From 12da677f85e99e3bd5992620228104960e62857a Mon Sep 17 00:00:00 2001
From: Laura Flores <ljflores@redhat.com>
Date: Tue, 13 Dec 2022 19:48:25 +0000
Subject: [PATCH 0001/2492] qa/workunits: update telemetry quincy workunits
 with basic_pool_options_bluestore collection

Signed-off-by: Laura Flores <ljflores@redhat.com>
---
 qa/workunits/test_telemetry_quincy.sh   | 3 ++-
 qa/workunits/test_telemetry_quincy_x.sh | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/qa/workunits/test_telemetry_quincy.sh b/qa/workunits/test_telemetry_quincy.sh
index e8b07ec13032..2ce268eadbbc 100755
--- a/qa/workunits/test_telemetry_quincy.sh
+++ b/qa/workunits/test_telemetry_quincy.sh
@@ -13,7 +13,8 @@ ceph telemetry preview-all
 
 # Assert that new collections are available
 COLLECTIONS=$(ceph telemetry collection ls)
-NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics")
+NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage"
+                 "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore")
 for col in ${NEW_COLLECTIONS[@]}; do
     if ! [[ $COLLECTIONS == *$col* ]];
     then
diff --git a/qa/workunits/test_telemetry_quincy_x.sh b/qa/workunits/test_telemetry_quincy_x.sh
index 4734132d024c..bfb050cfa59b 100755
--- a/qa/workunits/test_telemetry_quincy_x.sh
+++ b/qa/workunits/test_telemetry_quincy_x.sh
@@ -12,7 +12,8 @@ fi
 ceph -s
 
 COLLECTIONS=$(ceph telemetry collection ls)
-NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics")
+NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage"
+                 "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore")
 for col in ${NEW_COLLECTIONS[@]}; do
     if ! [[ $COLLECTIONS == *$col* ]];
     then

From 73ddc4d202113c3dd4b09d602287f09358e2495a Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Thu, 15 Dec 2022 16:58:51 +0000
Subject: [PATCH 0002/2492] osdc: fix the ENOCONN normalization in
 Objecter::_linger_reconnect()

Problem's description: https://tracker.ceph.com/issues/53789#note-12

Fixes: https://tracker.ceph.com/issues/53789
Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osdc/Objecter.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index eff79c5e9061..5d3c4e88b123 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -670,8 +670,8 @@ void Objecter::_linger_reconnect(LingerOp *info, bs::error_code ec)
 		 << " (last_error " << info->last_error << ")" << dendl;
   std::unique_lock wl(info->watch_lock);
   if (ec) {
+    ec = _normalize_watch_error(ec);
     if (!info->last_error) {
-      ec = _normalize_watch_error(ec);
       if (info->handle) {
 	boost::asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
       }

From 1c64c6be303f5ed4110468101a01576508468a74 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 16 Dec 2022 19:33:07 +0300
Subject: [PATCH 0003/2492] pybind/rados: fix missed changes for PEP484 style
 type annotations

originally brought by https://github.com/ceph/ceph/pull/36918

Fixes: https://tracker.ceph.com/issues/58304
Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/pybind/rados/rados.pyx    | 16 ++++++++--------
 src/test/pybind/test_rados.py |  5 +++++
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx
index b8ae8165c6c2..e0cf6d9422a6 100644
--- a/src/pybind/rados/rados.pyx
+++ b/src/pybind/rados/rados.pyx
@@ -1047,10 +1047,10 @@ Rados object in state %s." % self.state)
         # NOTE(sileht): looks weird but test_monmap_dump pass int
             target = str(target)
 
-        target = cstr(target, 'target', opt=True)
+        target_raw = cstr(target, 'target', opt=True)
 
         cdef:
-            char *_target = opt_str(target)
+            char *_target = opt_str(target_raw)
             char **_cmd = to_bytes_array(cmds)
             size_t _cmdlen = len(cmds)
 
@@ -1063,7 +1063,7 @@ Rados object in state %s." % self.state)
             size_t _outs_len
 
         try:
-            if target:
+            if target_raw:
                 with nogil:
                     ret = rados_mon_command_target(self.cluster, _target,
                                                 <const char **>_cmd, _cmdlen,
@@ -1148,10 +1148,10 @@ Rados object in state %s." % self.state)
         self.require_state("connected")
 
         cmds = [cstr(cmd, 'cmd')]
-        target = cstr(target, 'target', opt=True)
+        target_raw = cstr(target, 'target', opt=True)
 
         cdef:
-            char *_target = opt_str(target)
+            char *_target = opt_str(target_raw)
 
             char **_cmd = to_bytes_array(cmds)
             size_t _cmdlen = len(cmds)
@@ -1165,7 +1165,7 @@ Rados object in state %s." % self.state)
             size_t _outs_len
 
         try:
-            if target is not None:
+            if target_raw is not None:
                 with nogil:
                     ret = rados_mgr_command_target(self.cluster,
 		                            <const char*>_target,
@@ -3779,9 +3779,9 @@ returned %d, but should return zero on success." % (self.name, ret))
         :para max_return: list no more than max_return key/value pairs
         :returns: an iterator over the requested omap values, return value from this action
         """
-        start_after = cstr(start_after, 'start_after') if start_after else None
+        start_after_raw = cstr(start_after, 'start_after') if start_after else None
         cdef:
-            char *_start_after = opt_str(start_after)
+            char *_start_after = opt_str(start_after_raw)
             ReadOp _read_op = read_op
             rados_omap_iter_t iter_addr = NULL
             int _max_return = max_return
diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py
index e68269ff01ed..f6954e69a81c 100644
--- a/src/test/pybind/test_rados.py
+++ b/src/test/pybind/test_rados.py
@@ -556,6 +556,11 @@ def test_get_omap_keys(self):
             eq(ret, 0)
             with assert_raises(ObjectNotFound):
                 self.ioctx.operate_read_op(read_op, "no_such")
+        with ReadOpCtx() as read_op:
+            iter, ret = self.ioctx.get_omap_keys(read_op,"2",2)
+            eq(ret, 0)
+            self.ioctx.operate_read_op(read_op, "hw")
+            eq(list(iter), [("3", None)])
 
     def test_clear_omap(self):
         keys = ("1", "2", "3")

From 3c922133eee7466ed2169d0ff5c83e94c3a03cff Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@suse.com>
Date: Mon, 22 Aug 2022 17:29:27 +0300
Subject: [PATCH 0004/2492] os/bluestore: assert on improper releases in
 AvlAllocator

Let's assert on unexpected unit release before we damage internal data structures.
Plus uniforms some logging output for avl/hybrid allocator with bitmap one to enable replay
tool usage.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/AvlAllocator.cc    | 60 +++++++++++++++++++++--------
 src/os/bluestore/HybridAllocator.cc |  8 ++--
 2 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/src/os/bluestore/AvlAllocator.cc b/src/os/bluestore/AvlAllocator.cc
index 4584bfae713f..9d828ee313ce 100644
--- a/src/os/bluestore/AvlAllocator.cc
+++ b/src/os/bluestore/AvlAllocator.cc
@@ -105,6 +105,16 @@ void AvlAllocator::_add_to_tree(uint64_t start, uint64_t size)
     rs_before = std::prev(rs_after);
   }
 
+  if ((rs_before != range_tree.end() && rs_before->end > start) ||
+      (rs_after != range_tree.end() && rs_after->start < end)) {
+    derr << __func__ << " inconsistent tree state " << std::hex
+         << " 0x" << start << "~" << end - start
+         << std::dec << dendl;
+    _dump();
+  }
+  ceph_assert(rs_before == range_tree.end() || rs_before->end <= start);
+  ceph_assert(rs_after == range_tree.end() || rs_after->start >= end);
+
   bool merge_before = (rs_before != range_tree.end() && rs_before->end == start);
   bool merge_after = (rs_after != range_tree.end() && rs_after->start == end);
 
@@ -169,6 +179,14 @@ void AvlAllocator::_remove_from_tree(uint64_t start, uint64_t size)
 
   auto rs = range_tree.find(range_t{start, end}, range_tree.key_comp());
   /* Make sure we completely overlap with someone */
+  if (rs == range_tree.end() ||
+      rs->start > start ||
+      rs->end < end) {
+    derr << __func__ << " inconsistent tree state " << std::hex
+         << " 0x" << rs->start << "~" << rs->end - rs->start
+         << std::dec << dendl;
+    _dump();
+  }
   ceph_assert(rs != range_tree.end());
   ceph_assert(rs->start <= start);
   ceph_assert(rs->end >= end);
@@ -292,7 +310,9 @@ int AvlAllocator::_allocate(
   if (start == -1ULL) {
     return -ENOSPC;
   }
-
+  dout(20) << __func__ << " allocated 0x" << std::hex
+           << start << "~" << size
+           << std::dec << dendl;
   _remove_from_tree(start, size);
 
   *offset = start;
@@ -306,9 +326,9 @@ void AvlAllocator::_release(const interval_set<uint64_t>& release_set)
     const auto offset = p.get_start();
     const auto length = p.get_len();
     ceph_assert(offset + length <= uint64_t(device_size));
-    ldout(cct, 10) << __func__ << std::hex
-      << " offset 0x" << offset
-      << " length 0x" << length
+    ldout(cct, 20) << __func__ << std::hex
+      << " 0x" << offset
+      << "~" << length
       << std::dec << dendl;
     _add_to_tree(offset, length);
   }
@@ -316,9 +336,9 @@ void AvlAllocator::_release(const interval_set<uint64_t>& release_set)
 
 void AvlAllocator::_release(const PExtentVector& release_set) {
   for (auto& e : release_set) {
-    ldout(cct, 10) << __func__ << std::hex
-      << " offset 0x" << e.offset
-      << " length 0x" << e.length
+    ldout(cct, 20) << __func__ << std::hex
+      << " 0x" << e.offset
+      << "~" << e.length
       << std::dec << dendl;
     _add_to_tree(e.offset, e.length);
   }
@@ -346,14 +366,20 @@ AvlAllocator::AvlAllocator(CephContext* cct,
     cct->_conf.get_val<Option::size_t>("bluestore_avl_alloc_ff_max_search_bytes")),
   range_count_cap(max_mem / sizeof(range_seg_t)),
   cct(cct)
-{}
+{
+  ldout(cct, 10) << __func__ << " 0x" << std::hex << get_capacity() << "/"
+                 << get_block_size() << std::dec << dendl;
+}
 
 AvlAllocator::AvlAllocator(CephContext* cct,
 			   int64_t device_size,
 			   int64_t block_size,
 			   std::string_view name) :
   AvlAllocator(cct, device_size, block_size, 0 /* max_mem */, name)
-{}
+{
+  ldout(cct, 10) << __func__ << " 0x" << std::hex << get_capacity() << "/"
+                 << get_block_size() << std::dec << dendl;
+}
 
 AvlAllocator::~AvlAllocator()
 {
@@ -368,10 +394,10 @@ int64_t AvlAllocator::allocate(
   PExtentVector* extents)
 {
   ldout(cct, 10) << __func__ << std::hex
-                 << " want 0x" << want
-                 << " unit 0x" << unit
-                 << " max_alloc_size 0x" << max_alloc_size
-                 << " hint 0x" << hint
+                 << " 0x" << want
+                 << "/" << unit
+                 << "," << max_alloc_size
+                 << "," << hint
                  << std::dec << dendl;
   ceph_assert(std::has_single_bit(unit));
   ceph_assert(want % unit == 0);
@@ -450,8 +476,8 @@ void AvlAllocator::init_add_free(uint64_t offset, uint64_t length)
   std::lock_guard l(lock);
   ceph_assert(offset + length <= uint64_t(device_size));
   ldout(cct, 10) << __func__ << std::hex
-                 << " offset 0x" << offset
-                 << " length 0x" << length
+                 << " 0x" << offset
+                 << "~" << length
                  << std::dec << dendl;
   _add_to_tree(offset, length);
 }
@@ -463,8 +489,8 @@ void AvlAllocator::init_rm_free(uint64_t offset, uint64_t length)
   std::lock_guard l(lock);
   ceph_assert(offset + length <= uint64_t(device_size));
   ldout(cct, 10) << __func__ << std::hex
-                 << " offset 0x" << offset
-                 << " length 0x" << length
+                 << " 0x" << offset
+                 << "~" << length
                  << std::dec << dendl;
   _remove_from_tree(offset, length);
 }
diff --git a/src/os/bluestore/HybridAllocator.cc b/src/os/bluestore/HybridAllocator.cc
index 2201d5958246..cfb5858fd11d 100644
--- a/src/os/bluestore/HybridAllocator.cc
+++ b/src/os/bluestore/HybridAllocator.cc
@@ -23,10 +23,10 @@ int64_t HybridAllocator::allocate(
   PExtentVector* extents)
 {
   ldout(cct, 10) << __func__ << std::hex
-                 << " want 0x" << want
-                 << " unit 0x" << unit
-                 << " max_alloc_size 0x" << max_alloc_size
-                 << " hint 0x" << hint
+                 << " 0x" << want
+                 << "/" << unit
+                 << "," << max_alloc_size
+                 << "," << hint
                  << std::dec << dendl;
   ceph_assert(std::has_single_bit(unit));
   ceph_assert(want % unit == 0);

From 5b6be4565f5f2eae2b31a53b3d62837c5c213228 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@suse.com>
Date: Tue, 23 Aug 2022 20:18:31 +0300
Subject: [PATCH 0005/2492] os/bluestore: log values in hex in AvlAllocator

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/AvlAllocator.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/os/bluestore/AvlAllocator.cc b/src/os/bluestore/AvlAllocator.cc
index 9d828ee313ce..317291e65f49 100644
--- a/src/os/bluestore/AvlAllocator.cc
+++ b/src/os/bluestore/AvlAllocator.cc
@@ -293,12 +293,16 @@ int AvlAllocator::_allocate(
     ceph_assert(align != 0);
     uint64_t* cursor = &lbas[cbits(align) - 1];
     start = _pick_block_after(cursor, size, unit);
-    dout(20) << __func__ << " first fit=" << start << " size=" << size << dendl;
+    dout(20) << __func__
+             << std::hex << " first fit params: 0x" << start << "~" << size
+             << std::dec << dendl;
   }
   if (start == -1ULL) {
     do {
       start = _pick_block_fits(size, unit);
-      dout(20) << __func__ << " best fit=" << start << " size=" << size << dendl;
+      dout(20) << __func__
+               << std::hex << " best fit params: 0x" << start << "~" << size
+               << std::dec << dendl;
       if (start != uint64_t(-1ULL)) {
         break;
       }

From 36961d644ce11e89c789e9112e9f3702f8580829 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@suse.com>
Date: Tue, 23 Aug 2022 20:21:48 +0300
Subject: [PATCH 0006/2492] os/bluestore: do not call allocator's release on
 empty set

Signed-off-by: Igor Fedotov <ifedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 33d1d9983087..33eb0943c0a2 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -13055,7 +13055,8 @@ void BlueStore::_txc_release_alloc(TransContext *txc)
 {
   bool discard_queued = false;
   // it's expected we're called with lazy_release_lock already taken!
-  if (unlikely(cct->_conf->bluestore_debug_no_reuse_blocks)) {
+  if (unlikely(cct->_conf->bluestore_debug_no_reuse_blocks ||
+               txc->released.size() == 0)) {
       goto out;
   }
   discard_queued = bdev->try_discard(txc->released);

From 08d80006d07d98031125a1d854cb4f778eaee75b Mon Sep 17 00:00:00 2001
From: Nitzan Mordechai <nmordech@redhat.com>
Date: Sun, 12 Feb 2023 07:14:23 +0000
Subject: [PATCH 0007/2492] qa/*/test_envlibrados_for_rocksdb.sh: subscrib repo
 subscription-manager repos will fail with 'does not match a valid repository
 ID' we should use dnf config-manager --set-enabled instead

Fixes: https://tracker.ceph.com/issues/58560
Signed-off-by: Nitzan Mordechai <nmordec@redhat.com>
---
 qa/workunits/rados/test_envlibrados_for_rocksdb.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
index 371452f40429..e072b8299844 100755
--- a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
+++ b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
@@ -26,7 +26,7 @@ case $(distro_id) in
         case $(distro_id) in
             rhel)
                 # RHEL needs CRB repo for snappy-devel
-                sudo subscription-manager repos --enable "codeready-builder-for-rhel-8-x86_64-rpms"
+                sudo dnf config-manager --set-enabled "codeready-builder-for-rhel-8-x86_64-rpms" 
                 ;;
         esac
         install git gcc-c++.x86_64 snappy-devel zlib zlib-devel bzip2 bzip2-devel libradospp-devel.x86_64 cmake libarchive-3.3.3

From bf52c18043117cccb68c5d9dabe8672dd2b943b3 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 30 Jan 2023 14:38:41 -0500
Subject: [PATCH 0008/2492] qa/cephadm: basic test for monitoring stack

Testing that the monitoring stack daemons are
all basically functioning by checking their HTTP
APIs are responsive and and that putting down a mon
daemon, which should cause an alert, actually triggers
an alert that is viewable in the prometheus and
alertmanager API

Signed-off-by: Adam King <adking@redhat.com>
---
 .../task/test_monitoring_stack_basic.yaml     | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml

diff --git a/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml b/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
new file mode 100644
index 000000000000..62947ef65d9c
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
@@ -0,0 +1,55 @@
+roles:
+- - host.a
+  - mon.a
+  - mgr.a
+  - osd.0
+- - host.b
+  - mon.b
+  - mgr.b
+  - osd.1
+- - host.c
+  - mon.c
+  - osd.2
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+    host.a:
+      - |
+        set -e
+        set -x
+        ceph orch apply node-exporter
+        ceph orch apply grafana
+        ceph orch apply alertmanager
+        ceph orch apply prometheus
+        sleep 240
+        ceph orch ls
+        ceph orch ps
+        ceph orch host ls
+        MON_DAEMON=$(ceph orch ps --daemon-type mon -f json | jq -r 'last | .daemon_name')
+        GRAFANA_HOST=$(ceph orch ps --daemon-type grafana -f json | jq -e '.[]' | jq -r '.hostname')
+        PROM_HOST=$(ceph orch ps --daemon-type prometheus -f json | jq -e '.[]' | jq -r '.hostname')
+        ALERTM_HOST=$(ceph orch ps --daemon-type alertmanager -f json | jq -e '.[]' | jq -r '.hostname')
+        GRAFANA_IP=$(ceph orch host ls -f json | jq -r --arg GRAFANA_HOST "$GRAFANA_HOST" '.[] | select(.hostname==$GRAFANA_HOST) | .addr')
+        PROM_IP=$(ceph orch host ls -f json | jq -r --arg PROM_HOST "$PROM_HOST" '.[] | select(.hostname==$PROM_HOST) | .addr')
+        ALERTM_IP=$(ceph orch host ls -f json | jq -r --arg ALERTM_HOST "$ALERTM_HOST" '.[] | select(.hostname==$ALERTM_HOST) | .addr')
+        # check each host node-exporter metrics endpoint is responsive
+        ALL_HOST_IPS=$(ceph orch host ls -f json | jq -r '.[] | .addr')
+        for ip in $ALL_HOST_IPS; do
+          curl -s http://${ip}:9100/metric
+        done
+        # check grafana endpoints are responsive and database health is okay
+        curl -k -s https://${GRAFANA_IP}:3000/api/health
+        curl -k -s https://${GRAFANA_IP}:3000/api/health | jq -e '.database == "ok"'
+        # stop mon daemon in order to trigger an alert
+        ceph orch daemon stop $MON_DAEMON
+        sleep 120
+        # check prometheus endpoints are responsive and mon down alert is firing
+        curl -s http://${PROM_IP}:9095/api/v1/status/config
+        curl -s http://${PROM_IP}:9095/api/v1/status/config | jq -e '.status == "success"'
+        curl -s http://${PROM_IP}:9095/api/v1/alerts
+        curl -s http://${PROM_IP}:9095/api/v1/alerts | jq -e '.data | .alerts | .[] | select(.labels | .alertname == "CephMonDown") | .state == "firing"'
+        # check alertmanager endpoints are responsive and mon down alert is active
+        curl -s http://${ALERTM_IP}:9093/api/v1/status
+        curl -s http://${ALERTM_IP}:9093/api/v1/alerts
+        curl -s http://${ALERTM_IP}:9093/api/v1/alerts | jq -e '.data | .[] | select(.labels | .alertname == "CephMonDown") | .status | .state == "active"'

From 5b56098f17dd9abe4c15cbc7f487c0e94841beaf Mon Sep 17 00:00:00 2001
From: Neeraj Pratap Singh <neesingh@redhat.com>
Date: Thu, 17 Nov 2022 23:33:15 +0530
Subject: [PATCH 0009/2492] mds: scrub repair does not clear earlier damage
 health status

Fixes: https://tracker.ceph.com/issues/54557
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 src/mds/CDir.cc        |  1 +
 src/mds/CInode.cc      |  4 ++++
 src/mds/DamageTable.cc | 28 ++++++++++++++++++++++++++++
 src/mds/DamageTable.h  |  7 +++++++
 4 files changed, 40 insertions(+)

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index e6844cb7a4bd..f9aed746051f 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -3750,6 +3750,7 @@ bool CDir::scrub_local()
     mdcache->repair_dirfrag_stats(this);
     scrub_infop->header->set_repaired();
     good = true;
+    mdcache->mds->damage_table.remove_dentry_damage_entry(this);
   }
   return good;
 }
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 4ac963166e98..9aa3a8c67a89 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -4783,6 +4783,7 @@ void CInode::validate_disk_state(CInode::validated_data *results,
                            false);
         // Flag that we repaired this BT so that it won't go into damagetable
         results->backtrace.repaired = true;
+        in->mdcache->mds->damage_table.remove_backtrace_damage_entry(in->ino());
         if (in->mdcache->mds->logger)
           in->mdcache->mds->logger->inc(l_mds_scrub_backtrace_repaired);
       }
@@ -4921,6 +4922,9 @@ void CInode::validate_disk_state(CInode::validated_data *results,
 	    << "freshly-calculated rstats don't match existing ones (will be fixed)";
 	  in->mdcache->repair_inode_stats(in);
           results->raw_stats.repaired = true;
+          for (const auto &p : in->dirfrags){
+            in->mdcache->mds->damage_table.remove_dirfrag_damage_entry(p.second);
+          }
 	} else {
 	  results->raw_stats.error_str
 	    << "freshly-calculated rstats don't match existing ones";
diff --git a/src/mds/DamageTable.cc b/src/mds/DamageTable.cc
index 22802079d85d..2079d23333a8 100644
--- a/src/mds/DamageTable.cc
+++ b/src/mds/DamageTable.cc
@@ -15,6 +15,7 @@
 #include "common/debug.h"
 
 #include "mds/CDir.h"
+#include "mds/CInode.h"
 
 #include "DamageTable.h"
 
@@ -200,6 +201,33 @@ bool DamageTable::notify_remote_damaged(inodeno_t ino, std::string_view path)
   return false;
 }
 
+void DamageTable::remove_dentry_damage_entry(CDir *dir)
+{
+  if (dentries.count(
+        DirFragIdent(dir->inode->ino(), dir->frag)
+        ) > 0){
+          const auto frag_dentries =
+            dentries.at(DirFragIdent(dir->inode->ino(), dir->frag));
+          for(const auto &i : frag_dentries) {
+            erase(i.second->id);
+          }
+        }
+}
+
+void DamageTable::remove_dirfrag_damage_entry(CDir *dir)
+{
+  if (is_dirfrag_damaged(dir)){
+    erase(dirfrags.find(DirFragIdent(dir->inode->ino(), dir->frag))->second->id);
+  }
+}
+
+void DamageTable::remove_backtrace_damage_entry(inodeno_t ino)
+{  
+  if (is_remote_damaged(ino)){
+    erase(remotes.find(ino)->second->id);
+  }  
+}
+
 bool DamageTable::oversized() const
 {
   return by_id.size() > (size_t)(g_conf()->mds_damage_table_max_entries);
diff --git a/src/mds/DamageTable.h b/src/mds/DamageTable.h
index 18a61e08b122..a1b96fe22186 100644
--- a/src/mds/DamageTable.h
+++ b/src/mds/DamageTable.h
@@ -22,6 +22,7 @@
 #include "include/random.h"
 
 class CDir;
+class CInode;
 
 typedef uint64_t damage_entry_id_t;
 
@@ -155,6 +156,12 @@ class DamageTable
      */
     bool notify_remote_damaged(inodeno_t ino, std::string_view path);
 
+    void remove_dentry_damage_entry(CDir *dir);
+
+    void remove_dirfrag_damage_entry(CDir *dir);
+
+    void remove_backtrace_damage_entry(inodeno_t ino);
+
     bool is_dentry_damaged(
       const CDir *dir_frag,
       std::string_view dname,

From 81944f60478abe2d4253caaeb4165da5ccffaca1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 7 Mar 2023 13:30:11 +0100
Subject: [PATCH 0010/2492] osd: don't send stale hb msgr's addresses in
 MOSDBoot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

See comments in the ticket for the RCA.

NOTE: we can't just hold a reference to what `get_myaddrs()`
returns as the `safe_item_history` is involved:

```cpp
template<class T>
class safe_item_history {
  //...
  T *current = nullptr;

  // ...
  const T& operator=(const T& other) {
    std::lock_guard l(lock);
    history.push_back(other);
    current = &history.back();
    return *current;
  }
```

Fixes: https://tracker.ceph.com/issues/58915
Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/OSD.cc | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 7ab35bf0e080..f26e87fa00ee 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -6742,13 +6742,11 @@ void OSD::_send_boot()
     cluster_messenger->get_loopback_connection().get();
   entity_addrvec_t client_addrs = client_messenger->get_myaddrs();
   entity_addrvec_t cluster_addrs = cluster_messenger->get_myaddrs();
-  entity_addrvec_t hb_back_addrs = hb_back_server_messenger->get_myaddrs();
-  entity_addrvec_t hb_front_addrs = hb_front_server_messenger->get_myaddrs();
 
   dout(20) << " initial client_addrs " << client_addrs
 	   << ", cluster_addrs " << cluster_addrs
-	   << ", hb_back_addrs " << hb_back_addrs
-	   << ", hb_front_addrs " << hb_front_addrs
+	   << ", hb_back_addrs " << hb_back_server_messenger->get_myaddrs()
+	   << ", hb_front_addrs " << hb_front_server_messenger->get_myaddrs()
 	   << dendl;
   if (cluster_messenger->set_addr_unknowns(client_addrs)) {
     dout(10) << " assuming cluster_addrs match client_addrs "
@@ -6763,7 +6761,6 @@ void OSD::_send_boot()
   if (hb_back_server_messenger->set_addr_unknowns(cluster_addrs)) {
     dout(10) << " assuming hb_back_addrs match cluster_addrs "
 	     << cluster_addrs << dendl;
-    hb_back_addrs = hb_back_server_messenger->get_myaddrs();
   }
   if (auto session = local_connection->get_priv(); !session) {
     hb_back_server_messenger->ms_deliver_handle_fast_connect(local_connection);
@@ -6773,7 +6770,6 @@ void OSD::_send_boot()
   if (hb_front_server_messenger->set_addr_unknowns(client_addrs)) {
     dout(10) << " assuming hb_front_addrs match client_addrs "
 	     << client_addrs << dendl;
-    hb_front_addrs = hb_front_server_messenger->get_myaddrs();
   }
   if (auto session = local_connection->get_priv(); !session) {
     hb_front_server_messenger->ms_deliver_handle_fast_connect(local_connection);
@@ -6784,6 +6780,8 @@ void OSD::_send_boot()
   // are, so now is a good time!
   set_numa_affinity();
 
+  entity_addrvec_t hb_back_addrs = hb_back_server_messenger->get_myaddrs();
+  entity_addrvec_t hb_front_addrs = hb_front_server_messenger->get_myaddrs();
   MOSDBoot *mboot = new MOSDBoot(
     superblock, get_osdmap_epoch(), service.get_boot_epoch(),
     hb_back_addrs, hb_front_addrs, cluster_addrs,

From c7f5037293737322a20617bd7e43ab28da258d22 Mon Sep 17 00:00:00 2001
From: Andreas Schwab <schwab@suse.de>
Date: Thu, 30 Mar 2023 20:36:55 +0200
Subject: [PATCH 0011/2492] ceph.spec.in: enable build on riscv64 for openSUSE
 Factory

Signed-off-by: Andreas Schwab <schwab@suse.de>
---
 ceph.spec.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index f0dd8e8a941a..5583c59128e3 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -189,7 +189,7 @@ URL:		http://ceph.com/
 Source0:	%{?_remote_tarball_prefix}@TARBALL_BASENAME@.tar.bz2
 %if 0%{?suse_version}
 # _insert_obs_source_lines_here
-ExclusiveArch:  x86_64 aarch64 ppc64le s390x
+ExclusiveArch:  x86_64 aarch64 ppc64le s390x riscv64
 %endif
 #################################################################################
 # dependencies that apply across all distro families

From e2b2e8eb74c4d5cf06a5a2cb872a30b508bf75a2 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 12 Oct 2022 15:48:51 -0400
Subject: [PATCH 0012/2492] mds: trim cache during standby-replay

Fixes: 138fea6a7638697acb1a9e824db7b8d04ad8d671
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 77303ddd8ada..9d8bd04d66c5 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13595,7 +13595,7 @@ void MDCache::upkeep_main(void)
         if (active_with_clients) {
           trim_client_leases();
         }
-        if (is_open()) {
+        if (is_open() || mds->is_standby_replay()) {
           trim();
         }
         if (active_with_clients) {

From fe35c9b200c5b5c6066a4bbec4207cf9b19957f4 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 13 Oct 2022 15:50:49 -0400
Subject: [PATCH 0013/2492] include: remove unused lru method

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/include/lru.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/include/lru.h b/src/include/lru.h
index 3f5069ee3ed3..33f2f4e08ff9 100644
--- a/src/include/lru.h
+++ b/src/include/lru.h
@@ -185,10 +185,6 @@ class LRU {
     return NULL;
   }
 
-  void lru_status() {
-    //generic_dout(10) << "lru: " << lru_get_size() << " items, " << top.size() << " top, " << bottom.size() << " bot, " << pintail.size() << " pintail" << dendl;
-  }
-
 protected:
   // adjust top/bot balance, as necessary
   void adjust() {

From a1ca8e8b9ab280fd0bdb36a1c3a42ea82dfeaea6 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 13 Oct 2022 15:51:06 -0400
Subject: [PATCH 0014/2492] mds: log lru stats during trim

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 9d8bd04d66c5..59a8f0739e2e 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -6786,6 +6786,13 @@ std::pair<bool, uint64_t> MDCache::trim_lru(uint64_t count, expiremap& expiremap
           << " pinned=" << lru.lru_get_num_pinned()
           << dendl;
 
+  dout(20) << "bottom_lru: " << bottom_lru.lru_get_size() << " items"
+              ", " << bottom_lru.lru_get_top() << " top"
+              ", " << bottom_lru.lru_get_bot() << " bot"
+              ", " << bottom_lru.lru_get_pintail() << " pintail"
+              ", " << bottom_lru.lru_get_num_pinned() << " pinned"
+              << dendl;
+
   const uint64_t trim_counter_start = trim_counter.get();
   bool throttled = false;
   while (1) {
@@ -6806,6 +6813,13 @@ std::pair<bool, uint64_t> MDCache::trim_lru(uint64_t count, expiremap& expiremap
   }
   unexpirables.clear();
 
+  dout(20) << "lru: " << lru.lru_get_size() << " items"
+              ", " << lru.lru_get_top() << " top"
+              ", " << lru.lru_get_bot() << " bot"
+              ", " << lru.lru_get_pintail() << " pintail"
+              ", " << lru.lru_get_num_pinned() << " pinned"
+              << dendl;
+
   // trim dentries from the LRU until count is reached
   // if mds is in standby_replay and skip trimming the inodes
   while (!throttled && (cache_toofull() || count > 0 || is_standby_replay)) {

From 589e59af11e40164695ca13f4ce4f2bc140b18b8 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 10 Nov 2022 08:22:35 -0500
Subject: [PATCH 0015/2492] mds: revert standby-replay trimming changes

Revert "mds: do not trim the inodes from the lru list in standby_replay"
Revert "mds: trim cache during standby replay"
This reverts commit 79bb44c1b9f1715378a9550a81984e949e454ff4.
This reverts commit c0fe25bb2a87856c1281eddcb4da2efe0d7fbf75.

standby-replay daemons were changed to keep minimal metadata from the journal
in cache but the original intent of standby-replay was to have a cache that's
as warm as the rank itself.  This reverts the two commits which changed that
behavior.

Part of these reason for this is that the new rapid cache trimming behavior was
not correct at all. The trimming loop would break when it runs into a dentry
with non-null linkage. This would nearly always be the case.  It was thought
that this was a problem introduced by [2] as MDCache::standby_trim_segment has
a different trim check [4] but the original issue (tracker 48673) is as old as
[1], indicating the problem predates [2].

So, this commit reverts all of that. I have lingering suspicions that the
standby-replay daemon is not pinning some dentries properly which causes [5]
but this did not show up unless the MDS was rapidly evicting some dentries.
More research needs done there.

[1] c0fe25bb2a87856c1281eddcb4da2efe0d7fbf75
[2] 79bb44c1b9f1715378a9550a81984e949e454ff4
[3] https://github.com/ceph/ceph/blob/84fba097049ec4f72549588eaacc64f30c7a88a8/src/mds/MDCache.cc#L6816-L6820
[4] https://github.com/ceph/ceph/blob/84fba097049ec4f72549588eaacc64f30c7a88a8/src/mds/MDCache.cc#L7476-L7481
[5] https://tracker.ceph.com/issues/50246

Fixes: https://tracker.ceph.com/issues/48673
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 39 +++++----------------------------------
 1 file changed, 5 insertions(+), 34 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 59a8f0739e2e..d44ea8187c10 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -6821,19 +6821,17 @@ std::pair<bool, uint64_t> MDCache::trim_lru(uint64_t count, expiremap& expiremap
               << dendl;
 
   // trim dentries from the LRU until count is reached
-  // if mds is in standby_replay and skip trimming the inodes
-  while (!throttled && (cache_toofull() || count > 0 || is_standby_replay)) {
+  while (!throttled && (cache_toofull() || count > 0)) {
     throttled |= trim_counter_start+trimmed >= trim_threshold;
     if (throttled) break;
     CDentry *dn = static_cast<CDentry*>(lru.lru_expire());
     if (!dn) {
       break;
     }
-    if (is_standby_replay && dn->get_linkage()->inode) {
-      // we move the inodes that need to be trimmed to the end of the lru queue.
-      // refer to MDCache::standby_trim_segment
-      lru.lru_insert_bot(dn);
-      break;
+    if ((is_standby_replay && dn->get_linkage()->inode &&
+        dn->get_linkage()->inode->item_open_file.is_on_list())) {
+      dout(20) << "unexpirable: " << *dn << dendl;
+      unexpirables.push_back(dn);
     } else if (trim_dentry(dn, expiremap)) {
       unexpirables.push_back(dn);
     } else {
@@ -7479,69 +7477,42 @@ void MDCache::try_trim_non_auth_subtree(CDir *dir)
 
 void MDCache::standby_trim_segment(LogSegment *ls)
 {
-  auto try_trim_inode = [this](CInode *in) {
-    if (in->get_num_ref() == 0 &&
-	!in->item_open_file.is_on_list() &&
-	in->parent != NULL &&
-	in->parent->get_num_ref() == 0){
-      touch_dentry_bottom(in->parent);
-    }
-  };
-
-  auto try_trim_dentry = [this](CDentry *dn) {
-    if (dn->get_num_ref() > 0)
-      return;
-    auto in = dn->get_linkage()->inode;
-    if(in && in->item_open_file.is_on_list())
-      return;
-    touch_dentry_bottom(dn);
-  };
-  
   ls->new_dirfrags.clear_list();
   ls->open_files.clear_list();
 
   while (!ls->dirty_dirfrags.empty()) {
     CDir *dir = ls->dirty_dirfrags.front();
     dir->mark_clean();
-    if (dir->inode)
-      try_trim_inode(dir->inode);
   }
   while (!ls->dirty_inodes.empty()) {
     CInode *in = ls->dirty_inodes.front();
     in->mark_clean();
-    try_trim_inode(in);
   }
   while (!ls->dirty_dentries.empty()) {
     CDentry *dn = ls->dirty_dentries.front();
     dn->mark_clean();
-    try_trim_dentry(dn);
   }
   while (!ls->dirty_parent_inodes.empty()) {
     CInode *in = ls->dirty_parent_inodes.front();
     in->clear_dirty_parent();
-    try_trim_inode(in);
   }
   while (!ls->dirty_dirfrag_dir.empty()) {
     CInode *in = ls->dirty_dirfrag_dir.front();
     in->filelock.remove_dirty();
-    try_trim_inode(in);
   }
   while (!ls->dirty_dirfrag_nest.empty()) {
     CInode *in = ls->dirty_dirfrag_nest.front();
     in->nestlock.remove_dirty();
-    try_trim_inode(in);
   }
   while (!ls->dirty_dirfrag_dirfragtree.empty()) {
     CInode *in = ls->dirty_dirfrag_dirfragtree.front();
     in->dirfragtreelock.remove_dirty();
-    try_trim_inode(in);
   }
   while (!ls->truncating_inodes.empty()) {
     auto it = ls->truncating_inodes.begin();
     CInode *in = *it;
     ls->truncating_inodes.erase(it);
     in->put(CInode::PIN_TRUNCATING);
-    try_trim_inode(in);
   }
 }
 

From 4e20faa5bd3ee1d6607f662a1cf9f2d3a42a540c Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Fri, 23 Jun 2023 05:05:27 +0000
Subject: [PATCH 0016/2492] src/tools/ceph_dedup_tool: remove unused code

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index f3c942a97604..b8c79efa42ef 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -570,11 +570,6 @@ class SampleDedupWorkerThread : public Thread
       return cur_reference >= dedup_threshold && dedup_threshold != -1;
     }
 
-    void init(size_t dedup_threshold_) {
-      std::unique_lock lock(fingerprint_lock);
-      fp_map.clear();
-      dedup_threshold = dedup_threshold_;
-    }
     FpStore(size_t chunk_threshold) : dedup_threshold(chunk_threshold) { }
 
   private:

From 25890a22a929872cf073edab946d23ee5d6b5801 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 5 Jul 2023 06:01:10 -0500
Subject: [PATCH 0017/2492] test/rgw: annotating variables with maybe_unused

Some variables in test/rgw/rgw_cr_test.cc are only used in
asserts, while this file is also used in builds without
NDEBUG set.
Adding the [[maybe_unused]] attribute clears the compilation
warnings.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/rgw/rgw_cr_test.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/test/rgw/rgw_cr_test.cc b/src/test/rgw/rgw_cr_test.cc
index 37120925291a..2c250b390a98 100644
--- a/src/test/rgw/rgw_cr_test.cc
+++ b/src/test/rgw/rgw_cr_test.cc
@@ -55,12 +55,14 @@ struct TempPool {
     fmt::format("{}-{}-{}", ::time(nullptr), ::getpid(),num++);
 
   TempPool() {
-    auto r = store->getRados()->get_rados_handle()->pool_create(name.c_str());
+    [[maybe_unused]] auto r =
+        store->getRados()->get_rados_handle()->pool_create(name.c_str());
     assert(r == 0);
   }
 
   ~TempPool() {
-    auto r = store->getRados()->get_rados_handle()->pool_delete(name.c_str());
+    [[maybe_unused]] auto r =
+        store->getRados()->get_rados_handle()->pool_delete(name.c_str());
     assert(r == 0);
   }
 
@@ -70,8 +72,9 @@ struct TempPool {
 
   operator librados::IoCtx() {
     librados::IoCtx ioctx;
-    auto r = store->getRados()->get_rados_handle()->ioctx_create(name.c_str(),
-								 ioctx);
+    [[maybe_unused]] auto r =
+        store->getRados()->get_rados_handle()->ioctx_create(name.c_str(),
+                                                            ioctx);
     assert(r == 0);
     return ioctx;
   }

From cd323cb664a125834e41b77f711eac898e548382 Mon Sep 17 00:00:00 2001
From: Ville Ojamo <14869000+bluikko@users.noreply.github.com>
Date: Fri, 7 Jul 2023 17:02:19 +0700
Subject: [PATCH 0018/2492] doc/radosgw/admin.rst: use underscores in config
 var names

Following the current policy, config var names in `ceph.conf` etc. should use underscores instead of spaces.

Signed-off-by: Ville Ojamo <14869000+bluikko@users.noreply.github.com>
---
 doc/radosgw/admin.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index fc2651ec0d3d..9ba9be3e212b 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -434,9 +434,9 @@ Default Quotas
 You can set default quotas in the config.  These defaults are used when
 creating a new user and have no effect on existing users. If the
 relevant default quota is set in config, then that quota is set on the
-new user, and that quota is enabled.  See ``rgw bucket default quota max objects``,
-``rgw bucket default quota max size``, ``rgw user default quota max objects``, and
-``rgw user default quota max size`` in `Ceph Object Gateway Config Reference`_
+new user, and that quota is enabled.  See ``rgw_bucket_default_quota_max_objects``,
+``rgw_bucket_default_quota_max_size``, ``rgw_user_default_quota_max_objects``, and
+``rgw_user_default_quota_max_size`` in `Ceph Object Gateway Config Reference`_
 
 Quota Cache
 -----------
@@ -444,8 +444,8 @@ Quota Cache
 Quota statistics are cached on each RGW instance.  If there are multiple
 instances, then the cache can keep quotas from being perfectly enforced, as
 each instance will have a different view of quotas.  The options that control
-this are ``rgw bucket quota ttl``, ``rgw user quota bucket sync interval`` and
-``rgw user quota sync interval``.  The higher these values are, the more
+this are ``rgw_bucket_quota_ttl``, ``rgw_user_quota_bucket_sync_interval`` and
+``rgw_user_quota_sync_interval``.  The higher these values are, the more
 efficient quota operations are, but the more out-of-sync multiple instances
 will be.  The lower these values are, the closer to perfect enforcement
 multiple instances will achieve.  If all three are 0, then quota caching is
@@ -647,7 +647,7 @@ Usage
 The Ceph Object Gateway logs usage for each user. You can track
 user usage within date ranges too.
 
-- Add ``rgw enable usage log = true`` in [client.rgw] section of ceph.conf and restart the radosgw service. 
+- Add ``rgw_enable_usage_log = true`` in [client.rgw] section of ceph.conf and restart the radosgw service. 
 
 Options include: 
 

From f2b5f0727401fd3ab975976555864c21860f3579 Mon Sep 17 00:00:00 2001
From: Vedansh Bhartia <vedanshbhartia@gmail.com>
Date: Thu, 13 Jul 2023 17:17:06 +0530
Subject: [PATCH 0019/2492] rgw: Fix potential null dereference in
 rgw/driver/dbstore/sqlite/statement.cc

Signed-off-by: Vedansh Bhartia <vedanshbhartia@gmail.com>
---
 src/rgw/driver/dbstore/sqlite/statement.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/rgw/driver/dbstore/sqlite/statement.cc b/src/rgw/driver/dbstore/sqlite/statement.cc
index 3e44f4c0b6e5..199774f4d9ad 100644
--- a/src/rgw/driver/dbstore/sqlite/statement.cc
+++ b/src/rgw/driver/dbstore/sqlite/statement.cc
@@ -118,10 +118,10 @@ void eval0(const DoutPrefixProvider* dpp, const stmt_execution& stmt)
   if (ec != sqlite::errc::done) {
     const char* errmsg = ::sqlite3_errmsg(db);
     ldpp_dout(dpp, 20) << "evaluation failed: " << errmsg
-        << " (" << ec << ")\nstatement: " << sql.get() << dendl;
+        << " (" << ec << ")\nstatement: " << (sql ? sql.get() : "") << dendl;
     throw sqlite::error(errmsg, ec);
   }
-  ldpp_dout(dpp, 20) << "evaluation succeeded: " << sql.get() << dendl;
+  ldpp_dout(dpp, 20) << "evaluation succeeded: " << (sql ? sql.get() : "") << dendl;
 }
 
 void eval1(const DoutPrefixProvider* dpp, const stmt_execution& stmt)
@@ -137,10 +137,10 @@ void eval1(const DoutPrefixProvider* dpp, const stmt_execution& stmt)
     sqlite3* db = ::sqlite3_db_handle(stmt.get());
     const char* errmsg = ::sqlite3_errmsg(db);
     ldpp_dout(dpp, 1) << "evaluation failed: " << errmsg << " (" << ec
-        << ")\nstatement: " << sql.get() << dendl;
+        << ")\nstatement: " << (sql ? sql.get() : "") << dendl;
     throw sqlite::error(errmsg, ec);
   }
-  ldpp_dout(dpp, 20) << "evaluation succeeded: " << sql.get() << dendl;
+  ldpp_dout(dpp, 20) << "evaluation succeeded: " << (sql ? sql.get() : "") << dendl;
 }
 
 int column_int(const stmt_execution& stmt, int column)
@@ -181,14 +181,14 @@ auto read_text_rows(const DoutPrefixProvider* dpp,
       sqlite3* db = ::sqlite3_db_handle(stmt.get());
       const char* errmsg = ::sqlite3_errmsg(db);
       ldpp_dout(dpp, 1) << "evaluation failed: " << errmsg << " (" << ec
-          << ")\nstatement: " << sql.get() << dendl;
+          << ")\nstatement: " << (sql ? sql.get() : "") << dendl;
       throw sqlite::error(errmsg, ec);
     }
     entries[count] = column_text(stmt, 0);
     ++count;
   }
   ldpp_dout(dpp, 20) << "statement evaluation produced " << count
-      << " results: " << sql.get() << dendl;
+      << " results: " << (sql ? sql.get() : "") << dendl;
 
   return entries.first(count);
 }

From 1a2744282c46ca113f5f9f168d1f4d66c734e64d Mon Sep 17 00:00:00 2001
From: TomNewChao <chaotomzhu@gmail.com>
Date: Tue, 25 Jul 2023 15:08:44 +0800
Subject: [PATCH 0020/2492] Ceph dashboard supports multiple languages

mgr/dashboard/frontend:Ceph dashboard supports multiple languages

Signed-off-by: TomNewChao <chaotomzhu@gmail.com>
---
 ceph.spec.in | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 7af4123826a7..d1d11e8af451 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -24,6 +24,7 @@
 %bcond_with zbd
 %bcond_with cmake_verbose_logging
 %bcond_without ceph_test_package
+%bcond_without mgr_dashboard_frontend_support_multi_language
 %ifarch s390
 %bcond_with tcmalloc
 %else
@@ -470,6 +471,9 @@ BuildRequires:  libnuma-devel
 %if 0%{?rhel} >= 8
 BuildRequires:  /usr/bin/pathfix.py
 %endif
+%if 0%{with mgr_dashboard_frontend_support_multi_language}
+BuildRequires:  npm
+%endif
 
 %description
 Ceph is a massively scalable, open-source, distributed storage system that runs
@@ -1351,7 +1355,13 @@ cmake .. \
     -DSYSTEMD_SYSTEM_UNIT_DIR:PATH=%{_unitdir} \
     -DWITH_MANPAGE:BOOL=ON \
     -DWITH_PYTHON3:STRING=%{python3_version} \
+%if 0%{with mgr_dashboard_frontend_support_multi_language}
+    -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=ON \
+    -DDASHBOARD_FRONTEND_LANGS:STRING="cs,de,es,fr,id,it,ja,ko,pl,zh-Hans,zh-Hant,pt" \
+    -DWITH_SYSTEM_NPM:BOOL=ON \
+%else
     -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF \
+%endif
 %if 0%{?suse_version}
     -DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF \
 %endif

From 10edb2ffbdb7c4ef839f1ba7b88d7d85a682b7be Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 22 Aug 2023 14:10:46 +0800
Subject: [PATCH 0021/2492] crimson/osd/pg_recovery: avoiding duplicated object
 recovering

UrgentRecovery and other recoveries may collide when doing
`PGRecovery::add_recovering`, this is not an error. We should allow this
to happen

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg_recovery.cc     | 91 +++++++++++++++++++-----------
 src/crimson/osd/recovery_backend.h |  6 +-
 2 files changed, 60 insertions(+), 37 deletions(-)

diff --git a/src/crimson/osd/pg_recovery.cc b/src/crimson/osd/pg_recovery.cc
index 09b45779ec87..efbbf7e4f3ee 100644
--- a/src/crimson/osd/pg_recovery.cc
+++ b/src/crimson/osd/pg_recovery.cc
@@ -266,20 +266,27 @@ PGRecovery::recover_missing(
   RecoveryBackend::RecoveryBlockingEvent::TriggerI& trigger,
   const hobject_t &soid, eversion_t need)
 {
-  if (pg->get_peering_state().get_missing_loc().is_deleted(soid)) {
-    return pg->get_recovery_backend()->add_recovering(soid).wait_track_blocking(
-      trigger,
-      pg->get_recovery_backend()->recover_delete(soid, need));
+  logger().info("{} {} v {}", __func__, soid, need);
+  auto [recovering, added] = pg->get_recovery_backend()->add_recovering(soid);
+  if (added) {
+    logger().info("{} {} v {}, new recovery", __func__, soid, need);
+    if (pg->get_peering_state().get_missing_loc().is_deleted(soid)) {
+      return recovering.wait_track_blocking(
+	trigger,
+	pg->get_recovery_backend()->recover_delete(soid, need));
+    } else {
+      return recovering.wait_track_blocking(
+	trigger,
+	pg->get_recovery_backend()->recover_object(soid, need)
+	.handle_exception_interruptible(
+	  [=, this, soid = std::move(soid)] (auto e) {
+	  on_failed_recover({ pg->get_pg_whoami() }, soid, need);
+	  return seastar::make_ready_future<>();
+	})
+      );
+    }
   } else {
-    return pg->get_recovery_backend()->add_recovering(soid).wait_track_blocking(
-      trigger,
-      pg->get_recovery_backend()->recover_object(soid, need)
-      .handle_exception_interruptible(
-	[=, this, soid = std::move(soid)] (auto e) {
-	on_failed_recover({ pg->get_pg_whoami() }, soid, need);
-	return seastar::make_ready_future<>();
-      })
-    );
+    return recovering.wait_for_recovered();
   }
 }
 
@@ -288,16 +295,23 @@ RecoveryBackend::interruptible_future<> PGRecovery::prep_object_replica_deletes(
   const hobject_t& soid,
   eversion_t need)
 {
-  return pg->get_recovery_backend()->add_recovering(soid).wait_track_blocking(
-    trigger,
-    pg->get_recovery_backend()->push_delete(soid, need).then_interruptible(
-      [=, this] {
-      object_stat_sum_t stat_diff;
-      stat_diff.num_objects_recovered = 1;
-      on_global_recover(soid, stat_diff, true);
-      return seastar::make_ready_future<>();
-    })
-  );
+  logger().info("{} {} v {}", __func__, soid, need);
+  auto [recovering, added] = pg->get_recovery_backend()->add_recovering(soid);
+  if (added) {
+    logger().info("{} {} v {}, new recovery", __func__, soid, need);
+    return recovering.wait_track_blocking(
+      trigger,
+      pg->get_recovery_backend()->push_delete(soid, need).then_interruptible(
+	[=, this] {
+	object_stat_sum_t stat_diff;
+	stat_diff.num_objects_recovered = 1;
+	on_global_recover(soid, stat_diff, true);
+	return seastar::make_ready_future<>();
+      })
+    );
+  } else {
+    return recovering.wait_for_recovered();
+  }
 }
 
 RecoveryBackend::interruptible_future<> PGRecovery::prep_object_replica_pushes(
@@ -305,15 +319,22 @@ RecoveryBackend::interruptible_future<> PGRecovery::prep_object_replica_pushes(
   const hobject_t& soid,
   eversion_t need)
 {
-  return pg->get_recovery_backend()->add_recovering(soid).wait_track_blocking(
-    trigger,
-    pg->get_recovery_backend()->recover_object(soid, need)
-    .handle_exception_interruptible(
-      [=, this, soid = std::move(soid)] (auto e) {
-      on_failed_recover({ pg->get_pg_whoami() }, soid, need);
-      return seastar::make_ready_future<>();
-    })
-  );
+  logger().info("{} {} v {}", __func__, soid, need);
+  auto [recovering, added] = pg->get_recovery_backend()->add_recovering(soid);
+  if (added) {
+    logger().info("{} {} v {}, new recovery", __func__, soid, need);
+    return recovering.wait_track_blocking(
+      trigger,
+      pg->get_recovery_backend()->recover_object(soid, need)
+      .handle_exception_interruptible(
+	[=, this, soid = std::move(soid)] (auto e) {
+	on_failed_recover({ pg->get_pg_whoami() }, soid, need);
+	return seastar::make_ready_future<>();
+      })
+    );
+  } else {
+    return recovering.wait_for_recovered();
+  }
 }
 
 void PGRecovery::on_local_recover(
@@ -449,9 +470,11 @@ void PGRecovery::enqueue_push(
   const hobject_t& obj,
   const eversion_t& v)
 {
-  logger().debug("{}: obj={} v={}",
+  logger().info("{}: obj={} v={}",
                  __func__, obj, v);
-  pg->get_recovery_backend()->add_recovering(obj);
+  auto [recovering, added] = pg->get_recovery_backend()->add_recovering(obj);
+  if (!added)
+    return;
   std::ignore = pg->get_recovery_backend()->recover_object(obj, v).\
   handle_exception_interruptible([] (auto) {
     ceph_abort_msg("got exception on backfill's push");
diff --git a/src/crimson/osd/recovery_backend.h b/src/crimson/osd/recovery_backend.h
index 65e9bb01fbda..abf695891596 100644
--- a/src/crimson/osd/recovery_backend.h
+++ b/src/crimson/osd/recovery_backend.h
@@ -45,10 +45,10 @@ class RecoveryBackend {
       coll{coll},
       backend{backend} {}
   virtual ~RecoveryBackend() {}
-  WaitForObjectRecovery& add_recovering(const hobject_t& soid) {
+  std::pair<WaitForObjectRecovery&, bool> add_recovering(const hobject_t& soid) {
     auto [it, added] = recovering.emplace(soid, new WaitForObjectRecovery{});
-    assert(added);
-    return *(it->second);
+    assert(it->second);
+    return {*(it->second), added};
   }
   WaitForObjectRecovery& get_recovering(const hobject_t& soid) {
     assert(is_recovering(soid));

From 094af522c7062cb91c87991ec55cbf67ee1da4dc Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 24 Aug 2023 12:14:13 +0800
Subject: [PATCH 0022/2492] crimson/osd/pg: discard watches' states after
 iterating all cached obcs

Discarding watches' states while iterating cached obcs might have the
following problem:

1. discard a watch's state
2. the corresponding object context's use_count drops to zero
3. the object context is unreferenced by obc lru
4. obc is deleted from obc lru by lru's evict()
5. obc iteration is corrupted

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg.cc | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 85f6116e604b..696b6651a37f 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1454,14 +1454,19 @@ void PG::on_change(ceph::os::Transaction &t) {
 }
 
 void PG::context_registry_on_change() {
-    obc_registry.for_each([](ObjectContextRef obc) {
-      assert(obc);
-      for (auto j = obc->watchers.begin();
-           j != obc->watchers.end();
-           j = obc->watchers.erase(j)) {
-        j->second->discard_state();
-      }
+  std::vector<seastar::shared_ptr<crimson::osd::Watch>> watchers;
+  obc_registry.for_each([&watchers](ObjectContextRef obc) {
+    assert(obc);
+    for (auto j = obc->watchers.begin();
+         j != obc->watchers.end();
+         j = obc->watchers.erase(j)) {
+      watchers.emplace_back(j->second);
+    }
   });
+
+  for (auto &watcher : watchers) {
+    watcher->discard_state();
+  }
 }
 
 bool PG::can_discard_op(const MOSDOp& m) const {

From 46f01d832487d3a3183783d41450fd8f49347097 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@suse.com>
Date: Fri, 25 Aug 2023 16:10:26 +0300
Subject: [PATCH 0023/2492] test/store_test: get rid off assert_death.

Looks like death assertions aren't 100% reliable and might cause
deadlock sometimes.
Hence getting rid of them and enabling optional sending exception from
*Store::queue_transaction()

Fixes: https://tracker.ceph.com/issues/61193

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/common/options/global.yaml.in          |  6 ++++++
 src/os/bluestore/BlueStore.cc              | 17 +++++++++++++++--
 src/os/bluestore/BlueStore.h               |  6 ++++++
 src/os/kstore/KStore.cc                    | 16 ++++++++++++++--
 src/os/kstore/KStore.h                     |  5 +++++
 src/os/memstore/MemStore.cc                |  6 +++++-
 src/test/objectstore/store_test.cc         | 19 ++++++++++++++-----
 src/test/objectstore/store_test_fixture.cc |  4 ----
 src/test/objectstore/store_test_fixture.h  |  8 --------
 9 files changed, 65 insertions(+), 22 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 3a3a4a137291..cbe895941183 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -6343,3 +6343,9 @@ options:
   default: 0
   services:
   - mgr
+- name: objectstore_debug_throw_on_failed_txc
+  type: bool
+  level: dev
+  desc: Enables exception throwing instead of process abort on transaction submission error.
+  default: false
+  with_legacy: false
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index e3e4833f1a2d..3d9f680ba6c2 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -7852,6 +7852,7 @@ int BlueStore::_mount()
 
 int BlueStore::umount()
 {
+  dout(5) << __func__ << dendl;
   ceph_assert(_kv_only || mounted);
   _osr_drain_all();
 
@@ -14434,7 +14435,13 @@ void BlueStore::_txc_add_transaction(TransContext *txc, Transaction *t)
            << " not handled on operation " << op->op
            << " (op " << pos << ", counting from 0)" << dendl;
       _dump_transaction<0>(cct, t);
-      ceph_abort_msg("unexpected error");
+      if (!g_conf().get_val<bool>("objectstore_debug_throw_on_failed_txc")) {
+	ceph_abort_msg("unexpected error");
+      } else {
+	txc->osr->undo_queue(txc);
+	delete txc;
+	throw r;
+      }
     }
 
     // these operations implicity create the object
@@ -14680,7 +14687,13 @@ void BlueStore::_txc_add_transaction(TransContext *txc, Transaction *t)
              << dendl;
         derr << msg << dendl;
         _dump_transaction<0>(cct, t);
-	ceph_abort_msg("unexpected error");
+	if (!g_conf().get_val<bool>("objectstore_debug_throw_on_failed_txc")) {
+	  ceph_abort_msg("unexpected error");
+	} else {
+	  txc->osr->undo_queue(txc);
+	  delete txc;
+	  throw r;
+	}
       }
     }
   }
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index c3c53264ec1d..486c7cd4de09 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2055,6 +2055,12 @@ class BlueStore : public ObjectStore,
       txc->seq = ++last_seq;
       q.push_back(*txc);
     }
+    void undo_queue(TransContext* txc) {
+      std::lock_guard l(qlock);
+      ceph_assert(&q.back() == txc);
+      --last_seq;
+      q.pop_back();
+    }
 
     void drain() {
       std::unique_lock l(qlock);
diff --git a/src/os/kstore/KStore.cc b/src/os/kstore/KStore.cc
index 9526a756419c..7158486ca388 100644
--- a/src/os/kstore/KStore.cc
+++ b/src/os/kstore/KStore.cc
@@ -2285,7 +2285,13 @@ void KStore::_txc_add_transaction(TransContext *txc, Transaction *t)
       f.close_section();
       f.flush(*_dout);
       *_dout << dendl;
-      ceph_abort_msg("unexpected error");
+      if (!g_conf().get_val<bool>("objectstore_debug_throw_on_failed_txc")) {
+	ceph_abort_msg("unexpected error");
+      } else {
+	txc->osr->undo_queue(txc);
+	delete txc;
+	throw r;
+      }
     }
 
     // object operations
@@ -2534,7 +2540,13 @@ void KStore::_txc_add_transaction(TransContext *txc, Transaction *t)
 	f.close_section();
 	f.flush(*_dout);
 	*_dout << dendl;
-	ceph_abort_msg("unexpected error");
+	if (!g_conf().get_val<bool>("objectstore_debug_throw_on_failed_txc")) {
+	  ceph_abort_msg("unexpected error");
+	} else {
+	  txc->osr->undo_queue(txc);
+	  delete txc;
+	  throw r;
+	}
       }
     }
   }
diff --git a/src/os/kstore/KStore.h b/src/os/kstore/KStore.h
index 9e3c7acd73b4..30a7606fd2fb 100644
--- a/src/os/kstore/KStore.h
+++ b/src/os/kstore/KStore.h
@@ -276,6 +276,11 @@ class KStore : public ObjectStore {
       std::lock_guard<std::mutex> l(qlock);
       q.push_back(*txc);
     }
+    void undo_queue(TransContext* txc) {
+      std::lock_guard<std::mutex> l(qlock);
+      ceph_assert(&q.back() == txc);
+      q.pop_back();
+    }
 
     void flush() {
       std::unique_lock<std::mutex> l(qlock);
diff --git a/src/os/memstore/MemStore.cc b/src/os/memstore/MemStore.cc
index 99e99dcba041..8ada7524dbcd 100644
--- a/src/os/memstore/MemStore.cc
+++ b/src/os/memstore/MemStore.cc
@@ -1032,7 +1032,11 @@ void MemStore::_do_transaction(Transaction& t)
 	f.close_section();
 	f.flush(*_dout);
 	*_dout << dendl;
-	ceph_abort_msg("unexpected error");
+        if (!g_conf().get_val<bool>("objectstore_debug_throw_on_failed_txc")) {
+	  ceph_abort_msg("unexpected error");
+        } else {
+	  throw r;
+        }
       }
     }
 
diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 482d30283041..d23c17a019dc 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -3245,7 +3245,8 @@ TEST_P(StoreTest, SimpleCloneTest) {
   int r;
   coll_t cid;
 
-  SetDeathTestStyle("threadsafe");
+  SetVal(g_conf(), "objectstore_debug_throw_on_failed_txc", "true");
+  g_conf().apply_changes(nullptr);
 
   auto ch = store->create_new_collection(cid);
   {
@@ -3531,8 +3532,12 @@ TEST_P(StoreTest, SimpleCloneTest) {
     ObjectStore::Transaction t;
     t.remove_collection(cid);
     cerr << "Invalid rm coll" << std::endl;
-    PrCtl unset_dumpable;
-    EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), "");
+    try {
+      queue_transaction(store, ch, std::move(t));
+      FAIL() << "remove_collection failed to return ENOTEMPTY.";
+    } catch (int err) {
+      ASSERT_EQ(err, -ENOTEMPTY);
+    }
   }
   {
     ObjectStore::Transaction t;
@@ -3554,8 +3559,12 @@ TEST_P(StoreTest, SimpleCloneTest) {
     t.remove(cid, hoid);
     t.remove(cid, hoid2);
     t.remove_collection(cid);
-    PrCtl unset_dumpable;
-    EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), "");
+    try {
+      queue_transaction(store, ch, std::move(t));
+      FAIL() << "remove_collection failed to return ENOTEMPTY.";
+    } catch (int err) {
+      ASSERT_EQ(err, -ENOTEMPTY);
+    }
   }
   {
     ObjectStore::Transaction t;
diff --git a/src/test/objectstore/store_test_fixture.cc b/src/test/objectstore/store_test_fixture.cc
index a3bdc7a36ac3..0cffd79a709d 100644
--- a/src/test/objectstore/store_test_fixture.cc
+++ b/src/test/objectstore/store_test_fixture.cc
@@ -77,10 +77,6 @@ void StoreTestFixture::TearDown()
   // config settings. Hence setting it to 'unsafe' here as test case is closing.
   g_conf()._clear_safe_to_start_threads();
   PopSettings(0);
-  if (!orig_death_test_style.empty()) {
-    ::testing::FLAGS_gtest_death_test_style = orig_death_test_style;
-    orig_death_test_style.clear();
-  }
 }
 
 void StoreTestFixture::SetVal(ConfigProxy& _conf, const char* key, const char* val)
diff --git a/src/test/objectstore/store_test_fixture.h b/src/test/objectstore/store_test_fixture.h
index 3f25fd493d0d..0495c21bd327 100644
--- a/src/test/objectstore/store_test_fixture.h
+++ b/src/test/objectstore/store_test_fixture.h
@@ -13,8 +13,6 @@ class StoreTestFixture : virtual public ::testing::Test {
   std::stack<std::pair<std::string, std::string>> saved_settings;
   ConfigProxy* conf = nullptr;
 
-  std::string orig_death_test_style;
-
 public:
   std::unique_ptr<ObjectStore> store;
   ObjectStore::CollectionHandle ch;
@@ -25,12 +23,6 @@ class StoreTestFixture : virtual public ::testing::Test {
 
   void SetUp() override;
   void TearDown() override;
-  void SetDeathTestStyle(const char* new_style) {
-    if (orig_death_test_style.empty()) {
-      orig_death_test_style = ::testing::FLAGS_gtest_death_test_style;
-    }
-    ::testing::FLAGS_gtest_death_test_style = new_style;
-  }
 
   void SetVal(ConfigProxy& conf, const char* key, const char* val);
   struct SettingsBookmark {

From a778e1533a2e0e6edc75916c7005cfc5a4baa41d Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Fri, 23 Jun 2023 05:03:21 +0000
Subject: [PATCH 0024/2492] src/tools/ceph_dedup_tool: add prints to check the
 dedup progress

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 50 +++++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index b8c79efa42ef..fa6bd5e4d5dc 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -551,6 +551,17 @@ class SampleDedupWorkerThread : public Thread
   public:
     using dup_count_t = ssize_t;
 
+    void maybe_print_status() {
+      utime_t now = ceph_clock_now();
+      if (next_report != utime_t() && now > next_report) {
+	cerr << (int)(now - start) << "s : read "
+	     << total_bytes << " bytes so far..."
+	     << std::endl;
+	next_report = now;
+	next_report += report_period;
+      }
+    }
+
     bool find(string& fp) {
       std::shared_lock lock(fingerprint_lock);
       auto found_item = fp_map.find(fp);
@@ -562,6 +573,8 @@ class SampleDedupWorkerThread : public Thread
       std::unique_lock lock(fingerprint_lock);
       auto found_iter = fp_map.find(chunk.fingerprint);
       ssize_t cur_reference = 1;
+      total_bytes += chunk.size;
+      maybe_print_status();
       if (found_iter == fp_map.end()) {
         fp_map.insert({chunk.fingerprint, 1});
       } else {
@@ -570,12 +583,20 @@ class SampleDedupWorkerThread : public Thread
       return cur_reference >= dedup_threshold && dedup_threshold != -1;
     }
 
-    FpStore(size_t chunk_threshold) : dedup_threshold(chunk_threshold) { }
+    FpStore(size_t chunk_threshold, uint32_t report_period) :
+      dedup_threshold(chunk_threshold), report_period(report_period) {
+      next_report = start;
+      next_report += report_period;
+    }
 
   private:
     ssize_t dedup_threshold = -1;
     std::unordered_map<std::string, dup_count_t> fp_map;
     std::shared_mutex fingerprint_lock;
+    const utime_t start = ceph_clock_now();
+    utime_t next_report;
+    const uint32_t report_period = default_report_period;
+    size_t total_bytes = 0;
   };
 
   struct SampleDedupGlobal {
@@ -583,8 +604,9 @@ class SampleDedupWorkerThread : public Thread
     const double sampling_ratio = -1;
     SampleDedupGlobal(
       int chunk_threshold,
-      int sampling_ratio) :
-      fp_store(chunk_threshold),
+      int sampling_ratio,
+      uint32_t report_period) :
+      fp_store(chunk_threshold, report_period),
       sampling_ratio(static_cast<double>(sampling_ratio) / 100) { }
   };
 
@@ -608,6 +630,14 @@ class SampleDedupWorkerThread : public Thread
 
   ~SampleDedupWorkerThread() { };
 
+  size_t get_total_duplicated_size() const {
+    return total_duplicated_size;
+  }
+
+  size_t get_total_object_size() const {
+    return total_object_size;
+  }
+
 protected:
   void* entry() override {
     crawl();
@@ -1527,6 +1557,7 @@ int make_crawling_daemon(const po::variables_map &opts)
   string base_pool_name = get_opts_pool_name(opts);
   string chunk_pool_name = get_opts_chunk_pool(opts);
   unsigned max_thread = get_opts_max_thread(opts);
+  uint32_t report_period = default_report_period;
 
   bool loop = false;
   if (opts.count("loop")) {
@@ -1550,6 +1581,7 @@ int make_crawling_daemon(const po::variables_map &opts)
   }
 
   std::string chunk_algo = get_opts_chunk_algo(opts);
+  report_period = get_opts_report_period(opts);
 
   Rados rados;
   int ret = rados.init_with_context(g_ceph_context);
@@ -1640,9 +1672,11 @@ int make_crawling_daemon(const po::variables_map &opts)
     }
 
     SampleDedupWorkerThread::SampleDedupGlobal sample_dedup_global(
-      chunk_dedup_threshold, sampling_ratio);
+      chunk_dedup_threshold, sampling_ratio, report_period);
 
     std::list<SampleDedupWorkerThread> threads;
+    size_t total_size = 0;
+    size_t total_duplicate_size = 0;
     for (unsigned i = 0; i < max_thread; i++) {
       cout << " add thread.. " << std::endl;
       ObjectCursor shard_start;
@@ -1668,8 +1702,16 @@ int make_crawling_daemon(const po::variables_map &opts)
     }
 
     for (auto &p : threads) {
+      total_size += p.get_total_object_size();
+      total_duplicate_size += p.get_total_duplicated_size();
       p.join();
     }
+
+    cerr << "Summary: read "
+	 << total_size << " bytes so far and found saveable space ("
+	 << total_duplicate_size << " bytes)."
+	 << std::endl;
+
     if (loop) {
       sleep(wakeup_period);
     } else {

From ced1627fb25b0a42cd53152b06a8cd452ba31482 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Fri, 23 Jun 2023 07:07:07 +0000
Subject: [PATCH 0025/2492] src/tools/ceph_dedup_tool: print the progress in
 the process of chunk scrub

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index fa6bd5e4d5dc..c8432f87fa26 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -433,6 +433,7 @@ void EstimateDedupRatio::estimate_dedup_ratio()
   }
 }
 
+static void print_chunk_scrub();
 void ChunkScrub::chunk_scrub_common()
 {
   ObjectCursor shard_start;
@@ -459,6 +460,13 @@ void ChunkScrub::chunk_scrub_common()
     &shard_start,
     &shard_end);
 
+  const utime_t start = ceph_clock_now();
+  utime_t next_report;
+  if (report_period) {
+    next_report = start;
+    next_report += report_period;
+  }
+
   ObjectCursor c(shard_start);
   while(c < shard_end)
   {
@@ -477,6 +485,17 @@ void ChunkScrub::chunk_scrub_common()
 	delete formatter;
 	return;
       }
+
+      utime_t now = ceph_clock_now();
+      if (n == 0 && // first thread only
+	  next_report != utime_t() && now > next_report) {
+	cerr << (int)(now - start) << "s, interim findings is : "
+	     << std::endl;
+	print_chunk_scrub();
+	next_report = now;
+	next_report += report_period;
+      }
+
       auto oid = i.oid;
       cout << oid << std::endl;
       chunk_refs_t refs;

From bbce7f8ac8e9c5a096e59d6b388b0294ab8bff19 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Fri, 23 Jun 2023 13:32:15 +0000
Subject: [PATCH 0026/2492] src/tools/ceph_dedup_tool: verbose print only if
 debug is enabled

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index c8432f87fa26..b4a461aabed5 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -497,7 +497,9 @@ void ChunkScrub::chunk_scrub_common()
       }
 
       auto oid = i.oid;
-      cout << oid << std::endl;
+      if (debug) {
+	cout << oid << std::endl;
+      }
       chunk_refs_t refs;
       {
 	bufferlist t;

From 315ad2605ad15ff6bf4cfe525df65661b134844b Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Tue, 29 Aug 2023 08:04:17 +0000
Subject: [PATCH 0027/2492] src/tools/ceph_dedup_tool: move default values to
 options_description

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index b4a461aabed5..c394ec829f48 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -132,8 +132,6 @@ map<uint64_t, EstimateResult> dedup_estimates;  // chunk size -> result
 
 using namespace librados;
 unsigned default_op_size = 1 << 26;
-unsigned default_max_thread = 2;
-int32_t default_report_period = 10;
 ceph::mutex glock = ceph::make_mutex("glock");
 
 po::options_description make_usage() {
@@ -169,8 +167,8 @@ po::options_description make_usage() {
     ("chunk-algorithm", po::value<std::string>(), ": <fixed|fastcdc>, set chunk-algorithm")
     ("fingerprint-algorithm", po::value<std::string>(), ": <sha1|sha256|sha512>, set fingerprint-algorithm")
     ("chunk-pool", po::value<std::string>(), ": set chunk pool name")
-    ("max-thread", po::value<int>(), ": set max thread")
-    ("report-period", po::value<int>(), ": set report-period")
+    ("max-thread", po::value<int>()->default_value(2), ": set max thread")
+    ("report-period", po::value<int>()->default_value(10), ": set report-period")
     ("max-seconds", po::value<int>(), ": set max runtime")
     ("max-read-size", po::value<int>(), ": set max read size")
     ("pool", po::value<std::string>(), ": set pool name")
@@ -616,7 +614,7 @@ class SampleDedupWorkerThread : public Thread
     std::shared_mutex fingerprint_lock;
     const utime_t start = ceph_clock_now();
     utime_t next_report;
-    const uint32_t report_period = default_report_period;
+    const uint32_t report_period;
     size_t total_bytes = 0;
   };
 
@@ -1029,8 +1027,8 @@ int estimate_dedup_ratio(const po::variables_map &opts)
   uint64_t chunk_size = 8192;
   uint64_t min_chunk_size = 8192;
   uint64_t max_chunk_size = 4*1024*1024;
-  unsigned max_thread = default_max_thread;
-  uint32_t report_period = default_report_period;
+  unsigned max_thread = get_opts_max_thread(opts);
+  uint32_t report_period = get_opts_report_period(opts);
   uint64_t max_read_size = default_op_size;
   uint64_t max_seconds = 0;
   int ret;
@@ -1069,8 +1067,6 @@ int estimate_dedup_ratio(const po::variables_map &opts)
   } else {
     cout << "4MB is set as max chunk size by default" << std::endl;
   }
-  max_thread = get_opts_max_thread(opts);
-  report_period = get_opts_report_period(opts);
   if (opts.count("max-seconds")) {
     max_seconds = opts["max-seconds"].as<int>();
   } else {
@@ -1188,9 +1184,9 @@ int chunk_scrub_common(const po::variables_map &opts)
   std::string object_name, target_object_name;
   string chunk_pool_name, op_name;
   int ret;
-  unsigned max_thread = default_max_thread;
+  unsigned max_thread = get_opts_max_thread(opts);
   std::map<std::string, std::string>::const_iterator i;
-  uint32_t report_period = default_report_period;
+  uint32_t report_period = get_opts_report_period(opts);
   ObjectCursor begin;
   ObjectCursor end;
   librados::pool_stat_t s; 
@@ -1341,8 +1337,6 @@ int chunk_scrub_common(const po::variables_map &opts)
     return 0;
   }
 
-  max_thread = get_opts_max_thread(opts);
-  report_period = get_opts_report_period(opts);
   glock.lock();
   begin = chunk_io_ctx.object_list_begin();
   end = chunk_io_ctx.object_list_end();
@@ -1578,7 +1572,7 @@ int make_crawling_daemon(const po::variables_map &opts)
   string base_pool_name = get_opts_pool_name(opts);
   string chunk_pool_name = get_opts_chunk_pool(opts);
   unsigned max_thread = get_opts_max_thread(opts);
-  uint32_t report_period = default_report_period;
+  uint32_t report_period = get_opts_report_period(opts);
 
   bool loop = false;
   if (opts.count("loop")) {
@@ -1602,7 +1596,6 @@ int make_crawling_daemon(const po::variables_map &opts)
   }
 
   std::string chunk_algo = get_opts_chunk_algo(opts);
-  report_period = get_opts_report_period(opts);
 
   Rados rados;
   int ret = rados.init_with_context(g_ceph_context);

From 1b7a7a8df88ffac007dbafdecc131807de66c046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 30 Aug 2023 15:19:07 +0200
Subject: [PATCH 0028/2492] common/weighted_shuffle: don't feed
 std::discrete_distribution with all-zero weights
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This flaw results is the assertions like the following one:

```
/usr/include/c++/11/bits/random.tcc:2667: void std::discrete_distribution<_IntType>::param_type::_M_initialize() [with _IntType = int]: Assertion '__sum > 0' failed.
Aborted (core dumped)
```

The reason behind is that `std::discrete_distribution` sums
the weights and uses the result as a divisor.

Fixes: https://tracker.ceph.com/issues/62645
Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/common/weighted_shuffle.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/common/weighted_shuffle.h b/src/common/weighted_shuffle.h
index 10def0a011a4..dd8f22da014d 100644
--- a/src/common/weighted_shuffle.h
+++ b/src/common/weighted_shuffle.h
@@ -14,6 +14,8 @@ void weighted_shuffle(RandomIt first, RandomIt last,
 {
   if (first == last) {
     return;
+  } else if (std::accumulate(weight_first, weight_last, 0) == 0) {
+    return;
   } else {
     std::discrete_distribution d{weight_first, weight_last};
     if (auto n = d(g); n > 0) {

From d02b17ff84c61123ed27d79dc177c2cfbbe6a72f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 30 Aug 2023 15:23:34 +0200
Subject: [PATCH 0029/2492] test/test_weighted_shuffle: verify weights
 containing zeros
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/test/test_weighted_shuffle.cc | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/src/test/test_weighted_shuffle.cc b/src/test/test_weighted_shuffle.cc
index 9f92cbdc0951..efc1cdeb7cb1 100644
--- a/src/test/test_weighted_shuffle.cc
+++ b/src/test/test_weighted_shuffle.cc
@@ -37,3 +37,55 @@ TEST(WeightedShuffle, Basic) {
 		epsilon);
   }
 }
+
+TEST(WeightedShuffle, ZeroedWeights) {
+  std::array<char, 5> choices{'a', 'b', 'c', 'd', 'e'};
+  std::array<int, 5> weights{0, 0, 0, 0, 0};
+  std::map<char, std::array<unsigned, 5>> frequency {
+    {'a', {0, 0, 0, 0, 0}},
+    {'b', {0, 0, 0, 0, 0}},
+    {'c', {0, 0, 0, 0, 0}},
+    {'d', {0, 0, 0, 0, 0}},
+    {'e', {0, 0, 0, 0, 0}}
+  }; // count each element appearing in each position
+  const int samples = 10000;
+  std::random_device rd;
+  for (auto i = 0; i < samples; i++) {
+    weighted_shuffle(begin(choices), end(choices),
+		     begin(weights), end(weights),
+		     std::mt19937{rd()});
+    for (size_t j = 0; j < choices.size(); ++j)
+      ++frequency[choices[j]][j];
+  }
+
+  for (char ch : choices) {
+    // all samples on the diagonal
+    ASSERT_EQ(std::accumulate(begin(frequency[ch]), end(frequency[ch]), 0),
+	      samples);
+    ASSERT_EQ(frequency[ch][ch-'a'], samples);
+  }
+}
+
+TEST(WeightedShuffle, SingleNonZeroWeight) {
+  std::array<char, 5> choices{'a', 'b', 'c', 'd', 'e'};
+  std::array<int, 5> weights{0, 42, 0, 0, 0};
+  std::map<char, std::array<unsigned, 5>> frequency {
+    {'a', {0, 0, 0, 0, 0}},
+    {'b', {0, 0, 0, 0, 0}},
+    {'c', {0, 0, 0, 0, 0}},
+    {'d', {0, 0, 0, 0, 0}},
+    {'e', {0, 0, 0, 0, 0}}
+  }; // count each element appearing in each position
+  const int samples = 10000;
+  std::random_device rd;
+  for (auto i = 0; i < samples; i++) {
+    weighted_shuffle(begin(choices), end(choices),
+		     begin(weights), end(weights),
+		     std::mt19937{rd()});
+    for (size_t j = 0; j < choices.size(); ++j)
+      ++frequency[choices[j]][j];
+  }
+
+  // 'b' is always first
+  ASSERT_EQ(frequency['b'][0], samples);
+}

From f4e2c3351f6d871ffe38c66e95ac99688f1d28c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 5 Sep 2023 15:21:21 +0200
Subject: [PATCH 0030/2492] crimson: drop store from ECBackend to not shadow
 PGBackend::store
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This problem was leading to crashes like this one:

```
../src/crimson/os/futurized_store.h:113:41: runtime error: member access within misaligned address 0xbebebebebebebebe for type 'struct Shard', which requires 8 byte alignment
0xbebebebebebebebe: note: pointer points here
<memory cannot be printed>
--Type <RET> for more, q to quit, c to continue without paging--

Thread 1 "crimson-osd" received signal SIGSEGV, Segmentation fault.
0x000055558e7a1dca in crimson::os::FuturizedStore::Shard::do_transaction (txn=..., ch=..., this=0xbebebebebebebebe) at ../src/crimson/os/futurized_store.h:113
113	      return do_transaction_no_callbacks(
(gdb) bt
```

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/crimson/osd/ec_backend.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/crimson/osd/ec_backend.h b/src/crimson/osd/ec_backend.h
index 3dbcc4def2e0..56fbb4454231 100644
--- a/src/crimson/osd/ec_backend.h
+++ b/src/crimson/osd/ec_backend.h
@@ -33,7 +33,6 @@ class ECBackend : public PGBackend
 		      epoch_t min_epoch, epoch_t max_epoch,
 		      std::vector<pg_log_entry_t>&& log_entries) final;
   CollectionRef coll;
-  crimson::os::FuturizedStore::Shard* store;
   seastar::future<> request_committed(const osd_reqid_t& reqid,
 				       const eversion_t& version) final {
     return seastar::now();

From 44c24aaf33a6b3f552c49da7da63656f097c3914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 1 Sep 2023 19:39:49 +0200
Subject: [PATCH 0031/2492] mon/OSDMonitor: fix the hint for set-allow-crimson
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/mon/OSDMonitor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 4ad44d0309d5..ebb130a072cd 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3518,7 +3518,7 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
       if (!osdmap.get_allow_crimson()) {
 	mon.clog->info()
 	  << "Disallowing boot of crimson-osd without allow_crimson "
-	  << "OSDMap flag.  Run ceph osd set_allow_crimson to set "
+	  << "OSDMap flag.  Run ceph osd set-allow-crimson to set "
 	  << "allow_crimson flag.  Note that crimson-osd is "
 	  << "considered unstable and may result in crashes or "
 	  << "data loss.  Its usage should be restricted to "

From a78e660728c6c0442cdbfa65db776b5856aee933 Mon Sep 17 00:00:00 2001
From: Kim Minjong <make.dirty.code@gmail.com>
Date: Fri, 3 Feb 2023 11:47:47 +0900
Subject: [PATCH 0032/2492] ceph-volume: fix a bug in
 _check_generic_reject_reasons

The types of removable and ro are wrong. Here, both filters are not
working at all. Changed this from integer to string and corrected the test
data.

Delete redundant logic in get_block_devs_sysfs. Given the name of the
function, I think it is correct to judge from _check_generic_reject_reasons,
and in fact it was before v17.2.4.

Fixes: https://tracker.ceph.com/issues/58591
Signed-off-by: Kim Minjong <make.dirty.code@gmail.com>
---
 .../ceph_volume/tests/util/test_device.py     | 22 +++++++++----------
 src/ceph-volume/ceph_volume/util/device.py    |  4 ++--
 src/ceph-volume/ceph_volume/util/disk.py      |  2 --
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/tests/util/test_device.py b/src/ceph-volume/ceph_volume/tests/util/test_device.py
index e382981d9232..e2ea026286f9 100644
--- a/src/ceph-volume/ceph_volume/tests/util/test_device.py
+++ b/src/ceph-volume/ceph_volume/tests/util/test_device.py
@@ -241,7 +241,7 @@ def test_is_ceph_disk_member_not_available_blkid(self, fake_call, monkeypatch, p
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_reject_removable_device(self, fake_call, device_info):
-        data = {"/dev/sdb": {"removable": 1}}
+        data = {"/dev/sdb": {"removable": "1"}}
         lsblk = {"TYPE": "disk", "NAME": "sdb"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/sdb")
@@ -249,7 +249,7 @@ def test_reject_removable_device(self, fake_call, device_info):
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_reject_device_with_gpt_headers(self, fake_call, device_info):
-        data = {"/dev/sdb": {"removable": 0, "size": 5368709120}}
+        data = {"/dev/sdb": {"removable": "0", "size": 5368709120}}
         lsblk = {"TYPE": "disk", "NAME": "sdb"}
         blkid= {"PTTYPE": "gpt"}
         device_info(
@@ -262,7 +262,7 @@ def test_reject_device_with_gpt_headers(self, fake_call, device_info):
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_accept_non_removable_device(self, fake_call, device_info):
-        data = {"/dev/sdb": {"removable": 0, "size": 5368709120}}
+        data = {"/dev/sdb": {"removable": "0", "size": 5368709120}}
         lsblk = {"TYPE": "disk", "NAME": "sdb"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/sdb")
@@ -286,7 +286,7 @@ def test_accept_symlink_to_device(self,
                                       fake_call):
         m_os_path_islink.return_value = True
         m_os_path_realpath.return_value = '/dev/sdb'
-        data = {"/dev/sdb": {"ro": 0, "size": 5368709120}}
+        data = {"/dev/sdb": {"ro": "0", "size": 5368709120}}
         lsblk = {"TYPE": "disk"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/test_symlink")
@@ -304,7 +304,7 @@ def test_reject_symlink_to_device_mapper(self,
                                              fake_call):
         m_os_path_islink.return_value = True
         m_os_readlink.return_value = '/dev/dm-0'
-        data = {"/dev/mapper/mpatha": {"ro": 0, "size": 5368709120}}
+        data = {"/dev/mapper/mpatha": {"ro": "0", "size": 5368709120}}
         lsblk = {"TYPE": "disk"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/mapper/mpatha")
@@ -312,7 +312,7 @@ def test_reject_symlink_to_device_mapper(self,
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_reject_readonly_device(self, fake_call, device_info):
-        data = {"/dev/cdrom": {"ro": 1}}
+        data = {"/dev/cdrom": {"ro": "1"}}
         lsblk = {"TYPE": "disk", "NAME": "cdrom"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/cdrom")
@@ -328,7 +328,7 @@ def test_reject_smaller_than_5gb(self, fake_call, device_info):
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_accept_non_readonly_device(self, fake_call, device_info):
-        data = {"/dev/sda": {"ro": 0, "size": 5368709120}}
+        data = {"/dev/sda": {"ro": "0", "size": 5368709120}}
         lsblk = {"TYPE": "disk", "NAME": "sda"}
         device_info(devices=data,lsblk=lsblk)
         disk = device.Device("/dev/sda")
@@ -594,10 +594,10 @@ class TestDeviceOrdering(object):
 
     def setup_method(self):
         self.data = {
-                "/dev/sda": {"removable": 0},
-                "/dev/sdb": {"removable": 1}, # invalid
-                "/dev/sdc": {"removable": 0},
-                "/dev/sdd": {"removable": 1}, # invalid
+                "/dev/sda": {"removable": "0"},
+                "/dev/sdb": {"removable": "1"}, # invalid
+                "/dev/sdc": {"removable": "0"},
+                "/dev/sdd": {"removable": "1"}, # invalid
         }
 
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index d61222afe0a0..d01d395339d4 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -594,8 +594,8 @@ def has_partitions(self):
 
     def _check_generic_reject_reasons(self):
         reasons = [
-            ('removable', 1, 'removable'),
-            ('ro', 1, 'read-only'),
+            ('removable', '1', 'removable'),
+            ('ro', '1', 'read-only'),
         ]
         rejected = [reason for (k, v, reason) in reasons if
                     self.sys_api.get(k, '') == v]
diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index a96b0f0a7f84..fa900e5da0f3 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -779,8 +779,6 @@ def holder_inner_loop():
             continue
         type_ = 'disk'
         holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
-        if get_file_contents(os.path.join(_sys_block_path, dev, 'removable')) == "1":
-            continue
         if holder_inner_loop():
             continue
         dm_dir_path = os.path.join(_sys_block_path, dev, 'dm')

From bd5e1a83495e31e457827f564c56fba23f4da8c9 Mon Sep 17 00:00:00 2001
From: Kim Minjong <make.dirty.code@gmail.com>
Date: Fri, 3 Feb 2023 13:57:05 +0900
Subject: [PATCH 0033/2492] ceph-volume: allow removable devices but exclude
 USB

Changed the logic to exclude USB devices to pass hot swap devices that were
passed through before the bug.

Fixes: https://tracker.ceph.com/issues/57907
Fixes: https://tracker.ceph.com/issues/58189
Fixes: https://tracker.ceph.com/issues/58306
Fixes: https://tracker.ceph.com/issues/58591
Signed-off-by: Kim Minjong <make.dirty.code@gmail.com>
---
 src/ceph-volume/ceph_volume/util/device.py | 2 +-
 src/ceph-volume/ceph_volume/util/disk.py   | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index d01d395339d4..bb806292f2c4 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -594,7 +594,7 @@ def has_partitions(self):
 
     def _check_generic_reject_reasons(self):
         reasons = [
-            ('removable', '1', 'removable'),
+            ('id_bus', 'usb', 'id_bus'),
             ('ro', '1', 'read-only'),
         ]
         rejected = [reason for (k, v, reason) in reasons if
diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index fa900e5da0f3..dc1d9b1b7920 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -916,6 +916,10 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         metadata['path'] = diskname
         metadata['type'] = block[2]
 
+        # some facts from udevadm
+        p = udevadm_property(sysdir)
+        metadata['id_bus'] = p.get('ID_BUS', '')
+
         device_facts[diskname] = metadata
     return device_facts
 

From 0e95b27402e46c34586f460d2140af48d03fa305 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 7 Sep 2023 11:58:22 -0700
Subject: [PATCH 0034/2492] ceph-volume: Fix unbound var in disk.get_devices()

00ba00fdfab8 looks to have regressed.

Signed-off-by: Zack Cerza <zack@redhat.com>
---
 src/ceph-volume/ceph_volume/util/disk.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index a96b0f0a7f84..0a59cb0ba1cf 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -878,6 +878,7 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         for key, file_ in facts:
             metadata[key] = get_file_contents(os.path.join(sysdir, file_))
 
+        device_slaves = []
         if block[2] != 'part':
             device_slaves = os.listdir(os.path.join(sysdir, 'slaves'))
             metadata['partitions'] = get_partitions_facts(sysdir)

From 3758f6e7433c58b9e62ae35184659cffabdbd133 Mon Sep 17 00:00:00 2001
From: "David.Hall" <mcfadden7586@gmail.com>
Date: Fri, 8 Sep 2023 15:12:49 -0500
Subject: [PATCH 0035/2492] SignatureDoesNotMatch for certain RGW Admin Ops
 endpoints when using v4 auth

https://tracker.ceph.com/issues/62105
Change from std::map<> to std::multimap<> to allow for duplicates
rgwadmin submits duplicates in a very few cases, so we need to handle them.

Signed-off-by: David.Hall <mcfadden7586@gmail.com>
---
 src/rgw/rgw_auth_s3.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index a2def87040ef..ccbdfff0c427 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -574,7 +574,7 @@ std::string get_v4_canonical_qs(const req_info& info, const bool using_qs)
 
   /* Handle case when query string exists. Step 3 described in: http://docs.
    * aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html */
-  std::map<std::string, std::string> canonical_qs_map;
+  std::multimap<std::string, std::string> canonical_qs_map;
   for (const auto& s : get_str_vec<5>(*params, "&")) {
     std::string_view key, val;
     const auto parsed_pair = parse_key_value(s);
@@ -595,7 +595,7 @@ std::string get_v4_canonical_qs(const req_info& info, const bool using_qs)
     // while awsv4 specs ask for all slashes to be encoded, s3 itself is relaxed
     // in its implementation allowing non-url-encoded slashes to be present in
     // presigned urls for instance
-    canonical_qs_map[aws4_uri_recode(key, true)] = aws4_uri_recode(val, true);
+    canonical_qs_map.insert({{aws4_uri_recode(key, true), aws4_uri_recode(val, true)}});
   }
 
   /* Thanks to the early exist we have the guarantee that canonical_qs_map has

From 9215996586ba79302c07e5cf586ec2add1ac159e Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 12 Jul 2023 12:33:58 +0000
Subject: [PATCH 0036/2492] osd/osd_types: Introduce OSDSuperblock::maps

replace oldest/newest_map members with an interval_set in order to
support the tracking of an osdmap range gap.

Map gap example:
```
2023-08-28T18:21:05.452+0000 7f409bce2640  3 osd.4 84 handle_osd_map epochs [85,86], i have 84, src has [1,86]
2023-08-28T18:26:25.829+0000 7fcfea0c1640  3 osd.4 86 handle_osd_map epochs [208,208], i have 86, src has [208,400]
2023-08-28T18:26:25.829+0000 7fcfea0c1640 10 osd.4 86  superblock cluster_osdmap_trim_lower_bound new epoch is: 208
2023-08-28T18:26:25.829+0000 7fcfea0c1640 10 osd.4 86 handle_osd_map osd map gap [16~71,208~1]
2023-08-28T18:26:25.829+0000 7fcfea0c1640  3 osd.4 86 handle_osd_map epochs [209,248], i have 208, src has [208,400]
2023-08-28T18:26:25.833+0000 7fcfea0c1640 10 osd.4 86 handle_osd_map osd map gap [31~56,208~4]
2023-08-28T18:26:25.941+0000 7fcfea0c1640  3 osd.4 211 handle_osd_map epochs [212,248], i have 211, src has [208,400]
2023-08-28T18:26:25.945+0000 7fcfea0c1640 10 osd.4 211 handle_osd_map osd map gap [46~41,208~41]
2023-08-28T18:26:25.949+0000 7fcfea0c1640  3 osd.4 211 handle_osd_map epochs [209,248], i have 248, src has [208,400]
2023-08-28T18:26:25.949+0000 7fcfea0c1640  3 osd.4 211 handle_osd_map epochs [212,251], i have 248, src has [208,400]
2023-08-28T18:26:25.953+0000 7fcfea0c1640 10 osd.4 211 handle_osd_map osd map gap [61~26,208~44]
2023-08-28T18:26:26.073+0000 7fcfea0c1640  3 osd.4 251 handle_osd_map epochs [249,288], i have 251, src has [208,400]
2023-08-28T18:26:26.081+0000 7fcfea0c1640 10 osd.4 251 handle_osd_map osd map gap [76~11,208~48]
2023-08-28T18:26:26.081+0000 7fcfea0c1640  3 osd.4 251 handle_osd_map epochs [252,291], i have 255, src has [208,400]
```

Full example: https://gist.github.com/Matan-B/9b0eed8daee3bd6c3216bd3b6d11e8fb

Fixes: https://tracker.ceph.com/issues/61962

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc             | 23 +++++------
 src/crimson/osd/shard_services.cc  | 12 +++---
 src/mon/OSDMonitor.cc              |  2 +-
 src/osd/OSD.cc                     | 63 ++++++++++++++----------------
 src/osd/osd_types.cc               | 21 ++++++----
 src/osd/osd_types.h                | 28 ++++++++++++-
 src/tools/ceph_objectstore_tool.cc |  4 +-
 src/tools/rebuild_mondb.cc         |  8 ++--
 8 files changed, 93 insertions(+), 68 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index cfe4f54ab2e5..ccb7435332b3 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -708,19 +708,17 @@ void OSD::dump_status(Formatter* f) const
   f->dump_stream("osd_fsid") << superblock.osd_fsid;
   f->dump_unsigned("whoami", superblock.whoami);
   f->dump_string("state", pg_shard_manager.get_osd_state_string());
-  f->dump_unsigned("oldest_map", superblock.oldest_map);
+  f->dump_stream("maps") << superblock.maps;
   f->dump_unsigned("cluster_osdmap_trim_lower_bound",
                    superblock.cluster_osdmap_trim_lower_bound);
-  f->dump_unsigned("newest_map", superblock.newest_map);
   f->dump_unsigned("num_pgs", pg_shard_manager.get_num_pgs());
 }
 
 void OSD::print(std::ostream& out) const
 {
   out << "{osd." << superblock.whoami << " "
-      << superblock.osd_fsid << " [" << superblock.oldest_map
-      << "," << superblock.newest_map << "] "
-      << "tlb:" << superblock.cluster_osdmap_trim_lower_bound
+      << superblock.osd_fsid << " maps " << superblock.maps
+      << " tlb:" << superblock.cluster_osdmap_trim_lower_bound
       << " pgs:" << pg_shard_manager.get_num_pgs()
       << "}";
 }
@@ -934,16 +932,16 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
   const auto first = m->get_first();
   const auto last = m->get_last();
   logger().info("handle_osd_map epochs [{}..{}], i have {}, src has [{}..{}]",
-                first, last, superblock.newest_map,
+                first, last, superblock.get_newest_map(),
                 m->cluster_osdmap_trim_lower_bound, m->newest_map);
   // make sure there is something new, here, before we bother flushing
   // the queues and such
-  if (last <= superblock.newest_map) {
+  if (last <= superblock.get_newest_map()) {
     return seastar::now();
   }
   // missing some?
   bool skip_maps = false;
-  epoch_t start = superblock.newest_map + 1;
+  epoch_t start = superblock.get_newest_map() + 1;
   if (first > start) {
     logger().info("handle_osd_map message skips epochs {}..{}",
                   start, first - 1);
@@ -967,10 +965,13 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
     return pg_shard_manager.store_maps(t, start, m).then([=, this, &t] {
       // even if this map isn't from a mon, we may have satisfied our subscription
       monc->sub_got("osdmap", last);
-      if (!superblock.oldest_map || skip_maps) {
-        superblock.oldest_map = first;
+
+      if (!superblock.maps.empty()) {
+        // TODO: support osdmap trimming
+        // See: <tracker>
       }
-      superblock.newest_map = last;
+
+      superblock.insert_osdmap_epochs(first, last);
       superblock.current_epoch = last;
 
       // note in the superblock that we were clean thru the prior epoch
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index a6431305d806..404f28d7d7f3 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -711,16 +711,16 @@ seastar::future<> OSDSingletonState::send_incremental_map(
 {
   logger().info("{}: first osdmap: {} "
                 "superblock's oldest map: {}",
-                __func__, first, superblock.oldest_map);
-  if (first >= superblock.oldest_map) {
+                __func__, first, superblock.get_oldest_map());
+  if (first >= superblock.get_oldest_map()) {
     return load_map_bls(
-      first, superblock.newest_map
+      first, superblock.get_newest_map()
     ).then([this, &conn, first](auto&& bls) {
       auto m = crimson::make_message<MOSDMap>(
 	monc.get_fsid(),
 	osdmap->get_encoding_features());
       m->cluster_osdmap_trim_lower_bound = first;
-      m->newest_map = superblock.newest_map;
+      m->newest_map = superblock.get_newest_map();
       m->maps = std::move(bls);
       return conn.send(std::move(m));
     });
@@ -736,8 +736,8 @@ seastar::future<> OSDSingletonState::send_incremental_map(
        *       See: OSD::handle_osd_map for how classic updates the
        *       cluster's trim lower bound.
        */
-      m->cluster_osdmap_trim_lower_bound = superblock.oldest_map;
-      m->newest_map = superblock.newest_map;
+      m->cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
+      m->newest_map = superblock.get_newest_map();
       m->maps.emplace(osdmap->get_epoch(), std::move(bl));
       return conn.send(std::move(m));
     });
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 4e3f862b77eb..116e93680c6f 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3656,7 +3656,7 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op)
     }
 
     // fresh osd?
-    if (m->sb.newest_map == 0 && osdmap.exists(from)) {
+    if (m->sb.get_newest_map() == 0 && osdmap.exists(from)) {
       const osd_info_t& i = osdmap.get_info(from);
       if (i.up_from > i.lost_at) {
 	dout(10) << " fresh osd; marking lost_at too" << dendl;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 83c49a08a48d..93f5ca238fab 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1380,7 +1380,7 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
   MOSDMap *m = new MOSDMap(monc->get_fsid(),
 			   osdmap->get_encoding_features());
   m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
-  m->newest_map = sblock.newest_map;
+  m->newest_map = sblock.get_newest_map();
 
   int max = cct->_conf->osd_map_message_max;
   ssize_t max_bytes = cct->_conf->osd_map_message_max_bytes;
@@ -1459,12 +1459,12 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
   MOSDMap *m = NULL;
   while (!m) {
     OSDSuperblock sblock(get_superblock());
-    if (since < sblock.oldest_map) {
+    if (since < sblock.get_oldest_map()) {
       // just send latest full map
       MOSDMap *m = new MOSDMap(monc->get_fsid(),
 			       osdmap->get_encoding_features());
       m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
-      m->newest_map = sblock.newest_map;
+      m->newest_map = sblock.get_newest_map();
       get_map_bl(to, m->maps[to]);
       send_map(m, con);
       return;
@@ -1650,7 +1650,7 @@ void OSDService::handle_misdirected_op(PG *pg, OpRequestRef op)
        * splitting.  The simplest thing is to detect such cases here and drop
        * them without an error (the client will resend anyway).
        */
-    ceph_assert(m->get_map_epoch() <= superblock.newest_map);
+    ceph_assert(m->get_map_epoch() <= superblock.get_newest_map());
     OSDMapRef opmap = try_get_map(m->get_map_epoch());
     if (!opmap) {
       dout(7) << __func__ << ": " << *pg << " no longer have map for "
@@ -2705,10 +2705,9 @@ void OSD::asok_command(
     f->dump_stream("osd_fsid") << superblock.osd_fsid;
     f->dump_unsigned("whoami", superblock.whoami);
     f->dump_string("state", get_state_name(get_state()));
-    f->dump_unsigned("oldest_map", superblock.oldest_map);
+    f->dump_stream("maps") << superblock.maps;
     f->dump_unsigned("cluster_osdmap_trim_lower_bound",
                      superblock.cluster_osdmap_trim_lower_bound);
-    f->dump_unsigned("newest_map", superblock.newest_map);
     f->dump_unsigned("num_pgs", num_pgs);
     f->close_section();
   } else if (prefix == "flush_journal") {
@@ -3763,7 +3762,7 @@ int OSD::init()
     dout(5) << "Upgrading superblock adding: " << diff << dendl;
 
     if (!superblock.cluster_osdmap_trim_lower_bound) {
-      superblock.cluster_osdmap_trim_lower_bound = superblock.oldest_map;
+      superblock.cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
     }
 
     ObjectStore::Transaction t;
@@ -6277,7 +6276,7 @@ void OSD::tick_without_osd_lock()
     if (max_waiting_epoch > get_osdmap()->get_epoch()) {
       dout(20) << __func__ << " max_waiting_epoch " << max_waiting_epoch
 	       << ", requesting new map" << dendl;
-      osdmap_subscribe(superblock.newest_map + 1, false);
+      osdmap_subscribe(superblock.get_newest_map() + 1, false);
     }
   }
 
@@ -6638,8 +6637,7 @@ void OSD::start_boot()
   }
   dout(1) << __func__ << dendl;
   set_state(STATE_PREBOOT);
-  dout(10) << "start_boot - have maps " << superblock.oldest_map
-	   << ".." << superblock.newest_map << dendl;
+  dout(10) << "start_boot - have maps " << superblock.maps << dendl;
   monc->get_version("osdmap", CB_OSD_GetVersion(this));
 }
 
@@ -7952,20 +7950,20 @@ void OSD::trim_maps(epoch_t oldest, bool skip_maps)
    */
   epoch_t min = std::min(oldest, service.map_cache.cached_key_lower_bound());
   dout(20) <<  __func__ << ": min=" << min << " oldest_map="
-           << superblock.oldest_map << " skip_maps=" << skip_maps
+           << superblock.get_oldest_map() << " skip_maps=" << skip_maps
            << dendl;
-  if (min <= superblock.oldest_map)
+  if (min <= superblock.get_oldest_map())
     return;
 
   // Trim from the superblock's oldest_map up to `min`.
   // Break if we have exceeded the txn target size.
   // If skip_maps is true, we will trim up `min` unconditionally.
   ObjectStore::Transaction t;
-  while (superblock.oldest_map < min) {
-    dout(20) << " removing old osdmap epoch " << superblock.oldest_map << dendl;
-    t.remove(coll_t::meta(), get_osdmap_pobject_name(superblock.oldest_map));
-    t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(superblock.oldest_map));
-    ++superblock.oldest_map;
+  while (superblock.superblock.get_oldest_map() < min) {
+    dout(20) << " removing old osdmap epoch " << superblock.superblock.get_oldest_map() << dendl;
+    t.remove(coll_t::meta(), get_osdmap_pobject_name(superblock.superblock.get_oldest_map()));
+    t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(superblock.superblock.get_oldest_map()));
+    superblock.maps.erase(superblock.get_oldest_map());
     if (t.get_num_ops() > cct->_conf->osd_target_transaction_size) {
       service.publish_superblock(superblock);
       write_superblock(cct, superblock, t);
@@ -8057,15 +8055,15 @@ void OSD::handle_osd_map(MOSDMap *m)
   epoch_t first = m->get_first();
   epoch_t last = m->get_last();
   dout(3) << "handle_osd_map epochs [" << first << "," << last << "], i have "
-	  << superblock.newest_map
+	  << superblock.get_newest_map()
 	  << ", src has [" << m->cluster_osdmap_trim_lower_bound
           << "," << m->newest_map << "]"
 	  << dendl;
 
   logger->inc(l_osd_map);
   logger->inc(l_osd_mape, last - first + 1);
-  if (first <= superblock.newest_map)
-    logger->inc(l_osd_mape_dup, superblock.newest_map - first + 1);
+  if (first <= superblock.get_newest_map())
+    logger->inc(l_osd_mape_dup, superblock.get_newest_map() - first + 1);
 
   if (superblock.cluster_osdmap_trim_lower_bound <
       m->cluster_osdmap_trim_lower_bound) {
@@ -8074,12 +8072,12 @@ void OSD::handle_osd_map(MOSDMap *m)
     dout(10) << " superblock cluster_osdmap_trim_lower_bound new epoch is: "
              << superblock.cluster_osdmap_trim_lower_bound << dendl;
     ceph_assert(
-      superblock.cluster_osdmap_trim_lower_bound >= superblock.oldest_map);
+      superblock.cluster_osdmap_trim_lower_bound >= superblock.get_oldest_map());
   }
 
   // make sure there is something new, here, before we bother flushing
   // the queues and such
-  if (last <= superblock.newest_map) {
+  if (last <= superblock.get_newest_map()) {
     dout(10) << " no new maps here, dropping" << dendl;
     m->put();
     return;
@@ -8087,11 +8085,11 @@ void OSD::handle_osd_map(MOSDMap *m)
 
   // missing some?
   bool skip_maps = false;
-  if (first > superblock.newest_map + 1) {
+  if (first > superblock.get_newest_map() + 1) {
     dout(10) << "handle_osd_map message skips epochs "
-	     << superblock.newest_map + 1 << ".." << (first-1) << dendl;
-    if (m->cluster_osdmap_trim_lower_bound <= superblock.newest_map + 1) {
-      osdmap_subscribe(superblock.newest_map + 1, false);
+	     << superblock.get_newest_map() + 1 << ".." << (first-1) << dendl;
+    if (m->cluster_osdmap_trim_lower_bound <= superblock.get_newest_map() + 1) {
+      osdmap_subscribe(superblock.get_newest_map() + 1, false);
       m->put();
       return;
     }
@@ -8116,7 +8114,7 @@ void OSD::handle_osd_map(MOSDMap *m)
   map<epoch_t,mempool::osdmap::map<int64_t,snap_interval_set_t>> purged_snaps;
 
   // store new maps: queue for disk and put in the osdmap cache
-  epoch_t start = std::max(superblock.newest_map + 1, first);
+  epoch_t start = std::max(superblock.get_newest_map() + 1, first);
   for (epoch_t e = start; e <= last; e++) {
     if (txn_size >= t.get_num_bytes()) {
       derr << __func__ << " transaction size overflowed" << dendl;
@@ -8227,14 +8225,11 @@ void OSD::handle_osd_map(MOSDMap *m)
     rerequest_full_maps();
   }
 
-  if (superblock.oldest_map) {
+  if (!superblock.maps.empty()) {
     trim_maps(m->cluster_osdmap_trim_lower_bound, skip_maps);
-    pg_num_history.prune(superblock.oldest_map);
+    pg_num_history.prune(superblock.get_oldest_map());
   }
-
-  if (!superblock.oldest_map || skip_maps)
-    superblock.oldest_map = first;
-  superblock.newest_map = last;
+  superblock.insert_osdmap_epochs(first, last);
   superblock.current_epoch = last;
 
   // note in the superblock that we were clean thru the prior epoch
@@ -8360,7 +8355,7 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m)
   for (epoch_t cur = first; cur <= last; cur++) {
     dout(10) << " advance to epoch " << cur
 	     << " (<= last " << last
-	     << " <= newest_map " << superblock.newest_map
+	     << " <= newest_map " << superblock.get_newest_map()
 	     << ")" << dendl;
 
     OSDMapRef newmap = get_map(cur);
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 664d8a287406..948abeaafc8a 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -5705,12 +5705,12 @@ void pg_hit_set_history_t::generate_test_instances(list<pg_hit_set_history_t*>&
 
 void OSDSuperblock::encode(ceph::buffer::list &bl) const
 {
-  ENCODE_START(10, 5, bl);
+  ENCODE_START(11, 5, bl);
   encode(cluster_fsid, bl);
   encode(whoami, bl);
   encode(current_epoch, bl);
-  encode(oldest_map, bl);
-  encode(newest_map, bl);
+  encode((epoch_t)0, bl); // oldest_map
+  encode((epoch_t)0, bl); // newest_map
   encode(weight, bl);
   compat_features.encode(bl);
   encode(clean_thru, bl);
@@ -5721,12 +5721,13 @@ void OSDSuperblock::encode(ceph::buffer::list &bl) const
   encode(purged_snaps_last, bl);
   encode(last_purged_snaps_scrub, bl);
   encode(cluster_osdmap_trim_lower_bound, bl);
+  encode(maps, bl);
   ENCODE_FINISH(bl);
 }
 
 void OSDSuperblock::decode(ceph::buffer::list::const_iterator &bl)
 {
-  DECODE_START_LEGACY_COMPAT_LEN(10, 5, 5, bl);
+  DECODE_START_LEGACY_COMPAT_LEN(11, 5, 5, bl);
   if (struct_v < 3) {
     string magic;
     decode(magic, bl);
@@ -5734,6 +5735,7 @@ void OSDSuperblock::decode(ceph::buffer::list::const_iterator &bl)
   decode(cluster_fsid, bl);
   decode(whoami, bl);
   decode(current_epoch, bl);
+  epoch_t oldest_map, newest_map;
   decode(oldest_map, bl);
   decode(newest_map, bl);
   decode(weight, bl);
@@ -5765,6 +5767,11 @@ void OSDSuperblock::decode(ceph::buffer::list::const_iterator &bl)
   } else {
     cluster_osdmap_trim_lower_bound = 0;
   }
+  if (struct_v >= 11) {
+    decode(maps, bl);
+  } else {
+    insert_osdmap_epochs(oldest_map, newest_map);
+  }
   DECODE_FINISH(bl);
 }
 
@@ -5774,8 +5781,6 @@ void OSDSuperblock::dump(Formatter *f) const
   f->dump_stream("osd_fsid") << osd_fsid;
   f->dump_int("whoami", whoami);
   f->dump_int("current_epoch", current_epoch);
-  f->dump_int("oldest_map", oldest_map);
-  f->dump_int("newest_map", newest_map);
   f->dump_float("weight", weight);
   f->open_object_section("compat");
   compat_features.dump(f);
@@ -5786,6 +5791,7 @@ void OSDSuperblock::dump(Formatter *f) const
   f->dump_stream("last_purged_snaps_scrub") << last_purged_snaps_scrub;
   f->dump_int("cluster_osdmap_trim_lower_bound",
               cluster_osdmap_trim_lower_bound);
+  f->dump_stream("maps") << maps;
 }
 
 void OSDSuperblock::generate_test_instances(list<OSDSuperblock*>& o)
@@ -5796,8 +5802,7 @@ void OSDSuperblock::generate_test_instances(list<OSDSuperblock*>& o)
   z.osd_fsid.parse("02020202-0202-0202-0202-020202020202");
   z.whoami = 3;
   z.current_epoch = 4;
-  z.oldest_map = 5;
-  z.newest_map = 9;
+  z.insert_osdmap_epochs(5, 9);
   z.mounted = 8;
   z.clean_thru = 7;
   o.push_back(new OSDSuperblock(z));
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 16955ef5ef4d..8b86b0a36356 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -5454,7 +5454,31 @@ class OSDSuperblock {
   uuid_d cluster_fsid, osd_fsid;
   int32_t whoami = -1;    // my role in this fs.
   epoch_t current_epoch = 0;             // most recent epoch
-  epoch_t oldest_map = 0, newest_map = 0;    // oldest/newest maps we have.
+  interval_set<epoch_t> maps; // oldest/newest maps we have.
+
+  epoch_t get_oldest_map() const {
+    if (!maps.empty()) {
+      return maps.range_start();
+    }
+    return 0;
+  }
+
+  epoch_t get_newest_map() const {
+    if (!maps.empty()) {
+      // maps stores [oldest_map, newest_map) (exclusive)
+      return maps.range_end() - 1;
+    }
+    return 0;
+  }
+
+  void insert_osdmap_epochs(epoch_t first, epoch_t last) {
+    ceph_assert(std::cmp_less_equal(first, last));
+    interval_set<epoch_t> message_epochs;
+    message_epochs.insert(first, last - first + 1);
+    maps.union_of(message_epochs);
+    ceph_assert(last == get_newest_map());
+  }
+
   double weight = 0.0;
 
   CompatSet compat_features;
@@ -5481,7 +5505,7 @@ inline std::ostream& operator<<(std::ostream& out, const OSDSuperblock& sb)
              << " osd." << sb.whoami
 	     << " " << sb.osd_fsid
              << " e" << sb.current_epoch
-             << " [" << sb.oldest_map << "," << sb.newest_map << "]"
+             << " maps " << sb.maps
 	     << " lci=[" << sb.mounted << "," << sb.clean_thru << "]"
              << " tlb=" << sb.cluster_osdmap_trim_lower_bound
              << ")";
diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index df6d1f85c39f..19a445824834 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -1630,9 +1630,9 @@ int get_pg_metadata(ObjectStore *store, bufferlist &bl, metadata_section &ms,
     return -EINVAL;
   }
 
-  if (ms.osdmap.get_epoch() < sb.oldest_map) {
+  if (ms.osdmap.get_epoch() < sb.get_oldest_map()) {
     cerr << "PG export's map " << ms.osdmap.get_epoch()
-	 << " is older than OSD's oldest_map " << sb.oldest_map << std::endl;
+	 << " is older than OSD's oldest_map " << sb.get_oldest_map() << std::endl;
     if (!force) {
       cerr << " pass --force to proceed anyway (with incomplete PastIntervals)"
 	   << std::endl;
diff --git a/src/tools/rebuild_mondb.cc b/src/tools/rebuild_mondb.cc
index 17e4dadcfdd4..033f63aad22a 100644
--- a/src/tools/rebuild_mondb.cc
+++ b/src/tools/rebuild_mondb.cc
@@ -216,7 +216,7 @@ int update_osdmap(ObjectStore& fs, OSDSuperblock& sb, MonitorDBStore& ms)
   // osdmap starts at 1. if we have a "0" first_committed, then there is nothing
   // to trim. and "1 osdmaps trimmed" in the output message is misleading. so
   // let's make it an exception.
-  for (auto e = first_committed; first_committed && e < sb.oldest_map; e++) {
+  for (auto e = first_committed; first_committed && e < sb.get_oldest_map(); e++) {
     t->erase(prefix, e);
     t->erase(prefix, ms.combine_strings("full", e));
     ntrimmed++;
@@ -225,7 +225,7 @@ int update_osdmap(ObjectStore& fs, OSDSuperblock& sb, MonitorDBStore& ms)
   // because PaxosService::put_last_committed() set it to last_committed, if it
   // is zero. which breaks OSDMonitor::update_from_paxos(), in which we believe
   // that latest_full should always be greater than last_committed.
-  if (first_committed == 0 && sb.oldest_map < sb.newest_map) {
+  if (first_committed == 0 && sb.get_oldest_map() < sb.get_newest_map()) {
     first_committed = 1;
   } else if (ntrimmed) {
     first_committed += ntrimmed;
@@ -240,8 +240,8 @@ int update_osdmap(ObjectStore& fs, OSDSuperblock& sb, MonitorDBStore& ms)
 
   auto ch = fs.open_collection(coll_t::meta());
   OSDMap osdmap;
-  for (auto e = std::max(last_committed+1, sb.oldest_map);
-       e <= sb.newest_map; e++) {
+  for (auto e = std::max(last_committed+1, sb.get_oldest_map());
+       e <= sb.get_newest_map(); e++) {
     bool have_crc = false;
     uint32_t crc = -1;
     uint64_t features = 0;

From 05aeeeebe634213c882fb25842afb4679e6fd61d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 17 Jul 2023 13:12:27 +0000
Subject: [PATCH 0037/2492] osd/OSD: remove `skip_maps` comment

Now that oldest/newest maps are stored as an interval set we no longer
move the oldest_map forward to `first` epoch. We simply add the new osdmap range
from `first` to `last` regardless of `skip_maps`.
trim_maps now erases the oldest_map each iteration and support epoch gaps.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 93f5ca238fab..2c1d2441b769 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7932,22 +7932,6 @@ void OSD::osdmap_subscribe(version_t epoch, bool force_request)
 
 void OSD::trim_maps(epoch_t oldest, bool skip_maps)
 {
-  /* There's a possible leak here. skip_maps is set to true if the received
-   * MOSDMap message indicates that there's a discontinuity between
-   * the Monitor cluster's stored set of maps and our set of stored
-   * maps such that there is a "gap". This happens generally when an OSD
-   * is down for a while and the cluster has trimmed maps in the mean time.
-   *
-   * Because the superblock cannot represent two discontinuous sets of maps,
-   * OSD::handle_osd_map unconditionally sets superblock.oldest_map to the first
-   * map in the message. OSD::trim_maps here, however, will only trim up to
-   * service.map_cache.cached_key_lower_bound() resulting in the maps between
-   * service.map_cache.cached_key_lower_bound() and MOSDMap::get_first() being
-   * leaked. Note, trimming past service.map_cache.cached_key_lower_bound()
-   * here won't work as there may still be PGs with those map epochs recorded.
-   *
-   * Fixing this is future work: https://tracker.ceph.com/issues/61962
-   */
   epoch_t min = std::min(oldest, service.map_cache.cached_key_lower_bound());
   dout(20) <<  __func__ << ": min=" << min << " oldest_map="
            << superblock.get_oldest_map() << " skip_maps=" << skip_maps
@@ -7980,7 +7964,8 @@ void OSD::trim_maps(epoch_t oldest, bool skip_maps)
     int tr = store->queue_transaction(service.meta_ch, std::move(t), nullptr);
     ceph_assert(tr == 0);
   }
-  // we should not remove the cached maps
+  // we should not trim past service.map_cache.cached_key_lower_bound() 
+  // as there may still be PGs with those map epochs recorded.
   ceph_assert(min <= service.map_cache.cached_key_lower_bound());
 }
 

From 9897164265726895eeee2348ee679aed03e75ea1 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 18 Jul 2023 16:08:23 +0000
Subject: [PATCH 0038/2492] osd/OSD: remove `skip_maps`

the superblock now stores an interval_set which supports non-contiguous osdmap history.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc |  3 ---
 src/osd/OSD.cc         | 44 ++++++++++++++----------------------------
 src/osd/OSD.h          |  2 +-
 3 files changed, 15 insertions(+), 34 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index ccb7435332b3..157881ccbe40 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -940,7 +940,6 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
     return seastar::now();
   }
   // missing some?
-  bool skip_maps = false;
   epoch_t start = superblock.get_newest_map() + 1;
   if (first > start) {
     logger().info("handle_osd_map message skips epochs {}..{}",
@@ -956,8 +955,6 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
       return get_shard_services().osdmap_subscribe(
         m->cluster_osdmap_trim_lower_bound - 1, true);
     }
-    skip_maps = true;
-    start = first;
   }
 
   return seastar::do_with(ceph::os::Transaction{},
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2c1d2441b769..1fd6b8eec3a8 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7930,40 +7930,30 @@ void OSD::osdmap_subscribe(version_t epoch, bool force_request)
   }
 }
 
-void OSD::trim_maps(epoch_t oldest, bool skip_maps)
+void OSD::trim_maps(epoch_t oldest)
 {
   epoch_t min = std::min(oldest, service.map_cache.cached_key_lower_bound());
   dout(20) <<  __func__ << ": min=" << min << " oldest_map="
-           << superblock.get_oldest_map() << " skip_maps=" << skip_maps
-           << dendl;
+           << superblock.get_oldest_map() << dendl;
   if (min <= superblock.get_oldest_map())
     return;
 
   // Trim from the superblock's oldest_map up to `min`.
   // Break if we have exceeded the txn target size.
-  // If skip_maps is true, we will trim up `min` unconditionally.
   ObjectStore::Transaction t;
-  while (superblock.superblock.get_oldest_map() < min) {
-    dout(20) << " removing old osdmap epoch " << superblock.superblock.get_oldest_map() << dendl;
-    t.remove(coll_t::meta(), get_osdmap_pobject_name(superblock.superblock.get_oldest_map()));
-    t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(superblock.superblock.get_oldest_map()));
+  while (superblock.get_oldest_map() < min &&
+         t.get_num_ops() < cct->_conf->osd_target_transaction_size) {
+    dout(20) << " removing old osdmap epoch " << superblock.get_oldest_map() << dendl;
+    t.remove(coll_t::meta(), get_osdmap_pobject_name(superblock.get_oldest_map()));
+    t.remove(coll_t::meta(), get_inc_osdmap_pobject_name(superblock.get_oldest_map()));
     superblock.maps.erase(superblock.get_oldest_map());
-    if (t.get_num_ops() > cct->_conf->osd_target_transaction_size) {
-      service.publish_superblock(superblock);
-      write_superblock(cct, superblock, t);
-      int tr = store->queue_transaction(service.meta_ch, t.claim_and_reset(), nullptr);
-      ceph_assert(tr == 0);
-      if (skip_maps == false) {
-        break;
-      }
-    }
-  }
-  if (t.get_num_ops() > 0) {
-    service.publish_superblock(superblock);
-    write_superblock(cct, superblock, t);
-    int tr = store->queue_transaction(service.meta_ch, std::move(t), nullptr);
-    ceph_assert(tr == 0);
   }
+
+  service.publish_superblock(superblock);
+  write_superblock(cct, superblock, t);
+  int tr = store->queue_transaction(service.meta_ch, std::move(t), nullptr);
+  ceph_assert(tr == 0);
+
   // we should not trim past service.map_cache.cached_key_lower_bound() 
   // as there may still be PGs with those map epochs recorded.
   ceph_assert(min <= service.map_cache.cached_key_lower_bound());
@@ -8068,8 +8058,6 @@ void OSD::handle_osd_map(MOSDMap *m)
     return;
   }
 
-  // missing some?
-  bool skip_maps = false;
   if (first > superblock.get_newest_map() + 1) {
     dout(10) << "handle_osd_map message skips epochs "
 	     << superblock.get_newest_map() + 1 << ".." << (first-1) << dendl;
@@ -8087,10 +8075,6 @@ void OSD::handle_osd_map(MOSDMap *m)
       m->put();
       return;
     }
-    // The superblock's oldest_map should be moved forward (skipped)
-    // to the `first` osdmap of the incoming MOSDMap message.
-    // Trim all of the skipped osdmaps before updating the oldest_map.
-    skip_maps = true;
   }
 
   ObjectStore::Transaction t;
@@ -8211,7 +8195,7 @@ void OSD::handle_osd_map(MOSDMap *m)
   }
 
   if (!superblock.maps.empty()) {
-    trim_maps(m->cluster_osdmap_trim_lower_bound, skip_maps);
+    trim_maps(m->cluster_osdmap_trim_lower_bound);
     pg_num_history.prune(superblock.get_oldest_map());
   }
   superblock.insert_osdmap_epochs(first, last);
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 8ed960d96055..231050d8cc2f 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1685,7 +1685,7 @@ class OSD : public Dispatcher,
 
   void handle_osd_map(class MOSDMap *m);
   void _committed_osd_maps(epoch_t first, epoch_t last, class MOSDMap *m);
-  void trim_maps(epoch_t oldest, bool skip_maps);
+  void trim_maps(epoch_t oldest);
   void note_down_osd(int osd);
   void note_up_osd(int osd);
   friend struct C_OnMapCommit;

From 95a44bf08794d73f0c9cdd7caaa866e2d7ce5913 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 30 Aug 2023 08:17:06 +0000
Subject: [PATCH 0039/2492] osd/OSD: add log line on mapgaps

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1fd6b8eec3a8..d7b60e79cf82 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -8199,6 +8199,9 @@ void OSD::handle_osd_map(MOSDMap *m)
     pg_num_history.prune(superblock.get_oldest_map());
   }
   superblock.insert_osdmap_epochs(first, last);
+  if (superblock.maps.num_intervals() > 1) {
+    dout(10) << __func__ << " osd map gap " << superblock.maps << dendl;
+  }
   superblock.current_epoch = last;
 
   // note in the superblock that we were clean thru the prior epoch

From e7779fefc69d33cc129c792862667aa7b43a4dba Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Fri, 8 Sep 2023 17:48:39 -0400
Subject: [PATCH 0040/2492] cmake: BuildFIO.cmake should not introduce
 -std=gnu++17

Not correct in general, and a build bug because fio-objectstore
includes c++20 headers.

Fixes: https://tracker.ceph.com/issues/62778

Add CXX_EXTENSIONS ON, and cleanup INTERFACE_COMPILE_OPTIONS per Kefu review

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 cmake/modules/BuildFIO.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/modules/BuildFIO.cmake b/cmake/modules/BuildFIO.cmake
index 3a0694b543ee..49fcfb31d973 100644
--- a/cmake/modules/BuildFIO.cmake
+++ b/cmake/modules/BuildFIO.cmake
@@ -37,6 +37,7 @@ function(build_fio)
   add_library(fio INTERFACE IMPORTED)
   add_dependencies(fio fio_ext)
   set_target_properties(fio PROPERTIES
+    CXX_EXTENSIONS ON
     INTERFACE_INCLUDE_DIRECTORIES ${source_dir}
-    INTERFACE_COMPILE_OPTIONS "-include;${source_dir}/config-host.h;$<$<COMPILE_LANGUAGE:C>:-std=gnu99>$<$<COMPILE_LANGUAGE:CXX>:-std=gnu++17>")
+    INTERFACE_COMPILE_OPTIONS "-include;${source_dir}/config-host.h;$<$<COMPILE_LANGUAGE:C>:-std=gnu99>")
 endfunction()

From e80d11b3903c52d1a35112b822b7680b6d018ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 8 Sep 2023 13:50:01 +0200
Subject: [PATCH 0041/2492] ceph-run: fix string comparison around --no-restart
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`man test` says the stirng equality operator is `=`:

```
       STRING1 = STRING2
              the strings are equal
```

Also, we should handle the argument-less case which currently
fails:

```
$ [ $1 == "--no-restart" ]
bash: [: ==: unary operator expected
```

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/ceph-run | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ceph-run b/src/ceph-run
index 764101c564f8..c28244d90f83 100755
--- a/src/ceph-run
+++ b/src/ceph-run
@@ -3,7 +3,7 @@
 sleep=5
 no_restart=0
 
-if [ $1 == "--no-restart" ]; then
+if [ "$1" = "--no-restart" ]; then
     no_restart=1
     shift
 fi

From 622ef1af43a2bd635921f802f63147a9651d4961 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 12 Sep 2023 14:44:03 +0800
Subject: [PATCH 0042/2492] crimson/osd/pg_recovery: recover head before
 recovering clones

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg_recovery.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/pg_recovery.cc b/src/crimson/osd/pg_recovery.cc
index 09b45779ec87..18c00b68831f 100644
--- a/src/crimson/osd/pg_recovery.cc
+++ b/src/crimson/osd/pg_recovery.cc
@@ -140,13 +140,14 @@ size_t PGRecovery::start_primary_recovery_ops(
 
     hobject_t head = soid.get_head();
 
+    bool head_missing = missing.is_missing(head);
     logger().info(
       "{} {} item.need {} {} {} {} {}",
       __func__,
       soid,
       item.need,
       missing.is_missing(soid) ? " (missing)":"",
-      missing.is_missing(head) ? " (missing head)":"",
+      head_missing ? " (missing head)":"",
       pg->get_recovery_backend()->is_recovering(soid) ? " (recovering)":"",
       pg->get_recovery_backend()->is_recovering(head) ? " (recovering head)":"");
 
@@ -158,7 +159,15 @@ size_t PGRecovery::start_primary_recovery_ops(
     } else if (pg->get_recovery_backend()->is_recovering(head)) {
       ++skipped;
     } else {
-      out->emplace_back(recover_missing(trigger, soid, item.need));
+      if (head_missing) {
+	auto it = missing.get_items().find(head);
+	assert(it != missing.get_items().end());
+	auto head_need = it->second.need;
+	out->emplace_back(recover_missing(trigger, head, head_need));
+	++skipped;
+      } else {
+	out->emplace_back(recover_missing(trigger, soid, item.need));
+      }
       ++started;
     }
 

From 1d7cabf3db8b8d1863aab9c403e2935ed01a5e5d Mon Sep 17 00:00:00 2001
From: NitzanMordhai <nmordech@redhat.com>
Date: Tue, 27 Jun 2023 06:31:22 +0000
Subject: [PATCH 0043/2492] ceph-dencoder: Add missing common types to
 ceph-dencoder for accurate encode-decode comparison Currently, ceph-dencoder
 lacks certain common types, preventing us from accurately checking the ceph
 corpus for encode-decode mismatches. This pull request aims to address this
 issue by adding the missing types to ceph-dencoder.

To successfully incorporate these types into ceph-dencoder, we need to introduce the necessary dump and generate_test_instances functions that was missing in some types.
These functions are essential for proper encode and decode of the added types.

This PR will enhance the functionality of ceph-dencoder by including the missing types, enabling a comprehensive analysis of encode-decode consistency.
With the addition of these types, we can ensure the robustness and correctness of the ceph corpus.

This update will significantly contribute to improving the overall reliability and accuracy of ceph-dencoder.
It allows for a more comprehensive assessment of the encode-decode behavior,
leading to enhanced data integrity and stability within the ceph ecosystem.

Fixes: https://tracker.ceph.com/issues/61788
Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 src/auth/Auth.h                         |  60 +++++++++
 src/auth/Crypto.cc                      |  17 +++
 src/auth/Crypto.h                       |   2 +
 src/auth/cephx/CephxKeyServer.cc        |  10 ++
 src/auth/cephx/CephxKeyServer.h         |  40 +++++-
 src/auth/cephx/CephxProtocol.h          | 124 +++++++++++++++++
 src/cls/2pc_queue/cls_2pc_queue_ops.h   |  70 +++++++++-
 src/cls/2pc_queue/cls_2pc_queue_types.h |  36 +++++
 src/cls/cas/cls_cas_internal.h          |  11 ++
 src/cls/fifo/cls_fifo_ops.h             |  48 +++++++
 src/cls/fifo/cls_fifo_types.h           |  62 ++++++++-
 src/cls/log/cls_log_ops.h               |  56 ++++++++
 src/cls/log/cls_log_types.h             |  10 ++
 src/cls/queue/cls_queue_ops.h           |  74 ++++++++++-
 src/cls/queue/cls_queue_types.h         |  43 +++++-
 src/cls/timeindex/cls_timeindex_ops.h   |  38 ++++++
 src/cls/timeindex/cls_timeindex_types.h |   2 +-
 src/cls/version/cls_version_ops.h       |  55 ++++++++
 src/cls/version/cls_version_types.cc    |   6 -
 src/cls/version/cls_version_types.h     |  18 ++-
 src/common/ceph_json.h                  |  55 ++++++++
 src/common/entity_name.cc               |  63 ++++-----
 src/common/entity_name.h                |   3 +-
 src/include/cephfs/types.h              |   2 +
 src/include/frag.h                      |  14 ++
 src/include/fs_types.h                  |   7 +
 src/include/object.h                    |  18 +++
 src/include/types.h                     |  33 ++++-
 src/mds/mdstypes.cc                     |  15 +++
 src/messages/MClientReply.h             |  12 +-
 src/messages/MClientRequest.h           |  31 +++++
 src/messages/MMDSCacheRejoin.h          |  32 +++++
 src/messages/MMgrReport.h               |  21 +++
 src/os/bluestore/bluestore_types.cc     |   9 ++
 src/os/bluestore/bluestore_types.h      |   1 +
 src/tools/ceph-dencoder/common_types.h  | 170 +++++++++++++++++++++++-
 36 files changed, 1205 insertions(+), 63 deletions(-)

diff --git a/src/auth/Auth.h b/src/auth/Auth.h
index 5521c8d3fcf0..83e23b34dbe3 100644
--- a/src/auth/Auth.h
+++ b/src/auth/Auth.h
@@ -16,6 +16,7 @@
 #define CEPH_AUTHTYPES_H
 
 #include "Crypto.h"
+#include "common/ceph_json.h"
 #include "common/entity_name.h"
 
 // The _MAX values are a bit wonky here because we are overloading the first
@@ -59,6 +60,14 @@ struct EntityAuth {
       decode(pending_key, bl);
     }
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("key", key);
+    encode_json("caps", caps, f);
+    f->dump_object("pending_key", pending_key);
+  }
+  static void generate_test_instances(std::list<EntityAuth*>& ls) {
+    ls.push_back(new EntityAuth);
+  }
 };
 WRITE_CLASS_ENCODER(EntityAuth)
 
@@ -95,6 +104,19 @@ struct AuthCapsInfo {
     allow_all = (bool)a;
     decode(caps, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_bool("allow_all", allow_all);
+    encode_json("caps", caps, f);
+    f->dump_unsigned("caps_len", caps.length());
+  }
+  static void generate_test_instances(std::list<AuthCapsInfo*>& ls) {
+    ls.push_back(new AuthCapsInfo);
+    ls.push_back(new AuthCapsInfo);
+    ls.back()->allow_all = true;
+    ls.push_back(new AuthCapsInfo);
+    ls.back()->caps.append("foo");
+    ls.back()->caps.append("bar");
+  }
 };
 WRITE_CLASS_ENCODER(AuthCapsInfo)
 
@@ -147,6 +169,25 @@ struct AuthTicket {
     decode(caps, bl);
     decode(flags, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("name", name);
+    f->dump_unsigned("global_id", global_id);
+    f->dump_stream("created") << created;
+    f->dump_stream("renew_after") << renew_after;
+    f->dump_stream("expires") << expires;
+    f->dump_object("caps", caps);
+    f->dump_unsigned("flags", flags);
+  }
+  static void generate_test_instances(std::list<AuthTicket*>& ls) {
+    ls.push_back(new AuthTicket);
+    ls.push_back(new AuthTicket);
+    ls.back()->name.set_id("client.123");
+    ls.back()->global_id = 123;
+    ls.back()->init_timestamps(utime_t(123, 456), 7);
+    ls.back()->caps.caps.append("foo");
+    ls.back()->caps.caps.append("bar");
+    ls.back()->flags = 0x12345678;
+  }
 };
 WRITE_CLASS_ENCODER(AuthTicket)
 
@@ -231,6 +272,16 @@ struct ExpiringCryptoKey {
     decode(key, bl);
     decode(expiration, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("key", key);
+    f->dump_stream("expiration") << expiration;
+  }
+  static void generate_test_instances(std::list<ExpiringCryptoKey*>& ls) {
+    ls.push_back(new ExpiringCryptoKey);
+    ls.push_back(new ExpiringCryptoKey);
+    ls.back()->key.set_secret(
+      CEPH_CRYPTO_AES, bufferptr("1234567890123456", 16), utime_t(123, 456));
+  }
 };
 WRITE_CLASS_ENCODER(ExpiringCryptoKey)
 
@@ -295,6 +346,15 @@ struct RotatingSecrets {
   }
 
   void dump();
+  void dump(ceph::Formatter *f) const {
+    encode_json("secrets", secrets, f);
+  }
+  static void generate_test_instances(std::list<RotatingSecrets*>& ls) {
+    ls.push_back(new RotatingSecrets);
+    ls.push_back(new RotatingSecrets);
+    auto eck = new ExpiringCryptoKey;
+    ls.back()->add(*eck);
+  }
 };
 WRITE_CLASS_ENCODER(RotatingSecrets)
 
diff --git a/src/auth/Crypto.cc b/src/auth/Crypto.cc
index ce666e8bdc8c..5d68d3470bc0 100644
--- a/src/auth/Crypto.cc
+++ b/src/auth/Crypto.cc
@@ -511,6 +511,23 @@ void CryptoKey::decode(bufferlist::const_iterator& bl)
     throw ceph::buffer::malformed_input("malformed secret");
 }
 
+void CryptoKey::dump(Formatter *f) const
+{
+  f->dump_int("type", type);
+  f->dump_stream("created") << created;
+  f->dump_int("secret.length", secret.length());
+}
+
+void CryptoKey::generate_test_instances(std::list<CryptoKey*>& ls)
+{
+  ls.push_back(new CryptoKey);
+  ls.push_back(new CryptoKey);
+  ls.back()->type = CEPH_CRYPTO_AES;
+  ls.back()->set_secret(
+    CEPH_CRYPTO_AES, bufferptr("1234567890123456", 16), utime_t(123, 456));
+  ls.back()->created = utime_t(123, 456);
+}
+
 int CryptoKey::set_secret(int type, const bufferptr& s, utime_t c)
 {
   int r = _set_secret(type, s);
diff --git a/src/auth/Crypto.h b/src/auth/Crypto.h
index a29ac1abd811..3ce655a12562 100644
--- a/src/auth/Crypto.h
+++ b/src/auth/Crypto.h
@@ -111,6 +111,8 @@ class CryptoKey {
 
   void encode(ceph::buffer::list& bl) const;
   void decode(ceph::buffer::list::const_iterator& bl);
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<CryptoKey*>& ls);
 
   void clear() {
     *this = CryptoKey();
diff --git a/src/auth/cephx/CephxKeyServer.cc b/src/auth/cephx/CephxKeyServer.cc
index 236ac451add9..a5ad1f2b7973 100644
--- a/src/auth/cephx/CephxKeyServer.cc
+++ b/src/auth/cephx/CephxKeyServer.cc
@@ -257,6 +257,16 @@ std::map<EntityName,CryptoKey> KeyServer::get_used_pending_keys()
   return ret;
 }
 
+void KeyServer::dump(Formatter *f) const
+{
+  f->dump_object("data", data);
+}
+
+void KeyServer::generate_test_instances(std::list<KeyServer*>& ls)
+{
+  ls.push_back(new KeyServer(nullptr, nullptr));
+}
+
 bool KeyServer::generate_secret(CryptoKey& secret)
 {
   bufferptr bp;
diff --git a/src/auth/cephx/CephxKeyServer.h b/src/auth/cephx/CephxKeyServer.h
index 64915c8ce4aa..d147dd441ad2 100644
--- a/src/auth/cephx/CephxKeyServer.h
+++ b/src/auth/cephx/CephxKeyServer.h
@@ -21,15 +21,16 @@
 #include "include/common_fwd.h"
 
 struct KeyServerData {
-  version_t version;
+  version_t version{0};
 
   /* for each entity */
   std::map<EntityName, EntityAuth> secrets;
-  KeyRing *extra_secrets;
+  KeyRing *extra_secrets = nullptr;
 
   /* for each service type */
-  version_t rotating_ver;
+  version_t rotating_ver{0};
   std::map<uint32_t, RotatingSecrets> rotating_secrets;
+  KeyServerData() {}
 
   explicit KeyServerData(KeyRing *extra)
     : version(0),
@@ -70,7 +71,17 @@ struct KeyServerData {
     decode(rotating_ver, iter);
     decode(rotating_secrets, iter);
   }
-
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("version", version);
+    f->dump_unsigned("rotating_version", rotating_ver);
+    encode_json("secrets", secrets, f);
+    encode_json("rotating_secrets", rotating_secrets, f);
+  }
+  static void generate_test_instances(std::list<KeyServerData*>& ls) {
+    ls.push_back(new KeyServerData);
+    ls.push_back(new KeyServerData);
+    ls.back()->version = 1;
+  }
   bool contains(const EntityName& name) const {
     return (secrets.find(name) != secrets.end());
   }
@@ -159,8 +170,21 @@ struct KeyServerData {
 	decode(auth, bl);
       }
     }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("op", op);
+      f->dump_object("name", name);
+      f->dump_object("auth", auth);
+    }
+    static void generate_test_instances(std::list<Incremental*>& ls) {
+      ls.push_back(new Incremental);
+      ls.back()->op = AUTH_INC_DEL;
+      ls.push_back(new Incremental);
+      ls.back()->op = AUTH_INC_ADD;
+      ls.push_back(new Incremental);
+      ls.back()->op = AUTH_INC_SET_ROTATING;
+    }
   };
-
+  
   void apply_incremental(Incremental& inc) {
     switch (inc.op) {
     case AUTH_INC_ADD:
@@ -188,8 +212,6 @@ WRITE_CLASS_ENCODER(KeyServerData)
 WRITE_CLASS_ENCODER(KeyServerData::Incremental)
 
 
-
-
 class KeyServer : public KeyStore {
   CephContext *cct;
   KeyServerData data;
@@ -205,7 +227,9 @@ class KeyServer : public KeyStore {
   bool _get_service_caps(const EntityName& name, uint32_t service_id,
 	AuthCapsInfo& caps) const;
 public:
+  KeyServer() : lock{ceph::make_mutex("KeyServer::lock")} {}
   KeyServer(CephContext *cct_, KeyRing *extra_secrets);
+  KeyServer& operator=(const KeyServer&) = delete;
   bool generate_secret(CryptoKey& secret);
 
   bool get_secret(const EntityName& name, CryptoKey& secret) const override;
@@ -248,6 +272,8 @@ class KeyServer : public KeyStore {
     using ceph::decode;
     decode(data, bl);
   }
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<KeyServer*>& ls);
   bool contains(const EntityName& name) const;
   int encode_secrets(ceph::Formatter *f, std::stringstream *ds) const;
   void encode_formatted(std::string label, ceph::Formatter *f, ceph::buffer::list &bl);
diff --git a/src/auth/cephx/CephxProtocol.h b/src/auth/cephx/CephxProtocol.h
index aabfaaad10c9..260cb13ff5ab 100644
--- a/src/auth/cephx/CephxProtocol.h
+++ b/src/auth/cephx/CephxProtocol.h
@@ -55,6 +55,13 @@ struct CephXServerChallenge {
     decode(struct_v, bl);
     decode(server_challenge, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("server_challenge", server_challenge);
+  }
+  static void generate_test_instances(std::list<CephXServerChallenge*>& ls) {
+    ls.push_back(new CephXServerChallenge);
+    ls.back()->server_challenge = 1;
+  }
 };
 WRITE_CLASS_ENCODER(CephXServerChallenge)
 
@@ -72,6 +79,13 @@ struct CephXRequestHeader {
     using ceph::decode;
     decode(request_type, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("request_type", request_type);
+  }
+  static void generate_test_instances(std::list<CephXRequestHeader*>& ls) {
+    ls.push_back(new CephXRequestHeader);
+    ls.back()->request_type = 1;
+  }
 };
 WRITE_CLASS_ENCODER(CephXRequestHeader)
 
@@ -89,6 +103,15 @@ struct CephXResponseHeader {
     decode(request_type, bl);
     decode(status, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("request_type", request_type);
+    f->dump_int("status", status);
+  }
+  static void generate_test_instances(std::list<CephXResponseHeader*>& ls) {
+    ls.push_back(new CephXResponseHeader);
+    ls.back()->request_type = 1;
+    ls.back()->status = 0;
+  }
 };
 WRITE_CLASS_ENCODER(CephXResponseHeader)
 
@@ -113,6 +136,17 @@ struct CephXTicketBlob {
      decode(secret_id, bl);
      decode(blob, bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("secret_id", secret_id);
+    f->dump_unsigned("blob_len", blob.length());
+  }
+
+  static void generate_test_instances(std::list<CephXTicketBlob*>& ls) {
+    ls.push_back(new CephXTicketBlob);
+    ls.back()->secret_id = 123;
+    ls.back()->blob.append(std::string_view("this is a blob"));
+  }
 };
 WRITE_CLASS_ENCODER(CephXTicketBlob)
 
@@ -152,6 +186,25 @@ struct CephXAuthenticate {
     //   old_ticket both on reconnects and renewals
     old_ticket_may_be_omitted = struct_v < 3;
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("client_challenge", client_challenge);
+    f->dump_unsigned("key", key);
+    f->open_object_section("old_ticket");
+    old_ticket.dump(f);
+    f->close_section();
+    f->dump_unsigned("other_keys", other_keys);
+  }
+  static void generate_test_instances(std::list<CephXAuthenticate*>& ls) {
+    ls.push_back(new CephXAuthenticate);
+    ls.back()->client_challenge = 0;
+    ls.back()->key = 0;
+    ls.push_back(new CephXAuthenticate);
+    ls.back()->client_challenge = 1;
+    ls.back()->key = 2;
+    ls.back()->old_ticket.secret_id = 3;
+    ls.back()->old_ticket.blob.append(std::string_view("this is a blob"));
+    ls.back()->other_keys = 4;
+  }
 };
 WRITE_CLASS_ENCODER(CephXAuthenticate)
 
@@ -168,6 +221,15 @@ struct CephXChallengeBlob {
     decode(server_challenge, bl);
     decode(client_challenge, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("server_challenge", server_challenge);
+    f->dump_unsigned("client_challenge", client_challenge);
+  }
+  static void generate_test_instances(std::list<CephXChallengeBlob*>& ls) {
+    ls.push_back(new CephXChallengeBlob);
+    ls.back()->server_challenge = 123;
+    ls.back()->client_challenge = 456;
+  }
 };
 WRITE_CLASS_ENCODER(CephXChallengeBlob)
 
@@ -218,6 +280,15 @@ struct CephXServiceTicketRequest {
     decode(struct_v, bl);
     decode(keys, bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("keys", keys);
+  }
+
+  static void generate_test_instances(std::list<CephXServiceTicketRequest*>& ls) {
+    ls.push_back(new CephXServiceTicketRequest);
+    ls.back()->keys = 123;
+  }
 };
 WRITE_CLASS_ENCODER(CephXServiceTicketRequest)
 
@@ -251,6 +322,17 @@ struct CephXAuthorizeReply {
       decode(connection_secret, bl);
     }
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("nonce_plus_one", nonce_plus_one);
+    f->dump_string("connection_secret", connection_secret);
+  }
+  static void generate_test_instances(std::list<CephXAuthorizeReply*>& ls) {
+    ls.push_back(new CephXAuthorizeReply);
+    ls.back()->nonce_plus_one = 0;
+    ls.push_back(new CephXAuthorizeReply);
+    ls.back()->nonce_plus_one = 123;
+    ls.back()->connection_secret = "secret";
+  }
 };
 WRITE_CLASS_ENCODER(CephXAuthorizeReply)
 
@@ -353,6 +435,17 @@ struct CephXServiceTicket {
     decode(session_key, bl);
     decode(validity, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    session_key.dump(f);
+    validity.dump(f);
+  }
+  static void generate_test_instances(std::list<CephXServiceTicket*>& ls) {
+    ls.push_back(new CephXServiceTicket);
+    ls.push_back(new CephXServiceTicket);
+    ls.back()->session_key.set_secret(
+      CEPH_CRYPTO_AES, bufferptr("1234567890123456", 16), utime_t(123, 456));
+    ls.back()->validity = utime_t(123, 456);
+  }
 };
 WRITE_CLASS_ENCODER(CephXServiceTicket)
 
@@ -375,6 +468,18 @@ struct CephXServiceTicketInfo {
     decode(ticket, bl);
     decode(session_key, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    ticket.dump(f);
+    session_key.dump(f);
+  }
+  static void generate_test_instances(std::list<CephXServiceTicketInfo*>& ls) {
+    ls.push_back(new CephXServiceTicketInfo);
+    ls.push_back(new CephXServiceTicketInfo);
+    ls.back()->ticket.global_id = 1234;
+    ls.back()->ticket.init_timestamps(utime_t(123, 456), utime_t(123, 456));
+    ls.back()->session_key.set_secret(
+      CEPH_CRYPTO_AES, bufferptr("1234567890123456", 16), utime_t(123, 456));
+  }
 };
 WRITE_CLASS_ENCODER(CephXServiceTicketInfo)
 
@@ -392,6 +497,13 @@ struct CephXAuthorizeChallenge : public AuthAuthorizerChallenge {
     decode(struct_v, bl);
     decode(server_challenge, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("server_challenge", server_challenge);
+  }
+  static void generate_test_instances(std::list<CephXAuthorizeChallenge*>& ls) {
+    ls.push_back(new CephXAuthorizeChallenge);
+    ls.back()->server_challenge = 1234;
+  }
 };
 WRITE_CLASS_ENCODER(CephXAuthorizeChallenge)
 
@@ -417,6 +529,18 @@ struct CephXAuthorize {
       decode(server_challenge_plus_one, bl);
     }
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("nonce", nonce);
+    f->dump_unsigned("have_challenge", have_challenge);
+    f->dump_unsigned("server_challenge_plus_one", server_challenge_plus_one);
+  }
+  static void generate_test_instances(std::list<CephXAuthorize*>& ls) {
+    ls.push_back(new CephXAuthorize);
+    ls.push_back(new CephXAuthorize);
+    ls.back()->nonce = 1234;
+    ls.back()->have_challenge = true;
+    ls.back()->server_challenge_plus_one = 1234;
+  }
 };
 WRITE_CLASS_ENCODER(CephXAuthorize)
 
diff --git a/src/cls/2pc_queue/cls_2pc_queue_ops.h b/src/cls/2pc_queue/cls_2pc_queue_ops.h
index bb61ef341ac1..3cdfd9663e8d 100644
--- a/src/cls/2pc_queue/cls_2pc_queue_ops.h
+++ b/src/cls/2pc_queue/cls_2pc_queue_ops.h
@@ -3,12 +3,13 @@
 
 #pragma once
 
+#include "common/ceph_json.h"
 #include "include/types.h"
 #include "cls_2pc_queue_types.h"
 
 struct cls_2pc_queue_reserve_op {
   uint64_t size;
-  uint32_t entries;
+  uint32_t entries{0};
 
   void encode(ceph::buffer::list& bl) const {
     ENCODE_START(1, 1, bl);
@@ -23,6 +24,19 @@ struct cls_2pc_queue_reserve_op {
     decode(entries, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("size", size);
+    f->dump_unsigned("entries", entries);
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_queue_reserve_op*>& ls) {
+    ls.push_back(new cls_2pc_queue_reserve_op);
+    ls.back()->size = 0;
+    ls.push_back(new cls_2pc_queue_reserve_op);
+    ls.back()->size = 123;
+    ls.back()->entries = 456;
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_reserve_op)
 
@@ -40,6 +54,15 @@ struct cls_2pc_queue_reserve_ret {
     decode(id, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("id", id);
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_queue_reserve_ret*>& ls) {
+    ls.push_back(new cls_2pc_queue_reserve_ret);
+    ls.back()->id = 123;
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_reserve_ret)
 
@@ -61,6 +84,19 @@ struct cls_2pc_queue_commit_op {
     DECODE_FINISH(bl);
   }
 
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("id", id);
+    encode_json("bl_data_vec", bl_data_vec, f);
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_queue_commit_op*>& ls) {
+    ls.push_back(new cls_2pc_queue_commit_op);
+    ls.back()->id = 123;
+    ls.back()->bl_data_vec.push_back(ceph::buffer::list());
+    ls.back()->bl_data_vec.back().append("foo");
+    ls.back()->bl_data_vec.push_back(ceph::buffer::list());
+    ls.back()->bl_data_vec.back().append("bar");
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_commit_op)
 
@@ -78,6 +114,13 @@ struct cls_2pc_queue_abort_op {
     decode(id, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("id", id);
+  }
+  static void generate_test_instances(std::list<cls_2pc_queue_abort_op*>& ls) {
+    ls.push_back(new cls_2pc_queue_abort_op);
+    ls.back()->id = 1;
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_abort_op)
 
@@ -96,6 +139,14 @@ struct cls_2pc_queue_expire_op {
     decode(stale_time, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("stale_time") << stale_time;
+  }
+  static void generate_test_instances(std::list<cls_2pc_queue_expire_op*>& ls) {
+    ls.push_back(new cls_2pc_queue_expire_op);
+    ls.push_back(new cls_2pc_queue_expire_op);
+    ls.back()->stale_time = ceph::coarse_real_time::min();
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_expire_op)
 
@@ -113,6 +164,23 @@ struct cls_2pc_queue_reservations_ret {
     decode(reservations, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->open_array_section("reservations");
+    for (const auto& i : reservations) {
+      f->open_object_section("reservation");
+      f->dump_unsigned("id", i.first);
+      i.second.dump(f);
+      f->close_section();
+    }
+    f->close_section();
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_queue_reservations_ret*>& ls) {
+    ls.push_back(new cls_2pc_queue_reservations_ret);
+    ls.push_back(new cls_2pc_queue_reservations_ret);
+    ls.back()->reservations[1] = cls_2pc_reservation();
+    ls.back()->reservations[2] = cls_2pc_reservation();
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_queue_reservations_ret)
 
diff --git a/src/cls/2pc_queue/cls_2pc_queue_types.h b/src/cls/2pc_queue/cls_2pc_queue_types.h
index 2413fd7043da..b270c9d6e790 100644
--- a/src/cls/2pc_queue/cls_2pc_queue_types.h
+++ b/src/cls/2pc_queue/cls_2pc_queue_types.h
@@ -34,6 +34,19 @@ struct cls_2pc_reservation
     }
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("size", size);
+    f->dump_stream("timestamp") << timestamp;
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_reservation*>& ls) {
+    ls.push_back(new cls_2pc_reservation);
+    ls.back()->size = 0;
+    ls.push_back(new cls_2pc_reservation);
+    ls.back()->size = 123;
+    ls.back()->timestamp = ceph::coarse_real_clock::zero();
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_reservation)
 
@@ -68,5 +81,28 @@ struct cls_2pc_urgent_data
     }
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("reserved_size", reserved_size);
+    f->dump_unsigned("last_id", last_id);
+    f->open_array_section("reservations");
+    for (const auto& [id, res] : reservations) {
+      f->open_object_section("reservation");
+      f->dump_unsigned("id", id);
+      res.dump(f);
+      f->close_section();
+    }
+    f->close_section();
+    f->dump_bool("has_xattrs", has_xattrs);
+  }
+
+  static void generate_test_instances(std::list<cls_2pc_urgent_data*>& ls) {
+    ls.push_back(new cls_2pc_urgent_data);
+    ls.push_back(new cls_2pc_urgent_data);
+    ls.back()->reserved_size = 123;
+    ls.back()->last_id = 456;
+    ls.back()->reservations.emplace(789, cls_2pc_reservation(1, ceph::coarse_real_clock::zero(), 2));
+    ls.back()->has_xattrs = true;
+  }
 };
 WRITE_CLASS_ENCODER(cls_2pc_urgent_data)
diff --git a/src/cls/cas/cls_cas_internal.h b/src/cls/cas/cls_cas_internal.h
index 09e7f9f1f69d..037e60fffd9d 100644
--- a/src/cls/cas/cls_cas_internal.h
+++ b/src/cls/cas/cls_cas_internal.h
@@ -145,6 +145,12 @@ struct chunk_refs_by_object_t : public chunk_refs_t::refs_t {
     }
     f->close_section();
   }
+  static void generate_test_instances(std::list<chunk_refs_by_object_t*>& ls) {
+    ls.push_back(new chunk_refs_by_object_t());
+    ls.push_back(new chunk_refs_by_object_t());
+    ls.back()->by_object.insert(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
+    ls.back()->by_object.insert(hobject_t(sobject_t("bar", CEPH_NOSNAP)));
+  }
 };
 WRITE_CLASS_ENCODER(chunk_refs_by_object_t)
 
@@ -386,6 +392,11 @@ struct chunk_refs_count_t : public chunk_refs_t::refs_t {
     f->dump_string("type", "count");
     f->dump_unsigned("count", total);
   }
+  static void generate_test_instances(std::list<chunk_refs_count_t*>& o) {
+    o.push_back(new chunk_refs_count_t);
+    o.push_back(new chunk_refs_count_t);
+    o.back()->total = 123;
+  }
 };
 WRITE_CLASS_ENCODER(chunk_refs_count_t)
 
diff --git a/src/cls/fifo/cls_fifo_ops.h b/src/cls/fifo/cls_fifo_ops.h
index e850c635c0b8..d466122a9527 100644
--- a/src/cls/fifo/cls_fifo_ops.h
+++ b/src/cls/fifo/cls_fifo_ops.h
@@ -67,6 +67,31 @@ struct create_meta
     decode(exclusive, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("id", id);
+    f->dump_object("version", version.value_or(objv()));
+    f->dump_string("pool_name", pool.name);
+    f->dump_string("pool_ns", pool.ns);
+    f->dump_string("oid_prefix", oid_prefix.value_or(""));
+    f->dump_unsigned("max_part_size", max_part_size);
+    f->dump_unsigned("max_entry_size", max_entry_size);
+    f->dump_bool("exclusive", exclusive);
+  }
+  static void generate_test_instances(std::list<create_meta*>& o) {
+    o.push_back(new create_meta);
+    o.push_back(new create_meta);
+    o.back()->id = "id";
+    objv v1;
+    v1.instance = "inst1";
+    v1.ver = 1;
+    o.back()->version = v1;
+    o.back()->pool.name = "pool";
+    o.back()->pool.ns = "ns";
+    o.back()->oid_prefix = "prefix";
+    o.back()->max_part_size = 1024;
+    o.back()->max_entry_size = 1024;
+    o.back()->exclusive = true;
+  }
 };
 WRITE_CLASS_ENCODER(create_meta)
 
@@ -84,6 +109,17 @@ struct get_meta
     decode(version, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("version", version.value_or(objv()));
+  }
+  static void generate_test_instances(std::list<get_meta*>& o) {
+    o.push_back(new get_meta);
+    o.push_back(new get_meta);
+    objv v1;
+    v1.instance = "inst1";
+    v1.ver = 1;
+    o.back()->version = v1;
+  }
 };
 WRITE_CLASS_ENCODER(get_meta)
 
@@ -108,6 +144,18 @@ struct get_meta_reply
     decode(part_entry_overhead, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("info", info);
+    f->dump_unsigned("part_header_size", part_header_size);
+    f->dump_unsigned("part_entry_overhead", part_entry_overhead);
+  }
+  static void generate_test_instances(std::list<get_meta_reply*>& o) {
+    o.push_back(new get_meta_reply);
+    o.push_back(new get_meta_reply);
+    o.back()->info = fifo::info();
+    o.back()->part_header_size = 1024;
+    o.back()->part_entry_overhead = 1024;
+  }
 };
 WRITE_CLASS_ENCODER(get_meta_reply)
 
diff --git a/src/cls/fifo/cls_fifo_types.h b/src/cls/fifo/cls_fifo_types.h
index 1c69c1f08718..2ae601e4aafa 100644
--- a/src/cls/fifo/cls_fifo_types.h
+++ b/src/cls/fifo/cls_fifo_types.h
@@ -54,7 +54,16 @@ struct objv {
     decode(ver, bl);
     DECODE_FINISH(bl);
   }
-  void dump(ceph::Formatter* f) const;
+  void dump(ceph::Formatter* f) const {
+    f->dump_string("instance", instance);
+    f->dump_unsigned("ver", ver);
+  }
+  static void generate_test_instances(std::list<objv*>& o) {
+    o.push_back(new objv);
+    o.push_back(new objv);
+    o.back()->instance = "instance";
+    o.back()->ver = 1;
+  }
   void decode_json(JSONObj* obj);
 
   bool operator ==(const objv& rhs) const {
@@ -103,7 +112,18 @@ struct data_params {
     decode(full_size_threshold, bl);
     DECODE_FINISH(bl);
   }
-  void dump(ceph::Formatter* f) const;
+  void dump(ceph::Formatter* f) const {
+    f->dump_unsigned("max_part_size", max_part_size);
+    f->dump_unsigned("max_entry_size", max_entry_size);
+    f->dump_unsigned("full_size_threshold", full_size_threshold);
+  }
+  static void generate_test_instances(std::list<data_params*>& o) {
+    o.push_back(new data_params);
+    o.push_back(new data_params);
+    o.back()->max_part_size = 1;
+    o.back()->max_entry_size = 2;
+    o.back()->full_size_threshold = 3;
+  }
   void decode_json(JSONObj* obj);
 
   auto operator <=>(const data_params&) const = default;
@@ -161,7 +181,10 @@ struct journal_entry {
     decode(part_tag, bl);
     DECODE_FINISH(bl);
   }
-  void dump(ceph::Formatter* f) const;
+  void dump(ceph::Formatter* f) const {
+    f->dump_int("op", (int)op);
+    f->dump_int("part_num", part_num);
+  }
 
   auto operator <=>(const journal_entry&) const = default;
 };
@@ -397,7 +420,38 @@ struct info {
     decode_journal(bl);
     DECODE_FINISH(bl);
   }
-  void dump(ceph::Formatter* f) const;
+  void dump(ceph::Formatter* f) const {
+    f->dump_string("id", id);
+    f->dump_object("version", version);
+    f->dump_string("oid_prefix", oid_prefix);
+    f->dump_object("params", params);
+    f->dump_int("tail_part_num", tail_part_num);
+    f->dump_int("head_part_num", head_part_num);
+    f->dump_int("min_push_part_num", min_push_part_num);
+    f->dump_int("max_push_part_num", max_push_part_num);
+    f->open_array_section("journal");
+    for (const auto& entry : journal) {
+      f->open_object_section("entry");
+      f->dump_object("entry", entry);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  static void generate_test_instances(std::list<info*>& o) {
+    o.push_back(new info);
+    o.push_back(new info);
+    o.back()->id = "myid";
+    o.back()->version = objv();
+    o.back()->oid_prefix = "myprefix";
+    o.back()->params = data_params();
+    o.back()->tail_part_num = 123;
+    o.back()->head_part_num = 456;
+    o.back()->min_push_part_num = 789;
+    o.back()->max_push_part_num = 101112;
+    o.back()->journal.insert(journal_entry(journal_entry::Op::create, 1));
+    o.back()->journal.insert(journal_entry(journal_entry::Op::create, 2));
+    o.back()->journal.insert(journal_entry(journal_entry::Op::create, 3));
+  }
   void decode_json(JSONObj* obj);
 
   std::string part_oid(std::int64_t part_num) const {
diff --git a/src/cls/log/cls_log_ops.h b/src/cls/log/cls_log_ops.h
index 5a65892598b6..4d3b2f5d3091 100644
--- a/src/cls/log/cls_log_ops.h
+++ b/src/cls/log/cls_log_ops.h
@@ -4,6 +4,7 @@
 #ifndef CEPH_CLS_LOG_OPS_H
 #define CEPH_CLS_LOG_OPS_H
 
+#include "common/ceph_json.h"
 #include "cls_log_types.h"
 
 struct cls_log_add_op {
@@ -73,6 +74,21 @@ struct cls_log_list_op {
     decode(max_entries, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter* f) const {
+    f->dump_stream("from_time") << from_time;
+    f->dump_string("marker", marker);
+    f->dump_stream("to_time") << to_time;
+    f->dump_int("max_entries", max_entries);
+  }
+  static void generate_test_instances(std::list<cls_log_list_op*>& ls) {
+    ls.push_back(new cls_log_list_op);
+    ls.push_back(new cls_log_list_op);
+    ls.back()->from_time = utime_t(1, 2);
+    ls.back()->marker = "marker";
+    ls.back()->to_time = utime_t(3, 4);
+    ls.back()->max_entries = 5;
+  }
 };
 WRITE_CLASS_ENCODER(cls_log_list_op)
 
@@ -98,6 +114,25 @@ struct cls_log_list_ret {
     decode(truncated, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter* f) const {
+    encode_json("entries", entries, f);
+    f->dump_string("marker", marker);
+    f->dump_bool("truncated", truncated);
+  }
+  static void generate_test_instances(std::list<cls_log_list_ret*>& ls) {
+    ls.push_back(new cls_log_list_ret);
+    ls.push_back(new cls_log_list_ret);
+    ls.back()->entries.push_back(cls_log_entry());
+    ls.back()->entries.push_back(cls_log_entry());
+    ls.back()->entries.back().section = "section";
+    ls.back()->entries.back().name = "name";
+    ls.back()->entries.back().timestamp = utime_t(1, 2);
+    ls.back()->entries.back().data.append("data");
+    ls.back()->entries.back().id = "id";
+    ls.back()->marker = "marker";
+    ls.back()->truncated = true;
+  }
 };
 WRITE_CLASS_ENCODER(cls_log_list_ret)
 
@@ -133,6 +168,20 @@ struct cls_log_trim_op {
     }
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter* f) const {
+    f->dump_stream("from_time") << from_time;
+    f->dump_stream("to_time") << to_time;
+    f->dump_string("from_marker", from_marker);
+    f->dump_string("to_marker", to_marker);
+  }
+  static void generate_test_instances(std::list<cls_log_trim_op*>& ls) {
+    ls.push_back(new cls_log_trim_op);
+    ls.push_back(new cls_log_trim_op);
+    ls.back()->from_time = utime_t(1, 2);
+    ls.back()->to_time = utime_t(3, 4);
+    ls.back()->from_marker = "from_marker";
+    ls.back()->to_marker = "to_marker";
+  }
 };
 WRITE_CLASS_ENCODER(cls_log_trim_op)
 
@@ -150,6 +199,13 @@ struct cls_log_info_op {
     // currently empty request
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter* f) const {
+  }
+
+  static void generate_test_instances(std::list<cls_log_info_op*>& ls) {
+    ls.push_back(new cls_log_info_op);
+  }
 };
 WRITE_CLASS_ENCODER(cls_log_info_op)
 
diff --git a/src/cls/log/cls_log_types.h b/src/cls/log/cls_log_types.h
index 33b8cce51e5e..29aa2bae8fe9 100644
--- a/src/cls/log/cls_log_types.h
+++ b/src/cls/log/cls_log_types.h
@@ -92,6 +92,16 @@ struct cls_log_header {
     decode(max_time, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter* f) const {
+    f->dump_string("max_marker", max_marker);
+    f->dump_stream("max_time") << max_time;
+  }
+  static void generate_test_instances(std::list<cls_log_header*>& o) {
+    o.push_back(new cls_log_header);
+    o.push_back(new cls_log_header);
+    o.back()->max_marker = "test_marker";
+    o.back()->max_time = utime_t();
+  }
 };
 inline bool operator ==(const cls_log_header& lhs, const cls_log_header& rhs) {
   return (lhs.max_marker == rhs.max_marker &&
diff --git a/src/cls/queue/cls_queue_ops.h b/src/cls/queue/cls_queue_ops.h
index 8209659bda90..0b0b30b6f01c 100644
--- a/src/cls/queue/cls_queue_ops.h
+++ b/src/cls/queue/cls_queue_ops.h
@@ -4,6 +4,7 @@
 #ifndef CEPH_CLS_QUEUE_OPS_H
 #define CEPH_CLS_QUEUE_OPS_H
 
+#include "common/ceph_json.h"
 #include "cls/queue/cls_queue_types.h"
 
 struct cls_queue_init_op {
@@ -29,6 +30,19 @@ struct cls_queue_init_op {
     DECODE_FINISH(bl);
   }
 
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("queue_size", queue_size);
+    f->dump_unsigned("max_urgent_data_size", max_urgent_data_size);
+    f->dump_unsigned("urgent_data_len", bl_urgent_data.length());
+  }
+
+  static void generate_test_instances(std::list<cls_queue_init_op*>& o) {
+    o.push_back(new cls_queue_init_op);
+    o.push_back(new cls_queue_init_op);
+    o.back()->queue_size = 1024;
+    o.back()->max_urgent_data_size = 1024;
+    o.back()->bl_urgent_data.append(std::string_view("data"));
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_init_op)
 
@@ -47,12 +61,23 @@ struct cls_queue_enqueue_op {
     DECODE_START(1, bl);
     decode(bl_data_vec, bl);
     DECODE_FINISH(bl);
-  } 
+  }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("data_vec_len", bl_data_vec.size());
+  }
+
+  static void generate_test_instances(std::list<cls_queue_enqueue_op*>& o) {
+    o.push_back(new cls_queue_enqueue_op);
+    o.push_back(new cls_queue_enqueue_op);
+    o.back()->bl_data_vec.push_back(ceph::buffer::list());
+    o.back()->bl_data_vec.back().append(std::string_view("data"));
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_enqueue_op)
 
 struct cls_queue_list_op {
-  uint64_t max;
+  uint64_t max{0};
   std::string start_marker;
 
   cls_queue_list_op() {}
@@ -70,6 +95,18 @@ struct cls_queue_list_op {
     decode(start_marker, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("max", max);
+    f->dump_string("start_marker", start_marker);
+  }
+
+  static void generate_test_instances(std::list<cls_queue_list_op*>& o) {
+    o.push_back(new cls_queue_list_op);
+    o.push_back(new cls_queue_list_op);
+    o.back()->max = 123;
+    o.back()->start_marker = "foo";
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_list_op)
 
@@ -95,6 +132,22 @@ struct cls_queue_list_ret {
     decode(entries, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_bool("is_truncated", is_truncated);
+    f->dump_string("next_marker", next_marker);
+    encode_json("entries", entries, f);
+  }
+
+  static void generate_test_instances(std::list<cls_queue_list_ret*>& o) {
+    o.push_back(new cls_queue_list_ret);
+    o.back()->is_truncated = true;
+    o.back()->next_marker = "foo";
+    o.back()->entries.push_back(cls_queue_entry());
+    o.back()->entries.push_back(cls_queue_entry());
+    o.back()->entries.back().marker = "id";
+    o.back()->entries.back().data.append(std::string_view("data"));
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_list_ret)
 
@@ -114,6 +167,15 @@ struct cls_queue_remove_op {
     decode(end_marker, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("end_marker", end_marker);
+  }
+  static void generate_test_instances(std::list<cls_queue_remove_op*>& o) {
+    o.push_back(new cls_queue_remove_op);
+    o.push_back(new cls_queue_remove_op);
+    o.back()->end_marker = "foo";
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_remove_op)
 
@@ -133,6 +195,14 @@ struct cls_queue_get_capacity_ret {
     decode(queue_capacity, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("queue_capacity", queue_capacity);
+  }
+  static void generate_test_instances(std::list<cls_queue_get_capacity_ret*>& o) {
+    o.push_back(new cls_queue_get_capacity_ret);
+    o.back()->queue_capacity = 123;
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_get_capacity_ret)
 
diff --git a/src/cls/queue/cls_queue_types.h b/src/cls/queue/cls_queue_types.h
index cc46df405052..3c3e828edf0a 100644
--- a/src/cls/queue/cls_queue_types.h
+++ b/src/cls/queue/cls_queue_types.h
@@ -34,6 +34,17 @@ struct cls_queue_entry
     decode(marker, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("marker", marker);
+    f->dump_unsigned("data_len", data.length());
+  }
+  static void generate_test_instances(std::list<cls_queue_entry*>& o) {
+    o.push_back(new cls_queue_entry);
+    o.push_back(new cls_queue_entry);
+    o.back()->data.append(std::string_view("data"));
+    o.back()->marker = "marker";
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_entry)
 
@@ -80,7 +91,16 @@ struct cls_queue_marker
     }
     return 0;
   }
-
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("offset", offset);
+    f->dump_unsigned("gen", gen);
+  }
+  static void generate_test_instances(std::list<cls_queue_marker*>& o) {
+    o.push_back(new cls_queue_marker);
+    o.push_back(new cls_queue_marker);
+    o.back()->offset = 1024;
+    o.back()->gen = 0;
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_marker)
 
@@ -114,6 +134,27 @@ struct cls_queue_head
     decode(bl_urgent_data, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("max_head_size", max_head_size);
+    f->dump_unsigned("queue_size", queue_size);
+    f->dump_unsigned("max_urgent_data_size", max_urgent_data_size);
+    f->dump_unsigned("front_offset", front.offset);
+    f->dump_unsigned("front_gen", front.gen);
+    f->dump_unsigned("tail_offset", tail.offset);
+    f->dump_unsigned("tail_gen", tail.gen);
+  }
+  static void generate_test_instances(std::list<cls_queue_head*>& o) {
+    o.push_back(new cls_queue_head);
+    o.push_back(new cls_queue_head);
+    o.back()->max_head_size = 1024;
+    o.back()->front.offset = 1024;
+    o.back()->front.gen = 0;
+    o.back()->tail.offset = 1024;
+    o.back()->tail.gen = 0;
+    o.back()->queue_size = 1024;
+    o.back()->max_urgent_data_size = 0;
+  }
 };
 WRITE_CLASS_ENCODER(cls_queue_head)
 
diff --git a/src/cls/timeindex/cls_timeindex_ops.h b/src/cls/timeindex/cls_timeindex_ops.h
index f40058954dce..f0f0cc024751 100644
--- a/src/cls/timeindex/cls_timeindex_ops.h
+++ b/src/cls/timeindex/cls_timeindex_ops.h
@@ -4,6 +4,7 @@
 #ifndef CEPH_CLS_TIMEINDEX_OPS_H
 #define CEPH_CLS_TIMEINDEX_OPS_H
 
+#include "common/ceph_json.h"
 #include "cls_timeindex_types.h"
 
 struct cls_timeindex_add_op {
@@ -51,6 +52,26 @@ struct cls_timeindex_list_op {
     decode(max_entries, bl);
     DECODE_FINISH(bl);
   }
+
+   void dump(ceph::Formatter *f) const {
+    f->open_object_section("from_time");
+    from_time.dump(f);
+    f->close_section();
+    f->dump_string("marker", marker);
+    f->open_object_section("to_time");
+    to_time.dump(f);
+    f->close_section();
+    f->dump_int("max_entries", max_entries);
+  }
+
+  static void generate_test_instances(std::list<cls_timeindex_list_op*>& o) {
+    o.push_back(new cls_timeindex_list_op);
+    o.push_back(new cls_timeindex_list_op);
+    o.back()->from_time = utime_t(1, 2);
+    o.back()->marker = "marker";
+    o.back()->to_time = utime_t(3, 4);
+    o.back()->max_entries = 5;
+  }
 };
 WRITE_CLASS_ENCODER(cls_timeindex_list_op)
 
@@ -76,6 +97,23 @@ struct cls_timeindex_list_ret {
     decode(truncated, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    encode_json("entries", entries, f);
+    f->dump_string("marker", marker);
+    f->dump_bool("truncated", truncated);
+  }
+
+  static void generate_test_instances(std::list<cls_timeindex_list_ret*>& o) {
+    o.push_back(new cls_timeindex_list_ret);
+    o.push_back(new cls_timeindex_list_ret);
+    o.back()->entries.push_back(cls_timeindex_entry());
+    o.back()->entries.back().key_ts = utime_t(1, 2);
+    o.back()->entries.back().key_ext = "key_ext";
+    o.back()->entries.back().value.append("value");
+    o.back()->marker = "marker";
+    o.back()->truncated = true;
+  }
 };
 WRITE_CLASS_ENCODER(cls_timeindex_list_ret)
 
diff --git a/src/cls/timeindex/cls_timeindex_types.h b/src/cls/timeindex/cls_timeindex_types.h
index d33886881be5..ea8d6c93d2c8 100644
--- a/src/cls/timeindex/cls_timeindex_types.h
+++ b/src/cls/timeindex/cls_timeindex_types.h
@@ -4,9 +4,9 @@
 #ifndef CEPH_CLS_TIMEINDEX_TYPES_H
 #define CEPH_CLS_TIMEINDEX_TYPES_H
 
+#include "common/Formatter.h"
 #include "include/encoding.h"
 #include "include/types.h"
-
 #include "include/utime.h"
 
 class JSONObj;
diff --git a/src/cls/version/cls_version_ops.h b/src/cls/version/cls_version_ops.h
index 62cd1172982a..2eff788ce52c 100644
--- a/src/cls/version/cls_version_ops.h
+++ b/src/cls/version/cls_version_ops.h
@@ -5,6 +5,7 @@
 #define CEPH_CLS_VERSION_OPS_H
 
 #include "cls_version_types.h"
+#include "common/ceph_json.h"
 
 struct cls_version_set_op {
   obj_version objv;
@@ -22,6 +23,17 @@ struct cls_version_set_op {
     decode(objv, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("objv", objv);
+  }
+
+  static void generate_test_instances(std::list<cls_version_set_op*>& o) {
+    o.push_back(new cls_version_set_op);
+    o.push_back(new cls_version_set_op);
+    o.back()->objv.ver = 123;
+    o.back()->objv.tag = "foo";
+  }
 };
 WRITE_CLASS_ENCODER(cls_version_set_op)
 
@@ -44,6 +56,22 @@ struct cls_version_inc_op {
     decode(conds, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("objv", objv);
+    encode_json("conds", conds, f);
+  }
+
+  static void generate_test_instances(std::list<cls_version_inc_op*>& o) {
+    o.push_back(new cls_version_inc_op);
+    o.push_back(new cls_version_inc_op);
+    o.back()->objv.ver = 123;
+    o.back()->objv.tag = "foo";
+    o.back()->conds.push_back(obj_version_cond());
+    o.back()->conds.back().ver.ver = 123;
+    o.back()->conds.back().ver.tag = "foo";
+    o.back()->conds.back().cond = VER_COND_GE;
+  }
 };
 WRITE_CLASS_ENCODER(cls_version_inc_op)
 
@@ -66,6 +94,22 @@ struct cls_version_check_op {
     decode(conds, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("objv", objv);
+    encode_json("conds", conds, f);
+  }
+
+  static void generate_test_instances(std::list<cls_version_check_op*>& o) {
+    o.push_back(new cls_version_check_op);
+    o.push_back(new cls_version_check_op);
+    o.back()->objv.ver = 123;
+    o.back()->objv.tag = "foo";
+    o.back()->conds.push_back(obj_version_cond());
+    o.back()->conds.back().ver.ver = 123;
+    o.back()->conds.back().ver.tag = "foo";
+    o.back()->conds.back().cond = VER_COND_GE;
+  }
 };
 WRITE_CLASS_ENCODER(cls_version_check_op)
 
@@ -85,6 +129,17 @@ struct cls_version_read_ret {
     decode(objv, bl);
     DECODE_FINISH(bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("objv", objv);
+  }
+
+  static void generate_test_instances(std::list<cls_version_read_ret*>& o) {
+    o.push_back(new cls_version_read_ret);
+    o.push_back(new cls_version_read_ret);
+    o.back()->objv.ver = 123;
+    o.back()->objv.tag = "foo";
+  }
 };
 WRITE_CLASS_ENCODER(cls_version_read_ret)
 
diff --git a/src/cls/version/cls_version_types.cc b/src/cls/version/cls_version_types.cc
index b82f6aa8a5dd..735ef7c89857 100644
--- a/src/cls/version/cls_version_types.cc
+++ b/src/cls/version/cls_version_types.cc
@@ -6,12 +6,6 @@
 #include "common/ceph_json.h"
 
 
-void obj_version::dump(ceph::Formatter *f) const
-{
-  f->dump_int("ver", ver);
-  f->dump_string("tag", tag);
-}
-
 void obj_version::decode_json(JSONObj *obj)
 {
   JSONDecoder::decode_json("ver", ver, obj);
diff --git a/src/cls/version/cls_version_types.h b/src/cls/version/cls_version_types.h
index 62cc16e33d52..dafa866e1f35 100644
--- a/src/cls/version/cls_version_types.h
+++ b/src/cls/version/cls_version_types.h
@@ -53,7 +53,11 @@ struct obj_version {
             tag.compare(v.tag) == 0);
   }
 
-  void dump(ceph::Formatter *f) const;
+  void dump(ceph::Formatter *f) const {
+    f->dump_int("ver", ver);
+    f->dump_string("tag", tag);
+  }
+
   void decode_json(JSONObj *obj);
   static void generate_test_instances(std::list<obj_version*>& o);
 };
@@ -91,6 +95,18 @@ struct obj_version_cond {
     DECODE_FINISH(bl);
   }
 
+  void dump(ceph::Formatter *f) const {
+    f->dump_object("ver", ver);
+    f->dump_unsigned("cond", cond);
+  }
+
+  static void generate_test_instances(std::list<obj_version_cond*>& o) {
+    o.push_back(new obj_version_cond);
+    o.push_back(new obj_version_cond);
+    o.back()->ver.ver = 1;
+    o.back()->ver.tag = "foo";
+    o.back()->cond = VER_COND_EQ;
+  }
 };
 WRITE_CLASS_ENCODER(obj_version_cond)
 
diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h
index 08e8d9e46623..f7a899fd9356 100644
--- a/src/common/ceph_json.h
+++ b/src/common/ceph_json.h
@@ -836,6 +836,61 @@ class JSONFormattable : public ceph::JSONFormatter {
     DECODE_FINISH(bl);
   }
 
+  void dump(ceph::Formatter *f) const {
+    switch (type) {
+      case FMT_VALUE:
+        if (value.quoted) {
+          f->dump_string("value", value.str);
+        } else {
+          f->dump_format_unquoted("value", "%s", value.str.c_str());
+        }
+        break;
+      case FMT_ARRAY:
+        f->open_array_section("array");
+        for (auto& i : arr) {
+          i.dump(f);
+        }
+        f->close_section();
+        break;
+      case FMT_OBJ:
+        f->open_object_section("object");
+        for (auto& i : obj) {
+          f->dump_object(i.first.c_str(), i.second);
+        }
+        f->close_section();
+        break;
+      default:
+        break;
+    }
+  }
+  static void generate_test_instances(std::list<JSONFormattable*>& o) {
+    o.push_back(new JSONFormattable);
+    o.push_back(new JSONFormattable);
+    o.back()->set_type(FMT_VALUE);
+    o.back()->value.str = "foo";
+    o.back()->value.quoted = true;
+    o.push_back(new JSONFormattable);
+    o.back()->set_type(FMT_VALUE);
+    o.back()->value.str = "foo";
+    o.back()->value.quoted = false;
+    o.push_back(new JSONFormattable);
+    o.back()->set_type(FMT_ARRAY);
+    o.back()->arr.push_back(JSONFormattable());
+    o.back()->arr.back().set_type(FMT_VALUE);
+    o.back()->arr.back().value.str = "foo";
+    o.back()->arr.back().value.quoted = true;
+    o.back()->arr.push_back(JSONFormattable());
+    o.back()->arr.back().set_type(FMT_VALUE);
+    o.back()->arr.back().value.str = "bar";
+    o.back()->arr.back().value.quoted = true;
+    o.push_back(new JSONFormattable);
+    o.back()->set_type(FMT_OBJ);
+    o.back()->obj["foo"] = JSONFormattable();
+    o.back()->obj["foo"].set_type(FMT_VALUE);
+    o.back()->obj["foo"].value.str = "bar";
+    o.back()->obj["foo"].value.quoted = true;
+  }
+
   const std::string& val() const {
     return value.str;
   }
diff --git a/src/common/entity_name.cc b/src/common/entity_name.cc
index 5357b34eacb7..a9d6fb9c8b10 100644
--- a/src/common/entity_name.cc
+++ b/src/common/entity_name.cc
@@ -29,21 +29,30 @@ const std::array<EntityName::str_to_entity_type_t, 6> EntityName::STR_TO_ENTITY_
   { CEPH_ENTITY_TYPE_CLIENT, "client" },
 }};
 
-const std::string& EntityName::
-to_str() const
-{
+void EntityName::dump(ceph::Formatter *f) const {
+  f->dump_int("type", type);
+  f->dump_string("id", id);
+}
+
+void EntityName::generate_test_instances(std::list<EntityName*>& ls) {
+  ls.push_back(new EntityName);
+  ls.push_back(new EntityName);
+  ls.back()->set_type(CEPH_ENTITY_TYPE_OSD);
+  ls.back()->set_id("0");
+  ls.push_back(new EntityName);
+  ls.back()->set_type(CEPH_ENTITY_TYPE_MDS);
+  ls.back()->set_id("a");
+}
+
+const std::string& EntityName::to_str() const {
   return type_id;
 }
 
-const char* EntityName::
-to_cstr() const
-{
+const char* EntityName::to_cstr() const {
   return type_id.c_str();
 }
 
-bool EntityName::
-from_str(std::string_view s)
-{
+bool EntityName::from_str(std::string_view s) {
   size_t pos = s.find('.');
 
   if (pos == string::npos)
@@ -56,9 +65,7 @@ from_str(std::string_view s)
   return true;
 }
 
-void EntityName::
-set(uint32_t type_, std::string_view id_)
-{
+void EntityName::set(uint32_t type_, std::string_view id_) {
   type = type_;
   id = id_;
 
@@ -71,9 +78,7 @@ set(uint32_t type_, std::string_view id_)
   }
 }
 
-int EntityName::
-set(std::string_view type_, std::string_view id_)
-{
+int EntityName::set(std::string_view type_, std::string_view id_) {
   uint32_t t = str_to_ceph_entity_type(type_);
   if (t == CEPH_ENTITY_TYPE_ANY)
     return -EINVAL;
@@ -81,9 +86,7 @@ set(std::string_view type_, std::string_view id_)
   return 0;
 }
 
-void EntityName::
-set_type(uint32_t type_)
-{
+void EntityName::set_type(uint32_t type_) {
   set(type_, id);
 }
 
@@ -93,9 +96,7 @@ set_type(std::string_view type_)
   return set(type_, id);
 }
 
-void EntityName::
-set_id(std::string_view id_)
-{
+void EntityName::set_id(std::string_view id_) {
   set(type, id_);
 }
 
@@ -106,33 +107,23 @@ void EntityName::set_name(entity_name_t n)
   set(n.type(), s);
 }
 
-const char* EntityName::
-get_type_str() const
-{
+const char* EntityName::get_type_str() const {
   return ceph_entity_type_name(type);
 }
 
-std::string_view EntityName::
-get_type_name() const
-{
+std::string_view EntityName::get_type_name() const {
   return ceph_entity_type_name(type);
 }
 
-const std::string &EntityName::
-get_id() const
-{
+const std::string &EntityName::get_id() const {
   return id;
 }
 
-bool EntityName::
-has_default_id() const
-{
+bool EntityName::has_default_id() const {
   return (id == "admin");
 }
 
-std::string EntityName::
-get_valid_types_as_str()
-{
+std::string EntityName::get_valid_types_as_str() {
   std::ostringstream out;
   size_t i;
   for (i = 0; i < STR_TO_ENTITY_TYPE.size(); ++i) {
diff --git a/src/common/entity_name.h b/src/common/entity_name.h
index c88ebcbbabde..53f8cd4d5d09 100644
--- a/src/common/entity_name.h
+++ b/src/common/entity_name.h
@@ -41,7 +41,8 @@ struct EntityName
     decode(id_, bl);
     set(type_, id_);
   }
-
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<EntityName*>& ls);
   const std::string& to_str() const;
   const char *to_cstr() const;
   bool from_str(std::string_view s);
diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 049ef8e3fcf6..54d97d9d4a2f 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -349,6 +349,8 @@ struct inline_data_t {
   }
   void encode(ceph::buffer::list &bl) const;
   void decode(ceph::buffer::list::const_iterator& bl);
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<inline_data_t*>& ls);
 
   version_t version = 1;
 
diff --git a/src/include/frag.h b/src/include/frag.h
index ec18bddfbb1e..5ea2429aee8d 100644
--- a/src/include/frag.h
+++ b/src/include/frag.h
@@ -159,6 +159,15 @@ class frag_t {
     ceph::decode_raw(v, p);
     _enc = v;
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("value", value());
+    f->dump_unsigned("bits", bits());
+  }
+  static void generate_test_instances(std::list<frag_t*>& ls) {
+    ls.push_back(new frag_t);
+    ls.push_back(new frag_t(10, 2));
+    ls.push_back(new frag_t(11, 3));
+  }
   bool operator<(const frag_t& b) const
   {
     if (value() != b.value())
@@ -525,6 +534,11 @@ class fragtree_t {
     }
     f->close_section(); // splits
   }
+
+  static void generate_test_instances(std::list<fragtree_t*>& ls) {
+    ls.push_back(new fragtree_t);
+    ls.push_back(new fragtree_t);
+  }
 };
 WRITE_CLASS_ENCODER(fragtree_t)
 
diff --git a/src/include/fs_types.h b/src/include/fs_types.h
index c1932bfcc30e..606b9c6503d4 100644
--- a/src/include/fs_types.h
+++ b/src/include/fs_types.h
@@ -75,6 +75,13 @@ struct inodeno_t {
     using ceph::decode;
     decode(val, p);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("val", val);
+  }
+  static void generate_test_instances(std::list<inodeno_t*>& ls) {
+    ls.push_back(new inodeno_t(1));
+    ls.push_back(new inodeno_t(123456789));
+  }
 } __attribute__ ((__may_alias__));
 WRITE_CLASS_ENCODER(inodeno_t)
 
diff --git a/src/include/object.h b/src/include/object.h
index 4564af86e577..2e5fb471c139 100644
--- a/src/include/object.h
+++ b/src/include/object.h
@@ -25,6 +25,7 @@
 
 #include "include/rados.h"
 #include "include/unordered_map.h"
+#include "common/Formatter.h"
 
 #include "hash.h"
 #include "encoding.h"
@@ -58,6 +59,15 @@ struct object_t {
     using ceph::decode;
     decode(name, bl);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("name", name);
+  }
+
+  static void generate_test_instances(std::list<object_t*>& o) {
+    o.push_back(new object_t);
+    o.push_back(new object_t("myobject"));
+  }
 };
 WRITE_CLASS_ENCODER(object_t)
 
@@ -170,6 +180,14 @@ struct sobject_t {
     decode(oid, bl);
     decode(snap, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("oid") << oid;
+    f->dump_stream("snap") << snap;
+  }
+  static void generate_test_instances(std::list<sobject_t*>& o) {
+    o.push_back(new sobject_t);
+    o.push_back(new sobject_t(object_t("myobject"), 123));
+  }
 };
 WRITE_CLASS_ENCODER(sobject_t)
 
diff --git a/src/include/types.h b/src/include/types.h
index bdd09a53df10..a50a506eb0bc 100644
--- a/src/include/types.h
+++ b/src/include/types.h
@@ -371,6 +371,14 @@ struct client_t {
     using ceph::decode;
     decode(v, bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_int("id", v);
+  }
+  static void generate_test_instances(std::list<client_t*>& ls) {
+    ls.push_back(new client_t);
+    ls.push_back(new client_t(1));
+    ls.push_back(new client_t(123));
+  }
 };
 WRITE_CLASS_ENCODER(client_t)
 
@@ -517,7 +525,13 @@ struct shard_id_t {
     using ceph::decode;
     decode(id, bl);
   }
-
+  void dump(ceph::Formatter *f) const {
+    f->dump_int("id", id);
+  }
+  static void generate_test_instances(std::list<shard_id_t*>& ls) {
+    ls.push_back(new shard_id_t(1));
+    ls.push_back(new shard_id_t(2));
+  }
   bool operator==(const shard_id_t&) const = default;
   auto operator<=>(const shard_id_t&) const = default;
 };
@@ -561,6 +575,13 @@ struct errorcode32_t {
     decode(code, bl);
     code = ceph_to_hostos_errno(code);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_int("code", code);
+  }
+  static void generate_test_instances(std::list<errorcode32_t*>& ls) {
+    ls.push_back(new errorcode32_t(1));
+    ls.push_back(new errorcode32_t(2));
+  }
 };
 WRITE_CLASS_ENCODER(errorcode32_t)
 
@@ -602,6 +623,16 @@ struct sha_digest_t {
     decode(tmparr, bl);
     memcpy(v, tmparr.data(), SIZE);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("sha1", to_str());
+  }
+  static void generate_test_instances(std::list<sha_digest_t*>& ls) {
+    ls.push_back(new sha_digest_t);
+    ls.push_back(new sha_digest_t);
+    ls.back()->v[0] = 1;
+    ls.push_back(new sha_digest_t);
+    ls.back()->v[0] = 2;
+  }
 };
 
 template<uint8_t S>
diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 044c33459606..21e17ca0e5bc 100644
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -284,6 +284,21 @@ void inline_data_t::decode(bufferlist::const_iterator &p)
     free_data();
 }
 
+void inline_data_t::dump(Formatter *f) const
+{
+  f->dump_unsigned("version", version);
+  f->dump_unsigned("length", length());
+}
+
+void inline_data_t::generate_test_instances(std::list<inline_data_t*>& ls)
+{
+  ls.push_back(new inline_data_t);
+  ls.push_back(new inline_data_t);
+  bufferlist bl;
+  bl.append("inline data");
+  ls.back()->set_data(bl);
+}
+
 
 /*
  * fnode_t
diff --git a/src/messages/MClientReply.h b/src/messages/MClientReply.h
index be33fad49491..028c4200c149 100644
--- a/src/messages/MClientReply.h
+++ b/src/messages/MClientReply.h
@@ -291,7 +291,7 @@ struct InodeStat {
 };
 
 struct openc_response_t {
-  _inodeno_t			created_ino;
+  _inodeno_t			created_ino{0};
   interval_set<inodeno_t>	delegated_inos;
 
 public:
@@ -309,6 +309,16 @@ struct openc_response_t {
     decode(delegated_inos, p);
     DECODE_FINISH(p);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("created_ino", created_ino);
+    f->dump_stream("delegated_inos") << delegated_inos;
+  }
+  static void generate_test_instances(std::list<openc_response_t*>& ls) {
+    ls.push_back(new openc_response_t);
+    ls.push_back(new openc_response_t);
+    ls.back()->created_ino = 1;
+    ls.back()->delegated_inos.insert(1, 10);
+  }
 } __attribute__ ((__may_alias__));
 WRITE_CLASS_ENCODER(openc_response_t)
 
diff --git a/src/messages/MClientRequest.h b/src/messages/MClientRequest.h
index c62e183a7563..a51d73489186 100644
--- a/src/messages/MClientRequest.h
+++ b/src/messages/MClientRequest.h
@@ -60,6 +60,17 @@ struct SnapPayload {
     decode(metadata, iter);
     DECODE_FINISH(iter);
   }
+  void dump(ceph::Formatter *f) const {
+    for (const auto &i : metadata) {
+      f->dump_string(i.first.c_str(), i.second);
+    }
+  }
+  static void generate_test_instances(std::list<SnapPayload *> &o) {
+    o.push_back(new SnapPayload);
+    o.push_back(new SnapPayload);
+    o.back()->metadata["key1"] = "val1";
+    o.back()->metadata["key2"] = "val2";
+  }
 };
 
 WRITE_CLASS_ENCODER(SnapPayload)
@@ -95,6 +106,26 @@ class MClientRequest final : public MMDSOp {
       decode(item, bl);
       ceph::decode_nohead(item.dname_len, dname, bl);
     }
+
+    void dump(ceph::Formatter *f) const {
+      f->dump_string("dname", dname);
+      f->dump_unsigned("ino", item.ino);
+      f->dump_unsigned("cap_id", item.cap_id);
+      f->dump_unsigned("caps", item.caps);
+      f->dump_unsigned("wanted", item.wanted);
+      f->dump_unsigned("seq", item.seq);
+      f->dump_unsigned("issue_seq", item.issue_seq);
+      f->dump_unsigned("mseq", item.mseq);
+      f->dump_unsigned("dname_seq", item.dname_seq);
+      f->dump_unsigned("dname_len", item.dname_len);
+    }
+
+    static void generate_test_instances(std::list<Release*>& ls) {
+      ls.push_back(new Release);
+      ls.push_back(new Release);
+      ls.back()->item.dname_len = 4;
+      ls.back()->dname = "test";
+    }
   };
   mutable std::vector<Release> releases; /* XXX HACK! */
 
diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h
index a9211d53538a..9237c79b6e77 100644
--- a/src/messages/MMDSCacheRejoin.h
+++ b/src/messages/MMDSCacheRejoin.h
@@ -63,6 +63,17 @@ class MMDSCacheRejoin final : public MMDSOp {
       decode(nestlock, bl);
       decode(dftlock, bl);
     }
+    void dump(ceph::Formatter *f) const {
+      f->dump_int("nonce", nonce);
+      f->dump_int("caps_wanted", caps_wanted);
+      f->dump_int("filelock", filelock);
+      f->dump_int("nestlock", nestlock);
+      f->dump_int("dftlock", dftlock);
+    }
+    static void generate_test_instances(std::list<inode_strong*>& ls) {
+      ls.push_back(new inode_strong);
+      ls.push_back(new inode_strong(1, 2, 3, 4, 5));
+    }
   };
   WRITE_CLASS_ENCODER(inode_strong)
 
@@ -81,6 +92,14 @@ class MMDSCacheRejoin final : public MMDSOp {
       decode(nonce, bl);
       decode(dir_rep, bl);
     }
+    static void generate_test_instances(std::list<dirfrag_strong*>& ls) {
+      ls.push_back(new dirfrag_strong);
+      ls.push_back(new dirfrag_strong(1, 2));
+    }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("nonce", nonce);
+      f->dump_unsigned("dir_rep", dir_rep);
+    }
   };
   WRITE_CLASS_ENCODER(dirfrag_strong)
 
@@ -118,6 +137,19 @@ class MMDSCacheRejoin final : public MMDSOp {
       decode(lock, bl);
       decode(alternate_name, bl);
     }
+    static void generate_test_instances(std::list<dn_strong*>& ls) {
+      ls.push_back(new dn_strong);
+      ls.push_back(new dn_strong(1, "alternate_name", 2, 3, 4, 5, 6));
+    }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("first", first);
+      f->dump_string("alternate_name", alternate_name);
+      f->dump_unsigned("ino", ino);
+      f->dump_unsigned("remote_ino", remote_ino);
+      f->dump_unsigned("remote_d_type", remote_d_type);
+      f->dump_unsigned("nonce", nonce);
+      f->dump_unsigned("lock", lock);
+    }
   };
   WRITE_CLASS_ENCODER(dn_strong)
 
diff --git a/src/messages/MMgrReport.h b/src/messages/MMgrReport.h
index bd3c221a847e..e81282dc1a06 100644
--- a/src/messages/MMgrReport.h
+++ b/src/messages/MMgrReport.h
@@ -74,6 +74,27 @@ class PerfCounterType
     }
     DECODE_FINISH(p);
   }
+
+  void dump(ceph::Formatter *f) const
+  {
+    f->dump_string("path", path);
+    f->dump_string("description", description);
+    f->dump_string("nick", nick);
+    f->dump_int("type", type);
+    f->dump_int("priority", priority);
+    f->dump_int("unit", unit);
+  }
+  static void generate_test_instances(std::list<PerfCounterType*>& ls)
+  {
+    ls.push_back(new PerfCounterType);
+    ls.push_back(new PerfCounterType);
+    ls.back()->path = "mycounter";
+    ls.back()->description = "mycounter description";
+    ls.back()->nick = "mycounter nick";
+    ls.back()->type = PERFCOUNTER_COUNTER;
+    ls.back()->priority = PerfCountersBuilder::PRIO_CRITICAL;
+    ls.back()->unit = UNIT_BYTES;
+  }
 };
 WRITE_CLASS_ENCODER(PerfCounterType)
 
diff --git a/src/os/bluestore/bluestore_types.cc b/src/os/bluestore/bluestore_types.cc
index 904b6fbd3d98..80a8ea06a805 100644
--- a/src/os/bluestore/bluestore_types.cc
+++ b/src/os/bluestore/bluestore_types.cc
@@ -1087,6 +1087,15 @@ void bluestore_onode_t::shard_info::dump(Formatter *f) const
   f->dump_unsigned("bytes", bytes);
 }
 
+void bluestore_onode_t::shard_info::generate_test_instances(
+  list<shard_info*>& o)
+{
+  o.push_back(new shard_info);
+  o.push_back(new shard_info);
+  o.back()->offset = 123;
+  o.back()->bytes = 456;
+}
+
 ostream& operator<<(ostream& out, const bluestore_onode_t::shard_info& si)
 {
   return out << std::hex << "0x" << si.offset << "(0x" << si.bytes << " bytes"
diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h
index 4c96e8903260..cdedf223b055 100644
--- a/src/os/bluestore/bluestore_types.h
+++ b/src/os/bluestore/bluestore_types.h
@@ -954,6 +954,7 @@ struct bluestore_onode_t {
       denc_varint(v.bytes, p);
     }
     void dump(ceph::Formatter *f) const;
+    static void generate_test_instances(std::list<shard_info*>& ls);
   };
   std::vector<shard_info> extent_map_shards; ///< extent std::map shards (if any)
 
diff --git a/src/tools/ceph-dencoder/common_types.h b/src/tools/ceph-dencoder/common_types.h
index 3180e3476be2..e853321645ba 100644
--- a/src/tools/ceph-dencoder/common_types.h
+++ b/src/tools/ceph-dencoder/common_types.h
@@ -23,12 +23,29 @@ TYPE(filepath)
 
 #include "include/fs_types.h"
 TYPE_FEATUREFUL(file_layout_t)
+TYPE(inodeno_t)
 
 #include "include/util.h"
 TYPE(ceph_data_stats)
 
+#include "include/object.h"
+TYPE(object_t)
+TYPE(sobject_t)
+
+#include "include/frag.h"
+TYPE(fragtree_t)
+TYPE(frag_t)
+
+#include "include/types.h"
+TYPE(shard_id_t)
+TYPE(inline_data_t)
+TYPE(sha256_digest_t)
+TYPE(errorcode32_t)
+TYPE(client_t)
+
 #include "common/bit_vector.hpp"
 TYPE(BitVector<2>)
+TYPE(ceph::BitVector<2>)
 
 #include "common/bloom_filter.hpp"
 TYPE(bloom_filter)
@@ -37,6 +54,9 @@ TYPE(compressible_bloom_filter)
 #include "common/DecayCounter.h"
 TYPE(DecayCounter)
 
+#include "common/entity_name.h"
+TYPE(EntityName)
+
 #include "common/histogram.h"
 TYPE(pow2_hist_t)
 
@@ -71,11 +91,17 @@ TYPE(cls_cas_chunk_put_ref_op)
 
 #include "cls/cas/cls_cas_internal.h"
 TYPE(chunk_refs_t)
+TYPE(chunk_refs_count_t)
+TYPE(chunk_refs_by_object_t)
 
 #include "cls/lock/cls_lock_types.h"
 TYPE(rados::cls::lock::locker_id_t)
 TYPE_FEATUREFUL(rados::cls::lock::locker_info_t)
 TYPE_FEATUREFUL(rados::cls::lock::lock_info_t)
+using namespace rados::cls::lock;
+TYPE(locker_id_t)
+TYPE_FEATUREFUL(locker_info_t)
+TYPE_FEATUREFUL(lock_info_t)
 
 #include "cls/lock/cls_lock_ops.h"
 TYPE(cls_lock_lock_op)
@@ -98,6 +124,64 @@ TYPE(obj_refcount)
 #include "cls/timeindex/cls_timeindex_types.h"
 TYPE(cls_timeindex_entry)
 
+#include "cls/timeindex/cls_timeindex_ops.h"
+TYPE(cls_timeindex_list_op)
+TYPE(cls_timeindex_list_ret)
+
+#include "cls/queue/cls_queue_types.h"
+TYPE(cls_queue_entry)
+TYPE(cls_queue_marker)
+TYPE(cls_queue_head)
+
+#include "cls/queue/cls_queue_ops.h"
+TYPE(cls_queue_get_capacity_ret)
+TYPE(cls_queue_remove_op)
+TYPE(cls_queue_enqueue_op)
+TYPE(cls_queue_list_op)
+TYPE(cls_queue_list_ret)
+TYPE(cls_queue_init_op)
+
+#include "cls/2pc_queue/cls_2pc_queue_ops.h"
+TYPE(cls_2pc_queue_abort_op)
+TYPE(cls_2pc_queue_commit_op)
+TYPE(cls_2pc_queue_expire_op)
+TYPE_NONDETERMINISTIC(cls_2pc_queue_reservations_ret)
+TYPE(cls_2pc_queue_reserve_op)
+TYPE(cls_2pc_queue_reserve_ret)
+TYPE(cls_queue_init_op)
+
+#include "cls/2pc_queue/cls_2pc_queue_types.h"
+TYPE(cls_2pc_reservation)
+TYPE_NONDETERMINISTIC(cls_2pc_urgent_data)
+
+#include "cls/log/cls_log_types.h"
+TYPE(cls_log_header)
+
+#include "cls/log/cls_log_ops.h"
+TYPE(cls_log_info_op)
+TYPE(cls_log_list_op)
+TYPE(cls_log_list_ret)
+TYPE(cls_log_trim_op)
+
+#include "cls/version/cls_version_ops.h"
+TYPE(cls_version_check_op)
+TYPE(cls_version_read_ret)
+TYPE(cls_version_inc_op)
+TYPE(cls_version_set_op)
+
+
+#include "cls/fifo/cls_fifo_ops.h"
+using namespace rados::cls::fifo::op;
+TYPE(create_meta)
+TYPE(get_meta)
+TYPE(get_meta_reply)
+
+#include "cls/fifo/cls_fifo_types.h"
+using namespace rados::cls::fifo;
+TYPE(data_params)
+TYPE(objv)
+TYPE(info)
+
 #include "journal/Entry.h"
 TYPE(journal::Entry)
 
@@ -125,9 +209,12 @@ MESSAGE(MClientReconnect)
 
 #include "messages/MClientReply.h"
 MESSAGE(MClientReply)
+TYPE(openc_response_t)
 
 #include "messages/MClientRequest.h"
 MESSAGE(MClientRequest)
+TYPE(SnapPayload)
+TYPE(MClientRequest::Release)
 
 #include "messages/MClientRequestForward.h"
 MESSAGE(MClientRequestForward)
@@ -251,6 +338,9 @@ MESSAGE(MMDSBeacon)
 
 #include "messages/MMDSCacheRejoin.h"
 MESSAGE(MMDSCacheRejoin)
+TYPE(MMDSCacheRejoin::dirfrag_strong)
+TYPE(MMDSCacheRejoin::dn_strong)
+TYPE(MMDSCacheRejoin::inode_strong)
 
 #include "messages/MMDSFindIno.h"
 MESSAGE(MMDSFindIno)
@@ -265,10 +355,14 @@ MESSAGE(MMDSFragmentNotify)
 MESSAGE(MMDSLoadTargets)
 
 #include "messages/MMDSMap.h"
-MESSAGE(MMDSMap)
+MESSAGE(MMDSMap) 
+
+#include "messages/MMgrBeacon.h"
+MESSAGE(MMgrBeacon)
 
 #include "messages/MMgrReport.h"
 MESSAGE(MMgrReport)
+TYPE(PerfCounterType)
 
 #include "messages/MMDSResolve.h"
 MESSAGE(MMDSResolve)
@@ -321,6 +415,9 @@ MESSAGE(MMonGetVersionReply)
 #include "messages/MMonGlobalID.h"
 MESSAGE(MMonGlobalID)
 
+#include "messages/MMonHealth.h"
+MESSAGE(MMonHealth)
+
 #include "messages/MMonJoin.h"
 MESSAGE(MMonJoin)
 
@@ -452,3 +549,74 @@ MESSAGE(MWatchNotify)
 
 #include "messages/MMgrUpdate.h" 
 MESSAGE(MMgrUpdate)
+
+#include "messages/MOSDECSubOpRead.h"
+MESSAGE(MOSDECSubOpRead)
+
+#include "messages/MOSDECSubOpReadReply.h"
+MESSAGE(MOSDECSubOpReadReply)
+
+#include "messages/MOSDECSubOpWrite.h"
+MESSAGE(MOSDECSubOpWrite)
+
+#include "messages/MOSDECSubOpWriteReply.h"
+MESSAGE(MOSDECSubOpWriteReply)
+
+#include "messages/MOSDMarkMeDown.h"
+MESSAGE(MOSDMarkMeDown)
+
+#include "messages/MOSDPGCreated.h"
+MESSAGE(MOSDPGCreated)
+
+#include "messages/MOSDPGPush.h"
+MESSAGE(MOSDPGPush)
+
+#include "messages/MOSDPGPushReply.h"
+MESSAGE(MOSDPGPushReply)
+
+#include "messages/MOSDPGUpdateLogMissing.h"
+MESSAGE(MOSDPGUpdateLogMissing)
+
+#include "messages/MOSDPGUpdateLogMissingReply.h"
+MESSAGE(MOSDPGUpdateLogMissingReply)
+
+#include "messages/MOSDRepOp.h"
+MESSAGE(MOSDRepOp)
+
+#include "messages/MOSDRepOpReply.h"
+MESSAGE(MOSDRepOpReply)
+
+#include "messages/MRecoveryReserve.h"
+MESSAGE(MRecoveryReserve)
+
+
+#include "auth/cephx/CephxProtocol.h"
+TYPE(CephXAuthenticate)
+TYPE(CephXAuthorize)
+TYPE(CephXAuthorizeChallenge)
+TYPE(CephXAuthorizeReply)
+TYPE(CephXChallengeBlob)
+TYPE(CephXRequestHeader)
+TYPE(CephXResponseHeader)
+TYPE(CephXServerChallenge)
+TYPE(CephXServiceTicket)
+TYPE(CephXServiceTicketInfo)
+TYPE(CephXServiceTicketRequest)
+TYPE(CephXTicketBlob)
+
+#include "auth/cephx/CephxKeyServer.h"
+TYPE(KeyServerData)
+TYPE(KeyServerData::Incremental)
+
+#include "auth/Auth.h"
+TYPE(RotatingSecrets)
+TYPE(ExpiringCryptoKey)
+TYPE(AuthCapsInfo)
+TYPE(AuthTicket)
+TYPE(EntityAuth)
+
+#include "auth/Crypto.h"
+TYPE(CryptoKey)
+
+#include "common/ceph_json.h"
+TYPE(JSONFormattable)

From c64501d07a47989454a43bb1cb2a0d74bbfa5df7 Mon Sep 17 00:00:00 2001
From: NitzanMordhai <nmordech@redhat.com>
Date: Thu, 7 Sep 2023 05:56:55 +0000
Subject: [PATCH 0044/2492] submodule: update ceph-object-corpus submodule

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 ceph-object-corpus | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ceph-object-corpus b/ceph-object-corpus
index 038c72b5acec..530602c5f31d 160000
--- a/ceph-object-corpus
+++ b/ceph-object-corpus
@@ -1 +1 @@
-Subproject commit 038c72b5acec667e1aca4c79a8cfcae705d766fe
+Subproject commit 530602c5f31d68595495593027439838c459b1eb

From cc4b75718f2f9c2e57a40fd2ed395762bbff5bd9 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 30 Aug 2023 08:19:27 +0000
Subject: [PATCH 0045/2492] qa/suites/rados/thrash/thrashers/mapgap: Increase
 trimming probability

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/suites/rados/thrash/thrashers/mapgap.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/suites/rados/thrash/thrashers/mapgap.yaml b/qa/suites/rados/thrash/thrashers/mapgap.yaml
index 3b34f5b6b37e..ea7a0c911b5c 100644
--- a/qa/suites/rados/thrash/thrashers/mapgap.yaml
+++ b/qa/suites/rados/thrash/thrashers/mapgap.yaml
@@ -18,6 +18,7 @@ overrides:
         osd scrub max interval: 120
         osd scrub during recovery: false
         osd max backfills: 6
+        osd beacon report interval: 30
 tasks:
 - thrashosds:
     timeout: 1800
@@ -25,3 +26,4 @@ tasks:
     chance_pgnum_shrink: 0.25
     chance_pgpnum_fix: 0.25
     chance_test_map_discontinuity: 2
+    map_discontinuity_sleep_time: 200

From 527b34e6285719f163519f9562523db64012bcdf Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 10 Sep 2023 14:38:28 +0000
Subject: [PATCH 0046/2492] doc/dev/osd_internals/past_intervals.rst: add
 OSDSuperblock::maps

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 doc/dev/osd_internals/past_intervals.rst | 35 ++++++++++++++++++------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/doc/dev/osd_internals/past_intervals.rst b/doc/dev/osd_internals/past_intervals.rst
index 5b594df1ae0c..4d8e7d720933 100644
--- a/doc/dev/osd_internals/past_intervals.rst
+++ b/doc/dev/osd_internals/past_intervals.rst
@@ -1,9 +1,10 @@
-=============
-PastIntervals
-=============
+=================================
+OSDMap Trimming and PastIntervals
+=================================
+
 
-Purpose
--------
+PastIntervals
+-------------
 
 There are two situations where we need to consider the set of all acting-set
 OSDs for a PG back to some epoch ``e``:
@@ -81,13 +82,31 @@ trimmed up to epoch ``e``, we know that the PG must have been clean at some epoc
 
 This dependency also pops up in PeeringState::check_past_interval_bounds().
 PeeringState::get_required_past_interval_bounds takes as a parameter
-oldest_epoch, which comes from OSDSuperblock::cluster_osdmap_trim_lower_bound.
-We use cluster_osdmap_trim_lower_bound rather than a specific osd's oldest_map
+oldest epoch, which comes from OSDSuperblock::cluster_osdmap_trim_lower_bound.
+We use cluster_osdmap_trim_lower_bound rather than a specific osd's oldest map
 because we don't necessarily trim all MOSDMap::cluster_osdmap_trim_lower_bound.
 In order to avoid doing too much work at once we limit the amount of osdmaps
 trimmed using ``osd_target_transaction_size`` in OSD::trim_maps().
-For this reason, a specific OSD's oldest_map can lag behind
+For this reason, a specific OSD's oldest map can lag behind
 OSDSuperblock::cluster_osdmap_trim_lower_bound
 for a while.
 
 See https://tracker.ceph.com/issues/49689 for an example.
+
+OSDSuperblock::maps
+-------------------
+
+The OSDSuperblock holds an epoch interval set that represents the OSDMaps
+that are stored by the OSD. Each OSDMap epoch range that was handled
+is added to the set.
+Once an osdmap is trimmed, it will be erased from the set.
+As a result, the set's lower bound represent the oldest map that is
+stored. While the upper bound represents the newest map.
+
+The ``interval_set`` data structure supports non-contiguous epoch intervals
+which may occur in "map gap" events. Before using this data structure,
+``oldest_map`` and ``newest_map`` epochs were stored in the OSDSuperblock.
+However, holding a single and contiguous epoch range imposed constraints which
+may have resulted in an OSDMap leak.
+
+See: https://tracker.ceph.com/issues/61962

From 7d8db4954c2fedb4db9f0ccc8a9d304f35807b07 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Wed, 13 Sep 2023 12:12:26 +0000
Subject: [PATCH 0047/2492] dokan: simple case insensitive emulation

While CephFS is case sensitive, Windows software commonly assume
that the filesystem is case insensitive.

In order to unblock certain use cases, a short term solution is to
simply normalize paths when creating or accessing files or
directories.

This change adds two ceph-dokan parameters:

* --case-insensitive: if set, paths are normalized
* --force-lowercase: normalized paths are converted to lowercase
                     instead of uppercase

This trivial implementation has some limitations:

* the original case is not preserved
  * we could later on use xattr to store the original name
* can't access existing files that have a different case
* handled at ceph-dokan level

The advantage is that it's simple, shouldn't impact performance
and doesn't require libcephfs or mds changes.

In the future, we may conider a more robust implementation
at the mds and/or libcephfs level.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/dokan/ceph_dokan.cc | 40 +++++++++++++++----
 src/dokan/ceph_dokan.h  |  8 ++++
 src/dokan/options.cc    |  9 +++++
 src/test/dokan/dokan.cc | 86 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 136 insertions(+), 7 deletions(-)

diff --git a/src/dokan/ceph_dokan.cc b/src/dokan/ceph_dokan.cc
index 9e115222cab2..6459ea261bf1 100644
--- a/src/dokan/ceph_dokan.cc
+++ b/src/dokan/ceph_dokan.cc
@@ -77,9 +77,26 @@ typedef struct {
 static_assert(sizeof(fd_context) <= 8,
               "fd_context exceeds DOKAN_FILE_INFO.Context size.");
 
-string get_path(LPCWSTR path_w) {
+string get_path(LPCWSTR path_w, bool normalize_case=true) {
   string path = to_string(path_w);
   replace(path.begin(), path.end(), '\\', '/');
+
+  if (normalize_case && !g_cfg->case_sensitive) {
+    if (g_cfg->convert_to_uppercase) {
+      std::transform(
+        path.begin(), path.end(), path.begin(),
+        [](unsigned char c){
+          return std::toupper(c);
+        });
+    } else {
+      std::transform(
+        path.begin(), path.end(), path.begin(),
+        [](unsigned char c){
+          return std::tolower(c);
+        });
+    }
+  }
+
   return path;
 }
 
@@ -543,6 +560,11 @@ static NTSTATUS WinCephFindFiles(
     return cephfs_errno_to_ntstatus_map(ret);
   }
 
+  // TODO: retrieve the original case (e.g. using xattr) if configured
+  // to do so.
+  // TODO: provide aliases when case insensitive mounts cause collisions.
+  // For example, when having test.txt and Test.txt, the latter becomes
+  // TEST~1.txt
   WIN32_FIND_DATAW findData;
   int count = 0;
   while (1) {
@@ -794,14 +816,18 @@ static NTSTATUS WinCephGetVolumeInformation(
 {
   g_cfg->win_vol_name.copy(VolumeNameBuffer, VolumeNameSize);
   *VolumeSerialNumber = g_cfg->win_vol_serial;
-
   *MaximumComponentLength = g_cfg->max_path_len;
 
-  *FileSystemFlags = FILE_CASE_SENSITIVE_SEARCH |
-            FILE_CASE_PRESERVED_NAMES |
-            FILE_SUPPORTS_REMOTE_STORAGE |
-            FILE_UNICODE_ON_DISK |
-            FILE_PERSISTENT_ACLS;
+  *FileSystemFlags =
+    FILE_SUPPORTS_REMOTE_STORAGE |
+    FILE_UNICODE_ON_DISK |
+    FILE_PERSISTENT_ACLS;
+
+  if (g_cfg->case_sensitive) {
+    *FileSystemFlags |=
+      FILE_CASE_SENSITIVE_SEARCH |
+      FILE_CASE_PRESERVED_NAMES;
+  }
 
   wcscpy(FileSystemNameBuffer, L"Ceph");
   return 0;
diff --git a/src/dokan/ceph_dokan.h b/src/dokan/ceph_dokan.h
index 5957d4dead11..fe48aa458143 100644
--- a/src/dokan/ceph_dokan.h
+++ b/src/dokan/ceph_dokan.h
@@ -36,6 +36,14 @@ struct Config {
   unsigned long max_path_len = 256;
   mode_t file_mode = 0755;
   mode_t dir_mode = 0755;
+
+  bool case_sensitive = true;
+  // Convert new file paths to upper case in case of case insensitive mounts.
+  // Visual Studio recommends normalizing to uppercase in order to avoid
+  // locale issues (CA1308).
+  bool convert_to_uppercase = true;
+  // TODO: consider adding an option to preserve the original case.
+  // It could be stored using an extended attribute.
 };
 
 extern Config *g_cfg;
diff --git a/src/dokan/options.cc b/src/dokan/options.cc
index 1ed90ef9d34d..705e1117ca99 100644
--- a/src/dokan/options.cc
+++ b/src/dokan/options.cc
@@ -45,6 +45,11 @@ Map options:
   --max-path-len              The value of the maximum path length. Default: 256.
   --file-mode                 The access mode to be used when creating files.
   --dir-mode                  The access mode to be used when creating directories.
+  --case-insensitive          Emulate a case insensitive filesystem by normalizing
+                              paths. The original case is NOT preserved. Existing
+                              paths with a different case cannot be accessed.
+  --force-lowercase           Use lowercase when normalizing paths. Uppercase is
+                              used by default.
 
 Unmap options:
   -l [ --mountpoint ] arg     mountpoint (path or drive letter) (e.g -l x).
@@ -196,6 +201,10 @@ int parse_args(
         *err_msg << "ceph-dokan: Invalid argument for operation-timeout";
         return -EINVAL;
       }
+    } else if (ceph_argparse_flag(args, i, "--case-insensitive", (char *)NULL)) {
+      cfg->case_sensitive = false;
+    } else if (ceph_argparse_flag(args, i, "--force-lowercase", (char *)NULL)) {
+      cfg->convert_to_uppercase = false;
     } else {
       ++i;
     }
diff --git a/src/test/dokan/dokan.cc b/src/test/dokan/dokan.cc
index 18f206985e8b..eaa26557fe88 100644
--- a/src/test/dokan/dokan.cc
+++ b/src/test/dokan/dokan.cc
@@ -39,6 +39,18 @@ std::string get_uuid() {
     return suffix.to_string();
 }
 
+std::string to_upper(std::string& in) {
+    std::string out = in;
+
+    std::transform(
+        out.begin(), out.end(), out.begin(),
+        [](unsigned char c){
+          return std::toupper(c);
+        });
+
+    return out;
+}
+
 bool move_eof(HANDLE handle, LARGE_INTEGER offset) {
 
     // Move file pointer to FILE_BEGIN + offset
@@ -162,6 +174,22 @@ void map_dokan_with_maxpath(
     }
 }
 
+void map_dokan_case_insensitive(SubProcess** mount, const char* mountpoint,
+                                bool force_lowercase=false) {
+    SubProcess* new_mount = new SubProcess("ceph-dokan");
+
+    new_mount->add_cmd_args("map", "--win-vol-name", "TestCeph",
+                            "--win-vol-serial", TEST_VOL_SERIAL,
+                            "-l", mountpoint, "--case-insensitive", NULL);
+    if (force_lowercase) {
+        new_mount->add_cmd_args("--force-lowercase", NULL);
+    }
+
+    *mount = new_mount;
+    ASSERT_EQ(new_mount->spawn(), 0);
+    ASSERT_EQ(wait_for_mount(mountpoint), 0);
+}
+
 void unmap_dokan(SubProcess* mount, const char* mountpoint) {
     std::string ret = run_cmd("ceph-dokan", "unmap", "-l",
                               mountpoint, (char*)NULL);
@@ -762,3 +790,61 @@ TEST_F(DokanTests, test_create_dispositions) {
     // clean-up
     ASSERT_TRUE(fs::remove(file_path));
 }
+
+TEST_F(DokanTests, test_case_sensitive) {
+    std::string test_dir = DEFAULT_MOUNTPOINT"test_dir" + get_uuid() + "\\";
+    std::string lower_file_path = test_dir + "file_" + get_uuid();
+    std::string upper_file_path = to_upper(lower_file_path);
+
+    ASSERT_TRUE(fs::create_directory(test_dir));
+    std::ofstream{lower_file_path};
+
+    ASSERT_TRUE(fs::exists(lower_file_path));
+    ASSERT_FALSE(fs::exists(upper_file_path));
+
+    // clean-up
+    fs::remove_all(test_dir);
+}
+
+void test_case_insensitive(bool force_lowercase) {
+    std::string mountpoint = "Q:\\";
+    std::string test_dir = mountpoint + "test_dir" + get_uuid() + "/";
+    std::string file_name = "file_" + get_uuid();
+    std::string lower_file_path = test_dir + file_name;
+    std::string upper_file_path = to_upper(lower_file_path);
+
+    SubProcess* mount = nullptr;
+    map_dokan_case_insensitive(&mount, mountpoint.c_str(), force_lowercase);
+
+    ASSERT_TRUE(fs::create_directory(test_dir));
+    std::ofstream{upper_file_path};
+
+    ASSERT_TRUE(fs::exists(lower_file_path));
+    ASSERT_TRUE(fs::exists(upper_file_path));
+
+    std::vector<std::string> paths;
+    for (const auto & entry : fs::recursive_directory_iterator(test_dir)) {
+        paths.push_back(entry.path().filename().generic_string());
+    }
+
+    bool found_lowercase = std::find(
+        begin(paths), end(paths), file_name) != end(paths);
+    bool found_uppercase = std::find(
+        begin(paths), end(paths), to_upper(file_name)) != end(paths);
+
+    ASSERT_EQ(found_lowercase, force_lowercase);
+    ASSERT_NE(found_uppercase, force_lowercase);
+
+    // clean-up
+    fs::remove_all(test_dir);
+
+    unmap_dokan(mount, mountpoint.c_str());
+}
+
+TEST_F(DokanTests, test_case_insensitive_force_lower) {
+    test_case_insensitive(true);
+}
+
+TEST_F(DokanTests, test_case_insensitive_force_upper) {
+   test_case_insensitive(false);
+}

From 0357ab2ad8b398641b1e03727c9de6241d72ad83 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Thu, 14 Sep 2023 11:08:36 +0000
Subject: [PATCH 0048/2492] test/dokan: avoid unnecessary wait

One of the dokan tests expects some mount operations to fail.
The issue is that it polls the mount location for 10s, which
is unnecessary.

We'll update the test to check the mount process exit code
instead. This basically reduces the dokan test duration by
about 20s.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/test/dokan/dokan.cc | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/test/dokan/dokan.cc b/src/test/dokan/dokan.cc
index eaa26557fe88..baef44a49b65 100644
--- a/src/test/dokan/dokan.cc
+++ b/src/test/dokan/dokan.cc
@@ -157,7 +157,18 @@ void map_dokan_with_maxpath(
     const char* mountpoint,
     uint64_t max_path_len)
 {
-    SubProcess* new_mount = new SubProcess("ceph-dokan");
+    SubProcess* new_mount = nullptr;
+
+    bool expect_failure = max_path_len < 256 || max_path_len > 4096;
+    if (expect_failure) {
+        new_mount = new SubProcessTimed(
+            "ceph-dokan",
+            SubProcess::CLOSE, SubProcess::CLOSE, SubProcess::CLOSE,
+            MOUNT_POLL_ATTEMPT * MOUNT_POLL_INTERVAL_MS / 1000);
+    } else {
+        new_mount = new SubProcess("ceph-dokan");
+    }
+
     new_mount->add_cmd_args("map", "--debug", "--dokan-stderr",
                             "--win-vol-name", "TestCeph",
                             "--win-vol-serial", TEST_VOL_SERIAL,
@@ -167,10 +178,10 @@ void map_dokan_with_maxpath(
 
     *mount = new_mount;
     ASSERT_EQ(new_mount->spawn(), 0);
-    if (256 <= max_path_len && max_path_len <= 4096) {
-        ASSERT_EQ(wait_for_mount(mountpoint), 0);
+    if (expect_failure) {
+        ASSERT_NE(0, new_mount->join());
     } else {
-        ASSERT_NE(wait_for_mount(mountpoint), 0);
+        ASSERT_EQ(wait_for_mount(mountpoint), 0);
     }
 }
 

From 723c1b7c9e773a68b6c2586155092ca95ab4f3e7 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Fri, 15 Sep 2023 08:41:35 +0800
Subject: [PATCH 0049/2492] mds: record the internal client request and receive
 client reply

This will be used to avoid possible multiple reintegration issue
later.

Fixes: https://tracker.ceph.com/issues/62702
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/MDSMetaRequest.h | 33 +++++++++++++++++++++++++++++++++
 src/mds/MDSRank.cc       |  2 ++
 src/mds/MDSRank.h        |  3 +++
 src/mds/Server.cc        | 30 ++++++++++++++++++++++++++++++
 src/mds/Server.h         |  1 +
 5 files changed, 69 insertions(+)
 create mode 100644 src/mds/MDSMetaRequest.h

diff --git a/src/mds/MDSMetaRequest.h b/src/mds/MDSMetaRequest.h
new file mode 100644
index 000000000000..ad4720410686
--- /dev/null
+++ b/src/mds/MDSMetaRequest.h
@@ -0,0 +1,33 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MDS_META_REQUEST_H
+#define CEPH_MDS_META_REQUEST_H
+
+#include "include/types.h"
+
+struct MDSMetaRequest {
+private:
+  int op;
+  ceph_tid_t tid;
+public:
+  explicit MDSMetaRequest(int o, ceph_tid_t t) :
+    op(o), tid(t) { }
+  virtual ~MDSMetaRequest() { }
+
+  int get_op() { return op; }
+  ceph_tid_t get_tid() { return tid; }
+};
+
+#endif // !CEPH_MDS_META_REQUEST_H
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index d7d583f53274..fb78aa3c28ee 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1191,6 +1191,7 @@ bool MDSRank::is_valid_message(const cref_t<Message> &m) {
       type == CEPH_MSG_CLIENT_RECONNECT ||
       type == CEPH_MSG_CLIENT_RECLAIM ||
       type == CEPH_MSG_CLIENT_REQUEST ||
+      type == CEPH_MSG_CLIENT_REPLY ||
       type == MSG_MDS_PEER_REQUEST ||
       type == MSG_MDS_HEARTBEAT ||
       type == MSG_MDS_TABLE_REQUEST ||
@@ -1244,6 +1245,7 @@ void MDSRank::handle_message(const cref_t<Message> &m)
       ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_CLIENT);
       // fall-thru
     case CEPH_MSG_CLIENT_REQUEST:
+    case CEPH_MSG_CLIENT_REPLY:
       server->dispatch(m);
       break;
     case MSG_MDS_PEER_REQUEST:
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index d01a5894df47..48043df4baf6 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -43,6 +43,7 @@
 #include "Server.h"
 #include "MetricsHandler.h"
 #include "osdc/Journaler.h"
+#include "MDSMetaRequest.h"
 
 // Full .h import instead of forward declaration for PerfCounter, for the
 // benefit of those including this header and using MDSRank::logger
@@ -423,6 +424,8 @@ class MDSRank {
     PerfCounters *logger = nullptr, *mlogger = nullptr;
     OpTracker op_tracker;
 
+    std::map<ceph_tid_t, std::unique_ptr<MDSMetaRequest>> internal_client_requests;
+
     // The last different state I held before current
     MDSMap::DaemonState last_state = MDSMap::STATE_BOOT;
     // The state assigned to me by the MDSMap
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index bb5f0a30ac7c..780eafc43644 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -31,6 +31,7 @@
 #include "Mutation.h"
 #include "MetricsHandler.h"
 #include "cephfs_features.h"
+#include "MDSContext.h"
 
 #include "msg/Messenger.h"
 
@@ -360,6 +361,9 @@ void Server::dispatch(const cref_t<Message> &m)
   case CEPH_MSG_CLIENT_REQUEST:
     handle_client_request(ref_cast<MClientRequest>(m));
     return;
+  case CEPH_MSG_CLIENT_REPLY:
+    handle_client_reply(ref_cast<MClientReply>(m));
+    return;
   case CEPH_MSG_CLIENT_RECLAIM:
     handle_client_reclaim(ref_cast<MClientReclaim>(m));
     return;
@@ -2319,6 +2323,10 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
     mds->send_message_client(reply, session);
   }
 
+  if (client_inst.name.is_mds() && reply->get_op() == CEPH_MDS_OP_RENAME) {
+    mds->send_message(reply, mdr->client_request->get_connection());
+  }
+
   if (req->is_queued_for_replay() &&
       (mdr->has_completed || reply->get_result() < 0)) {
     if (reply->get_result() < 0) {
@@ -2551,6 +2559,28 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
   return;
 }
 
+void Server::handle_client_reply(const cref_t<MClientReply> &reply)
+{
+  dout(4) << "handle_client_reply " << *reply << dendl;
+
+  ceph_assert(reply->is_safe());
+  ceph_tid_t tid = reply->get_tid();
+
+  if (mds->internal_client_requests.count(tid) == 0) {
+    dout(1) << " no pending request on tid " << tid << dendl;
+    return;
+  }
+
+  switch (reply->get_op()) {
+  case CEPH_MDS_OP_RENAME:
+    break;
+  default:
+    dout(5) << " unknown client op " << reply->get_op() << dendl;
+  }
+
+  mds->internal_client_requests.erase(tid);
+}
+
 void Server::handle_osd_map()
 {
   /* Note that we check the OSDMAP_FULL flag directly rather than
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 61096a5b68a4..6b2f9c188f5a 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -159,6 +159,7 @@ class Server {
 
   // -- requests --
   void handle_client_request(const cref_t<MClientRequest> &m);
+  void handle_client_reply(const cref_t<MClientReply> &m);
 
   void journal_and_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn,
 			 LogEvent *le, MDSLogContextBase *fin);

From 53d9e657e42bdf2440cd93a8273eb3ce79d8bd33 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Fri, 15 Sep 2023 08:41:35 +0800
Subject: [PATCH 0050/2492] mds: fix issuing redundant
 reintegrate/migrate_stray requests

Just in case a CInode's nlink is 1, and then a unlink request comes
and then early replies and submits to the MDLogs, but just before
the MDlogs are flushed a link request comes, and the link request
also succeeds and early replies to client.

Later when the unlink/link requests' MDLog events are flushed and
the callbacks are called, which will fire a stray denty reintegration.
But it will pick the new dentry, which is from the link's request
and is a remote dentry, to do the reintegration. While in the
'rename' code when traversing the path it will trigger to call the
'dn->link_remote()', which later will fire a new stray dentry
reintegration.

The problem is if the first 'rename' request is retried several
times, and in each time it will fire a new reintegration, which
makes no sense and maybe blocked for a very long time dues to some
reasons and then will be reported as slow request warning.

Fixes: https://tracker.ceph.com/issues/62702
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/CDentry.h       |  2 ++
 src/mds/StrayManager.cc | 30 ++++++++++++++++++++++++++----
 src/mds/StrayManager.h  | 17 ++++++++++++++++-
 3 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h
index 4dca5816ae6f..1c2b6f892cec 100644
--- a/src/mds/CDentry.h
+++ b/src/mds/CDentry.h
@@ -376,6 +376,8 @@ class CDentry : public MDSCacheObject, public LRUObject, public Counter<CDentry>
   mempool::mds_co::map<client_t,ClientLease*> client_lease_map;
   std::map<int, std::unique_ptr<BatchOp>> batch_ops;
 
+  ceph_tid_t reintegration_reqid = 0;
+
 
 protected:
   friend class Migrator;
diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc
index 325209da6e0e..e9ec153d3fdb 100644
--- a/src/mds/StrayManager.cc
+++ b/src/mds/StrayManager.cc
@@ -673,24 +673,41 @@ void StrayManager::reintegrate_stray(CDentry *straydn, CDentry *rdn)
 {
   dout(10) << __func__ << " " << *straydn << " to " << *rdn << dendl;
 
+  if (straydn->reintegration_reqid) {
+    dout(20) << __func__ << ": stray dentry " << *straydn
+             << " is already under reintegrating" << dendl;
+    return;
+  }
+
   logger->inc(l_mdc_strays_reintegrated);
-  
+
   // rename it to remote linkage .
   filepath src(straydn->get_name(), straydn->get_dir()->ino());
   filepath dst(rdn->get_name(), rdn->get_dir()->ino());
 
+  ceph_tid_t tid = mds->issue_tid();
+
   auto req = make_message<MClientRequest>(CEPH_MDS_OP_RENAME);
   req->set_filepath(dst);
   req->set_filepath2(src);
-  req->set_tid(mds->issue_tid());
+  req->set_tid(tid);
+
+  auto ptr = std::make_unique<StrayEvalRequest>(CEPH_MDS_OP_RENAME, tid, straydn);
+  mds->internal_client_requests.emplace(tid, std::move(ptr));
 
   mds->send_message_mds(req, rdn->authority().first);
 }
- 
+
 void StrayManager::migrate_stray(CDentry *dn, mds_rank_t to)
 {
   dout(10) << __func__ << " " << *dn << " to mds." << to << dendl;
 
+  if (dn->reintegration_reqid) {
+    dout(20) << __func__ << ": stray dentry " << *dn
+             << " is already under migrating" << dendl;
+    return;
+  }
+
   logger->inc(l_mdc_strays_migrated);
 
   // rename it to another mds.
@@ -700,10 +717,15 @@ void StrayManager::migrate_stray(CDentry *dn, mds_rank_t to)
   filepath src(dn->get_name(), dirino);
   filepath dst(dn->get_name(), MDS_INO_STRAY(to, MDS_INO_STRAY_INDEX(dirino)));
 
+  ceph_tid_t tid = mds->issue_tid();
+
   auto req = make_message<MClientRequest>(CEPH_MDS_OP_RENAME);
   req->set_filepath(dst);
   req->set_filepath2(src);
-  req->set_tid(mds->issue_tid());
+  req->set_tid(tid);
+
+  auto ptr = std::make_unique<StrayEvalRequest>(CEPH_MDS_OP_RENAME, tid, dn);
+  mds->internal_client_requests.emplace(tid, std::move(ptr));
 
   mds->send_message_mds(req, to);
 }
diff --git a/src/mds/StrayManager.h b/src/mds/StrayManager.h
index 86b6941a5131..874fbbb9a8dc 100644
--- a/src/mds/StrayManager.h
+++ b/src/mds/StrayManager.h
@@ -19,15 +19,30 @@
 #include <list>
 #include "Mutation.h"
 #include "PurgeQueue.h"
+#include "MDSMetaRequest.h"
+#include "CDentry.h"
 
 class MDSRank;
 class CInode;
-class CDentry;
 
 class StrayManager
 {
   // My public interface is for consumption by MDCache
 public:
+  struct StrayEvalRequest : public MDSMetaRequest {
+    CDentry *dentry;
+  public:
+    explicit StrayEvalRequest(int o, ceph_tid_t t, CDentry *d) :
+      MDSMetaRequest(o, t), dentry(d) {
+      dentry->get(CDentry::PIN_PURGING);
+      dentry->reintegration_reqid = t;
+    }
+    ~StrayEvalRequest() {
+      dentry->reintegration_reqid = 0;
+      dentry->put(CDentry::PIN_PURGING);
+    }
+  };
+
   explicit StrayManager(MDSRank *mds, PurgeQueue &purge_queue_);
   void set_logger(PerfCounters *l) {logger = l;}
   void activate();

From dbb4daff404c5d2da32c33f4e852e84a257c0b8d Mon Sep 17 00:00:00 2001
From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Tue, 12 Sep 2023 17:45:05 +0530
Subject: [PATCH 0051/2492] rbd-nbd: fix stuck with disable request

Problem:
-------
Trying to disable any feature on an rbd image mapped with nbd leads to stuck
in rbd-nbd.

The rbd-nbd registers a watcher callback to detect image resize in
NBDWatchCtx::handle_notify(). The handle_notify calls image info method, which
calls refresh_if_required and it got stuck there.

It is getting stuck in ImageState::refresh_if_required() because
DisableFeaturesRequest issues update notifications while still holding onto
the exclusive lock with everything that has to do with it blocked.

Solution:
--------
Set only notify flag as part of NBDWatchCtx::handle_notify() and handle
the resize detection part as part of a different thread.

Fixes: https://tracker.ceph.com/issues/58740
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---
 qa/workunits/rbd/rbd-nbd.sh  |  10 ++++
 src/tools/rbd_nbd/rbd-nbd.cc | 112 +++++++++++++++++++++++++----------
 2 files changed, 91 insertions(+), 31 deletions(-)

diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh
index 122df3d6f35a..bc89e9be5a18 100755
--- a/qa/workunits/rbd/rbd-nbd.sh
+++ b/qa/workunits/rbd/rbd-nbd.sh
@@ -472,6 +472,16 @@ DEV=
 rbd feature disable ${POOL}/${IMAGE} journaling
 rbd config image rm ${POOL}/${IMAGE} rbd_discard_granularity_bytes
 
+# test that disabling a feature so that the op is proxied to rbd-nbd
+# (arranged here by blkdiscard before "rbd feature disable") doesn't hang
+DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
+get_pid ${POOL}
+rbd feature enable ${POOL}/${IMAGE} journaling
+_sudo blkdiscard --offset 0 --length 4096 ${DEV}
+rbd feature disable ${POOL}/${IMAGE} journaling
+unmap_device ${DEV} ${PID}
+DEV=
+
 # test that rbd_op_threads setting takes effect
 EXPECTED=`ceph-conf --show-config-value librados_thread_count`
 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc
index 3130e8bc750e..e348bd8fe431 100644
--- a/src/tools/rbd_nbd/rbd-nbd.cc
+++ b/src/tools/rbd_nbd/rbd-nbd.cc
@@ -738,7 +738,67 @@ class NBDWatchCtx : public librbd::UpdateWatchCtx
   bool use_netlink;
   librados::IoCtx &io_ctx;
   librbd::Image &image;
-  unsigned long size;
+  uint64_t size;
+  std::thread handle_notify_thread;
+  ceph::condition_variable cond;
+  ceph::mutex lock = ceph::make_mutex("NBDWatchCtx::Locker");
+  bool notify = false;
+  bool terminated = false;
+
+  bool wait_notify() {
+    dout(10) << __func__ << dendl;
+
+    std::unique_lock locker{lock};
+    cond.wait(locker, [this] { return notify || terminated; });
+
+    if (terminated) {
+      return false;
+    }
+
+    dout(10) << __func__ << ": got notify request" << dendl;
+    notify = false;
+    return true;
+  }
+
+  void handle_notify_entry() {
+    dout(10) << __func__ << dendl;
+
+    while (wait_notify()) {
+      uint64_t new_size;
+      int ret = image.size(&new_size);
+      if (ret < 0) {
+        derr << "getting image size failed: " << cpp_strerror(ret) << dendl;
+        continue;
+      }
+      if (new_size == size) {
+        continue;
+      }
+      dout(5) << "resize detected" << dendl;
+      if (ioctl(fd, BLKFLSBUF, NULL) < 0) {
+        derr << "invalidate page cache failed: " << cpp_strerror(errno)
+             << dendl;
+      }
+      if (use_netlink) {
+        ret = netlink_resize(nbd_index, new_size);
+      } else {
+        ret = ioctl(fd, NBD_SET_SIZE, new_size);
+        if (ret < 0) {
+          derr << "resize failed: " << cpp_strerror(errno) << dendl;
+        }
+      }
+      if (!ret) {
+        size = new_size;
+      }
+      if (ioctl(fd, BLKRRPART, NULL) < 0) {
+        derr << "rescan of partition table failed: " << cpp_strerror(errno)
+             << dendl;
+      }
+      if (image.invalidate_cache() < 0) {
+        derr << "invalidate rbd cache failed" << dendl;
+      }
+    }
+  }
+
 public:
   NBDWatchCtx(int _fd,
               int _nbd_index,
@@ -752,41 +812,31 @@ class NBDWatchCtx : public librbd::UpdateWatchCtx
     , io_ctx(_io_ctx)
     , image(_image)
     , size(_size)
-  { }
+  {
+    handle_notify_thread = make_named_thread("rbd_handle_notify",
+                                             &NBDWatchCtx::handle_notify_entry,
+                                             this);
+  }
 
-  ~NBDWatchCtx() override {}
+  ~NBDWatchCtx() override
+  {
+    dout(10) << __func__ << ": terminating" << dendl;
+    std::unique_lock locker{lock};
+    terminated = true;
+    cond.notify_all();
+    locker.unlock();
+
+    handle_notify_thread.join();
+    dout(10) << __func__ << ": finish" << dendl;
+  }
 
   void handle_notify() override
   {
-    librbd::image_info_t info;
-    if (image.stat(info, sizeof(info)) == 0) {
-      unsigned long new_size = info.size;
-      int ret;
-
-      if (new_size != size) {
-        dout(5) << "resize detected" << dendl;
-        if (ioctl(fd, BLKFLSBUF, NULL) < 0)
-          derr << "invalidate page cache failed: " << cpp_strerror(errno)
-               << dendl;
-	if (use_netlink) {
-	  ret = netlink_resize(nbd_index, new_size);
-	} else {
-          ret = ioctl(fd, NBD_SET_SIZE, new_size);
-          if (ret < 0)
-            derr << "resize failed: " << cpp_strerror(errno) << dendl;
-	}
-
-        if (!ret)
-          size = new_size;
+    dout(10) << __func__ << dendl;
 
-        if (ioctl(fd, BLKRRPART, NULL) < 0) {
-          derr << "rescan of partition table failed: " << cpp_strerror(errno)
-               << dendl;
-        }
-        if (image.invalidate_cache() < 0)
-          derr << "invalidate rbd cache failed" << dendl;
-      }
-    }
+    std::unique_lock locker{lock};
+    notify = true;
+    cond.notify_all();
   }
 };
 

From b30c35019d0128a0cf840b43d3059392effbb52c Mon Sep 17 00:00:00 2001
From: Patty8122 <divyapattisapu@uchicago.edu>
Date: Fri, 22 Sep 2023 16:33:38 -0500
Subject: [PATCH 0052/2492] doc/man/8/ceph-bluestore-tool.rst: Added valid
 options for fsck --deep

Signed-off-by: Patty8122 <divyapattisapu@uchicago.edu>
---
 doc/man/8/ceph-bluestore-tool.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/man/8/ceph-bluestore-tool.rst b/doc/man/8/ceph-bluestore-tool.rst
index f6c88da09b24..634d3a8bb6eb 100644
--- a/doc/man/8/ceph-bluestore-tool.rst
+++ b/doc/man/8/ceph-bluestore-tool.rst
@@ -44,7 +44,7 @@ Commands
 
    show help
 
-:command:`fsck` [ --deep ]
+:command:`fsck` [ --deep ] *(on|off) or (yes|no) or (1|0) or (true|false)*
 
    run consistency check on BlueStore metadata.  If *--deep* is specified, also read all object data and verify checksums.
 

From 9b9eff16a885284c106a368970479b54eedb9c19 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 19 Sep 2023 13:45:16 +0800
Subject: [PATCH 0053/2492] crimson/osd/replicated_backend: don't put pg's
 metadata updates in repops

Fixes: https://tracker.ceph.com/issues/62857
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg.cc                 |  4 +-
 src/crimson/osd/pg_backend.cc         |  4 +-
 src/crimson/osd/pg_backend.h          |  1 +
 src/crimson/osd/replicated_backend.cc | 65 ++++++++++++++++-----------
 src/crimson/osd/replicated_backend.h  |  3 ++
 5 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 7cf3b158c89c..2bd1bfc5bf31 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -115,6 +115,7 @@ PG::PG(
 	pgid.pgid,
 	pg_shard,
 	pool,
+        *this,
 	coll_ref,
 	shard_services,
 	profile,
@@ -761,8 +762,7 @@ PG::submit_transaction(
   ceph_assert(!has_reset_since(osd_op_p.at_version.epoch));
 
   peering_state.pre_submit_op(obc->obs.oi.soid, log_entries, osd_op_p.at_version);
-  peering_state.append_log_with_trim_to_updated(std::move(log_entries), osd_op_p.at_version,
-						txn, true, false);
+  peering_state.update_trim_to();
 
   auto [submitted, all_completed] = backend->mutate_object(
       peering_state.get_acting_recovery_backfill(),
diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index 02acb9a55d3f..06bcf7bbd036 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -23,6 +23,7 @@
 #include "crimson/os/futurized_store.h"
 #include "crimson/osd/osd_operation.h"
 #include "crimson/osd/object_context_loader.h"
+#include "crimson/osd/pg.h"
 #include "replicated_backend.h"
 #include "replicated_recovery_backend.h"
 #include "ec_backend.h"
@@ -43,6 +44,7 @@ std::unique_ptr<PGBackend>
 PGBackend::create(pg_t pgid,
 		  const pg_shard_t pg_shard,
 		  const pg_pool_t& pool,
+		  crimson::osd::PG& pg,
 		  crimson::os::CollectionRef coll,
 		  crimson::osd::ShardServices& shard_services,
 		  const ec_profile_t& ec_profile,
@@ -50,7 +52,7 @@ PGBackend::create(pg_t pgid,
 {
   switch (pool.type) {
   case pg_pool_t::TYPE_REPLICATED:
-    return std::make_unique<ReplicatedBackend>(pgid, pg_shard,
+    return std::make_unique<ReplicatedBackend>(pgid, pg_shard, pg,
 					       coll, shard_services,
 					       dpp);
   case pg_pool_t::TYPE_ERASURE:
diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h
index fbad37d4c71c..d681726576ff 100644
--- a/src/crimson/osd/pg_backend.h
+++ b/src/crimson/osd/pg_backend.h
@@ -70,6 +70,7 @@ class PGBackend
   static std::unique_ptr<PGBackend> create(pg_t pgid,
 					   const pg_shard_t pg_shard,
 					   const pg_pool_t& pool,
+					   crimson::osd::PG &pg,
 					   crimson::os::CollectionRef coll,
 					   crimson::osd::ShardServices& shard_services,
 					   const ec_profile_t& ec_profile,
diff --git a/src/crimson/osd/replicated_backend.cc b/src/crimson/osd/replicated_backend.cc
index 0ff4ad5730f5..caa86d46a5b1 100644
--- a/src/crimson/osd/replicated_backend.cc
+++ b/src/crimson/osd/replicated_backend.cc
@@ -8,6 +8,7 @@
 #include "crimson/common/exception.h"
 #include "crimson/common/log.h"
 #include "crimson/os/futurized_store.h"
+#include "crimson/osd/pg.h"
 #include "crimson/osd/shard_services.h"
 #include "osd/PeeringState.h"
 
@@ -15,12 +16,14 @@ SET_SUBSYS(osd);
 
 ReplicatedBackend::ReplicatedBackend(pg_t pgid,
                                      pg_shard_t whoami,
+				     crimson::osd::PG& pg,
                                      ReplicatedBackend::CollectionRef coll,
                                      crimson::osd::ShardServices& shard_services,
 				     DoutPrefixProvider &dpp)
   : PGBackend{whoami.shard, coll, shard_services, dpp},
     pgid{pgid},
-    whoami{whoami}
+    whoami{whoami},
+    pg(pg)
 {}
 
 ReplicatedBackend::ll_read_ierrorator::future<ceph::bufferlist>
@@ -41,6 +44,7 @@ ReplicatedBackend::_submit_transaction(std::set<pg_shard_t>&& pg_shards,
 				       std::vector<pg_log_entry_t>&& log_entries)
 {
   LOG_PREFIX(ReplicatedBackend::_submit_transaction);
+  DEBUGDPP("object {}, {}", dpp, hoid);
 
   const ceph_tid_t tid = shard_services.get_tid();
   auto pending_txn =
@@ -48,29 +52,6 @@ ReplicatedBackend::_submit_transaction(std::set<pg_shard_t>&& pg_shards,
   bufferlist encoded_txn;
   encode(txn, encoded_txn);
 
-  DEBUGDPP("object {}", dpp, hoid);
-  auto all_completed = interruptor::make_interruptible(
-    shard_services.get_store().do_transaction(coll, std::move(txn))
-  ).then_interruptible([FNAME, this,
-			peers=pending_txn->second.weak_from_this()] {
-    if (!peers) {
-      // for now, only actingset_changed can cause peers
-      // to be nullptr
-      ERRORDPP("peers is null, this should be impossible", dpp);
-      assert(0 == "impossible");
-    }
-    if (--peers->pending == 0) {
-      peers->all_committed.set_value();
-      peers->all_committed = {};
-      return seastar::now();
-    }
-    return peers->all_committed.get_shared_future();
-  }).then_interruptible([pending_txn, this] {
-    auto acked_peers = std::move(pending_txn->second.acked_peers);
-    pending_trans.erase(pending_txn);
-    return seastar::make_ready_future<crimson::osd::acked_peers_t>(std::move(acked_peers));
-  });
-
   auto sends = std::make_unique<std::vector<seastar::future<>>>();
   for (auto pg_shard : pg_shards) {
     if (pg_shard != whoami) {
@@ -91,9 +72,43 @@ ReplicatedBackend::_submit_transaction(std::set<pg_shard_t>&& pg_shards,
       m->min_last_complete_ondisk = osd_op_p.min_last_complete_ondisk;
       m->set_rollback_to(osd_op_p.at_version);
       // TODO: set more stuff. e.g., pg_states
-      sends->emplace_back(shard_services.send_to_osd(pg_shard.osd, std::move(m), map_epoch));
+      sends->emplace_back(
+	shard_services.send_to_osd(
+	  pg_shard.osd, std::move(m), map_epoch));
     }
   }
+
+  pg.log_operation(
+    std::move(log_entries),
+    osd_op_p.pg_trim_to,
+    osd_op_p.at_version,
+    osd_op_p.min_last_complete_ondisk,
+    true,
+    txn,
+    false);
+
+  auto all_completed = interruptor::make_interruptible(
+    shard_services.get_store().do_transaction(coll, std::move(txn))
+  ).then_interruptible([FNAME, this,
+			peers=pending_txn->second.weak_from_this()] {
+    if (!peers) {
+      // for now, only actingset_changed can cause peers
+      // to be nullptr
+      ERRORDPP("peers is null, this should be impossible", dpp);
+      assert(0 == "impossible");
+    }
+    if (--peers->pending == 0) {
+      peers->all_committed.set_value();
+      peers->all_committed = {};
+      return seastar::now();
+    }
+    return peers->all_committed.get_shared_future();
+  }).then_interruptible([pending_txn, this] {
+    auto acked_peers = std::move(pending_txn->second.acked_peers);
+    pending_trans.erase(pending_txn);
+    return seastar::make_ready_future<crimson::osd::acked_peers_t>(std::move(acked_peers));
+  });
+
   auto sends_complete = seastar::when_all_succeed(
     sends->begin(), sends->end()
   ).finally([sends=std::move(sends)] {});
diff --git a/src/crimson/osd/replicated_backend.h b/src/crimson/osd/replicated_backend.h
index f789a35eae69..78366060d894 100644
--- a/src/crimson/osd/replicated_backend.h
+++ b/src/crimson/osd/replicated_backend.h
@@ -14,12 +14,14 @@
 
 namespace crimson::osd {
   class ShardServices;
+  class PG;
 }
 
 class ReplicatedBackend : public PGBackend
 {
 public:
   ReplicatedBackend(pg_t pgid, pg_shard_t whoami,
+		    crimson::osd::PG& pg,
 		    CollectionRef coll,
 		    crimson::osd::ShardServices& shard_services,
 		    DoutPrefixProvider &dpp);
@@ -55,6 +57,7 @@ class ReplicatedBackend : public PGBackend
   };
   using pending_transactions_t = std::map<ceph_tid_t, pending_on_t>;
   pending_transactions_t pending_trans;
+  crimson::osd::PG& pg;
 
   seastar::future<> request_committed(
     const osd_reqid_t& reqid, const eversion_t& at_version) final;

From fcce984a74f0b34deea491d5b91cc6a1254f73e1 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Mon, 25 Sep 2023 17:58:53 +0300
Subject: [PATCH 0054/2492] osd/PeeringState: get rid off costly and redundant
 count() calls.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/osd/PeeringState.cc | 47 +++++++++++++++++++++++------------------
 src/osd/PeeringState.h  | 10 +++++----
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 5cd78afdeec8..2c41b7b71189 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -2749,8 +2749,9 @@ void PeeringState::activate(
 	 ++i) {
       if (*i == pg_whoami) continue;
       pg_shard_t peer = *i;
-      ceph_assert(peer_info.count(peer));
-      pg_info_t& pi = peer_info[peer];
+      auto pi_it = peer_info.find(peer);
+      ceph_assert(pi_it != peer_info.end());
+      pg_info_t& pi = pi_it->second;
 
       psdout(10) << "activate peer osd." << peer << " " << pi << dendl;
 
@@ -2759,8 +2760,9 @@ void PeeringState::activate(
       #else
       MRef<MOSDPGLog> m;
       #endif
-      ceph_assert(peer_missing.count(peer));
-      pg_missing_t& pm = peer_missing[peer];
+      auto pm_it = peer_missing.find(peer);
+      ceph_assert(pm_it != peer_missing.end());
+      pg_missing_t& pm = pm_it->second;
 
       bool needs_past_intervals = pi.dne();
 
@@ -2927,21 +2929,24 @@ void PeeringState::activate(
 	     ++i) {
 	  if (*i == pg_whoami) continue;
 	  psdout(10) << ": adding " << *i << " as a source" << dendl;
-	  ceph_assert(peer_missing.count(*i));
-	  ceph_assert(peer_info.count(*i));
+	  auto pi_it = peer_info.find(*i);
+	  ceph_assert(pi_it != peer_info.end());
+	  auto pm_it = peer_missing.find(*i);
+	  ceph_assert(pm_it != peer_missing.end());
 	  missing_loc.add_source_info(
 	    *i,
-	    peer_info[*i],
-	    peer_missing[*i],
+	    pi_it->second,
+	    pm_it->second,
             ctx.handle);
         }
       }
       for (auto i = peer_missing.begin(); i != peer_missing.end(); ++i) {
 	if (is_acting_recovery_backfill(i->first))
 	  continue;
-	ceph_assert(peer_info.count(i->first));
+	auto pi_it = peer_info.find(i->first);
+	ceph_assert(pi_it != peer_info.end());
 	search_for_missing(
-	  peer_info[i->first],
+	  pi_it->second,
 	  i->second,
 	  i->first,
 	  ctx);
@@ -3640,8 +3645,9 @@ void PeeringState::update_calc_stats()
       if (is_backfill_target(peer.first)) {
         missing = std::max((int64_t)0, num_objects - peer_num_objects);
       } else {
-        if (peer_missing.count(peer.first)) {
-          missing = peer_missing[peer.first].num_missing();
+	auto pm_it = peer_missing.find(peer.first);
+        if (pm_it != peer_missing.end()) {
+          missing = pm_it->second.num_missing();
         } else {
           psdout(20) << "no peer_missing found for "
 		     << peer.first << dendl;
@@ -4094,12 +4100,14 @@ void PeeringState::merge_new_log_entries(
        ++i) {
     pg_shard_t peer(*i);
     if (peer == pg_whoami) continue;
-    ceph_assert(peer_missing.count(peer));
-    ceph_assert(peer_info.count(peer));
-    pg_missing_t& pmissing(peer_missing[peer]);
+    auto pm_it = peer_missing.find(peer);
+    ceph_assert(pm_it != peer_missing.end());
+    auto pi_it = peer_info.find(peer);
+    ceph_assert(pi_it != peer_info.end());
+    pg_missing_t& pmissing(pm_it->second);
     psdout(20) << "peer_missing for " << peer
 	       << " = " << pmissing << dendl;
-    pg_info_t& pinfo(peer_info[peer]);
+    pg_info_t& pinfo = pi_it->second;
     bool invalidate_stats = PGLog::append_log_entries_update_missing(
       pinfo.last_backfill,
       entries,
@@ -6133,10 +6141,9 @@ boost::statechart::result PeeringState::Active::react(const MInfoRec& infoevt)
   // may be telling us they have activated (and committed) but we can't
   // share that until _everyone_ does the same.
   if (ps->is_acting_recovery_backfill(infoevt.from) &&
-      ps->peer_activated.count(infoevt.from) == 0) {
+      ps->peer_activated.insert(infoevt.from).second) {
     psdout(10) << " peer osd." << infoevt.from
 	       << " activated and committed" << dendl;
-    ps->peer_activated.insert(infoevt.from);
     ps->blocked_by.erase(infoevt.from.shard);
     pl->publish_stats_to_osd();
     if (ps->peer_activated.size() == ps->acting_recovery_backfill.size()) {
@@ -6220,8 +6227,8 @@ boost::statechart::result PeeringState::Active::react(
   const ActivateCommitted &evt)
 {
   DECLARE_LOCALS;
-  ceph_assert(!ps->peer_activated.count(ps->pg_whoami));
-  ps->peer_activated.insert(ps->pg_whoami);
+  auto p = ps->peer_activated.insert(ps->pg_whoami);
+  ceph_assert(p.second);
   psdout(10) << "_activate_committed " << evt.epoch
 	     << " peer_activated now " << ps->peer_activated
 	     << " last_interval_started "
diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index cf70fa1d11bc..b9f8401b05b6 100644
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -2333,13 +2333,15 @@ class PeeringState : public MissingLoc::MappingInfo {
     if (peer == pg_whoami) {
       return pg_log.get_missing();
     } else {
-      assert(peer_missing.count(peer));
-      return peer_missing.find(peer)->second;
+      auto it = peer_missing.find(peer);
+      assert(it != peer_missing.end());
+      return it->second;
     }
   }
   const pg_info_t&get_peer_info(pg_shard_t peer) const {
-    assert(peer_info.count(peer));
-    return peer_info.find(peer)->second;
+    auto it = peer_info.find(peer);
+    assert(it != peer_info.end());
+    return it->second;
   }
   bool has_peer_info(pg_shard_t peer) const {
     return peer_info.count(peer);

From 36870a557194dd647c03f99e033c4fc99c89dbe0 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Mon, 25 Sep 2023 18:00:37 +0300
Subject: [PATCH 0055/2492] osd/PrimaryLogPG: get rid off redundant assertion.

It's also performed inside get_peer_info() call below.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/osd/PrimaryLogPG.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index c28184f9c9b6..f615604eaeff 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -550,7 +550,6 @@ bool PrimaryLogPG::should_send_op(
   const hobject_t &hoid) {
   if (peer == get_primary())
     return true;
-  ceph_assert(recovery_state.has_peer_info(peer));
   bool should_send =
       hoid.pool != (int64_t)info.pgid.pool() ||
       hoid <= last_backfill_started ||

From d14752ff1f56da93ad0d9c94fcf101945fdacadd Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 21 Sep 2023 15:02:38 +0800
Subject: [PATCH 0056/2492] client: queue a delay cap flushing if there are
 ditry caps/snapcaps

We should queue a cap release anyway if there are dirty caps/snapcaps
for the inodes when trimming caps.

Fixes: https://tracker.ceph.com/issues/62979
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/client/Client.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 5820ef90ae0c..7d40bbc9440a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -4796,6 +4796,9 @@ void Client::trim_caps(MetaSession *s, uint64_t max)
     // is deleted inside remove_cap
     ++p;
 
+    if (in->dirty_caps || in->cap_snaps.size())
+      cap_delay_requeue(in.get());
+
     if (in->caps.size() > 1 && cap != in->auth_cap) {
       int mine = cap->issued | cap->implemented;
       int oissued = in->auth_cap ? in->auth_cap->issued : 0;
@@ -4833,7 +4836,8 @@ void Client::trim_caps(MetaSession *s, uint64_t max)
       }
       if (all && in->ino != CEPH_INO_ROOT) {
         ldout(cct, 20) << __func__ << " counting as trimmed: " << *in << dendl;
-	trimmed++;
+	if (!in->dirty_caps && !in->cap_snaps.size())
+	  trimmed++;
       }
     }
   }

From c20b4d706dfe32e08c5301676fd9144b052a4347 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Tue, 26 Sep 2023 16:20:50 +0530
Subject: [PATCH 0057/2492] mds/scrub: enqueue all child frags for a given
 fragset

Problem:
fragsets sent over to replicas for scrubbing are simplified i.e. they
are bit representation of the lease common ancestors of the frags that
need scrubbing on that replica. A search operation of a frag in the
frasget often fails to match exactly with the frags delegated to the
replica causing the scrub item to infinitely be held in the scrub stack.

Solution:
Test if the frag in the fragset sent over to the replica contains the
delegated frag as a child to accept it for scrubbing.

Fixes: https://tracker.ceph.com/issues/62658
Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/mds/ScrubStack.cc | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc
index 6d799343f149..047bf3ba8220 100644
--- a/src/mds/ScrubStack.cc
+++ b/src/mds/ScrubStack.cc
@@ -892,22 +892,30 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m)
 
       std::vector<CDir*> dfs;
       MDSGatherBuilder gather(g_ceph_context);
+      frag_vec_t frags;
+      diri->dirfragtree.get_leaves(frags);
       for (const auto& fg : m->get_frags()) {
-	CDir *dir = diri->get_dirfrag(fg);
-	if (!dir) {
-	  dout(10) << __func__ << " no frag " << fg << dendl;
-	  continue;
-	}
-	if (!dir->is_auth()) {
-	  dout(10) << __func__ << " not auth " << *dir << dendl;
-	  continue;
-	}
-	if (!dir->can_auth_pin()) {
-	  dout(10) << __func__ << " can't auth pin " << *dir <<  dendl;
-	  dir->add_waiter(CDir::WAIT_UNFREEZE, gather.new_sub());
-	  continue;
+	for (auto f : frags) {
+	  if (!fg.contains(f)) {
+	    dout(20) << __func__ << " skipping " << f << dendl;
+	    continue;
+	  }
+	  CDir *dir = diri->get_or_open_dirfrag(mdcache, f);
+	  if (!dir) {
+	    dout(10) << __func__ << " no frag " << f << dendl;
+	    continue;
+	  }
+	  if (!dir->is_auth()) {
+	    dout(10) << __func__ << " not auth " << *dir << dendl;
+	    continue;
+	  }
+	  if (!dir->can_auth_pin()) {
+	    dout(10) << __func__ << " can't auth pin " << *dir <<  dendl;
+	    dir->add_waiter(CDir::WAIT_UNFREEZE, gather.new_sub());
+	    continue;
+	  }
+	  dfs.push_back(dir);
 	}
-	dfs.push_back(dir);
       }
 
       if (gather.has_subs()) {

From d0226306f04284f99e95be51a55574f6b43e5a49 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 31 Aug 2023 13:15:18 +0800
Subject: [PATCH 0058/2492] test/crimson/seastore: add journal replay to
 fltree-onode-manager unittest

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../seastore/onode_tree/test_fltree_onode_manager.cc        | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
index 1f661cdca596..17ad975d5e87 100644
--- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
+++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
@@ -272,13 +272,14 @@ TEST_P(fltree_onode_manager_test_t, 2_synthetic)
   run_async([this] {
     uint64_t block_size = tm->get_block_size();
     auto pool = KVPool<onode_item_t>::create_range(
-        {0, 100}, {32, 64, 128, 256, 512}, block_size);
+        {0, 10000}, {32, 64, 128, 256, 512}, block_size);
     auto start = pool.begin();
     auto end = pool.end();
     with_onodes_write(start, end,
         [](auto& t, auto& onode, auto& item) {
       item.initialize(t, onode);
     });
+    restart();
     validate_onodes(start, end);
 
     validate_list_onodes(pool);
@@ -289,6 +290,7 @@ TEST_P(fltree_onode_manager_test_t, 2_synthetic)
         [](auto& t, auto& onode, auto& item) {
       item.modify(t, onode);
     });
+    restart();
     validate_onodes(start, end);
 
     pool.shuffle();
@@ -298,6 +300,7 @@ TEST_P(fltree_onode_manager_test_t, 2_synthetic)
         [](auto& t, auto& onode, auto& item) {
       item.modify(t, onode);
     });
+    restart();
     validate_onodes(start, end);
 
     pool.shuffle();
@@ -310,6 +313,7 @@ TEST_P(fltree_onode_manager_test_t, 2_synthetic)
         return manager->erase_onode(t, onode_ref);
       }).unsafe_get0();
     });
+    restart();
     validate_erased(rd_start, rd_end);
     pool.erase_from_random(rd_start, rd_end);
     start = pool.begin();

From 3ccd10f266cfd7ec6dd1ad930598bfe4ca422a90 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Thu, 17 Aug 2023 20:01:38 +0000
Subject: [PATCH 0059/2492] qa/suites/rados: Added
 wait_for_all_active_clean_pgs flag

Added flag to not allow rados suite to delete
the pool unless all pgs are active+clean
and all OSDs are up in the thrashosds side
of the test.

Fixes: https://tracker.ceph.com/issues/59172

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 .../thrash-erasure-code/thrashers/minsize_recovery.yaml    | 5 ++++-
 qa/tasks/rados.py                                          | 7 +++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/qa/suites/rados/thrash-erasure-code/thrashers/minsize_recovery.yaml b/qa/suites/rados/thrash-erasure-code/thrashers/minsize_recovery.yaml
index 771d9a1047a7..f7df20f313f0 100644
--- a/qa/suites/rados/thrash-erasure-code/thrashers/minsize_recovery.yaml
+++ b/qa/suites/rados/thrash-erasure-code/thrashers/minsize_recovery.yaml
@@ -13,7 +13,10 @@ overrides:
         osd scrub min interval: 60
         osd scrub max interval: 120
         osd max backfills: 2
+  rados:
+    wait_for_all_active_clean_pgs: true
+
 tasks:
 - thrashosds:
     timeout: 1200
-    chance_test_min_size: 3 
+    chance_test_min_size: 3
diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py
index a730a72993c2..625ea4d9d764 100644
--- a/qa/tasks/rados.py
+++ b/qa/tasks/rados.py
@@ -272,6 +272,13 @@ def thread():
                     )
                 tests[id_] = proc
             run.wait(tests.values())
+            wait_for_all_active_clean_pgs = config.get("wait_for_all_active_clean_pgs", False)
+            # usually set when we do min_size testing.
+            if  wait_for_all_active_clean_pgs:
+                # Make sure we finish the test first before deleting the pool.
+                # Mainly used for test_pool_min_size
+                manager.wait_for_clean()
+                manager.wait_for_all_osds_up(timeout=1800)
 
             for pool in created_pools:
                 manager.wait_snap_trimming_complete(pool);

From 8c68a503f1d87d4efedab53dd68d7c79529d27aa Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Fri, 29 Sep 2023 13:16:42 +0200
Subject: [PATCH 0060/2492] os/bluestore: test _extend_log sequence advance

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/test/objectstore/test_bluefs.cc | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc
index 5eac49938a68..007f47837cef 100644
--- a/src/test/objectstore/test_bluefs.cc
+++ b/src/test/objectstore/test_bluefs.cc
@@ -1559,6 +1559,30 @@ TEST(BlueFS, test_log_runway_3) {
   }
 }
 
+TEST(BlueFS, test_log_runway_advance_seq) {
+  uint64_t max_log_runway = 65536;
+  ConfSaver conf(g_ceph_context->_conf);
+  conf.SetVal("bluefs_alloc_size", "4096");
+  conf.SetVal("bluefs_shared_alloc_size", "4096");
+  conf.SetVal("bluefs_compact_log_sync", "false");
+  conf.SetVal("bluefs_min_log_runway", "32768");
+  conf.SetVal("bluefs_max_log_runway", std::to_string(max_log_runway).c_str());
+  conf.ApplyChanges();
+
+  uint64_t size = 1048576 * 128;
+  TempBdev bdev{size};
+  BlueFS fs(g_ceph_context);
+  ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
+  uuid_d fsid;
+  ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
+  ASSERT_EQ(0, fs.mount());
+  ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
+
+  std::string longdir(max_log_runway*2, 'A');
+  ASSERT_EQ(fs.mkdir(longdir), 0);
+  fs.compact_log();
+}
+
 int main(int argc, char **argv) {
   auto args = argv_to_vec(argc, argv);
   map<string,string> defaults = {

From 63f0a0df14c9d8e68be61e374438bc75cef45a1f Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Fri, 29 Sep 2023 13:17:03 +0200
Subject: [PATCH 0061/2492] os/bluestore: fix _extend_log seq advance

when extending the log, the sequence was left on a bad state because it would first create a transaction to update with the current seq number but leave the "real" transaction with the same sequence number which should be `extend_log_transaction.seq + 1`.

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueFS.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 53284355132f..e6ae5e70e48c 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -3120,12 +3120,13 @@ void BlueFS::_extend_log(uint64_t amount) {
   _pad_bl(bl, super.block_size);
   log.writer->append(bl);
   ceph_assert(allocated_before_extension >= log.writer->get_effective_write_pos());
-  log.t.seq = log.seq_live;
 
   // before sync_core we advance the seq
   {
     std::unique_lock<ceph::mutex> l(dirty.lock);
-    _log_advance_seq();
+    dirty.seq_live++;
+    log.seq_live++;
+    log.t.seq++;
   }
 }
 

From eace0b102c48a78a717fa30e58a68796c17b82b6 Mon Sep 17 00:00:00 2001
From: Patty8122 <divyapattisapu@uchicago.edu>
Date: Fri, 22 Sep 2023 17:37:10 -0500
Subject: [PATCH 0062/2492] mds: Updating the print statement to include
 filepath(root.c_str())

Signed-off-by: Patty8122 <divyapattisapu@uchicago.edu>
---
 src/mds/MDSRank.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index f93c8d7fd60a..984d90267297 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3183,7 +3183,7 @@ void MDSRank::command_dump_tree(const cmdmap_t &cmdmap, std::ostream &ss, Format
   std::lock_guard l(mds_lock);
   CInode *in = mdcache->cache_traverse(filepath(root.c_str()));
   if (!in) {
-    ss << "root inode is not in cache";
+    ss << "inode for path '" << filepath(root.c_str()) << "' is not in cache";
     return;
   }
   f->open_array_section("inodes");

From cf0450f8e148ea481f59a2f19149ae398230227e Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 22 Sep 2023 15:44:44 +0300
Subject: [PATCH 0063/2492] os/bluestore: add more latency tracking perf
 counters into BlueFS

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueFS.cc | 42 +++++++++++++++++++++++++++++++++++---
 src/os/bluestore/BlueFS.h  |  7 ++++++-
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 53284355132f..920739dec898 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -300,6 +300,10 @@ void BlueFS::_init_logger()
 		    "auwb",
 		    PerfCountersBuilder::PRIO_CRITICAL,
 		    unit_t(UNIT_BYTES));
+  b.add_time_avg   (l_bluefs_read_random_lat, "read_random_lat",
+                    "Average bluefs read_random latency",
+                    "rdrt",
+                    PerfCountersBuilder::PRIO_INTERESTING);
   b.add_u64_counter(l_bluefs_read_random_count, "read_random_count",
 		    "random read requests processed",
 		    NULL,
@@ -338,6 +342,10 @@ void BlueFS::_init_logger()
 		    "Bytes read from prefetch buffer in random read mode",
 		    NULL,
 		    PerfCountersBuilder::PRIO_USEFUL, unit_t(UNIT_BYTES));
+  b.add_time_avg   (l_bluefs_read_lat, "read_lat",
+                    "Average bluefs read latency",
+                    "rd_t",
+                    PerfCountersBuilder::PRIO_INTERESTING);
   b.add_u64_counter(l_bluefs_read_count, "read_count",
 		    "buffered read requests processed",
 		    NULL,
@@ -381,14 +389,30 @@ void BlueFS::_init_logger()
   b.add_u64_counter(l_bluefs_write_bytes, "write_bytes",
 		    "Bytes written", NULL,
 		    PerfCountersBuilder::PRIO_USEFUL, unit_t(UNIT_BYTES));
- b.add_time_avg     (l_bluefs_compaction_lat, "compact_lat",
+  b.add_time_avg   (l_bluefs_compaction_lat, "compact_lat",
                     "Average bluefs log compaction latency",
                     "c__t",
                     PerfCountersBuilder::PRIO_INTERESTING);
- b.add_time_avg     (l_bluefs_compaction_lock_lat, "compact_lock_lat",
+  b.add_time_avg   (l_bluefs_compaction_lock_lat, "compact_lock_lat",
                     "Average lock duration while compacting bluefs log",
                     "c_lt",
                     PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time_avg   (l_bluefs_fsync_lat, "fsync_lat",
+                    "Average bluefs fsync latency",
+                    "fs_t",
+                    PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time_avg   (l_bluefs_flush_lat, "flush_lat",
+                    "Average bluefs flush latency",
+                    "fl_t",
+                    PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time_avg   (l_bluefs_unlink_lat, "unlink_lat",
+                    "Average bluefs unlink latency",
+                    "unlt",
+                    PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time_avg   (l_bluefs_truncate_lat, "truncate_lat",
+                    "Average bluefs truncate latency",
+                    "trnt",
+                    PerfCountersBuilder::PRIO_INTERESTING);
   b.add_u64_counter(l_bluefs_alloc_shared_dev_fallbacks, "alloc_slow_fallback",
 		    "Amount of allocations that required fallback to "
                     " slow/shared device",
@@ -2086,6 +2110,7 @@ int64_t BlueFS::_read_random(
   uint64_t len,          ///< [in] this many bytes
   char *out)             ///< [out] copy it here
 {
+  auto t0 = mono_clock::now();
   auto* buf = &h->buf;
 
   int64_t ret = 0;
@@ -2171,6 +2196,7 @@ int64_t BlueFS::_read_random(
            << " got 0x" << ret
            << std::dec  << dendl;
   --h->file->num_reading;
+  logger->tinc(l_bluefs_read_random_lat, mono_clock::now() - t0);
   return ret;
 }
 
@@ -2181,6 +2207,7 @@ int64_t BlueFS::_read(
   bufferlist *outbl,     ///< [out] optional: reference the result here
   char *out)             ///< [out] optional: or copy it here
 {
+  auto t0 = mono_clock::now();
   FileReaderBuffer *buf = &(h->buf);
 
   bool prefetch = !outbl && !out;
@@ -2302,6 +2329,7 @@ int64_t BlueFS::_read(
            << std::dec  << dendl;
   ceph_assert(!outbl || (int)outbl->length() == ret);
   --h->file->num_reading;
+  logger->tinc(l_bluefs_read_lat, mono_clock::now() - t0);
   return ret;
 }
 
@@ -3369,6 +3397,7 @@ void BlueFS::flush_range(FileWriter *h, uint64_t offset, uint64_t length)/*_WF*/
 
 int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length)
 {
+  auto t0 = mono_clock::now();
   ceph_assert(ceph_mutex_is_locked(h->lock));
   ceph_assert(h->file->num_readers.load() == 0);
   ceph_assert(h->file->fnode.ino > 1);
@@ -3424,6 +3453,7 @@ int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length)
   dout(20) << __func__ << " file now, unflushed " << h->file->fnode << dendl;
   int res = _flush_data(h, offset, length, buffered);
   vselector->add_usage(h->file->vselector_hint, h->file->fnode);
+  logger->tinc(l_bluefs_flush_lat, mono_clock::now() - t0);
   return res;
 }
 
@@ -3645,6 +3675,7 @@ uint64_t BlueFS::_flush_special(FileWriter *h)
 
 int BlueFS::truncate(FileWriter *h, uint64_t offset)/*_WF_L*/
 {
+  auto t0 = mono_clock::now();
   std::lock_guard hl(h->lock);
   dout(10) << __func__ << " 0x" << std::hex << offset << std::dec
            << " file " << h->file->fnode << dendl;
@@ -3683,11 +3714,13 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset)/*_WF_L*/
   h->file->is_dirty = true;
   vselector->add_usage(h->file->vselector_hint, h->file->fnode.size);
   log.t.op_file_update_inc(h->file->fnode);
+  logger->tinc(l_bluefs_truncate_lat, mono_clock::now() - t0);
   return 0;
 }
 
 int BlueFS::fsync(FileWriter *h)/*_WF_WD_WLD_WLNF_WNF*/
 {
+  auto t0 = mono_clock::now();
   _maybe_check_vselector_LNF();
   std::unique_lock hl(h->lock);
   uint64_t old_dirty_seq = 0;
@@ -3715,7 +3748,7 @@ int BlueFS::fsync(FileWriter *h)/*_WF_WD_WLD_WLNF_WNF*/
     _flush_and_sync_log_LD(old_dirty_seq);
   }
   _maybe_compact_log_LNF_NF_LD_D();
-
+  logger->tinc(l_bluefs_fsync_lat, mono_clock::now() - t0);
   return 0;
 }
 
@@ -4357,6 +4390,7 @@ int BlueFS::readdir(std::string_view dirname, vector<string> *ls)/*_N*/
 
 int BlueFS::unlink(std::string_view dirname, std::string_view filename)/*_LND*/
 {
+  auto t0 = mono_clock::now();
   std::lock_guard ll(log.lock);
   std::lock_guard nl(nodes.lock);
   dout(10) << __func__ << " " << dirname << "/" << filename << dendl;
@@ -4381,6 +4415,8 @@ int BlueFS::unlink(std::string_view dirname, std::string_view filename)/*_LND*/
   dir->file_map.erase(string{filename});
   log.t.op_dir_unlink(dirname, filename);
   _drop_link_D(file);
+  logger->tinc(l_bluefs_unlink_lat, mono_clock::now() - t0);
+
   return 0;
 }
 
diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index 9c5fb4981592..f0493a758dab 100644
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -46,6 +46,7 @@ enum {
   l_bluefs_main_alloc_unit,
   l_bluefs_db_alloc_unit,
   l_bluefs_wal_alloc_unit,
+  l_bluefs_read_random_lat,
   l_bluefs_read_random_count,
   l_bluefs_read_random_bytes,
   l_bluefs_read_random_disk_count,
@@ -55,6 +56,7 @@ enum {
   l_bluefs_read_random_disk_bytes_slow,
   l_bluefs_read_random_buffer_count,
   l_bluefs_read_random_buffer_bytes,
+  l_bluefs_read_lat,
   l_bluefs_read_count,
   l_bluefs_read_bytes,
   l_bluefs_read_disk_count,
@@ -69,6 +71,10 @@ enum {
   l_bluefs_write_bytes,
   l_bluefs_compaction_lat,
   l_bluefs_compaction_lock_lat,
+  l_bluefs_fsync_lat,
+  l_bluefs_flush_lat,
+  l_bluefs_unlink_lat,
+  l_bluefs_truncate_lat,
   l_bluefs_alloc_shared_dev_fallbacks,
   l_bluefs_alloc_shared_size_fallbacks,
   l_bluefs_read_zeros_candidate,
@@ -445,7 +451,6 @@ class BlueFS {
   int _flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered);
   int _flush_F(FileWriter *h, bool force, bool *flushed = nullptr);
   uint64_t _flush_special(FileWriter *h);
-  int _fsync(FileWriter *h);
 
 #ifdef HAVE_LIBAIO
   void _claim_completed_aios(FileWriter *h, std::list<aio_t> *ls);

From b7b38897a411bcfd1f88a8bd0564fa4b81f23052 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 22 Sep 2023 15:59:18 +0300
Subject: [PATCH 0064/2492] os/bluestore: a bit more effective file_map
 handling in BlueFS

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueFS.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 920739dec898..337a022e7df0 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -4023,8 +4023,8 @@ int BlueFS::open_for_write(
     dir = p->second;
   }
 
-  map<string,FileRef>::iterator q = dir->file_map.find(filename);
-  if (q == dir->file_map.end()) {
+  map<string,FileRef>::iterator q = dir->file_map.lower_bound(filename);
+  if (q == dir->file_map.end() || q->first != filename) {
     if (overwrite) {
       dout(20) << __func__ << " dir " << dirname << " (" << dir
 	       << ") file " << filename
@@ -4034,7 +4034,7 @@ int BlueFS::open_for_write(
     file = ceph::make_ref<File>();
     file->fnode.ino = ++ino_last;
     nodes.file_map[ino_last] = file;
-    dir->file_map[string{filename}] = file;
+    dir->file_map.emplace_hint(q, string{filename}, file);
     ++file->refs;
     create = true;
     logger->set(l_bluefs_num_files, nodes.file_map.size());
@@ -4263,7 +4263,7 @@ int BlueFS::rmdir(std::string_view dirname)/*_LN*/
     dout(20) << __func__ << " dir " << dirname << " not empty" << dendl;
     return -ENOTEMPTY;
   }
-  nodes.dir_map.erase(string{dirname});
+  nodes.dir_map.erase(p);
   log.t.op_dir_remove(dirname);
   return 0;
 }
@@ -4317,9 +4317,9 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename,
     return -ENOENT;
   }
   DirRef dir = p->second;
-  auto q = dir->file_map.find(filename);
+  auto q = dir->file_map.lower_bound(filename);
   FileRef file;
-  if (q == dir->file_map.end()) {
+  if (q == dir->file_map.end() || q->first != filename) {
     dout(20) << __func__ << " dir " << dirname << " (" << dir
 	     << ") file " << filename
 	     << " not found, creating" << dendl;
@@ -4327,7 +4327,7 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename,
     file->fnode.ino = ++ino_last;
     file->fnode.mtime = ceph_clock_now();
     nodes.file_map[ino_last] = file;
-    dir->file_map[string{filename}] = file;
+    dir->file_map.emplace_hint(q, string{filename}, file);
     logger->set(l_bluefs_num_files, nodes.file_map.size());
     ++file->refs;
     log.t.op_file_update(file->fnode);
@@ -4412,7 +4412,7 @@ int BlueFS::unlink(std::string_view dirname, std::string_view filename)/*_LND*/
              << " is locked" << dendl;
     return -EBUSY;
   }
-  dir->file_map.erase(string{filename});
+  dir->file_map.erase(q);
   log.t.op_dir_unlink(dirname, filename);
   _drop_link_D(file);
   logger->tinc(l_bluefs_unlink_lat, mono_clock::now() - t0);

From acc26d7b17316d1e45bfc3a882355b46db19d053 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 9 May 2023 15:06:41 -0400
Subject: [PATCH 0065/2492] mgr/cephadm: make jaeger-collector urls a dep for
 jaeger-agent

the jaeger-agent's need to know the url for the collector(s)
that have been deployed. If a collector moves, or we deployed
the agents before the collector, we need to reconfig the agents
with updated info about the collectors. Failure to do so can
leave the jager-agents down reporting

```
Could not create collector proxy","error":"at least one collector hostPort address is required when resolver is not available"
```

Fixes: https://tracker.ceph.com/issues/59704

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py          | 7 +++++++
 src/pybind/mgr/cephadm/serve.py           | 5 +++++
 src/pybind/mgr/cephadm/services/jaeger.py | 3 +++
 3 files changed, 15 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 4b6f7cf7a567..70d66732f0f5 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -40,6 +40,7 @@
 
 
 from mgr_module import MgrModule, HandleCommandResult, Option, NotifyType
+from mgr_util import build_url
 import orchestrator
 from orchestrator.module import to_format, Format
 
@@ -2712,6 +2713,12 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
                 deps.append(f'{hash(alertmanager_user + alertmanager_password)}')
         elif daemon_type == 'promtail':
             deps += get_daemon_names(['loki'])
+        elif daemon_type == JaegerAgentService.TYPE:
+            for dd in self.cache.get_daemons_by_type(JaegerCollectorService.TYPE):
+                assert dd.hostname is not None
+                port = dd.ports[0] if dd.ports else JaegerCollectorService.DEFAULT_SERVICE_PORT
+                deps.append(build_url(host=dd.hostname, port=port).lstrip('/'))
+            deps = sorted(deps)
         else:
             # TODO(redo): some error message!
             pass
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index a17ac151e3a6..116e97238691 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -1060,6 +1060,11 @@ def _check_daemons(self) -> None:
                     diff = list(set(last_deps) - set(deps))
                     if any('secure_monitoring_stack' in e for e in diff):
                         action = 'redeploy'
+                elif dd.daemon_type == 'jaeger-agent':
+                    # changes to jaeger-agent deps affect the way the unit.run for
+                    # the daemon is written, which we rewrite on redeploy, but not
+                    # on reconfig.
+                    action = 'redeploy'
 
             elif spec is not None and hasattr(spec, 'extra_container_args') and dd.extra_container_args != spec.extra_container_args:
                 self.log.debug(
diff --git a/src/pybind/mgr/cephadm/services/jaeger.py b/src/pybind/mgr/cephadm/services/jaeger.py
index c136d20e612a..c83c765d0394 100644
--- a/src/pybind/mgr/cephadm/services/jaeger.py
+++ b/src/pybind/mgr/cephadm/services/jaeger.py
@@ -20,13 +20,16 @@ class JaegerAgentService(CephadmService):
     def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
         assert self.TYPE == daemon_spec.daemon_type
         collectors = []
+        deps: List[str] = []
         for dd in self.mgr.cache.get_daemons_by_type(JaegerCollectorService.TYPE):
             # scrape jaeger-collector nodes
             assert dd.hostname is not None
             port = dd.ports[0] if dd.ports else JaegerCollectorService.DEFAULT_SERVICE_PORT
             url = build_url(host=dd.hostname, port=port).lstrip('/')
             collectors.append(url)
+            deps.append(url)
         daemon_spec.final_config = {'collector_nodes': ",".join(collectors)}
+        daemon_spec.deps = sorted(deps)
         return daemon_spec
 
 

From 9a083b0935509744234082832d12ed2734bcb6e0 Mon Sep 17 00:00:00 2001
From: "Frank S. Filz" <ffilzlnx@mindspring.com>
Date: Tue, 3 Oct 2023 09:39:20 -0700
Subject: [PATCH 0066/2492] CLIENT: C_Read_Async_Finisher should assume
 client_lock is held

Client::C_Read_Async_Finisher::finish() doesn't need to take the
lock because ObjectCacher has already assured the lock is held.

Of course when we immediately complete, we don't need to unlock.

Signed-off-by: Frank S. Filz <ffilzlnx@mindspring.com>
---
 src/client/Client.cc | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 4e7e3961e8e1..1ce33f230538 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -10999,15 +10999,11 @@ void Client::do_readahead(Fh *f, Inode *in, uint64_t off, uint64_t len)
 
 void Client::C_Read_Async_Finisher::finish(int r)
 {
-  clnt->client_lock.lock();
-
   // Do read ahead as long as we aren't completing with 0 bytes
   if (r != 0)
     clnt->do_readahead(f, in, off, len);
 
   onfinish->complete(r);
-
-  clnt->client_lock.unlock();
 }
 
 int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl,
@@ -11039,9 +11035,7 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl,
     Context *crf = io_finish.release();
 
     // Complete the crf immediately with 0 bytes
-    client_lock.unlock();
     crf->complete(0);
-    client_lock.lock();
 
     // Signal async completion
     return 0;
@@ -11073,9 +11067,7 @@ int Client::_read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl,
     Context *crf = io_finish.release();
     if (r != 0) {
       // need to do readahead, so complete the crf
-      client_lock.unlock();
       crf->complete(r);
-      client_lock.lock();
     } else {
       get_cap_ref(in, CEPH_CAP_FILE_CACHE);
     }

From f004def5b7cb8aa31e7f428f5be9bcf1aee2f30c Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Tue, 19 Sep 2023 14:26:19 +0300
Subject: [PATCH 0067/2492] osd: do not assert on fast shutdown timeout

Fixes: https://tracker.ceph.com/issues/61140

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/osd/OSD.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index f01540c3a930..d813de2b7e20 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4496,10 +4496,12 @@ int OSD::shutdown()
     store->umount();
 
     utime_t end_time = ceph_clock_now();
-    if (cct->_conf->osd_fast_shutdown_timeout) {
-      ceph_assert(end_time - start_time_func < cct->_conf->osd_fast_shutdown_timeout);
-    }
     dout(0) <<"Fast Shutdown duration total     :" << end_time              - start_time_func       << " seconds" << dendl;
+    if (cct->_conf->osd_fast_shutdown_timeout &&
+        end_time - start_time_func > cct->_conf->osd_fast_shutdown_timeout) {
+      dout(0) << "Fast Shutdown duration exceeded :" << cct->_conf->osd_fast_shutdown_timeout       << " seconds"
+              << dendl;
+    }
     dout(0) <<"Fast Shutdown duration osd_drain :" << start_time_umount     - start_time_osd_drain  << " seconds" << dendl;
     dout(0) <<"Fast Shutdown duration umount    :" << end_time              - start_time_umount     << " seconds" << dendl;
     dout(0) <<"Fast Shutdown duration timer     :" << start_time_osd_drain  - start_time_timer      << " seconds" << dendl;

From b2c4e62afac32edda142a51eb601420a1a79bb2f Mon Sep 17 00:00:00 2001
From: pilem94 <pierre.lemay@dti.ulaval.ca>
Date: Tue, 3 Oct 2023 16:09:48 -0400
Subject: [PATCH 0068/2492] src/ceph-volume/ceph_volume/devices/lvm/listing.py
 : lvm list filters also on vg name

This commit fix the listing of LVs with the same name on multiple VG

Fixes: https://tracker.ceph.com/issues/62320
Signed-off-by: Pierre Lemay <pierre.lemay@gmail.com>
---
 src/ceph-volume/ceph_volume/devices/lvm/listing.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/listing.py b/src/ceph-volume/ceph_volume/devices/lvm/listing.py
index c16afdaa7672..8fb9d8ddcf87 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/listing.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/listing.py
@@ -153,7 +153,9 @@ def single_report(self, arg):
         elif arg[0] == '/':
             lv = api.get_lvs_from_path(arg)
         else:
-            lv = [api.get_single_lv(filters={'lv_name': arg.split('/')[1]})]
+            vg_name, lv_name = arg.split('/')
+            lv = [api.get_single_lv(filters={'lv_name': lv_name,
+                                             'vg_name': vg_name})]
 
         report = self.create_report(lv)
 

From 637613eb148157f30f8524dc99e508cf674b7b9f Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Tue, 3 Oct 2023 16:05:39 -0400
Subject: [PATCH 0069/2492] STS: when generating keys, take the trailing
 character into account

Otherwise, STS acckey has 19 and secret key has 39 alphanumeric chars.

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_sts.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_sts.cc b/src/rgw/rgw_sts.cc
index b552834426a9..2b31d5c5a297 100644
--- a/src/rgw/rgw_sts.cc
+++ b/src/rgw/rgw_sts.cc
@@ -54,7 +54,7 @@ int Credentials::generateCredentials(const DoutPrefixProvider *dpp,
                           rgw::auth::Identity* identity)
 {
   uuid_d accessKey, secretKey;
-  char accessKeyId_str[MAX_ACCESS_KEY_LEN], secretAccessKey_str[MAX_SECRET_KEY_LEN];
+  char accessKeyId_str[MAX_ACCESS_KEY_LEN + 1], secretAccessKey_str[MAX_SECRET_KEY_LEN + 1];
 
   //AccessKeyId
   gen_rand_alphanumeric_plain(cct, accessKeyId_str, sizeof(accessKeyId_str));

From b14ff07e6344d9f097259265d468f6300818b053 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 4 Oct 2023 01:11:32 +0200
Subject: [PATCH 0070/2492] ceph-volume: fix util.get_partitions

The current logic makes it report only the first
partitions of devices.

Fixes: https://tracker.ceph.com/issues/63086

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-volume/ceph_volume/util/disk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index ee061b724007..704b9e76600c 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -816,7 +816,7 @@ def get_partitions(_sys_dev_block_path ='/sys/dev/block'):
     result = dict()
     for device in devices:
         device_path = os.path.join(_sys_dev_block_path, device)
-        is_partition = get_file_contents(os.path.join(device_path, 'partition')) == "1"
+        is_partition = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0
         if not is_partition:
             continue
 

From 6f3f58cb8e4ce100cc7186858465b4d11d5c2c49 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Wed, 4 Oct 2023 12:24:13 +0530
Subject: [PATCH 0071/2492] mgr/dashboard: Consider null values as zero in
 grafana panels

After upgrading from RHCS4 to RHCS5..some of the grafana charts broke.
This is because in RHCS5 we do not generate the metrics if its value is
zero as a result the null value from that metric breaks the grafana
charts or graphs. This PR is to fix the above mentioned issue.

Fixes: https://tracker.ceph.com/issues/63088

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../ceph-mixin/dashboards/osd.libsonnet       |  2 +-
 .../ceph-mixin/dashboards/rbd.libsonnet       |  2 +-
 .../ceph-mixin/dashboards/rgw.libsonnet       |  4 ++--
 .../dashboards_out/osd-device-details.json    | 12 +++++------
 .../dashboards_out/radosgw-detail.json        |  6 +++---
 .../dashboards_out/radosgw-overview.json      | 20 +++++++++----------
 .../dashboards_out/rbd-overview.json          |  6 +++---
 7 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards/osd.libsonnet b/monitoring/ceph-mixin/dashboards/osd.libsonnet
index 0ea43c96ff9f..0015c7f398bd 100644
--- a/monitoring/ceph-mixin/dashboards/osd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/osd.libsonnet
@@ -342,7 +342,7 @@ local g = import 'grafonnet/grafana.libsonnet';
       $.graphPanelSchema({},
                          title,
                          description,
-                         'null',
+                         'null as zero',
                          false,
                          formatY1,
                          'short',
diff --git a/monitoring/ceph-mixin/dashboards/rbd.libsonnet b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
index 0eca5a877737..709d4e04f7e9 100644
--- a/monitoring/ceph-mixin/dashboards/rbd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
@@ -133,7 +133,7 @@ local u = import 'utils.libsonnet';
       $.graphPanelSchema({},
                          title,
                          '',
-                         'null',
+                         'null as zero',
                          false,
                          formatY1,
                          'short',
diff --git a/monitoring/ceph-mixin/dashboards/rgw.libsonnet b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
index 892480d1ca0f..49dcf9156884 100644
--- a/monitoring/ceph-mixin/dashboards/rgw.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
@@ -140,7 +140,7 @@ local u = import 'utils.libsonnet';
         {},
         title,
         description,
-        'null',
+        'null as zero',
         false,
         formatY1,
         formatY2,
@@ -658,7 +658,7 @@ local u = import 'utils.libsonnet';
       $.graphPanelSchema(aliasColors,
                          title,
                          description,
-                         'null',
+                         'null as zero',
                          false,
                          formatY1,
                          formatY2,
diff --git a/monitoring/ceph-mixin/dashboards_out/osd-device-details.json b/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
index 384516fb0195..811e6d57ef2e 100644
--- a/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
+++ b/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
@@ -87,7 +87,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -185,7 +185,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -283,7 +283,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -400,7 +400,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -498,7 +498,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -596,7 +596,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
index a0f8f3537c48..4568f9a4d854 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
@@ -93,7 +93,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -186,7 +186,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -285,7 +285,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
index 77d69e4f3152..a8256c1f5e16 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
@@ -87,7 +87,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -180,7 +180,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -266,7 +266,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -352,7 +352,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -445,7 +445,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -531,7 +531,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -636,7 +636,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -754,7 +754,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -893,7 +893,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -1000,7 +1000,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
diff --git a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
index e017280e02bf..86b354a6089c 100644
--- a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
@@ -80,7 +80,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -173,7 +173,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
@@ -266,7 +266,7 @@
          "lines": true,
          "linewidth": 1,
          "links": [ ],
-         "nullPointMode": "null",
+         "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,

From 0b8b98907c7f60cfe64636c3278e474bd03f27ce Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 20 Sep 2023 19:10:59 +0300
Subject: [PATCH 0072/2492] osd: introduce dump_osd_pg_stats admin socket
 command.

One can learn full OSD stats in a way they're reported to monitors using it.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/messages/MPGStats.h | 25 +++++++++++++++++++++++++
 src/osd/OSD.cc          | 12 ++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/src/messages/MPGStats.h b/src/messages/MPGStats.h
index 65cec5244788..2d9c2dcb5c2c 100644
--- a/src/messages/MPGStats.h
+++ b/src/messages/MPGStats.h
@@ -44,6 +44,31 @@ class MPGStats final : public PaxosServiceMessage {
   void print(std::ostream& out) const override {
     out << "pg_stats(" << pg_stat.size() << " pgs seq " << osd_stat.seq << " v " << version << ")";
   }
+  void dump_stats(ceph::Formatter *f) const {
+    f->open_object_section("stats");
+    {
+      f->open_array_section("pg_stat");
+      for(const auto& [_pg, _stat] : pg_stat) {
+        f->open_object_section("pg_stat");
+        _pg.dump(f);
+        _stat.dump(f);
+        f->close_section();
+      }
+      f->close_section();
+
+      f->dump_object("osd_stat", osd_stat);
+
+      f->open_array_section("pool_stat");
+      for(const auto& [_id, _stat] : pool_stat) {
+        f->open_object_section("pool");
+        f->dump_int("poolid", _id);
+        _stat.dump(f);
+        f->close_section();
+      }
+      f->close_section();
+    }
+    f->close_section();
+  }
 
   void encode_payload(uint64_t features) override {
     using ceph::encode;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 52f937d7ff4d..6b3dd52786a3 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3278,6 +3278,13 @@ will start to track new ops received afterwards.";
       st.dump(f);
       f->close_section();
     }
+  } else if (prefix == "dump_osd_pg_stats") {
+    lock_guard l(osd_lock);
+
+    MPGStats* m = collect_pg_stats();
+    ceph_assert(m);
+    m->dump_stats(f);
+    m->put();
   } else {
     ceph_abort_msg("broken asok registration");
   }
@@ -4155,6 +4162,11 @@ void OSD::final_init()
     "Dump store's statistics for the given pool");
   ceph_assert(r == 0);
 
+  r = admin_socket->register_command(
+    "dump_osd_pg_stats ", asok_hook,
+    "Dump OSD PGs' statistics");
+  ceph_assert(r == 0);
+
   test_ops_hook = new TestOpsSocketHook(&(this->service), this->store.get());
   // Note: pools are CephString instead of CephPoolname because
   // these commands traditionally support both pool names and numbers

From 7752b9019d4444a499051669539cfd67b83e0d44 Mon Sep 17 00:00:00 2001
From: Cory Snyder <csnyder@1111systems.com>
Date: Wed, 4 Oct 2023 05:42:30 -0400
Subject: [PATCH 0073/2492] rgw: add versioning info to radosgw-admin bucket
 stats output

This allows admins to more easily identify whether a bucket
has versioning / object lock/ mfa enabled.

Fixes: https://tracker.ceph.com/issues/63092
Signed-off-by: Cory Snyder <csnyder@1111systems.com>
---
 src/rgw/driver/rados/rgw_bucket.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index c4b89d6cd4c6..e58b554f790b 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1290,6 +1290,8 @@ static int bucket_stats(rgw::sal::Driver* driver,
     return ret;
   }
 
+  const RGWBucketInfo& bucket_info = bucket->get_info();
+
   const auto& index = bucket->get_info().get_current_index();
   if (is_layout_indexless(index)) {
     cerr << "error, indexless buckets do not maintain stats; bucket=" <<
@@ -1320,6 +1322,10 @@ static int bucket_stats(rgw::sal::Driver* driver,
   formatter->dump_string("id", bucket->get_bucket_id());
   formatter->dump_string("marker", bucket->get_marker());
   formatter->dump_stream("index_type") << bucket->get_info().layout.current_index.layout.type;
+  formatter->dump_bool("versioned", bucket_info.versioned());
+  formatter->dump_bool("versioning_enabled", bucket_info.versioning_enabled());
+  formatter->dump_bool("object_lock_enabled", bucket_info.obj_lock_enabled());
+  formatter->dump_bool("mfa_enabled", bucket_info.mfa_enabled());
   ::encode_json("owner", bucket->get_info().owner, formatter);
   formatter->dump_string("ver", bucket_ver);
   formatter->dump_string("master_ver", master_ver);

From ae11bbe6b2805740d3621cc47c68a5f0da493df0 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Thu, 28 Sep 2023 17:52:11 +0000
Subject: [PATCH 0074/2492] osd: fix logic in check_pg_upmaps

The logic was changed in check_pg_upmaps
in a Reef refactor, which results in recommendations
made by the upmap balancer even when it says there are
no optimizations.

Fixes: https://tracker.ceph.com/issues/63029
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/osd/OSDMap.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 4a2d400b6703..ce199e8ee8ac 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -2157,8 +2157,8 @@ bool OSDMap::check_pg_upmaps(
                        << j->first << " " << j->second
                        << dendl;
         to_cancel->push_back(pg);
-      } else {
-        //Josh--check partial no-op here.
+      } else if (newmap != j->second) {
+        // check partial no-op here.
         ldout(cct, 10) << __func__ << " simplifying partially no-op pg_upmap_items "
                        << j->first << " " << j->second
                        << " -> " << newmap

From 82a242672375965d7b3872b43c49285630d93402 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Thu, 5 Oct 2023 00:04:50 +0000
Subject: [PATCH 0075/2492] qa: enable test_librgw_file.sh to be run with
 vstart cluster

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 qa/workunits/rgw/test_librgw_file.sh | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/qa/workunits/rgw/test_librgw_file.sh b/qa/workunits/rgw/test_librgw_file.sh
index 1371ff711075..d56dc7b8258f 100755
--- a/qa/workunits/rgw/test_librgw_file.sh
+++ b/qa/workunits/rgw/test_librgw_file.sh
@@ -1,5 +1,11 @@
 #!/bin/sh -e
-
+#
+# To run this test script with a cluster created via vstart.sh:
+# $PATH needs to be set for radosgw-admin and ceph_test_librgw executables.
+# $KEYRING need to be set as the path for a vstart clusters Ceph keyring.
+#
+# Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw:
+# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_librgw_file.sh
 
 if [ -z ${AWS_ACCESS_KEY_ID} ]
 then
@@ -13,7 +19,10 @@ then
        --email librgw@example.com || echo "librgw user exists"
 
     # keyring override for teuthology env
-    KEYRING="/etc/ceph/ceph.keyring"
+    if [ -z ${KEYRING} ]
+    then
+      KEYRING="/etc/ceph/ceph.keyring"
+    fi
     K="-k ${KEYRING}"
 fi
 

From dd8f59a1e88313d61f16e4b43bb47247dc64e71f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 12:50:44 -0400
Subject: [PATCH 0076/2492] cephadm: add unit test coverage for deploying
 keepalived

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 55 ++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 6fd36cc6eb18..4c0b5c845f7e 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -1,3 +1,4 @@
+import os
 import pathlib
 import unittest
 from unittest import mock
@@ -84,3 +85,57 @@ def test_deploy_snmp_container(cephadm_fs, monkeypatch):
     assert basedir.is_dir()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()
+
+
+def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
+    _call = mock.MagicMock(return_value=('', '', 0))
+    monkeypatch.setattr('cephadmlib.container_types.call', _call)
+    _call_throws = mock.MagicMock(return_value=0)
+    monkeypatch.setattr(
+        'cephadmlib.container_types.call_throws', _call_throws
+    )
+    _firewalld = mock.MagicMock()
+    _firewalld().external_ports.get.return_value = []
+    monkeypatch.setattr('cephadm.Firewalld', _firewalld)
+    _extract_uid_gid = mock.MagicMock()
+    _extract_uid_gid.return_value = (8765, 8765)
+    monkeypatch.setattr('cephadm.extract_uid_gid', _extract_uid_gid)
+    _install_sysctl = mock.MagicMock()
+    monkeypatch.setattr('cephadm.install_sysctl', _install_sysctl)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'keepalived.uiop'
+        ctx.image = 'quay.io/eeranimated/keepalived:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'destination': '192.168.100.10:8899',
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'keepalived.conf': 'neversayneveragain',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/keepalived.uiop')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith('quay.io/eeranimated/keepalived:latest')
+    _firewalld().open_ports.assert_not_called()
+    assert not (basedir / 'config').exists()
+    assert not (basedir / 'keyring').exists()
+    with open(basedir / 'keepalived.conf') as f:
+        assert f.read() == 'neversayneveragain'
+    with open(basedir / 'keepalived.conf') as f:
+        assert f.read() == 'neversayneveragain'
+        si = os.fstat(f.fileno())
+        assert (si.st_uid, si.st_gid) == (8765, 8765)
+    assert (basedir / 'keepalived').is_dir()
+    si = (basedir / 'keepalived').stat()
+    assert (si.st_uid, si.st_gid) == (8765, 8765)
+    assert _install_sysctl.call_count == 1
+    assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1

From d365d605bd7fcffef66a4924e792594e08245bd2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 13:11:14 -0400
Subject: [PATCH 0077/2492] cephadm: move some common mocks to a function

Reduce frequently repeated mocks in these new deployment test functions.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 51 +++++++++++++++++---------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 4c0b5c845f7e..fff88ba63755 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -14,12 +14,33 @@
 _cephadm = import_cephadm()
 
 
-def test_deploy_nfs_container(cephadm_fs, monkeypatch):
+def _common_mp(monkeypatch):
+    mocks = {}
     _call = mock.MagicMock(return_value=('', '', 0))
     monkeypatch.setattr('cephadmlib.container_types.call', _call)
+    mocks['call'] = _call
+    _call_throws = mock.MagicMock(return_value=0)
+    monkeypatch.setattr(
+        'cephadmlib.container_types.call_throws', _call_throws
+    )
+    mocks['call_throws'] = _call_throws
     _firewalld = mock.MagicMock()
     _firewalld().external_ports.get.return_value = []
     monkeypatch.setattr('cephadm.Firewalld', _firewalld)
+    mocks['Firewalld'] = _firewalld
+    _extract_uid_gid = mock.MagicMock()
+    _extract_uid_gid.return_value = (8765, 8765)
+    monkeypatch.setattr('cephadm.extract_uid_gid', _extract_uid_gid)
+    mocks['extract_uid_gid'] = _extract_uid_gid
+    _install_sysctl = mock.MagicMock()
+    monkeypatch.setattr('cephadm.install_sysctl', _install_sysctl)
+    mocks['install_sysctl'] = _install_sysctl
+    return mocks
+
+
+def test_deploy_nfs_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()
@@ -51,15 +72,8 @@ def test_deploy_nfs_container(cephadm_fs, monkeypatch):
 
 
 def test_deploy_snmp_container(cephadm_fs, monkeypatch):
-    _call = mock.MagicMock(return_value=('', '', 0))
-    monkeypatch.setattr('cephadmlib.container_types.call', _call)
-    _call_throws = mock.MagicMock(return_value=0)
-    monkeypatch.setattr(
-        'cephadmlib.container_types.call_throws', _call_throws
-    )
-    _firewalld = mock.MagicMock()
-    _firewalld().external_ports.get.return_value = []
-    monkeypatch.setattr('cephadm.Firewalld', _firewalld)
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()
@@ -88,20 +102,9 @@ def test_deploy_snmp_container(cephadm_fs, monkeypatch):
 
 
 def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
-    _call = mock.MagicMock(return_value=('', '', 0))
-    monkeypatch.setattr('cephadmlib.container_types.call', _call)
-    _call_throws = mock.MagicMock(return_value=0)
-    monkeypatch.setattr(
-        'cephadmlib.container_types.call_throws', _call_throws
-    )
-    _firewalld = mock.MagicMock()
-    _firewalld().external_ports.get.return_value = []
-    monkeypatch.setattr('cephadm.Firewalld', _firewalld)
-    _extract_uid_gid = mock.MagicMock()
-    _extract_uid_gid.return_value = (8765, 8765)
-    monkeypatch.setattr('cephadm.extract_uid_gid', _extract_uid_gid)
-    _install_sysctl = mock.MagicMock()
-    monkeypatch.setattr('cephadm.install_sysctl', _install_sysctl)
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _install_sysctl = mocks['install_sysctl']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()

From 42991a1dc69abca1aec6fe7ec358d554dc17bce3 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 13:19:18 -0400
Subject: [PATCH 0078/2492] cephadm: add unit test coverage for deploying
 haproxy

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 42 ++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index fff88ba63755..fdc6f582dbdc 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -142,3 +142,45 @@ def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
     assert (si.st_uid, si.st_gid) == (8765, 8765)
     assert _install_sysctl.call_count == 1
     assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1
+
+
+def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _install_sysctl = mocks['install_sysctl']
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'haproxy.yyz'
+        ctx.image = 'quay.io/lfeuwbo/haproxy:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'haproxy.cfg': 'bifrost',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/haproxy.yyz')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith(
+        'quay.io/lfeuwbo/haproxy:latest haproxy -f /var/lib/haproxy/haproxy.cfg'
+    )
+    _firewalld().open_ports.assert_not_called()
+    assert not (basedir / 'config').exists()
+    assert not (basedir / 'keyring').exists()
+    assert (basedir / 'haproxy').is_dir()
+    si = (basedir / 'haproxy').stat()
+    assert (si.st_uid, si.st_gid) == (8765, 8765)
+    with open(basedir / 'haproxy/haproxy.cfg') as f:
+        assert f.read() == 'bifrost'
+        si = os.fstat(f.fileno())
+        assert (si.st_uid, si.st_gid) == (8765, 8765)
+    assert _install_sysctl.call_count == 1
+    assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1

From 9aecda0f9437d3ca9eb59bd0a9385357e838a6ce Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 13:31:26 -0400
Subject: [PATCH 0079/2492] cephadm: add unit test coverage for deploying iscsi

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 39 ++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index fdc6f582dbdc..c392c9f4d798 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -184,3 +184,42 @@ def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
         assert (si.st_uid, si.st_gid) == (8765, 8765)
     assert _install_sysctl.call_count == 1
     assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1
+
+
+def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'iscsi.wuzzy'
+        ctx.image = 'quay.io/ayeaye/iscsi:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'iscsi-gateway.cfg': 'portal',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/iscsi.wuzzy')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith('quay.io/ayeaye/iscsi:latest')
+    _firewalld().open_ports.assert_not_called()
+    with open(basedir / 'config') as f:
+        assert f.read() == 'XXXXXXX'
+    with open(basedir / 'keyring') as f:
+        assert f.read() == 'YYYYYY'
+    assert (basedir / 'configfs').is_dir()
+    si = (basedir / 'configfs').stat()
+    assert (si.st_uid, si.st_gid) == (8765, 8765)
+    with open(basedir / 'iscsi-gateway.cfg') as f:
+        assert f.read() == 'portal'
+        si = os.fstat(f.fileno())
+        assert (si.st_uid, si.st_gid) == (8765, 8765)

From 785e4a623d47bb1118914146d94411b4a1556ac1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 13:39:12 -0400
Subject: [PATCH 0080/2492] cephadm: add unit test coverage for deploying
 nvmeof

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 39 ++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index c392c9f4d798..d11aca065fbf 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -223,3 +223,42 @@ def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
         assert f.read() == 'portal'
         si = os.fstat(f.fileno())
         assert (si.st_uid, si.st_gid) == (8765, 8765)
+
+
+def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'nvmeof.andu'
+        ctx.image = 'quay.io/ownf/nmve:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'ceph-nvmeof.conf': 'icantbeliveitsnotiscsi',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/nvmeof.andu')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith('quay.io/ownf/nmve:latest')
+    _firewalld().open_ports.assert_not_called()
+    with open(basedir / 'config') as f:
+        assert f.read() == 'XXXXXXX'
+    with open(basedir / 'keyring') as f:
+        assert f.read() == 'YYYYYY'
+    assert (basedir / 'configfs').is_dir()
+    si = (basedir / 'configfs').stat()
+    assert (si.st_uid, si.st_gid) == (167, 167)
+    with open(basedir / 'ceph-nvmeof.conf') as f:
+        assert f.read() == 'icantbeliveitsnotiscsi'
+        si = os.fstat(f.fileno())
+        assert (si.st_uid, si.st_gid) == (167, 167)

From 5c49070a058beb96a95ed53395b7ae2b87fb3dde Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 13:56:36 -0400
Subject: [PATCH 0081/2492] cephadm: add unit test coverage for deploying
 monitoring

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 38 ++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index d11aca065fbf..fad142ebbabb 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -262,3 +262,41 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
         assert f.read() == 'icantbeliveitsnotiscsi'
         si = os.fstat(f.fileno())
         assert (si.st_uid, si.st_gid) == (167, 167)
+
+
+def test_deploy_a_monitoring_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _get_ip_addresses = mock.MagicMock(return_value=(['10.10.10.10'], []))
+    monkeypatch.setattr('cephadm.get_ip_addresses', _get_ip_addresses)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'prometheus.fire'
+        ctx.image = 'quay.io/titans/prometheus:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'prometheus.yml': 'bettercallherc',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/prometheus.fire')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith(
+        'quay.io/titans/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.listen-address=:9095 --storage.tsdb.retention.time=15d --storage.tsdb.retention.size=0 --web.external-url=http://10.10.10.10:9095'
+    )
+    _firewalld().open_ports.assert_not_called()
+    assert not (basedir / 'config').exists()
+    assert not (basedir / 'keyring').exists()
+    with open(basedir / 'etc/prometheus/prometheus.yml') as f:
+        assert f.read() == 'bettercallherc'
+        si = os.fstat(f.fileno())
+        assert (si.st_uid, si.st_gid) == (8765, 8765)

From 8f82ef32c8b991ba777d5436e3cc69937805bc9f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 14:17:02 -0400
Subject: [PATCH 0082/2492] cephadm: add unit test coverage for deploying
 tracing

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index fad142ebbabb..e6b4da8127fc 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -300,3 +300,33 @@ def test_deploy_a_monitoring_container(cephadm_fs, monkeypatch):
         assert f.read() == 'bettercallherc'
         si = os.fstat(f.fileno())
         assert (si.st_uid, si.st_gid) == (8765, 8765)
+
+
+def test_deploy_a_tracing_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'elasticsearch.band'
+        ctx.image = 'quay.io/rubber/elasticsearch:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'prometheus.yml': 'bettercallherc',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/elasticsearch.band')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith('quay.io/rubber/elasticsearch:latest')
+    _firewalld().open_ports.assert_not_called()
+    assert not (basedir / 'config').exists()
+    assert not (basedir / 'keyring').exists()

From 04e2ac04b68457824ec69bc0a712d595915e4f81 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 14:28:31 -0400
Subject: [PATCH 0083/2492] cephadm: add unit test coverage for deploying ceph
 container

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 38 ++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index e6b4da8127fc..a5511a34d1d3 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -330,3 +330,41 @@ def test_deploy_a_tracing_container(cephadm_fs, monkeypatch):
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()
+
+
+def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _make_var_run = mock.MagicMock()
+    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'mgr.foo'
+        ctx.image = 'quay.io/ceph/ceph:latest'
+        ctx.reconfig = False
+        ctx.allow_ptrace = False
+        ctx.osd_fsid = '00000000-0000-0000-0000-000000000000'
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/mgr.foo')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith(
+        'quay.io/ceph/ceph:latest -n mgr.foo -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false'
+    )
+    _firewalld().open_ports.assert_not_called()
+    with open(basedir / 'config') as f:
+        assert f.read() == 'XXXXXXX'
+    with open(basedir / 'keyring') as f:
+        assert f.read() == 'YYYYYY'
+    assert _make_var_run.call_count == 1
+    assert _make_var_run.call_args[0][2] == 8765
+    assert _make_var_run.call_args[0][3] == 8765

From 89dd3719a01323025da5ed7fd95ca27bd96af16b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 14:43:51 -0400
Subject: [PATCH 0084/2492] cephadm: add unit test coverage for deploying
 ceph-exporter

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 45 ++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index a5511a34d1d3..1f4eddf3a70d 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -368,3 +368,48 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
     assert _make_var_run.call_count == 1
     assert _make_var_run.call_args[0][2] == 8765
     assert _make_var_run.call_args[0][3] == 8765
+
+
+def test_deploy_ceph_exporter_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _get_ip_addresses = mock.MagicMock(return_value=(['10.10.10.10'], []))
+    monkeypatch.setattr('cephadm.get_ip_addresses', _get_ip_addresses)
+    _make_var_run = mock.MagicMock()
+    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'ceph-exporter.zaq'
+        ctx.image = 'quay.io/ceph/ceph:latest'
+        ctx.reconfig = False
+        ctx.allow_ptrace = False
+        ctx.osd_fsid = '00000000-0000-0000-0000-000000000000'
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'prio-limit': 12,
+        }
+
+        # ceph-exporter is weird and special. it requires the "sock dir"
+        # to already exist. that dir defaults to /var/run/ceph
+        vrc = pathlib.Path('/var/run/ceph')
+        (vrc / fsid).mkdir(parents=True)
+
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/ceph-exporter.zaq')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith(
+        'quay.io/ceph/ceph:latest -n client.ceph-exporter.zaq -f --sock-dir=/var/run/ceph/ --addrs=0.0.0.0 --port=9926 --prio-limit=12 --stats-period=5'
+    )
+    assert '--entrypoint /usr/bin/ceph-exporter' in runfile_lines[-1]
+    _firewalld().open_ports.assert_not_called()
+    with open(basedir / 'config') as f:
+        assert f.read() == 'XXXXXXX'
+    with open(basedir / 'keyring') as f:
+        assert f.read() == 'YYYYYY'

From 14664264e51eec8e62119439f92dffd3cc487607 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 16:33:46 -0400
Subject: [PATCH 0085/2492] cephadm: remove gateways property from Ceph type

The gateways list property of the Ceph type was used in exactly one
place. In order to provide a clearer structure to the classes in
cephadm, remove this property and simply list the daemon types
specifically where they are needed.

In the future, I hope to see these handled in a class/object based
manner but this is still better for now.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 4901abf42cd3..ed6841bbae89 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -211,7 +211,6 @@ def __eq__(self, other: Any) -> bool:
 class Ceph(DaemonForm):
     daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
                'crash', 'cephfs-mirror', 'ceph-exporter')
-    gateways = ('iscsi', 'nfs', 'nvmeof')
 
     @classmethod
     def for_daemon_type(cls, daemon_type: str) -> bool:
@@ -2553,7 +2552,11 @@ def get_container(
         envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
     if container_args is None:
         container_args = []
-    if daemon_type in Ceph.daemons or daemon_type in Ceph.gateways:
+    unlimited_daemons = set(Ceph.daemons)
+    unlimited_daemons.add(CephIscsi.daemon_type)
+    unlimited_daemons.add(CephNvmeof.daemon_type)
+    unlimited_daemons.add(NFSGanesha.daemon_type)
+    if daemon_type in unlimited_daemons:
         set_pids_limit_unlimited(ctx, container_args)
     if daemon_type in ['mon', 'osd']:
         # mon and osd need privileged in order for libudev to query devices

From 1e531f4854e742091d02d15c8bef7d7e0f70fe61 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:00:03 -0400
Subject: [PATCH 0086/2492] cephadm: convert keepalived type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 22 ++++++----------------
 src/cephadm/tests/test_ingress.py |  2 +-
 2 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ed6841bbae89..1d5c55369971 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1240,7 +1240,7 @@ def get_sysctl_settings() -> List[str]:
 
 
 @register_daemon_form
-class Keepalived(DaemonForm):
+class Keepalived(ContainerDaemonForm):
     """Defines an Keepalived container"""
     daemon_type = 'keepalived'
     required_files = ['keepalived.conf']
@@ -1335,7 +1335,7 @@ def get_sysctl_settings() -> List[str]:
             'net.ipv4.ip_nonlocal_bind = 1',
         ]
 
-    def extract_uid_gid_keepalived(self) -> Tuple[int, int]:
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         # better directory for this?
         return extract_uid_gid(self.ctx, file_path='/var/lib')
 
@@ -1345,6 +1345,10 @@ def get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
         return mounts
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        return get_deployment_container(ctx, self.identity)
+
+
 ##################################
 
 
@@ -5235,20 +5239,6 @@ def _dispatch_deploy(
             endpoints=daemon_endpoints,
         )
 
-    elif daemon_type == Keepalived.daemon_type:
-        keepalived = Keepalived.init(ctx, ident.fsid, ident.daemon_id)
-        uid, gid = keepalived.extract_uid_gid_keepalived()
-        c = get_deployment_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
-        )
-
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
         uid = os.getuid()
diff --git a/src/cephadm/tests/test_ingress.py b/src/cephadm/tests/test_ingress.py
index 798c73708686..51a6e113bc4a 100644
--- a/src/cephadm/tests/test_ingress.py
+++ b/src/cephadm/tests/test_ingress.py
@@ -331,7 +331,7 @@ def test_keepalived_extract_uid_gid_keepalived():
         )
         with mock.patch("cephadm.CephContainer") as cc:
             cc.return_value.run.return_value = "500 500"
-            uid, gid = kad.extract_uid_gid_keepalived()
+            uid, gid = kad.uid_gid(ctx)
             cc.return_value.run.assert_called()
         assert uid == 500
         assert gid == 500

From aaffe62c0b224039db6c70097b7122e0ad72013a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:06:12 -0400
Subject: [PATCH 0087/2492] cephadm: convert haproxy type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 20 +++++---------------
 src/cephadm/tests/test_ingress.py |  2 +-
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 1d5c55369971..bd116fd00636 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1139,7 +1139,7 @@ def validate(self) -> None:
 
 
 @register_daemon_form
-class HAproxy(DaemonForm):
+class HAproxy(ContainerDaemonForm):
     """Defines an HAproxy container"""
     daemon_type = 'haproxy'
     required_files = ['haproxy.cfg']
@@ -1218,7 +1218,7 @@ def get_container_name(self, desc=None):
             cname = '%s-%s' % (cname, desc)
         return cname
 
-    def extract_uid_gid_haproxy(self) -> Tuple[int, int]:
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         # better directory for this?
         return extract_uid_gid(self.ctx, file_path='/var/lib')
 
@@ -1236,6 +1236,9 @@ def get_sysctl_settings() -> List[str]:
             'net.ipv4.ip_nonlocal_bind = 1',
         ]
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        return get_deployment_container(ctx, self.identity)
+
 ##################################
 
 
@@ -5225,19 +5228,6 @@ def _dispatch_deploy(
             deployment_type=deployment_type,
             endpoints=daemon_endpoints,
         )
-    elif daemon_type == HAproxy.daemon_type:
-        haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
-        uid, gid = haproxy.extract_uid_gid_haproxy()
-        c = get_deployment_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
-        )
 
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
diff --git a/src/cephadm/tests/test_ingress.py b/src/cephadm/tests/test_ingress.py
index 51a6e113bc4a..08a9808ddbba 100644
--- a/src/cephadm/tests/test_ingress.py
+++ b/src/cephadm/tests/test_ingress.py
@@ -168,7 +168,7 @@ def test_haproxy_extract_uid_gid_haproxy():
         )
         with mock.patch("cephadm.CephContainer") as cc:
             cc.return_value.run.return_value = "500 500"
-            uid, gid = hap.extract_uid_gid_haproxy()
+            uid, gid = hap.uid_gid(ctx)
             cc.return_value.run.assert_called()
         assert uid == 500
         assert gid == 500

From 04b2f4cddd8c1d132e3b6c31357b7455ac4b02da Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:09:47 -0400
Subject: [PATCH 0088/2492] cephadm: convert tracing type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index bd116fd00636..5582e1ef692f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1356,7 +1356,7 @@ def container(self, ctx: CephadmContext) -> CephContainer:
 
 
 @register_daemon_form
-class Tracing(DaemonForm):
+class Tracing(ContainerDaemonForm):
     """Define the configs for the jaeger tracing containers"""
 
     components: Dict[str, Dict[str, Any]] = {
@@ -1404,6 +1404,14 @@ def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Tracing':
     def identity(self) -> DaemonIdentity:
         return self._identity
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        # TODO(jjm) this looks to be the only container for deployment
+        # not using get_deployment_container. Previous oversight?
+        return get_container(ctx, self.identity)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 65534, 65534
+
 ##################################
 
 
@@ -5216,18 +5224,6 @@ def _dispatch_deploy(
             deployment_type=deployment_type,
             endpoints=daemon_endpoints,
         )
-    elif daemon_type in Tracing.components:
-        uid, gid = 65534, 65534
-        c = get_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
-        )
 
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid

From 1fcacd6b20838968438b05d4cbb832f8c2c97a07 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:12:48 -0400
Subject: [PATCH 0089/2492] cephadm: convert nvmeof type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5582e1ef692f..7511b4cf17c2 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -928,7 +928,7 @@ def get_tcmu_runner_container(self):
 
 
 @register_daemon_form
-class CephNvmeof(DaemonForm):
+class CephNvmeof(ContainerDaemonForm):
     """Defines a Ceph-Nvmeof container"""
 
     daemon_type = 'nvmeof'
@@ -1061,6 +1061,17 @@ def get_sysctl_settings() -> List[str]:
             'vm.nr_hugepages = 4096',
         ]
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        return get_deployment_container(ctx, self.identity)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 167, 167  # TODO: need to get properly the uid/gid
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
 
 ##################################
 
@@ -5209,21 +5220,6 @@ def _dispatch_deploy(
             deployment_type=deployment_type,
             endpoints=daemon_endpoints
         )
-    elif daemon_type == CephNvmeof.daemon_type:
-        config, keyring = get_config_and_keyring(ctx)
-        uid, gid = 167, 167  # TODO: need to get properly the uid/gid
-        c = get_deployment_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            config=config,
-            keyring=keyring,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
-        )
 
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid

From abf39a669f04b130b248c2edc0e7102bbb61b2ec Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:15:37 -0400
Subject: [PATCH 0090/2492] cephadm: convert iscsi type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7511b4cf17c2..6faf348a7fde 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -728,7 +728,7 @@ def config_and_keyring(
 
 
 @register_daemon_form
-class CephIscsi(DaemonForm):
+class CephIscsi(ContainerDaemonForm):
     """Defines a Ceph-Iscsi container"""
 
     daemon_type = 'iscsi'
@@ -923,6 +923,17 @@ def get_tcmu_runner_container(self):
         tcmu_container.cname = self.get_container_name(desc='tcmu')
         return tcmu_container
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        return get_deployment_container(ctx, self.identity)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return extract_uid_gid(ctx)
+
 
 ##################################
 
@@ -5205,22 +5216,6 @@ def _dispatch_deploy(
             endpoints=daemon_endpoints
         )
 
-    elif daemon_type == CephIscsi.daemon_type:
-        config, keyring = get_config_and_keyring(ctx)
-        uid, gid = extract_uid_gid(ctx)
-        c = get_deployment_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            config=config,
-            keyring=keyring,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints
-        )
-
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
         uid = os.getuid()

From de6ccf0be4f08fb1fd3bdf301689d98f60295bc0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:21:51 -0400
Subject: [PATCH 0091/2492] cephadm: move extract_uid_gid_monitoring to
 Monitoring class

Make it a method of the class handling monitoring, such as other classes
already are.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 52 ++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 25 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6faf348a7fde..403575f0bf4b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -541,6 +541,28 @@ def get_version(ctx, container_id, daemon_type):
                 version = out.split(' ')[2]
         return version
 
+    @staticmethod
+    def extract_uid_gid(
+        ctx: CephadmContext, daemon_type: str
+    ) -> Tuple[int, int]:
+        if daemon_type == 'prometheus':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus')
+        elif daemon_type == 'node-exporter':
+            uid, gid = 65534, 65534
+        elif daemon_type == 'grafana':
+            uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana')
+        elif daemon_type == 'loki':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/loki')
+        elif daemon_type == 'promtail':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/promtail')
+        elif daemon_type == 'alertmanager':
+            uid, gid = extract_uid_gid(
+                ctx, file_path=['/etc/alertmanager', '/etc/prometheus']
+            )
+        else:
+            raise Error('{} not implemented yet'.format(daemon_type))
+        return uid, gid
+
     def __init__(self, ident: DaemonIdentity) -> None:
         self._identity = ident
 
@@ -2654,7 +2676,7 @@ def get_container(
         container_args.extend(cc.get_container_args())
 
     if daemon_type in Monitoring.components:
-        uid, gid = extract_uid_gid_monitoring(ctx, daemon_type)
+        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
         monitoring_args = [
             '--user',
             str(uid),
@@ -5013,26 +5035,6 @@ def command_registry_login(ctx: CephadmContext) -> int:
 ##################################
 
 
-def extract_uid_gid_monitoring(ctx, daemon_type):
-    # type: (CephadmContext, str) -> Tuple[int, int]
-
-    if daemon_type == 'prometheus':
-        uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus')
-    elif daemon_type == 'node-exporter':
-        uid, gid = 65534, 65534
-    elif daemon_type == 'grafana':
-        uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana')
-    elif daemon_type == 'loki':
-        uid, gid = extract_uid_gid(ctx, file_path='/etc/loki')
-    elif daemon_type == 'promtail':
-        uid, gid = extract_uid_gid(ctx, file_path='/etc/promtail')
-    elif daemon_type == 'alertmanager':
-        uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus'])
-    else:
-        raise Error('{} not implemented yet'.format(daemon_type))
-    return uid, gid
-
-
 def get_deployment_container(
     ctx: CephadmContext,
     ident: 'DaemonIdentity',
@@ -5204,7 +5206,7 @@ def _dispatch_deploy(
                 raise Error('{} deployment requires config-json which must '
                             'contain arg for {}'.format(daemon_type.capitalize(), ', '.join(required_args)))
 
-        uid, gid = extract_uid_gid_monitoring(ctx, daemon_type)
+        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
         c = get_deployment_container(ctx, ident)
         deploy_daemon(
             ctx,
@@ -6094,7 +6096,7 @@ def command_adopt_ceph(ctx, daemon_type, daemon_id, fsid):
 def command_adopt_prometheus(ctx, daemon_id, fsid):
     # type: (CephadmContext, str, str) -> None
     daemon_type = 'prometheus'
-    (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+    (uid, gid) = Monitoring.extract_uid_gid(ctx, daemon_type)
     # should try to set the ports we know cephadm defaults
     # to for these services in the firewall.
     ports = Monitoring.port_map['prometheus']
@@ -6141,7 +6143,7 @@ def command_adopt_grafana(ctx, daemon_id, fsid):
     # type: (CephadmContext, str, str) -> None
 
     daemon_type = 'grafana'
-    (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+    (uid, gid) = Monitoring.extract_uid_gid(ctx, daemon_type)
     # should try to set the ports we know cephadm defaults
     # to for these services in the firewall.
     ports = Monitoring.port_map['grafana']
@@ -6212,7 +6214,7 @@ def command_adopt_alertmanager(ctx, daemon_id, fsid):
     # type: (CephadmContext, str, str) -> None
 
     daemon_type = 'alertmanager'
-    (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+    (uid, gid) = Monitoring.extract_uid_gid(ctx, daemon_type)
     # should try to set the ports we know cephadm defaults
     # to for these services in the firewall.
     ports = Monitoring.port_map['alertmanager']

From 9015edc3f34bbaf1906b9e5c83be7c49ef7c42c8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 27 Sep 2023 18:30:17 -0400
Subject: [PATCH 0092/2492] cephadm: convert monitoring type to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 66 +++++++++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 29 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 403575f0bf4b..6c87d7983b6f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -422,7 +422,7 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
 
 ##################################
 @register_daemon_form
-class Monitoring(DaemonForm):
+class Monitoring(ContainerDaemonForm):
     """Define the configs for the monitoring containers"""
 
     port_map = {
@@ -574,6 +574,42 @@ def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Monitoring':
     def identity(self) -> DaemonIdentity:
         return self._identity
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        self._prevalidate(ctx)
+        return get_deployment_container(ctx, self.identity)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return self.extract_uid_gid(ctx, self.identity.daemon_type)
+
+    def _prevalidate(self, ctx: CephadmContext) -> None:
+        # before being refactored into a ContainerDaemonForm these checks were
+        # done inside the deploy function. This was the only "family" of daemons
+        # that performed these checks in that location
+        daemon_type = self.identity.daemon_type
+        config = fetch_configs(ctx)  # type: ignore
+        required_files = self.components[daemon_type].get(
+            'config-json-files', list()
+        )
+        required_args = self.components[daemon_type].get(
+            'config-json-args', list()
+        )
+        if required_files:
+            if not config or not all(c in config.get('files', {}).keys() for c in required_files):  # type: ignore
+                raise Error(
+                    '{} deployment requires config-json which must '
+                    'contain file content for {}'.format(
+                        daemon_type.capitalize(), ', '.join(required_files)
+                    )
+                )
+        if required_args:
+            if not config or not all(c in config.keys() for c in required_args):  # type: ignore
+                raise Error(
+                    '{} deployment requires config-json which must '
+                    'contain arg for {}'.format(
+                        daemon_type.capitalize(), ', '.join(required_args)
+                    )
+                )
+
 ##################################
 
 
@@ -5190,34 +5226,6 @@ def _dispatch_deploy(
             endpoints=daemon_endpoints,
         )
 
-    elif daemon_type in Monitoring.components:
-        # monitoring daemon - prometheus, grafana, alertmanager, node-exporter
-        # Default Checks
-        # make sure provided config-json is sufficient
-        config = fetch_configs(ctx)  # type: ignore
-        required_files = Monitoring.components[daemon_type].get('config-json-files', list())
-        required_args = Monitoring.components[daemon_type].get('config-json-args', list())
-        if required_files:
-            if not config or not all(c in config.get('files', {}).keys() for c in required_files):  # type: ignore
-                raise Error('{} deployment requires config-json which must '
-                            'contain file content for {}'.format(daemon_type.capitalize(), ', '.join(required_files)))
-        if required_args:
-            if not config or not all(c in config.keys() for c in required_args):  # type: ignore
-                raise Error('{} deployment requires config-json which must '
-                            'contain arg for {}'.format(daemon_type.capitalize(), ', '.join(required_args)))
-
-        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
-        c = get_deployment_container(ctx, ident)
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints
-        )
-
     elif daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
         uid = os.getuid()

From ffe1f2f8f159749905e224dbaa06f79681063b08 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 11:02:37 -0400
Subject: [PATCH 0093/2492] cephadm: update test to avoid using exception
 handling as an assertion

The use of an exception as an assertion mostly works but has the side
effect of hiding other errors. Hiding these errors can make it hard to
debug problems in this code path, as it did for me recently. Update the
test to use a standard assertion as well as asserting that the assertion
must have been called.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index ff474c23ccd9..7e31b26307c3 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -423,13 +423,12 @@ def test_mon_crush_location(self, _get_deployment_container, _migrate_sysctl, _m
         )
 
         def _crush_location_checker(ctx, ident, container, uid, gid, **kwargs):
-            print(container.args)
-            raise Exception(' '.join(container.args))
+            argval = ' '.join(container.args)
+            assert '--set-crush-location database=a' in argval
 
         _deploy_daemon.side_effect = _crush_location_checker
-
-        with pytest.raises(Exception, match='--set-crush-location database=a'):
-            _cephadm.command_deploy_from(ctx)
+        _cephadm.command_deploy_from(ctx)
+        _deploy_daemon.assert_called()
 
     @mock.patch('cephadm.logger')
     @mock.patch('cephadm.fetch_custom_config_files')

From b911cf42c058cde51ce5a57234845136e30743bd Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Sep 2023 20:46:50 -0400
Subject: [PATCH 0094/2492] qa: narrow search to debug_asok

To avoid matching debug_asok_assert_abort.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/workunits/mon/config.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/workunits/mon/config.sh b/qa/workunits/mon/config.sh
index 1b00201ae481..9a62081c6680 100755
--- a/qa/workunits/mon/config.sh
+++ b/qa/workunits/mon/config.sh
@@ -98,11 +98,11 @@ ceph tell osd.0 config unset debug_asok
 ceph tell osd.0 config unset debug_asok
 
 ceph config rm osd.0 debug_asok
-while ceph config show osd.0 | grep debug_asok | grep mon
+while ceph config show osd.0 | grep '^debug_asok[:[space]:]' | grep mon
 do
     sleep 1
 done
-ceph config show osd.0 | grep -c debug_asok | grep 0
+ceph config show osd.0 | grep -c '^debug_asok[:[space]:]' | grep 0
 
 ceph config set osd.0 osd_scrub_cost 123
 while ! ceph config show osd.0 | grep osd_scrub_cost | grep mon

From cd1833ab152d0231c1c6f3685238f4fcd3284f6e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Sep 2023 20:47:40 -0400
Subject: [PATCH 0095/2492] qa: add reproducer for obs removal deadlock

And hopefully this "config set" race test may catch future bugs!

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/workunits/mon/config.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/qa/workunits/mon/config.sh b/qa/workunits/mon/config.sh
index 9a62081c6680..7d97a452bd89 100755
--- a/qa/workunits/mon/config.sh
+++ b/qa/workunits/mon/config.sh
@@ -130,6 +130,21 @@ rm -f $t1 $t2
 
 expect_false ceph config reset
 expect_false ceph config reset -1
+
+
+# test parallel config set
+# reproducer for https://tracker.ceph.com/issues/62832
+ceph config reset 0
+for ((try = 0; try < 10; try++)); do
+    set +x
+    for ((i = 0; i < 100; i++)); do
+        # Use a config that will get "handled" by the Objecter instantiated by the ceph binary
+        ceph config set client rados_mon_op_timeout $((i+300)) &
+    done 2> /dev/null
+    set -x
+    wait
+done
+
 # we are at end of testing, so it's okay to revert everything
 ceph config reset 0
 

From a8bd314bbd78dbe73371e7a8beaaa1929577b76e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 3 Oct 2023 20:45:12 +0000
Subject: [PATCH 0096/2492] common/ceph_mutex: note whether mutex debug methods
 are usable

So we can do checks like:

    ceph_assert(!ceph::mutex_debugging || !ceph_mutex_is_locked_by_me(lock));

Without this boolean, this check:

    ceph_assert(!ceph_mutex_is_locked_by_me(lock));

will fail for all crimson/release builds because the method always returns true.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/ceph_mutex.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/common/ceph_mutex.h b/src/common/ceph_mutex.h
index 6ed8c56d5dad..059d81f2ac39 100644
--- a/src/common/ceph_mutex.h
+++ b/src/common/ceph_mutex.h
@@ -83,6 +83,7 @@ namespace ceph {
     return {};
   }
 
+  static constexpr bool mutex_debugging = false;
   #define ceph_mutex_is_locked(m) true
   #define ceph_mutex_is_locked_by_me(m) true
 }
@@ -130,6 +131,8 @@ namespace ceph {
     return {std::forward<Args>(args)...};
   }
 
+  static constexpr bool mutex_debugging = true;
+
   // debug methods
   #define ceph_mutex_is_locked(m) ((m).is_locked())
   #define ceph_mutex_is_not_locked(m) (!(m).is_locked())
@@ -183,6 +186,8 @@ namespace ceph {
     return {};
   }
 
+  static constexpr bool mutex_debugging = false;
+
   // debug methods.  Note that these can blindly return true
   // because any code that does anything other than assert these
   // are true is broken.

From 7b5076bafeb8981cd809437ee04fc570857a05f4 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 20 Sep 2023 22:00:03 -0400
Subject: [PATCH 0097/2492] common: add missing locks in config_proxy methods

It's not generally safe to access the md_config_t without these locks. Some
methods are probably harmless (accessing read-only state) but best to be
consistent.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/config_proxy.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/common/config_proxy.h b/src/common/config_proxy.h
index 02c670f60277..400aa4ed052d 100644
--- a/src/common/config_proxy.h
+++ b/src/common/config_proxy.h
@@ -150,12 +150,15 @@ class ConfigProxy {
 				       std::forward<Args>(args)...);
   }
   void config_options(ceph::Formatter *f) const {
+    std::lock_guard l{lock};
     config.config_options(f);
   }
   const decltype(md_config_t::schema)& get_schema() const {
+    std::lock_guard l{lock};
     return config.schema;
   }
   const Option* get_schema(const std::string_view key) const {
+    std::lock_guard l{lock};
     auto found = config.schema.find(key);
     if (found == config.schema.end()) {
       return nullptr;
@@ -164,6 +167,7 @@ class ConfigProxy {
     }
   }
   const Option *find_option(const std::string& name) const {
+    std::lock_guard l{lock};
     return config.find_option(name);
   }
   void diff(ceph::Formatter *f, const std::string& name = {}) const {
@@ -186,6 +190,7 @@ class ConfigProxy {
 					 sections, key, out, emeta);
   }
   unsigned get_osd_pool_default_min_size(uint8_t size) const {
+    std::lock_guard l{lock};
     return config.get_osd_pool_default_min_size(values, size);
   }
   void early_expand_meta(std::string &val,
@@ -225,9 +230,11 @@ class ConfigProxy {
     call_observers(locker, rev_obs);
   }
   void set_safe_to_start_threads() {
+    std::lock_guard l(lock);
     config.set_safe_to_start_threads();
   }
   void _clear_safe_to_start_threads() {
+    std::lock_guard l(lock);
     config._clear_safe_to_start_threads();
   }
   void show_config(std::ostream& out) {
@@ -319,12 +326,15 @@ class ConfigProxy {
 				     conf_files, warnings, flags);
   }
   bool has_parse_error() const {
+    std::lock_guard l(lock);
     return !config.parse_error.empty();
   }
   std::string get_parse_error() {
+    std::lock_guard l(lock);
     return config.parse_error;
   }
   void complain_about_parse_error(CephContext *cct) {
+    std::lock_guard l(lock);
     return config.complain_about_parse_error(cct);
   }
   void do_argv_commands() const {
@@ -342,9 +352,11 @@ class ConfigProxy {
     config.get_defaults_bl(values, bl);
   }
   const std::string& get_conf_path() const {
+    std::lock_guard l(lock);
     return config.get_conf_path();
   }
   std::optional<std::string> get_val_default(std::string_view key) {
+    std::lock_guard l(lock);
     return config.get_val_default(key);
   }
 };

From 6fa7167732085d651285a11fd85c1bc33a2afcd7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 11:11:09 -0400
Subject: [PATCH 0098/2492] qa/rgw/tempest: rearrange tasks and overrides

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/tempest/0-install.yaml          | 15 +++++++++
 qa/suites/rgw/tempest/overrides.yaml          | 14 ++++++++
 .../tasks/{rgw_tempest.yaml => tempest.yaml}  | 33 -------------------
 3 files changed, 29 insertions(+), 33 deletions(-)
 create mode 100644 qa/suites/rgw/tempest/0-install.yaml
 rename qa/suites/rgw/tempest/tasks/{rgw_tempest.yaml => tempest.yaml} (67%)

diff --git a/qa/suites/rgw/tempest/0-install.yaml b/qa/suites/rgw/tempest/0-install.yaml
new file mode 100644
index 000000000000..fc2cfcc7b6d2
--- /dev/null
+++ b/qa/suites/rgw/tempest/0-install.yaml
@@ -0,0 +1,15 @@
+tasks:
+- install:
+- ceph:
+- tox: [ client.0 ]
+- keystone:
+    client.0:
+      force-branch: stable/2023.1
+      services:
+        - name: swift
+          type: object-store
+          description: Swift Service
+- rgw:
+    client.0:
+      frontend_prefix: /swift
+      use-keystone-role: client.0
diff --git a/qa/suites/rgw/tempest/overrides.yaml b/qa/suites/rgw/tempest/overrides.yaml
index e7a292ffd1fd..9fb0e11e9493 100644
--- a/qa/suites/rgw/tempest/overrides.yaml
+++ b/qa/suites/rgw/tempest/overrides.yaml
@@ -1,7 +1,21 @@
 overrides:
   ceph:
     conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
       client:
         setuser: ceph
         setgroup: ceph
         debug rgw: 20
+        rgw keystone api version: 3
+        rgw keystone accepted roles: admin,member
+        rgw keystone implicit tenants: true
+        rgw keystone accepted admin roles: admin
+        rgw swift enforce content length: true
+        rgw swift account in url: true
+        rgw swift versioning enabled: true
+        rgw keystone admin domain: Default
+        rgw keystone admin user: admin
+        rgw keystone admin password: ADMIN
+        rgw keystone admin project: admin
diff --git a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml
similarity index 67%
rename from qa/suites/rgw/tempest/tasks/rgw_tempest.yaml
rename to qa/suites/rgw/tempest/tasks/tempest.yaml
index ad9dc9dd5025..560e41de5bbb 100644
--- a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml
+++ b/qa/suites/rgw/tempest/tasks/tempest.yaml
@@ -1,18 +1,4 @@
 tasks:
-- install:
-- ceph:
-- tox: [ client.0 ]
-- keystone:
-    client.0:
-      force-branch: stable/2023.1
-      services:
-        - name: swift
-          type: object-store
-          description: Swift Service
-- rgw:
-    client.0:
-      frontend_prefix: /swift
-      use-keystone-role: client.0
 - tempest:
     client.0:
       sha1: 34.1.0
@@ -51,22 +37,3 @@ tasks:
         - .*test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time
         - .*test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time
         - .*test_account_services.AccountTest.test_list_no_account_metadata
-
-overrides:
-  ceph:
-    conf:
-      global:
-        osd_min_pg_log_entries: 10
-        osd_max_pg_log_entries: 10
-      client:
-        rgw keystone api version: 3
-        rgw keystone accepted roles: admin,member
-        rgw keystone implicit tenants: true
-        rgw keystone accepted admin roles: admin
-        rgw swift enforce content length: true
-        rgw swift account in url: true
-        rgw swift versioning enabled: true
-        rgw keystone admin domain: Default
-        rgw keystone admin user: admin
-        rgw keystone admin password: ADMIN
-        rgw keystone admin project: admin

From f19d05dcab8093424d9cd50943972fa7f1942f92 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 17:03:40 -0400
Subject: [PATCH 0099/2492] qa/keystone: merge overrides into config

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/keystone.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py
index 7aa785055c21..f0f09842e605 100644
--- a/qa/tasks/keystone.py
+++ b/qa/tasks/keystone.py
@@ -463,6 +463,8 @@ def task(ctx, config):
         config = all_clients
     if isinstance(config, list):
         config = dict.fromkeys(config)
+    overrides = ctx.config.get('overrides', {})
+    teuthology.deep_merge(config, overrides.get('keystone', {}))
 
     log.debug('Keystone config is %s', config)
 

From 86b8b4b0198045b733235315c167ff1cb249a16b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:00:28 -0400
Subject: [PATCH 0100/2492] qa/keystone: extract os_auth_args to helper
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/keystone.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py
index f0f09842e605..27e808f68835 100644
--- a/qa/tasks/keystone.py
+++ b/qa/tasks/keystone.py
@@ -326,25 +326,26 @@ def dict_to_args(specials, items):
     args.extend(arg for arg in special_vals.values() if arg)
     return args
 
+def os_auth_args(host, port):
+    return [
+        '--os-username', 'admin',
+        '--os-password', 'ADMIN',
+        '--os-user-domain-id', 'default',
+        '--os-project-name', 'admin',
+        '--os-project-domain-id', 'default',
+        '--os-identity-api-version', '3',
+        '--os-auth-url', 'http://{host}:{port}/v3'.format(host=host, port=port),
+    ]
+
 def run_section_cmds(ctx, cclient, section_cmd, specials,
                      section_config_list):
     public_host, public_port = ctx.keystone.public_endpoints[cclient]
-
-    auth_section = [
-        ( 'os-username', 'admin' ),
-        ( 'os-password', 'ADMIN' ),
-        ( 'os-user-domain-id', 'default' ),
-        ( 'os-project-name', 'admin' ),
-        ( 'os-project-domain-id', 'default' ),
-        ( 'os-identity-api-version', '3' ),
-        ( 'os-auth-url', 'http://{host}:{port}/v3'.format(host=public_host,
-                                                          port=public_port) ),
-    ]
+    auth_args = os_auth_args(public_host, public_port)
 
     for section_item in section_config_list:
         run_in_keystone_venv(ctx, cclient,
-            [ 'openstack' ] + section_cmd.split() +
-            dict_to_args(specials, auth_section + list(section_item.items())) +
+            [ 'openstack' ] + section_cmd.split() + auth_args +
+            dict_to_args(specials, list(section_item.items())) +
             [ '--debug' ])
 
 def create_endpoint(ctx, cclient, service, url, adminurl=None):

From 1456d12a9723f509717d706c09b252f9b810b300 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:01:15 -0400
Subject: [PATCH 0101/2492] qa/keystone: config can create ec2 credentials

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/keystone.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py
index 27e808f68835..fc45b9c2938d 100644
--- a/qa/tasks/keystone.py
+++ b/qa/tasks/keystone.py
@@ -387,6 +387,8 @@ def fill_keystone(ctx, config):
                          cconfig.get('projects', []))
         run_section_cmds(ctx, cclient, 'user create --or-show', 'name',
                          cconfig.get('users', []))
+        run_section_cmds(ctx, cclient, 'ec2 credentials create', '',
+                         cconfig.get('ec2 credentials', []))
         run_section_cmds(ctx, cclient, 'role create --or-show', 'name',
                          cconfig.get('roles', []))
         run_section_cmds(ctx, cclient, 'role add', 'name',
@@ -441,6 +443,9 @@ def task(ctx, config):
               - name: custom
                 password: SECRET
                 project: custom
+            ec2 credentials:
+              - project: custom
+                user: custom
             roles: [ name: custom ]
             role-mappings:
               - name: custom

From f9872e58a1e5c0e0d1ca5d56e72e2333f5d8e810 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:02:24 -0400
Subject: [PATCH 0102/2492] qa/keystone: expose function to read a user's ec2
 creds

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/keystone.py | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py
index fc45b9c2938d..bffeeeae1811 100644
--- a/qa/tasks/keystone.py
+++ b/qa/tasks/keystone.py
@@ -3,6 +3,8 @@
 """
 import argparse
 import contextlib
+from io import StringIO
+import json
 import logging
 
 # still need this for python3.6
@@ -35,12 +37,12 @@ def toxvenv_sh(ctx, remote, args, **kwargs):
     activate = get_toxvenv_dir(ctx) + '/bin/activate'
     return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)
 
-def run_in_keystone_venv(ctx, client, args):
-    run_in_keystone_dir(ctx, client,
+def run_in_keystone_venv(ctx, client, args, **kwargs):
+    return run_in_keystone_dir(ctx, client,
                         [   'source',
                             '.tox/venv/bin/activate',
                             run.Raw('&&')
-                        ] + args)
+                        ] + args, **kwargs)
 
 def get_keystone_venved_cmd(ctx, cmd, args, env=[]):
     kbindir = get_keystone_dir(ctx) + '/.tox/venv/bin/'
@@ -420,6 +422,29 @@ def assign_ports(ctx, config, initial_port):
 
     return role_endpoints
 
+def read_ec2_credentials(ctx, client, user):
+    """
+    Look up EC2 credentials for the given user.
+
+    Returns a dictionary of the form:
+    {
+        "Access": "b2c9a792ff934b50b7e5c6d8f0fbbc96",
+        "Secret": "53b34a24a8e244ca89f1d754f089b63a",
+        "Project ID": "49208b6cc1864a0ea1cd7de3b456db11",
+        "User ID": "3276c0e0116a4a3ab1dd462ae4846416"
+    }
+    """
+    public_host, public_port = ctx.keystone.public_endpoints[client]
+    procs = run_in_keystone_venv(ctx, client,
+        ['openstack', 'ec2', 'credentials', 'list',
+         '--user', user, '--format', 'json', '--debug'] +
+        os_auth_args(public_host, public_port),
+        stdout=StringIO())
+    assert len(procs) == 1
+    response = json.loads(procs[0].stdout.getvalue())
+    assert len(response)
+    return response[0]
+
 @contextlib.contextmanager
 def task(ctx, config):
     """
@@ -476,6 +501,7 @@ def task(ctx, config):
 
     ctx.keystone = argparse.Namespace()
     ctx.keystone.public_endpoints = assign_ports(ctx, config, 5000)
+    ctx.keystone.read_ec2_credentials = read_ec2_credentials
 
     with contextutil.nested(
         lambda: download(ctx=ctx, config=config),

From ff590c5ae467d3fb3cff920a1c3a04cf060097ec Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:11:15 -0400
Subject: [PATCH 0103/2492] qa/s3tests: create_users() takes all yaml config

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/s3tests.py | 60 +++++++++++++++++++++------------------------
 1 file changed, 28 insertions(+), 32 deletions(-)

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 70b49c086b6f..ad7d1b0fb5ac 100644
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -90,22 +90,21 @@ def _config_user(s3tests_conf, section, user):
 
 
 @contextlib.contextmanager
-def create_users(ctx, config):
+def create_users(ctx, config, s3tests_conf):
     """
     Create a main and an alternate s3 user.
     """
-    assert isinstance(config, dict)
     log.info('Creating rgw users...')
     testdir = teuthology.get_testdir(ctx)
     
     users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'}
-    for client in config['clients']:
-        s3tests_conf = config['s3tests_conf'][client]
-        s3tests_conf.setdefault('fixtures', {})
-        s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
+    for client, cconfig in config.items():
+        conf = s3tests_conf[client]
+        conf.setdefault('fixtures', {})
+        conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
         for section, user in users.items():
-            _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
-            log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
+            _config_user(conf, section, '{user}.{client}'.format(user=user, client=client))
+            log.debug('Creating user {user} on {host}'.format(user=conf[section]['user_id'], host=client))
             cluster_name, daemon_type, client_id = teuthology.split_role(client)
             client_with_id = daemon_type + '.' + client_id
             # create user
@@ -117,12 +116,12 @@ def create_users(ctx, config):
                     'radosgw-admin',
                     '-n', client_with_id,
                     'user', 'create',
-                    '--uid', s3tests_conf[section]['user_id'],
-                    '--display-name', s3tests_conf[section]['display_name'],
-                    '--email', s3tests_conf[section]['email'],
+                    '--uid', conf[section]['user_id'],
+                    '--display-name', conf[section]['display_name'],
+                    '--email', conf[section]['email'],
                     '--caps', 'user-policy=*',
-                    '--access-key', s3tests_conf[section]['access_key'],
-                    '--secret', s3tests_conf[section]['secret_key'],
+                    '--access-key', conf[section]['access_key'],
+                    '--secret', conf[section]['secret_key'],
                     '--cluster', cluster_name,
                 ],
             )
@@ -136,10 +135,10 @@ def create_users(ctx, config):
                         'radosgw-admin',
                         '-n', client_with_id,
                         'mfa', 'create',
-                        '--uid', s3tests_conf[section]['user_id'],
-                        '--totp-serial', s3tests_conf[section]['totp_serial'],
-                        '--totp-seed', s3tests_conf[section]['totp_seed'],
-                        '--totp-seconds', s3tests_conf[section]['totp_seconds'],
+                        '--uid', conf[section]['user_id'],
+                        '--totp-serial', conf[section]['totp_serial'],
+                        '--totp-seed', conf[section]['totp_seed'],
+                        '--totp-seconds', conf[section]['totp_seconds'],
                         '--totp-window', '8',
                         '--totp-seed-type', 'base32',
                         '--cluster', cluster_name,
@@ -156,7 +155,7 @@ def create_users(ctx, config):
                         'radosgw-admin',
                         '-n', client_with_id,
                         'caps', 'add',
-                        '--uid', s3tests_conf[section]['user_id'],
+                        '--uid', conf[section]['user_id'],
                         '--caps', 'roles=*',
                         '--cluster', cluster_name,
                     ],
@@ -169,26 +168,26 @@ def create_users(ctx, config):
                         'radosgw-admin',
                         '-n', client_with_id,
                         'caps', 'add',
-                        '--uid', s3tests_conf[section]['user_id'],
+                        '--uid', conf[section]['user_id'],
                         '--caps', 'oidc-provider=*',
                         '--cluster', cluster_name,
                     ],
                 )
 
     if "TOKEN" in os.environ:
-        s3tests_conf.setdefault('webidentity', {})
-        s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN'])
-        s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD'])
-        s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB'])
-        s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP'])
-        s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
-        s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
-        s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
+        conf.setdefault('webidentity', {})
+        conf['webidentity'].setdefault('token',os.environ['TOKEN'])
+        conf['webidentity'].setdefault('aud',os.environ['AUD'])
+        conf['webidentity'].setdefault('sub',os.environ['SUB'])
+        conf['webidentity'].setdefault('azp',os.environ['AZP'])
+        conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
+        conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
+        conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
 
     try:
         yield
     finally:
-        for client in config['clients']:
+        for client in config.keys():
             for user in users.values():
                 uid = '{user}.{client}'.format(user=user, client=client)
                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
@@ -638,10 +637,7 @@ def task(ctx, config):
 
     with contextutil.nested(
         lambda: download(ctx=ctx, config=config),
-        lambda: create_users(ctx=ctx, config=dict(
-                clients=clients,
-                s3tests_conf=s3tests_conf,
-                )),
+        lambda: create_users(ctx=ctx, config=config, s3tests_conf=s3tests_conf),
         lambda: configure(ctx=ctx, config=dict(
                 clients=config,
                 s3tests_conf=s3tests_conf,

From fa70c782882a541b5beb1cb8ebfdc2a5119f1d88 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:25:19 -0400
Subject: [PATCH 0104/2492] qa/s3tests: enable testing against keystone users
 with ec2

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/s3tests.py | 51 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index ad7d1b0fb5ac..7006602f2bea 100644
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -102,8 +102,31 @@ def create_users(ctx, config, s3tests_conf):
         conf = s3tests_conf[client]
         conf.setdefault('fixtures', {})
         conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
+
+        keystone_users = cconfig.get('keystone users', {})
         for section, user in users.items():
             _config_user(conf, section, '{user}.{client}'.format(user=user, client=client))
+
+            # for keystone users, read ec2 credentials into s3tests.conf instead
+            # of creating a local user
+            keystone_user = keystone_users.get(section)
+            if keystone_user:
+                project_name = keystone_user.pop('project')
+                creds = ctx.keystone.read_ec2_credentials(ctx, **keystone_user)
+                access = creds['Access']
+                secret = creds['Secret']
+                project_id = creds['Project ID']
+
+                conf[section]['access_key'] = access
+                conf[section]['secret_key'] = secret
+                conf[section]['user_id'] = project_id
+                conf[section]['display_name'] = project_name
+
+                log.debug('Using keystone user {kuser} credentials ({access} : {secret}) for {pname}:{pid} on {host}'.format(
+                    kuser=keystone_user['user'], access=access, secret=secret,
+                    pname=project_name, pid=project_id, host=client))
+                continue
+
             log.debug('Creating user {user} on {host}'.format(user=conf[section]['user_id'], host=client))
             cluster_name, daemon_type, client_id = teuthology.split_role(client)
             client_with_id = daemon_type + '.' + client_id
@@ -189,6 +212,9 @@ def create_users(ctx, config, s3tests_conf):
     finally:
         for client in config.keys():
             for user in users.values():
+                # don't need to delete keystone users
+                if not user in keystone_users:
+                    continue
                 uid = '{user}.{client}'.format(user=user, client=client)
                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
                 client_with_id = daemon_type + '.' + client_id
@@ -501,6 +527,31 @@ def task(ctx, config):
               cloudtier_tests: True
               rgw_server: client.0
 
+    To test against Keystone users with EC2 credentials::
+
+        tasks:
+        - ceph:
+        - rgw: [client.0 client.1]
+        - keystone:
+          client.0:
+            projects:
+              - name: myproject
+                description: my project
+            users:
+              - name: myuser
+                password: SECRET
+                project: myproject
+            ec2 credentials:
+              - project: myproject
+                user: myuser
+        - s3tests:
+            client.0:
+              keystone users:
+                s3 main:
+                  client: client.0
+                  project: myproject
+                  user: myuser
+
     """
     assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
     assert hasattr(ctx, 'tox'), 's3tests must run after the tox task'

From 7b27f9dea979f6845e71dcc8eeffac7d1485eeab Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 4 Aug 2023 12:29:47 -0400
Subject: [PATCH 0105/2492] qa/rgw/tempest: run s3tests against keystone ec2

Fixes: https://tracker.ceph.com/issues/59424

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/tempest/s3tests-branch.yaml |  1 +
 qa/suites/rgw/tempest/tasks/+             |  0
 qa/suites/rgw/tempest/tasks/s3tests.yaml  | 35 +++++++++++++++++++++++
 3 files changed, 36 insertions(+)
 create mode 120000 qa/suites/rgw/tempest/s3tests-branch.yaml
 create mode 100644 qa/suites/rgw/tempest/tasks/+
 create mode 100644 qa/suites/rgw/tempest/tasks/s3tests.yaml

diff --git a/qa/suites/rgw/tempest/s3tests-branch.yaml b/qa/suites/rgw/tempest/s3tests-branch.yaml
new file mode 120000
index 000000000000..bdcaca48ae02
--- /dev/null
+++ b/qa/suites/rgw/tempest/s3tests-branch.yaml
@@ -0,0 +1 @@
+.qa/rgw/s3tests-branch.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/tempest/tasks/+ b/qa/suites/rgw/tempest/tasks/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/tempest/tasks/s3tests.yaml b/qa/suites/rgw/tempest/tasks/s3tests.yaml
new file mode 100644
index 000000000000..f1ee8a17c4ce
--- /dev/null
+++ b/qa/suites/rgw/tempest/tasks/s3tests.yaml
@@ -0,0 +1,35 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rgw s3 auth use keystone: true
+        rgw crypt s3 kms backend: testing
+        rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
+        rgw crypt require ssl: false
+  keystone:
+    client.0:
+      projects:
+        - name: s3tests
+          description: s3tests project
+      users:
+        - name: s3tests-main
+          password: SECRET
+          project: s3tests
+      ec2 credentials:
+        - project: s3tests
+          user: s3tests-main
+      roles: [ name: member ]
+      role-mappings:
+        - name: member
+          user: s3tests-main
+          project: s3tests
+
+tasks:
+- s3tests:
+    client.0:
+      rgw_server: client.0
+      keystone users:
+        s3 main:
+          client: client.0
+          project: s3tests
+          user: s3tests-main

From c19aab4c31e89defca9409440463f4654b6cafd5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 9 Aug 2023 14:58:49 -0400
Subject: [PATCH 0106/2492] qa/rgw/tempest: override frontend_prefix in tempest
 task

tempest relies on `frontend_prefix: /swift` to serve the swift api from
the root path

s3tests relies on the same for s3. move the frontend_prefix
override into the tempest task so it doesn't apply to s3tests

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/tempest/0-install.yaml     | 1 -
 qa/suites/rgw/tempest/tasks/+            | 0
 qa/suites/rgw/tempest/tasks/tempest.yaml | 6 ++++++
 3 files changed, 6 insertions(+), 1 deletion(-)
 delete mode 100644 qa/suites/rgw/tempest/tasks/+

diff --git a/qa/suites/rgw/tempest/0-install.yaml b/qa/suites/rgw/tempest/0-install.yaml
index fc2cfcc7b6d2..f968db20c2bf 100644
--- a/qa/suites/rgw/tempest/0-install.yaml
+++ b/qa/suites/rgw/tempest/0-install.yaml
@@ -11,5 +11,4 @@ tasks:
           description: Swift Service
 - rgw:
     client.0:
-      frontend_prefix: /swift
       use-keystone-role: client.0
diff --git a/qa/suites/rgw/tempest/tasks/+ b/qa/suites/rgw/tempest/tasks/+
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rgw/tempest/tasks/tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml
index 560e41de5bbb..988c7515b51f 100644
--- a/qa/suites/rgw/tempest/tasks/tempest.yaml
+++ b/qa/suites/rgw/tempest/tasks/tempest.yaml
@@ -1,3 +1,9 @@
+overrides:
+  rgw:
+    client.0:
+      # tempest tests expect the swift api at the root
+      frontend_prefix: /swift
+
 tasks:
 - tempest:
     client.0:

From 0c70dd8e39cc3d0cdef8bbcc8a0c6f214e54c770 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 20 Sep 2023 16:57:01 -0400
Subject: [PATCH 0107/2492] common: resolve config proxy deadlock using
 refcounted pointers

7e8c683 introduced some gymnastics with a "CallGate" to maintain a count for
each observer we may be "calling into" with a config change (namely:
handle_conf_change). This was to prevent remove_observer coming in and deleting
the observer in the middle of the call. More importantly, it was to avoid
holding the lock while traversing the observers so that the config_proxy lock
can be dropped while calling handle_conf_change. This is important as e.g. the
MDS may attempt to acquire the config_proxy lock in its
MDSRank::handle_conf_change method (what prompted the change).

However, this introduces a new deadlock:

- Thread 2 acquires the config_proxy lock and then removes an observer. It blocks
  waiting for the observer's CallGate to close.

- Thread 1 had dropped the config_proxy lock while traversing the observers to call each
  observer's handle_conf_change method. Those methods may attempt to reacquire the
  config_proxy lock. This creates the deadlock as it's waiting for Thread 2 to drop the lock
  while Thread 1 cannot release the CallGate.

The solution, I believe, is to properly refcount "uses" of the observers for the purposes
of flushing these changes. Use std::shared_ptr to effect this.

Reproducing this is fairly simply with several parallel calls to `config set`.
During the course of executing `config set`, the Objecter may receive config
updates that will be flushed and potentially race with cleanup of observers
during shutdown.

Fixes: https://tracker.ceph.com/issues/62832
Partial-revert: 7e8c683
Partial-revert: 4458a72
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/config_obs_mgr.h       |  39 ++++---
 src/common/config_proxy.h         | 180 ++++++++++++------------------
 src/crimson/common/config_proxy.h |  25 +++--
 3 files changed, 110 insertions(+), 134 deletions(-)

diff --git a/src/common/config_obs_mgr.h b/src/common/config_obs_mgr.h
index 06b3cf934a53..759930df92d9 100644
--- a/src/common/config_obs_mgr.h
+++ b/src/common/config_obs_mgr.h
@@ -14,13 +14,11 @@ class ConfigValues;
 // the changes of settings at runtime.
 template<class ConfigObs>
 class ObserverMgr : public ConfigTracker {
-  // Maps configuration options to the observer listening for them.
-  using obs_map_t = std::multimap<std::string, ConfigObs*>;
-  obs_map_t observers;
-
 public:
-  typedef std::map<ConfigObs*, std::set<std::string>> rev_obs_map;
-  typedef std::function<void(ConfigObs*, const std::string&)> config_gather_cb;
+  using config_obs_ptr = std::shared_ptr<ConfigObs*>;
+  using config_obs_wptr = std::weak_ptr<ConfigObs*>;
+  typedef std::map<config_obs_ptr, std::set<std::string>> rev_obs_map;
+  typedef std::function<void(config_obs_ptr, const std::string&)> config_gather_cb;
 
   // Adds a new observer to this configuration. You can do this at any time,
   // but it will only receive notifications for the changes that happen after
@@ -37,15 +35,18 @@ class ObserverMgr : public ConfigTracker {
   // you need to delete it yourself.
   // This function will assert if you try to delete an observer that isn't
   // there.
-  void remove_observer(ConfigObs* observer);
+  config_obs_wptr remove_observer(ConfigObs* observer);
   // invoke callback for every observers tracking keys
   void for_each_observer(config_gather_cb callback);
   // invoke callback for observers keys tracking the provided change set
-  template<class ConfigProxyT>
-  void for_each_change(const std::set<std::string>& changes,
-                       ConfigProxyT& proxy,
+  void for_each_change(const std::map<std::string,bool>& changes,
                        config_gather_cb callback, std::ostream *oss);
   bool is_tracking(const std::string& name) const override;
+
+private:
+  // Maps configuration options to the observer listening for them.
+  using obs_map_t = std::multimap<std::string, config_obs_ptr>;
+  obs_map_t observers;
 };
 
 // we could put the implementations in a .cc file, and only instantiate the
@@ -60,17 +61,20 @@ template<class ConfigObs>
 void ObserverMgr<ConfigObs>::add_observer(ConfigObs* observer)
 {
   const char **keys = observer->get_tracked_conf_keys();
+  auto ptr = std::make_shared<ConfigObs*>(observer);
   for (const char ** k = keys; *k; ++k) {
-    observers.emplace(*k, observer);
+    observers.emplace(*k, ptr);
   }
 }
 
 template<class ConfigObs>
-void ObserverMgr<ConfigObs>::remove_observer(ConfigObs* observer)
+typename ObserverMgr<ConfigObs>::config_obs_wptr ObserverMgr<ConfigObs>::remove_observer(ConfigObs* observer)
 {
   [[maybe_unused]] bool found_obs = false;
+  config_obs_ptr ptr;
   for (auto o = observers.begin(); o != observers.end(); ) {
-    if (o->second == observer) {
+    if (*o->second == observer) {
+      ptr = std::move(o->second);
       observers.erase(o++);
       found_obs = true;
     } else {
@@ -78,6 +82,7 @@ void ObserverMgr<ConfigObs>::remove_observer(ConfigObs* observer)
     }
   }
   ceph_assert(found_obs);
+  return config_obs_wptr(ptr);
 }
 
 template<class ConfigObs>
@@ -89,17 +94,15 @@ void ObserverMgr<ConfigObs>::for_each_observer(config_gather_cb callback)
 }
 
 template<class ConfigObs>
-template<class ConfigProxyT>
-void ObserverMgr<ConfigObs>::for_each_change(const std::set<std::string>& changes,
-                                             ConfigProxyT& proxy,
+void ObserverMgr<ConfigObs>::for_each_change(const std::map<std::string,bool>& changes,
                                              config_gather_cb callback, std::ostream *oss)
 {
   // create the reverse observer mapping, mapping observers to the set of
   // changed keys that they'll get.
   std::string val;
-  for (auto& key : changes) {
+  for (auto& [key, present] : changes) {
     auto [first, last] = observers.equal_range(key);
-    if ((oss) && !proxy.get_val(key, &val)) {
+    if ((oss) && present) {
       (*oss) << key << " = '" << val << "' ";
       if (first == last) {
         (*oss) << "(not observed, change may require restart) ";
diff --git a/src/common/config_proxy.h b/src/common/config_proxy.h
index 400aa4ed052d..b9b47d9cef47 100644
--- a/src/common/config_proxy.h
+++ b/src/common/config_proxy.h
@@ -18,91 +18,51 @@ class ConfigProxy {
    */
   ConfigValues values;
   using md_config_obs_t = ceph::md_config_obs_impl<ConfigProxy>;
-  ObserverMgr<md_config_obs_t> obs_mgr;
+  using ObsMgr = ObserverMgr<md_config_obs_t>;
+  ObsMgr obs_mgr;
   md_config_t config;
   /** A lock that protects the md_config_t internals. It is
    * recursive, for simplicity.
    * It is best if this lock comes first in the lock hierarchy. We will
    * hold this lock when calling configuration observers.  */
-  mutable ceph::recursive_mutex lock =
-    ceph::make_recursive_mutex("ConfigProxy::lock");
+  mutable ceph::mutex lock = ceph::make_mutex("ConfigProxy::lock");
+  ceph::condition_variable cond;
 
-  class CallGate {
-  private:
-    uint32_t call_count = 0;
-    ceph::mutex lock;
-    ceph::condition_variable cond;
-  public:
-    CallGate()
-      : lock(ceph::make_mutex("call::gate::lock")) {
-    }
+  using rev_obs_map_t = ObsMgr::rev_obs_map;
 
-    void enter() {
-      std::lock_guard<ceph::mutex> locker(lock);
-      ++call_count;
-    }
-    void leave() {
-      std::lock_guard<ceph::mutex> locker(lock);
-      ceph_assert(call_count > 0);
-      if (--call_count == 0) {
-        cond.notify_all();
-      }
+  void _call_observers(rev_obs_map_t& rev_obs) {
+    ceph_assert(!ceph::mutex_debugging || !ceph_mutex_is_locked_by_me(lock));
+    for (auto& [obs, keys] : rev_obs) {
+      (*obs)->handle_conf_change(*this, keys);
     }
-    void close() {
-      std::unique_lock<ceph::mutex> locker(lock);
-      while (call_count != 0) {
-        cond.wait(locker);
-      }
+    rev_obs.clear(); // drop shared_ptrs
+    {
+      std::lock_guard l{lock};
+      cond.notify_all();
     }
-  };
-
-  void call_gate_enter(md_config_obs_t *obs) {
-    auto p = obs_call_gate.find(obs);
-    ceph_assert(p != obs_call_gate.end());
-    p->second->enter();
-  }
-  void call_gate_leave(md_config_obs_t *obs) {
-    auto p = obs_call_gate.find(obs);
-    ceph_assert(p != obs_call_gate.end());
-    p->second->leave();
-  }
-  void call_gate_close(md_config_obs_t *obs) {
-    auto p = obs_call_gate.find(obs);
-    ceph_assert(p != obs_call_gate.end());
-    p->second->close();
   }
-
-  using rev_obs_map_t = ObserverMgr<md_config_obs_t>::rev_obs_map;
-  typedef std::unique_ptr<CallGate> CallGateRef;
-
-  std::map<md_config_obs_t*, CallGateRef> obs_call_gate;
-
-  void call_observers(std::unique_lock<ceph::recursive_mutex>& locker,
-                      rev_obs_map_t& rev_obs) {
-    // observers are notified outside of lock
-    locker.unlock();
-    for (auto& [obs, keys] : rev_obs) {
-      obs->handle_conf_change(*this, keys);
-    }
-    locker.lock();
-
-    for (auto& rev_ob : rev_obs) {
-      call_gate_leave(rev_ob.first);
+  void _gather_changes(std::set<std::string> &changes,
+                       rev_obs_map_t *rev_obs, std::ostream* oss) {
+    ceph_assert(ceph_mutex_is_locked_by_me(lock));
+    std::map<std::string,bool> changes_present;
+    for (auto& change : changes) {
+      std::string dummy;
+      changes_present[change] = (0 == config.get_val(values, change, &dummy));
     }
+    obs_mgr.for_each_change(
+      changes_present,
+      [this, rev_obs](auto obs, const std::string &key) {
+        _map_observer_changes(obs, key, rev_obs);
+      }, oss);
+    changes.clear();
   }
 
-  void map_observer_changes(md_config_obs_t *obs, const std::string &key,
+  void _map_observer_changes(ObsMgr::config_obs_ptr obs, const std::string& key,
                             rev_obs_map_t *rev_obs) {
-    ceph_assert(ceph_mutex_is_locked(lock));
+    ceph_assert(ceph_mutex_is_locked_by_me(lock));
 
     auto [it, new_entry] = rev_obs->emplace(obs, std::set<std::string>{});
     it->second.emplace(key);
-    if (new_entry) {
-      // this needs to be done under lock as once this lock is
-      // dropped (before calling observers) a remove_observer()
-      // can sneak in and cause havoc.
-      call_gate_enter(obs);
-    }
   }
 
 public:
@@ -200,34 +160,39 @@ class ConfigProxy {
   }
   // for those want to reexpand special meta, e.g, $pid
   void finalize_reexpand_meta() {
-    std::unique_lock locker(lock);
     rev_obs_map_t rev_obs;
-    if (config.finalize_reexpand_meta(values, obs_mgr)) {
-      _gather_changes(values.changed, &rev_obs, nullptr);
+    {
+      std::lock_guard locker(lock);
+      if (config.finalize_reexpand_meta(values, obs_mgr)) {
+        _gather_changes(values.changed, &rev_obs, nullptr);
+      }
     }
 
-    call_observers(locker, rev_obs);
+    _call_observers(rev_obs);
   }
   void add_observer(md_config_obs_t* obs) {
     std::lock_guard l(lock);
     obs_mgr.add_observer(obs);
-    obs_call_gate.emplace(obs, std::make_unique<CallGate>());
+    cond.notify_all();
   }
   void remove_observer(md_config_obs_t* obs) {
-    std::lock_guard l(lock);
-    call_gate_close(obs);
-    obs_call_gate.erase(obs);
-    obs_mgr.remove_observer(obs);
+    std::unique_lock l(lock);
+    auto wptr = obs_mgr.remove_observer(obs);
+    while (!wptr.expired()) {
+      cond.wait(l);
+    }
   }
   void call_all_observers() {
-    std::unique_lock locker(lock);
     rev_obs_map_t rev_obs;
-    obs_mgr.for_each_observer(
-      [this, &rev_obs](md_config_obs_t *obs, const std::string &key) {
-        map_observer_changes(obs, key, &rev_obs);
-      });
+    {
+      std::lock_guard locker(lock);
+      obs_mgr.for_each_observer(
+        [this, &rev_obs](auto obs, const std::string& key) {
+          _map_observer_changes(obs, key, &rev_obs);
+        });
+    }
 
-    call_observers(locker, rev_obs);
+    _call_observers(rev_obs);
   }
   void set_safe_to_start_threads() {
     std::lock_guard l(lock);
@@ -255,25 +220,18 @@ class ConfigProxy {
   }
   // Expand all metavariables. Make any pending observer callbacks.
   void apply_changes(std::ostream* oss) {
-    std::unique_lock locker(lock);
     rev_obs_map_t rev_obs;
 
-    // apply changes until the cluster name is assigned
-    if (!values.cluster.empty()) {
-      // meta expands could have modified anything.  Copy it all out again.
-      _gather_changes(values.changed, &rev_obs, oss);
+    {
+      std::lock_guard locker(lock);
+      // apply changes until the cluster name is assigned
+      if (!values.cluster.empty()) {
+        // meta expands could have modified anything.  Copy it all out again.
+        _gather_changes(values.changed, &rev_obs, oss);
+      }
     }
 
-    call_observers(locker, rev_obs);
-  }
-  void _gather_changes(std::set<std::string> &changes,
-                       rev_obs_map_t *rev_obs, std::ostream* oss) {
-    obs_mgr.for_each_change(
-      changes, *this,
-      [this, rev_obs](md_config_obs_t *obs, const std::string &key) {
-        map_observer_changes(obs, key, rev_obs);
-      }, oss);
-      changes.clear();
+    _call_observers(rev_obs);
   }
   int set_val(const std::string_view key, const std::string& s,
               std::stringstream* err_ss=nullptr) {
@@ -291,23 +249,27 @@ class ConfigProxy {
   int set_mon_vals(CephContext *cct,
 		   const std::map<std::string,std::string,std::less<>>& kv,
 		   md_config_t::config_callback config_cb) {
-    std::unique_lock locker(lock);
-    int ret = config.set_mon_vals(cct, values, obs_mgr, kv, config_cb);
-
+    int ret;
     rev_obs_map_t rev_obs;
-    _gather_changes(values.changed, &rev_obs, nullptr);
 
-    call_observers(locker, rev_obs);
+    {
+      std::lock_guard locker(lock);
+      ret = config.set_mon_vals(cct, values, obs_mgr, kv, config_cb);
+      _gather_changes(values.changed, &rev_obs, nullptr);
+    }
+
+    _call_observers(rev_obs);
     return ret;
   }
   int injectargs(const std::string &s, std::ostream *oss) {
-    std::unique_lock locker(lock);
-    int ret = config.injectargs(values, obs_mgr, s, oss);
-
+    int ret;
     rev_obs_map_t rev_obs;
-    _gather_changes(values.changed, &rev_obs, oss);
-
-    call_observers(locker, rev_obs);
+    {
+      std::lock_guard locker(lock);
+      ret = config.injectargs(values, obs_mgr, s, oss);
+      _gather_changes(values.changed, &rev_obs, oss);
+    }
+    _call_observers(rev_obs);
     return ret;
   }
   void parse_env(unsigned entity_type,
diff --git a/src/crimson/common/config_proxy.h b/src/crimson/common/config_proxy.h
index 4c0e655075ad..822db34f61a4 100644
--- a/src/crimson/common/config_proxy.h
+++ b/src/crimson/common/config_proxy.h
@@ -54,13 +54,18 @@ class ConfigProxy : public seastar::peering_sharded_service<ConfigProxy>
       // avoid racings with other do_change() calls in parallel.
       ObserverMgr<ConfigObserver>::rev_obs_map rev_obs;
       owner.values.reset(new_values);
-      owner.obs_mgr.for_each_change(owner.values->changed, owner,
-                                    [&rev_obs](ConfigObserver *obs,
+      std::map<std::string, bool> changes_present;
+      for (const auto& change : owner.values->changed) {
+        std::string dummy;
+        changes_present[change] = owner.get_val(change, &dummy);
+      }
+      owner.obs_mgr.for_each_change(changes_present,
+                                    [&rev_obs](auto obs,
                                                const std::string &key) {
                                       rev_obs[obs].insert(key);
                                     }, nullptr);
       for (auto& [obs, keys] : rev_obs) {
-        obs->handle_conf_change(owner, keys);
+        (*obs)->handle_conf_change(owner, keys);
       }
 
       return seastar::parallel_for_each(boost::irange(1u, seastar::smp::count),
@@ -70,13 +75,19 @@ class ConfigProxy : public seastar::peering_sharded_service<ConfigProxy>
             proxy.values.reset();
             proxy.values = std::move(foreign_values);
 
+            std::map<std::string, bool> changes_present;
+            for (const auto& change : proxy.values->changed) {
+              std::string dummy;
+              changes_present[change] = proxy.get_val(change, &dummy);
+            }
+
             ObserverMgr<ConfigObserver>::rev_obs_map rev_obs;
-            proxy.obs_mgr.for_each_change(proxy.values->changed, proxy,
-              [&rev_obs](ConfigObserver *obs, const std::string& key) {
+            proxy.obs_mgr.for_each_change(changes_present,
+              [&rev_obs](auto obs, const std::string& key) {
                 rev_obs[obs].insert(key);
               }, nullptr);
-            for (auto& obs_keys : rev_obs) {
-              obs_keys.first->handle_conf_change(proxy, obs_keys.second);
+            for (auto& [obs, keys] : rev_obs) {
+              (*obs)->handle_conf_change(proxy, keys);
             }
           });
         }).finally([new_values] {

From bfd12365979a702bd0575acbd3d2b3eb5f103ec2 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 27 Sep 2023 17:13:44 +0800
Subject: [PATCH 0108/2492] crimson/os/seastore/onode_manager: populate delta
 recorders for each onode modification

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../os/seastore/object_data_handler.cc        |  7 +-
 src/crimson/os/seastore/onode.h               | 10 +-
 .../staged-fltree/fltree_onode_manager.cc     | 12 +--
 .../staged-fltree/fltree_onode_manager.h      | 68 +++++++++++++-
 src/crimson/os/seastore/seastore.cc           | 94 ++++++++-----------
 src/crimson/os/seastore/seastore.h            |  3 +-
 .../onode_tree/test_fltree_onode_manager.cc   | 10 +-
 .../seastore/test_object_data_handler.cc      | 67 ++++++++++++-
 8 files changed, 192 insertions(+), 79 deletions(-)

diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 0d852696b714..025f91993efa 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -888,7 +888,7 @@ auto with_object_data(
       return std::invoke(f, object_data
       ).si_then([ctx, &object_data] {
 	if (object_data.must_update()) {
-	  ctx.onode.get_mutable_layout(ctx.t).object_data.update(object_data);
+	  ctx.onode.update_object_data(ctx.t, object_data);
 	}
 	return seastar::now();
       });
@@ -909,11 +909,10 @@ auto with_objects_data(
       return std::invoke(f, object_data, d_object_data
       ).si_then([ctx, &object_data, &d_object_data] {
 	if (object_data.must_update()) {
-	  ctx.onode.get_mutable_layout(ctx.t).object_data.update(object_data);
+	  ctx.onode.update_object_data(ctx.t, object_data);
 	}
 	if (d_object_data.must_update()) {
-	  ctx.d_onode->get_mutable_layout(
-	    ctx.t).object_data.update(d_object_data);
+	  ctx.d_onode->update_object_data(ctx.t, d_object_data);
 	}
 	return seastar::now();
       });
diff --git a/src/crimson/os/seastore/onode.h b/src/crimson/os/seastore/onode.h
index 069daa3df5b5..e803a2e4e74f 100644
--- a/src/crimson/os/seastore/onode.h
+++ b/src/crimson/os/seastore/onode.h
@@ -64,9 +64,17 @@ class Onode : public boost::intrusive_ref_counter<
 
   virtual bool is_alive() const = 0;
   virtual const onode_layout_t &get_layout() const = 0;
-  virtual onode_layout_t &get_mutable_layout(Transaction &t) = 0;
   virtual ~Onode() = default;
 
+  virtual void update_onode_size(Transaction&, uint32_t) = 0;
+  virtual void update_omap_root(Transaction&, omap_root_t&) = 0;
+  virtual void update_xattr_root(Transaction&, omap_root_t&) = 0;
+  virtual void update_object_data(Transaction&, object_data_t&) = 0;
+  virtual void update_object_info(Transaction&, ceph::bufferlist&) = 0;
+  virtual void update_snapset(Transaction&, ceph::bufferlist&) = 0;
+  virtual void clear_object_info(Transaction&) = 0;
+  virtual void clear_snapset(Transaction&) = 0;
+
   laddr_t get_metadata_hint(uint64_t block_size) const {
     assert(default_metadata_offset);
     assert(default_metadata_range);
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
index bff27ab65178..a1f593889d52 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
@@ -51,17 +51,17 @@ FLTreeOnodeManager::get_or_create_onode(
   ).si_then([this, &trans, &hoid, FNAME](auto p)
               -> get_or_create_onode_ret {
     auto [cursor, created] = std::move(p);
-    auto val = OnodeRef(new FLTreeOnode(
+    auto onode = new FLTreeOnode(
 	default_data_reservation,
 	default_metadata_range,
-	cursor.value()));
+	cursor.value());
     if (created) {
       DEBUGT("created onode for entry for {}", trans, hoid);
-      val->get_mutable_layout(trans) = onode_layout_t{};
+      onode->with_mutable_layout(trans, [](onode_layout_t &mlayout) {
+	mlayout = onode_layout_t{};
+      });
     }
-    return get_or_create_onode_iertr::make_ready_future<OnodeRef>(
-      val
-    );
+    return get_or_create_onode_iertr::make_ready_future<OnodeRef>(onode);
   });
 }
 
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
index 09998fbfaea9..619609419ad8 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
@@ -78,14 +78,16 @@ struct FLTreeOnode final : Onode, Value {
     return *read_payload<onode_layout_t>();
   }
 
-  onode_layout_t &get_mutable_layout(Transaction &t) final {
+  template <typename Func>
+  void with_mutable_layout(Transaction &t, Func&& f) {
     assert(status != status_t::DELETED);
     auto p = prepare_mutate_payload<
       onode_layout_t,
       Recorder>(t);
     status = status_t::MUTATED;
-    return *reinterpret_cast<onode_layout_t*>(p.first.get_write());
-  };
+    f(*reinterpret_cast<onode_layout_t*>(p.first.get_write()));
+    populate_recorder(t);
+  }
 
   void populate_recorder(Transaction &t) {
     assert(status == status_t::MUTATED);
@@ -99,6 +101,66 @@ struct FLTreeOnode final : Onode, Value {
     status = status_t::STABLE;
   }
 
+  void update_onode_size(Transaction &t, uint32_t size) final {
+    with_mutable_layout(t, [size](onode_layout_t &mlayout) {
+      mlayout.size = size;
+    });
+  }
+
+  void update_omap_root(Transaction &t, omap_root_t &oroot) final {
+    with_mutable_layout(t, [&oroot](onode_layout_t &mlayout) {
+      mlayout.omap_root.update(oroot);
+    });
+  }
+
+  void update_xattr_root(Transaction &t, omap_root_t &xroot) final {
+    with_mutable_layout(t, [&xroot](onode_layout_t &mlayout) {
+      mlayout.xattr_root.update(xroot);
+    });
+  }
+
+  void update_object_data(Transaction &t, object_data_t &odata) final {
+    with_mutable_layout(t, [&odata](onode_layout_t &mlayout) {
+      mlayout.object_data.update(odata);
+    });
+  }
+
+  void update_object_info(Transaction &t, ceph::bufferlist &oi_bl) final {
+    with_mutable_layout(t, [&oi_bl](onode_layout_t &mlayout) {
+      maybe_inline_memcpy(
+	&mlayout.oi[0],
+	oi_bl.c_str(),
+	oi_bl.length(),
+	onode_layout_t::MAX_OI_LENGTH);
+      mlayout.oi_size = oi_bl.length();
+    });
+  }
+
+  void clear_object_info(Transaction &t) final {
+    with_mutable_layout(t, [](onode_layout_t &mlayout) {
+      memset(&mlayout.oi[0], 0, mlayout.oi_size);
+      mlayout.oi_size = 0;
+    });
+  }
+
+  void update_snapset(Transaction &t, ceph::bufferlist &ss_bl) final {
+    with_mutable_layout(t, [&ss_bl](onode_layout_t &mlayout) {
+      maybe_inline_memcpy(
+	&mlayout.ss[0],
+	ss_bl.c_str(),
+	ss_bl.length(),
+	onode_layout_t::MAX_OI_LENGTH);
+      mlayout.ss_size = ss_bl.length();
+    });
+  }
+
+  void clear_snapset(Transaction &t) final {
+    with_mutable_layout(t, [](onode_layout_t &mlayout) {
+      memset(&mlayout.ss[0], 0, mlayout.ss_size);
+      mlayout.ss_size = 0;
+    });
+  }
+
   void mark_delete() {
     assert(status != status_t::DELETED);
     status = status_t::DELETED;
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 897a063e0fe6..edbc9a0115ec 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1529,10 +1529,10 @@ SeaStore::Shard::_write(
   LOG_PREFIX(SeaStore::_write);
   DEBUGT("onode={} {}~{}", *ctx.transaction, *onode, offset, len);
   {
-    auto &object_size = onode->get_mutable_layout(*ctx.transaction).size;
-    object_size = std::max<uint64_t>(
-      offset + len,
-      object_size);
+    const auto &object_size = onode->get_layout().size;
+    onode->update_onode_size(
+      *ctx.transaction,
+      std::max<uint64_t>(offset + len, object_size));
   }
   return seastar::do_with(
     std::move(_bl),
@@ -1563,8 +1563,7 @@ SeaStore::Shard::_clone(
     //TODO: currently, we only care about object data, leaving cloning
     //      of xattr/omap for future work
     auto &object_size = onode->get_layout().size;
-    auto &d_object_size = d_onode->get_mutable_layout(*ctx.transaction).size;
-    d_object_size = object_size;
+    d_onode->update_onode_size(*ctx.transaction, object_size);
     return objHandler.clone(
       ObjectDataHandler::context_t{
 	*transaction_manager,
@@ -1586,8 +1585,10 @@ SeaStore::Shard::_zero(
   if (offset + len >= max_object_size) {
     return crimson::ct_error::input_output_error::make();
   }
-  auto &object_size = onode->get_mutable_layout(*ctx.transaction).size;
-  object_size = std::max<uint64_t>(offset + len, object_size);
+  const auto &object_size = onode->get_layout().size;
+  onode->update_onode_size(
+    *ctx.transaction,
+    std::max<uint64_t>(offset + len, object_size));
   return seastar::do_with(
     ObjectDataHandler(max_object_size),
     [=, this, &ctx, &onode](auto &objhandler) {
@@ -1607,7 +1608,6 @@ SeaStore::Shard::_omap_set_kvs(
   OnodeRef &onode,
   const omap_root_le_t& omap_root,
   Transaction& t,
-  omap_root_le_t& mutable_omap_root,
   std::map<std::string, ceph::bufferlist>&& kvs)
 {
   return seastar::do_with(
@@ -1627,10 +1627,6 @@ SeaStore::Shard::_omap_set_kvs(
           return omap_manager.omap_set_keys(root, t, std::move(keys));
       }).si_then([&] {
         return tm_iertr::make_ready_future<omap_root_t>(std::move(root));
-      }).si_then([&mutable_omap_root](auto root) {
-        if (root.must_update()) {
-          mutable_omap_root.update(root);
-        }
       });
     }
   );
@@ -1648,8 +1644,12 @@ SeaStore::Shard::_omap_set_values(
     onode,
     onode->get_layout().omap_root,
     *ctx.transaction,
-    onode->get_mutable_layout(*ctx.transaction).omap_root,
-    std::move(aset));
+    std::move(aset)
+  ).si_then([onode, &ctx](auto root) {
+    if (root.must_update()) {
+      onode->update_omap_root(*ctx.transaction, root);
+    }
+  });
 }
 
 SeaStore::Shard::tm_ret
@@ -1691,8 +1691,7 @@ SeaStore::Shard::_omap_clear(
           *ctx.transaction)
         .si_then([&] {
           if (omap_root.must_update()) {
-            onode->get_mutable_layout(*ctx.transaction
-            ).omap_root.update(omap_root);
+	    onode->update_omap_root(*ctx.transaction, omap_root);
           }
         });
       });
@@ -1733,8 +1732,7 @@ SeaStore::Shard::_omap_rmkeys(
             }
           ).si_then([&] {
             if (omap_root.must_update()) {
-              onode->get_mutable_layout(*ctx.transaction
-              ).omap_root.update(omap_root);
+	      onode->update_omap_root(*ctx.transaction, omap_root);
             }
           });
       }
@@ -1782,8 +1780,7 @@ SeaStore::Shard::_omap_rmkeyrange(
 	config
       ).si_then([&] {
         if (omap_root.must_update()) {
-          onode->get_mutable_layout(*ctx.transaction
-          ).omap_root.update(omap_root);
+	  onode->update_omap_root(*ctx.transaction, omap_root);
         }
       });
     });
@@ -1798,7 +1795,7 @@ SeaStore::Shard::_truncate(
 {
   LOG_PREFIX(SeaStore::_truncate);
   DEBUGT("onode={} size={}", *ctx.transaction, *onode, size);
-  onode->get_mutable_layout(*ctx.transaction).size = size;
+  onode->update_onode_size(*ctx.transaction, size);
   return seastar::do_with(
     ObjectDataHandler(max_object_size),
     [=, this, &ctx, &onode](auto &objhandler) {
@@ -1822,45 +1819,34 @@ SeaStore::Shard::_setattrs(
   DEBUGT("onode={}", *ctx.transaction, *onode);
 
   auto fut = tm_iertr::now();
-  auto& layout = onode->get_mutable_layout(*ctx.transaction);
+  auto& layout = onode->get_layout();
   if (auto it = aset.find(OI_ATTR); it != aset.end()) {
     auto& val = it->second;
     if (likely(val.length() <= onode_layout_t::MAX_OI_LENGTH)) {
-      maybe_inline_memcpy(
-	&layout.oi[0],
-	val.c_str(),
-	val.length(),
-	onode_layout_t::MAX_OI_LENGTH);
 
       if (!layout.oi_size) {
 	// if oi was not in the layout, it probably exists in the omap,
 	// need to remove it first
 	fut = _xattr_rmattr(ctx, onode, OI_ATTR);
       }
-      layout.oi_size = val.length();
+      onode->update_object_info(*ctx.transaction, val);
       aset.erase(it);
     } else {
-      layout.oi_size = 0;
+      onode->clear_object_info(*ctx.transaction);
     }
   }
 
   if (auto it = aset.find(SS_ATTR); it != aset.end()) {
     auto& val = it->second;
     if (likely(val.length() <= onode_layout_t::MAX_SS_LENGTH)) {
-      maybe_inline_memcpy(
-	&layout.ss[0],
-	val.c_str(),
-	val.length(),
-	onode_layout_t::MAX_SS_LENGTH);
 
       if (!layout.ss_size) {
 	fut = _xattr_rmattr(ctx, onode, SS_ATTR);
       }
-      layout.ss_size = val.length();
-
+      onode->update_snapset(*ctx.transaction, val);
       aset.erase(it);
     } else {
-      layout.ss_size = 0;
+      onode->clear_snapset(*ctx.transaction);
     }
   }
 
@@ -1869,14 +1855,17 @@ SeaStore::Shard::_setattrs(
   }
 
   return fut.si_then(
-    [this, onode, &ctx, &layout,
-    aset=std::move(aset)]() mutable {
+    [this, onode, &ctx, aset=std::move(aset)]() mutable {
     return _omap_set_kvs(
       onode,
       onode->get_layout().xattr_root,
       *ctx.transaction,
-      layout.xattr_root,
-      std::move(aset));
+      std::move(aset)
+    ).si_then([onode, &ctx](auto root) {
+      if (root.must_update()) {
+	onode->update_xattr_root(*ctx.transaction, root);
+      }
+    });
   });
 }
 
@@ -1888,14 +1877,12 @@ SeaStore::Shard::_rmattr(
 {
   LOG_PREFIX(SeaStore::_rmattr);
   DEBUGT("onode={}", *ctx.transaction, *onode);
-  auto& layout = onode->get_mutable_layout(*ctx.transaction);
+  auto& layout = onode->get_layout();
   if ((name == OI_ATTR) && (layout.oi_size > 0)) {
-    memset(&layout.oi[0], 0, layout.oi_size);
-    layout.oi_size = 0;
+    onode->clear_object_info(*ctx.transaction);
     return tm_iertr::now();
   } else if ((name == SS_ATTR) && (layout.ss_size > 0)) {
-    memset(&layout.ss[0], 0, layout.ss_size);
-    layout.ss_size = 0;
+    onode->clear_snapset(*ctx.transaction);
     return tm_iertr::now();
   } else {
     return _xattr_rmattr(
@@ -1927,8 +1914,7 @@ SeaStore::Shard::_xattr_rmattr(
         return omap_manager.omap_rm_key(xattr_root, *ctx.transaction, name)
           .si_then([&] {
           if (xattr_root.must_update()) {
-              onode->get_mutable_layout(*ctx.transaction
-              ).xattr_root.update(xattr_root);
+	    onode->update_xattr_root(*ctx.transaction, xattr_root);
           }
         });
     });
@@ -1942,11 +1928,8 @@ SeaStore::Shard::_rmattrs(
 {
   LOG_PREFIX(SeaStore::_rmattrs);
   DEBUGT("onode={}", *ctx.transaction, *onode);
-  auto& layout = onode->get_mutable_layout(*ctx.transaction);
-  memset(&layout.oi[0], 0, layout.oi_size);
-  layout.oi_size = 0;
-  memset(&layout.ss[0], 0, layout.ss_size);
-  layout.ss_size = 0;
+  onode->clear_object_info(*ctx.transaction);
+  onode->clear_snapset(*ctx.transaction);
   return _xattr_clear(ctx, onode);
 }
 
@@ -1970,8 +1953,7 @@ SeaStore::Shard::_xattr_clear(
         return omap_manager.omap_clear(xattr_root, *ctx.transaction)
 	  .si_then([&] {
 	  if (xattr_root.must_update()) {
-              onode->get_mutable_layout(*ctx.transaction
-              ).xattr_root.update(xattr_root);
+	    onode->update_xattr_root(*ctx.transaction, xattr_root);
           }
         });
     });
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index 876fadca8c78..def40a282208 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -408,12 +408,11 @@ class SeaStore final : public FuturizedStore {
     tm_ret _remove_collection(
       internal_context_t &ctx,
       const coll_t& cid);
-    using omap_set_kvs_ret = tm_iertr::future<>;
+    using omap_set_kvs_ret = tm_iertr::future<omap_root_t>;
     omap_set_kvs_ret _omap_set_kvs(
       OnodeRef &onode,
       const omap_root_le_t& omap_root,
       Transaction& t,
-      omap_root_le_t& mutable_omap_root,
       std::map<std::string, ceph::bufferlist>&& kvs);
 
     boost::intrusive_ptr<SeastoreCollection> _get_collection(const coll_t& cid);
diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
index 17ad975d5e87..3d98dcf5aef2 100644
--- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
+++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
@@ -30,10 +30,12 @@ struct onode_item_t {
   uint32_t cnt_modify = 0;
 
   void initialize(Transaction& t, Onode& value) const {
-    auto& layout = value.get_mutable_layout(t);
-    layout.size = size;
-    layout.omap_root.update(omap_root_t(id, cnt_modify,
-      value.get_metadata_hint(block_size)));
+    auto &ftvalue = static_cast<FLTreeOnode&>(value);
+    ftvalue.with_mutable_layout(t, [this, &value](auto &mlayout) {
+      mlayout.size = size;
+      mlayout.omap_root.update(omap_root_t(id, cnt_modify,
+	value.get_metadata_hint(block_size)));
+    });
     validate(value);
   }
 
diff --git a/src/test/crimson/seastore/test_object_data_handler.cc b/src/test/crimson/seastore/test_object_data_handler.cc
index 6510cb5d93fe..47607643946e 100644
--- a/src/test/crimson/seastore/test_object_data_handler.cc
+++ b/src/test/crimson/seastore/test_object_data_handler.cc
@@ -30,9 +30,9 @@ class TestOnode final : public Onode {
   const onode_layout_t &get_layout() const final {
     return layout;
   }
-  onode_layout_t &get_mutable_layout(Transaction &t) final {
-    dirty = true;
-    return layout;
+  template <typename Func>
+  void with_mutable_layout(Transaction &t, Func&& f) {
+    f(layout);
   }
   bool is_alive() const {
     return true;
@@ -40,6 +40,67 @@ class TestOnode final : public Onode {
   bool is_dirty() const { return dirty; }
   laddr_t get_hint() const final {return L_ADDR_MIN; }
   ~TestOnode() final = default;
+
+  void update_onode_size(Transaction &t, uint32_t size) final {
+    with_mutable_layout(t, [size](onode_layout_t &mlayout) {
+      mlayout.size = size;
+    });
+  }
+
+  void update_omap_root(Transaction &t, omap_root_t &oroot) final {
+    with_mutable_layout(t, [&oroot](onode_layout_t &mlayout) {
+      mlayout.omap_root.update(oroot);
+    });
+  }
+
+  void update_xattr_root(Transaction &t, omap_root_t &xroot) final {
+    with_mutable_layout(t, [&xroot](onode_layout_t &mlayout) {
+      mlayout.xattr_root.update(xroot);
+    });
+  }
+
+  void update_object_data(Transaction &t, object_data_t &odata) final {
+    with_mutable_layout(t, [&odata](onode_layout_t &mlayout) {
+      mlayout.object_data.update(odata);
+    });
+  }
+
+  void update_object_info(Transaction &t, ceph::bufferlist &oi_bl) final {
+    with_mutable_layout(t, [&oi_bl](onode_layout_t &mlayout) {
+      maybe_inline_memcpy(
+	&mlayout.oi[0],
+	oi_bl.c_str(),
+	oi_bl.length(),
+	onode_layout_t::MAX_OI_LENGTH);
+      mlayout.oi_size = oi_bl.length();
+    });
+  }
+
+  void clear_object_info(Transaction &t) final {
+    with_mutable_layout(t, [](onode_layout_t &mlayout) {
+      memset(&mlayout.oi[0], 0, mlayout.oi_size);
+      mlayout.oi_size = 0;
+    });
+  }
+
+  void update_snapset(Transaction &t, ceph::bufferlist &ss_bl) final {
+    with_mutable_layout(t, [&ss_bl](onode_layout_t &mlayout) {
+      maybe_inline_memcpy(
+	&mlayout.ss[0],
+	ss_bl.c_str(),
+	ss_bl.length(),
+	onode_layout_t::MAX_OI_LENGTH);
+      mlayout.ss_size = ss_bl.length();
+    });
+  }
+
+  void clear_snapset(Transaction &t) final {
+    with_mutable_layout(t, [](onode_layout_t &mlayout) {
+      memset(&mlayout.ss[0], 0, mlayout.ss_size);
+      mlayout.ss_size = 0;
+    });
+  }
+
 };
 
 struct object_data_handler_test_t:

From 544985f08969704e29a52b9a0005c8233c08cc92 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 27 Sep 2023 17:15:29 +0800
Subject: [PATCH 0109/2492] crimson/os/seastore/onode_manager: drop write_dirty

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/onode_manager.h       |  6 ----
 .../staged-fltree/fltree_onode_manager.cc     | 28 -------------------
 .../staged-fltree/fltree_onode_manager.h      |  4 ---
 src/crimson/os/seastore/seastore.cc           |  2 --
 .../onode_tree/test_fltree_onode_manager.cc   | 10 -------
 5 files changed, 50 deletions(-)

diff --git a/src/crimson/os/seastore/onode_manager.h b/src/crimson/os/seastore/onode_manager.h
index 123c9e4f865e..5a454906dc6f 100644
--- a/src/crimson/os/seastore/onode_manager.h
+++ b/src/crimson/os/seastore/onode_manager.h
@@ -58,12 +58,6 @@ class OnodeManager {
     Transaction &trans,
     const std::vector<ghobject_t> &hoids) = 0;
 
-  using write_dirty_iertr = base_iertr;
-  using write_dirty_ret = write_dirty_iertr::future<>;
-  virtual write_dirty_ret write_dirty(
-    Transaction &trans,
-    const std::vector<OnodeRef> &onodes) = 0;
-
   using erase_onode_iertr = base_iertr;
   using erase_onode_ret = erase_onode_iertr::future<>;
   virtual erase_onode_ret erase_onode(
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
index a1f593889d52..dc6c183f5385 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
@@ -87,34 +87,6 @@ FLTreeOnodeManager::get_or_create_onodes(
     });
 }
 
-FLTreeOnodeManager::write_dirty_ret FLTreeOnodeManager::write_dirty(
-  Transaction &trans,
-  const std::vector<OnodeRef> &onodes)
-{
-  return trans_intr::do_for_each(
-    onodes,
-    [&trans](auto &onode) -> eagain_ifuture<> {
-      if (!onode) {
-	return eagain_iertr::make_ready_future<>();
-      }
-      auto &flonode = static_cast<FLTreeOnode&>(*onode);
-      if (!flonode.is_alive()) {
-	return eagain_iertr::make_ready_future<>();
-      }
-      switch (flonode.status) {
-      case FLTreeOnode::status_t::MUTATED: {
-        flonode.populate_recorder(trans);
-        return eagain_iertr::make_ready_future<>();
-      }
-      case FLTreeOnode::status_t::STABLE: {
-        return eagain_iertr::make_ready_future<>();
-      }
-      default:
-        __builtin_unreachable();
-      }
-    });
-}
-
 FLTreeOnodeManager::erase_onode_ret FLTreeOnodeManager::erase_onode(
   Transaction &trans,
   OnodeRef &onode)
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
index 619609419ad8..d5a0dfc7bd5a 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
@@ -212,10 +212,6 @@ class FLTreeOnodeManager : public crimson::os::seastore::OnodeManager {
     Transaction &trans,
     const std::vector<ghobject_t> &hoids) final;
 
-  write_dirty_ret write_dirty(
-    Transaction &trans,
-    const std::vector<OnodeRef> &onodes) final;
-
   erase_onode_ret erase_onode(
     Transaction &trans,
     OnodeRef &onode) final;
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index edbc9a0115ec..4a52fa22d998 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1226,8 +1226,6 @@ seastar::future<> SeaStore::Shard::do_transaction_no_callbacks(
                 return seastar::make_ready_future<seastar::stop_iteration>(
                   seastar::stop_iteration::yes);
               };
-            }).si_then([this, &ctx, &d_onodes] {
-              return onode_manager->write_dirty(*ctx.transaction, d_onodes);
             });
         }).si_then([this, &ctx] {
           return transaction_manager->submit_transaction(*ctx.transaction);
diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
index 3d98dcf5aef2..cd5dd4407f80 100644
--- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
+++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
@@ -121,13 +121,6 @@ struct fltree_onode_manager_test_t
         return manager->get_or_create_onode(t, p_kv->key);
       }).unsafe_get0();
       std::invoke(f, t, *onode, p_kv->value);
-      with_trans_intr(t, [&](auto &t) {
-	if (onode->is_alive()) {
-	  return manager->write_dirty(t, {onode});
-	} else {
-	  return OnodeManager::write_dirty_iertr::now();
-	}
-      }).unsafe_get0();
     });
   }
 
@@ -182,9 +175,6 @@ struct fltree_onode_manager_test_t
         boost::tie(onode, p_item) = tup;
         std::invoke(f, t, *onode, *p_item);
       }
-      with_trans_intr(t, [&](auto &t) {
-        return manager->write_dirty(t, onodes);
-      }).unsafe_get0();
     });
   }
 

From db726d7a44cde04f82f2e9d2cacd5ac0edfd86ac Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 6 Sep 2023 19:29:48 +0300
Subject: [PATCH 0110/2492] vstart.sh: make sure that --localhost does only
 bind to 127.0.0.1 in all cases

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/vstart.sh | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index a33bcde3234b..99ecdf510e9b 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -174,6 +174,7 @@ pmem_enabled=0
 zoned_enabled=0
 io_uring_enabled=0
 with_jaeger=0
+force_addr=0
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -344,6 +345,7 @@ case $1 in
         ;;
     -l | --localhost)
         ip="127.0.0.1"
+        force_addr=1
         ;;
     -i)
         [ -z "$2" ] && usage_exit
@@ -737,6 +739,12 @@ prepare_conf() {
         msgr_conf="ms bind msgr2 = false
                    ms bind msgr1 = true"
     fi
+    if [ $force_addr -eq 1 ]; then
+        msgr_conf+="
+                   public bind addr = $IP
+                   public addr = $IP
+                   cluster addr = $IP"
+    fi
 
     wconf <<EOF
 ; generated by vstart.sh on `date`
@@ -915,6 +923,26 @@ EOF
         osd pool default crimson = true
 EOF
     fi
+
+    # this is most probably a bug in ceph_mon
+    # but public_bind_addr set in [global] doesn't work
+    # when mon_host is also provided, resulting
+    # in different public and bind addresses (ports)
+    # As a result, no client can connect to the mon.
+    # The problematic code is in ceph_mon.cc, it looks like
+    #
+    #   // check if the public_bind_addr option is set
+    #  if (!g_conf()->public_bind_addr.is_blank_ip()) {
+    #    bind_addrs = make_mon_addrs(g_conf()->public_bind_addr);
+    #  }
+    #
+    if [ $force_addr -eq 1 ]; then
+        wconf <<EOF
+        ; this is to counter the explicit public_bind_addr in the [global] section
+        ; see src/vstart.sh for more info
+        public bind addr =
+EOF
+    fi   
 }
 
 write_logrotate_conf() {

From 7f7858748d39510a0d699f91df74a17ec9f4ff3d Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 28 Sep 2023 13:24:35 +0530
Subject: [PATCH 0111/2492] cephfs-shell: use pkg_resources rather than
 packaging module

`pkg_resources` is already being used by other py scripts.

Fixes: https://tracker.ceph.com/issues/62739
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/tools/cephfs/shell/cephfs-shell | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/tools/cephfs/shell/cephfs-shell b/src/tools/cephfs/shell/cephfs-shell
index bdd1bb4f0b2b..d1726940ffde 100755
--- a/src/tools/cephfs/shell/cephfs-shell
+++ b/src/tools/cephfs/shell/cephfs-shell
@@ -16,13 +16,14 @@ import shlex
 import stat
 import errno
 
-from packaging import version
+from pkg_resources import packaging  # type: ignore
 
 from cmd2 import Cmd
 from cmd2 import __version__ as cmd2_version
 # XXX: In cmd2 versions < 1.0.1, we'll get SystemExit(2) instead of
 # Cmd2ArgparseError
-if version.parse(cmd2_version) >= version.parse("1.0.1"):
+Version = packaging.version.Version
+if Version(cmd2_version) >= Version("1.0.1"):
     from cmd2.exceptions import Cmd2ArgparseError
 else:
     # HACK: so that we don't have check for version everywhere

From 5be9213738bfbde12172b3a5158667ebe9dfa13d Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 28 Sep 2023 13:25:46 +0530
Subject: [PATCH 0112/2492] doc/cephfs-shell: drop installing packaging module

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 doc/man/8/cephfs-shell.rst          |  2 +-
 src/tools/cephfs/shell/cephfs-shell | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/man/8/cephfs-shell.rst b/doc/man/8/cephfs-shell.rst
index 91136f123b6c..974ba37d9110 100644
--- a/doc/man/8/cephfs-shell.rst
+++ b/doc/man/8/cephfs-shell.rst
@@ -56,7 +56,7 @@ Options
 
 .. code:: bash
 
-    [build]$ python3 -m venv venv && source venv/bin/activate && pip3 install cmd2 colorama packaging
+    [build]$ python3 -m venv venv && source venv/bin/activate && pip3 install cmd2 colorama
     [build]$ source vstart_environment.sh && source venv/bin/activate && python3 ../src/tools/cephfs/shell/cephfs-shell
 
 Commands
diff --git a/src/tools/cephfs/shell/cephfs-shell b/src/tools/cephfs/shell/cephfs-shell
index d1726940ffde..d2c050f89316 100755
--- a/src/tools/cephfs/shell/cephfs-shell
+++ b/src/tools/cephfs/shell/cephfs-shell
@@ -1701,11 +1701,11 @@ def read_shell_conf(shell, shell_conf_file):
 
     sec = 'cephfs-shell'
     opts = []
-    if version.parse(cmd2_version) >= version.parse("0.10.0"):
+    if Version(cmd2_version) >= Version("0.10.0"):
         for attr in shell.settables.keys():
             opts.append(attr)
     else:
-        if version.parse(cmd2_version) <= version.parse("0.9.13"):
+        if Version(cmd2_version) <= Version("0.9.13"):
             # hardcoding options for 0.7.9 because -
             # 1. we use cmd2 v0.7.9 with teuthology and
             # 2. there's no way distinguish between a shell setting and shell
@@ -1714,7 +1714,7 @@ def read_shell_conf(shell, shell_conf_file):
                     'continuation_prompt', 'debug', 'echo', 'editor',
                     'feedback_to_output', 'locals_in_py', 'prompt', 'quiet',
                     'timing']
-        elif version.parse(cmd2_version) >= version.parse("0.9.23"):
+        elif Version(cmd2_version) >= Version("0.9.23"):
             opts.append('allow_style')
         # no equivalent option was defined by cmd2.
         else:
@@ -1769,7 +1769,7 @@ def manage_args():
     args.exe_and_quit = False    # Execute and quit, don't launch the shell.
 
     if args.batch:
-        if version.parse(cmd2_version) <= version.parse("0.9.13"):
+        if Version(cmd2_version) <= Version("0.9.13"):
             args.commands = ['load ' + args.batch, ',quit']
         else:
             args.commands = ['run_script ' + args.batch, ',quit']
@@ -1814,7 +1814,7 @@ def execute_cmds_and_quit(args):
     # value to indicate whether the execution of the commands should stop, but
     # since 0.9.7 it returns the return value of do_* methods only if it's
     # not None. When it is None it returns False instead of None.
-    if version.parse(cmd2_version) <= version.parse("0.9.6"):
+    if Version(cmd2_version) <= Version("0.9.6"):
         stop_exec_val = None
     else:
         stop_exec_val = False

From 6a6a9ddd46e5dd2135dfd241fc0dff8ff7472a06 Mon Sep 17 00:00:00 2001
From: avanthakkar <avanjohn@gmail.com>
Date: Thu, 5 Oct 2023 17:48:34 +0530
Subject: [PATCH 0113/2492] mgr/cephadm is not defining haproxy tcp
 healthchecks for Ganesha

Fixes: https://tracker.ceph.com/issues/62638
Signed-off-by: avanthakkar <avanjohn@gmail.com>
---
 .../mgr/cephadm/templates/services/ingress/haproxy.cfg.j2     | 2 +-
 src/pybind/mgr/cephadm/tests/test_services.py                 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
index 100acce401ba..c114a8cba115 100644
--- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
@@ -85,6 +85,6 @@ backend backend
     default-server {{ default_server_opts|join(" ") }}
 {% endif %}
     {% for server in servers %}
-    server {{ server.name }} {{ server.ip }}:{{ server.port }}
+    server {{ server.name }} {{ server.ip }}:{{ server.port }} check
     {% endfor %}
 {% endif %}
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index 21c62ffd06fb..52a3b3619668 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -1647,7 +1647,7 @@ def fake_get_addr(hostname: str) -> str:
         )
         if enable_haproxy_protocol:
             haproxy_txt += '    default-server send-proxy-v2\n'
-        haproxy_txt += '    server nfs.foo.0 192.168.122.111:12049\n'
+        haproxy_txt += '    server nfs.foo.0 192.168.122.111:12049 check\n'
         haproxy_expected_conf = {
             'files': {'haproxy.cfg': haproxy_txt}
         }
@@ -2425,7 +2425,7 @@ def fake_keys():
             '    balance     source\n'
             '    hash-type   consistent\n'
             '    default-server send-proxy-v2\n'
-            '    server nfs.foo.0 192.168.122.111:12049\n'
+            '    server nfs.foo.0 192.168.122.111:12049 check\n'
         )
         haproxy_expected_conf = {
             'files': {'haproxy.cfg': haproxy_txt}

From 6caf2c437128809b294506f61237f1aecf16aea8 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 9 Oct 2023 12:06:42 +0000
Subject: [PATCH 0114/2492] tools/ceph_monstore_tool: add ceph-specific option
 example

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/tools/ceph_monstore_tool.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc
index 9da7f5f5c40e..437b45cd6760 100644
--- a/src/tools/ceph_monstore_tool.cc
+++ b/src/tools/ceph_monstore_tool.cc
@@ -232,6 +232,7 @@ void usage(const char *n, po::options_description &d)
     << "\nPlease Note:\n"
     << "* Ceph-specific options should be in the format --option-name=VAL\n"
     << "  (specifically, do not forget the '='!!)\n"
+    << "  e.g., 'dump-keys --debug-rocksdb=0'\n"
     << "* Command-specific options need to be passed after a '--'\n"
     << "  e.g., 'get monmap -- --version 10 --out /tmp/foo'"
     << std::endl;

From 23c7d1e749022f7a15c92296f5367bc651cecc5d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 6 Aug 2023 13:04:11 +0000
Subject: [PATCH 0115/2492] tools/ceph_monstore_tool: add get-key command

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/tools/ceph_monstore_tool.cc | 98 +++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc
index 437b45cd6760..f6b0dc62ae55 100644
--- a/src/tools/ceph_monstore_tool.cc
+++ b/src/tools/ceph_monstore_tool.cc
@@ -210,6 +210,7 @@ void usage(const char *n, po::options_description &d)
   << "                                  (default: last committed)\n"
   << "  get crushmap [-- options]       get crushmap (version VER if specified)\n"
   << "                                  (default: last committed)\n"
+  << "  get-key PREFIX KEY [-- options] get key\n"
   << "  show-versions [-- options]      show the first&last committed version of map\n"
   << "                                  (show-versions -- --help for more info)\n"
   << "  dump-keys                       dumps store keys to FILE\n"
@@ -999,6 +1000,103 @@ int main(int argc, char **argv) {
                 << " version " << v << " to " << outpath
                 << std::endl;
     }
+} else if (cmd == "get-key") {
+    string outpath;
+    string prefix;
+    string key;
+
+    // visible options for this command
+    po::options_description op_desc("Allowed 'get-key' options");
+    op_desc.add_options()
+      ("help,h", "produce this help message")
+      ("out,o", po::value<string>(&outpath),
+       "output file (default: stdout)")
+      ("readable,r", "print the map information in human readable format")
+      ;
+    // this is going to be a positional argument; we don't want to show
+    // it as an option during --help, but we do want to have it captured
+    // when parsing.
+    po::options_description hidden_op_desc("Hidden 'get-key' options");
+    hidden_op_desc.add_options()
+      ("prefix", po::value<string>(&prefix),"prefix")
+      ("key", po::value<string>(&key),"key")
+      ;
+    po::positional_options_description op_positional;
+    op_positional.add("prefix", 1);
+    op_positional.add("key", 1);
+
+
+    po::variables_map op_vm;
+    int r = parse_cmd_args(&op_desc, &hidden_op_desc, &op_positional,
+                           subcmds, &op_vm);
+    if (r < 0) {
+      return -r;
+    }
+
+    if (op_vm.count("help") || prefix.empty()) {
+      usage(argv[0], op_desc);
+      return 0;
+    }
+
+    int fd = STDOUT_FILENO;
+    if (!outpath.empty()){
+      fd = ::open(outpath.c_str(), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0666);
+      if (fd < 0) {
+        std::cerr << "error opening output file: "
+          << cpp_strerror(errno) << std::endl;
+        return EINVAL;
+      }
+    }
+
+    auto close_fd = make_scope_guard([&] {
+      ::close(fd);
+      if (r < 0 && fd != STDOUT_FILENO) {
+        ::remove(outpath.c_str());
+      }
+    });
+    bufferlist bl;
+    r = 0;
+    std::cout << prefix << " " << key << std::endl;
+    r = st.get(prefix, key, bl);
+    if (r < 0) {
+      std::cerr << "Error getting key: " << cpp_strerror(r) << std::endl;
+      return EINVAL;
+    }
+
+    if (op_vm.count("readable")) {
+      try {
+        if (prefix == "osd_snap") {
+          auto p = bl.cbegin();
+          if (key.starts_with("purged_epoch_")) {
+            map<int64_t,snap_interval_set_t> val;
+            ceph::decode(val, p);
+            std::cout << val << std::endl;
+          } else if (key.starts_with("purged_snap_")) {
+            snapid_t first_snap, end_snap;
+            epoch_t epoch;
+            ceph::decode(first_snap, p);
+            ceph::decode(end_snap, p);
+            ceph::decode(epoch, p);
+            std::cout << "first_snap:" << first_snap
+                      << " end_snap: " << end_snap
+                      << " epoch: " << epoch
+                      << std::endl;
+          }
+        } else {
+          std::cerr << "This type of readable key does not exist: " << prefix
+                    << std::endl << "You can only specify[osd_snap]" << std::endl;
+        }
+      } catch (const buffer::error &err) {
+        std::cerr << "Could not decode for human readable output (you may still"
+	  " use non-readable mode).  Detail: " << err.what() << std::endl;
+      }
+    }
+
+    bl.write_fd(fd);
+
+    if (!outpath.empty()) {
+      std::cout << "wrote " << prefix << " " <<  key <<  " to " << outpath << std::endl;
+    }
   } else if (cmd == "show-versions") {
     string map_type; //map type:osdmap,monmap...
     // visible options for this command

From cfd2cc9449ddf291d0fc03d96eb18806afc39578 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 6 Aug 2023 13:03:19 +0000
Subject: [PATCH 0116/2492] tools/ceph_monstore_tool: add remove-key command

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/MonitorDBStore.h        |  8 ++++++++
 src/tools/ceph_monstore_tool.cc | 27 +++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index 11608e9e553c..b7dfc50b0a17 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -604,6 +604,14 @@ class MonitorDBStore
     return combine_strings(prefix, os.str());
   }
 
+  int clear_key(const std::string& prefix, const std::string& key) {
+    ceph_assert(!prefix.empty());
+    ceph_assert(!key.empty());
+    KeyValueDB::Transaction dbt = db->get_transaction();
+    dbt->rmkey(prefix, key);
+    return db->submit_transaction_sync(dbt);
+  }
+
   void clear(std::set<std::string>& prefixes) {
     KeyValueDB::Transaction dbt = db->get_transaction();
 
diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc
index f6b0dc62ae55..8c6c9c7233b2 100644
--- a/src/tools/ceph_monstore_tool.cc
+++ b/src/tools/ceph_monstore_tool.cc
@@ -211,6 +211,7 @@ void usage(const char *n, po::options_description &d)
   << "  get crushmap [-- options]       get crushmap (version VER if specified)\n"
   << "                                  (default: last committed)\n"
   << "  get-key PREFIX KEY [-- options] get key\n"
+  << "  remove-key PREFIX KEY           remove key\n"
   << "  show-versions [-- options]      show the first&last committed version of map\n"
   << "                                  (show-versions -- --help for more info)\n"
   << "  dump-keys                       dumps store keys to FILE\n"
@@ -1410,6 +1411,32 @@ int main(int argc, char **argv) {
     err = rewrite_crush(argv[0], subcmds, st);
   } else if (cmd == "rebuild") {
     err = rebuild_monstore(argv[0], subcmds, st);
+  } else if (cmd == "remove-key") {
+    string prefix, key;
+    // No visible options for this command
+    po::options_description op_desc("Allowed 'get' options");
+    po::options_description hidden_op_desc("Hidden 'get' options");
+    hidden_op_desc.add_options()
+      ("prefix", po::value<string>(&prefix),"prefix")
+      ("key", po::value<string>(&key),"key")
+      ;
+    po::positional_options_description op_positional;
+    op_positional.add("prefix", 1);
+    op_positional.add("key", 1);
+
+    po::variables_map op_vm;
+    int r = parse_cmd_args(&op_desc, &hidden_op_desc, &op_positional,
+                           subcmds, &op_vm);
+    if (r < 0) {
+      return -r;
+    }
+    r = st.clear_key(prefix, key);
+    if (r < 0) {
+      std::cerr << "error removing ("
+                << prefix << "," << key << ")"
+                << std::endl;
+      return r;
+    }
   } else {
     std::cerr << "Unrecognized command: " << cmd << std::endl;
     usage(argv[0], desc);

From 42f94dd88fab7d46fb7478ab911c9882b124af86 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 9 Oct 2023 13:43:28 +0000
Subject: [PATCH 0117/2492] doc/man/8/ceph-monstore-tool.rst: doc fixes

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 doc/man/8/ceph-monstore-tool.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/man/8/ceph-monstore-tool.rst b/doc/man/8/ceph-monstore-tool.rst
index 9396df0b6057..02700a50b22e 100644
--- a/doc/man/8/ceph-monstore-tool.rst
+++ b/doc/man/8/ceph-monstore-tool.rst
@@ -18,13 +18,13 @@ Description
 :program:`ceph-monstore-tool` is used to manipulate MonitorDBStore's data
 (monmap, osdmap, etc.) offline. It is similar to `ceph-kvstore-tool`.
 
-The default RocksDB debug level is `0`. This can be changed using `--debug`.
-
 Note:
     Ceph-specific options take the format `--option-name=VAL`
     DO NOT FORGET THE EQUALS SIGN. ('=')
+    for example, `dump-keys --debug-rocksdb=0`
+
     Command-specific options must be passed after a `--`
-    for example, `get monmap --debug -- --version 10 --out /tmp/foo`
+    for example, `get monmap -- --version 10 --out /tmp/foo`
 
 Commands
 ========
@@ -49,8 +49,11 @@ Commands
 :command:`get crushmap [-- options]`
     Get crushmap (version VER if specified) (default: last committed).
 
-:command:`get osd_snap <key> [-- options]`
-    Get osd_snap key (`purged_snap` or `purged_epoch`).
+:command:`get-key <prefix> <key> [-- options]`
+    Get key to FILE (default: stdout).
+
+:command:`remove-key <prefix> <key> [-- options]`
+    Remove key.
 
 :command:`dump-keys`
     Dump store keys to FILE (default: stdout).
@@ -73,9 +76,6 @@ Commands
 :command:`rebuild`
     Rebuild store.
 
-:command:`rm <prefix> <key>`
-    Remove specified key from the store.
-
 Availability
 ============
 

From a10f6e37e4291c682e9b7dccf1d819a04f9b3cbb Mon Sep 17 00:00:00 2001
From: Vedansh Bhartia <vedanshbhartia@gmail.com>
Date: Wed, 14 Jun 2023 18:14:10 +0530
Subject: [PATCH 0118/2492] rgw: Add a wrapper for librados::AioCompletion to
 prevent memory leaks

Signed-off-by: Vedansh Bhartia <vedanshbhartia@gmail.com>
---
 src/rgw/driver/rados/rgw_gc.cc | 30 +++++++++++++++---------------
 src/rgw/rgw_aio.cc             |  1 +
 src/rgw/rgw_common.h           |  6 ++++++
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_gc.cc b/src/rgw/driver/rados/rgw_gc.cc
index 4705c46fff4b..4b30219734cc 100644
--- a/src/rgw/driver/rados/rgw_gc.cc
+++ b/src/rgw/driver/rados/rgw_gc.cc
@@ -170,9 +170,9 @@ void RGWGC::on_defer_canceled(const cls_rgw_gc_obj_info& info)
   cls_rgw_gc_queue_defer_entry(op, cct->_conf->rgw_gc_obj_min_wait, info);
   cls_rgw_gc_remove(op, {tag});
 
-  auto c = librados::Rados::aio_create_completion(nullptr, nullptr);
-  store->gc_aio_operate(obj_names[i], c, &op);
-  c->release();
+  aio_completion_ptr c{librados::Rados::aio_create_completion(nullptr, nullptr)};
+
+  store->gc_aio_operate(obj_names[i], c.get(), &op);
 }
 
 int RGWGC::async_defer_chain(const string& tag, const cls_rgw_obj_chain& chain)
@@ -191,9 +191,9 @@ int RGWGC::async_defer_chain(const string& tag, const cls_rgw_obj_chain& chain)
     // enqueue succeeds
     cls_rgw_gc_remove(op, {tag});
 
-    auto c = librados::Rados::aio_create_completion(nullptr, nullptr);
-    int ret = store->gc_aio_operate(obj_names[i], c, &op);
-    c->release();
+    aio_completion_ptr c{librados::Rados::aio_create_completion(nullptr, nullptr)};
+
+    int ret = store->gc_aio_operate(obj_names[i], c.get(), &op);
     return ret;
   }
 
@@ -225,12 +225,11 @@ int RGWGC::remove(int index, const std::vector<string>& tags, AioCompletion **pc
   ObjectWriteOperation op;
   cls_rgw_gc_remove(op, tags);
 
-  auto c = librados::Rados::aio_create_completion(nullptr, nullptr);
-  int ret = store->gc_aio_operate(obj_names[index], c, &op);
-  if (ret < 0) {
-    c->release();
-  } else {
-    *pc = c;
+  aio_completion_ptr c{librados::Rados::aio_create_completion(nullptr, nullptr)};
+  int ret = store->gc_aio_operate(obj_names[index], c.get(), &op);
+  if (ret >= 0) {
+    *pc = c.get();
+    c.release();
   }
   return ret;
 }
@@ -391,12 +390,13 @@ class RGWGCIOManager {
       }
     }
 
-    auto c = librados::Rados::aio_create_completion(nullptr, nullptr);
-    int ret = ioctx->aio_operate(oid, c, op);
+    aio_completion_ptr c{librados::Rados::aio_create_completion(nullptr, nullptr)};
+    int ret = ioctx->aio_operate(oid, c.get(), op);
     if (ret < 0) {
       return ret;
     }
-    ios.push_back(IO{IO::TailIO, c, oid, index, tag});
+    ios.push_back(IO{IO::TailIO, c.get(), oid, index, tag});
+    c.release();
 
     return 0;
   }
diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index 02e3411858ac..cd85ea6d7fab 100644
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -33,6 +33,7 @@ struct state {
 
   state(Aio* aio, librados::IoCtx ctx, AioResult& r)
     : aio(aio), ctx(std::move(ctx)),
+    // coverity[ctor_dtor_leak:SUPPRESS]
       c(librados::Rados::aio_create_completion(&r, &cb)) {}
 };
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index de2f7443164a..c1d19e49d948 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1842,3 +1842,9 @@ rgw_global_init(const std::map<std::string,std::string> *defaults,
 		    std::vector < const char* >& args,
 		    uint32_t module_type, code_environment_t code_env,
 		    int flags);
+
+
+struct AioCompletionDeleter {
+  void operator()(librados::AioCompletion* c) { c->release(); }
+};
+using aio_completion_ptr = std::unique_ptr<librados::AioCompletion, AioCompletionDeleter>;

From 41029edcac9b2509414c8d2e5d51e23fe6590c8a Mon Sep 17 00:00:00 2001
From: Vedansh Bhartia <vedanshbhartia@gmail.com>
Date: Mon, 9 Oct 2023 21:21:23 +0530
Subject: [PATCH 0119/2492] rgw: Add coverity annotations for uncaught
 exceptions in standalone binaries

Signed-off-by: Vedansh Bhartia <vedanshbhartia@gmail.com>
---
 src/rgw/driver/dbstore/dbstore_main.cc | 3 +++
 src/rgw/rgw_admin.cc                   | 3 +++
 src/rgw/rgw_es_main.cc                 | 3 +++
 src/rgw/rgw_jsonparser.cc              | 4 +++-
 src/rgw/rgw_main.cc                    | 4 ++++
 src/rgw/rgw_multiparser.cc             | 4 ++++
 src/rgw/rgw_object_expirer.cc          | 3 +++
 src/rgw/rgw_polparser.cc               | 3 +++
 src/rgw/rgw_token.cc                   | 3 +++
 9 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/dbstore/dbstore_main.cc b/src/rgw/driver/dbstore/dbstore_main.cc
index 4fff38ced279..46d4106ca436 100644
--- a/src/rgw/driver/dbstore/dbstore_main.cc
+++ b/src/rgw/driver/dbstore/dbstore_main.cc
@@ -119,6 +119,9 @@ void* process(void *arg)
   return 0;
 }
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, char *argv[])
 {
   string tenant = "Redhat";
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index a4e17b8895ae..01a07f2c2dea 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -3305,6 +3305,9 @@ void init_realm_param(CephContext *cct, string& var, std::optional<string>& opt_
   }
 }
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, const char **argv)
 {
   auto args = argv_to_vec(argc, argv);
diff --git a/src/rgw/rgw_es_main.cc b/src/rgw/rgw_es_main.cc
index 6cfbc9352926..d84f9ecadd58 100644
--- a/src/rgw/rgw_es_main.cc
+++ b/src/rgw/rgw_es_main.cc
@@ -14,6 +14,9 @@
 
 using namespace std;
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, char *argv[])
 {
   auto args = argv_to_vec(argc, argv);
diff --git a/src/rgw/rgw_jsonparser.cc b/src/rgw/rgw_jsonparser.cc
index 6541630b286d..a6c99c639890 100644
--- a/src/rgw/rgw_jsonparser.cc
+++ b/src/rgw/rgw_jsonparser.cc
@@ -56,7 +56,9 @@ struct UserInfo {
   }
 };
 
-
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, char **argv) {
   JSONParser parser;
 
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 27b02f841951..4d31db8072f5 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -57,7 +57,11 @@ static int usage()
 
 /*
  * start up the RADOS connection and then handle HTTP messages as they come in
+ *
+ * This has an uncaught exception. Even if the exception is caught, the program
+ * would need to be terminated, so the warning is simply suppressed.
  */
+// coverity[root_function:SUPPRESS]
 int main(int argc, char *argv[])
 { 
   int r{0};
diff --git a/src/rgw/rgw_multiparser.cc b/src/rgw/rgw_multiparser.cc
index a8778abd9a08..c7a37213c4db 100644
--- a/src/rgw/rgw_multiparser.cc
+++ b/src/rgw/rgw_multiparser.cc
@@ -14,6 +14,10 @@
 
 using namespace std;
 
+
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, char **argv) {
   RGWMultiXMLParser parser;
 
diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc
index 7a49fc8d161e..0470b1a6e6fc 100644
--- a/src/rgw/rgw_object_expirer.cc
+++ b/src/rgw/rgw_object_expirer.cc
@@ -51,6 +51,9 @@ static void usage()
   generic_server_usage();
 }
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(const int argc, const char **argv)
 {
   auto args = argv_to_vec(argc, argv);
diff --git a/src/rgw/rgw_polparser.cc b/src/rgw/rgw_polparser.cc
index eca5066b3ce3..3991181809ee 100644
--- a/src/rgw/rgw_polparser.cc
+++ b/src/rgw/rgw_polparser.cc
@@ -50,6 +50,9 @@ void usage(std::string_view cmdname)
 	    << std::endl;
 }
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, const char** argv)
 {
   std::string_view cmdname = argv[0];
diff --git a/src/rgw/rgw_token.cc b/src/rgw/rgw_token.cc
index 999d46e0e229..8ffac69c8313 100644
--- a/src/rgw/rgw_token.cc
+++ b/src/rgw/rgw_token.cc
@@ -60,6 +60,9 @@ void usage()
   generic_client_usage();
 }
 
+// This has an uncaught exception. Even if the exception is caught, the program
+// would need to be terminated, so the warning is simply suppressed.
+// coverity[root_function:SUPPRESS]
 int main(int argc, char **argv)
 {
   auto args = argv_to_vec(argc, argv);

From 2ee29047ca564f04f1d27dc0aee09dcf42ab9c5a Mon Sep 17 00:00:00 2001
From: cuiming_yewu <cuiming_yewu@cmss.chinamobile.com>
Date: Fri, 22 Sep 2023 10:45:30 +0800
Subject: [PATCH 0120/2492] kv:resolve three complication warnings

Resolve three warning issues in the derived classes overriding base class virtual functions with mismatched parameter lists at lines 69, 72, and 74 in src/kv/rocksdb_cache/ShardedCache.h.
with ../src/kv/rocksdb_cache/ShardedCache.h:74:16: warning:   by 'virtual bool rocksdb_cache::ShardedCache::Release(rocksdb::Cache::Handle*, bool)' [-Woverloaded-virtual]
   virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
                ^~~~~~~
../src/kv/rocksdb_cache/ShardedCache.h:72:35: warning:   by 'virtual rocksdb::Cache::Handle* rocksdb_cache::ShardedCache::Lookup(const rocksdb::Slice&, rocksdb::Statistics*)' [-Woverloaded-virtual]
   virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
                                   ^~~~~~
../src/kv/rocksdb_cache/ShardedCache.h:69:27: warning:   by 'virtual rocksdb::Status rocksdb_cache::ShardedCache::Insert(const rocksdb::Slice&, void*, size_t, rocksdb::Cache::DeleterFn, rocksdb::Cache::Handle**, rocksdb::Cache::Priority)' [-Woverloaded-virtual]
   virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
                           ^~~~~~

Signed-off-by: cuiming <cuiming_yewu@cmss.chinamobile.com>
---
 src/kv/rocksdb_cache/ShardedCache.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/kv/rocksdb_cache/ShardedCache.h b/src/kv/rocksdb_cache/ShardedCache.h
index ef4b10d8f273..63a56c4577eb 100644
--- a/src/kv/rocksdb_cache/ShardedCache.h
+++ b/src/kv/rocksdb_cache/ShardedCache.h
@@ -66,11 +66,14 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
   virtual ~ShardedCache() = default;
   // rocksdb::Cache
   virtual const char* Name() const override = 0;
+  using rocksdb::Cache::Insert;
   virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
                                  DeleterFn,
                                  rocksdb::Cache::Handle** handle, Priority priority) override;
+  using rocksdb::Cache::Lookup;
   virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
   virtual bool Ref(rocksdb::Cache::Handle* handle) override;
+  using rocksdb::Cache::Release;
   virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
   virtual void* Value(Handle* handle) override = 0;
   virtual void Erase(const rocksdb::Slice& key) override;

From 4ff02f53fe722c20dbf0bb51d9b786d5f94d18ab Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Sat, 7 Oct 2023 14:01:32 +0800
Subject: [PATCH 0121/2492] crimson/os/seastore/onode_manager: avoid
 unnecessary delta related overhead

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../staged-fltree/fltree_onode_manager.cc     | 126 ++++++++++++-
 .../staged-fltree/fltree_onode_manager.h      | 175 ++++++++++++------
 .../onode_tree/test_fltree_onode_manager.cc   |   9 +-
 3 files changed, 246 insertions(+), 64 deletions(-)

diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
index dc6c183f5385..6243252682aa 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.cc
@@ -9,6 +9,125 @@ SET_SUBSYS(seastore_onode);
 
 namespace crimson::os::seastore::onode {
 
+void FLTreeOnode::Recorder::apply_value_delta(
+  ceph::bufferlist::const_iterator &bliter,
+  NodeExtentMutable &value,
+  laddr_t value_addr)
+{
+  LOG_PREFIX(FLTreeOnode::Recorder::apply_value_delta);
+  delta_op_t op;
+  try {
+    ceph::decode(op, bliter);
+    auto &mlayout = *reinterpret_cast<onode_layout_t*>(value.get_write());
+    switch (op) {
+    case delta_op_t::UPDATE_ONODE_SIZE:
+      DEBUG("update onode size");
+      bliter.copy(sizeof(mlayout.size), (char *)&mlayout.size);
+      break;
+    case delta_op_t::UPDATE_OMAP_ROOT:
+      DEBUG("update omap root");
+      bliter.copy(sizeof(mlayout.omap_root), (char *)&mlayout.omap_root);
+      break;
+    case delta_op_t::UPDATE_XATTR_ROOT:
+      DEBUG("update xattr root");
+      bliter.copy(sizeof(mlayout.xattr_root), (char *)&mlayout.xattr_root);
+      break;
+    case delta_op_t::UPDATE_OBJECT_DATA:
+      DEBUG("update object data");
+      bliter.copy(sizeof(mlayout.object_data), (char *)&mlayout.object_data);
+      break;
+    case delta_op_t::UPDATE_OBJECT_INFO:
+      DEBUG("update object info");
+      bliter.copy(onode_layout_t::MAX_OI_LENGTH, (char *)&mlayout.oi[0]);
+      ceph::decode(mlayout.oi_size, bliter);
+      break;
+    case delta_op_t::UPDATE_SNAPSET:
+      DEBUG("update snapset");
+      bliter.copy(onode_layout_t::MAX_SS_LENGTH, (char *)&mlayout.ss[0]);
+      ceph::decode(mlayout.ss_size, bliter);
+      break;
+    case delta_op_t::CLEAR_OBJECT_INFO:
+      DEBUG("clear object info");
+      memset(&mlayout.oi[0], 0, mlayout.oi_size);
+      mlayout.oi_size = 0;
+      break;
+    case delta_op_t::CLEAR_SNAPSET:
+      DEBUG("clear snapset");
+      memset(&mlayout.ss[0], 0, mlayout.ss_size);
+      mlayout.ss_size = 0;
+      break;
+    case delta_op_t::CREATE_DEFAULT:
+      mlayout = onode_layout_t{};
+      break;
+    default:
+      ceph_abort();
+    }
+  } catch (buffer::error& e) {
+    ceph_abort();
+  }
+}
+
+void FLTreeOnode::Recorder::encode_update(
+  NodeExtentMutable &payload_mut, delta_op_t op)
+{
+  LOG_PREFIX(FLTreeOnode::Recorder::encode_update);
+  auto &layout = *reinterpret_cast<const onode_layout_t*>(
+    payload_mut.get_read());
+  auto &encoded = get_encoded(payload_mut);
+  ceph::encode(op, encoded);
+  switch(op) {
+  case delta_op_t::UPDATE_ONODE_SIZE:
+    DEBUG("update onode size");
+    encoded.append(
+      (const char *)&layout.size,
+      sizeof(layout.size));
+    break;
+  case delta_op_t::UPDATE_OMAP_ROOT:
+    DEBUG("update omap root");
+    encoded.append(
+      (const char *)&layout.omap_root,
+      sizeof(layout.omap_root));
+    break;
+  case delta_op_t::UPDATE_XATTR_ROOT:
+    DEBUG("update xattr root");
+    encoded.append(
+      (const char *)&layout.xattr_root,
+      sizeof(layout.xattr_root));
+    break;
+  case delta_op_t::UPDATE_OBJECT_DATA:
+    DEBUG("update object data");
+    encoded.append(
+      (const char *)&layout.object_data,
+      sizeof(layout.object_data));
+    break;
+  case delta_op_t::UPDATE_OBJECT_INFO:
+    DEBUG("update object info");
+    encoded.append(
+      (const char *)&layout.oi[0],
+      onode_layout_t::MAX_OI_LENGTH);
+    ceph::encode(layout.oi_size, encoded);
+    break;
+  case delta_op_t::UPDATE_SNAPSET:
+    DEBUG("update snapset");
+    encoded.append(
+      (const char *)&layout.ss[0],
+      onode_layout_t::MAX_SS_LENGTH);
+    ceph::encode(layout.ss_size, encoded);
+    break;
+  case delta_op_t::CREATE_DEFAULT:
+    DEBUG("create default layout");
+    [[fallthrough]];
+  case delta_op_t::CLEAR_OBJECT_INFO:
+    DEBUG("clear object info");
+    [[fallthrough]];
+  case delta_op_t::CLEAR_SNAPSET:
+    DEBUG("clear snapset");
+    break;
+  default:
+    ceph_abort();
+  }
+}
+
 FLTreeOnodeManager::contains_onode_ret FLTreeOnodeManager::contains_onode(
   Transaction &trans,
   const ghobject_t &hoid)
@@ -57,9 +176,7 @@ FLTreeOnodeManager::get_or_create_onode(
 	cursor.value());
     if (created) {
       DEBUGT("created onode for entry for {}", trans, hoid);
-      onode->with_mutable_layout(trans, [](onode_layout_t &mlayout) {
-	mlayout = onode_layout_t{};
-      });
+      onode->create_default_layout(trans);
     }
     return get_or_create_onode_iertr::make_ready_future<OnodeRef>(onode);
   });
@@ -93,9 +210,6 @@ FLTreeOnodeManager::erase_onode_ret FLTreeOnodeManager::erase_onode(
 {
   auto &flonode = static_cast<FLTreeOnode&>(*onode);
   assert(flonode.is_alive());
-  if (flonode.status == FLTreeOnode::status_t::MUTATED) {
-    flonode.populate_recorder(trans);
-  }
   flonode.mark_delete();
   return tree.erase(trans, flonode);
 }
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
index d5a0dfc7bd5a..86f5cea883b2 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
@@ -47,6 +47,17 @@ struct FLTreeOnode final : Onode, Value {
       Value(std::forward<T>(args)...) {}
 
   struct Recorder : public ValueDeltaRecorder {
+    enum class delta_op_t : uint8_t {
+      UPDATE_ONODE_SIZE,
+      UPDATE_OMAP_ROOT,
+      UPDATE_XATTR_ROOT,
+      UPDATE_OBJECT_DATA,
+      UPDATE_OBJECT_INFO,
+      UPDATE_SNAPSET,
+      CLEAR_OBJECT_INFO,
+      CLEAR_SNAPSET,
+      CREATE_DEFAULT
+    };
     Recorder(bufferlist &bl) : ValueDeltaRecorder(bl) {}
 
     value_magic_t get_header_magic() const final {
@@ -56,18 +67,9 @@ struct FLTreeOnode final : Onode, Value {
     void apply_value_delta(
       ceph::bufferlist::const_iterator &bliter,
       NodeExtentMutable &value,
-      laddr_t) final {
-      assert(value.get_length() == sizeof(onode_layout_t));
-      bliter.copy(value.get_length(), value.get_write());
-    }
+      laddr_t value_addr) final;
 
-    void record_delta(NodeExtentMutable &value) {
-      // TODO: probably could use versioning, etc
-      assert(value.get_length() == sizeof(onode_layout_t));
-      ceph::buffer::ptr bptr(value.get_length());
-      memcpy(bptr.c_str(), value.get_read(), value.get_length());
-      get_encoded(value).append(bptr);
-    }
+    void encode_update(NodeExtentMutable &payload_mut, delta_op_t op);
   };
 
   bool is_alive() const {
@@ -78,86 +80,153 @@ struct FLTreeOnode final : Onode, Value {
     return *read_payload<onode_layout_t>();
   }
 
-  template <typename Func>
-  void with_mutable_layout(Transaction &t, Func&& f) {
+  template <typename layout_func_t>
+  void with_mutable_layout(
+    Transaction &t,
+    layout_func_t &&layout_func) {
     assert(status != status_t::DELETED);
     auto p = prepare_mutate_payload<
       onode_layout_t,
       Recorder>(t);
     status = status_t::MUTATED;
-    f(*reinterpret_cast<onode_layout_t*>(p.first.get_write()));
-    populate_recorder(t);
+    layout_func(p.first, p.second);
+    status = status_t::STABLE;
   }
 
-  void populate_recorder(Transaction &t) {
-    assert(status == status_t::MUTATED);
-    auto p = prepare_mutate_payload<
-      onode_layout_t,
-      Recorder>(t);
-    if (p.second) {
-      p.second->record_delta(
-        p.first);
-    }
-    status = status_t::STABLE;
+  void create_default_layout(Transaction &t) {
+    with_mutable_layout(
+      t,
+      [](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+	  payload_mut.get_write());
+	mlayout = onode_layout_t{};
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::CREATE_DEFAULT);
+	}
+    });
   }
 
   void update_onode_size(Transaction &t, uint32_t size) final {
-    with_mutable_layout(t, [size](onode_layout_t &mlayout) {
-      mlayout.size = size;
+    with_mutable_layout(
+      t,
+      [size](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	mlayout.size = size;
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_ONODE_SIZE);
+	}
     });
   }
 
   void update_omap_root(Transaction &t, omap_root_t &oroot) final {
-    with_mutable_layout(t, [&oroot](onode_layout_t &mlayout) {
-      mlayout.omap_root.update(oroot);
+    with_mutable_layout(
+      t,
+      [&oroot](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	mlayout.omap_root.update(oroot);
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_OMAP_ROOT);
+	}
     });
   }
 
   void update_xattr_root(Transaction &t, omap_root_t &xroot) final {
-    with_mutable_layout(t, [&xroot](onode_layout_t &mlayout) {
-      mlayout.xattr_root.update(xroot);
+    with_mutable_layout(
+      t,
+      [&xroot](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+	  payload_mut.get_write());
+	mlayout.xattr_root.update(xroot);
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_XATTR_ROOT);
+	}
     });
   }
 
   void update_object_data(Transaction &t, object_data_t &odata) final {
-    with_mutable_layout(t, [&odata](onode_layout_t &mlayout) {
-      mlayout.object_data.update(odata);
+    with_mutable_layout(
+      t,
+      [&odata](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	mlayout.object_data.update(odata);
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_OBJECT_DATA);
+	}
     });
   }
 
   void update_object_info(Transaction &t, ceph::bufferlist &oi_bl) final {
-    with_mutable_layout(t, [&oi_bl](onode_layout_t &mlayout) {
-      maybe_inline_memcpy(
-	&mlayout.oi[0],
-	oi_bl.c_str(),
-	oi_bl.length(),
-	onode_layout_t::MAX_OI_LENGTH);
-      mlayout.oi_size = oi_bl.length();
+    with_mutable_layout(
+      t,
+      [&oi_bl](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	maybe_inline_memcpy(
+	  &mlayout.oi[0],
+	  oi_bl.c_str(),
+	  oi_bl.length(),
+	  onode_layout_t::MAX_OI_LENGTH);
+	mlayout.oi_size = oi_bl.length();
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_OBJECT_INFO);
+	}
     });
   }
 
   void clear_object_info(Transaction &t) final {
-    with_mutable_layout(t, [](onode_layout_t &mlayout) {
-      memset(&mlayout.oi[0], 0, mlayout.oi_size);
-      mlayout.oi_size = 0;
+    with_mutable_layout(
+      t, [](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	memset(&mlayout.oi[0], 0, mlayout.oi_size);
+	mlayout.oi_size = 0;
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::CLEAR_OBJECT_INFO);
+	}
     });
   }
 
   void update_snapset(Transaction &t, ceph::bufferlist &ss_bl) final {
-    with_mutable_layout(t, [&ss_bl](onode_layout_t &mlayout) {
-      maybe_inline_memcpy(
-	&mlayout.ss[0],
-	ss_bl.c_str(),
-	ss_bl.length(),
-	onode_layout_t::MAX_OI_LENGTH);
-      mlayout.ss_size = ss_bl.length();
+    with_mutable_layout(
+      t,
+      [&ss_bl](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	maybe_inline_memcpy(
+	  &mlayout.ss[0],
+	  ss_bl.c_str(),
+	  ss_bl.length(),
+	  onode_layout_t::MAX_OI_LENGTH);
+	mlayout.ss_size = ss_bl.length();
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::UPDATE_SNAPSET);
+	}
     });
   }
 
   void clear_snapset(Transaction &t) final {
-    with_mutable_layout(t, [](onode_layout_t &mlayout) {
-      memset(&mlayout.ss[0], 0, mlayout.ss_size);
-      mlayout.ss_size = 0;
+    with_mutable_layout(
+      t,
+      [](NodeExtentMutable &payload_mut, Recorder *recorder) {
+	auto &mlayout = *reinterpret_cast<onode_layout_t*>(
+          payload_mut.get_write());
+	memset(&mlayout.ss[0], 0, mlayout.ss_size);
+	mlayout.ss_size = 0;
+	if (recorder) {
+	  recorder->encode_update(
+	    payload_mut, Recorder::delta_op_t::CLEAR_SNAPSET);
+	}
     });
   }
 
diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
index cd5dd4407f80..92ab147ef89b 100644
--- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
+++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
@@ -31,11 +31,10 @@ struct onode_item_t {
 
   void initialize(Transaction& t, Onode& value) const {
     auto &ftvalue = static_cast<FLTreeOnode&>(value);
-    ftvalue.with_mutable_layout(t, [this, &value](auto &mlayout) {
-      mlayout.size = size;
-      mlayout.omap_root.update(omap_root_t(id, cnt_modify,
-	value.get_metadata_hint(block_size)));
-    });
+    ftvalue.update_onode_size(t, size);
+    auto oroot = omap_root_t(id, cnt_modify,
+      value.get_metadata_hint(block_size));
+    ftvalue.update_omap_root(t, oroot);
     validate(value);
   }
 

From 23c73912c7ca4483b59558c3e1735d61e545a92e Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 10 Oct 2023 11:47:38 +0800
Subject: [PATCH 0122/2492] crimson/os/seastore/onode_manager: drop unnecessary
 status

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../onode_manager/staged-fltree/fltree_onode_manager.h     | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
index 86f5cea883b2..33109e50f6a6 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h
@@ -25,10 +25,9 @@ struct FLTreeOnode final : Onode, Value {
   };
 
   enum class status_t {
-    STABLE,
-    MUTATED,
+    ALIVE,
     DELETED
-  } status = status_t::STABLE;
+  } status = status_t::ALIVE;
 
   FLTreeOnode(FLTreeOnode&&) = default;
   FLTreeOnode& operator=(FLTreeOnode&&) = delete;
@@ -88,9 +87,7 @@ struct FLTreeOnode final : Onode, Value {
     auto p = prepare_mutate_payload<
       onode_layout_t,
       Recorder>(t);
-    status = status_t::MUTATED;
     layout_func(p.first, p.second);
-    status = status_t::STABLE;
   }
 
   void create_default_layout(Transaction &t) {

From 194dd09263b23a7c5e0a06cd59841bca5f89c7f4 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 30 Sep 2023 13:34:44 +0200
Subject: [PATCH 0123/2492] qa/suites/rbd: drop cache tiering workload tests

Cache tiering facets have been a constant source of job timeouts
accompanied by "slow request" warnings on the OSDs for at least two
years.  Same workloads pass without pool/small-cache-pool.yaml or
thrashers/cache.yaml.

See cache tiering deprecation note added in commit 535b8db33ea0 ("doc:
deprecate the cache tiering").

Fixes: https://tracker.ceph.com/issues/63149
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 qa/suites/rbd/basic/cachepool/.qa             |  1 -
 qa/suites/rbd/basic/cachepool/none.yaml       |  0
 qa/suites/rbd/basic/cachepool/small.yaml      | 17 ---------------
 qa/suites/rbd/cli/pool/small-cache-pool.yaml  | 17 ---------------
 qa/suites/rbd/cli_v1/pool/.qa                 |  1 -
 qa/suites/rbd/cli_v1/pool/none.yaml           |  0
 .../rbd/cli_v1/pool/small-cache-pool.yaml     | 17 ---------------
 .../rbd/encryption/pool/ec-cache-pool.yaml    | 21 -------------------
 .../rbd/encryption/pool/small-cache-pool.yaml | 17 ---------------
 .../rbd/librbd/pool/small-cache-pool.yaml     | 17 ---------------
 qa/suites/rbd/qemu/pool/ec-cache-pool.yaml    | 21 -------------------
 qa/suites/rbd/qemu/pool/small-cache-pool.yaml | 17 ---------------
 qa/suites/rbd/thrash/thrashers/cache.yaml     | 21 -------------------
 13 files changed, 167 deletions(-)
 delete mode 120000 qa/suites/rbd/basic/cachepool/.qa
 delete mode 100644 qa/suites/rbd/basic/cachepool/none.yaml
 delete mode 100644 qa/suites/rbd/basic/cachepool/small.yaml
 delete mode 100644 qa/suites/rbd/cli/pool/small-cache-pool.yaml
 delete mode 120000 qa/suites/rbd/cli_v1/pool/.qa
 delete mode 100644 qa/suites/rbd/cli_v1/pool/none.yaml
 delete mode 100644 qa/suites/rbd/cli_v1/pool/small-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/encryption/pool/ec-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/encryption/pool/small-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/librbd/pool/small-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/qemu/pool/ec-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/qemu/pool/small-cache-pool.yaml
 delete mode 100644 qa/suites/rbd/thrash/thrashers/cache.yaml

diff --git a/qa/suites/rbd/basic/cachepool/.qa b/qa/suites/rbd/basic/cachepool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/basic/cachepool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/basic/cachepool/none.yaml b/qa/suites/rbd/basic/cachepool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/basic/cachepool/small.yaml b/qa/suites/rbd/basic/cachepool/small.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/basic/cachepool/small.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/cli/pool/small-cache-pool.yaml b/qa/suites/rbd/cli/pool/small-cache-pool.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/cli/pool/small-cache-pool.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/cli_v1/pool/.qa b/qa/suites/rbd/cli_v1/pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/cli_v1/pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/cli_v1/pool/none.yaml b/qa/suites/rbd/cli_v1/pool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/cli_v1/pool/small-cache-pool.yaml b/qa/suites/rbd/cli_v1/pool/small-cache-pool.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/cli_v1/pool/small-cache-pool.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/encryption/pool/ec-cache-pool.yaml b/qa/suites/rbd/encryption/pool/ec-cache-pool.yaml
deleted file mode 100644
index a0f88b4096d0..000000000000
--- a/qa/suites/rbd/encryption/pool/ec-cache-pool.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
-      - sudo ceph osd pool create rbd 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
-      - rbd pool init rbd
diff --git a/qa/suites/rbd/encryption/pool/small-cache-pool.yaml b/qa/suites/rbd/encryption/pool/small-cache-pool.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/encryption/pool/small-cache-pool.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/librbd/pool/small-cache-pool.yaml b/qa/suites/rbd/librbd/pool/small-cache-pool.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/librbd/pool/small-cache-pool.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/qemu/pool/ec-cache-pool.yaml b/qa/suites/rbd/qemu/pool/ec-cache-pool.yaml
deleted file mode 100644
index a0f88b4096d0..000000000000
--- a/qa/suites/rbd/qemu/pool/ec-cache-pool.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
-      - sudo ceph osd pool create rbd 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
-      - rbd pool init rbd
diff --git a/qa/suites/rbd/qemu/pool/small-cache-pool.yaml b/qa/suites/rbd/qemu/pool/small-cache-pool.yaml
deleted file mode 100644
index bad95eaddf25..000000000000
--- a/qa/suites/rbd/qemu/pool/small-cache-pool.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
diff --git a/qa/suites/rbd/thrash/thrashers/cache.yaml b/qa/suites/rbd/thrash/thrashers/cache.yaml
deleted file mode 100644
index b434e28be8a8..000000000000
--- a/qa/suites/rbd/thrash/thrashers/cache.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - but it is still running
-      - objects unfound and apparently lost
-      - overall HEALTH_
-      - \(CACHE_POOL_NEAR_FULL\)
-      - \(CACHE_POOL_NO_HIT_SET\)
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create cache 4
-      - sudo ceph osd tier add rbd cache
-      - sudo ceph osd tier cache-mode cache writeback
-      - sudo ceph osd tier set-overlay rbd cache
-      - sudo ceph osd pool set cache hit_set_type bloom
-      - sudo ceph osd pool set cache hit_set_count 8
-      - sudo ceph osd pool set cache hit_set_period 60
-      - sudo ceph osd pool set cache target_max_objects 250
-- thrashosds:
-    timeout: 1200

From 83880580aa15a12fb238a8da2e9355da9cf69cbf Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 30 Sep 2023 11:39:32 +0200
Subject: [PATCH 0124/2492] qa/suites/rbd: deduplicate (data) pool facets

With cache tiering facets gone, "pool" facets are strictly about
--data-pool option now.  Rename to "data-pool" and create symlinks
to a common directory.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../data-pool/ec.yaml}                        |  0
 .../rbd/cli/pool => rbd/data-pool}/none.yaml  |  0
 .../data-pool/replicated.yaml}                |  0
 qa/suites/rbd/cli/data-pool                   |  1 +
 qa/suites/rbd/cli/pool/.qa                    |  1 -
 qa/suites/rbd/encryption/data-pool            |  1 +
 qa/suites/rbd/encryption/pool/.qa             |  1 -
 .../rbd/encryption/pool/ec-data-pool.yaml     | 24 -------------------
 qa/suites/rbd/encryption/pool/none.yaml       |  0
 .../encryption/pool/replicated-data-pool.yaml | 11 ---------
 qa/suites/rbd/librbd/data-pool                |  1 +
 qa/suites/rbd/librbd/pool/.qa                 |  1 -
 qa/suites/rbd/librbd/pool/ec-data-pool.yaml   | 24 -------------------
 qa/suites/rbd/librbd/pool/none.yaml           |  0
 .../rbd/librbd/pool/replicated-data-pool.yaml | 11 ---------
 qa/suites/rbd/migration/5-data-pool           |  1 +
 qa/suites/rbd/migration/5-pool/.qa            |  1 -
 .../rbd/migration/5-pool/ec-data-pool.yaml    | 24 -------------------
 qa/suites/rbd/migration/5-pool/none.yaml      |  0
 .../5-pool/replicated-data-pool.yaml          | 11 ---------
 qa/suites/rbd/qemu/data-pool                  |  1 +
 qa/suites/rbd/qemu/pool/.qa                   |  1 -
 qa/suites/rbd/qemu/pool/ec-data-pool.yaml     | 24 -------------------
 qa/suites/rbd/qemu/pool/none.yaml             |  0
 .../rbd/qemu/pool/replicated-data-pool.yaml   | 11 ---------
 25 files changed, 5 insertions(+), 145 deletions(-)
 rename qa/{suites/rbd/cli/pool/ec-data-pool.yaml => rbd/data-pool/ec.yaml} (100%)
 rename qa/{suites/rbd/cli/pool => rbd/data-pool}/none.yaml (100%)
 rename qa/{suites/rbd/cli/pool/replicated-data-pool.yaml => rbd/data-pool/replicated.yaml} (100%)
 create mode 120000 qa/suites/rbd/cli/data-pool
 delete mode 120000 qa/suites/rbd/cli/pool/.qa
 create mode 120000 qa/suites/rbd/encryption/data-pool
 delete mode 120000 qa/suites/rbd/encryption/pool/.qa
 delete mode 100644 qa/suites/rbd/encryption/pool/ec-data-pool.yaml
 delete mode 100644 qa/suites/rbd/encryption/pool/none.yaml
 delete mode 100644 qa/suites/rbd/encryption/pool/replicated-data-pool.yaml
 create mode 120000 qa/suites/rbd/librbd/data-pool
 delete mode 120000 qa/suites/rbd/librbd/pool/.qa
 delete mode 100644 qa/suites/rbd/librbd/pool/ec-data-pool.yaml
 delete mode 100644 qa/suites/rbd/librbd/pool/none.yaml
 delete mode 100644 qa/suites/rbd/librbd/pool/replicated-data-pool.yaml
 create mode 120000 qa/suites/rbd/migration/5-data-pool
 delete mode 120000 qa/suites/rbd/migration/5-pool/.qa
 delete mode 100644 qa/suites/rbd/migration/5-pool/ec-data-pool.yaml
 delete mode 100644 qa/suites/rbd/migration/5-pool/none.yaml
 delete mode 100644 qa/suites/rbd/migration/5-pool/replicated-data-pool.yaml
 create mode 120000 qa/suites/rbd/qemu/data-pool
 delete mode 120000 qa/suites/rbd/qemu/pool/.qa
 delete mode 100644 qa/suites/rbd/qemu/pool/ec-data-pool.yaml
 delete mode 100644 qa/suites/rbd/qemu/pool/none.yaml
 delete mode 100644 qa/suites/rbd/qemu/pool/replicated-data-pool.yaml

diff --git a/qa/suites/rbd/cli/pool/ec-data-pool.yaml b/qa/rbd/data-pool/ec.yaml
similarity index 100%
rename from qa/suites/rbd/cli/pool/ec-data-pool.yaml
rename to qa/rbd/data-pool/ec.yaml
diff --git a/qa/suites/rbd/cli/pool/none.yaml b/qa/rbd/data-pool/none.yaml
similarity index 100%
rename from qa/suites/rbd/cli/pool/none.yaml
rename to qa/rbd/data-pool/none.yaml
diff --git a/qa/suites/rbd/cli/pool/replicated-data-pool.yaml b/qa/rbd/data-pool/replicated.yaml
similarity index 100%
rename from qa/suites/rbd/cli/pool/replicated-data-pool.yaml
rename to qa/rbd/data-pool/replicated.yaml
diff --git a/qa/suites/rbd/cli/data-pool b/qa/suites/rbd/cli/data-pool
new file mode 120000
index 000000000000..3df827572804
--- /dev/null
+++ b/qa/suites/rbd/cli/data-pool
@@ -0,0 +1 @@
+.qa/rbd/data-pool/
\ No newline at end of file
diff --git a/qa/suites/rbd/cli/pool/.qa b/qa/suites/rbd/cli/pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/cli/pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/encryption/data-pool b/qa/suites/rbd/encryption/data-pool
new file mode 120000
index 000000000000..3df827572804
--- /dev/null
+++ b/qa/suites/rbd/encryption/data-pool
@@ -0,0 +1 @@
+.qa/rbd/data-pool/
\ No newline at end of file
diff --git a/qa/suites/rbd/encryption/pool/.qa b/qa/suites/rbd/encryption/pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/encryption/pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/encryption/pool/ec-data-pool.yaml b/qa/suites/rbd/encryption/pool/ec-data-pool.yaml
deleted file mode 100644
index f39a5bb4ca62..000000000000
--- a/qa/suites/rbd/encryption/pool/ec-data-pool.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool create datapool 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool set datapool allow_ec_overwrites true
-      - rbd pool init datapool
-
-overrides:
-  thrashosds:
-    bdev_inject_crash: 2
-    bdev_inject_crash_probability: .5
-  ceph:
-    fs: xfs
-    conf:
-      client:
-        rbd default data pool: datapool
-      osd: # force bluestore since it's required for ec overwrites
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        enable experimental unrecoverable data corrupting features: "*"
-        osd debug randomize hobject sort order: false
-# this doesn't work with failures bc the log writes are not atomic across the two backends
-#        bluestore bluefs env mirror: true
diff --git a/qa/suites/rbd/encryption/pool/none.yaml b/qa/suites/rbd/encryption/pool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/encryption/pool/replicated-data-pool.yaml b/qa/suites/rbd/encryption/pool/replicated-data-pool.yaml
deleted file mode 100644
index c5647dba1c68..000000000000
--- a/qa/suites/rbd/encryption/pool/replicated-data-pool.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create datapool 4
-      - rbd pool init datapool
-
-overrides:
-  ceph:
-    conf:
-      client:
-        rbd default data pool: datapool
diff --git a/qa/suites/rbd/librbd/data-pool b/qa/suites/rbd/librbd/data-pool
new file mode 120000
index 000000000000..3df827572804
--- /dev/null
+++ b/qa/suites/rbd/librbd/data-pool
@@ -0,0 +1 @@
+.qa/rbd/data-pool/
\ No newline at end of file
diff --git a/qa/suites/rbd/librbd/pool/.qa b/qa/suites/rbd/librbd/pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/librbd/pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/librbd/pool/ec-data-pool.yaml b/qa/suites/rbd/librbd/pool/ec-data-pool.yaml
deleted file mode 100644
index f39a5bb4ca62..000000000000
--- a/qa/suites/rbd/librbd/pool/ec-data-pool.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool create datapool 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool set datapool allow_ec_overwrites true
-      - rbd pool init datapool
-
-overrides:
-  thrashosds:
-    bdev_inject_crash: 2
-    bdev_inject_crash_probability: .5
-  ceph:
-    fs: xfs
-    conf:
-      client:
-        rbd default data pool: datapool
-      osd: # force bluestore since it's required for ec overwrites
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        enable experimental unrecoverable data corrupting features: "*"
-        osd debug randomize hobject sort order: false
-# this doesn't work with failures bc the log writes are not atomic across the two backends
-#        bluestore bluefs env mirror: true
diff --git a/qa/suites/rbd/librbd/pool/none.yaml b/qa/suites/rbd/librbd/pool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/librbd/pool/replicated-data-pool.yaml b/qa/suites/rbd/librbd/pool/replicated-data-pool.yaml
deleted file mode 100644
index c5647dba1c68..000000000000
--- a/qa/suites/rbd/librbd/pool/replicated-data-pool.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create datapool 4
-      - rbd pool init datapool
-
-overrides:
-  ceph:
-    conf:
-      client:
-        rbd default data pool: datapool
diff --git a/qa/suites/rbd/migration/5-data-pool b/qa/suites/rbd/migration/5-data-pool
new file mode 120000
index 000000000000..3df827572804
--- /dev/null
+++ b/qa/suites/rbd/migration/5-data-pool
@@ -0,0 +1 @@
+.qa/rbd/data-pool/
\ No newline at end of file
diff --git a/qa/suites/rbd/migration/5-pool/.qa b/qa/suites/rbd/migration/5-pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/migration/5-pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/migration/5-pool/ec-data-pool.yaml b/qa/suites/rbd/migration/5-pool/ec-data-pool.yaml
deleted file mode 100644
index f39a5bb4ca62..000000000000
--- a/qa/suites/rbd/migration/5-pool/ec-data-pool.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool create datapool 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool set datapool allow_ec_overwrites true
-      - rbd pool init datapool
-
-overrides:
-  thrashosds:
-    bdev_inject_crash: 2
-    bdev_inject_crash_probability: .5
-  ceph:
-    fs: xfs
-    conf:
-      client:
-        rbd default data pool: datapool
-      osd: # force bluestore since it's required for ec overwrites
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        enable experimental unrecoverable data corrupting features: "*"
-        osd debug randomize hobject sort order: false
-# this doesn't work with failures bc the log writes are not atomic across the two backends
-#        bluestore bluefs env mirror: true
diff --git a/qa/suites/rbd/migration/5-pool/none.yaml b/qa/suites/rbd/migration/5-pool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/migration/5-pool/replicated-data-pool.yaml b/qa/suites/rbd/migration/5-pool/replicated-data-pool.yaml
deleted file mode 100644
index c5647dba1c68..000000000000
--- a/qa/suites/rbd/migration/5-pool/replicated-data-pool.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create datapool 4
-      - rbd pool init datapool
-
-overrides:
-  ceph:
-    conf:
-      client:
-        rbd default data pool: datapool
diff --git a/qa/suites/rbd/qemu/data-pool b/qa/suites/rbd/qemu/data-pool
new file mode 120000
index 000000000000..3df827572804
--- /dev/null
+++ b/qa/suites/rbd/qemu/data-pool
@@ -0,0 +1 @@
+.qa/rbd/data-pool/
\ No newline at end of file
diff --git a/qa/suites/rbd/qemu/pool/.qa b/qa/suites/rbd/qemu/pool/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/rbd/qemu/pool/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/qemu/pool/ec-data-pool.yaml b/qa/suites/rbd/qemu/pool/ec-data-pool.yaml
deleted file mode 100644
index f39a5bb4ca62..000000000000
--- a/qa/suites/rbd/qemu/pool/ec-data-pool.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
-      - sudo ceph osd pool create datapool 4 4 erasure teuthologyprofile
-      - sudo ceph osd pool set datapool allow_ec_overwrites true
-      - rbd pool init datapool
-
-overrides:
-  thrashosds:
-    bdev_inject_crash: 2
-    bdev_inject_crash_probability: .5
-  ceph:
-    fs: xfs
-    conf:
-      client:
-        rbd default data pool: datapool
-      osd: # force bluestore since it's required for ec overwrites
-        osd objectstore: bluestore
-        bluestore block size: 96636764160
-        enable experimental unrecoverable data corrupting features: "*"
-        osd debug randomize hobject sort order: false
-# this doesn't work with failures bc the log writes are not atomic across the two backends
-#        bluestore bluefs env mirror: true
diff --git a/qa/suites/rbd/qemu/pool/none.yaml b/qa/suites/rbd/qemu/pool/none.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rbd/qemu/pool/replicated-data-pool.yaml b/qa/suites/rbd/qemu/pool/replicated-data-pool.yaml
deleted file mode 100644
index c5647dba1c68..000000000000
--- a/qa/suites/rbd/qemu/pool/replicated-data-pool.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-tasks:
-- exec:
-    client.0:
-      - sudo ceph osd pool create datapool 4
-      - rbd pool init datapool
-
-overrides:
-  ceph:
-    conf:
-      client:
-        rbd default data pool: datapool

From f987b4daa097a84ca35db4037de1985fc0acaf01 Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Tue, 10 Oct 2023 09:04:39 +0000
Subject: [PATCH 0125/2492] os/kv_test: Fix estimate functions

We need to use random content to estimate DB size.
Otherwise, compression will cause DB to be unreasonably small.

Fixes: https://tracker.ceph.com/issues/63121

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
---
 src/test/objectstore/test_kv.cc | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/test/objectstore/test_kv.cc b/src/test/objectstore/test_kv.cc
index 33ffd6ab3968..95c712ceffa5 100644
--- a/src/test/objectstore/test_kv.cc
+++ b/src/test/objectstore/test_kv.cc
@@ -29,6 +29,14 @@
 
 using namespace std;
 
+std::string gen_random_string(size_t size) {
+  std::string s;
+  for (size_t i = 0; i < size; i++) {
+    s.push_back(rand());
+  }
+  return s;
+}
+
 class KVTest : public ::testing::TestWithParam<const char*> {
 public:
   boost::scoped_ptr<KeyValueDB> db;
@@ -556,10 +564,11 @@ TEST_P(KVTest, RocksDB_estimate_size) {
   for(int test = 0; test < 20; test++)
   {
     KeyValueDB::Transaction t = db->get_transaction();
-    bufferlist v1;
-    v1.append(string(1000, '1'));
-    for (int i = 0; i < 100; i++)
+    for (int i = 0; i < 100; i++) {
+      bufferlist v1;
+      v1.append(gen_random_string(1000));
       t->set("A", to_string(rand()%100000), v1);
+    }
     db->submit_transaction_sync(t);
     db->compact();
 
@@ -588,10 +597,11 @@ TEST_P(KVTest, RocksDB_estimate_size_column_family) {
   for(int test = 0; test < 20; test++)
   {
     KeyValueDB::Transaction t = db->get_transaction();
-    bufferlist v1;
-    v1.append(string(1000, '1'));
-    for (int i = 0; i < 100; i++)
+    for (int i = 0; i < 100; i++) {
+      bufferlist v1;
+      v1.append(gen_random_string(1000));
       t->set("cf1", to_string(rand()%100000), v1);
+    }
     db->submit_transaction_sync(t);
     db->compact();
 

From f3c0424bb583423db80009c29f73e484aad840b4 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Mon, 25 Sep 2023 18:19:50 +0530
Subject: [PATCH 0126/2492] mds: do not simplify fragset

Problem:
Frags in simplified fragset aren't found as is on the replica during scrub.

fragset::simplify() computes the bit representation of the least common
ancestor of the frags.  When this representation is forwarded to the replicas,
the ScrubStack::handle_scrub() method in the OP_QUEUEDIR case simply searches
this simplified representation in the list of (unsimplified) frags ...
which it fails to find.
Hence we get to see the "no frag 10*" type of logs.

Solution:
Do not simplify fragset when forwarding fragset to replica for scrub.

Fixes: https://tracker.ceph.com/issues/62658
Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/mds/ScrubStack.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc
index 6d799343f149..742c464f4d37 100644
--- a/src/mds/ScrubStack.cc
+++ b/src/mds/ScrubStack.cc
@@ -320,7 +320,7 @@ void ScrubStack::scrub_dir_inode(CInode *in, bool *added_children, bool *done)
 
   frag_vec_t frags;
   in->dirfragtree.get_leaves(frags);
-  dout(20) << __func__ << "recursive mode, frags " << frags << dendl;
+  dout(20) << __func__ << " recursive mode, frags " << frags << dendl;
   for (auto &fg : frags) {
     if (queued.contains(fg))
       continue;
@@ -366,7 +366,6 @@ void ScrubStack::scrub_dir_inode(CInode *in, bool *added_children, bool *done)
     scrub_r.tag = header->get_tag();
 
     for (auto& p : scrub_remote) {
-      p.second.simplify();
       dout(20) << __func__ << " forward " << p.second  << " to mds." << p.first << dendl;
       auto r = make_message<MMDSScrub>(MMDSScrub::OP_QUEUEDIR, in->ino(),
 				       std::move(p.second), header->get_tag(),

From 90ae55f6d1ecc9c951e5dfa7d5a5d2169d3a917f Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Tue, 10 Oct 2023 11:54:44 +0200
Subject: [PATCH 0127/2492] mgr/rook: fixing rook-ceph-exporter daemon type
 handling Fixes: https://tracker.ceph.com/issues/63107

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index b67349d1bff6..32699fa0117a 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -457,7 +457,17 @@ def _list_daemons(self,
         for p in pods:
             sd = orchestrator.DaemonDescription()
             sd.hostname = p['hostname']
-            sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '')
+
+            # In Rook environments, the 'ceph-exporter' daemon is named 'exporter' whereas
+            # in the orchestrator interface, it is named 'ceph-exporter'. The purpose of the
+            # following adjustment is to ensure that the 'daemon_type' is correctly set.
+            # Without this adjustment, the 'service_to_daemon_types' lookup would fail, as
+            # it would be searching for a non-existent entry called 'exporter
+            if p['labels']['app'] == 'rook-ceph-exporter':
+                sd.daemon_type = 'ceph-exporter'
+            else:
+                sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '')
+
             status = {
                 'Pending': orchestrator.DaemonDescriptionStatus.starting,
                 'Running': orchestrator.DaemonDescriptionStatus.running,

From e40752ec25155ab232a68cd44093a9764ebf4091 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Tue, 10 Oct 2023 12:31:28 +0200
Subject: [PATCH 0128/2492] qa/suites/rbd: drop redundant ignorelist entries

CACHE_POOL_NO_HIT_SET is retained in *api_tests*.yaml and
rbd_mirror.yaml snippets for TestLibRBD.ListChildrenTiered and
TestClusterWatcher.CachePools tests.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 qa/rbd/data-pool/ec.yaml                   | 3 ---
 qa/suites/rbd/singleton/all/rbd_tasks.yaml | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/qa/rbd/data-pool/ec.yaml b/qa/rbd/data-pool/ec.yaml
index db289c7e7e60..f39a5bb4ca62 100644
--- a/qa/rbd/data-pool/ec.yaml
+++ b/qa/rbd/data-pool/ec.yaml
@@ -12,9 +12,6 @@ overrides:
     bdev_inject_crash_probability: .5
   ceph:
     fs: xfs
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NO_HIT_SET\)
     conf:
       client:
         rbd default data pool: datapool
diff --git a/qa/suites/rbd/singleton/all/rbd_tasks.yaml b/qa/suites/rbd/singleton/all/rbd_tasks.yaml
index 4723eb6800ce..782b0214135e 100644
--- a/qa/suites/rbd/singleton/all/rbd_tasks.yaml
+++ b/qa/suites/rbd/singleton/all/rbd_tasks.yaml
@@ -4,9 +4,6 @@ tasks:
 - install:
 - ceph:
     fs: xfs
-    log-ignorelist:
-      - overall HEALTH_
-      - \(CACHE_POOL_NO_HIT_SET\)
 - workunit:
     clients:
       all: [rbd/test_rbd_tasks.sh]

From be78b3ea68f75319da05700276630f7f331bbd63 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 3 Oct 2023 01:17:09 +0530
Subject: [PATCH 0129/2492] mds/FSMap: add logging facility

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/mds/FSMap.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index 5dfaaf0e3441..fe7ee5588919 100644
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -15,7 +15,7 @@
 #include <ostream>
 
 #include "FSMap.h"
-
+#include "common/debug.h"
 #include "common/StackStringStream.h"
 
 #ifdef WITH_SEASTAR
@@ -26,6 +26,11 @@
 #include "global/global_context.h"
 #include "mon/health_check.h"
 
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mds
+#undef dout_prefix
+#define dout_prefix *_dout << "FSMap "
+
 using std::list;
 using std::pair;
 using std::ostream;

From 69e4c9e4729fd0d29ed43fb1f6e94cb1804a3f74 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 3 Oct 2023 01:17:42 +0530
Subject: [PATCH 0130/2492] qa/cephfs: import only sleep() from time

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_admin.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index 923871f88e3a..b2e10e4d3cf4 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -1,10 +1,10 @@
 import errno
 import json
 import logging
-import time
 import uuid
 from io import StringIO
 from os.path import join as os_path_join
+from time import sleep
 
 from teuthology.exceptions import CommandFailedError
 
@@ -802,7 +802,7 @@ def test_fsmap_trim(self):
             self.fs.set_joinable(b)
             b = not b
 
-        time.sleep(10) # for tick/compaction
+        sleep(10) # for tick/compaction
 
         try:
             self.fs.status(epoch=epoch)
@@ -826,7 +826,7 @@ def test_fsmap_force_trim(self):
 
         # force a new fsmap
         self.fs.set_joinable(False)
-        time.sleep(10) # for tick/compaction
+        sleep(10) # for tick/compaction
 
         status = self.fs.status()
         log.debug(f"new epoch is {status['epoch']}")

From 6ac58b0a12324ea13c724cbba3107e1eae9a96c3 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 3 Oct 2023 13:41:54 +0530
Subject: [PATCH 0131/2492] qa/cephfs: minor improvement caps_helper.py

When assert fails for equality of two variables and when both the
variables are printed in error message, print each variable on a new
line.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/caps_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/caps_helper.py b/qa/tasks/cephfs/caps_helper.py
index f083c788337a..75a40ac83fe8 100644
--- a/qa/tasks/cephfs/caps_helper.py
+++ b/qa/tasks/cephfs/caps_helper.py
@@ -124,7 +124,7 @@ def get_fsnames_from_moncap(moncap):
 
 
 def assert_equal(first, second):
-    msg = f'Variables are not equal.\nfirst = {first}\nsecond = {second}'
+    msg = f'Variables are not equal.\nfirst -\n{first}\nsecond -\n{second}'
     assert first == second, msg
 
 

From f8acdf01cb38c208c2ca5f32ae34fa1c0dba8cfb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 11:04:49 -0400
Subject: [PATCH 0132/2492] cephadm: convert ceph class to a
 ContainerDaemonForm

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 83 ++++++++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 31 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6c87d7983b6f..f312ca663436 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -208,7 +208,7 @@ def __eq__(self, other: Any) -> bool:
 
 
 @register_daemon_form
-class Ceph(DaemonForm):
+class Ceph(ContainerDaemonForm):
     daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
                'crash', 'cephfs-mirror', 'ceph-exporter')
 
@@ -235,6 +235,37 @@ def firewall_service_name(self) -> str:
             return 'ceph'
         return ''
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        # previous to being a ContainerDaemonForm, this make_var_run
+        # call was hard coded in the deploy path. Eventually, it would be
+        # good to move this somwhere cleaner and avoid needing to know the
+        # uid/gid here.
+        uid, gid = self.uid_gid(ctx)
+        make_var_run(ctx, ctx.fsid, uid, gid)
+
+        ctr = get_deployment_container(ctx, self.identity)
+        config_json = fetch_configs(ctx)
+        if self.identity.daemon_type == 'mon' and config_json is not None:
+            if 'crush_location' in config_json:
+                c_loc = config_json['crush_location']
+                # was originally "c.args.extend(['--set-crush-location', c_loc])"
+                # but that doesn't seem to persist in the object after it's passed
+                # in further function calls
+                ctr.args = ctr.args + ['--set-crush-location', c_loc]
+        return ctr
+
+    _uid_gid: Optional[Tuple[int, int]] = None
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        if self._uid_gid is None:
+            self._uid_gid = extract_uid_gid(ctx)
+        return self._uid_gid
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
 ##################################
 
 
@@ -245,6 +276,21 @@ def for_daemon_type(cls, daemon_type: str) -> bool:
         # TODO: figure out a way to un-special-case osd
         return daemon_type == 'osd'
 
+    def __init__(
+        self, ident: DaemonIdentity, osd_fsid: Optional[str] = None
+    ) -> None:
+        super().__init__(ident)
+        self._osd_fsid = osd_fsid
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'OSD':
+        osd_fsid = getattr(ctx, 'osd_fsid', None)
+        if osd_fsid is None:
+            logger.info(
+                'Creating an OSD daemon form without an OSD FSID value'
+            )
+        return cls(ident, osd_fsid)
+
     @staticmethod
     def get_sysctl_settings() -> List[str]:
         return [
@@ -256,6 +302,10 @@ def get_sysctl_settings() -> List[str]:
     def firewall_service_name(self) -> str:
         return 'ceph'
 
+    @property
+    def osd_fsid(self) -> Optional[str]:
+        return self._osd_fsid
+
 
 ##################################
 
@@ -5196,37 +5246,8 @@ def _dispatch_deploy(
     deployment_type: DeploymentType,
 ) -> None:
     daemon_type = ident.daemon_type
-    if daemon_type in Ceph.daemons:
-        config, keyring = get_config_and_keyring(ctx)
-        uid, gid = extract_uid_gid(ctx)
-        make_var_run(ctx, ctx.fsid, uid, gid)
-
-        config_json = fetch_configs(ctx)
-
-        c = get_deployment_container(ctx, ident, ptrace=ctx.allow_ptrace)
-
-        if daemon_type == 'mon' and config_json is not None:
-            if 'crush_location' in config_json:
-                c_loc = config_json['crush_location']
-                # was originally "c.args.extend(['--set-crush-location', c_loc])"
-                # but that doesn't seem to persist in the object after it's passed
-                # in further function calls
-                c.args = c.args + ['--set-crush-location', c_loc]
-
-        deploy_daemon(
-            ctx,
-            ident,
-            c,
-            uid,
-            gid,
-            config=config,
-            keyring=keyring,
-            osd_fsid=ctx.osd_fsid,
-            deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
-        )
 
-    elif daemon_type == CephadmAgent.daemon_type:
+    if daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
         uid = os.getuid()
         gid = os.getgid()

From af2058536b49309d7408d6cb19d04dc292218419 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 14:15:55 -0400
Subject: [PATCH 0133/2492] cephadm: eliminate _dispatch_deploy function

Eliminate the _dispatch_deploy function, folding it into the
_common_deploy function, because the mass of if-elif lines have
been replaced and keeping it as a separate function no longer
serves much of a useful purpose.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index f312ca663436..f9041317f4db 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -5236,18 +5236,8 @@ def _common_deploy(ctx: CephadmContext) -> None:
 
     # Get and check ports explicitly required to be opened
     endpoints = fetch_endpoints(ctx)
-    _dispatch_deploy(ctx, ident, endpoints, deployment_type)
 
-
-def _dispatch_deploy(
-    ctx: CephadmContext,
-    ident: 'DaemonIdentity',
-    daemon_endpoints: List[EndPoint],
-    deployment_type: DeploymentType,
-) -> None:
-    daemon_type = ident.daemon_type
-
-    if daemon_type == CephadmAgent.daemon_type:
+    if ident.daemon_type == CephadmAgent.daemon_type:
         # get current user gid and uid
         uid = os.getuid()
         gid = os.getgid()
@@ -5258,17 +5248,15 @@ def _dispatch_deploy(
             uid,
             gid,
             deployment_type=deployment_type,
-            endpoints=daemon_endpoints,
+            endpoints=endpoints,
         )
 
     else:
         try:
-            _deploy_daemon_container(
-                ctx, ident, daemon_endpoints, deployment_type
-            )
+            _deploy_daemon_container(ctx, ident, endpoints, deployment_type)
         except UnexpectedDaemonTypeError:
             raise Error('daemon type {} not implemented in command_deploy function'
-                        .format(daemon_type))
+                        .format(ident.daemon_type))
 
 
 def _deploy_daemon_container(

From 28fe98bd5949c3a6bcae4e78cdae5bbcde4e8d3b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 16:43:59 -0400
Subject: [PATCH 0134/2492] cephadm: stop directly using Ceph.daemons property

The Ceph.daemons property has two unfortunate behaviors: most important,
it includes ceph-exporter which causes the other CephExporter class to
be over-shadowed the DaemonForms mechanism. Second, it couples all
functions that want to know the names of ceph daemon types to the Ceph
class preventing future refactoring of that class.

Break the existing coupling by adding a new `ceph_daemons` function
similar to `get_supported_daemons` but returning the same value that
Ceph.daemons used to provide. This will permit future fixes and
improvements.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 38 +++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index f9041317f4db..a3b2c22c4813 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -209,13 +209,13 @@ def __eq__(self, other: Any) -> bool:
 
 @register_daemon_form
 class Ceph(ContainerDaemonForm):
-    daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
-               'crash', 'cephfs-mirror', 'ceph-exporter')
+    _daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
+                'crash', 'cephfs-mirror', 'ceph-exporter')
 
     @classmethod
     def for_daemon_type(cls, daemon_type: str) -> bool:
         # TODO: figure out a way to un-special-case osd
-        return daemon_type in cls.daemons and daemon_type != 'osd'
+        return daemon_type in cls._daemons and daemon_type != 'osd'
 
     def __init__(self, ident: DaemonIdentity) -> None:
         self._identity = ident
@@ -1708,7 +1708,7 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
 
 def get_supported_daemons():
     # type: () -> List[str]
-    supported_daemons = list(Ceph.daemons)
+    supported_daemons = ceph_daemons()
     supported_daemons.extend(Monitoring.components)
     supported_daemons.append(NFSGanesha.daemon_type)
     supported_daemons.append(CephIscsi.daemon_type)
@@ -1722,6 +1722,10 @@ def get_supported_daemons():
     assert len(supported_daemons) == len(set(supported_daemons))
     return supported_daemons
 
+
+def ceph_daemons() -> List[str]:
+    return list(Ceph._daemons)
+
 ##################################
 
 
@@ -2014,7 +2018,7 @@ def infer_local_ceph_image(ctx: CephadmContext, container_path: str) -> Optional
 
     container_info = None
     daemon_name = ctx.name if ('name' in ctx and ctx.name and '.' in ctx.name) else None
-    daemons_ls = [daemon_name] if daemon_name is not None else Ceph.daemons  # daemon types: 'mon', 'mgr', etc
+    daemons_ls = [daemon_name] if daemon_name is not None else ceph_daemons()  # daemon types: 'mon', 'mgr', etc
     for daemon in daemons_ls:
         container_info = get_container_info(ctx, daemon, daemon_name is not None)
         if container_info is not None:
@@ -2190,7 +2194,7 @@ def get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     r = list()  # type: List[str]
 
     daemon_type = ident.daemon_type
-    if daemon_type in Ceph.daemons and daemon_type not in ['crash', 'ceph-exporter']:
+    if daemon_type in ceph_daemons() and daemon_type not in ['crash', 'ceph-exporter']:
         r += [
             '--setuser', 'ceph',
             '--setgroup', 'ceph',
@@ -2309,7 +2313,7 @@ def create_daemon_dirs(
     fsid, daemon_type = ident.fsid, ident.daemon_type
     data_dir = make_data_dir(ctx, ident, uid=uid, gid=gid)
 
-    if daemon_type in Ceph.daemons:
+    if daemon_type in ceph_daemons():
         make_log_dir(ctx, fsid, uid=uid, gid=gid)
 
     if config:
@@ -2479,7 +2483,7 @@ def _get_container_mounts_for_type(
     """
     mounts = dict()
 
-    if daemon_type in Ceph.daemons:
+    if daemon_type in ceph_daemons():
         if fsid:
             run_path = os.path.join('/var/run/ceph', fsid)
             if os.path.exists(run_path):
@@ -2548,7 +2552,7 @@ def get_container_mounts(
 
     assert ident.fsid
     assert ident.daemon_id
-    if daemon_type in Ceph.daemons:
+    if daemon_type in ceph_daemons():
         data_dir = ident.data_dir(ctx.data_dir)
         if daemon_type == 'rgw':
             cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (ident.daemon_id)
@@ -2693,11 +2697,11 @@ def get_container(
     host_network: bool = True
 
     daemon_type = ident.daemon_type
-    if daemon_type in Ceph.daemons:
+    if daemon_type in ceph_daemons():
         envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
     if container_args is None:
         container_args = []
-    unlimited_daemons = set(Ceph.daemons)
+    unlimited_daemons = set(ceph_daemons())
     unlimited_daemons.add(CephIscsi.daemon_type)
     unlimited_daemons.add(CephNvmeof.daemon_type)
     unlimited_daemons.add(NFSGanesha.daemon_type)
@@ -2777,7 +2781,7 @@ def get_container(
             container_args.extend(['--security-opt', 'label=disable'])
     elif daemon_type == 'crash':
         ceph_args = ['-n', name]
-    elif daemon_type in Ceph.daemons:
+    elif daemon_type in ceph_daemons():
         ceph_args = ['-n', name, '-f']
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
@@ -2966,7 +2970,7 @@ def deploy_daemon(
 
     # If this was a reconfig and the daemon is not a Ceph daemon, restart it
     # so it can pick up potential changes to its configuration files
-    if deployment_type == DeploymentType.RECONFIG and daemon_type not in Ceph.daemons:
+    if deployment_type == DeploymentType.RECONFIG and daemon_type not in ceph_daemons():
         # ceph daemons do not need a restart; others (presumably) do to pick
         # up the new config
         call_throws(ctx, ['systemctl', 'reset-failed', ident.unit_name])
@@ -3115,7 +3119,7 @@ def deploy_daemon_units(
 
         f.write('set -e\n')
 
-        if daemon_type in Ceph.daemons:
+        if daemon_type in ceph_daemons():
             install_path = find_program('install')
             f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid))
 
@@ -5320,7 +5324,7 @@ def command_shell(ctx):
         daemon_type = 'osd'  # get the most mounts
         daemon_id = None
 
-    if ctx.fsid and daemon_type in Ceph.daemons:
+    if ctx.fsid and daemon_type in ceph_daemons():
         make_log_dir(ctx, ctx.fsid)
 
     if daemon_id and not ctx.fsid:
@@ -5667,7 +5671,7 @@ def list_daemons(ctx, detail=True, legacy_dir=None):
                             if daemon_type == CephNvmeof.daemon_type:
                                 version = CephNvmeof.get_version(ctx, container_id)
                             elif not version:
-                                if daemon_type in Ceph.daemons:
+                                if daemon_type in ceph_daemons():
                                     out, err, code = call(ctx,
                                                           [container_path, 'exec', container_id,
                                                            'ceph', '-v'],
@@ -5856,7 +5860,7 @@ def command_adopt(ctx):
     lock.acquire()
 
     # call correct adoption
-    if daemon_type in Ceph.daemons:
+    if daemon_type in ceph_daemons():
         command_adopt_ceph(ctx, daemon_type, daemon_id, fsid)
     elif daemon_type == 'prometheus':
         command_adopt_prometheus(ctx, daemon_id, fsid)

From d9314780a59e991afce036236d425a5ed8368d09 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 16:51:49 -0400
Subject: [PATCH 0135/2492] cephadm: mock os.path.listdir in daemon forms test

Prevent classes that want to check the filesystem from breaking the
simple daemon forms instantiation test case. A better future fix would
be avoiding checking the file system during __init__ of the class but
that is left for future improvements.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_daemon_form.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/tests/test_daemon_form.py b/src/cephadm/tests/test_daemon_form.py
index 428183aaa3e1..07896cc58559 100644
--- a/src/cephadm/tests/test_daemon_form.py
+++ b/src/cephadm/tests/test_daemon_form.py
@@ -61,7 +61,7 @@ def test_is_sysctl_daemon_form(dt, is_sdf):
     assert isinstance(inst, daemon_form.SysctlDaemonForm) == is_sdf
 
 
-def test_can_create_all_daemon_forms():
+def test_can_create_all_daemon_forms(monkeypatch):
     uuid = 'daeb985e-58c7-11ee-a536-201e8814f771'
     ctx = mock.MagicMock()
     ctx.config_blobs = {
@@ -69,6 +69,8 @@ def test_can_create_all_daemon_forms():
         'pool': 'swimming',
         'destination': 'earth',
     }
+    _os_path_isdir = mock.MagicMock(return_value=True)
+    monkeypatch.setattr('os.path.isdir', _os_path_isdir)
     dtypes = _cephadm.get_supported_daemons()
     for daemon_type in dtypes:
         if daemon_type == 'agent':

From ead4cf25c959ea6edb04e68fcc45eed1a80be4f9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 3 Oct 2023 16:52:09 -0400
Subject: [PATCH 0136/2492] cephadm: convert ceph exporter type to a
 ContainerDaemonForm

CephExporter was being (partially) over-shadowed by the Ceph class as
the Ceph class listed 'ceph-exporter' as one of the daemon types it
handled. This change updates CephExporter to a ContainerDaemonForm while
simultaneously breaking the link between Ceph and 'ceph-exporter',
allowing CephExporter to handle all the duty of managing ceph-exporter,
continuing the process of having clearer logical responsibilities and
class hierarchy in cephadm.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index a3b2c22c4813..a64b0b49404d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -210,7 +210,7 @@ def __eq__(self, other: Any) -> bool:
 @register_daemon_form
 class Ceph(ContainerDaemonForm):
     _daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
-                'crash', 'cephfs-mirror', 'ceph-exporter')
+                'crash', 'cephfs-mirror')
 
     @classmethod
     def for_daemon_type(cls, daemon_type: str) -> bool:
@@ -1196,7 +1196,7 @@ def config_and_keyring(
 
 
 @register_daemon_form
-class CephExporter(DaemonForm):
+class CephExporter(ContainerDaemonForm):
     """Defines a Ceph exporter container"""
 
     daemon_type = 'ceph-exporter'
@@ -1264,6 +1264,17 @@ def validate(self) -> None:
         if not os.path.isdir(self.sock_dir):
             raise Error(f'Directory does not exist. Got: {self.sock_dir}')
 
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        return get_deployment_container(ctx, self.identity)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return extract_uid_gid(ctx)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
 
 ##################################
 
@@ -1724,7 +1735,9 @@ def get_supported_daemons():
 
 
 def ceph_daemons() -> List[str]:
-    return list(Ceph._daemons)
+    cds = list(Ceph._daemons)
+    cds.append(CephExporter.daemon_type)
+    return cds
 
 ##################################
 

From 2f2cd3bcff82afc3a4d251143eb462e700e7fc60 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Sun, 17 Sep 2023 22:52:56 -0400
Subject: [PATCH 0137/2492] qa/suites/rbd: add test to check rbd_support module
 recovery

... on repeated blocklisting of its client.

There were issues with rbd_support module not being able to recover
from its RADOS client being repeatedly blocklisted. This occured for
example in clusters with OSDs slow to process RBD requests while the
module's mirror_snapshot_scheduler was taking mirror snapshots by
requesting exclusive locks on the RBD images and workloads were running
on the snapshotted images via kernel clients.

Fixes: https://tracker.ceph.com/issues/62891
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 .../rbd_support_module_recovery.yaml          | 13 ++++
 .../rbd/rbd_support_module_recovery.sh        | 77 +++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 qa/suites/rbd/cli/workloads/rbd_support_module_recovery.yaml
 create mode 100755 qa/workunits/rbd/rbd_support_module_recovery.sh

diff --git a/qa/suites/rbd/cli/workloads/rbd_support_module_recovery.yaml b/qa/suites/rbd/cli/workloads/rbd_support_module_recovery.yaml
new file mode 100644
index 000000000000..aa4d0001fc09
--- /dev/null
+++ b/qa/suites/rbd/cli/workloads/rbd_support_module_recovery.yaml
@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    conf:
+      mgr:
+        debug rbd: 20
+tasks:
+- install:
+    extra_system_packages:
+      - fio
+- workunit:
+    clients:
+      client.0:
+        - rbd/rbd_support_module_recovery.sh
diff --git a/qa/workunits/rbd/rbd_support_module_recovery.sh b/qa/workunits/rbd/rbd_support_module_recovery.sh
new file mode 100755
index 000000000000..e9defced24a8
--- /dev/null
+++ b/qa/workunits/rbd/rbd_support_module_recovery.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+set -ex
+
+POOL=rbd
+IMAGE_PREFIX=image
+NUM_IMAGES=20
+RUN_TIME=3600
+
+rbd mirror pool enable ${POOL} image
+rbd mirror pool peer add ${POOL} dummy
+
+# Create images and schedule their mirror snapshots
+for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
+    rbd create -s 1G --image-feature exclusive-lock ${POOL}/${IMAGE_PREFIX}$i
+    rbd mirror image enable ${POOL}/${IMAGE_PREFIX}$i snapshot
+    rbd mirror snapshot schedule add -p ${POOL} --image ${IMAGE_PREFIX}$i 1m
+done
+
+# Run fio workloads on images via kclient
+# Test the recovery of the rbd_support module and its scheduler from their
+# librbd client being blocklisted while a exclusive lock gets passed around
+# between their librbd client and a kclient trying to take mirror snapshots
+# and perform I/O on the same image.
+for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
+    DEVS[$i]=$(sudo rbd device map ${POOL}/${IMAGE_PREFIX}$i)
+    fio --name=fiotest --filename=${DEVS[$i]} --rw=randrw --bs=4K --direct=1 \
+        --ioengine=libaio --iodepth=2 --runtime=43200 --time_based \
+        &> /dev/null &
+done
+
+# Repeatedly blocklist rbd_support module's client ~10s after the module
+# recovers from previous blocklisting
+CURRENT_TIME=$(date +%s)
+END_TIME=$((CURRENT_TIME + RUN_TIME))
+PREV_CLIENT_ADDR=""
+CLIENT_ADDR=""
+while ((CURRENT_TIME <= END_TIME)); do
+    if [[ -n "${CLIENT_ADDR}" ]] &&
+       [[ "${CLIENT_ADDR}" != "${PREV_CLIENT_ADDR}" ]]; then
+            ceph osd blocklist add ${CLIENT_ADDR}
+            # Confirm rbd_support module's client is blocklisted
+            ceph osd blocklist ls | grep -q ${CLIENT_ADDR}
+            PREV_CLIENT_ADDR=${CLIENT_ADDR}
+    fi
+    sleep 10
+    CLIENT_ADDR=$(ceph mgr dump |
+        jq .active_clients[] |
+        jq 'select(.name == "rbd_support")' |
+        jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
+    CURRENT_TIME=$(date +%s)
+done
+
+# Confirm that rbd_support module recovered from repeated blocklisting
+# Check that you can add a mirror snapshot schedule after a few retries
+for ((i = 1; i <= 24; i++)); do
+    rbd mirror snapshot schedule add -p ${POOL} \
+        --image ${IMAGE_PREFIX}1 2m && break
+    sleep 10
+done
+rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 |
+    grep 'every 2m'
+# Verify that the schedule present before client blocklisting is preserved
+rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 |
+    grep 'every 1m'
+rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}1 2m
+for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
+    rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}$i 1m
+done
+
+# cleanup
+killall fio || true
+wait
+for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
+    sudo rbd device unmap ${DEVS[$i]}
+done
+
+echo OK

From f328c2d7e75c4ce969d21f7c8b9ae2401baee5e1 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 10 Oct 2023 12:42:57 -0400
Subject: [PATCH 0138/2492] mgr/cephadm: add unit test to for upgrade check
 with --ceph-version

This is actually meant to make sure we don't screw
up the image base. See https://tracker.ceph.com/issues/63150
to see what we're trying to avoid happening again

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/tests/test_upgrade.py | 25 ++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/pybind/mgr/cephadm/tests/test_upgrade.py b/src/pybind/mgr/cephadm/tests/test_upgrade.py
index 7aa46f902769..3b5c305b5f0f 100644
--- a/src/pybind/mgr/cephadm/tests/test_upgrade.py
+++ b/src/pybind/mgr/cephadm/tests/test_upgrade.py
@@ -7,6 +7,7 @@
 from cephadm import CephadmOrchestrator
 from cephadm.upgrade import CephadmUpgrade, UpgradeState
 from cephadm.ssh import HostConnectionError
+from cephadm.utils import ContainerInspectInfo
 from orchestrator import OrchestratorError, DaemonDescription
 from .fixtures import _run_cephadm, wait, with_host, with_service, \
     receive_agent_metadata, async_side_effect
@@ -80,6 +81,30 @@ def test_upgrade_resume_clear_health_warnings(_rm_health_warning, cephadm_module
             _rm_health_warning.assert_has_calls(calls_list, any_order=True)
 
 
+@mock.patch('cephadm.upgrade.CephadmUpgrade._get_current_version', lambda _: (17, 2, 6))
+@mock.patch("cephadm.serve.CephadmServe._get_container_image_info")
+def test_upgrade_check_with_ceph_version(_get_img_info, cephadm_module: CephadmOrchestrator):
+    # This test was added to avoid screwing up the image base so that
+    # when the version was added to it it made an incorrect image
+    # The issue caused the image to come out as
+    # quay.io/ceph/ceph:v18:v18.2.0
+    # see https://tracker.ceph.com/issues/63150
+    _img = ''
+
+    def _fake_get_img_info(img_name):
+        nonlocal _img
+        _img = img_name
+        return ContainerInspectInfo(
+            'image_id',
+            '18.2.0',
+            'digest'
+        )
+
+    _get_img_info.side_effect = _fake_get_img_info
+    cephadm_module.upgrade_check('', '18.2.0')
+    assert _img == 'quay.io/ceph/ceph:v18.2.0'
+
+
 @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
 @pytest.mark.parametrize("use_repo_digest",
                          [

From 4fe08f167c1cf3df6a8d2a02573cfc328f1a9b77 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 3 Oct 2023 01:17:42 +0530
Subject: [PATCH 0139/2492] qa/cephfs: import only sleep() from time

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_admin.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index 923871f88e3a..b2e10e4d3cf4 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -1,10 +1,10 @@
 import errno
 import json
 import logging
-import time
 import uuid
 from io import StringIO
 from os.path import join as os_path_join
+from time import sleep
 
 from teuthology.exceptions import CommandFailedError
 
@@ -802,7 +802,7 @@ def test_fsmap_trim(self):
             self.fs.set_joinable(b)
             b = not b
 
-        time.sleep(10) # for tick/compaction
+        sleep(10) # for tick/compaction
 
         try:
             self.fs.status(epoch=epoch)
@@ -826,7 +826,7 @@ def test_fsmap_force_trim(self):
 
         # force a new fsmap
         self.fs.set_joinable(False)
-        time.sleep(10) # for tick/compaction
+        sleep(10) # for tick/compaction
 
         status = self.fs.status()
         log.debug(f"new epoch is {status['epoch']}")

From 56cbf3f0716b556c815487d719abe86021125925 Mon Sep 17 00:00:00 2001
From: shimin <shimin@kuaishou.com>
Date: Sun, 8 Oct 2023 19:15:09 +0800
Subject: [PATCH 0140/2492] mon: fix mds metadata lost in one case.

In most cases, peon's pending_metadata is inconsistent with mon's db.
When a peon turns into leader, and at the same time a active mds stops,
the new leader may flush wrong mds metadata into db. So we meed to
update mds metadata from db at every fsmap change.

This phenomenon can be reproduce like this:
A Cluster with 3 mon and 3 mds (one active, other two standby), 6 osd.
step 1. stop two standby mds;
step 2. restart all mon; (make pending_medata consistent with db)
step 3. start other two mds
step 4. stop leader mon
step 5. run "ceph mds metadata" command to check mds metadata
step 6. stop active mds
step 7. run "ceph mds metadata" command to check mds metadata again

In step 7, we would find mds metadata lost.

Fixes: https://tracker.ceph.com/issues/63166
Signed-off-by: shimin <shimin@kuaishou.com>
---
 src/mon/MDSMonitor.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 3042bdca30d8..e24d7388f9cd 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -136,6 +136,7 @@ void MDSMonitor::update_from_paxos(bool *need_bootstrap)
 	   << ", my e " << get_fsmap().get_epoch() << dendl;
   ceph_assert(version > get_fsmap().get_epoch());
 
+  load_metadata(pending_metadata);
   load_health();
 
   // read and decode

From a29e6a86733fbf8554089dd486ca6c9cf748886b Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 25 Sep 2023 17:05:50 +0530
Subject: [PATCH 0141/2492] mgr/dashboard: Show the OSD's Out and Down panels
 as red whenever an OSD is in Out or Down state in Ceph Cluster grafana
 dashboard

Fixes: https://tracker.ceph.com/issues/62969

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../dashboards_out/ceph-cluster.json          | 90 ++++++++++++++++++-
 1 file changed, 88 insertions(+), 2 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json b/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
index dc9e75382595..240c17677e3d 100644
--- a/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
+++ b/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
@@ -256,7 +256,93 @@
         }
       ],
       "title": "OSDs",
-      "type": "stat"
+      "type": "stat",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "percentage",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "All"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Out"
+            },
+            "properties": [
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "percentage",
+                  "steps": [
+                    {
+                      "color": "green",
+                      "value": null
+                    },
+                    {
+                      "color": "orange",
+                      "value": 0.1
+                    },
+                    {
+                      "value": 10,
+                      "color": "red"
+                    }
+                  ]
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Down"
+            },
+            "properties": [
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "percentage",
+                  "steps": [
+                    {
+                      "color": "green",
+                      "value": null
+                    },
+                    {
+                      "color": "orange",
+                      "value": 0.1
+                    },
+                    {
+                      "value": 10,
+                      "color": "red"
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      }
     },
     {
       "clusterName": "",
@@ -1276,4 +1362,4 @@
   "timezone": "",
   "title": "Ceph - Cluster",
   "version": 13
-    }
+}

From fca554cee093771646a0c4a37827d7fe3fe95a4a Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Mon, 25 Sep 2023 16:32:31 +0530
Subject: [PATCH 0142/2492] mgr/volumes: fix `subvolume group rm` error message

Currently, if we try to delete subvolumegroup using `fs subvolumegroup rm`
when there's one or more subvolume(s) present under that subvolumegroup we
see the error something like :
`Error ENOTEMPTY: error in rmdir /volumes/group1`
which causes confusion. Make it more descriptive

Fixes: https://tracker.ceph.com/issues/62968
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 src/pybind/mgr/volumes/fs/operations/group.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/pybind/mgr/volumes/fs/operations/group.py b/src/pybind/mgr/volumes/fs/operations/group.py
index 8b40610332dc..efc10e0797aa 100644
--- a/src/pybind/mgr/volumes/fs/operations/group.py
+++ b/src/pybind/mgr/volumes/fs/operations/group.py
@@ -269,6 +269,9 @@ def remove_group(fs, vol_spec, groupname):
     except cephfs.Error as e:
         if e.args[0] == errno.ENOENT:
             raise VolumeException(-errno.ENOENT, "subvolume group '{0}' does not exist".format(groupname))
+        elif e.args[0] == errno.ENOTEMPTY:
+            raise VolumeException(-errno.ENOTEMPTY, f"subvolume group {groupname} contains subvolume(s) "
+                                  "or retained snapshots of deleted subvolume(s)")
         raise VolumeException(-e.args[0], e.args[1])
 
 

From 1245e5c1dfd7bf78f1da534753c8954fa163138e Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Tue, 26 Sep 2023 08:43:43 +0530
Subject: [PATCH 0143/2492] qa: add test_subvolume_group_rm_when_its_not_empty

Fixes:  https://tracker.ceph.com/issues/62968
Signed-off-by: neeraj pratap singh <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
---
 qa/tasks/cephfs/test_volumes.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 44e28937bcbd..16ed00998d36 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -1871,6 +1871,32 @@ def test_subvolume_group_exists_without_subvolumegroup_and_with_subvolume(self):
         ret = self._fs_cmd("subvolumegroup", "exist", self.volname)
         self.assertEqual(ret.strip('\n'), "no subvolumegroup exists")
 
+    def test_subvolume_group_rm_when_its_not_empty(self):
+        group = self._generate_random_group_name()
+        subvolume = self._generate_random_subvolume_name()
+
+        # create subvolumegroup
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+        # create subvolume in group
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group)
+        # try, remove subvolume group
+        try:
+            self._fs_cmd("subvolumegroup", "rm", self.volname, group)
+        except CommandFailedError as ce:
+            self.assertEqual(ce.exitstatus, errno.ENOTEMPTY, "invalid error code on deleting "
+                             "subvolumegroup when it is not empty")
+        else:
+            self.fail("expected the 'fs subvolumegroup rm' command to fail")
+        
+        # delete subvolume
+        self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--group_name", group)
+
+        # delete subvolumegroup
+        self._fs_cmd("subvolumegroup", "rm", self.volname, group)
+
+        # verify trash dir is clean
+        self._wait_for_trash_empty()
+
 
 class TestSubvolumes(TestVolumesHelper):
     """Tests for FS subvolume operations, except snapshot and snapshot clone."""

From 3774d7319baec7a55ea98c54a205b23f52a33a7e Mon Sep 17 00:00:00 2001
From: Teng Jie <tengjie5@asiainfo.com>
Date: Wed, 23 Aug 2023 17:01:10 +0800
Subject: [PATCH 0144/2492] cephfs-shell: bump up acceptable xattr value len to
 64K Fixes: https://tracker.ceph.com/issues/62545 Signed-off-by: teng jie
 <tengjie5@asiainfo.com>

---
 src/tools/cephfs/shell/cephfs-shell | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/cephfs/shell/cephfs-shell b/src/tools/cephfs/shell/cephfs-shell
index 60ad79c6126c..96a606f24d44 100755
--- a/src/tools/cephfs/shell/cephfs-shell
+++ b/src/tools/cephfs/shell/cephfs-shell
@@ -1633,7 +1633,7 @@ class CephFSShell(Cmd):
         """
         try:
             poutput('{}'.format(cephfs.getxattr(args.path,
-                                to_bytes(args.name)).decode('utf-8')))
+                                to_bytes(args.name), size=65536).decode('utf-8')))
         except libcephfs.Error as e:
             set_exit_code_msg(msg=e)
 

From 15cebf2ad9edc453f17d7d5265e9e10f0d1ff7ba Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 11 Oct 2023 10:59:33 +0200
Subject: [PATCH 0145/2492] mgr/rook: Addint UT for rook cluster Fixes:
 https://tracker.ceph.com/issues/63107

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/tests/fixtures.py  |  11 +++
 src/pybind/mgr/rook/tests/test_rook.py | 120 +++++++++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 src/pybind/mgr/rook/tests/fixtures.py
 create mode 100644 src/pybind/mgr/rook/tests/test_rook.py

diff --git a/src/pybind/mgr/rook/tests/fixtures.py b/src/pybind/mgr/rook/tests/fixtures.py
new file mode 100644
index 000000000000..65a5197430c4
--- /dev/null
+++ b/src/pybind/mgr/rook/tests/fixtures.py
@@ -0,0 +1,11 @@
+from rook.module import RookOrchestrator
+from orchestrator import raise_if_exception, OrchResult
+
+try:
+    from typing import Any
+except ImportError:
+    pass
+
+
+def wait(m: RookOrchestrator, c: OrchResult) -> Any:
+    return raise_if_exception(c)
diff --git a/src/pybind/mgr/rook/tests/test_rook.py b/src/pybind/mgr/rook/tests/test_rook.py
new file mode 100644
index 000000000000..08028ba85502
--- /dev/null
+++ b/src/pybind/mgr/rook/tests/test_rook.py
@@ -0,0 +1,120 @@
+import orchestrator
+from .fixtures import wait
+import pytest
+from unittest.mock import patch, PropertyMock
+
+from rook.module import RookOrchestrator
+from rook.rook_cluster import RookCluster
+
+
+# we use this intermediate class as .rook_cluster property
+# is read only in the paretn class RookCluster
+class FakeRookCluster(RookCluster):
+    def __init__(self):
+        pass
+
+
+class TestRook(object):
+
+    @pytest.mark.parametrize("pods, expected_daemon_types", [
+        (
+            [
+                {
+                    'name': 'ceph-rook-exporter',
+                    'hostname': 'host1',
+                    "labels": {'app': 'rook-ceph-exporter',
+                               'ceph_daemon_id': 'exporter'},
+                    'phase': 'Pending',
+                    'container_image_name': 'quay.io/ceph/ceph:v18',
+                    'container_image_id': 'docker-pullable://quay.io/ceph/ceph@sha256:f239715e1c7756e32a202a572e2763a4ce15248e09fc6e8990985f8a09ffa784',
+                    'refreshed': 'pod1_ts',
+                    'started': 'pod1_ts',
+                    'created': 'pod1_1ts',
+                },
+                {
+                    'name': 'rook-ceph-mgr-a-68c7b9b6d8-vjjhl',
+                    'hostname': 'host1',
+                    "labels": {'app': 'rook-ceph-mgr',
+                               'ceph_daemon_type': 'mgr',
+                               'ceph_daemon_id': 'a'},
+                    'phase': 'Failed',
+                    'container_image_name': 'quay.io/ceph/ceph:v18',
+                    'container_image_id': '',
+                    'refreshed': 'pod2_ts',
+                    'started': 'pod2_ts',
+                    'created': 'pod2_1ts',
+                },
+                {
+                    'name': 'rook-ceph-mon-a-65fb8694b4-mmtl5',
+                    'hostname': 'host1',
+                    "labels": {'app': 'rook-ceph-mon',
+                               'ceph_daemon_type': 'mon',
+                               'ceph_daemon_id': 'b'},
+                    'phase': 'Running',
+                    'container_image_name': 'quay.io/ceph/ceph:v18',
+                    'container_image_id': '',
+                    'refreshed': 'pod3_ts',
+                    'started': 'pod3_ts',
+                    'created': 'pod3_1ts',
+                },
+                {
+                    'name': 'rook-ceph-osd-0-58cbd7b65c-6cjnr',
+                    'hostname': 'host1',
+                    "labels": {'app': 'rook-ceph-osd',
+                               'ceph-osd-id': '0',
+                               'ceph_daemon_type': 'osd',
+                               'ceph_daemon_id': '0'},
+                    'phase': 'Succeeded',
+                    'container_image_name': 'quay.io/ceph/ceph:v18',
+                    'container_image_id': '',
+                    'refreshed': 'pod4_ts',
+                    'started': 'pod4_ts',
+                    'created': 'pod4_1ts',
+                },
+                # unknown pod: has no labels are provided, it shouldn't
+                #  be part of the output
+                {
+                    'name': 'unknown-pod',
+                    'hostname': '',
+                    "labels": {'app': 'unkwon'},
+                    'phase': 'Pending',
+                    'container_image_name': 'quay.io/ceph/ceph:v18',
+                    'container_image_id': '',
+                    'refreshed': '',
+                    'started': '',
+                    'created': '',
+                }
+            ],
+            ['ceph-exporter', 'mgr', 'mon', 'osd']
+        )
+    ])
+    def test_list_daemons(self, pods, expected_daemon_types):
+
+        status = {
+            'Pending': orchestrator.DaemonDescriptionStatus.starting,
+            'Running': orchestrator.DaemonDescriptionStatus.running,
+            'Succeeded': orchestrator.DaemonDescriptionStatus.stopped,
+            'Failed': orchestrator.DaemonDescriptionStatus.error,
+            'Unknown': orchestrator.DaemonDescriptionStatus.unknown,
+        }
+
+        fake_rook_cluster = FakeRookCluster()
+        ro = RookOrchestrator('rook', None, self)
+        with patch('rook.RookOrchestrator.rook_cluster',
+                   new_callable=PropertyMock,
+                   return_value=fake_rook_cluster):
+            with patch.object(fake_rook_cluster, 'describe_pods') as mock_describe_pods:
+                mock_describe_pods.return_value = pods
+                dds = wait(ro, ro.list_daemons())
+                assert len(dds) == len(expected_daemon_types)
+                for i in range(0, len(dds)):
+                    assert dds[i].daemon_type == expected_daemon_types[i]
+                    assert dds[i].hostname == pods[i]['hostname']
+                    assert dds[i].status == status[pods[i]['phase']]
+                    assert dds[i].container_image_name == pods[i]['container_image_name']
+                    assert dds[i].container_image_id == pods[i]['container_image_id']
+                    assert dds[i].created == pods[i]['created']
+                    assert dds[i].last_configured == pods[i]['created']
+                    assert dds[i].last_deployed == pods[i]['created']
+                    assert dds[i].started == pods[i]['started']
+                    assert dds[i].last_refresh == pods[i]['refreshed']

From 8a5677f956d1b18ebae22c27b690b83e82db13cc Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 5 Oct 2023 17:41:38 +0530
Subject: [PATCH 0146/2492] mds: report clients laggy due laggy OSDs only after
 checking any OSD is laggy

Fixes: https://tracker.ceph.com/issues/63105
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/mds/Beacon.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/mds/Beacon.cc b/src/mds/Beacon.cc
index 861cfa4378f1..4990f0b249f9 100644
--- a/src/mds/Beacon.cc
+++ b/src/mds/Beacon.cc
@@ -500,13 +500,17 @@ void Beacon::notify_health(MDSRank const *mds)
 
   // Report laggy client(s) due to laggy OSDs
   {
+    bool defer_client_eviction =
+    g_conf().get_val<bool>("defer_client_eviction_on_laggy_osds")
+    && mds->objecter->with_osdmap([](const OSDMap &map) {
+      return map.any_osd_laggy(); });
     auto&& laggy_clients = mds->server->get_laggy_clients();
-    if (!laggy_clients.empty()) {
+    if (defer_client_eviction && !laggy_clients.empty()) {
       std::vector<MDSHealthMetric> laggy_clients_metrics;
       for (const auto& laggy_client: laggy_clients) {
         CachedStackStringStream css;
         *css << "Client " << laggy_client << " is laggy; not evicted"
-            << " because some OSD(s) is/are laggy";
+             << " because some OSD(s) is/are laggy";
         MDSHealthMetric m(MDS_HEALTH_CLIENTS_LAGGY, HEALTH_WARN, css->strv());
         laggy_clients_metrics.emplace_back(std::move(m));
       }

From 754b6022fb9fda075d38dcb1d058482f75dcff4d Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 11 Oct 2023 12:57:04 +0530
Subject: [PATCH 0147/2492] mds: erase clients getting evicted from
 laggy_clients

Fixes: https://tracker.ceph.com/issues/63105
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/mds/Server.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index f162025ddd1c..5e3110314b53 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1263,6 +1263,8 @@ void Server::find_idle_sessions()
       kill_session(session, NULL);
     }
   }
+  // clear as there's no use to keep the evicted clients in laggy_clients
+  clear_laggy_clients();
 }
 
 void Server::evict_cap_revoke_non_responders() {

From 17bd0c6aca24a8d10b1f346462d6df2e29afd80e Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Wed, 11 Oct 2023 21:08:19 +0000
Subject: [PATCH 0148/2492] src/mon: Added more loggings for disallowed_leaders

Added more loggings regarding the disallowed_leaders
set so that it is easier to debug problems like:

https://tracker.ceph.com/issues/63183

Fixes: https://tracker.ceph.com/issues/63183

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/mon/ElectionLogic.cc | 3 ++-
 src/mon/MonMap.cc        | 1 +
 src/mon/Monitor.cc       | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/mon/ElectionLogic.cc b/src/mon/ElectionLogic.cc
index e22a85bed56f..0c1b30c417c5 100644
--- a/src/mon/ElectionLogic.cc
+++ b/src/mon/ElectionLogic.cc
@@ -398,7 +398,8 @@ void ElectionLogic::propose_connectivity_handler(int from, epoch_t mepoch,
   ldout(cct, 10) << "propose from rank=" << from << ",from_score=" << from_score
 		 << "; my score=" << my_score
 		 << "; currently acked " << leader_acked
-		 << ",leader_score=" << leader_score << dendl;
+		 << ",leader_score=" << leader_score
+     << ",disallowed_leaders=" << elector->get_disallowed_leaders() << dendl;
 
   bool my_win = (my_score >= 0) && // My score is non-zero; I am allowed to lead
     ((my_rank < from && my_score >= from_score) || // We have same scores and I have lower rank, or
diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
index 33b9aa8fa288..bb8a4b19455d 100644
--- a/src/mon/MonMap.cc
+++ b/src/mon/MonMap.cc
@@ -369,6 +369,7 @@ void MonMap::print_summary(ostream& out) const
     has_printed = true;
   }
   out << "}" << " removed_ranks: {" << removed_ranks << "}";
+  out << " disallowed_leaders: {" << disallowed_leaders << "}";
 }
  
 void MonMap::print(ostream& out) const
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 349ca30a8981..6866536d0654 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2001,6 +2001,7 @@ void Monitor::handle_probe_reply(MonOpRequestRef op)
       dout(10) << " got newer/committed monmap epoch " << newmap->get_epoch()
 	       << ", mine was " << monmap->get_epoch() << dendl;
       int epoch_diff = newmap->get_epoch() - monmap->get_epoch();
+      dout(20) << " new monmap is " << *newmap  << dendl;
       delete newmap;
       monmap->decode(m->monmap_bl);
       dout(20) << "has_ever_joined: " << has_ever_joined << dendl;

From 163f79a810309c92ab7002ad92ea60175d8d836a Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Wed, 11 Oct 2023 15:07:02 -0700
Subject: [PATCH 0149/2492] qa/tests: fixed typo and added `--force-priority`
 in missing places

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index f169af75ac44..2b7435de608a 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -125,15 +125,15 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 ## !!!! the client suites below MUST use --suite-branch octopus, pacific (see https://tracker.ceph.com/issues/24021)
 
-20 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+20 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
 
-25 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+25 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
 
 22 14 * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:octopus-x --subset 111/120000 -e $CEPH_QA_EMAIL -k distro -p 100 --force-priority
 
 23 14 * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x --subset 111/120000 -e $CEPH_QA_EMAIL -k distro -p 100 --force-priority
 
-35 01 * * 7  CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL -p 75
+35 01 * * 7  CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL -p 75 --force-priority
 
 
 ### upgrade runs for reef release
@@ -144,13 +144,14 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 23 14 * * 6 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL -p 100 --force-priority
 
-20 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-reef -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+20 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-reef -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
 
-25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-reef -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-reef -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
 
-25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-quincy-reef -k distro -e $CEPH_QA_EMAIL --suite-branch quincy --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-quincy-reef -k distro -e $CEPH_QA_EMAIL --suite-branch quincy --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
 
 ### upgrade runs for squid release
 ###### on smithi
 
-25 02 * * 4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:reef-x -k distro -e $CEPH_QA_EMAIL --suite-branch quincy --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50
+25 02 * * 4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:reef-x -k distro -e $CEPH_QA_EMAIL --suite-branch main --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
+

From c992ab86d644410c4a03b48d4a7b372dbc2efd02 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Wed, 20 Sep 2023 09:46:54 +0000
Subject: [PATCH 0150/2492] rgw: adding request context structure

this structure should be created at the frontend and trickle all the way
to the RADOS layer. holding: dout prefix, optional yield and trace.
in this commit, so far it was only added to the "complete()" sal interface,
and to the "write_meta()" rados interface.

in the future, it should be added to more sal interfaces, replacing the
current way where dpp and optional yield are passed as sepearte
arguments to all functions.
in addition, if more information would be needed, it should be possible
to add that information to the request context struct without changing
many function prototypes

basic test instructions:
https://gist.github.com/yuvalif/1c7f1e80126bed5fa79345efb27fe1b1

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/common/tracer.h                          |  19 ++-
 src/rgw/driver/d4n/rgw_sal_d4n.cc            |   8 +-
 src/rgw/driver/d4n/rgw_sal_d4n.h             |   2 +-
 src/rgw/driver/daos/rgw_sal_daos.cc          |   2 +-
 src/rgw/driver/daos/rgw_sal_daos.h           |   2 +-
 src/rgw/driver/motr/rgw_sal_motr.cc          |   2 +-
 src/rgw/driver/motr/rgw_sal_motr.h           |   2 +-
 src/rgw/driver/posix/rgw_sal_posix.cc        |  18 +--
 src/rgw/driver/posix/rgw_sal_posix.h         |   4 +-
 src/rgw/driver/rados/rgw_cr_rados.cc         |   5 +-
 src/rgw/driver/rados/rgw_putobj_processor.cc |  28 ++--
 src/rgw/driver/rados/rgw_putobj_processor.h  |   6 +-
 src/rgw/driver/rados/rgw_rados.cc            | 160 ++++++++++---------
 src/rgw/driver/rados/rgw_rados.h             |  13 +-
 src/rgw/driver/rados/rgw_sal_rados.cc        |  27 ++--
 src/rgw/driver/rados/rgw_sal_rados.h         |   8 +-
 src/rgw/driver/rados/rgw_tools.cc            |   3 +-
 src/rgw/rgw_file.cc                          |   3 +-
 src/rgw/rgw_lc.cc                            |   4 +-
 src/rgw/rgw_op.cc                            |   9 +-
 src/rgw/rgw_req_context.h                    |  18 +++
 src/rgw/rgw_sal.h                            |   5 +-
 src/rgw/rgw_sal_dbstore.cc                   |   8 +-
 src/rgw/rgw_sal_dbstore.h                    |   8 +-
 src/rgw/rgw_sal_filter.cc                    |   4 +-
 src/rgw/rgw_sal_filter.h                     |   2 +-
 src/test/rgw/test_d4n_filter.cc              |  11 +-
 27 files changed, 209 insertions(+), 172 deletions(-)
 create mode 100644 src/rgw/rgw_req_context.h

diff --git a/src/common/tracer.h b/src/common/tracer.h
index 8a19db39021a..94efedbed6e1 100644
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -67,7 +67,8 @@ struct jspan_context {
   jspan_context(bool sampled_flag, bool is_remote) {}
 };
 
-struct span_stub {
+namespace opentelemetry::trace {
+struct Span {
   jspan_context _ctx;
   template <typename T>
   void SetAttribute(std::string_view key, const T& value) const noexcept {}
@@ -78,17 +79,21 @@ struct span_stub {
   void UpdateName(std::string_view) {}
   bool IsRecording() { return false; }
 };
+}
 
 class jspan {
-  span_stub span;
- public:
-  span_stub& operator*() { return span; }
-  const span_stub& operator*() const { return span; }
+  opentelemetry::trace::Span span;
+public:
+  opentelemetry::trace::Span& operator*() { return span; }
+  const opentelemetry::trace::Span& operator*() const { return span; }
 
-  span_stub* operator->() { return &span; }
-  const span_stub* operator->() const { return &span; }
+  opentelemetry::trace::Span* operator->() { return &span; }
+  const opentelemetry::trace::Span* operator->() const { return &span; }
 
   operator bool() const { return false; }
+
+  opentelemetry::trace::Span* get() { return &span; }
+  const opentelemetry::trace::Span* get() const { return &span; }
 };
 
 namespace tracing {
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index ff2ed7d9a204..3195d87eac42 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -445,7 +445,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   cache_block* temp_cache_block = filter->get_cache_block();
   RGWBlockDirectory* temp_block_dir = filter->get_block_dir();
@@ -467,9 +467,9 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
   RGWObjState* astate;
   int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, y);
-  obj->get_obj_attrs(y, save_dpp, NULL);
-  obj->get_obj_state(save_dpp, &astate, y);
+			canceled, rctx);
+  obj->get_obj_attrs(rctx.y, save_dpp, NULL);
+  obj->get_obj_state(save_dpp, &astate, rctx.y);
 
   /* Append additional metadata to attributes */ 
   rgw::sal::Attrs baseAttrs = obj->get_attrs();
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index 62c13f0abed6..5a2cd88896db 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -191,7 +191,7 @@ class D4NFilterWriter : public FilterWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; }
 };
diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 46db3dd654ca..73eec5b3e09c 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -2047,7 +2047,7 @@ int DaosMultipartWriter::complete(
     ceph::real_time set_mtime, std::map<std::string, bufferlist>& attrs,
     ceph::real_time delete_at, const char* if_match, const char* if_nomatch,
     const std::string* user_data, rgw_zone_set* zones_trace, bool* canceled,
-    optional_yield y) {
+    const req_context& rctx) {
   ldpp_dout(dpp, 20) << "DaosMultipartWriter::complete(): enter part="
                      << part_num_str << dendl;
 
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 0eaf495d2e2b..429c6160488d 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -748,7 +748,7 @@ class DaosAtomicWriter : public StoreWriter {
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 };
 
 class DaosMultipartWriter : public StoreWriter {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 06df127594e7..83c6153a92a5 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -2355,7 +2355,7 @@ int MotrAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   int rc = 0;
 
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index eee843d7effa..ce5fc2b95b1f 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -814,7 +814,7 @@ class MotrAtomicWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 
   unsigned populate_bvec(unsigned len, bufferlist::iterator &bi);
   void cleanup();
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 052471562e43..40091d06d0f9 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -2914,7 +2914,7 @@ int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   int ret;
   POSIXUploadPartInfo info;
@@ -2945,16 +2945,16 @@ int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& eta
   attrs[RGW_POSIX_ATTR_MPUPLOAD] = bl;
 
   for (auto& attr : attrs) {
-    ret = obj->write_attr(dpp, y, attr.first, attr.second);
+    ret = obj->write_attr(rctx.dpp, rctx.y, attr.first, attr.second);
     if (ret < 0) {
-      ldpp_dout(dpp, 20) << "ERROR: failed writing attr " << attr.first << dendl;
+      ldpp_dout(rctx.dpp, 20) << "ERROR: failed writing attr " << attr.first << dendl;
       return ret;
     }
   }
 
   ret = obj->close();
   if (ret < 0) {
-    ldpp_dout(dpp, 20) << "ERROR: failed closing file" << dendl;
+    ldpp_dout(rctx.dpp, 20) << "ERROR: failed closing file" << dendl;
     return ret;
   }
 
@@ -2981,7 +2981,7 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   int ret;
 
@@ -3023,14 +3023,14 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
   attrs[RGW_POSIX_ATTR_OWNER] = bl;
 
   for (auto attr : attrs) {
-    ret = obj.write_attr(dpp, y, attr.first, attr.second);
+    ret = obj.write_attr(rctx.dpp, rctx.y, attr.first, attr.second);
     if (ret < 0) {
-      ldpp_dout(dpp, 20) << "ERROR: POSIXAtomicWriter failed writing attr " << attr.first << dendl;
+      ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed writing attr " << attr.first << dendl;
       return ret;
     }
   }
 
-  ret = obj.link_temp_file(dpp, y);
+  ret = obj.link_temp_file(rctx.dpp, rctx.y);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << "ERROR: POSIXAtomicWriter failed writing temp file" << dendl;
     return ret;
@@ -3038,7 +3038,7 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
 
   ret = obj.close();
   if (ret < 0) {
-    ldpp_dout(dpp, 20) << "ERROR: POSIXAtomicWriter failed closing file" << dendl;
+    ldpp_dout(rctx.dpp, 20) << "ERROR: POSIXAtomicWriter failed closing file" << dendl;
     return ret;
   }
 
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index a2a5e5fdda9d..739e7ef7a610 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -625,7 +625,7 @@ class POSIXAtomicWriter : public StoreWriter {
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
-		       optional_yield y) override;
+		       const req_context& rctx) override;
 };
 
 class POSIXMultipartWriter : public StoreWriter {
@@ -664,7 +664,7 @@ class POSIXMultipartWriter : public StoreWriter {
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
-		       optional_yield y) override;
+		       const req_context& rctx) override;
 
 };
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index 6556d116b8f4..1baff16ff42e 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -752,6 +752,7 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
   std::string etag;
 
   std::optional<uint64_t> bytes_transferred;
+  const req_context rctx{dpp, null_yield, nullptr};
   int r = store->getRados()->fetch_remote_obj(obj_ctx,
                        user_id.value_or(rgw_user()),
                        NULL, /* req_info */
@@ -778,8 +779,8 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
                        &etag, /* string *petag, */
                        NULL, /* void (*progress_cb)(off_t, void *), */
                        NULL, /* void *progress_data*); */
-                       dpp,
-                       filter.get(), null_yield,
+                       rctx,
+                       filter.get(),
                        stat_follow_olh,
                        stat_dest_obj,
                        source_trace_entry,
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 9eb2ef266683..65fbd5791d5e 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -341,7 +341,8 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
                                     const char *if_nomatch,
                                     const std::string *user_data,
                                     rgw_zone_set *zones_trace,
-                                    bool *pcanceled, optional_yield y)
+                                    bool *pcanceled, 
+                                    const req_context& rctx)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -378,7 +379,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
 
   read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
 
-  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx);
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -482,7 +483,8 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
                                        const char *if_nomatch,
                                        const std::string *user_data,
                                        rgw_zone_set *zones_trace,
-                                       bool *pcanceled, optional_yield y)
+                                       bool *pcanceled, 
+                                       const req_context& rctx)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -506,7 +508,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx);
   if (r < 0)
     return r;
 
@@ -531,7 +533,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   bool compressed;
   r = rgw_compression_info_from_attrset(attrs, compressed, info.cs_info);
   if (r < 0) {
-    ldpp_dout(dpp, 1) << "cannot get compression info" << dendl;
+    ldpp_dout(rctx.dpp, 1) << "cannot get compression info" << dendl;
     return r;
   }
 
@@ -543,16 +545,16 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   store->obj_to_raw(bucket_info.placement_rule, meta_obj, &meta_raw_obj); 
 
   rgw_rados_ref meta_obj_ref;
-  r = store->get_raw_obj_ref(dpp, meta_raw_obj, &meta_obj_ref);
+  r = store->get_raw_obj_ref(rctx.dpp, meta_raw_obj, &meta_obj_ref);
   if (r < 0) {
-    ldpp_dout(dpp, -1) << "ERROR: failed to get obj ref of meta obj with ret=" << r << dendl;
+    ldpp_dout(rctx.dpp, -1) << "ERROR: failed to get obj ref of meta obj with ret=" << r << dendl;
     return r;
   }
 
   librados::ObjectWriteOperation op;
   cls_rgw_mp_upload_part_info_update(op, p, info);
-  r = rgw_rados_operate(dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, y);
-  ldpp_dout(dpp, 20) << "Update meta: " << meta_obj_ref.obj.oid << " part " << p << " prefix " << info.manifest.get_prefix() << " return " << r << dendl;
+  r = rgw_rados_operate(rctx.dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, rctx.y);
+  ldpp_dout(rctx.dpp, 20) << "Update meta: " << meta_obj_ref.obj.oid << " part " << p << " prefix " << info.manifest.get_prefix() << " return " << r << dendl;
 
   if (r == -EOPNOTSUPP) {
     // New CLS call to update part info is not yet supported. Fall back to the old handling.
@@ -565,7 +567,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
     op = librados::ObjectWriteOperation{};
     op.assert_exists(); // detect races with abort
     op.omap_set(m);
-    r = rgw_rados_operate(dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, y);
+    r = rgw_rados_operate(rctx.dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, rctx.y);
   }
   if (r < 0) {
     return r == -ENOENT ? -ERR_NO_SUCH_UPLOAD : r;
@@ -686,7 +688,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
                                     ceph::real_time set_mtime, rgw::sal::Attrs& attrs,
                                     ceph::real_time delete_at, const char *if_match, const char *if_nomatch,
                                     const string *user_data, rgw_zone_set *zones_trace, bool *pcanceled,
-                                    optional_yield y)
+                                    const req_context& rctx)
 {
   int r = writer.drain();
   if (r < 0)
@@ -742,9 +744,9 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
     etag_bl.append(final_etag_str, strlen(final_etag_str) + 1);
     attrs[RGW_ATTR_ETAG] = etag_bl;
   }
-  r = obj_op.write_meta(dpp, actual_size + cur_size,
+  r = obj_op.write_meta(actual_size + cur_size,
 			accounted_size + *cur_accounted_size,
-			attrs, y);
+			attrs, rctx);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index fa9200f32dae..9a21c0c793a3 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -191,7 +191,7 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                const char *if_match, const char *if_nomatch,
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
-               optional_yield y) override;
+               const req_context& rctx) override;
 
 };
 
@@ -238,7 +238,7 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                const char *if_match, const char *if_nomatch,
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
-               optional_yield y) override;
+               const req_context& rctx) override;
 
 };
 
@@ -274,7 +274,7 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                  std::map<std::string, bufferlist>& attrs, ceph::real_time delete_at,
                  const char *if_match, const char *if_nomatch, const std::string *user_data,
                  rgw_zone_set *zones_trace, bool *canceled,
-                 optional_yield y) override;
+                 const req_context& rctx) override;
   };
 
 } // namespace putobj
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 8814c5a1fbac..5437d12d4b76 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2665,8 +2665,8 @@ int RGWRados::fix_tail_obj_locator(const DoutPrefixProvider *dpp,
 
   RGWObjState *astate = nullptr;
   RGWObjManifest* manifest = nullptr;
-  RGWObjectCtx rctx(this->driver);
-  r = get_obj_state(dpp, &rctx, bucket_info, obj, &astate, &manifest, false, y);
+  RGWObjectCtx octx(this->driver);
+  r = get_obj_state(dpp, &octx, bucket_info, obj, &astate, &manifest, false, y);
   if (r < 0)
     return r;
 
@@ -3047,11 +3047,10 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
                                   handler, y);
 }
 
-int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
-                                           uint64_t size, uint64_t accounted_size,
+int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_size,
                                            map<string, bufferlist>& attrs,
                                            bool assume_noent, bool modify_tail,
-                                           void *_index_op, optional_yield y)
+                                           void *_index_op, const req_context& rctx)
 {
   RGWRados::Bucket::UpdateIndex *index_op = static_cast<RGWRados::Bucket::UpdateIndex *>(_index_op);
   RGWRados *store = target->get_store();
@@ -3070,19 +3069,19 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
 
   RGWObjState *state;
   RGWObjManifest *manifest = nullptr;
-  int r = target->get_state(dpp, &state, &manifest, false, y, assume_noent);
+  int r = target->get_state(rctx.dpp, &state, &manifest, false, rctx.y, assume_noent);
   if (r < 0)
     return r;
 
   rgw_obj& obj = target->get_obj();
 
   if (obj.get_oid().empty()) {
-    ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): cannot write object with empty name" << dendl;
+    ldpp_dout(rctx.dpp, 0) << "ERROR: " << __func__ << "(): cannot write object with empty name" << dendl;
     return -EIO;
   }
 
   rgw_rados_ref ref;
-  r = store->get_obj_head_ref(dpp, target->get_meta_placement_rule(), obj, &ref);
+  r = store->get_obj_head_ref(rctx.dpp, target->get_meta_placement_rule(), obj, &ref);
   if (r < 0)
     return r;
 
@@ -3094,7 +3093,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   if (!ptag && !index_op->get_optag()->empty()) {
     ptag = index_op->get_optag();
   }
-  r = target->prepare_atomic_modification(dpp, op, reset_obj, ptag, meta.if_match, meta.if_nomatch, false, modify_tail, y);
+  r = target->prepare_atomic_modification(rctx.dpp, op, reset_obj, ptag, meta.if_match, meta.if_nomatch, false, modify_tail, rctx.y);
   if (r < 0)
     return r;
 
@@ -3217,7 +3216,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
 
   if (!index_op->is_prepared()) {
     tracepoint(rgw_rados, prepare_enter, req_id.c_str());
-    r = index_op->prepare(dpp, CLS_RGW_OP_ADD, &state->write_tag, y);
+    r = index_op->prepare(rctx.dpp, CLS_RGW_OP_ADD, &state->write_tag, rctx.y);
     tracepoint(rgw_rados, prepare_exit, req_id.c_str());
     if (r < 0)
       return r;
@@ -3226,7 +3225,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   auto& ioctx = ref.pool.ioctx();
 
   tracepoint(rgw_rados, operate_enter, req_id.c_str());
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(rctx.dpp, ref.pool.ioctx(), ref.obj.oid, &op, rctx.y);
   tracepoint(rgw_rados, operate_exit, req_id.c_str());
   if (r < 0) { /* we can expect to get -ECANCELED if object was replaced under,
                 or -ENOENT if was removed, or -EEXIST if it did not exist
@@ -3241,16 +3240,16 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   epoch = ioctx.get_last_version();
   poolid = ioctx.get_id();
 
-  r = target->complete_atomic_modification(dpp, y);
+  r = target->complete_atomic_modification(rctx.dpp, rctx.y);
   if (r < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: complete_atomic_modification returned r=" << r << dendl;
+    ldpp_dout(rctx.dpp, 0) << "ERROR: complete_atomic_modification returned r=" << r << dendl;
   }
 
   tracepoint(rgw_rados, complete_enter, req_id.c_str());
-  r = index_op->complete(dpp, poolid, epoch, size, accounted_size,
+  r = index_op->complete(rctx.dpp, poolid, epoch, size, accounted_size,
                         meta.set_mtime, etag, content_type,
                         storage_class, &acl_bl,
-			 meta.category, meta.remove_objs, y,
+			 meta.category, meta.remove_objs, rctx.y,
 			 meta.user_data, meta.appendable);
   tracepoint(rgw_rados, complete_exit, req_id.c_str());
   if (r < 0)
@@ -3265,7 +3264,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   state = NULL;
 
   if (versioned_op && meta.olh_epoch) {
-    r = store->set_olh(dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, y, meta.zones_trace);
+    r = store->set_olh(rctx.dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, rctx.y, meta.zones_trace);
     if (r < 0) {
       return r;
     }
@@ -3275,10 +3274,10 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
     rgw_obj_index_key obj_key;
     obj.key.get_index_key(&obj_key);
 
-    r = store->obj_expirer->hint_add(dpp, meta.delete_at, obj.bucket.tenant, obj.bucket.name,
+    r = store->obj_expirer->hint_add(rctx.dpp, meta.delete_at, obj.bucket.tenant, obj.bucket.name,
                                      obj.bucket.bucket_id, obj_key);
     if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: objexp_hint_add() returned r=" << r << ", object will not get removed" << dendl;
+      ldpp_dout(rctx.dpp, 0) << "ERROR: objexp_hint_add() returned r=" << r << ", object will not get removed" << dendl;
       /* ignoring error, nothing we can do at this point */
     }
   }
@@ -3296,9 +3295,9 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   return 0;
 
 done_cancel:
-  int ret = index_op->cancel(dpp, meta.remove_objs, y);
+  int ret = index_op->cancel(rctx.dpp, meta.remove_objs, rctx.y);
   if (ret < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
+    ldpp_dout(rctx.dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
   }
 
   meta.canceled = true;
@@ -3340,8 +3339,8 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   return r;
 }
 
-int RGWRados::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size,
-                                           map<string, bufferlist>& attrs, optional_yield y)
+int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
+                                           map<string, bufferlist>& attrs, const req_context& rctx)
 {
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -3352,13 +3351,13 @@ int RGWRados::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t
   bool assume_noent = (meta.if_match == NULL && meta.if_nomatch == NULL);
   int r;
   if (assume_noent) {
-    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx);
     if (r == -EEXIST) {
       assume_noent = false;
     }
   }
   if (!assume_noent) {
-    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx);
   }
   return r;
 }
@@ -3624,11 +3623,11 @@ static void set_copy_attrs(map<string, bufferlist>& src_attrs,
 
 int RGWRados::rewrite_obj(RGWBucketInfo& dest_bucket_info, const rgw_obj& obj, const DoutPrefixProvider *dpp, optional_yield y)
 {
-  RGWObjectCtx rctx(this->driver);
+  RGWObjectCtx octx(this->driver);
   rgw::sal::Attrs attrset;
   uint64_t obj_size;
   ceph::real_time mtime;
-  RGWRados::Object op_target(this, dest_bucket_info, rctx, obj);
+  RGWRados::Object op_target(this, dest_bucket_info, octx, obj);
   RGWRados::Object::Read read_op(&op_target);
 
   read_op.params.attrs = &attrset;
@@ -3643,7 +3642,7 @@ int RGWRados::rewrite_obj(RGWBucketInfo& dest_bucket_info, const rgw_obj& obj, c
   attrset.erase(RGW_ATTR_TAIL_TAG);
   attrset.erase(RGW_ATTR_STORAGE_CLASS);
 
-  return copy_obj_data(rctx, dest_bucket_info, dest_bucket_info.placement_rule,
+  return copy_obj_data(octx, dest_bucket_info, dest_bucket_info.placement_rule,
                        read_op, obj_size - 1, obj, NULL, mtime,
                        attrset, 0, real_time(), NULL, dpp, y);
 }
@@ -4138,8 +4137,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
                string *petag,
                void (*progress_cb)(off_t, void *),
                void *progress_data,
-               const DoutPrefixProvider *dpp,
-               RGWFetchObjFilter *filter, optional_yield y,
+               const req_context& rctx,
+               RGWFetchObjFilter *filter,
                bool stat_follow_olh,
                const rgw_obj& stat_dest_obj,
                const rgw_zone_set_entry& source_trace_entry,
@@ -4160,7 +4159,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   using namespace rgw::putobj;
   AtomicObjectProcessor processor(&aio, this, dest_bucket_info, nullptr,
                                   user_id, obj_ctx, dest_obj, olh_epoch,
-				  tag, dpp, y);
+				  tag, rctx.dpp, rctx.y);
   RGWRESTConn *conn;
   auto& zone_conn_map = svc.zone->get_zone_conn_map();
   auto& zonegroup_conn_map = svc.zone->get_zonegroup_conn_map();
@@ -4171,7 +4170,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     } else {
       map<string, RGWRESTConn *>::iterator iter = zonegroup_conn_map.find(src_bucket_info->zonegroup);
       if (iter == zonegroup_conn_map.end()) {
-        ldpp_dout(dpp, 0) << "could not find zonegroup connection to zonegroup: " << source_zone << dendl;
+        ldpp_dout(rctx.dpp, 0) << "could not find zonegroup connection to zonegroup: " << source_zone << dendl;
         return -ENOENT;
       }
       conn = iter->second;
@@ -4179,7 +4178,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   } else {
     auto iter = zone_conn_map.find(source_zone);
     if (iter == zone_conn_map.end()) {
-      ldpp_dout(dpp, 0) << "could not find zone connection to zone: " << source_zone << dendl;
+      ldpp_dout(rctx.dpp, 0) << "could not find zone connection to zone: " << source_zone << dendl;
       return -ENOENT;
     }
     conn = iter->second;
@@ -4195,7 +4194,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   std::optional<rgw_user> override_owner;
 
-  RGWRadosPutObj cb(dpp, cct, plugin, compressor, &processor, progress_cb, progress_data,
+  RGWRadosPutObj cb(rctx.dpp, cct, plugin, compressor, &processor, progress_cb, progress_data,
                     [&](map<string, bufferlist>& obj_attrs) {
                       const rgw_placement_rule *ptail_rule;
 
@@ -4207,7 +4206,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 					       &override_owner,
                                                &ptail_rule);
                       if (ret < 0) {
-                        ldpp_dout(dpp, 5) << "Aborting fetch: source object filter returned ret=" << ret << dendl;
+                        ldpp_dout(rctx.dpp, 5) << "Aborting fetch: source object filter returned ret=" << ret << dendl;
                         return ret;
                       }
 
@@ -4217,12 +4216,12 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
                       if (compression_type != "none") {
                         plugin = Compressor::create(cct, compression_type);
                         if (!plugin) {
-                          ldpp_dout(dpp, 1) << "Cannot load plugin for compression type "
+                          ldpp_dout(rctx.dpp, 1) << "Cannot load plugin for compression type "
                                         << compression_type << dendl;
                         }
                       }
 
-                      ret = processor.prepare(y);
+                      ret = processor.prepare(rctx.y);
                       if (ret < 0) {
                         return ret;
                       }
@@ -4243,7 +4242,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   if (copy_if_newer) {
     /* need to get mtime for destination */
-    ret = get_obj_state(dpp, &obj_ctx, dest_bucket_info, stat_dest_obj, &dest_state, &manifest, stat_follow_olh, y);
+    ret = get_obj_state(rctx.dpp, &obj_ctx, dest_bucket_info, stat_dest_obj, &dest_state, &manifest, stat_follow_olh, rctx.y);
     if (ret < 0)
       goto set_err_state;
 
@@ -4259,7 +4258,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   static constexpr bool sync_manifest = true;
   static constexpr bool skip_decrypt = true;
   static constexpr bool sync_cloudtiered = true;
-  ret = conn->get_obj(dpp, user_id, info, src_obj, pmod, unmod_ptr,
+  ret = conn->get_obj(rctx.dpp, user_id, info, src_obj, pmod, unmod_ptr,
                       dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
                       prepend_meta, get_op, rgwx_stat,
                       sync_manifest, skip_decrypt, &dst_zone_trace,
@@ -4270,7 +4269,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   }
 
   ret = conn->complete_request(in_stream_req, &etag, &set_mtime,
-                               &expected_size, nullptr, nullptr, y);
+                               &expected_size, nullptr, nullptr, rctx.y);
   if (ret < 0) {
     goto set_err_state;
   }
@@ -4280,7 +4279,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   }
   if (cb.get_data_len() != expected_size) {
     ret = -EIO;
-    ldpp_dout(dpp, 0) << "ERROR: object truncated during fetching, expected "
+    ldpp_dout(rctx.dpp, 0) << "ERROR: object truncated during fetching, expected "
         << expected_size << " bytes but received " << cb.get_data_len() << dendl;
     goto set_err_state;
   }
@@ -4301,8 +4300,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     auto& obj_attrs = cb.get_attrs();
 
     RGWUserInfo owner_info;
-    if (ctl.user->get_info_by_uid(dpp, *override_owner, &owner_info, y) < 0) {
-      ldpp_dout(dpp, 10) << "owner info does not exist" << dendl;
+    if (ctl.user->get_info_by_uid(rctx.dpp, *override_owner, &owner_info, rctx.y) < 0) {
+      ldpp_dout(rctx.dpp, 10) << "owner info does not exist" << dendl;
       return -EINVAL;
     }
 
@@ -4310,14 +4309,14 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
     auto aiter = obj_attrs.find(RGW_ATTR_ACL);
     if (aiter == obj_attrs.end()) {
-      ldpp_dout(dpp, 0) << "WARNING: " << __func__ << "(): object doesn't have ACL attribute, setting default ACLs" << dendl;
+      ldpp_dout(rctx.dpp, 0) << "WARNING: " << __func__ << "(): object doesn't have ACL attribute, setting default ACLs" << dendl;
       acl.create_default(owner_info.user_id, owner_info.display_name);
     } else {
       auto iter = aiter->second.cbegin();
       try {
 	acl.decode(iter);
       } catch (buffer::error& err) {
-	ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): could not decode policy, caught buffer::error" << dendl;
+	ldpp_dout(rctx.dpp, 0) << "ERROR: " << __func__ << "(): could not decode policy, caught buffer::error" << dendl;
 	return -EIO;
       }
     }
@@ -4341,7 +4340,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
       try {
         decode(delete_at, iter->second);
       } catch (buffer::error& err) {
-        ldpp_dout(dpp, 0) << "ERROR: failed to decode delete_at field in intra zone copy" << dendl;
+        ldpp_dout(rctx.dpp, 0) << "ERROR: failed to decode delete_at field in intra zone copy" << dendl;
       }
     }
   }
@@ -4395,7 +4394,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
       try {
         decode(pg_ver, iter);
       } catch (buffer::error& err) {
-        ldpp_dout(dpp, 0) << "ERROR: failed to decode pg ver attribute, ignoring" << dendl;
+        ldpp_dout(rctx.dpp, 0) << "ERROR: failed to decode pg ver attribute, ignoring" << dendl;
         /* non critical error */
       }
     }
@@ -4413,7 +4412,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
     if (verifier_etag != trimmed_etag) {
       ret = -EIO;
-      ldpp_dout(dpp, 0) << "ERROR: source and destination objects don't match. Expected etag:"
+      ldpp_dout(rctx.dpp, 0) << "ERROR: source and destination objects don't match. Expected etag:"
         << trimmed_etag << " Computed etag:" << verifier_etag << dendl;
       goto set_err_state;
     }
@@ -4424,34 +4423,34 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     bool canceled = false;
     ret = processor.complete(cb.get_data_len(), etag, mtime, set_mtime,
                              attrs, delete_at, nullptr, nullptr, nullptr,
-                             zones_trace, &canceled, y);
+                             zones_trace, &canceled, rctx);
     if (ret < 0) {
       goto set_err_state;
     }
 
     if (copy_if_newer && canceled) {
-      ldpp_dout(dpp, 20) << "raced with another write of obj: " << dest_obj << dendl;
+      ldpp_dout(rctx.dpp, 20) << "raced with another write of obj: " << dest_obj << dendl;
       obj_ctx.invalidate(dest_obj); /* object was overwritten */
-      ret = get_obj_state(dpp, &obj_ctx, dest_bucket_info, stat_dest_obj, &dest_state, &manifest, stat_follow_olh, y);
+      ret = get_obj_state(rctx.dpp, &obj_ctx, dest_bucket_info, stat_dest_obj, &dest_state, &manifest, stat_follow_olh, rctx.y);
       if (ret < 0) {
-        ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": get_err_state() returned ret=" << ret << dendl;
+        ldpp_dout(rctx.dpp, 0) << "ERROR: " << __func__ << ": get_err_state() returned ret=" << ret << dendl;
         goto set_err_state;
       }
       dest_mtime_weight.init(dest_state);
       dest_mtime_weight.high_precision = high_precision_time;
       if (!dest_state->exists ||
         dest_mtime_weight < set_mtime_weight) {
-        ldpp_dout(dpp, 20) << "retrying writing object mtime=" << set_mtime << " dest_state->mtime=" << dest_state->mtime << " dest_state->exists=" << dest_state->exists << dendl;
+        ldpp_dout(rctx.dpp, 20) << "retrying writing object mtime=" << set_mtime << " dest_state->mtime=" << dest_state->mtime << " dest_state->exists=" << dest_state->exists << dendl;
         continue;
       } else {
-        ldpp_dout(dpp, 20) << "not retrying writing object mtime=" << set_mtime << " dest_state->mtime=" << dest_state->mtime << " dest_state->exists=" << dest_state->exists << dendl;
+        ldpp_dout(rctx.dpp, 20) << "not retrying writing object mtime=" << set_mtime << " dest_state->mtime=" << dest_state->mtime << " dest_state->exists=" << dest_state->exists << dendl;
       }
     }
     break;
   }
 
   if (i == MAX_COMPLETE_RETRY) {
-    ldpp_dout(dpp, 0) << "ERROR: retried object completion too many times, something is wrong!" << dendl;
+    ldpp_dout(rctx.dpp, 0) << "ERROR: retried object completion too many times, something is wrong!" << dendl;
     ret = -EIO;
     goto set_err_state;
   }
@@ -4466,8 +4465,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     // for OP_LINK_OLH to call set_olh() with a real olh_epoch
     if (olh_epoch && *olh_epoch > 0) {
       constexpr bool log_data_change = true;
-      ret = set_olh(dpp, obj_ctx, dest_bucket_info, dest_obj, false, nullptr,
-                    *olh_epoch, real_time(), false, y, zones_trace, log_data_change);
+      ret = set_olh(rctx.dpp, obj_ctx, dest_bucket_info, dest_obj, false, nullptr,
+                    *olh_epoch, real_time(), false, rctx.y, zones_trace, log_data_change);
     } else {
       // we already have the latest copy
       ret = 0;
@@ -4590,13 +4589,14 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
 
   if (remote_src || !source_zone.empty()) {
     rgw_zone_set_entry source_trace_entry{source_zone.id, std::nullopt};
+    const req_context rctx{dpp, y, nullptr};
     return fetch_remote_obj(obj_ctx, user_id, info, source_zone,
                dest_obj, src_obj, dest_bucket_info, &src_bucket_info,
                dest_placement, src_mtime, mtime, mod_ptr,
                unmod_ptr, high_precision_time,
                if_match, if_nomatch, attrs_mod, copy_if_newer, attrs, category,
-               olh_epoch, delete_at, ptag, petag, progress_cb, progress_data, dpp,
-               nullptr /* filter */, y, stat_follow_olh, stat_dest_obj, source_trace_entry);
+               olh_epoch, delete_at, ptag, petag, progress_cb, progress_data, rctx,
+               nullptr /* filter */, stat_follow_olh, stat_dest_obj, source_trace_entry);
   }
 
   map<string, bufferlist> src_attrs;
@@ -4756,6 +4756,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
     append_rand_alpha(cct, tag, tag, 32);
   }
 
+  const req_context rctx{dpp, y, nullptr};
   std::unique_ptr<rgw::Aio> aio;
   rgw::AioResultList all_results;
   if (!copy_itself) {
@@ -4828,7 +4829,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   write_op.meta.delete_at = delete_at;
   write_op.meta.modify_tail = !copy_itself;
 
-  ret = write_op.write_meta(dpp, obj_size, astate->accounted_size, attrs, y);
+  ret = write_op.write_meta(obj_size, astate->accounted_size, attrs, rctx);
   if (ret < 0) {
     goto done_ret;
   }
@@ -4951,8 +4952,9 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
     accounted_size = compressed ? cs_info.orig_size : ofs;
   }
 
+  const req_context rctx{dpp, y, nullptr};
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-                            nullptr, nullptr, nullptr, nullptr, nullptr, y);
+                            nullptr, nullptr, nullptr, nullptr, nullptr, rctx);
 }
 
 int RGWRados::transition_obj(RGWObjectCtx& obj_ctx,
@@ -5503,17 +5505,17 @@ int RGWRados::bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketI
   return r;
 }
 
-int RGWRados::defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y)
+int RGWRados::defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y)
 {
   std::string oid, key;
   get_obj_bucket_and_oid_loc(obj, oid, key);
-  if (!rctx)
+  if (!octx)
     return 0;
 
   RGWObjState *state = NULL;
   RGWObjManifest *manifest = nullptr;
 
-  int r = get_obj_state(dpp, rctx, bucket_info, obj, &state, &manifest, false, y);
+  int r = get_obj_state(dpp, octx, bucket_info, obj, &state, &manifest, false, y);
   if (r < 0)
     return r;
 
@@ -5895,7 +5897,7 @@ int RGWRados::get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx&
   return 0;
 }
 
-int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
+int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *octx,
 				 RGWBucketInfo& bucket_info, const rgw_obj& obj,
                                  RGWObjState **state, RGWObjManifest** manifest,
 				 bool follow_olh, optional_yield y, bool assume_noent)
@@ -5907,16 +5909,16 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rc
   bool need_follow_olh = follow_olh && obj.key.instance.empty();
   *manifest = nullptr;
 
-  RGWObjStateManifest *sm = rctx->get_state(obj);
+  RGWObjStateManifest *sm = octx->get_state(obj);
   RGWObjState *s = &(sm->state);
-  ldpp_dout(dpp, 20) << "get_obj_state: rctx=" << (void *)rctx << " obj=" << obj << " state=" << (void *)s << " s->prefetch_data=" << s->prefetch_data << dendl;
+  ldpp_dout(dpp, 20) << "get_obj_state: octx=" << (void *)octx << " obj=" << obj << " state=" << (void *)s << " s->prefetch_data=" << s->prefetch_data << dendl;
   *state = s;
   if (sm->manifest) {
     *manifest = &(*sm->manifest);
   }
   if (s->has_attrs) {
     if (s->is_olh && need_follow_olh) {
-      return get_olh_target_state(dpp, *rctx, bucket_info, obj, s, state, manifest, y);
+      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, state, manifest, y);
     }
     return 0;
   }
@@ -6068,7 +6070,7 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rc
     ldpp_dout(dpp, 20) << __func__ << ": setting s->olh_tag to " << string(s->olh_tag.c_str(), s->olh_tag.length()) << dendl;
 
     if (need_follow_olh) {
-      return get_olh_target_state(dpp, *rctx, bucket_info, obj, s, state, manifest, y);
+      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, state, manifest, y);
     } else if (obj.key.have_null_instance() && !sm->manifest) {
       // read null version, and the head object only have olh info
       s->exists = false;
@@ -6079,13 +6081,13 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rc
   return 0;
 }
 
-int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest,
+int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *octx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest,
                             bool follow_olh, optional_yield y, bool assume_noent)
 {
   int ret;
 
   do {
-    ret = get_obj_state_impl(dpp, rctx, bucket_info, obj, state, manifest, follow_olh, y, assume_noent);
+    ret = get_obj_state_impl(dpp, octx, bucket_info, obj, state, manifest, follow_olh, y, assume_noent);
   } while (ret == -EAGAIN);
 
   return ret;
@@ -6195,15 +6197,15 @@ int RGWRados::Object::Stat::finish(const DoutPrefixProvider *dpp)
   return 0;
 }
 
-int RGWRados::append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx,
+int RGWRados::append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx* octx,
                                  RGWBucketInfo& bucket_info, const rgw_obj& obj,
                                  ObjectOperation& op, RGWObjState **pstate,
 				 RGWObjManifest** pmanifest, optional_yield y)
 {
-  if (!rctx)
+  if (!octx)
     return 0;
 
-  int r = get_obj_state(dpp, rctx, bucket_info, obj, pstate, pmanifest, false, y);
+  int r = get_obj_state(dpp, octx, bucket_info, obj, pstate, pmanifest, false, y);
   if (r < 0)
     return r;
 
@@ -6339,14 +6341,14 @@ int RGWRados::Object::prepare_atomic_modification(const DoutPrefixProvider *dpp,
  * bl: the contents of the attr
  * Returns: 0 on success, -ERR# otherwise.
  */
-int RGWRados::set_attr(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, const char *name, bufferlist& bl, optional_yield y)
+int RGWRados::set_attr(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBucketInfo& bucket_info, const rgw_obj& obj, const char *name, bufferlist& bl, optional_yield y)
 {
   map<string, bufferlist> attrs;
   attrs[name] = bl;
-  return set_attrs(dpp, rctx, bucket_info, obj, attrs, NULL, y);
+  return set_attrs(dpp, octx, bucket_info, obj, attrs, NULL, y);
 }
 
-int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx, RGWBucketInfo& bucket_info, const rgw_obj& src_obj,
+int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBucketInfo& bucket_info, const rgw_obj& src_obj,
                         map<string, bufferlist>& attrs,
                         map<string, bufferlist>* rmattrs,
                         optional_yield y,
@@ -6367,7 +6369,7 @@ int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx, RGWBu
   RGWObjState *state = NULL;
   RGWObjManifest *manifest = nullptr;
 
-  r = append_atomic_test(dpp, rctx, bucket_info, obj, op, &state, &manifest, y);
+  r = append_atomic_test(dpp, octx, bucket_info, obj, op, &state, &manifest, y);
   if (r < 0)
     return r;
 
@@ -9885,8 +9887,8 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp,
 
   RGWObjState *astate = NULL;
   RGWObjManifest *manifest = nullptr;
-  RGWObjectCtx rctx(this->driver);
-  int r = get_obj_state(dpp, &rctx, bucket_info, obj, &astate, &manifest, false, y);
+  RGWObjectCtx octx(this->driver);
+  int r = get_obj_state(dpp, &octx, bucket_info, obj, &astate, &manifest, false, y);
   if (r < 0)
     return r;
 
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 5ca604c971f6..2dca3cfaad4e 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -816,13 +816,12 @@ class RGWRados
 
       explicit Write(RGWRados::Object *_target) : target(_target) {}
 
-      int _do_write_meta(const DoutPrefixProvider *dpp,
-                     uint64_t size, uint64_t accounted_size,
+      int _do_write_meta(uint64_t size, uint64_t accounted_size,
                      std::map<std::string, bufferlist>& attrs,
                      bool modify_tail, bool assume_noent,
-                     void *index_op, optional_yield y);
-      int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size,
-                     std::map<std::string, bufferlist>& attrs, optional_yield y);
+                     void *index_op, const req_context& rctx);
+      int write_meta(uint64_t size, uint64_t accounted_size,
+                     std::map<std::string, bufferlist>& attrs, const req_context& rctx);
       int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive);
       const req_state* get_req_state() {
         return nullptr;  /* XXX dang Only used by LTTng, and it handles null anyway */
@@ -1144,8 +1143,8 @@ class RGWRados
                        std::string *petag,
                        void (*progress_cb)(off_t, void *),
                        void *progress_data,
-                       const DoutPrefixProvider *dpp,
-                       RGWFetchObjFilter *filter, optional_yield y,
+                       const req_context& rctx,
+                       RGWFetchObjFilter *filter,
                        bool stat_follow_olh,
                        const rgw_obj& stat_dest_obj,
                        const rgw_zone_set_entry& source_trace_entry,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 0c24a36a0a58..fb684a921e6d 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2091,7 +2091,8 @@ int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp,
   attrs.erase(RGW_ATTR_ID_TAG);
   attrs.erase(RGW_ATTR_TAIL_TAG);
 
-  return obj_op.write_meta(dpp, 0, 0, attrs, y);
+  const req_context rctx{dpp, y, nullptr};
+  return obj_op.write_meta(0, 0, attrs, rctx);
 }
 
 int RadosObject::get_max_chunk_size(const DoutPrefixProvider* dpp, rgw_placement_rule placement_rule, uint64_t* max_chunk_size, uint64_t* alignment)
@@ -2178,12 +2179,12 @@ std::unique_ptr<Object::ReadOp> RadosObject::get_read_op()
   return std::make_unique<RadosObject::RadosReadOp>(this, rados_ctx);
 }
 
-RadosObject::RadosReadOp::RadosReadOp(RadosObject *_source, RGWObjectCtx *_rctx) :
+RadosObject::RadosReadOp::RadosReadOp(RadosObject *_source, RGWObjectCtx *_octx) :
 	source(_source),
-	rctx(_rctx),
+	octx(_octx),
 	op_target(_source->store->getRados(),
 		  _source->get_bucket()->get_info(),
-		  *static_cast<RGWObjectCtx *>(rctx),
+		  *static_cast<RGWObjectCtx *>(octx),
 		  _source->get_obj()),
 	parent_op(&op_target)
 { }
@@ -2502,6 +2503,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
   int ret;
   std::string oid = mp_obj.get_key();
   RGWObjectCtx obj_ctx(store);
+  const req_context rctx{dpp, y, nullptr};
 
   do {
     char buf[33];
@@ -2537,7 +2539,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     encode(upload_info, bl);
     obj_op.meta.data = &bl;
 
-    ret = obj_op.write_meta(dpp, bl.length(), 0, attrs, y);
+    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx);
   } while (ret == -EEXIST);
 
   return ret;
@@ -2820,7 +2822,8 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   obj_op.meta.completeMultipart = true;
   obj_op.meta.olh_epoch = olh_epoch;
 
-  ret = obj_op.write_meta(dpp, ofs, accounted_size, attrs, y);
+  const req_context rctx{dpp, y, nullptr};
+  ret = obj_op.write_meta(ofs, accounted_size, attrs, rctx);
   if (ret < 0)
     return ret;
 
@@ -3099,10 +3102,10 @@ int RadosAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, y);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
 }
 
 int RadosAppendWriter::prepare(optional_yield y)
@@ -3122,10 +3125,10 @@ int RadosAppendWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, y);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
 }
 
 int RadosMultipartWriter::prepare(optional_yield y)
@@ -3145,10 +3148,10 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, y);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
 }
 
 const std::string& RadosZoneGroup::get_endpoint() const
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 228ba532869c..e86a0870782a 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -324,7 +324,7 @@ class RadosObject : public StoreObject {
     struct RadosReadOp : public ReadOp {
     private:
       RadosObject* source;
-      RGWObjectCtx* rctx;
+      RGWObjectCtx* octx;
       RGWRados::Object op_target;
       RGWRados::Object::Read parent_op;
 
@@ -793,7 +793,7 @@ class RadosAtomicWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 };
 
 class RadosAppendWriter : public StoreWriter {
@@ -840,7 +840,7 @@ class RadosAppendWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 };
 
 class RadosMultipartWriter : public StoreWriter {
@@ -885,7 +885,7 @@ class RadosMultipartWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 };
 
 class RadosLuaManager : public StoreLuaManager {
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 66651da5cc8c..cdc4be83a054 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -417,12 +417,13 @@ int RGWDataAccess::Object::put(bufferlist& data,
     puser_data = &(*user_data);
   }
 
+  const req_context rctx{dpp, y, nullptr};
   return processor->complete(obj_size, etag,
 			    &mtime, mtime,
 			    attrs, delete_at,
                             nullptr, nullptr,
                             puser_data,
-                            nullptr, nullptr, y);
+                            nullptr, nullptr, rctx);
 }
 
 void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy)
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 6a55d3f1d8f5..227c212ac6be 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1937,6 +1937,7 @@ namespace rgw {
     char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
     unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
     req_state* state = get_state();
+    const req_context rctx{this, state->yield, nullptr};
 
     size_t osize = rgw_fh->get_size();
     struct timespec octime = rgw_fh->get_ctime();
@@ -2018,7 +2019,7 @@ namespace rgw {
     op_ret = processor->complete(state->obj_size, etag, &mtime, real_time(), attrs,
                                  (delete_at ? *delete_at : real_time()),
                                 if_match, if_nomatch, nullptr, nullptr, nullptr,
-                                state->yield);
+                                rctx);
     if (op_ret != 0) {
       /* revert attr updates */
       rgw_fh->set_mtime(omtime);
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index ab2dda48ec7b..4887c9d146a6 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -480,7 +480,7 @@ struct lc_op_ctx {
   LCObjsLister& ol;
 
   std::unique_ptr<rgw::sal::Object> obj;
-  RGWObjectCtx rctx;
+  RGWObjectCtx octx;
   const DoutPrefixProvider *dpp;
   WorkQ* wq;
 
@@ -493,7 +493,7 @@ struct lc_op_ctx {
     : cct(env.driver->ctx()), env(env), o(o), next_key_name(next_key_name),
       effective_mtime(effective_mtime),
       driver(env.driver), bucket(env.bucket), op(env.op), ol(env.ol),
-      rctx(env.driver), dpp(dpp), wq(wq)
+      octx(env.driver), dpp(dpp), wq(wq)
     {
       obj = bucket->get_object(o.key);
     }
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8c15e5bd2e3f..88c1a5abd1ab 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4390,10 +4390,11 @@ void RGWPutObj::execute(optional_yield y)
   }
 
   tracepoint(rgw_op, processor_complete_enter, s->req_id.c_str());
+  const req_context rctx{this, s->yield, s->trace.get()};
   op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs,
                                (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
                                (user_data.empty() ? nullptr : &user_data), nullptr, nullptr,
-                               s->yield);
+                               rctx);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
 
   // send request to notification manager
@@ -4658,10 +4659,11 @@ void RGWPostObj::execute(optional_yield y)
       emplace_attr(RGW_ATTR_COMPRESSION, std::move(tmp));
     }
 
+    const req_context rctx{this, s->yield, s->trace.get()};
     op_ret = processor->complete(s->obj_size, etag, nullptr, real_time(), attrs,
                                 (delete_at ? *delete_at : real_time()),
                                 nullptr, nullptr, nullptr, nullptr, nullptr,
-                                s->yield);
+                                rctx);
     if (op_ret < 0) {
       return;
     }
@@ -7772,10 +7774,11 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   }
 
   /* Complete the transaction. */
+  const req_context rctx{this, s->yield, s->trace.get()};
   op_ret = processor->complete(size, etag, nullptr, ceph::real_time(),
                               attrs, ceph::real_time() /* delete_at */,
                               nullptr, nullptr, nullptr, nullptr, nullptr,
-                              s->yield);
+                              rctx);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "processor::complete returned op_ret=" << op_ret << dendl;
   }
diff --git a/src/rgw/rgw_req_context.h b/src/rgw/rgw_req_context.h
new file mode 100644
index 000000000000..b0030ca1a94b
--- /dev/null
+++ b/src/rgw/rgw_req_context.h
@@ -0,0 +1,18 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "common/async/yield_context.h"
+namespace opentelemetry::trace {
+  class Span;
+}
+
+class DoutPrefixProvider;
+
+// this struct holds information which is created at the frontend
+// and should trickle down through all function calls to the backend
+struct req_context {
+  const DoutPrefixProvider* dpp{nullptr};
+  optional_yield y;
+  const opentelemetry::trace::Span* span{nullptr};
+};
+
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 84731f333d72..7cfd4923761f 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -20,6 +20,7 @@
 #include "rgw_user.h"
 #include "rgw_notify_event_type.h"
 #include "common/tracer.h"
+#include "rgw_req_context.h"
 #include "rgw_datalog_notify.h"
 #include "include/random.h"
 
@@ -244,7 +245,7 @@ class ObjectProcessor : public DataProcessor {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) = 0;
+                       const req_context& rctx) = 0;
 };
 
 /** A list of key-value attributes */
@@ -1387,7 +1388,7 @@ class Writer : public ObjectProcessor {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) = 0;
+                       const req_context& rctx) = 0;
 };
 
 
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index aa1243fe5982..36d76cc12e15 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -738,9 +738,9 @@ namespace rgw::sal {
     return std::make_unique<DBObject::DBReadOp>(this, nullptr);
   }
 
-  DBObject::DBReadOp::DBReadOp(DBObject *_source, RGWObjectCtx *_rctx) :
+  DBObject::DBReadOp::DBReadOp(DBObject *_source, RGWObjectCtx *_octx) :
     source(_source),
-    rctx(_rctx),
+    octx(_octx),
     op_target(_source->store->getDB(),
         _source->get_bucket()->get_info(),
         _source->get_obj()),
@@ -1323,7 +1323,7 @@ namespace rgw::sal {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
   {
     /* XXX: same as AtomicWriter..consolidate code */
     parent_op.meta.mtime = mtime;
@@ -1477,7 +1477,7 @@ namespace rgw::sal {
                          const char *if_match, const char *if_nomatch,
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
-                         optional_yield y)
+                         const req_context& rctx)
   {
     parent_op.meta.mtime = mtime;
     parent_op.meta.delete_at = delete_at;
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 65ffd9091093..0c75f4b98cbb 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -524,12 +524,12 @@ class DBNotification : public StoreNotification {
       struct DBReadOp : public ReadOp {
         private:
           DBObject* source;
-          RGWObjectCtx* rctx;
+          RGWObjectCtx* octx;
           DB::Object op_target;
           DB::Object::Read parent_op;
 
         public:
-          DBReadOp(DBObject *_source, RGWObjectCtx *_rctx);
+          DBReadOp(DBObject *_source, RGWObjectCtx *_octx);
 
           virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) override;
 
@@ -688,7 +688,7 @@ class DBNotification : public StoreNotification {
                          const char *if_match, const char *if_nomatch,
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
-                         optional_yield y) override;
+                         const req_context& rctx) override;
   };
 
   class DBMultipartWriter : public StoreWriter {
@@ -736,7 +736,7 @@ class DBNotification : public StoreNotification {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
   };
 
   class DBStore : public StoreDriver {
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 13e9155c524b..dbf688a22ab0 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -1276,11 +1276,11 @@ int FilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       const req_context& rctx)
 {
   return next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, y);
+			canceled, rctx);
 }
 
 int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 6db44a191003..dcc03df95194 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -879,7 +879,7 @@ class FilterWriter : public Writer {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       const req_context& rctx) override;
 };
 
 class FilterLuaManager : public LuaManager {
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index 30a508cf7097..7ceb7092c64f 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -25,6 +25,7 @@ string redisHost = "";
 vector<const char*> args;
 class Environment* env;
 const DoutPrefixProvider* dpp;
+const req_context rctx{dpp, null_yield, nullptr};
 
 class StoreObject : public rgw::sal::StoreObject {
   friend class D4NFilterFixture;
@@ -194,7 +195,7 @@ class D4NFilterFixture : public ::testing::Test {
                        &if_match, &if_nomatch,
                        &user_data,
                        &zones_trace, &canceled,
-                       null_yield);
+                       rctx);
 
       return ret;
     }
@@ -454,7 +455,7 @@ TEST_F(D4NFilterFixture, CopyObjectReplace) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   null_yield), 0);
+		   rctx), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -579,7 +580,7 @@ TEST_F(D4NFilterFixture, CopyObjectMerge) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   null_yield), 0);
+		   rctx), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -1913,7 +1914,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 null_yield), 0);
+		 rctx), 0);
  
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&data](cpp_redis::reply& reply) {
     if (reply.is_string()) {
@@ -1938,7 +1939,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 null_yield), 0);
+		 rctx), 0);
 
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&dataNew](cpp_redis::reply& reply) {
     if (reply.is_string()) {

From 8532f596e67f82016c1247fcd67385cd7890ee02 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 10 Oct 2023 17:30:54 +0000
Subject: [PATCH 0151/2492] tracing/osd/rgw: using jspan as alias to
 opentelemetry::Span

and jspan_ptr as jspan shared pointer

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/common/tracer.cc      | 10 ++++-----
 src/common/tracer.h       | 43 ++++++++++++++++++---------------------
 src/osd/OpRequest.h       |  2 +-
 src/rgw/rgw_common.h      |  4 ++--
 src/rgw/rgw_op.h          |  8 ++++----
 src/rgw/rgw_req_context.h |  6 ++----
 src/rgw/rgw_sal.h         |  2 +-
 7 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/src/common/tracer.cc b/src/common/tracer.cc
index 1146da319500..e98053735b48 100644
--- a/src/common/tracer.cc
+++ b/src/common/tracer.cc
@@ -17,7 +17,7 @@
 namespace tracing {
 
 const opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> Tracer::noop_tracer = opentelemetry::trace::Provider::GetTracerProvider()->GetTracer("no-op", OPENTELEMETRY_SDK_VERSION);
-const jspan Tracer::noop_span = noop_tracer->StartSpan("noop");
+const jspan_ptr Tracer::noop_span = noop_tracer->StartSpan("noop");
 
 using bufferlist = ceph::buffer::list;
 
@@ -38,7 +38,7 @@ void Tracer::init(CephContext* _cct, opentelemetry::nostd::string_view service_n
   }
 }
 
-jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name) {
+jspan_ptr Tracer::start_trace(opentelemetry::nostd::string_view trace_name) {
   ceph_assert(cct);
   if (is_enabled()) {
     ceph_assert(tracer);
@@ -48,7 +48,7 @@ jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name) {
   return noop_span;
 }
 
-jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool trace_is_enabled) {
+jspan_ptr Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool trace_is_enabled) {
   ceph_assert(cct);
   ldout(cct, 20) << "start trace enabled " << trace_is_enabled << " " << dendl;
   if (trace_is_enabled) {
@@ -59,7 +59,7 @@ jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool tra
   return noop_tracer->StartSpan(trace_name);
 }
 
-jspan Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan& parent_span) {
+jspan_ptr Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan_ptr& parent_span) {
   if (parent_span && parent_span->IsRecording()) {
     ceph_assert(tracer);
     opentelemetry::trace::StartSpanOptions span_opts;
@@ -70,7 +70,7 @@ jspan Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan&
   return noop_span;
 }
 
-jspan Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan_context& parent_ctx) {
+jspan_ptr Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan_context& parent_ctx) {
   if (parent_ctx.IsValid()) {
     ceph_assert(tracer);
     opentelemetry::trace::StartSpanOptions span_opts;
diff --git a/src/common/tracer.h b/src/common/tracer.h
index 94efedbed6e1..291ff9de25a7 100644
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -9,7 +9,8 @@
 #ifdef HAVE_JAEGER
 #include "opentelemetry/trace/provider.h"
 
-using jspan = opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>;
+using jspan = opentelemetry::trace::Span;
+using jspan_ptr = opentelemetry::nostd::shared_ptr<jspan>;
 using jspan_context = opentelemetry::trace::SpanContext;
 using jspan_attribute = opentelemetry::common::AttributeValue;
 
@@ -18,7 +19,7 @@ namespace tracing {
 class Tracer {
  private:
   const static opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> noop_tracer;
-  const static jspan noop_span;
+  const static jspan_ptr noop_span;
   CephContext* cct = nullptr;;
   opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> tracer;
 
@@ -30,18 +31,18 @@ class Tracer {
   bool is_enabled() const;
   // creates and returns a new span with `trace_name`
   // this span represents a trace, since it has no parent.
-  jspan start_trace(opentelemetry::nostd::string_view trace_name);
+  jspan_ptr start_trace(opentelemetry::nostd::string_view trace_name);
 
   // creates and returns a new span with `trace_name`
   // if false is given to `trace_is_enabled` param, noop span will be returned
-  jspan start_trace(opentelemetry::nostd::string_view trace_name, bool trace_is_enabled);
+  jspan_ptr start_trace(opentelemetry::nostd::string_view trace_name, bool trace_is_enabled);
 
   // creates and returns a new span with `span_name` which parent span is `parent_span'
-  jspan add_span(opentelemetry::nostd::string_view span_name, const jspan& parent_span);
+  jspan_ptr add_span(opentelemetry::nostd::string_view span_name, const jspan_ptr& parent_span);
   // creates and return a new span with `span_name`
   // the span is added to the trace which it's context is `parent_ctx`.
   // parent_ctx contains the required information of the trace.
-  jspan add_span(opentelemetry::nostd::string_view span_name, const jspan_context& parent_ctx);
+  jspan_ptr add_span(opentelemetry::nostd::string_view span_name, const jspan_context& parent_ctx);
 
 };
 
@@ -67,9 +68,9 @@ struct jspan_context {
   jspan_context(bool sampled_flag, bool is_remote) {}
 };
 
-namespace opentelemetry::trace {
-struct Span {
+class jspan {
   jspan_context _ctx;
+public:
   template <typename T>
   void SetAttribute(std::string_view key, const T& value) const noexcept {}
   void AddEvent(std::string_view) {}
@@ -79,21 +80,17 @@ struct Span {
   void UpdateName(std::string_view) {}
   bool IsRecording() { return false; }
 };
-}
 
-class jspan {
-  opentelemetry::trace::Span span;
+class jspan_ptr {
+  jspan span;
 public:
-  opentelemetry::trace::Span& operator*() { return span; }
-  const opentelemetry::trace::Span& operator*() const { return span; }
-
-  opentelemetry::trace::Span* operator->() { return &span; }
-  const opentelemetry::trace::Span* operator->() const { return &span; }
-
+  jspan& operator*() { return span; }
+  const jspan& operator*() const { return span; }
+  jspan* operator->() { return &span; }
+  const jspan* operator->() const { return &span; }
   operator bool() const { return false; }
-
-  opentelemetry::trace::Span* get() { return &span; }
-  const opentelemetry::trace::Span* get() const { return &span; }
+  jspan* get() { return &span; }
+  const jspan* get() const { return &span; }
 };
 
 namespace tracing {
@@ -101,9 +98,9 @@ namespace tracing {
 struct Tracer {
   void init(CephContext* _cct, std::string_view service_name) {}
   bool is_enabled() const { return false; }
-  jspan start_trace(std::string_view, bool enabled = true) { return {}; }
-  jspan add_span(std::string_view, const jspan&) { return {}; }
-  jspan add_span(std::string_view span_name, const jspan_context& parent_ctx) { return {}; }
+  jspan_ptr start_trace(std::string_view, bool enabled = true) { return {}; }
+  jspan_ptr add_span(std::string_view, const jspan_ptr&) { return {}; }
+  jspan_ptr add_span(std::string_view span_name, const jspan_context& parent_ctx) { return {}; }
 };
 
 inline void encode(const jspan_context& span, bufferlist& bl, uint64_t f=0) {}
diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h
index 8df4f25d53e5..23f3b1d932e9 100644
--- a/src/osd/OpRequest.h
+++ b/src/osd/OpRequest.h
@@ -94,7 +94,7 @@ struct OpRequest : public TrackedOp {
   epoch_t min_epoch = 0;      ///< min epoch needed to handle this msg
 
   bool hitset_inserted;
-  jspan osd_parent_span;
+  jspan_ptr osd_parent_span;
 
   template<class T>
   const T* get_req() const { return static_cast<const T*>(request); }
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index a71f1368070e..1094d0cd443e 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -25,6 +25,7 @@
 
 #include "common/ceph_crypto.h"
 #include "common/random_string.h"
+#include "common/tracer.h"
 #include "rgw_acl.h"
 #include "rgw_bucket_layout.h"
 #include "rgw_cors.h"
@@ -43,7 +44,6 @@
 #include "cls/rgw/cls_rgw_types.h"
 #include "include/rados/librados.hpp"
 #include "rgw_public_access.h"
-#include "common/tracer.h"
 #include "rgw_sal_fwd.h"
 
 namespace ceph {
@@ -1232,7 +1232,7 @@ struct req_state : DoutPrefixProvider {
 
   std::vector<rgw::IAM::Policy> session_policies;
 
-  jspan trace;
+  jspan_ptr trace;
   bool trace_enabled = false;
 
   //Principal tags that come in as part of AssumeRoleWithWebIdentity
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index b5ce737147f1..ffc65836ce62 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1220,7 +1220,7 @@ class RGWPutObj : public RGWOp {
   std::string multipart_upload_id;
   std::string multipart_part_str;
   int multipart_part_num = 0;
-  jspan multipart_trace;
+  jspan_ptr multipart_trace;
 
   boost::optional<ceph::real_time> delete_at;
   //append obj
@@ -1847,7 +1847,7 @@ class RGWInitMultipart : public RGWOp {
   std::string upload_id;
   RGWAccessControlPolicy policy;
   ceph::real_time mtime;
-  jspan multipart_trace;
+  jspan_ptr multipart_trace;
 
 public:
   RGWInitMultipart() {}
@@ -1875,7 +1875,7 @@ class RGWCompleteMultipart : public RGWOp {
   std::string version_id;
   bufferlist data;
   std::unique_ptr<rgw::sal::MPSerializer> serializer;
-  jspan multipart_trace;
+  jspan_ptr multipart_trace;
 
 public:
   RGWCompleteMultipart() {}
@@ -1896,7 +1896,7 @@ class RGWCompleteMultipart : public RGWOp {
 
 class RGWAbortMultipart : public RGWOp {
 protected:
-  jspan multipart_trace;
+  jspan_ptr multipart_trace;
 public:
   RGWAbortMultipart() {}
 
diff --git a/src/rgw/rgw_req_context.h b/src/rgw/rgw_req_context.h
index b0030ca1a94b..b8c284187c87 100644
--- a/src/rgw/rgw_req_context.h
+++ b/src/rgw/rgw_req_context.h
@@ -1,10 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#pragma once
 #include "common/async/yield_context.h"
-namespace opentelemetry::trace {
-  class Span;
-}
 
 class DoutPrefixProvider;
 
@@ -13,6 +11,6 @@ class DoutPrefixProvider;
 struct req_context {
   const DoutPrefixProvider* dpp{nullptr};
   optional_yield y;
-  const opentelemetry::trace::Span* span{nullptr};
+  const jspan* span;
 };
 
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 7cfd4923761f..e7a3dfe0aa52 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -15,11 +15,11 @@
 
 #pragma once
 
+#include "common/tracer.h"
 #include "rgw_sal_fwd.h"
 #include "rgw_lua.h"
 #include "rgw_user.h"
 #include "rgw_notify_event_type.h"
-#include "common/tracer.h"
 #include "rgw_req_context.h"
 #include "rgw_datalog_notify.h"
 #include "include/random.h"

From edec8f35abc2d7bb6aef68ea9b841e33e12012dd Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 10 Oct 2023 16:09:20 +0530
Subject: [PATCH 0152/2492] qa: minor improvement in ceph_test_case.py

When two values (say x and y) are being printed because assert for
equality of both failed (assert x == y), print both the values on a new
line.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/ceph_test_case.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py
index 649c0e53cf96..8926e5c7e1a5 100644
--- a/qa/tasks/ceph_test_case.py
+++ b/qa/tasks/ceph_test_case.py
@@ -76,8 +76,8 @@ def _verify(self, proc, exp_retval=None, exp_errmsgs=None):
 
         proc_stderr = proc.stderr.getvalue().lower()
         msg = ('didn\'t find any of the expected string in stderr.\n'
-               f'expected string: {exp_errmsgs}\n'
-               f'received error message: {proc_stderr}\n'
+               f'expected string -\n{exp_errmsgs}\n'
+               f'received error message -\n{proc_stderr}\n'
                'note: received error message is converted to lowercase')
         for e in exp_errmsgs:
             if e in proc_stderr:

From bd4cb58f7abca4fa20ec283f675f31d1ede14752 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 10 Oct 2023 16:12:13 +0530
Subject: [PATCH 0153/2492] qa: log stdout for commands being run through
 negtest_ceph_cmd

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/ceph_test_case.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py
index 8926e5c7e1a5..877f79d06a3b 100644
--- a/qa/tasks/ceph_test_case.py
+++ b/qa/tasks/ceph_test_case.py
@@ -105,6 +105,8 @@ def negtest_ceph_cmd(self, args, retval=None, errmsgs=None, **kwargs):
         # execution is needed to not halt on command failure because we are
         # conducting negative testing
         kwargs['check_status'] = False
+        # log stdout since it may contain something useful when command fails
+        kwargs['stdout'] = StringIO()
         # stderr is needed to check for expected error messages.
         kwargs['stderr'] = StringIO()
 

From 03df86b7c54325fd577bacec6a1f4c6441117e72 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 10 Oct 2023 16:30:00 +0530
Subject: [PATCH 0154/2492] qa/cephfs: log commands on INFO logging level

Commands issued by negtest_ceph_cmd() aren't printed because log level
(due to code for teuthology) changes from DEBUG to INFO in case of some
files.

This patch ensures that users can see commands being executed regardless
of whether log level is changed or not.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/vstart_runner.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index caf7a7fe3f79..96dc9fffba2f 100644
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -436,7 +436,13 @@ def _perform_checks_and_adjustments(self, args, omit_sudo):
 
         usr_args, args = self._omit_cmd_args(args, omit_sudo)
 
-        log.debug('> ' + usr_args)
+        # Let's print all commands on INFO log level since some logging level
+        # might be changed to INFO from DEBUG during a vstart_runner.py's
+        # execution due to code added for teuthology. This happened for
+        # ceph_test_case.RunCephCmd.negtest_ceph_cmd(). Commands it executes
+        # weren't printed in output because logging level for
+        # ceph_test_case.py is set to INFO by default.
+        log.info('> ' + usr_args)
 
         return args, usr_args
 

From afd899d29fbd86caf915db8c59be1fb77f9ebb2d Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 9 Oct 2023 22:13:59 +0300
Subject: [PATCH 0155/2492] doc/operations/monitoring: mention the new asok
 command 'raise'

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 doc/rados/operations/monitoring.rst | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/doc/rados/operations/monitoring.rst b/doc/rados/operations/monitoring.rst
index a9171f2d8419..2343e7e1f59c 100644
--- a/doc/rados/operations/monitoring.rst
+++ b/doc/rados/operations/monitoring.rst
@@ -626,6 +626,21 @@ For example, the following commands are equivalent to each other:
    ceph daemon osd.0 foo
    ceph daemon /var/run/ceph/ceph-osd.0.asok foo
 
+There are two methods of running admin socket commands: (1)
+using ``ceph daemon`` as described above, which bypasses
+the monitor and assumes a direct login to the daemon's host,
+and (2) using the ``ceph tell {daemon-type}.{id}`` command,
+which is relayed by monitors and does not require access
+to the daemon's host.
+
+Use the ``raise`` command to send a signal to a daemon, as if by running ``kill -X {daemon.pid}``.
+When run via ``ceph tell`` it allows signalling a daemon without access to its host:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name} raise HUP
+   ceph tell {daemon-type}.{id} raise -9
+
 To view the available admin-socket commands, run the following command:
 
 .. prompt:: bash $
@@ -634,11 +649,7 @@ To view the available admin-socket commands, run the following command:
 
 Admin-socket commands enable you to view and set your configuration at runtime.
 For more on viewing your configuration, see `Viewing a Configuration at
-Runtime`_. There are two methods of setting configuration value at runtime: (1)
-using the admin socket, which bypasses the monitor and requires a direct login
-to the host in question, and (2) using the ``ceph tell {daemon-type}.{id}
-config set`` command, which relies on the monitor and does not require a direct
-login.
+Runtime`_.
 
 .. _Viewing a Configuration at Runtime: ../../configuration/ceph-conf#viewing-a-configuration-at-runtime
 .. _Storage Capacity: ../../configuration/mon-config-ref#storage-capacity

From 7cc54b2b7323096bf7394d8502ff232a78e0c355 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 27 Sep 2023 00:54:26 +0300
Subject: [PATCH 0156/2492] common/admin_socket: add a command to raise a
 signal

The new command "raise <signal> [--after X]" accepts signals
in the forms: '9', '-9', 'kill', '-KILL'
When --after is specified, the program will fork to wait for the timeout

The forked instance will bail out if it detects that the parent PID has changed
which would indicate that the original parent has terminated.

Forking an instance allows to schedule delivery of signals even if the original
process is suspended, e.g.:

  ceph tell mds.a raise CONT --after 10
  ceph tell mds.a raise STOP

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
Fixes: https://tracker.ceph.com/issues/62882
---
 src/common/admin_socket.cc | 302 +++++++++++++++++++++++++++++++++++++
 src/common/admin_socket.h  |   1 +
 src/test/admin_socket.cc   | 214 ++++++++++++++++++++++++++
 3 files changed, 517 insertions(+)

diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index 8a7e0c721971..2ed3179e8070 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -13,6 +13,7 @@
  */
 #include <poll.h>
 #include <sys/un.h>
+#include <optional>
 
 #include "common/admin_socket.h"
 #include "common/admin_socket_client.h"
@@ -36,6 +37,7 @@
 #include "include/ceph_assert.h"
 #include "include/compat.h"
 #include "include/sock_compat.h"
+#include "fmt/format.h"
 
 #define dout_subsys ceph_subsys_asok
 #undef dout_prefix
@@ -693,6 +695,297 @@ class GetdescsHook : public AdminSocketHook {
   }
 };
 
+// Define a macro to simplify adding signals to the map
+#define ADD_SIGNAL(signalName)                 \
+  {                                            \
+    ((const char*)#signalName) + 3, signalName \
+  }
+
+static const std::map<std::string, int> known_signals = {
+  // the following 6 signals are recognized in windows according to
+  // https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/raise?view=msvc-170
+  ADD_SIGNAL(SIGABRT),
+  ADD_SIGNAL(SIGFPE),
+  ADD_SIGNAL(SIGILL),
+  ADD_SIGNAL(SIGINT),
+  ADD_SIGNAL(SIGSEGV),
+  ADD_SIGNAL(SIGTERM),
+#ifndef WIN32
+  ADD_SIGNAL(SIGTRAP),
+  ADD_SIGNAL(SIGHUP),
+  ADD_SIGNAL(SIGBUS),
+  ADD_SIGNAL(SIGQUIT),
+  ADD_SIGNAL(SIGKILL),
+  ADD_SIGNAL(SIGUSR1),
+  ADD_SIGNAL(SIGUSR2),
+  ADD_SIGNAL(SIGPIPE),
+  ADD_SIGNAL(SIGALRM),
+  ADD_SIGNAL(SIGCHLD),
+  ADD_SIGNAL(SIGCONT),
+  ADD_SIGNAL(SIGSTOP),
+  ADD_SIGNAL(SIGTSTP),
+  ADD_SIGNAL(SIGTTIN),
+  ADD_SIGNAL(SIGTTOU),
+#endif
+  // Add more signals as needed...
+};
+
+#undef ADD_SIGNAL
+
+static std::string strsignal_compat(int signal) {
+#ifndef WIN32
+  return strsignal(signal);
+#else
+  switch (signal) {
+    case SIGABRT: return "SIGABRT";
+    case SIGFPE: return "SIGFPE";
+    case SIGILL: return "SIGILL";
+    case SIGINT: return "SIGINT";
+    case SIGSEGV: return "SIGSEGV";
+    case SIGTERM: return "SIGTERM";
+    default: return fmt::format("Signal #{}", signal);
+  }
+#endif
+}
+
+class RaiseHook: public AdminSocketHook {
+  struct Killer {
+    CephContext* m_cct;
+    pid_t pid;
+    int signal;
+    ceph::coarse_mono_clock::time_point due;
+
+    std::string describe()
+    {
+      using std::chrono::duration_cast;
+      using std::chrono::seconds;
+      auto remaining = (due - coarse_mono_clock::now());
+      return fmt::format(
+        "pending signal ({}) due in {}", 
+        strsignal_compat(signal),
+        duration_cast<seconds>(remaining));
+    }
+
+    bool cancel()
+    {
+#   ifndef WIN32
+      int wstatus;
+      int status;
+      if (0 == (status = waitpid(pid, &wstatus, WNOHANG))) {
+        status = kill(pid, SIGKILL);
+        if (status) {
+          ldout(m_cct, 5) << __func__ << "couldn't kill the killer. Error: " << strerror(errno) << dendl;
+          return false;
+        }
+        while (pid == waitpid(pid, &wstatus, 0)) {
+          if (WIFEXITED(wstatus)) {
+            return false;
+          }
+          if (WIFSIGNALED(wstatus)) {
+            return true;
+          }
+        }
+      }
+      if (status < 0) {
+        ldout(m_cct, 5) << __func__ << "waitpid(killer, NOHANG) returned " << status << "; " << strerror(errno) << dendl;
+      } else {
+        ldout(m_cct, 20) << __func__ << "killer process " << pid << "\"" << describe() << "\" reaped. "
+                         << "WIFEXITED: " << WIFEXITED(wstatus)
+                         << "WIFSIGNALED: " << WIFSIGNALED(wstatus)
+                         << dendl;
+      }
+#   endif
+      return false;
+    }
+
+    static std::optional<Killer> fork(CephContext *m_cct, int signal_to_send, double delay) {
+#   ifndef WIN32
+      pid_t victim = getpid();
+      auto until = ceph::coarse_mono_clock::now() + ceph::make_timespan(delay);
+
+      int fresult = ::fork();
+      if (fresult < 0) {
+        ldout(m_cct, 5) << __func__ << "couldn't fork the killer. Error: " << strerror(errno) << dendl;
+        return std::nullopt;
+      }
+
+      if (fresult) {
+        // this is parent
+        return {{m_cct, fresult, signal_to_send, until}};
+      }
+
+      const auto poll_interval = ceph::make_timespan(0.1);
+      auto remaining = (until - ceph::coarse_mono_clock::now());
+      do {
+        using std::chrono::duration_cast;
+        using std::chrono::nanoseconds;
+        std::this_thread::sleep_for(duration_cast<nanoseconds>(std::min(remaining, poll_interval)));
+        if (getppid() != victim) {
+          // suicide if my parent has changed
+          // this means that the original parent process has terminated
+          _exit(1);
+        }
+        remaining = (until - ceph::coarse_mono_clock::now());
+      } while (remaining > ceph::signedspan::zero());
+
+      int status = kill(victim, signal_to_send);
+      if (0 != status) {
+        ldout(m_cct, 5) << __func__ << "couldn't kill the victim: " << strerror(errno) << dendl;
+      }
+      _exit(status);
+#   endif
+      return std::nullopt;
+    }
+  };
+
+  CephContext* m_cct;
+  std::optional<Killer> killer;
+
+  int parse_signal(std::string&& sigdesc, Formatter* f, std::ostream& errss)
+  {
+    int result = 0;
+    std::transform(sigdesc.begin(), sigdesc.end(), sigdesc.begin(),
+        [](unsigned char c) { return std::toupper(c); });
+    if (sigdesc.starts_with("-")) {
+      sigdesc.erase(0, 1);
+    }
+    if (sigdesc.starts_with("SIG")) {
+      sigdesc.erase(0, 3);
+    }
+
+    if (sigdesc == "L") {
+      f->open_object_section("known_signals");
+      for (auto& [name, num] : known_signals) {
+        f->dump_int(name, num);
+      }
+      f->close_section();
+    } else {
+      try {
+        result = std::stoi(sigdesc);
+        if (result < 1 || result > 64) {
+          errss << "signal number should be an integer in the range [1..64]" << std::endl;
+          return -EINVAL;
+        }
+      } catch (std::invalid_argument) {
+        auto sig_it = known_signals.find(sigdesc);
+        if (sig_it == known_signals.end()) {
+          errss << "unknown signal name; use -l to see recognized names" << std::endl;
+          return -EINVAL;
+        }
+        result = sig_it->second;
+      }
+    }
+    return result;
+  }
+
+public:
+  RaiseHook(CephContext* cct) : m_cct(cct) { }
+  static const char* get_cmddesc()
+  {
+    return "raise "
+           "name=signal,type=CephString,req=false "
+           "name=cancel,type=CephBool,req=false "
+           "name=after,type=CephFloat,range=0.0,req=false ";
+  }
+
+  static const char* get_help()
+  {
+    return "deliver the <signal> to the daemon process, optionally delaying <after> seconds; "
+           "when --after is used, the program will fork before sleeping, which allows to "
+           "schedule signal delivery to a stopped daemon; it's possible to --cancel a pending signal delivery. "
+           "<signal> can be in the forms '9', '-9', 'kill', '-KILL'. Use `raise -l` to list known signal names.";
+  }
+
+  int call(std::string_view command, const cmdmap_t& cmdmap,
+      const bufferlist&,
+      Formatter* f,
+      std::ostream& errss,
+      bufferlist& out) override
+  {
+    using std::endl;
+    string sigdesc;
+    bool cancel = cmd_getval_or<bool>(cmdmap, "cancel", false);
+    int signal_to_send = 0;
+
+    if (cmd_getval(cmdmap, "signal", sigdesc)) {
+      signal_to_send = parse_signal(std::move(sigdesc), f, errss);
+      if (signal_to_send < 0) {
+        return signal_to_send;
+      }
+    } else if (!cancel) {
+      errss << "signal name or number is required" << endl;
+      return -EINVAL;
+    }
+
+    if (cancel) {
+      if (killer) {
+        if (signal_to_send == 0 || signal_to_send == killer->signal) {
+          if (killer->cancel()) {
+            errss << "cancelled " << killer->describe() << endl;
+            return 0;
+          }
+          killer = std::nullopt;
+        }
+        if (signal_to_send) {
+          errss << "signal " << signal_to_send << " is not pending" << endl;
+        }
+      } else {
+        errss << "no pending signal" << endl;
+      }
+      return 1;
+    }
+
+    if (!signal_to_send) {
+      return 0;
+    }
+
+    double delay = 0;
+    if (cmd_getval(cmdmap, "after", delay)) {
+      #ifdef WIN32
+        errss << "'--after' functionality is unsupported on Windows" << endl;
+        return -ENOTSUP;
+      #endif
+      if (killer) {
+        if (killer->cancel()) {
+          errss << "cancelled " << killer->describe() << endl;
+        }
+      }
+
+      killer = Killer::fork(m_cct, signal_to_send, delay);
+
+      if (killer) {
+        errss << "scheduled " << killer->describe() << endl;
+        ldout(m_cct, 20) << __func__ << "scheduled " << killer->describe() << dendl;
+      } else {
+        errss << "couldn't fork the killer" << std::endl;
+        return -EAGAIN;
+      }
+    } else {
+      ldout(m_cct, 20) << __func__ << "raising "
+                      << " (" << strsignal_compat(signal_to_send) << ")" << dendl;
+      // raise the signal immediately
+      int status = raise(signal_to_send);
+
+      if (0 == status) {
+        errss << "raised signal "
+              << " (" << strsignal_compat(signal_to_send) << ")" << endl;
+      } else {
+        errss << "couldn't raise signal "
+              << " (" << strsignal_compat(signal_to_send) << ")."
+              << " Error: " << strerror(errno) << endl;
+
+        ldout(m_cct, 5) << __func__ << "couldn't raise signal "
+                << " (" << strsignal_compat(signal_to_send) << ")."
+                << " Error: " << strerror(errno) << dendl;
+
+        return 1;
+      }
+    }
+
+    return 0;
+  }
+};
+
 bool AdminSocket::init(const std::string& path)
 {
   ldout(m_cct, 5) << "init " << path << dendl;
@@ -745,6 +1038,12 @@ bool AdminSocket::init(const std::string& path)
   register_command("get_command_descriptions",
 		   getdescs_hook.get(), "list available commands");
 
+  raise_hook = std::make_unique<RaiseHook>(m_cct);
+  register_command(
+      RaiseHook::get_cmddesc(),
+      raise_hook.get(),
+      RaiseHook::get_help());
+
   th = make_named_thread("admin_socket", &AdminSocket::entry, this);
   add_cleanup_file(m_path.c_str());
   return true;
@@ -777,6 +1076,9 @@ void AdminSocket::shutdown()
   unregister_commands(getdescs_hook.get());
   getdescs_hook.reset();
 
+  unregister_commands(raise_hook.get());
+  raise_hook.reset();
+
   remove_cleanup_file(m_path);
   m_path.clear();
 }
diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h
index 3f364a5b711c..b95a52af7beb 100644
--- a/src/common/admin_socket.h
+++ b/src/common/admin_socket.h
@@ -190,6 +190,7 @@ class AdminSocket
   std::unique_ptr<AdminSocketHook> version_hook;
   std::unique_ptr<AdminSocketHook> help_hook;
   std::unique_ptr<AdminSocketHook> getdescs_hook;
+  std::unique_ptr<AdminSocketHook> raise_hook;
 
   std::mutex tell_lock;
   std::list<ceph::cref_t<MCommand>> tell_queue;
diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc
index 369e7abbf9b6..a8236271652c 100644
--- a/src/test/admin_socket.cc
+++ b/src/test/admin_socket.cc
@@ -17,12 +17,14 @@
 #include "common/admin_socket.h"
 #include "common/admin_socket_client.h"
 #include "common/ceph_argparse.h"
+#include "json_spirit/json_spirit.h"
 #include "gtest/gtest.h"
 
 #include <stdint.h>
 #include <string.h>
 #include <string>
 #include <sys/un.h>
+#include <signal.h>
 
 using namespace std;
 
@@ -328,6 +330,218 @@ TEST(AdminSocket, bind_and_listen) {
   }
 }
 
+class AdminSocketRaise: public ::testing::Test 
+{
+public:
+  struct TestSignal {
+    int sig;
+    const char * name;
+    std::atomic<int> count;
+  };
+
+  static void SetUpTestSuite() {
+    signal(sig1.sig, sighandler);
+    signal(sig2.sig, sighandler);
+  }
+  static void TearDownTestSuite()
+  {
+    signal(sig1.sig, SIG_DFL);
+    signal(sig2.sig, SIG_DFL);
+  }
+  void SetUp() override
+  {
+    std::string path = get_rand_socket_path();
+    asock = std::make_unique<AdminSocket>(g_ceph_context);
+    asock_client = std::make_unique<AdminSocketClient>(path);
+    ASSERT_TRUE(asock->init(path));
+    sig1.count = 0;
+    sig2.count = 0;
+  }
+  void TearDown() override
+  {
+    AdminSocketTest(asock.get()).shutdown();
+  }
+protected:
+  static TestSignal sig1;
+  static TestSignal sig2;
+
+  std::unique_ptr<AdminSocket> asock;
+  std::unique_ptr<AdminSocketClient> asock_client;
+
+  static void sighandler(int signal)
+  {
+    if (signal == sig1.sig) {
+      sig1.count++;
+    } else if (signal == sig2.sig) {
+      sig2.count++;
+    }
+
+    // Windows resets the handler upon signal delivery
+    // as apparently some linuxes do as well.
+    // The below shouldn't hurt in any case.
+    ::signal(signal, sighandler);
+  }
+  std::string send_raise(std::optional<std::string> arg, std::optional<double> after, bool cancel)
+  {
+    JSONFormatter f;
+    f.open_object_section("");
+    f.dump_string("prefix", "raise");
+    if (arg) {
+      f.dump_string("signal", *arg);
+    }
+    if (after) {
+      f.dump_float("after", *after);
+    }
+    if (cancel) {
+      f.dump_bool("cancel", true);
+    }
+    f.close_section();
+
+    bufferlist command;
+    f.flush(command);
+
+    std::string response;
+
+    asock_client->do_request(command.to_str(), &response);
+    return response;
+  }
+
+  std::string send_raise_cancel(std::optional<std::string> arg = std::nullopt) {
+    return send_raise(arg, std::nullopt, true);
+  }
+
+  std::string send_raise(std::string arg, std::optional<double> after = std::nullopt) {
+    return send_raise(arg, after, false);
+  }
+};
+
+AdminSocketRaise::TestSignal AdminSocketRaise::sig1 = { SIGINT, "INT", 0 };
+AdminSocketRaise::TestSignal AdminSocketRaise::sig2 = { SIGTERM, "TERM", 0 };
+
+TEST_F(AdminSocketRaise, List) {
+  auto r = send_raise("-l");
+  json_spirit::mValue v;
+  ASSERT_TRUE(json_spirit::read(r, v));
+  ASSERT_EQ(json_spirit::Value_type::obj_type, v.type());
+  EXPECT_EQ(sig1.sig, v.get_obj()[sig1.name].get_int());
+  EXPECT_EQ(sig2.sig, v.get_obj()[sig2.name].get_int());
+}
+
+TEST_F(AdminSocketRaise, ImmediateFormats) {
+  std::string name1, name2;
+
+  name1 = sig1.name;
+  std::transform(name1.begin(), name1.end(), name1.begin(), [](int c) { return std::tolower(c); });
+  name2 = fmt::format("-{}", sig2.name);
+  std::transform(name2.begin(), name2.end(), name2.begin(), [](int c) { return std::tolower(c); });
+
+  send_raise(fmt::format("-{}", sig1.sig));
+  send_raise(name1);
+  send_raise(name2);
+  send_raise(fmt::format("{}", sig2.sig));
+  EXPECT_EQ(2, sig1.count.load());
+  EXPECT_EQ(2, sig2.count.load());
+}
+
+TEST_F(AdminSocketRaise, Async)
+{
+  using std::chrono::milliseconds;
+
+#ifdef WIN32
+  GTEST_SKIP() << "Windows doesn't support --after behavior";
+#endif
+
+  ASSERT_EQ("", send_raise(fmt::format("{}", sig1.sig)));
+  ASSERT_EQ("", send_raise(sig2.name, 0.1));
+
+  EXPECT_EQ(1, sig1.count.load());
+  EXPECT_EQ(0, sig2.count.load());
+
+  this_thread::sleep_for(milliseconds(150));
+
+  EXPECT_EQ(1, sig1.count.load());
+  EXPECT_EQ(1, sig2.count.load());
+}
+
+TEST_F(AdminSocketRaise, AsyncReschedule)
+{
+  using std::chrono::milliseconds;
+
+#ifdef WIN32
+  GTEST_SKIP() << "Windows doesn't support --after behavior";
+#endif
+
+  ASSERT_EQ("", send_raise(sig1.name, 0.1));
+  ASSERT_EQ("", send_raise(sig2.name, 0.2));
+
+  EXPECT_EQ(0, sig1.count.load());
+  EXPECT_EQ(0, sig2.count.load());
+
+  this_thread::sleep_for(milliseconds(150));
+
+  // USR1 got overridden by the second async schedule
+  EXPECT_EQ(0, sig1.count.load());
+  EXPECT_EQ(0, sig2.count.load());
+
+  this_thread::sleep_for(milliseconds(100));
+  EXPECT_EQ(0, sig1.count.load());
+  EXPECT_EQ(1, sig2.count.load());
+}
+
+TEST_F(AdminSocketRaise, AsyncCancel)
+{
+  using std::chrono::milliseconds;
+
+#ifdef WIN32
+  GTEST_SKIP() << "Windows doesn't support --after behavior";
+#endif
+
+  ASSERT_EQ("", send_raise(sig1.name, 0.1));
+
+  EXPECT_EQ(0, sig1.count.load());
+  EXPECT_EQ(0, sig2.count.load());
+
+  ASSERT_EQ("", send_raise_cancel(sig2.name));
+
+  this_thread::sleep_for(milliseconds(150));
+
+  // cancel shouldn't have worked because the signals
+  // didn't match
+  EXPECT_EQ(1, sig1.count.load());
+
+  ASSERT_EQ("", send_raise(sig2.name, 0.1));
+  ASSERT_EQ("", send_raise_cancel(sig2.name));
+
+  this_thread::sleep_for(milliseconds(150));
+
+  // cancel must have worked
+  EXPECT_EQ(0, sig2.count.load());
+
+  ASSERT_EQ("", send_raise(sig1.name, 0.1));
+  ASSERT_EQ("", send_raise_cancel());
+
+  // cancel must have worked, the counter stays 1
+  EXPECT_EQ(1, sig1.count.load());
+}
+
+TEST_F(AdminSocketRaise, StopCont)
+{
+  using std::chrono::duration_cast;
+  using std::chrono::milliseconds;
+  using std::chrono::system_clock;
+
+#ifdef WIN32
+  GTEST_SKIP() << "Windows doesn't support SIGSTOP/SIGCONT and --after";
+#endif
+
+  auto then = system_clock::now();
+  ASSERT_EQ("", send_raise("CONT", 0.2));
+  ASSERT_EQ("", send_raise("STOP"));
+  auto elapsed = system_clock::now() - then;
+  // give it a 1% slack
+  EXPECT_LE(milliseconds(198), duration_cast<milliseconds>(elapsed));
+}
+
 /*
  * Local Variables:
  * compile-command: "cd .. ;

From fd3c941f30296995c8f03d8d29288943871c0baf Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 14 Sep 2023 00:24:46 +0530
Subject: [PATCH 0157/2492] mgr/nfs: report proper errno with err status

Fixes: https://tracker.ceph.com/issues/62641
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/nfs/export.py | 50 ++++++++++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py
index 5887c898fef9..aaa93c34f6c1 100644
--- a/src/pybind/mgr/nfs/export.py
+++ b/src/pybind/mgr/nfs/export.py
@@ -167,9 +167,22 @@ class AppliedExportResults:
     def __init__(self) -> None:
         self.changes: List[Dict[str, str]] = []
         self.has_error = False
+        self.exceptions: List[Exception] = []
+        self.faulty_export_block_indices = ""
+        self.num_errors = 0
+        self.status = ""
 
-    def append(self, value: Dict[str, str]) -> None:
+    def append(self, value: Dict[str, Any]) -> None:
         if value.get("state", "") == "error":
+            self.num_errors += 1
+            # If there is an error then there must be an exception in the dict.
+            self.exceptions.append(value.pop("exception"))
+            # Index is for indicating at which export block in the conf/json
+            # file did the export creation/update failed.
+            if len(self.faulty_export_block_indices) == 0:
+                self.faulty_export_block_indices = str(value.pop("index"))
+            else:
+                self.faulty_export_block_indices += f", {value.pop('index')}"
             self.has_error = True
         self.changes.append(value)
 
@@ -177,7 +190,29 @@ def to_simplified(self) -> List[Dict[str, str]]:
         return self.changes
 
     def mgr_return_value(self) -> int:
-        return -errno.EIO if self.has_error else 0
+        if self.has_error:
+            if len(self.exceptions) == 1:
+                ex = self.exceptions[0]
+                if isinstance(ex, NFSException):
+                    return ex.errno
+                # Some non-nfs exception occurred, this can be anything
+                # therefore return EAGAIN as a generalised errno.
+                return -errno.EAGAIN
+            # There are multiple failures so returning EIO as a generalised
+            # errno.
+            return -errno.EIO
+        return 0
+
+    def mgr_status_value(self) -> str:
+        if self.has_error:
+            if len(self.faulty_export_block_indices) == 1:
+                self.status = f"{str(self.exceptions[0])} for export block" \
+                              f" at index {self.faulty_export_block_indices}"
+            elif len(self.faulty_export_block_indices) > 1:
+                self.status = f"{self.num_errors} export blocks (at index" \
+                              f" {self.faulty_export_block_indices}) failed" \
+                              " to be created/updated"
+        return self.status
 
 
 class ExportMgr:
@@ -501,7 +536,12 @@ def apply_export(self, cluster_id: str, export_config: str) -> AppliedExportResu
 
         aeresults = AppliedExportResults()
         for export in exports:
-            aeresults.append(self._change_export(cluster_id, export))
+            changed_export = self._change_export(cluster_id, export)
+            # This will help figure out which export blocks in conf/json file
+            # are problematic.
+            if changed_export.get("state", "") == "error":
+                changed_export.update({"index": exports.index(export) + 1})
+            aeresults.append(changed_export)
         return aeresults
 
     def _read_export_config(self, cluster_id: str, export_config: str) -> List[Dict]:
@@ -525,7 +565,7 @@ def _read_export_config(self, cluster_id: str, export_config: str) -> List[Dict]
             return j  # j is already a list object
         return [j]  # return a single object list, with j as the only item
 
-    def _change_export(self, cluster_id: str, export: Dict) -> Dict[str, str]:
+    def _change_export(self, cluster_id: str, export: Dict) -> Dict[str, Any]:
         try:
             return self._apply_export(cluster_id, export)
         except NotImplementedError:
@@ -543,7 +583,7 @@ def _change_export(self, cluster_id: str, export: Dict) -> Dict[str, str]:
         except Exception as ex:
             msg = f'Failed to apply export: {ex}'
             log.exception(msg)
-            return {"state": "error", "msg": msg}
+            return {"state": "error", "msg": msg, "exception": ex}
 
     def _update_user_id(
             self,

From 3a5381326c3e5e88080f6b9e249383db67fcb622 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 14 Sep 2023 00:25:25 +0530
Subject: [PATCH 0158/2492] mgr: make object_format's Responder class capable
 of responding err status

Fixes: https://tracker.ceph.com/issues/62641
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/object_format.py | 49 ++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/object_format.py b/src/pybind/mgr/object_format.py
index b53bc3eb060a..8b60aa1b3dc8 100644
--- a/src/pybind/mgr/object_format.py
+++ b/src/pybind/mgr/object_format.py
@@ -235,6 +235,15 @@ def mgr_return_value(self) -> int:
         ...  # pragma: no cover
 
 
+class StatusValueProvider(Protocol):
+    def mgr_status_value(self) -> str:
+        """Return a string value to provide the Ceph MGR with an error status
+        for the MGR's response tuple. Empty string means success. Return a string
+        containing error info otherwise.
+        """
+        ...  # pragma: no cover
+
+
 class CommonFormatter(Protocol):
     """A protocol that indicates the type is a formatter for multiple
     possible formats.
@@ -276,6 +285,11 @@ def _is_return_value_provider(obj: ReturnValueProvider) -> bool:
     return callable(getattr(obj, 'mgr_return_value', None))
 
 
+def _is_status_value_provider(obj: StatusValueProvider) -> bool:
+    """Return true if obj is usable as a StatusValueProvider"""
+    return callable(getattr(obj, 'mgr_status_value', None))
+
+
 class ObjectFormatAdapter:
     """A format adapater for a single object.
     Given an input object, this type will adapt the object, or a simplified
@@ -366,6 +380,27 @@ def mgr_return_value(self) -> int:
         return self.default_return_value
 
 
+class StatusValueAdapter:
+    """A status-value adapter for an object.
+    Given an input object, this type will attempt to get a mgr status value
+    from the object if provides a `mgr_status_value` function.
+    If not it returns a default status value, typically an empty string.
+    """
+
+    def __init__(
+            self,
+            obj: Any,
+            default: str = "",
+    ) -> None:
+        self.obj = obj
+        self.default_status = default
+
+    def mgr_status_value(self) -> str:
+        if _is_status_value_provider(self.obj):
+            return str(self.obj.mgr_status_value())
+        return self.default_status
+
+
 class ErrorResponseBase(Exception):
     """An exception that can directly be converted to a mgr reponse."""
 
@@ -448,6 +483,7 @@ def wrap(
     Callable[..., JSONDataProvider],
     Callable[..., YAMLDataProvider],
     Callable[..., ReturnValueProvider],
+    Callable[..., StatusValueProvider],
 ]
 
 
@@ -487,6 +523,10 @@ def _retval_provider(self, obj: Any) -> ReturnValueProvider:
         """Return a ReturnValueProvider for the given object."""
         return ReturnValueAdapter(obj)
 
+    def _statusval_provider(self, obj: Any) -> StatusValueProvider:
+        """Return a StatusValueProvider for the given object."""
+        return StatusValueAdapter(obj)
+
     def _get_format_func(
         self, obj: Any, format_req: Optional[str] = None
     ) -> Callable:
@@ -515,6 +555,12 @@ def _return_value(self, obj: Any) -> int:
         """Return a mgr return-value for the given object (usually zero)."""
         return self._retval_provider(obj).mgr_return_value()
 
+    def _return_status(self, obj: Any) -> str:
+        """Return a mgr status-value for the given object (usually empty
+        string).
+        """
+        return self._statusval_provider(obj).mgr_status_value()
+
     def __call__(self, f: ObjectResponseFuncType) -> HandlerFuncType:
         """Wrap a python function so that the original function's return value
         becomes the source for an automatically formatted mgr response.
@@ -528,9 +574,10 @@ def _format_response(*args: Any, **kwargs: Any) -> Tuple[int, str, str]:
                 robj = f(*args, **kwargs)
                 body = self._formatted(robj, format_req)
                 retval = self._return_value(robj)
+                statusval = self._return_status(robj)
             except ErrorResponseBase as e:
                 return e.format_response()
-            return retval, body, ""
+            return retval, body, statusval
 
         # set the extra args on our wrapper function. this will be consumed by
         # the CLICommand decorator and added to the set of optional arguments

From 4d663e4e4484977fb90beeb05f67bc71215bddb3 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 14 Sep 2023 00:34:25 +0530
Subject: [PATCH 0159/2492] mgr/tests: test returning error status works as
 expected

Fixes: https://tracker.ceph.com/issues/62641
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
Signed-off-by: John Mulligan <jmulliga@redhat.com>
---
 src/pybind/mgr/tests/test_object_format.py | 26 +++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/tests/test_object_format.py b/src/pybind/mgr/tests/test_object_format.py
index d2fd20870e7a..2e674c69838c 100644
--- a/src/pybind/mgr/tests/test_object_format.py
+++ b/src/pybind/mgr/tests/test_object_format.py
@@ -115,12 +115,18 @@ def test_format_yaml(obj: Any, compatible: bool, yaml_val: str):
 
 
 class Retty:
-    def __init__(self, v) -> None:
+    def __init__(self, v, status="") -> None:
         self.value = v
+        self.status = status
 
     def mgr_return_value(self) -> int:
         return self.value
 
+    def mgr_status_value(self) -> str:
+        if self.status:
+            return self.status
+        return "NOPE"
+
 
 @pytest.mark.parametrize(
     "obj, ret",
@@ -139,6 +145,24 @@ def test_return_value(obj: Any, ret: int):
     assert rva.mgr_return_value() == ret
 
 
+@pytest.mark.parametrize(
+    "obj, ret",
+    [
+        ({}, ""),
+        ({"fish": "sticks"}, ""),
+        (-55, ""),
+        (Retty(0), "NOPE"),
+        (Retty(-55, "cake"), "cake"),
+        (Retty(-50, "pie"), "pie"),
+    ],
+)
+def test_return_status(obj: Any, ret: str):
+    rva = object_format.StatusValueAdapter(obj)
+    # a StatusValueAdapter instance meets the StatusValueProvider protocol.
+    assert object_format._is_status_value_provider(rva)
+    assert rva.mgr_status_value() == ret
+
+
 def test_valid_formats():
     ofa = object_format.ObjectFormatAdapter({"fred": "wilma"})
     vf = ofa.valid_formats()

From 106b88537ddd2a0ecc5b38eefa765af16fac14f6 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 14 Sep 2023 15:03:35 +0530
Subject: [PATCH 0160/2492] mgr: fix some doc strings in object_format.py

Fixes: https://tracker.ceph.com/issues/62641
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/object_format.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/object_format.py b/src/pybind/mgr/object_format.py
index 8b60aa1b3dc8..4a2b6fa8b69b 100644
--- a/src/pybind/mgr/object_format.py
+++ b/src/pybind/mgr/object_format.py
@@ -228,8 +228,8 @@ def format_yaml(self) -> str:
 
 class ReturnValueProvider(Protocol):
     def mgr_return_value(self) -> int:
-        """Return an integer value to provide the Ceph MGR with a error code
-        for the MGR's response tuple. Zero means success. Return an negative
+        """Return an integer value to provide the Ceph MGR with an error code
+        for the MGR's response tuple. Zero means success. Return a negative
         errno otherwise.
         """
         ...  # pragma: no cover
@@ -281,7 +281,7 @@ def _is_yaml_data_provider(obj: YAMLDataProvider) -> bool:
 
 
 def _is_return_value_provider(obj: ReturnValueProvider) -> bool:
-    """Return true if obj is usable as a YAMLDataProvider."""
+    """Return true if obj is usable as a ReturnValueProvider."""
     return callable(getattr(obj, 'mgr_return_value', None))
 
 

From 9a92242e997527a07c7abe065696887f5f761546 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 20 Sep 2023 17:52:50 +0530
Subject: [PATCH 0161/2492] qa: add test cases to verify error reporting works
 as expected

Fixes: https://tracker.ceph.com/issues/62641
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/cephfs/test_nfs.py | 198 ++++++++++++++++++++++++++++++++++++
 1 file changed, 198 insertions(+)

diff --git a/qa/tasks/cephfs/test_nfs.py b/qa/tasks/cephfs/test_nfs.py
index 5fe71054ad9a..94d3182f98b5 100644
--- a/qa/tasks/cephfs/test_nfs.py
+++ b/qa/tasks/cephfs/test_nfs.py
@@ -404,6 +404,13 @@ def _delete_cluster_with_fs(self, fs_name, mnt_pt=None, mode=None):
         self._cmd('fs', 'volume', 'rm', fs_name, '--yes-i-really-mean-it')
         self._test_delete_cluster()
 
+    def _nfs_export_apply(self, cluster, exports, raise_on_error=False):
+        return self.ctx.cluster.run(args=['ceph', 'nfs', 'export', 'apply',
+                                          cluster, '-i', '-'],
+                                    check_status=raise_on_error,
+                                    stdin=json.dumps(exports),
+                                    stdout=StringIO(), stderr=StringIO())
+
     def test_create_and_delete_cluster(self):
         '''
         Test successful creation and deletion of the nfs cluster.
@@ -876,3 +883,194 @@ def test_nfs_export_creation_at_symlink(self):
                 raise
         self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}/*'])
         self._delete_cluster_with_fs(self.fs_name, mnt_pt, preserve_mode)
+
+    def test_nfs_export_apply_multiple_exports(self):
+        """
+        Test multiple export creation/update with multiple
+        export blocks provided in the json/conf file using:
+        ceph nfs export apply <nfs_cluster> -i <{conf/json}_file>, and check
+        1) if there are multiple failure:
+        -> Return the EIO and error status to CLI (along with JSON output
+           containing status of every export).
+        2) if there is single failure:
+        -> Return the respective errno and error status to CLI (along with
+           JSON output containing status of every export).
+        """
+
+        mnt_pt = self._sys_cmd(['mktemp', '-d']).decode().strip()
+        self._create_cluster_with_fs(self.fs_name, mnt_pt)
+        try:
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir1'])
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir2'])
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir3'])
+            self._create_export(export_id='1',
+                                extra_cmd=['--pseudo-path', self.pseudo_path,
+                                           '--path', '/testdir1'])
+            self._create_export(export_id='2',
+                                extra_cmd=['--pseudo-path',
+                                           self.pseudo_path+'2',
+                                           '--path', '/testdir2'])
+            exports = [
+                {
+                    "export_id": 11,  # export_id change not allowed
+                    "path": "/testdir1",
+                    "pseudo": self.pseudo_path,
+                    "squash": "none",
+                    "access_type": "rw",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.1",
+                        "fs_name": self.fs_name
+                    }
+                },
+                {
+                    "export_id": 2,
+                    "path": "/testdir2",
+                    "pseudo": self.pseudo_path+'2',
+                    "squash": "none",
+                    "access_type": "rw",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.2",
+                        "fs_name": "invalid_fs_name"  # invalid fs
+                    }
+                },
+                {   # no error, export creation should succeed
+                    "export_id": 3,
+                    "path": "/testdir3",
+                    "pseudo": self.pseudo_path+'3',
+                    "squash": "none",
+                    "access_type": "rw",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.3",
+                        "fs_name": self.fs_name
+                    }
+                }
+            ]
+
+            # multiple failures
+            ret = self._nfs_export_apply(self.cluster_id, exports)
+            self.assertEqual(ret[0].returncode, errno.EIO)
+            self.assertIn("2 export blocks (at index 1, 2) failed to be "
+                          "created/updated", ret[0].stderr.getvalue())
+
+            # single failure
+            exports[1]["fsal"]["fs_name"] = self.fs_name  # correct the fs
+            ret = self._nfs_export_apply(self.cluster_id, exports)
+            self.assertEqual(ret[0].returncode, errno.EINVAL)
+            self.assertIn("Export ID changed, Cannot update export for "
+                          "export block at index 1", ret[0].stderr.getvalue())
+        finally:
+            self._delete_cluster_with_fs(self.fs_name, mnt_pt)
+            self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])
+
+    def test_nfs_export_apply_single_export(self):
+        """
+        Test that when single export creation/update fails with multiple
+        export blocks provided in the json/conf file using:
+        ceph nfs export apply <nfs_cluster> -i <{conf/json}_file>, it
+        returns the respective errno and error status to CLI (along with
+        JSON output containing status of every export).
+        """
+
+        mnt_pt = self._sys_cmd(['mktemp', '-d']).decode().strip()
+        self._create_cluster_with_fs(self.fs_name, mnt_pt)
+        try:
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir1'])
+            self._create_export(export_id='1',
+                                extra_cmd=['--pseudo-path', self.pseudo_path,
+                                           '--path', '/testdir1'])
+            export = {
+                "export_id": 1,
+                "path": "/testdir1",
+                "pseudo": self.pseudo_path,
+                "squash": "none",
+                "access_type": "rw",
+                "protocols": [4],
+                "fsal": {
+                    "name": "CEPH",
+                    "user_id": "nfs.test.1",
+                    "fs_name": "invalid_fs_name"  # invalid fs
+                }
+            }
+            ret = self._nfs_export_apply(self.cluster_id, export)
+            self.assertEqual(ret[0].returncode, errno.ENOENT)
+            self.assertIn("filesystem invalid_fs_name not found for "
+                          "export block at index 1", ret[0].stderr.getvalue())
+        finally:
+            self._delete_cluster_with_fs(self.fs_name, mnt_pt)
+            self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])
+
+    def test_nfs_export_apply_json_output_states(self):
+        """
+        If export creation/update is done using:
+        ceph nfs export apply <nfs_cluster> -i <{conf/json}_file> then the
+        "status" field in the json output maybe added, updated, error or
+        warning. Test different scenarios to make sure these states are
+        in the json output as expected.
+        """
+
+        mnt_pt = self._sys_cmd(['mktemp', '-d']).decode().strip()
+        self._create_cluster_with_fs(self.fs_name, mnt_pt)
+        try:
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir1'])
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir2'])
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir3'])
+            self._create_export(export_id='1',
+                                extra_cmd=['--pseudo-path', self.pseudo_path,
+                                           '--path', '/testdir1'])
+            exports = [
+                {   # change pseudo, state should be "updated"
+                    "export_id": 1,
+                    "path": "/testdir1",
+                    "pseudo": self.pseudo_path+'1',
+                    "squash": "none",
+                    "access_type": "rw",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.1",
+                        "fs_name": self.fs_name
+                    }
+                },
+                {   # a new export, state should be "added"
+                    "export_id": 2,
+                    "path": "/testdir2",
+                    "pseudo": self.pseudo_path+'2',
+                    "squash": "none",
+                    "access_type": "rw",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.2",
+                        "fs_name": self.fs_name
+                    }
+                },
+                {   # error in export block, state should be "error" since the
+                    # fs_name is invalid
+                    "export_id": 3,
+                    "path": "/testdir3",
+                    "pseudo": self.pseudo_path+'3',
+                    "squash": "none",
+                    "access_type": "RW",
+                    "protocols": [4],
+                    "fsal": {
+                        "name": "CEPH",
+                        "user_id": "nfs.test.3",
+                        "fs_name": "invalid_fs_name"
+                    }
+                }
+            ]
+            ret = self._nfs_export_apply(self.cluster_id, exports)
+            json_output = json.loads(ret[0].stdout.getvalue().strip())
+            self.assertEqual(len(json_output), 3)
+            self.assertEqual(json_output[0]["state"], "updated")
+            self.assertEqual(json_output[1]["state"], "added")
+            self.assertEqual(json_output[2]["state"], "error")
+        finally:
+            self._delete_cluster_with_fs(self.fs_name, mnt_pt)
+            self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])

From 3f11cd94c0add0ac182d03ed1fc1a8708d72a06d Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 12 Oct 2023 17:56:05 +0800
Subject: [PATCH 0162/2492] crimson/osd: store "final pool info" for pools that
 are deleted

Although "final pool info" is needed by ec which is not yet implemented
for crimson, we need this to make OSD reboot work

Fixes: https://tracker.ceph.com/issues/63186
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/osd_meta.cc       | 31 +++++++++++++++++++++++++++++++
 src/crimson/osd/osd_meta.h        |  4 ++++
 src/crimson/osd/shard_services.cc | 22 +++++++++++++++++-----
 3 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/src/crimson/osd/osd_meta.cc b/src/crimson/osd/osd_meta.cc
index e40b2b2464be..0a9b42cd594f 100644
--- a/src/crimson/osd/osd_meta.cc
+++ b/src/crimson/osd/osd_meta.cc
@@ -9,6 +9,7 @@
 #include "crimson/os/futurized_collection.h"
 #include "crimson/os/futurized_store.h"
 #include "os/Transaction.h"
+#include "osd/OSDMap.h"
 
 using std::string;
 using read_errorator = crimson::os::FuturizedStore::Shard::read_errorator;
@@ -80,6 +81,36 @@ OSDMeta::load_final_pool_info(int64_t pool) {
   }));
 }
 
+void OSDMeta::store_final_pool_info(
+  ceph::os::Transaction &t,
+  OSDMap* lastmap,
+  std::map<epoch_t, OSDMap*> &added_map)
+{
+  for (auto [e, map] : added_map) {
+    if (!lastmap) {
+      lastmap = map;
+      continue;
+    }
+    for (auto &[pool_id, pool] : lastmap->get_pools()) {
+      if (!map->have_pg_pool(pool_id)) {
+	ghobject_t obj = final_pool_info_oid(pool_id);
+	bufferlist bl;
+	encode(pool, bl, CEPH_FEATURES_ALL);
+	string name = lastmap->get_pool_name(pool_id);
+	encode(name, bl);
+	std::map<string, string> profile;
+	if (pool.is_erasure()) {
+	  profile = lastmap->get_erasure_code_profile(
+	    pool.erasure_code_profile);
+	}
+	encode(profile, bl);
+	t.write(coll->get_cid(), obj, 0, bl.length(), bl);
+      }
+    }
+    lastmap = map;
+  }
+}
+
 ghobject_t OSDMeta::osdmap_oid(epoch_t epoch)
 {
   string name = fmt::format("osdmap.{}", epoch);
diff --git a/src/crimson/osd/osd_meta.h b/src/crimson/osd/osd_meta.h
index 652266d9e201..af18991ff74b 100644
--- a/src/crimson/osd/osd_meta.h
+++ b/src/crimson/osd/osd_meta.h
@@ -53,6 +53,10 @@ class OSDMeta {
   seastar::future<std::tuple<pg_pool_t,
 			     std::string,
 			     ec_profile_t>> load_final_pool_info(int64_t pool);
+  void store_final_pool_info(
+    ceph::os::Transaction&,
+    OSDMap* lastmap,
+    std::map<epoch_t, OSDMap*>&);
 private:
   static ghobject_t osdmap_oid(epoch_t epoch);
   static ghobject_t final_pool_info_oid(int64_t pool);
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index a6431305d806..dd11cddce733 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -420,15 +420,19 @@ seastar::future<std::unique_ptr<OSDMap>> OSDSingletonState::load_map(epoch_t e)
 seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
                                   epoch_t start, Ref<MOSDMap> m)
 {
-  return seastar::do_for_each(
-    boost::make_counting_iterator(start),
-    boost::make_counting_iterator(m->get_last() + 1),
-    [&t, m, this](epoch_t e) {
+  return seastar::do_with(
+    std::map<epoch_t, OSDMap*>(),
+    [&t, m, start, this](auto &added_maps) {
+    return seastar::do_for_each(
+      boost::make_counting_iterator(start),
+      boost::make_counting_iterator(m->get_last() + 1),
+      [&t, m, this, &added_maps](epoch_t e) {
       if (auto p = m->maps.find(e); p != m->maps.end()) {
 	auto o = std::make_unique<OSDMap>();
 	o->decode(p->second);
 	logger().info("store_maps storing osdmap.{}", e);
 	store_map_bl(t, e, std::move(std::move(p->second)));
+	added_maps.emplace(e, o.get());
 	osdmaps.insert(e, std::move(o));
 	return seastar::now();
       } else if (auto p = m->incremental_maps.find(e);
@@ -436,7 +440,8 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	logger().info("store_maps found osdmap.{} incremental map, "
 	              "loading osdmap.{}", e, e - 1);
 	ceph_assert(std::cmp_greater(e, 0u));
-	return load_map(e - 1).then([e, bl=p->second, &t, this](auto o) {
+	return load_map(e - 1).then(
+	  [&added_maps, e, bl=p->second, &t, this](auto o) {
 	  OSDMap::Incremental inc;
 	  auto i = bl.cbegin();
 	  inc.decode(i);
@@ -445,6 +450,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	  o->encode(fbl, inc.encode_features | CEPH_FEATURE_RESERVED);
 	  logger().info("store_maps storing osdmap.{}", o->get_epoch());
 	  store_map_bl(t, e, std::move(fbl));
+	  added_maps.emplace(e, o.get());
 	  osdmaps.insert(e, std::move(o));
 	  return seastar::now();
 	});
@@ -452,7 +458,13 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	logger().error("MOSDMap lied about what maps it had?");
 	return seastar::now();
       }
+    }).then([&t, this, &added_maps] {
+      auto [e, map] = *added_maps.begin();
+      auto lastmap = osdmaps.find(e - 1).get();
+      meta_coll->store_final_pool_info(t, lastmap, added_maps);
+      return seastar::now();
     });
+  });
 }
 
 seastar::future<Ref<PG>> ShardServices::make_pg(

From 8fed26a185215d1a92c88544fd64f84a94bb106e Mon Sep 17 00:00:00 2001
From: "yite.gu" <yitegu0@gmail.com>
Date: Fri, 13 Oct 2023 02:50:35 +0800
Subject: [PATCH 0163/2492] os/bluestore: add bluestore fragmentation micros to
 prometheus

High fragmentation is one of the factors affecting performance decrease,
add it to prometheus, it helps to us monitor fragmentation changes.

Signed-off-by: Yite Gu <yitegu0@gmail.com>
---
 src/os/bluestore/BlueStore.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index ec03fcde14ae..67f9fb92c6fb 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6088,7 +6088,9 @@ void BlueStore::_init_logger()
 	    PerfCountersBuilder::PRIO_CRITICAL,
 	    unit_t(UNIT_BYTES));
   b.add_u64(l_bluestore_fragmentation, "fragmentation_micros",
-            "How fragmented bluestore free space is (free extents / max possible number of free extents) * 1000");
+            "How fragmented bluestore free space is (free extents / max possible number of free extents) * 1000",
+	    "fbss",
+	    PerfCountersBuilder::PRIO_USEFUL);
   b.add_u64(l_bluestore_alloc_unit, "alloc_unit",
 	    "allocation unit size in bytes",
 	    "au_b",

From cc373de90932a4f2dd54466062fae89f6598ea67 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Tue, 2 May 2023 16:22:19 -0400
Subject: [PATCH 0164/2492] common: perf counters cache + rgw op labeled
 counters

This commit contains the following features:
- a perf counters cache per CephContext that acts
as a wrapper around perf counters for storing and
modifying labeled perf counters per CephContext
- instrumentation of the rgw with labeled perf
injest counters for the major rgw ops

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 src/common/CMakeLists.txt            |    1 +
 src/common/options/rgw.yaml.in       |   18 +
 src/common/perf_counters.cc          |    7 +-
 src/common/perf_counters_cache.cc    |  116 +++
 src/common/perf_counters_cache.h     |   83 ++
 src/rgw/rgw_file.cc                  |    6 +-
 src/rgw/rgw_op.cc                    |   52 +-
 src/rgw/rgw_perf_counters.cc         |  188 +++--
 src/rgw/rgw_perf_counters.h          |   68 +-
 src/test/CMakeLists.txt              |    7 +
 src/test/test_perf_counters_cache.cc | 1063 ++++++++++++++++++++++++++
 11 files changed, 1543 insertions(+), 66 deletions(-)
 create mode 100644 src/common/perf_counters_cache.cc
 create mode 100644 src/common/perf_counters_cache.h
 create mode 100644 src/test/test_perf_counters_cache.cc

diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 695ea7a68b3e..5f07f7e8483c 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -83,6 +83,7 @@ set(common_srcs
   options.cc
   page.cc
   perf_counters.cc
+  perf_counters_cache.cc
   perf_counters_collection.cc
   perf_counters_key.cc
   perf_histogram.cc
diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 3971929e412d..f2f85b648f8b 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3829,8 +3829,26 @@ options:
     other form of policies that Amazon does, so if you are mirroring
     policies between RGW and AWS, you may wish to set this to false.
   default: true
+- name: rgw_perf_counters_cache
+  type: bool
+  level: dev
+  default: false
+  desc: enable rgw labeled perf counters cache
+  long desc: If set to true, rgw creates labeled perf counters and stores them 
+    in an rgw specific labeled perf counters cache.
+  see_also:
+  - rgw_perf_counters_cache_size
   services:
   - rgw
+  with_legacy: true
+- name: rgw_perf_counters_cache_size
+  type: uint
+  level: advanced
+  desc: Number of labeled perf counters the rgw perf counters cache can store
+  default: 10000
+  services:
+  - rgw
+  with_legacy: true
 - name: rgw_d4n_host
   type: str
   level: advanced
diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc
index b5e361b505cd..81bf3284bdae 100644
--- a/src/common/perf_counters.cc
+++ b/src/common/perf_counters.cc
@@ -135,7 +135,12 @@ void PerfCountersCollectionImpl::dump_formatted_generic(
     const std::string &counter) const
 {
   f->open_object_section("perfcounter_collection");
-  
+  // close out all of counters collection immediately if collection is empty
+  if (m_loggers.empty()) {
+    f->close_section(); // all of counters collection
+    return;
+  }
+
   if (dump_labeled) {
     std::string prev_key_name;
     for (auto l = m_loggers.begin(); l != m_loggers.end(); ++l) {
diff --git a/src/common/perf_counters_cache.cc b/src/common/perf_counters_cache.cc
new file mode 100644
index 000000000000..e0810508ce7f
--- /dev/null
+++ b/src/common/perf_counters_cache.cc
@@ -0,0 +1,116 @@
+#include "common/perf_counters_cache.h"
+#include "common/perf_counters_key.h"
+
+namespace ceph::perf_counters {
+
+void PerfCountersCache::check_key(const std::string &key) {
+  std::string_view key_name = ceph::perf_counters::key_name(key);
+  // return false for empty key name
+  assert(key_name != "");
+
+  // if there are no labels key name is not valid
+  auto key_labels = ceph::perf_counters::key_labels(key);
+  assert(key_labels.begin() != key_labels.end());
+
+  // don't accept keys where any labels have an empty label name
+  for (auto key_label : key_labels) {
+    assert(key_label.first != "");
+    assert(key_label.second != "");
+  }
+}
+
+std::shared_ptr<PerfCounters> PerfCountersCache::add(const std::string &key) {
+  check_key(key);
+
+  auto [ref, key_existed] = cache.get_or_create(key);
+  if (!key_existed) {
+    ref->counters = create_counters(key, cct);
+    assert(ref->counters);
+    ref->cct = cct;
+  }
+  return ref->counters;
+}
+
+
+std::shared_ptr<PerfCounters> PerfCountersCache::get(const std::string &key) {
+  std::lock_guard lock(m_lock);
+  return add(key);
+}
+
+void PerfCountersCache::inc(const std::string &key, int indx, uint64_t v) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->inc(indx, v);
+  }
+}
+
+void PerfCountersCache::dec(const std::string &key, int indx, uint64_t v) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->dec(indx, v);
+  }
+}
+
+void PerfCountersCache::tinc(const std::string &key, int indx, utime_t amt) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->tinc(indx, amt);
+  }
+}
+
+void PerfCountersCache::tinc(const std::string &key, int indx, ceph::timespan amt) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->tinc(indx, amt);
+  }
+}
+
+void PerfCountersCache::set_counter(const std::string &key, int indx, uint64_t val) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->set(indx, val);
+  }
+}
+
+uint64_t PerfCountersCache::get_counter(const std::string &key, int indx) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  uint64_t val = 0;
+  if (counters) {
+    val = counters->get(indx);
+  }
+  return val;
+}
+
+utime_t PerfCountersCache::tget(const std::string &key, int indx) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  utime_t val;
+  if (counters) {
+    val = counters->tget(indx);
+    return val;
+  } else {
+    return utime_t();
+  }
+}
+
+void PerfCountersCache::tset(const std::string &key, int indx, utime_t amt) {
+  std::lock_guard lock(m_lock);
+  auto counters = add(key);
+  if (counters) {
+    counters->tset(indx, amt);
+  }
+}
+
+PerfCountersCache::PerfCountersCache(CephContext *_cct, size_t _target_size,
+      std::function<std::shared_ptr<PerfCounters>(const std::string&, CephContext*)> _create_counters)
+      : cct(_cct), create_counters(_create_counters), m_lock(ceph::make_mutex("PerfCountersCache")) { cache.set_target_size(_target_size); }
+
+PerfCountersCache::~PerfCountersCache() { cache.set_target_size(0); }
+
+} // namespace ceph::perf_counters
diff --git a/src/common/perf_counters_cache.h b/src/common/perf_counters_cache.h
new file mode 100644
index 000000000000..866f56ee3502
--- /dev/null
+++ b/src/common/perf_counters_cache.h
@@ -0,0 +1,83 @@
+#pragma once
+
+#include "common/perf_counters.h"
+#include "common/ceph_context.h"
+#include "common/intrusive_lru.h"
+
+namespace ceph::perf_counters {
+
+struct perf_counters_cache_item_to_key;
+
+struct PerfCountersCacheEntry : public ceph::common::intrusive_lru_base<
+                                       ceph::common::intrusive_lru_config<
+                                       std::string, PerfCountersCacheEntry, perf_counters_cache_item_to_key>> {
+  std::string key;
+  std::shared_ptr<PerfCounters> counters;
+  CephContext *cct;
+
+  PerfCountersCacheEntry(const std::string &_key) : key(_key) {}
+
+  ~PerfCountersCacheEntry() {
+    if (counters) {
+      cct->get_perfcounters_collection()->remove(counters.get());
+    }
+  }
+};
+
+struct perf_counters_cache_item_to_key {
+  using type = std::string;
+  const type &operator()(const PerfCountersCacheEntry &entry) {
+    return entry.key;
+  }
+};
+
+class PerfCountersCache {
+private:
+  CephContext *cct;
+  std::function<std::shared_ptr<PerfCounters>(const std::string&, CephContext*)> create_counters;
+  PerfCountersCacheEntry::lru_t cache;
+  mutable ceph::mutex m_lock;
+
+  /* check to make sure key name is non-empty and non-empty labels
+   *
+   * A valid key has the the form 
+   * key\0label1\0val1\0label2\0val2 ... label\0valN
+   * The following 3 properties checked for in this function
+   * 1. A non-empty key
+   * 2. At least 1 set of labels
+   * 3. Each label has a non-empty key and value
+   *
+   * See perf_counters_key.h
+   */
+  void check_key(const std::string &key);
+
+  // adds a new entry to the cache and returns its respective PerfCounter*
+  // or returns the PerfCounter* of an existing entry in the cache
+  std::shared_ptr<PerfCounters> add(const std::string &key);
+
+public:
+
+  // get() and its associated shared_ptr reference counting should be avoided 
+  // unless the caller intends to modify multiple counter values at the same time.
+  // If multiple counter values will not be modified at the same time, inc/dec/etc. 
+  // are recommended.
+  std::shared_ptr<PerfCounters> get(const std::string &key);
+
+  void inc(const std::string &key, int indx, uint64_t v);
+  void dec(const std::string &key, int indx, uint64_t v);
+  void tinc(const std::string &key, int indx, utime_t amt);
+  void tinc(const std::string &key, int indx, ceph::timespan amt);
+  void set_counter(const std::string &key, int indx, uint64_t val);
+  uint64_t get_counter(const std::string &key, int indx);
+  utime_t tget(const std::string &key, int indx);
+  void tset(const std::string &key, int indx, utime_t amt);
+
+  // _create_counters should be a function that returns a valid, newly created perf counters instance
+  // Ceph components utilizing the PerfCountersCache are encouraged to pass in a factory function that would
+  // create and initialize different kinds of counters based on the name returned from ceph::perfcounters::key_name(key)
+  PerfCountersCache(CephContext *_cct, size_t _target_size,
+                    std::function<std::shared_ptr<PerfCounters>(const std::string&, CephContext*)> _create_counters);
+  ~PerfCountersCache();
+};
+
+} // namespace ceph::perf_counters
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 6a55d3f1d8f5..92aa66f060eb 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1838,7 +1838,7 @@ namespace rgw {
     ceph_assert(! dlo_manifest);
     ceph_assert(! slo_info);
 
-    perfcounter->inc(l_rgw_put);
+    rgw::op_counters::global_op_counters->inc(l_rgw_op_put);
     op_ret = -EINVAL;
 
     if (state->object->empty()) {
@@ -1944,7 +1944,7 @@ namespace rgw {
     real_time appx_t = real_clock::now();
 
     state->obj_size = bytes_written;
-    perfcounter->inc(l_rgw_put_b, state->obj_size);
+    rgw::op_counters::global_op_counters->inc(l_rgw_op_put_b, state->obj_size);
 
     // flush data in filters
     op_ret = filter->process({}, state->obj_size);
@@ -2027,7 +2027,7 @@ namespace rgw {
     }
 
   done:
-    perfcounter->tinc(l_rgw_put_lat, state->time_elapsed());
+    rgw::op_counters::global_op_counters->tinc(l_rgw_op_put_lat, state->time_elapsed());
     return op_ret;
   } /* exec_finish */
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8c15e5bd2e3f..cd347502e76c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -21,6 +21,7 @@
 #include "common/utf8.h"
 #include "common/ceph_json.h"
 #include "common/static_ptr.h"
+#include "common/perf_counters_key.h"
 #include "rgw_tracer.h"
 
 #include "rgw_rados.h"
@@ -1691,7 +1692,8 @@ int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
     return 0;
   }
 
-  perfcounter->inc(l_rgw_get_b, cur_end - cur_ofs);
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_get_b, cur_end - cur_ofs);
   filter->fixup_range(cur_ofs, cur_end);
   op_ret = read_op->iterate(this, cur_ofs, cur_end, filter, s->yield);
   if (op_ret >= 0)
@@ -1764,8 +1766,8 @@ static int iterate_user_manifest_parts(const DoutPrefixProvider *dpp,
 	found_end = true;
       }
 
-      perfcounter->tinc(l_rgw_get_lat,
-			(ceph_clock_now() - start_time));
+      rgw::op_counters::global_op_counters->tinc(l_rgw_op_get_lat, 
+                            (ceph_clock_now() - start_time));
 
       if (found_start && !handled_end) {
         len_count += end_ofs - start_ofs;
@@ -1860,8 +1862,8 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
       found_end = true;
     }
 
-    perfcounter->tinc(l_rgw_get_lat,
-		      (ceph_clock_now() - start_time));
+    rgw::op_counters::global_op_counters->tinc(l_rgw_op_get_lat, 
+                      (ceph_clock_now() - start_time));
 
     if (found_start) {
       if (cb) {
@@ -2208,7 +2210,8 @@ void RGWGetObj::execute(optional_yield y)
   std::unique_ptr<RGWGetObj_Filter> run_lua;
   map<string, bufferlist>::iterator attr_iter;
 
-  perfcounter->inc(l_rgw_get);
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_get, 1);
 
   std::unique_ptr<rgw::sal::Object::ReadOp> read_op(s->object->get_read_op());
 
@@ -2406,14 +2409,15 @@ void RGWGetObj::execute(optional_yield y)
     return;
   }
 
-  perfcounter->inc(l_rgw_get_b, end - ofs);
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_get_b, end-ofs);
 
   op_ret = read_op->iterate(this, ofs_x, end_x, filter, s->yield);
 
   if (op_ret >= 0)
     op_ret = filter->flush();
 
-  perfcounter->tinc(l_rgw_get_lat, s->time_elapsed());
+  rgw::op_counters::tinc(labeled_counters, l_rgw_op_get_lat, s->time_elapsed());
+
   if (op_ret < 0) {
     goto done_err;
   }
@@ -2489,6 +2493,9 @@ void RGWListBuckets::execute(optional_yield y)
 
   const uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
 
+  auto labeled_counters = rgw::op_counters::get({{"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_list_buckets, 1);
+
   auto g = make_scope_guard([this, &started] {
       if (!started) {
         send_response_begin(false);
@@ -2565,6 +2572,8 @@ void RGWListBuckets::execute(optional_yield y)
 
     handle_listing_chunk(listing.buckets);
   } while (!marker.empty() && !done);
+  
+  rgw::op_counters::tinc(labeled_counters, l_rgw_op_list_buckets_lat, s->time_elapsed());
 }
 
 void RGWGetUsage::execute(optional_yield y)
@@ -3049,6 +3058,10 @@ void RGWListBucket::execute(optional_yield y)
     objs = std::move(results.objs);
     common_prefixes = std::move(results.common_prefixes);
   }
+
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_list_obj, 1);
+  rgw::op_counters::tinc(labeled_counters, l_rgw_op_list_obj_lat, s->time_elapsed());
 }
 
 int RGWGetBucketLogging::verify_permission(optional_yield y)
@@ -3582,6 +3595,10 @@ void RGWDeleteBucket::execute(optional_yield y)
       op_ret = 0;
   }
 
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_del_bucket, 1);
+  rgw::op_counters::tinc(labeled_counters, l_rgw_op_del_bucket_lat, s->time_elapsed());
+
   return;
 }
 
@@ -4008,11 +4025,14 @@ void RGWPutObj::execute(optional_yield y)
   off_t fst;
   off_t lst;
 
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+
   bool need_calc_md5 = (dlo_manifest == NULL) && (slo_info == NULL);
-  perfcounter->inc(l_rgw_put);
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_put, 1);
+
   // report latency on return
   auto put_lat = make_scope_guard([&] {
-      perfcounter->tinc(l_rgw_put_lat, s->time_elapsed());
+      rgw::op_counters::tinc(labeled_counters, l_rgw_op_put_lat, s->time_elapsed());
     });
 
   op_ret = -EINVAL;
@@ -4287,7 +4307,7 @@ void RGWPutObj::execute(optional_yield y)
   s->obj_size = ofs;
   s->object->set_obj_size(ofs);
 
-  perfcounter->inc(l_rgw_put_b, s->obj_size);
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_put_b, s->obj_size);
 
   op_ret = do_aws4_auth_completion();
   if (op_ret < 0) {
@@ -5244,6 +5264,11 @@ void RGWDeleteObj::execute(optional_yield y)
       op_ret = 0;
     }
 
+    auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+    rgw::op_counters::inc(labeled_counters, l_rgw_op_del_obj, 1);
+    rgw::op_counters::inc(labeled_counters, l_rgw_op_del_obj_b, obj_size);
+    rgw::op_counters::tinc(labeled_counters, l_rgw_op_del_obj_lat, s->time_elapsed());
+
     // send request to notification manager
     int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
     if (ret < 0) {
@@ -5703,6 +5728,11 @@ void RGWCopyObj::execute(optional_yield y)
     ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
     // too late to rollback operation, hence op_ret is not set here
   }
+
+  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_copy_obj, 1);
+  rgw::op_counters::inc(labeled_counters, l_rgw_op_copy_obj_b, obj_size);
+  rgw::op_counters::tinc(labeled_counters, l_rgw_op_copy_obj_lat, s->time_elapsed());
 }
 
 int RGWGetACLs::verify_permission(optional_yield y)
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index 6757dd8913cf..aca56a60946d 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -3,69 +3,171 @@
 
 #include "rgw_perf_counters.h"
 #include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
 #include "common/ceph_context.h"
 
 PerfCounters *perfcounter = NULL;
+ceph::perf_counters::PerfCountersCache *perf_counters_cache = NULL;
+std::string rgw_op_counters_key = "rgw_op";
 
-int rgw_perf_start(CephContext *cct)
-{
-  PerfCountersBuilder plb(cct, "rgw", l_rgw_first, l_rgw_last);
-
+static void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
   // RGW emits comparatively few metrics, so let's be generous
   // and mark them all USEFUL to get transmission to ceph-mgr by default.
-  plb.set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+  pcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
 
-  plb.add_u64_counter(l_rgw_req, "req", "Requests");
-  plb.add_u64_counter(l_rgw_failed_req, "failed_req", "Aborted requests");
+  pcb->add_u64_counter(l_rgw_req, "req", "Requests");
+  pcb->add_u64_counter(l_rgw_failed_req, "failed_req", "Aborted requests");
 
-  plb.add_u64_counter(l_rgw_get, "get", "Gets");
-  plb.add_u64_counter(l_rgw_get_b, "get_b", "Size of gets");
-  plb.add_time_avg(l_rgw_get_lat, "get_initial_lat", "Get latency");
-  plb.add_u64_counter(l_rgw_put, "put", "Puts");
-  plb.add_u64_counter(l_rgw_put_b, "put_b", "Size of puts");
-  plb.add_time_avg(l_rgw_put_lat, "put_initial_lat", "Put latency");
+  pcb->add_u64(l_rgw_qlen, "qlen", "Queue length");
+  pcb->add_u64(l_rgw_qactive, "qactive", "Active requests queue");
 
-  plb.add_u64(l_rgw_qlen, "qlen", "Queue length");
-  plb.add_u64(l_rgw_qactive, "qactive", "Active requests queue");
+  pcb->add_u64_counter(l_rgw_cache_hit, "cache_hit", "Cache hits");
+  pcb->add_u64_counter(l_rgw_cache_miss, "cache_miss", "Cache miss");
 
-  plb.add_u64_counter(l_rgw_cache_hit, "cache_hit", "Cache hits");
-  plb.add_u64_counter(l_rgw_cache_miss, "cache_miss", "Cache miss");
+  pcb->add_u64_counter(l_rgw_keystone_token_cache_hit, "keystone_token_cache_hit", "Keystone token cache hits");
+  pcb->add_u64_counter(l_rgw_keystone_token_cache_miss, "keystone_token_cache_miss", "Keystone token cache miss");
 
-  plb.add_u64_counter(l_rgw_keystone_token_cache_hit, "keystone_token_cache_hit", "Keystone token cache hits");
-  plb.add_u64_counter(l_rgw_keystone_token_cache_miss, "keystone_token_cache_miss", "Keystone token cache miss");
+  pcb->add_u64_counter(l_rgw_gc_retire, "gc_retire_object", "GC object retires");
 
-  plb.add_u64_counter(l_rgw_gc_retire, "gc_retire_object", "GC object retires");
-
-  plb.add_u64_counter(l_rgw_lc_expire_current, "lc_expire_current",
+  pcb->add_u64_counter(l_rgw_lc_expire_current, "lc_expire_current",
 		      "Lifecycle current expiration");
-  plb.add_u64_counter(l_rgw_lc_expire_noncurrent, "lc_expire_noncurrent",
+  pcb->add_u64_counter(l_rgw_lc_expire_noncurrent, "lc_expire_noncurrent",
 		      "Lifecycle non-current expiration");
-  plb.add_u64_counter(l_rgw_lc_expire_dm, "lc_expire_dm",
+  pcb->add_u64_counter(l_rgw_lc_expire_dm, "lc_expire_dm",
 		      "Lifecycle delete-marker expiration");
-  plb.add_u64_counter(l_rgw_lc_transition_current, "lc_transition_current",
+  pcb->add_u64_counter(l_rgw_lc_transition_current, "lc_transition_current",
 		      "Lifecycle current transition");
-  plb.add_u64_counter(l_rgw_lc_transition_noncurrent,
+  pcb->add_u64_counter(l_rgw_lc_transition_noncurrent,
 		      "lc_transition_noncurrent",
 		      "Lifecycle non-current transition");
-  plb.add_u64_counter(l_rgw_lc_abort_mpu, "lc_abort_mpu",
+  pcb->add_u64_counter(l_rgw_lc_abort_mpu, "lc_abort_mpu",
 		      "Lifecycle abort multipart upload");
 
-  plb.add_u64_counter(l_rgw_pubsub_event_triggered, "pubsub_event_triggered", "Pubsub events with at least one topic");
-  plb.add_u64_counter(l_rgw_pubsub_event_lost, "pubsub_event_lost", "Pubsub events lost");
-  plb.add_u64_counter(l_rgw_pubsub_store_ok, "pubsub_store_ok", "Pubsub events successfully stored");
-  plb.add_u64_counter(l_rgw_pubsub_store_fail, "pubsub_store_fail", "Pubsub events failed to be stored");
-  plb.add_u64(l_rgw_pubsub_events, "pubsub_events", "Pubsub events in store");
-  plb.add_u64_counter(l_rgw_pubsub_push_ok, "pubsub_push_ok", "Pubsub events pushed to an endpoint");
-  plb.add_u64_counter(l_rgw_pubsub_push_failed, "pubsub_push_failed", "Pubsub events failed to be pushed to an endpoint");
-  plb.add_u64(l_rgw_pubsub_push_pending, "pubsub_push_pending", "Pubsub events pending reply from endpoint");
-  plb.add_u64_counter(l_rgw_pubsub_missing_conf, "pubsub_missing_conf", "Pubsub events could not be handled because of missing configuration");
-  
-  plb.add_u64_counter(l_rgw_lua_script_ok, "lua_script_ok", "Successfull executions of Lua scripts");
-  plb.add_u64_counter(l_rgw_lua_script_fail, "lua_script_fail", "Failed executions of Lua scripts");
-  plb.add_u64(l_rgw_lua_current_vms, "lua_current_vms", "Number of Lua VMs currently being executed");
+  pcb->add_u64_counter(l_rgw_pubsub_event_triggered, "pubsub_event_triggered", "Pubsub events with at least one topic");
+  pcb->add_u64_counter(l_rgw_pubsub_event_lost, "pubsub_event_lost", "Pubsub events lost");
+  pcb->add_u64_counter(l_rgw_pubsub_store_ok, "pubsub_store_ok", "Pubsub events successfully stored");
+  pcb->add_u64_counter(l_rgw_pubsub_store_fail, "pubsub_store_fail", "Pubsub events failed to be stored");
+  pcb->add_u64(l_rgw_pubsub_events, "pubsub_events", "Pubsub events in store");
+  pcb->add_u64_counter(l_rgw_pubsub_push_ok, "pubsub_push_ok", "Pubsub events pushed to an endpoint");
+  pcb->add_u64_counter(l_rgw_pubsub_push_failed, "pubsub_push_failed", "Pubsub events failed to be pushed to an endpoint");
+  pcb->add_u64(l_rgw_pubsub_push_pending, "pubsub_push_pending", "Pubsub events pending reply from endpoint");
+  pcb->add_u64_counter(l_rgw_pubsub_missing_conf, "pubsub_missing_conf", "Pubsub events could not be handled because of missing configuration");
   
-  perfcounter = plb.create_perf_counters();
-  cct->get_perfcounters_collection()->add(perfcounter);
+  pcb->add_u64_counter(l_rgw_lua_script_ok, "lua_script_ok", "Successfull executions of Lua scripts");
+  pcb->add_u64_counter(l_rgw_lua_script_fail, "lua_script_fail", "Failed executions of Lua scripts");
+  pcb->add_u64(l_rgw_lua_current_vms, "lua_current_vms", "Number of Lua VMs currently being executed");
+}
+
+static void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
+  // description must match general rgw counters description above
+  lpcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+
+  lpcb->add_u64_counter(l_rgw_op_put, "put_ops", "Puts");
+  lpcb->add_u64_counter(l_rgw_op_put_b, "put_b", "Size of puts");
+  lpcb->add_time_avg(l_rgw_op_put_lat, "put_initial_lat", "Put latency");
+
+  lpcb->add_u64_counter(l_rgw_op_get, "get_ops", "Gets");
+  lpcb->add_u64_counter(l_rgw_op_get_b, "get_b", "Size of gets");
+  lpcb->add_time_avg(l_rgw_op_get_lat, "get_initial_lat", "Get latency");
+
+  lpcb->add_u64_counter(l_rgw_op_del_obj, "del_obj_ops", "Delete objects");
+  lpcb->add_u64_counter(l_rgw_op_del_obj_b, "del_obj_bytes", "Size of delete objects");
+  lpcb->add_time_avg(l_rgw_op_del_obj_lat, "del_obj_lat", "Delete object latency");
+
+  lpcb->add_u64_counter(l_rgw_op_del_bucket, "del_bucket_ops", "Delete Buckets");
+  lpcb->add_time_avg(l_rgw_op_del_bucket_lat, "del_bucket_lat", "Delete bucket latency");
+
+  lpcb->add_u64_counter(l_rgw_op_copy_obj, "copy_obj_ops", "Copy objects");
+  lpcb->add_u64_counter(l_rgw_op_copy_obj_b, "copy_obj_bytes", "Size of copy objects");
+  lpcb->add_time_avg(l_rgw_op_copy_obj_lat, "copy_obj_lat", "Copy object latency");
+
+  lpcb->add_u64_counter(l_rgw_op_list_obj, "list_obj_ops", "List objects");
+  lpcb->add_time_avg(l_rgw_op_list_obj_lat, "list_obj_lat", "List objects latency");
+
+  lpcb->add_u64_counter(l_rgw_op_list_buckets, "list_buckets_ops", "List buckets");
+  lpcb->add_time_avg(l_rgw_op_list_buckets_lat, "list_buckets_lat", "List buckets latency");
+}
+
+std::shared_ptr<PerfCounters> create_rgw_counters(const std::string& name, CephContext *cct) {
+  std::string_view key = ceph::perf_counters::key_name(name);
+  if (rgw_op_counters_key.compare(key) == 0) {
+    PerfCountersBuilder pcb(cct, name, l_rgw_op_first, l_rgw_op_last);
+    add_rgw_op_counters(&pcb);
+    std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
+    cct->get_perfcounters_collection()->add(new_counters.get());
+    return new_counters;
+  } else {
+    PerfCountersBuilder pcb(cct, name, l_rgw_first, l_rgw_last);
+    add_rgw_frontend_counters(&pcb);
+    std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
+    cct->get_perfcounters_collection()->add(new_counters.get());
+    return new_counters;
+  }
+}
+
+void frontend_counters_init(CephContext *cct) {
+  PerfCountersBuilder pcb(cct, "rgw", l_rgw_first, l_rgw_last);
+  add_rgw_frontend_counters(&pcb);
+  PerfCounters *new_counters = pcb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(new_counters);
+  perfcounter = new_counters;
+}
+
+namespace rgw::op_counters {
+
+PerfCounters *global_op_counters = NULL;
+
+void global_op_counters_init(CephContext *cct) {
+  PerfCountersBuilder pcb(cct, rgw_op_counters_key, l_rgw_op_first, l_rgw_op_last);
+  add_rgw_op_counters(&pcb);
+  PerfCounters *new_counters = pcb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(new_counters);
+  global_op_counters = new_counters;
+}
+
+void inc(std::shared_ptr<PerfCounters> labeled_counters, int idx, uint64_t v) {
+  if (labeled_counters) {
+    PerfCounters *counter = labeled_counters.get();
+    counter->inc(idx, v);
+  }
+  if (global_op_counters) {
+    global_op_counters->inc(idx, v);
+  }
+}
+
+void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, utime_t amt) {
+  if (labeled_counters) {
+    PerfCounters *counter = labeled_counters.get();
+    counter->tinc(idx, amt);
+  }
+  if (global_op_counters) {
+    global_op_counters->tinc(idx, amt);
+  }
+}
+
+void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, ceph::timespan amt) {
+  if (labeled_counters) {
+    PerfCounters *counter = labeled_counters.get();
+    counter->tinc(idx, amt);
+  }
+  if (global_op_counters) {
+    global_op_counters->tinc(idx, amt);
+  }
+}
+
+} // namespace rgw::op_counters
+
+int rgw_perf_start(CephContext *cct)
+{
+  frontend_counters_init(cct);
+
+  bool cache_enabled = cct->_conf.get_val<bool>("rgw_perf_counters_cache");
+  if (cache_enabled) {
+    uint64_t target_size = cct->_conf.get_val<uint64_t>("rgw_perf_counters_cache_size");
+    perf_counters_cache = new ceph::perf_counters::PerfCountersCache(cct, target_size, create_rgw_counters); 
+  }
+
+  rgw::op_counters::global_op_counters_init(cct);
   return 0;
 }
 
@@ -74,5 +176,5 @@ void rgw_perf_stop(CephContext *cct)
   ceph_assert(perfcounter);
   cct->get_perfcounters_collection()->remove(perfcounter);
   delete perfcounter;
+  delete perf_counters_cache;
 }
-
diff --git a/src/rgw/rgw_perf_counters.h b/src/rgw/rgw_perf_counters.h
index 3c4e4e97f023..49f7e4d4218b 100644
--- a/src/rgw/rgw_perf_counters.h
+++ b/src/rgw/rgw_perf_counters.h
@@ -4,25 +4,23 @@
 #pragma once
 
 #include "include/common_fwd.h"
+#include "common/perf_counters_cache.h"
+#include "common/perf_counters_key.h"
 
 extern PerfCounters *perfcounter;
+extern ceph::perf_counters::PerfCountersCache *perf_counters_cache;
+extern std::string rgw_op_counters_key;
 
 extern int rgw_perf_start(CephContext *cct);
 extern void rgw_perf_stop(CephContext *cct);
+extern void frontend_counters_init(CephContext *cct);
+extern std::shared_ptr<PerfCounters> create_rgw_counters(const std::string& name, CephContext *cct);
 
 enum {
   l_rgw_first = 15000,
   l_rgw_req,
   l_rgw_failed_req,
 
-  l_rgw_get,
-  l_rgw_get_b,
-  l_rgw_get_lat,
-
-  l_rgw_put,
-  l_rgw_put_b,
-  l_rgw_put_lat,
-
   l_rgw_qlen,
   l_rgw_qactive,
 
@@ -58,3 +56,57 @@ enum {
   l_rgw_last,
 };
 
+enum {
+  l_rgw_op_first = 16000,
+
+  l_rgw_op_put,
+  l_rgw_op_put_b,
+  l_rgw_op_put_lat,
+
+  l_rgw_op_get,
+  l_rgw_op_get_b,
+  l_rgw_op_get_lat,
+
+  l_rgw_op_del_obj,
+  l_rgw_op_del_obj_b,
+  l_rgw_op_del_obj_lat,
+
+  l_rgw_op_del_bucket,
+  l_rgw_op_del_bucket_lat,
+
+  l_rgw_op_copy_obj,
+  l_rgw_op_copy_obj_b,
+  l_rgw_op_copy_obj_lat,
+
+  l_rgw_op_list_obj,
+  l_rgw_op_list_obj_lat,
+
+  l_rgw_op_list_buckets,
+  l_rgw_op_list_buckets_lat,
+
+  l_rgw_op_last
+};
+
+namespace rgw::op_counters {
+
+extern PerfCounters *global_op_counters;
+
+void global_op_counters_init(CephContext *cct);
+
+template <std::size_t Count>
+std::shared_ptr<PerfCounters> get(ceph::perf_counters::label_pair (&&labels)[Count]) {
+  if (perf_counters_cache) {
+    std::string key = ceph::perf_counters::key_create(rgw_op_counters_key, std::move(labels));
+    return perf_counters_cache->get(key);
+  } else {
+    return std::shared_ptr<PerfCounters>(nullptr);
+  }
+}
+
+void inc(std::shared_ptr<PerfCounters> labeled_counters, int idx, uint64_t v);
+
+void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, utime_t);
+
+void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, ceph::timespan amt);
+
+} // namespace rgw::op_counters
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 09281ab2dbf5..f8e2f2a1b011 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -834,6 +834,13 @@ add_executable(unittest_perf_counters
 add_ceph_unittest(unittest_perf_counters)
 target_link_libraries(unittest_perf_counters global)
 
+# unittest_perf_counters_cache
+add_executable(unittest_perf_counters_cache
+  test_perf_counters_cache.cc
+  )
+add_ceph_unittest(unittest_perf_counters_cache)
+target_link_libraries(unittest_perf_counters_cache global)
+
 # unittest_ceph_crypto
 add_executable(unittest_ceph_crypto
   ceph_crypto.cc)
diff --git a/src/test/test_perf_counters_cache.cc b/src/test/test_perf_counters_cache.cc
new file mode 100644
index 000000000000..16d92bd7d431
--- /dev/null
+++ b/src/test/test_perf_counters_cache.cc
@@ -0,0 +1,1063 @@
+#include "common/perf_counters_cache.h"
+#include "common/perf_counters_key.h"
+#include "common/admin_socket_client.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/msgr.h" // for CEPH_ENTITY_TYPE_CLIENT
+#include "gtest/gtest.h"
+
+using namespace ceph::perf_counters;
+
+int main(int argc, char **argv) {
+  std::map<std::string,std::string> defaults = {
+    { "admin_socket", get_rand_socket_path() }
+  };
+  std::vector<const char*> args;
+  auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
+			 CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE|
+			 CINIT_FLAG_NO_CCT_PERF_COUNTERS);
+  common_init_finish(g_ceph_context);
+  ::testing::FLAGS_gtest_death_test_style = "threadsafe";
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+enum {
+  TEST_PERFCOUNTERS1_ELEMENT_FIRST = 200,
+  TEST_PERFCOUNTERS_COUNTER,
+  TEST_PERFCOUNTERS_TIME,
+  TEST_PERFCOUNTERS_TIME_AVG,
+  TEST_PERFCOUNTERS1_ELEMENT_LAST,
+};
+
+std::string sd(const char *c)
+{
+  std::string ret(c);
+  std::string::size_type sz = ret.size();
+  for (std::string::size_type i = 0; i < sz; ++i) {
+    if (ret[i] == '\'') {
+      ret[i] = '\"';
+    }
+  }
+  return ret;
+}
+
+void add_test_counters(PerfCountersBuilder *pcb) {
+  pcb->add_u64(TEST_PERFCOUNTERS_COUNTER, "test_counter");
+  pcb->add_time(TEST_PERFCOUNTERS_TIME, "test_time");
+  pcb->add_time_avg(TEST_PERFCOUNTERS_TIME_AVG, "test_time_avg");
+}
+
+static std::shared_ptr<PerfCounters> create_test_counters(const std::string& name, CephContext *cct) {
+  PerfCountersBuilder pcb(cct, name, TEST_PERFCOUNTERS1_ELEMENT_FIRST, TEST_PERFCOUNTERS1_ELEMENT_LAST);
+  add_test_counters(&pcb);
+  std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
+  cct->get_perfcounters_collection()->add(new_counters.get());
+  return new_counters;
+}
+
+static PerfCountersCache* setup_test_perf_counters_cache(CephContext *cct, uint64_t target_size = 100)
+{
+  return new PerfCountersCache(cct, target_size, create_test_counters);
+}
+
+
+void cleanup_test(PerfCountersCache *pcc) {
+  delete pcc;
+}
+
+TEST(PerfCountersCache, NoCacheTest) {
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump" })", &message));
+  ASSERT_EQ("{}\n", message);
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter schema" })", &message));
+  ASSERT_EQ("{}\n", message);
+}
+
+TEST(PerfCountersCache, TestEviction) {
+  PerfCountersCache *pcc = setup_test_perf_counters_cache(g_ceph_context, 4);
+  std::string label1 = key_create("key1", {{"label1", "val1"}});
+  std::string label2 = key_create("key2", {{"label2", "val2"}});
+  std::string label3 = key_create("key3", {{"label3", "val3"}});
+  std::string label4 = key_create("key4", {{"label4", "val4"}});
+  std::string label5 = key_create("key5", {{"label5", "val5"}});
+  std::string label6 = key_create("key6", {{"label6", "val6"}});
+
+  pcc->set_counter(label1, TEST_PERFCOUNTERS_COUNTER, 0);
+  std::shared_ptr<PerfCounters> counter = pcc->get(label2);
+  counter->set(TEST_PERFCOUNTERS_COUNTER, 0);
+  pcc->set_counter(label3, TEST_PERFCOUNTERS_COUNTER, 0);
+  pcc->set_counter(label4, TEST_PERFCOUNTERS_COUNTER, 0);
+
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key3": [
+        {
+            "labels": {
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key4": [
+        {
+            "labels": {
+                "label4": "val4"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter schema", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key3": [
+        {
+            "labels": {
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key4": [
+        {
+            "labels": {
+                "label4": "val4"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  pcc->set_counter(label5, TEST_PERFCOUNTERS_COUNTER, 0);
+  pcc->set_counter(label6, TEST_PERFCOUNTERS_COUNTER, 0);
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key3": [
+        {
+            "labels": {
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key4": [
+        {
+            "labels": {
+                "label4": "val4"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key5": [
+        {
+            "labels": {
+                "label5": "val5"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key6": [
+        {
+            "labels": {
+                "label6": "val6"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter schema", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key3": [
+        {
+            "labels": {
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key4": [
+        {
+            "labels": {
+                "label4": "val4"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key5": [
+        {
+            "labels": {
+                "label5": "val5"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key6": [
+        {
+            "labels": {
+                "label6": "val6"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ]
+}
+)", message);
+  cleanup_test(pcc);
+}
+
+TEST(PerfCountersCache, TestLabeledCounters) {
+  PerfCountersCache *pcc = setup_test_perf_counters_cache(g_ceph_context);
+  std::string label1 = key_create("key1", {{"label1", "val1"}});
+  std::string label2 = key_create("key2", {{"label2", "val2"}});
+  std::string label3 = key_create("key3", {{"label3", "val3"}});
+
+  // test inc()
+  pcc->inc(label1, TEST_PERFCOUNTERS_COUNTER, 1);
+  pcc->inc(label2, TEST_PERFCOUNTERS_COUNTER, 2);
+
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 1,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": 2,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter schema", "format": "raw"  })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  // tests to ensure there is no interaction with normal perf counters
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "perf dump", "format": "raw" })", &message));
+  ASSERT_EQ("{}\n", message);
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "perf schema", "format": "raw" })", &message));
+  ASSERT_EQ("{}\n", message);
+
+  // test dec()
+  pcc->dec(label2, TEST_PERFCOUNTERS_COUNTER, 1);
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 1,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": 1,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+
+  // test set_counters()
+  pcc->set_counter(label3, TEST_PERFCOUNTERS_COUNTER, 4);
+  uint64_t val = pcc->get_counter(label3, TEST_PERFCOUNTERS_COUNTER);
+  ASSERT_EQ(val, 4);
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 1,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": 1,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "key3": [
+        {
+            "labels": {
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 4,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  cleanup_test(pcc);
+}
+
+TEST(PerfCountersCache, TestLabeledTimes) {
+  PerfCountersCache *pcc = setup_test_perf_counters_cache(g_ceph_context);
+  std::string label1 = key_create("key1", {{"label1", "val1"}});
+  std::string label2 = key_create("key2", {{"label2", "val2"}});
+  std::string label3 = key_create("key3", {{"label3", "val3"}});
+
+  // test inc()
+  pcc->tinc(label1, TEST_PERFCOUNTERS_TIME, utime_t(100,0));
+  pcc->tinc(label2, TEST_PERFCOUNTERS_TIME, utime_t(200,0));
+
+  //tinc() that takes a ceph_timespan
+  ceph::timespan ceph_timespan = std::chrono::seconds(10);
+  pcc->tinc(label1, TEST_PERFCOUNTERS_TIME, ceph_timespan);
+
+  pcc->tinc(label1, TEST_PERFCOUNTERS_TIME_AVG, utime_t(200,0));
+  pcc->tinc(label1, TEST_PERFCOUNTERS_TIME_AVG, utime_t(400,0));
+  pcc->tinc(label2, TEST_PERFCOUNTERS_TIME_AVG, utime_t(100,0));
+  pcc->tinc(label2, TEST_PERFCOUNTERS_TIME_AVG, utime_t(200,0));
+
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 110.000000000,
+                "test_time_avg": {
+                    "avgcount": 2,
+                    "sum": 600.000000000,
+                    "avgtime": 300.000000000
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 200.000000000,
+                "test_time_avg": {
+                    "avgcount": 2,
+                    "sum": 300.000000000,
+                    "avgtime": 150.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter schema", "format": "raw"  })", &message));
+  ASSERT_EQ(R"({
+    "key1": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ],
+    "key2": [
+        {
+            "labels": {
+                "label2": "val2"
+            },
+            "counters": {
+                "test_counter": {
+                    "type": 2,
+                    "metric_type": "gauge",
+                    "value_type": "integer",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time": {
+                    "type": 1,
+                    "metric_type": "gauge",
+                    "value_type": "real",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                },
+                "test_time_avg": {
+                    "type": 5,
+                    "metric_type": "gauge",
+                    "value_type": "real-integer-pair",
+                    "description": "",
+                    "nick": "",
+                    "priority": 0,
+                    "units": "none"
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  // test tset() & tget()
+  pcc->tset(label1, TEST_PERFCOUNTERS_TIME, utime_t(500,0));
+  utime_t label1_time = pcc->tget(label1, TEST_PERFCOUNTERS_TIME);
+  ASSERT_EQ(utime_t(500,0), label1_time);
+
+  cleanup_test(pcc);
+}
+
+TEST(PerfCountersCache, TestLabelStrings) {
+  AdminSocketClient client(get_rand_socket_path());
+  std::string message;
+  PerfCountersCache *pcc = setup_test_perf_counters_cache(g_ceph_context);
+  std::string empty_key = "";
+
+  // empty string as should not create a labeled entry
+  EXPECT_DEATH(pcc->set_counter(empty_key, TEST_PERFCOUNTERS_COUNTER, 1), "");
+  EXPECT_DEATH(pcc->get(empty_key), "");
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ("{}\n", message);
+
+  // key name but no labels at all should not create a labeled entry
+  std::string only_key = "only_key";
+  // run an op on an invalid key name to make sure nothing happens
+  EXPECT_DEATH(pcc->set_counter(only_key, TEST_PERFCOUNTERS_COUNTER, 4), "");
+  EXPECT_DEATH(pcc->get(only_key), "");
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ("{}\n", message);
+
+  // test valid key name with multiple valid label pairs
+  std::string label1 = key_create("good_ctrs", {{"label3", "val3"}, {"label2", "val4"}});
+  pcc->set_counter(label1, TEST_PERFCOUNTERS_COUNTER, 8);
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "good_ctrs": [
+        {
+            "labels": {
+                "label2": "val4",
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 8,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  // test empty val in a label pair will get the label pair added into the perf counters cache but empty key will not
+  std::string label2 = key_create("bad_ctrs1", {{"label3", "val4"}, {"label1", ""}});
+  EXPECT_DEATH(pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2), "");
+
+  std::string label3 = key_create("bad_ctrs2", {{"", "val4"}, {"label1", "val1"}});
+  EXPECT_DEATH(pcc->set_counter(label3, TEST_PERFCOUNTERS_COUNTER, 2), "");
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "good_ctrs": [
+        {
+            "labels": {
+                "label2": "val4",
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 8,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  // test empty keys in each of the label pairs will not get the label added into the perf counters cache
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "good_ctrs": [
+        {
+            "labels": {
+                "label2": "val4",
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 8,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  // a key with a somehow odd number of entries after the the key name will omit final unfinished label pair
+  std::string label5 = "too_many_delimiters";
+  label5 += '\0';
+  label5 += "label1";
+  label5 += '\0';
+  label5 += "val1";
+  label5 += '\0';
+  label5 += "label2";
+  label5 += '\0';
+  pcc->set_counter(label5, TEST_PERFCOUNTERS_COUNTER, 0);
+
+  ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
+  ASSERT_EQ(R"({
+    "good_ctrs": [
+        {
+            "labels": {
+                "label2": "val4",
+                "label3": "val3"
+            },
+            "counters": {
+                "test_counter": 8,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
+    "too_many_delimiters": [
+        {
+            "labels": {
+                "label1": "val1"
+            },
+            "counters": {
+                "test_counter": 0,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ]
+}
+)", message);
+
+  cleanup_test(pcc);
+}

From 6284322af41b2d76ca2e3fcf56faef4d55701250 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Wed, 23 Aug 2023 01:05:37 -0400
Subject: [PATCH 0165/2492] exporter: check key exists before json::object at()

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 src/exporter/DaemonMetricCollector.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/exporter/DaemonMetricCollector.cc b/src/exporter/DaemonMetricCollector.cc
index ebe85c3041e5..0ce5419a7764 100644
--- a/src/exporter/DaemonMetricCollector.cc
+++ b/src/exporter/DaemonMetricCollector.cc
@@ -160,8 +160,10 @@ void DaemonMetricCollector::dump_asok_metrics() {
             labels.insert(multisite_labels_and_name.first.begin(), multisite_labels_and_name.first.end());
             counter_name = multisite_labels_and_name.second;
           }
-          auto perf_values = counters_values.at(counter_name_init);
-          dump_asok_metric(counter_group, perf_values, counter_name, labels);
+          if (counters_values.find(counter_name_init) != counters_values.end()) {
+            auto perf_values = counters_values.at(counter_name_init);
+            dump_asok_metric(counter_group, perf_values, counter_name, labels);
+          }
         }
       }
     }

From 56c7cc8c59339164c052a432cd59a55aeca0dbab Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Fri, 8 Sep 2023 11:16:02 -0400
Subject: [PATCH 0166/2492] rgw: misc labeled op counters work

Highlights of this commit include:
- splitting the rgw perf counters cache int two
caches for bucket labeled and user labeled op counters
- add config overrides to verify suite for CI
- add tenant label for op counters
- misc cleanup cleanup
- add docs for rgw metrics

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 doc/dev/perf_counters.rst             |    4 +
 doc/radosgw/index.rst                 |    1 +
 doc/radosgw/metrics.rst               |  204 +++
 qa/suites/rgw/verify/overrides.yaml   |    2 +
 src/common/options/rgw.yaml.in        |   62 +-
 src/common/perf_counters.cc           |    7 +-
 src/common/perf_counters_cache.cc     |    7 +-
 src/exporter/DaemonMetricCollector.cc |    5 +-
 src/rgw/rgw_file.cc                   |    7 +-
 src/rgw/rgw_file_int.h                |    2 +
 src/rgw/rgw_op.cc                     |   62 +-
 src/rgw/rgw_perf_counters.cc          |  123 +-
 src/rgw/rgw_perf_counters.h           |   29 +-
 src/rgw/vstart.sh.swift               | 1930 +++++++++++++++++++++++++
 src/test/test_perf_counters_cache.cc  |   54 +-
 15 files changed, 2376 insertions(+), 123 deletions(-)
 create mode 100644 doc/radosgw/metrics.rst
 create mode 100755 src/rgw/vstart.sh.swift

diff --git a/doc/dev/perf_counters.rst b/doc/dev/perf_counters.rst
index a64d14d33bd0..1bcc6120e4af 100644
--- a/doc/dev/perf_counters.rst
+++ b/doc/dev/perf_counters.rst
@@ -1,3 +1,5 @@
+.. _Perf Counters:
+
 ===============
  Perf counters
 ===============
@@ -200,6 +202,8 @@ The actual dump is similar to the schema, except that average values are grouped
    }
  }
 
+.. _Labeled Perf Counters:
+
 Labeled Perf Counters
 ---------------------
 
diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst
index 70443620237a..ed67413646d8 100644
--- a/doc/radosgw/index.rst
+++ b/doc/radosgw/index.rst
@@ -84,4 +84,5 @@ Storage Cluster with one API and then retrieve that data with the other API.
    Lua Scripting <lua-scripting>
    D3N Data Cache <d3n_datacache>
    Cloud Transition <cloud-transition>
+   Metrics <metrics>
 
diff --git a/doc/radosgw/metrics.rst b/doc/radosgw/metrics.rst
new file mode 100644
index 000000000000..75ef782fa6aa
--- /dev/null
+++ b/doc/radosgw/metrics.rst
@@ -0,0 +1,204 @@
+=======
+Metrics
+=======
+
+The Ceph Object Gateway uses :ref:`Perf Counters` to track metrics. The counters can be labeled (:ref:`Labeled Perf Counters`). When counters are labeled, they are stored in the Ceph Object Gateway specific caches.
+
+These metrics can be sent to the time series database Prometheus to visualize a cluster wide view of usage data (ex: number of S3 put operations on a specific bucket) over time.
+
+.. contents::
+
+Op Metrics
+==========
+
+The following metrics related to S3 or Swift operations are tracked per Ceph Object Gateway.
+
+.. list-table:: Radosgw Op Metrics
+   :widths: 25 25 75
+   :header-rows: 1
+
+   * - Name
+     - Type
+     - Description
+   * - put_ops
+     - Counter
+     - Number of put operations
+   * - put_b
+     - Counter
+     - Number of bytes put
+   * - put_initial_lat
+     - Guage
+     - Total latency of put operations
+   * - get_ops
+     - Counter
+     - Number of get operations
+   * - get_b
+     - Counter
+     - Number of bytes from get requests
+   * - get_initial_lat
+     - Guage
+     - Total latency of get operations
+   * - del_obj_ops
+     - Counter
+     - Number of delete object operations
+   * - del_obj_bytes
+     - Counter
+     - Number of bytes deleted
+   * - del_obj_lat
+     - Guage
+     - Total latency of delete object operations
+   * - del_bucket_ops
+     - Counter
+     - Number of delete bucket operations
+   * - del_bucket_lat
+     - Guage
+     - Total latency of delete bucket operations
+   * - copy_obj_ops
+     - Counter
+     - Number of copy object operations
+   * - copy_obj_bytes
+     - Counter
+     - Number of bytes copied
+   * - copy_obj_lat
+     - Guage
+     - Total latency of copy object operations
+   * - list_object_ops
+     - Counter
+     - Number of list object operations
+   * - list_object_lat
+     - Guage
+     - Total latency of list object operations
+   * - list_bucket_ops
+     - Counter
+     - Number of list bucket operations
+   * - list_bucket_lat
+     - Guage
+     - Total latency of list bucket operations
+
+More information about op metrics can be seen in the ``rgw_op`` section of the output of the ``counter schema`` command.
+To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` section of the output of the ``counter dump`` command::
+
+    "rgw_op": [
+        {
+            "labels": {},
+            "counters": {
+                "put_ops": 2,
+                "put_b": 5327,
+                "put_initial_lat": {
+                    "avgcount": 2,
+                    "sum": 2.818064835,
+                    "avgtime": 1.409032417
+                },
+                "get_ops": 5,
+                "get_b": 5325,
+                "get_initial_lat": {
+                    "avgcount": 2,
+                    "sum": 0.003000069,
+                    "avgtime": 0.001500034
+                },
+                ...
+                "list_buckets_ops": 1,
+                "list_buckets_lat": {
+                    "avgcount": 1,
+                    "sum": 0.002300000,
+                    "avgtime": 0.002300000
+                }
+            }
+        },
+    ]
+
+Op Metrics Labels
+--------------------
+
+Op metrics can also be tracked per-user or per-bucket. These metrics are exported to Prometheus with labels like Bucket = {name} or User = {userid}::
+
+    "rgw_op": [
+        ...
+        {
+            "labels": {
+                "Bucket": "bucket1"
+            },
+            "counters": {
+                "put_ops": 2,
+                "put_b": 5327,
+                "put_initial_lat": {
+                    "avgcount": 2,
+                    "sum": 2.818064835,
+                    "avgtime": 1.409032417
+                },
+                "get_ops": 5,
+                "get_b": 5325,
+                "get_initial_lat": {
+                    "avgcount": 2,
+                    "sum": 0.003000069,
+                    "avgtime": 0.001500034
+                },
+                ...
+                "list_buckets_ops": 1,
+                "list_buckets_lat": {
+                    "avgcount": 1,
+                    "sum": 0.002300000,
+                    "avgtime": 0.002300000
+                }
+            }
+        },
+        ...
+    ]
+
+:ref:`rgw-multitenancy` allows to use buckets and users of the same name simultaneously. If a user or bucket lies under a tenant, a label for tenant in the form  Tenant = {tenantid} is added to the metric.
+
+In a large system with many users and buckets, it may not be tractable to export all metrics to Prometheus. For that reason, the collection of these labeled metrics is disabled by default.
+
+Once enabled, the working set of tracked users and buckets is constrained to limit memory and database usage. As a result, the collection of these labeled metrics will not always be reliable.
+
+
+User & Bucket Counter Caches
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To track op metrics by user the Ceph Object Gateway the config value ``rgw_user_counters_cache`` must be set to ``true``. 
+
+To track op metrics by bucket the Ceph Object Gateway the config value ``rgw_bucket_counters_cache`` must be set to ``true``. 
+
+These config values are set in Ceph via the command ``ceph config set client.rgw rgw_{user,bucket}_counters_cache true``
+
+Since the op metrics are labeled perf counters, they live in memory. If the Ceph Object Gateway is restarted or crashes, all counters in the Ceph Object Gateway, whether in a cache or not, are lost.
+
+User & Bucket Counter Cache Size & Eviction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Both ``rgw_user_counters_cache_size`` and ``rgw_bucket_counters_cache_size`` can be used to set number of entries in each cache.
+
+Counters are evicted from a cache once the number of counters in the cache are greater than the cache size config variable. The counters that are evicted are the least recently used (LRU). 
+
+For example if the number of buckets exceeded ``rgw_bucket_counters_cache_size`` by 1 and the counters with label ``bucket1`` were the last to be updated, the counters for ``bucket1`` would be evicted from the cache. If S3 operations tracked by the op metrics were done on ``bucket1`` after eviction, all of the metrics in the cache for ``bucket1`` would start at 0.
+
+Cache sizing can depend on a number of factors. These factors include:
+
+#. Number of users in the cluster
+#. Number of buckets in the cluster
+#. Memory usage of the Ceph Object Gateway
+#. Disk and memory usage of Promtheus. 
+
+To help calculate the Ceph Object Gateway's memory usage of a cache, it should be noted that each cache entry, encompassing all of the op metrics, is 1360 bytes. This is an estimate and subject to change if metrics are added or removed from the op metrics list.
+
+Sending Metrics to Prometheus
+=============================
+
+To get metrics from a Ceph Object Gateway into the time series database Prometheus, the ceph-exporter daemon must be running and configured to scrape the Radogw's admin socket.
+
+The ceph-exporter daemon scrapes the Ceph Object Gateway's admin socket at a regular interval, defined by the config variable ``exporter_stats_period``.
+
+Prometheus has a configurable interval in which it scrapes the exporter (see: https://prometheus.io/docs/prometheus/latest/configuration/configuration/).
+
+Config Reference
+================
+The following rgw op metrics related settings can be set via ``ceph config set client.rgw CONFIG_VARIABLE VALUE``.
+
+.. confval:: rgw_user_counters_cache
+.. confval:: rgw_user_counters_cache_size
+.. confval:: rgw_bucket_counters_cache
+.. confval:: rgw_bucket_counters_cache_size
+
+The following are notable ceph-exporter related settings can be set via ``ceph config set global CONFIG_VARIABLE VALUE``.
+
+.. confval:: exporter_stats_period
diff --git a/qa/suites/rgw/verify/overrides.yaml b/qa/suites/rgw/verify/overrides.yaml
index eac70f30c9f3..a881ce5cbe28 100644
--- a/qa/suites/rgw/verify/overrides.yaml
+++ b/qa/suites/rgw/verify/overrides.yaml
@@ -9,6 +9,8 @@ overrides:
         rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
         rgw crypt require ssl: false
         rgw torrent flag: true
+        rgw user counters cache: true
+        rgw bucket counters cache: true
   rgw:
     compression type: random
     storage classes: LUKEWARM, FROZEN
diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index f2f85b648f8b..1dfb96228944 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3829,26 +3829,8 @@ options:
     other form of policies that Amazon does, so if you are mirroring
     policies between RGW and AWS, you may wish to set this to false.
   default: true
-- name: rgw_perf_counters_cache
-  type: bool
-  level: dev
-  default: false
-  desc: enable rgw labeled perf counters cache
-  long desc: If set to true, rgw creates labeled perf counters and stores them 
-    in an rgw specific labeled perf counters cache.
-  see_also:
-  - rgw_perf_counters_cache_size
-  services:
-  - rgw
-  with_legacy: true
-- name: rgw_perf_counters_cache_size
-  type: uint
-  level: advanced
-  desc: Number of labeled perf counters the rgw perf counters cache can store
-  default: 10000
   services:
   - rgw
-  with_legacy: true
 - name: rgw_d4n_host
   type: str
   level: advanced
@@ -3914,3 +3896,47 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_user_counters_cache
+  type: bool
+  level: dev
+  default: false
+  desc: enable a rgw perf counters cache for counters with user label
+  long desc: If set to true, rgw creates perf counters with a label for the user and stores them
+    in a perf counters cache. This perf counters cache contains only perf counters labeled by user.
+  see_also:
+  - rgw_user_counters_cache_size
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_user_counters_cache_size
+  type: uint
+  level: advanced
+  desc: Number of labeled perf counters the user perf counters cache can store
+  default: 10000
+  services:
+  - rgw
+  see_also:
+  - rgw_user_counters_cache
+  with_legacy: true
+- name: rgw_bucket_counters_cache
+  type: bool
+  level: dev
+  default: false
+  desc: enable a rgw perf counters cache for counters with bucket label
+  long desc: If set to true, rgw creates perf counters with a label for the bucket and stores them
+    in a perf counters cache. This perf counters cache contains only perf counters labeled by bucket.
+  see_also:
+  - rgw_bucket_counters_cache_size
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_bucket_counters_cache_size
+  type: uint
+  level: advanced
+  desc: Number of labeled perf counters the bucket perf counters cache can store
+  default: 10000
+  services:
+  - rgw
+  see_also:
+  - rgw_bucket_counters_cache
+  with_legacy: true
diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc
index 81bf3284bdae..b5e361b505cd 100644
--- a/src/common/perf_counters.cc
+++ b/src/common/perf_counters.cc
@@ -135,12 +135,7 @@ void PerfCountersCollectionImpl::dump_formatted_generic(
     const std::string &counter) const
 {
   f->open_object_section("perfcounter_collection");
-  // close out all of counters collection immediately if collection is empty
-  if (m_loggers.empty()) {
-    f->close_section(); // all of counters collection
-    return;
-  }
-
+  
   if (dump_labeled) {
     std::string prev_key_name;
     for (auto l = m_loggers.begin(); l != m_loggers.end(); ++l) {
diff --git a/src/common/perf_counters_cache.cc b/src/common/perf_counters_cache.cc
index e0810508ce7f..946b5f5b8edb 100644
--- a/src/common/perf_counters_cache.cc
+++ b/src/common/perf_counters_cache.cc
@@ -5,17 +5,16 @@ namespace ceph::perf_counters {
 
 void PerfCountersCache::check_key(const std::string &key) {
   std::string_view key_name = ceph::perf_counters::key_name(key);
-  // return false for empty key name
+  // don't accept an empty key name
   assert(key_name != "");
 
-  // if there are no labels key name is not valid
+  // if there are no labels, key name is not valid
   auto key_labels = ceph::perf_counters::key_labels(key);
   assert(key_labels.begin() != key_labels.end());
 
-  // don't accept keys where any labels have an empty label name
+  // don't accept keys where any labels in the key have an empty key name
   for (auto key_label : key_labels) {
     assert(key_label.first != "");
-    assert(key_label.second != "");
   }
 }
 
diff --git a/src/exporter/DaemonMetricCollector.cc b/src/exporter/DaemonMetricCollector.cc
index 0ce5419a7764..23a0dd550f1c 100644
--- a/src/exporter/DaemonMetricCollector.cc
+++ b/src/exporter/DaemonMetricCollector.cc
@@ -160,8 +160,9 @@ void DaemonMetricCollector::dump_asok_metrics() {
             labels.insert(multisite_labels_and_name.first.begin(), multisite_labels_and_name.first.end());
             counter_name = multisite_labels_and_name.second;
           }
-          if (counters_values.find(counter_name_init) != counters_values.end()) {
-            auto perf_values = counters_values.at(counter_name_init);
+          auto counters_values_itr = counters_values.find(counter_name_init);
+          if (counters_values_itr != counters_values.end()) {
+            auto perf_values = counters_values_itr->value();
             dump_asok_metric(counter_group, perf_values, counter_name, labels);
           }
         }
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 92aa66f060eb..ca149ad836b8 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1838,7 +1838,8 @@ namespace rgw {
     ceph_assert(! dlo_manifest);
     ceph_assert(! slo_info);
 
-    rgw::op_counters::global_op_counters->inc(l_rgw_op_put);
+    counters = rgw::op_counters::get(state);
+    rgw::op_counters::inc(counters, l_rgw_op_put, 1);
     op_ret = -EINVAL;
 
     if (state->object->empty()) {
@@ -1944,7 +1945,7 @@ namespace rgw {
     real_time appx_t = real_clock::now();
 
     state->obj_size = bytes_written;
-    rgw::op_counters::global_op_counters->inc(l_rgw_op_put_b, state->obj_size);
+    rgw::op_counters::inc(counters, l_rgw_op_put_b, state->obj_size);
 
     // flush data in filters
     op_ret = filter->process({}, state->obj_size);
@@ -2027,7 +2028,7 @@ namespace rgw {
     }
 
   done:
-    rgw::op_counters::global_op_counters->tinc(l_rgw_op_put_lat, state->time_elapsed());
+    rgw::op_counters::tinc(counters, l_rgw_op_put_lat, state->time_elapsed());
     return op_ret;
   } /* exec_finish */
 
diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h
index 6ecd4b2447da..91c858e5b3bd 100644
--- a/src/rgw/rgw_file_int.h
+++ b/src/rgw/rgw_file_int.h
@@ -36,6 +36,7 @@
 #include "rgw_putobj_processor.h"
 #include "rgw_aio_throttle.h"
 #include "rgw_compression.h"
+#include "rgw_perf_counters.h"
 
 
 /* XXX
@@ -2485,6 +2486,7 @@ class RGWWriteRequest : public RGWLibContinuedReq,
   off_t real_ofs;
   size_t bytes_written;
   bool eio;
+  rgw::op_counters::CountersContainer counters;
 
   RGWWriteRequest(rgw::sal::Driver* driver, const RGWProcessEnv& penv,
 		  std::unique_ptr<rgw::sal::User> _user,
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index cd347502e76c..29a1f36c0273 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1692,8 +1692,8 @@ int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
     return 0;
   }
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_get_b, cur_end - cur_ofs);
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_get_b, cur_end - cur_ofs);
   filter->fixup_range(cur_ofs, cur_end);
   op_ret = read_op->iterate(this, cur_ofs, cur_end, filter, s->yield);
   if (op_ret >= 0)
@@ -1766,7 +1766,8 @@ static int iterate_user_manifest_parts(const DoutPrefixProvider *dpp,
 	found_end = true;
       }
 
-      rgw::op_counters::global_op_counters->tinc(l_rgw_op_get_lat, 
+      rgw::op_counters::CountersContainer counters;
+      rgw::op_counters::tinc(counters, l_rgw_op_get_lat, 
                             (ceph_clock_now() - start_time));
 
       if (found_start && !handled_end) {
@@ -1862,8 +1863,9 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
       found_end = true;
     }
 
-    rgw::op_counters::global_op_counters->tinc(l_rgw_op_get_lat, 
-                      (ceph_clock_now() - start_time));
+    rgw::op_counters::CountersContainer counters;
+    rgw::op_counters::tinc(counters, l_rgw_op_get_lat, 
+                          (ceph_clock_now() - start_time));
 
     if (found_start) {
       if (cb) {
@@ -2210,8 +2212,8 @@ void RGWGetObj::execute(optional_yield y)
   std::unique_ptr<RGWGetObj_Filter> run_lua;
   map<string, bufferlist>::iterator attr_iter;
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_get, 1);
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_get, 1);
 
   std::unique_ptr<rgw::sal::Object::ReadOp> read_op(s->object->get_read_op());
 
@@ -2409,14 +2411,14 @@ void RGWGetObj::execute(optional_yield y)
     return;
   }
 
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_get_b, end-ofs);
+  rgw::op_counters::inc(counters, l_rgw_op_get_b, end-ofs);
 
   op_ret = read_op->iterate(this, ofs_x, end_x, filter, s->yield);
 
   if (op_ret >= 0)
     op_ret = filter->flush();
 
-  rgw::op_counters::tinc(labeled_counters, l_rgw_op_get_lat, s->time_elapsed());
+  rgw::op_counters::tinc(counters, l_rgw_op_get_lat, s->time_elapsed());
 
   if (op_ret < 0) {
     goto done_err;
@@ -2493,8 +2495,8 @@ void RGWListBuckets::execute(optional_yield y)
 
   const uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
 
-  auto labeled_counters = rgw::op_counters::get({{"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_list_buckets, 1);
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_list_buckets, 1);
 
   auto g = make_scope_guard([this, &started] {
       if (!started) {
@@ -2573,7 +2575,7 @@ void RGWListBuckets::execute(optional_yield y)
     handle_listing_chunk(listing.buckets);
   } while (!marker.empty() && !done);
   
-  rgw::op_counters::tinc(labeled_counters, l_rgw_op_list_buckets_lat, s->time_elapsed());
+  rgw::op_counters::tinc(counters, l_rgw_op_list_buckets_lat, s->time_elapsed());
 }
 
 void RGWGetUsage::execute(optional_yield y)
@@ -3059,9 +3061,9 @@ void RGWListBucket::execute(optional_yield y)
     common_prefixes = std::move(results.common_prefixes);
   }
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_list_obj, 1);
-  rgw::op_counters::tinc(labeled_counters, l_rgw_op_list_obj_lat, s->time_elapsed());
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_list_obj, 1);
+  rgw::op_counters::tinc(counters, l_rgw_op_list_obj_lat, s->time_elapsed());
 }
 
 int RGWGetBucketLogging::verify_permission(optional_yield y)
@@ -3595,9 +3597,9 @@ void RGWDeleteBucket::execute(optional_yield y)
       op_ret = 0;
   }
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_del_bucket, 1);
-  rgw::op_counters::tinc(labeled_counters, l_rgw_op_del_bucket_lat, s->time_elapsed());
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_del_bucket, 1);
+  rgw::op_counters::tinc(counters, l_rgw_op_del_bucket_lat, s->time_elapsed());
 
   return;
 }
@@ -4025,14 +4027,14 @@ void RGWPutObj::execute(optional_yield y)
   off_t fst;
   off_t lst;
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
+  auto counters = rgw::op_counters::get(s);
 
   bool need_calc_md5 = (dlo_manifest == NULL) && (slo_info == NULL);
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_put, 1);
+  rgw::op_counters::inc(counters, l_rgw_op_put, 1);
 
   // report latency on return
   auto put_lat = make_scope_guard([&] {
-      rgw::op_counters::tinc(labeled_counters, l_rgw_op_put_lat, s->time_elapsed());
+      rgw::op_counters::tinc(counters, l_rgw_op_put_lat, s->time_elapsed());
     });
 
   op_ret = -EINVAL;
@@ -4307,7 +4309,7 @@ void RGWPutObj::execute(optional_yield y)
   s->obj_size = ofs;
   s->object->set_obj_size(ofs);
 
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_put_b, s->obj_size);
+  rgw::op_counters::inc(counters, l_rgw_op_put_b, s->obj_size);
 
   op_ret = do_aws4_auth_completion();
   if (op_ret < 0) {
@@ -5264,10 +5266,10 @@ void RGWDeleteObj::execute(optional_yield y)
       op_ret = 0;
     }
 
-    auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-    rgw::op_counters::inc(labeled_counters, l_rgw_op_del_obj, 1);
-    rgw::op_counters::inc(labeled_counters, l_rgw_op_del_obj_b, obj_size);
-    rgw::op_counters::tinc(labeled_counters, l_rgw_op_del_obj_lat, s->time_elapsed());
+    auto counters = rgw::op_counters::get(s);
+    rgw::op_counters::inc(counters, l_rgw_op_del_obj, 1);
+    rgw::op_counters::inc(counters, l_rgw_op_del_obj_b, obj_size);
+    rgw::op_counters::tinc(counters, l_rgw_op_del_obj_lat, s->time_elapsed());
 
     // send request to notification manager
     int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
@@ -5729,10 +5731,10 @@ void RGWCopyObj::execute(optional_yield y)
     // too late to rollback operation, hence op_ret is not set here
   }
 
-  auto labeled_counters = rgw::op_counters::get({{"Bucket", s->bucket_name}, {"User", s->user->get_id().id}});
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_copy_obj, 1);
-  rgw::op_counters::inc(labeled_counters, l_rgw_op_copy_obj_b, obj_size);
-  rgw::op_counters::tinc(labeled_counters, l_rgw_op_copy_obj_lat, s->time_elapsed());
+  auto counters = rgw::op_counters::get(s);
+  rgw::op_counters::inc(counters, l_rgw_op_copy_obj, 1);
+  rgw::op_counters::inc(counters, l_rgw_op_copy_obj_b, obj_size);
+  rgw::op_counters::tinc(counters, l_rgw_op_copy_obj_lat, s->time_elapsed());
 }
 
 int RGWGetACLs::verify_permission(optional_yield y)
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index aca56a60946d..f2d245c27d2e 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -5,12 +5,14 @@
 #include "common/perf_counters.h"
 #include "common/perf_counters_key.h"
 #include "common/ceph_context.h"
+#include "rgw_sal.h"
+
+using namespace ceph::perf_counters;
+using namespace rgw::op_counters;
 
 PerfCounters *perfcounter = NULL;
-ceph::perf_counters::PerfCountersCache *perf_counters_cache = NULL;
-std::string rgw_op_counters_key = "rgw_op";
 
-static void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
+void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
   // RGW emits comparatively few metrics, so let's be generous
   // and mark them all USEFUL to get transmission to ceph-mgr by default.
   pcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
@@ -58,7 +60,7 @@ static void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
   pcb->add_u64(l_rgw_lua_current_vms, "lua_current_vms", "Number of Lua VMs currently being executed");
 }
 
-static void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
+void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
   // description must match general rgw counters description above
   lpcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
 
@@ -88,23 +90,6 @@ static void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
   lpcb->add_time_avg(l_rgw_op_list_buckets_lat, "list_buckets_lat", "List buckets latency");
 }
 
-std::shared_ptr<PerfCounters> create_rgw_counters(const std::string& name, CephContext *cct) {
-  std::string_view key = ceph::perf_counters::key_name(name);
-  if (rgw_op_counters_key.compare(key) == 0) {
-    PerfCountersBuilder pcb(cct, name, l_rgw_op_first, l_rgw_op_last);
-    add_rgw_op_counters(&pcb);
-    std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
-    cct->get_perfcounters_collection()->add(new_counters.get());
-    return new_counters;
-  } else {
-    PerfCountersBuilder pcb(cct, name, l_rgw_first, l_rgw_last);
-    add_rgw_frontend_counters(&pcb);
-    std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
-    cct->get_perfcounters_collection()->add(new_counters.get());
-    return new_counters;
-  }
-}
-
 void frontend_counters_init(CephContext *cct) {
   PerfCountersBuilder pcb(cct, "rgw", l_rgw_first, l_rgw_last);
   add_rgw_frontend_counters(&pcb);
@@ -115,7 +100,20 @@ void frontend_counters_init(CephContext *cct) {
 
 namespace rgw::op_counters {
 
+ceph::perf_counters::PerfCountersCache *user_counters_cache = NULL;
+ceph::perf_counters::PerfCountersCache *bucket_counters_cache = NULL;
 PerfCounters *global_op_counters = NULL;
+const std::string rgw_op_counters_key = "rgw_op";
+
+std::shared_ptr<PerfCounters> create_rgw_op_counters(const std::string& name, CephContext *cct) {
+  std::string_view key = ceph::perf_counters::key_name(name);
+  ceph_assert(rgw_op_counters_key == key);
+  PerfCountersBuilder pcb(cct, name, l_rgw_op_first, l_rgw_op_last);
+  add_rgw_op_counters(&pcb);
+  std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
+  cct->get_perfcounters_collection()->add(new_counters.get());
+  return new_counters;
+}
 
 void global_op_counters_init(CephContext *cct) {
   PerfCountersBuilder pcb(cct, rgw_op_counters_key, l_rgw_op_first, l_rgw_op_last);
@@ -125,30 +123,67 @@ void global_op_counters_init(CephContext *cct) {
   global_op_counters = new_counters;
 }
 
-void inc(std::shared_ptr<PerfCounters> labeled_counters, int idx, uint64_t v) {
-  if (labeled_counters) {
-    PerfCounters *counter = labeled_counters.get();
-    counter->inc(idx, v);
+CountersContainer get(req_state *s) {
+  CountersContainer counters;
+  std::string key;
+
+  if (user_counters_cache && !s->user->get_id().id.empty()) {
+    if (s->user->get_tenant().empty()) {
+      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}}));
+    } else {
+      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}, {"Tenant", s->user->get_tenant()}}));
+    }
+    counters.user_counters = user_counters_cache->get(key);
+  }
+
+  if (bucket_counters_cache && !s->bucket_name.empty()) {
+    if (s->bucket_tenant.empty()) {
+      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}}));
+    } else {
+      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}, {"Tenant", s->bucket_tenant}}));
+    }
+    counters.bucket_counters = bucket_counters_cache->get(key);
+  }
+
+  return counters;
+}
+
+void inc(const CountersContainer &counters, int idx, uint64_t v) {
+  if (counters.user_counters) {
+    PerfCounters *user_counters = counters.user_counters.get();
+    user_counters->inc(idx, v);
+  }
+  if (counters.bucket_counters) {
+    PerfCounters *bucket_counters = counters.bucket_counters.get();
+    bucket_counters->inc(idx, v);
   }
   if (global_op_counters) {
     global_op_counters->inc(idx, v);
   }
 }
 
-void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, utime_t amt) {
-  if (labeled_counters) {
-    PerfCounters *counter = labeled_counters.get();
-    counter->tinc(idx, amt);
+void tinc(const CountersContainer &counters, int idx, utime_t amt) {
+  if (counters.user_counters) {
+    PerfCounters *user_counters = counters.user_counters.get();
+    user_counters->tinc(idx, amt);
+  }
+  if (counters.bucket_counters) {
+    PerfCounters *bucket_counters = counters.bucket_counters.get();
+    bucket_counters->tinc(idx, amt);
   }
   if (global_op_counters) {
     global_op_counters->tinc(idx, amt);
   }
 }
 
-void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, ceph::timespan amt) {
-  if (labeled_counters) {
-    PerfCounters *counter = labeled_counters.get();
-    counter->tinc(idx, amt);
+void tinc(const CountersContainer &counters, int idx, ceph::timespan amt) {
+  if (counters.user_counters) {
+    PerfCounters *user_counters = counters.user_counters.get();
+    user_counters->tinc(idx, amt);
+  }
+  if (counters.bucket_counters) {
+    PerfCounters *bucket_counters = counters.bucket_counters.get();
+    bucket_counters->tinc(idx, amt);
   }
   if (global_op_counters) {
     global_op_counters->tinc(idx, amt);
@@ -161,13 +196,19 @@ int rgw_perf_start(CephContext *cct)
 {
   frontend_counters_init(cct);
 
-  bool cache_enabled = cct->_conf.get_val<bool>("rgw_perf_counters_cache");
-  if (cache_enabled) {
-    uint64_t target_size = cct->_conf.get_val<uint64_t>("rgw_perf_counters_cache_size");
-    perf_counters_cache = new ceph::perf_counters::PerfCountersCache(cct, target_size, create_rgw_counters); 
+  bool user_counters_cache_enabled = cct->_conf.get_val<bool>("rgw_user_counters_cache");
+  if (user_counters_cache_enabled) {
+    uint64_t target_size = cct->_conf.get_val<uint64_t>("rgw_user_counters_cache_size");
+    user_counters_cache = new PerfCountersCache(cct, target_size, create_rgw_op_counters);
+  }
+
+  bool bucket_counters_cache_enabled = cct->_conf.get_val<bool>("rgw_bucket_counters_cache");
+  if (bucket_counters_cache_enabled) {
+    uint64_t target_size = cct->_conf.get_val<uint64_t>("rgw_bucket_counters_cache_size");
+    bucket_counters_cache = new PerfCountersCache(cct, target_size, create_rgw_op_counters);
   }
 
-  rgw::op_counters::global_op_counters_init(cct);
+  global_op_counters_init(cct);
   return 0;
 }
 
@@ -176,5 +217,9 @@ void rgw_perf_stop(CephContext *cct)
   ceph_assert(perfcounter);
   cct->get_perfcounters_collection()->remove(perfcounter);
   delete perfcounter;
-  delete perf_counters_cache;
+  ceph_assert(global_op_counters);
+  cct->get_perfcounters_collection()->remove(global_op_counters);
+  delete global_op_counters;
+  delete user_counters_cache;
+  delete bucket_counters_cache;
 }
diff --git a/src/rgw/rgw_perf_counters.h b/src/rgw/rgw_perf_counters.h
index 49f7e4d4218b..e9068f4c9ff2 100644
--- a/src/rgw/rgw_perf_counters.h
+++ b/src/rgw/rgw_perf_counters.h
@@ -4,17 +4,13 @@
 #pragma once
 
 #include "include/common_fwd.h"
+#include "rgw_common.h"
 #include "common/perf_counters_cache.h"
 #include "common/perf_counters_key.h"
 
 extern PerfCounters *perfcounter;
-extern ceph::perf_counters::PerfCountersCache *perf_counters_cache;
-extern std::string rgw_op_counters_key;
-
 extern int rgw_perf_start(CephContext *cct);
 extern void rgw_perf_stop(CephContext *cct);
-extern void frontend_counters_init(CephContext *cct);
-extern std::shared_ptr<PerfCounters> create_rgw_counters(const std::string& name, CephContext *cct);
 
 enum {
   l_rgw_first = 15000,
@@ -89,24 +85,17 @@ enum {
 
 namespace rgw::op_counters {
 
-extern PerfCounters *global_op_counters;
-
-void global_op_counters_init(CephContext *cct);
+struct CountersContainer {
+  std::shared_ptr<PerfCounters> user_counters;
+  std::shared_ptr<PerfCounters> bucket_counters;
+};
 
-template <std::size_t Count>
-std::shared_ptr<PerfCounters> get(ceph::perf_counters::label_pair (&&labels)[Count]) {
-  if (perf_counters_cache) {
-    std::string key = ceph::perf_counters::key_create(rgw_op_counters_key, std::move(labels));
-    return perf_counters_cache->get(key);
-  } else {
-    return std::shared_ptr<PerfCounters>(nullptr);
-  }
-}
+CountersContainer get(req_state *s);
 
-void inc(std::shared_ptr<PerfCounters> labeled_counters, int idx, uint64_t v);
+void inc(const CountersContainer &counters, int idx, uint64_t v);
 
-void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, utime_t);
+void tinc(const CountersContainer &counters, int idx, utime_t);
 
-void tinc(std::shared_ptr<PerfCounters> labeled_counters, int idx, ceph::timespan amt);
+void tinc(const CountersContainer &counters, int idx, ceph::timespan amt);
 
 } // namespace rgw::op_counters
diff --git a/src/rgw/vstart.sh.swift b/src/rgw/vstart.sh.swift
new file mode 100755
index 000000000000..46e46da0e4a5
--- /dev/null
+++ b/src/rgw/vstart.sh.swift
@@ -0,0 +1,1930 @@
+#!/usr/bin/env bash
+# -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*-
+# vim: softtabstop=4 shiftwidth=4 expandtab
+
+# abort on failure
+set -e
+
+quoted_print() {
+    for s in "$@"; do
+        if [[ "$s" =~ \  ]]; then
+            printf -- "'%s' " "$s"
+        else
+            printf -- "$s "
+        fi
+    done
+    printf '\n'
+}
+
+debug() {
+  "$@" >&2
+}
+
+prunb() {
+    debug quoted_print "$@" '&'
+    PATH=$CEPH_BIN:$PATH "$@" &
+}
+
+prun() {
+    debug quoted_print "$@"
+    PATH=$CEPH_BIN:$PATH "$@"
+}
+
+
+if [ -n "$VSTART_DEST" ]; then
+    SRC_PATH=`dirname $0`
+    SRC_PATH=`(cd $SRC_PATH; pwd)`
+
+    CEPH_DIR=$SRC_PATH
+    CEPH_BIN=${CEPH_BIN:-${PWD}/bin}
+    CEPH_LIB=${CEPH_LIB:-${PWD}/lib}
+
+    CEPH_CONF_PATH=$VSTART_DEST
+    CEPH_DEV_DIR=$VSTART_DEST/dev
+    CEPH_OUT_DIR=$VSTART_DEST/out
+    CEPH_ASOK_DIR=$VSTART_DEST/asok
+    CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR}
+fi
+
+get_cmake_variable() {
+    local variable=$1
+    grep "${variable}:" CMakeCache.txt | cut -d "=" -f 2
+}
+
+# for running out of the CMake build directory
+if [ -e CMakeCache.txt ]; then
+    # Out of tree build, learn source location from CMakeCache.txt
+    CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR)
+    CEPH_BUILD_DIR=`pwd`
+    [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
+fi
+
+# use CEPH_BUILD_ROOT to vstart from a 'make install'
+if [ -n "$CEPH_BUILD_ROOT" ]; then
+    [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin
+    [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
+    [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_ROOT/external/lib
+    [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
+    [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
+    # make install should install python extensions into PYTHONPATH
+elif [ -n "$CEPH_ROOT" ]; then
+    [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL=$CEPH_ROOT/src/tools/cephfs/shell/cephfs-shell
+    [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
+    [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
+    [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph
+    [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
+    [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib
+    [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_DIR/external/lib
+    [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB
+    [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB
+    [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON=$CEPH_ROOT/src/python-common
+fi
+
+if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
+    PATH=$(pwd):$PATH
+fi
+
+[ -z "$PYBIND" ] && PYBIND=./pybind
+
+[ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON="$CEPH_PYTHON_COMMON:"
+CYTHON_PYTHONPATH="$CEPH_LIB/cython_modules/lib.3"
+export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH
+
+export LD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$LD_LIBRARY_PATH
+export DYLD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$DYLD_LIBRARY_PATH
+# Suppress logging for regular use that indicated that we are using a
+# development version. vstart.sh is only used during testing and
+# development
+export CEPH_DEV=1
+
+[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
+[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
+[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
+[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR"
+[ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS="$FS"
+[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
+[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM="$NFS"
+
+# if none of the CEPH_NUM_* number is specified, kill the existing
+# cluster.
+if [ -z "$CEPH_NUM_MON" -a \
+     -z "$CEPH_NUM_OSD" -a \
+     -z "$CEPH_NUM_MDS" -a \
+     -z "$CEPH_NUM_MGR" -a \
+     -z "$GANESHA_DAEMON_NUM" ]; then
+    kill_all=1
+else
+    kill_all=0
+fi
+
+[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
+[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3
+[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
+[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1
+[ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS=1
+[ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1
+[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0
+[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM=0
+
+[ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD"
+[ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
+[ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
+[ -z "$CEPH_ASOK_DIR" ] && CEPH_ASOK_DIR="$CEPH_DIR/asok"
+[ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
+[ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR
+CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR}
+
+if [ $CEPH_NUM_OSD -gt 3 ]; then
+    OSD_POOL_DEFAULT_SIZE=3
+else
+    OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD
+fi
+
+extra_conf=""
+new=0
+standby=0
+debug=0
+trace=0
+ip=""
+nodaemon=0
+redirect=0
+smallmds=0
+short=0
+crimson=0
+ec=0
+cephadm=0
+parallel=true
+restart=1
+hitset=""
+overwrite_conf=0
+cephx=1 #turn cephx on by default
+gssapi_authx=0
+cache=""
+if [ `uname` = FreeBSD ]; then
+    objectstore="memstore"
+else
+    objectstore="bluestore"
+fi
+ceph_osd=ceph-osd
+rgw_frontend="beast prefix=/swift"
+rgw_compression=""
+lockdep=${LOCKDEP:-1}
+spdk_enabled=0 # disable SPDK by default
+pmem_enabled=0
+zoned_enabled=0
+io_uring_enabled=0
+with_jaeger=0
+
+with_mgr_dashboard=true
+if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
+   [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then
+    debug echo "ceph-mgr dashboard not built - disabling."
+    with_mgr_dashboard=false
+fi
+with_mgr_restful=false
+
+kstore_path=
+declare -a block_devs
+declare -a bluestore_db_devs
+declare -a bluestore_wal_devs
+declare -a secondary_block_devs
+secondary_block_devs_type="SSD"
+
+VSTART_SEC="client.vstart.sh"
+
+MON_ADDR=""
+DASH_URLS=""
+RESTFUL_URLS=""
+
+conf_fn="$CEPH_CONF_PATH/ceph.conf"
+keyring_fn="$CEPH_CONF_PATH/keyring"
+monmap_fn="/tmp/ceph_monmap.$$"
+inc_osd_num=0
+
+msgr="21"
+
+read -r -d '' usage <<EOF || true
+usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 $0 -n -d
+options:
+	-d, --debug
+	-t, --trace
+	-s, --standby_mds: Generate standby-replay MDS for each active
+	-l, --localhost: use localhost instead of hostname
+	-i <ip>: bind to specific ip
+	-n, --new
+	--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'
+	--nodaemon: use ceph-run as wrapper for mon/osd/mds
+	--redirect-output: only useful with nodaemon, directs output to log file
+	--smallmds: limit mds cache memory limit
+	-m ip:port		specify monitor address
+	-k keep old configuration files (default)
+	-x enable cephx (on by default)
+	-X disable cephx
+	-g --gssapi enable Kerberos/GSSApi authentication
+	-G disable Kerberos/GSSApi authentication
+	--hitset <pool> <hit_set_type>: enable hitset tracking
+	-e : create an erasure pool
+	-o config add extra config parameters to all sections
+	--rgw_port specify ceph rgw http listen port
+	--rgw_frontend specify the rgw frontend configuration
+	--rgw_arrow_flight start arrow flight frontend
+	--rgw_compression specify the rgw compression plugin
+	--seastore use seastore as crimson osd backend
+	-b, --bluestore use bluestore as the osd objectstore backend (default)
+	-K, --kstore use kstore as the osd objectstore backend
+	--cyanstore use cyanstore as the osd objectstore backend
+	--memstore use memstore as the osd objectstore backend
+	--cache <pool>: enable cache tiering on pool
+	--short: short object names only; necessary for ext4 dev
+	--nolockdep disable lockdep
+	--multimds <count> allow multimds with maximum active count
+	--without-dashboard: do not run using mgr dashboard
+	--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
+	--bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
+	--msgr1: use msgr1 only
+	--msgr2: use msgr2 only
+	--msgr21: use msgr2 and msgr1
+	--crimson: use crimson-osd instead of ceph-osd
+	--crimson-foreground: use crimson-osd, but run it in the foreground
+	--osd-args: specify any extra osd specific options
+	--bluestore-devs: comma-separated list of blockdevs to use for bluestore
+	--bluestore-db-devs: comma-separated list of db-devs to use for bluestore
+	--bluestore-wal-devs: comma-separated list of wal-devs to use for bluestore
+	--bluestore-zoned: blockdevs listed by --bluestore-devs are zoned devices (HM-SMR HDD or ZNS SSD)
+	--bluestore-io-uring: enable io_uring backend
+	--inc-osd: append some more osds into existing vcluster
+	--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]
+	--no-parallel: dont start all OSDs in parallel
+	--no-restart: dont restart process when using ceph-run
+	--jaeger: use jaegertracing for tracing
+	--seastore-devs: comma-separated list of blockdevs to use for seastore
+	--seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore
+	--seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
+	--crimson-smp: number of cores to use for crimson
+\n
+EOF
+
+usage_exit() {
+    printf "$usage"
+    exit
+}
+
+parse_block_devs() {
+    local opt_name=$1
+    shift
+    local devs=$1
+    shift
+    local dev
+    IFS=',' read -r -a block_devs <<< "$devs"
+    for dev in "${block_devs[@]}"; do
+        if [ ! -b $dev ] || [ ! -w $dev ]; then
+            echo "All $opt_name must refer to writable block devices"
+            exit 1
+        fi
+    done
+}
+
+parse_bluestore_db_devs() {
+    local opt_name=$1
+    shift
+    local devs=$1
+    shift
+    local dev
+    IFS=',' read -r -a bluestore_db_devs <<< "$devs"
+    for dev in "${bluestore_db_devs[@]}"; do
+        if [ ! -b $dev ] || [ ! -w $dev ]; then
+            echo "All $opt_name must refer to writable block devices"
+            exit 1
+        fi
+    done
+}
+
+parse_bluestore_wal_devs() {
+    local opt_name=$1
+    shift
+    local devs=$1
+    shift
+    local dev
+    IFS=',' read -r -a bluestore_wal_devs <<< "$devs"
+    for dev in "${bluestore_wal_devs[@]}"; do
+        if [ ! -b $dev ] || [ ! -w $dev ]; then
+            echo "All $opt_name must refer to writable block devices"
+            exit 1
+        fi
+    done
+}
+
+parse_secondary_devs() {
+    local opt_name=$1
+    shift
+    local devs=$1
+    shift
+    local dev
+    IFS=',' read -r -a secondary_block_devs <<< "$devs"
+    for dev in "${secondary_block_devs[@]}"; do
+        if [ ! -b $dev ] || [ ! -w $dev ]; then
+            echo "All $opt_name must refer to writable block devices"
+            exit 1
+        fi
+    done
+}
+
+crimson_smp=1
+while [ $# -ge 1 ]; do
+case $1 in
+    -d | --debug)
+        debug=1
+        ;;
+    -t | --trace)
+        trace=1
+        ;;
+    -s | --standby_mds)
+        standby=1
+        ;;
+    -l | --localhost)
+        ip="127.0.0.1"
+        ;;
+    -i)
+        [ -z "$2" ] && usage_exit
+        ip="$2"
+        shift
+        ;;
+    -e)
+        ec=1
+        ;;
+    --new | -n)
+        new=1
+        ;;
+    --inc-osd)
+        new=0
+        kill_all=0
+        inc_osd_num=$2
+        if [ "$inc_osd_num" == "" ]; then
+            inc_osd_num=1
+        else
+            shift
+        fi
+        ;;
+    --short)
+        short=1
+        ;;
+    --crimson)
+        crimson=1
+        ceph_osd=crimson-osd
+        nodaemon=1
+        msgr=2
+        ;;
+    --crimson-foreground)
+        crimson=1
+        ceph_osd=crimson-osd
+        nodaemon=0
+        msgr=2
+        ;;
+    --osd-args)
+        extra_osd_args="$2"
+        shift
+        ;;
+    --msgr1)
+        msgr="1"
+        ;;
+    --msgr2)
+        msgr="2"
+        ;;
+    --msgr21)
+        msgr="21"
+        ;;
+    --cephadm)
+        cephadm=1
+        ;;
+    --no-parallel)
+        parallel=false
+        ;;
+    --no-restart)
+        restart=0
+        ;;
+    --valgrind)
+        [ -z "$2" ] && usage_exit
+        valgrind=$2
+        shift
+        ;;
+    --valgrind_args)
+        valgrind_args="$2"
+        shift
+        ;;
+    --valgrind_mds)
+        [ -z "$2" ] && usage_exit
+        valgrind_mds=$2
+        shift
+        ;;
+    --valgrind_osd)
+        [ -z "$2" ] && usage_exit
+        valgrind_osd=$2
+        shift
+        ;;
+    --valgrind_mon)
+        [ -z "$2" ] && usage_exit
+        valgrind_mon=$2
+        shift
+        ;;
+    --valgrind_mgr)
+        [ -z "$2" ] && usage_exit
+        valgrind_mgr=$2
+        shift
+        ;;
+    --valgrind_rgw)
+        [ -z "$2" ] && usage_exit
+        valgrind_rgw=$2
+        shift
+        ;;
+    --nodaemon)
+        nodaemon=1
+        ;;
+    --redirect-output)
+        redirect=1
+        ;;
+    --smallmds)
+        smallmds=1
+        ;;
+    --rgw_port)
+        CEPH_RGW_PORT=$2
+        shift
+        ;;
+    --rgw_frontend)
+        rgw_frontend=$2
+        shift
+        ;;
+    --rgw_arrow_flight)
+        rgw_flight_frontend="yes"
+        ;;
+    --rgw_compression)
+        rgw_compression=$2
+        shift
+        ;;
+    --kstore_path)
+        kstore_path=$2
+        shift
+        ;;
+    -m)
+        [ -z "$2" ] && usage_exit
+        MON_ADDR=$2
+        shift
+        ;;
+    -x)
+        cephx=1 # this is on be default, flag exists for historical consistency
+        ;;
+    -X)
+        cephx=0
+        ;;
+
+    -g | --gssapi)
+        gssapi_authx=1
+        ;;
+    -G)
+        gssapi_authx=0
+        ;;
+
+    -k)
+        if [ ! -r $conf_fn ]; then
+            echo "cannot use old configuration: $conf_fn not readable." >&2
+            exit
+        fi
+        new=0
+        ;;
+    --memstore)
+        objectstore="memstore"
+        ;;
+    --cyanstore)
+        objectstore="cyanstore"
+        ;;
+    --seastore)
+        objectstore="seastore"
+        ;;
+    -b | --bluestore)
+        objectstore="bluestore"
+        ;;
+    -K | --kstore)
+        objectstore="kstore"
+        ;;
+    --hitset)
+        hitset="$hitset $2 $3"
+        shift
+        shift
+        ;;
+    -o)
+        extra_conf+=$'\n'"$2"
+        shift
+        ;;
+    --cache)
+        if [ -z "$cache" ]; then
+            cache="$2"
+        else
+            cache="$cache $2"
+        fi
+        shift
+        ;;
+    --nolockdep)
+        lockdep=0
+        ;;
+    --multimds)
+        CEPH_MAX_MDS="$2"
+        shift
+        ;;
+    --without-dashboard)
+        with_mgr_dashboard=false
+        ;;
+    --with-restful)
+        with_mgr_restful=true
+        ;;
+    --seastore-devs)
+        parse_block_devs --seastore-devs "$2"
+        shift
+        ;;
+    --seastore-secondary-devs)
+        parse_secondary_devs --seastore-devs "$2"
+        shift
+        ;;
+    --seastore-secondary-devs-type)
+        secondary_block_devs_type="$2"
+        shift
+        ;;
+    --crimson-smp)
+        crimson_smp=$2
+        shift
+        ;;
+    --bluestore-spdk)
+        [ -z "$2" ] && usage_exit
+        IFS=',' read -r -a bluestore_spdk_dev <<< "$2"
+        spdk_enabled=1
+        shift
+        ;;
+    --bluestore-pmem)
+        [ -z "$2" ] && usage_exit
+        bluestore_pmem_file="$2"
+        pmem_enabled=1
+        shift
+        ;;
+    --bluestore-devs)
+        parse_block_devs --bluestore-devs "$2"
+        shift
+        ;;
+    --bluestore-db-devs)
+        parse_bluestore_db_devs --bluestore-db-devs "$2"
+        shift
+        ;;
+    --bluestore-wal-devs)
+        parse_bluestore_wal_devs --bluestore-wal-devs "$2"
+        shift
+        ;;
+    --bluestore-zoned)
+        zoned_enabled=1
+        ;;
+    --bluestore-io-uring)
+        io_uring_enabled=1
+        shift
+        ;;
+    --jaeger)
+        with_jaeger=1
+        echo "with_jaeger $with_jaeger"
+        ;;
+    *)
+        usage_exit
+esac
+shift
+done
+
+if [ $kill_all -eq 1 ]; then
+    $SUDO $INIT_CEPH stop
+fi
+
+if [ "$new" -eq 0 ]; then
+    if [ -z "$CEPH_ASOK_DIR" ]; then
+        CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
+    fi
+    mkdir -p $CEPH_ASOK_DIR
+    MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \
+        CEPH_NUM_MON="$MON"
+    OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \
+        CEPH_NUM_OSD="$OSD"
+    MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \
+        CEPH_NUM_MDS="$MDS"
+    MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \
+        CEPH_NUM_MGR="$MGR"
+    RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \
+        CEPH_NUM_RGW="$RGW"
+    NFS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \
+        GANESHA_DAEMON_NUM="$NFS"
+else
+    # only delete if -n
+    if [ -e "$conf_fn" ]; then
+        asok_dir=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
+        rm -- "$conf_fn"
+        if [ $asok_dir != /var/run/ceph ]; then
+            [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
+        fi
+    fi
+    if [ -z "$CEPH_ASOK_DIR" ]; then
+        CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
+    fi
+fi
+
+ARGS="-c $conf_fn"
+
+run() {
+    type=$1
+    shift
+    num=$1
+    shift
+    eval "valg=\$valgrind_$type"
+    [ -z "$valg" ] && valg="$valgrind"
+
+    if [ -n "$valg" ]; then
+        prunb valgrind --tool="$valg" $valgrind_args "$@" -f
+        sleep 1
+    else
+        if [ "$nodaemon" -eq 0 ]; then
+            prun "$@"
+        else
+            if [ "$restart" -eq 0 ]; then
+                set -- '--no-restart' "$@"
+            fi
+            if [ "$redirect" -eq 0 ]; then
+                prunb ${CEPH_ROOT}/src/ceph-run "$@" -f
+            else
+                ( prunb ${CEPH_ROOT}/src/ceph-run "$@" -f ) >$CEPH_OUT_DIR/$type.$num.stdout 2>&1
+            fi
+        fi
+    fi
+}
+
+wconf() {
+    if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then
+        cat >> "$conf_fn"
+    fi
+}
+
+
+do_rgw_conf() {
+
+    if [ $CEPH_NUM_RGW -eq 0 ]; then
+        return 0
+    fi
+
+    # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
+    # individual rgw's ids will be their ports.
+    current_port=$CEPH_RGW_PORT
+    # allow only first rgw to start arrow_flight server/port
+    local flight_conf=$rgw_flight_frontend
+    for n in $(seq 1 $CEPH_NUM_RGW); do
+        wconf << EOF
+[client.rgw.${current_port}]
+        rgw frontends = $rgw_frontend port=${current_port}${flight_conf:+,arrow_flight}
+        admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
+        debug rgw_flight = 20
+        rgw keystone accepted admin roles = admin
+        rgw keystone accepted roles = admin,Member
+        rgw keystone admin domain = Default
+        rgw keystone admin password = ADMIN
+        rgw keystone admin project = admin
+        rgw keystone admin user = admin
+        rgw keystone api version = 3
+        rgw keystone implicit tenants = true
+        rgw swift account in url = true
+        rgw swift enforce content length = true
+        rgw swift versioning enabled = true
+EOF
+        current_port=$((current_port + 1))
+        unset flight_conf
+done
+
+}
+
+format_conf() {
+    local opts=$1
+    local indent="        "
+    local opt
+    local formatted
+    while read -r opt; do
+        if [ -z "$formatted" ]; then
+            formatted="${opt}"
+        else
+            formatted+=$'\n'${indent}${opt}
+        fi
+    done <<< "$opts"
+    echo "$formatted"
+}
+
+prepare_conf() {
+    local DAEMONOPTS="
+        log file = $CEPH_OUT_DIR/\$name.log
+        admin socket = $CEPH_ASOK_DIR/\$name.asok
+        chdir = \"\"
+        pid file = $CEPH_OUT_DIR/\$name.pid
+        heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
+"
+
+    local mgr_modules="iostat nfs"
+    if $with_mgr_dashboard; then
+        mgr_modules+=" dashboard"
+    fi
+    if $with_mgr_restful; then
+        mgr_modules+=" restful"
+    fi
+
+    local msgr_conf=''
+    if [ $msgr -eq 21 ]; then
+        msgr_conf="ms bind msgr2 = true
+                   ms bind msgr1 = true"
+    fi
+    if [ $msgr -eq 2 ]; then
+        msgr_conf="ms bind msgr2 = true
+                   ms bind msgr1 = false"
+    fi
+    if [ $msgr -eq 1 ]; then
+        msgr_conf="ms bind msgr2 = false
+                   ms bind msgr1 = true"
+    fi
+
+    wconf <<EOF
+; generated by vstart.sh on `date`
+[$VSTART_SEC]
+        num mon = $CEPH_NUM_MON
+        num osd = $CEPH_NUM_OSD
+        num mds = $CEPH_NUM_MDS
+        num mgr = $CEPH_NUM_MGR
+        num rgw = $CEPH_NUM_RGW
+        num ganesha = $GANESHA_DAEMON_NUM
+
+[global]
+        fsid = $(uuidgen)
+        osd failsafe full ratio = .99
+        mon osd full ratio = .99
+        mon osd nearfull ratio = .99
+        mon osd backfillfull ratio = .99
+        mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000}
+        erasure code dir = $EC_PATH
+        plugin dir = $CEPH_LIB
+        run dir = $CEPH_OUT_DIR
+        crash dir = $CEPH_OUT_DIR
+        enable experimental unrecoverable data corrupting features = *
+        osd_crush_chooseleaf_type = 0
+        debug asok assert abort = true
+        $(format_conf "${msgr_conf}")
+        $(format_conf "${extra_conf}")
+        $AUTOSCALER_OPTS
+EOF
+    if [ "$with_jaeger" -eq 1 ] ; then
+        wconf <<EOF
+        jaeger_agent_port = 6831
+EOF
+    fi
+    if [ "$lockdep" -eq 1 ] ; then
+        wconf <<EOF
+        lockdep = true
+EOF
+    fi
+    if [ "$cephx" -eq 1 ] ; then
+        wconf <<EOF
+        auth cluster required = cephx
+        auth service required = cephx
+        auth client required = cephx
+EOF
+    elif [ "$gssapi_authx" -eq 1 ] ; then
+        wconf <<EOF
+        auth cluster required = gss
+        auth service required = gss
+        auth client required = gss
+        gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab
+EOF
+    else
+        wconf <<EOF
+        auth cluster required = none
+        auth service required = none
+        auth client required = none
+        ms mon client mode = crc
+EOF
+    fi
+    if [ "$short" -eq 1 ]; then
+        COSDSHORT="        osd max object name len = 460
+        osd max object namespace len = 64"
+    fi
+    if [ "$objectstore" == "bluestore" ]; then
+        if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
+            BLUESTORE_OPTS="        bluestore_block_db_path = \"\"
+        bluestore_block_db_size = 0
+        bluestore_block_db_create = false
+        bluestore_block_wal_path = \"\"
+        bluestore_block_wal_size = 0
+        bluestore_block_wal_create = false
+        bluestore_spdk_mem = 2048"
+        else
+            BLUESTORE_OPTS="        bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file
+        bluestore block db size = 1073741824
+        bluestore block db create = true
+        bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file
+        bluestore block wal size = 1048576000
+        bluestore block wal create = true"
+            if [ ${#block_devs[@]} -gt 0 ] || \
+               [ ${#bluestore_db_devs[@]} -gt 0 ] || \
+               [ ${#bluestore_wal_devs[@]} -gt 0 ]; then
+                # when use physical disk, not create file for db/wal
+                BLUESTORE_OPTS=""
+            fi
+        fi
+        if [ "$zoned_enabled" -eq 1 ]; then
+            BLUESTORE_OPTS+="
+        bluestore min alloc size = 65536
+        bluestore prefer deferred size = 0
+        bluestore prefer deferred size hdd = 0
+        bluestore prefer deferred size ssd = 0
+        bluestore allocator = zoned"
+        fi
+        if [ "$io_uring_enabled" -eq 1 ]; then
+            BLUESTORE_OPTS+="
+        bdev ioring = true"
+        fi
+    fi
+    wconf <<EOF
+[client]
+$CCLIENTDEBUG
+        keyring = $keyring_fn
+        log file = $CEPH_OUT_CLIENT_DIR/\$name.\$pid.log
+        admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
+
+        ; needed for s3tests
+        rgw crypt s3 kms backend = testing
+        rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
+        rgw crypt require ssl = false
+        ; uncomment the following to set LC days as the value in seconds;
+        ; needed for passing lc time based s3-tests (can be verbose)
+        ; rgw lc debug interval = 10
+        $(format_conf "${extra_conf}")
+EOF
+	do_rgw_conf
+	wconf << EOF
+[mds]
+$CMDSDEBUG
+$DAEMONOPTS
+        mds data = $CEPH_DEV_DIR/mds.\$id
+        mds root ino uid = `id -u`
+        mds root ino gid = `id -g`
+        $(format_conf "${extra_conf}")
+[mgr]
+        mgr disabled modules = rook
+        mgr data = $CEPH_DEV_DIR/mgr.\$id
+        mgr module path = $MGR_PYTHON_PATH
+        cephadm path = $CEPH_BIN/cephadm
+$DAEMONOPTS
+        $(format_conf "${extra_conf}")
+[osd]
+$DAEMONOPTS
+        osd_check_max_object_name_len_on_startup = false
+        osd data = $CEPH_DEV_DIR/osd\$id
+        osd journal = $CEPH_DEV_DIR/osd\$id/journal
+        osd journal size = 100
+        osd class tmp = out
+        osd class dir = $OBJCLASS_PATH
+        osd class load list = *
+        osd class default list = *
+        osd fast shutdown = false
+
+        bluestore fsck on mount = true
+        bluestore block create = true
+$BLUESTORE_OPTS
+
+        ; kstore
+        kstore fsck on mount = true
+        osd objectstore = $objectstore
+$COSDSHORT
+        $(format_conf "${extra_conf}")
+[mon]
+        mon_data_avail_crit = 1
+        mgr initial modules = $mgr_modules
+$DAEMONOPTS
+$CMONDEBUG
+        $(format_conf "${extra_conf}")
+        mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
+        osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
+        auth allow insecure global id reclaim = false
+EOF
+
+    if [ "$crimson" -eq 1 ]; then
+        wconf <<EOF
+        osd pool default crimson = true
+EOF
+    fi
+}
+
+write_logrotate_conf() {
+    out_dir=$(pwd)"/out/*.log"
+
+    cat << EOF
+$out_dir
+{
+    rotate 5
+    size 1G
+    copytruncate
+    compress
+    notifempty
+    missingok
+    sharedscripts
+    postrotate
+        # NOTE: assuring that the absence of one of the following processes
+        # won't abort the logrotate command.
+        killall -u $USER -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror || echo ""
+    endscript
+}
+EOF
+}
+
+init_logrotate() {
+    logrotate_conf_path=$(pwd)"/logrotate.conf"
+    logrotate_state_path=$(pwd)"/logrotate.state"
+
+    if ! test -a $logrotate_conf_path; then
+        if test -a $logrotate_state_path; then
+            rm -f $logrotate_state_path
+        fi
+        write_logrotate_conf > $logrotate_conf_path
+    fi
+}
+
+start_mon() {
+    local MONS=""
+    local count=0
+    for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
+    do
+        [ $count -eq $CEPH_NUM_MON ] && break;
+        count=$(($count + 1))
+        if [ -z "$MONS" ]; then
+	    MONS="$f"
+        else
+	    MONS="$MONS $f"
+        fi
+    done
+
+    if [ "$new" -eq 1 ]; then
+        if [ `echo $IP | grep '^127\\.'` ]; then
+            echo
+            echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
+            echo "  connect.  either adjust /etc/hosts, or edit this script to use your"
+            echo "  machine's real IP."
+            echo
+        fi
+
+        prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *'
+        prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \
+             --cap mon 'allow *' \
+             --cap osd 'allow *' \
+             --cap mds 'allow *' \
+             --cap mgr 'allow *' \
+             "$keyring_fn"
+
+        # build a fresh fs monmap, mon fs
+        local params=()
+        local count=0
+        local mon_host=""
+        for f in $MONS
+        do
+            if [ $msgr -eq 1 ]; then
+                A="v1:$IP:$(($CEPH_PORT+$count+1))"
+            fi
+            if [ $msgr -eq 2 ]; then
+                A="v2:$IP:$(($CEPH_PORT+$count+1))"
+            fi
+            if [ $msgr -eq 21 ]; then
+                A="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]"
+            fi
+            params+=("--addv" "$f" "$A")
+            mon_host="$mon_host $A"
+            wconf <<EOF
+[mon.$f]
+        host = $HOSTNAME
+        mon data = $CEPH_DEV_DIR/mon.$f
+EOF
+            count=$(($count + 2))
+        done
+        wconf <<EOF
+[global]
+        mon host = $mon_host
+EOF
+        prun "$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn"
+
+        for f in $MONS
+        do
+            prun rm -rf -- "$CEPH_DEV_DIR/mon.$f"
+            prun mkdir -p "$CEPH_DEV_DIR/mon.$f"
+            prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
+        done
+
+        prun rm -- "$monmap_fn"
+    fi
+
+    # start monitors
+    for f in $MONS
+    do
+        run 'mon' $f $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
+    done
+
+    if [ "$crimson" -eq 1 ]; then
+        $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it
+    fi
+}
+
+start_osd() {
+    if [ $inc_osd_num -gt 0 ]; then
+        old_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
+        start=$old_maxosd
+        end=$(($start-1+$inc_osd_num))
+        overwrite_conf=1 # fake wconf
+    else
+        start=0
+        end=$(($CEPH_NUM_OSD-1))
+    fi
+    local osds_wait
+    for osd in `seq $start $end`
+    do
+	local extra_seastar_args
+	if [ "$ceph_osd" == "crimson-osd" ]; then
+        bottom_cpu=$(( osd * crimson_smp ))
+        top_cpu=$(( bottom_cpu + crimson_smp - 1 ))
+	    # set a single CPU nodes for each osd
+	    extra_seastar_args="--cpuset $bottom_cpu-$top_cpu"
+	    if [ "$debug" -ne 0 ]; then
+		extra_seastar_args+=" --debug"
+	    fi
+            if [ "$trace" -ne 0 ]; then
+                extra_seastar_args+=" --trace"
+            fi
+	fi
+	if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
+            wconf <<EOF
+[osd.$osd]
+        host = $HOSTNAME
+EOF
+            if [ "$spdk_enabled" -eq 1 ]; then
+                wconf <<EOF
+        bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
+EOF
+            elif [ "$pmem_enabled" -eq 1 ]; then
+                wconf <<EOF
+        bluestore_block_path = ${bluestore_pmem_file}
+EOF
+            fi
+            rm -rf $CEPH_DEV_DIR/osd$osd || true
+            if command -v btrfs > /dev/null; then
+                for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
+            fi
+            if [ -n "$kstore_path" ]; then
+                ln -s $kstore_path $CEPH_DEV_DIR/osd$osd
+            else
+                mkdir -p $CEPH_DEV_DIR/osd$osd
+                if [ -n "${block_devs[$osd]}" ]; then
+                    dd if=/dev/zero of=${block_devs[$osd]} bs=1M count=1
+                    ln -s ${block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block
+                fi
+                if [ -n "${bluestore_db_devs[$osd]}" ]; then
+                    dd if=/dev/zero of=${bluestore_db_devs[$osd]} bs=1M count=1
+                    ln -s ${bluestore_db_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.db
+                fi
+                if [ -n "${bluestore_wal_devs[$osd]}" ]; then
+                    dd if=/dev/zero of=${bluestore_wal_devs[$osd]} bs=1M count=1
+                    ln -s ${bluestore_wal_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.wal
+                fi
+                if [ -n "${secondary_block_devs[$osd]}" ]; then
+                    dd if=/dev/zero of=${secondary_block_devs[$osd]} bs=1M count=1
+                    mkdir -p $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1
+                    ln -s ${secondary_block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1/block
+                fi
+            fi
+            if [ "$objectstore" == "bluestore" ]; then
+                wconf <<EOF
+        bluestore fsck on mount = false
+EOF
+            fi
+
+            local uuid=`uuidgen`
+            echo "add osd$osd $uuid"
+            OSD_SECRET=$($CEPH_BIN/ceph-authtool --gen-print-key)
+            echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json
+            ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json
+            rm $CEPH_DEV_DIR/osd$osd/new.json
+            prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \
+                2>&1 | tee $CEPH_OUT_DIR/osd-mkfs.$osd.log
+
+            local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
+            cat > $key_fn<<EOF
+[osd.$osd]
+        key = $OSD_SECRET
+EOF
+        fi
+        echo start osd.$osd
+        local osd_pid
+        echo 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
+            $extra_seastar_args $extra_osd_args \
+            -i $osd $ARGS $COSD_ARGS
+        run 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
+            $extra_seastar_args $extra_osd_args \
+            -i $osd $ARGS $COSD_ARGS &
+        osd_pid=$!
+        if $parallel; then
+            osds_wait=$osd_pid
+        else
+            wait $osd_pid
+        fi
+    done
+    if $parallel; then
+        for p in $osds_wait; do
+            wait $p
+        done
+        debug echo OSDs started
+    fi
+    if [ $inc_osd_num -gt 0 ]; then
+        # update num osd
+        new_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
+        sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn
+    fi
+}
+
+create_mgr_restful_secret() {
+    while ! ceph_adm -h | grep -c -q ^restful ; do
+        debug echo 'waiting for mgr restful module to start'
+        sleep 1
+    done
+    local secret_file
+    if ceph_adm restful create-self-signed-cert > /dev/null; then
+        secret_file=`mktemp`
+        ceph_adm restful create-key admin -o $secret_file
+        RESTFUL_SECRET=`cat $secret_file`
+        rm $secret_file
+    else
+        debug echo MGR Restful is not working, perhaps the package is not installed?
+    fi
+}
+
+start_mgr() {
+    local mgr=0
+    local ssl=${DASHBOARD_SSL:-1}
+    # avoid monitors on nearby ports (which test/*.sh use extensively)
+    MGR_PORT=$(($CEPH_PORT + 1000))
+    PROMETHEUS_PORT=9283
+    for name in x y z a b c d e f g h i j k l m n o p
+    do
+        [ $mgr -eq $CEPH_NUM_MGR ] && break
+        mgr=$(($mgr + 1))
+        if [ "$new" -eq 1 ]; then
+            mkdir -p $CEPH_DEV_DIR/mgr.$name
+            key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
+            $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
+            ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *'
+
+            wconf <<EOF
+[mgr.$name]
+        host = $HOSTNAME
+EOF
+
+            if $with_mgr_dashboard ; then
+                local port_option="ssl_server_port"
+                local http_proto="https"
+                if [ "$ssl" == "0" ]; then
+                    port_option="server_port"
+                    http_proto="http"
+                    ceph_adm config set mgr mgr/dashboard/ssl false --force
+                fi
+                ceph_adm config set mgr mgr/dashboard/$name/$port_option $MGR_PORT --force
+                if [ $mgr -eq 1 ]; then
+                    DASH_URLS="$http_proto://$IP:$MGR_PORT"
+                else
+                    DASH_URLS+=", $http_proto://$IP:$MGR_PORT"
+                fi
+            fi
+	    MGR_PORT=$(($MGR_PORT + 1000))
+	    ceph_adm config set mgr mgr/prometheus/$name/server_port $PROMETHEUS_PORT --force
+	    PROMETHEUS_PORT=$(($PROMETHEUS_PORT + 1000))
+
+	    ceph_adm config set mgr mgr/restful/$name/server_port $MGR_PORT --force
+            if [ $mgr -eq 1 ]; then
+                RESTFUL_URLS="https://$IP:$MGR_PORT"
+            else
+                RESTFUL_URLS+=", https://$IP:$MGR_PORT"
+            fi
+	    MGR_PORT=$(($MGR_PORT + 1000))
+        fi
+
+        debug echo "Starting mgr.${name}"
+        run 'mgr' $name $CEPH_BIN/ceph-mgr -i $name $ARGS
+    done
+
+    while ! ceph_adm mgr stat | jq -e '.available'; do
+        debug echo 'waiting for mgr to become available'
+        sleep 1
+    done
+    
+    if [ "$new" -eq 1 ]; then
+        # setting login credentials for dashboard
+        if $with_mgr_dashboard; then
+            while ! ceph_adm -h | grep -c -q ^dashboard ; do
+                debug echo 'waiting for mgr dashboard module to start'
+                sleep 1
+            done
+            DASHBOARD_ADMIN_SECRET_FILE="${CEPH_CONF_PATH}/dashboard-admin-secret.txt"
+            printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}"
+            ceph_adm dashboard ac-user-create admin -i "${DASHBOARD_ADMIN_SECRET_FILE}" \
+                administrator --force-password
+            if [ "$ssl" != "0" ]; then
+                if ! ceph_adm dashboard create-self-signed-cert;  then
+                    debug echo dashboard module not working correctly!
+                fi
+            fi
+        fi
+        if $with_mgr_restful; then
+            create_mgr_restful_secret
+        fi
+    fi
+
+    if [ "$cephadm" -eq 1 ]; then
+        debug echo Enabling cephadm orchestrator
+	if [ "$new" -eq 1 ]; then
+		digest=$(curl -s \
+		https://hub.docker.com/v2/repositories/ceph/daemon-base/tags/latest-master-devel \
+		| jq -r '.images[0].digest')
+		ceph_adm config set global container_image "docker.io/ceph/daemon-base@$digest"
+	fi
+        ceph_adm config-key set mgr/cephadm/ssh_identity_key -i ~/.ssh/id_rsa
+        ceph_adm config-key set mgr/cephadm/ssh_identity_pub -i ~/.ssh/id_rsa.pub
+        ceph_adm mgr module enable cephadm
+        ceph_adm orch set backend cephadm
+        ceph_adm orch host add "$(hostname)"
+        ceph_adm orch apply crash '*'
+        ceph_adm config set mgr mgr/cephadm/allow_ptrace true
+    fi
+}
+
+start_mds() {
+    local mds=0
+    for name in a b c d e f g h i j k l m n o p
+    do
+        [ $mds -eq $CEPH_NUM_MDS ] && break
+        mds=$(($mds + 1))
+
+        if [ "$new" -eq 1 ]; then
+            prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
+            key_fn=$CEPH_DEV_DIR/mds.$name/keyring
+            wconf <<EOF
+[mds.$name]
+        host = $HOSTNAME
+EOF
+            if [ "$standby" -eq 1 ]; then
+                mkdir -p $CEPH_DEV_DIR/mds.${name}s
+                wconf <<EOF
+        mds standby for rank = $mds
+[mds.${name}s]
+        mds standby replay = true
+        mds standby for name = ${name}
+EOF
+            fi
+            prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
+            ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds 'allow' mgr 'allow profile mds'
+            if [ "$standby" -eq 1 ]; then
+                prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
+                     "$CEPH_DEV_DIR/mds.${name}s/keyring"
+                ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
+                             mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
+            fi
+        fi
+
+        run 'mds' $name $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
+        if [ "$standby" -eq 1 ]; then
+            run 'mds' $name $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
+        fi
+
+        #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
+        #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
+        #ceph_adm mds set max_mds 2
+    done
+
+    if [ $new -eq 1 ]; then
+        if [ "$CEPH_NUM_FS" -gt "0" ] ; then
+            sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation
+            if [ "$CEPH_NUM_FS" -gt "1" ] ; then
+                ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
+            fi
+
+	    # wait for volume module to load
+	    while ! ceph_adm fs volume ls ; do sleep 1 ; done
+            local fs=0
+            for name in a b c d e f g h i j k l m n o p
+            do
+                ceph_adm fs volume create ${name}
+                ceph_adm fs authorize ${name} "client.fs_${name}" / rwp >> "$keyring_fn"
+                fs=$(($fs + 1))
+                [ $fs -eq $CEPH_NUM_FS ] && break
+            done
+        fi
+    fi
+
+}
+
+# Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace
+# nfs-ganesha-rados-urls (version 3.3 and above) packages installed. On
+# Fedora>=31 these packages can be installed directly with 'dnf'. For CentOS>=8
+# the packages are available at
+# https://wiki.centos.org/SpecialInterestGroup/Storage
+# Similarly for Ubuntu>=16.04 follow the instructions on
+# https://launchpad.net/~nfs-ganesha
+
+start_ganesha() {
+    cluster_id="vstart"
+    GANESHA_PORT=$(($CEPH_PORT + 4000))
+    local ganesha=0
+    test_user="$cluster_id"
+    pool_name=".nfs"
+    namespace=$cluster_id
+    url="rados://$pool_name/$namespace/conf-nfs.$test_user"
+
+    prun ceph_adm auth get-or-create client.$test_user \
+        mon "allow r" \
+        osd "allow rw pool=$pool_name namespace=$namespace, allow rw tag cephfs data=a" \
+        mds "allow rw path=/" \
+        >> "$keyring_fn"
+
+    ceph_adm mgr module enable test_orchestrator
+    ceph_adm orch set backend test_orchestrator
+    ceph_adm test_orchestrator load_data -i $CEPH_ROOT/src/pybind/mgr/test_orchestrator/dummy_data.json
+    prun ceph_adm nfs cluster create $cluster_id
+    prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path "/cephfs"
+
+    for name in a b c d e f g h i j k l m n o p
+    do
+        [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break
+
+        port=$(($GANESHA_PORT + ganesha))
+        ganesha=$(($ganesha + 1))
+        ganesha_dir="$CEPH_DEV_DIR/ganesha.$name"
+        prun rm -rf $ganesha_dir
+        prun mkdir -p $ganesha_dir
+
+        echo "NFS_CORE_PARAM {
+            Enable_NLM = false;
+            Enable_RQUOTA = false;
+            Protocols = 4;
+            NFS_Port = $port;
+        }
+
+        MDCACHE {
+           Dir_Chunk = 0;
+        }
+
+        NFSv4 {
+           RecoveryBackend = rados_cluster;
+           Minor_Versions = 1, 2;
+        }
+
+        RADOS_KV {
+           pool = '$pool_name';
+           namespace = $namespace;
+           UserId = $test_user;
+           nodeid = $name;
+        }
+
+        RADOS_URLS {
+	   Userid = $test_user;
+	   watch_url = '$url';
+        }
+
+	%url $url" > "$ganesha_dir/ganesha-$name.conf"
+	wconf <<EOF
+[ganesha.$name]
+        host = $HOSTNAME
+        ip = $IP
+        port = $port
+        ganesha data = $ganesha_dir
+        pid file = $CEPH_OUT_DIR/ganesha-$name.pid
+EOF
+
+        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace add $name
+        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
+
+        prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_DEBUG
+
+        # Wait few seconds for grace period to be removed
+        sleep 2
+
+        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
+
+        echo "$test_user ganesha daemon $name started on port: $port"
+    done
+}
+
+if [ "$debug" -eq 0 ]; then
+    CMONDEBUG='
+        debug mon = 10
+        debug ms = 1'
+    CCLIENTDEBUG=''
+    CMDSDEBUG=''
+else
+    debug echo "** going verbose **"
+    CMONDEBUG='
+        debug osd = 20
+        debug mon = 20
+        debug osd = 20
+        debug paxos = 20
+        debug auth = 20
+        debug mgrc = 20
+        debug ms = 1'
+    CCLIENTDEBUG='
+        debug client = 20'
+    CMDSDEBUG='
+        debug mds = 20'
+fi
+
+# Crimson doesn't support PG merge/split yet.
+if [ "$ceph_osd" == "crimson-osd" ]; then
+    AUTOSCALER_OPTS='
+        osd_pool_default_pg_autoscale_mode = off'
+fi
+
+if [ -n "$MON_ADDR" ]; then
+    CMON_ARGS=" -m "$MON_ADDR
+    COSD_ARGS=" -m "$MON_ADDR
+    CMDS_ARGS=" -m "$MON_ADDR
+fi
+
+if [ -z "$CEPH_PORT" ]; then
+    while [ true ]
+    do
+        CEPH_PORT="$(echo $(( RANDOM % 1000 + 40000 )))"
+        ss -a -n | egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev/null 2>&1 || break
+    done
+fi
+
+[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
+
+# sudo if btrfs
+[ -d $CEPH_DEV_DIR/osd0/. ] && [ -e $CEPH_DEV_DIR/sudo ] && SUDO="sudo"
+
+if [ $inc_osd_num -eq 0 ]; then
+    prun $SUDO rm -f core*
+fi
+
+[ -d $CEPH_ASOK_DIR ] || mkdir -p $CEPH_ASOK_DIR
+[ -d $CEPH_OUT_DIR  ] || mkdir -p $CEPH_OUT_DIR
+[ -d $CEPH_DEV_DIR  ] || mkdir -p $CEPH_DEV_DIR
+[ -d $CEPH_OUT_CLIENT_DIR ] || mkdir -p $CEPH_OUT_CLIENT_DIR
+if [ $inc_osd_num -eq 0 ]; then
+    $SUDO find "$CEPH_OUT_DIR" -type f -delete
+fi
+[ -d gmon ] && $SUDO rm -rf gmon/*
+
+[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn
+
+
+# figure machine's ip
+HOSTNAME=`hostname -s`
+if [ -n "$ip" ]; then
+    IP="$ip"
+else
+    echo hostname $HOSTNAME
+    if [ -x "$(which ip 2>/dev/null)" ]; then
+        IP_CMD="ip addr"
+    else
+        IP_CMD="ifconfig"
+    fi
+    # filter out IPv4 and localhost addresses
+    IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
+    # if nothing left, try using localhost address, it might work
+    if [ -z "$IP" ]; then IP="127.0.0.1"; fi
+fi
+echo "ip $IP"
+echo "port $CEPH_PORT"
+
+
+[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
+
+ceph_adm() {
+    if [ "$cephx" -eq 1 ]; then
+        prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
+    else
+        prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
+    fi
+}
+
+if [ $inc_osd_num -gt 0 ]; then
+    start_osd
+    exit
+fi
+
+if [ "$new" -eq 1 ]; then
+    prepare_conf
+fi
+
+if [ $CEPH_NUM_MON -gt 0 ]; then
+    start_mon
+
+    debug echo Populating config ...
+    cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
+[global]
+osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE
+osd_pool_default_min_size = 1
+
+[mon]
+mon_osd_reporter_subtree_level = osd
+mon_data_avail_warn = 2
+mon_data_avail_crit = 1
+mon_allow_pool_delete = true
+mon_allow_pool_size_one = true
+
+[osd]
+osd_scrub_load_threshold = 2000
+osd_debug_op_order = true
+osd_debug_misdirected_ops = true
+osd_copyfrom_max_chunk = 524288
+
+[mds]
+mds_debug_frag = true
+mds_debug_auth_pins = true
+mds_debug_subtrees = true
+
+[mgr]
+mgr/telemetry/nag = false
+mgr/telemetry/enable = false
+
+EOF
+
+    if [ "$debug" -ne 0 ]; then
+        debug echo Setting debug configs ...
+        cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
+[mgr]
+debug_ms = 1
+debug_mgr = 20
+debug_monc = 20
+debug_mon = 20
+
+[osd]
+debug_ms = 1
+debug_osd = 25
+debug_objecter = 20
+debug_monc = 20
+debug_mgrc = 20
+debug_journal = 20
+debug_bluestore = 20
+debug_bluefs = 20
+debug_rocksdb = 20
+debug_bdev = 20
+debug_reserver = 10
+debug_objclass = 20
+
+[mds]
+debug_ms = 1
+debug_mds = 20
+debug_monc = 20
+debug_mgrc = 20
+mds_debug_scatterstat = true
+mds_verify_scatter = true
+EOF
+    fi
+    if [ "$cephadm" -gt 0 ]; then
+        debug echo Setting mon public_network ...
+        public_network=$(ip route list | grep -w "$IP" | awk '{print $1}')
+        ceph_adm config set mon public_network $public_network
+    fi
+fi
+
+if [ "$ceph_osd" == "crimson-osd" ]; then
+    $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp
+fi
+
+if [ $CEPH_NUM_MGR -gt 0 ]; then
+    start_mgr
+fi
+
+# osd
+if [ $CEPH_NUM_OSD -gt 0 ]; then
+    start_osd
+fi
+
+# mds
+if [ "$smallmds" -eq 1 ]; then
+    wconf <<EOF
+[mds]
+        mds log max segments = 2
+        # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB.
+        mds cache memory limit = 100M
+EOF
+fi
+
+if [ $CEPH_NUM_MDS -gt 0 ]; then
+    start_mds
+    # key with access to all FS
+    ceph_adm fs authorize \* "client.fs" / rwp >> "$keyring_fn"
+fi
+
+# Don't set max_mds until all the daemons are started, otherwise
+# the intended standbys might end up in active roles.
+if [ "$CEPH_MAX_MDS" -gt 1 ]; then
+    sleep 5  # wait for daemons to make it into FSMap before increasing max_mds
+fi
+fs=0
+for name in a b c d e f g h i j k l m n o p
+do
+    [ $fs -eq $CEPH_NUM_FS ] && break
+    fs=$(($fs + 1))
+    if [ "$CEPH_MAX_MDS" -gt 1 ]; then
+        ceph_adm fs set "${name}" max_mds "$CEPH_MAX_MDS"
+    fi
+done
+
+# mgr
+
+if [ "$ec" -eq 1 ]; then
+    ceph_adm <<EOF
+osd erasure-code-profile set ec-profile m=2 k=2
+osd pool create ec erasure ec-profile
+EOF
+fi
+
+do_cache() {
+    while [ -n "$*" ]; do
+        p="$1"
+        shift
+        debug echo "creating cache for pool $p ..."
+        ceph_adm <<EOF
+osd pool create ${p}-cache
+osd tier add $p ${p}-cache
+osd tier cache-mode ${p}-cache writeback
+osd tier set-overlay $p ${p}-cache
+EOF
+    done
+}
+do_cache $cache
+
+do_hitsets() {
+    while [ -n "$*" ]; do
+        pool="$1"
+        type="$2"
+        shift
+        shift
+        debug echo "setting hit_set on pool $pool type $type ..."
+        ceph_adm <<EOF
+osd pool set $pool hit_set_type $type
+osd pool set $pool hit_set_count 8
+osd pool set $pool hit_set_period 30
+EOF
+    done
+}
+do_hitsets $hitset
+
+do_rgw_create_bucket()
+{
+   # Create RGW Bucket
+   local rgw_python_file='rgw-create-bucket.py'
+   echo "import boto
+import boto.s3.connection
+
+conn = boto.connect_s3(
+        aws_access_key_id = '$s3_akey',
+        aws_secret_access_key = '$s3_skey',
+        host = '$HOSTNAME',
+        port = 80,
+        is_secure=False,
+        calling_format = boto.s3.connection.OrdinaryCallingFormat(),
+        )
+
+bucket = conn.create_bucket('nfs-bucket')
+print('created new bucket')" > "$CEPH_OUT_DIR/$rgw_python_file"
+   prun python $CEPH_OUT_DIR/$rgw_python_file
+}
+
+do_rgw_create_users()
+{
+    # Create S3 user
+    s3_akey='0555b35654ad1656d804'
+    s3_skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
+    debug echo "setting up user testid"
+    $CEPH_BIN/radosgw-admin user create --uid testid --access-key $s3_akey --secret $s3_skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null
+
+    # Create S3-test users
+    # See: https://github.com/ceph/s3-tests
+    debug echo "setting up s3-test users"
+    $CEPH_BIN/radosgw-admin user create \
+        --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
+        --access-key ABCDEFGHIJKLMNOPQRST \
+        --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
+        --display-name youruseridhere \
+        --email s3@example.com --caps="user-policy=*" -c $conf_fn > /dev/null
+    $CEPH_BIN/radosgw-admin user create \
+        --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
+        --access-key NOPQRSTUVWXYZABCDEFG \
+        --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
+        --display-name john.doe \
+        --email john.doe@example.com -c $conf_fn > /dev/null
+    $CEPH_BIN/radosgw-admin user create \
+	--tenant testx \
+        --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
+        --access-key HIJKLMNOPQRSTUVWXYZA \
+        --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
+        --display-name tenanteduser \
+        --email tenanteduser@example.com -c $conf_fn > /dev/null
+
+    # Create Swift user
+    debug echo "setting up user tester"
+    $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null
+
+    echo ""
+    echo "S3 User Info:"
+    echo "  access key:  $s3_akey"
+    echo "  secret key:  $s3_skey"
+    echo ""
+    echo "Swift User Info:"
+    echo "  account   : test"
+    echo "  user      : tester"
+    echo "  password  : testing"
+    echo ""
+}
+
+do_rgw()
+{
+    if [ "$new" -eq 1 ]; then
+        do_rgw_create_users
+        if [ -n "$rgw_compression" ]; then
+            debug echo "setting compression type=$rgw_compression"
+            $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null
+        fi
+    fi
+
+    if [ -n "$rgw_flight_frontend" ] ;then
+        debug echo "starting arrow_flight frontend on first rgw"
+    fi
+
+    # Start server
+    if [ "$cephadm" -gt 0 ]; then
+        ceph_adm orch apply rgw rgwTest
+        return
+    fi
+
+    RGWDEBUG=""
+    if [ "$debug" -ne 0 ]; then
+        RGWDEBUG="--debug-rgw=20 --debug-ms=1"
+    fi
+
+    local CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT}"
+    local CEPH_RGW_HTTPS="${CEPH_RGW_PORT: -1}"
+    if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then
+        CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT::-1}"
+    else
+        CEPH_RGW_HTTPS=""
+    fi
+    RGWSUDO=
+    [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo
+
+    current_port=$CEPH_RGW_PORT
+    # allow only first rgw to start arrow_flight server/port
+    local flight_conf=$rgw_flight_frontend
+    for n in $(seq 1 $CEPH_NUM_RGW); do
+        rgw_name="client.rgw.${current_port}"
+
+        ceph_adm auth get-or-create $rgw_name \
+            mon 'allow rw' \
+            osd 'allow rwx' \
+            mgr 'allow rw' \
+            >> "$keyring_fn"
+
+        debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
+        run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \
+            --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \
+            --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \
+            --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \
+            --rgw_luarocks_location=${CEPH_OUT_DIR}/luarocks \
+            --rgw_keystone_url=http://localhost:5000 \
+            ${RGWDEBUG} \
+            -n ${rgw_name} \
+            "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}${flight_conf:+,arrow_flight}"
+
+        i=$(($i + 1))
+        [ $i -eq $CEPH_NUM_RGW ] && break
+
+        current_port=$((current_port+1))
+        unset flight_conf
+    done
+}
+if [ "$CEPH_NUM_RGW" -gt 0 ]; then
+    do_rgw
+fi
+
+# Ganesha Daemons
+if [ $GANESHA_DAEMON_NUM -gt 0 ]; then
+    pseudo_path="/cephfs"
+    if [ "$cephadm" -gt 0 ]; then
+        cluster_id="vstart"
+	port="2049"
+        prun ceph_adm nfs cluster create $cluster_id
+	if [ $CEPH_NUM_MDS -gt 0 ]; then
+            prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path $pseudo_path
+	    echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint"
+	fi
+	if [ "$CEPH_NUM_RGW" -gt 0 ]; then
+            pseudo_path="/rgw"
+            do_rgw_create_bucket
+	    prun ceph_adm nfs export create rgw --cluster-id $cluster_id --pseudo-path $pseudo_path --bucket "nfs-bucket"
+            echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint"
+	fi
+    else
+        start_ganesha
+	echo "Mount using: mount -t nfs -o port=<ganesha-port-num> $IP:$pseudo_path mountpoint"
+    fi
+fi
+
+docker_service(){
+     local service=''
+     #prefer podman
+     if command -v podman > /dev/null; then
+	 service="podman"
+     elif pgrep -f docker > /dev/null; then
+	 service="docker"
+     fi
+     if [ -n "$service" ]; then
+       echo "using $service for deploying jaeger..."
+       #check for exited container, remove them and restart container
+       if [ "$($service ps -aq -f status=exited -f name=jaeger)" ]; then
+	 $service rm jaeger
+       fi
+       if [ ! "$(podman ps -aq -f name=jaeger)" ]; then
+         $service "$@"
+       fi
+     else
+         echo "cannot find docker or podman, please restart service and rerun."
+     fi
+}
+
+echo ""
+if [ $with_jaeger -eq 1 ]; then
+    debug echo "Enabling jaegertracing..."
+    docker_service run -d --name jaeger \
+  -p 5775:5775/udp \
+  -p 6831:6831/udp \
+  -p 6832:6832/udp \
+  -p 5778:5778 \
+  -p 16686:16686 \
+  -p 14268:14268 \
+  -p 14250:14250 \
+  quay.io/jaegertracing/all-in-one
+fi
+
+debug echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output."
+
+echo ""
+if [ "$new" -eq 1 ]; then
+    if $with_mgr_dashboard; then
+        cat <<EOF
+dashboard urls: $DASH_URLS
+  w/ user/pass: admin / admin
+EOF
+    fi
+    if $with_mgr_restful; then
+        cat <<EOF
+restful urls: $RESTFUL_URLS
+  w/ user/pass: admin / $RESTFUL_SECRET
+EOF
+    fi
+fi
+
+echo ""
+# add header to the environment file
+{
+    echo "#"
+    echo "# source this file into your shell to set up the environment."
+    echo "# For example:"
+    echo "# $ . $CEPH_DIR/vstart_environment.sh"
+    echo "#"
+} > $CEPH_DIR/vstart_environment.sh
+{
+    echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH"
+    echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH"
+    echo "export PATH=$CEPH_DIR/bin:\$PATH"
+    echo "export CEPH_CONF=$conf_fn"
+    # We cannot set CEPH_KEYRING if this is sourced by vstart_runner.py (API tests)
+    if [ "$CEPH_DIR" != "$PWD" ]; then
+        echo "export CEPH_KEYRING=$keyring_fn"
+    fi
+
+    if [ -n "$CEPHFS_SHELL" ]; then
+        echo "alias cephfs-shell=$CEPHFS_SHELL"
+    fi
+} | tee -a $CEPH_DIR/vstart_environment.sh
+
+echo "CEPH_DEV=1"
+
+# always keep this section at the very bottom of this file
+STRAY_CONF_PATH="/etc/ceph/ceph.conf"
+if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then
+    echo ""
+    echo ""
+    echo "WARNING:"
+    echo "    Please remove stray $STRAY_CONF_PATH if not needed."
+    echo "    Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync"
+    echo "    and may lead to undesired results."
+    echo ""
+    echo "NOTE:"
+    echo "    Remember to restart cluster after removing $STRAY_CONF_PATH"
+fi
+
+init_logrotate
diff --git a/src/test/test_perf_counters_cache.cc b/src/test/test_perf_counters_cache.cc
index 16d92bd7d431..8867896c02f2 100644
--- a/src/test/test_perf_counters_cache.cc
+++ b/src/test/test_perf_counters_cache.cc
@@ -960,13 +960,31 @@ TEST(PerfCountersCache, TestLabelStrings) {
 
   // test empty val in a label pair will get the label pair added into the perf counters cache but empty key will not
   std::string label2 = key_create("bad_ctrs1", {{"label3", "val4"}, {"label1", ""}});
-  EXPECT_DEATH(pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2), "");
+  //EXPECT_DEATH(pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2), "");
+  pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2);
 
   std::string label3 = key_create("bad_ctrs2", {{"", "val4"}, {"label1", "val1"}});
   EXPECT_DEATH(pcc->set_counter(label3, TEST_PERFCOUNTERS_COUNTER, 2), "");
 
   ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
   ASSERT_EQ(R"({
+    "bad_ctrs1": [
+        {
+            "labels": {
+                "label1": "",
+                "label3": "val4"
+            },
+            "counters": {
+                "test_counter": 2,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
     "good_ctrs": [
         {
             "labels": {
@@ -990,6 +1008,23 @@ TEST(PerfCountersCache, TestLabelStrings) {
   // test empty keys in each of the label pairs will not get the label added into the perf counters cache
   ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
   ASSERT_EQ(R"({
+    "bad_ctrs1": [
+        {
+            "labels": {
+                "label1": "",
+                "label3": "val4"
+            },
+            "counters": {
+                "test_counter": 2,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
     "good_ctrs": [
         {
             "labels": {
@@ -1023,6 +1058,23 @@ TEST(PerfCountersCache, TestLabelStrings) {
 
   ASSERT_EQ("", client.do_request(R"({ "prefix": "counter dump", "format": "raw" })", &message));
   ASSERT_EQ(R"({
+    "bad_ctrs1": [
+        {
+            "labels": {
+                "label1": "",
+                "label3": "val4"
+            },
+            "counters": {
+                "test_counter": 2,
+                "test_time": 0.000000000,
+                "test_time_avg": {
+                    "avgcount": 0,
+                    "sum": 0.000000000,
+                    "avgtime": 0.000000000
+                }
+            }
+        }
+    ],
     "good_ctrs": [
         {
             "labels": {

From 5d1b5da21591c57cb0cbbbc8775b6ea0ced953a4 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Wed, 11 Oct 2023 21:12:03 +0000
Subject: [PATCH 0167/2492] src/mon/Monitor: Fix set_elector_disallowed_leaders

Problem:

In the monitors we hold 2 copies of disallowed_leader ...
1. MonMap class 2. Elector class.
When computing the ConnectivityScore for the monitors during
the election, we use the `disallowed_leader` from Elector
class to determine which monitors we shouldn't allow to lead.

Now, we rely on the function `set_elector_disallowed_leaders`
to set the `disallowed_leader` of the Elector class, MonMap
class copy of the `disallowed_leader` contains the
`tiebreaker_monitor` so we inherit that plus we also add the
monitors that are dead due to a zone failure.

Hence, the `adding dead monitors` phase is only allowed if we can
enter stretch_mode. However, there is a problem when failing over a stretch cluster
zone and reviving the entire zone back up, the revived monitors
couldn't enter stretch_mode when they are at the state of "probing"
since PaxosServices like osdmon becomes unreadable (this is expected)

Solution:

We unconditionally add monitors that are in
`monmap->stretch_marked_down_mons` to the
`disallowed_leaders` list in
`Monitor::set_elector_disallowed_leaders` since
if the monitors are in `monmap->stretch_marked_down_mons`
we know that they probably belong in a marked down
zone and is not fit for lead.

This will fix the problem of newly revived monitors
having different disallowed_leaders set
and getting stuck in election.

Fixes: https://tracker.ceph.com/issues/63183

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/mon/Monitor.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 6866536d0654..27151e60b220 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -6658,14 +6658,16 @@ void Monitor::notify_new_monmap(bool can_change_external_state, bool remove_rank
 void Monitor::set_elector_disallowed_leaders(bool allow_election)
 {
   set<int> dl;
+  // inherit dl from monmap
   for (auto name : monmap->disallowed_leaders) {
     dl.insert(monmap->get_rank(name));
-  }
-  if (is_stretch_mode()) {
-    for (auto name : monmap->stretch_marked_down_mons) {
-      dl.insert(monmap->get_rank(name));
-    }
-    dl.insert(monmap->get_rank(monmap->tiebreaker_mon));
+  } // unconditionally add stretch_marked_down_mons to the new dl copy
+  for (auto name : monmap->stretch_marked_down_mons) {
+    dl.insert(monmap->get_rank(name));
+  } // add the tiebreaker_mon incase it is not in monmap->disallowed_leaders
+  if (!monmap->tiebreaker_mon.empty() &&
+      monmap->contains(monmap->tiebreaker_mon)) {
+      dl.insert(monmap->get_rank(monmap->tiebreaker_mon));
   }
 
   bool disallowed_changed = elector.set_disallowed_leaders(dl);

From 99d12712c566800d9a710bc281980d052c3ed144 Mon Sep 17 00:00:00 2001
From: galsalomon66 <gal.salomon@gmail.com>
Date: Sat, 9 Sep 2023 16:38:23 +0300
Subject: [PATCH 0168/2492] fix per QE defect. s3select submodule

s3select engine changes

s3select submodule
aggregation functions upon no data

Signed-off-by: galsalomon66 <gal.salomon@gmail.com>
---
 src/s3select | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/s3select b/src/s3select
index 8f86167c65cc..9ade26c63ef1 160000
--- a/src/s3select
+++ b/src/s3select
@@ -1 +1 @@
-Subproject commit 8f86167c65ccd4f134b6baec0eeb0ed7ea193bf8
+Subproject commit 9ade26c63ef1016dac868f53270a61e6232be9ba

From a5027e37ec1856b92e2af3fd19e23af537d040af Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Mon, 25 Sep 2023 18:58:06 +0530
Subject: [PATCH 0169/2492] mgr/dashboard: fix broken alert generator

Currently the alert generator is broken if you try to run `tox
-ealerts-fix`. I fixed it and ran the command and it built a new json
file as well.

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 monitoring/ceph-mixin/README.md                   | 6 ++++++
 monitoring/ceph-mixin/alerts.jsonnet              | 2 +-
 monitoring/ceph-mixin/jsonnetfile.lock.json       | 2 +-
 monitoring/ceph-mixin/prometheus_alerts.libsonnet | 2 +-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/monitoring/ceph-mixin/README.md b/monitoring/ceph-mixin/README.md
index 4772021939ef..f34d67f92a05 100644
--- a/monitoring/ceph-mixin/README.md
+++ b/monitoring/ceph-mixin/README.md
@@ -73,4 +73,10 @@ The jsonnet code located in this directory depends on some Jsonnet third party
 libraries. To update those libraries you can run `jb update` and then update
 the generated files using `tox -egrafonnet-fix`.
 
+### Building alerts from `prometheus_alerts.libsonnet`
+
+To rebuild the `prometheus_alerts.yml` file from the corresponding libsonnet,
+you can run `tox -ealerts-fix`.
+
+
 ##### Any upgrade or downgrade to different major versions of the recommended tools mentioned above is not supported.
diff --git a/monitoring/ceph-mixin/alerts.jsonnet b/monitoring/ceph-mixin/alerts.jsonnet
index ab7907c76fd1..13e70179f14d 100644
--- a/monitoring/ceph-mixin/alerts.jsonnet
+++ b/monitoring/ceph-mixin/alerts.jsonnet
@@ -1 +1 @@
-std.manifestYamlDoc(((import 'config.libsonnet') + (import 'alerts.libsonnet')).prometheusAlerts, indent_array_in_object=true, quote_keys=false)
+std.manifestYamlDoc((import 'mixin.libsonnet').prometheusAlerts, indent_array_in_object=true, quote_keys=false)
diff --git a/monitoring/ceph-mixin/jsonnetfile.lock.json b/monitoring/ceph-mixin/jsonnetfile.lock.json
index 3c9d38d935ce..480438230f39 100644
--- a/monitoring/ceph-mixin/jsonnetfile.lock.json
+++ b/monitoring/ceph-mixin/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "30280196507e0fe6fa978a3e0eaca3a62844f817",
+      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
       "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
     }
   ],
diff --git a/monitoring/ceph-mixin/prometheus_alerts.libsonnet b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
index a7c994ba9b6a..b90573be6c01 100644
--- a/monitoring/ceph-mixin/prometheus_alerts.libsonnet
+++ b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
@@ -236,7 +236,7 @@
           annotations: {
             documentation: 'https://docs.ceph.com/en/latest/rados/operations/health-checks#device-health-toomany',
             summary: 'Too many devices are predicted to fail, unable to resolve%(cluster)s' % $.MultiClusterSummary(),
-            description: 'The device health module has determined that devices predicted to fail can not be remediated automatically, since too many OSDs would be removed from the cluster to ensure performance and availabililty. Prevent data integrity issues by adding new OSDs so that data may be relocated.',
+            description: 'The device health module has determined that devices predicted to fail can not be remediated automatically, since too many OSDs would be removed from the cluster to ensure performance and availability. Prevent data integrity issues by adding new OSDs so that data may be relocated.',
           },
         },
         {

From b18a1441bebdb5c997ed3970d0a53dd39cebd277 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 13 Oct 2023 13:17:05 +0530
Subject: [PATCH 0170/2492] mgr/dashboard: disable hosts field while editing
 the filesystem

Even though the Placement field was disabled, the Host field was still
showing up in UI while Editing. That option is not possible in fs form.

Fixes: https://tracker.ceph.com/issues/63193
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../cephfs-form/cephfs-form.component.html    |  4 +--
 .../cephfs-form/cephfs-form.component.spec.ts | 29 +++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
index 76e51b2c5f39..05235d16ccd4 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
@@ -59,7 +59,7 @@
           </div>
 
           <!-- Label -->
-          <div *ngIf="form.controls.placement.value === 'label'"
+          <div *ngIf="form.controls.placement.value === 'label' && !editing"
                class="form-group row">
             <label i18n
                    class="cd-col-form-label"
@@ -79,7 +79,7 @@
           </div>
 
           <!-- Hosts -->
-          <div *ngIf="form.controls.placement.value === 'hosts'"
+          <div *ngIf="form.controls.placement.value === 'hosts' && !editing"
                class="form-group row">
             <label class="cd-col-form-label"
                    for="hosts"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
index 5409131d97bb..461f4bca052d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
@@ -7,10 +7,15 @@ import { FormHelper, configureTestBed } from '~/testing/unit-test-helper';
 import { SharedModule } from '~/app/shared/shared.module';
 import { ToastrModule } from 'ngx-toastr';
 import { ReactiveFormsModule } from '@angular/forms';
+import { By } from '@angular/platform-browser';
+import { OrchestratorService } from '~/app/shared/api/orchestrator.service';
+import { of } from 'rxjs';
+
 describe('CephfsVolumeFormComponent', () => {
   let component: CephfsVolumeFormComponent;
   let fixture: ComponentFixture<CephfsVolumeFormComponent>;
   let formHelper: FormHelper;
+  let orchService: OrchestratorService;
 
   configureTestBed({
     imports: [
@@ -27,6 +32,8 @@ describe('CephfsVolumeFormComponent', () => {
     fixture = TestBed.createComponent(CephfsVolumeFormComponent);
     component = fixture.componentInstance;
     formHelper = new FormHelper(component.form);
+    orchService = TestBed.inject(OrchestratorService);
+    spyOn(orchService, 'status').and.returnValue(of({ available: true }));
     fixture.detectChanges();
   });
 
@@ -50,4 +57,26 @@ describe('CephfsVolumeFormComponent', () => {
       formHelper.expectError('name', 'pattern');
     }
   }));
+
+  it('should show placement when orchestrator is available', () => {
+    const placement = fixture.debugElement.query(By.css('#placement'));
+    expect(placement).not.toBeNull();
+  });
+
+  describe('when editing', () => {
+    beforeEach(() => {
+      component.editing = true;
+      component.ngOnInit();
+      fixture.detectChanges();
+    });
+
+    it('should not show placement while editing even if orch is available', () => {
+      const placement = fixture.debugElement.query(By.css('#placement'));
+      const label = fixture.debugElement.query(By.css('#label'));
+      const hosts = fixture.debugElement.query(By.css('#hosts'));
+      expect(placement).toBeNull();
+      expect(label).toBeNull();
+      expect(hosts).toBeNull();
+    });
+  });
 });

From 3072b113242e01ed54a2154157f734e671d13a1a Mon Sep 17 00:00:00 2001
From: shimin <shimin@kuaishou.com>
Date: Fri, 13 Oct 2023 15:54:53 +0800
Subject: [PATCH 0171/2492] mgr: remove out&down osd from daemons

When we meet a bad disk problem, we just out the osd (alreay DOWN
 state). Before osd go to DOWN, it report slow ops to mgr, and `ceph -s`
command shows `18 slow ops, oldest one blocked for 259 sec...` warnings.
This warning never gone except we restart mgr.

Fixes: https://tracker.ceph.com/issues/63195
Signed-off-by: shimin <shimin@kuaishou.com>
---
 src/mgr/Mgr.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc
index cb988cf761cc..63ad530fae2f 100644
--- a/src/mgr/Mgr.cc
+++ b/src/mgr/Mgr.cc
@@ -525,7 +525,7 @@ void Mgr::handle_osd_map()
   cluster_state.with_osdmap_and_pgmap([this, &names_exist](const OSDMap &osd_map,
 							   const PGMap &pg_map) {
     for (int osd_id = 0; osd_id < osd_map.get_max_osd(); ++osd_id) {
-      if (!osd_map.exists(osd_id)) {
+      if (!osd_map.exists(osd_id) || (osd_map.is_out(osd_id) && osd_map.is_down(osd_id))) {
         continue;
       }
 

From e66e0b17f53ca3899ed082f4a010ec34561813a5 Mon Sep 17 00:00:00 2001
From: ivan <i.makarychev@tinkoff.ru>
Date: Mon, 2 Oct 2023 15:25:56 +0300
Subject: [PATCH 0172/2492] rgw: fix user.rgw.user-policy attr remove by modify
 user

Signed-off-by: ivan <i.makarychev@tinkoff.ru>
---
 src/rgw/driver/rados/rgw_user.cc | 10 ++++++++++
 src/rgw/driver/rados/rgw_user.h  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index 4e70c8552221..469fe6fe68fd 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -295,6 +295,15 @@ void RGWUserAdminOpState::set_user_version_tracker(RGWObjVersionTracker& objv_tr
   user->get_version_tracker() = objv_tracker;
 }
 
+void RGWUserAdminOpState::set_attrs(rgw::sal::Attrs& attrs)
+{
+  user->get_attrs() = attrs;
+}
+
+rgw::sal::Attrs RGWUserAdminOpState::get_attrs() {
+  return user->get_attrs();
+}
+
 const rgw_user& RGWUserAdminOpState::get_user_id()
 {
   return user->get_id();
@@ -1386,6 +1395,7 @@ int RGWUser::init(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state,
   
   op_state.set_existing_user(found);
   if (found) {
+    op_state.set_attrs(user->get_attrs());
     op_state.set_user_info(user->get_info());
     op_state.set_populated();
     op_state.objv = user->get_version_tracker();
diff --git a/src/rgw/driver/rados/rgw_user.h b/src/rgw/driver/rados/rgw_user.h
index aca905774a95..d5eb167daa3f 100644
--- a/src/rgw/driver/rados/rgw_user.h
+++ b/src/rgw/driver/rados/rgw_user.h
@@ -313,6 +313,10 @@ struct RGWUserAdminOpState {
     max_buckets_specified = true;
   }
 
+  rgw::sal::Attrs get_attrs();
+
+  void set_attrs(rgw::sal::Attrs& attrs);
+
   void set_gen_access() {
     gen_access = true;
     key_op = true;

From a8c8bb5d572125f867d182985a36368c428839c6 Mon Sep 17 00:00:00 2001
From: ivan <i.makarychev@tinkoff.ru>
Date: Fri, 13 Oct 2023 13:45:43 +0300
Subject: [PATCH 0173/2492] rgw: add subuser to user policy condition check

Signed-off-by: ivan <i.makarychev@tinkoff.ru>
---
 src/rgw/rgw_iam_policy_keywords.gperf | 2 ++
 src/rgw/rgw_iam_policy_keywords.h     | 1 +
 src/rgw/rgw_op.cc                     | 4 ++++
 3 files changed, 7 insertions(+)

diff --git a/src/rgw/rgw_iam_policy_keywords.gperf b/src/rgw/rgw_iam_policy_keywords.gperf
index af73dd130749..6b09a6aff818 100644
--- a/src/rgw/rgw_iam_policy_keywords.gperf
+++ b/src/rgw/rgw_iam_policy_keywords.gperf
@@ -115,6 +115,8 @@ Null, TokenKind::cond_op, TokenID::Null, (uint64_t) Type::null, true, true
 #s3:authType, TokenKind::cond_key, TokenID::s3authType, (uint64_t) Type::string, true, false
 #s3:signatureAge, TokenKind::cond_key, TokenID::s3signatureAge, (uint64_t) Type::number, true, false
 #s3:x-amz-content-sha256, TokenKind::cond_key, TokenID::s3x_amz_content_sha256, (uint64_t) Type::string, true, false
+# RGW
+#rgw:subuser, TokenKind::cond_key, TokenID::rgwsubuser, (uint64_t) Type::string, true, false
 # STS
 #sts:authentication, TokenKind::cond_key, TokenID::stsauthentication, (uint64_t) Type::boolean, true, false
 #
diff --git a/src/rgw/rgw_iam_policy_keywords.h b/src/rgw/rgw_iam_policy_keywords.h
index 8130ace456c6..c1cfa9052d60 100644
--- a/src/rgw/rgw_iam_policy_keywords.h
+++ b/src/rgw/rgw_iam_policy_keywords.h
@@ -89,6 +89,7 @@ enum class TokenID {
   s3authType,
   s3signatureAge,
   s3x_amz_content_sha256,
+  rgwsubuser,
 #else
   CondKey,
 #endif
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8c15e5bd2e3f..0f02ac9364b9 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -922,6 +922,10 @@ void rgw_build_iam_environment(rgw::sal::Driver* driver,
     s->env.emplace("aws:username", s->user->get_id().id);
   }
 
+  if (s->auth.identity) {
+    s->env.emplace("rgw:subuser", s->auth.identity->get_subuser().c_str());
+  }
+
   i = m.find("HTTP_X_AMZ_SECURITY_TOKEN");
   if (i != m.end()) {
     s->env.emplace("sts:authentication", "true");

From 37e7099267996a3075b4902a10a19d94fc738c08 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 12 Oct 2023 18:33:58 +1000
Subject: [PATCH 0174/2492] doc/rados: Edit troubleshooting-osd (2 of x)

Edit doc/rados/troubleshooting/troubleshooting.rst (2 of x).

Follows https://github.com/ceph/ceph/pull/53936.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-osd.rst   | 334 +++++++++++-------
 1 file changed, 203 insertions(+), 131 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-osd.rst b/doc/rados/troubleshooting/troubleshooting-osd.rst
index 27c9751f24f6..2e3bcd58ea94 100644
--- a/doc/rados/troubleshooting/troubleshooting-osd.rst
+++ b/doc/rados/troubleshooting/troubleshooting-osd.rst
@@ -267,206 +267,278 @@ If the cluster has started but an OSD isn't starting, check the following:
 An OSD Failed
 -------------
 
-When a ``ceph-osd`` process dies, surviving ``ceph-osd`` daemons will report
-to the mons that it appears down, which will in turn surface the new status
-via the ``ceph health`` command::
+When an OSD fails, this means that a ``ceph-osd`` process is unresponsive or
+has died and that the corresponding OSD has been marked ``down``. Surviving
+``ceph-osd`` daemons will report to the monitors that the OSD appears to be
+down, and a new status will be visible in the output of the ``ceph health``
+command, as in the following example:
 
-	ceph health
-	HEALTH_WARN 1/3 in osds are down
+.. prompt:: bash
+
+   ceph health
+
+::
+
+   HEALTH_WARN 1/3 in osds are down
 
-Specifically, you will get a warning whenever there are OSDs marked ``in``
-and ``down``.  You can identify which  are ``down`` with::
+This health alert is raised whenever there are one or more OSDs marked ``in``
+and ``down``. To see which OSDs are ``down``, add ``detail`` to the command as in
+the following example:
 
-	ceph health detail
-	HEALTH_WARN 1/3 in osds are down
-	osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
+.. prompt:: bash
 
-or ::
+   ceph health detail
 
-	ceph osd tree down
+::
+
+   HEALTH_WARN 1/3 in osds are down
+   osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
+
+Alternatively, run the following command:
+
+.. prompt:: bash
 
-If there is a drive
-failure or other fault preventing ``ceph-osd`` from functioning or
-restarting, an error message should be present in its log file under
-``/var/log/ceph``.
+    ceph osd tree down
 
-If the daemon stopped because of a heartbeat failure or ``suicide timeout``,
-the underlying drive or filesystem may be unresponsive. Check ``dmesg``
-and `syslog`  output for drive or other kernel errors.  You may need to
-specify something like ``dmesg -T`` to get timestamps, otherwise it's
-easy to mistake old errors for new.
+If there is a drive failure or another fault that is preventing a given
+``ceph-osd`` daemon from functioning or restarting, then there should be an
+error message present in its log file under ``/var/log/ceph``.
+
+If the ``ceph-osd`` daemon stopped because of a heartbeat failure or a
+``suicide timeout`` error, then the underlying drive or filesystem might be
+unresponsive. Check ``dmesg`` output and `syslog`  output for drive errors or
+kernel errors. It might be necessary to specify certain flags (for example,
+``dmesg -T`` to see human-readable timestamps) in order to avoid mistaking old
+errors for new errors.
+
+If an entire host's OSDs are ``down``, check to see if there is a network
+error or a hardware issue with the host.
+
+If the OSD problem is the result of a software error (for example, a failed
+assertion or another unexpected error), search for reports of the issue in the
+`bug tracker <https://tracker.ceph/com/projects/ceph>`_ , the `dev mailing list
+archives <https://lists.ceph.io/hyperkitty/list/dev@ceph.io/>`_, and the
+`ceph-users mailing list archives
+<https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/>`_.  If there is no
+clear fix or existing bug, then :ref:`report the problem to the ceph-devel
+email list <Get Involved>`.
 
-If the problem is a software error (failed assertion or other
-unexpected error), search the archives and tracker as above, and
-report it to the `ceph-devel`_ email list if there's no clear fix or
-existing bug.
 
 .. _no-free-drive-space:
 
 No Free Drive Space
 -------------------
 
-Ceph prevents you from writing to a full OSD so that you don't lose data.
-In an operational cluster, you should receive a warning when your cluster's OSDs
-and pools approach the full ratio. The ``mon_osd_full_ratio`` defaults to
-``0.95``, or 95% of capacity before it stops clients from writing data.
-The ``mon_osd_backfillfull_ratio`` defaults to ``0.90``, or 90 % of
-capacity above which backfills will not start. The
-OSD nearfull ratio defaults to ``0.85``, or 85% of capacity
-when it generates a health warning.
+If an OSD is full, Ceph prevents data loss by ensuring that no new data is
+written to the OSD. In an properly running cluster, health checks are raised
+when the cluster's OSDs and pools approach certain "fullness" ratios. The
+``mon_osd_full_ratio`` threshold defaults to ``0.95`` (or 95% of capacity):
+this is the point above which clients are prevented from writing data. The
+``mon_osd_backfillfull_ratio`` threshold defaults to ``0.90`` (or 90% of
+capacity): this is the point above which backfills will not start. The
+``mon_osd_nearfull_ratio`` threshold defaults to ``0.85`` (or 85% of capacity):
+this is the point at which it raises the ``OSD_NEARFULL`` health check.
+
+OSDs within a cluster will vary in how much data is allocated to them by Ceph.
+To check "fullness" by displaying data utilization for every OSD, run the
+following command:
+
+.. prompt:: bash
 
-Note that individual OSDs within a cluster will vary in how much data Ceph
-allocates to them.  This utilization can be displayed for each OSD with ::
+   ceph osd df
 
-	ceph osd df
+To check "fullness" by displaying a cluster’s overall data usage and data
+distribution among pools, run the following command:
 
-Overall cluster / pool fullness can be checked with ::
+.. prompt:: bash
 
-	ceph df 
+   ceph df 
 
-Pay close attention to the **most full** OSDs, not the percentage of raw space
-used as reported by ``ceph df``.  It only takes one outlier OSD filling up to
-fail writes to its pool.  The space available to each pool as reported by
-``ceph df`` considers the ratio settings relative to the *most full* OSD that
-is part of a given pool.  The distribution can be flattened by progressively
-moving data from overfull or to underfull OSDs using the ``reweight-by-utilization``
-command.  With Ceph releases beginning with later revisions of Luminous one can also
-exploit the ``ceph-mgr`` ``balancer`` module to perform this task automatically
-and rather effectively.
+When examining the output of the ``ceph df`` command, pay special attention to
+the **most full** OSDs, as opposed to the percentage of raw space used. If a
+single outlier OSD becomes full, all writes to this OSD's pool might fail as a
+result. When ``ceph df`` reports the space available to a pool, it considers
+the ratio settings relative to the *most full* OSD that is part of the pool. To
+flatten the distribution, two approaches are available: (1) Using the
+``reweight-by-utilization`` command to progressively move data from excessively
+full OSDs or move data to insufficiently full OSDs, and (2) in later revisions
+of Luminous and subsequent releases, exploiting the ``ceph-mgr`` ``balancer``
+module to perform the same task automatically.
 
-The ratios can be adjusted:
+To adjust the "fullness" ratios, run a command or commands of the following
+form:
+
+.. prompt:: bash
+
+   ceph osd set-nearfull-ratio <float[0.0-1.0]>
+   ceph osd set-full-ratio <float[0.0-1.0]>
+   ceph osd set-backfillfull-ratio <float[0.0-1.0]>
+
+Sometimes full cluster issues arise because an OSD has failed. This can happen
+either because of a test or because the cluster is small, very full, or
+unbalanced. When an OSD or node holds an excessive percentage of the cluster's
+data, component failures or natural growth can result in the ``nearfull`` and
+``full`` ratios being exceeded.  When testing Ceph's resilience to OSD failures
+on a small cluster, it is advised to leave ample free disk space and to
+consider temporarily lowering the OSD ``full ratio``, OSD ``backfillfull
+ratio``, and OSD ``nearfull ratio``.
+
+The "fullness" status of OSDs is visible in the output of the ``ceph health``
+command, as in the following example:
+
+.. prompt:: bash
+
+   ceph health
 
 ::
 
-    ceph osd set-nearfull-ratio <float[0.0-1.0]>
-    ceph osd set-full-ratio <float[0.0-1.0]>
-    ceph osd set-backfillfull-ratio <float[0.0-1.0]>
+  HEALTH_WARN 1 nearfull osd(s)
 
-Full cluster issues can arise when an OSD fails either as a test or organically
-within small and/or very full or unbalanced cluster. When an OSD or node
-holds an outsize percentage of the cluster's data, the ``nearfull`` and ``full``
-ratios may be exceeded as a result of component failures or even natural growth.
-If you are testing how Ceph reacts to OSD failures on a small
-cluster, you should leave ample free disk space and consider temporarily
-lowering the OSD ``full ratio``, OSD ``backfillfull ratio`` and
-OSD ``nearfull ratio``
+For details, add the ``detail`` command as in the following example:
 
-Full ``ceph-osds`` will be reported by ``ceph health``::
+.. prompt:: bash
 
-	ceph health
-	HEALTH_WARN 1 nearfull osd(s)
+    ceph health detail
 
-Or::
+::
 
-	ceph health detail
-	HEALTH_ERR 1 full osd(s); 1 backfillfull osd(s); 1 nearfull osd(s)
-	osd.3 is full at 97%
-	osd.4 is backfill full at 91%
-	osd.2 is near full at 87%
+    HEALTH_ERR 1 full osd(s); 1 backfillfull osd(s); 1 nearfull osd(s)
+    osd.3 is full at 97%
+    osd.4 is backfill full at 91%
+    osd.2 is near full at 87%
 
-The best way to deal with a full cluster is to add capacity via new OSDs, enabling
-the cluster to redistribute data to newly available storage.
+To address full cluster issues, it is recommended to add capacity by adding
+OSDs. Adding new OSDs allows the cluster to redistribute data to newly
+available storage. Search for ``rados bench`` orphans that are wasting space.
 
-If you cannot start a legacy Filestore OSD because it is full, you may reclaim
-some space deleting a few placement group directories in the full OSD.
+If a legacy Filestore OSD cannot be started because it is full, it is possible
+to reclaim space by deleting a small number of placement group directories in
+the full OSD.
 
-.. important:: If you choose to delete a placement group directory on a full OSD,
-   **DO NOT** delete the same placement group directory on another full OSD, or
-   **YOU WILL LOSE DATA**. You **MUST** maintain at least one copy of your data on
-   at least one OSD.  This is a rare and extreme intervention, and is not to be
-   undertaken lightly.
+.. important:: If you choose to delete a placement group directory on a full
+   OSD, **DO NOT** delete the same placement group directory on another full
+   OSD. **OTHERWISE YOU WILL LOSE DATA**. You **MUST** maintain at least one
+   copy of your data on at least one OSD. Deleting placement group directories
+   is a rare and extreme intervention. It is not to be undertaken lightly.
+
+See `Monitor Config Reference`_ for more information.
 
-See `Monitor Config Reference`_ for additional details.
 
 OSDs are Slow/Unresponsive
 ==========================
 
-A common issue involves slow or unresponsive OSDs. Ensure that you
-have eliminated other troubleshooting possibilities before delving into OSD
-performance issues. For example, ensure that your network(s) is working properly
-and your OSDs are running. Check to see if OSDs are throttling recovery traffic.
+OSDs are sometimes slow or unresponsive. When troubleshooting this common
+problem, it is advised to eliminate other possibilities before investigating
+OSD performance issues. For example, be sure to confirm that your network(s)
+are working properly, to verify that your OSDs are running, and to check
+whether OSDs are throttling recovery traffic.
+
+.. tip:: In pre-Luminous releases of Ceph, ``up`` and ``in`` OSDs were
+   sometimes not available or were otherwise slow because recovering OSDs were
+   consuming system resources. Newer releases provide better recovery handling
+   by preventing this phenomenon.
 
-.. tip:: Newer versions of Ceph provide better recovery handling by preventing
-   recovering OSDs from using up system resources so that ``up`` and ``in``
-   OSDs are not available or are otherwise slow.
 
 Networking Issues
 -----------------
 
-Ceph is a distributed storage system, so it relies upon networks for OSD peering
-and replication, recovery from faults, and periodic heartbeats. Networking
-issues can cause OSD latency and flapping OSDs. See `Flapping OSDs`_ for
-details.
+As a distributed storage system, Ceph relies upon networks for OSD peering and
+replication, recovery from faults, and periodic heartbeats. Networking issues
+can cause OSD latency and flapping OSDs. For more information, see `Flapping
+OSDs`_.
 
-Ensure that Ceph processes and Ceph-dependent processes are connected and/or
-listening. ::
+To make sure that Ceph processes and Ceph-dependent processes are connected and
+listening, run the following commands:
 
-	netstat -a | grep ceph
-	netstat -l | grep ceph
-	sudo netstat -p | grep ceph
+.. prompt:: bash
+
+   netstat -a | grep ceph
+   netstat -l | grep ceph
+   sudo netstat -p | grep ceph
 
-Check network statistics. ::
+To check network statistics, run the following command:
 
-	netstat -s
+.. prompt:: bash
+
+   netstat -s
 
 Drive Configuration
 -------------------
 
-A SAS or SATA storage drive should only house one OSD; NVMe drives readily
-handle two or more. Read and write throughput can bottleneck if other processes
-share the drive, including journals / metadata, operating systems, Ceph monitors,
-`syslog` logs, other OSDs, and non-Ceph processes.
+An SAS or SATA storage drive should house only one OSD, but a NVMe drive can
+easily house two or more. However, it is possible for read and write throughput
+to bottleneck if other processes share the drive. Such processes include:
+journals / metadata, operating systems, Ceph monitors, ``syslog`` logs, other
+OSDs, and non-Ceph processes.
 
-Ceph acknowledges writes *after* journaling, so fast SSDs are an
-attractive option to accelerate the response time--particularly when
-using the ``XFS`` or ``ext4`` file systems for legacy Filestore OSDs.
-By contrast, the ``Btrfs``
-file system can write and journal simultaneously.  (Note, however, that
-we recommend against using ``Btrfs`` for production deployments.)
+Because Ceph acknowledges writes *after* journaling, fast SSDs are an
+attractive option for accelerating response time -- particularly when using the
+``XFS`` or ``ext4`` filesystems for legacy FileStore OSDs.  By contrast, the
+``Btrfs`` file system can write and journal simultaneously. (However, use of
+``Btrfs`` is not recommended for production deployments.)
 
 .. note:: Partitioning a drive does not change its total throughput or
-   sequential read/write limits. Running a journal in a separate partition
-   may help, but you should prefer a separate physical drive.
+   sequential read/write limits. Throughput might be improved somewhat by
+   running a journal in a separate partition, but it is better still to run
+   such a journal in a separate physical drive.
+   
+.. warning:: Reef does not support FileStore. Releases after Reef do not
+   support FileStore. Any information that mentions FileStore is pertinent only
+   to the Quincy release of Ceph and to releases prior to Quincy.
+
 
 Bad Sectors / Fragmented Disk
 -----------------------------
 
-Check your drives for bad blocks, fragmentation, and other errors that can cause
-performance to drop substantially.  Invaluable tools include ``dmesg``, ``syslog``
-logs, and ``smartctl`` (from the ``smartmontools`` package).
+Check your drives for bad blocks, fragmentation, and other errors that can
+cause significantly degraded performance. Tools that are useful in checking for
+drive errors include ``dmesg``, ``syslog`` logs, and ``smartctl`` (found in the
+``smartmontools`` package).
+
+.. note:: ``smartmontools`` 7.0 and late provides NVMe stat passthrough and
+   JSON output.
+
 
 Co-resident Monitors/OSDs
 -------------------------
 
-Monitors are relatively lightweight processes, but they issue lots of
-``fsync()`` calls,
-which can interfere with other workloads, particularly if monitors run on the
-same drive as an OSD. Additionally, if you run monitors on the same host as
-OSDs, you may incur performance issues related to:
-
-- Running an older kernel (pre-3.0)
-- Running a kernel with no ``syncfs(2)`` syscall.
+Although monitors are relatively lightweight processes, performance issues can
+result when monitors are run on the same host machine as an OSD. Monitors issue
+many ``fsync()`` calls and this can interfere with other workloads. The danger
+of performance issues is especially acute when the monitors are co-resident on
+the same storage drive as an OSD. In addition, if the monitors are running an
+older kernel (pre-3.0) or a kernel with no ``syncfs(2)`` syscall, then multiple
+OSDs running on the same host might make so many commits as to undermine each
+other's performance.  This problem sometimes results in what is called "the
+bursty writes".
 
-In these cases, multiple OSDs running on the same host can drag each other down
-by doing lots of commits. That often leads to the bursty writes.
 
 Co-resident Processes
 ---------------------
 
-Spinning up co-resident processes (convergence) such as a cloud-based solution, virtual
-machines and other applications that write data to Ceph while operating on the
-same hardware as OSDs can introduce significant OSD latency. Generally, we
-recommend optimizing hosts for use with Ceph and using other hosts for other
-processes. The practice of separating Ceph operations from other applications
-may help improve performance and may streamline troubleshooting and maintenance.
+Significant OSD latency can result from processes that write data to Ceph (for
+example, cloud-based solutions and virtual machines) while operating on the
+same hardware as OSDs. For this reason, making such processes co-resident with
+OSDs is not generally recommended. Instead, the recommended practice is to
+optimize certain hosts for use with Ceph and use other hosts for other
+processes. This practice of separating Ceph operations from other applications
+might help improve performance and might also streamline troubleshooting and
+maintenance.
+
+Running co-resident processes on the same hardware is sometimes called
+"convergence". When using Ceph, engage in convergence only with expertise and
+after consideration.
+
 
 Logging Levels
 --------------
 
-If you turned logging levels up to track an issue and then forgot to turn
-logging levels back down, the OSD may be putting a lot of logs onto the disk. If
-you intend to keep logging levels high, you may consider mounting a drive to the
-default path for logging (i.e., ``/var/log/ceph/$cluster-$name.log``).
+Performance issues can result from high logging levels. Operators sometimes
+raise logging levels in order to track an issue and then forget to lower them
+afterwards. In such a situation, OSDs might consume valuable system resources to
+write needlessly verbose logs onto the disk. Anyone who does want to use high logging
+levels is advised to consider mounting a drive to the default path for logging
+(for example, ``/var/log/ceph/$cluster-$name.log``).
 
 Recovery Throttling
 -------------------

From a81bd2db3af4d7b53736be8e42a3eaa53028d60c Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 12 Oct 2023 19:03:10 +0200
Subject: [PATCH 0175/2492] pybind/rbd: don't produce info on errors in
 aio_mirror_image_get_info()

Check completion return value before attemting to decode c_info.
Otherwise we are guaranteed to access invalid memory in decode_cstr()
while trying to compute global_id string length when the client is
blocklisted for example.

Fixes: https://tracker.ceph.com/issues/63028
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/pybind/rbd/rbd.pyx | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx
index f9e5d8391d3e..fcb2fb347060 100644
--- a/src/pybind/rbd/rbd.pyx
+++ b/src/pybind/rbd/rbd.pyx
@@ -4511,13 +4511,18 @@ written." % (self.name, ret, length))
         def oncomplete_(completion_v):
             cdef:
                 Completion _completion_v = completion_v
-                rbd_mirror_image_info_t *c_info = <rbd_mirror_image_info_t *>_completion_v.buf
-            info = {
-                'global_id' : decode_cstr(c_info[0].global_id),
-                'state'     : int(c_info[0].state),
-                'primary'   : c_info[0].primary,
-            }
-            rbd_mirror_image_get_info_cleanup(c_info)
+                rbd_mirror_image_info_t *c_info
+            return_value = _completion_v.get_return_value()
+            if return_value == 0:
+                c_info = <rbd_mirror_image_info_t *>_completion_v.buf
+                info = {
+                    'global_id' : decode_cstr(c_info[0].global_id),
+                    'state'     : int(c_info[0].state),
+                    'primary'   : c_info[0].primary,
+                }
+                rbd_mirror_image_get_info_cleanup(c_info)
+            else:
+                info = None
             return oncomplete(_completion_v, info)
 
         completion = self.__get_completion(oncomplete_)

From 01fff6a72a328459c1d153e5dc1de6a34e48a82f Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 12 Oct 2023 21:32:53 +0200
Subject: [PATCH 0176/2492] mgr/rbd_support: make type hits on
 aio_mirror_image_*() callbacks better

Make it clear that mirror mode, mirror info and snap ID can be None if
the respective operation fails.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../rbd_support/mirror_snapshot_schedule.py   | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
index 122a3cd3997b..120b59318bac 100644
--- a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
+++ b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
@@ -125,7 +125,7 @@ def get_mirror_mode(self, image_spec: ImageSpec, image: rbd.Image) -> None:
         self.log.debug("CreateSnapshotRequests.get_mirror_mode: {}/{}/{}".format(
             pool_id, namespace, image_id))
 
-        def cb(comp: rados.Completion, mode: int) -> None:
+        def cb(comp: rados.Completion, mode: Optional[int]) -> None:
             self.handle_get_mirror_mode(image_spec, image, comp, mode)
 
         try:
@@ -140,14 +140,14 @@ def handle_get_mirror_mode(self,
                                image_spec: ImageSpec,
                                image: rbd.Image,
                                comp: rados.Completion,
-                               mode: int) -> None:
+                               mode: Optional[int]) -> None:
         pool_id, namespace, image_id = image_spec
 
         self.log.debug(
             "CreateSnapshotRequests.handle_get_mirror_mode {}/{}/{}: r={} mode={}".format(
                 pool_id, namespace, image_id, comp.get_return_value(), mode))
 
-        if comp.get_return_value() < 0:
+        if mode is None:
             if comp.get_return_value() != -errno.ENOENT:
                 self.log.error(
                     "error when getting mirror mode for {}/{}/{}: {}".format(
@@ -171,7 +171,7 @@ def get_mirror_info(self, image_spec: ImageSpec, image: rbd.Image) -> None:
         self.log.debug("CreateSnapshotRequests.get_mirror_info: {}/{}/{}".format(
             pool_id, namespace, image_id))
 
-        def cb(comp: rados.Completion, info: Dict[str, Union[str, int]]) -> None:
+        def cb(comp: rados.Completion, info: Optional[Dict[str, Union[str, int]]]) -> None:
             self.handle_get_mirror_info(image_spec, image, comp, info)
 
         try:
@@ -186,14 +186,14 @@ def handle_get_mirror_info(self,
                                image_spec: ImageSpec,
                                image: rbd.Image,
                                comp: rados.Completion,
-                               info: Dict[str, Union[str, int]]) -> None:
+                               info: Optional[Dict[str, Union[str, int]]]) -> None:
         pool_id, namespace, image_id = image_spec
 
         self.log.debug(
             "CreateSnapshotRequests.handle_get_mirror_info {}/{}/{}: r={} info={}".format(
                 pool_id, namespace, image_id, comp.get_return_value(), info))
 
-        if comp.get_return_value() < 0:
+        if info is None:
             if comp.get_return_value() != -errno.ENOENT:
                 self.log.error(
                     "error when getting mirror info for {}/{}/{}: {}".format(
@@ -218,7 +218,7 @@ def create_snapshot(self, image_spec: ImageSpec, image: rbd.Image) -> None:
             "CreateSnapshotRequests.create_snapshot for {}/{}/{}".format(
                 pool_id, namespace, image_id))
 
-        def cb(comp: rados.Completion, snap_id: int) -> None:
+        def cb(comp: rados.Completion, snap_id: Optional[int]) -> None:
             self.handle_create_snapshot(image_spec, image, comp, snap_id)
 
         try:
@@ -233,15 +233,14 @@ def handle_create_snapshot(self,
                                image_spec: ImageSpec,
                                image: rbd.Image,
                                comp: rados.Completion,
-                               snap_id: int) -> None:
+                               snap_id: Optional[int]) -> None:
         pool_id, namespace, image_id = image_spec
 
         self.log.debug(
             "CreateSnapshotRequests.handle_create_snapshot for {}/{}/{}: r={}, snap_id={}".format(
                 pool_id, namespace, image_id, comp.get_return_value(), snap_id))
 
-        if comp.get_return_value() < 0 and \
-           comp.get_return_value() != -errno.ENOENT:
+        if snap_id is None and comp.get_return_value() != -errno.ENOENT:
             self.log.error(
                 "error when creating snapshot for {}/{}/{}: {}".format(
                     pool_id, namespace, image_id, comp.get_return_value()))

From 073b0919ed9e413a04a7e12b7a0c23b5e058cd52 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 14 Oct 2023 04:17:44 +1000
Subject: [PATCH 0177/2492] doc/dev: remove cache-pool

Remove cache-pool, which is deprecated in Reef.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/dev/cache-pool.rst             | 200 -----------------------------
 doc/dev/osd_internals/manifest.rst |  34 -----
 2 files changed, 234 deletions(-)
 delete mode 100644 doc/dev/cache-pool.rst

diff --git a/doc/dev/cache-pool.rst b/doc/dev/cache-pool.rst
deleted file mode 100644
index 7dc71c828e9f..000000000000
--- a/doc/dev/cache-pool.rst
+++ /dev/null
@@ -1,200 +0,0 @@
-Cache pool
-==========
-
-Purpose
--------
-
-Use a pool of fast storage devices (probably SSDs) and use it as a
-cache for an existing slower and larger pool.
-
-Use a replicated pool as a front-end to service most I/O, and destage
-cold data to a separate erasure coded pool that does not currently (and
-cannot efficiently) handle the workload.
-
-We should be able to create and add a cache pool to an existing pool
-of data, and later remove it, without disrupting service or migrating
-data around.
-
-Use cases
----------
-
-Read-write pool, writeback
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-We have an existing data pool and put a fast cache pool "in front" of
-it.  Writes will go to the cache pool and immediately ack.  We flush
-them back to the data pool based on the defined policy.
-
-Read-only pool, weak consistency
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-We have an existing data pool and add one or more read-only cache
-pools.  We copy data to the cache pool(s) on read.  Writes are
-forwarded to the original data pool.  Stale data is expired from the
-cache pools based on the defined policy.
-
-This is likely only useful for specific applications with specific
-data access patterns.  It may be a match for rgw, for example.
-
-
-Interface
----------
-
-Set up a read/write cache pool foo-hot for pool foo::
-
- ceph osd tier add foo foo-hot
- ceph osd tier cache-mode foo-hot writeback
-
-Direct all traffic for foo to foo-hot::
-
- ceph osd tier set-overlay foo foo-hot
-
-Set the target size and enable the tiering agent for foo-hot::
-
- ceph osd pool set foo-hot hit_set_type bloom
- ceph osd pool set foo-hot hit_set_count 1
- ceph osd pool set foo-hot hit_set_period 3600   # 1 hour
- ceph osd pool set foo-hot target_max_bytes 1000000000000  # 1 TB
- ceph osd pool set foo-hot min_read_recency_for_promote 1
- ceph osd pool set foo-hot min_write_recency_for_promote 1
-
-Drain the cache in preparation for turning it off::
-
- ceph osd tier cache-mode foo-hot forward
- rados -p foo-hot cache-flush-evict-all
-
-When cache pool is finally empty, disable it::
-
- ceph osd tier remove-overlay foo
- ceph osd tier remove foo foo-hot
-
-Read-only pools with lazy consistency::
-
- ceph osd tier add foo foo-east
- ceph osd tier cache-mode foo-east readonly
- ceph osd tier add foo foo-west
- ceph osd tier cache-mode foo-west readonly
-
-
-
-Tiering agent
--------------
-
-The tiering policy is defined as properties on the cache pool itself.
-
-HitSet metadata
-~~~~~~~~~~~~~~~
-
-First, the agent requires HitSet information to be tracked on the
-cache pool in order to determine which objects in the pool are being
-accessed.  This is enabled with::
-
- ceph osd pool set foo-hot hit_set_type bloom
- ceph osd pool set foo-hot hit_set_count 1
- ceph osd pool set foo-hot hit_set_period 3600   # 1 hour
-
-The supported HitSet types include 'bloom' (a bloom filter, the
-default), 'explicit_hash', and 'explicit_object'.  The latter two
-explicitly enumerate accessed objects and are less memory efficient.
-They are there primarily for debugging and to demonstrate pluggability
-for the infrastructure.  For the bloom filter type, you can additionally
-define the false positive probability for the bloom filter (default is 0.05)::
-
- ceph osd pool set foo-hot hit_set_fpp 0.15
-
-The hit_set_count and hit_set_period define how much time each HitSet
-should cover, and how many such HitSets to store.  Binning accesses
-over time allows Ceph to independently determine whether an object was
-accessed at least once and whether it was accessed more than once over
-some time period ("age" vs "temperature").
-
-The ``min_read_recency_for_promote`` defines how many HitSets to check for the
-existence of an object when handling a read operation. The checking result is
-used to decide whether to promote the object asynchronously. Its value should be
-between 0 and ``hit_set_count``. If it's set to 0, the object is always promoted.
-If it's set to 1, the current HitSet is checked. And if this object is in the
-current HitSet, it's promoted. Otherwise not. For the other values, the exact
-number of archive HitSets are checked. The object is promoted if the object is
-found in any of the most recent ``min_read_recency_for_promote`` HitSets.
-
-A similar parameter can be set for the write operation, which is
-``min_write_recency_for_promote``. ::
-
- ceph osd pool set {cachepool} min_read_recency_for_promote 1
- ceph osd pool set {cachepool} min_write_recency_for_promote 1
-
-Note that the longer the ``hit_set_period`` and the higher the
-``min_read_recency_for_promote``/``min_write_recency_for_promote`` the more RAM
-will be consumed by the ceph-osd process. In particular, when the agent is active
-to flush or evict cache objects, all hit_set_count HitSets are loaded into RAM.
-
-Cache mode
-~~~~~~~~~~
-
-The most important policy is the cache mode:
-
- ceph osd pool set foo-hot cache-mode writeback
-
-The supported modes are 'none', 'writeback', 'forward', and
-'readonly'.  Most installations want 'writeback', which will write
-into the cache tier and only later flush updates back to the base
-tier.  Similarly, any object that is read will be promoted into the
-cache tier.
-
-The 'forward' mode is intended for when the cache is being disabled
-and needs to be drained.  No new objects will be promoted or written
-to the cache pool unless they are already present.  A background
-operation can then do something like::
-
-  rados -p foo-hot cache-try-flush-evict-all
-  rados -p foo-hot cache-flush-evict-all
-
-to force all data to be flushed back to the base tier.
-
-The 'readonly' mode is intended for read-only workloads that do not
-require consistency to be enforced by the storage system.  Writes will
-be forwarded to the base tier, but objects that are read will get
-promoted to the cache.  No attempt is made by Ceph to ensure that the
-contents of the cache tier(s) are consistent in the presence of object
-updates.
-
-Cache sizing
-~~~~~~~~~~~~
-
-The agent performs two basic functions: flushing (writing 'dirty'
-cache objects back to the base tier) and evicting (removing cold and
-clean objects from the cache).
-
-The thresholds at which Ceph will flush or evict objects is specified
-relative to a 'target size' of the pool.  For example::
-
- ceph osd pool set foo-hot cache_target_dirty_ratio .4
- ceph osd pool set foo-hot cache_target_dirty_high_ratio .6
- ceph osd pool set foo-hot cache_target_full_ratio .8
-
-will begin flushing dirty objects when 40% of the pool is dirty and begin
-evicting clean objects when we reach 80% of the target size.
-
-The target size can be specified either in terms of objects or bytes::
-
- ceph osd pool set foo-hot target_max_bytes 1000000000000  # 1 TB
- ceph osd pool set foo-hot target_max_objects 1000000       # 1 million objects
-
-Note that if both limits are specified, Ceph will begin flushing or
-evicting when either threshold is triggered.
-
-Other tunables
-~~~~~~~~~~~~~~
-
-You can specify a minimum object age before a recently updated object is
-flushed to the base tier::
-
- ceph osd pool set foo-hot cache_min_flush_age 600   # 10 minutes
-
-You can specify the minimum age of an object before it will be evicted from
-the cache tier::
-
- ceph osd pool set foo-hot cache_min_evict_age 1800   # 30 minutes
-
-
-
diff --git a/doc/dev/osd_internals/manifest.rst b/doc/dev/osd_internals/manifest.rst
index f998a04f2e7b..7be4350ead88 100644
--- a/doc/dev/osd_internals/manifest.rst
+++ b/doc/dev/osd_internals/manifest.rst
@@ -289,40 +289,6 @@ This seems complicated, but it gets us two valuable properties:
 All clone operations will need to consider adjacent ``chunk_maps``
 when adding or removing references.
 
-Cache/Tiering
--------------
-
-There already exists a cache/tiering mechanism based on whiteouts.
-One goal here should ultimately be for this manifest machinery to
-provide a complete replacement.
-
-See ``cache-pool.rst``
-
-The manifest machinery already shares some code paths with the
-existing cache/tiering code, mainly ``stat_flush``.
-
-In no particular order, here's in incomplete list of things that need
-to be wired up to provide feature parity:
-
-* Online object access information: The osd already has pool configs
-  for maintaining bloom filters which provide estimates of access
-  recency for objects.  We probably need to modify this to permit
-  hitset maintenance for a normal pool -- there are already
-  ``CEPH_OSD_OP_PG_HITSET*`` interfaces for querying them.
-* Tiering agent: The osd already has a background tiering agent which
-  would need to be modified to instead flush and evict using
-  manifests.
-
-* Use exiting existing features regarding the cache flush policy such as
-  histset, age, ratio.
-  - hitset
-  - age, ratio, bytes
-
-* Add tiering-mode to ``manifest-tiering``
-  - Writeback
-  - Read-only
-
-
 Data Structures
 ===============
 

From eab0be5ca513ca2dfaeb31911c175e88bcf4f9e4 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 14 Oct 2023 04:40:26 +1000
Subject: [PATCH 0178/2492] doc/rados: remove FileStore material (in Reef)

Remove FileStore-related material because FileStore is not supported in
Reef.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/filestore-config-ref.rst | 11 ++++++-----
 doc/releases/reef.rst                            |  2 ++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/doc/rados/configuration/filestore-config-ref.rst b/doc/rados/configuration/filestore-config-ref.rst
index 9d65d00a6cf6..7aefe26b38e0 100644
--- a/doc/rados/configuration/filestore-config-ref.rst
+++ b/doc/rados/configuration/filestore-config-ref.rst
@@ -4,11 +4,12 @@
 
 .. note:: Since the Luminous release of Ceph, Filestore has not been Ceph's
    default storage back end. Since the Luminous release of Ceph, BlueStore has
-   been Ceph's default storage back end.  However, Filestore OSDs are still
-   supported. See :ref:`OSD Back Ends
-   <rados_config_storage_devices_osd_backends>`. See :ref:`BlueStore Migration
-   <rados_operations_bluestore_migration>` for instructions explaining how to
-   replace an existing Filestore back end with a BlueStore back end.
+   been Ceph's default storage back end. However, Filestore OSDs are still
+   supported up to Quincy. Filestore OSDs are not supported in Reef. See
+   :ref:`OSD Back Ends <rados_config_storage_devices_osd_backends>`. See
+   :ref:`BlueStore Migration <rados_operations_bluestore_migration>` for
+   instructions explaining how to replace an existing Filestore back end with a
+   BlueStore back end.
 
 
 ``filestore_debug_omap_check``
diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
index 01e0f7b690bd..6f4f827830b7 100644
--- a/doc/releases/reef.rst
+++ b/doc/releases/reef.rst
@@ -27,6 +27,7 @@ Highlights
 
 See the relevant sections below for more details on these changes.
 
+* **RADOS** FileStore is not supported in Reef.
 * **RADOS:** RocksDB has been upgraded to version 7.9.2.
 * **RADOS:** There have been significant improvements to RocksDB iteration overhead and performance.
 * **RADOS:** The ``perf dump`` and ``perf schema`` commands have been deprecated in
@@ -115,6 +116,7 @@ MGR
 RADOS
 ~~~~~
 
+* FileStore is not supported in Reef.
 * RocksDB has been upgraded to version 7.9.2, which incorporates several
   performance improvements and features. This is the first release that can
   tune RocksDB settings per column family, which allows for more granular

From a402e34b625ca3030a36f4621c1924017c964c9b Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 2 Oct 2023 04:43:54 -0500
Subject: [PATCH 0179/2492] osd/scrub: modify 'a PG is reserving' to note PG

Only the PG that had set the 'I am in the process of
reserving my replicas' is allowed to clear that status.

That will simplify the follow-up commit, setting this flag from
a specific scrub-FSM state.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc       |  8 ++++----
 src/osd/scrubber/osd_scrub.h        |  4 ++--
 src/osd/scrubber/osd_scrub_sched.cc | 24 ++++++++++++++++++------
 src/osd/scrubber/osd_scrub_sched.h  | 15 +++++++++++----
 src/osd/scrubber/pg_scrubber.cc     |  8 ++++----
 5 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index 8da75233ebb9..e3a71e262347 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -466,12 +466,12 @@ int OsdScrub::get_blocked_pgs_count() const
   return m_queue.get_blocked_pgs_count();
 }
 
-bool OsdScrub::set_reserving_now()
+bool OsdScrub::set_reserving_now(spg_t reserving_id, utime_t now_is)
 {
-  return m_queue.set_reserving_now();
+  return m_queue.set_reserving_now(reserving_id, now_is);
 }
 
-void OsdScrub::clear_reserving_now()
+void OsdScrub::clear_reserving_now(spg_t reserving_id)
 {
-  m_queue.clear_reserving_now();
+  m_queue.clear_reserving_now(reserving_id);
 }
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 60e1f45adee9..570430660ed0 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -141,9 +141,9 @@ class OsdScrub {
    * and that PG is trying to acquire replica resources.
    * \retval false if the flag was already set (due to a race)
    */
-  bool set_reserving_now();
+  bool set_reserving_now(spg_t reserving_id, utime_t now_is);
 
-  void clear_reserving_now();
+  void clear_reserving_now(spg_t reserving_id);
 
   /**
    * \returns true if the current time is within the scrub time window
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index 691461860148..324899f29ab8 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -479,18 +479,30 @@ int ScrubQueue::get_blocked_pgs_count() const
 // ////////////////////////////////////////////////////////////////////////// //
 // ScrubQueue - maintaining the 'some PG is reserving' flag
 
-bool ScrubQueue::set_reserving_now()
+bool ScrubQueue::set_reserving_now(spg_t reserving_id, utime_t now_is)
 {
-  auto was_set = a_pg_is_reserving.exchange(true);
-  return !was_set;
+  std::unique_lock l{reserving_lock};
+
+  if (!reserving_pg.has_value()) {
+    reserving_pg = reserving_id;
+    reserving_since = now_is;
+    return true;
+  }
+  ceph_assert(reserving_id != *reserving_pg);
+  return false;
 }
 
-void ScrubQueue::clear_reserving_now()
+void ScrubQueue::clear_reserving_now(spg_t was_reserving_id)
 {
-  a_pg_is_reserving = false;
+  std::unique_lock l{reserving_lock};
+  if (reserving_pg && (*reserving_pg == was_reserving_id)) {
+    reserving_pg.reset();
+  }
+  // otherwise - ignore silently
 }
 
 bool ScrubQueue::is_reserving_now() const
 {
-  return a_pg_is_reserving;
+  // no lock needed, as set_reserving_now() will recheck
+  return reserving_pg.has_value();
 }
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index 9e222718c509..bd6de1c93478 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -246,8 +246,13 @@ class ScrubQueue {
    * (which is a possible result of a race between the check in OsdScrub and
    * the initiation of a scrub by some other PG)
    */
-  bool set_reserving_now();
-  void clear_reserving_now();
+  bool set_reserving_now(spg_t reserving_id, utime_t now_is);
+
+  /**
+   * silently ignore attempts to clear the flag if it was not set by
+   * the named pg.
+   */
+  void clear_reserving_now(spg_t reserving_id);
   bool is_reserving_now() const;
 
   /// counting the number of PGs stuck while scrubbing, waiting for objects
@@ -331,9 +336,11 @@ class ScrubQueue {
    * trying to secure its replicas' resources. We will refrain from initiating
    * any other scrub sessions until this one is done.
    *
-   * \todo keep the ID of the reserving PG; possibly also the time it started.
+   * \todo replace the local lock with regular osd-service locking
    */
-  std::atomic_bool a_pg_is_reserving{false};
+  ceph::mutex reserving_lock = ceph::make_mutex("ScrubQueue::reserving_lock");
+  std::optional<spg_t> reserving_pg;
+  utime_t reserving_since;
 
   /**
    * If the scrub job was not explicitly requested, we postpone it by some
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 22f5606d454b..e27e50b70d41 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -1687,14 +1687,14 @@ void PgScrubber::on_replica_reservation_timeout()
   }
 }
 
-bool PgScrubber::set_reserving_now()
-{
-  return m_osds->get_scrub_services().set_reserving_now();
+bool PgScrubber::set_reserving_now() {
+  return m_osds->get_scrub_services().set_reserving_now(m_pg_id,
+                                                        ceph_clock_now());
 }
 
 void PgScrubber::clear_reserving_now()
 {
-  m_osds->get_scrub_services().clear_reserving_now();
+  m_osds->get_scrub_services().clear_reserving_now(m_pg_id);
 }
 
 void PgScrubber::set_queued_or_active()

From 817fd00240cf8ead3f4333154824b2c262dd868b Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 2 Oct 2023 11:29:51 -0500
Subject: [PATCH 0180/2492] osd/scrub: group all scrub session states into a
 Session state

The Session state now includes the ReservingReplicas & Active
sub-states.

This new state will hold (in future commits) most of the scrub
state information that relates to a specific scrub session (and
should be cleaned up when that session terminates).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_machine.cc | 110 +++++++++++++++---------------
 src/osd/scrubber/scrub_machine.h  |  53 ++++++++++----
 2 files changed, 94 insertions(+), 69 deletions(-)

diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 0d52d5b76d77..efb091788206 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -109,25 +109,47 @@ sc::result NotActive::react(const AfterRepairScrub&)
   return transit<ReservingReplicas>();
 }
 
+// ----------------------- Session -----------------------------------------
+
+Session::Session(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session")
+{
+  dout(10) << "-- state -->> Session" << dendl;
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+
+  // while we've checked the 'someone is reserving' flag before queueing
+  // the start-scrub event, it's possible that the flag was set in the meantime.
+  // Handling this case here requires adding a new sub-state, and the
+  // complication of reporting a failure to the caller in a new failure
+  // path. On the other hand - ignoring an ongoing reservation on rare
+  // occasions will cause no harm.
+  // We choose ignorance.
+  std::ignore = scrbr->set_reserving_now();
+}
+
+Session::~Session()
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+
+  // note the interaction between clearing the 'queued' flag and two
+  // other states: the snap-mapper and the scrubber internal state.
+  // All of these must be cleared in the correct order, and the snap mapper
+  // (re-triggered by resetting the 'queued' flag) must not resume before
+  // the scrubber is reset.
+  scrbr->clear_pgscrub_state();
+}
+
+
 // ----------------------- ReservingReplicas ---------------------------------
 
 ReservingReplicas::ReservingReplicas(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReservingReplicas")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/ReservingReplicas")
 {
   dout(10) << "-- state -->> ReservingReplicas" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
-  // prevent the OSD from starting another scrub while we are trying to secure
-  // replicas resources
-  if (!scrbr->set_reserving_now()) {
-    dout(1) << "ReservingReplicas::ReservingReplicas() some other PG is "
-		"already reserving replicas resources"
-	     << dendl;
-    post_event(ReservationFailure{});
-    return;
-  }
-  m_holding_isreserving_flag = true;
   scrbr->reserve_replicas();
 
   auto timeout = scrbr->get_cct()->_conf.get_val<
@@ -143,9 +165,9 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
 ReservingReplicas::~ReservingReplicas()
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  if (m_holding_isreserving_flag) {
-    scrbr->clear_reserving_now();
-  }
+  // it's OK to try and clear the flag even if we don't hold it
+  // (the flag remembers the actual holder)
+  scrbr->clear_reserving_now();
 }
 
 sc::result ReservingReplicas::react(const ReservationTimeout&)
@@ -166,18 +188,6 @@ sc::result ReservingReplicas::react(const ReservationFailure&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReservingReplicas::react(const ReservationFailure&)" << dendl;
-
-  // the Scrubber must release all resources and abort the scrubbing
-  scrbr->clear_pgscrub_state();
-  return transit<NotActive>();
-}
-
-/**
- * note: the event poster is handling the scrubber reset
- */
-sc::result ReservingReplicas::react(const FullReset&)
-{
-  dout(10) << "ReservingReplicas::react(const FullReset&)" << dendl;
   return transit<NotActive>();
 }
 
@@ -185,7 +195,7 @@ sc::result ReservingReplicas::react(const FullReset&)
 
 ActiveScrubbing::ActiveScrubbing(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ActiveScrubbing")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/ActiveScrubbing")
 {
   dout(10) << "-- state -->> ActiveScrubbing" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
@@ -199,8 +209,6 @@ ActiveScrubbing::~ActiveScrubbing()
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(15) << __func__ << dendl;
-  scrbr->unreserve_replicas();
-  scrbr->clear_queued_or_active();
 }
 
 /*
@@ -212,14 +220,6 @@ sc::result ActiveScrubbing::react(const InternalError&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << __func__ << dendl;
-  scrbr->clear_pgscrub_state();
-  return transit<NotActive>();
-}
-
-sc::result ActiveScrubbing::react(const FullReset&)
-{
-  dout(10) << "ActiveScrubbing::react(const FullReset&)" << dendl;
-  // caller takes care of clearing the scrubber & FSM states
   return transit<NotActive>();
 }
 
@@ -237,9 +237,9 @@ sc::result ActiveScrubbing::react(const FullReset&)
  */
 RangeBlocked::RangeBlocked(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/RangeBlocked")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/RangeBlocked")
 {
-  dout(10) << "-- state -->> Act/RangeBlocked" << dendl;
+  dout(10) << "-- state -->> Session/Act/RangeBlocked" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   auto grace = scrbr->get_range_blocked_grace();
@@ -287,9 +287,9 @@ sc::result RangeBlocked::react(const RangeBlockedAlarm&)
  */
 PendingTimer::PendingTimer(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/PendingTimer")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/PendingTimer")
 {
-  dout(10) << "-- state -->> Act/PendingTimer" << dendl;
+  dout(10) << "-- state -->> Session/Act/PendingTimer" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   auto sleep_time = scrbr->get_scrub_sleep_time();
@@ -328,9 +328,9 @@ sc::result PendingTimer::react(const SleepComplete&)
  */
 NewChunk::NewChunk(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/NewChunk")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/NewChunk")
 {
-  dout(10) << "-- state -->> Act/NewChunk" << dendl;
+  dout(10) << "-- state -->> Session/Act/NewChunk" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   scrbr->get_preemptor().adjust_parameters();
@@ -355,9 +355,9 @@ sc::result NewChunk::react(const SelectedChunkFree&)
 
 WaitPushes::WaitPushes(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/WaitPushes")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/WaitPushes")
 {
-  dout(10) << " -- state -->> Act/WaitPushes" << dendl;
+  dout(10) << " -- state -->> Session/Act/WaitPushes" << dendl;
   post_event(ActivePushesUpd{});
 }
 
@@ -383,9 +383,9 @@ sc::result WaitPushes::react(const ActivePushesUpd&)
 
 WaitLastUpdate::WaitLastUpdate(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/WaitLastUpdate")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/WaitLastUpdate")
 {
-  dout(10) << " -- state -->> Act/WaitLastUpdate" << dendl;
+  dout(10) << " -- state -->> Session/Act/WaitLastUpdate" << dendl;
   post_event(UpdatesApplied{});
 }
 
@@ -427,9 +427,9 @@ sc::result WaitLastUpdate::react(const InternalAllUpdates&)
 
 BuildMap::BuildMap(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/BuildMap")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/BuildMap")
 {
-  dout(10) << " -- state -->> Act/BuildMap" << dendl;
+  dout(10) << " -- state -->> Session/Act/BuildMap" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   // no need to check for an epoch change, as all possible flows that brought
@@ -477,9 +477,9 @@ sc::result BuildMap::react(const IntLocalMapDone&)
 
 DrainReplMaps::DrainReplMaps(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/DrainReplMaps")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/DrainReplMaps")
 {
-  dout(10) << "-- state -->> Act/DrainReplMaps" << dendl;
+  dout(10) << "-- state -->> Session/Act/DrainReplMaps" << dendl;
   // we may have got all maps already. Send the event that will make us check.
   post_event(GotReplicas{});
 }
@@ -504,9 +504,9 @@ sc::result DrainReplMaps::react(const GotReplicas&)
 
 WaitReplicas::WaitReplicas(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/WaitReplicas")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/WaitReplicas")
 {
-  dout(10) << "-- state -->> Act/WaitReplicas" << dendl;
+  dout(10) << "-- state -->> Session/Act/WaitReplicas" << dendl;
   post_event(GotReplicas{});
 }
 
@@ -564,10 +564,10 @@ sc::result WaitReplicas::react(const DigestUpdate&)
 
 WaitDigestUpdate::WaitDigestUpdate(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Act/WaitDigestUpdate")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/Act/WaitDigestUpdate")
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "-- state -->> Act/WaitDigestUpdate" << dendl;
+  dout(10) << "-- state -->> Session/Act/WaitDigestUpdate" << dendl;
 
   // perform an initial check: maybe we already
   // have all the updates we need:
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 658abfa494f1..071a464ce130 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -141,6 +141,7 @@ MEV(ScrubFinished)
 //
 
 struct NotActive;	    ///< the quiescent state. No active scrubbing.
+struct Session;            ///< either reserving or actively scrubbing
 struct ReservingReplicas;   ///< securing scrub resources from replicas' OSDs
 struct ActiveScrubbing;	    ///< the active state for a Primary. A sub-machine.
 struct ReplicaIdle;         ///< Initial reserved replica state
@@ -163,13 +164,17 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
   [[nodiscard]] bool is_reserving() const;
   [[nodiscard]] bool is_accepting_updates() const;
 
+
+// ///////////////// aux declarations & functions //////////////////////// //
+
+
 private:
   /**
    * scheduled_event_state_t
    *
    * Heap allocated, ref-counted state shared between scheduled event callback
    * and timer_event_token_t.  Ensures that callback and timer_event_token_t
-   * can be safetly destroyed in either order while still allowing for
+   * can be safely destroyed in either order while still allowing for
    * cancellation.
    */
   struct scheduled_event_state_t {
@@ -183,7 +188,7 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
     ~scheduled_event_state_t() {
       /* For the moment, this assert encodes an assumption that we always
        * retain the token until the event either fires or is canceled.
-       * If a user needs/wants to relaxt that requirement, this assert can
+       * If a user needs/wants to relax that requirement, this assert can
        * be removed */
       assert(!cb_token);
     }
@@ -259,7 +264,7 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
    * schedule_timer_event_after
    *
    * Schedules event EventT{Args...} to be delivered duration in the future.
-   * The implementation implicitely drops the event on interval change.  The
+   * The implementation implicitly drops the event on interval change.  The
    * returned timer_event_token_t can be used to cancel the event prior to
    * its delivery -- it should generally be embedded as a member in the state
    * intended to handle the event.  See the comment on timer_event_token_t
@@ -284,6 +289,10 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
   }
 };
 
+
+// ///////////////// the states //////////////////////// //
+
+
 /**
  *  The Scrubber's base (quiescent) state.
  *  Scrubbing is triggered by one of the following events:
@@ -314,11 +323,34 @@ struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
   sc::result react(const AfterRepairScrub&);
 };
 
-struct ReservingReplicas : sc::state<ReservingReplicas, ScrubMachine>,
+
+/**
+ *  Session
+ *
+ *  This state encompasses the two main "active" states: ReservingReplicas and
+ *  ActiveScrubbing.
+ *  'Session' is the owner of all the resources that are allocated for a
+ *  scrub session performed as a Primary.
+ *
+ *  Exit from this state is either following an interval change, or with
+ *  'FullReset' (that would cover all other completion/termination paths).
+ *  Note that if terminating the session following an interval change - no
+ *  reservations are released. This is because we know that the replicas are
+ *  also resetting their reservations.
+ */
+struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>, NamedSimply {
+  explicit Session(my_context ctx);
+  ~Session();
+
+  using reactions = mpl::list<sc::transition<FullReset, NotActive>>;
+  /// \todo handle interval change
+};
+
+struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
 			   NamedSimply {
   explicit ReservingReplicas(my_context ctx);
   ~ReservingReplicas();
-  using reactions = mpl::list<sc::custom_reaction<FullReset>,
+  using reactions = mpl::list<
 			      // all replicas granted our resources request
 			      sc::transition<RemotesReserved, ActiveScrubbing>,
 			      sc::custom_reaction<ReservationTimeout>,
@@ -328,11 +360,6 @@ struct ReservingReplicas : sc::state<ReservingReplicas, ScrubMachine>,
     ceph::coarse_real_clock::now();
   ScrubMachine::timer_event_token_t m_timeout_token;
 
-  /// if true - we must 'clear_reserving_now()' upon exit
-  bool m_holding_isreserving_flag{false};
-
-  sc::result react(const FullReset&);
-
   sc::result react(const ReservationTimeout&);
 
   /// at least one replica denied us the scrub resources we've requested
@@ -364,15 +391,13 @@ struct WaitReplicas;
 struct WaitDigestUpdate;
 
 struct ActiveScrubbing
-    : sc::state<ActiveScrubbing, ScrubMachine, PendingTimer>, NamedSimply {
+    : sc::state<ActiveScrubbing, Session, PendingTimer>, NamedSimply {
 
   explicit ActiveScrubbing(my_context ctx);
   ~ActiveScrubbing();
 
-  using reactions = mpl::list<sc::custom_reaction<InternalError>,
-			      sc::custom_reaction<FullReset>>;
+  using reactions = mpl::list<sc::custom_reaction<InternalError>>;
 
-  sc::result react(const FullReset&);
   sc::result react(const InternalError&);
 };
 

From cbb6c33b41d51a7d46de5efaf4ee4141a62d0652 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 24 Sep 2023 07:34:14 -0500
Subject: [PATCH 0181/2492] osd/scrub: reserve replicas one by one, and in
 consistent order

Issuing the reservation requests one by one - waiting for
approval from the secondary before the next request is sent.

The requests are sent in ascending target pg-shard-id order, reducing the
chance of having two PGs repeatedly competing for the same set of
OSDs - and doing so in an interleaved sequence.

Modifying the Session state in the scrubber FSM to react to interval
changes by discarding replica reservations.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                          |   2 +-
 src/osd/scrubber/pg_scrubber.cc        | 394 +++++++++++--------------
 src/osd/scrubber/pg_scrubber.h         | 154 +++++-----
 src/osd/scrubber/scrub_machine.cc      |  19 +-
 src/osd/scrubber/scrub_machine.h       |  16 +-
 src/osd/scrubber/scrub_machine_lstnr.h |  16 +-
 src/osd/scrubber_common.h              |   5 -
 7 files changed, 294 insertions(+), 312 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 0d1f8d44e1c4..580b3add8484 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -359,7 +359,7 @@ void PG::clear_primary_state()
   release_pg_backoffs();
 
   if (m_scrubber) {
-    m_scrubber->discard_replica_reservations();
+    m_scrubber->on_new_interval();
   }
   scrub_after_recovery = false;
 
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index e27e50b70d41..294d050268d0 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -5,6 +5,7 @@
 
 #include <cmath>
 #include <iostream>
+#include <span>
 #include <vector>
 
 #include <fmt/ranges.h>
@@ -469,7 +470,7 @@ unsigned int PgScrubber::scrub_requeue_priority(
 
 /* on_new_interval
  *
- * Responsible for restting any scrub state and releasing any resources.
+ * Responsible for resetting any scrub state and releasing any resources.
  * Any inflight events will be ignored via check_interval/should_drop_message
  * or canceled.
  */
@@ -481,11 +482,13 @@ void PgScrubber::on_new_interval()
 		  is_scrub_active(), is_queued_or_active())
 	   << dendl;
 
+  // If in active session - the IntervalChanged handler takes care of
+  // discarding the remote reservations, and transitioning out of Session.
+  // That resets both the scrubber and the FSM.
+  m_fsm->process_event(IntervalChanged{});
+
+  // The 'FullReset' is only relevant if we are not an active Primary
   m_fsm->process_event(FullReset{});
-  // we may be the primary
-  if (is_queued_or_active()) {
-    clear_pgscrub_state();
-  }
   rm_from_osd_scrubbing();
 }
 
@@ -1569,7 +1572,7 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
    * replicas.  Unconditionally clear any existing state prior to handling
    * the new reservation. */
   m_fsm->process_event(FullReset{});
-  
+
   bool granted{false};
   if (m_pg->cct->_conf->osd_scrub_during_recovery ||
       !m_osds->is_recovery_active()) {
@@ -1600,6 +1603,9 @@ void PgScrubber::handle_scrub_reserve_grant(OpRequestRef op, pg_shard_t from)
   dout(10) << __func__ << " " << *op->get_req() << dendl;
   op->mark_started();
 
+  if (should_drop_message(op)) {
+    return;
+  }
   if (m_reservations.has_value()) {
     m_reservations->handle_reserve_grant(op, from);
   } else {
@@ -1613,6 +1619,9 @@ void PgScrubber::handle_scrub_reserve_reject(OpRequestRef op, pg_shard_t from)
   dout(10) << __func__ << " " << *op->get_req() << dendl;
   op->mark_started();
 
+  if (should_drop_message(op)) {
+    return;
+  }
   if (m_reservations.has_value()) {
     // there is an active reservation process. No action is required otherwise.
     m_reservations->handle_reserve_reject(op, from);
@@ -1623,6 +1632,11 @@ void PgScrubber::handle_scrub_reserve_release(OpRequestRef op)
 {
   dout(10) << __func__ << " " << *op->get_req() << dendl;
   op->mark_started();
+  if (should_drop_message(op)) {
+    // we might have turned into a Primary in the meantime. The interval
+    // change should have been noticed already, and caused us to reset.
+    return;
+  }
 
   /*
    * this specific scrub session has terminated. All incoming events carrying
@@ -1635,7 +1649,7 @@ void PgScrubber::discard_replica_reservations()
 {
   dout(10) << __func__ << dendl;
   if (m_reservations.has_value()) {
-    m_reservations->discard_all();
+    m_reservations->discard_remote_reservations();
   }
 }
 
@@ -1646,47 +1660,6 @@ void PgScrubber::clear_scrub_reservations()
   m_local_osd_resource.reset();	  // the local reservation
 }
 
-void PgScrubber::message_all_replicas(int32_t opcode, std::string_view op_text)
-{
-  ceph_assert(m_pg->recovery_state.get_backfill_targets().empty());
-
-  std::vector<pair<int, Message*>> messages;
-  messages.reserve(m_pg->get_actingset().size());
-
-  epoch_t epch = get_osdmap_epoch();
-
-  for (auto& p : m_pg->get_actingset()) {
-
-    if (p == m_pg_whoami)
-      continue;
-
-    dout(10) << "scrub requesting " << op_text << " from osd." << p
-	     << " Epoch: " << epch << dendl;
-    Message* m = new MOSDScrubReserve(spg_t(m_pg->info.pgid.pgid, p.shard),
-				      epch,
-				      opcode,
-				      m_pg_whoami);
-    messages.push_back(std::make_pair(p.osd, m));
-  }
-
-  if (!messages.empty()) {
-    m_osds->send_message_osd_cluster(messages, epch);
-  }
-}
-
-void PgScrubber::unreserve_replicas()
-{
-  dout(10) << __func__ << dendl;
-  m_reservations.reset();
-}
-
-void PgScrubber::on_replica_reservation_timeout()
-{
-  if (m_reservations) {
-    m_reservations->handle_no_reply_timeout();
-  }
-}
-
 bool PgScrubber::set_reserving_now() {
   return m_osds->get_scrub_services().set_reserving_now(m_pg_id,
                                                         ceph_clock_now());
@@ -1738,6 +1711,11 @@ void PgScrubber::clear_scrub_blocked()
   m_pg->publish_stats_to_osd();
 }
 
+void PgScrubber::flag_reservations_failure()
+{
+  m_scrub_job->resources_failure = true;
+}
+
 /*
  * note: only called for the Primary.
  */
@@ -2195,8 +2173,7 @@ void PgScrubber::set_scrub_duration()
 void PgScrubber::reserve_replicas()
 {
   dout(10) << __func__ << dendl;
-  m_reservations.emplace(
-    m_pg, m_pg_whoami, m_scrub_job, m_pg->get_cct()->_conf);
+  m_reservations.emplace(*this);
 }
 
 void PgScrubber::cleanup_on_finish()
@@ -2454,121 +2431,71 @@ void PgScrubber::preemption_data_t::reset()
 
 
 // ///////////////////// ReplicaReservations //////////////////////////////////
-namespace Scrub {
 
-void ReplicaReservations::release_replica(pg_shard_t peer, epoch_t epoch)
+#undef dout_prefix
+#define dout_prefix _prefix_fn(_dout, this, __func__)
+template <class T>
+static std::ostream& _prefix_fn(std::ostream* _dout, T* t, std::string fn = "")
 {
-  auto m = new MOSDScrubReserve(spg_t(m_pg_info.pgid.pgid, peer.shard),
-				epoch,
-				MOSDScrubReserve::RELEASE,
-				m_pg->pg_whoami);
-  m_osds->send_message_osd_cluster(peer.osd, m, epoch);
+  return t->gen_prefix(*_dout, fn);
 }
 
-ReplicaReservations::ReplicaReservations(
-  PG* pg,
-  pg_shard_t whoami,
-  Scrub::ScrubJobRef scrubjob,
-  const ConfigProxy& conf)
-    : m_pg{pg}
-    , m_acting_set{pg->get_actingset()}
+namespace Scrub {
+
+ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
+
+    : m_scrubber{scrbr}
+    , m_pg{m_scrubber.get_pg()}
+    , m_pgid{m_scrubber.get_spgid().pgid}
     , m_osds{m_pg->get_pg_osd(ScrubberPasskey())}
-    , m_pending{static_cast<int>(m_acting_set.size()) - 1}
-    , m_pg_info{m_pg->get_pg_info(ScrubberPasskey())}
-    , m_scrub_job{scrubjob}
-    , m_conf{conf}
 {
+  // the acting set is sorted by pg_shard_t. The reservations are to be issued
+  // in this order, so that the OSDs will receive the requests in a consistent
+  // order. This is done to reduce the chance of having two PGs that share some
+  // of their acting-set OSDs, consistently interfering with each other's
+  // reservation process.
+  auto acting = m_pg->get_actingset();
+  m_sorted_secondaries.reserve(acting.size());
+  std::copy_if(
+      acting.cbegin(), acting.cend(), std::back_inserter(m_sorted_secondaries),
+      [whoami = m_pg->pg_whoami](const pg_shard_t& shard) {
+	return shard != whoami;
+      });
+
+  m_next_to_request = m_sorted_secondaries.cbegin();
+  // send out the 1'st request (unless we have no replicas)
+  send_next_reservation_or_complete();
+}
+
+void ReplicaReservations::release_all()
+{
+  std::span<const pg_shard_t> replicas{
+      m_sorted_secondaries.cbegin(), m_next_to_request};
+  dout(10) << fmt::format("releasing {}", replicas) << dendl;
   epoch_t epoch = m_pg->get_osdmap_epoch();
-  m_log_msg_prefix = fmt::format(
-      "osd.{} ep: {} scrubber::ReplicaReservations pg[{}]: ", m_osds->whoami,
-      epoch, pg->pg_id);
 
-  m_timeout = conf.get_val<std::chrono::milliseconds>(
-      "osd_scrub_slow_reservation_response");
-
-  if (m_pending <= 0) {
-    // A special case of no replicas.
-    // just signal the scrub state-machine to continue
-    send_all_done();
-
-  } else {
-    // send the reservation requests
-    for (auto p : m_acting_set) {
-      if (p == whoami)
-	continue;
-      auto m = new MOSDScrubReserve(
-	spg_t(m_pg_info.pgid.pgid, p.shard), epoch, MOSDScrubReserve::REQUEST,
+  // send 'release' messages to all replicas we have managed to reserve
+  for (const auto& peer : replicas) {
+    auto m = make_message<MOSDScrubReserve>(
+	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::RELEASE,
 	m_pg->pg_whoami);
-      m_osds->send_message_osd_cluster(p.osd, m, epoch);
-      m_waited_for_peers.push_back(p);
-      dout(10) << __func__ << ": reserve " << p.osd << dendl;
-    }
+    m_pg->send_cluster_message(peer.osd, m, epoch, false);
   }
-}
-
-void ReplicaReservations::send_all_done()
-{
-  // stop any pending timeout timer
-  m_osds->queue_for_scrub_granted(m_pg, scrub_prio_t::low_priority);
-}
 
-void ReplicaReservations::send_reject()
-{
-  // stop any pending timeout timer
-  m_scrub_job->resources_failure = true;
-  m_osds->queue_for_scrub_denied(m_pg, scrub_prio_t::low_priority);
+  m_sorted_secondaries.clear();
+  m_next_to_request = m_sorted_secondaries.cbegin();
 }
 
-void ReplicaReservations::discard_all()
+void ReplicaReservations::discard_remote_reservations()
 {
-  dout(10) << __func__ << ": " << m_reserved_peers << dendl;
-
-  m_had_rejections = true;  // preventing late-coming responses from triggering
-			    // events
-  m_reserved_peers.clear();
-  m_waited_for_peers.clear();
-}
-
-/*
- * The following holds when update_latecomers() is called:
- * - we are still waiting for replies from some of the replicas;
- * - we might have already set a timer. If so, we should restart it.
- * - we might have received responses from 50% of the replicas.
- */
-std::optional<ReplicaReservations::tpoint_t>
-ReplicaReservations::update_latecomers(tpoint_t now_is)
-{
-  if (m_reserved_peers.size() > m_waited_for_peers.size()) {
-    // at least half of the replicas have already responded. Time we flag
-    // latecomers.
-    return now_is + m_timeout;
-  } else {
-    return std::nullopt;
-  }
+  dout(10) << "reset w/o issuing messages" << dendl;
+  m_sorted_secondaries.clear();
+  m_next_to_request = m_sorted_secondaries.cbegin();
 }
 
 ReplicaReservations::~ReplicaReservations()
 {
-  m_had_rejections = true;  // preventing late-coming responses from triggering
-			    // events
-
-  // send un-reserve messages to all reserved replicas. We do not wait for
-  // answer (there wouldn't be one). Other incoming messages will be discarded
-  // on the way, by our owner.
-  epoch_t epoch = m_pg->get_osdmap_epoch();
-
-  for (auto& p : m_reserved_peers) {
-    release_replica(p, epoch);
-  }
-  m_reserved_peers.clear();
-
-  // note: the release will follow on the heels of the request. When tried
-  // otherwise, grants that followed a reject arrived after the whole scrub
-  // machine-state was reset, causing leaked reservations.
-  for (auto& p : m_waited_for_peers) {
-    release_replica(p, epoch);
-  }
-  m_waited_for_peers.clear();
+  release_all();
 }
 
 /**
@@ -2577,108 +2504,121 @@ ReplicaReservations::~ReplicaReservations()
  */
 void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
 {
-  dout(10) << __func__ << ": granted by " << from << dendl;
-  op->mark_started();
-
-  {
-    // reduce the amount of extra release messages. Not a must, but the log is
-    // cleaner
-    auto w = find(m_waited_for_peers.begin(), m_waited_for_peers.end(), from);
-    if (w != m_waited_for_peers.end())
-      m_waited_for_peers.erase(w);
+  // verify that the grant is from the peer we expected. If not?
+  // for now - abort the OSD. \todo reconsider the reaction.
+  if (!get_last_sent().has_value() || from != *get_last_sent()) {
+    dout(1) << fmt::format(
+		   "unexpected grant from {} (expected {})", from,
+		   get_last_sent().value_or(pg_shard_t{}))
+	    << dendl;
+    ceph_assert(from == get_last_sent());
+    return;
   }
 
-  // are we forced to reject the reservation?
-  if (m_had_rejections) {
+  auto elapsed = clock::now() - m_last_request_sent_at;
 
-    dout(10) << __func__ << ": rejecting late-coming reservation from " << from
-	     << dendl;
-    release_replica(from, m_pg->get_osdmap_epoch());
-
-  } else if (std::find(m_reserved_peers.begin(),
-		       m_reserved_peers.end(),
-		       from) != m_reserved_peers.end()) {
+  // log a warning if the response was slow to arrive
+  auto warn_timeout = m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
+      "osd_scrub_slow_reservation_response");
+  if (!m_slow_response_warned && (elapsed > warn_timeout)) {
+    dout(1) << fmt::format(
+		   "slow reservation response from {} ({}ms)", from,
+		   duration_cast<milliseconds>(elapsed).count())
+	    << dendl;
+    // prevent additional warnings
+    m_slow_response_warned = true;
+  }
+  dout(10) << fmt::format(
+		  "granted by {} ({} of {}) in {}ms", from,
+		  active_requests_cnt(), m_sorted_secondaries.size(),
+		  duration_cast<milliseconds>(elapsed).count())
+	   << dendl;
+  send_next_reservation_or_complete();
+}
 
-    dout(10) << __func__ << ": already had osd." << from << " reserved"
-	     << dendl;
+void ReplicaReservations::send_next_reservation_or_complete()
+{
+  if (m_next_to_request == m_sorted_secondaries.cend()) {
+    // granted by all replicas
+    dout(10) << "remote reservation complete" << dendl;
+    m_osds->queue_for_scrub_granted(m_pg, scrub_prio_t::low_priority);
 
   } else {
-
-    dout(10) << __func__ << ": osd." << from << " scrub reserve = success"
+    // send the next reservation request
+    const auto peer = *m_next_to_request;
+    const auto epoch = m_pg->get_osdmap_epoch();
+    auto m = make_message<MOSDScrubReserve>(
+	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
+	m_pg->pg_whoami);
+    m_pg->send_cluster_message(peer.osd, m, epoch, false);
+    m_last_request_sent_at = clock::now();
+    dout(10) << fmt::format(
+		    "reserving {} (the {} of {} replicas)", *m_next_to_request,
+		    active_requests_cnt()+1, m_sorted_secondaries.size())
 	     << dendl;
-    m_reserved_peers.push_back(from);
-
-    // was this response late?
-    auto now_is = clock::now();
-    if (m_timeout_point && (now_is > *m_timeout_point)) {
-      m_osds->clog->warn() << fmt::format(
-	"osd.{} scrubber pg[{}]: late reservation from osd.{}",
-	m_osds->whoami,
-	m_pg->pg_id,
-	from);
-      m_timeout_point.reset();
-    } else {
-      // possibly set a timer to warn about late-coming reservations
-      m_timeout_point = update_latecomers(now_is);
-    }
-
-    if (--m_pending == 0) {
-      send_all_done();
-    }
+    m_next_to_request++;
   }
 }
 
-void ReplicaReservations::handle_reserve_reject(OpRequestRef op,
-						pg_shard_t from)
+// temporary comment: the part of handle_reserve_reject() that will not
+// be delegated to the FSM in the following commits
+void ReplicaReservations::verify_rejections_source(
+    OpRequestRef op,
+    pg_shard_t from)
 {
-  dout(10) << __func__ << ": rejected by " << from << dendl;
-  dout(15) << __func__ << ": " << *op->get_req() << dendl;
-  op->mark_started();
-
-  {
-    // reduce the amount of extra release messages. Not a must, but the log is
-    // cleaner
-    auto w = find(m_waited_for_peers.begin(), m_waited_for_peers.end(), from);
-    if (w != m_waited_for_peers.end())
-      m_waited_for_peers.erase(w);
-  }
-
-  if (m_had_rejections) {
-
-    // our failure was already handled when the first rejection arrived
-    dout(15) << __func__ << ": ignoring late-coming rejection from " << from
-	     << dendl;
-
-  } else if (std::find(m_reserved_peers.begin(),
-		       m_reserved_peers.end(),
-		       from) != m_reserved_peers.end()) {
+  // a convenient log message for the reservation process conclusion
+  // (matches the one in send_next_reservation_or_complete())
+  dout(10) << fmt::format(
+		  "remote reservation failure. Rejected by {} ({})", from,
+		  *op->get_req())
+	   << dendl;
 
-    dout(10) << __func__ << ": already had osd." << from << " reserved"
-	     << dendl;
+  // verify that the denial is from the peer we expected. If not?
+  // we should treat it as though the *correct* peer has rejected the request,
+  // but remember to release that peer, too.
 
+  ceph_assert(get_last_sent().has_value());
+  const auto expected = *get_last_sent();
+  if (from != expected) {
+    dout(1) << fmt::format(
+		   "unexpected rejection from {} (expected {})", from, expected)
+	    << dendl;
   } else {
-
-    dout(10) << __func__ << ": osd." << from << " scrub reserve = fail"
-	     << dendl;
-    m_had_rejections = true;  // preventing any additional notifications
-    send_reject();
+    // correct peer, wrong answer...
+    m_next_to_request--;  // no need to release this one
   }
 }
 
-void ReplicaReservations::handle_no_reply_timeout()
+// to be delegated to the FSM in the following commits
+void ReplicaReservations::handle_reserve_reject(
+    OpRequestRef op,
+    pg_shard_t from)
+{
+  verify_rejections_source(op, from);
+  release_all();
+  m_scrubber.flag_reservations_failure();
+  m_osds->queue_for_scrub_denied(m_pg, scrub_prio_t::low_priority);
+}
+
+std::optional<pg_shard_t> ReplicaReservations::get_last_sent() const
 {
-  dout(1) << fmt::format(
-	       "{}: timeout! no reply from {}", __func__, m_waited_for_peers)
-	  << dendl;
+  if (m_next_to_request == m_sorted_secondaries.cbegin()) {
+    return std::nullopt;
+  }
+  return *(m_next_to_request - 1);
+}
 
-  // treat reply timeout as if a REJECT was received
-  m_had_rejections = true;  // preventing any additional notifications
-  send_reject();
+size_t ReplicaReservations::active_requests_cnt() const
+{
+  return m_next_to_request - m_sorted_secondaries.cbegin();
 }
 
-std::ostream& ReplicaReservations::gen_prefix(std::ostream& out) const
+std::ostream& ReplicaReservations::gen_prefix(
+    std::ostream& out,
+    std::string fn) const
 {
-  return out << m_log_msg_prefix;
+  return m_pg->gen_prefix(out)
+	 << fmt::format("scrubber::ReplicaReservations:{}: ", fn);
 }
 
 
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 52428599514e..6f56390e627c 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -90,90 +90,119 @@ struct BuildMap;
 /**
  * Reserving/freeing scrub resources at the replicas.
  *
- * When constructed - sends reservation requests to the acting_set.
+ * When constructed - sends reservation requests to the acting_set OSDs, one
+ * by one.
+ * Once a replica's OSD replies with a 'grant'ed reservation, we send a
+ * reservation request to the next replica.
  * A rejection triggers a "couldn't acquire the replicas' scrub resources"
- * event. All previous requests, whether already granted or not, are explicitly
- * released.
+ * event. All granted reservations are released.
+ *
+ * Reserved replicas should be released at the end of the scrub session. The
+ * one exception is if the scrub terminates upon an interval change. In that
+ * scenario - the replicas discard their reservations on their own accord
+ * when noticing the change in interval, and there is no need (and no
+ * guaranteed way) to send them the release message.
  *
  * Timeouts:
  *
  *  Slow-Secondary Warning:
- *  Once at least half of the replicas have accepted the reservation, we start
- *  reporting any secondary that takes too long (more than <conf> milliseconds
- *  after the previous response received) to respond to the reservation request.
- *  (Why? because we have encountered real-life situations where a specific OSD
- *  was systematically very slow (e.g. 5 seconds) to respond to the reservation
- *  requests, slowing the scrub process to a crawl).
+ *  Warn if a replica takes more than <conf> milliseconds to reply to a
+ *  reservation request. Only one warning is issued per session.
  *
  *  Reservation Timeout:
  *  We limit the total time we wait for the replicas to respond to the
- *  reservation request. If we don't get all the responses (either Grant or
- *  Reject) within <conf> milliseconds, we give up and release all the
- *  reservations we have acquired so far.
+ *  reservation request. If the reservation back-and-forth does not complete
+ *  within <conf> milliseconds, we give up and release all the reservations
+ *  that have been acquired until that moment.
  *  (Why? because we have encountered instances where a reservation request was
  *  lost - either due to a bug or due to a network issue.)
- *
- * A note re performance: I've measured a few container alternatives for
- * m_reserved_peers, with its specific usage pattern. Std::set is extremely
- * slow, as expected. flat_set is only slightly better. Surprisingly -
- * std::vector (with no sorting) is better than boost::small_vec. And for
- * std::vector: no need to pre-reserve.
  */
 class ReplicaReservations {
-  using clock = std::chrono::system_clock;
-  using tpoint_t = std::chrono::time_point<clock>;
+  using clock = ceph::coarse_real_clock;
 
+  ScrubMachineListener& m_scrubber;
   PG* m_pg;
-  std::set<pg_shard_t> m_acting_set;
-  OSDService* m_osds;
-  std::vector<pg_shard_t> m_waited_for_peers;
-  std::vector<pg_shard_t> m_reserved_peers;
-  bool m_had_rejections{false};
-  int m_pending{-1};
-  const pg_info_t& m_pg_info;
-  Scrub::ScrubJobRef m_scrub_job;	///< a ref to this PG's scrub job
-  const ConfigProxy& m_conf;
 
-  // detecting slow peers (see 'slow-secondary' above)
-  std::chrono::milliseconds m_timeout;
-  std::optional<tpoint_t> m_timeout_point;
+  /// shorthand for m_scrubber.get_spgid().pgid
+  const pg_t m_pgid;
+
+  /// for dout && when queueing messages to the FSM
+  OSDService* m_osds;
 
-  void release_replica(pg_shard_t peer, epoch_t epoch);
+  /// the acting set (not including myself), sorted by pg_shard_t
+  std::vector<pg_shard_t> m_sorted_secondaries;
 
-  void send_all_done();	 ///< all reservations are granted
+  /// the next replica to which we will send a reservation request
+  std::vector<pg_shard_t>::const_iterator m_next_to_request;
 
-  /// notify the scrubber that we have failed to reserve replicas' resources
-  void send_reject();
+  /// for logs, and for detecting slow peers
+  clock::time_point m_last_request_sent_at;
 
-  std::optional<tpoint_t> update_latecomers(tpoint_t now_is);
+  /// used to prevent multiple "slow response" warnings
+  bool m_slow_response_warned{false};
 
  public:
-  std::string m_log_msg_prefix;
+  ReplicaReservations(ScrubMachineListener& scrubber);
+
+  ~ReplicaReservations();
 
   /**
-   *  quietly discard all knowledge about existing reservations. No messages
-   *  are sent to peers.
-   *  To be used upon interval change, as we know the the running scrub is no
-   *  longer relevant, and that the replicas had reset the reservations on
-   *  their side.
+   * The OK received from the replica (after verifying that it is indeed
+   * the replica we are expecting a reply from) is noted, and triggers
+   * one of two: either sending a reservation request to the next replica,
+   * or notifying the scrubber that we have reserved them all.
    */
-  void discard_all();
+  void handle_reserve_grant(OpRequestRef op, pg_shard_t from);
 
-  ReplicaReservations(PG* pg,
-                      pg_shard_t whoami,
-                      Scrub::ScrubJobRef scrubjob,
-                      const ConfigProxy& conf); 
+  /**
+   * Verify that the sender of the received rejection is the replica we
+   * were expecting a reply from.
+   * If this is so - just mark the fact that the specific peer need not
+   * be released.
+   *
+   * Note - the actual handling of scrub session termination and of
+   * releasing the reserved replicas is done by the caller (the FSM).
+   */
+  void verify_rejections_source(OpRequestRef op, pg_shard_t from);
 
-  ~ReplicaReservations();
+  void handle_reserve_reject(OpRequestRef op, pg_shard_t from);
 
-  void handle_reserve_grant(OpRequestRef op, pg_shard_t from);
+  /**
+   * Notifies implementation that it is no longer responsible for releasing
+   * tracked remote reservations.
+   *
+   * The intended usage is upon interval change.  In general, replicas are
+   * responsible for releasing their own resources upon interval change without
+   * coordination from the primary.
+   *
+   * Sends no messages.
+   */
+  void discard_remote_reservations();
 
-  void handle_reserve_reject(OpRequestRef op, pg_shard_t from);
+  // note: 'public', as accessed via the 'standard' dout_prefix() macro
+  std::ostream& gen_prefix(std::ostream& out, std::string fn) const;
+
+ private:
+  /// send 'release' messages to all replicas we have managed to reserve
+  void release_all();
+
+  /// send a reservation request to a replica's OSD
+  void send_request_to_replica(pg_shard_t peer, epoch_t epoch);
+
+  /// let the scrubber know that we have reserved all the replicas
+  void send_all_done();
+
+  /// the only replica we are expecting a reply from
+  std::optional<pg_shard_t> get_last_sent() const;
 
-  // if timing out on receiving replies from our replicas:
-  void handle_no_reply_timeout();
+  /// The number of requests that have been sent (and not rejected) so far.
+  size_t active_requests_cnt() const;
 
-  std::ostream& gen_prefix(std::ostream& out) const;
+  /**
+   * Either send a reservation request to the next replica, or notify the
+   * scrubber that we have reserved all the replicas.
+   */
+  void send_next_reservation_or_complete();
 };
 
 /**
@@ -348,9 +377,6 @@ class PgScrubber : public ScrubPgIF,
   void handle_scrub_reserve_release(OpRequestRef op) final;
   void discard_replica_reservations() final;
   void clear_scrub_reservations() final;  // PG::clear... fwds to here
-  void unreserve_replicas() final;
-  void on_replica_reservation_timeout() final;
-
 
   // managing scrub op registration
 
@@ -453,10 +479,14 @@ class PgScrubber : public ScrubPgIF,
   // the I/F used by the state-machine (i.e. the implementation of
   // ScrubMachineListener)
 
-  CephContext* get_cct() const final { return m_pg->cct; }
   LogChannelRef &get_clog() const final;
   int get_whoami() const final;
   spg_t get_spgid() const final { return m_pg->get_pgid(); }
+  PG* get_pg() const final { return m_pg; }
+
+  // temporary interface (to be discarded in a follow-up PR)
+  /// set the 'resources_failure' flag in the scrub-job object
+  void flag_reservations_failure();
 
   scrubber_callback_cancel_token_t schedule_callback_after(
     ceph::timespan duration, scrubber_callback_t &&cb);
@@ -871,14 +901,6 @@ class PgScrubber : public ScrubPgIF,
 
   std::list<Context*> m_callbacks;
 
-  /**
-   * send a replica (un)reservation request to the acting set
-   *
-   * @param opcode - one of MOSDScrubReserve::REQUEST
-   *                  or MOSDScrubReserve::RELEASE
-   */
-  void message_all_replicas(int32_t opcode, std::string_view op_text);
-
   hobject_t m_max_end;	///< Largest end that may have been sent to replicas
   ScrubMapBuilder m_primary_scrubmap_pos;
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index efb091788206..33924d134298 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -140,6 +140,16 @@ Session::~Session()
   scrbr->clear_pgscrub_state();
 }
 
+sc::result Session::react(const IntervalChanged&)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "Session::react(const IntervalChanged&)" << dendl;
+
+  /// \todo (future commit): the reservations will be local to this state
+  scrbr->discard_replica_reservations();
+  return transit<NotActive>();
+}
+
 
 // ----------------------- ReservingReplicas ---------------------------------
 
@@ -152,7 +162,7 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
 
   scrbr->reserve_replicas();
 
-  auto timeout = scrbr->get_cct()->_conf.get_val<
+  auto timeout = scrbr->get_pg_cct()->_conf.get_val<
     std::chrono::milliseconds>("osd_scrub_reservation_timeout");
   if (timeout.count() > 0) {
     // Start a timer to handle case where the replicas take a long time to
@@ -180,8 +190,11 @@ sc::result ReservingReplicas::react(const ReservationTimeout&)
       scrbr->get_spgid(), entered_at);
   dout(5) << msg << dendl;
   scrbr->get_clog()->warn() << "osd." << scrbr->get_whoami() << " " << msg;
-  scrbr->on_replica_reservation_timeout();
-  return discard_event();
+
+  // cause the scrubber to stop the scrub session, marking 'reservation
+  // failure' as the cause (affecting future scheduling)
+  scrbr->flag_reservations_failure();
+  return transit<NotActive>();
 }
 
 sc::result ReservingReplicas::react(const ReservationFailure&)
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 071a464ce130..f5a3bb677ba9 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -127,6 +127,11 @@ MEV(SchedReplica)
 /// that is in-flight to the local ObjectStore
 MEV(ReplicaPushesUpd)
 
+/// a new interval has dawned.
+/// For a Primary: Discards replica reservations, so that the FullReset that would
+/// follow it would not attempt to release them.
+MEV(IntervalChanged)
+
 /// guarantee that the FSM is in the quiescent state (i.e. NotActive)
 MEV(FullReset)
 
@@ -202,7 +207,7 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
    * from being delivered.  The intended usage is to invoke
    * schedule_timer_event_after in the constructor of the state machine state
    * intended to handle the event and assign the returned timer_event_token_t
-   * to a member of that state. That way, exiting the state will implicitely
+   * to a member of that state. That way, exiting the state will implicitly
    * cancel the event.  See RangedBlocked::m_timeout_token and
    * RangeBlockedAlarm for an example usage.
    */
@@ -338,12 +343,15 @@ struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
  *  reservations are released. This is because we know that the replicas are
  *  also resetting their reservations.
  */
-struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>, NamedSimply {
+struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>,
+                 NamedSimply {
   explicit Session(my_context ctx);
   ~Session();
 
-  using reactions = mpl::list<sc::transition<FullReset, NotActive>>;
-  /// \todo handle interval change
+  using reactions = mpl::list<sc::transition<FullReset, NotActive>,
+                              sc::custom_reaction<IntervalChanged>>;
+
+  sc::result react(const IntervalChanged&);
 };
 
 struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index cfef666e1b11..e9db62e2a2d8 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -48,10 +48,11 @@ struct preemption_t {
 }  // namespace Scrub
 
 struct ScrubMachineListener {
-  virtual CephContext *get_cct() const = 0;
+  virtual CephContext *get_pg_cct() const = 0;
   virtual LogChannelRef &get_clog() const = 0;
   virtual int get_whoami() const = 0;
   virtual spg_t get_spgid() const = 0;
+  virtual PG* get_pg() const = 0;
 
   using scrubber_callback_t = std::function<void(void)>;
   using scrubber_callback_cancel_token_t = Context*;
@@ -72,9 +73,9 @@ struct ScrubMachineListener {
   /**
    * cancel_callback
    *
-   * Attempts to cancel the callback to whcih the passed token is associated.
+   * Attempts to cancel the callback to which the passed token is associated.
    * cancel_callback is best effort, the callback may still fire.
-   * cancel_callback guarrantees that exactly one of the two things will happen:
+   * cancel_callback guarantees that exactly one of the two things will happen:
    * - the callback is destroyed and will not be invoked
    * - the callback will be invoked
    */
@@ -177,9 +178,8 @@ struct ScrubMachineListener {
    */
   virtual void reserve_replicas() = 0;
 
-  virtual void unreserve_replicas() = 0;
-
-  virtual void on_replica_reservation_timeout() = 0;
+  /// discard replica reservations without sending a message (on interval)
+  virtual void discard_replica_reservations() = 0;
 
   virtual void set_scrub_begin_time() = 0;
 
@@ -238,4 +238,8 @@ struct ScrubMachineListener {
 
   /// sending cluster-log warnings
   virtual void log_cluster_warning(const std::string& msg) const = 0;
+
+  // temporary interface (to be discarded in a follow-up PR)
+  /// set the 'resources_failure' flag in the scrub-job object
+  virtual void flag_reservations_failure() = 0;
 };
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index d5d4a8c278cf..929bec06e163 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -382,11 +382,6 @@ struct ScrubPgIF {
 
   // --------------- reservations -----------------------------------
 
-  /**
-   *  message all replicas with a request to "unreserve" scrub
-   */
-  virtual void unreserve_replicas() = 0;
-
   /**
    *  "forget" all replica reservations. No messages are sent to the
    *  previously-reserved.

From b427ab678caae5eab6d528638484148c1a6b932e Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Fri, 13 Oct 2023 07:48:44 -0500
Subject: [PATCH 0182/2492] osd/scrub: route grant/deny messages through the
 scrubber FSM

The scrubber FSM will now be responsible for handling the grant/deny
ops received from the replica OSDs.
For this temporary step - the scrubber FSM will simply forward a
call to the ReplicaReservations object in the Scrubber.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PrimaryLogPG.cc                | 24 ++-----
 src/osd/scrubber/pg_scrubber.cc        | 87 +++++++++++++++-----------
 src/osd/scrubber/pg_scrubber.h         | 24 ++++---
 src/osd/scrubber/scrub_machine.cc      | 22 +++++++
 src/osd/scrubber/scrub_machine.h       | 47 ++++++++++++++
 src/osd/scrubber/scrub_machine_lstnr.h |  5 ++
 src/osd/scrubber_common.h              | 15 +++--
 7 files changed, 148 insertions(+), 76 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index c28184f9c9b6..38757e7e7dfd 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1936,27 +1936,11 @@ void PrimaryLogPG::do_request(
     break;
 
   case MSG_OSD_SCRUB_RESERVE:
-    {
-      if (!m_scrubber) {
-        osd->reply_op_error(op, -EAGAIN);
-        return;
-      }
-      auto m = op->get_req<MOSDScrubReserve>();
-      switch (m->type) {
-      case MOSDScrubReserve::REQUEST:
-	m_scrubber->handle_scrub_reserve_request(op);
-	break;
-      case MOSDScrubReserve::GRANT:
-	m_scrubber->handle_scrub_reserve_grant(op, m->from);
-	break;
-      case MOSDScrubReserve::REJECT:
-	m_scrubber->handle_scrub_reserve_reject(op, m->from);
-	break;
-      case MOSDScrubReserve::RELEASE:
-	m_scrubber->handle_scrub_reserve_release(op);
-	break;
-      }
+    if (!m_scrubber) {
+      osd->reply_op_error(op, -EAGAIN);
+      return;
     }
+    m_scrubber->handle_scrub_reserve_msgs(op);
     break;
 
   case MSG_OSD_REP_SCRUB:
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 294d050268d0..8ee964c964b4 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -1553,10 +1553,39 @@ void PgScrubber::map_from_replica(OpRequestRef op)
   }
 }
 
-void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
+/**
+ * route incoming replica-reservations requests/responses to the
+ * appropriate handler.
+ * As the ReplicaReservations object is to be owned by the ScrubMachine, we
+ * send the relevant messages to the ScrubMachine.
+ */
+void PgScrubber::handle_scrub_reserve_msgs(OpRequestRef op)
 {
-  dout(10) << __func__ << " " << *op->get_req() << dendl;
+  dout(10) << fmt::format("{}: {}", __func__, *op->get_req()) << dendl;
   op->mark_started();
+  if (should_drop_message(op)) {
+    return;
+  }
+  auto m = op->get_req<MOSDScrubReserve>();
+  switch (m->type) {
+    case MOSDScrubReserve::REQUEST:
+      handle_scrub_reserve_request(op);
+      break;
+    case MOSDScrubReserve::GRANT:
+      m_fsm->process_event(ReplicaGrant{op, m->from});
+      break;
+    case MOSDScrubReserve::REJECT:
+      m_fsm->process_event(ReplicaReject{op, m->from});
+      break;
+    case MOSDScrubReserve::RELEASE:
+      handle_scrub_reserve_release(op);
+      break;
+  }
+}
+
+
+void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
+{
   auto request_ep = op->sent_epoch;
   dout(20) << fmt::format("{}: request_ep:{} recovery:{}",
 			  __func__,
@@ -1564,13 +1593,9 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
 			  m_osds->is_recovery_active())
 	   << dendl;
 
-  if (should_drop_message(op)) {
-    return;
-  }
-
-  /* The primary may unilaterally restart the scrub process without notifying
-   * replicas.  Unconditionally clear any existing state prior to handling
-   * the new reservation. */
+  // The primary may unilaterally restart the scrub process without notifying
+  // replicas. Unconditionally clear any existing state prior to handling
+  // the new reservation.
   m_fsm->process_event(FullReset{});
 
   bool granted{false};
@@ -1598,50 +1623,36 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
   m_osds->send_message_osd_cluster(reply, op->get_req()->get_connection());
 }
 
-void PgScrubber::handle_scrub_reserve_grant(OpRequestRef op, pg_shard_t from)
-{
-  dout(10) << __func__ << " " << *op->get_req() << dendl;
-  op->mark_started();
 
-  if (should_drop_message(op)) {
-    return;
-  }
-  if (m_reservations.has_value()) {
-    m_reservations->handle_reserve_grant(op, from);
-  } else {
-    dout(20) << __func__ << ": late/unsolicited reservation grant from osd "
-	 << from << " (" << op << ")" << dendl;
-  }
+/// temporary
+void PgScrubber::grant_from_replica(OpRequestRef op, pg_shard_t from)
+{
+  dout(10) << fmt::format("{}: {}", __func__, *op->get_req()) << dendl;
+  ceph_assert(m_reservations.has_value()); // the FSM should know
+  m_reservations->handle_reserve_grant(op, from);
 }
 
-void PgScrubber::handle_scrub_reserve_reject(OpRequestRef op, pg_shard_t from)
-{
-  dout(10) << __func__ << " " << *op->get_req() << dendl;
-  op->mark_started();
 
-  if (should_drop_message(op)) {
-    return;
-  }
-  if (m_reservations.has_value()) {
-    // there is an active reservation process. No action is required otherwise.
-    m_reservations->handle_reserve_reject(op, from);
-  }
+/// temporary
+void PgScrubber::reject_from_replica(OpRequestRef op, pg_shard_t from)
+{
+  dout(10) << fmt::format("{}: {}", __func__, *op->get_req()) << dendl;
+  ceph_assert(m_reservations.has_value()); // the FSM should know
+  m_reservations->handle_reserve_reject(op, from);
 }
 
+
 void PgScrubber::handle_scrub_reserve_release(OpRequestRef op)
 {
   dout(10) << __func__ << " " << *op->get_req() << dendl;
-  op->mark_started();
   if (should_drop_message(op)) {
     // we might have turned into a Primary in the meantime. The interval
     // change should have been noticed already, and caused us to reset.
     return;
   }
 
-  /*
-   * this specific scrub session has terminated. All incoming events carrying
-   *  the old tag will be discarded.
-   */
+  // this specific scrub session has terminated. All incoming events carrying
+  // the old tag will be discarded.
   m_fsm->process_event(FullReset{});
 }
 
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 6f56390e627c..1370fc953c0c 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -186,12 +186,6 @@ class ReplicaReservations {
   /// send 'release' messages to all replicas we have managed to reserve
   void release_all();
 
-  /// send a reservation request to a replica's OSD
-  void send_request_to_replica(pg_shard_t peer, epoch_t epoch);
-
-  /// let the scrubber know that we have reserved all the replicas
-  void send_all_done();
-
   /// the only replica we are expecting a reply from
   std::optional<pg_shard_t> get_last_sent() const;
 
@@ -366,15 +360,21 @@ class PgScrubber : public ScrubPgIF,
   bool range_intersects_scrub(const hobject_t& start,
 			      const hobject_t& end) final;
 
+  /**
+   * route incoming replica-reservations requests/responses to the
+   * appropriate handler.
+   * As the ReplicaReservations object is to be owned by the ScrubMachine, we
+   * send all relevant messages to the ScrubMachine.
+   */
+  void handle_scrub_reserve_msgs(OpRequestRef op) final;
+
   /**
    *  we are a replica being asked by the Primary to reserve OSD resources for
    *  scrubbing
    */
-  void handle_scrub_reserve_request(OpRequestRef op) final;
+  void handle_scrub_reserve_request(OpRequestRef op);
 
-  void handle_scrub_reserve_grant(OpRequestRef op, pg_shard_t from) final;
-  void handle_scrub_reserve_reject(OpRequestRef op, pg_shard_t from) final;
-  void handle_scrub_reserve_release(OpRequestRef op) final;
+  void handle_scrub_reserve_release(OpRequestRef op);
   void discard_replica_reservations() final;
   void clear_scrub_reservations() final;  // PG::clear... fwds to here
 
@@ -611,6 +611,10 @@ class PgScrubber : public ScrubPgIF,
 
   void log_cluster_warning(const std::string& warning) const final;
 
+  // temporary interface to handle forwarded reservation messages:
+  void grant_from_replica(OpRequestRef op, pg_shard_t from) final;
+  void reject_from_replica(OpRequestRef op, pg_shard_t from) final;
+
  protected:
   bool state_test(uint64_t m) const { return m_pg->state_test(m); }
   void state_set(uint64_t m) { m_pg->state_set(m); }
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 33924d134298..f6eb471530a2 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -180,6 +180,28 @@ ReservingReplicas::~ReservingReplicas()
   scrbr->clear_reserving_now();
 }
 
+sc::result ReservingReplicas::react(const ReplicaGrant& ev)
+{
+  // for now - route the message back to the scrubber, as it holds the
+  // ReplicaReservations object
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReservingReplicas::react(const ReplicaGrant&)" << dendl;
+
+  scrbr->grant_from_replica(ev.m_op, ev.m_from);
+  return discard_event();
+}
+
+sc::result ReservingReplicas::react(const ReplicaReject& ev)
+{
+  // for now - route the message back to the scrubber, as it holds the
+  // ReplicaReservations object
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReservingReplicas::react(const ReplicaReject&)" << dendl;
+
+  scrbr->reject_from_replica(ev.m_op, ev.m_from);
+  return discard_event();
+}
+
 sc::result ReservingReplicas::react(const ReservationTimeout&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index f5a3bb677ba9..806376e3032d 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -15,6 +15,11 @@
 #include <boost/statechart/transition.hpp>
 
 #include "common/version.h"
+#include "messages/MOSDOp.h"
+#include "messages/MOSDRepScrub.h"
+#include "messages/MOSDRepScrubMap.h"
+#include "messages/MOSDScrubReserve.h"
+
 #include "include/Context.h"
 #include "osd/scrubber_common.h"
 
@@ -42,6 +47,40 @@ namespace mpl = ::boost::mpl;
 void on_event_creation(std::string_view nm);
 void on_event_discard(std::string_view nm);
 
+// reservation grant/reject events carry the peer's response:
+
+/// a replica has granted our reservation request
+struct ReplicaGrant : sc::event<ReplicaGrant> {
+  OpRequestRef m_op;
+  pg_shard_t m_from;
+  ReplicaGrant(OpRequestRef op, pg_shard_t from) : m_op{op}, m_from{from}
+  {
+    on_event_creation("ReplicaGrant");
+  }
+  void print(std::ostream* out) const
+  {
+    *out << fmt::format("ReplicaGrant(from: {})", m_from);
+  }
+  std::string_view print() const { return "ReplicaGrant"; }
+  ~ReplicaGrant() { on_event_discard("ReplicaGrant"); }
+};
+
+/// a replica has denied our reservation request
+struct ReplicaReject : sc::event<ReplicaReject> {
+  OpRequestRef m_op;
+  pg_shard_t m_from;
+  ReplicaReject(OpRequestRef op, pg_shard_t from) : m_op{op}, m_from{from}
+  {
+    on_event_creation("ReplicaReject");
+  }
+  void print(std::ostream* out) const
+  {
+    *out << fmt::format("ReplicaReject(from: {})", m_from);
+  }
+  std::string_view print() const { return "ReplicaReject"; }
+  ~ReplicaReject() { on_event_discard("ReplicaReject"); }
+};
+
 #define MEV(E)                                          \
   struct E : sc::event<E> {                             \
     inline static int actv{0};                          \
@@ -360,6 +399,8 @@ struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
   ~ReservingReplicas();
   using reactions = mpl::list<
 			      // all replicas granted our resources request
+			      sc::custom_reaction<ReplicaGrant>,
+			      sc::custom_reaction<ReplicaReject>,
 			      sc::transition<RemotesReserved, ActiveScrubbing>,
 			      sc::custom_reaction<ReservationTimeout>,
 			      sc::custom_reaction<ReservationFailure>>;
@@ -368,6 +409,12 @@ struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
     ceph::coarse_real_clock::now();
   ScrubMachine::timer_event_token_t m_timeout_token;
 
+  /// a "raw" event carrying a peer's grant response
+  sc::result react(const ReplicaGrant&);
+
+  /// a "raw" event carrying a peer's denial response
+  sc::result react(const ReplicaReject&);
+
   sc::result react(const ReservationTimeout&);
 
   /// at least one replica denied us the scrub resources we've requested
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index e9db62e2a2d8..e760ddf0055d 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -239,6 +239,11 @@ struct ScrubMachineListener {
   /// sending cluster-log warnings
   virtual void log_cluster_warning(const std::string& msg) const = 0;
 
+  // temporary interface (to be discarded in a follow-up commit)
+  // to handle replica reservation messages routed thru the FSM
+  virtual void grant_from_replica(OpRequestRef op, pg_shard_t from) = 0;
+  virtual void reject_from_replica(OpRequestRef op, pg_shard_t from) = 0;
+
   // temporary interface (to be discarded in a follow-up PR)
   /// set the 'resources_failure' flag in the scrub-job object
   virtual void flag_reservations_failure() = 0;
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 929bec06e163..24ef8206980a 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -418,14 +418,13 @@ struct ScrubPgIF {
    */
   virtual void update_scrub_job(const requested_scrub_t& request_flags) = 0;
 
-  // on the replica:
-  virtual void handle_scrub_reserve_request(OpRequestRef op) = 0;
-  virtual void handle_scrub_reserve_release(OpRequestRef op) = 0;
-
-  // and on the primary:
-  virtual void handle_scrub_reserve_grant(OpRequestRef op, pg_shard_t from) = 0;
-  virtual void handle_scrub_reserve_reject(OpRequestRef op,
-					   pg_shard_t from) = 0;
+  /**
+   * route incoming replica-reservations requests/responses to the
+   * appropriate handler.
+   * As the ReplicaReservations object is to be owned by the ScrubMachine, we
+   * send all relevant messages to the ScrubMachine.
+   */
+  virtual void handle_scrub_reserve_msgs(OpRequestRef op) = 0;
 
   virtual void rm_from_osd_scrubbing() = 0;
 

From 3bb2e14eb8ab5426199f36b93f54c85d40ced70e Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Fri, 13 Oct 2023 11:07:56 -0500
Subject: [PATCH 0183/2492] osd/scrub: extract ReplicaReservations into
 separate files

As a preliminary step before ReplicaReservations ownership
is moved to the scrubber's FSM.

No code changes in this commit (apart from required 'include's).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/CMakeLists.txt                 |   1 +
 src/osd/scrubber/pg_scrubber.cc        | 192 +----------------------
 src/osd/scrubber/pg_scrubber.h         | 112 +-------------
 src/osd/scrubber/scrub_reservations.cc | 206 +++++++++++++++++++++++++
 src/osd/scrubber/scrub_reservations.h  | 130 ++++++++++++++++
 5 files changed, 339 insertions(+), 302 deletions(-)
 create mode 100644 src/osd/scrubber/scrub_reservations.cc
 create mode 100644 src/osd/scrubber/scrub_reservations.h

diff --git a/src/osd/CMakeLists.txt b/src/osd/CMakeLists.txt
index 7d19424b404a..aad0fdb15cd1 100644
--- a/src/osd/CMakeLists.txt
+++ b/src/osd/CMakeLists.txt
@@ -27,6 +27,7 @@ set(osd_srcs
   scrubber/PrimaryLogScrub.cc
   scrubber/scrub_job.cc
   scrubber/scrub_machine.cc
+  scrubber/scrub_reservations.cc
   scrubber/scrub_resources.cc
   scrubber/ScrubStore.cc
   scrubber/scrub_backend.cc
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 8ee964c964b4..609eb0a72e6a 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -2441,200 +2441,10 @@ void PgScrubber::preemption_data_t::reset()
 }
 
 
-// ///////////////////// ReplicaReservations //////////////////////////////////
-
-#undef dout_prefix
-#define dout_prefix _prefix_fn(_dout, this, __func__)
-template <class T>
-static std::ostream& _prefix_fn(std::ostream* _dout, T* t, std::string fn = "")
-{
-  return t->gen_prefix(*_dout, fn);
-}
+// ///////////////////// LocalReservation //////////////////////////////////
 
 namespace Scrub {
 
-ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
-
-    : m_scrubber{scrbr}
-    , m_pg{m_scrubber.get_pg()}
-    , m_pgid{m_scrubber.get_spgid().pgid}
-    , m_osds{m_pg->get_pg_osd(ScrubberPasskey())}
-{
-  // the acting set is sorted by pg_shard_t. The reservations are to be issued
-  // in this order, so that the OSDs will receive the requests in a consistent
-  // order. This is done to reduce the chance of having two PGs that share some
-  // of their acting-set OSDs, consistently interfering with each other's
-  // reservation process.
-  auto acting = m_pg->get_actingset();
-  m_sorted_secondaries.reserve(acting.size());
-  std::copy_if(
-      acting.cbegin(), acting.cend(), std::back_inserter(m_sorted_secondaries),
-      [whoami = m_pg->pg_whoami](const pg_shard_t& shard) {
-	return shard != whoami;
-      });
-
-  m_next_to_request = m_sorted_secondaries.cbegin();
-  // send out the 1'st request (unless we have no replicas)
-  send_next_reservation_or_complete();
-}
-
-void ReplicaReservations::release_all()
-{
-  std::span<const pg_shard_t> replicas{
-      m_sorted_secondaries.cbegin(), m_next_to_request};
-  dout(10) << fmt::format("releasing {}", replicas) << dendl;
-  epoch_t epoch = m_pg->get_osdmap_epoch();
-
-  // send 'release' messages to all replicas we have managed to reserve
-  for (const auto& peer : replicas) {
-    auto m = make_message<MOSDScrubReserve>(
-	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::RELEASE,
-	m_pg->pg_whoami);
-    m_pg->send_cluster_message(peer.osd, m, epoch, false);
-  }
-
-  m_sorted_secondaries.clear();
-  m_next_to_request = m_sorted_secondaries.cbegin();
-}
-
-void ReplicaReservations::discard_remote_reservations()
-{
-  dout(10) << "reset w/o issuing messages" << dendl;
-  m_sorted_secondaries.clear();
-  m_next_to_request = m_sorted_secondaries.cbegin();
-}
-
-ReplicaReservations::~ReplicaReservations()
-{
-  release_all();
-}
-
-/**
- *  @ATTN we would not reach here if the ReplicaReservation object managed by
- * the scrubber was reset.
- */
-void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
-{
-  // verify that the grant is from the peer we expected. If not?
-  // for now - abort the OSD. \todo reconsider the reaction.
-  if (!get_last_sent().has_value() || from != *get_last_sent()) {
-    dout(1) << fmt::format(
-		   "unexpected grant from {} (expected {})", from,
-		   get_last_sent().value_or(pg_shard_t{}))
-	    << dendl;
-    ceph_assert(from == get_last_sent());
-    return;
-  }
-
-  auto elapsed = clock::now() - m_last_request_sent_at;
-
-  // log a warning if the response was slow to arrive
-  auto warn_timeout = m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
-      "osd_scrub_slow_reservation_response");
-  if (!m_slow_response_warned && (elapsed > warn_timeout)) {
-    dout(1) << fmt::format(
-		   "slow reservation response from {} ({}ms)", from,
-		   duration_cast<milliseconds>(elapsed).count())
-	    << dendl;
-    // prevent additional warnings
-    m_slow_response_warned = true;
-  }
-  dout(10) << fmt::format(
-		  "granted by {} ({} of {}) in {}ms", from,
-		  active_requests_cnt(), m_sorted_secondaries.size(),
-		  duration_cast<milliseconds>(elapsed).count())
-	   << dendl;
-  send_next_reservation_or_complete();
-}
-
-void ReplicaReservations::send_next_reservation_or_complete()
-{
-  if (m_next_to_request == m_sorted_secondaries.cend()) {
-    // granted by all replicas
-    dout(10) << "remote reservation complete" << dendl;
-    m_osds->queue_for_scrub_granted(m_pg, scrub_prio_t::low_priority);
-
-  } else {
-    // send the next reservation request
-    const auto peer = *m_next_to_request;
-    const auto epoch = m_pg->get_osdmap_epoch();
-    auto m = make_message<MOSDScrubReserve>(
-	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
-	m_pg->pg_whoami);
-    m_pg->send_cluster_message(peer.osd, m, epoch, false);
-    m_last_request_sent_at = clock::now();
-    dout(10) << fmt::format(
-		    "reserving {} (the {} of {} replicas)", *m_next_to_request,
-		    active_requests_cnt()+1, m_sorted_secondaries.size())
-	     << dendl;
-    m_next_to_request++;
-  }
-}
-
-// temporary comment: the part of handle_reserve_reject() that will not
-// be delegated to the FSM in the following commits
-void ReplicaReservations::verify_rejections_source(
-    OpRequestRef op,
-    pg_shard_t from)
-{
-  // a convenient log message for the reservation process conclusion
-  // (matches the one in send_next_reservation_or_complete())
-  dout(10) << fmt::format(
-		  "remote reservation failure. Rejected by {} ({})", from,
-		  *op->get_req())
-	   << dendl;
-
-  // verify that the denial is from the peer we expected. If not?
-  // we should treat it as though the *correct* peer has rejected the request,
-  // but remember to release that peer, too.
-
-  ceph_assert(get_last_sent().has_value());
-  const auto expected = *get_last_sent();
-  if (from != expected) {
-    dout(1) << fmt::format(
-		   "unexpected rejection from {} (expected {})", from, expected)
-	    << dendl;
-  } else {
-    // correct peer, wrong answer...
-    m_next_to_request--;  // no need to release this one
-  }
-}
-
-// to be delegated to the FSM in the following commits
-void ReplicaReservations::handle_reserve_reject(
-    OpRequestRef op,
-    pg_shard_t from)
-{
-  verify_rejections_source(op, from);
-  release_all();
-  m_scrubber.flag_reservations_failure();
-  m_osds->queue_for_scrub_denied(m_pg, scrub_prio_t::low_priority);
-}
-
-std::optional<pg_shard_t> ReplicaReservations::get_last_sent() const
-{
-  if (m_next_to_request == m_sorted_secondaries.cbegin()) {
-    return std::nullopt;
-  }
-  return *(m_next_to_request - 1);
-}
-
-size_t ReplicaReservations::active_requests_cnt() const
-{
-  return m_next_to_request - m_sorted_secondaries.cbegin();
-}
-
-std::ostream& ReplicaReservations::gen_prefix(
-    std::ostream& out,
-    std::string fn) const
-{
-  return m_pg->gen_prefix(out)
-	 << fmt::format("scrubber::ReplicaReservations:{}: ", fn);
-}
-
-
-// ///////////////////// LocalReservation //////////////////////////////////
-
 // note: no dout()s in LocalReservation functions. Client logs interactions.
 LocalReservation::LocalReservation(OSDService* osds) : m_osds{osds}
 {
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 1370fc953c0c..fc7b735d219e 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -82,122 +82,12 @@ Main Scrubber interfaces:
 #include "osd_scrub_sched.h"
 #include "scrub_backend.h"
 #include "scrub_machine_lstnr.h"
+#include "scrub_reservations.h"
 
 namespace Scrub {
 class ScrubMachine;
 struct BuildMap;
 
-/**
- * Reserving/freeing scrub resources at the replicas.
- *
- * When constructed - sends reservation requests to the acting_set OSDs, one
- * by one.
- * Once a replica's OSD replies with a 'grant'ed reservation, we send a
- * reservation request to the next replica.
- * A rejection triggers a "couldn't acquire the replicas' scrub resources"
- * event. All granted reservations are released.
- *
- * Reserved replicas should be released at the end of the scrub session. The
- * one exception is if the scrub terminates upon an interval change. In that
- * scenario - the replicas discard their reservations on their own accord
- * when noticing the change in interval, and there is no need (and no
- * guaranteed way) to send them the release message.
- *
- * Timeouts:
- *
- *  Slow-Secondary Warning:
- *  Warn if a replica takes more than <conf> milliseconds to reply to a
- *  reservation request. Only one warning is issued per session.
- *
- *  Reservation Timeout:
- *  We limit the total time we wait for the replicas to respond to the
- *  reservation request. If the reservation back-and-forth does not complete
- *  within <conf> milliseconds, we give up and release all the reservations
- *  that have been acquired until that moment.
- *  (Why? because we have encountered instances where a reservation request was
- *  lost - either due to a bug or due to a network issue.)
- */
-class ReplicaReservations {
-  using clock = ceph::coarse_real_clock;
-
-  ScrubMachineListener& m_scrubber;
-  PG* m_pg;
-
-  /// shorthand for m_scrubber.get_spgid().pgid
-  const pg_t m_pgid;
-
-  /// for dout && when queueing messages to the FSM
-  OSDService* m_osds;
-
-  /// the acting set (not including myself), sorted by pg_shard_t
-  std::vector<pg_shard_t> m_sorted_secondaries;
-
-  /// the next replica to which we will send a reservation request
-  std::vector<pg_shard_t>::const_iterator m_next_to_request;
-
-  /// for logs, and for detecting slow peers
-  clock::time_point m_last_request_sent_at;
-
-  /// used to prevent multiple "slow response" warnings
-  bool m_slow_response_warned{false};
-
- public:
-  ReplicaReservations(ScrubMachineListener& scrubber);
-
-  ~ReplicaReservations();
-
-  /**
-   * The OK received from the replica (after verifying that it is indeed
-   * the replica we are expecting a reply from) is noted, and triggers
-   * one of two: either sending a reservation request to the next replica,
-   * or notifying the scrubber that we have reserved them all.
-   */
-  void handle_reserve_grant(OpRequestRef op, pg_shard_t from);
-
-  /**
-   * Verify that the sender of the received rejection is the replica we
-   * were expecting a reply from.
-   * If this is so - just mark the fact that the specific peer need not
-   * be released.
-   *
-   * Note - the actual handling of scrub session termination and of
-   * releasing the reserved replicas is done by the caller (the FSM).
-   */
-  void verify_rejections_source(OpRequestRef op, pg_shard_t from);
-
-  void handle_reserve_reject(OpRequestRef op, pg_shard_t from);
-
-  /**
-   * Notifies implementation that it is no longer responsible for releasing
-   * tracked remote reservations.
-   *
-   * The intended usage is upon interval change.  In general, replicas are
-   * responsible for releasing their own resources upon interval change without
-   * coordination from the primary.
-   *
-   * Sends no messages.
-   */
-  void discard_remote_reservations();
-
-  // note: 'public', as accessed via the 'standard' dout_prefix() macro
-  std::ostream& gen_prefix(std::ostream& out, std::string fn) const;
-
- private:
-  /// send 'release' messages to all replicas we have managed to reserve
-  void release_all();
-
-  /// the only replica we are expecting a reply from
-  std::optional<pg_shard_t> get_last_sent() const;
-
-  /// The number of requests that have been sent (and not rejected) so far.
-  size_t active_requests_cnt() const;
-
-  /**
-   * Either send a reservation request to the next replica, or notify the
-   * scrubber that we have reserved all the replicas.
-   */
-  void send_next_reservation_or_complete();
-};
 
 /**
  *  wraps the local OSD scrub resource reservation in an RAII wrapper
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
new file mode 100644
index 000000000000..bd5f61c9b9ec
--- /dev/null
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -0,0 +1,206 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "./scrub_reservations.h"
+
+#include <span>
+
+#include "common/ceph_time.h"
+#include "messages/MOSDScrubReserve.h"
+#include "osd/OSD.h"
+#include "osd/PG.h"
+#include "osd/osd_types_fmt.h"
+
+#include "pg_scrubber.h"
+
+using namespace Scrub;
+using namespace std::chrono;
+using namespace std::chrono_literals;
+
+#define dout_context (m_osds->cct)
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix_fn(_dout, this, __func__)
+template <class T>
+static std::ostream& _prefix_fn(std::ostream* _dout, T* t, std::string fn = "")
+{
+  return t->gen_prefix(*_dout, fn);
+}
+
+namespace Scrub {
+
+ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
+    : m_scrubber{scrbr}
+    , m_pg{m_scrubber.get_pg()}
+    , m_pgid{m_scrubber.get_spgid().pgid}
+    , m_osds{m_pg->get_pg_osd(ScrubberPasskey())}
+{
+  // the acting set is sorted by pg_shard_t. The reservations are to be issued
+  // in this order, so that the OSDs will receive the requests in a consistent
+  // order. This is done to reduce the chance of having two PGs that share some
+  // of their acting-set OSDs, consistently interfering with each other's
+  // reservation process.
+  auto acting = m_pg->get_actingset();
+  m_sorted_secondaries.reserve(acting.size());
+  std::copy_if(
+      acting.cbegin(), acting.cend(), std::back_inserter(m_sorted_secondaries),
+      [whoami = m_pg->pg_whoami](const pg_shard_t& shard) {
+	return shard != whoami;
+      });
+
+  m_next_to_request = m_sorted_secondaries.cbegin();
+  // send out the 1'st request (unless we have no replicas)
+  send_next_reservation_or_complete();
+}
+
+void ReplicaReservations::release_all()
+{
+  std::span<const pg_shard_t> replicas{
+      m_sorted_secondaries.cbegin(), m_next_to_request};
+  dout(10) << fmt::format("releasing {}", replicas) << dendl;
+  epoch_t epoch = m_pg->get_osdmap_epoch();
+
+  // send 'release' messages to all replicas we have managed to reserve
+  for (const auto& peer : replicas) {
+    auto m = make_message<MOSDScrubReserve>(
+	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::RELEASE,
+	m_pg->pg_whoami);
+    m_pg->send_cluster_message(peer.osd, m, epoch, false);
+  }
+
+  m_sorted_secondaries.clear();
+  m_next_to_request = m_sorted_secondaries.cbegin();
+}
+
+void ReplicaReservations::discard_remote_reservations()
+{
+  dout(10) << "reset w/o issuing messages" << dendl;
+  m_sorted_secondaries.clear();
+  m_next_to_request = m_sorted_secondaries.cbegin();
+}
+
+ReplicaReservations::~ReplicaReservations()
+{
+  release_all();
+}
+
+void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
+{
+  // verify that the grant is from the peer we expected. If not?
+  // for now - abort the OSD. \todo reconsider the reaction.
+  if (!get_last_sent().has_value() || from != *get_last_sent()) {
+    dout(1) << fmt::format(
+		   "unexpected grant from {} (expected {})", from,
+		   get_last_sent().value_or(pg_shard_t{}))
+	    << dendl;
+    ceph_assert(from == get_last_sent());
+    return;
+  }
+
+  auto elapsed = clock::now() - m_last_request_sent_at;
+
+  // log a warning if the response was slow to arrive
+  auto warn_timeout = m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
+      "osd_scrub_slow_reservation_response");
+  if (!m_slow_response_warned && (elapsed > warn_timeout)) {
+    dout(1) << fmt::format(
+		   "slow reservation response from {} ({}ms)", from,
+		   duration_cast<milliseconds>(elapsed).count())
+	    << dendl;
+    // prevent additional warnings
+    m_slow_response_warned = true;
+  }
+  dout(10) << fmt::format(
+		  "granted by {} ({} of {}) in {}ms", from,
+		  active_requests_cnt(), m_sorted_secondaries.size(),
+		  duration_cast<milliseconds>(elapsed).count())
+	   << dendl;
+  send_next_reservation_or_complete();
+}
+
+void ReplicaReservations::send_next_reservation_or_complete()
+{
+  if (m_next_to_request == m_sorted_secondaries.cend()) {
+    // granted by all replicas
+    dout(10) << "remote reservation complete" << dendl;
+    m_osds->queue_for_scrub_granted(m_pg, scrub_prio_t::low_priority);
+
+  } else {
+    // send the next reservation request
+    const auto peer = *m_next_to_request;
+    const auto epoch = m_pg->get_osdmap_epoch();
+    auto m = make_message<MOSDScrubReserve>(
+	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
+	m_pg->pg_whoami);
+    m_pg->send_cluster_message(peer.osd, m, epoch, false);
+    m_last_request_sent_at = clock::now();
+    dout(10) << fmt::format(
+		    "reserving {} (the {} of {} replicas)", *m_next_to_request,
+		    active_requests_cnt()+1, m_sorted_secondaries.size())
+	     << dendl;
+    m_next_to_request++;
+  }
+}
+
+// temporary comment: the part of handle_reserve_reject() that will not
+// be delegated to the FSM in the following commits
+void ReplicaReservations::verify_rejections_source(
+    OpRequestRef op,
+    pg_shard_t from)
+{
+  // a convenient log message for the reservation process conclusion
+  // (matches the one in send_next_reservation_or_complete())
+  dout(10) << fmt::format(
+		  "remote reservation failure. Rejected by {} ({})", from,
+		  *op->get_req())
+	   << dendl;
+
+  // verify that the denial is from the peer we expected. If not?
+  // we should treat it as though the *correct* peer has rejected the request,
+  // but remember to release that peer, too.
+
+  ceph_assert(get_last_sent().has_value());
+  const auto expected = *get_last_sent();
+  if (from != expected) {
+    dout(1) << fmt::format(
+		   "unexpected rejection from {} (expected {})", from, expected)
+	    << dendl;
+  } else {
+    // correct peer, wrong answer...
+    m_next_to_request--;  // no need to release this one
+  }
+}
+
+// to be delegated to the FSM in the following commits
+void ReplicaReservations::handle_reserve_reject(
+    OpRequestRef op,
+    pg_shard_t from)
+{
+  verify_rejections_source(op, from);
+  release_all();
+  m_scrubber.flag_reservations_failure();
+  m_osds->queue_for_scrub_denied(m_pg, scrub_prio_t::low_priority);
+}
+
+std::optional<pg_shard_t> ReplicaReservations::get_last_sent() const
+{
+  if (m_next_to_request == m_sorted_secondaries.cbegin()) {
+    return std::nullopt;
+  }
+  return *(m_next_to_request - 1);
+}
+
+size_t ReplicaReservations::active_requests_cnt() const
+{
+  return m_next_to_request - m_sorted_secondaries.cbegin();
+}
+
+std::ostream& ReplicaReservations::gen_prefix(
+    std::ostream& out,
+    std::string fn) const
+{
+  return m_pg->gen_prefix(out)
+	 << fmt::format("scrubber::ReplicaReservations:{}: ", fn);
+}
+
+} // namespace Scrub
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
new file mode 100644
index 000000000000..a52f385a4fdf
--- /dev/null
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -0,0 +1,130 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#pragma once
+
+#include <cassert>
+#include <chrono>
+#include <optional>
+#include <string_view>
+#include <vector>
+
+#include "osd/scrubber_common.h"
+
+#include "osd_scrub_sched.h"
+#include "scrub_machine_lstnr.h"
+
+namespace Scrub {
+
+/**
+ * Reserving/freeing scrub resources at the replicas.
+ *
+ * When constructed - sends reservation requests to the acting_set OSDs, one
+ * by one.
+ * Once a replica's OSD replies with a 'grant'ed reservation, we send a
+ * reservation request to the next replica.
+ * A rejection triggers a "couldn't acquire the replicas' scrub resources"
+ * event. All granted reservations are released.
+ *
+ * Reserved replicas should be released at the end of the scrub session. The
+ * one exception is if the scrub terminates upon an interval change. In that
+ * scenario - the replicas discard their reservations on their own accord
+ * when noticing the change in interval, and there is no need (and no
+ * guaranteed way) to send them the release message.
+ *
+ * Timeouts:
+ *
+ *  Slow-Secondary Warning:
+ *  Warn if a replica takes more than <conf> milliseconds to reply to a
+ *  reservation request. Only one warning is issued per session.
+ *
+ *  Reservation Timeout:
+ *  We limit the total time we wait for the replicas to respond to the
+ *  reservation request. If the reservation back-and-forth does not complete
+ *  within <conf> milliseconds, we give up and release all the reservations
+ *  that have been acquired until that moment.
+ *  (Why? because we have encountered instances where a reservation request was
+ *  lost - either due to a bug or due to a network issue.)
+ */
+class ReplicaReservations {
+  using clock = ceph::coarse_real_clock;
+
+  ScrubMachineListener& m_scrubber;
+  PG* m_pg;
+
+  /// shorthand for m_scrubber.get_spgid().pgid
+  const pg_t m_pgid;
+
+  /// for dout && when queueing messages to the FSM
+  OSDService* m_osds;
+
+  /// the acting set (not including myself), sorted by pg_shard_t
+  std::vector<pg_shard_t> m_sorted_secondaries;
+
+  /// the next replica to which we will send a reservation request
+  std::vector<pg_shard_t>::const_iterator m_next_to_request;
+
+  /// for logs, and for detecting slow peers
+  clock::time_point m_last_request_sent_at;
+
+  /// used to prevent multiple "slow response" warnings
+  bool m_slow_response_warned{false};
+
+ public:
+  ReplicaReservations(ScrubMachineListener& scrubber);
+
+  ~ReplicaReservations();
+
+  /**
+   * The OK received from the replica (after verifying that it is indeed
+   * the replica we are expecting a reply from) is noted, and triggers
+   * one of two: either sending a reservation request to the next replica,
+   * or notifying the scrubber that we have reserved them all.
+   */
+  void handle_reserve_grant(OpRequestRef op, pg_shard_t from);
+
+  /**
+   * Verify that the sender of the received rejection is the replica we
+   * were expecting a reply from.
+   * If this is so - just mark the fact that the specific peer need not
+   * be released.
+   *
+   * Note - the actual handling of scrub session termination and of
+   * releasing the reserved replicas is done by the caller (the FSM).
+   */
+  void verify_rejections_source(OpRequestRef op, pg_shard_t from);
+
+  void handle_reserve_reject(OpRequestRef op, pg_shard_t from);
+
+  /**
+   * Notifies implementation that it is no longer responsible for releasing
+   * tracked remote reservations.
+   *
+   * The intended usage is upon interval change.  In general, replicas are
+   * responsible for releasing their own resources upon interval change without
+   * coordination from the primary.
+   *
+   * Sends no messages.
+   */
+  void discard_remote_reservations();
+
+  // note: 'public', as accessed via the 'standard' dout_prefix() macro
+  std::ostream& gen_prefix(std::ostream& out, std::string fn) const;
+
+ private:
+  /// send 'release' messages to all replicas we have managed to reserve
+  void release_all();
+
+  /// the only replica we are expecting a reply from
+  std::optional<pg_shard_t> get_last_sent() const;
+
+  /// The number of requests that have been sent (and not rejected) so far.
+  size_t active_requests_cnt() const;
+
+  /**
+   * Either send a reservation request to the next replica, or notify the
+   * scrubber that we have reserved all the replicas.
+   */
+  void send_next_reservation_or_complete();
+};
+
+} // namespace Scrub

From 10401db4a67443ff6579cfce31135917dd19fec3 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Fri, 13 Oct 2023 12:14:31 -0500
Subject: [PATCH 0184/2492] osd/scrub: move ReplicaReservations into the
 Scrubber FSM

Handle grant/deny messages within the FSM.
One exception at this point: the handling of "granted by everyone"
(due to the technical inconvenience of having to handle the
"0 replicas" case in the FSM state constructor).

Note: after this commit, ScrubMachineListener - an API which is
a subset of the Scrubber API to be used by the Scrubber FSM - does
no longer make sense. The FSM should now have full access to the
scrubber, and that interface will be removed in a subsequent PR.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc                         |  6 ---
 src/osd/OSD.h                          |  3 --
 src/osd/PG.h                           |  6 ---
 src/osd/scheduler/OpSchedulerItem.cc   |  9 -----
 src/osd/scheduler/OpSchedulerItem.h    | 11 ------
 src/osd/scrubber/pg_scrubber.cc        | 54 +-------------------------
 src/osd/scrubber/pg_scrubber.h         | 12 +-----
 src/osd/scrubber/scrub_machine.cc      | 45 +++++++++++----------
 src/osd/scrubber/scrub_machine.h       | 18 ++++-----
 src/osd/scrubber/scrub_machine_lstnr.h | 18 ++-------
 src/osd/scrubber/scrub_reservations.cc | 13 -------
 src/osd/scrubber/scrub_reservations.h  |  2 -
 src/osd/scrubber_common.h              | 22 +----------
 13 files changed, 36 insertions(+), 183 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index d6948cf05a76..88a0bc037564 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1805,12 +1805,6 @@ void OSDService::queue_for_scrub_granted(PG* pg, Scrub::scrub_prio_t with_priori
   queue_scrub_event_msg<PGScrubResourcesOK>(pg, with_priority);
 }
 
-void OSDService::queue_for_scrub_denied(PG* pg, Scrub::scrub_prio_t with_priority)
-{
-  // Resulting scrub event: 'ReservationFailure'
-  queue_scrub_event_msg<PGScrubDenied>(pg, with_priority);
-}
-
 void OSDService::queue_for_scrub_resched(PG* pg, Scrub::scrub_prio_t with_priority)
 {
   // Resulting scrub event: 'InternalSchedScrub'
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 4ba3d7681194..d86443351cf2 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -507,9 +507,6 @@ class OSDService : public Scrub::ScrubSchedListener {
   /// queue the message (-> event) that all replicas have reserved scrub resources for us
   void queue_for_scrub_granted(PG* pg, Scrub::scrub_prio_t with_priority);
 
-  /// queue the message (-> event) that some replicas denied our scrub resources request
-  void queue_for_scrub_denied(PG* pg, Scrub::scrub_prio_t with_priority);
-
   /// Signals either (a) the end of a sleep period, or (b) a recheck of the availability
   /// of the primary map being created by the backend.
   void queue_for_scrub_resched(PG* pg, Scrub::scrub_prio_t with_priority);
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 56ac9a65bf1a..fe335b85e000 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -454,12 +454,6 @@ class PG : public DoutPrefixProvider,
     forward_scrub_event(&ScrubPgIF::send_remotes_reserved, queued, "RemotesReserved");
   }
 
-  void scrub_send_resources_denied(epoch_t queued, ThreadPool::TPHandle& handle)
-  {
-    forward_scrub_event(&ScrubPgIF::send_reservation_failure, queued,
-			"ReservationFailure");
-  }
-
   void scrub_send_scrub_resched(epoch_t queued, ThreadPool::TPHandle& handle)
   {
     forward_scrub_event(&ScrubPgIF::send_scrub_resched, queued, "InternalSchedScrub");
diff --git a/src/osd/scheduler/OpSchedulerItem.cc b/src/osd/scheduler/OpSchedulerItem.cc
index d1abc264a8f8..0641aafdc1c9 100644
--- a/src/osd/scheduler/OpSchedulerItem.cc
+++ b/src/osd/scheduler/OpSchedulerItem.cc
@@ -86,15 +86,6 @@ void PGScrubResourcesOK::run(OSD* osd,
   pg->unlock();
 }
 
-void PGScrubDenied::run(OSD* osd,
-			OSDShard* sdata,
-			PGRef& pg,
-			ThreadPool::TPHandle& handle)
-{
-  pg->scrub_send_resources_denied(epoch_queued, handle);
-  pg->unlock();
-}
-
 void PGScrubPushesUpdate::run(OSD* osd,
 			      OSDShard* sdata,
 			      PGRef& pg,
diff --git a/src/osd/scheduler/OpSchedulerItem.h b/src/osd/scheduler/OpSchedulerItem.h
index 3222901dc706..2803169a9bf9 100644
--- a/src/osd/scheduler/OpSchedulerItem.h
+++ b/src/osd/scheduler/OpSchedulerItem.h
@@ -384,17 +384,6 @@ class PGScrubResourcesOK : public PGScrubItem {
   void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
 };
 
-/**
- *  scrub resources requests denied by replica(s)
- */
-class PGScrubDenied : public PGScrubItem {
- public:
-  PGScrubDenied(spg_t pg, epoch_t epoch_queued)
-      : PGScrubItem{pg, epoch_queued, "PGScrubDenied"}
-  {}
-  void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
-};
-
 /**
  *  called when a repair process completes, to initiate scrubbing. No local/remote
  *  resources are allocated.
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 609eb0a72e6a..98290a06ac45 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -370,16 +370,6 @@ void PgScrubber::send_remotes_reserved(epoch_t epoch_queued)
   dout(10) << "scrubber event --<< " << __func__ << dendl;
 }
 
-void PgScrubber::send_reservation_failure(epoch_t epoch_queued)
-{
-  dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued
-	   << dendl;
-  if (check_interval(epoch_queued)) {  // do not check for 'active'!
-    m_fsm->process_event(ReservationFailure{});
-  }
-  dout(10) << "scrubber event --<< " << __func__ << dendl;
-}
-
 void PgScrubber::send_chunk_free(epoch_t epoch_queued)
 {
   dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued
@@ -1623,25 +1613,6 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
   m_osds->send_message_osd_cluster(reply, op->get_req()->get_connection());
 }
 
-
-/// temporary
-void PgScrubber::grant_from_replica(OpRequestRef op, pg_shard_t from)
-{
-  dout(10) << fmt::format("{}: {}", __func__, *op->get_req()) << dendl;
-  ceph_assert(m_reservations.has_value()); // the FSM should know
-  m_reservations->handle_reserve_grant(op, from);
-}
-
-
-/// temporary
-void PgScrubber::reject_from_replica(OpRequestRef op, pg_shard_t from)
-{
-  dout(10) << fmt::format("{}: {}", __func__, *op->get_req()) << dendl;
-  ceph_assert(m_reservations.has_value()); // the FSM should know
-  m_reservations->handle_reserve_reject(op, from);
-}
-
-
 void PgScrubber::handle_scrub_reserve_release(OpRequestRef op)
 {
   dout(10) << __func__ << " " << *op->get_req() << dendl;
@@ -1656,21 +1627,6 @@ void PgScrubber::handle_scrub_reserve_release(OpRequestRef op)
   m_fsm->process_event(FullReset{});
 }
 
-void PgScrubber::discard_replica_reservations()
-{
-  dout(10) << __func__ << dendl;
-  if (m_reservations.has_value()) {
-    m_reservations->discard_remote_reservations();
-  }
-}
-
-void PgScrubber::clear_scrub_reservations()
-{
-  dout(10) << __func__ << dendl;
-  m_reservations.reset();	  // the remote reservations
-  m_local_osd_resource.reset();	  // the local reservation
-}
-
 bool PgScrubber::set_reserving_now() {
   return m_osds->get_scrub_services().set_reserving_now(m_pg_id,
                                                         ceph_clock_now());
@@ -2181,12 +2137,6 @@ void PgScrubber::set_scrub_duration()
   });
 }
 
-void PgScrubber::reserve_replicas()
-{
-  dout(10) << __func__ << dendl;
-  m_reservations.emplace(*this);
-}
-
 void PgScrubber::cleanup_on_finish()
 {
   dout(10) << __func__ << dendl;
@@ -2195,7 +2145,7 @@ void PgScrubber::cleanup_on_finish()
   state_clear(PG_STATE_SCRUBBING);
   state_clear(PG_STATE_DEEP_SCRUB);
 
-  clear_scrub_reservations();
+  m_local_osd_resource.reset();
   requeue_waiting();
 
   reset_internal_state();
@@ -2229,7 +2179,7 @@ void PgScrubber::clear_pgscrub_state()
 
   state_clear(PG_STATE_REPAIR);
 
-  clear_scrub_reservations();
+  m_local_osd_resource.reset();
   requeue_waiting();
 
   reset_internal_state();
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index fc7b735d219e..0c8fa8c34fb7 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -265,8 +265,6 @@ class PgScrubber : public ScrubPgIF,
   void handle_scrub_reserve_request(OpRequestRef op);
 
   void handle_scrub_reserve_release(OpRequestRef op);
-  void discard_replica_reservations() final;
-  void clear_scrub_reservations() final;  // PG::clear... fwds to here
 
   // managing scrub op registration
 
@@ -448,7 +446,6 @@ class PgScrubber : public ScrubPgIF,
   void send_preempted_replica() final;
 
   void send_remotes_reserved(epoch_t epoch_queued) final;
-  void send_reservation_failure(epoch_t epoch_queued) final;
 
   /**
    *  does the PG have newer updates than what we (the scrubber) know?
@@ -465,8 +462,6 @@ class PgScrubber : public ScrubPgIF,
 
   int build_replica_map_chunk() final;
 
-  void reserve_replicas() final;
-
   bool set_reserving_now() final;
   void clear_reserving_now() final;
 
@@ -501,10 +496,6 @@ class PgScrubber : public ScrubPgIF,
 
   void log_cluster_warning(const std::string& warning) const final;
 
-  // temporary interface to handle forwarded reservation messages:
-  void grant_from_replica(OpRequestRef op, pg_shard_t from) final;
-  void reject_from_replica(OpRequestRef op, pg_shard_t from) final;
-
  protected:
   bool state_test(uint64_t m) const { return m_pg->state_test(m); }
   void state_set(uint64_t m) { m_pg->state_set(m); }
@@ -633,9 +624,8 @@ class PgScrubber : public ScrubPgIF,
 
   epoch_t m_last_aborted{};  // last time we've noticed a request to abort
 
-  // 'optional', as 'ReplicaReservations' & 'LocalReservation' are
+  // 'optional', as 'LocalReservation' is
   // 'RAII-designed' to guarantee un-reserving when deleted.
-  std::optional<Scrub::ReplicaReservations> m_reservations;
   std::optional<Scrub::LocalReservation> m_local_osd_resource;
 
   void cleanup_on_finish();  // scrub_clear_state() as called for a Primary when
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index f6eb471530a2..cc257a47f0e3 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -131,6 +131,7 @@ Session::Session(my_context ctx)
 Session::~Session()
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  m_reservations.reset();
 
   // note the interaction between clearing the 'queued' flag and two
   // other states: the snap-mapper and the scrubber internal state.
@@ -145,8 +146,7 @@ sc::result Session::react(const IntervalChanged&)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "Session::react(const IntervalChanged&)" << dendl;
 
-  /// \todo (future commit): the reservations will be local to this state
-  scrbr->discard_replica_reservations();
+  m_reservations->discard_remote_reservations();
   return transit<NotActive>();
 }
 
@@ -160,10 +160,11 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
   dout(10) << "-- state -->> ReservingReplicas" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
-  scrbr->reserve_replicas();
+  // initiate the reservation process
+  context<Session>().m_reservations.emplace(*scrbr);
 
-  auto timeout = scrbr->get_pg_cct()->_conf.get_val<
-    std::chrono::milliseconds>("osd_scrub_reservation_timeout");
+  auto timeout = scrbr->get_pg_cct()->_conf.get_val<milliseconds>(
+      "osd_scrub_reservation_timeout");
   if (timeout.count() > 0) {
     // Start a timer to handle case where the replicas take a long time to
     // ack the reservation.  See ReservationTimeout handler below.
@@ -182,24 +183,29 @@ ReservingReplicas::~ReservingReplicas()
 
 sc::result ReservingReplicas::react(const ReplicaGrant& ev)
 {
-  // for now - route the message back to the scrubber, as it holds the
-  // ReplicaReservations object
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReservingReplicas::react(const ReplicaGrant&)" << dendl;
 
-  scrbr->grant_from_replica(ev.m_op, ev.m_from);
+  context<Session>().m_reservations->handle_reserve_grant(ev.m_op, ev.m_from);
   return discard_event();
 }
 
 sc::result ReservingReplicas::react(const ReplicaReject& ev)
 {
-  // for now - route the message back to the scrubber, as it holds the
-  // ReplicaReservations object
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReservingReplicas::react(const ReplicaReject&)" << dendl;
 
-  scrbr->reject_from_replica(ev.m_op, ev.m_from);
-  return discard_event();
+  // manipulate the 'next to reserve' iterator to exclude
+  // the rejecting replica from the set of replicas requiring release
+  context<Session>().m_reservations->verify_rejections_source(
+      ev.m_op, ev.m_from);
+
+  // set 'reservation failure' as the scrub termination cause (affecting
+  // the rescheduling of this PG)
+  scrbr->flag_reservations_failure();
+
+  // 'Session' state dtor stops the scrubber
+  return transit<NotActive>();
 }
 
 sc::result ReservingReplicas::react(const ReservationTimeout&)
@@ -208,10 +214,10 @@ sc::result ReservingReplicas::react(const ReservationTimeout&)
   dout(10) << "ReservingReplicas::react(const ReservationTimeout&)" << dendl;
 
   const auto msg = fmt::format(
-      "PgScrubber: {} timeout on reserving replicas (since {})",
-      scrbr->get_spgid(), entered_at);
-  dout(5) << msg << dendl;
-  scrbr->get_clog()->warn() << "osd." << scrbr->get_whoami() << " " << msg;
+      "osd.{} PgScrubber: {} timeout on reserving replicas (since {})",
+      scrbr->get_whoami(), scrbr->get_spgid(), entered_at);
+  dout(1) << msg << dendl;
+  scrbr->get_clog()->warn() << msg;
 
   // cause the scrubber to stop the scrub session, marking 'reservation
   // failure' as the cause (affecting future scheduling)
@@ -219,13 +225,6 @@ sc::result ReservingReplicas::react(const ReservationTimeout&)
   return transit<NotActive>();
 }
 
-sc::result ReservingReplicas::react(const ReservationFailure&)
-{
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "ReservingReplicas::react(const ReservationFailure&)" << dendl;
-  return transit<NotActive>();
-}
-
 // ----------------------- ActiveScrubbing -----------------------------------
 
 ActiveScrubbing::ActiveScrubbing(my_context ctx)
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 806376e3032d..2f73cbbefb5d 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -24,6 +24,7 @@
 #include "osd/scrubber_common.h"
 
 #include "scrub_machine_lstnr.h"
+#include "scrub_reservations.h"
 
 /// a wrapper that sets the FSM state description used by the
 /// PgScrubber
@@ -101,9 +102,6 @@ struct ReplicaReject : sc::event<ReplicaReject> {
 /// all replicas have granted our reserve request
 MEV(RemotesReserved)
 
-/// a reservation request has failed
-MEV(ReservationFailure)
-
 /// reservations have timed out
 MEV(ReservationTimeout)
 
@@ -391,19 +389,20 @@ struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>,
                               sc::custom_reaction<IntervalChanged>>;
 
   sc::result react(const IntervalChanged&);
+
+  /// managing the scrub session's reservations (optional, as
+  /// it's an RAII wrapper around the state of 'holding reservations')
+  std::optional<ReplicaReservations> m_reservations{std::nullopt};
 };
 
 struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
 			   NamedSimply {
   explicit ReservingReplicas(my_context ctx);
   ~ReservingReplicas();
-  using reactions = mpl::list<
-			      // all replicas granted our resources request
-			      sc::custom_reaction<ReplicaGrant>,
+  using reactions = mpl::list<sc::custom_reaction<ReplicaGrant>,
 			      sc::custom_reaction<ReplicaReject>,
 			      sc::transition<RemotesReserved, ActiveScrubbing>,
-			      sc::custom_reaction<ReservationTimeout>,
-			      sc::custom_reaction<ReservationFailure>>;
+			      sc::custom_reaction<ReservationTimeout>>;
 
   ceph::coarse_real_clock::time_point entered_at =
     ceph::coarse_real_clock::now();
@@ -416,9 +415,6 @@ struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
   sc::result react(const ReplicaReject&);
 
   sc::result react(const ReservationTimeout&);
-
-  /// at least one replica denied us the scrub resources we've requested
-  sc::result react(const ReservationFailure&);
 };
 
 
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index e760ddf0055d..4206c789f91a 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -5,11 +5,13 @@
 /**
  * \file the PgScrubber interface used by the scrub FSM
  */
+#include "common/LogClient.h"
 #include "common/version.h"
 #include "include/Context.h"
 #include "osd/osd_types.h"
+#include "osd/scrubber_common.h"
 
-struct ScrubMachineListener;
+class PG;
 
 namespace Scrub {
 
@@ -172,15 +174,6 @@ struct ScrubMachineListener {
    */
   virtual void maps_compare_n_cleanup() = 0;
 
-  /**
-   * order the PgScrubber to initiate the process of reserving replicas' scrub
-   * resources.
-   */
-  virtual void reserve_replicas() = 0;
-
-  /// discard replica reservations without sending a message (on interval)
-  virtual void discard_replica_reservations() = 0;
-
   virtual void set_scrub_begin_time() = 0;
 
   virtual void set_scrub_duration() = 0;
@@ -239,11 +232,6 @@ struct ScrubMachineListener {
   /// sending cluster-log warnings
   virtual void log_cluster_warning(const std::string& msg) const = 0;
 
-  // temporary interface (to be discarded in a follow-up commit)
-  // to handle replica reservation messages routed thru the FSM
-  virtual void grant_from_replica(OpRequestRef op, pg_shard_t from) = 0;
-  virtual void reject_from_replica(OpRequestRef op, pg_shard_t from) = 0;
-
   // temporary interface (to be discarded in a follow-up PR)
   /// set the 'resources_failure' flag in the scrub-job object
   virtual void flag_reservations_failure() = 0;
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index bd5f61c9b9ec..011ace2a2836 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -142,8 +142,6 @@ void ReplicaReservations::send_next_reservation_or_complete()
   }
 }
 
-// temporary comment: the part of handle_reserve_reject() that will not
-// be delegated to the FSM in the following commits
 void ReplicaReservations::verify_rejections_source(
     OpRequestRef op,
     pg_shard_t from)
@@ -171,17 +169,6 @@ void ReplicaReservations::verify_rejections_source(
   }
 }
 
-// to be delegated to the FSM in the following commits
-void ReplicaReservations::handle_reserve_reject(
-    OpRequestRef op,
-    pg_shard_t from)
-{
-  verify_rejections_source(op, from);
-  release_all();
-  m_scrubber.flag_reservations_failure();
-  m_osds->queue_for_scrub_denied(m_pg, scrub_prio_t::low_priority);
-}
-
 std::optional<pg_shard_t> ReplicaReservations::get_last_sent() const
 {
   if (m_next_to_request == m_sorted_secondaries.cbegin()) {
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index a52f385a4fdf..634e7e580027 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -93,8 +93,6 @@ class ReplicaReservations {
    */
   void verify_rejections_source(OpRequestRef op, pg_shard_t from);
 
-  void handle_reserve_reject(OpRequestRef op, pg_shard_t from);
-
   /**
    * Notifies implementation that it is no longer responsible for releasing
    * tracked remote reservations.
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 24ef8206980a..61be0bd5a62c 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -20,7 +20,7 @@ namespace Scrub {
   class ReplicaReservations;
 }
 
-/// Facilitating scrub-realated object access to private PG data
+/// Facilitating scrub-related object access to private PG data
 class ScrubberPasskey {
 private:
   friend class Scrub::ReplicaReservations;
@@ -362,12 +362,6 @@ struct ScrubPgIF {
    */
   virtual void send_remotes_reserved(epoch_t epoch_queued) = 0;
 
-  /**
-   * triggers the 'ReservationFailure' (at least one replica denied us the
-   * requested resources) state-machine event
-   */
-  virtual void send_reservation_failure(epoch_t epoch_queued) = 0;
-
   virtual void cleanup_store(ObjectStore::Transaction* t) = 0;
 
   virtual bool get_store_errors(const scrub_ls_arg_t& arg,
@@ -382,20 +376,6 @@ struct ScrubPgIF {
 
   // --------------- reservations -----------------------------------
 
-  /**
-   *  "forget" all replica reservations. No messages are sent to the
-   *  previously-reserved.
-   *
-   *  Used upon interval change. The replicas' state is guaranteed to
-   *  be reset separately by the interval-change event.
-   */
-  virtual void discard_replica_reservations() = 0;
-
-  /**
-   * clear both local and OSD-managed resource reservation flags
-   */
-  virtual void clear_scrub_reservations() = 0;
-
   /**
    * Reserve local scrub resources (managed by the OSD)
    *

From 13b84c856e797c872eab3231a9e352ea3ad99564 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 14 Oct 2023 07:36:06 -0500
Subject: [PATCH 0185/2492] osd/scrub: handle reservation completion within the
 Scrubber FSM

with special handling for the 0-replica case.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc                         |  6 ----
 src/osd/OSD.h                          |  3 --
 src/osd/PG.h                           |  5 ----
 src/osd/scheduler/OpSchedulerItem.cc   |  9 ------
 src/osd/scheduler/OpSchedulerItem.h    | 11 -------
 src/osd/scrubber/pg_scrubber.cc        | 11 -------
 src/osd/scrubber/pg_scrubber.h         |  2 --
 src/osd/scrubber/scrub_machine.cc      | 29 +++++++++++++------
 src/osd/scrubber/scrub_reservations.cc | 40 +++++++++++++-------------
 src/osd/scrubber/scrub_reservations.h  | 17 ++++++-----
 src/osd/scrubber_common.h              |  6 ----
 11 files changed, 51 insertions(+), 88 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 88a0bc037564..fa938c082781 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1799,12 +1799,6 @@ void OSDService::queue_for_rep_scrub_resched(PG* pg,
 					   act_token);
 }
 
-void OSDService::queue_for_scrub_granted(PG* pg, Scrub::scrub_prio_t with_priority)
-{
-  // Resulting scrub event: 'RemotesReserved'
-  queue_scrub_event_msg<PGScrubResourcesOK>(pg, with_priority);
-}
-
 void OSDService::queue_for_scrub_resched(PG* pg, Scrub::scrub_prio_t with_priority)
 {
   // Resulting scrub event: 'InternalSchedScrub'
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index d86443351cf2..38f9a6ca8475 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -504,9 +504,6 @@ class OSDService : public Scrub::ScrubSchedListener {
 
   void queue_scrub_after_repair(PG* pg, Scrub::scrub_prio_t with_priority);
 
-  /// queue the message (-> event) that all replicas have reserved scrub resources for us
-  void queue_for_scrub_granted(PG* pg, Scrub::scrub_prio_t with_priority);
-
   /// Signals either (a) the end of a sleep period, or (b) a recheck of the availability
   /// of the primary map being created by the backend.
   void queue_for_scrub_resched(PG* pg, Scrub::scrub_prio_t with_priority);
diff --git a/src/osd/PG.h b/src/osd/PG.h
index fe335b85e000..70c1d12b2105 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -449,11 +449,6 @@ class PG : public DoutPrefixProvider,
 			"SchedReplica");
   }
 
-  void scrub_send_resources_granted(epoch_t queued, ThreadPool::TPHandle& handle)
-  {
-    forward_scrub_event(&ScrubPgIF::send_remotes_reserved, queued, "RemotesReserved");
-  }
-
   void scrub_send_scrub_resched(epoch_t queued, ThreadPool::TPHandle& handle)
   {
     forward_scrub_event(&ScrubPgIF::send_scrub_resched, queued, "InternalSchedScrub");
diff --git a/src/osd/scheduler/OpSchedulerItem.cc b/src/osd/scheduler/OpSchedulerItem.cc
index 0641aafdc1c9..750fc2a4f58e 100644
--- a/src/osd/scheduler/OpSchedulerItem.cc
+++ b/src/osd/scheduler/OpSchedulerItem.cc
@@ -77,15 +77,6 @@ void PGScrubResched::run(OSD* osd,
   pg->unlock();
 }
 
-void PGScrubResourcesOK::run(OSD* osd,
-			     OSDShard* sdata,
-			     PGRef& pg,
-			     ThreadPool::TPHandle& handle)
-{
-  pg->scrub_send_resources_granted(epoch_queued, handle);
-  pg->unlock();
-}
-
 void PGScrubPushesUpdate::run(OSD* osd,
 			      OSDShard* sdata,
 			      PGRef& pg,
diff --git a/src/osd/scheduler/OpSchedulerItem.h b/src/osd/scheduler/OpSchedulerItem.h
index 2803169a9bf9..7fb7125a1416 100644
--- a/src/osd/scheduler/OpSchedulerItem.h
+++ b/src/osd/scheduler/OpSchedulerItem.h
@@ -373,17 +373,6 @@ class PGScrubResched : public PGScrubItem {
   void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
 };
 
-/**
- *  all replicas have granted our scrub resources request
- */
-class PGScrubResourcesOK : public PGScrubItem {
- public:
-  PGScrubResourcesOK(spg_t pg, epoch_t epoch_queued)
-      : PGScrubItem{pg, epoch_queued, "PGScrubResourcesOK"}
-  {}
-  void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
-};
-
 /**
  *  called when a repair process completes, to initiate scrubbing. No local/remote
  *  resources are allocated.
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 98290a06ac45..b090ec113ee3 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -359,17 +359,6 @@ void PgScrubber::send_replica_pushes_upd(epoch_t epoch_queued)
   dout(10) << "scrubber event --<< " << __func__ << dendl;
 }
 
-void PgScrubber::send_remotes_reserved(epoch_t epoch_queued)
-{
-  dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued
-	   << dendl;
-  // note: scrub is not active yet
-  if (check_interval(epoch_queued)) {
-    m_fsm->process_event(RemotesReserved{});
-  }
-  dout(10) << "scrubber event --<< " << __func__ << dendl;
-}
-
 void PgScrubber::send_chunk_free(epoch_t epoch_queued)
 {
   dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 0c8fa8c34fb7..97bf7da8f22a 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -445,8 +445,6 @@ class PgScrubber : public ScrubPgIF,
 
   void send_preempted_replica() final;
 
-  void send_remotes_reserved(epoch_t epoch_queued) final;
-
   /**
    *  does the PG have newer updates than what we (the scrubber) know?
    */
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index cc257a47f0e3..40b43b6e0770 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -163,13 +163,22 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
   // initiate the reservation process
   context<Session>().m_reservations.emplace(*scrbr);
 
-  auto timeout = scrbr->get_pg_cct()->_conf.get_val<milliseconds>(
-      "osd_scrub_reservation_timeout");
-  if (timeout.count() > 0) {
-    // Start a timer to handle case where the replicas take a long time to
-    // ack the reservation.  See ReservationTimeout handler below.
-    m_timeout_token = machine.schedule_timer_event_after<ReservationTimeout>(
-      timeout);
+  if (context<Session>().m_reservations->get_last_sent()) {
+    // the 1'st reservation request was sent
+
+    auto timeout = scrbr->get_pg_cct()->_conf.get_val<milliseconds>(
+	"osd_scrub_reservation_timeout");
+    if (timeout.count() > 0) {
+      // Start a timer to handle case where the replicas take a long time to
+      // ack the reservation.  See ReservationTimeout handler below.
+      m_timeout_token =
+	  machine.schedule_timer_event_after<ReservationTimeout>(timeout);
+    }
+  } else {
+    // no replicas to reserve
+    dout(10) << "no replicas to reserve" << dendl;
+    // can't transit directly from here
+    post_event(RemotesReserved{});
   }
 }
 
@@ -186,7 +195,11 @@ sc::result ReservingReplicas::react(const ReplicaGrant& ev)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReservingReplicas::react(const ReplicaGrant&)" << dendl;
 
-  context<Session>().m_reservations->handle_reserve_grant(ev.m_op, ev.m_from);
+  if (context<Session>().m_reservations->handle_reserve_grant(
+	  ev.m_op, ev.m_from)) {
+    // we are done with the reservation process
+    return transit<ActiveScrubbing>();
+  }
   return discard_event();
 }
 
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 011ace2a2836..4d187a2a644f 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -84,7 +84,7 @@ ReplicaReservations::~ReplicaReservations()
   release_all();
 }
 
-void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
+bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
 {
   // verify that the grant is from the peer we expected. If not?
   // for now - abort the OSD. \todo reconsider the reaction.
@@ -94,7 +94,7 @@ void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
 		   get_last_sent().value_or(pg_shard_t{}))
 	    << dendl;
     ceph_assert(from == get_last_sent());
-    return;
+    return false;
   }
 
   auto elapsed = clock::now() - m_last_request_sent_at;
@@ -115,31 +115,31 @@ void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
 		  active_requests_cnt(), m_sorted_secondaries.size(),
 		  duration_cast<milliseconds>(elapsed).count())
 	   << dendl;
-  send_next_reservation_or_complete();
+  return send_next_reservation_or_complete();
 }
 
-void ReplicaReservations::send_next_reservation_or_complete()
+bool ReplicaReservations::send_next_reservation_or_complete()
 {
   if (m_next_to_request == m_sorted_secondaries.cend()) {
     // granted by all replicas
     dout(10) << "remote reservation complete" << dendl;
-    m_osds->queue_for_scrub_granted(m_pg, scrub_prio_t::low_priority);
-
-  } else {
-    // send the next reservation request
-    const auto peer = *m_next_to_request;
-    const auto epoch = m_pg->get_osdmap_epoch();
-    auto m = make_message<MOSDScrubReserve>(
-	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
-	m_pg->pg_whoami);
-    m_pg->send_cluster_message(peer.osd, m, epoch, false);
-    m_last_request_sent_at = clock::now();
-    dout(10) << fmt::format(
-		    "reserving {} (the {} of {} replicas)", *m_next_to_request,
-		    active_requests_cnt()+1, m_sorted_secondaries.size())
-	     << dendl;
-    m_next_to_request++;
+    return true;  // done
   }
+
+  // send the next reservation request
+  const auto peer = *m_next_to_request;
+  const auto epoch = m_pg->get_osdmap_epoch();
+  auto m = make_message<MOSDScrubReserve>(
+      spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
+      m_pg->pg_whoami);
+  m_pg->send_cluster_message(peer.osd, m, epoch, false);
+  m_last_request_sent_at = clock::now();
+  dout(10) << fmt::format(
+		  "reserving {} (the {} of {} replicas)", *m_next_to_request,
+		  active_requests_cnt() + 1, m_sorted_secondaries.size())
+	   << dendl;
+  m_next_to_request++;
+  return false;
 }
 
 void ReplicaReservations::verify_rejections_source(
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index 634e7e580027..a603c7073563 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -79,8 +79,11 @@ class ReplicaReservations {
    * the replica we are expecting a reply from) is noted, and triggers
    * one of two: either sending a reservation request to the next replica,
    * or notifying the scrubber that we have reserved them all.
+   *
+   * \returns true if there are no more replicas to send reservation requests
+   * (i.e., the scrubber should proceed to the next phase), false otherwise.
    */
-  void handle_reserve_grant(OpRequestRef op, pg_shard_t from);
+  bool handle_reserve_grant(OpRequestRef op, pg_shard_t from);
 
   /**
    * Verify that the sender of the received rejection is the replica we
@@ -105,6 +108,9 @@ class ReplicaReservations {
    */
   void discard_remote_reservations();
 
+  /// the only replica we are expecting a reply from
+  std::optional<pg_shard_t> get_last_sent() const;
+
   // note: 'public', as accessed via the 'standard' dout_prefix() macro
   std::ostream& gen_prefix(std::ostream& out, std::string fn) const;
 
@@ -112,17 +118,14 @@ class ReplicaReservations {
   /// send 'release' messages to all replicas we have managed to reserve
   void release_all();
 
-  /// the only replica we are expecting a reply from
-  std::optional<pg_shard_t> get_last_sent() const;
-
   /// The number of requests that have been sent (and not rejected) so far.
   size_t active_requests_cnt() const;
 
   /**
-   * Either send a reservation request to the next replica, or notify the
-   * scrubber that we have reserved all the replicas.
+   * Send a reservation request to the next replica.
+   * - if there are no more replicas to send requests to, return true
    */
-  void send_next_reservation_or_complete();
+  bool send_next_reservation_or_complete();
 };
 
 } // namespace Scrub
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 61be0bd5a62c..745ea2388b67 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -356,12 +356,6 @@ struct ScrubPgIF {
    */
   virtual void clear_pgscrub_state() = 0;
 
-  /**
-   *  triggers the 'RemotesReserved' (all replicas granted scrub resources)
-   *  state-machine event
-   */
-  virtual void send_remotes_reserved(epoch_t epoch_queued) = 0;
-
   virtual void cleanup_store(ObjectStore::Transaction* t) = 0;
 
   virtual bool get_store_errors(const scrub_ls_arg_t& arg,

From 9e5868f33b25cf11be42e615cb22e23d158a6c86 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 14 Oct 2023 07:55:42 -0500
Subject: [PATCH 0186/2492] osd/scrub: modify slow-replica-reply warning
 implementation

Merging the 'do once' functionality into the timeout duration.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_reservations.cc | 11 +++++++----
 src/osd/scrubber/scrub_reservations.h  |  5 +++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 4d187a2a644f..3194c9871d57 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -51,6 +51,10 @@ ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
   m_next_to_request = m_sorted_secondaries.cbegin();
   // send out the 1'st request (unless we have no replicas)
   send_next_reservation_or_complete();
+
+  m_slow_response_warn_timeout =
+      m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
+	  "osd_scrub_slow_reservation_response");
 }
 
 void ReplicaReservations::release_all()
@@ -100,15 +104,14 @@ bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
   auto elapsed = clock::now() - m_last_request_sent_at;
 
   // log a warning if the response was slow to arrive
-  auto warn_timeout = m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
-      "osd_scrub_slow_reservation_response");
-  if (!m_slow_response_warned && (elapsed > warn_timeout)) {
+  if ((m_slow_response_warn_timeout > 0ms) &&
+      (elapsed > m_slow_response_warn_timeout)) {
     dout(1) << fmt::format(
 		   "slow reservation response from {} ({}ms)", from,
 		   duration_cast<milliseconds>(elapsed).count())
 	    << dendl;
     // prevent additional warnings
-    m_slow_response_warned = true;
+    m_slow_response_warn_timeout = 0ms;
   }
   dout(10) << fmt::format(
 		  "granted by {} ({} of {}) in {}ms", from,
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index a603c7073563..1f2628579dcc 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -66,8 +66,9 @@ class ReplicaReservations {
   /// for logs, and for detecting slow peers
   clock::time_point m_last_request_sent_at;
 
-  /// used to prevent multiple "slow response" warnings
-  bool m_slow_response_warned{false};
+  /// the 'slow response' timeout (in milliseconds) - as configured.
+  /// Doubles as a 'do once' flag for the warning.
+  std::chrono::milliseconds m_slow_response_warn_timeout;
 
  public:
   ReplicaReservations(ScrubMachineListener& scrubber);

From 09aa83426c3446dac33205083c36e890db5d0a73 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Sat, 14 Oct 2023 16:50:07 -0400
Subject: [PATCH 0187/2492] rgw: cleanup from previous commit

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 src/rgw/vstart.sh.swift              | 1930 --------------------------
 src/test/test_perf_counters_cache.cc |    1 -
 2 files changed, 1931 deletions(-)
 delete mode 100755 src/rgw/vstart.sh.swift

diff --git a/src/rgw/vstart.sh.swift b/src/rgw/vstart.sh.swift
deleted file mode 100755
index 46e46da0e4a5..000000000000
--- a/src/rgw/vstart.sh.swift
+++ /dev/null
@@ -1,1930 +0,0 @@
-#!/usr/bin/env bash
-# -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*-
-# vim: softtabstop=4 shiftwidth=4 expandtab
-
-# abort on failure
-set -e
-
-quoted_print() {
-    for s in "$@"; do
-        if [[ "$s" =~ \  ]]; then
-            printf -- "'%s' " "$s"
-        else
-            printf -- "$s "
-        fi
-    done
-    printf '\n'
-}
-
-debug() {
-  "$@" >&2
-}
-
-prunb() {
-    debug quoted_print "$@" '&'
-    PATH=$CEPH_BIN:$PATH "$@" &
-}
-
-prun() {
-    debug quoted_print "$@"
-    PATH=$CEPH_BIN:$PATH "$@"
-}
-
-
-if [ -n "$VSTART_DEST" ]; then
-    SRC_PATH=`dirname $0`
-    SRC_PATH=`(cd $SRC_PATH; pwd)`
-
-    CEPH_DIR=$SRC_PATH
-    CEPH_BIN=${CEPH_BIN:-${PWD}/bin}
-    CEPH_LIB=${CEPH_LIB:-${PWD}/lib}
-
-    CEPH_CONF_PATH=$VSTART_DEST
-    CEPH_DEV_DIR=$VSTART_DEST/dev
-    CEPH_OUT_DIR=$VSTART_DEST/out
-    CEPH_ASOK_DIR=$VSTART_DEST/asok
-    CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR}
-fi
-
-get_cmake_variable() {
-    local variable=$1
-    grep "${variable}:" CMakeCache.txt | cut -d "=" -f 2
-}
-
-# for running out of the CMake build directory
-if [ -e CMakeCache.txt ]; then
-    # Out of tree build, learn source location from CMakeCache.txt
-    CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR)
-    CEPH_BUILD_DIR=`pwd`
-    [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr
-fi
-
-# use CEPH_BUILD_ROOT to vstart from a 'make install'
-if [ -n "$CEPH_BUILD_ROOT" ]; then
-    [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin
-    [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib
-    [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_ROOT/external/lib
-    [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code
-    [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes
-    # make install should install python extensions into PYTHONPATH
-elif [ -n "$CEPH_ROOT" ]; then
-    [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL=$CEPH_ROOT/src/tools/cephfs/shell/cephfs-shell
-    [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind
-    [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin
-    [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph
-    [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
-    [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib
-    [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_DIR/external/lib
-    [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB
-    [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB
-    [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON=$CEPH_ROOT/src/python-common
-fi
-
-if [ -z "${CEPH_VSTART_WRAPPER}" ]; then
-    PATH=$(pwd):$PATH
-fi
-
-[ -z "$PYBIND" ] && PYBIND=./pybind
-
-[ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON="$CEPH_PYTHON_COMMON:"
-CYTHON_PYTHONPATH="$CEPH_LIB/cython_modules/lib.3"
-export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH
-
-export LD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$LD_LIBRARY_PATH
-export DYLD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$DYLD_LIBRARY_PATH
-# Suppress logging for regular use that indicated that we are using a
-# development version. vstart.sh is only used during testing and
-# development
-export CEPH_DEV=1
-
-[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON"
-[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD"
-[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS"
-[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR"
-[ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS="$FS"
-[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW"
-[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM="$NFS"
-
-# if none of the CEPH_NUM_* number is specified, kill the existing
-# cluster.
-if [ -z "$CEPH_NUM_MON" -a \
-     -z "$CEPH_NUM_OSD" -a \
-     -z "$CEPH_NUM_MDS" -a \
-     -z "$CEPH_NUM_MGR" -a \
-     -z "$GANESHA_DAEMON_NUM" ]; then
-    kill_all=1
-else
-    kill_all=0
-fi
-
-[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
-[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3
-[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3
-[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1
-[ -z "$CEPH_NUM_FS"  ] && CEPH_NUM_FS=1
-[ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1
-[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0
-[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM=0
-
-[ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD"
-[ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev"
-[ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out"
-[ -z "$CEPH_ASOK_DIR" ] && CEPH_ASOK_DIR="$CEPH_DIR/asok"
-[ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000
-[ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR
-CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR}
-
-if [ $CEPH_NUM_OSD -gt 3 ]; then
-    OSD_POOL_DEFAULT_SIZE=3
-else
-    OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD
-fi
-
-extra_conf=""
-new=0
-standby=0
-debug=0
-trace=0
-ip=""
-nodaemon=0
-redirect=0
-smallmds=0
-short=0
-crimson=0
-ec=0
-cephadm=0
-parallel=true
-restart=1
-hitset=""
-overwrite_conf=0
-cephx=1 #turn cephx on by default
-gssapi_authx=0
-cache=""
-if [ `uname` = FreeBSD ]; then
-    objectstore="memstore"
-else
-    objectstore="bluestore"
-fi
-ceph_osd=ceph-osd
-rgw_frontend="beast prefix=/swift"
-rgw_compression=""
-lockdep=${LOCKDEP:-1}
-spdk_enabled=0 # disable SPDK by default
-pmem_enabled=0
-zoned_enabled=0
-io_uring_enabled=0
-with_jaeger=0
-
-with_mgr_dashboard=true
-if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
-   [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then
-    debug echo "ceph-mgr dashboard not built - disabling."
-    with_mgr_dashboard=false
-fi
-with_mgr_restful=false
-
-kstore_path=
-declare -a block_devs
-declare -a bluestore_db_devs
-declare -a bluestore_wal_devs
-declare -a secondary_block_devs
-secondary_block_devs_type="SSD"
-
-VSTART_SEC="client.vstart.sh"
-
-MON_ADDR=""
-DASH_URLS=""
-RESTFUL_URLS=""
-
-conf_fn="$CEPH_CONF_PATH/ceph.conf"
-keyring_fn="$CEPH_CONF_PATH/keyring"
-monmap_fn="/tmp/ceph_monmap.$$"
-inc_osd_num=0
-
-msgr="21"
-
-read -r -d '' usage <<EOF || true
-usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 $0 -n -d
-options:
-	-d, --debug
-	-t, --trace
-	-s, --standby_mds: Generate standby-replay MDS for each active
-	-l, --localhost: use localhost instead of hostname
-	-i <ip>: bind to specific ip
-	-n, --new
-	--valgrind[_{osd,mds,mon,rgw}] 'toolname args...'
-	--nodaemon: use ceph-run as wrapper for mon/osd/mds
-	--redirect-output: only useful with nodaemon, directs output to log file
-	--smallmds: limit mds cache memory limit
-	-m ip:port		specify monitor address
-	-k keep old configuration files (default)
-	-x enable cephx (on by default)
-	-X disable cephx
-	-g --gssapi enable Kerberos/GSSApi authentication
-	-G disable Kerberos/GSSApi authentication
-	--hitset <pool> <hit_set_type>: enable hitset tracking
-	-e : create an erasure pool
-	-o config add extra config parameters to all sections
-	--rgw_port specify ceph rgw http listen port
-	--rgw_frontend specify the rgw frontend configuration
-	--rgw_arrow_flight start arrow flight frontend
-	--rgw_compression specify the rgw compression plugin
-	--seastore use seastore as crimson osd backend
-	-b, --bluestore use bluestore as the osd objectstore backend (default)
-	-K, --kstore use kstore as the osd objectstore backend
-	--cyanstore use cyanstore as the osd objectstore backend
-	--memstore use memstore as the osd objectstore backend
-	--cache <pool>: enable cache tiering on pool
-	--short: short object names only; necessary for ext4 dev
-	--nolockdep disable lockdep
-	--multimds <count> allow multimds with maximum active count
-	--without-dashboard: do not run using mgr dashboard
-	--bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0)
-	--bluestore-pmem: enable PMEM and with path to a file mapped to PMEM
-	--msgr1: use msgr1 only
-	--msgr2: use msgr2 only
-	--msgr21: use msgr2 and msgr1
-	--crimson: use crimson-osd instead of ceph-osd
-	--crimson-foreground: use crimson-osd, but run it in the foreground
-	--osd-args: specify any extra osd specific options
-	--bluestore-devs: comma-separated list of blockdevs to use for bluestore
-	--bluestore-db-devs: comma-separated list of db-devs to use for bluestore
-	--bluestore-wal-devs: comma-separated list of wal-devs to use for bluestore
-	--bluestore-zoned: blockdevs listed by --bluestore-devs are zoned devices (HM-SMR HDD or ZNS SSD)
-	--bluestore-io-uring: enable io_uring backend
-	--inc-osd: append some more osds into existing vcluster
-	--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]
-	--no-parallel: dont start all OSDs in parallel
-	--no-restart: dont restart process when using ceph-run
-	--jaeger: use jaegertracing for tracing
-	--seastore-devs: comma-separated list of blockdevs to use for seastore
-	--seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore
-	--seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
-	--crimson-smp: number of cores to use for crimson
-\n
-EOF
-
-usage_exit() {
-    printf "$usage"
-    exit
-}
-
-parse_block_devs() {
-    local opt_name=$1
-    shift
-    local devs=$1
-    shift
-    local dev
-    IFS=',' read -r -a block_devs <<< "$devs"
-    for dev in "${block_devs[@]}"; do
-        if [ ! -b $dev ] || [ ! -w $dev ]; then
-            echo "All $opt_name must refer to writable block devices"
-            exit 1
-        fi
-    done
-}
-
-parse_bluestore_db_devs() {
-    local opt_name=$1
-    shift
-    local devs=$1
-    shift
-    local dev
-    IFS=',' read -r -a bluestore_db_devs <<< "$devs"
-    for dev in "${bluestore_db_devs[@]}"; do
-        if [ ! -b $dev ] || [ ! -w $dev ]; then
-            echo "All $opt_name must refer to writable block devices"
-            exit 1
-        fi
-    done
-}
-
-parse_bluestore_wal_devs() {
-    local opt_name=$1
-    shift
-    local devs=$1
-    shift
-    local dev
-    IFS=',' read -r -a bluestore_wal_devs <<< "$devs"
-    for dev in "${bluestore_wal_devs[@]}"; do
-        if [ ! -b $dev ] || [ ! -w $dev ]; then
-            echo "All $opt_name must refer to writable block devices"
-            exit 1
-        fi
-    done
-}
-
-parse_secondary_devs() {
-    local opt_name=$1
-    shift
-    local devs=$1
-    shift
-    local dev
-    IFS=',' read -r -a secondary_block_devs <<< "$devs"
-    for dev in "${secondary_block_devs[@]}"; do
-        if [ ! -b $dev ] || [ ! -w $dev ]; then
-            echo "All $opt_name must refer to writable block devices"
-            exit 1
-        fi
-    done
-}
-
-crimson_smp=1
-while [ $# -ge 1 ]; do
-case $1 in
-    -d | --debug)
-        debug=1
-        ;;
-    -t | --trace)
-        trace=1
-        ;;
-    -s | --standby_mds)
-        standby=1
-        ;;
-    -l | --localhost)
-        ip="127.0.0.1"
-        ;;
-    -i)
-        [ -z "$2" ] && usage_exit
-        ip="$2"
-        shift
-        ;;
-    -e)
-        ec=1
-        ;;
-    --new | -n)
-        new=1
-        ;;
-    --inc-osd)
-        new=0
-        kill_all=0
-        inc_osd_num=$2
-        if [ "$inc_osd_num" == "" ]; then
-            inc_osd_num=1
-        else
-            shift
-        fi
-        ;;
-    --short)
-        short=1
-        ;;
-    --crimson)
-        crimson=1
-        ceph_osd=crimson-osd
-        nodaemon=1
-        msgr=2
-        ;;
-    --crimson-foreground)
-        crimson=1
-        ceph_osd=crimson-osd
-        nodaemon=0
-        msgr=2
-        ;;
-    --osd-args)
-        extra_osd_args="$2"
-        shift
-        ;;
-    --msgr1)
-        msgr="1"
-        ;;
-    --msgr2)
-        msgr="2"
-        ;;
-    --msgr21)
-        msgr="21"
-        ;;
-    --cephadm)
-        cephadm=1
-        ;;
-    --no-parallel)
-        parallel=false
-        ;;
-    --no-restart)
-        restart=0
-        ;;
-    --valgrind)
-        [ -z "$2" ] && usage_exit
-        valgrind=$2
-        shift
-        ;;
-    --valgrind_args)
-        valgrind_args="$2"
-        shift
-        ;;
-    --valgrind_mds)
-        [ -z "$2" ] && usage_exit
-        valgrind_mds=$2
-        shift
-        ;;
-    --valgrind_osd)
-        [ -z "$2" ] && usage_exit
-        valgrind_osd=$2
-        shift
-        ;;
-    --valgrind_mon)
-        [ -z "$2" ] && usage_exit
-        valgrind_mon=$2
-        shift
-        ;;
-    --valgrind_mgr)
-        [ -z "$2" ] && usage_exit
-        valgrind_mgr=$2
-        shift
-        ;;
-    --valgrind_rgw)
-        [ -z "$2" ] && usage_exit
-        valgrind_rgw=$2
-        shift
-        ;;
-    --nodaemon)
-        nodaemon=1
-        ;;
-    --redirect-output)
-        redirect=1
-        ;;
-    --smallmds)
-        smallmds=1
-        ;;
-    --rgw_port)
-        CEPH_RGW_PORT=$2
-        shift
-        ;;
-    --rgw_frontend)
-        rgw_frontend=$2
-        shift
-        ;;
-    --rgw_arrow_flight)
-        rgw_flight_frontend="yes"
-        ;;
-    --rgw_compression)
-        rgw_compression=$2
-        shift
-        ;;
-    --kstore_path)
-        kstore_path=$2
-        shift
-        ;;
-    -m)
-        [ -z "$2" ] && usage_exit
-        MON_ADDR=$2
-        shift
-        ;;
-    -x)
-        cephx=1 # this is on be default, flag exists for historical consistency
-        ;;
-    -X)
-        cephx=0
-        ;;
-
-    -g | --gssapi)
-        gssapi_authx=1
-        ;;
-    -G)
-        gssapi_authx=0
-        ;;
-
-    -k)
-        if [ ! -r $conf_fn ]; then
-            echo "cannot use old configuration: $conf_fn not readable." >&2
-            exit
-        fi
-        new=0
-        ;;
-    --memstore)
-        objectstore="memstore"
-        ;;
-    --cyanstore)
-        objectstore="cyanstore"
-        ;;
-    --seastore)
-        objectstore="seastore"
-        ;;
-    -b | --bluestore)
-        objectstore="bluestore"
-        ;;
-    -K | --kstore)
-        objectstore="kstore"
-        ;;
-    --hitset)
-        hitset="$hitset $2 $3"
-        shift
-        shift
-        ;;
-    -o)
-        extra_conf+=$'\n'"$2"
-        shift
-        ;;
-    --cache)
-        if [ -z "$cache" ]; then
-            cache="$2"
-        else
-            cache="$cache $2"
-        fi
-        shift
-        ;;
-    --nolockdep)
-        lockdep=0
-        ;;
-    --multimds)
-        CEPH_MAX_MDS="$2"
-        shift
-        ;;
-    --without-dashboard)
-        with_mgr_dashboard=false
-        ;;
-    --with-restful)
-        with_mgr_restful=true
-        ;;
-    --seastore-devs)
-        parse_block_devs --seastore-devs "$2"
-        shift
-        ;;
-    --seastore-secondary-devs)
-        parse_secondary_devs --seastore-devs "$2"
-        shift
-        ;;
-    --seastore-secondary-devs-type)
-        secondary_block_devs_type="$2"
-        shift
-        ;;
-    --crimson-smp)
-        crimson_smp=$2
-        shift
-        ;;
-    --bluestore-spdk)
-        [ -z "$2" ] && usage_exit
-        IFS=',' read -r -a bluestore_spdk_dev <<< "$2"
-        spdk_enabled=1
-        shift
-        ;;
-    --bluestore-pmem)
-        [ -z "$2" ] && usage_exit
-        bluestore_pmem_file="$2"
-        pmem_enabled=1
-        shift
-        ;;
-    --bluestore-devs)
-        parse_block_devs --bluestore-devs "$2"
-        shift
-        ;;
-    --bluestore-db-devs)
-        parse_bluestore_db_devs --bluestore-db-devs "$2"
-        shift
-        ;;
-    --bluestore-wal-devs)
-        parse_bluestore_wal_devs --bluestore-wal-devs "$2"
-        shift
-        ;;
-    --bluestore-zoned)
-        zoned_enabled=1
-        ;;
-    --bluestore-io-uring)
-        io_uring_enabled=1
-        shift
-        ;;
-    --jaeger)
-        with_jaeger=1
-        echo "with_jaeger $with_jaeger"
-        ;;
-    *)
-        usage_exit
-esac
-shift
-done
-
-if [ $kill_all -eq 1 ]; then
-    $SUDO $INIT_CEPH stop
-fi
-
-if [ "$new" -eq 0 ]; then
-    if [ -z "$CEPH_ASOK_DIR" ]; then
-        CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
-    fi
-    mkdir -p $CEPH_ASOK_DIR
-    MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \
-        CEPH_NUM_MON="$MON"
-    OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \
-        CEPH_NUM_OSD="$OSD"
-    MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \
-        CEPH_NUM_MDS="$MDS"
-    MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \
-        CEPH_NUM_MGR="$MGR"
-    RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \
-        CEPH_NUM_RGW="$RGW"
-    NFS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \
-        GANESHA_DAEMON_NUM="$NFS"
-else
-    # only delete if -n
-    if [ -e "$conf_fn" ]; then
-        asok_dir=`dirname $($CEPH_BIN/ceph-conf  -c $conf_fn --show-config-value admin_socket)`
-        rm -- "$conf_fn"
-        if [ $asok_dir != /var/run/ceph ]; then
-            [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir
-        fi
-    fi
-    if [ -z "$CEPH_ASOK_DIR" ]; then
-        CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"`
-    fi
-fi
-
-ARGS="-c $conf_fn"
-
-run() {
-    type=$1
-    shift
-    num=$1
-    shift
-    eval "valg=\$valgrind_$type"
-    [ -z "$valg" ] && valg="$valgrind"
-
-    if [ -n "$valg" ]; then
-        prunb valgrind --tool="$valg" $valgrind_args "$@" -f
-        sleep 1
-    else
-        if [ "$nodaemon" -eq 0 ]; then
-            prun "$@"
-        else
-            if [ "$restart" -eq 0 ]; then
-                set -- '--no-restart' "$@"
-            fi
-            if [ "$redirect" -eq 0 ]; then
-                prunb ${CEPH_ROOT}/src/ceph-run "$@" -f
-            else
-                ( prunb ${CEPH_ROOT}/src/ceph-run "$@" -f ) >$CEPH_OUT_DIR/$type.$num.stdout 2>&1
-            fi
-        fi
-    fi
-}
-
-wconf() {
-    if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then
-        cat >> "$conf_fn"
-    fi
-}
-
-
-do_rgw_conf() {
-
-    if [ $CEPH_NUM_RGW -eq 0 ]; then
-        return 0
-    fi
-
-    # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT.
-    # individual rgw's ids will be their ports.
-    current_port=$CEPH_RGW_PORT
-    # allow only first rgw to start arrow_flight server/port
-    local flight_conf=$rgw_flight_frontend
-    for n in $(seq 1 $CEPH_NUM_RGW); do
-        wconf << EOF
-[client.rgw.${current_port}]
-        rgw frontends = $rgw_frontend port=${current_port}${flight_conf:+,arrow_flight}
-        admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok
-        debug rgw_flight = 20
-        rgw keystone accepted admin roles = admin
-        rgw keystone accepted roles = admin,Member
-        rgw keystone admin domain = Default
-        rgw keystone admin password = ADMIN
-        rgw keystone admin project = admin
-        rgw keystone admin user = admin
-        rgw keystone api version = 3
-        rgw keystone implicit tenants = true
-        rgw swift account in url = true
-        rgw swift enforce content length = true
-        rgw swift versioning enabled = true
-EOF
-        current_port=$((current_port + 1))
-        unset flight_conf
-done
-
-}
-
-format_conf() {
-    local opts=$1
-    local indent="        "
-    local opt
-    local formatted
-    while read -r opt; do
-        if [ -z "$formatted" ]; then
-            formatted="${opt}"
-        else
-            formatted+=$'\n'${indent}${opt}
-        fi
-    done <<< "$opts"
-    echo "$formatted"
-}
-
-prepare_conf() {
-    local DAEMONOPTS="
-        log file = $CEPH_OUT_DIR/\$name.log
-        admin socket = $CEPH_ASOK_DIR/\$name.asok
-        chdir = \"\"
-        pid file = $CEPH_OUT_DIR/\$name.pid
-        heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat
-"
-
-    local mgr_modules="iostat nfs"
-    if $with_mgr_dashboard; then
-        mgr_modules+=" dashboard"
-    fi
-    if $with_mgr_restful; then
-        mgr_modules+=" restful"
-    fi
-
-    local msgr_conf=''
-    if [ $msgr -eq 21 ]; then
-        msgr_conf="ms bind msgr2 = true
-                   ms bind msgr1 = true"
-    fi
-    if [ $msgr -eq 2 ]; then
-        msgr_conf="ms bind msgr2 = true
-                   ms bind msgr1 = false"
-    fi
-    if [ $msgr -eq 1 ]; then
-        msgr_conf="ms bind msgr2 = false
-                   ms bind msgr1 = true"
-    fi
-
-    wconf <<EOF
-; generated by vstart.sh on `date`
-[$VSTART_SEC]
-        num mon = $CEPH_NUM_MON
-        num osd = $CEPH_NUM_OSD
-        num mds = $CEPH_NUM_MDS
-        num mgr = $CEPH_NUM_MGR
-        num rgw = $CEPH_NUM_RGW
-        num ganesha = $GANESHA_DAEMON_NUM
-
-[global]
-        fsid = $(uuidgen)
-        osd failsafe full ratio = .99
-        mon osd full ratio = .99
-        mon osd nearfull ratio = .99
-        mon osd backfillfull ratio = .99
-        mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000}
-        erasure code dir = $EC_PATH
-        plugin dir = $CEPH_LIB
-        run dir = $CEPH_OUT_DIR
-        crash dir = $CEPH_OUT_DIR
-        enable experimental unrecoverable data corrupting features = *
-        osd_crush_chooseleaf_type = 0
-        debug asok assert abort = true
-        $(format_conf "${msgr_conf}")
-        $(format_conf "${extra_conf}")
-        $AUTOSCALER_OPTS
-EOF
-    if [ "$with_jaeger" -eq 1 ] ; then
-        wconf <<EOF
-        jaeger_agent_port = 6831
-EOF
-    fi
-    if [ "$lockdep" -eq 1 ] ; then
-        wconf <<EOF
-        lockdep = true
-EOF
-    fi
-    if [ "$cephx" -eq 1 ] ; then
-        wconf <<EOF
-        auth cluster required = cephx
-        auth service required = cephx
-        auth client required = cephx
-EOF
-    elif [ "$gssapi_authx" -eq 1 ] ; then
-        wconf <<EOF
-        auth cluster required = gss
-        auth service required = gss
-        auth client required = gss
-        gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab
-EOF
-    else
-        wconf <<EOF
-        auth cluster required = none
-        auth service required = none
-        auth client required = none
-        ms mon client mode = crc
-EOF
-    fi
-    if [ "$short" -eq 1 ]; then
-        COSDSHORT="        osd max object name len = 460
-        osd max object namespace len = 64"
-    fi
-    if [ "$objectstore" == "bluestore" ]; then
-        if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then
-            BLUESTORE_OPTS="        bluestore_block_db_path = \"\"
-        bluestore_block_db_size = 0
-        bluestore_block_db_create = false
-        bluestore_block_wal_path = \"\"
-        bluestore_block_wal_size = 0
-        bluestore_block_wal_create = false
-        bluestore_spdk_mem = 2048"
-        else
-            BLUESTORE_OPTS="        bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file
-        bluestore block db size = 1073741824
-        bluestore block db create = true
-        bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file
-        bluestore block wal size = 1048576000
-        bluestore block wal create = true"
-            if [ ${#block_devs[@]} -gt 0 ] || \
-               [ ${#bluestore_db_devs[@]} -gt 0 ] || \
-               [ ${#bluestore_wal_devs[@]} -gt 0 ]; then
-                # when use physical disk, not create file for db/wal
-                BLUESTORE_OPTS=""
-            fi
-        fi
-        if [ "$zoned_enabled" -eq 1 ]; then
-            BLUESTORE_OPTS+="
-        bluestore min alloc size = 65536
-        bluestore prefer deferred size = 0
-        bluestore prefer deferred size hdd = 0
-        bluestore prefer deferred size ssd = 0
-        bluestore allocator = zoned"
-        fi
-        if [ "$io_uring_enabled" -eq 1 ]; then
-            BLUESTORE_OPTS+="
-        bdev ioring = true"
-        fi
-    fi
-    wconf <<EOF
-[client]
-$CCLIENTDEBUG
-        keyring = $keyring_fn
-        log file = $CEPH_OUT_CLIENT_DIR/\$name.\$pid.log
-        admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok
-
-        ; needed for s3tests
-        rgw crypt s3 kms backend = testing
-        rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
-        rgw crypt require ssl = false
-        ; uncomment the following to set LC days as the value in seconds;
-        ; needed for passing lc time based s3-tests (can be verbose)
-        ; rgw lc debug interval = 10
-        $(format_conf "${extra_conf}")
-EOF
-	do_rgw_conf
-	wconf << EOF
-[mds]
-$CMDSDEBUG
-$DAEMONOPTS
-        mds data = $CEPH_DEV_DIR/mds.\$id
-        mds root ino uid = `id -u`
-        mds root ino gid = `id -g`
-        $(format_conf "${extra_conf}")
-[mgr]
-        mgr disabled modules = rook
-        mgr data = $CEPH_DEV_DIR/mgr.\$id
-        mgr module path = $MGR_PYTHON_PATH
-        cephadm path = $CEPH_BIN/cephadm
-$DAEMONOPTS
-        $(format_conf "${extra_conf}")
-[osd]
-$DAEMONOPTS
-        osd_check_max_object_name_len_on_startup = false
-        osd data = $CEPH_DEV_DIR/osd\$id
-        osd journal = $CEPH_DEV_DIR/osd\$id/journal
-        osd journal size = 100
-        osd class tmp = out
-        osd class dir = $OBJCLASS_PATH
-        osd class load list = *
-        osd class default list = *
-        osd fast shutdown = false
-
-        bluestore fsck on mount = true
-        bluestore block create = true
-$BLUESTORE_OPTS
-
-        ; kstore
-        kstore fsck on mount = true
-        osd objectstore = $objectstore
-$COSDSHORT
-        $(format_conf "${extra_conf}")
-[mon]
-        mon_data_avail_crit = 1
-        mgr initial modules = $mgr_modules
-$DAEMONOPTS
-$CMONDEBUG
-        $(format_conf "${extra_conf}")
-        mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log
-        osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd
-        auth allow insecure global id reclaim = false
-EOF
-
-    if [ "$crimson" -eq 1 ]; then
-        wconf <<EOF
-        osd pool default crimson = true
-EOF
-    fi
-}
-
-write_logrotate_conf() {
-    out_dir=$(pwd)"/out/*.log"
-
-    cat << EOF
-$out_dir
-{
-    rotate 5
-    size 1G
-    copytruncate
-    compress
-    notifempty
-    missingok
-    sharedscripts
-    postrotate
-        # NOTE: assuring that the absence of one of the following processes
-        # won't abort the logrotate command.
-        killall -u $USER -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror || echo ""
-    endscript
-}
-EOF
-}
-
-init_logrotate() {
-    logrotate_conf_path=$(pwd)"/logrotate.conf"
-    logrotate_state_path=$(pwd)"/logrotate.state"
-
-    if ! test -a $logrotate_conf_path; then
-        if test -a $logrotate_state_path; then
-            rm -f $logrotate_state_path
-        fi
-        write_logrotate_conf > $logrotate_conf_path
-    fi
-}
-
-start_mon() {
-    local MONS=""
-    local count=0
-    for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
-    do
-        [ $count -eq $CEPH_NUM_MON ] && break;
-        count=$(($count + 1))
-        if [ -z "$MONS" ]; then
-	    MONS="$f"
-        else
-	    MONS="$MONS $f"
-        fi
-    done
-
-    if [ "$new" -eq 1 ]; then
-        if [ `echo $IP | grep '^127\\.'` ]; then
-            echo
-            echo "NOTE: hostname resolves to loopback; remote hosts will not be able to"
-            echo "  connect.  either adjust /etc/hosts, or edit this script to use your"
-            echo "  machine's real IP."
-            echo
-        fi
-
-        prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *'
-        prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \
-             --cap mon 'allow *' \
-             --cap osd 'allow *' \
-             --cap mds 'allow *' \
-             --cap mgr 'allow *' \
-             "$keyring_fn"
-
-        # build a fresh fs monmap, mon fs
-        local params=()
-        local count=0
-        local mon_host=""
-        for f in $MONS
-        do
-            if [ $msgr -eq 1 ]; then
-                A="v1:$IP:$(($CEPH_PORT+$count+1))"
-            fi
-            if [ $msgr -eq 2 ]; then
-                A="v2:$IP:$(($CEPH_PORT+$count+1))"
-            fi
-            if [ $msgr -eq 21 ]; then
-                A="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]"
-            fi
-            params+=("--addv" "$f" "$A")
-            mon_host="$mon_host $A"
-            wconf <<EOF
-[mon.$f]
-        host = $HOSTNAME
-        mon data = $CEPH_DEV_DIR/mon.$f
-EOF
-            count=$(($count + 2))
-        done
-        wconf <<EOF
-[global]
-        mon host = $mon_host
-EOF
-        prun "$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn"
-
-        for f in $MONS
-        do
-            prun rm -rf -- "$CEPH_DEV_DIR/mon.$f"
-            prun mkdir -p "$CEPH_DEV_DIR/mon.$f"
-            prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn"
-        done
-
-        prun rm -- "$monmap_fn"
-    fi
-
-    # start monitors
-    for f in $MONS
-    do
-        run 'mon' $f $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
-    done
-
-    if [ "$crimson" -eq 1 ]; then
-        $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it
-    fi
-}
-
-start_osd() {
-    if [ $inc_osd_num -gt 0 ]; then
-        old_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
-        start=$old_maxosd
-        end=$(($start-1+$inc_osd_num))
-        overwrite_conf=1 # fake wconf
-    else
-        start=0
-        end=$(($CEPH_NUM_OSD-1))
-    fi
-    local osds_wait
-    for osd in `seq $start $end`
-    do
-	local extra_seastar_args
-	if [ "$ceph_osd" == "crimson-osd" ]; then
-        bottom_cpu=$(( osd * crimson_smp ))
-        top_cpu=$(( bottom_cpu + crimson_smp - 1 ))
-	    # set a single CPU nodes for each osd
-	    extra_seastar_args="--cpuset $bottom_cpu-$top_cpu"
-	    if [ "$debug" -ne 0 ]; then
-		extra_seastar_args+=" --debug"
-	    fi
-            if [ "$trace" -ne 0 ]; then
-                extra_seastar_args+=" --trace"
-            fi
-	fi
-	if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
-            wconf <<EOF
-[osd.$osd]
-        host = $HOSTNAME
-EOF
-            if [ "$spdk_enabled" -eq 1 ]; then
-                wconf <<EOF
-        bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}
-EOF
-            elif [ "$pmem_enabled" -eq 1 ]; then
-                wconf <<EOF
-        bluestore_block_path = ${bluestore_pmem_file}
-EOF
-            fi
-            rm -rf $CEPH_DEV_DIR/osd$osd || true
-            if command -v btrfs > /dev/null; then
-                for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done
-            fi
-            if [ -n "$kstore_path" ]; then
-                ln -s $kstore_path $CEPH_DEV_DIR/osd$osd
-            else
-                mkdir -p $CEPH_DEV_DIR/osd$osd
-                if [ -n "${block_devs[$osd]}" ]; then
-                    dd if=/dev/zero of=${block_devs[$osd]} bs=1M count=1
-                    ln -s ${block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block
-                fi
-                if [ -n "${bluestore_db_devs[$osd]}" ]; then
-                    dd if=/dev/zero of=${bluestore_db_devs[$osd]} bs=1M count=1
-                    ln -s ${bluestore_db_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.db
-                fi
-                if [ -n "${bluestore_wal_devs[$osd]}" ]; then
-                    dd if=/dev/zero of=${bluestore_wal_devs[$osd]} bs=1M count=1
-                    ln -s ${bluestore_wal_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.wal
-                fi
-                if [ -n "${secondary_block_devs[$osd]}" ]; then
-                    dd if=/dev/zero of=${secondary_block_devs[$osd]} bs=1M count=1
-                    mkdir -p $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1
-                    ln -s ${secondary_block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1/block
-                fi
-            fi
-            if [ "$objectstore" == "bluestore" ]; then
-                wconf <<EOF
-        bluestore fsck on mount = false
-EOF
-            fi
-
-            local uuid=`uuidgen`
-            echo "add osd$osd $uuid"
-            OSD_SECRET=$($CEPH_BIN/ceph-authtool --gen-print-key)
-            echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json
-            ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json
-            rm $CEPH_DEV_DIR/osd$osd/new.json
-            prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \
-                2>&1 | tee $CEPH_OUT_DIR/osd-mkfs.$osd.log
-
-            local key_fn=$CEPH_DEV_DIR/osd$osd/keyring
-            cat > $key_fn<<EOF
-[osd.$osd]
-        key = $OSD_SECRET
-EOF
-        fi
-        echo start osd.$osd
-        local osd_pid
-        echo 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
-            $extra_seastar_args $extra_osd_args \
-            -i $osd $ARGS $COSD_ARGS
-        run 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \
-            $extra_seastar_args $extra_osd_args \
-            -i $osd $ARGS $COSD_ARGS &
-        osd_pid=$!
-        if $parallel; then
-            osds_wait=$osd_pid
-        else
-            wait $osd_pid
-        fi
-    done
-    if $parallel; then
-        for p in $osds_wait; do
-            wait $p
-        done
-        debug echo OSDs started
-    fi
-    if [ $inc_osd_num -gt 0 ]; then
-        # update num osd
-        new_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
-        sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn
-    fi
-}
-
-create_mgr_restful_secret() {
-    while ! ceph_adm -h | grep -c -q ^restful ; do
-        debug echo 'waiting for mgr restful module to start'
-        sleep 1
-    done
-    local secret_file
-    if ceph_adm restful create-self-signed-cert > /dev/null; then
-        secret_file=`mktemp`
-        ceph_adm restful create-key admin -o $secret_file
-        RESTFUL_SECRET=`cat $secret_file`
-        rm $secret_file
-    else
-        debug echo MGR Restful is not working, perhaps the package is not installed?
-    fi
-}
-
-start_mgr() {
-    local mgr=0
-    local ssl=${DASHBOARD_SSL:-1}
-    # avoid monitors on nearby ports (which test/*.sh use extensively)
-    MGR_PORT=$(($CEPH_PORT + 1000))
-    PROMETHEUS_PORT=9283
-    for name in x y z a b c d e f g h i j k l m n o p
-    do
-        [ $mgr -eq $CEPH_NUM_MGR ] && break
-        mgr=$(($mgr + 1))
-        if [ "$new" -eq 1 ]; then
-            mkdir -p $CEPH_DEV_DIR/mgr.$name
-            key_fn=$CEPH_DEV_DIR/mgr.$name/keyring
-            $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn
-            ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *'
-
-            wconf <<EOF
-[mgr.$name]
-        host = $HOSTNAME
-EOF
-
-            if $with_mgr_dashboard ; then
-                local port_option="ssl_server_port"
-                local http_proto="https"
-                if [ "$ssl" == "0" ]; then
-                    port_option="server_port"
-                    http_proto="http"
-                    ceph_adm config set mgr mgr/dashboard/ssl false --force
-                fi
-                ceph_adm config set mgr mgr/dashboard/$name/$port_option $MGR_PORT --force
-                if [ $mgr -eq 1 ]; then
-                    DASH_URLS="$http_proto://$IP:$MGR_PORT"
-                else
-                    DASH_URLS+=", $http_proto://$IP:$MGR_PORT"
-                fi
-            fi
-	    MGR_PORT=$(($MGR_PORT + 1000))
-	    ceph_adm config set mgr mgr/prometheus/$name/server_port $PROMETHEUS_PORT --force
-	    PROMETHEUS_PORT=$(($PROMETHEUS_PORT + 1000))
-
-	    ceph_adm config set mgr mgr/restful/$name/server_port $MGR_PORT --force
-            if [ $mgr -eq 1 ]; then
-                RESTFUL_URLS="https://$IP:$MGR_PORT"
-            else
-                RESTFUL_URLS+=", https://$IP:$MGR_PORT"
-            fi
-	    MGR_PORT=$(($MGR_PORT + 1000))
-        fi
-
-        debug echo "Starting mgr.${name}"
-        run 'mgr' $name $CEPH_BIN/ceph-mgr -i $name $ARGS
-    done
-
-    while ! ceph_adm mgr stat | jq -e '.available'; do
-        debug echo 'waiting for mgr to become available'
-        sleep 1
-    done
-    
-    if [ "$new" -eq 1 ]; then
-        # setting login credentials for dashboard
-        if $with_mgr_dashboard; then
-            while ! ceph_adm -h | grep -c -q ^dashboard ; do
-                debug echo 'waiting for mgr dashboard module to start'
-                sleep 1
-            done
-            DASHBOARD_ADMIN_SECRET_FILE="${CEPH_CONF_PATH}/dashboard-admin-secret.txt"
-            printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}"
-            ceph_adm dashboard ac-user-create admin -i "${DASHBOARD_ADMIN_SECRET_FILE}" \
-                administrator --force-password
-            if [ "$ssl" != "0" ]; then
-                if ! ceph_adm dashboard create-self-signed-cert;  then
-                    debug echo dashboard module not working correctly!
-                fi
-            fi
-        fi
-        if $with_mgr_restful; then
-            create_mgr_restful_secret
-        fi
-    fi
-
-    if [ "$cephadm" -eq 1 ]; then
-        debug echo Enabling cephadm orchestrator
-	if [ "$new" -eq 1 ]; then
-		digest=$(curl -s \
-		https://hub.docker.com/v2/repositories/ceph/daemon-base/tags/latest-master-devel \
-		| jq -r '.images[0].digest')
-		ceph_adm config set global container_image "docker.io/ceph/daemon-base@$digest"
-	fi
-        ceph_adm config-key set mgr/cephadm/ssh_identity_key -i ~/.ssh/id_rsa
-        ceph_adm config-key set mgr/cephadm/ssh_identity_pub -i ~/.ssh/id_rsa.pub
-        ceph_adm mgr module enable cephadm
-        ceph_adm orch set backend cephadm
-        ceph_adm orch host add "$(hostname)"
-        ceph_adm orch apply crash '*'
-        ceph_adm config set mgr mgr/cephadm/allow_ptrace true
-    fi
-}
-
-start_mds() {
-    local mds=0
-    for name in a b c d e f g h i j k l m n o p
-    do
-        [ $mds -eq $CEPH_NUM_MDS ] && break
-        mds=$(($mds + 1))
-
-        if [ "$new" -eq 1 ]; then
-            prun mkdir -p "$CEPH_DEV_DIR/mds.$name"
-            key_fn=$CEPH_DEV_DIR/mds.$name/keyring
-            wconf <<EOF
-[mds.$name]
-        host = $HOSTNAME
-EOF
-            if [ "$standby" -eq 1 ]; then
-                mkdir -p $CEPH_DEV_DIR/mds.${name}s
-                wconf <<EOF
-        mds standby for rank = $mds
-[mds.${name}s]
-        mds standby replay = true
-        mds standby for name = ${name}
-EOF
-            fi
-            prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn"
-            ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds 'allow' mgr 'allow profile mds'
-            if [ "$standby" -eq 1 ]; then
-                prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \
-                     "$CEPH_DEV_DIR/mds.${name}s/keyring"
-                ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \
-                             mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds'
-            fi
-        fi
-
-        run 'mds' $name $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
-        if [ "$standby" -eq 1 ]; then
-            run 'mds' $name $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
-        fi
-
-        #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
-        #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
-        #ceph_adm mds set max_mds 2
-    done
-
-    if [ $new -eq 1 ]; then
-        if [ "$CEPH_NUM_FS" -gt "0" ] ; then
-            sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation
-            if [ "$CEPH_NUM_FS" -gt "1" ] ; then
-                ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it
-            fi
-
-	    # wait for volume module to load
-	    while ! ceph_adm fs volume ls ; do sleep 1 ; done
-            local fs=0
-            for name in a b c d e f g h i j k l m n o p
-            do
-                ceph_adm fs volume create ${name}
-                ceph_adm fs authorize ${name} "client.fs_${name}" / rwp >> "$keyring_fn"
-                fs=$(($fs + 1))
-                [ $fs -eq $CEPH_NUM_FS ] && break
-            done
-        fi
-    fi
-
-}
-
-# Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace
-# nfs-ganesha-rados-urls (version 3.3 and above) packages installed. On
-# Fedora>=31 these packages can be installed directly with 'dnf'. For CentOS>=8
-# the packages are available at
-# https://wiki.centos.org/SpecialInterestGroup/Storage
-# Similarly for Ubuntu>=16.04 follow the instructions on
-# https://launchpad.net/~nfs-ganesha
-
-start_ganesha() {
-    cluster_id="vstart"
-    GANESHA_PORT=$(($CEPH_PORT + 4000))
-    local ganesha=0
-    test_user="$cluster_id"
-    pool_name=".nfs"
-    namespace=$cluster_id
-    url="rados://$pool_name/$namespace/conf-nfs.$test_user"
-
-    prun ceph_adm auth get-or-create client.$test_user \
-        mon "allow r" \
-        osd "allow rw pool=$pool_name namespace=$namespace, allow rw tag cephfs data=a" \
-        mds "allow rw path=/" \
-        >> "$keyring_fn"
-
-    ceph_adm mgr module enable test_orchestrator
-    ceph_adm orch set backend test_orchestrator
-    ceph_adm test_orchestrator load_data -i $CEPH_ROOT/src/pybind/mgr/test_orchestrator/dummy_data.json
-    prun ceph_adm nfs cluster create $cluster_id
-    prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path "/cephfs"
-
-    for name in a b c d e f g h i j k l m n o p
-    do
-        [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break
-
-        port=$(($GANESHA_PORT + ganesha))
-        ganesha=$(($ganesha + 1))
-        ganesha_dir="$CEPH_DEV_DIR/ganesha.$name"
-        prun rm -rf $ganesha_dir
-        prun mkdir -p $ganesha_dir
-
-        echo "NFS_CORE_PARAM {
-            Enable_NLM = false;
-            Enable_RQUOTA = false;
-            Protocols = 4;
-            NFS_Port = $port;
-        }
-
-        MDCACHE {
-           Dir_Chunk = 0;
-        }
-
-        NFSv4 {
-           RecoveryBackend = rados_cluster;
-           Minor_Versions = 1, 2;
-        }
-
-        RADOS_KV {
-           pool = '$pool_name';
-           namespace = $namespace;
-           UserId = $test_user;
-           nodeid = $name;
-        }
-
-        RADOS_URLS {
-	   Userid = $test_user;
-	   watch_url = '$url';
-        }
-
-	%url $url" > "$ganesha_dir/ganesha-$name.conf"
-	wconf <<EOF
-[ganesha.$name]
-        host = $HOSTNAME
-        ip = $IP
-        port = $port
-        ganesha data = $ganesha_dir
-        pid file = $CEPH_OUT_DIR/ganesha-$name.pid
-EOF
-
-        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace add $name
-        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
-
-        prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_DEBUG
-
-        # Wait few seconds for grace period to be removed
-        sleep 2
-
-        prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
-
-        echo "$test_user ganesha daemon $name started on port: $port"
-    done
-}
-
-if [ "$debug" -eq 0 ]; then
-    CMONDEBUG='
-        debug mon = 10
-        debug ms = 1'
-    CCLIENTDEBUG=''
-    CMDSDEBUG=''
-else
-    debug echo "** going verbose **"
-    CMONDEBUG='
-        debug osd = 20
-        debug mon = 20
-        debug osd = 20
-        debug paxos = 20
-        debug auth = 20
-        debug mgrc = 20
-        debug ms = 1'
-    CCLIENTDEBUG='
-        debug client = 20'
-    CMDSDEBUG='
-        debug mds = 20'
-fi
-
-# Crimson doesn't support PG merge/split yet.
-if [ "$ceph_osd" == "crimson-osd" ]; then
-    AUTOSCALER_OPTS='
-        osd_pool_default_pg_autoscale_mode = off'
-fi
-
-if [ -n "$MON_ADDR" ]; then
-    CMON_ARGS=" -m "$MON_ADDR
-    COSD_ARGS=" -m "$MON_ADDR
-    CMDS_ARGS=" -m "$MON_ADDR
-fi
-
-if [ -z "$CEPH_PORT" ]; then
-    while [ true ]
-    do
-        CEPH_PORT="$(echo $(( RANDOM % 1000 + 40000 )))"
-        ss -a -n | egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev/null 2>&1 || break
-    done
-fi
-
-[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph
-
-# sudo if btrfs
-[ -d $CEPH_DEV_DIR/osd0/. ] && [ -e $CEPH_DEV_DIR/sudo ] && SUDO="sudo"
-
-if [ $inc_osd_num -eq 0 ]; then
-    prun $SUDO rm -f core*
-fi
-
-[ -d $CEPH_ASOK_DIR ] || mkdir -p $CEPH_ASOK_DIR
-[ -d $CEPH_OUT_DIR  ] || mkdir -p $CEPH_OUT_DIR
-[ -d $CEPH_DEV_DIR  ] || mkdir -p $CEPH_DEV_DIR
-[ -d $CEPH_OUT_CLIENT_DIR ] || mkdir -p $CEPH_OUT_CLIENT_DIR
-if [ $inc_osd_num -eq 0 ]; then
-    $SUDO find "$CEPH_OUT_DIR" -type f -delete
-fi
-[ -d gmon ] && $SUDO rm -rf gmon/*
-
-[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn
-
-
-# figure machine's ip
-HOSTNAME=`hostname -s`
-if [ -n "$ip" ]; then
-    IP="$ip"
-else
-    echo hostname $HOSTNAME
-    if [ -x "$(which ip 2>/dev/null)" ]; then
-        IP_CMD="ip addr"
-    else
-        IP_CMD="ifconfig"
-    fi
-    # filter out IPv4 and localhost addresses
-    IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)"
-    # if nothing left, try using localhost address, it might work
-    if [ -z "$IP" ]; then IP="127.0.0.1"; fi
-fi
-echo "ip $IP"
-echo "port $CEPH_PORT"
-
-
-[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph
-
-ceph_adm() {
-    if [ "$cephx" -eq 1 ]; then
-        prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@"
-    else
-        prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@"
-    fi
-}
-
-if [ $inc_osd_num -gt 0 ]; then
-    start_osd
-    exit
-fi
-
-if [ "$new" -eq 1 ]; then
-    prepare_conf
-fi
-
-if [ $CEPH_NUM_MON -gt 0 ]; then
-    start_mon
-
-    debug echo Populating config ...
-    cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
-[global]
-osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE
-osd_pool_default_min_size = 1
-
-[mon]
-mon_osd_reporter_subtree_level = osd
-mon_data_avail_warn = 2
-mon_data_avail_crit = 1
-mon_allow_pool_delete = true
-mon_allow_pool_size_one = true
-
-[osd]
-osd_scrub_load_threshold = 2000
-osd_debug_op_order = true
-osd_debug_misdirected_ops = true
-osd_copyfrom_max_chunk = 524288
-
-[mds]
-mds_debug_frag = true
-mds_debug_auth_pins = true
-mds_debug_subtrees = true
-
-[mgr]
-mgr/telemetry/nag = false
-mgr/telemetry/enable = false
-
-EOF
-
-    if [ "$debug" -ne 0 ]; then
-        debug echo Setting debug configs ...
-        cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i -
-[mgr]
-debug_ms = 1
-debug_mgr = 20
-debug_monc = 20
-debug_mon = 20
-
-[osd]
-debug_ms = 1
-debug_osd = 25
-debug_objecter = 20
-debug_monc = 20
-debug_mgrc = 20
-debug_journal = 20
-debug_bluestore = 20
-debug_bluefs = 20
-debug_rocksdb = 20
-debug_bdev = 20
-debug_reserver = 10
-debug_objclass = 20
-
-[mds]
-debug_ms = 1
-debug_mds = 20
-debug_monc = 20
-debug_mgrc = 20
-mds_debug_scatterstat = true
-mds_verify_scatter = true
-EOF
-    fi
-    if [ "$cephadm" -gt 0 ]; then
-        debug echo Setting mon public_network ...
-        public_network=$(ip route list | grep -w "$IP" | awk '{print $1}')
-        ceph_adm config set mon public_network $public_network
-    fi
-fi
-
-if [ "$ceph_osd" == "crimson-osd" ]; then
-    $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp
-fi
-
-if [ $CEPH_NUM_MGR -gt 0 ]; then
-    start_mgr
-fi
-
-# osd
-if [ $CEPH_NUM_OSD -gt 0 ]; then
-    start_osd
-fi
-
-# mds
-if [ "$smallmds" -eq 1 ]; then
-    wconf <<EOF
-[mds]
-        mds log max segments = 2
-        # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB.
-        mds cache memory limit = 100M
-EOF
-fi
-
-if [ $CEPH_NUM_MDS -gt 0 ]; then
-    start_mds
-    # key with access to all FS
-    ceph_adm fs authorize \* "client.fs" / rwp >> "$keyring_fn"
-fi
-
-# Don't set max_mds until all the daemons are started, otherwise
-# the intended standbys might end up in active roles.
-if [ "$CEPH_MAX_MDS" -gt 1 ]; then
-    sleep 5  # wait for daemons to make it into FSMap before increasing max_mds
-fi
-fs=0
-for name in a b c d e f g h i j k l m n o p
-do
-    [ $fs -eq $CEPH_NUM_FS ] && break
-    fs=$(($fs + 1))
-    if [ "$CEPH_MAX_MDS" -gt 1 ]; then
-        ceph_adm fs set "${name}" max_mds "$CEPH_MAX_MDS"
-    fi
-done
-
-# mgr
-
-if [ "$ec" -eq 1 ]; then
-    ceph_adm <<EOF
-osd erasure-code-profile set ec-profile m=2 k=2
-osd pool create ec erasure ec-profile
-EOF
-fi
-
-do_cache() {
-    while [ -n "$*" ]; do
-        p="$1"
-        shift
-        debug echo "creating cache for pool $p ..."
-        ceph_adm <<EOF
-osd pool create ${p}-cache
-osd tier add $p ${p}-cache
-osd tier cache-mode ${p}-cache writeback
-osd tier set-overlay $p ${p}-cache
-EOF
-    done
-}
-do_cache $cache
-
-do_hitsets() {
-    while [ -n "$*" ]; do
-        pool="$1"
-        type="$2"
-        shift
-        shift
-        debug echo "setting hit_set on pool $pool type $type ..."
-        ceph_adm <<EOF
-osd pool set $pool hit_set_type $type
-osd pool set $pool hit_set_count 8
-osd pool set $pool hit_set_period 30
-EOF
-    done
-}
-do_hitsets $hitset
-
-do_rgw_create_bucket()
-{
-   # Create RGW Bucket
-   local rgw_python_file='rgw-create-bucket.py'
-   echo "import boto
-import boto.s3.connection
-
-conn = boto.connect_s3(
-        aws_access_key_id = '$s3_akey',
-        aws_secret_access_key = '$s3_skey',
-        host = '$HOSTNAME',
-        port = 80,
-        is_secure=False,
-        calling_format = boto.s3.connection.OrdinaryCallingFormat(),
-        )
-
-bucket = conn.create_bucket('nfs-bucket')
-print('created new bucket')" > "$CEPH_OUT_DIR/$rgw_python_file"
-   prun python $CEPH_OUT_DIR/$rgw_python_file
-}
-
-do_rgw_create_users()
-{
-    # Create S3 user
-    s3_akey='0555b35654ad1656d804'
-    s3_skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
-    debug echo "setting up user testid"
-    $CEPH_BIN/radosgw-admin user create --uid testid --access-key $s3_akey --secret $s3_skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null
-
-    # Create S3-test users
-    # See: https://github.com/ceph/s3-tests
-    debug echo "setting up s3-test users"
-    $CEPH_BIN/radosgw-admin user create \
-        --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
-        --access-key ABCDEFGHIJKLMNOPQRST \
-        --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \
-        --display-name youruseridhere \
-        --email s3@example.com --caps="user-policy=*" -c $conf_fn > /dev/null
-    $CEPH_BIN/radosgw-admin user create \
-        --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \
-        --access-key NOPQRSTUVWXYZABCDEFG \
-        --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \
-        --display-name john.doe \
-        --email john.doe@example.com -c $conf_fn > /dev/null
-    $CEPH_BIN/radosgw-admin user create \
-	--tenant testx \
-        --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \
-        --access-key HIJKLMNOPQRSTUVWXYZA \
-        --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \
-        --display-name tenanteduser \
-        --email tenanteduser@example.com -c $conf_fn > /dev/null
-
-    # Create Swift user
-    debug echo "setting up user tester"
-    $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null
-
-    echo ""
-    echo "S3 User Info:"
-    echo "  access key:  $s3_akey"
-    echo "  secret key:  $s3_skey"
-    echo ""
-    echo "Swift User Info:"
-    echo "  account   : test"
-    echo "  user      : tester"
-    echo "  password  : testing"
-    echo ""
-}
-
-do_rgw()
-{
-    if [ "$new" -eq 1 ]; then
-        do_rgw_create_users
-        if [ -n "$rgw_compression" ]; then
-            debug echo "setting compression type=$rgw_compression"
-            $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null
-        fi
-    fi
-
-    if [ -n "$rgw_flight_frontend" ] ;then
-        debug echo "starting arrow_flight frontend on first rgw"
-    fi
-
-    # Start server
-    if [ "$cephadm" -gt 0 ]; then
-        ceph_adm orch apply rgw rgwTest
-        return
-    fi
-
-    RGWDEBUG=""
-    if [ "$debug" -ne 0 ]; then
-        RGWDEBUG="--debug-rgw=20 --debug-ms=1"
-    fi
-
-    local CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT}"
-    local CEPH_RGW_HTTPS="${CEPH_RGW_PORT: -1}"
-    if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then
-        CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT::-1}"
-    else
-        CEPH_RGW_HTTPS=""
-    fi
-    RGWSUDO=
-    [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo
-
-    current_port=$CEPH_RGW_PORT
-    # allow only first rgw to start arrow_flight server/port
-    local flight_conf=$rgw_flight_frontend
-    for n in $(seq 1 $CEPH_NUM_RGW); do
-        rgw_name="client.rgw.${current_port}"
-
-        ceph_adm auth get-or-create $rgw_name \
-            mon 'allow rw' \
-            osd 'allow rwx' \
-            mgr 'allow rw' \
-            >> "$keyring_fn"
-
-        debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
-        run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \
-            --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \
-            --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \
-            --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \
-            --rgw_luarocks_location=${CEPH_OUT_DIR}/luarocks \
-            --rgw_keystone_url=http://localhost:5000 \
-            ${RGWDEBUG} \
-            -n ${rgw_name} \
-            "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}${flight_conf:+,arrow_flight}"
-
-        i=$(($i + 1))
-        [ $i -eq $CEPH_NUM_RGW ] && break
-
-        current_port=$((current_port+1))
-        unset flight_conf
-    done
-}
-if [ "$CEPH_NUM_RGW" -gt 0 ]; then
-    do_rgw
-fi
-
-# Ganesha Daemons
-if [ $GANESHA_DAEMON_NUM -gt 0 ]; then
-    pseudo_path="/cephfs"
-    if [ "$cephadm" -gt 0 ]; then
-        cluster_id="vstart"
-	port="2049"
-        prun ceph_adm nfs cluster create $cluster_id
-	if [ $CEPH_NUM_MDS -gt 0 ]; then
-            prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path $pseudo_path
-	    echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint"
-	fi
-	if [ "$CEPH_NUM_RGW" -gt 0 ]; then
-            pseudo_path="/rgw"
-            do_rgw_create_bucket
-	    prun ceph_adm nfs export create rgw --cluster-id $cluster_id --pseudo-path $pseudo_path --bucket "nfs-bucket"
-            echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint"
-	fi
-    else
-        start_ganesha
-	echo "Mount using: mount -t nfs -o port=<ganesha-port-num> $IP:$pseudo_path mountpoint"
-    fi
-fi
-
-docker_service(){
-     local service=''
-     #prefer podman
-     if command -v podman > /dev/null; then
-	 service="podman"
-     elif pgrep -f docker > /dev/null; then
-	 service="docker"
-     fi
-     if [ -n "$service" ]; then
-       echo "using $service for deploying jaeger..."
-       #check for exited container, remove them and restart container
-       if [ "$($service ps -aq -f status=exited -f name=jaeger)" ]; then
-	 $service rm jaeger
-       fi
-       if [ ! "$(podman ps -aq -f name=jaeger)" ]; then
-         $service "$@"
-       fi
-     else
-         echo "cannot find docker or podman, please restart service and rerun."
-     fi
-}
-
-echo ""
-if [ $with_jaeger -eq 1 ]; then
-    debug echo "Enabling jaegertracing..."
-    docker_service run -d --name jaeger \
-  -p 5775:5775/udp \
-  -p 6831:6831/udp \
-  -p 6832:6832/udp \
-  -p 5778:5778 \
-  -p 16686:16686 \
-  -p 14268:14268 \
-  -p 14250:14250 \
-  quay.io/jaegertracing/all-in-one
-fi
-
-debug echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output."
-
-echo ""
-if [ "$new" -eq 1 ]; then
-    if $with_mgr_dashboard; then
-        cat <<EOF
-dashboard urls: $DASH_URLS
-  w/ user/pass: admin / admin
-EOF
-    fi
-    if $with_mgr_restful; then
-        cat <<EOF
-restful urls: $RESTFUL_URLS
-  w/ user/pass: admin / $RESTFUL_SECRET
-EOF
-    fi
-fi
-
-echo ""
-# add header to the environment file
-{
-    echo "#"
-    echo "# source this file into your shell to set up the environment."
-    echo "# For example:"
-    echo "# $ . $CEPH_DIR/vstart_environment.sh"
-    echo "#"
-} > $CEPH_DIR/vstart_environment.sh
-{
-    echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH"
-    echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH"
-    echo "export PATH=$CEPH_DIR/bin:\$PATH"
-    echo "export CEPH_CONF=$conf_fn"
-    # We cannot set CEPH_KEYRING if this is sourced by vstart_runner.py (API tests)
-    if [ "$CEPH_DIR" != "$PWD" ]; then
-        echo "export CEPH_KEYRING=$keyring_fn"
-    fi
-
-    if [ -n "$CEPHFS_SHELL" ]; then
-        echo "alias cephfs-shell=$CEPHFS_SHELL"
-    fi
-} | tee -a $CEPH_DIR/vstart_environment.sh
-
-echo "CEPH_DEV=1"
-
-# always keep this section at the very bottom of this file
-STRAY_CONF_PATH="/etc/ceph/ceph.conf"
-if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then
-    echo ""
-    echo ""
-    echo "WARNING:"
-    echo "    Please remove stray $STRAY_CONF_PATH if not needed."
-    echo "    Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync"
-    echo "    and may lead to undesired results."
-    echo ""
-    echo "NOTE:"
-    echo "    Remember to restart cluster after removing $STRAY_CONF_PATH"
-fi
-
-init_logrotate
diff --git a/src/test/test_perf_counters_cache.cc b/src/test/test_perf_counters_cache.cc
index 8867896c02f2..1fa147ee2732 100644
--- a/src/test/test_perf_counters_cache.cc
+++ b/src/test/test_perf_counters_cache.cc
@@ -960,7 +960,6 @@ TEST(PerfCountersCache, TestLabelStrings) {
 
   // test empty val in a label pair will get the label pair added into the perf counters cache but empty key will not
   std::string label2 = key_create("bad_ctrs1", {{"label3", "val4"}, {"label1", ""}});
-  //EXPECT_DEATH(pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2), "");
   pcc->set_counter(label2, TEST_PERFCOUNTERS_COUNTER, 2);
 
   std::string label3 = key_create("bad_ctrs2", {{"", "val4"}, {"label1", "val1"}});

From d3aff2574f33d6f5a1bf533ec0cee79f485e8b3f Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Wed, 28 Jun 2023 08:25:32 +0000
Subject: [PATCH 0188/2492] tool/ceph-dedup-tool: perform dedup on snapshotted
 object if snapshot option is enabled

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/tools/ceph_dedup_tool.cc | 55 +++++++++++++++++++++++++++---------
 1 file changed, 42 insertions(+), 13 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index f3c942a97604..69dc50e5702e 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -601,15 +601,18 @@ class SampleDedupWorkerThread : public Thread
     size_t chunk_size,
     std::string &fp_algo,
     std::string &chunk_algo,
-    SampleDedupGlobal &sample_dedup_global) :
-    io_ctx(io_ctx),
+    SampleDedupGlobal &sample_dedup_global,
+    bool snap) :
     chunk_io_ctx(chunk_io_ctx),
     chunk_size(chunk_size),
     fp_type(pg_pool_t::get_fingerprint_from_str(fp_algo)),
     chunk_algo(chunk_algo),
     sample_dedup_global(sample_dedup_global),
     begin(begin),
-    end(end) { }
+    end(end),
+    snap(snap) {
+      this->io_ctx.dup(io_ctx);
+    }
 
   ~SampleDedupWorkerThread() { };
 
@@ -626,9 +629,9 @@ class SampleDedupWorkerThread : public Thread
     ObjectCursor end,
     size_t max_object_count);
   std::vector<size_t> sample_object(size_t count);
-  void try_dedup_and_accumulate_result(ObjectItem &object);
+  void try_dedup_and_accumulate_result(ObjectItem &object, snap_t snap = 0);
   bool ok_to_dedup_all();
-  int do_chunk_dedup(chunk_t &chunk);
+  int do_chunk_dedup(chunk_t &chunk, snap_t snap);
   bufferlist read_object(ObjectItem &object);
   std::vector<std::tuple<bufferlist, pair<uint64_t, uint64_t>>> do_cdc(
     ObjectItem &object,
@@ -641,13 +644,14 @@ class SampleDedupWorkerThread : public Thread
   size_t total_duplicated_size = 0;
   size_t total_object_size = 0;
 
-  std::set<std::string> oid_for_evict;
+  std::set<std::pair<std::string, snap_t>> oid_for_evict;
   const size_t chunk_size = 0;
   pg_pool_t::fingerprint_t fp_type = pg_pool_t::TYPE_FINGERPRINT_NONE;
   std::string chunk_algo;
   SampleDedupGlobal &sample_dedup_global;
   ObjectCursor begin;
   ObjectCursor end;
+  bool snap;
 };
 
 void SampleDedupWorkerThread::crawl()
@@ -666,14 +670,33 @@ void SampleDedupWorkerThread::crawl()
     auto sampled_indexes = sample_object(objects.size());
     for (size_t index : sampled_indexes) {
       ObjectItem target = objects[index];
-      try_dedup_and_accumulate_result(target);
+      if (snap) {
+	io_ctx.snap_set_read(librados::SNAP_DIR);
+	snap_set_t snap_set;
+	int snap_ret;
+	ObjectReadOperation op;
+	op.list_snaps(&snap_set, &snap_ret);
+	io_ctx.operate(target.oid, &op, NULL);
+
+	for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
+	  r != snap_set.clones.end();
+	  ++r) {
+	  io_ctx.snap_set_read(r->cloneid);
+	  try_dedup_and_accumulate_result(target, r->cloneid);
+	}
+      } else {
+	try_dedup_and_accumulate_result(target);
+      }
     }
   }
 
   vector<AioCompRef> evict_completions(oid_for_evict.size());
   int i = 0;
   for (auto &oid : oid_for_evict) {
-    evict_completions[i] = do_async_evict(oid);
+    if (snap) {
+      io_ctx.snap_set_read(oid.second);
+    }
+    evict_completions[i] = do_async_evict(oid.first);
     i++;
   }
   for (auto &completion : evict_completions) {
@@ -731,7 +754,8 @@ std::vector<size_t> SampleDedupWorkerThread::sample_object(size_t count)
   return indexes;
 }
 
-void SampleDedupWorkerThread::try_dedup_and_accumulate_result(ObjectItem &object)
+void SampleDedupWorkerThread::try_dedup_and_accumulate_result(
+  ObjectItem &object, snap_t snap)
 {
   bufferlist data = read_object(object);
   if (data.length() == 0) {
@@ -780,7 +804,7 @@ void SampleDedupWorkerThread::try_dedup_and_accumulate_result(ObjectItem &object
 
   // perform chunk-dedup
   for (auto &p : redundant_chunks) {
-    do_chunk_dedup(p);
+    do_chunk_dedup(p, snap);
   }
   total_duplicated_size += duplicated_size;
   total_object_size += object_size;
@@ -848,7 +872,7 @@ std::string SampleDedupWorkerThread::generate_fingerprint(bufferlist chunk_data)
   return ret;
 }
 
-int SampleDedupWorkerThread::do_chunk_dedup(chunk_t &chunk)
+int SampleDedupWorkerThread::do_chunk_dedup(chunk_t &chunk, snap_t snap)
 {
   uint64_t size;
   time_t mtime;
@@ -874,7 +898,7 @@ int SampleDedupWorkerThread::do_chunk_dedup(chunk_t &chunk)
       0,
       CEPH_OSD_OP_FLAG_WITH_REFERENCE);
   ret = io_ctx.operate(chunk.oid, &op, nullptr);
-  oid_for_evict.insert(chunk.oid);
+  oid_for_evict.insert(make_pair(chunk.oid, snap));
   return ret;
 }
 
@@ -1548,6 +1572,10 @@ int make_crawling_daemon(const po::variables_map &opts)
   } else {
     cout << "8192 is set as chunk size by default" << std::endl;
   }
+  bool snap = false;
+  if (opts.count("snap")) {
+    snap = true;
+  }
 
   uint32_t chunk_dedup_threshold = -1;
   if (opts.count("chunk-dedup-threshold")) {
@@ -1668,7 +1696,8 @@ int make_crawling_daemon(const po::variables_map &opts)
 	chunk_size,
 	fp_algo,
 	chunk_algo,
-	sample_dedup_global);
+	sample_dedup_global,
+	snap);
       threads.back().create("sample_dedup");
     }
 

From 0da2318e7069f373ac53ca93b75446da83429d73 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Thu, 29 Jun 2023 15:53:14 +0000
Subject: [PATCH 0189/2492] qa/workunits/test_dedup_tool.sh: add snapshot test
 during sample dedup

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 qa/workunits/rados/test_dedup_tool.sh | 71 +++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh
index 18deb331b60a..352e9ef09c19 100755
--- a/qa/workunits/rados/test_dedup_tool.sh
+++ b/qa/workunits/rados/test_dedup_tool.sh
@@ -446,11 +446,82 @@ function test_sample_dedup()
   $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
 }
 
+function test_sample_dedup_snap()
+{
+  CHUNK_POOL=dedup_chunk_pool
+  $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+  $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+
+  sleep 2
+
+  run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+  run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
+  run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_tier "$CHUNK_POOL"
+  run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_chunk_algorithm fastcdc
+  run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_cdc_chunk_size 8192
+  run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" fingerprint_algorithm sha1
+
+  # 8 Dedupable objects
+  CONTENT_1="There hiHI"
+  echo $CONTENT_1 > foo
+  for num in `seq 1 8`
+  do
+    $RADOS_TOOL -p $POOL put foo_$num ./foo
+  done
+
+  # 1 Unique object
+  CONTENT_2="There hiHI3"
+  echo $CONTENT_2 > foo3
+  $RADOS_TOOL -p $POOL put foo3_1 ./foo3
+
+  $RADOS_TOOL -p $POOL mksnap mysnap
+
+  SNAP_CONTENT="There HIHIHI" 
+  echo $SNAP_CONTENT > foo3_new
+  $RADOS_TOOL -p $POOL put foo3_1 ./foo3_new
+
+  $RADOS_TOOL -p $POOL mksnap mysnap2
+  $RADOS_TOOL -p $POOL put foo3_1 ./foo3_new
+
+  sleep 2
+
+  # Execute dedup crawler
+  RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 1 --sampling-ratio 100 --snap)
+
+  CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}')
+  SNAP_CONTENT_OID=$(echo $SNAP_CONTENT | sha1sum | awk '{print $1}')
+
+  # Find chunk object has references of 8 dedupable meta objects
+  RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $SNAP_CONTENT_OID | grep foo3_1)
+  if [ -z "$RESULT" ] ; then
+    $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+    $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+    die "There is no expected chunk object"
+  fi
+
+  RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_2 | grep foo3_1)
+  if [ -z "$RESULT" ] ; then
+    $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+    $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+    die "There is no expected chunk object"
+  fi
+
+  rm -rf ./foo ./foo3 ./foo3_new
+  for num in `seq 1 8`
+  do
+    $RADOS_TOOL -p $POOL rm foo_$num
+  done
+
+  $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+}
+
+
 test_dedup_ratio_fixed
 test_dedup_chunk_scrub
 test_dedup_chunk_repair
 test_dedup_object
 test_sample_dedup
+test_sample_dedup_snap
 
 $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
 

From 1a547ca4640fd9c3f4f25fe1d85fb09043ebd5bc Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 15 Oct 2023 10:50:52 +0000
Subject: [PATCH 0190/2492] tools/ceph_objectstore_tool: Support
 get-/set-/superblock

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/tools/ceph_objectstore_tool.cc | 47 ++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index df6d1f85c39f..0147d2b0c66a 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -3376,11 +3376,12 @@ int main(int argc, char **argv)
      "Pool name")
     ("op", po::value<string>(&op),
      "Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, list-slow-omap, fix-lost, list-pgs, dump-super, meta-list, "
-     "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, update-mon-db, dump-export, trim-pg-log, trim-pg-log-dups statfs]")
+     "get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, update-mon-db, dump-export, "
+     "trim-pg-log, trim-pg-log-dups statfs]")
     ("epoch", po::value<unsigned>(&epoch),
      "epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified")
     ("file", po::value<string>(&file),
-     "path of file to export, export-remove, import, get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap")
+     "path of file to export, export-remove, import, get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap or set-inc-osdmap")
     ("mon-store-path", po::value<string>(&mon_store_path),
      "path of monstore to update-mon-db")
     ("fsid", po::value<string>(&fsid),
@@ -3534,7 +3535,7 @@ int main(int argc, char **argv)
   outistty = isatty(STDOUT_FILENO) || tty;
 
   file_fd = fd_none;
-  if ((op == "export" || op == "export-remove" || op == "get-osdmap" || op == "get-inc-osdmap") && !dry_run) {
+  if ((op == "export" || op == "export-remove" || op == "get-osdmap" || op == "get-inc-osdmap" || op == "get-superblock") && !dry_run) {
     if (!vm.count("file") || file == "-") {
       if (outistty) {
         cerr << "stdout is a tty and no --file filename specified" << std::endl;
@@ -3544,7 +3545,7 @@ int main(int argc, char **argv)
     } else {
       file_fd = open(file.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
     }
-  } else if (op == "import" || op == "dump-export" || op == "set-osdmap" || op == "set-inc-osdmap" || op == "pg-log-inject-dups") {
+  } else if (op == "import" || op == "dump-export" || op == "set-osdmap" || op == "set-inc-osdmap" || op == "pg-log-inject-dups" || op == "set-superblock") {
     if (!vm.count("file") || file == "-") {
       if (isatty(STDIN_FILENO)) {
         cerr << "stdin is a tty and no --file filename specified" << std::endl;
@@ -3560,7 +3561,7 @@ int main(int argc, char **argv)
 
   if (vm.count("file") && file_fd == fd_none && !dry_run) {
     cerr << "--file option only applies to import, dump-export, export, export-remove, "
-	 << "get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap" << std::endl;
+	 << "get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap or set-inc-osdmap" << std::endl;
     return 1;
   }
 
@@ -3977,6 +3978,39 @@ int main(int argc, char **argv)
       ret = set_inc_osdmap(fs.get(), epoch, bl, force);
     }
     goto out;
+  } else if (op == "get-superblock") {
+    bufferlist bl;
+    ceph_assert(superblock != nullptr);
+    encode(*superblock, bl);
+    ret = bl.write_fd(file_fd);
+    if (ret) {
+      cerr << "Failed to write to " << file << ": " << cpp_strerror(ret) << std::endl;
+      goto out;
+    } else {
+      cout << "superblock exported." << std::endl;
+    }
+    goto out;
+  } else if (op == "set-superblock") {
+    bufferlist bl;
+    ret = get_fd_data(file_fd, bl);
+    if (ret < 0) {
+      cerr << "Failed to read superblock " << cpp_strerror(ret) << std::endl;
+    } else {
+      // OSD::write_superblock
+      ObjectStore::Transaction t;
+      t.write(coll_t::meta(), OSD_SUPERBLOCK_GOBJECT, 0, bl.length(), bl);
+      std::map<std::string, ceph::buffer::list> attrs;
+      attrs.emplace(OSD_SUPERBLOCK_OMAP_KEY, bl);
+      t.omap_setkeys(coll_t::meta(), OSD_SUPERBLOCK_GOBJECT, attrs);
+      auto ch = fs.get()->open_collection(coll_t::meta());
+      ret = fs.get()->queue_transaction(ch, std::move(t));
+      if (ret < 0) {
+        cerr << "Error setting superblock" << cpp_strerror(ret) << std::endl;
+        goto out;
+      }
+      cout << "Superblock was set" << std::endl;
+    }
+    goto out;
   } else if (op == "update-mon-db") {
     if (!vm.count("mon-store-path")) {
       cerr << "Please specify the path to monitor db to update" << std::endl;
@@ -4112,7 +4146,8 @@ int main(int argc, char **argv)
   // before complaining about a bad pgid
   if (!vm.count("objcmd") && op != "export" && op != "export-remove" && op != "info" && op != "log" && op != "mark-complete" && op != "trim-pg-log" && op != "trim-pg-log-dups" && op != "pg-log-inject-dups") {
     cerr << "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-super, meta-list, "
-      "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, dump-export, trim-pg-log, trim-pg-log-dups statfs)"
+      "get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, dump-export, trim-pg-log, "
+      "trim-pg-log-dups statfs)"
 	 << std::endl;
     usage(desc);
     ret = 1;

From fc1049764c0bb2158278b309764c4a298da70d41 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 15 Oct 2023 14:23:36 +0000
Subject: [PATCH 0191/2492] doc/man/8/ceph-objectstore-tool.rst: add get/set
 superblock doc

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 doc/man/8/ceph-objectstore-tool.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/man/8/ceph-objectstore-tool.rst b/doc/man/8/ceph-objectstore-tool.rst
index de1304733bc7..88b0da6ca5bb 100644
--- a/doc/man/8/ceph-objectstore-tool.rst
+++ b/doc/man/8/ceph-objectstore-tool.rst
@@ -59,6 +59,8 @@ Possible -op commands::
 * meta-list
 * get-osdmap
 * set-osdmap
+* get-superblock
+* set-superblock
 * get-inc-osdmap
 * set-inc-osdmap
 * mark-complete
@@ -404,7 +406,7 @@ Options
 
 .. option:: --op arg
 
-   Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, fix-lost, list-pgs, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, update-mon-db, dump-export, trim-pg-log]
+   Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, fix-lost, list-pgs, dump-super, meta-list, get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, update-mon-db, dump-export, trim-pg-log]
 
 .. option:: --epoch arg
 
@@ -412,7 +414,7 @@ Options
 
 .. option:: --file arg             
    
-   path of file to export, export-remove, import, get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap
+   path of file to export, export-remove, import, get-osdmap, set-osdmap, get-superblock, set-superblock, get-inc-osdmap or set-inc-osdmap
 
 .. option:: --mon-store-path arg
 

From 7d808bbab2f6057ddbf53de2b385db8c14ae7d2c Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 15 Oct 2023 14:28:19 +0000
Subject: [PATCH 0192/2492] osd/osd_types: don't increment snap_seq on removal

Avoid adjacent snap ids removals discontinuity in monitor's purged_snap and OSD's PSN entries.

Fixes: https://tracker.ceph.com/issues/62983

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/osd_types.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 664d8a287406..8d4ba74789fc 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -1755,19 +1755,13 @@ void pg_pool_t::remove_snap(snapid_t s)
 {
   ceph_assert(snaps.count(s));
   snaps.erase(s);
-  snap_seq = snap_seq + 1;
 }
 
 void pg_pool_t::remove_unmanaged_snap(snapid_t s, bool preoctopus_compat)
 {
   ceph_assert(is_unmanaged_snaps_mode());
-  ++snap_seq;
   if (preoctopus_compat) {
     removed_snaps.insert(s);
-    // try to add in the new seq, just to try to keep the interval_set contiguous
-    if (!removed_snaps.contains(get_snap_seq())) {
-      removed_snaps.insert(get_snap_seq());
-    }
   }
 }
 

From f04d42380bb640cd38e4a3170ff51b3d5e0141f6 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 15 Oct 2023 14:28:53 +0000
Subject: [PATCH 0193/2492] mon/OSDMonitor: remove pending_pseudo_purged_snaps

pending_pseudo_purged_snaps was introduced in order to avoid discontinuity in the purged_snap monitor entries.
After this change there shouldn't be any discontinuity in the first place.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 12 ------------
 src/mon/OSDMonitor.h  |  1 -
 2 files changed, 13 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index ac6a3f4466c2..d7e67e49103f 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -1162,7 +1162,6 @@ void OSDMonitor::create_pending()
   pending_inc.fsid = mon.monmap->fsid;
   pending_metadata.clear();
   pending_metadata_rm.clear();
-  pending_pseudo_purged_snaps.clear();
 
   dout(10) << "create_pending e " << pending_inc.epoch << dendl;
 
@@ -2066,13 +2065,6 @@ void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 				t);
     }
   }
-  for (auto& [pool, snaps] : pending_pseudo_purged_snaps) {
-    for (auto snap : snaps) {
-      insert_purged_snap_update(pool, snap, snap + 1,
-				pending_inc.epoch,
-				t);
-    }
-  }
 
   // health
   health_check_map_t next;
@@ -14303,10 +14295,6 @@ bool OSDMonitor::prepare_pool_op(MonOpRequestRef op)
 	m->snapid,
 	osdmap.require_osd_release < ceph_release_t::octopus);
       pending_inc.new_removed_snaps[m->pool].insert(m->snapid);
-      // also record the new seq as purged: this avoids a discontinuity
-      // after all of the snaps have been purged, since the seq assigned
-      // during removal lives in the same namespace as the actual snaps.
-      pending_pseudo_purged_snaps[m->pool].insert(pp.get_snap_seq());
       changed = true;
     }
     break;
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 3f6226c057a1..08aa4f61783b 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -217,7 +217,6 @@ class OSDMonitor : public PaxosService,
   std::map<int, failure_info_t> failure_info;
   std::map<int,utime_t>    down_pending_out;  // osd down -> out
   bool priority_convert = false;
-  std::map<int64_t,std::set<snapid_t>> pending_pseudo_purged_snaps;
   std::shared_ptr<PriorityCache::PriCache> rocksdb_binned_kv_cache = nullptr;
   std::shared_ptr<PriorityCache::Manager> pcm = nullptr;
   ceph::mutex balancer_lock = ceph::make_mutex("OSDMonitor::balancer_lock");

From 01b5aa5d27a97a02cef03b9eb0ee9dbcd55684f1 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 15 Oct 2023 22:43:39 +1000
Subject: [PATCH 0194/2492] doc/rados: Edit troubleshooting-osd (3 of x)

Edit doc/rados/troubleshooting/troubleshooting-osd.rst (3 of x).

Follows https://github.com/ceph/ceph/pull/53986.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-osd.rst   | 217 ++++++++++--------
 1 file changed, 123 insertions(+), 94 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-osd.rst b/doc/rados/troubleshooting/troubleshooting-osd.rst
index 2e3bcd58ea94..035947d7ec94 100644
--- a/doc/rados/troubleshooting/troubleshooting-osd.rst
+++ b/doc/rados/troubleshooting/troubleshooting-osd.rst
@@ -544,33 +544,39 @@ Recovery Throttling
 -------------------
 
 Depending upon your configuration, Ceph may reduce recovery rates to maintain
-performance or it may increase recovery rates to the point that recovery
-impacts OSD performance. Check to see if the OSD is recovering.
+client or OSD performance, or it may increase recovery rates to the point that
+recovery impacts client or OSD performance. Check to see if the client or OSD
+is recovering.
+
 
 Kernel Version
 --------------
 
-Check the kernel version you are running. Older kernels may not receive
-new backports that Ceph depends upon for better performance.
+Check the kernel version that you are running. Older kernels may lack updates
+that improve Ceph performance. 
+
 
 Kernel Issues with SyncFS
 -------------------------
 
-Try running one OSD per host to see if performance improves. Old kernels
-might not have a recent enough version of ``glibc`` to support ``syncfs(2)``.
+If you have kernel issues with SyncFS, try running one OSD per host to see if
+performance improves. Old kernels might not have a recent enough version of
+``glibc`` to support ``syncfs(2)``.
+
 
 Filesystem Issues
 -----------------
 
-Currently, we recommend deploying clusters with the BlueStore back end.
-When running a pre-Luminous release or if you have a specific reason to deploy
-OSDs with the previous Filestore backend, we recommend ``XFS``.
+In post-Luminous releases, we recommend deploying clusters with the BlueStore
+back end.  When running a pre-Luminous release, or if you have a specific
+reason to deploy OSDs with the previous Filestore backend, we recommend
+``XFS``.
 
 We recommend against using ``Btrfs`` or ``ext4``.  The ``Btrfs`` filesystem has
-many attractive features, but bugs may lead to
-performance issues and spurious ENOSPC errors.  We do not recommend
-``ext4`` for Filestore OSDs because ``xattr`` limitations break support for long
-object names, which are needed for RGW.
+many attractive features, but bugs may lead to performance issues and spurious
+ENOSPC errors.  We do not recommend ``ext4`` for Filestore OSDs because
+``xattr`` limitations break support for long object names, which are needed for
+RGW.
 
 For more information, see `Filesystem Recommendations`_.
 
@@ -579,31 +585,32 @@ For more information, see `Filesystem Recommendations`_.
 Insufficient RAM
 ----------------
 
-We recommend a *minimum* of 4GB of RAM per OSD daemon and suggest rounding up
-from 6-8GB.  You may notice that during normal operations, ``ceph-osd``
-processes only use a fraction of that amount.
-Unused RAM makes it tempting to use the excess RAM for co-resident
-applications or to skimp on each node's memory capacity.  However,
-when OSDs experience recovery their memory utilization spikes. If
-there is insufficient RAM available, OSD performance will slow considerably
-and the daemons may even crash or be killed by the Linux ``OOM Killer``.
+We recommend a *minimum* of 4GB of RAM per OSD daemon and we suggest rounding
+up from 6GB to 8GB. During normal operations, you may notice that ``ceph-osd``
+processes use only a fraction of that amount.  You might be tempted to use the
+excess RAM for co-resident applications or to skimp on each node's memory
+capacity. However, when OSDs experience recovery their memory utilization
+spikes. If there is insufficient RAM available during recovery, OSD performance
+will slow considerably and the daemons may even crash or be killed by the Linux
+``OOM Killer``.
+
 
 Blocked Requests or Slow Requests
 ---------------------------------
 
-If a ``ceph-osd`` daemon is slow to respond to a request, messages will be logged
-noting ops that are taking too long.  The warning threshold
+When a ``ceph-osd`` daemon is slow to respond to a request, the cluster log
+receives messages reporting ops that are taking too long. The warning threshold
 defaults to 30 seconds and is configurable via the ``osd_op_complaint_time``
-setting.  When this happens, the cluster log will receive messages.
+setting.
 
 Legacy versions of Ceph complain about ``old requests``::
 
-	osd.0 192.168.106.220:6800/18813 312 : [WRN] old request osd_op(client.5099.0:790 fatty_26485_object789 [write 0~4096] 2.5e54f643) v4 received at 2012-03-06 15:42:56.054801 currently waiting for sub ops
+    osd.0 192.168.106.220:6800/18813 312 : [WRN] old request osd_op(client.5099.0:790 fatty_26485_object789 [write 0~4096] 2.5e54f643) v4 received at 2012-03-06 15:42:56.054801 currently waiting for sub ops
 
-New versions of Ceph complain about ``slow requests``::
+Newer versions of Ceph complain about ``slow requests``::
 
-	{date} {osd.num} [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.005692 secs
-	{date} {osd.num}  [WRN] slow request 30.005692 seconds old, received at {date-time}: osd_op(client.4240.0:8 benchmark_data_ceph-1_39426_object7 [write 0~4194304] 0.69848840) v4 currently waiting for subops from [610]
+    {date} {osd.num} [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.005692 secs
+    {date} {osd.num}  [WRN] slow request 30.005692 seconds old, received at {date-time}: osd_op(client.4240.0:8 benchmark_data_ceph-1_39426_object7 [write 0~4194304] 0.69848840) v4 currently waiting for subops from [610]
 
 Possible causes include:
 
@@ -623,27 +630,27 @@ Possible solutions:
 Debugging Slow Requests
 -----------------------
 
-If you run ``ceph daemon osd.<id> dump_historic_ops`` or ``ceph daemon osd.<id> dump_ops_in_flight``,
-you will see a set of operations and a list of events each operation went
-through. These are briefly described below.
+If you run ``ceph daemon osd.<id> dump_historic_ops`` or ``ceph daemon osd.<id>
+dump_ops_in_flight``, you will see a set of operations and a list of events
+each operation went through. These are briefly described below.
 
 Events from the Messenger layer:
 
-- ``header_read``: When the messenger first started reading the message off the wire.
-- ``throttled``: When the messenger tried to acquire memory throttle space to read
+- ``header_read``: The time that the messenger first started reading the message off the wire.
+- ``throttled``: The time that the messenger tried to acquire memory throttle space to read
   the message into memory.
-- ``all_read``: When the messenger finished reading the message off the wire.
-- ``dispatched``: When the messenger gave the message to the OSD.
+- ``all_read``: The time that the messenger finished reading the message off the wire.
+- ``dispatched``: The time that the messenger gave the message to the OSD.
 - ``initiated``: This is identical to ``header_read``. The existence of both is a
   historical oddity.
 
 Events from the OSD as it processes ops:
 
 - ``queued_for_pg``: The op has been put into the queue for processing by its PG.
-- ``reached_pg``: The PG has started doing the op.
-- ``waiting for \*``: The op is waiting for some other work to complete before it
-  can proceed (e.g. a new OSDMap; for its object target to scrub; for the PG to
-  finish peering; all as specified in the message).
+- ``reached_pg``: The PG has started performing the op.
+- ``waiting for \*``: The op is waiting for some other work to complete before
+  it can proceed (for example, a new OSDMap; the scrubbing of its object
+  target; the completion of a PG's peering; all as specified in the message).
 - ``started``: The op has been accepted as something the OSD should do and 
   is now being performed.
 - ``waiting for subops from``: The op has been sent to replica OSDs.
@@ -651,95 +658,115 @@ Events from the OSD as it processes ops:
 Events from ```Filestore```:
 
 - ``commit_queued_for_journal_write``: The op has been given to the FileStore.
-- ``write_thread_in_journal_buffer``: The op is in the journal's buffer and waiting
+- ``write_thread_in_journal_buffer``: The op is in the journal's buffer and is waiting
   to be persisted (as the next disk write).
 - ``journaled_completion_queued``: The op was journaled to disk and its callback
-  queued for invocation.
+  has been queued for invocation.
 
 Events from the OSD after data has been given to underlying storage:
 
-- ``op_commit``: The op has been committed (i.e. written to journal) by the
+- ``op_commit``: The op has been committed (that is, written to journal) by the
   primary OSD.
-- ``op_applied``: The op has been `write()'en <https://www.freebsd.org/cgi/man.cgi?write(2)>`_ to the backing FS (i.e.   applied in memory but not flushed out to disk) on the primary.
+- ``op_applied``: The op has been `write()'en
+  <https://www.freebsd.org/cgi/man.cgi?write(2)>`_ to the backing FS (that is,
+  applied in memory but not flushed out to disk) on the primary.
 - ``sub_op_applied``: ``op_applied``, but for a replica's "subop".
 - ``sub_op_committed``: ``op_commit``, but for a replica's subop (only for EC pools).
 - ``sub_op_commit_rec/sub_op_apply_rec from <X>``: The primary marks this when it
   hears about the above, but for a particular replica (i.e. ``<X>``).
 - ``commit_sent``: We sent a reply back to the client (or primary OSD, for sub ops).
 
-Many of these events are seemingly redundant, but cross important boundaries in
-the internal code (such as passing data across locks into new threads).
+Some of these events may appear redundant, but they cross important boundaries
+in the internal code (such as passing data across locks into new threads).
+
 
 Flapping OSDs
 =============
 
-When OSDs peer and check heartbeats, they use the cluster (back-end)
-network when it's available. See `Monitor/OSD Interaction`_ for details.
+"Flapping" is the term for the phenomenon of an OSD being repeatedly marked
+``up`` and then ``down`` in rapid succession.  This section explains how to
+recognize flapping, and how to mitigate it.
 
-We have traditionally recommended separate *public* (front-end) and *private*
-(cluster / back-end / replication) networks:
+When OSDs peer and check heartbeats, they use the cluster (back-end) network
+when it is available. See `Monitor/OSD Interaction`_ for details.
 
-#. Segregation of heartbeat and replication / recovery traffic (private)
-   from client and OSD <-> mon traffic (public).  This helps keep one
-   from DoS-ing the other, which could in turn result in a cascading failure.
+The upstream Ceph community has traditionally recommended separate *public*
+(front-end) and *private* (cluster / back-end / replication) networks. This
+provides the following benefits:
+
+#. Segregation of (1) heartbeat traffic and replication/recovery traffic
+   (private) from (2) traffic from clients and between OSDs and monitors
+   (public). This helps keep one stream of traffic from DoS-ing the other,
+   which could in turn result in a cascading failure.
 
 #. Additional throughput for both public and private traffic.
 
-When common networking technologies were 100Mb/s and 1Gb/s, this separation
-was often critical.  With today's 10Gb/s, 40Gb/s, and 25/50/100Gb/s
-networks, the above capacity concerns are often diminished or even obviated.
-For example, if your OSD nodes have two network ports, dedicating one to
-the public and the other to the private network means no path redundancy.
-This degrades your ability to weather network maintenance and failures without
-significant cluster or client impact.  Consider instead using both links
-for just a public network:  with bonding (LACP) or equal-cost routing (e.g. FRR)
-you reap the benefits of increased throughput headroom, fault tolerance, and
-reduced OSD flapping.
+In the past, when common networking technologies were measured in a range
+encompassing 100Mb/s and 1Gb/s, this separation was often critical. But with
+today's 10Gb/s, 40Gb/s, and 25/50/100Gb/s networks, the above capacity concerns
+are often diminished or even obviated.  For example, if your OSD nodes have two
+network ports, dedicating one to the public and the other to the private
+network means that you have no path redundancy.  This degrades your ability to
+endure network maintenance and network failures without significant cluster or
+client impact. In situations like this, consider instead using both links for
+only a public network: with bonding (LACP) or equal-cost routing (for example,
+FRR) you reap the benefits of increased throughput headroom, fault tolerance,
+and reduced OSD flapping.
 
 When a private network (or even a single host link) fails or degrades while the
-public network operates normally, OSDs may not handle this situation well. What
-happens is that OSDs use the public network to report each other ``down`` to
-the monitors, while marking themselves ``up``. The monitors then send out,
-again on the public network, an updated cluster map with affected OSDs marked
-`down`. These OSDs reply to the monitors "I'm not dead yet!", and the cycle
-repeats.  We call this scenario 'flapping`, and it can be difficult to isolate
-and remediate.  With no private network, this irksome dynamic is avoided:
-OSDs are generally either ``up`` or ``down`` without flapping.
-
-If something does cause OSDs to 'flap' (repeatedly getting marked ``down`` and
+public network continues operating normally, OSDs may not handle this situation
+well. In such situations, OSDs use the public network to report each other
+``down`` to the monitors, while marking themselves ``up``. The monitors then
+send out-- again on the public network--an updated cluster map with the
+affected OSDs marked `down`. These OSDs reply to the monitors "I'm not dead
+yet!", and the cycle repeats. We call this scenario 'flapping`, and it can be
+difficult to isolate and remediate. Without a private network, this irksome
+dynamic is avoided: OSDs are generally either ``up`` or ``down`` without
+flapping.
+
+If something does cause OSDs to 'flap' (repeatedly being marked ``down`` and
 then ``up`` again), you can force the monitors to halt the flapping by
-temporarily freezing their states::
+temporarily freezing their states:
 
-	ceph osd set noup      # prevent OSDs from getting marked up
-	ceph osd set nodown    # prevent OSDs from getting marked down
+.. prompt:: bash
 
-These flags are recorded in the osdmap::
+   ceph osd set noup      # prevent OSDs from getting marked up
+   ceph osd set nodown    # prevent OSDs from getting marked down
 
-	ceph osd dump | grep flags
-	flags no-up,no-down
+These flags are recorded in the osdmap:
 
-You can clear the flags with::
+.. prompt:: bash
 
-	ceph osd unset noup
-	ceph osd unset nodown
+   ceph osd dump | grep flags
 
-Two other flags are supported, ``noin`` and ``noout``, which prevent
-booting OSDs from being marked ``in`` (allocated data) or protect OSDs
-from eventually being marked ``out`` (regardless of what the current value for
-``mon_osd_down_out_interval`` is).
+::
 
-.. note:: ``noup``, ``noout``, and ``nodown`` are temporary in the
-   sense that once the flags are cleared, the action they were blocking
-   should occur shortly after.  The ``noin`` flag, on the other hand,
-   prevents OSDs from being marked ``in`` on boot, and any daemons that
-   started while the flag was set will remain that way.
+   flags no-up,no-down
 
-.. note:: The causes and effects of flapping can be somewhat mitigated through
-   careful adjustments to the ``mon_osd_down_out_subtree_limit``,
+You can clear these flags with:
+
+.. prompt:: bash
+
+   ceph osd unset noup
+   ceph osd unset nodown
+
+Two other flags are available, ``noin`` and ``noout``, which prevent booting
+OSDs from being marked ``in`` (allocated data) or protect OSDs from eventually
+being marked ``out`` (regardless of the current value of
+``mon_osd_down_out_interval``).
+
+.. note:: ``noup``, ``noout``, and ``nodown`` are temporary in the sense that
+   after the flags are cleared, the action that they were blocking should be
+   possible shortly thereafter. But the ``noin`` flag prevents OSDs from being
+   marked ``in`` on boot, and any daemons that started while the flag was set
+   will remain that way.
+
+.. note:: The causes and effects of flapping can be mitigated somewhat by
+   making careful adjustments to ``mon_osd_down_out_subtree_limit``,
    ``mon_osd_reporter_subtree_level``, and ``mon_osd_min_down_reporters``.
    Derivation of optimal settings depends on cluster size, topology, and the
-   Ceph  release in use. Their interactions are subtle and beyond the scope of
-   this document.
+   Ceph release in use. The interaction of all of these factors is subtle and
+   is beyond the scope of this document.
 
 
 .. _iostat: https://en.wikipedia.org/wiki/Iostat
@@ -749,7 +776,9 @@ from eventually being marked ``out`` (regardless of what the current value for
 .. _Monitor/OSD Interaction: ../../configuration/mon-osd-interaction
 .. _Monitor Config Reference: ../../configuration/mon-config-ref
 .. _monitoring your OSDs: ../../operations/monitoring-osd-pg
+
 .. _monitoring OSDs: ../../operations/monitoring-osd-pg/#monitoring-osds
+
 .. _subscribe to the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=subscribe+ceph-devel
 .. _unsubscribe from the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=unsubscribe+ceph-devel
 .. _subscribe to the ceph-users email list: mailto:ceph-users-join@lists.ceph.com

From 7e55b60060c71ad945a31d788f59f3531079f919 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Tue, 16 Nov 2021 15:18:24 -0500
Subject: [PATCH 0195/2492] mon/ConfigMonitor: clean up pending_* clear()s

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 782dbdc2a73fc7a95140285e1141cff897b4f398)
---
 src/mon/ConfigMonitor.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc
index 4b7322d2e8f2..7f52b9d7ef73 100644
--- a/src/mon/ConfigMonitor.cc
+++ b/src/mon/ConfigMonitor.cc
@@ -71,7 +71,7 @@ void ConfigMonitor::create_initial()
 {
   dout(10) << __func__ << dendl;
   version = 0;
-  pending.clear();
+  create_pending();
 }
 
 void ConfigMonitor::update_from_paxos(bool *need_bootstrap)
@@ -89,6 +89,7 @@ void ConfigMonitor::create_pending()
 {
   dout(10) << " " << version << dendl;
   pending.clear();
+  pending_cleanup.clear();
   pending_description.clear();
 }
 
@@ -779,12 +780,12 @@ void ConfigMonitor::load_config()
     { "mds_session_blacklist_on_evict", "mds_session_blocklist_on_evict" },
   };
 
+  config_map.clear();
+  current.clear();
+
   unsigned num = 0;
   KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX);
   it->lower_bound(KEY_PREFIX);
-  config_map.clear();
-  current.clear();
-  pending_cleanup.clear();
   while (it->valid() &&
 	 it->key().compare(0, KEY_PREFIX.size(), KEY_PREFIX) == 0) {
     string key = it->key().substr(KEY_PREFIX.size());

From b039f8e1e74d956c8a30d0c3c7d43316a7df5b8c Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Wed, 17 Nov 2021 18:02:29 -0500
Subject: [PATCH 0196/2492] test/mon: add unittest_config_map

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit fe3a1dad9a74f89ed6d50a628ca739dddee75523)
---
 src/test/mon/CMakeLists.txt     |   7 ++
 src/test/mon/test_config_map.cc | 143 ++++++++++++++++++++++++++++++++
 2 files changed, 150 insertions(+)
 create mode 100644 src/test/mon/test_config_map.cc

diff --git a/src/test/mon/CMakeLists.txt b/src/test/mon/CMakeLists.txt
index 943ca99a3fe9..8c271407e982 100644
--- a/src/test/mon/CMakeLists.txt
+++ b/src/test/mon/CMakeLists.txt
@@ -18,6 +18,13 @@ add_executable(ceph_test_mon_msg
   )
 target_link_libraries(ceph_test_mon_msg os osdc global ${UNITTEST_LIBS})
 
+# unittest_config_map
+add_executable(unittest_config_map
+  test_config_map.cc
+  )
+add_ceph_unittest(unittest_config_map)
+target_link_libraries(unittest_config_map mon global)
+
 # unittest_mon_moncap
 add_executable(unittest_mon_moncap
   moncap.cc
diff --git a/src/test/mon/test_config_map.cc b/src/test/mon/test_config_map.cc
new file mode 100644
index 000000000000..ab148a10bca6
--- /dev/null
+++ b/src/test/mon/test_config_map.cc
@@ -0,0 +1,143 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "mon/ConfigMap.h"
+
+#include <iostream>
+#include <string>
+#include "crush/CrushWrapper.h"
+#include "common/ceph_context.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+
+TEST(ConfigMap, parse_key)
+{
+  ConfigMap cm;
+  {
+    std::string name, who;
+    cm.parse_key("global/foo", &name, &who);
+    ASSERT_EQ("foo", name);
+    ASSERT_EQ("global", who);
+  }
+  {
+    std::string name, who;
+    cm.parse_key("mon/foo", &name, &who);
+    ASSERT_EQ("foo", name);
+    ASSERT_EQ("mon", who);
+  }
+  {
+    std::string name, who;
+    cm.parse_key("mon.a/foo", &name, &who);
+    ASSERT_EQ("foo", name);
+    ASSERT_EQ("mon.a", who);
+  }
+  {
+    std::string name, who;
+    cm.parse_key("mon.a/mgr/foo", &name, &who);
+    ASSERT_EQ("mgr/foo", name);
+    ASSERT_EQ("mon.a", who);
+  }
+  {
+    std::string name, who;
+    cm.parse_key("mon.a/a=b/foo", &name, &who);
+    ASSERT_EQ("foo", name);
+    ASSERT_EQ("mon.a/a=b", who);
+  }
+  {
+    std::string name, who;
+    cm.parse_key("mon.a/a=b/c=d/foo", &name, &who);
+    ASSERT_EQ("foo", name);
+    ASSERT_EQ("mon.a/a=b/c=d", who);
+  }
+}
+
+TEST(ConfigMap, add_option)
+{
+  ConfigMap cm;
+  auto cct = new CephContext(CEPH_ENTITY_TYPE_MON);
+  int r;
+
+  r = cm.add_option(
+    cct, "foo", "global", "fooval",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.global.options.size());
+
+  r = cm.add_option(
+    cct, "foo", "mon", "fooval",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.by_type.size());
+  ASSERT_EQ(1, cm.by_type["mon"].options.size());
+  
+  r = cm.add_option(
+    cct, "foo", "mon.a", "fooval",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.by_id.size());
+  ASSERT_EQ(1, cm.by_id["mon.a"].options.size());
+}
+
+
+TEST(ConfigMap, result_sections)
+{
+  ConfigMap cm;
+  auto cct = new CephContext(CEPH_ENTITY_TYPE_MON);
+  auto crush = new CrushWrapper;
+  crush->finalize();
+
+  int r;
+
+  r = cm.add_option(
+    cct, "foo", "global", "g",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.global.options.size());
+
+  r = cm.add_option(
+    cct, "foo", "mon", "m",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.by_type.size());
+  ASSERT_EQ(1, cm.by_type["mon"].options.size());
+
+  r = cm.add_option(
+    cct, "foo", "mon.a", "a",
+    [&](const std::string& name) {
+      return nullptr;
+    });
+  ASSERT_EQ(0, r);
+  ASSERT_EQ(1, cm.by_id.size());
+  ASSERT_EQ(1, cm.by_id["mon.a"].options.size());
+
+  EntityName n;
+  n.set(CEPH_ENTITY_TYPE_MON, "a");
+  auto c = cm.generate_entity_map(
+    n, {}, crush, "none", nullptr);
+  ASSERT_EQ(1, c.size());
+  ASSERT_EQ("a", c["foo"]);
+
+  n.set(CEPH_ENTITY_TYPE_MON, "b");
+  c = cm.generate_entity_map(
+    n, {}, crush, "none", nullptr);
+  ASSERT_EQ(1, c.size());
+  ASSERT_EQ("m", c["foo"]);
+
+  n.set(CEPH_ENTITY_TYPE_MDS, "c");
+  c = cm.generate_entity_map(
+    n, {}, crush, "none", nullptr);
+  ASSERT_EQ(1, c.size());
+  ASSERT_EQ("g", c["foo"]);
+}
+

From e65f676cd17836b2144fca82408412fe9e21fdaf Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Tue, 16 Nov 2021 16:04:01 -0500
Subject: [PATCH 0197/2492] mon/ConfigMap: use ValueSource type for
 generate_entity_map()

Changes from original commit:
- switch to unordered map
- use nullptr instead of 0

Signed-off-by: Sage Weil <sage@newdream.net>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
(cherry picked from commit e36da92118eac4d2af11d8a8a1795e7b08d08874)
---
 src/mgr/ActivePyModules.cc |  4 +---
 src/mon/ConfigMap.cc       |  4 ++--
 src/mon/ConfigMap.h        | 11 ++++++++++-
 src/mon/ConfigMonitor.cc   | 16 ++++++++--------
 4 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 45038e734afc..701b41bef8e1 100644
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -1198,13 +1198,11 @@ PyObject *ActivePyModules::get_foreign_config(
 		 << " class " << device_class << dendl;
       }
 
-      std::map<std::string,pair<std::string,const MaskedOption*>> src;
       config = config_map.generate_entity_map(
 	entity,
 	crush_location,
 	osdmap.crush.get(),
-	device_class,
-	&src);
+	device_class);
     });
 
   // get a single value
diff --git a/src/mon/ConfigMap.cc b/src/mon/ConfigMap.cc
index 7a639947bdef..dff1a94c1e6f 100644
--- a/src/mon/ConfigMap.cc
+++ b/src/mon/ConfigMap.cc
@@ -136,7 +136,7 @@ ConfigMap::generate_entity_map(
   const map<std::string,std::string>& crush_location,
   const CrushWrapper *crush,
   const std::string& device_class,
-  std::map<std::string,pair<std::string,const MaskedOption*>> *src)
+  std::unordered_map<std::string, ValueSource> *src)
 {
   // global, then by type, then by name prefix component(s), then name.
   // name prefix components are .-separated,
@@ -185,7 +185,7 @@ ConfigMap::generate_entity_map(
       }
       out[i.first] = o.raw_value;
       if (src) {
-	(*src)[i.first] = make_pair(s.first, &o);
+	(*src).emplace(i.first, ConfigMap::ValueSource(s.first, &o));
       }
       prev = &o;
     }
diff --git a/src/mon/ConfigMap.h b/src/mon/ConfigMap.h
index 34af942a61e0..bf2bd97c0ba3 100644
--- a/src/mon/ConfigMap.h
+++ b/src/mon/ConfigMap.h
@@ -99,6 +99,14 @@ struct Section {
 };
 
 struct ConfigMap {
+  struct ValueSource {
+    std::string section;
+    const MaskedOption *option = nullptr;
+    ValueSource() {}
+    ValueSource(const std::string& s, const MaskedOption *o)
+      : section(s), option(o) {}
+  };
+
   Section global;
   std::map<std::string,Section, std::less<>> by_type;
   std::map<std::string,Section, std::less<>> by_id;
@@ -125,12 +133,13 @@ struct ConfigMap {
     stray_options.clear();
   }
   void dump(ceph::Formatter *f) const;
+
   std::map<std::string,std::string,std::less<>> generate_entity_map(
     const EntityName& name,
     const std::map<std::string,std::string>& crush_location,
     const CrushWrapper *crush,
     const std::string& device_class,
-    std::map<std::string,std::pair<std::string,const MaskedOption*>> *src=0);
+    std::unordered_map<std::string,ValueSource> *src = nullptr);
 
   void parse_key(
     const std::string& key,
diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc
index 7f52b9d7ef73..a5f61cbea529 100644
--- a/src/mon/ConfigMonitor.cc
+++ b/src/mon/ConfigMonitor.cc
@@ -314,7 +314,7 @@ bool ConfigMonitor::preprocess_command(MonOpRequestRef op)
 	       << " class " << device_class << dendl;
     }
 
-    std::map<std::string,pair<std::string,const MaskedOption*>> src;
+    std::unordered_map<std::string,ConfigMap::ValueSource> src;
     auto config = config_map.generate_entity_map(
       entity,
       crush_location,
@@ -378,20 +378,20 @@ bool ConfigMonitor::preprocess_command(MonOpRequestRef op)
 	  continue;
 	}
 	if (!f) {
-	  tbl << q->second.first;
-	  tbl << q->second.second->mask.to_str();
-	  tbl << Option::level_to_str(q->second.second->opt->level);
+	  tbl << q->second.section;
+	  tbl << q->second.option->mask.to_str();
+	  tbl << Option::level_to_str(q->second.option->opt->level);
 	  tbl << p->first;
 	  tbl << p->second;
-	  tbl << (q->second.second->opt->can_update_at_runtime() ? "" : "*");
+	  tbl << (q->second.option->opt->can_update_at_runtime() ? "" : "*");
 	  tbl << TextTable::endrow;
 	} else {
 	  f->open_object_section(p->first.c_str());
 	  f->dump_string("value", p->second);
-	  f->dump_string("section", q->second.first);
-	  f->dump_object("mask", q->second.second->mask);
+	  f->dump_string("section", q->second.section);
+	  f->dump_object("mask", q->second.option->mask);
 	  f->dump_bool("can_update_at_runtime",
-		       q->second.second->opt->can_update_at_runtime());
+		       q->second.option->opt->can_update_at_runtime());
 	  f->close_section();
 	}
       }

From 4b1e82eaaa602dfa3e21b5adc0e9a7ae24359974 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Tue, 16 Nov 2021 17:48:27 -0500
Subject: [PATCH 0198/2492] mon/ConfigMap: push kv parsing logic into
 add_option()

Push shared code into ConfigMap from mon + mgr.  Pass in a function to
fetch the Option*, since in the mon case it may come from somewhere other
than g_conf.

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit d8ccd221549cdd28702932deb148f075cb6d88b3)

Fixed conflict: `mopt.localized_name = name;` is added to add_option()
(due to 3821722e5660437298a7c0f41e1061d363090103)
---
 src/mgr/ActivePyModules.cc | 41 +++--------------------
 src/mon/ConfigMap.cc       | 57 ++++++++++++++++++++++++++++++++
 src/mon/ConfigMap.h        |  7 ++++
 src/mon/ConfigMonitor.cc   | 68 +++++++++-----------------------------
 4 files changed, 84 insertions(+), 89 deletions(-)

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 701b41bef8e1..4f75670591af 100644
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -866,42 +866,11 @@ void ActivePyModules::_refresh_config_map()
     string who;
     config_map.parse_key(key, &name, &who);
 
-    const Option *opt = g_conf().find_option(name);
-    if (!opt) {
-      config_map.stray_options.push_back(
-	std::unique_ptr<Option>(
-	  new Option(name, Option::TYPE_STR, Option::LEVEL_UNKNOWN)));
-      opt = config_map.stray_options.back().get();
-    }
-
-    string err;
-    int r = opt->pre_validate(&value, &err);
-    if (r < 0) {
-      dout(10) << __func__ << " pre-validate failed on '" << name << "' = '"
-	       << value << "' for " << name << dendl;
-    }
-
-    MaskedOption mopt(opt);
-    mopt.raw_value = value;
-    string section_name;
-    if (who.size() &&
-	!ConfigMap::parse_mask(who, &section_name, &mopt.mask)) {
-      derr << __func__ << " invalid mask for key " << key << dendl;
-    } else if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
-      dout(10) << __func__ << " NO_MON_UPDATE option '"
-	       << name << "' = '" << value << "' for " << name
-	       << dendl;
-    } else {
-      Section *section = &config_map.global;;
-      if (section_name.size() && section_name != "global") {
-	if (section_name.find('.') != std::string::npos) {
-	  section = &config_map.by_id[section_name];
-	} else {
-	  section = &config_map.by_type[section_name];
-	}
-      }
-      section->options.insert(make_pair(name, std::move(mopt)));
-    }
+    config_map.add_option(
+      g_ceph_context, name, who, value,
+      [&](const std::string& name) {
+	return  g_conf().find_option(name);
+      });
   }
 }
 
diff --git a/src/mon/ConfigMap.cc b/src/mon/ConfigMap.cc
index dff1a94c1e6f..86528c1dedfe 100644
--- a/src/mon/ConfigMap.cc
+++ b/src/mon/ConfigMap.cc
@@ -7,6 +7,10 @@
 #include "crush/CrushWrapper.h"
 #include "common/entity_name.h"
 
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#include "common/dout.h"
+
 using namespace std::literals;
 
 using std::cerr;
@@ -250,6 +254,59 @@ void ConfigMap::parse_key(
   }
 }
 
+int ConfigMap::add_option(
+  CephContext *cct,
+  const std::string& name,
+  const std::string& who,
+  const std::string& orig_value,
+  std::function<const Option *(const std::string&)> get_opt)
+{
+  const Option *opt = get_opt(name);
+  if (!opt) {
+    ldout(cct, 10) << __func__ << " unrecognized option '" << name << "'" << dendl;
+    stray_options.push_back(
+      std::unique_ptr<Option>(
+	new Option(name, Option::TYPE_STR, Option::LEVEL_UNKNOWN)));
+    opt = stray_options.back().get();
+  }
+
+  string err;
+  string value = orig_value;
+  int r = opt->pre_validate(&value, &err);
+  if (r < 0) {
+    ldout(cct, 10) << __func__ << " pre-validate failed on '" << name << "' = '"
+		   << value << "' for " << name << dendl;
+  }
+
+  int ret = 0;
+  MaskedOption mopt(opt);
+  mopt.raw_value = value;
+  mopt.localized_name = name;
+  string section_name;
+  if (who.size() &&
+      !ConfigMap::parse_mask(who, &section_name, &mopt.mask)) {
+    lderr(cct) << __func__ << " invalid mask for option " << name << " mask " << who
+	       << dendl;
+    ret = -EINVAL;
+  } else if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
+    ldout(cct, 10) << __func__ << " NO_MON_UPDATE option '"
+		   << name << "' = '" << value << "' for " << name
+		   << dendl;
+    ret = -EINVAL;
+  } else {
+    Section *section = &global;;
+    if (section_name.size() && section_name != "global") {
+      if (section_name.find('.') != std::string::npos) {
+	section = &by_id[section_name];
+      } else {
+	section = &by_type[section_name];
+      }
+    }
+    section->options.insert(make_pair(name, std::move(mopt)));
+  }
+  return ret;
+}
+
 
 // --------------
 
diff --git a/src/mon/ConfigMap.h b/src/mon/ConfigMap.h
index bf2bd97c0ba3..5a14d089a720 100644
--- a/src/mon/ConfigMap.h
+++ b/src/mon/ConfigMap.h
@@ -149,6 +149,13 @@ struct ConfigMap {
     const std::string& in,
     std::string *section,
     OptionMask *mask);
+
+  int add_option(
+    CephContext *cct,
+    const std::string& name,
+    const std::string& who,
+    const std::string& value,
+    std::function<const Option *(const std::string&)> get_opt);
 };
 
 
diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc
index a5f61cbea529..6480674926da 100644
--- a/src/mon/ConfigMonitor.cc
+++ b/src/mon/ConfigMonitor.cc
@@ -785,9 +785,10 @@ void ConfigMonitor::load_config()
 
   unsigned num = 0;
   KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX);
-  it->lower_bound(KEY_PREFIX);
-  while (it->valid() &&
-	 it->key().compare(0, KEY_PREFIX.size(), KEY_PREFIX) == 0) {
+  for (it->lower_bound(KEY_PREFIX);
+       it->valid() &&
+	 it->key().compare(0, KEY_PREFIX.size(), KEY_PREFIX) == 0;
+       it->next(), ++num) {
     string key = it->key().substr(KEY_PREFIX.size());
     string value = it->value().to_str();
 
@@ -811,58 +812,19 @@ void ConfigMonitor::load_config()
       }
     }
 
-    const Option *opt = g_conf().find_option(name);
-    if (!opt) {
-      opt = mon.mgrmon()->find_module_option(name);
-    }
-    if (!opt) {
-      dout(10) << __func__ << " unrecognized option '" << name << "'" << dendl;
-      config_map.stray_options.push_back(
-	std::unique_ptr<Option>(
-	  new Option(name, Option::TYPE_STR, Option::LEVEL_UNKNOWN)));
-      opt = config_map.stray_options.back().get();
-    }
-
-    string err;
-    int r = opt->pre_validate(&value, &err);
-    if (r < 0) {
-      dout(10) << __func__ << " pre-validate failed on '" << name << "' = '"
-	       << value << "' for " << name << dendl;
-    }
-    
-    MaskedOption mopt(opt);
-    mopt.raw_value = value;
-    mopt.localized_name = name;
-    string section_name;
-    if (who.size() &&
-	!ConfigMap::parse_mask(who, &section_name, &mopt.mask)) {
-      derr << __func__ << " invalid mask for key " << key << dendl;
-      pending_cleanup[key].reset();
-    } else if (opt->has_flag(Option::FLAG_NO_MON_UPDATE)) {
-      dout(10) << __func__ << " NO_MON_UPDATE option '"
-	       << name << "' = '" << value << "' for " << name
-	       << dendl;
-      pending_cleanup[key].reset();
-    } else {
-      if (section_name.empty()) {
-	// we prefer global/$option instead of just $option
-	derr << __func__ << " adding global/ prefix to key '" << key << "'"
-	     << dendl;
-	pending_cleanup[key].reset();
-	pending_cleanup["global/"s + key] = it->value();
-      }
-      Section *section = &config_map.global;;
-      if (section_name.size() && section_name != "global") {
-	if (section_name.find('.') != std::string::npos) {
-	  section = &config_map.by_id[section_name];
-	} else {
-	  section = &config_map.by_type[section_name];
+    int r = config_map.add_option(
+      g_ceph_context, name, who, value,
+      [&](const std::string& name) {
+	const Option *opt = g_conf().find_option(name);
+	if (!opt) {
+	  opt = mon.mgrmon()->find_module_option(name);
 	}
-      }
-      section->options.insert(make_pair(name, std::move(mopt)));
-      ++num;
+	return opt;
+      });
+    if (r == -EINVAL) {
+      dout(10) << __func__ << " will clean up key " << key << dendl;
+      pending_cleanup[key].reset();
     }
-    it->next();
   }
   dout(10) << __func__ << " got " << num << " keys" << dendl;
 

From 76755dad4cee1bcf8283652296c84d436f766374 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 16 Oct 2023 19:16:04 +0530
Subject: [PATCH 0199/2492] qa: fix error reporting string in
 assert_cluster_log

Fixes: https://tracker.ceph.com/issues/63217
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/ceph_test_case.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py
index 649c0e53cf96..f5f1445a059b 100644
--- a/qa/tasks/ceph_test_case.py
+++ b/qa/tasks/ceph_test_case.py
@@ -255,7 +255,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
                 if present and not self.match():
                     log.error(f"Log output: \n{self.watcher_process.stdout.getvalue()}\n")
-                    raise AssertionError(f"Expected log message found: '{expected_pattern}'")
+                    raise AssertionError(f"Expected log message not found: '{expected_pattern}'")
                 elif fail or (not present and self.match()):
                     log.error(f"Log output: \n{self.watcher_process.stdout.getvalue()}\n")
                     raise AssertionError(f"Unexpected log message found: '{expected_pattern}'")

From b8c6ba01b262d33084b9c607d829503ac8ae25c5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 3 Oct 2023 16:01:40 +0000
Subject: [PATCH 0200/2492] CMakeLists: warn if CMAKE_BUILD_TYPE is set/unset

At some point the debug builds for wip branches no longer had the .git
directory available so the Debug build type was unset. This meant we are
no longer doing numerous checks (like mutex ownership checks) that we
would normally be doing in the qa suite.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9e2af5e35634..34d035b4df86 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,9 +24,12 @@ endforeach()
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
 if(NOT CMAKE_BUILD_TYPE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
+  message(WARNING "CMAKE_BUILD_TYPE not specified, assuming CMAKE_BUILD_TYPE=Debug because .git exists.")
   set(default_build_type "Debug")
   set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
       STRING "Default BUILD_TYPE is Debug, other options are: RelWithDebInfo, Release, and MinSizeRel." FORCE)
+else()
+  message(WARNING "CMAKE_BUILD_TYPE not specified, leaving unset because .git does NOT exist.")
 endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")

From 83bd3a8dfb83c46a2ba84d9f309a6dbe72c64ea9 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sun, 15 Oct 2023 21:27:44 -0400
Subject: [PATCH 0201/2492] doc/start: Modernize and clarify
 hardware-recommendations.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/start/hardware-recommendations.rst | 391 ++++++++++++++++---------
 1 file changed, 252 insertions(+), 139 deletions(-)

diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index c759d7495a0c..f3f8afdf164a 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -1,66 +1,83 @@
 .. _hardware-recommendations:
 
 ==========================
- Hardware Recommendations
+ hardware recommendations
 ==========================
 
-Ceph was designed to run on commodity hardware, which makes building and
-maintaining petabyte-scale data clusters economically feasible. 
-When planning out your cluster hardware, you will need to balance a number 
-of considerations, including failure domains and potential performance
-issues. Hardware planning should include distributing Ceph daemons and 
+Ceph is designed to run on commodity hardware, which makes building and
+maintaining petabyte-scale data clusters flexible and economically feasible. 
+When planning your cluster's hardware, you will need to balance a number 
+of considerations, including failure domains, cost, and performance.
+Hardware planning should include distributing Ceph daemons and 
 other processes that use Ceph across many hosts. Generally, we recommend 
 running Ceph daemons of a specific type on a host configured for that type 
-of daemon. We recommend using other hosts for processes that utilize your 
-data cluster (e.g., OpenStack, CloudStack, etc).
+of daemon. We recommend using separate hosts for processes that utilize your 
+data cluster (e.g., OpenStack, CloudStack, Kubernetes, etc).
 
+The requirements of one Ceph cluster are not the same as the requirements of
+another, but below are some general guidelines. 
 
-.. tip:: Check out the `Ceph blog`_ too.
-
+.. tip:: check out the `ceph blog`_ too.
 
 CPU
 ===
 
-CephFS metadata servers (MDS) are CPU-intensive. CephFS metadata servers (MDS)
-should therefore have quad-core (or better) CPUs and high clock rates (GHz). OSD
-nodes need enough processing power to run the RADOS service, to calculate data
+CephFS Metadata Servers (MDS) are CPU-intensive. They are
+are single-threaded and perform best with CPUs with a high clock rate (GHz). MDS
+servers do not need a large number of CPU cores unless they are also hosting other
+services, such as SSD OSDs for the CephFS metadata pool.
+OSD nodes need enough processing power to run the RADOS service, to calculate data
 placement with CRUSH, to replicate data, and to maintain their own copies of the
 cluster map.
 
-The requirements of one Ceph cluster are not the same as the requirements of
-another, but here are some general guidelines. 
-
-In earlier versions of Ceph, we would make hardware recommendations based on
-the number of cores per OSD, but this cores-per-OSD metric is no longer as
-useful a metric as the number of cycles per IOP and the number of IOPs per OSD.
-For example, for NVMe drives, Ceph can easily utilize five or six cores on real
+With earlier releases of Ceph, we would make hardware recommendations based on
+the number of cores per OSD, but this cores-per-osd metric is no longer as
+useful a metric as the number of cycles per IOP and the number of IOPS per OSD.
+For example, with NVMe OSD drives, Ceph can easily utilize five or six cores on real
 clusters and up to about fourteen cores on single OSDs in isolation. So cores
 per OSD are no longer as pressing a concern as they were. When selecting
-hardware, select for IOPs per core.
+hardware, select for IOPS per core.
 
-Monitor nodes and manager nodes have no heavy CPU demands and require only
-modest processors. If your host machines will run CPU-intensive processes in
+.. tip:: When we speak of CPU _cores_, we mean _threads_ when hyperthreading
+	 is enabled.  Hyperthreading is usually beneficial for Ceph servers.
+
+Monitor nodes and Manager nodes do not have heavy CPU demands and require only
+modest processors. if your hosts will run CPU-intensive processes in
 addition to Ceph daemons, make sure that you have enough processing power to
 run both the CPU-intensive processes and the Ceph daemons. (OpenStack Nova is
-one such example of a CPU-intensive process.) We recommend that you run
+one example of a CPU-intensive process.) We recommend that you run
 non-Ceph CPU-intensive processes on separate hosts (that is, on hosts that are
-not your monitor and manager nodes) in order to avoid resource contention.
+not your Monitor and Manager nodes) in order to avoid resource contention.
+If your cluster deployes the Ceph Object Gateway, RGW daemons may co-reside
+with your Mon and Manager services if the nodes have sufficient resources.
 
 RAM
 ===
 
-Generally, more RAM is better.  Monitor / manager nodes for a modest cluster
+Generally, more RAM is better.  Monitor / Manager nodes for a modest cluster
 might do fine with 64GB; for a larger cluster with hundreds of OSDs 128GB
-is a reasonable target.  There is a memory target for BlueStore OSDs that
+is advised.
+
+.. tip:: when we speak of RAM and storage requirements, we often describe
+	 the needs of a single daemon of a given type.  A given server as
+	 a whole will thus need at least the sum of the needs of the
+	 daemons that it hosts as well as resources for logs and other operating
+	 system components.  Keep in mind that a server's need for RAM
+	 and storage will be greater at startup and when components
+	 fail or are added and the cluster rebalances.  In other words,
+	 allow headroom past what you might see used during a calm period
+	 on a small initial cluster footprint.
+
+There is an :confval:`osd_memory_target` setting for BlueStore OSDs that
 defaults to 4GB.  Factor in a prudent margin for the operating system and
 administrative tasks (like monitoring and metrics) as well as increased
-consumption during recovery:  provisioning ~8GB per BlueStore OSD
-is advised.
+consumption during recovery:  provisioning ~8GB *per BlueStore OSD* is thus
+advised.
 
 Monitors and managers (ceph-mon and ceph-mgr)
 ---------------------------------------------
 
-Monitor and manager daemon memory usage generally scales with the size of the
+Monitor and manager daemon memory usage scales with the size of the
 cluster.  Note that at boot-time and during topology changes and recovery these
 daemons will need more RAM than they do during steady-state operation, so plan
 for peak usage. For very small clusters, 32 GB suffices. For clusters of up to,
@@ -75,8 +92,8 @@ tuning the following settings:
 Metadata servers (ceph-mds)
 ---------------------------
 
-The metadata daemon memory utilization depends on how much memory its cache is
-configured to consume.  We recommend 1 GB as a minimum for most systems.  See
+CephFS metadata daemon memory utilization depends on the configured size of
+its cache. We recommend 1 GB as a minimum for most systems.  See
 :confval:`mds_cache_memory_limit`.
 
 
@@ -88,23 +105,24 @@ operating system's page cache. In Bluestore you can adjust the amount of memory
 that the OSD attempts to consume by changing the :confval:`osd_memory_target`
 configuration option.
 
-- Setting the :confval:`osd_memory_target` below 2GB is typically not
-  recommended (Ceph may fail to keep the memory consumption under 2GB and 
-  this may cause extremely slow performance).
+- Setting the :confval:`osd_memory_target` below 2GB is not
+  recommended.  eph may fail to keep the memory consumption under 2GB and 
+  extremely slow performance is likely.
 
 - Setting the memory target between 2GB and 4GB typically works but may result
-  in degraded performance: metadata may be read from disk during IO unless the
-  active data set is relatively small.
+  in degraded performance: metadata may need to be read from disk during IO
+  unless the active data set is relatively small.
 
-- 4GB is the current default :confval:`osd_memory_target` size. This default
-  was chosen for typical use cases, and is intended to balance memory
-  requirements and OSD performance.
+- 4GB is the current default value for :confval:`osd_memory_target` This default
+  was chosen for typical use cases, and is intended to balance RAM cost and
+  OSD performance.
 
 - Setting the :confval:`osd_memory_target` higher than 4GB can improve
   performance when there many (small) objects or when large (256GB/OSD 
-  or more) data sets are processed.
+  or more) data sets are processed.  This is especially true with fast
+  NVMe OSDs.
 
-.. important:: OSD memory autotuning is "best effort". Although the OSD may
+.. important:: OSD memory management is "best effort". Although the OSD may
    unmap memory to allow the kernel to reclaim it, there is no guarantee that
    the kernel will actually reclaim freed memory within a specific time
    frame. This applies especially in older versions of Ceph, where transparent
@@ -113,14 +131,19 @@ configuration option.
    pages at the application level to avoid this, but that does not
    guarantee that the kernel will immediately reclaim unmapped memory. The OSD
    may still at times exceed its memory target. We recommend budgeting 
-   approximately 20% extra memory on your system to prevent OSDs from going OOM
+   at least 20% extra memory on your system to prevent OSDs from going OOM
    (**O**\ut **O**\f **M**\emory) during temporary spikes or due to delay in
    the kernel reclaiming freed pages. That 20% value might be more or less than
    needed, depending on the exact configuration of the system.
 
-When using the legacy FileStore back end, the page cache is used for caching
-data, so no tuning is normally needed. When using the legacy FileStore backend,
-the OSD memory consumption is related to the number of PGs per daemon in the
+.. tip:: Configuring the operating system with swap to provide additional
+	 virtual memory for daemons is not advised for modern systems.  Doing
+	 may result in lower performance, and your Ceph cluster may well be
+	 happier with a daemon that crashes vs one that slows to a crawl.
+
+When using the legacy FileStore back end, the OS page cache was used for caching
+data, so tuning was not normally needed. When using the legacy FileStore backend,
+the OSD memory consumption was related to the number of PGs per daemon in the
 system.
 
 
@@ -130,13 +153,34 @@ Data Storage
 Plan your data storage configuration carefully. There are significant cost and
 performance tradeoffs to consider when planning for data storage. Simultaneous
 OS operations and simultaneous requests from multiple daemons for read and
-write operations against a single drive can slow performance.
+write operations against a single drive can impact performance.
+
+OSDs require substantial storage drive space for RADOS data. We recommend a
+minimum drive size of 1 terabyte. OSD drives much smaller than one terabyte
+use a significant fraction of their capacity for metadata, and drives smaller
+than 100 gigabytes will not be effective at all.
+
+It is *strongly* suggested that (enterprise-class) SSDs are provisioned for, at a
+minimum, Ceph Monitor and Ceph Manager hosts, as well as CephFS Metadata Server
+metadata pools and Ceph Object Gateway (RGW) index pools, even if HDDs are to
+be provisioned for bulk OSD data.
+
+To get the best performance out of Ceph, provision the following on separate
+drives:
+
+* The operating systems
+* OSD data
+* BlueStore WAL+DB
+
+For more
+information on how to effectively use a mix of fast drives and slow drives in
+your Ceph cluster, see the `block and block.db`_ section of the Bluestore
+Configuration Reference.
 
 Hard Disk Drives
 ----------------
 
-OSDs should have plenty of storage drive space for object data. We recommend a
-minimum disk drive size of 1 terabyte. Consider the cost-per-gigabyte advantage
+Consider carefully the cost-per-gigabyte advantage
 of larger disks. We recommend dividing the price of the disk drive by the
 number of gigabytes to arrive at a cost per gigabyte, because larger drives may
 have a significant impact on the cost-per-gigabyte. For example, a 1 terabyte
@@ -146,11 +190,10 @@ per gigabyte (i.e., $150 / 3072 = 0.0488). In the foregoing example, using the
 1 terabyte disks would generally increase the cost per gigabyte by
 40%--rendering your cluster substantially less cost efficient.
 
-.. tip:: Running multiple OSDs on a single SAS / SATA drive
-   is **NOT** a good idea.  NVMe drives, however, can achieve
-   improved performance by being split into two or more OSDs.
+.. tip:: Hosting multiple OSDs on a single SAS / SATA HDD
+   is **NOT** a good idea.
 
-.. tip:: Running an OSD and a monitor or a metadata server on a single 
+.. tip:: Hosting an OSD with monitor, manager, or MDS data on a single 
    drive is also **NOT** a good idea.
 
 .. tip:: With spinning disks, the SATA and SAS interface increasingly
@@ -162,35 +205,36 @@ Storage drives are subject to limitations on seek time, access time, read and
 write times, as well as total throughput. These physical limitations affect
 overall system performance--especially during recovery. We recommend using a
 dedicated (ideally mirrored) drive for the operating system and software, and
-one drive for each Ceph OSD Daemon you run on the host (modulo NVMe above).
+one drive for each Ceph OSD Daemon you run on the host.
 Many "slow OSD" issues (when they are not attributable to hardware failure)
 arise from running an operating system and multiple OSDs on the same drive.
+Also be aware that today's 22TB HDD uses the same SATA interface as a
+3TB HDD from ten years ago: more than seven times the data to squeeze
+through the same same interface.  For this reason, when using HDDs for
+OSDs, drives larger than 8TB may be best suited for storage of large
+files / objects that are not at all performance-sensitive.
 
-It is technically possible to run multiple Ceph OSD Daemons per SAS / SATA
-drive, but this will lead to resource contention and diminish overall
-throughput.
-
-To get the best performance out of Ceph, run the following on separate drives:
-(1) operating systems, (2) OSD data, and (3) BlueStore db.  For more
-information on how to effectively use a mix of fast drives and slow drives in
-your Ceph cluster, see the `block and block.db`_ section of the Bluestore
-Configuration Reference.
 
 Solid State Drives
 ------------------
 
-Ceph performance can be improved by using solid-state drives (SSDs). This
-reduces random access time and reduces latency while accelerating throughput. 
+Ceph performance is much improved when using solid-state drives (SSDs). This
+reduces random access time and reduces latency while increasing throughput. 
 
-SSDs cost more per gigabyte than do hard disk drives, but SSDs often offer
-access times that are, at a minimum, 100 times faster than hard disk drives.
+SSDs cost more per gigabyte than do HDDs but SSDs often offer
+access times that are, at a minimum, 100 times faster than HDDs.
 SSDs avoid hotspot issues and bottleneck issues within busy clusters, and
-they may offer better economics when TCO is evaluated holistically.
-
-SSDs do not have moving mechanical parts, so they are not necessarily subject
-to the same types of limitations as hard disk drives. SSDs do have significant
+they may offer better economics when TCO is evaluated holistically. Notably,
+the amortized drive cost for a given number of IOPS is much lower with SSDs
+than with HDDs.  SSDs do not suffer rotational or seek latency and in addition
+to improved client performance, they substantially improve the speed and
+client impact of cluster changes including rebalancing when OSDs or Monitors
+are added, removed, or fail.
+
+SSDs do not have moving mechanical parts, so they are not subject
+to many of the limitations of HDDs.  SSDs do have significant
 limitations though. When evaluating SSDs, it is important to consider the
-performance of sequential reads and writes.
+performance of sequential and random reads and writes.
 
 .. important:: We recommend exploring the use of SSDs to improve performance. 
    However, before making a significant investment in SSDs, we **strongly
@@ -198,16 +242,36 @@ performance of sequential reads and writes.
    SSD in a test configuration in order to gauge performance. 
 
 Relatively inexpensive SSDs may appeal to your sense of economy. Use caution.
-Acceptable IOPS are not the only factor to consider when selecting an SSD for
-use with Ceph. 
-
-SSDs have historically been cost prohibitive for object storage, but emerging
-QLC drives are closing the gap, offering greater density with lower power
-consumption and less power spent on cooling. HDD OSDs may see a significant
-performance improvement by offloading WAL+DB onto an SSD.
-
-To get a better sense of the factors that determine the cost of storage, you
-might use the `Storage Networking Industry Association's Total Cost of
+Acceptable IOPS are not the only factor to consider when selecting SSDs for
+use with Ceph. Bargain SSDs are often a false economy: they may experience
+"cliffing", which means that after an initial burst, sustained performance
+once a limited cache is filled declines considerably.  Consider also durability:
+a drive rated for 0.3 Drive Writes Per Day (DWPD or equivalent) may be fine for
+OSDs dedicated to certain types of sequentially-written read-mostly data, but
+are not a good choice for Ceph Monitor duty.  Enterprise-class SSDs are best
+for Ceph:  they almost always feature power less protection (PLP) and do
+not suffer the dramatic cliffing that client (desktop) models may experience.
+
+When using a single (or mirrored pair) SSD for both operating system boot
+and Ceph Monitor / Manager purposes, a minimum capacity of 256GB is advised
+and at least 480GB is recommended. A drive model rated at 1+ DWPD (or the
+equivalent in TBW (TeraBytes Written) is suggested.  However, for a given write
+workload, a larger drive than technically required will provide more endurance
+because it effectively has greater overprovsioning. We stress that
+enterprise-class drives are best for production use, as they feature power
+loss protection and increased durability compared to client (desktop) SKUs
+that are intended for much lighter and intermittent duty cycles.
+
+SSDs were historically been cost prohibitive for object storage, but
+QLC SSDs are closing the gap, offering greater density with lower power
+consumption and less power spent on cooling. Also, HDD OSDs may see a
+significant write latency improvement by offloading WAL+DB onto an SSD.
+Many Ceph OSD deployments do not require an SSD with greater endurance than
+1 DWPD (aka "read-optimized").  "Mixed-use" SSDs in the 3 DWPD class are
+often overkill for this purpose and cost signficantly more.
+
+To get a better sense of the factors that determine the total cost of storage,
+you might use the `Storage Networking Industry Association's Total Cost of
 Ownership calculator`_
 
 Partition Alignment
@@ -222,11 +286,11 @@ alignment and example commands that show how to align partitions properly, see
 CephFS Metadata Segregation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-One way that Ceph accelerates CephFS file system performance is by segregating
+One way that Ceph accelerates CephFS file system performance is by separating
 the storage of CephFS metadata from the storage of the CephFS file contents.
 Ceph provides a default ``metadata`` pool for CephFS metadata. You will never
-have to create a pool for CephFS metadata, but you can create a CRUSH map
-hierarchy for your CephFS metadata pool that points only to SSD storage media.
+have to manually create a pool for CephFS metadata, but you can create a CRUSH map
+hierarchy for your CephFS metadata pool that includes only SSD storage media.
 See :ref:`CRUSH Device Class<crush-map-device-class>` for details.
 
 
@@ -237,8 +301,20 @@ Disk controllers (HBAs) can have a significant impact on write throughput.
 Carefully consider your selection of HBAs to ensure that they do not create a
 performance bottleneck. Notably, RAID-mode (IR) HBAs may exhibit higher latency
 than simpler "JBOD" (IT) mode HBAs. The RAID SoC, write cache, and battery
-backup can substantially increase hardware and maintenance costs. Some RAID
-HBAs can be configured with an IT-mode "personality".
+backup can substantially increase hardware and maintenance costs. Many RAID
+HBAs can be configured with an IT-mode "personality" or "JBOD mode" for
+streamlined operation.
+
+You do not need an RoC (RAID-capable) HBA. ZFS or Linux MD software mirroring
+serve well for boot volume durability.  When using SAS or SATA data drives,
+forgoing HBA RAID capabilities can reduce the gap between HDD and SSD
+media cost.  Moreover, when using NVMe SSDs, you do not need *any* HBA.  This
+additionally reduces the HDD vs SSD cost gap when the system as a whole is
+considered. The initial cost of a fancy RAID HBA plus onboard cache plus
+battery backup (BBU or supercapacitor) can easily exceed more than 1000 US
+dollars even after discounts - a sum that goes a log way toward SSD cost parity.
+An HBA-free system may also cost hundreds of US dollars less every year if one
+purchases an annual maintenance contract or extended warranty.
 
 .. tip:: The `Ceph blog`_ is often an excellent source of information on Ceph
    performance issues. See `Ceph Write Throughput 1`_ and `Ceph Write 
@@ -248,10 +324,10 @@ HBAs can be configured with an IT-mode "personality".
 Benchmarking
 ------------
 
-BlueStore opens block devices in O_DIRECT and uses fsync frequently to ensure
-that data is safely persisted to media. You can evaluate a drive's low-level
-write performance using ``fio``. For example, 4kB random write performance is
-measured as follows:
+BlueStore opens storage devices with ``O_DIRECT`` and issues ``fsync()``
+frequently to ensure that data is safely persisted to media. You can evaluate a
+drive's low-level write performance using ``fio``. For example, 4kB random write
+performance is measured as follows:
 
 .. code-block:: console
 
@@ -261,6 +337,7 @@ Write Caches
 ------------
 
 Enterprise SSDs and HDDs normally include power loss protection features which
+ensure data durability when power is lost while operating, and
 use multi-level caches to speed up direct or synchronous writes.  These devices
 can be toggled between two caching modes -- a volatile cache flushed to
 persistent media with fsync, or a non-volatile cache written synchronously.
@@ -269,9 +346,9 @@ These two modes are selected by either "enabling" or "disabling" the write
 (volatile) cache.  When the volatile cache is enabled, Linux uses a device in
 "write back" mode, and when disabled, it uses "write through".
 
-The default configuration (normally caching enabled) may not be optimal, and
+The default configuration (usually: caching is enabled) may not be optimal, and
 OSD performance may be dramatically increased in terms of increased IOPS and
-decreased commit_latency by disabling the write cache.
+decreased commit latency by disabling this write cache.
 
 Users are therefore encouraged to benchmark their devices with ``fio`` as
 described earlier and persist the optimal cache configuration for their
@@ -319,11 +396,11 @@ The write cache can be disabled with those same tools:
   === START OF ENABLE/DISABLE COMMANDS SECTION ===
   Write cache disabled
 
-Normally, disabling the cache using ``hdparm``, ``sdparm``, or ``smartctl``
+In most cases, disabling this cache  using ``hdparm``, ``sdparm``, or ``smartctl``
 results in the cache_type changing automatically to "write through". If this is
-not the case, you can try setting it directly as follows. (Users should note
+not the case, you can try setting it directly as follows. (Users should ensure
 that setting cache_type also correctly persists the caching mode of the device
-until the next reboot):
+until the next reboot as some drives require this to be repeated at every boot):
 
 .. code-block:: console
 
@@ -367,13 +444,13 @@ until the next reboot):
 Additional Considerations
 -------------------------
 
-You typically will run multiple OSDs per host, but you should ensure that the
-aggregate throughput of your OSD drives doesn't exceed the network bandwidth
-required to service a client's need to read or write data. You should also
-consider what percentage of the overall data the cluster stores on each host. If
-the percentage on a particular host is large and the host fails, it can lead to
-problems such as exceeding the ``full ratio``,  which causes Ceph to halt
-operations as a safety precaution that prevents data loss.
+Ceph operators typically provision  multiple OSDs per host, but you should
+ensure that the aggregate throughput of your OSD drives doesn't exceed the
+network bandwidth required to service a client's read and write operations.
+You should also each host's percentage of the cluster's overall capacity. If
+the percentage located on a particular host is large and the host fails, it
+can lead to problems such as recovery causing OSDs to exceed the ``full ratio``,
+which in turn causes Ceph to halt operations to prevent data loss.
 
 When you run multiple OSDs per host, you also need to ensure that the kernel
 is up to date. See `OS Recommendations`_ for notes on ``glibc`` and
@@ -384,7 +461,11 @@ multiple OSDs per host.
 Networks
 ========
 
-Provision at least 10 Gb/s networking in your racks.
+Provision at least 10 Gb/s networking in your datacenter, both among Ceph
+hosts and between clients and your Ceph cluster.  Network link active/active
+bonding across separate network switches is strongly recommended both for
+increased throughput and for tolerance of network failures and maintenance.
+Take care that your bonding hash policy distributes traffic across links.
 
 Speed
 -----
@@ -392,13 +473,20 @@ Speed
 It takes three hours to replicate 1 TB of data across a 1 Gb/s network and it
 takes thirty hours to replicate 10 TB across a 1 Gb/s network. But it takes only
 twenty minutes to replicate 1 TB across a 10 Gb/s network, and it takes
-only one hour to replicate 10 TB across a 10 Gb/s network. 
+only one hour to replicate 10 TB across a 10 Gb/s network.
+
+Note that a 40 Gb/s network link is effectively four 10 Gb/s channels in
+parallel, and that a 100Gb/s network link is effectively four 25 Gb/s channels
+in parallel.  Thus, and perhaps somewhat counterintuitively, an individual
+packet on a 25 Gb/s network has slightly lower latency compared to a 40 Gb/s
+network.
+
 
 Cost
 ----
 
 The larger the Ceph cluster, the more common OSD failures will be.
-The faster that a placement group (PG) can recover from a ``degraded`` state to
+The faster that a placement group (PG) can recover from a degraded state to
 an ``active + clean`` state, the better. Notably, fast recovery minimizes
 the likelihood of multiple, overlapping failures that can cause data to become
 temporarily unavailable or even lost. Of course, when provisioning your
@@ -410,10 +498,10 @@ switches. The added expense of this hardware may be offset by the operational
 cost savings on network setup and maintenance. When using VLANs to handle VM
 traffic between the cluster and compute stacks (e.g., OpenStack, CloudStack,
 etc.), there is additional value in using 10 Gb/s Ethernet or better; 40 Gb/s or
-25/50/100 Gb/s networking as of 2022 is common for production clusters.
+increasingly 25/50/100 Gb/s networking as of 2022 is common for production clusters.
 
-Top-of-rack (TOR) switches also need fast and redundant uplinks to spind
-spine switches / routers, often at least 40 Gb/s.
+Top-of-rack (TOR) switches also need fast and redundant uplinks to
+core / spine network switches or routers, often at least 40 Gb/s.
 
 
 Baseboard Management Controller (BMC)
@@ -425,78 +513,103 @@ Administration and deployment tools may also use BMCs extensively, especially
 via IPMI or Redfish, so consider the cost/benefit tradeoff of an out-of-band
 network for security and administration.  Hypervisor SSH access, VM image uploads,
 OS image installs, management sockets, etc. can impose significant loads on a network.
-Running three networks may seem like overkill, but each traffic path represents
+Running multiple networks may seem like overkill, but each traffic path represents
 a potential capacity, throughput and/or performance bottleneck that you should
 carefully consider before deploying a large scale data cluster.
+
+Additionally BMCs as of 2023 rarely sport network connections faster than 1 Gb/s,
+so dedicated and inexpensive 1 Gb/s switches for BMC administrative traffic
+may reduce costs by wasting fewer expenive ports on faster host switches.
  
 
 Failure Domains
 ===============
 
-A failure domain is any failure that prevents access to one or more OSDs. That
-could be a stopped daemon on a host; a disk failure, an OS crash, a
-malfunctioning NIC, a failed power supply, a network outage, a power outage,
-and so forth. When planning out your hardware needs, you must balance the
-temptation to reduce costs by placing too many responsibilities into too few
-failure domains, and the added costs of isolating every potential failure
-domain.
+A failure domain can be thought of as any component loss that prevents access to
+one or more OSDs or other Ceph daemons. These could be a stopped daemon on a host;
+a storage drive failure, an OS crash, a malfunctioning NIC, a failed power supply,
+a network outage, a power outage, and so forth. When planning your hardware
+deployment, you must balance the risk of reducing costs by placing too many
+responsibilities into too few failure domains against the added costs of
+isolating every potential failure domain.
 
 
 Minimum Hardware Recommendations
 ================================
 
 Ceph can run on inexpensive commodity hardware. Small production clusters
-and development clusters can run successfully with modest hardware.
+and development clusters can run successfully with modest hardware.  As
+we noted above: when we speak of CPU _cores_, we mean _threads_ when
+hyperthreading (HT) is enabled.  Each modern physical x64 CPU core typically
+provides two logical CPU threads; other CPU architectures may vary.
+
+Take care that there are many factors that influence resource choices.  The
+minimum resources that suffice for one purpose will not necessarily suffice for
+another.  A sandbox cluster with one OSD built on a laptop with VirtualBox or on
+a trio of Raspberry PIs will get by with fewer resources than a production
+deployment with a thousand OSDs serving five thousand of RBD clients.  The
+classic Fisher Price PXL 2000 captures video, as does an IMAX or RED camera.
+One would not expect the former to do the job of the latter.  We especially
+cannot stress enough the criticality of using enterprise-quality storage
+media for production workloads.
+
+Additional insights into resource planning for production clusters are
+found above and elsewhere within this documentation.
 
 +--------------+----------------+-----------------------------------------+
-|  Process     | Criteria       | Minimum Recommended                     |
+|  Process     | Criteria       | Bare Minimum and Recommended            |
 +==============+================+=========================================+
-| ``ceph-osd`` | Processor      | - 1 core minimum                        |
-|              |                | - 1 core per 200-500 MB/s               |
+| ``ceph-osd`` | Processor      | - 1 core minimum, 2 recommended         |
+|              |                | - 1 core per 200-500 MB/s throughput    |
 |              |                | - 1 core per 1000-3000 IOPS             |
 |              |                |                                         |
 |              |                | * Results are before replication.       |
-|              |                | * Results may vary with different       |
-|              |                |   CPU models and Ceph features.         |
+|              |                | * Results may vary across CPU and drive |
+|              |                |   models and Ceph configuration:        |
 |              |                |   (erasure coding, compression, etc)    |
 |              |                | * ARM processors specifically may       |
-|              |                |   require additional cores.             |
+|              |                |   require more cores for performance.   |
+|              |                | * SSD OSDs, especially NVMe, will       |
+|              |                |   benefit from additional cores per OSD.|
 |              |                | * Actual performance depends on many    |
 |              |                |   factors including drives, net, and    |
 |              |                |   client throughput and latency.        |
 |              |                |   Benchmarking is highly recommended.   |
 |              +----------------+-----------------------------------------+
 |              | RAM            | - 4GB+ per daemon (more is better)      |
-|              |                | - 2-4GB often functions (may be slow)   |
-|              |                | - Less than 2GB not recommended         |
+|              |                | - 2-4GB may function but may be slow    |
+|              |                | - Less than 2GB is not recommended      |
 |              +----------------+-----------------------------------------+
-|              | Volume Storage |  1x storage drive per daemon            |
+|              | Storage Drives |  1x storage drive per OSD               |
 |              +----------------+-----------------------------------------+
-|              | DB/WAL         |  1x SSD partition per daemon (optional) |
+|              | DB/WAL         |  1x SSD partion per HDD OSD             |
+|              | (optional)     |  4-5x HDD OSDs per DB/WAL SATA SSD      |
+|              |                |  <= 10 HDD OSDss per DB/WAL NVMe SSD    |
 |              +----------------+-----------------------------------------+
-|              | Network        |  1x 1GbE+ NICs (10GbE+ recommended)     |
+|              | Network        |  1x 1Gb/s (bonded 10+ Gb/s recommended) |
 +--------------+----------------+-----------------------------------------+
 | ``ceph-mon`` | Processor      | - 2 cores minimum                       |
 |              +----------------+-----------------------------------------+
-|              | RAM            |  2-4GB+ per daemon                      |
+|              | RAM            |  5GB+ per daemon (large / production    |
+|              |                |  clusters need more)                    |
 |              +----------------+-----------------------------------------+
-|              | Disk Space     |  60 GB per daemon                       |
+|              | Storage        |  100 GB per daemon, SSD is recommended  |
 |              +----------------+-----------------------------------------+
-|              | Network        |  1x 1GbE+ NICs                          |
+|              | Network        |  1x 1Gb/s (10+ Gb/s recommended)        |
 +--------------+----------------+-----------------------------------------+
 | ``ceph-mds`` | Processor      | - 2 cores minimum                       |
 |              +----------------+-----------------------------------------+
-|              | RAM            |  2GB+ per daemon                        |
+|              | RAM            |  2GB+ per daemon (more for production)  |
 |              +----------------+-----------------------------------------+
-|              | Disk Space     |  1 MB per daemon                        |
+|              | Disk Space     |  1 GB per daemon                        |
 |              +----------------+-----------------------------------------+
-|              | Network        |  1x 1GbE+ NICs                          |
+|              | Network        |  1x 1Gb/s (10+ Gb/s recommended)        |
 +--------------+----------------+-----------------------------------------+
 
-.. tip:: If you are running an OSD with a single disk, create a
-   partition for your volume storage that is separate from the partition
-   containing the OS. Generally, we recommend separate disks for the
-   OS and the volume storage.
+.. tip:: If you are running an OSD node with a single storage drive, create a
+   partition for your OSD that is separate from the partition
+   containing the OS. We recommend separate drives for the
+   OS and for OSD storage.
 
 
 

From afba0ffbf5ce50d41150c783b24a1d16b7361249 Mon Sep 17 00:00:00 2001
From: shreyanshjain7174 <ssanchet@redhat.com>
Date: Tue, 17 Oct 2023 08:55:42 +0200
Subject: [PATCH 0202/2492] RadosGW: RGW Swift info not providing
 max_deletes_per_request in bulk_delete section

RGW implements the info endpoint for the Swift protocol which presents a key knowns as bulk_delete, but upon doing swift info the max_deletes_per_request is not shown which is necessary for user to ge the information of a global paramter rgw_delete_multi_obj_max_num in radosgw.

Fixes: https://tracker.ceph.com/issues/62646
Signed-off-by: shreyanshjain7174 <ssanchet@redhat.com>
---
 src/rgw/rgw_rest_swift.cc | 12 +++++++++++-
 src/rgw/rgw_rest_swift.h  |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 1f9657111599..6efc7bd8e1cb 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -1818,7 +1818,7 @@ void RGWGetHealthCheck_ObjStore_SWIFT::send_response()
 
 const vector<pair<string, RGWInfo_ObjStore_SWIFT::info>> RGWInfo_ObjStore_SWIFT::swift_info =
 {
-    {"bulk_delete", {false, nullptr}},
+    {"bulk_delete", {false, RGWInfo_ObjStore_SWIFT::list_bulk_delete}},
     {"container_quotas", {false, nullptr}},
     {"swift", {false, RGWInfo_ObjStore_SWIFT::list_swift_data}},
     {"tempurl", { false, RGWInfo_ObjStore_SWIFT::list_tempurl_data}},
@@ -1870,6 +1870,16 @@ void RGWInfo_ObjStore_SWIFT::send_response()
   rgw_flush_formatter_and_reset(s, s->formatter);
 }
 
+void RGWInfo_ObjStore_SWIFT::list_bulk_delete(Formatter& formatter,
+                                                const ConfigProxy& config,
+                                                rgw::sal::Driver* driver)
+{
+  formatter.open_object_section("bulk_delete");
+  formatter.dump_int("max_deletes_per_request", config->rgw_delete_multi_obj_max_num); 
+  formatter.close_section();
+
+}
+
 void RGWInfo_ObjStore_SWIFT::list_swift_data(Formatter& formatter,
                                               const ConfigProxy& config,
                                               rgw::sal::Driver* driver)
diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h
index 25b20a9700d1..08b5a77dabbe 100644
--- a/src/rgw/rgw_rest_swift.h
+++ b/src/rgw/rgw_rest_swift.h
@@ -238,6 +238,7 @@ class RGWInfo_ObjStore_SWIFT : public RGWInfo_ObjStore {
 
   void execute(optional_yield y) override;
   void send_response() override;
+  static void list_bulk_delete(Formatter& formatter, const ConfigProxy& config, rgw::sal::Driver* driver);
   static void list_swift_data(Formatter& formatter, const ConfigProxy& config, rgw::sal::Driver* driver);
   static void list_tempauth_data(Formatter& formatter, const ConfigProxy& config, rgw::sal::Driver* driver);
   static void list_tempurl_data(Formatter& formatter, const ConfigProxy& config, rgw::sal::Driver* driver);

From a7502cb5125dda8d052dc3991ed5d7013606aca2 Mon Sep 17 00:00:00 2001
From: tanchangzhi <544463199@qq.com>
Date: Tue, 17 Oct 2023 16:48:51 +0800
Subject: [PATCH 0203/2492] doc: Update mClock QOS documentation to discard
 osd_mclock_cost_per_*

The cost parameters (osd_mclock_cost_per_*) have been removed.
The cost of an operation is now determined using the random IOPS
and maximum sequential bandwidth capability of the OSD's underlying device.

Fixes: https://tracker.ceph.com/issues/58529
Signed-off-by: tanchangzhi <544463199@qq.com>
---
 .../osd_internals/mclock_wpq_cmp_study.rst    | 23 -------------------
 1 file changed, 23 deletions(-)

diff --git a/doc/dev/osd_internals/mclock_wpq_cmp_study.rst b/doc/dev/osd_internals/mclock_wpq_cmp_study.rst
index 88e350c862f1..31ad184098ea 100644
--- a/doc/dev/osd_internals/mclock_wpq_cmp_study.rst
+++ b/doc/dev/osd_internals/mclock_wpq_cmp_study.rst
@@ -114,29 +114,6 @@ baseline throughput for each device type was determined:
           256 KiB. For HDDs, it was 40MiB. The above throughput was obtained
           by running 4 KiB random writes at a queue depth of 64 for 300 secs.
 
-Factoring I/O Cost in mClock
-============================
-
-The services using mClock have a cost associated with them. The cost can be
-different for each service type. The mClock scheduler factors in the cost
-during calculations for parameters like *reservation*, *weight* and *limit*.
-The calculations determine when the next op for the service type can be
-dequeued from the operation queue. In general, the higher the cost, the longer
-an op remains in the operation queue.
-
-A cost modeling study was performed to determine the cost per I/O and the cost
-per byte for SSD and HDD device types. The following cost specific options are
-used under the hood by mClock,
-
-- :confval:`osd_mclock_cost_per_io_usec`
-- :confval:`osd_mclock_cost_per_io_usec_hdd`
-- :confval:`osd_mclock_cost_per_io_usec_ssd`
-- :confval:`osd_mclock_cost_per_byte_usec`
-- :confval:`osd_mclock_cost_per_byte_usec_hdd`
-- :confval:`osd_mclock_cost_per_byte_usec_ssd`
-
-See :doc:`/rados/configuration/mclock-config-ref` for more details.
-
 MClock Profile Allocations
 ==========================
 

From 7a54ba3cc1071aeac23709b5f7b91693a158abb6 Mon Sep 17 00:00:00 2001
From: cloudbehl <cloudbehl@gmail.com>
Date: Wed, 9 Aug 2023 12:39:47 +0530
Subject: [PATCH 0204/2492] mgr/dashboard: Left navigation improvements

Fixes: https://tracker.ceph.com/issues/62366

Signed-off-by: cloudbehl <cloudbehl@gmail.com>
---
 .../frontend/cypress/e2e/ui/navigation.po.ts  |  57 ++--
 .../frontend/src/app/app-routing.module.ts    |  26 +-
 .../navigation/navigation.component.html      | 288 +++++++++++-------
 .../navigation/navigation.component.scss      |  25 +-
 .../navigation/navigation.component.spec.ts   |  28 +-
 .../navigation/navigation.component.ts        |   8 +-
 .../src/app/shared/enum/icons.enum.ts         |   9 +-
 7 files changed, 277 insertions(+), 164 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
index f797bbc26a90..7d84939b8807 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
@@ -6,33 +6,18 @@ export class NavigationPageHelper extends PageHelper {
   };
 
   navigations = [
-    { menu: 'NFS', component: 'cd-error' },
-    {
-      menu: 'Object Gateway',
-      submenus: [
-        { menu: 'Gateways', component: 'cd-rgw-daemon-list' },
-        { menu: 'Users', component: 'cd-rgw-user-list' },
-        { menu: 'Buckets', component: 'cd-rgw-bucket-list' }
-      ]
-    },
     { menu: 'Dashboard', component: 'cd-dashboard' },
     {
       menu: 'Cluster',
       submenus: [
+        { menu: 'Pools', component: 'cd-pool-list' },
         { menu: 'Hosts', component: 'cd-hosts' },
-        { menu: 'Physical Disks', component: 'cd-error' },
-        { menu: 'Monitors', component: 'cd-monitor' },
-        { menu: 'Services', component: 'cd-error' },
         { menu: 'OSDs', component: 'cd-osd-list' },
-        { menu: 'Configuration', component: 'cd-configuration' },
+        { menu: 'Physical Disks', component: 'cd-error' },
         { menu: 'CRUSH map', component: 'cd-crushmap' },
-        { menu: 'Manager Modules', component: 'cd-mgr-module-list' },
-        { menu: 'Ceph Users', component: 'cd-crud-table' },
-        { menu: 'Logs', component: 'cd-logs' },
-        { menu: 'Alerts', component: 'cd-prometheus-tabs' }
+        { menu: 'Monitors', component: 'cd-monitor' }
       ]
     },
-    { menu: 'Pools', component: 'cd-pool-list' },
     {
       menu: 'Block',
       submenus: [
@@ -41,7 +26,41 @@ export class NavigationPageHelper extends PageHelper {
         { menu: 'iSCSI', component: 'cd-iscsi' }
       ]
     },
-    { menu: 'File Systems', component: 'cd-cephfs-list' }
+    {
+      menu: 'Object',
+      submenus: [
+        { menu: 'Overview', component: 'cd-rgw-overview-dashboard' },
+        { menu: 'Buckets', component: 'cd-rgw-bucket-list' },
+        { menu: 'Users', component: 'cd-rgw-user-list' },
+        { menu: 'Multi-site', component: 'cd-rgw-multisite-details' },
+        { menu: 'Gateways', component: 'cd-rgw-daemon-list' },
+        { menu: 'NFS', component: 'cd-error' }
+      ]
+    },
+    {
+      menu: 'File',
+      submenus: [
+        { menu: 'File Systems', component: 'cd-cephfs-list' },
+        { menu: 'NFS', component: 'cd-error' }
+      ]
+    },
+    {
+      menu: 'Observability',
+      submenus: [
+        { menu: 'Logs', component: 'cd-logs' },
+        { menu: 'Alerts', component: 'cd-prometheus-tabs' }
+      ]
+    },
+    {
+      menu: 'Administration',
+      submenus: [
+        { menu: 'Services', component: 'cd-error' },
+        { menu: 'Upgrade', component: 'cd-error' },
+        { menu: 'Ceph Users', component: 'cd-crud-table' },
+        { menu: 'Manager Modules', component: 'cd-mgr-module-list' },
+        { menu: 'Configuration', component: 'cd-configuration' }
+      ]
+    }
   ];
 
   getVerticalMenu() {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
index c9ffe72227fb..38ae3a4affd0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
@@ -124,7 +124,7 @@ const routes: Routes = [
         path: 'ceph-users',
         component: CRUDTableComponent,
         data: {
-          breadcrumbs: 'Cluster/Ceph Users',
+          breadcrumbs: 'Administration/Ceph Users',
           resource: 'api.cluster.user@1.0'
         }
       },
@@ -132,7 +132,7 @@ const routes: Routes = [
         path: 'cluster/user/create',
         component: CrudFormComponent,
         data: {
-          breadcrumbs: 'Cluster/Ceph Users/Create',
+          breadcrumbs: 'Administration/Ceph Users/Create',
           resource: 'api.cluster.user@1.0'
         }
       },
@@ -140,7 +140,7 @@ const routes: Routes = [
         path: 'cluster/user/import',
         component: CrudFormComponent,
         data: {
-          breadcrumbs: 'Cluster/Ceph Users/Import',
+          breadcrumbs: 'Administration/Ceph Users/Import',
           resource: 'api.cluster.user@1.0'
         }
       },
@@ -148,7 +148,7 @@ const routes: Routes = [
         path: 'cluster/user/edit',
         component: CrudFormComponent,
         data: {
-          breadcrumbs: 'Cluster/Ceph Users/Edit',
+          breadcrumbs: 'Administration/Ceph Users/Edit',
           resource: 'api.cluster.user@1.0'
         }
       },
@@ -169,7 +169,7 @@ const routes: Routes = [
             section_info: 'Orchestrator',
             header: 'Orchestrator is not available'
           },
-          breadcrumbs: 'Cluster/Services'
+          breadcrumbs: 'Administration/Services'
         },
         children: [
           {
@@ -213,7 +213,7 @@ const routes: Routes = [
       },
       {
         path: 'configuration',
-        data: { breadcrumbs: 'Cluster/Configuration' },
+        data: { breadcrumbs: 'Administration/Configuration' },
         children: [
           { path: '', component: ConfigurationComponent },
           {
@@ -231,7 +231,7 @@ const routes: Routes = [
       {
         path: 'logs',
         component: LogsComponent,
-        data: { breadcrumbs: 'Cluster/Logs' }
+        data: { breadcrumbs: 'Observability/Logs' }
       },
       {
         path: 'telemetry',
@@ -240,7 +240,7 @@ const routes: Routes = [
       },
       {
         path: 'monitoring',
-        data: { breadcrumbs: 'Cluster/Alerts' },
+        data: { breadcrumbs: 'Observability/Alerts' },
         children: [
           { path: '', redirectTo: 'active-alerts', pathMatch: 'full' },
           {
@@ -297,7 +297,7 @@ const routes: Routes = [
             section_info: 'Orchestrator',
             header: 'Orchestrator is not available'
           },
-          breadcrumbs: 'Cluster/Upgrade'
+          breadcrumbs: 'Administration/Upgrade'
         },
         children: [
           {
@@ -321,7 +321,7 @@ const routes: Routes = [
       // Mgr modules
       {
         path: 'mgr-modules',
-        data: { breadcrumbs: 'Cluster/Manager Modules' },
+        data: { breadcrumbs: 'Administrator/Manager Modules' },
         children: [
           {
             path: '',
@@ -339,7 +339,7 @@ const routes: Routes = [
       // Pools
       {
         path: 'pool',
-        data: { breadcrumbs: 'Pools' },
+        data: { breadcrumbs: 'Cluster/Pools' },
         loadChildren: () => import('./ceph/pool/pool.module').then((m) => m.RoutedPoolModule)
       },
       // Block
@@ -352,7 +352,7 @@ const routes: Routes = [
       {
         path: 'cephfs',
         canActivate: [FeatureTogglesGuardService],
-        data: { breadcrumbs: 'File Systems' },
+        data: { breadcrumbs: 'File/File Systems' },
         children: [
           { path: '', component: CephfsListComponent },
           {
@@ -380,7 +380,7 @@ const routes: Routes = [
             header: 'The Object Gateway Service is not configured'
           },
           breadcrumbs: true,
-          text: 'Object Gateway',
+          text: 'Object',
           path: null
         },
         loadChildren: () => import('./ceph/rgw/rgw.module').then((m) => m.RoutedRgwModule)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
index 9c436f7044d5..abd2e1ae6506 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
@@ -5,10 +5,10 @@
   <cd-notifications-sidebar></cd-notifications-sidebar>
   <div class="cd-navbar-top">
     <nav class="navbar navbar-expand-md navbar-dark cd-navbar-brand">
-      <button class="btn btn-link py-0 ms-3"
+      <button class="btn btn-link py-0"
               (click)="showMenuSidebar = !showMenuSidebar"
               aria-label="toggle sidebar visibility">
-        <i class="fa fa-bars fa-2x"
+        <i [ngClass]="[icons.bars, icons.large2x]"
            aria-hidden="true"></i>
       </button>
 
@@ -23,8 +23,8 @@
               (click)="toggleRightSidebar()">
         <span i18n
               class="sr-only">Toggle navigation</span>
-        <span class="">
-          <i class="fa fa-navicon fa-lg"></i>
+        <span>
+          <i [ngClass]="[icons.navicon, icons.large]"></i>
         </span>
       </button>
 
@@ -81,9 +81,14 @@
           class="nav-item tc_menuitem_dashboard">
         <a routerLink="/dashboard"
            class="nav-link">
-          <span i18n>Dashboard</span>&nbsp;
-          <i [ngClass]="[icons.health]"
-             [ngStyle]="summaryData?.health_status | healthColor"></i>
+          <span i18n>
+            <i [ngClass]="[icons.areaChart]"></i>
+            Dashboard</span>
+          <i
+            *ngIf="summaryData?.health_status !== 'HEALTH_OK'"
+            [ngClass]="[icons.circle]"
+            [ngStyle]="summaryData?.health_status | healthColor">
+          </i>
         </a>
       </li>
 
@@ -91,124 +96,79 @@
       <li routerLinkActive="active"
           class="nav-item tc_menuitem_cluster"
           *ngIf="permissions.hosts.read || permissions.monitor.read ||
-          permissions.osd.read || permissions.configOpt.read ||
-          permissions.log.read || permissions.prometheus.read">
+          permissions.osd.read || permissions.pool.read">
         <a (click)="toggleSubMenu('cluster')"
            class="nav-link dropdown-toggle"
-           [attr.aria-expanded]="displayedSubMenu === 'cluster'"
+           [attr.aria-expanded]="displayedSubMenu.cluster"
            aria-controls="cluster-nav"
            role="button">
-          <ng-container i18n>Cluster</ng-container>
+          <ng-container i18n>
+            <i [ngClass]="[icons.sitemap]"></i>
+            Cluster
+          </ng-container>
         </a>
         <ul class="list-unstyled"
             id="cluster-nav"
-            [ngbCollapse]="displayedSubMenu !== 'cluster'">
+            [ngbCollapse]="!displayedSubMenu.cluster">
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_hosts"
-              *ngIf="permissions.hosts.read">
-            <a i18n
-               routerLink="/hosts">Hosts</a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_cluster_inventory"
-              *ngIf="permissions.hosts.read">
-            <a i18n
-               routerLink="/inventory">Physical Disks</a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_cluster_monitor"
-              *ngIf="permissions.monitor.read">
+              class="tc_submenuitem tc_submenuitem_cluster_pool"
+              *ngIf="permissions.pool.read">
             <a i18n
-               routerLink="/monitor/">Monitors</a>
+               routerLink="/pool">Pools</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_cluster_services"
+              class="tc_submenuitem tc_submenuitem_cluster_hosts"
               *ngIf="permissions.hosts.read">
             <a i18n
-               routerLink="/services/">Services</a>
+               routerLink="/hosts">Hosts</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_osds"
+              class="tc_submenuitem tc_submenuitem_cluster_osds"
               *ngIf="permissions.osd.read">
             <a i18n
                routerLink="/osd">OSDs</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_configuration"
-              *ngIf="permissions.configOpt.read">
+              class="tc_submenuitem tc_submenuitem_cluster_inventory"
+              *ngIf="permissions.hosts.read">
             <a i18n
-               routerLink="/configuration">Configuration</a>
+               routerLink="/inventory">Physical Disks</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_crush"
+              class="tc_submenuitem tc_submenuitem_cluster_crush"
               *ngIf="permissions.osd.read">
             <a i18n
                routerLink="/crush-map">CRUSH map</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_modules"
-              *ngIf="permissions.configOpt.read">
-            <a i18n
-               routerLink="/mgr-modules">Manager Modules</a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_users"
-              *ngIf="permissions.configOpt.read">
-            <a i18n
-               routerLink="/ceph-users">Ceph Users</a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_log"
-              *ngIf="permissions.log.read">
-            <a i18n
-               routerLink="/logs">Logs</a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_monitoring"
-              *ngIf="permissions.prometheus.read">
-            <a routerLink="/monitoring">
-              <ng-container i18n>Alerts</ng-container>
-              <small *ngIf="prometheusAlertService.activeCriticalAlerts > 0"
-                     class="badge badge-danger ms-1">{{ prometheusAlertService.activeCriticalAlerts }}</small>
-              <small *ngIf="prometheusAlertService.activeWarningAlerts > 0"
-                     class="badge badge-warning ms-1">{{ prometheusAlertService.activeWarningAlerts }}</small>
-            </a>
-          </li>
-          <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_upgrade"
-              *ngIf="permissions.configOpt.read">
+              class="tc_submenuitem tc_submenuitem_cluster_monitor"
+              *ngIf="permissions.monitor.read">
             <a i18n
-               routerLink="/upgrade">Upgrade</a>
+               routerLink="/monitor/">Monitors</a>
           </li>
         </ul>
       </li>
 
-      <!-- Pools -->
-      <li routerLinkActive="active"
-          class="nav-item tc_menuitem_pool"
-          *ngIf="permissions.pool.read">
-        <a class="nav-link"
-           i18n
-           routerLink="/pool">Pools</a>
-      </li>
-
-      <!-- Block -->
+      <!-- Block Storage -->
       <li routerLinkActive="active"
           class="nav-item tc_menuitem_block"
           *ngIf="(permissions.rbdImage.read || permissions.rbdMirroring.read || permissions.iscsi.read) &&
           (enabledFeature.rbd || enabledFeature.mirroring || enabledFeature.iscsi)">
         <a class="nav-link dropdown-toggle"
            (click)="toggleSubMenu('block')"
-           [attr.aria-expanded]="displayedSubMenu === 'block'"
+           [attr.aria-expanded]="displayedSubMenu.block"
            aria-controls="block-nav"
            role="button"
            [ngStyle]="blockHealthColor()">
-          <ng-container i18n>Block</ng-container>
+          <ng-container i18n>
+            <i [ngClass]="[icons.database]"></i>
+            Block
+          </ng-container>
         </a>
 
         <ul class="list-unstyled"
             id="block-nav"
-            [ngbCollapse]="displayedSubMenu !== 'block'">
+            [ngbCollapse]="!displayedSubMenu.block">
           <li routerLinkActive="active"
               class="tc_submenuitem tc_submenuitem_block_images"
               *ngIf="permissions.rbdImage.read && enabledFeature.rbd">
@@ -237,47 +197,32 @@
         </ul>
       </li>
 
-      <!-- NFS -->
-      <li routerLinkActive="active"
-          class="nav-item tc_menuitem_nfs"
-          *ngIf="permissions.nfs.read && enabledFeature.nfs">
-        <a i18n
-           class="nav-link"
-           routerLink="/nfs">NFS</a>
-      </li>
-
-      <!-- Filesystem -->
-      <li routerLinkActive="active"
-          class="nav-item tc_menuitem_cephfs"
-          *ngIf="permissions.cephfs.read && enabledFeature.cephfs">
-        <a i18n
-           class="nav-link"
-           routerLink="/cephfs">File Systems</a>
-      </li>
-
-      <!-- Object Gateway -->
+      <!-- Object Storage -->
       <li routerLinkActive="active"
           class="nav-item tc_menuitem_rgw"
           *ngIf="permissions.rgw.read && enabledFeature.rgw">
         <a class="nav-link dropdown-toggle"
-           (click)="toggleSubMenu('rgw')"
-           [attr.aria-expanded]="displayedSubMenu === 'rgw'"
+           (click)="toggleSubMenu('object')"
+           [attr.aria-expanded]="displayedSubMenu.object"
            aria-controls="gateway-nav"
            role="button">
-          <ng-container i18n>Object Gateway</ng-container>
+          <ng-container i18n>
+            <i [ngClass]="[icons.cubes]"></i>
+            Object
+          </ng-container>
         </a>
         <ul class="list-unstyled"
             id="gateway-nav"
-            [ngbCollapse]="displayedSubMenu !== 'rgw'">
+            [ngbCollapse]="!displayedSubMenu.object">
           <li routerLinkActive="active"
               class="tc_submenuitem tc_submenuitem_rgw_overview">
             <a i18n
                routerLink="/rgw/overview">Overview</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_rgw_daemons">
+              class="tc_submenuitem tc_submenuitem_rgw_buckets">
             <a i18n
-               routerLink="/rgw/daemon">Gateways</a>
+               routerLink="/rgw/bucket">Buckets</a>
           </li>
           <li routerLinkActive="active"
               class="tc_submenuitem tc_submenuitem_rgw_users">
@@ -287,12 +232,141 @@
           <li routerLinkActive="active"
               class="tc_submenuitem tc_submenuitem_rgw_buckets">
             <a i18n
-               routerLink="/rgw/bucket">Buckets</a>
+               routerLink="/rgw/multisite">Multi-site</a>
           </li>
           <li routerLinkActive="active"
-              class="tc_submenuitem tc_submenuitem_rgw_buckets">
+              class="tc_submenuitem tc_submenuitem_rgw_daemons">
+            <a i18n
+               routerLink="/rgw/daemon">Gateways</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_rgw_nfs"
+              *ngIf="permissions.nfs.read && enabledFeature.nfs">
             <a i18n
-               routerLink="/rgw/multisite">Multi-Site</a>
+               class="nav-link"
+               routerLink="/nfs">NFS</a>
+          </li>
+        </ul>
+      </li>
+
+      <!-- Filesystem -->
+      <li routerLinkActive="active"
+          class="nav-item tc_menuitem_file"
+          *ngIf="permissions.nfs.read && enabledFeature.nfs
+          || permissions.cephfs.read && enabledFeature.cephfs">
+        <a class="nav-link dropdown-toggle"
+           (click)="toggleSubMenu('file')"
+           [attr.aria-expanded]="displayedSubMenu.file"
+           aria-controls="filesystem-nav"
+           role="button">
+          <ng-container i18n>
+            <i [ngClass]="[icons.text]"></i>
+            File
+          </ng-container>
+        </a>
+        <ul class="list-unstyled"
+            id="filesystem-nav"
+            [ngbCollapse]="!displayedSubMenu.file">
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_file_cephfs"
+              *ngIf="permissions.cephfs.read && enabledFeature.cephfs">
+            <a i18n
+               class="nav-link"
+               routerLink="/cephfs">File Systems</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_file_nfs"
+              *ngIf="permissions.nfs.read && enabledFeature.nfs">
+            <a i18n
+               class="nav-link"
+               routerLink="/nfs">NFS</a>
+          </li>
+        </ul>
+      </li>
+
+
+      <!-- Observability -->
+      <li routerLinkActive="active"
+          class="nav-item tc_menuitem_observe"
+          *ngIf="permissions.log.read || permissions.prometheus.read">
+        <a class="nav-link dropdown-toggle"
+           (click)="toggleSubMenu('observe')"
+           [attr.aria-expanded]="displayedSubMenu.observe"
+           aria-controls="observe-nav"
+           role="button">
+          <ng-container i18n>
+            <i [ngClass]="[icons.eye]"></i>
+            Observability
+          </ng-container>
+        </a>
+        <ul class="list-unstyled"
+            id="observe-nav"
+            [ngbCollapse]="!displayedSubMenu.observe">
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_observe_log"
+              *ngIf="permissions.log.read">
+            <a i18n
+               routerLink="/logs">Logs</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_observe_monitoring"
+              *ngIf="permissions.prometheus.read">
+            <a routerLink="/monitoring">
+              <ng-container i18n>Alerts</ng-container>
+              <small *ngIf="prometheusAlertService.activeCriticalAlerts > 0"
+                     class="badge badge-danger ms-1">{{ prometheusAlertService.activeCriticalAlerts }}</small>
+              <small *ngIf="prometheusAlertService.activeWarningAlerts > 0"
+                     class="badge badge-warning ms-1">{{ prometheusAlertService.activeWarningAlerts }}</small>
+            </a>
+          </li>
+        </ul>
+      </li>
+      <li routerLinkActive="active"
+          class="nav-item tc_menuitem_admin"
+          *ngIf="permissions.configOpt.read ||
+          permissions.hosts.read">
+        <a class="nav-link dropdown-toggle"
+           (click)="toggleSubMenu('admin')"
+           [attr.aria-expanded]="displayedSubMenu.admin"
+           aria-controls="admin-nav"
+           role="button">
+          <ng-container i18n>
+            <i [ngClass]="[icons.cogs]"></i>
+            Administration
+          </ng-container>
+        </a>
+        <ul class="list-unstyled"
+            id="admin-nav"
+            [ngbCollapse]="!displayedSubMenu.admin">
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_admin_services"
+              *ngIf="permissions.hosts.read">
+            <a i18n
+               routerLink="/services/">Services</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_admin_upgrade"
+              *ngIf="permissions.configOpt.read">
+            <a i18n
+               routerLink="/upgrade">Upgrade</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_admin_users"
+              *ngIf="permissions.configOpt.read">
+            <a i18n
+               routerLink="/ceph-users">Ceph Users</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_admin_modules"
+              *ngIf="permissions.configOpt.read">
+            <a i18n
+               routerLink="/mgr-modules">Manager Modules</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_admin_configuration"
+              *ngIf="permissions.configOpt.read">
+            <a i18n
+               routerLink="/configuration">Configuration</a>
           </li>
         </ul>
       </li>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
index 9cc5b5d1ac19..8e0dfa364cd9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
@@ -195,9 +195,7 @@ $sidebar-width: 200px;
         color: vv.$white;
         display: block;
         font-size: 1.3em;
-        padding: 10px;
-        padding-left: 27px;
-
+        padding: 10px 23px 10px 10px;
         text-decoration: none;
 
         &:hover {
@@ -208,6 +206,25 @@ $sidebar-width: 200px;
         > .badge {
           margin-left: 5px;
         }
+
+        i.fa.fa-circle {
+          animation: blink 2s ease-in infinite;
+          font-size: 0.875em;
+          margin-top: 4px;
+          position: absolute;
+          right: 35px;
+        }
+
+        @keyframes blink {
+          from,
+          to {
+            opacity: 1;
+          }
+
+          50% {
+            opacity: 0;
+          }
+        }
       }
 
       li.active > a,
@@ -239,7 +256,7 @@ $sidebar-width: 200px;
   ul ul a {
     background: lighten(vv.$secondary, 10);
     font-size: 1.1em !important;
-    padding-left: 40px !important;
+    padding-left: 35px !important;
   }
 
   .cd-navbar-primary a:focus {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.spec.ts
index c8873186eb84..92d9a28878a0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.spec.ts
@@ -102,31 +102,31 @@ describe('NavigationComponent', () => {
       [
         ['hosts'],
         [
-          '.tc_submenuitem_hosts',
+          '.tc_submenuitem_cluster_hosts',
           '.tc_submenuitem_cluster_inventory',
-          '.tc_submenuitem_cluster_services'
+          '.tc_submenuitem_admin_services'
         ]
       ],
       [['monitor'], ['.tc_submenuitem_cluster_monitor']],
-      [['osd'], ['.tc_submenuitem_osds', '.tc_submenuitem_crush']],
+      [['osd'], ['.tc_submenuitem_cluster_osds', '.tc_submenuitem_cluster_crush']],
       [
         ['configOpt'],
         [
-          '.tc_submenuitem_configuration',
-          '.tc_submenuitem_modules',
-          '.tc_submenuitem_users',
-          '.tc_submenuitem_upgrade'
+          '.tc_submenuitem_admin_configuration',
+          '.tc_submenuitem_admin_modules',
+          '.tc_submenuitem_admin_users',
+          '.tc_submenuitem_admin_upgrade'
         ]
       ],
-      [['log'], ['.tc_submenuitem_log']],
-      [['prometheus'], ['.tc_submenuitem_monitoring']],
-      [['pool'], ['.tc_menuitem_pool']],
+      [['log'], ['.tc_submenuitem_observe_log']],
+      [['prometheus'], ['.tc_submenuitem_observe_monitoring']],
+      [['pool'], ['.tc_submenuitem_cluster_pool']],
       [['rbdImage'], ['.tc_submenuitem_block_images']],
       [['rbdMirroring'], ['.tc_submenuitem_block_mirroring']],
       [['iscsi'], ['.tc_submenuitem_block_iscsi']],
       [['rbdImage', 'rbdMirroring', 'iscsi'], ['.tc_menuitem_block']],
-      [['nfs'], ['.tc_menuitem_nfs']],
-      [['cephfs'], ['.tc_menuitem_cephfs']],
+      [['nfs'], ['.tc_submenuitem_file_nfs']],
+      [['cephfs'], ['.tc_submenuitem_file_cephfs']],
       [
         ['rgw'],
         [
@@ -173,8 +173,8 @@ describe('NavigationComponent', () => {
       [['mirroring'], ['.tc_submenuitem_block_mirroring']],
       [['iscsi'], ['.tc_submenuitem_block_iscsi']],
       [['rbd', 'mirroring', 'iscsi'], ['.tc_menuitem_block']],
-      [['nfs'], ['.tc_menuitem_nfs']],
-      [['cephfs'], ['.tc_menuitem_cephfs']],
+      [['nfs'], ['.tc_submenuitem_file_nfs']],
+      [['cephfs'], ['.tc_submenuitem_file_cephfs']],
       [
         ['rgw'],
         [
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index a7cc40f5a823..e5615012e370 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -33,11 +33,11 @@ export class NavigationComponent implements OnInit, OnDestroy {
 
   rightSidebarOpen = false; // rightSidebar only opens when width is less than 768px
   showMenuSidebar = true;
-  displayedSubMenu = '';
 
   simplebar = {
     autoHide: false
   };
+  displayedSubMenu = {};
   private subs = new Subscription();
 
   constructor(
@@ -97,11 +97,7 @@ export class NavigationComponent implements OnInit, OnDestroy {
   }
 
   toggleSubMenu(menu: string) {
-    if (this.displayedSubMenu === menu) {
-      this.displayedSubMenu = '';
-    } else {
-      this.displayedSubMenu = menu;
-    }
+    this.displayedSubMenu[menu] = !this.displayedSubMenu[menu];
   }
 
   toggleRightSidebar() {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
index 2e59f9e9be9c..c851e1505da3 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
@@ -20,6 +20,7 @@ export enum Icons {
   stop = 'fa fa-stop', // Disable
   analyse = 'fa fa-stethoscope', // Scrub
   deepCheck = 'fa fa-cog', // Deep Scrub, Setting, Configuration
+  cogs = 'fa fa-cogs', // Multiple Settings, Configurations
   reweight = 'fa fa-balance-scale', // Reweight
   up = 'fa fa-arrow-up', // Up
   left = 'fa fa-arrow-left', // Mark out
@@ -74,7 +75,13 @@ export enum Icons {
   exit = 'fa fa-sign-out', // Exit
   restart = 'fa fa-history', // Restart
   deploy = 'fa fa-cube', // Deploy, Redeploy
-  cubes = 'fa fa-cubes',
+  cubes = 'fa fa-cubes', // Object storage
+  sitemap = 'fa fa-sitemap', // Cluster, network, connections
+  database = 'fa fa-database', // Database, Block storage
+  bars = 'fa fa-bars', // Stack, bars
+  navicon = 'fa fa-navicon', // Navigation
+  areaChart = 'fa fa-area-chart', // Area Chart, dashboard
+  eye = 'fa fa-eye', // Observability
 
   /* Icons for special effect */
   large = 'fa fa-lg', // icon becomes 33% larger

From 3438de58f2af599901a19cf984e2e7cb1e9f6944 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 17 Oct 2023 12:59:41 -0400
Subject: [PATCH 0205/2492] cmake: promote uring package search to top-level

For use by multiple projects, rocksdb in particular.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 CMakeLists.txt         | 9 +++++++++
 src/blk/CMakeLists.txt | 6 ------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 34d035b4df86..28c9209c5579 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -256,6 +256,15 @@ set(HAVE_LIBURING ${WITH_LIBURING})
 CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_LIBURING "Require and build with system liburing" OFF
   "HAVE_LIBAIO;WITH_BLUESTORE" OFF)
 
+if(WITH_LIBURING)
+  if(WITH_SYSTEM_LIBURING)
+    find_package(uring REQUIRED)
+  else()
+    include(Builduring)
+    build_uring()
+  endif()
+endif()
+
 CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
   "WITH_BLUESTORE" OFF)
 if(WITH_BLUESTORE_PMEM)
diff --git a/src/blk/CMakeLists.txt b/src/blk/CMakeLists.txt
index 288955dd0547..5d9fd6a52953 100644
--- a/src/blk/CMakeLists.txt
+++ b/src/blk/CMakeLists.txt
@@ -57,11 +57,5 @@ if(WITH_EVENTTRACE)
 endif()
 
 if(WITH_LIBURING)
-  if(WITH_SYSTEM_LIBURING)
-    find_package(uring REQUIRED)
-  else()
-    include(Builduring)
-    build_uring()
-  endif()
   target_link_libraries(blk PRIVATE uring::uring)
 endif()

From 8addb3598701410002688ce8ca8bdfcde2026ece Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 16 Oct 2023 10:07:04 -0400
Subject: [PATCH 0206/2492] cmake: populate liburing include and library paths
 down to rocksdb external project

There are two problems:

1) The librocksdb does not use the built liburing if present.
2) You cannot turn off rocksdb's use of liburing with WITH_LIBURING=OFF.

This is to fix errors like:

	FAILED: bin/ceph_test_keyvaluedb_iterators
	: && /opt/rh/gcc-toolset-11/root/usr/bin/g++ -Og -g -rdynamic -pie src/test/ObjectMap/CMakeFiles/ceph_test_keyvaluedb_iterators.dir/test_keyvaluedb_iterators.cc.o src/test/ObjectMap/CMakeFiles/ceph_test_keyvaluedb_iterators.dir/KeyValueDBMemory.cc.o -o bin/ceph_test_keyvaluedb_iterators  -Wl,-rpath,/home/pdonnell/scratch/build/lib  lib/libos.a  lib/libgmock_maind.a  lib/libgmockd.	a  lib/libgtestd.a  -lpthread  -ldl  lib/libglobal.a  -ldl  /usr/lib64/librt.so  -lresolv  -ldl  lib/libblk.a  /lib64/libaio.so  src/liburing/src/liburing.a  lib/libkv.a  lib/libheap_profiler.a  /lib64/libtcmalloc.so  src/rocksdb/librocksdb.a  /lib64/libsnappy.so  /usr/lib64/liblz4.so  /usr/lib64/libz.so  /usr/lib64/libfuse.so  lib/libceph-common.so.2  src/opentelemetry-cpp/sdk/src/trace/libopentelemetry_trace.a  src/opentelemetry-cpp/sdk/src/resource/libopentelemetry_resources.a  src/opentelemetry-cpp/sdk/src/common/libopentelemetry_common.a  src/opentelemetry-cpp/exporters/jaeger/libopentelemetry_exporter_jaeger_trace.a  src/opentelemetry-cpp/ext/src/http/client/curl/libopentelemetry_http_client_curl.a  /usr/lib64/libcurl.so  /usr/lib64/libthrift.so  lib/libjson_spirit.a  lib/libcommon_utf8.a  lib/liberasure_code.a  lib/libextblkdev.a  -lcap  boost/lib/libboost_thread.a  boost/lib/libboost_chrono.a  boost/lib/libboost_atomic.a  boost/lib/libboost_system.a  boost/lib/libboost_random.a  boost/lib/libboost_program_options.a  boost/lib/libboost_date_time.a  boost/lib/libboost_iostreams.a  boost/lib/libboost_regex.a  lib/libfmtd.a  /usr/lib64/libblkid.so  -lpthread  /usr/lib64/libcrypto.so  /usr/lib64/libudev.so  /usr/lib64/libz.so  -ldl  -lresolv   -Wl,--as-needed -latomic && :
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(fs_posix.cc.o): in function `io_uring_wait_cqe_nr':
	/home/pdonnell/scratch/build/src/liburing/src/include/liburing.h:494: undefined reference to `__io_uring_get_cqe'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(fs_posix.cc.o): in function `rocksdb::(anonymous namespace)::PosixFileSystem::AbortIO(std::vector<void*, std::allocator<void*> >&)':
	/home/pdonnell/ceph/src/rocksdb/env/fs_posix.cc:1125: undefined reference to `io_uring_get_sqe'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: /home/pdonnell/ceph/src/rocksdb/env/fs_posix.cc:1134: undefined reference to `io_uring_submit'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(fs_posix.cc.o): in function `rocksdb::CreateIOUring()':
	/home/pdonnell/ceph/src/rocksdb/env/io_posix.h:272: undefined reference to `io_uring_queue_init'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(io_posix.cc.o): in function `io_uring_wait_cqe_nr':
	/home/pdonnell/scratch/build/src/liburing/src/include/liburing.h:494: undefined reference to `__io_uring_get_cqe'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(io_posix.cc.o): in function `rocksdb::PosixRandomAccessFile::MultiRead(rocksdb::FSReadRequest*, unsigned long, rocksdb::IOOptions const&, rocksdb::IODebugContext*)':
	/home/pdonnell/ceph/src/rocksdb/env/io_posix.cc:674: undefined reference to `io_uring_get_sqe'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: /home/pdonnell/ceph/src/rocksdb/env/io_posix.cc:684: undefined reference to `io_uring_submit_and_wait'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: src/rocksdb/librocksdb.a(io_posix.cc.o): in function `rocksdb::PosixRandomAccessFile::ReadAsync(rocksdb::FSReadRequest&, rocksdb::IOOptions const&, std::function<void (rocksdb::FSReadRequest const&, void*)>, void*, void**, std::function<void (void*)>*, rocksdb::IODebugContext*)':
	/home/pdonnell/ceph/src/rocksdb/env/io_posix.cc:901: undefined reference to `io_uring_get_sqe'
	/opt/rh/gcc-toolset-11/root/usr/bin/ld: /home/pdonnell/ceph/src/rocksdb/env/io_posix.cc:910: undefined reference to `io_uring_submit'
	collect2: error: ld returned 1 exit status

Fixes: https://tracker.ceph.com/issues/63218
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 cmake/modules/BuildRocksDB.cmake | 7 +++++++
 cmake/modules/Builduring.cmake   | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/cmake/modules/BuildRocksDB.cmake b/cmake/modules/BuildRocksDB.cmake
index f71f2bb6cc4d..97a101edfec3 100644
--- a/cmake/modules/BuildRocksDB.cmake
+++ b/cmake/modules/BuildRocksDB.cmake
@@ -11,6 +11,13 @@ function(build_rocksdb)
          -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
   endif()
 
+  list(APPEND rocksdb_CMAKE_ARGS -DWITH_LIBURING=${WITH_LIBURING})
+  if(WITH_LIBURING)
+    list(APPEND rocksdb_CMAKE_ARGS -During_INCLUDE_DIR=${URING_INCLUDE_DIR})
+    list(APPEND rocksdb_CMAKE_ARGS -During_LIBRARIES=${URING_LIBRARY_DIR})
+    list(APPEND rocksdb_INTERFACE_LINK_LIBRARIES uring::uring)
+  endif()
+
   if(ALLOCATOR STREQUAL "jemalloc")
     list(APPEND rocksdb_CMAKE_ARGS -DWITH_JEMALLOC=ON)
     list(APPEND rocksdb_INTERFACE_LINK_LIBRARIES JeMalloc::JeMalloc)
diff --git a/cmake/modules/Builduring.cmake b/cmake/modules/Builduring.cmake
index 8683880f7116..4b27a989ac4b 100644
--- a/cmake/modules/Builduring.cmake
+++ b/cmake/modules/Builduring.cmake
@@ -32,6 +32,8 @@ function(build_uring)
   ExternalProject_Get_Property(liburing_ext source_dir)
   set(URING_INCLUDE_DIR "${source_dir}/src/include")
   set(URING_LIBRARY_DIR "${source_dir}/src")
+  set(URING_INCLUDE_DIR ${URING_INCLUDE_DIR} PARENT_SCOPE)
+  set(URING_LIBRARY_DIR ${URING_LIBRARY_DIR} PARENT_SCOPE)
 
   add_library(uring::uring STATIC IMPORTED GLOBAL)
   add_dependencies(uring::uring liburing_ext)

From fbde0a441ee753c06afe3723584d6c9686ceef21 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 17 Oct 2023 19:13:01 +1000
Subject: [PATCH 0207/2492] doc/rados: edit troubleshooting-pg.rst (1 of x)

Edit doc/rados/troubleshooting/troubleshooting-pg.rst (1 of x).

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-pg.rst    | 307 ++++++++++--------
 1 file changed, 169 insertions(+), 138 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
index b7ca679ae6f0..34f960498226 100644
--- a/doc/rados/troubleshooting/troubleshooting-pg.rst
+++ b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -1,120 +1,128 @@
-=====================
+====================
  Troubleshooting PGs
-=====================
+====================
 
 Placement Groups Never Get Clean
 ================================
 
-When you create a cluster and your cluster remains in ``active``,
-``active+remapped`` or ``active+degraded`` status and never achieves an
-``active+clean`` status, you likely have a problem with your configuration.
+If, after you have created your cluster, any Placement Groups (PGs) remain in
+the ``active`` status, the ``active+remapped`` status or the
+``active+degraded`` status and never achieves an ``active+clean`` status, you
+likely have a problem with your configuration.
 
-You may need to review settings in the `Pool, PG and CRUSH Config Reference`_
-and make appropriate adjustments.
+In such a situation, it may be necessary to review the settings in the `Pool,
+PG and CRUSH Config Reference`_ and make appropriate adjustments.
 
-As a general rule, you should run your cluster with more than one OSD and a
-pool size greater than 1 object replica.
+As a general rule, run your cluster with more than one OSD and a pool size
+greater than two object replicas.
 
 .. _one-node-cluster:
 
 One Node Cluster
 ----------------
 
-Ceph no longer provides documentation for operating on a single node, because
-you would never deploy a system designed for distributed computing on a single
-node. Additionally, mounting client kernel modules on a single node containing a
-Ceph  daemon may cause a deadlock due to issues with the Linux kernel itself
-(unless you use VMs for the clients). You can experiment with Ceph in a 1-node
+Ceph no longer provides documentation for operating on a single node.  Systems
+designed for distributed computing by definition do not run on a single node.
+The mounting of client kernel modules on a single node that contains a Ceph
+daemon may cause a deadlock due to issues with the Linux kernel itself (unless
+VMs are used as clients). You can experiment with Ceph in a one-node
 configuration, in spite of the limitations as described herein.
 
-If you are trying to create a cluster on a single node, you must change the
-default of the ``osd_crush_chooseleaf_type`` setting from ``1`` (meaning
+To create a cluster on a single node, you must change the
+``osd_crush_chooseleaf_type`` setting from the default of ``1`` (meaning
 ``host`` or ``node``) to ``0`` (meaning ``osd``) in your Ceph configuration
-file before you create your monitors and OSDs. This tells Ceph that an OSD
-can peer with another OSD on the same host. If you are trying to set up a
-1-node cluster and ``osd_crush_chooseleaf_type`` is greater than ``0``,
-Ceph will try to peer the PGs of one OSD with the PGs of another OSD on
-another node, chassis, rack, row, or even datacenter depending on the setting.
+file before you create your monitors and OSDs. This tells Ceph that an OSD is
+permitted to place another OSD on the same host. If you are trying to set up a
+single-node cluster and ``osd_crush_chooseleaf_type`` is greater than ``0``,
+Ceph will attempt to place the PGs of one OSD with the PGs of another OSD on
+another node, chassis, rack, row, or datacenter depending on the setting.
 
-.. tip:: DO NOT mount kernel clients directly on the same node as your
-   Ceph Storage Cluster, because kernel conflicts can arise. However, you
-   can mount kernel clients within virtual machines (VMs) on a single node.
+.. tip:: DO NOT mount kernel clients directly on the same node as your Ceph
+   Storage Cluster. Kernel conflicts can arise. However, you can mount kernel
+   clients within virtual machines (VMs) on a single node.
 
-If you are creating OSDs using a single disk, you must create directories
-for the data manually first.
+If you are creating OSDs using a single disk, you must manually create
+directories for the data first.
 
 
 Fewer OSDs than Replicas
 ------------------------
 
-If you have brought up two OSDs to an ``up`` and ``in`` state, but you still
-don't see ``active + clean`` placement groups, you may have an
-``osd_pool_default_size`` set to greater than ``2``.
+If two OSDs are in an ``up`` and ``in`` state, but the placement gropus are not
+in an ``active + clean`` state, you may have an ``osd_pool_default_size`` set
+to greater than ``2``.
 
 There are a few ways to address this situation. If you want to operate your
 cluster in an ``active + degraded`` state with two replicas, you can set the
-``osd_pool_default_min_size`` to ``2`` so that you can write objects in
-an ``active + degraded`` state. You may also set the ``osd_pool_default_size``
-setting to ``2`` so that you only have two stored replicas (the original and
-one replica), in which case the cluster should achieve an ``active + clean``
+``osd_pool_default_min_size`` to ``2`` so that you can write objects in an
+``active + degraded`` state. You may also set the ``osd_pool_default_size``
+setting to ``2`` so that you have only two stored replicas (the original and
+one replica). In such a case, the cluster should achieve an ``active + clean``
 state.
 
-.. note:: You can make the changes at runtime. If you make the changes in
-   your Ceph configuration file, you may need to restart your cluster.
+.. note:: You can make the changes while the cluster is running. If you make
+   the changes in your Ceph configuration file, you might need to restart your
+   cluster.
 
 
 Pool Size = 1
 -------------
 
-If you have the ``osd_pool_default_size`` set to ``1``, you will only have
-one copy of the object. OSDs rely on other OSDs to tell them which objects
-they should have. If a first OSD has a copy of an object and there is no
-second copy, then no second OSD can tell the first OSD that it should have
-that copy. For each placement group mapped to the first OSD (see
-``ceph pg dump``), you can force the first OSD to notice the placement groups
-it needs by running::
+If you have ``osd_pool_default_size`` set to ``1``, you will have only one copy
+of the object. OSDs rely on other OSDs to tell them which objects they should
+have. If one OSD has a copy of an object and there is no second copy, then
+there is no second OSD to tell the first OSD that it should have that copy. For
+each placement group mapped to the first OSD (see ``ceph pg dump``), you can
+force the first OSD to notice the placement groups it needs by running a
+command of the following form:
 
-   	ceph osd force-create-pg <pgid>
+.. prompt:: bash
+
+   ceph osd force-create-pg <pgid>
 
 
 CRUSH Map Errors
 ----------------
 
-Another candidate for placement groups remaining unclean involves errors
+If any placement groups in your cluster are unclean, then there might be errors
 in your CRUSH map.
 
 
 Stuck Placement Groups
 ======================
 
-It is normal for placement groups to enter states like "degraded" or "peering"
-following a failure.  Normally these states indicate the normal progression
-through the failure recovery process. However, if a placement group stays in one
-of these states for a long time this may be an indication of a larger problem.
-For this reason, the monitor will warn when placement groups get "stuck" in a
-non-optimal state.  Specifically, we check for:
+It is normal for placement groups to enter "degraded" or "peering" states after
+a component failure. Normally, these states reflect the expected progression
+through the failure recovery process. However, a placement group that stays in
+one of these states for a long time might be an indication of a larger problem.
+For this reason, the Ceph Monitors will warn when placement groups get "stuck"
+in a non-optimal state. Specifically, we check for:
+
+* ``inactive`` - The placement group has not been ``active`` for too long (that
+  is, it hasn't been able to service read/write requests).
 
-* ``inactive`` - The placement group has not been ``active`` for too long
-  (i.e., it hasn't been able to service read/write requests).
+* ``unclean`` - The placement group has not been ``clean`` for too long (that
+  is, it hasn't been able to completely recover from a previous failure).
 
-* ``unclean`` - The placement group has not been ``clean`` for too long
-  (i.e., it hasn't been able to completely recover from a previous failure).
+* ``stale`` - The placement group status has not been updated by a
+  ``ceph-osd``.  This indicates that all nodes storing this placement group may
+  be ``down``.
 
-* ``stale`` - The placement group status has not been updated by a ``ceph-osd``,
-  indicating that all nodes storing this placement group may be ``down``.
+List stuck placement groups by running one of the following commands:
 
-You can explicitly list stuck placement groups with one of::
+.. prompt:: bash
 
-	ceph pg dump_stuck stale
-	ceph pg dump_stuck inactive
-	ceph pg dump_stuck unclean
+   ceph pg dump_stuck stale
+   ceph pg dump_stuck inactive
+   ceph pg dump_stuck unclean
 
-For stuck ``stale`` placement groups, it is normally a matter of getting the
-right ``ceph-osd`` daemons running again.  For stuck ``inactive`` placement
-groups, it is usually a peering problem (see :ref:`failures-osd-peering`).  For
-stuck ``unclean`` placement groups, there is usually something preventing
-recovery from completing, like unfound objects (see
-:ref:`failures-osd-unfound`);
+- Stuck ``stale`` placement groups usually indicate that key ``ceph-osd``
+  daemons are not running.
+- Stuck ``inactive`` placement groups usually indicate a peering problem (see
+  :ref:`failures-osd-peering`).
+- Stuck ``unclean`` placement groups usually indicate that something is
+  preventing recovery from completing, possibly unfound objects (see
+  :ref:`failures-osd-unfound`);
 
 
 
@@ -123,21 +131,28 @@ recovery from completing, like unfound objects (see
 Placement Group Down - Peering Failure
 ======================================
 
-In certain cases, the ``ceph-osd`` `Peering` process can run into
-problems, preventing a PG from becoming active and usable.  For
-example, ``ceph health`` might report::
+In certain cases, the ``ceph-osd`` `peering` process can run into problems,
+which can prevent a PG from becoming active and usable. In such a case, running
+the command ``ceph health detail`` will report something similar to the following:
 
-	ceph health detail
-	HEALTH_ERR 7 pgs degraded; 12 pgs down; 12 pgs peering; 1 pgs recovering; 6 pgs stuck unclean; 114/3300 degraded (3.455%); 1/3 in osds are down
-	...
-	pg 0.5 is down+peering
-	pg 1.4 is down+peering
-	...
-	osd.1 is down since epoch 69, last address 192.168.106.220:6801/8651
+.. prompt:: bash
+
+   ceph health detail
+
+::
+
+    HEALTH_ERR 7 pgs degraded; 12 pgs down; 12 pgs peering; 1 pgs recovering; 6 pgs stuck unclean; 114/3300 degraded (3.455%); 1/3 in osds are down
+    ...
+    pg 0.5 is down+peering
+    pg 1.4 is down+peering
+    ...
+    osd.1 is down since epoch 69, last address 192.168.106.220:6801/8651
 
-We can query the cluster to determine exactly why the PG is marked ``down`` with::
+Query the cluster to determine exactly why the PG is marked ``down`` by running a command of the following form:
 
-	ceph pg 0.5 query
+.. prompt:: bash
+
+   ceph pg 0.5 query
 
 .. code-block:: javascript
 
@@ -164,21 +179,24 @@ We can query the cluster to determine exactly why the PG is marked ``down`` with
     ]
  }
 
-The ``recovery_state`` section tells us that peering is blocked due to
-down ``ceph-osd`` daemons, specifically ``osd.1``.  In this case, we can start that ``ceph-osd``
-and things will recover.
+The ``recovery_state`` section tells us that peering is blocked due to down
+``ceph-osd`` daemons, specifically ``osd.1``. In this case, we can start that
+particular ``ceph-osd`` and recovery will proceed.
+
+Alternatively, if there is a catastrophic failure of ``osd.1`` (for example, if
+there has been a disk failure), the cluster can be informed that the OSD is
+``lost`` and the cluster can be instructed that it must cope as best it can.
 
-Alternatively, if there is a catastrophic failure of ``osd.1`` (e.g., disk
-failure), we can tell the cluster that it is ``lost`` and to cope as
-best it can.
+.. important:: Informing the cluster that an OSD has been lost is dangerous
+   because the cluster cannot guarantee that the other copies of the data are
+   consistent and up to date.
 
-.. important:: This is dangerous in that the cluster cannot
-   guarantee that the other copies of the data are consistent
-   and up to date.
+To report an OSD ``lost`` and to instruct Ceph to continue to attempt recovery
+anyway, run a command of the following form:
 
-To instruct Ceph to continue anyway::
+.. prompt:: bash
 
-	ceph osd lost 1
+   ceph osd lost 1
 
 Recovery will proceed.
 
@@ -188,32 +206,43 @@ Recovery will proceed.
 Unfound Objects
 ===============
 
-Under certain combinations of failures Ceph may complain about
-``unfound`` objects::
+Under certain combinations of failures, Ceph may complain about ``unfound``
+objects, as in this example:
 
-	ceph health detail
-	HEALTH_WARN 1 pgs degraded; 78/3778 unfound (2.065%)
-	pg 2.4 is active+degraded, 78 unfound
+.. prompt:: bash
+
+   ceph health detail
+
+::
 
-This means that the storage cluster knows that some objects (or newer
-copies of existing objects) exist, but it hasn't found copies of them.
-One example of how this might come about for a PG whose data is on ceph-osds
-1 and 2:
+   HEALTH_WARN 1 pgs degraded; 78/3778 unfound (2.065%)
+   pg 2.4 is active+degraded, 78 unfound
+
+This means that the storage cluster knows that some objects (or newer copies of
+existing objects) exist, but it hasn't found copies of them.  Here is an
+example of how this might come about for a PG whose data is on two OSDS, which
+we will call "1" and "2":
 
 * 1 goes down
 * 2 handles some writes, alone
 * 1 comes up
-* 1 and 2 repeer, and the objects missing on 1 are queued for recovery.
+* 1 and 2 re-peer, and the objects missing on 1 are queued for recovery.
 * Before the new objects are copied, 2 goes down.
 
-Now 1 knows that these object exist, but there is no live ``ceph-osd`` who
-has a copy.  In this case, IO to those objects will block, and the
-cluster will hope that the failed node comes back soon; this is
-assumed to be preferable to returning an IO error to the user.
+At this point, 1 knows that these objects exist, but there is no live
+``ceph-osd`` that has a copy of the objects. In this case, IO to those objects
+will block, and the cluster will hope that the failed node comes back soon.
+This is assumed to be preferable to returning an IO error to the user.
+
+.. note:: The situation described immediately above is one reason that setting
+   ``size=2`` on a replicated pool and ``m=1`` on an erasure coded pool risks
+   data loss.
 
-First, you can identify which objects are unfound with::
+Identify which objects are unfound by running a command of the following form:
 
-	ceph pg 2.4 list_unfound [starting offset, in json]
+.. prompt:: bash
+
+   ceph pg 2.4 list_unfound [starting offset, in json]
 
 .. code-block:: javascript
 
@@ -252,22 +281,24 @@ First, you can identify which objects are unfound with::
     "more": false
   }
 
-If there are too many objects to list in a single result, the ``more``
-field will be true and you can query for more.  (Eventually the
-command line tool will hide this from you, but not yet.)
+If there are too many objects to list in a single result, the ``more`` field
+will be true and you can query for more.  (Eventually the command line tool
+will hide this from you, but not yet.)
+
+Now you can identify which OSDs have been probed or might contain data.
 
-Second, you can identify which OSDs have been probed or might contain
-data.
+At the end of the listing (before ``more: false``), ``might_have_unfound`` is
+provided when ``available_might_have_unfound`` is true.  This is equivalent to
+the output of ``ceph pg #.# query``.  This eliminates the need to use ``query``
+directly.  The ``might_have_unfound`` information given behaves the same way as
+that ``query`` does, which is described below.  The only difference is that
+OSDs that have the status of ``already probed`` are ignored.
 
-At the end of the listing (before ``more`` is false), ``might_have_unfound`` is provided
-when ``available_might_have_unfound`` is true.  This is equivalent to the output
-of ``ceph pg #.# query``.  This eliminates the need to use ``query`` directly.
-The ``might_have_unfound`` information given behaves the same way as described below for ``query``.
-The only difference is that OSDs that have ``already probed`` status are ignored.
+Use of ``query``:
 
-Use of ``query``::
+.. prompt:: bash
 
-	ceph pg 2.4 query
+   ceph pg 2.4 query
 
 .. code-block:: javascript
 
@@ -278,8 +309,8 @@ Use of ``query``::
                 { "osd": 1,
                   "status": "osd is down"}]},
 
-In this case, for example, the cluster knows that ``osd.1`` might have
-data, but it is ``down``.  The full range of possible states include:
+In this case, the cluster knows that ``osd.1`` might have data, but it is
+``down``. Here is the full range of possible states:
 
 * already probed
 * querying
@@ -289,31 +320,31 @@ data, but it is ``down``.  The full range of possible states include:
 Sometimes it simply takes some time for the cluster to query possible
 locations.
 
-It is possible that there are other locations where the object can
-exist that are not listed.  For example, if a ceph-osd is stopped and
-taken out of the cluster, the cluster fully recovers, and due to some
-future set of failures ends up with an unfound object, it won't
-consider the long-departed ceph-osd as a potential location to
-consider.  (This scenario, however, is unlikely.)
+It is possible that there are other locations where the object might exist that
+are not listed. For example: if an OSD is stopped and taken out of the cluster
+and then the cluster fully recovers, and then through a subsequent set of
+failures the cluster ends up with an unfound object, the cluster will ignore
+the removed OSD. (This scenario, however, is unlikely.)
 
-If all possible locations have been queried and objects are still
-lost, you may have to give up on the lost objects. This, again, is
-possible given unusual combinations of failures that allow the cluster
-to learn about writes that were performed before the writes themselves
-are recovered.  To mark the "unfound" objects as "lost"::
+If all possible locations have been queried and objects are still lost, you may
+have to give up on the lost objects. This, again, is possible only when unusual
+combinations of failures have occurred that allow the cluster to learn about
+writes that were performed before the writes themselves have been recovered. To
+mark the "unfound" objects as "lost", run a command of the following form:
 
-	ceph pg 2.5 mark_unfound_lost revert|delete
+.. prompt:: bash
 
-This the final argument specifies how the cluster should deal with
-lost objects.
+   ceph pg 2.5 mark_unfound_lost revert|delete
 
-The "delete" option will forget about them entirely.
+Here the final argument (``revert|delete``) specifies how the cluster should
+deal with lost objects.
 
-The "revert" option (not available for erasure coded pools) will
-either roll back to a previous version of the object or (if it was a
-new object) forget about it entirely.  Use this with caution, as it
-may confuse applications that expected the object to exist.
+The ``delete`` option will cause the cluster to forget about them entirely.
 
+The ``revert`` option (which is not available for erasure coded pools) will
+either roll back to a previous version of the object or (if it was a new
+object) forget about the object entirely. Use ``revert`` with caution, as it
+may confuse applications that expect the object to exist.
 
 Homeless Placement Groups
 =========================

From 84ec19442b2db4f4e389810efbc01674d9824408 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Fri, 13 Oct 2023 13:53:23 +0530
Subject: [PATCH 0208/2492] mgr/dashboard: add port and zone endpoints to
 import realm token form in rgw multisite

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   |   4 +-
 .../rgw-multisite-import.component.html       | 122 +++++++++++++++++-
 .../rgw-multisite-import.component.ts         | 115 +++++++++++++++--
 .../src/app/shared/api/rgw-realm.service.ts   |  16 +--
 src/pybind/mgr/dashboard/openapi.yaml         |   6 +-
 .../mgr/dashboard/services/ceph_service.py    |   5 +-
 src/pybind/mgr/rgw/module.py                  |  13 +-
 7 files changed, 249 insertions(+), 32 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 65c809ebec02..9ccf4b36b2b1 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -812,10 +812,10 @@ def get_realm_tokens(self):
     @UpdatePermission
     @allow_empty_body
     # pylint: disable=W0613
-    def import_realm_token(self, realm_token, zone_name, daemon_name=None):
+    def import_realm_token(self, realm_token, zone_name, port, placement_spec):
         try:
             multisite_instance = RgwMultisite()
-            result = CephService.import_realm_token(realm_token, zone_name)
+            result = CephService.import_realm_token(realm_token, zone_name, port, placement_spec)
             multisite_instance.update_period()
             return result
         except NoRgwDaemonsException as e:
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
index bf11e04029aa..a001e4b00c7c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
@@ -13,9 +13,9 @@
           <li>This feature allows you to configure a connection between your primary and secondary Ceph clusters for data replication. By importing a token, you establish a link between the clusters, enabling data synchronization.</li>
           <li>To obtain the token, generate it from your secondary Ceph cluster. This token includes encoded information about the secondary cluster's endpoint, access key, and secret key.</li>
           <li>The secondary zone represents the destination cluster where your data will be replicated.</li>
-          <li>Please create an RGW service using the secondary zone (created after submitting this form) to start the replication between zones.</li>
         </ul>
       </cd-alert-panel>
+      <legend i18n>Zone Details</legend>
       <div class="form-group row">
         <label class="cd-col-form-label required"
                for="realmToken"
@@ -51,6 +51,126 @@
                 i18n>The chosen zone name is already in use.</span>
         </div>
       </div>
+
+      <legend i18n>Service Details</legend>
+      <div class="form-group row">
+        <div class="cd-col-form-offset">
+          <div class="custom-control custom-checkbox">
+            <input class="custom-control-input"
+                   id="unmanaged"
+                   type="checkbox"
+                   formControlName="unmanaged">
+            <label class="custom-control-label"
+                   for="unmanaged"
+                   i18n>Unmanaged</label>
+            <cd-helper i18n>If set to true, the orchestrator will not start nor stop any daemon associated with this service.
+               Placement and all other properties will be ignored.</cd-helper>
+          </div>
+        </div>
+      </div>
+
+      <!-- Placement -->
+      <div *ngIf="!importTokenForm.controls.unmanaged.value"
+           class="form-group row">
+        <label class="cd-col-form-label"
+               for="placement"
+               i18n>Placement</label>
+        <div class="cd-col-form-input">
+          <select id="placement"
+                  class="form-select"
+                  formControlName="placement">
+            <option i18n
+                    value="hosts">Hosts</option>
+            <option i18n
+                    value="label">Label</option>
+          </select>
+        </div>
+      </div>
+
+      <!-- Label -->
+      <div *ngIf="!importTokenForm.controls.unmanaged.value && importTokenForm.controls.placement.value === 'label'"
+           class="form-group row">
+        <label i18n
+               class="cd-col-form-label"
+               for="label">Label</label>
+        <div class="cd-col-form-input">
+          <input id="label"
+                 class="form-control"
+                 type="text"
+                 formControlName="label"
+                 [ngbTypeahead]="searchLabels"
+                 (focus)="labelFocus.next($any($event).target.value)"
+                 (click)="labelClick.next($any($event).target.value)">
+          <span class="invalid-feedback"
+                *ngIf="importTokenForm.showError('label', frm, 'required')"
+                i18n>This field is required.</span>
+        </div>
+      </div>
+
+      <!-- Hosts -->
+      <div *ngIf="!importTokenForm.controls.unmanaged.value && importTokenForm.controls.placement.value === 'hosts'"
+           class="form-group row">
+        <label class="cd-col-form-label"
+               for="hosts"
+               i18n>Hosts</label>
+        <div class="cd-col-form-input">
+          <cd-select-badges id="hosts"
+                            [data]="importTokenForm.controls.hosts.value"
+                            [options]="hosts.options"
+                            [messages]="hosts.messages">
+          </cd-select-badges>
+        </div>
+      </div>
+
+      <!-- count -->
+      <div *ngIf="!importTokenForm.controls.unmanaged.value"
+           class="form-group row">
+        <label class="cd-col-form-label"
+               for="count">
+          <span i18n>Count</span>
+          <cd-helper i18n>Only that number of daemons will be created.</cd-helper>
+        </label>
+        <div class="cd-col-form-input">
+          <input id="count"
+                 class="form-control"
+                 type="number"
+                 formControlName="count"
+                 min="1">
+          <span class="invalid-feedback"
+                *ngIf="importTokenForm.showError('count', frm, 'min')"
+                i18n>The value must be at least 1.</span>
+          <span class="invalid-feedback"
+                *ngIf="importTokenForm.showError('count', frm, 'pattern')"
+                i18n>The entered value needs to be a number.</span>
+        </div>
+      </div>
+
+      <!-- RGW -->
+      <ng-container *ngIf="!importTokenForm.controls.unmanaged.value">
+        <!-- rgw_frontend_port -->
+        <div class="form-group row">
+          <label i18n
+                 class="cd-col-form-label"
+                 for="rgw_frontend_port">Port</label>
+          <div class="cd-col-form-input">
+            <input id="rgw_frontend_port"
+                   class="form-control"
+                   type="number"
+                   formControlName="rgw_frontend_port"
+                   min="1"
+                   max="65535">
+            <span class="invalid-feedback"
+                  *ngIf="importTokenForm.showError('rgw_frontend_port', frm, 'pattern')"
+                  i18n>The entered value needs to be a number.</span>
+            <span class="invalid-feedback"
+                  *ngIf="importTokenForm.showError('rgw_frontend_port', frm, 'min')"
+                  i18n>The value must be at least 1.</span>
+            <span class="invalid-feedback"
+                  *ngIf="importTokenForm.showError('rgw_frontend_port', frm, 'max')"
+                  i18n>The value cannot exceed 65535.</span>
+          </div>
+        </div>
+      </ng-container>
     </div>
     <div class="modal-footer">
       <cd-form-button-panel (submitActionEvent)="onSubmit()"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.ts
index 5581a80bfe1a..deda89016707 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.ts
@@ -1,6 +1,6 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, ViewChild } from '@angular/core';
 import { FormControl, Validators } from '@angular/forms';
-import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { NgbActiveModal, NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
 import { RgwRealmService } from '~/app/shared/api/rgw-realm.service';
 import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { NotificationType } from '~/app/shared/enum/notification-type.enum';
@@ -9,6 +9,12 @@ import { CdValidators } from '~/app/shared/forms/cd-validators';
 import { NotificationService } from '~/app/shared/services/notification.service';
 import { RgwZone } from '../models/rgw-multisite';
 import _ from 'lodash';
+import { SelectMessages } from '~/app/shared/components/select/select-messages.model';
+import { HostService } from '~/app/shared/api/host.service';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
+import { SelectOption } from '~/app/shared/components/select/select-option.model';
+import { Observable, Subject, merge } from 'rxjs';
+import { debounceTime, distinctUntilChanged, filter, map } from 'rxjs/operators';
 
 @Component({
   selector: 'cd-rgw-multisite-import',
@@ -19,18 +25,33 @@ export class RgwMultisiteImportComponent implements OnInit {
   readonly endpoints = /^((https?:\/\/)|(www.))(?:([a-zA-Z]+)|(\d+\.\d+.\d+.\d+)):\d{2,4}$/;
   readonly ipv4Rgx = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/i;
   readonly ipv6Rgx = /^(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4}$/i;
+  @ViewChild(NgbTypeahead, { static: false })
+  typeahead: NgbTypeahead;
 
   importTokenForm: CdFormGroup;
   multisiteInfo: object[] = [];
   zoneList: RgwZone[] = [];
   zoneNames: string[];
+  hosts: any;
+  labels: string[];
+  labelClick = new Subject<string>();
+  labelFocus = new Subject<string>();
 
   constructor(
     public activeModal: NgbActiveModal,
+    public hostService: HostService,
+
     public rgwRealmService: RgwRealmService,
     public actionLabels: ActionLabelsI18n,
     public notificationService: NotificationService
   ) {
+    this.hosts = {
+      options: [],
+      messages: new SelectMessages({
+        empty: $localize`There are no hosts.`,
+        filter: $localize`Filter hosts`
+      })
+    };
     this.createForm();
   }
   ngOnInit(): void {
@@ -41,6 +62,20 @@ export class RgwMultisiteImportComponent implements OnInit {
     this.zoneNames = this.zoneList.map((zone) => {
       return zone['name'];
     });
+    const hostContext = new CdTableFetchDataContext(() => undefined);
+    this.hostService.list(hostContext.toParams(), 'false').subscribe((resp: object[]) => {
+      const options: SelectOption[] = [];
+      _.forEach(resp, (host: object) => {
+        if (_.get(host, 'sources.orchestrator', false)) {
+          const option = new SelectOption(false, _.get(host, 'hostname'), '');
+          options.push(option);
+        }
+      });
+      this.hosts.options = [...options];
+    });
+    this.hostService.getLabels().subscribe((resp: string[]) => {
+      this.labels = resp;
+    });
   }
 
   createForm() {
@@ -55,23 +90,75 @@ export class RgwMultisiteImportComponent implements OnInit {
             return this.zoneNames && this.zoneNames.indexOf(zoneName) !== -1;
           })
         ]
-      })
+      }),
+      rgw_frontend_port: new FormControl(null, {
+        validators: [Validators.required, Validators.pattern('^[0-9]*$')]
+      }),
+      placement: new FormControl('hosts'),
+      label: new FormControl(null, [
+        CdValidators.requiredIf({
+          placement: 'label',
+          unmanaged: false
+        })
+      ]),
+      hosts: new FormControl([]),
+      count: new FormControl(null, [CdValidators.number(false)]),
+      unmanaged: new FormControl(false)
     });
   }
 
   onSubmit() {
     const values = this.importTokenForm.value;
-    this.rgwRealmService.importRealmToken(values['realmToken'], values['zoneName']).subscribe(
-      () => {
-        this.notificationService.show(
-          NotificationType.success,
-          $localize`Realm token import successfull`
-        );
-        this.activeModal.close();
-      },
-      () => {
-        this.importTokenForm.setErrors({ cdSubmitButton: true });
+    const placementSpec: object = {
+      placement: {}
+    };
+    if (!values['unmanaged']) {
+      switch (values['placement']) {
+        case 'hosts':
+          if (values['hosts'].length > 0) {
+            placementSpec['placement']['hosts'] = values['hosts'];
+          }
+          break;
+        case 'label':
+          placementSpec['placement']['label'] = values['label'];
+          break;
       }
-    );
+      if (_.isNumber(values['count']) && values['count'] > 0) {
+        placementSpec['placement']['count'] = values['count'];
+      }
+    }
+    this.rgwRealmService
+      .importRealmToken(
+        values['realmToken'],
+        values['zoneName'],
+        values['rgw_frontend_port'],
+        placementSpec
+      )
+      .subscribe(
+        () => {
+          this.notificationService.show(
+            NotificationType.success,
+            $localize`Realm token import successfull`
+          );
+          this.activeModal.close();
+        },
+        () => {
+          this.importTokenForm.setErrors({ cdSubmitButton: true });
+        }
+      );
   }
+
+  searchLabels = (text$: Observable<string>) => {
+    return merge(
+      text$.pipe(debounceTime(200), distinctUntilChanged()),
+      this.labelFocus,
+      this.labelClick.pipe(filter(() => !this.typeahead.isPopupOpen()))
+    ).pipe(
+      map((value) =>
+        this.labels
+          .filter((label: string) => label.toLowerCase().indexOf(value.toLowerCase()) > -1)
+          .slice(0, 10)
+      )
+    );
+  };
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-realm.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-realm.service.ts
index efa882c8b34c..e81731cd5203 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-realm.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-realm.service.ts
@@ -66,14 +66,14 @@ export class RgwRealmService {
     };
   }
 
-  importRealmToken(realm_token: string, zone_name: string) {
-    return this.rgwDaemonService.request((params: HttpParams) => {
-      params = params.appendAll({
-        realm_token: realm_token,
-        zone_name: zone_name
-      });
-      return this.http.post(`${this.url}/import_realm_token`, null, { params: params });
-    });
+  importRealmToken(realm_token: string, zone_name: string, port: number, placementSpec: object) {
+    let requestBody = {
+      realm_token: realm_token,
+      zone_name: zone_name,
+      port: port,
+      placement_spec: placementSpec
+    };
+    return this.http.post(`${this.url}/import_realm_token`, requestBody);
   }
 
   getRealmTokens() {
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index da0e9236e6d4..d35ea87e15ae 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -9394,7 +9394,9 @@ paths:
           application/json:
             schema:
               properties:
-                daemon_name:
+                placement_spec:
+                  type: string
+                port:
                   type: string
                 realm_token:
                   type: string
@@ -9403,6 +9405,8 @@ paths:
               required:
               - realm_token
               - zone_name
+              - port
+              - placement_spec
               type: object
       responses:
         '201':
diff --git a/src/pybind/mgr/dashboard/services/ceph_service.py b/src/pybind/mgr/dashboard/services/ceph_service.py
index 135f88ca2c97..53cd0e7ad936 100644
--- a/src/pybind/mgr/dashboard/services/ceph_service.py
+++ b/src/pybind/mgr/dashboard/services/ceph_service.py
@@ -317,9 +317,10 @@ def get_realm_tokens(cls):
         return tokens_info
 
     @classmethod
-    def import_realm_token(cls, realm_token, zone_name):
+    def import_realm_token(cls, realm_token, zone_name, port, placement_spec):
         tokens_info = mgr.remote('rgw', 'import_realm_token', zone_name=zone_name,
-                                 realm_token=realm_token, start_radosgw=True)
+                                 realm_token=realm_token, port=port, placement=placement_spec,
+                                 start_radosgw=True)
         return tokens_info
 
     @classmethod
diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py
index 079e7e817ca5..f48e2e09fc32 100644
--- a/src/pybind/mgr/rgw/module.py
+++ b/src/pybind/mgr/rgw/module.py
@@ -307,10 +307,11 @@ def rgw_zone_create(self,
                         zone_name: Optional[str] = None,
                         realm_token: Optional[str] = None,
                         port: Optional[int] = None,
-                        placement: Optional[str] = None,
+                        placement: Optional[Union[str, Dict[str, Any]]] = None,
                         start_radosgw: Optional[bool] = True,
                         zone_endpoints: Optional[str] = None,
                         inbuf: Optional[str] = None) -> Any:
+
         if inbuf:
             try:
                 rgw_specs = self._parse_rgw_specs(inbuf)
@@ -318,7 +319,10 @@ def rgw_zone_create(self,
                 return HandleCommandResult(retval=-errno.EINVAL, stderr=f'{e}')
         elif (zone_name and realm_token):
             token = RealmToken.from_base64_str(realm_token)
-            placement_spec = PlacementSpec.from_string(placement) if placement else None
+            if isinstance(placement, dict):
+                placement_spec = PlacementSpec.from_json(placement) if placement else None
+            elif isinstance(placement, str):
+                placement_spec = PlacementSpec.from_string(placement) if placement else None
             rgw_specs = [RGWSpec(rgw_realm=token.realm_name,
                                  rgw_zone=zone_name,
                                  rgw_realm_token=realm_token,
@@ -371,8 +375,9 @@ def import_realm_token(self,
                            zone_name: Optional[str] = None,
                            realm_token: Optional[str] = None,
                            port: Optional[int] = None,
-                           placement: Optional[str] = None,
+                           placement: Optional[dict] = None,
                            start_radosgw: Optional[bool] = True,
                            zone_endpoints: Optional[str] = None) -> None:
-        self.rgw_zone_create(zone_name, realm_token, port, placement, start_radosgw,
+        placement_spec = placement.get('placement') if placement else None
+        self.rgw_zone_create(zone_name, realm_token, port, placement_spec, start_radosgw,
                              zone_endpoints)

From ee4f5d7de2f900553c791066137388ae1a6e85c7 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Tue, 10 Oct 2023 11:24:08 -0400
Subject: [PATCH 0209/2492] rgw: check if bucket index layout can be resharded
 before progressing

Adds a test for a reshardable index layout when a bucket is considered
for immediate, scheduled, or dynamic resharding.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc   |  4 ++++
 src/rgw/driver/rados/rgw_reshard.cc |  8 ++++----
 src/rgw/driver/rados/rgw_reshard.h  |  4 ++--
 src/rgw/rgw_admin.cc                | 11 ++++++++++-
 src/rgw/rgw_bucket_layout.h         |  9 +++++++++
 5 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 5437d12d4b76..e08804379893 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -10065,6 +10065,10 @@ int RGWRados::check_bucket_shards(const RGWBucketInfo& bucket_info,
       return 0;
   }
 
+  if (! is_layout_reshardable(bucket_info.layout)) {
+    return 0;
+  }
+
   bool need_resharding = false;
   uint32_t num_source_shards = rgw::current_num_shards(bucket_info.layout);
   const uint32_t max_dynamic_shards =
diff --git a/src/rgw/driver/rados/rgw_reshard.cc b/src/rgw/driver/rados/rgw_reshard.cc
index 4369bc8a05c8..7a54da7fe7d0 100644
--- a/src/rgw/driver/rados/rgw_reshard.cc
+++ b/src/rgw/driver/rados/rgw_reshard.cc
@@ -994,11 +994,11 @@ int RGWBucketReshard::execute(int num_shards,
   return 0;
 } // execute
 
-bool RGWBucketReshard::can_reshard(const RGWBucketInfo& bucket,
-                                   const RGWSI_Zone* zone_svc)
+bool RGWBucketReshard::should_zone_reshard_now(const RGWBucketInfo& bucket,
+					       const RGWSI_Zone* zone_svc)
 {
   return !zone_svc->need_to_log_data() ||
-      bucket.layout.logs.size() < max_bilog_history;
+    bucket.layout.logs.size() < max_bilog_history;
 }
 
 
@@ -1240,7 +1240,7 @@ int RGWReshard::process_entry(const cls_rgw_reshard_entry& entry,
     return 0;
   }
 
-  if (!RGWBucketReshard::can_reshard(bucket_info, store->svc()->zone)) {
+  if (!RGWBucketReshard::should_zone_reshard_now(bucket_info, store->svc()->zone)) {
     ldpp_dout(dpp, 1) << "Bucket " << bucket_info.bucket << " is not "
         "eligible for resharding until peer zones finish syncing one "
         "or more of its old log generations" << dendl;
diff --git a/src/rgw/driver/rados/rgw_reshard.h b/src/rgw/driver/rados/rgw_reshard.h
index 768e6c8b3593..0497414566ad 100644
--- a/src/rgw/driver/rados/rgw_reshard.h
+++ b/src/rgw/driver/rados/rgw_reshard.h
@@ -175,8 +175,8 @@ class RGWBucketReshard {
   // too large by refusing to reshard the bucket until the old logs get trimmed
   static constexpr size_t max_bilog_history = 4;
 
-  static bool can_reshard(const RGWBucketInfo& bucket,
-                          const RGWSI_Zone* zone_svc);
+  static bool should_zone_reshard_now(const RGWBucketInfo& bucket,
+				      const RGWSI_Zone* zone_svc);
 }; // RGWBucketReshard
 
 
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index a4e17b8895ae..cc7f5811c9ef 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -3006,6 +3006,14 @@ int check_reshard_bucket_params(rgw::sal::Driver* driver,
     return ret;
   }
 
+  if (! is_layout_reshardable((*bucket)->get_info().layout)) {
+    std::cerr << "Bucket '" << (*bucket)->get_name() <<
+      "' currently has layout '" <<
+      current_layout_desc((*bucket)->get_info().layout) <<
+      "', which does not support resharding." << std::endl;
+    return -EINVAL;
+  }
+
   int num_source_shards = rgw::current_num_shards((*bucket)->get_info().layout);
 
   if (num_shards <= num_source_shards && !yes_i_really_mean_it) {
@@ -8105,7 +8113,8 @@ int main(int argc, const char **argv)
           "have the resharding feature enabled." << std::endl;
       return ENOTSUP;
     }
-    if (!RGWBucketReshard::can_reshard(bucket->get_info(), zone_svc) &&
+
+    if (!RGWBucketReshard::should_zone_reshard_now(bucket->get_info(), zone_svc) &&
         !yes_i_really_mean_it) {
       std::cerr << "Bucket '" << bucket->get_name() << "' already has too many "
           "log generations (" << bucket->get_info().layout.logs.size() << ") "
diff --git a/src/rgw/rgw_bucket_layout.h b/src/rgw/rgw_bucket_layout.h
index 40aafd4dd8de..114f1f1ff589 100644
--- a/src/rgw/rgw_bucket_layout.h
+++ b/src/rgw/rgw_bucket_layout.h
@@ -278,5 +278,14 @@ inline uint32_t current_num_shards(const BucketLayout& layout) {
 inline bool is_layout_indexless(const bucket_index_layout_generation& layout) {
   return layout.layout.type == BucketIndexType::Indexless;
 }
+inline bool is_layout_reshardable(const bucket_index_layout_generation& layout) {
+  return layout.layout.type == BucketIndexType::Normal;
+}
+inline bool is_layout_reshardable(const BucketLayout& layout) {
+  return is_layout_reshardable(layout.current_index);
+}
+inline std::string_view current_layout_desc(const BucketLayout& layout) {
+  return rgw::to_string(layout.current_index.layout.type);
+}
 
 } // namespace rgw

From 13512cc202c90abd6c5f1e2747d121cc07689d1b Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 18 Oct 2023 14:00:05 -0400
Subject: [PATCH 0210/2492] mgr/cephadm: update timestamp on repeat
 daemon/service events

If you have a daemon/service event and then an identical
event happens later (e.g. the same daemon is redeployed
multiple times) the events are not updated on the repeat
instances. In cases like this I think it makes more
sense to update the timestamp so users can see the most
recent time the event happened.

Fixes: https://tracker.ceph.com/issues/63238

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/inventory.py          |  2 ++
 src/pybind/mgr/cephadm/tests/test_cephadm.py | 36 ++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 7153ca6dcde3..329501fc7dcb 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1507,6 +1507,8 @@ def add(self, event: OrchestratorEvent) -> None:
 
         for e in self.events[event.kind_subject()]:
             if e.message == event.message:
+                # if subject and message match, just update the timestamp
+                e.created = event.created
                 return
 
         self.events[event.kind_subject()].append(event)
diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py
index 24fcb0280949..b6aef018cf09 100644
--- a/src/pybind/mgr/cephadm/tests/test_cephadm.py
+++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py
@@ -400,6 +400,42 @@ def test_daemon_action_fail(self, cephadm_module: CephadmOrchestrator):
 
                     assert 'myerror' in ''.join(evs)
 
+    @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('[]'))
+    def test_daemon_action_event_timestamp_update(self, cephadm_module: CephadmOrchestrator):
+        # Test to make sure if a new daemon event is created with the same subject
+        # and message that the timestamp of the event is updated to let users know
+        # when it most recently occurred.
+        cephadm_module.service_cache_timeout = 10
+        with with_host(cephadm_module, 'test'):
+            with with_service(cephadm_module, RGWSpec(service_id='myrgw.foobar', unmanaged=True)) as _, \
+                    with_daemon(cephadm_module, RGWSpec(service_id='myrgw.foobar'), 'test') as daemon_id:
+
+                d_name = 'rgw.' + daemon_id
+
+                now = str_to_datetime('2023-10-18T22:45:29.119250Z')
+                with mock.patch("cephadm.inventory.datetime_now", lambda: now):
+                    c = cephadm_module.daemon_action('redeploy', d_name)
+                    assert wait(cephadm_module,
+                                c) == f"Scheduled to redeploy rgw.{daemon_id} on host 'test'"
+
+                    CephadmServe(cephadm_module)._check_daemons()
+
+                d_events = cephadm_module.events.get_for_daemon(d_name)
+                assert len(d_events) == 1
+                assert d_events[0].created == now
+
+                later = str_to_datetime('2023-10-18T23:46:37.119250Z')
+                with mock.patch("cephadm.inventory.datetime_now", lambda: later):
+                    c = cephadm_module.daemon_action('redeploy', d_name)
+                    assert wait(cephadm_module,
+                                c) == f"Scheduled to redeploy rgw.{daemon_id} on host 'test'"
+
+                    CephadmServe(cephadm_module)._check_daemons()
+
+                d_events = cephadm_module.events.get_for_daemon(d_name)
+                assert len(d_events) == 1
+                assert d_events[0].created == later
+
     @pytest.mark.parametrize(
         "action",
         [

From 602564d784ea33a729dc67002ac6cbaf706e0d4d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 17:00:47 -0400
Subject: [PATCH 0211/2492] cephadm: import types from the modules that defines
 them

For the test fixtures created for Podman and Docker, import the Podman
and Docker types from the module that defines them, rather than the
cephadm.py file which now imports them.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/fixtures.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index ebed7b3d21ca..788913ae6b8e 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -17,15 +17,17 @@ def import_cephadm():
 
 
 def mock_docker():
-    _cephadm = import_cephadm()
-    docker = mock.Mock(_cephadm.Docker)
+    from cephadmlib.container_engines import Docker
+
+    docker = mock.Mock(Docker)
     docker.path = '/usr/bin/docker'
     return docker
 
 
 def mock_podman():
-    _cephadm = import_cephadm()
-    podman = mock.Mock(_cephadm.Podman)
+    from cephadmlib.container_engines import Podman
+
+    podman = mock.Mock(Podman)
     podman.path = '/usr/bin/podman'
     podman.version = (2, 1, 0)
     return podman

From 48846540c8581e66b8c954ca14164038a8d69d63 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 11:39:05 -0400
Subject: [PATCH 0212/2492] cephadm: add supports_split_cgroups property to
 Podman

Add a new method, supports_split_cgroups, to the Podman type. This
function returns a boolean indicating the the podman instance in
use supports split cgroups. This function will be later used to replace
various instances of `podman.version >= CGROUPS_SPLIT_PODMAN_VERSION`.
This encapsulates the check and allows it to be more easily modified
in the future.
It is also shorter, which is nicer to read and to type.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_engines.py | 11 ++++++++++-
 src/cephadm/tests/fixtures.py               |  6 ++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/container_engines.py b/src/cephadm/cephadmlib/container_engines.py
index 396161906431..dc7834de36ac 100644
--- a/src/cephadm/cephadmlib/container_engines.py
+++ b/src/cephadm/cephadmlib/container_engines.py
@@ -7,7 +7,11 @@
 from .call_wrappers import call_throws, CallVerbosity
 from .context import CephadmContext
 from .container_engine_base import ContainerEngine
-from .constants import DEFAULT_MODE, MIN_PODMAN_VERSION
+from .constants import (
+    CGROUPS_SPLIT_PODMAN_VERSION,
+    DEFAULT_MODE,
+    MIN_PODMAN_VERSION,
+)
 from .exceptions import Error
 
 
@@ -36,6 +40,11 @@ def __str__(self) -> str:
         version = '.'.join(map(str, self.version))
         return f'{self.EXE} ({self.path}) version {version}'
 
+    @property
+    def supports_split_cgroups(self) -> bool:
+        """Return true if this version of podman supports split cgroups."""
+        return self.version >= CGROUPS_SPLIT_PODMAN_VERSION
+
 
 class Docker(ContainerEngine):
     EXE = 'docker'
diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index 788913ae6b8e..940703733db7 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -30,6 +30,12 @@ def mock_podman():
     podman = mock.Mock(Podman)
     podman.path = '/usr/bin/podman'
     podman.version = (2, 1, 0)
+    # This next little bit of black magic was adapated from the mock docs for
+    # PropertyMock. We don't use a PropertyMock but the suggestion to call
+    # type(...) from the doc allows us to "borrow" the real
+    # supports_split_cgroups attribute:
+    # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock
+    type(podman).supports_split_cgroups = Podman.supports_split_cgroups
     return podman
 
 

From ba863fd61e9dc0eb4b0bc37d8c7acf263f83abc1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 14 Oct 2023 11:21:05 -0400
Subject: [PATCH 0213/2492] cephadm: change version checks to use
 supports_split_cgroups method

Use the recently added supports_split_cgroups method instead of checking
for an exact version.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index a64b0b49404d..622b8aa41577 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -59,7 +59,6 @@
     CEPH_DEFAULT_PUBKEY,
     CEPH_KEYRING,
     CEPH_PUBKEY,
-    CGROUPS_SPLIT_PODMAN_VERSION,
     CONTAINER_INIT,
     CUSTOM_PS1,
     DATA_DIR,
@@ -2814,7 +2813,7 @@ def get_container(
             '--cidfile',
             f'{runtime_dir}/{service_name}-cid',
         ])
-        if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION and not ctx.no_cgroups_split:
+        if ctx.container_engine.supports_split_cgroups and not ctx.no_cgroups_split:
             container_args.append('--cgroups=split')
         # if /etc/hosts doesn't exist, we can be confident
         # users aren't using it for host name resolution
@@ -3406,7 +3405,7 @@ def get_unit_file(ctx, fsid):
                       'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid\n'
                       'Type=forking\n'
                       'PIDFile=%t/%n-pid\n')
-        if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION:
+        if ctx.container_engine.supports_split_cgroups:
             extra_args += 'Delegate=yes\n'
 
     docker = isinstance(ctx.container_engine, Docker)

From 86422b65d537c94605c2d4803132f6b464d808fa Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 14 Oct 2023 11:21:05 -0400
Subject: [PATCH 0214/2492] cephadm: change version checks to use
 supports_split_cgroups method

Use the recently added supports_split_cgroups method instead of checking
for an exact version.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/context_getters.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/context_getters.py b/src/cephadm/cephadmlib/context_getters.py
index a78c67a7812e..0bd0c0e5e6c4 100644
--- a/src/cephadm/cephadmlib/context_getters.py
+++ b/src/cephadm/cephadmlib/context_getters.py
@@ -6,7 +6,6 @@
 
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from .constants import CGROUPS_SPLIT_PODMAN_VERSION
 from .container_engines import Podman
 from .context import CephadmContext
 from .exceptions import Error
@@ -186,5 +185,5 @@ def should_log_to_journald(ctx: CephadmContext) -> bool:
         return ctx.log_to_journald
     return (
         isinstance(ctx.container_engine, Podman)
-        and ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION
+        and ctx.container_engine.supports_split_cgroups
     )

From 621e108af51d98eb7d2899f48190767e71ce7430 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 13:41:04 -0400
Subject: [PATCH 0215/2492] cephadm: move code adding extra args for podman
 service to a func

Move the code that checks for podman/podman features and adds extra args
based on that to a separate function. This more cleanly divides the
"business logic" from the mechanism of running a service container under
podman.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 622b8aa41577..175ecad7defc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2801,6 +2801,25 @@ def get_container(
             f'--env-file={sg.conf_file_path}'
         )
 
+    _update_container_args_for_podman(ctx, ident, container_args)
+    return CephContainer.for_daemon(
+        ctx,
+        ident=ident,
+        entrypoint=entrypoint,
+        args=ceph_args + get_daemon_args(ctx, ident),
+        container_args=container_args,
+        volume_mounts=get_container_mounts(ctx, ident),
+        bind_mounts=get_container_binds(ctx, ident),
+        envs=envs,
+        privileged=privileged,
+        ptrace=ptrace,
+        host_network=host_network,
+    )
+
+
+def _update_container_args_for_podman(
+    ctx: CephadmContext, ident: DaemonIdentity, container_args: List[str]
+) -> None:
     # if using podman, set -d, --conmon-pidfile & --cidfile flags
     # so service can have Type=Forking
     if isinstance(ctx.container_engine, Podman):
@@ -2824,20 +2843,6 @@ def get_container(
         if not os.path.exists('/etc/hosts'):
             container_args.extend(['--no-hosts'])
 
-    return CephContainer.for_daemon(
-        ctx,
-        ident=ident,
-        entrypoint=entrypoint,
-        args=ceph_args + get_daemon_args(ctx, ident),
-        container_args=container_args,
-        volume_mounts=get_container_mounts(ctx, ident),
-        bind_mounts=get_container_binds(ctx, ident),
-        envs=envs,
-        privileged=privileged,
-        ptrace=ptrace,
-        host_network=host_network,
-    )
-
 
 def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
     # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int]

From 19f5249029fdb3e14bac0c462db422469c680cd6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 13:42:33 -0400
Subject: [PATCH 0216/2492] cephadm: move podman service args selection to
 Podman class

Move code that chooses what additional args should be added to a service
container under podman to the Podman class. This continues the effort to
separate and encapsulate code and associate it with general types best
suited to them.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                      | 28 ++++-------------
 src/cephadm/cephadmlib/container_engines.py | 33 +++++++++++++++++++++
 src/cephadm/tests/fixtures.py               |  1 +
 3 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 175ecad7defc..bb16d1c8c224 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2820,28 +2820,12 @@ def get_container(
 def _update_container_args_for_podman(
     ctx: CephadmContext, ident: DaemonIdentity, container_args: List[str]
 ) -> None:
-    # if using podman, set -d, --conmon-pidfile & --cidfile flags
-    # so service can have Type=Forking
-    if isinstance(ctx.container_engine, Podman):
-        runtime_dir = '/run'
-        service_name = f'{ident.unit_name}.service'
-        container_args.extend([
-            '-d', '--log-driver', 'journald',
-            '--conmon-pidfile',
-            f'{runtime_dir}/{service_name}-pid',
-            '--cidfile',
-            f'{runtime_dir}/{service_name}-cid',
-        ])
-        if ctx.container_engine.supports_split_cgroups and not ctx.no_cgroups_split:
-            container_args.append('--cgroups=split')
-        # if /etc/hosts doesn't exist, we can be confident
-        # users aren't using it for host name resolution
-        # and adding --no-hosts avoids bugs created in certain daemons
-        # by modifications podman makes to /etc/hosts
-        # https://tracker.ceph.com/issues/58532
-        # https://tracker.ceph.com/issues/57018
-        if not os.path.exists('/etc/hosts'):
-            container_args.extend(['--no-hosts'])
+    if not isinstance(ctx.container_engine, Podman):
+        return
+    service_name = f'{ident.unit_name}.service'
+    container_args.extend(
+        ctx.container_engine.service_args(ctx, service_name)
+    )
 
 
 def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
diff --git a/src/cephadm/cephadmlib/container_engines.py b/src/cephadm/cephadmlib/container_engines.py
index dc7834de36ac..99d64ff015cb 100644
--- a/src/cephadm/cephadmlib/container_engines.py
+++ b/src/cephadm/cephadmlib/container_engines.py
@@ -45,6 +45,39 @@ def supports_split_cgroups(self) -> bool:
         """Return true if this version of podman supports split cgroups."""
         return self.version >= CGROUPS_SPLIT_PODMAN_VERSION
 
+    def service_args(
+        self, ctx: CephadmContext, service_name: str
+    ) -> List[str]:
+        """Return a list of arguments that should be added to the engine's run
+        command when starting a long-term service (aka daemon) container.
+        """
+        args = []
+        # if using podman, set -d, --conmon-pidfile & --cidfile flags
+        # so service can have Type=Forking
+        runtime_dir = '/run'
+        args.extend(
+            [
+                '-d',
+                '--log-driver',
+                'journald',
+                '--conmon-pidfile',
+                f'{runtime_dir}/{service_name}-pid',
+                '--cidfile',
+                f'{runtime_dir}/{service_name}-cid',
+            ]
+        )
+        if self.supports_split_cgroups and not ctx.no_cgroups_split:
+            args.append('--cgroups=split')
+        # if /etc/hosts doesn't exist, we can be confident
+        # users aren't using it for host name resolution
+        # and adding --no-hosts avoids bugs created in certain daemons
+        # by modifications podman makes to /etc/hosts
+        # https://tracker.ceph.com/issues/58532
+        # https://tracker.ceph.com/issues/57018
+        if not os.path.exists('/etc/hosts'):
+            args.append('--no-hosts')
+        return args
+
 
 class Docker(ContainerEngine):
     EXE = 'docker'
diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index 940703733db7..f84a2d7ba205 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -36,6 +36,7 @@ def mock_podman():
     # supports_split_cgroups attribute:
     # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock
     type(podman).supports_split_cgroups = Podman.supports_split_cgroups
+    type(podman).service_args = Podman.service_args
     return podman
 
 

From 4ecec7a777e90fce31317bc805fc70e1b40cba74 Mon Sep 17 00:00:00 2001
From: Prashant D <pdhange@redhat.com>
Date: Wed, 18 Oct 2023 16:07:47 -0400
Subject: [PATCH 0217/2492] qa/smoke,orch,perf-basic: add POOL_APP_NOT_ENABLED
 to ignorelist

Some of the smoke, orch and perf-basic tests are failing due
to POOL_APP_NOT_ENABLED health check failure. Add
POOL_APP_NOT_ENABLED to ignorelist for these tests.

Signed-off-by: Prashant D <pdhange@redhat.com>
---
 qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml  | 3 ++-
 qa/suites/perf-basic/objectstore/bluestore.yaml                | 2 ++
 .../basic/tasks/test/cfuse_workunit_suites_blogbench.yaml      | 2 ++
 .../smoke/basic/tasks/test/cfuse_workunit_suites_fsstress.yaml | 2 ++
 .../smoke/basic/tasks/test/cfuse_workunit_suites_iozone.yaml   | 2 ++
 .../smoke/basic/tasks/test/cfuse_workunit_suites_pjd.yaml      | 2 ++
 .../smoke/basic/tasks/test/kclient_workunit_direct_io.yaml     | 2 ++
 .../smoke/basic/tasks/test/kclient_workunit_suites_dbench.yaml | 2 ++
 .../basic/tasks/test/kclient_workunit_suites_fsstress.yaml     | 2 ++
 .../smoke/basic/tasks/test/kclient_workunit_suites_pjd.yaml    | 2 ++
 .../smoke/basic/tasks/test/libcephfs_interface_tests.yaml      | 2 ++
 qa/suites/smoke/basic/tasks/test/rados_cls_all.yaml            | 2 ++
 qa/suites/smoke/basic/tasks/test/rbd_cli_import_export.yaml    | 2 ++
 qa/suites/smoke/basic/tasks/test/rbd_python_api_tests.yaml     | 2 ++
 .../smoke/basic/tasks/test/rbd_workunit_suites_iozone.yaml     | 2 ++
 qa/suites/smoke/basic/tasks/test/rgw_ec_s3tests.yaml           | 2 ++
 qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml              | 2 ++
 17 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
index 564a2eb0252d..3e6e7f95536f 100644
--- a/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
+++ b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
@@ -13,6 +13,7 @@ tasks:
         - \(PG_
         - replacing it with standby
         - No standby daemons available
+        - \(POOL_APP_NOT_ENABLED\)
   - cephfs_test_runner:
       modules:
-        - tasks.mgr.test_orchestrator_cli
\ No newline at end of file
+        - tasks.mgr.test_orchestrator_cli
diff --git a/qa/suites/perf-basic/objectstore/bluestore.yaml b/qa/suites/perf-basic/objectstore/bluestore.yaml
index f5793d76ee5d..699db42d74dc 100644
--- a/qa/suites/perf-basic/objectstore/bluestore.yaml
+++ b/qa/suites/perf-basic/objectstore/bluestore.yaml
@@ -1,6 +1,8 @@
 overrides:
   ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
     conf:
       osd:
         osd objectstore: bluestore
diff --git a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_blogbench.yaml b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_blogbench.yaml
index 0e51ed0e4fd0..bc40416ffb21 100644
--- a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_blogbench.yaml
+++ b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_blogbench.yaml
@@ -1,6 +1,8 @@
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_fsstress.yaml b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_fsstress.yaml
index 8f0bc592c903..e21286d595e1 100644
--- a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_fsstress.yaml
+++ b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_fsstress.yaml
@@ -1,5 +1,7 @@
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_iozone.yaml b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_iozone.yaml
index fedc23081b4d..871606ab87c7 100644
--- a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_iozone.yaml
+++ b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_iozone.yaml
@@ -1,5 +1,7 @@
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse: [client.0]
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_pjd.yaml b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_pjd.yaml
index 03501ecd3c0f..0f4469c9300b 100644
--- a/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_pjd.yaml
+++ b/qa/suites/smoke/basic/tasks/test/cfuse_workunit_suites_pjd.yaml
@@ -1,6 +1,8 @@
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
     conf:
       mds:
         debug mds: 20
diff --git a/qa/suites/smoke/basic/tasks/test/kclient_workunit_direct_io.yaml b/qa/suites/smoke/basic/tasks/test/kclient_workunit_direct_io.yaml
index 6d3ccdf05567..3720d418c828 100644
--- a/qa/suites/smoke/basic/tasks/test/kclient_workunit_direct_io.yaml
+++ b/qa/suites/smoke/basic/tasks/test/kclient_workunit_direct_io.yaml
@@ -5,6 +5,8 @@ overrides:
         ms die on skipped message: false
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - kclient:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_dbench.yaml b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_dbench.yaml
index 2a9b991a9720..256d1f1feeb3 100644
--- a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_dbench.yaml
+++ b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_dbench.yaml
@@ -6,6 +6,8 @@ overrides:
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - kclient:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_fsstress.yaml b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_fsstress.yaml
index 96a7a215db76..649ea8e14592 100644
--- a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_fsstress.yaml
+++ b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_fsstress.yaml
@@ -6,6 +6,8 @@ overrides:
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - kclient:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_pjd.yaml b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_pjd.yaml
index 60723a4164f4..7dea45b805d6 100644
--- a/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_pjd.yaml
+++ b/qa/suites/smoke/basic/tasks/test/kclient_workunit_suites_pjd.yaml
@@ -6,6 +6,8 @@ overrides:
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - kclient:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/libcephfs_interface_tests.yaml b/qa/suites/smoke/basic/tasks/test/libcephfs_interface_tests.yaml
index 2e332f0137d4..3be975b6bf2c 100644
--- a/qa/suites/smoke/basic/tasks/test/libcephfs_interface_tests.yaml
+++ b/qa/suites/smoke/basic/tasks/test/libcephfs_interface_tests.yaml
@@ -9,6 +9,8 @@ overrides:
         debug mds: 20
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/rados_cls_all.yaml b/qa/suites/smoke/basic/tasks/test/rados_cls_all.yaml
index 07e28a7a46d9..c4d55728c31d 100644
--- a/qa/suites/smoke/basic/tasks/test/rados_cls_all.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rados_cls_all.yaml
@@ -7,6 +7,8 @@ overrides:
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - workunit:
     clients:
       client.0:
diff --git a/qa/suites/smoke/basic/tasks/test/rbd_cli_import_export.yaml b/qa/suites/smoke/basic/tasks/test/rbd_cli_import_export.yaml
index 9c2ad68a311e..79ff9418d821 100644
--- a/qa/suites/smoke/basic/tasks/test/rbd_cli_import_export.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rbd_cli_import_export.yaml
@@ -1,6 +1,8 @@
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/rbd_python_api_tests.yaml b/qa/suites/smoke/basic/tasks/test/rbd_python_api_tests.yaml
index a3cfaa2bc624..73e64bb2c5b0 100644
--- a/qa/suites/smoke/basic/tasks/test/rbd_python_api_tests.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rbd_python_api_tests.yaml
@@ -5,6 +5,8 @@ overrides:
       - python3-pytest
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - ceph-fuse:
 - workunit:
     clients:
diff --git a/qa/suites/smoke/basic/tasks/test/rbd_workunit_suites_iozone.yaml b/qa/suites/smoke/basic/tasks/test/rbd_workunit_suites_iozone.yaml
index 1cbaebc89c5b..8602447aad2f 100644
--- a/qa/suites/smoke/basic/tasks/test/rbd_workunit_suites_iozone.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rbd_workunit_suites_iozone.yaml
@@ -7,6 +7,8 @@ overrides:
         rbd default features: 5
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - rbd:
     all:
       image_size: 20480
diff --git a/qa/suites/smoke/basic/tasks/test/rgw_ec_s3tests.yaml b/qa/suites/smoke/basic/tasks/test/rgw_ec_s3tests.yaml
index f35771e78a3c..8f824838cb10 100644
--- a/qa/suites/smoke/basic/tasks/test/rgw_ec_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rgw_ec_s3tests.yaml
@@ -4,6 +4,8 @@ overrides:
     cache-pools: true
 tasks:
 - ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - rgw: [client.0]
 - tox: [client.0]
 - s3tests:
diff --git a/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
index e3a1167c6ee8..ab98e5819e0a 100644
--- a/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
@@ -1,6 +1,8 @@
 tasks:
 - ceph:
     fs: xfs
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
 - rgw: [client.0]
 - tox: [client.0]
 - s3tests:

From 9f19dffc93463513e03908f3506c62e65364c0cd Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Mon, 25 Sep 2023 21:29:35 +0000
Subject: [PATCH 0218/2492] qa/tasks/ceph_manager.py: init
 test_min_size_duration

Added comment about test_min_size_duration
in qa/tasks/thrashosds.

But also use the variable in ceph_manager.py

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 qa/tasks/ceph_manager.py | 3 ++-
 qa/tasks/thrashosds.py   | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 4702bd471afc..a12d8c750116 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -881,7 +881,8 @@ def test_pool_min_size(self):
         self.ceph_manager.wait_for_clean(timeout=180)
         assert self.ceph_manager.is_clean(), \
             'not clean before minsize thrashing starts'
-        while not self.stopping:
+        start = time.time()
+        while time.time() - start < self.config.get("test_min_size_duration", 1800):
             # look up k and m from all the pools on each loop, in case it
             # changes as the cluster runs
             k = 0
diff --git a/qa/tasks/thrashosds.py b/qa/tasks/thrashosds.py
index aa7ec437a600..476e0e4755a7 100644
--- a/qa/tasks/thrashosds.py
+++ b/qa/tasks/thrashosds.py
@@ -63,6 +63,9 @@ def task(ctx, config):
        - kills that osd
        - revives all other osds
        - verifies that the osds fully recover
+    
+    test_min_size_duration: (1800) the number of seconds for
+        test_pool_min_size to last.
 
     timeout: (360) the number of seconds to wait for the cluster
        to become clean after each cluster change. If this doesn't

From 92bf1a8aa8d0d208577c4076d4a86644c01548d5 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Thu, 28 Sep 2023 18:03:45 +0000
Subject: [PATCH 0219/2492] qa/tasks/rados.py: Allow rados task to override
 config

Problem:

Currently, no option override the config in rados task.

Solution:

Enable override of the config file in rados task.

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 qa/tasks/rados.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py
index 625ea4d9d764..7e786fb68e7e 100644
--- a/qa/tasks/rados.py
+++ b/qa/tasks/rados.py
@@ -130,6 +130,12 @@ def task(ctx, config):
     assert isinstance(config, dict), \
         "please list clients to run on"
 
+    log.info("config is {config}".format(config=str(config)))
+    overrides = ctx.config.get('overrides', {})
+    log.info("overrides is {overrides}".format(overrides=str(overrides)))
+    teuthology.deep_merge(config, overrides.get('rados', {}))
+    log.info("config is {config}".format(config=str(config)))
+
     object_size = int(config.get('object_size', 4000000))
     op_weights = config.get('op_weights', {})
     testdir = teuthology.get_testdir(ctx)

From e46dc20cdfb157f94781032451057d1e138535cc Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 19 Oct 2023 10:20:55 +0800
Subject: [PATCH 0220/2492] client: use the fs' full path instead of from
 mountpoint's root

The mountpoint's root ino# possibly not be the full CephFS
filesystem root, it's just the mountpoint of this particular client.
Just prepend the mountpoint path to the full path.

Introduced-by: c1bf8d88e9d client: check the cephx mds auth access for setattr
Introduced-by: ce216595c03 client: check the cephx mds auth access for open
Fixes: https://github.com/ceph/ceph/pull/48027#issuecomment-1741019086
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/client/Client.cc | 20 ++++++++++++++++----
 src/client/Client.h  |  1 +
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 4e7e3961e8e1..84b30a2d3677 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -8000,6 +8000,20 @@ int Client::_getvxattr(
   return res;
 }
 
+bool Client::make_absolute_path_string(Inode *in, std::string& path)
+{
+  if (!metadata.count("root") || !in)
+    return false;
+
+  path = metadata["root"].data();
+  if (!in->make_path_string(path)) {
+    path.clear();
+    return false;
+  }
+
+  return true;
+}
+
 int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask,
 			const UserPerm& perms, InodeRef *inp,
 			std::vector<uint8_t>* aux)
@@ -8042,8 +8056,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask,
   int res;
   {
     std::string path;
-    res = in->make_path_string(path);
-    if (res) {
+    if (make_absolute_path_string(in, path)) {
       ldout(cct, 20) << " absolute path: " << path << dendl;
       if (path.length())
         path = path.substr(1);    // drop leading /
@@ -10272,8 +10285,7 @@ int Client::_open(Inode *in, int flags, mode_t mode, Fh **fhp,
     }
 
     std::string path;
-    int result = in->make_path_string(path);
-    if (result) {
+    if (make_absolute_path_string(in, path)) {
       ldout(cct, 20) << __func__ << " absolute path: " << path << dendl;
       if (path.length())
         path = path.substr(1);    // drop leading /
diff --git a/src/client/Client.h b/src/client/Client.h
index 63df6b815bd2..9c1303384be8 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -1625,6 +1625,7 @@ class Client : public Dispatcher, public md_config_obs_t {
 	       const UserPerm& perms, std::string alternate_name, InodeRef *inp = 0);
   int _mknod(Inode *dir, const char *name, mode_t mode, dev_t rdev,
 	     const UserPerm& perms, InodeRef *inp = 0);
+  bool make_absolute_path_string(Inode *in, std::string& path);
   int _do_setattr(Inode *in, struct ceph_statx *stx, int mask,
 		  const UserPerm& perms, InodeRef *inp,
 		  std::vector<uint8_t>* aux=nullptr);

From c7bc4523ffec65d132eef217ce87eaf344144e35 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Mon, 9 Oct 2023 12:15:02 +0800
Subject: [PATCH 0221/2492] client: return false if cannot link all the way to
 mountpoint

Just in case we cannot link all the way to our mount point we will
force to do sync requests instead and let the MDS do the auth access
check.

Introduced-by: fd7aaab2907 client: add make_path_string() helpers support
Fixes: https://github.com/ceph/ceph/pull/48027#issuecomment-1752324457
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/client/Dentry.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/client/Dentry.h b/src/client/Dentry.h
index c66aca6f1e04..47d320ecbbcf 100644
--- a/src/client/Dentry.h
+++ b/src/client/Dentry.h
@@ -84,7 +84,8 @@ class Dentry : public LRUObject {
     if (dir) {
       ret = dir->parent_inode->make_path_string(s);
     } else {
-      s = "???";
+      // Couldn't link all the way to our mount point
+      return false;
     }
     s += "/";
     s.append(name.data(), name.length());

From 81d88bc242639479d9de6600f54d26190f23a061 Mon Sep 17 00:00:00 2001
From: Cyril Duval <cyril.duval@diabolocom.com>
Date: Thu, 12 Oct 2023 13:11:15 +0200
Subject: [PATCH 0222/2492] cephadm: add option to install systemd unit only

Cephadm is responsible for installing systemd units on the system. If
for some reasons the units are missing, this newly added option cephadm
    unit-install will install them (and do nothing else).

This is useful in some cases where going through the others means of
creating such unit are not possible, or do extra unwanted things.

Signed-off-by: Cyril Duval <cyril.duval@diabolocom.com>
---
 src/cephadm/cephadm.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 4901abf42cd3..e58f523c0fc9 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -5493,6 +5493,24 @@ def command_ceph_volume(ctx):
 ##################################
 
 
+def command_unit_install(ctx):
+    # type: (CephadmContext) -> int
+    if not ctx.fsid:
+        raise Error('must pass --fsid to specify cluster')
+
+    fsid = ctx.fsid
+    install_base_units(ctx, fsid)
+    unit = get_unit_file(ctx, fsid)
+    unit_file = 'ceph-%s@.service' % (fsid)
+    with open(ctx.unit_dir + '/' + unit_file + '.new', 'w') as f:
+        f.write(unit)
+        os.rename(ctx.unit_dir + '/' + unit_file + '.new',
+                  ctx.unit_dir + '/' + unit_file)
+    call_throws(ctx, ['systemctl', 'daemon-reload'])
+
+    return 0
+
+
 @infer_fsid
 def command_unit(ctx):
     # type: (CephadmContext) -> int
@@ -7223,6 +7241,10 @@ def _get_parser():
         required=True,
         help='daemon name (type.id)')
 
+    parser_unit_install = subparsers.add_parser(
+        'unit-install', help="Install the daemon's systemd unit")
+    parser_unit_install.set_defaults(func=command_unit_install)
+
     parser_logs = subparsers.add_parser(
         'logs', help='print journald logs for a daemon container')
     parser_logs.set_defaults(func=command_logs)

From 9005451882371948359a1466fca10256476c5c37 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 12 Oct 2023 17:59:04 +0530
Subject: [PATCH 0223/2492] qa: enhance test cases

Fixes: https://tracker.ceph.com/issues/63105
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/cephfs/test_client_recovery.py | 37 +++++++++++++++----------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/qa/tasks/cephfs/test_client_recovery.py b/qa/tasks/cephfs/test_client_recovery.py
index 703166129098..245e25f34df5 100644
--- a/qa/tasks/cephfs/test_client_recovery.py
+++ b/qa/tasks/cephfs/test_client_recovery.py
@@ -13,6 +13,7 @@
 import string
 import os
 
+from teuthology import contextutil
 from teuthology.orchestra import run
 from teuthology.exceptions import CommandFailedError
 from tasks.cephfs.fuse_mount import FuseMount
@@ -808,24 +809,27 @@ def test_client_eviction_if_config_is_set(self):
             # it takes time to have laggy clients entries in cluster log,
             # wait for 6 minutes to see if it is visible, finally restart
             # the client
-            tries = 6
-            while True:
-                try:
-                    with self.assert_cluster_log("1 client(s) laggy due to laggy OSDs",
-                                                 timeout=55):
-                        # make sure clients weren't evicted
-                        self.assert_session_count(2)
-                        break
-                except AssertionError:
-                    tries -= 1
-                    if tries:
-                        continue
-                    raise
+            with contextutil.safe_while(sleep=5, tries=6) as proceed:
+                while proceed():
+                    try:
+                        with self.assert_cluster_log("1 client(s) laggy due to"
+                                                     " laggy OSDs",
+                                                     timeout=55):
+                            # make sure clients weren't evicted
+                            self.assert_session_count(2)
+                            break
+                    except (AssertionError, CommandFailedError) as e:
+                        log.debug(f'{e}, retrying')
+
+            # clear lagginess, expect to get the warning cleared and make sure
+            # client gets evicted
+            self.clear_laggy_params(osd)
+            self.wait_for_health_clear(60)
+            self.assert_session_count(1)
         finally:
             self.mount_a.kill_cleanup()
             self.mount_a.mount_wait()
             self.mount_a.create_destroy()
-            self.clear_laggy_params(osd)
 
     def test_client_eviction_if_config_is_unset(self):
         """
@@ -857,6 +861,11 @@ def test_client_eviction_if_config_is_unset(self):
 
             time.sleep(session_timeout)
             self.assert_session_count(1)
+
+            # make sure warning wasn't seen in cluster log
+            with self.assert_cluster_log("laggy due to laggy OSDs",
+                                         timeout=120, present=False):
+                pass
         finally:
             self.mount_a.kill_cleanup()
             self.mount_a.mount_wait()

From e0fb615c9757da60ad1545529b44df3eda5331b7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 6 Sep 2023 11:46:01 +0000
Subject: [PATCH 0224/2492] crimson/osd/pg: do_osd_ops_execute refactor

add verbosity around returned futures, no change in behavior.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 7cf3b158c89c..0c1ed5b863cd 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -902,13 +902,20 @@ PG::do_osd_ops_execute(
   }, OpsExecuter::osd_op_errorator::all_same_way(
     [rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
-    return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
-        seastar::now(),
-        e.value() == ENOENT ? (*failure_func_ptr)(e) :
-        rollbacker.rollback_obc_if_modified(e).then_interruptible(
-          [e, failure_func_ptr] {
+
+    auto submitted_fut = seastar::now();
+
+    auto all_completed_fut = e.value() == ENOENT ?
+      (*failure_func_ptr)(e) :
+      rollbacker.rollback_obc_if_modified(e).then_interruptible(
+      [e, failure_func_ptr] {
           return (*failure_func_ptr)(e);
-        }));
+      });
+
+    return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
+      std::move(submitted_fut),
+      std::move(all_completed_fut)
+    );
   }));
 }
 seastar::future<> PG::submit_error_log(

From a8b28d69de9e1a54dfeb9c4a02db45359479052e Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 6 Sep 2023 12:09:49 +0000
Subject: [PATCH 0225/2492] crimson/osd/pg: do_osd_ops_execute refactor pt. 2

be more explicit around returned futures, no change in behavior.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 55 +++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 0c1ed5b863cd..6ba56b36e4e2 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -872,33 +872,36 @@ PG::do_osd_ops_execute(
     });
   }).safe_then_unpack_interruptible(
     [success_func=std::move(success_func), rollbacker, this, failure_func_ptr]
-    (auto submitted_fut, auto all_completed_fut) mutable {
+    (auto submitted_fut, auto _all_completed_fut) mutable {
+
+    auto all_completed_fut = _all_completed_fut.safe_then_interruptible_tuple(
+      std::move(success_func),
+      crimson::ct_error::object_corrupted::handle(
+      [rollbacker, this] (const std::error_code& e) mutable {
+      // this is a path for EIO. it's special because we want to fix the obejct
+      // and try again. that is, the layer above `PG::do_osd_ops` is supposed to
+      // restart the execution.
+      return rollbacker.rollback_obc_if_modified(e).then_interruptible(
+      [obc=rollbacker.get_obc(), this] {
+        return repair_object(obc->obs.oi.soid,
+                             obc->obs.oi.version
+        ).then_interruptible([] {
+          return do_osd_ops_iertr::future<Ret>{crimson::ct_error::eagain::make()};
+        });
+      });
+    }), OpsExecuter::osd_op_errorator::all_same_way(
+        [rollbacker, failure_func_ptr]
+        (const std::error_code& e) mutable {
+          return rollbacker.rollback_obc_if_modified(e).then_interruptible(
+          [e, failure_func_ptr] {
+            return (*failure_func_ptr)(e);
+          });
+    }));
+
     return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
-        std::move(submitted_fut),
-        all_completed_fut.safe_then_interruptible_tuple(
-          std::move(success_func),
-          crimson::ct_error::object_corrupted::handle(
-            [rollbacker, this] (const std::error_code& e) mutable {
-            // this is a path for EIO. it's special because we want to fix the obejct
-            // and try again. that is, the layer above `PG::do_osd_ops` is supposed to
-            // restart the execution.
-            return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-              [obc=rollbacker.get_obc(), this] {
-              return repair_object(obc->obs.oi.soid,
-                                   obc->obs.oi.version).then_interruptible([] {
-                return do_osd_ops_iertr::future<Ret>{crimson::ct_error::eagain::make()};
-              });
-            });
-          }), OpsExecuter::osd_op_errorator::all_same_way(
-            [rollbacker, failure_func_ptr]
-            (const std::error_code& e) mutable {
-            return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-              [e, failure_func_ptr] {
-              return (*failure_func_ptr)(e);
-            });
-          })
-        )
-      );
+      std::move(submitted_fut),
+      std::move(all_completed_fut)
+    );
   }, OpsExecuter::osd_op_errorator::all_same_way(
     [rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {

From 49524eca5896101e09f81fc9c92e9b535ed37152 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 6 Sep 2023 12:12:49 +0000
Subject: [PATCH 0226/2492] crimson/osd/pg: add comments around
 do_osd_ops_execute parameters

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 6ba56b36e4e2..54a0eb4dde9d 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -994,6 +994,7 @@ PG::do_osd_ops(
     seastar::make_lw_shared<OpsExecuter>(
       Ref<PG>{this}, obc, op_info, *m, conn, snapc),
     m->ops,
+    // success_func
     [this, m, obc, may_write = op_info.may_write(),
      may_read = op_info.may_read(), rvec = op_info.allows_returnvec()] {
       // TODO: should stop at the first op which returns a negative retval,
@@ -1032,6 +1033,7 @@ PG::do_osd_ops(
       return do_osd_ops_iertr::make_ready_future<MURef<MOSDOpReply>>(
         std::move(reply));
     },
+    // failure_func
     [m, &op_info, obc, this] (const std::error_code& e) {
     return seastar::do_with(eversion_t(), [m, &op_info, obc, e, this](auto &version) {
       auto fut = seastar::now();

From 81fc5c8b40bd897e0c831ac81ce3db91d02b976a Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 6 Sep 2023 12:14:53 +0000
Subject: [PATCH 0227/2492] crimson/osd/pg.cc: add pending tracker comment

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 54a0eb4dde9d..4009117e8140 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1041,6 +1041,9 @@ PG::do_osd_ops(
       ceph_tid_t rep_tid = shard_services.get_tid();
       auto last_complete = peering_state.get_info().last_complete;
       if (op_info.may_write()) {
+        // This should be executed as OrderedExclusivePhaseT so that
+        // successive ops will not reorder.
+        // TODO: https://tracker.ceph.com/issues/61651
         fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
       }
       return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete,  this] {

From da57ee1d115848c6228b73d19d1d0dbb6b0468a0 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 19 Oct 2023 10:59:28 +0000
Subject: [PATCH 0228/2492] crimson/osd/pg: do_osd_ops remove redundant
 log_reply()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 4009117e8140..a64530e801e6 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1085,7 +1085,6 @@ PG::do_osd_ops(
             if (version != eversion_t()) {
               peering_state.complete_write(version, last_complete);
             }
-            return log_reply();
           } else {
             return it->second.all_committed.get_shared_future()
               .then([this, &version, last_complete, log_reply = std::move(log_reply)] {
@@ -1095,9 +1094,8 @@ PG::do_osd_ops(
               return log_reply();
             });
           }
-        } else {
-          return log_reply();
         }
+        return log_reply();
       });
     });
   });

From b002d5a4413ef281f79870bf5b545a5930b09d77 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 19 Oct 2023 11:24:03 +0000
Subject: [PATCH 0229/2492] crimson/osd/pg: log_reply() into seperate method

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 64 ++++++++++++++++++++++++-------------------
 src/crimson/osd/pg.h  |  3 ++
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index a64530e801e6..3f9a9e0f7a2f 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1047,31 +1047,9 @@ PG::do_osd_ops(
         fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
       }
       return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete,  this] {
-        auto log_reply = [m, e, this] {
-          auto reply = crimson::make_message<MOSDOpReply>(
-            m.get(), -e.value(), get_osdmap_epoch(), 0, false);
-          if (m->ops.empty() ? 0 :
-            m->ops.back().op.flags & CEPH_OSD_OP_FLAG_FAILOK) {
-            reply->set_result(0);
-          }
-          // For all ops except for CMPEXT, the correct error value is encoded
-          // in e.value(). For CMPEXT, osdop.rval has the actual error value.
-          if (e.value() == ct_error::cmp_fail_error_value) {
-            assert(!m->ops.empty());
-            for (auto &osdop : m->ops) {
-              if (osdop.rval < 0) {
-                reply->set_result(osdop.rval);
-                break;
-              }
-            }
-          }
-          reply->set_enoent_reply_versions(
-          peering_state.get_info().last_update,
-          peering_state.get_info().last_user_version);
-          reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
-          return do_osd_ops_iertr::make_ready_future<MURef<MOSDOpReply>>(
-            std::move(reply));
-	};
+        auto log_reply_fut = [m, e, this] {
+          return log_reply(m, e);
+        };
 
         if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
           auto it = log_entry_update_waiting_on.find(rep_tid);
@@ -1087,20 +1065,50 @@ PG::do_osd_ops(
             }
           } else {
             return it->second.all_committed.get_shared_future()
-              .then([this, &version, last_complete, log_reply = std::move(log_reply)] {
+              .then([this, &version, last_complete, log_reply_fut = std::move(log_reply_fut)] {
               if (version != eversion_t()) {
                 peering_state.complete_write(version, last_complete);
               }
-              return log_reply();
+              return log_reply_fut();
             });
           }
         }
-        return log_reply();
+        return log_reply_fut();
       });
     });
   });
 }
 
+PG::do_osd_ops_iertr::future<MURef<MOSDOpReply>>
+PG::log_reply(
+  Ref<MOSDOp> m,
+  const std::error_code& e)
+{
+  auto reply = crimson::make_message<MOSDOpReply>(
+    m.get(), -e.value(), get_osdmap_epoch(), 0, false);
+  if (m->ops.empty() ? 0 :
+    m->ops.back().op.flags & CEPH_OSD_OP_FLAG_FAILOK) {
+      reply->set_result(0);
+    }
+  // For all ops except for CMPEXT, the correct error value is encoded
+  // in e.value(). For CMPEXT, osdop.rval has the actual error value.
+  if (e.value() == ct_error::cmp_fail_error_value) {
+    assert(!m->ops.empty());
+    for (auto &osdop : m->ops) {
+      if (osdop.rval < 0) {
+        reply->set_result(osdop.rval);
+        break;
+      }
+    }
+  }
+  reply->set_enoent_reply_versions(
+    peering_state.get_info().last_update,
+    peering_state.get_info().last_user_version);
+  reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
+  return do_osd_ops_iertr::make_ready_future<MURef<MOSDOpReply>>(
+    std::move(reply));
+}
+
 PG::do_osd_ops_iertr::future<PG::pg_rep_op_fut_t<>>
 PG::do_osd_ops(
   ObjectContextRef obc,
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 380820e8250d..6a0231e452f6 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -576,6 +576,9 @@ class PG : public boost::intrusive_ref_counter<
   using do_osd_ops_failure_func_t =
     std::function<do_osd_ops_iertr::future<>(const std::error_code&)>;
   struct do_osd_ops_params_t;
+  do_osd_ops_iertr::future<MURef<MOSDOpReply>> log_reply(
+    Ref<MOSDOp> m,
+    const std::error_code& e);
   do_osd_ops_iertr::future<pg_rep_op_fut_t<>> do_osd_ops(
     ObjectContextRef obc,
     std::vector<OSDOp>& ops,

From 10069b769c84019f67d4a029adb2df94eb717645 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Oct 2023 13:04:14 +0000
Subject: [PATCH 0230/2492] mgr/dashboard: bump @babel/traverse

Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.23.0 to 7.23.2.
- [Release notes](https://github.com/babel/babel/releases)
- [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md)
- [Commits](https://github.com/babel/babel/commits/v7.23.2/packages/babel-traverse)

---
updated-dependencies:
- dependency-name: "@babel/traverse"
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 src/pybind/mgr/dashboard/frontend/package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json
index 15aaff89db92..bc07bd98b98a 100644
--- a/src/pybind/mgr/dashboard/frontend/package-lock.json
+++ b/src/pybind/mgr/dashboard/frontend/package-lock.json
@@ -3201,9 +3201,9 @@
       }
     },
     "node_modules/@babel/traverse": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.0.tgz",
-      "integrity": "sha512-t/QaEvyIoIkwzpiZ7aoSKK8kObQYeF7T2v+dazAYCb8SXtp58zEVkWW7zAnju8FNKNdr4ScAOEDmMItbyOmEYw==",
+      "version": "7.23.2",
+      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz",
+      "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==",
       "dependencies": {
         "@babel/code-frame": "^7.22.13",
         "@babel/generator": "^7.23.0",

From 1ffe4aca2a870600d2946e9432740848600a8ac4 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Sun, 15 Oct 2023 11:31:54 +0000
Subject: [PATCH 0231/2492] rgw/notifications/test: use dnf and systemctl to
 setup rabbitmq

Fixes: https://tracker.ceph.com/issues/63205

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 qa/tasks/rabbitmq.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/qa/tasks/rabbitmq.py b/qa/tasks/rabbitmq.py
index c78ac1e568fa..944233d9775d 100644
--- a/qa/tasks/rabbitmq.py
+++ b/qa/tasks/rabbitmq.py
@@ -23,7 +23,7 @@ def install_rabbitmq(ctx, config):
         (remote,) = ctx.cluster.only(client).remotes.keys()
 
         ctx.cluster.only(client).run(args=[
-             'sudo', 'yum', '-y', 'install', 'epel-release'
+             'sudo', 'dnf', '-y', 'install', 'epel-release'
         ])
 
         link1 = 'https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh'
@@ -33,7 +33,7 @@ def install_rabbitmq(ctx, config):
         ])
 
         ctx.cluster.only(client).run(args=[
-             'sudo', 'yum', '-y', 'install', 'erlang'
+             'sudo', 'dnf', '-y', 'install', 'erlang'
         ])
 
         link2 = 'https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh'
@@ -43,7 +43,7 @@ def install_rabbitmq(ctx, config):
         ])
 
         ctx.cluster.only(client).run(args=[
-             'sudo', 'yum', '-y', 'install', 'rabbitmq-server'
+             'sudo', 'dnf', '-y', 'install', 'rabbitmq-server'
         ])
 
     try:
@@ -53,7 +53,7 @@ def install_rabbitmq(ctx, config):
 
         for (client, _) in config.items():
             ctx.cluster.only(client).run(args=[
-                 'sudo', 'yum', '-y', 'remove', 'rabbitmq-server.noarch'
+                 'sudo', 'dnf', '-y', 'remove', 'rabbitmq-server.noarch'
             ])
 
 
@@ -70,7 +70,7 @@ def run_rabbitmq(ctx, config):
         (remote,) = ctx.cluster.only(client).remotes.keys()
 
         ctx.cluster.only(client).run(args=[
-             'sudo', 'chkconfig', 'rabbitmq-server', 'on'
+             'sudo', 'systemctl', 'enable', 'rabbitmq-server.service'
             ],
         )
 

From 1ca356a916910a4016d5d3460d68ea3732f3bad9 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Thu, 19 Oct 2023 11:28:07 -0400
Subject: [PATCH 0232/2492] .github: Clarify checklist details

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 .github/pull_request_template.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 834ef742832f..214b2114e1a3 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -22,7 +22,9 @@
 ## Contribution Guidelines
 - To sign and title your commits, please refer to [Submitting Patches to Ceph](https://github.com/ceph/ceph/blob/main/SubmittingPatches.rst).
 
-- If you are submitting a fix for a stable branch (e.g. "pacific"), please refer to [Submitting Patches to Ceph - Backports](https://github.com/ceph/ceph/blob/master/SubmittingPatches-backports.rst) for the proper workflow.
+- If you are submitting a fix for a stable branch (e.g. "quincy"), please refer to [Submitting Patches to Ceph - Backports](https://github.com/ceph/ceph/blob/master/SubmittingPatches-backports.rst) for the proper workflow.
+
+- When filling out the below checklist, you may click boxes directly in the GitHub web UI.  When entering or editing the entire PR message in the GitHub web UI editor, you may also select a checklist item by adding an `x` between the brackets: `[x]`.  Spaces and capitalization matter when checking off items this way.
 
 ## Checklist
 - Tracker (select at least one)

From 060e1239dda97d26fad79fd0f09d778c71263a0b Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Thu, 19 Oct 2023 10:26:55 -0700
Subject: [PATCH 0233/2492] qa/tests: enabled smoke tests on main, quincy and
 reef

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index 2b7435de608a..bc2a10fcef57 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -31,10 +31,10 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 @daily SUITE_NAME=~/src/ceph-qa-suite_main/suites/ceph-ansible; crontab=$(teuthology-describe-tests --show-facet no $SUITE_NAME | perl -p -e 's/</&lt;/g; s/>/&gt;/g; s/&/&amp;/g') ; header=$(echo h4. $SUITE_NAME ; echo " "; echo " ") ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '<?xml version="1.0"?><wiki_page><text>'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;</text></wiki_page>' http://tracker.ceph.com/projects/ceph-qa-suite/wiki/ceph-ansible.xml?key=$(cat /etc/redmine-key)
 
 
-## ********** smoke tests on main, octopus, and pacific branches
-# 0 5  * * 0,2,4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-# 0 8  * * 5 CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-# 7 8  * * 6 CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+## ********** smoke tests on main, quincy, reef branches
+0 5  * * 0,2,4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+0 8  * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+7 8  * * 6 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
 
 
 ## ********** windows tests on main branch - weekly

From d0cdfc1d72c4edefb060f8ba9ab130189617cab1 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 19 Oct 2023 08:25:38 +1000
Subject: [PATCH 0234/2492] doc/rados: edit troubleshooting-pg (2 of x)

Edit doc/rados/troubleshooting/troubleshooting-pg.rst (1 of x).

Follows https://github.com/ceph/ceph/pull/54052.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/mon-config-ref.rst    |   2 +
 .../configuration/pool-pg-config-ref.rst      |   2 +
 doc/rados/operations/placement-groups.rst     |   1 +
 .../troubleshooting/troubleshooting-pg.rst    | 349 +++++++++++-------
 4 files changed, 212 insertions(+), 142 deletions(-)

diff --git a/doc/rados/configuration/mon-config-ref.rst b/doc/rados/configuration/mon-config-ref.rst
index 0b4118bdb1cc..0bc2d4180421 100644
--- a/doc/rados/configuration/mon-config-ref.rst
+++ b/doc/rados/configuration/mon-config-ref.rst
@@ -538,6 +538,8 @@ Trimming requires that the placement groups are ``active+clean``.
 
 .. index:: Ceph Monitor; clock
 
+.. _mon-config-ref-clock:
+
 Clock
 -----
 
diff --git a/doc/rados/configuration/pool-pg-config-ref.rst b/doc/rados/configuration/pool-pg-config-ref.rst
index 6fedd7c78c58..902c80346bbe 100644
--- a/doc/rados/configuration/pool-pg-config-ref.rst
+++ b/doc/rados/configuration/pool-pg-config-ref.rst
@@ -1,3 +1,5 @@
+.. _rados_config_pool_pg_crush_ref:
+
 ======================================
  Pool, PG and CRUSH Config Reference
 ======================================
diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index feb937651249..dda4a017715d 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -664,6 +664,7 @@ In releases of Ceph that are Nautilus and later (inclusive), when the
 ``pg_num``. This process manifests as periods of remapping of PGs and of
 backfill, and is expected behavior and normal.
 
+.. _rados_ops_pgs_get_pg_num:
 
 Get the Number of PGs
 =====================
diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
index 34f960498226..4af2cf347efb 100644
--- a/doc/rados/troubleshooting/troubleshooting-pg.rst
+++ b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -349,77 +349,106 @@ may confuse applications that expect the object to exist.
 Homeless Placement Groups
 =========================
 
-It is possible for all OSDs that had copies of a given placement groups to fail.
-If that's the case, that subset of the object store is unavailable, and the
-monitor will receive no status updates for those placement groups.  To detect
-this situation, the monitor marks any placement group whose primary OSD has
-failed as ``stale``.  For example::
+It is possible that every OSD that has copies of a given placement group fails.
+If this happens, then the subset of the object store that contains those
+placement groups becomes unavailable and the monitor will receive no status
+updates for those placement groups. The monitor marks as ``stale`` any
+placement group whose primary OSD has failed. For example:
 
-	ceph health
-	HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+.. prompt:: bash
+
+   ceph health
+
+::
 
-You can identify which placement groups are ``stale``, and what the last OSDs to
-store them were, with::
+    HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+
+Identify which placement groups are ``stale`` and which were the last OSDs to
+store the ``stale`` placement groups by running the following command:
+
+.. prompt:: bash
 
-	ceph health detail
-	HEALTH_WARN 24 pgs stale; 3/300 in osds are down
-	...
-	pg 2.5 is stuck stale+active+remapped, last acting [2,0]
-	...
-	osd.10 is down since epoch 23, last address 192.168.106.220:6800/11080
-	osd.11 is down since epoch 13, last address 192.168.106.220:6803/11539
-	osd.12 is down since epoch 24, last address 192.168.106.220:6806/11861
+   ceph health detail
+
+::
+
+   HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+   ...
+   pg 2.5 is stuck stale+active+remapped, last acting [2,0]
+   ...
+   osd.10 is down since epoch 23, last address 192.168.106.220:6800/11080
+   osd.11 is down since epoch 13, last address 192.168.106.220:6803/11539
+   osd.12 is down since epoch 24, last address 192.168.106.220:6806/11861
 
-If we want to get placement group 2.5 back online, for example, this tells us that
-it was last managed by ``osd.0`` and ``osd.2``.  Restarting those ``ceph-osd``
-daemons will allow the cluster to recover that placement group (and, presumably,
-many others).
+This output indicates that placement group 2.5 (``pg 2.5``) was last managed by
+``osd.0`` and ``osd.2``. Restart those OSDs to allow the cluster to recover
+that placement group.
 
 
 Only a Few OSDs Receive Data
 ============================
 
-If you have many nodes in your cluster and only a few of them receive data,
-`check`_ the number of placement groups in your pool. Since placement groups get
-mapped to OSDs, a small number of placement groups will not distribute across
-your cluster. Try creating a pool with a placement group count that is a
-multiple of the number of OSDs. See `Placement Groups`_ for details. The default
-placement group count for pools is not useful, but you can change it `here`_.
+If only a few of the nodes in the cluster are receiving data, check the number
+of placement groups in the pool as instructed in the :ref:`Placement Groups
+<rados_ops_pgs_get_pg_num>` documentation. Since placement groups get mapped to
+OSDs in an operation involving dividing the number of placement groups in the
+cluster by the number of OSDs in the cluster, a small number of placement
+groups (the remainder, in this operation) are sometimes not distributed across
+the cluster. In situations like this, create a pool with a placement group
+count that is a multiple of the number of OSDs. See `Placement Groups`_ for
+details. See the :ref:`Pool, PG, and CRUSH Config Reference
+<rados_config_pool_pg_crush_ref>` for instructions on changing the default
+values used to determine how many placement groups are assigned to each pool.
 
 
 Can't Write Data
 ================
 
-If your cluster is up, but some OSDs are down and you cannot write data,
-check to ensure that you have the minimum number of OSDs running for the
-placement group. If you don't have the minimum number of OSDs running,
-Ceph will not allow you to write data because there is no guarantee
-that Ceph can replicate your data. See ``osd_pool_default_min_size``
-in the `Pool, PG and CRUSH Config Reference`_ for details.
+If the cluster is up, but some OSDs are down and you cannot write data, make
+sure that you have the minimum number of OSDs running for the placement group.
+If you don't have the minimum number of OSDs running, Ceph will not allow you
+to write data because there is no guarantee that Ceph can replicate your data.
+See ``osd_pool_default_min_size`` in the :ref:`Pool, PG, and CRUSH Config
+Reference <rados_config_pool_pg_crush_ref>` for details.
 
 
 PGs Inconsistent
 ================
 
-If you receive an ``active + clean + inconsistent`` state, this may happen
-due to an error during scrubbing. As always, we can identify the inconsistent
-placement group(s) with::
+If the command ``ceph health detail`` returns an ``active + clean +
+inconsistent`` state, this might indicate an error during scrubbing. Identify
+the inconsistent placement group or placement groups by running the following
+command:
+
+.. prompt:: bash
 
     $ ceph health detail
+
+::
+
     HEALTH_ERR 1 pgs inconsistent; 2 scrub errors
     pg 0.6 is active+clean+inconsistent, acting [0,1,2]
     2 scrub errors
 
-Or if you prefer inspecting the output in a programmatic way::
+Alternatively, run this command if you prefer to inspect the output in a
+programmatic way:
+
+.. prompt:: bash
+
+   $ rados list-inconsistent-pg rbd
+
+::
 
-    $ rados list-inconsistent-pg rbd
     ["0.6"]
 
 There is only one consistent state, but in the worst case, we could have
 different inconsistencies in multiple perspectives found in more than one
-objects. If an object named ``foo`` in PG ``0.6`` is truncated, we will have::
+object. If an object named ``foo`` in PG ``0.6`` is truncated, the output of
+``rados list-inconsistent-pg rbd`` will look something like this:
 
-    $ rados list-inconsistent-obj 0.6 --format=json-pretty
+.. prompt:: bash
+
+   rados list-inconsistent-obj 0.6 --format=json-pretty
 
 .. code-block:: javascript
 
@@ -473,82 +502,103 @@ objects. If an object named ``foo`` in PG ``0.6`` is truncated, we will have::
         ]
     }
 
-In this case, we can learn from the output:
+In this case, the output indicates the following:
 
-* The only inconsistent object is named ``foo``, and it is its head that has
+* The only inconsistent object is named ``foo``, and its head has
   inconsistencies.
 * The inconsistencies fall into two categories:
 
-  * ``errors``: these errors indicate inconsistencies between shards without a
-    determination of which shard(s) are bad. Check for the ``errors`` in the
-    `shards` array, if available, to pinpoint the problem.
-
-    * ``data_digest_mismatch``: the digest of the replica read from OSD.2 is
-      different from the ones of OSD.0 and OSD.1
-    * ``size_mismatch``: the size of the replica read from OSD.2 is 0, while
-      the size reported by OSD.0 and OSD.1 is 968.
-  * ``union_shard_errors``: the union of all shard specific ``errors`` in
-    ``shards`` array. The ``errors`` are set for the given shard that has the
-    problem. They include errors like ``read_error``. The ``errors`` ending in
-    ``oi`` indicate a comparison with ``selected_object_info``. Look at the
-    ``shards`` array to determine which shard has which error(s).
+  #. ``errors``: these errors indicate inconsistencies between shards, without
+     an indication of which shard(s) are bad. Check for the ``errors`` in the
+     ``shards`` array, if available, to pinpoint the problem.
+
+     * ``data_digest_mismatch``: the digest of the replica read from ``OSD.2``
+       is different from the digests of the replica reads of ``OSD.0`` and
+       ``OSD.1``
+     * ``size_mismatch``: the size of the replica read from ``OSD.2`` is ``0``,
+       but the size reported by ``OSD.0`` and ``OSD.1`` is ``968``.
+
+  #. ``union_shard_errors``: the union of all shard-specific ``errors`` in the
+     ``shards`` array. The ``errors`` are set for the shard with the problem.
+     These errors include ``read_error`` and other similar errors. The
+     ``errors`` ending in ``oi`` indicate a comparison with
+     ``selected_object_info``. Examine the ``shards`` array to determine
+     which shard has which error or errors.
+
+     * ``data_digest_mismatch_info``: the digest stored in the ``object-info``
+       is not ``0xffffffff``, which is calculated from the shard read from
+       ``OSD.2``
+     * ``size_mismatch_info``: the size stored in the ``object-info`` is
+       different from the size read from ``OSD.2``. The latter is ``0``.
+
+.. warning:: If ``read_error`` is listed in a shard's ``errors`` attribute, the
+   inconsistency is likely due to physical storage errors. In cases like this,
+   check the storage used by that OSD. 
+   
+   Examine the output of ``dmesg`` and ``smartctl`` before attempting a drive
+   repair.
+
+To repair the inconsistent placement group, run a command of the following
+form:
 
-    * ``data_digest_mismatch_info``: the digest stored in the object-info is not
-      ``0xffffffff``, which is calculated from the shard read from OSD.2
-    * ``size_mismatch_info``: the size stored in the object-info is different
-      from the one read from OSD.2. The latter is 0.
-
-You can repair the inconsistent placement group by executing::
-
-	ceph pg repair {placement-group-ID}
+.. prompt:: bash
 
-Which overwrites the `bad` copies with the `authoritative` ones. In most cases,
-Ceph is able to choose authoritative copies from all available replicas using
-some predefined criteria. But this does not always work. For example, the stored
-data digest could be missing, and the calculated digest will be ignored when
-choosing the authoritative copies. So, please use the above command with caution.
+   ceph pg repair {placement-group-ID}
+    
+.. warning: This command overwrites the "bad" copies with "authoritative"
+   copies. In most cases, Ceph is able to choose authoritative copies from all
+   the available replicas by using some predefined criteria. This, however,
+   does not work in every case. For example, it might be the case that the
+   stored data digest is missing, which means that the calculated digest is
+   ignored when Ceph chooses the authoritative copies. Be aware of this, and
+   use the above command with caution.
 
-If ``read_error`` is listed in the ``errors`` attribute of a shard, the
-inconsistency is likely due to disk errors. You might want to check your disk
-used by that OSD.
 
 If you receive ``active + clean + inconsistent`` states periodically due to
-clock skew, you may consider configuring your `NTP`_ daemons on your
-monitor hosts to act as peers. See `The Network Time Protocol`_ and Ceph
-`Clock Settings`_ for additional details.
+clock skew, consider configuring the `NTP
+<https://en.wikipedia.org/wiki/Network_Time_Protocol>`_ daemons on your monitor
+hosts to act as peers. See `The Network Time Protocol <http://www.ntp.org>`_
+and Ceph :ref:`Clock Settings <mon-config-ref-clock>` for more information.
 
 
 Erasure Coded PGs are not active+clean
 ======================================
 
-When CRUSH fails to find enough OSDs to map to a PG, it will show as a
-``2147483647`` which is ITEM_NONE or ``no OSD found``. For instance::
+If CRUSH fails to find enough OSDs to map to a PG, it will show as a
+``2147483647`` which is ``ITEM_NONE`` or ``no OSD found``. For example::
 
      [2,1,6,0,5,8,2147483647,7,4]
 
 Not enough OSDs
 ---------------
 
-If the Ceph cluster only has 8 OSDs and the erasure coded pool needs
-9, that is what it will show. You can either create another erasure
-coded pool that requires less OSDs::
+If the Ceph cluster has only eight OSDs and an erasure coded pool needs nine
+OSDs, the cluster will show "Not enough OSDs". In this case, you either create
+another erasure coded pool that requires fewer OSDs, by running commands of the
+following form:
+
+.. prompt:: bash
 
      ceph osd erasure-code-profile set myprofile k=5 m=3
      ceph osd pool create erasurepool erasure myprofile
 
-or add a new OSDs and the PG will automatically use them.
+or add new OSDs, and the PG will automatically use them.
 
 CRUSH constraints cannot be satisfied
 -------------------------------------
 
-If the cluster has enough OSDs, it is possible that the CRUSH rule
-imposes constraints that cannot be satisfied. If there are 10 OSDs on
-two hosts and the CRUSH rule requires that no two OSDs from the
-same host are used in the same PG, the mapping may fail because only
-two OSDs will be found. You can check the constraint by displaying ("dumping")
-the rule::
+If the cluster has enough OSDs, it is possible that the CRUSH rule is imposing
+constraints that cannot be satisfied. If there are ten OSDs on two hosts and
+the CRUSH rule requires that no two OSDs from the same host are used in the
+same PG, the mapping may fail because only two OSDs will be found. Check the
+constraint by displaying ("dumping") the rule, as shown here:
+
+.. prompt:: bash
+
+   ceph osd crush rule ls
+
+::
 
-    $ ceph osd crush rule ls
     [
         "replicated_rule",
         "erasurepool"]
@@ -566,36 +616,43 @@ the rule::
             { "op": "emit"}]}
 
 
-You can resolve the problem by creating a new pool in which PGs are allowed
-to have OSDs residing on the same host with::
+Resolve this problem by creating a new pool in which PGs are allowed to have
+OSDs residing on the same host by running the following commands:
 
-     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd
-     ceph osd pool create erasurepool erasure myprofile
+.. prompt:: bash
+
+   ceph osd erasure-code-profile set myprofile crush-failure-domain=osd
+   ceph osd pool create erasurepool erasure myprofile
 
 CRUSH gives up too soon
 -----------------------
 
-If the Ceph cluster has just enough OSDs to map the PG (for instance a
-cluster with a total of 9 OSDs and an erasure coded pool that requires
-9 OSDs per PG), it is possible that CRUSH gives up before finding a
-mapping. It can be resolved by:
+If the Ceph cluster has just enough OSDs to map the PG (for instance a cluster
+with a total of nine OSDs and an erasure coded pool that requires nine OSDs per
+PG), it is possible that CRUSH gives up before finding a mapping. This problem
+can be resolved by:
 
-* lowering the erasure coded pool requirements to use less OSDs per PG
-  (that requires the creation of another pool as erasure code profiles
-  cannot be dynamically modified).
+* lowering the erasure coded pool requirements to use fewer OSDs per PG (this
+  requires the creation of another pool, because erasure code profiles cannot
+  be modified dynamically).
 
-* adding more OSDs to the cluster (that does not require the erasure
-  coded pool to be modified, it will become clean automatically)
+* adding more OSDs to the cluster (this does not require the erasure coded pool
+  to be modified, because it will become clean automatically)
 
-* use a handmade CRUSH rule that tries more times to find a good
-  mapping. This can be done by setting ``set_choose_tries`` to a value
-  greater than the default.
+* using a handmade CRUSH rule that tries more times to find a good mapping.
+  This can be modified for an existing CRUSH rule by setting
+  ``set_choose_tries`` to a value greater than the default.
 
-You should first verify the problem with ``crushtool`` after
-extracting the crushmap from the cluster so your experiments do not
-modify the Ceph cluster and only work on a local files::
+First, verify the problem by using  ``crushtool`` after extracting the crushmap
+from the cluster. This ensures that your experiments do not modify the Ceph
+cluster and that they operate only on local files:
+
+.. prompt:: bash
+
+   ceph osd crush rule dump erasurepool
+
+::
 
-    $ ceph osd crush rule dump erasurepool
     { "rule_id": 1,
       "rule_name": "erasurepool",
       "type": 3,
@@ -617,44 +674,54 @@ modify the Ceph cluster and only work on a local files::
     bad mapping rule 8 x 79 num_rep 9 result [6,0,2,1,4,7,2147483647,5,8]
     bad mapping rule 8 x 173 num_rep 9 result [0,4,6,8,2,1,3,7,2147483647]
 
-Where ``--num-rep`` is the number of OSDs the erasure code CRUSH
-rule needs, ``--rule`` is the value of the ``rule_id`` field
-displayed by ``ceph osd crush rule dump``.  The test will try mapping
-one million values (i.e. the range defined by ``[--min-x,--max-x]``)
-and must display at least one bad mapping. If it outputs nothing it
-means all mappings are successful and you can stop right there: the
-problem is elsewhere.
+Here, ``--num-rep`` is the number of OSDs that the erasure code CRUSH rule
+needs, ``--rule`` is the value of the ``rule_id`` field that was displayed by
+``ceph osd crush rule dump``. This test will attempt to map one million values
+(in this example, the range defined by ``[--min-x,--max-x]``) and must display
+at least one bad mapping. If this test outputs nothing, all mappings have been
+successful and you can be assured that the problem with your cluster is not
+caused by bad mappings.
 
-The CRUSH rule can be edited by decompiling the crush map::
+Changing the value of set_choose_tries
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    $ crushtool --decompile crush.map > crush.txt
+#. Decompile the CRUSH map to edit the CRUSH rule by running the following
+   command:
 
-and adding the following line to the rule::
+   .. prompt:: bash
 
-    step set_choose_tries 100
+      crushtool --decompile crush.map > crush.txt
 
-The relevant part of the ``crush.txt`` file should look something
-like::
+#. Add the following line to the rule::
 
-     rule erasurepool {
-             id 1
-             type erasure
-             step set_chooseleaf_tries 5
-             step set_choose_tries 100
-             step take default
-             step chooseleaf indep 0 type host
-             step emit
-     }
+      step set_choose_tries 100
 
-It can then be compiled and tested again::
+   The relevant part of the ``crush.txt`` file will resemble this::
 
-    $ crushtool --compile crush.txt -o better-crush.map
+      rule erasurepool {
+              id 1
+              type erasure
+              step set_chooseleaf_tries 5
+              step set_choose_tries 100
+              step take default
+              step chooseleaf indep 0 type host
+              step emit
+      }
 
-When all mappings succeed, an histogram of the number of tries that
-were necessary to find all of them can be displayed with the
-``--show-choose-tries`` option of ``crushtool``::
+#. Recompile and retest the CRUSH rule:
 
-    $ crushtool -i better-crush.map --test --show-bad-mappings \
+   .. prompt:: bash
+
+      crushtool --compile crush.txt -o better-crush.map
+
+#. When all mappings succeed, display a histogram of the number of tries that
+   were necessary to find all of the mapping by using the
+   ``--show-choose-tries`` option of the ``crushtool`` command, as in the
+   following example:
+
+   .. prompt:: bash
+
+      crushtool -i better-crush.map --test --show-bad-mappings \
        --show-choose-tries \
        --rule 1 \
        --num-rep 9 \
@@ -704,14 +771,12 @@ were necessary to find all of them can be displayed with the
     104:         0
     ...
 
-It took 11 tries to map 42 PGs, 12 tries to map 44 PGs etc. The highest number of tries is the minimum value of ``set_choose_tries`` that prevents bad mappings (i.e. 103 in the above output because it did not take more than 103 tries for any PG to be mapped).
+   This output indicates that it took eleven tries to map forty-two PGs, twelve
+   tries to map forty-four PGs etc. The highest number of tries is the minimum
+   value of ``set_choose_tries`` that prevents bad mappings (for example,
+   ``103`` in the above output, because it did not take more than 103 tries for
+   any PG to be mapped).
 
 .. _check: ../../operations/placement-groups#get-the-number-of-placement-groups
-.. _here: ../../configuration/pool-pg-config-ref
 .. _Placement Groups: ../../operations/placement-groups
 .. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
-.. _NTP: https://en.wikipedia.org/wiki/Network_Time_Protocol
-.. _The Network Time Protocol: http://www.ntp.org/
-.. _Clock Settings: ../../configuration/mon-config-ref/#clock
-
-

From 9bb63bdc8969f2ecdfeedfc8396890ad59f0d796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Miguel=20Olmo=20Mart=C3=ADnez?= <jolmomar@redhat.com>
Date: Wed, 11 Oct 2023 18:00:15 +0200
Subject: [PATCH 0235/2492] doc: Documentation about main Ceph metrics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Co-authored-by: Zac Dover <zac.dover@proton.me>
Signed-off-by: Juan Miguel Olmo Martínez <jolmomar@ibm.com>
---
 doc/index.rst            |   9 +-
 doc/monitoring/index.rst | 474 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 479 insertions(+), 4 deletions(-)
 create mode 100644 doc/monitoring/index.rst

diff --git a/doc/index.rst b/doc/index.rst
index d4ccd087afba..8edc2cb09e0e 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -4,11 +4,11 @@
 
 Ceph delivers **object, block, and file storage in one unified system**.
 
-.. warning:: 
+.. warning::
 
-   :ref:`If this is your first time using Ceph, read the "Basic Workflow" 
-   page in the Ceph Developer Guide to learn how to contribute to the 
-   Ceph project. (Click anywhere in this paragraph to read the "Basic 
+   :ref:`If this is your first time using Ceph, read the "Basic Workflow"
+   page in the Ceph Developer Guide to learn how to contribute to the
+   Ceph project. (Click anywhere in this paragraph to read the "Basic
    Workflow" page of the Ceph Developer Guide.) <basic workflow dev guide>`.
 
 .. note::
@@ -110,6 +110,7 @@ about Ceph, see our `Architecture`_ section.
    radosgw/index
    mgr/index
    mgr/dashboard
+   monitoring/index
    api/index
    architecture
    Developer Guide <dev/developer_guide/index>
diff --git a/doc/monitoring/index.rst b/doc/monitoring/index.rst
new file mode 100644
index 000000000000..2bf2aca90f2f
--- /dev/null
+++ b/doc/monitoring/index.rst
@@ -0,0 +1,474 @@
+.. _monitoring:
+
+===================
+Monitoring overview
+===================
+
+The aim of this part of the documentation is to explain the Ceph monitoring
+stack and the meaning of the main Ceph metrics.
+
+With a good understand of the Ceph monitoring stack and metrics users can
+create customized monitoring tools, like Prometheus queries, Grafana
+dashboards, or scripts.
+
+
+Ceph Monitoring stack
+=====================
+
+Ceph provides a default monitoring stack wich is installed by cephadm and
+explained in the :ref:`Monitoring Services <mgr-cephadm-monitoring>` section of
+the cephadm documentation.
+
+
+Ceph metrics
+============
+
+The main source for Ceph metrics are the performance counters exposed by each
+Ceph daemon. The :doc:`../dev/perf_counters` are native Ceph monitoring data
+
+Performance counters are transformed into standard Prometheus metrics by the
+Ceph exporter daemon. This daemon runs on every Ceph cluster host and exposes a
+metrics end point where all the performance counters exposed by all the Ceph
+daemons running in the host are published in the form of Prometheus metrics.
+
+In addition to the Ceph exporter, there is another agent to expose Ceph
+metrics. It is the Prometheus manager module, wich exposes metrics related to
+the whole cluster, basically metrics that are not produced by individual Ceph
+daemons.
+
+The main source for obtaining Ceph metrics is the metrics endpoint exposed by
+the Cluster Prometheus server.  Ceph can provide you with the Prometheus
+endpoint where you can obtain the complete list of metrics (coming from Ceph
+exporter daemons and Prometheus manager module) and exeute queries.
+
+Use the following command to obtain the Prometheus server endpoint in your
+cluster:
+
+Example:
+
+.. code-block:: bash
+
+  # ceph orch ps --service_name prometheus
+  NAME                         HOST                          PORTS   STATUS          REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID
+  prometheus.cephtest-node-00  cephtest-node-00.cephlab.com  *:9095  running (103m)    50s ago   5w     142M        -  2.33.4   514e6a882f6e  efe3cbc2e521
+
+With this information you can connect to
+``http://cephtest-node-00.cephlab.com:9095`` to access the Prometheus server
+interface.
+
+And the complete list of metrics (with help) for your cluster will be available
+in:
+
+``http://cephtest-node-00.cephlab.com:9095/api/v1/targets/metadata``
+
+
+It is good to outline that the main tool allowing users to observe and monitor a Ceph cluster is the **Ceph dashboard**. It provides graphics where the most important cluster and service metrics are represented. Most of the examples in this document are extracted from the dashboard graphics or extrapolated from the metrics exposed by the Ceph dashboard.
+
+
+Performance metrics
+===================
+
+Main metrics used to measure Cluster Ceph performance:
+
+All metrics have the following labels:
+``ceph_daemon``: identifier of the OSD daemon generating the metric
+``instance``: the IP address of the ceph exporter instance exposing the metric.
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_osd_op_r{ceph_daemon="osd.0", instance="192.168.122.7:9283", job="ceph"} = 73981
+
+*Cluster I/O (throughput):*
+Use ``ceph_osd_op_r_out_bytes`` and ``ceph_osd_op_w_in_bytes`` to obtain the cluster throughput generated by clients
+
+Example:
+
+.. code-block:: bash
+
+  Writes (B/s):
+  sum(irate(ceph_osd_op_w_in_bytes[1m]))
+
+  Reads (B/s):
+  sum(irate(ceph_osd_op_r_out_bytes[1m]))
+
+
+*Cluster I/O (operations):*
+Use ``ceph_osd_op_r``, ``ceph_osd_op_w`` to obtain the number of operations generated by clients
+
+Example:
+
+.. code-block:: bash
+
+  Writes (ops/s):
+  sum(irate(ceph_osd_op_w[1m]))
+
+  Reads (ops/s):
+  sum(irate(ceph_osd_op_r[1m]))
+
+*Latency:*
+Use ``ceph_osd_op_latency_sum`` wich represents the delay before a OSD transfer of data begins following a client instruction for its transfer
+
+Example:
+
+.. code-block:: bash
+
+  sum(irate(ceph_osd_op_latency_sum[1m]))
+
+
+OSD performance
+===============
+
+The previous explained cluster performance metrics are based in OSD metrics, selecting the right label we can obtain for a single OSD the same performance information explained for the cluster:
+
+Example:
+
+.. code-block:: bash
+
+  OSD 0 read latency
+  irate(ceph_osd_op_r_latency_sum{ceph_daemon=~"osd.0"}[1m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[1m])
+
+  OSD 0 write IOPS
+  irate(ceph_osd_op_w{ceph_daemon=~"osd.0"}[1m])
+
+  OSD 0 write thughtput (bytes)
+  irate(ceph_osd_op_w_in_bytes{ceph_daemon=~"osd.0"}[1m])
+
+  OSD.0 total raw capacity available
+  ceph_osd_stat_bytes{ceph_daemon="osd.0", instance="cephtest-node-00.cephlab.com:9283", job="ceph"} = 536451481
+
+
+Physical disk performance:
+==========================
+
+Combining Prometheus ``node_exporter`` metrics with Ceph metrics we can have
+information about the performance provided by physical disks used by OSDs.
+
+Example:
+
+.. code-block:: bash
+
+  Read latency of device used by OSD 0:
+  label_replace(irate(node_disk_read_time_seconds_total[1m]) / irate(node_disk_reads_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Write latency of device used by OSD 0
+  label_replace(irate(node_disk_write_time_seconds_total[1m]) / irate(node_disk_writes_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  IOPS (device used by OSD.0)
+  reads:
+  label_replace(irate(node_disk_reads_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  writes:
+  label_replace(irate(node_disk_writes_completed_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Throughput (device used by OSD.0)
+  reads:
+  label_replace(irate(node_disk_read_bytes_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  writes:
+  label_replace(irate(node_disk_written_bytes_total[1m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+  Physical Device Utilization (%) for OSD.0 in the last 5 minutes
+  label_replace(irate(node_disk_io_time_seconds_total[5m]), "instance", "$1", "instance", "([^:.]*).*") and on (instance, device) label_replace(label_replace(ceph_disk_occupation_human{ceph_daemon=~"osd.0"}, "device", "$1", "device", "/dev/(.*)"), "instance", "$1", "instance", "([^:.]*).*")
+
+Pool metrics
+============
+
+These metrics have the following labels:
+``instance``: the ip address of the Ceph exporter daemon producing the metric.
+``pool_id``: identifier of the pool
+``job``: prometheus scrape job
+
+
+- ``ceph_pool_metadata``: Information about the pool It can be used together
+  with other metrics to provide more contextual information in queries and
+  graphs.  Apart of the three common labels this metric provide the following
+  extra labels:
+
+  - ``compression_mode``: compression used in the pool (lz4, snappy, zlib,
+    zstd, none). Example: compression_mode="none"
+
+  - ``description``: brief description of the pool type (replica:number of
+    replicas or Erasure code: ec profile). Example: description="replica:3"
+  - ``name``: name of the pool. Example: name=".mgr"
+  - ``type``: type of pool (replicated/erasure code). Example: type="replicated"
+
+- ``ceph_pool_bytes_used``: Total raw capacity consumed by user data and associated overheads by pool (metadata + redundancy):
+
+- ``ceph_pool_stored``: Total of CLIENT data stored in the pool
+
+- ``ceph_pool_compress_under_bytes``: Data eligible to be compressed in the pool
+
+- ``ceph_pool_compress_bytes_used``:  Data compressed in the pool
+
+- ``ceph_pool_rd``: CLIENT read operations per pool (reads per second)
+
+- ``ceph_pool_rd_bytes``: CLIENT read operations in bytes per pool
+
+- ``ceph_pool_wr``: CLIENT write operations per pool (writes per second)
+
+- ``ceph_pool_wr_bytes``: CLIENT write operation in bytes per pool
+
+
+**Useful queries**:
+
+.. code-block:: bash
+
+  Total raw capacity available in the cluster:
+  sum(ceph_osd_stat_bytes)
+
+  Total raw capacity consumed in the cluster (including metadata + redundancy):
+  sum(ceph_pool_bytes_used)
+
+  Total of CLIENT data stored in the cluster:
+  sum(ceph_pool_stored)
+
+  Compression savings:
+  sum(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used)
+
+  CLIENT IOPS for a pool (testrbdpool)
+  reads: irate(ceph_pool_rd[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+  writes: irate(ceph_pool_wr[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+
+  CLIENT Throughput for a pool
+  reads: irate(ceph_pool_rd_bytes[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+  writes: irate(ceph_pool_wr_bytes[1m]) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~"testrbdpool"}
+
+Object metrics
+==============
+
+These metrics have the following labels:
+``instance``: the ip address of the ceph exporter daemon providing the metric
+``instance_id``: identifier of the rgw daemon
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_rgw_req{instance="192.168.122.7:9283", instance_id="154247", job="ceph"} = 12345
+
+
+Generic metrics
+---------------
+- ``ceph_rgw_metadata``: Provides generic information about the RGW daemon.  It
+  can be used together with other metrics to provide more contextual
+  information in queries and graphs. Apart from the three common labels, this
+  metric provides the following extra labels:
+
+  - ``ceph_daemon``: Name of the Ceph daemon. Example:
+    ceph_daemon="rgw.rgwtest.cephtest-node-00.sxizyq",
+  - ``ceph_version``: Version of Ceph daemon. Example: ceph_version="ceph
+    version 17.2.6 (d7ff0d10654d2280e08f1ab989c7cdf3064446a5) quincy (stable)",
+  - ``hostname``: Name of the host where the daemon runs. Example:
+    hostname:"cephtest-node-00.cephlab.com",
+
+- ``ceph_rgw_req``: Number total of requests for the daemon (GET+PUT+DELETE)
+    Useful to detect bottlenecks and optimize load distribution.
+
+- ``ceph_rgw_qlen``: RGW operations queue length for the daemon.
+    Useful to detect bottlenecks and optimize load distribution.
+
+- ``ceph_rgw_failed_req``: Aborted requests.
+    Useful to detect daemon errors
+
+
+GET operations: related metrics
+-------------------------------
+- ``ceph_rgw_get_initial_lat_count``: Number of get operations
+
+- ``ceph_rgw_get_initial_lat_sum``: Total latency time for the GET operations
+
+- ``ceph_rgw_get``: Number total of GET requests
+
+- ``ceph_rgw_get_b``: Total bytes transferred in GET operations
+
+
+Put operations: related metrics
+-------------------------------
+- ``ceph_rgw_put_initial_lat_count``: Number of get operations
+
+- ``ceph_rgw_put_initial_lat_sum``: Total latency time for the PUT operations
+
+- ``ceph_rgw_put``: Total number of PUT operations
+
+- ``ceph_rgw_get_b``: Total bytes transferred in PUT operations
+
+
+Useful queries
+--------------
+
+.. code-block:: bash
+
+  The average of get latencies:
+  rate(ceph_rgw_get_initial_lat_sum[30s]) / rate(ceph_rgw_get_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  The average of put latencies:
+  rate(ceph_rgw_put_initial_lat_sum[30s]) / rate(ceph_rgw_put_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Total requests per second:
+  rate(ceph_rgw_req[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Total number of "other" operations (LIST, DELETE)
+  rate(ceph_rgw_req[30s]) -  (rate(ceph_rgw_get[30s]) + rate(ceph_rgw_put[30s]))
+
+  GET latencies
+  rate(ceph_rgw_get_initial_lat_sum[30s]) /  rate(ceph_rgw_get_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  PUT latencies
+  rate(ceph_rgw_put_initial_lat_sum[30s]) /  rate(ceph_rgw_put_initial_lat_count[30s]) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Bandwidth consumed by GET operations
+  sum(rate(ceph_rgw_get_b[30s]))
+
+  Bandwidth consumed by PUT operations
+  sum(rate(ceph_rgw_put_b[30s]))
+
+  Bandwidth consumed by RGW instance (PUTs + GETs)
+  sum by (instance_id) (rate(ceph_rgw_get_b[30s]) + rate(ceph_rgw_put_b[30s])) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata
+
+  Http errors:
+  rate(ceph_rgw_failed_req[30s])
+
+
+Filesystem Metrics
+==================
+
+These metrics have the following labels:
+``ceph_daemon``: The name of the MDS daemon
+``instance``: the ip address (and port) of of the Ceph exporter daemon exposing the metric
+``job``: prometheus scrape job
+
+Example:
+
+.. code-block:: bash
+
+  ceph_mds_request{ceph_daemon="mds.test.cephtest-node-00.hmhsoh", instance="192.168.122.7:9283", job="ceph"} = 1452
+
+
+Main metrics
+------------
+
+- ``ceph_mds_metadata``: Provides general information about the MDS daemon.  It
+  can be used together with other metrics to provide more contextual
+  information in queries and graphs.  It provides the following extra labels:
+
+  - ``ceph_version``: MDS daemon Ceph version
+  - ``fs_id``: filesystem cluster id
+  - ``hostname``: Host name where the MDS daemon runs
+  - ``public_addr``: Public address where the MDS daemon runs
+  - ``rank``: Rank of the MDS daemon
+
+Example:
+
+.. code-block:: bash
+
+ ceph_mds_metadata{ceph_daemon="mds.test.cephtest-node-00.hmhsoh", ceph_version="ceph version 17.2.6 (d7ff0d10654d2280e08f1ab989c7cdf3064446a5) quincy (stable)", fs_id="-1", hostname="cephtest-node-00.cephlab.com", instance="cephtest-node-00.cephlab.com:9283", job="ceph", public_addr="192.168.122.145:6801/118896446", rank="-1"}
+
+
+- ``ceph_mds_request``: Total number of requests for the MDs daemon
+
+- ``ceph_mds_reply_latency_sum``: Reply latency total
+
+- ``ceph_mds_reply_latency_count``: Reply latency count
+
+- ``ceph_mds_server_handle_client_request``: Number of client requests
+
+- ``ceph_mds_sessions_session_count``: Session count
+
+- ``ceph_mds_sessions_total_load``: Total load
+
+- ``ceph_mds_sessions_sessions_open``: Sessions currently open
+
+- ``ceph_mds_sessions_sessions_stale``: Sessions currently stale
+
+- ``ceph_objecter_op_r``: Number of read operations
+
+- ``ceph_objecter_op_w``: Number of write operations
+
+- ``ceph_mds_root_rbytes``: Total number of bytes managed by the daemon
+
+- ``ceph_mds_root_rfiles``: Total number of files managed by the daemon
+
+
+Useful queries:
+---------------
+
+.. code-block:: bash
+
+  Total MDS daemons read workload:
+  sum(rate(ceph_objecter_op_r[1m]))
+
+  Total MDS daemons write workload:
+  sum(rate(ceph_objecter_op_w[1m]))
+
+  MDS daemon read workload: (daemon name is "mdstest")
+  sum(rate(ceph_objecter_op_r{ceph_daemon=~"mdstest"}[1m]))
+
+  MDS daemon write workload: (daemon name is "mdstest")
+  sum(rate(ceph_objecter_op_r{ceph_daemon=~"mdstest"}[1m]))
+
+  The average of reply latencies:
+  rate(ceph_mds_reply_latency_sum[30s]) / rate(ceph_mds_reply_latency_count[30s])
+
+  Total requests per second:
+  rate(ceph_mds_request[30s]) * on (instance) group_right (ceph_daemon) ceph_mds_metadata
+
+
+Block metrics
+=============
+
+By default RBD metrics for images are not available in order to provide the
+best performance in the prometheus manager module.
+
+To produce metrics for RBD images it is needed to configure properly the
+manager option ``mgr/prometheus/rbd_stats_pools``. For more information please
+see :ref:`prometheus-rbd-io-statistics`
+
+
+These metrics have the following labels:
+``image``: Name of the image which produces the metric value.
+``instance``: Node where the rbd metric is produced. (It points to the Ceph exporter daemon)
+``job``: Name of the Prometheus scrape job.
+``pool``: Image pool name.
+
+Example:
+
+.. code-block:: bash
+
+  ceph_rbd_read_bytes{image="test2", instance="cephtest-node-00.cephlab.com:9283", job="ceph", pool="testrbdpool"}
+
+
+Main metrics
+------------
+
+- ``ceph_rbd_read_bytes``: RBD image bytes read
+
+- ``ceph_rbd_read_latency_count``: RBD image reads latency count
+
+- ``ceph_rbd_read_latency_sum``: RBD image reads latency total
+
+- ``ceph_rbd_read_ops``: RBD image reads count
+
+- ``ceph_rbd_write_bytes``: RBD image bytes written
+
+- ``ceph_rbd_write_latency_count``: RBD image writes latency count
+
+- ``ceph_rbd_write_latency_sum``: RBD image writes latency total
+
+- ``ceph_rbd_write_ops``: RBD image writes count
+
+
+Useful queries
+--------------
+
+.. code-block:: bash
+
+  The average of read latencies:
+  rate(ceph_rbd_read_latency_sum[30s]) / rate(ceph_rbd_read_latency_count[30s]) * on (instance) group_left (ceph_daemon) ceph_rgw_metadata
+
+
+
+

From 445724fa61a27071bce02afa084bfc1437acd271 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 19 Oct 2023 11:29:36 +0200
Subject: [PATCH 0236/2492] mgr/dashboard: dashboard area chart unit test

Fixes: https://tracker.ceph.com/issues/63243
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../dashboard-area-chart.component.spec.ts    | 58 ++++++++++++++++++-
 .../dashboard-area-chart.component.ts         | 20 +++----
 .../app/shared/services/formatter.service.ts  |  5 +-
 3 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
index 0501ac75dde6..ec71070544fa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
@@ -1,4 +1,4 @@
-import { NO_ERRORS_SCHEMA } from '@angular/core';
+import { NO_ERRORS_SCHEMA, SimpleChange } from '@angular/core';
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
 import { CssHelper } from '~/app/shared/classes/css-helper';
@@ -28,9 +28,65 @@ describe('DashboardAreaChartComponent', () => {
   beforeEach(() => {
     fixture = TestBed.createComponent(DashboardAreaChartComponent);
     component = fixture.componentInstance;
+    component.data = [
+      [1, '110'],
+      [3, '130']
+    ];
   });
 
   it('should create', () => {
     expect(component).toBeTruthy();
   });
+
+  it('should have a chart', () => {
+    const chartElement = fixture.debugElement.nativeElement.querySelector('canvas');
+    expect(chartElement).toBeTruthy();
+  });
+
+  it('should have two datasets', () => {
+    component.data2 = [
+      [2, '120'],
+      [4, '140']
+    ];
+    expect(component.chartData.dataset[0].data).toBeDefined();
+    expect(component.chartData.dataset[1].data).toBeDefined();
+  });
+
+  it('should set label', () => {
+    component.label = 'Write';
+    expect(component.label).toBe('Write');
+  });
+
+  it('should transform and update data', () => {
+    expect(component.chartData.dataset[0].data).toEqual([{ x: 0, y: 0 }]);
+
+    component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
+
+    expect(component.chartData.dataset[0].data).toEqual([
+      { x: 1000, y: 110 },
+      { x: 3000, y: 130 }
+    ]);
+  });
+
+  it('should set currentData to last value', () => {
+    component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
+    expect(component.currentData).toBe('130');
+  });
+
+  it('should keep data units consistency', () => {
+    // Timeout to be able to access chart object
+    setTimeout(() => {
+      fixture.detectChanges();
+
+      component.data = [
+        [1, '1100'],
+        [3, '1300']
+      ];
+      component.dataUnits = 'B';
+      component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
+
+      expect(component.currentDataUnits).toBe('KiB');
+      expect(component.chartDataUnits).toBe('KiB');
+    }, 1000);
+  });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index c2ed2f35b4fd..70157ab6fa5c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -1,4 +1,4 @@
-import { AfterViewInit, Component, Input, OnChanges, ViewChild } from '@angular/core';
+import { Component, Input, OnChanges, SimpleChanges, ViewChild } from '@angular/core';
 
 import { CssHelper } from '~/app/shared/classes/css-helper';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
@@ -13,7 +13,7 @@ import { NumberFormatterService } from '~/app/shared/services/number-formatter.s
   templateUrl: './dashboard-area-chart.component.html',
   styleUrls: ['./dashboard-area-chart.component.scss']
 })
-export class DashboardAreaChartComponent implements OnChanges, AfterViewInit {
+export class DashboardAreaChartComponent implements OnChanges {
   @ViewChild(BaseChartDirective) chart: BaseChartDirective;
 
   @Input()
@@ -178,19 +178,16 @@ export class DashboardAreaChartComponent implements OnChanges, AfterViewInit {
     };
   }
 
-  ngOnChanges(): void {
-    this.updateChartData();
+  ngOnChanges(changes: SimpleChanges): void {
+    this.updateChartData(changes);
   }
 
-  ngAfterViewInit(): void {
-    this.updateChartData();
-  }
-
-  private updateChartData(): void {
+  private updateChartData(changes: SimpleChanges): void {
     this.chartData.dataset[0].label = this.label;
     this.chartData.dataset[1].label = this.label2;
     this.setChartTicks();
-    if (this.data) {
+    if (changes.data && changes.data.currentValue) {
+      this.data = changes.data.currentValue;
       this.chartData.dataset[0].data = this.formatData(this.data);
       [this.currentData, this.currentDataUnits] = this.convertUnits(
         this.data[this.data.length - 1][1]
@@ -199,7 +196,8 @@ export class DashboardAreaChartComponent implements OnChanges, AfterViewInit {
         this.maxValue
       ).split(' ');
     }
-    if (this.data2) {
+    if (changes.data2 && changes.data2.currentValue) {
+      this.data2 = changes.data2.currentValue;
       this.chartData.dataset[1].data = this.formatData(this.data2);
       [this.currentData2, this.currentDataUnits2] = this.convertUnits(
         this.data2[this.data2.length - 1][1]
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/formatter.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/formatter.service.ts
index b5e0b9475a44..c1ad14b47423 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/formatter.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/formatter.service.ts
@@ -40,7 +40,7 @@ export class FormatterService {
    */
   formatNumberFromTo(
     n: any,
-    units: any,
+    units: string = '',
     targetedUnits: string = '',
     conversionFactor: number,
     unitsArray: string[],
@@ -52,6 +52,9 @@ export class FormatterService {
     if (!_.isNumber(n)) {
       return '-';
     }
+    if (!unitsArray) {
+      return '-';
+    }
     const unitsArrayLowerCase = unitsArray.map((str) => str.toLowerCase());
     if (
       !unitsArrayLowerCase.includes(units.toLowerCase()) ||

From 670118abfb9ef6135972a56514173e72b64ef8f4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 31 Mar 2023 14:18:21 -0400
Subject: [PATCH 0237/2492] cmake: fix Finduring.cmake

liburing-devel package provides `/usr/lib64/liburing.so`. without this
fix, `find_package(uring)` failed with this `CMAKE_FIND_DEBUG_MODE`
output:
```
  find_library considered the following locations:

    /home/cbodley/ceph/build/virtualenv/bin//(lib)liburing.a(\.so|\.a)
    /home/cbodley/ceph/build/virtualenv/bin/(lib)liburing.a(\.so|\.a)
    /usr/local/bin//(lib)liburing.a(\.so|\.a)
    /usr/local/bin/(lib)liburing.a(\.so|\.a)
    /usr/local/sbin//(lib)liburing.a(\.so|\.a)
    /usr/local/sbin/(lib)liburing.a(\.so|\.a)
    /usr/bin//(lib)liburing.a(\.so|\.a)
    /usr/bin/(lib)liburing.a(\.so|\.a)
    /usr/sbin//(lib)liburing.a(\.so|\.a)
    /usr/sbin/(lib)liburing.a(\.so|\.a)
    /usr/local/lib64//(lib)liburing.a(\.so|\.a)
    /usr/local/lib64/(lib)liburing.a(\.so|\.a)
    /usr/local/lib//(lib)liburing.a(\.so|\.a)
    /usr/local/lib/(lib)liburing.a(\.so|\.a)
    /usr/local//(lib)liburing.a(\.so|\.a)
    /usr/local/(lib)liburing.a(\.so|\.a)
    /usr/lib64//(lib)liburing.a(\.so|\.a)
    /usr/lib64/(lib)liburing.a(\.so|\.a)
    /usr/lib//(lib)liburing.a(\.so|\.a)
    /usr/lib/(lib)liburing.a(\.so|\.a)
    /usr//(lib)liburing.a(\.so|\.a)
    /usr/(lib)liburing.a(\.so|\.a)
    /lib64//(lib)liburing.a(\.so|\.a)
    /lib64/(lib)liburing.a(\.so|\.a)
    /lib//(lib)liburing.a(\.so|\.a)
    /lib/(lib)liburing.a(\.so|\.a)
    /opt//(lib)liburing.a(\.so|\.a)
    /opt/(lib)liburing.a(\.so|\.a)
    /usr/lib64/X11//(lib)liburing.a(\.so|\.a)
    /usr/lib64/X11/(lib)liburing.a(\.so|\.a)
```

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 cmake/modules/Finduring.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/Finduring.cmake b/cmake/modules/Finduring.cmake
index 10c8de425501..8a6267ef2f31 100644
--- a/cmake/modules/Finduring.cmake
+++ b/cmake/modules/Finduring.cmake
@@ -5,7 +5,7 @@
 # uring_FOUND - True if uring found.
 
 find_path(URING_INCLUDE_DIR liburing.h)
-find_library(URING_LIBRARIES liburing.a liburing)
+find_library(URING_LIBRARIES uring)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(uring DEFAULT_MSG URING_LIBRARIES URING_INCLUDE_DIR)

From 05c341b30deab327444eac464e24a840dae25083 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 31 Mar 2023 14:33:17 -0400
Subject: [PATCH 0238/2492] cmake: define BOOST_ASIO_HAS_IO_URING for
 WITH_LIBURING

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 28c9209c5579..f5ea2c49b9a6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -263,6 +263,8 @@ if(WITH_LIBURING)
     include(Builduring)
     build_uring()
   endif()
+  # enable uring in boost::asio
+  add_compile_definitions("BOOST_ASIO_HAS_IO_URING")
 endif()
 
 CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF

From f479bbc4bc118989849fb663b5dcf5f46f05097d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 18 Apr 2023 11:06:50 -0400
Subject: [PATCH 0239/2492] common: hide asio header from common/error_code.h

this added a dependency on uring headers where it wasn't necessary.
pick_address.cc was relying on that for the IFF_UP define

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/error_code.cc   | 1 +
 src/common/error_code.h    | 3 +--
 src/common/pick_address.cc | 6 ++++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/common/error_code.cc b/src/common/error_code.cc
index 60086c550aeb..ed0e681b22bf 100644
--- a/src/common/error_code.cc
+++ b/src/common/error_code.cc
@@ -15,6 +15,7 @@
 
 #include <exception>
 
+#include <boost/asio/error.hpp>
 #include "common/error_code.h"
 
 #pragma GCC diagnostic push
diff --git a/src/common/error_code.h b/src/common/error_code.h
index 6bcd8cb1791c..e39122f8ce31 100644
--- a/src/common/error_code.h
+++ b/src/common/error_code.h
@@ -18,8 +18,7 @@
 
 #include <netdb.h>
 
-#include <boost/system/error_code.hpp>
-#include <boost/asio.hpp>
+#include <boost/system.hpp>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index 2fd076808ac7..70a18c25dc31 100644
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -17,6 +17,12 @@
 #include <bitset>
 #include <netdb.h>
 #include <netinet/in.h>
+#ifdef _WIN32
+#include <ws2ipdef.h>
+#else
+#include <arpa/inet.h> // inet_pton()
+#include <net/if.h> // IFF_UP
+#endif
 #include <string>
 #include <string.h>
 #include <vector>

From efe8145467c4e92f4d31144c24dcc8b3a2fc6845 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 10:26:20 -0400
Subject: [PATCH 0240/2492] librbd: include posix/stream_descriptor.hpp header

the use of boost::asio::posix::stream_descriptor requires the
concrete header instead of posix/basic_stream_descriptor.hpp

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/librbd/migration/FileStream.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/librbd/migration/FileStream.h b/src/librbd/migration/FileStream.h
index 32face71e1c5..1a7e20ac7079 100644
--- a/src/librbd/migration/FileStream.h
+++ b/src/librbd/migration/FileStream.h
@@ -8,7 +8,7 @@
 #include "librbd/migration/StreamInterface.h"
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>
-#include <boost/asio/posix/basic_stream_descriptor.hpp>
+#include <boost/asio/posix/stream_descriptor.hpp>
 #include <json_spirit/json_spirit.h>
 #include <memory>
 #include <string>

From 49998361f9a3117559f78b7a66ec299c8568542d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 10:29:13 -0400
Subject: [PATCH 0241/2492] neorados: use granular includes instead of
 boost/asio.hpp

avoid the boost/asio.hpp convenience header, as it pulls in a dependency
on liburing

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/async/bind_handler.h            | 3 ++-
 src/common/async/completion.h              | 2 ++
 src/common/async/forward_handler.h         | 3 ++-
 src/include/neorados/RADOS.hpp             | 3 ++-
 src/mon/MonClient.cc                       | 2 ++
 src/mon/MonClient.h                        | 4 ++++
 src/neorados/RADOSImpl.h                   | 2 +-
 src/osdc/Objecter.h                        | 6 +++++-
 src/test/common/test_async_completion.cc   | 2 ++
 src/test/common/test_blocked_completion.cc | 4 +++-
 src/test/neorados/completions.cc           | 4 +---
 src/test/neorados/start_stop.cc            | 2 ++
 src/tools/neorados.cc                      | 1 -
 13 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/src/common/async/bind_handler.h b/src/common/async/bind_handler.h
index 516d8a5e8b41..69128501a07f 100644
--- a/src/common/async/bind_handler.h
+++ b/src/common/async/bind_handler.h
@@ -16,7 +16,8 @@
 #define CEPH_ASYNC_BIND_HANDLER_H
 
 #include <tuple>
-#include <boost/asio.hpp>
+#include <boost/asio/associated_allocator.hpp>
+#include <boost/asio/associated_executor.hpp>
 
 namespace ceph::async {
 
diff --git a/src/common/async/completion.h b/src/common/async/completion.h
index 6af9109d5479..384c85f0fe38 100644
--- a/src/common/async/completion.h
+++ b/src/common/async/completion.h
@@ -17,6 +17,8 @@
 
 #include <memory>
 
+#include <boost/asio/executor_work_guard.hpp>
+
 #include "bind_handler.h"
 #include "forward_handler.h"
 
diff --git a/src/common/async/forward_handler.h b/src/common/async/forward_handler.h
index ae88cc83f464..1491ef6085d4 100644
--- a/src/common/async/forward_handler.h
+++ b/src/common/async/forward_handler.h
@@ -15,7 +15,8 @@
 #ifndef CEPH_ASYNC_FORWARD_HANDLER_H
 #define CEPH_ASYNC_FORWARD_HANDLER_H
 
-#include <boost/asio.hpp>
+#include <boost/asio/associated_allocator.hpp>
+#include <boost/asio/associated_executor.hpp>
 
 namespace ceph::async {
 
diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index fa1ac92ae518..b4f48d12ffc1 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -24,7 +24,8 @@
 #include <type_traits>
 #include <variant>
 
-#include <boost/asio.hpp>
+#include <boost/asio/async_result.hpp>
+#include <boost/asio/io_context.hpp>
 
 #include <boost/container/flat_map.hpp>
 #include <boost/container/flat_set.hpp>
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index c60268701970..909d964c6658 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -15,6 +15,8 @@
 #include <algorithm>
 #include <iterator>
 #include <random>
+
+#include <boost/asio/post.hpp>
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/adaptor/filtered.hpp>
 #include <boost/range/algorithm/copy.hpp>
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index b72bf1f65749..081893ae9c4a 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -22,6 +22,10 @@
 #include <string>
 #include <vector>
 
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/steady_timer.hpp>
+
 #include "msg/Messenger.h"
 
 #include "MonMap.h"
diff --git a/src/neorados/RADOSImpl.h b/src/neorados/RADOSImpl.h
index d45ca948123e..53853e37e969 100644
--- a/src/neorados/RADOSImpl.h
+++ b/src/neorados/RADOSImpl.h
@@ -18,7 +18,7 @@
 #include <memory>
 #include <string>
 
-#include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/intrusive_ptr.hpp>
 
 #include "common/ceph_context.h"
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index d9d723dca747..a0d9d0b32e76 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -27,7 +27,11 @@
 #include <variant>
 
 #include <boost/container/small_vector.hpp>
-#include <boost/asio.hpp>
+#include <boost/asio/async_result.hpp>
+#include <boost/asio/defer.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/post.hpp>
 
 #include <fmt/format.h>
 
diff --git a/src/test/common/test_async_completion.cc b/src/test/common/test_async_completion.cc
index 4cf4394e1cc0..6ea516d1058b 100644
--- a/src/test/common/test_async_completion.cc
+++ b/src/test/common/test_async_completion.cc
@@ -13,6 +13,8 @@
  */
 
 #include "common/async/completion.h"
+#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
 #include <optional>
 #include <boost/intrusive/list.hpp>
 #include <gtest/gtest.h>
diff --git a/src/test/common/test_blocked_completion.cc b/src/test/common/test_blocked_completion.cc
index 71e5784af7e4..9a694de79e28 100644
--- a/src/test/common/test_blocked_completion.cc
+++ b/src/test/common/test_blocked_completion.cc
@@ -13,7 +13,9 @@
  */
 
 
-#include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/steady_timer.hpp>
 #include <boost/system/error_code.hpp>
 
 #include <gtest/gtest.h>
diff --git a/src/test/neorados/completions.cc b/src/test/neorados/completions.cc
index d9c0e087005e..b6286130bbea 100644
--- a/src/test/neorados/completions.cc
+++ b/src/test/neorados/completions.cc
@@ -1,6 +1,4 @@
-#include <cassert>
-#include <boost/asio.hpp>
-#include <boost/system/system_error.hpp>
+#include <boost/asio/io_context.hpp>
 
 constexpr int max_completions = 10'000'000;
 int completed = 0;
diff --git a/src/test/neorados/start_stop.cc b/src/test/neorados/start_stop.cc
index 4ea0ae5644fb..12ef9b5aa50b 100644
--- a/src/test/neorados/start_stop.cc
+++ b/src/test/neorados/start_stop.cc
@@ -16,6 +16,8 @@
 #include <thread>
 #include <vector>
 
+#include <boost/asio/use_future.hpp>
+
 #include "include/neorados/RADOS.hpp"
 
 #include "common/async/context_pool.h"
diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
index 16eed9ba9ff7..4c5f8475d394 100644
--- a/src/tools/neorados.cc
+++ b/src/tools/neorados.cc
@@ -23,7 +23,6 @@
 #include <tuple>
 #include <vector>
 
-#include <boost/asio.hpp>
 #include <boost/io/ios_state.hpp>
 #include <boost/program_options.hpp>
 #include <boost/system/system_error.hpp>

From bfc591f529c6e06bacc45913d7d650e54a634e6b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 13:02:35 -0400
Subject: [PATCH 0242/2492] common: Graylog uses granular asio headers

and use the updated class name io_context

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/Graylog.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/common/Graylog.h b/src/common/Graylog.h
index c8c50131999d..f70ac754cfa7 100644
--- a/src/common/Graylog.h
+++ b/src/common/Graylog.h
@@ -4,7 +4,8 @@
 #ifndef __CEPH_LOG_GRAYLOG_H
 #define __CEPH_LOG_GRAYLOG_H
 
-#include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/udp.hpp>
 #include <boost/iostreams/filtering_stream.hpp>
 #include <boost/iostreams/filter/zlib.hpp>
 
@@ -66,7 +67,7 @@ class Graylog
   std::string m_logger;
 
   boost::asio::ip::udp::endpoint m_endpoint;
-  boost::asio::io_service m_io_service;
+  boost::asio::io_context m_io_service;
 
   std::unique_ptr<Formatter> m_formatter;
   std::unique_ptr<Formatter> m_formatter_section;

From b56eb6a19503ee0c8fc1c0e72d16ae266a3caf7c Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 14:07:52 -0400
Subject: [PATCH 0243/2492] exporter: use granular asio headers

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/exporter/DaemonMetricCollector.cc | 3 ++-
 src/exporter/DaemonMetricCollector.h  | 2 +-
 src/exporter/http_server.cc           | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/exporter/DaemonMetricCollector.cc b/src/exporter/DaemonMetricCollector.cc
index ebe85c3041e5..3716592a199f 100644
--- a/src/exporter/DaemonMetricCollector.cc
+++ b/src/exporter/DaemonMetricCollector.cc
@@ -1,5 +1,6 @@
 #include "DaemonMetricCollector.h"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/json/src.hpp>
 #include <chrono>
 #include <filesystem>
@@ -43,7 +44,7 @@ void DaemonMetricCollector::request_loop(boost::asio::steady_timer &timer) {
 void DaemonMetricCollector::main() {
   // time to wait before sending requests again
 
-  boost::asio::io_service io;
+  boost::asio::io_context io;
   boost::asio::steady_timer timer{io, std::chrono::seconds(0)};
   request_loop(timer);
   io.run();
diff --git a/src/exporter/DaemonMetricCollector.h b/src/exporter/DaemonMetricCollector.h
index e906fb13a597..77761aaea04f 100644
--- a/src/exporter/DaemonMetricCollector.h
+++ b/src/exporter/DaemonMetricCollector.h
@@ -5,7 +5,7 @@
 #include <string>
 #include <vector>
 
-#include <boost/asio.hpp>
+#include <boost/asio/steady_timer.hpp>
 #include <boost/json/object.hpp>
 #include <filesystem>
 #include <map>
diff --git a/src/exporter/http_server.cc b/src/exporter/http_server.cc
index 317d877e88c1..3eb48a2a1f04 100644
--- a/src/exporter/http_server.cc
+++ b/src/exporter/http_server.cc
@@ -5,7 +5,7 @@
 #include "global/global_context.h"
 #include "exporter/DaemonMetricCollector.h"
 
-#include <boost/asio.hpp>
+#include <boost/asio/ip/tcp.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/http.hpp>
 #include <boost/beast/version.hpp>

From cf85121fd684b228f3139253d9570ba59f4c70d2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 14:41:24 -0400
Subject: [PATCH 0244/2492] immutable-object-cache: use granular asio headers

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/tools/immutable_object_cache/CacheClient.cc  |  8 ++++++--
 src/tools/immutable_object_cache/CacheClient.h   | 16 ++++++++--------
 src/tools/immutable_object_cache/CacheServer.cc  |  2 ++
 src/tools/immutable_object_cache/CacheServer.h   | 10 +++++-----
 src/tools/immutable_object_cache/CacheSession.cc |  6 +++++-
 src/tools/immutable_object_cache/CacheSession.h  | 12 ++++++------
 6 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/src/tools/immutable_object_cache/CacheClient.cc b/src/tools/immutable_object_cache/CacheClient.cc
index 2b837be512c3..44686529547d 100644
--- a/src/tools/immutable_object_cache/CacheClient.cc
+++ b/src/tools/immutable_object_cache/CacheClient.cc
@@ -1,6 +1,10 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include <boost/asio/error.hpp>
+#include <boost/asio/placeholders.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/write.hpp>
 #include <boost/bind/bind.hpp>
 #include "CacheClient.h"
 #include "common/Cond.h"
@@ -25,8 +29,8 @@ namespace immutable_obj_cache {
         "immutable_object_cache_client_dedicated_thread_num");
 
     if (m_worker_thread_num != 0) {
-      m_worker = new boost::asio::io_service();
-      m_worker_io_service_work = new boost::asio::io_service::work(*m_worker);
+      m_worker = new boost::asio::io_context();
+      m_worker_io_service_work = new boost::asio::io_context::work(*m_worker);
       for (uint64_t i = 0; i < m_worker_thread_num; i++) {
         std::thread* thd = new std::thread([this](){m_worker->run();});
         m_worker_threads.push_back(thd);
diff --git a/src/tools/immutable_object_cache/CacheClient.h b/src/tools/immutable_object_cache/CacheClient.h
index b2f749631258..7dc4aa76c132 100644
--- a/src/tools/immutable_object_cache/CacheClient.h
+++ b/src/tools/immutable_object_cache/CacheClient.h
@@ -5,8 +5,8 @@
 #define CEPH_CACHE_CACHE_CLIENT_H
 
 #include <atomic>
-#include <boost/asio.hpp>
-#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/local/stream_protocol.hpp>
 #include <boost/algorithm/string.hpp>
 
 #include "include/ceph_assert.h"
@@ -16,11 +16,11 @@
 #include "SocketCommon.h"
 
 
-using boost::asio::local::stream_protocol;
-
 namespace ceph {
 namespace immutable_obj_cache {
 
+using boost::asio::local::stream_protocol;
+
 class CacheClient {
  public:
   CacheClient(const std::string& file, CephContext* ceph_ctx);
@@ -57,17 +57,17 @@ class CacheClient {
 
  private:
   CephContext* m_cct;
-  boost::asio::io_service m_io_service;
-  boost::asio::io_service::work m_io_service_work;
+  boost::asio::io_context m_io_service;
+  boost::asio::io_context::work m_io_service_work;
   stream_protocol::socket m_dm_socket;
   stream_protocol::endpoint m_ep;
   std::shared_ptr<std::thread> m_io_thread;
   std::atomic<bool> m_session_work;
 
   uint64_t m_worker_thread_num;
-  boost::asio::io_service* m_worker;
+  boost::asio::io_context* m_worker;
   std::vector<std::thread*> m_worker_threads;
-  boost::asio::io_service::work* m_worker_io_service_work;
+  boost::asio::io_context::work* m_worker_io_service_work;
 
   std::atomic<bool> m_writing;
   std::atomic<bool> m_reading;
diff --git a/src/tools/immutable_object_cache/CacheServer.cc b/src/tools/immutable_object_cache/CacheServer.cc
index e94a47c7a5cb..14deddce561b 100644
--- a/src/tools/immutable_object_cache/CacheServer.cc
+++ b/src/tools/immutable_object_cache/CacheServer.cc
@@ -1,6 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include <boost/asio/error.hpp>
+#include <boost/asio/placeholders.hpp>
 #include <boost/bind/bind.hpp>
 #include "common/debug.h"
 #include "common/ceph_context.h"
diff --git a/src/tools/immutable_object_cache/CacheServer.h b/src/tools/immutable_object_cache/CacheServer.h
index 31d8599340d5..8affc92ddd5c 100644
--- a/src/tools/immutable_object_cache/CacheServer.h
+++ b/src/tools/immutable_object_cache/CacheServer.h
@@ -4,19 +4,19 @@
 #ifndef CEPH_CACHE_CACHE_SERVER_H
 #define CEPH_CACHE_CACHE_SERVER_H
 
-#include <boost/asio.hpp>
-#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/local/stream_protocol.hpp>
 
 #include "Types.h"
 #include "SocketCommon.h"
 #include "CacheSession.h"
 
 
-using boost::asio::local::stream_protocol;
-
 namespace ceph {
 namespace immutable_obj_cache {
 
+using boost::asio::local::stream_protocol;
+
 class CacheServer {
  public:
   CacheServer(CephContext* cct, const std::string& file, ProcessMsg processmsg);
@@ -33,7 +33,7 @@ class CacheServer {
 
  private:
   CephContext* cct;
-  boost::asio::io_service m_io_service;
+  boost::asio::io_context m_io_service;
   ProcessMsg m_server_process_msg;
   stream_protocol::endpoint m_local_path;
   stream_protocol::acceptor m_acceptor;
diff --git a/src/tools/immutable_object_cache/CacheSession.cc b/src/tools/immutable_object_cache/CacheSession.cc
index 38c38c97d44e..aace2023e675 100644
--- a/src/tools/immutable_object_cache/CacheSession.cc
+++ b/src/tools/immutable_object_cache/CacheSession.cc
@@ -1,6 +1,10 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include <boost/asio/error.hpp>
+#include <boost/asio/placeholders.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/write.hpp>
 #include <boost/bind/bind.hpp>
 #include "common/debug.h"
 #include "common/ceph_context.h"
@@ -16,7 +20,7 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-CacheSession::CacheSession(io_service& io_service,
+CacheSession::CacheSession(io_context& io_service,
                            ProcessMsg processmsg,
                            CephContext* cct)
     : m_dm_socket(io_service),
diff --git a/src/tools/immutable_object_cache/CacheSession.h b/src/tools/immutable_object_cache/CacheSession.h
index 0826e8a2b9a1..23d83d8a0355 100644
--- a/src/tools/immutable_object_cache/CacheSession.h
+++ b/src/tools/immutable_object_cache/CacheSession.h
@@ -4,21 +4,21 @@
 #ifndef CEPH_CACHE_SESSION_H
 #define CEPH_CACHE_SESSION_H
 
-#include <boost/asio.hpp>
-#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/local/stream_protocol.hpp>
 
 #include "Types.h"
 #include "SocketCommon.h"
 
-using boost::asio::local::stream_protocol;
-using boost::asio::io_service;
-
 namespace ceph {
 namespace immutable_obj_cache {
 
+using boost::asio::local::stream_protocol;
+using boost::asio::io_context;
+
 class CacheSession : public std::enable_shared_from_this<CacheSession> {
  public:
-  CacheSession(io_service& io_service, ProcessMsg process_msg,
+  CacheSession(io_context& io_service, ProcessMsg process_msg,
                 CephContext* ctx);
   ~CacheSession();
   stream_protocol::socket& socket();

From 47b4532a3a6fc547a232af5baf7540fb979b6ee7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 26 Apr 2023 15:25:43 -0400
Subject: [PATCH 0245/2492] rgw: use granular asio includes

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_notify.cc         | 3 +++
 src/rgw/rgw_asio_frontend.cc               | 8 +++++++-
 src/rgw/rgw_coroutine.h                    | 2 +-
 src/rgw/rgw_dmclock_async_scheduler.h      | 3 ++-
 src/test/common/test_async_shared_mutex.cc | 3 +++
 src/test/rgw/bench_rgw_ratelimit.cc        | 5 +++--
 src/test/rgw/test_rgw_throttle.cc          | 4 ++++
 7 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 46b3a2302cd4..6e5eeeb2fabb 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -6,6 +6,9 @@
 #include "cls/lock/cls_lock_client.h"
 #include <memory>
 #include <boost/algorithm/hex.hpp>
+#include <boost/asio/basic_waitable_timer.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/context/protected_fixedsize_stack.hpp>
 #include <spawn/spawn.hpp>
 #include "rgw_sal_rados.h"
diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index e373e4b4cf77..e7d39d7cece6 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -6,7 +6,13 @@
 #include <thread>
 #include <vector>
 
-#include <boost/asio.hpp>
+#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/ip/v6_only.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/write.hpp>
+
 #include <boost/intrusive/list.hpp>
 #include <boost/smart_ptr/intrusive_ref_counter.hpp>
 
diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h
index eb3216640c64..9c0915a3b1e4 100644
--- a/src/rgw/rgw_coroutine.h
+++ b/src/rgw/rgw_coroutine.h
@@ -8,7 +8,7 @@
 #pragma push_macro("_ASSERT_H")
 #endif
 
-#include <boost/asio.hpp>
+#include <boost/asio/coroutine.hpp>
 #include <boost/intrusive_ptr.hpp>
 
 #ifdef NEED_ASSERT_H
diff --git a/src/rgw/rgw_dmclock_async_scheduler.h b/src/rgw/rgw_dmclock_async_scheduler.h
index 7bde75870a5a..f18318f30298 100644
--- a/src/rgw/rgw_dmclock_async_scheduler.h
+++ b/src/rgw/rgw_dmclock_async_scheduler.h
@@ -16,7 +16,8 @@
 
 #include "common/async/completion.h"
 
-#include <boost/asio.hpp>
+#include <boost/asio/basic_waitable_timer.hpp>
+#include <boost/asio/io_context.hpp>
 #include "rgw_dmclock_scheduler.h"
 #include "rgw_dmclock_scheduler_ctx.h"
 
diff --git a/src/test/common/test_async_shared_mutex.cc b/src/test/common/test_async_shared_mutex.cc
index aed6e7b00ee0..ed3a55a70edc 100644
--- a/src/test/common/test_async_shared_mutex.cc
+++ b/src/test/common/test_async_shared_mutex.cc
@@ -13,7 +13,10 @@
  */
 
 #include "common/async/shared_mutex.h"
+#include <future>
 #include <optional>
+#include <boost/asio/bind_executor.hpp>
+#include <boost/asio/io_context.hpp>
 #include <gtest/gtest.h>
 
 namespace ceph::async {
diff --git a/src/test/rgw/bench_rgw_ratelimit.cc b/src/test/rgw/bench_rgw_ratelimit.cc
index 2bf7753ad3e6..1ea8714f9df0 100644
--- a/src/test/rgw/bench_rgw_ratelimit.cc
+++ b/src/test/rgw/bench_rgw_ratelimit.cc
@@ -3,9 +3,10 @@
 #include "random"
 #include <cstdlib>
 #include <string>
-#include <boost/asio.hpp>
-#include <spawn/spawn.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/steady_timer.hpp>
+#include <spawn/spawn.hpp>
 #include <chrono>
 #include <mutex>
 #include <unordered_map>
diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc
index 72dae286cdd6..5e18fc3c1854 100644
--- a/src/test/rgw/test_rgw_throttle.cc
+++ b/src/test/rgw/test_rgw_throttle.cc
@@ -16,6 +16,10 @@
 
 #include <optional>
 #include <thread>
+#include <boost/asio/basic_waitable_timer.hpp>
+#include <boost/asio/error.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
 #include "include/scope_guard.h"
 
 #include <spawn/spawn.hpp>

From e67943e0c5f1959b9de31955060e60831840b000 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 17 Aug 2023 19:39:29 -0400
Subject: [PATCH 0246/2492] test/librados: include io_context.hpp and rename
 io_service

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/librados/asio.cc | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/test/librados/asio.cc b/src/test/librados/asio.cc
index 9f86b4472b5a..459ce6896a54 100644
--- a/src/test/librados/asio.cc
+++ b/src/test/librados/asio.cc
@@ -22,6 +22,7 @@
 #include <boost/range/begin.hpp>
 #include <boost/range/end.hpp>
 #include <spawn/spawn.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/use_future.hpp>
 
 #define dout_subsys ceph_subsys_rados
@@ -74,7 +75,7 @@ librados::IoCtx AsioRados::snapio;
 
 TEST_F(AsioRados, AsyncReadCallback)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   auto success_cb = [&] (boost::system::error_code ec, bufferlist bl) {
     EXPECT_FALSE(ec);
@@ -92,7 +93,7 @@ TEST_F(AsioRados, AsyncReadCallback)
 
 TEST_F(AsioRados, AsyncReadFuture)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   std::future<bufferlist> f1 = librados::async_read(service, io, "exist", 256,
                                                     0, boost::asio::use_future);
@@ -110,7 +111,7 @@ TEST_F(AsioRados, AsyncReadFuture)
 
 TEST_F(AsioRados, AsyncReadYield)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   auto success_cr = [&] (spawn::yield_context yield) {
     boost::system::error_code ec;
@@ -132,7 +133,7 @@ TEST_F(AsioRados, AsyncReadYield)
 
 TEST_F(AsioRados, AsyncWriteCallback)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");
@@ -154,7 +155,7 @@ TEST_F(AsioRados, AsyncWriteCallback)
 
 TEST_F(AsioRados, AsyncWriteFuture)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");
@@ -172,7 +173,7 @@ TEST_F(AsioRados, AsyncWriteFuture)
 
 TEST_F(AsioRados, AsyncWriteYield)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");
@@ -199,7 +200,7 @@ TEST_F(AsioRados, AsyncWriteYield)
 
 TEST_F(AsioRados, AsyncReadOperationCallback)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
   {
     librados::ObjectReadOperation op;
     op.read(0, 0, nullptr, nullptr);
@@ -222,7 +223,7 @@ TEST_F(AsioRados, AsyncReadOperationCallback)
 
 TEST_F(AsioRados, AsyncReadOperationFuture)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
   std::future<bufferlist> f1;
   {
     librados::ObjectReadOperation op;
@@ -248,7 +249,7 @@ TEST_F(AsioRados, AsyncReadOperationFuture)
 
 TEST_F(AsioRados, AsyncReadOperationYield)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   auto success_cr = [&] (spawn::yield_context yield) {
     librados::ObjectReadOperation op;
@@ -276,7 +277,7 @@ TEST_F(AsioRados, AsyncReadOperationYield)
 
 TEST_F(AsioRados, AsyncWriteOperationCallback)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");
@@ -302,7 +303,7 @@ TEST_F(AsioRados, AsyncWriteOperationCallback)
 
 TEST_F(AsioRados, AsyncWriteOperationFuture)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");
@@ -329,7 +330,7 @@ TEST_F(AsioRados, AsyncWriteOperationFuture)
 
 TEST_F(AsioRados, AsyncWriteOperationYield)
 {
-  boost::asio::io_service service;
+  boost::asio::io_context service;
 
   bufferlist bl;
   bl.append("hello");

From fea26180d372264a709b76c367800472b2289b5c Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 12 Oct 2023 14:39:50 -0400
Subject: [PATCH 0247/2492] rgw/admin: remove unused asio.hpp include

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_admin.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index cc7f5811c9ef..fc39d0f7fb0c 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -6,7 +6,6 @@
 #include <sstream>
 #include <string>
 
-#include <boost/asio.hpp>
 #include <boost/optional.hpp>
 
 extern "C" {

From 078ecaa42b98f9858d2e3a045aedb51153b39e34 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 15 Jul 2022 16:39:00 -0400
Subject: [PATCH 0248/2492] mds: ensure next replay is queued on req drop

Not all client replay requests are queued at once since [1]. We require
the next request by queued when completed (unsafely) or during cleanup.
Not all code paths seem to handle this [2] so move it to a generic
location, MDCache::request_cleanup. Even so, this doesn't handle all
errors (so we must still be careful) as sometimes we must queue the next
replay request before an MDRequest is constructed [3] during some error
conditions.

Additionally, preserve the behavior of Server::journal_and_reply
queueing the next replay op. Otherwise, must wait for the request to be
durable before moving onto the next one, unnecessarily.

For reproducing, two specific cases are highlighted (thanks to @Mer1997 on
Github for locating these):

- The request is killed by a session close / eviction while a replayed request
  is queued and waiting for a journal flush (e.g. dirty inest locks).

- The request construction fails because the request is already in the
  active_requests. This could happen theoretically if a client resends the same
  request (same reqid) twice.

The first case is most probable but very difficult to reproduce for testing
purposes. The replayed op would need to wait on a journal flush (to be
restarted by C_MDS_RetryRequest).  Then, the request would need killed by a
session close.

[1] ed6a18d90fdd1dc869369fb92c2aad43bc5c9a34
[2] https://github.com/ceph/ceph/blob/a6f1a1c6c09d74f5918c715b05789f34f2ea0e90/src/mds/Server.cc#L2253-L2262
[3] https://github.com/ceph/ceph/blob/a6f1a1c6c09d74f5918c715b05789f34f2ea0e90/src/mds/Server.cc#L2380

Fixes: https://tracker.ceph.com/issues/56577
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc |  6 ++++++
 src/mds/MDSRank.cc |  2 ++
 src/mds/Mutation.h |  7 +++++++
 src/mds/Server.cc  | 40 ++++++++++++++++++++++++----------------
 4 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 0c812936fdcc..35fca081a6be 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -9887,6 +9887,12 @@ void MDCache::request_cleanup(MDRequestRef& mdr)
   // remove from map
   active_requests.erase(mdr->reqid);
 
+  // queue next replay op?
+  if (mdr->is_queued_for_replay() && !mdr->get_queued_next_replay_op()) {
+    mdr->set_queued_next_replay_op();
+    mds->queue_one_replay();
+  }
+
   if (mds->logger)
     log_stat();
 
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index ad1a7bc7f3f2..d0cdadd58748 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2069,6 +2069,7 @@ bool MDSRank::queue_one_replay()
   if (!replay_queue.empty()) {
     queue_waiter(replay_queue.front());
     replay_queue.pop_front();
+    dout(10) << " queued next replay op" << dendl;
     return true;
   }
   if (!replaying_requests_done) {
@@ -2076,6 +2077,7 @@ bool MDSRank::queue_one_replay()
     mdlog->flush();
   }
   maybe_clientreplay_done();
+  dout(10) << " journaled last replay op" << dendl;
   return false;
 }
 
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index b963dee08420..bc83f2191514 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -387,6 +387,12 @@ struct MDRequestImpl : public MutationImpl {
   void set_filepath(const filepath& fp);
   void set_filepath2(const filepath& fp);
   bool is_queued_for_replay() const;
+  bool get_queued_next_replay_op() const {
+    return queued_next_replay_op;
+  }
+  void set_queued_next_replay_op() {
+    queued_next_replay_op = true;
+  }
   int compare_paths();
 
   bool can_batch();
@@ -460,6 +466,7 @@ struct MDRequestImpl : public MutationImpl {
   }
   void _dump(ceph::Formatter *f, bool has_mds_lock) const;
   void _dump_op_descriptor(std::ostream& stream) const override;
+  bool queued_next_replay_op = false;
 };
 
 struct MDPeerUpdate {
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index ecbb781188ee..fac0f5b3f42b 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -306,6 +306,7 @@ void Server::dispatch(const cref_t<Message> &m)
 	return;
       }
       bool queue_replay = false;
+      dout(5) << "dispatch request in up:reconnect: " << *req << dendl;
       if (req->is_replay() || req->is_async()) {
 	dout(3) << "queuing replayed op" << dendl;
 	queue_replay = true;
@@ -324,10 +325,13 @@ void Server::dispatch(const cref_t<Message> &m)
 	// process completed request in clientreplay stage. The completed request
 	// might have created new file/directorie. This guarantees MDS sends a reply
 	// to client before other request modifies the new file/directorie.
-	if (session->have_completed_request(req->get_reqid().tid, NULL)) {
-	  dout(3) << "queuing completed op" << dendl;
+        bool r = session->have_completed_request(req->get_reqid().tid, NULL);
+	if (r) {
+	  dout(3) << __func__ << ": queuing completed op" << dendl;
 	  queue_replay = true;
-	}
+	} else {
+          dout(20) << __func__  << ": request not complete" << dendl;
+        }
 	// this request was created before the cap reconnect message, drop any embedded
 	// cap releases.
 	req->releases.clear();
@@ -2037,12 +2041,15 @@ void Server::journal_and_reply(MDRequestRef& mdr, CInode *in, CDentry *dn, LogEv
   mdr->committing = true;
   submit_mdlog_entry(le, fin, mdr, __func__);
   
-  if (mdr->client_request && mdr->client_request->is_queued_for_replay()) {
-    if (mds->queue_one_replay()) {
-      dout(10) << " queued next replay op" << dendl;
-    } else {
-      dout(10) << " journaled last replay op" << dendl;
-    }
+  if (mdr->is_queued_for_replay()) {
+
+    /* We want to queue the next replay op while waiting for the journaling, so
+     * do it now when the early (unsafe) replay is dispatched. Don't wait until
+     * this request is cleaned up in MDCache.cc.
+     */
+
+    mdr->set_queued_next_replay_op();
+    mds->queue_one_replay();
   } else if (mdr->did_early_reply)
     mds->locker->drop_rdlocks_for_early_reply(mdr.get());
   else
@@ -2350,15 +2357,12 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
     mds->send_message(reply, mdr->client_request->get_connection());
   }
 
-  if (req->is_queued_for_replay() &&
-      (mdr->has_completed || reply->get_result() < 0)) {
-    if (reply->get_result() < 0) {
-      int r = reply->get_result();
+  if (req->is_queued_for_replay()) {
+    if (int r = reply->get_result(); r < 0) {
       derr << "reply_client_request: failed to replay " << *req
-	   << " error " << r << " (" << cpp_strerror(r)  << ")" << dendl;
+           << " error " << r << " (" << cpp_strerror(r)  << ")" << dendl;
       mds->clog->warn() << "failed to replay " << req->get_reqid() << " error " << r;
     }
-    mds->queue_one_replay();
   }
 
   // clean up request
@@ -2557,8 +2561,12 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
 
   // register + dispatch
   MDRequestRef mdr = mdcache->request_start(req);
-  if (!mdr.get())
+  if (!mdr.get()) {
+    dout(5) << __func__ << ": possibly duplicate op " << *req << dendl;
+    if (req->is_queued_for_replay())
+      mds->queue_one_replay();
     return;
+  }
 
   if (session) {
     mdr->session = session;

From 167fdd22e38d9febef8d7e0969403261314e3c98 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 16 Oct 2023 00:11:19 -0400
Subject: [PATCH 0249/2492] client: always refresh mds feature bits on session
 open

Fixes: http://tracker.ceph.com/issues/63188
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/client/Client.cc | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index b7c645abe3cb..7d223018c762 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -2384,6 +2384,12 @@ void Client::_closed_mds_session(MetaSession *s, int err, bool rejected)
     mds_sessions.erase(s->mds_num);
 }
 
+static void reinit_mds_features(MetaSession *session,
+				const MConstRef<MClientSession>& m) {
+  session->mds_features = std::move(m->supported_features);
+  session->mds_metric_flags = std::move(m->metric_spec.metric_flags);
+}
+
 void Client::handle_client_session(const MConstRef<MClientSession>& m)
 {
   mds_rank_t from = mds_rank_t(m->get_source().num());
@@ -2402,6 +2408,13 @@ void Client::handle_client_session(const MConstRef<MClientSession>& m)
       if (session->state == MetaSession::STATE_OPEN) {
         ldout(cct, 10) << "mds." << from << " already opened, ignore it"
                        << dendl;
+	// The MDS could send a client_session(open) message even when
+	// the session state is STATE_OPEN. Normally, its fine to
+	// ignore this message, but, if the MDS sent this message just
+	// after it got upgraded, the MDS feature bits could differ
+	// than the one before the upgrade - so, refresh the feature
+	// bits the client holds.
+	reinit_mds_features(session.get(), m);
         return;
       }
       /*
@@ -2411,8 +2424,7 @@ void Client::handle_client_session(const MConstRef<MClientSession>& m)
       if (!session->seq && m->get_seq())
         session->seq = m->get_seq();
 
-      session->mds_features = std::move(m->supported_features);
-      session->mds_metric_flags = std::move(m->metric_spec.metric_flags);
+      reinit_mds_features(session.get(), m);
       cap_auths = std::move(m->cap_auths);
 
       renew_caps(session.get());

From be6ce973a66aa21b35760dc3b4271e682a4d596b Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Thu, 19 Oct 2023 20:44:52 +0000
Subject: [PATCH 0250/2492] rgw: remove move's, rename put/get counters, revert
 exporter changes

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 doc/radosgw/metrics.rst               | 38 +++++++++++++--------------
 src/exporter/DaemonMetricCollector.cc |  7 ++---
 src/rgw/rgw_file.cc                   |  6 ++---
 src/rgw/rgw_op.cc                     | 18 ++++++-------
 src/rgw/rgw_perf_counters.cc          | 20 +++++++-------
 src/rgw/rgw_perf_counters.h           | 12 ++++-----
 6 files changed, 49 insertions(+), 52 deletions(-)

diff --git a/doc/radosgw/metrics.rst b/doc/radosgw/metrics.rst
index 75ef782fa6aa..dd5c56c1752a 100644
--- a/doc/radosgw/metrics.rst
+++ b/doc/radosgw/metrics.rst
@@ -20,22 +20,22 @@ The following metrics related to S3 or Swift operations are tracked per Ceph Obj
    * - Name
      - Type
      - Description
-   * - put_ops
+   * - put_obj_ops
      - Counter
      - Number of put operations
-   * - put_b
+   * - put_obj_bytes
      - Counter
      - Number of bytes put
-   * - put_initial_lat
+   * - put_obj_lat
      - Guage
      - Total latency of put operations
-   * - get_ops
+   * - get_obj_ops
      - Counter
      - Number of get operations
-   * - get_b
+   * - get_obj_bytes
      - Counter
      - Number of bytes from get requests
-   * - get_initial_lat
+   * - get_obj_lat
      - Guage
      - Total latency of get operations
    * - del_obj_ops
@@ -82,16 +82,16 @@ To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` section of th
         {
             "labels": {},
             "counters": {
-                "put_ops": 2,
-                "put_b": 5327,
-                "put_initial_lat": {
+                "put_obj_ops": 2,
+                "put_obj_bytes": 5327,
+                "put_obj_lat": {
                     "avgcount": 2,
                     "sum": 2.818064835,
                     "avgtime": 1.409032417
                 },
-                "get_ops": 5,
-                "get_b": 5325,
-                "get_initial_lat": {
+                "get_obj_ops": 5,
+                "get_obj_bytes": 5325,
+                "get_obj_lat": {
                     "avgcount": 2,
                     "sum": 0.003000069,
                     "avgtime": 0.001500034
@@ -108,7 +108,7 @@ To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` section of th
     ]
 
 Op Metrics Labels
---------------------
+-----------------
 
 Op metrics can also be tracked per-user or per-bucket. These metrics are exported to Prometheus with labels like Bucket = {name} or User = {userid}::
 
@@ -119,16 +119,16 @@ Op metrics can also be tracked per-user or per-bucket. These metrics are exporte
                 "Bucket": "bucket1"
             },
             "counters": {
-                "put_ops": 2,
-                "put_b": 5327,
-                "put_initial_lat": {
+                "put_obj_ops": 2,
+                "put_obj_bytes": 5327,
+                "put_obj_lat": {
                     "avgcount": 2,
                     "sum": 2.818064835,
                     "avgtime": 1.409032417
                 },
-                "get_ops": 5,
-                "get_b": 5325,
-                "get_initial_lat": {
+                "get_obj_ops": 5,
+                "get_obj_bytes": 5325,
+                "get_obj_lat": {
                     "avgcount": 2,
                     "sum": 0.003000069,
                     "avgtime": 0.001500034
diff --git a/src/exporter/DaemonMetricCollector.cc b/src/exporter/DaemonMetricCollector.cc
index 23a0dd550f1c..ebe85c3041e5 100644
--- a/src/exporter/DaemonMetricCollector.cc
+++ b/src/exporter/DaemonMetricCollector.cc
@@ -160,11 +160,8 @@ void DaemonMetricCollector::dump_asok_metrics() {
             labels.insert(multisite_labels_and_name.first.begin(), multisite_labels_and_name.first.end());
             counter_name = multisite_labels_and_name.second;
           }
-          auto counters_values_itr = counters_values.find(counter_name_init);
-          if (counters_values_itr != counters_values.end()) {
-            auto perf_values = counters_values_itr->value();
-            dump_asok_metric(counter_group, perf_values, counter_name, labels);
-          }
+          auto perf_values = counters_values.at(counter_name_init);
+          dump_asok_metric(counter_group, perf_values, counter_name, labels);
         }
       }
     }
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index ca149ad836b8..045c18cafe2a 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1839,7 +1839,7 @@ namespace rgw {
     ceph_assert(! slo_info);
 
     counters = rgw::op_counters::get(state);
-    rgw::op_counters::inc(counters, l_rgw_op_put, 1);
+    rgw::op_counters::inc(counters, l_rgw_op_put_obj, 1);
     op_ret = -EINVAL;
 
     if (state->object->empty()) {
@@ -1945,7 +1945,7 @@ namespace rgw {
     real_time appx_t = real_clock::now();
 
     state->obj_size = bytes_written;
-    rgw::op_counters::inc(counters, l_rgw_op_put_b, state->obj_size);
+    rgw::op_counters::inc(counters, l_rgw_op_put_obj_b, state->obj_size);
 
     // flush data in filters
     op_ret = filter->process({}, state->obj_size);
@@ -2028,7 +2028,7 @@ namespace rgw {
     }
 
   done:
-    rgw::op_counters::tinc(counters, l_rgw_op_put_lat, state->time_elapsed());
+    rgw::op_counters::tinc(counters, l_rgw_op_put_obj_lat, state->time_elapsed());
     return op_ret;
   } /* exec_finish */
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 29a1f36c0273..5e0fd42f7da0 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1693,7 +1693,7 @@ int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
   }
 
   auto counters = rgw::op_counters::get(s);
-  rgw::op_counters::inc(counters, l_rgw_op_get_b, cur_end - cur_ofs);
+  rgw::op_counters::inc(counters, l_rgw_op_get_obj_b, cur_end - cur_ofs);
   filter->fixup_range(cur_ofs, cur_end);
   op_ret = read_op->iterate(this, cur_ofs, cur_end, filter, s->yield);
   if (op_ret >= 0)
@@ -1767,7 +1767,7 @@ static int iterate_user_manifest_parts(const DoutPrefixProvider *dpp,
       }
 
       rgw::op_counters::CountersContainer counters;
-      rgw::op_counters::tinc(counters, l_rgw_op_get_lat, 
+      rgw::op_counters::tinc(counters, l_rgw_op_get_obj_lat,
                             (ceph_clock_now() - start_time));
 
       if (found_start && !handled_end) {
@@ -1864,7 +1864,7 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
     }
 
     rgw::op_counters::CountersContainer counters;
-    rgw::op_counters::tinc(counters, l_rgw_op_get_lat, 
+    rgw::op_counters::tinc(counters, l_rgw_op_get_obj_lat,
                           (ceph_clock_now() - start_time));
 
     if (found_start) {
@@ -2213,7 +2213,7 @@ void RGWGetObj::execute(optional_yield y)
   map<string, bufferlist>::iterator attr_iter;
 
   auto counters = rgw::op_counters::get(s);
-  rgw::op_counters::inc(counters, l_rgw_op_get, 1);
+  rgw::op_counters::inc(counters, l_rgw_op_get_obj, 1);
 
   std::unique_ptr<rgw::sal::Object::ReadOp> read_op(s->object->get_read_op());
 
@@ -2411,14 +2411,14 @@ void RGWGetObj::execute(optional_yield y)
     return;
   }
 
-  rgw::op_counters::inc(counters, l_rgw_op_get_b, end-ofs);
+  rgw::op_counters::inc(counters, l_rgw_op_get_obj_b, end-ofs);
 
   op_ret = read_op->iterate(this, ofs_x, end_x, filter, s->yield);
 
   if (op_ret >= 0)
     op_ret = filter->flush();
 
-  rgw::op_counters::tinc(counters, l_rgw_op_get_lat, s->time_elapsed());
+  rgw::op_counters::tinc(counters, l_rgw_op_get_obj_lat, s->time_elapsed());
 
   if (op_ret < 0) {
     goto done_err;
@@ -4030,11 +4030,11 @@ void RGWPutObj::execute(optional_yield y)
   auto counters = rgw::op_counters::get(s);
 
   bool need_calc_md5 = (dlo_manifest == NULL) && (slo_info == NULL);
-  rgw::op_counters::inc(counters, l_rgw_op_put, 1);
+  rgw::op_counters::inc(counters, l_rgw_op_put_obj, 1);
 
   // report latency on return
   auto put_lat = make_scope_guard([&] {
-      rgw::op_counters::tinc(counters, l_rgw_op_put_lat, s->time_elapsed());
+      rgw::op_counters::tinc(counters, l_rgw_op_put_obj_lat, s->time_elapsed());
     });
 
   op_ret = -EINVAL;
@@ -4309,7 +4309,7 @@ void RGWPutObj::execute(optional_yield y)
   s->obj_size = ofs;
   s->object->set_obj_size(ofs);
 
-  rgw::op_counters::inc(counters, l_rgw_op_put_b, s->obj_size);
+  rgw::op_counters::inc(counters, l_rgw_op_put_obj_b, s->obj_size);
 
   op_ret = do_aws4_auth_completion();
   if (op_ret < 0) {
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index f2d245c27d2e..c3e89211cca6 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -64,13 +64,13 @@ void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
   // description must match general rgw counters description above
   lpcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
 
-  lpcb->add_u64_counter(l_rgw_op_put, "put_ops", "Puts");
-  lpcb->add_u64_counter(l_rgw_op_put_b, "put_b", "Size of puts");
-  lpcb->add_time_avg(l_rgw_op_put_lat, "put_initial_lat", "Put latency");
+  lpcb->add_u64_counter(l_rgw_op_put_obj, "put_obj_ops", "Puts");
+  lpcb->add_u64_counter(l_rgw_op_put_obj_b, "put_obj_bytes", "Size of puts");
+  lpcb->add_time_avg(l_rgw_op_put_obj_lat, "put_obj_lat", "Put latency");
 
-  lpcb->add_u64_counter(l_rgw_op_get, "get_ops", "Gets");
-  lpcb->add_u64_counter(l_rgw_op_get_b, "get_b", "Size of gets");
-  lpcb->add_time_avg(l_rgw_op_get_lat, "get_initial_lat", "Get latency");
+  lpcb->add_u64_counter(l_rgw_op_get_obj, "get_obj_ops", "Gets");
+  lpcb->add_u64_counter(l_rgw_op_get_obj_b, "get_obj_bytes", "Size of gets");
+  lpcb->add_time_avg(l_rgw_op_get_obj_lat, "get_obj_lat", "Get latency");
 
   lpcb->add_u64_counter(l_rgw_op_del_obj, "del_obj_ops", "Delete objects");
   lpcb->add_u64_counter(l_rgw_op_del_obj_b, "del_obj_bytes", "Size of delete objects");
@@ -129,18 +129,18 @@ CountersContainer get(req_state *s) {
 
   if (user_counters_cache && !s->user->get_id().id.empty()) {
     if (s->user->get_tenant().empty()) {
-      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}}));
+      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}});
     } else {
-      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}, {"Tenant", s->user->get_tenant()}}));
+      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}, {"Tenant", s->user->get_tenant()}});
     }
     counters.user_counters = user_counters_cache->get(key);
   }
 
   if (bucket_counters_cache && !s->bucket_name.empty()) {
     if (s->bucket_tenant.empty()) {
-      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}}));
+      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}});
     } else {
-      key = std::move(ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}, {"Tenant", s->bucket_tenant}}));
+      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}, {"Tenant", s->bucket_tenant}});
     }
     counters.bucket_counters = bucket_counters_cache->get(key);
   }
diff --git a/src/rgw/rgw_perf_counters.h b/src/rgw/rgw_perf_counters.h
index e9068f4c9ff2..5abd7e44b13d 100644
--- a/src/rgw/rgw_perf_counters.h
+++ b/src/rgw/rgw_perf_counters.h
@@ -55,13 +55,13 @@ enum {
 enum {
   l_rgw_op_first = 16000,
 
-  l_rgw_op_put,
-  l_rgw_op_put_b,
-  l_rgw_op_put_lat,
+  l_rgw_op_put_obj,
+  l_rgw_op_put_obj_b,
+  l_rgw_op_put_obj_lat,
 
-  l_rgw_op_get,
-  l_rgw_op_get_b,
-  l_rgw_op_get_lat,
+  l_rgw_op_get_obj,
+  l_rgw_op_get_obj_b,
+  l_rgw_op_get_obj_lat,
 
   l_rgw_op_del_obj,
   l_rgw_op_del_obj_b,

From ebbc50c90794f6d8eef6dbf9b55fbc68e01c7b15 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Fri, 6 Oct 2023 21:41:58 -0400
Subject: [PATCH 0251/2492] RGW/Roles: use the target/new max-session-duration
 value when validating it

If we validate before updating the role's max-session-duration, the
validator function wrongly uses the on-disk (existing/old) value for
validation. Note that the "role" object being updated is in-memory and
only after validation passes, it's persisted on-disk. So, calling role
object's update_max_session_duration API function is OK before the
role->validate_max_session_duration call.

validate_max_session_duration is used by both "role creation" and "role
update". The latter wrongly uses existing role's max_session_duration
value for validation instead of the new/target duration:

$ radosgw-admin ... role create --role-name=myrole ...
$ radosgw-admin ... role get --role-name=myrole | jq
'.MaxSessionDuration'
3600

where 3600 seconds is the default value.

$ radosgw-admin ... role update --role-name=myrole
--max_session_duration=100000
Max session duration updated successfully for role: myrole

Although above update call should have failed since 100K is higher than
43200 (the default max), it succeeded.

$ radosgw-admin ... role get --role-name=myrole | jq
'.MaxSessionDuration'
100000

Fixes: https://tracker.ceph.com/issues/63109

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_admin.cc     | 2 +-
 src/rgw/rgw_rest_role.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index cc7f5811c9ef..25955a52213c 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -6922,11 +6922,11 @@ int main(int argc, const char **argv)
       if (ret < 0) {
         return -ret;
       }
+      role->update_max_session_duration(max_session_duration);
       if (!role->validate_max_session_duration(dpp())) {
         ret = -EINVAL;
         return ret;
       }
-      role->update_max_session_duration(max_session_duration);
       ret = role->update(dpp(), null_yield);
       if (ret < 0) {
         return -ret;
diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index e71dff5708ff..14e164553665 100644
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -1005,12 +1005,12 @@ void RGWUpdateRole::execute(optional_yield y)
     }
   }
 
+  _role->update_max_session_duration(max_session_duration);
   if (!_role->validate_max_session_duration(this)) {
     op_ret = -EINVAL;
     return;
   }
 
-  _role->update_max_session_duration(max_session_duration);
   op_ret = _role->update(this, y);
 
   s->formatter->open_object_section("UpdateRoleResponse");

From 08dffc23921fe1be98bce72d16d8bcc73e5e6b63 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Fri, 6 Oct 2023 21:53:23 -0400
Subject: [PATCH 0252/2492] RGW/Roles: honor "--max-session-duration" option
 when CREATING a role

Currently, this option is not honored and the default (3600s) is used
regardless of whether this option is provided when creating a role:

$ radosgw-admin role create  --role-name=myrole --max-session-duration=43200 ...
$ radosgw-admin role get --role-name=myrole | jq '.MaxSessionDuration'
3600

With this commit, the value given by the --max-session-duration is
considered when creating the role. This would reduce the need for
updating the role's max-session-duration using a separate "role update"
radosgw-admin command call after the role is created:

$ radosgw-admin role create  --role-name=myrole --max-session-duration=43200 ...
$ radosgw-admin role get --role-name=myrole | jq '.MaxSessionDuration'
43200

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_admin.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 25955a52213c..4b4b5977aa6c 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -6712,7 +6712,8 @@ int main(int argc, const char **argv)
         cerr << "failed to parse policy: " << e.what() << std::endl;
         return -EINVAL;
       }
-      std::unique_ptr<rgw::sal::RGWRole> role = driver->get_role(role_name, tenant, path, assume_role_doc);
+      std::unique_ptr<rgw::sal::RGWRole> role = driver->get_role(role_name, tenant, path,
+                                                                 assume_role_doc, max_session_duration);
       ret = role->create(dpp(), true, "", null_yield);
       if (ret < 0) {
         return -ret;

From e5abfff9380ff4369bc6a55f79842b6fad558366 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Fri, 6 Oct 2023 22:09:23 -0400
Subject: [PATCH 0253/2492] RGW/Roles: add more details to the cfg items
 rgw_sts_{max, min}_session_duration

Fixes: https://tracker.ceph.com/issues/63109

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/common/options/rgw.yaml.in | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 3971929e412d..7b4cba0a1c63 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3295,8 +3295,11 @@ options:
   type: uint
   level: advanced
   desc: Session token max duration
-  long_desc: Max duration in seconds for which the session token is valid.
+  long_desc: This option can be used to configure the upper limit of the
+    durationSeconds of temporary credentials returned by 'GetSessionToken'.
   default: 43200
+  see_also:
+  - rgw_sts_min_session_duration
   services:
   - rgw
   with_legacy: true
@@ -3304,10 +3307,14 @@ options:
   type: uint
   level: advanced
   desc: Minimum allowed duration of a session
+  long_desc: This option can be used to configure the lower limit of
+    durationSeconds of temporary credentials returned by 'AssumeRole*' calls.
   default: 900
   services:
   - rgw
   with_legacy: true
+  see_also:
+  - rgw_sts_max_session_duration
 - name: rgw_max_listing_results
   type: uint
   level: advanced

From f0e8565b4908bf2e3f40ef7750cfc90812653a2b Mon Sep 17 00:00:00 2001
From: Javier <sjavierlopez@gmail.com>
Date: Fri, 19 May 2023 18:32:13 -0600
Subject: [PATCH 0254/2492] monitoring: update libsonnet files for generate
 ceph-cluster.json

add ceph-cluster.libsonnet file to generate ceph-cluster.json

Fixes: https://tracker.ceph.com/issues/61443
Signed-off-by: Javier <sjavierlopez@gmail.com>
---
 monitoring/ceph-mixin/dashboards.libsonnet    |    1 +
 .../dashboards/ceph-cluster.libsonnet         | 1682 ++++++++
 .../dashboards/timeseries_panel.libsonnet     |  141 +
 .../ceph-mixin/dashboards/utils.libsonnet     |  409 +-
 .../dashboards_out/ceph-cluster-advanced.json | 3792 +++++++++++++++++
 .../features/ceph-cluster.feature             |  562 ++-
 .../ceph-mixin/tests_dashboards/util.py       |   12 +-
 7 files changed, 6541 insertions(+), 58 deletions(-)
 create mode 100644 monitoring/ceph-mixin/dashboards/ceph-cluster.libsonnet
 create mode 100644 monitoring/ceph-mixin/dashboards/timeseries_panel.libsonnet
 create mode 100644 monitoring/ceph-mixin/dashboards_out/ceph-cluster-advanced.json

diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet
index 5cae183294f9..4c67f21c039e 100644
--- a/monitoring/ceph-mixin/dashboards.libsonnet
+++ b/monitoring/ceph-mixin/dashboards.libsonnet
@@ -6,5 +6,6 @@
     (import 'dashboards/pool.libsonnet') +
     (import 'dashboards/rbd.libsonnet') +
     (import 'dashboards/rgw.libsonnet') +
+    (import 'dashboards/ceph-cluster.libsonnet') +
     { _config:: $._config },
 }
diff --git a/monitoring/ceph-mixin/dashboards/ceph-cluster.libsonnet b/monitoring/ceph-mixin/dashboards/ceph-cluster.libsonnet
new file mode 100644
index 000000000000..4e916d32427e
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/ceph-cluster.libsonnet
@@ -0,0 +1,1682 @@
+local g = import 'grafonnet/grafana.libsonnet';
+local u = import 'utils.libsonnet';
+
+(import 'utils.libsonnet') {
+  'ceph-cluster-advanced.json': u.dashboardSchema(
+    'Ceph Cluster - Advanced',
+    'Ceph cluster overview',
+    '',
+    'now-6h',
+    '1m',
+    38,
+    $._config.dashboardTags,
+    ''
+  ).addAnnotation(
+    u.addAnnotationSchema(
+      1,
+      '-- Grafana --',
+      true,  // enable
+      true,  // hide
+      'rgba(0, 211, 255, 1)',
+      'Annotations & Alerts',
+      'dashboard'
+    )
+  ).addRequired(
+    type='grafana', id='grafana', name='Grafana', version='5.3.2'
+  ).addRequired(
+    type='panel', id='graph', name='Graph', version='5.0.0'
+  ).addRequired(
+    type='panel', id='heatmap', name='Heatmap', version='5.0.0'
+  ).addRequired(
+    type='panel', id='singlestat', name='Singlestat', version='5.0.0'
+  ).
+    addTemplate(g.template.datasource('DS_PROMETHEUS', 'prometheus', 'Prometheus', label='Data Source')).
+    addTemplate(
+    u.addCustomTemplate(
+      name='interval',
+      query='5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d',
+      current='$__auto_interval_interval',
+      refresh=2,
+      label='Interval',
+      auto_count=10,
+      auto_min='1m',
+      options=[
+        { selected: true, text: 'auto', value: '$__auto_interval_interval' },
+        { selected: false, text: '5s', value: '5s' },
+        { selected: false, text: '10s', value: '10s' },
+        { selected: false, text: '30s', value: '30s' },
+        { selected: false, text: '1m', value: '1m' },
+        { selected: false, text: '10m', value: '10m' },
+        { selected: false, text: '30m', value: '30m' },
+        { selected: false, text: '1h', value: '1h' },
+        { selected: false, text: '6h', value: '6h' },
+        { selected: false, text: '12h', value: '12h' },
+        { selected: false, text: '1d', value: '1d' },
+        { selected: false, text: '7d', value: '7d' },
+        { selected: false, text: '14d', value: '14d' },
+        { selected: false, text: '30d', value: '30d' },
+      ],
+      auto=true,
+    )
+  ).addPanels(
+    [
+      u.addRowSchema(collapse=false, showTitle=true, title='CLUSTER STATE') + { gridPos: { x: 0, y: 0, w: 24, h: 1 } },
+      u.addStatPanel(
+        title='Ceph health status',
+        unit='none',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 0, y: 1, w: 3, h: 3 },
+        colorMode='value',
+        interval='1m',
+        transparent=true,
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7'
+      ).addMappings([
+        {
+          options: {
+            '0': { text: 'HEALTHY' },
+            '1': { text: 'WARNING' },
+            '2': { text: 'ERROR' },
+          },
+          type: 'value',
+        },
+        { options: { match: null, result: { text: 'N/A' } }, type: 'special' },
+      ])
+      .addThresholds([
+        { color: '#9ac48a' },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 1 },
+        { color: 'rgba(245, 54, 54, 0.9)', value: 2 },
+      ])
+      .addTarget(u.addTargetSchema(
+        expr='ceph_health_status{}',
+        instant=true,
+        interval='$interval',
+        datasource='${DS_PROMETHEUS}',
+        step=300,
+      )),
+
+      u.addGaugePanel(
+        title='Available Capacity',
+        gridPosition={ h: 6, w: 3, x: 3, y: 1 },
+        unit='percentunit',
+        max=1,
+        min=0,
+        interval='1m',
+        pluginVersion='9.4.7'
+      ).addMappings([
+        { options: { match: null, result: { text: 'N/A' } }, type: 'special' },
+      ])
+      .addThresholds([
+        { color: 'rgba(245, 54, 54, 0.9)' },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 0.1 },
+        { color: 'rgba(50, 172, 45, 0.97)', value: 0.3 },
+      ])
+      .addTarget(u.addTargetSchema(
+        expr='(ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{})/ceph_cluster_total_bytes{}',
+        instant=true,
+        interval='$interval',
+        datasource='${DS_PROMETHEUS}',
+        step=300
+      )),
+
+      u.addStatPanel(
+        title='Cluster Capacity',
+        unit='decbytes',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 6, y: 1, w: 3, h: 3 },
+        graphMode='area',
+        decimals=2,
+        interval='1m',
+        color={ fixedColor: 'rgb(31, 120, 193)', mode: 'fixed' },
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addMappings([
+        { options: { match: null, result: { text: 'N/A' } }, type: 'special' },
+      ]).addThresholds([
+        { color: 'rgba(50, 172, 45, 0.97)' },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 0.025 },
+        { color: 'rgba(245, 54, 54, 0.9)', value: 1.0 },
+      ])
+      .addTarget(u.addTargetSchema(
+        expr='ceph_cluster_total_bytes{}',
+        instant=true,
+        interval='$interval',
+        datasource='${DS_PROMETHEUS}',
+        step=300
+      )),
+
+      u.addStatPanel(
+        title='Write Throughput',
+        unit='Bps',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 9, y: 1, w: 3, h: 3 },
+        decimals=1,
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addMappings([
+        { options: { match: null, result: { text: 'N/A' } }, type: 'special' },
+      ]).addThresholds([
+        { color: 'green' },
+      ])
+      .addTarget(u.addTargetSchema(
+        expr='sum(irate(ceph_osd_op_w_in_bytes{}[5m]))',
+        instant=true,
+        interval='$interval',
+        datasource='${DS_PROMETHEUS}',
+      )),
+
+      u.addStatPanel(
+        title='Read Throughput',
+        unit='Bps',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 12, y: 1, w: 3, h: 3 },
+        decimals=1,
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addMappings([
+        { options: { match: null, result: { text: 'N/A' } }, type: 'special' },
+      ]).addThresholds([
+        { color: '#d44a3a' },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 0 },
+        { color: '#9ac48a', value: 0 },
+      ])
+      .addTarget(u.addTargetSchema(
+        expr='sum(irate(ceph_osd_op_r_out_bytes{}[5m]))',
+        instant=true,
+        interval='$interval',
+        datasource='${DS_PROMETHEUS}',
+      )),
+
+      u.addStatPanel(
+        title='OSDs',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 6, x: 15, y: 1 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='auto',
+        rootColorMode='Panel',
+        displayName='',
+        rootColors={
+          crit: 'rgb(255, 0, 0)',
+          disable: 'rgba(128, 128, 128, 0.9)',
+          ok: 'rgba(50, 128, 45, 0.9)',
+          warn: 'rgba(237, 129, 40, 0.9)',
+        },
+        cornerRadius=0,
+        flipCard=false,
+        flipTime=5,
+        isAutoScrollOnOverflow=false,
+        isGrayOnNoData=false,
+        isHideAlertsOnDisable=false,
+        isIgnoreOKColors=false,
+        fontFormat='Regular',
+        colorMode='background',
+        unit='none',
+        pluginVersion='9.4.7',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='All',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ceph_osd_metadata)',
+          legendFormat='All',
+          interval='$interval',
+          datasource='${DS_PROMETHEUS}',
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='In',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ceph_osd_in)',
+          legendFormat='In',
+          interval='$interval',
+          datasource='${DS_PROMETHEUS}',
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Out',
+          decimals=2,
+          displayAliasType='Warning / Critical',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='sum(ceph_osd_in == bool 0)',
+          legendFormat='Out',
+          interval='',
+          warn=1,
+          datasource='${DS_PROMETHEUS}',
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Up',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='sum(ceph_osd_up)',
+          legendFormat='Up',
+          interval='',
+          datasource='${DS_PROMETHEUS}',
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Down',
+          decimals=2,
+          displayAliasType='Warning / Critical',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='sum(ceph_osd_up == bool 0)',
+          legendFormat='Down',
+          interval='',
+          warn=1,
+          datasource='${DS_PROMETHEUS}',
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='MGRs',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 6, w: 3, x: 21, y: 1 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='auto',
+        rootColorMode='Panel',
+        displayName='',
+        rootColors={
+          crit: 'rgba(245, 54, 54, 0.9)',
+          disable: 'rgba(128, 128, 128, 0.9)',
+          ok: 'rgba(50, 128, 45, 0.9)',
+          warn: 'rgba(237, 129, 40, 0.9)',
+        },
+        cornerRadius=1,
+        flipCard=false,
+        flipTime=5,
+        isAutoScrollOnOverflow=false,
+        isGrayOnNoData=false,
+        isHideAlertsOnDisable=false,
+        isIgnoreOKColors=false,
+        fontFormat='Regular',
+        colorMode='background',
+        unit='none',
+        pluginVersion='9.4.7',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Active',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ceph_mgr_status == 1) or vector(0)',
+          legendFormat='Active',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Standby',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ceph_mgr_status == 0) or vector(0)',
+          legendFormat='Standby',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='Firing Alerts',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 3, x: 0, y: 4 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='auto',
+        rootColorMode='Panel',
+        displayName='',
+        rootColors={
+          crit: 'rgba(245, 54, 54, 0.9)',
+          disable: 'rgba(128, 128, 128, 0.9)',
+          ok: 'rgba(50, 128, 45, 0.9)',
+          warn: 'rgba(237, 129, 40, 0.9)',
+        },
+        cornerRadius=1,
+        flipCard=false,
+        flipTime=5,
+        isAutoScrollOnOverflow=false,
+        isGrayOnNoData=false,
+        isHideAlertsOnDisable=false,
+        isIgnoreOKColors=false,
+        fontFormat='Regular',
+        colorMode='background',
+        unit='none',
+        pluginVersion='9.4.7',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 1 },
+      ])
+      .addOverrides([
+        { matcher: { id: 'byName', options: 'Critical' }, properties: [
+          { id: 'color', value: { fixedColor: 'red', mode: 'fixed' } },
+        ] },
+        { matcher: { id: 'byName', options: 'Warning' }, properties: [
+          { id: 'color', value: { fixedColor: '#987d24', mode: 'fixed' } },
+        ] },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Active',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ALERTS{alertstate="firing",alertname=~"^Ceph.+", severity="critical"}) OR vector(0)',
+          legendFormat='Critical',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Standby',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ALERTS{alertstate="firing",alertname=~"^Ceph.+", severity="warning"}) OR vector(0)',
+          legendFormat='Warning',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='Used Capacity',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 3, x: 6, y: 4 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='horizontal',
+        graphMode='area',
+        displayName='',
+        maxDataPoints=100,
+        colorMode='none',
+        unit='decbytes',
+        pluginVersion='9.4.7',
+      )
+      .addMappings([
+        { options: { result: { text: 'N/A' } }, type: 'special' },
+      ])
+      .addThresholds([
+        { color: 'rgba(50, 172, 45, 0.97)', value: null },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 0.025 },
+        { color: 'rgba(245, 54, 54, 0.9)', value: 0.1 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          expr='ceph_cluster_total_used_bytes{}',
+          legendFormat='',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='Write IOPS',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 3, x: 9, y: 4 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='horizontal',
+        graphMode='area',
+        displayName='',
+        maxDataPoints=100,
+        colorMode='none',
+        unit='ops',
+        pluginVersion='9.4.7',
+      )
+      .addMappings([
+        { options: { result: { text: 'N/A' } }, type: 'special' },
+      ])
+      .addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          expr='sum(irate(ceph_osd_op_w{}[1m]))',
+          legendFormat='',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='Read IOPS',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 3, x: 12, y: 4 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='horizontal',
+        graphMode='area',
+        displayName='',
+        maxDataPoints=100,
+        colorMode='none',
+        unit='ops',
+        pluginVersion='9.4.7',
+      )
+      .addMappings([
+        { options: { result: { text: 'N/A' } }, type: 'special' },
+      ])
+      .addThresholds([
+        { color: '#d44a3a', value: null },
+        { color: 'rgba(237, 129, 40, 0.89)', value: 0 },
+        { color: '#9ac48a', value: 0 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          expr='sum(irate(ceph_osd_op_r{}[1m]))',
+          legendFormat='',
+          datasource='${DS_PROMETHEUS}',
+          instant=true,
+        ),
+      ]),
+
+      u.addStatPanel(
+        title='Monitors',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 3, w: 6, x: 15, y: 4 },
+        color={ mode: 'thresholds' },
+        thresholdsMode='absolute',
+        orientation='auto',
+        rootColorMode='Panel',
+        displayName='',
+        rootColors={
+          crit: 'rgba(245, 54, 54, 0.9)',
+          disable: 'rgba(128, 128, 128, 0.9)',
+          ok: 'rgba(50, 128, 45, 0.9)',
+          warn: 'rgba(237, 129, 40, 0.9)',
+        },
+        cornerRadius=1,
+        flipCard=false,
+        flipTime=5,
+        isAutoScrollOnOverflow=false,
+        isGrayOnNoData=false,
+        isHideAlertsOnDisable=false,
+        isIgnoreOKColors=false,
+        fontFormat='Regular',
+        colorMode='background',
+        unit='none',
+        pluginVersion='9.4.7',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='In Quorum',
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Text Only',
+          expr='sum(ceph_mon_quorum_status)',
+          legendFormat='In Quorum',
+          datasource='${DS_PROMETHEUS}',
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='Total',
+          crit=1,
+          decimals=2,
+          displayAliasType='Always',
+          displayType='Regular',
+          displayValueWithAlias='When Alias Displayed',
+          units='none',
+          valueHandler='Text Only',
+          expr='count(ceph_mon_quorum_status)',
+          legendFormat='Total',
+          datasource='${DS_PROMETHEUS}',
+          warn=2,
+        ),
+        u.addTargetSchema(
+          aggregation='Last',
+          alias='MONs out of Quorum',
+          crit=1.6,
+          decimals=2,
+          displayAliasType='Warning / Critical',
+          displayType='Annotation',
+          displayValueWithAlias='Never',
+          units='none',
+          valueHandler='Number Threshold',
+          expr='count(ceph_mon_quorum_status) - sum(ceph_mon_quorum_status)',
+          legendFormat='MONs out of Quorum',
+          datasource='${DS_PROMETHEUS}',
+          warn=1.1,
+          range=true,
+        ),
+      ]),
+      u.addRowSchema(collapse=false, showTitle=true, title='CLUSTER STATS') + { gridPos: { x: 0, y: 7, w: 24, h: 1 } },
+      u.addAlertListPanel(
+        title='Alerts',
+        datasource={
+          type: 'datasource',
+          uid: 'grafana',
+        },
+        gridPosition={ h: 8, w: 8, x: 0, y: 8 },
+        alertInstanceLabelFilter='{alertname=~"^Ceph.+"}',
+        alertName='',
+        dashboardAlerts=false,
+        groupBy=[],
+        groupMode='default',
+        maxItems=20,
+        sortOrder=1,
+        stateFilter={
+          'error': true,
+          firing: true,
+          noData: false,
+          normal: false,
+          pending: true,
+        },
+      ),
+
+      u.timeSeriesPanel(
+        title='Capacity',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 8, w: 8, x: 8, y: 8 },
+        fillOpacity=40,
+        pointSize=5,
+        showPoints='never',
+        unit='bytes',
+        displayMode='table',
+        tooltip={ mode: 'multi', sort: 'desc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=2,
+        thresholdsMode='percentage',
+        sortBy='Last',
+        sortDesc=true,
+      )
+      .addCalcs(['last'])
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: '#c0921f', value: 75 },
+        { color: '#E02F44', value: 85 },
+      ])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byName', options: 'Total Capacity' },
+            properties: [{
+              id: 'color',
+              value: { fixedColor: 'red', mode: 'fixed' },
+            }],
+          },
+          {
+            matcher: { id: 'byName', options: 'Used' },
+            properties: [
+              {
+                id: 'color',
+                value: { fixedColor: 'green', mode: 'fixed' },
+              },
+              {
+                id: 'custom.thresholdsStyle',
+                value: { mode: 'dashed' },
+              },
+            ],
+          },
+        ]
+      )
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='ceph_cluster_total_bytes{}',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            instant=false,
+            legendFormat='Total Capacity',
+            step=300,
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='ceph_cluster_total_used_bytes{}',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            instant=false,
+            legendFormat='Used',
+            step=300,
+            range=true,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Cluster Throughput',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 8, w: 8, x: 16, y: 8 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='decbytes',
+        displayMode='table',
+        tooltip={ mode: 'multi', sort: 'desc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+      ).addCalcs(['mean', 'lastNotNull', 'max', 'min'])
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 85 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='sum(irate(ceph_osd_op_w_in_bytes{}[5m]))',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Write',
+            step=300,
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(irate(ceph_osd_op_r_out_bytes{}[5m]))',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Read',
+            step=300,
+            range=true,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='IOPS',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 8, w: 8, x: 0, y: 16 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='decbytes',
+        displayMode='table',
+        tooltip={ mode: 'multi', sort: 'desc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+      )
+      .addCalcs(['mean', 'lastNotNull', 'max', 'min'])
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='sum(irate(ceph_osd_op_w{}[1m]))',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Write',
+            step=300,
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(irate(ceph_osd_op_r{}[1m]))',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Read',
+            step=300,
+            range=true,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Pool Used Bytes',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 8, w: 8, x: 8, y: 16 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='bytes',
+        tooltip={ mode: 'multi', sort: 'desc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='right',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='(ceph_pool_bytes_used{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='{{name}}',
+            step=300,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Pool Used RAW Bytes',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 8, w: 8, x: 16, y: 16 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='bytes',
+        tooltip={ mode: 'multi', sort: 'desc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='table',
+        placement='right',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byName', options: 'rbd Stored' },
+            properties: [{
+              id: 'color',
+              value: { fixedColor: 'transparent', mode: 'fixed' },
+            }],
+          },
+        ]
+      )
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='(ceph_pool_stored_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='',
+            legendFormat='{{name}}',
+            step=300,
+            range=true,
+            hide=false,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Pool Objects Quota',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 7, w: 8, x: 0, y: 24 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'none' },
+        interval='$interval',
+        stackingMode='none',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='bottom',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='(ceph_pool_quota_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='',
+            legendFormat='{{name}}',
+            step=300,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Pool Quota Bytes',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 7, w: 8, x: 8, y: 24 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='bytes',
+        tooltip={ mode: 'multi', sort: 'none' },
+        interval='$interval',
+        stackingMode='none',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='bottom',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='(ceph_pool_quota_bytes{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='',
+            legendFormat='{{name}}',
+            step=300,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Objects Per Pool',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 7, w: 8, x: 16, y: 24 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=1,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'none' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=false,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='right',
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='(ceph_pool_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='',
+            legendFormat='{{name}}',
+          ),
+        ]
+      ),
+
+      u.addRowSchema(collapse=false, showTitle=true, title='OBJECTS') + { gridPos: { x: 0, y: 31, w: 24, h: 1 } },
+
+      u.timeSeriesPanel(
+        title='OSD Type Count',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 12, w: 6, x: 0, y: 32 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=2,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'asc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='bottom',
+        showLegend=false,
+      )
+      .addThresholds([
+        { color: 'green' },
+        { color: 'red', value: 80 },
+      ])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byRegexp', options: '/^Total.*$/' },
+            properties: [{
+              id: 'custom.stacking',
+              value: { group: false, mode: 'normal' },
+            }],
+          },
+        ]
+      )
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='sum(ceph_pool_objects)',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Total',
+            range=true,
+            step=200
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='PGs State',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 12, w: 8, x: 6, y: 32 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=2,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'asc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='table',
+        placement='right',
+        showLegend=true,
+      )
+      .addThresholds([
+        { color: 'green' },
+        { color: 'red', value: 80 },
+      ])
+      .addCalcs(['lastNotNull'])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byRegexp', options: '/^Total.*$/' },
+            properties: [{
+              id: 'custom.stacking',
+              value: { group: false, mode: 'normal' },
+            }],
+          },
+        ]
+      )
+      .addTargets(
+        [
+          u.addTargetSchema(
+            expr='sum(ceph_pg_active{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Active',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_clean{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Clean',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_peering{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Peering',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_degraded{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Degraded',
+            range=true,
+            step=300,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_stale{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Stale',
+            range=true,
+            step=300,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_unclean_pgs{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Unclean',
+            range=true,
+            step=300,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_undersized{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Undersized',
+            range=true,
+            step=300,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_incomplete{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Incomplete',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_forced_backfill{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Forced Backfill',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_forced_recovery{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Forced Recovery',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_creating{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Creating',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_wait_backfill{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Wait Backfill',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_deep{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Deep',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_scrubbing{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Scrubbing',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_recovering{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Recovering',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_repair{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Repair',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_down{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Down',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_peered{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Peered',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_backfill{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Backfill',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_remapped{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Remapped',
+            range=true,
+          ),
+          u.addTargetSchema(
+            expr='sum(ceph_pg_backfill_toofull{})',
+            datasource='${DS_PROMETHEUS}',
+            interval='$interval',
+            legendFormat='Backfill Toofull',
+            range=true,
+          ),
+        ]
+      ),
+
+      u.timeSeriesPanel(
+        title='Stuck PGs',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 6, w: 10, x: 14, y: 32 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=2,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'asc' },
+        interval='$interval',
+        stackingMode='normal',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='table',
+        placement='right',
+        showLegend=true,
+      )
+      .addCalcs(['mean', 'lastNotNull'])
+      .addThresholds([
+        { color: 'green' },
+        { color: 'red', value: 80 },
+      ])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byRegexp', options: '/^Total.*$/' },
+            properties: [{
+              id: 'custom.stacking',
+              value: { group: false, mode: 'normal' },
+            }],
+          },
+        ]
+      )
+      .addTargets([
+        u.addTargetSchema(
+          expr='sum(ceph_pg_degraded{})',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          legendFormat='Degraded',
+          range=true,
+          step=300,
+        ),
+        u.addTargetSchema(
+          expr='sum(ceph_pg_stale{})',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          legendFormat='Stale',
+          range=true,
+          step=300,
+        ),
+        u.addTargetSchema(
+          expr='sum(ceph_pg_undersized{})',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          legendFormat='Undersized',
+          range=true,
+          step=300,
+        ),
+      ]),
+
+      u.timeSeriesPanel(
+        title='Recovery Operations',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ h: 6, w: 10, x: 14, y: 38 },
+        fillOpacity=10,
+        pointSize=5,
+        lineWidth=2,
+        showPoints='never',
+        unit='short',
+        tooltip={ mode: 'multi', sort: 'none' },
+        interval='$interval',
+        stackingMode='none',
+        spanNulls=true,
+        decimals=null,
+        thresholdsMode='absolute',
+        displayMode='list',
+        placement='bottom',
+        showLegend=false,
+      )
+      .addThresholds([
+        { color: 'green' },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        u.addTargetSchema(
+          expr='sum(irate(ceph_osd_recovery_ops{}[$interval]))',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          legendFormat='OPS',
+          step=300,
+        ),
+      ]),
+      u.addRowSchema(false, true, 'LATENCY', collapsed=true)
+      .addPanels([
+        u.heatMapPanel(
+          title='OSD Apply Latency Distribution',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 42 },
+          colorMode='opacity',
+          legendShow=true,
+          optionsCalculate=true,
+          optionsColor={
+            exponent: 0.5,
+            fill: '#b4ff00',
+            mode: 'opacity',
+            reverse: false,
+            scale: 'exponential',
+            scheme: 'Oranges',
+            steps: 128,
+          },
+          optionsExemplars={ color: 'rgba(255,0,255,0.7)' },
+          optionsFilterValues={ le: 1e-9 },
+          optionsLegend={ show: true },
+          optionsRowFrame={ layout: 'auto' },
+          optionsToolTip={
+            show: true,
+            yHistogram: false,
+          },
+          optionsYAxis={
+            axisPlacement: 'left',
+            min: '0',
+            reverse: false,
+            unit: 'ms',
+          },
+          xBucketSize='',
+          yAxisFormat='ms',
+          yAxisLogBase=2,
+          yAxisMin='0',
+          yBucketSize=10,
+          pluginVersion='9.4.7',
+        ).addTarget(u.addTargetSchema(
+          expr='ceph_osd_apply_latency_ms{}',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          instant=false,
+        )),
+        u.heatMapPanel(
+          title='OSD Commit Latency Distribution',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 8, w: 12, x: 12, y: 42 },
+          colorMode='opacity',
+          legendShow=true,
+          cardColor='#65c5db',
+          optionsColor={
+            exponent: 0.5,
+            fill: '#65c5db',
+            mode: 'opacity',
+            reverse: false,
+            scale: 'exponential',
+            scheme: 'Oranges',
+            steps: 128,
+          },
+          optionsCalculate=true,
+          optionsCalculation={
+            yBuckets: {
+              mode: 'count',
+              scale: { log: 2, type: 'log' },
+            },
+          },
+          optionsExemplars={ color: 'rgba(255,0,255,0.7)' },
+          optionsFilterValues={ le: 1e-9 },
+          optionsLegend={ show: true },
+          optionsRowFrame={ layout: 'auto' },
+          optionsToolTip={
+            show: true,
+            yHistogram: false,
+          },
+          optionsYAxis={
+            axisPlacement: 'left',
+            min: '0',
+            reverse: false,
+            unit: 'ms',
+          },
+          xBucketSize='',
+          yAxisFormat='ms',
+          yAxisLogBase=2,
+          yAxisMin='0',
+          yBucketSize=10,
+          pluginVersion='9.4.7',
+        ).addTarget(u.addTargetSchema(
+          expr='ceph_osd_commit_latency_ms{}',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          instant=false,
+        )),
+        u.heatMapPanel(
+          title='OSD Read Op Latency Distribution',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 50 },
+          colorMode='opacity',
+          legendShow=true,
+          cardColor='#806eb7',
+          optionsColor={
+            exponent: 0.5,
+            fill: '#806eb7',
+            mode: 'opacity',
+            reverse: false,
+            scale: 'exponential',
+            scheme: 'Oranges',
+            steps: 128,
+          },
+          optionsCalculate=true,
+          optionsCalculation={
+            yBuckets: {
+              mode: 'count',
+              scale: { log: 2, type: 'log' },
+            },
+          },
+          optionsExemplars={ color: 'rgba(255,0,255,0.7)' },
+          optionsFilterValues={ le: 1e-9 },
+          optionsLegend={ show: true },
+          optionsRowFrame={ layout: 'auto' },
+          optionsToolTip={
+            show: true,
+            yHistogram: false,
+          },
+          optionsYAxis={
+            axisPlacement: 'left',
+            decimals: 2,
+            min: '0',
+            reverse: false,
+            unit: 'ms',
+          },
+          xBucketSize='',
+          yAxisFormat='ms',
+          yAxisLogBase=2,
+          yAxisMin='0',
+          yBucketSize=null,
+          pluginVersion='9.4.7',
+        ).addTarget(u.addTargetSchema(
+          expr='rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          instant=false,
+        )),
+
+        u.heatMapPanel(
+          title='OSD Write Op Latency Distribution',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 8, w: 12, x: 12, y: 50 },
+          colorMode='opacity',
+          legendShow=true,
+          cardColor='#f9934e',
+          optionsColor={
+            exponent: 0.5,
+            fill: '#f9934e',
+            mode: 'opacity',
+            reverse: false,
+            scale: 'exponential',
+            scheme: 'Oranges',
+            steps: 128,
+          },
+          optionsCalculate=true,
+          optionsCalculation={
+            yBuckets: {
+              mode: 'count',
+              scale: { log: 2, type: 'log' },
+            },
+          },
+          optionsExemplars={ color: 'rgba(255,0,255,0.7)' },
+          optionsFilterValues={ le: 1e-9 },
+          optionsLegend={ show: true },
+          optionsRowFrame={ layout: 'auto' },
+          optionsToolTip={
+            show: true,
+            yHistogram: false,
+          },
+          optionsYAxis={
+            axisPlacement: 'left',
+            decimals: 2,
+            min: '0',
+            reverse: false,
+            unit: 'ms',
+          },
+          xBucketSize='',
+          yAxisFormat='ms',
+          yAxisLogBase=2,
+          yAxisMin='0',
+          yBucketSize=null,
+          pluginVersion='9.4.7',
+        ).addTarget(u.addTargetSchema(
+          expr='rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0',
+          datasource='${DS_PROMETHEUS}',
+          interval='$interval',
+          legendFormat='',
+          instant=false,
+        )),
+        u.timeSeriesPanel(
+          title='Recovery Operations',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 7, w: 12, x: 0, y: 58 },
+          fillOpacity=10,
+          pointSize=5,
+          lineWidth=1,
+          showPoints='never',
+          unit='ms',
+          tooltip={ mode: 'multi', sort: 'none' },
+          interval='$interval',
+          stackingMode='none',
+          spanNulls=false,
+          decimals=null,
+          thresholdsMode='absolute',
+          displayMode='table',
+          placement='bottom',
+          showLegend=true,
+        )
+        .addThresholds([
+          { color: 'green' },
+          { color: 'red', value: 80 },
+        ])
+        .addTargets([
+          u.addTargetSchema(
+            expr='avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='Read',
+          ),
+          u.addTargetSchema(
+            expr='avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='Write',
+          ),
+        ]),
+
+        u.timeSeriesPanel(
+          title='AVG OSD Apply + Commit Latency',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 7, w: 12, x: 12, y: 58 },
+          fillOpacity=10,
+          pointSize=5,
+          lineWidth=1,
+          showPoints='never',
+          unit='ms',
+          tooltip={ mode: 'multi', sort: 'none' },
+          interval='$interval',
+          stackingMode='none',
+          spanNulls=false,
+          decimals=null,
+          thresholdsMode='absolute',
+          displayMode='table',
+          placement='bottom',
+          showLegend=true,
+        )
+        .addCalcs(['lastNotNull', 'max'])
+        .addThresholds([
+          { color: 'green' },
+          { color: 'red', value: 80 },
+        ])
+        .addTargets([
+          u.addTargetSchema(
+            expr='avg(ceph_osd_apply_latency_ms{})',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='apply',
+            interval='$interval',
+            metric='ceph_osd_perf_apply_latency_seconds',
+            step=4,
+          ),
+          u.addTargetSchema(
+            expr='avg(ceph_osd_commit_latency_ms{})',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='commit',
+            interval='$interval',
+            metric='ceph_osd_perf_commit_latency_seconds',
+            step=4,
+          ),
+        ]),
+      ])
+      + { gridPos: { x: 0, y: 44, w: 24, h: 1 } },
+      u.addRowSchema(collapse=true, showTitle=true, title='', collapsed=false) + { gridPos: { x: 0, y: 45, w: 24, h: 1 } },
+
+      u.addTableExtended(
+        datasource='${DS_PROMETHEUS}',
+        title='Ceph Versions',
+        gridPosition={ h: 6, w: 24, x: 0, y: 46 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+        },
+        custom={ align: 'left', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green' },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byName', options: 'Time' },
+          properties: [
+            { id: 'custom.hidden', value: true },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {},
+            indexByName: {},
+            renameByName: {
+              Time: '',
+              'Value #A': 'OSD Services',
+              'Value #B': 'Mon Services',
+              'Value #C': 'MDS Services',
+              'Value #D': 'RGW Services',
+              'Value #E': 'MGR Services',
+              ceph_version: 'Ceph Version',
+            },
+          },
+        },
+      ]).addTargets([
+        u.addTargetSchema(
+          expr='count by (ceph_version)(ceph_osd_metadata{})',
+          datasource='${DS_PROMETHEUS}',
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='OSD Services',
+          range=false,
+        ),
+        u.addTargetSchema(
+          expr='count by (ceph_version)(ceph_mon_metadata{})',
+          datasource='${DS_PROMETHEUS}',
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Mon Services',
+          range=false,
+        ),
+        u.addTargetSchema(
+          expr='count by (ceph_version)(ceph_mds_metadata{})',
+          datasource='${DS_PROMETHEUS}',
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          legendFormat='MDS Services',
+          range=false,
+        ),
+        u.addTargetSchema(
+          expr='count by (ceph_version)(ceph_rgw_metadata{})',
+          datasource='${DS_PROMETHEUS}',
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='RGW Services',
+          range=false,
+        ),
+        u.addTargetSchema(
+          expr='count by (ceph_version)(ceph_mgr_metadata{})',
+          datasource='${DS_PROMETHEUS}',
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='MGR Services',
+          range=false,
+        ),
+      ]),
+
+
+    ]  //end panels
+  ),
+}
diff --git a/monitoring/ceph-mixin/dashboards/timeseries_panel.libsonnet b/monitoring/ceph-mixin/dashboards/timeseries_panel.libsonnet
new file mode 100644
index 000000000000..7da147cf5680
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/timeseries_panel.libsonnet
@@ -0,0 +1,141 @@
+{
+  /**
+   * Creates a [Time series panel](https://grafana.com/docs/grafana/latest/panels-visualizations/visualizations/time-series/).
+   *
+   * @name timeseries_panel.new
+   *
+   * @param title (default `''`) Panel title.
+   * @param description (default null) Panel description.
+   */
+  new(
+    title='',
+    description=null,
+    pluginVersion='9.1.3',
+    gridPos={},
+    datasource='',
+    colorMode='palette-classic',
+    axisCenteredZero=false,
+    axisColorMode='text',
+    axisLabel='',
+    axisPlacement='auto',
+    barAlignment=0,
+    drawStyle='line',
+    fillOpacity=0,
+    gradientMode='none',
+    lineInterpolation='linear',
+    lineWidth=0,
+    pointSize=0,
+    scaleDistributionType='linear',
+    showPoints='',
+    spanNulls=false,
+    stackingGroup='A',
+    stackingMode='none',
+    thresholdsStyleMode='off',
+    decimals=null,
+    thresholdsMode='absolute',
+    unit='none',
+    tooltip={},
+    legend={},
+    displayMode='list',
+    placement='bottom',
+    showLegend=true,
+    min=null,
+    scaleDistributionLog=null,
+    sortBy=null,
+    sortDesc=null,
+  ):: {
+    title: title,
+    type: 'timeseries',
+    [if description != null then 'description']: description,
+    pluginVersion: pluginVersion,
+    gridPos: gridPos,
+    datasource: datasource,
+    fieldConfig: {
+      defaults: {
+        color: { mode: colorMode },
+        custom: {
+          axisCenteredZero: axisCenteredZero,
+          axisColorMode: axisColorMode,
+          axisLabel: axisLabel,
+          axisPlacement: axisPlacement,
+          barAlignment: barAlignment,
+          drawStyle: drawStyle,
+          fillOpacity: fillOpacity,
+          gradientMode: gradientMode,
+          hideFrom: {
+            legend: false,
+            tooltip: false,
+            viz: false,
+          },
+          lineInterpolation: lineInterpolation,
+          lineWidth: lineWidth,
+          pointSize: pointSize,
+          scaleDistribution: {
+            [if scaleDistributionLog != null then 'scaleDistributionLog']: scaleDistributionLog,
+            type: scaleDistributionType,
+          },
+          showPoints: showPoints,
+          spanNulls: spanNulls,
+          stacking: {
+            group: stackingGroup,
+            mode: stackingMode,
+          },
+          thresholdsStyle: {
+            mode: thresholdsStyleMode,
+          },
+        },
+        [if decimals != null then 'decimals']: decimals,
+        [if min != null then 'min']: min,
+        thresholds: {
+          mode: thresholdsMode,
+          steps: [],
+        },
+        unit: unit,
+      },
+      overrides: [],
+    },
+    options: {
+      legend: {
+        calcs: [],
+        displayMode: displayMode,
+        placement: placement,
+        showLegend: showLegend,
+        [if sortBy != null then 'sortBy']: sortBy,
+        [if sortDesc != null then 'sortDesc']: sortDesc,
+      },
+      tooltip: tooltip,
+    },
+    // Overrides
+    addOverride(
+      matcher=null,
+      properties=null,
+    ):: self {
+      fieldConfig+: {
+        overrides+: [
+          {
+            [if matcher != null then 'matcher']: matcher,
+            [if properties != null then 'properties']: properties,
+          },
+        ],
+      },
+    },
+    // thresholds
+    addThreshold(step):: self {
+      fieldConfig+: { defaults+: { thresholds+: { steps+: [step] } } },
+    },
+    addCalc(calc):: self {
+      options+: { legend+: { calcs+: [calc] } },
+    },
+    _nextTarget:: 0,
+    addTarget(target):: self {
+      // automatically ref id in added targets.
+      local nextTarget = super._nextTarget,
+      _nextTarget: nextTarget + 1,
+      targets+: [target { refId: std.char(std.codepoint('A') + nextTarget) }],
+    },
+    addTargets(targets):: std.foldl(function(p, t) p.addTarget(t), targets, self),
+    addThresholds(steps):: std.foldl(function(p, s) p.addThreshold(s), steps, self),
+    addCalcs(calcs):: std.foldl(function(p, t) p.addCalc(t), calcs, self),
+    addOverrides(overrides):: std.foldl(function(p, o) p.addOverride(o.matcher, o.properties), overrides, self),
+  },
+}
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index a7774c7ceed8..257b4a44f660 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -1,4 +1,5 @@
 local g = import 'grafonnet/grafana.libsonnet';
+local timeSeries = import 'timeseries_panel.libsonnet';
 
 {
   _config:: error 'must provide _config',
@@ -58,12 +59,53 @@ local g = import 'grafonnet/grafana.libsonnet';
                      legend_values=legend_values),
 
 
-  addTargetSchema(expr, legendFormat='', format='time_series', intervalFactor=1, instant=null)::
+  addTargetSchema(
+    expr,
+    legendFormat='',
+    format='time_series',
+    intervalFactor=1,
+    instant=null,
+    datasource=null,
+    step=null,
+    interval=null,
+    range=null,
+    hide=null,
+    metric=null,
+    aggregation=null,
+    alias=null,
+    decimals=null,
+    displayAliasType=null,
+    displayType=null,
+    displayValueWithAlias=null,
+    units=null,
+    valueHandler=null,
+    warn=null,
+    crit=null,
+    exemplar=null,
+  )::
     g.prometheus.target(expr=expr,
                         legendFormat=legendFormat,
                         format=format,
                         intervalFactor=intervalFactor,
-                        instant=instant),
+                        instant=instant,
+                        datasource=datasource) + {
+      [if step != null then 'step']: step,
+      [if interval != null then 'interval']: interval,
+      [if range != null then 'range']: range,
+      [if hide != null then 'hide']: hide,
+      [if metric != null then 'metric']: metric,
+      [if aggregation != null then 'aggregation']: aggregation,
+      [if alias != null then 'alias']: alias,
+      [if decimals != null then 'decimals']: decimals,
+      [if displayAliasType != null then 'displayAliasType']: displayAliasType,
+      [if displayType != null then 'displayType']: displayType,
+      [if displayValueWithAlias != null then 'displayValueWithAlias']: displayValueWithAlias,
+      [if units != null then 'units']: units,
+      [if valueHandler != null then 'valueHandler']: valueHandler,
+      [if warn != null then 'warn']: warn,
+      [if crit != null then 'crit']: crit,
+      [if exemplar != null then 'exemplar']: exemplar,
+    },
 
   addTemplateSchema(name,
                     datasource,
@@ -103,8 +145,15 @@ local g = import 'grafonnet/grafana.libsonnet';
                             name=name,
                             type=type),
 
-  addRowSchema(collapse, showTitle, title)::
-    g.row.new(collapse=collapse, showTitle=showTitle, title=title),
+  addRowSchema(
+    collapse,
+    showTitle,
+    title,
+    collapsed=null
+  )::
+    g.row.new(collapse=collapse, showTitle=showTitle, title=title) + {
+      [if collapsed != null then 'collapsed']: collapsed,
+    },
 
   addSingleStatSchema(colors,
                       datasource,
@@ -176,7 +225,6 @@ local g = import 'grafonnet/grafana.libsonnet';
       unit: unit,
       valueMaps: valueMaps,
     },
-
   matchers()::
     local jobMatcher = 'job=~"$job"';
     local clusterMatcher = '%s=~"$cluster"' % $._config.clusterLabel;
@@ -330,4 +378,355 @@ local g = import 'grafonnet/grafana.libsonnet';
                         'pie',
                         title,
                         'current'),
+
+  addStatPanel(
+    title,
+    description='',
+    transparent=false,
+    datasource=null,
+    color={},
+    unit='none',
+    overrides=[],
+    gridPosition={},
+    colorMode='none',
+    graphMode='none',
+    justifyMode='auto',
+    orientation='horizontal',
+    textMode='auto',
+    reducerFunction='lastNotNull',
+    pluginVersion='9.1.3',
+    decimals=0,
+    interval=null,
+    maxDataPoints=null,
+    thresholdsMode='absolute',
+    rootColorMode=null,
+    rootColors=null,
+    cornerRadius=null,
+    flipCard=null,
+    flipTime=null,
+    fontFormat=null,
+    displayName=null,
+    isAutoScrollOnOverflow=null,
+    isGrayOnNoData=null,
+    isHideAlertsOnDisable=null,
+    isIgnoreOKColors=null,
+  )::
+    g.statPanel.new(
+      title=title,
+      description=description,
+      transparent=transparent,
+      datasource=datasource,
+      unit=unit,
+      colorMode=colorMode,
+      graphMode=graphMode,
+      justifyMode=justifyMode,
+      orientation=orientation,
+      textMode=textMode,
+      reducerFunction=reducerFunction,
+      pluginVersion=pluginVersion,
+      decimals=decimals,
+      thresholdsMode=thresholdsMode,
+    ) + {
+      [if interval != null then 'interval']: interval,
+      [if maxDataPoints != null then 'maxDataPoints']: maxDataPoints,
+      [if gridPosition != {} then 'gridPos']: gridPosition,
+      [if rootColorMode != null then 'colorMode']: rootColorMode,
+      [if rootColors != {} then 'colors']: rootColors,
+      [if cornerRadius != null then 'cornerRadius']: cornerRadius,
+      [if flipCard != null then 'flipCard']: flipCard,
+      [if flipTime != null then 'flipTime']: flipTime,
+      [if fontFormat != null then 'fontFormat']: fontFormat,
+      [if displayName != null then 'displayName']: displayName,
+      [if isAutoScrollOnOverflow != null then 'isAutoScrollOnOverflow']: isAutoScrollOnOverflow,
+      [if isGrayOnNoData != null then 'isGrayOnNoData']: isGrayOnNoData,
+      [if isHideAlertsOnDisable != null then 'isHideAlertsOnDisable']: isHideAlertsOnDisable,
+      [if isIgnoreOKColors != null then 'isIgnoreOKColors']: isIgnoreOKColors,
+    },
+
+  addAlertListPanel(
+    title,
+    datasource=null,
+    gridPosition={},
+    alertInstanceLabelFilter=null,
+    alertName=null,
+    dashboardAlerts=null,
+    groupBy=null,
+    groupMode=null,
+    maxItems=null,
+    sortOrder=null,
+    stateFilter=null,
+    viewMode='list'
+  )::
+    g.alertlist.new(
+      title=title,
+      datasource=datasource,
+    ) + {
+      gridPos: gridPosition,
+      options: {
+        [if alertInstanceLabelFilter != null then 'alertInstanceLabelFilter']: alertInstanceLabelFilter,
+        [if alertName != null then 'alertName']: alertName,
+        [if dashboardAlerts != null then 'dashboardAlerts']: dashboardAlerts,
+        [if groupBy != null then 'groupBy']: groupBy,
+        [if groupMode != null then 'groupMode']: groupMode,
+        [if maxItems != null then 'maxItems']: maxItems,
+        [if sortOrder != null then 'sortOrder']: sortOrder,
+        [if stateFilter != null then 'stateFilter']: stateFilter,
+        viewMode: viewMode,
+      },
+    },
+
+  addCustomTemplate(name='',
+                    query='',
+                    current='',
+                    valuelabels={},
+                    refresh=0,
+                    label='Interval',
+                    auto_count=10,
+                    auto_min='2m',
+                    options=[],
+                    auto=null)::
+    g.template.interval(name=name,
+                        query=query,
+                        current=current,
+                        label=label,
+                        auto_count=auto_count,
+                        auto_min=auto_min,) + {
+      options: options,
+      refresh: refresh,
+      valuelabels: valuelabels,
+      [if auto != null then 'auto']: auto,
+    },
+
+  addGaugePanel(title='',
+                description='',
+                transparent=false,
+                datasource='${DS_PROMETHEUS}',
+                gridPosition={},
+                pluginVersion='9.1.3',
+                unit='percentunit',
+                instant=false,
+                reducerFunction='lastNotNull',
+                steps=[],
+                max=1,
+                min=0,
+                maxDataPoints=100,
+                interval='1m')::
+    g.gaugePanel.new(title=title,
+                     description=description,
+                     transparent=transparent,
+                     datasource=datasource,
+                     pluginVersion=pluginVersion,
+                     unit=unit,
+                     reducerFunction=reducerFunction,
+                     max=max,
+                     min=min) + {
+      gridPos: gridPosition,
+      maxDataPoints: maxDataPoints,
+      interval: interval,
+    },
+  addTableExtended(
+    title='',
+    datasource=null,
+    description=null,
+    sort=null,
+    styles='',
+    transform=null,
+    pluginVersion='9.1.3',
+    options=null,
+    gridPosition={},
+    custom=null,
+    decimals=null,
+    thresholds=null,
+    unit=null,
+    overrides=[],
+    color=null
+  )::
+    g.tablePanel.new(datasource=datasource,
+                     description=description,
+                     sort=sort,
+                     styles=styles,
+                     title=title,
+                     transform=transform) + {
+      pluginVersion: pluginVersion,
+      gridPos: gridPosition,
+      [if options != null then 'options']: options,
+      fieldConfig+: {
+        defaults+: {
+          [if custom != null then 'custom']: custom,
+          [if decimals != null then 'decimals']: decimals,
+          [if thresholds != null then 'thresholds']: thresholds,
+          [if unit != null then 'unit']: unit,
+          [if color != null then 'color']: color,
+
+        },
+        overrides: overrides,
+      },
+    },
+  timeSeriesPanel(
+    title='',
+    datasource=null,
+    gridPosition={},
+    colorMode='palette-classic',
+    axisCenteredZero=false,
+    axisColorMode='text',
+    axisLabel='',
+    axisPlacement='auto',
+    barAlignment=0,
+    drawStyle='line',
+    fillOpacity=0,
+    gradientMode='none',
+    lineInterpolation='linear',
+    lineWidth=0,
+    pointSize=0,
+    scaleDistributionType='linear',
+    showPoints='',
+    spanNulls=false,
+    stackingGroup='A',
+    stackingMode='none',
+    thresholdsStyleMode='off',
+    decimals=null,
+    thresholdsMode='absolute',
+    unit='none',
+    tooltip={ mode: 'multi', sort: 'none' },
+    pluginVersion='9.1.3',
+    displayMode='list',
+    placement='bottom',
+    showLegend=true,
+    interval=null,
+    min=null,
+    scaleDistributionLog=null,
+    sortBy=null,
+    sortDesc=null,
+  )::
+    timeSeries.new(
+      title=title,
+      gridPos=gridPosition,
+      datasource=datasource,
+      colorMode=colorMode,
+      axisCenteredZero=axisCenteredZero,
+      axisColorMode=axisColorMode,
+      axisLabel=axisLabel,
+      axisPlacement=axisPlacement,
+      barAlignment=barAlignment,
+      drawStyle=drawStyle,
+      fillOpacity=fillOpacity,
+      gradientMode=gradientMode,
+      lineInterpolation=lineInterpolation,
+      lineWidth=lineWidth,
+      pointSize=pointSize,
+      scaleDistributionType=scaleDistributionType,
+      showPoints=showPoints,
+      spanNulls=spanNulls,
+      stackingGroup=stackingGroup,
+      stackingMode=stackingMode,
+      thresholdsStyleMode=thresholdsStyleMode,
+      decimals=decimals,
+      thresholdsMode=thresholdsMode,
+      unit=unit,
+      displayMode=displayMode,
+      placement=placement,
+      showLegend=showLegend,
+      tooltip=tooltip,
+      min=min,
+      scaleDistributionLog=scaleDistributionLog,
+      sortBy=sortBy,
+      sortDesc=sortDesc,
+    ) + {
+      pluginVersion: pluginVersion,
+      [if interval != null then 'interval']: interval,
+    },
+
+  heatMapPanel(
+    title='',
+    datasource=null,
+    gridPosition={},
+    colorMode='spectrum',
+    cardColor='#b4ff00',
+    colorScale='sqrt',
+    colorScheme='interpolateOranges',
+    colorExponent=0.5,
+    pluginVersion='9.1.3',
+    dataFormat='timeseries',
+    hideFrom={ legend: false, tooltip: false, viz: false },
+    scaleDistributionType='linear',
+    legendShow=false,
+    optionsCalculate=false,
+    optionsCalculation={
+      yBuckets: {
+        mode: 'count',
+        scale: { log: 2, type: 'log' },
+        value: '1',
+      },
+    },
+    optionsCellGap=2,
+    optionsCellValues={},
+    optionsColor={},
+    optionsExemplars={},
+    optionsFilterValues={},
+    optionsLegend={},
+    optionsRowFrame={},
+    optionsShowValue='never',
+    optionsToolTip={},
+    optionsYAxis={},
+    xBucketSize=null,
+    yAxisDecimals=null,
+    yAxisFormat='short',
+    yAxisLogBase=1,
+    yAxisMin=null,
+    yAxisMax=null,
+    yAxisShow=true,
+    yAxisSplitFactor=1,
+    yBucketSize=null,
+    yBucketBound='auto'
+  )
+  :: g.heatmapPanel.new(
+    title=title,
+    datasource=datasource,
+    color_mode=colorMode,
+    color_cardColor=cardColor,
+    color_colorScale=colorScale,
+    color_colorScheme=colorScheme,
+    color_exponent=colorExponent,
+    legend_show=legendShow,
+    xBucketSize=xBucketSize,
+    yAxis_decimals=yAxisDecimals,
+    yAxis_format=yAxisFormat,
+    yAxis_logBase=yAxisLogBase,
+    yAxis_min=yAxisMin,
+    yAxis_max=yAxisMax,
+    yAxis_show=yAxisShow,
+    yAxis_splitFactor=yAxisSplitFactor,
+    yBucketSize=yBucketSize,
+    yBucketBound=yBucketBound
+  ) + {
+    gridPos: gridPosition,
+    pluginVersion: pluginVersion,
+    color+: {
+      colorScheme: colorScheme,
+    },
+    fieldConfig: {
+      defaults: {
+        custom: {
+          hideFrom: hideFrom,
+          scaleDistribution: {
+            type: scaleDistributionType,
+          },
+        },
+      },
+    },
+    options: {
+      calculate: optionsCalculate,
+      calculation: optionsCalculation,
+      cellGap: optionsCellGap,
+      cellValues: optionsCellValues,
+      color: optionsColor,
+      exemplars: optionsExemplars,
+      filterValues: optionsFilterValues,
+      legend: optionsLegend,
+      rowsFrame: optionsRowFrame,
+      showValue: optionsShowValue,
+      tooltip: optionsToolTip,
+      yAxis: optionsYAxis,
+    },
+  },
 }
diff --git a/monitoring/ceph-mixin/dashboards_out/ceph-cluster-advanced.json b/monitoring/ceph-mixin/dashboards_out/ceph-cluster-advanced.json
new file mode 100644
index 000000000000..216e02ed1a30
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards_out/ceph-cluster-advanced.json
@@ -0,0 +1,3792 @@
+{
+   "__inputs": [ ],
+   "__requires": [
+      {
+         "id": "grafana",
+         "name": "Grafana",
+         "type": "grafana",
+         "version": "5.3.2"
+      },
+      {
+         "id": "graph",
+         "name": "Graph",
+         "type": "panel",
+         "version": "5.0.0"
+      },
+      {
+         "id": "heatmap",
+         "name": "Heatmap",
+         "type": "panel",
+         "version": "5.0.0"
+      },
+      {
+         "id": "singlestat",
+         "name": "Singlestat",
+         "type": "panel",
+         "version": "5.0.0"
+      }
+   ],
+   "annotations": {
+      "list": [
+         {
+            "builtIn": 1,
+            "datasource": "-- Grafana --",
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "showIn": 0,
+            "tags": [ ],
+            "type": "dashboard"
+         }
+      ]
+   },
+   "description": "Ceph cluster overview",
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 2,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "CLUSTER STATE",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "0": {
+                           "text": "HEALTHY"
+                        },
+                        "1": {
+                           "text": "WARNING"
+                        },
+                        "2": {
+                           "text": "ERROR"
+                        }
+                     },
+                     "type": "value"
+                  },
+                  {
+                     "id": 1,
+                     "options": {
+                        "match": null,
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "#9ac48a"
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 1
+                     },
+                     {
+                        "color": "rgba(245, 54, 54, 0.9)",
+                        "value": 2
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 0,
+            "y": 1
+         },
+         "id": 3,
+         "interval": "1m",
+         "links": [ ],
+         "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "ceph_health_status{}",
+               "format": "time_series",
+               "instant": true,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Ceph health status",
+         "transparent": true,
+         "type": "stat"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "match": null,
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "max": 1,
+               "min": 0,
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "rgba(245, 54, 54, 0.9)"
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 0.10000000000000001
+                     },
+                     {
+                        "color": "rgba(50, 172, 45, 0.97)",
+                        "value": 0.29999999999999999
+                     }
+                  ]
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 6,
+            "w": 3,
+            "x": 3,
+            "y": 1
+         },
+         "id": 4,
+         "interval": "1m",
+         "links": [ ],
+         "maxDataPoints": 100,
+         "options": {
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "showThresholdLabels": false,
+            "showThresholdMarkers": true
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_cluster_total_bytes{}-ceph_cluster_total_used_bytes{})/ceph_cluster_total_bytes{}",
+               "format": "time_series",
+               "instant": true,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Available Capacity",
+         "transparent": false,
+         "type": "gauge"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 2,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "match": null,
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "rgba(50, 172, 45, 0.97)"
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 0.025000000000000001
+                     },
+                     {
+                        "color": "rgba(245, 54, 54, 0.9)",
+                        "value": 1
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 6,
+            "y": 1
+         },
+         "id": 5,
+         "interval": "1m",
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "ceph_cluster_total_bytes{}",
+               "format": "time_series",
+               "instant": true,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Cluster Capacity",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 1,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "match": null,
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     }
+                  ]
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 9,
+            "y": 1
+         },
+         "id": 6,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+               "format": "time_series",
+               "instant": true,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "title": "Write Throughput",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 1,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "match": null,
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "#d44a3a"
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 0
+                     },
+                     {
+                        "color": "#9ac48a",
+                        "value": 0
+                     }
+                  ]
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 12,
+            "y": 1
+         },
+         "id": 7,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+               "format": "time_series",
+               "instant": true,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "title": "Read Throughput",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colorMode": "Panel",
+         "colors": {
+            "crit": "rgb(255, 0, 0)",
+            "disable": "rgba(128, 128, 128, 0.9)",
+            "ok": "rgba(50, 128, 45, 0.9)",
+            "warn": "rgba(237, 129, 40, 0.9)"
+         },
+         "cornerRadius": 0,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "flipCard": false,
+         "flipTime": 5,
+         "fontFormat": "Regular",
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 15,
+            "y": 1
+         },
+         "id": 8,
+         "isAutoScrollOnOverflow": false,
+         "isGrayOnNoData": false,
+         "isHideAlertsOnDisable": false,
+         "isIgnoreOKColors": false,
+         "links": [ ],
+         "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "aggregation": "Last",
+               "alias": "All",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ceph_osd_metadata)",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "All",
+               "refId": "A",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "In",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ceph_osd_in)",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "In",
+               "refId": "B",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Out",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Warning / Critical",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "sum(ceph_osd_in == bool 0)",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Out",
+               "refId": "C",
+               "units": "none",
+               "valueHandler": "Number Threshold",
+               "warn": 1
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Up",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "sum(ceph_osd_up)",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Up",
+               "refId": "D",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Down",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Warning / Critical",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "sum(ceph_osd_up == bool 0)",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Down",
+               "refId": "E",
+               "units": "none",
+               "valueHandler": "Number Threshold",
+               "warn": 1
+            }
+         ],
+         "title": "OSDs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colorMode": "Panel",
+         "colors": {
+            "crit": "rgba(245, 54, 54, 0.9)",
+            "disable": "rgba(128, 128, 128, 0.9)",
+            "ok": "rgba(50, 128, 45, 0.9)",
+            "warn": "rgba(237, 129, 40, 0.9)"
+         },
+         "cornerRadius": 1,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "flipCard": false,
+         "flipTime": 5,
+         "fontFormat": "Regular",
+         "gridPos": {
+            "h": 6,
+            "w": 3,
+            "x": 21,
+            "y": 1
+         },
+         "id": 9,
+         "isAutoScrollOnOverflow": false,
+         "isGrayOnNoData": false,
+         "isHideAlertsOnDisable": false,
+         "isIgnoreOKColors": false,
+         "links": [ ],
+         "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "aggregation": "Last",
+               "alias": "Active",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ceph_mgr_status == 1) or vector(0)",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Active",
+               "refId": "A",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Standby",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ceph_mgr_status == 0) or vector(0)",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Standby",
+               "refId": "B",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            }
+         ],
+         "title": "MGRs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colorMode": "Panel",
+         "colors": {
+            "crit": "rgba(245, 54, 54, 0.9)",
+            "disable": "rgba(128, 128, 128, 0.9)",
+            "ok": "rgba(50, 128, 45, 0.9)",
+            "warn": "rgba(237, 129, 40, 0.9)"
+         },
+         "cornerRadius": 1,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 1
+                     }
+                  ]
+               },
+               "unit": "none"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Critical"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "red",
+                           "mode": "fixed"
+                        }
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Warning"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "#987d24",
+                           "mode": "fixed"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "flipCard": false,
+         "flipTime": 5,
+         "fontFormat": "Regular",
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 0,
+            "y": 4
+         },
+         "id": 10,
+         "isAutoScrollOnOverflow": false,
+         "isGrayOnNoData": false,
+         "isHideAlertsOnDisable": false,
+         "isIgnoreOKColors": false,
+         "links": [ ],
+         "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "aggregation": "Last",
+               "alias": "Active",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\", severity=\"critical\"}) OR vector(0)",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Critical",
+               "refId": "A",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Standby",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ALERTS{alertstate=\"firing\",alertname=~\"^Ceph.+\", severity=\"warning\"}) OR vector(0)",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Warning",
+               "refId": "B",
+               "units": "none",
+               "valueHandler": "Number Threshold"
+            }
+         ],
+         "title": "Firing Alerts",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "rgba(50, 172, 45, 0.97)",
+                        "value": null
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 0.025000000000000001
+                     },
+                     {
+                        "color": "rgba(245, 54, 54, 0.9)",
+                        "value": 0.10000000000000001
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 6,
+            "y": 4
+         },
+         "id": 11,
+         "links": [ ],
+         "maxDataPoints": 100,
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "ceph_cluster_total_used_bytes{}",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "title": "Used Capacity",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "ops"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 9,
+            "y": 4
+         },
+         "id": 12,
+         "links": [ ],
+         "maxDataPoints": 100,
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_w{}[1m]))",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "title": "Write IOPS",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [
+                  {
+                     "id": 0,
+                     "options": {
+                        "result": {
+                           "text": "N/A"
+                        }
+                     },
+                     "type": "special"
+                  }
+               ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "#d44a3a",
+                        "value": null
+                     },
+                     {
+                        "color": "rgba(237, 129, 40, 0.89)",
+                        "value": 0
+                     },
+                     {
+                        "color": "#9ac48a",
+                        "value": 0
+                     }
+                  ]
+               },
+               "unit": "ops"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 12,
+            "y": 4
+         },
+         "id": 13,
+         "links": [ ],
+         "maxDataPoints": 100,
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_r{}[1m]))",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "",
+               "refId": "A"
+            }
+         ],
+         "title": "Read IOPS",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colorMode": "Panel",
+         "colors": {
+            "crit": "rgba(245, 54, 54, 0.9)",
+            "disable": "rgba(128, 128, 128, 0.9)",
+            "ok": "rgba(50, 128, 45, 0.9)",
+            "warn": "rgba(237, 129, 40, 0.9)"
+         },
+         "cornerRadius": 1,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "displayName": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "flipCard": false,
+         "flipTime": 5,
+         "fontFormat": "Regular",
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 15,
+            "y": 4
+         },
+         "id": 14,
+         "isAutoScrollOnOverflow": false,
+         "isGrayOnNoData": false,
+         "isHideAlertsOnDisable": false,
+         "isIgnoreOKColors": false,
+         "links": [ ],
+         "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "aggregation": "Last",
+               "alias": "In Quorum",
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "sum(ceph_mon_quorum_status)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "In Quorum",
+               "refId": "A",
+               "units": "none",
+               "valueHandler": "Text Only"
+            },
+            {
+               "aggregation": "Last",
+               "alias": "Total",
+               "crit": 1,
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Always",
+               "displayType": "Regular",
+               "displayValueWithAlias": "When Alias Displayed",
+               "expr": "count(ceph_mon_quorum_status)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Total",
+               "refId": "B",
+               "units": "none",
+               "valueHandler": "Text Only",
+               "warn": 2
+            },
+            {
+               "aggregation": "Last",
+               "alias": "MONs out of Quorum",
+               "crit": 1.6000000000000001,
+               "datasource": "${DS_PROMETHEUS}",
+               "decimals": 2,
+               "displayAliasType": "Warning / Critical",
+               "displayType": "Annotation",
+               "displayValueWithAlias": "Never",
+               "expr": "count(ceph_mon_quorum_status) - sum(ceph_mon_quorum_status)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "MONs out of Quorum",
+               "range": true,
+               "refId": "C",
+               "units": "none",
+               "valueHandler": "Number Threshold",
+               "warn": 1.1000000000000001
+            }
+         ],
+         "title": "Monitors",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 7
+         },
+         "id": 15,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "CLUSTER STATS",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "grafana"
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 0,
+            "y": 8
+         },
+         "id": 16,
+         "limit": 10,
+         "onlyAlertsOnDashboard": true,
+         "options": {
+            "alertInstanceLabelFilter": "{alertname=~\"^Ceph.+\"}",
+            "alertName": "",
+            "dashboardAlerts": false,
+            "groupBy": [ ],
+            "groupMode": "default",
+            "maxItems": 20,
+            "sortOrder": 1,
+            "stateFilter": {
+               "error": true,
+               "firing": true,
+               "noData": false,
+               "normal": false,
+               "pending": true
+            },
+            "viewMode": "list"
+         },
+         "show": "current",
+         "sortOrder": 1,
+         "stateFilter": [ ],
+         "title": "Alerts",
+         "type": "alertlist"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 40,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 0,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "decimals": 2,
+               "thresholds": {
+                  "mode": "percentage",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "#c0921f",
+                        "value": 75
+                     },
+                     {
+                        "color": "#E02F44",
+                        "value": 85
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Total Capacity"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "red",
+                           "mode": "fixed"
+                        }
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Used"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "green",
+                           "mode": "fixed"
+                        }
+                     },
+                     {
+                        "id": "custom.thresholdsStyle",
+                        "value": {
+                           "mode": "dashed"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 8,
+            "y": 8
+         },
+         "id": 17,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [
+                  "last"
+               ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true,
+               "sortBy": "Last",
+               "sortDesc": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "desc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "ceph_cluster_total_bytes{}",
+               "format": "time_series",
+               "instant": false,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Total Capacity",
+               "range": true,
+               "refId": "A",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "ceph_cluster_total_used_bytes{}",
+               "format": "time_series",
+               "instant": false,
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Used",
+               "range": true,
+               "refId": "B",
+               "step": 300
+            }
+         ],
+         "title": "Capacity",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 85
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 8
+         },
+         "id": 18,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [
+                  "mean",
+                  "lastNotNull",
+                  "max",
+                  "min"
+               ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "desc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_w_in_bytes{}[5m]))",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Write",
+               "range": true,
+               "refId": "A",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_r_out_bytes{}[5m]))",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Read",
+               "range": true,
+               "refId": "B",
+               "step": 300
+            }
+         ],
+         "title": "Cluster Throughput",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 0,
+            "y": 16
+         },
+         "id": 19,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [
+                  "mean",
+                  "lastNotNull",
+                  "max",
+                  "min"
+               ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "desc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_w{}[1m]))",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Write",
+               "range": true,
+               "refId": "A",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_op_r{}[1m]))",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Read",
+               "range": true,
+               "refId": "B",
+               "step": 300
+            }
+         ],
+         "title": "IOPS",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 8,
+            "y": 16
+         },
+         "id": 20,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "desc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_pool_bytes_used{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "{{name}}",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Pool Used Bytes",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "rbd Stored"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "transparent",
+                           "mode": "fixed"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 16
+         },
+         "id": 21,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "desc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_pool_stored_raw{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+               "format": "time_series",
+               "hide": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "{{name}}",
+               "range": true,
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Pool Used RAW Bytes",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "none"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 0,
+            "y": 24
+         },
+         "id": 22,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "bottom",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "none"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_pool_quota_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "{{name}}",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Pool Objects Quota",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "none"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 8,
+            "y": 24
+         },
+         "id": 23,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "bottom",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "none"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_pool_quota_bytes{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "{{name}}",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Pool Quota Bytes",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 1,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": false,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 16,
+            "y": 24
+         },
+         "id": 24,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "none"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "(ceph_pool_objects{}) *on (pool_id) group_left(name)(ceph_pool_metadata{})",
+               "format": "time_series",
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "{{name}}",
+               "refId": "A"
+            }
+         ],
+         "title": "Objects Per Pool",
+         "type": "timeseries"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 31
+         },
+         "id": 25,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "OBJECTS",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 2,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/^Total.*$/"
+                  },
+                  "properties": [
+                     {
+                        "id": "custom.stacking",
+                        "value": {
+                           "group": false,
+                           "mode": "normal"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 12,
+            "w": 6,
+            "x": 0,
+            "y": 32
+         },
+         "id": 26,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "bottom",
+               "showLegend": false
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "asc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pool_objects)",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Total",
+               "range": true,
+               "refId": "A",
+               "step": 200
+            }
+         ],
+         "title": "OSD Type Count",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 2,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/^Total.*$/"
+                  },
+                  "properties": [
+                     {
+                        "id": "custom.stacking",
+                        "value": {
+                           "group": false,
+                           "mode": "normal"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 12,
+            "w": 8,
+            "x": 6,
+            "y": 32
+         },
+         "id": 27,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "asc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_active{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Active",
+               "range": true,
+               "refId": "A"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_clean{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Clean",
+               "range": true,
+               "refId": "B"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_peering{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Peering",
+               "range": true,
+               "refId": "C"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_degraded{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Degraded",
+               "range": true,
+               "refId": "D",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_stale{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Stale",
+               "range": true,
+               "refId": "E",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_unclean_pgs{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Unclean",
+               "range": true,
+               "refId": "F",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_undersized{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Undersized",
+               "range": true,
+               "refId": "G",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_incomplete{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Incomplete",
+               "range": true,
+               "refId": "H"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_forced_backfill{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Forced Backfill",
+               "range": true,
+               "refId": "I"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_forced_recovery{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Forced Recovery",
+               "range": true,
+               "refId": "J"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_creating{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Creating",
+               "range": true,
+               "refId": "K"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_wait_backfill{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Wait Backfill",
+               "range": true,
+               "refId": "L"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_deep{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Deep",
+               "range": true,
+               "refId": "M"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_scrubbing{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Scrubbing",
+               "range": true,
+               "refId": "N"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_recovering{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Recovering",
+               "range": true,
+               "refId": "O"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_repair{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Repair",
+               "range": true,
+               "refId": "P"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_down{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Down",
+               "range": true,
+               "refId": "Q"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_peered{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Peered",
+               "range": true,
+               "refId": "R"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_backfill{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Backfill",
+               "range": true,
+               "refId": "S"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_remapped{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Remapped",
+               "range": true,
+               "refId": "T"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_backfill_toofull{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Backfill Toofull",
+               "range": true,
+               "refId": "U"
+            }
+         ],
+         "title": "PGs State",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 2,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/^Total.*$/"
+                  },
+                  "properties": [
+                     {
+                        "id": "custom.stacking",
+                        "value": {
+                           "group": false,
+                           "mode": "normal"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 6,
+            "w": 10,
+            "x": 14,
+            "y": 32
+         },
+         "id": 28,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [
+                  "mean",
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "asc"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_degraded{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Degraded",
+               "range": true,
+               "refId": "A",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_stale{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Stale",
+               "range": true,
+               "refId": "B",
+               "step": 300
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_pg_undersized{})",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "Undersized",
+               "range": true,
+               "refId": "C",
+               "step": 300
+            }
+         ],
+         "title": "Stuck PGs",
+         "type": "timeseries"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "drawStyle": "line",
+                  "fillOpacity": 10,
+                  "gradientMode": "none",
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  },
+                  "lineInterpolation": "linear",
+                  "lineWidth": 2,
+                  "pointSize": 5,
+                  "scaleDistribution": {
+                     "type": "linear"
+                  },
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "group": "A",
+                     "mode": "none"
+                  },
+                  "thresholdsStyle": {
+                     "mode": "off"
+                  }
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "short"
+            },
+            "overrides": [ ]
+         },
+         "gridPos": {
+            "h": 6,
+            "w": 10,
+            "x": 14,
+            "y": 38
+         },
+         "id": 29,
+         "interval": "$interval",
+         "options": {
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "list",
+               "placement": "bottom",
+               "showLegend": false
+            },
+            "tooltip": {
+               "mode": "multi",
+               "sort": "none"
+            }
+         },
+         "pluginVersion": "9.1.3",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(irate(ceph_osd_recovery_ops{}[$interval]))",
+               "format": "time_series",
+               "interval": "$interval",
+               "intervalFactor": 1,
+               "legendFormat": "OPS",
+               "refId": "A",
+               "step": 300
+            }
+         ],
+         "title": "Recovery Operations",
+         "type": "timeseries"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 44
+         },
+         "id": 30,
+         "panels": [
+            {
+               "cards": {
+                  "cardPadding": null,
+                  "cardRound": null
+               },
+               "color": {
+                  "cardColor": "#b4ff00",
+                  "colorScale": "sqrt",
+                  "colorScheme": "interpolateOranges",
+                  "exponent": 0.5,
+                  "mode": "opacity"
+               },
+               "dataFormat": "timeseries",
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "scaleDistribution": {
+                           "type": "linear"
+                        }
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 42
+               },
+               "heatmap": { },
+               "hideZeroBuckets": false,
+               "highlightCards": true,
+               "id": 31,
+               "legend": {
+                  "show": true
+               },
+               "options": {
+                  "calculate": true,
+                  "calculation": {
+                     "yBuckets": {
+                        "mode": "count",
+                        "scale": {
+                           "log": 2,
+                           "type": "log"
+                        },
+                        "value": "1"
+                     }
+                  },
+                  "cellGap": 2,
+                  "cellValues": { },
+                  "color": {
+                     "exponent": 0.5,
+                     "fill": "#b4ff00",
+                     "mode": "opacity",
+                     "reverse": false,
+                     "scale": "exponential",
+                     "scheme": "Oranges",
+                     "steps": 128
+                  },
+                  "exemplars": {
+                     "color": "rgba(255,0,255,0.7)"
+                  },
+                  "filterValues": {
+                     "le": 1.0000000000000001e-09
+                  },
+                  "legend": {
+                     "show": true
+                  },
+                  "rowsFrame": {
+                     "layout": "auto"
+                  },
+                  "showValue": "never",
+                  "tooltip": {
+                     "show": true,
+                     "yHistogram": false
+                  },
+                  "yAxis": {
+                     "axisPlacement": "left",
+                     "min": "0",
+                     "reverse": false,
+                     "unit": "ms"
+                  }
+               },
+               "pluginVersion": "9.4.7",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "ceph_osd_apply_latency_ms{}",
+                     "format": "time_series",
+                     "instant": false,
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "",
+                     "refId": "A"
+                  }
+               ],
+               "title": "OSD Apply Latency Distribution",
+               "tooltip": {
+                  "show": true,
+                  "showHistogram": false
+               },
+               "type": "heatmap",
+               "xAxis": {
+                  "show": true
+               },
+               "xBucketNumber": null,
+               "xBucketSize": "",
+               "yAxis": {
+                  "decimals": null,
+                  "format": "ms",
+                  "logBase": 2,
+                  "max": null,
+                  "min": "0",
+                  "show": true,
+                  "splitFactor": 1
+               },
+               "yBucketBound": "auto",
+               "yBucketNumber": null,
+               "yBucketSize": 10
+            },
+            {
+               "cards": {
+                  "cardPadding": null,
+                  "cardRound": null
+               },
+               "color": {
+                  "cardColor": "#65c5db",
+                  "colorScale": "sqrt",
+                  "colorScheme": "interpolateOranges",
+                  "exponent": 0.5,
+                  "mode": "opacity"
+               },
+               "dataFormat": "timeseries",
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "scaleDistribution": {
+                           "type": "linear"
+                        }
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 42
+               },
+               "heatmap": { },
+               "hideZeroBuckets": false,
+               "highlightCards": true,
+               "id": 32,
+               "legend": {
+                  "show": true
+               },
+               "options": {
+                  "calculate": true,
+                  "calculation": {
+                     "yBuckets": {
+                        "mode": "count",
+                        "scale": {
+                           "log": 2,
+                           "type": "log"
+                        }
+                     }
+                  },
+                  "cellGap": 2,
+                  "cellValues": { },
+                  "color": {
+                     "exponent": 0.5,
+                     "fill": "#65c5db",
+                     "mode": "opacity",
+                     "reverse": false,
+                     "scale": "exponential",
+                     "scheme": "Oranges",
+                     "steps": 128
+                  },
+                  "exemplars": {
+                     "color": "rgba(255,0,255,0.7)"
+                  },
+                  "filterValues": {
+                     "le": 1.0000000000000001e-09
+                  },
+                  "legend": {
+                     "show": true
+                  },
+                  "rowsFrame": {
+                     "layout": "auto"
+                  },
+                  "showValue": "never",
+                  "tooltip": {
+                     "show": true,
+                     "yHistogram": false
+                  },
+                  "yAxis": {
+                     "axisPlacement": "left",
+                     "min": "0",
+                     "reverse": false,
+                     "unit": "ms"
+                  }
+               },
+               "pluginVersion": "9.4.7",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "ceph_osd_commit_latency_ms{}",
+                     "format": "time_series",
+                     "instant": false,
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "",
+                     "refId": "A"
+                  }
+               ],
+               "title": "OSD Commit Latency Distribution",
+               "tooltip": {
+                  "show": true,
+                  "showHistogram": false
+               },
+               "type": "heatmap",
+               "xAxis": {
+                  "show": true
+               },
+               "xBucketNumber": null,
+               "xBucketSize": "",
+               "yAxis": {
+                  "decimals": null,
+                  "format": "ms",
+                  "logBase": 2,
+                  "max": null,
+                  "min": "0",
+                  "show": true,
+                  "splitFactor": 1
+               },
+               "yBucketBound": "auto",
+               "yBucketNumber": null,
+               "yBucketSize": 10
+            },
+            {
+               "cards": {
+                  "cardPadding": null,
+                  "cardRound": null
+               },
+               "color": {
+                  "cardColor": "#806eb7",
+                  "colorScale": "sqrt",
+                  "colorScheme": "interpolateOranges",
+                  "exponent": 0.5,
+                  "mode": "opacity"
+               },
+               "dataFormat": "timeseries",
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "scaleDistribution": {
+                           "type": "linear"
+                        }
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 50
+               },
+               "heatmap": { },
+               "hideZeroBuckets": false,
+               "highlightCards": true,
+               "id": 33,
+               "legend": {
+                  "show": true
+               },
+               "options": {
+                  "calculate": true,
+                  "calculation": {
+                     "yBuckets": {
+                        "mode": "count",
+                        "scale": {
+                           "log": 2,
+                           "type": "log"
+                        }
+                     }
+                  },
+                  "cellGap": 2,
+                  "cellValues": { },
+                  "color": {
+                     "exponent": 0.5,
+                     "fill": "#806eb7",
+                     "mode": "opacity",
+                     "reverse": false,
+                     "scale": "exponential",
+                     "scheme": "Oranges",
+                     "steps": 128
+                  },
+                  "exemplars": {
+                     "color": "rgba(255,0,255,0.7)"
+                  },
+                  "filterValues": {
+                     "le": 1.0000000000000001e-09
+                  },
+                  "legend": {
+                     "show": true
+                  },
+                  "rowsFrame": {
+                     "layout": "auto"
+                  },
+                  "showValue": "never",
+                  "tooltip": {
+                     "show": true,
+                     "yHistogram": false
+                  },
+                  "yAxis": {
+                     "axisPlacement": "left",
+                     "decimals": 2,
+                     "min": "0",
+                     "reverse": false,
+                     "unit": "ms"
+                  }
+               },
+               "pluginVersion": "9.4.7",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0",
+                     "format": "time_series",
+                     "instant": false,
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "",
+                     "refId": "A"
+                  }
+               ],
+               "title": "OSD Read Op Latency Distribution",
+               "tooltip": {
+                  "show": true,
+                  "showHistogram": false
+               },
+               "type": "heatmap",
+               "xAxis": {
+                  "show": true
+               },
+               "xBucketNumber": null,
+               "xBucketSize": "",
+               "yAxis": {
+                  "decimals": null,
+                  "format": "ms",
+                  "logBase": 2,
+                  "max": null,
+                  "min": "0",
+                  "show": true,
+                  "splitFactor": 1
+               },
+               "yBucketBound": "auto",
+               "yBucketNumber": null,
+               "yBucketSize": null
+            },
+            {
+               "cards": {
+                  "cardPadding": null,
+                  "cardRound": null
+               },
+               "color": {
+                  "cardColor": "#f9934e",
+                  "colorScale": "sqrt",
+                  "colorScheme": "interpolateOranges",
+                  "exponent": 0.5,
+                  "mode": "opacity"
+               },
+               "dataFormat": "timeseries",
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "scaleDistribution": {
+                           "type": "linear"
+                        }
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 50
+               },
+               "heatmap": { },
+               "hideZeroBuckets": false,
+               "highlightCards": true,
+               "id": 34,
+               "legend": {
+                  "show": true
+               },
+               "options": {
+                  "calculate": true,
+                  "calculation": {
+                     "yBuckets": {
+                        "mode": "count",
+                        "scale": {
+                           "log": 2,
+                           "type": "log"
+                        }
+                     }
+                  },
+                  "cellGap": 2,
+                  "cellValues": { },
+                  "color": {
+                     "exponent": 0.5,
+                     "fill": "#f9934e",
+                     "mode": "opacity",
+                     "reverse": false,
+                     "scale": "exponential",
+                     "scheme": "Oranges",
+                     "steps": 128
+                  },
+                  "exemplars": {
+                     "color": "rgba(255,0,255,0.7)"
+                  },
+                  "filterValues": {
+                     "le": 1.0000000000000001e-09
+                  },
+                  "legend": {
+                     "show": true
+                  },
+                  "rowsFrame": {
+                     "layout": "auto"
+                  },
+                  "showValue": "never",
+                  "tooltip": {
+                     "show": true,
+                     "yHistogram": false
+                  },
+                  "yAxis": {
+                     "axisPlacement": "left",
+                     "decimals": 2,
+                     "min": "0",
+                     "reverse": false,
+                     "unit": "ms"
+                  }
+               },
+               "pluginVersion": "9.4.7",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0",
+                     "format": "time_series",
+                     "instant": false,
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "",
+                     "refId": "A"
+                  }
+               ],
+               "title": "OSD Write Op Latency Distribution",
+               "tooltip": {
+                  "show": true,
+                  "showHistogram": false
+               },
+               "type": "heatmap",
+               "xAxis": {
+                  "show": true
+               },
+               "xBucketNumber": null,
+               "xBucketSize": "",
+               "yAxis": {
+                  "decimals": null,
+                  "format": "ms",
+                  "logBase": 2,
+                  "max": null,
+                  "min": "0",
+                  "show": true,
+                  "splitFactor": 1
+               },
+               "yBucketBound": "auto",
+               "yBucketNumber": null,
+               "yBucketSize": null
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 10,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "never",
+                        "spanNulls": false,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           },
+                           {
+                              "color": "red",
+                              "value": 80
+                           }
+                        ]
+                     },
+                     "unit": "ms"
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 12,
+                  "x": 0,
+                  "y": 58
+               },
+               "id": 35,
+               "interval": "$interval",
+               "options": {
+                  "legend": {
+                     "calcs": [ ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true
+                  },
+                  "tooltip": {
+                     "mode": "multi",
+                     "sort": "none"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "avg(rate(ceph_osd_op_r_latency_sum{}[5m]) / rate(ceph_osd_op_r_latency_count{}[5m]) >= 0)",
+                     "format": "time_series",
+                     "intervalFactor": 1,
+                     "legendFormat": "Read",
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "avg(rate(ceph_osd_op_w_latency_sum{}[5m]) / rate(ceph_osd_op_w_latency_count{}[5m]) >= 0)",
+                     "format": "time_series",
+                     "intervalFactor": 1,
+                     "legendFormat": "Write",
+                     "refId": "B"
+                  }
+               ],
+               "title": "Recovery Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 10,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "never",
+                        "spanNulls": false,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           },
+                           {
+                              "color": "red",
+                              "value": 80
+                           }
+                        ]
+                     },
+                     "unit": "ms"
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 12,
+                  "x": 12,
+                  "y": 58
+               },
+               "id": 36,
+               "interval": "$interval",
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull",
+                        "max"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true
+                  },
+                  "tooltip": {
+                     "mode": "multi",
+                     "sort": "none"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "avg(ceph_osd_apply_latency_ms{})",
+                     "format": "time_series",
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "apply",
+                     "metric": "ceph_osd_perf_apply_latency_seconds",
+                     "refId": "A",
+                     "step": 4
+                  },
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "avg(ceph_osd_commit_latency_ms{})",
+                     "format": "time_series",
+                     "interval": "$interval",
+                     "intervalFactor": 1,
+                     "legendFormat": "commit",
+                     "metric": "ceph_osd_perf_commit_latency_seconds",
+                     "refId": "B",
+                     "step": 4
+                  }
+               ],
+               "title": "AVG OSD Apply + Commit Latency",
+               "type": "timeseries"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "LATENCY",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": true,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 45
+         },
+         "id": 37,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "left",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green"
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Time"
+                  },
+                  "properties": [
+                     {
+                        "id": "custom.hidden",
+                        "value": true
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 6,
+            "w": 24,
+            "x": 0,
+            "y": 46
+         },
+         "id": 38,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "exemplar": false,
+               "expr": "count by (ceph_version)(ceph_osd_metadata{})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "OSD Services",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "exemplar": false,
+               "expr": "count by (ceph_version)(ceph_mon_metadata{})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Mon Services",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "exemplar": false,
+               "expr": "count by (ceph_version)(ceph_mds_metadata{})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "MDS Services",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "exemplar": false,
+               "expr": "count by (ceph_version)(ceph_rgw_metadata{})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "RGW Services",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "exemplar": false,
+               "expr": "count by (ceph_version)(ceph_mgr_metadata{})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "MGR Services",
+               "range": false,
+               "refId": "E"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Ceph Versions",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": { },
+                  "indexByName": { },
+                  "renameByName": {
+                     "Time": "",
+                     "Value #A": "OSD Services",
+                     "Value #B": "Mon Services",
+                     "Value #C": "MDS Services",
+                     "Value #D": "RGW Services",
+                     "Value #E": "MGR Services",
+                     "ceph_version": "Ceph Version"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      }
+   ],
+   "refresh": "1m",
+   "rows": [ ],
+   "schemaVersion": 38,
+   "style": "dark",
+   "tags": [
+      "ceph-mixin"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "text": "Prometheus",
+               "value": "Prometheus"
+            },
+            "hide": 0,
+            "label": "Data Source",
+            "name": "DS_PROMETHEUS",
+            "options": [ ],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "auto": true,
+            "auto_count": 10,
+            "auto_min": "1m",
+            "current": {
+               "text": "$__auto_interval_interval",
+               "value": "$__auto_interval_interval"
+            },
+            "hide": 0,
+            "label": "Interval",
+            "name": "interval",
+            "options": [
+               {
+                  "selected": true,
+                  "text": "auto",
+                  "value": "$__auto_interval_interval"
+               },
+               {
+                  "selected": false,
+                  "text": "5s",
+                  "value": "5s"
+               },
+               {
+                  "selected": false,
+                  "text": "10s",
+                  "value": "10s"
+               },
+               {
+                  "selected": false,
+                  "text": "30s",
+                  "value": "30s"
+               },
+               {
+                  "selected": false,
+                  "text": "1m",
+                  "value": "1m"
+               },
+               {
+                  "selected": false,
+                  "text": "10m",
+                  "value": "10m"
+               },
+               {
+                  "selected": false,
+                  "text": "30m",
+                  "value": "30m"
+               },
+               {
+                  "selected": false,
+                  "text": "1h",
+                  "value": "1h"
+               },
+               {
+                  "selected": false,
+                  "text": "6h",
+                  "value": "6h"
+               },
+               {
+                  "selected": false,
+                  "text": "12h",
+                  "value": "12h"
+               },
+               {
+                  "selected": false,
+                  "text": "1d",
+                  "value": "1d"
+               },
+               {
+                  "selected": false,
+                  "text": "7d",
+                  "value": "7d"
+               },
+               {
+                  "selected": false,
+                  "text": "14d",
+                  "value": "14d"
+               },
+               {
+                  "selected": false,
+                  "text": "30d",
+                  "value": "30d"
+               }
+            ],
+            "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+            "refresh": 2,
+            "type": "interval",
+            "valuelabels": { }
+         }
+      ]
+   },
+   "time": {
+      "from": "now-6h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "",
+   "title": "Ceph Cluster - Advanced",
+   "version": 0
+}
diff --git a/monitoring/ceph-mixin/tests_dashboards/features/ceph-cluster.feature b/monitoring/ceph-mixin/tests_dashboards/features/ceph-cluster.feature
index 1a446cd2c535..c00107ec7ae3 100644
--- a/monitoring/ceph-mixin/tests_dashboards/features/ceph-cluster.feature
+++ b/monitoring/ceph-mixin/tests_dashboards/features/ceph-cluster.feature
@@ -1,54 +1,526 @@
 Feature: Ceph Cluster Dashboard
 
-Scenario: "Test total PG States"
+  Scenario: "Test cluster health"
   Given the following series:
     | metrics                  | values |
-    | ceph_pg_total{foo="var"} | 10 100 |
-    | ceph_pg_total{foo="bar"} | 20 200 |
-  Then Grafana panel `PG States` with legend `Total` shows:
+    | ceph_health_status{}   | 1.0 |
+  Then Grafana panel `Health Status` with legend `EMPTY` shows:
     | metrics | values |
-    | {}      | 300    |
+    | ceph_health_status{}      | 1.0 |
 
-Scenario: "Test OSDs in"
-  Given the following series:
-    | metrics                          | values |
-    | ceph_osd_in{ceph_daemon="osd.0"} | 1.0    |
-    | ceph_osd_in{ceph_daemon="osd.1"} | 0.0    |
-    | ceph_osd_in{ceph_daemon="osd.2"} | 1.0    |
-  When variable `instance` is `.*`
-  Then Grafana panel `OSDs` with legend `In` shows:
-    | metrics | values |
-    | {}      | 2      |
+  Scenario: "Test Firing Alerts Warning"
+    Given the following series:
+      | metrics                  | values |
+      | ALERTS{alertstate="firing",alertname="Ceph.1", severity="warning"}  | 1 |
+      | ALERTS{alertstate="firing",alertname="Ceph.2", severity="critical"}  | 1 |
+    Then Grafana panel `Firing Alerts` with legend `Warning` shows:
+      | metrics | values |
+      | {}      | 1 |
 
-Scenario: "Test OSDs down"
-  Given the following series:
-    | metrics                                                  | values |
-    | ceph_osd_up{ceph_daemon="osd.0", instance="127.0.0.1"} | 0.0    |
-    | ceph_osd_up{ceph_daemon="osd.1", instance="127.0.0.1"} | 0.0    |
-    | ceph_osd_up{ceph_daemon="osd.2", instance="127.0.0.1"} | 0.0    |
-  When variable `instance` is `127.0.0.1`
-  Then Grafana panel `OSDs` with legend `Down` shows:
-    | metrics | values |
-    | {}      | 3      |
+  Scenario: "Test Firing Alerts Critical"
+    Given the following series:
+      | metrics                  | values |
+      | ALERTS{alertstate="firing",alertname="Ceph.1", severity="warning"}  | 1 |
+      | ALERTS{alertstate="firing",alertname="Ceph.2", severity="critical"}  | 1 |
+    Then Grafana panel `Firing Alerts` with legend `Critical` shows:
+      | metrics | values |
+      | {}      | 1 |
 
-Scenario: "Test OSDs out"
-  Given the following series:
-    | metrics                                                  | values |
-    | ceph_osd_in{ceph_daemon="osd.0", instance="127.0.0.1"} | 0.0    |
-    | ceph_osd_in{ceph_daemon="osd.1", instance="127.0.0.1"} | 1.0    |
-    | ceph_osd_in{ceph_daemon="osd.2", instance="127.0.0.1"} | 0.0    |
-  When variable `instance` is `127.0.0.1`
-  Then Grafana panel `OSDs` with legend `Out` shows:
-    | metrics | values |
-    | {}      | 2      |
+    Scenario: "Test Available Capacity"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_cluster_total_bytes{}| 100 |
+      | ceph_cluster_total_used_bytes{}| 70 |
+    Then Grafana panel `Available Capacity` with legend `EMPTY` shows:
+      | metrics | values |
+      | {} | 0.3  |
 
-Scenario: "Test OSDs all"
-  Given the following series:
-    | metrics | values |
-    | ceph_osd_metadata{ceph_daemon="osd.0", instance="127.0.0.1"} | 1.0 |
-    | ceph_osd_metadata{ceph_daemon="osd.1", instance="127.0.0.1"} | 1.0 |
-    | ceph_osd_metadata{ceph_daemon="osd.2", instance="127.0.0.1"} | 1.0 |
-  When variable `instance` is `127.0.0.1`
-  Then Grafana panel `OSDs` with legend `All` shows:
-    | metrics | values |
-    | {}      | 3      |
+  Scenario: "Test Cluster Capacity"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_cluster_total_bytes{}| 100 |
+    Then Grafana panel `Cluster Capacity` with legend `EMPTY` shows:
+      | metrics | values |
+      | ceph_cluster_total_bytes{} | 100  |
+
+  Scenario: "Test Used Capacity"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_cluster_total_used_bytes{}| 100 |
+    Then Grafana panel `Used Capacity` with legend `EMPTY` shows:
+      | metrics | values |
+      | ceph_cluster_total_used_bytes{} | 100  |
+
+  Scenario: "Test Write Throughput"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_op_w_in_bytes{cluster="mycluster", osd="osd.0"} | 500 500 500 |
+      | ceph_osd_op_w_in_bytes{cluster="mycluster", osd="osd.1"}  | 500 120 110 |
+    Then Grafana panel `Write Throughput` with legend `EMPTY` shows:
+      | metrics | values |
+      | {}   | 2    |
+
+  Scenario: "Test Write IOPS"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_op_w{cluster="mycluster", osd="osd.0"} | 500 500 500 |
+      | ceph_osd_op_w{cluster="mycluster", osd="osd.1"}  | 500 120 110 |
+    Then Grafana panel `Write IOPS` with legend `EMPTY` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test Read Throughput"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_op_r_out_bytes{cluster="mycluster", osd="osd.0"} | 500 500 500 |
+      | ceph_osd_op_r_out_bytes{cluster="mycluster", osd="osd.1"}  | 500 120 110 |
+    Then Grafana panel `Read Throughput` with legend `EMPTY` shows:
+      | metrics | values |
+      | {}   | 2    |
+
+  Scenario: "Test Read IOPS"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_op_r{cluster="mycluster", osd="osd.0"} | 500 500 500 |
+      | ceph_osd_op_r{cluster="mycluster", osd="osd.1"}  | 500 120 110 |
+    Then Grafana panel `Read IOPS` with legend `EMPTY` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test OSDs All"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_metadata{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_osd_metadata{cluster="mycluster", osd="osd.2"} | 1 |
+      | ceph_osd_metadata{cluster="mycluster", osd="osd.3"} | 1 |
+    Then Grafana panel `OSDs` with legend `All` shows:
+      | metrics | values |
+      | {}   | 3 |
+
+  Scenario: "Test OSDs In"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_in{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_osd_in{cluster="mycluster", osd="osd.1"} | 1 |
+      | ceph_osd_in{cluster="mycluster", osd="osd.2"} | 1 |
+    Then Grafana panel `OSDs` with legend `In` shows:
+      | metrics | values |
+      | {}   | 3 |
+
+  Scenario: "Test OSDs Out"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_in{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_osd_in{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_osd_in{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `OSDs` with legend `Out` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test OSDs Up"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_up{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_osd_up{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_osd_up{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `OSDs` with legend `Up` shows:
+      | metrics | values |
+      | {}   | 1 |
+
+  Scenario: "Test OSDs Down"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_osd_up{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_osd_up{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_osd_up{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `OSDs` with legend `Down` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test MGRs Standby"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_mgr_status{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_mgr_status{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_mgr_status{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `MGRs` with legend `Standby` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test MGRs Active"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_mgr_status{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_mgr_status{cluster="mycluster", osd="osd.1"} | 0 |
+    Then Grafana panel `MGRs` with legend `Active` shows:
+      | metrics | values |
+      | {}   | 1 |
+
+  Scenario: "Test Monitors Total"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `Monitors` with legend `Total` shows:
+      | metrics | values |
+      | {}   | 3 |
+
+  Scenario: "Test Monitors In Quorum"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `Monitors` with legend `In Quorum` shows:
+      | metrics | values |
+      | {}   | 1 |
+
+  Scenario: "Test Monitors out of Quorum"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.0"} | 1 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.1"} | 0 |
+      | ceph_mon_quorum_status{cluster="mycluster", osd="osd.2"} | 0 |
+    Then Grafana panel `Monitors` with legend `MONs out of Quorum` shows:
+      | metrics | values |
+      | {}   | 2 |
+
+  Scenario: "Test Total Capacity"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_cluster_total_bytes{cluster="mycluster", osd="osd.0"} | 100 |
+    Then Grafana panel `Capacity` with legend `Total Capacity` shows:
+      | metrics | values |
+      | ceph_cluster_total_bytes{cluster="mycluster", osd="osd.0"}   | 100 |
+
+  Scenario: "Test Used Capacity"
+    Given the following series:
+      | metrics                  | values |
+      | ceph_cluster_total_used_bytes{cluster="mycluster", osd="osd.0"} | 100 |
+    Then Grafana panel `Capacity` with legend `Used` shows:
+      | metrics | values |
+      | ceph_cluster_total_used_bytes{cluster="mycluster", osd="osd.0"}   | 100 |
+
+    Scenario: "Test Cluster Throughput Write"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_op_w_in_bytes{cluster="mycluster", osd="osd.0"} | 1000 1000|
+      | ceph_osd_op_w_in_bytes{cluster="mycluster", osd="osd.1"} | 2000 1500 |
+    Then Grafana panel `Cluster Throughput` with legend `Write` shows:
+      | metrics | values |
+      | {}      | 25  |
+
+  Scenario: "Test Cluster Throughput Read"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_op_r_out_bytes{cluster="mycluster", osd="osd.0"} | 1000 1000|
+      | ceph_osd_op_r_out_bytes{cluster="mycluster", osd="osd.1"} | 2000 1500 |
+    Then Grafana panel `Cluster Throughput` with legend `Read` shows:
+      | metrics | values |
+      | {}      | 25  |
+
+    Scenario: "Test IOPS Read"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_op_r{cluster="mycluster", osd="osd.0"} | 1000 1000|
+      | ceph_osd_op_r{cluster="mycluster", osd="osd.1"} | 2000 1500 |
+    Then Grafana panel `IOPS` with legend `Read` shows:
+      | metrics | values |
+      | {}      | 25  |
+
+  Scenario: "Test IOPS Write"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_op_w{cluster="mycluster", osd="osd.0"} | 1000 1000|
+      | ceph_osd_op_w{cluster="mycluster", osd="osd.1"} | 2000 1500 |
+    Then Grafana panel `IOPS` with legend `Write` shows:
+      | metrics | values |
+      | {}      | 25  |
+
+    Scenario: "Test Pool Used Bytes"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_bytes_used{cluster="mycluster", pool_id="1"} | 10000 |
+      | ceph_pool_bytes_used{cluster="mycluster", pool_id="2"} | 20000 |
+      | ceph_pool_bytes_used{cluster="mycluster", pool_id="3"} | 30000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="1", name="pool1"} | 2000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="2", name="pool2"} | 4000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="3", name="pool3"} | 6000 |
+    Then Grafana panel `Pool Used Bytes` with legend `{{name}}` shows:
+      | metrics | values |
+      | {cluster="mycluster", name="pool1", pool_id="1"}       | 20000000 |
+      | {cluster="mycluster", name="pool2", pool_id="2"}       | 80000000 |
+      | {cluster="mycluster", name="pool3", pool_id="3"}       | 180000000 |
+
+  Scenario: "Test Pool Used RAW Bytes"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_stored_raw{cluster="mycluster", pool_id="1"} | 10000 |
+      | ceph_pool_stored_raw{cluster="mycluster", pool_id="2"} | 20000 |
+      | ceph_pool_stored_raw{cluster="mycluster", pool_id="3"} | 30000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="1", name="pool1"} | 2000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="2", name="pool2"} | 4000 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="3", name="pool3"} | 6000 |
+    Then Grafana panel `Pool Used RAW Bytes` with legend `{{name}}` shows:
+      | metrics | values |
+      | {cluster="mycluster", name="pool1", pool_id="1"}       | 20000000 |
+      | {cluster="mycluster", name="pool2", pool_id="2"}       | 80000000 |
+      | {cluster="mycluster", name="pool3", pool_id="3"}       | 180000000 |
+
+  Scenario: "Test Pool Objects Quota"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_quota_objects{cluster="mycluster", pool_id="1"} | 10 |
+      | ceph_pool_quota_objects{cluster="mycluster", pool_id="2"} | 20 |
+      | ceph_pool_quota_objects{cluster="mycluster", pool_id="3"} | 30 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="1", name="pool1"} | 10 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="2", name="pool2"} | 15 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="3", name="pool3"} | 15 |
+    Then Grafana panel `Pool Objects Quota` with legend `{{name}}` shows:
+      | metrics | values |
+      | {cluster="mycluster", name="pool1", pool_id="1"}       | 100 |
+      | {cluster="mycluster", name="pool2", pool_id="2"}       | 300 |
+      | {cluster="mycluster", name="pool3", pool_id="3"}       | 450|
+
+    Scenario: "Test Pool Quota Bytes"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_quota_bytes{cluster="mycluster", pool_id="1"} | 100 |
+      | ceph_pool_quota_bytes{cluster="mycluster", pool_id="2"} | 200 |
+      | ceph_pool_quota_bytes{cluster="mycluster", pool_id="3"} | 300 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="1", name="pool1"} | 100 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="2", name="pool2"} | 150 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="3", name="pool3"} | 150 |
+    Then Grafana panel `Pool Quota Bytes` with legend `{{name}}` shows:
+      | metrics | values |
+      | {cluster="mycluster", name="pool1", pool_id="1"}       | 10000 |
+      | {cluster="mycluster", name="pool2", pool_id="2"}       | 30000 |
+      | {cluster="mycluster", name="pool3", pool_id="3"}       | 45000 |
+
+  Scenario: "Test Objects Per Pool"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_objects{cluster="mycluster", pool_id="1"} | 100 |
+      | ceph_pool_objects{cluster="mycluster", pool_id="2"} | 200 |
+      | ceph_pool_objects{cluster="mycluster", pool_id="3"} | 300 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="1", name="pool1"} | 100 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="2", name="pool2"} | 150 |
+      | ceph_pool_metadata{cluster="mycluster", pool_id="3", name="pool3"} | 150 |
+    Then Grafana panel `Objects Per Pool` with legend `{{name}}` shows:
+      | metrics | values |
+      | {cluster="mycluster", name="pool1", pool_id="1"}       | 10000 |
+      | {cluster="mycluster", name="pool2", pool_id="2"}       | 30000 |
+      | {cluster="mycluster", name="pool3", pool_id="3"}       | 45000|
+
+  Scenario: "Test OSD Type Count"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pool_objects{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pool_objects{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `OSD Type Count` with legend `Total` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Backfill Toofull"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_backfill_toofull{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_backfill_toofull{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Backfill Toofull` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Remapped"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_remapped{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_remapped{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Remapped` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Backfill"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_backfill{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_backfill{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Backfill` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Peered"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_peered{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_peered{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Peered` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Down"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_down{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_down{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Down` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Repair"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_repair{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_repair{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Repair` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Recovering"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_recovering{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_recovering{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Recovering` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Deep"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_deep{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_deep{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Deep` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Wait Backfill"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_wait_backfill{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_wait_backfill{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Wait Backfill` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Creating"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_creating{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_creating{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Creating` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Forced Recovery"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_forced_recovery{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_forced_recovery{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Forced Recovery` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Forced Backfill"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_forced_backfill{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_forced_backfill{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Forced Backfill` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Incomplete"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_incomplete{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_incomplete{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Incomplete` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test PGs State Undersized"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_undersized{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_undersized{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `PGs State` with legend `Undersized` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test Stuck PGs Undersized"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_undersized{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_undersized{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `Stuck PGs` with legend `Undersized` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test Stuck PGs Stale"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_stale{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_stale{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `Stuck PGs` with legend `Stale` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test Stuck PGs Degraded"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_pg_degraded{cluster="mycluster", osd="osd.0"} | 10 |
+      | ceph_pg_degraded{cluster="mycluster", osd="osd.1"} | 20 |
+    Then Grafana panel `Stuck PGs` with legend `Degraded` shows:
+      | metrics | values |
+      | {}      | 30 |
+
+  Scenario: "Test Recovery Operations"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_recovery_ops{cluster="mycluster", osd="osd.0"}| 250 200 |
+      | ceph_osd_recovery_ops{cluster="mycluster", osd="osd.1"} | 800 100 |
+    When variable `interval` is `120s`
+    Then Grafana panel `Recovery Operations` with legend `OPS` shows:
+      | metrics | values |
+      |  {} | 5 |
+
+  Scenario: "Test Ceph Versions OSD"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_osd_metadata{cluster="mycluster", osd="osd.0"}| 17 |
+    Then Grafana panel `Ceph Versions` with legend `OSD Services` shows:
+      | metrics | values |
+      |  {} | 1 |
+
+  Scenario: "Test Ceph Versions Mon"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_mon_metadata{cluster="mycluster", osd="osd.0"}| 17 |
+    Then Grafana panel `Ceph Versions` with legend `Mon Services` shows:
+      | metrics | values |
+      |  {} | 1 |
+
+  Scenario: "Test Ceph Versions MDS"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_mds_metadata{cluster="mycluster", osd="osd.0"}| 17 |
+    Then Grafana panel `Ceph Versions` with legend `MDS Services` shows:
+      | metrics | values |
+      |  {} | 1 |
+
+  Scenario: "Test Ceph Versions RGW"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_rgw_metadata{cluster="mycluster", osd="osd.0"}| 17 |
+    Then Grafana panel `Ceph Versions` with legend `RGW Services` shows:
+      | metrics | values |
+      |  {} | 1 |
+
+  Scenario: "Test Ceph Versions MGR"
+    Given the following series:
+      | metrics                          | values |
+      | ceph_mgr_metadata{cluster="mycluster", osd="osd.0"}| 17 |
+    Then Grafana panel `Ceph Versions` with legend `MGR Services` shows:
+      | metrics | values |
+      |  {} | 1 |
\ No newline at end of file
diff --git a/monitoring/ceph-mixin/tests_dashboards/util.py b/monitoring/ceph-mixin/tests_dashboards/util.py
index 835dedee7eb5..4213ff6d5a1e 100644
--- a/monitoring/ceph-mixin/tests_dashboards/util.py
+++ b/monitoring/ceph-mixin/tests_dashboards/util.py
@@ -43,6 +43,7 @@ def add_dashboard_queries(data: Dict[str, Any], dashboard_data: Dict[str, Any],
     if 'panels' not in dashboard_data:
         return
     error = 0
+    panel_ids_in_file = set()
     for panel in dashboard_data['panels']:
         if (
                 'title' in panel
@@ -54,18 +55,13 @@ def add_dashboard_queries(data: Dict[str, Any], dashboard_data: Dict[str, Any],
                 title = panel['title']
                 legend_format = target['legendFormat'] if 'legendFormat' in target else ""
                 query_id = f'{title}-{legend_format}'
-                if query_id in data['queries']:
-                    # NOTE: If two or more panels have the same name and legend it
-                    # might suggest a refactoring is needed or add something else
-                    # to identify each query.
-                    conflict_file = Path(data['queries'][query_id]['path']).name
-                    file = Path(path).name
+                if query_id in panel_ids_in_file:
                     cprint((f'ERROR: Query in panel "{title}" with legend "{legend_format}"'
-                                       f' already exists. Conflict "{conflict_file}" '
-                                       f'with: "{file}"'), 'red')
+                            f' already exists in the same file: "{path}"'), 'red')
                     error = 1
                 data['queries'][query_id] = {'query': target['expr'], 'path': path}
                 data['stats'][path]['total'] += 1
+                panel_ids_in_file.add(query_id)
     if error:
         raise ValueError('Missing legend_format in queries, please add a proper value.')
 

From c479f2e6caa1f8f736eedb6f0036dbaa4649a204 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Sun, 22 Oct 2023 07:09:22 +0000
Subject: [PATCH 0255/2492] doc/rgw: pubsub capabilities reference was removed
 from docs

most pubsub docs were removed in
0fecbe2476afbb06de7ea731b7c92066ae40cfc4
however, the text removed described functionality that is only
supported bia pubsub

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 doc/radosgw/s3-notification-compatibility.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
index 9a101306aa3d..1627ed0c4db0 100644
--- a/doc/radosgw/s3-notification-compatibility.rst
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -13,7 +13,7 @@ Supported Destination
 ---------------------
 
 AWS supports: **SNS**, **SQS** and **Lambda** as possible destinations (AWS internal destinations). 
-Currently, we support: **HTTP/S**, **Kafka** and **AMQP**. And also support pulling and acking of events stored in Ceph (as an internal destination).
+Currently, we support: **HTTP/S**, **Kafka** and **AMQP**.
 
 We are using the **SNS** ARNs to represent the **HTTP/S**, **Kafka** and **AMQP** destinations.
 

From 2ab7c16737dbf97402907db74d7432a056f2fa97 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Sun, 15 Oct 2023 15:54:07 +0000
Subject: [PATCH 0256/2492] rgw/notifications/test: install java for kafka
 build

also use kafka binaries insted of building from source

Fixes: https://tracker.ceph.com/issues/63205

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 .../rgw/notifications/tasks/0-install.yaml    |  7 +++++++
 qa/tasks/kafka.py                             | 21 +++++++------------
 src/test/rgw/bucket_notification/test_bn.py   |  3 +--
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/qa/suites/rgw/notifications/tasks/0-install.yaml b/qa/suites/rgw/notifications/tasks/0-install.yaml
index 9e7852d4d25f..5c83d5c0d23f 100644
--- a/qa/suites/rgw/notifications/tasks/0-install.yaml
+++ b/qa/suites/rgw/notifications/tasks/0-install.yaml
@@ -6,6 +6,13 @@ tasks:
     client.0:
 
 overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        rpm:
+        - java
+        deb:
+        - default-jre
   ceph:
     conf:
       global:
diff --git a/qa/tasks/kafka.py b/qa/tasks/kafka.py
index 48bf3611f5a1..5e6c208ca30e 100644
--- a/qa/tasks/kafka.py
+++ b/qa/tasks/kafka.py
@@ -17,9 +17,11 @@ def get_kafka_version(config):
             kafka_version = client_config.get('kafka_version')
     return kafka_version
 
+kafka_prefix = 'kafka_2.13-'
+
 def get_kafka_dir(ctx, config):
     kafka_version = get_kafka_version(config)
-    current_version = 'kafka-' + kafka_version + '-src'
+    current_version = kafka_prefix + kafka_version
     return '{tdir}/{ver}'.format(tdir=teuthology.get_testdir(ctx),ver=current_version)
 
 
@@ -36,14 +38,15 @@ def install_kafka(ctx, config):
         test_dir=teuthology.get_testdir(ctx)
         current_version = get_kafka_version(config)
 
-        link1 = 'https://archive.apache.org/dist/kafka/' + current_version + '/kafka-' + current_version + '-src.tgz'
+        kafka_file =  kafka_prefix + current_version + '.tgz'
+
+        link1 = 'https://archive.apache.org/dist/kafka/' + current_version + '/' + kafka_file
         ctx.cluster.only(client).run(
             args=['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'wget', link1],
         )
 
-        file1 = 'kafka-' + current_version + '-src.tgz'
         ctx.cluster.only(client).run(
-            args=['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'tar', '-xvzf', file1],
+            args=['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'tar', '-xvzf', kafka_file],
         )
 
     try:
@@ -61,9 +64,8 @@ def install_kafka(ctx, config):
                 args=['rm', '-rf', test_dir],
             )
 
-            rmfile1 = 'kafka-' + current_version + '-src.tgz'
             ctx.cluster.only(client).run(
-                args=['rm', '-rf', '{tdir}/{doc}'.format(tdir=teuthology.get_testdir(ctx),doc=rmfile1)],
+                args=['rm', '-rf', '{tdir}/{doc}'.format(tdir=teuthology.get_testdir(ctx),doc=kafka_file)],
             )
 
 
@@ -79,13 +81,6 @@ def run_kafka(ctx,config):
     for (client,_) in config.items():
         (remote,) = ctx.cluster.only(client).remotes.keys()
 
-        ctx.cluster.only(client).run(
-            args=['cd', '{tdir}'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
-             './gradlew', 'jar', 
-             '-PscalaVersion=2.13.2'
-            ],
-        )
-
         ctx.cluster.only(client).run(
             args=['cd', '{tdir}/bin'.format(tdir=get_kafka_dir(ctx, config)), run.Raw('&&'),
              './zookeeper-server-start.sh',
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 041ca3ebe114..fc5887869fcb 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -1615,8 +1615,7 @@ def test_ps_s3_notification_push_kafka_on_master():
         time.sleep(5)
         receiver.verify_s3_events(keys, exact_match=True, deletions=True, etags=etags)
     except Exception as e:
-        print(e)
-        assert False
+        assert False, str(e)
     finally:
         # cleanup
         if s3_notification_conf is not None:

From 22cd766ca905dd66c8502c457c8262538c88a085 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 19 Oct 2023 12:05:40 +0000
Subject: [PATCH 0257/2492] crimson/osd/pg: do_osd_ops failure_func refactor

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 3f9a9e0f7a2f..892201024a66 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1046,11 +1046,11 @@ PG::do_osd_ops(
         // TODO: https://tracker.ceph.com/issues/61651
         fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
       }
-      return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete,  this] {
+      return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
         auto log_reply_fut = [m, e, this] {
           return log_reply(m, e);
         };
-
+        auto fut2 = seastar::now();
         if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
           auto it = log_entry_update_waiting_on.find(rep_tid);
           ceph_assert(it != log_entry_update_waiting_on.end());
@@ -1064,16 +1064,18 @@ PG::do_osd_ops(
               peering_state.complete_write(version, last_complete);
             }
           } else {
-            return it->second.all_committed.get_shared_future()
-              .then([this, &version, last_complete, log_reply_fut = std::move(log_reply_fut)] {
+            fut2 = it->second.all_committed.get_shared_future().then(
+              [this, &version, last_complete] {
               if (version != eversion_t()) {
                 peering_state.complete_write(version, last_complete);
               }
-              return log_reply_fut();
+              return seastar::now();
             });
           }
         }
-        return log_reply_fut();
+        return fut2.then([this, log_reply_fut = std::move(log_reply_fut)] {
+          return log_reply_fut();
+        });
       });
     });
   });

From dd5c18ad575f5cff5a8cbbce69983096a0ebdfa5 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 19 Oct 2023 12:11:56 +0000
Subject: [PATCH 0258/2492] crimson/osd/pg: do_osd_ops failure_func refactor

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 892201024a66..0cd6c503e4d7 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1047,9 +1047,6 @@ PG::do_osd_ops(
         fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
       }
       return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
-        auto log_reply_fut = [m, e, this] {
-          return log_reply(m, e);
-        };
         auto fut2 = seastar::now();
         if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
           auto it = log_entry_update_waiting_on.find(rep_tid);
@@ -1057,7 +1054,6 @@ PG::do_osd_ops(
           auto it2 = it->second.waiting_on.find(pg_whoami);
           ceph_assert(it2 != it->second.waiting_on.end());
           it->second.waiting_on.erase(it2);
-
           if (it->second.waiting_on.empty()) {
             log_entry_update_waiting_on.erase(it);
             if (version != eversion_t()) {
@@ -1073,8 +1069,8 @@ PG::do_osd_ops(
             });
           }
         }
-        return fut2.then([this, log_reply_fut = std::move(log_reply_fut)] {
-          return log_reply_fut();
+        return fut2.then([this, m, e] {
+          return log_reply(m, e);
         });
       });
     });

From e4db99a699aa040a5c102322a75aa399cd9de19f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 15:53:34 -0400
Subject: [PATCH 0259/2492] cephadm: assert that ceph specific env vars get set

Assert that ceph based services set a tcmalloc related env var.
Also assert that a few assorted services do not set the env var.

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 1f4eddf3a70d..5a8950f3171a 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -62,6 +62,7 @@ def test_deploy_nfs_container(cephadm_fs, monkeypatch):
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ceph/ceph:latest -F -L STDERR')
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     _firewalld().open_ports.assert_called_with([2049])
     with open(f'/var/lib/ceph/{fsid}/nfs.fun/config') as f:
         assert f.read() == 'BALONEY'
@@ -211,6 +212,7 @@ def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ayeaye/iscsi:latest')
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -250,6 +252,7 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ownf/nmve:latest')
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -360,6 +363,7 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith(
         'quay.io/ceph/ceph:latest -n mgr.foo -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false'
     )
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -408,6 +412,7 @@ def test_deploy_ceph_exporter_container(cephadm_fs, monkeypatch):
         'quay.io/ceph/ceph:latest -n client.ceph-exporter.zaq -f --sock-dir=/var/run/ceph/ --addrs=0.0.0.0 --port=9926 --prio-limit=12 --stats-period=5'
     )
     assert '--entrypoint /usr/bin/ceph-exporter' in runfile_lines[-1]
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'

From 4c7051de098abec04afdcdc27b735ab868776fa1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:04:48 -0400
Subject: [PATCH 0260/2492] cephadm: add test assertions for unlimited pids
 option

Ensure that future changes continue to set/not set option as
appropriate.

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 5a8950f3171a..bdadcf9906bc 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -63,6 +63,7 @@ def test_deploy_nfs_container(cephadm_fs, monkeypatch):
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ceph/ceph:latest -F -L STDERR')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
     _firewalld().open_ports.assert_called_with([2049])
     with open(f'/var/lib/ceph/{fsid}/nfs.fun/config') as f:
         assert f.read() == 'BALONEY'
@@ -95,6 +96,7 @@ def test_deploy_snmp_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith(
         'quay.io/aaabbb/snmp:latest --web.listen-address=:9464 --snmp.destination=192.168.100.10:8899 --snmp.version=V2c --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
     )
+    assert '--pids-limit' not in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/snmp-gateway.sunmop')
     assert basedir.is_dir()
@@ -173,6 +175,7 @@ def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith(
         'quay.io/lfeuwbo/haproxy:latest haproxy -f /var/lib/haproxy/haproxy.cfg'
     )
+    assert '--pids-limit' not in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()
@@ -213,6 +216,7 @@ def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ayeaye/iscsi:latest')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -253,6 +257,7 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/ownf/nmve:latest')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -364,6 +369,7 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
         'quay.io/ceph/ceph:latest -n mgr.foo -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false'
     )
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'
@@ -413,6 +419,7 @@ def test_deploy_ceph_exporter_container(cephadm_fs, monkeypatch):
     )
     assert '--entrypoint /usr/bin/ceph-exporter' in runfile_lines[-1]
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'

From 43c4c16a4267887a9c5b3672b7d4d64db0ac8180 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:08:25 -0400
Subject: [PATCH 0261/2492] cephadm: add test assertions for ceph mgr
 entrypoint, mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index bdadcf9906bc..1c877df6bf52 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -370,6 +370,9 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
     )
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
     assert '--pids-limit' in runfile_lines[-1]
+    assert '--entrypoint /usr/bin/ceph-mgr' in runfile_lines[-1]
+    assert f'-v /var/lib/ceph/{fsid}/mgr.foo:/var/lib/ceph/mgr/ceph-foo:z' in runfile_lines[-1]
+    assert f'-v /var/log/ceph/{fsid}:/var/log/ceph:z' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'

From 2cd155eb42020efb570406817d140f07c8e57841 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:10:32 -0400
Subject: [PATCH 0262/2492] cephadm: add a test assertion for tracing env var

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 1c877df6bf52..d7712beb85f7 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -334,6 +334,7 @@ def test_deploy_a_tracing_container(cephadm_fs, monkeypatch):
     with open(basedir / 'unit.run') as f:
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
+    assert '-e discovery.type=single-node' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/rubber/elasticsearch:latest')
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()

From 022dd4b4680888ddb6e3745b39fc9c4706f3aac1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:35:32 -0400
Subject: [PATCH 0263/2492] cephadm: add test assertion for snmp env file
 option

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index d7712beb85f7..098bdfc3f1da 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -97,6 +97,7 @@ def test_deploy_snmp_container(cephadm_fs, monkeypatch):
         'quay.io/aaabbb/snmp:latest --web.listen-address=:9464 --snmp.destination=192.168.100.10:8899 --snmp.version=V2c --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
     )
     assert '--pids-limit' not in runfile_lines[-1]
+    assert f'--env-file=/var/lib/ceph/{fsid}/snmp-gateway.sunmop/snmp-gateway.conf' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/snmp-gateway.sunmop')
     assert basedir.is_dir()

From e6c44ef2597cf936a85728d2796e1e378bb8f46f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:32:15 -0400
Subject: [PATCH 0264/2492] cephadm: add assertions for monitoring options,
 mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 098bdfc3f1da..98a2b2a6d1fb 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -302,6 +302,8 @@ def test_deploy_a_monitoring_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith(
         'quay.io/titans/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.listen-address=:9095 --storage.tsdb.retention.time=15d --storage.tsdb.retention.size=0 --web.external-url=http://10.10.10.10:9095'
     )
+    assert '--user 8765' in runfile_lines[-1]
+    assert f'-v /var/lib/ceph/{fsid}/prometheus.fire/etc/prometheus:/etc/prometheus:Z' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()

From 03c29598d1065f156c5126d655fc34a5ed6d681b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:12:40 -0400
Subject: [PATCH 0265/2492] cephadm: add test assertions for nfs env vars,
 mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 98a2b2a6d1fb..c45e05166f64 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -64,6 +64,8 @@ def test_deploy_nfs_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith('quay.io/ceph/ceph:latest -F -L STDERR')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     assert '--pids-limit' in runfile_lines[-1]
+    assert '-e CEPH_CONF=' in runfile_lines[-1]
+    assert f'-v /var/lib/ceph/{fsid}/nfs.fun/etc/ganesha:/etc/ganesha:z' in runfile_lines[-1]
     _firewalld().open_ports.assert_called_with([2049])
     with open(f'/var/lib/ceph/{fsid}/nfs.fun/config') as f:
         assert f.read() == 'BALONEY'

From 351326fe9eda6ae0da89a6fbb5fa1cc66434f430 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:16:32 -0400
Subject: [PATCH 0266/2492] cephadm: add test assertion for haproxy options,
 mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index c45e05166f64..94f133e8db97 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -179,6 +179,8 @@ def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
         'quay.io/lfeuwbo/haproxy:latest haproxy -f /var/lib/haproxy/haproxy.cfg'
     )
     assert '--pids-limit' not in runfile_lines[-1]
+    assert '--user=root' in runfile_lines[-1]
+    assert f'-v {basedir}/haproxy:/var/lib/haproxy' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()

From 93217f61ad8843855520380bcd760eeca869412c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:20:10 -0400
Subject: [PATCH 0267/2492] cephadm: add test assertions for keepalived
 options, mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 94f133e8db97..1dd685358643 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -134,6 +134,11 @@ def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith('quay.io/eeranimated/keepalived:latest')
+    assert '-e KEEPALIVED_AUTOCONF=false' in runfile_lines[-1]
+    assert '-e KEEPALIVED_DEBUG=false' in runfile_lines[-1]
+    assert '--cap-add=NET_ADMIN' in runfile_lines[-1]
+    assert '--cap-add=NET_RAW' in runfile_lines[-1]
+    assert f'-v {basedir}/keepalived.conf:/etc/keepalived/keepalived.conf' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     assert not (basedir / 'config').exists()
     assert not (basedir / 'keyring').exists()

From bd2c1b751329044901503c232f5a75982a5f7f42 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:29:38 -0400
Subject: [PATCH 0268/2492] cephadm: add test assertions for iscsi options,
 mounts

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 1dd685358643..872b2bfc7531 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -227,6 +227,9 @@ def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith('quay.io/ayeaye/iscsi:latest')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     assert '--pids-limit' in runfile_lines[-1]
+    assert '--privileged' in runfile_lines[-1]
+    assert f'-v {basedir}/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z' in runfile_lines[-1]
+    assert '--mount type=bind,source=/lib/modules,destination=/lib/modules' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'

From ef775debb5a44797337d304a536e74a859a5cdb8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 16:27:13 -0400
Subject: [PATCH 0269/2492] cephadm: add test assertions for nvmeof options,
 mount

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 872b2bfc7531..ced9a0bb7f5d 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -271,6 +271,11 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
     assert runfile_lines[-1].endswith('quay.io/ownf/nmve:latest')
     assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' not in runfile_lines[-1]
     assert '--pids-limit' in runfile_lines[-1]
+    assert '--ulimit memlock=-1:-1' in runfile_lines[-1]
+    assert '--cap-add=SYS_ADMIN' in runfile_lines[-1]
+    assert '--cap-add=CAP_SYS_NICE' in runfile_lines[-1]
+    assert f'-v {basedir}/ceph-nvmeof.conf:/src/ceph-nvmeof.conf:z' in runfile_lines[-1]
+    assert '--mount type=bind,source=/lib/modules,destination=/lib/modules' in runfile_lines[-1]
     _firewalld().open_ports.assert_not_called()
     with open(basedir / 'config') as f:
         assert f.read() == 'XXXXXXX'

From e771a8f06d613e58160d0f1dc45c1e2ff7dbc0f1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 22 Oct 2023 08:14:24 -0400
Subject: [PATCH 0270/2492] cephadm: add deployment test for osd

Add a deployment test case for OSD. OSD has some special properties that
we have extra assertions for.

Part of a series of commits to increase coverage of deployment
path features with regards to container engine options, env vars
and mounts. This will serve future refactoring efforts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 45 ++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index ced9a0bb7f5d..c77b243dfa77 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -404,6 +404,51 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
     assert _make_var_run.call_args[0][3] == 8765
 
 
+def test_deploy_ceph_osd_container(cephadm_fs, monkeypatch):
+    mocks = _common_mp(monkeypatch)
+    _firewalld = mocks['Firewalld']
+    _make_var_run = mock.MagicMock()
+    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'osd.quux'
+        ctx.image = 'quay.io/ceph/ceph:latest'
+        ctx.reconfig = False
+        ctx.allow_ptrace = False
+        ctx.osd_fsid = '00000000-0000-0000-0000-000000000000'
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/osd.quux')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith(
+        'quay.io/ceph/ceph:latest -n osd.quux -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false'
+    )
+    assert '-e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES' in runfile_lines[-1]
+    assert '--privileged' in runfile_lines[-1]
+    assert '--pids-limit' in runfile_lines[-1]
+    assert '--entrypoint /usr/bin/ceph-osd' in runfile_lines[-1]
+    assert f'-v /var/lib/ceph/{fsid}/osd.quux:/var/lib/ceph/osd/ceph-quux:z' in runfile_lines[-1]
+    assert f'-v /var/log/ceph/{fsid}:/var/log/ceph:z' in runfile_lines[-1]
+    assert '-v /dev:/dev' in runfile_lines[-1]
+    _firewalld().open_ports.assert_not_called()
+    with open(basedir / 'config') as f:
+        assert f.read() == 'XXXXXXX'
+    with open(basedir / 'keyring') as f:
+        assert f.read() == 'YYYYYY'
+    assert _make_var_run.call_count == 1
+    assert _make_var_run.call_args[0][2] == 8765
+    assert _make_var_run.call_args[0][3] == 8765
+
+
 def test_deploy_ceph_exporter_container(cephadm_fs, monkeypatch):
     mocks = _common_mp(monkeypatch)
     _firewalld = mocks['Firewalld']

From 1a281ccd71bb00606e4d317d46b0144e673f53a9 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 23 Oct 2023 12:02:47 +0100
Subject: [PATCH 0271/2492] mgr/dashboard: fixed edit user quota form error

Fixes: https://tracker.ceph.com/issues/63287

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../dashboard/frontend/src/app/shared/forms/cd-form-group.ts    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-form-group.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-form-group.ts
index 64820d82a412..09aac7136c46 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-form-group.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-form-group.ts
@@ -68,7 +68,7 @@ export class CdFormGroup extends UntypedFormGroup {
   showError(controlName: string, form: NgForm, errorName?: string): boolean {
     const control = this.get(controlName);
     return (
-      (form.submitted || control.dirty) &&
+      (form?.submitted || control.dirty) &&
       (errorName ? control.hasError(errorName) : control.invalid)
     );
   }

From 8cfadb7e982a4d16e058dc73a3bd8b2d7b560434 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Thu, 5 Oct 2023 12:14:35 -0400
Subject: [PATCH 0272/2492] rgw/notification: Add support to set topic policy
 while topic creation

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 doc/radosgw/notifications.rst               |  77 ++++
 src/common/options/rgw.yaml.in              |  15 +
 src/rgw/rgw_auth_s3.cc                      |   1 +
 src/rgw/rgw_iam_policy.cc                   |  22 ++
 src/rgw/rgw_iam_policy.h                    |   9 +-
 src/rgw/rgw_op_type.h                       |   1 +
 src/rgw/rgw_pubsub.cc                       |   8 +-
 src/rgw/rgw_pubsub.h                        |  11 +-
 src/rgw/rgw_rest_pubsub.cc                  | 394 ++++++++++++++++++--
 src/test/rgw/bucket_notification/api.py     |  15 +-
 src/test/rgw/bucket_notification/test_bn.py |  70 ++++
 src/test/rgw/test_rgw_iam_policy.cc         |   3 +-
 12 files changed, 580 insertions(+), 46 deletions(-)

diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
index 16f4847c5dc0..a38ad9e01bdf 100644
--- a/doc/radosgw/notifications.rst
+++ b/doc/radosgw/notifications.rst
@@ -159,6 +159,7 @@ updating, use the name of an existing topic and different endpoint values).
    [&Attributes.entry.12.key=time_to_live&Attributes.entry.12.value=<seconds to live>]
    [&Attributes.entry.13.key=max_retries&Attributes.entry.13.value=<retries number>]
    [&Attributes.entry.14.key=retry_sleep_duration&Attributes.entry.14.value=<sleep seconds>]
+   [&Attributes.entry.14.key=Policy&Attributes.entry.14.value=<policy-JSON-string>]
 
 Request parameters:
 
@@ -179,6 +180,25 @@ Request parameters:
   default value is taken from `rgw_topic_persistency_sleep_duration`.
   providing a value overrides the global value.
   zero value mean there is no delay between retries.
+- Policy: This will control who can access the topic in addition to the owner of the topic.
+  The policy passed needs to be a JSON string similar to bucket policy.
+  For example, one can send a policy string as follows::
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [{
+        "Effect": "Allow",
+        "Principal": {"AWS": ["arn:aws:iam::usfolks:user/fred:subuser"]},
+        "Action": ["sns:GetTopicAttributes","sns:Publish"],
+        "Resource": ["arn:aws:sns:default::mytopic"],
+      }]
+    }
+
+  Currently, we support only the following actions:
+  - sns:GetTopicAttributes  To list or get existing topics
+  - sns:SetTopicAttributes  To set attributes for the existing topic
+  - sns:DeleteTopic         To delete the existing topic
+  - sns:Publish             To be able to create/subscribe notification on existing topic
 
 - HTTP endpoint
 
@@ -340,6 +360,7 @@ The response has the following format:
 - TopicArn: topic `ARN
   <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
 - OpaqueData: The opaque data set on the topic.
+- Policy: Any access permission set on the topic.
 
 Get Topic Information
 `````````````````````
@@ -393,6 +414,7 @@ The response has the following format:
 - TopicArn: topic `ARN
   <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
 - OpaqueData: the opaque data set on the topic.
+- Policy: Any access permission set on the topic.
 
 Delete Topic
 ````````````
@@ -463,6 +485,61 @@ The response has the following format:
   topic, the request must be made over HTTPS. The "topic list" request will
   otherwise be rejected.
 
+Set Topic Attributes
+````````````````````
+
+::
+
+   POST
+
+   Action=SetTopicAttributes
+   &TopicArn=<topic-arn>&AttributeName=<attribute-name>&AttributeValue=<attribute-value>
+
+This allows to set/modify existing attributes on the specified topic.
+
+.. note::
+
+  - The AttributeName passed will either be updated or created (if not exist) with AttributeValue passed.
+  - Any unsupported AttributeName passed will result in error 400.
+
+The response has the following format:
+
+::
+
+    <SetTopicAttributesResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </SetTopicAttributesResponse>
+
+Valid AttributeName that can be passed:
+
+  - push-endpoint: This is the URI of an endpoint to send push notifications to.
+  - OpaqueData: Opaque data is set in the topic configuration and added to all
+    notifications that are triggered by the topic.
+  - persistent: This indicates whether notifications to this endpoint are
+    persistent (=asynchronous) or not persistent. (This is "false" by default.)
+  - time_to_live: This will limit the time (in seconds) to retain the notifications.
+  - max_retries: This will limit the max retries before expiring notifications.
+  - retry_sleep_duration: This will control the frequency of retrying the notifications.
+  - Policy: This will control who can access the topic other than owner of the topic.
+  - verify-ssl: This indicates whether the server certificates must be validated by
+    the client. This is "true" by default.
+  - ``use-ssl``: If this is set to "true", a secure connection is used to
+    connect to the broker. This is "false" by default.
+  - cloudevents: This indicates whether the HTTP header should contain
+    attributes according to the `S3 CloudEvents Spec`_. 
+  - amqp-exchange: The exchanges must exist and must be able to route messages
+    based on topics.
+  - amqp-ack-level: No end2end acknowledgement is required. Messages may persist in the
+    broker before being delivered to their final destinations. 
+  - ``ca-location``: If this is provided and a secure connection is used, the
+    specified CA will be used instead of the default CA to authenticate the
+    broker. 
+  - mechanism: may be provided together with user/password (default: ``PLAIN``).
+  - kafka-ack-level: No end2end acknowledgement is required. Messages may persist in the
+    broker before being delivered to their final destinations. 
+
 Notifications
 ~~~~~~~~~~~~~
 
diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 8928e853e64d..37453605260b 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3850,3 +3850,18 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: mandatory_topic_permissions
+  type: bool
+  level: basic
+  desc: Whether to validate user permissions to access notification topics.
+  long_desc: If true, all users (other then the owner of the topic) will need
+    to have a policy to access topics.
+    The topic policy can be set by owner via CreateTopic() or SetTopicAttribute().
+    Following permissions can be granted "sns:Publish", "sns:GetTopicAttributes",
+    "sns:SetTopicAttributes" and "sns:DeleteTopic" via Policy.
+    NOTE that even if set to "false" topics will still follow the policies if set on them.
+  default: false
+  services:
+  - rgw
+  with_legacy: true
+
diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index a2def87040ef..d06c83813382 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -482,6 +482,7 @@ bool is_non_s3_op(RGWOpType op_type)
       op_type == RGW_OP_PUBSUB_TOPIC_CREATE ||
       op_type == RGW_OP_PUBSUB_TOPICS_LIST ||
       op_type == RGW_OP_PUBSUB_TOPIC_GET ||
+      op_type == RGW_OP_PUBSUB_TOPIC_SET ||
       op_type == RGW_OP_PUBSUB_TOPIC_DELETE ||
       op_type == RGW_OP_TAG_ROLE ||
       op_type == RGW_OP_LIST_ROLE_TAGS ||
diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc
index 35aeb15fcdc5..0bfcccff638e 100644
--- a/src/rgw/rgw_iam_policy.cc
+++ b/src/rgw/rgw_iam_policy.cc
@@ -157,6 +157,10 @@ static const actpair actpairs[] =
  { "sts:AssumeRoleWithWebIdentity", stsAssumeRoleWithWebIdentity},
  { "sts:GetSessionToken", stsGetSessionToken},
  { "sts:TagSession", stsTagSession},
+ { "sns:GetTopicAttributes", snsGetTopicAttributes},
+ { "sns:DeleteTopic", snsDeleteTopic},
+ { "sns:Publish", snsPublish},
+ { "sns:SetTopicAttributes", snsSetTopicAttributes},
 };
 
 struct PolicyParser;
@@ -600,6 +604,12 @@ bool ParseState::do_string(CephContext* cct, const char* s, size_t l) {
         if ((t->notaction & stsAllValue) == stsAllValue) {
           t->notaction[stsAll] = 1;
         }
+        if ((t->action & snsAllValue) == snsAllValue) {
+          t->action[snsAll] = 1;
+        }
+        if ((t->notaction & snsAllValue) == snsAllValue) {
+          t->notaction[snsAll] = 1;
+        }
       }
     }
   } else if (w->id == TokenID::Resource || w->id == TokenID::NotResource) {
@@ -1452,6 +1462,18 @@ const char* action_bit_string(uint64_t action) {
 
   case stsTagSession:
     return "sts:TagSession";
+
+  case snsSetTopicAttributes:
+    return "sns:SetTopicAttributes";
+
+  case snsGetTopicAttributes:
+    return "sns:GetTopicAttributes";
+
+  case snsDeleteTopic:
+    return "sns:DeleteTopic";
+
+  case snsPublish:
+    return "sns:Publish";
   }
   return "s3Invalid";
 }
diff --git a/src/rgw/rgw_iam_policy.h b/src/rgw/rgw_iam_policy.h
index c60872850a30..e528d1515c77 100644
--- a/src/rgw/rgw_iam_policy.h
+++ b/src/rgw/rgw_iam_policy.h
@@ -141,8 +141,14 @@ static constexpr std::uint64_t stsGetSessionToken = iamAll + 3;
 static constexpr std::uint64_t stsTagSession = iamAll + 4;
 static constexpr std::uint64_t stsAll = iamAll + 5;
 
+static constexpr std::uint64_t snsGetTopicAttributes = stsAll + 1;
+static constexpr std::uint64_t snsDeleteTopic = stsAll + 2;
+static constexpr std::uint64_t snsPublish = stsAll + 3;
+static constexpr std::uint64_t snsSetTopicAttributes = stsAll + 4;
+static constexpr std::uint64_t snsAll = stsAll + 5;
+
 static constexpr std::uint64_t s3Count = s3All;
-static constexpr std::uint64_t allCount = stsAll + 1;
+static constexpr std::uint64_t allCount = snsAll + 1;
 
 using Action_t = std::bitset<allCount>;
 using NotAction_t = Action_t;
@@ -164,6 +170,7 @@ static const Action_t None(0);
 static const Action_t s3AllValue = set_cont_bits<allCount>(0,s3All);
 static const Action_t iamAllValue = set_cont_bits<allCount>(s3All+1,iamAll);
 static const Action_t stsAllValue = set_cont_bits<allCount>(iamAll+1,stsAll);
+static const Action_t snsAllValue = set_cont_bits<allCount>(stsAll + 1, snsAll);
 static const Action_t allValue = set_cont_bits<allCount>(0,allCount);
 
 namespace {
diff --git a/src/rgw/rgw_op_type.h b/src/rgw/rgw_op_type.h
index 375c7348b248..a7a68d4ade5e 100644
--- a/src/rgw/rgw_op_type.h
+++ b/src/rgw/rgw_op_type.h
@@ -104,6 +104,7 @@ enum RGWOpType {
   RGW_OP_PUBSUB_TOPIC_CREATE,
   RGW_OP_PUBSUB_TOPICS_LIST,
   RGW_OP_PUBSUB_TOPIC_GET,
+  RGW_OP_PUBSUB_TOPIC_SET,
   RGW_OP_PUBSUB_TOPIC_DELETE,
   RGW_OP_PUBSUB_SUB_CREATE,
   RGW_OP_PUBSUB_SUB_GET,
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 51c8b0281c68..14d85abe3b62 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -337,6 +337,7 @@ void rgw_pubsub_topic::dump(Formatter *f) const
   encode_json("dest", dest, f);
   encode_json("arn", arn, f);
   encode_json("opaqueData", opaque_data, f);
+  encode_json("policy", policy_text, f);
 }
 
 void rgw_pubsub_topic::dump_xml(Formatter *f) const
@@ -346,6 +347,7 @@ void rgw_pubsub_topic::dump_xml(Formatter *f) const
   encode_xml("EndPoint", dest, f);
   encode_xml("TopicArn", arn, f);
   encode_xml("OpaqueData", opaque_data, f);
+  encode_xml("Policy", policy_text, f);
 }
 
 void encode_xml_key_value_entry(const std::string& key, const std::string& value, Formatter *f) {
@@ -365,6 +367,7 @@ void rgw_pubsub_topic::dump_xml_as_attributes(Formatter *f) const
   encode_xml_key_value_entry("EndPoint", dest.to_json_str(), f);
   encode_xml_key_value_entry("TopicArn", arn, f);
   encode_xml_key_value_entry("OpaqueData", opaque_data, f);
+  encode_xml_key_value_entry("Policy", policy_text, f);
   f->close_section(); // Attributes
 }
 
@@ -695,7 +698,9 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const std::string& name,
                             const rgw_pubsub_dest& dest, const std::string& arn,
                             const std::string& opaque_data,
-                            const rgw_user& user, optional_yield y) const {
+                            const rgw_user& user,
+                            const std::string& policy_text,
+                            optional_yield y) const {
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topics topics;
 
@@ -712,6 +717,7 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
   new_topic.dest = dest;
   new_topic.arn = arn;
   new_topic.opaque_data = opaque_data;
+  new_topic.policy_text = policy_text;
 
   ret = write_topics(dpp, topics, &objv_tracker, y);
   if (ret < 0) {
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index 3a87d777d6cb..ddc72f99b076 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -398,19 +398,21 @@ struct rgw_pubsub_topic {
   rgw_pubsub_dest dest;
   std::string arn;
   std::string opaque_data;
+  std::string policy_text;
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(3, 1, bl);
+    ENCODE_START(4, 1, bl);
     encode(user, bl);
     encode(name, bl);
     encode(dest, bl);
     encode(arn, bl);
     encode(opaque_data, bl);
+    encode(policy_text, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(3, bl);
+    DECODE_START(4, bl);
     decode(user, bl);
     decode(name, bl);
     if (struct_v >= 2) {
@@ -420,6 +422,9 @@ struct rgw_pubsub_topic {
     if (struct_v >= 3) {
       decode(opaque_data, bl);
     }
+    if (struct_v >= 4) {
+      decode(policy_text, bl);
+    }
     DECODE_FINISH(bl);
   }
 
@@ -629,7 +634,7 @@ class RGWPubSub
   int create_topic(const DoutPrefixProvider* dpp, const std::string& name,
                    const rgw_pubsub_dest& dest, const std::string& arn,
                    const std::string& opaque_data, const rgw_user& user,
-                   optional_yield y) const;
+                   const std::string& policy_text, optional_yield y) const;
   // remove a topic according to its name
   // if the topic does not exists it is a no-op (considered success)
   // return 0 on success, error code otherwise
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 9bc7a2897632..da81711511c3 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -67,6 +67,53 @@ bool topics_has_endpoint_secret(const rgw_pubsub_topics& topics) {
     return false;
 }
 
+std::optional<rgw::IAM::Policy> get_policy_from_text(req_state* const s,
+                                                     std::string& policy_text) {
+  const auto bl = bufferlist::static_from_string(policy_text);
+  try {
+    return rgw::IAM::Policy(
+        s->cct, s->owner.get_id().tenant, bl,
+        s->cct->_conf.get_val<bool>("rgw_policy_reject_invalid_principals"));
+  } catch (rgw::IAM::PolicyParseException& e) {
+    ldout(s->cct, 1) << "failed to parse policy:' " << policy_text
+                     << " ' with error: " << e.what() << dendl;
+    s->err.message = e.what();
+    return std::nullopt;
+  }
+}
+
+int verify_topic_owner_or_policy(req_state* const s,
+                                 const rgw_pubsub_topic& topic,
+                                 const std::string& zonegroup_name,
+                                 const uint64_t op) {
+  if (topic.user == s->owner.get_id()) {
+    return 0;
+  }
+  // no policy set.
+  if (topic.policy_text.empty()) {
+    // if mandatory_topic_permissions is true, then validate all users for
+    // permission.
+    if (s->cct->_conf->mandatory_topic_permissions) {
+      return -EACCES;
+    } else {
+      return 0;
+    }
+  }
+  // bufferlist::static_from_string wants non const string
+  std::string policy_text(topic.policy_text);
+  const auto p = get_policy_from_text(s, policy_text);
+  rgw::IAM::PolicyPrincipal princ_type = rgw::IAM::PolicyPrincipal::Other;
+  const rgw::ARN arn(rgw::Partition::aws, rgw::Service::sns, zonegroup_name,
+                     s->user->get_tenant(), topic.name);
+  if (!p || p->eval(s->env, *s->auth.identity, op, arn, princ_type) !=
+                rgw::IAM::Effect::Allow) {
+    ldout(s->cct, 1) << "topic_policy failed validation, topic_policy: " << p
+                     << dendl;
+    return -EACCES;
+  }
+  return 0;
+}
+
 // command (AWS compliant): 
 // POST
 // Action=CreateTopic&Name=<topic-name>[&OpaqueData=data][&push-endpoint=<endpoint>[&persistent][&<arg1>=<value1>]]
@@ -76,7 +123,8 @@ class RGWPSCreateTopicOp : public RGWOp {
   rgw_pubsub_dest dest;
   std::string topic_arn;
   std::string opaque_data;
-  
+  std::string policy_text;
+
   int get_params() {
     topic_name = s->info.args.get("Name");
     if (topic_name.empty()) {
@@ -84,17 +132,32 @@ class RGWPSCreateTopicOp : public RGWOp {
       return -EINVAL;
     }
 
+    // Remove the args that are parsed, so the push_endpoint_args only contains
+    // necessary one's.
     opaque_data = s->info.args.get("OpaqueData");
+    s->info.args.remove("OpaqueData");
 
     dest.push_endpoint = s->info.args.get("push-endpoint");
+    s->info.args.remove("push-endpoint");
     s->info.args.get_bool("persistent", &dest.persistent, false);
+    s->info.args.remove("persistent");
     s->info.args.get_int("time_to_live", reinterpret_cast<int *>(&dest.time_to_live), rgw::notify::DEFAULT_GLOBAL_VALUE);
+    s->info.args.remove("time_to_live");
     s->info.args.get_int("max_retries", reinterpret_cast<int *>(&dest.max_retries), rgw::notify::DEFAULT_GLOBAL_VALUE);
+    s->info.args.remove("max_retries");
     s->info.args.get_int("retry_sleep_duration", reinterpret_cast<int *>(&dest.retry_sleep_duration), rgw::notify::DEFAULT_GLOBAL_VALUE);
+    s->info.args.remove("retry_sleep_duration");
 
     if (!validate_and_update_endpoint_secret(dest, s->cct, *(s->info.env))) {
       return -EINVAL;
     }
+    // Store topic Policy.
+    policy_text = url_decode(s->info.args.get("Policy"), true);
+    if (!policy_text.empty() && !get_policy_from_text(s, policy_text)) {
+      return -ERR_MALFORMED_DOC;
+    }
+    s->info.args.remove("Policy");
+
     for (const auto& param : s->info.args.get_params()) {
       if (param.first == "Action" || param.first == "Name" || param.first == "PayloadHash") {
         continue;
@@ -106,14 +169,7 @@ class RGWPSCreateTopicOp : public RGWOp {
       // remove last separator
       dest.push_endpoint_args.pop_back();
     }
-    if (!dest.push_endpoint.empty() && dest.persistent) {
-      const auto ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
-      if (ret < 0) {
-        ldpp_dout(this, 1) << "CreateTopic Action failed to create queue for persistent topics. error:" << ret << dendl;
-        return ret;
-      }
-    }
-    
+
     // dest object only stores endpoint info
     dest.arn_topic = topic_name;
     // the topic ARN will be sent in the reply
@@ -125,9 +181,32 @@ class RGWPSCreateTopicOp : public RGWOp {
   }
 
   public:
-  int verify_permission(optional_yield) override {
-    return 0;
-  }
+   int verify_permission(optional_yield y) override {
+    auto ret = get_params();
+    if (ret < 0) {
+      return ret;
+    }
+
+    const RGWPubSub ps(driver, s->owner.get_id().tenant);
+    rgw_pubsub_topic result;
+    ret = ps.get_topic(this, topic_name, result, y);
+    if (ret == -ENOENT) {
+      // topic not present
+      return 0;
+    }
+    if (ret == 0) {
+      if (result.user == s->owner.get_id() ||
+          !s->cct->_conf->mandatory_topic_permissions) {
+        return 0;
+      }
+      ldpp_dout(this, 1) << "failed to create topic '" << topic_name
+                         << "', topic already exist." << dendl;
+      return -EPERM;
+    }
+    ldpp_dout(this, 1) << "failed to read topic '" << topic_name
+                       << "', with error:" << ret << dendl;
+    return ret;
+   }
 
   void pre_exec() override {
     rgw_bucket_object_pre_exec(s);
@@ -163,14 +242,18 @@ class RGWPSCreateTopicOp : public RGWOp {
 };
 
 void RGWPSCreateTopicOp::execute(optional_yield y) {
-  op_ret = get_params();
-  if (op_ret < 0) {
-    return;
+  if (!dest.push_endpoint.empty() && dest.persistent) {
+    op_ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "CreateTopic Action failed to create queue for "
+                            "persistent topics. error:"
+                         << op_ret << dendl;
+      return;
+    }
   }
-
   const RGWPubSub ps(driver, s->owner.get_id().tenant);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
-                           s->owner.get_id(), y);
+                           s->owner.get_id(), policy_text, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to create topic '" << topic_name << "', ret=" << op_ret << dendl;
     return;
@@ -236,6 +319,15 @@ void RGWPSListTopicsOp::execute(optional_yield y) {
     op_ret = -EPERM;
     return;
   }
+  for (auto it = result.topics.cbegin(); it != result.topics.cend();) {
+    if (verify_topic_owner_or_policy(
+            s, it->second, driver->get_zone()->get_zonegroup().get_name(),
+            rgw::IAM::snsGetTopicAttributes) != 0) {
+      result.topics.erase(it++);
+    } else {
+      ++it;
+    }
+  }
   ldpp_dout(this, 20) << "successfully got topics" << dendl;
 }
 
@@ -312,6 +404,14 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
     op_ret = -EPERM;
     return;
   }
+  op_ret = verify_topic_owner_or_policy(
+      s, result, driver->get_zone()->get_zonegroup().get_name(),
+      rgw::IAM::snsGetTopicAttributes);
+  if (op_ret != 0) {
+    ldpp_dout(this, 1) << "failed to get topic '" << topic_name
+                       << "', topic owned by other user" << dendl;
+    return;
+  }
   ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
 }
 
@@ -388,9 +488,202 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
     op_ret = -EPERM;
     return;
   }
+  op_ret = verify_topic_owner_or_policy(
+      s, result, driver->get_zone()->get_zonegroup().get_name(),
+      rgw::IAM::snsGetTopicAttributes);
+  if (op_ret != 0) {
+    ldpp_dout(this, 1) << "failed to get topic '" << topic_name
+                       << "', topic owned by other user" << dendl;
+    return;
+  }
   ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
 }
 
+// command (AWS compliant):
+// POST
+// Action=SetTopicAttributes&TopicArn=<topic-arn>&AttributeName=<attribute-name>&AttributeValue=<attribute-value>
+class RGWPSSetTopicAttributesOp : public RGWOp {
+ private:
+  std::string topic_name;
+  std::string topic_arn;
+  std::string opaque_data;
+  std::string policy_text;
+  rgw_pubsub_dest dest;
+  rgw_user topic_owner;
+  std::string attribute_name;
+
+  int get_params() {
+    const auto arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
+
+    if (!arn || arn->resource.empty()) {
+      ldpp_dout(this, 1) << "SetTopicAttribute Action 'TopicArn' argument is "
+                            "missing or invalid"
+                         << dendl;
+      return -EINVAL;
+    }
+    topic_arn = arn->to_string();
+    topic_name = arn->resource;
+    attribute_name = s->info.args.get("AttributeName");
+    if (attribute_name.empty()) {
+      ldpp_dout(this, 1)
+          << "SetTopicAttribute Action 'AttributeName' argument is "
+             "missing or invalid"
+          << dendl;
+      return -EINVAL;
+    }
+    return 0;
+  }
+
+  int map_attributes(const rgw_pubsub_topic& topic) {
+    // update the default values that is stored in topic currently.
+    opaque_data = topic.opaque_data;
+    policy_text = topic.policy_text;
+    dest = topic.dest;
+
+    if (attribute_name == "OpaqueData") {
+      opaque_data = s->info.args.get("AttributeValue");
+    } else if (attribute_name == "persistent") {
+      s->info.args.get_bool("AttributeValue", &dest.persistent, false);
+    } else if (attribute_name == "time_to_live") {
+      s->info.args.get_int("AttributeValue",
+                           reinterpret_cast<int*>(&dest.time_to_live),
+                           rgw::notify::DEFAULT_GLOBAL_VALUE);
+    } else if (attribute_name == "max_retries") {
+      s->info.args.get_int("AttributeValue",
+                           reinterpret_cast<int*>(&dest.max_retries),
+                           rgw::notify::DEFAULT_GLOBAL_VALUE);
+    } else if (attribute_name == "retry_sleep_duration") {
+      s->info.args.get_int("AttributeValue",
+                           reinterpret_cast<int*>(&dest.retry_sleep_duration),
+                           rgw::notify::DEFAULT_GLOBAL_VALUE);
+    } else if (attribute_name == "push-endpoint") {
+      dest.push_endpoint = s->info.args.get("AttributeValue");
+      if (!validate_and_update_endpoint_secret(dest, s->cct, *(s->info.env))) {
+        return -EINVAL;
+      }
+    } else if (attribute_name == "Policy") {
+      policy_text = url_decode(s->info.args.get("AttributeValue"), true);
+      if (!policy_text.empty() && !get_policy_from_text(s, policy_text)) {
+        return -ERR_MALFORMED_DOC;
+      }
+    } else {
+      // replace the push_endpoint_args if passed in SetAttribute.
+      const auto replace_str = [&](const std::string& param,
+                                   const std::string& val) {
+        auto& push_endpoint_args = dest.push_endpoint_args;
+        const std::string replaced_str = param + "=" + val;
+        const auto pos = push_endpoint_args.find(param);
+        if (pos == std::string::npos) {
+          dest.push_endpoint_args.append("&" + replaced_str);
+          return;
+        }
+        auto end_pos = dest.push_endpoint_args.find("&", pos);
+        end_pos = end_pos == std::string::npos ? push_endpoint_args.length()
+                                               : end_pos;
+        push_endpoint_args.replace(pos, end_pos - pos, replaced_str);
+      };
+      const std::unordered_set<std::string> push_endpoint_args = {
+          "verify-ssl",    "use-ssl",         "ca-location", "amqp-ack-level",
+          "amqp-exchange", "kafka-ack-level", "mechanism",   "cloudevents"};
+      if (push_endpoint_args.count(attribute_name) == 1) {
+        replace_str(attribute_name, s->info.args.get("AttributeValue"));
+        return 0;
+      }
+      ldpp_dout(this, 1)
+          << "SetTopicAttribute Action 'AttributeName' argument is "
+             "invalid: 'AttributeName' = "
+          << attribute_name << dendl;
+      return -EINVAL;
+    }
+    return 0;
+  }
+
+ public:
+  int verify_permission(optional_yield y) override {
+    auto ret = get_params();
+    if (ret < 0) {
+      return ret;
+    }
+    rgw_pubsub_topic result;
+    const RGWPubSub ps(driver, s->owner.get_id().tenant);
+    ret = ps.get_topic(this, topic_name, result, y);
+    if (ret < 0) {
+      ldpp_dout(this, 1) << "failed to get topic '" << topic_name
+                         << "', ret=" << ret << dendl;
+      return ret;
+    }
+    topic_owner = result.user;
+    ret = verify_topic_owner_or_policy(
+        s, result, driver->get_zone()->get_zonegroup().get_name(),
+        rgw::IAM::snsSetTopicAttributes);
+    if (ret != 0) {
+      ldpp_dout(this, 1) << "failed to set attributes for topic '" << topic_name
+                         << "', topic owned by other user" << dendl;
+      return ret;
+    }
+
+    return map_attributes(result);
+  }
+
+  void pre_exec() override { rgw_bucket_object_pre_exec(s); }
+  void execute(optional_yield) override;
+
+  const char* name() const override { return "pubsub_topic_set"; }
+  RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_SET; }
+  uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
+
+  void send_response() override {
+    if (op_ret) {
+      set_req_state_err(s, op_ret);
+    }
+    dump_errno(s);
+    end_header(s, this, "application/xml");
+
+    if (op_ret < 0) {
+      return;
+    }
+
+    const auto f = s->formatter;
+    f->open_object_section_in_ns("SetTopicAttributesResponse", AWS_SNS_NS);
+    f->open_object_section("ResponseMetadata");
+    encode_xml("RequestId", s->req_id, f);
+    f->close_section();  // ResponseMetadata
+    f->close_section();  // SetTopicAttributesResponse
+    rgw_flush_formatter_and_reset(s, f);
+  }
+};
+
+void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
+  if (!dest.push_endpoint.empty() && dest.persistent) {
+    op_ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1)
+          << "SetTopicAttributes Action failed to create queue for "
+             "persistent topics. error:"
+          << op_ret << dendl;
+      return;
+    }
+  } else {  // changing the persistent topic to non-persistent.
+    op_ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
+    if (op_ret != -ENOENT && op_ret < 0) {
+      ldpp_dout(this, 1) << "SetTopicAttributes Action failed to remove queue "
+                            "for persistent topics. error:"
+                         << op_ret << dendl;
+      return;
+    }
+  }
+  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
+                           topic_owner, policy_text, y);
+  if (op_ret < 0) {
+    ldpp_dout(this, 1) << "failed to SetAttributes for topic '" << topic_name
+                       << "', ret=" << op_ret << dendl;
+    return;
+  }
+  ldpp_dout(this, 20) << "successfully set the attributes for topic '"
+                      << topic_name << "'" << dendl;
+}
+
 // command (AWS compliant): 
 // POST
 // Action=DeleteTopic&TopicArn=<topic-arn>
@@ -407,19 +700,6 @@ class RGWPSDeleteTopicOp : public RGWOp {
     }
 
     topic_name = topic_arn->resource;
-
-    // upon deletion it is not known if topic is persistent or not
-    // will try to delete the persistent topic anyway
-    const auto ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
-    if (ret == -ENOENT) {
-      // topic was not persistent, or already deleted
-      return 0;
-    }
-    if (ret < 0) {
-      ldpp_dout(this, 1) << "DeleteTopic Action failed to remove queue for persistent topics. error:" << ret << dendl;
-      return ret;
-    }
-
     return 0;
   }
 
@@ -463,6 +743,35 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
     return;
   }
   const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  rgw_pubsub_topic result;
+  op_ret = ps.get_topic(this, topic_name, result, y);
+  if (op_ret == 0) {
+    op_ret = verify_topic_owner_or_policy(
+        s, result, driver->get_zone()->get_zonegroup().get_name(),
+        rgw::IAM::snsDeleteTopic);
+    if (op_ret != 0) {
+      ldpp_dout(this, 1) << "failed to remove topic '" << topic_name
+                         << "' topic owned by other user" << dendl;
+      return;
+    }
+  } else {
+    ldpp_dout(this, 1) << "failed to fetch topic '" << topic_name
+                       << "' with error: " << op_ret << dendl;
+    if (op_ret == -ENOENT) {
+      // its not an error if no topics exist, just a no-op
+      op_ret = 0;
+    }
+    return;
+  }
+  // upon deletion it is not known if topic is persistent or not
+  // will try to delete the persistent topic anyway
+  op_ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
+  if (op_ret != -ENOENT && op_ret < 0) {
+    ldpp_dout(this, 1) << "DeleteTopic Action failed to remove queue for "
+                          "persistent topics. error:"
+                       << op_ret << dendl;
+    return;
+  }
   op_ret = ps.remove_topic(this, topic_name, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to remove topic '" << topic_name << ", ret=" << op_ret << dendl;
@@ -473,12 +782,14 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
 
 using op_generator = RGWOp*(*)();
 static const std::unordered_map<std::string, op_generator> op_generators = {
-  {"CreateTopic", []() -> RGWOp* {return new RGWPSCreateTopicOp;}},
-  {"DeleteTopic", []() -> RGWOp* {return new RGWPSDeleteTopicOp;}},
-  {"ListTopics", []() -> RGWOp* {return new RGWPSListTopicsOp;}},
-  {"GetTopic", []() -> RGWOp* {return new RGWPSGetTopicOp;}},
-  {"GetTopicAttributes", []() -> RGWOp* {return new RGWPSGetTopicAttributesOp;}}
-};
+    {"CreateTopic", []() -> RGWOp* { return new RGWPSCreateTopicOp; }},
+    {"DeleteTopic", []() -> RGWOp* { return new RGWPSDeleteTopicOp; }},
+    {"ListTopics", []() -> RGWOp* { return new RGWPSListTopicsOp; }},
+    {"GetTopic", []() -> RGWOp* { return new RGWPSGetTopicOp; }},
+    {"GetTopicAttributes",
+     []() -> RGWOp* { return new RGWPSGetTopicAttributesOp; }},
+    {"SetTopicAttributes",
+     []() -> RGWOp* { return new RGWPSSetTopicAttributesOp; }}};
 
 bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s) 
 {
@@ -710,6 +1021,15 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
       return;
     }
+    op_ret = verify_topic_owner_or_policy(
+        s, topic_info, driver->get_zone()->get_zonegroup().get_name(),
+        rgw::IAM::snsPublish);
+    if (op_ret != 0) {
+      ldpp_dout(this, 1) << "failed to create notification for topic '"
+                         << topic_name << "' topic owned by other user"
+                         << dendl;
+      return;
+    }
     // make sure that full topic configuration match
     // TODO: use ARN match function
     
@@ -722,7 +1042,7 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     // ARN is cached to make the "GET" method faster
     op_ret = ps.create_topic(this, unique_topic_name, topic_info.dest,
                              topic_info.arn, topic_info.opaque_data,
-                             s->owner.get_id(), y);
+                             s->owner.get_id(), topic_info.policy_text, y);
     if (op_ret < 0) {
       ldpp_dout(this, 1) << "failed to auto-generate unique topic '" << unique_topic_name << 
         "', ret=" << op_ret << dendl;
diff --git a/src/test/rgw/bucket_notification/api.py b/src/test/rgw/bucket_notification/api.py
index fe38576fb351..30fce41a1bb0 100644
--- a/src/test/rgw/bucket_notification/api.py
+++ b/src/test/rgw/bucket_notification/api.py
@@ -78,7 +78,7 @@ class PSTopicS3:
     POST ?Action=GetTopic&TopicArn=<topic-arn>
     POST ?Action=DeleteTopic&TopicArn=<topic-arn>
     """
-    def __init__(self, conn, topic_name, region, endpoint_args=None, opaque_data=None):
+    def __init__(self, conn, topic_name, region, endpoint_args=None, opaque_data=None, policy_text=None):
         self.conn = conn
         self.topic_name = topic_name.strip()
         assert self.topic_name
@@ -88,6 +88,8 @@ def __init__(self, conn, topic_name, region, endpoint_args=None, opaque_data=Non
             self.attributes = {nvp[0] : nvp[1] for nvp in urlparse.parse_qsl(endpoint_args, keep_blank_values=True)}
         if opaque_data is not None:
             self.attributes['OpaqueData'] = opaque_data
+        if policy_text is not None:
+            self.attributes['Policy'] = policy_text
         protocol = 'https' if conn.is_secure else 'http'
         self.client = boto3.client('sns',
                            endpoint_url=protocol+'://'+conn.host+':'+str(conn.port),
@@ -96,9 +98,9 @@ def __init__(self, conn, topic_name, region, endpoint_args=None, opaque_data=Non
                            region_name=region,
                            verify='./cert.pem')
 
-    def get_config(self):
+    def get_config(self, topic_arn=None):
         """get topic info"""
-        parameters = {'Action': 'GetTopic', 'TopicArn': self.topic_arn}
+        parameters = {'Action': 'GetTopic', 'TopicArn': (topic_arn if topic_arn is not None else self.topic_arn)}
         body = urlparse.urlencode(parameters)
         string_date = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
         content_type = 'application/x-www-form-urlencoded; charset=utf-8'
@@ -131,6 +133,13 @@ def set_config(self):
         result = self.client.create_topic(Name=self.topic_name, Attributes=self.attributes)
         self.topic_arn = result['TopicArn']
         return self.topic_arn
+    
+    def set_attributes(self, attribute_name, attribute_val, topic_arn=None):
+        """set topic attributes."""
+        result = self.client.set_topic_attributes(TopicArn=(
+            topic_arn if topic_arn is not None else self.topic_arn), AttributeName=attribute_name, AttributeValue=attribute_val)
+        return result['ResponseMetadata']['HTTPStatusCode']
+
 
     def del_config(self, topic_arn=None):
         """delete topic"""
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index ffb8e488bbff..f5017885493a 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -4283,6 +4283,76 @@ def test_ps_s3_multiple_topics_notification():
     conn.delete_bucket(bucket_name)
     http_server.close()
 
+@attr('basic_test')
+def test_ps_s3_topic_permissions():
+    """ test s3 topic set/get/delete permissions """
+    conn1 = connection()
+    conn2 = another_user()
+    zonegroup = 'default'
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name + TOPIC_SUFFIX
+    topic_policy = json.dumps({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "Statement",
+                "Effect": "Deny",
+                "Principal": "*",
+                "Action": ["sns:Publish", "sns:SetTopicAttributes", "sns:GetTopicAttributes"],
+                "Resource": f"arn:aws:sns:{zonegroup}::{topic_name}"
+            }
+        ]
+    })
+    # create s3 topic with DENY policy
+    endpoint_address = 'amqp://127.0.0.1:7001'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args, policy_text=topic_policy)
+    topic_arn = topic_conf.set_config()
+
+    # 2nd user tries to fetch the topic
+    topic_conf2 = PSTopicS3(conn2, topic_name, zonegroup, endpoint_args=endpoint_args)
+    _, status = topic_conf2.get_config(topic_arn=topic_arn)
+    assert_equal(status, 403)
+    try:
+        # 2nd user tries to set the attribute
+        status = topic_conf2.set_attributes(attribute_name="persistent", attribute_val="false", topic_arn=topic_arn)
+        assert False, "'AccessDenied' error is expected"
+    except Exception as err:
+        print(err)
+
+    # create bucket for conn2 and try publishing notification to topic
+    _ = conn2.create_bucket(bucket_name)
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
+                         'Events': []
+                       }]
+    try:
+        s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
+        _, status = s3_notification_conf2.set_config()
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as error:
+        assert_equal(error.response['Error']['Code'], 'AccessDenied')
+
+    # Topic policy is now added by the 1st user to allow 2nd user.
+    topic_policy  = topic_policy.replace("Deny", "Allow")
+    topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args, policy_text=topic_policy)
+    topic_arn = topic_conf.set_config()
+    # 2nd user try to fetch topic again
+    _, status = topic_conf2.get_config(topic_arn=topic_arn)
+    assert_equal(status, 200)
+    # 2nd user tries to set the attribute again
+    status = topic_conf2.set_attributes(attribute_name="persistent", attribute_val="false", topic_arn=topic_arn)
+    assert_equal(status, 200)
+    # 2nd user tries to publish notification again
+    s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf2.set_config()
+    assert_equal(status, 200)
+
+    # cleanup
+    s3_notification_conf2.del_config()
+    topic_conf.del_config()
+    # delete the bucket
+    conn2.delete_bucket(bucket_name)
 
 def kafka_security(security_type, mechanism='PLAIN'):
     """ test pushing kafka s3 notification securly to master """
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index f4c3c6aff6f1..c630646c9128 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -94,6 +94,7 @@ using rgw::IAM::iamCreateRole;
 using rgw::IAM::iamDeleteRole;
 using rgw::IAM::iamAll;
 using rgw::IAM::stsAll;
+using rgw::IAM::snsAll;
 using rgw::IAM::allCount;
 
 class FakeIdentity : public Identity {
@@ -642,7 +643,7 @@ TEST_F(PolicyTest, Parse6) {
   EXPECT_TRUE(p->statements[0].noprinc.empty());
   EXPECT_EQ(p->statements[0].effect, Effect::Allow);
   Action_t act;
-  for (auto i = 0U; i <= stsAll; i++)
+  for (auto i = 0U; i <= snsAll; i++)
     act[i] = 1;
   EXPECT_EQ(p->statements[0].action, act);
   EXPECT_EQ(p->statements[0].notaction, None);

From 9999249876d4a4c5ab8f4c8a204442df2902fc36 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 23 Oct 2023 18:38:18 +0300
Subject: [PATCH 0273/2492] osd/scrub: extend scrub reservation timeout

As replicas are now reserved sequentially, the scrub reservation
timeout should be extended.

Taking into account the low priority of scrub-related messages,
modify both 'osd_scrub_reservation_timeout' and
'osd_scrub_slow_reservation_response'.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/options/osd.yaml.in   |  8 ++++----
 src/osd/scrubber/scrub_machine.h | 16 +++++++++++++---
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 5d8d40cf12d1..198224423585 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -511,9 +511,9 @@ options:
   type: millisecs
   level: advanced
   desc: Maximum wait (milliseconds) for scrub reservations before issuing a cluster-log warning
-  long_desc: Waiting too long for a replica to respond to scrub resource reservation request
-   (after at least half of the replicas have responded). Disable by setting to a very large value.
-  default: 2200
+  long_desc: Waiting too long for a replica to respond to scrub resource reservation request.
+    Disable by setting to a very large value.
+  default: 30000
   min: 500
   see_also:
   - osd_scrub_reservation_timeout
@@ -525,7 +525,7 @@ options:
   long_desc: Maximum wait (milliseconds) for all replicas to respond to
     scrub reservation requests, before the scrub session is aborted. Disable by setting
     to a very large value.
-  default: 5000
+  default: 300000
   min: 2000
   see_also:
   - osd_scrub_slow_reservation_response
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 2f73cbbefb5d..cbce07fe183c 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -164,9 +164,19 @@ MEV(SchedReplica)
 /// that is in-flight to the local ObjectStore
 MEV(ReplicaPushesUpd)
 
-/// a new interval has dawned.
-/// For a Primary: Discards replica reservations, so that the FullReset that would
-/// follow it would not attempt to release them.
+/**
+ * IntervalChanged
+ *
+ * This event notifies the ScrubMachine that it is no longer responsible for
+ * releasing replica state.  It will generally be submitted upon a PG interval
+ * change.
+ *
+ * This event is distinct from FullReset because replicas are always responsible
+ * for releasing any interval specific state (including but certainly not limited to
+ * scrub reservations) upon interval change, without coordination from the
+ * Primary.  This event notifies the ScrubMachine that it can forget about
+ * such remote state.
+ */
 MEV(IntervalChanged)
 
 /// guarantee that the FSM is in the quiescent state (i.e. NotActive)

From 033e3d4c7e1c16e98ff9c40c76dd9b39c5fb4e5f Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 21 Mar 2023 16:23:18 -0400
Subject: [PATCH 0274/2492] mds: add debugging for standby-replay journal
 positions

This adds debugging for https://tracker.ceph.com/issues/59119

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSRank.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index ad1a7bc7f3f2..ee331c4f58c9 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1816,7 +1816,11 @@ class MDSRank::C_MDS_StandbyReplayRestartFinish : public MDSIOContext {
 
 void MDSRank::_standby_replay_restart_finish(int r, uint64_t old_read_pos)
 {
-  if (old_read_pos < mdlog->get_journaler()->get_trimmed_pos()) {
+  auto trimmed_pos = mdlog->get_journaler()->get_trimmed_pos();
+  dout(20) << __func__ << ":"
+           << " old_read_pos=" << old_read_pos
+           << " trimmed_pos=" << trimmed_pos << dendl;
+  if (old_read_pos < trimmed_pos) {
     dout(0) << "standby MDS fell behind active MDS journal's expire_pos, restarting" << dendl;
     respawn(); /* we're too far back, and this is easier than
 		  trying to reset everything in the cache, etc */

From 4364ebdfa4a218a61528f7579e45233cc4725326 Mon Sep 17 00:00:00 2001
From: jrchyang <yujrchyang@outlook.com>
Date: Tue, 24 Oct 2023 10:03:09 +0800
Subject: [PATCH 0275/2492] os/bluestore: fix crash caused by dividing by 0

In Allocator::create() => ZonedAllocator::ZonedAllocator()
num_zones is calculated by size / zone_size,
causing a crash when zone_size is set to 0

Signed-off-by: Jrchyang Yu <yuzhiqiang_yewu@cmss.chinamobile.com>
---
 src/os/bluestore/BlueStore.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index ec03fcde14ae..d031b2e4c010 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6890,7 +6890,7 @@ int BlueStore::_create_alloc()
       cct, cct->_conf->bluestore_allocator,
       bdev->get_conventional_region_size(),
       alloc_size,
-      0, 0,
+      zone_size, 0,
       "zoned_block");
     if (!a) {
       lderr(cct) << __func__ << " failed to create " << cct->_conf->bluestore_allocator

From 8d32e586dcbf355ea7155a4a8c06389be07a2fe8 Mon Sep 17 00:00:00 2001
From: Raimund Sacherer <rsachere@redhat.com>
Date: Fri, 26 May 2023 17:15:43 +0200
Subject: [PATCH 0276/2492] osd: make _set_cache_sizes ratio aware of
 cache_kv_onode_ratio

The formula checking if all the cache sizes sum up to <= 1 did not take
cache_kv_onode_ratio into account.

The last dout line did also not take cache_kv_onode_ratio into account.

Signed-off-by: Raimund Sacherer <rsachere@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index a489cb65d2e1..66faedaddf67 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4955,10 +4955,11 @@ int BlueStore::_set_cache_sizes()
     return -EINVAL;
   }
 
-  if (cache_meta_ratio + cache_kv_ratio > 1.0) {
+  if (cache_meta_ratio + cache_kv_ratio + cache_kv_onode_ratio > 1.0) {
     derr << __func__ << " bluestore_cache_meta_ratio (" << cache_meta_ratio
          << ") + bluestore_cache_kv_ratio (" << cache_kv_ratio
-         << ") = " << cache_meta_ratio + cache_kv_ratio << "; must be <= 1.0"
+         << ") + bluestore_cache_kv_onode_ratio (" << cache_kv_onode_ratio
+         << ") = " << cache_meta_ratio + cache_kv_ratio + cache_kv_onode_ratio << "; must be <= 1.0"
          << dendl;
     return -EINVAL;
   }
@@ -4975,6 +4976,7 @@ int BlueStore::_set_cache_sizes()
   dout(1) << __func__ << " cache_size " << cache_size
           << " meta " << cache_meta_ratio
 	  << " kv " << cache_kv_ratio
+	  << " kv_onode " << cache_kv_onode_ratio
 	  << " data " << cache_data_ratio
 	  << dendl;
   return 0;

From f2c392ec0dce39050060f097b82603f8c9759d5c Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Mon, 23 Oct 2023 17:18:57 +0200
Subject: [PATCH 0277/2492] mgr/rook: remove all the hardcoded refs to the
 'rook-ceph' namespace Fixes: https://tracker.ceph.com/issues/63291

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/rook_cluster.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index c89b38f8d868..8f7ac3493147 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -97,9 +97,10 @@ def wrapper(*args: Any, **kwargs: Any) -> threading.Thread:
 
 
 class DefaultFetcher():
-    def __init__(self, storage_class: str, coreV1_api: 'client.CoreV1Api'):
+    def __init__(self, storage_class: str, coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv'):
         self.storage_class = storage_class
         self.coreV1_api = coreV1_api
+        self.rook_env = rook_env
 
     def fetch(self) -> None:
         self.inventory: KubernetesResource[client.V1PersistentVolumeList] = KubernetesResource(self.coreV1_api.list_persistent_volume)
@@ -151,8 +152,8 @@ def device(self, i: 'client.V1PersistentVolume') -> Tuple[str, Device]:
         
 
 class LSOFetcher(DefaultFetcher):
-    def __init__(self, storage_class: 'str', coreV1_api: 'client.CoreV1Api', customObjects_api: 'client.CustomObjectsApi', nodenames: 'Optional[List[str]]' = None):
-        super().__init__(storage_class, coreV1_api)
+    def __init__(self, storage_class: 'str', coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv', customObjects_api: 'client.CustomObjectsApi', nodenames: 'Optional[List[str]]' = None):
+        super().__init__(storage_class, coreV1_api, rook_env)
         self.customObjects_api = customObjects_api
         self.nodenames = nodenames
 
@@ -219,13 +220,13 @@ def device(self, i: Any) -> Tuple[str, Device]:
 
 class PDFetcher(DefaultFetcher):
     """ Physical Devices Fetcher"""
-    def __init__(self, coreV1_api: 'client.CoreV1Api'):
-        self.coreV1_api = coreV1_api
+    def __init__(self, coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv'):
+        super().__init__('', coreV1_api, rook_env)
 
     def fetch(self) -> None:
         """ Collect the devices information from k8s configmaps"""
         self.dev_cms: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_config_map,
-                                                              namespace='rook-ceph',
+                                                              namespace=self.rook_env.namespace,
                                                               label_selector='app=rook-discover')
 
     def devices(self) -> Dict[str, List[Device]]:
@@ -759,15 +760,15 @@ def get_storage_class(self) -> 'client.V1StorageClass':
 
     def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[str, List[Device]]:
         self.fetcher: Optional[DefaultFetcher] = None
-        op_settings = self.coreV1_api.read_namespaced_config_map(name="rook-ceph-operator-config", namespace='rook-ceph').data
+        op_settings = self.coreV1_api.read_namespaced_config_map(name="rook-ceph-operator-config", namespace=self.rook_env.namespace).data
         if op_settings.get('ROOK_ENABLE_DISCOVERY_DAEMON', 'false').lower() == 'true':
-            self.fetcher = PDFetcher(self.coreV1_api)
+            self.fetcher = PDFetcher(self.coreV1_api, self.rook_env)
         else:
             storage_class = self.get_storage_class()
             if storage_class.metadata.labels and ('local.storage.openshift.io/owner-name' in storage_class.metadata.labels):
                 self.fetcher = LSOFetcher(self.storage_class, self.coreV1_api, self.customObjects_api, nodenames)
             else:
-                self.fetcher = DefaultFetcher(self.storage_class, self.coreV1_api)
+                self.fetcher = DefaultFetcher(self.storage_class, self.coreV1_api, self.rook_env)
 
         self.fetcher.fetch()
         return self.fetcher.devices()

From a55bf557e2c90f9cb84cebdb0662563d0397b45d Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Tue, 24 Oct 2023 14:06:41 +0200
Subject: [PATCH 0278/2492] mgr/rook: fixing the namespaces used to access some
 resources/config

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/rook_cluster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index 8f7ac3493147..f455581656e0 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -226,7 +226,7 @@ def __init__(self, coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv'):
     def fetch(self) -> None:
         """ Collect the devices information from k8s configmaps"""
         self.dev_cms: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_config_map,
-                                                              namespace=self.rook_env.namespace,
+                                                              namespace=self.rook_env.operator_namespace,
                                                               label_selector='app=rook-discover')
 
     def devices(self) -> Dict[str, List[Device]]:
@@ -760,7 +760,7 @@ def get_storage_class(self) -> 'client.V1StorageClass':
 
     def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[str, List[Device]]:
         self.fetcher: Optional[DefaultFetcher] = None
-        op_settings = self.coreV1_api.read_namespaced_config_map(name="rook-ceph-operator-config", namespace=self.rook_env.namespace).data
+        op_settings = self.coreV1_api.read_namespaced_config_map(name="rook-ceph-operator-config", namespace=self.rook_env.operator_namespace).data
         if op_settings.get('ROOK_ENABLE_DISCOVERY_DAEMON', 'false').lower() == 'true':
             self.fetcher = PDFetcher(self.coreV1_api, self.rook_env)
         else:

From da268858650c9c0abe8f667e3aaf0015d8f3ee1f Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 24 Oct 2023 13:39:01 +0000
Subject: [PATCH 0279/2492] crimson/osd/pg: add comments around
 do_osd_ops_execute parameters

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/internal_client_request.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc
index 1e9b842b2ec7..87438d4a1468 100644
--- a/src/crimson/osd/osd_operations/internal_client_request.cc
+++ b/src/crimson/osd/osd_operations/internal_client_request.cc
@@ -89,9 +89,11 @@ seastar::future<> InternalClientRequest::start()
                   osd_ops,
                   std::as_const(op_info),
                   get_do_osd_ops_params(),
+                  // success_func
                   [] {
                     return PG::do_osd_ops_iertr::now();
                   },
+                  // failure_func
                   [] (const std::error_code& e) {
                     return PG::do_osd_ops_iertr::now();
                   }

From e83bf4821bfb604694b03a67d7390bfb38655506 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 24 Oct 2023 13:40:27 +0000
Subject: [PATCH 0280/2492] crimson/osd/pg: rename submit_error_log returned
 future

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 0cd6c503e4d7..42052a4c84d9 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1036,7 +1036,7 @@ PG::do_osd_ops(
     // failure_func
     [m, &op_info, obc, this] (const std::error_code& e) {
     return seastar::do_with(eversion_t(), [m, &op_info, obc, e, this](auto &version) {
-      auto fut = seastar::now();
+      auto error_log_fut = seastar::now();
       epoch_t epoch = get_osdmap_epoch();
       ceph_tid_t rep_tid = shard_services.get_tid();
       auto last_complete = peering_state.get_info().last_complete;
@@ -1044,10 +1044,10 @@ PG::do_osd_ops(
         // This should be executed as OrderedExclusivePhaseT so that
         // successive ops will not reorder.
         // TODO: https://tracker.ceph.com/issues/61651
-        fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
+        error_log_fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
       }
-      return fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
-        auto fut2 = seastar::now();
+      return error_log_fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
+        auto fut = seastar::now();
         if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
           auto it = log_entry_update_waiting_on.find(rep_tid);
           ceph_assert(it != log_entry_update_waiting_on.end());
@@ -1060,7 +1060,7 @@ PG::do_osd_ops(
               peering_state.complete_write(version, last_complete);
             }
           } else {
-            fut2 = it->second.all_committed.get_shared_future().then(
+            fut = it->second.all_committed.get_shared_future().then(
               [this, &version, last_complete] {
               if (version != eversion_t()) {
                 peering_state.complete_write(version, last_complete);
@@ -1069,7 +1069,7 @@ PG::do_osd_ops(
             });
           }
         }
-        return fut2.then([this, m, e] {
+        return fut.then([this, m, e] {
           return log_reply(m, e);
         });
       });

From 43f32a46aa9095b19525357ba7ca215e842b4f77 Mon Sep 17 00:00:00 2001
From: Alexander Mikhalitsyn <aleksandr.mikhalitsyn@canonical.com>
Date: Mon, 23 Oct 2023 14:34:26 +0200
Subject: [PATCH 0281/2492] MClientRequest: handle ext_num_retry and
 ext_num_fwd from ceph_mds_request_head_legacy

When a client is too old and uses struct ceph_mds_request_head_legacy we must
fill new ext_num_retry and ext_num_fwd fields from an old num_retry and num_fwd.

Fixes: https://github.com/ceph/ceph/pull/45669
Fixes: https://tracker.ceph.com/issues/63288
Fixes: commit cbd7e3040208 ("ceph_fs.h: add 32 bits extended num_retry and num_fwd support")

Signed-off-by: Alexander Mikhalitsyn <aleksandr.mikhalitsyn@canonical.com>
---
 src/messages/MClientRequest.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/messages/MClientRequest.h b/src/messages/MClientRequest.h
index c62e183a7563..3c7d83ec11b3 100644
--- a/src/messages/MClientRequest.h
+++ b/src/messages/MClientRequest.h
@@ -234,6 +234,9 @@ class MClientRequest final : public MMDSOp {
       copy_from_legacy_head(&head, &old_mds_head);
       head.version = 0;
 
+      head.ext_num_retry = head.num_retry;
+      head.ext_num_fwd = head.num_fwd;
+
       /* Can't set the btime from legacy struct */
       if (head.op == CEPH_MDS_OP_SETATTR) {
 	int localmask = head.args.setattr.mask;

From a70a70f589214d6e2a5b477a61005b13ba2fec46 Mon Sep 17 00:00:00 2001
From: Alexander Mikhalitsyn <aleksandr.mikhalitsyn@canonical.com>
Date: Mon, 23 Oct 2023 14:37:44 +0200
Subject: [PATCH 0282/2492] MClientRequest: handle owner_uid and owner_gid from
 ceph_mds_request_head_legacy

When a client is too old and uses struct ceph_mds_request_head_legacy we must
fill new owner_uid and owner_gid fields from an old client_uid and client_gid.

Fixes: https://github.com/ceph/ceph/pull/52575
Fixes: https://tracker.ceph.com/issues/63288
Fixes: commit 46cb244b9c839 ("ceph_fs.h: add separate owner_{u,g}id fields")

Signed-off-by: Alexander Mikhalitsyn <aleksandr.mikhalitsyn@canonical.com>
---
 src/messages/MClientRequest.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/messages/MClientRequest.h b/src/messages/MClientRequest.h
index 3c7d83ec11b3..f63657d2e253 100644
--- a/src/messages/MClientRequest.h
+++ b/src/messages/MClientRequest.h
@@ -237,6 +237,9 @@ class MClientRequest final : public MMDSOp {
       head.ext_num_retry = head.num_retry;
       head.ext_num_fwd = head.num_fwd;
 
+      head.owner_uid = head.caller_uid;
+      head.owner_gid = head.caller_gid;
+
       /* Can't set the btime from legacy struct */
       if (head.op == CEPH_MDS_OP_SETATTR) {
 	int localmask = head.args.setattr.mask;

From 8bf0820061a7b9d76f31904cb6c0781d1ac12dd4 Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Tue, 24 Oct 2023 12:13:21 -0400
Subject: [PATCH 0283/2492] rgw/multisite: set versioning status in archive
 zone upon bucket creation

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index d6d83eab3dcc..eb410fe0582e 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -2575,15 +2575,16 @@ int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dp
     }
     bci.info.layout.current_index.layout.type = rule_info.index_type;
   } else {
-    /* always keep bucket versioning enabled on archive zone */
-    if (bihandler->driver->get_zone()->get_tier_type() == "archive") {
-      bci.info.flags = (bci.info.flags & ~BUCKET_VERSIONS_SUSPENDED) | BUCKET_VERSIONED;
-    }
     /* existing bucket, keep its placement */
     bci.info.bucket.explicit_placement = old_bci->info.bucket.explicit_placement;
     bci.info.placement_rule = old_bci->info.placement_rule;
   }
 
+  //always keep bucket versioning enabled on archive zone
+  if (bihandler->driver->get_zone()->get_tier_type() == "archive") {
+    bci.info.flags = (bci.info.flags & ~BUCKET_VERSIONS_SUSPENDED) | BUCKET_VERSIONED;
+  }
+
   /* record the read version (if any), store the new version */
   bci.info.objv_tracker.read_version = objv_tracker.read_version;
   bci.info.objv_tracker.write_version = objv_tracker.write_version;

From db1e785b2c1a55f4d232b1642573e5672a136190 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 11 Oct 2023 14:05:17 -0400
Subject: [PATCH 0284/2492] cephadm: add a --dry-run option to cephadm shell

Instead of creating the shell, the --dry-run option prints the container
command that would be used. This can be used as a starting point for
creating custom container commands similar to what cephadm shell would
generate but with tweaks.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index a64b0b49404d..04104c846b85 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -5417,6 +5417,10 @@ def command_shell(ctx):
         privileged=True)
     command = c.shell_cmd(command)
 
+    if ctx.dry_run:
+        print(' '.join(shlex.quote(arg) for arg in command))
+        return 0
+
     return call_timeout(ctx, command, ctx.timeout)
 
 ##################################
@@ -7165,6 +7169,10 @@ def _get_parser():
         '--no-hosts',
         action='store_true',
         help='dont pass /etc/hosts through to the container')
+    parser_shell.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='print, but do not execute, the container command to start the shell')
 
     parser_enter = subparsers.add_parser(
         'enter', help='run an interactive shell inside a running daemon container')

From 4a033788f87859be4da706e38621009474ed3b82 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 11 Sep 2023 14:58:43 -0400
Subject: [PATCH 0285/2492] doc/cephadm: clean up debugging section

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 doc/cephadm/troubleshooting.rst | 53 ++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 14 deletions(-)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index 5ec692881661..1ea153424461 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -335,16 +335,22 @@ Deploy the daemon::
 
   cephadm --image <container-image> deploy --fsid <fsid> --name mgr.hostname.smfvfd --config-json config-json.json
 
-Analyzing Core Dumps
+Capturing Core Dumps
 ---------------------
 
-When a Ceph daemon crashes, cephadm supports analyzing core dumps. To enable core dumps, run
+A Ceph cluster that uses cephadm can be configured to capture core dumps.
+Initial capture and processing of the coredump is performed by
+`systemd-coredump <https://www.man7.org/linux/man-pages/man8/systemd-coredump.8.html>`_.
+
+
+To enable coredump handling, run:
 
 .. prompt:: bash #
 
   ulimit -c unlimited
 
-Core dumps will now be written to ``/var/lib/systemd/coredump``.
+Core dumps will be written to ``/var/lib/systemd/coredump``.
+This will persist until the system is rebooted.
 
 .. note::
 
@@ -352,16 +358,35 @@ Core dumps will now be written to ``/var/lib/systemd/coredump``.
   they will be written to ``/var/lib/systemd/coredump`` on
   the container host. 
 
-Now, wait for the crash to happen again. To simulate the crash of a daemon, run e.g. ``killall -3 ceph-mon``.
+Now, wait for the crash to happen again. To simulate the crash of a daemon, run
+e.g. ``killall -3 ceph-mon``.
+
+
+Running the Debugger with cephadm
+----------------------------------
+
+Running a single debugging session
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One can initiate a debugging session using the ``cephadm shell`` command.
+From within the shell container we need to install the debugger and debuginfo
+packages. To debug a core file captured by systemd, run the following:
+
+.. prompt:: bash #
+
+    # start the shell session
+    cephadm shell --mount /var/lib/system/coredump
+    # within the shell:
+    dnf install ceph-debuginfo gdb zstd
+    unzstd /var/lib/systemd/coredump/core.ceph-*.zst
+    gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-*.zst
+
+You can then run debugger commands at gdb's prompt.
 
-Install debug packages including ``ceph-debuginfo`` by entering the cephadm shelll::
+.. prompt::
 
-  # cephadm shell --mount /var/lib/systemd/coredump
-  [ceph: root@host1 /]# dnf install ceph-debuginfo gdb zstd
-  [ceph: root@host1 /]# unzstd /mnt/coredump/core.ceph-*.zst
-  [ceph: root@host1 /]# gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-...
-  (gdb) bt
-  #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
-  #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
-  #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
-  #3  0x0000563085ca3d7e in main ()
+    (gdb) bt
+    #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
+    #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
+    #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
+    #3  0x0000563085ca3d7e in main ()

From b4e2b3991774fc15f677752a344181920fc1216c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 11 Sep 2023 17:55:36 -0400
Subject: [PATCH 0286/2492] doc/cephadm: extend container debugging to cover
 live processes

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 doc/cephadm/troubleshooting.rst | 75 +++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index 1ea153424461..3d4965deda03 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -390,3 +390,78 @@ You can then run debugger commands at gdb's prompt.
     #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
     #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
     #3  0x0000563085ca3d7e in main ()
+
+
+Running repeated debugging sessions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When using ``cephadm shell``, like in the example above, the changes made to
+the container the shell command spawned are ephemeral. Once the shell session
+exits all of the files that were downloaded and installed are no longer
+available. One can simply re-run the same commands every time ``cephadm shell``
+is invoked, but in order to save time and resources one can create a new
+container image and use it for repeated debugging sessions.
+
+In the following example we create a simple file for constructing the
+container image. The command below uses podman but it should work correctly
+if ``podman`` is replaced with ``docker``.
+
+.. prompt:: bash
+
+  cat >Containerfile <<EOF
+  ARG BASE_IMG=quay.io/ceph/ceph:v18
+  FROM \${BASE_IMG}
+  # install ceph debuginfo packages, gdb and other potentially useful packages
+  RUN dnf install --enablerepo='*debug*' -y ceph-debuginfo gdb zstd strace python3-debuginfo
+  EOF
+  podman build -t ceph:debugging -f Containerfile .
+  # pass --build-arg=BASE_IMG=<your image> to customize the base image
+
+The result should be a new local image named ``ceph:debugging``. This image can
+be used on the same machine that built it. Later, the image could be pushed to
+a container repository, or saved and copied to a node runing other ceph
+containers. Please consult the documentation for ``podman`` or ``docker`` for
+more details on the general container workflow.
+
+Once the image has been built it can be used to initiate repeat debugging
+sessions without having to re-install the debug tools and debuginfo packages.
+To debug a core file using this image, in the same way as previously described,
+run:
+
+.. prompt:: bash #
+
+    cephadm --image ceph:debugging shell --mount /var/lib/system/coredump
+
+
+Debugging live processes
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The gdb debugger has the ability to attach to running processes to debug them.
+For a containerized process this can be accomplished by using the debug image
+and attaching it to the same PID namespace as the process to be debugged.
+
+This requires running a container command with some custom arguments. We can generate a script that can debug a process in a running container.
+
+.. prompt:: bash #
+
+   cephadm --image ceph:debugging shell --dry-run > /tmp/debug.sh
+
+This creates a script with the container command cephadm would use to create a
+shell. Now, modify the script by removing the ``--init`` argument and replace
+that with the argument to join to the namespace used for a running running
+container.  For example, let's assume we want to debug the MGR, and have
+determnined that the MGR is running in a container named
+``ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``. The new
+argument
+``--pid=container:ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``
+should be used.
+
+Now, we can run our debugging container with ``sh /tmp/debug.sh``. Within the shell
+we can run commands such as ``ps`` to get the PID of the MGR process. In the following
+example this will be ``2``. Running gdb, we can now attach to the running process:
+
+.. prompt:: bash (gdb)
+
+   attach 2
+   info threads
+   bt

From 999d539f76bdb4baa0b8342fff74a9e3fd1cdc12 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 24 Oct 2023 16:45:47 -0400
Subject: [PATCH 0287/2492] rgw: rename 'expected' to 'accounted' size in
 fetch_remote_obj()

and use it instead of cb.get_data_len() after testing for equality

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index e08804379893..b72bf3a856b8 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4230,7 +4230,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   string etag;
   real_time set_mtime;
-  uint64_t expected_size = 0;
+  uint64_t accounted_size = 0;
 
   RGWObjState *dest_state = NULL;
   RGWObjManifest *manifest = nullptr;
@@ -4269,7 +4269,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   }
 
   ret = conn->complete_request(in_stream_req, &etag, &set_mtime,
-                               &expected_size, nullptr, nullptr, rctx.y);
+                               &accounted_size, nullptr, nullptr, rctx.y);
   if (ret < 0) {
     goto set_err_state;
   }
@@ -4277,17 +4277,18 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   if (ret < 0) {
     goto set_err_state;
   }
-  if (cb.get_data_len() != expected_size) {
+  if (cb.get_data_len() != accounted_size) {
     ret = -EIO;
     ldpp_dout(rctx.dpp, 0) << "ERROR: object truncated during fetching, expected "
-        << expected_size << " bytes but received " << cb.get_data_len() << dendl;
+        << accounted_size << " bytes but received " << cb.get_data_len() << dendl;
     goto set_err_state;
   }
+
   if (compressor && compressor->is_compressed()) {
     bufferlist tmp;
     RGWCompressionInfo cs_info;
     cs_info.compression_type = plugin->get_type_name();
-    cs_info.orig_size = cb.get_data_len();
+    cs_info.orig_size = accounted_size;
     cs_info.compressor_message = compressor->get_compressor_message();
     cs_info.blocks = std::move(compressor->get_compression_blocks());
     encode(cs_info, tmp);
@@ -4421,7 +4422,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 #define MAX_COMPLETE_RETRY 100
   for (i = 0; i < MAX_COMPLETE_RETRY; i++) {
     bool canceled = false;
-    ret = processor.complete(cb.get_data_len(), etag, mtime, set_mtime,
+    ret = processor.complete(accounted_size, etag, mtime, set_mtime,
                              attrs, delete_at, nullptr, nullptr, nullptr,
                              zones_trace, &canceled, rctx);
     if (ret < 0) {

From 526643826246fa8dbe35ea2f65bf9ad0ef9566ff Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 24 Oct 2023 16:48:06 -0400
Subject: [PATCH 0288/2492] rgw: fetch_remote_obj() uses uncompressed size for
 encrypted objects

use the original size from RGW_ATTR_COMPRESSION as the accounted size in
the bucket index for objects that were transferred in their
encrypted/compressed form

Fixes: https://tracker.ceph.com/issues/63306

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b72bf3a856b8..b935610f9ad5 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4293,6 +4293,20 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     cs_info.blocks = std::move(compressor->get_compression_blocks());
     encode(cs_info, tmp);
     cb.get_attrs()[RGW_ATTR_COMPRESSION] = tmp;
+  } else if (auto c = cb.get_attrs().find(RGW_ATTR_COMPRESSION);
+             c != cb.get_attrs().end()) {
+    // if the object was transferred in its compressed+encrypted form, use its
+    // original uncompressed size
+    try {
+      RGWCompressionInfo info;
+      auto p = c->second.cbegin();
+      decode(info, p);
+      accounted_size = info.orig_size;
+    } catch (const buffer::error&) {
+      ldpp_dout(rctx.dpp, 0) << "ERROR: could not decode compression attr for "
+          "replicated object " << dest_obj << dendl;
+      // decode error isn't fatal, but we might put the wrong size in the index
+    }
   }
 
   if (override_owner) {

From fca3a61ce231f6498e961432b07692117cdd59ae Mon Sep 17 00:00:00 2001
From: Michaela Lang <milang@redhat.com>
Date: Tue, 24 Oct 2023 13:24:44 +0200
Subject: [PATCH 0289/2492] added check for `--skip-firewalld` to section on
 adding  explicit Ports to firewalld

Signed-off-by: Michaela Lang <milang@redhat.com>
---
 src/cephadm/cephadm.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index a64b0b49404d..d8a938c659ef 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2976,10 +2976,11 @@ def deploy_daemon(
     update_firewalld(ctx, daemon_form_create(ctx, ident))
 
     # Open ports explicitly required for the daemon
-    if endpoints:
-        fw = Firewalld(ctx)
-        fw.open_ports([e.port for e in endpoints] + fw.external_ports.get(daemon_type, []))
-        fw.apply_rules()
+    if not ('skip_firewalld' in ctx and ctx.skip_firewalld):
+        if endpoints:
+            fw = Firewalld(ctx)
+            fw.open_ports([e.port for e in endpoints] + fw.external_ports.get(daemon_type, []))
+            fw.apply_rules()
 
     # If this was a reconfig and the daemon is not a Ceph daemon, restart it
     # so it can pick up potential changes to its configuration files

From 9e2b8b0e8235b36e55310aab49b8f760e8d57cad Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 25 Oct 2023 02:24:18 -0500
Subject: [PATCH 0290/2492] osd: fix use-after-move in
 build_incremental_map_msg()

Fixes: https://tracker.ceph.com/issues/63310

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a9d08cd25364..862ee5b18c85 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1405,14 +1405,14 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
   for (epoch_t e = since + 1; e <= to; ++e) {
     bufferlist bl;
     if (get_inc_map_bl(e, bl)) {
-      m->incremental_maps[e] = std::move(bl);
+      m->incremental_maps[e] = bl;
     } else {
       dout(10) << __func__ << " missing incremental map " << e << dendl;
       if (!get_map_bl(e, bl)) {
 	derr << __func__ << " also missing full map " << e << dendl;
 	goto panic;
       }
-      m->maps[e] = std::move(bl);
+      m->maps[e] = bl;
     }
     max--;
     max_bytes -= bl.length();

From 9c7e8295a14394f77a12a4475382c8a04b2c9868 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 25 Oct 2023 19:46:56 +0300
Subject: [PATCH 0291/2492] common/buffer: silent compiler warning

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/common/buffer.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index b363b99573f6..b46409792890 100644
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -827,8 +827,9 @@ static ceph::spinlock debug_lock;
   {
     length = std::min<size_t>(length, get_remaining());
     while (length > 0) {
-      const char *p;
+      const char *p = nullptr;
       size_t l = get_ptr_and_advance(length, &p);
+      ceph_assert(p);
       crc = ceph_crc32c(crc, (unsigned char*)p, l);
       length -= l;
     }

From 35715ce798cd6ca43483fa9348ad88e7a5a99286 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 25 Oct 2023 19:47:29 +0300
Subject: [PATCH 0292/2492] os/bluestore: silent compiler warning

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index d031b2e4c010..901648f1af87 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4528,7 +4528,7 @@ BlueStore::ExtentMap::debug_list_disk_layout()
     }
 
     unsigned csum_i = 0;
-    size_t csum_cnt;
+    size_t csum_cnt = 0;
     uint32_t length;
     if (bblob.has_csum()) {
       csum_cnt = bblob.get_csum_count();

From 5eb14cf18067cad35e413ca2a917ae05d4a5c691 Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Wed, 25 Oct 2023 15:05:56 -0700
Subject: [PATCH 0293/2492] docs: revised Distro Codemap, removed `octopus`,
 added `reef`

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 doc/dev/release-process.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/dev/release-process.rst b/doc/dev/release-process.rst
index 9cd64ece932b..269983de6eb5 100644
--- a/doc/dev/release-process.rst
+++ b/doc/dev/release-process.rst
@@ -103,15 +103,15 @@ NOTE: if for some reason the build has to be restarted (for example if one distr
 
 4. Use https://docs.ceph.com/en/latest/start/os-recommendations/?highlight=debian#platforms to determine the ``DISTROS`` parameter.  For example,
 
-    +-------------------+-------------------------------------------+
-    | Release           | Distro Codemap                            |
-    +===================+===========================================+
-    | octopus (15.X.X)  | ``focal bionic centos7 centos8 buster``   |
-    +-------------------+-------------------------------------------+
-    | pacific (16.X.X)  | ``focal bionic centos8 buster bullseye``  |
-    +-------------------+-------------------------------------------+
-    | quincy (17.X.X)   | ``focal centos8 centos9 bullseye``        |
-    +-------------------+-------------------------------------------+
+    +-------------------+--------------------------------------------------+
+    | Release           | Distro Codemap                                   |
+    +===================+==================================================+
+    | pacific (16.X.X)  | ``focal bionic centos8 buster bullseye``         |
+    +-------------------+--------------------------------------------------+
+    | quincy (17.X.X)   | ``focal centos8 centos9 bullseye``               |
+    +-------------------+--------------------------------------------------+
+    | reef (18.X.X)     | ``jammy focal centos8 centos9 windows bookworm`` |
+    +-------------------+--------------------------------------------------+
 
 5. Click ``Build``.
 

From ccf2bba418f2b10ea39d3699aea79baf9563a68f Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Thu, 26 Oct 2023 10:58:53 +0530
Subject: [PATCH 0294/2492] qa: use Remote.run_unit_test when "unit_test_scan"
 config is used

"unit_test_scan" optional config uses Remote.run_unit_test()
to scan xml files generated by unit tests to throw better failure
messages (for s3tests and gtests run by workunit)
It also creates "unit_test_summary.yaml" for more exception details
from xml files.

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 .../monthrash/workloads/rados_api_tests.yaml  |  2 +-
 .../smoke/basic/tasks/test/rgw_s3tests.yaml   |  1 +
 qa/tasks/s3tests.py                           | 22 +++++++++++++++----
 qa/tasks/workunit.py                          | 20 ++++++++++++-----
 qa/workunits/rados/test.sh                    |  4 +++-
 5 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/qa/suites/rados/monthrash/workloads/rados_api_tests.yaml b/qa/suites/rados/monthrash/workloads/rados_api_tests.yaml
index f14c2c2a60d7..a3fd80843d4b 100644
--- a/qa/suites/rados/monthrash/workloads/rados_api_tests.yaml
+++ b/qa/suites/rados/monthrash/workloads/rados_api_tests.yaml
@@ -25,4 +25,4 @@ tasks:
 - workunit:
     clients:
       client.0:
-        - rados/test.sh
+        - rados/test.sh unit_test_scan
diff --git a/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml b/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
index e3a1167c6ee8..4d4f4fe51f75 100644
--- a/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
+++ b/qa/suites/smoke/basic/tasks/test/rgw_s3tests.yaml
@@ -6,6 +6,7 @@ tasks:
 - s3tests:
     client.0:
       force-branch: ceph-master
+      unit_test_scan: True
       rgw_server: client.0
 overrides:
   ceph:
diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py
index 70b49c086b6f..27ca1a03bc53 100644
--- a/qa/tasks/s3tests.py
+++ b/qa/tasks/s3tests.py
@@ -9,6 +9,8 @@
 import os
 import random
 import string
+import datetime
+import uuid
 
 from teuthology import misc as teuthology
 from teuthology import contextutil
@@ -392,10 +394,22 @@ def run_tests(ctx, config):
             attrs += ['not sse_s3']
        
         attrs += client_config.get('extra_attrs', [])
-        args += ['tox', '--', '-v', '-m', ' and '.join(attrs)]
-        args += client_config.get('extra_args', [])
-
-        toxvenv_sh(ctx, remote, args, label="s3 tests against rgw")
+        if 'unit_test_scan' in client_config and client_config['unit_test_scan']:
+            xmlfile_id = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S--") + str(uuid.uuid4())
+            xmlpath= f'{testdir}/archive/s3test-{xmlfile_id}.xml'
+            args += ['tox', '--', '-v', f'--junitxml={xmlpath}', '-m', ' and '.join(attrs)]
+            args += client_config.get('extra_args', [])
+            activate = get_toxvenv_dir(ctx) + '/bin/activate'
+            remote.run_unit_test(
+                args=['source', activate, run.Raw('&&')] + args,
+                label="s3 tests against rgw",
+                xml_path_regex=f'{testdir}/archive/s3test-*.xml',
+                output_yaml=os.path.join(ctx.archive, 'unit_test_summary.yaml'),
+            )
+        else:
+            args += ['tox', '--', '-v', '-m', ' and '.join(attrs)]
+            args += client_config.get('extra_args', [])
+            toxvenv_sh(ctx, remote, args, label="s3 tests against rgw")
     yield
 
 @contextlib.contextmanager
diff --git a/qa/tasks/workunit.py b/qa/tasks/workunit.py
index 23d436fc0c5c..1876b7842192 100644
--- a/qa/tasks/workunit.py
+++ b/qa/tasks/workunit.py
@@ -421,11 +421,21 @@ def _run_tests(ctx, refspec, role, tests, env, basedir,
                         workunit=workunit,
                     ),
                 ])
-                remote.run(
-                    logger=log.getChild(role),
-                    args=args + optional_args,
-                    label="workunit test {workunit}".format(workunit=workunit)
-                )
+                if 'unit_test_scan' in optional_args:
+                    optional_args.remove('unit_test_scan')
+                    remote.run_unit_test(
+                        logger=log.getChild(role),
+                        args=args + optional_args,
+                        label="workunit test {workunit}".format(workunit=workunit),
+                        xml_path_regex=f'{testdir}/archive/gtest_xml_report-*.xml',
+                        output_yaml=os.path.join(ctx.archive, 'unit_test_summary.yaml'),
+                    )
+                else:
+                    remote.run(
+                        logger=log.getChild(role),
+                        args=args + optional_args,
+                        label="workunit test {workunit}".format(workunit=workunit)
+                    )
                 if cleanup:
                     args=['sudo', 'rm', '-rf', '--', scratch_tmp]
                     remote.run(logger=log.getChild(role), args=args, timeout=(60*60))
diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
index daa25fe4dfd8..a88826cf619c 100755
--- a/qa/workunits/rados/test.sh
+++ b/qa/workunits/rados/test.sh
@@ -12,6 +12,8 @@ function cleanup() {
 }
 trap cleanup EXIT ERR HUP INT QUIT
 
+GTEST_OUTPUT="xml:/home/ubuntu/cephtest/archive/gtest_xml_report"
+
 declare -A pids
 
 for f in \
@@ -37,7 +39,7 @@ do
     if [ $parallel -eq 1 ]; then
 	r=`printf '%25s' $f`
 	ff=`echo $f | awk '{print $1}'`
-	bash -o pipefail -exc "ceph_test_rados_$f $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" &
+	bash -o pipefail -exc "ceph_test_rados_$f --gtest_output=$GTEST_OUTPUT-$f.xml $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" &
 	pid=$!
 	echo "test $f on pid $pid"
 	pids[$f]=$pid

From 397e85a5f8ee6b984077e94184404370497237a9 Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Thu, 26 Oct 2023 11:16:29 +0530
Subject: [PATCH 0295/2492] qa: rewrite "valgrind_post" to use ValgrindScanner

previously, valgrind_post() func used grep to find error
from valgrind logs.
now, it uses ValgrindScanner to log better exceptions with
traceback and exception kind, along with creating a more detailed
summary in valgrind.yaml in archive.

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 qa/tasks/ceph.py | 40 +++++++++-------------------------------
 1 file changed, 9 insertions(+), 31 deletions(-)

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index d487c4296105..a7b3ad4c5eec 100644
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -28,6 +28,7 @@
 from teuthology import contextutil
 from teuthology import exceptions
 from teuthology.orchestra import run
+from teuthology.util.scanner import ValgrindScanner
 from tasks import ceph_client as cclient
 from teuthology.orchestra.daemon import DaemonGroup
 from tasks.daemonwatchdog import DaemonWatchdog
@@ -327,38 +328,15 @@ def valgrind_post(ctx, config):
     try:
         yield
     finally:
-        lookup_procs = list()
-        log.info('Checking for errors in any valgrind logs...')
-        for remote in ctx.cluster.remotes.keys():
-            # look at valgrind logs for each node
-            proc = remote.run(
-                args="sudo zgrep '<kind>' /var/log/ceph/valgrind/* "
-                     # include a second file so that we always get
-                     # a filename prefix on the output
-                     "/dev/null | sort | uniq",
-                wait=False,
-                check_status=False,
-                stdout=StringIO(),
-            )
-            lookup_procs.append((proc, remote))
-
         valgrind_exception = None
-        for (proc, remote) in lookup_procs:
-            proc.wait()
-            out = proc.stdout.getvalue()
-            for line in out.split('\n'):
-                if line == '':
-                    continue
-                try:
-                    (file, kind) = line.split(':')
-                except Exception:
-                    log.error('failed to split line %s', line)
-                    raise
-                log.debug('file %s kind %s', file, kind)
-                if (file.find('mds') >= 0) and kind.find('Lost') > 0:
-                    continue
-                log.error('saw valgrind issue %s in %s', kind, file)
-                valgrind_exception = Exception('saw valgrind issues')
+        valgrind_yaml = os.path.join(ctx.archive, 'valgrind.yaml')
+        for remote in ctx.cluster.remotes.keys():
+            scanner = ValgrindScanner(remote)
+            errors = scanner.scan_all_files('/var/log/ceph/valgrind/*')
+            scanner.write_summary(valgrind_yaml)
+            if errors and not valgrind_exception:
+                log.debug('valgrind exception message: %s', errors[0])
+                valgrind_exception = Exception(errors[0])
 
         if config.get('expect_valgrind_errors'):
             if not valgrind_exception:

From 94a287d8fd50fa13669790689d473e1429824dfc Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 24 Aug 2023 00:25:58 +0530
Subject: [PATCH 0296/2492] qa/cephfs: update dependencies for xfstests-dev

Dependencies listed in xfstests_dev.py for xfstests-dev project are
outdated. This leads the xfstests_dev.py based integration tests to
fail. Update this dependency list using README of xfstests-dev project.

Also, remove code which is not relevant anymore (specifically, if-block
that checks and deals for 'python' and 'btrfs-progs-devel').

Fixes: https://tracker.ceph.com/issues/62556
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/xfstests_dev.py | 62 ++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/qa/tasks/cephfs/xfstests_dev.py b/qa/tasks/cephfs/xfstests_dev.py
index 36d5e4a21df8..cc41744c6f43 100644
--- a/qa/tasks/cephfs/xfstests_dev.py
+++ b/qa/tasks/cephfs/xfstests_dev.py
@@ -189,15 +189,14 @@ def install_deps(self):
                                                    # number
         log.info(f'distro and version detected is "{distro}" and "{version}".')
 
-        # we keep fedora here so that right deps are installed when this test
-        # is run locally by a dev.
-        if distro in ('redhatenterpriseserver', 'redhatenterprise', 'fedora',
-                      'centos', 'centosstream', 'rhel'):
-            deps = """acl attr automake bc dbench dump e2fsprogs fio \
-            gawk gcc indent libtool lvm2 make psmisc quota sed \
-            xfsdump xfsprogs \
-            libacl-devel libattr-devel libaio-devel libuuid-devel \
-            xfsprogs-devel btrfs-progs-devel python3 sqlite""".split()
+        if distro in ('redhatenterpriseserver', 'redhatenterprise', 'centos',
+                      'centosstream', 'rhel'):
+            deps = """\
+                    acl attr automake bc dbench dump e2fsprogs fio gawk gcc \
+                    gdbm-devel git indent kernel-devel libacl-devel \
+                    libaio-devel libcap-devel libtool libuuid-devel lvm2 \
+                    make psmisc python3 quota sed sqlite udftools \
+                    xfsprogs""".split()
 
             if self.install_xfsprogs:
                 if distro == 'centosstream' and major_ver_num == 8:
@@ -206,25 +205,46 @@ def install_deps(self):
                          'gettext', 'libedit-devel', 'libattr-devel',
                          'device-mapper-devel', 'libicu-devel']
 
-            deps_old_distros = ['xfsprogs-qa-devel']
-
-            if distro != 'fedora' and major_ver_num > 7:
-                    deps.remove('btrfs-progs-devel')
-
-            args = ['sudo', 'yum', 'install', '-y'] + deps + deps_old_distros
+            args = ['sudo', 'yum', 'install', '-y'] + deps
+        elif distro == 'fedora':
+            deps = """\
+                   acl attr automake bc dbench dump e2fsprogs fio gawk gcc \
+                   gdbm-devel git indent kernel-devel libacl-devel \
+                   libaio-devel libcap-devel libtool liburing-devel \
+                   libuuid-devel lvm2 make psmisc python3 quota sed sqlite \
+                   udftools xfsprogs \
+                   \
+                   btrfs-progs exfatprogs f2fs-tools ocfs2-tools xfsdump \
+                   xfsprogs-devel""".split()
+
+            args = ['sudo', 'yum', 'install', '-y'] + deps
         elif distro == 'ubuntu':
-            deps = """xfslibs-dev uuid-dev libtool-bin \
-            e2fsprogs automake gcc libuuid1 quota attr libattr1-dev make \
-            libacl1-dev libaio-dev xfsprogs libgdbm-dev gawk fio dbench \
-            uuid-runtime python sqlite3""".split()
+            deps = """\
+                   acl attr automake bc dbench dump e2fsprogs fio gawk \
+                   gcc git indent libacl1-dev libaio-dev libcap-dev \
+                   libgdbm-dev libtool libtool-bin liburing-dev libuuid1 \
+                   lvm2 make psmisc python3 quota sed uuid-dev uuid-runtime \
+                   xfsprogs sqlite3 \
+                   \
+                   exfatprogs f2fs-tools ocfs2-tools udftools xfsdump \
+                   xfslibs-dev""".split()
+                   # NOTE: Acc to xfstests-dev project's README we need the
+                   # following package, but it is not available for machines
+                   # where CephFS tests are run, since a custom version of
+                   # kernel is installed for testing. The default version of
+                   # kernel that comes with OS. Since all tests in generic
+                   # test-suite are running fine without this packages, no
+                   # effort is being made to build and install this package
+                   # before running tests from xfstests-dev.
+                   #
+                   # + [f'linux-headers-{k_rel}']
+                   # k_rel stands for kernel release number.
 
             if self.install_xfsprogs:
                 deps += ['libinih-dev', 'liburcu-dev', 'libblkid-dev',
                          'gettext', 'libedit-dev', 'libattr1-dev',
                          'libdevmapper-dev', 'libicu-dev', 'pkg-config']
 
-            if major_ver_num >= 19:
-                deps[deps.index('python')] ='python2'
             args = ['sudo', 'apt-get', 'install', '-y'] + deps
         else:
             raise RuntimeError('expected a yum based or a apt based system')

From e5abcc6e0865acf7faa8aaa1235b977016d9f7cd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 26 Oct 2023 11:17:40 -0400
Subject: [PATCH 0297/2492] rgw-check-policy: remove check for nonempty tenant

the tool should be usable with the default empty tenant to check
principals like `arn:aws:iam:::user/my-user-id`

Fixes: https://tracker.ceph.com/issues/63333

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_polparser.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/rgw/rgw_polparser.cc b/src/rgw/rgw_polparser.cc
index eca5066b3ce3..c26862478e9b 100644
--- a/src/rgw/rgw_polparser.cc
+++ b/src/rgw/rgw_polparser.cc
@@ -78,11 +78,6 @@ int main(int argc, const char** argv)
     }
   }
 
-  if (tenant.empty()) {
-    std::cerr << cmdname << ": must specify tenant name" << std::endl;
-    helpful_exit(cmdname);
-  }
-
   bool success = true;
 
   if (args.empty()) {

From 3082cd4701a6825a5a0c9ad0cb69e62361379957 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Wed, 25 Oct 2023 16:53:58 -0400
Subject: [PATCH 0298/2492] rgw: add layout generation info to "bucket stats"

Adds a new field to bucket stats. Also rearranges existing information
slightly to make more sense. Also removes the redundancy of showing
bucket versioning status in two different ways.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index d6d83eab3dcc..e0f26d33f462 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1312,18 +1312,20 @@ static int bucket_stats(rgw::sal::Driver* driver,
 
   formatter->open_object_section("stats");
   formatter->dump_string("bucket", bucket->get_name());
-  formatter->dump_int("num_shards",
-		      bucket->get_info().layout.current_index.layout.normal.num_shards);
   formatter->dump_string("tenant", bucket->get_tenant());
-  formatter->dump_string("versioning", bucket->versioned() ? (bucket->versioning_enabled() ? "enabled" : "suspended") : "off");
+  formatter->dump_string("versioning",
+			 bucket->versioned()
+			 ? (bucket->versioning_enabled() ? "enabled" : "suspended")
+			 : "off");
   formatter->dump_string("zonegroup", bucket->get_info().zonegroup);
   formatter->dump_string("placement_rule", bucket->get_info().placement_rule.to_str());
   ::encode_json("explicit_placement", bucket->get_key().explicit_placement, formatter);
   formatter->dump_string("id", bucket->get_bucket_id());
   formatter->dump_string("marker", bucket->get_marker());
   formatter->dump_stream("index_type") << bucket->get_info().layout.current_index.layout.type;
-  formatter->dump_bool("versioned", bucket_info.versioned());
-  formatter->dump_bool("versioning_enabled", bucket_info.versioning_enabled());
+  formatter->dump_int("index_generation", bucket->get_info().layout.current_index.gen);
+  formatter->dump_int("num_shards",
+		      bucket->get_info().layout.current_index.layout.normal.num_shards);
   formatter->dump_bool("object_lock_enabled", bucket_info.obj_lock_enabled());
   formatter->dump_bool("mfa_enabled", bucket_info.mfa_enabled());
   ::encode_json("owner", bucket->get_info().owner, formatter);

From 60fb818a1cb43d028c76e80baa6bea345d5e0b9d Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Mon, 23 Oct 2023 12:50:26 -0700
Subject: [PATCH 0299/2492] doc: 17.2.7 Release Notes

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
Signed-off-by: Laura Flores <lflores@ibm.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/releases/index.rst    |   1 +
 doc/releases/quincy.rst   | 590 +++++++++++++++++++++++++++++++++++++-
 doc/releases/releases.yml |   2 +
 3 files changed, 591 insertions(+), 2 deletions(-)

diff --git a/doc/releases/index.rst b/doc/releases/index.rst
index 2b8c9587a946..a078e5e47ce4 100644
--- a/doc/releases/index.rst
+++ b/doc/releases/index.rst
@@ -73,6 +73,7 @@ Release timeline
 .. _17.2.4: quincy#v17-2-4-quincy
 .. _17.2.5: quincy#v17-2-5-quincy
 .. _17.2.6: quincy#v17-2-6-quincy
+.. _17.2.7: quincy#v17-2-7-quincy
 
 .. _Pacific: pacific
 .. _16.2.14: pacific#v16-2-14-pacific
diff --git a/doc/releases/quincy.rst b/doc/releases/quincy.rst
index 18f3a3ff573b..89653ae935a1 100644
--- a/doc/releases/quincy.rst
+++ b/doc/releases/quincy.rst
@@ -5,6 +5,594 @@ Quincy
 Quincy is the 17th stable release of Ceph.  It is named after Squidward
 Quincy Tentacles from Spongebob Squarepants.
 
+v17.2.7 Quincy
+==============
+
+This is the seventh backport release in the Quincy series. We recommend
+that all users update to this release.
+
+Notable Changes
+---------------
+
+* `ceph mgr dump` command now displays the name of the Manager module that
+  registered a RADOS client in the `name` field added to elements of the
+  `active_clients` array. Previously, only the address of a module's RADOS
+  client was shown in the `active_clients` array.
+
+* mClock Scheduler: The mClock scheduler (default scheduler in Quincy) has
+  undergone significant usability and design improvements to address the slow
+  backfill issue. Some important changes are:
+
+  * The 'balanced' profile is set as the default mClock profile because it
+    represents a compromise between prioritizing client IO or recovery IO. Users
+    can then choose either the 'high_client_ops' profile to prioritize client IO
+    or the 'high_recovery_ops' profile to prioritize recovery IO.
+
+  * QoS parameters including reservation and limit are now specified in terms
+    of a fraction (range: 0.0 to 1.0) of the OSD's IOPS capacity.
+
+  * The cost parameters (osd_mclock_cost_per_io_usec_* and
+    osd_mclock_cost_per_byte_usec_*) have been removed. The cost of an operation
+    is now determined using the random IOPS and maximum sequential bandwidth
+    capability of the OSD's underlying device.
+
+  * Degraded object recovery is given higher priority when compared to misplaced
+    object recovery because degraded objects present a data safety issue not
+    present with objects that are merely misplaced. Therefore, backfilling
+    operations with the 'balanced' and 'high_client_ops' mClock profiles may
+    progress slower than what was seen with the 'WeightedPriorityQueue' (WPQ)
+    scheduler.
+
+  * The QoS allocations in all mClock profiles are optimized based on the above
+    fixes and enhancements.
+
+  * For more detailed information see:
+    https://docs.ceph.com/en/quincy/rados/configuration/mclock-config-ref/
+
+* RGW: S3 multipart uploads using Server-Side Encryption now replicate
+  correctly in multi-site. Previously, the replicas of such objects were
+  corrupted on decryption.  A new tool, ``radosgw-admin bucket resync encrypted
+  multipart``, can be used to identify these original multipart uploads. The
+  ``LastModified`` timestamp of any identified object is incremented by 1
+  nanosecond to cause peer zones to replicate it again.  For multi-site
+  deployments that make any use of Server-Side Encryption, we recommended
+  running this command against every bucket in every zone after all zones have
+  upgraded.
+
+* CephFS: MDS evicts clients which are not advancing their request tids which
+  causes a large buildup of session metadata resulting in the MDS going
+  read-only due to the RADOS operation exceeding the size threshold.
+  `mds_session_metadata_threshold` config controls the maximum size that a
+  (encoded) session metadata can grow.
+
+* CephFS: After recovering a Ceph File System post following the disaster
+  recovery procedure, the recovered files under `lost+found` directory can now
+  be deleted.
+
+Changelog
+---------
+
+* .github: Clarify checklist details (`pr#54131 <https://github.com/ceph/ceph/pull/54131>`_, Anthony D'Atri)
+* .github: Give folks 30 seconds to fill out the checklist (`pr#51944 <https://github.com/ceph/ceph/pull/51944>`_, David Galloway)
+* [CVE-2023-43040] rgw: Fix bucket validation against POST policies (`pr#53757 <https://github.com/ceph/ceph/pull/53757>`_, Joshua Baergen)
+* backport commit 70425c7 -- client/fuse: set max_idle_threads to the correct value (critical, ceph-fuse with libfuse3 is nearly useless without it) (`pr#50668 <https://github.com/ceph/ceph/pull/50668>`_, Zhansong Gao)
+* blk/kernel: Add O_EXCL for block devices (`pr#53566 <https://github.com/ceph/ceph/pull/53566>`_, Adam Kupczyk)
+* blk/kernel: Fix error code mapping in KernelDevice::read (`pr#49984 <https://github.com/ceph/ceph/pull/49984>`_, Joshua Baergen)
+* blk/KernelDevice: Modify the rotational and discard check log message (`pr#50323 <https://github.com/ceph/ceph/pull/50323>`_, Vikhyat Umrao)
+* Bluestore: fix bluestore collection_list latency perf counter (`pr#52951 <https://github.com/ceph/ceph/pull/52951>`_, Wangwenjuan)
+* build: make it possible to build w/o ceph-mgr (`pr#54132 <https://github.com/ceph/ceph/pull/54132>`_, J. Eric Ivancich)
+* build: Remove ceph-libboost\* packages in install-deps (`pr#52564 <https://github.com/ceph/ceph/pull/52564>`_, Nizamudeen A, Adam Emerson)
+* ceph-volume/cephadm: support lv devices in inventory (`pr#53287 <https://github.com/ceph/ceph/pull/53287>`_, Guillaume Abrioux)
+* ceph-volume: add --osd-id option to raw prepare (`pr#52929 <https://github.com/ceph/ceph/pull/52929>`_, Guillaume Abrioux)
+* ceph-volume: fix a bug in `get_lvm_fast_allocs()` (batch) (`pr#52062 <https://github.com/ceph/ceph/pull/52062>`_, Guillaume Abrioux)
+* ceph-volume: fix batch refactor issue (`pr#51206 <https://github.com/ceph/ceph/pull/51206>`_, Guillaume Abrioux)
+* ceph-volume: fix drive-group issue that expects the batch_args to be a string (`pr#51210 <https://github.com/ceph/ceph/pull/51210>`_, Mohan Sharma)
+* ceph-volume: fix inventory with device arg (`pr#48125 <https://github.com/ceph/ceph/pull/48125>`_, Guillaume Abrioux)
+* ceph-volume: fix issue with fast device allocs when there are multiple PVs per VG (`pr#50879 <https://github.com/ceph/ceph/pull/50879>`_, Cory Snyder)
+* ceph-volume: fix mpath device support (`pr#53540 <https://github.com/ceph/ceph/pull/53540>`_, Guillaume Abrioux)
+* ceph-volume: fix raw list for lvm devices (`pr#52620 <https://github.com/ceph/ceph/pull/52620>`_, Guillaume Abrioux)
+* ceph-volume: quick fix in zap.py (`pr#51195 <https://github.com/ceph/ceph/pull/51195>`_, Guillaume Abrioux)
+* ceph-volume: set lvm membership for mpath type devices (`pr#52079 <https://github.com/ceph/ceph/pull/52079>`_, Guillaume Abrioux)
+* ceph-volume: update the OS before deploying Ceph (quincy) (`pr#50995 <https://github.com/ceph/ceph/pull/50995>`_, Guillaume Abrioux)
+* ceph: allow xlock state to be LOCK_PREXLOCK when putting it (`pr#53663 <https://github.com/ceph/ceph/pull/53663>`_, Xiubo Li)
+* ceph_volume: support encrypted volumes for lvm new-db/new-wal/migrate commands (`pr#52874 <https://github.com/ceph/ceph/pull/52874>`_, Igor Fedotov)
+* cephadm: eliminate duplication of sections (`pr#51432 <https://github.com/ceph/ceph/pull/51432>`_, Rongqi Sun)
+* cephadm: fix call timeout argument (`pr#52909 <https://github.com/ceph/ceph/pull/52909>`_, John Mulligan)
+* cephadm: handle exceptions applying extra services during bootstrap (`pr#50904 <https://github.com/ceph/ceph/pull/50904>`_, Adam King)
+* cephadm: mount host /etc/hosts for daemon containers in podman deployments (`pr#50902 <https://github.com/ceph/ceph/pull/50902>`_, Adam King, Ilya Dryomov)
+* cephadm: reschedule haproxy from an offline host (`pr#51216 <https://github.com/ceph/ceph/pull/51216>`_, Michael Fritch)
+* cephadm: set --ulimit nofiles with Docker (`pr#50890 <https://github.com/ceph/ceph/pull/50890>`_, Michal Nasiadka)
+* cephadm: Split multicast interface and unicast_ip in keepalived.conf (`pr#53098 <https://github.com/ceph/ceph/pull/53098>`_, Luis Domingues)
+* cephadm: using ip instead of short hostname for prometheus urls (`pr#50905 <https://github.com/ceph/ceph/pull/50905>`_, Redouane Kachach)
+* cephfs-journal-tool: disambiguate usage of all keyword (in tool help) (`pr#53285 <https://github.com/ceph/ceph/pull/53285>`_, Manish M Yathnalli)
+* cephfs-mirror: do not run concurrent C_RestartMirroring context (`issue#62072 <http://tracker.ceph.com/issues/62072>`_, `pr#53639 <https://github.com/ceph/ceph/pull/53639>`_, Venky Shankar)
+* cephfs-top: check the minimum compatible python version (`pr#51354 <https://github.com/ceph/ceph/pull/51354>`_, Jos Collin)
+* cephfs-top: dump values to stdout and -d [--delay] option fix (`pr#50717 <https://github.com/ceph/ceph/pull/50717>`_, Jos Collin, Neeraj Pratap Singh, wangxinyu, Rishabh Dave)
+* cephfs-top: Handle `METRIC_TYPE_NONE` fields for sorting (`pr#50595 <https://github.com/ceph/ceph/pull/50595>`_, Neeraj Pratap Singh)
+* cephfs-top: include the missing fields in --dump output (`pr#53454 <https://github.com/ceph/ceph/pull/53454>`_, Jos Collin)
+* cephfs-top: navigate to home screen when no fs (`pr#50731 <https://github.com/ceph/ceph/pull/50731>`_, Jos Collin)
+* cephfs-top: Some fixes in `choose_field()` for sorting (`pr#50365 <https://github.com/ceph/ceph/pull/50365>`_, Neeraj Pratap Singh)
+* cephfs_mirror: correctly set top level dir permissions (`pr#50528 <https://github.com/ceph/ceph/pull/50528>`_, Milind Changire)
+* client: clear the suid/sgid in fallocate path (`pr#50989 <https://github.com/ceph/ceph/pull/50989>`_, Lucian Petrut, Xiubo Li)
+* client: do not send metrics until the MDS rank is ready (`pr#52502 <https://github.com/ceph/ceph/pull/52502>`_, Xiubo Li)
+* client: force sending cap revoke ack always (`pr#52508 <https://github.com/ceph/ceph/pull/52508>`_, Xiubo Li)
+* client: issue a cap release immediately if no cap exists (`pr#52851 <https://github.com/ceph/ceph/pull/52851>`_, Xiubo Li)
+* client: move the Inode to new auth mds session when changing auth cap (`pr#53664 <https://github.com/ceph/ceph/pull/53664>`_, Xiubo Li)
+* client: only wait for write MDS OPs when unmounting (`pr#52303 <https://github.com/ceph/ceph/pull/52303>`_, Xiubo Li)
+* client: trigger to flush the buffer when making snapshot (`pr#52498 <https://github.com/ceph/ceph/pull/52498>`_, Xiubo Li)
+* client: use deep-copy when setting permission during make_request (`pr#51486 <https://github.com/ceph/ceph/pull/51486>`_, Mer Xuanyi)
+* client: wait rename to finish (`pr#52503 <https://github.com/ceph/ceph/pull/52503>`_, Xiubo Li)
+* common: avoid redefining clock type on Windows (`pr#50573 <https://github.com/ceph/ceph/pull/50573>`_, Lucian Petrut)
+* Consider setting "bulk" autoscale pool flag when automatically creating a data pool for CephFS (`pr#52902 <https://github.com/ceph/ceph/pull/52902>`_, Leonid Usov)
+* debian: install cephfs-mirror systemd unit files and man page (`pr#52074 <https://github.com/ceph/ceph/pull/52074>`_, Jos Collin)
+* doc,test: clean up crush rule min/max_size leftovers (`pr#52169 <https://github.com/ceph/ceph/pull/52169>`_, Ilya Dryomov)
+* doc/architecture.rst - edit a sentence (`pr#53373 <https://github.com/ceph/ceph/pull/53373>`_, Zac Dover)
+* doc/architecture.rst - edit up to "Cluster Map" (`pr#53367 <https://github.com/ceph/ceph/pull/53367>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (`pr#53620 <https://github.com/ceph/ceph/pull/53620>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (one of several) (`pr#53586 <https://github.com/ceph/ceph/pull/53586>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (one of several) (`pr#53492 <https://github.com/ceph/ceph/pull/53492>`_, Zac Dover)
+* doc/architecture: edit "Calculating PG IDs" (`pr#53749 <https://github.com/ceph/ceph/pull/53749>`_, Zac Dover)
+* doc/architecture: edit "Cluster Map" (`pr#53435 <https://github.com/ceph/ceph/pull/53435>`_, Zac Dover)
+* doc/architecture: edit "Data Scrubbing" (`pr#53731 <https://github.com/ceph/ceph/pull/53731>`_, Zac Dover)
+* doc/architecture: Edit "HA Auth" (`pr#53489 <https://github.com/ceph/ceph/pull/53489>`_, Zac Dover)
+* doc/architecture: edit "HA Authentication" (`pr#53633 <https://github.com/ceph/ceph/pull/53633>`_, Zac Dover)
+* doc/architecture: edit "High Avail. Monitors" (`pr#53452 <https://github.com/ceph/ceph/pull/53452>`_, Zac Dover)
+* doc/architecture: edit "OSD Membership and Status" (`pr#53728 <https://github.com/ceph/ceph/pull/53728>`_, Zac Dover)
+* doc/architecture: edit "OSDs service clients directly" (`pr#53687 <https://github.com/ceph/ceph/pull/53687>`_, Zac Dover)
+* doc/architecture: edit "Peering and Sets" (`pr#53872 <https://github.com/ceph/ceph/pull/53872>`_, Zac Dover)
+* doc/architecture: edit "Replication" (`pr#53739 <https://github.com/ceph/ceph/pull/53739>`_, Zac Dover)
+* doc/architecture: edit "SDEH" (`pr#53660 <https://github.com/ceph/ceph/pull/53660>`_, Zac Dover)
+* doc/architecture: edit several sections (`pr#53743 <https://github.com/ceph/ceph/pull/53743>`_, Zac Dover)
+* doc/architecture: repair RBD sentence (`pr#53878 <https://github.com/ceph/ceph/pull/53878>`_, Zac Dover)
+* doc/cephadm: add ssh note to install.rst (`pr#53200 <https://github.com/ceph/ceph/pull/53200>`_, Zac Dover)
+* doc/cephadm: edit "Adding Hosts" in install.rst (`pr#53226 <https://github.com/ceph/ceph/pull/53226>`_, Zac Dover)
+* doc/cephadm: edit sentence in mgr.rst (`pr#53165 <https://github.com/ceph/ceph/pull/53165>`_, Zac Dover)
+* doc/cephadm: fix typo in cephadm initial crush location section (`pr#52888 <https://github.com/ceph/ceph/pull/52888>`_, John Mulligan)
+* doc/cephfs: add note to isolate metadata pool osds (`pr#52464 <https://github.com/ceph/ceph/pull/52464>`_, Patrick Donnelly)
+* doc/cephfs: edit fs-volumes.rst (1 of x) (`pr#51466 <https://github.com/ceph/ceph/pull/51466>`_, Zac Dover)
+* doc/cephfs: explain cephfs data and metadata set (`pr#51236 <https://github.com/ceph/ceph/pull/51236>`_, Zac Dover)
+* doc/cephfs: fix prompts in fs-volumes.rst (`pr#51435 <https://github.com/ceph/ceph/pull/51435>`_, Zac Dover)
+* doc/cephfs: Improve fs-volumes.rst (`pr#50831 <https://github.com/ceph/ceph/pull/50831>`_, Anthony D'Atri)
+* doc/cephfs: line-edit "Mirroring Module" (`pr#51543 <https://github.com/ceph/ceph/pull/51543>`_, Zac Dover)
+* doc/cephfs: rectify prompts in fs-volumes.rst (`pr#51459 <https://github.com/ceph/ceph/pull/51459>`_, Zac Dover)
+* doc/cephfs: repairing inaccessible FSes (`pr#51372 <https://github.com/ceph/ceph/pull/51372>`_, Zac Dover)
+* doc/cephfs: write cephfs commands fully in docs (`pr#53401 <https://github.com/ceph/ceph/pull/53401>`_, Rishabh Dave)
+* doc/configuration: edit "bg" in mon-config-ref.rst (`pr#53348 <https://github.com/ceph/ceph/pull/53348>`_, Zac Dover)
+* doc/dev/encoding.txt: update per std::optional (`pr#51398 <https://github.com/ceph/ceph/pull/51398>`_, Radoslaw Zarzynski)
+* doc/dev: backport deduplication.rst to Quincy (`pr#53533 <https://github.com/ceph/ceph/pull/53533>`_, Zac Dover)
+* doc/dev: fix "deploying dev cluster" link (`pr#52035 <https://github.com/ceph/ceph/pull/52035>`_, Zac Dover)
+* doc/dev: Fix typos in files cephfs-mirroring.rst and deduplication.rst (`pr#53541 <https://github.com/ceph/ceph/pull/53541>`_, Daniel Parkes)
+* doc/dev: format command in cephfs-mirroring (`pr#51108 <https://github.com/ceph/ceph/pull/51108>`_, Zac Dover)
+* doc/dev: remove seqdiag assets (`pr#52310 <https://github.com/ceph/ceph/pull/52310>`_, Zac Dover)
+* doc/foundation: Updating foundation members for July 2023 (`pr#54064 <https://github.com/ceph/ceph/pull/54064>`_, Mike Perez)
+* doc/glossary: add "Hybrid Storage" (`pr#51097 <https://github.com/ceph/ceph/pull/51097>`_, Zac Dover)
+* doc/glossary: add "primary affinity" to glossary (`pr#53428 <https://github.com/ceph/ceph/pull/53428>`_, Zac Dover)
+* doc/glossary: add "Scrubbing" (`pr#50702 <https://github.com/ceph/ceph/pull/50702>`_, Zac Dover)
+* doc/glossary: add "User" (`pr#50672 <https://github.com/ceph/ceph/pull/50672>`_, Zac Dover)
+* doc/glossary: improve "CephX" entry (`pr#51064 <https://github.com/ceph/ceph/pull/51064>`_, Zac Dover)
+* doc/glossary: link to CephX Config ref (`pr#50708 <https://github.com/ceph/ceph/pull/50708>`_, Zac Dover)
+* doc/glossary: update bluestore entry (`pr#51694 <https://github.com/ceph/ceph/pull/51694>`_, Zac Dover)
+* doc/man/8: improve radosgw-admin.rst (`pr#53268 <https://github.com/ceph/ceph/pull/53268>`_, Anthony D'Atri)
+* doc/man: radosgw-admin.rst typo (`pr#53316 <https://github.com/ceph/ceph/pull/53316>`_, Zac Dover)
+* doc/man: remove docs about support for unix domain sockets (`pr#53313 <https://github.com/ceph/ceph/pull/53313>`_, Zac Dover)
+* doc/mgr/ceph_api: Promptify example commands in index.rst (`pr#52696 <https://github.com/ceph/ceph/pull/52696>`_, Ville Ojamo)
+* doc/mgr/dashboard: fix a typo (`pr#52142 <https://github.com/ceph/ceph/pull/52142>`_, Guido Santella)
+* doc/mgr/prometheus: fix confval reference (`pr#51093 <https://github.com/ceph/ceph/pull/51093>`_, Piotr Parczewski)
+* doc/mgr/rgw.rst: add missing "ceph" command in cli specification (`pr#52487 <https://github.com/ceph/ceph/pull/52487>`_, Ville Ojamo)
+* doc/mgr/rgw.rst: multisite typed wrong (`pr#52479 <https://github.com/ceph/ceph/pull/52479>`_, Ville Ojamo)
+* doc/mgr: edit "leaderboard" in telemetry.rst (`pr#51721 <https://github.com/ceph/ceph/pull/51721>`_, Zac Dover)
+* doc/mgr: update prompts in prometheus.rst (`pr#51310 <https://github.com/ceph/ceph/pull/51310>`_, Zac Dover)
+* doc/msgr2: update dual stack status (`pr#50800 <https://github.com/ceph/ceph/pull/50800>`_, Dan van der Ster)
+* doc/operations: fix prompt in bluestore-migration (`pr#50662 <https://github.com/ceph/ceph/pull/50662>`_, Zac Dover)
+* doc/rados/config: edit auth-config-ref (`pr#50950 <https://github.com/ceph/ceph/pull/50950>`_, Zac Dover)
+* doc/rados/configuration: add links to MON DNS (`pr#52613 <https://github.com/ceph/ceph/pull/52613>`_, Ville Ojamo)
+* doc/rados/configuration: Avoid repeating "support" in msgr2.rst (`pr#52999 <https://github.com/ceph/ceph/pull/52999>`_, Ville Ojamo)
+* doc/rados/operations: Acting Set question (`pr#51740 <https://github.com/ceph/ceph/pull/51740>`_, Zac Dover)
+* doc/rados/operations: edit monitoring.rst (`pr#51036 <https://github.com/ceph/ceph/pull/51036>`_, Zac Dover)
+* doc/rados/operations: Fix erasure-code-jerasure.rst fix (`pr#51743 <https://github.com/ceph/ceph/pull/51743>`_, Anthony D'Atri)
+* doc/rados/operations: fix typo in balancer.rst (`pr#51938 <https://github.com/ceph/ceph/pull/51938>`_, Pierre Riteau)
+* doc/rados/operations: Fix typo in erasure-code.rst (`pr#50752 <https://github.com/ceph/ceph/pull/50752>`_, Sainithin Artham)
+* doc/rados/operations: Improve formatting in crush-map.rst (`pr#52140 <https://github.com/ceph/ceph/pull/52140>`_, Anthony D'Atri)
+* doc/rados/ops: add ceph-medic documentation (`pr#50853 <https://github.com/ceph/ceph/pull/50853>`_, Zac Dover)
+* doc/rados/ops: add hyphen to mon-osd-pg.rst (`pr#50960 <https://github.com/ceph/ceph/pull/50960>`_, Zac Dover)
+* doc/rados/ops: edit health checks.rst (5 of x) (`pr#50967 <https://github.com/ceph/ceph/pull/50967>`_, Zac Dover)
+* doc/rados/ops: edit health-checks.rst (1 of x) (`pr#50797 <https://github.com/ceph/ceph/pull/50797>`_, Zac Dover)
+* doc/rados/ops: edit health-checks.rst (2 of x) (`pr#50912 <https://github.com/ceph/ceph/pull/50912>`_, Zac Dover)
+* doc/rados/ops: edit health-checks.rst (3 of x) (`pr#50953 <https://github.com/ceph/ceph/pull/50953>`_, Zac Dover)
+* doc/rados/ops: edit health-checks.rst (4 of x) (`pr#50956 <https://github.com/ceph/ceph/pull/50956>`_, Zac Dover)
+* doc/rados/ops: edit health-checks.rst (6 of x) (`pr#50970 <https://github.com/ceph/ceph/pull/50970>`_, Zac Dover)
+* doc/rados/ops: edit monitoring-osd-pg.rst (1 of x) (`pr#50865 <https://github.com/ceph/ceph/pull/50865>`_, Zac Dover)
+* doc/rados/ops: edit monitoring-osd-pg.rst (2 of x) (`pr#50946 <https://github.com/ceph/ceph/pull/50946>`_, Zac Dover)
+* doc/rados/ops: edit user-management.rst (3 of x) (`pr#51240 <https://github.com/ceph/ceph/pull/51240>`_, Zac Dover)
+* doc/rados/ops: line-edit operating.rst (`pr#50934 <https://github.com/ceph/ceph/pull/50934>`_, Zac Dover)
+* doc/rados/ops: remove ceph-medic from monitoring (`pr#51088 <https://github.com/ceph/ceph/pull/51088>`_, Zac Dover)
+* doc/rados: add bulk flag to pools.rst (`pr#53318 <https://github.com/ceph/ceph/pull/53318>`_, Zac Dover)
+* doc/rados: add link to ops/health-checks.rst (`pr#50762 <https://github.com/ceph/ceph/pull/50762>`_, Zac Dover)
+* doc/rados: add math markup to placement-groups.rst (`pr#52038 <https://github.com/ceph/ceph/pull/52038>`_, Zac Dover)
+* doc/rados: clean up ops/bluestore-migration.rst (`pr#50678 <https://github.com/ceph/ceph/pull/50678>`_, Zac Dover)
+* doc/rados: edit add-or-rm-osds (1 of x) (`pr#52384 <https://github.com/ceph/ceph/pull/52384>`_, Zac Dover)
+* doc/rados: edit add-or-rm-osds (2 of x) (`pr#52451 <https://github.com/ceph/ceph/pull/52451>`_, Zac Dover)
+* doc/rados: edit balancer.rst (`pr#51825 <https://github.com/ceph/ceph/pull/51825>`_, Zac Dover)
+* doc/rados: edit bluestore-config-ref.rst (1 of x) (`pr#51790 <https://github.com/ceph/ceph/pull/51790>`_, Zac Dover)
+* doc/rados: edit bluestore-config-ref.rst (2 of x) (`pr#51793 <https://github.com/ceph/ceph/pull/51793>`_, Zac Dover)
+* doc/rados: edit ceph-conf.rst (`pr#52449 <https://github.com/ceph/ceph/pull/52449>`_, Zac Dover)
+* doc/rados: edit ceph-conf.rst (2 of x) (`pr#52471 <https://github.com/ceph/ceph/pull/52471>`_, Zac Dover)
+* doc/rados: edit ceph-conf.rst (3 of x) (`pr#52589 <https://github.com/ceph/ceph/pull/52589>`_, Zac Dover)
+* doc/rados: edit ceph-conf.rst (4 of x) (`pr#52594 <https://github.com/ceph/ceph/pull/52594>`_, Zac Dover)
+* doc/rados: edit change-mon-elections (`pr#51999 <https://github.com/ceph/ceph/pull/51999>`_, Zac Dover)
+* doc/rados: edit control.rst (1 of x) (`pr#52153 <https://github.com/ceph/ceph/pull/52153>`_, Zac Dover)
+* doc/rados: edit crush-map-edits (2 of x) (`pr#52312 <https://github.com/ceph/ceph/pull/52312>`_, Zac Dover)
+* doc/rados: edit crush-map-edits.rst (1 of x) (`pr#52180 <https://github.com/ceph/ceph/pull/52180>`_, Zac Dover)
+* doc/rados: edit crush-map.rst (1 of x) (`pr#52031 <https://github.com/ceph/ceph/pull/52031>`_, Zac Dover)
+* doc/rados: edit crush-map.rst (2 of x) (`pr#52070 <https://github.com/ceph/ceph/pull/52070>`_, Zac Dover)
+* doc/rados: edit crush-map.rst (3 of x) (`pr#52094 <https://github.com/ceph/ceph/pull/52094>`_, Zac Dover)
+* doc/rados: edit crush-map.rst (4 of x) (`pr#52099 <https://github.com/ceph/ceph/pull/52099>`_, Zac Dover)
+* doc/rados: edit data-placement.rst (`pr#51596 <https://github.com/ceph/ceph/pull/51596>`_, Zac Dover)
+* doc/rados: edit devices.rst (`pr#51478 <https://github.com/ceph/ceph/pull/51478>`_, Zac Dover)
+* doc/rados: edit filestore-config-ref.rst (`pr#51752 <https://github.com/ceph/ceph/pull/51752>`_, Zac Dover)
+* doc/rados: edit firefly tunables section (`pr#52103 <https://github.com/ceph/ceph/pull/52103>`_, Zac Dover)
+* doc/rados: edit log-and-debug.rst (1 of x) (`pr#51903 <https://github.com/ceph/ceph/pull/51903>`_, Zac Dover)
+* doc/rados: edit log-and-debug.rst (2 of x) (`pr#51907 <https://github.com/ceph/ceph/pull/51907>`_, Zac Dover)
+* doc/rados: edit memory-profiling.rst (`pr#53933 <https://github.com/ceph/ceph/pull/53933>`_, Zac Dover)
+* doc/rados: edit operations/add-or-rm-mons (1 of x) (`pr#52890 <https://github.com/ceph/ceph/pull/52890>`_, Zac Dover)
+* doc/rados: edit operations/add-or-rm-mons (2 of x) (`pr#52826 <https://github.com/ceph/ceph/pull/52826>`_, Zac Dover)
+* doc/rados: edit operations/bs-migration (1 of x) (`pr#50587 <https://github.com/ceph/ceph/pull/50587>`_, Zac Dover)
+* doc/rados: edit operations/bs-migration (2 of x) (`pr#50590 <https://github.com/ceph/ceph/pull/50590>`_, Zac Dover)
+* doc/rados: edit ops/control.rst (1 of x) (`pr#53812 <https://github.com/ceph/ceph/pull/53812>`_, zdover23, Zac Dover)
+* doc/rados: edit ops/control.rst (2 of x) (`pr#53816 <https://github.com/ceph/ceph/pull/53816>`_, Zac Dover)
+* doc/rados: edit ops/monitoring.rst (1 of 3) (`pr#50823 <https://github.com/ceph/ceph/pull/50823>`_, Zac Dover)
+* doc/rados: edit ops/monitoring.rst (2 of 3) (`pr#50849 <https://github.com/ceph/ceph/pull/50849>`_, Zac Dover)
+* doc/rados: edit placement-groups.rst (1 of x) (`pr#51985 <https://github.com/ceph/ceph/pull/51985>`_, Zac Dover)
+* doc/rados: edit placement-groups.rst (2 of x) (`pr#51997 <https://github.com/ceph/ceph/pull/51997>`_, Zac Dover)
+* doc/rados: edit placement-groups.rst (3 of x) (`pr#52002 <https://github.com/ceph/ceph/pull/52002>`_, Zac Dover)
+* doc/rados: edit pools.rst (1 of x) (`pr#51913 <https://github.com/ceph/ceph/pull/51913>`_, Zac Dover)
+* doc/rados: edit pools.rst (2 of x) (`pr#51940 <https://github.com/ceph/ceph/pull/51940>`_, Zac Dover)
+* doc/rados: edit pools.rst (3 of x) (`pr#51957 <https://github.com/ceph/ceph/pull/51957>`_, Zac Dover)
+* doc/rados: edit pools.rst (4 of x) (`pr#51971 <https://github.com/ceph/ceph/pull/51971>`_, Zac Dover)
+* doc/rados: edit stretch-mode procedure (`pr#51290 <https://github.com/ceph/ceph/pull/51290>`_, Zac Dover)
+* doc/rados: edit stretch-mode.rst (`pr#51338 <https://github.com/ceph/ceph/pull/51338>`_, Zac Dover)
+* doc/rados: edit stretch-mode.rst (`pr#51303 <https://github.com/ceph/ceph/pull/51303>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (1 of x) (`pr#51905 <https://github.com/ceph/ceph/pull/51905>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (2 of x) (`pr#52840 <https://github.com/ceph/ceph/pull/52840>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (3 of x) (`pr#53880 <https://github.com/ceph/ceph/pull/53880>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (4 of x) (`pr#53898 <https://github.com/ceph/ceph/pull/53898>`_, Zac Dover)
+* doc/rados: edit troubleshooting-osd (1 of x) (`pr#53983 <https://github.com/ceph/ceph/pull/53983>`_, Zac Dover)
+* doc/rados: Edit troubleshooting-osd (2 of x) (`pr#54001 <https://github.com/ceph/ceph/pull/54001>`_, Zac Dover)
+* doc/rados: Edit troubleshooting-osd (3 of x) (`pr#54027 <https://github.com/ceph/ceph/pull/54027>`_, Zac Dover)
+* doc/rados: edit troubleshooting-pg (2 of x) (`pr#54115 <https://github.com/ceph/ceph/pull/54115>`_, Zac Dover)
+* doc/rados: edit troubleshooting-pg.rst (1 of x) (`pr#54074 <https://github.com/ceph/ceph/pull/54074>`_, Zac Dover)
+* doc/rados: edit troubleshooting.rst (`pr#53838 <https://github.com/ceph/ceph/pull/53838>`_, Zac Dover)
+* doc/rados: edit troubleshooting/community.rst (`pr#53882 <https://github.com/ceph/ceph/pull/53882>`_, Zac Dover)
+* doc/rados: edit user-management (2 of x) (`pr#51156 <https://github.com/ceph/ceph/pull/51156>`_, Zac Dover)
+* doc/rados: edit user-management.rst (1 of x) (`pr#50641 <https://github.com/ceph/ceph/pull/50641>`_, Zac Dover)
+* doc/rados: fix link in common.rst (`pr#51756 <https://github.com/ceph/ceph/pull/51756>`_, Zac Dover)
+* doc/rados: fix list in crush-map.rst (`pr#52066 <https://github.com/ceph/ceph/pull/52066>`_, Zac Dover)
+* doc/rados: fix typos in pg-repair.rst (`pr#51898 <https://github.com/ceph/ceph/pull/51898>`_, Zac Dover)
+* doc/rados: introduce emdash (`pr#52382 <https://github.com/ceph/ceph/pull/52382>`_, Zac Dover)
+* doc/rados: line edit mon-lookup-dns top matter (`pr#50582 <https://github.com/ceph/ceph/pull/50582>`_, Zac Dover)
+* doc/rados: line-edit common.rst (`pr#50943 <https://github.com/ceph/ceph/pull/50943>`_, Zac Dover)
+* doc/rados: line-edit devices.rst (`pr#51577 <https://github.com/ceph/ceph/pull/51577>`_, Zac Dover)
+* doc/rados: line-edit erasure-code.rst (`pr#50619 <https://github.com/ceph/ceph/pull/50619>`_, Zac Dover)
+* doc/rados: line-edit pg-repair.rst (`pr#50803 <https://github.com/ceph/ceph/pull/50803>`_, Zac Dover)
+* doc/rados: line-edit upmap.rst (`pr#50566 <https://github.com/ceph/ceph/pull/50566>`_, Zac Dover)
+* doc/rados: m-config-ref: edit "background" (`pr#51273 <https://github.com/ceph/ceph/pull/51273>`_, Zac Dover)
+* doc/rados: pools.rst: "decreaesed" (`pr#51920 <https://github.com/ceph/ceph/pull/51920>`_, Zac Dover)
+* doc/rados: remove git tag in placement-groups in q (`pr#51990 <https://github.com/ceph/ceph/pull/51990>`_, Zac Dover)
+* doc/rados: stretch-mode.rst (other commands) (`pr#51390 <https://github.com/ceph/ceph/pull/51390>`_, Zac Dover)
+* doc/rados: stretch-mode: stretch cluster issues (`pr#51378 <https://github.com/ceph/ceph/pull/51378>`_, Zac Dover)
+* doc/rados: update monitoring-osd-pg.rst (`pr#52959 <https://github.com/ceph/ceph/pull/52959>`_, Zac Dover)
+* doc/radosgw: Add missing space to date option spec in admin.rst (`pr#52694 <https://github.com/ceph/ceph/pull/52694>`_, Ville Ojamo)
+* doc/radosgw: add Zonegroup policy explanation (`pr#52362 <https://github.com/ceph/ceph/pull/52362>`_, Zac Dover)
+* doc/radosgw: add Zonegroup purpose (`pr#52349 <https://github.com/ceph/ceph/pull/52349>`_, Zac Dover)
+* doc/radosgw: correct emphasis in rate limit section (`pr#52713 <https://github.com/ceph/ceph/pull/52713>`_, Piotr Parczewski)
+* doc/radosgw: edit "Basic Workflow" in s3select.rst (`pr#52263 <https://github.com/ceph/ceph/pull/52263>`_, Zac Dover)
+* doc/radosgw: edit "Overview" in s3select.rst (`pr#52220 <https://github.com/ceph/ceph/pull/52220>`_, Zac Dover)
+* doc/radosgw: explain multisite dynamic sharding (`pr#51586 <https://github.com/ceph/ceph/pull/51586>`_, Zac Dover)
+* doc/radosgw: fix command error blank (`pr#53656 <https://github.com/ceph/ceph/pull/53656>`_, stevenhua)
+* doc/radosgw: format part of s3select (`pr#51117 <https://github.com/ceph/ceph/pull/51117>`_, Cole Mitchell)
+* doc/radosgw: format part of s3select (`pr#51105 <https://github.com/ceph/ceph/pull/51105>`_, Cole Mitchell)
+* doc/radosgw: Improve language and formatting in config-ref.rst (`pr#52836 <https://github.com/ceph/ceph/pull/52836>`_, Ville Ojamo)
+* doc/radosgw: multisite - edit "migrating a single-site" (`pr#53262 <https://github.com/ceph/ceph/pull/53262>`_, Qi Tao)
+* doc/radosgw: rabbitmq - push-endpoint edit (`pr#51306 <https://github.com/ceph/ceph/pull/51306>`_, Zac Dover)
+* doc/radosgw: refine "Zones" in multisite.rst (`pr#52282 <https://github.com/ceph/ceph/pull/52282>`_, Zac Dover)
+* doc/radosgw: remove pipes from s3select.rst (`pr#52188 <https://github.com/ceph/ceph/pull/52188>`_, Zac Dover)
+* doc/radosgw: remove pipes from s3select.rst (`pr#52184 <https://github.com/ceph/ceph/pull/52184>`_, Zac Dover)
+* doc/radosgw: s/s3select/S3 Select/ (`pr#52279 <https://github.com/ceph/ceph/pull/52279>`_, Zac Dover)
+* doc/radosgw: update rate limit management (`pr#52911 <https://github.com/ceph/ceph/pull/52911>`_, Zac Dover)
+* doc/README.md - edit "Building Ceph" (`pr#53058 <https://github.com/ceph/ceph/pull/53058>`_, Zac Dover)
+* doc/README.md - improve "Running a test cluster" (`pr#53259 <https://github.com/ceph/ceph/pull/53259>`_, Zac Dover)
+* doc/rgw/lua: add info uploading a script in cephadm deployment (`pr#52299 <https://github.com/ceph/ceph/pull/52299>`_, Yuval Lifshitz)
+* doc/rgw: refine "Setting a Zonegroup" (`pr#51072 <https://github.com/ceph/ceph/pull/51072>`_, Zac Dover)
+* doc/rgw: several response headers are supported (`pr#52804 <https://github.com/ceph/ceph/pull/52804>`_, Casey Bodley)
+* doc/start/os-recommendations: drop 4.14 kernel and reword guidance (`pr#51490 <https://github.com/ceph/ceph/pull/51490>`_, Ilya Dryomov)
+* doc/start: documenting-ceph - add squash procedure (`pr#50740 <https://github.com/ceph/ceph/pull/50740>`_, Zac Dover)
+* doc/start: edit first 150 lines of documenting-ceph (`pr#51182 <https://github.com/ceph/ceph/pull/51182>`_, Zac Dover)
+* doc/start: edit os-recommendations.rst (`pr#53180 <https://github.com/ceph/ceph/pull/53180>`_, Zac Dover)
+* doc/start: fix "Planet Ceph" link (`pr#51420 <https://github.com/ceph/ceph/pull/51420>`_, Zac Dover)
+* doc/start: format procedure in documenting-ceph (`pr#50788 <https://github.com/ceph/ceph/pull/50788>`_, Zac Dover)
+* doc/start: KRBD feature flag support note (`pr#51503 <https://github.com/ceph/ceph/pull/51503>`_, Zac Dover)
+* doc/start: Modernize and clarify hardware-recommendations.rst (`pr#54072 <https://github.com/ceph/ceph/pull/54072>`_, Anthony D'Atri)
+* doc/start: rewrite intro paragraph (`pr#51221 <https://github.com/ceph/ceph/pull/51221>`_, Zac Dover)
+* doc/start: update "notify us" section (`pr#50770 <https://github.com/ceph/ceph/pull/50770>`_, Zac Dover)
+* doc/start: update linking conventions (`pr#52913 <https://github.com/ceph/ceph/pull/52913>`_, Zac Dover)
+* doc/start: update linking conventions (`pr#52842 <https://github.com/ceph/ceph/pull/52842>`_, Zac Dover)
+* doc/troubleshooting: edit cpu-profiling.rst (`pr#53060 <https://github.com/ceph/ceph/pull/53060>`_, Zac Dover)
+* doc: Add a note on possible deadlock on volume deletion (`pr#52947 <https://github.com/ceph/ceph/pull/52947>`_, Kotresh HR)
+* doc: add information on expediting MDS recovery (`pr#52368 <https://github.com/ceph/ceph/pull/52368>`_, Patrick Donnelly)
+* doc: add link to "documenting ceph" to index.rst (`pr#51470 <https://github.com/ceph/ceph/pull/51470>`_, Zac Dover)
+* doc: Add missing `ceph` command in documentation section `REPLACING A… (`pr#51620 <https://github.com/ceph/ceph/pull/51620>`_, Alexander Proschek)
+* doc: add note for removing (automatic) partitioning policy (`pr#53570 <https://github.com/ceph/ceph/pull/53570>`_, Venky Shankar)
+* doc: Add warning on manual CRUSH rule removal (`pr#53421 <https://github.com/ceph/ceph/pull/53421>`_, Alvin Owyong)
+* doc: deprecate the cache tiering (`pr#51653 <https://github.com/ceph/ceph/pull/51653>`_, Radosław Zarzyński)
+* doc: Documentation about main Ceph metrics (`pr#54112 <https://github.com/ceph/ceph/pull/54112>`_, Juan Miguel Olmo Martínez)
+* doc: edit README.md - contributing code (`pr#53050 <https://github.com/ceph/ceph/pull/53050>`_, Zac Dover)
+* doc: expand and consolidate mds placement (`pr#53147 <https://github.com/ceph/ceph/pull/53147>`_, Patrick Donnelly)
+* doc: explain cephfs mirroring `peer_add` step in detail (`pr#51521 <https://github.com/ceph/ceph/pull/51521>`_, Venky Shankar)
+* doc: Fix doc for mds cap acquisition throttle (`pr#53025 <https://github.com/ceph/ceph/pull/53025>`_, Kotresh HR)
+* doc: for EC we recommend K+1 (`pr#52780 <https://github.com/ceph/ceph/pull/52780>`_, Dan van der Ster)
+* doc: governance.rst - update D Orman (`pr#52573 <https://github.com/ceph/ceph/pull/52573>`_, Zac Dover)
+* doc: improve doc/dev/encoding.rst (`pr#52759 <https://github.com/ceph/ceph/pull/52759>`_, Radosław Zarzyński)
+* doc: improve submodule update command - README.md (`pr#53001 <https://github.com/ceph/ceph/pull/53001>`_, Zac Dover)
+* doc: remove egg fragment from dev/developer_guide/running-tests-locally (`pr#53854 <https://github.com/ceph/ceph/pull/53854>`_, Dhairya Parmar)
+* doc: Update jerasure.org references (`pr#51726 <https://github.com/ceph/ceph/pull/51726>`_, Anthony D'Atri)
+* doc: Update mClock QOS documentation to discard osd_mclock_cost_per\_\* (`pr#54080 <https://github.com/ceph/ceph/pull/54080>`_, tanchangzhi)
+* doc: update multisite doc (`pr#51401 <https://github.com/ceph/ceph/pull/51401>`_, parth-gr)
+* doc: update rados.cc (`pr#52968 <https://github.com/ceph/ceph/pull/52968>`_, Zac Dover)
+* doc: update README.md (`pr#52642 <https://github.com/ceph/ceph/pull/52642>`_, Zac Dover)
+* doc: update README.md install procedure (`pr#52680 <https://github.com/ceph/ceph/pull/52680>`_, Zac Dover)
+* doc: update test cluster commands in README.md (`pr#53350 <https://github.com/ceph/ceph/pull/53350>`_, Zac Dover)
+* doc: Use `ceph osd crush tree` command to display weight set weights (`pr#51350 <https://github.com/ceph/ceph/pull/51350>`_, James Lakin)
+* docs: fix nfs cluster create syntax (`pr#52424 <https://github.com/ceph/ceph/pull/52424>`_, Paul Cuzner)
+* docs: Update the Prometheus endpoint info (`pr#51287 <https://github.com/ceph/ceph/pull/51287>`_, Paul Cuzner)
+* Fix FTBFS on gcc 13 (`pr#52120 <https://github.com/ceph/ceph/pull/52120>`_, Tim Serong)
+* install-deps: remove the legacy resolver flags (`pr#53706 <https://github.com/ceph/ceph/pull/53706>`_, Nizamudeen A)
+* kv/RocksDBStore: Add CompactOnDeletion support (`pr#50893 <https://github.com/ceph/ceph/pull/50893>`_, Mark Nelson)
+* kv/RocksDBStore: cumulative backport for rm_range_keys and around (`pr#50636 <https://github.com/ceph/ceph/pull/50636>`_, Igor Fedotov)
+* kv/RocksDBStore: don't use real wholespace iterator for prefixed access (`pr#50495 <https://github.com/ceph/ceph/pull/50495>`_, Igor Fedotov)
+* libcephsqlite: fill 0s in unread portion of buffer (`pr#53102 <https://github.com/ceph/ceph/pull/53102>`_, Patrick Donnelly)
+* librados: aio operate functions can set times (`pr#52118 <https://github.com/ceph/ceph/pull/52118>`_, Casey Bodley)
+* librbd/managed_lock/GetLockerRequest: Fix no valid lockers case (`pr#52288 <https://github.com/ceph/ceph/pull/52288>`_, Ilya Dryomov, Matan Breizman)
+* librbd: avoid decrementing iterator before first element (`pr#51854 <https://github.com/ceph/ceph/pull/51854>`_, Lucian Petrut)
+* librbd: avoid object map corruption in snapshots taken under I/O (`pr#52286 <https://github.com/ceph/ceph/pull/52286>`_, Ilya Dryomov)
+* librbd: don't wait for a watch in send_acquire_lock() if client is blocklisted (`pr#50920 <https://github.com/ceph/ceph/pull/50920>`_, Ilya Dryomov, Christopher Hoffman)
+* librbd: fix wrong attribute for rbd_quiesce_complete api (`pr#50873 <https://github.com/ceph/ceph/pull/50873>`_, Dongsheng Yang)
+* librbd: kick ExclusiveLock state machine on client being blocklisted when waiting for lock (`pr#53294 <https://github.com/ceph/ceph/pull/53294>`_, Ramana Raja)
+* librbd: kick ExclusiveLock state machine stalled waiting for lock from reacquire_lock() (`pr#53920 <https://github.com/ceph/ceph/pull/53920>`_, Ramana Raja)
+* librbd: localize snap_remove op for mirror snapshots (`pr#51428 <https://github.com/ceph/ceph/pull/51428>`_, Christopher Hoffman)
+* librbd: make CreatePrimaryRequest remove any unlinked mirror snapshots (`pr#53275 <https://github.com/ceph/ceph/pull/53275>`_, Ilya Dryomov)
+* librbd: remove previous incomplete primary snapshot after successfully creating a new one (`pr#51173 <https://github.com/ceph/ceph/pull/51173>`_, Ilya Dryomov, Prasanna Kumar Kalever)
+* librbd: report better errors when failing to enable mirroring on an image (`pr#50837 <https://github.com/ceph/ceph/pull/50837>`_, Prasanna Kumar Kalever)
+* log: writes to stderr (pipe) may not be atomic (`pr#50777 <https://github.com/ceph/ceph/pull/50777>`_, Lucian Petrut, Patrick Donnelly)
+* MDS imported_inodes metric is not updated (`pr#51697 <https://github.com/ceph/ceph/pull/51697>`_, Yongseok Oh)
+* mds/FSMap: allow upgrades if no up mds (`pr#53852 <https://github.com/ceph/ceph/pull/53852>`_, Patrick Donnelly)
+* mds/Server: mark a cap acquisition throttle event in the request (`pr#53167 <https://github.com/ceph/ceph/pull/53167>`_, Leonid Usov)
+* mds: acquire inode snaplock in open (`pr#53184 <https://github.com/ceph/ceph/pull/53184>`_, Patrick Donnelly)
+* mds: add event for batching getattr/lookup (`pr#53557 <https://github.com/ceph/ceph/pull/53557>`_, Patrick Donnelly)
+* mds: allow unlink from lost+found directory (`issue#59569 <http://tracker.ceph.com/issues/59569>`_, `pr#51689 <https://github.com/ceph/ceph/pull/51689>`_, Venky Shankar)
+* mds: blocklist clients with "bloated" session metadata (`issue#61947 <http://tracker.ceph.com/issues/61947>`_, `issue#62873 <http://tracker.ceph.com/issues/62873>`_, `pr#53330 <https://github.com/ceph/ceph/pull/53330>`_, Venky Shankar)
+* mds: catch damage to CDentry's first member before persisting (`issue#58482 <http://tracker.ceph.com/issues/58482>`_, `pr#50779 <https://github.com/ceph/ceph/pull/50779>`_, Patrick Donnelly)
+* mds: display sane hex value (0x0) for empty feature bit (`pr#52127 <https://github.com/ceph/ceph/pull/52127>`_, Jos Collin)
+* mds: do not send split_realms for CEPH_SNAP_OP_UPDATE msg (`pr#52849 <https://github.com/ceph/ceph/pull/52849>`_, Xiubo Li)
+* mds: do not take the ino which has been used (`pr#51507 <https://github.com/ceph/ceph/pull/51507>`_, Xiubo Li)
+* mds: drop locks and retry when lock set changes (`pr#53242 <https://github.com/ceph/ceph/pull/53242>`_, Patrick Donnelly)
+* mds: fix stray evaluation using scrub and introduce new option (`pr#50815 <https://github.com/ceph/ceph/pull/50815>`_, Dhairya Parmar)
+* mds: Fix the linkmerge assert check (`pr#52725 <https://github.com/ceph/ceph/pull/52725>`_, Kotresh HR)
+* mds: force replay sessionmap version (`pr#50724 <https://github.com/ceph/ceph/pull/50724>`_, Xiubo Li)
+* mds: make num_fwd and num_retry to __u32 (`pr#50732 <https://github.com/ceph/ceph/pull/50732>`_, Xiubo Li)
+* mds: MDLog::_recovery_thread: handle the errors gracefully (`pr#52514 <https://github.com/ceph/ceph/pull/52514>`_, Jos Collin)
+* mds: rdlock_path_xlock_dentry supports returning auth target inode (`pr#51688 <https://github.com/ceph/ceph/pull/51688>`_, Zhansong Gao)
+* mds: record and dump last tid for trimming completed requests (or flushes) (`issue#57985 <http://tracker.ceph.com/issues/57985>`_, `pr#50785 <https://github.com/ceph/ceph/pull/50785>`_, Venky Shankar)
+* mds: session ls command appears twice in command listing (`pr#52516 <https://github.com/ceph/ceph/pull/52516>`_, Neeraj Pratap Singh)
+* mds: skip forwarding request if the session were removed (`pr#52845 <https://github.com/ceph/ceph/pull/52845>`_, Xiubo Li)
+* mds: update mdlog perf counters during replay (`pr#52683 <https://github.com/ceph/ceph/pull/52683>`_, Patrick Donnelly)
+* mds: wait for unlink operation to finish (`pr#50985 <https://github.com/ceph/ceph/pull/50985>`_, Xiubo Li)
+* mds: wait reintegrate to finish when unlinking (`pr#51685 <https://github.com/ceph/ceph/pull/51685>`_, Xiubo Li)
+* mgr/cephadm: add commands to set services to managed/unmanaged (`pr#50897 <https://github.com/ceph/ceph/pull/50897>`_, Adam King)
+* mgr/cephadm: add more aggressive force flag for host maintenance enter (`pr#50901 <https://github.com/ceph/ceph/pull/50901>`_, Adam King)
+* mgr/cephadm: allow configuring anonymous access for grafana (`pr#51617 <https://github.com/ceph/ceph/pull/51617>`_, Adam King)
+* mgr/cephadm: allow setting mon crush locations through mon service spec (`pr#51217 <https://github.com/ceph/ceph/pull/51217>`_, Adam King)
+* mgr/cephadm: also don't write client files/tuned profiles to maintenance hosts (`pr#53705 <https://github.com/ceph/ceph/pull/53705>`_, Adam King)
+* mgr/cephadm: asyncio based universal timeout for ssh/cephadm commands (`pr#51218 <https://github.com/ceph/ceph/pull/51218>`_, Adam King)
+* mgr/cephadm: be aware of host's shortname and FQDN (`pr#50888 <https://github.com/ceph/ceph/pull/50888>`_, Adam King)
+* mgr/cephadm: don't add mgr into iscsi trusted_ip_list if it's already there (`pr#50521 <https://github.com/ceph/ceph/pull/50521>`_, Mykola Golub)
+* mgr/cephadm: handle HostConnectionError when checking for valid addr (`pr#50900 <https://github.com/ceph/ceph/pull/50900>`_, Adam King)
+* mgr/cephadm: increasing container stop timeout for OSDs (`pr#50903 <https://github.com/ceph/ceph/pull/50903>`_, Redouane Kachach)
+* mgr/cephadm: make upgrade respect use_repo_digest (`pr#50898 <https://github.com/ceph/ceph/pull/50898>`_, Adam King)
+* mgr/cephadm: support for nfs backed by VIP (`pr#51616 <https://github.com/ceph/ceph/pull/51616>`_, Adam King)
+* mgr/cephadm: update monitoring stack versions (`pr#51356 <https://github.com/ceph/ceph/pull/51356>`_, Nizamudeen A)
+* mgr/cephadm: use a dedicated cephadm tmp dir to copy remote files (`pr#50906 <https://github.com/ceph/ceph/pull/50906>`_, Redouane Kachach)
+* mgr/dashboard CRUD component backport (`pr#51367 <https://github.com/ceph/ceph/pull/51367>`_, Pedro Gonzalez Gomez, Pere Diaz Bou, Nizamudeen A, Ernesto Puerta)
+* mgr/dashboard: Add more decimals in latency graph (`pr#52728 <https://github.com/ceph/ceph/pull/52728>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: add popover to cluster status card (`pr#52027 <https://github.com/ceph/ceph/pull/52027>`_, Nizamudeen A)
+* mgr/dashboard: align charts of landing page (`pr#53544 <https://github.com/ceph/ceph/pull/53544>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: allow PUT in CORS (`pr#52706 <https://github.com/ceph/ceph/pull/52706>`_, Nizamudeen A)
+* mgr/dashboard: batch backport hackathon prs (`pr#51768 <https://github.com/ceph/ceph/pull/51768>`_, Nizamudeen A, Pedro Gonzalez Gomez, Ankush Behl, Pere Diaz Bou, Aashish Sharma, avanthakkar)
+* mgr/dashboard: bump moment from 2.29.3 to 2.29.4 in /src/pybind/mgr/dashboard/frontend (`pr#51358 <https://github.com/ceph/ceph/pull/51358>`_, dependabot[bot])
+* mgr/dashboard: disable promote on mirroring not enabled (`pr#52537 <https://github.com/ceph/ceph/pull/52537>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: disable protect if layering is not enabled on the image (`pr#53174 <https://github.com/ceph/ceph/pull/53174>`_, avanthakkar)
+* mgr/dashboard: enable protect option if layering enabled (`pr#53796 <https://github.com/ceph/ceph/pull/53796>`_, avanthakkar)
+* mgr/dashboard: expose more grafana configs in service form (`pr#51112 <https://github.com/ceph/ceph/pull/51112>`_, Nizamudeen A)
+* mgr/dashboard: fix a bug where data would plot wrongly (`pr#52332 <https://github.com/ceph/ceph/pull/52332>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: fix cephadm e2e expression changed error (`pr#51079 <https://github.com/ceph/ceph/pull/51079>`_, Nizamudeen A)
+* mgr/dashboard: fix CephPGImbalance alert (`pr#51252 <https://github.com/ceph/ceph/pull/51252>`_, Aashish Sharma)
+* mgr/dashboard: fix create osd default selected as recommended not working (`pr#51007 <https://github.com/ceph/ceph/pull/51007>`_, Nizamudeen A)
+* mgr/dashboard: fix displaying mirror image progress (`pr#50871 <https://github.com/ceph/ceph/pull/50871>`_, Pere Diaz Bou)
+* mgr/dashboard: fix eviction of all FS clients (`pr#51011 <https://github.com/ceph/ceph/pull/51011>`_, Pere Diaz Bou)
+* mgr/dashboard: fix image columns naming (`pr#53253 <https://github.com/ceph/ceph/pull/53253>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: fix issues with read-only user on landing page (`pr#51809 <https://github.com/ceph/ceph/pull/51809>`_, Pedro Gonzalez Gomez, Nizamudeen A)
+* mgr/dashboard: Fix rbd snapshot creation (`pr#51076 <https://github.com/ceph/ceph/pull/51076>`_, Aashish Sharma)
+* mgr/dashboard: fix regression caused by cephPgImabalance alert (`pr#51525 <https://github.com/ceph/ceph/pull/51525>`_, Aashish Sharma)
+* mgr/dashboard: fix rgw page issues when hostname not resolvable (`pr#53216 <https://github.com/ceph/ceph/pull/53216>`_, Nizamudeen A)
+* mgr/dashboard: fix test_dashboard_e2e.sh failure (`pr#51866 <https://github.com/ceph/ceph/pull/51866>`_, Nizamudeen A)
+* mgr/dashboard: fix the rbd mirroring configure check (`pr#51325 <https://github.com/ceph/ceph/pull/51325>`_, Nizamudeen A)
+* mgr/dashboard: fix the rgw roles page (`pr#51867 <https://github.com/ceph/ceph/pull/51867>`_, Nizamudeen A)
+* mgr/dashboard: force TLS 1.3 (`pr#50526 <https://github.com/ceph/ceph/pull/50526>`_, Ernesto Puerta)
+* mgr/dashboard: hide notification on force promote (`pr#51164 <https://github.com/ceph/ceph/pull/51164>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: images -> edit -> disable checkboxes for layering and deef-flatten (`pr#53387 <https://github.com/ceph/ceph/pull/53387>`_, avanthakkar)
+* mgr/dashboard: Landing page v3 (`pr#50608 <https://github.com/ceph/ceph/pull/50608>`_, Pedro Gonzalez Gomez, Nizamudeen A, bryanmontalvan)
+* mgr/dashboard: move cephadm e2e cleanup to jenkins job config (`pr#52388 <https://github.com/ceph/ceph/pull/52388>`_, Nizamudeen A)
+* mgr/dashboard: n/a entries behind primary snapshot mode (`pr#53225 <https://github.com/ceph/ceph/pull/53225>`_, Pere Diaz Bou)
+* mgr/dashboard: paginate hosts (`pr#52917 <https://github.com/ceph/ceph/pull/52917>`_, Pere Diaz Bou)
+* mgr/dashboard: rbd-mirror force promotion (`pr#51057 <https://github.com/ceph/ceph/pull/51057>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: remove unncessary hyperlink in landing page (`pr#51119 <https://github.com/ceph/ceph/pull/51119>`_, Nizamudeen A)
+* mgr/dashboard: remove used and total used columns in favor of usage bar (`pr#53303 <https://github.com/ceph/ceph/pull/53303>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: set CORS header for unauthorized access (`pr#53203 <https://github.com/ceph/ceph/pull/53203>`_, Nizamudeen A)
+* mgr/dashboard: skip Create OSDs step in Cluster expansion (`pr#51149 <https://github.com/ceph/ceph/pull/51149>`_, Nizamudeen A)
+* mgr/dashboard: SSO error: AttributeError: 'str' object has no attribute 'decode' (`pr#51952 <https://github.com/ceph/ceph/pull/51952>`_, Volker Theile)
+* mgr/nfs: disallow non-existent paths when creating export (`pr#50807 <https://github.com/ceph/ceph/pull/50807>`_, Dhairya Parmar)
+* mgr/orchestrator: allow deploying raw mode OSDs with --all-available-devices (`pr#50891 <https://github.com/ceph/ceph/pull/50891>`_, Adam King)
+* mgr/orchestrator: fix device size in `orch device ls` output (`pr#50899 <https://github.com/ceph/ceph/pull/50899>`_, Adam King)
+* mgr/prometheus: avoid duplicates and deleted entries for rbd_stats_pools (`pr#48523 <https://github.com/ceph/ceph/pull/48523>`_, Avan Thakkar)
+* mgr/prometheus: fix pool_objects_repaired and daemon_health_metrics format (`pr#51671 <https://github.com/ceph/ceph/pull/51671>`_, banuchka)
+* mgr/rbd_support: add user-friendly stderr message when module is not ready (`pr#52189 <https://github.com/ceph/ceph/pull/52189>`_, Ramana Raja)
+* mgr/rbd_support: recover from "double blocklisting" (`pr#51758 <https://github.com/ceph/ceph/pull/51758>`_, Ramana Raja)
+* mgr/rbd_support: recover from rados client blocklisting (`pr#51455 <https://github.com/ceph/ceph/pull/51455>`_, Ramana Raja)
+* mgr/rgw: initial multisite deployment work (`pr#50887 <https://github.com/ceph/ceph/pull/50887>`_, Redouane Kachach)
+* mgr/snap_schedule: add debug log for paths failing snapshot creation (`pr#50780 <https://github.com/ceph/ceph/pull/50780>`_, Milind Changire)
+* mgr/snap_schedule: allow retention spec 'n' to be user defined (`pr#52749 <https://github.com/ceph/ceph/pull/52749>`_, Milind Changire, Jakob Haufe)
+* mgr/snap_schedule: catch all exceptions for cli (`pr#52752 <https://github.com/ceph/ceph/pull/52752>`_, Milind Changire)
+* mgr/telemetry: compile all channels and collections in selftest (`pr#51761 <https://github.com/ceph/ceph/pull/51761>`_, Laura Flores)
+* mgr/telemetry: fixed log exceptions as "exception" instead of "error" (`pr#51244 <https://github.com/ceph/ceph/pull/51244>`_, Vonesha Frost)
+* mgr/telemetry: make sure histograms are formatted in `all` commands (`pr#50480 <https://github.com/ceph/ceph/pull/50480>`_, Laura Flores)
+* mgr/volumes: avoid returning -ESHUTDOWN back to cli (`issue#58651 <http://tracker.ceph.com/issues/58651>`_, `pr#50786 <https://github.com/ceph/ceph/pull/50786>`_, Venky Shankar)
+* mgr/volumes: Fix pending_subvolume_deletions in volume info (`pr#53573 <https://github.com/ceph/ceph/pull/53573>`_, Kotresh HR)
+* mgr: Add one finisher thread per module (`pr#51044 <https://github.com/ceph/ceph/pull/51044>`_, Kotresh HR, Patrick Donnelly)
+* mgr: add urllib3==1.26.15 to mgr/requirements.txt (`pr#51335 <https://github.com/ceph/ceph/pull/51335>`_, Laura Flores)
+* mgr: register OSDs in ms_handle_accept (`pr#53188 <https://github.com/ceph/ceph/pull/53188>`_, Patrick Donnelly)
+* mgr: store names of modules that register RADOS clients in the MgrMap (`pr#50964 <https://github.com/ceph/ceph/pull/50964>`_, Ramana Raja)
+* MgrMonitor: batch commit OSDMap and MgrMap mutations (`pr#50979 <https://github.com/ceph/ceph/pull/50979>`_, Patrick Donnelly, Kefu Chai, Radosław Zarzyński)
+* mon, qa: issue pool application warning even if pool is empty (`pr#53042 <https://github.com/ceph/ceph/pull/53042>`_, Prashant D)
+* mon/ConfigMonitor: update crush_location from osd entity (`pr#52467 <https://github.com/ceph/ceph/pull/52467>`_, Didier Gazen)
+* mon/MDSMonitor: batch last_metadata update with pending (`pr#52228 <https://github.com/ceph/ceph/pull/52228>`_, Patrick Donnelly)
+* mon/MDSMonitor: check fscid in pending exists in current (`pr#52234 <https://github.com/ceph/ceph/pull/52234>`_, Patrick Donnelly)
+* mon/MDSMonitor: do not propose on error in prepare_update (`pr#52239 <https://github.com/ceph/ceph/pull/52239>`_, Patrick Donnelly)
+* mon/MDSMonitor: ignore extraneous up:boot messages (`pr#52243 <https://github.com/ceph/ceph/pull/52243>`_, Patrick Donnelly)
+* mon/MDSMonitor: plug paxos when maybe manipulating osdmap (`pr#52983 <https://github.com/ceph/ceph/pull/52983>`_, Patrick Donnelly)
+* mon/MonClient: before complete auth with error, reopen session (`pr#52134 <https://github.com/ceph/ceph/pull/52134>`_, Nitzan Mordechai)
+* mon/MonClient: resurrect original client_mount_timeout handling (`pr#52534 <https://github.com/ceph/ceph/pull/52534>`_, Ilya Dryomov)
+* mon/Monitor.cc: exit function if !osdmon()->is_writeable() && mon/OSDMonitor: Added extra check before mon.go_recovery_stretch_mode() (`pr#51413 <https://github.com/ceph/ceph/pull/51413>`_, Kamoltat)
+* mon: avoid exception when setting require-osd-release more than 2 (`pr#51102 <https://github.com/ceph/ceph/pull/51102>`_, Igor Fedotov)
+* mon: block osd pool mksnap for fs pools (`pr#52398 <https://github.com/ceph/ceph/pull/52398>`_, Milind Changire)
+* mon: Fix ceph versions command (`pr#52161 <https://github.com/ceph/ceph/pull/52161>`_, Prashant D)
+* mon: fix iterator mishandling in PGMap::apply_incremental (`pr#52553 <https://github.com/ceph/ceph/pull/52553>`_, Oliver Schmidt)
+* msg/async: don't abort when public addrs mismatch bind addrs (`pr#50575 <https://github.com/ceph/ceph/pull/50575>`_, Radosław Zarzyński)
+* orchestrator: add `--no-destroy` arg to `ceph orch osd rm` (`pr#51215 <https://github.com/ceph/ceph/pull/51215>`_, Guillaume Abrioux)
+* orchestrator: improvements to the orch host ls command (`pr#50889 <https://github.com/ceph/ceph/pull/50889>`_, Paul Cuzner)
+* os/bluestore/bluefs: fix dir_link might add link that already exists in compact log (`pr#51002 <https://github.com/ceph/ceph/pull/51002>`_, ethanwu, Adam Kupczyk)
+* os/bluestore: Add bluefs write op count metrics (`pr#51777 <https://github.com/ceph/ceph/pull/51777>`_, Joshua Baergen)
+* os/bluestore: allow 'fit_to_fast' selector for single-volume osd (`pr#51412 <https://github.com/ceph/ceph/pull/51412>`_, Igor Fedotov)
+* os/bluestore: do not signal deleted dirty file to bluefs log (`pr#48171 <https://github.com/ceph/ceph/pull/48171>`_, Igor Fedotov)
+* os/bluestore: don't require bluestore_db_block_size when attaching new (`pr#52941 <https://github.com/ceph/ceph/pull/52941>`_, Igor Fedotov)
+* os/bluestore: fix no metadata update on truncate+fsync (`pr#48169 <https://github.com/ceph/ceph/pull/48169>`_, Igor Fedotov)
+* os/bluestore: fix spillover alert (`pr#50931 <https://github.com/ceph/ceph/pull/50931>`_, Igor Fedotov)
+* os/bluestore: log before assert in AvlAllocator (`pr#50319 <https://github.com/ceph/ceph/pull/50319>`_, Igor Fedotov)
+* os/bluestore: proper locking for Allocators' dump methods (`pr#48170 <https://github.com/ceph/ceph/pull/48170>`_, Igor Fedotov)
+* os/bluestore: proper override rocksdb::WritableFile::Allocate (`pr#51774 <https://github.com/ceph/ceph/pull/51774>`_, Igor Fedotov)
+* os/bluestore: report min_alloc_size through "ceph osd metadata" (`pr#50505 <https://github.com/ceph/ceph/pull/50505>`_, Igor Fedotov)
+* os/bluestore: use direct write in BlueStore::_write_bdev_label (`pr#48279 <https://github.com/ceph/ceph/pull/48279>`_, luo rixin)
+* osd, mon: add pglog dups length (`pr#47840 <https://github.com/ceph/ceph/pull/47840>`_, Nitzan Mordechai)
+* osd/OpRequest: Add detailed description for delayed op in osd log file (`pr#53690 <https://github.com/ceph/ceph/pull/53690>`_, Yite Gu)
+* osd/OSDCap: allow rbd.metadata_list method under rbd-read-only profile (`pr#51877 <https://github.com/ceph/ceph/pull/51877>`_, Ilya Dryomov)
+* osd/PeeringState: fix missed `recheck_readable` from laggy (`pr#49304 <https://github.com/ceph/ceph/pull/49304>`_, 胡玮文)
+* osd/scheduler/mClockScheduler: Use same profile and client ids for all clients to ensure allocated QoS limit consumption (`pr#53092 <https://github.com/ceph/ceph/pull/53092>`_, Sridhar Seshasayee)
+* osd/scheduler: Reset ephemeral changes to mClock built-in profile (`pr#51664 <https://github.com/ceph/ceph/pull/51664>`_, Sridhar Seshasayee)
+* osd/scrub: verify SnapMapper consistency (`pr#52256 <https://github.com/ceph/ceph/pull/52256>`_, Ronen Friedman, Tim Serong, Kefu Chai, Adam C. Emerson)
+* osd: bring the missed fmt::formatter for snapid_t to address FTBFS (`pr#54175 <https://github.com/ceph/ceph/pull/54175>`_, Radosław Zarzyński)
+* osd: Change scrub cost in case of mClock scheduler (`pr#51728 <https://github.com/ceph/ceph/pull/51728>`_, Aishwarya Mathuria)
+* OSD: during test start, not all osds started due to consum map hang (`pr#51807 <https://github.com/ceph/ceph/pull/51807>`_, Nitzan Mordechai)
+* OSD: Fix check_past_interval_bounds() (`pr#51512 <https://github.com/ceph/ceph/pull/51512>`_, Matan Breizman, Samuel Just)
+* osd: fix: slow scheduling when item_cost is large (`pr#53860 <https://github.com/ceph/ceph/pull/53860>`_, Jrchyang Yu)
+* osd: mClock recovery/backfill cost fixes (`pr#49973 <https://github.com/ceph/ceph/pull/49973>`_, Sridhar Seshasayee, Samuel Just)
+* osd: set per_pool_stats true when OSD has no PG (`pr#48249 <https://github.com/ceph/ceph/pull/48249>`_, jindengke, lmgdlmgd)
+* PendingReleaseNotes: Document mClock scheduler fixes and enhancements (`pr#51978 <https://github.com/ceph/ceph/pull/51978>`_, Sridhar Seshasayee)
+* pybind/argparse: blocklist ip validation (`pr#51811 <https://github.com/ceph/ceph/pull/51811>`_, Nitzan Mordechai)
+* pybind/mgr/devicehealth: do not crash if db not ready (`pr#52215 <https://github.com/ceph/ceph/pull/52215>`_, Patrick Donnelly)
+* pybind/mgr/pg_autoscaler: fix warn when not too few pgs (`pr#53675 <https://github.com/ceph/ceph/pull/53675>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: noautoscale flag retains individual pool configs (`pr#53677 <https://github.com/ceph/ceph/pull/53677>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: Reorderd if statement for the func: _maybe_adjust (`pr#50693 <https://github.com/ceph/ceph/pull/50693>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: Use bytes_used for actual_raw_used (`pr#53725 <https://github.com/ceph/ceph/pull/53725>`_, Kamoltat)
+* pybind: drop GIL during library callouts (`pr#52322 <https://github.com/ceph/ceph/pull/52322>`_, Ilya Dryomov, Patrick Donnelly)
+* python-common: drive_selection: fix KeyError when osdspec_affinity is not set (`pr#53158 <https://github.com/ceph/ceph/pull/53158>`_, Guillaume Abrioux)
+* qa/cephfs: add 'rhel' to family of RH OS in xfstest_dev.py (`pr#52585 <https://github.com/ceph/ceph/pull/52585>`_, Rishabh Dave)
+* qa/rgw: add new POOL_APP_NOT_ENABLED failures to log-ignorelist (`pr#53895 <https://github.com/ceph/ceph/pull/53895>`_, Casey Bodley)
+* qa/smoke,rados,perf-basic: add POOL_APP_NOT_ENABLED to ignorelist (`pr#54065 <https://github.com/ceph/ceph/pull/54065>`_, Prashant D)
+* qa/standalone/osd/divergent-prior.sh: Divergent test 3 with pg_autoscale_mode on pick divergent osd (`pr#52722 <https://github.com/ceph/ceph/pull/52722>`_, Nitzan Mordechai)
+* qa/suites/krbd: stress test for recovering from watch errors (`pr#53785 <https://github.com/ceph/ceph/pull/53785>`_, Ilya Dryomov)
+* qa/suites/rados: remove rook coverage from the rados suite (`pr#52016 <https://github.com/ceph/ceph/pull/52016>`_, Laura Flores)
+* qa/suites/rados: whitelist POOL_APP_NOT_ENABLED for cls tests (`pr#52137 <https://github.com/ceph/ceph/pull/52137>`_, Laura Flores)
+* qa/suites/rbd: install qemu-utils in addition to qemu-block-extra on Ubuntu (`pr#51060 <https://github.com/ceph/ceph/pull/51060>`_, Ilya Dryomov)
+* qa/suites/upgrade/octopus-x: skip TestClsRbd.mirror_snapshot test (`pr#52992 <https://github.com/ceph/ceph/pull/52992>`_, Ilya Dryomov)
+* qa/suites/upgrade/quincy-p2p: skip TestClsRbd.mirror_snapshot test (`pr#53338 <https://github.com/ceph/ceph/pull/53338>`_, Ilya Dryomov)
+* qa/suites/{rbd,krbd}: disable POOL_APP_NOT_ENABLED health check (`pr#53598 <https://github.com/ceph/ceph/pull/53598>`_, Ilya Dryomov)
+* qa/tasks: Changing default mClock profile to high_recovery_ops (`pr#51568 <https://github.com/ceph/ceph/pull/51568>`_, Aishwarya Mathuria)
+* qa/upgrade/quincy-p2p: remove s3tests (`pr#54078 <https://github.com/ceph/ceph/pull/54078>`_, Casey Bodley)
+* qa/upgrade: consistently use the tip of the branch as the start version (`pr#50747 <https://github.com/ceph/ceph/pull/50747>`_, Yuri Weinstein)
+* qa/workunits/rados/test_dedup_tool.sh: reset dedup tier during tests (`pr#51780 <https://github.com/ceph/ceph/pull/51780>`_, Myoungwon Oh)
+* qa: add `POOL_APP_NOT_ENABLED` to ignorelist for cephfs tests (`issue#62508 <http://tracker.ceph.com/issues/62508>`_, `issue#62482 <http://tracker.ceph.com/issues/62482>`_, `pr#53863 <https://github.com/ceph/ceph/pull/53863>`_, Venky Shankar, Patrick Donnelly)
+* qa: check each fs for health (`pr#52241 <https://github.com/ceph/ceph/pull/52241>`_, Patrick Donnelly)
+* qa: cleanup volumes on unwind (`pr#50766 <https://github.com/ceph/ceph/pull/50766>`_, Patrick Donnelly)
+* qa: enable kclient test for newop test (`pr#50991 <https://github.com/ceph/ceph/pull/50991>`_, Xiubo Li, Dhairya Parmar)
+* qa: fix cephfs-mirror unwinding and 'fs volume create/rm' order (`pr#52653 <https://github.com/ceph/ceph/pull/52653>`_, Jos Collin)
+* qa: ignore expected cluster warning from damage tests (`pr#53485 <https://github.com/ceph/ceph/pull/53485>`_, Patrick Donnelly)
+* qa: ignore expected scrub error (`pr#50774 <https://github.com/ceph/ceph/pull/50774>`_, Patrick Donnelly)
+* qa: ignore MDS_TRIM warnings when osd thrashing (`pr#50768 <https://github.com/ceph/ceph/pull/50768>`_, Patrick Donnelly)
+* qa: output higher debugging for cephfs-journal-tool/cephfs-data-scan (`pr#50772 <https://github.com/ceph/ceph/pull/50772>`_, Patrick Donnelly)
+* qa: run scrub post file system recovery (`issue#59527 <http://tracker.ceph.com/issues/59527>`_, `pr#51690 <https://github.com/ceph/ceph/pull/51690>`_, Venky Shankar)
+* qa: test_rebuild_simple checks status on wrong file system (`pr#50922 <https://github.com/ceph/ceph/pull/50922>`_, Patrick Donnelly)
+* qa: test_recovery_pool uses wrong recovery procedure (`pr#50767 <https://github.com/ceph/ceph/pull/50767>`_, Patrick Donnelly)
+* qa: use parallel gzip for compressing logs (`pr#52952 <https://github.com/ceph/ceph/pull/52952>`_, Patrick Donnelly)
+* qa: wait for file to have correct size (`pr#52743 <https://github.com/ceph/ceph/pull/52743>`_, Patrick Donnelly)
+* qa: wait for MDSMonitor tick to replace daemons (`pr#52236 <https://github.com/ceph/ceph/pull/52236>`_, Patrick Donnelly)
+* radosgw-admin: try reshard even if bucket is resharding (`pr#51835 <https://github.com/ceph/ceph/pull/51835>`_, Casey Bodley)
+* rbd-mirror: fix image replayer shut down description on force promote (`pr#52879 <https://github.com/ceph/ceph/pull/52879>`_, Prasanna Kumar Kalever)
+* rbd-mirror: fix race preventing local image deletion (`pr#52626 <https://github.com/ceph/ceph/pull/52626>`_, N Balachandran)
+* rbd-wnbd: improve image map error message (`pr#52289 <https://github.com/ceph/ceph/pull/52289>`_, Lucian Petrut)
+* RGW - Fix NoSuchTagSet error (`pr#50103 <https://github.com/ceph/ceph/pull/50103>`_, Daniel Gryniewicz)
+* RGW - Use correct multipart upload time (`pr#51834 <https://github.com/ceph/ceph/pull/51834>`_, Daniel Gryniewicz)
+* rgw multisite: complete fix for metadata sync issue (`pr#51496 <https://github.com/ceph/ceph/pull/51496>`_, Shilpa Jagannath, gengjichao)
+* rgw/admin: 'bucket stats' displays non-empty time (`pr#50485 <https://github.com/ceph/ceph/pull/50485>`_, Casey Bodley)
+* rgw/lua: allow bucket name override in pre request (`pr#51300 <https://github.com/ceph/ceph/pull/51300>`_, Yuval Lifshitz)
+* rgw/notifications: send mtime in complete multipart upload event (`pr#50962 <https://github.com/ceph/ceph/pull/50962>`_, yuval Lifshitz)
+* rgw/notifications: sending metadata in COPY and CompleteMultipartUpload (`pr#49808 <https://github.com/ceph/ceph/pull/49808>`_, yuval Lifshitz)
+* rgw/rados: check_quota() uses real bucket owner (`pr#51329 <https://github.com/ceph/ceph/pull/51329>`_, Mykola Golub, Casey Bodley)
+* rgw/swift: check position of first slash in slo manifest files (`pr#51598 <https://github.com/ceph/ceph/pull/51598>`_, Marcio Roberto Starke)
+* rgw/sync-policy: Correct "sync status" & "sync group" commands (`pr#53396 <https://github.com/ceph/ceph/pull/53396>`_, Soumya Koduri)
+* rgw: add radosgw-admin bucket check olh/unlinked commands (`pr#53821 <https://github.com/ceph/ceph/pull/53821>`_, Cory Snyder)
+* rgw: avoid string_view to temporary in RGWBulkUploadOp (`pr#52158 <https://github.com/ceph/ceph/pull/52158>`_, Casey Bodley)
+* rgw: concurrency for multi object deletes (`pr#50208 <https://github.com/ceph/ceph/pull/50208>`_, Casey Bodley, Cory Snyder)
+* rgw: D3N cache objects which oid contains slash (`pr#52320 <https://github.com/ceph/ceph/pull/52320>`_, Mark Kogan)
+* rgw: fetch_remote_obj() preserves original part lengths for BlockDecrypt (`pr#52818 <https://github.com/ceph/ceph/pull/52818>`_, Casey Bodley)
+* rgw: fix 2 null versionID after convert_plain_entry_to_versioned (`pr#53399 <https://github.com/ceph/ceph/pull/53399>`_, rui ma, zhuo li)
+* rgw: fix consistency bug with OLH objects (`pr#52538 <https://github.com/ceph/ceph/pull/52538>`_, Cory Snyder)
+* rgw: fix FP error when calculating enteries per bi shard (`pr#53592 <https://github.com/ceph/ceph/pull/53592>`_, J. Eric Ivancich)
+* rgw: fix rgw rate limiting RGWRateLimitInfo class decode_json max_rea… (`pr#53766 <https://github.com/ceph/ceph/pull/53766>`_, xiangrui meng)
+* rgw: fix SignatureDoesNotMatch when extra headers start with 'x-amz' (`pr#53771 <https://github.com/ceph/ceph/pull/53771>`_, rui ma)
+* rgw: fix unwatch crash at radosgw startup (`pr#53761 <https://github.com/ceph/ceph/pull/53761>`_, lichaochao)
+* rgw: handle http options CORS with v4 auth (`pr#53414 <https://github.com/ceph/ceph/pull/53414>`_, Tobias Urdin)
+* rgw: improve buffer list utilization in the chunkupload scenario (`pr#53774 <https://github.com/ceph/ceph/pull/53774>`_, liubingrun)
+* rgw: LDAP fix resource leak with wrong credentials (`pr#50562 <https://github.com/ceph/ceph/pull/50562>`_, Johannes Liebl, Johannes)
+* rgw: optimizations for handling ECANCELED errors from within get_obj_state (`pr#50892 <https://github.com/ceph/ceph/pull/50892>`_, Cory Snyder)
+* rgw: pick http_date in case of http_x_amz_date absence (`pr#53441 <https://github.com/ceph/ceph/pull/53441>`_, Seena Fallah, Mohamed Awnallah)
+* rgw: retry metadata cache notifications with INVALIDATE_OBJ (`pr#52799 <https://github.com/ceph/ceph/pull/52799>`_, Casey Bodley)
+* rgw: rgw_parse_url_bucket() rejects empty bucket names after 'tenant:' (`pr#50625 <https://github.com/ceph/ceph/pull/50625>`_, Casey Bodley)
+* rgw: s3website doesn't prefetch for web_dir() check (`pr#53768 <https://github.com/ceph/ceph/pull/53768>`_, Casey Bodley)
+* rgw: set keys from from master zone on admin api user create (`pr#51601 <https://github.com/ceph/ceph/pull/51601>`_, Ali Maredia)
+* rgw: swift : check for valid key in POST forms (`pr#52739 <https://github.com/ceph/ceph/pull/52739>`_, Abhishek Lekshmanan)
+* rgw: under fips & openssl 3.x allow md5 usage in select rgw ops (`pr#51269 <https://github.com/ceph/ceph/pull/51269>`_, Mark Kogan)
+* rgwlc: prevent lc for one bucket from exceeding time budget (`pr#53561 <https://github.com/ceph/ceph/pull/53561>`_, Matt Benjamin)
+* test/cli-integration/rbd: iSCSI REST API responses aren't pretty-printed anymore (`pr#52283 <https://github.com/ceph/ceph/pull/52283>`_, Ilya Dryomov)
+* test: correct osd pool default size (`pr#51804 <https://github.com/ceph/ceph/pull/51804>`_, Nitzan Mordechai)
+* test: monitor thrasher wait until quorum (`pr#51801 <https://github.com/ceph/ceph/pull/51801>`_, Nitzan Mordechai)
+* tools/ceph-dencoder: Fix incorrect type define for trash_watcher (`pr#51779 <https://github.com/ceph/ceph/pull/51779>`_, Chen Yuanrun)
+* tools/cephfs-data-scan: support for multi-datapool (`pr#50522 <https://github.com/ceph/ceph/pull/50522>`_, Mykola Golub)
+* tools/cephfs: add basic detection/cleanup tool for dentry first damage (`pr#52245 <https://github.com/ceph/ceph/pull/52245>`_, Patrick Donnelly)
+* tools/cephfs: include lost+found in scan_links (`pr#50783 <https://github.com/ceph/ceph/pull/50783>`_, Patrick Donnelly)
+* vstart: check mgr status after starting mgr (`pr#51603 <https://github.com/ceph/ceph/pull/51603>`_, Rongqi Sun)
+* vstart: fix text format (`pr#51124 <https://github.com/ceph/ceph/pull/51124>`_, Rongqi Sun)
+* win32_deps_build: avoid pip (`pr#51129 <https://github.com/ceph/ceph/pull/51129>`_, Lucian Petrut, Ken Dreyer)
+* Wip doc 2023 04 23 backport 51178 to quincy (`pr#51185 <https://github.com/ceph/ceph/pull/51185>`_, Zac Dover)
+* Wip nitzan fixing few rados/test.sh (`pr#49938 <https://github.com/ceph/ceph/pull/49938>`_, Nitzan Mordechai)
+* Wip nitzan pglog ec getattr error (`pr#49936 <https://github.com/ceph/ceph/pull/49936>`_, Nitzan Mordechai)
 
 v17.2.6 Quincy
 ==============
@@ -33,7 +621,6 @@ Changelog
 * Add per OSD crush_device_class definition (`pr#50444 <https://github.com/ceph/ceph/pull/50444>`_, Francesco Pantano)
 * ceph-crash: drop privileges to run as "ceph" user, rather than root (CVE-2022-3650) (`pr#48805 <https://github.com/ceph/ceph/pull/48805>`_, Tim Serong, Guillaume Abrioux)
 * ceph-dencoder: Add erasure_code to denc-mod-osd's target_link_libraries (`pr#48028 <https://github.com/ceph/ceph/pull/48028>`_, Tim Serong)
-* ceph-exporter: cephadm changes (`pr#49771 <https://github.com/ceph/ceph/pull/49771>`_, Avan Thakkar)
 * ceph-mixing: fix ceph_hosts variable (`pr#48934 <https://github.com/ceph/ceph/pull/48934>`_, Tatjana Dehler)
 * ceph-volume/tests: add allowlist_externals to tox.ini (`pr#49788 <https://github.com/ceph/ceph/pull/49788>`_, Guillaume Abrioux)
 * ceph-volume/tests: fix lvm centos8-filestore-create job (`pr#48122 <https://github.com/ceph/ceph/pull/48122>`_, Guillaume Abrioux)
@@ -483,7 +1070,6 @@ Changelog
 * pybind/rados: notify callback reconnect (`pr#48113 <https://github.com/ceph/ceph/pull/48113>`_, Nitzan Mordechai)
 * python-common: Add 'KB' to supported suffixes in SizeMatcher (`pr#48242 <https://github.com/ceph/ceph/pull/48242>`_, Tim Serong)
 * qa/cephadm: remove fsid dir before bootstrap in test_cephadm.sh (`pr#47949 <https://github.com/ceph/ceph/pull/47949>`_, Adam King)
-* qa/fs/mixed-clients: specify distros for tests (`pr#49957 <https://github.com/ceph/ceph/pull/49957>`_, Dhairya Parmar)
 * qa/suites/rbd: fix sporadic "rx-only direction" test failures (`pr#50113 <https://github.com/ceph/ceph/pull/50113>`_, Ilya Dryomov)
 * qa/suites/rgw: fix and update tempest and barbican tests (`pr#50002 <https://github.com/ceph/ceph/pull/50002>`_, Tobias Urdin)
 * qa/tasks/cephadm.py: fix pulling cephadm from git.ceph.com (`pr#49858 <https://github.com/ceph/ceph/pull/49858>`_, Adam King)
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
index 2e471bcbc264..ff60a3e7b607 100644
--- a/doc/releases/releases.yml
+++ b/doc/releases/releases.yml
@@ -20,6 +20,8 @@ releases:
   quincy:
     target_eol: 2024-06-01
     releases:
+      - version: 17.2.7
+        released: 2023-10-30
       - version: 17.2.6
         released: 2023-04-10
       - version: 17.2.5

From 78878511a3e782d5cde63bfb71409c46244402f7 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 26 Oct 2023 09:48:57 +1000
Subject: [PATCH 0300/2492] doc/rados: remove HitSet-related key information

Remove HitSet-related key information from
doc/rados/operations/pools.rst. HitSet-related keys are relevant only to
releases of Ceph that support cache tiering. Only Quincy and earlier
(inclusive) releases of Ceph support cache tiering. Backport this commit
from main to Reef, but not to Quincy or to release branches earlier than
Quincy.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/mon-config-ref.rst |  1 -
 doc/rados/operations/pools.rst             | 87 ----------------------
 2 files changed, 88 deletions(-)

diff --git a/doc/rados/configuration/mon-config-ref.rst b/doc/rados/configuration/mon-config-ref.rst
index 0bc2d4180421..d7e02a0683fe 100644
--- a/doc/rados/configuration/mon-config-ref.rst
+++ b/doc/rados/configuration/mon-config-ref.rst
@@ -287,7 +287,6 @@ by setting it in the ``[mon]`` section of the configuration file.
 .. confval:: mon_data_size_warn
 .. confval:: mon_data_avail_warn
 .. confval:: mon_data_avail_crit
-.. confval:: mon_warn_on_cache_pools_without_hit_sets
 .. confval:: mon_warn_on_crush_straw_calc_version_zero
 .. confval:: mon_warn_on_legacy_crush_tunables
 .. confval:: mon_crush_min_required_version
diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index 900780587428..3e1ff510a6dd 100644
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -499,47 +499,6 @@ You may set values for the following keys:
    :Type: Integer
    :Valid Range: ``1`` sets flag, ``0`` unsets flag
 
-.. _hit_set_type:
-
-.. describe:: hit_set_type
-
-   :Description: Enables HitSet tracking for cache pools.
-                 For additional information, see `Bloom Filter`_.
-   :Type: String
-   :Valid Settings: ``bloom``, ``explicit_hash``, ``explicit_object``
-   :Default: ``bloom``. Other values are for testing.
-
-.. _hit_set_count:
-
-.. describe:: hit_set_count
-
-   :Description: Determines the number of HitSets to store for cache pools. The
-                 higher the value, the more RAM is consumed by the ``ceph-osd``
-                 daemon.
-   :Type: Integer
-   :Valid Range: ``1``. Agent doesn't handle > ``1`` yet.
-
-.. _hit_set_period:
-
-.. describe:: hit_set_period
-
-   :Description: Determines the duration of a HitSet period (in seconds) for
-                 cache pools. The higher the value, the more RAM is consumed
-                 by the ``ceph-osd`` daemon.
-   :Type: Integer
-   :Example: ``3600`` (3600 seconds: one hour)
-
-.. _hit_set_fpp:
-
-.. describe:: hit_set_fpp
-
-   :Description: Determines the probability of false positives for the
-                 ``bloom`` HitSet type. For additional information, see `Bloom
-                 Filter`_.
-   :Type: Double
-   :Valid Range: ``0.0`` - ``1.0``
-   :Default: ``0.05``
-
 .. _cache_target_dirty_ratio:
 
 .. describe:: cache_target_dirty_ratio
@@ -593,23 +552,6 @@ You may set values for the following keys:
    :Type: Integer
    :Example: ``1000000`` #1M objects
 
-
-.. describe:: hit_set_grade_decay_rate
-   
-   :Description: Sets the temperature decay rate between two successive 
-                 HitSets.
-   :Type: Integer
-   :Valid Range: 0 - 100
-   :Default: ``20``
-
-.. describe:: hit_set_search_last_n
-   
-   :Description: Count at most N appearances in HitSets. Used for temperature 
-                 calculation.
-   :Type: Integer
-   :Valid Range: 0 - hit_set_count
-   :Default: ``1``
-
 .. _cache_min_flush_age:
 
 .. describe:: cache_min_flush_age
@@ -739,35 +681,6 @@ You may get values from the following keys:
 :Description: See crush_rule_.
 
 
-``hit_set_type``
-
-:Description: See hit_set_type_.
-
-:Type: String
-:Valid Settings: ``bloom``, ``explicit_hash``, ``explicit_object``
-
-
-``hit_set_count``
-
-:Description: See hit_set_count_.
-
-:Type: Integer
-
-
-``hit_set_period``
-
-:Description: See hit_set_period_.
-
-:Type: Integer
-
-
-``hit_set_fpp``
-
-:Description: See hit_set_fpp_.
-
-:Type: Double
-
-
 ``cache_target_dirty_ratio``
 
 :Description: See cache_target_dirty_ratio_.

From 97512a3e3c28d9aae7e61c30bbe66987298960a9 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 27 Oct 2023 15:22:34 +1000
Subject: [PATCH 0301/2492] doc/rados: edit troubleshooting-pg.rst

s/placement group/pool/ in a sentence that, prior to this change, was
confusing. Suitable for backports prior to Reef.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-pg.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
index 4af2cf347efb..74d04bd9ffe3 100644
--- a/doc/rados/troubleshooting/troubleshooting-pg.rst
+++ b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -405,11 +405,11 @@ Can't Write Data
 ================
 
 If the cluster is up, but some OSDs are down and you cannot write data, make
-sure that you have the minimum number of OSDs running for the placement group.
-If you don't have the minimum number of OSDs running, Ceph will not allow you
-to write data because there is no guarantee that Ceph can replicate your data.
-See ``osd_pool_default_min_size`` in the :ref:`Pool, PG, and CRUSH Config
-Reference <rados_config_pool_pg_crush_ref>` for details.
+sure that you have the minimum number of OSDs running in the pool. If you don't
+have the minimum number of OSDs running in the pool, Ceph will not allow you to
+write data to it because there is no guarantee that Ceph can replicate your
+data. See ``osd_pool_default_min_size`` in the :ref:`Pool, PG, and CRUSH
+Config Reference <rados_config_pool_pg_crush_ref>` for details.
 
 
 PGs Inconsistent

From 6f3d0f570f1a262b06d4c661582091d8ddb11bfa Mon Sep 17 00:00:00 2001
From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Fri, 20 Oct 2023 15:41:05 +0530
Subject: [PATCH 0302/2492] test/librbd/fsx: wait for resize to propagate in
 krbd_resize()

With this changes resize request will not be blocked until the resize is
completed. Because of this the fsx test fails as it assumes that the
request to resize immediately implies changes on the device size.

Hence we have to add a wait in resize handler of fsx for the device to
actually get resized.

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---
 src/test/librbd/fsx.cc | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/test/librbd/fsx.cc b/src/test/librbd/fsx.cc
index acea8ede2708..cbf7111a457b 100644
--- a/src/test/librbd/fsx.cc
+++ b/src/test/librbd/fsx.cc
@@ -1162,6 +1162,8 @@ int
 krbd_resize(struct rbd_ctx *ctx, uint64_t size)
 {
 	int ret;
+	int count = 0;
+	uint64_t effective_size;
 
 	ceph_assert(size % truncbdy == 0);
 
@@ -1183,7 +1185,29 @@ krbd_resize(struct rbd_ctx *ctx, uint64_t size)
 	if (ret < 0)
 		return ret;
 
-	return __librbd_resize(ctx, size);
+	ret = __librbd_resize(ctx, size);
+	if (ret < 0)
+		return ret;
+
+	for (;;) {
+		ret = krbd_get_size(ctx, &effective_size);
+		if (ret < 0)
+			return ret;
+
+		if (effective_size == size)
+			break;
+
+		if (count++ >= 15) {
+			prt("BLKGETSIZE64 size error: expected 0x%llx, actual 0x%llx\n",
+			    (unsigned long long)size,
+			    (unsigned long long)effective_size);
+			return -EINVAL;
+		}
+
+		usleep(count * 250 * 1000);
+	}
+
+	return 0;
 }
 
 int

From d8e5b87a313f6b7b98b93b61327f94af4d8f2bed Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 27 Oct 2023 16:58:28 +1000
Subject: [PATCH 0303/2492] doc/rados: remove cache-tiering-related keys

Remove information related to cache-tiering-related keys from
doc/rados/operations/pools.rst. Cache-tiering is deprecated in Reef.
This PR is suitable for backporting to the Reef release branch, but not
to release branches prior to Reef.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/pools.rst | 88 ----------------------------------
 1 file changed, 88 deletions(-)

diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index 3e1ff510a6dd..dda9e844e965 100644
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -499,41 +499,6 @@ You may set values for the following keys:
    :Type: Integer
    :Valid Range: ``1`` sets flag, ``0`` unsets flag
 
-.. _cache_target_dirty_ratio:
-
-.. describe:: cache_target_dirty_ratio
-
-   :Description: Sets a flush threshold for the percentage of the cache pool
-                 containing modified (dirty) objects. When this threshold is
-                 reached, the cache-tiering agent will flush these objects to
-                 the backing storage pool.
-   :Type: Double
-   :Default: ``.4``
-
-.. _cache_target_dirty_high_ratio:
-
-.. describe:: cache_target_dirty_high_ratio
-   
-   :Description: Sets a flush threshold for the percentage of the cache pool
-                 containing modified (dirty) objects. When this threshold is
-                 reached, the cache-tiering agent will flush these objects to
-                 the backing storage pool with a higher speed (as compared with
-                 ``cache_target_dirty_ratio``).
-   :Type: Double
-   :Default: ``.6``
-
-.. _cache_target_full_ratio:
-
-.. describe:: cache_target_full_ratio
-   
-   :Description: Sets an eviction threshold for the percentage of the cache
-                 pool containing unmodified (clean) objects. When this
-                 threshold is reached, the cache-tiering agent will evict 
-                 these objects from the cache pool.
-
-   :Type: Double
-   :Default: ``.8``
-
 .. _target_max_bytes:
 
 .. describe:: target_max_bytes
@@ -552,24 +517,6 @@ You may set values for the following keys:
    :Type: Integer
    :Example: ``1000000`` #1M objects
 
-.. _cache_min_flush_age:
-
-.. describe:: cache_min_flush_age
-   
-   :Description: Sets the time (in seconds) before the cache-tiering agent
-                 flushes an object from the cache pool to the storage pool.
-   :Type: Integer
-   :Example: ``600`` (600 seconds: ten minutes)
-
-.. _cache_min_evict_age:
-
-.. describe:: cache_min_evict_age
-   
-   :Description: Sets the time (in seconds) before the cache-tiering agent
-                 evicts an object from the cache pool.
-   :Type: Integer
-   :Example: ``1800`` (1800 seconds: thirty minutes)
-
 .. _fast_read:
 
 .. describe:: fast_read
@@ -681,27 +628,6 @@ You may get values from the following keys:
 :Description: See crush_rule_.
 
 
-``cache_target_dirty_ratio``
-
-:Description: See cache_target_dirty_ratio_.
-
-:Type: Double
-
-
-``cache_target_dirty_high_ratio``
-
-:Description: See cache_target_dirty_high_ratio_.
-
-:Type: Double
-
-
-``cache_target_full_ratio``
-
-:Description: See cache_target_full_ratio_.
-
-:Type: Double
-
-
 ``target_max_bytes``
 
 :Description: See target_max_bytes_.
@@ -716,20 +642,6 @@ You may get values from the following keys:
 :Type: Integer
 
 
-``cache_min_flush_age``
-
-:Description: See cache_min_flush_age_.
-
-:Type: Integer
-
-
-``cache_min_evict_age``
-
-:Description: See cache_min_evict_age_.
-
-:Type: Integer
-
-
 ``fast_read``
 
 :Description: See fast_read_.

From a0491a4f63885be9ea14a15c67cabd8903f19641 Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Wed, 4 Oct 2023 15:22:41 -0400
Subject: [PATCH 0304/2492] rgw/multisite: fix sync_error_trim command. without
 --marker option, we trim everthing. if --marker option is passed in, we trim
 upto that marker.

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 src/rgw/rgw_admin.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index cc7f5811c9ef..cc7f2b704d89 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -9579,13 +9579,16 @@ int main(int argc, const char **argv)
       return -EINVAL;
     }
     if (!start_marker.empty()) {
-      std::cerr << "end-date not allowed." << std::endl;
+      std::cerr << "start-marker not allowed." << std::endl;
       return -EINVAL;
     }
     if (!end_marker.empty()) {
-      std::cerr << "end-date not allowed." << std::endl;
+      std::cerr << "end_marker not allowed." << std::endl;
       return -EINVAL;
     }
+    if (marker.empty()) {
+      marker = "9"; // trims everything
+    }
 
     if (shard_id < 0) {
       shard_id = 0;

From 87f4a8914c297c505d8d79fc88063926d0113ff0 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Fri, 27 Oct 2023 17:01:35 -0400
Subject: [PATCH 0305/2492] build: Don't kill install-deps.sh if we can't get
 ceph-libboost

We can still use the submodule.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 install-deps.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/install-deps.sh b/install-deps.sh
index 50a62682765c..d01bc6464e6f 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -216,7 +216,9 @@ function install_boost_on_ubuntu {
         ceph-libboost-system${boost_ver}-dev \
         ceph-libboost-test${boost_ver}-dev \
         ceph-libboost-thread${boost_ver}-dev \
-        ceph-libboost-timer${boost_ver}-dev
+        ceph-libboost-timer${boost_ver}-dev \
+	|| ci_debug "ceph-libboost package unavailable, you can build the submodule"
+
 }
 
 function install_libzbd_on_ubuntu {

From e75643f717a00454678ff0882f38da9d24281168 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 28 Oct 2023 11:42:34 -0500
Subject: [PATCH 0306/2492] osd/scrub: do not clear PG_STATE_REPAIR
 unconditionally

As we now call clear_pgscrub_state() at the end of each
'Session' state, we must not clear PG_STATE_REPAIR
unconditionally.

Previously - scrubs that reached normal completion, i.e.
reached PgScrubber::scrub_finish(), would have only cleared
that PG flag under specific conditions. That was changed in
previous commits of this PR, and is now fixed.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index b090ec113ee3..5afc2ced2725 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -2166,8 +2166,6 @@ void PgScrubber::clear_pgscrub_state()
   state_clear(PG_STATE_SCRUBBING);
   state_clear(PG_STATE_DEEP_SCRUB);
 
-  state_clear(PG_STATE_REPAIR);
-
   m_local_osd_resource.reset();
   requeue_waiting();
 

From 7219897180894027df70accfc8b184181ba34bd9 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 29 Oct 2023 21:47:04 +1000
Subject: [PATCH 0307/2492] doc/install: simplify cephadm docs

Simplify the explanation of cephadm.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/install/index.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install/index.rst b/doc/install/index.rst
index 58ef92a270c1..d8e9ca3a63e9 100644
--- a/doc/install/index.rst
+++ b/doc/install/index.rst
@@ -9,15 +9,15 @@ There are multiple ways to install Ceph.
 Recommended methods
 ~~~~~~~~~~~~~~~~~~~
 
-:ref:`Cephadm <cephadm_deploying_new_cluster>` installs and manages a Ceph
-cluster that uses containers and systemd and is tightly integrated with the CLI
-and dashboard GUI.
+:ref:`Cephadm <cephadm_deploying_new_cluster>` is a tool that can be used to
+install and manage a Ceph cluster. 
 
 * cephadm supports only Octopus and newer releases.
 * cephadm is fully integrated with the orchestration API and fully supports the
   CLI and dashboard features that are used to manage cluster deployment.
 * cephadm requires container support (in the form of Podman or Docker) and
   Python 3.
+* cephadm requires systemd.
 
 `Rook <https://rook.io/>`_ deploys and manages Ceph clusters running
 in Kubernetes, while also enabling management of storage resources and

From e58bc3e4df97d950d96d006b6336e10dbe9327a2 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 29 Oct 2023 22:01:11 +1000
Subject: [PATCH 0308/2492] doc/ceph-volume: explain idempotence

Improve the grammar of a note explaining that the execution of the call
that activates ceph-volume is idempotent.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/ceph-volume/lvm/activate.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/ceph-volume/lvm/activate.rst b/doc/ceph-volume/lvm/activate.rst
index d5129def11d5..3f86e98b70cc 100644
--- a/doc/ceph-volume/lvm/activate.rst
+++ b/doc/ceph-volume/lvm/activate.rst
@@ -10,8 +10,9 @@ This activation process enables a systemd unit that persists the OSD ID and its
 UUID (also called ``fsid`` in Ceph CLI tools), so that at boot time it can
 understand what OSD is enabled and needs to be mounted.
 
-.. note:: The execution of this call is fully idempotent, and there is no
-          side-effects when running multiple times
+.. note:: The execution of this call is fully idempotent. This means that the
+   call can be executed multiple times without changing the result of its first
+   successful execution.
 
 For OSDs deployed by cephadm, please refer to :ref:`cephadm-osd-activate`
 instead.

From 23190dd91861d1dcb8d03b78fb378e042394dfad Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 29 Oct 2023 22:15:13 +1000
Subject: [PATCH 0309/2492] doc/ceph-volume: improve front matter

Simplify and improve the readability of the front matter of
doc/ceph-volume/lvm/activate.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/ceph-volume/lvm/activate.rst | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/ceph-volume/lvm/activate.rst b/doc/ceph-volume/lvm/activate.rst
index d5129def11d5..bf7dab06aef3 100644
--- a/doc/ceph-volume/lvm/activate.rst
+++ b/doc/ceph-volume/lvm/activate.rst
@@ -3,18 +3,19 @@
 ``activate``
 ============
 
-Once :ref:`ceph-volume-lvm-prepare` is completed, and all the various steps
-that entails are done, the volume is ready to get "activated".
+After :ref:`ceph-volume-lvm-prepare` has completed its run, the volume can be
+activated. 
 
-This activation process enables a systemd unit that persists the OSD ID and its
-UUID (also called ``fsid`` in Ceph CLI tools), so that at boot time it can
-understand what OSD is enabled and needs to be mounted.
+Activating the volume involves enabling a ``systemd`` unit that persists the
+``OSD ID`` and its ``UUID`` (which is also called the ``fsid`` in the Ceph CLI
+tools). After this information has been persisted, the cluster can determine
+which OSD is enabled and must be mounted.
 
 .. note:: The execution of this call is fully idempotent, and there is no
           side-effects when running multiple times
 
-For OSDs deployed by cephadm, please refer to :ref:`cephadm-osd-activate`
-instead.
+For information about OSDs deployed by cephadm, refer to
+:ref:`cephadm-osd-activate`.
 
 New OSDs
 --------

From a36f5ed185cbcd788e36ad22a00cfe9dcdfdfe7f Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Wed, 18 Oct 2023 18:29:06 +0530
Subject: [PATCH 0310/2492] client: call _getattr() for -ENODATA returned
 _getvxattr() calls

_getvxattr rpc doesn't handle ceph.mirror.* attributes and always
returns -ENODATA for ceph.mirror.* attributes. This prevents
incremental syncing of snapshots. So let them call _getattr()
and proceed.

Fixes: https://tracker.ceph.com/issues/63099
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 src/client/Client.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 7d223018c762..9fb806b95840 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -13716,7 +13716,9 @@ int Client::_getxattr(Inode *in, const char *name, void *value, size_t size,
 
   if (!strncmp(name, "ceph.", 5)) {
     r = _getvxattr(in, perms, name, size, value, MDS_RANK_NONE);
-    goto out;
+    if (r != -ENODATA) {
+      goto out;
+    }
   }
 
   if (acl_type == NO_ACL && !strncmp(name, "system.", 7)) {

From be551a6c68018207b6c730ae9d12341c796f5deb Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Thu, 12 Oct 2023 14:53:16 +0530
Subject: [PATCH 0311/2492] qa: tests set/get ceph.mirror.dirty_snap_id
 attribute succeeds in a remote filesystem

Fixes: https://tracker.ceph.com/issues/63099
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 qa/tasks/cephfs/test_mirroring.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py
index b7f436774ddb..24dbd61b17fb 100644
--- a/qa/tasks/cephfs/test_mirroring.py
+++ b/qa/tasks/cephfs/test_mirroring.py
@@ -1305,3 +1305,13 @@ def test_local_and_remote_dir_root_mode(self):
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
         self.mount_a.run_shell(["rmdir", "l1/.snap/snap0"])
         self.mount_a.run_shell(["rmdir", "l1"])
+
+    def test_get_set_mirror_dirty_snap_id(self):
+        """
+        That get/set ceph.mirror.dirty_snap_id attribute succeeds in a remote filesystem.
+        """
+        self.mount_b.run_shell(["mkdir", "-p", "d1/d2/d3"])
+        attr = str(random.randint(1, 10))
+        self.mount_b.setfattr("d1/d2/d3", "ceph.mirror.dirty_snap_id", attr)
+        val = self.mount_b.getfattr("d1/d2/d3", "ceph.mirror.dirty_snap_id")
+        self.assertEqual(attr, val, f"Mismatch for ceph.mirror.dirty_snap_id value: {attr} vs {val}")

From 551281bdc2a049c58ff1d035014312d8eaa99920 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 30 Oct 2023 13:09:25 +0530
Subject: [PATCH 0312/2492] mgr/dashboard: Block Ui fails in angular with
 target es2022

The BlockUi element is failing in angular with target es2022..change it to es2020

Fixes: https://tracker.ceph.com/issues/63347

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/frontend/tsconfig.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/tsconfig.json b/src/pybind/mgr/dashboard/frontend/tsconfig.json
index 0f454e30cd73..e0cf323fd54f 100644
--- a/src/pybind/mgr/dashboard/frontend/tsconfig.json
+++ b/src/pybind/mgr/dashboard/frontend/tsconfig.json
@@ -17,7 +17,7 @@
     "noImplicitReturns": true,
     "noImplicitAny": true,
     "suppressImplicitAnyIndexErrors": true,
-    "target": "ES2022",
+    "target": "ES2020",
     "module": "es2020",
     "baseUrl": "./",
     "resolveJsonModule": true,

From 6d066e3d7eefc3f4aceccde6b53fc5265668b3de Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 30 Oct 2023 13:17:37 +0530
Subject: [PATCH 0313/2492] mgr/dashboard: update rgw multisite import form
 helper info

Change 'To obtain the token, generate it from your secondary Ceph cluster' to 'To obtain the token, generate it from your primary Ceph cluster' in rgw multisite import form helper

Fixes: https://tracker.ceph.com/issues/63348

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../rgw-multisite-import/rgw-multisite-import.component.html    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
index a001e4b00c7c..ee9816dc5cbc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
@@ -11,7 +11,7 @@
                       spacingClass="mb-3">
         <ul>
           <li>This feature allows you to configure a connection between your primary and secondary Ceph clusters for data replication. By importing a token, you establish a link between the clusters, enabling data synchronization.</li>
-          <li>To obtain the token, generate it from your secondary Ceph cluster. This token includes encoded information about the secondary cluster's endpoint, access key, and secret key.</li>
+          <li>To obtain the token, generate it from your primary Ceph cluster. This token includes encoded information about the secondary cluster's endpoint, access key, and secret key.</li>
           <li>The secondary zone represents the destination cluster where your data will be replicated.</li>
         </ul>
       </cd-alert-panel>

From 3a386d52a40582536655859003f85bc0f77f8d4a Mon Sep 17 00:00:00 2001
From: Ernesto Puerta <37327689+epuertat@users.noreply.github.com>
Date: Mon, 30 Oct 2023 12:36:04 +0100
Subject: [PATCH 0314/2492] create-backport-trackers: Python >= 3.12 deprecated
 distutils

See https://docs.python.org/3.10/whatsnew/3.10.html#distutils-deprecated.

Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
---
 .github/workflows/create-backport-trackers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create-backport-trackers.yml b/.github/workflows/create-backport-trackers.yml
index b3525d9e94e1..fe05d09b0ec1 100644
--- a/.github/workflows/create-backport-trackers.yml
+++ b/.github/workflows/create-backport-trackers.yml
@@ -41,7 +41,7 @@ jobs:
           files: src/script/backport-create-issue src/script/requirements.backport-create-issue.txt
       - uses: actions/setup-python@v4
         with:
-          python-version: '>=3.6'
+          python-version: '>=3.6 <3.12'
           cache: 'pip'
           cache-dependency-path: src/script/requirements.backport-create-issue.txt
       - run: pip install -r src/script/requirements.backport-create-issue.txt

From b2fadb65ae6a9eb3eaade56e04b08f2b694b220c Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Thu, 26 Oct 2023 15:57:54 -0400
Subject: [PATCH 0315/2492] rgw/notification: Fix Kafka persistent notification
  breakage that are not retried.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/rgw/rgw_kafka.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_kafka.cc b/src/rgw/rgw_kafka.cc
index 832c4667b94e..256678c49704 100644
--- a/src/rgw/rgw_kafka.cc
+++ b/src/rgw/rgw_kafka.cc
@@ -100,8 +100,9 @@ struct connection_t {
     // fire all remaining callbacks (if not fired by rd_kafka_flush)
     std::for_each(callbacks.begin(), callbacks.end(), [this](auto& cb_tag) {
         cb_tag.cb(status);
-        ldout(cct, 20) << "Kafka destroy: invoking callback with tag=" << cb_tag.tag << 
-          " for: " << broker << dendl;
+        ldout(cct, 20) << "Kafka destroy: invoking callback with tag="
+                       << cb_tag.tag << " for: " << broker
+                       << " with status: " << status << dendl;
       });
     callbacks.clear();
     delivery_tag = 1;
@@ -418,7 +419,9 @@ class Manager {
     if (tag) {
       auto const q_len = conn->callbacks.size();
       if (q_len < max_inflight) {
-        ldout(conn->cct, 20) << "Kafka publish (with callback, tag=" << *tag << "): OK. Queue has: " << q_len << " callbacks" << dendl;
+        ldout(conn->cct, 20)
+            << "Kafka publish (with callback, tag=" << *tag
+            << "): OK. Queue has: " << q_len + 1 << " callbacks" << dendl;
         conn->callbacks.emplace_back(*tag, message->cb);
       } else {
         // immediately invoke callback with error - this is not a connection error
@@ -463,6 +466,7 @@ class Manager {
         if(conn->timestamp.sec() + max_idle_time < ceph_clock_now()) {
           ldout(conn->cct, 20) << "kafka run: deleting a connection due to idle behaviour: " << ceph_clock_now() << dendl;
           std::lock_guard lock(connections_lock);
+          conn->status = STATUS_CONNECTION_IDLE;
           conn_it = connections.erase(conn_it);
           --connection_count; \
           continue;

From 686dd3d838b6de6bf1101f0b61f729426b0bccad Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Sat, 28 Oct 2023 13:29:59 -0400
Subject: [PATCH 0316/2492] build: Fix fmt version check

Currently, when attempting to build ceph on a system with fmt
installed, we try to build against it whatever the version. This
constantly breaks people's builds, since newer versions of fmt often
change the API.

This change specifies that versions must be below 10 as well as at or
above 8.1.1, so that on systems with a new format, we fall back to
using the submodule.

It also removes the `Findfmt.cmake` module, as that does not check
the installed version. Instead, we use the cmake config file installed by
the system package of fmt and does support version checking.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 cmake/modules/Findfmt.cmake | 61 -------------------------------------
 src/CMakeLists.txt          | 12 +++++---
 2 files changed, 7 insertions(+), 66 deletions(-)
 delete mode 100644 cmake/modules/Findfmt.cmake

diff --git a/cmake/modules/Findfmt.cmake b/cmake/modules/Findfmt.cmake
deleted file mode 100644
index 734c2b0571c2..000000000000
--- a/cmake/modules/Findfmt.cmake
+++ /dev/null
@@ -1,61 +0,0 @@
-find_path(fmt_INCLUDE_DIR NAMES fmt/format.h)
-
-if(fmt_INCLUDE_DIR)
-  set(_fmt_version_file "${fmt_INCLUDE_DIR}/fmt/core.h")
-  if(NOT EXISTS "${_fmt_version_file}")
-    set(_fmt_version_file "${fmt_INCLUDE_DIR}/fmt/format.h")
-  endif()
-  if(EXISTS "${_fmt_version_file}")
-    # parse "#define FMT_VERSION 40100" to 4.1.0
-    file(STRINGS "${_fmt_version_file}" fmt_VERSION_LINE
-      REGEX "^#define[ \t]+FMT_VERSION[ \t]+[0-9]+$")
-    string(REGEX REPLACE "^#define[ \t]+FMT_VERSION[ \t]+([0-9]+)$"
-      "\\1" fmt_VERSION "${fmt_VERSION_LINE}")
-    foreach(ver "fmt_VERSION_PATCH" "fmt_VERSION_MINOR" "fmt_VERSION_MAJOR")
-      math(EXPR ${ver} "${fmt_VERSION} % 100")
-      math(EXPR fmt_VERSION "(${fmt_VERSION} - ${${ver}}) / 100")
-    endforeach()
-    set(fmt_VERSION
-      "${fmt_VERSION_MAJOR}.${fmt_VERSION_MINOR}.${fmt_VERSION_PATCH}")
-  endif()
-endif()
-
-find_library(fmt_LIBRARY NAMES fmt)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(fmt
-  REQUIRED_VARS fmt_INCLUDE_DIR fmt_LIBRARY
-  VERSION_VAR fmt_VERSION)
-mark_as_advanced(
-  fmt_INCLUDE_DIR
-  fmt_LIBRARY
-  fmt_VERSION_MAJOR
-  fmt_VERSION_MINOR
-  fmt_VERSION_PATCH
-  fmt_VERSION_STRING)
-
-if(fmt_FOUND AND NOT (TARGET fmt::fmt))
-  add_library(fmt-header-only INTERFACE)
-  set_target_properties(fmt-header-only PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES "${fmt_INCLUDE_DIR}"
-    INTERFACE_COMPILE_DEFINITIONS FMT_HEADER_ONLY=1
-    INTERFACE_COMPILE_FEATURES cxx_std_11)
-
-  add_library(fmt UNKNOWN IMPORTED GLOBAL)
-  set_target_properties(fmt PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES "${fmt_INCLUDE_DIR}"
-    INTERFACE_COMPILE_FEATURES cxx_std_11
-    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-    IMPORTED_LOCATION "${fmt_LIBRARY}")
-
-  if(WITH_FMT_HEADER_ONLY)
-    # please note, this is different from how upstream defines fmt::fmt.
-    # in order to force 3rd party libraries to link against fmt-header-only if
-    # WITH_FMT_HEADER_ONLY is ON, we have to point fmt::fmt to fmt-header-only
-    # in this case.
-    add_library(fmt::fmt ALIAS fmt-header-only)
-  else()
-    add_library(fmt::fmt ALIAS fmt)
-  endif()
-
-endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 8aa271a2b5b2..deb27a1c36cf 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -326,13 +326,15 @@ if(NOT TARGET RapidJSON::RapidJSON)
 endif()
 
 option(WITH_FMT_HEADER_ONLY "use header-only version of fmt library" OFF)
-set(WITH_FMT_VERSION "8.1.1" CACHE
-  STRING "build with fmt version")
-find_package(fmt ${WITH_FMT_VERSION} QUIET)
+find_package(fmt 8.1.1...<10.0.0)
 if(fmt_FOUND)
-  include_directories(SYSTEM "${fmt_INCLUDE_DIR}")
+  message(STATUS "Building with system fmt.")
+  if (WITH_FMT_HEADER_ONLY)
+    message(STATUS "Using fmt header-only.")
+    add_library(fmt::fmt ALIAS fmt::fmt-header-only)
+  endif()
 else()
-  message(STATUS "Could not find fmt, will build it")
+  message(STATUS "Building fmt as submodule")
   set(old_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
   set(BUILD_SHARED_LIBS FALSE)
   add_subdirectory(fmt)

From 421bf61fee35cb8158b4e7ee8a8e40a4089554d9 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Sat, 28 Oct 2023 13:42:59 -0400
Subject: [PATCH 0317/2492] build: Add `WITH_SYSTEM_FMT` option

The `WITH_SYSTEM_FMT` option builds against system fmt and will fail
if a suitable version is not found. It is defaulted to `OFF`.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index deb27a1c36cf..dca5e6e6d4ab 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -326,7 +326,10 @@ if(NOT TARGET RapidJSON::RapidJSON)
 endif()
 
 option(WITH_FMT_HEADER_ONLY "use header-only version of fmt library" OFF)
-find_package(fmt 8.1.1...<10.0.0)
+option(WITH_SYSTEM_FMT "build against system fmt" OFF)
+if(WITH_SYSTEM_FMT)
+  find_package(fmt 8.1.1...<10.0.0 REQUIRED)
+endif()
 if(fmt_FOUND)
   message(STATUS "Building with system fmt.")
   if (WITH_FMT_HEADER_ONLY)

From 7f57798b7f82b07eb69d87312114d41e44d3c7f3 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 30 Oct 2023 18:09:09 +0530
Subject: [PATCH 0318/2492] mgr/dashboard: fix rgw multisite error when no rgw
 entity is present

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../frontend/src/app/ceph/rgw/rgw.module.ts    | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
index df7e019ed7ce..c16c13a81bd7 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
@@ -28,8 +28,6 @@ import { RgwUserTabsComponent } from './rgw-user-tabs/rgw-user-tabs.component';
 import { RgwMultisiteDetailsComponent } from './rgw-multisite-details/rgw-multisite-details.component';
 import { TreeModule } from '@circlon/angular-tree-component';
 import { DataTableModule } from '~/app/shared/datatable/datatable.module';
-import { FeatureTogglesGuardService } from '~/app/shared/services/feature-toggles-guard.service';
-import { ModuleStatusGuardService } from '~/app/shared/services/module-status-guard.service';
 import { RgwMultisiteRealmFormComponent } from './rgw-multisite-realm-form/rgw-multisite-realm-form.component';
 import { RgwMultisiteZonegroupFormComponent } from './rgw-multisite-zonegroup-form/rgw-multisite-zonegroup-form.component';
 import { RgwMultisiteZoneFormComponent } from './rgw-multisite-zone-form/rgw-multisite-zone-form.component';
@@ -185,22 +183,6 @@ const routes: Routes = [
   },
   {
     path: 'multisite',
-    canActivate: [FeatureTogglesGuardService, ModuleStatusGuardService],
-    data: {
-      moduleStatusGuardConfig: {
-        uiApiPath: 'rgw/multisite',
-        redirectTo: 'error',
-        header: 'Multi-site not configured',
-        button_name: 'Add Multi-site Configuration',
-        button_route: '/rgw/multisite/create',
-        button_title: 'Add multi-site configuration (realms/zonegroups/zones)',
-        secondary_button_name: 'Import Multi-site Configuration',
-        secondary_button_route: 'rgw/multisite/import',
-        secondary_button_title:
-          'Import multi-site configuration (import realm token from a secondary cluster)'
-      },
-      breadcrumbs: 'Multi-Site'
-    },
     children: [{ path: '', component: RgwMultisiteDetailsComponent }]
   }
 ];

From bdb69c7f17374cd5b1903f4b04ae136e4b3922f8 Mon Sep 17 00:00:00 2001
From: barakda <barak.davidov@gmail.com>
Date: Mon, 30 Oct 2023 17:49:55 +0200
Subject: [PATCH 0319/2492] Update default nvmeof gw version to v0.0.5

Signed-off-by: barakda <barak.davidov@gmail.com>
---
 src/pybind/mgr/cephadm/module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 70d66732f0f5..ac6e39c469ed 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -108,7 +108,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_IMAGE = 'quay.io/ceph/ceph'
 DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.43.0'
 DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.5.0'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:0.0.3'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:v0.0.5'
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'

From 4452bc22d1c6c8499cf55d6e39090adf7ae1dcbf Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 26 Oct 2023 13:18:52 -0400
Subject: [PATCH 0320/2492] mgr/rbd_support: fix recursive locking on
 CreateSnapshotRequests lock

The MirrorSnapshotScheduleHandler's run thread issues asynchronous
create snapshot requests using a CreateSnapshotRequests instance. When
the thread invokes a CreateSnapshotRequests instance's get_ioctx(),
the instance's class variable lock is acquired. With the class
variable lock held, the garbage collection of a CreateSnapshotRequests
instance may race in the thread. The thread would then call
CreateSnapshotRequests __del__() that tries to acquire the class
variable lock that the thread already holds. Fix this
recursive deadlock by converting the CreateSnapshotRequests lock from
a class variable to an instance variable. There is no need to share
the lock across CreateSnapshotRequests instances.

Also convert MirrorSnapshotScheduleHandler, PerfHandler and
TrashPurgeScheduleHandler class variables to instance variables
that don't need to be shared across the instances.

Fixes: https://tracker.ceph.com/issues/62994
Signed-off-by: Ramana Raja <rraja@redhat.com>
Co-Authored-By: Ilya Dryomov <idryomov@gmail.com>
---
 .../rbd_support/mirror_snapshot_schedule.py   | 10 ++++------
 src/pybind/mgr/rbd_support/perf.py            | 19 ++++++++++---------
 .../mgr/rbd_support/trash_purge_schedule.py   |  5 ++---
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
index 120b59318bac..0ddc2e6e734c 100644
--- a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
+++ b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
@@ -33,10 +33,9 @@ class ImageSpec(NamedTuple):
 
 class CreateSnapshotRequests:
 
-    lock = Lock()
-    condition = Condition(lock)
-
     def __init__(self, handler: Any) -> None:
+        self.lock = Lock()
+        self.condition = Condition(self.lock)
         self.handler = handler
         self.rados = handler.module.rados
         self.log = handler.log
@@ -331,10 +330,9 @@ class MirrorSnapshotScheduleHandler:
     SCHEDULE_OID = "rbd_mirror_snapshot_schedule"
     REFRESH_DELAY_SECONDS = 60.0
 
-    lock = Lock()
-    condition = Condition(lock)
-
     def __init__(self, module: Any) -> None:
+        self.lock = Lock()
+        self.condition = Condition(self.lock)
         self.module = module
         self.log = module.log
         self.last_refresh_images = datetime(1970, 1, 1)
diff --git a/src/pybind/mgr/rbd_support/perf.py b/src/pybind/mgr/rbd_support/perf.py
index 68cbbd3b5f48..20815721de50 100644
--- a/src/pybind/mgr/rbd_support/perf.py
+++ b/src/pybind/mgr/rbd_support/perf.py
@@ -65,15 +65,6 @@
 
 
 class PerfHandler:
-    user_queries: Dict[PoolKeyT, Dict[str, Any]] = {}
-    image_cache: Dict[str, str] = {}
-
-    lock = Lock()
-    query_condition = Condition(lock)
-    refresh_condition = Condition(lock)
-
-    image_name_cache: Dict[Tuple[int, str], Dict[str, str]] = {}
-    image_name_refresh_time = datetime.fromtimestamp(0)
 
     @classmethod
     def prepare_regex(cls, value: Any) -> str:
@@ -114,6 +105,16 @@ def submatch_pool_key(cls, pool_key: PoolKeyT, search_key: str) -> bool:
                 and (pool_key[0] == search_key[0] or not search_key[0]))
 
     def __init__(self, module: Any) -> None:
+        self.user_queries: Dict[PoolKeyT, Dict[str, Any]] = {}
+        self.image_cache: Dict[str, str] = {}
+
+        self.lock = Lock()
+        self.query_condition = Condition(self.lock)
+        self.refresh_condition = Condition(self.lock)
+
+        self.image_name_cache: Dict[Tuple[int, str], Dict[str, str]] = {}
+        self.image_name_refresh_time = datetime.fromtimestamp(0)
+
         self.module = module
         self.log = module.log
 
diff --git a/src/pybind/mgr/rbd_support/trash_purge_schedule.py b/src/pybind/mgr/rbd_support/trash_purge_schedule.py
index b2f7b1614f13..abc50ec394f4 100644
--- a/src/pybind/mgr/rbd_support/trash_purge_schedule.py
+++ b/src/pybind/mgr/rbd_support/trash_purge_schedule.py
@@ -16,10 +16,9 @@ class TrashPurgeScheduleHandler:
     SCHEDULE_OID = "rbd_trash_purge_schedule"
     REFRESH_DELAY_SECONDS = 60.0
 
-    lock = Lock()
-    condition = Condition(lock)
-
     def __init__(self, module: Any) -> None:
+        self.lock = Lock()
+        self.condition = Condition(self.lock)
         self.module = module
         self.log = module.log
         self.last_refresh_pools = datetime(1970, 1, 1)

From fed1e87685a698876cf167b3681327e5b0066ee6 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Mon, 30 Oct 2023 11:05:27 -0400
Subject: [PATCH 0321/2492] mgr/rbd_support: remove CreateSnapshotRequests
 __del__()

There is no need for CreateSnapshotRequests.__del__() that calls
CreateSnapshotRequests.wait_for_pending().
MirrorSnapshotScheduleHandler.shutdown() already calls
CreateSnapshotRequests.wait_for_pending().

Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
index 0ddc2e6e734c..e5b19f36228d 100644
--- a/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
+++ b/src/pybind/mgr/rbd_support/mirror_snapshot_schedule.py
@@ -43,9 +43,6 @@ def __init__(self, handler: Any) -> None:
         self.queue: List[ImageSpec] = []
         self.ioctxs: Dict[Tuple[str, str], Tuple[rados.Ioctx, Set[ImageSpec]]] = {}
 
-    def __del__(self) -> None:
-        self.wait_for_pending()
-
     def wait_for_pending(self) -> None:
         with self.lock:
             while self.pending:

From 29be020aa623e174f9880f8fbb9c290b167221ba Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Fri, 29 Sep 2023 16:17:51 +0200
Subject: [PATCH 0322/2492] mgr/rook: Adding support to automatically discover
 storage classes Fixes: https://tracker.ceph.com/issues/63038 Fixes:
 https://tracker.ceph.com/issues/63338

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py       |  4 +-
 src/pybind/mgr/rook/rook_cluster.py | 73 +++++++++++++++++++----------
 2 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index 32699fa0117a..fa75db2cf9cf 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -130,7 +130,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
 
         self._load_drive_groups()
         self._shutdown = threading.Event()
-        
+
     def config_notify(self) -> None:
         """
         This method is called whenever one of our config options is changed.
@@ -147,7 +147,7 @@ def config_notify(self) -> None:
         assert isinstance(self.drive_group_interval, float)
 
         if self._rook_cluster:
-            self._rook_cluster.storage_class = self.storage_class
+            self._rook_cluster.storage_class_name = self.storage_class
 
     def shutdown(self) -> None:
         self._shutdown.set()
diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index f455581656e0..5c7c9fc0477f 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -97,14 +97,15 @@ def wrapper(*args: Any, **kwargs: Any) -> threading.Thread:
 
 
 class DefaultFetcher():
-    def __init__(self, storage_class: str, coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv'):
-        self.storage_class = storage_class
+    def __init__(self, storage_class_name: str, coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv'):
+        self.storage_class_name = storage_class_name
         self.coreV1_api = coreV1_api
         self.rook_env = rook_env
+        self.pvs_in_sc: List[client.V1PersistentVolumeList] = []
 
     def fetch(self) -> None:
         self.inventory: KubernetesResource[client.V1PersistentVolumeList] = KubernetesResource(self.coreV1_api.list_persistent_volume)
-        self.pvs_in_sc = [i for i in self.inventory.items if i.spec.storage_class_name == self.storage_class]
+        self.pvs_in_sc = [i for i in self.inventory.items if i.spec.storage_class_name == self.storage_class_name]
 
     def convert_size(self, size_str: str) -> int:
         units = ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "", "K", "M", "G", "T", "P", "E")
@@ -149,7 +150,7 @@ def device(self, i: 'client.V1PersistentVolume') -> Tuple[str, Device]:
                 available = state,
         )
         return (node, device)
-        
+
 
 class LSOFetcher(DefaultFetcher):
     def __init__(self, storage_class: 'str', coreV1_api: 'client.CoreV1Api', rook_env: 'RookEnv', customObjects_api: 'client.CustomObjectsApi', nodenames: 'Optional[List[str]]' = None):
@@ -372,9 +373,9 @@ def get_item_name(self, item: Any) -> Any:
                             self.api_func))
 
 class DefaultCreator():
-    def __init__(self, inventory: 'Dict[str, List[Device]]', coreV1_api: 'client.CoreV1Api', storage_class: 'str'):
+    def __init__(self, inventory: 'Dict[str, List[Device]]', coreV1_api: 'client.CoreV1Api', storage_class_name: 'str'):
         self.coreV1_api = coreV1_api
-        self.storage_class = storage_class
+        self.storage_class_name = storage_class_name
         self.inventory = inventory
 
     def device_to_device_set(self, drive_group: DriveGroupSpec, d: Device) -> ccl.StorageClassDeviceSetsItem:
@@ -391,7 +392,7 @@ def device_to_device_set(self, drive_group: DriveGroupSpec, d: Device) -> ccl.St
                     name="data"
                 ),
                 spec=ccl.Spec(
-                    storageClassName=self.storage_class,
+                    storageClassName=self.storage_class_name,
                     volumeMode="Block",
                     accessModes=ccl.CrdObjectList(["ReadWriteOnce"]),
                     resources={
@@ -700,7 +701,7 @@ def __init__(
         storageV1_api: 'client.StorageV1Api',
         appsV1_api: 'client.AppsV1Api',
         rook_env: 'RookEnv',
-        storage_class: 'str'
+        storage_class_name: 'str'
     ):
         self.rook_env = rook_env  # type: RookEnv
         self.coreV1_api = coreV1_api  # client.CoreV1Api
@@ -708,10 +709,11 @@ def __init__(
         self.customObjects_api = customObjects_api
         self.storageV1_api = storageV1_api  # client.StorageV1Api
         self.appsV1_api = appsV1_api  # client.AppsV1Api
-        self.storage_class = storage_class # type: str
+        self.storage_class_name = storage_class_name # type: str
 
         #  TODO: replace direct k8s calls with Rook API calls
-        self.storage_classes : KubernetesResource = KubernetesResource(self.storageV1_api.list_storage_class)
+        self.available_storage_classes : KubernetesResource = KubernetesResource(self.storageV1_api.list_storage_class)
+        self.configured_storage_classes = self.list_storage_classes()
 
         self.rook_pods: KubernetesResource[client.V1Pod] = KubernetesResource(self.coreV1_api.list_namespaced_pod,
                                                                               namespace=self.rook_env.namespace,
@@ -751,34 +753,57 @@ def rook_api_patch(self, path: str, **kwargs: Any) -> Any:
     def rook_api_post(self, path: str, **kwargs: Any) -> Any:
         return self.rook_api_call("POST", path, **kwargs)
 
+    def list_storage_classes(self) -> List[str]:
+        try:
+            crd = self.customObjects_api.get_namespaced_custom_object(
+                group="ceph.rook.io",
+                version="v1",
+                namespace=self.rook_env.namespace,
+                plural="cephclusters",
+                name=self.rook_env.cluster_name)
+
+            sc_devicesets = crd['spec']['storage']['storageClassDeviceSets']
+            sc_names = [vct['spec']['storageClassName'] for sc in sc_devicesets for vct in sc['volumeClaimTemplates']]
+            log.info(f"the cluster has the following configured sc: {sc_names}")
+            return sc_names
+        except Exception as e:
+            log.error(f"unable to list storage classes: {e}")
+            return []
+
+    # TODO: remove all the calls to code that uses rook_cluster.storage_class_name
     def get_storage_class(self) -> 'client.V1StorageClass':
-        matching_sc = [i for i in self.storage_classes.items if self.storage_class == i.metadata.name]
+        matching_sc = [i for i in self.available_storage_classes.items if self.storage_class_name == i.metadata.name]
         if len(matching_sc) == 0:
-            log.error(f"No storage class exists matching configured Rook orchestrator storage class which currently is <{self.storage_class}>. This storage class can be set in ceph config (mgr/rook/storage_class)")
+            log.error(f"No storage class exists matching configured Rook orchestrator storage class which currently is <{self.storage_class_name}>. This storage class can be set in ceph config (mgr/rook/storage_class)")
             raise Exception('No storage class exists matching name provided in ceph config at mgr/rook/storage_class')
         return matching_sc[0]
 
     def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[str, List[Device]]:
-        self.fetcher: Optional[DefaultFetcher] = None
+        discovered_devices: Dict[str, List[Device]] = {}
         op_settings = self.coreV1_api.read_namespaced_config_map(name="rook-ceph-operator-config", namespace=self.rook_env.operator_namespace).data
+        fetcher: Optional[DefaultFetcher] = None
         if op_settings.get('ROOK_ENABLE_DISCOVERY_DAEMON', 'false').lower() == 'true':
-            self.fetcher = PDFetcher(self.coreV1_api, self.rook_env)
+            fetcher = PDFetcher(self.coreV1_api, self.rook_env)
+            fetcher.fetch()
+            discovered_devices = fetcher.devices()
         else:
-            storage_class = self.get_storage_class()
-            if storage_class.metadata.labels and ('local.storage.openshift.io/owner-name' in storage_class.metadata.labels):
-                self.fetcher = LSOFetcher(self.storage_class, self.coreV1_api, self.customObjects_api, nodenames)
-            else:
-                self.fetcher = DefaultFetcher(self.storage_class, self.coreV1_api, self.rook_env)
+            active_storage_classes = [sc for sc in self.available_storage_classes.items if sc.metadata.name in self.configured_storage_classes]
+            for sc in active_storage_classes:
+                if sc.metadata.labels and ('local.storage.openshift.io/owner-name' in sc.metadata.labels):
+                    fetcher = LSOFetcher(sc.metadata.name, self.coreV1_api, self.customObjects_api, nodenames)
+                else:
+                    fetcher = DefaultFetcher(sc.metadata.name, self.coreV1_api, self.rook_env)
+                fetcher.fetch()
+                discovered_devices.update(fetcher.devices())
 
-        self.fetcher.fetch()
-        return self.fetcher.devices()
+        return discovered_devices
 
     def get_osds(self) -> List:
         osd_pods: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_pod,
                                                           namespace=self.rook_env.namespace,
                                                           label_selector='app=rook-ceph-osd')
         return list(osd_pods.items)
-        
+
     def get_nfs_conf_url(self, nfs_cluster: str, instance: str) -> Optional[str]:
         #
         # Fetch cephnfs object for "nfs_cluster" and then return a rados://
@@ -1174,9 +1199,9 @@ def add_osds(self, drive_group, matching_hosts):
             storage_class.metadata.labels
             and 'local.storage.openshift.io/owner-name' in storage_class.metadata.labels
         ):
-            creator = LSOCreator(inventory, self.coreV1_api, self.storage_class)    
+            creator = LSOCreator(inventory, self.coreV1_api, self.storage_class_name)
         else:
-            creator = DefaultCreator(inventory, self.coreV1_api, self.storage_class)
+            creator = DefaultCreator(inventory, self.coreV1_api, self.storage_class_name)
         return self._patch(
             ccl.CephCluster,
             'cephclusters',

From aa57b9c309208fa6b32005e7be5adf1b31455052 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 26 Oct 2023 19:11:12 -0400
Subject: [PATCH 0323/2492] build: Fix `WITH_FMT_HEADER_ONLY`

Previously it was still linking in the submodule's static libraries
and ldd showed it linking to the system fmt's shared libraries.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/CMakeLists.txt                       | 18 +++++++++---------
 src/common/CMakeLists.txt                |  2 +-
 src/crimson/os/alienstore/CMakeLists.txt |  2 +-
 src/crimson/osd/CMakeLists.txt           |  2 +-
 src/mon/CMakeLists.txt                   |  2 +-
 src/msg/CMakeLists.txt                   |  2 +-
 src/neorados/CMakeLists.txt              |  6 +++---
 src/osd/CMakeLists.txt                   |  2 +-
 src/rgw/CMakeLists.txt                   |  2 +-
 src/test/neorados/CMakeLists.txt         |  6 +++---
 src/tools/CMakeLists.txt                 |  2 +-
 11 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index dca5e6e6d4ab..5b1300bb84a3 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -330,20 +330,20 @@ option(WITH_SYSTEM_FMT "build against system fmt" OFF)
 if(WITH_SYSTEM_FMT)
   find_package(fmt 8.1.1...<10.0.0 REQUIRED)
 endif()
+if (WITH_FMT_HEADER_ONLY)
+  message(STATUS "Using fmt header-only.")
+  set(FMT_LIB fmt::fmt-header-only)
+else()
+  message(STATUS "Linking to fmt library.")
+  set(FMT_LIB fmt::fmt)
+endif()
 if(fmt_FOUND)
   message(STATUS "Building with system fmt.")
-  if (WITH_FMT_HEADER_ONLY)
-    message(STATUS "Using fmt header-only.")
-    add_library(fmt::fmt ALIAS fmt::fmt-header-only)
-  endif()
 else()
   message(STATUS "Building fmt as submodule")
   set(old_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
   set(BUILD_SHARED_LIBS FALSE)
   add_subdirectory(fmt)
-  if (WITH_FMT_HEADER_ONLY)
-    add_library(fmt::fmt ALIAS fmt-header-only)
-  endif()
   set(BUILD_SHARED_LIBS ${old_BUILD_SHARED_LIBS})
   unset(old_BUILD_SHARED_LIBS)
   include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/fmt/include")
@@ -439,7 +439,7 @@ set_source_files_properties(ceph_ver.c
   APPEND PROPERTY OBJECT_DEPENDS ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h)
 add_library(common-objs OBJECT ${libcommon_files})
 target_compile_definitions(common-objs PRIVATE
-  $<TARGET_PROPERTY:fmt::fmt,INTERFACE_COMPILE_DEFINITIONS>)
+  $<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
 add_dependencies(common-objs legacy-option-headers)
 
 if(WITH_JAEGER)
@@ -482,7 +482,7 @@ set(ceph_common_deps
   Boost::date_time
   Boost::iostreams
   StdFilesystem::filesystem
-  fmt::fmt
+  ${FMT_LIB}
   ${BLKID_LIBRARIES}
   ${Backtrace_LIBRARIES}
   ${BLKIN_LIBRARIES}
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 5f07f7e8483c..3deecc3f3d6b 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -192,7 +192,7 @@ target_compile_definitions(common-common-objs PRIVATE
   "CMAKE_INSTALL_LIBDIR=\"${CMAKE_INSTALL_LIBDIR}\""
   "CEPH_INSTALL_FULL_PKGLIBDIR=\"${CEPH_INSTALL_FULL_PKGLIBDIR}\""
   "CEPH_INSTALL_DATADIR=\"${CEPH_INSTALL_DATADIR}\""
-  $<TARGET_PROPERTY:fmt::fmt,INTERFACE_COMPILE_DEFINITIONS>)
+  $<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
 add_dependencies(common-common-objs legacy-option-headers)
 
 set(common_mountcephfs_srcs
diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt
index c881f4fbccbc..20e03d3305ff 100644
--- a/src/crimson/os/alienstore/CMakeLists.txt
+++ b/src/crimson/os/alienstore/CMakeLists.txt
@@ -76,7 +76,7 @@ endif()
 target_link_libraries(crimson-alienstore
   PRIVATE
     alien::cflags
-    fmt::fmt
+    ${FMT_LIB}
     kv
     heap_profiler
     crimson-alien-common
diff --git a/src/crimson/osd/CMakeLists.txt b/src/crimson/osd/CMakeLists.txt
index f521e0244d7a..817027c18ffc 100644
--- a/src/crimson/osd/CMakeLists.txt
+++ b/src/crimson/osd/CMakeLists.txt
@@ -61,7 +61,7 @@ target_link_libraries(crimson-osd
   crimson-common
   crimson-os
   crimson
-  fmt::fmt
+  ${FMT_LIB}
   Boost::MPL
   dmclock::dmclock)
 set_target_properties(crimson-osd PROPERTIES
diff --git a/src/mon/CMakeLists.txt b/src/mon/CMakeLists.txt
index 784b4c3ee0b3..3f2b8605344b 100644
--- a/src/mon/CMakeLists.txt
+++ b/src/mon/CMakeLists.txt
@@ -40,7 +40,7 @@ add_library(mon STATIC
 target_link_libraries(mon
   kv
   heap_profiler
-  fmt::fmt)
+  ${FMT_LIB})
 if(WITH_JAEGER)
   target_link_libraries(mon jaeger_base)
 endif()
diff --git a/src/msg/CMakeLists.txt b/src/msg/CMakeLists.txt
index e8cc3fdabdf6..ff2d47276f25 100644
--- a/src/msg/CMakeLists.txt
+++ b/src/msg/CMakeLists.txt
@@ -46,7 +46,7 @@ endif()
 
 add_library(common-msg-objs OBJECT ${msg_srcs})
 target_compile_definitions(common-msg-objs PRIVATE
-  $<TARGET_PROPERTY:fmt::fmt,INTERFACE_COMPILE_DEFINITIONS>)
+  $<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
 target_include_directories(common-msg-objs PRIVATE ${OPENSSL_INCLUDE_DIR})
 
 if(WITH_DPDK)
diff --git a/src/neorados/CMakeLists.txt b/src/neorados/CMakeLists.txt
index c66043ac9f94..119554f67d10 100644
--- a/src/neorados/CMakeLists.txt
+++ b/src/neorados/CMakeLists.txt
@@ -1,17 +1,17 @@
 add_library(neorados_objs OBJECT
   RADOSImpl.cc)
 target_compile_definitions(neorados_objs PRIVATE
-  $<TARGET_PROPERTY:fmt::fmt,INTERFACE_COMPILE_DEFINITIONS>)
+  $<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
 add_library(neorados_api_obj OBJECT
   RADOS.cc)
 target_compile_definitions(neorados_api_obj PRIVATE
-  $<TARGET_PROPERTY:fmt::fmt,INTERFACE_COMPILE_DEFINITIONS>)
+  $<TARGET_PROPERTY:${FMT_LIB},INTERFACE_COMPILE_DEFINITIONS>)
 
 add_library(libneorados STATIC
   $<TARGET_OBJECTS:neorados_api_obj>
   $<TARGET_OBJECTS:neorados_objs>)
 target_link_libraries(libneorados PRIVATE
-  osdc ceph-common cls_lock_client fmt::fmt
+  osdc ceph-common cls_lock_client ${FMT_LIB}
   ${BLKID_LIBRARIES} ${CRYPTO_LIBS} ${EXTRALIBS})
 
 # if(ENABLE_SHARED)
diff --git a/src/osd/CMakeLists.txt b/src/osd/CMakeLists.txt
index 7d19424b404a..85f65f7f0e19 100644
--- a/src/osd/CMakeLists.txt
+++ b/src/osd/CMakeLists.txt
@@ -56,7 +56,7 @@ endif()
 add_library(osd STATIC ${osd_srcs})
 target_link_libraries(osd
   PUBLIC dmclock::dmclock Boost::MPL
-  PRIVATE os heap_profiler cpu_profiler fmt::fmt ${CMAKE_DL_LIBS})
+  PRIVATE os heap_profiler cpu_profiler ${FMT_LIB} ${CMAKE_DL_LIBS})
 if(WITH_LTTNG)
   add_dependencies(osd osd-tp pg-tp)
 endif()
diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index c90a553abee0..9b55081dbaaa 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -284,7 +284,7 @@ target_link_libraries(rgw_common
     ${LUA_LIBRARIES}
     RapidJSON::RapidJSON
     spawn
-    fmt::fmt)
+    ${FMT_LIB})
 target_include_directories(rgw_common
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/services"
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index 31e79a6615c4..ba96fb479d29 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -17,12 +17,12 @@ target_link_libraries(ceph_test_neorados_completions Boost::system pthread
 
 add_executable(ceph_test_neorados_op_speed op_speed.cc)
 target_link_libraries(ceph_test_neorados_op_speed
-  libneorados fmt::fmt ${unittest_libs})
+  libneorados ${FMT_LIB} ${unittest_libs})
 
 add_library(neoradostest-support STATIC common_tests.cc)
 target_link_libraries(neoradostest-support
-  libneorados fmt::fmt)
+  libneorados ${FMT_LIB})
 
 add_executable(ceph_test_neorados_list_pool list_pool.cc)
 target_link_libraries(ceph_test_neorados_list_pool
-  libneorados neoradostest-support global fmt::fmt ${unittest_libs})
+  libneorados neoradostest-support global ${FMT_LIB} ${unittest_libs})
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index aeb9d0248ea6..d1133798c5c9 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -20,7 +20,7 @@ if(NOT WIN32)
   set(neorados_srcs
       neorados.cc)
   add_executable(neorados ${neorados_srcs})
-  target_link_libraries(neorados libneorados spawn fmt::fmt ${CMAKE_DL_LIBS})
+  target_link_libraries(neorados libneorados spawn ${FMT_LIB} ${CMAKE_DL_LIBS})
   #install(TARGETS neorados DESTINATION bin)
 endif()
 

From 65cc32e1abef4ca737207878fac18395af0aac70 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Mon, 30 Oct 2023 21:22:24 +0530
Subject: [PATCH 0324/2492] doc: Release notes 17.2.7 with updated dashboard
 info

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 doc/releases/quincy.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/releases/quincy.rst b/doc/releases/quincy.rst
index 89653ae935a1..4954a6fed9be 100644
--- a/doc/releases/quincy.rst
+++ b/doc/releases/quincy.rst
@@ -69,6 +69,10 @@ Notable Changes
   recovery procedure, the recovered files under `lost+found` directory can now
   be deleted.
 
+* Dashboard: There is a new Dashboard page with an improved layout. Active alerts
+  and some important charts are now displayed inside cards. This new dashboard can
+  be disabled and the older layout brought back by setting ``ceph dashboard feature disable dashboard``.
+
 Changelog
 ---------
 

From b88cecdc7c3dc048550f634ecadc04f661f0fabc Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Tue, 6 Sep 2022 19:40:43 +0530
Subject: [PATCH 0325/2492] DaemonServer.cc: fix config show command for RGW
 daemons

RGW daemons register in the servicemap by gid which allows multiple radosgw instances to share an auth key/identity. The daemon name is sent as part of the metadata.  (https://github.com/ceph/ceph/commit/84c265238b796935b3aa66d191593b2e8655f384).
All other daemons register by the daemon name and the manager stores all daemon state information with daemon name as key. The 'config show' command looks up the daemon_state map with the daemon name the user mentions as key (for example: 'osd.0', 'client.rgw', 'mon.a').
Due to the change in RGW daemon registration, the key used for storing daemon state has become rgw.gid and 'config show client.rgw' no longer works.

This change will take care of going through the daemon metadata to look for the RGW daemon name when a user enters the config show command for a RGW daemon. Once the correct daemon is found, we retrieve the corresponding daemon key (rgw.gid) and use that to query the daemon_state map.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2011756
Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 qa/suites/rados/singleton/all/mon-config.yaml |  3 +-
 qa/workunits/mon/config.sh                    |  7 +++++
 src/mgr/DaemonServer.cc                       | 31 +++++++++++++++++++
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/qa/suites/rados/singleton/all/mon-config.yaml b/qa/suites/rados/singleton/all/mon-config.yaml
index ab1eb81b0948..5e36a34a6c1b 100644
--- a/qa/suites/rados/singleton/all/mon-config.yaml
+++ b/qa/suites/rados/singleton/all/mon-config.yaml
@@ -6,7 +6,7 @@ roles:
   - osd.0
   - osd.1
   - osd.2
-  - client.0
+  - client.rgw
 openstack:
   - volumes: # attached to each instance
       count: 3
@@ -18,6 +18,7 @@ tasks:
       - sudo ceph config set mgr mgr_pool false --force
     log-ignorelist:
       - \(POOL_APP_NOT_ENABLED\)
+- rgw: [client.rgw]
 - workunit:
     clients:
       all:
diff --git a/qa/workunits/mon/config.sh b/qa/workunits/mon/config.sh
index 1b00201ae481..42abcf91e212 100755
--- a/qa/workunits/mon/config.sh
+++ b/qa/workunits/mon/config.sh
@@ -111,6 +111,13 @@ do
 done
 ceph config rm osd.0 osd_scrub_cost
 
+#RGW daemons test config set
+ceph config set client.rgw debug_rgw 22
+while ! ceph config show client.rgw | grep debug_rgw | grep 22 | grep mon
+do
+    sleep 1
+done
+
 # show-with-defaults
 ceph config show-with-defaults osd.0 | grep debug_asok
 
diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index a4e85e2fc421..39c200aaa368 100644
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -1991,6 +1991,37 @@ bool DaemonServer::_handle_command(
       cmdctx->reply(-EINVAL, ss);
       return true;
     }
+    /*
+     *  RGW has the daemon name stored in the daemon metadata
+     *  and uses the GID as key in the service_map.
+     *  We need to match the user's query with the daemon name to
+     *  find the correct key for retrieving daemon state.
+     */
+    string daemon_name = key.name;
+    auto p = daemon_name.find("rgw");
+    if (p != daemon_name.npos) {
+      auto rgw_daemons = daemon_state.get_by_service("rgw");
+      for (auto& rgw_daemon : rgw_daemons) {
+	DaemonStatePtr daemon = rgw_daemon.second;
+	string name = daemon->metadata.find("id")->second;
+	/*
+	 * The id stored in the metadata is the port number
+	 * for the RGW daemon.
+	 * In the case of multiple RGW daemons, the user might
+	 * use the port number (rgw.8000) to specify the daemon.
+	 */
+	auto p = daemon_name.find('.');
+	if (p == key.name.npos) {
+          key = daemon->key;
+	} else {
+	  // if user has specified port number in the query
+	  if (daemon_name.substr(p + 1) == name) {
+	    key = daemon->key;
+	    break;
+	  }
+        }
+      }
+    }
     DaemonStatePtr daemon = daemon_state.get(key);
     if (!daemon) {
       ss << "no config state for daemon " << who;

From 48acf4712a36936306acfc887f0ab205d3bfe8b4 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Mon, 30 Oct 2023 15:57:04 -0400
Subject: [PATCH 0326/2492] rgw: handle instances w/ leading hyphens correctly

Some instance identifiers have leading hyphens, which broke the
rgw-restore-bucket-index script. This fixes those issues and adds a
few features to making debugging easier in the future.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/rgw-restore-bucket-index | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/rgw/rgw-restore-bucket-index b/src/rgw/rgw-restore-bucket-index
index 512ca24d9ec2..243a4f49068e 100755
--- a/src/rgw/rgw-restore-bucket-index
+++ b/src/rgw/rgw-restore-bucket-index
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# version 2023-07-06
+# version 2023-10-30
 
 # rgw-restore-bucket-index is an EXPERIMENTAL tool to use in case
 # bucket index entries for objects in the bucket are somehow lost. It
@@ -37,13 +37,18 @@ export obj_reindex_script=/tmp/rgwrbi-object-list-script.$$
 export zone_info=/tmp/rgwrbi-zone-info.$$
 export olh_info_enc=/tmp/rgwrbi-olh-info-enc.$$
 export olh_info_json=/tmp/rgwrbi-olh-info-json.$$
+export debug_log=/tmp/rgwrbi-debug-log.$$
 
 export clean_temps=1
 
+# make explicit tabs easier to see in code
+export TAB="	"
+
 
 clean() {
-  if [ -n "$clean_temps" ] ;then
-    rm -f $bkt_entry $bkt_inst $marker_ls $obj_list $obj_list_ver $obj_reindex_script $zone_info $olh_info_enc $olh_info_json
+  if [ "$clean_temps" == 1 ] ;then
+    rm -f $bkt_entry $bkt_inst $marker_ls $obj_list $obj_list_ver \
+       $obj_reindex_script $zone_info $olh_info_enc $olh_info_json
   fi
 }
 
@@ -178,8 +183,9 @@ while getopts "b:l:p:r:g:z:yd" o; do
 	    proceed=1
 	    ;;
 	d)
-	    echo setting debug to 1
+	    echo Debugging On
 	    debug=1
+	    clean_temps=0
 	    ;;
 	*)
 	    echo
@@ -217,15 +223,24 @@ handle_versioned() {
 	    filter_out_last_instance="$last_instance"
 	fi
 
+	if [ "$debug" == 1 ] ;then
+	    echo "working on versioned $o"
+	    echo "last instance is $last_instance"
+	    echo "filter_out_last_instance is $filter_out_last_instance"
+	fi >>$debug_log
+
 	# we currently don't need the delete marker, but we can have access to it
 	# delete_marker=$(jq -r ".removed" $olh_info_json) # true or false
 
 	IFS='\t' grep -E "(__:.*[^_])?_$o(\t.*)?$" $marker_ls | # versioned head objects
 	    while read obj loc ;do
+		if [ "$debug" == 1 ] ;then
+		    echo "obj=$obj ; loc=$loc" >>$debug_log
+		fi
 		rados -p $pool stat2 $obj --object-locator "$loc"
 	    done | # output of stat2, which includes mtime
 	    sort -k 3 | # stat2 but sorted by mtime earlier to later
-	    grep -v "$filter_out_last_instance" | # remove the final instance in case it's not last
+	    grep -v -e "$filter_out_last_instance" | # remove the final instance in case it's not last
 
 	    # sed 1) removes pool and marker, 2) removes indicator of
 	    # version id, 3) removes obj name including escaped
@@ -238,9 +253,9 @@ handle_versioned() {
 		-e 's/^__://' \
 		-e "s/_+${o}.*//" \
 		-e "s/^/${o}\t/"
-	echo "$o	$last_instance" # now add the final instance; could be delete marker; note TAB
+	echo "${o}${TAB}$last_instance" # now add the final instance; could be delete marker
     done <$obj_list 2>/dev/null | sed 's/\t$//' >$obj_list_ver
-}
+} # handle_versioned
 
 if [ -z "$bucket" ]; then
   echo

From 188aed9d779a168d999be9f652bcceecbfd64f05 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 31 Oct 2023 10:30:18 +0800
Subject: [PATCH 0327/2492] mds: add trim_completed_request_list() helper

Fixes: https://tracker.ceph.com/issues/63364
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/Server.cc | 55 ++++++++++++++++++++++++++---------------------
 src/mds/Server.h  |  1 +
 2 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index ecbb781188ee..dcd5daafc251 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2448,6 +2448,35 @@ void Server::set_trace_dist(const ref_t<MClientReply> &reply,
   reply->set_trace(bl);
 }
 
+// trim completed_request list
+void Server::trim_completed_request_list(ceph_tid_t tid, Session *session)
+{
+  if (tid == UINT64_MAX || !session)
+    return;
+
+  dout(15) << " oldest_client_tid=" << tid << dendl;
+  if (session->trim_completed_requests(tid)) {
+    // Sessions 'completed_requests' was dirtied, mark it to be
+    // potentially flushed at segment expiry.
+    mdlog->get_current_segment()->touched_sessions.insert(session->info.inst.name);
+
+    if (session->get_num_trim_requests_warnings() > 0 &&
+        session->get_num_completed_requests() * 2 < g_conf()->mds_max_completed_requests)
+      session->reset_num_trim_requests_warnings();
+  } else {
+    if (session->get_num_completed_requests() >=
+        (g_conf()->mds_max_completed_requests << session->get_num_trim_requests_warnings())) {
+      session->inc_num_trim_requests_warnings();
+      CachedStackStringStream css;
+      *css << "client." << session->get_client() << " does not advance its oldest_client_tid ("
+         << tid << "), " << session->get_num_completed_requests()
+         << " completed requests recorded in session\n";
+      mds->clog->warn() << css->strv();
+      dout(20) << __func__ << " " << css->strv() << dendl;
+    }
+  }
+}
+
 void Server::handle_client_request(const cref_t<MClientRequest> &req)
 {
   dout(4) << "handle_client_request " << *req << dendl;
@@ -2529,31 +2558,7 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
   }
 
   // trim completed_request list
-  if (req->get_oldest_client_tid() > 0) {
-    dout(15) << " oldest_client_tid=" << req->get_oldest_client_tid() << dendl;
-    ceph_assert(session);
-    if (session->trim_completed_requests(req->get_oldest_client_tid())) {
-      // Sessions 'completed_requests' was dirtied, mark it to be
-      // potentially flushed at segment expiry.
-      mdlog->get_current_segment()->touched_sessions.insert(session->info.inst.name);
-
-      if (session->get_num_trim_requests_warnings() > 0 &&
-	  session->get_num_completed_requests() * 2 < g_conf()->mds_max_completed_requests)
-	session->reset_num_trim_requests_warnings();
-    } else {
-      if (session->get_num_completed_requests() >=
-	  (g_conf()->mds_max_completed_requests << session->get_num_trim_requests_warnings())) {
-	session->inc_num_trim_requests_warnings();
-	CachedStackStringStream css;
-	*css << "client." << session->get_client() << " does not advance its oldest_client_tid ("
-	   << req->get_oldest_client_tid() << "), "
-	   << session->get_num_completed_requests()
-	   << " completed requests recorded in session\n";
-	mds->clog->warn() << css->strv();
-	dout(20) << __func__ << " " << css->strv() << dendl;
-      }
-    }
-  }
+  trim_completed_request_list(req->get_oldest_client_tid(), session);
 
   // register + dispatch
   MDRequestRef mdr = mdcache->request_start(req);
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 6b2f9c188f5a..4715a0880c83 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -158,6 +158,7 @@ class Server {
   void force_clients_readonly();
 
   // -- requests --
+  void trim_completed_request_list(ceph_tid_t tid, Session *session);
   void handle_client_request(const cref_t<MClientRequest> &m);
   void handle_client_reply(const cref_t<MClientReply> &m);
 

From 0cff6d5d487972a850e7c64b1e3b8ca283bf3c03 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 31 Oct 2023 10:23:09 +0800
Subject: [PATCH 0328/2492] mds,client: update the oldest_client_tid via the
 renew caps

Update the oldest_client_tid via the session renew caps msg to
make sure that the MDSs won't pile up the completed request list
in a very large size.

Fixes: https://tracker.ceph.com/issues/63364
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/client/Client.cc          | 4 +++-
 src/mds/Server.cc             | 1 +
 src/messages/MClientSession.h | 7 ++++++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 7d223018c762..4a0cb61fd17a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -7254,7 +7254,9 @@ void Client::renew_caps(MetaSession *session)
   ldout(cct, 10) << "renew_caps mds." << session->mds_num << dendl;
   session->last_cap_renew_request = ceph_clock_now();
   uint64_t seq = ++session->cap_renew_seq;
-  session->con->send_message2(make_message<MClientSession>(CEPH_SESSION_REQUEST_RENEWCAPS, seq));
+  auto m = make_message<MClientSession>(CEPH_SESSION_REQUEST_RENEWCAPS, seq);
+  m->oldest_client_tid = oldest_tid;
+  session->con->send_message2(std::move(m));
 }
 
 
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index dcd5daafc251..cbcaf1a6b330 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -785,6 +785,7 @@ void Server::handle_client_session(const cref_t<MClientSession> &m)
 	mds->locker->resume_stale_caps(session);
 	mds->sessionmap.touch_session(session);
       }
+      trim_completed_request_list(m->oldest_client_tid, session);
       auto reply = make_message<MClientSession>(CEPH_SESSION_RENEWCAPS, m->get_seq());
       mds->send_message_client(reply, session);
     } else {
diff --git a/src/messages/MClientSession.h b/src/messages/MClientSession.h
index 1034707faacd..3e70370a52d9 100644
--- a/src/messages/MClientSession.h
+++ b/src/messages/MClientSession.h
@@ -21,7 +21,7 @@
 
 class MClientSession final : public SafeMessage {
 private:
-  static constexpr int HEAD_VERSION = 6;
+  static constexpr int HEAD_VERSION = 7;
   static constexpr int COMPAT_VERSION = 1;
 
 public:
@@ -33,6 +33,7 @@ class MClientSession final : public SafeMessage {
   feature_bitset_t supported_features;
   metric_spec_t metric_spec;
   std::vector<MDSCapAuth> cap_auths;
+  ceph_tid_t oldest_client_tid = UINT64_MAX;
 
   int get_op() const { return head.op; }
   version_t get_seq() const { return head.seq; }
@@ -88,6 +89,9 @@ class MClientSession final : public SafeMessage {
     if (header.version >= 6) {
       decode(cap_auths, p);
     }
+    if (header.version >= 7) {
+      decode(oldest_client_tid, p);
+    }
   }
   void encode_payload(uint64_t features) override { 
     using ceph::encode;
@@ -104,6 +108,7 @@ class MClientSession final : public SafeMessage {
       encode(metric_spec, payload);
       encode(flags, payload);
       encode(cap_auths, payload);
+      encode(oldest_client_tid, payload);
     }
   }
 private:

From 1c579f2469be3083b9a99bc20298769f1bc24892 Mon Sep 17 00:00:00 2001
From: Jiffin Tony Thottan <thottanjiffin@gmail.com>
Date: Tue, 31 Oct 2023 12:50:52 +0530
Subject: [PATCH 0329/2492] rgw/rgw_period : remove realm name from period
 config

The realm name is used in period configuration, only initialised in the
beginning. Only value which is needed is the realm_id, hence removing
realm_name from period config.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>
---
 src/rgw/driver/rados/rgw_rest_realm.cc |  5 ++---
 src/rgw/driver/rados/rgw_trim_mdlog.cc |  7 +++----
 src/rgw/driver/rados/rgw_zone.cc       |  1 -
 src/rgw/driver/rados/rgw_zone.h        |  6 +-----
 src/rgw/rgw_period.cc                  | 12 ++++--------
 src/rgw/rgw_period_pusher.cc           |  2 +-
 src/rgw/rgw_realm.cc                   |  6 +++---
 src/rgw/services/svc_zone.cc           |  3 +--
 8 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rest_realm.cc b/src/rgw/driver/rados/rgw_rest_realm.cc
index 79640a2a1081..91381d772954 100644
--- a/src/rgw/driver/rados/rgw_rest_realm.cc
+++ b/src/rgw/driver/rados/rgw_rest_realm.cc
@@ -65,17 +65,16 @@ class RGWOp_Period_Get : public RGWOp_Period_Base {
 
 void RGWOp_Period_Get::execute(optional_yield y)
 {
-  string realm_id, realm_name, period_id;
+  string realm_id, period_id;
   epoch_t epoch = 0;
   RESTArgs::get_string(s, "realm_id", realm_id, &realm_id);
-  RESTArgs::get_string(s, "realm_name", realm_name, &realm_name);
   RESTArgs::get_string(s, "period_id", period_id, &period_id);
   RESTArgs::get_uint32(s, "epoch", 0, &epoch);
 
   period.set_id(period_id);
   period.set_epoch(epoch);
 
-  op_ret = period.init(this, driver->ctx(), static_cast<rgw::sal::RadosStore*>(driver)->svc()->sysobj, realm_id, y, realm_name);
+  op_ret = period.init(this, driver->ctx(), static_cast<rgw::sal::RadosStore*>(driver)->svc()->sysobj, realm_id, y);
   if (op_ret < 0)
     ldpp_dout(this, 5) << "failed to read period" << dendl;
 }
diff --git a/src/rgw/driver/rados/rgw_trim_mdlog.cc b/src/rgw/driver/rados/rgw_trim_mdlog.cc
index d97538f98321..da649366c0fd 100644
--- a/src/rgw/driver/rados/rgw_trim_mdlog.cc
+++ b/src/rgw/driver/rados/rgw_trim_mdlog.cc
@@ -729,8 +729,8 @@ bool sanity_check_endpoints(const DoutPrefixProvider *dpp, rgw::sal::RadosStore*
 	<< __PRETTY_FUNCTION__ << ":" << __LINE__
 	<< " WARNING: Cluster is is misconfigured! "
 	<< " Zonegroup " << zonegroup.get_name()
-	<< " (" << zonegroup.get_id() << ") in Realm "
-	<< period.get_realm_name() << " ( " << period.get_realm() << ") "
+	<< " (" << zonegroup.get_id() << ") in Realm id ( "
+  << period.get_realm() << ") "
 	<< " has no endpoints!" << dendl;
     }
     for (const auto& [_, zone] : zonegroup.zones) {
@@ -740,8 +740,7 @@ bool sanity_check_endpoints(const DoutPrefixProvider *dpp, rgw::sal::RadosStore*
 	  << " ERROR: Cluster is is misconfigured! "
 	  << " Zone " << zone.name << " (" << zone.id << ") in Zonegroup "
 	  << zonegroup.get_name() << " ( " << zonegroup.get_id()
-	  << ") in Realm " << period.get_realm_name()
-	  << " ( " << period.get_realm() << ") "
+	  << ") in Realm id ( " << period.get_realm() << ") "
 	  << " has no endpoints! Trimming is impossible." << dendl;
 	retval = false;
       }
diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index 8c237f6bedac..850748c8ff1f 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -516,7 +516,6 @@ int create_realm(const DoutPrefixProvider* dpp, optional_yield y,
     period->period_map.id = period->id;
     period->epoch = FIRST_EPOCH;
     period->realm_id = info.id;
-    period->realm_name = info.name;
 
     r = cfgstore->create_period(dpp, y, true, *period);
     if (r < 0) {
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 140678fcdce5..fea7bdb21838 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -642,7 +642,6 @@ class RGWPeriod
   rgw_zone_id master_zone;
 
   std::string realm_id;
-  std::string realm_name;
   epoch_t realm_epoch{1}; //< realm epoch when period was made current
 
   CephContext *cct{nullptr};
@@ -678,7 +677,6 @@ class RGWPeriod
   const rgw_zone_id& get_master_zone() const { return master_zone; }
   const std::string& get_master_zonegroup() const { return master_zonegroup; }
   const std::string& get_realm() const { return realm_id; }
-  const std::string& get_realm_name() const { return realm_name; }
   const RGWPeriodMap& get_map() const { return period_map; }
   RGWPeriodConfig& get_config() { return period_config; }
   const RGWPeriodConfig& get_config() const { return period_config; }
@@ -750,7 +748,7 @@ class RGWPeriod
   int update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, optional_yield y);
 
   int init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, const std::string &period_realm_id, optional_yield y,
-	   const std::string &period_realm_name = "", bool setup_obj = true);
+	    bool setup_obj = true);
   int init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, optional_yield y, bool setup_obj = true);  
 
   int create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive = true);
@@ -780,7 +778,6 @@ class RGWPeriod
     encode(master_zonegroup, bl);
     encode(period_config, bl);
     encode(realm_id, bl);
-    encode(realm_name, bl);
     ENCODE_FINISH(bl);
   }
 
@@ -796,7 +793,6 @@ class RGWPeriod
     decode(master_zonegroup, bl);
     decode(period_config, bl);
     decode(realm_id, bl);
-    decode(realm_name, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc
index 1e7de60ea6fa..67ce98039513 100644
--- a/src/rgw/rgw_period.cc
+++ b/src/rgw/rgw_period.cc
@@ -22,10 +22,10 @@ int RGWPeriod::init(const DoutPrefixProvider *dpp,
     return 0;
 
   if (id.empty()) {
-    RGWRealm realm(realm_id, realm_name);
+    RGWRealm realm(realm_id);
     int ret = realm.init(dpp, cct, sysobj_svc, y);
     if (ret < 0) {
-      ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm " << realm_name << " id " << realm_id << " : " <<
+      ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm  id " << realm_id << " : " <<
 	cpp_strerror(-ret) << dendl;
       return ret;
     }
@@ -36,7 +36,7 @@ int RGWPeriod::init(const DoutPrefixProvider *dpp,
   if (!epoch) {
     int ret = use_latest_epoch(dpp, y);
     if (ret < 0) {
-      ldpp_dout(dpp, 0) << "failed to use_latest_epoch period id " << id << " realm " << realm_name  << " id " << realm_id
+      ldpp_dout(dpp, 0) << "failed to use_latest_epoch period id " << id << " realm id " << realm_id
 	   << " : " << cpp_strerror(-ret) << dendl;
       return ret;
     }
@@ -46,14 +46,12 @@ int RGWPeriod::init(const DoutPrefixProvider *dpp,
 }
 
 int RGWPeriod::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc,
-		    const string& period_realm_id, optional_yield y,
-		    const string& period_realm_name, bool setup_obj)
+		    const string& period_realm_id, optional_yield y, bool setup_obj)
 {
   cct = _cct;
   sysobj_svc = _sysobj_svc;
 
   realm_id = period_realm_id;
-  realm_name = period_realm_name;
 
   if (!setup_obj)
     return 0;
@@ -244,7 +242,6 @@ void RGWPeriod::dump(Formatter *f) const
   encode_json("master_zone", master_zone, f);
   encode_json("period_config", period_config, f);
   encode_json("realm_id", realm_id, f);
-  encode_json("realm_name", realm_name, f);
   encode_json("realm_epoch", realm_epoch, f);
 }
 
@@ -259,7 +256,6 @@ void RGWPeriod::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("master_zone", master_zone, obj);
   JSONDecoder::decode_json("period_config", period_config, obj);
   JSONDecoder::decode_json("realm_id", realm_id, obj);
-  JSONDecoder::decode_json("realm_name", realm_name, obj);
   JSONDecoder::decode_json("realm_epoch", realm_epoch, obj);
 }
 
diff --git a/src/rgw/rgw_period_pusher.cc b/src/rgw/rgw_period_pusher.cc
index d9c899e5c1c5..0fc9efa85e75 100644
--- a/src/rgw/rgw_period_pusher.cc
+++ b/src/rgw/rgw_period_pusher.cc
@@ -175,7 +175,7 @@ RGWPeriodPusher::RGWPeriodPusher(const DoutPrefixProvider *dpp, rgw::sal::Driver
   // always send out the current period on startup
   RGWPeriod period;
   // XXX dang
-  int r = period.init(dpp, cct, static_cast<rgw::sal::RadosStore* >(driver)->svc()->sysobj, realm_id, y, zone->get_realm_name());
+  int r = period.init(dpp, cct, static_cast<rgw::sal::RadosStore* >(driver)->svc()->sysobj, realm_id, y);
   if (r < 0) {
     ldpp_dout(dpp, -1) << "failed to load period for realm " << realm_id << dendl;
     return;
diff --git a/src/rgw/rgw_realm.cc b/src/rgw/rgw_realm.cc
index 8dd6d6f50b99..2d854e7244f5 100644
--- a/src/rgw/rgw_realm.cc
+++ b/src/rgw/rgw_realm.cc
@@ -64,7 +64,7 @@ int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclu
   RGWPeriod period;
   if (current_period.empty()) {
     /* create new period for the realm */
-    ret = period.init(dpp, cct, sysobj_svc, id, y, name, false);
+    ret = period.init(dpp, cct, sysobj_svc, id, y, false);
     if (ret < 0 ) {
       return ret;
     }
@@ -75,7 +75,7 @@ int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclu
     }
   } else {
     period = RGWPeriod(current_period, 0);
-    int ret = period.init(dpp, cct, sysobj_svc, id, y, name);
+    int ret = period.init(dpp, cct, sysobj_svc, id, y);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "ERROR: failed to init period " << current_period << dendl;
       return ret;
@@ -228,7 +228,7 @@ int RGWRealm::find_zone(const DoutPrefixProvider *dpp,
   epoch_t epoch = 0;
 
   RGWPeriod period(period_id, epoch);
-  int r = period.init(dpp, cct, sysobj_svc, get_id(), y, get_name());
+  int r = period.init(dpp, cct, sysobj_svc, get_id(), y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "WARNING: period init failed: " << cpp_strerror(-r) << " ... skipping" << dendl;
     return r;
diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc
index 180d9371222b..c0530ae67431 100644
--- a/src/rgw/services/svc_zone.cc
+++ b/src/rgw/services/svc_zone.cc
@@ -146,8 +146,7 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp)
   }
 
   ldpp_dout(dpp, 20) << "realm  " << realm->get_name() << " " << realm->get_id() << dendl;
-  ret = current_period->init(dpp, cct, sysobj_svc, realm->get_id(), y,
-                             realm->get_name());
+  ret = current_period->init(dpp, cct, sysobj_svc, realm->get_id(), y);
   if (ret < 0 && ret != -ENOENT) {
     ldpp_dout(dpp, 0) << "failed reading current period info: " << " " << cpp_strerror(-ret) << dendl;
     return ret;

From 4af9f8eb6c189fcaec07ac64949c3037a92e7e8c Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Wed, 18 Oct 2023 14:22:56 +0530
Subject: [PATCH 0330/2492] qa: fix rank_asok() to handle errors from asok
 commands

Fixes: https://tracker.ceph.com/issues/63176
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 qa/tasks/cephfs/filesystem.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 3516bf4b86cd..a49d2d05c912 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -272,8 +272,14 @@ def get_nonnumeric_values(value):
                      "-Infinity": -float("inf")}
                 return c[value]
 
-            j = json.loads(response_data.replace('inf', 'Infinity'),
-                           parse_constant=get_nonnumeric_values)
+            
+            j = {}
+            try:
+                j = json.loads(response_data.replace('inf', 'Infinity'),
+                            parse_constant=get_nonnumeric_values)
+            except json.decoder.JSONDecodeError:
+                raise RuntimeError(response_data) # assume it is an error message, pass it up
+            
             pretty = json.dumps(j, sort_keys=True, indent=2)
             log.debug(f"_json_asok output\n{pretty}")
             return j

From 8713cca328c9373636efdb92449d743b5bd56584 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 30 Oct 2023 12:37:39 +1000
Subject: [PATCH 0331/2492] doc/glossary: improve "BlueStore" entry

Initially s/backend/back end/ but then I added a little more information
about BlueStore's use of RocksDB to map object names to block locations
on disk.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 524dd01a20ec..d5619e464c8b 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -15,10 +15,12 @@
                 introduced in the Ceph Kraken release. The Luminous release of
                 Ceph promoted BlueStore to the default OSD back end,
                 supplanting FileStore. As of the Reef release, FileStore is no
-                longer available as a storage backend.
+                longer available as a storage back end.
                 
-                BlueStore stores objects directly on Ceph block devices without
-                a mounted file system.  
+                BlueStore stores objects directly on raw block devices or
+                partitions, and does not interact with mounted file systems.
+                BlueStore uses RocksDB's key/value database to map object names
+                to block locations on disk.
 
         Bucket
                 In the context of :term:`RGW`, a bucket is a group of objects.

From 4719bfbda61fbdca225f96981d70b2b63ec262b7 Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Tue, 31 Oct 2023 13:56:53 +0530
Subject: [PATCH 0332/2492] doc/dev/developer_guide: mention
 unit_test_summary.yaml & valgrind.yaml

In tests-integration-testing-teuthology-workflow.rst,
briefly mention unit_test_summary.yaml and valgrind.yaml
as other files that exist in teuthology archive.

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 .../tests-integration-testing-teuthology-workflow.rst     | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
index 64b006c57fb4..6b8877886a78 100644
--- a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
@@ -209,6 +209,14 @@ For example: for the above test ID, the path is::
 
 This method can be used to view the log more quickly than would be possible through a browser.
 
+In addition to teuthology.log, there are other files included for debugging purposes:
+
+* unit_test_summary.yaml: It provides a summary of all unit test failures and 
+  is optionally generated when the ``unit_test_scan`` configuration option is 
+  used in the job's YAML file.
+
+* valgrind.yaml: It summarizes any Valgrind errors that may occur.
+
 .. note:: To access archives more conveniently, ``/a/`` has been symbolically
    linked to ``/ceph/teuthology-archive/``. For instance, to access the previous
    example, we can use something like::

From 19b139932d8aef9b47e1fa9c400c3f73b5cb3d7b Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 30 Oct 2023 12:19:07 +1000
Subject: [PATCH 0333/2492] doc/rados: improve "scrubbing" explanation

Improve the English describing the operation of scrubbing PGs in
doc/rados/configuration/osd-config-ref.rst.

Co-authored-by: Ronen Friedman <rfriedma@redhat.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/osd-config-ref.rst | 25 ++++++++++++----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
index 3c3b378e7b4e..060121200838 100644
--- a/doc/rados/configuration/osd-config-ref.rst
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -145,17 +145,20 @@ See `Pool & PG Config Reference`_ for details.
 Scrubbing
 =========
 
-In addition to making multiple copies of objects, Ceph ensures data integrity by
-scrubbing placement groups. Ceph scrubbing is analogous to ``fsck`` on the
-object storage layer. For each placement group, Ceph generates a catalog of all
-objects and compares each primary object and its replicas to ensure that no
-objects are missing or mismatched. Light scrubbing (daily) checks the object
-size and attributes.  Deep scrubbing (weekly) reads the data and uses checksums
-to ensure data integrity.
-
-Scrubbing is important for maintaining data integrity, but it can reduce
-performance. You can adjust the following settings to increase or decrease
-scrubbing operations.
+One way that Ceph ensures data integrity is by "scrubbing" placement groups.
+Ceph scrubbing is analogous to ``fsck`` on the object storage layer. Ceph
+generates a catalog of all objects in each placement group and compares each
+primary object to its replicas, ensuring that no objects are missing or
+mismatched. Light scrubbing checks the object size and attributes, and is
+usually done daily. Deep scrubbing reads the data and uses checksums to ensure
+data integrity, and is usually done weekly. The freqeuncies of both light
+scrubbing and deep scrubbing are determined by the cluster's configuration,
+which is fully under your control and subject to the settings explained below
+in this section.
+
+Although scrubbing is important for maintaining data integrity, it can reduce
+the performance of the Ceph cluster. You can adjust the following settings to
+increase or decrease the frequency and depth of scrubbing operations.
 
 
 .. confval:: osd_max_scrubs

From 23505441b077579581995126cf99088da815f9d2 Mon Sep 17 00:00:00 2001
From: barakda <barak.davidov@gmail.com>
Date: Tue, 31 Oct 2023 15:15:41 +0200
Subject: [PATCH 0334/2492] Update default nvmeof gw version to v0.0.5

Signed-off-by: barakda <barak.davidov@gmail.com>
---
 src/cephadm/cephadmlib/constants.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index d1e0aa4425c5..811aae24c700 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -12,7 +12,7 @@
 DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.7'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:0.0.1'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:v0.0.5'
 DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'
 DEFAULT_ELASTICSEARCH_IMAGE = 'quay.io/omrizeneva/elasticsearch:6.8.23'
 DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29'

From 536b054167d7aff1d18490e94ed98fd41d3003a3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 31 Oct 2023 10:54:37 -0400
Subject: [PATCH 0335/2492] cmake: do not issue CMAKE_BUILD_TYPE warning if set

Fixes: b8c6ba01b262d33084b9c607d829503ac8ae25c5
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 CMakeLists.txt | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f5ea2c49b9a6..4feb22c135e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,13 +23,15 @@ endforeach()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
-if(NOT CMAKE_BUILD_TYPE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
-  message(WARNING "CMAKE_BUILD_TYPE not specified, assuming CMAKE_BUILD_TYPE=Debug because .git exists.")
-  set(default_build_type "Debug")
-  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
-      STRING "Default BUILD_TYPE is Debug, other options are: RelWithDebInfo, Release, and MinSizeRel." FORCE)
-else()
-  message(WARNING "CMAKE_BUILD_TYPE not specified, leaving unset because .git does NOT exist.")
+if(NOT CMAKE_BUILD_TYPE)
+  if (EXISTS "${CMAKE_SOURCE_DIR}/.git")
+    message(WARNING "CMAKE_BUILD_TYPE not specified, assuming CMAKE_BUILD_TYPE=Debug because .git exists.")
+    set(default_build_type "Debug")
+    set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
+        STRING "Default BUILD_TYPE is Debug, other options are: RelWithDebInfo, Release, and MinSizeRel." FORCE)
+  else()
+    message(WARNING "CMAKE_BUILD_TYPE not specified, leaving unset because .git does NOT exist.")
+  endif()
 endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")

From eb6995436f9db1f0da17d71aa08ffefa6a358152 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 26 Oct 2023 16:18:31 +0530
Subject: [PATCH 0336/2492] cmake: add --progress flag to git submodule update
 commands

Ceph has lots of submodules that needs to be cloned before building
binaries from the repository. Seeing the progress when these submodules
are being cloned is useful, especially when developers/users have a
network issue or a slow network.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 README.md                                          | 2 +-
 do_cmake.sh                                        | 2 +-
 doc/dev/corpus.rst                                 | 2 +-
 doc/install/clone-source.rst                       | 4 ++--
 make-dist                                          | 2 +-
 qa/workunits/rados/test_envlibrados_for_rocksdb.sh | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 9db4161c793d..3a0274b53756 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ following commands to move into the cloned `ceph/ceph` repository and to check
 out the git submodules associated with it:
 
     cd ceph
-	git submodule update --init --recursive
+	git submodule update --init --recursive --progress
 
 
 ## Build Prerequisites
diff --git a/do_cmake.sh b/do_cmake.sh
index 6936a5596ebd..e838bca8b976 100755
--- a/do_cmake.sh
+++ b/do_cmake.sh
@@ -2,7 +2,7 @@
 set -ex
 
 if [ -d .git ]; then
-    git submodule update --init --recursive
+    git submodule update --init --recursive --progress
 fi
 
 : ${BUILD_DIR:=build}
diff --git a/doc/dev/corpus.rst b/doc/dev/corpus.rst
index 4005f70c0384..85cfc0ef2c90 100644
--- a/doc/dev/corpus.rst
+++ b/doc/dev/corpus.rst
@@ -27,7 +27,7 @@ script of ``script/gen-corpus.sh``, or by following the instructions below:
 
 	git clone ceph.git
 	cd ceph
-	git submodule update --init --recursive
+	git submodule update --init --recursive --progress
 
 #. Build with flag to dump objects to ``/tmp/foo``::
 
diff --git a/doc/install/clone-source.rst b/doc/install/clone-source.rst
index 2d09ef9ebabe..8e783fed8e83 100644
--- a/doc/install/clone-source.rst
+++ b/doc/install/clone-source.rst
@@ -154,13 +154,13 @@ Updating Submodules
 
    .. prompt:: bash $
 
-      git submodule update --force --init --recursive
+      git submodule update --force --init --recursive --progress
       git clean -fdx
       git submodule foreach git clean -fdx
 
    If you still have problems with a submodule directory, use ``rm -rf
    [directory name]`` to remove the directory. Then run ``git submodule update
-   --init --recursive`` again.
+   --init --recursive --progress`` again.
 
 #. Run ``git status`` again:
 
diff --git a/make-dist b/make-dist
index f69a969ada0c..4add880180eb 100755
--- a/make-dist
+++ b/make-dist
@@ -35,7 +35,7 @@ echo "version $version"
 # update submodules
 echo "updating submodules..."
 force=$(if git submodule usage 2>&1 | grep --quiet 'update.*--force'; then echo --force ; fi)
-if ! git submodule sync || ! git submodule update $force --init --recursive; then
+if ! git submodule sync || ! git submodule update $force --init --recursive --progress; then
     echo "Error: could not initialize submodule projects"
     echo "  Network connectivity might be required."
     exit 1
diff --git a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
index 371452f40429..ccabbd8a6a6c 100755
--- a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
+++ b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
@@ -58,7 +58,7 @@ if [ -e rocksdb ]; then
 fi
 
 pushd $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../
-git submodule update --init src/rocksdb
+git submodule update --init --progress src/rocksdb
 popd
 git clone $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../src/rocksdb rocksdb
 

From 69472c26af5419faa9ed93c071ed5933d03fa67f Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 1 Nov 2023 11:53:59 +1000
Subject: [PATCH 0337/2492] doc/cephadm: edit troubleshooting.rst (1 of x)

Edit doc/cephadm/troubleshooting.rst. This commit and the PR of which it
is a part was raised in response to
https://github.com/ceph/ceph/pull/53976. The limits of reStructuredText
are particularly visible here in every instance of a BASH for-loop and
in every instance of a command stretched over multiple lines.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/cephadm/troubleshooting.rst | 147 ++++++++++++++++++--------------
 1 file changed, 81 insertions(+), 66 deletions(-)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index 3d4965deda03..a7ee9168da05 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -1,60 +1,56 @@
 Troubleshooting
 ===============
 
-You may wish to investigate why a cephadm command failed
-or why a certain service no longer runs properly.
+This section explains how to investigate why a cephadm command failed or why a
+certain service no longer runs properly.
 
-Cephadm deploys daemons within containers. This means that
-troubleshooting those containerized daemons will require
-a different process than traditional package-install daemons.
+Cephadm deploys daemons within containers. Troubleshooting containerized
+daemons requires a different process than does troubleshooting traditional
+daemons that were installed by means of packages.
 
-Here are some tools and commands to help you troubleshoot
-your Ceph environment.
+Here are some tools and commands to help you troubleshoot your Ceph
+environment.
 
 .. _cephadm-pause:
 
 Pausing or Disabling cephadm
 ----------------------------
 
-If something goes wrong and cephadm is behaving badly, you can
-pause most of the Ceph cluster's background activity by running
-the following command: 
+If something goes wrong and cephadm is behaving badly, pause most of the Ceph
+cluster's background activity by running the following command: 
 
 .. prompt:: bash #
 
   ceph orch pause
 
-This stops all changes in the Ceph cluster, but cephadm will
-still periodically check hosts to refresh its inventory of
-daemons and devices.  You can disable cephadm completely by
-running the following commands:
+This stops all changes in the Ceph cluster, but cephadm will still periodically
+check hosts to refresh its inventory of daemons and devices. Disable cephadm
+completely by running the following commands:
 
 .. prompt:: bash #
 
   ceph orch set backend ''
   ceph mgr module disable cephadm
 
-These commands disable all of the ``ceph orch ...`` CLI commands.
-All previously deployed daemon containers continue to exist and
-will start as they did before you ran these commands.
+These commands disable all of the ``ceph orch ...`` CLI commands. All
+previously deployed daemon containers continue to run and will start just as
+they were before you ran these commands.
 
-See :ref:`cephadm-spec-unmanaged` for information on disabling
-individual services.
+See :ref:`cephadm-spec-unmanaged` for more on disabling individual services.
 
 
 Per-service and Per-daemon Events
 ---------------------------------
 
-In order to facilitate debugging failed daemons,
-cephadm stores events per service and per daemon.
-These events often contain information relevant to
-troubleshooting your Ceph cluster. 
+To make it easier to debug failed daemons, cephadm stores events per service
+and per daemon. These events often contain information relevant to
+the troubleshooting of your Ceph cluster. 
 
 Listing Service Events
 ~~~~~~~~~~~~~~~~~~~~~~
 
-To see the events associated with a certain service, run a
-command of the and following form:
+To see the events associated with a certain service, run a command of the 
+following form:
 
 .. prompt:: bash #
 
@@ -81,8 +77,8 @@ This will return something in the following form:
 Listing Daemon Events
 ~~~~~~~~~~~~~~~~~~~~~
 
-To see the events associated with a certain daemon, run a
-command of the and following form:
+To see the events associated with a certain daemon, run a command of the
+following form:
 
 .. prompt:: bash #
 
@@ -105,32 +101,41 @@ This will return something in the following form:
 Checking Cephadm Logs
 ---------------------
 
-To learn how to monitor cephadm logs as they are generated, read :ref:`watching_cephadm_logs`.
+To learn how to monitor cephadm logs as they are generated, read
+:ref:`watching_cephadm_logs`.
 
-If your Ceph cluster has been configured to log events to files, there will be a
-``ceph.cephadm.log`` file on all monitor hosts (see
-:ref:`cephadm-logs` for a more complete explanation).
+If your Ceph cluster has been configured to log events to files, there will be
+a ``ceph.cephadm.log`` file on all monitor hosts. See :ref:`cephadm-logs` for a
+more complete explanation.
 
 Gathering Log Files
 -------------------
 
-Use journalctl to gather the log files of all daemons:
+Use ``journalctl`` to gather the log files of all daemons:
 
 .. note:: By default cephadm now stores logs in journald. This means
    that you will no longer find daemon logs in ``/var/log/ceph/``.
 
-To read the log file of one specific daemon, run::
+To read the log file of one specific daemon, run a command of the following
+form:
 
-    cephadm logs --name <name-of-daemon>
+.. prompt:: bash
+
+   cephadm logs --name <name-of-daemon>
+
+.. Note:: This works only when run on the same host that is running the daemon.
+   To get the logs of a daemon that is running on a different host, add the
+   ``--fsid`` option to the command, as in the following example:
 
-Note: this only works when run on the same host where the daemon is running. To
-get logs of a daemon running on a different host, give the ``--fsid`` option::
+   .. prompt:: bash
 
-    cephadm logs --fsid <fsid> --name <name-of-daemon>
+      cephadm logs --fsid <fsid> --name <name-of-daemon>
 
-where the ``<fsid>`` corresponds to the cluster ID printed by ``ceph status``.
+   In this example, ``<fsid>`` corresponds to the cluster ID returned by the
+   ``ceph status`` command.
 
-To fetch all log files of all daemons on a given host, run::
+To fetch all log files of all daemons on a given host, run the following
+for-loop::
 
     for name in $(cephadm ls | jq -r '.[].name') ; do
       cephadm logs --fsid <fsid> --name "$name" > $name;
@@ -139,39 +144,41 @@ To fetch all log files of all daemons on a given host, run::
 Collecting Systemd Status
 -------------------------
 
-To print the state of a systemd unit, run::
+To print the state of a systemd unit, run a command of the following form: 
 
-      systemctl status "ceph-$(cephadm shell ceph fsid)@<service name>.service";
+.. prompt:: bash
 
+   systemctl status "ceph-$(cephadm shell ceph fsid)@<service name>.service";
 
-To fetch all state of all daemons of a given host, run::
 
-    fsid="$(cephadm shell ceph fsid)"
-    for name in $(cephadm ls | jq -r '.[].name') ; do
-      systemctl status "ceph-$fsid@$name.service" > $name;
-    done
+To fetch the state of all daemons of a given host, run the following shell
+script::
+
+   fsid="$(cephadm shell ceph fsid)"
+   for name in $(cephadm ls | jq -r '.[].name') ; do
+     systemctl status "ceph-$fsid@$name.service" > $name;
+   done
 
 
 List all Downloaded Container Images
 ------------------------------------
 
-To list all container images that are downloaded on a host:
+To list all container images that are downloaded on a host, run the following
+commands:
 
-.. note:: ``Image`` might also be called `ImageID`
+.. prompt:: bash #
 
-::
+   podman ps -a --format json | jq '.[].Image' "docker.io/library/centos:8" "registry.opensuse.org/opensuse/leap:15.2"
 
-    podman ps -a --format json | jq '.[].Image'
-    "docker.io/library/centos:8"
-    "registry.opensuse.org/opensuse/leap:15.2"
+.. note:: ``Image`` might also be called ``ImageID``.
 
 
 Manually Running Containers
 ---------------------------
 
 Cephadm uses small wrappers when running containers. Refer to
-``/var/lib/ceph/<cluster-fsid>/<service-name>/unit.run`` for the
-container execution command.
+``/var/lib/ceph/<cluster-fsid>/<service-name>/unit.run`` for the container
+execution command.
 
 .. _cephadm-ssh-errors:
 
@@ -187,9 +194,10 @@ Error message::
   Please make sure that the host is reachable and accepts connections using the cephadm SSH key
   ...
 
-Things Ceph administrators can do:
+If you receive the above error message, try the following things to
+troubleshoot the SSH connection between ``cephadm`` and the monitor:
 
-1. Ensure cephadm has an SSH identity key::
+1. Ensure that ``cephadm`` has an SSH identity key::
 
      [root@mon1~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key
      INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98
@@ -208,14 +216,15 @@ Things Ceph administrators can do:
 
      [root@mon1 ~]# cephadm shell -- ceph cephadm get-ssh-config > config
 
-3. Verify that we can connect to the host::
+3. Verify that it is possible to connect to the host::
 
      [root@mon1 ~]# ssh -F config -i ~/cephadm_private_key root@mon1
 
 Verifying that the Public Key is Listed in the authorized_keys file
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To verify that the public key is in the authorized_keys file, run the following commands::
+To verify that the public key is in the ``authorized_keys`` file, run the
+following commands::
 
      [root@mon1 ~]# cephadm shell -- ceph cephadm get-pub-key > ~/ceph.pub
      [root@mon1 ~]# grep "`cat ~/ceph.pub`"  /root/.ssh/authorized_keys
@@ -231,22 +240,28 @@ Or this error::
 
    Must set public_network config option or specify a CIDR network, ceph addrvec, or plain IP
 
-This means that you must run a command of this form::
+This means that you must run a command of this form:
 
-  ceph config set mon public_network <mon_network>
+.. prompt:: bash
 
-For more detail on operations of this kind, see :ref:`deploy_additional_monitors`
+   ceph config set mon public_network <mon_network>
+
+For more detail on operations of this kind, see
+:ref:`deploy_additional_monitors`.
 
 Accessing the Admin Socket
 --------------------------
 
-Each Ceph daemon provides an admin socket that bypasses the
-MONs (See :ref:`rados-monitoring-using-admin-socket`).
+Each Ceph daemon provides an admin socket that bypasses the MONs (See
+:ref:`rados-monitoring-using-admin-socket`).
+
+#. To access the admin socket, enter the daemon container on the host::
 
-To access the admin socket, first enter the daemon container on the host::
+   [root@mon1 ~]# cephadm enter --name <daemon-name>
 
-    [root@mon1 ~]# cephadm enter --name <daemon-name>
-    [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok config show
+#. Run a command of the following form to see the admin socket's configuration::
+  
+   [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok config show
 
 Running Various Ceph Tools
 --------------------------------

From 3de9912171d4d63b8a41e51f9b73f0bfc10187ce Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 11:57:39 +0800
Subject: [PATCH 0338/2492] crimson/os/seastore/btree: drop unused lba mapping
 method

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/btree/btree_range_pin.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/crimson/os/seastore/btree/btree_range_pin.h b/src/crimson/os/seastore/btree/btree_range_pin.h
index 68188e9ff550..7a08b6d89472 100644
--- a/src/crimson/os/seastore/btree/btree_range_pin.h
+++ b/src/crimson/os/seastore/btree/btree_range_pin.h
@@ -153,10 +153,6 @@ class BtreeNodeMapping : public PhysicalNodeMapping<key_t, val_t> {
     return parent;
   }
 
-  void set_parent(CachedExtentRef ext) {
-    parent = ext;
-  }
-
   uint16_t get_pos() const final {
     return pos;
   }

From 4b52ca8a6bc67838fb19f908f9f9903190878281 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 11:59:05 +0800
Subject: [PATCH 0339/2492] crimson/os/seastore/lba_manager: copy
 intermediate_length when dupicating lba mappings

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
index 892600ed0e94..5496a4b19682 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
@@ -147,6 +147,7 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     pin->key = key;
     pin->intermediate_base = intermediate_base;
     pin->intermediate_key = intermediate_key;
+    pin->intermediate_length = intermediate_length;
     pin->indirect = indirect;
     pin->raw_val = raw_val;
     pin->map_val = map_val;

From 42a3901f0f9a68fa46963c321de8740d45468485 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 16:16:58 +0800
Subject: [PATCH 0340/2492] test/crimson/seastore/lba_manager: skip indirect
 lba mappings

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/btree/fixed_kv_btree.h                | 4 +++-
 .../os/seastore/lba_manager/btree/btree_lba_manager.cc        | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h
index 2970d0440850..4133be627c10 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_btree.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h
@@ -505,7 +505,9 @@ class FixedKVBtree {
           i->get_val().maybe_relative_to(node->get_paddr()),
           &child_node);
       } else {
-        assert(i->get_val().pladdr.is_paddr());
+        if (i->get_val().pladdr.is_laddr()) {
+          continue;
+        }
         ret = c.trans.get_extent(
           i->get_val().pladdr.get_paddr().maybe_relative_to(node->get_paddr()),
           &child_node);
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
index a607cd612a59..bb43bdb2c4f4 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
@@ -496,7 +496,9 @@ BtreeLBAManager::scan_mappings(
 	      seastar::stop_iteration::yes);
 	  }
 	  ceph_assert((pos.get_key() + pos.get_val().len) > begin);
-	  f(pos.get_key(), pos.get_val().pladdr.get_paddr(), pos.get_val().len);
+	  if (pos.get_val().pladdr.is_paddr()) {
+	    f(pos.get_key(), pos.get_val().pladdr.get_paddr(), pos.get_val().len);
+	  }
 	  return LBABtree::iterate_repeat_ret_inner(
 	    interruptible::ready_future_marker{},
 	    seastar::stop_iteration::no);

From 41406dc3ec5940ec8b9db8dcbf8e4640ce5c3eae Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 25 Oct 2023 13:04:14 +0300
Subject: [PATCH 0341/2492] mds/MDBalancer: ignore queued callbacks if MDS is
 not active

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
Fixes: https://tracker.ceph.com/issues/63281
---
 src/mds/MDBalancer.cc | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 0f4f76b16d25..cd2e7d9bd002 100644
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -565,7 +565,8 @@ double MDBalancer::try_match(balance_state_t& state, mds_rank_t ex, double& maxe
 
 void MDBalancer::queue_split(const CDir *dir, bool fast)
 {
-  dout(10) << __func__ << " enqueuing " << *dir
+  constexpr const auto &_func_ = __func__;
+  dout(10) << _func_ << " enqueuing " << *dir
                        << " (fast=" << fast << ")" << dendl;
 
   const dirfrag_t df = dir->dirfrag();
@@ -579,6 +580,16 @@ void MDBalancer::queue_split(const CDir *dir, bool fast)
       return;
     }
 
+    if (mds->is_stopping()) {
+      // not a good time. This could have been (!mds->is_active())
+      // or at least (mds->is_stopping() || mds->is_stopped()), but
+      // is_stopped() is never true because an MDS respawns as soon as it's removed from the map;
+      // the narrow is_stopping check is to avoid potential regressions
+      // due to unknown coupling with other parts of the MDS (especially multiple ranks).
+      dout(5) << "ignoring the " << _func_ << " callback because the MDS state is '" << ceph_mds_state_name(mds->get_state()) << "'" << dendl;
+      return;
+    }
+
     auto mdcache = mds->mdcache;
 
     CDir *dir = mdcache->get_dirfrag(df);
@@ -593,7 +604,7 @@ void MDBalancer::queue_split(const CDir *dir, bool fast)
 
     // Pass on to MDCache: note that the split might still not
     // happen if the checks in MDCache::can_fragment fail.
-    dout(10) << __func__ << " splitting " << *dir << dendl;
+    dout(10) << _func_ << " splitting " << *dir << dendl;
     int bits = g_conf()->mds_bal_split_bits;
     if (dir->inode->is_ephemeral_dist()) {
       unsigned min_frag_bits = mdcache->get_ephemeral_dist_frag_bits();
@@ -623,6 +634,7 @@ void MDBalancer::queue_split(const CDir *dir, bool fast)
 void MDBalancer::queue_merge(CDir *dir)
 {
   const auto frag = dir->dirfrag();
+  constexpr const auto &_func_ = __func__;
   auto callback = [this, frag](int r) {
     ceph_assert(frag.frag != frag_t());
 
@@ -631,6 +643,16 @@ void MDBalancer::queue_merge(CDir *dir)
     // starting one), and this context is the only one that erases it.
     merge_pending.erase(frag);
 
+    if (mds->is_stopping()) {
+      // not a good time. This could have been (!mds->is_active())
+      // or at least (mds->is_stopping() || mds->is_stopped()), but
+      // is_stopped() is never true because an MDS respawns as soon as it's removed from the map;
+      // the narrow is_stopping check is to avoid potential regressions
+      // due to unknown coupling with other parts of the MDS (especially multiple ranks).
+      dout(5) << "ignoring the " << _func_ << " callback because the MDS state is '" << ceph_mds_state_name(mds->get_state()) << "'" << dendl;
+      return;
+    }
+
     auto mdcache = mds->mdcache;
     CDir *dir = mdcache->get_dirfrag(frag);
     if (!dir) {

From 647b5d67a8a800091acea68d20e87354373b0fac Mon Sep 17 00:00:00 2001
From: Theofilos Mouratidis <theofilos.mouratidis@switch.ch>
Date: Wed, 1 Nov 2023 10:55:37 +0100
Subject: [PATCH 0342/2492] mgr/dashboard: discovery service (port 8765) fails
 on ipv6 only clusters

Having ms_bind_ipv4=false and ipv6=true the code that the Ceph dashboard runs
for the discovery service (port 8765) fails, because it requests the address
of the mgr container which returns ipv6 and the mgr code expects ipv4 address

Fixes: https://tracker.ceph.com/issues/63388
Signed-off-by: Theofilos Mouratidis <mtheofilos@gmail.com>
---
 src/pybind/mgr/cephadm/ssl_cert_utils.py | 9 +++------
 src/pybind/mgr/mgr_util.py               | 5 ++++-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/cephadm/ssl_cert_utils.py b/src/pybind/mgr/cephadm/ssl_cert_utils.py
index fcc6f00eab9d..6295152c7c1b 100644
--- a/src/pybind/mgr/cephadm/ssl_cert_utils.py
+++ b/src/pybind/mgr/cephadm/ssl_cert_utils.py
@@ -46,7 +46,7 @@ def generate_root_cert(self, addr: str) -> Tuple[str, str]:
         root_builder = root_builder.public_key(root_public_key)
         root_builder = root_builder.add_extension(
             x509.SubjectAlternativeName(
-                [x509.IPAddress(ipaddress.IPv4Address(addr))]
+                [x509.IPAddress(ipaddress.ip_address(addr))]
             ),
             critical=False
         )
@@ -70,12 +70,9 @@ def generate_root_cert(self, addr: str) -> Tuple[str, str]:
     def generate_cert(self, host: str, addr: str) -> Tuple[str, str]:
         have_ip = True
         try:
-            ip = x509.IPAddress(ipaddress.IPv4Address(addr))
+            ip = x509.IPAddress(ipaddress.ip_address(addr))
         except Exception:
-            try:
-                ip = x509.IPAddress(ipaddress.IPv6Address(addr))
-            except Exception:
-                have_ip = False
+            have_ip = False
 
         private_key = rsa.generate_private_key(
             public_exponent=65537, key_size=4096, backend=default_backend())
diff --git a/src/pybind/mgr/mgr_util.py b/src/pybind/mgr/mgr_util.py
index 8684f8013184..05ec6496682f 100644
--- a/src/pybind/mgr/mgr_util.py
+++ b/src/pybind/mgr/mgr_util.py
@@ -12,6 +12,7 @@
 import time
 import logging
 import sys
+from ipaddress import ip_address
 from threading import Lock, Condition, Event
 from typing import no_type_check, NewType
 import urllib
@@ -413,7 +414,9 @@ def test_port_allocation(addr: str, port: int) -> None:
     If no exception is raised, the port can be assumed available
     """
     try:
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        ip_version = ip_address(addr).version
+        addr_family = socket.AF_INET if ip_version == 4 else socket.AF_INET6
+        sock = socket.socket(addr_family, socket.SOCK_STREAM)
         sock.bind((addr, port))
         sock.close()
     except socket.error as e:

From 165ab1e06ab760d06f8bd0d9982051071e7c12ae Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Wed, 1 Nov 2023 11:31:58 -0400
Subject: [PATCH 0343/2492] rgw/multisite: correct the marker_tracker->flush()
 call and only call it when we have lost bid.

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 src/rgw/driver/rados/rgw_data_sync.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index bbd3aaf448da..b7ab04bdd24f 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -2122,9 +2122,9 @@ class RGWDataIncSyncShardCR : public RGWDataBaseSyncShardCR {
       } while (true);
 
       drain_all();
-      yield marker_tracker->flush();
 
       if (lost_bid) {
+        yield call(marker_tracker->flush());
         return set_cr_error(-EBUSY);
       } else if (lost_lock) {
         return set_cr_error(-ECANCELED);

From c4b4ba554d285d9342cf16e4ce6782f18bd405ce Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 1 Nov 2023 13:16:46 -0400
Subject: [PATCH 0344/2492] rgw: link only radosgw with ALLOC_LIBS

In particular, do not link intermediate dependencies nor librgw.so.2
with a custom allocator (normally tcmalloc).

This prevents illegal behavior due to mismatched allocators when run
under nfs-ganesha or other consumers.

Fixes: https://tracker.ceph.com/issues/63394

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/CMakeLists.txt | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 9b55081dbaaa..e4ff8c9199bc 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -279,7 +279,6 @@ target_link_libraries(rgw_common
     ${ARROW_LIBRARIES}
     ${ARROW_FLIGHT_LIBRARIES}
     ${LMDB_LIBRARIES}
-    ${ALLOC_LIBS}
   PUBLIC
     ${LUA_LIBRARIES}
     RapidJSON::RapidJSON
@@ -459,7 +458,12 @@ target_include_directories(radosgw
 
 target_include_directories(radosgw SYSTEM PUBLIC "../rapidjson/include")
 
-target_link_libraries(radosgw PRIVATE ${rgw_libs} rgw_schedulers kmip)
+target_link_libraries(radosgw PRIVATE
+  ${rgw_libs}
+  rgw_schedulers
+  kmip
+  ${ALLOC_LIBS})
+
 if(WITH_RADOSGW_BEAST_OPENSSL)
   # used by rgw_asio_frontend.cc
   target_link_libraries(radosgw PRIVATE OpenSSL::SSL)
@@ -509,7 +513,7 @@ set(radosgw_token_srcs
   rgw_token.cc)
 add_executable(radosgw-token ${radosgw_token_srcs})
 target_link_libraries(radosgw-token librados
-  global ${ALLOC_LIBS})
+  global)
 install(TARGETS radosgw-token DESTINATION bin)
 
 set(radosgw_object_expirer_srcs

From 3bcaf7b0045974e474ebdc4a2bbb9adc0316569d Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 17 Oct 2023 10:04:33 +0800
Subject: [PATCH 0345/2492] osd: clean up the sparse-read related code

Just clean up the code and nothing fixed to make the code to be
more readable.

Fixes: https://tracker.ceph.com/issues/63318
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/osd/PGBackend.h          | 2 +-
 src/osd/PrimaryLogPG.cc      | 2 +-
 src/osd/ReplicatedBackend.cc | 2 +-
 src/osd/ReplicatedBackend.h  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h
index a0919e1d87f5..ac17f05035db 100644
--- a/src/osd/PGBackend.h
+++ b/src/osd/PGBackend.h
@@ -568,7 +568,7 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
 
    virtual int objects_readv_sync(
      const hobject_t &hoid,
-     std::map<uint64_t, uint64_t>&& m,
+     std::map<uint64_t, uint64_t>& m,
      uint32_t op_flags,
      ceph::buffer::list *bl) {
      return -EOPNOTSUPP;
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index c1384412a35c..f548fc544e38 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -5931,7 +5931,7 @@ int PrimaryLogPG::do_sparse_read(OpContext *ctx, OSDOp& osd_op) {
     }
 
     bufferlist data_bl;
-    r = pgbackend->objects_readv_sync(soid, std::move(m), op.flags, &data_bl);
+    r = pgbackend->objects_readv_sync(soid, m, op.flags, &data_bl);
     if (r == -EIO) {
       r = rep_repair_primary_object(soid, ctx);
     }
diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc
index a2a3312399c7..8fd2d2022f81 100644
--- a/src/osd/ReplicatedBackend.cc
+++ b/src/osd/ReplicatedBackend.cc
@@ -275,7 +275,7 @@ int ReplicatedBackend::objects_read_sync(
 
 int ReplicatedBackend::objects_readv_sync(
   const hobject_t &hoid,
-  map<uint64_t, uint64_t>&& m,
+  map<uint64_t, uint64_t>& m,
   uint32_t op_flags,
   bufferlist *bl)
 {
diff --git a/src/osd/ReplicatedBackend.h b/src/osd/ReplicatedBackend.h
index 0800c8ab024d..41b634269ba0 100644
--- a/src/osd/ReplicatedBackend.h
+++ b/src/osd/ReplicatedBackend.h
@@ -142,7 +142,7 @@ class ReplicatedBackend : public PGBackend {
 
   int objects_readv_sync(
     const hobject_t &hoid,
-    std::map<uint64_t, uint64_t>&& m,
+    std::map<uint64_t, uint64_t>& m,
     uint32_t op_flags,
     ceph::buffer::list *bl) override;
 

From 764da1be698e4a88f4afa74c5cdf9fcfbbc18f9b Mon Sep 17 00:00:00 2001
From: Ionut Balutoiu <ibalutoiu@cloudbasesolutions.com>
Date: Wed, 1 Nov 2023 18:07:58 +0200
Subject: [PATCH 0346/2492] rgw: fix cloud-sync multi-tenancy scenario

At the moment, we cannot set buckets prefixed with tenant ID in the
`source_bucket` field from cloud-sync profiles (non-trivial config):
https://docs.ceph.com/en/latest/radosgw/cloud-sync-module/#non-trivial-configuration

This is because the `do_find_profile` function only searches in the
profiles configured using `bucket.name`, and it ignores `bucket.tenant`.

This is problematic in the RGW multi-tenancy scenario:
https://docs.ceph.com/en/latest/radosgw/multitenancy/#rgw-multi-tenancy

At the moment, we can only configure bucket name in the profile
`source_bucket` field. In the multi-tenancy scenario, this would sync
all the buckets (from all the tenants).

Without this fix, we cannot configure a cloud-sync profile that syncs
all the buckets from a tenant to a particular S3 target.

For example, we cannot do this:

* `tenantA/test-bucket` -> S3 target A
* `tenantB/test-bucket` -> S3 target B
* `tenantC/test-bucket` -> S3 target C

We can only do this at the moment:

* `test-bucket` -> S3 target A

If `test-bucket` is present in both `tenantA` and `tenantB`, both
buckets will be synced to S3 target A.

The idea would be to be able to do this:

* `tenantA/*` -> S3 target A
* `tenantB/*` -> S3 target B
* `tenantC/*` -> S3 target C

If `test-bucket` is present in all tenants, each tenant bucket is
synced to its own S3 target.

Fixes: https://tracker.ceph.com/issues/63395

Signed-off-by: Ionut Balutoiu <ibalutoiu@cloudbasesolutions.com>
---
 src/rgw/driver/rados/rgw_sync_module_aws.cc | 2 +-
 src/rgw/rgw_bucket_types.h                  | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_sync_module_aws.cc b/src/rgw/driver/rados/rgw_sync_module_aws.cc
index cefcd9dd10c6..cdcd831e9892 100644
--- a/src/rgw/driver/rados/rgw_sync_module_aws.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_aws.cc
@@ -487,7 +487,7 @@ struct AWSSyncConfig {
   }
 
   bool do_find_profile(const rgw_bucket bucket, std::shared_ptr<AWSSyncConfig_Profile> *result) {
-    const string& name = bucket.name;
+    const string& name = bucket.get_namespaced_name();
     auto iter = explicit_profiles.upper_bound(name);
     if (iter == explicit_profiles.begin()) {
       return false;
diff --git a/src/rgw/rgw_bucket_types.h b/src/rgw/rgw_bucket_types.h
index 61acc58bbeb1..52ac5dc34bd1 100644
--- a/src/rgw/rgw_bucket_types.h
+++ b/src/rgw/rgw_bucket_types.h
@@ -136,6 +136,13 @@ struct rgw_bucket {
     DECODE_FINISH(bl);
   }
 
+  std::string get_namespaced_name() const {
+    if (tenant.empty()) {
+      return name;
+    }
+    return tenant + std::string("/") + name;
+  }
+
   void update_bucket_id(const std::string& new_bucket_id) {
     bucket_id = new_bucket_id;
   }

From 563734198c4dfb91c70e90d57240b5eb90ea37a9 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 16:19:52 +0800
Subject: [PATCH 0347/2492] crimson/os/seastore/transaction_manager: indirect
 lba mappings should always have the entire original mappings as their targets

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/transaction_manager.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index dd1898ba77c7..dfce85c5e1b8 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -401,6 +401,9 @@ class TransactionManager : public ExtentCallbackInterface {
             auto remap_len = remap.len;
             auto remap_laddr = original_laddr + remap_offset;
             auto remap_paddr = original_paddr.add_offset(remap_offset);
+	    if (intermediate_key != L_ADDR_NULL) {
+	      remap_paddr = original_paddr;
+	    }
             ceph_assert(remap_len < original_len);
             ceph_assert(remap_offset + remap_len <= original_len);
             ceph_assert(remap_len != 0);

From 683c0274a900560afa7ecf4da5a8f213be810f4c Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 16:22:31 +0800
Subject: [PATCH 0348/2492] test/crimson/seastore: add unittests for
 clone/remap/read_pin

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../seastore/test_transaction_manager.cc      | 145 ++++++++++++++++--
 1 file changed, 133 insertions(+), 12 deletions(-)

diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index 1148884a0c19..914eea9bbbfd 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -239,12 +239,15 @@ struct transaction_manager_test_t :
 
     void check_hint(
       laddr_t hint,
+      laddr_t intermediate_hint,
       laddr_t addr,
       extent_len_t len,
       delta_t &delta) const {
       delta_overlay_t overlay(*this, delta);
-      auto iter = overlay.lower_bound(hint);
-      laddr_t last = hint;
+      auto real_hint = intermediate_hint == L_ADDR_NULL
+	? hint : intermediate_hint;
+      auto iter = overlay.lower_bound(real_hint);
+      laddr_t last = real_hint;
       while (true) {
 	if (iter == overlay.end() || iter->first > addr) {
 	  EXPECT_EQ(addr, last);
@@ -286,8 +289,18 @@ struct transaction_manager_test_t :
 	test_extent_record_t{extent.get_desc(), 1};
     }
 
-    void alloced(laddr_t hint, TestBlock &extent, delta_t &delta) const {
-      check_hint(hint, extent.get_laddr(), extent.get_length(), delta);
+    void alloced(
+      laddr_t hint,
+      TestBlock &extent,
+      delta_t &delta,
+      laddr_t intermediate_hint = L_ADDR_NULL) const
+    {
+      check_hint(
+	hint,
+	intermediate_hint,
+	extent.get_laddr(),
+	extent.get_length(),
+	delta);
       insert(extent, delta);
     }
 
@@ -413,6 +426,25 @@ struct transaction_manager_test_t :
     check_mappings(t);
   }
 
+  TestBlockRef read_pin(
+    test_transaction_t &t,
+    LBAMappingRef pin) {
+    auto addr = pin->is_indirect()
+      ? pin->get_intermediate_base()
+      : pin->get_key();
+    auto len = pin->is_indirect()
+      ? pin->get_intermediate_length()
+      : pin->get_length();
+    ceph_assert(test_mappings.contains(addr, t.mapping_delta));
+    ceph_assert(test_mappings.get(addr, t.mapping_delta).desc.len == len);
+
+    auto ext = with_trans_intr(*(t.t), [&](auto& trans) {
+      return tm->read_pin<TestBlock>(trans, std::move(pin));
+    }).unsafe_get0();
+    EXPECT_EQ(addr, ext->get_laddr());
+    return ext;
+  }
+
   TestBlockRef get_extent(
     test_transaction_t &t,
     laddr_t addr,
@@ -484,7 +516,9 @@ struct transaction_manager_test_t :
     LBAMappingRef &&pin) {
     using ertr = with_trans_ertr<TransactionManager::base_iertr>;
     using ret = ertr::future<TestBlockRef>;
+    bool indirect = pin->is_indirect();
     auto addr = pin->get_key();
+    auto im_addr = indirect ? pin->get_intermediate_base() : L_ADDR_NULL;
     auto ext = with_trans_intr(*(t.t), [&](auto& trans) {
       return tm->read_pin<TestBlock>(trans, std::move(pin));
     }).safe_then([](auto ext) -> ret {
@@ -498,7 +532,11 @@ struct transaction_manager_test_t :
       }
     ).get0();
     if (ext) {
-      EXPECT_EQ(addr, ext->get_laddr());
+      if (indirect) {
+	EXPECT_EQ(im_addr, ext->get_laddr());
+      } else {
+	EXPECT_EQ(addr, ext->get_laddr());
+      }
     }
     if (t.t->is_conflicted()) {
       return nullptr;
@@ -544,6 +582,20 @@ struct transaction_manager_test_t :
     return pin;
   }
 
+  LBAMappingRef clone_pin(
+    test_transaction_t &t,
+    laddr_t offset,
+    const LBAMapping &mapping) {
+    auto pin = with_trans_intr(*(t.t), [&](auto &trans) {
+      return tm->clone_pin(trans, offset, mapping);
+    }).unsafe_get0();
+    EXPECT_EQ(offset, pin->get_key());
+    EXPECT_EQ(mapping.get_key(), pin->get_intermediate_key());
+    EXPECT_EQ(mapping.get_key(), pin->get_intermediate_base());
+    test_mappings.inc_ref(pin->get_intermediate_key(), t.mapping_delta);
+    return pin;
+  }
+
   LBAMappingRef try_get_pin(
     test_transaction_t &t,
     laddr_t offset) {
@@ -597,7 +649,7 @@ struct transaction_manager_test_t :
     for (const auto &i: overlay) {
       logger().debug("check_mappings: {}->{}", i.first, i.second);
       auto ext = get_extent(t, i.first, i.second.desc.len);
-      EXPECT_EQ(i.second, ext->get_desc());
+      assert(i.second == ext->get_desc());
     }
     with_trans_intr(
       *t.t,
@@ -992,6 +1044,9 @@ struct transaction_manager_test_t :
       return nullptr;
     }
     auto o_laddr = opin->get_key();
+    auto data_laddr = opin->is_indirect()
+      ? opin->get_intermediate_base()
+      : o_laddr;
     auto pin = with_trans_intr(*(t.t), [&](auto& trans) {
       return tm->remap_pin<TestBlock>(
         trans, std::move(opin), std::array{
@@ -1006,16 +1061,24 @@ struct transaction_manager_test_t :
     if (t.t->is_conflicted()) {
       return nullptr;
     }
-    test_mappings.dec_ref(o_laddr, t.mapping_delta);
-    EXPECT_FALSE(test_mappings.contains(o_laddr, t.mapping_delta));
+    if (opin->is_indirect()) {
+      test_mappings.inc_ref(data_laddr, t.mapping_delta);
+    } else {
+      test_mappings.dec_ref(data_laddr, t.mapping_delta);
+      EXPECT_FALSE(test_mappings.contains(data_laddr, t.mapping_delta));
+    }
     EXPECT_TRUE(pin);
     EXPECT_EQ(pin->get_length(), new_len);
     EXPECT_EQ(pin->get_key(), o_laddr + new_offset);
 
     auto extent = try_read_pin(t, pin->duplicate());
     if (extent) {
-      test_mappings.alloced(pin->get_key(), *extent, t.mapping_delta);
-      EXPECT_TRUE(extent->is_exist_clean());
+      if (!pin->is_indirect()) {
+	test_mappings.alloced(pin->get_key(), *extent, t.mapping_delta);
+	EXPECT_TRUE(extent->is_exist_clean());
+      } else {
+	EXPECT_TRUE(extent->is_stable());
+      }
     } else {
       ceph_assert(t.t->is_conflicted());
       return nullptr;
@@ -1220,7 +1283,7 @@ struct transaction_manager_test_t :
         ASSERT_TRUE(pin2);
         auto pin3 = remap_pin(t, std::move(pin2), 0, 4 << 10);
         ASSERT_TRUE(pin3);
-        auto lext = get_extent(t, pin3->get_key(), pin3->get_length());
+        auto lext = read_pin(t, std::move(pin3));
         EXPECT_EQ('l', lext->get_bptr().c_str()[0]);
 	auto mlext = mutate_extent(t, lext);
 	ASSERT_TRUE(mlext->is_exist_mutation_pending());
@@ -1233,7 +1296,7 @@ struct transaction_manager_test_t :
         ASSERT_TRUE(pin5);
         auto pin6 = remap_pin(t, std::move(pin5), 4 << 10, 4 << 10);
         ASSERT_TRUE(pin6);
-        auto rext = get_extent(t, pin6->get_key(), pin6->get_length());
+        auto rext = read_pin(t, std::move(pin6));
         EXPECT_EQ('r', rext->get_bptr().c_str()[0]);
 	auto mrext = mutate_extent(t, rext);
 	ASSERT_TRUE(mrext->is_exist_mutation_pending());
@@ -1247,6 +1310,59 @@ struct transaction_manager_test_t :
     });
   }
 
+  void test_clone_and_remap_pin() {
+    run_async([this] {
+      constexpr size_t l_offset = 32 << 10;
+      constexpr size_t l_len = 32 << 10;
+      constexpr size_t r_offset = 64 << 10;
+      constexpr size_t r_len = 32 << 10;
+      constexpr size_t l_clone_offset = 96 << 10;
+      constexpr size_t r_clone_offset = 128 << 10;
+      {
+	auto t = create_transaction();
+	auto lext = alloc_extent(t, l_offset, l_len);
+        lext->set_contents('l', 0, 16 << 10);
+	test_mappings.update(l_offset, lext->get_desc(), t.mapping_delta);
+        auto rext = alloc_extent(t, r_offset, r_len);
+        rext->set_contents('r', 16 << 10, 16 << 10);
+	test_mappings.update(r_offset, rext->get_desc(), t.mapping_delta);
+	submit_transaction(std::move(t));
+      }
+      {
+	auto t = create_transaction();
+        auto lpin = get_pin(t, l_offset);
+        auto rpin = get_pin(t, r_offset);
+	auto l_clone_pin = clone_pin(t, l_clone_offset, *lpin);
+	auto r_clone_pin = clone_pin(t, r_clone_offset, *rpin);
+        //split left
+        auto pin1 = remap_pin(t, std::move(l_clone_pin), 0, 16 << 10);
+        ASSERT_TRUE(pin1);
+        auto pin2 = remap_pin(t, std::move(pin1), 0, 8 << 10);  
+        ASSERT_TRUE(pin2);
+        auto pin3 = remap_pin(t, std::move(pin2), 0, 4 << 10);
+        ASSERT_TRUE(pin3);
+        auto lext = read_pin(t, std::move(pin3));
+        EXPECT_EQ('l', lext->get_bptr().c_str()[0]);
+
+        //split right
+        auto pin4 = remap_pin(t, std::move(r_clone_pin), 16 << 10, 16 << 10);
+        ASSERT_TRUE(pin4);
+        auto pin5 = remap_pin(t, std::move(pin4), 8 << 10, 8 << 10);  
+        ASSERT_TRUE(pin5);
+        auto pin6 = remap_pin(t, std::move(pin5), 4 << 10, 4 << 10);
+        ASSERT_TRUE(pin6);
+	auto int_offset = pin6->get_intermediate_offset();
+        auto rext = read_pin(t, std::move(pin6));
+        EXPECT_EQ('r', rext->get_bptr().c_str()[int_offset]);
+
+	submit_transaction(std::move(t));
+	check();
+      }
+      replay();
+      check();
+    });
+  }
+
   void test_overwrite_pin() {
     run_async([this] {
       constexpr size_t m_offset = 8 << 10;
@@ -1954,6 +2070,11 @@ TEST_P(tm_single_device_test_t, test_remap_pin)
   test_remap_pin();
 }
 
+TEST_P(tm_single_device_test_t, test_clone_and_remap_pin)
+{
+  test_clone_and_remap_pin();
+}
+
 TEST_P(tm_single_device_test_t, test_overwrite_pin)
 {
   test_overwrite_pin();

From 3e066d9d32abdcaf500c1fa90ef64735e3390f95 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Thu, 22 Jun 2023 04:42:55 +0000
Subject: [PATCH 0349/2492] tools/ceph_dedup_tool: limit memory used for
 fingerprint database

Adds an FpMap capable of discarding any entries which have not met the
dedup threshold.

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/tools/ceph_dedup_tool.cc | 182 ++++++++++++++++++++++++++++++-----
 1 file changed, 160 insertions(+), 22 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index b57c74cf45dd..0cb90699925d 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -185,6 +185,7 @@ po::options_description make_usage() {
     ("daemon", ": execute sample dedup in daemon mode")
     ("loop", ": execute sample dedup in a loop until terminated. Sleeps 'wakeup-period' seconds between iterations")
     ("wakeup-period", po::value<int>(), ": set the wakeup period of crawler thread (sec)")
+    ("fpstore-threshold", po::value<int>()->default_value(100_M), ": set max size of in-memory fingerprint store (bytes)")
   ;
   desc.add(op_desc);
   return desc;
@@ -566,10 +567,135 @@ class SampleDedupWorkerThread : public Thread
     bufferlist data;
   };
 
-  class FpStore {
+  using dup_count_t = ssize_t;
+
+  template <typename K, typename V>
+  class FpMap {
+    using map_t = std::unordered_map<K, V>;
   public:
-    using dup_count_t = ssize_t;
+    /// Represents a nullable reference into logical container
+    class entry_t {
+      /// Entry may be into one of two maps or NONE, indicates which
+      enum entry_into_t {
+	UNDER, OVER, NONE
+      } entry_into = NONE;
+
+      /// Valid iterator into map for UNDER|OVER, default for NONE
+      map_t::iterator iter;
+
+      entry_t(entry_into_t entry_into, map_t::iterator iter) :
+	entry_into(entry_into), iter(iter) {
+	ceph_assert(entry_into != NONE);
+      }
+
+    public:
+      entry_t() = default;
+
+      auto &operator*() {
+	ceph_assert(entry_into != NONE);
+	return *iter;
+      }
+      auto operator->() {
+	ceph_assert(entry_into != NONE);
+	return iter.operator->();
+      }
+      bool is_valid() const {
+	return entry_into != NONE;
+      }
+      bool is_above_threshold() const {
+	return entry_into == entry_t::OVER;
+      }
+      friend class FpMap;
+    };
+
+    /// inserts str, count into container, must not already be present
+    entry_t insert(const K &str, V count) {
+      std::pair<typename map_t::iterator, bool> r;
+      typename entry_t::entry_into_t s;
+      if (count < dedup_threshold) {
+       r = under_threshold_fp_map.insert({str, count});
+       s = entry_t::UNDER;
+      } else {
+       r = over_threshold_fp_map.insert({str, count});
+       s = entry_t::OVER;
+      }
+      ceph_assert(r.second);
+      return entry_t{s, r.first};
+    }
+
+    /// increments refcount for entry, promotes as necessary, entry must be valid
+    entry_t increment_reference(entry_t entry) {
+      ceph_assert(entry.is_valid());
+      entry.iter->second++;
+      if (entry.entry_into == entry_t::OVER ||
+	  entry.iter->second < dedup_threshold) {
+	return entry;
+      } else {
+	auto [over_iter, inserted] = over_threshold_fp_map.insert(
+	  *entry);
+	ceph_assert(inserted);
+	under_threshold_fp_map.erase(entry.iter);
+	return entry_t{entry_t::OVER, over_iter};
+      }
+    }
+
+    /// returns entry for fp, return will be !is_valid() if not present
+    auto find(const K &fp) {
+      if (auto iter = under_threshold_fp_map.find(fp);
+	  iter != under_threshold_fp_map.end()) {
+	return entry_t{entry_t::UNDER, iter};
+      } else if (auto iter = over_threshold_fp_map.find(fp);
+		 iter != over_threshold_fp_map.end()) {
+	return entry_t{entry_t::OVER, iter};
+      }  else {
+	return entry_t{};
+      }
+    }
 
+    /// true if container contains fp
+    bool contains(const K &fp) {
+      return find(fp).is_valid();
+    }
+
+    /// returns number of items
+    size_t get_num_items() const {
+      return under_threshold_fp_map.size() + over_threshold_fp_map.size();
+    }
+
+    /// returns estimate of total in-memory size (bytes)
+    size_t estimate_total_size() const {
+      size_t total = 0;
+      if (!under_threshold_fp_map.empty()) {
+	total += under_threshold_fp_map.size() *
+	  (under_threshold_fp_map.begin()->first.size() + sizeof(V));
+      }
+      if (!over_threshold_fp_map.empty()) {
+	total += over_threshold_fp_map.size() *
+	  (over_threshold_fp_map.begin()->first.size() + sizeof(V));
+      }
+      return total;
+    }
+
+    /// true if empty
+    bool empty() const {
+      return under_threshold_fp_map.empty() && over_threshold_fp_map.empty();
+    }
+
+    /// instructs container to drop entries with refcounts below threshold
+    void drop_entries_below_threshold() {
+      under_threshold_fp_map.clear();
+    }
+
+    FpMap(ssize_t dedup_threshold) : dedup_threshold(dedup_threshold) {}
+    FpMap() = delete;
+  private:
+    map_t under_threshold_fp_map;
+    map_t over_threshold_fp_map;
+    const ssize_t dedup_threshold;
+  };
+
+  class FpStore {
+  public:
     void maybe_print_status() {
       utime_t now = ceph_clock_now();
       if (next_report != utime_t() && now > next_report) {
@@ -581,41 +707,50 @@ class SampleDedupWorkerThread : public Thread
       }
     }
 
-    bool find(string& fp) {
+    bool contains(string& fp) {
       std::shared_lock lock(fingerprint_lock);
-      auto found_item = fp_map.find(fp);
-      return found_item != fp_map.end();
+      return fp_map.contains(fp);
     }
 
     // return true if the chunk is duplicate
     bool add(chunk_t& chunk) {
       std::unique_lock lock(fingerprint_lock);
-      auto found_iter = fp_map.find(chunk.fingerprint);
-      ssize_t cur_reference = 1;
+      auto entry = fp_map.find(chunk.fingerprint);
       total_bytes += chunk.size;
-      maybe_print_status();
-      if (found_iter == fp_map.end()) {
-        fp_map.insert({chunk.fingerprint, 1});
+      if (!entry.is_valid()) {
+	if (is_fpmap_full()) {
+	  fp_map.drop_entries_below_threshold();
+	  if (is_fpmap_full()) {
+	    return false;
+	  }
+	}
+	entry = fp_map.insert(chunk.fingerprint, 1);
       } else {
-	cur_reference = ++found_iter->second;
+	entry = fp_map.increment_reference(entry);
       }
-      return cur_reference >= dedup_threshold && dedup_threshold != -1;
+      return entry.is_above_threshold();
     }
 
-    FpStore(size_t chunk_threshold, uint32_t report_period) :
-      dedup_threshold(chunk_threshold), report_period(report_period) {
-      next_report = start;
-      next_report += report_period;
+    bool is_fpmap_full() const {
+      return fp_map.estimate_total_size() >= memory_threshold;
     }
 
+    FpStore(size_t chunk_threshold,
+      uint32_t report_period,	
+      ssize_t memory_threshold) :
+      report_period(report_period),
+      memory_threshold(memory_threshold),
+      fp_map(chunk_threshold) { }
+    FpStore() = delete;
+
   private:
-    ssize_t dedup_threshold = -1;
-    std::unordered_map<std::string, dup_count_t> fp_map;
     std::shared_mutex fingerprint_lock;
     const utime_t start = ceph_clock_now();
     utime_t next_report;
     const uint32_t report_period;
     size_t total_bytes = 0;
+    const uint64_t memory_threshold;
+    FpMap<std::string, dup_count_t> fp_map;
   };
 
   struct SampleDedupGlobal {
@@ -624,8 +759,9 @@ class SampleDedupWorkerThread : public Thread
     SampleDedupGlobal(
       int chunk_threshold,
       int sampling_ratio,
-      uint32_t report_period) :
-      fp_store(chunk_threshold, report_period),
+      uint32_t report_period,
+      unsigned fpstore_threshold) :
+      fp_store(chunk_threshold, report_period, fpstore_threshold),
       sampling_ratio(static_cast<double>(sampling_ratio) / 100) { }
   };
 
@@ -836,7 +972,7 @@ void SampleDedupWorkerThread::try_dedup_and_accumulate_result(
       .data = chunk_data
       };
 
-    if (sample_dedup_global.fp_store.find(fingerprint)) {
+    if (sample_dedup_global.fp_store.contains(fingerprint)) {
       duplicated_size += chunk_data.length();
     }
     if (sample_dedup_global.fp_store.add(chunk_info)) {
@@ -1643,6 +1779,8 @@ int make_crawling_daemon(const po::variables_map &opts)
     cout << "100 second is set as wakeup period by default" << std::endl;
   }
 
+  const unsigned fp_threshold = opts["fpstore-threshold"].as<int>();
+
   std::string fp_algo = get_opts_fp_algo(opts);
 
   list<string> pool_names;
@@ -1714,7 +1852,7 @@ int make_crawling_daemon(const po::variables_map &opts)
     }
 
     SampleDedupWorkerThread::SampleDedupGlobal sample_dedup_global(
-      chunk_dedup_threshold, sampling_ratio, report_period);
+      chunk_dedup_threshold, sampling_ratio, report_period, fp_threshold);
 
     std::list<SampleDedupWorkerThread> threads;
     size_t total_size = 0;

From c841c761ce6d3ef86a4390b4e6367f72bb323fb2 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 6 Oct 2023 15:54:28 -0700
Subject: [PATCH 0350/2492] tools/ceph_dedup_tool: use size_t for
 fpstore-threshold

At various points, this value was an int, an unsigned, an ssize_t, or a
uint64_t.  Change all of these to size_t.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/tools/ceph_dedup_tool.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index 0cb90699925d..65c2b9ac3c60 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -185,7 +185,7 @@ po::options_description make_usage() {
     ("daemon", ": execute sample dedup in daemon mode")
     ("loop", ": execute sample dedup in a loop until terminated. Sleeps 'wakeup-period' seconds between iterations")
     ("wakeup-period", po::value<int>(), ": set the wakeup period of crawler thread (sec)")
-    ("fpstore-threshold", po::value<int>()->default_value(100_M), ": set max size of in-memory fingerprint store (bytes)")
+    ("fpstore-threshold", po::value<size_t>()->default_value(100_M), ": set max size of in-memory fingerprint store (bytes)")
   ;
   desc.add(op_desc);
   return desc;
@@ -737,7 +737,7 @@ class SampleDedupWorkerThread : public Thread
 
     FpStore(size_t chunk_threshold,
       uint32_t report_period,	
-      ssize_t memory_threshold) :
+      size_t memory_threshold) :
       report_period(report_period),
       memory_threshold(memory_threshold),
       fp_map(chunk_threshold) { }
@@ -749,7 +749,7 @@ class SampleDedupWorkerThread : public Thread
     utime_t next_report;
     const uint32_t report_period;
     size_t total_bytes = 0;
-    const uint64_t memory_threshold;
+    const size_t memory_threshold;
     FpMap<std::string, dup_count_t> fp_map;
   };
 
@@ -760,7 +760,7 @@ class SampleDedupWorkerThread : public Thread
       int chunk_threshold,
       int sampling_ratio,
       uint32_t report_period,
-      unsigned fpstore_threshold) :
+      size_t fpstore_threshold) :
       fp_store(chunk_threshold, report_period, fpstore_threshold),
       sampling_ratio(static_cast<double>(sampling_ratio) / 100) { }
   };
@@ -1779,7 +1779,7 @@ int make_crawling_daemon(const po::variables_map &opts)
     cout << "100 second is set as wakeup period by default" << std::endl;
   }
 
-  const unsigned fp_threshold = opts["fpstore-threshold"].as<int>();
+  const size_t fp_threshold = opts["fpstore-threshold"].as<size_t>();
 
   std::string fp_algo = get_opts_fp_algo(opts);
 

From 1e4eb389ac22201b1614437ecbcb3f70bfe4d3ef Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 6 Oct 2023 15:58:01 -0700
Subject: [PATCH 0351/2492] tools/ceph_dedup_tool: use size_t for dup_count_t

Duplicate counts cannot be negative.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/tools/ceph_dedup_tool.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index 65c2b9ac3c60..4808058b866c 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -567,7 +567,7 @@ class SampleDedupWorkerThread : public Thread
     bufferlist data;
   };
 
-  using dup_count_t = ssize_t;
+  using dup_count_t = size_t;
 
   template <typename K, typename V>
   class FpMap {

From 08d71e82a34e7ab4eb79b1c81b331e03d5437155 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 6 Oct 2023 15:59:44 -0700
Subject: [PATCH 0352/2492] tools/ceph_dedup_tool: make dedup_threshold a
 size_t

It was variously an int, a uint32_t, or an ssize_t.  Standardize it to
size_t.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/tools/ceph_dedup_tool.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index 4808058b866c..c67368f9318c 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -180,7 +180,7 @@ po::options_description make_usage() {
     ("snap", ": deduplciate snapshotted object")
     ("debug", ": enable debug")
     ("pgid", ": set pgid")
-    ("chunk-dedup-threshold", po::value<uint32_t>(), ": set the threshold for chunk dedup (number of duplication) ")
+    ("chunk-dedup-threshold", po::value<size_t>(), ": set the threshold for chunk dedup (number of duplication) ")
     ("sampling-ratio", po::value<int>(), ": set the sampling ratio (percentile)")
     ("daemon", ": execute sample dedup in daemon mode")
     ("loop", ": execute sample dedup in a loop until terminated. Sleeps 'wakeup-period' seconds between iterations")
@@ -686,12 +686,12 @@ class SampleDedupWorkerThread : public Thread
       under_threshold_fp_map.clear();
     }
 
-    FpMap(ssize_t dedup_threshold) : dedup_threshold(dedup_threshold) {}
+    FpMap(size_t dedup_threshold) : dedup_threshold(dedup_threshold) {}
     FpMap() = delete;
   private:
     map_t under_threshold_fp_map;
     map_t over_threshold_fp_map;
-    const ssize_t dedup_threshold;
+    const size_t dedup_threshold;
   };
 
   class FpStore {
@@ -757,7 +757,7 @@ class SampleDedupWorkerThread : public Thread
     FpStore fp_store;
     const double sampling_ratio = -1;
     SampleDedupGlobal(
-      int chunk_threshold,
+      size_t chunk_threshold,
       int sampling_ratio,
       uint32_t report_period,
       size_t fpstore_threshold) :
@@ -1756,7 +1756,7 @@ int make_crawling_daemon(const po::variables_map &opts)
 
   uint32_t chunk_dedup_threshold = -1;
   if (opts.count("chunk-dedup-threshold")) {
-    chunk_dedup_threshold = opts["chunk-dedup-threshold"].as<uint32_t>();
+    chunk_dedup_threshold = opts["chunk-dedup-threshold"].as<size_t>();
   }
 
   std::string chunk_algo = get_opts_chunk_algo(opts);

From 231c7acc55071afa3c42241dcbd20bf5e9f84acc Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 16 Oct 2023 11:58:32 +0800
Subject: [PATCH 0353/2492] crimson/os/seastore/lba_manager: hide lba mapping
 ref count update away from users of TransactionManager

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../os/seastore/btree/btree_range_pin.cc      | 10 +++
 .../os/seastore/btree/btree_range_pin.h       |  1 +
 src/crimson/os/seastore/btree/fixed_kv_node.h | 38 +++++++++-
 src/crimson/os/seastore/cached_extent.cc      | 11 ++-
 src/crimson/os/seastore/cached_extent.h       |  4 +-
 .../flat_collection_manager.cc                |  2 +-
 .../lba_manager/btree/btree_lba_manager.cc    | 62 +++++++--------
 .../lba_manager/btree/btree_lba_manager.h     | 75 ++++++++++++-------
 .../os/seastore/object_data_handler.cc        |  4 +-
 .../omap_manager/btree/btree_omap_manager.cc  |  4 +-
 .../btree/omap_btree_node_impl.cc             |  2 +-
 .../node_extent_manager/seastore.h            |  2 +-
 .../os/seastore/transaction_manager.cc        | 10 +--
 src/crimson/os/seastore/transaction_manager.h | 39 +++++-----
 .../seastore/test_transaction_manager.cc      |  2 +-
 15 files changed, 173 insertions(+), 93 deletions(-)

diff --git a/src/crimson/os/seastore/btree/btree_range_pin.cc b/src/crimson/os/seastore/btree/btree_range_pin.cc
index 2f801dcf1ec5..1fe79eafa817 100644
--- a/src/crimson/os/seastore/btree/btree_range_pin.cc
+++ b/src/crimson/os/seastore/btree/btree_range_pin.cc
@@ -22,6 +22,16 @@ BtreeNodeMapping<key_t, val_t>::get_logical_extent(
   return v;
 }
 
+template <typename key_t, typename val_t>
+bool BtreeNodeMapping<key_t, val_t>::is_stable() const
+{
+  assert(parent);
+  assert(parent->is_valid());
+  assert(pos != std::numeric_limits<uint16_t>::max());
+  auto &p = (FixedKVNode<key_t>&)*parent;
+  return p.is_child_stable(pos);
+}
+
 template class BtreeNodeMapping<laddr_t, paddr_t>;
 template class BtreeNodeMapping<paddr_t, laddr_t>;
 } // namespace crimson::os::seastore
diff --git a/src/crimson/os/seastore/btree/btree_range_pin.h b/src/crimson/os/seastore/btree/btree_range_pin.h
index 7a08b6d89472..b23a50bf4bad 100644
--- a/src/crimson/os/seastore/btree/btree_range_pin.h
+++ b/src/crimson/os/seastore/btree/btree_range_pin.h
@@ -195,6 +195,7 @@ class BtreeNodeMapping : public PhysicalNodeMapping<key_t, val_t> {
   }
 
   get_child_ret_t<LogicalCachedExtent> get_logical_extent(Transaction&) final;
+  bool is_stable() const final;
 };
 
 }
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index 956a1824e2a5..0ae23b2f4dea 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -157,7 +157,7 @@ struct FixedKVNode : ChildableCachedExtent {
       (get_node_size() - offset - 1) * sizeof(ChildableCachedExtent*));
   }
 
-  FixedKVNode& get_stable_for_key(node_key_t key) {
+  FixedKVNode& get_stable_for_key(node_key_t key) const {
     ceph_assert(is_pending());
     if (is_mutation_pending()) {
       return (FixedKVNode&)*get_prior_instance();
@@ -229,6 +229,8 @@ struct FixedKVNode : ChildableCachedExtent {
   virtual get_child_ret_t<LogicalCachedExtent>
   get_logical_child(op_context_t<node_key_t> c, uint16_t pos) = 0;
 
+  virtual bool is_child_stable(uint16_t pos) const = 0;
+
   template <typename T, typename iter_t>
   get_child_ret_t<T> get_child(op_context_t<node_key_t> c, iter_t iter) {
     auto pos = iter.get_offset();
@@ -592,6 +594,11 @@ struct FixedKVInternalNode
     return get_child_ret_t<LogicalCachedExtent>(child_pos_t(nullptr, 0));
   }
 
+  bool is_child_stable(uint16_t pos) const final {
+    ceph_abort("impossible");
+    return false;
+  }
+
   bool validate_stable_children() final {
     LOG_PREFIX(FixedKVInternalNode::validate_stable_children);
     if (this->children.empty()) {
@@ -984,6 +991,35 @@ struct FixedKVLeafNode
     }
   }
 
+  // children are considered stable if any of the following case is true:
+  // 1. Not in memory
+  // 2. being stable
+  // 3. being mutation pending and under-io
+  bool is_child_stable(uint16_t pos) const final {
+    auto child = this->children[pos];
+    if (is_valid_child_ptr(child)) {
+      ceph_assert(child->is_logical());
+      return child->is_stable() ||
+	(child->is_mutation_pending() &&
+	 child->is_pending_io());
+    } else if (this->is_pending()) {
+      auto key = this->iter_idx(pos).get_key();
+      auto &sparent = this->get_stable_for_key(key);
+      auto spos = sparent.child_pos_for_key(key);
+      auto child = sparent.children[spos];
+      if (is_valid_child_ptr(child)) {
+	ceph_assert(child->is_logical());
+	return child->is_stable() ||
+	  (child->is_mutation_pending() &&
+	   child->is_pending_io());
+      } else {
+	return true;
+      }
+    } else {
+      return true;
+    }
+  }
+
   bool validate_stable_children() override {
     return true;
   }
diff --git a/src/crimson/os/seastore/cached_extent.cc b/src/crimson/os/seastore/cached_extent.cc
index 769b0446a5d6..37884227186f 100644
--- a/src/crimson/os/seastore/cached_extent.cc
+++ b/src/crimson/os/seastore/cached_extent.cc
@@ -158,8 +158,15 @@ parent_tracker_t::~parent_tracker_t() {
 
 std::ostream &operator<<(std::ostream &out, const LBAMapping &rhs)
 {
-  return out << "LBAMapping(" << rhs.get_key() << "~" << rhs.get_length()
-	     << "->" << rhs.get_val();
+  out << "LBAMapping(" << rhs.get_key() << "~" << rhs.get_length()
+      << "->" << rhs.get_val();
+  if (rhs.is_indirect()) {
+    out << " indirect(" << rhs.get_intermediate_base() << "~"
+	<< rhs.get_intermediate_key() << "~"
+	<< rhs.get_intermediate_length() << ")";
+  }
+  out << ")";
+  return out;
 }
 
 std::ostream &operator<<(std::ostream &out, const lba_pin_list_t &rhs)
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 02f8ae46c95c..c73839cf1fe8 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -595,7 +595,7 @@ class CachedExtent
 
   // a rewrite extent has an invalid prior_instance,
   // and a mutation_pending extent has a valid prior_instance
-  CachedExtentRef get_prior_instance() {
+  CachedExtentRef get_prior_instance() const {
     return prior_instance;
   }
 
@@ -1046,6 +1046,8 @@ class PhysicalNodeMapping {
     child_pos->link_child(c);
   }
 
+  virtual bool is_stable() const = 0;
+
   virtual ~PhysicalNodeMapping() {}
 protected:
   std::optional<child_pos_t> child_pos = std::nullopt;
diff --git a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
index decb095f6f98..3c65ed0e2c18 100644
--- a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
+++ b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
@@ -84,7 +84,7 @@ FlatCollectionManager::create(coll_root_t &coll_root, Transaction &t,
 	    get_coll_context(t), cid, info.split_bits
 	  ).si_then([=, this, &t](auto result) {
 	    assert(result == CollectionNode::create_result_t::SUCCESS);
-	    return tm.dec_ref(t, extent->get_laddr());
+	    return tm.remove(t, extent->get_laddr());
 	  }).si_then([] (auto) {
             return create_iertr::make_ready_future<>();
           });
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
index bb43bdb2c4f4..1b7f927ec0fe 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
@@ -197,7 +197,7 @@ BtreeLBAManager::_get_original_mappings(
 	    pin->get_key(), pin->get_length(),
 	    pin->get_raw_val().get_laddr());
 	  auto &btree_new_pin = static_cast<BtreeLBAMapping&>(*new_pin);
-	  btree_new_pin.set_key_for_indirect(
+	  btree_new_pin.make_indirect(
 	    pin->get_key(),
 	    pin->get_length(),
 	    pin->get_raw_val().get_laddr());
@@ -287,7 +287,7 @@ BtreeLBAManager::_get_mapping(
 	      c.trans, pin->get_raw_val().get_laddr()
 	    ).si_then([&pin](auto new_pin) {
 	      ceph_assert(pin->get_length() == new_pin->get_length());
-	      new_pin->set_key_for_indirect(
+	      new_pin->make_indirect(
 		pin->get_key(),
 		pin->get_length());
 	      return new_pin;
@@ -307,7 +307,6 @@ BtreeLBAManager::_alloc_extent(
   extent_len_t len,
   pladdr_t addr,
   paddr_t actual_addr,
-  laddr_t intermediate_base,
   LogicalCachedExtent* nextent)
 {
   struct state_t {
@@ -321,6 +320,8 @@ BtreeLBAManager::_alloc_extent(
 
   LOG_PREFIX(BtreeLBAManager::_alloc_extent);
   TRACET("{}~{}, hint={}", t, addr, len, hint);
+
+  ceph_assert(actual_addr != P_ADDR_NULL ? addr.is_laddr() : addr.is_paddr());
   auto c = get_context(t);
   ++stats.num_alloc_extents;
   auto lookup_attempts = stats.num_alloc_extents_iter_nexts;
@@ -384,17 +385,9 @@ BtreeLBAManager::_alloc_extent(
 	    state.ret = iter;
 	  });
 	});
-    }).si_then([c, actual_addr, addr, intermediate_base](auto &&state) {
-      auto ret_pin = state.ret->get_pin(c);
-      if (actual_addr != P_ADDR_NULL) {
-	ceph_assert(addr.is_laddr());
-	ret_pin->set_paddr(actual_addr);
-	ret_pin->set_intermediate_base(intermediate_base);
-      } else {
-	ceph_assert(addr.is_paddr());
-      }
-      return alloc_extent_iertr::make_ready_future<LBAMappingRef>(
-	std::move(ret_pin));
+    }).si_then([c](auto &&state) {
+      return alloc_extent_iertr::make_ready_future<
+	LBAMappingRef>(state.ret->get_pin(c));
     });
 }
 
@@ -556,7 +549,8 @@ BtreeLBAManager::update_mapping(
       return ret;
     },
     nextent
-  ).si_then([&t, laddr, prev_addr, addr, FNAME](auto result) {
+  ).si_then([&t, laddr, prev_addr, addr, FNAME](auto p) {
+      auto &result = p.first;
       DEBUGT("laddr={}, paddr {} => {} done -- {}",
              t, laddr, prev_addr, addr, result);
     },
@@ -687,7 +681,9 @@ BtreeLBAManager::update_refcount(
       return out;
     },
     nullptr
-  ).si_then([&t, addr, delta, FNAME, this, cascade_remove](auto result) {
+  ).si_then([&t, addr, delta, FNAME, this, cascade_remove](auto p) {
+    auto &result = p.first;
+    auto &mapping = p.second;
     DEBUGT("laddr={}, delta={} done -- {}", t, addr, delta, result);
     auto fut = ref_iertr::make_ready_future<
       std::optional<std::pair<paddr_t, extent_len_t>>>();
@@ -698,19 +694,23 @@ BtreeLBAManager::update_refcount(
 	result.len
       );
     }
-    return fut.si_then([result](auto removed) {
+    return fut.si_then([result, mapping=std::move(mapping)]
+		       (auto removed) mutable {
       if (result.pladdr.is_laddr()
 	  && removed) {
-	return ref_update_result_t{
-	  result.refcount,
-	  removed->first,
-	  removed->second};
+	return std::make_pair(
+	    ref_update_result_t{
+	      result.refcount,
+	      removed->first,
+	      removed->second},
+	    std::move(mapping));
       } else {
-	return ref_update_result_t{
-	  result.refcount,
-	  result.pladdr,
-	  result.len
-	};
+	return std::make_pair(
+	    ref_update_result_t{
+	      result.refcount,
+	      result.pladdr,
+	      result.len},
+	    std::move(mapping));
       }
     });
   });
@@ -724,7 +724,7 @@ BtreeLBAManager::_update_mapping(
   LogicalCachedExtent* nextent)
 {
   auto c = get_context(t);
-  return with_btree_ret<LBABtree, lba_map_val_t>(
+  return with_btree_ret<LBABtree, _update_mapping_ret_bare>(
     cache,
     c,
     [f=std::move(f), c, addr, nextent](auto &btree) mutable {
@@ -744,7 +744,8 @@ BtreeLBAManager::_update_mapping(
 	    c,
 	    iter
 	  ).si_then([ret] {
-	    return ret;
+	    return std::make_pair(
+		std::move(ret), BtreeLBAMappingRef(nullptr));
 	  });
 	} else {
 	  return btree.update(
@@ -752,8 +753,9 @@ BtreeLBAManager::_update_mapping(
 	    iter,
 	    ret,
 	    nextent
-	  ).si_then([ret](auto) {
-	    return ret;
+	  ).si_then([c, ret](auto iter) {
+	    return std::make_pair(
+		std::move(ret), iter.get_pin(c));
 	  });
 	}
       });
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
index 5496a4b19682..7895f806abd5 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
@@ -51,7 +51,8 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
 // 	3. intermediate_base: the laddr key of the physical lba mapping, intermediate_key
 // 	   and intermediate_base should be the same when doing cloning
 // 	4. intermediate_offset: intermediate_key - intermediate_base
-// 	5. paddr: the paddr recorded in the physical lba mapping pointed to by the
+// 	5. intermediate_length: the length of the actual physical lba mapping
+// 	6. paddr: the paddr recorded in the physical lba mapping pointed to by the
 // 	   indirect lba mapping being queried;
 //
 // NOTE THAT, for direct BtreeLBAMappings, their intermediate_keys are the same as
@@ -73,7 +74,7 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
 	val.len,
 	meta),
       key(meta.begin),
-      indirect(val.pladdr.is_laddr() ? true : false),
+      indirect(val.pladdr.is_laddr()),
       intermediate_key(indirect ? val.pladdr.get_laddr() : L_ADDR_NULL),
       intermediate_length(indirect ? val.len : 0),
       raw_val(val.pladdr),
@@ -88,12 +89,16 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     return indirect;
   }
 
-  void set_key_for_indirect(
+  void make_indirect(
     laddr_t new_key,
     extent_len_t length,
     laddr_t interkey = L_ADDR_NULL)
   {
-    turn_indirect(interkey);
+    assert(!indirect);
+    assert(value.is_paddr());
+    intermediate_base = key;
+    intermediate_key = (interkey == L_ADDR_NULL ? key : interkey);
+    indirect = true;
     key = new_key;
     intermediate_length = len;
     len = length;
@@ -107,10 +112,6 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     return raw_val;
   }
 
-  void set_paddr(paddr_t addr) {
-    value = addr;
-  }
-
   laddr_t get_intermediate_key() const final {
     assert(is_indirect());
     assert(intermediate_key != L_ADDR_NULL);
@@ -136,10 +137,6 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     return intermediate_length;
   }
 
-  void set_intermediate_base(laddr_t base) {
-    intermediate_base = base;
-  }
-
 protected:
   std::unique_ptr<BtreeNodeMapping<laddr_t, paddr_t>> _duplicate(
     op_context_t<laddr_t> ctx) const final {
@@ -154,12 +151,6 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     return pin;
   }
 private:
-  void turn_indirect(laddr_t interkey) {
-    assert(value.is_paddr());
-    intermediate_base = key;
-    intermediate_key = (interkey == L_ADDR_NULL ? key : interkey);
-    indirect = true;
-  }
   laddr_t key = L_ADDR_NULL;
   bool indirect = false;
   laddr_t intermediate_key = L_ADDR_NULL;
@@ -226,7 +217,6 @@ class BtreeLBAManager : public LBAManager {
       len,
       P_ADDR_ZERO,
       P_ADDR_NULL,
-      L_ADDR_NULL,
       nullptr);
   }
 
@@ -238,14 +228,32 @@ class BtreeLBAManager : public LBAManager {
     paddr_t actual_addr,
     laddr_t intermediate_base)
   {
+    assert(intermediate_key != L_ADDR_NULL);
+    assert(intermediate_base != L_ADDR_NULL);
     return _alloc_extent(
       t,
       hint,
       len,
       intermediate_key,
       actual_addr,
-      intermediate_base,
-      nullptr);
+      nullptr
+    ).si_then([&t, this, intermediate_base](auto indirect_mapping) {
+      assert(indirect_mapping->is_indirect());
+      return update_refcount(t, intermediate_base, 1, false
+      ).si_then([imapping=std::move(indirect_mapping)](auto p) mutable {
+	auto mapping = std::move(p.second);
+	ceph_assert(mapping->is_stable());
+	mapping->make_indirect(
+	  imapping->get_key(),
+	  imapping->get_length(),
+	  imapping->get_intermediate_key());
+	return seastar::make_ready_future<
+	  LBAMappingRef>(std::move(mapping));
+      });
+    }).handle_error_interruptible(
+      crimson::ct_error::input_output_error::pass_further{},
+      crimson::ct_error::assert_all{"unexpect enoent"}
+    );
   }
 
   alloc_extent_ret alloc_extent(
@@ -261,7 +269,6 @@ class BtreeLBAManager : public LBAManager {
       len,
       addr,
       P_ADDR_NULL,
-      L_ADDR_NULL,
       &ext);
   }
 
@@ -269,13 +276,19 @@ class BtreeLBAManager : public LBAManager {
     Transaction &t,
     laddr_t addr,
     bool cascade_remove) final {
-    return update_refcount(t, addr, -1, cascade_remove);
+    return update_refcount(t, addr, -1, cascade_remove
+    ).si_then([](auto p) {
+      return std::move(p.first);
+    });
   }
 
   ref_ret incref_extent(
     Transaction &t,
     laddr_t addr) final {
-    return update_refcount(t, addr, 1, false);
+    return update_refcount(t, addr, 1, false
+    ).si_then([](auto p) {
+      return std::move(p.first);
+    });
   }
 
   ref_ret incref_extent(
@@ -283,7 +296,10 @@ class BtreeLBAManager : public LBAManager {
     laddr_t addr,
     int delta) final {
     ceph_assert(delta > 0);
-    return update_refcount(t, addr, delta, false);
+    return update_refcount(t, addr, delta, false
+    ).si_then([](auto p) {
+      return std::move(p.first);
+    });
   }
 
   /**
@@ -344,7 +360,10 @@ class BtreeLBAManager : public LBAManager {
    *
    * Updates refcount, returns resulting refcount
    */
-  using update_refcount_ret = ref_ret;
+  using update_refcount_ret_bare = std::pair<ref_update_result_t, BtreeLBAMappingRef>;
+  using update_refcount_iertr = ref_iertr;
+  using update_refcount_ret = update_refcount_iertr::future<
+    update_refcount_ret_bare>;
   update_refcount_ret update_refcount(
     Transaction &t,
     laddr_t addr,
@@ -357,7 +376,8 @@ class BtreeLBAManager : public LBAManager {
    * Updates mapping, removes if f returns nullopt
    */
   using _update_mapping_iertr = ref_iertr;
-  using _update_mapping_ret = ref_iertr::future<lba_map_val_t>;
+  using _update_mapping_ret_bare = std::pair<lba_map_val_t, BtreeLBAMappingRef>;
+  using _update_mapping_ret = ref_iertr::future<_update_mapping_ret_bare>;
   using update_func_t = std::function<
     lba_map_val_t(const lba_map_val_t &v)
     >;
@@ -373,7 +393,6 @@ class BtreeLBAManager : public LBAManager {
     extent_len_t len,
     pladdr_t addr,
     paddr_t actual_addr,
-    laddr_t intermediate_base,
     LogicalCachedExtent*);
 
   using _get_mapping_ret = get_mapping_iertr::future<BtreeLBAMappingRef>;
diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 025f91993efa..1b0ae5c814ae 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -387,7 +387,7 @@ ObjectDataHandler::write_ret do_removals(
       DEBUGT("decreasing ref: {}",
 	     ctx.t,
 	     pin->get_key());
-      return ctx.tm.dec_ref(
+      return ctx.tm.remove(
 	ctx.t,
 	pin->get_key()
       ).si_then(
@@ -1524,7 +1524,7 @@ ObjectDataHandler::clone_ret ObjectDataHandler::clone_extents(
     object_data.get_reserved_data_base(),
     object_data.get_reserved_data_len(),
     data_base);
-  return ctx.tm.dec_ref(
+  return ctx.tm.remove(
     ctx.t,
     object_data.get_reserved_data_base()
   ).si_then(
diff --git a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
index 1782d7ee66ef..77dc270a5323 100644
--- a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
@@ -84,7 +84,7 @@ BtreeOMapManager::handle_root_merge(
     omap_root.hint);
   oc.t.get_omap_tree_stats().depth = omap_root.depth;
   oc.t.get_omap_tree_stats().extents_num_delta--;
-  return oc.tm.dec_ref(oc.t, root->get_laddr()
+  return oc.tm.remove(oc.t, root->get_laddr()
   ).si_then([](auto &&ret) -> handle_root_merge_ret {
     return seastar::now();
   }).handle_error_interruptible(
@@ -274,7 +274,7 @@ BtreeOMapManager::omap_clear(
   ).si_then([this, &t, &omap_root](auto extent) {
     return extent->clear(get_omap_context(t, omap_root.hint));
   }).si_then([this, &omap_root, &t] {
-    return tm.dec_ref(
+    return tm.remove(
       t, omap_root.get_location()
     ).si_then([&omap_root] (auto ret) {
       omap_root.update(
diff --git a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
index 4db58414a6ec..96115f13237c 100644
--- a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
@@ -36,7 +36,7 @@ using dec_ref_iertr = OMapInnerNode::base_iertr;
 using dec_ref_ret = dec_ref_iertr::future<>;
 template <typename T>
 dec_ref_ret dec_ref(omap_context_t oc, T&& addr) {
-  return oc.tm.dec_ref(oc.t, std::forward<T>(addr)).handle_error_interruptible(
+  return oc.tm.remove(oc.t, std::forward<T>(addr)).handle_error_interruptible(
     dec_ref_iertr::pass_further{},
     crimson::ct_error::assert_all{
       "Invalid error in OMapInnerNode helper dec_ref"
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
index f7cfa8c2112d..c12e583bd566 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
@@ -165,7 +165,7 @@ class SeastoreNodeExtentManager final: public TransactionManagerHandle {
         return retire_iertr::now();
       }
     }
-    return tm.dec_ref(t, extent).si_then([addr, len, &t] (unsigned cnt) {
+    return tm.remove(t, extent).si_then([addr, len, &t] (unsigned cnt) {
       assert(cnt == 0);
       SUBTRACET(seastore_onode, "retired {}B at {:#x} ...", t, len, addr);
     });
diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index ad8e5f1a65f4..7fbe119fcdd7 100644
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -210,11 +210,11 @@ TransactionManager::ref_ret TransactionManager::inc_ref(
   });
 }
 
-TransactionManager::ref_ret TransactionManager::dec_ref(
+TransactionManager::ref_ret TransactionManager::remove(
   Transaction &t,
   LogicalCachedExtentRef &ref)
 {
-  LOG_PREFIX(TransactionManager::dec_ref);
+  LOG_PREFIX(TransactionManager::remove);
   TRACET("{}", t, *ref);
   return lba_manager->decref_extent(t, ref->get_laddr(), true
   ).si_then([this, FNAME, &t, ref](auto result) {
@@ -253,17 +253,17 @@ TransactionManager::ref_ret TransactionManager::_dec_ref(
   });
 }
 
-TransactionManager::refs_ret TransactionManager::dec_ref(
+TransactionManager::refs_ret TransactionManager::remove(
   Transaction &t,
   std::vector<laddr_t> offsets)
 {
-  LOG_PREFIX(TransactionManager::dec_ref);
+  LOG_PREFIX(TransactionManager::remove);
   DEBUG("{} offsets", offsets.size());
   return seastar::do_with(std::move(offsets), std::vector<unsigned>(),
       [this, &t] (auto &&offsets, auto &refcnt) {
       return trans_intr::do_for_each(offsets.begin(), offsets.end(),
         [this, &t, &refcnt] (auto &laddr) {
-        return this->dec_ref(t, laddr).si_then([&refcnt] (auto ref) {
+        return this->remove(t, laddr).si_then([&refcnt] (auto ref) {
           refcnt.push_back(ref);
           return ref_iertr::now();
         });
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index dfce85c5e1b8..f30dea3bd779 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -235,6 +235,7 @@ class TransactionManager : public ExtentCallbackInterface {
   using ref_iertr = LBAManager::ref_iertr;
   using ref_ret = ref_iertr::future<unsigned>;
 
+#ifdef UNIT_TESTS_BUILT
   /// Add refcount for ref
   ref_ret inc_ref(
     Transaction &t,
@@ -244,14 +245,28 @@ class TransactionManager : public ExtentCallbackInterface {
   ref_ret inc_ref(
     Transaction &t,
     laddr_t offset);
+#endif
 
-  /// Remove refcount for ref
-  ref_ret dec_ref(
+  /** 
+   * remove
+   *
+   * Remove the extent and the corresponding lba mapping,
+   * users must make sure that lba mapping's refcount is 1
+   */
+  ref_ret remove(
     Transaction &t,
     LogicalCachedExtentRef &ref);
 
-  /// Remove refcount for offset
-  ref_ret dec_ref(
+  /**
+   * remove
+   *
+   * 1. Remove the indirect mapping(s), and if refcount drops to 0,
+   *    also remove the direct mapping and retire the extent.
+   * 
+   * 2. Remove the direct mapping(s) and retire the extent if
+   * 	refcount drops to 0.
+   */
+  ref_ret remove(
     Transaction &t,
     laddr_t offset) {
     return _dec_ref(t, offset, true);
@@ -259,7 +274,7 @@ class TransactionManager : public ExtentCallbackInterface {
 
   /// remove refcount for list of offset
   using refs_ret = ref_iertr::future<std::vector<unsigned>>;
-  refs_ret dec_ref(
+  refs_ret remove(
     Transaction &t,
     std::vector<laddr_t> offsets);
 
@@ -487,10 +502,6 @@ class TransactionManager : public ExtentCallbackInterface {
       mapping.is_indirect()
 	? mapping.get_intermediate_key()
 	: mapping.get_key();
-    auto intermediate_base =
-      mapping.is_indirect()
-      ? mapping.get_intermediate_base()
-      : mapping.get_key();
 
     LOG_PREFIX(TransactionManager::clone_pin);
     SUBDEBUGT(seastore_tm, "len={}, laddr_hint={}, clone_offset {}",
@@ -503,15 +514,7 @@ class TransactionManager : public ExtentCallbackInterface {
       intermediate_key,
       mapping.get_val(),
       intermediate_key
-    ).si_then([this, &t, intermediate_base](auto pin) {
-      return inc_ref(t, intermediate_base
-      ).si_then([pin=std::move(pin)](auto) mutable {
-	return std::move(pin);
-      }).handle_error_interruptible(
-	crimson::ct_error::input_output_error::pass_further(),
-	crimson::ct_error::assert_all("not possible")
-      );
-    });
+    );
   }
 
   /* alloc_extents
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index 914eea9bbbfd..54bd27b8c181 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -636,7 +636,7 @@ struct transaction_manager_test_t :
     ceph_assert(test_mappings.get(offset, t.mapping_delta).refcount > 0);
 
     auto refcnt = with_trans_intr(*(t.t), [&](auto& trans) {
-      return tm->dec_ref(trans, offset);
+      return tm->remove(trans, offset);
     }).unsafe_get0();
     auto check_refcnt = test_mappings.dec_ref(offset, t.mapping_delta);
     EXPECT_EQ(refcnt, check_refcnt);

From 4010045691ba940a409d2cc78db336fa4a1193a3 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 31 Oct 2023 20:42:56 +0800
Subject: [PATCH 0354/2492] crimson/os/seastore/lba_manager: rename
 clone_extent to clone_mapping

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/lba_manager.h                  |  2 +-
 .../os/seastore/lba_manager/btree/btree_lba_manager.h  |  2 +-
 src/crimson/os/seastore/transaction_manager.h          | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h
index d7adf2304fbb..d4d1826bcf90 100644
--- a/src/crimson/os/seastore/lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager.h
@@ -89,7 +89,7 @@ class LBAManager {
     paddr_t addr,
     LogicalCachedExtent &nextent) = 0;
 
-  virtual alloc_extent_ret clone_extent(
+  virtual alloc_extent_ret clone_mapping(
     Transaction &t,
     laddr_t hint,
     extent_len_t len,
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
index 7895f806abd5..79d21b363af3 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
@@ -220,7 +220,7 @@ class BtreeLBAManager : public LBAManager {
       nullptr);
   }
 
-  alloc_extent_ret clone_extent(
+  alloc_extent_ret clone_mapping(
     Transaction &t,
     laddr_t hint,
     extent_len_t len,
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index f30dea3bd779..b708e0a9f203 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -485,12 +485,12 @@ class TransactionManager : public ExtentCallbackInterface {
   }
 
   /*
-   * clone_pin
+   * clone_mapping
    *
    * create an indirect lba mapping pointing to the physical
    * lba mapping whose key is intermediate_key. Resort to btree_lba_manager.h
-   * for the definition of "indirect lba mapping" and "physical lba mapping"
-   *
+   * for the definition of "indirect lba mapping" and "physical lba mapping".
+   * Note that the cloned extent must be stable
    */
   using clone_extent_iertr = alloc_extent_iertr;
   using clone_extent_ret = clone_extent_iertr::future<LBAMappingRef>;
@@ -507,7 +507,7 @@ class TransactionManager : public ExtentCallbackInterface {
     SUBDEBUGT(seastore_tm, "len={}, laddr_hint={}, clone_offset {}",
       t, mapping.get_length(), hint, intermediate_key);
     ceph_assert(is_aligned(hint, epm->get_block_size()));
-    return lba_manager->clone_extent(
+    return lba_manager->clone_mapping(
       t,
       hint,
       mapping.get_length(),
@@ -887,7 +887,7 @@ class TransactionManager : public ExtentCallbackInterface {
       fut = lba_manager->alloc_extent(
 	t, remap_laddr, remap_length, remap_paddr, *ext);
     } else {
-      fut = lba_manager->clone_extent(
+      fut = lba_manager->clone_mapping(
 	t,
 	remap_laddr,
 	remap_length,

From 40f053aee0d3504d34545101a546b3eaf64f50d1 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 17 Oct 2023 10:04:33 +0530
Subject: [PATCH 0355/2492] mgr/dashboard: get object bucket policies for a
 bucket

Getting the bucket details will also fetch the bucket policy if its set.

Fixes: https://tracker.ceph.com/issues/63221
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   |   5 +
 .../rgw-bucket-details.component.html         | 211 ++++++++++--------
 .../rgw-bucket-details.component.spec.ts      |   3 +-
 .../rgw-bucket-details.component.ts           |   1 +
 .../mgr/dashboard/services/rgw_client.py      |  21 ++
 5 files changed, 150 insertions(+), 91 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 65c809ebec02..f7b9ed8ecf37 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -276,6 +276,10 @@ def _set_locking(self, owner, daemon_name, bucket_name, mode,
                                              retention_period_days,
                                              retention_period_years)
 
+    def _get_policy(self, bucket: str):
+        rgw_client = RgwClient.admin_instance()
+        return rgw_client.get_bucket_policy(bucket)
+
     @staticmethod
     def strip_tenant_from_bucket_name(bucket_name):
         # type (str) -> str
@@ -328,6 +332,7 @@ def get(self, bucket, daemon_name=None):
         result['encryption'] = encryption['Status']
         result['versioning'] = versioning['Status']
         result['mfa_delete'] = versioning['MfaDelete']
+        result['policy'] = self._get_policy(bucket_name)
 
         # Append the locking configuration.
         locking = self._get_locking(result['owner'], daemon_name, bucket_name)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
index c947e4490625..f2447feab264 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
@@ -1,94 +1,125 @@
 <ng-container *ngIf="selection">
-  <table class="table table-striped table-bordered">
-    <tbody>
-      <tr>
-        <td i18n
-            class="bold w-25">Versioning</td>
-        <td class="w-75">{{ selection.versioning }}</td>
-      </tr>
-      <tr>
-        <td i18n
-            class="bold">Encryption</td>
-        <td>{{ selection.encryption }}</td>
-      </tr>
-      <tr>
-        <td i18n
-            class="bold">MFA Delete</td>
-        <td>{{ selection.mfa_delete }}</td>
-      </tr>
-      <tr>
-        <td i18n
-            class="bold">Index type</td>
-        <td>{{ selection.index_type }}</td>
-      </tr>
-      <tr>
-        <td i18n
-            class="bold">Placement rule</td>
-        <td>{{ selection.placement_rule }}</td>
-      </tr>
-      <tr>
-        <td i18n
-            class="bold">Last modification time</td>
-        <td>{{ selection.mtime | cdDate }}</td>
-      </tr>
-    </tbody>
-  </table>
+  <nav ngbNav
+       #nav="ngbNav"
+       class="nav-tabs"
+       cdStatefulTab="rgw-bucket-details">
+    <ng-container ngbNavItem="details">
+      <a ngbNavLink
+         i18n>Details</a>
+      <ng-template ngbNavContent>
 
-  <!-- Bucket quota -->
-  <div>
-    <legend i18n>Bucket quota</legend>
-    <table class="table table-striped table-bordered">
-      <tbody>
-        <tr>
-          <td i18n
-              class="bold w-25">Enabled</td>
-          <td class="w-75">{{ selection.bucket_quota.enabled | booleanText }}</td>
-        </tr>
-        <ng-container *ngIf="selection.bucket_quota.enabled">
-          <tr>
-            <td i18n
-                class="bold">Maximum size</td>
-            <td *ngIf="selection.bucket_quota.max_size <= -1"
-                i18n>Unlimited</td>
-            <td *ngIf="selection.bucket_quota.max_size > -1">
-              {{ selection.bucket_quota.max_size | dimless }}
-            </td>
-          </tr>
-          <tr>
-            <td i18n
-                class="bold">Maximum objects</td>
-            <td *ngIf="selection.bucket_quota.max_objects <= -1"
-                i18n>Unlimited</td>
-            <td *ngIf="selection.bucket_quota.max_objects > -1">
-              {{ selection.bucket_quota.max_objects }}
-            </td>
-          </tr>
-        </ng-container>
-      </tbody>
-    </table>
-  </div>
+        <table class="table table-striped table-bordered">
+          <tbody>
+            <tr>
+              <td i18n
+                  class="bold w-25">Versioning</td>
+              <td class="w-75">{{ selection.versioning }}</td>
+            </tr>
+            <tr>
+              <td i18n
+                  class="bold">Encryption</td>
+              <td>{{ selection.encryption }}</td>
+            </tr>
+            <tr>
+              <td i18n
+                  class="bold">MFA Delete</td>
+              <td>{{ selection.mfa_delete }}</td>
+            </tr>
+            <tr>
+              <td i18n
+                  class="bold">Index type</td>
+              <td>{{ selection.index_type }}</td>
+            </tr>
+            <tr>
+              <td i18n
+                  class="bold">Placement rule</td>
+              <td>{{ selection.placement_rule }}</td>
+            </tr>
+            <tr>
+              <td i18n
+                  class="bold">Last modification time</td>
+              <td>{{ selection.mtime | cdDate }}</td>
+            </tr>
+          </tbody>
+        </table>
 
-  <!-- Locking -->
-  <legend i18n>Locking</legend>
-  <table class="table table-striped table-bordered">
-    <tbody>
-      <tr>
-        <td i18n
-            class="bold w-25">Enabled</td>
-        <td class="w-75">{{ selection.lock_enabled | booleanText }}</td>
-      </tr>
-      <ng-container *ngIf="selection.lock_enabled">
-        <tr>
-          <td i18n
-              class="bold">Mode</td>
-          <td>{{ selection.lock_mode }}</td>
-        </tr>
-        <tr>
-          <td i18n
-              class="bold">Days</td>
-          <td>{{ selection.lock_retention_period_days }}</td>
-        </tr>
-      </ng-container>
-    </tbody>
-  </table>
+        <!-- Bucket quota -->
+        <div>
+          <legend i18n>Bucket quota</legend>
+          <table class="table table-striped table-bordered">
+            <tbody>
+              <tr>
+                <td i18n
+                    class="bold w-25">Enabled</td>
+                <td class="w-75">{{ selection.bucket_quota.enabled | booleanText }}</td>
+              </tr>
+              <ng-container *ngIf="selection.bucket_quota.enabled">
+                <tr>
+                  <td i18n
+                      class="bold">Maximum size</td>
+                  <td *ngIf="selection.bucket_quota.max_size <= -1"
+                      i18n>Unlimited</td>
+                  <td *ngIf="selection.bucket_quota.max_size > -1">
+                    {{ selection.bucket_quota.max_size | dimless }}
+                  </td>
+                </tr>
+                <tr>
+                  <td i18n
+                      class="bold">Maximum objects</td>
+                  <td *ngIf="selection.bucket_quota.max_objects <= -1"
+                      i18n>Unlimited</td>
+                  <td *ngIf="selection.bucket_quota.max_objects > -1">
+                    {{ selection.bucket_quota.max_objects }}
+                  </td>
+                </tr>
+              </ng-container>
+            </tbody>
+          </table>
+        </div>
+
+        <!-- Locking -->
+        <legend i18n>Locking</legend>
+        <table class="table table-striped table-bordered">
+          <tbody>
+            <tr>
+              <td i18n
+                  class="bold w-25">Enabled</td>
+              <td class="w-75">{{ selection.lock_enabled | booleanText }}</td>
+            </tr>
+            <ng-container *ngIf="selection.lock_enabled">
+              <tr>
+                <td i18n
+                    class="bold">Mode</td>
+                <td>{{ selection.lock_mode }}</td>
+              </tr>
+              <tr>
+                <td i18n
+                    class="bold">Days</td>
+                <td>{{ selection.lock_retention_period_days }}</td>
+              </tr>
+            </ng-container>
+          </tbody>
+        </table>
+      </ng-template>
+    </ng-container>
+
+    <ng-container ngbNavItem="permissions">
+      <a ngbNavLink
+         i18n>Permissions</a>
+      <ng-template ngbNavContent>
+
+        <table class="table table-striped table-bordered">
+          <tbody>
+            <tr>
+              <td i18n
+                  class="bold w-25">Policy</td>
+              <td><pre>{{ selection.policy | json}}</pre></td>
+            </tr>
+          </tbody>
+        </table>
+      </ng-template>
+    </ng-container>
+  </nav>
+
+  <div [ngbNavOutlet]="nav"></div>
 </ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
index 59f62952a507..be6aa09182ca 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.spec.ts
@@ -8,6 +8,7 @@ import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
 import { SharedModule } from '~/app/shared/shared.module';
 import { configureTestBed } from '~/testing/unit-test-helper';
 import { RgwBucketDetailsComponent } from './rgw-bucket-details.component';
+import { NgbNavModule } from '@ng-bootstrap/ng-bootstrap';
 
 describe('RgwBucketDetailsComponent', () => {
   let component: RgwBucketDetailsComponent;
@@ -17,7 +18,7 @@ describe('RgwBucketDetailsComponent', () => {
 
   configureTestBed({
     declarations: [RgwBucketDetailsComponent],
-    imports: [SharedModule, HttpClientTestingModule]
+    imports: [SharedModule, HttpClientTestingModule, NgbNavModule]
   });
 
   beforeEach(() => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
index f9a351367daa..0ecbe0536dff 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
@@ -18,6 +18,7 @@ export class RgwBucketDetailsComponent implements OnChanges {
       this.rgwBucketService.get(this.selection.bid).subscribe((bucket: object) => {
         bucket['lock_retention_period_days'] = this.rgwBucketService.getLockDays(bucket);
         this.selection = bucket;
+        this.selection.policy = JSON.parse(this.selection.policy) || {};
       });
     }
   }
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index 20c0397458b2..2d3226bab3e0 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -852,6 +852,27 @@ def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: st
                    f' For more information about the format look at {link}')
             raise DashboardException(msg=msg, component='rgw')
 
+    @RestClient.api_get('/{bucket_name}?policy')
+    def get_bucket_policy(self, bucket_name: str, request=None):
+        """
+        Gets the bucket policy for a bucket.
+        :param bucket_name: The name of the bucket.
+        :type bucket_name: str
+        :rtype: None
+        """
+        # pylint: disable=unused-argument
+
+        try:
+            request = request()
+            return request
+        except RequestException as e:
+            if e.content:
+                content = json_str_to_object(e.content)
+                if content.get(
+                        'Code') == 'NoSuchBucketPolicy':
+                    return None
+            raise e
+
     def perform_validations(self, retention_period_days, retention_period_years, mode):
         try:
             retention_period_days = int(retention_period_days) if retention_period_days else 0

From f59a0dad06b11ad248e1c5a9c30ed919c40d7263 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 22 Aug 2023 11:23:47 +0800
Subject: [PATCH 0356/2492] crimson/common/operation: cleanup, drop the unused
 PipelineExitBarrierI::cancel()

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/operation.h | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h
index 6df2c99fd2a0..b0ac033a8cd3 100644
--- a/src/crimson/common/operation.h
+++ b/src/crimson/common/operation.h
@@ -478,13 +478,10 @@ class PipelineExitBarrierI {
   /// Waits for exit barrier
   virtual std::optional<seastar::future<>> wait() = 0;
 
-  /// Releases pipeline stage, can only be called after wait
+  /// Releases pipeline resources, after or without waiting
   virtual void exit() = 0;
 
-  /// Releases pipeline resources without waiting on barrier
-  virtual void cancel() = 0;
-
-  /// Must ensure that resources are released, likely by calling cancel()
+  /// Must ensure that resources are released, likely by calling exit()
   virtual ~PipelineExitBarrierI() {}
 };
 
@@ -606,12 +603,8 @@ class OrderedExclusivePhaseT : public PipelineStageIT<T> {
       }
     }
 
-    void cancel() final {
-      exit();
-    }
-
     ~ExitBarrier() final {
-      cancel();
+      exit();
     }
   };
 
@@ -717,12 +710,8 @@ class OrderedConcurrentPhaseT : public PipelineStageIT<T> {
       }
     }
 
-    void cancel() final {
-      exit();
-    }
-
     ~ExitBarrier() final {
-      cancel();
+      exit();
     }
   };
 
@@ -756,8 +745,6 @@ class UnorderedStageT : public PipelineStageIT<T> {
 
     void exit() final {}
 
-    void cancel() final {}
-
     ~ExitBarrier() final {}
   };
 

From 2e172d94bf093731f838882e4285ba9eb160aca3 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 14 Sep 2023 16:52:26 +0800
Subject: [PATCH 0357/2492] crimson/common/operation: document a potential
 trivial issue

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/operation.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h
index b0ac033a8cd3..3a65a793a9e1 100644
--- a/src/crimson/common/operation.h
+++ b/src/crimson/common/operation.h
@@ -479,6 +479,8 @@ class PipelineExitBarrierI {
   virtual std::optional<seastar::future<>> wait() = 0;
 
   /// Releases pipeline resources, after or without waiting
+  // FIXME: currently, exit() will discard the associated future even if it is
+  // still unresolved, which is discouraged by seastar.
   virtual void exit() = 0;
 
   /// Must ensure that resources are released, likely by calling exit()

From f8f3792cd325299147bf2d0c15caccd318cb134a Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 14 Sep 2023 16:55:51 +0800
Subject: [PATCH 0358/2492] crimson/osd/pg_shard_manager: relate
 remove_from/add_to_registry to cross-core

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/pg_shard_manager.h | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index 2f3a3015d1cd..fb5e93392caf 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -161,7 +161,9 @@ class PGShardManager {
         target_shard_services,
         std::move(op));
     }
-    return op->prepare_remote_submission(
+    auto &opref = *op;
+    get_local_state().registry.remove_from_registry(opref);
+    return opref.prepare_remote_submission(
     ).then([op=std::move(op), f=std::move(f), this, core
            ](auto f_conn) mutable {
       return shard_services.invoke_on(
@@ -169,9 +171,9 @@ class PGShardManager {
         [f=std::move(f), op=std::move(op), f_conn=std::move(f_conn)
         ](auto &target_shard_services) mutable {
         op->finish_remote_submission(std::move(f_conn));
+        target_shard_services.local_state.registry.add_to_registry(*op);
         return std::invoke(
           std::move(f),
-          target_shard_services.local_state,
           target_shard_services,
           std::move(op));
       });
@@ -188,16 +190,13 @@ class PGShardManager {
     static_assert(T::can_create());
     logger.debug("{}: can_create", *op);
 
-    get_local_state().registry.remove_from_registry(*op);
     return get_pg_to_shard_mapping().maybe_create_pg(
       op->get_pgid()
     ).then([this, op = std::move(op)](auto core) mutable {
       return this->template with_remote_shard_state_and_op<T>(
         core, std::move(op),
-        [](PerShardState &per_shard_state,
-           ShardServices &shard_services,
+        [](ShardServices &shard_services,
            typename T::IRef op) {
-	per_shard_state.registry.add_to_registry(*op);
 	auto &logger = crimson::get_logger(ceph_subsys_osd);
 	auto &opref = *op;
 	return opref.template with_blocking_event<
@@ -232,16 +231,13 @@ class PGShardManager {
     static_assert(!T::can_create());
     logger.debug("{}: !can_create", *op);
 
-    get_local_state().registry.remove_from_registry(*op);
     return get_pg_to_shard_mapping().maybe_create_pg(
       op->get_pgid()
     ).then([this, op = std::move(op)](auto core) mutable {
       return this->template with_remote_shard_state_and_op<T>(
         core, std::move(op),
-        [](PerShardState &per_shard_state,
-           ShardServices &shard_services,
+        [](ShardServices &shard_services,
            typename T::IRef op) {
-	per_shard_state.registry.add_to_registry(*op);
 	auto &logger = crimson::get_logger(ceph_subsys_osd);
 	auto &opref = *op;
 	return opref.template with_blocking_event<

From 08c9fbca19b62b0a3d0dc3892e6191cdeae73483 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 14 Sep 2023 17:02:03 +0800
Subject: [PATCH 0359/2492] crimson/osd/pg_shard_manager: cleanups around the
 remote pg submission

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/pg_shard_manager.h | 133 +++++++++++++----------------
 1 file changed, 57 insertions(+), 76 deletions(-)

diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index fb5e93392caf..71a9cf1a9c80 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -153,11 +153,11 @@ class PGShardManager {
       core_id_t core,
       typename T::IRef &&op,
       F &&f) {
+    ceph_assert(op->use_count() == 1);
     if (seastar::this_shard_id() == core) {
       auto &target_shard_services = shard_services.local();
       return std::invoke(
         std::move(f),
-        target_shard_services.local_state,
         target_shard_services,
         std::move(op));
     }
@@ -183,80 +183,54 @@ class PGShardManager {
   /// Runs opref on the appropriate core, creating the pg as necessary.
   template <typename T>
   seastar::future<> run_with_pg_maybe_create(
-    typename T::IRef op
+    typename T::IRef op,
+    ShardServices &target_shard_services
   ) {
-    ceph_assert(op->use_count() == 1);
-    auto &logger = crimson::get_logger(ceph_subsys_osd);
     static_assert(T::can_create());
-    logger.debug("{}: can_create", *op);
-
-    return get_pg_to_shard_mapping().maybe_create_pg(
-      op->get_pgid()
-    ).then([this, op = std::move(op)](auto core) mutable {
-      return this->template with_remote_shard_state_and_op<T>(
-        core, std::move(op),
-        [](ShardServices &shard_services,
-           typename T::IRef op) {
-	auto &logger = crimson::get_logger(ceph_subsys_osd);
-	auto &opref = *op;
-	return opref.template with_blocking_event<
-	  PGMap::PGCreationBlockingEvent
-	  >([&shard_services, &opref](
-	      auto &&trigger) {
-	    return shard_services.get_or_create_pg(
-	      std::move(trigger),
-	      opref.get_pgid(),
-	      std::move(opref.get_create_info())
-	    );
-	  }).safe_then([&logger, &shard_services, &opref](Ref<PG> pgref) {
-	    logger.debug("{}: have_pg", opref);
-	    return opref.with_pg(shard_services, pgref);
-	  }).handle_error(
-	    crimson::ct_error::ecanceled::handle([&logger, &opref](auto) {
-	      logger.debug("{}: pg creation canceled, dropping", opref);
-	      return seastar::now();
-	    })
-	  ).then([op=std::move(op)] {});
-      });
-    });
+    auto &logger = crimson::get_logger(ceph_subsys_osd);
+    auto &opref = *op;
+    return opref.template with_blocking_event<
+      PGMap::PGCreationBlockingEvent
+    >([&target_shard_services, &opref](auto &&trigger) {
+      return target_shard_services.get_or_create_pg(
+        std::move(trigger),
+        opref.get_pgid(),
+        std::move(opref.get_create_info())
+      );
+    }).safe_then([&logger, &target_shard_services, &opref](Ref<PG> pgref) {
+      logger.debug("{}: have_pg", opref);
+      return opref.with_pg(target_shard_services, pgref);
+    }).handle_error(
+      crimson::ct_error::ecanceled::handle([&logger, &opref](auto) {
+        logger.debug("{}: pg creation canceled, dropping", opref);
+        return seastar::now();
+      })
+    ).then([op=std::move(op)] {});
   }
 
   /// Runs opref on the appropriate core, waiting for pg as necessary
   template <typename T>
   seastar::future<> run_with_pg_maybe_wait(
-    typename T::IRef op
+    typename T::IRef op,
+    ShardServices &target_shard_services
   ) {
-    ceph_assert(op->use_count() == 1);
-    auto &logger = crimson::get_logger(ceph_subsys_osd);
     static_assert(!T::can_create());
-    logger.debug("{}: !can_create", *op);
-
-    return get_pg_to_shard_mapping().maybe_create_pg(
-      op->get_pgid()
-    ).then([this, op = std::move(op)](auto core) mutable {
-      return this->template with_remote_shard_state_and_op<T>(
-        core, std::move(op),
-        [](ShardServices &shard_services,
-           typename T::IRef op) {
-	auto &logger = crimson::get_logger(ceph_subsys_osd);
-	auto &opref = *op;
-	return opref.template with_blocking_event<
-	  PGMap::PGCreationBlockingEvent
-	  >([&shard_services, &opref](
-	      auto &&trigger) {
-	    return shard_services.wait_for_pg(
-	      std::move(trigger), opref.get_pgid());
-	  }).safe_then([&logger, &shard_services, &opref](Ref<PG> pgref) {
-	    logger.debug("{}: have_pg", opref);
-	    return opref.with_pg(shard_services, pgref);
-	  }).handle_error(
-	    crimson::ct_error::ecanceled::handle([&logger, &opref](auto) {
-	      logger.debug("{}: pg creation canceled, dropping", opref);
-	      return seastar::now();
-	    })
-	  ).then([op=std::move(op)] {});
-      });
-    });
+    auto &logger = crimson::get_logger(ceph_subsys_osd);
+    auto &opref = *op;
+    return opref.template with_blocking_event<
+      PGMap::PGCreationBlockingEvent
+    >([&target_shard_services, &opref](auto &&trigger) {
+      return target_shard_services.wait_for_pg(
+        std::move(trigger), opref.get_pgid());
+    }).safe_then([&logger, &target_shard_services, &opref](Ref<PG> pgref) {
+      logger.debug("{}: have_pg", opref);
+      return opref.with_pg(target_shard_services, pgref);
+    }).handle_error(
+      crimson::ct_error::ecanceled::handle([&logger, &opref](auto) {
+        logger.debug("{}: pg creation canceled, dropping", opref);
+        return seastar::now();
+      })
+    ).then([op=std::move(op)] {});
   }
 
   seastar::future<> load_pgs(crimson::os::FuturizedStore& store);
@@ -364,16 +338,23 @@ class PGShardManager {
       logger.debug("{}: got map {}, entering get_pg", opref, epoch);
       return opref.template enter_stage<>(
 	opref.get_connection_pipeline().get_pg);
-    }).then([this, &logger, &opref, op=std::move(op)]() mutable {
-      logger.debug("{}: in get_pg core {}", opref, seastar::this_shard_id());
-      logger.debug("{}: in get_pg", opref);
-      if constexpr (T::can_create()) {
-	logger.debug("{}: can_create", opref);
-	return run_with_pg_maybe_create<T>(std::move(op));
-      } else {
-	logger.debug("{}: !can_create", opref);
-	return run_with_pg_maybe_wait<T>(std::move(op));
-      }
+    }).then([this, &opref] {
+      return get_pg_to_shard_mapping().maybe_create_pg(opref.get_pgid());
+    }).then([this, &logger, op=std::move(op)](auto core) mutable {
+      logger.debug("{}: can_create={}, target-core={}",
+                   *op, T::can_create(), core);
+      return this->template with_remote_shard_state_and_op<T>(
+        core, std::move(op),
+        [this](ShardServices &target_shard_services,
+           typename T::IRef op) {
+        if constexpr (T::can_create()) {
+          return this->template run_with_pg_maybe_create<T>(
+              std::move(op), target_shard_services);
+        } else {
+          return this->template run_with_pg_maybe_wait<T>(
+              std::move(op), target_shard_services);
+        }
+      });
     });
     return std::make_pair(id, std::move(fut));
   }

From 4c7e55d84dd386ec2419cd139ed623919c346065 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Wed, 20 Sep 2023 14:09:22 +0800
Subject: [PATCH 0360/2492] crimson/osd/osd_operations: fix the usages of
 PipelineHandle::complete() and exit()

complete() should be called to leave the last phase in the normal path,
and exit() to be called in finally() to release the resources under all
circumstances.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 .../osd/osd_operations/background_recovery.cc |  4 ++
 .../osd/osd_operations/client_request.cc      | 17 ++++--
 .../osd_operations/internal_client_request.cc | 14 +++--
 .../osd/osd_operations/logmissing_request.cc  | 10 +++-
 .../logmissing_request_reply.cc               | 13 ++++-
 .../osd/osd_operations/peering_event.cc       | 10 ++--
 .../osd/osd_operations/pg_advance_map.cc      |  6 ++-
 .../osd/osd_operations/recovery_subrequest.cc | 10 +++-
 .../osd/osd_operations/replicated_request.cc  | 10 +++-
 .../osd/osd_operations/snaptrim_event.cc      | 54 ++++++++-----------
 .../osd/osd_operations/snaptrim_event.h       |  4 --
 src/crimson/osd/pg_shard_manager.h            |  9 +++-
 12 files changed, 104 insertions(+), 57 deletions(-)

diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc
index 953ec9595dae..74bd238c987b 100644
--- a/src/crimson/osd/osd_operations/background_recovery.cc
+++ b/src/crimson/osd/osd_operations/background_recovery.cc
@@ -196,7 +196,11 @@ BackfillRecovery::do_recovery()
     peering_pp(*pg).process
   ).then_interruptible([this] {
     pg->get_recovery_handler()->dispatch_backfill_event(std::move(evt));
+    return handle.complete();
+  }).then_interruptible([] {
     return seastar::make_ready_future<bool>(false);
+  }).finally([this] {
+    handle.exit();
   });
 }
 
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 9374fbde2cc0..f01f0c491f1a 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -143,6 +143,9 @@ seastar::future<> ClientRequest::with_pg_int(
 	} else {
 	  return process_op(ihref, pgref);
 	}
+      }).then_interruptible([this, this_instance_id, &ihref] {
+        logger().debug("{}.{}: complete", *this, this_instance_id);
+        return ihref.handle.complete();
       }).then_interruptible([this, this_instance_id, pgref] {
 	logger().debug("{}.{}: after process*", *this, this_instance_id);
 	pgref->client_request_orderer.remove_request(*this);
@@ -151,11 +154,15 @@ seastar::future<> ClientRequest::with_pg_int(
     }, [this, this_instance_id, pgref](std::exception_ptr eptr) {
       // TODO: better debug output
       logger().debug("{}.{}: interrupted {}", *this, this_instance_id, eptr);
-    }, pgref).finally(
-      [opref=std::move(opref), pgref=std::move(pgref),
-       instance_handle=std::move(instance_handle), &ihref] {
-      ihref.handle.exit();
-    });
+    },
+    pgref
+  ).finally(
+    [opref=std::move(opref), pgref,
+     instance_handle=std::move(instance_handle), &ihref,
+     this_instance_id, this] {
+    logger().debug("{}.{}: exit", *this, this_instance_id);
+    ihref.handle.exit();
+  });
 }
 
 seastar::future<> ClientRequest::with_pg(
diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc
index 87438d4a1468..790eb3f932d0 100644
--- a/src/crimson/osd/osd_operations/internal_client_request.cc
+++ b/src/crimson/osd/osd_operations/internal_client_request.cc
@@ -110,9 +110,14 @@ seastar::future<> InternalClientRequest::start()
               });
             });
           });
-        }).handle_error_interruptible(PG::load_obc_ertr::all_same_way([] {
-          return seastar::now();
-        })).then_interruptible([] {
+        }).si_then([this] {
+          logger().debug("{}: complete", *this);
+          return handle.complete();
+        }).handle_error_interruptible(
+          PG::load_obc_ertr::all_same_way([] {
+            return seastar::now();
+          })
+        ).then_interruptible([] {
           return seastar::stop_iteration::yes;
         });
       }, [this](std::exception_ptr eptr) {
@@ -124,6 +129,9 @@ seastar::future<> InternalClientRequest::start()
       }, pg);
     }).then([this] {
       track_event<CompletionEvent>();
+    }).finally([this] {
+      logger().debug("{}: exit", *this);
+      handle.exit();
     });
   });
 }
diff --git a/src/crimson/osd/osd_operations/logmissing_request.cc b/src/crimson/osd/osd_operations/logmissing_request.cc
index 739b46406500..ee83977cd8a2 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request.cc
@@ -72,8 +72,16 @@ seastar::future<> LogMissingRequest::with_pg(
       });
     }).then_interruptible([this, pg](auto) {
       return pg->do_update_log_missing(req, conn);
+    }).then_interruptible([this] {
+      logger().debug("{}: complete", *this);
+      return handle.complete();
     });
-  }, [ref](std::exception_ptr) { return seastar::now(); }, pg);
+  }, [](std::exception_ptr) {
+    return seastar::now();
+  }, pg).finally([this, ref] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
+  });
 }
 
 }
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.cc b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
index b4bf2938e05b..16e61ab4a985 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
@@ -61,8 +61,17 @@ seastar::future<> LogMissingRequestReply::with_pg(
 
   IRef ref = this;
   return interruptor::with_interruption([this, pg] {
-    return pg->do_update_log_missing_reply(std::move(req));
-  }, [ref](std::exception_ptr) { return seastar::now(); }, pg);
+    return pg->do_update_log_missing_reply(std::move(req)
+    ).then_interruptible([this] {
+      logger().debug("{}: complete", *this);
+      return handle.complete();
+    });
+  }, [](std::exception_ptr) {
+    return seastar::now();
+  }, pg).finally([this, ref] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
+  });
 }
 
 }
diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc
index ea4662bd01e0..0712147ab2b7 100644
--- a/src/crimson/osd/osd_operations/peering_event.cc
+++ b/src/crimson/osd/osd_operations/peering_event.cc
@@ -85,8 +85,9 @@ seastar::future<> PeeringEvent<T>::with_pg(
       return this->template enter_stage<interruptor>(peering_pp(*pg).process);
     }).then_interruptible([this, pg, &shard_services] {
       return pg->do_peering_event(evt, ctx
-      ).then_interruptible([this, pg, &shard_services] {
-	that()->get_handle().exit();
+      ).then_interruptible([this] {
+	return that()->get_handle().complete();
+      }).then_interruptible([this, pg, &shard_services] {
 	return complete_rctx(shard_services, pg);
       });
     }).then_interruptible([pg, &shard_services]()
@@ -100,7 +101,10 @@ seastar::future<> PeeringEvent<T>::with_pg(
     });
   }, [this](std::exception_ptr ep) {
     logger().debug("{}: interrupted with {}", *this, ep);
-  }, pg);
+  }, pg).finally([this] {
+    logger().debug("{}: exit", *this);
+    that()->get_handle().exit();
+  });
 }
 
 template <class T>
diff --git a/src/crimson/osd/osd_operations/pg_advance_map.cc b/src/crimson/osd/osd_operations/pg_advance_map.cc
index 3706af810557..ba63212fc7fc 100644
--- a/src/crimson/osd/osd_operations/pg_advance_map.cc
+++ b/src/crimson/osd/osd_operations/pg_advance_map.cc
@@ -122,8 +122,12 @@ seastar::future<> PGAdvanceMap::start()
 	  return shard_services.send_pg_temp();
 	});
     });
-  }).then([this, ref=std::move(ref)] {
+  }).then([this] {
     logger().debug("{}: complete", *this);
+    return handle.complete();
+  }).finally([this, ref=std::move(ref)] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
   });
 }
 
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.cc b/src/crimson/osd/osd_operations/recovery_subrequest.cc
index 68655b8da517..dd310d8d7274 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.cc
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.cc
@@ -30,11 +30,17 @@ seastar::future<> RecoverySubRequest::with_pg(
   track_event<StartEvent>();
   IRef opref = this;
   return interruptor::with_interruption([this, pgref] {
-    return pgref->get_recovery_backend()->handle_recovery_op(m, conn);
+    return pgref->get_recovery_backend()->handle_recovery_op(m, conn
+    ).then_interruptible([this] {
+      logger().debug("{}: complete", *this);
+      return handle.complete();
+    });
   }, [](std::exception_ptr) {
     return seastar::now();
-  }, pgref).finally([this, opref, pgref] {
+  }, pgref).finally([this, opref=std::move(opref), pgref] {
+    logger().debug("{}: exit", *this);
     track_event<CompletionEvent>();
+    handle.exit();
   });
 }
 
diff --git a/src/crimson/osd/osd_operations/replicated_request.cc b/src/crimson/osd/osd_operations/replicated_request.cc
index 09217575c8ff..7e16b2ebd06a 100644
--- a/src/crimson/osd/osd_operations/replicated_request.cc
+++ b/src/crimson/osd/osd_operations/replicated_request.cc
@@ -71,10 +71,16 @@ seastar::future<> RepRequest::with_pg(
       });
     }).then_interruptible([this, pg] (auto) {
       return pg->handle_rep_op(req);
+    }).then_interruptible([this] {
+      logger().debug("{}: complete", *this);
+      return handle.complete();
     });
-  }, [ref](std::exception_ptr) {
+  }, [](std::exception_ptr) {
     return seastar::now();
-  }, pg);
+  }, pg).finally([this, ref=std::move(ref)] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
+  });
 }
 
 }
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index e4a1b04df142..ffd43d736ad3 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -80,27 +80,15 @@ void SnapTrimEvent::dump_detail(Formatter *f) const
   f->close_section();
 }
 
-SnapTrimEvent::snap_trim_ertr::future<seastar::stop_iteration>
-SnapTrimEvent::start()
-{
-  logger().debug("{}: {}", *this, __func__);
-  return with_pg(
-    pg->get_shard_services(), pg
-  ).finally([ref=IRef{this}, this] {
-    logger().debug("{}: complete", *ref);
-    return handle.complete();
-  });
-}
-
 CommonPGPipeline& SnapTrimEvent::client_pp()
 {
   return pg->request_pg_pipeline;
 }
 
 SnapTrimEvent::snap_trim_ertr::future<seastar::stop_iteration>
-SnapTrimEvent::with_pg(
-  ShardServices &shard_services, Ref<PG> _pg)
+SnapTrimEvent::start()
 {
+  ShardServices &shard_services = pg->get_shard_services();
   return interruptor::with_interruption([&shard_services, this] {
     return enter_stage<interruptor>(
       client_pp().wait_for_active
@@ -176,7 +164,7 @@ SnapTrimEvent::with_pg(
           return subop_blocker.wait_completion();
         }).finally([this] {
 	  pg->snaptrim_mutex.unlock();
-	}).safe_then_interruptible([this] {
+	}).si_then([this] {
           if (!needs_pause) {
             return interruptor::now();
           }
@@ -193,17 +181,24 @@ SnapTrimEvent::with_pg(
             return seastar::sleep(
               std::chrono::milliseconds(std::lround(time_to_sleep * 1000)));
           });
-        }).safe_then_interruptible([this] {
+        }).si_then([this] {
           logger().debug("{}: all completed", *this);
           return snap_trim_iertr::make_ready_future<seastar::stop_iteration>(
             seastar::stop_iteration::no);
         });
+      }).si_then([this](auto stop) {
+        return handle.complete().then([stop] {
+          return snap_trim_iertr::make_ready_future<seastar::stop_iteration>(stop);
+        });
       });
     });
   }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
-  }, pg);
+  }, pg).finally([this] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
+  });
 }
 
 
@@ -212,18 +207,6 @@ CommonPGPipeline& SnapTrimObjSubEvent::client_pp()
   return pg->request_pg_pipeline;
 }
 
-SnapTrimObjSubEvent::remove_or_update_iertr::future<>
-SnapTrimObjSubEvent::start()
-{
-  logger().debug("{}: start", *this);
-  return with_pg(
-    pg->get_shard_services(), pg
-  ).finally([ref=IRef{this}, this] {
-    logger().debug("{}: complete", *ref);
-    return handle.complete();
-  });
-}
-
 SnapTrimObjSubEvent::remove_or_update_iertr::future<>
 SnapTrimObjSubEvent::remove_clone(
   ObjectContextRef obc,
@@ -466,7 +449,7 @@ SnapTrimObjSubEvent::remove_or_update(
                   *this, coid, old_snaps, new_snaps);
     ret = adjust_snaps(obc, head_obc, new_snaps, txn, log_entries);
   }
-  return std::move(ret).safe_then_interruptible(
+  return std::move(ret).si_then(
     [&txn, obc, num_objects_before_trim, log_entries=std::move(log_entries), head_obc=std::move(head_obc), this]() mutable {
     osd_op_p.at_version = pg->next_version();
 
@@ -484,7 +467,7 @@ SnapTrimObjSubEvent::remove_or_update(
       //  num_objects_before_trim - delta_stats.num_objects;
       //add_objects_trimmed_count(num_objects_trimmed);
     }
-  }).safe_then_interruptible(
+  }).si_then(
     [&txn, log_entries=std::move(log_entries)] () mutable {
     return remove_or_update_iertr::make_ready_future<remove_or_update_ret_t>(
       std::make_pair(std::move(txn), std::move(log_entries)));
@@ -493,8 +476,7 @@ SnapTrimObjSubEvent::remove_or_update(
 }
 
 SnapTrimObjSubEvent::remove_or_update_iertr::future<>
-SnapTrimObjSubEvent::with_pg(
-  ShardServices &shard_services, Ref<PG> _pg)
+SnapTrimObjSubEvent::start()
 {
   return enter_stage<interruptor>(
     client_pp().wait_for_active
@@ -544,10 +526,16 @@ SnapTrimObjSubEvent::with_pg(
           });
         });
       });
+    }).si_then([this] {
+      logger().debug("{}: completed", *this);
+      return handle.complete();
     }).handle_error_interruptible(
       remove_or_update_iertr::pass_further{},
       crimson::ct_error::assert_all{"unexpected error in SnapTrimObjSubEvent"}
     );
+  }).finally([this] {
+    logger().debug("{}: exit", *this);
+    handle.exit();
   });
 }
 
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index a3a970a04c7d..afb24952a045 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -53,8 +53,6 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
   void print(std::ostream &) const final;
   void dump_detail(ceph::Formatter* f) const final;
   snap_trim_ertr::future<seastar::stop_iteration> start();
-  snap_trim_ertr::future<seastar::stop_iteration> with_pg(
-    ShardServices &shard_services, Ref<PG> pg);
 
 private:
   CommonPGPipeline& client_pp();
@@ -140,8 +138,6 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
   void print(std::ostream &) const final;
   void dump_detail(ceph::Formatter* f) const final;
   remove_or_update_iertr::future<> start();
-  remove_or_update_iertr::future<> with_pg(
-    ShardServices &shard_services, Ref<PG> pg);
 
   CommonPGPipeline& client_pp();
 
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index 71a9cf1a9c80..e080dc43e4ad 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -340,7 +340,14 @@ class PGShardManager {
 	opref.get_connection_pipeline().get_pg);
     }).then([this, &opref] {
       return get_pg_to_shard_mapping().maybe_create_pg(opref.get_pgid());
-    }).then([this, &logger, op=std::move(op)](auto core) mutable {
+    }).then_wrapped([this, &logger, op=std::move(op)](auto fut) mutable {
+      if (unlikely(fut.failed())) {
+        logger.error("{}: failed before with_pg", *op);
+        op->get_handle().exit();
+        return seastar::make_exception_future<>(fut.get_exception());
+      }
+
+      auto core = fut.get();
       logger.debug("{}: can_create={}, target-core={}",
                    *op, T::can_create(), core);
       return this->template with_remote_shard_state_and_op<T>(

From 8bfd699de5b4fb4cb1efb76030436556652e6845 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 25 Sep 2023 10:22:49 +0800
Subject: [PATCH 0361/2492] crimson/osd/osd_connection_priv: make
 get_osd_priv() inline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

To address warning:
/root/ceph/src/crimson/osd/osd_connection_priv.h:89:27: warning:
‘crimson::osd::OSDConnectionPriv&
crimson::osd::get_osd_priv(crimson::net::Connection*)’ defined but not
used [-Wunused-function]
   89 | static OSDConnectionPriv &get_osd_priv(crimson::net::Connection
      *conn) { |                           ^~~~~~~~~~~~

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_connection_priv.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_connection_priv.h b/src/crimson/osd/osd_connection_priv.h
index 69edf94b88fe..99f394b1e83f 100644
--- a/src/crimson/osd/osd_connection_priv.h
+++ b/src/crimson/osd/osd_connection_priv.h
@@ -17,7 +17,7 @@ struct OSDConnectionPriv : public crimson::net::Connection::user_private_t {
   ConnectionPipeline replicated_request_conn_pipeline;
 };
 
-static OSDConnectionPriv &get_osd_priv(crimson::net::Connection *conn) {
+static inline OSDConnectionPriv &get_osd_priv(crimson::net::Connection *conn) {
   if (!conn->has_user_private()) {
     conn->set_user_private(std::make_unique<OSDConnectionPriv>());
   }

From 76cd51a2c1d9673d4a157a97f3f9be0d66355a26 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 25 Sep 2023 10:25:57 +0800
Subject: [PATCH 0362/2492] crimson/.../peering_event: drop the unused
 OSDPipeline

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_operations/peering_event.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/crimson/osd/osd_operations/peering_event.h b/src/crimson/osd/osd_operations/peering_event.h
index e94caead1992..a780a26768ef 100644
--- a/src/crimson/osd/osd_operations/peering_event.h
+++ b/src/crimson/osd/osd_operations/peering_event.h
@@ -120,14 +120,6 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
   ) override;
 
 public:
-  class OSDPipeline {
-    struct AwaitActive : OrderedExclusivePhaseT<AwaitActive> {
-      static constexpr auto type_name =
-	"PeeringRequest::OSDPipeline::await_active";
-    } await_active;
-    friend class RemotePeeringEvent;
-  };
-
   template <typename... Args>
   RemotePeeringEvent(crimson::net::ConnectionRef conn, Args&&... args) :
     PeeringEvent(std::forward<Args>(args)...),
@@ -144,7 +136,6 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
     PGPeeringPipeline::AwaitMap::BlockingEvent,
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent,
     PGPeeringPipeline::Process::BlockingEvent,
-    OSDPipeline::AwaitActive::BlockingEvent,
     CompletionEvent
   > tracking_events;
 

From c3c29c50035d3d5d0fd67a94c7cf80578aec083e Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 25 Sep 2023 11:08:22 +0800
Subject: [PATCH 0363/2492] crimson/osd: split ConnectionPipeline::get_pg into
 2 phases

Split the cross-core phase into 2 independent core-local phases, and
preserve the ordering using sequential ID instead.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_connection_priv.h         | 71 +++++++++++++++++
 src/crimson/osd/osd_operation.h               | 13 +++-
 .../osd/osd_operation_external_tracking.h     | 24 ++++--
 .../osd/osd_operations/client_request.cc      |  6 ++
 .../osd/osd_operations/client_request.h       | 11 ++-
 .../osd/osd_operations/logmissing_request.cc  |  6 ++
 .../osd/osd_operations/logmissing_request.h   | 11 ++-
 .../logmissing_request_reply.cc               |  6 ++
 .../osd_operations/logmissing_request_reply.h | 11 ++-
 .../osd/osd_operations/peering_event.cc       |  6 ++
 .../osd/osd_operations/peering_event.h        | 11 ++-
 .../osd/osd_operations/recovery_subrequest.cc |  6 ++
 .../osd/osd_operations/recovery_subrequest.h  | 11 ++-
 .../osd/osd_operations/replicated_request.cc  |  6 ++
 .../osd/osd_operations/replicated_request.h   | 11 ++-
 src/crimson/osd/pg_shard_manager.h            | 78 ++++++++++++++-----
 src/crimson/osd/shard_services.h              | 16 ++++
 17 files changed, 270 insertions(+), 34 deletions(-)

diff --git a/src/crimson/osd/osd_connection_priv.h b/src/crimson/osd/osd_connection_priv.h
index 99f394b1e83f..2d2a459017bb 100644
--- a/src/crimson/osd/osd_connection_priv.h
+++ b/src/crimson/osd/osd_connection_priv.h
@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include <seastar/core/smp.hh>
+
 #include "crimson/net/Connection.h"
 #include "crimson/osd/osd_operation.h"
 #include "crimson/osd/osd_operations/client_request.h"
@@ -11,10 +13,79 @@
 
 namespace crimson::osd {
 
+/**
+ * crosscore_ordering_t
+ *
+ * To preserve the event order from 1 source to n target cores.
+ */
+class crosscore_ordering_t {
+public:
+  using seq_t = uint64_t;
+
+  crosscore_ordering_t()
+    : out_seqs(seastar::smp::count, 0),
+      in_controls(seastar::smp::count) {}
+
+  ~crosscore_ordering_t() = default;
+
+  // Called by the original core to get the ordering sequence
+  seq_t prepare_submit(core_id_t target_core) {
+    auto &out_seq = out_seqs[target_core];
+    ++out_seq;
+    return out_seq;
+  }
+
+  /*
+   * Called by the target core to preserve the ordering
+   */
+
+  seq_t get_in_seq() const {
+    auto core = seastar::this_shard_id();
+    return in_controls[core].seq;
+  }
+
+  bool proceed_or_wait(seq_t seq) {
+    auto core = seastar::this_shard_id();
+    auto &in_control = in_controls[core];
+    if (seq == in_control.seq + 1) {
+      ++in_control.seq;
+      if (unlikely(in_control.pr_wait.has_value())) {
+        in_control.pr_wait->set_value();
+        in_control.pr_wait = std::nullopt;
+      }
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  seastar::future<> wait(seq_t seq) {
+    auto core = seastar::this_shard_id();
+    auto &in_control = in_controls[core];
+    assert(seq != in_control.seq + 1);
+    if (!in_control.pr_wait.has_value()) {
+      in_control.pr_wait = seastar::shared_promise<>();
+    }
+    return in_control.pr_wait->get_shared_future();
+  }
+
+private:
+  struct in_control_t {
+    seq_t seq = 0;
+    std::optional<seastar::shared_promise<>> pr_wait;
+  };
+
+  // source-side
+  std::vector<seq_t> out_seqs;
+  // target-side
+  std::vector<in_control_t> in_controls;
+};
+
 struct OSDConnectionPriv : public crimson::net::Connection::user_private_t {
   ConnectionPipeline client_request_conn_pipeline;
   ConnectionPipeline peering_request_conn_pipeline;
   ConnectionPipeline replicated_request_conn_pipeline;
+  crosscore_ordering_t crosscore_ordering;
 };
 
 static inline OSDConnectionPriv &get_osd_priv(crimson::net::Connection *conn) {
diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h
index 8ef44ee9e789..7174143fe01e 100644
--- a/src/crimson/osd/osd_operation.h
+++ b/src/crimson/osd/osd_operation.h
@@ -27,10 +27,17 @@ struct ConnectionPipeline {
       "ConnectionPipeline::await_map";
   } await_map;
 
-  struct GetPG : OrderedExclusivePhaseT<GetPG> {
+  struct GetPGMapping : OrderedExclusivePhaseT<GetPGMapping> {
     static constexpr auto type_name =
-      "ConnectionPipeline::get_pg";
-  } get_pg;
+      "ConnectionPipeline::get_pg_mapping";
+  } get_pg_mapping;
+};
+
+struct PerShardPipeline {
+  struct CreateOrWaitPG : OrderedExclusivePhaseT<CreateOrWaitPG> {
+    static constexpr auto type_name =
+      "PerShardPipeline::create_or_wait_pg";
+  } create_or_wait_pg;
 };
 
 enum class OperationTypeCode {
diff --git a/src/crimson/osd/osd_operation_external_tracking.h b/src/crimson/osd/osd_operation_external_tracking.h
index 4b6dbf4b7100..d5e2ed453284 100644
--- a/src/crimson/osd/osd_operation_external_tracking.h
+++ b/src/crimson/osd/osd_operation_external_tracking.h
@@ -22,7 +22,8 @@ struct LttngBackend
   : ClientRequest::StartEvent::Backend,
     ConnectionPipeline::AwaitActive::BlockingEvent::Backend,
     ConnectionPipeline::AwaitMap::BlockingEvent::Backend,
-    ConnectionPipeline::GetPG::BlockingEvent::Backend,
+    ConnectionPipeline::GetPGMapping::BlockingEvent::Backend,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent::Backend,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent::Backend,
     PGMap::PGCreationBlockingEvent::Backend,
     ClientRequest::PGPipeline::AwaitMap::BlockingEvent::Backend,
@@ -55,9 +56,14 @@ struct LttngBackend
               const OSD_OSDMapGate::OSDMapBlocker&) override {
   }
 
-  void handle(ConnectionPipeline::GetPG::BlockingEvent& ev,
+  void handle(ConnectionPipeline::GetPGMapping::BlockingEvent& ev,
               const Operation& op,
-              const ConnectionPipeline::GetPG& blocker) override {
+              const ConnectionPipeline::GetPGMapping& blocker) override {
+  }
+
+  void handle(PerShardPipeline::CreateOrWaitPG::BlockingEvent& ev,
+              const Operation& op,
+              const PerShardPipeline::CreateOrWaitPG& blocker) override {
   }
 
   void handle(PGMap::PGCreationBlockingEvent&,
@@ -122,7 +128,8 @@ struct HistoricBackend
   : ClientRequest::StartEvent::Backend,
     ConnectionPipeline::AwaitActive::BlockingEvent::Backend,
     ConnectionPipeline::AwaitMap::BlockingEvent::Backend,
-    ConnectionPipeline::GetPG::BlockingEvent::Backend,
+    ConnectionPipeline::GetPGMapping::BlockingEvent::Backend,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent::Backend,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent::Backend,
     PGMap::PGCreationBlockingEvent::Backend,
     ClientRequest::PGPipeline::AwaitMap::BlockingEvent::Backend,
@@ -155,9 +162,14 @@ struct HistoricBackend
               const OSD_OSDMapGate::OSDMapBlocker&) override {
   }
 
-  void handle(ConnectionPipeline::GetPG::BlockingEvent& ev,
+  void handle(ConnectionPipeline::GetPGMapping::BlockingEvent& ev,
+              const Operation& op,
+              const ConnectionPipeline::GetPGMapping& blocker) override {
+  }
+
+  void handle(PerShardPipeline::CreateOrWaitPG::BlockingEvent& ev,
               const Operation& op,
-              const ConnectionPipeline::GetPG& blocker) override {
+              const PerShardPipeline::CreateOrWaitPG& blocker) override {
   }
 
   void handle(PGMap::PGCreationBlockingEvent&,
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index f01f0c491f1a..287072642953 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -81,6 +81,12 @@ ConnectionPipeline &ClientRequest::get_connection_pipeline()
   return get_osd_priv(conn.get()).client_request_conn_pipeline;
 }
 
+PerShardPipeline &ClientRequest::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_client_request_pipeline();
+}
+
 ClientRequest::PGPipeline &ClientRequest::client_pp(PG &pg)
 {
   return pg.request_pg_pipeline;
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index b2dce1e873e1..d534fd6ac4fa 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -81,7 +81,8 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
     CompletionEvent
   > tracking_events;
@@ -206,6 +207,14 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
   epoch_t get_epoch() const { return m->get_min_epoch(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
diff --git a/src/crimson/osd/osd_operations/logmissing_request.cc b/src/crimson/osd/osd_operations/logmissing_request.cc
index ee83977cd8a2..7c8e1d7e499a 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request.cc
@@ -49,6 +49,12 @@ ConnectionPipeline &LogMissingRequest::get_connection_pipeline()
   return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
 }
 
+PerShardPipeline &LogMissingRequest::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_replicated_request_pipeline();
+}
+
 ClientRequest::PGPipeline &LogMissingRequest::client_pp(PG &pg)
 {
   return pg.request_pg_pipeline;
diff --git a/src/crimson/osd/osd_operations/logmissing_request.h b/src/crimson/osd/osd_operations/logmissing_request.h
index 71d0816fd201..5b01fee17b86 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.h
+++ b/src/crimson/osd/osd_operations/logmissing_request.h
@@ -38,6 +38,14 @@ class LogMissingRequest final : public PhasedOperationT<LogMissingRequest> {
   epoch_t get_epoch() const { return req->get_min_epoch(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
@@ -58,7 +66,8 @@ class LogMissingRequest final : public PhasedOperationT<LogMissingRequest> {
     StartEvent,
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     ClientRequest::PGPipeline::AwaitMap::BlockingEvent,
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.cc b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
index 16e61ab4a985..5cfe5b215307 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
@@ -49,6 +49,12 @@ ConnectionPipeline &LogMissingRequestReply::get_connection_pipeline()
   return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
 }
 
+PerShardPipeline &LogMissingRequestReply::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_replicated_request_pipeline();
+}
+
 ClientRequest::PGPipeline &LogMissingRequestReply::client_pp(PG &pg)
 {
   return pg.request_pg_pipeline;
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.h b/src/crimson/osd/osd_operations/logmissing_request_reply.h
index c89131fec1d7..b01cae15421d 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.h
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.h
@@ -38,6 +38,14 @@ class LogMissingRequestReply final : public PhasedOperationT<LogMissingRequestRe
   epoch_t get_epoch() const { return req->get_min_epoch(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
@@ -58,7 +66,8 @@ class LogMissingRequestReply final : public PhasedOperationT<LogMissingRequestRe
     StartEvent,
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent
   > tracking_events;
diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc
index 0712147ab2b7..9139e337f80a 100644
--- a/src/crimson/osd/osd_operations/peering_event.cc
+++ b/src/crimson/osd/osd_operations/peering_event.cc
@@ -128,6 +128,12 @@ ConnectionPipeline &RemotePeeringEvent::get_connection_pipeline()
   return get_osd_priv(conn.get()).peering_request_conn_pipeline;
 }
 
+PerShardPipeline &RemotePeeringEvent::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_peering_request_pipeline();
+}
+
 void RemotePeeringEvent::on_pg_absent(ShardServices &shard_services)
 {
   if (auto& e = get_event().get_event();
diff --git a/src/crimson/osd/osd_operations/peering_event.h b/src/crimson/osd/osd_operations/peering_event.h
index a780a26768ef..6bbfe6c91174 100644
--- a/src/crimson/osd/osd_operations/peering_event.h
+++ b/src/crimson/osd/osd_operations/peering_event.h
@@ -131,7 +131,8 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
     PGPeeringPipeline::AwaitMap::BlockingEvent,
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent,
@@ -148,6 +149,14 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
   epoch_t get_epoch() const { return evt.get_epoch_sent(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.cc b/src/crimson/osd/osd_operations/recovery_subrequest.cc
index dd310d8d7274..2e939880cbea 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.cc
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.cc
@@ -49,4 +49,10 @@ ConnectionPipeline &RecoverySubRequest::get_connection_pipeline()
   return get_osd_priv(conn.get()).peering_request_conn_pipeline;
 }
 
+PerShardPipeline &RecoverySubRequest::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_peering_request_pipeline();
+}
+
 }
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.h b/src/crimson/osd/osd_operations/recovery_subrequest.h
index 07c7c95b5e0f..31e6045cb0ee 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.h
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.h
@@ -41,6 +41,14 @@ class RecoverySubRequest final : public PhasedOperationT<RecoverySubRequest> {
   epoch_t get_epoch() const { return m->get_min_epoch(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
@@ -61,7 +69,8 @@ class RecoverySubRequest final : public PhasedOperationT<RecoverySubRequest> {
     StartEvent,
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
     OSD_OSDMapGate::OSDMapBlocker::BlockingEvent,
     CompletionEvent
diff --git a/src/crimson/osd/osd_operations/replicated_request.cc b/src/crimson/osd/osd_operations/replicated_request.cc
index 7e16b2ebd06a..7b8592b1e02d 100644
--- a/src/crimson/osd/osd_operations/replicated_request.cc
+++ b/src/crimson/osd/osd_operations/replicated_request.cc
@@ -49,6 +49,12 @@ ConnectionPipeline &RepRequest::get_connection_pipeline()
   return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
 }
 
+PerShardPipeline &RepRequest::get_pershard_pipeline(
+    ShardServices &shard_services)
+{
+  return shard_services.get_replicated_request_pipeline();
+}
+
 ClientRequest::PGPipeline &RepRequest::client_pp(PG &pg)
 {
   return pg.request_pg_pipeline;
diff --git a/src/crimson/osd/osd_operations/replicated_request.h b/src/crimson/osd/osd_operations/replicated_request.h
index c742888d9390..32cf271788ba 100644
--- a/src/crimson/osd/osd_operations/replicated_request.h
+++ b/src/crimson/osd/osd_operations/replicated_request.h
@@ -38,6 +38,14 @@ class RepRequest final : public PhasedOperationT<RepRequest> {
   epoch_t get_epoch() const { return req->get_min_epoch(); }
 
   ConnectionPipeline &get_connection_pipeline();
+
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
   seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
     assert(conn);
     return conn.get_foreign(
@@ -58,7 +66,8 @@ class RepRequest final : public PhasedOperationT<RepRequest> {
     StartEvent,
     ConnectionPipeline::AwaitActive::BlockingEvent,
     ConnectionPipeline::AwaitMap::BlockingEvent,
-    ConnectionPipeline::GetPG::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
     ClientRequest::PGPipeline::AwaitMap::BlockingEvent,
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent,
     PGMap::PGCreationBlockingEvent,
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index e080dc43e4ad..cf13cb52bbf7 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -7,6 +7,7 @@
 #include <seastar/core/shared_future.hh>
 #include <seastar/core/sharded.hh>
 
+#include "crimson/osd/osd_connection_priv.h"
 #include "crimson/osd/shard_services.h"
 #include "crimson/osd/pg_map.h"
 
@@ -148,6 +149,31 @@ class PGShardManager {
       });
   }
 
+  template <typename T, typename F>
+  auto process_ordered_op_remotely(
+      crosscore_ordering_t::seq_t cc_seq,
+      ShardServices &target_shard_services,
+      typename T::IRef &&op,
+      F &&f) {
+    auto &crosscore_ordering = get_osd_priv(&op->get_connection()).crosscore_ordering;
+    if (crosscore_ordering.proceed_or_wait(cc_seq)) {
+      return std::invoke(
+        std::move(f),
+        target_shard_services,
+        std::move(op));
+    } else {
+      auto &logger = crimson::get_logger(ceph_subsys_osd);
+      logger.debug("{} got {} at the remote pg, wait at {}",
+                   *op, cc_seq, crosscore_ordering.get_in_seq());
+      return crosscore_ordering.wait(cc_seq
+      ).then([this, cc_seq, &target_shard_services,
+              op=std::move(op), f=std::move(f)]() mutable {
+        return this->template process_ordered_op_remotely<T>(
+            cc_seq, target_shard_services, std::move(op), std::move(f));
+      });
+    }
+  }
+
   template <typename T, typename F>
   auto with_remote_shard_state_and_op(
       core_id_t core,
@@ -161,21 +187,28 @@ class PGShardManager {
         target_shard_services,
         std::move(op));
     }
+    // Note: the ordering in only preserved until f is invoked.
     auto &opref = *op;
-    get_local_state().registry.remove_from_registry(opref);
-    return opref.prepare_remote_submission(
-    ).then([op=std::move(op), f=std::move(f), this, core
-           ](auto f_conn) mutable {
+    auto &crosscore_ordering = get_osd_priv(&opref.get_connection()).crosscore_ordering;
+    auto cc_seq = crosscore_ordering.prepare_submit(core);
+    auto &logger = crimson::get_logger(ceph_subsys_osd);
+    logger.debug("{}: send {} to the remote pg core {}",
+                 opref, cc_seq, core);
+    return opref.get_handle().complete(
+    ).then([&opref, this] {
+      get_local_state().registry.remove_from_registry(opref);
+      return opref.prepare_remote_submission();
+    }).then([op=std::move(op), f=std::move(f), this, core, cc_seq
+            ](auto f_conn) mutable {
       return shard_services.invoke_on(
         core,
-        [f=std::move(f), op=std::move(op), f_conn=std::move(f_conn)
+        [this, cc_seq,
+         f=std::move(f), op=std::move(op), f_conn=std::move(f_conn)
         ](auto &target_shard_services) mutable {
         op->finish_remote_submission(std::move(f_conn));
         target_shard_services.local_state.registry.add_to_registry(*op);
-        return std::invoke(
-          std::move(f),
-          target_shard_services,
-          std::move(op));
+        return this->template process_ordered_op_remotely<T>(
+            cc_seq, target_shard_services, std::move(op), std::move(f));
       });
     });
   }
@@ -335,9 +368,9 @@ class PGShardManager {
 	      &get_shard_services());
       });
     }).then([&logger, &opref](auto epoch) {
-      logger.debug("{}: got map {}, entering get_pg", opref, epoch);
+      logger.debug("{}: got map {}, entering get_pg_mapping", opref, epoch);
       return opref.template enter_stage<>(
-	opref.get_connection_pipeline().get_pg);
+	opref.get_connection_pipeline().get_pg_mapping);
     }).then([this, &opref] {
       return get_pg_to_shard_mapping().maybe_create_pg(opref.get_pgid());
     }).then_wrapped([this, &logger, op=std::move(op)](auto fut) mutable {
@@ -353,14 +386,21 @@ class PGShardManager {
       return this->template with_remote_shard_state_and_op<T>(
         core, std::move(op),
         [this](ShardServices &target_shard_services,
-           typename T::IRef op) {
-        if constexpr (T::can_create()) {
-          return this->template run_with_pg_maybe_create<T>(
-              std::move(op), target_shard_services);
-        } else {
-          return this->template run_with_pg_maybe_wait<T>(
-              std::move(op), target_shard_services);
-        }
+               typename T::IRef op) {
+        auto &opref = *op;
+        auto &logger = crimson::get_logger(ceph_subsys_osd);
+        logger.debug("{}: entering create_or_wait_pg", opref);
+        return opref.template enter_stage<>(
+          opref.get_pershard_pipeline(target_shard_services).create_or_wait_pg
+        ).then([this, &target_shard_services, op=std::move(op)]() mutable {
+          if constexpr (T::can_create()) {
+            return this->template run_with_pg_maybe_create<T>(
+                std::move(op), target_shard_services);
+          } else {
+            return this->template run_with_pg_maybe_wait<T>(
+                std::move(op), target_shard_services);
+          }
+        });
       });
     });
     return std::make_pair(id, std::move(fut));
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 8786ec9626fd..d71513a6645e 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -70,6 +70,10 @@ class PerShardState {
   OSDState &osd_state;
   OSD_OSDMapGate osdmap_gate;
 
+  PerShardPipeline client_request_pipeline;
+  PerShardPipeline peering_request_pipeline;
+  PerShardPipeline replicated_request_pipeline;
+
   PerfCounters *perf = nullptr;
   PerfCounters *recoverystate_perf = nullptr;
 
@@ -453,6 +457,18 @@ class ShardServices : public OSDMapService {
     return dispatch_context({}, std::move(ctx));
   }
 
+  PerShardPipeline &get_client_request_pipeline() {
+    return local_state.client_request_pipeline;
+  }
+
+  PerShardPipeline &get_peering_request_pipeline() {
+    return local_state.peering_request_pipeline;
+  }
+
+  PerShardPipeline &get_replicated_request_pipeline() {
+    return local_state.replicated_request_pipeline;
+  }
+
   /// Return per-core tid
   ceph_tid_t get_tid() { return local_state.get_tid(); }
 

From 510cd5dc41d7d890941ad5eeca69935a3364a948 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 25 Sep 2023 11:34:47 +0800
Subject: [PATCH 0364/2492] crimson/common/operation: drop cross-core supports
 in ExitBarrier

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/operation.h | 54 ++++++++++++++++------------------
 1 file changed, 25 insertions(+), 29 deletions(-)

diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h
index 3a65a793a9e1..32e617ed5aee 100644
--- a/src/crimson/common/operation.h
+++ b/src/crimson/common/operation.h
@@ -489,9 +489,10 @@ class PipelineExitBarrierI {
 
 template <class T>
 class PipelineStageIT : public BlockerT<T> {
-  const core_id_t core = seastar::this_shard_id();
 public:
-  core_id_t get_core() const { return core; }
+#ifndef NDEBUG
+  const core_id_t core = seastar::this_shard_id();
+#endif
 
   template <class... Args>
   decltype(auto) enter(Args&&... args) {
@@ -523,7 +524,7 @@ class PipelineHandle {
   template <typename OpT, typename T>
   seastar::future<>
   enter(T &stage, typename T::BlockingEvent::template Trigger<OpT>&& t) {
-    ceph_assert(stage.get_core() == seastar::this_shard_id());
+    assert(stage.core == seastar::this_shard_id());
     auto wait_fut = wait_barrier();
     if (wait_fut.has_value()) {
       return wait_fut.value().then([this, &stage, t=std::move(t)] () mutable {
@@ -585,8 +586,8 @@ class OrderedExclusivePhaseT : public PipelineStageIT<T> {
     OrderedExclusivePhaseT *phase;
     Operation::id_t op_id;
   public:
-    ExitBarrier(OrderedExclusivePhaseT *phase, Operation::id_t id)
-      : phase(phase), op_id(id) {}
+    ExitBarrier(OrderedExclusivePhaseT &phase, Operation::id_t id)
+      : phase(&phase), op_id(id) {}
 
     std::optional<seastar::future<>> wait() final {
       return std::nullopt;
@@ -594,14 +595,9 @@ class OrderedExclusivePhaseT : public PipelineStageIT<T> {
 
     void exit() final {
       if (phase) {
-	auto *p = phase;
-	auto id = op_id;
-	phase = nullptr;
-	std::ignore = seastar::smp::submit_to(
-	  p->get_core(),
-	  [p, id] {
-	    p->exit(id);
-	  });
+        assert(phase->core == seastar::this_shard_id());
+        phase->exit(op_id);
+        phase = nullptr;
       }
     }
 
@@ -623,7 +619,7 @@ class OrderedExclusivePhaseT : public PipelineStageIT<T> {
       ceph_assert_always(waiting > 0);
       --waiting;
       set_held_by(op_id);
-      return PipelineExitBarrierI::Ref(new ExitBarrier{this, op_id});
+      return PipelineExitBarrierI::Ref(new ExitBarrier{*this, op_id});
     });
   }
 
@@ -683,9 +679,9 @@ class OrderedConcurrentPhaseT : public PipelineStageIT<T> {
     TriggerT trigger;
   public:
     ExitBarrier(
-      OrderedConcurrentPhaseT *phase,
+      OrderedConcurrentPhaseT &phase,
       seastar::future<> &&barrier,
-      TriggerT& trigger) : phase(phase), barrier(std::move(barrier)), trigger(trigger) {}
+      TriggerT& trigger) : phase(&phase), barrier(std::move(barrier)), trigger(trigger) {}
 
     std::optional<seastar::future<>> wait() final {
       assert(phase);
@@ -697,18 +693,18 @@ class OrderedConcurrentPhaseT : public PipelineStageIT<T> {
 
     void exit() final {
       if (barrier) {
-	static_cast<void>(
-	  std::move(*barrier).then([phase=this->phase] { phase->mutex.unlock(); }));
-	barrier = std::nullopt;
-	phase = nullptr;
-      }
-      if (phase) {
-	std::ignore = seastar::smp::submit_to(
-	  phase->get_core(),
-	  [this] {
-	    phase->mutex.unlock();
-	    phase = nullptr;
-	  });
+        assert(phase);
+        assert(phase->core == seastar::this_shard_id());
+        std::ignore = std::move(*barrier
+        ).then([phase=this->phase] {
+          phase->mutex.unlock();
+        });
+        barrier = std::nullopt;
+        phase = nullptr;
+      } else if (phase) {
+        assert(phase->core == seastar::this_shard_id());
+        phase->mutex.unlock();
+        phase = nullptr;
       }
     }
 
@@ -721,7 +717,7 @@ class OrderedConcurrentPhaseT : public PipelineStageIT<T> {
   template <class TriggerT>
   seastar::future<PipelineExitBarrierI::Ref> enter(TriggerT& t) {
     return seastar::make_ready_future<PipelineExitBarrierI::Ref>(
-      new ExitBarrier<TriggerT>{this, mutex.lock(), t});
+      new ExitBarrier<TriggerT>{*this, mutex.lock(), t});
   }
 
 private:

From c20e0a4f4895cbc25e4eaf179d9455824d538a0c Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 13 Oct 2023 16:05:52 +0800
Subject: [PATCH 0365/2492] crimson/osd: drop thread_unsafe_counter for the
 operations

They are supposed to be used cross-core.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/operation.h                  | 3 +--
 src/crimson/osd/osd_operations/client_request.h | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h
index 32e617ed5aee..bd3d79eec42b 100644
--- a/src/crimson/common/operation.h
+++ b/src/crimson/common/operation.h
@@ -317,8 +317,7 @@ struct AggregateBlockingEvent {
  * an interface for registering ops in flight and dumping
  * diagnostic information.
  */
-class Operation : public boost::intrusive_ref_counter<
-  Operation, boost::thread_unsafe_counter> {
+class Operation : public boost::intrusive_ref_counter<Operation> {
  public:
   using id_t = uint64_t;
   static constexpr id_t NULL_ID = std::numeric_limits<uint64_t>::max();
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index d534fd6ac4fa..3c8c146a4f2f 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -87,8 +87,7 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
     CompletionEvent
   > tracking_events;
 
-  class instance_handle_t : public boost::intrusive_ref_counter<
-    instance_handle_t, boost::thread_unsafe_counter> {
+  class instance_handle_t : public boost::intrusive_ref_counter<instance_handle_t> {
   public:
     // intrusive_ptr because seastar::lw_shared_ptr includes a cpu debug check
     // that we will fail since the core on which we allocate the request may not

From 0fd2e75baa6dce57e26b88db395ab0e804808a09 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 16 Oct 2023 14:41:55 +0800
Subject: [PATCH 0366/2492] 
 crimson/osd/osd_operations/logmissing_request_reply: do not move away req

So that print can always deal with a valid req.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_operations/logmissing_request_reply.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.cc b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
index 5cfe5b215307..b2d447d194af 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
@@ -67,7 +67,7 @@ seastar::future<> LogMissingRequestReply::with_pg(
 
   IRef ref = this;
   return interruptor::with_interruption([this, pg] {
-    return pg->do_update_log_missing_reply(std::move(req)
+    return pg->do_update_log_missing_reply(req
     ).then_interruptible([this] {
       logger().debug("{}: complete", *this);
       return handle.complete();

From 9fcb98411bfeb9147f61bef4424a78d98c63cadc Mon Sep 17 00:00:00 2001
From: Jaanus Torp <jaanus@saun.ee>
Date: Wed, 1 Nov 2023 12:23:47 +0200
Subject: [PATCH 0367/2492] doc/cephfs: edit mount-using-fuse.rst

Fixes missing dash that lead to unnecessary debugging.

Signed-off-by: Jaanus Torp <jaanus@saun.ee>
---
 doc/cephfs/mount-using-fuse.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephfs/mount-using-fuse.rst b/doc/cephfs/mount-using-fuse.rst
index bd098dc91de3..f3ac054c9be1 100644
--- a/doc/cephfs/mount-using-fuse.rst
+++ b/doc/cephfs/mount-using-fuse.rst
@@ -28,7 +28,7 @@ To FUSE-mount the Ceph file system, use the ``ceph-fuse`` command::
     mkdir /mnt/mycephfs
     ceph-fuse --id foo /mnt/mycephfs
 
-Option ``-id`` passes the name of the CephX user whose keyring we intend to
+Option ``--id`` passes the name of the CephX user whose keyring we intend to
 use for mounting CephFS. In the above command, it's ``foo``. You can also use
 ``-n`` instead, although ``--id`` is evidently easier::
 

From 3fd8a03887354a0c5a11d1eaae386b09242c14a9 Mon Sep 17 00:00:00 2001
From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Mon, 17 Oct 2022 13:35:22 +0530
Subject: [PATCH 0368/2492] qa/workunits/rbd: merge journal and snapshot test
 scripts

The idea is to avoid the maintenance of duplicate code in both the journal
and snapshot test scripts.

Usage:
   RBD_MIRROR_MODE=journal rbd_mirror.sh

Use environment variable RBD_MIRROR_MODE to set the mode
Available modes: snapshot | journal

Fixes: https://tracker.ceph.com/issues/54312
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---
 .../rbd-mirror-journal-workunit.yaml          |   5 +-
 ...rror-snapshot-workunit-exclusive-lock.yaml |   5 +-
 ...bd-mirror-snapshot-workunit-fast-diff.yaml |   5 +-
 ...d-mirror-snapshot-workunit-journaling.yaml |   5 +-
 .../rbd-mirror-snapshot-workunit-minimum.yaml |   5 +-
 .../rbd-mirror-workunit-config-key.yaml       |   5 +-
 ...or-workunit-min-compat-client-octopus.yaml |   5 +-
 .../rbd-mirror-workunit-policy-none.yaml      |   5 +-
 .../rbd-mirror-workunit-policy-simple.yaml    |   5 +-
 .../{rbd_mirror_journal.sh => rbd_mirror.sh}  | 351 +++++++-----
 qa/workunits/rbd/rbd_mirror_helpers.sh        |  68 ++-
 qa/workunits/rbd/rbd_mirror_snapshot.sh       | 517 ------------------
 12 files changed, 293 insertions(+), 688 deletions(-)
 rename qa/workunits/rbd/{rbd_mirror_journal.sh => rbd_mirror.sh} (69%)
 delete mode 100755 qa/workunits/rbd/rbd_mirror_snapshot.sh

diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml
index 5f12b22399ba..4fe1d6c10a5d 100644
--- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_journal.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: "journal"
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_USE_RBD_MIRROR: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml
index 29047a77d791..f5edab179103 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_CONFIG_KEY: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml
index af13c92b5758..e41596681fad 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_CONFIG_KEY: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml
index 5ea2bb105f06..ee6fc41ae8cb 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_CONFIG_KEY: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml
index e21d57b2b682..6bb17151b800 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_CONFIG_KEY: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml
index 0102050eb007..e5fbfd5610bc 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml
@@ -1,12 +1,13 @@
 meta:
-- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_journal.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'journal'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_CONFIG_KEY: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml
index 5bd024d2da81..2a1d06358b21 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml
@@ -1,11 +1,12 @@
 meta:
-- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_journal.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'journal'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_MIN_COMPAT_CLIENT: 'octopus'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml
index 0a610ea2f01d..cba93ad3afc2 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml
@@ -1,5 +1,5 @@
 meta:
-- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode
 overrides:
   ceph:
     conf:
@@ -8,9 +8,10 @@ overrides:
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_journal.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'journal'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml
index 883e8abd31e1..5c7d8486bdd2 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml
@@ -1,5 +1,5 @@
 meta:
-- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon
+- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode
 overrides:
   ceph:
     conf:
@@ -8,9 +8,10 @@ overrides:
 tasks:
 - workunit:
     clients:
-      cluster1.client.mirror: [rbd/rbd_mirror_journal.sh]
+      cluster1.client.mirror: [rbd/rbd_mirror.sh]
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_MODE: 'journal'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror.sh
similarity index 69%
rename from qa/workunits/rbd/rbd_mirror_journal.sh
rename to qa/workunits/rbd/rbd_mirror.sh
index 54f6aeec8e00..bc3237dfc089 100755
--- a/qa/workunits/rbd/rbd_mirror_journal.sh
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -1,26 +1,42 @@
 #!/bin/sh -ex
 #
-# rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode
+# rbd_mirror.sh - test rbd-mirror daemon in snapshot or journal mirroring mode
+#
+# Usage:
+#   RBD_MIRROR_MODE=journal rbd_mirror.sh
+#
+# Use environment variable RBD_MIRROR_MODE to set the mode
+# Available modes: snapshot | journal
 #
 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
 # creates a temporary directory, used for cluster configs, daemon logs, admin
 # socket, temporary files, and launches rbd-mirror daemon.
 #
 
-. $(dirname $0)/rbd_mirror_helpers.sh
+if [ "${#}" -gt 0 ]; then
+  echo "unnecessary arguments: ${@}"
+  exit 100
+fi
+
+if [ "${RBD_MIRROR_MODE}" != "snapshot" ] && [ "${RBD_MIRROR_MODE}" != "journal" ]; then
+  echo "unknown mode: ${RBD_MIRROR_MODE}"
+  echo "set RBD_MIRROR_MODE env variable, available modes: snapshot | journal"
+  exit 100
+fi
 
+. $(dirname $0)/rbd_mirror_helpers.sh
 setup
 
 testlog "TEST: add image and test replay"
 start_mirrors ${CLUSTER1}
 image=test
-create_image ${CLUSTER2} ${POOL} ${image}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE}
 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
 set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 write_image ${CLUSTER2} ${POOL} ${image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
 fi
@@ -31,12 +47,12 @@ compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
 testlog "TEST: stop mirror, add image, start mirror and test replay"
 stop_mirrors ${CLUSTER1}
 image1=test1
-create_image ${CLUSTER2} ${POOL} ${image1}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} ${RBD_MIRROR_MODE}
 write_image ${CLUSTER2} ${POOL} ${image1} 100
 start_mirrors ${CLUSTER1}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1}
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
 fi
@@ -46,7 +62,7 @@ testlog "TEST: test the first image is replaying after restart"
 write_image ${CLUSTER2} ${POOL} ${image} 100
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 compare_images ${POOL} ${image}
 
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
@@ -124,6 +140,9 @@ remove_image_retry ${CLUSTER2} ${POOL} ${image1}
 testlog "TEST: test image rename"
 new_name="${image}_RENAMED"
 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name}
+fi
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
@@ -131,13 +150,16 @@ admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${POOL} ${image}
+fi
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 testlog "TEST: test trash move restore"
 image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
 trash_move ${CLUSTER2} ${POOL} ${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
-trash_restore ${CLUSTER2} ${POOL} ${image_id}
+trash_restore ${CLUSTER2} ${POOL} ${image} ${image_id} ${RBD_MIRROR_MODE}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)"
@@ -146,7 +168,7 @@ remove_image_retry ${CLUSTER2} ${POOL} ${image}
 wait_for_image_in_omap ${CLUSTER1} ${POOL}
 wait_for_image_in_omap ${CLUSTER2} ${POOL}
 
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 write_image ${CLUSTER2} ${POOL} ${image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
@@ -165,7 +187,7 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 write_image ${CLUSTER2} ${POOL} ${image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 compare_images ${POOL} ${image}
 
 # failover (unmodified)
@@ -184,7 +206,7 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 promote_image ${CLUSTER2} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
@@ -198,7 +220,7 @@ wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 write_image ${CLUSTER1} ${POOL} ${image} 100
 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} ${image}
 compare_images ${POOL} ${image}
 
 # failback
@@ -210,7 +232,7 @@ promote_image ${CLUSTER2} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 write_image ${CLUSTER2} ${POOL} ${image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
@@ -235,15 +257,20 @@ for i in `seq 1 20`; do
   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 done
+# check that demote (or other mirror snapshots) don't pile up
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3
+  test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3
+fi
 
 testlog "TEST: force promote"
 force_promote_image=test_force_promote
-create_image ${CLUSTER2} ${POOL} ${force_promote_image}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} ${RBD_MIRROR_MODE}
 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image}
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
@@ -259,7 +286,7 @@ testlog "TEST: cloned images"
 testlog " - default"
 parent_image=test_parent
 parent_snap=snap
-create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${RBD_MIRROR_MODE}
 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
@@ -267,24 +294,29 @@ protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 clone_image=test_clone
 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
-
-enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  enable_mirror ${CLUSTER2} ${POOL} ${clone_image} ${RBD_MIRROR_MODE}
+else
+  enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${RBD_MIRROR_MODE}
+fi
 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 compare_images ${PARENT_POOL} ${parent_image}
 
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image}
 compare_images ${POOL} ${clone_image}
 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}
 
 testlog " - clone v1"
-clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
-
-clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
-            ${clone_image}_v1 --rbd-default-clone-format 1
+clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} \
+    ${parent_image} ${parent_snap} ${POOL} ${clone_image}1 \
+    ${RBD_MIRROR_MODE}
+clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} \
+    ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v1 \
+    ${RBD_MIRROR_MODE} --rbd-default-clone-format 1
 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
@@ -296,13 +328,20 @@ remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 testlog " - clone v2"
 parent_snap=snap_v2
 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
-            ${clone_image}_v2 --rbd-default-clone-format 2
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+fi
+clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} \
+    ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v2 \
+    ${RBD_MIRROR_MODE} --rbd-default-clone-format 2
 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
 
 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+fi
 test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
@@ -312,11 +351,18 @@ wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${pa
 
 testlog " - clone v2 non-primary"
 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+fi
 wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
-            ${clone_image}_v2 --rbd-default-clone-format 2
+clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} \
+    ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v2 \
+    ${RBD_MIRROR_MODE} --rbd-default-clone-format 2
 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+fi
 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
@@ -324,7 +370,8 @@ remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 
 testlog "TEST: data pool"
 dp_image=test_data_pool
-create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} \
+    ${RBD_MIRROR_MODE} 128 --data-pool ${PARENT_POOL}
 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 test "${data_pool}" = "${PARENT_POOL}"
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
@@ -335,7 +382,7 @@ write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image}
 compare_images ${POOL} ${dp_image}@snap1
 compare_images ${POOL} ${dp_image}@snap2
 compare_images ${POOL} ${dp_image}
@@ -347,7 +394,7 @@ image3=test3
 image4=test4
 image5=test5
 for i in ${image2} ${image3} ${image4} ${image5}; do
-  create_image ${CLUSTER2} ${POOL} ${i}
+  create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i} ${RBD_MIRROR_MODE}
   write_image ${CLUSTER2} ${POOL} ${i} 100
   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
@@ -356,6 +403,9 @@ for i in ${image2} ${image3} ${image4} ${image5}; do
     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
   fi
   write_image ${CLUSTER2} ${POOL} ${i} 100
+  if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+    mirror_image_snapshot ${CLUSTER2} ${POOL} ${i}
+  fi
   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 done
@@ -377,87 +427,100 @@ for i in ${image2} ${image3} ${image4} ${image5}; do
   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 done
 
-set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
-for i in ${image2} ${image4}; do
-  enable_journaling ${CLUSTER2} ${POOL} ${i}
-  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
-  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
-  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
-  compare_images ${POOL} ${i}
-done
-
-testlog "TEST: remove mirroring pool"
-pool=pool_to_remove
-for cluster in ${CLUSTER1} ${CLUSTER2}; do
+if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
+  set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
+  for i in ${image2} ${image4}; do
+    enable_journaling ${CLUSTER2} ${POOL} ${i}
+    wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
+    wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
+    wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
+    wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
+    compare_images ${POOL} ${i}
+  done
+
+  testlog "TEST: remove mirroring pool"
+  pool=pool_to_remove
+  for cluster in ${CLUSTER1} ${CLUSTER2}; do
     CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16
     CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool}
     rbd --cluster ${cluster} mirror pool enable ${pool} pool
-done
-peer_add ${CLUSTER1} ${pool} ${CLUSTER2}
-peer_add ${CLUSTER2} ${pool} ${CLUSTER1}
-rdp_image=test_remove_data_pool
-create_image ${CLUSTER2} ${pool} ${image} 128
-create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
-write_image ${CLUSTER2} ${pool} ${image} 100
-write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position'
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position'
-for cluster in ${CLUSTER1} ${CLUSTER2}; do
+  done
+  peer_add ${CLUSTER1} ${pool} ${CLUSTER2}
+  peer_add ${CLUSTER2} ${pool} ${CLUSTER1}
+  rdp_image=test_remove_data_pool
+  create_image ${CLUSTER2} ${pool} ${image} 128
+  create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
+  write_image ${CLUSTER2} ${pool} ${image} 100
+  write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
+  wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${pool} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
+  wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image}
+  for cluster in ${CLUSTER1} ${CLUSTER2}; do
     CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
-done
-remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
-wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
-for i in 0 1 2 4 8 8 8 8 16 16; do
+  done
+  remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
+  for i in 0 1 2 4 8 8 8 8 16 16; do
     sleep $i
     admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break
-done
-admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
+  done
+  admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
+fi
 
 testlog "TEST: snapshot rename"
 snap_name='snap_rename'
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  enable_mirror ${CLUSTER2} ${POOL} ${image2}
+fi
 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 for i in `seq 1 20`; do
   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 done
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2}
+fi
 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 
 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1'
 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2'
 for i in ${image2} ${image4}; do
-    remove_image_retry ${CLUSTER2} ${POOL} ${i}
+  remove_image_retry ${CLUSTER2} ${POOL} ${i}
 done
 
 testlog "TEST: disable mirror while daemon is stopped"
 stop_mirrors ${CLUSTER1}
 stop_mirrors ${CLUSTER2}
-set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
+if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
+  set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
+fi
 disable_mirror ${CLUSTER2} ${POOL} ${image}
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 fi
 start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
-set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
-enable_journaling ${CLUSTER2} ${POOL} ${image}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  enable_mirror ${CLUSTER2} ${POOL} ${image}
+else
+  set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
+  enable_journaling ${CLUSTER2} ${POOL} ${image}
+fi
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 testlog "TEST: non-default namespace image mirroring"
 testlog " - replay"
-create_image ${CLUSTER2} ${POOL}/${NS1} ${image}
-create_image ${CLUSTER2} ${POOL}/${NS2} ${image}
-enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal
+create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} ${RBD_MIRROR_MODE}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} ${RBD_MIRROR_MODE}
 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image}
 write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image}
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image}
 compare_images ${POOL}/${NS1} ${image}
 compare_images ${POOL}/${NS2} ${image}
 
@@ -470,7 +533,7 @@ remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image}
 
 testlog " - data pool"
 dp_image=test_data_pool
-create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL}
+create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} ${RBD_MIRROR_MODE} 128 --data-pool ${PARENT_POOL}
 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image})
 test "${data_pool}" = "${PARENT_POOL}"
 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
@@ -478,7 +541,7 @@ data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image})
 test "${data_pool}" = "${PARENT_POOL}"
 write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
 compare_images ${POOL}/${NS1} ${dp_image}
 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
 
@@ -487,11 +550,11 @@ request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 compare_images ${POOL} ${image}
 
-testlog "TEST: image resync while replayer is stopped"
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  testlog "TEST: image resync while replayer is stopped"
   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
   request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
@@ -500,7 +563,7 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+  wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
   compare_images ${POOL} ${image}
 fi
 
@@ -511,86 +574,88 @@ start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 compare_images ${POOL} ${image}
 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 
-testlog "TEST: client disconnect"
-image=laggy
-create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
-write_image ${CLUSTER2} ${POOL} ${image} 10
+if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
+  testlog "TEST: client disconnect"
+  image=laggy
+  create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
+  write_image ${CLUSTER2} ${POOL} ${image} 10
 
-testlog " - replay stopped after disconnect"
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-disconnect_image ${CLUSTER2} ${POOL} ${image}
-test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+  testlog " - replay stopped after disconnect"
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+  disconnect_image ${CLUSTER2} ${POOL} ${image}
+  test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 
-testlog " - replay started after resync requested"
-request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-compare_images ${POOL} ${image}
+  testlog " - replay started after resync requested"
+  request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+  compare_images ${POOL} ${image}
 
-testlog " - disconnected after max_concurrent_object_sets reached"
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  testlog " - disconnected after max_concurrent_object_sets reached"
+  if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+    admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+    wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+    test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+    set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	    conf_rbd_journal_max_concurrent_object_sets 1
+    write_image ${CLUSTER2} ${POOL} ${image} 20 16384
+    write_image ${CLUSTER2} ${POOL} ${image} 20 16384
+    test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+    set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	    conf_rbd_journal_max_concurrent_object_sets 0
+
+    testlog " - replay is still stopped (disconnected) after restart"
+    admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+    wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+    wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+  fi
+
+  testlog " - replay started after resync requested"
+  request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
   test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+  compare_images ${POOL} ${image}
+
+  testlog " - rbd_mirroring_resync_after_disconnect config option"
   set_image_meta ${CLUSTER2} ${POOL} ${image} \
-	         conf_rbd_journal_max_concurrent_object_sets 1
-  write_image ${CLUSTER2} ${POOL} ${image} 20 16384
-  write_image ${CLUSTER2} ${POOL} ${image} 20 16384
-  test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+	  conf_rbd_mirroring_resync_after_disconnect true
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
+  disconnect_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
+  compare_images ${POOL} ${image}
   set_image_meta ${CLUSTER2} ${POOL} ${image} \
-	         conf_rbd_journal_max_concurrent_object_sets 0
-
-  testlog " - replay is still stopped (disconnected) after restart"
-  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+	  conf_rbd_mirroring_resync_after_disconnect false
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  disconnect_image ${CLUSTER2} ${POOL} ${image}
+  test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+  remove_image_retry ${CLUSTER2} ${POOL} ${image}
 fi
 
-testlog " - replay started after resync requested"
-request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-compare_images ${POOL} ${image}
-
-testlog " - rbd_mirroring_resync_after_disconnect config option"
-set_image_meta ${CLUSTER2} ${POOL} ${image} \
-	       conf_rbd_mirroring_resync_after_disconnect true
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
-disconnect_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-compare_images ${POOL} ${image}
-set_image_meta ${CLUSTER2} ${POOL} ${image} \
-	       conf_rbd_mirroring_resync_after_disconnect false
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-disconnect_image ${CLUSTER2} ${POOL} ${image}
-test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
-remove_image_retry ${CLUSTER2} ${POOL} ${image}
-
 testlog "TEST: split-brain"
 image=split-brain
-create_image ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE}
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 promote_image ${CLUSTER1} ${POOL} ${image} --force
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
@@ -598,7 +663,7 @@ write_image ${CLUSTER1} ${POOL} ${image} 10
 demote_image ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
+wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image}
 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 
 testlog "TEST: check if removed images' OMAP are removed"
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index f4961b925e6f..7183a2cba258 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -24,7 +24,7 @@
 # The cleanup can be done as a separate step, running the script with
 # `cleanup ${RBD_MIRROR_TEMDIR}' arguments.
 #
-# Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries
+# Note, as other workunits tests, rbd_mirror_helpers.sh expects to find ceph binaries
 # in PATH.
 #
 # Thus a typical troubleshooting session:
@@ -35,7 +35,7 @@
 #   cd $CEPH_SRC_PATH
 #   PATH=$CEPH_SRC_PATH:$PATH
 #   RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
-#     ../qa/workunits/rbd/rbd_mirror_journal.sh
+#     RBD_MIRROR_MODE=journal ../qa/workunits/rbd/rbd_mirror.sh
 #
 # After the test failure cd to TEMPDIR and check the current state:
 #
@@ -49,21 +49,27 @@
 #   ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help
 #   ...
 #
+# To setup the environment without actually running the tests:
+#
+#   cd $CEPH_SRC_PATH
+#   RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
+#     ../qa/workunits/rbd_mirror_helpers.sh setup
+#
 # Also you can execute commands (functions) from the script:
 #
 #   cd $CEPH_SRC_PATH
 #   export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
-#   ../qa/workunits/rbd/rbd_mirror_journal.sh status
-#   ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1
-#   ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2
-#   ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2
+#   ../qa/workunits/rbd/rbd_mirror_helpers.sh status
+#   ../qa/workunits/rbd/rbd_mirror_helpers.sh stop_mirror cluster1
+#   ../qa/workunits/rbd/rbd_mirror_helpers.sh start_mirror cluster2
+#   ../qa/workunits/rbd/rbd_mirror_helpers.sh flush cluster2
 #   ...
 #
 # Eventually, run the cleanup:
 #
 #   cd $CEPH_SRC_PATH
 #   RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
-#     ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup
+#     ../qa/workunits/rbd_mirror_helpers.sh cleanup
 #
 
 if type xmlstarlet > /dev/null 2>&1; then
@@ -90,6 +96,10 @@ RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling}
 MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.}
 MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool}
 MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal}
+if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
+  MIRROR_POOL_MODE=image
+  MIRROR_IMAGE_MODE=snapshot
+fi
 
 export CEPH_ARGS="--id ${CEPH_ID}"
 
@@ -880,6 +890,20 @@ wait_for_status_in_pool_dir()
     return 1
 }
 
+wait_for_replaying_status_in_pool_dir()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
+        wait_for_status_in_pool_dir ${cluster} ${pool} ${image} 'up+replaying' \
+                                    'primary_position'
+    else
+        wait_for_status_in_pool_dir ${cluster} ${pool} ${image} 'up+replaying'
+    fi
+}
+
 create_image()
 {
     local cluster=$1 ; shift
@@ -896,6 +920,23 @@ create_image()
         --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image}
 }
 
+is_pool_mirror_mode_image()
+{
+    local pool=$1
+
+    if [ "${MIRROR_POOL_MODE}" = "image" ]; then
+        return 0
+    fi
+
+    case "${pool}" in
+        */${NS2} | ${PARENT_POOL})
+            return 0
+            ;;
+    esac
+
+    return 1
+}
+
 create_image_and_enable_mirror()
 {
     local cluster=$1 ; shift
@@ -907,7 +948,7 @@ create_image_and_enable_mirror()
     fi
 
     create_image ${cluster} ${pool} ${image} $@
-    if [ "${MIRROR_POOL_MODE}" = "image" ] || [ "$pool" = "${PARENT_POOL}" ]; then
+    if is_pool_mirror_mode_image ${pool}; then
         enable_mirror ${cluster} ${pool} ${image} ${mode}
     fi
 }
@@ -987,9 +1028,14 @@ trash_move() {
 trash_restore() {
     local cluster=$1
     local pool=$2
-    local image_id=$3
+    local image=$3
+    local image_id=$4
+    local mode=${5:-${RBD_MIRROR_MODE}}
 
     rbd --cluster=${cluster} trash restore ${pool}/${image_id}
+    if is_pool_mirror_mode_image ${pool}; then
+        enable_mirror ${cluster} ${pool} ${image} ${mode}
+    fi
 }
 
 clone_image()
@@ -1024,7 +1070,9 @@ clone_image_and_enable_mirror()
     fi
 
     clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@
-    enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode}
+    if is_pool_mirror_mode_image ${clone_pool}; then
+      enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode}
+    fi
 }
 
 disconnect_image()
diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh
deleted file mode 100755
index c70d48b09db4..000000000000
--- a/qa/workunits/rbd/rbd_mirror_snapshot.sh
+++ /dev/null
@@ -1,517 +0,0 @@
-#!/bin/sh -ex
-#
-# rbd_mirror_snapshot.sh - test rbd-mirror daemon in snapshot-based mirroring mode
-#
-# The scripts starts two ("local" and "remote") clusters using mstart.sh script,
-# creates a temporary directory, used for cluster configs, daemon logs, admin
-# socket, temporary files, and launches rbd-mirror daemon.
-#
-
-MIRROR_POOL_MODE=image
-MIRROR_IMAGE_MODE=snapshot
-
-. $(dirname $0)/rbd_mirror_helpers.sh
-
-setup
-
-testlog "TEST: add image and test replay"
-start_mirrors ${CLUSTER1}
-image=test
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image}
-set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
-set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-write_image ${CLUSTER2} ${POOL} ${image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
-fi
-compare_images ${POOL} ${image}
-compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
-compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
-
-testlog "TEST: stop mirror, add image, start mirror and test replay"
-stop_mirrors ${CLUSTER1}
-image1=test1
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1}
-write_image ${CLUSTER2} ${POOL} ${image1} 100
-start_mirrors ${CLUSTER1}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
-fi
-compare_images ${POOL} ${image1}
-
-testlog "TEST: test the first image is replaying after restart"
-write_image ${CLUSTER2} ${POOL} ${image} 100
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-compare_images ${POOL} ${image}
-
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  testlog "TEST: stop/start/restart mirror via admin socket"
-  all_admin_daemons ${CLUSTER1} rbd mirror stop
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror start
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror restart
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror stop
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror restart
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-  admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-
-  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
-
-  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
-
-  flush ${CLUSTER1}
-  all_admin_daemons ${CLUSTER1} rbd mirror status
-fi
-
-remove_image_retry ${CLUSTER2} ${POOL} ${image1}
-
-testlog "TEST: test image rename"
-new_name="${image}_RENAMED"
-rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
-mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
-admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
-admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
-rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
-mirror_image_snapshot ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-
-testlog "TEST: test trash move restore"
-image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
-trash_move ${CLUSTER2} ${POOL} ${image}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
-trash_restore ${CLUSTER2} ${POOL} ${image_id}
-enable_mirror ${CLUSTER2} ${POOL} ${image} snapshot
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-
-testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)"
-remove_image_retry ${CLUSTER2} ${POOL} ${image}
-
-wait_for_image_in_omap ${CLUSTER1} ${POOL}
-wait_for_image_in_omap ${CLUSTER2} ${POOL}
-
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-write_image ${CLUSTER2} ${POOL} ${image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-
-testlog "TEST: failover and failback"
-start_mirrors ${CLUSTER2}
-
-# demote and promote same cluster
-demote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-promote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-write_image ${CLUSTER2} ${POOL} ${image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-compare_images ${POOL} ${image}
-
-# failover (unmodified)
-demote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-promote_image ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
-
-# failback (unmodified)
-demote_image ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-promote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
-compare_images ${POOL} ${image}
-
-# failover
-demote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-promote_image ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
-write_image ${CLUSTER1} ${POOL} ${image} 100
-wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
-compare_images ${POOL} ${image}
-
-# failback
-demote_image ${CLUSTER1} ${POOL} ${image}
-wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-promote_image ${CLUSTER2} ${POOL} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-write_image ${CLUSTER2} ${POOL} ${image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
-compare_images ${POOL} ${image}
-
-testlog "TEST: failover / failback loop"
-for i in `seq 1 20`; do
-  demote_image ${CLUSTER2} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-  promote_image ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
-  wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
-  demote_image ${CLUSTER1} ${POOL} ${image}
-  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
-  promote_image ${CLUSTER2} ${POOL} ${image}
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-done
-# check that demote (or other mirror snapshots) don't pile up
-test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3
-test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3
-
-testlog "TEST: force promote"
-force_promote_image=test_force_promote
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image}
-write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
-wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
-promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
-wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
-wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
-write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
-write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
-remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image}
-remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image}
-
-testlog "TEST: cloned images"
-testlog " - default"
-parent_image=test_parent
-parent_snap=snap
-create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
-create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-
-clone_image=test_clone
-clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
-write_image ${CLUSTER2} ${POOL} ${clone_image} 100
-enable_mirror ${CLUSTER2} ${POOL} ${clone_image} snapshot
-
-wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying'
-compare_images ${PARENT_POOL} ${parent_image}
-
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying'
-compare_images ${POOL} ${clone_image}
-remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}
-
-testlog " - clone v1"
-clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \
-    ${parent_snap} ${POOL} ${clone_image}1
-
-clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \
-    ${parent_snap} ${POOL} ${clone_image}_v1 snapshot --rbd-default-clone-format 1
-test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
-test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
-remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1
-remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1
-unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-
-testlog " - clone v2"
-parent_snap=snap_v2
-create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \
-    ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2
-test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
-test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
-
-remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
-wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted'
-test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-
-testlog " - clone v2 non-primary"
-create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \
-    ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2
-remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
-mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
-wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
-remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image}
-
-testlog "TEST: data pool"
-dp_image=test_data_pool
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL}
-data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
-test "${data_pool}" = "${PARENT_POOL}"
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
-data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
-test "${data_pool}" = "${PARENT_POOL}"
-create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
-write_image ${CLUSTER2} ${POOL} ${dp_image} 100
-create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
-write_image ${CLUSTER2} ${POOL} ${dp_image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying'
-compare_images ${POOL} ${dp_image}@snap1
-compare_images ${POOL} ${dp_image}@snap2
-compare_images ${POOL} ${dp_image}
-remove_image_retry ${CLUSTER2} ${POOL} ${dp_image}
-
-testlog "TEST: disable mirroring / delete non-primary image"
-image2=test2
-image3=test3
-image4=test4
-image5=test5
-for i in ${image2} ${image3} ${image4} ${image5}; do
-  create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i}
-  write_image ${CLUSTER2} ${POOL} ${i} 100
-  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
-  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
-  if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
-    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
-    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
-  fi
-  write_image ${CLUSTER2} ${POOL} ${i} 100
-  mirror_image_snapshot ${CLUSTER2} ${POOL} ${i}
-  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
-  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
-done
-
-set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
-for i in ${image2} ${image4}; do
-  disable_mirror ${CLUSTER2} ${POOL} ${i}
-done
-
-unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
-unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
-for i in ${image3} ${image5}; do
-  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
-  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
-  remove_image_retry ${CLUSTER2} ${POOL} ${i}
-done
-
-for i in ${image2} ${image3} ${image4} ${image5}; do
-  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
-done
-
-testlog "TEST: snapshot rename"
-snap_name='snap_rename'
-enable_mirror ${CLUSTER2} ${POOL} ${image2}
-create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
-for i in `seq 1 20`; do
-  rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
-done
-mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2}
-wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
-
-unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1'
-unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2'
-for i in ${image2} ${image4}; do
-    remove_image_retry ${CLUSTER2} ${POOL} ${i}
-done
-
-testlog "TEST: disable mirror while daemon is stopped"
-stop_mirrors ${CLUSTER1}
-stop_mirrors ${CLUSTER2}
-disable_mirror ${CLUSTER2} ${POOL} ${image}
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-fi
-start_mirrors ${CLUSTER1}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
-enable_mirror ${CLUSTER2} ${POOL} ${image}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-
-testlog "TEST: non-default namespace image mirroring"
-testlog " - replay"
-create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image}
-create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image}
-wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image}
-write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
-write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image}
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying'
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying'
-compare_images ${POOL}/${NS1} ${image}
-compare_images ${POOL}/${NS2} ${image}
-
-testlog " - disable mirroring / delete image"
-remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image}
-disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image}
-wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted'
-wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted'
-remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image}
-
-testlog " - data pool"
-dp_image=test_data_pool
-create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL}
-data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image})
-test "${data_pool}" = "${PARENT_POOL}"
-wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
-data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image})
-test "${data_pool}" = "${PARENT_POOL}"
-write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100
-wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying'
-compare_images ${POOL}/${NS1} ${dp_image}
-remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
-
-testlog "TEST: simple image resync"
-request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-compare_images ${POOL} ${image}
-
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  testlog "TEST: image resync while replayer is stopped"
-  admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
-  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-  request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
-  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
-  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-  compare_images ${POOL} ${image}
-fi
-
-testlog "TEST: request image resync while daemon is offline"
-stop_mirrors ${CLUSTER1}
-request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-start_mirrors ${CLUSTER1}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
-wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-compare_images ${POOL} ${image}
-remove_image_retry ${CLUSTER2} ${POOL} ${image}
-
-testlog "TEST: split-brain"
-image=split-brain
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-promote_image ${CLUSTER1} ${POOL} ${image} --force
-wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
-write_image ${CLUSTER1} ${POOL} ${image} 10
-demote_image ${CLUSTER1} ${POOL} ${image}
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
-request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
-remove_image_retry ${CLUSTER2} ${POOL} ${image}
-
-testlog "TEST: check if removed images' OMAP are removed"
-start_mirrors ${CLUSTER2}
-wait_for_image_in_omap ${CLUSTER1} ${POOL}
-wait_for_image_in_omap ${CLUSTER2} ${POOL}
-
-if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
-  # teuthology will trash the daemon
-  testlog "TEST: no blocklists"
-  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
-  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
-fi

From b096c21c7c4df2bf177f3ba84093cc75ecd71d49 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 2 Nov 2023 16:07:39 +1000
Subject: [PATCH 0369/2492] doc/cephadm: edit troubleshooting.rst (2 of x)

Edit doc/cephadm/troubleshooting.rst. This commit and the PR of which it
is a part was raised in response to https://github.com/ceph/ceph/pull/53976.

Co-authored-by: John Mulligan <jmulligan@redhat.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/cephadm/troubleshooting.rst | 218 ++++++++++++++++++--------------
 1 file changed, 125 insertions(+), 93 deletions(-)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index a7ee9168da05..df66b3ae0385 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -266,7 +266,7 @@ Each Ceph daemon provides an admin socket that bypasses the MONs (See
 Running Various Ceph Tools
 --------------------------------
 
-To run Ceph tools like ``ceph-objectstore-tool`` or 
+To run Ceph tools such as ``ceph-objectstore-tool`` or 
 ``ceph-monstore-tool``, invoke the cephadm CLI with
 ``cephadm shell --name <daemon-name>``.  For example::
 
@@ -283,98 +283,114 @@ To run Ceph tools like ``ceph-objectstore-tool`` or
     election_strategy: 1
     0: [v2:127.0.0.1:3300/0,v1:127.0.0.1:6789/0] mon.myhostname
 
-The cephadm shell sets up the environment in a way that is suitable
-for extended daemon maintenance and running daemons interactively. 
+The cephadm shell sets up the environment in a way that is suitable for
+extended daemon maintenance and for the interactive running of daemons. 
 
 .. _cephadm-restore-quorum:
 
 Restoring the Monitor Quorum
 ----------------------------
 
-If the Ceph monitor daemons (mons) cannot form a quorum, cephadm will not be
-able to manage the cluster until quorum is restored.
+If the Ceph Monitor daemons (mons) cannot form a quorum, ``cephadm`` will not
+be able to manage the cluster until quorum is restored.
 
 In order to restore the quorum, remove unhealthy monitors
 form the monmap by following these steps:
 
-1. Stop all mons. For each mon host::
+1. Stop all Monitors. Use ``ssh`` to connect to each Monitor's host, and then
+   while connected to the Monitor's host use ``cephadm`` to stop the Monitor
+   daemon:
+
+   .. prompt:: bash
+
+      ssh {mon-host}
+      cephadm unit --name {mon.hostname} stop
 
-    ssh {mon-host}
-    cephadm unit --name mon.`hostname` stop
 
+2. Identify a surviving Monitor and log in to its host:
 
-2. Identify a surviving monitor and log in to that host::
+   .. prompt:: bash
 
-    ssh {mon-host}
-    cephadm enter --name mon.`hostname`
+      ssh {mon-host}
+      cephadm enter --name {mon.hostname}
 
-3. Follow the steps in :ref:`rados-mon-remove-from-unhealthy`
+3. Follow the steps in :ref:`rados-mon-remove-from-unhealthy`.
 
 .. _cephadm-manually-deploy-mgr:
 
 Manually Deploying a Manager Daemon
 -----------------------------------
-At least one manager (mgr) daemon is required by cephadm in order to manage the
-cluster. If the last mgr in a cluster has been removed, follow these steps in
-order to deploy a manager called (for example)
-``mgr.hostname.smfvfd`` on a random host of your cluster manually. 
+At least one Manager (``mgr``) daemon is required by cephadm in order to manage
+the cluster. If the last remaining Manager has been removed from the Ceph
+cluster, follow these steps in order to deploy a fresh Manager on an arbitrary
+host in your cluster. In this example, the freshly-deployed Manager daemon is
+called ``mgr.hostname.smfvfd``.
+
+#. Disable the cephadm scheduler, in order to prevent ``cephadm`` from removing
+   the new Manager. See :ref:`cephadm-enable-cli`:
+
+   .. prompt:: bash #
 
-Disable the cephadm scheduler, in order to prevent cephadm from removing the new 
-manager. See :ref:`cephadm-enable-cli`::
+      ceph config-key set mgr/cephadm/pause true
 
-  ceph config-key set mgr/cephadm/pause true
+#. Retrieve or create the "auth entry" for the new Manager:
 
-Then get or create the auth entry for the new manager::
+   .. prompt:: bash #
 
-  ceph auth get-or-create mgr.hostname.smfvfd mon "profile mgr" osd "allow *" mds "allow *"
+      ceph auth get-or-create mgr.hostname.smfvfd mon "profile mgr" osd "allow *" mds "allow *"
 
-Get the ceph.conf::
+#. Retrieve the Monitor's configuration:
 
-  ceph config generate-minimal-conf
+   .. prompt:: bash #
 
-Get the container image::
+      ceph config generate-minimal-conf
 
-  ceph config get "mgr.hostname.smfvfd" container_image
+#. Retrieve the container image:
 
-Create a file ``config-json.json`` which contains the information necessary to deploy
-the daemon:
+   .. prompt:: bash #
 
-.. code-block:: json
+      ceph config get "mgr.hostname.smfvfd" container_image
 
-  {
-    "config": "# minimal ceph.conf for 8255263a-a97e-4934-822c-00bfe029b28f\n[global]\n\tfsid = 8255263a-a97e-4934-822c-00bfe029b28f\n\tmon_host = [v2:192.168.0.1:40483/0,v1:192.168.0.1:40484/0]\n",
-    "keyring": "[mgr.hostname.smfvfd]\n\tkey = V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4=\n"
-  }
+#. Create a file called ``config-json.json``, which contains the information
+   necessary to deploy the daemon:
 
-Deploy the daemon::
+   .. code-block:: json
 
-  cephadm --image <container-image> deploy --fsid <fsid> --name mgr.hostname.smfvfd --config-json config-json.json
+     {
+       "config": "# minimal ceph.conf for 8255263a-a97e-4934-822c-00bfe029b28f\n[global]\n\tfsid = 8255263a-a97e-4934-822c-00bfe029b28f\n\tmon_host = [v2:192.168.0.1:40483/0,v1:192.168.0.1:40484/0]\n",
+       "keyring": "[mgr.hostname.smfvfd]\n\tkey = V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4=\n"
+     }
+
+#. Deploy the Manager daemon:
+
+   .. prompt:: bash #
+
+      cephadm --image <container-image> deploy --fsid <fsid> --name mgr.hostname.smfvfd --config-json config-json.json
 
 Capturing Core Dumps
 ---------------------
 
-A Ceph cluster that uses cephadm can be configured to capture core dumps.
-Initial capture and processing of the coredump is performed by
-`systemd-coredump <https://www.man7.org/linux/man-pages/man8/systemd-coredump.8.html>`_.
+A Ceph cluster that uses ``cephadm`` can be configured to capture core dumps.
+The initial capture and processing of the coredump is performed by
+`systemd-coredump
+<https://www.man7.org/linux/man-pages/man8/systemd-coredump.8.html>`_.
 
 
-To enable coredump handling, run:
+To enable coredump handling, run the following command
 
 .. prompt:: bash #
 
-  ulimit -c unlimited
+   ulimit -c unlimited
 
-Core dumps will be written to ``/var/lib/systemd/coredump``.
-This will persist until the system is rebooted.
 
 .. note::
 
-  Core dumps are not namespaced by the kernel, which means
-  they will be written to ``/var/lib/systemd/coredump`` on
-  the container host. 
+  Core dumps are not namespaced by the kernel. This means that core dumps are
+  written to ``/var/lib/systemd/coredump`` on the container host. The ``ulimit
+  -c unlimited`` setting  will persist  only until the system is rebooted.
 
-Now, wait for the crash to happen again. To simulate the crash of a daemon, run
-e.g. ``killall -3 ceph-mon``.
+Wait for the crash to happen again. To simulate the crash of a daemon, run for
+example ``killall -3 ceph-mon``.
 
 
 Running the Debugger with cephadm
@@ -383,45 +399,58 @@ Running the Debugger with cephadm
 Running a single debugging session
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-One can initiate a debugging session using the ``cephadm shell`` command.
+Initiate a debugging session by using the ``cephadm shell`` command.
 From within the shell container we need to install the debugger and debuginfo
 packages. To debug a core file captured by systemd, run the following:
 
-.. prompt:: bash #
 
-    # start the shell session
-    cephadm shell --mount /var/lib/system/coredump
-    # within the shell:
-    dnf install ceph-debuginfo gdb zstd
+#. Start the shell session:
+
+   .. prompt:: bash #
+
+      cephadm shell --mount /var/lib/system/coredump
+
+#. From within the shell session, run the following commands:
+
+   .. prompt:: bash #
+
+      dnf install ceph-debuginfo gdb zstd
+
+   .. prompt:: bash #
+      
     unzstd /var/lib/systemd/coredump/core.ceph-*.zst
+
+   .. prompt:: bash #
+
     gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-*.zst
 
-You can then run debugger commands at gdb's prompt.
+#. Run debugger commands at gdb's prompt:
+
+   .. prompt:: bash (gdb)
 
-.. prompt::
+      bt
+      
+   ::
 
-    (gdb) bt
-    #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
-    #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
-    #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
-    #3  0x0000563085ca3d7e in main ()
+      #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
+      #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
+      #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
+      #3  0x0000563085ca3d7e in main ()
 
 
 Running repeated debugging sessions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When using ``cephadm shell``, like in the example above, the changes made to
-the container the shell command spawned are ephemeral. Once the shell session
-exits all of the files that were downloaded and installed are no longer
-available. One can simply re-run the same commands every time ``cephadm shell``
-is invoked, but in order to save time and resources one can create a new
-container image and use it for repeated debugging sessions.
+When using ``cephadm shell``, as in the example above, any changes made to the
+container that is spawned by the shell command are ephemeral. After the shell
+session exits, the files that were downloaded and installed cease to be
+available. You can simply re-run the same commands every time ``cephadm
+shell`` is invoked, but in order to save time and resources one can create a
+new container image and use it for repeated debugging sessions.
 
-In the following example we create a simple file for constructing the
-container image. The command below uses podman but it should work correctly
-if ``podman`` is replaced with ``docker``.
-
-.. prompt:: bash
+In the following example, we create a simple file that will construct the
+container image. The command below uses podman but it is expected to work
+correctly even if ``podman`` is replaced with ``docker``::
 
   cat >Containerfile <<EOF
   ARG BASE_IMG=quay.io/ceph/ceph:v18
@@ -432,16 +461,17 @@ if ``podman`` is replaced with ``docker``.
   podman build -t ceph:debugging -f Containerfile .
   # pass --build-arg=BASE_IMG=<your image> to customize the base image
 
-The result should be a new local image named ``ceph:debugging``. This image can
-be used on the same machine that built it. Later, the image could be pushed to
-a container repository, or saved and copied to a node runing other ceph
-containers. Please consult the documentation for ``podman`` or ``docker`` for
-more details on the general container workflow.
+The above file creates a new local image named ``ceph:debugging``. This image
+can be used on the same machine that built it. The image can also be pushed to
+a container repository or saved and copied to a node runing other Ceph
+containers. Consult the ``podman`` or ``docker`` documentation for more
+information about the container workflow.
 
-Once the image has been built it can be used to initiate repeat debugging
-sessions without having to re-install the debug tools and debuginfo packages.
-To debug a core file using this image, in the same way as previously described,
-run:
+After the image has been built, it can be used to initiate repeat debugging
+sessions. By using an image in this way, you avoid the trouble of having to
+re-install the debug tools and debuginfo packages every time you need to run a
+debug session. To debug a core file using this image, in the same way as
+previously described, run:
 
 .. prompt:: bash #
 
@@ -451,29 +481,31 @@ run:
 Debugging live processes
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-The gdb debugger has the ability to attach to running processes to debug them.
-For a containerized process this can be accomplished by using the debug image
-and attaching it to the same PID namespace as the process to be debugged.
+The gdb debugger can attach to running processes to debug them. This can be
+achieved with a containerized process by using the debug image and attaching it
+to the same PID namespace in which the process to be debugged resides.
 
-This requires running a container command with some custom arguments. We can generate a script that can debug a process in a running container.
+This requires running a container command with some custom arguments. We can
+generate a script that can debug a process in a running container.
 
 .. prompt:: bash #
 
    cephadm --image ceph:debugging shell --dry-run > /tmp/debug.sh
 
-This creates a script with the container command cephadm would use to create a
-shell. Now, modify the script by removing the ``--init`` argument and replace
-that with the argument to join to the namespace used for a running running
-container.  For example, let's assume we want to debug the MGR, and have
-determnined that the MGR is running in a container named
-``ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``. The new
-argument
+This creates a script that includes the container command that ``cephadm``
+would use to create a shell. Modify the script by removing the ``--init``
+argument and replace it with the argument that joins to the namespace used for
+a running running container. For example, assume we want to debug the Manager
+and have determnined that the Manager is running in a container named
+``ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``. In this case,
+the argument
 ``--pid=container:ceph-bc615290-685b-11ee-84a6-525400220000-mgr-ceph0-sluwsk``
 should be used.
 
-Now, we can run our debugging container with ``sh /tmp/debug.sh``. Within the shell
-we can run commands such as ``ps`` to get the PID of the MGR process. In the following
-example this will be ``2``. Running gdb, we can now attach to the running process:
+We can run our debugging container with ``sh /tmp/debug.sh``. Within the shell,
+we can run commands such as ``ps`` to get the PID of the Manager process. In
+the following example this is ``2``. While running gdb, we can attach to the
+running process:
 
 .. prompt:: bash (gdb)
 

From fe3a4b9683d7b6f72f57f5ed8bc324bdbb24351f Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 10 Oct 2023 01:28:36 +0530
Subject: [PATCH 0370/2492] cephfs,mon: allow FS rename only if FS is offline

Reject the attempt to rename the CephFS is the CephFS is not offline.
Add new tests for this and update current tests (test_admin.py and
test_volumes.py) accordingly.

Fixes: https://tracker.ceph.com/issues/63154
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_admin.py   | 52 +++++++++++++++++++++++++++++++++
 qa/tasks/cephfs/test_volumes.py | 28 ++++++++++++++++++
 src/mon/FSCommands.cc           |  8 +++++
 3 files changed, 88 insertions(+)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index b2e10e4d3cf4..fe224b6831dd 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -627,7 +627,11 @@ def test_fs_rename(self):
         new_fs_name = 'new_cephfs'
         client_id = 'test_new_cephfs'
 
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
+        self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        sleep(5)
 
         # authorize a cephx ID access to the renamed file system.
         # use the ID to write to the file system.
@@ -660,8 +664,12 @@ def test_fs_rename_idempotency(self):
         orig_fs_name = self.fs.name
         new_fs_name = 'new_cephfs'
 
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
+        self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        sleep(5)
 
         # original file system name does not appear in `fs ls` command
         self.assertFalse(self.fs.exists())
@@ -680,7 +688,11 @@ def test_fs_rename_fs_new_fails_with_old_fsname_existing_pools(self):
         new_fs_name = 'new_cephfs'
         data_pool = self.fs.get_data_pool_name()
         metadata_pool = self.fs.get_metadata_pool_name()
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
+        self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        sleep(5)
 
         try:
             self.run_ceph_cmd(f"fs new {orig_fs_name} {metadata_pool} {data_pool}")
@@ -717,6 +729,8 @@ def test_fs_rename_fails_without_yes_i_really_mean_it_flag(self):
         """
         That renaming a file system without '--yes-i-really-mean-it' flag fails.
         """
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         try:
             self.run_ceph_cmd(f"fs rename {self.fs.name} new_fs")
         except CommandFailedError as ce:
@@ -726,11 +740,14 @@ def test_fs_rename_fails_without_yes_i_really_mean_it_flag(self):
         else:
             self.fail("expected renaming of file system without the "
                       "'--yes-i-really-mean-it' flag to fail ")
+        self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
 
     def test_fs_rename_fails_for_non_existent_fs(self):
         """
         That renaming a non-existent file system fails.
         """
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         try:
             self.run_ceph_cmd("fs rename non_existent_fs new_fs --yes-i-really-mean-it")
         except CommandFailedError as ce:
@@ -744,6 +761,8 @@ def test_fs_rename_fails_new_name_already_in_use(self):
         """
         self.fs2 = self.mds_cluster.newfs(name='cephfs2', create=True)
 
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         try:
             self.run_ceph_cmd(f"fs rename {self.fs.name} {self.fs2.name} --yes-i-really-mean-it")
         except CommandFailedError as ce:
@@ -751,6 +770,7 @@ def test_fs_rename_fails_new_name_already_in_use(self):
                              "invalid error code on renaming to a fs name that is already in use")
         else:
             self.fail("expected renaming to a new file system name that is already in use to fail.")
+        self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
 
     def test_fs_rename_fails_with_mirroring_enabled(self):
         """
@@ -760,6 +780,8 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
         new_fs_name = 'new_cephfs'
 
         self.run_ceph_cmd(f'fs mirror enable {orig_fs_name}')
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        sleep(5)
         try:
             self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         except CommandFailedError as ce:
@@ -767,6 +789,36 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
         else:
             self.fail("expected renaming of a mirrored file system to fail")
         self.run_ceph_cmd(f'fs mirror disable {orig_fs_name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
+
+    def test_rename_when_fs_is_online(self):
+        '''
+        Test that the command "ceph fs swap" command fails when first of the
+        two of FSs isn't failed/down.
+        '''
+        client_id = 'test_new_cephfs'
+        new_fs_name = 'new_cephfs'
+        self.negtest_ceph_cmd(
+            args=(f'fs rename {self.fs.name} {new_fs_name} '
+                   '--yes-i-really-mean-it'),
+            errmsgs=(f"CephFS '{self.fs.name}' is not offline. Before "
+                      "renaming a CephFS, it must be marked as down. See "
+                      "`ceph fs fail`."),
+            retval=errno.EPERM)
+
+        self.fs.getinfo()
+        keyring = self.fs.authorize(client_id, ('/', 'rw'))
+        keyring_path = self.mount_a.client_remote.mktemp(data=keyring)
+        self.mount_a.remount(client_id=client_id,
+                             client_keyring_path=keyring_path,
+                             cephfs_mntpt='/',
+                             cephfs_name=self.fs.name)
+
+        self.check_pool_application_metadata_key_value(
+            self.fs.get_data_pool_name(), 'cephfs', 'data', self.fs.name)
+        self.check_pool_application_metadata_key_value(
+            self.fs.get_metadata_pool_name(), 'cephfs', 'metadata',
+            self.fs.name)
 
 
 class TestDump(CephFSTestCase):
diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 44e28937bcbd..b11125d7247a 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -571,8 +571,12 @@ def test_volume_rename(self):
         oldvolname = self.volname
         newvolname = self._generate_random_volume_name()
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
+
+        self.run_ceph_cmd(f'fs fail {oldvolname}')
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
+        self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
         # volume name changed
@@ -592,10 +596,14 @@ def test_volume_rename_idempotency(self):
         oldvolname = self.volname
         newvolname = self._generate_random_volume_name()
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
+
+        self.run_ceph_cmd(f'fs fail {oldvolname}')
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
+        self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
         self.assertIn(newvolname, volnames)
@@ -609,6 +617,8 @@ def test_volume_rename_fails_without_confirmation_flag(self):
         That renaming volume fails without --yes-i-really-mean-it flag.
         """
         newvolname = self._generate_random_volume_name()
+
+        self.run_ceph_cmd(f'fs fail {self.volname}')
         try:
             self._fs_cmd("volume", "rename", self.volname, newvolname)
         except CommandFailedError as ce:
@@ -618,6 +628,7 @@ def test_volume_rename_fails_without_confirmation_flag(self):
         else:
             self.fail("expected renaming of FS volume to fail without the "
                       "'--yes-i-really-mean-it' flag")
+        self.run_ceph_cmd(f'fs set {self.volname} joinable true')
 
     def test_volume_rename_for_more_than_one_data_pool(self):
         """
@@ -632,8 +643,12 @@ def test_volume_rename_for_more_than_one_data_pool(self):
         self.fs.get_pool_names(refresh=True)
         orig_data_pool_names = list(self.fs.data_pools.values())
         new_metadata_pool = f"cephfs.{newvolname}.meta"
+
+        self.run_ceph_cmd(f'fs fail {oldvolname}')
         self._fs_cmd("volume", "rename", self.volname, newvolname,
                      "--yes-i-really-mean-it")
+        self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
         # volume name changed
@@ -645,6 +660,19 @@ def test_volume_rename_for_more_than_one_data_pool(self):
         # data pool names unchanged
         self.assertCountEqual(orig_data_pool_names, list(self.fs.data_pools.values()))
 
+    def test_rename_when_fs_is_online(self):
+        for m in self.mounts:
+            m.umount_wait()
+        newvolname = self._generate_random_volume_name()
+
+        self.negtest_ceph_cmd(
+            args=(f'fs volume rename {self.volname} {newvolname} '
+                   '--yes-i-really-mean-it'),
+            errmsgs=(f"CephFS '{self.volname}' is not offline. Before "
+                      "renaming a CephFS, it must be marked as down. See "
+                      "`ceph fs fail`."),
+            retval=errno.EPERM)
+
     def test_volume_info(self):
         """
         Tests the 'fs volume info' command
diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index de8695fb783c..906505224ea6 100644
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -1246,6 +1246,14 @@ class RenameFilesystemHandler : public FileSystemCommandHandler
       mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
       return -EAGAIN;
     }
+
+    // Check that no MDS daemons is up for this CephFS.
+    if (fsp->get_mds_map().get_num_up_mds() > 0) {
+      ss << "CephFS '" << fs_name << "' is not offline. Before renaming "
+	 << "a CephFS, it must be marked as down. See `ceph fs fail`.";
+      return -EPERM;
+    }
+
     for (const auto p : fsp->get_mds_map().get_data_pools()) {
       mon->osdmon()->do_application_enable(p,
 					   pg_pool_t::APPLICATION_NAME_CEPHFS,

From e268908a5a0d9954c71ee3e2d3e5606df0a19c59 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 31 Oct 2023 23:45:59 +0530
Subject: [PATCH 0371/2492] qa/cephfs: create new class for "volume fs volume
 rename" tests

Move tests for "ceph fs volume rename" command to a new class. This
makes it possible to run this group of tests in a single command.

This provides a convenient way to execute these tests which is necessary
after the changes has been made to the code for the "ceph fs volume
rename" command.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py | 190 ++++++++++++++++----------------
 1 file changed, 96 insertions(+), 94 deletions(-)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index b11125d7247a..22bfd4808e43 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -562,6 +562,102 @@ def test_volume_rm_when_mon_delete_pool_false(self):
             self.assertNotIn(pool["name"], pools,
                              "pool {0} exists after volume removal".format(pool["name"]))
 
+    def test_volume_info(self):
+        """
+        Tests the 'fs volume info' command
+        """
+        vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
+        group = self._generate_random_group_name()
+        # create subvolumegroup
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+        # get volume metadata
+        vol_info = json.loads(self._get_volume_info(self.volname))
+        for md in vol_fields:
+            self.assertIn(md, vol_info,
+                          f"'{md}' key not present in metadata of volume")
+        self.assertEqual(vol_info["used_size"], 0,
+                         "Size should be zero when volumes directory is empty")
+
+    def test_volume_info_pending_subvol_deletions(self):
+        """
+        Tests the pending_subvolume_deletions in 'fs volume info' command
+        """
+        subvolname = self._generate_random_subvolume_name()
+        # create subvolume
+        self._fs_cmd("subvolume", "create", self.volname, subvolname, "--mode=777")
+        # create 3K zero byte files
+        self._do_subvolume_io(subvolname, number_of_files=3000, file_size=0)
+        # Delete the subvolume
+        self._fs_cmd("subvolume", "rm", self.volname, subvolname)
+        # get volume metadata
+        vol_info = json.loads(self._get_volume_info(self.volname))
+        self.assertNotEqual(vol_info['pending_subvolume_deletions'], 0,
+                            "pending_subvolume_deletions should be 1")
+        # verify trash dir is clean
+        self._wait_for_trash_empty()
+
+    def test_volume_info_without_subvolumegroup(self):
+        """
+        Tests the 'fs volume info' command without subvolume group
+        """
+        vol_fields = ["pools", "mon_addrs"]
+        # get volume metadata
+        vol_info = json.loads(self._get_volume_info(self.volname))
+        for md in vol_fields:
+            self.assertIn(md, vol_info,
+                          f"'{md}' key not present in metadata of volume")
+        self.assertNotIn("used_size", vol_info,
+                         "'used_size' should not be present in absence of subvolumegroup")
+        self.assertNotIn("pending_subvolume_deletions", vol_info,
+                         "'pending_subvolume_deletions' should not be present in absence"
+                         " of subvolumegroup")
+
+    def test_volume_info_with_human_readable_flag(self):
+        """
+        Tests the 'fs volume info --human_readable' command
+        """
+        vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
+        group = self._generate_random_group_name()
+        # create subvolumegroup
+        self._fs_cmd("subvolumegroup", "create", self.volname, group)
+        # get volume metadata
+        vol_info = json.loads(self._get_volume_info(self.volname, "--human_readable"))
+        for md in vol_fields:
+            self.assertIn(md, vol_info,
+                          f"'{md}' key not present in metadata of volume")
+        units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
+        assert vol_info["used_size"][-1] in units, "unit suffix in used_size is absent"
+        assert vol_info["pools"]["data"][0]["avail"][-1] in units, "unit suffix in avail data is absent"
+        assert vol_info["pools"]["data"][0]["used"][-1] in units, "unit suffix in used data is absent"
+        assert vol_info["pools"]["metadata"][0]["avail"][-1] in units, "unit suffix in avail metadata is absent"
+        assert vol_info["pools"]["metadata"][0]["used"][-1] in units, "unit suffix in used metadata is absent"
+        self.assertEqual(int(vol_info["used_size"]), 0,
+                         "Size should be zero when volumes directory is empty")
+
+    def test_volume_info_with_human_readable_flag_without_subvolumegroup(self):
+        """
+        Tests the 'fs volume info --human_readable' command without subvolume group
+        """
+        vol_fields = ["pools", "mon_addrs"]
+        # get volume metadata
+        vol_info = json.loads(self._get_volume_info(self.volname, "--human_readable"))
+        for md in vol_fields:
+            self.assertIn(md, vol_info,
+                          f"'{md}' key not present in metadata of volume")
+        units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
+        assert vol_info["pools"]["data"][0]["avail"][-1] in units, "unit suffix in avail data is absent"
+        assert vol_info["pools"]["data"][0]["used"][-1] in units, "unit suffix in used data is absent"
+        assert vol_info["pools"]["metadata"][0]["avail"][-1] in units, "unit suffix in avail metadata is absent"
+        assert vol_info["pools"]["metadata"][0]["used"][-1] in units, "unit suffix in used metadata is absent"
+        self.assertNotIn("used_size", vol_info,
+                         "'used_size' should not be present in absence of subvolumegroup")
+        self.assertNotIn("pending_subvolume_deletions", vol_info,
+                         "'pending_subvolume_deletions' should not be present in absence"
+                         " of subvolumegroup")
+
+
+class TestRenameCmd(TestVolumesHelper):
+
     def test_volume_rename(self):
         """
         That volume, its file system and pools, can be renamed.
@@ -673,100 +769,6 @@ def test_rename_when_fs_is_online(self):
                       "`ceph fs fail`."),
             retval=errno.EPERM)
 
-    def test_volume_info(self):
-        """
-        Tests the 'fs volume info' command
-        """
-        vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
-        group = self._generate_random_group_name()
-        # create subvolumegroup
-        self._fs_cmd("subvolumegroup", "create", self.volname, group)
-        # get volume metadata
-        vol_info = json.loads(self._get_volume_info(self.volname))
-        for md in vol_fields:
-            self.assertIn(md, vol_info,
-                          f"'{md}' key not present in metadata of volume")
-        self.assertEqual(vol_info["used_size"], 0,
-                         "Size should be zero when volumes directory is empty")
-
-    def test_volume_info_pending_subvol_deletions(self):
-        """
-        Tests the pending_subvolume_deletions in 'fs volume info' command
-        """
-        subvolname = self._generate_random_subvolume_name()
-        # create subvolume
-        self._fs_cmd("subvolume", "create", self.volname, subvolname, "--mode=777")
-        # create 3K zero byte files
-        self._do_subvolume_io(subvolname, number_of_files=3000, file_size=0)
-        # Delete the subvolume
-        self._fs_cmd("subvolume", "rm", self.volname, subvolname)
-        # get volume metadata
-        vol_info = json.loads(self._get_volume_info(self.volname))
-        self.assertNotEqual(vol_info['pending_subvolume_deletions'], 0,
-                            "pending_subvolume_deletions should be 1")
-        # verify trash dir is clean
-        self._wait_for_trash_empty()
-
-    def test_volume_info_without_subvolumegroup(self):
-        """
-        Tests the 'fs volume info' command without subvolume group
-        """
-        vol_fields = ["pools", "mon_addrs"]
-        # get volume metadata
-        vol_info = json.loads(self._get_volume_info(self.volname))
-        for md in vol_fields:
-            self.assertIn(md, vol_info,
-                          f"'{md}' key not present in metadata of volume")
-        self.assertNotIn("used_size", vol_info,
-                         "'used_size' should not be present in absence of subvolumegroup")
-        self.assertNotIn("pending_subvolume_deletions", vol_info,
-                         "'pending_subvolume_deletions' should not be present in absence"
-                         " of subvolumegroup")
-
-    def test_volume_info_with_human_readable_flag(self):
-        """
-        Tests the 'fs volume info --human_readable' command
-        """
-        vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
-        group = self._generate_random_group_name()
-        # create subvolumegroup
-        self._fs_cmd("subvolumegroup", "create", self.volname, group)
-        # get volume metadata
-        vol_info = json.loads(self._get_volume_info(self.volname, "--human_readable"))
-        for md in vol_fields:
-            self.assertIn(md, vol_info,
-                          f"'{md}' key not present in metadata of volume")
-        units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
-        assert vol_info["used_size"][-1] in units, "unit suffix in used_size is absent"
-        assert vol_info["pools"]["data"][0]["avail"][-1] in units, "unit suffix in avail data is absent"
-        assert vol_info["pools"]["data"][0]["used"][-1] in units, "unit suffix in used data is absent"
-        assert vol_info["pools"]["metadata"][0]["avail"][-1] in units, "unit suffix in avail metadata is absent"
-        assert vol_info["pools"]["metadata"][0]["used"][-1] in units, "unit suffix in used metadata is absent"
-        self.assertEqual(int(vol_info["used_size"]), 0,
-                         "Size should be zero when volumes directory is empty")
-
-    def test_volume_info_with_human_readable_flag_without_subvolumegroup(self):
-        """
-        Tests the 'fs volume info --human_readable' command without subvolume group
-        """
-        vol_fields = ["pools", "mon_addrs"]
-        # get volume metadata
-        vol_info = json.loads(self._get_volume_info(self.volname, "--human_readable"))
-        for md in vol_fields:
-            self.assertIn(md, vol_info,
-                          f"'{md}' key not present in metadata of volume")
-        units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
-        assert vol_info["pools"]["data"][0]["avail"][-1] in units, "unit suffix in avail data is absent"
-        assert vol_info["pools"]["data"][0]["used"][-1] in units, "unit suffix in used data is absent"
-        assert vol_info["pools"]["metadata"][0]["avail"][-1] in units, "unit suffix in avail metadata is absent"
-        assert vol_info["pools"]["metadata"][0]["used"][-1] in units, "unit suffix in used metadata is absent"
-        self.assertNotIn("used_size", vol_info,
-                         "'used_size' should not be present in absence of subvolumegroup")
-        self.assertNotIn("pending_subvolume_deletions", vol_info,
-                         "'pending_subvolume_deletions' should not be present in absence"
-                         " of subvolumegroup")
-
-
 class TestSubvolumeGroups(TestVolumesHelper):
     """Tests for FS subvolume group operations."""
     def test_default_uid_gid_subvolume_group(self):

From 3f93d740677419756ca94dac99abc16c4e43a8cc Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Wed, 1 Nov 2023 16:09:25 +0530
Subject: [PATCH 0372/2492] cephfs: allow FS fail only if client_refuse_session
 is set

client_refuse_session must be set for a CephFS before an attempt to
rename a CephFS can be made. Add a new test for this, and update current
tests (test_admin.py and test_volumes.py) accordingly.

Fixes: https://tracker.ceph.com/issues/63154
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_admin.py   | 33 +++++++++++++++++++++++++++++++++
 qa/tasks/cephfs/test_volumes.py | 27 ++++++++++++++++++++++++++-
 src/mon/FSCommands.cc           |  8 ++++++++
 3 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index fe224b6831dd..6edea8d8bb15 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -628,9 +628,11 @@ def test_fs_rename(self):
         client_id = 'test_new_cephfs'
 
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        self.run_ceph_cmd(f'fs set {new_fs_name} refuse_client_session false')
         sleep(5)
 
         # authorize a cephx ID access to the renamed file system.
@@ -665,10 +667,12 @@ def test_fs_rename_idempotency(self):
         new_fs_name = 'new_cephfs'
 
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        self.run_ceph_cmd(f'fs set {new_fs_name} refuse_client_session false')
         sleep(5)
 
         # original file system name does not appear in `fs ls` command
@@ -689,9 +693,11 @@ def test_fs_rename_fs_new_fails_with_old_fsname_existing_pools(self):
         data_pool = self.fs.get_data_pool_name()
         metadata_pool = self.fs.get_metadata_pool_name()
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
         self.run_ceph_cmd(f'fs set {new_fs_name} joinable true')
+        self.run_ceph_cmd(f'fs set {new_fs_name} refuse_client_session false')
         sleep(5)
 
         try:
@@ -730,6 +736,7 @@ def test_fs_rename_fails_without_yes_i_really_mean_it_flag(self):
         That renaming a file system without '--yes-i-really-mean-it' flag fails.
         """
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         try:
             self.run_ceph_cmd(f"fs rename {self.fs.name} new_fs")
@@ -741,12 +748,14 @@ def test_fs_rename_fails_without_yes_i_really_mean_it_flag(self):
             self.fail("expected renaming of file system without the "
                       "'--yes-i-really-mean-it' flag to fail ")
         self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session false')
 
     def test_fs_rename_fails_for_non_existent_fs(self):
         """
         That renaming a non-existent file system fails.
         """
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         try:
             self.run_ceph_cmd("fs rename non_existent_fs new_fs --yes-i-really-mean-it")
@@ -762,6 +771,7 @@ def test_fs_rename_fails_new_name_already_in_use(self):
         self.fs2 = self.mds_cluster.newfs(name='cephfs2', create=True)
 
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         try:
             self.run_ceph_cmd(f"fs rename {self.fs.name} {self.fs2.name} --yes-i-really-mean-it")
@@ -771,6 +781,7 @@ def test_fs_rename_fails_new_name_already_in_use(self):
         else:
             self.fail("expected renaming to a new file system name that is already in use to fail.")
         self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session false')
 
     def test_fs_rename_fails_with_mirroring_enabled(self):
         """
@@ -781,6 +792,7 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
 
         self.run_ceph_cmd(f'fs mirror enable {orig_fs_name}')
         self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         sleep(5)
         try:
             self.run_ceph_cmd(f'fs rename {orig_fs_name} {new_fs_name} --yes-i-really-mean-it')
@@ -790,6 +802,7 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
             self.fail("expected renaming of a mirrored file system to fail")
         self.run_ceph_cmd(f'fs mirror disable {orig_fs_name}')
         self.run_ceph_cmd(f'fs set {self.fs.name} joinable true')
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session false')
 
     def test_rename_when_fs_is_online(self):
         '''
@@ -798,6 +811,8 @@ def test_rename_when_fs_is_online(self):
         '''
         client_id = 'test_new_cephfs'
         new_fs_name = 'new_cephfs'
+
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session true')
         self.negtest_ceph_cmd(
             args=(f'fs rename {self.fs.name} {new_fs_name} '
                    '--yes-i-really-mean-it'),
@@ -805,6 +820,7 @@ def test_rename_when_fs_is_online(self):
                       "renaming a CephFS, it must be marked as down. See "
                       "`ceph fs fail`."),
             retval=errno.EPERM)
+        self.run_ceph_cmd(f'fs set {self.fs.name} refuse_client_session false')
 
         self.fs.getinfo()
         keyring = self.fs.authorize(client_id, ('/', 'rw'))
@@ -820,6 +836,23 @@ def test_rename_when_fs_is_online(self):
             self.fs.get_metadata_pool_name(), 'cephfs', 'metadata',
             self.fs.name)
 
+    def test_rename_when_clients_not_refused(self):
+        '''
+        Test that "ceph fs rename" fails when client_refuse_session is not
+        set.
+        '''
+        self.mount_a.umount_wait(require_clean=True)
+
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+        self.negtest_ceph_cmd(
+            args=f"fs rename {self.fs.name} new_fs --yes-i-really-mean-it",
+            errmsgs=(f"CephFS '{self.fs.name}' doesn't refuse clients. "
+                      "Before renaming a CephFS, flag "
+                      "'refuse_client_session' must be set. See "
+                      "`ceph fs set`."),
+            retval=errno.EPERM)
+        self.run_ceph_cmd(f'fs fail {self.fs.name}')
+
 
 class TestDump(CephFSTestCase):
     CLIENTS_REQUIRED = 0
diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 22bfd4808e43..d203e66db301 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -669,9 +669,11 @@ def test_volume_rename(self):
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
 
         self.run_ceph_cmd(f'fs fail {oldvolname}')
+        self.run_ceph_cmd(f'fs set {oldvolname} refuse_client_session true')
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
         self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+        self.run_ceph_cmd(f'fs set {newvolname} refuse_client_session false')
 
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
@@ -694,11 +696,13 @@ def test_volume_rename_idempotency(self):
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
 
         self.run_ceph_cmd(f'fs fail {oldvolname}')
+        self.run_ceph_cmd(f'fs set {oldvolname} refuse_client_session true')
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
         self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
         self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+        self.run_ceph_cmd(f'fs set {newvolname} refuse_client_session false')
 
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
@@ -715,6 +719,7 @@ def test_volume_rename_fails_without_confirmation_flag(self):
         newvolname = self._generate_random_volume_name()
 
         self.run_ceph_cmd(f'fs fail {self.volname}')
+        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session true')
         try:
             self._fs_cmd("volume", "rename", self.volname, newvolname)
         except CommandFailedError as ce:
@@ -725,6 +730,7 @@ def test_volume_rename_fails_without_confirmation_flag(self):
             self.fail("expected renaming of FS volume to fail without the "
                       "'--yes-i-really-mean-it' flag")
         self.run_ceph_cmd(f'fs set {self.volname} joinable true')
+        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session false')
 
     def test_volume_rename_for_more_than_one_data_pool(self):
         """
@@ -741,9 +747,11 @@ def test_volume_rename_for_more_than_one_data_pool(self):
         new_metadata_pool = f"cephfs.{newvolname}.meta"
 
         self.run_ceph_cmd(f'fs fail {oldvolname}')
-        self._fs_cmd("volume", "rename", self.volname, newvolname,
+        self.run_ceph_cmd(f'fs set {oldvolname} refuse_client_session true')
+        self._fs_cmd("volume", "rename", oldvolname, newvolname,
                      "--yes-i-really-mean-it")
         self.run_ceph_cmd(f'fs set {newvolname} joinable true')
+        self.run_ceph_cmd(f'fs set {newvolname} refuse_client_session false')
 
         volumels = json.loads(self._fs_cmd('volume', 'ls'))
         volnames = [volume['name'] for volume in volumels]
@@ -761,6 +769,7 @@ def test_rename_when_fs_is_online(self):
             m.umount_wait()
         newvolname = self._generate_random_volume_name()
 
+        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session true')
         self.negtest_ceph_cmd(
             args=(f'fs volume rename {self.volname} {newvolname} '
                    '--yes-i-really-mean-it'),
@@ -768,6 +777,22 @@ def test_rename_when_fs_is_online(self):
                       "renaming a CephFS, it must be marked as down. See "
                       "`ceph fs fail`."),
             retval=errno.EPERM)
+        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session false')
+
+    def test_rename_when_clients_arent_refused(self):
+        newvolname = self._generate_random_volume_name()
+        for m in self.mounts:
+            m.umount_wait()
+
+        self.run_ceph_cmd(f'fs fail {self.volname}')
+        self.negtest_ceph_cmd(
+            args=(f'fs volume rename {self.volname} {newvolname} '
+                   '--yes-i-really-mean-it'),
+            errmsgs=(f"CephFS '{self.volname}' doesn't refuse clients. "
+                      "Before renaming a CephFS, flag "
+                      "'refuse_client_session' must be set. See "
+                      "`ceph fs set`."),
+            retval=errno.EPERM)
 
 class TestSubvolumeGroups(TestVolumesHelper):
     """Tests for FS subvolume group operations."""
diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 906505224ea6..17cc9501c786 100644
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -1254,6 +1254,14 @@ class RenameFilesystemHandler : public FileSystemCommandHandler
       return -EPERM;
     }
 
+    // Check that refuse_client_session is set.
+    if (!fsp->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+      ss << "CephFS '" << fs_name << "' doesn't refuse clients. Before "
+	 << "renaming a CephFS, flag 'refuse_client_session' must be set. "
+	 << "See `ceph fs set`.";
+      return -EPERM;
+    }
+
     for (const auto p : fsp->get_mds_map().get_data_pools()) {
       mon->osdmon()->do_application_enable(p,
 					   pg_pool_t::APPLICATION_NAME_CEPHFS,

From f637e53eb62f4f940918a793865db6629fc21275 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Thu, 2 Nov 2023 13:45:12 -0400
Subject: [PATCH 0373/2492] rgwfile: fix lock_guard decl

Reported by Leonid Usov <leonid.usov@ibm.com>.

Fixes: https://tracker.ceph.com/issues/63409

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_file.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 44ccc56e6d84..6130019df032 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -345,7 +345,7 @@ namespace rgw {
     int rc = g_rgwlib->get_fe()->execute_req(&req);
     if ((rc == 0) &&
         ((rc = req.get_ret()) == 0)) {
-      lock_guard(rgw_fh->mtx);
+      lock_guard guard(rgw_fh->mtx);
       rgw_fh->set_atime(real_clock::to_timespec(real_clock::now()));
       *bytes_read = req.nread;
     }

From abba1a8b2c0829a824ac11c75afa83e40ed230fc Mon Sep 17 00:00:00 2001
From: Gabriel BenHanokh <gbenhano@redhat.com>
Date: Wed, 14 Dec 2022 12:55:19 +0000
Subject: [PATCH 0374/2492] =?UTF-8?q?osd/SnapMapper:=20maintain=20the=20pr?=
 =?UTF-8?q?efix=5Fitr=20between=20calls=20to=20SnapMapper::get=5Fnext=5Fob?=
 =?UTF-8?q?jects=5Fto=5Ftrim()=20Maintain=20the=20prefix=5Fitr=20between?=
 =?UTF-8?q?=20calls=20to=20SnapMapper::get=5Fnext=5Fobjects=5Fto=5Ftrim()?=
 =?UTF-8?q?=20to=20prevent=20searching=20depleted=20prefixes.=20We=20got?=
 =?UTF-8?q?=208=20distinct=20hash=20prefixes=20used=20for=20searching=20ob?=
 =?UTF-8?q?jects=20owned=20by=20a=20given=20PG.=20On=20each=20call=20to=20?=
 =?UTF-8?q?SnapMapper::get=5Fnext=5Fobjects=5Fto=5Ftrim()=20we=20start=20f?=
 =?UTF-8?q?rom=20the=20first=20prefix=20even=20after=20all=20objects=20map?=
 =?UTF-8?q?ped=20to=20it=20were=20depleted.=20This=20means=20that=20we=20w?=
 =?UTF-8?q?ill=20be=20searching=20for=201=20non-existing=20prefix=20after?=
 =?UTF-8?q?=20the=20first=20prefix=20was=20depleted,=202=20after=20the=20f?=
 =?UTF-8?q?irst=20two=20prefixes=20were=20depleted...=20and=20so=20on=20un?=
 =?UTF-8?q?til=20we=20will=20search=207=20non-existing=20prefixes=20after?=
 =?UTF-8?q?=20the=20first=207=20prefixes=20were=C2=A0depleted.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is a performance improvement PR only!
It maintains the existing behavior and does not try to fix/change any of the TRIM logic.
I added an extra step after the last object is trimmed doing a full scan of the DB and only if no object was found it will return ENOENT.
This should make the new code no-worse than existing code which returns ENOENT after a full scan found no object.
It should not impact performance in real life snaps as it should only happen once per-snap.

added snap-mapper tests to rados-test-suite
disabled osd_debug_trim_objects when running (SnapMapperTest, prefix_itr) to prevent asserts(as this code does illegal inserts into DELETED snaps)
Code beautifing

Disabled the assert as there is a corner case when we retrieve the last valid object/s in a snap
The prefix_itr is advanced past the last valid value (as we completed a full scan)
If the OSD will call get_next_objects_to_trim() before the retrieved object/s was processed and removed from the SnapMapper DB it won't be found by the next call (as the prefix_itr is invalid).
The object will be found in the second-pass which will seems as if it was added after the trim was started (which is illegal) and will trigger an ASSERT

Signed-off-by: Gabriel BenHanokh <gbenhano@redhat.com>
---
 .../singleton-nomsgr/all/ceph-snapmapper.yaml |  22 ++
 qa/tasks/ceph.conf.template                   |   3 +-
 src/common/options/osd.yaml.in                |   5 +
 src/osd/SnapMapper.cc                         | 100 ++++--
 src/osd/SnapMapper.h                          |  22 +-
 src/test/CMakeLists.txt                       |   4 +
 src/test/test_snap_mapper.cc                  | 324 ++++++++++++++++--
 7 files changed, 427 insertions(+), 53 deletions(-)
 create mode 100644 qa/suites/rados/singleton-nomsgr/all/ceph-snapmapper.yaml

diff --git a/qa/suites/rados/singleton-nomsgr/all/ceph-snapmapper.yaml b/qa/suites/rados/singleton-nomsgr/all/ceph-snapmapper.yaml
new file mode 100644
index 000000000000..cbfd08620e76
--- /dev/null
+++ b/qa/suites/rados/singleton-nomsgr/all/ceph-snapmapper.yaml
@@ -0,0 +1,22 @@
+openstack:
+  - volumes: # attached to each instance
+      count: 3
+      size: 10 # GB
+roles:
+- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0]
+
+overrides:
+  ceph:
+    pre-mgr-commands:
+      - sudo ceph config set mgr mgr_pool false --force
+    log-ignorelist:
+    - but it is still running
+    - overall HEALTH_
+    - \(POOL_APP_NOT_ENABLED\)
+
+tasks:
+- install:
+- ceph:
+- exec:
+    client.0:
+    - ceph_test_snap_mapper
diff --git a/qa/tasks/ceph.conf.template b/qa/tasks/ceph.conf.template
index a9cce29539aa..7ae2b83c2951 100644
--- a/qa/tasks/ceph.conf.template
+++ b/qa/tasks/ceph.conf.template
@@ -55,8 +55,9 @@
 	osd debug shutdown = true
         osd debug op order = true
         osd debug verify stray on activate = true
+        osd debug trim objects = true
 
-	osd open classes on start = true
+        osd open classes on start = true
         osd debug pg log writeout = true
 
 	osd deep scrub update digest min age = 30
diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 5d8d40cf12d1..f4c60db26b77 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -194,6 +194,11 @@ options:
     load on busy clusters.
   default: false
   with_legacy: true
+- name: osd_debug_trim_objects
+  type: bool
+  level: advanced
+  desc: Asserts that no clone-objects were added to a snap after we start trimming it
+  default: false
 - name: osd_repair_during_recovery
   type: bool
   level: advanced
diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index 7893bc08fdcb..e41cd1df08e3 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -438,7 +438,7 @@ void SnapMapper::clear_snaps(
   to_remove.insert(to_object_key(oid));
   if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
     for (auto& i : to_remove) {
-      dout(20) << __func__ << " rm " << i << dendl;
+      dout(20) << __func__ << "::rm " << i << dendl;
     }
   }
   backend.remove_keys(to_remove, t);
@@ -457,7 +457,7 @@ void SnapMapper::set_snaps(
   dout(20) << __func__ << " " << oid << " " << in.snaps << dendl;
   if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
     for (auto& i : to_set) {
-      dout(20) << __func__ << " set " << i.first << dendl;
+      dout(20) << __func__ << "::set " << i.first << dendl;
     }
   }
   backend.set_keys(to_set, t);
@@ -540,36 +540,48 @@ void SnapMapper::add_oid(
   backend.set_keys(to_add, t);
 }
 
-int SnapMapper::get_next_objects_to_trim(
+// reset the prefix iterator to the first prefix hash
+void SnapMapper::reset_prefix_itr(snapid_t snap, const char *s)
+{
+  if (prefix_itr_snap == CEPH_NOSNAP) {
+    dout(10) << __func__ << "::from <CEPH_NOSNAP> to <" << snap << "> ::" << s << dendl;
+  }
+  else if (snap == CEPH_NOSNAP) {
+    dout(10) << __func__ << "::from <"<< prefix_itr_snap << "> to <CEPH_NOSNAP> ::" << s << dendl;
+  }
+  else if (prefix_itr_snap == snap) {
+    dout(10) << __func__ << "::with the same snapid <" << snap << "> ::" << s << dendl;
+  }
+  else {
+    // This is unexpected!!
+    dout(10) << __func__ << "::from <"<< prefix_itr_snap << "> to <" << snap << "> ::" << s << dendl;
+  }
+  prefix_itr_snap = snap;
+  prefix_itr      = prefixes.begin();
+}
+
+void SnapMapper::get_objects_by_prefixes(
   snapid_t snap,
   unsigned max,
   vector<hobject_t> *out)
 {
-  ceph_assert(out);
-  ceph_assert(out->empty());
-
-  // if max would be 0, we return ENOENT and the caller would mistakenly
-  // trim the snaptrim queue
-  ceph_assert(max > 0);
-  int r = 0;
-
-  /// \todo cache the prefixes-set in update_bits()
-  for (set<string>::iterator i = prefixes.begin();
-       i != prefixes.end() && out->size() < max && r == 0;
-       ++i) {
-    string prefix(get_prefix(pool, snap) + *i);
+  /// maintain the prefix_itr between calls to avoid searching depleted prefixes
+  for ( ; prefix_itr != prefixes.end(); prefix_itr++) {
+    string prefix(get_prefix(pool, snap) + *prefix_itr);
     string pos = prefix;
     while (out->size() < max) {
       pair<string, ceph::buffer::list> next;
-      r = backend.get_next(pos, &next);
+      // access RocksDB (an expensive operation!)
+      int r = backend.get_next(pos, &next);
       dout(20) << __func__ << " get_next(" << pos << ") returns " << r
 	       << " " << next << dendl;
       if (r != 0) {
-	break; // Done
+	return; // Done
       }
 
       if (next.first.substr(0, prefix.size()) !=
 	  prefix) {
+	// TBD: we access the DB twice for the first object of each iterator...
 	break; // Done with this prefix
       }
 
@@ -583,7 +595,56 @@ int SnapMapper::get_next_objects_to_trim(
       out->push_back(next_decoded.second);
       pos = next.first;
     }
+
+    if (out->size() >= max) {
+      return;
+    }
   }
+}
+
+int SnapMapper::get_next_objects_to_trim(
+  snapid_t snap,
+  unsigned max,
+  vector<hobject_t> *out)
+{
+  dout(20) << __func__ << "::snapid=" << snap << dendl;
+  ceph_assert(out);
+  ceph_assert(out->empty());
+
+  // if max would be 0, we return ENOENT and the caller would mistakenly
+  // trim the snaptrim queue
+  ceph_assert(max > 0);
+
+  // The prefix_itr is bound to a prefix_itr_snap so if we trim another snap
+  // we must reset the prefix_itr (should not happen normally)
+  if (prefix_itr_snap != snap) {
+    if (prefix_itr_snap == CEPH_NOSNAP) {
+      reset_prefix_itr(snap, "Trim begins");
+    }
+    else {
+      reset_prefix_itr(snap, "Unexpected snap change");
+    }
+  }
+
+  // when reaching the end of the DB reset the prefix_ptr and verify
+  // we didn't miss objects which were added after we started trimming
+  // This should never happen in reality because the snap was logically deleted
+  // before trimming starts (and so no new clone-objects could be added)
+  // For more info see PG::filter_snapc()
+  //
+  // We still like to be extra careful and run one extra loop over all prefixes
+  get_objects_by_prefixes(snap, max, out);
+  if (unlikely(out->size() == 0)) {
+    reset_prefix_itr(snap, "Second pass trim");
+    get_objects_by_prefixes(snap, max, out);
+
+    if (unlikely(out->size() > 0)) {
+      derr << __func__ << "::New Clone-Objects were added to Snap " << snap
+	   << " after trimming was started" << dendl;
+    }
+    reset_prefix_itr(CEPH_NOSNAP, "Trim was completed successfully");
+  }
+
   if (out->size() == 0) {
     return -ENOENT;
   } else {
@@ -591,7 +652,6 @@ int SnapMapper::get_next_objects_to_trim(
   }
 }
 
-
 int SnapMapper::remove_oid(
   const hobject_t &oid,
   MapCacher::Transaction<std::string, ceph::buffer::list> *t)
@@ -621,7 +681,7 @@ int SnapMapper::_remove_oid(
   }
   if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
     for (auto& i : to_remove) {
-      dout(20) << __func__ << " rm " << i << dendl;
+      dout(20) << __func__ << "::rm " << i << dendl;
     }
   }
   backend.remove_keys(to_remove, t);
diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index eb43a23c2b0e..a28b25970fb4 100644
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -281,6 +281,20 @@ class SnapMapper : public Scrub::SnapMapReaderI {
   tl::expected<object_snaps, SnapMapReaderI::result_t> get_snaps_common(
     const hobject_t &hoid) const;
 
+  /// file @out vector with the first objects with @snap as a snap
+  void get_objects_by_prefixes(
+    snapid_t snap,
+    unsigned max,
+    std::vector<hobject_t> *out);
+
+  std::set<std::string>           prefixes;
+  // maintain a current active prefix
+  std::set<std::string>::iterator prefix_itr;
+  // associate the active prefix with a snap
+  snapid_t                        prefix_itr_snap;
+
+  // reset the prefix iterator to the first prefix hash
+  void reset_prefix_itr(snapid_t snap, const char *s);
  public:
   static std::string make_shard_prefix(shard_id_t shard) {
     if (shard == shard_id_t::NO_SHARD)
@@ -290,6 +304,7 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     ceph_assert(r < (int)sizeof(buf));
     return std::string(buf, r) + '_';
   }
+
   uint32_t mask_bits;
   const uint32_t match;
   std::string last_key_checked;
@@ -309,7 +324,6 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     update_bits(mask_bits);
   }
 
-  std::set<std::string> prefixes;
   /// Update bits in case of pg split or merge
   void update_bits(
     uint32_t new_bits  ///< [in] new split bits
@@ -323,6 +337,12 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     for (auto i = _prefixes.begin(); i != _prefixes.end(); ++i) {
       prefixes.insert(shard_prefix + *i);
     }
+
+    reset_prefix_itr(CEPH_NOSNAP, "update_bits");
+  }
+
+  const std::set<std::string>::iterator get_prefix_itr() {
+    return prefix_itr;
   }
 
   /// Update snaps for oid, empty new_snaps removes the mapping
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index f8e2f2a1b011..ecd7dddda8d3 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -466,6 +466,10 @@ add_executable(ceph_test_snap_mapper
   $<TARGET_OBJECTS:unit-main>
   )
 target_link_libraries(ceph_test_snap_mapper osd global ${BLKID_LIBRARIES} ${UNITTEST_LIBS})
+
+install(TARGETS
+  ceph_test_snap_mapper
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif(NOT WIN32)
 
 add_executable(ceph_test_stress_watch
diff --git a/src/test/test_snap_mapper.cc b/src/test/test_snap_mapper.cc
index e502892cc42f..9fe726afb1f9 100644
--- a/src/test/test_snap_mapper.cc
+++ b/src/test/test_snap_mapper.cc
@@ -461,7 +461,19 @@ class MapperVerifier {
     uint32_t bits)
     : driver(driver),
       mapper(new SnapMapper(g_ceph_context, driver, mask, bits, 0, shard_id_t(1))),
-             mask(mask), bits(bits) {}
+	     mask(mask), bits(bits) {}
+
+  hobject_t create_hobject(
+    unsigned           idx,
+    snapid_t           snapid,
+    int64_t            pool,
+    const std::string& nspace) {
+    const object_t    oid("OID" + std::to_string(idx));
+    const std::string key("KEY" + std::to_string(idx));
+    const uint32_t    hash = (idx & ((~0)<<bits)) | (mask & ~((~0)<<bits));
+
+    return hobject_t(oid, key, snapid, hash, pool, nspace);
+  }
 
   hobject_t random_hobject() {
     return hobject_t(
@@ -480,9 +492,28 @@ class MapperVerifier {
     }
   }
 
-  void create_snap() {
-    snap_to_hobject[next];
+  snapid_t create_snap() {
+    snapid_t snapid = next;
+    snap_to_hobject[snapid];
     ++next;
+
+    return snapid;
+  }
+
+  // must be called with lock held to protect access to
+  // hobject_to_snap and snap_to_hobject
+  void add_object_to_snaps(const hobject_t & obj, const set<snapid_t> &snaps) {
+    hobject_to_snap[obj] = snaps;
+    for (auto snap : snaps) {
+      map<snapid_t, set<hobject_t> >::iterator j = snap_to_hobject.find(snap);
+      ceph_assert(j != snap_to_hobject.end());
+      j->second.insert(obj);
+    }
+    {
+      PausyAsyncMap::Transaction t;
+      mapper->add_oid(obj, snaps, &t);
+      driver->submit(&t);
+    }
   }
 
   void create_object() {
@@ -494,20 +525,9 @@ class MapperVerifier {
       obj = random_hobject();
     } while (hobject_to_snap.count(obj));
 
-    set<snapid_t> &snaps = hobject_to_snap[obj];
+    set<snapid_t> snaps;
     choose_random_snaps(1 + (rand() % 20), &snaps);
-    for (set<snapid_t>::iterator i = snaps.begin();
-	 i != snaps.end();
-	 ++i) {
-      map<snapid_t, set<hobject_t> >::iterator j = snap_to_hobject.find(*i);
-      ceph_assert(j != snap_to_hobject.end());
-      j->second.insert(obj);
-    }
-    {
-      PausyAsyncMap::Transaction t;
-      mapper->add_oid(obj, snaps, &t);
-      driver->submit(&t);
-    }
+    add_object_to_snaps(obj, snaps);
   }
 
   std::pair<std::string, ceph::buffer::list> to_raw(
@@ -535,27 +555,23 @@ class MapperVerifier {
     return mapper->make_purged_snap_key(std::forward<Args>(args)...);
   }
 
-  void trim_snap() {
-    std::lock_guard l{lock};
-    if (snap_to_hobject.empty())
-      return;
-    map<snapid_t, set<hobject_t> >::iterator snap =
-      rand_choose(snap_to_hobject);
-    set<hobject_t> hobjects = snap->second;
-
+  // must be called with lock held to protect access to
+  // snap_to_hobject and hobject_to_snap
+  int trim_snap(snapid_t snapid, unsigned max_count, vector<hobject_t> & out) {
+    set<hobject_t>&   hobjects = snap_to_hobject[snapid];
     vector<hobject_t> hoids;
-    while (mapper->get_next_objects_to_trim(
-	     snap->first, rand() % 5 + 1, &hoids) == 0) {
+    int ret = mapper->get_next_objects_to_trim(snapid, max_count, &hoids);
+    if (ret == 0) {
+      out.insert(out.end(), hoids.begin(), hoids.end());
       for (auto &&hoid: hoids) {
 	ceph_assert(!hoid.is_max());
 	ceph_assert(hobjects.count(hoid));
 	hobjects.erase(hoid);
 
-	map<hobject_t, set<snapid_t>>::iterator j =
-	  hobject_to_snap.find(hoid);
-	ceph_assert(j->second.count(snap->first));
+	map<hobject_t, set<snapid_t>>::iterator j = hobject_to_snap.find(hoid);
+	ceph_assert(j->second.count(snapid));
 	set<snapid_t> old_snaps(j->second);
-	j->second.erase(snap->first);
+	j->second.erase(snapid);
 
 	{
 	  PausyAsyncMap::Transaction t;
@@ -573,6 +589,48 @@ class MapperVerifier {
       }
       hoids.clear();
     }
+    return ret;
+  }
+
+  // must be called with lock held to protect access to
+  // snap_to_hobject and hobject_to_snap in trim_snap
+  // will keep trimming until reaching max_count or failing a call to trim_snap()
+  void trim_snap_force(snapid_t           snapid,
+		       unsigned           max_count,
+		       vector<hobject_t>& out) {
+    int               guard = 1000;
+    vector<hobject_t> tmp;
+    unsigned          prev_size = 0;
+    while (tmp.size() < max_count) {
+      unsigned req_size = max_count - tmp.size();
+      // each call adds more objects into the tmp vector
+      trim_snap(snapid, req_size, tmp);
+      if (prev_size < tmp.size()) {
+	prev_size = tmp.size();
+      }
+      else{
+	// the tmp vector size was not increased in the last call
+	// which means we were unable to find anything to trim
+	break;
+      }
+      ceph_assert(--guard > 0);
+    }
+    out.insert(out.end(), tmp.begin(), tmp.end());
+  }
+
+  void trim_snap() {
+    std::lock_guard l{lock};
+    if (snap_to_hobject.empty()) {
+      return;
+    }
+    int ret = 0;
+    map<snapid_t, set<hobject_t> >::iterator snap = rand_choose(snap_to_hobject);
+    do {
+      int max_count = rand() % 5 + 1;
+      vector<hobject_t> out;
+      ret = trim_snap(snap->first, max_count, out);
+    } while(ret == 0);
+    set<hobject_t> hobjects = snap->second;
     ceph_assert(hobjects.empty());
     snap_to_hobject.erase(snap);
   }
@@ -612,6 +670,189 @@ class MapperVerifier {
     ceph_assert(r == 0);
     ASSERT_EQ(snaps, obj->second);
   }
+
+  void test_prefix_itr() {
+    // protects access to snap_to_hobject and hobject_to_snap
+    std::lock_guard   l{lock};
+    snapid_t          snapid = create_snap();
+    // we initialize 32 PGS
+    ceph_assert(bits == 5);
+
+    const int64_t     pool(0);
+    const std::string nspace("GBH");
+    set<snapid_t>     snaps = { snapid };
+    set<hobject_t>&   hobjects = snap_to_hobject[snapid];
+    vector<hobject_t> trimmed_objs;
+    vector<hobject_t> stored_objs;
+
+    // add objects 0, 32, 64, 96, 128, 160, 192, 224
+    // which should hit all the prefixes
+    for (unsigned idx = 0; idx < 8; idx++) {
+      hobject_t hobj = create_hobject(idx * 32, snapid, pool, nspace);
+      add_object_to_snaps(hobj, snaps);
+      stored_objs.push_back(hobj);
+    }
+    ceph_assert(hobjects.size() == 8);
+
+    // trim 0, 32, 64, 96
+    trim_snap(snapid, 4, trimmed_objs);
+    ceph_assert(hobjects.size() == 4);
+
+    // add objects (3, 35, 67) before the prefix_itr position
+    // to force an iteartor reset later
+    for (unsigned idx = 0; idx < 3; idx++) {
+      hobject_t hobj = create_hobject(idx * 32 + 3, snapid, pool, nspace);
+      add_object_to_snaps(hobj, snaps);
+      stored_objs.push_back(hobj);
+    }
+    ceph_assert(hobjects.size() == 7);
+
+    // will now trim 128, 160, 192, 224
+    trim_snap(snapid, 4, trimmed_objs);
+    ceph_assert(hobjects.size() == 3);
+
+    // finally, trim 3, 35, 67
+    // This will force a reset to the prefix_itr (which is what we test here)
+    trim_snap(snapid, 3, trimmed_objs);
+    ceph_assert(hobjects.size() == 0);
+
+    ceph_assert(trimmed_objs.size() == 11);
+    // trimmed objs must be in the same order they were inserted
+    // this will prove that the second call to add_object_to_snaps inserted
+    // them before the current prefix_itr
+    ceph_assert(trimmed_objs.size() == stored_objs.size());
+    ceph_assert(std::equal(trimmed_objs.begin(), trimmed_objs.end(),
+			   stored_objs.begin()));
+    snap_to_hobject.erase(snapid);
+  }
+
+  // insert 256 objects which should populate multiple prefixes
+  // trim until we change prefix and then insert an old object
+  // which we know for certain belongs to a prefix before prefix_itr
+  void test_prefix_itr2() {
+    // protects access to snap_to_hobject and hobject_to_snap
+    std::lock_guard   l{lock};
+    snapid_t          snapid = create_snap();
+    // we initialize 32 PGS
+    ceph_assert(bits == 5);
+
+    const int64_t     pool(0);
+    const std::string nspace("GBH");
+    set<snapid_t>     snaps = { snapid };
+    vector<hobject_t> trimmed_objs;
+    vector<hobject_t> stored_objs;
+
+    constexpr unsigned MAX_IDX = 256;
+    for (unsigned idx = 0; idx < MAX_IDX; idx++) {
+      hobject_t hobj = create_hobject(idx, snapid, pool, nspace);
+      add_object_to_snaps(hobj, snaps);
+      stored_objs.push_back(hobj);
+    }
+
+    hobject_t dup_hobj;
+    bool      found = false;
+    trim_snap(snapid, 1, trimmed_objs);
+    const std::set<std::string>::iterator itr = mapper->get_prefix_itr();
+    for (unsigned idx = 1; idx < MAX_IDX + 1; idx++) {
+      trim_snap(snapid, 1, trimmed_objs);
+      if (!found && mapper->get_prefix_itr() != itr) {
+	// we changed prefix -> insert an OBJ belonging to perv prefix
+	dup_hobj = create_hobject(idx - 1, snapid, pool, nspace);
+	add_object_to_snaps(dup_hobj, snaps);
+	stored_objs.push_back(dup_hobj);
+	found = true;
+      }
+    }
+    ceph_assert(found);
+
+    sort(trimmed_objs.begin(), trimmed_objs.end());
+    sort(stored_objs.begin(),  stored_objs.end());
+    ceph_assert(trimmed_objs.size() == MAX_IDX+1);
+    ceph_assert(trimmed_objs.size() == stored_objs.size());
+    ceph_assert(std::equal(trimmed_objs.begin(), trimmed_objs.end(),
+			   stored_objs.begin()));
+    snap_to_hobject.erase(snapid);
+  }
+
+  void add_rand_hobjects(unsigned           count,
+			 snapid_t           snapid,
+			 int64_t            pool,
+			 const std::string& nspace,
+			 vector<hobject_t>& stored_objs) {
+    constexpr unsigned MAX_VAL = 1000;
+    set<snapid_t> snaps = { snapid };
+    for (unsigned i = 0; i < count; i++) {
+      hobject_t hobj;
+      do {
+	unsigned val = rand() % MAX_VAL;
+	hobj = create_hobject(val, snapid, pool, nspace);
+      }while (hobject_to_snap.count(hobj));
+      add_object_to_snaps(hobj, snaps);
+      stored_objs.push_back(hobj);
+    }
+  }
+
+  // Start with a set of random objects then run a partial trim
+  // followed by another random insert
+  // This should cause *some* objects to be added before the prefix_itr
+  // and will verify that we still remove them
+  void test_prefix_itr_rand() {
+    // protects access to snap_to_hobject and hobject_to_snap
+    std::lock_guard   l{lock};
+    snapid_t          snapid = create_snap();
+    // we initialize 32 PGS
+    ceph_assert(bits == 5);
+
+    const int64_t     pool(0);
+    const std::string nspace("GBH");
+    vector<hobject_t> trimmed_objs;
+    vector<hobject_t> stored_objs;
+    set<hobject_t>&   hobjects = snap_to_hobject[snapid];
+    ceph_assert(hobjects.size() == 0);
+
+    // add 100 random objects
+    add_rand_hobjects(100, snapid, pool, nspace, stored_objs);
+    ceph_assert(hobjects.size() == 100);
+
+    // trim the first 75 objects leaving 25 objects
+    trim_snap(snapid, 75, trimmed_objs);
+    ceph_assert(hobjects.size() == 25);
+
+    // add another 25 random objects (now we got 50 objects)
+    add_rand_hobjects(25, snapid, pool, nspace, stored_objs);
+    ceph_assert(hobjects.size() == 50);
+
+    // trim 49 objects leaving a single object
+    // we must use a wrapper function to keep trimming while until -ENOENT
+    trim_snap_force(snapid, 49, trimmed_objs);
+    ceph_assert(hobjects.size() == 1);
+
+    // add another 9 random objects (now we got 10 objects)
+    add_rand_hobjects(9, snapid, pool, nspace, stored_objs);
+    ceph_assert(hobjects.size() == 10);
+
+    // trim 10 objects leaving no object in store
+    trim_snap_force(snapid, 10, trimmed_objs);
+    ceph_assert(hobjects.size() == 0);
+
+    // add 10 random objects (now we got 10 objects)
+    add_rand_hobjects(10, snapid, pool, nspace, stored_objs);
+    ceph_assert(hobjects.size() == 10);
+
+    // trim 10 objects leaving no object in store
+    trim_snap_force(snapid, 10, trimmed_objs);
+    ceph_assert(hobjects.size() == 0);
+
+    sort(trimmed_objs.begin(), trimmed_objs.end());
+    sort(stored_objs.begin(),  stored_objs.end());
+    ceph_assert(trimmed_objs.size() == 144);
+    ceph_assert(trimmed_objs.size() == stored_objs.size());
+
+    bool are_equal = std::equal(trimmed_objs.begin(), trimmed_objs.end(),
+				stored_objs.begin());
+    ceph_assert(are_equal);
+    snap_to_hobject.erase(snapid);
+  }
 };
 
 class SnapMapperTest : public ::testing::Test {
@@ -675,6 +916,27 @@ class SnapMapperTest : public ::testing::Test {
   }
 };
 
+// This test creates scenarios which are impossible to get with normal code.
+// The normal code deletes the snap before calling TRIM and so no new clones
+// can be added to that snap.
+// Our test calls get_next_objects_to_trim() *without* deleting the snap first.
+// This allows us to add objects to the (non-deleted) snap after trimming began.
+// We test that SnapTrim will find them even when added into positions before the prefix_itr.
+// Since those tests are doing illegal inserts we must disable osd_debug_trim_objects
+// during those tests as otherwise the code will assert.
+TEST_F(SnapMapperTest, prefix_itr) {
+  bool orig_val = g_ceph_context->_conf.get_val<bool>("osd_debug_trim_objects");
+  std::cout << "osd_debug_trim_objects = " << orig_val << std::endl;
+  g_ceph_context->_conf.set_val("osd_debug_trim_objects", std::to_string(false));
+  init(32);
+  get_tester().test_prefix_itr();
+  get_tester().test_prefix_itr2();
+  get_tester().test_prefix_itr_rand();
+  g_ceph_context->_conf.set_val("osd_debug_trim_objects", std::to_string(orig_val));
+  bool curr_val = g_ceph_context->_conf.get_val<bool>("osd_debug_trim_objects");
+  ceph_assert(curr_val == orig_val);
+}
+
 TEST_F(SnapMapperTest, Simple) {
   init(1);
   get_tester().create_snap();
@@ -806,7 +1068,7 @@ class DirectMapper {
   DirectMapper(
     uint32_t mask,
     uint32_t bits)
-   : mapper(new SnapMapper(g_ceph_context, driver.get(), mask, bits, 0, shard_id_t(1))), 
+    : mapper(new SnapMapper(g_ceph_context, driver.get(), mask, bits, 0, shard_id_t(1))),
              mask(mask), bits(bits) {}
 
   hobject_t random_hobject() {

From 6b2e616b9321774ac2ccc54ab19233894b9793c5 Mon Sep 17 00:00:00 2001
From: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
Date: Fri, 3 Nov 2023 00:41:09 +0200
Subject: [PATCH 0375/2492] Do not encode slashes

Encoding the slash in object names breaks when the object we're trying
to fetch is inside a folder. A side effect of this is that the cloud
sync module fails to sync folders and objects nested within.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
---
 src/rgw/rgw_rest_client.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index b0b8fcc84f30..153f637c67bf 100644
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -803,7 +803,8 @@ static void send_prepare_convert(const rgw_obj& obj, string *resource)
 {
   string urlsafe_bucket, urlsafe_object;
   url_encode(obj.bucket.get_key(':', 0), urlsafe_bucket);
-  url_encode(obj.key.name, urlsafe_object);
+  // do not encode slash. It leads to 404 errors when fetching objects inside folders.
+  url_encode(obj.key.name, urlsafe_object, false);
   *resource = urlsafe_bucket + "/" + urlsafe_object;
 }
 

From 822e6b01f909dff79b336bb2fc029de4663b428a Mon Sep 17 00:00:00 2001
From: Dan van der Ster <dan.vanderster@clyso.com>
Date: Thu, 2 Nov 2023 16:47:48 -0700
Subject: [PATCH 0376/2492] qa/tasks/cephadm: enable mon_cluster_log_to_file

Without cluster_log_to_file we have nothing to grep for errors:

2023-10-27T16:06:59.111 DEBUG:teuthology.orchestra.run.smithi150:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/38cc7fce-74d9-11ee-8db9-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1
2023-10-27T16:06:59.141 INFO:teuthology.orchestra.run.smithi150.stderr:grep: /var/log/ceph/38cc7fce-74d9-11ee-8db9-212e2dc638e7/ceph.log: No such file or directory

Set mon_cluster_log_to_file = true.

Fixes: https://tracker.ceph.com/issues/63425
Signed-off-by: Dan van der Ster <dan.vanderster@clyso.com>
---
 qa/tasks/cephadm.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/tasks/cephadm.conf b/qa/tasks/cephadm.conf
index 9ec08a346ec2..374adfbab948 100644
--- a/qa/tasks/cephadm.conf
+++ b/qa/tasks/cephadm.conf
@@ -3,6 +3,7 @@
 log_to_file = true
 log_to_stderr = false
 log to journald = false
+mon cluster log to file = true
 mon cluster log file level = debug
 
 mon clock drift allowed = 1.000

From c4275c9842cbd5f80b87b84a34bcb56d480129fc Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Fri, 3 Nov 2023 03:18:32 +0000
Subject: [PATCH 0377/2492] tools/ceph_dedup_tool: add a test to check the
 memory limit works correctly

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 qa/workunits/rados/test_dedup_tool.sh | 71 +++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh
index 352e9ef09c19..381094c57c27 100755
--- a/qa/workunits/rados/test_dedup_tool.sh
+++ b/qa/workunits/rados/test_dedup_tool.sh
@@ -515,6 +515,76 @@ function test_sample_dedup_snap()
   $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
 }
 
+function test_dedup_memory_limit()
+{
+  CHUNK_POOL=dedup_chunk_pool
+  $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+  $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+
+  sleep 2
+
+  run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+  run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
+
+  # 6 dedupable objects
+  CONTENT_1="There hiHI"
+  echo $CONTENT_1 > foo
+  for num in `seq 1 6`
+  do
+    $RADOS_TOOL -p $POOL put foo_$num ./foo
+  done
+
+  # 3 Unique objects
+  for num in `seq 7 9`
+  do
+    CONTENT_="There hiHI"$num
+    echo $CONTENT_ > foo
+    $RADOS_TOOL -p $POOL put foo_$num ./foo
+  done
+
+  # 6 dedupable objects
+  CONTENT_2="There hiHIhi"
+  echo $CONTENT_2 > foo
+  for num in `seq 10 15`
+  do
+    $RADOS_TOOL -p $POOL put foo_$num ./foo
+  done
+
+  #Since the memory limit is 100 bytes, adding 3 unique objects causes a memory drop, leaving
+  #the chunk of the 6 dupable objects. If we then add 6 dedupable objects to the pool,
+  #the crawler should find dedupable chunks because it free memory space through the memory drop before.
+  # 1 entry == 46 bytes
+
+  sleep 2
+
+  # Execute dedup crawler
+  RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 2 --sampling-ratio 100 --fpstore-threshold 100)
+
+  CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}')
+  CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}')
+
+  RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_1 | grep foo)
+  if [ -z "$RESULT" ] ; then
+    $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+    $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+    die "There is no expected chunk object"
+  fi
+
+  RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_2 | grep foo)
+  if [ -z "$RESULT" ] ; then
+    $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+    $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+    die "There is no expected chunk object"
+  fi
+
+  rm -rf ./foo
+  for num in `seq 1 15`
+  do
+    $RADOS_TOOL -p $POOL rm foo_$num
+  done
+
+  $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+}
 
 test_dedup_ratio_fixed
 test_dedup_chunk_scrub
@@ -522,6 +592,7 @@ test_dedup_chunk_repair
 test_dedup_object
 test_sample_dedup
 test_sample_dedup_snap
+test_dedup_memory_limit
 
 $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
 

From 329df4959d08e9bc90d6e1d83f99bd344a13dc1e Mon Sep 17 00:00:00 2001
From: Ville Ojamo <14869000+bluikko@users.noreply.github.com>
Date: Fri, 3 Nov 2023 12:44:00 +0700
Subject: [PATCH 0378/2492] doc/cephadm/services: remove excess rendered
 indentation in osd.rst

Start bash command blocks at the left margin, removing
excessive padding/indentation that would render the
block too much towards the right.

At the same time ident the source consistently:
- Two spaces for command blocks and output blocks.
- Four spaces for notes, code blocks.

There seems to be no uniform style for this, sometimes
commands are indented with three spaces but it would
seem two spaces is common. In the end it all renders
the same I guess.

Signed-off-by: Ville Ojamo <14869000+bluikko@users.noreply.github.com>
---
 doc/cephadm/services/osd.rst | 81 ++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 41 deletions(-)

diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst
index 4031257bf582..96d05d5dc0bd 100644
--- a/doc/cephadm/services/osd.rst
+++ b/doc/cephadm/services/osd.rst
@@ -15,10 +15,9 @@ To print a list of devices discovered by ``cephadm``, run this command:
 
 .. prompt:: bash #
 
-    ceph orch device ls [--hostname=...] [--wide] [--refresh]
+  ceph orch device ls [--hostname=...] [--wide] [--refresh]
 
-Example
-::
+Example::
 
   Hostname  Path      Type  Serial              Size   Health   Ident  Fault  Available
   srv-01    /dev/sdb  hdd   15P0A0YFFRD6         300G  Unknown  N/A    N/A    No
@@ -44,7 +43,7 @@ enable cephadm's "enhanced device scan" option as follows;
 
 .. prompt:: bash #
 
-    ceph config set mgr mgr/cephadm/device_enhanced_scan true
+  ceph config set mgr mgr/cephadm/device_enhanced_scan true
 
 .. warning::
     Although the libstoragemgmt library performs standard SCSI inquiry calls,
@@ -175,16 +174,16 @@ will happen without actually creating the OSDs.
 
 For example:
 
-   .. prompt:: bash #
+.. prompt:: bash #
 
-     ceph orch apply osd --all-available-devices --dry-run
+  ceph orch apply osd --all-available-devices --dry-run
 
-   ::
+::
 
-     NAME                  HOST  DATA      DB  WAL
-     all-available-devices node1 /dev/vdb  -   -
-     all-available-devices node2 /dev/vdc  -   -
-     all-available-devices node3 /dev/vdd  -   -
+  NAME                  HOST  DATA      DB  WAL
+  all-available-devices node1 /dev/vdb  -   -
+  all-available-devices node2 /dev/vdc  -   -
+  all-available-devices node3 /dev/vdd  -   -
 
 .. _cephadm-osd-declarative:
 
@@ -199,9 +198,9 @@ command completes will be automatically found and added to the cluster.
 
 We will examine the effects of the following command:
 
-   .. prompt:: bash #
+.. prompt:: bash #
 
-     ceph orch apply osd --all-available-devices
+  ceph orch apply osd --all-available-devices
 
 After running the above command: 
 
@@ -214,17 +213,17 @@ If you want to avoid this behavior (disable automatic creation of OSD on availab
 
 .. prompt:: bash #
 
-   ceph orch apply osd --all-available-devices --unmanaged=true
+  ceph orch apply osd --all-available-devices --unmanaged=true
 
 .. note::
 
-  Keep these three facts in mind:
+    Keep these three facts in mind:
 
-  - The default behavior of ``ceph orch apply`` causes cephadm constantly to reconcile. This means that cephadm creates OSDs as soon as new drives are detected.
+    - The default behavior of ``ceph orch apply`` causes cephadm constantly to reconcile. This means that cephadm creates OSDs as soon as new drives are detected.
 
-  - Setting ``unmanaged: True`` disables the creation of OSDs. If ``unmanaged: True`` is set, nothing will happen even if you apply a new OSD service.
+    - Setting ``unmanaged: True`` disables the creation of OSDs. If ``unmanaged: True`` is set, nothing will happen even if you apply a new OSD service.
 
-  - ``ceph orch daemon add`` creates OSDs, but does not add an OSD service.
+    - ``ceph orch daemon add`` creates OSDs, but does not add an OSD service.
 
 * For cephadm, see also :ref:`cephadm-spec-unmanaged`.
 
@@ -252,7 +251,7 @@ Example:
 
 Expected output::
 
-   Scheduled OSD(s) for removal
+  Scheduled OSD(s) for removal
 
 OSDs that are not safe to destroy will be rejected.
 
@@ -275,14 +274,14 @@ You can query the state of OSD operation with the following command:
 
 .. prompt:: bash #
 
-   ceph orch osd rm status
+  ceph orch osd rm status
 
 Expected output::
 
-    OSD_ID  HOST         STATE                    PG_COUNT  REPLACE  FORCE  STARTED_AT
-    2       cephadm-dev  done, waiting for purge  0         True     False  2020-07-17 13:01:43.147684
-    3       cephadm-dev  draining                 17        False    True   2020-07-17 13:01:45.162158
-    4       cephadm-dev  started                  42        False    True   2020-07-17 13:01:45.162158
+  OSD_ID  HOST         STATE                    PG_COUNT  REPLACE  FORCE  STARTED_AT
+  2       cephadm-dev  done, waiting for purge  0         True     False  2020-07-17 13:01:43.147684
+  3       cephadm-dev  draining                 17        False    True   2020-07-17 13:01:45.162158
+  4       cephadm-dev  started                  42        False    True   2020-07-17 13:01:45.162158
 
 
 When no PGs are left on the OSD, it will be decommissioned and removed from the cluster.
@@ -304,11 +303,11 @@ Example:
 
 .. prompt:: bash #
 
-    ceph orch osd rm stop 4
+  ceph orch osd rm stop 4
 
 Expected output::
 
-    Stopped OSD(s) removal
+  Stopped OSD(s) removal
 
 This resets the initial state of the OSD and takes it off the removal queue.
 
@@ -329,7 +328,7 @@ Example:
 
 Expected output::
 
-   Scheduled OSD(s) for replacement
+  Scheduled OSD(s) for replacement
 
 This follows the same procedure as the procedure in the "Remove OSD" section, with
 one exception: the OSD is not permanently removed from the CRUSH hierarchy, but is
@@ -436,10 +435,10 @@ the ``ceph orch ps`` output in the ``MEM LIMIT`` column::
 To exclude an OSD from memory autotuning, disable the autotune option
 for that OSD and also set a specific memory target.  For example,
 
-  .. prompt:: bash #
+.. prompt:: bash #
 
-    ceph config set osd.123 osd_memory_target_autotune false
-    ceph config set osd.123 osd_memory_target 16G
+  ceph config set osd.123 osd_memory_target_autotune false
+  ceph config set osd.123 osd_memory_target 16G
 
 
 .. _drivegroups:
@@ -502,7 +501,7 @@ Example
 
 .. prompt:: bash [monitor.1]#
 
-   ceph orch apply -i /path/to/osd_spec.yml --dry-run
+  ceph orch apply -i /path/to/osd_spec.yml --dry-run
 
 
 
@@ -512,9 +511,9 @@ Filters
 -------
 
 .. note::
-   Filters are applied using an `AND` gate by default. This means that a drive
-   must fulfill all filter criteria in order to get selected. This behavior can
-   be adjusted by setting ``filter_logic: OR`` in the OSD specification. 
+    Filters are applied using an `AND` gate by default. This means that a drive
+    must fulfill all filter criteria in order to get selected. This behavior can
+    be adjusted by setting ``filter_logic: OR`` in the OSD specification. 
 
 Filters are used to assign disks to groups, using their attributes to group
 them. 
@@ -524,7 +523,7 @@ information about the attributes with this command:
 
 .. code-block:: bash
 
-  ceph-volume inventory </path/to/disk>
+    ceph-volume inventory </path/to/disk>
 
 Vendor or Model
 ^^^^^^^^^^^^^^^
@@ -633,9 +632,9 @@ but want to use only the first two, you could use `limit`:
 
 .. code-block:: yaml
 
-  data_devices:
-    vendor: VendorA
-    limit: 2
+    data_devices:
+      vendor: VendorA
+      limit: 2
 
 .. note:: `limit` is a last resort and shouldn't be used if it can be avoided.
 
@@ -858,8 +857,8 @@ See :ref:`orchestrator-cli-placement-spec`
 
 .. note::
 
-   Assuming each host has a unique disk layout, each OSD 
-   spec needs to have a different service id
+    Assuming each host has a unique disk layout, each OSD 
+    spec needs to have a different service id
 
 
 Dedicated wal + db
@@ -989,7 +988,7 @@ activates all existing OSDs on a host.
 
 .. prompt:: bash #
 
-   ceph cephadm osd activate <host>...
+  ceph cephadm osd activate <host>...
 
 This will scan all existing disks for OSDs and deploy corresponding daemons.
 

From 717031454243898071bdb254b2cadc7048057903 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Mon, 30 Jan 2023 02:18:54 +0530
Subject: [PATCH 0379/2492] cephfs: add command "ceph fs swap"

Add a FS command that enables users to swap names of two file systems in
a single PAXOS transaction. Add an option to this command that swaps
FSCIDS along with FS names. This commands also updates the application
pool tags and fails when mirroring is enabled on either or both FSs.

Fixes: https://tracker.ceph.com/issues/58129
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 PendingReleaseNotes           |   7 +
 doc/cephfs/administration.rst |  40 ++
 doc/man/8/ceph.rst            |  11 +-
 qa/tasks/cephfs/test_admin.py | 807 ++++++++++++++++++++++++++++++++++
 src/mds/FSMap.cc              |  17 +
 src/mds/FSMap.h               |   9 +
 src/mon/FSCommands.cc         | 205 ++++++++-
 src/mon/MonCommands.h         |   8 +
 8 files changed, 1102 insertions(+), 2 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 1fc9c4510d3e..b7290fb3938a 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -50,6 +50,13 @@
   recommend that users with versioned buckets, especially those that existed
   on prior releases, use these new tools to check whether their buckets are
   affected and to clean them up accordingly.
+* CephFS: Two FS names can now be swapped, optionally along with their IDs,
+  using "ceph fs swap" command. The function of this API is to facilitate
+  file system swaps for disaster recovery. In particular, it avoids situations
+  where a named file system is temporarily missing which would prompt a higher
+  level storage operator (like Rook) to recreate the missing file system.
+  See https://docs.ceph.com/en/latest/cephfs/administration/#file-systems
+  docs for more information.
 
 >=18.0.0
 
diff --git a/doc/cephfs/administration.rst b/doc/cephfs/administration.rst
index cd912b42aeaf..dd9a8e25b168 100644
--- a/doc/cephfs/administration.rst
+++ b/doc/cephfs/administration.rst
@@ -92,6 +92,46 @@ The CephX IDs authorized to the old file system name need to be reauthorized
 to the new name. Any on-going operations of the clients using these IDs may be
 disrupted. Mirroring is expected to be disabled on the file system.
 
+::
+
+    fs swap <fs1-name> <fs1_id> <fs2-name> <fs2_id> [--swap-fscids=yes|no] [--yes-i-really-mean-it]
+
+Swaps names of two Ceph file sytems and updates the application tags on all
+pools of both FSs accordingly. Certain tools that track FSCIDs of the file
+systems, besides the FS names, might get confused due to this operation. For
+this reason, mandatory option ``--swap-fscids`` has been provided that must be
+used to indicate whether or not FSCIDs must be swapped.
+
+.. note:: FSCID stands for "File System Cluster ID".
+
+Before the swap, mirroring should be disabled on both the CephFSs
+(because the cephfs-mirror daemon uses the fscid internally and changing it
+while the daemon is running could result in undefined behaviour), both the
+CephFSs should be offline and the file system flag ``refuse_client_sessions``
+must be set for both the CephFS.
+
+The function of this API is to facilitate disaster recovery where a new file
+system reconstructed from the previous one is ready to take over for the
+possibly damaged file system. Instead of two ``fs rename`` operations, the
+operator can use a swap so there is no FSMap epoch where the primary (or
+production) named file system does not exist. This is important when Ceph is
+monitored by automatic storage operators like (Rook) which try to reconcile
+the storage system continuously. That operator may attempt to recreate the
+file system as soon as it is seen to not exist.
+
+After the swap, CephX credentials may need to be reauthorized if the existing
+mounts should "follow" the old file system to its new name. Generally, for
+disaster recovery, its desirable for the existing mounts to continue using
+the same file system name. Any active file system mounts for either CephFSs
+must remount. Existing unflushed operations will be lost. When it is judged
+that one of the swapped file systems is ready for clients, run::
+
+    ceph fs set <fs> joinable true
+    ceph fs set <fs> refuse_client_sessions false
+
+Keep in mind that one of the swapped file systems may be left offline for
+future analysis if doing a disaster recovery swap.
+
 
 Settings
 --------
diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst
index 05db4092523b..478d3c6b3ce9 100644
--- a/doc/man/8/ceph.rst
+++ b/doc/man/8/ceph.rst
@@ -23,7 +23,7 @@ Synopsis
 
 | **ceph** **df** *{detail}*
 
-| **ceph** **fs** [ *add_data_pool* \| *authorize* \| *dump* \| *feature ls* \| *flag set* \| *get* \| *ls* \| *lsflags* \| *new* \| *rename* \| *reset* \| *required_client_features add* \| *required_client_features rm* \| *rm* \| *rm_data_pool* \| *set*] ...
+| **ceph** **fs** [ *add_data_pool* \| *authorize* \| *dump* \| *feature ls* \| *flag set* \| *get* \| *ls* \| *lsflags* \| *new* \| *rename* \| *reset* \| *required_client_features add* \| *required_client_features rm* \| *rm* \| *rm_data_pool* \| *set* \| *swap* ] ...
 
 | **ceph** **fsid**
 
@@ -474,6 +474,15 @@ Usage::
 
     ceph fs set <fs-name> <fs-setting> <value>
 
+Subcommand ``swap`` swaps the names of two Ceph file system and updates
+application tags on the pool of the file systems accordingly. Optionally,
+FSIDs of the filesystems can also be swapped along with names by passing
+``--swap-fscids``.
+
+Usage::
+
+    ceph fs swap <fs1-name> <fs1-id> <fs2-name> <fs2-id> [--swap-fscids] {--yes-i-really-meant-it}
+
 fsid
 ----
 
diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index b2e10e4d3cf4..79a35574dd7b 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -769,6 +769,813 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
         self.run_ceph_cmd(f'fs mirror disable {orig_fs_name}')
 
 
+class TestSwapCmd(TestAdminCommands):
+    """
+    Tests for "ceph fs swap" command.
+    """
+
+    MDSS_REQUIRED = 3
+    CLIENTS_REQUIRED = 2
+    client_id = 'testuser'
+    client_name = f'client.{client_id}'
+
+    def setUp(self):
+        super(self.__class__, self).setUp()
+        self.fs1 = self.fs
+        self.fs2 = self.mds_cluster.newfs(name='testcephfs2', create=True)
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self.orig_fs_id_name = {self.fs1.id: self.fs1.name,
+                                self.fs2.id: self.fs2.name}
+
+        self.mount_a.remount(cephfs_name=self.fs1.name)
+        self.mount_b.remount(cephfs_name=self.fs2.name)
+
+        self.captesters = (CapTester(self.mount_a), CapTester(self.mount_b))
+
+        self.mount_a.umount_wait()
+        self.mount_b.umount_wait()
+
+    def tearDown(self):
+        self.mount_a.umount_wait()
+        self.mount_b.umount_wait()
+        self.run_ceph_cmd(args=f'auth rm {self.client_name}')
+        super(self.__class__, self).tearDown()
+
+    def _reauthorize_client(self):
+        moncap = gen_mon_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        osdcap = gen_osd_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        mdscap = gen_mds_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        self.run_ceph_cmd(args=f'auth add {self.client_name} mon "{moncap}" '
+                               f'osd "{osdcap}" mds "{mdscap}"')
+
+    def _remount_both_cephfss(self):
+        keyring = self.fs.mon_manager.get_keyring(self.client_id) + '\n'
+        keyring_path_a = self.mount_a.client_remote.mktemp(data=keyring)
+        keyring_path_b = self.mount_b.client_remote.mktemp(data=keyring)
+
+        self.mount_a.mount(client_id=self.client_id,
+                           client_keyring_path=keyring_path_a,
+                           cephfs_mntpt='/', cephfs_name=self.fs1.name)
+        self.mount_b.mount(client_id=self.client_id,
+                           client_keyring_path=keyring_path_b,
+                           cephfs_mntpt='/', cephfs_name=self.fs2.name)
+
+    def run_rw_tests(self):
+        for captester in self.captesters:
+            captester.conduct_pos_test_for_read_caps()
+            captester.conduct_pos_test_for_write_caps()
+
+    def _check_fs_name_on_fs_pools(self, fss):
+        for fs in fss:
+            for pool in fs.get_data_pool_names(True):
+                self.check_pool_application_metadata_key_value(pool, 'cephfs',
+                    'data', fs.name)
+            self.check_pool_application_metadata_key_value(
+                fs.get_metadata_pool_name(), 'cephfs', 'metadata', fs.name)
+
+    def _are_fsnames_and_fscids_together(self):
+        '''
+        Are FS names and FSIDs together on same the FS as they were before
+        running "ceph fs swap" command?
+        '''
+        fs1_id_swapped = self.orig_fs_id_name[self.fs1.id] == self.fs1.name
+        fs2_id_swapped = self.orig_fs_id_name[self.fs2.id] == self.fs2.name
+
+        if fs1_id_swapped and fs2_id_swapped:
+            return True
+        elif not fs1_id_swapped and not fs2_id_swapped:
+            return False
+        else:
+            raise RuntimeError(
+                'Unexpected situation occured: FSID for one FS swapped but '
+                'not for the other FS.')
+
+    def _bring_both_cephfss_down(self):
+        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
+        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
+
+    def _bring_both_cephfss_up(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
+
+    def _refuse_clients_for_both_cephfss(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
+
+    def _accept_clients_for_both_cephfss(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
+                           'false')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
+                           'false')
+
+    def test_swap_fsnames_but_not_fscids(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=no" swaps the FS names but not
+        the FSCIDs.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        # log output to help debug test failures
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=no '
+                           '--yes_i_really_mean_it')
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # FS names were swapped but not FSIDs, so both can't be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), False)
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.run_rw_tests()
+
+    def test_swap_fsnames_and_fscids(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=yes" swaps the FS names as well
+        as the FSCIDs.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=yes '
+                           '--yes_i_really_mean_it')
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        # XXX: Let's ensure that FS mounted on a mountpoint is same before
+        # and after swapping of FS name and FSCIDs. This ensures that data
+        # available on mountpoints before and after the swap is same. This
+        # prevents self.run_rw_tests() from breaking.
+        #
+        # At the beginning of test, testcephfs1 has data (let's say) 'abc1'
+        # and testcephfs2 has data 'abc2'. self.fs1 is mapped to testcephfs1
+        # and self.fs2 mapped to testcephfs2. After swap, data of testcephfs1
+        # and testcephfs2 will be 'abc2' and 'abc1' respectively.
+        #
+        # However, running self.fs1.getinfo() after swap will map self.fs1 to
+        # FS with FSCID 1 i.e. testcephfs1 and not testcephfs2. Thus, data
+        # under self.fs1 will be different than data before swapping. This
+        # breaks self.run_rw_tests() because self.fs1 is always mounted on
+        # the mountpoint of self.mount_a.
+
+        # To prevent this, therefore, make sure that data on
+        # self.fs1/self.mount_a is same after and before the swap. To ensure
+        # this, swap FS that is represented by self.fs1. Instead of
+        # testcephfs1 it should be testcephfs2 because, after swap,
+        # testcephfs2 containts the data of testcephfs1. This will ensure that
+        # self.mount_rw_tests doesn't break.
+        #
+        # Same for self.fs2.
+        self.fs1.id, self.fs2.id = None, None
+        self.fs1.name, self.fs2.name = self.fs2.name, self.fs1.name
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # both FS name and FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.run_rw_tests()
+
+    def test_swap_without_confirmation_option(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=yes" without the option
+        "--yes-i-really-mean-it" fails.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('This is a potentially disruptive operation, client\'s cephx '
+               'credentials may need to be reauthorized to access the file '
+               'systems and its pools. Add --yes-i-really-mean-it if you are '
+               'sure you wish to continue.')
+        self.negtest_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                              f'{self.fs2.name} {self.fs2.id} '
+                               '--swap-fscids=no',
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    ###################################################
+    #
+    # Tests for "fs swap" when either FS name is false.
+    #
+    ###################################################
+
+    def test_swap_when_fs1_is_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there is no CephFS on cluster by the name "<fs1name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs = 'random_fsname_654'
+        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
+                "cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {absent_cephfs} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_is_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there is no CephFS on cluster by the name "<fs2name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs = 'random_fsname_654'
+        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
+                "cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs2.id} {absent_cephfs} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_fss_are_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there are no CephFSs on the cluster by the name "<fs1name>" and
+        "<fs2name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs1 = 'random_fsname_65'
+        absent_cephfs2 = 'random_fsname_66'
+        msg = (f"Neither file system '{absent_cephfs1}' nor file system "
+               f"'{absent_cephfs2}' exists on this Ceph cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {absent_cephfs1} 123 {absent_cephfs2} 1234 '
+                   '--swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    ###################################################
+    #
+    # Tests for "fs swap" when either FSCID is wrong.
+    #
+    ###################################################
+
+    def test_swap_when_fs1_id_is_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs1id>" is not the FSCID of the CephFS named "<fs1nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"FSCID provided for '{self.fs1.name}' is incorrect.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_id_is_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs2id>" is not the FSCID of the CephFS named "<fs2nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"FSCID provided for '{self.fs2.name}' is incorrect.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'123 --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_fscids_are_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs1id>" and "<fs2id>", respectively, are not the FSCIDs of the
+        CephFSs named "<fs1name>" and "<fs2nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('FSCIDs provided for both the CephFSs is incorrect.')
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} 1234 '
+                  f'--swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_user_swaps_fscids_in_cmd_args(self):
+        '''
+        Test that "ceph fs swap" fails and prints relevant error message when
+        FSCIDs are exchange while writing the command. That is user write the
+        command as -
+
+        "ceph fs swap <fs1name> <fs2id> <fs2name> <fs1id>"
+
+        instead of writing -
+
+        "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>"
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('FSCIDs provided in command arguments are swapped; perhaps '
+               '`ceph fs swap` has been run before.')
+        proc = self.run_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs2.id} {self.fs2.name} '
+                  f'{self.fs1.id} --swap-fscids=no --yes_i_really_mean_it'),
+            stderr=StringIO())
+        self.assertIn(msg.lower(), proc.stderr.getvalue().lower())
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    #####################################################
+    #
+    # Tests for "fs swap" when mirroring is enabled on FS
+    #
+    #####################################################
+
+    def test_swap_when_mirroring_enabled_for_1st_FS(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for the CephFS named "<fs1name>".
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"Mirroring is enabled on file system '{self.fs1.name}'. "
+                "Disable mirroring on the file system after ensuring it's "
+                "OK to do so, and then re-try swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_mirroring_enabled_for_2nd_FS(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for the CephFS named "<fs2name>".
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+
+        self.run_ceph_cmd('fs', 'dump')
+        msg = (f"Mirroring is enabled on file system '{self.fs2.name}'. "
+                "Disable mirroring on the file system after ensuring it's "
+                "OK to do so, and then re-try swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_mirroring_enabled_for_both_FSs(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for both the CephFSs.
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
+        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"Mirroring is enabled on file systems '{self.fs1.name}' and "
+               f"'{self.fs2.name}'. Disable mirroring on both the file "
+                "systems after ensuring it's OK to do so, and then re-try "
+                "swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    ##########################################################
+    #
+    # Tests for "fs swap" when either FS is not down/failed.
+    #
+    #########################################################
+
+    def test_swap_when_fs1_is_online(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        CephFS named "<fs1name>" is online (i.e. is not failed).
+        '''
+        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs1.name}' is not offline. Before swapping "
+                "CephFS names, both CephFSs should be marked as failed."
+                "See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_is_not_down(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        CephFS named "<fs2name>" is online (i.e. is not failed).
+        '''
+        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs2.name}' is not offline. Before swapping "
+                "CephFS names, both CephFSs should be marked as failed. "
+                "See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_FSs_are_not_down(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        both the CephFSs are online (i.e. is not failed).
+        '''
+        self._refuse_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' are not "
+                "offline. Before swapping CephFS names, both CephFSs should "
+                "be marked as failed. See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._accept_clients_for_both_cephfss()
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    ##################################################
+    #
+    # Tests for "fs swap" when either FS is offline.
+    #
+    ################################################
+
+
+    def test_swap_when_FS1_doesnt_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for the first of the two of FSs .
+        '''
+        self._bring_both_cephfss_down()
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs1.name}' doesn't refuse clients. Before "
+                "swapping CephFS names, flag 'refuse_client_session' must "
+                "be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
+                           'false')
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_FS2_doesnt_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for the second of the two of FSs .
+        '''
+        self._bring_both_cephfss_down()
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs2.name}' doesn't refuse clients. Before "
+                "swapping CephFS names, flag 'refuse_client_session' must "
+                "be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
+                           'false')
+        sleep(5)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_FSs_do_not_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for both the CephFSs.
+        '''
+        self.run_ceph_cmd('fs', 'dump')
+        self._bring_both_cephfss_down()
+        sleep(5)
+        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' do not "
+                "refuse clients. Before swapping CephFS names, flag "
+                "'refuse_client_session' must be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd('fs', 'dump')
+        sleep(5)
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
 class TestDump(CephFSTestCase):
     CLIENTS_REQUIRED = 0
     MDSS_REQUIRED = 1
diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index fe7ee5588919..2f14a780b84e 100644
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -1209,3 +1209,20 @@ void FSMap::erase_filesystem(fs_cluster_id_t fscid)
     }
   }
 }
+
+void FSMap::swap_fscids(fs_cluster_id_t fscid1, fs_cluster_id_t fscid2)
+{
+  auto fs1 = std::move(filesystems.at(fscid1));
+  filesystems[fscid1] = std::move(filesystems.at(fscid2));
+  filesystems[fscid2] = std::move(fs1);
+
+  auto set_fs1_fscid = [fscid1](auto&& fs) {
+    fs.set_fscid(fscid1);
+  };
+  modify_filesystem(fscid1, std::move(set_fs1_fscid));
+
+  auto set_fs2_fscid = [fscid2](auto&& fs) {
+    fs.set_fscid(fscid2);
+  };
+  modify_filesystem(fscid2, std::move(set_fs2_fscid));
+}
diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h
index 9720eb90d0d2..a6aa92f218be 100644
--- a/src/mds/FSMap.h
+++ b/src/mds/FSMap.h
@@ -253,6 +253,10 @@ class Filesystem
   }
 
 private:
+  void set_fscid(fs_cluster_id_t new_fscid) {
+    fscid = new_fscid;
+  }
+
   friend class FSMap;
 
   fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
@@ -480,6 +484,11 @@ class FSMap {
     fs.mds_map.modified = ceph_clock_now();
   }
 
+  /* This is method is written for the option of "ceph fs swap" commmand
+   * that intiates swap of FSCIDs.
+   */
+  void swap_fscids(fs_cluster_id_t fscid1, fs_cluster_id_t fscid2);
+
   /**
    * Apply a mutation to the mds_info_t structure for a particular
    * daemon (identified by GID), and make appropriate updates to epochs.
diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 06447a877337..7944aa4bd386 100644
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -1261,6 +1261,207 @@ class RenameFilesystemHandler : public FileSystemCommandHandler
   Paxos *m_paxos;
 };
 
+class SwapFilesystemHandler : public FileSystemCommandHandler
+{
+  public:
+  explicit SwapFilesystemHandler(Paxos *paxos)
+    : FileSystemCommandHandler("fs swap"), m_paxos(paxos)
+  {
+  }
+
+  int handle(Monitor *mon, FSMap& fsmap, MonOpRequestRef op,
+	     const cmdmap_t& cmdmap, std::ostream &ss) override
+  {
+    ceph_assert(m_paxos->is_plugged());
+
+    // Check for confirmation flag
+    bool confirmation_flag = false;
+    cmd_getval(cmdmap, "yes_i_really_mean_it", confirmation_flag);
+    if (!confirmation_flag) {
+      ss << "This is a potentially disruptive operation, client\'s cephx "
+	"credentials may need to be reauthorized to access the file systems "
+	"and its pools. Add --yes-i-really-mean-it if you are sure you wish "
+	"to continue.";
+      return -EPERM;
+    }
+
+    string fs1_name, fs2_name;
+    int64_t fs1_id = FS_CLUSTER_ID_NONE;
+    int64_t fs2_id = FS_CLUSTER_ID_NONE;
+    string swap_fscids_flag;
+    cmd_getval(cmdmap, "fs1_name", fs1_name);
+    cmd_getval(cmdmap, "fs2_name", fs2_name);
+    cmd_getval(cmdmap, "fs1_id", fs1_id);
+    cmd_getval(cmdmap, "fs2_id", fs2_id);
+    cmd_getval(cmdmap, "swap_fscids", swap_fscids_flag);
+    auto fs1p = fsmap.get_filesystem(fs1_name);
+    auto fs2p = fsmap.get_filesystem(fs2_name);
+
+    // Check that CephFSs exists for both given names.
+    if (fs1p == nullptr || fs2p == nullptr) {
+      if (fs1p == nullptr && fs2p != nullptr) {
+	ss << "File system '" << fs1_name << "' doesn\'t exist on this "
+	      "Ceph cluster.";
+	return -ENOENT;
+      } else if (fs1p != nullptr && fs2p == nullptr) {
+	ss << "File system '" << fs2_name << "' doesn\'t exist on this "
+	      "Ceph cluster.";
+	return -ENOENT;
+      } else {
+	ss << "Neither file system '" << fs1_name << "' nor file "
+	      "system '" << fs2_name << "' exists on this Ceph cluster.";
+	return -ENOENT;
+      }
+    }
+
+    // Check that FSCID provided for both CephFSs is correct.
+    if (fs1_id != fs1p->get_fscid() || fs2_id != fs2p->get_fscid()) {
+      if (fs1_id != fs1p->get_fscid() && fs2_id == fs2p->get_fscid()) {
+	ss << "FSCID provided for '" << fs1_name << "' is incorrect.";
+	return -EINVAL;
+      } else if (fs1_id == fs1p->get_fscid() && fs2_id != fs2p->get_fscid()) {
+	ss << "FSCID provided for '" << fs2_name << "' is incorrect.";
+	return -EINVAL;
+      } else if (fs1_id != fs1p->get_fscid() && fs2_id != fs2p->get_fscid()) {
+	if (fs1_id == fs2p->get_fscid() && fs2_id == fs1p->get_fscid()) {
+	  ss << "FSCIDs provided in command arguments are swapped; perhaps "
+	     << "`ceph fs swap` has been run before.";
+	  return 0;
+	} else {
+	ss << "FSCIDs provided for both the CephFSs is incorrect.";
+	return -EINVAL;
+	}
+      }
+    }
+
+    // Check that CephFS mirroring for both CephFSs is disabled.
+    if (fs1p->get_mirror_info().mirrored || fs2p->get_mirror_info().mirrored) {
+      if (fs1p->get_mirror_info().mirrored &&
+	  !fs2p->get_mirror_info().mirrored) {
+	ss << "Mirroring is enabled on file system '"<< fs1_name << "'. "
+	   << "Disable mirroring on the file system after ensuring it's OK "
+	   << "to do so, and then re-try swapping.";
+	return -EPERM;
+      } else if (!fs1p->get_mirror_info().mirrored &&
+		 fs2p->get_mirror_info().mirrored) {
+	ss << "Mirroring is enabled on file system '"<< fs2_name << "'. "
+	   << "Disable mirroring on the file system after ensuring it's OK "
+	   << "to do so, and then re-try swapping.";
+	return -EPERM;
+      } else {
+	ss << "Mirroring is enabled on file systems '" << fs1_name << "' "
+	   << "and '" << fs2_name << "'. Disable mirroring on both the "
+	   << "file systems after ensuring it's OK to do so, and then re-try "
+	   << "swapping.";
+	return -EPERM;
+      }
+    }
+
+    if (!mon->osdmon()->is_writeable()) {
+      // not allowed to write yet, so retry when we can
+      mon->osdmon()->wait_for_writeable(
+	op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
+      return -EAGAIN;
+    }
+
+    // Check that both CephFS have been marked as down, IOW has no MDS
+    // associated with it.
+    if (fs1p->get_mds_map().get_num_up_mds() > 0 ||
+        fs2p->get_mds_map().get_num_up_mds() > 0) {
+      if (fs1p->get_mds_map().get_num_up_mds() > 0 &&
+          fs2p->get_mds_map().get_num_up_mds() == 0) {
+	ss << "CephFS '" << fs1_name << "' is not offline. Before swapping "
+	   << "CephFS names, both CephFSs should be marked as failed. See "
+	   << "`ceph fs fail`.";
+	return -EPERM;
+      } else if (fs1p->get_mds_map().get_num_up_mds() == 0 &&
+		 fs2p->get_mds_map().get_num_up_mds() > 0) {
+	ss << "CephFS '" << fs2_name << "' is not offline. Before swapping "
+	   << "CephFS names, both CephFSs should be marked as failed. See "
+	   << "`ceph fs fail`.";
+	return -EPERM;
+      } else {
+	ss << "CephFSs '" << fs1_name << "' and '" << fs2_name << "' "
+	   << "are not offline. Before swapping CephFS names, both CephFSs "
+	   << "should be marked as failed. See `ceph fs fail`.";
+	return -EPERM;
+      }
+    }
+
+    // Check that refuse_client_session is set.
+    if (!fs1p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION) ||
+	!fs2p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+      if (!fs1p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION) &&
+          fs2p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+	ss << "CephFS '" << fs1_name << "' doesn't refuse clients. Before "
+	   << "swapping CephFS names, flag 'refuse_client_session' must be "
+	    << "set. See `ceph fs set`.";
+	return -EPERM;
+      } else if (
+          fs1p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION) &&
+	  !fs2p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+	ss << "CephFS '" << fs2_name << "' doesn't refuse clients. Before "
+	   << "swapping CephFS names, flag 'refuse_client_session' must be "
+	    << "set. See `ceph fs set`.";
+	return -EPERM;
+      } else if (
+          !fs1p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION) &&
+	  !fs2p->get_mds_map().test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) {
+	ss << "CephFSs '" << fs1_name << "' and '" << fs2_name << "' do not "
+	   << "refuse clients. Before swapping CephFS names, flag "
+	   << "'refuse_client_session' must be set. See `ceph fs set`.";
+	return -EPERM;
+      }
+    }
+
+    // Finally, the swap begins.
+    // Swap CephFS names on OSD pool application tag
+    for (const auto p : fs1p->get_mds_map().get_data_pools()) {
+      mon->osdmon()->do_application_enable(p,
+					   pg_pool_t::APPLICATION_NAME_CEPHFS,
+					   "data", fs2_name, true);
+    }
+    mon->osdmon()->do_application_enable(fs1p->get_mds_map().get_metadata_pool(),
+					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+					 "metadata", fs2_name, true);
+
+    for (const auto p : fs2p->get_mds_map().get_data_pools()) {
+      mon->osdmon()->do_application_enable(p,
+					   pg_pool_t::APPLICATION_NAME_CEPHFS,
+					   "data", fs1_name, true);
+    }
+    mon->osdmon()->do_application_enable(fs2p->get_mds_map().get_metadata_pool(),
+					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+					 "metadata", fs1_name, true);
+    mon->osdmon()->propose_pending();
+
+    // Now swap CephFS names and, optionally, FSCIDs.
+    auto renamefunc1 = [fs2_name](auto&& fs) {
+      fs.get_mds_map().set_fs_name(fs2_name);
+    };
+    fsmap.modify_filesystem(fs1_id, std::move(renamefunc1));
+    auto renamefunc2 = [fs1_name](auto&& fs) {
+      fs.get_mds_map().set_fs_name(fs1_name);
+    };
+    fsmap.modify_filesystem(fs2_id, std::move(renamefunc2));
+
+    if (swap_fscids_flag == "yes") {
+      fsmap.swap_fscids(fs1_id, fs2_id);
+    }
+
+    ss << "File system names ";
+    if (swap_fscids_flag == "yes") {
+      ss << "and FSCIDs ";
+    }
+    ss << " have been swapped; cephx credentials may need an upgrade.";
+
+    return 0;
+  }
+
+private:
+  Paxos *m_paxos;
+};
+
 class RemoveDataPoolHandler : public FileSystemCommandHandler
 {
   public:
@@ -1581,6 +1782,7 @@ FileSystemCommandHandler::load(Paxos *paxos)
   handlers.push_back(std::make_shared<RemoveFilesystemHandler>());
   handlers.push_back(std::make_shared<ResetFilesystemHandler>());
   handlers.push_back(std::make_shared<RenameFilesystemHandler>(paxos));
+  handlers.push_back(std::make_shared<SwapFilesystemHandler>(paxos));
 
   handlers.push_back(std::make_shared<SetDefaultHandler>());
   handlers.push_back(std::make_shared<AliasHandler<SetDefaultHandler> >(
@@ -1719,7 +1921,8 @@ int FileSystemCommandHandler::is_op_allowed(
     if (fsp == nullptr) {
       auto prefix = get_prefix();
       /* let "fs rm" and "fs rename" handle idempotent cases where file systems do not exist */
-      if (!(prefix == "fs rm" || prefix == "fs rename") && fsmap.get_filesystem(fs_name) == nullptr) {
+      if (!(prefix == "fs rm" || prefix == "fs rename" || prefix == "fs swap") &&
+	  fsmap.get_filesystem(fs_name) == nullptr) {
         ss << "Filesystem not found: '" << fs_name << "'";
         return -ENOENT;
       }
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 74ef2206c02b..71a8bec76ac0 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -444,6 +444,14 @@ COMMAND("fs rename "
 	"name=new_fs_name,type=CephString,goodchars=" FS_NAME_GOODCHARS
 	" name=yes_i_really_mean_it,type=CephBool,req=false",
 	"rename a ceph file system", "mds", "rw")
+COMMAND("fs swap "
+	"name=fs1_name,type=CephString "
+	"name=fs1_id,type=CephInt,range=0 "
+	"name=fs2_name,type=CephString "
+	"name=fs2_id,type=CephInt,range=0 "
+	"name=swap_fscids,type=CephChoices,strings=yes|no,req=true "
+	"name=yes_i_really_mean_it,type=CephBool,req=false",
+	"swap ceph file system names", "mds", "rw")
 
 /*
  * Monmap commands

From 4a12f6eb6c16722f1d68e0949390cd7d30f44bc6 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 28 Sep 2023 18:56:22 +0530
Subject: [PATCH 0380/2492] mon/FSCommands: use alias for higher brevity

Create an alias so "APP_NAME_CEPHFS" can be written instead of
"pg_pool_t::APPLICATION_NAME_CEPHFS".

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/mon/FSCommands.cc | 53 ++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 7944aa4bd386..300c025486a4 100644
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -32,6 +32,8 @@ using std::vector;
 using std::ostream;
 
 
+static const auto& APP_NAME_CEPHFS = pg_pool_t::APPLICATION_NAME_CEPHFS;
+
 class FlagSetHandler : public FileSystemCommandHandler
 {
   public:
@@ -255,11 +257,9 @@ class FsNewHandler : public FileSystemCommandHandler
       mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
       return -EAGAIN;
     }
-    mon->osdmon()->do_application_enable(data,
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
-					 "data", fs_name, true);
-    mon->osdmon()->do_application_enable(metadata,
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
+    mon->osdmon()->do_application_enable(data, APP_NAME_CEPHFS, "data",
+					 fs_name, true);
+    mon->osdmon()->do_application_enable(metadata, APP_NAME_CEPHFS,
 					 "metadata", fs_name, true);
     mon->osdmon()->do_set_pool_opt(metadata,
 				   pool_opts_t::RECOVERY_PRIORITY,
@@ -1004,9 +1004,8 @@ class AddDataPoolHandler : public FileSystemCommandHandler
       mon->osdmon()->wait_for_writeable(op, new PaxosService::C_RetryMessage(mon->mdsmon(), op));
       return -EAGAIN;
     }
-    mon->osdmon()->do_application_enable(poolid,
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
-					 "data", fs_name, true);
+    mon->osdmon()->do_application_enable(poolid, APP_NAME_CEPHFS, "data",
+					 fs_name, true);
     mon->osdmon()->propose_pending();
 
     fsmap.modify_filesystem(
@@ -1235,14 +1234,13 @@ class RenameFilesystemHandler : public FileSystemCommandHandler
       return -EAGAIN;
     }
     for (const auto p : fsp->get_mds_map().get_data_pools()) {
-      mon->osdmon()->do_application_enable(p,
-					   pg_pool_t::APPLICATION_NAME_CEPHFS,
-					   "data", new_fs_name, true);
+      mon->osdmon()->do_application_enable(p, APP_NAME_CEPHFS, "data",
+					   new_fs_name, true);
     }
 
-    mon->osdmon()->do_application_enable(fsp->get_mds_map().get_metadata_pool(),
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
-					 "metadata", new_fs_name, true);
+    mon->osdmon()->do_application_enable(
+      fsp->get_mds_map().get_metadata_pool(), APP_NAME_CEPHFS, "metadata",
+      new_fs_name, true);
     mon->osdmon()->propose_pending();
 
     auto f = [new_fs_name](auto&& fs) {
@@ -1417,22 +1415,20 @@ class SwapFilesystemHandler : public FileSystemCommandHandler
     // Finally, the swap begins.
     // Swap CephFS names on OSD pool application tag
     for (const auto p : fs1p->get_mds_map().get_data_pools()) {
-      mon->osdmon()->do_application_enable(p,
-					   pg_pool_t::APPLICATION_NAME_CEPHFS,
-					   "data", fs2_name, true);
+      mon->osdmon()->do_application_enable(p, APP_NAME_CEPHFS, "data",
+					   fs2_name, true);
     }
-    mon->osdmon()->do_application_enable(fs1p->get_mds_map().get_metadata_pool(),
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
-					 "metadata", fs2_name, true);
+    mon->osdmon()->do_application_enable(
+      fs1p->get_mds_map().get_metadata_pool(), APP_NAME_CEPHFS, "metadata",
+      fs2_name, true);
 
     for (const auto p : fs2p->get_mds_map().get_data_pools()) {
-      mon->osdmon()->do_application_enable(p,
-					   pg_pool_t::APPLICATION_NAME_CEPHFS,
-					   "data", fs1_name, true);
+      mon->osdmon()->do_application_enable(p, APP_NAME_CEPHFS, "data",
+					   fs1_name, true);
     }
-    mon->osdmon()->do_application_enable(fs2p->get_mds_map().get_metadata_pool(),
-					 pg_pool_t::APPLICATION_NAME_CEPHFS,
-					 "metadata", fs1_name, true);
+    mon->osdmon()->do_application_enable(
+      fs2p->get_mds_map().get_metadata_pool(), APP_NAME_CEPHFS, "metadata",
+      fs1_name, true);
     mon->osdmon()->propose_pending();
 
     // Now swap CephFS names and, optionally, FSCIDs.
@@ -1821,7 +1817,7 @@ int FileSystemCommandHandler::_check_pool(
   auto app_map = pool->application_metadata;
 
   if (!allow_overlay && !force && !app_map.empty()) {
-    auto app = app_map.find(pg_pool_t::APPLICATION_NAME_CEPHFS);
+    auto app = app_map.find(APP_NAME_CEPHFS);
     if (app != app_map.end()) {
       auto& [app_name, app_metadata] = *app;
       auto itr = app_metadata.find("data");
@@ -1887,8 +1883,7 @@ int FileSystemCommandHandler::_check_pool(
   }
 
   if (!force && !pool->application_metadata.empty() &&
-      pool->application_metadata.count(
-        pg_pool_t::APPLICATION_NAME_CEPHFS) == 0) {
+      pool->application_metadata.count(APP_NAME_CEPHFS) == 0) {
     *ss << " pool '" << pool_name << "' (id '" << pool_id
         << "') has a non-CephFS application enabled.";
     return -EINVAL;

From ab994d09e0f68d2e9be13129c898b6612395c19d Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 3 Nov 2023 17:24:54 +0800
Subject: [PATCH 0381/2492] crimson/os/seastore/transaction_manager: add the
 missing UNIT_TESTS_BUILT guard

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/transaction_manager.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index 7fbe119fcdd7..7261c84d2df7 100644
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -178,6 +178,7 @@ TransactionManager::close_ertr::future<> TransactionManager::close() {
   });
 }
 
+#ifdef UNIT_TESTS_BUILT
 TransactionManager::ref_ret TransactionManager::inc_ref(
   Transaction &t,
   LogicalCachedExtentRef &ref)
@@ -209,6 +210,7 @@ TransactionManager::ref_ret TransactionManager::inc_ref(
     return result.refcount;
   });
 }
+#endif
 
 TransactionManager::ref_ret TransactionManager::remove(
   Transaction &t,

From 3ef0dfaf12d4ad7481516c32c278d497c9a5bd2a Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Tue, 17 Oct 2023 09:31:59 +0200
Subject: [PATCH 0382/2492] mgr/rook: adding e2e version based on minikube

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/ci/Dockerfile             |   3 +
 src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh  |   9 ++
 .../rook/ci/scripts/bootstrap-rook-cluster.sh | 135 ++++++++++++++++++
 .../mgr/rook/ci/tests/features/rook.feature   |  12 ++
 .../ci/tests/features/steps/implementation.py |  21 +++
 .../mgr/rook/ci/tests/features/steps/utils.py |  29 ++++
 6 files changed, 209 insertions(+)
 create mode 100644 src/pybind/mgr/rook/ci/Dockerfile
 create mode 100755 src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
 create mode 100755 src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
 create mode 100644 src/pybind/mgr/rook/ci/tests/features/rook.feature
 create mode 100644 src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
 create mode 100644 src/pybind/mgr/rook/ci/tests/features/steps/utils.py

diff --git a/src/pybind/mgr/rook/ci/Dockerfile b/src/pybind/mgr/rook/ci/Dockerfile
new file mode 100644
index 000000000000..30ebea5749c8
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/Dockerfile
@@ -0,0 +1,3 @@
+FROM quay.io/ceph/daemon-base:latest-main
+COPY ./tmp_build/orchestrator /usr/share/ceph/mgr/orchestrator
+COPY ./tmp_build/rook /usr/share/ceph/mgr/rook
diff --git a/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
new file mode 100755
index 000000000000..a43e01a89861
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Execute tests
+: ${CEPH_DEV_FOLDER:=${PWD}}
+${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/tests
+behave
diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
new file mode 100755
index 000000000000..4b97df6bac14
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+
+set -eEx
+
+: ${CEPH_DEV_FOLDER:=${PWD}}
+KUBECTL="minikube kubectl --"
+
+# We build a local ceph image that contains the latest code
+# plus changes from the PR. This image will be used by the docker
+# running inside the minikube to start the different ceph pods
+LOCAL_CEPH_IMG="local/ceph"
+
+on_error() {
+    echo "on error"
+    minikube delete
+}
+
+configure_libvirt(){
+    sudo usermod -aG libvirt $(id -un)
+    sudo su -l $USER  # Avoid having to log out and log in for group addition to take effect.
+    sudo systemctl enable --now libvirtd
+    sudo systemctl restart libvirtd
+    sleep 10 # wait some time for libvirtd service to restart
+}
+
+setup_minikube_env() {
+
+    # Check if Minikube is running
+    if minikube status > /dev/null 2>&1; then
+	echo "Minikube is running"
+	minikube stop
+	minikube delete
+    else
+	echo "Minikube is not running"
+    fi
+
+    rm -rf ~/.minikube
+    minikube start --memory="4096" --cpus="2" --disk-size=10g --extra-disks=1 --driver kvm2
+    # point Docker env to use docker daemon running on minikube
+    eval $(minikube docker-env -p minikube)
+}
+
+build_ceph_image() {
+    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
+    CURR_CEPH_IMG=$(grep -E '^\s*image:\s+' cluster-test.yaml | sed 's/.*image: *\([^ ]*\)/\1/')
+
+    cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci
+    mkdir -p tmp_build/rook
+    mkdir -p tmp_build/orchestrator
+    cp ./../../orchestrator/*.py tmp_build/orchestrator
+    cp ../*.py tmp_build/rook
+
+    # we use the following tag to trick the Docker
+    # running inside minikube so it uses this image instead
+    # of pulling it from the registry
+    docker build --tag ${LOCAL_CEPH_IMG} .
+    docker tag ${LOCAL_CEPH_IMG} ${CURR_CEPH_IMG}
+
+    # cleanup
+    rm -rf tmp_build
+    cd ${CEPH_DEV_FOLDER}
+}
+
+create_rook_cluster() {
+    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
+    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/crds.yaml
+    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/common.yaml
+    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/operator.yaml
+    $KUBECTL create -f cluster-test.yaml
+    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/dashboard-external-http.yaml
+    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/toolbox.yaml
+}
+
+wait_for_rook_operator() {
+    local max_attempts=10
+    local sleep_interval=20
+    local attempts=0
+    $KUBECTL rollout status deployment rook-ceph-operator -n rook-ceph --timeout=180s
+    PHASE=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
+    echo "PHASE: $PHASE"
+    while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do
+	echo "Waiting for cluster to be ready..."
+	sleep $sleep_interval
+	attempts=$((attempts+1))
+        if [ $attempts -ge $max_attempts ]; then
+            echo "Maximum number of attempts ($max_attempts) reached. Exiting..."
+            return 1
+        fi
+    done
+}
+
+wait_for_ceph_cluster() {
+    local max_attempts=10
+    local sleep_interval=20
+    local attempts=0
+    $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=30s
+    while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do
+	echo "Waiting for Ceph cluster installed"
+	sleep $sleep_interval
+	attempts=$((attempts+1))
+        if [ $attempts -ge $max_attempts ]; then
+            echo "Maximum number of attempts ($max_attempts) reached. Exiting..."
+            return 1
+        fi
+    done
+    echo "Ceph cluster installed and running"
+}
+
+show_info() {
+    DASHBOARD_PASSWORD=$($KUBECTL -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo)
+    IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n rook-ceph --output jsonpath='{.items[*].status.hostIP}')
+    PORT="$($KUBECTL -n rook-ceph -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)"
+    BASE_URL="http://$IP_ADDR:$PORT"
+    echo "==========================="
+    echo "Ceph Dashboard:  "
+    echo "   IP_ADDRESS: $BASE_URL"
+    echo "   PASSWORD: $DASHBOARD_PASSWORD"
+    echo "==========================="
+}
+
+####################################################################
+####################################################################
+
+trap 'on_error $? $LINENO' ERR
+
+configure_libvirt
+setup_minikube_env
+build_ceph_image
+create_rook_cluster
+wait_for_rook_operator
+wait_for_ceph_cluster
+show_info
+
+####################################################################
+####################################################################
diff --git a/src/pybind/mgr/rook/ci/tests/features/rook.feature b/src/pybind/mgr/rook/ci/tests/features/rook.feature
new file mode 100644
index 000000000000..ae0478f8b9cd
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/tests/features/rook.feature
@@ -0,0 +1,12 @@
+Feature: Testing Rook orchestrator commands
+    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml and
+
+    Scenario: Verify ceph cluster health
+      When I run
+          """
+          ceph health | grep HEALTH
+          """
+      Then I get
+          """
+          HEALTH_OK
+          """
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
new file mode 100644
index 000000000000..adde61afd384
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
@@ -0,0 +1,21 @@
+from behave import *
+from utils import *
+import re
+
+@when("I run")
+def run_step(context):
+    context.output = run_commands(context.text)
+
+@then("I get")
+def verify_result_step(context):
+    print(f"Output is:\n{context.output}\n--------------\n")
+    assert context.text == context.output
+
+@then("I get something like")
+def verify_fuzzy_result_step(context):
+    output_lines = context.output.split("\n")
+    expected_lines = context.text.split("\n")
+    num_lines = min(len(output_lines), len(expected_lines))
+    for n in range(num_lines):
+        if not re.match(expected_lines[n], output_lines[n]):
+            raise
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/utils.py b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
new file mode 100644
index 000000000000..41a71d0fb1fc
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
@@ -0,0 +1,29 @@
+import subprocess
+
+ROOK_CEPH_COMMAND = "minikube kubectl -- -n rook-ceph exec -it deploy/rook-ceph-tools -- "
+CLUSTER_COMMAND = "minikube kubectl -- "
+
+
+def execute_command(command: str) -> str:
+    output = ""
+    try:
+        proc = subprocess.run(command, shell=True, capture_output=True, text=True)
+        output = proc.stdout
+    except Exception as ex:
+        output = f"Error executing command: {ex}"
+
+    return output
+
+
+def run_commands(commands: str) -> str:
+    commands_list = commands.split("\n")
+    output = ""
+    for cmd in commands_list:
+        if cmd.startswith("ceph"):
+            prefix = ROOK_CEPH_COMMAND
+        else:
+            prefix = CLUSTER_COMMAND
+        command = prefix + cmd
+        output = execute_command(command)
+
+    return output.strip("\n")

From 9c547ad2a570a325ea453ccc9419be2b8bc06b28 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Fri, 3 Nov 2023 00:32:06 +0530
Subject: [PATCH 0383/2492] qa/cephfs: allow finer control on test runs

When working with large group tests (18 in this case), it gets very
tedious to debug and fix tests when all 18 have to be run again for
every mistake. Cheap fix for this to split these 18 tests into several
classes.

But when modification are made to the feature, all these 18 tests needs
to exercised and previous solution forces developer to intitiate all
these test classes to run one by one.

Best of both worlds can be achieved if we split tests into group but
move all these related group to a new file.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/suites/fs/functional/tasks/admin.yaml |   1 +
 qa/tasks/cephfs/admin/test_fs_swap.py    | 829 +++++++++++++++++++++++
 qa/tasks/cephfs/test_admin.py            | 807 ----------------------
 3 files changed, 830 insertions(+), 807 deletions(-)
 create mode 100644 qa/tasks/cephfs/admin/test_fs_swap.py

diff --git a/qa/suites/fs/functional/tasks/admin.yaml b/qa/suites/fs/functional/tasks/admin.yaml
index 97ecc4cf41e4..c51cc68fed46 100644
--- a/qa/suites/fs/functional/tasks/admin.yaml
+++ b/qa/suites/fs/functional/tasks/admin.yaml
@@ -10,3 +10,4 @@ tasks:
       fail_on_skip: false
       modules:
         - tasks.cephfs.test_admin
+        - tasks.cephfs.admin.test_fs_swap
diff --git a/qa/tasks/cephfs/admin/test_fs_swap.py b/qa/tasks/cephfs/admin/test_fs_swap.py
new file mode 100644
index 000000000000..11b044916d0e
--- /dev/null
+++ b/qa/tasks/cephfs/admin/test_fs_swap.py
@@ -0,0 +1,829 @@
+import errno
+
+from io import StringIO
+from time import sleep
+from logging import getLogger
+
+from tasks.cephfs.cephfs_test_case import CephFSTestCase
+from tasks.cephfs.caps_helper import (CapTester, gen_mon_cap_str,
+                                      gen_osd_cap_str, gen_mds_cap_str)
+
+
+log = getLogger(__name__)
+
+
+class SwapHelper(CephFSTestCase):
+    '''
+    Helper methods for testing "ceph fs swap" command.
+    '''
+
+    MDSS_REQUIRED = 3
+    CLIENTS_REQUIRED = 2
+    client_id = 'testuser'
+    client_name = f'client.{client_id}'
+
+    def setUp(self):
+        super(SwapHelper, self).setUp()
+        self.fs1 = self.fs
+        self.fs2 = self.mds_cluster.newfs(name='testcephfs2', create=True)
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self.orig_fs_id_name = {self.fs1.id: self.fs1.name,
+                                self.fs2.id: self.fs2.name}
+
+        self.mount_a.remount(cephfs_name=self.fs1.name)
+        self.mount_b.remount(cephfs_name=self.fs2.name)
+
+        self.captesters = (CapTester(self.mount_a), CapTester(self.mount_b))
+
+        self.mount_a.umount_wait()
+        self.mount_b.umount_wait()
+
+    def tearDown(self):
+        self.mount_a.umount_wait()
+        self.mount_b.umount_wait()
+        self.run_ceph_cmd(args=f'auth rm {self.client_name}')
+        super(SwapHelper, self).tearDown()
+
+    def _reauthorize_client(self):
+        moncap = gen_mon_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        osdcap = gen_osd_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        mdscap = gen_mds_cap_str((("rw", self.fs1.name),
+                                  ("rw", self.fs2.name)))
+        self.run_ceph_cmd(args=f'auth add {self.client_name} mon "{moncap}" '
+                               f'osd "{osdcap}" mds "{mdscap}"')
+
+    def _remount_both_cephfss(self):
+        keyring = self.fs.mon_manager.get_keyring(self.client_id) + '\n'
+        keyring_path_a = self.mount_a.client_remote.mktemp(data=keyring)
+        keyring_path_b = self.mount_b.client_remote.mktemp(data=keyring)
+
+        self.mount_a.mount(client_id=self.client_id,
+                           client_keyring_path=keyring_path_a,
+                           cephfs_mntpt='/', cephfs_name=self.fs1.name)
+        self.mount_b.mount(client_id=self.client_id,
+                           client_keyring_path=keyring_path_b,
+                           cephfs_mntpt='/', cephfs_name=self.fs2.name)
+
+    def run_rw_tests(self):
+        for captester in self.captesters:
+            captester.conduct_pos_test_for_read_caps()
+            captester.conduct_pos_test_for_write_caps()
+
+    def check_pool_application_metadata_key_value(self, pool, app, key, value):
+        output = self.get_ceph_cmd_stdout(
+            'osd', 'pool', 'application', 'get', pool, app, key)
+        self.assertEqual(str(output.strip()), value)
+
+    def _check_fs_name_on_fs_pools(self, fss):
+        for fs in fss:
+            for pool in fs.get_data_pool_names(True):
+                self.check_pool_application_metadata_key_value(pool, 'cephfs',
+                    'data', fs.name)
+            self.check_pool_application_metadata_key_value(
+                fs.get_metadata_pool_name(), 'cephfs', 'metadata', fs.name)
+
+    def _are_fsnames_and_fscids_together(self):
+        '''
+        Are FS names and FSIDs together on same the FS as they were before
+        running "ceph fs swap" command?
+        '''
+        fs1_id_swapped = self.orig_fs_id_name[self.fs1.id] == self.fs1.name
+        fs2_id_swapped = self.orig_fs_id_name[self.fs2.id] == self.fs2.name
+
+        if fs1_id_swapped and fs2_id_swapped:
+            return True
+        elif not fs1_id_swapped and not fs2_id_swapped:
+            return False
+        else:
+            raise RuntimeError(
+                'Unexpected situation occured: FSID for one FS swapped but '
+                'not for the other FS.')
+
+    def _bring_both_cephfss_down(self):
+        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
+        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
+
+    def _bring_both_cephfss_up(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
+
+    def _refuse_clients_for_both_cephfss(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
+
+    def _accept_clients_for_both_cephfss(self):
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
+                           'false')
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
+                           'false')
+
+
+class TestSwap(SwapHelper):
+    '''
+    Tests for "ceph fs swap" command.
+    '''
+
+    def test_swap_fsnames_but_not_fscids(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=no" swaps the FS names but not
+        the FSCIDs.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        # log output to help debug test failures
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=no '
+                           '--yes_i_really_mean_it')
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # FS names were swapped but not FSIDs, so both can't be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), False)
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.run_rw_tests()
+
+    def test_swap_fsnames_and_fscids(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=yes" swaps the FS names as well
+        as the FSCIDs.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=yes '
+                           '--yes_i_really_mean_it')
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        # XXX: Let's ensure that FS mounted on a mountpoint is same before
+        # and after swapping of FS name and FSCIDs. This ensures that data
+        # available on mountpoints before and after the swap is same. This
+        # prevents self.run_rw_tests() from breaking.
+        #
+        # At the beginning of test, testcephfs1 has data (let's say) 'abc1'
+        # and testcephfs2 has data 'abc2'. self.fs1 is mapped to testcephfs1
+        # and self.fs2 mapped to testcephfs2. After swap, data of testcephfs1
+        # and testcephfs2 will be 'abc2' and 'abc1' respectively.
+        #
+        # However, running self.fs1.getinfo() after swap will map self.fs1 to
+        # FS with FSCID 1 i.e. testcephfs1 and not testcephfs2. Thus, data
+        # under self.fs1 will be different than data before swapping. This
+        # breaks self.run_rw_tests() because self.fs1 is always mounted on
+        # the mountpoint of self.mount_a.
+
+        # To prevent this, therefore, make sure that data on
+        # self.fs1/self.mount_a is same after and before the swap. To ensure
+        # this, swap FS that is represented by self.fs1. Instead of
+        # testcephfs1 it should be testcephfs2 because, after swap,
+        # testcephfs2 containts the data of testcephfs1. This will ensure that
+        # self.mount_rw_tests doesn't break.
+        #
+        # Same for self.fs2.
+        self.fs1.id, self.fs2.id = None, None
+        self.fs1.name, self.fs2.name = self.fs2.name, self.fs1.name
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # both FS name and FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.run_rw_tests()
+
+    def test_swap_without_confirmation_option(self):
+        '''
+        Test that "ceph fs swap --swap-fscids=yes" without the option
+        "--yes-i-really-mean-it" fails.
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('This is a potentially disruptive operation, client\'s cephx '
+               'credentials may need to be reauthorized to access the file '
+               'systems and its pools. Add --yes-i-really-mean-it if you are '
+               'sure you wish to continue.')
+        self.negtest_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
+                              f'{self.fs2.name} {self.fs2.id} '
+                               '--swap-fscids=no',
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
+class TestSwapFsAbsent(SwapHelper):
+    '''
+    Tests for "fs swap" when either FS name is false.
+    '''
+
+    def test_swap_when_fs1_is_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there is no CephFS on cluster by the name "<fs1name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs = 'random_fsname_654'
+        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
+                "cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {absent_cephfs} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_is_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there is no CephFS on cluster by the name "<fs2name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs = 'random_fsname_654'
+        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
+                "cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs2.id} {absent_cephfs} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_fss_are_absent(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when there are no CephFSs on the cluster by the name "<fs1name>" and
+        "<fs2name>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        absent_cephfs1 = 'random_fsname_65'
+        absent_cephfs2 = 'random_fsname_66'
+        msg = (f"Neither file system '{absent_cephfs1}' nor file system "
+               f"'{absent_cephfs2}' exists on this Ceph cluster")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {absent_cephfs1} 123 {absent_cephfs2} 1234 '
+                   '--swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.ENOENT, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
+class TestSwapFscidWrong(SwapHelper):
+    '''
+    Tests for "fs swap" when either FSCID is wrong.
+    '''
+
+    def test_swap_when_fs1_id_is_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs1id>" is not the FSCID of the CephFS named "<fs1nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"FSCID provided for '{self.fs1.name}' is incorrect.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_id_is_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs2id>" is not the FSCID of the CephFS named "<fs2nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"FSCID provided for '{self.fs2.name}' is incorrect.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'123 --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_fscids_are_wrong(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when "<fs1id>" and "<fs2id>", respectively, are not the FSCIDs of the
+        CephFSs named "<fs1name>" and "<fs2nmae>".
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('FSCIDs provided for both the CephFSs is incorrect.')
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} 1234 '
+                  f'--swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EINVAL, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_user_swaps_fscids_in_cmd_args(self):
+        '''
+        Test that "ceph fs swap" fails and prints relevant error message when
+        FSCIDs are exchange while writing the command. That is user write the
+        command as -
+
+        "ceph fs swap <fs1name> <fs2id> <fs2name> <fs1id>"
+
+        instead of writing -
+
+        "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>"
+        '''
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = ('FSCIDs provided in command arguments are swapped; perhaps '
+               '`ceph fs swap` has been run before.')
+        proc = self.run_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs2.id} {self.fs2.name} '
+                  f'{self.fs1.id} --swap-fscids=no --yes_i_really_mean_it'),
+            stderr=StringIO())
+        self.assertIn(msg.lower(), proc.stderr.getvalue().lower())
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
+class TestSwapMirroringOn(SwapHelper):
+    '''
+    # Tests for "fs swap" when mirroring is enabled on FS
+    '''
+
+    def test_swap_when_mirroring_enabled_for_1st_FS(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for the CephFS named "<fs1name>".
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"Mirroring is enabled on file system '{self.fs1.name}'. "
+                "Disable mirroring on the file system after ensuring it's "
+                "OK to do so, and then re-try swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_mirroring_enabled_for_2nd_FS(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for the CephFS named "<fs2name>".
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+
+        self.run_ceph_cmd('fs', 'dump')
+        msg = (f"Mirroring is enabled on file system '{self.fs2.name}'. "
+                "Disable mirroring on the file system after ensuring it's "
+                "OK to do so, and then re-try swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_mirroring_enabled_for_both_FSs(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
+        when mirroring is enabled for both the CephFSs.
+        '''
+        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
+        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
+        self._bring_both_cephfss_down()
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"Mirroring is enabled on file systems '{self.fs1.name}' and "
+               f"'{self.fs2.name}'. Disable mirroring on both the file "
+                "systems after ensuring it's OK to do so, and then re-try "
+                "swapping.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
+        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
+class TestSwapFsOnline(SwapHelper):
+    '''
+    Tests for "fs swap" when either FS is not down/failed.
+    '''
+
+    def test_swap_when_fs1_is_online(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        CephFS named "<fs1name>" is online (i.e. is not failed).
+        '''
+        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs1.name}' is not offline. Before swapping "
+                "CephFS names, both CephFSs should be marked as failed. "
+                "See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_fs2_is_online(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        CephFS named "<fs2name>" is online (i.e. is not failed).
+        '''
+        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs2.name}' is not offline. Before swapping "
+                "CephFS names, both CephFSs should be marked as failed. "
+                "See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_FSs_are_online(self):
+        '''
+        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
+        both the CephFSs are online (i.e. is not failed).
+        '''
+        self._refuse_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' are not "
+                "offline. Before swapping CephFS names, both CephFSs should "
+                "be marked as failed. See `ceph fs fail`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._accept_clients_for_both_cephfss()
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+
+class TestSwapDoesntRefuseClients(SwapHelper):
+    '''
+    Tests for "fs swap" when either FS is offline.
+    '''
+
+    def test_swap_when_FS1_doesnt_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for the first of the two of FSs .
+        '''
+        self._bring_both_cephfss_down()
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs1.name}' doesn't refuse clients. Before "
+                "swapping CephFS names, flag 'refuse_client_session' must "
+                "be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
+                           'false')
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_FS2_doesnt_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for the second of the two of FSs .
+        '''
+        self._bring_both_cephfss_down()
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        msg = (f"CephFS '{self.fs2.name}' doesn't refuse clients. Before "
+                "swapping CephFS names, flag 'refuse_client_session' must "
+                "be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
+                           'false')
+        sleep(2)
+        self.run_ceph_cmd('fs', 'dump')
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
+
+    def test_swap_when_both_FSs_do_not_refuse_clients(self):
+        '''
+        Test that the command "ceph fs swap" command fails when
+        "refuse_client_session" is not set for both the CephFSs.
+        '''
+        self.run_ceph_cmd('fs', 'dump')
+        self._bring_both_cephfss_down()
+        sleep(2)
+        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' do not "
+                "refuse clients. Before swapping CephFS names, flag "
+                "'refuse_client_session' must be set. See `ceph fs set`.")
+        self.negtest_ceph_cmd(
+            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
+                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
+            retval=errno.EPERM, errmsgs=msg)
+        self._bring_both_cephfss_up()
+        self.run_ceph_cmd('fs', 'dump')
+        sleep(2)
+
+        self.fs1.getinfo()
+        self.fs2.getinfo()
+        self._reauthorize_client()
+        self._remount_both_cephfss()
+
+        # check that content of both CephFSs is unaffected by this failure.
+        self.run_rw_tests()
+        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
+        # neither FS name nor FSIDs were swapped, so both must be together
+        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index 79a35574dd7b..b2e10e4d3cf4 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -769,813 +769,6 @@ def test_fs_rename_fails_with_mirroring_enabled(self):
         self.run_ceph_cmd(f'fs mirror disable {orig_fs_name}')
 
 
-class TestSwapCmd(TestAdminCommands):
-    """
-    Tests for "ceph fs swap" command.
-    """
-
-    MDSS_REQUIRED = 3
-    CLIENTS_REQUIRED = 2
-    client_id = 'testuser'
-    client_name = f'client.{client_id}'
-
-    def setUp(self):
-        super(self.__class__, self).setUp()
-        self.fs1 = self.fs
-        self.fs2 = self.mds_cluster.newfs(name='testcephfs2', create=True)
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self.orig_fs_id_name = {self.fs1.id: self.fs1.name,
-                                self.fs2.id: self.fs2.name}
-
-        self.mount_a.remount(cephfs_name=self.fs1.name)
-        self.mount_b.remount(cephfs_name=self.fs2.name)
-
-        self.captesters = (CapTester(self.mount_a), CapTester(self.mount_b))
-
-        self.mount_a.umount_wait()
-        self.mount_b.umount_wait()
-
-    def tearDown(self):
-        self.mount_a.umount_wait()
-        self.mount_b.umount_wait()
-        self.run_ceph_cmd(args=f'auth rm {self.client_name}')
-        super(self.__class__, self).tearDown()
-
-    def _reauthorize_client(self):
-        moncap = gen_mon_cap_str((("rw", self.fs1.name),
-                                  ("rw", self.fs2.name)))
-        osdcap = gen_osd_cap_str((("rw", self.fs1.name),
-                                  ("rw", self.fs2.name)))
-        mdscap = gen_mds_cap_str((("rw", self.fs1.name),
-                                  ("rw", self.fs2.name)))
-        self.run_ceph_cmd(args=f'auth add {self.client_name} mon "{moncap}" '
-                               f'osd "{osdcap}" mds "{mdscap}"')
-
-    def _remount_both_cephfss(self):
-        keyring = self.fs.mon_manager.get_keyring(self.client_id) + '\n'
-        keyring_path_a = self.mount_a.client_remote.mktemp(data=keyring)
-        keyring_path_b = self.mount_b.client_remote.mktemp(data=keyring)
-
-        self.mount_a.mount(client_id=self.client_id,
-                           client_keyring_path=keyring_path_a,
-                           cephfs_mntpt='/', cephfs_name=self.fs1.name)
-        self.mount_b.mount(client_id=self.client_id,
-                           client_keyring_path=keyring_path_b,
-                           cephfs_mntpt='/', cephfs_name=self.fs2.name)
-
-    def run_rw_tests(self):
-        for captester in self.captesters:
-            captester.conduct_pos_test_for_read_caps()
-            captester.conduct_pos_test_for_write_caps()
-
-    def _check_fs_name_on_fs_pools(self, fss):
-        for fs in fss:
-            for pool in fs.get_data_pool_names(True):
-                self.check_pool_application_metadata_key_value(pool, 'cephfs',
-                    'data', fs.name)
-            self.check_pool_application_metadata_key_value(
-                fs.get_metadata_pool_name(), 'cephfs', 'metadata', fs.name)
-
-    def _are_fsnames_and_fscids_together(self):
-        '''
-        Are FS names and FSIDs together on same the FS as they were before
-        running "ceph fs swap" command?
-        '''
-        fs1_id_swapped = self.orig_fs_id_name[self.fs1.id] == self.fs1.name
-        fs2_id_swapped = self.orig_fs_id_name[self.fs2.id] == self.fs2.name
-
-        if fs1_id_swapped and fs2_id_swapped:
-            return True
-        elif not fs1_id_swapped and not fs2_id_swapped:
-            return False
-        else:
-            raise RuntimeError(
-                'Unexpected situation occured: FSID for one FS swapped but '
-                'not for the other FS.')
-
-    def _bring_both_cephfss_down(self):
-        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
-        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
-
-    def _bring_both_cephfss_up(self):
-        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
-        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
-
-    def _refuse_clients_for_both_cephfss(self):
-        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
-        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
-
-    def _accept_clients_for_both_cephfss(self):
-        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
-                           'false')
-        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
-                           'false')
-
-    def test_swap_fsnames_but_not_fscids(self):
-        '''
-        Test that "ceph fs swap --swap-fscids=no" swaps the FS names but not
-        the FSCIDs.
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        # log output to help debug test failures
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
-                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=no '
-                           '--yes_i_really_mean_it')
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # FS names were swapped but not FSIDs, so both can't be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), False)
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        self.run_rw_tests()
-
-    def test_swap_fsnames_and_fscids(self):
-        '''
-        Test that "ceph fs swap --swap-fscids=yes" swaps the FS names as well
-        as the FSCIDs.
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.run_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
-                          f'{self.fs2.name} {self.fs2.id} --swap-fscids=yes '
-                           '--yes_i_really_mean_it')
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        # XXX: Let's ensure that FS mounted on a mountpoint is same before
-        # and after swapping of FS name and FSCIDs. This ensures that data
-        # available on mountpoints before and after the swap is same. This
-        # prevents self.run_rw_tests() from breaking.
-        #
-        # At the beginning of test, testcephfs1 has data (let's say) 'abc1'
-        # and testcephfs2 has data 'abc2'. self.fs1 is mapped to testcephfs1
-        # and self.fs2 mapped to testcephfs2. After swap, data of testcephfs1
-        # and testcephfs2 will be 'abc2' and 'abc1' respectively.
-        #
-        # However, running self.fs1.getinfo() after swap will map self.fs1 to
-        # FS with FSCID 1 i.e. testcephfs1 and not testcephfs2. Thus, data
-        # under self.fs1 will be different than data before swapping. This
-        # breaks self.run_rw_tests() because self.fs1 is always mounted on
-        # the mountpoint of self.mount_a.
-
-        # To prevent this, therefore, make sure that data on
-        # self.fs1/self.mount_a is same after and before the swap. To ensure
-        # this, swap FS that is represented by self.fs1. Instead of
-        # testcephfs1 it should be testcephfs2 because, after swap,
-        # testcephfs2 containts the data of testcephfs1. This will ensure that
-        # self.mount_rw_tests doesn't break.
-        #
-        # Same for self.fs2.
-        self.fs1.id, self.fs2.id = None, None
-        self.fs1.name, self.fs2.name = self.fs2.name, self.fs1.name
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # both FS name and FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        self.run_rw_tests()
-
-    def test_swap_without_confirmation_option(self):
-        '''
-        Test that "ceph fs swap --swap-fscids=yes" without the option
-        "--yes-i-really-mean-it" fails.
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = ('This is a potentially disruptive operation, client\'s cephx '
-               'credentials may need to be reauthorized to access the file '
-               'systems and its pools. Add --yes-i-really-mean-it if you are '
-               'sure you wish to continue.')
-        self.negtest_ceph_cmd(f'fs swap {self.fs1.name} {self.fs1.id} '
-                              f'{self.fs2.name} {self.fs2.id} '
-                               '--swap-fscids=no',
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    ###################################################
-    #
-    # Tests for "fs swap" when either FS name is false.
-    #
-    ###################################################
-
-    def test_swap_when_fs1_is_absent(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when there is no CephFS on cluster by the name "<fs1name>".
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        absent_cephfs = 'random_fsname_654'
-        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
-                "cluster")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {absent_cephfs} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.ENOENT, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_fs2_is_absent(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when there is no CephFS on cluster by the name "<fs2name>".
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        absent_cephfs = 'random_fsname_654'
-        msg = (f"File system '{absent_cephfs}' doesn't exist on this Ceph "
-                "cluster")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs2.id} {absent_cephfs} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.ENOENT, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_both_fss_are_absent(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when there are no CephFSs on the cluster by the name "<fs1name>" and
-        "<fs2name>".
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        absent_cephfs1 = 'random_fsname_65'
-        absent_cephfs2 = 'random_fsname_66'
-        msg = (f"Neither file system '{absent_cephfs1}' nor file system "
-               f"'{absent_cephfs2}' exists on this Ceph cluster")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {absent_cephfs1} 123 {absent_cephfs2} 1234 '
-                   '--swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.ENOENT, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    ###################################################
-    #
-    # Tests for "fs swap" when either FSCID is wrong.
-    #
-    ###################################################
-
-    def test_swap_when_fs1_id_is_wrong(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when "<fs1id>" is not the FSCID of the CephFS named "<fs1nmae>".
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"FSCID provided for '{self.fs1.name}' is incorrect.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EINVAL, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_fs2_id_is_wrong(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when "<fs2id>" is not the FSCID of the CephFS named "<fs2nmae>".
-        '''
-        self._bring_both_cephfss_down()
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"FSCID provided for '{self.fs2.name}' is incorrect.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'123 --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EINVAL, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_both_fscids_are_wrong(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when "<fs1id>" and "<fs2id>", respectively, are not the FSCIDs of the
-        CephFSs named "<fs1name>" and "<fs2nmae>".
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = ('FSCIDs provided for both the CephFSs is incorrect.')
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} 123 {self.fs2.name} 1234 '
-                  f'--swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EINVAL, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_user_swaps_fscids_in_cmd_args(self):
-        '''
-        Test that "ceph fs swap" fails and prints relevant error message when
-        FSCIDs are exchange while writing the command. That is user write the
-        command as -
-
-        "ceph fs swap <fs1name> <fs2id> <fs2name> <fs1id>"
-
-        instead of writing -
-
-        "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>"
-        '''
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = ('FSCIDs provided in command arguments are swapped; perhaps '
-               '`ceph fs swap` has been run before.')
-        proc = self.run_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs2.id} {self.fs2.name} '
-                  f'{self.fs1.id} --swap-fscids=no --yes_i_really_mean_it'),
-            stderr=StringIO())
-        self.assertIn(msg.lower(), proc.stderr.getvalue().lower())
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    #####################################################
-    #
-    # Tests for "fs swap" when mirroring is enabled on FS
-    #
-    #####################################################
-
-    def test_swap_when_mirroring_enabled_for_1st_FS(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when mirroring is enabled for the CephFS named "<fs1name>".
-        '''
-        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"Mirroring is enabled on file system '{self.fs1.name}'. "
-                "Disable mirroring on the file system after ensuring it's "
-                "OK to do so, and then re-try swapping.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_mirroring_enabled_for_2nd_FS(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when mirroring is enabled for the CephFS named "<fs2name>".
-        '''
-        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-
-        self.run_ceph_cmd('fs', 'dump')
-        msg = (f"Mirroring is enabled on file system '{self.fs2.name}'. "
-                "Disable mirroring on the file system after ensuring it's "
-                "OK to do so, and then re-try swapping.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_mirroring_enabled_for_both_FSs(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" fails
-        when mirroring is enabled for both the CephFSs.
-        '''
-        self.run_ceph_cmd(f'fs mirror enable {self.fs1.name}')
-        self.run_ceph_cmd(f'fs mirror enable {self.fs2.name}')
-        self._bring_both_cephfss_down()
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"Mirroring is enabled on file systems '{self.fs1.name}' and "
-               f"'{self.fs2.name}'. Disable mirroring on both the file "
-                "systems after ensuring it's OK to do so, and then re-try "
-                "swapping.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-        self.run_ceph_cmd(f'fs mirror disable {self.fs1.name}')
-        self.run_ceph_cmd(f'fs mirror disable {self.fs2.name}')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    ##########################################################
-    #
-    # Tests for "fs swap" when either FS is not down/failed.
-    #
-    #########################################################
-
-    def test_swap_when_fs1_is_online(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
-        CephFS named "<fs1name>" is online (i.e. is not failed).
-        '''
-        self.run_ceph_cmd(f'fs fail {self.fs2.name}')
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"CephFS '{self.fs1.name}' is not offline. Before swapping "
-                "CephFS names, both CephFSs should be marked as failed."
-                "See `ceph fs fail`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self.run_ceph_cmd(f'fs set {self.fs2.name} joinable true')
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_fs2_is_not_down(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
-        CephFS named "<fs2name>" is online (i.e. is not failed).
-        '''
-        self.run_ceph_cmd(f'fs fail {self.fs1.name}')
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"CephFS '{self.fs2.name}' is not offline. Before swapping "
-                "CephFS names, both CephFSs should be marked as failed. "
-                "See `ceph fs fail`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self.run_ceph_cmd(f'fs set {self.fs1.name} joinable true')
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_both_FSs_are_not_down(self):
-        '''
-        Test that "ceph fs swap <fs1name> <fs1id> <fs2name> <fs2id>" when
-        both the CephFSs are online (i.e. is not failed).
-        '''
-        self._refuse_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' are not "
-                "offline. Before swapping CephFS names, both CephFSs should "
-                "be marked as failed. See `ceph fs fail`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._accept_clients_for_both_cephfss()
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    ##################################################
-    #
-    # Tests for "fs swap" when either FS is offline.
-    #
-    ################################################
-
-
-    def test_swap_when_FS1_doesnt_refuse_clients(self):
-        '''
-        Test that the command "ceph fs swap" command fails when
-        "refuse_client_session" is not set for the first of the two of FSs .
-        '''
-        self._bring_both_cephfss_down()
-        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session true')
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"CephFS '{self.fs1.name}' doesn't refuse clients. Before "
-                "swapping CephFS names, flag 'refuse_client_session' must "
-                "be set. See `ceph fs set`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self.run_ceph_cmd(f'fs set {self.fs2.name} refuse_client_session '
-                           'false')
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_FS2_doesnt_refuse_clients(self):
-        '''
-        Test that the command "ceph fs swap" command fails when
-        "refuse_client_session" is not set for the second of the two of FSs .
-        '''
-        self._bring_both_cephfss_down()
-        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session true')
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        msg = (f"CephFS '{self.fs2.name}' doesn't refuse clients. Before "
-                "swapping CephFS names, flag 'refuse_client_session' must "
-                "be set. See `ceph fs set`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-
-        self._bring_both_cephfss_up()
-        self.run_ceph_cmd(f'fs set {self.fs1.name} refuse_client_session '
-                           'false')
-        sleep(5)
-        self.run_ceph_cmd('fs', 'dump')
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-    def test_swap_when_both_FSs_do_not_refuse_clients(self):
-        '''
-        Test that the command "ceph fs swap" command fails when
-        "refuse_client_session" is not set for both the CephFSs.
-        '''
-        self.run_ceph_cmd('fs', 'dump')
-        self._bring_both_cephfss_down()
-        sleep(5)
-        msg = (f"CephFSs '{self.fs1.name}' and '{self.fs2.name}' do not "
-                "refuse clients. Before swapping CephFS names, flag "
-                "'refuse_client_session' must be set. See `ceph fs set`.")
-        self.negtest_ceph_cmd(
-            args=(f'fs swap {self.fs1.name} {self.fs1.id} {self.fs2.name} '
-                  f'{self.fs2.id} --swap-fscids=no --yes_i_really_mean_it'),
-            retval=errno.EPERM, errmsgs=msg)
-        self._bring_both_cephfss_up()
-        self.run_ceph_cmd('fs', 'dump')
-        sleep(5)
-
-        self.fs1.getinfo()
-        self.fs2.getinfo()
-        self._reauthorize_client()
-        self._remount_both_cephfss()
-
-        # check that content of both CephFSs is unaffected by this failure.
-        self.run_rw_tests()
-        self._check_fs_name_on_fs_pools((self.fs1, self.fs2))
-        # neither FS name nor FSIDs were swapped, so both must be together
-        self.assertEqual(self._are_fsnames_and_fscids_together(), True)
-
-
 class TestDump(CephFSTestCase):
     CLIENTS_REQUIRED = 0
     MDSS_REQUIRED = 1

From dd9f9e268b5562bae8972bce03ec0970f353ae14 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 2 Nov 2023 14:38:02 -0400
Subject: [PATCH 0384/2492] mgr/Mgr: remove shutdown() and handle_signal()

Mgr::shutdown() and Mgr::handle_signal() are never called since
3363a10. The mgr just exits on SIGINT or SIGTERM.

Also found and removed other unreachable code associated with
shutdown in mgr codebase.

Fixes: https://tracker.ceph.com/issues/63410
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 src/mgr/ActivePyModules.cc  | 36 ----------------------------
 src/mgr/ActivePyModules.h   |  1 -
 src/mgr/ClusterState.cc     |  8 -------
 src/mgr/ClusterState.h      |  1 -
 src/mgr/DaemonServer.cc     | 19 ---------------
 src/mgr/DaemonServer.h      |  2 --
 src/mgr/Mgr.cc              | 27 ---------------------
 src/mgr/Mgr.h               |  3 ---
 src/mgr/MgrStandby.cc       | 35 ---------------------------
 src/mgr/MgrStandby.h        |  1 -
 src/mgr/PyModuleRegistry.cc | 47 -------------------------------------
 src/mgr/PyModuleRegistry.h  |  3 ---
 12 files changed, 183 deletions(-)

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 45038e734afc..2f4f6bd1e80b 100644
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -555,42 +555,6 @@ void ActivePyModules::start_one(PyModuleRef py_module)
   }));
 }
 
-void ActivePyModules::shutdown()
-{
-  std::lock_guard locker(lock);
-
-  // Stop per active module finisher thread
-  for (auto& [name, module] : modules) {
-      dout(4) << "Stopping active module " << name << " finisher thread" << dendl;
-      module->finisher.wait_for_empty();
-      module->finisher.stop();
-  }
-
-  // Signal modules to drop out of serve() and/or tear down resources
-  for (auto& [name, module] : modules) {
-    lock.unlock();
-    dout(10) << "calling module " << name << " shutdown()" << dendl;
-    module->shutdown();
-    dout(10) << "module " << name << " shutdown() returned" << dendl;
-    lock.lock();
-  }
-
-  // For modules implementing serve(), finish the threads where we
-  // were running that.
-  for (auto& [name, module] : modules) {
-    lock.unlock();
-    dout(10) << "joining module " << name << dendl;
-    module->thread.join();
-    dout(10) << "joined module " << name << dendl;
-    lock.lock();
-  }
-
-  cmd_finisher.wait_for_empty();
-  cmd_finisher.stop();
-
-  modules.clear();
-}
-
 void ActivePyModules::notify_all(const std::string &notify_type,
                      const std::string &notify_id)
 {
diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h
index 283f96a6ed95..d6ade4849f78 100644
--- a/src/mgr/ActivePyModules.h
+++ b/src/mgr/ActivePyModules.h
@@ -216,7 +216,6 @@ class ActivePyModules
       std::string *err);
 
   int init();
-  void shutdown();
 
   void start_one(PyModuleRef py_module);
 
diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 7f811a5e4159..6b106268efc4 100644
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -225,14 +225,6 @@ void ClusterState::final_init()
   ceph_assert(r == 0);
 }
 
-void ClusterState::shutdown()
-{
-  // unregister commands
-  g_ceph_context->get_admin_socket()->unregister_commands(asok_hook);
-  delete asok_hook;
-  asok_hook = NULL;
-}
-
 bool ClusterState::asok_command(
   std::string_view admin_command,
   const cmdmap_t& cmdmap,
diff --git a/src/mgr/ClusterState.h b/src/mgr/ClusterState.h
index 7939cd8eb8f0..2beac362b472 100644
--- a/src/mgr/ClusterState.h
+++ b/src/mgr/ClusterState.h
@@ -152,7 +152,6 @@ class ClusterState
   }
 
   void final_init();
-  void shutdown();
   bool asok_command(std::string_view admin_command,
 		    const cmdmap_t& cmdmap,
 		    Formatter *f,
diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index a4e85e2fc421..b1781316f828 100644
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -98,7 +98,6 @@ DaemonServer::DaemonServer(MonClient *monc_,
       audit_clog(audit_clog_),
       pgmap_ready(false),
       timer(g_ceph_context, lock),
-      shutting_down(false),
       tick_event(nullptr),
       osd_perf_metric_collector_listener(this),
       osd_perf_metric_collector(osd_perf_metric_collector_listener),
@@ -358,11 +357,6 @@ void DaemonServer::schedule_tick_locked(double delay_sec)
     tick_event = nullptr;
   }
 
-  // on shutdown start rejecting explicit requests to send reports that may
-  // originate from python land which may still be running.
-  if (shutting_down)
-    return;
-
   tick_event = timer.add_event_after(delay_sec,
     new LambdaContext([this](int r) {
       tick();
@@ -407,19 +401,6 @@ void DaemonServer::handle_mds_perf_metric_query_updated()
       }));
 }
 
-void DaemonServer::shutdown()
-{
-  dout(10) << "begin" << dendl;
-  msgr->shutdown();
-  msgr->wait();
-  cluster_state.shutdown();
-  dout(10) << "done" << dendl;
-
-  std::lock_guard l(lock);
-  shutting_down = true;
-  timer.shutdown();
-}
-
 static DaemonKey key_from_service(
   const std::string& service_name,
   int peer_type,
diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index a7b645610043..43125533e745 100644
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -190,7 +190,6 @@ class DaemonServer : public Dispatcher, public md_config_obs_t
   void maybe_ready(int32_t osd_id);
 
   SafeTimer timer;
-  bool shutting_down;
   Context *tick_event;
   void tick();
   void schedule_tick_locked(double delay_sec);
@@ -255,7 +254,6 @@ class DaemonServer : public Dispatcher, public md_config_obs_t
 
 public:
   int init(uint64_t gid, entity_addrvec_t client_addrs);
-  void shutdown();
 
   entity_addrvec_t get_myaddrs() const;
 
diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc
index 63ad530fae2f..5bd2ffb246c7 100644
--- a/src/mgr/Mgr.cc
+++ b/src/mgr/Mgr.cc
@@ -214,12 +214,6 @@ std::map<std::string, std::string> Mgr::load_store()
   return loaded;
 }
 
-void Mgr::handle_signal(int signum)
-{
-  ceph_assert(signum == SIGINT || signum == SIGTERM);
-  shutdown();
-}
-
 static void handle_mgr_signal(int signum)
 {
   derr << " *** Got signal " << sig_str(signum) << " ***" << dendl;
@@ -490,27 +484,6 @@ void Mgr::load_all_metadata()
   }
 }
 
-
-void Mgr::shutdown()
-{
-  dout(10) << "mgr shutdown init" << dendl;
-  finisher.queue(new LambdaContext([&](int) {
-    {
-      std::lock_guard l(lock);
-      // First stop the server so that we're not taking any more incoming
-      // requests
-      server.shutdown();
-    }
-    // after the messenger is stopped, signal modules to shutdown via finisher
-    py_module_registry->active_shutdown();
-  }));
-
-  // Then stop the finisher to ensure its enqueued contexts aren't going
-  // to touch references to the things we're about to tear down
-  finisher.wait_for_empty();
-  finisher.stop();
-}
-
 void Mgr::handle_osd_map()
 {
   ceph_assert(ceph_mutex_is_locked_by_me(lock));
diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h
index 22ebdb680410..65931c331f36 100644
--- a/src/mgr/Mgr.h
+++ b/src/mgr/Mgr.h
@@ -94,9 +94,6 @@ class Mgr : public AdminSocketHook {
   bool ms_dispatch2(const ceph::ref_t<Message>& m);
 
   void background_init(Context *completion);
-  void shutdown();
-
-  void handle_signal(int signum);
 
   std::map<std::string, std::string> get_services() const;
 
diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 545624eb79b0..052e68681772 100644
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -295,41 +295,6 @@ void MgrStandby::tick()
   ));
 }
 
-void MgrStandby::shutdown()
-{
-  finisher.queue(new LambdaContext([&](int) {
-    std::lock_guard l(lock);
-
-    dout(4) << "Shutting down" << dendl;
-
-    py_module_registry.shutdown();
-    // stop sending beacon first, I use monc to talk with monitors
-    timer.shutdown();
-    // client uses monc and objecter
-    client.shutdown();
-    mgrc.shutdown();
-    // Stop asio threads, so leftover events won't call into shut down
-    // monclient/objecter.
-    poolctx.finish();
-    // stop monc, so mon won't be able to instruct me to shutdown/activate after
-    // the active_mgr is stopped
-    monc.shutdown();
-    if (active_mgr) {
-      active_mgr->shutdown();
-    }
-    // objecter is used by monc and active_mgr
-    objecter.shutdown();
-    // client_messenger is used by all of them, so stop it in the end
-    client_messenger->shutdown();
-  }));
-
-  // Then stop the finisher to ensure its enqueued contexts aren't going
-  // to touch references to the things we're about to tear down
-  finisher.wait_for_empty();
-  finisher.stop();
-  mgr_perf_stop(g_ceph_context);
-}
-
 void MgrStandby::respawn()
 {
   // --- WARNING TO FUTURE COPY/PASTERS ---
diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h
index 0f06e3074a00..5d238c855779 100644
--- a/src/mgr/MgrStandby.h
+++ b/src/mgr/MgrStandby.h
@@ -79,7 +79,6 @@ class MgrStandby : public Dispatcher,
   bool ms_handle_refused(Connection *con) override;
 
   int init();
-  void shutdown();
   void respawn();
   int main(std::vector<const char *> args);
   void tick();
diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
index f5f5008023fc..eb2d2babe75f 100644
--- a/src/mgr/PyModuleRegistry.cc
+++ b/src/mgr/PyModuleRegistry.cc
@@ -217,53 +217,6 @@ void PyModuleRegistry::active_start(
   }
 }
 
-void PyModuleRegistry::active_shutdown()
-{
-  std::lock_guard locker(lock);
-
-  if (active_modules != nullptr) {
-    active_modules->shutdown();
-    active_modules.reset();
-  }
-}
-
-void PyModuleRegistry::shutdown()
-{
-  std::lock_guard locker(lock);
-
-  if (standby_modules != nullptr) {
-    standby_modules->shutdown();
-    standby_modules.reset();
-  }
-
-  // Ideally, now, we'd be able to do this for all modules:
-  //
-  //    Py_EndInterpreter(pMyThreadState);
-  //    PyThreadState_Swap(pMainThreadState);
-  //
-  // Unfortunately, if the module has any other *python* threads active
-  // at this point, Py_EndInterpreter() will abort with:
-  //
-  //    Fatal Python error: Py_EndInterpreter: not the last thread
-  //
-  // This can happen when using CherryPy in a module, becuase CherryPy
-  // runs an extra thread as a timeout monitor, which spends most of its
-  // life inside a time.sleep(60).  Unless you are very, very lucky with
-  // the timing calling this destructor, that thread will still be stuck
-  // in a sleep, and Py_EndInterpreter() will abort.
-  //
-  // This could of course also happen with a poorly written module which
-  // made no attempt to clean up any additional threads it created.
-  //
-  // The safest thing to do is just not call Py_EndInterpreter(), and
-  // let Py_Finalize() kill everything after all modules are shut down.
-
-  modules.clear();
-
-  PyEval_RestoreThread(pMainThreadState);
-  Py_Finalize();
-}
-
 std::vector<std::string> PyModuleRegistry::probe_modules(const std::string &path) const
 {
   const auto opt = g_conf().get_val<std::string>("mgr_disabled_modules");
diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h
index 9af9abb57625..9d6d9c2cdd02 100644
--- a/src/mgr/PyModuleRegistry.h
+++ b/src/mgr/PyModuleRegistry.h
@@ -122,9 +122,6 @@ class PyModuleRegistry
     return standby_modules != nullptr;
   }
 
-  void active_shutdown();
-  void shutdown();
-
   std::vector<MonCommand> get_commands() const;
   std::vector<ModuleCommand> get_py_commands() const;
 

From 16d12f3c667f7e11b700879406011ffe4bc2d37f Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Fri, 3 Nov 2023 15:06:08 +0100
Subject: [PATCH 0385/2492] mgr/rook: adding behave as testing python
 dependency

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini
index 0f2fe777f055..6270dbdca51e 100644
--- a/src/pybind/mgr/tox.ini
+++ b/src/pybind/mgr/tox.ini
@@ -49,6 +49,7 @@ setenv =
     UNITTEST = true
     PYTHONPATH = $PYTHONPATH:..
 deps =
+    behave
     -rrequirements.txt
     -rrook/requirements.txt
 commands =

From e96cef78bb97491e6488999d6bc4a068cd1c5a8d Mon Sep 17 00:00:00 2001
From: Aliaksei Makarau <aliaksei.makarau@ibm.com>
Date: Tue, 31 Oct 2023 16:20:05 +0100
Subject: [PATCH 0386/2492] doc: Update dynamicresharding.rst

"3, After resharding completed:" should be
"3. After resharding completed:"

Signed-off-by: Aliaksei Makarau <aliaksei.makarau@ibm.com>
---
 doc/radosgw/dynamicresharding.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/radosgw/dynamicresharding.rst b/doc/radosgw/dynamicresharding.rst
index db4ab57befe0..b58a94980c40 100644
--- a/doc/radosgw/dynamicresharding.rst
+++ b/doc/radosgw/dynamicresharding.rst
@@ -122,7 +122,7 @@ For example, the output at different Dynamic Resharding stages is shown below:
     }
   ]
 
-``3, After resharding completed:``
+``3. After resharding completed:``
 ::
 
   [

From 5aeded3afc2b83e5acb472a68b33cb3650d35073 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Thu, 2 Nov 2023 13:00:34 +0100
Subject: [PATCH 0387/2492] mgr/dashboard: fix secure_monitoring_stack check
 Fixes: https://tracker.ceph.com/issues/63326

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 .../mgr/dashboard/controllers/prometheus.py   | 32 ++++++------
 .../mgr/dashboard/tests/test_prometheus.py    | 49 ++++++++++++++-----
 2 files changed, 55 insertions(+), 26 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/prometheus.py b/src/pybind/mgr/dashboard/controllers/prometheus.py
index 7222b14f7b5c..b639d8826273 100644
--- a/src/pybind/mgr/dashboard/controllers/prometheus.py
+++ b/src/pybind/mgr/dashboard/controllers/prometheus.py
@@ -61,20 +61,24 @@ def get_access_info(self, module_name):
         user = None
         password = None
         cert_file = None
-        secure_monitoring_stack = bool(mgr.get_module_option_ex('cephadm',
-                                                                'secure_monitoring_stack',
-                                                                'false'))
-        if secure_monitoring_stack:
-            cmd = {'prefix': f'orch {module_name} get-credentials'}
-            ret, out, _ = mgr.mon_command(cmd)
-            if ret == 0 and out is not None:
-                access_info = json.loads(out)
-                user = access_info['user']
-                password = access_info['password']
-                certificate = access_info['certificate']
-                cert_file = tempfile.NamedTemporaryFile(delete=False)
-                cert_file.write(certificate.encode('utf-8'))
-                cert_file.flush()
+
+        orch_backend = mgr.get_module_option_ex('orchestrator', 'orchestrator')
+        if orch_backend == 'cephadm':
+            secure_monitoring_stack = mgr.get_module_option_ex('cephadm',
+                                                               'secure_monitoring_stack',
+                                                               False)
+            if secure_monitoring_stack:
+                cmd = {'prefix': f'orch {module_name} get-credentials'}
+                ret, out, _ = mgr.mon_command(cmd)
+                if ret == 0 and out is not None:
+                    access_info = json.loads(out)
+                    user = access_info['user']
+                    password = access_info['password']
+                    certificate = access_info['certificate']
+                    cert_file = tempfile.NamedTemporaryFile(delete=False)
+                    cert_file.write(certificate.encode('utf-8'))
+                    cert_file.flush()
+
         return user, password, cert_file
 
     def _get_api_url(self, host):
diff --git a/src/pybind/mgr/dashboard/tests/test_prometheus.py b/src/pybind/mgr/dashboard/tests/test_prometheus.py
index 21c4a0b10e9c..10aa8669ec0e 100644
--- a/src/pybind/mgr/dashboard/tests/test_prometheus.py
+++ b/src/pybind/mgr/dashboard/tests/test_prometheus.py
@@ -26,28 +26,53 @@ def setup_server(cls):
         mgr.get_module_option.side_effect = settings.get
         cls.setup_controllers([Prometheus, PrometheusNotifications, PrometheusReceiver])
 
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
-    def test_rules(self):
-        with patch('requests.request') as mock_request:
-            self._get('/api/prometheus/rules')
-            mock_request.assert_called_with('GET', self.prometheus_host_api + '/rules',
-                                            json=None, params={}, verify=True, auth=None)
-
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", return_value='cephadm')
+    @patch("dashboard.controllers.prometheus.mgr.mon_command", return_value=(1, {}, None))
+    @patch('requests.request')
+    def test_rules_cephadm(self, mock_request, mock_mon_command, mock_get_module_option_ex):
+        # in this test we use:
+        # in the first call to get_module_option_ex we return 'cephadm' as backend
+        # in the second call we return 'True' for 'secure_monitoring_stack' option
+        mock_get_module_option_ex.side_effect = lambda module, key, default=None: 'cephadm' \
+            if module == 'orchestrator' else True
+        self._get('/api/prometheus/rules')
+        mock_request.assert_called_with('GET',
+                                        self.prometheus_host_api + '/rules',
+                                        json=None, params={},
+                                        verify=True, auth=None)
+        assert mock_mon_command.called
+
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", return_value='cephadm')
+    @patch("dashboard.controllers.prometheus.mgr.mon_command", return_value=(1, {}, None))
+    @patch('requests.request')
+    def test_rules_rook(self, mock_request, mock_mon_command, mock_get_module_option_ex):
+        # in this test we use:
+        # in the first call to get_module_option_ex we return 'rook' as backend
+        mock_get_module_option_ex.side_effect = lambda module, key, default=None: 'rook' \
+            if module == 'orchestrator' else None
+        self._get('/api/prometheus/rules')
+        mock_request.assert_called_with('GET',
+                                        self.prometheus_host_api + '/rules',
+                                        json=None,
+                                        params={},
+                                        verify=True, auth=None)
+        assert not mock_mon_command.called
+
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c=None: None)
     def test_list(self):
         with patch('requests.request') as mock_request:
             self._get('/api/prometheus')
             mock_request.assert_called_with('GET', self.alert_host_api + '/alerts',
                                             json=None, params={}, verify=True, auth=None)
 
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c=None: None)
     def test_get_silences(self):
         with patch('requests.request') as mock_request:
             self._get('/api/prometheus/silences')
             mock_request.assert_called_with('GET', self.alert_host_api + '/silences',
                                             json=None, params={}, verify=True, auth=None)
 
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c=None: None)
     def test_add_silence(self):
         with patch('requests.request') as mock_request:
             self._post('/api/prometheus/silence', {'id': 'new-silence'})
@@ -55,7 +80,7 @@ def test_add_silence(self):
                                             params=None, json={'id': 'new-silence'},
                                             verify=True, auth=None)
 
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c=None: None)
     def test_update_silence(self):
         with patch('requests.request') as mock_request:
             self._post('/api/prometheus/silence', {'id': 'update-silence'})
@@ -63,7 +88,7 @@ def test_update_silence(self):
                                             params=None, json={'id': 'update-silence'},
                                             verify=True, auth=None)
 
-    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c: False)
+    @patch("dashboard.controllers.prometheus.mgr.get_module_option_ex", lambda a, b, c=None: None)
     def test_expire_silence(self):
         with patch('requests.request') as mock_request:
             self._delete('/api/prometheus/silence/0')

From 19378b11756e5d985421e1978bb2d59159bb3f21 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 09:42:34 -0400
Subject: [PATCH 0388/2492] cephadm: move extract_uid_gid func to
 container_types module

While extract_uid_gid isn't a perfect fit for container_types it is a
fairly fundamental function for working with containers in cephadm and
doesn't require anything beyond types in containers_types and that
module's existing imports.  Moving extract_uid_gid should allow us to
more easily move other functions in the future.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                    | 32 +---------------------
 src/cephadm/cephadmlib/container_types.py | 33 ++++++++++++++++++++++-
 src/cephadm/tests/test_ingress.py         |  4 +--
 3 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ca068c13757c..47de7e5627be 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -153,6 +153,7 @@
     CephContainer,
     InitContainer,
     is_container_running,
+    extract_uid_gid,
 )
 from cephadmlib.decorators import (
     deprecated_command,
@@ -2828,37 +2829,6 @@ def _update_container_args_for_podman(
     )
 
 
-def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
-    # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int]
-
-    if not img:
-        img = ctx.image
-
-    if isinstance(file_path, str):
-        paths = [file_path]
-    else:
-        paths = file_path
-
-    ex: Optional[Tuple[str, RuntimeError]] = None
-
-    for fp in paths:
-        try:
-            out = CephContainer(
-                ctx,
-                image=img,
-                entrypoint='stat',
-                args=['-c', '%u %g', fp]
-            ).run(verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
-            uid, gid = out.split(' ')
-            return int(uid), int(gid)
-        except RuntimeError as e:
-            ex = (fp, e)
-    if ex:
-        raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}')
-
-    raise RuntimeError('uid/gid not found')
-
-
 def deploy_daemon(
     ctx: CephadmContext,
     ident: 'DaemonIdentity',
diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 34c7ed29ada0..913a9cfe312e 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -2,7 +2,7 @@
 
 import os
 
-from typing import Dict, List, Optional, Any
+from typing import Dict, List, Optional, Any, Union, Tuple
 
 from .call_wrappers import call, call_throws, CallVerbosity
 from .constants import DEFAULT_TIMEOUT
@@ -485,3 +485,34 @@ def get_running_container_name(
         if out.strip() == 'running':
             return name
     return None
+
+
+def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
+    # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int]
+
+    if not img:
+        img = ctx.image
+
+    if isinstance(file_path, str):
+        paths = [file_path]
+    else:
+        paths = file_path
+
+    ex: Optional[Tuple[str, RuntimeError]] = None
+
+    for fp in paths:
+        try:
+            out = CephContainer(
+                ctx,
+                image=img,
+                entrypoint='stat',
+                args=['-c', '%u %g', fp]
+            ).run(verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
+            uid, gid = out.split(' ')
+            return int(uid), int(gid)
+        except RuntimeError as e:
+            ex = (fp, e)
+    if ex:
+        raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}')
+
+    raise RuntimeError('uid/gid not found')
diff --git a/src/cephadm/tests/test_ingress.py b/src/cephadm/tests/test_ingress.py
index 08a9808ddbba..698305aa4f38 100644
--- a/src/cephadm/tests/test_ingress.py
+++ b/src/cephadm/tests/test_ingress.py
@@ -166,7 +166,7 @@ def test_haproxy_extract_uid_gid_haproxy():
             good_haproxy_json(),
             SAMPLE_HAPROXY_IMAGE,
         )
-        with mock.patch("cephadm.CephContainer") as cc:
+        with mock.patch("cephadmlib.container_types.CephContainer") as cc:
             cc.return_value.run.return_value = "500 500"
             uid, gid = hap.uid_gid(ctx)
             cc.return_value.run.assert_called()
@@ -329,7 +329,7 @@ def test_keepalived_extract_uid_gid_keepalived():
             good_keepalived_json(),
             SAMPLE_KEEPALIVED_IMAGE,
         )
-        with mock.patch("cephadm.CephContainer") as cc:
+        with mock.patch("cephadmlib.container_types.CephContainer") as cc:
             cc.return_value.run.return_value = "500 500"
             uid, gid = kad.uid_gid(ctx)
             cc.return_value.run.assert_called()

From d8860c1336d2bf8c32d2105072ce153827452d74 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 09:46:58 -0400
Subject: [PATCH 0389/2492] cephadm: update extract_uid_gid function
 annotations & format

Update the type annotations of extract_uid_gid to the type annotations
form used in more recent python versions. Reformat the function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 913a9cfe312e..7ec323acf314 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -487,8 +487,11 @@ def get_running_container_name(
     return None
 
 
-def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
-    # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int]
+def extract_uid_gid(
+    ctx: CephadmContext,
+    img: str = '',
+    file_path: Union[str, List[str]] = '/var/lib/ceph',
+) -> Tuple[int, int]:
 
     if not img:
         img = ctx.image
@@ -503,10 +506,7 @@ def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
     for fp in paths:
         try:
             out = CephContainer(
-                ctx,
-                image=img,
-                entrypoint='stat',
-                args=['-c', '%u %g', fp]
+                ctx, image=img, entrypoint='stat', args=['-c', '%u %g', fp]
             ).run(verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
             uid, gid = out.split(' ')
             return int(uid), int(gid)

From fc45a32734d72e2e7b569242581cb2adcd0738bf Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Fri, 3 Feb 2023 16:46:05 -0500
Subject: [PATCH 0390/2492] rgwlc: dispatch S3 notifications on transition and
 mpu abort

Fixes: https://tracker.ceph.com/issues/58641

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 doc/radosgw/s3-notification-compatibility.rst |   2 +-
 src/rgw/rgw_lc.cc                             | 103 ++++++++-
 src/test/rgw/bucket_notification/test_bn.py   | 215 +++++++++++-------
 3 files changed, 228 insertions(+), 92 deletions(-)

diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
index 1627ed0c4db0..cced60924d09 100644
--- a/doc/radosgw/s3-notification-compatibility.rst
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -91,7 +91,7 @@ Event Types
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectLifecycle:Expiration:DeleteMarker``         | Ceph extension                          |
 +--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Expiration:AbortMultipartUpload`` | Defined, Ceph extension (not generated) |
+| ``s3:ObjectLifecycle:Expiration:AbortMultipartUpload`` | Ceph extension                          |
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectLifecycle:Transition:Current``              | Ceph extension                          |
 +--------------------------------------------------------+-----------------------------------------+
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 4887c9d146a6..cf0b65110507 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -828,15 +828,44 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
   params.ns = RGW_OBJ_NS_MULTIPART;
   params.access_list_filter = &mp_filter;
 
+  auto event_type = rgw::notify::ObjectExpirationAbortMPU;
+  std::string version_id;
+
   auto pf = [&](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) {
+    int ret{0};
     auto wt = boost::get<std::tuple<lc_op, rgw_bucket_dir_entry>>(wi);
     auto& [rule, obj] = wt;
     if (obj_has_expired(this, cct, obj.meta.mtime, rule.mp_expiration)) {
       rgw_obj_key key(obj.key);
       std::unique_ptr<rgw::sal::MultipartUpload> mpu = target->get_multipart_upload(key.name);
-      int ret = mpu->abort(this, cct, null_yield);
+      std::unique_ptr<rgw::sal::Object> sal_obj
+	= target->get_object(key);
+      std::unique_ptr<rgw::sal::Notification> notify
+	= driver->get_notification(
+	  this, sal_obj.get(), nullptr, event_type,
+	  target, lc_id,
+	  const_cast<std::string&>(target->get_tenant()),
+	  lc_req_id, null_yield);
+
+      ret = notify->publish_reserve(this, nullptr);
+      if (ret != 0) {
+	ldpp_dout(wk->get_lc(), 0)
+	  << "ERROR: reserving persistent notification for abort_multipart_upload, ret=" << ret
+	  << ", thread:" << wq->thr_name()
+	  << ", meta:" << obj.key
+	  << dendl;
+      }
+
+      ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
-        if (perfcounter) {
+
+        (void) notify->publish_commit(
+	  this, sal_obj->get_obj_size(),
+	  ceph::real_clock::now(),
+	  sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
+	  version_id);
+
+	if (perfcounter) {
           perfcounter->inc(l_rgw_lc_abort_mpu, 1);
         }
       } else {
@@ -1268,27 +1297,78 @@ class LCOpAction_Transition : public LCOpAction {
     /* If bucket is versioned, create delete_marker for current version
      */
     if (oc.bucket->versioned() && oc.o.is_current() && !oc.o.is_delete_marker()) {
-      ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectExpiration);
+      ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransitionCurrent);
       ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") current & not delete_marker" << " versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
     } else {
-      ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectExpiration);
+      ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransitionNoncurrent);
       ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not current " << "versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
     }
     return ret;
   }
 
   int transition_obj_to_cloud(lc_op_ctx& oc) {
+    int ret{0};
     /* If CurrentVersion object, remove it & create delete marker */
     bool delete_object = (!oc.tier->retain_head_object() ||
                      (oc.o.is_current() && oc.bucket->versioned()));
 
-    int ret = oc.obj->transition_to_cloud(oc.bucket, oc.tier.get(), oc.o,
-					  oc.env.worker->get_cloud_targets(), oc.cct,
-					  !delete_object, oc.dpp, null_yield);
+    /* notifications */
+    std::unique_ptr<rgw::sal::Bucket> bucket;
+    std::unique_ptr<rgw::sal::Object> obj;
+    auto& bucket_info = oc.bucket->get_info();
+    std::string version_id;
+
+    ret = oc.driver->get_bucket(nullptr, bucket_info, &bucket);
     if (ret < 0) {
       return ret;
     }
 
+    std::unique_ptr<rgw::sal::User> user;
+    if (! bucket->get_owner()) {
+      auto& bucket_info = bucket->get_info();
+      user = oc.driver->get_user(bucket_info.owner);
+      if (user) {
+	bucket->set_owner(user.get());
+      }
+    }
+
+    obj = bucket->get_object(oc.o.key);
+
+    auto event_type = (oc.bucket->versioned() &&
+		       oc.o.is_current() && !oc.o.is_delete_marker()) ?
+      rgw::notify::ObjectTransitionCurrent :
+      rgw::notify::ObjectTransitionNoncurrent;
+
+    std::unique_ptr<rgw::sal::Notification> notify
+      = oc.driver->get_notification(
+	oc.dpp, obj.get(), nullptr, event_type,
+	bucket.get(), lc_id,
+	const_cast<std::string&>(oc.bucket->get_tenant()),
+	lc_req_id, null_yield);
+
+    ret = notify->publish_reserve(oc.dpp, nullptr);
+    if (ret < 0) {
+      ldpp_dout(oc.dpp, 1)
+	<< "ERROR: notify reservation failed, deferring transition of object k="
+	<< oc.o.key
+	<< dendl;
+      return ret;
+    }
+
+    ret = oc.obj->transition_to_cloud(oc.bucket, oc.tier.get(), oc.o,
+				      oc.env.worker->get_cloud_targets(),
+				      oc.cct, !delete_object, oc.dpp,
+				      null_yield);
+    if (ret < 0) {
+      return ret;
+    } else {
+      // send request to notification manager
+      (void) notify->publish_commit(oc.dpp, obj->get_obj_size(),
+				    ceph::real_clock::now(),
+				    obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
+				    version_id);
+    }
+
     if (delete_object) {
       ret = delete_tier_obj(oc);
       if (ret < 0) {
@@ -1661,6 +1741,15 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     worker->workpool->drain();
   }
 
+  std::unique_ptr<rgw::sal::User> user;
+  if (! bucket->get_owner()) {
+    auto& bucket_info = bucket->get_info();
+    std::unique_ptr<rgw::sal::User> user = driver->get_user(bucket_info.owner);
+      if (user) {
+	bucket->set_owner(user.get());
+      }
+  }
+
   ret = handle_multipart_expiration(bucket.get(), prefix_map, worker, stop_at, once);
   return ret;
 }
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 4fd9cca12d62..87d3ca8eeaa5 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -7,18 +7,20 @@
 import socket
 import time
 import os
+import io
 import string
 import boto
-from botocore.exceptions import ClientError
 from http import server as http_server
 from random import randint
 import hashlib
+# XXX this should be converted to use pytest
 from nose.plugins.attrib import attr
 import boto3
 import datetime
 from cloudevents.http import from_http
 from dateutil import parser
 
+# XXX this should be converted to use boto3
 from boto.s3.connection import S3Connection
 
 from . import(
@@ -553,8 +555,17 @@ def another_user(tenant=None):
 @attr('basic_test')
 def test_ps_s3_topic_on_master():
     """ test s3 topics set/get/delete on master """
+    
+    access_key = str(time.time())
+    secret_key = str(time.time())
+    uid = 'superman' + str(time.time())
     tenant = 'kaboom'
-    conn = another_user(tenant)
+    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+    assert_equal(result, 0)
+    conn = S3Connection(aws_access_key_id=access_key,
+                  aws_secret_access_key=secret_key,
+                      is_secure=False, port=get_config_port(), host=get_config_host(), 
+                      calling_format='boto.s3.connection.OrdinaryCallingFormat')
     zonegroup = 'default' 
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
@@ -625,8 +636,17 @@ def test_ps_s3_topic_on_master():
 @attr('basic_test')
 def test_ps_s3_topic_admin_on_master():
     """ test s3 topics set/get/delete on master """
+    
+    access_key = str(time.time())
+    secret_key = str(time.time())
+    uid = 'superman' + str(time.time())
     tenant = 'kaboom'
-    conn = another_user(tenant)
+    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+    assert_equal(result, 0)
+    conn = S3Connection(aws_access_key_id=access_key,
+                  aws_secret_access_key=secret_key,
+                      is_secure=False, port=get_config_port(), host=get_config_host(), 
+                      calling_format='boto.s3.connection.OrdinaryCallingFormat')
     zonegroup = 'default' 
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
@@ -1216,87 +1236,6 @@ def test_ps_s3_notification_errors_on_master():
     # delete the bucket
     conn.delete_bucket(bucket_name)
 
-@attr('basic_test')
-def test_ps_s3_notification_permissions():
-    """ test s3 notification set/get/delete permissions """
-    conn1 = connection()
-    conn2 = another_user()
-    zonegroup = 'default'
-    bucket_name = gen_bucket_name()
-    # create bucket
-    bucket = conn1.create_bucket(bucket_name)
-    topic_name = bucket_name + TOPIC_SUFFIX
-    # create s3 topic
-    endpoint_address = 'amqp://127.0.0.1:7001'
-    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
-    topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args)
-    topic_arn = topic_conf.set_config()
-
-    # one user create a notification
-    notification_name = bucket_name + NOTIFICATION_SUFFIX
-    topic_conf_list = [{'Id': notification_name,
-                        'TopicArn': topic_arn,
-                        'Events': []
-                       }]
-    s3_notification_conf1 = PSNotificationS3(conn1, bucket_name, topic_conf_list)
-    _, status = s3_notification_conf1.set_config()
-    assert_equal(status, 200)
-    # another user try to fetch it
-    s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
-    try:
-        _, _ = s3_notification_conf2.get_config()
-        assert False, "'AccessDenied' error is expected"
-    except ClientError as error:
-        assert_equal(error.response['Error']['Code'], 'AccessDenied')
-    # other user try to delete the notification
-    _, status = s3_notification_conf2.del_config()
-    assert_equal(status, 403)
-
-    # bucket policy is added by the 1st user
-    client = boto3.client('s3',
-            endpoint_url='http://'+conn1.host+':'+str(conn1.port),
-            aws_access_key_id=conn1.aws_access_key_id,
-            aws_secret_access_key=conn1.aws_secret_access_key)
-    bucket_policy = json.dumps({
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Sid": "Statement",
-                "Effect": "Allow",
-                "Principal": "*",
-                "Action": ["s3:GetBucketNotification", "s3:PutBucketNotification"],
-                "Resource": f"arn:aws:s3:::{bucket_name}"
-            }
-        ]
-    })
-    response = client.put_bucket_policy(Bucket=bucket_name, Policy=bucket_policy)
-    assert_equal(int(response['ResponseMetadata']['HTTPStatusCode']/100), 2) 
-    result = client.get_bucket_policy(Bucket=bucket_name)
-    print(result['Policy'])
-    
-    # 2nd user try to fetch it again
-    _, status = s3_notification_conf2.get_config()
-    assert_equal(status, 200)
-
-    # 2nd user try to delete it again
-    result, status = s3_notification_conf2.del_config()
-    assert_equal(status, 200)
-
-    # 2nd user try to add another notification
-    topic_conf_list = [{'Id': notification_name+"2",
-                        'TopicArn': topic_arn,
-                        'Events': []
-                       }]
-    s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
-    result, status = s3_notification_conf2.set_config()
-    assert_equal(status, 200)
-
-    # cleanup
-    s3_notification_conf1.del_config()
-    s3_notification_conf2.del_config()
-    topic_conf.del_config()
-    # delete the bucket
-    conn1.delete_bucket(bucket_name)
 
 @attr('amqp_test')
 def test_ps_s3_notification_push_amqp_on_master():
@@ -2039,6 +1978,114 @@ def test_ps_s3_lifecycle_on_master():
     conn.delete_bucket(bucket_name)
     http_server.close()
 
+def start_and_abandon_multipart_upload(bucket, key_name, content):
+    try:
+        mp = bucket.initiate_multipart_upload(key_name)
+        part_data = io.StringIO(content)
+        mp.upload_part_from_file(part_data, 1)
+        # mp.complete_upload()
+    except Exception as e:
+        print('Error: ' + str(e))
+
+@attr('http_test')
+def test_ps_s3_lifecycle_abort_mpu_on_master():
+    """ test that when a multipart upload is aborted by lifecycle policy, notification is sent on master """
+    hostname = get_ip()
+    conn = connection()
+    zonegroup = 'default'
+
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    number_of_objects = 1
+    http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
+
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = conn.create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    opaque_data = 'http://1.2.3.4:8888'
+    topic_conf = PSTopicS3(conn, topic_name, zonegroup, endpoint_args=endpoint_args, opaque_data=opaque_data)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': ['s3:ObjectLifecycle:Expiration:*']
+                       }]
+    s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # start and abandon a multpart upload
+    # create objects in the bucket
+    obj_prefix = 'ooo'
+    start_time = time.time()
+    content = 'bar'
+
+    key_name = obj_prefix + str(1)
+    thr = threading.Thread(target = start_and_abandon_multipart_upload, args=(bucket, key_name, content,))
+    thr.start()
+    thr.join()    
+
+    time_diff = time.time() - start_time
+    print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
+    
+    # create lifecycle policy -- assume rgw_lc_debug_interval=10 is in effect
+    client = boto3.client('s3',
+            endpoint_url='http://'+conn.host+':'+str(conn.port),
+            aws_access_key_id=conn.aws_access_key_id,
+            aws_secret_access_key=conn.aws_secret_access_key)
+    response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, 
+            LifecycleConfiguration={'Rules': [
+                {
+                    'ID': 'abort1',
+                    'Filter': {'Prefix': obj_prefix},
+                    'Status': 'Enabled',
+                    'AbortIncompleteMultipartUpload': {'DaysAfterInitiation': 1},
+                }
+            ]
+        }
+    )
+
+    # start lifecycle processing
+    admin(['lc', 'process'])
+    print('wait for 20s (2 days) for the messages...')
+    time.sleep(20)
+
+    # check http receiver does not have messages
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    event_keys = []
+    events = http_server.get_and_reset_events()
+    for event in events:
+        # I hope Boto doesn't gak on the unknown eventName
+        assert_equal(event['Records'][0]['eventName'], 'ObjectLifecycle:Expiration:AbortMultipartUpload')
+        event_keys.append(event['Records'][0]['s3']['object']['key'])
+    for key in keys:
+        key_found = False
+        for event_key in event_keys:
+            if event_key == key:
+                key_found = True
+                break
+        if not key_found:
+            err = 'no lifecycle event found for key: ' + str(key)
+            log.error(events)
+            assert False, err
+
+    # cleanup
+    for key in keys:
+        key.delete()
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    conn.delete_bucket(bucket_name)
+    http_server.close()
 
 def ps_s3_creation_triggers_on_master(external_endpoint_address=None, ca_location=None, verify_ssl='true'):
     """ test object creation s3 notifications in using put/copy/post on master"""

From 540eec2a069443da097007169cf9032af3ed9a7e Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 10 Apr 2023 08:03:44 -0400
Subject: [PATCH 0391/2492] rgwlc: log error on failure to publish_commit

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index cf0b65110507..9be41feb1ec7 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -573,13 +573,17 @@ static int remove_expired_obj(
   ret =  del_op->delete_obj(dpp, null_yield);
   if (ret < 0) {
     ldpp_dout(dpp, 1) <<
-      "ERROR: publishing notification failed, with error: " << ret << dendl;
+      fmt::format("ERROR: {} failed, with error: {}", __func__, ret) << dendl;
   } else {
     // send request to notification manager
-    (void) notify->publish_commit(dpp, obj_state->size,
-				  ceph::real_clock::now(),
-				  obj_state->attrset[RGW_ATTR_ETAG].to_str(),
-				  version_id);
+    ret = notify->publish_commit(dpp, obj_state->size,
+				 ceph::real_clock::now(),
+				 obj_state->attrset[RGW_ATTR_ETAG].to_str(),
+				 version_id);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1) <<
+	"ERROR: notify publish_commit failed, with error: " << ret << dendl;
+    }
   }
 
   return ret;
@@ -859,12 +863,15 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
       ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
 
-        (void) notify->publish_commit(
+        ret = notify->publish_commit(
 	  this, sal_obj->get_obj_size(),
 	  ceph::real_clock::now(),
 	  sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 	  version_id);
-
+	if (ret < 0) {
+	  ldpp_dout(dpp, 1) <<
+	    "ERROR: notify publish_commit failed, with error: " << ret << dendl;
+	}
 	if (perfcounter) {
           perfcounter->inc(l_rgw_lc_abort_mpu, 1);
         }
@@ -1363,10 +1370,14 @@ class LCOpAction_Transition : public LCOpAction {
       return ret;
     } else {
       // send request to notification manager
-      (void) notify->publish_commit(oc.dpp, obj->get_obj_size(),
+      ret =  notify->publish_commit(oc.dpp, obj->get_obj_size(),
 				    ceph::real_clock::now(),
 				    obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 				    version_id);
+      if (ret < 0) {
+	ldpp_dout(dpp, 1) <<
+	  "ERROR: notify publish_commit failed, with error: " << ret << dendl;
+      }
     }
 
     if (delete_object) {

From bf11c8cf69466c0a76f963bb3e2b64ab96ac500f Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 10 Apr 2023 10:58:17 -0400
Subject: [PATCH 0392/2492] rgwlc: remove extra handles to sal bucket and
 object

I missed the addition of those to lc_op_ctx.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 35 ++++++++++-------------------------
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 9be41feb1ec7..3ab419172b03 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -528,17 +528,9 @@ static int remove_expired_obj(
     obj_key.instance = "null";
   }
 
+  auto& obj = oc.obj;
   std::unique_ptr<rgw::sal::User> user;
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-  std::unique_ptr<rgw::sal::Object> obj;
-
   user = driver->get_user(bucket_info.owner);
-  ret = driver->get_bucket(user.get(), bucket_info, &bucket);
-  if (ret < 0) {
-    return ret;
-  }
-
-  obj = bucket->get_object(obj_key);
 
   RGWObjState* obj_state{nullptr};
   ret = obj->get_obj_state(dpp, &obj_state, null_yield, true);
@@ -557,8 +549,8 @@ static int remove_expired_obj(
   del_op->params.marker_version_id = version_id;
 
   // notification supported only for RADOS driver for now
-  notify = driver->get_notification(dpp, obj.get(), nullptr, event_type,
-				   bucket.get(), lc_id,
+  notify = driver->get_notification(dpp, oc.obj.get(), nullptr, event_type,
+				   oc.bucket, lc_id,
 				   const_cast<std::string&>(oc.bucket->get_tenant()),
 				   lc_req_id, null_yield);
 
@@ -869,7 +861,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 	  sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 	  version_id);
 	if (ret < 0) {
-	  ldpp_dout(dpp, 1) <<
+	  ldpp_dout(wk->get_lc(), 1) <<
 	    "ERROR: notify publish_commit failed, with error: " << ret << dendl;
 	}
 	if (perfcounter) {
@@ -1320,28 +1312,21 @@ class LCOpAction_Transition : public LCOpAction {
                      (oc.o.is_current() && oc.bucket->versioned()));
 
     /* notifications */
-    std::unique_ptr<rgw::sal::Bucket> bucket;
-    std::unique_ptr<rgw::sal::Object> obj;
+    auto& bucket = oc.bucket;
     auto& bucket_info = oc.bucket->get_info();
     std::string version_id;
 
-    ret = oc.driver->get_bucket(nullptr, bucket_info, &bucket);
-    if (ret < 0) {
-      return ret;
-    }
-
     std::unique_ptr<rgw::sal::User> user;
+    user = oc.driver->get_user(bucket_info.owner);
     if (! bucket->get_owner()) {
-      auto& bucket_info = bucket->get_info();
-      user = oc.driver->get_user(bucket_info.owner);
       if (user) {
 	bucket->set_owner(user.get());
       }
     }
 
-    obj = bucket->get_object(oc.o.key);
+    auto& obj = oc.obj;
 
-    auto event_type = (oc.bucket->versioned() &&
+    auto event_type = (bucket->versioned() &&
 		       oc.o.is_current() && !oc.o.is_delete_marker()) ?
       rgw::notify::ObjectTransitionCurrent :
       rgw::notify::ObjectTransitionNoncurrent;
@@ -1349,7 +1334,7 @@ class LCOpAction_Transition : public LCOpAction {
     std::unique_ptr<rgw::sal::Notification> notify
       = oc.driver->get_notification(
 	oc.dpp, obj.get(), nullptr, event_type,
-	bucket.get(), lc_id,
+	bucket, lc_id,
 	const_cast<std::string&>(oc.bucket->get_tenant()),
 	lc_req_id, null_yield);
 
@@ -1375,7 +1360,7 @@ class LCOpAction_Transition : public LCOpAction {
 				    obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 				    version_id);
       if (ret < 0) {
-	ldpp_dout(dpp, 1) <<
+	ldpp_dout(oc.dpp, 1) <<
 	  "ERROR: notify publish_commit failed, with error: " << ret << dendl;
       }
     }

From ab2133cf67ab14a6191b0268af1dbaf699102e87 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 10 Apr 2023 11:22:56 -0400
Subject: [PATCH 0393/2492] rgwlc: constify notification event_type

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 3ab419172b03..77410c9f0108 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -824,7 +824,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
   params.ns = RGW_OBJ_NS_MULTIPART;
   params.access_list_filter = &mp_filter;
 
-  auto event_type = rgw::notify::ObjectExpirationAbortMPU;
+  const auto event_type = rgw::notify::ObjectExpirationAbortMPU;
   std::string version_id;
 
   auto pf = [&](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) {
@@ -1326,8 +1326,8 @@ class LCOpAction_Transition : public LCOpAction {
 
     auto& obj = oc.obj;
 
-    auto event_type = (bucket->versioned() &&
-		       oc.o.is_current() && !oc.o.is_delete_marker()) ?
+    const auto event_type = (bucket->versioned() &&
+			     oc.o.is_current() && !oc.o.is_delete_marker()) ?
       rgw::notify::ObjectTransitionCurrent :
       rgw::notify::ObjectTransitionNoncurrent;
 

From c566782db0837af37337078fc62f5403ef0fb91b Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 10 Apr 2023 14:08:39 -0400
Subject: [PATCH 0394/2492] rgwlc: split versioned and non-versioned transition
 notifications

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 77410c9f0108..42ce1b9cffad 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1295,13 +1295,20 @@ class LCOpAction_Transition : public LCOpAction {
 
     /* If bucket is versioned, create delete_marker for current version
      */
-    if (oc.bucket->versioned() && oc.o.is_current() && !oc.o.is_delete_marker()) {
-      ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransitionCurrent);
-      ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") current & not delete_marker" << " versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+    if (! oc.bucket->versioned()) {
+	ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransition);
+	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not versioned flags: " << oc.o.flags << dendl;
     } else {
-      ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransitionNoncurrent);
-      ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not current " << "versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+      /* versioned */
+      if (oc.o.is_current() && !oc.o.is_delete_marker()) {
+	ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransitionCurrent);
+	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") current & not delete_marker" << " versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+      } else {
+	ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransitionNoncurrent);
+	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not current " << "versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+      }
     }
+
     return ret;
   }
 

From ccf29b6cf86fbc64b4dec96105e5b83327632714 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Thu, 5 Oct 2023 11:11:03 -0400
Subject: [PATCH 0395/2492] rgwlc: fix remove expired for non-versioned case

Plus, reformat some adjacent code.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 42ce1b9cffad..e4ba182b6721 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -573,8 +573,8 @@ static int remove_expired_obj(
 				 obj_state->attrset[RGW_ATTR_ETAG].to_str(),
 				 version_id);
     if (ret < 0) {
-      ldpp_dout(dpp, 1) <<
-	"ERROR: notify publish_commit failed, with error: " << ret << dendl;
+      ldpp_dout(dpp, 1) << "ERROR: notify publish_commit failed, with error: "
+												<< ret << dendl;
     }
   }
 
@@ -844,12 +844,12 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 	  lc_req_id, null_yield);
 
       ret = notify->publish_reserve(this, nullptr);
-      if (ret != 0) {
-	ldpp_dout(wk->get_lc(), 0)
-	  << "ERROR: reserving persistent notification for abort_multipart_upload, ret=" << ret
-	  << ", thread:" << wq->thr_name()
-	  << ", meta:" << obj.key
-	  << dendl;
+      if (ret < 0) {
+				ldpp_dout(wk->get_lc(), 0)
+					<< "ERROR: reserving persistent notification for abort_multipart_upload, ret=" << ret
+					<< ", thread:" << wq->thr_name()
+					<< ", meta:" << obj.key
+					<< dendl;
       }
 
       ret = mpu->abort(this, cct, null_yield);
@@ -1296,16 +1296,25 @@ class LCOpAction_Transition : public LCOpAction {
     /* If bucket is versioned, create delete_marker for current version
      */
     if (! oc.bucket->versioned()) {
-	ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransition);
-	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not versioned flags: " << oc.o.flags << dendl;
+      ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransition);
+      ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key
+                            << ") not versioned flags: " << oc.o.flags << dendl;
     } else {
       /* versioned */
       if (oc.o.is_current() && !oc.o.is_delete_marker()) {
-	ret = remove_expired_obj(oc.dpp, oc, false, rgw::notify::ObjectTransitionCurrent);
-	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") current & not delete_marker" << " versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+        ret = remove_expired_obj(oc.dpp, oc, false,
+                                 rgw::notify::ObjectTransitionCurrent);
+        ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key
+                              << ") current & not delete_marker"
+                              << " versioned_epoch:  " << oc.o.versioned_epoch
+                              << "flags: " << oc.o.flags << dendl;
       } else {
-	ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransitionNoncurrent);
-	ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not current " << "versioned_epoch:  " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl;
+        ret = remove_expired_obj(oc.dpp, oc, true,
+                                 rgw::notify::ObjectTransitionNoncurrent);
+        ldpp_dout(oc.dpp, 20)
+            << "delete_tier_obj Object(key:" << oc.o.key << ") not current "
+            << "versioned_epoch:  " << oc.o.versioned_epoch
+            << "flags: " << oc.o.flags << dendl;
       }
     }
 

From 472259f69ce48c458bdbb90d469cfd84556eec56 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Fri, 6 Oct 2023 09:01:06 -0400
Subject: [PATCH 0396/2492] rgwlc: treat persistent notify reservation as
 failure as fatal

(Because that's consistent with what we have been doing for expiration.)

This is possibly a controversial choice, and it might be better to allow
setting a site policy, in future.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 57 ++++++++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 30 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index e4ba182b6721..b6781ce058c3 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -827,7 +827,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
   const auto event_type = rgw::notify::ObjectExpirationAbortMPU;
   std::string version_id;
 
-  auto pf = [&](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) {
+  auto pf = [&](RGWLC::LCWorker *wk, WorkQ *wq, WorkItem &wi) {
     int ret{0};
     auto wt = boost::get<std::tuple<lc_op, rgw_bucket_dir_entry>>(wi);
     auto& [rule, obj] = wt;
@@ -845,44 +845,41 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 
       ret = notify->publish_reserve(this, nullptr);
       if (ret < 0) {
-				ldpp_dout(wk->get_lc(), 0)
-					<< "ERROR: reserving persistent notification for abort_multipart_upload, ret=" << ret
-					<< ", thread:" << wq->thr_name()
-					<< ", meta:" << obj.key
-					<< dendl;
+        ldpp_dout(wk->get_lc(), 0)
+            << "ERROR: reserving persistent notification for "
+               "abort_multipart_upload, ret="
+            << ret << ", thread:" << wq->thr_name()
+            << ", deferring mpu cleanup for meta:" << obj.key << dendl;
+        return ret;
       }
 
       ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
-
         ret = notify->publish_commit(
-	  this, sal_obj->get_obj_size(),
-	  ceph::real_clock::now(),
-	  sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
-	  version_id);
-	if (ret < 0) {
-	  ldpp_dout(wk->get_lc(), 1) <<
-	    "ERROR: notify publish_commit failed, with error: " << ret << dendl;
-	}
-	if (perfcounter) {
+            this, sal_obj->get_obj_size(), ceph::real_clock::now(),
+            sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(), version_id);
+        if (ret < 0) {
+          ldpp_dout(wk->get_lc(), 1)
+              << "ERROR: notify publish_commit failed, with error: " << ret
+              << dendl;
+        }
+        if (perfcounter) {
           perfcounter->inc(l_rgw_lc_abort_mpu, 1);
         }
       } else {
-	if (ret == -ERR_NO_SUCH_UPLOAD) {
-	  ldpp_dout(wk->get_lc(), 5)
-	    << "ERROR: abort_multipart_upload failed, ret=" << ret
-	    << ", thread:" << wq->thr_name()
-	    << ", meta:" << obj.key
-	    << dendl;
-	} else {
-	  ldpp_dout(wk->get_lc(), 0)
-	    << "ERROR: abort_multipart_upload failed, ret=" << ret
-	    << ", thread:" << wq->thr_name()
-	    << ", meta:" << obj.key
-	    << dendl;
-	}
+        if (ret == -ERR_NO_SUCH_UPLOAD) {
+          ldpp_dout(wk->get_lc(), 5) << "ERROR: abort_multipart_upload "
+                                        "failed, ret="
+                                     << ret << ", thread:" << wq->thr_name()
+                                     << ", meta:" << obj.key << dendl;
+        } else {
+          ldpp_dout(wk->get_lc(), 0) << "ERROR: abort_multipart_upload "
+                                        "failed, ret="
+                                     << ret << ", thread:" << wq->thr_name()
+                                     << ", meta:" << obj.key << dendl;
+        }
       } /* abort failed */
-    } /* expired */
+    }   /* expired */
   };
 
   worker->workpool->setf(pf);

From ff446a877b8057da87f72a12f8bcdd177830bb93 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 11 Oct 2023 12:32:21 -0400
Subject: [PATCH 0397/2492] rgwlc: send version_id with notifications

Also fix return value from handle_multipart_expiration bucket-entry
lambda.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index b6781ce058c3..15e70977e726 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -519,7 +519,7 @@ static int remove_expired_obj(
   auto obj_key = o.key;
   auto& meta = o.meta;
   int ret;
-  std::string version_id;
+  auto& version_id = obj_key.instance;
   std::unique_ptr<rgw::sal::Notification> notify;
 
   if (!remove_indeed) {
@@ -546,7 +546,6 @@ static int remove_expired_obj(
   del_op->params.obj_owner.set_name(meta.owner_display_name);
   del_op->params.bucket_owner.set_id(bucket_info.owner);
   del_op->params.unmod_since = meta.mtime;
-  del_op->params.marker_version_id = version_id;
 
   // notification supported only for RADOS driver for now
   notify = driver->get_notification(dpp, oc.obj.get(), nullptr, event_type,
@@ -825,7 +824,6 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
   params.access_list_filter = &mp_filter;
 
   const auto event_type = rgw::notify::ObjectExpirationAbortMPU;
-  std::string version_id;
 
   auto pf = [&](RGWLC::LCWorker *wk, WorkQ *wq, WorkItem &wi) {
     int ret{0};
@@ -842,6 +840,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 	  target, lc_id,
 	  const_cast<std::string&>(target->get_tenant()),
 	  lc_req_id, null_yield);
+			auto& version_id = obj.key.instance;
 
       ret = notify->publish_reserve(this, nullptr);
       if (ret < 0) {
@@ -857,7 +856,8 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
       if (ret == 0) {
         ret = notify->publish_commit(
             this, sal_obj->get_obj_size(), ceph::real_clock::now(),
-            sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(), version_id);
+            sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
+						version_id);
         if (ret < 0) {
           ldpp_dout(wk->get_lc(), 1)
               << "ERROR: notify publish_commit failed, with error: " << ret
@@ -880,6 +880,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
         }
       } /* abort failed */
     }   /* expired */
+		return ret;
   };
 
   worker->workpool->setf(pf);

From fe1ebb87c23916839c26bbf00f193c144004d5dd Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 1 Nov 2023 12:10:28 -0400
Subject: [PATCH 0398/2492] rgwlc: fix notification test imports

Found by Yuval Lifshitz review.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/test/rgw/bucket_notification/test_bn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 87d3ca8eeaa5..32fe262bdc31 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -10,6 +10,7 @@
 import io
 import string
 import boto
+from botocore.exceptions import ClientError
 from http import server as http_server
 from random import randint
 import hashlib

From 59a7f01b5c3cc48aa1e15ee44204d3552ffd593a Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Fri, 3 Nov 2023 16:50:19 +0100
Subject: [PATCH 0399/2492] doc/rados/configuration/bluestore-config-ref: Fix
 lowcase typo

Letters "L P" got lowcased to "l p".
Reverted that.

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
---
 doc/rados/configuration/bluestore-config-ref.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst
index 3707be1aa9bc..cb43c0a021b5 100644
--- a/doc/rados/configuration/bluestore-config-ref.rst
+++ b/doc/rados/configuration/bluestore-config-ref.rst
@@ -358,7 +358,7 @@ OSD and run the following command:
 
        ceph-bluestore-tool \
         --path <data path> \
-        --sharding="m(3) p(3,0-12) o(3,0-13)=block_cache={type=binned_lru} l p" \
+        --sharding="m(3) p(3,0-12) o(3,0-13)=block_cache={type=binned_lru} L P" \
         reshard
 
 .. confval:: bluestore_rocksdb_cf

From fa0f62aa57755c45c713367620dc834530276b25 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 29 Sep 2023 14:39:10 -0400
Subject: [PATCH 0400/2492] mgr/cephadm: add --rm-crush-entry flag to host
 removal

This will tell cephadm to try and remove the
crush bucket for the host at the end of the host
removal process. If this fails, we still consider the
host as having been successfully remove from
cephadm's POV, but the user will get back an error
message telling them we failed to remove the
host from the crush map

Fixes: https://tracker.ceph.com/issues/63031

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py           | 21 +++++++++++++++++++--
 src/pybind/mgr/orchestrator/_interface.py  |  2 +-
 src/pybind/mgr/orchestrator/module.py      |  4 ++--
 src/pybind/mgr/test_orchestrator/module.py |  2 +-
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index ac6e39c469ed..a83256d0bb7d 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -39,7 +39,13 @@
 from cephadm.agent import CephadmAgentHelpers
 
 
-from mgr_module import MgrModule, HandleCommandResult, Option, NotifyType
+from mgr_module import (
+    MgrModule,
+    HandleCommandResult,
+    Option,
+    NotifyType,
+    MonCommandFailed,
+)
 from mgr_util import build_url
 import orchestrator
 from orchestrator.module import to_format, Format
@@ -1630,7 +1636,7 @@ def add_host(self, spec: HostSpec) -> str:
         return self._add_host(spec)
 
     @handle_orch_error
-    def remove_host(self, host: str, force: bool = False, offline: bool = False) -> str:
+    def remove_host(self, host: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> str:
         """
         Remove a host from orchestrator management.
 
@@ -1710,6 +1716,17 @@ def run_cmd(cmd_args: dict) -> None:
             }
             run_cmd(cmd_args)
 
+        if rm_crush_entry:
+            try:
+                self.check_mon_command({
+                    'prefix': 'osd crush remove',
+                    'name': host,
+                })
+            except MonCommandFailed as e:
+                self.log.error(f'Couldn\'t remove host {host} from CRUSH map: {str(e)}')
+                return (f'Cephadm failed removing host {host}\n'
+                        f'Failed to remove host {host} from the CRUSH map: {str(e)}')
+
         self.inventory.rm_host(host)
         self.cache.rm_host(host)
         self.ssh.reset_con(host)
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index 2c7776280028..5bde317d19e6 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -359,7 +359,7 @@ def add_host(self, host_spec: HostSpec) -> OrchResult[str]:
         """
         raise NotImplementedError()
 
-    def remove_host(self, host: str, force: bool, offline: bool) -> OrchResult[str]:
+    def remove_host(self, host: str, force: bool, offline: bool, rm_crush_entry: bool) -> OrchResult[str]:
         """
         Remove a host from the orchestrator inventory.
 
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index de4777e0defa..d6f36e81b718 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -488,9 +488,9 @@ def _add_host(self,
         return self._apply_misc([s], False, Format.plain)
 
     @_cli_write_command('orch host rm')
-    def _remove_host(self, hostname: str, force: bool = False, offline: bool = False) -> HandleCommandResult:
+    def _remove_host(self, hostname: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> HandleCommandResult:
         """Remove a host"""
-        completion = self.remove_host(hostname, force, offline)
+        completion = self.remove_host(hostname, force, offline, rm_crush_entry)
         raise_if_exception(completion)
         return HandleCommandResult(stdout=completion.result_str())
 
diff --git a/src/pybind/mgr/test_orchestrator/module.py b/src/pybind/mgr/test_orchestrator/module.py
index d89c23bf1593..a0721250c7f9 100644
--- a/src/pybind/mgr/test_orchestrator/module.py
+++ b/src/pybind/mgr/test_orchestrator/module.py
@@ -284,7 +284,7 @@ def add_host(self, spec):
         return ''
 
     @handle_orch_error
-    def remove_host(self, host, force: bool, offline: bool):
+    def remove_host(self, host, force: bool, offline: bool, rm_crush_entry: bool):
         assert isinstance(host, str)
         return 'done'
 

From c2ecc9d95fe5a211cb55fc68aba9692e39e85668 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 29 Sep 2023 15:32:53 -0400
Subject: [PATCH 0401/2492] doc/cephadm: document --rm-crush-entry host rm flag

Signed-off-by: Adam King <adking@redhat.com>
---
 doc/cephadm/host-management.rst | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst
index 4b964c5f455a..c1aa88cd60aa 100644
--- a/doc/cephadm/host-management.rst
+++ b/doc/cephadm/host-management.rst
@@ -316,10 +316,33 @@ create a new CRUSH host located in the specified hierarchy.
 
   The ``location`` attribute will be only affect the initial CRUSH location. Subsequent
   changes of the ``location`` property will be ignored. Also, removing a host will not remove
-  any CRUSH buckets.
+  any CRUSH buckets unless the ``--rm-crush-entry`` flag is provided to the ``orch host rm`` command
 
 See also :ref:`crush_map_default_types`.
 
+Removing a host from the CRUSH map
+==================================
+
+The ``ceph orch host rm`` command has support for removing the bucket entry for the host
+in the CRUSH map. This is done by providing the ``--rm-crush-entry`` flag.
+
+.. prompt:: bash [ceph:root@host1/]#
+
+   ceph orch host rm host1 --rm-crush-entry
+
+When this flag is specified, cephadm will attempt to remove the bucket entry
+for the host from the CRUSH map as part of the host removal process. Note that if
+it fails to do so, cephadm will report the failure and the host will remain under
+cephadm control.
+
+.. note:: 
+
+  The removal from the CRUSH map will fail if there are OSDs deployed on the
+  host. If you would like to remove all the host's OSDs as well, please start
+  by using  the ``ceph orch host drain`` command to do so. Once the OSDs
+  are all gone, then you may have cephadm remove the CRUSH entry along with the
+  host using the ``--rm-crush-entry`` flag.
+
 OS Tuning Profiles
 ==================
 

From 13f16e8d7bb029980d6688680390521253970e9a Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 29 Sep 2023 16:09:48 -0400
Subject: [PATCH 0402/2492] qa/cephadm: add teuthology test for host draining

This was a gap in our testing in general, but I'm
adding it here right now specifically to use it
to test the "--rm-crush-entry" flag in a follow
up commit

Signed-off-by: Adam King <adking@redhat.com>
---
 .../workunits/task/test_host_drain.yaml       | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml

diff --git a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
new file mode 100644
index 000000000000..c73d2cdb2788
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
@@ -0,0 +1,46 @@
+roles:
+- - host.a
+  - mon.a
+  - mgr.a
+  - osd.0
+  - osd.1
+- - host.b
+  - mon.b
+  - mgr.b
+  - osd.2
+  - osd.3
+- - host.c
+  - mon.c
+  - osd.4
+  - osd.5
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+    host.a:
+      - |
+        set -ex
+        HOSTNAMES=$(ceph orch host ls --format json | jq -r '.[] | .hostname')
+        for host in $HOSTNAMES; do
+          # find the hostname for "host.c" which will have no mgr
+          HAS_MGRS=$(ceph orch ps --hostname ${host} --format json | jq 'any(.daemon_type == "mgr")')
+          if [ "$HAS_MGRS" == "false" ]; then
+            HOST_C="${host}"
+          fi
+        done
+        # now drain that host
+        ceph orch host drain $HOST_C --zap-osd-devices
+        # wait for drain to complete
+        HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)
+        while [ "$HOST_C_DAEMONS" != "No daemons reported" ]; do
+          sleep 15
+          HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)
+        done
+        # If the drain was successful, we should be able to remove the
+        # host without force with no issues. If there are still daemons
+        # we will get a response telling us to drain the host and a
+        # non-zero return code
+        ceph orch host rm $HOST_C
+        
+        
+        

From 7870538dc1e19760cd96a3d343ae3d3235f71eb2 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 29 Sep 2023 16:52:37 -0400
Subject: [PATCH 0403/2492] qa/cephadm: test --rm-crush-entry host rm flag in
 host drain test

Signed-off-by: Adam King <adking@redhat.com>
---
 .../workunits/task/test_host_drain.yaml       | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
index c73d2cdb2788..83173c9f8b22 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
@@ -36,11 +36,27 @@ tasks:
           sleep 15
           HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)
         done
+        # we want to check the ability to remove the host from
+        # the CRUSH map, so we should first verify the host is in
+        # the CRUSH map.
+        ceph osd getcrushmap -o compiled-crushmap
+        crushtool -d compiled-crushmap -o crushmap.txt
+        CRUSH_MAP=$(cat crushmap.txt)
+        if ! grep -q "$HOST_C" <<< "$CRUSH_MAP"; then
+          printf "Expected to see $HOST_C in CRUSH map. Saw:\n\n$CRUSH_MAP"
+          exit 1
+        fi
         # If the drain was successful, we should be able to remove the
         # host without force with no issues. If there are still daemons
         # we will get a response telling us to drain the host and a
         # non-zero return code
-        ceph orch host rm $HOST_C
-        
-        
-        
+        ceph orch host rm $HOST_C --rm-crush-entry
+        # verify we've successfully removed the host from the CRUSH map
+        sleep 30
+        ceph osd getcrushmap -o compiled-crushmap
+        crushtool -d compiled-crushmap -o crushmap.txt
+        CRUSH_MAP=$(cat crushmap.txt)
+        if grep -q "$HOST_C" <<< "$CRUSH_MAP"; then
+          printf "Saw $HOST_C in CRUSH map after it should have been removed.\n\n$CRUSH_MAP"
+          exit 1
+        fi

From f317cef9e3f6255898a67528ab0e421899d5d3f5 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 3 Nov 2023 13:42:51 -0400
Subject: [PATCH 0404/2492] mgr/dashboard: update fedora link for
 dashboard-cephadm-e2e test

The old link points to a directory that now only contains a
README saying

---

ATTENTION
======================================
The contents of this directory have been moved to our archives available at:

http://archives.fedoraproject.org/pub/archive/fedora/

If you are having troubles finding something there please stop by #fedora-admin on irc.freenode.net

---

so I think we either need to start using a newer fedora or point
to the archive link. This commit is currently doing the latter.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh b/src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh
index 56be5b15964f..d5aa56efc9f5 100755
--- a/src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh
+++ b/src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh
@@ -60,7 +60,7 @@ npm run build ${FRONTEND_BUILD_OPTS} &
 
 cd ${CEPH_DEV_FOLDER}
 : ${VM_IMAGE:='fedora36'}
-: ${VM_IMAGE_URL:='https://download.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2'}
+: ${VM_IMAGE_URL:='https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2'}
 kcli download image -p ceph-dashboard -u ${VM_IMAGE_URL} ${VM_IMAGE}
 kcli delete plan -y ceph || true
 # Compile cephadm locally for the shared_ceph_folder to pick it up

From a375e5d9aa87c0409940388bda382252a81e4936 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 14 Jul 2023 15:41:54 -0400
Subject: [PATCH 0405/2492] cephadm: disable wheels and C compilers when
 building cephadm zipapp

We can not rely on any particular python version (py 3.6+ is supported)
and can not assume any particular architecture. So using wheels
based on the build system is pointless. Installing binary .so files
compiled from C/C++ similarly so. Attempt to block the behaviors
when adding dependencies to the zipapp.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 0680abad21a0..f5a2ce27479f 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -120,6 +120,11 @@ def _install_deps(tempdir):
     """Install dependencies with pip."""
     # TODO we could explicitly pass a python version here
     log.info("Installing dependencies")
+    # best effort to disable compilers, packages in the zipapp
+    # must be pure python.
+    env = os.environ.copy()
+    env['CC'] = '/bin/false'
+    env['CXX'] = '/bin/false'
     # apparently pip doesn't have an API, just a cli.
     subprocess.check_call(
         [
@@ -127,11 +132,14 @@ def _install_deps(tempdir):
             "-m",
             "pip",
             "install",
+            "--no-binary",
+            ":all:",
             "--requirement",
             "requirements.txt",
             "--target",
             tempdir,
-        ]
+        ],
+        env=env,
     )
 
 

From 742dadb285eefe90aab1995a2b3a657390fe221c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 1 Nov 2023 18:14:34 -0400
Subject: [PATCH 0406/2492] cephadm: update the build.py script to work on
 multiple distros

Unfortunately, a single simple call to pip does not work on all the
distributions that ceph is built on. In particular, Ubuntu 20.04 and
Ubuntu 22.04 come with pip versions that can not correctly handle
disabling wheels and installing Jinja2 (it tries to use the markupsafe
dependency before it is installed). This can be worked around by using a
virtual env and updating pip before proceeding. However, this is not
enough because CentOS/RHEL 8 uses python 3.6 and there is no version of
pip that supports 3.6 that we can update to that is new enough to  fix
the issue with disabling wheels. The workaround in this case is to
install each dependency one at a time through multiple calls to pip.

Because of this extra complexity is it simpler to eschew the use of a
requirements.txt file in build.py entirely. Thus the zipapp is built
using build.py only. Requirements files for cephadm are for setting up
the tox environments *only*.

For completeness a new option is added that gives the caller control
over when build.py uses a virtualenv or not. Thus the build.py script
requires at least one of: a working pip that handles disabling wheels;
or, a virtualenv (venv) and the ability to update to a working version
of pip. If the list of distros ceph supports (and the python versions
they use) ever becomes simpler/newer some of this complexity could be
removed from the build.py script.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 141 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 120 insertions(+), 21 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index f5a2ce27479f..87d990e72a35 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -7,12 +7,14 @@
 
 import argparse
 import compileall
+import enum
 import logging
 import os
 import pathlib
 import shutil
 import subprocess
 import tempfile
+import shlex
 import sys
 
 HAS_ZIPAPP = False
@@ -27,6 +29,20 @@
 log = logging.getLogger(__name__)
 
 
+PY36_REQUIREMENTS = [
+    'MarkupSafe >= 2.0.1, <2.2',
+    'Jinja2 >= 3.0.2, <3.2',
+]
+PY_REQUIREMENTS = [
+    'MarkupSafe >= 2.1.3, <2.2',
+    'Jinja2 >= 3.1.2, <3.2',
+]
+# IMPORTANT to be fully compatible with all the distros ceph is built for we
+# need to work around various old versions of python/pip. As such it's easier
+# to repeat our requirements in this script than it is to parse zipapp-reqs.txt.
+# You *must* keep the PY_REQUIREMENTS list in sync with the contents of
+# zipapp-reqs.txt manually.
+
 _VALID_VERS_VARS = [
     "CEPH_GIT_VER",
     "CEPH_GIT_NICE_VER",
@@ -36,6 +52,35 @@
 ]
 
 
+class PipEnv(enum.Enum):
+    never = enum.auto()
+    auto = enum.auto()
+    required = enum.auto()
+
+    @property
+    def enabled(self):
+        return self == self.auto or self == self.required
+
+
+class Config:
+    def __init__(self, cli_args):
+        self.cli_args = cli_args
+        self._maj_min = sys.version_info[0:2]
+        self.install_dependencies = True
+        if self._maj_min == (3, 6):
+            self.pip_split = True
+            self.requirements = PY36_REQUIREMENTS
+        else:
+            self.pip_split = False
+            self.requirements = PY_REQUIREMENTS
+        self.pip_venv = PipEnv[cli_args.pip_use_venv]
+
+
+def _run(command, *args, **kwargs):
+    log.info('Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command))
+    return subprocess.run(command, *args, **kwargs)
+
+
 def _reexec(python):
     """Switch to the selected version of python by exec'ing into the desired
     python path.
@@ -54,14 +99,14 @@ def _did_rexec():
     return bool(os.environ.get("_BUILD_PYTHON_SET", ""))
 
 
-def _build(dest, src, versioning_vars=None):
+def _build(dest, src, config):
     """Build the binary."""
     os.chdir(src)
     tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
     log.debug("working in %s", tempdir)
     try:
-        if os.path.isfile("requirements.txt"):
-            _install_deps(tempdir)
+        if config.install_dependencies:
+            _install_deps(tempdir, config)
         log.info("Copying contents")
         # cephadmlib is cephadm's private library of modules
         shutil.copytree(
@@ -70,6 +115,7 @@ def _build(dest, src, versioning_vars=None):
         # cephadm.py is cephadm's main script for the "binary"
         # this must be renamed to __main__.py for the zipapp
         shutil.copy("cephadm.py", tempdir / "__main__.py")
+        versioning_vars = config.cli_args.version_vars
         if versioning_vars:
             generate_version_file(versioning_vars, tempdir / "_version.py")
         _compile(dest, tempdir)
@@ -116,31 +162,77 @@ def _compile(dest, tempdir):
         log.info("Zipapp created without compression")
 
 
-def _install_deps(tempdir):
+def _install_deps(tempdir, config):
     """Install dependencies with pip."""
     # TODO we could explicitly pass a python version here
     log.info("Installing dependencies")
+
+    executable = sys.executable
+    venv = config.pip_venv
+    has_venv = _has_python_venv(sys.executable) if venv.enabled else False
+    venv = None
+    if venv == PipEnv.required and not has_venv:
+        raise RuntimeError('venv (virtual environment) module not found')
+    if has_venv:
+        log.info('Attempting to create a virtualenv')
+        venv = tempdir / "_venv_"
+        _run([sys.executable, '-m', 'venv', str(venv)])
+        executable = str(venv / "bin" / pathlib.Path(executable).name)
+        # try to upgrade pip in the virtualenv. if it fails ignore the error
+        _run([executable, '-m', 'pip', 'install', '-U', 'pip'])
+    else:
+        log.info('Continuing without a virtualenv...')
+    if not _has_python_pip(executable):
+        raise RuntimeError('pip module not found')
+
     # best effort to disable compilers, packages in the zipapp
     # must be pure python.
     env = os.environ.copy()
     env['CC'] = '/bin/false'
     env['CXX'] = '/bin/false'
-    # apparently pip doesn't have an API, just a cli.
-    subprocess.check_call(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "--no-binary",
-            ":all:",
-            "--requirement",
-            "requirements.txt",
-            "--target",
-            tempdir,
-        ],
-        env=env,
+    if env.get('PYTHONPATH'):
+        env['PYTHONPATH'] = env['PYTHONPATH'] + f':{tempdir}'
+    else:
+        env['PYTHONPATH'] = f'{tempdir}'
+    if config.pip_split:
+        # a list of single item lists; so that pip run once for each
+        # requirement
+        req_batches = [[r] for r in config.requirements]
+    else:
+        # a list containing another list of the requirements, so we only
+        # need to run pip once
+        req_batches = [list(config.requirements)]
+    for batch in req_batches:
+        _run(
+            [
+                executable,
+                "-m",
+                "pip",
+                "install",
+                "--no-binary",
+                ":all:",
+                "--target",
+                tempdir,
+            ] + batch,
+            env=env,
+            check=True,
+        )
+    if venv:
+        shutil.rmtree(venv)
+
+
+def _has_python_venv(executable):
+    res = _run(
+        [executable, '-m', 'venv', '--help'], stdout=subprocess.DEVNULL
     )
+    return res.returncode == 0
+
+
+def _has_python_pip(executable):
+    res = _run(
+        [executable, '-m', 'venv', '--help'], stdout=subprocess.DEVNULL
+    )
+    return res.returncode == 0
 
 
 def generate_version_file(versioning_vars, dest):
@@ -186,6 +278,12 @@ def main():
         action="append",
         help="Set a key=value pair in the generated version info file",
     )
+    parser.add_argument(
+        '--pip-use-venv',
+        choices=[e.name for e in PipEnv],
+        default=PipEnv.auto.name,
+        help='Configure pip to use a virtual environment when bundling dependencies',
+    )
     args = parser.parse_args()
 
     if not _did_rexec() and args.python:
@@ -196,7 +294,8 @@ def main():
             v=sys.version_info
         )
     )
-    log.info("Args: %s", vars(args))
+    for argkey, argval in vars(args).items():
+        log.info("Argument: %s=%r", argkey, argval)
     if not HAS_ZIPAPP:
         # Unconditionally display an error that the version of python
         # lacks zipapp (probably too old).
@@ -214,7 +313,7 @@ def main():
     dest = pathlib.Path(args.dest).absolute()
     log.info("Source Dir: %s", source)
     log.info("Destination Path: %s", dest)
-    _build(dest, source, versioning_vars=args.version_vars)
+    _build(dest, source, Config(args))
 
 
 if __name__ == "__main__":

From f30aad0d9e0f24e6b7c07503d9f9149b25b58f0e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 1 Nov 2023 18:14:34 -0400
Subject: [PATCH 0407/2492] cephadm: add tests for build.py script

Add tests that cover the four main distros that ceph is built on (in
the ceph infra). These tests should not be run by automation as they
are slow and have special requirements like a working podman.
Instead, these are provided to be run by a dev when build.py is updated.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/build/__init__.py           |  0
 src/cephadm/tests/build/test_cephadm_build.py | 96 +++++++++++++++++++
 src/cephadm/tox.ini                           |  9 ++
 3 files changed, 105 insertions(+)
 create mode 100644 src/cephadm/tests/build/__init__.py
 create mode 100644 src/cephadm/tests/build/test_cephadm_build.py

diff --git a/src/cephadm/tests/build/__init__.py b/src/cephadm/tests/build/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/cephadm/tests/build/test_cephadm_build.py b/src/cephadm/tests/build/test_cephadm_build.py
new file mode 100644
index 000000000000..b6fea0e8a9ea
--- /dev/null
+++ b/src/cephadm/tests/build/test_cephadm_build.py
@@ -0,0 +1,96 @@
+# tests for building cephadm into a zipapp using build.py
+#
+# these should not be run automatically as they require the use of podman,
+# which should not be assumed to exist on a typical test node
+
+import os
+import pathlib
+import pytest
+import subprocess
+import sys
+
+
+CONTAINERS = {
+    'centos-8': {
+        'name': 'cephadm-build-test:centos8-py36',
+        'base_image': 'quay.io/centos/centos:stream8',
+        'script': 'dnf install -y python36',
+    },
+    'centos-9': {
+        'name': 'cephadm-build-test:centos9-py3',
+        'base_image': 'quay.io/centos/centos:stream9',
+        'script': 'dnf install -y python3',
+    },
+    'ubuntu-20.04': {
+        'name': 'cephadm-build-test:ubuntu-20-04-py3',
+        'base_image': 'docker.io/library/ubuntu:20.04',
+        'script': 'apt update && apt install -y python3-venv',
+    },
+    'ubuntu-22.04': {
+        'name': 'cephadm-build-test:ubuntu-22-04-py3',
+        'base_image': 'docker.io/library/ubuntu:22.04',
+        'script': 'apt update && apt install -y python3-venv',
+    },
+}
+
+BUILD_PY = 'src/cephadm/build.py'
+
+
+def _print(*args):
+    """Print with a highlight prefix."""
+    print('----->', *args)
+    sys.stdout.flush()
+
+
+def container_cmd(image, cmd, ceph_dir, out_dir):
+    return [
+        'podman',
+        'run',
+        '--rm',
+        f'--volume={ceph_dir}:/ceph:ro',
+        f'--volume={out_dir}:/out',
+        image,
+    ] + list(cmd)
+
+
+def run_container_cmd(image, cmd, ceph_dir, out_dir):
+    full_cmd = container_cmd(image, cmd, ceph_dir, out_dir)
+    _print("CMD", full_cmd)
+    return subprocess.run(full_cmd)
+
+
+def build_container(src_image, dst_image, build_script, workdir):
+    cfile = pathlib.Path(workdir) / 'Dockerfile'
+    with open(cfile, 'w') as fh:
+        fh.write(f'FROM {src_image}\n')
+        fh.write(f'RUN {build_script}\n')
+    cmd = ['podman', 'build', '-t', str(dst_image), '-f', str(cfile)]
+    _print("BUILD CMD", cmd)
+    subprocess.run(cmd, check=True)
+
+
+def build_in(alias, ceph_dir, out_dir, args):
+    ctr = CONTAINERS[alias]
+    build_container(ctr['base_image'], ctr['name'], ctr['script'], out_dir)
+    cmd = ['/ceph/' + BUILD_PY] + list(args or []) + ['/out/cephadm']
+    run_container_cmd(ctr['name'], cmd, ceph_dir, out_dir)
+
+
+@pytest.fixture
+def source_dir():
+    return pathlib.Path(__file__).parents[4].absolute()
+
+
+@pytest.mark.parametrize(
+    'env',
+    [
+        'centos-8',
+        'centos-9',
+        'ubuntu-20.04',
+        'ubuntu-22.04',
+    ],
+)
+def test_cephadm_build(env, source_dir, tmp_path):
+    build_in(env, source_dir, tmp_path, [])
+    assert (tmp_path / 'cephadm').is_file()
+    # TODO: verify contents of zip
diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 266520ff5727..cea84554e129 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -87,3 +87,12 @@ deps =
     black>=23,<24
 commands =
     black -q -l78 -t py36 --skip-string-normalization cephadmlib/
+
+# test_build env is intentionally left out of the envlist. It is here for developers
+# to run locally as it has some unusual requirements: needs podman, etc
+[testenv:test_build]
+skip_install=true
+deps =
+  {[testenv]deps}
+commands =
+  pytest {posargs} tests/build

From 519a5f01b623fbf583f81d62593368e6b9a8949c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 11 Jul 2023 15:45:13 -0400
Subject: [PATCH 0408/2492] cephadm: move unit file related tests to dedicated
 file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py   |  9 --------
 src/cephadm/tests/test_unit_file.py | 33 +++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 9 deletions(-)
 create mode 100644 src/cephadm/tests/test_unit_file.py

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 7e31b26307c3..9d5cbf291e50 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -38,15 +38,6 @@ def get_ceph_conf(
 
 class TestCephAdm(object):
 
-    def test_docker_unit_file(self):
-        ctx = _cephadm.CephadmContext()
-        ctx.container_engine = mock_docker()
-        r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
-        assert 'Requires=docker.service' in r
-        ctx.container_engine = mock_podman()
-        r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
-        assert 'Requires=docker.service' not in r
-
     @mock.patch('cephadm.logger')
     def test_attempt_bind(self, _logger):
         from cephadmlib.net_utils import PortOccupiedError, attempt_bind
diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
new file mode 100644
index 000000000000..cf094c97bc2f
--- /dev/null
+++ b/src/cephadm/tests/test_unit_file.py
@@ -0,0 +1,33 @@
+# Tests for various assorted utility functions found within cephadm
+#
+from unittest import mock
+
+import functools
+import io
+import os
+import sys
+
+import pytest
+
+from tests.fixtures import (
+    import_cephadm,
+    mock_docker,
+    mock_podman,
+    with_cephadm_ctx,
+)
+
+_cephadm = import_cephadm()
+
+
+def test_docker_engine_requires_docker():
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_docker()
+    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert 'Requires=docker.service' in r
+
+
+def test_podman_engine_does_not_req_docker():
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_podman()
+    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert 'Requires=docker.service' not in r

From f14ef59984e556fc0fefee7c5eac2964f9b64c8b Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Fri, 3 Nov 2023 20:32:28 +0100
Subject: [PATCH 0409/2492] doc/rados/configuration/bluestore-config-ref: Fix
 lowcase typo

Letter "O" got lowcased to "o".
Reverted that.

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
---
 doc/rados/configuration/bluestore-config-ref.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst
index cb43c0a021b5..55270b5205f6 100644
--- a/doc/rados/configuration/bluestore-config-ref.rst
+++ b/doc/rados/configuration/bluestore-config-ref.rst
@@ -358,7 +358,7 @@ OSD and run the following command:
 
        ceph-bluestore-tool \
         --path <data path> \
-        --sharding="m(3) p(3,0-12) o(3,0-13)=block_cache={type=binned_lru} L P" \
+        --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" \
         reshard
 
 .. confval:: bluestore_rocksdb_cf

From e53c18fae5ec48916cef1ab805423608d1888f21 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 10 Oct 2023 14:00:27 -0400
Subject: [PATCH 0410/2492] cephadm: remove --cleanup-on-failure flag

As discussed in the orch weekly, instead of having the
two flags, we'll just have the --no-cleanup-on-failure
flag on its own. This commit does not change the behavior
at all. It will still do the cleanup if --no-cleanup-on-failure
is not provided and not do the cleanup if it was. This just
removes the additional flag.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py | 40 ++++++++++++++--------------------------
 1 file changed, 14 insertions(+), 26 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ca068c13757c..e5f33421ee36 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4799,26 +4799,26 @@ def _rollback(ctx: CephadmContext) -> Any:
             raise
         except (KeyboardInterrupt, Exception) as e:
             logger.error(f'{type(e).__name__}: {e}')
-            if ctx.cleanup_on_failure:
+            if ctx.no_cleanup_on_failure:
                 logger.info('\n\n'
                             '\t***************\n'
-                            '\tCephadm hit an issue during cluster installation. Current cluster files will be deleted automatically,\n'
-                            '\tto disable this behaviour you can pass the --no-cleanup-on-failure flag. In case of any previous\n'
-                            '\tbroken installation user must use the following command to completely delete the broken cluster:\n\n'
-                            '\t> cephadm rm-cluster --force --zap-osds --fsid <fsid>\n\n'
+                            '\tCephadm hit an issue during cluster installation. Current cluster files will NOT BE DELETED automatically. To change\n'
+                            '\tthis behaviour do not pass the --no-cleanup-on-failure flag. To remove this broken cluster manually please run:\n\n'
+                            f'\t   > cephadm rm-cluster --force --fsid {ctx.fsid}\n\n'
+                            '\tin case of any previous broken installation, users must use the rm-cluster command to delete the broken cluster:\n\n'
+                            '\t   > cephadm rm-cluster --force --zap-osds --fsid <fsid>\n\n'
                             '\tfor more information please refer to https://docs.ceph.com/en/latest/cephadm/operations/#purging-a-cluster\n'
                             '\t***************\n\n')
-                _rm_cluster(ctx, keep_logs=False, zap_osds=False)
             else:
                 logger.info('\n\n'
                             '\t***************\n'
-                            '\tCephadm hit an issue during cluster installation. Current cluster files will NOT BE DELETED automatically to change\n'
-                            '\tthis behaviour you can pass the --cleanup-on-failure. To remove this broken cluster manually please run:\n\n'
-                            f'\t   > cephadm rm-cluster --force --fsid {ctx.fsid}\n\n'
-                            '\tin case of any previous broken installation user must use the rm-cluster command to delete the broken cluster:\n\n'
-                            '\t   > cephadm rm-cluster --force --zap-osds --fsid <fsid>\n\n'
+                            '\tCephadm hit an issue during cluster installation. Current cluster files will be deleted automatically.\n'
+                            '\tTo disable this behaviour you can pass the --no-cleanup-on-failure flag. In case of any previous\n'
+                            '\tbroken installation, users must use the following command to completely delete the broken cluster:\n\n'
+                            '\t> cephadm rm-cluster --force --zap-osds --fsid <fsid>\n\n'
                             '\tfor more information please refer to https://docs.ceph.com/en/latest/cephadm/operations/#purging-a-cluster\n'
                             '\t***************\n\n')
+                _rm_cluster(ctx, keep_logs=False, zap_osds=False)
             raise
     return cast(FuncT, _rollback)
 
@@ -7379,22 +7379,10 @@ def _get_parser():
         '--allow-overwrite',
         action='store_true',
         help='allow overwrite of existing --output-* config/keyring/ssh files')
-    # following logic to have both '--cleanup-on-failure' and '--no-cleanup-on-failure'
-    # has been included in argparse of python v3.9, however since we have to support
-    # older python versions the following is more generic. Once python v3.9 becomes
-    # the minium supported version we can implement the same by using the new option
-    # argparse.BooleanOptionalAction
-    group = parser_bootstrap.add_mutually_exclusive_group()
-    group.add_argument(
-        '--cleanup-on-failure',
-        action='store_true',
-        default=True,
-        help='Delete cluster files in case of a failed installation')
-    group.add_argument(
+    parser_bootstrap.add_argument(
         '--no-cleanup-on-failure',
-        action='store_const',
-        const=False,
-        dest='cleanup_on_failure',
+        action='store_true',
+        default=False,
         help='Do not delete cluster files in case of a failed installation')
     parser_bootstrap.add_argument(
         '--allow-fqdn-hostname',

From 8b8941c7d18bf6e4f88cada6e7bfe733f2c1ce4c Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Wed, 25 Oct 2023 17:02:12 -0400
Subject: [PATCH 0411/2492] rgw: log errors during "bi list"

Currently errors are displayed on STDERR but do not get into the
appropriate log. This duplicates the STDERR messages to the log.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/rgw_admin.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index cc7f5811c9ef..62fafdfcfb1b 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -7670,7 +7670,8 @@ int main(int argc, const char **argv)
 
     int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
-      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+      ldpp_dout(dpp(), 0) << "ERROR: could not init bucket: " << cpp_strerror(-ret) <<
+	dendl;
       return -ret;
     }
 
@@ -7690,13 +7691,13 @@ int main(int argc, const char **argv)
     int i = (specified_shard_id ? shard_id : 0);
     for (; i < max_shards; i++) {
       ldpp_dout(dpp(), 20) << "INFO: " << __func__ << ": starting shard=" << i << dendl;
+      marker.clear();
 
       RGWRados::BucketShard bs(static_cast<rgw::sal::RadosStore*>(driver)->getRados());
       int ret = bs.init(dpp(), bucket->get_info(), index, i, null_yield);
-      marker.clear();
-
       if (ret < 0) {
-        cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << i << "): " << cpp_strerror(-ret) << std::endl;
+	ldpp_dout(dpp(), 0) << "ERROR: bs.init(bucket=" << bucket << ", shard=" << i <<
+	  "): " << cpp_strerror(-ret) << dendl;
         return -ret;
       }
 
@@ -7705,7 +7706,7 @@ int main(int argc, const char **argv)
 	// if object is specified, we use that as a filter to only retrieve some some entries
         ret = static_cast<rgw::sal::RadosStore*>(driver)->getRados()->bi_list(bs, object, marker, max_entries, &entries, &is_truncated, null_yield);
         if (ret < 0) {
-          cerr << "ERROR: bi_list(): " << cpp_strerror(-ret) << std::endl;
+          ldpp_dout(dpp(), 0) << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl;
           return -ret;
         }
 	ldpp_dout(dpp(), 20) << "INFO: " << __func__ <<

From 95d0385b06f946ef6c964f85aed3cbfea5ba2948 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 11 Jul 2023 15:51:19 -0400
Subject: [PATCH 0412/2492] cephadm: add additional test cases for unit file
 contents

Asserts more about the unit files.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_unit_file.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
index cf094c97bc2f..193baff35a78 100644
--- a/src/cephadm/tests/test_unit_file.py
+++ b/src/cephadm/tests/test_unit_file.py
@@ -16,6 +16,8 @@
     with_cephadm_ctx,
 )
 
+from cephadmlib.constants import CGROUPS_SPLIT_PODMAN_VERSION
+
 _cephadm = import_cephadm()
 
 
@@ -31,3 +33,24 @@ def test_podman_engine_does_not_req_docker():
     ctx.container_engine = mock_podman()
     r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Requires=docker.service' not in r
+
+
+def test_podman_engine_forking_service():
+    # verity that the podman service uses the forking service type
+    # and related parameters
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_podman()
+    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert 'Type=forking' in r
+    assert 'PIDFile=' in r
+    assert 'ExecStartPre' in r
+    assert 'ExecStopPost' in r
+
+
+def test_podman_with_split_cgroups_sets_delegate():
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_podman()
+    ctx.container_engine.version = CGROUPS_SPLIT_PODMAN_VERSION
+    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert 'Type=forking' in r
+    assert 'Delegate=yes' in r

From 8b94a25a0330f50163e9feba4da0ffdf80efe7ca Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 17 Jul 2023 09:24:14 -0400
Subject: [PATCH 0413/2492] cephadm: add more thorough test coverage to unit
 file generation

Add tests that check the generation of the standard systemd unit
for cephadm services. This test ensures that non trivial changes
to the content of these files are noticed.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_unit_file.py | 99 +++++++++++++++++++++++++++--
 1 file changed, 95 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
index 193baff35a78..0ebd84be3e84 100644
--- a/src/cephadm/tests/test_unit_file.py
+++ b/src/cephadm/tests/test_unit_file.py
@@ -21,17 +21,21 @@
 _cephadm = import_cephadm()
 
 
+def _get_unit_file(ctx, fsid):
+    return str(_cephadm.get_unit_file(ctx, fsid))
+
+
 def test_docker_engine_requires_docker():
     ctx = _cephadm.CephadmContext()
     ctx.container_engine = mock_docker()
-    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Requires=docker.service' in r
 
 
 def test_podman_engine_does_not_req_docker():
     ctx = _cephadm.CephadmContext()
     ctx.container_engine = mock_podman()
-    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Requires=docker.service' not in r
 
 
@@ -40,7 +44,7 @@ def test_podman_engine_forking_service():
     # and related parameters
     ctx = _cephadm.CephadmContext()
     ctx.container_engine = mock_podman()
-    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Type=forking' in r
     assert 'PIDFile=' in r
     assert 'ExecStartPre' in r
@@ -51,6 +55,93 @@ def test_podman_with_split_cgroups_sets_delegate():
     ctx = _cephadm.CephadmContext()
     ctx.container_engine = mock_podman()
     ctx.container_engine.version = CGROUPS_SPLIT_PODMAN_VERSION
-    r = _cephadm.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Type=forking' in r
     assert 'Delegate=yes' in r
+
+
+def _ignore_blank_lines(value):
+    return [v for v in value.splitlines() if v]
+
+
+def test_new_docker():
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_docker()
+    ru = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert _ignore_blank_lines(ru) == [
+        '# generated by cephadm',
+        '[Unit]',
+        'Description=Ceph %i for 9b9d7609-f4d5-4aba-94c8-effa764d96c9',
+        '# According to:',
+        '#   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget',
+        '# these can be removed once ceph-mon will dynamically change network',
+        '# configuration.',
+        'After=network-online.target local-fs.target time-sync.target docker.service',
+        'Wants=network-online.target local-fs.target time-sync.target',
+        'Requires=docker.service',
+        'PartOf=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+        'Before=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+        '[Service]',
+        'LimitNOFILE=1048576',
+        'LimitNPROC=1048576',
+        'EnvironmentFile=-/etc/environment',
+        'ExecStart=/bin/bash '
+        '/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.run',
+        "ExecStop=-/bin/bash -c 'bash "
+        "/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.stop'",
+        'ExecStopPost=-/bin/bash '
+        '/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.poststop',
+        'KillMode=none',
+        'Restart=on-failure',
+        'RestartSec=10s',
+        'TimeoutStartSec=200',
+        'TimeoutStopSec=120',
+        'StartLimitInterval=30min',
+        'StartLimitBurst=5',
+        '[Install]',
+        'WantedBy=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+    ]
+
+
+def test_new_podman():
+    ctx = _cephadm.CephadmContext()
+    ctx.container_engine = mock_podman()
+    ctx.container_engine.version = CGROUPS_SPLIT_PODMAN_VERSION
+    ru = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+    assert _ignore_blank_lines(ru) == [
+        '# generated by cephadm',
+        '[Unit]',
+        'Description=Ceph %i for 9b9d7609-f4d5-4aba-94c8-effa764d96c9',
+        '# According to:',
+        '#   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget',
+        '# these can be removed once ceph-mon will dynamically change network',
+        '# configuration.',
+        'After=network-online.target local-fs.target time-sync.target',
+        'Wants=network-online.target local-fs.target time-sync.target',
+        'PartOf=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+        'Before=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+        '[Service]',
+        'LimitNOFILE=1048576',
+        'LimitNPROC=1048576',
+        'EnvironmentFile=-/etc/environment',
+        'ExecStart=/bin/bash '
+        '/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.run',
+        "ExecStop=-/bin/bash -c 'bash "
+        "/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.stop'",
+        'ExecStopPost=-/bin/bash '
+        '/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/%i/unit.poststop',
+        'KillMode=none',
+        'Restart=on-failure',
+        'RestartSec=10s',
+        'TimeoutStartSec=200',
+        'TimeoutStopSec=120',
+        'StartLimitInterval=30min',
+        'StartLimitBurst=5',
+        'ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid',
+        'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid',
+        'Type=forking',
+        'PIDFile=%t/%n-pid',
+        'Delegate=yes',
+        '[Install]',
+        'WantedBy=ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9.target',
+    ]

From e708f2e15973ae9d4745c50f22bd753c42c5ed88 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 14 Jul 2023 15:44:09 -0400
Subject: [PATCH 0414/2492] cephadm: add zipapp-reqs.txt, mirroring cephadm's
 zipapp dependencies

Add Jinja2 and MarkupSafe dependencies to a requirements.txt style file.
This file tracks the dependencies needed to run the cephadm libs
in the unit test framework. The actual dependencies that get added
to the ziapp are managed by build.py but mirrored here.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/zipapp-reqs.txt | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 src/cephadm/zipapp-reqs.txt

diff --git a/src/cephadm/zipapp-reqs.txt b/src/cephadm/zipapp-reqs.txt
new file mode 100644
index 000000000000..cf36f87b2bc0
--- /dev/null
+++ b/src/cephadm/zipapp-reqs.txt
@@ -0,0 +1,16 @@
+# Requirements for the cephadm zipapp (aka the binary).
+#
+# IMPORTANT: The cephadm binary is expected to be portable across python
+# versions and CPU architectures. Dependencies are copied into the zipapp
+# by the build script and must not require compiled C (or C++, Rust, etc)
+# modules. Modules that have an optional C accelerator but can fall back
+# to pure python are OK. When you add a package to this list verify that
+# build.py creates the zipapp with only python files.
+#
+# IMPORTANT: This file is only used for installing the requirements that
+# cephaadm needs for the tox/unit tests. The actual zipapp is build using
+# the build.py script. The PY_REQUIREMENTS value in that script *must*
+# be kept in sync with this list.
+#
+MarkupSafe >= 2.1.3, <2.2
+Jinja2 >= 3.1.2, <3.2

From a789a03035e9e93d740d728bf0b841954985e739 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 16 Jul 2023 16:14:53 -0400
Subject: [PATCH 0415/2492] cephadm: add new requirements file to tox.ini envs

Add `-rzipapp-reqs.txt` to the unit tests and mypy environments in
tox.ini, enabling the use of dependencies outside the stdlib.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index cea84554e129..67ed18c97249 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -33,6 +33,7 @@ addopts =
 [testenv]
 skip_install=true
 deps =
+  -rzipapp-reqs.txt
   pyfakefs == 4.5.6 ; python_version < "3.7"
   pyfakefs >= 5, < 6 ; python_version >= "3.7"
   mock
@@ -43,6 +44,7 @@ commands=pytest {posargs}
 basepython = python3
 deps =
     mypy
+    -rzipapp-reqs.txt
     -c{toxinidir}/../mypy-constrains.txt
 commands = mypy --config-file ../mypy.ini {posargs:cephadm.py cephadmlib}
 

From def7059495f380caf1029b0ea44a1f9e3823c28a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 14 Jul 2023 15:44:50 -0400
Subject: [PATCH 0416/2492] cephadm: add wrapper func for jinja templating

Add a simple wrapper function for templating from a string to a string.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 src/cephadm/cephadmlib/templating.py

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
new file mode 100644
index 000000000000..29e0f825288d
--- /dev/null
+++ b/src/cephadm/cephadmlib/templating.py
@@ -0,0 +1,20 @@
+# templating.py - functions to wrap string/file templating libs
+
+
+from typing import Any
+
+import jinja2
+
+from .context import CephadmContext
+
+
+def template_str(ctx: CephadmContext, template: str, **kwargs: Any) -> str:
+    loader = jinja2.BaseLoader()
+    return (
+        jinja2.Environment(loader=loader)
+        .from_string(template)
+        .render(
+            ctx=ctx,
+            **kwargs,
+        )
+    )

From 4cba7a118c583a822d83b6ad50eb96e24319ab91 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 16 Jul 2023 16:15:49 -0400
Subject: [PATCH 0417/2492] cephadm: update get_unit_file to use jinja template

The somewhat complex string assembly of the main systemd unit file
for cephadm services can benefit from using a standard templating
approach.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 71 ++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 31 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ca068c13757c..84b4a03175ff 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -171,6 +171,7 @@
 from cephadmlib.container_daemon_form import ContainerDaemonForm
 from cephadmlib.sysctl import install_sysctl, migrate_sysctl_dir
 from cephadmlib.firewalld import Firewalld, update_firewalld
+from cephadmlib import templating
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -3386,40 +3387,30 @@ def install_base_units(ctx, fsid):
 """ % (fsid, ' '.join(targets), '|'.join(targets)))
 
 
-def get_unit_file(ctx, fsid):
-    # type: (CephadmContext, str) -> str
-    extra_args = ''
-    if isinstance(ctx.container_engine, Podman):
-        extra_args = ('ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid\n'
-                      'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid\n'
-                      'Type=forking\n'
-                      'PIDFile=%t/%n-pid\n')
-        if ctx.container_engine.supports_split_cgroups:
-            extra_args += 'Delegate=yes\n'
-
-    docker = isinstance(ctx.container_engine, Docker)
-    u = """# generated by cephadm
+_TMPL_DAEMON_UNIT = """# generated by cephadm
 [Unit]
-Description=Ceph %i for {fsid}
+Description=Ceph %i for {{fsid}}
 
 # According to:
 #   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget
 # these can be removed once ceph-mon will dynamically change network
 # configuration.
-After=network-online.target local-fs.target time-sync.target{docker_after}
+After=network-online.target local-fs.target time-sync.target{% if has_docker_engine %} docker.service{% endif %}
 Wants=network-online.target local-fs.target time-sync.target
-{docker_requires}
+{%- if has_docker_engine %}
+Requires=docker.service
+{%- endif %}
 
-PartOf=ceph-{fsid}.target
-Before=ceph-{fsid}.target
+PartOf=ceph-{{fsid}}.target
+Before=ceph-{{fsid}}.target
 
 [Service]
 LimitNOFILE=1048576
 LimitNPROC=1048576
 EnvironmentFile=-/etc/environment
-ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run
-ExecStop=-/bin/bash -c 'bash {data_dir}/{fsid}/%i/unit.stop'
-ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop
+ExecStart=/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.run
+ExecStop=-/bin/bash -c 'bash {{ctx.data_dir}}/{{fsid}}/%i/unit.stop'
+ExecStopPost=-/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.poststop
 KillMode=none
 Restart=on-failure
 RestartSec=10s
@@ -3427,17 +3418,35 @@ def get_unit_file(ctx, fsid):
 TimeoutStopSec=120
 StartLimitInterval=30min
 StartLimitBurst=5
-{extra_args}
+{%- if has_podman_engine %}
+ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
+ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid
+Type=forking
+PIDFile=%t/%n-pid
+{%- if has_podman_split_version %}
+Delegate=yes
+{%- endif %}
+{%- endif %}
+
 [Install]
-WantedBy=ceph-{fsid}.target
-""".format(fsid=fsid,
-           data_dir=ctx.data_dir,
-           extra_args=extra_args,
-           # if docker, we depend on docker.service
-           docker_after=' docker.service' if docker else '',
-           docker_requires='Requires=docker.service\n' if docker else '')
-
-    return u
+WantedBy=ceph-{{fsid}}.target
+"""
+
+
+def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
+    has_docker_engine = isinstance(ctx.container_engine, Docker)
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    has_podman_split_version = (
+        has_podman_engine and ctx.container_engine.supports_split_cgroups
+    )
+    return templating.template_str(
+        ctx,
+        _TMPL_DAEMON_UNIT,
+        fsid=fsid,
+        has_docker_engine=has_docker_engine,
+        has_podman_engine=has_podman_engine,
+        has_podman_split_version=has_podman_split_version,
+    )
 
 ##################################
 

From 51f2d64ee268a8b656a1f20346b604ca2ea1050f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 17 Jul 2023 09:28:35 -0400
Subject: [PATCH 0418/2492] cephadm: update agent unit file func to use jinja
 template

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 84b4a03175ff..8f189becaa12 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3619,25 +3619,23 @@ def unit_run(self) -> str:
         return ('set -e\n' + f'{py3} {binary_path} agent --fsid {self.fsid} --daemon-id {self.daemon_id} &\n')
 
     def unit_file(self) -> str:
-        return """#generated by cephadm
+        ts = """#generated by cephadm
 [Unit]
-Description=cephadm agent for cluster {fsid}
+Description=cephadm agent for cluster {{agent.fsid}}
 
-PartOf=ceph-{fsid}.target
-Before=ceph-{fsid}.target
+PartOf=ceph-{{agent.fsid}}.target
+Before=ceph-{{agent.fsid}}.target
 
 [Service]
 Type=forking
-ExecStart=/bin/bash {data_dir}/unit.run
+ExecStart=/bin/bash {{agent.daemon_dir}}/unit.run
 Restart=on-failure
 RestartSec=10s
 
 [Install]
-WantedBy=ceph-{fsid}.target
-""".format(
-            fsid=self.fsid,
-            data_dir=self.daemon_dir
-        )
+WantedBy=ceph-{{agent.fsid}}.target
+"""
+        return templating.template_str(self.ctx, ts, agent=self)
 
     def shutdown(self) -> None:
         self.stop = True

From 167ba5b22494b5365f22a8365bea8f311107f4a5 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 5 Oct 2023 15:58:22 -0400
Subject: [PATCH 0419/2492] cephadm: workaround jinja2 package loader issue

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/fixtures.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index f84a2d7ba205..86a8c6119ea8 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -78,6 +78,13 @@ def cephadm_fs(
     """
     from cephadmlib import constants
 
+    # the following is a workaround for the fakefs interfering with jinja2's
+    # package loader when run in the pytest suite when this fixture is used.
+    # it effectively maps what is `src/cephadm` as a real fs into the fake fs.`
+    # See: https://pytest-pyfakefs.readthedocs.io/en/stable/usage.html#access-to-files-in-the-real-file-system
+    srcdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    fs.add_real_directory(srcdir)
+
     uid = os.getuid()
     gid = os.getgid()
 

From eca2c763204f9800b602cfd6d2d9409d9edf1127 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 5 Oct 2023 16:02:06 -0400
Subject: [PATCH 0420/2492] cephadm: expand the templating module

Expand the templating module so that templates can be source from the
python package. Add (more) convenient to use methods.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 63 +++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 11 deletions(-)

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 29e0f825288d..ceb2f6fbddf3 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -1,20 +1,61 @@
 # templating.py - functions to wrap string/file templating libs
 
 
-from typing import Any
+from typing import Any, Optional, IO
 
 import jinja2
 
 from .context import CephadmContext
 
+_PKG = __name__.rsplit('.', 1)[0]
+_DIR = 'templates'
 
-def template_str(ctx: CephadmContext, template: str, **kwargs: Any) -> str:
-    loader = jinja2.BaseLoader()
-    return (
-        jinja2.Environment(loader=loader)
-        .from_string(template)
-        .render(
-            ctx=ctx,
-            **kwargs,
-        )
-    )
+
+class Templater:
+    """Cephadm's generic templater class. Based on jinja2."""
+
+    # defaults that can be overridden for testing purposes
+    # and are lazily acquired
+    _jinja2_loader: Optional[jinja2.BaseLoader] = None
+    _jinja2_env: Optional[jinja2.Environment] = None
+    _pkg = _PKG
+    _dir = _DIR
+
+    @property
+    def _env(self) -> jinja2.Environment:
+        if self._jinja2_env is None:
+            self._jinja2_env = jinja2.Environment(loader=self._loader)
+        return self._jinja2_env
+
+    @property
+    def _loader(self) -> jinja2.BaseLoader:
+        if self._jinja2_loader is None:
+            self._jinja2_loader = jinja2.PackageLoader(self._pkg, self._dir)
+        return self._jinja2_loader
+
+    def render_str(
+        self, ctx: CephadmContext, template: str, **kwargs: Any
+    ) -> str:
+        return self._env.from_string(template).render(ctx=ctx, **kwargs)
+
+    def render(self, ctx: CephadmContext, name: str, **kwargs: Any) -> str:
+        return self._env.get_template(name).render(ctx=ctx, **kwargs)
+
+    def render_to_file(
+        self, fp: IO, ctx: CephadmContext, name: str, **kwargs: Any
+    ) -> None:
+        self._env.get_template(name).stream(ctx=ctx, **kwargs).dump(fp)
+
+
+# create a defaultTemplater instace from the Templater class that will
+# be used to provide a simple set of methods
+defaultTemplater = Templater()
+
+# (temporary alias) template_str is currently used by the cephadm code
+template_str = defaultTemplater.render_str
+
+# alias methods as module level functions for convenience. most callers do
+# not need to care that these are implemented via a class
+render_str = defaultTemplater.render_str
+render = defaultTemplater.render
+render_to_file = defaultTemplater.render_to_file

From 45fca896bf667f2f2387d5ff5cab9d4c933bb0c7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 5 Oct 2023 16:41:12 -0400
Subject: [PATCH 0421/2492] cephadm: copy template strings into template files

Copy the current template strings into files under the `templates` dir.
Add a enum for holding the names of known template files.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadmlib/templates/agent.service.j2     | 15 +++++++
 .../cephadmlib/templates/ceph.service.j2      | 43 +++++++++++++++++++
 src/cephadm/cephadmlib/templating.py          | 18 +++++++-
 3 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/templates/agent.service.j2
 create mode 100644 src/cephadm/cephadmlib/templates/ceph.service.j2

diff --git a/src/cephadm/cephadmlib/templates/agent.service.j2 b/src/cephadm/cephadmlib/templates/agent.service.j2
new file mode 100644
index 000000000000..4a494e10d624
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/agent.service.j2
@@ -0,0 +1,15 @@
+# generated by cephadm
+[Unit]
+Description=cephadm agent for cluster {{agent.fsid}}
+
+PartOf=ceph-{{agent.fsid}}.target
+Before=ceph-{{agent.fsid}}.target
+
+[Service]
+Type=forking
+ExecStart=/bin/bash {{agent.daemon_dir}}/unit.run
+Restart=on-failure
+RestartSec=10s
+
+[Install]
+WantedBy=ceph-{{agent.fsid}}.target
diff --git a/src/cephadm/cephadmlib/templates/ceph.service.j2 b/src/cephadm/cephadmlib/templates/ceph.service.j2
new file mode 100644
index 000000000000..1c043dcdd6ad
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/ceph.service.j2
@@ -0,0 +1,43 @@
+# generated by cephadm
+[Unit]
+Description=Ceph %i for {{fsid}}
+
+# According to:
+#   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget
+# these can be removed once ceph-mon will dynamically change network
+# configuration.
+After=network-online.target local-fs.target time-sync.target{% if has_docker_engine %} docker.service{% endif %}
+Wants=network-online.target local-fs.target time-sync.target
+{%- if has_docker_engine %}
+Requires=docker.service
+{%- endif %}
+
+PartOf=ceph-{{fsid}}.target
+Before=ceph-{{fsid}}.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+EnvironmentFile=-/etc/environment
+ExecStart=/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.run
+ExecStop=-/bin/bash -c 'bash {{ctx.data_dir}}/{{fsid}}/%i/unit.stop'
+ExecStopPost=-/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.poststop
+KillMode=none
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=200
+TimeoutStopSec=120
+StartLimitInterval=30min
+StartLimitBurst=5
+{%- if has_podman_engine %}
+ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
+ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid
+Type=forking
+PIDFile=%t/%n-pid
+{%- if has_podman_split_version %}
+Delegate=yes
+{%- endif %}
+{%- endif %}
+
+[Install]
+WantedBy=ceph-{{fsid}}.target
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index ceb2f6fbddf3..b48b5deeaab6 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -1,5 +1,6 @@
 # templating.py - functions to wrap string/file templating libs
 
+import enum
 
 from typing import Any, Optional, IO
 
@@ -11,6 +12,19 @@
 _DIR = 'templates'
 
 
+class Templates(str, enum.Enum):
+    """Known template files."""
+
+    ceph_service = 'ceph.service.j2'
+    agent_service = 'agent.service.j2'
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return repr(self.value)
+
+
 class Templater:
     """Cephadm's generic templater class. Based on jinja2."""
 
@@ -39,12 +53,12 @@ def render_str(
         return self._env.from_string(template).render(ctx=ctx, **kwargs)
 
     def render(self, ctx: CephadmContext, name: str, **kwargs: Any) -> str:
-        return self._env.get_template(name).render(ctx=ctx, **kwargs)
+        return self._env.get_template(str(name)).render(ctx=ctx, **kwargs)
 
     def render_to_file(
         self, fp: IO, ctx: CephadmContext, name: str, **kwargs: Any
     ) -> None:
-        self._env.get_template(name).stream(ctx=ctx, **kwargs).dump(fp)
+        self._env.get_template(str(name)).stream(ctx=ctx, **kwargs).dump(fp)
 
 
 # create a defaultTemplater instace from the Templater class that will

From ae8a2183c11ad077e65334d8f05c445f0fb36921 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 1 Nov 2023 17:58:19 -0400
Subject: [PATCH 0422/2492] cephadm: switch to using template files

Switch off of the embedded template strings to using the recently
added template files.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py               | 70 ++--------------------------
 src/cephadm/cephadmlib/templating.py |  3 --
 2 files changed, 5 insertions(+), 68 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8f189becaa12..8b44e1c8f543 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3387,61 +3387,15 @@ def install_base_units(ctx, fsid):
 """ % (fsid, ' '.join(targets), '|'.join(targets)))
 
 
-_TMPL_DAEMON_UNIT = """# generated by cephadm
-[Unit]
-Description=Ceph %i for {{fsid}}
-
-# According to:
-#   http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget
-# these can be removed once ceph-mon will dynamically change network
-# configuration.
-After=network-online.target local-fs.target time-sync.target{% if has_docker_engine %} docker.service{% endif %}
-Wants=network-online.target local-fs.target time-sync.target
-{%- if has_docker_engine %}
-Requires=docker.service
-{%- endif %}
-
-PartOf=ceph-{{fsid}}.target
-Before=ceph-{{fsid}}.target
-
-[Service]
-LimitNOFILE=1048576
-LimitNPROC=1048576
-EnvironmentFile=-/etc/environment
-ExecStart=/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.run
-ExecStop=-/bin/bash -c 'bash {{ctx.data_dir}}/{{fsid}}/%i/unit.stop'
-ExecStopPost=-/bin/bash {{ctx.data_dir}}/{{fsid}}/%i/unit.poststop
-KillMode=none
-Restart=on-failure
-RestartSec=10s
-TimeoutStartSec=200
-TimeoutStopSec=120
-StartLimitInterval=30min
-StartLimitBurst=5
-{%- if has_podman_engine %}
-ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
-ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid
-Type=forking
-PIDFile=%t/%n-pid
-{%- if has_podman_split_version %}
-Delegate=yes
-{%- endif %}
-{%- endif %}
-
-[Install]
-WantedBy=ceph-{{fsid}}.target
-"""
-
-
 def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
     has_docker_engine = isinstance(ctx.container_engine, Docker)
     has_podman_engine = isinstance(ctx.container_engine, Podman)
     has_podman_split_version = (
         has_podman_engine and ctx.container_engine.supports_split_cgroups
     )
-    return templating.template_str(
+    return templating.render(
         ctx,
-        _TMPL_DAEMON_UNIT,
+        templating.Templates.ceph_service,
         fsid=fsid,
         has_docker_engine=has_docker_engine,
         has_podman_engine=has_podman_engine,
@@ -3619,23 +3573,9 @@ def unit_run(self) -> str:
         return ('set -e\n' + f'{py3} {binary_path} agent --fsid {self.fsid} --daemon-id {self.daemon_id} &\n')
 
     def unit_file(self) -> str:
-        ts = """#generated by cephadm
-[Unit]
-Description=cephadm agent for cluster {{agent.fsid}}
-
-PartOf=ceph-{{agent.fsid}}.target
-Before=ceph-{{agent.fsid}}.target
-
-[Service]
-Type=forking
-ExecStart=/bin/bash {{agent.daemon_dir}}/unit.run
-Restart=on-failure
-RestartSec=10s
-
-[Install]
-WantedBy=ceph-{{agent.fsid}}.target
-"""
-        return templating.template_str(self.ctx, ts, agent=self)
+        return templating.render(
+            self.ctx, templating.Templates.agent_service, agent=self
+        )
 
     def shutdown(self) -> None:
         self.stop = True
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index b48b5deeaab6..3b7c6f9657e0 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -65,9 +65,6 @@ def render_to_file(
 # be used to provide a simple set of methods
 defaultTemplater = Templater()
 
-# (temporary alias) template_str is currently used by the cephadm code
-template_str = defaultTemplater.render_str
-
 # alias methods as module level functions for convenience. most callers do
 # not need to care that these are implemented via a class
 render_str = defaultTemplater.render_str

From a9bf0c13e3dc866bd1cb724a10e23ae215e15437 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sat, 4 Nov 2023 11:06:25 -0400
Subject: [PATCH 0423/2492] doc/radosgw: Improve dynamicresharding.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/radosgw/dynamicresharding.rst | 85 ++++++++++++++++---------------
 1 file changed, 43 insertions(+), 42 deletions(-)

diff --git a/doc/radosgw/dynamicresharding.rst b/doc/radosgw/dynamicresharding.rst
index b58a94980c40..b8bd68d9e0c1 100644
--- a/doc/radosgw/dynamicresharding.rst
+++ b/doc/radosgw/dynamicresharding.rst
@@ -6,38 +6,39 @@ RGW Dynamic Bucket Index Resharding
 
 .. versionadded:: Luminous
 
-A large bucket index can lead to performance problems. In order
-to address this problem we introduced bucket index sharding.
+A large bucket index can lead to performance problems, which can
+be addressed by sharding bucket indexes.
 Until Luminous, changing the number of bucket shards (resharding)
-needed to be done offline. Starting with Luminous we support
-online bucket resharding.
+needed to be done offline, with RGW services disabled.
+Since the Luminous release Ceph has supported online bucket resharding.
 
 Each bucket index shard can handle its entries efficiently up until
-reaching a certain threshold number of entries. If this threshold is
+reaching a certain threshold. If this threshold is
 exceeded the system can suffer from performance issues. The dynamic
 resharding feature detects this situation and automatically increases
-the number of shards used by the bucket index, resulting in a
-reduction of the number of entries in each bucket index shard. This
-process is transparent to the user. Write I/Os to the target bucket
-are blocked and read I/Os are not during resharding process.
+the number of shards used by a bucket's index, resulting in a
+reduction of the number of entries in each shard. This
+process is transparent to the user. Writes to the target bucket
+are blocked (but reads are not) briefly during resharding process.
 
 By default dynamic bucket index resharding can only increase the
 number of bucket index shards to 1999, although this upper-bound is a
 configuration parameter (see Configuration below). When
-possible, the process chooses a prime number of bucket index shards to
-spread the number of bucket index entries across the bucket index
+possible, the process chooses a prime number of shards in order to
+spread the number of entries across the bucket index
 shards more evenly.
 
-The detection process runs in a background process that periodically
-scans all the buckets. A bucket that requires resharding is added to
-the resharding queue and will be scheduled to be resharded later. The
-reshard thread runs in the background and execute the scheduled
-resharding tasks, one at a time.
+Detection of resharding opportunities runs as a background process
+that periodically
+scans all buckets. A bucket that requires resharding is added to
+a queue. A thread runs in the background and processes the queueued
+resharding tasks, one at a time and in order.
 
 Multisite
 =========
 
-Prior to the Reef release, RGW does not support dynamic resharding in a
+With Ceph releases Prior to Reef, the Ceph Object Gateway (RGW) does not support
+dynamic resharding in a
 multisite environment. For information on dynamic resharding, see
 :ref:`Resharding <feature_resharding>` in the RGW multisite documentation.
 
@@ -50,11 +51,11 @@ Enable/Disable dynamic bucket index resharding:
 
 Configuration options that control the resharding process:
 
-- ``rgw_max_objs_per_shard``: maximum number of objects per bucket index shard before resharding is triggered, default: 100000 objects
+- ``rgw_max_objs_per_shard``: maximum number of objects per bucket index shard before resharding is triggered, default: 100000
 
-- ``rgw_max_dynamic_shards``: maximum number of shards that dynamic bucket index resharding can increase to, default: 1999
+- ``rgw_max_dynamic_shards``: maximum number of bucket index shards that dynamic resharding can increase to, default: 1999
 
-- ``rgw_reshard_bucket_lock_duration``: duration, in seconds, of lock on bucket obj during resharding, default: 360 seconds (i.e., 6 minutes)
+- ``rgw_reshard_bucket_lock_duration``: duration, in seconds, that writes to the bucket are locked during resharding, default: 360 (i.e., 6 minutes)
 
 - ``rgw_reshard_thread_interval``: maximum time, in seconds, between rounds of resharding queue processing, default: 600 seconds (i.e., 10 minutes)
 
@@ -91,9 +92,9 @@ Bucket resharding status
 
    # radosgw-admin reshard status --bucket <bucket_name>
 
-The output is a json array of 3 objects (reshard_status, new_bucket_instance_id, num_shards) per shard.
+The output is a JSON array of 3 objects (reshard_status, new_bucket_instance_id, num_shards) per shard.
 
-For example, the output at different Dynamic Resharding stages is shown below:
+For example, the output at each dynamic resharding stage is shown below:
 
 ``1. Before resharding occurred:``
 ::
@@ -142,7 +143,7 @@ For example, the output at different Dynamic Resharding stages is shown below:
 Cancel pending bucket resharding
 --------------------------------
 
-Note: Ongoing bucket resharding operations cannot be cancelled. ::
+Note: Bucket resharding operations cannot be cancelled while executing. ::
 
    # radosgw-admin reshard cancel --bucket <bucket_name>
 
@@ -153,25 +154,24 @@ Manual immediate bucket resharding
 
    # radosgw-admin bucket reshard --bucket <bucket_name> --num-shards <new number of shards>
 
-When choosing a number of shards, the administrator should keep a
-number of items in mind. Ideally the administrator is aiming for no
-more than 100000 entries per shard, now and through some future point
-in time.
+When choosing a number of shards, the administrator must anticipate each
+bucket's peak number of objects. Ideally one should aim for no
+more than 100000 entries per shard at any given time.
 
-Additionally, bucket index shards that are prime numbers tend to work
-better in evenly distributing bucket index entries across the
-shards. For example, 7001 bucket index shards is better than 7000
+Additionally, bucket index shards that are prime numbers are more effective
+in evenly distributing bucket index entries.
+For example, 7001 bucket index shards is better than 7000
 since the former is prime. A variety of web sites have lists of prime
-numbers; search for "list of prime numbers" withy your favorite web
+numbers; search for "list of prime numbers" with your favorite
 search engine to locate some web sites.
 
 Troubleshooting
 ===============
 
 Clusters prior to Luminous 12.2.11 and Mimic 13.2.5 left behind stale bucket
-instance entries, which were not automatically cleaned up. The issue also affected
-LifeCycle policies, which were not applied to resharded buckets anymore. Both of
-these issues can be worked around using a couple of radosgw-admin commands.
+instance entries, which were not automatically cleaned up. This issue also affected
+LifeCycle policies, which were no longer applied to resharded buckets. Both of
+these issues could be worked around by running ``radosgw-admin`` commands.
 
 Stale instance management
 -------------------------
@@ -183,7 +183,7 @@ List the stale instances in a cluster that are ready to be cleaned up.
    # radosgw-admin reshard stale-instances list
 
 Clean up the stale instances in a cluster. Note: cleanup of these
-instances should only be done on a single site cluster.
+instances should only be done on a single-site cluster.
 
 ::
 
@@ -193,11 +193,12 @@ instances should only be done on a single site cluster.
 Lifecycle fixes
 ---------------
 
-For clusters that had resharded instances, it is highly likely that the old
+For clusters with resharded instances, it is highly likely that the old
 lifecycle processes would have flagged and deleted lifecycle processing as the
-bucket instance changed during a reshard. While this is fixed for newer clusters
-(from Mimic 13.2.6 and Luminous 12.2.12), older buckets that had lifecycle policies and
-that have undergone resharding will have to be manually fixed.
+bucket instance changed during a reshard. While this is fixed for buckets
+deployed on newer Ceph releases (from Mimic 13.2.6 and Luminous 12.2.12),
+older buckets that had lifecycle policies and that have undergone
+resharding must be fixed manually.
 
 The command to do so is:
 
@@ -206,8 +207,8 @@ The command to do so is:
    # radosgw-admin lc reshard fix --bucket {bucketname}
 
 
-As a convenience wrapper, if the ``--bucket`` argument is dropped then this
-command will try and fix lifecycle policies for all the buckets in the cluster.
+If the ``--bucket`` argument is not provided, this
+command will try to fix lifecycle policies for all the buckets in the cluster.
 
 Object Expirer fixes
 --------------------
@@ -217,7 +218,7 @@ been dropped from the log pool and never deleted after the bucket was
 resharded. This would happen if their expiration time was before the
 cluster was upgraded, but if their expiration was after the upgrade
 the objects would be correctly handled. To manage these expire-stale
-objects, radosgw-admin provides two subcommands.
+objects, ``radosgw-admin`` provides two subcommands.
 
 Listing:
 

From e0974395e0030dcfdf31c27e92255ebbb8d43d00 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:33:57 -0400
Subject: [PATCH 0424/2492] cephadm: fix mistaken call to
 get_container_mounts_for_type

When this function was refactored two variants were created:
get_container_mounts_for_type and _get_container_mounts_for_type.  The
latter doesn't append podman specific mounts so that it can be called
from other functions that do append podman specific mounts.
Unfortunately, I didn't call the correct function from
get_container_mounts and therefore it was redundantly doing the podman
mounts update. This corrects that error.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 47de7e5627be..4ea82676573a 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2561,7 +2561,7 @@ def get_container_mounts(
     """
     # unpack fsid and daemon_type from ident because they're used very frequently
     fsid, daemon_type = ident.fsid, ident.daemon_type
-    mounts = get_container_mounts_for_type(ctx, fsid, daemon_type)
+    mounts = _get_container_mounts_for_type(ctx, fsid, daemon_type)
 
     assert ident.fsid
     assert ident.daemon_id

From 4516be40fe5947f62fe0712501451a7583ae8d3b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 10:43:04 -0400
Subject: [PATCH 0425/2492] cephadm: enhance test case with env vars

Add some env vars to the custom container test to help assure
future changes don't overlook the ability to set them.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_custom_container.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/tests/test_custom_container.py b/src/cephadm/tests/test_custom_container.py
index cff217a84044..c0f2a7966c44 100644
--- a/src/cephadm/tests/test_custom_container.py
+++ b/src/cephadm/tests/test_custom_container.py
@@ -115,6 +115,9 @@ def test_deploy_custom_container(cephadm_fs):
             '--servers',
             '192.168.8.42,192.168.8.43,192.168.12.11',
         ]
+        ctx.config_blobs = {
+            'envs': ['FOO=1', 'BAR=77'],
+        }
 
         _cephadm._common_deploy(ctx)
 
@@ -132,6 +135,8 @@ def test_deploy_custom_container(cephadm_fs):
             ' --cgroups=split --no-hosts'
             ' -e CONTAINER_IMAGE=quay.io/foobar/quux:latest'
             ' -e NODE_NAME=host1'
+            ' -e FOO=1'
+            ' -e BAR=77'
             ' quay.io/foobar/quux:latest'
             ' --label frobnicationist --servers 192.168.8.42,192.168.8.43,192.168.12.11'
         )

From 18832a3e94f55d0abb3c22290a7dc4dce9156944 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 14:12:40 -0400
Subject: [PATCH 0426/2492] cephadm: split get_deployment_container function

Split get_deployment_container into to_depoyment_container and
a call to get_container. A future change will remove
get_deployment_container entirely.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 4ea82676573a..23420c2b473d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -5106,25 +5106,37 @@ def get_deployment_container(
     # wrapper for get_container specifically for containers made during the `cephadm deploy`
     # command. Adds some extra things such as extra container args and custom config files
     c = get_container(ctx, ident, privileged, ptrace, container_args)
+    return to_deployment_container(ctx, c)
+
+
+def to_deployment_container(
+    ctx: CephadmContext, ctr: CephContainer
+) -> CephContainer:
+    """Given a standard ceph container instance return a CephContainer
+    prepared for a deployment as a daemon, having the extra args and
+    custom configurations added.
+    NOTE: The `ctr` object is mutated before being returned.
+    """
     if 'extra_container_args' in ctx and ctx.extra_container_args:
-        c.container_args.extend(ctx.extra_container_args)
+        ctr.container_args.extend(ctx.extra_container_args)
     if 'extra_entrypoint_args' in ctx and ctx.extra_entrypoint_args:
-        c.args.extend(ctx.extra_entrypoint_args)
+        ctr.args.extend(ctx.extra_entrypoint_args)
     ccfiles = fetch_custom_config_files(ctx)
     if ccfiles:
         mandatory_keys = ['mount_path', 'content']
         for conf in ccfiles:
             if all(k in conf for k in mandatory_keys):
                 mount_path = conf['mount_path']
+                assert ctr.identity
                 file_path = os.path.join(
                     ctx.data_dir,
-                    ident.fsid,
+                    ctr.identity.fsid,
                     'custom_config_files',
-                    ident.daemon_name,
+                    ctr.identity.daemon_name,
                     os.path.basename(mount_path)
                 )
-                c.volume_mounts[file_path] = mount_path
-    return c
+                ctr.volume_mounts[file_path] = mount_path
+    return ctr
 
 
 def get_deployment_type(

From 7494c56e4bb4d7114bf24ea2098259595f678826 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 14:22:49 -0400
Subject: [PATCH 0427/2492] cephadm: remove get_deployment_container

Remove get_deployment_container replacing all calls to it with calls to
to_deployment_container. Now, callers can inject modifications to the
container object between calls or even have to_deployment_container
update container objects that were constructed in some other way.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 52 +++++++++++++++----------------
 src/cephadm/tests/test_cephadm.py | 14 +++++----
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 23420c2b473d..bd0a1c97f4b3 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -243,7 +243,8 @@ def container(self, ctx: CephadmContext) -> CephContainer:
         uid, gid = self.uid_gid(ctx)
         make_var_run(ctx, ctx.fsid, uid, gid)
 
-        ctr = get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        ctr = to_deployment_container(ctx, ctr)
         config_json = fetch_configs(ctx)
         if self.identity.daemon_type == 'mon' and config_json is not None:
             if 'crush_location' in config_json:
@@ -464,7 +465,8 @@ def validate(self) -> None:
             raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return self.uid, self.gid
@@ -626,7 +628,8 @@ def identity(self) -> DaemonIdentity:
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         self._prevalidate(ctx)
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return self.extract_uid_gid(ctx, self.identity.daemon_type)
@@ -814,7 +817,8 @@ def firewall_service_name(self) -> str:
         return 'nfs'
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def customize_container_endpoints(
         self, endpoints: List[EndPoint], deployment_type: DeploymentType
@@ -1023,7 +1027,9 @@ def get_tcmu_runner_container(self):
         subident = DaemonSubIdentity(
             self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
         )
-        tcmu_container = get_deployment_container(self.ctx, subident)
+        tcmu_container = to_deployment_container(
+            self.ctx, get_container(self.ctx, subident)
+        )
         # TODO: Eventually we don't want to run tcmu-runner through this script.
         # This is intended to be a workaround backported to older releases
         # and should eventually be removed in at least squid onward
@@ -1032,7 +1038,8 @@ def get_tcmu_runner_container(self):
         return tcmu_container
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def config_and_keyring(
         self, ctx: CephadmContext
@@ -1181,7 +1188,8 @@ def get_sysctl_settings() -> List[str]:
         ]
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return 167, 167  # TODO: need to get properly the uid/gid
@@ -1265,7 +1273,8 @@ def validate(self) -> None:
             raise Error(f'Directory does not exist. Got: {self.sock_dir}')
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return extract_uid_gid(ctx)
@@ -1378,7 +1387,8 @@ def get_sysctl_settings() -> List[str]:
         ]
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
 ##################################
 
@@ -1490,7 +1500,8 @@ def get_container_mounts(data_dir: str) -> Dict[str, str]:
         return mounts
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        return get_deployment_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
 
 ##################################
@@ -1546,9 +1557,8 @@ def identity(self) -> DaemonIdentity:
         return self._identity
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        # TODO(jjm) this looks to be the only container for deployment
-        # not using get_deployment_container. Previous oversight?
-        return get_container(ctx, self.identity)
+        ctr = get_container(ctx, self.identity)
+        return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return 65534, 65534
@@ -1686,12 +1696,13 @@ def get_container_binds(self, data_dir: str) -> List[List[str]]:
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         if self._container is None:
-            self._container = get_deployment_container(
+            ctr = get_container(
                 ctx,
                 self.identity,
                 privileged=self.privileged,
                 ptrace=ctx.allow_ptrace,
             )
+            self._container = to_deployment_container(ctx, ctr)
         return self._container
 
     def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
@@ -5096,19 +5107,6 @@ def command_registry_login(ctx: CephadmContext) -> int:
 ##################################
 
 
-def get_deployment_container(
-    ctx: CephadmContext,
-    ident: 'DaemonIdentity',
-    privileged: bool = False,
-    ptrace: bool = False,
-    container_args: Optional[List[str]] = None,
-) -> 'CephContainer':
-    # wrapper for get_container specifically for containers made during the `cephadm deploy`
-    # command. Adds some extra things such as extra container args and custom config files
-    c = get_container(ctx, ident, privileged, ptrace, container_args)
-    return to_deployment_container(ctx, c)
-
-
 def to_deployment_container(
     ctx: CephadmContext, ctr: CephContainer
 ) -> CephContainer:
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 7e31b26307c3..f933e3bc4708 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -329,9 +329,9 @@ def test_skip_firewalld(self, _logger, cephadm_fs):
     @mock.patch('cephadm.logger')
     @mock.patch('cephadm.fetch_custom_config_files')
     @mock.patch('cephadm.get_container')
-    def test_get_deployment_container(self, _get_container, _get_config, _logger):
+    def test_to_deployment_container(self, _get_container, _get_config, _logger):
         """
-        test get_deployment_container properly makes use of extra container args and custom conf files
+        test to_deployment_container properly makes use of extra container args and custom conf files
         """
 
         ctx = _cephadm.CephadmContext()
@@ -365,7 +365,8 @@ def test_get_deployment_container(self, _get_container, _get_config, _logger):
             ptrace=False,
             host_network=True,
         )
-        c = _cephadm.get_deployment_container(ctx, ident)
+        c = _cephadm.get_container(ctx, ident)
+        c = _cephadm.to_deployment_container(ctx, c)
 
         assert '--pids-limit=12345' in c.container_args
         assert '--something' in c.container_args
@@ -380,9 +381,9 @@ def test_get_deployment_container(self, _get_container, _get_config, _logger):
     @mock.patch('cephadm.check_unit', lambda *args, **kwargs: (None, 'running', None))
     @mock.patch('cephadm.get_unit_name', lambda *args, **kwargs: 'mon-unit-name')
     @mock.patch('cephadm.extract_uid_gid', lambda *args, **kwargs: (0, 0))
-    @mock.patch('cephadm.get_deployment_container')
+    @mock.patch('cephadm.get_container')
     @mock.patch('cephadm.apply_deploy_config_to_ctx', lambda d, c: None)
-    def test_mon_crush_location(self, _get_deployment_container, _migrate_sysctl, _make_var_run, _deploy_daemon, _file_lock, _logger, monkeypatch):
+    def test_mon_crush_location(self, _get_container, _migrate_sysctl, _make_var_run, _deploy_daemon, _file_lock, _logger, monkeypatch):
         """
         test that crush location for mon is set if it is included in config_json
         """
@@ -390,6 +391,7 @@ def test_mon_crush_location(self, _get_deployment_container, _migrate_sysctl, _m
         monkeypatch.setattr('cephadmlib.context_getters.fetch_configs', _fetch_configs)
         monkeypatch.setattr('cephadm.fetch_configs', _fetch_configs)
         monkeypatch.setattr('cephadm.read_configuration_source', lambda c: {})
+        monkeypatch.setattr('cephadm.fetch_custom_config_files', mock.MagicMock())
 
         ctx = _cephadm.CephadmContext()
         ctx.name = 'mon.test'
@@ -404,7 +406,7 @@ def test_mon_crush_location(self, _get_deployment_container, _migrate_sysctl, _m
             'crush_location': 'database=a'
         }
 
-        _get_deployment_container.return_value = _cephadm.CephContainer.for_daemon(
+        _get_container.return_value = _cephadm.CephContainer.for_daemon(
             ctx,
             ident=_cephadm.DaemonIdentity(
                 fsid='9b9d7609-f4d5-4aba-94c8-effa764d96c9',

From 66901a6265f4ca13c71e9b4ef85b5f40d0ddce9e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 14:28:46 -0400
Subject: [PATCH 0428/2492] cephadm: move all pids-limit logic to renamed
 function

Repurpose set_pids_limit_unlimited as _update_pids_limit, moving the
logic to decide if the daemon should have unlimited pids into the
function. This was the only caller of set_pids_limit_unlimited.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index bd0a1c97f4b3..326c7aaa1063 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2693,11 +2693,20 @@ def get_ceph_volume_container(ctx: CephadmContext,
     )
 
 
-def set_pids_limit_unlimited(ctx: CephadmContext, container_args: List[str]) -> None:
+def _update_pids_limit(ctx: CephadmContext, daemon_type: str, container_args: List[str]) -> None:
+    """Update container_args to contain a pids limit option if the daemon_type
+    is of a suitable match.
+    """
     # set container's pids-limit to unlimited rather than default (Docker 4096 / Podman 2048)
     # Useful for daemons like iscsi where the default pids-limit limits the number of luns
     # per iscsi target or rgw where increasing the rgw_thread_pool_size to a value near
     # the default pids-limit may cause the container to crash.
+    unlimited_daemons = set(ceph_daemons())
+    unlimited_daemons.add(CephIscsi.daemon_type)
+    unlimited_daemons.add(CephNvmeof.daemon_type)
+    unlimited_daemons.add(NFSGanesha.daemon_type)
+    if daemon_type not in unlimited_daemons:
+        return
     if (
         isinstance(ctx.container_engine, Podman)
         and ctx.container_engine.version >= PIDS_LIMIT_UNLIMITED_PODMAN_VERSION
@@ -2725,12 +2734,7 @@ def get_container(
         envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
     if container_args is None:
         container_args = []
-    unlimited_daemons = set(ceph_daemons())
-    unlimited_daemons.add(CephIscsi.daemon_type)
-    unlimited_daemons.add(CephNvmeof.daemon_type)
-    unlimited_daemons.add(NFSGanesha.daemon_type)
-    if daemon_type in unlimited_daemons:
-        set_pids_limit_unlimited(ctx, container_args)
+    _update_pids_limit(ctx, daemon_type, container_args)
     if daemon_type in ['mon', 'osd']:
         # mon and osd need privileged in order for libudev to query devices
         privileged = True

From 5f25a2a161571bfa3386a19d2c26d4e4093b348f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 14:48:17 -0400
Subject: [PATCH 0429/2492] cephadm: consolidate if-blocks in get_container
 function

Instead of having a number of separate if-statement blocks try to
reduce the number of sections so that you can look at one block
and see what the function is doing for that daemon type.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 326c7aaa1063..b96e3be6d2cf 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2741,20 +2741,38 @@ def get_container(
     if daemon_type == 'rgw':
         entrypoint = '/usr/bin/radosgw'
         name = 'client.rgw.%s' % ident.daemon_id
+        ceph_args = ['-n', name, '-f']
     elif daemon_type == 'rbd-mirror':
         entrypoint = '/usr/bin/rbd-mirror'
         name = 'client.rbd-mirror.%s' % ident.daemon_id
+        ceph_args = ['-n', name, '-f']
     elif daemon_type == 'cephfs-mirror':
         entrypoint = '/usr/bin/cephfs-mirror'
         name = 'client.cephfs-mirror.%s' % ident.daemon_id
+        ceph_args = ['-n', name, '-f']
     elif daemon_type == 'crash':
         entrypoint = '/usr/bin/ceph-crash'
         name = 'client.crash.%s' % ident.daemon_id
+        ceph_args = ['-n', name]
     elif daemon_type in ['mon', 'mgr', 'mds', 'osd']:
         entrypoint = '/usr/bin/ceph-' + daemon_type
         name = ident.daemon_name
+        ceph_args = ['-n', name, '-f']
     elif daemon_type in Monitoring.components:
         entrypoint = ''
+        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
+        monitoring_args = [
+            '--user',
+            str(uid),
+            # FIXME: disable cpu/memory limits for the time being (not supported
+            # by ubuntu 18.04 kernel!)
+        ]
+        container_args.extend(monitoring_args)
+        if daemon_type == 'node-exporter':
+            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
+            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
+            # between the node-exporter container and the host to avoid selinux denials
+            container_args.extend(['--security-opt', 'label=disable'])
     elif daemon_type in Tracing.components:
         entrypoint = ''
         name = ident.daemon_name
@@ -2768,6 +2786,7 @@ def get_container(
     elif daemon_type == CephExporter.daemon_type:
         entrypoint = CephExporter.entrypoint
         name = 'client.ceph-exporter.%s' % ident.daemon_id
+        ceph_args = ['-n', name, '-f']
     elif daemon_type == HAproxy.daemon_type:
         name = ident.daemon_name
         container_args.extend(['--user=root'])  # haproxy 2.4 defaults to a different user
@@ -2792,25 +2811,6 @@ def get_container(
         host_network = False
         envs.extend(cc.get_container_envs())
         container_args.extend(cc.get_container_args())
-
-    if daemon_type in Monitoring.components:
-        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
-        monitoring_args = [
-            '--user',
-            str(uid),
-            # FIXME: disable cpu/memory limits for the time being (not supported
-            # by ubuntu 18.04 kernel!)
-        ]
-        container_args.extend(monitoring_args)
-        if daemon_type == 'node-exporter':
-            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
-            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
-            # between the node-exporter container and the host to avoid selinux denials
-            container_args.extend(['--security-opt', 'label=disable'])
-    elif daemon_type == 'crash':
-        ceph_args = ['-n', name]
-    elif daemon_type in ceph_daemons():
-        ceph_args = ['-n', name, '-f']
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
         container_args.append(

From 87b03e61753d0c388abc50e41f40a632b1522562 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:03:22 -0400
Subject: [PATCH 0430/2492] cephadm: move ceph get_daemon_args logic to a
 method on the class

Move the logic used to get deamon args into a method of the Ceph class.
This requires an update to save a reference to ctx.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 77 ++++++++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 33 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b96e3be6d2cf..fd571ffd5b2b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -217,12 +217,13 @@ def for_daemon_type(cls, daemon_type: str) -> bool:
         # TODO: figure out a way to un-special-case osd
         return daemon_type in cls._daemons and daemon_type != 'osd'
 
-    def __init__(self, ident: DaemonIdentity) -> None:
+    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
+        self.ctx = ctx
         self._identity = ident
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Ceph':
-        return cls(ident)
+        return cls(ctx, ident)
 
     @property
     def identity(self) -> DaemonIdentity:
@@ -267,6 +268,38 @@ def config_and_keyring(
     ) -> Tuple[Optional[str], Optional[str]]:
         return get_config_and_keyring(ctx)
 
+    def get_daemon_args(self) -> List[str]:
+        if self.identity.daemon_type == 'crash':
+            return []
+        r = [
+            '--setuser', 'ceph',
+            '--setgroup', 'ceph',
+            '--default-log-to-file=false',
+        ]
+        log_to_journald = should_log_to_journald(self.ctx)
+        if log_to_journald:
+            r += [
+                '--default-log-to-journald=true',
+                '--default-log-to-stderr=false',
+            ]
+        else:
+            r += [
+                '--default-log-to-stderr=true',
+                '--default-log-stderr-prefix=debug ',
+            ]
+        if self.identity.daemon_type == 'mon':
+            r += [
+                '--default-mon-cluster-log-to-file=false',
+            ]
+            if log_to_journald:
+                r += [
+                    '--default-mon-cluster-log-to-journald=true',
+                    '--default-mon-cluster-log-to-stderr=false',
+                ]
+            else:
+                r += ['--default-mon-cluster-log-to-stderr=true']
+        return r
+
 ##################################
 
 
@@ -278,9 +311,12 @@ def for_daemon_type(cls, daemon_type: str) -> bool:
         return daemon_type == 'osd'
 
     def __init__(
-        self, ident: DaemonIdentity, osd_fsid: Optional[str] = None
+        self,
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        osd_fsid: Optional[str] = None,
     ) -> None:
-        super().__init__(ident)
+        super().__init__(ctx, ident)
         self._osd_fsid = osd_fsid
 
     @classmethod
@@ -290,7 +326,7 @@ def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'OSD':
             logger.info(
                 'Creating an OSD daemon form without an OSD FSID value'
             )
-        return cls(ident, osd_fsid)
+        return cls(ctx, ident, osd_fsid)
 
     @staticmethod
     def get_sysctl_settings() -> List[str]:
@@ -2218,34 +2254,9 @@ def get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     r = list()  # type: List[str]
 
     daemon_type = ident.daemon_type
-    if daemon_type in ceph_daemons() and daemon_type not in ['crash', 'ceph-exporter']:
-        r += [
-            '--setuser', 'ceph',
-            '--setgroup', 'ceph',
-            '--default-log-to-file=false',
-        ]
-        log_to_journald = should_log_to_journald(ctx)
-        if log_to_journald:
-            r += [
-                '--default-log-to-journald=true',
-                '--default-log-to-stderr=false',
-            ]
-        else:
-            r += [
-                '--default-log-to-stderr=true',
-                '--default-log-stderr-prefix=debug ',
-            ]
-        if daemon_type == 'mon':
-            r += [
-                '--default-mon-cluster-log-to-file=false',
-            ]
-            if log_to_journald:
-                r += [
-                    '--default-mon-cluster-log-to-journald=true',
-                    '--default-mon-cluster-log-to-stderr=false',
-                ]
-            else:
-                r += ['--default-mon-cluster-log-to-stderr=true']
+    if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
+        daemon = Ceph.create(ctx, ident)
+        r += daemon.get_daemon_args()
     elif daemon_type in Monitoring.components:
         metadata = Monitoring.components[daemon_type]
         r += metadata.get('args', list())

From c9ea1d7849f9d1b74caeed6fb0f0b8ae7a129335 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:11:26 -0400
Subject: [PATCH 0431/2492] cephadm: move monitoring get_daemon_args logic to
 class

Move the logic used to get deamon args into a method of the Monitoring
class.
This requires an update to save a reference to ctx.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 126 ++++++++++++++++++++++-------------------
 1 file changed, 67 insertions(+), 59 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fd571ffd5b2b..c56eaf2f287e 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -651,12 +651,13 @@ def extract_uid_gid(
             raise Error('{} not implemented yet'.format(daemon_type))
         return uid, gid
 
-    def __init__(self, ident: DaemonIdentity) -> None:
+    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
+        self.ctx = ctx
         self._identity = ident
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Monitoring':
-        return cls(ident)
+        return cls(ctx, ident)
 
     @property
     def identity(self) -> DaemonIdentity:
@@ -699,6 +700,68 @@ def _prevalidate(self, ctx: CephadmContext) -> None:
                     )
                 )
 
+    def get_daemon_args(self) -> List[str]:
+        ctx = self.ctx
+        daemon_type = self.identity.daemon_type
+        metadata = self.components[daemon_type]
+        r = list(metadata.get('args', []))
+        # set ip and port to bind to for nodeexporter,alertmanager,prometheus
+        if daemon_type not in ['grafana', 'loki', 'promtail']:
+            ip = ''
+            port = self.port_map[daemon_type][0]
+            meta = fetch_meta(ctx)
+            if meta:
+                if 'ip' in meta and meta['ip']:
+                    ip = meta['ip']
+                if 'ports' in meta and meta['ports']:
+                    port = meta['ports'][0]
+            r += [f'--web.listen-address={ip}:{port}']
+            if daemon_type == 'prometheus':
+                config = fetch_configs(ctx)
+                retention_time = config.get('retention_time', '15d')
+                retention_size = config.get('retention_size', '0')  # default to disabled
+                r += [f'--storage.tsdb.retention.time={retention_time}']
+                r += [f'--storage.tsdb.retention.size={retention_size}']
+                scheme = 'http'
+                host = get_fqdn()
+                # in case host is not an fqdn then we use the IP to
+                # avoid producing a broken web.external-url link
+                if '.' not in host:
+                    ipv4_addrs, ipv6_addrs = get_ip_addresses(get_hostname())
+                    # use the first ipv4 (if any) otherwise use the first ipv6
+                    addr = next(iter(ipv4_addrs or ipv6_addrs), None)
+                    host = wrap_ipv6(addr) if addr else host
+                r += [f'--web.external-url={scheme}://{host}:{port}']
+        if daemon_type == 'alertmanager':
+            config = fetch_configs(ctx)
+            peers = config.get('peers', list())  # type: ignore
+            for peer in peers:
+                r += ['--cluster.peer={}'.format(peer)]
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+            # some alertmanager, by default, look elsewhere for a config
+            r += ['--config.file=/etc/alertmanager/alertmanager.yml']
+        if daemon_type == 'promtail':
+            r += ['--config.expand-env']
+        if daemon_type == 'prometheus':
+            config = fetch_configs(ctx)
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+        if daemon_type == 'node-exporter':
+            config = fetch_configs(ctx)
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+            r += ['--path.procfs=/host/proc',
+                  '--path.sysfs=/host/sys',
+                  '--path.rootfs=/rootfs']
+        return r
+
 ##################################
 
 
@@ -2258,63 +2321,8 @@ def get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
         daemon = Ceph.create(ctx, ident)
         r += daemon.get_daemon_args()
     elif daemon_type in Monitoring.components:
-        metadata = Monitoring.components[daemon_type]
-        r += metadata.get('args', list())
-        # set ip and port to bind to for nodeexporter,alertmanager,prometheus
-        if daemon_type not in ['grafana', 'loki', 'promtail']:
-            ip = ''
-            port = Monitoring.port_map[daemon_type][0]
-            meta = fetch_meta(ctx)
-            if meta:
-                if 'ip' in meta and meta['ip']:
-                    ip = meta['ip']
-                if 'ports' in meta and meta['ports']:
-                    port = meta['ports'][0]
-            r += [f'--web.listen-address={ip}:{port}']
-            if daemon_type == 'prometheus':
-                config = fetch_configs(ctx)
-                retention_time = config.get('retention_time', '15d')
-                retention_size = config.get('retention_size', '0')  # default to disabled
-                r += [f'--storage.tsdb.retention.time={retention_time}']
-                r += [f'--storage.tsdb.retention.size={retention_size}']
-                scheme = 'http'
-                host = get_fqdn()
-                # in case host is not an fqdn then we use the IP to
-                # avoid producing a broken web.external-url link
-                if '.' not in host:
-                    ipv4_addrs, ipv6_addrs = get_ip_addresses(get_hostname())
-                    # use the first ipv4 (if any) otherwise use the first ipv6
-                    addr = next(iter(ipv4_addrs or ipv6_addrs), None)
-                    host = wrap_ipv6(addr) if addr else host
-                r += [f'--web.external-url={scheme}://{host}:{port}']
-        if daemon_type == 'alertmanager':
-            config = fetch_configs(ctx)
-            peers = config.get('peers', list())  # type: ignore
-            for peer in peers:
-                r += ['--cluster.peer={}'.format(peer)]
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-            # some alertmanager, by default, look elsewhere for a config
-            r += ['--config.file=/etc/alertmanager/alertmanager.yml']
-        if daemon_type == 'promtail':
-            r += ['--config.expand-env']
-        if daemon_type == 'prometheus':
-            config = fetch_configs(ctx)
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-        if daemon_type == 'node-exporter':
-            config = fetch_configs(ctx)
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-            r += ['--path.procfs=/host/proc',
-                  '--path.sysfs=/host/sys',
-                  '--path.rootfs=/rootfs']
+        monitoring = Monitoring.create(ctx, ident)
+        r += monitoring.get_daemon_args()
     elif daemon_type == 'jaeger-agent':
         r.extend(Tracing.components[daemon_type]['daemon_args'])
     elif daemon_type == NFSGanesha.daemon_type:

From adf31c30da97c31362f1323ef30f11571094cd89 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:19:21 -0400
Subject: [PATCH 0432/2492] cephadm: move tracing get_daemon_args logic to
 class

Move the logic used to get deamon args into a method of the
Tracing class.
This requires an update to save a reference to ctx.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c56eaf2f287e..088234afb5fc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1662,6 +1662,11 @@ def container(self, ctx: CephadmContext) -> CephContainer:
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return 65534, 65534
 
+    def get_daemon_args(self) -> List[str]:
+        return self.components[self.identity.daemon_type].get(
+            'daemon_args', []
+        )
+
 ##################################
 
 
@@ -2324,7 +2329,8 @@ def get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
         monitoring = Monitoring.create(ctx, ident)
         r += monitoring.get_daemon_args()
     elif daemon_type == 'jaeger-agent':
-        r.extend(Tracing.components[daemon_type]['daemon_args'])
+        tracing = Tracing.create(ctx, ident)
+        r += tracing.get_daemon_args()
     elif daemon_type == NFSGanesha.daemon_type:
         nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
         r += nfs_ganesha.get_daemon_args()

From bf35bf769bbf36cfe8e3c08b062a043249ad2f2b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:22:37 -0400
Subject: [PATCH 0433/2492] cephadm: use ceph specific get_daemon_args calls

Use the new Ceph class specific get_daemon_args method instead of the
generic get_daemon_args calls that check all sorts of daemon types when
we're running specialized mon containers, which we know for certain are
ceph based.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 088234afb5fc..f8b0243e37bc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2924,7 +2924,7 @@ def deploy_daemon(
                 '--fsid', ident.fsid,
                 '-c', '/tmp/config',
                 '--keyring', '/tmp/keyring',
-            ] + get_daemon_args(ctx, ident),
+            ] + Ceph.create(ctx, ident).get_daemon_args(),
             volume_mounts={
                 log_dir: '/var/log/ceph:z',
                 mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (ident.daemon_id),
@@ -4305,7 +4305,7 @@ def prepare_create_mon(
             '-c', '/dev/null',
             '--monmap', '/tmp/monmap',
             '--keyring', '/tmp/keyring',
-        ] + get_daemon_args(ctx, ident),
+        ] + Ceph.create(ctx, ident).get_daemon_args(),
         volume_mounts={
             log_dir: '/var/log/ceph:z',
             mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id),

From 3bb4545d6b304bd2ee622c3778fc94280a519a0e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 16:25:17 -0400
Subject: [PATCH 0434/2492] cephadm: make get_daemon_args private

The get_daemon_args function now only has one caller. To double check
that and to prepare for this function's eventual removal we prefix the
name with an underscore to take it private.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 4 ++--
 src/cephadm/tests/test_cephadm.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index f8b0243e37bc..9a36360bf2dc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2318,7 +2318,7 @@ def get_legacy_daemon_fsid(ctx, cluster,
     return fsid
 
 
-def get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
+def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     r = list()  # type: List[str]
 
     daemon_type = ident.daemon_type
@@ -2847,7 +2847,7 @@ def get_container(
         ctx,
         ident=ident,
         entrypoint=entrypoint,
-        args=ceph_args + get_daemon_args(ctx, ident),
+        args=ceph_args + _get_daemon_args(ctx, ident),
         container_args=container_args,
         volume_mounts=get_container_mounts(ctx, ident),
         bind_mounts=get_container_binds(ctx, ident),
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index f933e3bc4708..c12bb9ec5c91 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1229,9 +1229,9 @@ def test_prometheus_external_url(self):
         daemon_type = 'prometheus'
         daemon_id = 'home'
         fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704'
-        args = _cephadm.get_daemon_args(
+        args = _cephadm.Monitoring.create(
             ctx, _cephadm.DaemonIdentity(fsid, daemon_type, daemon_id)
-        )
+        ).get_daemon_args()
         assert any([x.startswith('--web.external-url=http://') for x in args])
 
     @mock.patch('cephadm.call')

From eaacc0e29eea372a21bbf868a57cbcc4cfbe2ce7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 10:30:16 -0400
Subject: [PATCH 0435/2492] cephadm: ensure entrypoint is assigned a string

Ensure that the entrypoint variable is assigned a string. The
custom container class can have None and that can confuse future
refactoring.
The mypy command we're using does not notice this.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 9a36360bf2dc..b47d12f92493 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2832,7 +2832,7 @@ def get_container(
         privileged = True
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
-        entrypoint = cc.entrypoint
+        entrypoint = cc.entrypoint or ''
         host_network = False
         envs.extend(cc.get_container_envs())
         container_args.extend(cc.get_container_args())

From 43051cf5fe9bdf96e9f602691eab1a40ef980cd1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 10:52:56 -0400
Subject: [PATCH 0436/2492] cephadm: move daemon args assignment to a variable

Move the call to _get_daemon_args out of the function call.
This will aid with the next refactoring steps, were the content
of _get_daemon_args will be lifted into get_container.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b47d12f92493..3aa030f23953 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2843,11 +2843,12 @@ def get_container(
         )
 
     _update_container_args_for_podman(ctx, ident, container_args)
+    d_args = _get_daemon_args(ctx, ident)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,
         entrypoint=entrypoint,
-        args=ceph_args + _get_daemon_args(ctx, ident),
+        args=ceph_args + d_args,
         container_args=container_args,
         volume_mounts=get_container_mounts(ctx, ident),
         bind_mounts=get_container_binds(ctx, ident),

From 6a072f1b403c0af1255c4128bcb455cb38030a69 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 10:59:55 -0400
Subject: [PATCH 0437/2492] cephadm: move snmp daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 3aa030f23953..c3f7d9468de1 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2343,9 +2343,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
         r.extend(cc.get_daemon_args())
-    elif daemon_type == SNMPGateway.daemon_type:
-        sc = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
-        r.extend(sc.get_daemon_args())
 
     return r
 
@@ -2751,6 +2748,7 @@ def get_container(
     entrypoint: str = ''
     name: str = ''
     ceph_args: List[str] = []
+    d_args: List[str] = []
     envs: List[str] = []
     host_network: bool = True
 
@@ -2841,9 +2839,10 @@ def get_container(
         container_args.append(
             f'--env-file={sg.conf_file_path}'
         )
+        d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
-    d_args = _get_daemon_args(ctx, ident)
+    d_args.extend(_get_daemon_args(ctx, ident))
     return CephContainer.for_daemon(
         ctx,
         ident=ident,

From b8a171e34f09e2251bb807ef96b4481522f5aa80 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:03:52 -0400
Subject: [PATCH 0438/2492] cephadm: move custom container daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c3f7d9468de1..0d820cb2128f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2340,9 +2340,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type == HAproxy.daemon_type:
         haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
         r += haproxy.get_daemon_args()
-    elif daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
-        r.extend(cc.get_daemon_args())
 
     return r
 
@@ -2834,6 +2831,7 @@ def get_container(
         host_network = False
         envs.extend(cc.get_container_envs())
         container_args.extend(cc.get_container_args())
+        d_args.extend(cc.get_daemon_args())
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
         container_args.append(

From 3df35b03b94062a8ee79ca1f043356e84ad7c0f6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:04:32 -0400
Subject: [PATCH 0439/2492] cephadm: move haproxy daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 0d820cb2128f..6b25635b7912 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2337,9 +2337,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type == CephExporter.daemon_type:
         ceph_exporter = CephExporter.init(ctx, ident.fsid, ident.daemon_id)
         r.extend(ceph_exporter.get_daemon_args())
-    elif daemon_type == HAproxy.daemon_type:
-        haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
-        r += haproxy.get_daemon_args()
 
     return r
 
@@ -2810,6 +2807,8 @@ def get_container(
     elif daemon_type == HAproxy.daemon_type:
         name = ident.daemon_name
         container_args.extend(['--user=root'])  # haproxy 2.4 defaults to a different user
+        haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
+        d_args.extend(haproxy.get_daemon_args())
     elif daemon_type == Keepalived.daemon_type:
         name = ident.daemon_name
         envs.extend(Keepalived.get_container_envs())

From d52c684014e037815a9919ca8c0af9cae66e6706 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:08:09 -0400
Subject: [PATCH 0440/2492] cephadm: move ceph exporter daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6b25635b7912..a40d022beefc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2334,9 +2334,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type == NFSGanesha.daemon_type:
         nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
         r += nfs_ganesha.get_daemon_args()
-    elif daemon_type == CephExporter.daemon_type:
-        ceph_exporter = CephExporter.init(ctx, ident.fsid, ident.daemon_id)
-        r.extend(ceph_exporter.get_daemon_args())
 
     return r
 
@@ -2804,6 +2801,8 @@ def get_container(
         entrypoint = CephExporter.entrypoint
         name = 'client.ceph-exporter.%s' % ident.daemon_id
         ceph_args = ['-n', name, '-f']
+        ceph_exporter = CephExporter.init(ctx, ident.fsid, ident.daemon_id)
+        d_args.extend(ceph_exporter.get_daemon_args())
     elif daemon_type == HAproxy.daemon_type:
         name = ident.daemon_name
         container_args.extend(['--user=root'])  # haproxy 2.4 defaults to a different user

From a7f5479a8ac6c6df52652311f5a9b370da9039b2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:09:52 -0400
Subject: [PATCH 0441/2492] cephadm: move nfs daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index a40d022beefc..644f4fa60cf5 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2331,9 +2331,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type == 'jaeger-agent':
         tracing = Tracing.create(ctx, ident)
         r += tracing.get_daemon_args()
-    elif daemon_type == NFSGanesha.daemon_type:
-        nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
-        r += nfs_ganesha.get_daemon_args()
 
     return r
 
@@ -2797,6 +2794,8 @@ def get_container(
         entrypoint = NFSGanesha.entrypoint
         name = ident.daemon_name
         envs.extend(NFSGanesha.get_container_envs())
+        nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
+        d_args.extend(nfs_ganesha.get_daemon_args())
     elif daemon_type == CephExporter.daemon_type:
         entrypoint = CephExporter.entrypoint
         name = 'client.ceph-exporter.%s' % ident.daemon_id

From 4c20c7096de3f77968537f0b6dd9f8263bf08b4b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:11:27 -0400
Subject: [PATCH 0442/2492] cephadm: move jaeger tracing daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 644f4fa60cf5..3c95123046f9 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2328,9 +2328,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     elif daemon_type in Monitoring.components:
         monitoring = Monitoring.create(ctx, ident)
         r += monitoring.get_daemon_args()
-    elif daemon_type == 'jaeger-agent':
-        tracing = Tracing.create(ctx, ident)
-        r += tracing.get_daemon_args()
 
     return r
 
@@ -2790,6 +2787,9 @@ def get_container(
         config = fetch_configs(ctx)
         Tracing.set_configuration(config, daemon_type)
         envs.extend(Tracing.components[daemon_type].get('envs', []))
+        if daemon_type == 'jaeger-agent':
+            tracing = Tracing.create(ctx, ident)
+            d_args.extend(tracing.get_daemon_args())
     elif daemon_type == NFSGanesha.daemon_type:
         entrypoint = NFSGanesha.entrypoint
         name = ident.daemon_name

From e6cb7695655af6b06c51dc8788b22751dd976cba Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:13:52 -0400
Subject: [PATCH 0443/2492] cephadm: move monitoring daemon args out of
 _get_daemon_args

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 3c95123046f9..2c3e325c0a58 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2325,9 +2325,6 @@ def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
     if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
         daemon = Ceph.create(ctx, ident)
         r += daemon.get_daemon_args()
-    elif daemon_type in Monitoring.components:
-        monitoring = Monitoring.create(ctx, ident)
-        r += monitoring.get_daemon_args()
 
     return r
 
@@ -2781,6 +2778,8 @@ def get_container(
             # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
             # between the node-exporter container and the host to avoid selinux denials
             container_args.extend(['--security-opt', 'label=disable'])
+        monitoring = Monitoring.create(ctx, ident)
+        d_args.extend(monitoring.get_daemon_args())
     elif daemon_type in Tracing.components:
         entrypoint = ''
         name = ident.daemon_name

From bd226050066b126f66b299bcf94ccc9e22d8df8c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 11:20:36 -0400
Subject: [PATCH 0444/2492] cephadm: eliminate the _get_daemon_args function

Move the last remaining bit of daemon args logic (ceph) out of
the _get_daemon_args function and remove the function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 2c3e325c0a58..32d229a3169a 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2318,17 +2318,6 @@ def get_legacy_daemon_fsid(ctx, cluster,
     return fsid
 
 
-def _get_daemon_args(ctx: CephadmContext, ident: 'DaemonIdentity') -> List[str]:
-    r = list()  # type: List[str]
-
-    daemon_type = ident.daemon_type
-    if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
-        daemon = Ceph.create(ctx, ident)
-        r += daemon.get_daemon_args()
-
-    return r
-
-
 def create_daemon_dirs(
     ctx: CephadmContext,
     ident: 'DaemonIdentity',
@@ -2740,6 +2729,9 @@ def get_container(
     if container_args is None:
         container_args = []
     _update_pids_limit(ctx, daemon_type, container_args)
+    if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
+        ceph_daemon = Ceph.create(ctx, ident)
+        d_args.extend(ceph_daemon.get_daemon_args())
     if daemon_type in ['mon', 'osd']:
         # mon and osd need privileged in order for libudev to query devices
         privileged = True
@@ -2836,7 +2828,6 @@ def get_container(
         d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
-    d_args.extend(_get_daemon_args(ctx, ident))
     return CephContainer.for_daemon(
         ctx,
         ident=ident,

From aadb87aeb5cd8b413937ddfe4562411d424c4bb1 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 5 Nov 2023 16:24:16 +1000
Subject: [PATCH 0445/2492] doc/rados: format Q&A list in t-mon.rst

Format a Q&A list in doc/rados/troubleshooting/troubleshooting.rst, in
the "Clock Skews" section.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 3849149fb51a..ca2f19631a84 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -377,12 +377,15 @@ following:
 * EndRun `https://endruntechnologies.com <https://endruntechnologies.com/products/ntp-time-servers>`_
 * Netburner `https://www.netburner.com <https://www.netburner.com/products/network-time-server/pk70-ex-ntp-network-time-server>`_
 
-What's the maximum tolerated clock skew?
+Clock Skew Questions and Answers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**What's the maximum tolerated clock skew?**
 
   By default, monitors allow clocks to drift up to a maximum of 0.05 seconds
   (50 milliseconds).
 
-Can I increase the maximum tolerated clock skew?
+**Can I increase the maximum tolerated clock skew?**
 
   Yes, but we strongly recommend against doing so. The maximum tolerated clock
   skew is configurable via the ``mon-clock-drift-allowed`` option, but it is
@@ -393,7 +396,7 @@ Can I increase the maximum tolerated clock skew?
   unforeseen effects on the stability of the monitors and overall cluster
   health.
 
-How do I know whether there is a clock skew?
+**How do I know whether there is a clock skew?**
 
   The monitors will warn you via the cluster status ``HEALTH_WARN``. When clock
   skew is present, the ``ceph health detail`` and ``ceph status`` commands
@@ -410,7 +413,7 @@ How do I know whether there is a clock skew?
   the reported offsets of other monitors are relative to the lead monitor, not
   to any external reference source.
 
-What should I do if there is a clock skew?
+**What should I do if there is a clock skew?**
 
   Synchronize your clocks. Using an NTP client might help. However, if you
   are already using an NTP client and you still encounter clock skew problems,

From 5496bd426f35c0ab91ae7d544ae92ed3b517c7eb Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 5 Nov 2023 22:28:39 +1000
Subject: [PATCH 0446/2492] doc/rados: edit t-mon.rst text

Clarify the text in the "Clock Skew" section of
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 3849149fb51a..fb115658008f 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -350,12 +350,13 @@ Inject a monmap into the monitor
 Clock Skews
 -----------
 
-The Paxos consensus algorithm requires tight time alignment. For this reason,
-clock skew among the monitors in the quorum can have a serious effect on
-monitor operation. The resulting behavior can be very puzzling. To avoid
-this kind of issue, run a clock synchronization tool on your monitor nodes:
-for example, ``Chrony`` or the legacy ``ntpd`` utility. Be sure to configure
-the monitor nodes with the `iburst` option and multiple peers:
+The Paxos consensus algorithm requires close time synchroniziation, which means
+that clock skew among the monitors in the quorum can have a serious effect on
+monitor operation. The resulting behavior can be puzzling. To avoid this issue,
+run a clock synchronization tool on your monitor nodes: for example, use
+``Chrony`` or the legacy ``ntpd`` utility. Configure each monitor nodes so that
+the `iburst` option is in effect and so that each monitor has multiple peers,
+including the following: 
 
 * Each other
 * Internal ``NTP`` servers
@@ -366,12 +367,12 @@ the monitor nodes with the `iburst` option and multiple peers:
    into initial synchronization.
 
 Furthermore, it is advisable to synchronize *all* nodes in your cluster against
-internal and external servers, and perhaps even against your monitors. ``NTP``
-servers should run on bare metal; VM virtualized clocks are not suitable for
-steady timekeeping. For more information, visit `https://www.ntp.org
-<https://www.ntp.org>`_.  Your organization might already have quality internal
-``NTP`` servers available.  Sources for ``NTP`` server appliances include the
-following:
+internal and external servers, and perhaps even against your monitors. Run
+``NTP`` servers on bare metal: VM-virtualized clocks are not suitable for
+steady timekeeping. See `https://www.ntp.org <https://www.ntp.org>`_ for more
+information about the Network Time Protocol (NTP). Your organization might
+already have quality internal ``NTP`` servers available.  Sources for ``NTP``
+server appliances include the following:
 
 * Microsemi (formerly Symmetricom) `https://microsemi.com <https://www.microsemi.com/product-directory/3425-timing-synchronization>`_
 * EndRun `https://endruntechnologies.com <https://endruntechnologies.com/products/ntp-time-servers>`_

From d7999aba5acfdc8b088ad76eb2ad98c12846117a Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Sun, 5 Nov 2023 16:52:30 +0200
Subject: [PATCH 0447/2492] rgw: adding y2k38 annotations to supress coverity
 issues

the supression comments must be directly above the line
where the issue is.

this is a followup on these commits:
edfb5b9049ee897779863d780bfbaa130a7fbc40
aca2668c88a048ab556c53411dcf2ccd43099d50

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/driver/rados/rgw_trim_datalog.cc | 4 ++--
 src/rgw/driver/rados/rgw_trim_mdlog.cc   | 7 ++++---
 src/rgw/rgw_admin.cc                     | 7 ++++---
 src/test/cls_lock/test_cls_lock.cc       | 1 +
 src/test/cls_log/test_cls_log.cc         | 2 ++
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_trim_datalog.cc b/src/rgw/driver/rados/rgw_trim_datalog.cc
index 12adc6fbaa54..bac0cda8dd64 100644
--- a/src/rgw/driver/rados/rgw_trim_datalog.cc
+++ b/src/rgw/driver/rados/rgw_trim_datalog.cc
@@ -224,11 +224,11 @@ int DataLogTrimPollCR::operate(const DoutPrefixProvider *dpp)
       // prevent other gateways from attempting to trim for the duration
       set_status("acquiring trim lock");
 
-      // interval is a small number and unlikely to overflow
-      // coverity[store_truncates_time_t:SUPPRESS]
       yield call(new RGWSimpleRadosLockCR(store->svc()->rados->get_async_processor(), store,
                                           rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, lock_oid),
                                           "data_trim", lock_cookie,
+                                          // interval is a small number and unlikely to overflow
+                                          // coverity[store_truncates_time_t:SUPPRESS]
                                           interval.sec()));
       if (retcode < 0) {
         // if the lock is already held, go back to sleep and try again later
diff --git a/src/rgw/driver/rados/rgw_trim_mdlog.cc b/src/rgw/driver/rados/rgw_trim_mdlog.cc
index d97538f98321..5a2e07154d28 100644
--- a/src/rgw/driver/rados/rgw_trim_mdlog.cc
+++ b/src/rgw/driver/rados/rgw_trim_mdlog.cc
@@ -669,10 +669,11 @@ int MetaTrimPollCR::operate(const DoutPrefixProvider *dpp)
       // prevent others from trimming for our entire wait interval
       set_status("acquiring trim lock");
 
-      // interval is a small number and unlikely to overflow
-      // coverity[store_truncates_time_t:SUPPRESS]
       yield call(new RGWSimpleRadosLockCR(store->svc()->rados->get_async_processor(), store,
-                                          obj, name, cookie, interval.sec()));
+                                          obj, name, cookie, 
+                                          // interval is a small number and unlikely to overflow
+                                          // coverity[store_truncates_time_t:SUPPRESS]
+                                          interval.sec()));
       if (retcode < 0) {
         ldout(cct, 4) << "failed to lock: " << cpp_strerror(retcode) << dendl;
         continue;
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 40146293a2c8..ecf16abf2cb9 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -3039,19 +3039,20 @@ static int scan_totp(CephContext *cct, ceph::real_time& now, rados::cls::otp::ot
   uint32_t max_skew = MAX_TOTP_SKEW_HOURS * 3600;
 
   while (time_ofs_abs < max_skew) {
+    // coverity supression: oath_totp_validate2 is an external library function, cannot fix internally
+    // Further, step_size is a small number and unlikely to overflow
     int rc = oath_totp_validate2(totp.seed_bin.c_str(), totp.seed_bin.length(),
                              start_time, 
+                             // coverity[store_truncates_time_t:SUPPRESS]
                              step_size,
                              time_ofs,
                              1,
                              nullptr,
                              pins[0].c_str());
     if (rc != OATH_INVALID_OTP) {
-      // oath_totp_validate2 is an external library function, cannot fix internally
-      // Further, step_size is a small number and unlikely to overflow
-      // coverity[store_truncates_time_t:SUPPRESS]
       rc = oath_totp_validate2(totp.seed_bin.c_str(), totp.seed_bin.length(),
                                start_time, 
+                               // coverity[store_truncates_time_t:SUPPRESS]
                                step_size,
                                time_ofs - step_size, /* smaller time_ofs moves time forward */
                                1,
diff --git a/src/test/cls_lock/test_cls_lock.cc b/src/test/cls_lock/test_cls_lock.cc
index b915de9c2362..ef543ce8647c 100644
--- a/src/test/cls_lock/test_cls_lock.cc
+++ b/src/test/cls_lock/test_cls_lock.cc
@@ -298,6 +298,7 @@ TEST(ClsLock, TestLockDuration) {
     ASSERT_EQ(-EEXIST, r);
   }
 
+  // coverity[store_truncates_time_t:SUPPRESS]
   sleep(dur.sec());
   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 
diff --git a/src/test/cls_log/test_cls_log.cc b/src/test/cls_log/test_cls_log.cc
index e8777ac5fedb..f8c1a32494a1 100644
--- a/src/test/cls_log/test_cls_log.cc
+++ b/src/test/cls_log/test_cls_log.cc
@@ -79,6 +79,7 @@ void generate_log(librados::IoCtx& ioctx, string& oid, int max, utime_t& start_t
   int i;
 
   for (i = 0; i < max; i++) {
+    // coverity[store_truncates_time_t:SUPPRESS]
     uint32_t secs = start_time.sec();
     if (modify_time)
       secs += i;
@@ -94,6 +95,7 @@ void generate_log(librados::IoCtx& ioctx, string& oid, int max, utime_t& start_t
 
 utime_t get_time(utime_t& start_time, int i, bool modify_time)
 {
+  // coverity[store_truncates_time_t:SUPPRESS]
   uint32_t secs = start_time.sec();
   if (modify_time)
     secs += i;

From e723f6d2aadf41071b8acd47debc6f0ca4733af4 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 3 Nov 2023 17:00:07 +0800
Subject: [PATCH 0448/2492] crimson/os/seastore: cleanup, be explicit about
 stable and stable_written

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/btree/fixed_kv_btree.h      |  2 +-
 src/crimson/os/seastore/btree/fixed_kv_node.h       | 13 ++++---------
 src/crimson/os/seastore/cached_extent.h             | 11 +++++++++--
 .../crimson/seastore/test_transaction_manager.cc    |  2 +-
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h
index 4133be627c10..a2a97e30a503 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_btree.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h
@@ -513,7 +513,7 @@ class FixedKVBtree {
           &child_node);
       }
       if (ret == Transaction::get_extent_ret::PRESENT) {
-        if (child_node->is_stable()) {
+        if (child_node->is_stable_written()) {
           assert(child_node->is_valid());
           auto cnode = child_node->template cast<child_node_t>();
           assert(cnode->has_parent_tracker());
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index 0ae23b2f4dea..4d32475ee24d 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -992,16 +992,13 @@ struct FixedKVLeafNode
   }
 
   // children are considered stable if any of the following case is true:
-  // 1. Not in memory
-  // 2. being stable
-  // 3. being mutation pending and under-io
+  // 1. The child extent is absent in cache
+  // 2. The child extent is stable
   bool is_child_stable(uint16_t pos) const final {
     auto child = this->children[pos];
     if (is_valid_child_ptr(child)) {
       ceph_assert(child->is_logical());
-      return child->is_stable() ||
-	(child->is_mutation_pending() &&
-	 child->is_pending_io());
+      return child->is_stable();
     } else if (this->is_pending()) {
       auto key = this->iter_idx(pos).get_key();
       auto &sparent = this->get_stable_for_key(key);
@@ -1009,9 +1006,7 @@ struct FixedKVLeafNode
       auto child = sparent.children[spos];
       if (is_valid_child_ptr(child)) {
 	ceph_assert(child->is_logical());
-	return child->is_stable() ||
-	  (child->is_mutation_pending() &&
-	   child->is_pending_io());
+	return child->is_stable();
       } else {
 	return true;
       }
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index c73839cf1fe8..6ab19282637f 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -416,13 +416,20 @@ class CachedExtent
     return is_mutable() || state == extent_state_t::EXIST_CLEAN;
   }
 
-  /// Returns true if extent is stable and shared among transactions
-  bool is_stable() const {
+  /// Returns true if extent is stable, written and shared among transactions
+  bool is_stable_written() const {
     return state == extent_state_t::CLEAN_PENDING ||
       state == extent_state_t::CLEAN ||
       state == extent_state_t::DIRTY;
   }
 
+  /// Returns true if extent is stable and shared among transactions
+  bool is_stable() const {
+    return is_stable_written() ||
+           (is_mutation_pending() &&
+            is_pending_io());
+  }
+
   /// Returns true if extent has a pending delta
   bool is_mutation_pending() const {
     return state == extent_state_t::MUTATION_PENDING;
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index 54bd27b8c181..dfe599177056 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -1077,7 +1077,7 @@ struct transaction_manager_test_t :
 	test_mappings.alloced(pin->get_key(), *extent, t.mapping_delta);
 	EXPECT_TRUE(extent->is_exist_clean());
       } else {
-	EXPECT_TRUE(extent->is_stable());
+	EXPECT_TRUE(extent->is_stable_written());
       }
     } else {
       ceph_assert(t.t->is_conflicted());

From 0d953d27cb6503dfde7f4f7ecb9a386c533c30c9 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 6 Nov 2023 14:31:20 +1000
Subject: [PATCH 0449/2492] doc/rados: format Q&A list in tshooting-mon.rst

Bold the questions in a Q&A list in the "Most Common Monitor Issues"
section of doc/rados/troubleshooting/troubleshooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 4ccdbc4ef99e..64db2462ae12 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -190,7 +190,7 @@ you should be seeing something similar to::
       [snip]
       mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum)
 
-How to troubleshoot this?
+**How to troubleshoot this?**
 
   First, make sure ``mon.a`` is running.
 
@@ -213,7 +213,7 @@ How to troubleshoot this?
   troubleshooting the monitor, so check you ``ceph status`` again just to make
   sure. Proceed if the monitor is not yet in the quorum.
 
-What if the state is ``probing``?
+**What if the state is ``probing``?**
 
   This means the monitor is still looking for the other monitors. Every time
   you start a monitor, the monitor will stay in this state for some time while
@@ -240,7 +240,7 @@ What if the state is ``probing``?
   to `Preparing your logs`_ on how to best prepare your logs).
 
 
-What if state is ``electing``?
+**What if state is ``electing``?**
 
   This means the monitor is in the middle of an election. With recent Ceph
   releases these typically complete quickly, but at times the monitors can
@@ -253,7 +253,7 @@ What if state is ``electing``?
   why this would happen.  Worst case, if there are enough surviving mons,
   down the problematic one while you investigate.
 
-What if state is ``synchronizing``?
+**What if state is ``synchronizing``?**
 
   This means the monitor is catching up with the rest of the cluster in
   order to join the quorum. Time to synchronize is a function of the size
@@ -269,7 +269,7 @@ What if state is ``synchronizing``?
   this in later versions please let us know via a bug tracker. And bring some logs
   (see `Preparing your logs`_).
 
-What if state is ``leader`` or ``peon``?
+**What if state is ``leader`` or ``peon``?**
 
   This should not happen:  famous last words.  If it does, however, it likely
   has a lot to do with clock skew -- see `Clock Skews`_. If you are not

From 9c026fa18cf9390401e930e30d4dbf85d35340c3 Mon Sep 17 00:00:00 2001
From: Nobuto Murata <nobuto.murata@canonical.com>
Date: Thu, 2 Nov 2023 13:33:07 +0900
Subject: [PATCH 0450/2492] mgr/dashboard: info on why RBD graphs are empty

Those RBD IO statistics graphs are empty out of the box and it's on
purpose. Instead of giving an impression that those graphs are broken,
point users to a documentation explaining about optional steps to enable
those statistics.
https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics

Signed-off-by: Nobuto Murata <nobuto.murata@canonical.com>
---
 .../ceph-mixin/dashboards/rbd.libsonnet       | 29 +++++++++++++++----
 .../dashboards_out/rbd-details.json           |  6 ++--
 .../dashboards_out/rbd-overview.json          | 12 ++++----
 3 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards/rbd.libsonnet b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
index 709d4e04f7e9..8ae9d272e50d 100644
--- a/monitoring/ceph-mixin/dashboards/rbd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
@@ -1,12 +1,22 @@
 local g = import 'grafonnet/grafana.libsonnet';
 local u = import 'utils.libsonnet';
 
+local info_rbd_stats = std.join(
+  '',
+  [
+    'RBD per-image IO statistics are disabled by default.\n\n',
+    'Please refer to ',
+    'https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics ',
+    'for information about how to enable those optionally.',
+  ]
+);
+
 (import 'utils.libsonnet') {
   'rbd-details.json':
-    local RbdDetailsPanel(title, formatY1, expr1, expr2, x, y, w, h) =
+    local RbdDetailsPanel(title, description, formatY1, expr1, expr2, x, y, w, h) =
       $.graphPanelSchema({},
                          title,
-                         '',
+                         description,
                          'null as zero',
                          false,
                          formatY1,
@@ -83,6 +93,7 @@ local u = import 'utils.libsonnet';
     .addPanels([
       RbdDetailsPanel(
         'IOPS',
+        info_rbd_stats,
         'iops',
         'rate(ceph_rbd_write_ops{%(matchers)s, pool="$pool", image="$image"}[$__rate_interval])' % $.matchers()
         ,
@@ -94,6 +105,7 @@ local u = import 'utils.libsonnet';
       ),
       RbdDetailsPanel(
         'Throughput',
+        info_rbd_stats,
         'Bps',
         'rate(ceph_rbd_write_bytes{%(matchers)s, pool="$pool", image="$image"}[$__rate_interval])' % $.matchers(),
         'rate(ceph_rbd_read_bytes{%(matchers)s, pool="$pool", image="$image"}[$__rate_interval])' % $.matchers(),
@@ -104,6 +116,7 @@ local u = import 'utils.libsonnet';
       ),
       RbdDetailsPanel(
         'Average Latency',
+        info_rbd_stats,
         'ns',
         |||
           rate(ceph_rbd_write_latency_sum{%(matchers)s, pool="$pool", image="$image"}[$__rate_interval]) /
@@ -121,6 +134,7 @@ local u = import 'utils.libsonnet';
     ]),
   'rbd-overview.json':
     local RbdOverviewPanel(title,
+                           description,
                            formatY1,
                            expr1,
                            expr2,
@@ -132,7 +146,7 @@ local u = import 'utils.libsonnet';
                            h) =
       $.graphPanelSchema({},
                          title,
-                         '',
+                         description,
                          'null as zero',
                          false,
                          formatY1,
@@ -196,6 +210,7 @@ local u = import 'utils.libsonnet';
     .addPanels([
       RbdOverviewPanel(
         'IOPS',
+        info_rbd_stats,
         'short',
         'round(sum(rate(ceph_rbd_write_ops{%(matchers)s}[$__rate_interval])))' % $.matchers(),
         'round(sum(rate(ceph_rbd_read_ops{%(matchers)s}[$__rate_interval])))' % $.matchers(),
@@ -208,6 +223,7 @@ local u = import 'utils.libsonnet';
       ),
       RbdOverviewPanel(
         'Throughput',
+        info_rbd_stats,
         'Bps',
         'round(sum(rate(ceph_rbd_write_bytes{%(matchers)s}[$__rate_interval])))' % $.matchers(),
         'round(sum(rate(ceph_rbd_read_bytes{%(matchers)s}[$__rate_interval])))' % $.matchers(),
@@ -220,6 +236,7 @@ local u = import 'utils.libsonnet';
       ),
       RbdOverviewPanel(
         'Average Latency',
+        info_rbd_stats,
         'ns',
         |||
           round(
@@ -242,7 +259,7 @@ local u = import 'utils.libsonnet';
       ),
       $.addTableSchema(
         '$datasource',
-        '',
+        info_rbd_stats,
         { col: 3, desc: true },
         [
           $.overviewStyle('Pool', 'pool', 'string', 'short'),
@@ -273,7 +290,7 @@ local u = import 'utils.libsonnet';
       ) + { gridPos: { x: 0, y: 7, w: 8, h: 7 } },
       $.addTableSchema(
         '$datasource',
-        '',
+        info_rbd_stats,
         { col: 3, desc: true },
         [
           $.overviewStyle('Pool', 'pool', 'string', 'short'),
@@ -304,7 +321,7 @@ local u = import 'utils.libsonnet';
       ) + { gridPos: { x: 8, y: 7, w: 8, h: 7 } },
       $.addTableSchema(
         '$datasource',
-        '',
+        info_rbd_stats,
         { col: 3, desc: true },
         [
           $.overviewStyle('Pool', 'pool', 'string', 'short'),
diff --git a/monitoring/ceph-mixin/dashboards_out/rbd-details.json b/monitoring/ceph-mixin/dashboards_out/rbd-details.json
index f64de312af52..2d5d4ecac4a5 100644
--- a/monitoring/ceph-mixin/dashboards_out/rbd-details.json
+++ b/monitoring/ceph-mixin/dashboards_out/rbd-details.json
@@ -43,7 +43,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -136,7 +136,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -229,7 +229,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
diff --git a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
index 86b354a6089c..c367bd16fa38 100644
--- a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
@@ -55,7 +55,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -148,7 +148,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -241,7 +241,7 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -331,7 +331,7 @@
       {
          "columns": [ ],
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "gridPos": {
             "h": 7,
             "w": 8,
@@ -433,7 +433,7 @@
       {
          "columns": [ ],
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "gridPos": {
             "h": 7,
             "w": 8,
@@ -535,7 +535,7 @@
       {
          "columns": [ ],
          "datasource": "$datasource",
-         "description": "",
+         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
          "gridPos": {
             "h": 7,
             "w": 8,

From f4caa0e11e311df990fe788b1b338eec81764c2d Mon Sep 17 00:00:00 2001
From: Ali Masarwa <ali.saed.masarwa@gmail.com>
Date: Thu, 12 Oct 2023 14:58:49 +0300
Subject: [PATCH 0451/2492] RGW: perf counters for topic stats

Signed-off-by: Ali Masarwa <ali.saed.masarwa@gmail.com>
---
 src/rgw/driver/rados/rgw_notify.cc | 16 ++++++++++++++-
 src/rgw/rgw_perf_counters.cc       | 33 ++++++++++++++++++++++++++++++
 src/rgw/rgw_perf_counters.h        | 26 +++++++++++++++++++++++
 3 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 6e5eeeb2fabb..7abb0d84a7d1 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -75,6 +75,7 @@ struct persistency_tracker {
 using queues_t = std::set<std::string>;
 using entries_persistency_tracker = ceph::unordered_map<std::string, persistency_tracker>;
 using queues_persistency_tracker = ceph::unordered_map<std::string, entries_persistency_tracker>;
+using rgw::persistent_topic_counters::CountersManager;
 
 // use mmap/mprotect to allocate 128k coroutine stacks
 auto make_stack_allocator() {
@@ -313,7 +314,9 @@ class Manager : public DoutPrefixProvider {
     spawn::spawn(io_context, [this, queue_name](yield_context yield) {
             cleanup_queue(queue_name, yield);
             }, make_stack_allocator());
-    
+
+    CountersManager queue_counters_container(queue_name, this->get_cct());
+
     while (true) {
       // if queue was empty the last time, sleep for idle timeout
       if (is_idle) {
@@ -521,6 +524,17 @@ class Manager : public DoutPrefixProvider {
           }
         }
       }
+
+      // updating perfcounters with topic stats
+      uint64_t entries_size;
+      uint32_t entries_number;
+      const auto ret = cls_2pc_queue_get_topic_stats(rados_ioctx, queue_name, entries_number, entries_size);
+      if (ret < 0) {
+        ldpp_dout(this, 1) << "ERROR: topic stats for topic: " << queue_name << ". error: " << ret << dendl;
+      } else {
+        queue_counters_container.set(l_rgw_persistent_topic_len, entries_number);
+        queue_counters_container.set(l_rgw_persistent_topic_size, entries_size);
+      }
     }
   }
 
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index c3e89211cca6..8cbda840c5ad 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -9,6 +9,7 @@
 
 using namespace ceph::perf_counters;
 using namespace rgw::op_counters;
+using namespace rgw::persistent_topic_counters;
 
 PerfCounters *perfcounter = NULL;
 
@@ -90,6 +91,14 @@ void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
   lpcb->add_time_avg(l_rgw_op_list_buckets_lat, "list_buckets_lat", "List buckets latency");
 }
 
+void add_rgw_topic_counters(PerfCountersBuilder *lpcb) {
+  lpcb->set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+
+  lpcb->add_u64(l_rgw_persistent_topic_len, "persistent_topic_len", "Persistent topic queue length");
+  lpcb->add_u64(l_rgw_persistent_topic_size, "persistent_topic_size", "Persistent topic queue size");
+
+}
+
 void frontend_counters_init(CephContext *cct) {
   PerfCountersBuilder pcb(cct, "rgw", l_rgw_first, l_rgw_last);
   add_rgw_frontend_counters(&pcb);
@@ -192,6 +201,30 @@ void tinc(const CountersContainer &counters, int idx, ceph::timespan amt) {
 
 } // namespace rgw::op_counters
 
+namespace rgw::persistent_topic_counters {
+
+const std::string rgw_topic_counters_key = "rgw_topic";
+
+CountersManager::CountersManager(const std::string& topic_name, CephContext *cct)
+    : cct(cct)
+{
+  const std::string topic_key = ceph::perf_counters::key_create(rgw_topic_counters_key, {{"Topic", topic_name}});
+  PerfCountersBuilder pcb(cct, topic_key, l_rgw_topic_first, l_rgw_topic_last);
+  add_rgw_topic_counters(&pcb);
+  topic_counters = std::unique_ptr<PerfCounters>(pcb.create_perf_counters());
+  cct->get_perfcounters_collection()->add(topic_counters.get());
+}
+
+void CountersManager::set(int idx, uint64_t v) {
+  topic_counters->set(idx, v);
+}
+
+CountersManager::~CountersManager() {
+  cct->get_perfcounters_collection()->remove(topic_counters.get());
+}
+
+} // namespace rgw::persistent_topic_counters
+
 int rgw_perf_start(CephContext *cct)
 {
   frontend_counters_init(cct);
diff --git a/src/rgw/rgw_perf_counters.h b/src/rgw/rgw_perf_counters.h
index 5abd7e44b13d..7eb11c926c17 100644
--- a/src/rgw/rgw_perf_counters.h
+++ b/src/rgw/rgw_perf_counters.h
@@ -83,6 +83,15 @@ enum {
   l_rgw_op_last
 };
 
+enum {
+  l_rgw_topic_first = 17000,
+
+  l_rgw_persistent_topic_len,
+  l_rgw_persistent_topic_size,
+
+  l_rgw_topic_last
+};
+
 namespace rgw::op_counters {
 
 struct CountersContainer {
@@ -99,3 +108,20 @@ void tinc(const CountersContainer &counters, int idx, utime_t);
 void tinc(const CountersContainer &counters, int idx, ceph::timespan amt);
 
 } // namespace rgw::op_counters
+
+namespace rgw::persistent_topic_counters {
+
+class CountersManager {
+  std::unique_ptr<PerfCounters> topic_counters;
+  CephContext *cct;
+
+public:
+  CountersManager(const std::string& name, CephContext *cct);
+
+  void set(int idx, uint64_t v);
+
+  ~CountersManager();
+
+};
+
+} // namespace rgw::persistent_topic_counters

From 83ffaca69e531bfa0da1b1d34d2013bd6b053ed7 Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 6 Nov 2023 14:27:59 +0530
Subject: [PATCH 0452/2492] librbd: additional logs for debugging

Added image and object information to the deep_copy class constructors
debugging easier.

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 src/librbd/deep_copy/ImageCopyRequest.cc      |  7 +++++++
 src/librbd/deep_copy/MetadataCopyRequest.cc   |  4 ++++
 src/librbd/deep_copy/ObjectCopyRequest.cc     | 10 ++++++----
 src/librbd/deep_copy/SetHeadRequest.cc        |  6 ++++++
 src/librbd/deep_copy/SnapshotCopyRequest.cc   |  9 +++++++++
 src/librbd/deep_copy/SnapshotCreateRequest.cc |  8 ++++++++
 6 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/src/librbd/deep_copy/ImageCopyRequest.cc b/src/librbd/deep_copy/ImageCopyRequest.cc
index 08e959dd5723..9cce590ec183 100644
--- a/src/librbd/deep_copy/ImageCopyRequest.cc
+++ b/src/librbd/deep_copy/ImageCopyRequest.cc
@@ -39,6 +39,13 @@ ImageCopyRequest<I>::ImageCopyRequest(I *src_image_ctx, I *dst_image_ctx,
     m_flatten(flatten), m_object_number(object_number), m_snap_seqs(snap_seqs),
     m_handler(handler), m_on_finish(on_finish), m_cct(dst_image_ctx->cct),
     m_lock(ceph::make_mutex(unique_lock_name("ImageCopyRequest::m_lock", this))) {
+
+    ldout(m_cct, 20) << "src_image_id=" << m_src_image_ctx->id
+		     << ", dst_image_id=" << m_dst_image_ctx->id
+	             << ", src_snap_id_start=" << m_src_snap_id_start
+                     << ", src_snap_id_end=" << m_src_snap_id_end
+		     << ", dst_snap_id_start=" << m_dst_snap_id_start
+		     << dendl;
 }
 
 template <typename I>
diff --git a/src/librbd/deep_copy/MetadataCopyRequest.cc b/src/librbd/deep_copy/MetadataCopyRequest.cc
index c584bea54b80..d696ea5ccc8a 100644
--- a/src/librbd/deep_copy/MetadataCopyRequest.cc
+++ b/src/librbd/deep_copy/MetadataCopyRequest.cc
@@ -30,6 +30,10 @@ MetadataCopyRequest<I>::MetadataCopyRequest(I *src_image_ctx, I *dst_image_ctx,
                                             Context *on_finish)
   : m_src_image_ctx(src_image_ctx), m_dst_image_ctx(dst_image_ctx),
     m_on_finish(on_finish), m_cct(dst_image_ctx->cct) {
+
+  ldout(m_cct, 20) << "src_image_id=" << m_src_image_ctx->id
+                   << ", dst_image_id=" << m_dst_image_ctx->id
+		   << dendl;
 }
 
 template <typename I>
diff --git a/src/librbd/deep_copy/ObjectCopyRequest.cc b/src/librbd/deep_copy/ObjectCopyRequest.cc
index 4c380c642c15..3c1b9460140a 100644
--- a/src/librbd/deep_copy/ObjectCopyRequest.cc
+++ b/src/librbd/deep_copy/ObjectCopyRequest.cc
@@ -58,10 +58,12 @@ ObjectCopyRequest<I>::ObjectCopyRequest(I *src_image_ctx,
 
   m_dst_oid = m_dst_image_ctx->get_object_name(dst_object_number);
 
-  ldout(m_cct, 20) << "dst_oid=" << m_dst_oid << ", "
-                   << "src_snap_id_start=" << m_src_snap_id_start << ", "
-                   << "dst_snap_id_start=" << m_dst_snap_id_start << ", "
-                   << "snap_map=" << m_snap_map << dendl;
+  ldout(m_cct, 20) << "src_image_id=" << m_src_image_ctx->id
+		   << ", dst_image_id=" << m_dst_image_ctx->id
+	           << ", dst_oid=" << m_dst_oid
+		   << ", src_snap_id_start=" << m_src_snap_id_start
+		   << ", dst_snap_id_start=" << m_dst_snap_id_start
+                   << ", snap_map=" << m_snap_map << dendl;
 }
 
 template <typename I>
diff --git a/src/librbd/deep_copy/SetHeadRequest.cc b/src/librbd/deep_copy/SetHeadRequest.cc
index 1e056b9580cb..a82b93592dea 100644
--- a/src/librbd/deep_copy/SetHeadRequest.cc
+++ b/src/librbd/deep_copy/SetHeadRequest.cc
@@ -30,6 +30,12 @@ SetHeadRequest<I>::SetHeadRequest(I *image_ctx, uint64_t size,
     m_parent_overlap(parent_overlap), m_on_finish(on_finish),
     m_cct(image_ctx->cct) {
   ceph_assert(m_parent_overlap <= m_size);
+
+  ldout(m_cct, 20) << "image_id=" << m_image_ctx->id
+                   << ", size=" << m_size
+                   << ", parent_spec=" << m_parent_spec
+                   << ", parent_overlap=" << m_parent_overlap
+                   << dendl;
 }
 
 template <typename I>
diff --git a/src/librbd/deep_copy/SnapshotCopyRequest.cc b/src/librbd/deep_copy/SnapshotCopyRequest.cc
index 1aadd34db36c..5b7f3b7a83ef 100644
--- a/src/librbd/deep_copy/SnapshotCopyRequest.cc
+++ b/src/librbd/deep_copy/SnapshotCopyRequest.cc
@@ -76,6 +76,15 @@ SnapshotCopyRequest<I>::SnapshotCopyRequest(I *src_image_ctx,
     m_src_snap_ids.erase(m_src_snap_ids.upper_bound(m_src_snap_id_end),
                          m_src_snap_ids.end());
   }
+
+  ldout(m_cct, 20) << "src_image_id=" << m_src_image_ctx->id
+                   << ", dst_image_id=" << m_dst_image_ctx->id
+                   << ", src_snap_id_start=" << m_src_snap_id_start
+                   << ", src_snap_id_end=" << m_src_snap_id_end
+                   << ", dst_snap_id_start=" << m_dst_snap_id_start
+                   << ", src_snap_ids=" << m_src_snap_ids
+                   << ", dst_snap_ids=" << m_dst_snap_ids
+		   << dendl;
 }
 
 template <typename I>
diff --git a/src/librbd/deep_copy/SnapshotCreateRequest.cc b/src/librbd/deep_copy/SnapshotCreateRequest.cc
index d437bd3552fd..394aee287251 100644
--- a/src/librbd/deep_copy/SnapshotCreateRequest.cc
+++ b/src/librbd/deep_copy/SnapshotCreateRequest.cc
@@ -33,6 +33,14 @@ SnapshotCreateRequest<I>::SnapshotCreateRequest(
     m_snap_namespace(snap_namespace), m_size(size),
     m_parent_spec(spec), m_parent_overlap(parent_overlap),
     m_on_finish(on_finish), m_cct(dst_image_ctx->cct) {
+
+  ldout(m_cct, 20) << "dst_image_id=" << m_dst_image_ctx->id
+                   << ", snap_name=" << m_snap_name
+                   << ", snap_namespace=" << m_snap_namespace
+                   << ", size=" << m_size
+                   << ", parent_spec=" << m_parent_spec
+                   << ", parent_overlap=" << m_parent_overlap
+		   << dendl;
 }
 
 template <typename I>

From 48f9a6d0042d1867af55c8395ecf9d17867a8892 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 6 Nov 2023 03:33:22 -0600
Subject: [PATCH 0453/2492] osd/scrub: modify scrub_requested() to return the
 deduced scrub level

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                   |  3 ++-
 src/osd/scrubber/pg_scrubber.cc | 30 +++++++++++++++++-------------
 src/osd/scrubber/pg_scrubber.h  |  2 +-
 src/osd/scrubber_common.h       |  7 ++++---
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 580b3add8484..d2f97a129a24 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1708,7 +1708,8 @@ void PG::on_scrub_schedule_input_change()
 void PG::scrub_requested(scrub_level_t scrub_level, scrub_type_t scrub_type)
 {
   ceph_assert(m_scrubber);
-  m_scrubber->scrub_requested(scrub_level, scrub_type, m_planned_scrub);
+  std::ignore =
+      m_scrubber->scrub_requested(scrub_level, scrub_type, m_planned_scrub);
 }
 
 void PG::clear_ready_to_merge() {
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 5afc2ced2725..0c9b0585088e 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -552,29 +552,33 @@ void PgScrubber::update_scrub_job(const requested_scrub_t& request_flags)
   dout(15) << __func__ << ": done " << registration_state() << dendl;
 }
 
-void PgScrubber::scrub_requested(scrub_level_t scrub_level,
-				 scrub_type_t scrub_type,
-				 requested_scrub_t& req_flags)
+scrub_level_t PgScrubber::scrub_requested(
+    scrub_level_t scrub_level,
+    scrub_type_t scrub_type,
+    requested_scrub_t& req_flags)
 {
-  dout(10) << __func__
-	   << (scrub_level == scrub_level_t::deep ? " deep " : " shallow ")
-	   << (scrub_type == scrub_type_t::do_repair ? " repair-scrub "
-						     : " not-repair ")
-	   << " prev stamp: " << m_scrub_job->get_sched_time()
-	   << " registered? " << registration_state() << dendl;
+  const bool deep_requested = (scrub_level == scrub_level_t::deep) ||
+			      (scrub_type == scrub_type_t::do_repair);
+  dout(10) << fmt::format(
+		  "{}: {} {} scrub requested. Prev stamp: {}. Registered? {}",
+		  __func__,
+		  (scrub_type == scrub_type_t::do_repair ? " repair + "
+							 : " not-repair + "),
+		  (deep_requested ? "deep" : "shallow"),
+		  m_scrub_job->get_sched_time(), registration_state())
+	   << dendl;
 
   req_flags.must_scrub = true;
-  req_flags.must_deep_scrub = (scrub_level == scrub_level_t::deep) ||
-			      (scrub_type == scrub_type_t::do_repair);
+  req_flags.must_deep_scrub = deep_requested;
   req_flags.must_repair = (scrub_type == scrub_type_t::do_repair);
   // User might intervene, so clear this
   req_flags.need_auto = false;
   req_flags.req_scrub = true;
 
-  dout(20) << __func__ << " pg(" << m_pg_id << ") planned:" << req_flags
-	   << dendl;
+  dout(20) << fmt::format("{}: planned scrub:{}", __func__, req_flags) << dendl;
 
   update_scrub_job(req_flags);
+  return deep_requested ? scrub_level_t::deep : scrub_level_t::shallow;
 }
 
 
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 97bf7da8f22a..ab82e2b1714c 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -274,7 +274,7 @@ class PgScrubber : public ScrubPgIF,
 
   void on_pg_activate(const requested_scrub_t& request_flags) final;
 
-  void scrub_requested(
+  scrub_level_t scrub_requested(
       scrub_level_t scrub_level,
       scrub_type_t scrub_type,
       requested_scrub_t& req_flags) final;
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 745ea2388b67..f717d41fa41f 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -402,9 +402,10 @@ struct ScrubPgIF {
 
   virtual void rm_from_osd_scrubbing() = 0;
 
-  virtual void scrub_requested(scrub_level_t scrub_level,
-			       scrub_type_t scrub_type,
-			       requested_scrub_t& req_flags) = 0;
+  virtual scrub_level_t scrub_requested(
+      scrub_level_t scrub_level,
+      scrub_type_t scrub_type,
+      requested_scrub_t& req_flags) = 0;
 
   // --------------- debugging via the asok ------------------------------
 

From 455f835c737a4b5268b32135ed0b7c26da9fc1ee Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 6 Nov 2023 03:40:27 -0600
Subject: [PATCH 0454/2492] osd/scrub: introduce populate_config_params()

to compute the scrub-related configuration parameters from both
cluster and pool options.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc | 51 +++++++++++++++++++++++++
 src/osd/scrubber/pg_scrubber.h  |  6 +++
 src/osd/scrubber/scrub_job.h    | 68 +++++++++++++++++++++++++++++++++
 3 files changed, 125 insertions(+)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 0c9b0585088e..babd8782e9ca 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -606,6 +606,57 @@ bool PgScrubber::reserve_local()
   return false;
 }
 
+Scrub::sched_conf_t PgScrubber::populate_config_params() const
+{
+  const pool_opts_t& pool_conf = m_pg->get_pgpool().info.opts;
+  auto& conf = get_pg_cct()->_conf;  // for brevity
+  Scrub::sched_conf_t configs;
+
+  // deep-scrub optimal interval
+  configs.deep_interval =
+      pool_conf.value_or(pool_opts_t::DEEP_SCRUB_INTERVAL, 0.0);
+  if (configs.deep_interval <= 0.0) {
+    configs.deep_interval = conf->osd_deep_scrub_interval;
+  }
+
+  // shallow-scrub interval
+  configs.shallow_interval =
+      pool_conf.value_or(pool_opts_t::SCRUB_MIN_INTERVAL, 0.0);
+  if (configs.shallow_interval <= 0.0) {
+    configs.shallow_interval = conf->osd_scrub_min_interval;
+  }
+
+  // the max allowed delay between scrubs.
+  // For deep scrubs - there is no equivalent of scrub_max_interval. Per the
+  // documentation, once deep_scrub_interval has passed, we are already
+  // "overdue", at least as far as the "ignore allowed load" window is
+  // concerned.
+
+  configs.max_deep = configs.deep_interval + configs.shallow_interval;
+
+  auto max_shallow = pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0);
+  if (max_shallow <= 0.0) {
+    max_shallow = conf->osd_scrub_max_interval;
+  }
+  if (max_shallow > 0.0) {
+    configs.max_shallow = max_shallow;
+    // otherwise - we're left with the default nullopt
+  }
+
+  // but seems like our tests require: \todo fix!
+  configs.max_deep =
+      std::max(configs.max_shallow.value_or(0.0), configs.deep_interval);
+
+  configs.interval_randomize_ratio = conf->osd_scrub_interval_randomize_ratio;
+  configs.deep_randomize_ratio = conf->osd_deep_scrub_randomize_ratio;
+  configs.mandatory_on_invalid = conf->osd_scrub_invalid_stats;
+
+  dout(15) << fmt::format("updated config:{}", configs) << dendl;
+  return configs;
+}
+
+
+
 // ----------------------------------------------------------------------------
 
 bool PgScrubber::has_pg_marked_new_updates() const
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index ab82e2b1714c..ca26e359578c 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -769,6 +769,12 @@ class PgScrubber : public ScrubPgIF,
    */
   void request_rescrubbing(requested_scrub_t& req_flags);
 
+  /**
+   * combine cluster & pool configuration options into a single struct
+   * of scrub-related parameters.
+   */
+  Scrub::sched_conf_t populate_config_params() const;
+
   /*
    * Select a range of objects to scrub.
    *
diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index c2391a788929..57ff1400f9cb 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -142,6 +142,59 @@ class ScrubJob final : public RefCountedObject {
 
 using ScrubJobRef = ceph::ref_t<ScrubJob>;
 using ScrubQContainer = std::vector<ScrubJobRef>;
+
+/**
+ *  A collection of the configuration parameters (pool & OSD) that affect
+ *  scrub scheduling.
+ */
+struct sched_conf_t {
+  /// the desired interval between shallow scrubs
+  double shallow_interval{0.0};
+
+  /// the desired interval between deep scrubs
+  double deep_interval{0.0};
+
+  /**
+   * the maximum interval between shallow scrubs, as determined by either the
+   * OSD or the pool configuration. Empty if no limit is configured.
+   */
+  std::optional<double> max_shallow;
+
+  /**
+   * the maximum interval between deep scrubs.
+   * For deep scrubs - there is no equivalent of scrub_max_interval. Per the
+   * documentation, once deep_scrub_interval has passed, we are already
+   * "overdue", at least as far as the "ignore allowed load" window is
+   * concerned. \todo based on users complaints (and the fact that the
+   * interaction between the configuration parameters is clear to no one),
+   * this will be revised shortly.
+   */
+  double max_deep{0.0};
+
+  /**
+   * interval_randomize_ratio
+   *
+   * We add an extra random duration to the configured times when doing
+   * scheduling. An event configured with an interval of <interval> will
+   * actually be scheduled at a time selected uniformly from
+   * [<interval>, (1+<interval_randomize_ratio>) * <interval>)
+   */
+  double interval_randomize_ratio{0.0};
+
+  /**
+   * a randomization factor aimed at preventing 'thundering herd' problems
+   * upon deep-scrubs common intervals. If polling a random number smaller
+   * than that percentage, the next shallow scrub is upgraded to deep.
+   */
+  double deep_randomize_ratio{0.0};
+
+  /**
+   * must we schedule a scrub with high urgency if we do not have a valid
+   * last scrub stamp?
+   */
+  bool mandatory_on_invalid{true};
+};
+
 }  // namespace Scrub
 
 namespace std {
@@ -178,4 +231,19 @@ struct formatter<Scrub::ScrubJob> {
 	sjob.state.load(std::memory_order_relaxed));
   }
 };
+
+template <>
+struct formatter<Scrub::sched_conf_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+  template <typename FormatContext>
+  auto format(const Scrub::sched_conf_t& cf, FormatContext& ctx)
+  {
+    return format_to(
+	ctx.out(),
+	"periods: s:{}/{} d:{}/{} iv-ratio:{} deep-rand:{} on-inv:{}",
+	cf.shallow_interval, cf.max_shallow.value_or(-1.0), cf.deep_interval,
+	cf.max_deep, cf.interval_randomize_ratio, cf.deep_randomize_ratio,
+	cf.mandatory_on_invalid);
+  }
+};
 }  // namespace fmt

From 5e60c0f4ddd6084402652007a7ad0a7ac14260d3 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 6 Nov 2023 04:03:08 -0600
Subject: [PATCH 0455/2492] osd/scrub: minor tidy-ups

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.h                   | 2 +-
 src/osd/PrimaryLogPG.cc        | 2 +-
 src/osd/PrimaryLogPG.h         | 2 +-
 src/osd/scrubber/pg_scrubber.h | 4 ++--
 src/osd/scrubber_common.h      | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 70c1d12b2105..2e82e74ab012 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -765,7 +765,7 @@ class PG : public DoutPrefixProvider,
 
   virtual void snap_trimmer(epoch_t epoch_queued) = 0;
   virtual void do_command(
-    const std::string_view& prefix,
+    std::string_view prefix,
     const cmdmap_t& cmdmap,
     const ceph::buffer::list& idata,
     std::function<void(int,const std::string&,ceph::buffer::list&)> on_finish) = 0;
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 656b7ab4b0ac..ba13b9afa9b8 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -999,7 +999,7 @@ PrimaryLogPG::get_pgls_filter(bufferlist::const_iterator& iter)
 // ==========================================================
 
 void PrimaryLogPG::do_command(
-  const string_view& orig_prefix,
+  string_view orig_prefix,
   const cmdmap_t& cmdmap,
   const bufferlist& idata,
   std::function<void(int,const std::string&,bufferlist&)> on_finish)
diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 334ecb0d4180..6ed29927463d 100644
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -1478,7 +1478,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   ~PrimaryLogPG() override;
 
   void do_command(
-    const std::string_view& prefix,
+    std::string_view prefix,
     const cmdmap_t& cmdmap,
     const ceph::buffer::list& idata,
     std::function<void(int,const std::string&,ceph::buffer::list&)> on_finish) override;
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index ca26e359578c..996b26781be1 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -20,7 +20,7 @@ Main Scrubber interfaces:
 └────────────────────────────────┬──────────────────┘
                                  │
                                  │
-                                 │ ownes & uses
+                                 │ owns & uses
                                  │
                                  │
                                  │
@@ -43,7 +43,7 @@ Main Scrubber interfaces:
 │         PrimaryLogScrub                           │       │
 └─────┬───────────────────┬─────────────────────────┘       │
       │                   │                         implements
-      │    ownes & uses   │                                 │
+      │    owns & uses    │                                 │
       │                   │       ┌─────────────────────────▼──────┐
       │                   │       │    <<ScrubMachineListener>>    │
       │                   │       └─────────▲──────────────────────┘
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index f717d41fa41f..52890a8d1f04 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -92,7 +92,7 @@ struct PgScrubBeListener {
   // query the PG backend for the on-disk size of an object
   virtual uint64_t logical_to_ondisk_size(uint64_t logical_size) const = 0;
 
-  // used to verify our "cleaness" before scrubbing
+  // used to verify our "cleanliness" before scrubbing
   virtual bool is_waiting_for_unreadable_object() const = 0;
 };
 

From 1a13e7407a043b151d02d6d65ad37d87788fa3a5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 3 Nov 2023 14:44:38 -0400
Subject: [PATCH 0456/2492] pybind/mgr/devicehealth: replace SMART data if
 exists for same DATETIME

Where DATETIME means "seconds since epoch".

Fixes: https://tracker.ceph.com/issues/63433
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/pybind/mgr/devicehealth/module.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index 07768db75656..86ea1d280720 100644
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -502,8 +502,8 @@ def _create_device(self, devid: str) -> None:
 
     def put_device_metrics(self, devid: str, data: Any) -> None:
         SQL = """
-        INSERT INTO DeviceHealthMetrics (devid, raw_smart)
-            VALUES (?, ?);
+        INSERT OR REPLACE INTO DeviceHealthMetrics (devid, raw_smart, time)
+            VALUES (?, ?, strftime('%s', 'now'));
         """
 
         with self._db_lock, self.db:

From e227c39547e2bbcc213fb1fa865cdbfc377edaa6 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Fri, 29 Sep 2023 19:03:18 +0530
Subject: [PATCH 0457/2492] qa/upgrade/upgraded_client: upgrade
 nautilus->pacific and pacific->reef

Fixes: https://tracker.ceph.com/issues/62953
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 .../upgraded_client/branch/nautilus.yaml      |  2 +
 .../upgraded_client/branch/pacific.yaml       |  2 +
 .../upgraded_client/tasks/0-from/pacific.yaml | 53 ----------------
 .../{0-from/nautilus.yaml => 0-install.yaml}  | 19 +++---
 .../upgraded_client/tasks/2-upgrade.yaml      | 63 +++++++++++++++++++
 .../stress_tests/0-client-upgrade.yaml        | 14 -----
 .../tasks/{2-workload => 3-workload}/.qa      |  0
 .../{2-workload => 3-workload}/new_ops/%      |  0
 .../new_ops/0-clients/fuse-upgrade.yaml       |  0
 .../new_ops/0-clients/kclient.yaml            |  0
 .../new_ops/1-client-sanity.yaml              |  0
 .../{2-workload => 3-workload}/stress_tests/% |  0
 .../stress_tests/.qa                          |  0
 .../stress_tests/0-client-upgrade.yaml        | 24 +++++++
 .../stress_tests/1-tests/.qa                  |  0
 .../stress_tests/1-tests/blogbench.yaml       |  0
 .../stress_tests/1-tests/dbench.yaml          |  0
 .../stress_tests/1-tests/fsstress.yaml        |  0
 .../stress_tests/1-tests/iozone.yaml          |  0
 .../1-tests/kernel_untar_build.yaml           |  0
 20 files changed, 103 insertions(+), 74 deletions(-)
 create mode 100644 qa/suites/fs/upgrade/upgraded_client/branch/nautilus.yaml
 create mode 100644 qa/suites/fs/upgrade/upgraded_client/branch/pacific.yaml
 delete mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/0-from/pacific.yaml
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{0-from/nautilus.yaml => 0-install.yaml} (64%)
 create mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
 delete mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/0-client-upgrade.yaml
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/.qa (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/new_ops/% (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/new_ops/0-clients/fuse-upgrade.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/new_ops/0-clients/kclient.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/new_ops/1-client-sanity.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/% (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/.qa (100%)
 create mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/.qa (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/blogbench.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/dbench.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/fsstress.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/iozone.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{2-workload => 3-workload}/stress_tests/1-tests/kernel_untar_build.yaml (100%)

diff --git a/qa/suites/fs/upgrade/upgraded_client/branch/nautilus.yaml b/qa/suites/fs/upgrade/upgraded_client/branch/nautilus.yaml
new file mode 100644
index 000000000000..9b1db62e47f5
--- /dev/null
+++ b/qa/suites/fs/upgrade/upgraded_client/branch/nautilus.yaml
@@ -0,0 +1,2 @@
+teuthology:
+  branch: nautilus
diff --git a/qa/suites/fs/upgrade/upgraded_client/branch/pacific.yaml b/qa/suites/fs/upgrade/upgraded_client/branch/pacific.yaml
new file mode 100644
index 000000000000..d86c55a34f9d
--- /dev/null
+++ b/qa/suites/fs/upgrade/upgraded_client/branch/pacific.yaml
@@ -0,0 +1,2 @@
+teuthology:
+  branch: pacific
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/0-from/pacific.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/0-from/pacific.yaml
deleted file mode 100644
index defb0392259a..000000000000
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/0-from/pacific.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-meta:
-- desc: |
-   install ceph/pacific latest
-tasks:
-- install:
-    branch: pacific
-    exclude_packages:
-      - cephadm
-      - ceph-mgr-cephadm
-      - ceph-immutable-object-cache
-      - python3-rados
-      - python3-rgw
-      - python3-rbd
-      - python3-cephfs
-      - ceph-volume
-    extra_packages:
-      - python-rados
-      - python-rgw
-      - python-rbd
-      - python-cephfs
-    # For kernel_untar_build workunit
-    extra_system_packages:
-      - bison
-      - flex
-      - elfutils-libelf-devel
-      - openssl-devel
-      - NetworkManager
-      - iproute
-      - util-linux
-- print: "**** done installing pacific"
-- ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(FS_
-      - \(MDS_
-      - \(OSD_
-      - \(MON_DOWN\)
-      - \(CACHE_POOL_
-      - \(POOL_
-      - \(MGR_DOWN\)
-      - \(PG_
-      - \(SMALLER_PGP_NUM\)
-      - Monitor daemon marked osd
-      - Behind on trimming
-      - Manager daemon
-    conf:
-      global:
-        mon warn on pool no app: false
-        ms bind msgr2: false
-- exec:
-    osd.0:
-      - ceph osd set-require-min-compat-client pacific
-- print: "**** done ceph"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/0-from/nautilus.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/0-install.yaml
similarity index 64%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/0-from/nautilus.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/0-install.yaml
index 02f541eaf30f..5b250375a00e 100644
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/0-from/nautilus.yaml
+++ b/qa/suites/fs/upgrade/upgraded_client/tasks/0-install.yaml
@@ -1,9 +1,15 @@
-meta:
-- desc: |
-   install ceph/nautilus latest
+teuthology:
+  premerge: |
+            local branch = yaml.teuthology.branch
+            yaml_fragment['tasks'][0]['install']['branch'] = tostring(branch)
+            print_branch = "**** done installing "..tostring(branch)
+            yaml_fragment['tasks'][1]['print'] = print_branch
+            local exec = yaml_fragment['tasks'][3]['exec']['osd.0']
+            local set_compat = "ceph osd set-require-min-compat-client "..tostring(branch)
+            py_attrgetter(exec).append(set_compat)
 tasks:
 - install:
-    branch: nautilus
+    branch: []
     exclude_packages:
       - cephadm
       - ceph-mgr-cephadm
@@ -27,7 +33,7 @@ tasks:
       - NetworkManager
       - iproute
       - util-linux
-- print: "**** done installing nautilus"
+- print: []
 - ceph:
     log-ignorelist:
       - overall HEALTH_
@@ -48,6 +54,5 @@ tasks:
         mon warn on pool no app: false
         ms bind msgr2: false
 - exec:
-    osd.0:
-      - ceph osd set-require-min-compat-client nautilus
+    osd.0: []
 - print: "**** done ceph"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
new file mode 100644
index 000000000000..1ff967d4da6b
--- /dev/null
+++ b/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
@@ -0,0 +1,63 @@
+teuthology:
+  premerge: |
+            local branch = yaml.teuthology.branch
+            local upgrade_branch = ""
+            local osd_release = ""
+            if tostring(branch) == "nautilus" then
+              upgrade_branch = "pacific"
+              osd_release = "ceph osd require-osd-release pacific"
+            elseif tostring(branch) == "pacific" then
+              upgrade_branch = "reef"
+              osd_release = "ceph osd require-osd-release reef"
+            else
+              error("invalid source branch")
+            end
+            yaml_fragment['tasks'][2]['install.upgrade']['mon.a']['branch'] = upgrade_branch
+            py_attrgetter(yaml_fragment['tasks'][9]['exec']['mon.a']).append(osd_release)
+overrides:
+  ceph:
+    log-ignorelist:
+    - scrub mismatch
+    - ScrubResult
+    - wrongly marked
+    - \(POOL_APP_NOT_ENABLED\)
+    - \(SLOW_OPS\)
+    - overall HEALTH_
+    - \(MON_MSGR2_NOT_ENABLED\)
+    - slow request
+    conf:
+      global:
+        bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
+      mon:
+        mon warn on osd down out interval zero: false
+
+tasks:
+- mds_pre_upgrade:
+- print: "**** done mds pre-upgrade sequence"
+- install.upgrade:
+    # upgrade the single cluster node, which is running all the mon/mds/osd/mgr daemons
+    mon.a:
+      branch: []
+- print: "**** done install.upgrade the host"
+- ceph.restart:
+    daemons: [mon.*, mgr.*]
+    mon-health-to-clog: false
+    wait-for-healthy: false
+- ceph.healthy:
+- ceph.restart:
+    daemons: [osd.*]
+    wait-for-healthy: false
+    wait-for-osds-up: true
+- ceph.stop: [mds.*]
+- ceph.restart:
+    daemons: [mds.*]
+    wait-for-healthy: false
+    wait-for-osds-up: true
+- exec:
+    mon.a:
+    - ceph osd dump -f json-pretty
+    - ceph versions
+    - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done
+- ceph.healthy:
+- print: "**** done ceph.restart"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/0-client-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/0-client-upgrade.yaml
deleted file mode 100644
index 2d948af19598..000000000000
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/0-client-upgrade.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-teuthology:
-  postmerge:
-    - if not is_fuse() then reject() end
-tasks:
-- ceph-fuse:
-    client.0:
-      mounted: false
-- print: "**** done unmount client.0"
-- install.upgrade:
-    client.0:
-- print: "**** done install.upgrade on client.0"
-- ceph-fuse:
-    client.0:
-- print: "**** done remount client.0"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/.qa b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/.qa
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/.qa
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/.qa
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/% b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/%
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/%
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/%
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/0-clients/fuse-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/0-clients/fuse-upgrade.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/0-clients/kclient.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/kclient.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/0-clients/kclient.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/kclient.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/1-client-sanity.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/1-client-sanity.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/new_ops/1-client-sanity.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/1-client-sanity.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/% b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/%
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/%
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/%
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/.qa b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/.qa
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/.qa
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/.qa
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml
new file mode 100644
index 000000000000..d91a2b90790c
--- /dev/null
+++ b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml
@@ -0,0 +1,24 @@
+teuthology:
+  premerge: |
+            local branch = yaml.teuthology.branch
+            if tostring(branch) == "nautilus" then
+              yaml_fragment['tasks'][2]['install.upgrade']['client.0']['branch'] = "pacific"
+            elseif tostring(branch) == "pacific" then
+              yaml_fragment['tasks'][2]['install.upgrade']['client.0']['branch'] = "reef"
+            else
+              error("invalid source branch")
+            end
+  postmerge:
+    - if not is_fuse() then reject() end
+tasks:
+- ceph-fuse:
+    client.0:
+      mounted: false
+- print: "**** done unmount client.0"
+- install.upgrade:
+    client.0:
+      branch: []
+- print: "**** done install.upgrade on client.0"
+- ceph-fuse:
+    client.0:
+- print: "**** done remount client.0"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/.qa b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/.qa
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/blogbench.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/blogbench.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/blogbench.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/blogbench.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/dbench.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/dbench.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/dbench.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/dbench.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/fsstress.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/fsstress.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/fsstress.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/fsstress.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/iozone.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/iozone.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/iozone.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/iozone.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/kernel_untar_build.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/kernel_untar_build.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/2-workload/stress_tests/1-tests/kernel_untar_build.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/kernel_untar_build.yaml

From 66fd89a860e3a3258849105d7f9149f5baf804b8 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Sun, 1 Oct 2023 03:13:18 +0530
Subject: [PATCH 0458/2492] qa/upgrade/nofs: upgrade pacific->reef

Fixes: https://tracker.ceph.com/issues/62953
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml b/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml
index 858142871383..e31e4b43ca8d 100644
--- a/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml
+++ b/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml
@@ -23,6 +23,7 @@ tasks:
     - ceph fs dump
 - install.upgrade:
     mon.a:
+      branch: reef
 - print: "**** done install.upgrade"
 - ceph.restart:
     daemons: [mon.*, mgr.*]

From 11df09dbbe2a12b5b4a838925ace4cc0f5fb31fd Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Mon, 6 Nov 2023 15:43:43 +0100
Subject: [PATCH 0459/2492] github: adding command for rook e2e jenkins job

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 .github/pull_request_template.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 214b2114e1a3..494a3f23e06a 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -64,4 +64,5 @@
 - `jenkins test ceph-volume all`
 - `jenkins test ceph-volume tox`
 - `jenkins test windows`
+- `jenkins test rook e2e`
 </details>

From bf0bfef19594bf3748fed9ec957b5c868ab50b36 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 11:28:30 -0400
Subject: [PATCH 0460/2492] cephadm: move some podman specific mount logic to
 podman class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                      | 11 +----------
 src/cephadm/cephadmlib/container_engines.py | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c56bd6bc2a2c..641b269c1324 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2647,17 +2647,8 @@ def get_container_mounts(
 
 def _update_podman_mounts(ctx: CephadmContext, mounts: Dict[str, str]) -> None:
     """Update the given mounts dict with mounts specific to podman."""
-    # Modifications podman makes to /etc/hosts causes issues with
-    # certain daemons (specifically referencing "host.containers.internal" entry
-    # being added to /etc/hosts in this case). To avoid that, but still
-    # allow users to use /etc/hosts for hostname resolution, we can
-    # mount the host's /etc/hosts file.
-    # https://tracker.ceph.com/issues/58532
-    # https://tracker.ceph.com/issues/57018
     if isinstance(ctx.container_engine, Podman):
-        if os.path.exists('/etc/hosts'):
-            if '/etc/hosts' not in mounts:
-                mounts['/etc/hosts'] = '/etc/hosts:ro'
+        ctx.container_engine.update_mounts(ctx, mounts)
 
 
 def get_ceph_volume_container(ctx: CephadmContext,
diff --git a/src/cephadm/cephadmlib/container_engines.py b/src/cephadm/cephadmlib/container_engines.py
index 99d64ff015cb..8ced8ab3ff4b 100644
--- a/src/cephadm/cephadmlib/container_engines.py
+++ b/src/cephadm/cephadmlib/container_engines.py
@@ -2,7 +2,7 @@
 
 import os
 
-from typing import Tuple, List, Optional
+from typing import Tuple, List, Optional, Dict
 
 from .call_wrappers import call_throws, CallVerbosity
 from .context import CephadmContext
@@ -78,6 +78,21 @@ def service_args(
             args.append('--no-hosts')
         return args
 
+    def update_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        """Update mounts adding entries that are specific to podman."""
+        # Modifications podman makes to /etc/hosts causes issues with certain
+        # daemons (specifically referencing "host.containers.internal" entry
+        # being added to /etc/hosts in this case). To avoid that, but still
+        # allow users to use /etc/hosts for hostname resolution, we can mount
+        # the host's /etc/hosts file.
+        # https://tracker.ceph.com/issues/58532
+        # https://tracker.ceph.com/issues/57018
+        if os.path.exists('/etc/hosts'):
+            if '/etc/hosts' not in mounts:
+                mounts['/etc/hosts'] = '/etc/hosts:ro'
+
 
 class Docker(ContainerEngine):
     EXE = 'docker'

From 69c7670b6c6c7f0df386b48ce30b2f25c705d8b5 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 11:41:15 -0400
Subject: [PATCH 0461/2492] cephadm: move option for setting unlimited pids
 into engine classes

Move the option for setting unlimited pids for the container engine
into the container engine classes. This continues the attempts to
improve the locality of items specific to certain classes to be
by making them part of the classes.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                          |  9 +--------
 src/cephadm/cephadmlib/container_engine_base.py |  7 +++++++
 src/cephadm/cephadmlib/container_engines.py     | 10 ++++++++++
 src/cephadm/tests/fixtures.py                   |  2 ++
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 641b269c1324..650e080843ed 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -70,7 +70,6 @@
     LOGROTATE_DIR,
     LOG_DIR,
     LOG_DIR_MODE,
-    PIDS_LIMIT_UNLIMITED_PODMAN_VERSION,
     SYSCTL_DIR,
     UNIT_DIR,
 )
@@ -2692,13 +2691,7 @@ def _update_pids_limit(ctx: CephadmContext, daemon_type: str, container_args: Li
     unlimited_daemons.add(NFSGanesha.daemon_type)
     if daemon_type not in unlimited_daemons:
         return
-    if (
-        isinstance(ctx.container_engine, Podman)
-        and ctx.container_engine.version >= PIDS_LIMIT_UNLIMITED_PODMAN_VERSION
-    ):
-        container_args.append('--pids-limit=-1')
-    else:
-        container_args.append('--pids-limit=0')
+    container_args.append(ctx.container_engine.unlimited_pids_option)
 
 
 def get_container(
diff --git a/src/cephadm/cephadmlib/container_engine_base.py b/src/cephadm/cephadmlib/container_engine_base.py
index 135b2f4f3210..c8d4bfbcf290 100644
--- a/src/cephadm/cephadmlib/container_engine_base.py
+++ b/src/cephadm/cephadmlib/container_engine_base.py
@@ -11,5 +11,12 @@ def __init__(self) -> None:
     def EXE(self) -> str:
         raise NotImplementedError()
 
+    @property
+    def unlimited_pids_option(self) -> str:
+        """The option to pass to the container engine for allowing unlimited
+        pids (processes).
+        """
+        return '--pids-limit=0'
+
     def __str__(self) -> str:
         return f'{self.EXE} ({self.path})'
diff --git a/src/cephadm/cephadmlib/container_engines.py b/src/cephadm/cephadmlib/container_engines.py
index 8ced8ab3ff4b..98019fa820b3 100644
--- a/src/cephadm/cephadmlib/container_engines.py
+++ b/src/cephadm/cephadmlib/container_engines.py
@@ -11,6 +11,7 @@
     CGROUPS_SPLIT_PODMAN_VERSION,
     DEFAULT_MODE,
     MIN_PODMAN_VERSION,
+    PIDS_LIMIT_UNLIMITED_PODMAN_VERSION,
 )
 from .exceptions import Error
 
@@ -45,6 +46,15 @@ def supports_split_cgroups(self) -> bool:
         """Return true if this version of podman supports split cgroups."""
         return self.version >= CGROUPS_SPLIT_PODMAN_VERSION
 
+    @property
+    def unlimited_pids_option(self) -> str:
+        """The option to pass to the container engine for allowing unlimited
+        pids (processes).
+        """
+        if self.version >= PIDS_LIMIT_UNLIMITED_PODMAN_VERSION:
+            return '--pids-limit=-1'
+        return '--pids-limit=0'
+
     def service_args(
         self, ctx: CephadmContext, service_name: str
     ) -> List[str]:
diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index 86a8c6119ea8..d25dffa9e3b4 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -21,6 +21,7 @@ def mock_docker():
 
     docker = mock.Mock(Docker)
     docker.path = '/usr/bin/docker'
+    type(docker).unlimited_pids_option = Docker.unlimited_pids_option
     return docker
 
 
@@ -37,6 +38,7 @@ def mock_podman():
     # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock
     type(podman).supports_split_cgroups = Podman.supports_split_cgroups
     type(podman).service_args = Podman.service_args
+    type(podman).unlimited_pids_option = Podman.unlimited_pids_option
     return podman
 
 

From 1dc1a0d1481e7e5d1ca5a9830e809b4bfd2bfd73 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 21 Oct 2023 12:05:43 -0400
Subject: [PATCH 0462/2492] cephadm: move logic to build pull command to
 container engines

Move the logic needed to build a pull command to a function in the
container_engines.py file. This continues the effort to improve the
locality of items with regards to specific options for container
engines. I had thought to move the whole pull function but the logging
and retry code made me change my mind. We can always move it later if it
makes sense.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                      | 11 +++--------
 src/cephadm/cephadmlib/container_engines.py | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 650e080843ed..478de27f0b5e 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -103,6 +103,7 @@
     Podman,
     check_container_engine,
     find_container_engine,
+    pull_command,
     registry_login,
 )
 from cephadmlib.data_utils import (
@@ -3926,14 +3927,7 @@ def _pull_image(ctx, image, insecure=False):
         'Digest did not match, expected',
     ]
 
-    cmd = [ctx.container_engine.path, 'pull', image]
-    if isinstance(ctx.container_engine, Podman):
-        if insecure:
-            cmd.append('--tls-verify=false')
-
-        if os.path.exists('/etc/ceph/podman-auth.json'):
-            cmd.append('--authfile=/etc/ceph/podman-auth.json')
-    cmd_str = ' '.join(cmd)
+    cmd = pull_command(ctx, image, insecure=insecure)
 
     for sleep_secs in [1, 4, 25]:
         out, err, ret = call(ctx, cmd, verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
@@ -3943,6 +3937,7 @@ def _pull_image(ctx, image, insecure=False):
         if 'unauthorized' in err:
             raise UnauthorizedRegistryError()
 
+        cmd_str = ' '.join(cmd)
         if not any(pattern in err for pattern in ignorelist):
             raise Error('Failed command: %s' % cmd_str)
 
diff --git a/src/cephadm/cephadmlib/container_engines.py b/src/cephadm/cephadmlib/container_engines.py
index 98019fa820b3..64ce7ae821ab 100644
--- a/src/cephadm/cephadmlib/container_engines.py
+++ b/src/cephadm/cephadmlib/container_engines.py
@@ -175,3 +175,17 @@ def registry_login(
             'Failed to login to custom registry @ %s as %s with given password'
             % (ctx.registry_url, ctx.registry_username)
         )
+
+
+def pull_command(
+    ctx: CephadmContext, image: str, insecure: bool = False
+) -> List[str]:
+    """Return a command that can be run to pull an image."""
+    cmd = [ctx.container_engine.path, 'pull', image]
+    if isinstance(ctx.container_engine, Podman):
+        if insecure:
+            cmd.append('--tls-verify=false')
+
+        if os.path.exists('/etc/ceph/podman-auth.json'):
+            cmd.append('--authfile=/etc/ceph/podman-auth.json')
+    return cmd

From 57a1a9fa9659fd3aa0f3b7e4c2beeee273f64bd9 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 17 Mar 2023 11:11:05 -0400
Subject: [PATCH 0463/2492] rgw: remove unused RGWBucketCreateLocalCR

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_cr_tools.cc | 125 ---------------------------
 src/rgw/driver/rados/rgw_cr_tools.h  |   8 --
 2 files changed, 133 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_cr_tools.cc b/src/rgw/driver/rados/rgw_cr_tools.cc
index 94665a35aaa6..31119a25b2eb 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.cc
+++ b/src/rgw/driver/rados/rgw_cr_tools.cc
@@ -103,131 +103,6 @@ int RGWGetBucketInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
   return store->get_bucket(dpp, nullptr, params.tenant, params.bucket_name, &result->bucket, null_yield);
 }
 
-template<>
-int RGWBucketCreateLocalCR::Request::_send_request(const DoutPrefixProvider *dpp)
-{
-  CephContext *cct = store->ctx();
-  auto& zone_svc = store->svc()->zone;
-
-  const auto& user_info = params.user_info.get();
-  const auto& user = user_info->user_id;
-  const auto& bucket_name = params.bucket_name;
-  auto& placement_rule = params.placement_rule;
-
-  if (!placement_rule.empty() &&
-      !zone_svc->get_zone_params().valid_placement(placement_rule)) {
-    ldpp_dout(dpp, 0) << "placement target (" << placement_rule << ")"
-      << " doesn't exist in the placement targets of zonegroup"
-      << " (" << zone_svc->get_zonegroup().api_name << ")" << dendl;
-    return -ERR_INVALID_LOCATION_CONSTRAINT;
-  }
-
-  /* we need to make sure we read bucket info, it's not read before for this
-   * specific request */
-  RGWBucketInfo bucket_info;
-  map<string, bufferlist> bucket_attrs;
-
-  int ret = store->getRados()->get_bucket_info(store->svc(), user.tenant, bucket_name,
-				  bucket_info, nullptr, null_yield, dpp, &bucket_attrs);
-  if (ret < 0 && ret != -ENOENT)
-    return ret;
-  bool bucket_exists = (ret != -ENOENT);
-
-  RGWAccessControlPolicy old_policy(cct);
-  ACLOwner bucket_owner;
-  bucket_owner.set_id(user);
-  bucket_owner.set_name(user_info->display_name);
-  if (bucket_exists) {
-    ret = rgw_op_get_bucket_policy_from_attr(dpp, cct, store, bucket_info,
-                                             bucket_attrs, &old_policy, null_yield);
-    if (ret >= 0)  {
-      if (old_policy.get_owner().get_id().compare(user) != 0) {
-        return -EEXIST;
-      }
-    }
-  }
-
-  RGWBucketInfo master_info;
-  rgw_bucket *pmaster_bucket = nullptr;
-  uint32_t *pmaster_num_shards = nullptr;
-  real_time creation_time;
-
-  string zonegroup_id = zone_svc->get_zonegroup().get_id();
-
-  if (bucket_exists) {
-    rgw_placement_rule selected_placement_rule;
-    rgw_bucket bucket;
-    bucket.tenant = user.tenant;
-    bucket.name = bucket_name;
-    ret = zone_svc->select_bucket_placement(dpp, *user_info, zonegroup_id,
-					    placement_rule,
-					    &selected_placement_rule, nullptr, null_yield);
-    if (selected_placement_rule != bucket_info.placement_rule) {
-      ldpp_dout(dpp, 0) << "bucket already exists on a different placement rule: "
-        << " selected_rule= " << selected_placement_rule
-        << " existing_rule= " << bucket_info.placement_rule << dendl;
-      return -EEXIST;
-    }
-  }
-
-  /* Encode special metadata first as we're using std::map::emplace under
-   * the hood. This method will add the new items only if the map doesn't
-   * contain such keys yet. */
-  RGWAccessControlPolicy_S3 policy(cct);
-  policy.create_canned(bucket_owner, bucket_owner, string()); /* default private policy */
-  bufferlist aclbl;
-  policy.encode(aclbl);
-  map<string, buffer::list> attrs;
-  attrs.emplace(std::move(RGW_ATTR_ACL), std::move(aclbl));
-
-  RGWQuotaInfo quota_info;
-  const RGWQuotaInfo * pquota_info = nullptr;
-
-  rgw_bucket bucket;
-  bucket.tenant = user.tenant;
-  bucket.name = bucket_name;
-
-  RGWBucketInfo info;
-  obj_version ep_objv;
-
-  ret = store->getRados()->create_bucket(*user_info, bucket, zonegroup_id,
-                                placement_rule, bucket_info.swift_ver_location,
-                                pquota_info, attrs,
-                                info, nullptr, &ep_objv, creation_time,
-				pmaster_bucket, pmaster_num_shards, null_yield, dpp, true);
-
-
-  if (ret && ret != -EEXIST)
-    return ret;
-
-  bool existed = (ret == -EEXIST);
-
-  if (existed) {
-    if (info.owner != user) {
-      ldpp_dout(dpp, 20) << "NOTICE: bucket already exists under a different user (bucket=" << bucket << " user=" << user << " bucket_owner=" << info.owner << dendl;
-      return -EEXIST;
-    }
-    bucket = info.bucket;
-  }
-
-  ret = store->ctl()->bucket->link_bucket(user, bucket, info.creation_time, null_yield, dpp, false);
-  if (ret && !existed && ret != -EEXIST) {
-    /* if it exists (or previously existed), don't remove it! */
-    int r = store->ctl()->bucket->unlink_bucket(user, bucket, null_yield, dpp);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << r << dendl;
-    }
-  } else if (ret == -EEXIST || (ret == 0 && existed)) {
-    ret = -ERR_BUCKET_EXISTS;
-  }
-
-  if (ret < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: bucket creation (bucket=" << bucket << ") return ret=" << ret << dendl;
-  }
-
-  return ret;
-}
-
 template<>
 int RGWObjectSimplePutCR::Request::_send_request(const DoutPrefixProvider *dpp)
 {
diff --git a/src/rgw/driver/rados/rgw_cr_tools.h b/src/rgw/driver/rados/rgw_cr_tools.h
index 4cd97aa82f51..e68e386fb836 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.h
+++ b/src/rgw/driver/rados/rgw_cr_tools.h
@@ -45,14 +45,6 @@ struct rgw_get_bucket_info_result {
 
 using RGWGetBucketInfoCR = RGWSimpleAsyncCR<rgw_get_bucket_info_params, rgw_get_bucket_info_result>;
 
-struct rgw_bucket_create_local_params {
-  std::shared_ptr<RGWUserInfo> user_info;
-  std::string bucket_name;
-  rgw_placement_rule placement_rule;
-};
-
-using RGWBucketCreateLocalCR = RGWSimpleWriteOnlyAsyncCR<rgw_bucket_create_local_params>;
-
 struct rgw_object_simple_put_params {
   RGWDataAccess::BucketRef bucket;
   rgw_obj_key key;

From 019e60d68050f53fbd8b1679e6e835f4c164473d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 12:01:49 -0400
Subject: [PATCH 0464/2492] rgw: remove unused RGWObjectSimplePutCR

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_cr_tools.cc | 25 -------------------------
 src/rgw/driver/rados/rgw_cr_tools.h  | 10 ----------
 2 files changed, 35 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_cr_tools.cc b/src/rgw/driver/rados/rgw_cr_tools.cc
index 31119a25b2eb..65cb67505049 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.cc
+++ b/src/rgw/driver/rados/rgw_cr_tools.cc
@@ -103,31 +103,6 @@ int RGWGetBucketInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
   return store->get_bucket(dpp, nullptr, params.tenant, params.bucket_name, &result->bucket, null_yield);
 }
 
-template<>
-int RGWObjectSimplePutCR::Request::_send_request(const DoutPrefixProvider *dpp)
-{
-  RGWDataAccess::ObjectRef obj;
-
-  CephContext *cct = store->ctx();
-
-  int ret = params.bucket->get_object(params.key, &obj);
-  if (ret < 0) {
-    lderr(cct) << "ERROR: failed to get object: " << cpp_strerror(-ret) << dendl;
-    return -ret;
-  }
-
-  if (params.user_data) {
-    obj->set_user_data(*params.user_data);
-  }
-
-  ret = obj->put(params.data, params.attrs, dpp, null_yield);
-  if (ret < 0) {
-    ldpp_dout(dpp, -1) << "ERROR: put object returned error: " << cpp_strerror(-ret) << dendl;
-  }
-
-  return 0;
-}
-
 template<>
 int RGWBucketLifecycleConfigCR::Request::_send_request(const DoutPrefixProvider *dpp)
 {
diff --git a/src/rgw/driver/rados/rgw_cr_tools.h b/src/rgw/driver/rados/rgw_cr_tools.h
index e68e386fb836..3599e4505e4a 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.h
+++ b/src/rgw/driver/rados/rgw_cr_tools.h
@@ -45,16 +45,6 @@ struct rgw_get_bucket_info_result {
 
 using RGWGetBucketInfoCR = RGWSimpleAsyncCR<rgw_get_bucket_info_params, rgw_get_bucket_info_result>;
 
-struct rgw_object_simple_put_params {
-  RGWDataAccess::BucketRef bucket;
-  rgw_obj_key key;
-  bufferlist data;
-  std::map<std::string, bufferlist> attrs;
-  std::optional<std::string> user_data;
-};
-
-using RGWObjectSimplePutCR = RGWSimpleWriteOnlyAsyncCR<rgw_object_simple_put_params>;
-
 
 struct rgw_bucket_lifecycle_config_params {
   rgw::sal::Bucket* bucket;

From bac9bbeaac4c661cd930d0184422d7b4733e8310 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 11:48:05 -0400
Subject: [PATCH 0465/2492] rgw: don't use other zone's shard count for
 forwarded CreateBucket requests

multisite no longer requires shard counts to match between zones. each
zone can configure its own default index shard count, so ignore the shard
counts that come from the metadata master zone

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc    | 7 ++-----
 src/rgw/driver/rados/rgw_bucket.h     | 1 -
 src/rgw/driver/rados/rgw_rados.cc     | 6 +-----
 src/rgw/driver/rados/rgw_rados.h      | 3 +--
 src/rgw/driver/rados/rgw_sal_rados.cc | 6 +-----
 5 files changed, 5 insertions(+), 18 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index d6d83eab3dcc..fc16316efa4c 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -2506,7 +2506,6 @@ int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op,
 
 void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout,
 				const RGWZone& zone,
-				std::optional<uint32_t> shards,
 				std::optional<rgw::BucketIndexType> type) {
   layout.current_index.gen = 0;
   layout.current_index.layout.normal.hash_type = rgw::BucketHashType::Mod;
@@ -2514,9 +2513,7 @@ void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout,
   layout.current_index.layout.type =
     type.value_or(rgw::BucketIndexType::Normal);
 
-  if (shards) {
-    layout.current_index.layout.normal.num_shards = *shards;
-  } else if (cct->_conf->rgw_override_bucket_index_max_shards > 0) {
+  if (cct->_conf->rgw_override_bucket_index_max_shards > 0) {
     layout.current_index.layout.normal.num_shards =
       cct->_conf->rgw_override_bucket_index_max_shards;
   } else {
@@ -2548,7 +2545,7 @@ int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dp
       bci.info.layout = rgw::BucketLayout{};
       init_default_bucket_layout(cct, bci.info.layout,
 				 bihandler->svc.zone->get_zone(),
-				 std::nullopt, std::nullopt);
+				 std::nullopt);
     } else {
       bci.info.layout = old_bci->info.layout;
     }
diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h
index ba0d1cf005c8..0110eff33439 100644
--- a/src/rgw/driver/rados/rgw_bucket.h
+++ b/src/rgw/driver/rados/rgw_bucket.h
@@ -45,7 +45,6 @@ extern bool rgw_bucket_object_check_filter(const std::string& oid);
 
 void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout,
 				const RGWZone& zone,
-				std::optional<uint32_t> shards,
 				std::optional<rgw::BucketIndexType> type);
 
 struct RGWBucketCompleteInfo {
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b935610f9ad5..2df4f77b673e 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2282,8 +2282,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
                             obj_version *pobjv,
                             obj_version *pep_objv,
                             real_time creation_time,
-                            rgw_bucket *pmaster_bucket,
-                            uint32_t *pmaster_num_shards,
+                            const rgw_bucket* pmaster_bucket,
 			    optional_yield y,
                             const DoutPrefixProvider *dpp,
 			    bool exclusive)
@@ -2325,9 +2324,6 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
     info.swift_versioning = (!swift_ver_location.empty());
 
     init_default_bucket_layout(cct, info.layout, svc.zone->get_zone(),
-			       pmaster_num_shards ?
-			       std::optional{*pmaster_num_shards} :
-			       std::nullopt,
 			       rule_info.index_type);
 
     info.requester_pays = false;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 2dca3cfaad4e..d312e7355e22 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -638,8 +638,7 @@ class RGWRados
 		    obj_version *pobjv,
 		    obj_version *pep_objv,
 		    ceph::real_time creation_time,
-		    rgw_bucket *master_bucket,
-		    uint32_t *master_num_shards,
+		    const rgw_bucket *master_bucket,
 		    optional_yield y,
                     const DoutPrefixProvider *dpp,
 		    bool exclusive = true);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index fb684a921e6d..f085afb2beed 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -186,7 +186,6 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
   bufferlist in_data;
   RGWBucketInfo master_info;
   rgw_bucket* pmaster_bucket;
-  uint32_t* pmaster_num_shards;
   real_time creation_time;
   std::unique_ptr<Bucket> bucket;
   obj_version objv,* pobjv = NULL;
@@ -232,14 +231,12 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
     ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl;
     pmaster_bucket= &master_info.bucket;
     creation_time = master_info.creation_time;
-    pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards;
     pobjv = &objv;
     if (master_info.obj_lock_enabled()) {
       info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
     }
   } else {
     pmaster_bucket = NULL;
-    pmaster_num_shards = NULL;
     if (obj_lock_enabled)
       info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
   }
@@ -264,8 +261,7 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
     ret = store->getRados()->create_bucket(this->get_info(), bucket->get_key(),
 				    zid, placement_rule, swift_ver_location, pquota_info,
 				    attrs, info, pobjv, &ep_objv, creation_time,
-				    pmaster_bucket, pmaster_num_shards, y, dpp,
-				    exclusive);
+				    pmaster_bucket, y, dpp, exclusive);
     if (ret == -EEXIST) {
       *existed = true;
       /* bucket already existed, might have raced with another bucket creation,

From 59708637fac2dc5b036b81fb3df7b0eaebdac7c3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 12:29:08 -0400
Subject: [PATCH 0466/2492] rgw: RGWBulkDelete calls forward_to_master before
 remove_bucket()

fix the path of the forwarded request the same way that RGWBulkUpload
does, and call forward_request_to_master() before remove_bucket()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8d2f564b8e34..324164572135 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -7322,7 +7322,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
     goto auth_fail;
   }
 
-  if (!path.obj_key.empty()) {
+  if (!path.obj_key.empty()) { // object deletion
     ACLOwner bucket_owner;
 
     bucket_owner.set_id(bucket->get_info().owner);
@@ -7338,8 +7338,20 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
     if (ret < 0) {
       goto delop_fail;
     }
-  } else {
-    ret = bucket->remove_bucket(dpp, false, true, &s->info, s->yield);
+  } else { // bucket deletion
+    if (!driver->is_meta_master()) {
+      // apply bucket deletion on the master zone first
+      req_info req = s->info;
+      forward_req_info(dpp, s->cct, req, path.bucket_name);
+
+      bufferlist data;
+      ret = driver->forward_request_to_master(dpp, s->user.get(), nullptr,
+                                              data, nullptr, req, y);
+      if (ret < 0) {
+        goto delop_fail;
+      }
+    }
+    ret = bucket->remove_bucket(dpp, false, false, nullptr, s->yield);
     if (ret < 0) {
       goto delop_fail;
     }

From b69757ddddfb52113d5a910546a004ef4e258465 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 12:49:55 -0400
Subject: [PATCH 0467/2492] rgw/sal: User::remove_bucket() no longer supports
 forward_to_master

callers of remove_bucket() are now responsible for calling
forward_request_to_master() themselves

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc     |  8 ++------
 src/rgw/driver/daos/rgw_sal_daos.h      |  1 -
 src/rgw/driver/motr/rgw_sal_motr.cc     |  2 +-
 src/rgw/driver/motr/rgw_sal_motr.h      |  2 +-
 src/rgw/driver/posix/rgw_sal_posix.cc   |  8 +++-----
 src/rgw/driver/posix/rgw_sal_posix.h    |  1 -
 src/rgw/driver/rados/rgw_bucket.cc      |  3 +--
 src/rgw/driver/rados/rgw_rest_bucket.cc | 18 ++++++++++++++----
 src/rgw/driver/rados/rgw_sal_rados.cc   | 17 +----------------
 src/rgw/driver/rados/rgw_sal_rados.h    |  2 +-
 src/rgw/driver/rados/rgw_user.cc        |  2 +-
 src/rgw/rgw_op.cc                       |  4 ++--
 src/rgw/rgw_sal.h                       |  2 +-
 src/rgw/rgw_sal_dbstore.cc              |  2 +-
 src/rgw/rgw_sal_dbstore.h               |  2 +-
 src/rgw/rgw_sal_filter.cc               |  4 +---
 src/rgw/rgw_sal_filter.h                |  1 -
 17 files changed, 31 insertions(+), 48 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 73eec5b3e09c..c4deba7f11f9 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -405,13 +405,9 @@ std::unique_ptr<struct ds3_bucket_info> DaosBucket::get_encoded_info(
 }
 
 int DaosBucket::remove_bucket(const DoutPrefixProvider* dpp,
-                              bool delete_children, bool forward_to_master,
-                              req_info* req_info, optional_yield y) {
+                              bool delete_children, optional_yield y) {
   ldpp_dout(dpp, 20) << "DEBUG: remove_bucket, delete_children="
-                    
-                     << delete_children
-                    
-                     << " forward_to_master=" << forward_to_master << dendl;
+                     << delete_children << dendl;
 
   return ds3_bucket_destroy(get_name().c_str(), delete_children, store->ds3,
                             nullptr);
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 429c6160488d..84b640b8a6e0 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -290,7 +290,6 @@ class DaosBucket : public StoreBucket {
   virtual int list(const DoutPrefixProvider* dpp, ListParams&, int,
                    ListResults&, optional_yield y) override;
   virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-                            bool forward_to_master, req_info* req_info,
                             optional_yield y) override;
   virtual int remove_bucket_bypass_gc(int concurrent_max,
                                       bool keep_index_consistent,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 83c6153a92a5..e9377f945360 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -556,7 +556,7 @@ int MotrUser::verify_mfa(const std::string& mfa_str, bool* verified, const DoutP
   return 0;
 }
 
-int MotrBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y)
+int MotrBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
 {
   int ret;
 
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index ce5fc2b95b1f..6e6b9908061f 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -352,7 +352,7 @@ class MotrBucket : public StoreBucket {
 
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
     virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override;
-    virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) override;
+    virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
     virtual int remove_bucket_bypass_gc(int concurrent_max, bool
         keep_index_consistent,
         optional_yield y, const
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 40091d06d0f9..b999839bf4e7 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -950,8 +950,6 @@ int POSIXBucket::merge_and_store_attrs(const DoutPrefixProvider* dpp,
 
 int POSIXBucket::remove_bucket(const DoutPrefixProvider* dpp,
 				bool delete_children,
-				bool forward_to_master,
-				req_info* req_info,
 				optional_yield y)
 {
   return delete_directory(parent_fd, get_fname().c_str(),
@@ -963,7 +961,7 @@ int POSIXBucket::remove_bucket_bypass_gc(int concurrent_max,
 					 optional_yield y,
 					 const DoutPrefixProvider *dpp)
 {
-  return remove_bucket(dpp, true, false, nullptr, y);
+  return remove_bucket(dpp, true, y);
 }
 
 int POSIXBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y)
@@ -1547,7 +1545,7 @@ int POSIXObject::delete_object(const DoutPrefixProvider* dpp,
 
   if (!b->versioned()) {
     if (shadow) {
-      ret = shadow->remove_bucket(dpp, true, false, nullptr, y);
+      ret = shadow->remove_bucket(dpp, true, y);
       if (ret < 0) {
 	return ret;
       }
@@ -2686,7 +2684,7 @@ int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
     return ret;
   }
 
-  shadow->remove_bucket(dpp, true, false, nullptr, y);
+  shadow->remove_bucket(dpp, true, y);
 
   return 0;
 }
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 739e7ef7a610..3695763c45f9 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -196,7 +196,6 @@ class POSIXBucket : public StoreBucket {
   virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp,
 				    Attrs& new_attrs, optional_yield y) override;
   virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-			    bool forward_to_master, req_info* req_info,
 			    optional_yield y) override;
   virtual int remove_bucket_bypass_gc(int concurrent_max,
 				      bool keep_index_consistent,
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index fc16316efa4c..04fa15381390 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1250,8 +1250,7 @@ int RGWBucketAdminOp::remove_bucket(rgw::sal::Driver* driver, RGWBucketAdminOpSt
   if (bypass_gc)
     ret = bucket->remove_bucket_bypass_gc(op_state.get_max_aio(), keep_index_consistent, y, dpp);
   else
-    ret = bucket->remove_bucket(dpp, op_state.will_delete_children(),
-				false, nullptr, y);
+    ret = bucket->remove_bucket(dpp, op_state.will_delete_children(), y);
 
   return ret;
 }
diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index 69e3f42f15e2..9c811efa42d7 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -220,9 +220,19 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
   RESTArgs::get_string(s, "bucket", bucket_name, &bucket_name);
   RESTArgs::get_bool(s, "purge-objects", false, &delete_children);
 
-  /* FIXME We're abusing the owner of the bucket to pass the user, so that it can be forwarded to
-   * the master.  This user is actually the OP caller, not the bucket owner. */
-  op_ret = driver->get_bucket(s, s->user.get(), string(), bucket_name, &bucket, y);
+  bufferlist data;
+  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  if (op_ret < 0) {
+    ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
+    if (op_ret == -ENOENT) {
+      /* adjust error, we want to return with NoSuchBucket and not
+       * NoSuchKey */
+      op_ret = -ERR_NO_SUCH_BUCKET;
+    }
+    return;
+  }
+
+  op_ret = driver->get_bucket(s, nullptr, string(), bucket_name, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "get_bucket returned ret=" << op_ret << dendl;
     if (op_ret == -ENOENT) {
@@ -231,7 +241,7 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
     return;
   }
 
-  op_ret = bucket->remove_bucket(s, delete_children, true, &s->info, s->yield);
+  op_ret = bucket->remove_bucket(s, delete_children, s->yield);
 }
 
 class RGWOp_Set_Bucket_Quota : public RGWRESTOp {
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index f085afb2beed..1de9756fc0a1 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -404,8 +404,6 @@ RadosBucket::~RadosBucket() {}
 
 int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp,
 			       bool delete_children,
-			       bool forward_to_master,
-			       req_info* req_info,
 			       optional_yield y)
 {
   int ret;
@@ -488,19 +486,6 @@ int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp,
     ldpp_dout(dpp, -1) << "ERROR: unable to remove user bucket information" << dendl;
   }
 
-  if (forward_to_master) {
-    bufferlist in_data;
-    ret = store->forward_request_to_master(dpp, owner, &ot.read_version, in_data, nullptr, *req_info, y);
-    if (ret < 0) {
-      if (ret == -ENOENT) {
-	/* adjust error, we want to return with NoSuchBucket and not
-	 * NoSuchKey */
-	ret = -ERR_NO_SUCH_BUCKET;
-      }
-      return ret;
-    }
-  }
-
   return ret;
 }
 
@@ -631,7 +616,7 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool
   // this function can only be run if caller wanted children to be
   // deleted, so we can ignore the check for children as any that
   // remain are detritus from a prior bug
-  ret = remove_bucket(dpp, true, false, nullptr, y);
+  ret = remove_bucket(dpp, true, y);
   if (ret < 0) {
     ldpp_dout(dpp, -1) << "ERROR: could not remove bucket " << this << dendl;
     return ret;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index e86a0870782a..ebf6531f1471 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -537,7 +537,7 @@ class RadosBucket : public StoreBucket {
     virtual ~RadosBucket();
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
     virtual int list(const DoutPrefixProvider* dpp, ListParams&, int, ListResults&, optional_yield y) override;
-    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) override;
+    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) override;
     virtual int remove_bucket_bypass_gc(int concurrent_max, bool
 					keep_index_consistent,
 					optional_yield y, const
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index 469fe6fe68fd..f81254bcfb0e 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -1791,7 +1791,7 @@ int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
         return ret;
       }
 
-      ret = bucket->remove_bucket(dpp, true, false, nullptr, y);
+      ret = bucket->remove_bucket(dpp, true, y);
       if (ret < 0) {
         set_err_msg(err_msg, "unable to delete user data");
         return ret;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 324164572135..a830ff726af8 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3594,7 +3594,7 @@ void RGWDeleteBucket::execute(optional_yield y)
       // do nothing; it will already have been logged
   }
 
-  op_ret = s->bucket->remove_bucket(this, false, false, nullptr, y);
+  op_ret = s->bucket->remove_bucket(this, false, y);
   if (op_ret < 0 && op_ret == -ECANCELED) {
       // lost a race, either with mdlog sync or another delete bucket operation.
       // in either case, we've already called ctl.bucket->unlink_bucket()
@@ -7351,7 +7351,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
         goto delop_fail;
       }
     }
-    ret = bucket->remove_bucket(dpp, false, false, nullptr, s->yield);
+    ret = bucket->remove_bucket(dpp, false, s->yield);
     if (ret < 0) {
       goto delop_fail;
     }
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index e7a3dfe0aa52..73122e06b4b7 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -660,7 +660,7 @@ class Bucket {
     /** Set the cached attributes on this bucket */
     virtual int set_attrs(Attrs a) = 0;
     /** Remove this bucket from the backing store */
-    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) = 0;
+    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) = 0;
     /** Remove this bucket, bypassing garbage collection.  May be removed */
     virtual int remove_bucket_bypass_gc(int concurrent_max, bool
 					keep_index_consistent,
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 36d76cc12e15..97da47eb6d38 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -236,7 +236,7 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y)
+  int DBBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
   {
     int ret;
 
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 0c75f4b98cbb..68fe4c443199 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -171,7 +171,7 @@ class DBNotification : public StoreNotification {
 
       virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
       virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override;
-      virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, bool forward_to_master, req_info* req_info, optional_yield y) override;
+      virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
       virtual int remove_bucket_bypass_gc(int concurrent_max, bool
 					keep_index_consistent,
 					optional_yield y, const
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index dbf688a22ab0..fb9e7aa1a286 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -649,11 +649,9 @@ int FilterBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int ma
 
 int FilterBucket::remove_bucket(const DoutPrefixProvider* dpp,
 				bool delete_children,
-				bool forward_to_master,
-				req_info* req_info,
 				optional_yield y)
 {
-  return next->remove_bucket(dpp, delete_children, forward_to_master, req_info, y);
+  return next->remove_bucket(dpp, delete_children, y);
 }
 
 int FilterBucket::remove_bucket_bypass_gc(int concurrent_max,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index dcc03df95194..81452f22cd7d 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -413,7 +413,6 @@ class FilterBucket : public Bucket {
   virtual Attrs& get_attrs(void) override { return next->get_attrs(); }
   virtual int set_attrs(Attrs a) override { return next->set_attrs(a); }
   virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-			    bool forward_to_master, req_info* req_info,
 			    optional_yield y) override;
   virtual int remove_bucket_bypass_gc(int concurrent_max, bool
 				      keep_index_consistent,

From e2a770818a88dd8627545635511363e3337cce92 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 14:52:25 -0400
Subject: [PATCH 0468/2492] rgw: move rgw_op_get_bucket_policy_from_attr() out
 of sal

this was duplicated in rgw_op.cc and rgw_sal_rados.cc

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 54 ---------------------------
 src/rgw/rgw_op.cc                     | 23 +++++++++---
 src/rgw/rgw_op.h                      |  6 +--
 3 files changed, 21 insertions(+), 62 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 1de9756fc0a1..4cc3ab5972bd 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -77,51 +77,6 @@ namespace rgw::sal {
 static constexpr size_t listing_max_entries = 1000;
 static std::string pubsub_oid_prefix = "pubsub.";
 
-static int decode_policy(CephContext* cct,
-                         bufferlist& bl,
-                         RGWAccessControlPolicy* policy)
-{
-  auto iter = bl.cbegin();
-  try {
-    policy->decode(iter);
-  } catch (buffer::error& err) {
-    ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
-    return -EIO;
-  }
-  if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
-    ldout(cct, 15) << __func__ << " Read AccessControlPolicy";
-    RGWAccessControlPolicy_S3* s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
-    s3policy->to_xml(*_dout);
-    *_dout << dendl;
-  }
-  return 0;
-}
-
-static int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider* dpp,
-					      RadosStore* store,
-					      User* user,
-					      Attrs& bucket_attrs,
-					      RGWAccessControlPolicy* policy,
-					      optional_yield y)
-{
-  auto aiter = bucket_attrs.find(RGW_ATTR_ACL);
-
-  if (aiter != bucket_attrs.end()) {
-    int ret = decode_policy(store->ctx(), aiter->second, policy);
-    if (ret < 0)
-      return ret;
-  } else {
-    ldout(store->ctx(), 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl;
-    /* object exists, but policy is broken */
-    int r = user->load_user(dpp, y);
-    if (r < 0)
-      return r;
-
-    policy->create_default(user->get_id(), user->get_display_name());
-  }
-  return 0;
-}
-
 static int drain_aio(std::list<librados::AioCompletion*>& handles)
 {
   int ret = 0;
@@ -196,20 +151,11 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
     return ret;
 
   if (ret != -ENOENT) {
-    RGWAccessControlPolicy old_policy(store->ctx());
     *existed = true;
     if (swift_ver_location.empty()) {
       swift_ver_location = bucket->get_info().swift_ver_location;
     }
     placement_rule.inherit_from(bucket->get_info().placement_rule);
-
-    // don't allow changes to the acl policy
-    int r = rgw_op_get_bucket_policy_from_attr(dpp, store, this, bucket->get_attrs(),
-					       &old_policy, y);
-    if (r >= 0 && old_policy != policy) {
-      bucket_out->swap(bucket);
-      return -EEXIST;
-    }
   } else {
     bucket = std::unique_ptr<Bucket>(new RadosBucket(store, b, this));
     *existed = false;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index a830ff726af8..fec40c250735 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -227,7 +227,7 @@ static int get_user_policy_from_attr(const DoutPrefixProvider *dpp,
 int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp, 
                                        CephContext *cct,
 				       rgw::sal::Driver* driver,
-				       RGWBucketInfo& bucket_info,
+				       const rgw_user& bucket_owner,
 				       map<string, bufferlist>& bucket_attrs,
 				       RGWAccessControlPolicy *policy,
 				       optional_yield y)
@@ -240,13 +240,13 @@ int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
       return ret;
   } else {
     ldpp_dout(dpp, 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl;
-    std::unique_ptr<rgw::sal::User> user = driver->get_user(bucket_info.owner);
+    std::unique_ptr<rgw::sal::User> user = driver->get_user(bucket_owner);
     /* object exists, but policy is broken */
     int r = user->load_user(dpp, y);
     if (r < 0)
       return r;
 
-    policy->create_default(bucket_info.owner, user->get_display_name());
+    policy->create_default(user->get_id(), user->get_display_name());
   }
   return 0;
 }
@@ -360,7 +360,8 @@ static int read_bucket_policy(const DoutPrefixProvider *dpp,
     return 0;
   }
 
-  int ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info, bucket_attrs, policy, y);
+  int ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info.owner,
+                                               bucket_attrs, policy, y);
   if (ret == -ENOENT) {
       ret = -ERR_NO_SUCH_BUCKET;
   }
@@ -411,7 +412,8 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
     /* object does not exist checking the bucket's ACL to make sure
        that we send a proper error code */
     RGWAccessControlPolicy bucket_policy(s->cct);
-    ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info, bucket_attrs, &bucket_policy, y);
+    ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info.owner,
+                                             bucket_attrs, &bucket_policy, y);
     if (ret < 0) {
       return ret;
     }
@@ -3385,6 +3387,17 @@ void RGWCreateBucket::execute(optional_yield y)
       }
       /* Initialize info from req_state */
       info = tmp_bucket->get_info();
+
+      // don't allow changes to the acl policy
+      RGWAccessControlPolicy old_policy(get_cct());
+      int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
+                                                 tmp_bucket->get_attrs(),
+                                                 &old_policy, y);
+      if (r >= 0 && old_policy != policy) {
+        s->err.message = "Cannot modify existing access control policy";
+        op_ret = -EEXIST;
+        return;
+      }
     }
   }
 
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index ffc65836ce62..e1189cb236d4 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -79,11 +79,11 @@ class StrategyRegistry;
 
 int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
                                        CephContext *cct,
-				       rgw::sal::Driver* driver,
-                                       RGWBucketInfo& bucket_info,
+                                       rgw::sal::Driver* driver,
+                                       const rgw_user& bucket_owner,
                                        std::map<std::string, bufferlist>& bucket_attrs,
                                        RGWAccessControlPolicy *policy,
-				       optional_yield y);
+                                       optional_yield y);
 
 class RGWHandler {
 protected:

From de364986ffc83b409c309413713ffa25e2407cbf Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 15:09:42 -0400
Subject: [PATCH 0469/2492] rgw: RGWCreateBucket cleanup

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index fec40c250735..0307658aa0a6 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3328,10 +3328,6 @@ static void filter_out_website(std::map<std::string, ceph::bufferlist>& add_attr
 
 void RGWCreateBucket::execute(optional_yield y)
 {
-  buffer::list aclbl;
-  buffer::list corsbl;
-  string bucket_name = rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name);
-
   op_ret = get_params(y);
   if (op_ret < 0)
     return;
@@ -3418,10 +3414,12 @@ void RGWCreateBucket::execute(optional_yield y)
   /* Encode special metadata first as we're using std::map::emplace under
    * the hood. This method will add the new items only if the map doesn't
    * contain such keys yet. */
+  buffer::list aclbl;
   policy.encode(aclbl);
   emplace_attr(RGW_ATTR_ACL, std::move(aclbl));
 
   if (has_cors) {
+    buffer::list corsbl;
     cors_config.encode(corsbl);
     emplace_attr(RGW_ATTR_CORS, std::move(corsbl));
   }
@@ -3441,9 +3439,8 @@ void RGWCreateBucket::execute(optional_yield y)
     op_ret = filter_out_quota_info(attrs, rmattr_names, quota_info);
     if (op_ret < 0) {
       return;
-    } else {
-      pquota_info = &quota_info;
     }
+    pquota_info = &quota_info;
 
     /* Web site of Swift API. */
     filter_out_website(attrs, rmattr_names, info.website_conf);

From 6decabf0a2f0d5c78dc22024103a19daa3d2f72b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 16:05:51 -0400
Subject: [PATCH 0470/2492] rgw: read existing placement and swift_ver_location
 outside of sal

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc     |  4 ++--
 src/rgw/driver/d4n/rgw_sal_d4n.h      |  4 ++--
 src/rgw/driver/motr/rgw_sal_motr.cc   | 21 ++---------------
 src/rgw/driver/motr/rgw_sal_motr.h    |  4 ++--
 src/rgw/driver/posix/rgw_sal_posix.cc |  4 ++--
 src/rgw/driver/posix/rgw_sal_posix.h  |  4 ++--
 src/rgw/driver/rados/rgw_sal_rados.cc |  8 ++-----
 src/rgw/driver/rados/rgw_sal_rados.h  |  4 ++--
 src/rgw/rgw_op.cc                     |  5 ++++
 src/rgw/rgw_sal.h                     |  4 ++--
 src/rgw/rgw_sal_dbstore.cc            | 33 ++++++++-------------------
 src/rgw/rgw_sal_dbstore.h             |  4 ++--
 src/rgw/rgw_sal_filter.cc             |  4 ++--
 src/rgw/rgw_sal_filter.h              |  4 ++--
 src/test/rgw/test_rgw_lua.cc          |  2 +-
 15 files changed, 40 insertions(+), 69 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index 3195d87eac42..93cd8f2d4159 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -62,8 +62,8 @@ std::unique_ptr<Object> D4NFilterBucket::get_object(const rgw_obj_key& k)
 int D4NFilterUser::create_bucket(const DoutPrefixProvider* dpp,
                               const rgw_bucket& b,
                               const std::string& zonegroup_id,
-                              rgw_placement_rule& placement_rule,
-                              std::string& swift_ver_location,
+                              const rgw_placement_rule& placement_rule,
+                              const std::string& swift_ver_location,
                               const RGWQuotaInfo * pquota_info,
                               const RGWAccessControlPolicy& policy,
                               Attrs& attrs,
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index 5a2cd88896db..459e885fc240 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -75,8 +75,8 @@ class D4NFilterUser : public FilterUser {
     virtual int create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo* pquota_info,
                             const RGWAccessControlPolicy& policy,
                             Attrs& attrs,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index e9377f945360..156fb6d6031c 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -210,8 +210,8 @@ int MotrUser::list_buckets(const DoutPrefixProvider *dpp, const string& marker,
 int MotrUser::create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo* pquota_info,
                             const RGWAccessControlPolicy& policy,
                             Attrs& attrs,
@@ -234,24 +234,7 @@ int MotrUser::create_bucket(const DoutPrefixProvider* dpp,
 
   if (ret != -ENOENT) {
     *existed = true;
-    // if (swift_ver_location.empty()) {
-    //   swift_ver_location = bucket->get_info().swift_ver_location;
-    // }
-    // placement_rule.inherit_from(bucket->get_info().placement_rule);
-
-    // TODO: ACL policy
-    // // don't allow changes to the acl policy
-    //RGWAccessControlPolicy old_policy(ctx());
-    //int rc = rgw_op_get_bucket_policy_from_attr(
-    //           dpp, this, u, bucket->get_attrs(), &old_policy, y);
-    //if (rc >= 0 && old_policy != policy) {
-    //    bucket_out->swap(bucket);
-    //    return -EEXIST;
-    //}
   } else {
-
-    placement_rule.name = "default";
-    placement_rule.storage_class = "STANDARD";
     bucket = std::make_unique<MotrBucket>(store, b, this);
     bucket->set_attrs(attrs);
     *existed = false;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 6e6b9908061f..23854b466dea 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -222,8 +222,8 @@ class MotrUser : public StoreUser {
     virtual int create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo* pquota_info,
                             const RGWAccessControlPolicy& policy,
                             Attrs& attrs,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index b999839bf4e7..0831ebd97c5b 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -594,8 +594,8 @@ int POSIXUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma
 int POSIXUser::create_bucket(const DoutPrefixProvider* dpp,
 			      const rgw_bucket& b,
 			      const std::string& zonegroup_id,
-			      rgw_placement_rule& placement_rule,
-			      std::string& swift_ver_location,
+			      const rgw_placement_rule& placement_rule,
+			      const std::string& swift_ver_location,
 			      const RGWQuotaInfo * pquota_info,
 			      const RGWAccessControlPolicy& policy,
 			      Attrs& attrs,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 3695763c45f9..f78bbadbfe9c 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -128,8 +128,8 @@ class POSIXUser : public FilterUser {
   virtual int create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo* pquota_info,
                             const RGWAccessControlPolicy& policy,
                             Attrs& attrs,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 4cc3ab5972bd..b7a7732388bf 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -123,8 +123,8 @@ int RadosUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma
 int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
 				 const rgw_bucket& b,
 				 const std::string& zonegroup_id,
-				 rgw_placement_rule& placement_rule,
-				 std::string& swift_ver_location,
+				 const rgw_placement_rule& placement_rule,
+				 const std::string& swift_ver_location,
 				 const RGWQuotaInfo * pquota_info,
 				 const RGWAccessControlPolicy& policy,
 				 Attrs& attrs,
@@ -152,10 +152,6 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
 
   if (ret != -ENOENT) {
     *existed = true;
-    if (swift_ver_location.empty()) {
-      swift_ver_location = bucket->get_info().swift_ver_location;
-    }
-    placement_rule.inherit_from(bucket->get_info().placement_rule);
   } else {
     bucket = std::unique_ptr<Bucket>(new RadosBucket(store, b, this));
     *existed = false;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index ebf6531f1471..f542a68d0479 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -277,8 +277,8 @@ class RadosUser : public StoreUser {
     virtual int create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo * pquota_info,
                             const RGWAccessControlPolicy& policy,
 			    Attrs& attrs,
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 0307658aa0a6..6f42f03a118c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3384,6 +3384,11 @@ void RGWCreateBucket::execute(optional_yield y)
       /* Initialize info from req_state */
       info = tmp_bucket->get_info();
 
+      if (!swift_ver_location) {
+        swift_ver_location = info.swift_ver_location;
+      }
+      placement_rule.inherit_from(info.placement_rule);
+
       // don't allow changes to the acl policy
       RGWAccessControlPolicy old_policy(get_cct());
       int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 73122e06b4b7..d8072ca6f6f9 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -491,8 +491,8 @@ class User {
     virtual int create_bucket(const DoutPrefixProvider* dpp,
                             const rgw_bucket& b,
                             const std::string& zonegroup_id,
-                            rgw_placement_rule& placement_rule,
-                            std::string& swift_ver_location,
+                            const rgw_placement_rule& placement_rule,
+                            const std::string& swift_ver_location,
                             const RGWQuotaInfo* pquota_info,
                             const RGWAccessControlPolicy& policy,
 			    Attrs& attrs,
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 97da47eb6d38..5e429ba5c4c8 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -61,8 +61,8 @@ namespace rgw::sal {
   int DBUser::create_bucket(const DoutPrefixProvider *dpp,
       const rgw_bucket& b,
       const string& zonegroup_id,
-      rgw_placement_rule& placement_rule,
-      string& swift_ver_location,
+      const rgw_placement_rule& placement_rule,
+      const string& swift_ver_location,
       const RGWQuotaInfo * pquota_info,
       const RGWAccessControlPolicy& policy,
       Attrs& attrs,
@@ -79,7 +79,6 @@ namespace rgw::sal {
     bufferlist in_data;
     RGWBucketInfo master_info;
     rgw_bucket *pmaster_bucket = nullptr;
-    uint32_t *pmaster_num_shards = nullptr;
     real_time creation_time;
     std::unique_ptr<Bucket> bucket;
     obj_version objv, *pobjv = NULL;
@@ -90,28 +89,11 @@ namespace rgw::sal {
       return ret;
 
     if (ret != -ENOENT) {
-      RGWAccessControlPolicy old_policy(store->ctx());
       *existed = true;
-      if (swift_ver_location.empty()) {
-        swift_ver_location = bucket->get_info().swift_ver_location;
-      }
-      placement_rule.inherit_from(bucket->get_info().placement_rule);
-
-      // don't allow changes to the acl policy
-      /*    int r = rgw_op_get_bucket_policy_from_attr(dpp, this, this, bucket->get_attrs(),
-            &old_policy, y);
-            if (r >= 0 && old_policy != policy) {
-            bucket_out->swap(bucket);
-            return -EEXIST;
-            }*/
     } else {
       bucket = std::make_unique<DBBucket>(store, b, this);
       *existed = false;
       bucket->set_attrs(attrs);
-      // XXX: For now single default zone and STANDARD storage class
-      // supported.
-      placement_rule.name = "default";
-      placement_rule.storage_class = "STANDARD";
     }
 
     /*
@@ -123,8 +105,13 @@ namespace rgw::sal {
        zid = svc()->zone->get_zonegroup().get_id();
        } */
 
+    // XXX: For now single default zone and STANDARD storage class
+    // supported.
+    rgw_placement_rule selected_placement_rule;
+    selected_placement_rule.name = "default";
+    selected_placement_rule.storage_class = "STANDARD";
+
     if (*existed) {
-      rgw_placement_rule selected_placement_rule;
       /* XXX: Handle this when zone is implemented
          ret = svc()->zone->select_bucket_placement(this.get_info(),
          zid, placement_rule,
@@ -138,9 +125,9 @@ namespace rgw::sal {
 
       /* XXX: We may not need to send all these params. Cleanup the unused ones */
       ret = store->getDB()->create_bucket(dpp, this->get_info(), bucket->get_key(),
-          zid, placement_rule, swift_ver_location, pquota_info,
+          zid, selected_placement_rule, swift_ver_location, pquota_info,
           attrs, info, pobjv, &ep_objv, creation_time,
-          pmaster_bucket, pmaster_num_shards, y, exclusive);
+          pmaster_bucket, nullptr, y, exclusive);
       if (ret == -EEXIST) {
         *existed = true;
         ret = 0;
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 68fe4c443199..375a4524517c 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -91,8 +91,8 @@ class DBNotification : public StoreNotification {
       virtual int create_bucket(const DoutPrefixProvider* dpp,
           const rgw_bucket& b,
           const std::string& zonegroup_id,
-          rgw_placement_rule& placement_rule,
-          std::string& swift_ver_location,
+          const rgw_placement_rule& placement_rule,
+          const std::string& swift_ver_location,
           const RGWQuotaInfo* pquota_info,
           const RGWAccessControlPolicy& policy,
           Attrs& attrs,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index fb9e7aa1a286..3f3c5102cb42 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -543,8 +543,8 @@ int FilterUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& m
 int FilterUser::create_bucket(const DoutPrefixProvider* dpp,
 			      const rgw_bucket& b,
 			      const std::string& zonegroup_id,
-			      rgw_placement_rule& placement_rule,
-			      std::string& swift_ver_location,
+			      const rgw_placement_rule& placement_rule,
+			      const std::string& swift_ver_location,
 			      const RGWQuotaInfo * pquota_info,
 			      const RGWAccessControlPolicy& policy,
 			      Attrs& attrs,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 81452f22cd7d..5d81bea0e420 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -332,8 +332,8 @@ class FilterUser : public User {
   virtual int create_bucket(const DoutPrefixProvider* dpp,
 			    const rgw_bucket& b,
 			    const std::string& zonegroup_id,
-			    rgw_placement_rule& placement_rule,
-			    std::string& swift_ver_location,
+			    const rgw_placement_rule& placement_rule,
+			    const std::string& swift_ver_location,
 			    const RGWQuotaInfo* pquota_info,
 			    const RGWAccessControlPolicy& policy,
 			    Attrs& attrs,
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index e8656ac8e39e..05f0d291961a 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -78,7 +78,7 @@ class TestUser : public sal::StoreUser {
     return 0;
   }
 
-  virtual int create_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, const std::string& zonegroup_id, rgw_placement_rule& placement_rule, std::string& swift_ver_location, const RGWQuotaInfo* pquota_info, const RGWAccessControlPolicy& policy, sal::Attrs& attrs, RGWBucketInfo& info, obj_version& ep_objv, bool exclusive, bool obj_lock_enabled, bool* existed, req_info& req_info, std::unique_ptr<sal::Bucket>* bucket, optional_yield y) override {
+  virtual int create_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, const std::string& zonegroup_id, const rgw_placement_rule& placement_rule, const std::string& swift_ver_location, const RGWQuotaInfo* pquota_info, const RGWAccessControlPolicy& policy, sal::Attrs& attrs, RGWBucketInfo& info, obj_version& ep_objv, bool exclusive, bool obj_lock_enabled, bool* existed, req_info& req_info, std::unique_ptr<sal::Bucket>* bucket, optional_yield y) override {
     return 0;
   }
 

From ab2b45aa184ed3e2b48a98853b6eea87bfc2b31b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Jul 2022 17:25:32 -0400
Subject: [PATCH 0471/2492] rgw/rados: pass obj_lock_enabled into
 RGWRados::create_bucket

there's a retry loop here that may end up overwriting RGWBucketInfo, so
pass obj_lock_enabled and use it to update info.flags on each iteration

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc     |  4 ++++
 src/rgw/driver/rados/rgw_rados.h      |  1 +
 src/rgw/driver/rados/rgw_sal_rados.cc | 13 ++++---------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 2df4f77b673e..e2754eead727 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2276,6 +2276,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
                             const string& zonegroup_id,
                             const rgw_placement_rule& placement_rule,
                             const string& swift_ver_location,
+                            bool obj_lock_enabled,
                             const RGWQuotaInfo * pquota_info,
 			    map<std::string, bufferlist>& attrs,
                             RGWBucketInfo& info,
@@ -2322,6 +2323,9 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
     info.placement_rule = selected_placement_rule;
     info.swift_ver_location = swift_ver_location;
     info.swift_versioning = (!swift_ver_location.empty());
+    if (obj_lock_enabled) {
+      info.flags |= BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
+    }
 
     init_default_bucket_layout(cct, info.layout, svc.zone->get_zone(),
 			       rule_info.index_type);
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index d312e7355e22..1727bd84758f 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -632,6 +632,7 @@ class RGWRados
 		    const std::string& zonegroup_id,
 		    const rgw_placement_rule& placement_rule,
 		    const std::string& swift_ver_location,
+		    bool obj_lock_enabled,
 		    const RGWQuotaInfo * pquota_info,
 		    std::map<std::string,bufferlist>& attrs,
 		    RGWBucketInfo& bucket_info,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index b7a7732388bf..bc6efa461968 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -140,7 +140,7 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
   int ret;
   bufferlist in_data;
   RGWBucketInfo master_info;
-  rgw_bucket* pmaster_bucket;
+  rgw_bucket* pmaster_bucket = nullptr;
   real_time creation_time;
   std::unique_ptr<Bucket> bucket;
   obj_version objv,* pobjv = NULL;
@@ -174,13 +174,7 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
     pmaster_bucket= &master_info.bucket;
     creation_time = master_info.creation_time;
     pobjv = &objv;
-    if (master_info.obj_lock_enabled()) {
-      info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
-    }
-  } else {
-    pmaster_bucket = NULL;
-    if (obj_lock_enabled)
-      info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
+    obj_lock_enabled = master_info.obj_lock_enabled();
   }
 
   std::string zid = zonegroup_id;
@@ -201,7 +195,8 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
   } else {
 
     ret = store->getRados()->create_bucket(this->get_info(), bucket->get_key(),
-				    zid, placement_rule, swift_ver_location, pquota_info,
+				    zid, placement_rule, swift_ver_location,
+				    obj_lock_enabled, pquota_info,
 				    attrs, info, pobjv, &ep_objv, creation_time,
 				    pmaster_bucket, y, dpp, exclusive);
     if (ret == -EEXIST) {

From 392bc13b5658fc43660a8ebca23b3f433babd882 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 27 Jul 2022 09:27:40 -0400
Subject: [PATCH 0472/2492] rgw: remove legacy bucket placement and
 'radosgw-admin pool' commands

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_admin.cc         |  48 ----------
 src/rgw/services/svc_zone.cc | 169 ++---------------------------------
 src/rgw/services/svc_zone.h  |   7 --
 3 files changed, 5 insertions(+), 219 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 9e6f4c640008..ced41f87822c 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -686,9 +686,6 @@ enum class OPT {
   BUCKET_OBJECT_SHARD,
   BUCKET_RESYNC_ENCRYPTED_MULTIPART,
   POLICY,
-  POOL_ADD,
-  POOL_RM,
-  POOLS_LIST,
   LOG_LIST,
   LOG_SHOW,
   LOG_RM,
@@ -910,10 +907,6 @@ static SimpleCmd::Commands all_cmds = {
   { "bucket object shard", OPT::BUCKET_OBJECT_SHARD },
   { "bucket resync encrypted multipart", OPT::BUCKET_RESYNC_ENCRYPTED_MULTIPART },
   { "policy", OPT::POLICY },
-  { "pool add", OPT::POOL_ADD },
-  { "pool rm", OPT::POOL_RM },
-  { "pool list", OPT::POOLS_LIST },
-  { "pools list", OPT::POOLS_LIST },
   { "log list", OPT::LOG_LIST },
   { "log show", OPT::LOG_SHOW },
   { "log rm", OPT::LOG_RM },
@@ -7413,47 +7406,6 @@ int main(int argc, const char **argv)
     }
   }
 
-  if (opt_cmd == OPT::POOL_ADD) {
-    if (pool_name.empty()) {
-      cerr << "need to specify pool to add!" << std::endl;
-      exit(1);
-    }
-
-    int ret = static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->add_bucket_placement(dpp(), pool, null_yield);
-    if (ret < 0)
-      cerr << "failed to add bucket placement: " << cpp_strerror(-ret) << std::endl;
-  }
-
-  if (opt_cmd == OPT::POOL_RM) {
-    if (pool_name.empty()) {
-      cerr << "need to specify pool to remove!" << std::endl;
-      exit(1);
-    }
-
-    int ret = static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->remove_bucket_placement(dpp(), pool, null_yield);
-    if (ret < 0)
-      cerr << "failed to remove bucket placement: " << cpp_strerror(-ret) << std::endl;
-  }
-
-  if (opt_cmd == OPT::POOLS_LIST) {
-    set<rgw_pool> pools;
-    int ret = static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->list_placement_set(dpp(), pools, null_yield);
-    if (ret < 0) {
-      cerr << "could not list placement set: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
-    }
-    formatter->reset();
-    formatter->open_array_section("pools");
-    for (auto siter = pools.begin(); siter != pools.end(); ++siter) {
-      formatter->open_object_section("pool");
-      formatter->dump_string("name",  siter->to_str());
-      formatter->close_section();
-    }
-    formatter->close_section();
-    formatter->flush(cout);
-    cout << std::endl;
-  }
-
   if (opt_cmd == OPT::USAGE_SHOW) {
     uint64_t start_epoch = 0;
     uint64_t end_epoch = (uint64_t)-1;
diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc
index 180d9371222b..acf3b5b31cec 100644
--- a/src/rgw/services/svc_zone.cc
+++ b/src/rgw/services/svc_zone.cc
@@ -871,14 +871,7 @@ int RGWSI_Zone::select_new_bucket_location(const DoutPrefixProvider *dpp, const
 int RGWSI_Zone::select_bucket_location_by_rule(const DoutPrefixProvider *dpp, const rgw_placement_rule& location_rule, RGWZonePlacementInfo *rule_info, optional_yield y)
 {
   if (location_rule.name.empty()) {
-    /* we can only reach here if we're trying to set a bucket location from a bucket
-     * created on a different zone, using a legacy / default pool configuration
-     */
-    if (rule_info) {
-      return select_legacy_bucket_placement(dpp, rule_info, y);
-    }
-
-    return 0;
+    return -EINVAL;
   }
 
   /*
@@ -915,164 +908,12 @@ int RGWSI_Zone::select_bucket_placement(const DoutPrefixProvider *dpp, const RGW
                                         rgw_placement_rule *pselected_rule, RGWZonePlacementInfo *rule_info,
 					optional_yield y)
 {
-  if (!zone_params->placement_pools.empty()) {
-    return select_new_bucket_location(dpp, user_info, zonegroup_id, placement_rule,
-                                      pselected_rule, rule_info, y);
-  }
-
-  if (pselected_rule) {
-    pselected_rule->clear();
-  }
-
-  if (rule_info) {
-    return select_legacy_bucket_placement(dpp, rule_info, y);
-  }
-
-  return 0;
-}
-
-int RGWSI_Zone::select_legacy_bucket_placement(const DoutPrefixProvider *dpp, RGWZonePlacementInfo *rule_info,
-					       optional_yield y)
-{
-  bufferlist map_bl;
-  map<string, bufferlist> m;
-  string pool_name;
-  bool write_map = false;
-
-  rgw_raw_obj obj(zone_params->domain_root, avail_pools);
-
-  auto sysobj = sysobj_svc->get_obj(obj);
-  int ret = sysobj.rop().read(dpp, &map_bl, y);
-  if (ret < 0) {
-    goto read_omap;
-  }
-
-  try {
-    auto iter = map_bl.cbegin();
-    decode(m, iter);
-  } catch (buffer::error& err) {
-    ldpp_dout(dpp, 0) << "ERROR: couldn't decode avail_pools" << dendl;
-  }
-
-read_omap:
-  if (m.empty()) {
-    ret = sysobj.omap().get_all(dpp, &m, y);
-
-    write_map = true;
-  }
-
-  if (ret < 0 || m.empty()) {
-    vector<rgw_pool> pools;
-    string s = string("default.") + default_storage_pool_suffix;
-    pools.push_back(rgw_pool(s));
-    vector<int> retcodes;
-    bufferlist bl;
-    ret = rados_svc->pool().create(dpp, pools, &retcodes);
-    if (ret < 0)
-      return ret;
-    ret = sysobj.omap().set(dpp, s, bl, y);
-    if (ret < 0)
-      return ret;
-    m[s] = bl;
-  }
-
-  if (write_map) {
-    bufferlist new_bl;
-    encode(m, new_bl);
-    ret = sysobj.wop().write(dpp, new_bl, y);
-    if (ret < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl;
-    }
-  }
-
-  auto miter = m.begin();
-  if (m.size() > 1) {
-    // choose a pool at random
-    auto r = ceph::util::generate_random_number<size_t>(0, m.size() - 1);
-    std::advance(miter, r);
-  }
-  pool_name = miter->first;
-
-  rgw_pool pool = pool_name;
-
-  rule_info->storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr);
-  rule_info->data_extra_pool = pool_name;
-  rule_info->index_pool = pool_name;
-  rule_info->index_type = rgw::BucketIndexType::Normal;
-
-  return 0;
-}
-
-int RGWSI_Zone::update_placement_map(const DoutPrefixProvider *dpp, optional_yield y)
-{
-  bufferlist header;
-  map<string, bufferlist> m;
-  rgw_raw_obj obj(zone_params->domain_root, avail_pools);
-
-  auto sysobj = sysobj_svc->get_obj(obj);
-  int ret = sysobj.omap().get_all(dpp, &m, y);
-  if (ret < 0)
-    return ret;
-
-  bufferlist new_bl;
-  encode(m, new_bl);
-  ret = sysobj.wop().write(dpp, new_bl, y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl;
-  }
-
-  return ret;
-}
-
-int RGWSI_Zone::add_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& new_pool, optional_yield y)
-{
-  int ret = rados_svc->pool(new_pool).lookup();
-  if (ret < 0) { // DNE, or something
-    return ret;
-  }
-
-  rgw_raw_obj obj(zone_params->domain_root, avail_pools);
-  auto sysobj = sysobj_svc->get_obj(obj);
-
-  bufferlist empty_bl;
-  ret = sysobj.omap().set(dpp, new_pool.to_str(), empty_bl, y);
-
-  // don't care about return value
-  update_placement_map(dpp, y);
-
-  return ret;
-}
-
-int RGWSI_Zone::remove_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& old_pool, optional_yield y)
-{
-  rgw_raw_obj obj(zone_params->domain_root, avail_pools);
-  auto sysobj = sysobj_svc->get_obj(obj);
-  int ret = sysobj.omap().del(dpp, old_pool.to_str(), y);
-
-  // don't care about return value
-  update_placement_map(dpp, y);
-
-  return ret;
-}
-
-int RGWSI_Zone::list_placement_set(const DoutPrefixProvider *dpp, set<rgw_pool>& names, optional_yield y)
-{
-  bufferlist header;
-  map<string, bufferlist> m;
-
-  rgw_raw_obj obj(zone_params->domain_root, avail_pools);
-  auto sysobj = sysobj_svc->get_obj(obj);
-  int ret = sysobj.omap().get_all(dpp, &m, y);
-  if (ret < 0)
-    return ret;
-
-  names.clear();
-  map<string, bufferlist>::iterator miter;
-  for (miter = m.begin(); miter != m.end(); ++miter) {
-    names.insert(rgw_pool(miter->first));
+  if (zone_params->placement_pools.empty()) {
+    return -EINVAL; // legacy placement no longer supported
   }
 
-  return names.size();
+  return select_new_bucket_location(dpp, user_info, zonegroup_id, placement_rule,
+                                    pselected_rule, rule_info, y);
 }
 
 bool RGWSI_Zone::get_redirect_zone_endpoint(string *endpoint)
diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h
index 7b0a277c439f..3891b73ddb4d 100644
--- a/src/rgw/services/svc_zone.h
+++ b/src/rgw/services/svc_zone.h
@@ -67,8 +67,6 @@ class RGWSI_Zone : public RGWServiceInstance
   int init_zg_from_period(const DoutPrefixProvider *dpp, optional_yield y);
   int init_zg_from_local(const DoutPrefixProvider *dpp, optional_yield y);
 
-  int update_placement_map(const DoutPrefixProvider *dpp, optional_yield y);
-
   int create_default_zg(const DoutPrefixProvider *dpp, optional_yield y);
   int init_default_zone(const DoutPrefixProvider *dpp, optional_yield y);
 
@@ -137,17 +135,12 @@ class RGWSI_Zone : public RGWServiceInstance
   int select_bucket_placement(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const std::string& zonegroup_id,
                               const rgw_placement_rule& rule,
                               rgw_placement_rule *pselected_rule, RGWZonePlacementInfo *rule_info, optional_yield y);
-  int select_legacy_bucket_placement(const DoutPrefixProvider *dpp, RGWZonePlacementInfo *rule_info, optional_yield y);
   int select_new_bucket_location(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const std::string& zonegroup_id,
                                  const rgw_placement_rule& rule,
                                  rgw_placement_rule *pselected_rule_name, RGWZonePlacementInfo *rule_info,
 				 optional_yield y);
   int select_bucket_location_by_rule(const DoutPrefixProvider *dpp, const rgw_placement_rule& location_rule, RGWZonePlacementInfo *rule_info, optional_yield y);
 
-  int add_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& new_pool, optional_yield y);
-  int remove_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& old_pool, optional_yield y);
-  int list_placement_set(const DoutPrefixProvider *dpp, std::set<rgw_pool>& names, optional_yield y);
-
   bool is_meta_master() const;
 
   bool need_to_sync() const;

From 1b4e09eed15dab29e9b220747b7f976f85f0f6b2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 10:41:12 -0400
Subject: [PATCH 0473/2492] rgw/services: make set_attrs const correct

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.h   | 8 ++++----
 src/rgw/driver/rados/rgw_rados.cc   | 4 ++--
 src/rgw/driver/rados/rgw_rados.h    | 4 ++--
 src/rgw/driver/rados/rgw_tools.cc   | 2 +-
 src/rgw/driver/rados/rgw_tools.h    | 2 +-
 src/rgw/services/svc_bucket.h       | 4 ++--
 src/rgw/services/svc_bucket_sobj.cc | 4 ++--
 src/rgw/services/svc_bucket_sobj.h  | 4 ++--
 src/rgw/services/svc_meta_be_sobj.h | 6 +++---
 src/rgw/services/svc_sys_obj.h      | 4 ++--
 10 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h
index 0110eff33439..857d530aab50 100644
--- a/src/rgw/driver/rados/rgw_bucket.h
+++ b/src/rgw/driver/rados/rgw_bucket.h
@@ -498,7 +498,7 @@ class RGWBucketCtl {
       RGWObjVersionTracker *objv_tracker{nullptr};
       ceph::real_time mtime;
       bool exclusive{false};
-      std::map<std::string, bufferlist> *attrs{nullptr};
+      const std::map<std::string, bufferlist> *attrs{nullptr};
 
       PutParams() {}
 
@@ -517,7 +517,7 @@ class RGWBucketCtl {
         return *this;
       }
 
-      PutParams& set_attrs(std::map<std::string, bufferlist> *_attrs) {
+      PutParams& set_attrs(const std::map<std::string, bufferlist> *_attrs) {
         attrs = _attrs;
         return *this;
       }
@@ -582,7 +582,7 @@ class RGWBucketCtl {
                                                    nullptr: orig_info was not found (new bucket instance */
       ceph::real_time mtime;
       bool exclusive{false};
-      std::map<std::string, bufferlist> *attrs{nullptr};
+      const std::map<std::string, bufferlist> *attrs{nullptr};
       RGWObjVersionTracker *objv_tracker{nullptr};
 
       PutParams() {}
@@ -602,7 +602,7 @@ class RGWBucketCtl {
         return *this;
       }
 
-      PutParams& set_attrs(std::map<std::string, bufferlist> *_attrs) {
+      PutParams& set_attrs(const std::map<std::string, bufferlist> *_attrs) {
         attrs = _attrs;
         return *this;
       }
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index e2754eead727..1ec176374350 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -8587,7 +8587,7 @@ int RGWRados::try_refresh_bucket_info(RGWBucketInfo& info,
 }
 
 int RGWRados::put_bucket_instance_info(RGWBucketInfo& info, bool exclusive,
-                              real_time mtime, map<string, bufferlist> *pattrs,
+                              real_time mtime, const map<string, bufferlist> *pattrs,
                               const DoutPrefixProvider *dpp, optional_yield y)
 {
   return ctl.bucket->store_bucket_instance_info(info.bucket, info, y, dpp,
@@ -8598,7 +8598,7 @@ int RGWRados::put_bucket_instance_info(RGWBucketInfo& info, bool exclusive,
 }
 
 int RGWRados::put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, real_time mtime, obj_version *pep_objv,
-                                     map<string, bufferlist> *pattrs, bool create_entry_point,
+                                     const map<string, bufferlist> *pattrs, bool create_entry_point,
                                      const DoutPrefixProvider *dpp, optional_yield y)
 {
   bool create_head = !info.has_instance_obj || create_entry_point;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 1727bd84758f..18e818c4e893 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -1395,7 +1395,7 @@ class RGWRados
       std::map<RGWObjCategory, RGWStorageStats>& stats, std::string *max_marker, bool* syncstopped = NULL);
   int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB *cb);
 
-  int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, std::map<std::string, bufferlist> *pattrs, const DoutPrefixProvider *dpp, optional_yield y);
+  int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, const std::map<std::string, bufferlist> *pattrs, const DoutPrefixProvider *dpp, optional_yield y);
   /* xxx dang obj_ctx -> svc */
   int get_bucket_instance_info(const std::string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, std::map<std::string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp);
   int get_bucket_instance_info(const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, std::map<std::string, bufferlist> *pattrs, optional_yield y, const DoutPrefixProvider *dpp);
@@ -1420,7 +1420,7 @@ class RGWRados
 			      std::map<std::string, bufferlist> *pattrs = nullptr);
 
   int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, obj_version *pep_objv,
-			     std::map<std::string, bufferlist> *pattrs, bool create_entry_point,
+			     const std::map<std::string, bufferlist> *pattrs, bool create_entry_point,
                              const DoutPrefixProvider *dpp, optional_yield y);
 
   int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, std::string& tag, rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr);
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index cdc4be83a054..17c72012ab60 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -102,7 +102,7 @@ map<string, bufferlist>* no_change_attrs() {
 
 int rgw_put_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj,
                        const rgw_pool& pool, const string& oid, bufferlist& data, bool exclusive,
-                       RGWObjVersionTracker *objv_tracker, real_time set_mtime, optional_yield y, map<string, bufferlist> *pattrs)
+                       RGWObjVersionTracker *objv_tracker, real_time set_mtime, optional_yield y, const map<string, bufferlist> *pattrs)
 {
   map<string,bufferlist> no_attrs;
   if (!pattrs) {
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 66600856d162..607b064b1775 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -66,7 +66,7 @@ int rgw_put_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj,
                        bufferlist& data, bool exclusive,
                        RGWObjVersionTracker *objv_tracker,
                        real_time set_mtime, optional_yield y,
-                       std::map<std::string, bufferlist> *pattrs = nullptr);
+                       const std::map<std::string, bufferlist> *pattrs = nullptr);
 int rgw_get_system_obj(RGWSI_SysObj* svc_sysobj, const rgw_pool& pool,
                        const std::string& key, bufferlist& bl,
                        RGWObjVersionTracker *objv_tracker, real_time *pmtime,
diff --git a/src/rgw/services/svc_bucket.h b/src/rgw/services/svc_bucket.h
index 4a526e4f2480..caf6e029452b 100644
--- a/src/rgw/services/svc_bucket.h
+++ b/src/rgw/services/svc_bucket.h
@@ -49,7 +49,7 @@ class RGWSI_Bucket : public RGWServiceInstance
                                    RGWBucketEntryPoint& info,
                                    bool exclusive,
                                    real_time mtime,
-                                   std::map<std::string, bufferlist> *pattrs,
+                                   const std::map<std::string, bufferlist> *pattrs,
                                    RGWObjVersionTracker *objv_tracker,
                                    optional_yield y,
                                    const DoutPrefixProvider *dpp) = 0;
@@ -86,7 +86,7 @@ class RGWSI_Bucket : public RGWServiceInstance
                                                                               nullptr: orig_info was not found (new bucket instance */
                                  bool exclusive,
                                  real_time mtime,
-                                 std::map<std::string, bufferlist> *pattrs,
+                                 const std::map<std::string, bufferlist> *pattrs,
                                  optional_yield y,
                                  const DoutPrefixProvider *dpp) = 0;
 
diff --git a/src/rgw/services/svc_bucket_sobj.cc b/src/rgw/services/svc_bucket_sobj.cc
index 08a528015035..9927cc2d3a97 100644
--- a/src/rgw/services/svc_bucket_sobj.cc
+++ b/src/rgw/services/svc_bucket_sobj.cc
@@ -241,7 +241,7 @@ int RGWSI_Bucket_SObj::store_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx,
                                                     RGWBucketEntryPoint& info,
                                                     bool exclusive,
                                                     real_time mtime,
-                                                    map<string, bufferlist> *pattrs,
+                                                    const map<string, bufferlist> *pattrs,
                                                     RGWObjVersionTracker *objv_tracker,
                                                     optional_yield y,
                                                     const DoutPrefixProvider *dpp)
@@ -492,7 +492,7 @@ int RGWSI_Bucket_SObj::store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx,
                                                   std::optional<RGWBucketInfo *> orig_info,
                                                   bool exclusive,
                                                   real_time mtime,
-                                                  map<string, bufferlist> *pattrs,
+                                                  const map<string, bufferlist> *pattrs,
                                                   optional_yield y,
                                                   const DoutPrefixProvider *dpp)
 {
diff --git a/src/rgw/services/svc_bucket_sobj.h b/src/rgw/services/svc_bucket_sobj.h
index 8e9fe063c1b1..7a466ca37dd8 100644
--- a/src/rgw/services/svc_bucket_sobj.h
+++ b/src/rgw/services/svc_bucket_sobj.h
@@ -118,7 +118,7 @@ class RGWSI_Bucket_SObj : public RGWSI_Bucket
                                    RGWBucketEntryPoint& info,
                                    bool exclusive,
                                    real_time mtime,
-                                   std::map<std::string, bufferlist> *pattrs,
+                                   const std::map<std::string, bufferlist> *pattrs,
                                    RGWObjVersionTracker *objv_tracker,
                                    optional_yield y,
                                    const DoutPrefixProvider *dpp) override;
@@ -155,7 +155,7 @@ class RGWSI_Bucket_SObj : public RGWSI_Bucket
                                                                               nullptr: orig_info was not found (new bucket instance */
                                  bool exclusive,
                                  real_time mtime,
-                                 std::map<std::string, bufferlist> *pattrs,
+                                 const std::map<std::string, bufferlist> *pattrs,
                                  optional_yield y,
                                  const DoutPrefixProvider *dpp) override;
 
diff --git a/src/rgw/services/svc_meta_be_sobj.h b/src/rgw/services/svc_meta_be_sobj.h
index 304afc8bf2ad..bf1ad1e68c74 100644
--- a/src/rgw/services/svc_meta_be_sobj.h
+++ b/src/rgw/services/svc_meta_be_sobj.h
@@ -70,15 +70,15 @@ struct RGWSI_MBSObj_GetParams : public RGWSI_MetaBackend::GetParams {
 
 struct RGWSI_MBSObj_PutParams : public RGWSI_MetaBackend::PutParams {
   bufferlist bl;
-  std::map<std::string, bufferlist> *pattrs{nullptr};
+  const std::map<std::string, bufferlist> *pattrs{nullptr};
   bool exclusive{false};
 
   RGWSI_MBSObj_PutParams() {}
-  RGWSI_MBSObj_PutParams(std::map<std::string, bufferlist> *_pattrs,
+  RGWSI_MBSObj_PutParams(const std::map<std::string, bufferlist> *_pattrs,
                          const ceph::real_time& _mtime) : RGWSI_MetaBackend::PutParams(_mtime),
                                                           pattrs(_pattrs) {}
   RGWSI_MBSObj_PutParams(bufferlist& _bl,
-                         std::map<std::string, bufferlist> *_pattrs,
+                         const std::map<std::string, bufferlist> *_pattrs,
                          const ceph::real_time& _mtime,
                          bool _exclusive) : RGWSI_MetaBackend::PutParams(_mtime),
                                             bl(_bl),
diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h
index f3e217dbde94..292340dcbe32 100644
--- a/src/rgw/services/svc_sys_obj.h
+++ b/src/rgw/services/svc_sys_obj.h
@@ -108,13 +108,13 @@ class RGWSI_SysObj : public RGWServiceInstance
         return *this;
       }
 
-      WOp& set_attrs(std::map<std::string, bufferlist>& _attrs) {
+      WOp& set_attrs(const std::map<std::string, bufferlist>& _attrs) {
         attrs = _attrs;
         return *this;
       }
 
       WOp& set_attrs(std::map<std::string, bufferlist>&& _attrs) {
-        attrs = _attrs;
+        attrs = std::move(_attrs);
         return *this;
       }
 

From ad7c577a6731aa980d78def45f5d0a975915852d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 11:47:29 -0400
Subject: [PATCH 0474/2492] rgw/sal: rename loading get_bucket()s to
 load_bucket()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc           | 43 +++------
 src/rgw/driver/daos/rgw_sal_daos.h            | 17 ++--
 src/rgw/driver/motr/rgw_sal_motr.cc           | 34 ++-----
 src/rgw/driver/motr/rgw_sal_motr.h            |  9 +-
 src/rgw/driver/posix/rgw_sal_posix.cc         | 33 ++-----
 src/rgw/driver/posix/rgw_sal_posix.h          | 15 ++-
 src/rgw/driver/rados/rgw_bucket.cc            | 36 +++----
 src/rgw/driver/rados/rgw_cr_rados.h           |  2 +-
 src/rgw/driver/rados/rgw_cr_tools.cc          |  2 +-
 .../driver/rados/rgw_object_expirer_core.cc   |  2 +-
 src/rgw/driver/rados/rgw_rest_bucket.cc       |  4 +-
 src/rgw/driver/rados/rgw_rest_log.cc          | 10 +-
 src/rgw/driver/rados/rgw_sal_rados.cc         | 36 +++----
 src/rgw/driver/rados/rgw_sal_rados.h          |  9 +-
 src/rgw/driver/rados/rgw_tools.cc             |  5 +-
 src/rgw/driver/rados/rgw_user.cc              |  4 +-
 src/rgw/rgw_admin.cc                          |  8 +-
 src/rgw/rgw_flight.cc                         |  4 +-
 src/rgw/rgw_lc.cc                             |  2 +-
 src/rgw/rgw_op.cc                             | 94 +++++++++----------
 src/rgw/rgw_orphan.cc                         | 11 +--
 src/rgw/rgw_quota.cc                          |  6 +-
 src/rgw/rgw_rest_pubsub.cc                    |  6 +-
 src/rgw/rgw_rest_ratelimit.cc                 |  4 +-
 src/rgw/rgw_rest_swift.cc                     |  4 +-
 src/rgw/rgw_rest_usage.cc                     | 10 +-
 src/rgw/rgw_sal.h                             | 14 ++-
 src/rgw/rgw_sal_dbstore.cc                    | 36 +++----
 src/rgw/rgw_sal_dbstore.h                     |  9 +-
 src/rgw/rgw_sal_filter.cc                     | 46 +++------
 src/rgw/rgw_sal_filter.h                      | 15 ++-
 src/rgw/rgw_swift_auth.cc                     |  2 +-
 src/rgw/rgw_user.cc                           |  2 +-
 33 files changed, 218 insertions(+), 316 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index c4deba7f11f9..167c16d6be97 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -101,7 +101,7 @@ int DaosUser::create_bucket(
   std::unique_ptr<Bucket> bucket;
 
   // Look up the bucket. Create it if it doesn't exist.
-  ret = this->store->get_bucket(dpp, this, b, &bucket, y);
+  ret = this->store->load_bucket(dpp, this, b, &bucket, y);
   if (ret != 0 && ret != -ENOENT) {
     return ret;
   }
@@ -2249,45 +2249,30 @@ inline std::ostream& operator<<(std::ostream& out, const rgw_user* u) {
   return out << s;
 }
 
-int DaosStore::get_bucket(const DoutPrefixProvider* dpp, User* u,
-                          const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-                          optional_yield y) {
-  ldpp_dout(dpp, 20) << "DEBUG: get_bucket1: User: " << u << dendl;
-  int ret;
-  Bucket* bp;
-
-  bp = new DaosBucket(this, b, u);
-  ret = bp->load_bucket(dpp, y);
-  if (ret != 0) {
-    delete bp;
-    return ret;
-  }
-
-  bucket->reset(bp);
-  return 0;
+std::unique_ptr<Bucket> DaosStore::get_bucket(User* u, const RGWBucketInfo& i) {
+  /* Don't need to fetch the bucket info, use the provided one */
+  return std::make_unique<DaosBucket>(this, i, u);
 }
 
-int DaosStore::get_bucket(User* u, const RGWBucketInfo& i,
-                          std::unique_ptr<Bucket>* bucket) {
-  DaosBucket* bp;
-
-  bp = new DaosBucket(this, i, u);
-  /* Don't need to fetch the bucket info, use the provided one */
+int DaosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
+                           const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
+                           optional_yield y) {
+  ldpp_dout(dpp, 20) << "DEBUG: get_bucket1: User: " << u << dendl;
 
-  bucket->reset(bp);
-  return 0;
+  *bucket = std::make_unique<DaosBucket>(this, b, u);
+  return (*bucket)->load_bucket(dpp, y);
 }
 
-int DaosStore::get_bucket(const DoutPrefixProvider* dpp, User* u,
-                          const std::string& tenant, const std::string& name,
-                          std::unique_ptr<Bucket>* bucket, optional_yield y) {
+int DaosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
+                           const std::string& tenant, const std::string& name,
+                           std::unique_ptr<Bucket>* bucket, optional_yield y) {
   ldpp_dout(dpp, 20) << "get_bucket" << dendl;
   rgw_bucket b;
 
   b.tenant = tenant;
   b.name = name;
 
-  return get_bucket(dpp, u, b, bucket, y);
+  return load_bucket(dpp, u, b, bucket, y);
 }
 
 bool DaosStore::is_meta_master() { return true; }
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 84b640b8a6e0..23f723c0ce7c 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -895,15 +895,14 @@ class DaosStore : public StoreDriver {
                                 const std::string& user_str, optional_yield y,
                                 std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u,
-                         const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-                         optional_yield y) override;
-  virtual int get_bucket(User* u, const RGWBucketInfo& i,
-                         std::unique_ptr<Bucket>* bucket) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u,
-                         const std::string& tenant, const std::string& name,
-                         std::unique_ptr<Bucket>* bucket,
-                         optional_yield y) override;
+  std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
+  int load_bucket(const DoutPrefixProvider* dpp, User* u,
+                  const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
+                  optional_yield y) override;
+  int load_bucket(const DoutPrefixProvider* dpp, User* u,
+                  const std::string& tenant, const std::string& name,
+                  std::unique_ptr<Bucket>* bucket,
+                  optional_yield y) override;
   virtual bool is_meta_master() override;
   virtual int forward_request_to_master(const DoutPrefixProvider* dpp,
                                         User* user, obj_version* objv,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 156fb6d6031c..c7e4fc49bd48 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -228,7 +228,7 @@ int MotrUser::create_bucket(const DoutPrefixProvider* dpp,
   std::unique_ptr<Bucket> bucket;
 
   // Look up the bucket. Create it if it doesn't exist.
-  ret = this->store->get_bucket(dpp, this, b, &bucket, y);
+  ret = this->store->load_bucket(dpp, this, b, &bucket, y);
   if (ret < 0 && ret != -ENOENT)
     return ret;
 
@@ -3242,41 +3242,27 @@ std::unique_ptr<Object> MotrStore::get_object(const rgw_obj_key& k)
 }
 
 
-int MotrStore::get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+std::unique_ptr<Bucket> MotrStore::get_bucket(User* u, const RGWBucketInfo& i)
 {
-  int ret;
-  Bucket* bp;
-
-  bp = new MotrBucket(this, b, u);
-  ret = bp->load_bucket(dpp, y);
-  if (ret < 0) {
-    delete bp;
-    return ret;
-  }
-
-  bucket->reset(bp);
-  return 0;
+  /* Don't need to fetch the bucket info, use the provided one */
+  return std::make_unique<MotrBucket>(this, i, u);
 }
 
-int MotrStore::get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket)
+int MotrStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+                           std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
-  Bucket* bp;
-
-  bp = new MotrBucket(this, i, u);
-  /* Don't need to fetch the bucket info, use the provided one */
-
-  bucket->reset(bp);
-  return 0;
+  *bucket = std::make_unique<MotrBucket>(this, b, u);
+  return (*bucket)->load_bucket(dpp, y);
 }
 
-int MotrStore::get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int MotrStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   rgw_bucket b;
 
   b.tenant = tenant;
   b.name = name;
 
-  return get_bucket(dpp, u, b, bucket, y);
+  return load_bucket(dpp, u, b, bucket, y);
 }
 
 bool MotrStore::is_meta_master()
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 23854b466dea..ec5eb347cf22 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -992,9 +992,12 @@ class MotrStore : public StoreDriver {
     virtual int get_user_by_email(const DoutPrefixProvider *dpp, const std::string& email, optional_yield y, std::unique_ptr<User>* user) override;
     virtual int get_user_by_swift(const DoutPrefixProvider *dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-    virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-    virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
-    virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+    std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
+    int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+    int load_bucket(const DoutPrefixProvider *dpp, User* u,
+                    const std::string& tenant, const std::string& name,
+                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
     virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
         bufferlist& in_data, JSONParser *jp, req_info& info,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 0831ebd97c5b..40f509b28d74 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -393,41 +393,26 @@ std::unique_ptr<Object> POSIXDriver::get_object(const rgw_obj_key& k)
   return std::make_unique<POSIXObject>(this, k);
 }
 
-int POSIXDriver::get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
-  int ret;
-  Bucket* bp;
-
-  bp = new POSIXBucket(this, root_fd, b, u);
-  ret = bp->load_bucket(dpp, y);
-  if (ret < 0) {
-    delete bp;
-    return ret;
-  }
-
-  bucket->reset(bp);
-  return 0;
+  *bucket = std::make_unique<POSIXBucket>(this, root_fd, b, u);
+  return (*bucket)->load_bucket(dpp, y);
 }
 
-int POSIXDriver::get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket)
+std::unique_ptr<Bucket> POSIXDriver::get_bucket(User* u, const RGWBucketInfo& i)
 {
-  Bucket* bp;
-
-  bp = new POSIXBucket(this, root_fd, i, u);
   /* Don't need to fetch the bucket info, use the provided one */
-
-  bucket->reset(bp);
-  return 0;
+  return std::make_unique<POSIXBucket>(this, root_fd, i, u);
 }
 
-int POSIXDriver::get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   rgw_bucket b;
 
   b.tenant = tenant;
   b.name = name;
 
-  return get_bucket(dpp, u, b, bucket, y);
+  return load_bucket(dpp, u, b, bucket, y);
 }
 
 std::string POSIXDriver::zone_unique_trans_id(const uint64_t unique_num)
@@ -612,7 +597,7 @@ int POSIXUser::create_bucket(const DoutPrefixProvider* dpp,
   {
     std::unique_ptr<rgw::sal::Bucket> bucket;
 
-    int ret = driver->get_bucket(dpp, this, b, &bucket, y);
+    int ret = driver->load_bucket(dpp, this, b, &bucket, y);
     if (ret >= 0) {
       *existed = true;
       // Bucket exists.  Check owner comparison
@@ -730,7 +715,7 @@ int POSIXDriver::mint_listing_entry(const std::string &bname,
     POSIXObject *pobj;
     int ret;
 
-    ret = get_bucket(nullptr, nullptr, std::string(), bname, &b, null_yield);
+    ret = load_bucket(nullptr, nullptr, std::string(), bname, &b, null_yield);
     if (ret < 0)
       return ret;
 
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index f78bbadbfe9c..53cd05823b6c 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -52,14 +52,13 @@ class POSIXDriver : public FilterDriver {
 				std::string& user_str, optional_yield y,
 				std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-  virtual int get_bucket(User* u, const RGWBucketInfo& i,
-			 std::unique_ptr<Bucket>* bucket) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const
-			 rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-			 optional_yield y) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const
-			 std::string& tenant, const std::string& name,
-			 std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+  virtual std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i)  override;
+  virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const
+			  rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
+			  optional_yield y) override;
+  virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const
+			  std::string& tenant, const std::string& name,
+			  std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
 
   virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 04fa15381390..1c4341f438f3 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -109,7 +109,7 @@ void check_bad_user_bucket_mapping(rgw::sal::Driver* driver, rgw::sal::User& use
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      int r = driver->get_bucket(dpp, &user, user.get_tenant(), ent.bucket.name, &bucket, y);
+      int r = driver->load_bucket(dpp, &user, user.get_tenant(), ent.bucket.name, &bucket, y);
       if (r < 0) {
         ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << bucket << dendl;
         continue;
@@ -181,7 +181,7 @@ int RGWBucket::init(rgw::sal::Driver* _driver, RGWBucketAdminOpState& op_state,
     bucket_name = bucket_name.substr(pos + 1);
   }
 
-  int r = driver->get_bucket(dpp, user.get(), tenant, bucket_name, &bucket, y);
+  int r = driver->load_bucket(dpp, user.get(), tenant, bucket_name, &bucket, y);
   if (r < 0) {
       set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name);
       return r;
@@ -1242,8 +1242,8 @@ int RGWBucketAdminOp::remove_bucket(rgw::sal::Driver* driver, RGWBucketAdminOpSt
   std::unique_ptr<rgw::sal::Bucket> bucket;
   std::unique_ptr<rgw::sal::User> user = driver->get_user(op_state.get_user_id());
 
-  int ret = driver->get_bucket(dpp, user.get(), user->get_tenant(), op_state.get_bucket_name(),
-			      &bucket, y);
+  int ret = driver->load_bucket(dpp, user.get(), user->get_tenant(),
+                                op_state.get_bucket_name(), &bucket, y);
   if (ret < 0)
     return ret;
 
@@ -1284,7 +1284,7 @@ static int bucket_stats(rgw::sal::Driver* driver,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   map<RGWObjCategory, RGWStorageStats> stats;
 
-  int ret = driver->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
+  int ret = driver->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
   if (ret < 0) {
     return ret;
   }
@@ -1407,7 +1407,7 @@ int RGWBucketAdminOp::limit_check(rgw::sal::Driver* driver,
 	uint64_t num_objects = 0;
 
 	std::unique_ptr<rgw::sal::Bucket> bucket;
-	ret = driver->get_bucket(dpp, user.get(), ent.bucket, &bucket, y);
+	ret = driver->load_bucket(dpp, user.get(), ent.bucket, &bucket, y);
 	if (ret < 0)
 	  continue;
 
@@ -1591,7 +1591,7 @@ void get_stale_instances(rgw::sal::Driver* driver, const std::string& bucket_nam
     std::unique_ptr<rgw::sal::Bucket> bucket;
     rgw_bucket rbucket;
     rgw_bucket_parse_bucket_key(driver->ctx(), bucket_instance, &rbucket, nullptr);
-    int r = driver->get_bucket(dpp, nullptr, rbucket, &bucket, y);
+    int r = driver->load_bucket(dpp, nullptr, rbucket, &bucket, y);
     if (r < 0){
       // this can only happen if someone deletes us right when we're processing
       ldpp_dout(dpp, -1) << "Bucket instance is invalid: " << bucket_instance
@@ -1611,7 +1611,7 @@ void get_stale_instances(rgw::sal::Driver* driver, const std::string& bucket_nam
   auto [tenant, bname] = split_tenant(bucket_name);
   RGWBucketInfo cur_bucket_info;
   std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-  int r = driver->get_bucket(dpp, nullptr, tenant, bname, &cur_bucket, y);
+  int r = driver->load_bucket(dpp, nullptr, tenant, bname, &cur_bucket, y);
   if (r < 0) {
     if (r == -ENOENT) {
       // bucket doesn't exist, everything is stale then
@@ -1745,8 +1745,7 @@ int RGWBucketAdminOp::clear_stale_instances(rgw::sal::Driver* driver,
                       Formatter *formatter,
                       rgw::sal::Driver* driver){
                      for (const auto &binfo: lst) {
-		       std::unique_ptr<rgw::sal::Bucket> bucket;
-		       driver->get_bucket(nullptr, binfo, &bucket);
+		       auto bucket = driver->get_bucket(nullptr, binfo);
 		       int ret = bucket->purge_instance(dpp, y);
                        if (ret == 0){
                          auto md_key = "bucket.instance:" + binfo.bucket.get_key();
@@ -1768,7 +1767,7 @@ static int fix_single_bucket_lc(rgw::sal::Driver* driver,
                                 const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
+  int ret = driver->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
   if (ret < 0) {
     // TODO: Should we handle the case where the bucket could've been removed between
     // listing and fetching?
@@ -1938,11 +1937,7 @@ int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::Driver* driver,
     ldpp_dout(dpp, -1) << "failed to initialize bucket" << dendl;
     return ret;
   }
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = driver->get_bucket(nullptr, admin_bucket.get_bucket_info(), &bucket);
-  if (ret < 0) {
-    return ret;
-  }
+  auto bucket = driver->get_bucket(nullptr, admin_bucket.get_bucket_info());
 
   return fix_bucket_obj_expiry(dpp, driver, bucket.get(), flusher, dry_run, y);
 }
@@ -2623,14 +2618,7 @@ int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp
 
   /* update lifecyle policy */
   {
-    std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = bihandler->driver->get_bucket(nullptr, bci.info, &bucket);
-    if (ret < 0) {
-      ldpp_dout(dpp, 0) << __func__ << " failed to get_bucket(...) for "
-			<< bci.info.bucket.name
-			<< dendl;
-      return ret;
-    }
+    auto bucket = bihandler->driver->get_bucket(nullptr, bci.info);
 
     auto lc = bihandler->driver->get_rgwlc();
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index 3b192f198183..ff7a0efb481a 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -1350,7 +1350,7 @@ class RGWAsyncRemoveObj : public RGWAsyncRadosRequest {
     if (_zones_trace) {
       zones_trace = *_zones_trace;
     }
-    store->get_bucket(nullptr, _bucket_info, &bucket);
+    bucket = store->get_bucket(nullptr, _bucket_info);
     obj = bucket->get_object(_key);
   }
 };
diff --git a/src/rgw/driver/rados/rgw_cr_tools.cc b/src/rgw/driver/rados/rgw_cr_tools.cc
index 65cb67505049..10d17a49b380 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.cc
+++ b/src/rgw/driver/rados/rgw_cr_tools.cc
@@ -100,7 +100,7 @@ int RGWGetUserInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 template<>
 int RGWGetBucketInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 {
-  return store->get_bucket(dpp, nullptr, params.tenant, params.bucket_name, &result->bucket, null_yield);
+  return store->load_bucket(dpp, nullptr, params.tenant, params.bucket_name, &result->bucket, null_yield);
 }
 
 template<>
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc
index 2c931cf909d5..ba6580b89b07 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.cc
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc
@@ -201,7 +201,7 @@ int RGWObjectExpirer::garbage_single_object(const DoutPrefixProvider *dpp, objex
   RGWBucketInfo bucket_info;
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int ret = driver->get_bucket(dpp, nullptr, rgw_bucket(hint.tenant, hint.bucket_name, hint.bucket_id), &bucket, null_yield);
+  int ret = driver->load_bucket(dpp, nullptr, rgw_bucket(hint.tenant, hint.bucket_name, hint.bucket_id), &bucket, null_yield);
   if (-ENOENT == ret) {
     ldpp_dout(dpp, 15) << "NOTICE: cannot find bucket = " \
         << hint.bucket_name << ". The object must be already removed" << dendl;
diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index 9c811efa42d7..66aceabee96f 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -232,7 +232,7 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->get_bucket(s, nullptr, string(), bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, string(), bucket_name, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "get_bucket returned ret=" << op_ret << dendl;
     if (op_ret == -ENOENT) {
@@ -299,7 +299,7 @@ void RGWOp_Set_Bucket_Quota::execute(optional_yield y)
   }
   if (use_http_params) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->get_bucket(s, nullptr, uid.tenant, bucket_name, &bucket, s->yield);
+    op_ret = driver->load_bucket(s, nullptr, uid.tenant, bucket_name, &bucket, s->yield);
     if (op_ret < 0) {
       return;
     }
diff --git a/src/rgw/driver/rados/rgw_rest_log.cc b/src/rgw/driver/rados/rgw_rest_log.cc
index c2e9b7cfa187..d331097f10b9 100644
--- a/src/rgw/driver/rados/rgw_rest_log.cc
+++ b/src/rgw/driver/rados/rgw_rest_log.cc
@@ -414,7 +414,7 @@ void RGWOp_BILog_List::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->get_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -541,7 +541,7 @@ void RGWOp_BILog_Info::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->get_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -635,7 +635,7 @@ void RGWOp_BILog_Delete::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->get_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -985,7 +985,7 @@ void RGWOp_BILog_Status::execute(optional_yield y)
 
   // read the bucket instance info for num_shards
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->get_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 4) << "failed to read bucket info: " << cpp_strerror(op_ret) << dendl;
     return;
@@ -1069,7 +1069,7 @@ void RGWOp_BILog_Status::execute(optional_yield y)
     if (*pipe.dest.bucket != pinfo->bucket) {
       opt_dest_info.emplace();
       std::unique_ptr<rgw::sal::Bucket> dest_bucket;
-      op_ret = driver->get_bucket(s, nullptr, *pipe.dest.bucket, &dest_bucket, y);
+      op_ret = driver->load_bucket(s, nullptr, *pipe.dest.bucket, &dest_bucket, y);
       if (op_ret < 0) {
         ldpp_dout(this, 4) << "failed to read target bucket info (bucket=: " << cpp_strerror(op_ret) << dendl;
         return;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index bc6efa461968..2c526ec940d2 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -146,7 +146,7 @@ int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
   obj_version objv,* pobjv = NULL;
 
   /* If it exists, look it up; otherwise create it */
-  ret = store->get_bucket(dpp, this, b, &bucket, y);
+  ret = store->load_bucket(dpp, this, b, &bucket, y);
   if (ret < 0 && ret != -ENOENT)
     return ret;
 
@@ -1055,41 +1055,29 @@ std::unique_ptr<Object> RadosStore::get_object(const rgw_obj_key& k)
   return std::make_unique<RadosObject>(this, k);
 }
 
-int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+std::unique_ptr<Bucket> RadosStore::get_bucket(User* u, const RGWBucketInfo& i)
 {
-  int ret;
-  Bucket* bp;
-
-  bp = new RadosBucket(this, b, u);
-  ret = bp->load_bucket(dpp, y);
-  if (ret < 0) {
-    delete bp;
-    return ret;
-  }
-
-  bucket->reset(bp);
-  return 0;
+  /* Don't need to fetch the bucket info, use the provided one */
+  return std::make_unique<RadosBucket>(this, i, u);
 }
 
-int RadosStore::get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket)
+int RadosStore::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+                            std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
-  Bucket* bp;
-
-  bp = new RadosBucket(this, i, u);
-  /* Don't need to fetch the bucket info, use the provided one */
-
-  bucket->reset(bp);
-  return 0;
+  *bucket = std::make_unique<RadosBucket>(this, b, u);
+  return (*bucket)->load_bucket(dpp, y);
 }
 
-int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int RadosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
+                            const std::string& tenant, const std::string& name,
+                            std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   rgw_bucket b;
 
   b.tenant = tenant;
   b.name = name;
 
-  return get_bucket(dpp, u, b, bucket, y);
+  return load_bucket(dpp, u, b, bucket, y);
 }
 
 bool RadosStore::is_meta_master()
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index f542a68d0479..84a0b6bf9513 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -137,9 +137,12 @@ class RadosStore : public StoreDriver {
     virtual int get_user_by_email(const DoutPrefixProvider* dpp, const std::string& email, optional_yield y, std::unique_ptr<User>* user) override;
     virtual int get_user_by_swift(const DoutPrefixProvider* dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-    virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-    virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
-    virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+    std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
+    int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+    int load_bucket(const DoutPrefixProvider* dpp, User* u,
+                    const std::string& tenant, const std::string& name,
+                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
     virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
 					  bufferlist& in_data, JSONParser* jp, req_info& info,
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 17c72012ab60..a3877c0b4678 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -285,7 +285,7 @@ int RGWDataAccess::Bucket::finish_init()
 int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = sd->driver->get_bucket(dpp, nullptr, tenant, name, &bucket, y);
+  int ret = sd->driver->load_bucket(dpp, nullptr, tenant, name, &bucket, y);
   if (ret < 0) {
     return ret;
   }
@@ -327,8 +327,7 @@ int RGWDataAccess::Object::put(bufferlist& data,
 
   rgw::BlockingAioThrottle aio(driver->ctx()->_conf->rgw_put_obj_min_window_size);
 
-  std::unique_ptr<rgw::sal::Bucket> b;
-  driver->get_bucket(NULL, bucket_info, &b);
+  std::unique_ptr<rgw::sal::Bucket> b = driver->get_bucket(nullptr, bucket_info);
   std::unique_ptr<rgw::sal::Object> obj = b->get_object(key);
 
   auto& owner = bucket->policy.get_owner();
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index f81254bcfb0e..97776f9d5a21 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -1579,7 +1579,7 @@ int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->get_bucket(dpp, old_user.get(), ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, old_user.get(), ent.bucket, &bucket, y);
       if (ret < 0) {
         set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name());
         return ret;
@@ -1785,7 +1785,7 @@ int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->get_bucket(dpp, user, ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, user, ent.bucket, &bucket, y);
       if (ret < 0) {
         set_err_msg(err_msg, "unable to load bucket " + ent.bucket.name);
         return ret;
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index ced41f87822c..061464f0c2cc 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1192,7 +1192,7 @@ class StoreDestructor {
 static int init_bucket(rgw::sal::User* user, const rgw_bucket& b,
                        std::unique_ptr<rgw::sal::Bucket>* bucket)
 {
-  return driver->get_bucket(dpp(), user, b, bucket, null_yield);
+  return driver->load_bucket(dpp(), user, b, bucket, null_yield);
 }
 
 static int init_bucket(rgw::sal::User* user,
@@ -1407,7 +1407,7 @@ int set_bucket_quota(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_size, bool have_max_objects)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->get_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
@@ -1431,7 +1431,7 @@ int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_read_bytes, bool have_max_write_bytes)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->get_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
@@ -1534,7 +1534,7 @@ int show_bucket_ratelimit(rgw::sal::Driver* driver, const string& tenant_name,
                           const string& bucket_name, Formatter *formatter)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->get_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
diff --git a/src/rgw/rgw_flight.cc b/src/rgw/rgw_flight.cc
index f37d934b333d..7cc8a460dc4c 100644
--- a/src/rgw/rgw_flight.cc
+++ b/src/rgw/rgw_flight.cc
@@ -688,8 +688,8 @@ arw::Status FlightServer::DoGet(const flt::ServerCallContext &context,
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  ret = driver->get_bucket(&dp, &(*user), fd.tenant_name, fd.bucket_name,
-			   &bucket, null_yield);
+  ret = driver->load_bucket(&dp, &(*user), fd.tenant_name, fd.bucket_name,
+                            &bucket, null_yield);
   if (ret < 0) {
     ERROR << "get_bucket returned " << ret << dendl;
     // TODO return something
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 15e70977e726..862a2d0191c9 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1615,7 +1615,7 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     return 0;
   }
 
-  int ret = driver->get_bucket(this, nullptr, bucket_tenant, bucket_name, &bucket, null_yield);
+  int ret = driver->load_bucket(this, nullptr, bucket_tenant, bucket_name, &bucket, null_yield);
   if (ret < 0) {
     ldpp_dout(this, 0) << "LC:get_bucket for " << bucket_name
 		       << " failed" << dendl;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6f42f03a118c..607633534f10 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -490,10 +490,10 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   /* check if copy source is within the current domain */
   if (!s->src_bucket_name.empty()) {
     std::unique_ptr<rgw::sal::Bucket> src_bucket;
-    ret = driver->get_bucket(dpp, nullptr,
-                             rgw_bucket_key(s->src_tenant_name,
-                                            s->src_bucket_name),
-                             &src_bucket, y);
+    ret = driver->load_bucket(dpp, nullptr,
+                              rgw_bucket_key(s->src_tenant_name,
+                                             s->src_bucket_name),
+                              &src_bucket, y);
     if (ret == 0) {
       string& zonegroup = src_bucket->get_info().zonegroup;
       s->local_source = driver->get_zone()->get_zonegroup().equals(zonegroup);
@@ -513,7 +513,10 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 
     /* This is the only place that s->bucket is created.  It should never be
      * overwritten. */
-    ret = driver->get_bucket(dpp, s->user.get(), rgw_bucket(s->bucket_tenant, s->bucket_name, s->bucket_instance_id), &s->bucket, y);
+    ret = driver->load_bucket(dpp, s->user.get(),
+                              rgw_bucket(s->bucket_tenant, s->bucket_name,
+                                         s->bucket_instance_id),
+                              &s->bucket, y);
     if (ret < 0) {
       if (ret != -ENOENT) {
 	string bucket_log;
@@ -1936,7 +1939,8 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
     map<string, bufferlist> bucket_attrs;
-    r = driver->get_bucket(this, s->user.get(), s->user->get_tenant(), bucket_name, &ubucket, y);
+    r = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
+                            bucket_name, &ubucket, y);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
 		       << bucket_name << dendl;
@@ -2066,7 +2070,8 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
 	RGWAccessControlPolicy& _bucket_acl = allocated_acls.back();
 
 	std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-	int r = driver->get_bucket(this, s->user.get(), s->user->get_tenant(), bucket_name, &tmp_bucket, y);
+	int r = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
+                                    bucket_name, &tmp_bucket, y);
         if (r < 0) {
           ldpp_dout(this, 0) << "could not get bucket info for bucket="
 			   << bucket_name << dendl;
@@ -2974,7 +2979,7 @@ void RGWStatBucket::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->get_bucket(this, s->user.get(), s->bucket->get_key(), &bucket, y);
+  op_ret = driver->load_bucket(this, s->user.get(), s->bucket->get_key(), &bucket, y);
   if (op_ret) {
     return;
   }
@@ -3368,8 +3373,8 @@ void RGWCreateBucket::execute(optional_yield y)
    * specific request */
   {
     std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-    op_ret = driver->get_bucket(this, s->user.get(), s->bucket_tenant,
-			       s->bucket_name, &tmp_bucket, y);
+    op_ret = driver->load_bucket(this, s->user.get(), s->bucket_tenant,
+                                 s->bucket_name, &tmp_bucket, y);
     if (op_ret < 0 && op_ret != -ENOENT)
       return;
     s->bucket_exists = (op_ret != -ENOENT);
@@ -3668,8 +3673,8 @@ int RGWPutObj::init_processing(optional_yield y) {
       }
     }
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = driver->get_bucket(this, s->user.get(), copy_source_tenant_name, copy_source_bucket_name,
-			      &bucket, y);
+    ret = driver->load_bucket(this, s->user.get(), copy_source_tenant_name,
+                              copy_source_bucket_name, &bucket, y);
     if (ret < 0) {
       ldpp_dout(this, 5) << __func__ << "(): get_bucket() returned ret=" << ret << dendl;
       if (ret == -ENOENT) {
@@ -3730,20 +3735,16 @@ int RGWPutObj::verify_permission(optional_yield y)
     RGWAccessControlPolicy cs_acl(s->cct);
     boost::optional<Policy> policy;
     map<string, bufferlist> cs_attrs;
-    std::unique_ptr<rgw::sal::Bucket> cs_bucket;
-    int ret = driver->get_bucket(NULL, copy_source_bucket_info, &cs_bucket);
-    if (ret < 0)
-      return ret;
-
-    std::unique_ptr<rgw::sal::Object> cs_object =
-      cs_bucket->get_object(rgw_obj_key(copy_source_object_name, copy_source_version_id));
-
+    auto cs_bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+    auto cs_object = cs_bucket->get_object(rgw_obj_key(copy_source_object_name,
+                                                       copy_source_version_id));
     cs_object->set_atomic();
     cs_object->set_prefetch_data();
 
     /* check source object permissions */
-    if (ret = read_obj_policy(this, driver, s, copy_source_bucket_info, cs_attrs, &cs_acl, nullptr,
-			policy, cs_bucket.get(), cs_object.get(), y, true); ret < 0) {
+    int ret = read_obj_policy(this, driver, s, copy_source_bucket_info, cs_attrs, &cs_acl, nullptr,
+                              policy, cs_bucket.get(), cs_object.get(), y, true);
+    if (ret < 0) {
       return ret;
     }
 
@@ -3797,10 +3798,10 @@ int RGWPutObj::verify_permission(optional_yield y)
       return -EACCES;
   }
 
-  auto op_ret = get_params(y);
-  if (op_ret < 0) {
-    ldpp_dout(this, 20) << "get_params() returned ret=" << op_ret << dendl;
-    return op_ret;
+  int ret = get_params(y);
+  if (ret < 0) {
+    ldpp_dout(this, 20) << "get_params() returned ret=" << ret << dendl;
+    return ret;
   }
 
   if (s->iam_policy || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
@@ -3920,13 +3921,10 @@ int RGWPutObj::get_data(const off_t fst, const off_t lst, bufferlist& bl)
   new_ofs = fst;
   new_end = lst;
 
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = driver->get_bucket(nullptr, copy_source_bucket_info, &bucket);
-  if (ret < 0)
-    return ret;
-
-  std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(rgw_obj_key(copy_source_object_name, copy_source_version_id));
-  std::unique_ptr<rgw::sal::Object::ReadOp> read_op(obj->get_read_op());
+  auto bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+  auto obj = bucket->get_object(rgw_obj_key(copy_source_object_name,
+                                            copy_source_version_id));
+  auto read_op = obj->get_read_op();
 
   ret = read_op->prepare(s->yield, this);
   if (ret < 0)
@@ -4184,14 +4182,9 @@ void RGWPutObj::execute(optional_yield y)
     return;
   }
   if ((! copy_source.empty()) && !copy_source_range) {
-    std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->get_bucket(nullptr, copy_source_bucket_info, &bucket);
-    if (op_ret < 0) {
-      ldpp_dout(this, 0) << "ERROR: failed to get bucket with error" << op_ret << dendl;
-      return;
-    }
-    std::unique_ptr<rgw::sal::Object> obj =
-      bucket->get_object(rgw_obj_key(copy_source_object_name, copy_source_version_id));
+    auto bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+    auto obj = bucket->get_object(rgw_obj_key(copy_source_object_name,
+                                              copy_source_version_id));
 
     RGWObjState *astate;
     op_ret = obj->get_obj_state(this, &astate, s->yield);
@@ -5363,10 +5356,10 @@ int RGWCopyObj::init_processing(optional_yield y)
     return op_ret;
   }
 
-  op_ret = driver->get_bucket(this, s->user.get(),
-                              rgw_bucket_key(s->src_tenant_name,
-                                             s->src_bucket_name),
-                              &src_bucket, y);
+  op_ret = driver->load_bucket(this, s->user.get(),
+                               rgw_bucket_key(s->src_tenant_name,
+                                              s->src_bucket_name),
+                               &src_bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
       op_ret = -ERR_NO_SUCH_BUCKET;
@@ -7322,12 +7315,8 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
   ACLOwner bowner;
   RGWObjVersionTracker ot;
 
-  int ret = driver->get_bucket(dpp, s->user.get(), s->user->get_tenant(), path.bucket_name, &bucket, y);
-  if (ret < 0) {
-    goto binfo_fail;
-  }
-
-  ret = bucket->load_bucket(dpp, s->yield);
+  int ret = driver->load_bucket(dpp, s->user.get(), s->user->get_tenant(),
+                                path.bucket_name, &bucket, y);
   if (ret < 0) {
     goto binfo_fail;
   }
@@ -7700,7 +7689,8 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   ACLOwner bowner;
 
-  op_ret = driver->get_bucket(this, s->user.get(), rgw_bucket(rgw_bucket_key(s->user->get_tenant(), bucket_name)), &bucket, y);
+  op_ret = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
+                               bucket_name, &bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
       ldpp_dout(this, 20) << "non existent directory=" << bucket_name << dendl;
diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 75c7f9b0907d..491b25dacb63 100644
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -502,7 +502,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp,
   }
 
   std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-  ret = store->get_bucket(dpp, nullptr, orphan_bucket, &cur_bucket, null_yield);
+  ret = store->load_bucket(dpp, nullptr, orphan_bucket, &cur_bucket, null_yield);
   if (ret < 0) {
     if (ret == -ENOENT) {
       /* probably raced with bucket removal */
@@ -529,7 +529,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp,
   rgw_bucket b;
   rgw_bucket_parse_bucket_key(store->ctx(), bucket_instance_id, &b, nullptr);
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = store->get_bucket(dpp, nullptr, b, &bucket, null_yield);
+  ret = store->load_bucket(dpp, nullptr, b, &bucket, null_yield);
   if (ret < 0) {
     if (ret == -ENOENT) {
       /* probably raced with bucket removal */
@@ -1241,8 +1241,7 @@ int RGWRadosList::process_bucket(
 	continue;
       }
 
-      std::unique_ptr<rgw::sal::Bucket> bucket;
-      store->get_bucket(nullptr, bucket_info, &bucket);
+      auto bucket = store->get_bucket(nullptr, bucket_info);
       // we need to do this in two cases below, so use a lambda
       auto do_stat_key =
 	[&](const rgw_obj_key& key) -> int {
@@ -1389,7 +1388,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
     bucket_process_map.erase(front);
 
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = store->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield);
+    ret = store->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield);
     if (ret == -ENOENT) {
       std::cerr << "WARNING: bucket " << bucket_name <<
 	" does not exist; could it have been deleted very recently?" <<
@@ -1460,7 +1459,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
   // initial bucket
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = store->get_bucket(dpp, nullptr, tenant_name, start_bucket_name, &bucket, null_yield);
+  ret = store->load_bucket(dpp, nullptr, tenant_name, start_bucket_name, &bucket, null_yield);
   if (ret == -ENOENT) {
     // bucket deletion race?
     return 0;
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index 9df4c36d112a..3786ccd6729a 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -265,7 +265,7 @@ int BucketAsyncRefreshHandler::init_fetch()
   std::unique_ptr<rgw::sal::Bucket> rbucket;
 
   const DoutPrefix dp(driver->ctx(), dout_subsys, "rgw bucket async refresh handler: ");
-  int r = driver->get_bucket(&dp, nullptr, bucket, &rbucket, null_yield);
+  int r = driver->load_bucket(&dp, nullptr, bucket, &rbucket, null_yield);
   if (r < 0) {
     ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
     return r;
@@ -340,7 +340,7 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_
   std::unique_ptr<rgw::sal::User> user = driver->get_user(_u);
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int r = driver->get_bucket(dpp, user.get(), _b, &bucket, y);
+  int r = driver->load_bucket(dpp, user.get(), _b, &bucket, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << _b << " r=" << r << dendl;
     return r;
@@ -613,7 +613,7 @@ int RGWUserStatsCache::sync_bucket(const rgw_user& _u, rgw_bucket& _b, optional_
   std::unique_ptr<rgw::sal::User> user = driver->get_user(_u);
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int r = driver->get_bucket(dpp, user.get(), _b, &bucket, y);
+  int r = driver->load_bucket(dpp, user.get(), _b, &bucket, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << _b << " r=" << r << dendl;
     return r;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index da81711511c3..c3214123fb88 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -962,7 +962,7 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->get_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
@@ -1110,7 +1110,7 @@ void RGWPSDeleteNotifOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->get_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
@@ -1207,7 +1207,7 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->get_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc
index b482b4f82c49..5e37ad2c74c5 100644
--- a/src/rgw/rgw_rest_ratelimit.cc
+++ b/src/rgw/rgw_rest_ratelimit.cc
@@ -36,7 +36,7 @@ void RGWOp_Ratelimit_Info::execute(optional_yield y)
 
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    int r = driver->get_bucket(s, nullptr, tenant_name, bucket_name, &bucket, y);
+    int r = driver->load_bucket(s, nullptr, tenant_name, bucket_name, &bucket, y);
     if (r != 0) {
       op_ret = r;
       ldpp_dout(this, 0) << "Error on getting bucket info" << dendl;
@@ -273,7 +273,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y)
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     ldpp_dout(this, 0) << "getting bucket info" << dendl;
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->get_bucket(this, nullptr, tenant_name, bucket_name, &bucket, y);
+    op_ret = driver->load_bucket(this, nullptr, tenant_name, bucket_name, &bucket, y);
     if (op_ret) {
       ldpp_dout(this, 0) << "Error on getting bucket info" << dendl;
       return;
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 6efc7bd8e1cb..082bccf7fb2a 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -857,7 +857,7 @@ int RGWPutObj_ObjStore_SWIFT::update_slo_segment_size(rgw_slo_entry& entry) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
-    r = driver->get_bucket(s, s->user.get(), s->user->get_id().tenant, bucket_name, &bucket, s->yield);
+    r = driver->load_bucket(s, s->user.get(), s->user->get_id().tenant, bucket_name, &bucket, s->yield);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
 			 << bucket_name << dendl;
@@ -2120,7 +2120,7 @@ void RGWFormPost::get_owner_info(const req_state* const s,
 
   /* Need to get user info of bucket owner. */
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->get_bucket(s, user.get(), bucket_tenant, bucket_name, &bucket, s->yield);
+  int ret = driver->load_bucket(s, user.get(), bucket_tenant, bucket_name, &bucket, s->yield);
   if (ret < 0) {
     throw ret;
   }
diff --git a/src/rgw/rgw_rest_usage.cc b/src/rgw/rgw_rest_usage.cc
index 0ab93fd93406..5f9d5c8a5298 100644
--- a/src/rgw/rgw_rest_usage.cc
+++ b/src/rgw/rgw_rest_usage.cc
@@ -40,7 +40,10 @@ void RGWOp_Usage_Get::execute(optional_yield y) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    driver->get_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    op_ret = driver->load_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    if (op_ret < 0) {
+      return;
+    }
   }
 
   RESTArgs::get_epoch(s, "start", 0, &start);
@@ -87,7 +90,10 @@ void RGWOp_Usage_Delete::execute(optional_yield y) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    driver->get_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    op_ret = driver->load_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    if (op_ret < 0) {
+      return;
+    }
   }
 
   RESTArgs::get_epoch(s, "start", 0, &start);
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index d8072ca6f6f9..de5a2f7dda13 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -290,11 +290,15 @@ class Driver {
      * there is a Bucket, otherwise use the get_object() in the Bucket class. */
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) = 0;
     /** Get a Bucket by info.  Does not query the driver, just uses the give bucket info. */
-    virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) = 0;
-    /** Lookup a Bucket by key.  Queries driver for bucket info. */
-    virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
-    /** Lookup a Bucket by name.  Queries driver for bucket info. */
-    virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
+    virtual std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) = 0;
+    /** Load a Bucket by key.  Queries driver for bucket info.  On -ENOENT, the
+     * bucket must still be allocated to support bucket->create(). */
+    virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+                            std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
+    /** Load a Bucket by name.  Queries driver for bucket info. */
+    virtual int load_bucket(const DoutPrefixProvider* dpp, User* u,
+                            const std::string& tenant, const std::string& name,
+                            std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
     /** For multisite, this driver is the zone's master */
     virtual bool is_meta_master() = 0;
     /** For multisite, forward an OP to the zone's master */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 5e429ba5c4c8..7f9ad515ec21 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -84,7 +84,7 @@ namespace rgw::sal {
     obj_version objv, *pobjv = NULL;
 
     /* If it exists, look it up; otherwise create it */
-    ret = store->get_bucket(dpp, this, b, &bucket, y);
+    ret = store->load_bucket(dpp, this, b, &bucket, y);
     if (ret < 0 && ret != -ENOENT)
       return ret;
 
@@ -1633,41 +1633,29 @@ namespace rgw::sal {
   }
 
 
-  int DBStore::get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+  std::unique_ptr<Bucket> DBStore::get_bucket(User* u, const RGWBucketInfo& i)
   {
-    int ret;
-    Bucket* bp;
-
-    bp = new DBBucket(this, b, u);
-    ret = bp->load_bucket(dpp, y);
-    if (ret < 0) {
-      delete bp;
-      return ret;
-    }
-
-    bucket->reset(bp);
-    return 0;
+    /* Don't need to fetch the bucket info, use the provided one */
+    return std::make_unique<DBBucket>(this, i, u);
   }
 
-  int DBStore::get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket)
+  int DBStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+                           std::unique_ptr<Bucket>* bucket, optional_yield y)
   {
-    Bucket* bp;
-
-    bp = new DBBucket(this, i, u);
-    /* Don't need to fetch the bucket info, use the provided one */
-
-    bucket->reset(bp);
-    return 0;
+    *bucket = std::make_unique<DBBucket>(this, b, u);
+    return (*bucket)->load_bucket(dpp, y);
   }
 
-  int DBStore::get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
+  int DBStore::load_bucket(const DoutPrefixProvider *dpp, User* u,
+                           const std::string& tenant, const std::string& name,
+                           std::unique_ptr<Bucket>* bucket, optional_yield y)
   {
     rgw_bucket b;
 
     b.tenant = tenant;
     b.name = name;
 
-    return get_bucket(dpp, u, b, bucket, y);
+    return load_bucket(dpp, u, b, bucket, y);
   }
 
   bool DBStore::is_meta_master()
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 375a4524517c..e352af98b143 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -778,9 +778,12 @@ class DBNotification : public StoreNotification {
       virtual int get_user_by_swift(const DoutPrefixProvider *dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
       virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
       virtual std::string get_cluster_id(const DoutPrefixProvider* dpp, optional_yield y);
-      virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-      virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
-      virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+      std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
+      int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+                      std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+      int load_bucket(const DoutPrefixProvider *dpp, User* u,
+                      const std::string& tenant, const std::string&name,
+                      std::unique_ptr<Bucket>* bucket, optional_yield y) override;
       virtual bool is_meta_master() override;
       virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
           bufferlist& in_data, JSONParser *jp, req_info& info,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 3f3c5102cb42..fc5eecc008fd 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -161,49 +161,27 @@ std::unique_ptr<Object> FilterDriver::get_object(const rgw_obj_key& k)
   return std::make_unique<FilterObject>(std::move(o));
 }
 
-int FilterDriver::get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+std::unique_ptr<Bucket> FilterDriver::get_bucket(User* u, const RGWBucketInfo& i)
 {
-  std::unique_ptr<Bucket> nb;
-  int ret;
-  User* nu = nextUser(u);
-
-  ret = next->get_bucket(dpp, nu, b, &nb, y);
-  if (ret != 0)
-    return ret;
-
-  Bucket* fb = new FilterBucket(std::move(nb), u);
-  bucket->reset(fb);
-  return 0;
+  return std::make_unique<FilterBucket>(next->get_bucket(nextUser(u), i), u);
 }
 
-int FilterDriver::get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket)
+int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   std::unique_ptr<Bucket> nb;
-  int ret;
-  User* nu = nextUser(u);
-
-  ret = next->get_bucket(nu, i, &nb);
-  if (ret != 0)
-    return ret;
-
-  Bucket* fb = new FilterBucket(std::move(nb), u);
-  bucket->reset(fb);
-  return 0;
+  const int ret = next->load_bucket(dpp, nextUser(u), b, &nb, y);
+  *bucket = std::make_unique<FilterBucket>(std::move(nb), u);
+  return ret;
 }
 
-int FilterDriver::get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, User* u,
+                              const std::string& tenant, const std::string& name,
+                              std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   std::unique_ptr<Bucket> nb;
-  int ret;
-  User* nu = nextUser(u);
-
-  ret = next->get_bucket(dpp, nu, tenant, name, &nb, y);
-  if (ret != 0)
-    return ret;
-
-  Bucket* fb = new FilterBucket(std::move(nb), u);
-  bucket->reset(fb);
-  return 0;
+  const int ret = next->load_bucket(dpp, nextUser(u), tenant, name, &nb, y);
+  *bucket = std::make_unique<FilterBucket>(std::move(nb), u);
+  return ret;
 }
 
 bool FilterDriver::is_meta_master()
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 5d81bea0e420..e33c84871a7a 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -159,14 +159,13 @@ class FilterDriver : public Driver {
 				std::string& user_str, optional_yield y,
 				std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-  virtual int get_bucket(User* u, const RGWBucketInfo& i,
-			 std::unique_ptr<Bucket>* bucket) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const
-			 rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-			 optional_yield y) override;
-  virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const
-			 std::string& tenant, const std::string& name,
-			 std::unique_ptr<Bucket>* bucket, optional_yield y) override;
+  std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
+  int load_bucket(const DoutPrefixProvider* dpp, User* u, const
+                  rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
+                  optional_yield y) override;
+  int load_bucket(const DoutPrefixProvider* dpp, User* u, const
+                  std::string& tenant, const std::string& name,
+                  std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual bool is_meta_master() override;
   virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user,
 					obj_version* objv, bufferlist& in_data,
diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc
index 05d4b28c124e..852449ad20f6 100644
--- a/src/rgw/rgw_swift_auth.cc
+++ b/src/rgw/rgw_swift_auth.cc
@@ -119,7 +119,7 @@ void TempURLEngine::get_owner_info(const DoutPrefixProvider* dpp, const req_stat
   b.tenant = std::move(bucket_tenant);
   b.name = std::move(bucket_name);
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->get_bucket(dpp, nullptr, b, &bucket, s->yield);
+  int ret = driver->load_bucket(dpp, nullptr, b, &bucket, s->yield);
   if (ret < 0) {
     throw ret;
   }
diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index 175b2e6b23fe..1f88779d935d 100644
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -30,7 +30,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Driver* dri
 
     for (auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->get_bucket(dpp, user, ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, user, ent.bucket, &bucket, y);
       if (ret < 0) {
         ldpp_dout(dpp, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl;
         continue;

From 89882f24a0cea3507f8b4bbee5a2f66fef63a777 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 3 Oct 2023 15:23:49 -0400
Subject: [PATCH 0475/2492] rgw/sal: remove load_bucket() overload with
 tenant/name

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc   | 12 ------------
 src/rgw/driver/daos/rgw_sal_daos.h    |  4 ----
 src/rgw/driver/motr/rgw_sal_motr.cc   | 10 ----------
 src/rgw/driver/motr/rgw_sal_motr.h    |  3 ---
 src/rgw/driver/posix/rgw_sal_posix.cc | 10 ----------
 src/rgw/driver/posix/rgw_sal_posix.h  |  3 ---
 src/rgw/driver/rados/rgw_sal_rados.cc | 12 ------------
 src/rgw/driver/rados/rgw_sal_rados.h  |  3 ---
 src/rgw/rgw_sal.h                     |  4 ----
 src/rgw/rgw_sal_dbstore.cc            | 12 ------------
 src/rgw/rgw_sal_dbstore.h             |  3 ---
 src/rgw/rgw_sal_filter.cc             | 10 ----------
 src/rgw/rgw_sal_filter.h              |  7 ++-----
 13 files changed, 2 insertions(+), 91 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 167c16d6be97..a6aacf024b31 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -2263,18 +2263,6 @@ int DaosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
   return (*bucket)->load_bucket(dpp, y);
 }
 
-int DaosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
-                           const std::string& tenant, const std::string& name,
-                           std::unique_ptr<Bucket>* bucket, optional_yield y) {
-  ldpp_dout(dpp, 20) << "get_bucket" << dendl;
-  rgw_bucket b;
-
-  b.tenant = tenant;
-  b.name = name;
-
-  return load_bucket(dpp, u, b, bucket, y);
-}
-
 bool DaosStore::is_meta_master() { return true; }
 
 int DaosStore::forward_request_to_master(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 23f723c0ce7c..72d3bdb6cfbe 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -899,10 +899,6 @@ class DaosStore : public StoreDriver {
   int load_bucket(const DoutPrefixProvider* dpp, User* u,
                   const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
                   optional_yield y) override;
-  int load_bucket(const DoutPrefixProvider* dpp, User* u,
-                  const std::string& tenant, const std::string& name,
-                  std::unique_ptr<Bucket>* bucket,
-                  optional_yield y) override;
   virtual bool is_meta_master() override;
   virtual int forward_request_to_master(const DoutPrefixProvider* dpp,
                                         User* user, obj_version* objv,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index c7e4fc49bd48..b53516bb0653 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -3255,16 +3255,6 @@ int MotrStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_buc
   return (*bucket)->load_bucket(dpp, y);
 }
 
-int MotrStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
-{
-  rgw_bucket b;
-
-  b.tenant = tenant;
-  b.name = name;
-
-  return load_bucket(dpp, u, b, bucket, y);
-}
-
 bool MotrStore::is_meta_master()
 {
   return true;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index ec5eb347cf22..7935462ac3f0 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -995,9 +995,6 @@ class MotrStore : public StoreDriver {
     std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
     int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
                     std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-    int load_bucket(const DoutPrefixProvider *dpp, User* u,
-                    const std::string& tenant, const std::string& name,
-                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
     virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
         bufferlist& in_data, JSONParser *jp, req_info& info,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 40f509b28d74..065d6d2d95db 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -405,16 +405,6 @@ std::unique_ptr<Bucket> POSIXDriver::get_bucket(User* u, const RGWBucketInfo& i)
   return std::make_unique<POSIXBucket>(this, root_fd, i, u);
 }
 
-int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr<Bucket>* bucket, optional_yield y)
-{
-  rgw_bucket b;
-
-  b.tenant = tenant;
-  b.name = name;
-
-  return load_bucket(dpp, u, b, bucket, y);
-}
-
 std::string POSIXDriver::zone_unique_trans_id(const uint64_t unique_num)
 {
   char buf[41]; /* 2 + 21 + 1 + 16 (timestamp can consume up to 16) + 1 */
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 53cd05823b6c..e132f240812b 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -56,9 +56,6 @@ class POSIXDriver : public FilterDriver {
   virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const
 			  rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
 			  optional_yield y) override;
-  virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const
-			  std::string& tenant, const std::string& name,
-			  std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
 
   virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 2c526ec940d2..328b4f82c8f2 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1068,18 +1068,6 @@ int RadosStore::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bu
   return (*bucket)->load_bucket(dpp, y);
 }
 
-int RadosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
-                            const std::string& tenant, const std::string& name,
-                            std::unique_ptr<Bucket>* bucket, optional_yield y)
-{
-  rgw_bucket b;
-
-  b.tenant = tenant;
-  b.name = name;
-
-  return load_bucket(dpp, u, b, bucket, y);
-}
-
 bool RadosStore::is_meta_master()
 {
   return svc()->zone->is_meta_master();
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 84a0b6bf9513..8c1926f3fbef 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -140,9 +140,6 @@ class RadosStore : public StoreDriver {
     std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
     int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
                     std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-    int load_bucket(const DoutPrefixProvider* dpp, User* u,
-                    const std::string& tenant, const std::string& name,
-                    std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
     virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
 					  bufferlist& in_data, JSONParser* jp, req_info& info,
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index de5a2f7dda13..d69942532881 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -295,10 +295,6 @@ class Driver {
      * bucket must still be allocated to support bucket->create(). */
     virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
                             std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
-    /** Load a Bucket by name.  Queries driver for bucket info. */
-    virtual int load_bucket(const DoutPrefixProvider* dpp, User* u,
-                            const std::string& tenant, const std::string& name,
-                            std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
     /** For multisite, this driver is the zone's master */
     virtual bool is_meta_master() = 0;
     /** For multisite, forward an OP to the zone's master */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 7f9ad515ec21..792fa457f895 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -1646,18 +1646,6 @@ namespace rgw::sal {
     return (*bucket)->load_bucket(dpp, y);
   }
 
-  int DBStore::load_bucket(const DoutPrefixProvider *dpp, User* u,
-                           const std::string& tenant, const std::string& name,
-                           std::unique_ptr<Bucket>* bucket, optional_yield y)
-  {
-    rgw_bucket b;
-
-    b.tenant = tenant;
-    b.name = name;
-
-    return load_bucket(dpp, u, b, bucket, y);
-  }
-
   bool DBStore::is_meta_master()
   {
     return true;
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index e352af98b143..41a0b89ea502 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -781,9 +781,6 @@ class DBNotification : public StoreNotification {
       std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
       int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
                       std::unique_ptr<Bucket>* bucket, optional_yield y) override;
-      int load_bucket(const DoutPrefixProvider *dpp, User* u,
-                      const std::string& tenant, const std::string&name,
-                      std::unique_ptr<Bucket>* bucket, optional_yield y) override;
       virtual bool is_meta_master() override;
       virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
           bufferlist& in_data, JSONParser *jp, req_info& info,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index fc5eecc008fd..c70d9e372dc3 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -174,16 +174,6 @@ int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_
   return ret;
 }
 
-int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, User* u,
-                              const std::string& tenant, const std::string& name,
-                              std::unique_ptr<Bucket>* bucket, optional_yield y)
-{
-  std::unique_ptr<Bucket> nb;
-  const int ret = next->load_bucket(dpp, nextUser(u), tenant, name, &nb, y);
-  *bucket = std::make_unique<FilterBucket>(std::move(nb), u);
-  return ret;
-}
-
 bool FilterDriver::is_meta_master()
 {
   return next->is_meta_master();
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index e33c84871a7a..b2aa1be399f3 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -160,12 +160,9 @@ class FilterDriver : public Driver {
 				std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
   std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
-  int load_bucket(const DoutPrefixProvider* dpp, User* u, const
-                  rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
+  int load_bucket(const DoutPrefixProvider* dpp, User* u,
+                  const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
                   optional_yield y) override;
-  int load_bucket(const DoutPrefixProvider* dpp, User* u, const
-                  std::string& tenant, const std::string& name,
-                  std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual bool is_meta_master() override;
   virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user,
 					obj_version* objv, bufferlist& in_data,

From 8b7947debc41dea83f8bd6304ac6ecf82ca6ae10 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 3 Oct 2023 15:25:04 -0400
Subject: [PATCH 0476/2492] rgw: add rgw_bucket() ctor for tenant+name

this avoids the need to construct a temporary rgw_bucket_key just to
construct a rgw_bucket without an instance id

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_bucket_types.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/rgw/rgw_bucket_types.h b/src/rgw/rgw_bucket_types.h
index 52ac5dc34bd1..ea379678ebe3 100644
--- a/src/rgw/rgw_bucket_types.h
+++ b/src/rgw/rgw_bucket_types.h
@@ -61,6 +61,9 @@ struct rgw_bucket {
 	     const std::string& _bucket_id) : tenant(_tenant),
                                               name(_name),
                                               bucket_id(_bucket_id) {}
+  rgw_bucket(const std::string& _tenant,
+	     const std::string& _name)
+      : tenant(_tenant), name(_name) {}
   rgw_bucket(const rgw_bucket_key& bk) : tenant(bk.tenant),
                                          name(bk.name),
                                          bucket_id(bk.bucket_id) {}

From 015c4153bef0cbef4f66672e7c6cff6aeabf47cc Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 3 Oct 2023 15:51:13 -0400
Subject: [PATCH 0477/2492] rgw: use load_bucket()'s rgw_bucket overload

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/posix/rgw_sal_posix.cc   |  3 +-
 src/rgw/driver/rados/rgw_bucket.cc      | 24 ++++++++++-----
 src/rgw/driver/rados/rgw_cr_tools.cc    |  4 ++-
 src/rgw/driver/rados/rgw_rest_bucket.cc |  6 ++--
 src/rgw/driver/rados/rgw_tools.cc       |  2 +-
 src/rgw/rgw_admin.cc                    | 12 ++++++--
 src/rgw/rgw_lc.cc                       |  4 ++-
 src/rgw/rgw_op.cc                       | 39 +++++++++++++++----------
 src/rgw/rgw_orphan.cc                   |  7 +++--
 src/rgw/rgw_rest_pubsub.cc              | 12 ++++++--
 src/rgw/rgw_rest_ratelimit.cc           |  8 +++--
 src/rgw/rgw_rest_swift.cc               |  8 +++--
 src/rgw/rgw_rest_usage.cc               |  8 +++--
 13 files changed, 94 insertions(+), 43 deletions(-)

diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 065d6d2d95db..5204017274a9 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -705,7 +705,8 @@ int POSIXDriver::mint_listing_entry(const std::string &bname,
     POSIXObject *pobj;
     int ret;
 
-    ret = load_bucket(nullptr, nullptr, std::string(), bname, &b, null_yield);
+    ret = load_bucket(nullptr, nullptr, rgw_bucket(std::string(), bname),
+                      &b, null_yield);
     if (ret < 0)
       return ret;
 
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 1c4341f438f3..d0bdc025b7a6 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -109,7 +109,9 @@ void check_bad_user_bucket_mapping(rgw::sal::Driver* driver, rgw::sal::User& use
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      int r = driver->load_bucket(dpp, &user, user.get_tenant(), ent.bucket.name, &bucket, y);
+      int r = driver->load_bucket(dpp, &user,
+                                  rgw_bucket(user.get_tenant(), ent.bucket.name),
+                                  &bucket, y);
       if (r < 0) {
         ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << bucket << dendl;
         continue;
@@ -181,7 +183,8 @@ int RGWBucket::init(rgw::sal::Driver* _driver, RGWBucketAdminOpState& op_state,
     bucket_name = bucket_name.substr(pos + 1);
   }
 
-  int r = driver->load_bucket(dpp, user.get(), tenant, bucket_name, &bucket, y);
+  int r = driver->load_bucket(dpp, user.get(), rgw_bucket(tenant, bucket_name),
+                              &bucket, y);
   if (r < 0) {
       set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name);
       return r;
@@ -1242,8 +1245,10 @@ int RGWBucketAdminOp::remove_bucket(rgw::sal::Driver* driver, RGWBucketAdminOpSt
   std::unique_ptr<rgw::sal::Bucket> bucket;
   std::unique_ptr<rgw::sal::User> user = driver->get_user(op_state.get_user_id());
 
-  int ret = driver->load_bucket(dpp, user.get(), user->get_tenant(),
-                                op_state.get_bucket_name(), &bucket, y);
+  int ret = driver->load_bucket(dpp, user.get(),
+                                rgw_bucket(user->get_tenant(),
+                                           op_state.get_bucket_name()),
+                                &bucket, y);
   if (ret < 0)
     return ret;
 
@@ -1284,7 +1289,9 @@ static int bucket_stats(rgw::sal::Driver* driver,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   map<RGWObjCategory, RGWStorageStats> stats;
 
-  int ret = driver->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
+  int ret = driver->load_bucket(dpp, nullptr,
+                                rgw_bucket(tenant_name, bucket_name),
+                                &bucket, y);
   if (ret < 0) {
     return ret;
   }
@@ -1611,7 +1618,8 @@ void get_stale_instances(rgw::sal::Driver* driver, const std::string& bucket_nam
   auto [tenant, bname] = split_tenant(bucket_name);
   RGWBucketInfo cur_bucket_info;
   std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-  int r = driver->load_bucket(dpp, nullptr, tenant, bname, &cur_bucket, y);
+  int r = driver->load_bucket(dpp, nullptr, rgw_bucket(tenant, bname),
+                              &cur_bucket, y);
   if (r < 0) {
     if (r == -ENOENT) {
       // bucket doesn't exist, everything is stale then
@@ -1767,7 +1775,9 @@ static int fix_single_bucket_lc(rgw::sal::Driver* driver,
                                 const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, y);
+  int ret = driver->load_bucket(dpp, nullptr,
+                                rgw_bucket(tenant_name, bucket_name),
+                                &bucket, y);
   if (ret < 0) {
     // TODO: Should we handle the case where the bucket could've been removed between
     // listing and fetching?
diff --git a/src/rgw/driver/rados/rgw_cr_tools.cc b/src/rgw/driver/rados/rgw_cr_tools.cc
index 10d17a49b380..928cc9c0c988 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.cc
+++ b/src/rgw/driver/rados/rgw_cr_tools.cc
@@ -100,7 +100,9 @@ int RGWGetUserInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 template<>
 int RGWGetBucketInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 {
-  return store->load_bucket(dpp, nullptr, params.tenant, params.bucket_name, &result->bucket, null_yield);
+  return store->load_bucket(dpp, nullptr,
+                            rgw_bucket(params.tenant, params.bucket_name),
+                            &result->bucket, null_yield);
 }
 
 template<>
diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index 66aceabee96f..6853173be4ff 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -232,7 +232,8 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->load_bucket(s, nullptr, string(), bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(s, nullptr, rgw_bucket("", bucket_name),
+                               &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "get_bucket returned ret=" << op_ret << dendl;
     if (op_ret == -ENOENT) {
@@ -299,7 +300,8 @@ void RGWOp_Set_Bucket_Quota::execute(optional_yield y)
   }
   if (use_http_params) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->load_bucket(s, nullptr, uid.tenant, bucket_name, &bucket, s->yield);
+    op_ret = driver->load_bucket(s, nullptr, rgw_bucket(uid.tenant, bucket_name),
+                                 &bucket, s->yield);
     if (op_ret < 0) {
       return;
     }
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index a3877c0b4678..72a7da1c31de 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -285,7 +285,7 @@ int RGWDataAccess::Bucket::finish_init()
 int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = sd->driver->load_bucket(dpp, nullptr, tenant, name, &bucket, y);
+  int ret = sd->driver->load_bucket(dpp, nullptr, rgw_bucket(tenant, name), &bucket, y);
   if (ret < 0) {
     return ret;
   }
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 061464f0c2cc..6ec1e0c6a384 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1407,7 +1407,9 @@ int set_bucket_quota(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_size, bool have_max_objects)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr,
+                              rgw_bucket(tenant_name, bucket_name),
+                              &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
@@ -1431,7 +1433,9 @@ int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_read_bytes, bool have_max_write_bytes)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr,
+                              rgw_bucket(tenant_name, bucket_name),
+                              &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
@@ -1534,7 +1538,9 @@ int show_bucket_ratelimit(rgw::sal::Driver* driver, const string& tenant_name,
                           const string& bucket_name, Formatter *formatter)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr, tenant_name, bucket_name, &bucket, null_yield);
+  int r = driver->load_bucket(dpp(), nullptr,
+                              rgw_bucket(tenant_name, bucket_name),
+                              &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 862a2d0191c9..c2234912e84f 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1615,7 +1615,9 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     return 0;
   }
 
-  int ret = driver->load_bucket(this, nullptr, bucket_tenant, bucket_name, &bucket, null_yield);
+  int ret = driver->load_bucket(this, nullptr,
+                                rgw_bucket(bucket_tenant, bucket_name),
+                                &bucket, null_yield);
   if (ret < 0) {
     ldpp_dout(this, 0) << "LC:get_bucket for " << bucket_name
 		       << " failed" << dendl;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 607633534f10..067a939d7535 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -491,8 +491,7 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   if (!s->src_bucket_name.empty()) {
     std::unique_ptr<rgw::sal::Bucket> src_bucket;
     ret = driver->load_bucket(dpp, nullptr,
-                              rgw_bucket_key(s->src_tenant_name,
-                                             s->src_bucket_name),
+                              rgw_bucket(s->src_tenant_name, s->src_bucket_name),
                               &src_bucket, y);
     if (ret == 0) {
       string& zonegroup = src_bucket->get_info().zonegroup;
@@ -1939,8 +1938,9 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
     map<string, bufferlist> bucket_attrs;
-    r = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
-                            bucket_name, &ubucket, y);
+    r = driver->load_bucket(this, s->user.get(),
+                            rgw_bucket(s->user->get_tenant(), bucket_name),
+                            &ubucket, y);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
 		       << bucket_name << dendl;
@@ -2070,8 +2070,9 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
 	RGWAccessControlPolicy& _bucket_acl = allocated_acls.back();
 
 	std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-	int r = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
-                                    bucket_name, &tmp_bucket, y);
+	int r = driver->load_bucket(this, s->user.get(),
+                                    rgw_bucket(s->user->get_tenant(), bucket_name),
+                                    &tmp_bucket, y);
         if (r < 0) {
           ldpp_dout(this, 0) << "could not get bucket info for bucket="
 			   << bucket_name << dendl;
@@ -3373,8 +3374,9 @@ void RGWCreateBucket::execute(optional_yield y)
    * specific request */
   {
     std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-    op_ret = driver->load_bucket(this, s->user.get(), s->bucket_tenant,
-                                 s->bucket_name, &tmp_bucket, y);
+    op_ret = driver->load_bucket(this, s->user.get(),
+                                 rgw_bucket(s->bucket_tenant, s->bucket_name),
+                                 &tmp_bucket, y);
     if (op_ret < 0 && op_ret != -ENOENT)
       return;
     s->bucket_exists = (op_ret != -ENOENT);
@@ -3673,8 +3675,10 @@ int RGWPutObj::init_processing(optional_yield y) {
       }
     }
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = driver->load_bucket(this, s->user.get(), copy_source_tenant_name,
-                              copy_source_bucket_name, &bucket, y);
+    ret = driver->load_bucket(this, s->user.get(),
+                              rgw_bucket(copy_source_tenant_name,
+                                         copy_source_bucket_name),
+                              &bucket, y);
     if (ret < 0) {
       ldpp_dout(this, 5) << __func__ << "(): get_bucket() returned ret=" << ret << dendl;
       if (ret == -ENOENT) {
@@ -5357,8 +5361,8 @@ int RGWCopyObj::init_processing(optional_yield y)
   }
 
   op_ret = driver->load_bucket(this, s->user.get(),
-                               rgw_bucket_key(s->src_tenant_name,
-                                              s->src_bucket_name),
+                               rgw_bucket(s->src_tenant_name,
+                                          s->src_bucket_name),
                                &src_bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
@@ -7315,8 +7319,10 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
   ACLOwner bowner;
   RGWObjVersionTracker ot;
 
-  int ret = driver->load_bucket(dpp, s->user.get(), s->user->get_tenant(),
-                                path.bucket_name, &bucket, y);
+  int ret = driver->load_bucket(dpp, s->user.get(),
+                                rgw_bucket(s->user->get_tenant(),
+                                           path.bucket_name),
+                                &bucket, y);
   if (ret < 0) {
     goto binfo_fail;
   }
@@ -7689,8 +7695,9 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   ACLOwner bowner;
 
-  op_ret = driver->load_bucket(this, s->user.get(), s->user->get_tenant(),
-                               bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, s->user.get(),
+                               rgw_bucket(s->user->get_tenant(), bucket_name),
+                               &bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
       ldpp_dout(this, 20) << "non existent directory=" << bucket_name << dendl;
diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 491b25dacb63..8b9125f29067 100644
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -1388,7 +1388,8 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
     bucket_process_map.erase(front);
 
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = store->load_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield);
+    ret = store->load_bucket(dpp, nullptr, rgw_bucket(tenant_name, bucket_name),
+                             &bucket, null_yield);
     if (ret == -ENOENT) {
       std::cerr << "WARNING: bucket " << bucket_name <<
 	" does not exist; could it have been deleted very recently?" <<
@@ -1459,7 +1460,9 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
   // initial bucket
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = store->load_bucket(dpp, nullptr, tenant_name, start_bucket_name, &bucket, null_yield);
+  ret = store->load_bucket(dpp, nullptr,
+                           rgw_bucket(tenant_name, start_bucket_name),
+                           &bucket, null_yield);
   if (ret == -ENOENT) {
     // bucket deletion race?
     return 0;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index c3214123fb88..d2a1c72f3aa7 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -962,7 +962,9 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(),
+                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
@@ -1110,7 +1112,9 @@ void RGWPSDeleteNotifOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(),
+                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
@@ -1207,7 +1211,9 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
 
   std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(), s->bucket_tenant, s->bucket_name, &bucket, y);
+  op_ret = driver->load_bucket(this, user.get(),
+                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
       (s->bucket_tenant.empty() ? s->bucket_name : s->bucket_tenant + ":" + s->bucket_name) << 
diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc
index 5e37ad2c74c5..e5d1aabb1503 100644
--- a/src/rgw/rgw_rest_ratelimit.cc
+++ b/src/rgw/rgw_rest_ratelimit.cc
@@ -36,7 +36,9 @@ void RGWOp_Ratelimit_Info::execute(optional_yield y)
 
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    int r = driver->load_bucket(s, nullptr, tenant_name, bucket_name, &bucket, y);
+    int r = driver->load_bucket(s, nullptr,
+                                rgw_bucket(tenant_name, bucket_name),
+                                &bucket, y);
     if (r != 0) {
       op_ret = r;
       ldpp_dout(this, 0) << "Error on getting bucket info" << dendl;
@@ -273,7 +275,9 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y)
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     ldpp_dout(this, 0) << "getting bucket info" << dendl;
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->load_bucket(this, nullptr, tenant_name, bucket_name, &bucket, y);
+    op_ret = driver->load_bucket(this, nullptr,
+                                 rgw_bucket(tenant_name, bucket_name),
+                                 &bucket, y);
     if (op_ret) {
       ldpp_dout(this, 0) << "Error on getting bucket info" << dendl;
       return;
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 082bccf7fb2a..7ef73285583b 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -857,7 +857,9 @@ int RGWPutObj_ObjStore_SWIFT::update_slo_segment_size(rgw_slo_entry& entry) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
-    r = driver->load_bucket(s, s->user.get(), s->user->get_id().tenant, bucket_name, &bucket, s->yield);
+    r = driver->load_bucket(s, s->user.get(),
+                            rgw_bucket(s->user->get_id().tenant, bucket_name),
+                            &bucket, s->yield);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
 			 << bucket_name << dendl;
@@ -2120,7 +2122,9 @@ void RGWFormPost::get_owner_info(const req_state* const s,
 
   /* Need to get user info of bucket owner. */
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->load_bucket(s, user.get(), bucket_tenant, bucket_name, &bucket, s->yield);
+  int ret = driver->load_bucket(s, user.get(),
+                                rgw_bucket(bucket_tenant, bucket_name),
+                                &bucket, s->yield);
   if (ret < 0) {
     throw ret;
   }
diff --git a/src/rgw/rgw_rest_usage.cc b/src/rgw/rgw_rest_usage.cc
index 5f9d5c8a5298..65b035c00727 100644
--- a/src/rgw/rgw_rest_usage.cc
+++ b/src/rgw/rgw_rest_usage.cc
@@ -40,7 +40,9 @@ void RGWOp_Usage_Get::execute(optional_yield y) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    op_ret = driver->load_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    op_ret = driver->load_bucket(nullptr, user.get(),
+                                 rgw_bucket(std::string(), bucket_name),
+                                 &bucket, null_yield);
     if (op_ret < 0) {
       return;
     }
@@ -90,7 +92,9 @@ void RGWOp_Usage_Delete::execute(optional_yield y) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    op_ret = driver->load_bucket(nullptr, user.get(), std::string(), bucket_name, &bucket, null_yield);
+    op_ret = driver->load_bucket(nullptr, user.get(),
+                                 rgw_bucket(std::string(), bucket_name),
+                                 &bucket, null_yield);
     if (op_ret < 0) {
       return;
     }

From d2756f1509c79c992d29c8157584bea90d2d900a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 3 Oct 2023 14:12:35 -0400
Subject: [PATCH 0478/2492] rgw: add free function
 rgw::get_zonegroup_endpoint()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_zone.h |  3 +++
 src/rgw/rgw_zone.cc             | 13 +++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 140678fcdce5..6d55fd52c575 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -885,6 +885,9 @@ int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y,
                           sal::ConfigStore* cfgstore, const RGWZoneGroup& info,
                           bool exclusive = false);
 
+/// Return an endpoint from the zonegroup or its master zone.
+std::string get_zonegroup_endpoint(const RGWZoneGroup& info);
+
 /// Add a zone to the zonegroup, or update an existing zone entry.
 int add_zone_to_group(const DoutPrefixProvider* dpp,
                       RGWZoneGroup& zonegroup,
diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc
index 784c9c53fdc6..f41d987ec549 100644
--- a/src/rgw/rgw_zone.cc
+++ b/src/rgw/rgw_zone.cc
@@ -1268,6 +1268,19 @@ int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y,
   return 0;
 }
 
+std::string get_zonegroup_endpoint(const RGWZoneGroup& info)
+{
+  if (!info.endpoints.empty()) {
+    return info.endpoints.front();
+  }
+  // use zonegroup's master zone endpoints
+  auto z = info.zones.find(info.master_zone);
+  if (z != info.zones.end() && !z->second.endpoints.empty()) {
+    return z->second.endpoints.front();
+  }
+  return "";
+}
+
 int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup,
                       const RGWZoneParams& zone_params,
                       const bool *pis_master, const bool *pread_only,

From 6b3b3d3ef1fc6c354c30d5cfd683f5341ece53bf Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 6 Nov 2023 11:19:44 -0500
Subject: [PATCH 0479/2492] common: admin socket catches exception by reference
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

probably not a real bug, but resolves a compiler warning:

src/common/admin_socket.cc: In member function ‘int RaiseHook::parse_signal(std::string&&, ceph::Formatter*, std::ostream&)’:
src/common/admin_socket.cc:869:21: warning: catching polymorphic type ‘class std::invalid_argument’ by value [-Wcatch-value=]
  869 |       } catch (std::invalid_argument) {
      |                     ^~~~~~~~~~~~~~~~

Fixes: https://tracker.ceph.com/issues/63455

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/admin_socket.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index 2ed3179e8070..c4a37b29ed98 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -866,7 +866,7 @@ class RaiseHook: public AdminSocketHook {
           errss << "signal number should be an integer in the range [1..64]" << std::endl;
           return -EINVAL;
         }
-      } catch (std::invalid_argument) {
+      } catch (const std::invalid_argument&) {
         auto sig_it = known_signals.find(sigdesc);
         if (sig_it == known_signals.end()) {
           errss << "unknown signal name; use -l to see recognized names" << std::endl;

From 3398d2318ff24701de35bc945b5ca8f5204339b5 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 11 Oct 2023 18:27:08 -0400
Subject: [PATCH 0480/2492] cephadm: add modes for bundled deps and track what
 is installed

Add additional modes for how to handle bundled dependencies.
Add a mode to disable bundled dependencies entirely.
Add a mode to source dependencies from installed RPM packages. This mode
will likely be the basis for some distributions of ceph were the exact
version of the package can be used via a controlled build environment.

Add the capability to store metadata about the bundled dependencies
in the zipapp. Move all the metadata created by build.py to a synthetic
package `_cephadmmeta` that gets created when building the zipapp.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 177 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 167 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 87d990e72a35..1cd70d6da92b 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -8,14 +8,15 @@
 import argparse
 import compileall
 import enum
+import json
 import logging
 import os
 import pathlib
+import shlex
 import shutil
 import subprocess
-import tempfile
-import shlex
 import sys
+import tempfile
 
 HAS_ZIPAPP = False
 try:
@@ -62,22 +63,70 @@ def enabled(self):
         return self == self.auto or self == self.required
 
 
+class DependencyMode(enum.Enum):
+    pip = enum.auto()
+    rpm = enum.auto()
+    none = enum.auto()
+
+
 class Config:
     def __init__(self, cli_args):
         self.cli_args = cli_args
         self._maj_min = sys.version_info[0:2]
         self.install_dependencies = True
+        self.deps_mode = DependencyMode[cli_args.bundled_dependencies]
+        if self.deps_mode == DependencyMode.none:
+            self.install_dependencies = False
+        if self.deps_mode == DependencyMode.pip:
+            self._setup_pip()
+        elif self.deps_mode == DependencyMode.rpm:
+            self._setup_rpm()
+
+    def _setup_pip(self):
         if self._maj_min == (3, 6):
             self.pip_split = True
             self.requirements = PY36_REQUIREMENTS
         else:
             self.pip_split = False
             self.requirements = PY_REQUIREMENTS
-        self.pip_venv = PipEnv[cli_args.pip_use_venv]
+        self.pip_venv = PipEnv[self.cli_args.pip_use_venv]
+
+    def _setup_rpm(self):
+        self.requirements = [s.split()[0] for s in PY_REQUIREMENTS]
+
+
+class DependencyInfo:
+    """Type for tracking bundled dependencies."""
+
+    def __init__(self, config):
+        self._config = config
+        self._deps = []
+        self._reqs = {s.split()[0]: s for s in self._config.requirements}
+
+    @property
+    def requirements(self):
+        """Return requirements."""
+        return self._config.requirements
+
+    def add(self, name, **fields):
+        """Add a new bundled dependency to track."""
+        vals = {'name': name}
+        vals.update({k: v for k, v in fields.items() if v is not None})
+        if name in self._reqs:
+            vals['requirements_entry'] = self._reqs[name]
+        self._deps.append(vals)
+
+    def save(self, path):
+        """Record bundled dependency meta-data to the supplied file."""
+        with open(path, 'w') as fh:
+            json.dump(self._deps, fh)
+
 
 
 def _run(command, *args, **kwargs):
-    log.info('Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command))
+    log.info(
+        'Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command)
+    )
     return subprocess.run(command, *args, **kwargs)
 
 
@@ -104,9 +153,10 @@ def _build(dest, src, config):
     os.chdir(src)
     tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
     log.debug("working in %s", tempdir)
+    dinfo = None
     try:
         if config.install_dependencies:
-            _install_deps(tempdir, config)
+            dinfo = _install_deps(tempdir, config)
         log.info("Copying contents")
         # cephadmlib is cephadm's private library of modules
         shutil.copytree(
@@ -115,9 +165,14 @@ def _build(dest, src, config):
         # cephadm.py is cephadm's main script for the "binary"
         # this must be renamed to __main__.py for the zipapp
         shutil.copy("cephadm.py", tempdir / "__main__.py")
+        mdir = tempdir / "_cephadmmeta"
+        mdir.mkdir(parents=True, exist_ok=True)
+        (mdir / "__init__.py").touch(exist_ok=True)
         versioning_vars = config.cli_args.version_vars
         if versioning_vars:
-            generate_version_file(versioning_vars, tempdir / "_version.py")
+            generate_version_file(versioning_vars, mdir / "version.py")
+        if dinfo:
+            dinfo.save(mdir / "deps.json")
         _compile(dest, tempdir)
     finally:
         shutil.rmtree(tempdir)
@@ -128,7 +183,9 @@ def _ignore_cephadmlib(source_dir, names):
     return [
         name
         for name in names
-        if name.endswith(("~", ".old", ".swp", ".pyc", ".pyo", "__pycache__"))
+        if name.endswith(
+            ("~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__")
+        )
     ]
 
 
@@ -163,9 +220,16 @@ def _compile(dest, tempdir):
 
 
 def _install_deps(tempdir, config):
+    if config.deps_mode == DependencyMode.pip:
+        return _install_pip_deps(tempdir, config)
+    if config.deps_mode == DependencyMode.rpm:
+        return _install_rpm_deps(tempdir, config)
+    raise ValueError(f'unexpected deps mode: {deps.mode}')
+
+
+def _install_pip_deps(tempdir, config):
     """Install dependencies with pip."""
-    # TODO we could explicitly pass a python version here
-    log.info("Installing dependencies")
+    log.info("Installing dependencies using pip")
 
     executable = sys.executable
     venv = config.pip_venv
@@ -213,12 +277,29 @@ def _install_deps(tempdir, config):
                 ":all:",
                 "--target",
                 tempdir,
-            ] + batch,
+            ]
+            + batch,
             env=env,
             check=True,
         )
+
+    dinfo = DependencyInfo(config)
+    res = _run(
+        [executable, '-m', 'pip', 'list', '--format=json', '--path', tempdir],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    pkgs = json.loads(res.stdout)
+    for pkg in pkgs:
+        dinfo.add(
+            pkg['name'],
+            version=pkg['version'],
+            package_source='pip',
+        )
+
     if venv:
         shutil.rmtree(venv)
+    return dinfo
 
 
 def _has_python_venv(executable):
@@ -235,6 +316,75 @@ def _has_python_pip(executable):
     return res.returncode == 0
 
 
+def _install_rpm_deps(tempdir, config):
+    log.info("Installing dependencies using RPMs")
+    dinfo = DependencyInfo(config)
+    for pkg in config.requirements:
+        log.info(f"Looking for rpm package for: {pkg!r}")
+        _deps_from_rpm(tempdir, config, dinfo, pkg)
+    return dinfo
+
+
+def _deps_from_rpm(tempdir, config, dinfo, pkg):
+    # first, figure out what rpm provides a particular python lib
+    dist = f'python3dist({pkg})'.lower()
+    try:
+        res = subprocess.run(
+            ['rpm', '-q', '--whatprovides', dist],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError as err:
+        log.error(f"Command failed: {err.args[1]!r}")
+        log.error(f"An installed RPM package for {pkg} was not found")
+        sys.exit(1)
+    rpmname = res.stdout.strip().decode('utf8')
+    # get version information about said rpm
+    res = subprocess.run(
+        ['rpm', '-q', '--qf', '%{version} %{release} %{epoch}\\n', rpmname],
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    vers = res.stdout.decode('utf8').splitlines()[0].split()
+    log.info(f"RPM Package: {rpmname} ({vers})")
+    dinfo.add(
+        pkg,
+        rpm_name=rpmname,
+        version=vers[0],
+        rpm_release=vers[1],
+        rpm_epoch=vers[2],
+        package_source='rpm',
+    )
+    # get the list of files provided by the rpm
+    res = subprocess.run(
+        ['rpm', '-ql', rpmname], check=True, stdout=subprocess.PIPE
+    )
+    paths = [l.decode('utf8') for l in res.stdout.splitlines()]
+    # the top_level.txt file can be used to determine where the python packages
+    # actually are. We need all of those and the meta-data dir (parent of
+    # top_level.txt) to be included in our zipapp
+    top_level = None
+    for path in paths:
+        if path.endswith('top_level.txt'):
+            top_level = pathlib.Path(path)
+    if not top_level:
+        raise ValueError('top_level not found')
+    meta_dir = top_level.parent
+    pkg_dirs = [
+        top_level.parent.parent / p
+        for p in top_level.read_text().splitlines()
+    ]
+    meta_dest = tempdir / meta_dir.name
+    log.info(f"Copying {meta_dir} to {meta_dest}")
+    # copy the meta data directory
+    shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib)
+    # copy all the package directories
+    for pkg_dir in pkg_dirs:
+        pkg_dest = tempdir / pkg_dir.name
+        log.info(f"Copying {pkg_dir} to {pkg_dest}")
+        shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib)
+
+
 def generate_version_file(versioning_vars, dest):
     log.info("Generating version file")
     log.debug("versioning_vars=%r", versioning_vars)
@@ -284,6 +434,13 @@ def main():
         default=PipEnv.auto.name,
         help='Configure pip to use a virtual environment when bundling dependencies',
     )
+    parser.add_argument(
+        "--bundled-dependencies",
+        "-B",
+        choices=[e.name for e in DependencyMode],
+        default=DependencyMode.pip.name,
+        help="Source for bundled dependencies",
+    )
     args = parser.parse_args()
 
     if not _did_rexec() and args.python:

From d762e7156348a99d471e706cc50c39645c1f0579 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 12 Oct 2023 09:44:47 -0400
Subject: [PATCH 0481/2492] script/cpatch.py: support passing extra args to
 cephadm's build.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/cpatch.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/script/cpatch.py b/src/script/cpatch.py
index cbca0587dde8..b22cbb4f7e62 100755
--- a/src/script/cpatch.py
+++ b/src/script/cpatch.py
@@ -217,6 +217,10 @@ def strip_binaries(self):
     def components_selected(self):
         return bool(self._cli.components)
 
+    @property
+    def cephadm_build_args(self):
+        return list(self._cli.cephadm_build_arg or [])
+
     def build_components(self):
         if self._cli.components:
             return self._cli.components
@@ -305,6 +309,12 @@ def parse(cls):
             const=logging.WARNING,
             help="Only print errors and warnings",
         )
+        parser.add_argument(
+            "--cephadm-build-arg",
+            "-A",
+            action="append",
+            help="Pass additional arguments to cephadm build script.",
+        )
         # selectors
         component_selections = [
             # aggregated components:
@@ -596,7 +606,9 @@ def _cephadm_job(self, component):
             if not build_cephadm_path.is_file():
                 raise ValueError("no cephadm build script found")
             log.debug("found cephadm compilation script: compiling cephadm")
-            _run([build_cephadm_path, dst_path]).check_returncode()
+            build_cmd = [build_cephadm_path] + self._ctx.cephadm_build_args
+            build_cmd += [dst_path]
+            _run(build_cmd).check_returncode()
         return ["ADD cephadm /usr/sbin/cephadm"]
 
     def _pybind_job(self, component):

From d54df0d03d3733607466a09554ac4c2fd56a8f1c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 12 Oct 2023 10:06:53 -0400
Subject: [PATCH 0482/2492] cephadm: update cmake to pass bundling parameter

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/CMakeLists.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/CMakeLists.txt b/src/cephadm/CMakeLists.txt
index 8b969bc33e7b..c33572dd4efb 100644
--- a/src/cephadm/CMakeLists.txt
+++ b/src/cephadm/CMakeLists.txt
@@ -5,6 +5,10 @@ endif()
 
 set(bin_target_file ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/cephadm)
 
+if(NOT DEFINED CEPHADM_BUNDLED_DEPENDENCIES)
+  set(CEPHADM_BUNDLED_DEPENDENCIES "pip")
+endif()
+
 add_custom_command(
   OUTPUT "${bin_target_file}"
   DEPENDS
@@ -17,6 +21,7 @@ add_custom_command(
     --set-version-var=CEPH_RELEASE=${CEPH_RELEASE}
     --set-version-var=CEPH_RELEASE_NAME=${CEPH_RELEASE_NAME}
     --set-version-var=CEPH_RELEASE_TYPE=${CEPH_RELEASE_TYPE}
+    --bundled-dependencies=${CEPHADM_BUNDLED_DEPENDENCIES}
     ${bin_target_file}
 )
 

From dc52a352f7699a611d8fabefaeec073b185cd4e8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 12 Oct 2023 16:48:09 -0400
Subject: [PATCH 0483/2492] ceph.spec.in: add build conditions to control
 cephadm bundling

Add two conditions to either disable bundling or switch to rpm based
depdencies. The default is to bundle deps from pip.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 ceph.spec.in | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 6496a0cec785..aef26acf451d 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -124,6 +124,8 @@
 %{!?_selinux_policy_version: %global _selinux_policy_version 0.0.0}
 %endif
 %endif
+%bcond_without cephadm_bundling
+%bcond_without cephadm_pip_deps
 
 %{!?_udevrulesdir: %global _udevrulesdir /lib/udev/rules.d}
 %{!?tmpfiles_create: %global tmpfiles_create systemd-tmpfiles --create}
@@ -527,6 +529,13 @@ Requires:       which
 %if 0%{?weak_deps}
 Recommends:     podman >= 2.0.2
 %endif
+%if 0%{with cephadm_bundling}
+%if 0%{without cephadm_pip_deps}
+BuildRequires: python3-jinja2 >= 2.10
+%endif
+%else
+Requires: python3-jinja2 >= 2.10
+%endif
 %description -n cephadm
 Utility to bootstrap a Ceph cluster and manage Ceph daemons deployed
 with systemd and podman.
@@ -1431,7 +1440,16 @@ cmake .. \
     -DWITH_SEASTAR:BOOL=ON \
     -DWITH_JAEGER:BOOL=OFF \
 %endif
-    -DWITH_GRAFANA:BOOL=ON
+    -DWITH_GRAFANA:BOOL=ON \
+%if 0%{with cephadm_bundling}
+%if 0%{with cephadm_pip_deps}
+    -DCEPHADM_BUNDLED_DEPENDENCIES=pip
+%else
+    -DCEPHADM_BUNDLED_DEPENDENCIES=rpm
+%endif
+%else
+    -DCEPHADM_BUNDLED_DEPENDENCIES=none
+%endif
 
 %if %{with cmake_verbose_logging}
 cat ./CMakeFiles/CMakeOutput.log

From 1b8e8be4e79d76662ad8c97c0788504894762f3c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 23 Oct 2023 18:34:46 -0400
Subject: [PATCH 0484/2492] cephadm: add cephadm version --verbose option

The command `cephadm version --verbose` will emit JSON formatted output
detailing the metadata in the versions module as well as information
about what dependencies were bundled into the zipapp and a summary of
the contents of the zipapp. This should make it fairly easy to
determine the provenance of a given cephadm zipapp without needing to
unpack and analyze all the contents of the zip.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 77 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 67 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c56bd6bc2a2c..8f093001e9dc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3901,21 +3901,73 @@ def command_agent(ctx: CephadmContext) -> None:
 def command_version(ctx):
     # type: (CephadmContext) -> int
     import importlib
+    import zipimport
+    import types
 
+    vmod: Optional[types.ModuleType]
+    zmod: Optional[types.ModuleType]
     try:
-        vmod = importlib.import_module('_version')
+        vmod = importlib.import_module('_cephadmmeta.version')
+        zmod = vmod
     except ImportError:
-        print('cephadm version UNKNOWN')
-        return 1
-    _unset = '<UNSET>'
-    print('cephadm version {0} ({1}) {2} ({3})'.format(
-        getattr(vmod, 'CEPH_GIT_NICE_VER', _unset),
-        getattr(vmod, 'CEPH_GIT_VER', _unset),
-        getattr(vmod, 'CEPH_RELEASE_NAME', _unset),
-        getattr(vmod, 'CEPH_RELEASE_TYPE', _unset),
-    ))
+        vmod = zmod = None
+    if vmod is None:
+        # fallback to earlier location
+        try:
+            vmod = importlib.import_module('_version')
+        except ImportError:
+            pass
+    if zmod is None:
+        # fallback to outer package, for zip import module
+        try:
+            zmod = importlib.import_module('_cephadmmeta')
+        except ImportError:
+            zmod = None
+
+    if not ctx.verbose:
+        if vmod is None:
+            print('cephadm version UNKNOWN')
+            return 1
+        _unset = '<UNSET>'
+        print(
+            'cephadm version {0} ({1}) {2} ({3})'.format(
+                getattr(vmod, 'CEPH_GIT_NICE_VER', _unset),
+                getattr(vmod, 'CEPH_GIT_VER', _unset),
+                getattr(vmod, 'CEPH_RELEASE_NAME', _unset),
+                getattr(vmod, 'CEPH_RELEASE_TYPE', _unset),
+            )
+        )
+        return 0
+
+    out: Dict[str, Any] = {'name': 'cephadm'}
+    ceph_vars = [
+        'CEPH_GIT_NICE_VER',
+        'CEPH_GIT_VER',
+        'CEPH_RELEASE_NAME',
+        'CEPH_RELEASE_TYPE',
+    ]
+    for var in ceph_vars:
+        value = getattr(vmod, var, None)
+        if value is not None:
+            out[var.lower()] = value
+
+    loader = getattr(zmod, '__loader__', None)
+    if loader and isinstance(loader, zipimport.zipimporter):
+        try:
+            deps_info = json.loads(loader.get_data('_cephadmmeta/deps.json'))
+            out['bundled_packages'] = deps_info
+        except OSError:
+            pass
+        files = getattr(loader, '_files', {})
+        out['zip_root_entries'] = sorted(
+            {p.split('/')[0] for p in files.keys()}
+        )
+
+    json.dump(out, sys.stdout, indent=2)
+    print()
     return 0
 
+
 ##################################
 
 
@@ -6947,6 +6999,11 @@ def _get_parser():
     parser_version = subparsers.add_parser(
         'version', help='get cephadm version')
     parser_version.set_defaults(func=command_version)
+    parser_version.add_argument(
+        '--verbose',
+        action='store_true',
+        help='Detailed version information',
+    )
 
     parser_pull = subparsers.add_parser(
         'pull', help='pull the default container image')

From fed4066a859044ba272beee95d9deb577b82918b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 24 Oct 2023 16:48:22 -0400
Subject: [PATCH 0485/2492] doc/dev/cephadm: describe advanced build options

Describe the advanced build options provided by build.py and how
to use `cephadm version --verbose` to print the internal metadata.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 doc/dev/cephadm/developing-cephadm.rst | 61 ++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/doc/dev/cephadm/developing-cephadm.rst b/doc/dev/cephadm/developing-cephadm.rst
index 49b771caa5c8..aebe002da88c 100644
--- a/doc/dev/cephadm/developing-cephadm.rst
+++ b/doc/dev/cephadm/developing-cephadm.rst
@@ -401,3 +401,64 @@ own copy of the cephadm "binary" use the script located at
 ``./src/cephadm/build.py [output]``.
 
 .. _Python Zip Application: https://peps.python.org/pep-0441/
+
+You can pass a limited set of version metadata values to be stored in the
+compiled cepadm. These options can be passed to the build script with
+the ``--set-version-var`` or ``-S`` option. The values should take the form
+``KEY=VALUE`` and valid keys include:
+* ``CEPH_GIT_VER``
+* ``CEPH_GIT_NICE_VER``
+* ``CEPH_RELEASE``
+* ``CEPH_RELEASE_NAME``
+* ``CEPH_RELEASE_TYPE``
+
+Example: ``./src/cephadm/build.py -SCEPH_GIT_VER=$(git rev-parse HEAD) -SCEPH_GIT_NICE_VER=$(git describe) /tmp/cephadm``
+
+Typically these values will be passed to build.py by other, higher level, build
+tools - such as cmake.
+
+The compiled version of the binary may include a curated set of dependencies
+within the zipapp. The tool used to fetch the bundled dependencies can be
+Python's ``pip``, locally installed RPMs, or bundled dependencies can be
+disabled. To select the mode for bundled dependencies use the
+``--bundled-dependencies`` or ``-B`` option with a value of ``pip``, ``rpm``,
+or ``none``.
+
+The compiled cephadm zipapp file retains metadata about how it was built. This
+can be displayed by running ``cephadm version --verbose``.  The command will
+emit a JSON formatted object showing version metadata (if available), a list of
+the bundled dependencies generated by the build script (if bundled dependencies
+were enabled), and a summary of the top-level contents of the zipapp. Example::
+
+  $ ./cephadm version --verbose
+  {
+    "name": "cephadm",
+    "ceph_git_nice_ver": "18.0.0-6867-g6a1df2d0b01",
+    "ceph_git_ver": "6a1df2d0b01da581bfef3357940e1e88d5ce70ce",
+    "ceph_release_name": "reef",
+    "ceph_release_type": "dev",
+    "bundled_packages": [
+      {
+        "name": "Jinja2",
+        "version": "3.1.2",
+        "package_source": "pip",
+        "requirements_entry": "Jinja2 == 3.1.2"
+      },
+      {
+        "name": "MarkupSafe",
+        "version": "2.1.3",
+        "package_source": "pip",
+        "requirements_entry": "MarkupSafe == 2.1.3"
+      }
+    ],
+    "zip_root_entries": [
+      "Jinja2-3.1.2-py3.9.egg-info",
+      "MarkupSafe-2.1.3-py3.9.egg-info",
+      "__main__.py",
+      "__main__.pyc",
+      "_cephadmmeta",
+      "cephadmlib",
+      "jinja2",
+      "markupsafe"
+    ]
+  }

From 957b8bd443b7891114c4fc2ed0fe0574eb924c66 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 2 Nov 2023 17:19:34 -0400
Subject: [PATCH 0486/2492] cephadm: extend the cephadm build test to cover
 version --verbose

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/build/test_cephadm_build.py | 34 +++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/tests/build/test_cephadm_build.py b/src/cephadm/tests/build/test_cephadm_build.py
index b6fea0e8a9ea..11c85f1f3250 100644
--- a/src/cephadm/tests/build/test_cephadm_build.py
+++ b/src/cephadm/tests/build/test_cephadm_build.py
@@ -3,6 +3,7 @@
 # these should not be run automatically as they require the use of podman,
 # which should not be assumed to exist on a typical test node
 
+import json
 import os
 import pathlib
 import pytest
@@ -92,5 +93,34 @@ def source_dir():
 )
 def test_cephadm_build(env, source_dir, tmp_path):
     build_in(env, source_dir, tmp_path, [])
-    assert (tmp_path / 'cephadm').is_file()
-    # TODO: verify contents of zip
+    binary = tmp_path / 'cephadm'
+    assert binary.is_file()
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version'],
+        stdout=subprocess.PIPE,
+    )
+    out = res.stdout.decode('utf8')
+    assert 'version' in out
+    assert 'UNKNOWN' in out
+    assert res.returncode != 0
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version', '--verbose'],
+        stdout=subprocess.PIPE,
+    )
+    data = json.loads(res.stdout)
+    assert isinstance(data, dict)
+    assert 'bundled_packages' in data
+    assert all(v['package_source'] == 'pip' for v in data['bundled_packages'])
+    assert all(
+        v['name'] in ('Jinja2', 'MarkupSafe')
+        for v in data['bundled_packages']
+    )
+    assert all('requirements_entry' in v for v in data['bundled_packages'])
+    assert 'zip_root_entries' in data
+    zre = data['zip_root_entries']
+    assert any(e.startswith('Jinja2') for e in zre)
+    assert any(e.startswith('MarkupSafe') for e in zre)
+    assert any(e.startswith('jinja2') for e in zre)
+    assert any(e.startswith('markupsafe') for e in zre)
+    assert any(e.startswith('cephadmlib') for e in zre)
+    assert any(e.startswith('_cephadmmeta') for e in zre)

From 8da38f303eec26385fc5131d23709986f5aa6edd Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 2 Nov 2023 18:04:42 -0400
Subject: [PATCH 0487/2492] cephadm: add cephadm build tests for bundled
 sourced from rpms

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/build/test_cephadm_build.py | 68 ++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/tests/build/test_cephadm_build.py b/src/cephadm/tests/build/test_cephadm_build.py
index 11c85f1f3250..7cf51961ffca 100644
--- a/src/cephadm/tests/build/test_cephadm_build.py
+++ b/src/cephadm/tests/build/test_cephadm_build.py
@@ -22,6 +22,16 @@
         'base_image': 'quay.io/centos/centos:stream9',
         'script': 'dnf install -y python3',
     },
+    'centos-8-plusdeps': {
+        'name': 'cephadm-build-test:centos8-py36-deps',
+        'base_image': 'quay.io/centos/centos:stream8',
+        'script': 'dnf install -y python36 python3-jinja2',
+    },
+    'centos-9-plusdeps': {
+        'name': 'cephadm-build-test:centos9-py3-deps',
+        'base_image': 'quay.io/centos/centos:stream9',
+        'script': 'dnf install -y python3 python3-jinja2',
+    },
     'ubuntu-20.04': {
         'name': 'cephadm-build-test:ubuntu-20-04-py3',
         'base_image': 'docker.io/library/ubuntu:20.04',
@@ -74,7 +84,7 @@ def build_in(alias, ceph_dir, out_dir, args):
     ctr = CONTAINERS[alias]
     build_container(ctr['base_image'], ctr['name'], ctr['script'], out_dir)
     cmd = ['/ceph/' + BUILD_PY] + list(args or []) + ['/out/cephadm']
-    run_container_cmd(ctr['name'], cmd, ceph_dir, out_dir)
+    return run_container_cmd(ctr['name'], cmd, ceph_dir, out_dir)
 
 
 @pytest.fixture
@@ -124,3 +134,59 @@ def test_cephadm_build(env, source_dir, tmp_path):
     assert any(e.startswith('markupsafe') for e in zre)
     assert any(e.startswith('cephadmlib') for e in zre)
     assert any(e.startswith('_cephadmmeta') for e in zre)
+
+
+@pytest.mark.parametrize(
+    'env',
+    [
+        'centos-8-plusdeps',
+        'centos-9-plusdeps',
+        'centos-9',
+    ],
+)
+def test_cephadm_build_from_rpms(env, source_dir, tmp_path):
+    res = build_in(
+        env,
+        source_dir,
+        tmp_path,
+        ['-Brpm', '-SCEPH_GIT_VER=0', '-SCEPH_GIT_NICE_VER=foobar'],
+    )
+    if 'plusdeps' not in env:
+        assert res.returncode != 0
+        return
+    binary = tmp_path / 'cephadm'
+    if 'centos-8' in env and sys.version_info[:2] >= (3, 10):
+        # The version of markupsafe in centos 8 is incompatible with
+        # python>=3.10 due to changes in the stdlib therefore we can't execute
+        # the cephadm binary, so we quit the test early.
+        return
+    assert binary.is_file()
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version'],
+        stdout=subprocess.PIPE,
+    )
+    out = res.stdout.decode('utf8')
+    assert 'version' in out
+    assert 'foobar' in out
+    assert res.returncode == 0
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version', '--verbose'],
+        stdout=subprocess.PIPE,
+    )
+    data = json.loads(res.stdout)
+    assert isinstance(data, dict)
+    assert 'bundled_packages' in data
+    assert all(v['package_source'] == 'rpm' for v in data['bundled_packages'])
+    assert all(
+        v['name'] in ('Jinja2', 'MarkupSafe')
+        for v in data['bundled_packages']
+    )
+    assert all('requirements_entry' in v for v in data['bundled_packages'])
+    assert 'zip_root_entries' in data
+    zre = data['zip_root_entries']
+    assert any(e.startswith('Jinja2') for e in zre)
+    assert any(e.startswith('MarkupSafe') for e in zre)
+    assert any(e.startswith('jinja2') for e in zre)
+    assert any(e.startswith('markupsafe') for e in zre)
+    assert any(e.startswith('cephadmlib') for e in zre)
+    assert any(e.startswith('_cephadmmeta') for e in zre)

From 6ae78884d8f071d6a92d375bd2befbaea61d671f Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 6 Nov 2023 15:11:58 -0500
Subject: [PATCH 0488/2492] rgwlc: fix delete marker regression in
 currentversion lc expiration

Fixes a regression introduced in https://github.com/ceph/ceph/pull/50680,
which attempted to re-use an already created sal object handle as
an optimiztion.  This change suppressed creation of delete markers in
current version expiration when buckets are versioned.

Fixes: https://tracker.ceph.com/issues/63458

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 15e70977e726..a375b63e0c2d 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -522,15 +522,19 @@ static int remove_expired_obj(
   auto& version_id = obj_key.instance;
   std::unique_ptr<rgw::sal::Notification> notify;
 
+  std::unique_ptr<rgw::sal::User> user;
+  user = driver->get_user(bucket_info.owner);
+
+  /* per discussion w/Daniel, Casey,and Eric, we *do need*
+   * a new sal object handle, based on the following decision
+   * to clear obj_key.instance--which happens in the case
+   * where a delete marker should be created */
   if (!remove_indeed) {
     obj_key.instance.clear();
   } else if (obj_key.instance.empty()) {
     obj_key.instance = "null";
   }
-
-  auto& obj = oc.obj;
-  std::unique_ptr<rgw::sal::User> user;
-  user = driver->get_user(bucket_info.owner);
+  auto obj = oc.bucket->get_object(obj_key);
 
   RGWObjState* obj_state{nullptr};
   ret = obj->get_obj_state(dpp, &obj_state, null_yield, true);

From d821821e8324726c755ee40e9420299c59443261 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 17 Aug 2023 11:53:20 +0800
Subject: [PATCH 0489/2492] crimson/osd: interrupt_cond based logs

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/common/interruptible_future.h     |  4 +
 src/crimson/common/log.h                      | 32 ++++++++
 .../osd/osd_operations/background_recovery.cc | 23 ++++--
 .../osd/osd_operations/client_request.cc      | 78 ++++++++++++-------
 .../osd_operations/internal_client_request.cc | 13 +++-
 .../osd/osd_operations/logmissing_request.cc  |  8 +-
 .../osd/osd_operations/peering_event.cc       | 28 +++++--
 .../osd/osd_operations/recovery_subrequest.cc | 12 ++-
 .../osd/osd_operations/replicated_request.cc  |  8 +-
 9 files changed, 151 insertions(+), 55 deletions(-)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index c0e2c346c88b..1975295ef94b 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -1060,6 +1060,10 @@ struct interruptor
 public:
   using condition = InterruptCond;
 
+  static const void *get_interrupt_cond() {
+    return (const void*)interrupt_cond<InterruptCond>.interrupt_cond.get();
+  }
+
   template <typename FutureType>
   [[gnu::always_inline]]
   static interruptible_future_detail<InterruptCond, FutureType>
diff --git a/src/crimson/common/log.h b/src/crimson/common/log.h
index 27ff550d86be..4f564ac044d0 100644
--- a/src/crimson/common/log.h
+++ b/src/crimson/common/log.h
@@ -52,37 +52,69 @@ static inline seastar::log_level to_log_level(int level) {
   LOCAL_LOGGER.log(level_, "{}: " MSG, FNAME , ##__VA_ARGS__)
 #define SUBLOG(subname_, level_, MSG, ...) \
   LOGGER(subname_).log(level_, "{}: " MSG, FNAME , ##__VA_ARGS__)
+#define LOGI(level_, MSG, ...) \
+  LOCAL_LOGGER.log(level_, "{} {}: " MSG, \
+    interruptor::get_interrupt_cond(), FNAME , ##__VA_ARGS__)
+#define SUBLOGI(subname_, level_, MSG, ...) \
+  LOGGER(subname_).log(level_, "{} {}: " MSG, \
+    interruptor::get_interrupt_cond(), FNAME , ##__VA_ARGS__)
 
 #define TRACE(...) LOG(seastar::log_level::trace, __VA_ARGS__)
+#define TRACEI(...) LOGI(seastar::log_level::trace, __VA_ARGS__)
 #define SUBTRACE(subname_, ...) SUBLOG(subname_, seastar::log_level::trace, __VA_ARGS__)
+#define SUBTRACEI(subname_, ...) SUBLOGI(subname_, seastar::log_level::trace, __VA_ARGS__)
 
 #define DEBUG(...) LOG(seastar::log_level::debug, __VA_ARGS__)
+#define DEBUGI(...) LOGI(seastar::log_level::debug, __VA_ARGS__)
 #define SUBDEBUG(subname_, ...) SUBLOG(subname_, seastar::log_level::debug, __VA_ARGS__)
+#define SUBDEBUGI(subname_, ...) SUBLOGI(subname_, seastar::log_level::debug, __VA_ARGS__)
 
 #define INFO(...) LOG(seastar::log_level::info, __VA_ARGS__)
+#define INFOI(...) LOGI(seastar::log_level::info, __VA_ARGS__)
 #define SUBINFO(subname_, ...) SUBLOG(subname_, seastar::log_level::info, __VA_ARGS__)
+#define SUBINFOI(subname_, ...) SUBLOGI(subname_, seastar::log_level::info, __VA_ARGS__)
 
 #define WARN(...) LOG(seastar::log_level::warn, __VA_ARGS__)
+#define WARNI(...) LOGI(seastar::log_level::warn, __VA_ARGS__)
 #define SUBWARN(subname_, ...) SUBLOG(subname_, seastar::log_level::warn, __VA_ARGS__)
+#define SUBWARNI(subname_, ...) SUBLOGI(subname_, seastar::log_level::warn, __VA_ARGS__)
 
 #define ERROR(...) LOG(seastar::log_level::error, __VA_ARGS__)
+#define ERRORI(...) LOGI(seastar::log_level::error, __VA_ARGS__)
 #define SUBERROR(subname_, ...) SUBLOG(subname_, seastar::log_level::error, __VA_ARGS__)
+#define SUBERRORI(subname_, ...) SUBLOGI(subname_, seastar::log_level::error, __VA_ARGS__)
 
 // *DPP macros are intended to take DoutPrefixProvider implementations, but anything with
 // an operator<< will work as a prefix
 
 #define SUBLOGDPP(subname_, level_, MSG, dpp, ...) \
   LOGGER(subname_).log(level_, "{} {}: " MSG, dpp, FNAME , ##__VA_ARGS__)
+#define SUBLOGDPPI(subname_, level_, MSG, dpp, ...) \
+  LOGGER(subname_).log(level_, "{} {}: " MSG, \
+  interruptor::get_interrupt_cond(), dpp, FNAME , ##__VA_ARGS__)
 #define SUBTRACEDPP(subname_, ...) SUBLOGDPP(subname_, seastar::log_level::trace, __VA_ARGS__)
+#define SUBTRACEDPPI(subname_, ...) SUBLOGDPPI(subname_, seastar::log_level::trace, __VA_ARGS__)
 #define SUBDEBUGDPP(subname_, ...) SUBLOGDPP(subname_, seastar::log_level::debug, __VA_ARGS__)
+#define SUBDEBUGDPPI(subname_, ...) SUBLOGDPPI(subname_, seastar::log_level::debug, __VA_ARGS__)
 #define SUBINFODPP(subname_, ...) SUBLOGDPP(subname_, seastar::log_level::info, __VA_ARGS__)
+#define SUBINFODPPI(subname_, ...) SUBLOGDPPI(subname_, seastar::log_level::info, __VA_ARGS__)
 #define SUBWARNDPP(subname_, ...) SUBLOGDPP(subname_, seastar::log_level::warn, __VA_ARGS__)
+#define SUBWARNDPPI(subname_, ...) SUBLOGDPPI(subname_, seastar::log_level::warn, __VA_ARGS__)
 #define SUBERRORDPP(subname_, ...) SUBLOGDPP(subname_, seastar::log_level::error, __VA_ARGS__)
+#define SUBERRORDPPI(subname_, ...) SUBLOGDPPI(subname_, seastar::log_level::error, __VA_ARGS__)
 
 #define LOGDPP(level_, MSG, dpp, ...) \
   LOCAL_LOGGER.log(level_, "{} {}: " MSG, dpp, FNAME , ##__VA_ARGS__)
+#define LOGDPPI(level_, MSG, dpp, ...) \
+  LOCAL_LOGGER.log(level_, "{} {}: " MSG, \
+  interruptor::get_interrupt_cond(), dpp, FNAME , ##__VA_ARGS__)
 #define TRACEDPP(...) LOGDPP(seastar::log_level::trace, __VA_ARGS__)
+#define TRACEDPPI(...) LOGDPPI(seastar::log_level::trace, __VA_ARGS__)
 #define DEBUGDPP(...) LOGDPP(seastar::log_level::debug, __VA_ARGS__)
+#define DEBUGDPPI(...) LOGDPPI(seastar::log_level::debug, __VA_ARGS__)
 #define INFODPP(...) LOGDPP(seastar::log_level::info, __VA_ARGS__)
+#define INFODPPI(...) LOGDPPI(seastar::log_level::info, __VA_ARGS__)
 #define WARNDPP(...) LOGDPP(seastar::log_level::warn, __VA_ARGS__)
+#define WARNDPPI(...) LOGDPPI(seastar::log_level::warn, __VA_ARGS__)
 #define ERRORDPP(...) LOGDPP(seastar::log_level::error, __VA_ARGS__)
+#define ERRORDPPI(...) LOGDPPI(seastar::log_level::error, __VA_ARGS__)
diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc
index 74bd238c987b..1b9c5ba9b49d 100644
--- a/src/crimson/osd/osd_operations/background_recovery.cc
+++ b/src/crimson/osd/osd_operations/background_recovery.cc
@@ -34,6 +34,8 @@ namespace crimson {
   };
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 template <class T>
@@ -70,9 +72,11 @@ void BackgroundRecoveryT<T>::dump_detail(Formatter *f) const
 template <class T>
 seastar::future<> BackgroundRecoveryT<T>::start()
 {
-  logger().debug("{}: start", *this);
-
   typename T::IRef ref = static_cast<T*>(this);
+  using interruptor = typename T::interruptor;
+
+  LOG_PREFIX(BackgroundRecoveryT<T>::start);
+  DEBUGDPPI("{}: start", *pg, *this);
   auto maybe_delay = seastar::now();
   if (delay) {
     maybe_delay = seastar::sleep(
@@ -84,14 +88,15 @@ seastar::future<> BackgroundRecoveryT<T>::start()
       return ss.with_throttle_while(
         std::move(trigger),
         this, get_scheduler_params(), [this] {
-          return T::interruptor::with_interruption([this] {
+          return interruptor::with_interruption([this] {
             return do_recovery();
           }, [](std::exception_ptr) {
             return seastar::make_ready_future<bool>(false);
           }, pg);
         }).handle_exception_type([ref, this](const std::system_error& err) {
+	  LOG_PREFIX(BackgroundRecoveryT<T>::start);
           if (err.code() == std::make_error_code(std::errc::interrupted)) {
-            logger().debug("{} recovery interruped: {}", *pg, err.what());
+            DEBUGDPPI("recovery interruped: {}", *pg, err.what());
             return seastar::now();
           }
           return seastar::make_exception_future<>(err);
@@ -115,7 +120,8 @@ UrgentRecovery::UrgentRecovery(
 UrgentRecovery::interruptible_future<bool>
 UrgentRecovery::do_recovery()
 {
-  logger().debug("{}: {}", __func__, *this);
+  LOG_PREFIX(UrgentRecovery::do_recovery);
+  DEBUGDPPI("{}: {}", *pg, __func__, *this);
   if (!pg->has_reset_since(epoch_started)) {
     return with_blocking_event<RecoveryBackend::RecoveryBlockingEvent,
 			       interruptor>([this] (auto&& trigger) {
@@ -180,11 +186,12 @@ PGPeeringPipeline &BackfillRecovery::peering_pp(PG &pg)
 BackfillRecovery::interruptible_future<bool>
 BackfillRecovery::do_recovery()
 {
-  logger().debug("{}", __func__);
+  LOG_PREFIX(BackfillRecovery::do_recovery);
+  DEBUGDPPI("{}", *pg, __func__);
 
   if (pg->has_reset_since(epoch_started)) {
-    logger().debug("{}: pg got reset since epoch_started={}",
-                   __func__, epoch_started);
+    DEBUGDPPI("{}: pg got reset since epoch_started={}",
+		*pg, __func__, epoch_started);
     return seastar::make_ready_future<bool>(false);
   }
   // TODO: limits
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 287072642953..d208e2e53d97 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -19,14 +19,17 @@ namespace {
   }
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 
 void ClientRequest::Orderer::requeue(
   ShardServices &shard_services, Ref<PG> pg)
 {
+  LOG_PREFIX(ClientRequest::Orderer::requeue);
   for (auto &req: list) {
-    logger().debug("{}: {} requeueing {}", __func__, *pg, req);
+    DEBUGI("{}: {} requeueing {}", __func__, *pg, req);
     req.reset_instance_handle();
     std::ignore = req.with_pg_int(shard_services, pg);
   }
@@ -34,8 +37,9 @@ void ClientRequest::Orderer::requeue(
 
 void ClientRequest::Orderer::clear_and_cancel()
 {
+  LOG_PREFIX(ClientRequest::Orderer::clear_and_cancel);
   for (auto i = list.begin(); i != list.end(); ) {
-    logger().debug(
+    DEBUGI(
       "ClientRequest::Orderer::clear_and_cancel: {}",
       *i);
     i->complete_request();
@@ -60,7 +64,8 @@ ClientRequest::ClientRequest(
 
 ClientRequest::~ClientRequest()
 {
-  logger().debug("{}: destroying", *this);
+  LOG_PREFIX(ClientRequest::~ClientRequest);
+  DEBUGI("{}: destroying", *this);
 }
 
 void ClientRequest::print(std::ostream &lhs) const
@@ -70,7 +75,8 @@ void ClientRequest::print(std::ostream &lhs) const
 
 void ClientRequest::dump_detail(Formatter *f) const
 {
-  logger().debug("{}: dumping", *this);
+  LOG_PREFIX(ClientRequest::dump_detail);
+  DEBUGI("{}: dumping", *this);
   std::apply([f] (auto... event) {
     (..., event.dump(f));
   }, tracking_events);
@@ -102,8 +108,9 @@ bool ClientRequest::is_pg_op() const
 seastar::future<> ClientRequest::with_pg_int(
   ShardServices &shard_services, Ref<PG> pgref)
 {
+  LOG_PREFIX(ClientRequest::with_pg_int);
   epoch_t same_interval_since = pgref->get_interval_start_epoch();
-  logger().debug("{} same_interval_since: {}", *this, same_interval_since);
+  DEBUGI("{} same_interval_since: {}", *this, same_interval_since);
   if (m->finish_decode()) {
     m->clear_payload();
   }
@@ -113,12 +120,15 @@ seastar::future<> ClientRequest::with_pg_int(
   auto &ihref = *instance_handle;
   return interruptor::with_interruption(
     [this, pgref, this_instance_id, &ihref, &shard_services]() mutable {
+      LOG_PREFIX(ClientRequest::with_pg_int);
+      DEBUGI("{} start", *this);
       PG &pg = *pgref;
       if (pg.can_discard_op(*m)) {
 	return shard_services.send_incremental_map(
 	  std::ref(*conn), m->get_map_epoch()
 	).then([this, this_instance_id, pgref] {
-	  logger().debug("{}.{}: discarding", *this, this_instance_id);
+          LOG_PREFIX(ClientRequest::with_pg_int);
+	  DEBUGI("{}.{}: discarding", *this, this_instance_id);
 	  pgref->client_request_orderer.remove_request(*this);
 	  complete_request();
 	  return interruptor::now();
@@ -126,24 +136,26 @@ seastar::future<> ClientRequest::with_pg_int(
       }
       return ihref.enter_stage<interruptor>(client_pp(pg).await_map, *this
       ).then_interruptible([this, this_instance_id, &pg, &ihref] {
-	logger().debug("{}.{}: after await_map stage", *this, this_instance_id);
+        LOG_PREFIX(ClientRequest::with_pg_int);
+	DEBUGI("{}.{}: after await_map stage", *this, this_instance_id);
 	return ihref.enter_blocker(
 	  *this, pg.osdmap_gate, &decltype(pg.osdmap_gate)::wait_for_map,
 	  m->get_min_epoch(), nullptr);
       }).then_interruptible([this, this_instance_id, &pg, &ihref](auto map) {
-	logger().debug("{}.{}: after wait_for_map", *this, this_instance_id);
+        LOG_PREFIX(ClientRequest::with_pg_int);
+	DEBUGI("{}.{}: after wait_for_map", *this, this_instance_id);
 	return ihref.enter_stage<interruptor>(client_pp(pg).wait_for_active, *this);
       }).then_interruptible([this, this_instance_id, &pg, &ihref]() {
-	logger().debug(
-	  "{}.{}: after wait_for_active stage", *this, this_instance_id);
+        LOG_PREFIX(ClientRequest::with_pg_int);
+	DEBUGI("{}.{}: after wait_for_active stage", *this, this_instance_id);
 	return ihref.enter_blocker(
 	  *this,
 	  pg.wait_for_active_blocker,
 	  &decltype(pg.wait_for_active_blocker)::wait);
       }).then_interruptible([this, pgref, this_instance_id, &ihref]() mutable
 			    -> interruptible_future<> {
-	logger().debug(
-	  "{}.{}: after wait_for_active", *this, this_instance_id);
+        LOG_PREFIX(ClientRequest::with_pg_int);
+	DEBUGI("{}.{}: after wait_for_active", *this, this_instance_id);
 	if (is_pg_op()) {
 	  return process_pg_op(pgref);
 	} else {
@@ -153,13 +165,15 @@ seastar::future<> ClientRequest::with_pg_int(
         logger().debug("{}.{}: complete", *this, this_instance_id);
         return ihref.handle.complete();
       }).then_interruptible([this, this_instance_id, pgref] {
-	logger().debug("{}.{}: after process*", *this, this_instance_id);
+        LOG_PREFIX(ClientRequest::with_pg_int);
+	DEBUGI("{}.{}: after process*", *this, this_instance_id);
 	pgref->client_request_orderer.remove_request(*this);
 	complete_request();
       });
     }, [this, this_instance_id, pgref](std::exception_ptr eptr) {
+      LOG_PREFIX(ClientRequest::with_pg_int);
       // TODO: better debug output
-      logger().debug("{}.{}: interrupted {}", *this, this_instance_id, eptr);
+      DEBUGI("{}.{}: interrupted {}", *this, this_instance_id, eptr);
     },
     pgref
   ).finally(
@@ -196,7 +210,8 @@ ClientRequest::process_pg_op(
 
 auto ClientRequest::reply_op_error(const Ref<PG>& pg, int err)
 {
-  logger().debug("{}: replying with error {}", *this, err);
+  LOG_PREFIX(ClientRequest::reply_op_error);
+  DEBUGI("{}: replying with error {}", *this, err);
   auto reply = crimson::make_message<MOSDOpReply>(
     m.get(), err, pg->get_osdmap_epoch(),
     m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK),
@@ -214,10 +229,11 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
     *this
   ).then_interruptible(
     [this, pg]() mutable {
+    LOG_PREFIX(ClientRequest::process_op);
     if (pg->is_primary()) {
       return do_recover_missing(pg, m->get_hobj());
     } else {
-      logger().debug("process_op: Skipping do_recover_missing"
+      DEBUGI("process_op: Skipping do_recover_missing"
                      "on non primary pg");
       return interruptor::now();
     }
@@ -235,12 +251,14 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
         return ihref.enter_stage<interruptor>(client_pp(*pg).get_obc, *this
 	).then_interruptible(
           [this, pg, &ihref]() mutable -> PG::load_obc_iertr::future<> {
-          logger().debug("{}: in get_obc stage", *this);
+          LOG_PREFIX(ClientRequest::process_op);
+          DEBUGI("{}: in get_obc stage", *this);
           op_info.set_from_op(&*m, *pg->get_osdmap());
           return pg->with_locked_obc(
             m->get_hobj(), op_info,
             [this, pg, &ihref](auto obc) mutable {
-              logger().debug("{}: got obc {}", *this, obc->obs);
+              LOG_PREFIX(ClientRequest::process_op);
+              DEBUGI("{}: got obc {}", *this, obc->obs);
               return ihref.enter_stage<interruptor>(
                 client_pp(*pg).process, *this
               ).then_interruptible([this, pg, obc, &ihref]() mutable {
@@ -252,7 +270,8 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
     });
   }).handle_error_interruptible(
     PG::load_obc_ertr::all_same_way([this, pg=std::move(pg)](const auto &code) {
-      logger().error("ClientRequest saw error code {}", code);
+      LOG_PREFIX(ClientRequest::process_op);
+      ERRORI("ClientRequest saw error code {}", code);
       assert(code.value() > 0);
       return reply_op_error(pg, -code.value());
   }));
@@ -263,6 +282,7 @@ ClientRequest::do_process(
   instance_handle_t &ihref,
   Ref<PG>& pg, crimson::osd::ObjectContextRef obc)
 {
+  LOG_PREFIX(ClientRequest::do_process);
   if (m->has_flag(CEPH_OSD_FLAG_PARALLELEXEC)) {
     return reply_op_error(pg, -EINVAL);
   }
@@ -270,10 +290,10 @@ ClientRequest::do_process(
   if (pool.has_flag(pg_pool_t::FLAG_EIO)) {
     // drop op on the floor; the client will handle returning EIO
     if (m->has_flag(CEPH_OSD_FLAG_SUPPORTSPOOLEIO)) {
-      logger().debug("discarding op due to pool EIO flag");
+      DEBUGI("discarding op due to pool EIO flag");
       return seastar::now();
     } else {
-      logger().debug("replying EIO due to pool EIO flag");
+      DEBUGI("replying EIO due to pool EIO flag");
       return reply_op_error(pg, -EIO);
     }
   }
@@ -301,7 +321,7 @@ ClientRequest::do_process(
 
   if ((m->has_flag(CEPH_OSD_FLAG_ORDERSNAP)) &&
        snapc.seq < obc->ssc->snapset.seq) {
-        logger().debug("{} ORDERSNAP flag set and snapc seq {}",
+        DEBUGI("{} ORDERSNAP flag set and snapc seq {}",
                        " < snapset seq {} on {}",
                        __func__, snapc.seq, obc->ssc->snapset.seq,
                        obc->obs.oi.soid);
@@ -311,16 +331,16 @@ ClientRequest::do_process(
   if (!pg->is_primary()) {
     // primary can handle both normal ops and balanced reads
     if (is_misdirected(*pg)) {
-      logger().trace("do_process: dropping misdirected op");
+      TRACEI("do_process: dropping misdirected op");
       return seastar::now();
     } else if (const hobject_t& hoid = m->get_hobj();
                !pg->get_peering_state().can_serve_replica_read(hoid)) {
-      logger().debug("{}: unstable write on replica, "
+      DEBUGI("{}: unstable write on replica, "
 	             "bouncing to primary",
                      __func__);
       return reply_op_error(pg, -EAGAIN);
     } else {
-      logger().debug("{}: serving replica read on oid {}",
+      DEBUGI("{}: serving replica read on oid {}",
                      __func__, m->get_hobj());
     }
   }
@@ -335,7 +355,8 @@ ClientRequest::do_process(
 	      return ihref.enter_stage<interruptor>(client_pp(*pg).send_reply, *this
 	      ).then_interruptible(
 		[this, reply=std::move(reply)]() mutable {
-		  logger().debug("{}: sending response", *this);
+                  LOG_PREFIX(ClientRequest::do_process);
+		  DEBUGI("{}: sending response", *this);
 		  return conn->send(std::move(reply));
 		});
 	    }, crimson::ct_error::eagain::handle([this, pg, &ihref]() mutable {
@@ -378,20 +399,21 @@ const SnapContext ClientRequest::get_snapc(
   Ref<PG>& pg,
   crimson::osd::ObjectContextRef obc) const
 {
+  LOG_PREFIX(ClientRequest::get_snapc);
   SnapContext snapc;
   if (op_info.may_write() || op_info.may_cache()) {
     // snap
     if (pg->get_pgpool().info.is_pool_snaps_mode()) {
       // use pool's snapc
       snapc = pg->get_pgpool().snapc;
-      logger().debug("{} using pool's snapc snaps={}",
+      DEBUGI("{} using pool's snapc snaps={}",
                      __func__, snapc.snaps);
 
     } else {
       // client specified snapc
       snapc.seq = m->get_snap_seq();
       snapc.snaps = m->get_snaps();
-      logger().debug("{} client specified snapc seq={} snaps={}",
+      DEBUGI("{} client specified snapc seq={} snaps={}",
                      __func__, snapc.seq, snapc.snaps);
     }
   }
diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc
index 790eb3f932d0..c2a371d274f6 100644
--- a/src/crimson/osd/osd_operations/internal_client_request.cc
+++ b/src/crimson/osd/osd_operations/internal_client_request.cc
@@ -20,6 +20,7 @@ namespace crimson {
   };
 }
 
+SET_SUBSYS(osd);
 
 namespace crimson::osd {
 
@@ -32,7 +33,8 @@ InternalClientRequest::InternalClientRequest(Ref<PG> pg)
 
 InternalClientRequest::~InternalClientRequest()
 {
-  logger().debug("{}: destroying", *this);
+  LOG_PREFIX(InternalClientRequest::~InternalClientRequest);
+  DEBUGI("{}: destroying", *this);
 }
 
 void InternalClientRequest::print(std::ostream &) const
@@ -53,7 +55,8 @@ seastar::future<> InternalClientRequest::start()
   track_event<StartEvent>();
   return crimson::common::handle_system_shutdown([this] {
     return seastar::repeat([this] {
-      logger().debug("{}: in repeat", *this);
+      LOG_PREFIX(InternalClientRequest::start);
+      DEBUGI("{}: in repeat", *this);
       return interruptor::with_interruption([this]() mutable {
         return enter_stage<interruptor>(
 	  client_pp().wait_for_active
@@ -71,10 +74,12 @@ seastar::future<> InternalClientRequest::start()
           return enter_stage<interruptor>(
             client_pp().get_obc);
         }).then_interruptible([this] () -> PG::load_obc_iertr::future<> {
-          logger().debug("{}: getting obc lock", *this);
+          LOG_PREFIX(InternalClientRequest::start);
+          DEBUGI("{}: getting obc lock", *this);
           return seastar::do_with(create_osd_ops(),
             [this](auto& osd_ops) mutable {
-            logger().debug("InternalClientRequest: got {} OSDOps to execute",
+            LOG_PREFIX(InternalClientRequest::start);
+            DEBUGI("InternalClientRequest: got {} OSDOps to execute",
                            std::size(osd_ops));
             [[maybe_unused]] const int ret = op_info.set_from_op(
               std::as_const(osd_ops), pg->get_pgid().pgid, *pg->get_osdmap());
diff --git a/src/crimson/osd/osd_operations/logmissing_request.cc b/src/crimson/osd/osd_operations/logmissing_request.cc
index 7c8e1d7e499a..797625da629a 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request.cc
@@ -16,6 +16,8 @@ namespace {
   }
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 LogMissingRequest::LogMissingRequest(crimson::net::ConnectionRef&& conn,
@@ -63,11 +65,13 @@ ClientRequest::PGPipeline &LogMissingRequest::client_pp(PG &pg)
 seastar::future<> LogMissingRequest::with_pg(
   ShardServices &shard_services, Ref<PG> pg)
 {
-  logger().debug("{}: LogMissingRequest::with_pg", *this);
+  LOG_PREFIX(LogMissingRequest::with_pg);
+  DEBUGI("{}: LogMissingRequest::with_pg", *this);
 
   IRef ref = this;
   return interruptor::with_interruption([this, pg] {
-    logger().debug("{}: pg present", *this);
+    LOG_PREFIX(LogMissingRequest::with_pg);
+    DEBUGI("{}: pg present", *this);
     return this->template enter_stage<interruptor>(client_pp(*pg).await_map
     ).then_interruptible([this, pg] {
       return this->template with_blocking_event<
diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc
index 9139e337f80a..f12699e5e6a2 100644
--- a/src/crimson/osd/osd_operations/peering_event.cc
+++ b/src/crimson/osd/osd_operations/peering_event.cc
@@ -19,6 +19,8 @@ namespace {
   }
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 template <class T>
@@ -63,16 +65,19 @@ template <class T>
 seastar::future<> PeeringEvent<T>::with_pg(
   ShardServices &shard_services, Ref<PG> pg)
 {
+  using interruptor = typename T::interruptor;
+  LOG_PREFIX(PeeringEvent<T>::with_pg);
   if (!pg) {
-    logger().warn("{}: pg absent, did not create", *this);
+    WARNI("{}: pg absent, did not create", *this);
     on_pg_absent(shard_services);
     that()->get_handle().exit();
     return complete_rctx_no_pg(shard_services);
   }
+  DEBUGI("start");
 
-  using interruptor = typename T::interruptor;
   return interruptor::with_interruption([this, pg, &shard_services] {
-    logger().debug("{}: pg present", *this);
+    LOG_PREFIX(PeeringEvent<T>::with_pg);
+    DEBUGI("{} {}: pg present", interruptor::get_interrupt_cond(), *this);
     return this->template enter_stage<interruptor>(peering_pp(*pg).await_map
     ).then_interruptible([this, pg] {
       return this->template with_blocking_event<
@@ -100,7 +105,8 @@ seastar::future<> PeeringEvent<T>::with_pg(
       return shard_services.send_pg_temp();
     });
   }, [this](std::exception_ptr ep) {
-    logger().debug("{}: interrupted with {}", *this, ep);
+    LOG_PREFIX(PeeringEvent<T>::with_pg);
+    DEBUGI("{}: interrupted with {}", *this, ep);
   }, pg).finally([this] {
     logger().debug("{}: exit", *this);
     that()->get_handle().exit();
@@ -110,14 +116,18 @@ seastar::future<> PeeringEvent<T>::with_pg(
 template <class T>
 void PeeringEvent<T>::on_pg_absent(ShardServices &)
 {
-  logger().debug("{}: pg absent, dropping", *this);
+  using interruptor = typename T::interruptor;
+  LOG_PREFIX(PeeringEvent<T>::on_pg_absent);
+  DEBUGI("{}: pg absent, dropping", *this);
 }
 
 template <class T>
 typename PeeringEvent<T>::template interruptible_future<>
 PeeringEvent<T>::complete_rctx(ShardServices &shard_services, Ref<PG> pg)
 {
-  logger().debug("{}: submitting ctx", *this);
+  using interruptor = typename T::interruptor;
+  LOG_PREFIX(PeeringEvent<T>::complete_rctx);
+  DEBUGI("{}: submitting ctx", *this);
   return shard_services.dispatch_context(
     pg->get_collection_ref(),
     std::move(ctx));
@@ -176,7 +186,8 @@ seastar::future<> RemotePeeringEvent::complete_rctx_no_pg(
 
 seastar::future<> LocalPeeringEvent::start()
 {
-  logger().debug("{}: start", *this);
+  LOG_PREFIX(LocalPeeringEvent::start);
+  DEBUGI("{}: start", *this);
 
   IRef ref = this;
   auto maybe_delay = seastar::now();
@@ -187,7 +198,8 @@ seastar::future<> LocalPeeringEvent::start()
   return maybe_delay.then([this] {
     return with_pg(pg->get_shard_services(), pg);
   }).finally([ref=std::move(ref)] {
-    logger().debug("{}: complete", *ref);
+    LOG_PREFIX(LocalPeeringEvent::start);
+    DEBUGI("{}: complete", *ref);
   });
 }
 
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.cc b/src/crimson/osd/osd_operations/recovery_subrequest.cc
index 2e939880cbea..1ea7ae12869e 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.cc
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.cc
@@ -1,3 +1,6 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
 #include <fmt/format.h>
 #include <fmt/ostream.h>
 
@@ -20,19 +23,22 @@ namespace crimson {
   };
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 seastar::future<> RecoverySubRequest::with_pg(
   ShardServices &shard_services, Ref<PG> pgref)
 {
-  logger().debug("{}: {}", "RecoverySubRequest::with_pg", *this);
-
   track_event<StartEvent>();
   IRef opref = this;
   return interruptor::with_interruption([this, pgref] {
+    LOG_PREFIX(RecoverySubRequest::with_pg);
+    DEBUGI("{}: {}", "RecoverySubRequest::with_pg", *this);
     return pgref->get_recovery_backend()->handle_recovery_op(m, conn
     ).then_interruptible([this] {
-      logger().debug("{}: complete", *this);
+      LOG_PREFIX(RecoverySubRequest::with_pg);
+      DEBUGI("{}: complete", *this);
       return handle.complete();
     });
   }, [](std::exception_ptr) {
diff --git a/src/crimson/osd/osd_operations/replicated_request.cc b/src/crimson/osd/osd_operations/replicated_request.cc
index 7b8592b1e02d..4e97db5be982 100644
--- a/src/crimson/osd/osd_operations/replicated_request.cc
+++ b/src/crimson/osd/osd_operations/replicated_request.cc
@@ -16,6 +16,8 @@ namespace {
   }
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
 RepRequest::RepRequest(crimson::net::ConnectionRef&& conn,
@@ -63,10 +65,12 @@ ClientRequest::PGPipeline &RepRequest::client_pp(PG &pg)
 seastar::future<> RepRequest::with_pg(
   ShardServices &shard_services, Ref<PG> pg)
 {
-  logger().debug("{}: RepRequest::with_pg", *this);
+  LOG_PREFIX(RepRequest::with_pg);
+  DEBUGI("{}: RepRequest::with_pg", *this);
   IRef ref = this;
   return interruptor::with_interruption([this, pg] {
-    logger().debug("{}: pg present", *this);
+    LOG_PREFIX(RepRequest::with_pg);
+    DEBUGI("{}: pg present", *this);
     return this->template enter_stage<interruptor>(client_pp(*pg).await_map
     ).then_interruptible([this, pg] {
       return this->template with_blocking_event<

From 9dd6bc6198c5093410a6368c5fff44a349fba1a4 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 18 Aug 2023 11:17:45 +0800
Subject: [PATCH 0490/2492] crimson/common: add new log type
 "ceph_subsys_interrupt"

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/common/subsys.h                       | 1 +
 src/crimson/common/interruptible_future.h | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/common/subsys.h b/src/common/subsys.h
index e798f987aa0d..4892d202cf56 100644
--- a/src/common/subsys.h
+++ b/src/common/subsys.h
@@ -83,6 +83,7 @@ SUBSYS(prioritycache, 1, 5)
 SUBSYS(test, 0, 5)
 SUBSYS(cephfs_mirror, 0, 5)
 SUBSYS(cephsqlite, 0, 5)
+SUBSYS(crimson_interrupt, 0, 5)
 SUBSYS(seastore, 0, 5)       // logs above seastore tm
 SUBSYS(seastore_onode, 0, 5)
 SUBSYS(seastore_odata, 0, 5)
diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 1975295ef94b..ef780907ec88 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -11,7 +11,8 @@
 #include "crimson/common/log.h"
 #include "crimson/common/errorator.h"
 #ifndef NDEBUG
-#define INTR_FUT_DEBUG(FMT_MSG, ...) crimson::get_logger(ceph_subsys_).trace(FMT_MSG, ##__VA_ARGS__)
+#define INTR_FUT_DEBUG(FMT_MSG, ...) crimson::get_logger(\
+  ceph_subsys_crimson_interrupt).trace(FMT_MSG, ##__VA_ARGS__)
 #else
 #define INTR_FUT_DEBUG(FMT_MSG, ...)
 #endif

From 35474408980f404ea54d1a786dea3032b8305ba0 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 18 Aug 2023 11:19:10 +0800
Subject: [PATCH 0491/2492] crimson/common/interruptible_future: make sure
 interrupt_cond_t::ref_count won't overflow

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/common/interruptible_future.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index ef780907ec88..cac6f65d1fd2 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -114,6 +114,7 @@ struct interrupt_cond_t {
       ref_count);
   }
   void reset() {
+    assert(ref_count >= 1);
     if (--ref_count == 0) {
       INTR_FUT_DEBUG(
 	"{}: clearing interrupt_cond: {},{}",

From dc9a05c7a308f62d4f0e8d3742aa86370b751624 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 18 Aug 2023 11:21:03 +0800
Subject: [PATCH 0492/2492] crimson/common/interruptible_future: deal with
 exceptions thrown from seastar::future::get() and seastar::thread::yield()

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/common/interruptible_future.h | 117 ++++++++++++++++------
 1 file changed, 87 insertions(+), 30 deletions(-)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index cac6f65d1fd2..6ed11088e1a7 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -307,10 +307,17 @@ Result non_futurized_call_with_interruption(
       return std::forward<Result>(err);
     }
   } catch (std::exception& e) {
+    INTR_FUT_DEBUG(
+      "non_futurized_call_with_interruption catched exception: {}, "
+      "interrupt_condition: {}, interrupt_cond: {},{}",
+      e,
+      (void*)interrupt_condition.get(),
+      (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+      typeid(InterruptCond).name());
     // Clear the global "interrupt_cond" to prevent it from interfering other
     // continuation chains.
     interrupt_cond<InterruptCond>.reset();
-    throw e;
+    std::throw_with_nested(std::runtime_error("failed to run interruptible continuation"));
   }
 }
 
@@ -421,13 +428,25 @@ class [[nodiscard]] interruptible_future_detail<InterruptCond, seastar::future<T
 	(void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
 	typeid(InterruptCond).name());
       interrupt_cond<InterruptCond>.reset();
-      auto&& value = core_type::get();
-      interrupt_cond<InterruptCond>.set(interruption_condition);
-      INTR_FUT_DEBUG(
-	"interruptible_future_detail::get() got, interrupt_cond: {},{}",
-	(void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
-	typeid(InterruptCond).name());
-      return std::move(value);
+      try {
+	auto&& value = core_type::get();
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "interruptible_future_detail::get() got, interrupt_cond: {},{}",
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+	return std::move(value);
+      } catch (std::exception &e) {
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "interruptible_future_detail::get() error {}, interrupt_cond: {},{}",
+	  e,
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+	std::throw_with_nested(
+	  std::runtime_error(
+	    "failed to run interruptible continuation"));
+      }
     }
   }
 
@@ -1113,11 +1132,13 @@ struct interruptor
 	    typename... Params>
   static inline auto with_interruption_cond(
     OpFunc&& opfunc, OnInterrupt&& efunc, InterruptCond &&cond, Params&&... params) {
+    auto ic = seastar::make_lw_shared<InterruptCond>(std::move(cond));
     INTR_FUT_DEBUG(
-      "with_interruption_cond: interrupt_cond: {}",
-      (void*)interrupt_cond<InterruptCond>.interrupt_cond.get());
+      "with_interruption_cond: interrupt_cond: {}, ic: {}",
+      (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+      (void*)ic.get());
     return internal::call_with_interruption_impl(
-      seastar::make_lw_shared<InterruptCond>(std::move(cond)),
+      std::move(ic),
       std::forward<OpFunc>(opfunc),
       std::forward<Params>(params)...
     ).template handle_interruption(std::move(efunc));
@@ -1431,13 +1452,25 @@ struct interruptor
         (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
         typeid(InterruptCond).name());
       interrupt_cond<InterruptCond>.reset();
-      auto&& value = fut.get();
-      interrupt_cond<InterruptCond>.set(interruption_condition);
-      INTR_FUT_DEBUG(
-        "green_get() got, interrupt_cond: {},{}",
-        (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
-        typeid(InterruptCond).name());
-      return std::move(value);
+      try {
+	auto&& value = fut.get();
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "green_get() got, interrupt_cond: {},{}",
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+	return std::move(value);
+      } catch (std::exception &e) {
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "green_get() error {}, interrupt_cond: {},{}",
+	  e,
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+	std::throw_with_nested(
+	  std::runtime_error(
+	    "failed to run interruptible continuation"));
+      }
     }
   }
 
@@ -1450,12 +1483,24 @@ struct interruptor
       (void*)interruption_condition.get(),
       typeid(InterruptCond).name());
     interrupt_cond<InterruptCond>.reset();
-    seastar::thread::yield();
-    interrupt_cond<InterruptCond>.set(interruption_condition);
-    INTR_FUT_DEBUG(
-      "interruptible_future_detail::yield() yield back, interrupt_cond: {},{}",
-      (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
-      typeid(InterruptCond).name());
+    try {
+      seastar::thread::yield();
+      interrupt_cond<InterruptCond>.set(interruption_condition);
+      INTR_FUT_DEBUG(
+	"interruptible_future_detail::yield() yield back, interrupt_cond: {},{}",
+	(void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	typeid(InterruptCond).name());
+    } catch (std::exception &e) {
+      interrupt_cond<InterruptCond>.set(interruption_condition);
+      INTR_FUT_DEBUG(
+	"interruptible_future_detail::yield() error {}, interrupt_cond: {},{}",
+	e,
+	(void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	typeid(InterruptCond).name());
+      std::throw_with_nested(
+	std::runtime_error(
+	  "failed to run interruptible continuation"));
+    }
   }
 
   static void maybe_yield() {
@@ -1468,12 +1513,24 @@ struct interruptor
 	(void*)interruption_condition.get(),
 	typeid(InterruptCond).name());
       interrupt_cond<InterruptCond>.reset();
-      seastar::thread::yield();
-      interrupt_cond<InterruptCond>.set(interruption_condition);
-      INTR_FUT_DEBUG(
-	"interruptible_future_detail::may_yield() yield back, interrupt_cond: {},{}",
-	(void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
-	typeid(InterruptCond).name());
+      try {
+	seastar::thread::yield();
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "interruptible_future_detail::may_yield() yield back, interrupt_cond: {},{}",
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+      } catch (std::exception &e) {
+	interrupt_cond<InterruptCond>.set(interruption_condition);
+	INTR_FUT_DEBUG(
+	  "interruptible_future_detail::may_yield() error {}, interrupt_cond: {},{}",
+	  e,
+	  (void*)interrupt_cond<InterruptCond>.interrupt_cond.get(),
+	  typeid(InterruptCond).name());
+	std::throw_with_nested(
+	  std::runtime_error(
+	    "failed to run interruptible continuation"));
+      }
     }
   }
 };

From cc73ba6d8c01cddc80e4ffd0158af8fde7d2f91d Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Sat, 19 Aug 2023 15:08:20 +0800
Subject: [PATCH 0493/2492] crimson/osd/pg: set PG::projected_last_update when
 submitting transactions

Otherwise, PG::projected_last_update's epoch is always outdated, which
would cause failure of asserts in the backfilling code

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/ops_executer.cc               |   6 +-
 .../osd/osd_operations/snaptrim_event.cc      | 164 ++++++++----------
 .../osd/osd_operations/snaptrim_event.h       |  37 ++--
 src/crimson/osd/pg.cc                         |   6 +-
 src/crimson/osd/pg.h                          |   4 +-
 5 files changed, 108 insertions(+), 109 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 040870203bd9..868a7a3fb13d 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -797,7 +797,7 @@ void OpsExecuter::fill_op_params_bump_pg_version()
 {
   osd_op_params->req_id = msg->get_reqid();
   osd_op_params->mtime = msg->get_mtime();
-  osd_op_params->at_version = pg->next_version();
+  osd_op_params->at_version = pg->get_next_version();
   osd_op_params->pg_trim_to = pg->get_pg_trim_to();
   osd_op_params->min_last_complete_ondisk = pg->get_min_last_complete_ondisk();
   osd_op_params->last_complete = pg->get_info().last_complete;
@@ -820,6 +820,7 @@ std::vector<pg_log_entry_t> OpsExecuter::prepare_transaction(
     osd_op_params->req_id,
     osd_op_params->mtime,
     op_info.allows_returnvec() && !ops.empty() ? ops.back().rval.code : 0);
+  osd_op_params->at_version.version++;
   if (op_info.allows_returnvec()) {
     // also the per-op values are recorded in the pg log
     log_entries.back().set_op_returns(ops);
@@ -957,6 +958,7 @@ std::unique_ptr<OpsExecuter::CloningContext> OpsExecuter::execute_clone(
     initial_obs.oi.mtime, // will be replaced in `apply_to()`
     0
   };
+  osd_op_params->at_version.version++;
   encode(cloned_snaps, cloning_ctx->log_entry.snaps);
 
   // TODO: update most recent clone_overlap and usage stats
@@ -1012,7 +1014,7 @@ std::pair<object_info_t, ObjectContextRef> OpsExecuter::prepare_clone(
   const hobject_t& coid)
 {
   object_info_t static_snap_oi(coid);
-  static_snap_oi.version = pg->next_version();
+  static_snap_oi.version = osd_op_params->at_version;
   static_snap_oi.prior_version = obc->obs.oi.version;
   static_snap_oi.copy_user_bits(obc->obs.oi);
   if (static_snap_oi.is_whiteout()) {
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index ffd43d736ad3..263cdab37a8a 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -211,8 +211,7 @@ SnapTrimObjSubEvent::remove_or_update_iertr::future<>
 SnapTrimObjSubEvent::remove_clone(
   ObjectContextRef obc,
   ObjectContextRef head_obc,
-  ceph::os::Transaction& txn,
-  std::vector<pg_log_entry_t>& log_entries
+  ceph::os::Transaction& txn
 ) {
   const auto p = std::find(
     head_obc->ssc->snapset.clones.begin(),
@@ -259,17 +258,14 @@ SnapTrimObjSubEvent::remove_clone(
   head_obc->ssc->snapset.clone_size.erase(last);
   head_obc->ssc->snapset.clone_snaps.erase(last);
 
-  log_entries.emplace_back(
-    pg_log_entry_t{
-      pg_log_entry_t::DELETE,
-      coid,
-      osd_op_p.at_version,
-      obc->obs.oi.version,
-      0,
-      osd_reqid_t(),
-      obc->obs.oi.mtime, // will be replaced in `apply_to()`
-      0}
-    );
+  add_log_entry(
+    pg_log_entry_t::DELETE,
+    coid,
+    obc->obs.oi.version,
+    0,
+    osd_reqid_t(),
+    obc->obs.oi.mtime, // will be replaced in `apply_to()`
+    0);
   txn.remove(
     pg->get_collection_ref()->get_cid(),
     ghobject_t{coid, ghobject_t::NO_GEN, shard_id_t::NO_SHARD});
@@ -280,8 +276,7 @@ SnapTrimObjSubEvent::remove_clone(
 void SnapTrimObjSubEvent::remove_head_whiteout(
   ObjectContextRef obc,
   ObjectContextRef head_obc,
-  ceph::os::Transaction& txn,
-  std::vector<pg_log_entry_t>& log_entries
+  ceph::os::Transaction& txn
 ) {
   // NOTE: this arguably constitutes minor interference with the
   // tiering agent if this is a cache tier since a snap trim event
@@ -290,17 +285,14 @@ void SnapTrimObjSubEvent::remove_head_whiteout(
   const auto head_oid = coid.get_head();
   logger().info("{}: {} removing {}",
                 *this, coid, head_oid);
-  log_entries.emplace_back(
-    pg_log_entry_t{
-      pg_log_entry_t::DELETE,
-      head_oid,
-      osd_op_p.at_version,
-      head_obc->obs.oi.version,
-      0,
-      osd_reqid_t(),
-      obc->obs.oi.mtime, // will be replaced in `apply_to()`
-      0}
-    );
+  add_log_entry(
+    pg_log_entry_t::DELETE,
+    head_oid,
+    head_obc->obs.oi.version,
+    0,
+    osd_reqid_t(),
+    obc->obs.oi.mtime, // will be replaced in `apply_to()`
+    0);
   logger().info("{}: remove snap head", *this);
   object_info_t& oi = head_obc->obs.oi;
   delta_stats.num_objects--;
@@ -326,8 +318,7 @@ SnapTrimObjSubEvent::adjust_snaps(
   ObjectContextRef obc,
   ObjectContextRef head_obc,
   const std::set<snapid_t>& new_snaps,
-  ceph::os::Transaction& txn,
-  std::vector<pg_log_entry_t>& log_entries
+  ceph::os::Transaction& txn
 ) {
   head_obc->ssc->snapset.clone_snaps[coid.snap] =
     std::vector<snapid_t>(new_snaps.rbegin(), new_snaps.rend());
@@ -345,17 +336,14 @@ SnapTrimObjSubEvent::adjust_snaps(
     ghobject_t{coid, ghobject_t::NO_GEN, shard_id_t::NO_SHARD},
     OI_ATTR,
     bl);
-  log_entries.emplace_back(
-    pg_log_entry_t{
-      pg_log_entry_t::MODIFY,
-      coid,
-      obc->obs.oi.version,
-      obc->obs.oi.prior_version,
-      0,
-      osd_reqid_t(),
-      obc->obs.oi.mtime,
-      0}
-    );
+  add_log_entry(
+    pg_log_entry_t::MODIFY,
+    coid,
+    obc->obs.oi.prior_version,
+    0,
+    osd_reqid_t(),
+    obc->obs.oi.mtime,
+    0);
   return OpsExecuter::snap_map_modify(
     coid, new_snaps, pg->snap_mapper, pg->osdriver, txn);
 }
@@ -363,23 +351,19 @@ SnapTrimObjSubEvent::adjust_snaps(
 void SnapTrimObjSubEvent::update_head(
   ObjectContextRef obc,
   ObjectContextRef head_obc,
-  ceph::os::Transaction& txn,
-  std::vector<pg_log_entry_t>& log_entries
+  ceph::os::Transaction& txn
 ) {
   const auto head_oid = coid.get_head();
   logger().info("{}: writing updated snapset on {}, snapset is {}",
                 *this, head_oid, head_obc->ssc->snapset);
-  log_entries.emplace_back(
-    pg_log_entry_t{
-      pg_log_entry_t::MODIFY,
-      head_oid,
-      osd_op_p.at_version,
-      head_obc->obs.oi.version,
-      0,
-      osd_reqid_t(),
-      obc->obs.oi.mtime,
-      0}
-    );
+  add_log_entry(
+    pg_log_entry_t::MODIFY,
+    head_oid,
+    head_obc->obs.oi.version,
+    0,
+    osd_reqid_t(),
+    obc->obs.oi.mtime,
+    0);
 
   head_obc->obs.oi.prior_version = head_obc->obs.oi.version;
   head_obc->obs.oi.version = osd_op_p.at_version;
@@ -399,8 +383,7 @@ void SnapTrimObjSubEvent::update_head(
     attrs);
 }
 
-SnapTrimObjSubEvent::remove_or_update_iertr::future<
-  SnapTrimObjSubEvent::remove_or_update_ret_t>
+SnapTrimObjSubEvent::remove_or_update_iertr::future<ceph::os::Transaction>
 SnapTrimObjSubEvent::remove_or_update(
   ObjectContextRef obc,
   ObjectContextRef head_obc)
@@ -433,45 +416,41 @@ SnapTrimObjSubEvent::remove_or_update(
   }
 
   return seastar::do_with(ceph::os::Transaction{}, [=, this](auto &txn) {
-  std::vector<pg_log_entry_t> log_entries{};
-
-  int64_t num_objects_before_trim = delta_stats.num_objects;
-  osd_op_p.at_version = pg->next_version();
-  auto ret = remove_or_update_iertr::now();
-  if (new_snaps.empty()) {
-    // remove clone from snapset
-    logger().info("{}: {} snaps {} -> {} ... deleting",
-                  *this, coid, old_snaps, new_snaps);
-    ret = remove_clone(obc, head_obc, txn, log_entries);
-  } else {
-    // save adjusted snaps for this object
-    logger().info("{}: {} snaps {} -> {}",
-                  *this, coid, old_snaps, new_snaps);
-    ret = adjust_snaps(obc, head_obc, new_snaps, txn, log_entries);
-  }
-  return std::move(ret).si_then(
-    [&txn, obc, num_objects_before_trim, log_entries=std::move(log_entries), head_obc=std::move(head_obc), this]() mutable {
-    osd_op_p.at_version = pg->next_version();
-
-    // save head snapset
-    logger().debug("{}: {} new snapset {} on {}",
-                   *this, coid, head_obc->ssc->snapset, head_obc->obs.oi);
-    if (head_obc->ssc->snapset.clones.empty() && head_obc->obs.oi.is_whiteout()) {
-      remove_head_whiteout(obc, head_obc, txn, log_entries);
+    int64_t num_objects_before_trim = delta_stats.num_objects;
+    osd_op_p.at_version = pg->get_next_version();
+    auto ret = remove_or_update_iertr::now();
+    if (new_snaps.empty()) {
+      // remove clone from snapset
+      logger().info("{}: {} snaps {} -> {} ... deleting",
+		    *this, coid, old_snaps, new_snaps);
+      ret = remove_clone(obc, head_obc, txn);
     } else {
-      update_head(obc, head_obc, txn, log_entries);
-    }
-    // Stats reporting - Set number of objects trimmed
-    if (num_objects_before_trim > delta_stats.num_objects) {
-      //int64_t num_objects_trimmed =
-      //  num_objects_before_trim - delta_stats.num_objects;
-      //add_objects_trimmed_count(num_objects_trimmed);
+      // save adjusted snaps for this object
+      logger().info("{}: {} snaps {} -> {}",
+		    *this, coid, old_snaps, new_snaps);
+      ret = adjust_snaps(obc, head_obc, new_snaps, txn);
     }
-  }).si_then(
-    [&txn, log_entries=std::move(log_entries)] () mutable {
-    return remove_or_update_iertr::make_ready_future<remove_or_update_ret_t>(
-      std::make_pair(std::move(txn), std::move(log_entries)));
-  });
+    return std::move(ret).si_then(
+      [&txn, obc, num_objects_before_trim,
+      head_obc=std::move(head_obc), this]() mutable {
+      // save head snapset
+      logger().debug("{}: {} new snapset {} on {}",
+		     *this, coid, head_obc->ssc->snapset, head_obc->obs.oi);
+      if (head_obc->ssc->snapset.clones.empty() && head_obc->obs.oi.is_whiteout()) {
+	remove_head_whiteout(obc, head_obc, txn);
+      } else {
+	update_head(obc, head_obc, txn);
+      }
+      // Stats reporting - Set number of objects trimmed
+      if (num_objects_before_trim > delta_stats.num_objects) {
+	//int64_t num_objects_trimmed =
+	//  num_objects_before_trim - delta_stats.num_objects;
+	//add_objects_trimmed_count(num_objects_trimmed);
+      }
+    }).si_then(
+      [&txn] () mutable {
+      return std::move(txn);
+    });
   });
 }
 
@@ -509,8 +488,7 @@ SnapTrimObjSubEvent::start()
         logger().debug("{}: processing clone_obc={}", *this, clone_obc->get_oid());
         return remove_or_update(
           clone_obc, head_obc
-        ).safe_then_unpack_interruptible([clone_obc, this]
-                                         (auto&& txn, auto&& log_entries) mutable {
+        ).safe_then_interruptible([clone_obc, this](auto&& txn) mutable {
           auto [submitted, all_completed] = pg->submit_transaction(
             std::move(clone_obc),
             std::move(txn),
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index afb24952a045..288bd207e985 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -147,28 +147,22 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
   remove_or_update_iertr::future<> remove_clone(
     ObjectContextRef obc,
     ObjectContextRef head_obc,
-    ceph::os::Transaction& txn,
-    std::vector<pg_log_entry_t>& log_entries);
+    ceph::os::Transaction& txn);
   void remove_head_whiteout(
     ObjectContextRef obc,
     ObjectContextRef head_obc,
-    ceph::os::Transaction& txn,
-    std::vector<pg_log_entry_t>& log_entries);
+    ceph::os::Transaction& txn);
   interruptible_future<> adjust_snaps(
     ObjectContextRef obc,
     ObjectContextRef head_obc,
     const std::set<snapid_t>& new_snaps,
-    ceph::os::Transaction& txn,
-    std::vector<pg_log_entry_t>& log_entries);
+    ceph::os::Transaction& txn);
   void update_head(
     ObjectContextRef obc,
     ObjectContextRef head_obc,
-    ceph::os::Transaction& txn,
-    std::vector<pg_log_entry_t>& log_entries);
+    ceph::os::Transaction& txn);
 
-  using remove_or_update_ret_t =
-    std::pair<ceph::os::Transaction, std::vector<pg_log_entry_t>>;
-  remove_or_update_iertr::future<remove_or_update_ret_t>
+  remove_or_update_iertr::future<ceph::os::Transaction>
   remove_or_update(ObjectContextRef obc, ObjectContextRef head_obc);
 
   // we don't need to synchronize with other instances started by
@@ -177,11 +171,32 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
     static constexpr auto type_name = "SnapTrimObjSubEvent::wait_repop";
   } wait_repop;
 
+  void add_log_entry(
+    int _op,
+    const hobject_t& _soid,
+    const eversion_t& pv,
+    version_t uv,
+    const osd_reqid_t& rid,
+    const utime_t& mt,
+    int return_code) {
+    log_entries.emplace_back(
+      _op,
+      _soid,
+      osd_op_p.at_version,
+      pv,
+      uv,
+      rid,
+      mt,
+      return_code);
+    osd_op_p.at_version.version++;
+  }
+
   Ref<PG> pg;
   PipelineHandle handle;
   osd_op_params_t osd_op_p;
   const hobject_t coid;
   const snapid_t snap_to_trim;
+  std::vector<pg_log_entry_t> log_entries;
 
 public:
   PipelineHandle& get_handle() { return handle; }
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 42052a4c84d9..9aa92359151f 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -764,6 +764,10 @@ PG::submit_transaction(
   peering_state.append_log_with_trim_to_updated(std::move(log_entries), osd_op_p.at_version,
 						txn, true, false);
 
+  ceph_assert(!log_entries.empty());
+  ceph_assert(log_entries.rbegin()->version >= projected_last_update);
+  projected_last_update = log_entries.rbegin()->version;
+
   auto [submitted, all_completed] = backend->mutate_object(
       peering_state.get_acting_recovery_backfill(),
       std::move(obc),
@@ -933,7 +937,7 @@ seastar::future<> PG::submit_error_log(
   mempool::osd_pglog::list<pg_log_entry_t> log_entries;
   log_entries.push_back(pg_log_entry_t(pg_log_entry_t::ERROR,
                                        obc->obs.oi.soid,
-                                       next_version(),
+                                       get_next_version(),
                                        eversion_t(), 0,
                                        reqid, utime_t(),
                                        -e.value()));
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 6a0231e452f6..02e7ef3804ef 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -611,9 +611,9 @@ class PG : public boost::intrusive_ref_counter<
 
 public:
   cached_map_t get_osdmap() { return peering_state.get_osdmap(); }
-  eversion_t next_version() {
+  eversion_t get_next_version() {
     return eversion_t(get_osdmap_epoch(),
-		      ++projected_last_update.version);
+		      projected_last_update.version + 1);
   }
   ShardServices& get_shard_services() final {
     return shard_services;

From 96c11d26c9e61c76840dda33eabc489767125648 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 29 Aug 2023 14:05:02 +0800
Subject: [PATCH 0494/2492] crimson/osd/ops_executor: queue pg log entry for
 CLONE at the front

This is consistent with the order of ops in ceph::os::Transaction

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/ops_executer.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 868a7a3fb13d..09b1a492576f 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -970,7 +970,7 @@ void OpsExecuter::CloningContext::apply_to(
   ObjectContext& processed_obc) &&
 {
   log_entry.mtime = processed_obc.obs.oi.mtime;
-  log_entries.emplace_back(std::move(log_entry));
+  log_entries.insert(log_entries.begin(), std::move(log_entry));
   processed_obc.ssc->snapset = std::move(new_snapset);
 }
 
@@ -985,7 +985,7 @@ OpsExecuter::flush_clone_metadata(
   auto maybe_snap_mapped = interruptor::now();
   if (cloning_ctx) {
     std::move(*cloning_ctx).apply_to(log_entries, *obc);
-    const auto& coid = log_entries.back().soid;
+    const auto& coid = log_entries.front().soid;
     const auto& cloned_snaps = obc->ssc->snapset.clone_snaps[coid.snap];
     maybe_snap_mapped = snap_map_clone(
       coid,

From e9654b77eda12a3862eefb93c65dc5f15715833d Mon Sep 17 00:00:00 2001
From: Piotr Parczewski <piotr@stackhpc.com>
Date: Tue, 7 Nov 2023 12:48:01 +0100
Subject: [PATCH 0495/2492] doc/cephadm: fix typo in set ssh key command

Signed-off-by: Piotr Parczewski <piotr@stackhpc.com>
---
 doc/cephadm/troubleshooting.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index df66b3ae0385..d891ebaf2e03 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -210,7 +210,7 @@ troubleshoot the SSH connection between ``cephadm`` and the monitor:
 
  or::
 
-     [root@mon1 ~]# cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssk-key -i -
+     [root@mon1 ~]# cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssh-key -i -
 
 2. Ensure that the SSH config is correct::
 

From 38f5b55a95c35db36a7baeb653fbefd1efdd36dc Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 16:36:23 -0400
Subject: [PATCH 0496/2492] cephadm: move where _get_container_mounts_for_type
 is called

The _get_container_mounts_for_type method is only useful for ceph based
daemon types. There's no need to call this method in a universal
context.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 478de27f0b5e..e9e85a96b19d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2566,11 +2566,12 @@ def get_container_mounts(
     """
     # unpack fsid and daemon_type from ident because they're used very frequently
     fsid, daemon_type = ident.fsid, ident.daemon_type
-    mounts = _get_container_mounts_for_type(ctx, fsid, daemon_type)
+    mounts: Dict[str, str] = {}
 
     assert ident.fsid
     assert ident.daemon_id
     if daemon_type in ceph_daemons():
+        mounts = _get_container_mounts_for_type(ctx, fsid, daemon_type)
         data_dir = ident.data_dir(ctx.data_dir)
         if daemon_type == 'rgw':
             cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (ident.daemon_id)

From c1c666d8f53e3e044fd25d8d7ceb246827e28c7d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 17:24:44 -0400
Subject: [PATCH 0497/2492] cephadm: remove unused get_container_mounts
 CephExporter method

The get_container_mounts method on CephExporter is never called.
Remove it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index e9e85a96b19d..68c29d51e713 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1352,12 +1352,6 @@ def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephExporter':
     def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
-    @staticmethod
-    def get_container_mounts() -> Dict[str, str]:
-        mounts = dict()
-        mounts['/var/run/ceph'] = '/var/run/ceph:z'
-        return mounts
-
     def get_daemon_args(self) -> List[str]:
         args = [
             f'--sock-dir={self.sock_dir}',

From 01a21ad871300e7b25ba5e2d51382bdbedfa0aad Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 17:24:57 -0400
Subject: [PATCH 0498/2492] cephadm: move code setting mounts for ceph daemon
 types

Move the code in an if-block for ceph daemon types within get_container_mounts
to a new method of the Ceph type. The logic is moved to a staticmethod
so that it can be reused within the CephExporter type. This can be fixed
up in the future, but currently the staticmethod handles both Ceph and
CephExporter. Both types gain a get_container_mounts for common usage.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 64 +++++++++++++++++++++++++++++++++---------
 1 file changed, 51 insertions(+), 13 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 68c29d51e713..8c5d835b705f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -221,6 +221,7 @@ def for_daemon_type(cls, daemon_type: str) -> bool:
     def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
         self.ctx = ctx
         self._identity = ident
+        self.user_supplied_config = False
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Ceph':
@@ -301,6 +302,52 @@ def get_daemon_args(self) -> List[str]:
                 r += ['--default-mon-cluster-log-to-stderr=true']
         return r
 
+    @staticmethod
+    def get_ceph_mounts(
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        no_config: bool = False,
+    ) -> Dict[str, str]:
+        # Warning: This is a hack done for more expedient refactoring
+        mounts = _get_container_mounts_for_type(
+            ctx, ident.fsid, ident.daemon_type
+        )
+        data_dir = ident.data_dir(ctx.data_dir)
+        if ident.daemon_type == 'rgw':
+            cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (
+                ident.daemon_id
+            )
+        else:
+            cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (
+                ident.daemon_type,
+                ident.daemon_id,
+            )
+        if ident.daemon_type != 'crash':
+            mounts[data_dir] = cdata_dir + ':z'
+        if not no_config:
+            mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z'
+        if ident.daemon_type in [
+            'rbd-mirror',
+            'cephfs-mirror',
+            'crash',
+            'ceph-exporter',
+        ]:
+            # these do not search for their keyrings in a data directory
+            mounts[
+                data_dir + '/keyring'
+            ] = '/etc/ceph/ceph.client.%s.%s.keyring' % (
+                ident.daemon_type,
+                ident.daemon_id,
+            )
+        return mounts
+
+    def get_container_mounts(self) -> Dict[str, str]:
+        return self.get_ceph_mounts(
+            self.ctx,
+            self.identity,
+            no_config=self.ctx.config and self.user_supplied_config,
+        )
+
 ##################################
 
 
@@ -1378,6 +1425,9 @@ def config_and_keyring(
     ) -> Tuple[Optional[str], Optional[str]]:
         return get_config_and_keyring(ctx)
 
+    def get_container_mounts(self) -> Dict[str, str]:
+        return Ceph.get_ceph_mounts(self.ctx, self.identity)
+
 
 ##################################
 
@@ -2565,19 +2615,7 @@ def get_container_mounts(
     assert ident.fsid
     assert ident.daemon_id
     if daemon_type in ceph_daemons():
-        mounts = _get_container_mounts_for_type(ctx, fsid, daemon_type)
-        data_dir = ident.data_dir(ctx.data_dir)
-        if daemon_type == 'rgw':
-            cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (ident.daemon_id)
-        else:
-            cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (daemon_type, ident.daemon_id)
-        if daemon_type != 'crash':
-            mounts[data_dir] = cdata_dir + ':z'
-        if not no_config:
-            mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z'
-        if daemon_type in ['rbd-mirror', 'cephfs-mirror', 'crash', 'ceph-exporter']:
-            # these do not search for their keyrings in a data directory
-            mounts[data_dir + '/keyring'] = '/etc/ceph/ceph.client.%s.%s.keyring' % (daemon_type, ident.daemon_id)
+        mounts = Ceph.get_ceph_mounts(ctx, ident, no_config=no_config)
 
     if daemon_type in Monitoring.components:
         data_dir = ident.data_dir(ctx.data_dir)

From a3ca7216c2e3ddf383a768acb935a913bd6940f7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 19 Oct 2023 17:30:27 -0400
Subject: [PATCH 0499/2492] cephadm: move code setting up mounts for monitoring
 daemon types

Move the code used to set up mounts for monitoring daemon types to
a method on the Monitoring class.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 68 ++++++++++++++++++++++++++++--------------
 1 file changed, 45 insertions(+), 23 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8c5d835b705f..5cfc4938ffc9 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -810,6 +810,49 @@ def get_daemon_args(self) -> List[str]:
                   '--path.rootfs=/rootfs']
         return r
 
+    def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+        ctx = self.ctx
+        daemon_type = self.identity.daemon_type
+        mounts: Dict[str, str] = {}
+        log_dir = get_log_dir(self.identity.fsid, ctx.log_dir)
+        if daemon_type == 'prometheus':
+            mounts[
+                os.path.join(data_dir, 'etc/prometheus')
+            ] = '/etc/prometheus:Z'
+            mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z'
+        elif daemon_type == 'loki':
+            mounts[os.path.join(data_dir, 'etc/loki')] = '/etc/loki:Z'
+            mounts[os.path.join(data_dir, 'data')] = '/loki:Z'
+        elif daemon_type == 'promtail':
+            mounts[os.path.join(data_dir, 'etc/promtail')] = '/etc/promtail:Z'
+            mounts[log_dir] = '/var/log/ceph:z'
+            mounts[os.path.join(data_dir, 'data')] = '/promtail:Z'
+        elif daemon_type == 'node-exporter':
+            mounts[
+                os.path.join(data_dir, 'etc/node-exporter')
+            ] = '/etc/node-exporter:Z'
+            mounts['/proc'] = '/host/proc:ro'
+            mounts['/sys'] = '/host/sys:ro'
+            mounts['/'] = '/rootfs:ro'
+        elif daemon_type == 'grafana':
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/grafana.ini')
+            ] = '/etc/grafana/grafana.ini:Z'
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/provisioning/datasources')
+            ] = '/etc/grafana/provisioning/datasources:Z'
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/certs')
+            ] = '/etc/grafana/certs:Z'
+            mounts[
+                os.path.join(data_dir, 'data/grafana.db')
+            ] = '/var/lib/grafana/grafana.db:Z'
+        elif daemon_type == 'alertmanager':
+            mounts[
+                os.path.join(data_dir, 'etc/alertmanager')
+            ] = '/etc/alertmanager:Z'
+        return mounts
+
 ##################################
 
 
@@ -2619,29 +2662,8 @@ def get_container_mounts(
 
     if daemon_type in Monitoring.components:
         data_dir = ident.data_dir(ctx.data_dir)
-        log_dir = get_log_dir(fsid, ctx.log_dir)
-        if daemon_type == 'prometheus':
-            mounts[os.path.join(data_dir, 'etc/prometheus')] = '/etc/prometheus:Z'
-            mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z'
-        elif daemon_type == 'loki':
-            mounts[os.path.join(data_dir, 'etc/loki')] = '/etc/loki:Z'
-            mounts[os.path.join(data_dir, 'data')] = '/loki:Z'
-        elif daemon_type == 'promtail':
-            mounts[os.path.join(data_dir, 'etc/promtail')] = '/etc/promtail:Z'
-            mounts[log_dir] = '/var/log/ceph:z'
-            mounts[os.path.join(data_dir, 'data')] = '/promtail:Z'
-        elif daemon_type == 'node-exporter':
-            mounts[os.path.join(data_dir, 'etc/node-exporter')] = '/etc/node-exporter:Z'
-            mounts['/proc'] = '/host/proc:ro'
-            mounts['/sys'] = '/host/sys:ro'
-            mounts['/'] = '/rootfs:ro'
-        elif daemon_type == 'grafana':
-            mounts[os.path.join(data_dir, 'etc/grafana/grafana.ini')] = '/etc/grafana/grafana.ini:Z'
-            mounts[os.path.join(data_dir, 'etc/grafana/provisioning/datasources')] = '/etc/grafana/provisioning/datasources:Z'
-            mounts[os.path.join(data_dir, 'etc/grafana/certs')] = '/etc/grafana/certs:Z'
-            mounts[os.path.join(data_dir, 'data/grafana.db')] = '/var/lib/grafana/grafana.db:Z'
-        elif daemon_type == 'alertmanager':
-            mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/etc/alertmanager:Z'
+        monitoring = Monitoring.create(ctx, ident)
+        mounts.update(monitoring.get_container_mounts(data_dir))
 
     if daemon_type == NFSGanesha.daemon_type:
         data_dir = ident.data_dir(ctx.data_dir)

From bdae273235104c8d7e1d46add3b64d78511f4d0c Mon Sep 17 00:00:00 2001
From: shimin <shimin@kuaishou.com>
Date: Fri, 13 Oct 2023 15:28:58 +0800
Subject: [PATCH 0500/2492] mds/purgequeue: add l_pq_executed_ops counter

We need to know cluster deleting pressure better. And only pq_executed
 is not enough, because file size is different.

Fixes: https://tracker.ceph.com/issues/63468
Signed-off-by: shimin <shimin@kuaishou.com>
---
 src/mds/PurgeQueue.cc | 6 +++++-
 src/mds/PurgeQueue.h  | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc
index e8f416bd97ac..639c7e85277a 100644
--- a/src/mds/PurgeQueue.cc
+++ b/src/mds/PurgeQueue.cc
@@ -138,6 +138,8 @@ void PurgeQueue::create_logger()
 {
   PerfCountersBuilder pcb(g_ceph_context, "purge_queue", l_pq_first, l_pq_last);
 
+  pcb.add_u64_counter(l_pq_executed_ops, "pq_executed_ops", "Purge queue ops executed",
+                      "puro", PerfCountersBuilder::PRIO_INTERESTING);
   pcb.add_u64_counter(l_pq_executed, "pq_executed", "Purge queue tasks executed",
                       "purg", PerfCountersBuilder::PRIO_INTERESTING);
 
@@ -710,7 +712,8 @@ void PurgeQueue::_execute_item_complete(
     pending_expire.insert(expire_to);
   }
 
-  ops_in_flight -= _calculate_ops(iter->second);
+  auto executed_ops = _calculate_ops(iter->second);
+  ops_in_flight -= executed_ops;
   logger->set(l_pq_executing_ops, ops_in_flight);
   ops_high_water = std::max(ops_high_water, ops_in_flight);
   logger->set(l_pq_executing_ops_high_water, ops_high_water);
@@ -735,6 +738,7 @@ void PurgeQueue::_execute_item_complete(
     << "/" << expire_pos << ")" << dendl;
 
   logger->set(l_pq_item_in_journal, item_num);
+  logger->inc(l_pq_executed_ops, executed_ops);
   logger->inc(l_pq_executed);
 }
 
diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h
index 6e953d3e6333..7bc101e31c4f 100644
--- a/src/mds/PurgeQueue.h
+++ b/src/mds/PurgeQueue.h
@@ -89,6 +89,7 @@ enum {
   l_pq_executing_ops_high_water,
   l_pq_executing,
   l_pq_executing_high_water,
+  l_pq_executed_ops,
   l_pq_executed,
   l_pq_item_in_journal,
   l_pq_last

From 157f21022e5a065e1457e5ac534d0dd7c66aaab1 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Tue, 7 Nov 2023 18:57:24 +0530
Subject: [PATCH 0501/2492] mgr/dashboard: fix rgw multi-site import form
 helper

Before : To obtain the token, generate it from your primary Ceph cluster. This token includes encoded information about the secondary cluster's endpoint, access key, and secret key.
Fix: To obtain the token, generate it from your primary Ceph cluster. This token includes encoded information about the primary cluster's endpoint, access key, and secret key.

Fixes: https://tracker.ceph.com/issues/63469

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../rgw-multisite-import/rgw-multisite-import.component.html    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
index ee9816dc5cbc..70c07e8acadc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
@@ -11,7 +11,7 @@
                       spacingClass="mb-3">
         <ul>
           <li>This feature allows you to configure a connection between your primary and secondary Ceph clusters for data replication. By importing a token, you establish a link between the clusters, enabling data synchronization.</li>
-          <li>To obtain the token, generate it from your primary Ceph cluster. This token includes encoded information about the secondary cluster's endpoint, access key, and secret key.</li>
+          <li>To obtain the token, generate it from your primary Ceph cluster. This token includes encoded information about the primary cluster's endpoint, access key, and secret key.</li>
           <li>The secondary zone represents the destination cluster where your data will be replicated.</li>
         </ul>
       </cd-alert-panel>

From e2eb66a36175dbd7ee0cdc706b1965c978807416 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 11:48:38 -0400
Subject: [PATCH 0502/2492] rgw/sal: move User::create_bucket() to
 Bucket::create()

the CreateBucket operation always has to deal with existing buckets, so
we have to load the Bucket handle first anyway

also moves the related placement and forward_request_to_master() logic
out of sal entirely

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc             |  30 +-
 src/rgw/driver/d4n/rgw_sal_d4n.h              |  20 +-
 src/rgw/driver/dbstore/common/dbstore.cc      |  60 ++-
 src/rgw/driver/dbstore/common/dbstore.h       |  18 +-
 src/rgw/driver/dbstore/tests/dbstore_tests.cc |  42 +-
 src/rgw/driver/posix/rgw_sal_posix.cc         | 135 ++----
 src/rgw/driver/posix/rgw_sal_posix.h          |  19 +-
 src/rgw/driver/rados/rgw_rados.cc             | 101 ++---
 src/rgw/driver/rados/rgw_rados.h              |  29 +-
 src/rgw/driver/rados/rgw_sal_rados.cc         | 138 ++----
 src/rgw/driver/rados/rgw_sal_rados.h          |  18 +-
 src/rgw/driver/rados/rgw_zone.cc              |  21 +
 src/rgw/driver/rados/rgw_zone.h               |   6 +
 src/rgw/rgw_op.cc                             | 400 +++++++++++-------
 src/rgw/rgw_op.h                              |  24 +-
 src/rgw/rgw_quota_types.h                     |   2 -
 src/rgw/rgw_rest_s3.cc                        |  11 +-
 src/rgw/rgw_rest_swift.cc                     |   9 +-
 src/rgw/rgw_sal.h                             |  39 +-
 src/rgw/rgw_sal_dbstore.cc                    | 100 +----
 src/rgw/rgw_sal_dbstore.h                     |  19 +-
 src/rgw/rgw_sal_filter.cc                     |  36 +-
 src/rgw/rgw_sal_filter.h                      |  19 +-
 src/test/rgw/test_d4n_filter.cc               |  46 +-
 src/test/rgw/test_rgw_lua.cc                  |   4 -
 25 files changed, 540 insertions(+), 806 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index 93cd8f2d4159..efeb281c3cea 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -59,33 +59,11 @@ std::unique_ptr<Object> D4NFilterBucket::get_object(const rgw_obj_key& k)
   return std::make_unique<D4NFilterObject>(std::move(o), this, filter);
 }
 
-int D4NFilterUser::create_bucket(const DoutPrefixProvider* dpp,
-                              const rgw_bucket& b,
-                              const std::string& zonegroup_id,
-                              const rgw_placement_rule& placement_rule,
-                              const std::string& swift_ver_location,
-                              const RGWQuotaInfo * pquota_info,
-                              const RGWAccessControlPolicy& policy,
-                              Attrs& attrs,
-                              RGWBucketInfo& info,
-                              obj_version& ep_objv,
-                              bool exclusive,
-                              bool obj_lock_enabled,
-                              bool* existed,
-                              req_info& req_info,
-                              std::unique_ptr<Bucket>* bucket_out,
-                              optional_yield y)
+int D4NFilterBucket::create(const DoutPrefixProvider* dpp,
+                            const CreateParams& params,
+                            optional_yield y)
 {
-  std::unique_ptr<Bucket> nb;
-  int ret;
-
-  ret = next->create_bucket(dpp, b, zonegroup_id, placement_rule, swift_ver_location, pquota_info, policy, attrs, info, ep_objv, exclusive, obj_lock_enabled, existed, req_info, &nb, y);
-  if (ret < 0)
-    return ret;
-
-  Bucket* fb = new D4NFilterBucket(std::move(nb), this, filter);
-  bucket_out->reset(fb);
-  return 0;
+  return next->create(dpp, params, y);
 }
 
 int D4NFilterObject::copy_object(User* user,
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index 459e885fc240..a54371f6cc64 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -71,23 +71,6 @@ class D4NFilterUser : public FilterUser {
       FilterUser(std::move(_next)),
       filter(_filter) {}
     virtual ~D4NFilterUser() = default;
-
-    virtual int create_bucket(const DoutPrefixProvider* dpp,
-                            const rgw_bucket& b,
-                            const std::string& zonegroup_id,
-                            const rgw_placement_rule& placement_rule,
-                            const std::string& swift_ver_location,
-                            const RGWQuotaInfo* pquota_info,
-                            const RGWAccessControlPolicy& policy,
-                            Attrs& attrs,
-                            RGWBucketInfo& info,
-                            obj_version& ep_objv,
-                            bool exclusive,
-                            bool obj_lock_enabled,
-                            bool* existed,
-                            req_info& req_info,
-                            std::unique_ptr<Bucket>* bucket,
-                            optional_yield y) override;
 };
 
 class D4NFilterBucket : public FilterBucket {
@@ -101,6 +84,9 @@ class D4NFilterBucket : public FilterBucket {
     virtual ~D4NFilterBucket() = default;
    
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& key) override;
+    virtual int create(const DoutPrefixProvider* dpp,
+                       const CreateParams& params,
+                       optional_yield y) override;
 };
 
 class D4NFilterObject : public FilterObject {
diff --git a/src/rgw/driver/dbstore/common/dbstore.cc b/src/rgw/driver/dbstore/common/dbstore.cc
index 54202862c8d6..5a4ae021ead3 100644
--- a/src/rgw/driver/dbstore/common/dbstore.cc
+++ b/src/rgw/driver/dbstore/common/dbstore.cc
@@ -474,20 +474,16 @@ int DB::get_bucket_info(const DoutPrefixProvider *dpp, const std::string& query_
 }
 
 int DB::create_bucket(const DoutPrefixProvider *dpp,
-    const RGWUserInfo& owner, rgw_bucket& bucket,
-    const string& zonegroup_id,
+    const rgw_user& owner, const rgw_bucket& bucket,
+    const std::string& zonegroup_id,
     const rgw_placement_rule& placement_rule,
-    const string& swift_ver_location,
-    const RGWQuotaInfo * pquota_info,
-    map<std::string, bufferlist>& attrs,
-    RGWBucketInfo& info,
-    obj_version *pobjv,
+    const std::map<std::string, bufferlist>& attrs,
+    const std::optional<std::string>& swift_ver_location,
+    const std::optional<RGWQuotaInfo>& quota,
+    std::optional<ceph::real_time> creation_time,
     obj_version *pep_objv,
-    real_time creation_time,
-    rgw_bucket *pmaster_bucket,
-    uint32_t *pmaster_num_shards,
-    optional_yield y,
-    bool exclusive)
+    RGWBucketInfo& info,
+    optional_yield y)
 {
   /*
    * XXX: Simple creation for now.
@@ -506,50 +502,48 @@ int DB::create_bucket(const DoutPrefixProvider *dpp,
   orig_info.bucket.name = bucket.name;
   ret = get_bucket_info(dpp, string("name"), "", orig_info, nullptr, nullptr, nullptr);
 
-  if (!ret && !orig_info.owner.id.empty() && exclusive) {
+  if (!ret && !orig_info.owner.id.empty()) {
     /* already exists. Return the old info */
-
     info = std::move(orig_info);
     return ret;
   }
 
   RGWObjVersionTracker& objv_tracker = info.objv_tracker;
-
   objv_tracker.read_version.clear();
+  objv_tracker.generate_new_write_ver(cct);
 
-  if (pobjv) {
-    objv_tracker.write_version = *pobjv;
-  } else {
-    objv_tracker.generate_new_write_ver(cct);
-  }
   params.op.bucket.bucket_version = objv_tracker.write_version;
   objv_tracker.read_version = params.op.bucket.bucket_version;
 
-  uint64_t bid = next_bucket_id();
-  string s = getDBname() + "." + std::to_string(bid);
-  bucket.marker = bucket.bucket_id = s;
-
   info.bucket = bucket;
-  info.owner = owner.user_id;
+  if (info.bucket.marker.empty()) {
+    uint64_t bid = next_bucket_id();
+    string s = getDBname() + "." + std::to_string(bid);
+    info.bucket.marker = info.bucket.bucket_id = s;
+  }
+
+  info.owner = owner;
   info.zonegroup = zonegroup_id;
   info.placement_rule = placement_rule;
-  info.swift_ver_location = swift_ver_location;
-  info.swift_versioning = (!swift_ver_location.empty());
+  if (swift_ver_location) {
+    info.swift_ver_location = *swift_ver_location;
+  }
+  info.swift_versioning = swift_ver_location.has_value();
 
   info.requester_pays = false;
-  if (real_clock::is_zero(creation_time)) {
-    info.creation_time = ceph::real_clock::now();
+  if (creation_time) {
+    info.creation_time = *creation_time;
   } else {
-    info.creation_time = creation_time;
+    info.creation_time = ceph::real_clock::now();
   }
-  if (pquota_info) {
-    info.quota = *pquota_info;
+  if (quota) {
+    info.quota = *quota;
   }
 
   params.op.bucket.info = info;
   params.op.bucket.bucket_attrs = attrs;
   params.op.bucket.mtime = ceph::real_time();
-  params.op.user.uinfo.user_id.id = owner.user_id.id;
+  params.op.user.uinfo.user_id.id = owner.id;
 
   ret = ProcessOp(dpp, "InsertBucket", &params);
 
diff --git a/src/rgw/driver/dbstore/common/dbstore.h b/src/rgw/driver/dbstore/common/dbstore.h
index b26cc116e82d..79f312cb8446 100644
--- a/src/rgw/driver/dbstore/common/dbstore.h
+++ b/src/rgw/driver/dbstore/common/dbstore.h
@@ -1595,20 +1595,16 @@ class DB {
         RGWBucketInfo& info, rgw::sal::Attrs* pattrs, ceph::real_time* pmtime,
         obj_version* pbucket_version);
     int create_bucket(const DoutPrefixProvider *dpp,
-        const RGWUserInfo& owner, rgw_bucket& bucket,
+        const rgw_user& owner, const rgw_bucket& bucket,
         const std::string& zonegroup_id,
         const rgw_placement_rule& placement_rule,
-        const std::string& swift_ver_location,
-        const RGWQuotaInfo * pquota_info,
-        std::map<std::string, bufferlist>& attrs,
-        RGWBucketInfo& info,
-        obj_version *pobjv,
+        const std::map<std::string, bufferlist>& attrs,
+        const std::optional<std::string>& swift_ver_location,
+        const std::optional<RGWQuotaInfo>& quota,
+        std::optional<ceph::real_time> creation_time,
         obj_version *pep_objv,
-        real_time creation_time,
-        rgw_bucket *pmaster_bucket,
-        uint32_t *pmaster_num_shards,
-        optional_yield y,
-        bool exclusive);
+        RGWBucketInfo& info,
+        optional_yield y);
 
     int next_bucket_id() { return ++max_bucket_id; };
 
diff --git a/src/rgw/driver/dbstore/tests/dbstore_tests.cc b/src/rgw/driver/dbstore/tests/dbstore_tests.cc
index 27edb7b85739..14fe9c37e754 100644
--- a/src/rgw/driver/dbstore/tests/dbstore_tests.cc
+++ b/src/rgw/driver/dbstore/tests/dbstore_tests.cc
@@ -461,45 +461,37 @@ TEST_F(DBStoreTest, CreateBucket) {
   struct DBOpParams params = GlobalParams;
   int ret = -1;
   RGWBucketInfo info;
-  RGWUserInfo owner;
+  rgw_user owner;
   rgw_bucket bucket;
   obj_version objv;
   rgw_placement_rule rule;
   map<std::string, bufferlist> attrs;
 
-  owner.user_id.id = "user_id1";
+  owner.id = "user_id1";
   bucket.name = "bucket1";
   bucket.tenant = "tenant";
 
-  objv.ver = 2;
-  objv.tag = "write_tag";
-
   rule.name = "rule1";
   rule.storage_class = "sc1";
 
-  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, "swift_ver", NULL,
-      attrs, info, &objv, NULL, bucket_mtime, NULL, NULL,
-      null_yield, false);
+  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, attrs, "swift_ver",
+      std::nullopt, bucket_mtime, nullptr, info, null_yield);
   ASSERT_EQ(ret, 0);
   bucket.name = "bucket2";
-  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, "swift_ver", NULL,
-      attrs, info, &objv, NULL, bucket_mtime, NULL, NULL,
-      null_yield, false);
+  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, attrs, "swift_ver",
+      std::nullopt, bucket_mtime, nullptr, info, null_yield);
   ASSERT_EQ(ret, 0);
   bucket.name = "bucket3";
-  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, "swift_ver", NULL,
-      attrs, info, &objv, NULL, bucket_mtime, NULL, NULL,
-      null_yield, false);
+  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, attrs, "swift_ver",
+      std::nullopt, bucket_mtime, nullptr, info, null_yield);
   ASSERT_EQ(ret, 0);
   bucket.name = "bucket4";
-  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, "swift_ver", NULL,
-      attrs, info, &objv, NULL, bucket_mtime, NULL, NULL,
-      null_yield, false);
+  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, attrs, "swift_ver",
+      std::nullopt, bucket_mtime, nullptr, info, null_yield);
   ASSERT_EQ(ret, 0);
   bucket.name = "bucket5";
-  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, "swift_ver", NULL,
-      attrs, info, &objv, NULL, bucket_mtime, NULL, NULL,
-      null_yield, false);
+  ret = db->create_bucket(dpp, owner, bucket, "zid", rule, attrs, "swift_ver",
+      std::nullopt, bucket_mtime, nullptr, info, null_yield);
   ASSERT_EQ(ret, 0);
 }
 
@@ -516,14 +508,14 @@ TEST_F(DBStoreTest, GetBucketQueryByName) {
   ASSERT_EQ(binfo.bucket.name, "bucket2");
   ASSERT_EQ(binfo.bucket.tenant, "tenant");
   ASSERT_EQ(binfo.owner.id, "user_id1");
-  ASSERT_EQ(binfo.objv_tracker.read_version.ver, 2);
-  ASSERT_EQ(binfo.objv_tracker.read_version.tag, "write_tag");
+  ASSERT_EQ(binfo.objv_tracker.read_version.ver, 1);
+  ASSERT_FALSE(binfo.objv_tracker.read_version.tag.empty());
   ASSERT_EQ(binfo.zonegroup, "zid");
   ASSERT_EQ(binfo.creation_time, bucket_mtime);
   ASSERT_EQ(binfo.placement_rule.name, "rule1");
   ASSERT_EQ(binfo.placement_rule.storage_class, "sc1");
-  ASSERT_EQ(objv.ver, 2);
-  ASSERT_EQ(objv.tag, "write_tag");
+  ASSERT_EQ(objv.ver, 1);
+  ASSERT_FALSE(objv.tag.empty());
 
   marker1 = binfo.bucket.marker;
 }
@@ -575,7 +567,7 @@ TEST_F(DBStoreTest, BucketChown) {
 
   ret = db->update_bucket(dpp, "owner", info, false, &user, nullptr, &bucket_mtime, nullptr);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(info.objv_tracker.read_version.ver, 3);
+  ASSERT_EQ(info.objv_tracker.read_version.ver, 2);
 }
 
 TEST_F(DBStoreTest, ListAllBuckets) {
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 5204017274a9..e4e640b02034 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -37,51 +37,6 @@ const std::string MP_OBJ_PART_PFX = "part-";
 const std::string MP_OBJ_PART_FMT = "{:0>5}";
 const std::string MP_OBJ_HEAD_NAME = MP_OBJ_PART_PFX + "00000";
 
-static int decode_policy(CephContext* cct,
-                         bufferlist& bl,
-                         RGWAccessControlPolicy* policy)
-{
-  auto iter = bl.cbegin();
-  try {
-    policy->decode(iter);
-  } catch (buffer::error& err) {
-    ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
-    return -EIO;
-  }
-  if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
-    ldout(cct, 15) << __func__ << " POSIX Read AccessControlPolicy";
-    RGWAccessControlPolicy_S3* s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
-    s3policy->to_xml(*_dout);
-    *_dout << dendl;
-  }
-  return 0;
-}
-
-static int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider* dpp,
-					      POSIXDriver* driver,
-					      User* user,
-					      Attrs& bucket_attrs,
-					      RGWAccessControlPolicy* policy,
-					      optional_yield y)
-{
-  auto aiter = bucket_attrs.find(RGW_ATTR_ACL);
-
-  if (aiter != bucket_attrs.end()) {
-    int ret = decode_policy(driver->ctx(), aiter->second, policy);
-    if (ret < 0)
-      return ret;
-  } else {
-    ldout(driver->ctx(), 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl;
-    /* object exists, but policy is broken */
-    int r = user->load_user(dpp, y);
-    if (r < 0)
-      return r;
-
-    policy->create_default(user->get_id(), user->get_display_name());
-  }
-  return 0;
-}
-
 static inline bool get_attr(Attrs& attrs, const char* name, bufferlist& bl)
 {
   auto iter = attrs.find(name);
@@ -566,74 +521,46 @@ int POSIXUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma
   return 0;
 }
 
-int POSIXUser::create_bucket(const DoutPrefixProvider* dpp,
-			      const rgw_bucket& b,
-			      const std::string& zonegroup_id,
-			      const rgw_placement_rule& placement_rule,
-			      const std::string& swift_ver_location,
-			      const RGWQuotaInfo * pquota_info,
-			      const RGWAccessControlPolicy& policy,
-			      Attrs& attrs,
-			      RGWBucketInfo& binfo,
-			      obj_version& ep_objv,
-			      bool exclusive,
-			      bool obj_lock_enabled,
-			      bool* existed,
-			      req_info& req_info,
-			      std::unique_ptr<Bucket>* bucket_out,
-			      optional_yield y)
-{
-  /* Check for existence */
-  {
-    std::unique_ptr<rgw::sal::Bucket> bucket;
+int POSIXBucket::create(const DoutPrefixProvider* dpp,
+			const CreateParams& params,
+			optional_yield y)
+{
+  ceph_assert(owner);
+  info.owner = owner->get_id();
 
-    int ret = driver->load_bucket(dpp, this, b, &bucket, y);
-    if (ret >= 0) {
-      *existed = true;
-      // Bucket exists.  Check owner comparison
-      if (bucket->get_info().owner.compare(this->get_id()) != 0) {
-	return -EEXIST;
-      }
-      // Don't allow changes to ACL policy
-      RGWAccessControlPolicy old_policy(driver->ctx());
-      ret = rgw_op_get_bucket_policy_from_attr(
-          dpp, driver, this, bucket->get_attrs(), &old_policy, y);
-      if (ret >= 0 && old_policy != policy) {
-        bucket_out->swap(bucket);
-        return -EEXIST;
-      }
-    } else {
-      *existed = false;
-    }
-  }
+  info.bucket.marker = params.marker;
+  info.bucket.bucket_id = params.bucket_id;
 
-  binfo.bucket = b;
-  binfo.owner = get_id();
-  binfo.zonegroup = zonegroup_id;
-  binfo.placement_rule = placement_rule;
-  binfo.swift_ver_location = swift_ver_location;
-  binfo.swift_versioning = (!swift_ver_location.empty());
-  binfo.requester_pays = false;
-  binfo.creation_time = ceph::real_clock::now();
-  if (pquota_info) {
-    binfo.quota = *pquota_info;
+  info.zonegroup = params.zonegroup_id;
+  info.placement_rule = params.placement_rule;
+  info.swift_versioning = params.swift_ver_location.has_value();
+  if (params.swift_ver_location) {
+    info.swift_ver_location = *params.swift_ver_location;
+  }
+  if (params.obj_lock_enabled) {
+    info.flags |= BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
+  }
+  info.requester_pays = false;
+  if (params.creation_time) {
+    info.creation_time = *params.creation_time;
+  } else {
+    info.creation_time = ceph::real_clock::now();
+  }
+  if (params.quota) {
+    info.quota = *params.quota;
   }
 
-  POSIXBucket* fb = new POSIXBucket(driver, driver->get_root_fd(), binfo, this);
-
-  int ret = fb->set_attrs(attrs);
+  int ret = set_attrs(attrs);
   if (ret < 0) {
-    delete fb;
-    return  ret;
+    return ret;
   }
 
-  ret = fb->create(dpp, y, existed);
+  bool existed = false;
+  ret = create(dpp, y, &existed);
   if (ret < 0) {
-    delete fb;
-    return  ret;
+    return ret;
   }
 
-  bucket_out->reset(fb);
   return 0;
 }
 
@@ -1240,7 +1167,7 @@ int POSIXBucket::create(const DoutPrefixProvider* dpp, optional_yield y, bool* e
     } else if (existed != nullptr) {
       *existed = true;
     }
-    return ret;
+    return -ret;
   }
 
   return write_attrs(dpp, y);
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index e132f240812b..c5360aa90f9a 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -121,22 +121,6 @@ class POSIXUser : public FilterUser {
 			   const std::string& marker, const std::string& end_marker,
 			   uint64_t max, bool need_stats, BucketList& buckets,
 			   optional_yield y) override;
-  virtual int create_bucket(const DoutPrefixProvider* dpp,
-                            const rgw_bucket& b,
-                            const std::string& zonegroup_id,
-                            const rgw_placement_rule& placement_rule,
-                            const std::string& swift_ver_location,
-                            const RGWQuotaInfo* pquota_info,
-                            const RGWAccessControlPolicy& policy,
-                            Attrs& attrs,
-                            RGWBucketInfo& info,
-                            obj_version& ep_objv,
-                            bool exclusive,
-                            bool obj_lock_enabled,
-                            bool* existed,
-                            req_info& req_info,
-                            std::unique_ptr<Bucket>* bucket,
-                            optional_yield y) override;
   virtual Attrs& get_attrs() override { return next->get_attrs(); }
   virtual void set_attrs(Attrs& _attrs) override { next->set_attrs(_attrs); }
   virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
@@ -197,6 +181,9 @@ class POSIXBucket : public StoreBucket {
 				      bool keep_index_consistent,
 				      optional_yield y,
 				      const DoutPrefixProvider *dpp) override;
+  virtual int create(const DoutPrefixProvider* dpp,
+		     const CreateParams& params,
+		     optional_yield y) override;
   virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
   virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl,
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 1ec176374350..3632e66af595 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2272,79 +2272,70 @@ void RGWRados::create_bucket_id(string *bucket_id)
   *bucket_id = buf;
 }
 
-int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
-                            const string& zonegroup_id,
+int RGWRados::create_bucket(const DoutPrefixProvider* dpp,
+                            optional_yield y,
+                            const rgw_bucket& bucket,
+                            const rgw_user& owner,
+                            const std::string& zonegroup_id,
                             const rgw_placement_rule& placement_rule,
-                            const string& swift_ver_location,
+                            const RGWZonePlacementInfo* zone_placement,
+                            const std::map<std::string, bufferlist>& attrs,
                             bool obj_lock_enabled,
-                            const RGWQuotaInfo * pquota_info,
-			    map<std::string, bufferlist>& attrs,
-                            RGWBucketInfo& info,
-                            obj_version *pobjv,
-                            obj_version *pep_objv,
-                            real_time creation_time,
-                            const rgw_bucket* pmaster_bucket,
-			    optional_yield y,
-                            const DoutPrefixProvider *dpp,
-			    bool exclusive)
+                            const std::optional<std::string>& swift_ver_location,
+                            const std::optional<RGWQuotaInfo>& quota,
+                            std::optional<ceph::real_time> creation_time,
+                            obj_version* pep_objv,
+                            RGWBucketInfo& info)
 {
-#define MAX_CREATE_RETRIES 20 /* need to bound retries */
-  rgw_placement_rule selected_placement_rule;
-  RGWZonePlacementInfo rule_info;
+  int ret = 0;
 
+#define MAX_CREATE_RETRIES 20 /* need to bound retries */
   for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
-    int ret = 0;
-    ret = svc.zone->select_bucket_placement(dpp, owner, zonegroup_id, placement_rule,
-                                            &selected_placement_rule, &rule_info, y);
-    if (ret < 0)
-      return ret;
-
-    if (!pmaster_bucket) {
-      create_bucket_id(&bucket.marker);
-      bucket.bucket_id = bucket.marker;
-    } else {
-      bucket.marker = pmaster_bucket->marker;
-      bucket.bucket_id = pmaster_bucket->bucket_id;
-    }
-
     RGWObjVersionTracker& objv_tracker = info.objv_tracker;
-
     objv_tracker.read_version.clear();
+    objv_tracker.generate_new_write_ver(cct);
 
-    if (pobjv) {
-      objv_tracker.write_version = *pobjv;
+    if (bucket.marker.empty()) {
+      create_bucket_id(&info.bucket.marker);
+      info.bucket.bucket_id = info.bucket.marker;
     } else {
-      objv_tracker.generate_new_write_ver(cct);
+      info.bucket = bucket;
     }
 
-    info.bucket = bucket;
-    info.owner = owner.user_id;
+    info.owner = owner;
     info.zonegroup = zonegroup_id;
-    info.placement_rule = selected_placement_rule;
-    info.swift_ver_location = swift_ver_location;
-    info.swift_versioning = (!swift_ver_location.empty());
+    info.placement_rule = placement_rule;
+    info.swift_versioning = swift_ver_location.has_value();
+    if (swift_ver_location) {
+      info.swift_ver_location = *swift_ver_location;
+    }
     if (obj_lock_enabled) {
       info.flags |= BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
     }
 
-    init_default_bucket_layout(cct, info.layout, svc.zone->get_zone(),
-			       rule_info.index_type);
+    if (zone_placement) {
+      init_default_bucket_layout(cct, info.layout, svc.zone->get_zone(),
+                                 zone_placement->index_type);
+    }
 
     info.requester_pays = false;
-    if (real_clock::is_zero(creation_time)) {
-      info.creation_time = ceph::real_clock::now();
+    if (creation_time) {
+      info.creation_time = *creation_time;
     } else {
-      info.creation_time = creation_time;
+      info.creation_time = ceph::real_clock::now();
     }
-    if (pquota_info) {
-      info.quota = *pquota_info;
+    if (quota) {
+      info.quota = *quota;
     }
 
-    int r = svc.bi->init_index(dpp, info, info.layout.current_index);
-    if (r < 0) {
-      return r;
+    if (zone_placement) {
+      ret = svc.bi->init_index(dpp, info, info.layout.current_index);
+      if (ret < 0) {
+        return ret;
+      }
     }
 
+    constexpr bool exclusive = true;
     ret = put_linked_bucket_info(info, exclusive, ceph::real_time(), pep_objv, &attrs, true, dpp, y);
     if (ret == -ECANCELED) {
       ret = -EEXIST;
@@ -2352,7 +2343,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
     if (ret == -EEXIST) {
        /* we need to reread the info and return it, caller will have a use for it */
       RGWBucketInfo orig_info;
-      r = get_bucket_info(&svc, bucket.tenant, bucket.name, orig_info, NULL, y, NULL);
+      int r = get_bucket_info(&svc, bucket.tenant, bucket.name, orig_info, NULL, y, NULL);
       if (r < 0) {
         if (r == -ENOENT) {
           continue;
@@ -2363,10 +2354,12 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
 
       /* only remove it if it's a different bucket instance */
       if (orig_info.bucket.bucket_id != bucket.bucket_id) {
-	int r = svc.bi->clean_index(dpp, info, info.layout.current_index);
-	if (r < 0) {
-	  ldpp_dout(dpp, 0) << "WARNING: could not remove bucket index (r=" << r << ")" << dendl;
-	}
+        if (zone_placement) {
+          r = svc.bi->clean_index(dpp, info, info.layout.current_index);
+          if (r < 0) {
+            ldpp_dout(dpp, 0) << "WARNING: could not remove bucket index (r=" << r << ")" << dendl;
+          }
+        }
         r = ctl.bucket->remove_bucket_instance_info(info.bucket, info, y, dpp);
         if (r < 0) {
           ldpp_dout(dpp, 0) << "WARNING: " << __func__ << "(): failed to remove bucket instance info: bucket instance=" << info.bucket.get_key() << ": r=" << r << dendl;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 18e818c4e893..77a08c068a64 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -628,21 +628,20 @@ class RGWRados
   bool get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool);
   bool obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_raw_obj *raw_obj);
 
-  int create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket,
-		    const std::string& zonegroup_id,
-		    const rgw_placement_rule& placement_rule,
-		    const std::string& swift_ver_location,
-		    bool obj_lock_enabled,
-		    const RGWQuotaInfo * pquota_info,
-		    std::map<std::string,bufferlist>& attrs,
-		    RGWBucketInfo& bucket_info,
-		    obj_version *pobjv,
-		    obj_version *pep_objv,
-		    ceph::real_time creation_time,
-		    const rgw_bucket *master_bucket,
-		    optional_yield y,
-                    const DoutPrefixProvider *dpp,
-		    bool exclusive = true);
+  int create_bucket(const DoutPrefixProvider* dpp,
+                    optional_yield y,
+                    const rgw_bucket& bucket,
+                    const rgw_user& owner,
+                    const std::string& zonegroup_id,
+                    const rgw_placement_rule& placement_rule,
+                    const RGWZonePlacementInfo* zone_placement,
+                    const std::map<std::string, bufferlist>& attrs,
+                    bool obj_lock_enabled,
+                    const std::optional<std::string>& swift_ver_location,
+                    const std::optional<RGWQuotaInfo>& quota,
+                    std::optional<ceph::real_time> creation_time,
+                    obj_version* pep_objv,
+                    RGWBucketInfo& info);
 
   RGWCoroutinesManagerRegistry *get_cr_registry() { return cr_registry; }
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 328b4f82c8f2..fd8964e83ba1 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -120,120 +120,52 @@ int RadosUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma
   return 0;
 }
 
-int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
-				 const rgw_bucket& b,
-				 const std::string& zonegroup_id,
-				 const rgw_placement_rule& placement_rule,
-				 const std::string& swift_ver_location,
-				 const RGWQuotaInfo * pquota_info,
-				 const RGWAccessControlPolicy& policy,
-				 Attrs& attrs,
-				 RGWBucketInfo& info,
-				 obj_version& ep_objv,
-				 bool exclusive,
-				 bool obj_lock_enabled,
-				 bool* existed,
-				 req_info& req_info,
-				 std::unique_ptr<Bucket>* bucket_out,
-				 optional_yield y)
-{
-  int ret;
-  bufferlist in_data;
-  RGWBucketInfo master_info;
-  rgw_bucket* pmaster_bucket = nullptr;
-  real_time creation_time;
-  std::unique_ptr<Bucket> bucket;
-  obj_version objv,* pobjv = NULL;
-
-  /* If it exists, look it up; otherwise create it */
-  ret = store->load_bucket(dpp, this, b, &bucket, y);
-  if (ret < 0 && ret != -ENOENT)
-    return ret;
-
-  if (ret != -ENOENT) {
-    *existed = true;
-  } else {
-    bucket = std::unique_ptr<Bucket>(new RadosBucket(store, b, this));
-    *existed = false;
-    bucket->set_attrs(attrs);
-  }
-
-  if (!store->svc()->zone->is_meta_master()) {
-    JSONParser jp;
-    ret = store->forward_request_to_master(dpp, this, NULL, in_data, &jp, req_info, y);
-    if (ret < 0) {
-      return ret;
-    }
-
-    JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp);
-    JSONDecoder::decode_json("object_ver", objv, &jp);
-    JSONDecoder::decode_json("bucket_info", master_info, &jp);
-    ldpp_dout(dpp, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl;
-    std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time);
-    ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl;
-    pmaster_bucket= &master_info.bucket;
-    creation_time = master_info.creation_time;
-    pobjv = &objv;
-    obj_lock_enabled = master_info.obj_lock_enabled();
-  }
-
-  std::string zid = zonegroup_id;
-  if (zid.empty()) {
-    zid = store->svc()->zone->get_zonegroup().get_id();
-  }
-
-  if (*existed) {
-    rgw_placement_rule selected_placement_rule;
-    ret = store->svc()->zone->select_bucket_placement(dpp, this->get_info(),
-					       zid, placement_rule,
-					       &selected_placement_rule, nullptr, y);
-    if (selected_placement_rule != info.placement_rule) {
-      ret = -EEXIST;
-      bucket_out->swap(bucket);
-      return ret;
-    }
-  } else {
-
-    ret = store->getRados()->create_bucket(this->get_info(), bucket->get_key(),
-				    zid, placement_rule, swift_ver_location,
-				    obj_lock_enabled, pquota_info,
-				    attrs, info, pobjv, &ep_objv, creation_time,
-				    pmaster_bucket, y, dpp, exclusive);
-    if (ret == -EEXIST) {
-      *existed = true;
-      /* bucket already existed, might have raced with another bucket creation,
-       * or might be partial bucket creation that never completed. Read existing
-       * bucket info, verify that the reported bucket owner is the current user.
-       * If all is ok then update the user's list of buckets.  Otherwise inform
-       * client about a name conflict.
-       */
-      if (info.owner.compare(this->get_id()) != 0) {
-	return -EEXIST;
-      }
-      ret = 0;
-    } else if (ret != 0) {
-      return ret;
+int RadosBucket::create(const DoutPrefixProvider* dpp,
+                        const CreateParams& params,
+                        optional_yield y)
+{
+  ceph_assert(owner);
+  const rgw_user& owner_id = owner->get_id();
+
+  rgw_bucket key = get_key();
+  key.marker = params.marker;
+  key.bucket_id = params.bucket_id;
+
+  int ret = store->getRados()->create_bucket(
+      dpp, y, key, owner_id, params.zonegroup_id,
+      params.placement_rule, params.zone_placement, params.attrs,
+      params.obj_lock_enabled, params.swift_ver_location,
+      params.quota, params.creation_time, &bucket_version, info);
+
+  bool existed = false;
+  if (ret == -EEXIST) {
+    existed = true;
+    /* bucket already existed, might have raced with another bucket creation,
+     * or might be partial bucket creation that never completed. Read existing
+     * bucket info, verify that the reported bucket owner is the current user.
+     * If all is ok then update the user's list of buckets.  Otherwise inform
+     * client about a name conflict.
+     */
+    if (info.owner != owner_id) {
+      return -ERR_BUCKET_EXISTS;
     }
+    ret = 0;
+  } else if (ret != 0) {
+    return ret;
   }
 
-  bucket->set_version(ep_objv);
-  bucket->get_info() = info;
-
-  RadosBucket* rbucket = static_cast<RadosBucket*>(bucket.get());
-  ret = rbucket->link(dpp, this, y, false);
-  if (ret && !*existed && ret != -EEXIST) {
+  ret = link(dpp, owner, y, false);
+  if (ret && !existed && ret != -EEXIST) {
     /* if it exists (or previously existed), don't remove it! */
-    ret = rbucket->unlink(dpp, this, y);
+    ret = unlink(dpp, owner, y);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret
 		       << dendl;
     }
-  } else if (ret == -EEXIST || (ret == 0 && *existed)) {
+  } else if (ret == -EEXIST || (ret == 0 && existed)) {
     ret = -ERR_BUCKET_EXISTS;
   }
 
-  bucket_out->swap(bucket);
-
   return ret;
 }
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 8c1926f3fbef..c8fa876260bf 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -274,22 +274,6 @@ class RadosUser : public StoreUser {
     int list_buckets(const DoutPrefixProvider* dpp, const std::string& marker, const std::string& end_marker,
 		     uint64_t max, bool need_stats, BucketList& buckets,
 		     optional_yield y) override;
-    virtual int create_bucket(const DoutPrefixProvider* dpp,
-                            const rgw_bucket& b,
-                            const std::string& zonegroup_id,
-                            const rgw_placement_rule& placement_rule,
-                            const std::string& swift_ver_location,
-                            const RGWQuotaInfo * pquota_info,
-                            const RGWAccessControlPolicy& policy,
-			    Attrs& attrs,
-                            RGWBucketInfo& info,
-                            obj_version& ep_objv,
-			    bool exclusive,
-			    bool obj_lock_enabled,
-			    bool* existed,
-			    req_info& req_info,
-			    std::unique_ptr<Bucket>* bucket,
-			    optional_yield y) override;
     virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
     virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp, Attrs& new_attrs, optional_yield y) override;
     virtual int read_stats(const DoutPrefixProvider *dpp,
@@ -544,6 +528,8 @@ class RadosBucket : public StoreBucket {
 					DoutPrefixProvider *dpp) override;
     virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
     virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
+    int create(const DoutPrefixProvider* dpp, const CreateParams& params,
+               optional_yield y) override;
     virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) override;
     virtual int read_stats(const DoutPrefixProvider *dpp,
                            const bucket_index_layout_generation& idx_layout,
diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index 8c237f6bedac..a9cc179f06c2 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -1094,6 +1094,27 @@ int delete_zone(const DoutPrefixProvider* dpp, optional_yield y,
   return writer.remove(dpp, y);
 }
 
+auto find_zone_placement(const DoutPrefixProvider* dpp,
+                         const RGWZoneParams& info,
+                         const rgw_placement_rule& rule)
+    -> const RGWZonePlacementInfo*
+{
+  auto i = info.placement_pools.find(rule.name);
+  if (i == info.placement_pools.end()) {
+    ldpp_dout(dpp, 0) << "ERROR: This zone does not contain placement rule "
+        << rule.name << dendl;
+    return nullptr;
+  }
+
+  const std::string& storage_class = rule.get_storage_class();
+  if (!i->second.storage_class_exists(storage_class)) {
+    ldpp_dout(dpp, 5) << "ERROR: The zone placement for rule " << rule.name
+        << " does not contain storage class " << storage_class << dendl;
+    return nullptr;
+  }
+
+  return &i->second;
+}
 
 static int read_or_create_default_zone(const DoutPrefixProvider* dpp,
                                        optional_yield y,
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 6d55fd52c575..b57b05ada857 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -943,6 +943,12 @@ int delete_zone(const DoutPrefixProvider* dpp, optional_yield y,
                 sal::ConfigStore* cfgstore, const RGWZoneParams& info,
                 sal::ZoneWriter& writer);
 
+/// Return the zone placement corresponding to the given rule, or nullptr.
+auto find_zone_placement(const DoutPrefixProvider* dpp,
+                         const RGWZoneParams& info,
+                         const rgw_placement_rule& rule)
+    -> const RGWZonePlacementInfo*;
+
 
 /// Global state about the site configuration. Initialized once during
 /// startup and may be reinitialized by RGWRealmReloader, but is otherwise
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 067a939d7535..1bd1fd85ba0c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -487,6 +487,8 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
     s->bucket_acl = std::make_unique<RGWAccessControlPolicy>(s->cct);
   }
 
+  const RGWZoneGroup& zonegroup = s->penv.site->get_zonegroup();
+
   /* check if copy source is within the current domain */
   if (!s->src_bucket_name.empty()) {
     std::unique_ptr<rgw::sal::Bucket> src_bucket;
@@ -494,8 +496,7 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
                               rgw_bucket(s->src_tenant_name, s->src_bucket_name),
                               &src_bucket, y);
     if (ret == 0) {
-      string& zonegroup = src_bucket->get_info().zonegroup;
-      s->local_source = driver->get_zone()->get_zonegroup().equals(zonegroup);
+      s->local_source = zonegroup.equals(src_bucket->get_info().zonegroup);
     }
   }
 
@@ -543,20 +544,13 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 
     s->bucket_owner = s->bucket_acl->get_owner();
 
-    std::unique_ptr<rgw::sal::ZoneGroup> zonegroup;
-    int r = driver->get_zonegroup(s->bucket->get_info().zonegroup, &zonegroup);
-    if (!r) {
-      s->zonegroup_endpoint = zonegroup->get_endpoint();
-      s->zonegroup_name = zonegroup->get_name();
-    }
-    if (r < 0 && ret == 0) {
-      ret = r;
-    }
+    s->zonegroup_endpoint = rgw::get_zonegroup_endpoint(zonegroup);
+    s->zonegroup_name = zonegroup.get_name();
 
-    if (!driver->get_zone()->get_zonegroup().equals(s->bucket->get_info().zonegroup)) {
+    if (!zonegroup.equals(s->bucket->get_info().zonegroup)) {
       ldpp_dout(dpp, 0) << "NOTICE: request for data in a different zonegroup ("
           << s->bucket->get_info().zonegroup << " != "
-          << driver->get_zone()->get_zonegroup().get_id() << ")" << dendl;
+          << zonegroup.get_id() << ")" << dendl;
       /* we now need to make sure that the operation actually requires copy source, that is
        * it's a copy operation
        */
@@ -3331,6 +3325,47 @@ static void filter_out_website(std::map<std::string, ceph::bufferlist>& add_attr
   }
 }
 
+static int select_bucket_placement(const DoutPrefixProvider* dpp,
+                                   const RGWZoneGroup& zonegroup,
+                                   const RGWUserInfo& user,
+                                   rgw_placement_rule& rule)
+{
+  std::string_view selected = "requested";
+
+  // select placement: requested rule > user default > zonegroup default
+  if (rule.name.empty()) {
+    selected = "user-default";
+    rule.inherit_from(user.default_placement);
+    if (rule.name.empty()) {
+      selected = "zonegroup-default";
+      rule.inherit_from(zonegroup.default_placement);
+      if (rule.name.empty()) {
+        ldpp_dout(dpp, 0) << "ERROR: misconfigured zonegroup " << zonegroup.id
+            << ", default placement should not be empty" << dendl;
+        return -ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION;
+      }
+    }
+  }
+
+  // look up the zonegroup placement target
+  auto target = zonegroup.placement_targets.find(rule.name);
+  if (target == zonegroup.placement_targets.end()) {
+    ldpp_dout(dpp, 0) << "could not find " << selected << " placement target "
+        << rule.name << " within zonegroup" << dendl;
+    return -ERR_INVALID_LOCATION_CONSTRAINT;
+  }
+
+  // check the user's permission tags
+  if (!target->second.user_permitted(user.placement_tags)) {
+    ldpp_dout(dpp, 0) << "user not permitted to use placement rule "
+        << target->first << dendl;
+    return -EPERM;
+  }
+
+  ldpp_dout(dpp, 20) << "using " << selected << " placement target "
+      << rule.name << dendl;
+  return 0;
+}
 
 void RGWCreateBucket::execute(optional_yield y)
 {
@@ -3338,149 +3373,181 @@ void RGWCreateBucket::execute(optional_yield y)
   if (op_ret < 0)
     return;
 
-  if (!relaxed_region_enforcement &&
-      !location_constraint.empty() &&
-      !driver->get_zone()->has_zonegroup_api(location_constraint)) {
-      ldpp_dout(this, 0) << "location constraint (" << location_constraint << ")"
-                       << " can't be found." << dendl;
+  const rgw::SiteConfig& site = *s->penv.site;
+  const std::optional<RGWPeriod>& period = site.get_period();
+  const RGWZoneGroup& my_zonegroup = site.get_zonegroup();
+
+  if (s->system_request) {
+    // allow system requests to override the target zonegroup. for forwarded
+    // requests, we'll create the bucket for the originating zonegroup
+    createparams.zonegroup_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "zonegroup");
+  }
+
+  const RGWZoneGroup* bucket_zonegroup = &my_zonegroup;
+  if (createparams.zonegroup_id.empty()) {
+    // default to the local zonegroup
+    createparams.zonegroup_id = my_zonegroup.id;
+  } else if (period) {
+    auto z = period->period_map.zonegroups.find(createparams.zonegroup_id);
+    if (z == period->period_map.zonegroups.end()) {
+      ldpp_dout(this, 0) << "could not find zonegroup "
+          << createparams.zonegroup_id << " in current period" << dendl;
+      op_ret = -ENOENT;
+      return;
+    }
+    bucket_zonegroup = &z->second;
+  } else if (createparams.zonegroup_id != my_zonegroup.id) {
+    ldpp_dout(this, 0) << "zonegroup does not match current zonegroup "
+        << createparams.zonegroup_id << dendl;
+    op_ret = -ENOENT;
+    return;
+  }
+
+  // validate the LocationConstraint
+  if (!location_constraint.empty() && !relaxed_region_enforcement) {
+    // on the master zonegroup, allow any valid api_name. otherwise it has to
+    // match the bucket's zonegroup
+    if (period && my_zonegroup.is_master) {
+      if (!period->period_map.zonegroups_by_api.count(location_constraint)) {
+        ldpp_dout(this, 0) << "location constraint (" << location_constraint
+            << ") can't be found." << dendl;
+        op_ret = -ERR_INVALID_LOCATION_CONSTRAINT;
+        s->err.message = "The specified location-constraint is not valid";
+        return;
+      }
+    } else if (bucket_zonegroup->api_name != location_constraint) {
+      ldpp_dout(this, 0) << "location constraint (" << location_constraint
+          << ") doesn't match zonegroup (" << bucket_zonegroup->api_name
+          << ')' << dendl;
       op_ret = -ERR_INVALID_LOCATION_CONSTRAINT;
       s->err.message = "The specified location-constraint is not valid";
       return;
+    }
   }
 
-  if (!relaxed_region_enforcement && !driver->get_zone()->get_zonegroup().is_master_zonegroup() && !location_constraint.empty() &&
-      driver->get_zone()->get_zonegroup().get_api_name() != location_constraint) {
-    ldpp_dout(this, 0) << "location constraint (" << location_constraint << ")"
-                     << " doesn't match zonegroup" << " (" << driver->get_zone()->get_zonegroup().get_api_name() << ")"
-                     << dendl;
-    op_ret = -ERR_INVALID_LOCATION_CONSTRAINT;
-    s->err.message = "The specified location-constraint is not valid";
+  // select and validate the placement target
+  op_ret = select_bucket_placement(this, *bucket_zonegroup, s->user->get_info(),
+                                   createparams.placement_rule);
+  if (op_ret < 0) {
     return;
   }
 
-  std::set<std::string> names;
-  driver->get_zone()->get_zonegroup().get_placement_target_names(names);
-  if (!placement_rule.name.empty() &&
-      !names.count(placement_rule.name)) {
-    ldpp_dout(this, 0) << "placement target (" << placement_rule.name << ")"
-                     << " doesn't exist in the placement targets of zonegroup"
-                     << " (" << driver->get_zone()->get_zonegroup().get_api_name() << ")" << dendl;
-    op_ret = -ERR_INVALID_LOCATION_CONSTRAINT;
-    s->err.message = "The specified placement target does not exist";
-    return;
+  if (bucket_zonegroup == &my_zonegroup) {
+    // look up the zone placement pool
+    createparams.zone_placement = rgw::find_zone_placement(
+        this, site.get_zone_params(), createparams.placement_rule);
+    if (!createparams.zone_placement) {
+      op_ret = -ERR_INVALID_LOCATION_CONSTRAINT;
+      return;
+    }
   }
 
-  /* we need to make sure we read bucket info, it's not read before for this
-   * specific request */
-  {
-    std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-    op_ret = driver->load_bucket(this, s->user.get(),
-                                 rgw_bucket(s->bucket_tenant, s->bucket_name),
-                                 &tmp_bucket, y);
-    if (op_ret < 0 && op_ret != -ENOENT)
-      return;
-    s->bucket_exists = (op_ret != -ENOENT);
+  // read the bucket info if it exists
+  op_ret = driver->load_bucket(this, s->user.get(),
+                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &s->bucket, y);
+  if (op_ret < 0 && op_ret != -ENOENT)
+    return;
+  s->bucket_exists = (op_ret != -ENOENT);
+  ceph_assert(s->bucket); // creates handle even on ENOENT
 
-    if (s->bucket_exists) {
-      if (!s->system_request &&
-	  driver->get_zone()->get_zonegroup().get_id() !=
-	  tmp_bucket->get_info().zonegroup) {
-	op_ret = -EEXIST;
-	return;
-      }
-      /* Initialize info from req_state */
-      info = tmp_bucket->get_info();
+  if (s->bucket_exists) {
+    const RGWBucketInfo& info = s->bucket->get_info();
 
-      if (!swift_ver_location) {
-        swift_ver_location = info.swift_ver_location;
-      }
-      placement_rule.inherit_from(info.placement_rule);
-
-      // don't allow changes to the acl policy
-      RGWAccessControlPolicy old_policy(get_cct());
-      int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
-                                                 tmp_bucket->get_attrs(),
-                                                 &old_policy, y);
-      if (r >= 0 && old_policy != policy) {
-        s->err.message = "Cannot modify existing access control policy";
-        op_ret = -EEXIST;
-        return;
-      }
+    if (!s->system_request && createparams.zonegroup_id != info.zonegroup) {
+      s->err.message = "Cannot modify existing bucket's zonegroup";
+      op_ret = -EEXIST;
+      return;
     }
-  }
 
-  s->bucket_owner.set_id(s->user->get_id());
-  s->bucket_owner.set_name(s->user->get_display_name());
+    if (!createparams.swift_ver_location) {
+      createparams.swift_ver_location = info.swift_ver_location;
+    }
 
-  string zonegroup_id;
+    // don't allow changes to placement
+    if (createparams.placement_rule != info.placement_rule) {
+      s->err.message = "Cannot modify existing bucket's placement rule";
+      op_ret = -EEXIST;
+      return;
+    }
 
-  if (s->system_request) {
-    zonegroup_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "zonegroup");
-    if (zonegroup_id.empty()) {
-      zonegroup_id = driver->get_zone()->get_zonegroup().get_id();
+    // don't allow changes to the acl policy
+    RGWAccessControlPolicy old_policy(get_cct());
+    int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
+                                               s->bucket->get_attrs(),
+                                               &old_policy, y);
+    if (r >= 0 && old_policy != policy) {
+      s->err.message = "Cannot modify existing access control policy";
+      op_ret = -EEXIST;
+      return;
     }
-  } else {
-    zonegroup_id = driver->get_zone()->get_zonegroup().get_id();
   }
 
-  /* Encode special metadata first as we're using std::map::emplace under
-   * the hood. This method will add the new items only if the map doesn't
-   * contain such keys yet. */
+  s->bucket_owner.set_id(s->user->get_id());
+  s->bucket_owner.set_name(s->user->get_display_name());
+
   buffer::list aclbl;
   policy.encode(aclbl);
-  emplace_attr(RGW_ATTR_ACL, std::move(aclbl));
+  createparams.attrs[RGW_ATTR_ACL] = std::move(aclbl);
 
   if (has_cors) {
     buffer::list corsbl;
     cors_config.encode(corsbl);
-    emplace_attr(RGW_ATTR_CORS, std::move(corsbl));
+    createparams.attrs[RGW_ATTR_CORS] = std::move(corsbl);
   }
 
-  RGWQuotaInfo quota_info;
-  const RGWQuotaInfo * pquota_info = nullptr;
   if (need_metadata_upload()) {
     /* It's supposed that following functions WILL NOT change any special
      * attributes (like RGW_ATTR_ACL) if they are already present in attrs. */
-    op_ret = rgw_get_request_metadata(this, s->cct, s->info, attrs, false);
+    op_ret = rgw_get_request_metadata(this, s->cct, s->info,
+                                      createparams.attrs, false);
     if (op_ret < 0) {
       return;
     }
-    prepare_add_del_attrs(s->bucket_attrs, rmattr_names, attrs);
-    populate_with_generic_attrs(s, attrs);
+    prepare_add_del_attrs(s->bucket_attrs, rmattr_names, createparams.attrs);
+    populate_with_generic_attrs(s, createparams.attrs);
 
-    op_ret = filter_out_quota_info(attrs, rmattr_names, quota_info);
+    RGWQuotaInfo quota;
+    op_ret = filter_out_quota_info(createparams.attrs, rmattr_names, quota);
     if (op_ret < 0) {
       return;
     }
-    pquota_info = &quota_info;
+    createparams.quota = quota;
 
     /* Web site of Swift API. */
-    filter_out_website(attrs, rmattr_names, info.website_conf);
+    RGWBucketInfo& info = s->bucket->get_info();
+    filter_out_website(createparams.attrs, rmattr_names, info.website_conf);
     info.has_website = !info.website_conf.is_empty();
   }
 
-  rgw_bucket tmp_bucket;
-  tmp_bucket.tenant = s->bucket_tenant; /* ignored if bucket exists */
-  tmp_bucket.name = s->bucket_name;
+  if (!driver->is_meta_master()) {
+    // apply bucket creation on the master zone first
+    bufferlist in_data;
+    JSONParser jp;
+    op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr,
+                                               in_data, &jp, s->info, y);
+    if (op_ret < 0) {
+      return;
+    }
+
+    RGWBucketInfo master_info;
+    JSONDecoder::decode_json("bucket_info", master_info, &jp);
 
-  /* Handle updates of the metadata for Swift's object versioning. */
-  if (swift_ver_location) {
-    info.swift_ver_location = *swift_ver_location;
-    info.swift_versioning = (! swift_ver_location->empty());
+    // update params with info from the master
+    createparams.marker = master_info.bucket.marker;
+    createparams.bucket_id = master_info.bucket.bucket_id;
+    createparams.zonegroup_id = master_info.zonegroup;
+    createparams.obj_lock_enabled = master_info.obj_lock_enabled();
+    createparams.quota = master_info.quota;
+    createparams.creation_time = master_info.creation_time;
   }
 
-  /* We're replacing bucket with the newly created one */
-  ldpp_dout(this, 10) << "user=" << s->user << " bucket=" << tmp_bucket << dendl;
-  op_ret = s->user->create_bucket(this, tmp_bucket, zonegroup_id,
-				placement_rule,
-				info.swift_ver_location,
-				pquota_info, policy, attrs, info, ep_objv,
-				true, obj_lock_enabled, &s->bucket_exists, s->info,
-				&s->bucket, y);
+  ldpp_dout(this, 10) << "user=" << s->user << " bucket=" << s->bucket << dendl;
+  op_ret = s->bucket->create(this, createparams, y);
 
   /* continue if EEXIST and create_bucket will fail below.  this way we can
    * recover from a partial create by retrying it. */
-  ldpp_dout(this, 20) << "rgw_create_bucket returned ret=" << op_ret << " bucket=" << s->bucket.get() << dendl;
+  ldpp_dout(this, 20) << "Bucket::create() returned ret=" << op_ret << " bucket=" << s->bucket << dendl;
 
   if (op_ret)
     return;
@@ -3506,31 +3573,33 @@ void RGWCreateBucket::execute(optional_yield y)
         s->bucket_attrs = s->bucket->get_attrs();
       }
 
-      attrs.clear();
+      createparams.attrs.clear();
 
-      op_ret = rgw_get_request_metadata(this, s->cct, s->info, attrs, false);
+      op_ret = rgw_get_request_metadata(this, s->cct, s->info, createparams.attrs, false);
       if (op_ret < 0) {
         return;
       }
-      prepare_add_del_attrs(s->bucket_attrs, rmattr_names, attrs);
-      populate_with_generic_attrs(s, attrs);
-      op_ret = filter_out_quota_info(attrs, rmattr_names, s->bucket->get_info().quota);
+      prepare_add_del_attrs(s->bucket_attrs, rmattr_names, createparams.attrs);
+      populate_with_generic_attrs(s, createparams.attrs);
+      op_ret = filter_out_quota_info(createparams.attrs, rmattr_names,
+                                     s->bucket->get_info().quota);
       if (op_ret < 0) {
         return;
       }
 
       /* Handle updates of the metadata for Swift's object versioning. */
-      if (swift_ver_location) {
-        s->bucket->get_info().swift_ver_location = *swift_ver_location;
-        s->bucket->get_info().swift_versioning = (! swift_ver_location->empty());
+      if (createparams.swift_ver_location) {
+        s->bucket->get_info().swift_ver_location = *createparams.swift_ver_location;
+        s->bucket->get_info().swift_versioning = !createparams.swift_ver_location->empty();
       }
 
       /* Web site of Swift API. */
-      filter_out_website(attrs, rmattr_names, s->bucket->get_info().website_conf);
+      filter_out_website(createparams.attrs, rmattr_names,
+                         s->bucket->get_info().website_conf);
       s->bucket->get_info().has_website = !s->bucket->get_info().website_conf.is_empty();
 
       /* This will also set the quota on the bucket. */
-      op_ret = s->bucket->merge_and_store_attrs(this, attrs, y);
+      op_ret = s->bucket->merge_and_store_attrs(this, createparams.attrs, y);
     } while (op_ret == -ECANCELED && tries++ < 20);
 
     /* Restore the proper return code. */
@@ -7561,54 +7630,73 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y)
 {
   ldpp_dout(this, 20) << "got directory=" << path << dendl;
 
-  op_ret = handle_dir_verify_permission(y);
-  if (op_ret < 0) {
-    return op_ret;
+  int ret = handle_dir_verify_permission(y);
+  if (ret < 0) {
+    return ret;
   }
 
   std::string bucket_name;
   rgw_obj_key object_junk;
   std::tie(bucket_name, object_junk) =  *parse_path(path);
 
-  /* we need to make sure we read bucket info, it's not read before for this
-   * specific request */
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-
-  /* Create metadata: ACLs. */
-  std::map<std::string, ceph::bufferlist> attrs;
-  RGWAccessControlPolicy policy;
-  policy.create_default(s->user->get_id(), s->user->get_display_name());
-  ceph::bufferlist aclbl;
-  policy.encode(aclbl);
-  attrs.emplace(RGW_ATTR_ACL, std::move(aclbl));
-
-  obj_version objv, ep_objv;
-  bool bucket_exists;
-  RGWQuotaInfo quota_info;
-  const RGWQuotaInfo* pquota_info = nullptr;
-  RGWBucketInfo out_info;
-  string swift_ver_location;
   rgw_bucket new_bucket;
-  req_info info = s->info;
   new_bucket.tenant = s->bucket_tenant; /* ignored if bucket exists */
   new_bucket.name = bucket_name;
-  rgw_placement_rule placement_rule;
-  placement_rule.storage_class = s->info.storage_class;
-  forward_req_info(this, s->cct, info, bucket_name);
-
-  op_ret = s->user->create_bucket(this, new_bucket,
-                                driver->get_zone()->get_zonegroup().get_id(),
-                                placement_rule, swift_ver_location,
-                                pquota_info, policy, attrs,
-                                out_info, ep_objv,
-                                true, false, &bucket_exists,
-				info, &bucket, y);
-  /* continue if EEXIST and create_bucket will fail below.  this way we can
-   * recover from a partial create by retrying it. */
-  ldpp_dout(this, 20) << "rgw_create_bucket returned ret=" << op_ret
-      << ", bucket=" << bucket << dendl;
 
-  return op_ret;
+  // load the bucket
+  std::unique_ptr<rgw::sal::Bucket> bucket;
+  ret = driver->load_bucket(this, s->user.get(), new_bucket, &bucket, y);
+
+  // return success if it exists
+  if (ret != -ENOENT) {
+    return ret;
+  }
+  ceph_assert(bucket); // creates handle even on ENOENT
+
+  const auto& zonegroup = s->penv.site->get_zonegroup();
+
+  rgw::sal::Bucket::CreateParams createparams;
+  createparams.zonegroup_id = zonegroup.id;
+  createparams.placement_rule.storage_class = s->info.storage_class;
+  op_ret = select_bucket_placement(this, zonegroup, s->user->get_info(),
+                                   createparams.placement_rule);
+  createparams.zone_placement = rgw::find_zone_placement(
+      this, s->penv.site->get_zone_params(), createparams.placement_rule);
+
+  {
+    // create a default acl
+    RGWAccessControlPolicy policy;
+    policy.create_default(s->user->get_id(), s->user->get_display_name());
+    ceph::bufferlist aclbl;
+    policy.encode(aclbl);
+    createparams.attrs[RGW_ATTR_ACL] = std::move(aclbl);
+  }
+
+  if (!driver->is_meta_master()) {
+    // apply bucket creation on the master zone first
+    bufferlist in_data;
+    JSONParser jp;
+    req_info req = s->info;
+    forward_req_info(this, s->cct, req, bucket_name);
+
+    ret = driver->forward_request_to_master(this, s->user.get(), nullptr,
+                                            in_data, &jp, req, y);
+    if (ret < 0) {
+      return ret;
+    }
+
+    RGWBucketInfo master_info;
+    JSONDecoder::decode_json("bucket_info", master_info, &jp);
+
+    // update params with info from the master
+    createparams.marker = master_info.bucket.marker;
+    createparams.bucket_id = master_info.bucket.bucket_id;
+    createparams.obj_lock_enabled = master_info.obj_lock_enabled();
+    createparams.quota = master_info.quota;
+    createparams.creation_time = master_info.creation_time;
+  }
+
+  return bucket->create(this, createparams, y);
 }
 
 
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index e1189cb236d4..21044046680a 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1076,31 +1076,21 @@ class RGWStatBucket : public RGWOp {
 };
 
 class RGWCreateBucket : public RGWOp {
-protected:
+ protected:
+  rgw::sal::Bucket::CreateParams createparams;
   RGWAccessControlPolicy policy;
   std::string location_constraint;
-  rgw_placement_rule placement_rule;
-  RGWBucketInfo info;
-  obj_version ep_objv;
-  bool has_cors;
-  bool relaxed_region_enforcement;
-  bool obj_lock_enabled;
+  bool has_cors = false;
+  bool relaxed_region_enforcement = false;
   RGWCORSConfiguration cors_config;
-  boost::optional<std::string> swift_ver_location;
-  std::map<std::string, buffer::list> attrs;
   std::set<std::string> rmattr_names;
 
-  bufferlist in_data;
-
   virtual bool need_metadata_upload() const { return false; }
 
-public:
-  RGWCreateBucket() : has_cors(false), relaxed_region_enforcement(false), obj_lock_enabled(false) {}
-
+ public:
   void emplace_attr(std::string&& key, buffer::list&& bl) {
-    attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
+    createparams.attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
   }
-
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
@@ -1403,7 +1393,7 @@ class RGWPutMetadataBucket : public RGWOp {
   RGWAccessControlPolicy policy;
   RGWCORSConfiguration cors_config;
   rgw_placement_rule placement_rule;
-  boost::optional<std::string> swift_ver_location;
+  std::optional<std::string> swift_ver_location;
 
 public:
   RGWPutMetadataBucket()
diff --git a/src/rgw/rgw_quota_types.h b/src/rgw/rgw_quota_types.h
index f7d06657ed68..48d46dc62b43 100644
--- a/src/rgw/rgw_quota_types.h
+++ b/src/rgw/rgw_quota_types.h
@@ -28,8 +28,6 @@ static inline int64_t rgw_rounded_kb(int64_t bytes)
 class JSONObj;
 
 struct RGWQuotaInfo {
-  template<class T> friend class RGWQuotaCache;
-public:
   int64_t max_size;
   int64_t max_objects;
   bool enabled;
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6c3a8c8adc16..971b63e85d79 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2456,8 +2456,6 @@ int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y)
   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
     return op_ret;
 
-  in_data.append(data);
-
   if (data.length()) {
     RGWCreateBucketParser parser;
 
@@ -2486,17 +2484,18 @@ int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y)
 
   size_t pos = location_constraint.find(':');
   if (pos != string::npos) {
-    placement_rule.init(location_constraint.substr(pos + 1), s->info.storage_class);
+    createparams.placement_rule.init(location_constraint.substr(pos + 1),
+                                     s->info.storage_class);
     location_constraint = location_constraint.substr(0, pos);
   } else {
-    placement_rule.storage_class = s->info.storage_class;
+    createparams.placement_rule.storage_class = s->info.storage_class;
   }
   auto iter = s->info.x_meta_map.find("x-amz-bucket-object-lock-enabled");
   if (iter != s->info.x_meta_map.end()) {
     if (!boost::algorithm::iequals(iter->second, "true") && !boost::algorithm::iequals(iter->second, "false")) {
       return -EINVAL;
     }
-    obj_lock_enabled = boost::algorithm::iequals(iter->second, "true");
+    createparams.obj_lock_enabled = boost::algorithm::iequals(iter->second, "true");
   }
   return 0;
 }
@@ -2516,6 +2515,8 @@ void RGWCreateBucket_ObjStore_S3::send_response()
   if (s->system_request) {
     JSONFormatter f; /* use json formatter for system requests output */
 
+    const RGWBucketInfo& info = s->bucket->get_info();
+    const obj_version& ep_objv = s->bucket->get_version();
     f.open_object_section("info");
     encode_json("entry_point_object_ver", ep_objv, &f);
     encode_json("object_ver", info.objv_tracker.read_version, &f);
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 7ef73285583b..b941d4bb0992 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -682,14 +682,14 @@ static void get_rmattrs_from_headers(const req_state * const s,
 
 static int get_swift_versioning_settings(
   req_state * const s,
-  boost::optional<std::string>& swift_ver_location)
+  std::optional<std::string>& swift_ver_location)
 {
   /* Removing the Swift's versions location has lower priority than setting
    * a new one. That's the reason why we're handling it first. */
   const std::string vlocdel =
     s->info.env->get("HTTP_X_REMOVE_VERSIONS_LOCATION", "");
   if (vlocdel.size()) {
-    swift_ver_location = boost::in_place(std::string());
+    swift_ver_location.emplace();
   }
 
   if (s->info.env->exists("HTTP_X_VERSIONS_LOCATION")) {
@@ -724,9 +724,10 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params(optional_yield y)
   location_constraint = driver->get_zone()->get_zonegroup().get_api_name();
   get_rmattrs_from_headers(s, CONT_PUT_ATTR_PREFIX,
                            CONT_REMOVE_ATTR_PREFIX, rmattr_names);
-  placement_rule.init(s->info.env->get("HTTP_X_STORAGE_POLICY", ""), s->info.storage_class);
+  createparams.placement_rule.init(s->info.env->get("HTTP_X_STORAGE_POLICY", ""),
+                                   s->info.storage_class);
 
-  return get_swift_versioning_settings(s, swift_ver_location);
+  return get_swift_versioning_settings(s, createparams.swift_ver_location);
 }
 
 static inline int handle_metadata_errors(req_state* const s, const int op_ret)
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index d69942532881..d449535c3986 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -39,6 +39,7 @@ typedef std::shared_ptr<RGWSyncModuleInstance> RGWSyncModuleInstanceRef;
 class RGWCompressionInfo;
 struct rgw_pubsub_topics;
 struct rgw_pubsub_bucket_topics;
+class RGWZonePlacementInfo;
 
 
 using RGWBucketListNameFilter = std::function<bool (const std::string&)>;
@@ -487,23 +488,6 @@ class User {
 			     const std::string& marker, const std::string& end_marker,
 			     uint64_t max, bool need_stats, BucketList& buckets,
 			     optional_yield y) = 0;
-    /** Create a new bucket owned by this user.  Creates in the backing store, not just the instantiation. */
-    virtual int create_bucket(const DoutPrefixProvider* dpp,
-                            const rgw_bucket& b,
-                            const std::string& zonegroup_id,
-                            const rgw_placement_rule& placement_rule,
-                            const std::string& swift_ver_location,
-                            const RGWQuotaInfo* pquota_info,
-                            const RGWAccessControlPolicy& policy,
-			    Attrs& attrs,
-                            RGWBucketInfo& info,
-                            obj_version& ep_objv,
-			    bool exclusive,
-			    bool obj_lock_enabled,
-			    bool* existed,
-			    req_info& req_info,
-			    std::unique_ptr<Bucket>* bucket,
-			    optional_yield y) = 0;
 
     /** Get the display name for this User */
     virtual std::string& get_display_name() = 0;
@@ -674,6 +658,27 @@ class Bucket {
     // XXXX hack
     virtual void set_owner(rgw::sal::User* _owner) = 0;
 
+    /// Input parameters for create().
+    struct CreateParams {
+      std::string zonegroup_id;
+      rgw_placement_rule placement_rule;
+      // zone placement is optional on buckets created for another zonegroup
+      const RGWZonePlacementInfo* zone_placement;
+      RGWAccessControlPolicy policy;
+      Attrs attrs;
+      bool obj_lock_enabled = false;
+      std::string marker;
+      std::string bucket_id;
+      std::optional<std::string> swift_ver_location;
+      std::optional<RGWQuotaInfo> quota;
+      std::optional<ceph::real_time> creation_time;
+    };
+
+    /// Create this bucket in the backing store.
+    virtual int create(const DoutPrefixProvider* dpp,
+                       const CreateParams& params,
+                       optional_yield y) = 0;
+
     /** Load this bucket from the backing store.  Requires the key to be set, fills other fields. */
     virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) = 0;
     /** Read the bucket stats from the backing Store, synchronous */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 792fa457f895..c8046a65a00d 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -58,90 +58,22 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBUser::create_bucket(const DoutPrefixProvider *dpp,
-      const rgw_bucket& b,
-      const string& zonegroup_id,
-      const rgw_placement_rule& placement_rule,
-      const string& swift_ver_location,
-      const RGWQuotaInfo * pquota_info,
-      const RGWAccessControlPolicy& policy,
-      Attrs& attrs,
-      RGWBucketInfo& info,
-      obj_version& ep_objv,
-      bool exclusive,
-      bool obj_lock_enabled,
-      bool *existed,
-      req_info& req_info,
-      std::unique_ptr<Bucket>* bucket_out,
-      optional_yield y)
-  {
-    int ret;
-    bufferlist in_data;
-    RGWBucketInfo master_info;
-    rgw_bucket *pmaster_bucket = nullptr;
-    real_time creation_time;
-    std::unique_ptr<Bucket> bucket;
-    obj_version objv, *pobjv = NULL;
-
-    /* If it exists, look it up; otherwise create it */
-    ret = store->load_bucket(dpp, this, b, &bucket, y);
-    if (ret < 0 && ret != -ENOENT)
-      return ret;
-
-    if (ret != -ENOENT) {
-      *existed = true;
-    } else {
-      bucket = std::make_unique<DBBucket>(store, b, this);
-      *existed = false;
-      bucket->set_attrs(attrs);
-    }
-
-    /*
-     * XXX: If not master zone, fwd the request to master zone.
-     * For now DBStore has single zone.
-     */
-    std::string zid = zonegroup_id;
-    /* if (zid.empty()) {
-       zid = svc()->zone->get_zonegroup().get_id();
-       } */
-
-    // XXX: For now single default zone and STANDARD storage class
-    // supported.
-    rgw_placement_rule selected_placement_rule;
-    selected_placement_rule.name = "default";
-    selected_placement_rule.storage_class = "STANDARD";
-
-    if (*existed) {
-      /* XXX: Handle this when zone is implemented
-         ret = svc()->zone->select_bucket_placement(this.get_info(),
-         zid, placement_rule,
-         &selected_placement_rule, nullptr, y);
-         if (selected_placement_rule != info.placement_rule) {
-         ret = -EEXIST;
-         bucket_out->swap(bucket);
-         return ret;
-         } */
-    } else {
-
-      /* XXX: We may not need to send all these params. Cleanup the unused ones */
-      ret = store->getDB()->create_bucket(dpp, this->get_info(), bucket->get_key(),
-          zid, selected_placement_rule, swift_ver_location, pquota_info,
-          attrs, info, pobjv, &ep_objv, creation_time,
-          pmaster_bucket, nullptr, y, exclusive);
-      if (ret == -EEXIST) {
-        *existed = true;
-        ret = 0;
-      } else if (ret != 0) {
-        return ret;
-      }
-    }
-
-    bucket->set_version(ep_objv);
-    bucket->get_info() = info;
-
-    bucket_out->swap(bucket);
-
-    return ret;
+  int DBBucket::create(const DoutPrefixProvider *dpp,
+                       const CreateParams& params,
+                       optional_yield y)
+  {
+    ceph_assert(owner);
+    const rgw_user& owner_id = owner->get_id();
+
+    rgw_bucket key = get_key();
+    key.marker = params.marker;
+    key.bucket_id = params.bucket_id;
+
+    /* XXX: We may not need to send all these params. Cleanup the unused ones */
+    return store->getDB()->create_bucket(dpp, owner_id, key,
+        params.zonegroup_id, params.placement_rule, params.attrs,
+        params.swift_ver_location, params.quota, params.creation_time,
+        &bucket_version, info, y);
   }
 
   int DBUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y)
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 41a0b89ea502..83f3ff965782 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -88,22 +88,6 @@ class DBNotification : public StoreNotification {
       }
       int list_buckets(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& end_marker,
           uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override;
-      virtual int create_bucket(const DoutPrefixProvider* dpp,
-          const rgw_bucket& b,
-          const std::string& zonegroup_id,
-          const rgw_placement_rule& placement_rule,
-          const std::string& swift_ver_location,
-          const RGWQuotaInfo* pquota_info,
-          const RGWAccessControlPolicy& policy,
-          Attrs& attrs,
-          RGWBucketInfo& info,
-          obj_version& ep_objv,
-          bool exclusive,
-          bool obj_lock_enabled,
-          bool* existed,
-          req_info& req_info,
-          std::unique_ptr<Bucket>* bucket,
-          optional_yield y) override;
       virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
       virtual int read_stats(const DoutPrefixProvider *dpp,
           optional_yield y, RGWStorageStats* stats,
@@ -178,6 +162,9 @@ class DBNotification : public StoreNotification {
 					DoutPrefixProvider *dpp) override;
       virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
       virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
+      int create(const DoutPrefixProvider* dpp,
+                 const CreateParams& params,
+                 optional_yield y) override;
       virtual int load_bucket(const DoutPrefixProvider *dpp, optional_yield y) override;
       virtual int read_stats(const DoutPrefixProvider *dpp,
 			     const bucket_index_layout_generation& idx_layout,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index c70d9e372dc3..01a113513672 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -508,35 +508,6 @@ int FilterUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& m
                             need_stats, buckets, y);
 }
 
-int FilterUser::create_bucket(const DoutPrefixProvider* dpp,
-			      const rgw_bucket& b,
-			      const std::string& zonegroup_id,
-			      const rgw_placement_rule& placement_rule,
-			      const std::string& swift_ver_location,
-			      const RGWQuotaInfo * pquota_info,
-			      const RGWAccessControlPolicy& policy,
-			      Attrs& attrs,
-			      RGWBucketInfo& info,
-			      obj_version& ep_objv,
-			      bool exclusive,
-			      bool obj_lock_enabled,
-			      bool* existed,
-			      req_info& req_info,
-			      std::unique_ptr<Bucket>* bucket_out,
-			      optional_yield y)
-{
-  std::unique_ptr<Bucket> nb;
-  int ret;
-
-  ret = next->create_bucket(dpp, b, zonegroup_id, placement_rule, swift_ver_location, pquota_info, policy, attrs, info, ep_objv, exclusive, obj_lock_enabled, existed, req_info, &nb, y);
-  if (ret < 0)
-    return ret;
-
-  Bucket* fb = new FilterBucket(std::move(nb), this);
-  bucket_out->reset(fb);
-  return 0;
-}
-
 int FilterUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y)
 {
   return next->read_attrs(dpp, y);
@@ -636,6 +607,13 @@ int FilterBucket::set_acl(const DoutPrefixProvider* dpp,
   return next->set_acl(dpp, acl, y);
 }
 
+int FilterBucket::create(const DoutPrefixProvider* dpp,
+                         const CreateParams& params,
+                         optional_yield y)
+{
+  return next->create(dpp, params, y);
+}
+
 int FilterBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y)
 {
   return next->load_bucket(dpp, y);
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index b2aa1be399f3..1711723bdd26 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -325,22 +325,6 @@ class FilterUser : public User {
 			   const std::string& marker, const std::string& end_marker,
 			   uint64_t max, bool need_stats, BucketList& buckets,
 			   optional_yield y) override;
-  virtual int create_bucket(const DoutPrefixProvider* dpp,
-			    const rgw_bucket& b,
-			    const std::string& zonegroup_id,
-			    const rgw_placement_rule& placement_rule,
-			    const std::string& swift_ver_location,
-			    const RGWQuotaInfo* pquota_info,
-			    const RGWAccessControlPolicy& policy,
-			    Attrs& attrs,
-			    RGWBucketInfo& info,
-			    obj_version& ep_objv,
-			    bool exclusive,
-			    bool obj_lock_enabled,
-			    bool* existed,
-			    req_info& req_info,
-			    std::unique_ptr<Bucket>* bucket,
-			    optional_yield y) override;
 
   virtual std::string& get_display_name() override { return next->get_display_name(); }
   virtual const std::string& get_tenant() override { return next->get_tenant(); }
@@ -419,6 +403,9 @@ class FilterBucket : public Bucket {
 		      optional_yield y) override;
 
   virtual void set_owner(rgw::sal::User* _owner) override { next->set_owner(_owner); }
+  virtual int create(const DoutPrefixProvider* dpp,
+		     const CreateParams& params,
+		     optional_yield y) override;
   virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual int read_stats(const DoutPrefixProvider *dpp,
 			 const bucket_index_layout_generation& idx_layout,
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index 7ceb7092c64f..2d39981806c8 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -114,43 +114,17 @@ class D4NFilterFixture : public ::testing::Test {
     }
 
     int createBucket() {
-      rgw_bucket b;
-      string zonegroup_id = "test_id";
-      rgw_placement_rule placement_rule;
-      string swift_ver_location = "test_location";
-      const RGWAccessControlPolicy policy;
-      rgw::sal::Attrs attrs;
       RGWBucketInfo info;
-      obj_version ep_objv;
-      bool bucket_exists;
-      int ret;
-      
-      CephContext* cct = get_pointer(env->cct);
-      RGWProcessEnv penv;
-      RGWEnv rgw_env;
-      req_state s(cct->get(), penv, &rgw_env, 0);
-      req_info _req_info = s.info;
-
-      b.name = "test_bucket";
-      placement_rule.storage_class = "test_sc";
-
-      ret = testUser->create_bucket(dpp, b,
-	    zonegroup_id,
-	    placement_rule,
-	    swift_ver_location,
-	    nullptr,
-	    policy,
-	    attrs,
-	    info,
-	    ep_objv,
-	    false,
-	    false,
-	    &bucket_exists,
-	    _req_info,
-	    &testBucket,
-	    null_yield);
-	
-      return ret;
+      info.bucket.name = "test_bucket";
+
+      testBucket = driver->get_bucket(testUser.get(), info);
+
+      rgw::sal::Bucket::CreateParams params;
+      params.zonegroup_id = "test_id";
+      params.placement_rule.storage_class = "test_sc";
+      params.swift_ver_location = "test_location";
+
+      return testBucket->create(dpp, params, null_yield);
     }
 
     int putObject(string name) {
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 05f0d291961a..8502ce81ef5a 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -78,10 +78,6 @@ class TestUser : public sal::StoreUser {
     return 0;
   }
 
-  virtual int create_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, const std::string& zonegroup_id, const rgw_placement_rule& placement_rule, const std::string& swift_ver_location, const RGWQuotaInfo* pquota_info, const RGWAccessControlPolicy& policy, sal::Attrs& attrs, RGWBucketInfo& info, obj_version& ep_objv, bool exclusive, bool obj_lock_enabled, bool* existed, req_info& req_info, std::unique_ptr<sal::Bucket>* bucket, optional_yield y) override {
-    return 0;
-  }
-
   virtual int read_attrs(const DoutPrefixProvider *dpp, optional_yield y) override {
     return 0;
   }

From 5e344c7c51e06e26bf3f06252d1b306ebb16dda8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 3 Oct 2023 14:13:24 -0400
Subject: [PATCH 0503/2492] rgw/sal: remove ZoneGroup::get_endpoint()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc   | 13 -------------
 src/rgw/driver/daos/rgw_sal_daos.h    |  2 --
 src/rgw/driver/motr/rgw_sal_motr.cc   | 14 --------------
 src/rgw/driver/motr/rgw_sal_motr.h    |  2 --
 src/rgw/driver/rados/rgw_sal_rados.cc | 14 --------------
 src/rgw/driver/rados/rgw_sal_rados.h  |  2 --
 src/rgw/rgw_sal.h                     |  2 --
 src/rgw/rgw_sal_dbstore.cc            | 13 -------------
 src/rgw/rgw_sal_dbstore.h             |  2 --
 src/rgw/rgw_sal_filter.h              |  2 --
 10 files changed, 66 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index a6aacf024b31..8be268fe17cc 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -800,19 +800,6 @@ int DaosStore::initialize(CephContext* cct, const DoutPrefixProvider* dpp) {
   return ret;
 }
 
-const std::string& DaosZoneGroup::get_endpoint() const {
-  if (!group.endpoints.empty()) {
-    return group.endpoints.front();
-  } else {
-    // use zonegroup's master zone endpoints
-    auto z = group.zones.find(group.master_zone);
-    if (z != group.zones.end() && !z->second.endpoints.empty()) {
-      return z->second.endpoints.front();
-    }
-  }
-  return empty;
-}
-
 bool DaosZoneGroup::placement_target_exists(std::string& target) const {
   return !!group.placement_targets.count(target);
 }
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 72d3bdb6cfbe..4d5e6c831079 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -402,8 +402,6 @@ class DaosZoneGroup : public StoreZoneGroup {
   virtual int equals(const std::string& other_zonegroup) const override {
     return group.equals(other_zonegroup);
   };
-  /** Get the endpoint from zonegroup, or from master zone if not set */
-  virtual const std::string& get_endpoint() const override;
   virtual bool placement_target_exists(std::string& target) const override;
   virtual bool is_master_zonegroup() const override {
     return group.is_master_zonegroup();
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index b53516bb0653..d6a5eb1bc3ed 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -1047,20 +1047,6 @@ void MotrStore::finalize(void)
   m0_client_fini(this->instance, true);
 }
 
-const std::string& MotrZoneGroup::get_endpoint() const
-{
-  if (!group.endpoints.empty()) {
-      return group.endpoints.front();
-  } else {
-    // use zonegroup's master zone endpoints
-    auto z = group.zones.find(group.master_zone);
-    if (z != group.zones.end() && !z->second.endpoints.empty()) {
-      return z->second.endpoints.front();
-    }
-  }
-  return empty;
-}
-
 bool MotrZoneGroup::placement_target_exists(std::string& target) const
 {
   return !!group.placement_targets.count(target);
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 7935462ac3f0..66ca8314fc0b 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -440,8 +440,6 @@ class MotrZoneGroup : public StoreZoneGroup {
   virtual int equals(const std::string& other_zonegroup) const override {
     return group.equals(other_zonegroup);
   };
-  /** Get the endpoint from zonegroup, or from master zone if not set */
-  virtual const std::string& get_endpoint() const override;
   virtual bool placement_target_exists(std::string& target) const override;
   virtual bool is_master_zonegroup() const override {
     return group.is_master_zonegroup();
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index fd8964e83ba1..68b6d998eca4 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2980,20 +2980,6 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta
 			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
 }
 
-const std::string& RadosZoneGroup::get_endpoint() const
-{
-  if (!group.endpoints.empty()) {
-      return group.endpoints.front();
-  } else {
-    // use zonegroup's master zone endpoints
-    auto z = group.zones.find(group.master_zone);
-    if (z != group.zones.end() && !z->second.endpoints.empty()) {
-      return z->second.endpoints.front();
-    }
-  }
-  return empty;
-}
-
 bool RadosZoneGroup::placement_target_exists(std::string& target) const
 {
   return !!group.placement_targets.count(target);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index c8fa876260bf..c46ba8e68f0b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -55,8 +55,6 @@ class RadosZoneGroup : public StoreZoneGroup {
   virtual int equals(const std::string& other_zonegroup) const override {
     return group.equals(other_zonegroup);
   };
-  /** Get the endpoint from zonegroup, or from master zone if not set */
-  virtual const std::string& get_endpoint() const override;
   virtual bool placement_target_exists(std::string& target) const override;
   virtual bool is_master_zonegroup() const override {
     return group.is_master_zonegroup();
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index d449535c3986..a4b0219dd411 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1431,8 +1431,6 @@ class ZoneGroup {
   virtual const std::string& get_name() const = 0;
   /** Determine if two zonegroups are the same */
   virtual int equals(const std::string& other_zonegroup) const = 0;
-  /** Get the endpoint from zonegroup, or from master zone if not set */
-  virtual const std::string& get_endpoint() const = 0;
   /** Check if a placement target (by name) exists in this zonegroup */
   virtual bool placement_target_exists(std::string& target) const = 0;
   /** Check if this is the master zonegroup */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index c8046a65a00d..47fa0f06ba75 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -428,19 +428,6 @@ namespace rgw::sal {
       dbsm->destroyAllHandles();
   }
 
-  const std::string&  DBZoneGroup::get_endpoint() const {
-    if (!group->endpoints.empty()) {
-      return group->endpoints.front();
-    } else {
-      // use zonegroup's master zone endpoints
-      auto z = group->zones.find(group->master_zone);
-      if (z != group->zones.end() && !z->second.endpoints.empty()) {
-	return z->second.endpoints.front();
-      }
-    }
-    return empty;
-  }
-
   bool DBZoneGroup::placement_target_exists(std::string& target) const {
     return !!group->placement_targets.count(target);
   }
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 83f3ff965782..7ab98f6286b4 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -239,8 +239,6 @@ class DBNotification : public StoreNotification {
     virtual int equals(const std::string& other_zonegroup) const override {
       return group->equals(other_zonegroup);
     };
-    /** Get the endpoint from zonegroup, or from master zone if not set */
-    virtual const std::string& get_endpoint() const override;
     virtual bool placement_target_exists(std::string& target) const override;
     virtual bool is_master_zonegroup() const override {
       return group->is_master_zonegroup();
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 1711723bdd26..aec8cd64c2ca 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -50,8 +50,6 @@ class FilterZoneGroup : public ZoneGroup {
     { return next->get_name(); }
   virtual int equals(const std::string& other_zonegroup) const override
     { return next->equals(other_zonegroup); }
-  virtual const std::string& get_endpoint() const override
-    { return next->get_endpoint(); }
   virtual bool placement_target_exists(std::string& target) const override
     { return next->placement_target_exists(target); }
   virtual bool is_master_zonegroup() const override

From 0b737d59cc0dac58f68ed900b8113647284c8824 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 14:25:32 -0400
Subject: [PATCH 0504/2492] rgw: forward_request takes const req_info

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_common.cc      | 2 +-
 src/rgw/rgw_common.h       | 2 +-
 src/rgw/rgw_rest_client.cc | 6 +++---
 src/rgw/rgw_rest_client.h  | 2 +-
 src/rgw/rgw_rest_conn.cc   | 4 ++--
 src/rgw/rgw_rest_conn.h    | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 536f48d43f26..a8a4dea5c8a9 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -251,7 +251,7 @@ req_info::req_info(CephContext *cct, const class RGWEnv *env) : env(env) {
   }
 }
 
-void req_info::rebuild_from(req_info& src)
+void req_info::rebuild_from(const req_info& src)
 {
   method = src.method;
   script_uri = src.script_uri;
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 1094d0cd443e..5165f9008b9f 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1073,7 +1073,7 @@ struct req_info {
   std::string storage_class;
 
   req_info(CephContext *cct, const RGWEnv *env);
-  void rebuild_from(req_info& src);
+  void rebuild_from(const req_info& src);
   void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta);
 };
 
diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index b0b8fcc84f30..1ccb813a1098 100644
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -363,7 +363,7 @@ static void scope_from_api_name(const DoutPrefixProvider *dpp,
   }
 }
 
-int RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y, std::string service)
+int RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y, std::string service)
 {
 
   string date_str;
@@ -413,7 +413,7 @@ int RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const R
   }
 
   if (s == "iam") {
-    info.args.remove("PayloadHash");
+    new_info.args.remove("PayloadHash");
   }
 
   for (const auto& kv: new_env.get_map()) {
@@ -426,7 +426,7 @@ int RGWRESTSimpleRequest::forward_request(const DoutPrefixProvider *dpp, const R
   }
 
   string params_str;
-  get_params_str(info.args.get_params(), params_str);
+  get_params_str(new_info.args.get_params(), params_str);
 
   string new_url = url;
   string& resource = new_info.request_uri;
diff --git a/src/rgw/rgw_rest_client.h b/src/rgw/rgw_rest_client.h
index 97cf899fdb53..6e27576c6768 100644
--- a/src/rgw/rgw_rest_client.h
+++ b/src/rgw/rgw_rest_client.h
@@ -65,7 +65,7 @@ class RGWRESTSimpleRequest : public RGWHTTPSimpleRequest {
                        param_vec_t *_headers, param_vec_t *_params,
                        std::optional<std::string> _api_name) : RGWHTTPSimpleRequest(_cct, _method, _url, _headers, _params), api_name(_api_name) {}
 
-  int forward_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y, std::string service="");
+  int forward_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y, std::string service="");
 };
 
 class RGWWriteDrainCB {
diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 44663078d284..a9a6bc7c56cb 100644
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -91,7 +91,7 @@ void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, cons
   populate_zonegroup(params, zonegroup);
 }
 
-int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
+int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 {
   string url;
   int ret = get_url(url);
@@ -109,7 +109,7 @@ int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req
   return req.forward_request(dpp, key, info, max_response, inbl, outbl, y);
 }
 
-int RGWRESTConn::forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
+int RGWRESTConn::forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 {
   string url;
   int ret = get_url(url);
diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 81f839f49a23..4e3c88af219b 100644
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -125,10 +125,10 @@ class RGWRESTConn
   virtual void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup);
 
   /* sync request */
-  int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
+  int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 
   /* sync request */
-  int forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
+  int forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 
 
   /* async requests */

From d2dbe7550296da6db885b5344c71f77f9acbfd8f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 16:55:58 -0400
Subject: [PATCH 0505/2492] rgw: add free function
 rgw_forward_request_to_master()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rest_bucket.cc |  13 +--
 src/rgw/driver/rados/rgw_rest_user.cc   |  77 +++++++-------
 src/rgw/driver/rados/rgw_zone.h         |   4 +
 src/rgw/rgw_op.cc                       | 128 +++++++++++++++++-------
 src/rgw/rgw_op.h                        |  15 ++-
 src/rgw/rgw_rest_user_policy.cc         |   9 +-
 6 files changed, 151 insertions(+), 95 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index 6853173be4ff..a530d4056735 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -3,6 +3,7 @@
 
 #include "rgw_op.h"
 #include "driver/rados/rgw_bucket.h"
+#include "rgw_process_env.h"
 #include "rgw_rest_bucket.h"
 #include "rgw_sal.h"
 
@@ -150,8 +151,8 @@ void RGWOp_Bucket_Link::execute(optional_yield y)
   op_state.set_bucket_id(bucket_id);
   op_state.set_new_bucket_name(new_bucket_name);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -188,8 +189,8 @@ void RGWOp_Bucket_Unlink::execute(optional_yield y)
   op_state.set_user_id(uid);
   op_state.set_bucket_name(bucket);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -220,8 +221,8 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
   RESTArgs::get_string(s, "bucket", bucket_name, &bucket_name);
   RESTArgs::get_bool(s, "purge-objects", false, &delete_children);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     if (op_ret == -ENOENT) {
diff --git a/src/rgw/driver/rados/rgw_rest_user.cc b/src/rgw/driver/rados/rgw_rest_user.cc
index 361ceb0f70fc..200f1c03d2e1 100644
--- a/src/rgw/driver/rados/rgw_rest_user.cc
+++ b/src/rgw/driver/rados/rgw_rest_user.cc
@@ -5,6 +5,7 @@
 
 #include "rgw_op.h"
 #include "rgw_user.h"
+#include "rgw_process_env.h"
 #include "rgw_rest_user.h"
 #include "rgw_sal.h"
 
@@ -19,19 +20,23 @@
 
 using namespace std;
 
-int fetch_access_keys_from_master(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, RGWUserAdminOpState &op_state, req_state *s, optional_yield y) {
-    bufferlist data;
-    JSONParser jp;
-    RGWUserInfo ui;
-    int op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, &jp, s->info, y);
-    if (op_ret < 0) {
-      ldpp_dout(dpp, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
-      return op_ret;
-    }
-    ui.decode_json(&jp);
-    op_state.op_access_keys = std::move(ui.access_keys);
+int fetch_access_keys_from_master(const DoutPrefixProvider* dpp, req_state* s,
+                                  std::map<std::string, RGWAccessKey>& keys,
+                                  optional_yield y)
+{
+  bufferlist data;
+  JSONParser jp;
+  int ret = rgw_forward_request_to_master(dpp, *s->penv.site, s->user->get_id(),
+                                          &data, &jp, s->info, y);
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << "forward_request_to_master returned ret=" << ret << dendl;
+    return ret;
+  }
 
-    return 0;
+  RGWUserInfo ui;
+  ui.decode_json(&jp);
+  keys = std::move(ui.access_keys);
+  return 0;
 }
 
 class RGWOp_User_List : public RGWRESTOp {
@@ -238,15 +243,13 @@ void RGWOp_User_Create::execute(optional_yield y)
     op_state.set_placement_tags(placement_tags_list);
   }
 
-  if(!(driver->is_meta_master())) {
-    op_ret = fetch_access_keys_from_master(this, driver, op_state, s, y);
-
-    if(op_ret < 0) {
+  if (!s->penv.site->is_meta_master()) {
+    op_ret = fetch_access_keys_from_master(this, s, op_state.op_access_keys, y);
+    if (op_ret < 0) {
       return;
-    } else {
-      // set_generate_key() is not set if keys have already been fetched from master zone
-      gen_key = false;
     }
+    // set_generate_key() is not set if keys have already been fetched from master zone
+    gen_key = false;
   }
 
   if (gen_key) {
@@ -384,15 +387,13 @@ void RGWOp_User_Modify::execute(optional_yield y)
     op_state.set_placement_tags(placement_tags_list);
   }
   
-  if(!(driver->is_meta_master())) {
-    op_ret = fetch_access_keys_from_master(this, driver, op_state, s, y);
-
-    if(op_ret < 0) {
+  if (!s->penv.site->is_meta_master()) {
+    op_ret = fetch_access_keys_from_master(this, s, op_state.op_access_keys, y);
+    if (op_ret < 0) {
       return;
-    } else {
-      // set_generate_key() is not set if keys have already been fetched from master zone
-      gen_key = false;
     }
+    // set_generate_key() is not set if keys have already been fetched from master zone
+    gen_key = false;
   }
 
   if (gen_key) {
@@ -434,8 +435,8 @@ void RGWOp_User_Remove::execute(optional_yield y)
 
   op_state.set_purge_data(purge_data);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -509,8 +510,8 @@ void RGWOp_Subuser_Create::execute(optional_yield y)
   }
   op_state.set_key_type(key_type);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -576,8 +577,8 @@ void RGWOp_Subuser_Modify::execute(optional_yield y)
   }
   op_state.set_key_type(key_type);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -619,8 +620,8 @@ void RGWOp_Subuser_Remove::execute(optional_yield y)
   if (purge_keys)
     op_state.set_purge_keys();
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -760,8 +761,8 @@ void RGWOp_Caps_Add::execute(optional_yield y)
   op_state.set_user_id(uid);
   op_state.set_caps(caps);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -798,8 +799,8 @@ void RGWOp_Caps_Remove::execute(optional_yield y)
   op_state.set_user_id(uid);
   op_state.set_caps(caps);
 
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(s, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index b57b05ada857..4c6b82a29952 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -965,6 +965,10 @@ class SiteConfig {
   const RGWZoneGroup& get_zonegroup() const { return *zonegroup; }
   /// Return the public zone configuration.
   const RGWZone& get_zone() const { return *zone; }
+  /// Return true if the local zone can write metadata.
+  bool is_meta_master() const {
+    return zonegroup->is_master && zonegroup->master_zone == zone->id;
+  }
 
   /// Load or reload the multisite configuration from storage. This is not
   /// thread-safe, so requires careful coordination with the RGWRealmReloader.
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 1bd1fd85ba0c..1fbd2ec7105a 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -110,6 +110,47 @@ static constexpr auto S3_EXISTING_OBJTAG = "s3:ExistingObjectTag";
 static constexpr auto S3_RESOURCE_TAG = "s3:ResourceTag";
 static constexpr auto S3_RUNTIME_RESOURCE_VAL = "${s3:ResourceTag";
 
+int rgw_forward_request_to_master(const DoutPrefixProvider* dpp,
+                                  const rgw::SiteConfig& site,
+                                  const rgw_user& uid,
+                                  bufferlist* indata, JSONParser* jp,
+                                  req_info& req, optional_yield y)
+{
+  const auto& period = site.get_period();
+  if (!period) {
+    return 0; // not multisite
+  }
+  if (site.is_meta_master()) {
+    return 0; // don't need to forward metadata requests
+  }
+  const auto& pmap = period->period_map;
+  auto zg = pmap.zonegroups.find(pmap.master_zonegroup);
+  if (zg == pmap.zonegroups.end()) {
+    return -EINVAL;
+  }
+  auto z = zg->second.zones.find(zg->second.master_zone);
+  if (z == zg->second.zones.end()) {
+    return -EINVAL;
+  }
+  const RGWAccessKey& creds = site.get_zone_params().system_key;
+
+  // use the master zone's endpoints
+  auto conn = RGWRESTConn{dpp->get_cct(), z->second.id, z->second.endpoints,
+                          creds, zg->second.id, zg->second.api_name};
+  bufferlist outdata;
+  constexpr size_t max_response_size = 128 * 1024; // we expect a very small response
+  int ret = conn.forward(dpp, uid, req, nullptr, max_response_size,
+                         indata, &outdata, y);
+  if (ret < 0) {
+    return ret;
+  }
+  if (jp && !jp->parse(outdata.c_str(), outdata.length())) {
+    ldpp_dout(dpp, 0) << "failed parsing response from master zonegroup" << dendl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
 int RGWGetObj::parse_range(void)
 {
   int r = -ERANGE;
@@ -1215,9 +1256,11 @@ void RGWPutBucketTags::execute(optional_yield y)
   if (op_ret < 0) 
     return;
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
+    return;
   }
 
   op_ret = retry_raced_bucket_write(this, s->bucket.get(), [this, y] {
@@ -1244,8 +1287,8 @@ int RGWDeleteBucketTags::verify_permission(optional_yield y)
 
 void RGWDeleteBucketTags::execute(optional_yield y)
 {
-  bufferlist in_data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -1300,7 +1343,8 @@ void RGWPutBucketReplication::execute(optional_yield y) {
   if (op_ret < 0) 
     return;
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -1341,8 +1385,8 @@ int RGWDeleteBucketReplication::verify_permission(optional_yield y)
 
 void RGWDeleteBucketReplication::execute(optional_yield y)
 {
-  bufferlist in_data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -2781,7 +2825,8 @@ void RGWSetBucketVersioning::execute(optional_yield y)
     }
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -2870,7 +2915,8 @@ void RGWSetBucketWebsite::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << " forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -2911,9 +2957,8 @@ void RGWDeleteBucketWebsite::execute(optional_yield y)
     return;
   }
 
-  bufferlist in_data;
-
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "NOTICE: forward_to_master failed on bucket=" << s->bucket->get_name()
       << "returned err=" << op_ret << dendl;
@@ -3524,8 +3569,8 @@ void RGWCreateBucket::execute(optional_yield y)
     // apply bucket creation on the master zone first
     bufferlist in_data;
     JSONParser jp;
-    op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr,
-                                               in_data, &jp, s->info, y);
+    op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                           &in_data, &jp, s->info, y);
     if (op_ret < 0) {
       return;
     }
@@ -3669,8 +3714,8 @@ void RGWDeleteBucket::execute(optional_yield y)
     return;
   }
 
-  bufferlist in_data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), &ot.read_version, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
       /* adjust error, we want to return with NoSuchBucket and not
@@ -6038,7 +6083,8 @@ void RGWPutACLs::execute(optional_yield y)
     if (s->canned_acl.empty()) {
       in_data.append(data);
     }
-    op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+    op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                           &in_data, nullptr, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
       return;
@@ -6167,7 +6213,8 @@ void RGWPutLC::execute(optional_yield y)
     ldpp_dout(this, 15) << "New LifecycleConfiguration:" << ss.str() << dendl;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -6182,8 +6229,8 @@ void RGWPutLC::execute(optional_yield y)
 
 void RGWDeleteLC::execute(optional_yield y)
 {
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -6235,7 +6282,8 @@ void RGWPutCORS::execute(optional_yield y)
   if (op_ret < 0)
     return;
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -6260,8 +6308,8 @@ int RGWDeleteCORS::verify_permission(optional_yield y)
 
 void RGWDeleteCORS::execute(optional_yield y)
 {
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -6383,7 +6431,8 @@ void RGWSetRequestPayment::execute(optional_yield y)
   if (op_ret < 0)
     return;
   
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &in_data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -7423,9 +7472,8 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
       req_info req = s->info;
       forward_req_info(dpp, s->cct, req, path.bucket_name);
 
-      bufferlist data;
-      ret = driver->forward_request_to_master(dpp, s->user.get(), nullptr,
-                                              data, nullptr, req, y);
+      ret = rgw_forward_request_to_master(dpp, *s->penv.site, s->user->get_id(),
+                                          nullptr, nullptr, req, y);
       if (ret < 0) {
         goto delop_fail;
       }
@@ -7679,8 +7727,8 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y)
     req_info req = s->info;
     forward_req_info(this, s->cct, req, bucket_name);
 
-    ret = driver->forward_request_to_master(this, s->user.get(), nullptr,
-                                            in_data, &jp, req, y);
+    ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                        &in_data, &jp, req, y);
     if (ret < 0) {
       return ret;
     }
@@ -8390,7 +8438,8 @@ void RGWPutBucketPolicy::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -8491,8 +8540,8 @@ int RGWDeleteBucketPolicy::verify_permission(optional_yield y)
 
 void RGWDeleteBucketPolicy::execute(optional_yield y)
 {
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -8558,7 +8607,8 @@ void RGWPutBucketObjectLock::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << __func__ << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -8920,7 +8970,8 @@ void RGWPutBucketPublicAccessBlock::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -8995,8 +9046,8 @@ int RGWDeleteBucketPublicAccessBlock::verify_permission(optional_yield y)
 
 void RGWDeleteBucketPublicAccessBlock::execute(optional_yield y)
 {
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -9051,7 +9102,8 @@ void RGWPutBucketEncryption::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         &data, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -9105,8 +9157,8 @@ int RGWDeleteBucketEncryption::verify_permission(optional_yield y)
 
 void RGWDeleteBucketEncryption::execute(optional_yield y)
 {
-  bufferlist data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
     return;
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 21044046680a..09fe6cf4d7af 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -66,16 +66,13 @@ class RGWRados;
 class RGWMultiCompleteUpload;
 class RGWPutObj_Torrent;
 
+namespace rgw::auth::registry { class StrategyRegistry; }
 
-namespace rgw {
-namespace auth {
-namespace registry {
-
-class StrategyRegistry;
-
-}
-}
-}
+int rgw_forward_request_to_master(const DoutPrefixProvider* dpp,
+                                  const rgw::SiteConfig& site,
+                                  const rgw_user& uid,
+                                  bufferlist* indata, JSONParser* jp,
+                                  req_info& req, optional_yield y);
 
 int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
                                        CephContext *cct,
diff --git a/src/rgw/rgw_rest_user_policy.cc b/src/rgw/rgw_rest_user_policy.cc
index 2e300468b95c..ddca86a95d86 100644
--- a/src/rgw/rgw_rest_user_policy.cc
+++ b/src/rgw/rgw_rest_user_policy.cc
@@ -13,6 +13,7 @@
 
 #include "rgw_common.h"
 #include "rgw_op.h"
+#include "rgw_process_env.h"
 #include "rgw_rest.h"
 #include "rgw_rest_user_policy.h"
 #include "rgw_sal.h"
@@ -131,8 +132,8 @@ void RGWPutUserPolicy::execute(optional_yield y)
     return;
   }
 
-  ceph::bufferlist in_data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "ERROR: forward_request_to_master returned ret=" << op_ret << dendl;
     return;
@@ -362,8 +363,8 @@ void RGWDeleteUserPolicy::execute(optional_yield y)
     return;
   }
 
-  ceph::bufferlist in_data;
-  op_ret = driver->forward_request_to_master(this, s->user.get(), nullptr, in_data, nullptr, s->info, y);
+  op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
+                                         nullptr, nullptr, s->info, y);
   if (op_ret < 0) {
     // a policy might've been uploaded to this site when there was no sync
     // req. in earlier releases, proceed deletion

From d3ad0efaee6a19c84b2af9b3dfa1b6c902674774 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 17:42:56 -0400
Subject: [PATCH 0506/2492] rgw: add free function
 forward_iam_request_to_master()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_rest_role.cc | 173 ++++++++++++++++++++-------------------
 1 file changed, 88 insertions(+), 85 deletions(-)

diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index 14e164553665..2b8d3920acea 100644
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -13,15 +13,68 @@
 
 #include "rgw_common.h"
 #include "rgw_op.h"
+#include "rgw_process_env.h"
 #include "rgw_rest.h"
-#include "rgw_role.h"
+#include "rgw_rest_conn.h"
 #include "rgw_rest_role.h"
+#include "rgw_role.h"
 #include "rgw_sal.h"
 
 #define dout_subsys ceph_subsys_rgw
 
 using namespace std;
 
+int forward_iam_request_to_master(const DoutPrefixProvider* dpp,
+                                  const rgw::SiteConfig& site,
+                                  const RGWUserInfo& user,
+                                  bufferlist& indata,
+                                  RGWXMLDecoder::XMLParser& parser,
+                                  req_info& req, optional_yield y)
+{
+  const auto& period = site.get_period();
+  if (!period) {
+    return 0; // not multisite
+  }
+  if (site.is_meta_master()) {
+    return 0; // don't need to forward metadata requests
+  }
+  const auto& pmap = period->period_map;
+  auto zg = pmap.zonegroups.find(pmap.master_zonegroup);
+  if (zg == pmap.zonegroups.end()) {
+    return -EINVAL;
+  }
+  auto z = zg->second.zones.find(zg->second.master_zone);
+  if (z == zg->second.zones.end()) {
+    return -EINVAL;
+  }
+
+  RGWAccessKey creds;
+  if (auto i = user.access_keys.begin(); i != user.access_keys.end()) {
+    creds.id = i->first;
+    creds.key = i->second.key;
+  }
+
+  // use the master zone's endpoints
+  auto conn = RGWRESTConn{dpp->get_cct(), z->second.id, z->second.endpoints,
+                          std::move(creds), zg->second.id, zg->second.api_name};
+  bufferlist outdata;
+  constexpr size_t max_response_size = 128 * 1024; // we expect a very small response
+  int ret = conn.forward_iam_request(dpp, creds, req, nullptr, max_response_size,
+                                     &indata, &outdata, y);
+  if (ret < 0) {
+    return ret;
+  }
+
+  std::string r = outdata.to_str();
+  boost::replace_all(r, "&quot;", "\"");
+
+  if (!parser.parse(r.c_str(), r.length(), 1)) {
+    ldpp_dout(dpp, 0) << "ERROR: failed to parse response from master zonegroup" << dendl;
+    return -EIO;
+  }
+  return 0;
+}
+
 int RGWRestRole::verify_permission(optional_yield y)
 {
   if (s->auth.identity->is_anonymous()) {
@@ -209,7 +262,8 @@ void RGWCreateRole::execute(optional_yield y)
 
   std::string role_id;
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -231,15 +285,8 @@ void RGWCreateRole::execute(optional_yield y)
       }
     }
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -316,13 +363,13 @@ int RGWDeleteRole::get_params()
 void RGWDeleteRole::execute(optional_yield y)
 {
   bool is_master = true;
-  int master_op_ret = 0;
   op_ret = get_params();
   if (op_ret < 0) {
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     is_master = false;
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
@@ -335,17 +382,9 @@ void RGWDeleteRole::execute(optional_yield y)
     s->info.args.remove("Action");
     s->info.args.remove("Version");
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    master_op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
-    if (master_op_ret < 0) {
-      op_ret = master_op_ret;
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
+    if (op_ret < 0) {
       ldpp_dout(this, 0) << "forward_iam_request_to_master returned ret=" << op_ret << dendl;
       return;
     }
@@ -355,7 +394,7 @@ void RGWDeleteRole::execute(optional_yield y)
 
   if (op_ret == -ENOENT) {
     //Role has been deleted since metadata from master has synced up
-    if (!is_master && master_op_ret == 0) {
+    if (!is_master) {
       op_ret = 0;
     } else {
       op_ret = -ERR_NO_ROLE_FOUND;
@@ -466,7 +505,8 @@ void RGWModifyRoleTrustPolicy::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -480,15 +520,8 @@ void RGWModifyRoleTrustPolicy::execute(optional_yield y)
     s->info.args.remove("Action");
     s->info.args.remove("Version");
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -590,7 +623,8 @@ void RGWPutRolePolicy::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -605,15 +639,8 @@ void RGWPutRolePolicy::execute(optional_yield y)
     s->info.args.remove("Action");
     s->info.args.remove("Version");
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -723,7 +750,8 @@ void RGWDeleteRolePolicy::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -737,15 +765,8 @@ void RGWDeleteRolePolicy::execute(optional_yield y)
     s->info.args.remove("Action");
     s->info.args.remove("Version");
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -792,7 +813,8 @@ void RGWTagRole::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -811,15 +833,8 @@ void RGWTagRole::execute(optional_yield y)
       }
     }
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -906,7 +921,8 @@ void RGWUntagRole::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -929,15 +945,8 @@ void RGWUntagRole::execute(optional_yield y)
     for (auto& it : iters) {
       val_map.erase(it);
     }
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;
@@ -976,7 +985,8 @@ void RGWUpdateRole::execute(optional_yield y)
     return;
   }
 
-  if (!driver->is_meta_master()) {
+  const rgw::SiteConfig& site = *s->penv.site;
+  if (!site.is_meta_master()) {
     RGWXMLDecoder::XMLParser parser;
     if (!parser.init()) {
       ldpp_dout(this, 0) << "ERROR: failed to initialize xml parser" << dendl;
@@ -990,15 +1000,8 @@ void RGWUpdateRole::execute(optional_yield y)
     s->info.args.remove("Action");
     s->info.args.remove("Version");
 
-    RGWUserInfo info = s->user->get_info();
-    const auto& it = info.access_keys.begin();
-    RGWAccessKey key;
-    if (it != info.access_keys.end()) {
-      key.id = it->first;
-      RGWAccessKey cred = it->second;
-      key.key = cred.key;
-    }
-    op_ret = driver->forward_iam_request_to_master(s, key, nullptr, bl_post_body, &parser, s->info, y);
+    op_ret = forward_iam_request_to_master(this, site, s->user->get_info(),
+                                           bl_post_body, parser, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 20) << "ERROR: forward_iam_request_to_master failed with error code: " << op_ret << dendl;
       return;

From e93b6bb956829785b1e9daddc1679882d6e5901f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Mar 2023 17:51:25 -0400
Subject: [PATCH 0507/2492] rgw/sal: remove forward-to-master functions from
 Driver

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc   | 16 ------
 src/rgw/driver/daos/rgw_sal_daos.h    |  9 ----
 src/rgw/driver/motr/rgw_sal_motr.cc   | 16 ------
 src/rgw/driver/motr/rgw_sal_motr.h    |  7 ---
 src/rgw/driver/rados/rgw_sal_rados.cc | 72 ---------------------------
 src/rgw/driver/rados/rgw_sal_rados.h  |  7 ---
 src/rgw/rgw_sal.h                     |  8 ---
 src/rgw/rgw_sal_dbstore.cc            | 16 ------
 src/rgw/rgw_sal_dbstore.h             |  7 ---
 src/rgw/rgw_sal_filter.cc             | 20 --------
 src/rgw/rgw_sal_filter.h              | 11 ----
 11 files changed, 189 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 8be268fe17cc..1bade3752032 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -2252,22 +2252,6 @@ int DaosStore::load_bucket(const DoutPrefixProvider* dpp, User* u,
 
 bool DaosStore::is_meta_master() { return true; }
 
-int DaosStore::forward_request_to_master(const DoutPrefixProvider* dpp,
-                                         User* user, obj_version* objv,
-                                         bufferlist& in_data, JSONParser* jp,
-                                         req_info& info, optional_yield y) {
-  return DAOS_NOT_IMPLEMENTED_LOG(dpp);
-}
-
-int DaosStore::forward_iam_request_to_master(const DoutPrefixProvider* dpp,
-                                             const RGWAccessKey& key,
-                                             obj_version* objv,
-                                             bufferlist& in_data,
-                                             RGWXMLDecoder::XMLParser* parser,
-                                             req_info& info, optional_yield y) {
-  return DAOS_NOT_IMPLEMENTED_LOG(dpp);
-}
-
 std::string DaosStore::zone_unique_id(uint64_t unique_num) { return ""; }
 
 std::string DaosStore::zone_unique_trans_id(const uint64_t unique_num) {
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 4d5e6c831079..8394fb2f7570 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -898,15 +898,6 @@ class DaosStore : public StoreDriver {
                   const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
                   optional_yield y) override;
   virtual bool is_meta_master() override;
-  virtual int forward_request_to_master(const DoutPrefixProvider* dpp,
-                                        User* user, obj_version* objv,
-                                        bufferlist& in_data, JSONParser* jp,
-                                        req_info& info,
-                                        optional_yield y) override;
-  virtual int forward_iam_request_to_master(
-      const DoutPrefixProvider* dpp, const RGWAccessKey& key, obj_version* objv,
-      bufferlist& in_data, RGWXMLDecoder::XMLParser* parser, req_info& info,
-      optional_yield y) override;
   virtual Zone* get_zone() { return &zone; }
   virtual std::string zone_unique_id(uint64_t unique_num) override;
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index d6a5eb1bc3ed..1e110ae73a3c 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -3246,22 +3246,6 @@ bool MotrStore::is_meta_master()
   return true;
 }
 
-int MotrStore::forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version *objv,
-    bufferlist& in_data,
-    JSONParser *jp, req_info& info,
-    optional_yield y)
-{
-  return 0;
-}
-
-int MotrStore::forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y)
-{
-    return 0;
-}
-
 std::string MotrStore::zone_unique_id(uint64_t unique_num)
 {
   return "";
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 66ca8314fc0b..92ddc9a115f2 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -994,13 +994,6 @@ class MotrStore : public StoreDriver {
     int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
                     std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
-    virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
-        bufferlist& in_data, JSONParser *jp, req_info& info,
-        optional_yield y) override;
-    virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y) override;
     virtual Zone* get_zone() { return &zone; }
     virtual std::string zone_unique_id(uint64_t unique_num) override;
     virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 68b6d998eca4..f96378ada012 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1005,78 +1005,6 @@ bool RadosStore::is_meta_master()
   return svc()->zone->is_meta_master();
 }
 
-int RadosStore::forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
-					     bufferlist& in_data,
-					     JSONParser* jp, req_info& info,
-					     optional_yield y)
-{
-  if (is_meta_master()) {
-    /* We're master, don't forward */
-    return 0;
-  }
-
-  if (!svc()->zone->get_master_conn()) {
-    ldpp_dout(dpp, 0) << "rest connection is invalid" << dendl;
-    return -EINVAL;
-  }
-  ldpp_dout(dpp, 0) << "sending request to master zonegroup" << dendl;
-  bufferlist response;
-  std::string uid_str = user->get_id().to_str();
-#define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
-  int ret = svc()->zone->get_master_conn()->forward(dpp, rgw_user(uid_str), info,
-                                                    objv, MAX_REST_RESPONSE,
-						    &in_data, &response, y);
-  if (ret < 0)
-    return ret;
-
-  ldpp_dout(dpp, 20) << "response: " << response.c_str() << dendl;
-  if (jp && !jp->parse(response.c_str(), response.length())) {
-    ldpp_dout(dpp, 0) << "failed parsing response from master zonegroup" << dendl;
-    return -EINVAL;
-  }
-
-  return 0;
-}
-
-int RadosStore::forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y)
-{
-  if (is_meta_master()) {
-    /* We're master, don't forward */
-    return 0;
-  }
-
-  if (!svc()->zone->get_master_conn()) {
-    ldpp_dout(dpp, 0) << "rest connection is invalid" << dendl;
-    return -EINVAL;
-  }
-  ldpp_dout(dpp, 0) << "sending request to master zonegroup" << dendl;
-  bufferlist response;
-#define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
-  int ret = svc()->zone->get_master_conn()->forward_iam_request(dpp, key, info,
-                                                    objv, MAX_REST_RESPONSE,
-						                                        &in_data, &response, y);
-  if (ret < 0)
-    return ret;
-
-  ldpp_dout(dpp, 20) << "response: " << response.c_str() << dendl;
-
-  std::string r = response.c_str();
-  std::string str_to_search = "&quot;";
-  std::string str_to_replace = "\"";
-  boost::replace_all(r, str_to_search, str_to_replace);
-  ldpp_dout(dpp, 20) << "r: " << r.c_str() << dendl;
-
-  if (parser && !parser->parse(r.c_str(), r.length(), 1)) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to parse response from master zonegroup" << dendl;
-    return -EIO;
-  }
-
-  return 0;
-}
-
 std::string RadosStore::zone_unique_id(uint64_t unique_num)
 {
   return svc()->zone_utils->unique_id(unique_num);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index c46ba8e68f0b..0ad6e8fa083a 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -139,13 +139,6 @@ class RadosStore : public StoreDriver {
     int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
                     std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
-    virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
-					  bufferlist& in_data, JSONParser* jp, req_info& info,
-					  optional_yield y) override;
-    virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y) override;
     virtual Zone* get_zone() { return zone.get(); }
     virtual std::string zone_unique_id(uint64_t unique_num) override;
     virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index a4b0219dd411..1019f48dab46 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -298,14 +298,6 @@ class Driver {
                             std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
     /** For multisite, this driver is the zone's master */
     virtual bool is_meta_master() = 0;
-    /** For multisite, forward an OP to the zone's master */
-    virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
-					  bufferlist& in_data, JSONParser* jp, req_info& info,
-					  optional_yield y) = 0;
-    virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y) = 0;
     /** Get zone info for this driver */
     virtual Zone* get_zone() = 0;
     /** Get a unique ID specific to this zone. */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 47fa0f06ba75..bce27b912d8f 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -1570,22 +1570,6 @@ namespace rgw::sal {
     return true;
   }
 
-  int DBStore::forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version *objv,
-      bufferlist& in_data,
-      JSONParser *jp, req_info& info,
-      optional_yield y)
-  {
-    return 0;
-  }
-
-  int DBStore::forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y)
-  {
-      return 0;
-  }
-
   std::string DBStore::zone_unique_id(uint64_t unique_num)
   {
     return "";
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 7ab98f6286b4..bd359d038a28 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -767,13 +767,6 @@ class DBNotification : public StoreNotification {
       int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
                       std::unique_ptr<Bucket>* bucket, optional_yield y) override;
       virtual bool is_meta_master() override;
-      virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
-          bufferlist& in_data, JSONParser *jp, req_info& info,
-          optional_yield y) override;
-      virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp, const RGWAccessKey& key, obj_version* objv,
-					     bufferlist& in_data,
-					     RGWXMLDecoder::XMLParser* parser, req_info& info,
-					     optional_yield y) override;
       virtual Zone* get_zone() { return &zone; }
       virtual std::string zone_unique_id(uint64_t unique_num) override;
       virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 01a113513672..b27f1123995e 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -179,26 +179,6 @@ bool FilterDriver::is_meta_master()
   return next->is_meta_master();
 }
 
-int FilterDriver::forward_request_to_master(const DoutPrefixProvider *dpp,
-					   User* user, obj_version* objv,
-					   bufferlist& in_data,
-					   JSONParser* jp, req_info& info,
-					   optional_yield y)
-{
-  return next->forward_request_to_master(dpp, user, objv, in_data, jp, info, y);
-}
-
-int FilterDriver::forward_iam_request_to_master(const DoutPrefixProvider *dpp,
-					       const RGWAccessKey& key,
-					       obj_version* objv,
-					       bufferlist& in_data,
-					       RGWXMLDecoder::XMLParser* parser,
-					       req_info& info,
-					       optional_yield y)
-{
-  return next->forward_iam_request_to_master(dpp, key, objv, in_data, parser, info, y);
-}
-
 std::string FilterDriver::zone_unique_id(uint64_t unique_num)
 {
   return next->zone_unique_id(unique_num);
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index aec8cd64c2ca..8ca3a470e448 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -162,17 +162,6 @@ class FilterDriver : public Driver {
                   const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
                   optional_yield y) override;
   virtual bool is_meta_master() override;
-  virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user,
-					obj_version* objv, bufferlist& in_data,
-					JSONParser* jp, req_info& info,
-					optional_yield y) override;
-  virtual int forward_iam_request_to_master(const DoutPrefixProvider *dpp,
-					    const RGWAccessKey& key,
-					    obj_version* objv,
-					    bufferlist& in_data,
-					    RGWXMLDecoder::XMLParser* parser,
-					    req_info& info,
-					    optional_yield y) override;
   virtual Zone* get_zone() override { return zone.get(); }
   virtual std::string zone_unique_id(uint64_t unique_num) override;
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;

From 0ead6ff010b1b7a68205227706833418283cf4d2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 4 Oct 2023 12:49:10 -0400
Subject: [PATCH 0508/2492] rgw/sal: rename Bucket::remove_bucket() to remove()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc     | 12 ++++++------
 src/rgw/driver/daos/rgw_sal_daos.h      | 12 ++++++------
 src/rgw/driver/motr/rgw_sal_motr.cc     |  4 ++--
 src/rgw/driver/motr/rgw_sal_motr.h      |  4 ++--
 src/rgw/driver/posix/rgw_sal_posix.cc   | 20 ++++++++++----------
 src/rgw/driver/posix/rgw_sal_posix.h    | 12 ++++++------
 src/rgw/driver/rados/rgw_bucket.cc      |  4 ++--
 src/rgw/driver/rados/rgw_rest_bucket.cc |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc   | 16 ++++++++--------
 src/rgw/driver/rados/rgw_sal_rados.h    | 10 +++++-----
 src/rgw/driver/rados/rgw_user.cc        |  2 +-
 src/rgw/rgw_op.cc                       |  4 ++--
 src/rgw/rgw_sal.h                       | 10 +++++-----
 src/rgw/rgw_sal_dbstore.cc              | 10 +++++-----
 src/rgw/rgw_sal_dbstore.h               | 10 +++++-----
 src/rgw/rgw_sal_filter.cc               | 18 +++++++++---------
 src/rgw/rgw_sal_filter.h                | 12 ++++++------
 17 files changed, 81 insertions(+), 81 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 1bade3752032..3687f57b8c89 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -404,8 +404,8 @@ std::unique_ptr<struct ds3_bucket_info> DaosBucket::get_encoded_info(
   return bucket_info;
 }
 
-int DaosBucket::remove_bucket(const DoutPrefixProvider* dpp,
-                              bool delete_children, optional_yield y) {
+int DaosBucket::remove(const DoutPrefixProvider* dpp,
+                       bool delete_children, optional_yield y) {
   ldpp_dout(dpp, 20) << "DEBUG: remove_bucket, delete_children="
                      << delete_children << dendl;
 
@@ -413,10 +413,10 @@ int DaosBucket::remove_bucket(const DoutPrefixProvider* dpp,
                             nullptr);
 }
 
-int DaosBucket::remove_bucket_bypass_gc(int concurrent_max,
-                                        bool keep_index_consistent,
-                                        optional_yield y,
-                                        const DoutPrefixProvider* dpp) {
+int DaosBucket::remove_bypass_gc(int concurrent_max,
+                                 bool keep_index_consistent,
+                                 optional_yield y,
+                                 const DoutPrefixProvider* dpp) {
   ldpp_dout(dpp, 20) << "DEBUG: remove_bucket_bypass_gc, concurrent_max="
                     
                      << concurrent_max
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 8394fb2f7570..66f570064632 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -289,12 +289,12 @@ class DaosBucket : public StoreBucket {
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
   virtual int list(const DoutPrefixProvider* dpp, ListParams&, int,
                    ListResults&, optional_yield y) override;
-  virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-                            optional_yield y) override;
-  virtual int remove_bucket_bypass_gc(int concurrent_max,
-                                      bool keep_index_consistent,
-                                      optional_yield y,
-                                      const DoutPrefixProvider* dpp) override;
+  virtual int remove(const DoutPrefixProvider* dpp, bool delete_children,
+                     optional_yield y) override;
+  virtual int remove_bypass_gc(int concurrent_max,
+                               bool keep_index_consistent,
+                               optional_yield y,
+                               const DoutPrefixProvider* dpp) override;
   virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
   virtual int set_acl(const DoutPrefixProvider* dpp,
                       RGWAccessControlPolicy& acl, optional_yield y) override;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 1e110ae73a3c..8cc8df987d2b 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -539,7 +539,7 @@ int MotrUser::verify_mfa(const std::string& mfa_str, bool* verified, const DoutP
   return 0;
 }
 
-int MotrBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
+int MotrBucket::remove(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
 {
   int ret;
 
@@ -669,7 +669,7 @@ int MotrBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_childre
   return ret;
 }
 
-int MotrBucket::remove_bucket_bypass_gc(int concurrent_max, bool
+int MotrBucket::remove_bypass_gc(int concurrent_max, bool
         keep_index_consistent,
         optional_yield y, const
         DoutPrefixProvider *dpp) {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 92ddc9a115f2..3a110e36f173 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -352,8 +352,8 @@ class MotrBucket : public StoreBucket {
 
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
     virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override;
-    virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
-    virtual int remove_bucket_bypass_gc(int concurrent_max, bool
+    virtual int remove(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
+    virtual int remove_bypass_gc(int concurrent_max, bool
         keep_index_consistent,
         optional_yield y, const
         DoutPrefixProvider *dpp) override;
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index e4e640b02034..b72c7868600b 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -851,20 +851,20 @@ int POSIXBucket::merge_and_store_attrs(const DoutPrefixProvider* dpp,
   return write_attrs(dpp, y);
 }
 
-int POSIXBucket::remove_bucket(const DoutPrefixProvider* dpp,
-				bool delete_children,
-				optional_yield y)
+int POSIXBucket::remove(const DoutPrefixProvider* dpp,
+			bool delete_children,
+			optional_yield y)
 {
   return delete_directory(parent_fd, get_fname().c_str(),
 			  delete_children, dpp);
 }
 
-int POSIXBucket::remove_bucket_bypass_gc(int concurrent_max,
-					 bool keep_index_consistent,
-					 optional_yield y,
-					 const DoutPrefixProvider *dpp)
+int POSIXBucket::remove_bypass_gc(int concurrent_max,
+				  bool keep_index_consistent,
+				  optional_yield y,
+				  const DoutPrefixProvider *dpp)
 {
-  return remove_bucket(dpp, true, y);
+  return remove(dpp, true, y);
 }
 
 int POSIXBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y)
@@ -1448,7 +1448,7 @@ int POSIXObject::delete_object(const DoutPrefixProvider* dpp,
 
   if (!b->versioned()) {
     if (shadow) {
-      ret = shadow->remove_bucket(dpp, true, y);
+      ret = shadow->remove(dpp, true, y);
       if (ret < 0) {
 	return ret;
       }
@@ -2587,7 +2587,7 @@ int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
     return ret;
   }
 
-  shadow->remove_bucket(dpp, true, y);
+  shadow->remove(dpp, true, y);
 
   return 0;
 }
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index c5360aa90f9a..16f88729ab4a 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -175,12 +175,12 @@ class POSIXBucket : public StoreBucket {
 		   ListResults&, optional_yield y) override;
   virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp,
 				    Attrs& new_attrs, optional_yield y) override;
-  virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-			    optional_yield y) override;
-  virtual int remove_bucket_bypass_gc(int concurrent_max,
-				      bool keep_index_consistent,
-				      optional_yield y,
-				      const DoutPrefixProvider *dpp) override;
+  virtual int remove(const DoutPrefixProvider* dpp, bool delete_children,
+		     optional_yield y) override;
+  virtual int remove_bypass_gc(int concurrent_max,
+			       bool keep_index_consistent,
+			       optional_yield y,
+			       const DoutPrefixProvider *dpp) override;
   virtual int create(const DoutPrefixProvider* dpp,
 		     const CreateParams& params,
 		     optional_yield y) override;
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index d0bdc025b7a6..4af11fc46464 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1253,9 +1253,9 @@ int RGWBucketAdminOp::remove_bucket(rgw::sal::Driver* driver, RGWBucketAdminOpSt
     return ret;
 
   if (bypass_gc)
-    ret = bucket->remove_bucket_bypass_gc(op_state.get_max_aio(), keep_index_consistent, y, dpp);
+    ret = bucket->remove_bypass_gc(op_state.get_max_aio(), keep_index_consistent, y, dpp);
   else
-    ret = bucket->remove_bucket(dpp, op_state.will_delete_children(), y);
+    ret = bucket->remove(dpp, op_state.will_delete_children(), y);
 
   return ret;
 }
diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index a530d4056735..4fac8ef64f7c 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -243,7 +243,7 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
     return;
   }
 
-  op_ret = bucket->remove_bucket(s, delete_children, s->yield);
+  op_ret = bucket->remove(s, delete_children, s->yield);
 }
 
 class RGWOp_Set_Bucket_Quota : public RGWRESTOp {
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index f96378ada012..c555ead9745e 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -271,9 +271,9 @@ int RadosUser::verify_mfa(const std::string& mfa_str, bool* verified,
 
 RadosBucket::~RadosBucket() {}
 
-int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp,
-			       bool delete_children,
-			       optional_yield y)
+int RadosBucket::remove(const DoutPrefixProvider* dpp,
+			bool delete_children,
+			optional_yield y)
 {
   int ret;
 
@@ -358,10 +358,10 @@ int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp,
   return ret;
 }
 
-int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool
-					 keep_index_consistent,
-					 optional_yield y, const
-					 DoutPrefixProvider *dpp)
+int RadosBucket::remove_bypass_gc(int concurrent_max, bool
+				  keep_index_consistent,
+				  optional_yield y, const
+				  DoutPrefixProvider *dpp)
 {
   int ret;
   map<RGWObjCategory, RGWStorageStats> stats;
@@ -485,7 +485,7 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool
   // this function can only be run if caller wanted children to be
   // deleted, so we can ignore the check for children as any that
   // remain are detritus from a prior bug
-  ret = remove_bucket(dpp, true, y);
+  ret = remove(dpp, true, y);
   if (ret < 0) {
     ldpp_dout(dpp, -1) << "ERROR: could not remove bucket " << this << dendl;
     return ret;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 0ad6e8fa083a..c67d23289ed2 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -512,11 +512,11 @@ class RadosBucket : public StoreBucket {
     virtual ~RadosBucket();
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
     virtual int list(const DoutPrefixProvider* dpp, ListParams&, int, ListResults&, optional_yield y) override;
-    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) override;
-    virtual int remove_bucket_bypass_gc(int concurrent_max, bool
-					keep_index_consistent,
-					optional_yield y, const
-					DoutPrefixProvider *dpp) override;
+    virtual int remove(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) override;
+    virtual int remove_bypass_gc(int concurrent_max, bool
+				 keep_index_consistent,
+				 optional_yield y, const
+				 DoutPrefixProvider *dpp) override;
     virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
     virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
     int create(const DoutPrefixProvider* dpp, const CreateParams& params,
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index 97776f9d5a21..c27be158c334 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -1791,7 +1791,7 @@ int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
         return ret;
       }
 
-      ret = bucket->remove_bucket(dpp, true, y);
+      ret = bucket->remove(dpp, true, y);
       if (ret < 0) {
         set_err_msg(err_msg, "unable to delete user data");
         return ret;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 1fbd2ec7105a..00715db9219d 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3730,7 +3730,7 @@ void RGWDeleteBucket::execute(optional_yield y)
       // do nothing; it will already have been logged
   }
 
-  op_ret = s->bucket->remove_bucket(this, false, y);
+  op_ret = s->bucket->remove(this, false, y);
   if (op_ret < 0 && op_ret == -ECANCELED) {
       // lost a race, either with mdlog sync or another delete bucket operation.
       // in either case, we've already called ctl.bucket->unlink_bucket()
@@ -7478,7 +7478,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
         goto delop_fail;
       }
     }
-    ret = bucket->remove_bucket(dpp, false, s->yield);
+    ret = bucket->remove(dpp, false, s->yield);
     if (ret < 0) {
       goto delop_fail;
     }
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 1019f48dab46..b222fcd03cbc 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -636,12 +636,12 @@ class Bucket {
     /** Set the cached attributes on this bucket */
     virtual int set_attrs(Attrs a) = 0;
     /** Remove this bucket from the backing store */
-    virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) = 0;
+    virtual int remove(const DoutPrefixProvider* dpp, bool delete_children, optional_yield y) = 0;
     /** Remove this bucket, bypassing garbage collection.  May be removed */
-    virtual int remove_bucket_bypass_gc(int concurrent_max, bool
-					keep_index_consistent,
-					optional_yield y, const
-					DoutPrefixProvider *dpp) = 0;
+    virtual int remove_bypass_gc(int concurrent_max, bool
+				 keep_index_consistent,
+				 optional_yield y, const
+				 DoutPrefixProvider *dpp) = 0;
     /** Get then ACL for this bucket */
     virtual RGWAccessControlPolicy& get_acl(void) = 0;
     /** Set the ACL for this bucket */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index bce27b912d8f..5140a3bfbe8e 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -155,7 +155,7 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBBucket::remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
+  int DBBucket::remove(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y)
   {
     int ret;
 
@@ -196,10 +196,10 @@ namespace rgw::sal {
     return ret;
   }
 
-  int DBBucket::remove_bucket_bypass_gc(int concurrent_max, bool
-					keep_index_consistent,
-					optional_yield y, const
-					DoutPrefixProvider *dpp) {
+  int DBBucket::remove_bypass_gc(int concurrent_max, bool
+				 keep_index_consistent,
+				 optional_yield y, const
+				 DoutPrefixProvider *dpp) {
     return 0;
   }
 
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index bd359d038a28..26a8dfbcafdd 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -155,11 +155,11 @@ class DBNotification : public StoreNotification {
 
       virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
       virtual int list(const DoutPrefixProvider *dpp, ListParams&, int, ListResults&, optional_yield y) override;
-      virtual int remove_bucket(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
-      virtual int remove_bucket_bypass_gc(int concurrent_max, bool
-					keep_index_consistent,
-					optional_yield y, const
-					DoutPrefixProvider *dpp) override;
+      virtual int remove(const DoutPrefixProvider *dpp, bool delete_children, optional_yield y) override;
+      virtual int remove_bypass_gc(int concurrent_max, bool
+				   keep_index_consistent,
+				   optional_yield y, const
+				   DoutPrefixProvider *dpp) override;
       virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
       virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
       int create(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index b27f1123995e..f3ee09fe0c98 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -566,19 +566,19 @@ int FilterBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int ma
   return next->list(dpp, params, max, results, y);
 }
 
-int FilterBucket::remove_bucket(const DoutPrefixProvider* dpp,
-				bool delete_children,
-				optional_yield y)
+int FilterBucket::remove(const DoutPrefixProvider* dpp,
+			 bool delete_children,
+			 optional_yield y)
 {
-  return next->remove_bucket(dpp, delete_children, y);
+  return next->remove(dpp, delete_children, y);
 }
 
-int FilterBucket::remove_bucket_bypass_gc(int concurrent_max,
-					  bool keep_index_consistent,
-					  optional_yield y,
-					  const DoutPrefixProvider *dpp)
+int FilterBucket::remove_bypass_gc(int concurrent_max,
+				   bool keep_index_consistent,
+				   optional_yield y,
+				   const DoutPrefixProvider *dpp)
 {
-  return next->remove_bucket_bypass_gc(concurrent_max, keep_index_consistent, y, dpp);
+  return next->remove_bypass_gc(concurrent_max, keep_index_consistent, y, dpp);
 }
 
 int FilterBucket::set_acl(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 8ca3a470e448..34739c3426a2 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -379,12 +379,12 @@ class FilterBucket : public Bucket {
 		   ListResults&, optional_yield y) override;
   virtual Attrs& get_attrs(void) override { return next->get_attrs(); }
   virtual int set_attrs(Attrs a) override { return next->set_attrs(a); }
-  virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children,
-			    optional_yield y) override;
-  virtual int remove_bucket_bypass_gc(int concurrent_max, bool
-				      keep_index_consistent,
-				      optional_yield y, const
-				      DoutPrefixProvider *dpp) override;
+  virtual int remove(const DoutPrefixProvider* dpp, bool delete_children,
+		     optional_yield y) override;
+  virtual int remove_bypass_gc(int concurrent_max, bool
+			       keep_index_consistent,
+			       optional_yield y, const
+			       DoutPrefixProvider *dpp) override;
   virtual RGWAccessControlPolicy& get_acl(void) override { return next->get_acl(); }
   virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl,
 		      optional_yield y) override;

From 24882deb0264401dba49b10fd3474d5944674961 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 7 Nov 2023 12:32:45 -0500
Subject: [PATCH 0509/2492] cephadm: work around pip failure on some envs

Work around an encoding/locale issue when the dashboard tests are run
(ubuntu 20.04).
The build.py changes brought in a9d1c62ca86 were validated for package
builds, teuthology, and other CI jobs but a different error was masking
this failure in the dashboard ci job.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 87d990e72a35..122767a8ecc8 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -190,6 +190,7 @@ def _install_deps(tempdir, config):
     env = os.environ.copy()
     env['CC'] = '/bin/false'
     env['CXX'] = '/bin/false'
+    env['LC_ALL'] = 'C.UTF-8'  # work around some env issues with pip
     if env.get('PYTHONPATH'):
         env['PYTHONPATH'] = env['PYTHONPATH'] + f':{tempdir}'
     else:

From 99fc4a8d406291b65a53f157442bc54bc67e8b0d Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 7 Nov 2023 15:49:57 -0500
Subject: [PATCH 0510/2492] mgr/cephadm: fix reweighting of OSD when OSD
 removal is stopped

Previously, when you ran "ceph orch osd rm stop <osd-id>"
cephadm would pass in a new OSD object to the removal
queue that would not have any of the fields set previously
for the OSD. This was mostly fine when removing it from
the queue as those fields were no longer needed, but an
exception was the initial weight, which you need if
you want to set the weight back when you stop removal.

This patch changes it so it will now remove the actual
OSD object the removal queue stores so that we will
get to use the previously set original weight. It also
changes when we grab the original weight to make it
happen earlier and adds it to the to_json so it survives
any potential mgr failovers.

Fixes: https://tracker.ceph.com/issues/63481

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py             |  3 +-
 src/pybind/mgr/cephadm/services/osd.py       | 13 ++++++++-
 src/pybind/mgr/cephadm/tests/test_cephadm.py | 30 ++++++++++++++------
 3 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index a83256d0bb7d..05cd8d6b7855 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -3379,8 +3379,7 @@ def stop_remove_osds(self, osd_ids: List[str]) -> str:
         """
         for osd_id in osd_ids:
             try:
-                self.to_remove_osds.rm(OSD(osd_id=int(osd_id),
-                                           remove_util=self.to_remove_osds.rm_util))
+                self.to_remove_osds.rm_by_osd_id(int(osd_id))
             except (NotFoundError, KeyError, ValueError):
                 return f'Unable to find OSD in the queue: {osd_id}'
 
diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py
index bfecc57230ab..3616d42deee4 100644
--- a/src/pybind/mgr/cephadm/services/osd.py
+++ b/src/pybind/mgr/cephadm/services/osd.py
@@ -664,6 +664,7 @@ def start(self) -> None:
             return None
         self.started = True
         self.stopped = False
+        self.original_weight = self.rm_util.get_weight(self)
 
     def start_draining(self) -> bool:
         if self.stopped:
@@ -672,7 +673,6 @@ def start_draining(self) -> bool:
         if self.replace:
             self.rm_util.set_osd_flag([self], 'out')
         else:
-            self.original_weight = self.rm_util.get_weight(self)
             self.rm_util.reweight_osd(self, 0.0)
         self.drain_started_at = datetime.utcnow()
         self.draining = True
@@ -761,6 +761,7 @@ def to_json(self) -> dict:
         out['force'] = self.force
         out['zap'] = self.zap
         out['hostname'] = self.hostname  # type: ignore
+        out['original_weight'] = self.original_weight
 
         for k in ['drain_started_at', 'drain_stopped_at', 'drain_done_at', 'process_started_at']:
             if getattr(self, k):
@@ -953,6 +954,16 @@ def enqueue(self, osd: "OSD") -> None:
             self.osds.add(osd)
         osd.start()
 
+    def rm_by_osd_id(self, osd_id: int) -> None:
+        osd: Optional["OSD"] = None
+        for o in self.osds:
+            if o.osd_id == osd_id:
+                osd = o
+        if not osd:
+            logger.debug(f"Could not find osd with id {osd_id} in queue.")
+            raise KeyError(f'No osd with id {osd_id} in removal queue')
+        self.rm(osd)
+
     def rm(self, osd: "OSD") -> None:
         if not osd.exists:
             raise NotFoundError()
diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py
index b6aef018cf09..ebdf7eea5aa0 100644
--- a/src/pybind/mgr/cephadm/tests/test_cephadm.py
+++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py
@@ -1287,7 +1287,11 @@ def test_raw_driveselection_to_ceph_volume(self, cephadm_module, devices, previe
     ))
     @mock.patch("cephadm.services.osd.OSD.exists", True)
     @mock.patch("cephadm.services.osd.RemoveUtil.get_pg_count", lambda _, __: 0)
-    def test_remove_osds(self, cephadm_module):
+    @mock.patch("cephadm.services.osd.RemoveUtil.get_weight")
+    @mock.patch("cephadm.services.osd.RemoveUtil.reweight_osd")
+    def test_remove_osds(self, _reweight_osd, _get_weight, cephadm_module):
+        osd_initial_weight = 2.1
+        _get_weight.return_value = osd_initial_weight
         with with_host(cephadm_module, 'test'):
             CephadmServe(cephadm_module)._refresh_host_daemons('test')
             c = cephadm_module.list_daemons()
@@ -1297,13 +1301,23 @@ def test_remove_osds(self, cephadm_module):
             out = wait(cephadm_module, c)
             assert out == ["Removed osd.0 from host 'test'"]
 
-            cephadm_module.to_remove_osds.enqueue(OSD(osd_id=0,
-                                                      replace=False,
-                                                      force=False,
-                                                      hostname='test',
-                                                      process_started_at=datetime_now(),
-                                                      remove_util=cephadm_module.to_remove_osds.rm_util
-                                                      ))
+            osd_0 = OSD(osd_id=0,
+                        replace=False,
+                        force=False,
+                        hostname='test',
+                        process_started_at=datetime_now(),
+                        remove_util=cephadm_module.to_remove_osds.rm_util
+                        )
+
+            cephadm_module.to_remove_osds.enqueue(osd_0)
+            _get_weight.assert_called()
+
+            # test that OSD is properly reweighted on removal
+            cephadm_module.stop_remove_osds([0])
+            _reweight_osd.assert_called_with(mock.ANY, osd_initial_weight)
+
+            # add OSD back to queue and test normal removal queue processing
+            cephadm_module.to_remove_osds.enqueue(osd_0)
             cephadm_module.to_remove_osds.process_removal_queue()
             assert cephadm_module.to_remove_osds == OSDRemovalQueue(cephadm_module)
 

From 2ed3d7ad01c4f85b183ed57ec1cdbaeeb13a94c1 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 7 Nov 2023 11:32:12 -0800
Subject: [PATCH 0511/2492] crimson/tools/store_nbd/tm_driver: use
 TransactionManager::remove

231c7ac removed TransactionManager::dec_ref, adjust store_nbd/tm_driver
to use TransactionManager::remove instead.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/tools/store_nbd/tm_driver.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc
index bd216fd58ec4..62891aa864c8 100644
--- a/src/crimson/tools/store_nbd/tm_driver.cc
+++ b/src/crimson/tools/store_nbd/tm_driver.cc
@@ -27,7 +27,7 @@ seastar::future<> TMDriver::write(
         "write",
         [this, offset, &ptr](auto& t)
       {
-        return tm->dec_ref(t, offset
+        return tm->remove(t, offset
         ).si_then([](auto){}).handle_error_interruptible(
           crimson::ct_error::enoent::handle([](auto) { return seastar::now(); }),
           crimson::ct_error::pass_further_all{}

From bd3e01926fc472cfddb0b6bb79b822595632b378 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 7 Nov 2023 13:17:12 +0800
Subject: [PATCH 0512/2492] mds: skip uneccessary caller gid list check

Once the caller_gid is matched it will make no sense to do the
following caller gid list check.

Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/MDSAuthCaps.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MDSAuthCaps.cc b/src/mds/MDSAuthCaps.cc
index 7ddad41a08fa..ca4add2f323b 100644
--- a/src/mds/MDSAuthCaps.cc
+++ b/src/mds/MDSAuthCaps.cc
@@ -145,7 +145,7 @@ bool MDSCapMatch::match(string_view target_path,
       bool gid_matched = false;
       if (std::find(gids.begin(), gids.end(), caller_gid) != gids.end())
 	gid_matched = true;
-      if (caller_gid_list) {
+      else if (caller_gid_list) {
 	for (auto i = caller_gid_list->begin(); i != caller_gid_list->end(); ++i) {
 	  if (std::find(gids.begin(), gids.end(), *i) != gids.end()) {
 	    gid_matched = true;

From d5834c9af6013a23324a808570b22c819988ce03 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 8 Nov 2023 11:50:24 +0530
Subject: [PATCH 0513/2492] qa: enable debug mds/client for fs/nfs suite

Fixes: http://tracker.ceph.com/issues/63482
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/suites/fs/nfs/conf | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 qa/suites/fs/nfs/conf

diff --git a/qa/suites/fs/nfs/conf b/qa/suites/fs/nfs/conf
new file mode 120000
index 000000000000..9de8742be7b8
--- /dev/null
+++ b/qa/suites/fs/nfs/conf
@@ -0,0 +1 @@
+./.qa/cephfs/conf
\ No newline at end of file

From 434d8967a37d49bdce3effae4e9b4c69a8586a56 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 8 Nov 2023 19:07:40 +1000
Subject: [PATCH 0514/2492] doc/rados: edit t-mon "common issues" (1 of x)

Edit the first part of the section "Most Common Monitor Issues" in
doc/rados/troubleshooting/troublehsooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 41 +++++++++++--------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 64db2462ae12..3beb42fdf01a 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -180,38 +180,43 @@ the quorum is formed by only two monitors, and *c* is in the quorum as a
 Most Common Monitor Issues
 ===========================
 
-Have Quorum but at least one Monitor is down
----------------------------------------------
+The Cluster Has Quorum but at Least One Monitor is Down
+-------------------------------------------------------
 
-When this happens, depending on the version of Ceph you are running,
-you should be seeing something similar to::
+When the cluster has quorum but at least one monitor is down, ``ceph health
+detail`` returns a message similar to the following::
 
       $ ceph health detail
       [snip]
       mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum)
 
-**How to troubleshoot this?**
+**How do I troubleshoot a Ceph cluster that has quorum but also has at least one monitor down?**
 
-  First, make sure ``mon.a`` is running.
+  #. Make sure that ``mon.a`` is running.
 
-  Second, make sure you are able to connect to ``mon.a``'s node from the
-  other mon nodes. Check the TCP ports as well. Check ``iptables`` and
-  ``nf_conntrack`` on all nodes and ensure that you are not
-  dropping/rejecting connections.
+  #. Make sure that you can connect to ``mon.a``'s node from the
+     other Monitor nodes. Check the TCP ports as well. Check ``iptables`` and
+     ``nf_conntrack`` on all nodes and make sure that you are not
+     dropping/rejecting connections.
 
-  If this initial troubleshooting doesn't solve your problems, then it's
-  time to go deeper.
+  If this initial troubleshooting doesn't solve your problem, then further
+  investigation is necessary.
 
   First, check the problematic monitor's ``mon_status`` via the admin
   socket as explained in `Using the monitor's admin socket`_ and
   `Understanding mon_status`_.
 
-  If the monitor is out of the quorum, its state should be one of ``probing``,
-  ``electing`` or ``synchronizing``. If it happens to be either ``leader`` or
-  ``peon``, then the monitor believes to be in quorum, while the remaining
-  cluster is sure it is not; or maybe it got into the quorum while we were
-  troubleshooting the monitor, so check you ``ceph status`` again just to make
-  sure. Proceed if the monitor is not yet in the quorum.
+  If the Monitor is out of the quorum, then its state will be one of the
+  following: ``probing``, ``electing`` or ``synchronizing``. If the state of
+  the Monitor is ``leader`` or ``peon``, then the Monitor believes itself to be
+  in quorum but the rest of the cluster believes that it is not in quorum. It
+  is possible that a Monitor that is in one of the ``probing``, ``electing``,
+  or ``synchronizing`` states has entered the quorum during the process of
+  troubleshooting. Check ``ceph status`` again to determine whether the Monitor
+  has entered quorum during your troubleshooting. If the Monitor remains out of
+  the quorum, then proceed with the investigations described in this section of
+  the documentation.
+  
 
 **What if the state is ``probing``?**
 

From 7dcfa9132ce6ca97d2c414014fc5de0a0c7e106e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 8 Nov 2023 23:24:06 +1000
Subject: [PATCH 0515/2492] doc/rados: edit t-mon "common issues" (2 of x)

Edit the second part of the section "Most Common Monitor Issues" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 57 +++++++++++--------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 64db2462ae12..6375df50dfb9 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -213,31 +213,38 @@ you should be seeing something similar to::
   troubleshooting the monitor, so check you ``ceph status`` again just to make
   sure. Proceed if the monitor is not yet in the quorum.
 
-**What if the state is ``probing``?**
-
-  This means the monitor is still looking for the other monitors. Every time
-  you start a monitor, the monitor will stay in this state for some time while
-  trying to connect the rest of the monitors specified in the ``monmap``.  The
-  time a monitor will spend in this state can vary. For instance, when on a
-  single-monitor cluster (never do this in production), the monitor will pass
-  through the probing state almost instantaneously.  In a multi-monitor
-  cluster, the monitors will stay in this state until they find enough monitors
-  to form a quorum |---| this means that if you have 2 out of 3 monitors down, the
-  one remaining monitor will stay in this state indefinitely until you bring
-  one of the other monitors up.
-
-  If you have a quorum the starting daemon should be able to find the
-  other monitors quickly, as long as they can be reached. If your
-  monitor is stuck probing and you have gone through with all the communication
-  troubleshooting, then there is a fair chance that the monitor is trying
-  to reach the other monitors on a wrong address. ``mon_status`` outputs the
-  ``monmap`` known to the monitor: check if the other monitor's locations
-  match reality. If they don't, jump to
-  `Recovering a Monitor's Broken monmap`_; if they do, then it may be related
-  to severe clock skews amongst the monitor nodes and you should refer to
-  `Clock Skews`_ first, but if that doesn't solve your problem then it is
-  the time to prepare some logs and reach out to the community (please refer
-  to `Preparing your logs`_ on how to best prepare your logs).
+**What does it mean if  a Monitor's state is ``probing``?**
+
+  If ``ceph health detail`` shows that a Monitor's state is
+  ``probing``, then the Monitor is still looking for the other Monitors. Every
+  Monitor remains in this state for some time when it is started. When a
+  Monitor has connected to the other Monitors specified in the ``monmap``, it
+  ceases to be in the ``probing`` state. The amount of time that a Monitor is
+  in the ``probing`` state depends upon the parameters of the cluster of which
+  it is a part. For example, when a Monitor is a part of a single-monitor
+  cluster (never do this in production), the monitor passes through the probing
+  state almost instantaneously. In a multi-monitor cluster, the Monitors stay
+  in the ``probing`` state until they find enough monitors to form a quorum
+  |---| this means that if two out of three Monitors in the cluster are
+  ``down``, the one remaining Monitor stays in the ``probing``  state
+  indefinitely until you bring one of the other monitors up.
+
+  If quorum has been established, then the Monitor daemon should be able to
+  find the other Monitors quickly, as long as they can be reached. If a Monitor
+  is stuck in the ``probing`` state and you have exhausted the procedures above
+  that describe the troubleshooting of communications between the Monitors,
+  then it is possible that the problem Monitor is trying to reach the other
+  Monitors at a wrong address. ``mon_status`` outputs the ``monmap`` that is
+  known to the monitor: determine whether the other Monitors' locations as
+  specified in the ``monmap`` match the locations of the Monitors in the
+  network. If they do not, see `Recovering a Monitor's Broken monmap`_.
+  If the locations of the Monitors as specified in the ``monmap`` match the
+  locations of the Monitors in the network, then the persistent
+  ``probing`` state could  be related to severe clock skews amongst the monitor
+  nodes.  See `Clock Skews`_.  If the information in `Clock Skews`_ does not
+  bring the Monitor out of the ``probing`` state, then prepare your system logs
+  and ask the Ceph community for help. See `Preparing your logs`_ for
+  information about the proper preparation of logs.
 
 
 **What if state is ``electing``?**

From e6e3fda2efb8f4a4345ba5f939baade844c5d582 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 8 Nov 2023 17:56:29 +1000
Subject: [PATCH 0516/2492] doc/rados: edit t-shooting-mon.rst

Improve the front matter of the section "Troubleshooting Monitors" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 3beb42fdf01a..39e07f2fff2a 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -6,14 +6,13 @@
 
 .. index:: monitor, high availability
 
-If a cluster encounters monitor-related problems, this does not necessarily
-mean that the cluster is in danger of going down. Even if multiple monitors are
-lost, the cluster can still be up and running, as long as there are enough
-surviving monitors to form a quorum.
-
-However serious your cluster's monitor-related problems might be, we recommend
-that you take the following troubleshooting steps.
-
+Even if a cluster experiences monitor-related problems, the cluster is not
+necessarily in danger of going down. If a cluster has lost multiple monitors,
+it can still remain up and running as long as there are enough surviving
+monitors to form a quorum.
+   
+If your cluster is having monitor-related problems, we recommend that you
+consult the following troubleshooting information.
 
 Initial Troubleshooting
 =======================

From 916a22ef031953056771eceb1f49cab7eb746978 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 7 Nov 2023 14:39:50 +0100
Subject: [PATCH 0517/2492] ceph-volume: Revert "ceph-volume: fix raw list for
 lvm devices"

This reverts commit e5e429617c1c27dcd631171f65d30571e32f7266.
This commit introduced a regression, see linked tracker for details.

Fixes: https://tracker.ceph.com/issues/63391

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-volume/ceph_volume/devices/raw/list.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/devices/raw/list.py b/src/ceph-volume/ceph_volume/devices/raw/list.py
index 0f801701b806..c86353b90ecc 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/list.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/list.py
@@ -69,7 +69,7 @@ def __init__(self, argv):
     def generate(self, devs=None):
         logger.debug('Listing block devices via lsblk...')
         info_devices = disk.lsblk_all(abspath=True)
-        if devs is None or devs == []:
+        if not devs or not any(devs):
             # If no devs are given initially, we want to list ALL devices including children and
             # parents. Parent disks with child partitions may be the appropriate device to return if
             # the parent disk has a bluestore header, but children may be the most appropriate
@@ -89,9 +89,6 @@ def generate(self, devs=None):
             # determine whether a parent is bluestore, we should err on the side of not reporting
             # the child so as not to give a false negative.
             info_device = [info for info in info_devices if info['NAME'] == dev][0]
-            if info_device['TYPE'] == 'lvm':
-                # lvm devices are not raw devices
-                continue
             if 'PKNAME' in info_device and info_device['PKNAME'] != "":
                 parent = info_device['PKNAME']
                 try:

From 1357a2412cacba8da8adc54efa0a6b4816a70409 Mon Sep 17 00:00:00 2001
From: Jinkyu Yi <jincreator@jincreator.net>
Date: Wed, 8 Nov 2023 22:08:22 +0900
Subject: [PATCH 0518/2492] cls: fix unable to link with system-installed fmt

Signed-off-by: Jinkyu Yi <jincreator@jincreator.net>
---
 src/cls/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cls/CMakeLists.txt b/src/cls/CMakeLists.txt
index 57d0dace67c5..af2249adeae2 100644
--- a/src/cls/CMakeLists.txt
+++ b/src/cls/CMakeLists.txt
@@ -201,7 +201,7 @@ if (WITH_RADOSGW)
     rgw/cls_rgw_types.cc
     ${CMAKE_SOURCE_DIR}/src/common/ceph_json.cc)
   add_library(cls_rgw SHARED ${cls_rgw_srcs})
-  target_link_libraries(cls_rgw fmt json_spirit)
+  target_link_libraries(cls_rgw ${FMT_LIB} json_spirit)
   target_include_directories(cls_rgw
 	  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
 	  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
@@ -360,7 +360,7 @@ set_target_properties(cls_fifo PROPERTIES
   SOVERSION "1"
   INSTALL_RPATH ""
   CXX_VISIBILITY_PRESET hidden)
-target_link_libraries(cls_fifo fmt)
+target_link_libraries(cls_fifo ${FMT_LIB})
 install(TARGETS cls_fifo DESTINATION ${cls_dir})
 
 # cls_test_remote_reads

From 0e17f8b5aebb51ee69ce0689dabe2a9111055716 Mon Sep 17 00:00:00 2001
From: Matthew Booth <mbooth@redhat.com>
Date: Tue, 7 Nov 2023 19:58:16 +0000
Subject: [PATCH 0519/2492] ceph-volume: fix raw list for non-existent device

ceph-volume should not crash when given a device which doesn't exist.

Fixes: https://tracker.ceph.com/issues/63391

Signed-off-by: Matthew Booth <mbooth@redhat.com>
(cherry picked from commit e009b1a319c834a504947423ae1dd55387809235)
---
 src/ceph-volume/ceph_volume/devices/raw/list.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ceph-volume/ceph_volume/devices/raw/list.py b/src/ceph-volume/ceph_volume/devices/raw/list.py
index c86353b90ecc..dacd980fad60 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/list.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/list.py
@@ -88,7 +88,11 @@ def generate(self, devs=None):
             # parent isn't bluestore, then the child could be a valid bluestore OSD. If we fail to
             # determine whether a parent is bluestore, we should err on the side of not reporting
             # the child so as not to give a false negative.
-            info_device = [info for info in info_devices if info['NAME'] == dev][0]
+            matched_info_devices = [info for info in info_devices if info['NAME'] == dev]
+            if not matched_info_devices:
+                logger.warning('device {} does not exist'.format(dev))
+                continue
+            info_device = matched_info_devices[0]
             if 'PKNAME' in info_device and info_device['PKNAME'] != "":
                 parent = info_device['PKNAME']
                 try:

From 37f8a2c00bab2ee70531cda4240ffaf9b5141c47 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 4 Nov 2023 16:40:21 -0400
Subject: [PATCH 0520/2492] rgw/sal: remove Bucket::get_acl_owner()

was only used under the line by rados and dbstores

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 2 +-
 src/rgw/rgw_sal.h                     | 2 --
 src/rgw/rgw_sal_dbstore.cc            | 2 +-
 src/rgw/rgw_sal_filter.h              | 1 -
 src/rgw/rgw_sal_store.h               | 1 -
 5 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index c555ead9745e..08722cc24b51 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2229,7 +2229,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
   }
 
   std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = meta_obj->get_delete_op();
-  del_op->params.bucket_owner = bucket->get_acl_owner();
+  del_op->params.bucket_owner = bucket->get_info().owner;
   del_op->params.versioning_status = 0;
   if (!remove_objs.empty()) {
     del_op->params.remove_objs = &remove_objs;
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index b222fcd03cbc..2fb2ec98a06e 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -699,8 +699,6 @@ class Bucket {
     virtual bool is_owner(User* user) = 0;
     /** Get the owner of this bucket */
     virtual User* get_owner(void) = 0;
-    /** Get the owner of this bucket in the form of an ACLOwner object */
-    virtual ACLOwner get_acl_owner(void) = 0;
     /** Check in the backing store if this bucket is empty */
     virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) = 0;
     /** Chec k if the given size fits within the quota */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 5140a3bfbe8e..d406289f0488 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -796,7 +796,7 @@ namespace rgw::sal {
     int ret;
 
     std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = meta_obj->get_delete_op();
-    del_op->params.bucket_owner = bucket->get_acl_owner();
+    del_op->params.bucket_owner = bucket->get_info().owner;
     del_op->params.versioning_status = 0;
 
     // Since the data objects are associated with meta obj till
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 34739c3426a2..943170dd0820 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -413,7 +413,6 @@ class FilterBucket : public Bucket {
 		       ceph::real_time mtime, optional_yield y) override;
   virtual bool is_owner(User* user) override;
   virtual User* get_owner(void) override { return user; }
-  virtual ACLOwner get_acl_owner(void) override { return next->get_acl_owner(); }
   virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota,
 			  uint64_t obj_size, optional_yield y,
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index f5efdc3494fe..943830c77fe4 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -104,7 +104,6 @@ class StoreBucket : public Bucket {
     virtual User* get_owner(void) override { return owner; };
     /* Make sure to call get_bucket_info() if you need it first */
     virtual bool is_owner(User* user) override { return (info.owner.compare(user->get_id()) == 0); }
-    virtual ACLOwner get_acl_owner(void) override { return ACLOwner(info.owner); };
     virtual bool empty() const override { return info.bucket.name.empty(); }
     virtual const std::string& get_name() const override { return info.bucket.name; }
     virtual const std::string& get_tenant() const override { return info.bucket.tenant; }

From cd706e469699c4b90ffd3802e64ce2c0543944d4 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Wed, 8 Nov 2023 09:20:53 +0800
Subject: [PATCH 0521/2492] mds: fix and improve MDSCapMatch::match_path()

Just in case the 'target_path' is tailed with '/' it may fail to
match the 'path', such as in case 'path=/foo/' and 'target_path=/foo'
it will fail.

Just try to remove the tailing '/' from 'target_path' and then
also we can simplify the code to make it more readable.

Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/MDSAuthCaps.cc | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/mds/MDSAuthCaps.cc b/src/mds/MDSAuthCaps.cc
index ca4add2f323b..e5d8c4854017 100644
--- a/src/mds/MDSAuthCaps.cc
+++ b/src/mds/MDSAuthCaps.cc
@@ -167,14 +167,29 @@ bool MDSCapMatch::match(string_view target_path,
 
 bool MDSCapMatch::match_path(string_view target_path) const
 {
-  if (path.length()) {
-    if (target_path.find(path) != 0)
+  string _path = path;
+  // drop any tailing /
+  while (_path.length() && _path[_path.length() - 1] == '/') {
+    _path = path.substr(0, _path.length() - 1);
+  }
+
+  if (_path.length()) {
+    if (target_path.find(_path) != 0)
       return false;
-    // if path doesn't already have a trailing /, make sure the target
-    // does so that path=/foo doesn't match target_path=/food
-    if (target_path.length() > path.length() &&
-	path[path.length()-1] != '/' &&
-	target_path[path.length()] != '/')
+    /* In case target_path.find(_path) == 0 && target_path.length() == _path.length():
+     *  path=/foo  _path=/foo target_path=/foo     --> match
+     *  path=/foo/ _path=/foo target_path=/foo     --> match
+     *
+     * In case target_path.find(_path) == 0 && target_path.length() > _path.length():
+     *  path=/foo/ _path=/foo target_path=/foo/    --> match
+     *  path=/foo  _path=/foo target_path=/foo/    --> match
+     *  path=/foo/ _path=/foo target_path=/foo/d   --> match
+     *  path=/foo  _path=/foo target_path=/food    --> mismatch
+     *
+     * All the other cases                         --> mismatch
+     */
+    if (target_path.length() > _path.length() &&
+	target_path[_path.length()] != '/')
       return false;
   }
 

From 516b0999f5bfcdc98c659db2f5aea591a71f7bf4 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 9 Nov 2023 09:02:39 +0800
Subject: [PATCH 0522/2492] test/libcephfs: add test case for slash tailing
 path for access

Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/test/libcephfs/access.cc | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/test/libcephfs/access.cc b/src/test/libcephfs/access.cc
index 1260a23e548d..7f0b1120693a 100644
--- a/src/test/libcephfs/access.cc
+++ b/src/test/libcephfs/access.cc
@@ -114,6 +114,7 @@ TEST(AccessTest, Foo) {
 
 TEST(AccessTest, Path) {
   string good = get_unique_dir("good");
+  string good_slash = get_unique_dir("good_slash") + "/";
   string bad = get_unique_dir("bad");
   string user = "libcephfs_path_test." + stringify(rand());
   struct ceph_mount_info *admin;
@@ -122,11 +123,15 @@ TEST(AccessTest, Path) {
   ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
   ASSERT_EQ(0, ceph_mount(admin, "/"));
   ASSERT_EQ(0, ceph_mkdir(admin, good.c_str(), 0755));
+  ASSERT_EQ(0, ceph_mkdir(admin, good_slash.c_str(), 0755));
   ASSERT_EQ(0, ceph_mkdir(admin, string(good + "/p").c_str(), 0755));
+  ASSERT_EQ(0, ceph_mkdir(admin, string(good_slash + "/p").c_str(), 0755));
   ASSERT_EQ(0, ceph_mkdir(admin, bad.c_str(), 0755));
   ASSERT_EQ(0, ceph_mkdir(admin, string(bad + "/p").c_str(), 0755));
   int fd = ceph_open(admin, string(good + "/q").c_str(), O_CREAT|O_WRONLY, 0755);
   ceph_close(admin, fd);
+  fd = ceph_open(admin, string(good_slash + "/q").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_close(admin, fd);
   fd = ceph_open(admin, string(bad + "/q").c_str(), O_CREAT|O_WRONLY, 0755);
   ceph_close(admin, fd);
   fd = ceph_open(admin, string(bad + "/z").c_str(), O_CREAT|O_WRONLY, 0755);
@@ -137,7 +142,7 @@ TEST(AccessTest, Path) {
   ASSERT_EQ(0, do_mon_command(
       "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
       "\"caps\": [\"mon\", \"allow r\", \"osd\", \"allow rwx\", "
-      "\"mds\", \"allow r, allow rw path=" + good + "\""
+      "\"mds\", \"allow r, allow rw path=" + good + ", allow rw path=" + good_slash + "\""
       "], \"format\": \"json\"}", &key));
 
   struct ceph_mount_info *cmount;
@@ -158,6 +163,16 @@ TEST(AccessTest, Path) {
   ASSERT_GE(ceph_unlink(cmount, string(good + "/y").c_str()), 0);
   ASSERT_GE(ceph_rmdir(cmount, string(good + "/x").c_str()), 0);
 
+  ASSERT_GE(ceph_mkdir(cmount, string(good_slash + "/x").c_str(), 0755), 0);
+  ASSERT_GE(ceph_rmdir(cmount, string(good_slash + "/p").c_str()), 0);
+  ASSERT_GE(ceph_unlink(cmount, string(good_slash + "/q").c_str()), 0);
+  fd = ceph_open(cmount, string(good_slash + "/y").c_str(), O_CREAT|O_WRONLY, 0755);
+  ASSERT_GE(fd, 0);
+  ceph_write(cmount, fd, "bar", 3, 0);
+  ceph_close(cmount, fd);
+  ASSERT_GE(ceph_unlink(cmount, string(good_slash + "/y").c_str()), 0);
+  ASSERT_GE(ceph_rmdir(cmount, string(good_slash + "/x").c_str()), 0);
+
   fd = ceph_open(cmount, string(bad + "/z").c_str(), O_RDONLY, 0644);
   ASSERT_GE(fd, 0);
   ceph_close(cmount, fd);

From 91ed10bc734bd93605b60c87037393d2704a16bc Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 9 Nov 2023 20:20:20 +1000
Subject: [PATCH 0523/2492] doc/rados: edit t-mon "common issues" (3 of x)

Edit the second part of the section "Most Common Monitor Issues" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Follows https://github.com/ceph/ceph/pull/54417.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/dev/mon-elections.rst                     |  2 ++
 .../troubleshooting/troubleshooting-mon.rst   | 32 ++++++++++++-------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/doc/dev/mon-elections.rst b/doc/dev/mon-elections.rst
index 86cfc3803e72..1f346aece4d7 100644
--- a/doc/dev/mon-elections.rst
+++ b/doc/dev/mon-elections.rst
@@ -1,3 +1,5 @@
+.. _dev_mon_elections:
+
 =================
 Monitor Elections
 =================
diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 428f08d1b02a..740de9be017d 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -251,18 +251,26 @@ detail`` returns a message similar to the following::
   information about the proper preparation of logs.
 
 
-**What if state is ``electing``?**
-
-  This means the monitor is in the middle of an election. With recent Ceph
-  releases these typically complete quickly, but at times the monitors can
-  get stuck in what is known as an *election storm*. This can indicate
-  clock skew among the monitor nodes; jump to
-  `Clock Skews`_ for more information. If all your clocks are properly
-  synchronized, you should search the mailing lists and tracker.
-  This is not a state that is likely to persist and aside from
-  (*really*) old bugs there is not an obvious reason besides clock skews on
-  why this would happen.  Worst case, if there are enough surviving mons,
-  down the problematic one while you investigate.
+**What does it mean when a Monitor's state is ``electing``?**
+
+  If ``ceph health detail`` shows that a Monitor's state is ``electing``, the
+  monitor is in the middle of an election. Elections typically complete
+  quickly, but sometimes the monitors can get stuck in what is known as an
+  *election storm*. See :ref:`Monitor Elections <dev_mon_elections>` for more
+  on monitor elections.
+  
+  The presence of election storm might indicate clock skew among the monitor
+  nodes. See `Clock Skews`_ for more information. 
+  
+  If your clocks are properly synchronized, search the mailing lists and bug
+  tracker for issues similar to your issue. The ``electing`` state is not
+  likely to persist. In versions of Ceph after the release of Cuttlefish, there
+  is no obvious reason other than clock skew that explains why an ``electing``
+  state would persist.  
+  
+  It is possible to investigate the cause of a persistent ``electing`` state if
+  you put the problematic Monitor into a ``down`` state while you investigate.
+  This is possible only if there are enough surviving Monitors to form quorum. 
 
 **What if state is ``synchronizing``?**
 

From d0456a9df9856ad253639274702e7d2ad2a25431 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Wed, 20 Sep 2023 10:05:56 +0900
Subject: [PATCH 0524/2492] crimson/os/seastore: store data as a delta using
 mutable extent when overwriting

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/common/options/crimson.yaml.in            |   5 +
 src/crimson/os/seastore/cached_extent.h       |   3 +
 .../os/seastore/object_data_handler.cc        | 162 +++++++++++++++---
 src/crimson/os/seastore/object_data_handler.h |  38 +++-
 src/crimson/os/seastore/transaction_manager.h |  15 ++
 5 files changed, 188 insertions(+), 35 deletions(-)

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index 1007998fade9..8f0af93f3ea8 100644
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -117,3 +117,8 @@ options:
   level: advanced
   desc: Begin fast eviction when the used ratio of the main tier reaches this value.
   default: 0.7
+- name: seastore_data_delta_based_overwrite
+  type: size
+  level: dev
+  desc: overwrite the existing data block based on delta if the original size is smaller than the value, otherwise do overwrite based on remapping, set to 0 to enforce the remap-based overwrite.
+  default: 0
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 6ab19282637f..c3010efe6cb4 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1054,6 +1054,9 @@ class PhysicalNodeMapping {
   }
 
   virtual bool is_stable() const = 0;
+  bool is_zero_reserved() const {
+    return !get_val().is_real();
+  }
 
   virtual ~PhysicalNodeMapping() {}
 protected:
diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 1b0ae5c814ae..29e89d3ddf0d 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -98,7 +98,8 @@ using extent_to_write_list_t = std::list<extent_to_write_t>;
 // Encapsulates extents to be written out using do_remappings.
 struct extent_to_remap_t {
   enum class type_t {
-    REMAP,
+    REMAP1,
+    REMAP2,
     OVERWRITE
   };
   type_t type;
@@ -114,54 +115,75 @@ struct extent_to_remap_t {
   extent_to_remap_t(const extent_to_remap_t &) = delete;
   extent_to_remap_t(extent_to_remap_t &&) = default;
 
-  bool is_remap() const {
-    return type == type_t::REMAP;
+  bool is_remap1() const {
+    return type == type_t::REMAP1;
   }
 
-  bool is_overwrite() const {
+  bool is_remap2() const {
     assert((new_offset != 0) && (pin->get_length() != new_offset + new_len));
+    return type == type_t::REMAP2;
+  }
+
+  bool is_overwrite() const {
     return type == type_t::OVERWRITE;
   }
 
   using remap_entry = TransactionManager::remap_entry;
   remap_entry create_remap_entry() {
-    assert(is_remap());
+    assert(is_remap1());
     return remap_entry(
       new_offset,
       new_len);
   }
 
   remap_entry create_left_remap_entry() {
-    assert(is_overwrite());
+    assert(is_remap2());
     return remap_entry(
       0,
       new_offset);
   }
 
   remap_entry create_right_remap_entry() {
-    assert(is_overwrite());
+    assert(is_remap2());
     return remap_entry(
       new_offset + new_len,
       pin->get_length() - new_offset - new_len);
   }
 
-  static extent_to_remap_t create_remap(
+  static extent_to_remap_t create_remap1(
     LBAMappingRef &&pin, extent_len_t new_offset, extent_len_t new_len) {
-    return extent_to_remap_t(type_t::REMAP,
+    return extent_to_remap_t(type_t::REMAP1,
       std::move(pin), new_offset, new_len);
   }
 
-  static extent_to_remap_t create_overwrite(
+  static extent_to_remap_t create_remap2(
     LBAMappingRef &&pin, extent_len_t new_offset, extent_len_t new_len) {
-    return extent_to_remap_t(type_t::OVERWRITE,
+    return extent_to_remap_t(type_t::REMAP2,
       std::move(pin), new_offset, new_len);
   }
 
+  static extent_to_remap_t create_overwrite(
+    extent_len_t new_offset, extent_len_t new_len, LBAMappingRef p,
+    bufferlist b) {
+    return extent_to_remap_t(type_t::OVERWRITE,
+      nullptr, new_offset, new_len, p->get_key(), p->get_length(), b);
+  }
+
+  uint64_t laddr_start;
+  extent_len_t length;
+  std::optional<bufferlist> bl;
+
 private:
   extent_to_remap_t(type_t type,
     LBAMappingRef &&pin, extent_len_t new_offset, extent_len_t new_len)
     : type(type),
       pin(std::move(pin)), new_offset(new_offset), new_len(new_len) {}
+  extent_to_remap_t(type_t type,
+    LBAMappingRef &&pin, extent_len_t new_offset, extent_len_t new_len,
+    uint64_t ori_laddr, extent_len_t ori_len, std::optional<bufferlist> b)
+    : type(type),
+      pin(std::move(pin)), new_offset(new_offset), new_len(new_len),
+      laddr_start(ori_laddr), length(ori_len), bl(b) {}
 };
 using extent_to_remap_list_t = std::list<extent_to_remap_t>;
 
@@ -222,7 +244,8 @@ struct overwrite_ops_t {
 // prepare to_remap, to_retire, to_insert list
 overwrite_ops_t prepare_ops_list(
   lba_pin_list_t &pins_to_remove,
-  extent_to_write_list_t &to_write) {
+  extent_to_write_list_t &to_write,
+  size_t delta_based_overwrite_max_extent_size) {
   assert(pins_to_remove.size() != 0);
   overwrite_ops_t ops;
   ops.to_remove.swap(pins_to_remove);
@@ -241,7 +264,7 @@ overwrite_ops_t prepare_ops_list(
       assert(to_write.size() > 2);
       assert(front.addr == front.pin->get_key());
       assert(back.addr > back.pin->get_key());
-      ops.to_remap.push_back(extent_to_remap_t::create_overwrite(
+      ops.to_remap.push_back(extent_to_remap_t::create_remap2(
 	std::move(front.pin),
 	front.len,
 	back.addr - front.addr - front.len));
@@ -252,7 +275,7 @@ overwrite_ops_t prepare_ops_list(
       visitted++;
       assert(to_write.size() > 1);
       assert(front.addr == front.pin->get_key());
-      ops.to_remap.push_back(extent_to_remap_t::create_remap(
+      ops.to_remap.push_back(extent_to_remap_t::create_remap1(
 	std::move(front.pin),
 	0,
 	front.len));
@@ -263,7 +286,7 @@ overwrite_ops_t prepare_ops_list(
       assert(to_write.size() > 1);
       assert(back.addr + back.len ==
 	back.pin->get_key() + back.pin->get_length());
-      ops.to_remap.push_back(extent_to_remap_t::create_remap(
+      ops.to_remap.push_back(extent_to_remap_t::create_remap1(
 	std::move(back.pin),
 	back.addr - back.pin->get_key(),
 	back.len));
@@ -271,13 +294,65 @@ overwrite_ops_t prepare_ops_list(
     }
   }
 
-  // prepare to_insert
+  interval_set<uint64_t> pre_alloc_addr_removed, pre_alloc_addr_remapped;
+  if (delta_based_overwrite_max_extent_size) {
+    for (auto &r : ops.to_remove) {
+      if (r->is_stable() && !r->is_zero_reserved()) {
+	pre_alloc_addr_removed.insert(r->get_key(), r->get_length());
+
+      }
+    }
+    for (auto &r : ops.to_remap) {
+      if (r.pin && r.pin->is_stable() && !r.pin->is_zero_reserved()) {
+	pre_alloc_addr_remapped.insert(r.pin->get_key(), r.pin->get_length());
+      }
+    }
+  }
+
+  // prepare to insert
+  extent_to_remap_list_t to_remap;
   for (auto &region : to_write) {
     if (region.is_data()) {
       visitted++;
       assert(region.to_write.has_value());
-      ops.to_insert.push_back(extent_to_insert_t::create_data(
-	region.addr, region.len, region.to_write));
+      int erased_num = 0;
+      if (pre_alloc_addr_removed.contains(region.addr, region.len) &&
+	  region.len <= delta_based_overwrite_max_extent_size) {
+	erased_num = std::erase_if(
+	  ops.to_remove,
+	  [&region, &to_remap](auto &r) {
+	    interval_set<uint64_t> range;
+	    range.insert(r->get_key(), r->get_length());
+	    if (range.contains(region.addr, region.len)) {
+	      to_remap.push_back(extent_to_remap_t::create_overwrite(
+		0, region.len, std::move(r), *region.to_write));
+	      return true;
+	    }
+	    return false;
+	  });
+	// if the size of the region is wider than the ragne from the enry in to_remove,
+	// we create a separated extent in the original way.
+      } else if (pre_alloc_addr_remapped.contains(region.addr, region.len) &&
+		 region.len <= delta_based_overwrite_max_extent_size) {
+	erased_num = std::erase_if(
+	  ops.to_remap,
+	  [&region, &to_remap](auto &r) {
+	    interval_set<uint64_t> range;
+	    range.insert(r.pin->get_key(), r.pin->get_length());
+	    if (range.contains(region.addr, region.len)) {
+	      to_remap.push_back(extent_to_remap_t::create_overwrite(
+		region.addr - range.begin().get_start(), region.len,
+		std::move(r.pin), *region.to_write));
+	      return true;
+	    }
+	    return false;
+	  });
+	assert(erased_num > 0);
+      }
+      if (erased_num == 0)  {
+	ops.to_insert.push_back(extent_to_insert_t::create_data(
+	  region.addr, region.len, region.to_write));
+      }
     } else if (region.is_zero()) {
       visitted++;
       assert(!(region.to_write.has_value()));
@@ -285,6 +360,7 @@ overwrite_ops_t prepare_ops_list(
 	region.addr, region.len));
     }
   }
+  ops.to_remap.splice(ops.to_remap.end(), to_remap);
 
   logger().debug(
     "to_remap list size: {}"
@@ -334,6 +410,22 @@ void splice_extent_to_write(
   }
 }
 
+ceph::bufferlist ObjectDataBlock::get_delta() {
+  ceph::bufferlist bl;
+  encode(delta, bl);
+  return bl;
+}
+
+void ObjectDataBlock::apply_delta(const ceph::bufferlist &bl) {
+  auto biter = bl.begin();
+  decltype(delta) deltas;
+  decode(deltas, biter);
+  for (auto &&d : deltas) {
+    auto iter = d.bl.cbegin();
+    iter.copy(d.len, get_bptr().c_str() + d.offset);
+  }
+}
+
 /// Creates remap extents in to_remap
 ObjectDataHandler::write_ret do_remappings(
   context_t ctx,
@@ -342,7 +434,7 @@ ObjectDataHandler::write_ret do_remappings(
   return trans_intr::do_for_each(
     to_remap,
     [ctx](auto &region) {
-      if (region.is_remap()) {
+      if (region.is_remap1()) {
         return ctx.tm.remap_pin<ObjectDataBlock, 1>(
           ctx.t,
           std::move(region.pin),
@@ -355,6 +447,22 @@ ObjectDataHandler::write_ret do_remappings(
           return ObjectDataHandler::write_iertr::now();
         });
       } else if (region.is_overwrite()) {
+	return ctx.tm.get_mutable_extent_by_laddr<ObjectDataBlock>(
+	  ctx.t,
+	  region.laddr_start,
+	  region.length
+	).handle_error_interruptible(
+	  TransactionManager::base_iertr::pass_further{},
+	  crimson::ct_error::assert_all{
+	    "ObjectDataHandler::do_remapping hit invalid error"
+	  }
+	).si_then([&region](auto extent) {
+	  extent_len_t off = region.new_offset;
+	  assert(region.bl->length() == region.new_len);
+	  extent->overwrite(off, *region.bl);
+	  return ObjectDataHandler::write_iertr::now();
+	});
+      } else if (region.is_remap2()) {
         return ctx.tm.remap_pin<ObjectDataBlock, 2>(
           ctx.t,
           std::move(region.pin),
@@ -960,7 +1068,7 @@ ObjectDataHandler::clear_ret ObjectDataHandler::trim_data_reservation(
   return seastar::do_with(
     lba_pin_list_t(),
     extent_to_write_list_t(),
-    [ctx, size, &object_data](auto &pins, auto &to_write) {
+    [ctx, size, &object_data, this](auto &pins, auto &to_write) {
       LOG_PREFIX(ObjectDataHandler::trim_data_reservation);
       DEBUGT("object_data: {}~{}",
 	     ctx.t,
@@ -1038,9 +1146,10 @@ ObjectDataHandler::clear_ret ObjectDataHandler::trim_data_reservation(
             });
           }
 	}
-      }).si_then([ctx, size, &to_write, &object_data, &pins] {
+      }).si_then([ctx, size, &to_write, &object_data, &pins, this] {
         return seastar::do_with(
-          prepare_ops_list(pins, to_write),
+          prepare_ops_list(pins, to_write,
+	    delta_based_overwrite_max_extent_size),
           [ctx, size, &object_data](auto &ops) {
             return do_remappings(ctx, ops.to_remap
             ).si_then([ctx, &ops] {
@@ -1162,7 +1271,7 @@ ObjectDataHandler::write_ret ObjectDataHandler::overwrite(
   return seastar::do_with(
     std::move(_pins),
     extent_to_write_list_t(),
-    [ctx, len, offset, overwrite_plan, bl=std::move(bl)]
+    [ctx, len, offset, overwrite_plan, bl=std::move(bl), this]
     (auto &pins, auto &to_write) mutable
   {
     LOG_PREFIX(ObjectDataHandler::overwrite);
@@ -1178,7 +1287,7 @@ ObjectDataHandler::write_ret ObjectDataHandler::overwrite(
       pins.front(),
       overwrite_plan
     ).si_then([ctx, len, offset, overwrite_plan, bl=std::move(bl),
-               &to_write, &pins](auto p) mutable {
+               &to_write, &pins, this](auto p) mutable {
       auto &[left_extent, headptr] = p;
       if (left_extent) {
         ceph_assert(left_extent->addr == overwrite_plan.pin_begin);
@@ -1195,7 +1304,7 @@ ObjectDataHandler::write_ret ObjectDataHandler::overwrite(
                  pin_begin=overwrite_plan.pin_begin,
                  pin_end=overwrite_plan.pin_end,
                  bl=std::move(bl), headptr=std::move(headptr),
-                 &to_write, &pins](auto p) mutable {
+                 &to_write, &pins, this](auto p) mutable {
         auto &[right_extent, tailptr] = p;
         if (bl.has_value()) {
           auto write_offset = offset;
@@ -1232,7 +1341,8 @@ ObjectDataHandler::write_ret ObjectDataHandler::overwrite(
         assert(pin_end == to_write.back().get_end_addr());
 
         return seastar::do_with(
-          prepare_ops_list(pins, to_write),
+          prepare_ops_list(pins, to_write,
+	    delta_based_overwrite_max_extent_size),
           [ctx](auto &ops) {
             return do_remappings(ctx, ops.to_remap
             ).si_then([ctx, &ops] {
diff --git a/src/crimson/os/seastore/object_data_handler.h b/src/crimson/os/seastore/object_data_handler.h
index b5f432d5ac77..eaa05da8d545 100644
--- a/src/crimson/os/seastore/object_data_handler.h
+++ b/src/crimson/os/seastore/object_data_handler.h
@@ -16,9 +16,25 @@
 
 namespace crimson::os::seastore {
 
+struct block_delta_t {
+  uint64_t offset = 0;
+  extent_len_t len = 0;
+  bufferlist bl;
+
+  DENC(block_delta_t, v, p) {
+    DENC_START(1, 1, p);
+    denc(v.offset, p);
+    denc(v.len, p);
+    denc(v.bl, p);
+    DENC_FINISH(p);
+  }
+};
+
 struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
   using Ref = TCachedExtentRef<ObjectDataBlock>;
 
+  std::vector<block_delta_t> delta = {};
+
   explicit ObjectDataBlock(ceph::bufferptr &&ptr)
     : LogicalCachedExtent(std::move(ptr)) {}
   explicit ObjectDataBlock(const ObjectDataBlock &other)
@@ -35,16 +51,15 @@ struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
     return TYPE;
   }
 
-  ceph::bufferlist get_delta() final {
-    /* Currently, we always allocate fresh ObjectDataBlock's rather than
-     * mutating existing ones. */
-    ceph_assert(0 == "Should be impossible");
+  void overwrite(extent_len_t offset, bufferlist bl) {
+    auto iter = bl.cbegin();
+    iter.copy(bl.length(), get_bptr().c_str() + offset);
+    delta.push_back({offset, bl.length(), bl});
   }
 
-  void apply_delta(const ceph::bufferlist &bl) final {
-    // See get_delta()
-    ceph_assert(0 == "Should be impossible");
-  }
+  ceph::bufferlist get_delta() final;
+
+  void apply_delta(const ceph::bufferlist &bl) final;
 };
 using ObjectDataBlockRef = TCachedExtentRef<ObjectDataBlock>;
 
@@ -52,7 +67,9 @@ class ObjectDataHandler {
 public:
   using base_iertr = TransactionManager::base_iertr;
 
-  ObjectDataHandler(uint32_t mos) : max_object_size(mos) {}
+  ObjectDataHandler(uint32_t mos) : max_object_size(mos),
+    delta_based_overwrite_max_extent_size(
+      crimson::common::get_conf<Option::size_t>("seastore_data_delta_based_overwrite")) {}
 
   struct context_t {
     TransactionManager &tm;
@@ -147,10 +164,13 @@ class ObjectDataHandler {
    * these regions and remove this assumption.
    */
   const uint32_t max_object_size = 0;
+  extent_len_t delta_based_overwrite_max_extent_size = 0; // enable only if rbm is used
 };
 
 }
 
+WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::block_delta_t)
+
 #if FMT_VERSION >= 90000
 template <> struct fmt::formatter<crimson::os::seastore::ObjectDataBlock> : fmt::ostream_formatter {};
 #endif
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index b708e0a9f203..34c87a404c4d 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -316,6 +316,21 @@ class TransactionManager : public ExtentCallbackInterface {
     });
   }
 
+  template <typename T>
+  read_extent_ret<T> get_mutable_extent_by_laddr(Transaction &t, laddr_t laddr, extent_len_t len) {
+    return get_pin(t, laddr
+    ).si_then([this, &t, len](auto pin) {
+      ceph_assert(pin->is_stable());
+      ceph_assert(!pin->is_clone());
+      ceph_assert(pin->get_length() == len);
+      return this->read_pin<T>(t, std::move(pin));
+    }).si_then([this, &t](auto extent) {
+      auto ext = get_mutable_extent(t, extent)->template cast<T>();
+      return alloc_extent_iertr::make_ready_future<TCachedExtentRef<T>>(
+	std::move(ext));
+    });
+  }
+
   /**
    * remap_pin
    *

From b48e662a9ee38f87bb77b661612d56a08326c728 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Fri, 22 Sep 2023 18:03:18 +0900
Subject: [PATCH 0525/2492] test/crimsons/seastore/object_data_handler: add
 overwrite test cases

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 .../seastore/test_object_data_handler.cc      | 381 +++++++++++++-----
 1 file changed, 285 insertions(+), 96 deletions(-)

diff --git a/src/test/crimson/seastore/test_object_data_handler.cc b/src/test/crimson/seastore/test_object_data_handler.cc
index 47607643946e..ae44cb94a62d 100644
--- a/src/test/crimson/seastore/test_object_data_handler.cc
+++ b/src/test/crimson/seastore/test_object_data_handler.cc
@@ -127,14 +127,19 @@ struct object_data_handler_test_t:
 	offset,
 	len));
     with_trans_intr(t, [&](auto &t) {
-      return ObjectDataHandler(MAX_OBJECT_SIZE).write(
-        ObjectDataHandler::context_t{
-          *tm,
-          t,
-          *onode,
-        },
-        offset,
-        bl);
+      return seastar::do_with(
+	std::move(bl),
+	ObjectDataHandler(MAX_OBJECT_SIZE),
+	[=, this, &t](auto &bl, auto &objhandler) {
+	  return objhandler.write(
+	    ObjectDataHandler::context_t{
+	      *tm,
+	      t,
+	      *onode,
+	    },
+	    offset,
+	    bl);
+	});
     }).unsafe_get0();
   }
   void write(objaddr_t offset, extent_len_t len, char fill) {
@@ -150,13 +155,17 @@ struct object_data_handler_test_t:
 	0,
 	size - offset);
       with_trans_intr(t, [&](auto &t) {
-        return ObjectDataHandler(MAX_OBJECT_SIZE).truncate(
-          ObjectDataHandler::context_t{
-            *tm,
-            t,
-            *onode
-          },
-          offset);
+      return seastar::do_with(
+	ObjectDataHandler(MAX_OBJECT_SIZE),
+	[=, this, &t](auto &objhandler) {
+	  return objhandler.truncate(
+	    ObjectDataHandler::context_t{
+	      *tm,
+	      t,
+	      *onode
+	    },
+	    offset);
+	});
       }).unsafe_get0();
     }
     size = offset;
@@ -222,21 +231,15 @@ struct object_data_handler_test_t:
     size = 0;
     return tm_teardown();
   }
-};
 
-TEST_P(object_data_handler_test_t, single_write)
-{
-  run_async([this] {
-    write(1<<20, 8<<10, 'c');
-
-    read_near(1<<20, 8<<10, 1);
-    read_near(1<<20, 8<<10, 512);
-  });
-}
+  void set_overwrite_threshold() {
+    crimson::common::local_conf().set_val("seastore_data_delta_based_overwrite", "131072").get();
+  }
+  void unset_overwrite_threshold() {
+    crimson::common::local_conf().set_val("seastore_data_delta_based_overwrite", "0").get();
+  }
 
-TEST_P(object_data_handler_test_t, multi_write)
-{
-  run_async([this] {
+  void test_multi_write() {
     write((1<<20) - (4<<10), 4<<10, 'a');
     write(1<<20, 4<<10, 'b');
     write((1<<20) + (4<<10), 4<<10, 'c');
@@ -246,12 +249,9 @@ TEST_P(object_data_handler_test_t, multi_write)
 
     read_near((1<<20)-(4<<10), 12<<10, 1);
     read_near((1<<20)-(4<<10), 12<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, write_hole)
-{
-  run_async([this] {
+  void test_write_hole() {
     write((1<<20) - (4<<10), 4<<10, 'a');
     // hole at 1<<20
     write((1<<20) + (4<<10), 4<<10, 'c');
@@ -261,23 +261,17 @@ TEST_P(object_data_handler_test_t, write_hole)
 
     read_near((1<<20)-(4<<10), 12<<10, 1);
     read_near((1<<20)-(4<<10), 12<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, overwrite_single)
-{
-  run_async([this] {
+  void test_overwrite_single() {
     write((1<<20), 4<<10, 'a');
     write((1<<20), 4<<10, 'c');
 
     read_near(1<<20, 4<<10, 1);
     read_near(1<<20, 4<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, overwrite_double)
-{
-  run_async([this] {
+  void test_overwrite_double() {
     write((1<<20), 4<<10, 'a');
     write((1<<20)+(4<<10), 4<<10, 'c');
     write((1<<20), 8<<10, 'b');
@@ -290,12 +284,9 @@ TEST_P(object_data_handler_test_t, overwrite_double)
 
     read_near((1<<20) + (4<<10), 4<<10, 1);
     read_near((1<<20) + (4<<10), 4<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, overwrite_partial)
-{
-  run_async([this] {
+  void test_overwrite_partial() {
     write((1<<20), 12<<10, 'a');
     read_near(1<<20, 12<<10, 1);
 
@@ -315,12 +306,9 @@ TEST_P(object_data_handler_test_t, overwrite_partial)
 
     read_near((1<<20) + (4<<10), 4<<10, 1);
     read_near((1<<20) + (4<<10), 4<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, unaligned_write)
-{
-  run_async([this] {
+  void test_unaligned_write() {
     objaddr_t base = 1<<20;
     write(base, (4<<10)+(1<<10), 'a');
     read_near(base-(4<<10), 12<<10, 512);
@@ -332,12 +320,9 @@ TEST_P(object_data_handler_test_t, unaligned_write)
     base = (1<<20) + (128<<10);
     write(base-(1<<10), (4<<10)+(2<<20), 'c');
     read_near(base-(4<<10), 12<<10, 512);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, unaligned_overwrite)
-{
-  run_async([this] {
+  void test_unaligned_overwrite() {
     objaddr_t base = 1<<20;
     write(base, (128<<10) + (16<<10), 'x');
 
@@ -353,12 +338,9 @@ TEST_P(object_data_handler_test_t, unaligned_overwrite)
     read_near(base-(4<<10), 12<<10, 2<<10);
 
     read(base, (128<<10) + (16<<10));
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, truncate)
-{
-  run_async([this] {
+  void test_truncate() {
     objaddr_t base = 1<<20;
     write(base, 8<<10, 'a');
     write(base+(8<<10), 8<<10, 'b');
@@ -375,11 +357,9 @@ TEST_P(object_data_handler_test_t, truncate)
 
     truncate(base - (12<<10));
     read(base, 64<<10);
-  });
-}
+  }
 
-TEST_P(object_data_handler_test_t, no_split) {
-  run_async([this] {
+  void write_same() {
     write(0, 8<<10, 'x');
     write(0, 8<<10, 'a');
 
@@ -387,14 +367,197 @@ TEST_P(object_data_handler_test_t, no_split) {
     EXPECT_EQ(pins.size(), 1);
 
     read(0, 8<<10);
+  }
+
+  void write_right() {
+    write(0, 128<<10, 'x');
+    write(64<<10, 60<<10, 'a');
+  }
+
+  void write_left() {
+    write(0, 128<<10, 'x');
+    write(4<<10, 60<<10, 'a');
+  }
+
+  void write_right_left() {
+    write(0, 128<<10, 'x');
+    write(48<<10, 32<<10, 'a');
+  }
+
+  void multiple_write() {
+    write(0, 128<<10, 'x');
+
+    auto t = create_mutate_transaction();
+    // normal split
+    write(*t, 120<<10, 4<<10, 'a');
+    // not aligned right
+    write(*t, 4<<10, 5<<10, 'b');
+    // split right extent of last split result
+    write(*t, 32<<10, 4<<10, 'c');
+    // non aligned overwrite
+    write(*t, 13<<10, 4<<10, 'd');
+
+    write(*t, 64<<10, 32<<10, 'e');
+    // not split right
+    write(*t, 60<<10, 8<<10, 'f');
+
+    submit_transaction(std::move(t));
+  }
+};
+
+TEST_P(object_data_handler_test_t, single_write)
+{
+  run_async([this] {
+    write(1<<20, 8<<10, 'c');
+
+    read_near(1<<20, 8<<10, 1);
+    read_near(1<<20, 8<<10, 512);
   });
 }
 
-TEST_P(object_data_handler_test_t, split_left) {
+TEST_P(object_data_handler_test_t, multi_write)
+{
   run_async([this] {
-    write(0, 128<<10, 'x');
+    test_multi_write();
+  });
+}
 
-    write(64<<10, 60<<10, 'a');
+TEST_P(object_data_handler_test_t, delta_over_multi_write)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_multi_write();
+  });
+}
+
+TEST_P(object_data_handler_test_t, write_hole)
+{
+  run_async([this] {
+    test_write_hole();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_write_hole)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_write_hole();
+  });
+}
+
+TEST_P(object_data_handler_test_t, overwrite_single)
+{
+  run_async([this] {
+    test_overwrite_single();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_overwrite_single)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_overwrite_single();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, overwrite_double)
+{
+  run_async([this] {
+    test_overwrite_double();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_overwrite_double)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_overwrite_double();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, overwrite_partial)
+{
+  run_async([this] {
+    test_overwrite_partial();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_overwrite_partial)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_overwrite_partial();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, unaligned_write)
+{
+  run_async([this] {
+    test_unaligned_write();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_unaligned_write)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_unaligned_write();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, unaligned_overwrite)
+{
+  run_async([this] {
+    test_unaligned_overwrite();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_unaligned_overwrite)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_unaligned_overwrite();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, truncate)
+{
+  run_async([this] {
+    test_truncate();
+  });
+}
+
+TEST_P(object_data_handler_test_t, delta_over_truncate)
+{
+  run_async([this] {
+    set_overwrite_threshold();
+    test_truncate();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, no_remap) {
+  run_async([this] {
+    write_same();
+  });
+}
+
+TEST_P(object_data_handler_test_t, no_overwrite) {
+  run_async([this] {
+    set_overwrite_threshold();
+    write_same();
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, remap_left) {
+  run_async([this] {
+    write_right();
 
     auto pins = get_mappings(0, 128<<10);
     EXPECT_EQ(pins.size(), 2);
@@ -410,10 +573,21 @@ TEST_P(object_data_handler_test_t, split_left) {
   });
 }
 
-TEST_P(object_data_handler_test_t, split_right) {
+TEST_P(object_data_handler_test_t, overwrite_right) {
   run_async([this] {
-    write(0, 128<<10, 'x');
-    write(4<<10, 60<<10, 'a');
+    set_overwrite_threshold();
+    write_right();
+
+    auto pins = get_mappings(0, 128<<10);
+    EXPECT_EQ(pins.size(), 1);
+    read(0, 128<<10);
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, remap_right) {
+  run_async([this] {
+    write_left();
 
     auto pins = get_mappings(0, 128<<10);
     EXPECT_EQ(pins.size(), 2);
@@ -428,10 +602,21 @@ TEST_P(object_data_handler_test_t, split_right) {
     read(0, 128<<10);
   });
 }
-TEST_P(object_data_handler_test_t, split_left_right) {
+
+TEST_P(object_data_handler_test_t, overwrite_left) {
   run_async([this] {
-    write(0, 128<<10, 'x');
-    write(48<<10, 32<<10, 'a');
+    set_overwrite_threshold();
+    write_left();
+    auto pins = get_mappings(0, 128<<10);
+    EXPECT_EQ(pins.size(), 1);
+    read(0, 128<<10);
+    unset_overwrite_threshold();
+  });
+}
+
+TEST_P(object_data_handler_test_t, remap_right_left) {
+  run_async([this] {
+    write_right_left();
 
     auto pins = get_mappings(0, 128<<10);
     EXPECT_EQ(pins.size(), 3);
@@ -445,26 +630,21 @@ TEST_P(object_data_handler_test_t, split_left_right) {
     }
   });
 }
-TEST_P(object_data_handler_test_t, multiple_split) {
-  run_async([this] {
-    write(0, 128<<10, 'x');
-
-    auto t = create_mutate_transaction();
-    // normal split
-    write(*t, 120<<10, 4<<10, 'a');
-    // not aligned right
-    write(*t, 4<<10, 5<<10, 'b');
-    // split right extent of last split result
-    write(*t, 32<<10, 4<<10, 'c');
-    // non aligned overwrite
-    write(*t, 13<<10, 4<<10, 'd');
-
-    write(*t, 64<<10, 32<<10, 'e');
-    // not split right
-    write(*t, 60<<10, 8<<10, 'f');
 
-    submit_transaction(std::move(t));
+TEST_P(object_data_handler_test_t, overwrite_right_left) {
+  run_async([this] {
+    set_overwrite_threshold();
+    write_right_left();
+    auto pins = get_mappings(0, 128<<10);
+    EXPECT_EQ(pins.size(), 1);
+    read(0, 128<<10);
+    unset_overwrite_threshold();
+  });
+}
 
+TEST_P(object_data_handler_test_t, multiple_remap) {
+  run_async([this] {
+    multiple_write();
     auto pins = get_mappings(0, 128<<10);
     EXPECT_EQ(pins.size(), 10);
 
@@ -480,6 +660,17 @@ TEST_P(object_data_handler_test_t, multiple_split) {
   });
 }
 
+TEST_P(object_data_handler_test_t, multiple_overwrite) {
+  run_async([this] {
+    set_overwrite_threshold();
+    multiple_write();
+    auto pins = get_mappings(0, 128<<10);
+    EXPECT_EQ(pins.size(), 1);
+    read(0, 128<<10);
+    unset_overwrite_threshold();
+  });
+}
+
 INSTANTIATE_TEST_SUITE_P(
   object_data_handler_test,
   object_data_handler_test_t,
@@ -488,5 +679,3 @@ INSTANTIATE_TEST_SUITE_P(
     "circularbounded"
   )
 );
-
-

From 99a6a32fa4b7d3a49520889da4b9c81be71a0ab4 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Fri, 3 Nov 2023 15:47:24 +0900
Subject: [PATCH 0526/2492] crimson/os/seastore/object_data_handler: prevent
 from being overwritten if cloned

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/backref/btree_backref_manager.h       | 4 ++++
 src/crimson/os/seastore/cached_extent.h                       | 1 +
 src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h | 4 ++++
 src/crimson/os/seastore/object_data_handler.cc                | 4 ++--
 src/crimson/os/seastore/transaction_manager.h                 | 2 +-
 5 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/seastore/backref/btree_backref_manager.h b/src/crimson/os/seastore/backref/btree_backref_manager.h
index 952e78b65189..38084bb00e69 100644
--- a/src/crimson/os/seastore/backref/btree_backref_manager.h
+++ b/src/crimson/os/seastore/backref/btree_backref_manager.h
@@ -35,6 +35,10 @@ class BtreeBackrefMapping : public BtreeNodeMapping<paddr_t, laddr_t> {
     return type;
   }
 
+  bool is_clone() const final {
+    return false;
+  }
+
 protected:
   std::unique_ptr<BtreeNodeMapping<paddr_t, laddr_t>> _duplicate(
     op_context_t<paddr_t> ctx) const final {
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index c3010efe6cb4..5fcb8142527d 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1054,6 +1054,7 @@ class PhysicalNodeMapping {
   }
 
   virtual bool is_stable() const = 0;
+  virtual bool is_clone() const = 0;
   bool is_zero_reserved() const {
     return !get_val().is_real();
   }
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
index 79d21b363af3..1c907f76d311 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
@@ -137,6 +137,10 @@ class BtreeLBAMapping : public BtreeNodeMapping<laddr_t, paddr_t> {
     return intermediate_length;
   }
 
+  bool is_clone() const final {
+    return get_map_val().refcount > 1;
+  }
+
 protected:
   std::unique_ptr<BtreeNodeMapping<laddr_t, paddr_t>> _duplicate(
     op_context_t<laddr_t> ctx) const final {
diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 29e89d3ddf0d..3333fcfad9d4 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -323,7 +323,7 @@ overwrite_ops_t prepare_ops_list(
 	  [&region, &to_remap](auto &r) {
 	    interval_set<uint64_t> range;
 	    range.insert(r->get_key(), r->get_length());
-	    if (range.contains(region.addr, region.len)) {
+	    if (range.contains(region.addr, region.len) && !r->is_clone()) {
 	      to_remap.push_back(extent_to_remap_t::create_overwrite(
 		0, region.len, std::move(r), *region.to_write));
 	      return true;
@@ -339,7 +339,7 @@ overwrite_ops_t prepare_ops_list(
 	  [&region, &to_remap](auto &r) {
 	    interval_set<uint64_t> range;
 	    range.insert(r.pin->get_key(), r.pin->get_length());
-	    if (range.contains(region.addr, region.len)) {
+	    if (range.contains(region.addr, region.len) && !r.pin->is_clone()) {
 	      to_remap.push_back(extent_to_remap_t::create_overwrite(
 		region.addr - range.begin().get_start(), region.len,
 		std::move(r.pin), *region.to_write));
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 34c87a404c4d..911ebbf6008f 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -320,7 +320,7 @@ class TransactionManager : public ExtentCallbackInterface {
   read_extent_ret<T> get_mutable_extent_by_laddr(Transaction &t, laddr_t laddr, extent_len_t len) {
     return get_pin(t, laddr
     ).si_then([this, &t, len](auto pin) {
-      ceph_assert(pin->is_stable());
+      ceph_assert(pin->is_stable() && !pin->is_zero_reserved());
       ceph_assert(!pin->is_clone());
       ceph_assert(pin->get_length() == len);
       return this->read_pin<T>(t, std::move(pin));

From eb4cac740e75474b944bb3bed842776df1e516d4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 6 Nov 2023 20:26:28 -0500
Subject: [PATCH 0527/2492] rgw/sal: Bucket owner as rgw_user

stop maintaining a User pointer for the bucket owner, and use the
stored rgw_user owner directly

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc             |  21 ----
 src/rgw/driver/d4n/rgw_sal_d4n.h              |   4 +-
 src/rgw/driver/posix/rgw_sal_posix.cc         |  21 ++--
 src/rgw/driver/posix/rgw_sal_posix.h          |  17 ++-
 src/rgw/driver/rados/rgw_bucket.cc            |  36 +++---
 src/rgw/driver/rados/rgw_cr_rados.h           |   2 +-
 src/rgw/driver/rados/rgw_cr_tools.cc          |   3 +-
 src/rgw/driver/rados/rgw_notify.cc            |   3 +-
 .../driver/rados/rgw_object_expirer_core.cc   |   2 +-
 src/rgw/driver/rados/rgw_rest_bucket.cc       |   4 +-
 src/rgw/driver/rados/rgw_rest_log.cc          |  10 +-
 src/rgw/driver/rados/rgw_sal_rados.cc         |  50 ++++----
 src/rgw/driver/rados/rgw_sal_rados.h          |  28 +----
 src/rgw/driver/rados/rgw_tools.cc             |   4 +-
 src/rgw/driver/rados/rgw_user.cc              |   4 +-
 src/rgw/rgw_admin.cc                          | 107 +++++++++---------
 src/rgw/rgw_bucket.cc                         |   2 +-
 src/rgw/rgw_lc.cc                             |  28 +----
 src/rgw/rgw_lua_request.cc                    |   2 +-
 src/rgw/rgw_op.cc                             |  60 ++++------
 src/rgw/rgw_orphan.cc                         |  11 +-
 src/rgw/rgw_quota.cc                          |   6 +-
 src/rgw/rgw_rest_pubsub.cc                    |  12 +-
 src/rgw/rgw_rest_ratelimit.cc                 |   6 +-
 src/rgw/rgw_rest_s3.cc                        |   2 +-
 src/rgw/rgw_rest_swift.cc                     |   6 +-
 src/rgw/rgw_rest_usage.cc                     |  10 +-
 src/rgw/rgw_sal.h                             |  14 +--
 src/rgw/rgw_sal_dbstore.cc                    |  17 ++-
 src/rgw/rgw_sal_dbstore.h                     |  24 +---
 src/rgw/rgw_sal_filter.cc                     |  19 ++--
 src/rgw/rgw_sal_filter.h                      |  21 ++--
 src/rgw/rgw_sal_store.h                       |  12 +-
 src/rgw/rgw_swift_auth.cc                     |   2 +-
 src/rgw/rgw_user.cc                           |   2 +-
 src/test/rgw/test_d4n_filter.cc               |   9 +-
 src/test/rgw/test_rgw_lua.cc                  |  24 ++--
 37 files changed, 221 insertions(+), 384 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index efeb281c3cea..a25d5bbdfd95 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -326,7 +326,6 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix
     ldpp_dout(dpp, 20) << "D4N Filter: Cache get object operation failed." << dendl;
   } else {
     /* Set metadata locally */
-    RGWQuotaInfo quota_info;
     RGWObjState* astate;
     source->get_obj_state(dpp, &astate, y);
 
@@ -343,16 +342,9 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix
 	source->set_instance(it->second);
       } else if (!std::strcmp(it->first.data(), "source_zone_short_id")) {
 	astate->zone_short_id = static_cast<uint32_t>(std::stoul(it->second));
-      } else if (!std::strcmp(it->first.data(), "user_quota.max_size")) {
-        quota_info.max_size = std::stoull(it->second);
-      } else if (!std::strcmp(it->first.data(), "user_quota.max_objects")) {
-        quota_info.max_objects = std::stoull(it->second);
-      } else if (!std::strcmp(it->first.data(), "max_buckets")) {
-        source->get_bucket()->get_owner()->set_max_buckets(std::stoull(it->second));
       }
     }
 
-    source->get_bucket()->get_owner()->set_info(quota_info);
     source->set_obj_state(*astate);
    
     /* Set attributes locally */
@@ -491,19 +483,6 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
     bl.clear();
   }
 
-  RGWUserInfo info = obj->get_bucket()->get_owner()->get_info();
-  bl.append(std::to_string(info.quota.user_quota.max_size));
-  baseAttrs.insert({"user_quota.max_size", bl});
-  bl.clear();
-
-  bl.append(std::to_string(info.quota.user_quota.max_objects));
-  baseAttrs.insert({"user_quota.max_objects", bl});
-  bl.clear();
-
-  bl.append(std::to_string(obj->get_bucket()->get_owner()->get_max_buckets()));
-  baseAttrs.insert({"max_buckets", bl});
-  bl.clear();
-
   baseAttrs.insert(attrs.begin(), attrs.end());
 
   int setObjReturn = filter->get_d4n_cache()->setObject(obj->get_key().get_oid(), &baseAttrs);
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index a54371f6cc64..cebae1eaf6ba 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -78,8 +78,8 @@ class D4NFilterBucket : public FilterBucket {
     D4NFilterDriver* filter;
 
   public:
-    D4NFilterBucket(std::unique_ptr<Bucket> _next, User* _user, D4NFilterDriver* _filter) :
-      FilterBucket(std::move(_next), _user), 
+    D4NFilterBucket(std::unique_ptr<Bucket> _next, D4NFilterDriver* _filter) :
+      FilterBucket(std::move(_next)),
       filter(_filter) {}
     virtual ~D4NFilterBucket() = default;
    
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index b72c7868600b..fc48c4f8fa8c 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -348,16 +348,16 @@ std::unique_ptr<Object> POSIXDriver::get_object(const rgw_obj_key& k)
   return std::make_unique<POSIXObject>(this, k);
 }
 
-int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int POSIXDriver::load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
-  *bucket = std::make_unique<POSIXBucket>(this, root_fd, b, u);
+  *bucket = std::make_unique<POSIXBucket>(this, root_fd, b);
   return (*bucket)->load_bucket(dpp, y);
 }
 
-std::unique_ptr<Bucket> POSIXDriver::get_bucket(User* u, const RGWBucketInfo& i)
+std::unique_ptr<Bucket> POSIXDriver::get_bucket(const RGWBucketInfo& i)
 {
   /* Don't need to fetch the bucket info, use the provided one */
-  return std::make_unique<POSIXBucket>(this, root_fd, i, u);
+  return std::make_unique<POSIXBucket>(this, root_fd, i);
 }
 
 std::string POSIXDriver::zone_unique_trans_id(const uint64_t unique_num)
@@ -525,8 +525,7 @@ int POSIXBucket::create(const DoutPrefixProvider* dpp,
 			const CreateParams& params,
 			optional_yield y)
 {
-  ceph_assert(owner);
-  info.owner = owner->get_id();
+  info.owner = params.owner;
 
   info.bucket.marker = params.marker;
   info.bucket.bucket_id = params.bucket_id;
@@ -632,7 +631,7 @@ int POSIXDriver::mint_listing_entry(const std::string &bname,
     POSIXObject *pobj;
     int ret;
 
-    ret = load_bucket(nullptr, nullptr, rgw_bucket(std::string(), bname),
+    ret = load_bucket(nullptr, rgw_bucket(std::string(), bname),
                       &b, null_yield);
     if (ret < 0)
       return ret;
@@ -883,10 +882,6 @@ int POSIXBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y)
   mtime = ceph::real_clock::from_time_t(stx.stx_mtime.tv_sec);
   info.creation_time = ceph::real_clock::from_time_t(stx.stx_btime.tv_sec);
 
-  if (owner) {
-    info.owner = owner->get_id();
-  }
-
   ret = open(dpp);
   if (ret < 0) {
     return ret;
@@ -981,7 +976,7 @@ int POSIXBucket::check_bucket_shards(const DoutPrefixProvider* dpp,
   return 0;
 }
 
-int POSIXBucket::chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y)
+int POSIXBucket::chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y)
 {
   /* TODO map user to UID/GID, and change it */
   return 0;
@@ -1204,7 +1199,7 @@ int POSIXBucket::get_shadow_bucket(const DoutPrefixProvider* dpp, optional_yield
 
   open(dpp);
 
-  bp = new POSIXBucket(driver, dir_fd, b, owner, ons);
+  bp = new POSIXBucket(driver, dir_fd, b, ons);
   ret = bp->load_bucket(dpp, y);
   if (ret == -ENOENT && create) {
     /* Create it if it doesn't exist */
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 16f88729ab4a..1599808beff3 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -52,10 +52,9 @@ class POSIXDriver : public FilterDriver {
 				std::string& user_str, optional_yield y,
 				std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-  virtual std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i)  override;
-  virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const
-			  rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-			  optional_yield y) override;
+  virtual std::unique_ptr<Bucket> get_bucket(const RGWBucketInfo& i)  override;
+  virtual int load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
+                          std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
 
   virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
@@ -143,16 +142,16 @@ class POSIXBucket : public StoreBucket {
   std::optional<std::string> ns;
 
 public:
-  POSIXBucket(POSIXDriver *_dr, int _p_fd, const rgw_bucket& _b, User* _u, std::optional<std::string> _ns = std::nullopt)
-    : StoreBucket(_b, _u),
+  POSIXBucket(POSIXDriver *_dr, int _p_fd, const rgw_bucket& _b, std::optional<std::string> _ns = std::nullopt)
+    : StoreBucket(_b),
     driver(_dr),
     parent_fd(_p_fd),
     acls(),
     ns(_ns)
     { }
 
-  POSIXBucket(POSIXDriver *_dr, int _p_fd, const RGWBucketInfo& _i, User* _u)
-    : StoreBucket(_i, _u),
+  POSIXBucket(POSIXDriver *_dr, int _p_fd, const RGWBucketInfo& _i)
+    : StoreBucket(_i),
     driver(_dr),
     parent_fd(_p_fd),
     acls()
@@ -201,7 +200,7 @@ class POSIXBucket : public StoreBucket {
                               RGWBucketEnt* ent) override;
   virtual int check_bucket_shards(const DoutPrefixProvider* dpp,
                                   uint64_t num_objs, optional_yield y) override;
-  virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override;
+  virtual int chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y) override;
   virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive,
 		       ceph::real_time mtime, optional_yield y) override;
   virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override;
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 0e1eb274d403..066654adfa8c 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -109,8 +109,7 @@ void check_bad_user_bucket_mapping(rgw::sal::Driver* driver, rgw::sal::User& use
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      int r = driver->load_bucket(dpp, &user,
-                                  rgw_bucket(user.get_tenant(), ent.bucket.name),
+      int r = driver->load_bucket(dpp, rgw_bucket(user.get_tenant(), ent.bucket.name),
                                   &bucket, y);
       if (r < 0) {
         ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << bucket << dendl;
@@ -122,7 +121,7 @@ void check_bad_user_bucket_mapping(rgw::sal::Driver* driver, rgw::sal::User& use
             << " got " << bucket << std::endl;
         if (fix) {
           cout << "fixing" << std::endl;
-	  r = bucket->chown(dpp, user, y);
+	  r = bucket->chown(dpp, user.get_id(), y);
           if (r < 0) {
             cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
           }
@@ -183,7 +182,7 @@ int RGWBucket::init(rgw::sal::Driver* _driver, RGWBucketAdminOpState& op_state,
     bucket_name = bucket_name.substr(pos + 1);
   }
 
-  int r = driver->load_bucket(dpp, user.get(), rgw_bucket(tenant, bucket_name),
+  int r = driver->load_bucket(dpp, rgw_bucket(tenant, bucket_name),
                               &bucket, y);
   if (r < 0) {
       set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name);
@@ -246,11 +245,6 @@ bool rgw_find_bucket_by_id(const DoutPrefixProvider *dpp, CephContext *cct, rgw:
 int RGWBucket::chown(RGWBucketAdminOpState& op_state, const string& marker,
                      optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg)
 {
-  /* User passed in by rgw_admin is the new user; get the current user and set it in
-   * the bucket */
-  std::unique_ptr<rgw::sal::User> old_user = driver->get_user(bucket->get_info().owner);
-  bucket->set_owner(old_user.get());
-
   return rgw_chown_bucket_and_objects(driver, bucket.get(), user.get(), marker, err_msg, dpp, y);
 }
 
@@ -1243,11 +1237,9 @@ int RGWBucketAdminOp::remove_bucket(rgw::sal::Driver* driver, RGWBucketAdminOpSt
                                     bool bypass_gc, bool keep_index_consistent)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(op_state.get_user_id());
 
-  int ret = driver->load_bucket(dpp, user.get(),
-                                rgw_bucket(user->get_tenant(),
-                                           op_state.get_bucket_name()),
+  int ret = driver->load_bucket(dpp, rgw_bucket(op_state.get_tenant(),
+                                                op_state.get_bucket_name()),
                                 &bucket, y);
   if (ret < 0)
     return ret;
@@ -1289,8 +1281,7 @@ static int bucket_stats(rgw::sal::Driver* driver,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   map<RGWObjCategory, RGWStorageStats> stats;
 
-  int ret = driver->load_bucket(dpp, nullptr,
-                                rgw_bucket(tenant_name, bucket_name),
+  int ret = driver->load_bucket(dpp, rgw_bucket(tenant_name, bucket_name),
                                 &bucket, y);
   if (ret < 0) {
     return ret;
@@ -1416,7 +1407,7 @@ int RGWBucketAdminOp::limit_check(rgw::sal::Driver* driver,
 	uint64_t num_objects = 0;
 
 	std::unique_ptr<rgw::sal::Bucket> bucket;
-	ret = driver->load_bucket(dpp, user.get(), ent.bucket, &bucket, y);
+	ret = driver->load_bucket(dpp, ent.bucket, &bucket, y);
 	if (ret < 0)
 	  continue;
 
@@ -1600,7 +1591,7 @@ void get_stale_instances(rgw::sal::Driver* driver, const std::string& bucket_nam
     std::unique_ptr<rgw::sal::Bucket> bucket;
     rgw_bucket rbucket;
     rgw_bucket_parse_bucket_key(driver->ctx(), bucket_instance, &rbucket, nullptr);
-    int r = driver->load_bucket(dpp, nullptr, rbucket, &bucket, y);
+    int r = driver->load_bucket(dpp, rbucket, &bucket, y);
     if (r < 0){
       // this can only happen if someone deletes us right when we're processing
       ldpp_dout(dpp, -1) << "Bucket instance is invalid: " << bucket_instance
@@ -1620,7 +1611,7 @@ void get_stale_instances(rgw::sal::Driver* driver, const std::string& bucket_nam
   auto [tenant, bname] = split_tenant(bucket_name);
   RGWBucketInfo cur_bucket_info;
   std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-  int r = driver->load_bucket(dpp, nullptr, rgw_bucket(tenant, bname),
+  int r = driver->load_bucket(dpp, rgw_bucket(tenant, bname),
                               &cur_bucket, y);
   if (r < 0) {
     if (r == -ENOENT) {
@@ -1755,7 +1746,7 @@ int RGWBucketAdminOp::clear_stale_instances(rgw::sal::Driver* driver,
                       Formatter *formatter,
                       rgw::sal::Driver* driver){
                      for (const auto &binfo: lst) {
-		       auto bucket = driver->get_bucket(nullptr, binfo);
+		       auto bucket = driver->get_bucket(binfo);
 		       int ret = bucket->purge_instance(dpp, y);
                        if (ret == 0){
                          auto md_key = "bucket.instance:" + binfo.bucket.get_key();
@@ -1777,8 +1768,7 @@ static int fix_single_bucket_lc(rgw::sal::Driver* driver,
                                 const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->load_bucket(dpp, nullptr,
-                                rgw_bucket(tenant_name, bucket_name),
+  int ret = driver->load_bucket(dpp, rgw_bucket(tenant_name, bucket_name),
                                 &bucket, y);
   if (ret < 0) {
     // TODO: Should we handle the case where the bucket could've been removed between
@@ -1949,7 +1939,7 @@ int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::Driver* driver,
     ldpp_dout(dpp, -1) << "failed to initialize bucket" << dendl;
     return ret;
   }
-  auto bucket = driver->get_bucket(nullptr, admin_bucket.get_bucket_info());
+  auto bucket = driver->get_bucket(admin_bucket.get_bucket_info());
 
   return fix_bucket_obj_expiry(dpp, driver, bucket.get(), flusher, dry_run, y);
 }
@@ -2630,7 +2620,7 @@ int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp
 
   /* update lifecyle policy */
   {
-    auto bucket = bihandler->driver->get_bucket(nullptr, bci.info);
+    auto bucket = bihandler->driver->get_bucket(bci.info);
 
     auto lc = bihandler->driver->get_rgwlc();
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index ff7a0efb481a..676fd3d3cae2 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -1350,7 +1350,7 @@ class RGWAsyncRemoveObj : public RGWAsyncRadosRequest {
     if (_zones_trace) {
       zones_trace = *_zones_trace;
     }
-    bucket = store->get_bucket(nullptr, _bucket_info);
+    bucket = store->get_bucket(_bucket_info);
     obj = bucket->get_object(_key);
   }
 };
diff --git a/src/rgw/driver/rados/rgw_cr_tools.cc b/src/rgw/driver/rados/rgw_cr_tools.cc
index 928cc9c0c988..a46ba1ac6e12 100644
--- a/src/rgw/driver/rados/rgw_cr_tools.cc
+++ b/src/rgw/driver/rados/rgw_cr_tools.cc
@@ -100,8 +100,7 @@ int RGWGetUserInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 template<>
 int RGWGetBucketInfoCR::Request::_send_request(const DoutPrefixProvider *dpp)
 {
-  return store->load_bucket(dpp, nullptr,
-                            rgw_bucket(params.tenant, params.bucket_name),
+  return store->load_bucket(dpp, rgw_bucket(params.tenant, params.bucket_name),
                             &result->bucket, null_yield);
 }
 
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 7abb0d84a7d1..3935957bc5c3 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -873,8 +873,7 @@ static inline void populate_event(reservation_t& res,
   event.x_amz_id_2 = res.store->getRados()->host_id; // RGW on which the change was made
   // configurationId is filled from notification configuration
   event.bucket_name = res.bucket->get_name();
-  event.bucket_ownerIdentity = res.bucket->get_owner() ?
-    res.bucket->get_owner()->get_id().id : res.bucket->get_info().owner.id;
+  event.bucket_ownerIdentity = res.bucket->get_owner().id;
   const auto region = res.store->get_zone()->get_zonegroup().get_api_name();
   rgw::ARN bucket_arn(res.bucket->get_key());
   bucket_arn.region = region; 
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc
index ba6580b89b07..72c21c1d262a 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.cc
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc
@@ -201,7 +201,7 @@ int RGWObjectExpirer::garbage_single_object(const DoutPrefixProvider *dpp, objex
   RGWBucketInfo bucket_info;
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int ret = driver->load_bucket(dpp, nullptr, rgw_bucket(hint.tenant, hint.bucket_name, hint.bucket_id), &bucket, null_yield);
+  int ret = driver->load_bucket(dpp, rgw_bucket(hint.tenant, hint.bucket_name, hint.bucket_id), &bucket, null_yield);
   if (-ENOENT == ret) {
     ldpp_dout(dpp, 15) << "NOTICE: cannot find bucket = " \
         << hint.bucket_name << ". The object must be already removed" << dendl;
diff --git a/src/rgw/driver/rados/rgw_rest_bucket.cc b/src/rgw/driver/rados/rgw_rest_bucket.cc
index 4fac8ef64f7c..dc71e40335ff 100644
--- a/src/rgw/driver/rados/rgw_rest_bucket.cc
+++ b/src/rgw/driver/rados/rgw_rest_bucket.cc
@@ -233,7 +233,7 @@ void RGWOp_Bucket_Remove::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->load_bucket(s, nullptr, rgw_bucket("", bucket_name),
+  op_ret = driver->load_bucket(s, rgw_bucket("", bucket_name),
                                &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 0) << "get_bucket returned ret=" << op_ret << dendl;
@@ -301,7 +301,7 @@ void RGWOp_Set_Bucket_Quota::execute(optional_yield y)
   }
   if (use_http_params) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->load_bucket(s, nullptr, rgw_bucket(uid.tenant, bucket_name),
+    op_ret = driver->load_bucket(s, rgw_bucket(uid.tenant, bucket_name),
                                  &bucket, s->yield);
     if (op_ret < 0) {
       return;
diff --git a/src/rgw/driver/rados/rgw_rest_log.cc b/src/rgw/driver/rados/rgw_rest_log.cc
index d331097f10b9..9c0467ec36a4 100644
--- a/src/rgw/driver/rados/rgw_rest_log.cc
+++ b/src/rgw/driver/rados/rgw_rest_log.cc
@@ -414,7 +414,7 @@ void RGWOp_BILog_List::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -541,7 +541,7 @@ void RGWOp_BILog_Info::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -635,7 +635,7 @@ void RGWOp_BILog_Delete::execute(optional_yield y) {
     b.name = bn;
     b.bucket_id = bucket_instance;
   }
-  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 5) << "could not get bucket info for bucket=" << bucket_name << dendl;
     return;
@@ -985,7 +985,7 @@ void RGWOp_BILog_Status::execute(optional_yield y)
 
   // read the bucket instance info for num_shards
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(s, nullptr, b, &bucket, y);
+  op_ret = driver->load_bucket(s, b, &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 4) << "failed to read bucket info: " << cpp_strerror(op_ret) << dendl;
     return;
@@ -1069,7 +1069,7 @@ void RGWOp_BILog_Status::execute(optional_yield y)
     if (*pipe.dest.bucket != pinfo->bucket) {
       opt_dest_info.emplace();
       std::unique_ptr<rgw::sal::Bucket> dest_bucket;
-      op_ret = driver->load_bucket(s, nullptr, *pipe.dest.bucket, &dest_bucket, y);
+      op_ret = driver->load_bucket(s, *pipe.dest.bucket, &dest_bucket, y);
       if (op_ret < 0) {
         ldpp_dout(this, 4) << "failed to read target bucket info (bucket=: " << cpp_strerror(op_ret) << dendl;
         return;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 08722cc24b51..ed6e2525ac2b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -124,15 +124,12 @@ int RadosBucket::create(const DoutPrefixProvider* dpp,
                         const CreateParams& params,
                         optional_yield y)
 {
-  ceph_assert(owner);
-  const rgw_user& owner_id = owner->get_id();
-
   rgw_bucket key = get_key();
   key.marker = params.marker;
   key.bucket_id = params.bucket_id;
 
   int ret = store->getRados()->create_bucket(
-      dpp, y, key, owner_id, params.zonegroup_id,
+      dpp, y, key, params.owner, params.zonegroup_id,
       params.placement_rule, params.zone_placement, params.attrs,
       params.obj_lock_enabled, params.swift_ver_location,
       params.quota, params.creation_time, &bucket_version, info);
@@ -146,7 +143,7 @@ int RadosBucket::create(const DoutPrefixProvider* dpp,
      * If all is ok then update the user's list of buckets.  Otherwise inform
      * client about a name conflict.
      */
-    if (info.owner != owner_id) {
+    if (info.owner != params.owner) {
       return -ERR_BUCKET_EXISTS;
     }
     ret = 0;
@@ -154,10 +151,10 @@ int RadosBucket::create(const DoutPrefixProvider* dpp,
     return ret;
   }
 
-  ret = link(dpp, owner, y, false);
+  ret = link(dpp, params.owner, y, false);
   if (ret && !existed && ret != -EEXIST) {
     /* if it exists (or previously existed), don't remove it! */
-    ret = unlink(dpp, owner, y);
+    ret = unlink(dpp, params.owner, y);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret
 		       << dendl;
@@ -542,7 +539,7 @@ int RadosBucket::read_stats_async(const DoutPrefixProvider *dpp,
 int RadosBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                                  RGWBucketEnt* ent)
 {
-  return store->ctl()->bucket->sync_user_stats(dpp, owner->get_id(), info, y, ent);
+  return store->ctl()->bucket->sync_user_stats(dpp, info.owner, info, y, ent);
 }
 
 int RadosBucket::check_bucket_shards(const DoutPrefixProvider* dpp,
@@ -551,17 +548,17 @@ int RadosBucket::check_bucket_shards(const DoutPrefixProvider* dpp,
   return store->getRados()->check_bucket_shards(info, num_objs, dpp, y);
 }
 
-int RadosBucket::link(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint, RGWObjVersionTracker* objv)
+int RadosBucket::link(const DoutPrefixProvider* dpp, const rgw_user& new_user, optional_yield y, bool update_entrypoint, RGWObjVersionTracker* objv)
 {
   RGWBucketEntryPoint ep;
   ep.bucket = info.bucket;
-  ep.owner = new_user->get_id();
+  ep.owner = new_user;
   ep.creation_time = get_creation_time();
   ep.linked = true;
   Attrs ep_attrs;
   rgw_ep_info ep_data{ep, ep_attrs};
 
-  int r = store->ctl()->bucket->link_bucket(new_user->get_id(), info.bucket,
+  int r = store->ctl()->bucket->link_bucket(new_user, info.bucket,
 					    get_creation_time(), y, dpp, update_entrypoint,
 					    &ep_data);
   if (r < 0)
@@ -573,27 +570,20 @@ int RadosBucket::link(const DoutPrefixProvider* dpp, User* new_user, optional_yi
   return r;
 }
 
-int RadosBucket::unlink(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint)
+int RadosBucket::unlink(const DoutPrefixProvider* dpp, const rgw_user& owner, optional_yield y, bool update_entrypoint)
 {
-  return store->ctl()->bucket->unlink_bucket(new_user->get_id(), info.bucket, y, dpp, update_entrypoint);
+  return store->ctl()->bucket->unlink_bucket(owner, info.bucket, y, dpp, update_entrypoint);
 }
 
-int RadosBucket::chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y)
+int RadosBucket::chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y)
 {
   std::string obj_marker;
-  int r;
-
-  if (!owner) {
-      ldpp_dout(dpp, 0) << __func__ << " Cannot chown without an owner " << dendl;
-      return -EINVAL;
-  }
-
-  r = this->unlink(dpp, owner, y);
+  int r = this->unlink(dpp, info.owner, y);
   if (r < 0) {
     return r;
   }
 
-  return this->link(dpp, &new_user, y);
+  return this->link(dpp, new_owner, y);
 }
 
 int RadosBucket::put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time _mtime, optional_yield y)
@@ -633,14 +623,14 @@ int RadosBucket::read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch,
 			       RGWUsageIter& usage_iter,
 			       map<rgw_user_bucket, rgw_usage_log_entry>& usage)
 {
-  return store->getRados()->read_usage(dpp, owner->get_id(), get_name(), start_epoch,
+  return store->getRados()->read_usage(dpp, info.owner, get_name(), start_epoch,
 				       end_epoch, max_entries, is_truncated,
 				       usage_iter, usage);
 }
 
 int RadosBucket::trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, optional_yield y)
 {
-  return store->getRados()->trim_usage(dpp, owner->get_id(), get_name(), start_epoch, end_epoch, y);
+  return store->getRados()->trim_usage(dpp, info.owner, get_name(), start_epoch, end_epoch, y);
 }
 
 int RadosBucket::remove_objs_from_index(const DoutPrefixProvider *dpp, std::list<rgw_obj_index_key>& objs_to_unlink)
@@ -987,16 +977,16 @@ std::unique_ptr<Object> RadosStore::get_object(const rgw_obj_key& k)
   return std::make_unique<RadosObject>(this, k);
 }
 
-std::unique_ptr<Bucket> RadosStore::get_bucket(User* u, const RGWBucketInfo& i)
+std::unique_ptr<Bucket> RadosStore::get_bucket(const RGWBucketInfo& i)
 {
   /* Don't need to fetch the bucket info, use the provided one */
-  return std::make_unique<RadosBucket>(this, i, u);
+  return std::make_unique<RadosBucket>(this, i);
 }
 
-int RadosStore::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+int RadosStore::load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
                             std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
-  *bucket = std::make_unique<RadosBucket>(this, b, u);
+  *bucket = std::make_unique<RadosBucket>(this, b);
   return (*bucket)->load_bucket(dpp, y);
 }
 
@@ -2102,7 +2092,7 @@ int RadosObject::swift_versioning_restore(bool& restored,
 {
   rgw_obj obj = get_obj();
   return store->getRados()->swift_versioning_restore(*rados_ctx,
-						     bucket->get_owner()->get_id(),
+						     bucket->get_owner(),
 						     bucket->get_info(),
 						     obj,
 						     restored,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index c67d23289ed2..7efd7757c3c4 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -135,8 +135,8 @@ class RadosStore : public StoreDriver {
     virtual int get_user_by_email(const DoutPrefixProvider* dpp, const std::string& email, optional_yield y, std::unique_ptr<User>* user) override;
     virtual int get_user_by_swift(const DoutPrefixProvider* dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-    std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
-    int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+    std::unique_ptr<Bucket> get_bucket(const RGWBucketInfo& i) override;
+    int load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
                     std::unique_ptr<Bucket>* bucket, optional_yield y) override;
     virtual bool is_meta_master() override;
     virtual Zone* get_zone() { return zone.get(); }
@@ -479,12 +479,6 @@ class RadosBucket : public StoreBucket {
         acls() {
     }
 
-    RadosBucket(RadosStore *_st, User* _u)
-      : StoreBucket(_u),
-	store(_st),
-        acls() {
-    }
-
     RadosBucket(RadosStore *_st, const rgw_bucket& _b)
       : StoreBucket(_b),
 	store(_st),
@@ -497,18 +491,6 @@ class RadosBucket : public StoreBucket {
         acls() {
     }
 
-    RadosBucket(RadosStore *_st, const rgw_bucket& _b, User* _u)
-      : StoreBucket(_b, _u),
-	store(_st),
-        acls() {
-    }
-
-    RadosBucket(RadosStore *_st, const RGWBucketInfo& _i, User* _u)
-      : StoreBucket(_i, _u),
-	store(_st),
-        acls() {
-    }
-
     virtual ~RadosBucket();
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
     virtual int list(const DoutPrefixProvider* dpp, ListParams&, int, ListResults&, optional_yield y) override;
@@ -535,7 +517,7 @@ class RadosBucket : public StoreBucket {
                         RGWBucketEnt* ent) override;
     int check_bucket_shards(const DoutPrefixProvider* dpp, uint64_t num_objs,
                             optional_yield y) override;
-    virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override;
+    virtual int chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y) override;
     virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override;
     virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override;
     virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override;
@@ -575,8 +557,8 @@ class RadosBucket : public StoreBucket {
         optional_yield y, const DoutPrefixProvider *dpp) override;
 
   private:
-    int link(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true, RGWObjVersionTracker* objv = nullptr);
-    int unlink(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true);
+    int link(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y, bool update_entrypoint = true, RGWObjVersionTracker* objv = nullptr);
+    int unlink(const DoutPrefixProvider* dpp, const rgw_user& owner, optional_yield y, bool update_entrypoint = true);
     friend class RadosUser;
 };
 
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 72a7da1c31de..635e5ce887d0 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -285,7 +285,7 @@ int RGWDataAccess::Bucket::finish_init()
 int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = sd->driver->load_bucket(dpp, nullptr, rgw_bucket(tenant, name), &bucket, y);
+  int ret = sd->driver->load_bucket(dpp, rgw_bucket(tenant, name), &bucket, y);
   if (ret < 0) {
     return ret;
   }
@@ -327,7 +327,7 @@ int RGWDataAccess::Object::put(bufferlist& data,
 
   rgw::BlockingAioThrottle aio(driver->ctx()->_conf->rgw_put_obj_min_window_size);
 
-  std::unique_ptr<rgw::sal::Bucket> b = driver->get_bucket(nullptr, bucket_info);
+  std::unique_ptr<rgw::sal::Bucket> b = driver->get_bucket(bucket_info);
   std::unique_ptr<rgw::sal::Object> obj = b->get_object(key);
 
   auto& owner = bucket->policy.get_owner();
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index c27be158c334..506ac0acf112 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -1579,7 +1579,7 @@ int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->load_bucket(dpp, old_user.get(), ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, ent.bucket, &bucket, y);
       if (ret < 0) {
         set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name());
         return ret;
@@ -1785,7 +1785,7 @@ int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
 
     for (const auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->load_bucket(dpp, user, ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, ent.bucket, &bucket, y);
       if (ret < 0) {
         set_err_msg(err_msg, "unable to load bucket " + ent.bucket.name);
         return ret;
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 46aef1ee2d24..d44e66c700cd 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1189,20 +1189,19 @@ class StoreDestructor {
   }
 };
 
-static int init_bucket(rgw::sal::User* user, const rgw_bucket& b,
+static int init_bucket(const rgw_bucket& b,
                        std::unique_ptr<rgw::sal::Bucket>* bucket)
 {
-  return driver->load_bucket(dpp(), user, b, bucket, null_yield);
+  return driver->load_bucket(dpp(), b, bucket, null_yield);
 }
 
-static int init_bucket(rgw::sal::User* user,
-		       const string& tenant_name,
+static int init_bucket(const string& tenant_name,
 		       const string& bucket_name,
 		       const string& bucket_id,
                        std::unique_ptr<rgw::sal::Bucket>* bucket)
 {
   rgw_bucket b{tenant_name, bucket_name, bucket_id};
-  return init_bucket(user, b, bucket);
+  return init_bucket(b, bucket);
 }
 
 static int read_input(const string& infile, bufferlist& bl)
@@ -1407,8 +1406,7 @@ int set_bucket_quota(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_size, bool have_max_objects)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr,
-                              rgw_bucket(tenant_name, bucket_name),
+  int r = driver->load_bucket(dpp(), rgw_bucket(tenant_name, bucket_name),
                               &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
@@ -1433,8 +1431,7 @@ int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd,
                      bool have_max_read_bytes, bool have_max_write_bytes)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr,
-                              rgw_bucket(tenant_name, bucket_name),
+  int r = driver->load_bucket(dpp(), rgw_bucket(tenant_name, bucket_name),
                               &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
@@ -1538,8 +1535,7 @@ int show_bucket_ratelimit(rgw::sal::Driver* driver, const string& tenant_name,
                           const string& bucket_name, Formatter *formatter)
 {
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int r = driver->load_bucket(dpp(), nullptr,
-                              rgw_bucket(tenant_name, bucket_name),
+  int r = driver->load_bucket(dpp(), rgw_bucket(tenant_name, bucket_name),
                               &bucket, null_yield);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
@@ -1724,7 +1720,7 @@ int do_check_object_locator(const string& tenant_name, const string& bucket_name
 
   f->open_object_section("bucket");
   f->dump_string("bucket", bucket_name);
-  int ret = init_bucket(nullptr, tenant_name, bucket_name, bucket_id, &bucket);
+  int ret = init_bucket(tenant_name, bucket_name, bucket_id, &bucket);
   if (ret < 0) {
     cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
     return ret;
@@ -2041,12 +2037,11 @@ static int update_period(rgw::sal::ConfigStore* cfgstore,
   return 0;
 }
 
-static int init_bucket_for_sync(rgw::sal::User* user,
-				const string& tenant, const string& bucket_name,
+static int init_bucket_for_sync(const string& tenant, const string& bucket_name,
                                 const string& bucket_id,
 				std::unique_ptr<rgw::sal::Bucket>* bucket)
 {
-  int ret = init_bucket(user, tenant, bucket_name, bucket_id, bucket);
+  int ret = init_bucket(tenant, bucket_name, bucket_id, bucket);
   if (ret < 0) {
     cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
     return ret;
@@ -2522,7 +2517,7 @@ static int bucket_source_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Ra
   }
 
   std::unique_ptr<rgw::sal::Bucket> source_bucket;
-  int r = init_bucket(nullptr, *pipe.source.bucket, &source_bucket);
+  int r = init_bucket(*pipe.source.bucket, &source_bucket);
   if (r < 0) {
     ldpp_dout(dpp, -1) << "failed to read source bucket info: " << cpp_strerror(r) << dendl;
     return r;
@@ -2651,7 +2646,7 @@ static void get_hint_entities(const std::set<rgw_zone_id>& zones, const std::set
   for (auto& zone_id : zones) {
     for (auto& b : buckets) {
       std::unique_ptr<rgw::sal::Bucket> hint_bucket;
-      int ret = init_bucket(nullptr, b, &hint_bucket);
+      int ret = init_bucket(b, &hint_bucket);
       if (ret < 0) {
 	ldpp_dout(dpp(), 20) << "could not init bucket info for hint bucket=" << b << " ... skipping" << dendl;
 	continue;
@@ -2694,7 +2689,7 @@ static int sync_info(std::optional<rgw_zone_id> opt_target_zone, std::optional<r
   if (eff_bucket) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
 
-    int ret = init_bucket(nullptr, *eff_bucket, &bucket);
+    int ret = init_bucket(*eff_bucket, &bucket);
     if (ret < 0 && ret != -ENOENT) {
       cerr << "ERROR: init_bucket failed: " << cpp_strerror(-ret) << std::endl;
       return ret;
@@ -2998,7 +2993,7 @@ int check_reshard_bucket_params(rgw::sal::Driver* driver,
     return -EINVAL;
   }
 
-  int ret = init_bucket(nullptr, tenant, bucket_name, bucket_id, bucket);
+  int ret = init_bucket(tenant, bucket_name, bucket_id, bucket);
   if (ret < 0) {
     cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
     return ret;
@@ -3181,7 +3176,7 @@ class SyncPolicyContext
       return 0;
     }
 
-    ret = init_bucket(nullptr, *b, &bucket);
+    ret = init_bucket(*b, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return ret;
@@ -7020,7 +7015,7 @@ int main(int argc, const char **argv)
       bucket_op.marker = marker;
       RGWBucketAdminOp::info(driver, bucket_op, stream_flusher, null_yield, dpp());
     } else {
-      int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+      int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
       if (ret < 0) {
         cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
         return -ret;
@@ -7113,7 +7108,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -7227,7 +7222,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -7435,7 +7430,7 @@ int main(int argc, const char **argv)
 
 
     if (!bucket_name.empty()) {
-      int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+      int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
       if (ret < 0) {
 	cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
 	return -ret;
@@ -7479,7 +7474,7 @@ int main(int argc, const char **argv)
     }
 
     if (!bucket_name.empty()) {
-      int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+      int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
       if (ret < 0) {
 	cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
 	return -ret;
@@ -7518,7 +7513,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::OLH_GET) {
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7535,7 +7530,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::OLH_READLOG) {
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7573,7 +7568,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: object not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7599,7 +7594,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket name not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7627,7 +7622,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       ldpp_dout(dpp(), 0) << "ERROR: could not init bucket: " << cpp_strerror(-ret) <<
 	dendl;
@@ -7697,14 +7692,14 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket name not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
 
     std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-    ret = init_bucket(user.get(), tenant, bucket_name, string(), &cur_bucket);
+    ret = init_bucket(tenant, bucket_name, string(), &cur_bucket);
     if (ret == -ENOENT) {
       // no bucket entrypoint
     } else if (ret < 0) {
@@ -7782,7 +7777,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::OBJECT_RM) {
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7806,7 +7801,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -7849,7 +7844,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) <<
 	"." << std::endl;
@@ -7942,7 +7937,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -8184,7 +8179,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -8221,7 +8216,7 @@ int main(int argc, const char **argv)
     }
 
     bool bucket_initable = true;
-    ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       if (yes_i_really_mean_it) {
         bucket_initable = false;
@@ -8283,7 +8278,7 @@ int main(int argc, const char **argv)
   } // OPT_RESHARD_CANCEL
 
   if (opt_cmd == OPT::OBJECT_UNLINK) {
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -8303,7 +8298,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::OBJECT_STAT) {
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -8509,7 +8504,7 @@ int main(int argc, const char **argv)
     }
 
     RGWLifecycleConfiguration config;
-    ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -8535,7 +8530,7 @@ int main(int argc, const char **argv)
   if (opt_cmd == OPT::LC_PROCESS) {
     if ((! bucket_name.empty()) ||
 	(! bucket_id.empty())) {
-        int ret = init_bucket(nullptr, tenant, bucket_name, bucket_id, &bucket);
+        int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
 	if (ret < 0) {
 	  cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret)
 	       << std::endl;
@@ -8696,7 +8691,7 @@ int main(int argc, const char **argv)
 
     if (sync_stats) {
       if (!bucket_name.empty()) {
-        int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+        int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
         if (ret < 0) {
           cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
           return -ret;
@@ -9219,7 +9214,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket_for_sync(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9227,7 +9222,7 @@ int main(int argc, const char **argv)
     if (opt_sb && opt_sb->bucket_id.empty()) {
       string sbid;
       std::unique_ptr<rgw::sal::Bucket> sbuck;
-      int ret = init_bucket_for_sync(user.get(), opt_sb->tenant, opt_sb->name, sbid, &sbuck);
+      int ret = init_bucket_for_sync(opt_sb->tenant, opt_sb->name, sbid, &sbuck);
       if (ret < 0) {
         return -ret;
       }
@@ -9258,7 +9253,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9310,7 +9305,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9322,7 +9317,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9338,7 +9333,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket_for_sync(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9371,7 +9366,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket_for_sync(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       return -ret;
     }
@@ -9396,7 +9391,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -9906,7 +9901,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -9929,7 +9924,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket not specified" << std::endl;
       return EINVAL;
     }
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -10514,7 +10509,7 @@ int main(int argc, const char **argv)
     RGWPubSub ps(driver, tenant);
 
     rgw_pubsub_bucket_topics result;
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -10581,7 +10576,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -10635,7 +10630,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    int ret = init_bucket(user.get(), tenant, bucket_name, bucket_id, &bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 852469b7eac9..93cd2ea76349 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -137,7 +137,7 @@ int rgw_chown_bucket_and_objects(rgw::sal::Driver* driver, rgw::sal::Bucket* buc
 				 const DoutPrefixProvider *dpp, optional_yield y)
 {
   /* Chown on the bucket */
-  int ret = bucket->chown(dpp, *new_user, y);
+  int ret = bucket->chown(dpp, new_user->get_id(), y);
   if (ret < 0) {
     set_err_msg(err_msg, "Failed to change object ownership: " + cpp_strerror(-ret));
   }
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index ca8330c39624..978f9736b8eb 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1331,17 +1331,8 @@ class LCOpAction_Transition : public LCOpAction {
 
     /* notifications */
     auto& bucket = oc.bucket;
-    auto& bucket_info = oc.bucket->get_info();
     std::string version_id;
 
-    std::unique_ptr<rgw::sal::User> user;
-    user = oc.driver->get_user(bucket_info.owner);
-    if (! bucket->get_owner()) {
-      if (user) {
-	bucket->set_owner(user.get());
-      }
-    }
-
     auto& obj = oc.obj;
 
     const auto event_type = (bucket->versioned() &&
@@ -1619,8 +1610,7 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     return 0;
   }
 
-  int ret = driver->load_bucket(this, nullptr,
-                                rgw_bucket(bucket_tenant, bucket_name),
+  int ret = driver->load_bucket(this, rgw_bucket(bucket_tenant, bucket_name),
                                 &bucket, null_yield);
   if (ret < 0) {
     ldpp_dout(this, 0) << "LC:get_bucket for " << bucket_name
@@ -1628,13 +1618,6 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     return ret;
   }
 
-  ret = bucket->load_bucket(this, null_yield);
-  if (ret < 0) {
-    ldpp_dout(this, 0) << "LC:load_bucket for " << bucket_name
-		       << " failed" << dendl;
-    return ret;
-  }
-
   auto stack_guard = make_scope_guard(
     [&worker]
       {
@@ -1757,15 +1740,6 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     worker->workpool->drain();
   }
 
-  std::unique_ptr<rgw::sal::User> user;
-  if (! bucket->get_owner()) {
-    auto& bucket_info = bucket->get_info();
-    std::unique_ptr<rgw::sal::User> user = driver->get_user(bucket_info.owner);
-      if (user) {
-	bucket->set_owner(user.get());
-      }
-  }
-
   ret = handle_multipart_expiration(bucket.get(), prefix_map, worker, stop_at, once);
   return ret;
 }
diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc
index 058384929b3c..cb819deff14c 100644
--- a/src/rgw/rgw_lua_request.cc
+++ b/src/rgw/rgw_lua_request.cc
@@ -305,7 +305,7 @@ struct BucketMetaTable : public EmptyMetaTable {
       create_metatable<PlacementRuleMetaTable>(L, name, index, false, &(bucket->get_info().placement_rule));
     } else if (strcasecmp(index, "User") == 0) {
       create_metatable<UserMetaTable>(L, name, index, false, 
-          const_cast<rgw_user*>(&bucket->get_owner()->get_id()));
+          const_cast<rgw_user*>(&bucket->get_owner()));
     } else {
       return error_unknown_field(L, index, name);
     }
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 00715db9219d..dc802b4c4528 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -533,8 +533,8 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   /* check if copy source is within the current domain */
   if (!s->src_bucket_name.empty()) {
     std::unique_ptr<rgw::sal::Bucket> src_bucket;
-    ret = driver->load_bucket(dpp, nullptr,
-                              rgw_bucket(s->src_tenant_name, s->src_bucket_name),
+    ret = driver->load_bucket(dpp, rgw_bucket(s->src_tenant_name,
+                                              s->src_bucket_name),
                               &src_bucket, y);
     if (ret == 0) {
       s->local_source = zonegroup.equals(src_bucket->get_info().zonegroup);
@@ -554,9 +554,8 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 
     /* This is the only place that s->bucket is created.  It should never be
      * overwritten. */
-    ret = driver->load_bucket(dpp, s->user.get(),
-                              rgw_bucket(s->bucket_tenant, s->bucket_name,
-                                         s->bucket_instance_id),
+    ret = driver->load_bucket(dpp, rgw_bucket(s->bucket_tenant, s->bucket_name,
+                                              s->bucket_instance_id),
                               &s->bucket, y);
     if (ret < 0) {
       if (ret != -ENOENT) {
@@ -1976,8 +1975,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
     map<string, bufferlist> bucket_attrs;
-    r = driver->load_bucket(this, s->user.get(),
-                            rgw_bucket(s->user->get_tenant(), bucket_name),
+    r = driver->load_bucket(this, rgw_bucket(s->user->get_tenant(), bucket_name),
                             &ubucket, y);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
@@ -2108,8 +2106,8 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
 	RGWAccessControlPolicy& _bucket_acl = allocated_acls.back();
 
 	std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
-	int r = driver->load_bucket(this, s->user.get(),
-                                    rgw_bucket(s->user->get_tenant(), bucket_name),
+	int r = driver->load_bucket(this, rgw_bucket(s->user->get_tenant(),
+                                                     bucket_name),
                                     &tmp_bucket, y);
         if (r < 0) {
           ldpp_dout(this, 0) << "could not get bucket info for bucket="
@@ -3019,7 +3017,7 @@ void RGWStatBucket::execute(optional_yield y)
     return;
   }
 
-  op_ret = driver->load_bucket(this, s->user.get(), s->bucket->get_key(), &bucket, y);
+  op_ret = driver->load_bucket(this, s->bucket->get_key(), &bucket, y);
   if (op_ret) {
     return;
   }
@@ -3488,8 +3486,7 @@ void RGWCreateBucket::execute(optional_yield y)
   }
 
   // read the bucket info if it exists
-  op_ret = driver->load_bucket(this, s->user.get(),
-                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &s->bucket, y);
   if (op_ret < 0 && op_ret != -ENOENT)
     return;
@@ -3530,6 +3527,7 @@ void RGWCreateBucket::execute(optional_yield y)
 
   s->bucket_owner.set_id(s->user->get_id());
   s->bucket_owner.set_name(s->user->get_display_name());
+  createparams.owner = s->user->get_id();
 
   buffer::list aclbl;
   policy.encode(aclbl);
@@ -3610,7 +3608,7 @@ void RGWCreateBucket::execute(optional_yield y)
       op_ret = s->bucket->load_bucket(this, y);
       if (op_ret < 0) {
         return;
-      } else if (!s->bucket->is_owner(s->user.get())) {
+      } else if (s->bucket->get_owner() != s->user->get_id()) {
         /* New bucket doesn't belong to the account we're operating on. */
         op_ret = -EEXIST;
         return;
@@ -3789,23 +3787,16 @@ int RGWPutObj::init_processing(optional_yield y) {
       }
     }
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = driver->load_bucket(this, s->user.get(),
-                              rgw_bucket(copy_source_tenant_name,
-                                         copy_source_bucket_name),
+    ret = driver->load_bucket(this, rgw_bucket(copy_source_tenant_name,
+                                               copy_source_bucket_name),
                               &bucket, y);
     if (ret < 0) {
-      ldpp_dout(this, 5) << __func__ << "(): get_bucket() returned ret=" << ret << dendl;
+      ldpp_dout(this, 5) << __func__ << "(): load_bucket() returned ret=" << ret << dendl;
       if (ret == -ENOENT) {
         ret = -ERR_NO_SUCH_BUCKET;
       }
       return ret;
     }
-
-    ret = bucket->load_bucket(this, y);
-    if (ret < 0) {
-      ldpp_dout(this, 5) << __func__ << "(): load_bucket() returned ret=" << ret << dendl;
-      return ret;
-    }
     copy_source_bucket_info = bucket->get_info();
 
     /* handle x-amz-copy-source-range */
@@ -3853,7 +3844,7 @@ int RGWPutObj::verify_permission(optional_yield y)
     RGWAccessControlPolicy cs_acl(s->cct);
     boost::optional<Policy> policy;
     map<string, bufferlist> cs_attrs;
-    auto cs_bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+    auto cs_bucket = driver->get_bucket(copy_source_bucket_info);
     auto cs_object = cs_bucket->get_object(rgw_obj_key(copy_source_object_name,
                                                        copy_source_version_id));
     cs_object->set_atomic();
@@ -4039,7 +4030,7 @@ int RGWPutObj::get_data(const off_t fst, const off_t lst, bufferlist& bl)
   new_ofs = fst;
   new_end = lst;
 
-  auto bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+  auto bucket = driver->get_bucket(copy_source_bucket_info);
   auto obj = bucket->get_object(rgw_obj_key(copy_source_object_name,
                                             copy_source_version_id));
   auto read_op = obj->get_read_op();
@@ -4300,7 +4291,7 @@ void RGWPutObj::execute(optional_yield y)
     return;
   }
   if ((! copy_source.empty()) && !copy_source_range) {
-    auto bucket = driver->get_bucket(nullptr, copy_source_bucket_info);
+    auto bucket = driver->get_bucket(copy_source_bucket_info);
     auto obj = bucket->get_object(rgw_obj_key(copy_source_object_name,
                                               copy_source_version_id));
 
@@ -5474,9 +5465,8 @@ int RGWCopyObj::init_processing(optional_yield y)
     return op_ret;
   }
 
-  op_ret = driver->load_bucket(this, s->user.get(),
-                               rgw_bucket(s->src_tenant_name,
-                                          s->src_bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->src_tenant_name,
+                                                s->src_bucket_name),
                                &src_bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
@@ -7437,9 +7427,8 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
   ACLOwner bowner;
   RGWObjVersionTracker ot;
 
-  int ret = driver->load_bucket(dpp, s->user.get(),
-                                rgw_bucket(s->user->get_tenant(),
-                                           path.bucket_name),
+  int ret = driver->load_bucket(dpp, rgw_bucket(s->user->get_tenant(),
+                                                path.bucket_name),
                                 &bucket, y);
   if (ret < 0) {
     goto binfo_fail;
@@ -7693,7 +7682,7 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y)
 
   // load the bucket
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = driver->load_bucket(this, s->user.get(), new_bucket, &bucket, y);
+  ret = driver->load_bucket(this, new_bucket, &bucket, y);
 
   // return success if it exists
   if (ret != -ENOENT) {
@@ -7704,6 +7693,7 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y)
   const auto& zonegroup = s->penv.site->get_zonegroup();
 
   rgw::sal::Bucket::CreateParams createparams;
+  createparams.owner = s->user->get_id();
   createparams.zonegroup_id = zonegroup.id;
   createparams.placement_rule.storage_class = s->info.storage_class;
   op_ret = select_bucket_placement(this, zonegroup, s->user->get_info(),
@@ -7831,8 +7821,8 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   std::unique_ptr<rgw::sal::Bucket> bucket;
   ACLOwner bowner;
 
-  op_ret = driver->load_bucket(this, s->user.get(),
-                               rgw_bucket(s->user->get_tenant(), bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->user->get_tenant(),
+                                                bucket_name),
                                &bucket, y);
   if (op_ret < 0) {
     if (op_ret == -ENOENT) {
diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 8b9125f29067..b7dc562c721b 100644
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -502,7 +502,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp,
   }
 
   std::unique_ptr<rgw::sal::Bucket> cur_bucket;
-  ret = store->load_bucket(dpp, nullptr, orphan_bucket, &cur_bucket, null_yield);
+  ret = store->load_bucket(dpp, orphan_bucket, &cur_bucket, null_yield);
   if (ret < 0) {
     if (ret == -ENOENT) {
       /* probably raced with bucket removal */
@@ -529,7 +529,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp,
   rgw_bucket b;
   rgw_bucket_parse_bucket_key(store->ctx(), bucket_instance_id, &b, nullptr);
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = store->load_bucket(dpp, nullptr, b, &bucket, null_yield);
+  ret = store->load_bucket(dpp, b, &bucket, null_yield);
   if (ret < 0) {
     if (ret == -ENOENT) {
       /* probably raced with bucket removal */
@@ -1241,7 +1241,7 @@ int RGWRadosList::process_bucket(
 	continue;
       }
 
-      auto bucket = store->get_bucket(nullptr, bucket_info);
+      auto bucket = store->get_bucket(bucket_info);
       // we need to do this in two cases below, so use a lambda
       auto do_stat_key =
 	[&](const rgw_obj_key& key) -> int {
@@ -1388,7 +1388,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
     bucket_process_map.erase(front);
 
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    ret = store->load_bucket(dpp, nullptr, rgw_bucket(tenant_name, bucket_name),
+    ret = store->load_bucket(dpp, rgw_bucket(tenant_name, bucket_name),
                              &bucket, null_yield);
     if (ret == -ENOENT) {
       std::cerr << "WARNING: bucket " << bucket_name <<
@@ -1460,8 +1460,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp,
   // initial bucket
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  ret = store->load_bucket(dpp, nullptr,
-                           rgw_bucket(tenant_name, start_bucket_name),
+  ret = store->load_bucket(dpp, rgw_bucket(tenant_name, start_bucket_name),
                            &bucket, null_yield);
   if (ret == -ENOENT) {
     // bucket deletion race?
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index 3786ccd6729a..c2f300b1fdb8 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -265,7 +265,7 @@ int BucketAsyncRefreshHandler::init_fetch()
   std::unique_ptr<rgw::sal::Bucket> rbucket;
 
   const DoutPrefix dp(driver->ctx(), dout_subsys, "rgw bucket async refresh handler: ");
-  int r = driver->load_bucket(&dp, nullptr, bucket, &rbucket, null_yield);
+  int r = driver->load_bucket(&dp, bucket, &rbucket, null_yield);
   if (r < 0) {
     ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
     return r;
@@ -340,7 +340,7 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_
   std::unique_ptr<rgw::sal::User> user = driver->get_user(_u);
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int r = driver->load_bucket(dpp, user.get(), _b, &bucket, y);
+  int r = driver->load_bucket(dpp, _b, &bucket, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << _b << " r=" << r << dendl;
     return r;
@@ -613,7 +613,7 @@ int RGWUserStatsCache::sync_bucket(const rgw_user& _u, rgw_bucket& _b, optional_
   std::unique_ptr<rgw::sal::User> user = driver->get_user(_u);
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
-  int r = driver->load_bucket(dpp, user.get(), _b, &bucket, y);
+  int r = driver->load_bucket(dpp, _b, &bucket, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "could not get bucket info for bucket=" << _b << " r=" << r << dendl;
     return r;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index d2a1c72f3aa7..04f21d9b87d9 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -960,10 +960,8 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     return;
   }
 
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(),
-                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
@@ -1110,10 +1108,8 @@ void RGWPSDeleteNotifOp::execute(optional_yield y) {
     return;
   }
 
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(),
-                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
@@ -1209,10 +1205,8 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
     return;
   }
 
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  op_ret = driver->load_bucket(this, user.get(),
-                               rgw_bucket(s->bucket_tenant, s->bucket_name),
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get bucket '" << 
diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc
index e5d1aabb1503..3de8ca2618dc 100644
--- a/src/rgw/rgw_rest_ratelimit.cc
+++ b/src/rgw/rgw_rest_ratelimit.cc
@@ -36,8 +36,7 @@ void RGWOp_Ratelimit_Info::execute(optional_yield y)
 
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    int r = driver->load_bucket(s, nullptr,
-                                rgw_bucket(tenant_name, bucket_name),
+    int r = driver->load_bucket(s, rgw_bucket(tenant_name, bucket_name),
                                 &bucket, y);
     if (r != 0) {
       op_ret = r;
@@ -275,8 +274,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y)
   if (ratelimit_scope == "bucket" && !bucket_name.empty() && !global) {
     ldpp_dout(this, 0) << "getting bucket info" << dendl;
     std::unique_ptr<rgw::sal::Bucket> bucket;
-    op_ret = driver->load_bucket(this, nullptr,
-                                 rgw_bucket(tenant_name, bucket_name),
+    op_ret = driver->load_bucket(this, rgw_bucket(tenant_name, bucket_name),
                                  &bucket, y);
     if (op_ret) {
       ldpp_dout(this, 0) << "Error on getting bucket info" << dendl;
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 971b63e85d79..6d56766d10eb 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2343,7 +2343,7 @@ static void dump_bucket_metadata(req_state *s, rgw::sal::Bucket* bucket,
   dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(stats.size));
 
   // only bucket's owner is allowed to get the quota settings of the account
-  if (bucket->is_owner(s->user.get())) {
+  if (bucket->get_owner() == s->user->get_id()) {
     auto user_info = s->user->get_info();
     auto bucket_quota = s->bucket->get_info().quota; // bucket quota
     dump_header(s, "X-RGW-Quota-User-Size", static_cast<long long>(user_info.quota.user_quota.max_size));
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index b941d4bb0992..53815e11d1f2 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -858,8 +858,7 @@ int RGWPutObj_ObjStore_SWIFT::update_slo_segment_size(rgw_slo_entry& entry) {
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (bucket_name.compare(s->bucket->get_name()) != 0) {
-    r = driver->load_bucket(s, s->user.get(),
-                            rgw_bucket(s->user->get_id().tenant, bucket_name),
+    r = driver->load_bucket(s, rgw_bucket(s->user->get_tenant(), bucket_name),
                             &bucket, s->yield);
     if (r < 0) {
       ldpp_dout(this, 0) << "could not get bucket info for bucket="
@@ -2123,8 +2122,7 @@ void RGWFormPost::get_owner_info(const req_state* const s,
 
   /* Need to get user info of bucket owner. */
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->load_bucket(s, user.get(),
-                                rgw_bucket(bucket_tenant, bucket_name),
+  int ret = driver->load_bucket(s, rgw_bucket(bucket_tenant, bucket_name),
                                 &bucket, s->yield);
   if (ret < 0) {
     throw ret;
diff --git a/src/rgw/rgw_rest_usage.cc b/src/rgw/rgw_rest_usage.cc
index 65b035c00727..806affa6f883 100644
--- a/src/rgw/rgw_rest_usage.cc
+++ b/src/rgw/rgw_rest_usage.cc
@@ -30,18 +30,19 @@ void RGWOp_Usage_Get::execute(optional_yield y) {
 
   string uid_str;
   string bucket_name;
+  string tenant;
   uint64_t start, end;
   bool show_entries;
   bool show_summary;
 
   RESTArgs::get_string(s, "uid", uid_str, &uid_str);
   RESTArgs::get_string(s, "bucket", bucket_name, &bucket_name);
+  RESTArgs::get_string(s, "tenant", "", &tenant);
   std::unique_ptr<rgw::sal::User> user = driver->get_user(rgw_user(uid_str));
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    op_ret = driver->load_bucket(nullptr, user.get(),
-                                 rgw_bucket(std::string(), bucket_name),
+    op_ret = driver->load_bucket(this, rgw_bucket(tenant, bucket_name),
                                  &bucket, null_yield);
     if (op_ret < 0) {
       return;
@@ -84,16 +85,17 @@ class RGWOp_Usage_Delete : public RGWRESTOp {
 void RGWOp_Usage_Delete::execute(optional_yield y) {
   string uid_str;
   string bucket_name;
+  string tenant;
   uint64_t start, end;
 
   RESTArgs::get_string(s, "uid", uid_str, &uid_str);
   RESTArgs::get_string(s, "bucket", bucket_name, &bucket_name);
+  RESTArgs::get_string(s, "tenant", "", &tenant);
   std::unique_ptr<rgw::sal::User> user = driver->get_user(rgw_user(uid_str));
   std::unique_ptr<rgw::sal::Bucket> bucket;
 
   if (!bucket_name.empty()) {
-    op_ret = driver->load_bucket(nullptr, user.get(),
-                                 rgw_bucket(std::string(), bucket_name),
+    op_ret = driver->load_bucket(this, rgw_bucket(tenant, bucket_name),
                                  &bucket, null_yield);
     if (op_ret < 0) {
       return;
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 2fb2ec98a06e..fc93fa10431f 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -291,10 +291,10 @@ class Driver {
      * there is a Bucket, otherwise use the get_object() in the Bucket class. */
     virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) = 0;
     /** Get a Bucket by info.  Does not query the driver, just uses the give bucket info. */
-    virtual std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) = 0;
+    virtual std::unique_ptr<Bucket> get_bucket(const RGWBucketInfo& i) = 0;
     /** Load a Bucket by key.  Queries driver for bucket info.  On -ENOENT, the
      * bucket must still be allocated to support bucket->create(). */
-    virtual int load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b,
+    virtual int load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
                             std::unique_ptr<Bucket>* bucket, optional_yield y) = 0;
     /** For multisite, this driver is the zone's master */
     virtual bool is_meta_master() = 0;
@@ -647,11 +647,9 @@ class Bucket {
     /** Set the ACL for this bucket */
     virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) = 0;
 
-    // XXXX hack
-    virtual void set_owner(rgw::sal::User* _owner) = 0;
-
     /// Input parameters for create().
     struct CreateParams {
+      rgw_user owner;
       std::string zonegroup_id;
       rgw_placement_rule placement_rule;
       // zone placement is optional on buckets created for another zonegroup
@@ -692,13 +690,11 @@ class Bucket {
                                     uint64_t num_objs, optional_yield y) = 0;
     /** Change the owner of this bucket in the backing store.  Current owner must be set.  Does not
      * change ownership of the objects in the bucket. */
-    virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) = 0;
+    virtual int chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y) = 0;
     /** Store the cached bucket info into the backing store */
     virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime, optional_yield y) = 0;
-    /** Check to see if the given user is the owner of this bucket */
-    virtual bool is_owner(User* user) = 0;
     /** Get the owner of this bucket */
-    virtual User* get_owner(void) = 0;
+    virtual const rgw_user& get_owner() const = 0;
     /** Check in the backing store if this bucket is empty */
     virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) = 0;
     /** Chec k if the given size fits within the quota */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index d406289f0488..6c8778b87238 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -62,15 +62,12 @@ namespace rgw::sal {
                        const CreateParams& params,
                        optional_yield y)
   {
-    ceph_assert(owner);
-    const rgw_user& owner_id = owner->get_id();
-
     rgw_bucket key = get_key();
     key.marker = params.marker;
     key.bucket_id = params.bucket_id;
 
     /* XXX: We may not need to send all these params. Cleanup the unused ones */
-    return store->getDB()->create_bucket(dpp, owner_id, key,
+    return store->getDB()->create_bucket(dpp, params.owner, key,
         params.zonegroup_id, params.placement_rule, params.attrs,
         params.swift_ver_location, params.quota, params.creation_time,
         &bucket_version, info, y);
@@ -241,11 +238,11 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBBucket::chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y)
+  int DBBucket::chown(const DoutPrefixProvider *dpp, const rgw_user& new_owner, optional_yield y)
   {
     int ret;
 
-    ret = store->getDB()->update_bucket(dpp, "owner", info, false, &(new_user.get_id()), nullptr, nullptr, nullptr);
+    ret = store->getDB()->update_bucket(dpp, "owner", info, false, &new_owner, nullptr, nullptr, nullptr);
     return ret;
   }
 
@@ -1552,16 +1549,16 @@ namespace rgw::sal {
   }
 
 
-  std::unique_ptr<Bucket> DBStore::get_bucket(User* u, const RGWBucketInfo& i)
+  std::unique_ptr<Bucket> DBStore::get_bucket(const RGWBucketInfo& i)
   {
     /* Don't need to fetch the bucket info, use the provided one */
-    return std::make_unique<DBBucket>(this, i, u);
+    return std::make_unique<DBBucket>(this, i);
   }
 
-  int DBStore::load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+  int DBStore::load_bucket(const DoutPrefixProvider *dpp, const rgw_bucket& b,
                            std::unique_ptr<Bucket>* bucket, optional_yield y)
   {
-    *bucket = std::make_unique<DBBucket>(this, b, u);
+    *bucket = std::make_unique<DBBucket>(this, b);
     return (*bucket)->load_bucket(dpp, y);
   }
 
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 26a8dfbcafdd..646ee23f8875 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -121,12 +121,6 @@ class DBNotification : public StoreNotification {
         acls() {
         }
 
-      DBBucket(DBStore *_st, User* _u)
-        : StoreBucket(_u),
-        store(_st),
-        acls() {
-        }
-
       DBBucket(DBStore *_st, const rgw_bucket& _b)
         : StoreBucket(_b),
         store(_st),
@@ -139,18 +133,6 @@ class DBNotification : public StoreNotification {
         acls() {
         }
 
-      DBBucket(DBStore *_st, const rgw_bucket& _b, User* _u)
-        : StoreBucket(_b, _u),
-        store(_st),
-        acls() {
-        }
-
-      DBBucket(DBStore *_st, const RGWBucketInfo& _i, User* _u)
-        : StoreBucket(_i, _u),
-        store(_st),
-        acls() {
-        }
-
       ~DBBucket() { }
 
       virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
@@ -178,7 +160,7 @@ class DBNotification : public StoreNotification {
                           RGWBucketEnt* ent) override;
       int check_bucket_shards(const DoutPrefixProvider *dpp,
                               uint64_t num_objs, optional_yield y) override;
-      virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override;
+      virtual int chown(const DoutPrefixProvider *dpp, const rgw_user& new_owner, optional_yield y) override;
       virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override;
       virtual int check_empty(const DoutPrefixProvider *dpp, optional_yield y) override;
       virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override;
@@ -763,8 +745,8 @@ class DBNotification : public StoreNotification {
       virtual int get_user_by_swift(const DoutPrefixProvider *dpp, const std::string& user_str, optional_yield y, std::unique_ptr<User>* user) override;
       virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
       virtual std::string get_cluster_id(const DoutPrefixProvider* dpp, optional_yield y);
-      std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
-      int load_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b,
+      std::unique_ptr<Bucket> get_bucket(const RGWBucketInfo& i) override;
+      int load_bucket(const DoutPrefixProvider *dpp, const rgw_bucket& b,
                       std::unique_ptr<Bucket>* bucket, optional_yield y) override;
       virtual bool is_meta_master() override;
       virtual Zone* get_zone() { return &zone; }
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index f3ee09fe0c98..b49c6887b62e 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -161,16 +161,17 @@ std::unique_ptr<Object> FilterDriver::get_object(const rgw_obj_key& k)
   return std::make_unique<FilterObject>(std::move(o));
 }
 
-std::unique_ptr<Bucket> FilterDriver::get_bucket(User* u, const RGWBucketInfo& i)
+std::unique_ptr<Bucket> FilterDriver::get_bucket(const RGWBucketInfo& i)
 {
-  return std::make_unique<FilterBucket>(next->get_bucket(nextUser(u), i), u);
+  return std::make_unique<FilterBucket>(next->get_bucket(i));
 }
 
-int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y)
+int FilterDriver::load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
+                              std::unique_ptr<Bucket>* bucket, optional_yield y)
 {
   std::unique_ptr<Bucket> nb;
-  const int ret = next->load_bucket(dpp, nextUser(u), b, &nb, y);
-  *bucket = std::make_unique<FilterBucket>(std::move(nb), u);
+  const int ret = next->load_bucket(dpp, b, &nb, y);
+  *bucket = std::make_unique<FilterBucket>(std::move(nb));
   return ret;
 }
 
@@ -629,9 +630,9 @@ int FilterBucket::check_bucket_shards(const DoutPrefixProvider* dpp,
   return next->check_bucket_shards(dpp, num_objs, y);
 }
 
-int FilterBucket::chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y)
+int FilterBucket::chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner, optional_yield y)
 {
-  return next->chown(dpp, new_user, y);
+  return next->chown(dpp, new_owner, y);
 }
 
 int FilterBucket::put_info(const DoutPrefixProvider* dpp, bool exclusive,
@@ -640,9 +641,9 @@ int FilterBucket::put_info(const DoutPrefixProvider* dpp, bool exclusive,
   return next->put_info(dpp, exclusive, _mtime, y);
 }
 
-bool FilterBucket::is_owner(User* user)
+const rgw_user& FilterBucket::get_owner() const
 {
-  return next->is_owner(nextUser(user));
+  return next->get_owner();
 }
 
 int FilterBucket::check_empty(const DoutPrefixProvider* dpp, optional_yield y)
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 943170dd0820..239a80a64653 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -157,10 +157,9 @@ class FilterDriver : public Driver {
 				std::string& user_str, optional_yield y,
 				std::unique_ptr<User>* user) override;
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& k) override;
-  std::unique_ptr<Bucket> get_bucket(User* u, const RGWBucketInfo& i) override;
-  int load_bucket(const DoutPrefixProvider* dpp, User* u,
-                  const rgw_bucket& b, std::unique_ptr<Bucket>* bucket,
-                  optional_yield y) override;
+  std::unique_ptr<Bucket> get_bucket(const RGWBucketInfo& i) override;
+  int load_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b,
+                  std::unique_ptr<Bucket>* bucket, optional_yield y) override;
   virtual bool is_meta_master() override;
   virtual Zone* get_zone() override { return zone.get(); }
   virtual std::string zone_unique_id(uint64_t unique_num) override;
@@ -365,13 +364,10 @@ class FilterUser : public User {
 class FilterBucket : public Bucket {
 protected:
   std::unique_ptr<Bucket> next;
-private:
-  User* user;
 
 public:
 
-  FilterBucket(std::unique_ptr<Bucket> _next, User* _user) :
-    next(std::move(_next)), user(_user) {}
+  FilterBucket(std::unique_ptr<Bucket> _next) : next(std::move(_next)) {}
   virtual ~FilterBucket() = default;
 
   virtual std::unique_ptr<Object> get_object(const rgw_obj_key& key) override;
@@ -389,7 +385,6 @@ class FilterBucket : public Bucket {
   virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl,
 		      optional_yield y) override;
 
-  virtual void set_owner(rgw::sal::User* _owner) override { next->set_owner(_owner); }
   virtual int create(const DoutPrefixProvider* dpp,
 		     const CreateParams& params,
 		     optional_yield y) override;
@@ -407,12 +402,11 @@ class FilterBucket : public Bucket {
                       RGWBucketEnt* ent) override;
   int check_bucket_shards(const DoutPrefixProvider* dpp,
                           uint64_t num_objs, optional_yield y) override;
-  virtual int chown(const DoutPrefixProvider* dpp, User& new_user,
+  virtual int chown(const DoutPrefixProvider* dpp, const rgw_user& new_owner,
 		    optional_yield y) override;
   virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive,
 		       ceph::real_time mtime, optional_yield y) override;
-  virtual bool is_owner(User* user) override;
-  virtual User* get_owner(void) override { return user; }
+  virtual const rgw_user& get_owner() const override;
   virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota,
 			  uint64_t obj_size, optional_yield y,
@@ -453,8 +447,7 @@ class FilterBucket : public Bucket {
   virtual bool versioning_enabled() override { return next->versioning_enabled(); }
 
   virtual std::unique_ptr<Bucket> clone() override {
-    std::unique_ptr<Bucket> nb = next->clone();
-    return std::make_unique<FilterBucket>(std::move(nb), user);
+    return std::make_unique<FilterBucket>(next->clone());
   }
 
   virtual std::unique_ptr<MultipartUpload> get_multipart_upload(
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index 943830c77fe4..b1592413d782 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -80,7 +80,6 @@ class StoreUser : public User {
 class StoreBucket : public Bucket {
   protected:
     RGWBucketInfo info;
-    User* owner = nullptr;
     Attrs attrs;
     obj_version bucket_version;
     ceph::real_time mtime;
@@ -88,22 +87,13 @@ class StoreBucket : public Bucket {
   public:
 
     StoreBucket() = default;
-    StoreBucket(User* u) : owner(u) { }
     StoreBucket(const rgw_bucket& b) { info.bucket = b; }
     StoreBucket(const RGWBucketInfo& i) : info(i) {}
-    StoreBucket(const rgw_bucket& b, User* u) : owner(u) { info.bucket = b; }
-    StoreBucket(const RGWBucketInfo& i, User* u) : info(i), owner(u) {}
     virtual ~StoreBucket() = default;
 
     virtual Attrs& get_attrs(void) override { return attrs; }
     virtual int set_attrs(Attrs a) override { attrs = a; return 0; }
-    virtual void set_owner(rgw::sal::User* _owner) override {
-      owner = _owner;
-      info.owner = owner->get_id();
-    }
-    virtual User* get_owner(void) override { return owner; };
-    /* Make sure to call get_bucket_info() if you need it first */
-    virtual bool is_owner(User* user) override { return (info.owner.compare(user->get_id()) == 0); }
+    virtual const rgw_user& get_owner() const override { return info.owner; };
     virtual bool empty() const override { return info.bucket.name.empty(); }
     virtual const std::string& get_name() const override { return info.bucket.name; }
     virtual const std::string& get_tenant() const override { return info.bucket.tenant; }
diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc
index 852449ad20f6..b58f0b45d8e7 100644
--- a/src/rgw/rgw_swift_auth.cc
+++ b/src/rgw/rgw_swift_auth.cc
@@ -119,7 +119,7 @@ void TempURLEngine::get_owner_info(const DoutPrefixProvider* dpp, const req_stat
   b.tenant = std::move(bucket_tenant);
   b.name = std::move(bucket_name);
   std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = driver->load_bucket(dpp, nullptr, b, &bucket, s->yield);
+  int ret = driver->load_bucket(dpp, b, &bucket, s->yield);
   if (ret < 0) {
     throw ret;
   }
diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index 1f88779d935d..02403e5f342e 100644
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -30,7 +30,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Driver* dri
 
     for (auto& ent : listing.buckets) {
       std::unique_ptr<rgw::sal::Bucket> bucket;
-      ret = driver->load_bucket(dpp, user, ent.bucket, &bucket, y);
+      ret = driver->load_bucket(dpp, ent.bucket, &bucket, y);
       if (ret < 0) {
         ldpp_dout(dpp, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl;
         continue;
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index 2d39981806c8..4f54d9b86bb0 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -117,7 +117,7 @@ class D4NFilterFixture : public ::testing::Test {
       RGWBucketInfo info;
       info.bucket.name = "test_bucket";
 
-      testBucket = driver->get_bucket(testUser.get(), info);
+      testBucket = driver->get_bucket(info);
 
       rgw::sal::Bucket::CreateParams params;
       params.zonegroup_id = "test_id";
@@ -1699,9 +1699,6 @@ TEST_F(D4NFilterFixture, StoreGetMetadata) {
   value.push_back(make_pair("source_zone_short_id", "300"));
   value.push_back(make_pair("bucket_count", "10"));
   value.push_back(make_pair("bucket_size", "20"));
-  value.push_back(make_pair("user_quota.max_size", "0"));
-  value.push_back(make_pair("user_quota.max_objects", "0"));
-  value.push_back(make_pair("max_buckets", "2000"));
 
   client.hmset("rgw-object:test_object_StoreGetMetadata:cache", value, [](cpp_redis::reply& reply) {
     if (!reply.is_null()) {
@@ -1730,7 +1727,6 @@ TEST_F(D4NFilterFixture, StoreGetMetadata) {
   ASSERT_EQ(testROp->prepare(null_yield, dpp), 0);
 
   /* Check updated metadata values */ 
-  RGWUserInfo info = testObject_StoreGetMetadata->get_bucket()->get_owner()->get_info();
   static StoreObject* storeObject = static_cast<StoreObject*>(dynamic_cast<rgw::sal::FilterObject*>(testObject_StoreGetMetadata.get())->get_next());
 
   EXPECT_EQ(to_iso_8601(storeObject->state.mtime), "2021-11-08T21:13:38.334696731Z");
@@ -1738,9 +1734,6 @@ TEST_F(D4NFilterFixture, StoreGetMetadata) {
   EXPECT_EQ(storeObject->state.accounted_size, (uint64_t)200);
   EXPECT_EQ(storeObject->state.epoch, (uint64_t)3);
   EXPECT_EQ(storeObject->state.zone_short_id, (uint32_t)300);
-  EXPECT_EQ(info.quota.user_quota.max_size, (int64_t)0);
-  EXPECT_EQ(info.quota.user_quota.max_objects, (int64_t)0);
-  EXPECT_EQ(testObject_StoreGetMetadata->get_bucket()->get_owner()->get_max_buckets(), (int32_t)2000);
 }
 
 TEST_F(D4NFilterFixture, StoreModifyAttr) {
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 8502ce81ef5a..be9fa6aed162 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -338,13 +338,14 @@ TEST(TestRGWLua, Bucket)
 
   DEFINE_REQ_STATE;
 
-  rgw_bucket b;
-  b.tenant = "mytenant";
-  b.name = "myname";
-  b.marker = "mymarker";
-  b.bucket_id = "myid"; 
-  s.bucket.reset(new sal::RadosBucket(nullptr, b));
-  s.bucket->set_owner(new sal::RadosUser(nullptr, rgw_user("mytenant", "myuser")));
+  RGWBucketInfo info;
+  info.bucket.tenant = "mytenant";
+  info.bucket.name = "myname";
+  info.bucket.marker = "mymarker";
+  info.bucket.bucket_id = "myid";
+  info.owner.id = "myuser";
+  info.owner.tenant = "mytenant";
+  s.bucket.reset(new sal::RadosBucket(nullptr, info));
 
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
@@ -1558,10 +1559,11 @@ TEST(TestRGWLua, DifferentContextUser)
   DEFINE_REQ_STATE;
 
   s.user.reset(new sal::RadosUser(nullptr, rgw_user("tenant1", "user1")));
-  rgw_bucket b;
-  b.name = "bucket1";
-  s.bucket.reset(new sal::RadosBucket(nullptr, b));
-  s.bucket->set_owner(new sal::RadosUser(nullptr, rgw_user("tenant2", "user2")));
+  RGWBucketInfo info;
+  info.bucket.name = "bucket1";
+  info.owner.id = "user2";
+  info.owner.tenant = "tenant2";
+  s.bucket.reset(new sal::RadosBucket(nullptr, info));
 
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);

From 434e5fe6aa69cad11454d437002015cff55b727a Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 16 Oct 2023 15:15:54 -0400
Subject: [PATCH 0528/2492] mgr/cephadm: warn when draining host explicitly
 listed in placement

In the case you apply a spec like

```
service_type: node-exporter
placement:
  hosts:
  - host3
```

and then you run `ceph orch host drain host3`, cephadm will remove
the daemon from that host and the placement would now match nothing.

This is definitely an issue that should be able to be bypassed as
it generally isn't serious, but it would be good to let users
know they have the host listed explicitly in placements like this
when they want to drain it.

Fixes: https://tracker.ceph.com/issues/63220

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index a83256d0bb7d..e54a5fe207f8 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -3414,6 +3414,18 @@ def drain_host(self, hostname: str, force: bool = False, keep_conf_keyring: bool
                                                   f"It is recommended to add the {SpecialHostLabels.ADMIN} label to another host"
                                                   " before completing this operation.\nIf you're certain this is"
                                                   " what you want rerun this command with --force.")
+            # if the user has specified the host we are going to drain
+            # explicitly in any service spec, warn the user. Having a
+            # drained host listed in a placement provides no value, so
+            # they may want to fix it.
+            services_matching_drain_host: List[str] = []
+            for sname, sspec in self.spec_store.all_specs.items():
+                if sspec.placement.hosts and hostname in [h.hostname for h in sspec.placement.hosts]:
+                    services_matching_drain_host.append(sname)
+            if services_matching_drain_host:
+                raise OrchestratorValidationError(f'Host {hostname} was found explicitly listed in the placements '
+                                                  f'of services:\n  {services_matching_drain_host}.\nPlease update those '
+                                                  'specs to not list this host.\nThis warning can be bypassed with --force')
 
         self.add_host_label(hostname, '_no_schedule')
         if not keep_conf_keyring:

From b4db5e4ffcf0fb345c99986718b16853f76b148a Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 6 Nov 2023 11:19:09 -0500
Subject: [PATCH 0529/2492] qa/cephadm: adjust host drain test to handle
 explicit placement warning

Since we're adding a warning if any host is listed explicitly
in the placement of any service when removing the host,
we need to adjust the host drain test that removes a host
without the --force flag to not have the explicit hostname
in the placement for the mon service.

Signed-off-by: Adam King <adking@redhat.com>
---
 .../orch/cephadm/workunits/task/test_host_drain.yaml   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
index 83173c9f8b22..c195bc052b7e 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
@@ -28,6 +28,16 @@ tasks:
             HOST_C="${host}"
           fi
         done
+        # One last thing to worry about before draining the host
+        # is that the teuthology test tends to put the explicit
+        # hostnames in the placement for the mon service.
+        # We want to make sure we can drain without providing
+        # --force and there is a check for the host being removed
+        # being listed explicitly in the placements. Therefore,
+        # we should remove it from the mon placement.
+        ceph orch ls mon --export > mon.yaml
+        sed /"$HOST_C"/d mon.yaml > mon_adjusted.yaml
+        ceph orch apply -i mon_adjusted.yaml
         # now drain that host
         ceph orch host drain $HOST_C --zap-osd-devices
         # wait for drain to complete

From 5c96c510db6b0abd4ec0f4b8325dd1732e9b4ce2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 16:59:33 -0400
Subject: [PATCH 0530/2492] cephadm: always pass ctx to
 customize_container_{binds,mounts}

These functions often derive the binds and/or mounts from the context
variable. Thus we should have the base class method accept the context.
Not all subclassess will use it but it will be there for those that do.
Also, fix the type for customize_container_mounts - it should be a dict
not a list.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_daemon_form.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index 5aef951f37c7..4da0624698bc 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -2,7 +2,7 @@
 
 import abc
 
-from typing import List, Tuple, Optional
+from typing import List, Tuple, Optional, Dict
 
 from .container_types import CephContainer, InitContainer
 from .context import CephadmContext
@@ -39,13 +39,17 @@ def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
         """
         return []
 
-    def customize_container_binds(self, binds: List[List[str]]) -> None:
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
         """Given a list of container binds this function can update, delete,
         or otherwise mutate the binds that the container will use.
         """
         pass
 
-    def customize_container_mounts(self, mounts: List[str]) -> None:
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
         """Given a list of container mounts this function can update, delete,
         or otherwise mutate the mounts that the container will use.
         """

From f80cf438b3f774e46da55b4cdc1b0417692eeed7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 17:57:59 -0400
Subject: [PATCH 0531/2492] cephadm: ensure the iscsi test can create iscsi
 daemon class

Ensure that the iscsi test can call CephIscsi.create inside other
functions by passing a required, but fake, gateway config file.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index f57bd11cab20..3f6ed70a39cd 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1773,6 +1773,11 @@ def test_get_container(self):
         """
         fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
         with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx:
+            ctx.config_blobs = {
+                'files': {
+                    'iscsi-gateway.cfg': 'portal',
+                },
+            }
             ctx.fsid = fsid
             c = _cephadm.get_container(
                 ctx, _cephadm.DaemonIdentity(fsid, 'iscsi', 'something')

From 21563528d1071654b8b77af0a65585915e73843f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 18:03:40 -0400
Subject: [PATCH 0532/2492] cephadm: covert iscsi type to provide a
 customize_container_binds method

Convert the CephIscsi type's get_container_binds to
customize_container_binds and use it in the generic method.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index db075949f803..9e2f8dab1ee2 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1087,16 +1087,16 @@ def get_container_mounts(data_dir, log_dir):
         mounts['/dev'] = '/dev'
         return mounts
 
-    @staticmethod
-    def get_container_binds():
-        # type: () -> List[List[str]]
-        binds = []
-        lib_modules = ['type=bind',
-                       'source=/lib/modules',
-                       'destination=/lib/modules',
-                       'ro=true']
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        lib_modules = [
+            'type=bind',
+            'source=/lib/modules',
+            'destination=/lib/modules',
+            'ro=true',
+        ]
         binds.append(lib_modules)
-        return binds
 
     @staticmethod
     def get_version(ctx, container_id):
@@ -2554,10 +2554,11 @@ def _write_custom_conf_files(
 def get_container_binds(
     ctx: CephadmContext, ident: 'DaemonIdentity'
 ) -> List[List[str]]:
-    binds = list()
+    binds: List[List[str]] = list()
 
     if ident.daemon_type == CephIscsi.daemon_type:
-        binds.extend(CephIscsi.get_container_binds())
+        iscsi = CephIscsi.create(ctx, ident)
+        iscsi.customize_container_binds(ctx, binds)
     if ident.daemon_type == CephNvmeof.daemon_type:
         binds.extend(CephNvmeof.get_container_binds())
     elif ident.daemon_type == CustomContainer.daemon_type:

From 34ef3f7554345ca1054ff8e1d9c8383a639b5b84 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 18:08:34 -0400
Subject: [PATCH 0533/2492] cephadm: covert nvmeof type to provide a
 customize_container_binds method

Convert the CephNvmeof type's get_container_binds to
customize_container_binds and use it in the generic method.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 9e2f8dab1ee2..d01619cb5aff 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1298,16 +1298,16 @@ def get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         return mounts
 
-    @staticmethod
-    def get_container_binds():
-        # type: () -> List[List[str]]
-        binds = []
-        lib_modules = ['type=bind',
-                       'source=/lib/modules',
-                       'destination=/lib/modules',
-                       'ro=true']
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        lib_modules = [
+            'type=bind',
+            'source=/lib/modules',
+            'destination=/lib/modules',
+            'ro=true',
+        ]
         binds.append(lib_modules)
-        return binds
 
     @staticmethod
     def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
@@ -2560,7 +2560,8 @@ def get_container_binds(
         iscsi = CephIscsi.create(ctx, ident)
         iscsi.customize_container_binds(ctx, binds)
     if ident.daemon_type == CephNvmeof.daemon_type:
-        binds.extend(CephNvmeof.get_container_binds())
+        nvmeof = CephNvmeof.create(ctx, ident)
+        nvmeof.customize_container_binds(ctx, binds)
     elif ident.daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
         data_dir = ident.data_dir(ctx.data_dir)

From 93ad2993bdd4a6701badc729b3e52e93f6ca384a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 18:19:57 -0400
Subject: [PATCH 0534/2492] cephadm: add customize_container_binds method to
 custom container

Add a customize_container_binds function to the CustomContainer daemon
type class and use it from the common get_container_binds function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 13 +++++++++----
 src/cephadm/tests/test_custom_container.py |  5 ++++-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index d01619cb5aff..7d6855778afa 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1854,7 +1854,7 @@ def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
             mounts[source] = destination
         return mounts
 
-    def get_container_binds(self, data_dir: str) -> List[List[str]]:
+    def _get_container_binds(self, data_dir: str) -> List[List[str]]:
         """
         Get the bind mounts. Relative `source=...` paths will be located below
         `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
@@ -1882,6 +1882,12 @@ def get_container_binds(self, data_dir: str) -> List[List[str]]:
                         data_dir, match.group(1)))
         return binds
 
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        binds.extend(self._get_container_binds(data_dir))
+
     # Cache the container so we don't need to rebuild it again when calling
     # into init_containers
     _container: Optional[CephContainer] = None
@@ -2563,9 +2569,8 @@ def get_container_binds(
         nvmeof = CephNvmeof.create(ctx, ident)
         nvmeof.customize_container_binds(ctx, binds)
     elif ident.daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
-        data_dir = ident.data_dir(ctx.data_dir)
-        binds.extend(cc.get_container_binds(data_dir))
+        cc = CustomContainer.create(ctx, ident)
+        cc.customize_container_binds(ctx, binds)
 
     return binds
 
diff --git a/src/cephadm/tests/test_custom_container.py b/src/cephadm/tests/test_custom_container.py
index c0f2a7966c44..0c020732cc7d 100644
--- a/src/cephadm/tests/test_custom_container.py
+++ b/src/cephadm/tests/test_custom_container.py
@@ -79,7 +79,10 @@ def test_get_container_mounts(self):
         })
 
     def test_get_container_binds(self):
-        result = self.cc.get_container_binds('/xyz')
+        # TODO: get_container_binds was made private. test the private func for
+        # now. in the future update to test base class fune
+        # customize_container_binds
+        result = self.cc._get_container_binds('/xyz')
         self.assertEqual(result, [
             [
                 'type=bind',

From 9dd73e4de0d16aaa6cb308b04f654a089ac147f8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 13:49:50 -0400
Subject: [PATCH 0535/2492] cephadm: move bind mounts assignment to a variable

Move the call to get_container_binds out of the function call.
This will aid with the next refactoring steps, so that the uses
of get_container_binds can be brought into the get_container call
directly.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7d6855778afa..d99af63ddefb 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2876,6 +2876,7 @@ def get_container(
         d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
+    binds = get_container_binds(ctx, ident)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,
@@ -2883,7 +2884,7 @@ def get_container(
         args=ceph_args + d_args,
         container_args=container_args,
         volume_mounts=get_container_mounts(ctx, ident),
-        bind_mounts=get_container_binds(ctx, ident),
+        bind_mounts=binds,
         envs=envs,
         privileged=privileged,
         ptrace=ptrace,

From 451305dc8d2719628e402f06750480165e82f751 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 13:56:31 -0400
Subject: [PATCH 0536/2492] cephadm: only call get_container_binds on types
 that have binds

This is a step towards not calling get_container_binds in get_container.
A future commit will replace uses of get_container_binds with direct
uses of common class methods.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index d99af63ddefb..03887b46e6b1 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2770,6 +2770,7 @@ def get_container(
     d_args: List[str] = []
     envs: List[str] = []
     host_network: bool = True
+    binds: List[List[str]] = []
 
     daemon_type = ident.daemon_type
     if daemon_type in ceph_daemons():
@@ -2855,12 +2856,14 @@ def get_container(
         container_args.extend(['--ulimit', 'memlock=-1:-1'])
         container_args.extend(['--ulimit', 'nofile=10240'])
         container_args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
+        binds = get_container_binds(ctx, ident)
     elif daemon_type == CephIscsi.daemon_type:
         entrypoint = CephIscsi.entrypoint
         name = ident.daemon_name
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
         privileged = True
+        binds = get_container_binds(ctx, ident)
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
         entrypoint = cc.entrypoint or ''
@@ -2868,6 +2871,7 @@ def get_container(
         envs.extend(cc.get_container_envs())
         container_args.extend(cc.get_container_args())
         d_args.extend(cc.get_daemon_args())
+        binds = get_container_binds(ctx, ident)
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
         container_args.append(
@@ -2876,7 +2880,6 @@ def get_container(
         d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
-    binds = get_container_binds(ctx, ident)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,

From f04c4b6c6f2b047260a003f5b47cb6b81a998d93 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 14:22:40 -0400
Subject: [PATCH 0537/2492] cephadm: convert get_container_binds to use class
 based approach

Since all types affected by get_container_binds now have the common
customize_container_binds, use a generic class-based approach by
creating an instance of ContainerDaemonForm and calling the method.
All other classes have a customize_container_binds that is a no-op.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 14 +++-----------
 src/cephadm/tests/test_cephadm.py |  4 ++--
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 03887b46e6b1..9a1989a3c956 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2561,17 +2561,9 @@ def get_container_binds(
     ctx: CephadmContext, ident: 'DaemonIdentity'
 ) -> List[List[str]]:
     binds: List[List[str]] = list()
-
-    if ident.daemon_type == CephIscsi.daemon_type:
-        iscsi = CephIscsi.create(ctx, ident)
-        iscsi.customize_container_binds(ctx, binds)
-    if ident.daemon_type == CephNvmeof.daemon_type:
-        nvmeof = CephNvmeof.create(ctx, ident)
-        nvmeof.customize_container_binds(ctx, binds)
-    elif ident.daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.create(ctx, ident)
-        cc.customize_container_binds(ctx, binds)
-
+    daemon = daemon_form_create(ctx, ident)
+    assert isinstance(daemon, ContainerDaemonForm)
+    daemon.customize_container_binds(ctx, binds)
     return binds
 
 
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 3f6ed70a39cd..8db8edd0c1bf 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1575,7 +1575,7 @@ def test_name(self, cephadm_fs):
             retval = _cephadm.command_shell(ctx)
             assert retval == 0
 
-        cmd = ['shell', '--name', 'foo.bar']
+        cmd = ['shell', '--name', 'mgr.bar']
         with with_cephadm_ctx(cmd) as ctx:
             err = r'must pass --fsid'
             with pytest.raises(_cephadm.Error, match=err):
@@ -1583,7 +1583,7 @@ def test_name(self, cephadm_fs):
                 assert retval == 1
 
         fsid = '00000000-0000-0000-0000-0000deadbeef'
-        cmd = ['shell', '--name', 'foo.bar', '--fsid', fsid]
+        cmd = ['shell', '--name', 'mgr.bar', '--fsid', fsid]
         with with_cephadm_ctx(cmd) as ctx:
             retval = _cephadm.command_shell(ctx)
             assert retval == 0

From f6b686ac3118507baab719f1587c8c360499e929 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 10 Nov 2023 05:50:21 +1000
Subject: [PATCH 0538/2492] doc/rados: edit t-mon "common issues" (4 of x)

Edit the fourth part of the section "Most Common Monitor Issues" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Follows https://github.com/ceph/ceph/pull/54432.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 37 +++++++++++--------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 740de9be017d..3a6423fef109 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -272,21 +272,28 @@ detail`` returns a message similar to the following::
   you put the problematic Monitor into a ``down`` state while you investigate.
   This is possible only if there are enough surviving Monitors to form quorum. 
 
-**What if state is ``synchronizing``?**
-
-  This means the monitor is catching up with the rest of the cluster in
-  order to join the quorum. Time to synchronize is a function of the size
-  of your monitor store and thus of cluster size and state, so if you have a
-  large or degraded cluster this may take a while.
-
-  If you notice that the monitor jumps from ``synchronizing`` to
-  ``electing`` and then back to ``synchronizing``, then you do have a
-  problem: the cluster state may be advancing (i.e., generating new maps)
-  too fast for the synchronization process to keep up. This was a more common
-  thing in early days (Cuttlefish), but since then the synchronization process
-  has been refactored and enhanced to avoid this dynamic. If you experience
-  this in later versions please let us know via a bug tracker. And bring some logs
-  (see `Preparing your logs`_).
+**What does it mean when a Monitor's state is ``synchronizing``?**
+
+  If ``ceph health detail`` shows that the Monitor is ``synchronizing``, the
+  monitor is catching up with the rest of the cluster so that it can join the
+  quorum. The amount of time that it takes for the Monitor to synchronize with
+  the rest of the quorum is a function of the size of the cluster's monitor
+  store, the cluster's size, and the state of the cluster. Larger and degraded
+  clusters generally keep Monitors in the ``synchronizing`` state longer than
+  do smaller, new clusters.
+
+  A Monitor that changes its state from ``synchronizing`` to ``electing`` and
+  then back to ``synchronizing`` indicates a problem: the cluster state may be
+  advancing (that is, generating new maps) too fast for the synchronization
+  process to keep up with the pace of the creation of the new maps. This issue
+  presented more frequently prior to the Cuttlefish release than it does in
+  more recent releases, because the synchronization process has since been
+  refactored and enhanced to avoid this dynamic. If you experience this in
+  later versions, report the issue in the `Ceph bug tracker
+  <https://tracker.ceph.com>`_. Prepare and provide logs to substantiate any
+  bug you raise. See `Preparing your logs`_ for information about the proper
+  preparation of logs.
+
 
 **What if state is ``leader`` or ``peon``?**
 

From 78948e78d5f62d7c4dd13eec74464304c8dca0e5 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Thu, 9 Nov 2023 16:41:04 -0500
Subject: [PATCH 0539/2492] rgw: make subclass dependencies explicit

As part of the Zipper project generic back-end code is being teased
apart from rados-specific back-end code. This is a work in progress,
so currently generic code and other subclasses of StoreDriver (and
related high-level classes) depend on the rados-specific declarations.
Some of these dependencies are not always obvious since
src/rgw/driver/rados was put on the include path. That is now removed,
so any includes needing files from that subclass have to give a more
fully specified path.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/CMakeLists.txt                  | 1 -
 src/rgw/driver/dbstore/common/dbstore.h | 3 ++-
 src/rgw/driver/dbstore/config/sqlite.cc | 3 ++-
 src/rgw/rgw_basic_types.h               | 3 ++-
 src/rgw/rgw_multi.h                     | 3 ++-
 src/rgw/rgw_sal.h                       | 6 ++++--
 src/rgw/services/svc_bucket_sync.h      | 2 +-
 src/rgw/services/svc_mdlog.cc           | 3 ++-
 src/rgw/services/svc_meta_be.h          | 3 ++-
 src/rgw/services/svc_user_rados.h       | 3 ++-
 10 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 9b55081dbaaa..ddd9be20d2df 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -287,7 +287,6 @@ target_link_libraries(rgw_common
     ${FMT_LIB})
 target_include_directories(rgw_common
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/services"
-  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
   PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
   PUBLIC "${LUA_INCLUDE_DIR}")
 
diff --git a/src/rgw/driver/dbstore/common/dbstore.h b/src/rgw/driver/dbstore/common/dbstore.h
index 79f312cb8446..d3bdf8805381 100644
--- a/src/rgw/driver/dbstore/common/dbstore.h
+++ b/src/rgw/driver/dbstore/common/dbstore.h
@@ -18,9 +18,10 @@
 #include "global/global_context.h"
 #include "global/global_init.h"
 #include "common/ceph_context.h"
-#include "rgw_obj_manifest.h"
 #include "rgw_multi.h"
 
+#include "driver/rados/rgw_obj_manifest.h" // FIXME: subclass dependency
+
 namespace rgw { namespace store {
 
 class DB;
diff --git a/src/rgw/driver/dbstore/config/sqlite.cc b/src/rgw/driver/dbstore/config/sqlite.cc
index 1f759c69178e..aa2d1a354ffc 100644
--- a/src/rgw/driver/dbstore/config/sqlite.cc
+++ b/src/rgw/driver/dbstore/config/sqlite.cc
@@ -25,7 +25,8 @@
 #include "include/encoding.h"
 #include "common/dout.h"
 #include "common/random_string.h"
-#include "rgw_zone.h"
+
+#include "driver/rados/rgw_zone.h" // FIXME: subclass dependency
 
 #include "common/connection_pool.h"
 #include "sqlite/connection.h"
diff --git a/src/rgw/rgw_basic_types.h b/src/rgw/rgw_basic_types.h
index cd30d2344bad..4cffb333288e 100644
--- a/src/rgw/rgw_basic_types.h
+++ b/src/rgw/rgw_basic_types.h
@@ -31,7 +31,8 @@
 #include "rgw_user_types.h"
 #include "rgw_bucket_types.h"
 #include "rgw_obj_types.h"
-#include "rgw_obj_manifest.h"
+
+#include "driver/rados/rgw_obj_manifest.h" // FIXME: subclass dependency
 
 #include "common/Formatter.h"
 
diff --git a/src/rgw/rgw_multi.h b/src/rgw/rgw_multi.h
index f57c90e74895..06a192fc72d8 100644
--- a/src/rgw/rgw_multi.h
+++ b/src/rgw/rgw_multi.h
@@ -6,11 +6,12 @@
 #include <map>
 #include "rgw_xml.h"
 #include "rgw_obj_types.h"
-#include "rgw_obj_manifest.h"
 #include "rgw_compression_types.h"
 #include "common/dout.h"
 #include "rgw_sal_fwd.h"
 
+#include "driver/rados/rgw_obj_manifest.h" // FIXME: subclass dependency
+
 #define MULTIPART_UPLOAD_ID_PREFIX_LEGACY "2/"
 #define MULTIPART_UPLOAD_ID_PREFIX "2~" // must contain a unique char that may not come up in gen_rand_alpha()
 
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index b222fcd03cbc..9e047ad04b43 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -18,12 +18,14 @@
 #include "common/tracer.h"
 #include "rgw_sal_fwd.h"
 #include "rgw_lua.h"
-#include "rgw_user.h"
 #include "rgw_notify_event_type.h"
 #include "rgw_req_context.h"
-#include "rgw_datalog_notify.h"
 #include "include/random.h"
 
+// FIXME: following subclass dependencies
+#include "driver/rados/rgw_user.h"
+#include "driver/rados/rgw_datalog_notify.h"
+
 struct RGWBucketEnt;
 class RGWRESTMgr;
 class RGWAccessListFilter;
diff --git a/src/rgw/services/svc_bucket_sync.h b/src/rgw/services/svc_bucket_sync.h
index 7975e062bb6f..37dada55ecd8 100644
--- a/src/rgw/services/svc_bucket_sync.h
+++ b/src/rgw/services/svc_bucket_sync.h
@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "rgw_service.h"
+#include "driver/rados/rgw_service.h" // FIXME: subclass dependency
 
 #include "svc_bucket_types.h"
 
diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc
index 06459f5239ea..da723f3e1295 100644
--- a/src/rgw/services/svc_mdlog.cc
+++ b/src/rgw/services/svc_mdlog.cc
@@ -10,7 +10,8 @@
 #include "rgw_mdlog.h"
 #include "rgw_coroutine.h"
 #include "rgw_cr_rados.h"
-#include "rgw_zone.h"
+
+#include "driver/rados/rgw_zone.h" // FIXME: subclass dependency
 
 #include "common/errno.h"
 
diff --git a/src/rgw/services/svc_meta_be.h b/src/rgw/services/svc_meta_be.h
index 97267a4e7e3e..b580d173f220 100644
--- a/src/rgw/services/svc_meta_be.h
+++ b/src/rgw/services/svc_meta_be.h
@@ -19,9 +19,10 @@
 
 #include "svc_meta_be_params.h"
 
-#include "rgw_service.h"
 #include "rgw_mdlog_types.h"
 
+#include "driver/rados/rgw_service.h" // FIXME: subclass dependency
+
 class RGWMetadataLogData;
 
 class RGWSI_MDLog;
diff --git a/src/rgw/services/svc_user_rados.h b/src/rgw/services/svc_user_rados.h
index 177f720d6b18..4712d1fdafd4 100644
--- a/src/rgw/services/svc_user_rados.h
+++ b/src/rgw/services/svc_user_rados.h
@@ -20,7 +20,8 @@
 
 #include "svc_meta_be.h"
 #include "svc_user.h"
-#include "rgw_bucket.h"
+
+#include "driver/rados/rgw_bucket.h" // FIXME: subclass dependency
 
 class RGWSI_RADOS;
 class RGWSI_Zone;

From 91c0334f62ee615e185f78a63c7c5ef9c1ce7b09 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 10 Nov 2023 09:20:11 -0500
Subject: [PATCH 0540/2492] test/rgw: clean up unused include dirs

store/rados was renamed to driver/rados, so these include directories
aren't required

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/rgw/CMakeLists.txt | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
index 2e29f4b70b15..d6bb3db1c690 100644
--- a/src/test/rgw/CMakeLists.txt
+++ b/src/test/rgw/CMakeLists.txt
@@ -211,15 +211,13 @@ target_link_libraries(unittest_rgw_iam_policy
 add_executable(unittest_rgw_string test_rgw_string.cc)
 add_ceph_unittest(unittest_rgw_string)
 target_include_directories(unittest_rgw_string
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 
 # unitttest_rgw_dmclock_queue
 add_executable(unittest_rgw_dmclock_scheduler test_rgw_dmclock_scheduler.cc $<TARGET_OBJECTS:unit-main>)
 add_ceph_unittest(unittest_rgw_dmclock_scheduler)
 target_include_directories(unittest_rgw_dmclock_scheduler
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 
 target_link_libraries(unittest_rgw_dmclock_scheduler rgw_schedulers global ${UNITTEST_LIBS})
 
@@ -227,8 +225,7 @@ if(WITH_RADOSGW_AMQP_ENDPOINT)
   add_executable(unittest_rgw_amqp test_rgw_amqp.cc)
   add_ceph_unittest(unittest_rgw_amqp)
   target_include_directories(unittest_rgw_amqp
-    SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-    SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+    SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
   target_link_libraries(unittest_rgw_amqp ${rgw_libs})
 endif()
 
@@ -236,16 +233,14 @@ endif()
 add_executable(unittest_rgw_xml test_rgw_xml.cc)
 add_ceph_unittest(unittest_rgw_xml)
 target_include_directories(unittest_rgw_xml
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES})
 
 # unittest_rgw_lc
 add_executable(unittest_rgw_lc test_rgw_lc.cc)
 add_ceph_unittest(unittest_rgw_lc)
 target_include_directories(unittest_rgw_lc
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_lc
   rgw_common ${rgw_libs} ${EXPAT_LIBRARIES})
 
@@ -253,30 +248,26 @@ target_link_libraries(unittest_rgw_lc
 add_executable(unittest_rgw_arn test_rgw_arn.cc)
 add_ceph_unittest(unittest_rgw_arn)
 target_include_directories(unittest_rgw_arn
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_arn ${rgw_libs})
 
 # unittest_rgw_kms
 add_executable(unittest_rgw_kms test_rgw_kms.cc)
 add_ceph_unittest(unittest_rgw_kms)
 target_include_directories(unittest_rgw_kms
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_kms ${rgw_libs})
 
 # unittest_rgw_url
 add_executable(unittest_rgw_url test_rgw_url.cc)
 add_ceph_unittest(unittest_rgw_url)
 target_include_directories(unittest_rgw_url
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_url ${rgw_libs})
 
 add_executable(ceph_test_rgw_gc_log test_rgw_gc_log.cc $<TARGET_OBJECTS:unit-main>)
 target_include_directories(ceph_test_rgw_gc_log
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(ceph_test_rgw_gc_log ${rgw_libs} radostest-cxx)
 install(TARGETS ceph_test_rgw_gc_log DESTINATION ${CMAKE_INSTALL_BINDIR})
 
@@ -286,8 +277,7 @@ add_ceph_test(test-ceph-diff-sorted.sh
 # unittest_cls_fifo_legacy
 add_executable(unittest_cls_fifo_legacy test_cls_fifo_legacy.cc)
 target_include_directories(unittest_cls_fifo_legacy
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_cls_fifo_legacy radostest-cxx ${UNITTEST_LIBS}
   ${rgw_libs})
 
@@ -302,8 +292,7 @@ target_link_libraries(unittest_log_backing radostest-cxx ${UNITTEST_LIBS}
 add_executable(unittest_rgw_lua test_rgw_lua.cc)
 add_ceph_unittest(unittest_rgw_lua)
 target_include_directories(unittest_rgw_lua
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw"
-  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados")
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_lua ${rgw_libs})
 
 add_executable(radosgw-cr-test rgw_cr_test.cc)

From cc747a2d0732776af8af8c3f6d6bbc39205ed095 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 3 Nov 2023 13:46:08 -0400
Subject: [PATCH 0541/2492] script: enable checkout of specific sha1 as base

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/script/ptl-tool.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py
index 095fceb41fc0..176f61f22c8c 100755
--- a/src/script/ptl-tool.py
+++ b/src/script/ptl-tool.py
@@ -302,12 +302,13 @@ def build_branch(args):
         try:
             base_path = args.base_path + base
             base = next(ref for ref in G.refs if ref.path == base_path)
+            # So we know that we're not on an old test branch, detach HEAD onto ref:
+            base.checkout()
         except StopIteration:
-            log.error("Branch " + base + " does not exist!")
-            sys.exit(1)
-
-        # So we know that we're not on an old test branch, detach HEAD onto ref:
-        base.checkout()
+            log.info(f"Trying to checkout uninterpreted base {base}")
+            c = G.commit(base)
+            G.git.checkout(c)
+        assert G.head.is_detached
 
     for pr in prs:
         pr = int(pr)

From 33899fdaac57db5266940f59f5ef956932aa1714 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Sat, 11 Nov 2023 09:09:55 +0530
Subject: [PATCH 0542/2492] qa: bump up scrub status command timeout

A journal flush sometimes takes more than 120 seconds and so the 'scrub
status' command after blocking for more than 120 seconds is declared
failed causing the job to be declared as failed.

This bumping up of the timeout gives more time for the 'scrub status'
command to wait and eventually let the journal flush to complete.

Fixes: https://tracker.ceph.com/issues/63411
Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 qa/tasks/ceph_manager.py      | 15 +++++++++------
 qa/tasks/cephfs/filesystem.py | 12 ++++++------
 qa/tasks/vstart_runner.py     |  4 +++-
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 86b57028ee27..fa909c1a77ca 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -1540,11 +1540,9 @@ def tmp(x):
         self.cephadm = cephadm
         self.testdir = teuthology.get_testdir(self.ctx)
         # prefix args for ceph cmds to be executed
-        pre = ['adjust-ulimits', 'ceph-coverage',
-               f'{self.testdir}/archive/coverage']
-        self.CEPH_CMD = ['sudo'] + pre + ['timeout', '120', 'ceph',
-                                          '--cluster', self.cluster]
-        self.RADOS_CMD = pre + ['rados', '--cluster', self.cluster]
+        self.pre = ['adjust-ulimits', 'ceph-coverage',
+                    f'{self.testdir}/archive/coverage']
+        self.RADOS_CMD = self.pre + ['rados', '--cluster', self.cluster]
 
         pools = self.list_pools()
         self.pools = {}
@@ -1555,6 +1553,11 @@ def tmp(x):
             except CommandFailedError:
                 self.log('Failed to get pg_num from pool %s, ignoring' % pool)
 
+    def get_ceph_cmd(self, **kwargs):
+        timeout = kwargs.pop('timeout', 120)
+        return ['sudo'] + self.pre + ['timeout', f'{timeout}', 'ceph',
+                                      '--cluster', self.cluster]
+
     def ceph(self, cmd, **kwargs):
         """
         Simple Ceph admin command wrapper around run_cluster_cmd.
@@ -1598,7 +1601,7 @@ def run_cluster_cmd(self, **kwargs):
                            stdout=StringIO(),
                            check_status=kwargs.get('check_status', True))
         else:
-            kwargs['args'] = prefixcmd + self.CEPH_CMD + kwargs['args']
+            kwargs['args'] = prefixcmd + self.get_ceph_cmd(**kwargs) + kwargs['args']
             return self.controller.run(**kwargs)
 
     def raw_cluster_cmd(self, *args, **kwargs) -> str:
diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 3516bf4b86cd..7de5653021bf 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -1295,9 +1295,9 @@ def rank_asok(self, command, rank=0, status=None, timeout=None):
         info = self.get_rank(rank=rank, status=status)
         return self.json_asok(command, 'mds', info['name'], timeout=timeout)
 
-    def rank_tell(self, command, rank=0, status=None):
+    def rank_tell(self, command, rank=0, status=None, timeout=120):
         try:
-            out = self.get_ceph_cmd_stdout("tell", f"mds.{self.id}:{rank}", *command)
+            out = self.get_ceph_cmd_stdout("tell", f"mds.{self.id}:{rank}", *command, timeout=timeout)
             return json.loads(out)
         except json.decoder.JSONDecodeError:
             log.error("could not decode: {}".format(out))
@@ -1712,11 +1712,11 @@ def shrink(self, new_max_mds, status=None):
         self.set_max_mds(new_max_mds)
         return self.wait_for_daemons()
 
-    def run_scrub(self, cmd, rank=0):
-        return self.rank_tell(["scrub"] + cmd, rank)
+    def run_scrub(self, cmd, rank=0, timeout=300):
+        return self.rank_tell(["scrub"] + cmd, rank=rank, timeout=timeout)
 
     def get_scrub_status(self, rank=0):
-        return self.run_scrub(["status"], rank)
+        return self.run_scrub(["status"], rank=rank, timeout=300)
 
     def flush(self, rank=0):
         return self.rank_tell(["flush", "journal"], rank=rank)
@@ -1728,7 +1728,7 @@ def wait_until_scrub_complete(self, result=None, tag=None, rank=0, sleep=30,
             result = "no active scrubs running"
         with contextutil.safe_while(sleep=sleep, tries=timeout//sleep) as proceed:
             while proceed():
-                out_json = self.rank_tell(["scrub", "status"], rank=rank)
+                out_json = self.rank_tell(["scrub", "status"], rank=rank, timeout=timeout)
                 assert out_json is not None
                 if not reverse:
                     if result in out_json['status']:
diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index 96dc9fffba2f..8d9dc6f1845c 100644
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -804,9 +804,11 @@ def __init__(self, ctx=None):
         self.cephadm = False
         self.rook = False
         self.testdir = None
-        self.CEPH_CMD = [CEPH_CMD]
         self.RADOS_CMD = [RADOS_CMD]
 
+    def get_ceph_cmd(self, **kwargs):
+        return [CEPH_CMD]
+
     def find_remote(self, daemon_type, daemon_id):
         """
         daemon_type like 'mds', 'osd'

From d98222db9b4436841102d23c979f05a9d958bb1f Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 11 Nov 2023 13:45:42 +1000
Subject: [PATCH 0543/2492] doc/rados: edit t-mon "common issues" (5 of x)

Edit the fifth part of the section "Most Common Monitor Issues" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Follows https://github.com/ceph/ceph/pull/54440.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst     | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 3a6423fef109..6add6127e16f 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -294,13 +294,16 @@ detail`` returns a message similar to the following::
   bug you raise. See `Preparing your logs`_ for information about the proper
   preparation of logs.
 
-
-**What if state is ``leader`` or ``peon``?**
-
-  This should not happen:  famous last words.  If it does, however, it likely
-  has a lot to do with clock skew -- see `Clock Skews`_. If you are not
-  suffering from clock skew, then please prepare your logs (see
-  `Preparing your logs`_) and reach out to the community.
+**What does it mean when the Monitor state is ``leader`` or ``peon``?**
+
+  If ``ceph health detail`` shows that the Monitor is in the ``leader`` state
+  or in the ``peon`` state, it is likely that clock skew is present. Follow the
+  instructions in `Clock Skews`_. If you have followed those instructions and
+  ``ceph health detail`` still shows that the Monitor is in the ``leader``
+  state or the ``peon`` state, report the issue in the `Ceph bug tracker
+  <https://tracker.ceph.com>`_. If you raise an issue, provide logs to
+  substantiate it. See `Preparing your logs`_ for information about the
+  proper preparation of logs.
 
 
 Recovering a Monitor's Broken ``monmap``

From 60cee3876a94391e1d6817d8810991e3956d5ee6 Mon Sep 17 00:00:00 2001
From: Sascha Lucas <sascha_lucas@web.de>
Date: Fri, 10 Nov 2023 18:40:57 +0100
Subject: [PATCH 0544/2492] ceph-volume: fix blkid on inaccessible devices

There exists block devices which can not be accessed. For example DRBD
/dev/drbdX in secondary state. `blkid` can not read from this:

  $ blkid -c /dev/null -p /dev/drbd4
  blkid: error: /dev/drbd4: Wrong medium type

Since release 17.2.7 this results in "CEPHADM_REFRESH_FAILED: failed to
probe daemons or devices"[1] and the following trace:

  File "/usr/lib/python3.6/site-packages/ceph_volume/util/device.py", line 482, in is_partition
    return self.blkid_api['TYPE'] == 'part'

Fix this by using `get()` method, so that it defaults to `None` if the
key does not exist. And while at it also switch to `get()` in an other
reference to this key.

Probably caused by commit 2422ad867dff9d526d7e8be543178c897991097f ???

[1] https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/thread/ZSD4OPDMCEXOW74IJH4L4D5PQXFFINGS/

Signed-off-by: Sascha Lucas <sascha_lucas@web.de>
---
 src/ceph-volume/ceph_volume/util/device.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index bb806292f2c4..a9acb7f20437 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -464,7 +464,7 @@ def device_type(self):
         elif self.disk_api:
             return self.disk_api['TYPE']
         elif self.blkid_api:
-            return self.blkid_api['TYPE']
+            return self.blkid_api.get('TYPE')
 
     @property
     def is_mpath(self):
@@ -480,7 +480,7 @@ def is_partition(self):
         if self.disk_api:
             return self.disk_api['TYPE'] == 'part'
         elif self.blkid_api:
-            return self.blkid_api['TYPE'] == 'part'
+            return self.blkid_api.get('TYPE') == 'part'
         return False
 
     @property

From 653ae9ebc143617d99997fa872ddaf0c5fd60223 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 6 Nov 2023 04:08:57 -0600
Subject: [PATCH 0545/2492] osd/scrub: modify (deep)scrub ASOK commands

Fix all instances of 'deep_scrub' to the documented format (deep-scrub);

Make all formats of the documented command ('ceph tell $pg
[deep-]scrub', and the old 'ceph pg scrub $pgid') trigger an
operator-priority scrub;

Introduce a dev/tests-only format for the required functionality
of "trigger a scrub by modifying the timestamps".

Change some implementation details.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc                  | 44 +++++++++++++-------
 src/osd/PrimaryLogPG.cc         | 50 +++++++++--------------
 src/osd/scrubber/pg_scrubber.cc | 72 +++++++++++++++++++++++++++++++++
 src/osd/scrubber/pg_scrubber.h  | 10 +++++
 src/osd/scrubber_common.h       | 15 +++++++
 5 files changed, 145 insertions(+), 46 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 8a13353db98e..8608c1365655 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2633,7 +2633,9 @@ void OSD::asok_command(
       prefix == "mark_unfound_lost" ||
       prefix == "list_unfound" ||
       prefix == "scrub" ||
-      prefix == "deep_scrub"
+      prefix == "deep-scrub" ||
+      prefix == "schedule-scrub" ||      ///< dev/tests only!
+      prefix == "schedule-deep-scrub"    ///< dev/tests only!
     ) {
     string pgidstr;
     pg_t pgid;
@@ -4363,18 +4365,16 @@ void OSD::final_init()
     "");
   ceph_assert(r == 0);
   r = admin_socket->register_command(
-    "pg "			   \
-    "name=pgid,type=CephPgid "	   \
-    "name=cmd,type=CephChoices,strings=scrub " \
-    "name=time,type=CephInt,req=false",
+    "pg "
+    "name=pgid,type=CephPgid "
+    "name=cmd,type=CephChoices,strings=scrub",
     asok_hook,
     "");
   ceph_assert(r == 0);
   r = admin_socket->register_command(
-    "pg "			   \
-    "name=pgid,type=CephPgid "	   \
-    "name=cmd,type=CephChoices,strings=deep_scrub " \
-    "name=time,type=CephInt,req=false",
+    "pg "
+    "name=pgid,type=CephPgid "
+    "name=cmd,type=CephChoices,strings=deep-scrub",
     asok_hook,
     "");
   ceph_assert(r == 0);
@@ -4403,19 +4403,33 @@ void OSD::final_init()
     asok_hook,
     "list unfound objects on this pg, perhaps starting at an offset given in JSON");
   ceph_assert(r == 0);
+  // the operator commands (force a scrub)
+  r = admin_socket->register_command(
+    "scrub "
+    "name=pgid,type=CephPgid,req=false",
+    asok_hook,
+    "Trigger a scrub");
+  ceph_assert(r == 0);
+  r = admin_socket->register_command(
+    "deep-scrub "
+    "name=pgid,type=CephPgid,req=false",
+    asok_hook,
+    "Trigger a deep scrub");
+  ceph_assert(r == 0);
+  // debug/test commands (faking the timestamps)
   r = admin_socket->register_command(
-    "scrub "				\
-    "name=pgid,type=CephPgid,req=false "	\
+    "schedule-scrub "
+    "name=pgid,type=CephPgid,req=false "
     "name=time,type=CephInt,req=false",
     asok_hook,
-    "Trigger a scheduled scrub ");
+    "Schedule a scrub");
   ceph_assert(r == 0);
   r = admin_socket->register_command(
-    "deep_scrub "			\
-    "name=pgid,type=CephPgid,req=false "	\
+    "schedule-deep-scrub "
+    "name=pgid,type=CephPgid,req=false "
     "name=time,type=CephInt,req=false",
     asok_hook,
-    "Trigger a scheduled deep scrub ");
+    "Schedule a deep scrub");
   ceph_assert(r == 0);
 }
 
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index ba13b9afa9b8..a2c2f15d739e 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1158,39 +1158,26 @@ void PrimaryLogPG::do_command(
     f->close_section();
   }
 
-  else if (prefix == "scrub" ||
-	   prefix == "deep_scrub") {
-    bool deep = (prefix == "deep_scrub");
-    int64_t time = cmd_getval_or<int64_t>(cmdmap, "time", 0);
+  else if (prefix == "scrub" || prefix == "deep-scrub") {
+    if (is_primary()) {
+      scrub_level_t deep = (prefix == "deep-scrub") ? scrub_level_t::deep
+						    : scrub_level_t::shallow;
+      m_scrubber->on_operator_forced_scrub(f.get(), deep, m_planned_scrub);
+    } else {
+      ss << "Not primary";
+      ret = -EPERM;
+    }
+    outbl.append(ss.str());
+  }
 
+  // the test/debug commands that schedule a scrub by modifying timestamps
+  else if (prefix == "schedule-scrub" || prefix == "schedule-deep-scrub") {
     if (is_primary()) {
-      const pg_pool_t *p = &pool.info;
-      double pool_scrub_max_interval = 0;
-      double scrub_max_interval;
-      if (deep) {
-        p->opts.get(pool_opts_t::DEEP_SCRUB_INTERVAL, &pool_scrub_max_interval);
-        scrub_max_interval = pool_scrub_max_interval > 0 ?
-          pool_scrub_max_interval : g_conf()->osd_deep_scrub_interval;
-      } else {
-        p->opts.get(pool_opts_t::SCRUB_MAX_INTERVAL, &pool_scrub_max_interval);
-        scrub_max_interval = pool_scrub_max_interval > 0 ?
-          pool_scrub_max_interval : g_conf()->osd_scrub_max_interval;
-      }
-      // Instead of marking must_scrub force a schedule scrub
-      utime_t stamp = ceph_clock_now();
-      if (time == 0)
-        stamp -= scrub_max_interval;
-      else
-        stamp -=  (float)time;
-      stamp -= 100.0;  // push back last scrub more for good measure
-      if (deep) {
-        set_last_deep_scrub_stamp(stamp);
-      }
-      set_last_scrub_stamp(stamp); // for 'deep' as well, as we use this value to order scrubs
-      f->open_object_section("result");
-      f->dump_bool("deep", deep);
-      f->dump_stream("stamp") << stamp;
-      f->close_section();
+      scrub_level_t deep = (prefix == "schedule-deep-scrub")
+			       ? scrub_level_t::deep
+			       : scrub_level_t::shallow;
+      const int64_t offst = cmd_getval_or<int64_t>(cmdmap, "time", 0);
+      m_scrubber->on_operator_periodic_cmd(f.get(), deep, offst);
     } else {
       ss << "Not primary";
       ret = -EPERM;
@@ -1214,6 +1201,7 @@ void PrimaryLogPG::do_command(
     }
     outbl.append(ss.str());
   }
+
   else {
     ret = -ENOSYS;
     ss << "prefix '" << prefix << "' not implemented";
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index babd8782e9ca..4cd861b89c8f 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -656,6 +656,78 @@ Scrub::sched_conf_t PgScrubber::populate_config_params() const
 }
 
 
+// handling Asok's "scrub" & "deep-scrub" commands
+
+namespace {
+void asok_response_section(
+    ceph::Formatter* f,
+    bool is_periodic,
+    scrub_level_t scrub_level,
+    utime_t stamp = utime_t{})
+{
+  f->open_object_section("result");
+  f->dump_bool("deep", (scrub_level == scrub_level_t::deep));
+  f->dump_bool("must", !is_periodic);
+  f->dump_stream("stamp") << stamp;
+  f->close_section();
+}
+}  // namespace
+
+// when asked to force a "periodic" scrub by faking the timestamps
+void PgScrubber::on_operator_periodic_cmd(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    int64_t offset)
+{
+  const auto cnf = populate_config_params();
+  dout(10) << fmt::format(
+		  "{}: {} (cmd offset:{}) conf:{}", __func__,
+		  (scrub_level == scrub_level_t::deep ? "deep" : "shallow"), offset,
+		  cnf)
+	   << dendl;
+
+  // move the relevant time-stamp backwards - enough to trigger a scrub
+
+  utime_t now_is = ceph_clock_now();
+  utime_t stamp = now_is;
+
+  if (offset > 0) {
+    stamp -= offset;
+  } else {
+    double max_iv =
+	(scrub_level == scrub_level_t::deep)
+	    ? 2 * cnf.max_deep
+	    : (cnf.max_shallow ? *cnf.max_shallow : cnf.shallow_interval);
+    dout(20) << fmt::format(
+		    "{}: stamp:{:s} ms:{}/{}/{}", __func__, stamp,
+		    (cnf.max_shallow ? "ms+" : "ms-"),
+		    (cnf.max_shallow ? *cnf.max_shallow : -999.99),
+		    cnf.shallow_interval)
+	     << dendl;
+    stamp -= max_iv;
+  }
+  stamp -= 100.0;  // for good measure
+
+  dout(10) << fmt::format("{}: stamp:{:s} ", __func__, stamp) << dendl;
+  asok_response_section(f, true, scrub_level, stamp);
+
+  if (scrub_level == scrub_level_t::deep) {
+    m_pg->set_last_deep_scrub_stamp(stamp);
+  }
+  // and in both cases:
+  m_pg->set_last_scrub_stamp(stamp);
+}
+
+// when asked to force a high-priority scrub
+void PgScrubber::on_operator_forced_scrub(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    requested_scrub_t& request_flags)
+{
+  auto deep_req = scrub_requested(scrub_level, scrub_type_t::not_repair, request_flags);
+  asok_response_section(f, false, deep_req);
+}
+
 
 // ----------------------------------------------------------------------------
 
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 996b26781be1..9946f9ce5ee9 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -291,6 +291,16 @@ class PgScrubber : public ScrubPgIF,
 
   pg_scrubbing_status_t get_schedule() const final;
 
+  void on_operator_periodic_cmd(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    int64_t offset) final;
+
+  void on_operator_forced_scrub(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    requested_scrub_t& request_flags) final;
+
   void dump_scrubber(ceph::Formatter* f,
 		     const requested_scrub_t& request_flags) const final;
 
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 52890a8d1f04..16810bba15c6 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -316,6 +316,21 @@ struct ScrubPgIF {
 
   virtual pg_scrubbing_status_t get_schedule() const = 0;
 
+
+  // // perform 'scrub'/'deep_scrub' asok commands
+
+  /// ... by faking the "last scrub" stamps
+  virtual void on_operator_periodic_cmd(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    int64_t offset) = 0;
+
+  /// ... by requesting an "operator initiated" scrub
+  virtual void on_operator_forced_scrub(
+    ceph::Formatter* f,
+    scrub_level_t scrub_level,
+    requested_scrub_t& request_flags) = 0;
+
   virtual void dump_scrubber(ceph::Formatter* f,
 			     const requested_scrub_t& request_flags) const = 0;
 

From 77e45e5395c87b2f87ae7680e20d81de6c8cb9d2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 18:39:07 -0400
Subject: [PATCH 0546/2492] cephadm: move volume mounts assignment to a
 variable

Move the call to get_container_mounts out of the function call block.
This will aid with the next refactoring steps, so that the uses
of get_container_mounts can be brought into the get_container call
directly.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 9a1989a3c956..79e718afefda 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2872,13 +2872,14 @@ def get_container(
         d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
+    mounts = get_container_mounts(ctx, ident)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,
         entrypoint=entrypoint,
         args=ceph_args + d_args,
         container_args=container_args,
-        volume_mounts=get_container_mounts(ctx, ident),
+        volume_mounts=mounts,
         bind_mounts=binds,
         envs=envs,
         privileged=privileged,

From 895d5d92e56264fe3b436288fb337e26cf47411c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 14:39:28 -0400
Subject: [PATCH 0547/2492] cephadm: only call get_container_mounts for types
 that have mounts

This helps clarify what types will eventually have to be converted
away from calling get_container_mounts directly and to using common
methods.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 79e718afefda..d888cd260b6f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2763,10 +2763,12 @@ def get_container(
     envs: List[str] = []
     host_network: bool = True
     binds: List[List[str]] = []
+    mounts: Dict[str, str] = {}
 
     daemon_type = ident.daemon_type
     if daemon_type in ceph_daemons():
         envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+        mounts = get_container_mounts(ctx, ident)
     if container_args is None:
         container_args = []
     _update_pids_limit(ctx, daemon_type, container_args)
@@ -2813,6 +2815,7 @@ def get_container(
             container_args.extend(['--security-opt', 'label=disable'])
         monitoring = Monitoring.create(ctx, ident)
         d_args.extend(monitoring.get_daemon_args())
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type in Tracing.components:
         entrypoint = ''
         name = ident.daemon_name
@@ -2828,6 +2831,7 @@ def get_container(
         envs.extend(NFSGanesha.get_container_envs())
         nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
         d_args.extend(nfs_ganesha.get_daemon_args())
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephExporter.daemon_type:
         entrypoint = CephExporter.entrypoint
         name = 'client.ceph-exporter.%s' % ident.daemon_id
@@ -2839,16 +2843,19 @@ def get_container(
         container_args.extend(['--user=root'])  # haproxy 2.4 defaults to a different user
         haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
         d_args.extend(haproxy.get_daemon_args())
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == Keepalived.daemon_type:
         name = ident.daemon_name
         envs.extend(Keepalived.get_container_envs())
         container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephNvmeof.daemon_type:
         name = ident.daemon_name
         container_args.extend(['--ulimit', 'memlock=-1:-1'])
         container_args.extend(['--ulimit', 'nofile=10240'])
         container_args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
         binds = get_container_binds(ctx, ident)
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephIscsi.daemon_type:
         entrypoint = CephIscsi.entrypoint
         name = ident.daemon_name
@@ -2856,6 +2863,7 @@ def get_container(
         # to configfs we need to make this a privileged container.
         privileged = True
         binds = get_container_binds(ctx, ident)
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
         entrypoint = cc.entrypoint or ''
@@ -2864,6 +2872,7 @@ def get_container(
         container_args.extend(cc.get_container_args())
         d_args.extend(cc.get_daemon_args())
         binds = get_container_binds(ctx, ident)
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
         container_args.append(
@@ -2872,7 +2881,6 @@ def get_container(
         d_args.extend(sg.get_daemon_args())
 
     _update_container_args_for_podman(ctx, ident, container_args)
-    mounts = get_container_mounts(ctx, ident)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,

From abddb3b0582331df93a7b87133485e7763505635 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:05:35 -0400
Subject: [PATCH 0548/2492] cephadm: add customize_container_mounts method to
 custom container

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 13 +++++++++----
 src/cephadm/tests/test_custom_container.py |  5 ++++-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index d888cd260b6f..95cf15ea5659 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1832,7 +1832,7 @@ def get_container_args(self) -> List[str]:
     def get_container_envs(self) -> List[str]:
         return self.envs
 
-    def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
         """
         Get the volume mounts. Relative source paths will be located below
         `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
@@ -1854,6 +1854,12 @@ def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
             mounts[source] = destination
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
     def _get_container_binds(self, data_dir: str) -> List[List[str]]:
         """
         Get the bind mounts. Relative `source=...` paths will be located below
@@ -2691,9 +2697,8 @@ def get_container_mounts(
         mounts.update(Keepalived.get_container_mounts(data_dir))
 
     if daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.init(ctx, fsid, ident.daemon_id)
-        data_dir = ident.data_dir(ctx.data_dir)
-        mounts.update(cc.get_container_mounts(data_dir))
+        cc = CustomContainer.create(ctx, ident)
+        cc.customize_container_mounts(ctx, mounts)
 
     _update_podman_mounts(ctx, mounts)
     return mounts
diff --git a/src/cephadm/tests/test_custom_container.py b/src/cephadm/tests/test_custom_container.py
index 0c020732cc7d..2bcfa0aae6c4 100644
--- a/src/cephadm/tests/test_custom_container.py
+++ b/src/cephadm/tests/test_custom_container.py
@@ -72,7 +72,10 @@ def test_get_container_envs(self):
         self.assertEqual(result, ['SECRET=password'])
 
     def test_get_container_mounts(self):
-        result = self.cc.get_container_mounts('/xyz')
+        # TODO: get_container_mounts was made private. test the private func for
+        # now. in the future update to test base class func
+        # customize_container_mounts
+        result = self.cc._get_container_mounts('/xyz')
         self.assertDictEqual(result, {
             '/CONFIG_DIR': '/foo/conf',
             '/xyz/bar/config': '/bar:ro'

From fe0ed1c541bf47f3fd1bcd2eb69e1ae2e2e19afb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 14:57:50 -0400
Subject: [PATCH 0549/2492] cephadm: add customize_container_mounts method to
 iscsi class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 95cf15ea5659..88f26e2abff2 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1075,7 +1075,7 @@ def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
     @staticmethod
-    def get_container_mounts(data_dir, log_dir):
+    def _get_container_mounts(data_dir, log_dir):
         # type: (str, str) -> Dict[str, str]
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
@@ -1087,6 +1087,17 @@ def get_container_mounts(data_dir, log_dir):
         mounts['/dev'] = '/dev'
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        # Removes ending ".tcmu" from data_dir a tcmu-runner uses the same
+        # data_dir as rbd-runner-api
+        if data_dir.endswith('.tcmu'):
+            data_dir = re.sub(r'\.tcmu$', '', data_dir)
+        log_dir = get_log_dir(self.identity.fsid, ctx.log_dir)
+        mounts.update(CephIscsi._get_container_mounts(data_dir, log_dir))
+
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]
     ) -> None:
@@ -1239,7 +1250,6 @@ def config_and_keyring(
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return extract_uid_gid(ctx)
 
-
 ##################################
 
 
@@ -2684,13 +2694,8 @@ def get_container_mounts(
         mounts.update(CephNvmeof.get_container_mounts(data_dir))
 
     if daemon_type == CephIscsi.daemon_type:
-        data_dir = ident.data_dir(ctx.data_dir)
-        # Removes ending ".tcmu" from data_dir a tcmu-runner uses the same data_dir
-        # as rbd-runner-api
-        if data_dir.endswith('.tcmu'):
-            data_dir = re.sub(r'\.tcmu$', '', data_dir)
-        log_dir = get_log_dir(fsid, ctx.log_dir)
-        mounts.update(CephIscsi.get_container_mounts(data_dir, log_dir))
+        iscsi = CephIscsi.create(ctx, ident)
+        iscsi.customize_container_mounts(ctx, mounts)
 
     if daemon_type == Keepalived.daemon_type:
         data_dir = ident.data_dir(ctx.data_dir)

From 7ea698c6ca819711330466ffccfe1f51f929b7d2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 15:00:25 -0400
Subject: [PATCH 0550/2492] cephadm: add customize_container_mounts method to
 nvmeof class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 88f26e2abff2..fe67904d800f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1298,7 +1298,7 @@ def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
     @staticmethod
-    def get_container_mounts(data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
         mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
@@ -1308,6 +1308,12 @@ def get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]
     ) -> None:
@@ -2690,8 +2696,8 @@ def get_container_mounts(
         mounts.update(HAproxy.get_container_mounts(data_dir))
 
     if daemon_type == CephNvmeof.daemon_type:
-        data_dir = ident.data_dir(ctx.data_dir)
-        mounts.update(CephNvmeof.get_container_mounts(data_dir))
+        nvmeof = CephNvmeof.create(ctx, ident)
+        nvmeof.customize_container_mounts(ctx, mounts)
 
     if daemon_type == CephIscsi.daemon_type:
         iscsi = CephIscsi.create(ctx, ident)

From f76822a15ec813e6268da36c0ae40e7b259eec9f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 15:03:11 -0400
Subject: [PATCH 0551/2492] cephadm: add customize_container_mounts method to
 keepalived class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 12 +++++++++---
 src/cephadm/tests/test_ingress.py |  2 +-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fe67904d800f..83698a7f0be3 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1697,11 +1697,17 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return extract_uid_gid(self.ctx, file_path='/var/lib')
 
     @staticmethod
-    def get_container_mounts(data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
     def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = get_container(ctx, self.identity)
         return to_deployment_container(ctx, ctr)
@@ -2704,8 +2710,8 @@ def get_container_mounts(
         iscsi.customize_container_mounts(ctx, mounts)
 
     if daemon_type == Keepalived.daemon_type:
-        data_dir = ident.data_dir(ctx.data_dir)
-        mounts.update(Keepalived.get_container_mounts(data_dir))
+        keepalive = Keepalived.create(ctx, ident)
+        keepalive.customize_container_mounts(ctx, mounts)
 
     if daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.create(ctx, ident)
diff --git a/src/cephadm/tests/test_ingress.py b/src/cephadm/tests/test_ingress.py
index 698305aa4f38..d2bb7deb6eeb 100644
--- a/src/cephadm/tests/test_ingress.py
+++ b/src/cephadm/tests/test_ingress.py
@@ -244,7 +244,7 @@ def test_keepalived_container_mounts():
             good_keepalived_json(),
             SAMPLE_KEEPALIVED_IMAGE,
         )
-        cmounts = kad.get_container_mounts("/var/tmp")
+        cmounts = kad._get_container_mounts("/var/tmp")
         assert len(cmounts) == 1
         assert (
             cmounts["/var/tmp/keepalived.conf"]

From 7817a6f4292cf283a119d7f5de57832594482e59 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:18:24 -0400
Subject: [PATCH 0552/2492] cephadm: add customize_container_mounts method to
 haproxy class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 12 +++++++++---
 src/cephadm/tests/test_ingress.py |  2 +-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 83698a7f0be3..0ad3bb8b0192 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1576,11 +1576,17 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return extract_uid_gid(self.ctx, file_path='/var/lib')
 
     @staticmethod
-    def get_container_mounts(data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
     @staticmethod
     def get_sysctl_settings() -> List[str]:
         return [
@@ -2698,8 +2704,8 @@ def get_container_mounts(
         mounts.update(nfs_ganesha.get_container_mounts(data_dir))
 
     if daemon_type == HAproxy.daemon_type:
-        data_dir = ident.data_dir(ctx.data_dir)
-        mounts.update(HAproxy.get_container_mounts(data_dir))
+        haproxy = HAproxy.create(ctx, ident)
+        haproxy.customize_container_mounts(ctx, mounts)
 
     if daemon_type == CephNvmeof.daemon_type:
         nvmeof = CephNvmeof.create(ctx, ident)
diff --git a/src/cephadm/tests/test_ingress.py b/src/cephadm/tests/test_ingress.py
index d2bb7deb6eeb..7f23f64f51fc 100644
--- a/src/cephadm/tests/test_ingress.py
+++ b/src/cephadm/tests/test_ingress.py
@@ -90,7 +90,7 @@ def test_haproxy_container_mounts():
             good_haproxy_json(),
             SAMPLE_HAPROXY_IMAGE,
         )
-        cmounts = hap.get_container_mounts("/var/tmp")
+        cmounts = hap._get_container_mounts("/var/tmp")
         assert len(cmounts) == 1
         assert cmounts["/var/tmp/haproxy"] == "/var/lib/haproxy"
 

From 8acdacd789790892b8e958e3d293b3133161ceb6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:24:48 -0400
Subject: [PATCH 0553/2492] cephadm: add customize_container_mounts method to
 nfs class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py        | 17 +++++++++++------
 src/cephadm/tests/test_nfs.py |  4 ++--
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 0ad3bb8b0192..48c50e779b86 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -910,7 +910,7 @@ def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'NFSGanesha':
     def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
-    def get_container_mounts(self, data_dir):
+    def _get_container_mounts(self, data_dir):
         # type: (str) -> Dict[str, str]
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
@@ -923,6 +923,12 @@ def get_container_mounts(self, data_dir):
                 '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
     @staticmethod
     def get_container_envs():
         # type: () -> List[str]
@@ -2684,8 +2690,8 @@ def get_container_mounts(
     paths given a daemon identity.
     Setting `no_config` will skip mapping a daemon specific ceph.conf file.
     """
-    # unpack fsid and daemon_type from ident because they're used very frequently
-    fsid, daemon_type = ident.fsid, ident.daemon_type
+    # unpack daemon_type from ident because they're used very frequently
+    daemon_type = ident.daemon_type
     mounts: Dict[str, str] = {}
 
     assert ident.fsid
@@ -2699,9 +2705,8 @@ def get_container_mounts(
         mounts.update(monitoring.get_container_mounts(data_dir))
 
     if daemon_type == NFSGanesha.daemon_type:
-        data_dir = ident.data_dir(ctx.data_dir)
-        nfs_ganesha = NFSGanesha.init(ctx, fsid, ident.daemon_id)
-        mounts.update(nfs_ganesha.get_container_mounts(data_dir))
+        nfs_ganesha = NFSGanesha.create(ctx, ident)
+        nfs_ganesha.customize_container_mounts(ctx, mounts)
 
     if daemon_type == HAproxy.daemon_type:
         haproxy = HAproxy.create(ctx, ident)
diff --git a/src/cephadm/tests/test_nfs.py b/src/cephadm/tests/test_nfs.py
index 0649ef934c16..94ab6afcfdf9 100644
--- a/src/cephadm/tests/test_nfs.py
+++ b/src/cephadm/tests/test_nfs.py
@@ -117,7 +117,7 @@ def test_nfsganesha_container_mounts():
             "fred",
             good_nfs_json(),
         )
-        cmounts = nfsg.get_container_mounts("/var/tmp")
+        cmounts = nfsg._get_container_mounts("/var/tmp")
         assert len(cmounts) == 3
         assert cmounts["/var/tmp/config"] == "/etc/ceph/ceph.conf:z"
         assert cmounts["/var/tmp/keyring"] == "/etc/ceph/keyring:z"
@@ -130,7 +130,7 @@ def test_nfsganesha_container_mounts():
             "fred",
             nfs_json(pool=True, files=True, rgw=True),
         )
-        cmounts = nfsg.get_container_mounts("/var/tmp")
+        cmounts = nfsg._get_container_mounts("/var/tmp")
         assert len(cmounts) == 4
         assert cmounts["/var/tmp/config"] == "/etc/ceph/ceph.conf:z"
         assert cmounts["/var/tmp/keyring"] == "/etc/ceph/keyring:z"

From 44c8e55b5cb3b00bbb70ef780ba08ff97aee9076 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:33:07 -0400
Subject: [PATCH 0554/2492] cephadm: add customize_container_mounts method to
 monitoring class

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 48c50e779b86..7bfbbeedb0a4 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -810,7 +810,7 @@ def get_daemon_args(self) -> List[str]:
                   '--path.rootfs=/rootfs']
         return r
 
-    def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
         ctx = self.ctx
         daemon_type = self.identity.daemon_type
         mounts: Dict[str, str] = {}
@@ -853,6 +853,12 @@ def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
             ] = '/etc/alertmanager:Z'
         return mounts
 
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
 ##################################
 
 
@@ -2700,9 +2706,8 @@ def get_container_mounts(
         mounts = Ceph.get_ceph_mounts(ctx, ident, no_config=no_config)
 
     if daemon_type in Monitoring.components:
-        data_dir = ident.data_dir(ctx.data_dir)
         monitoring = Monitoring.create(ctx, ident)
-        mounts.update(monitoring.get_container_mounts(data_dir))
+        monitoring.customize_container_mounts(ctx, mounts)
 
     if daemon_type == NFSGanesha.daemon_type:
         nfs_ganesha = NFSGanesha.create(ctx, ident)

From 3411e10c5ca9b740cd25d02ff0c5c078bf7ce64f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:41:25 -0400
Subject: [PATCH 0555/2492] cephadm: change ceph & exporter to
 customize_container_mounts method

Unlike the other types Ceph and CephExporter share the underlying
method. There was no other use of get_container_mounts on the class
so it could be converted to be customize_container_mounts.
Because there's an extra arg that passes from get_container_mounts
top-level function to Ceph.get_ceph_mounts, that function was not
changed.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7bfbbeedb0a4..2bf31527fbfd 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -341,12 +341,15 @@ def get_ceph_mounts(
             )
         return mounts
 
-    def get_container_mounts(self) -> Dict[str, str]:
-        return self.get_ceph_mounts(
-            self.ctx,
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        cm = self.get_ceph_mounts(
+            ctx,
             self.identity,
             no_config=self.ctx.config and self.user_supplied_config,
         )
+        mounts.update(cm)
 
 ##################################
 
@@ -1496,8 +1499,11 @@ def config_and_keyring(
     ) -> Tuple[Optional[str], Optional[str]]:
         return get_config_and_keyring(ctx)
 
-    def get_container_mounts(self) -> Dict[str, str]:
-        return Ceph.get_ceph_mounts(self.ctx, self.identity)
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        cm = Ceph.get_ceph_mounts(ctx, self.identity)
+        mounts.update(cm)
 
 
 ##################################

From 77ee6ab8d4428d007519a430c5c1fd17e7027638 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 4 Nov 2023 19:48:08 -0400
Subject: [PATCH 0556/2492] cephadm: use customize_container_mounts in
 get_container_mounts

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 34 ++++++----------------------------
 1 file changed, 6 insertions(+), 28 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 2bf31527fbfd..c9bc3a787579 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2708,36 +2708,14 @@ def get_container_mounts(
 
     assert ident.fsid
     assert ident.daemon_id
+    # Ceph daemon types are special cased here beacause of the no_config
+    # option which JJM thinks is *only* used by cephadm shell
     if daemon_type in ceph_daemons():
         mounts = Ceph.get_ceph_mounts(ctx, ident, no_config=no_config)
-
-    if daemon_type in Monitoring.components:
-        monitoring = Monitoring.create(ctx, ident)
-        monitoring.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == NFSGanesha.daemon_type:
-        nfs_ganesha = NFSGanesha.create(ctx, ident)
-        nfs_ganesha.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == HAproxy.daemon_type:
-        haproxy = HAproxy.create(ctx, ident)
-        haproxy.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == CephNvmeof.daemon_type:
-        nvmeof = CephNvmeof.create(ctx, ident)
-        nvmeof.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == CephIscsi.daemon_type:
-        iscsi = CephIscsi.create(ctx, ident)
-        iscsi.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == Keepalived.daemon_type:
-        keepalive = Keepalived.create(ctx, ident)
-        keepalive.customize_container_mounts(ctx, mounts)
-
-    if daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.create(ctx, ident)
-        cc.customize_container_mounts(ctx, mounts)
+    else:
+        daemon = daemon_form_create(ctx, ident)
+        assert isinstance(daemon, ContainerDaemonForm)
+        daemon.customize_container_mounts(ctx, mounts)
 
     _update_podman_mounts(ctx, mounts)
     return mounts

From c886e05404ce9c50454d40e692ababdc55f346c6 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 12 Nov 2023 20:21:41 +1000
Subject: [PATCH 0557/2492] doc/config: edit "ceph-conf.rst"

Edit the first section of doc/rados/configuration/ceph-conf.rst.
Initially I just wanted to change "series" to "set", but once I got my
hands dirty I ended up simplifying some sentences.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/ceph-conf.rst | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/doc/rados/configuration/ceph-conf.rst b/doc/rados/configuration/ceph-conf.rst
index f62a215458d3..d8d5c9d03431 100644
--- a/doc/rados/configuration/ceph-conf.rst
+++ b/doc/rados/configuration/ceph-conf.rst
@@ -4,7 +4,7 @@
  Configuring Ceph
 ==================
 
-When Ceph services start, the initialization process activates a series of
+When Ceph services start, the initialization process activates a set of
 daemons that run in the background. A :term:`Ceph Storage Cluster` runs at
 least three types of daemons:
 
@@ -12,15 +12,16 @@ least three types of daemons:
 - :term:`Ceph Manager` (``ceph-mgr``)
 - :term:`Ceph OSD Daemon` (``ceph-osd``)
 
-Ceph Storage Clusters that support the :term:`Ceph File System` also run at
-least one :term:`Ceph Metadata Server` (``ceph-mds``). Clusters that support
-:term:`Ceph Object Storage` run Ceph RADOS Gateway daemons (``radosgw``).
+Any Ceph Storage Cluster that supports the :term:`Ceph File System` also runs
+at least one :term:`Ceph Metadata Server` (``ceph-mds``). Any Cluster that
+supports :term:`Ceph Object Storage` runs Ceph RADOS Gateway daemons
+(``radosgw``).
 
-Each daemon has a number of configuration options, each of which has a default
-value. You may adjust the behavior of the system by changing these
-configuration options. Be careful to understand the consequences before
-overriding default values, as it is possible to significantly degrade the
-performance and stability of your cluster. Note too that default values
+Each daemon has a number of configuration options, and each of those options
+has a default value. Adjust the behavior of the system by changing these
+configuration options. Make sure to understand the consequences before
+overriding the default values, as it is possible to significantly degrade the
+performance and stability of your cluster. Remember that default values
 sometimes change between releases. For this reason, it is best to review the
 version of this documentation that applies to your Ceph release.
 

From 010e5a13b9e4e6a860c676f1b3090a495592d715 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 12 Nov 2023 20:52:09 +1000
Subject: [PATCH 0558/2492] doc/rados: parallelize t-mon headings

Give parallel structure to the questions in the Q&A section of the "The
Cluster Has Quorum But At Least One Monitor Is Down" subsection of the
"Most Common Monitor Issues" section of
doc/rados/troubleshooting/troubleshooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 6add6127e16f..49c0ac1a980f 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -217,7 +217,7 @@ detail`` returns a message similar to the following::
   the documentation.
   
 
-**What does it mean if  a Monitor's state is ``probing``?**
+**What does it mean when a Monitor's state is ``probing``?**
 
   If ``ceph health detail`` shows that a Monitor's state is
   ``probing``, then the Monitor is still looking for the other Monitors. Every
@@ -294,7 +294,7 @@ detail`` returns a message similar to the following::
   bug you raise. See `Preparing your logs`_ for information about the proper
   preparation of logs.
 
-**What does it mean when the Monitor state is ``leader`` or ``peon``?**
+**What does it mean when a Monitor's state is ``leader`` or ``peon``?**
 
   If ``ceph health detail`` shows that the Monitor is in the ``leader`` state
   or in the ``peon`` state, it is likely that clock skew is present. Follow the

From cdee9b928ff9a410e6e0f38aa791756829d68358 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 9 Nov 2023 09:39:23 +0000
Subject: [PATCH 0559/2492] crimson/osd/osd_operations/snaptrim_event: fix
 lifetime management

The operation's id and future returned when starting SnapTrimObjSubEvent
is emplaced into subop_blocker.
Later on, we await the completion of all the started operations futures.
Before this patch, we only stored the op id in the subop_blocker vector
which allowed `op` to go out of scope and lose all its references
(and get deleted) before exiting.

Storing the operation as a reference instead of the id
will maintain the SnapTrimObjSubEvent operation lifetime.

Fixes: https://tracker.ceph.com/issues/63299

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 4 ++--
 src/crimson/osd/osd_operations/snaptrim_event.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index ffd43d736ad3..21fd6e94da94 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -44,7 +44,7 @@ void SnapTrimEvent::SubOpBlocker::dump_detail(Formatter *f) const
   f->open_array_section("dependent_operations");
   {
     for (const auto &kv : subops) {
-      f->dump_unsigned("op_id", kv.first);
+      f->dump_unsigned("op_id", kv.first->get_id());
     }
   }
   f->close_section();
@@ -152,7 +152,7 @@ SnapTrimEvent::start()
 	      object,
 	      snapid);
 	    subop_blocker.emplace_back(
-	      op->get_id(),
+	      std::move(op),
 	      std::move(fut)
 	    );
 	  }
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index afb24952a045..f7c512881de3 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -61,7 +61,7 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
   struct SubOpBlocker : crimson::BlockerT<SubOpBlocker> {
     static constexpr const char* type_name = "CompoundOpBlocker";
 
-    using id_done_t = std::pair<crimson::Operation::id_t,
+    using id_done_t = std::pair<crimson::OperationRef,
                                 remove_or_update_iertr::future<>>;
 
     void dump_detail(Formatter *f) const final;

From 994ae0d633843e61eae91f409ec3b0336db71dc3 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Mon, 13 Nov 2023 12:56:50 +0530
Subject: [PATCH 0560/2492] qa: use correct imports to resolve fuse_mount and
 kernel_mount

Fixes: https://tracker.ceph.com/issues/62706
Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 qa/tasks/cephfs/test_snap_schedules.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/test_snap_schedules.py b/qa/tasks/cephfs/test_snap_schedules.py
index d82404982dd8..8f9132566da5 100644
--- a/qa/tasks/cephfs/test_snap_schedules.py
+++ b/qa/tasks/cephfs/test_snap_schedules.py
@@ -588,8 +588,8 @@ def remove_snapshots(self, dir_path, sdn):
             self.mount_a.run_shell(['rmdir', snapshot_path])
 
     def get_snap_dir_name(self):
-        from tasks.cephfs.fuse_mount import FuseMount
-        from tasks.cephfs.kernel_mount import KernelMount
+        from .fuse_mount import FuseMount
+        from .kernel_mount import KernelMount
 
         if isinstance(self.mount_a, KernelMount):
             sdn = self.mount_a.client_config.get('snapdirname', '.snap')

From c581018caf626fa0dd50bd244766bfa9755c9a16 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 13 Nov 2023 20:57:07 +1000
Subject: [PATCH 0561/2492] doc/rados: format "initial troubleshooting"

Format the steps in the "Initial Troubleshooting" section of
doc/rados/troubleshooting/troubleshooting-mon.rst. A near-future PR (not
this one) will add context to this section and explain that the steps
described here are the first steps that you should undertake when you
determine that you have an unresponsive or down Monitor. This PR is
merely for formatting.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 106 +++++++++---------
 1 file changed, 54 insertions(+), 52 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 49c0ac1a980f..ad745a3a73de 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -17,58 +17,60 @@ consult the following troubleshooting information.
 Initial Troubleshooting
 =======================
 
-**Are the monitors running?**
-
-  First, make sure that the monitor (*mon*) daemon processes (``ceph-mon``) are
-  running. Sometimes Ceph admins either forget to start the mons or forget to
-  restart the mons after an upgrade. Checking for this simple oversight can
-  save hours of painstaking troubleshooting. It is also important to make sure
-  that the manager daemons (``ceph-mgr``) are running. Remember that typical
-  cluster configurations provide one ``ceph-mgr`` for each ``ceph-mon``.
-
-  .. note:: Rook will not run more than two managers.
-
-**Can you reach the monitor nodes?**
-
-  In certain rare cases, there may be ``iptables`` rules that block access to
-  monitor nodes or TCP ports. These rules might be left over from earlier
-  stress testing or rule development. To check for the presence of such rules,
-  SSH into the server and then try to connect to the monitor's ports
-  (``tcp/3300`` and ``tcp/6789``) using ``telnet``, ``nc``, or a similar tool.
-
-**Does the ``ceph status`` command run and receive a reply from the cluster?**
-
-  If the ``ceph status`` command does receive a reply from the cluster, then the
-  cluster is up and running. The monitors will answer to a ``status`` request
-  only if there is a formed quorum. Confirm that one or more ``mgr`` daemons
-  are reported as running. Under ideal conditions, all ``mgr`` daemons will be
-  reported as running.
-
-
-  If the ``ceph status`` command does not receive a reply from the cluster, then
-  there are probably not enough monitors ``up`` to form a quorum.  The ``ceph
-  -s`` command with no further options specified connects to an arbitrarily
-  selected monitor. In certain cases, however, it might be helpful to connect
-  to a specific monitor (or to several specific monitors in sequence) by adding
-  the ``-m`` flag to the command: for example, ``ceph status -m mymon1``.
-
-
-**None of this worked. What now?**
-
-  If the above solutions have not resolved your problems, you might find it
-  helpful to examine each individual monitor in turn. Whether or not a quorum
-  has been formed, it is possible to contact each monitor individually and
-  request its status by using the ``ceph tell mon.ID mon_status`` command (here
-  ``ID`` is the monitor's identifier).
-
-  Run the ``ceph tell mon.ID mon_status`` command for each monitor in the
-  cluster. For more on this command's output, see :ref:`Understanding
-  mon_status
-  <rados_troubleshoting_troubleshooting_mon_understanding_mon_status>`.
-
-  There is also an alternative method: SSH into each monitor node and query the
-  daemon's admin socket. See :ref:`Using the Monitor's Admin
-  Socket<rados_troubleshoting_troubleshooting_mon_using_admin_socket>`.
+#. **Make sure that the monitors are running.**
+
+    First, make sure that the monitor (*mon*) daemon processes (``ceph-mon``)
+    are running. Sometimes Ceph admins either forget to start the mons or
+    forget to restart the mons after an upgrade. Checking for this simple
+    oversight can save hours of painstaking troubleshooting. It is also
+    important to make sure that the manager daemons (``ceph-mgr``) are running.
+    Remember that typical cluster configurations provide one ``ceph-mgr`` for
+    each ``ceph-mon``.
+
+    .. note:: Rook will not run more than two managers.
+
+#. **Make sure that you can reach the monitor nodes.**
+
+    In certain rare cases, there may be ``iptables`` rules that block access to
+    monitor nodes or TCP ports. These rules might be left over from earlier
+    stress testing or rule development. To check for the presence of such
+    rules, SSH into the server and then try to connect to the monitor's ports
+    (``tcp/3300`` and ``tcp/6789``) using ``telnet``, ``nc``, or a similar
+    tool.
+
+#. **Make sure that the ``ceph status`` command runs  and receives a reply from the cluster.**
+
+    If the ``ceph status`` command does receive a reply from the cluster, then
+    the cluster is up and running. The monitors will answer to a ``status``
+    request only if there is a formed quorum. Confirm that one or more ``mgr``
+    daemons are reported as running. Under ideal conditions, all ``mgr``
+    daemons will be reported as running.
+
+
+    If the ``ceph status`` command does not receive a reply from the cluster,
+    then there are probably not enough monitors ``up`` to form a quorum.  The
+    ``ceph -s`` command with no further options specified connects to an
+    arbitrarily selected monitor. In certain cases, however, it might be
+    helpful to connect to a specific monitor (or to several specific monitors
+    in sequence) by adding the ``-m`` flag to the command: for example, ``ceph
+    status -m mymon1``.
+
+#. **None of this worked. What now?**
+
+    If the above solutions have not resolved your problems, you might find it
+    helpful to examine each individual monitor in turn. Whether or not a quorum
+    has been formed, it is possible to contact each monitor individually and
+    request its status by using the ``ceph tell mon.ID mon_status`` command
+    (here ``ID`` is the monitor's identifier).
+
+    Run the ``ceph tell mon.ID mon_status`` command for each monitor in the
+    cluster. For more on this command's output, see :ref:`Understanding
+    mon_status
+    <rados_troubleshoting_troubleshooting_mon_understanding_mon_status>`.
+
+    There is also an alternative method: SSH into each monitor node and query
+    the daemon's admin socket. See :ref:`Using the Monitor's Admin
+    Socket<rados_troubleshoting_troubleshooting_mon_using_admin_socket>`.
 
 .. _rados_troubleshoting_troubleshooting_mon_using_admin_socket:
 

From 48fd7ff1e13ccfeca04b9283b13641ce59316723 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Mon, 13 Nov 2023 08:58:44 -0500
Subject: [PATCH 0562/2492] doc/start: fix typo in hardware-recommendations.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/start/hardware-recommendations.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index f3f8afdf164a..a63b5a457964 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -106,7 +106,7 @@ that the OSD attempts to consume by changing the :confval:`osd_memory_target`
 configuration option.
 
 - Setting the :confval:`osd_memory_target` below 2GB is not
-  recommended.  eph may fail to keep the memory consumption under 2GB and 
+  recommended. Ceph may fail to keep the memory consumption under 2GB and 
   extremely slow performance is likely.
 
 - Setting the memory target between 2GB and 4GB typically works but may result

From 454a2b9285319d561046be0b482029f5130adfc7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 8 Nov 2023 13:07:29 +0000
Subject: [PATCH 0563/2492] crimson/osd/pg: extend pg lifetime on snap_trimq
 iteration

Beacuse the loop's returned future is ignored,
we should cover the scenario where the pg is removed and the
snap_trimq iteration didn't complete yet.

Fixes: https://tracker.ceph.com/issues/61653

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 42052a4c84d9..46daa402f81f 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -471,6 +471,7 @@ void PG::on_active_actmap()
   logger().debug("{}: {} snap_trimq={}", *this, __func__, snap_trimq);
   peering_state.state_clear(PG_STATE_SNAPTRIM_ERROR);
   // loops until snap_trimq is empty or SNAPTRIM_ERROR.
+  Ref<PG> pg_ref = this;
   std::ignore = seastar::do_until(
     [this] { return snap_trimq.empty()
                     || peering_state.state_test(PG_STATE_SNAPTRIM_ERROR);
@@ -505,7 +506,7 @@ void PG::on_active_actmap()
       }).then([this, trimmed=to_trim] {
         logger().debug("{}: trimmed snap={}", *this, trimmed);
       });
-    }).finally([this] {
+    }).finally([this, pg_ref] {
       logger().debug("{}: PG::on_active_actmap() finished trimming",
                      *this);
       peering_state.state_clear(PG_STATE_SNAPTRIM);

From f5300331e504cbde3d3e8d79c0d363c677df33e4 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 12 Nov 2023 13:53:59 +0000
Subject: [PATCH 0564/2492] crimson/osd/pg: PG::on_active_actmap() indention

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 46daa402f81f..78ae559e8cd5 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -506,13 +506,14 @@ void PG::on_active_actmap()
       }).then([this, trimmed=to_trim] {
         logger().debug("{}: trimmed snap={}", *this, trimmed);
       });
-    }).finally([this, pg_ref] {
-      logger().debug("{}: PG::on_active_actmap() finished trimming",
-                     *this);
-      peering_state.state_clear(PG_STATE_SNAPTRIM);
-      peering_state.state_clear(PG_STATE_SNAPTRIM_ERROR);
-      publish_stats_to_osd();
-    });
+    }
+  ).finally([this, pg_ref] {
+    logger().debug("{}: PG::on_active_actmap() finished trimming",
+                   *this);
+    peering_state.state_clear(PG_STATE_SNAPTRIM);
+    peering_state.state_clear(PG_STATE_SNAPTRIM_ERROR);
+    publish_stats_to_osd();
+  });
 }
 
 void PG::on_active_advmap(const OSDMapRef &osdmap)

From 6d8e0a82f6e6bb6a5d7833fa431b750c1b17b644 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 2 Nov 2023 17:44:10 -0400
Subject: [PATCH 0565/2492] pybind/mgr: remove __del__() of mgr modules

It's strongly recommended for objects that have references to
external resources (e.g., files) to explicitly release them.
Python doesn't guarantee garbage collection of objects and hence
doesn't guarantee freeing of external resources that occur on
garbage collection.

The __del__() methods in the python mgr modules may not even be
called since garbage collection of objects is not guaranteed in python.
And some of the __del__() methods try to cleanup that seem redundant.

- In volumes/module.py, vc.shutdown() is called in Module.shutdown().
  No need to call it again in Module.__del__()

- In telegraf/basesocket.py, BaseSocker.close() is called in
  BaseSocket.__exit__(). No need to call it again in
  BaseSocket.__del__().

- In mgr_module.py, MgrModuleLoggingMixin._unconfigure_logging() is
  called in MgrModule.__init__() and MgrStandbyModule.__init__(). No
  need to call it in MgrModule.__del__() and
  MgrStandbyModule.__del__().|

- In dashboard/services/cephfs.py, the libcephfs mount is not
  shutdown explicitly by the mgr module. However, the cython libcephfs
  bindings has a LibCephFS.__dealloc__() finalizer method that calls
  LibCephFS.shutdown(). This should unmount and cleanup the ceph mount
  handle.

Remove the __del__() of the python mgr modules.

Fixes: https://tracker.ceph.com/issues/63421
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 src/pybind/mgr/dashboard/services/cephfs.py |  4 ----
 src/pybind/mgr/mgr_module.py                | 10 ----------
 src/pybind/mgr/telegraf/basesocket.py       |  3 ---
 src/pybind/mgr/volumes/module.py            |  3 ---
 4 files changed, 20 deletions(-)

diff --git a/src/pybind/mgr/dashboard/services/cephfs.py b/src/pybind/mgr/dashboard/services/cephfs.py
index 07b339cc9217..ffbf9d0c8165 100644
--- a/src/pybind/mgr/dashboard/services/cephfs.py
+++ b/src/pybind/mgr/dashboard/services/cephfs.py
@@ -45,10 +45,6 @@ def __init__(self, fs_name=None):
             self.cfs.mount()
         logger.debug("mounted cephfs filesystem")
 
-    def __del__(self):
-        logger.debug("shutting down cephfs filesystem")
-        self.cfs.shutdown()
-
     @contextmanager
     def opendir(self, dirpath):
         d = None
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 5a7b9bfc6f6c..6c83f2771619 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -836,13 +836,6 @@ def __init__(self, module_name: str, capsule: Any):
         # for backwards compatibility
         self._logger = self.getLogger()
 
-    def __del__(self) -> None:
-        self._cleanup()
-        self._unconfigure_logging()
-
-    def _cleanup(self) -> None:
-        pass
-
     @classmethod
     def _register_options(cls, module_name: str) -> None:
         cls.MODULE_OPTIONS.append(
@@ -1045,9 +1038,6 @@ def __init__(self, module_name: str, py_modules_ptr: object, this_ptr: object):
 
         self._db_lock = threading.Lock()
 
-    def __del__(self) -> None:
-        self._unconfigure_logging()
-
     @classmethod
     def _register_options(cls, module_name: str) -> None:
         cls.MODULE_OPTIONS.append(
diff --git a/src/pybind/mgr/telegraf/basesocket.py b/src/pybind/mgr/telegraf/basesocket.py
index 5caea3be7259..463cf326dd05 100644
--- a/src/pybind/mgr/telegraf/basesocket.py
+++ b/src/pybind/mgr/telegraf/basesocket.py
@@ -38,9 +38,6 @@ def close(self) -> None:
     def send(self, data: str, flags: int = 0) -> int:
         return self.sock.send(data.encode('utf-8') + b'\n', flags)
 
-    def __del__(self) -> None:
-        self.sock.close()
-
     def __enter__(self) -> 'BaseSocket':
         self.connect()
         return self
diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index ff7256eebfd3..6227276fcaf5 100644
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -506,9 +506,6 @@ def __init__(self, *args, **kwargs):
             self.vc = VolumeClient(self)
             self.inited = True
 
-    def __del__(self):
-        self.vc.shutdown()
-
     def shutdown(self):
         self.vc.shutdown()
 

From 28a26f7b0441649deef0692e48d4cbe2d95e2cfe Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Mon, 9 Oct 2023 23:42:32 +0530
Subject: [PATCH 0566/2492] common, osd: Remove unused implementation of
 mClockPriorityQueue

mClockPriorityQueue (mClockQueue class) is an older mClock implementation
of the OpQueue abstraction. This was replaced by a simpler implementation
of the OpScheduler abstraction as part of
https://github.com/ceph/ceph/pull/30650.

The simpler implementation of mClockScheduler is being currently used.
This commit removes the unused src/common/mClockPriorityQueue.h along
with the associated unit test file: test_mclock_priority_queue.cc.

Other miscellaneous changes,
 - Remove the cmake references to the unit test file
 - Remove the inclusion of the header file in mClockScheduler.h

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/common/mClockPriorityQueue.h              | 369 ------------------
 src/osd/scheduler/mClockScheduler.h           |   1 -
 src/test/common/CMakeLists.txt                |  11 -
 src/test/common/test_mclock_priority_queue.cc | 320 ---------------
 4 files changed, 701 deletions(-)
 delete mode 100644 src/common/mClockPriorityQueue.h
 delete mode 100644 src/test/common/test_mclock_priority_queue.cc

diff --git a/src/common/mClockPriorityQueue.h b/src/common/mClockPriorityQueue.h
deleted file mode 100644
index c1f9f3c2517d..000000000000
--- a/src/common/mClockPriorityQueue.h
+++ /dev/null
@@ -1,369 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2016 Red Hat Inc.
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#pragma once
-
-
-#include <functional>
-#include <map>
-#include <list>
-#include <cmath>
-
-#include "common/Formatter.h"
-#include "common/OpQueue.h"
-
-#include "dmclock/src/dmclock_server.h"
-
-// the following is done to unclobber _ASSERT_H so it returns to the
-// way ceph likes it
-#include "include/ceph_assert.h"
-
-
-namespace ceph {
-
-  namespace dmc = crimson::dmclock;
-
-  template <typename T, typename K>
-  class mClockQueue : public OpQueue <T, K> {
-
-    using priority_t = unsigned;
-    using cost_t = unsigned;
-
-    typedef std::list<std::pair<cost_t, T> > ListPairs;
-
-    static void filter_list_pairs(ListPairs *l,
-				  std::function<bool (T&&)> f) {
-      for (typename ListPairs::iterator i = l->end();
-	   i != l->begin();
-	   /* no inc */
-	) {
-	auto next = i;
-	--next;
-	if (f(std::move(next->second))) {
-	  l->erase(next);
-	} else {
-	  i = next;
-	}
-      }
-    }
-
-    struct SubQueue {
-    private:
-      typedef std::map<K, ListPairs> Classes;
-      // client-class to ordered queue
-      Classes q;
-
-      unsigned tokens, max_tokens;
-
-      typename Classes::iterator cur;
-
-    public:
-
-      SubQueue(const SubQueue &other)
-	: q(other.q),
-	  tokens(other.tokens),
-	  max_tokens(other.max_tokens),
-	  cur(q.begin()) {}
-
-      SubQueue()
-	: tokens(0),
-	  max_tokens(0),
-	  cur(q.begin()) {}
-
-      void set_max_tokens(unsigned mt) {
-	max_tokens = mt;
-      }
-
-      unsigned get_max_tokens() const {
-	return max_tokens;
-      }
-
-      unsigned num_tokens() const {
-	return tokens;
-      }
-
-      void put_tokens(unsigned t) {
-	tokens += t;
-	if (tokens > max_tokens) {
-	  tokens = max_tokens;
-	}
-      }
-
-      void take_tokens(unsigned t) {
-	if (tokens > t) {
-	  tokens -= t;
-	} else {
-	  tokens = 0;
-	}
-      }
-
-      void enqueue(K cl, cost_t cost, T&& item) {
-	q[cl].emplace_back(cost, std::move(item));
-	if (cur == q.end())
-	  cur = q.begin();
-      }
-
-      void enqueue_front(K cl, cost_t cost, T&& item) {
-	q[cl].emplace_front(cost, std::move(item));
-	if (cur == q.end())
-	  cur = q.begin();
-      }
-
-      const std::pair<cost_t, T>& front() const {
-	ceph_assert(!(q.empty()));
-	ceph_assert(cur != q.end());
-	return cur->second.front();
-      }
-
-      std::pair<cost_t, T>& front() {
-	ceph_assert(!(q.empty()));
-	ceph_assert(cur != q.end());
-	return cur->second.front();
-      }
-
-      void pop_front() {
-	ceph_assert(!(q.empty()));
-	ceph_assert(cur != q.end());
-	cur->second.pop_front();
-	if (cur->second.empty()) {
-	  auto i = cur;
-	  ++cur;
-	  q.erase(i);
-	} else {
-	  ++cur;
-	}
-	if (cur == q.end()) {
-	  cur = q.begin();
-	}
-      }
-
-      unsigned get_size_slow() const {
-	unsigned count = 0;
-	for (const auto& cls : q) {
-	  count += cls.second.size();
-	}
-	return count;
-      }
-
-      bool empty() const {
-	return q.empty();
-      }
-
-      void remove_by_filter(std::function<bool (T&&)> f) {
-	for (typename Classes::iterator i = q.begin();
-	     i != q.end();
-	     /* no-inc */) {
-	  filter_list_pairs(&(i->second), f);
-	  if (i->second.empty()) {
-	    if (cur == i) {
-	      ++cur;
-	    }
-	    i = q.erase(i);
-	  } else {
-	    ++i;
-	  }
-	}
-	if (cur == q.end()) cur = q.begin();
-      }
-
-      void remove_by_class(K k, std::list<T> *out) {
-	typename Classes::iterator i = q.find(k);
-	if (i == q.end()) {
-	  return;
-	}
-	if (i == cur) {
-	  ++cur;
-	}
-	if (out) {
-	  for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) {
-	    out->push_front(std::move(j->second));
-	  }
-	}
-	q.erase(i);
-	if (cur == q.end()) cur = q.begin();
-      }
-
-      void dump(ceph::Formatter *f) const {
-	f->dump_int("size", get_size_slow());
-	f->dump_int("num_keys", q.size());
-      }
-    };
-
-    using SubQueues = std::map<priority_t, SubQueue>;
-
-    SubQueues high_queue;
-
-    using Queue = dmc::PullPriorityQueue<K,T,false>;
-    Queue queue;
-
-    // when enqueue_front is called, rather than try to re-calc tags
-    // to put in mClock priority queue, we'll just keep a separate
-    // list from which we dequeue items first, and only when it's
-    // empty do we use queue.
-    std::list<std::pair<K,T>> queue_front;
-
-  public:
-
-    mClockQueue(
-      const typename Queue::ClientInfoFunc& info_func,
-      double anticipation_timeout = 0.0) :
-      queue(info_func, dmc::AtLimit::Allow, anticipation_timeout)
-    {
-      // empty
-    }
-
-    unsigned get_size_slow() const {
-      unsigned total = 0;
-      total += queue_front.size();
-      total += queue.request_count();
-      for (auto i = high_queue.cbegin(); i != high_queue.cend(); ++i) {
-	ceph_assert(i->second.get_size_slow());
-	total += i->second.get_size_slow();
-      }
-      return total;
-    }
-
-    // be sure to do things in reverse priority order and push_front
-    // to the list so items end up on list in front-to-back priority
-    // order
-    void remove_by_filter(std::function<bool (T&&)> filter_accum) {
-      queue.remove_by_req_filter([&] (std::unique_ptr<T>&& r) {
-          return filter_accum(std::move(*r));
-        }, true);
-
-      for (auto i = queue_front.rbegin(); i != queue_front.rend(); /* no-inc */) {
-	if (filter_accum(std::move(i->second))) {
-	  i = decltype(i){ queue_front.erase(std::next(i).base()) };
-	} else {
-	  ++i;
-	}
-      }
-
-      for (typename SubQueues::iterator i = high_queue.begin();
-	   i != high_queue.end();
-	   /* no-inc */ ) {
-	i->second.remove_by_filter(filter_accum);
-	if (i->second.empty()) {
-	  i = high_queue.erase(i);
-	} else {
-	  ++i;
-	}
-      }
-    }
-
-    void remove_by_class(K k, std::list<T> *out = nullptr) override final {
-      if (out) {
-	queue.remove_by_client(k,
-			       true,
-			       [&out] (std::unique_ptr<T>&& t) {
-				 out->push_front(std::move(*t));
-			       });
-      } else {
-	queue.remove_by_client(k, true);
-      }
-
-      for (auto i = queue_front.rbegin(); i != queue_front.rend(); /* no-inc */) {
-	if (k == i->first) {
-	  if (nullptr != out) out->push_front(std::move(i->second));
-	  i = decltype(i){ queue_front.erase(std::next(i).base()) };
-	} else {
-	  ++i;
-	}
-      }
-
-      for (auto i = high_queue.begin(); i != high_queue.end(); /* no-inc */) {
-	i->second.remove_by_class(k, out);
-	if (i->second.empty()) {
-	  i = high_queue.erase(i);
-	} else {
-	  ++i;
-	}
-      }
-    }
-
-    void enqueue_strict(K cl, unsigned priority, T&& item) override final {
-      high_queue[priority].enqueue(cl, 1, std::move(item));
-    }
-
-    void enqueue_strict_front(K cl, unsigned priority, T&& item) override final {
-      high_queue[priority].enqueue_front(cl, 1, std::move(item));
-    }
-
-    void enqueue(K cl, unsigned priority, unsigned cost, T&& item) override final {
-      // priority is ignored
-      queue.add_request(std::move(item), cl, cost);
-    }
-
-    void enqueue_front(K cl,
-		       unsigned priority,
-		       unsigned cost,
-		       T&& item) override final {
-      queue_front.emplace_front(std::pair<K,T>(cl, std::move(item)));
-    }
-
-    bool empty() const override final {
-      return queue.empty() && high_queue.empty() && queue_front.empty();
-    }
-
-    T dequeue() override final {
-      ceph_assert(!empty());
-
-      if (!high_queue.empty()) {
-	T ret = std::move(high_queue.rbegin()->second.front().second);
-	high_queue.rbegin()->second.pop_front();
-	if (high_queue.rbegin()->second.empty()) {
-	  high_queue.erase(high_queue.rbegin()->first);
-	}
-	return ret;
-      }
-
-      if (!queue_front.empty()) {
-	T ret = std::move(queue_front.front().second);
-	queue_front.pop_front();
-	return ret;
-      }
-
-      auto pr = queue.pull_request();
-      ceph_assert(pr.is_retn());
-      auto& retn = pr.get_retn();
-      return std::move(*(retn.request));
-    }
-
-    void dump(ceph::Formatter *f) const override final {
-      f->open_array_section("high_queues");
-      for (typename SubQueues::const_iterator p = high_queue.begin();
-	   p != high_queue.end();
-	   ++p) {
-	f->open_object_section("subqueue");
-	f->dump_int("priority", p->first);
-	p->second.dump(f);
-	f->close_section();
-      }
-      f->close_section();
-
-      f->open_object_section("queue_front");
-      f->dump_int("size", queue_front.size());
-      f->close_section();
-
-      f->open_object_section("queue");
-      f->dump_int("size", queue.request_count());
-      f->close_section();
-    } // dump
-
-    void print(std::ostream &os) const final {
-      os << "mClockPriorityQueue";
-    }
-  };
-
-} // namespace ceph
diff --git a/src/osd/scheduler/mClockScheduler.h b/src/osd/scheduler/mClockScheduler.h
index f708b1d7a1e3..c1aca5da5d14 100644
--- a/src/osd/scheduler/mClockScheduler.h
+++ b/src/osd/scheduler/mClockScheduler.h
@@ -27,7 +27,6 @@
 #include "osd/scheduler/OpScheduler.h"
 #include "common/config.h"
 #include "common/ceph_context.h"
-#include "common/mClockPriorityQueue.h"
 #include "osd/scheduler/OpSchedulerItem.h"
 
 
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index c044daf662ab..450827116dca 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -75,17 +75,6 @@ add_executable(unittest_prioritized_queue
 target_link_libraries(unittest_prioritized_queue ceph-common)
 add_ceph_unittest(unittest_prioritized_queue)
 
-if(NOT WIN32)
-# unittest_mclock_priority_queue
-add_executable(unittest_mclock_priority_queue
-  test_mclock_priority_queue.cc
-  )
-add_ceph_unittest(unittest_mclock_priority_queue)
-target_link_libraries(unittest_mclock_priority_queue
-  ceph-common
-  dmclock::dmclock)
-endif(NOT WIN32)
-
 # unittest_str_map
 add_executable(unittest_str_map
   test_str_map.cc
diff --git a/src/test/common/test_mclock_priority_queue.cc b/src/test/common/test_mclock_priority_queue.cc
deleted file mode 100644
index 8e8bcdf38cf2..000000000000
--- a/src/test/common/test_mclock_priority_queue.cc
+++ /dev/null
@@ -1,320 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2017 Red Hat Inc.
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <thread>
-#include <chrono>
-#include <iostream>
-#include "gtest/gtest.h"
-#include "common/mClockPriorityQueue.h"
-
-
-struct Request {
-  int value;
-  Request() :
-    value(0)
-  {}
-  Request(const Request& o) = default;
-  explicit Request(int value) :
-    value(value)
-  {}
-};
-
-
-struct Client {
-  int client_num;
-  Client() :
-    Client(-1)
-  {}
-  Client(int client_num) :
-    client_num(client_num)
-  {}
-  friend bool operator<(const Client& r1, const Client& r2) {
-    return r1.client_num < r2.client_num;
-  }
-  friend bool operator==(const Client& r1, const Client& r2) {
-    return r1.client_num == r2.client_num;
-  }
-};
-
-
-const crimson::dmclock::ClientInfo* client_info_func(const Client& c) {
-  static const crimson::dmclock::ClientInfo
-    the_info(10.0, 10.0, 10.0);
-  return &the_info;
-}
-
-
-TEST(mClockPriorityQueue, Create)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-}
-
-
-TEST(mClockPriorityQueue, Sizes)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  ASSERT_TRUE(q.empty());
-  ASSERT_EQ(0u, q.get_size_slow());
-
-  Client c1(1);
-  Client c2(2);
-
-  q.enqueue_strict(c1, 1, Request(1));
-  q.enqueue_strict(c2, 2, Request(2));
-  q.enqueue_strict(c1, 2, Request(3));
-  q.enqueue(c2, 1, 1u, Request(4));
-  q.enqueue(c1, 2, 1u, Request(5));
-  q.enqueue_strict(c2, 1, Request(6));
-
-  ASSERT_FALSE(q.empty());
-  ASSERT_EQ(6u, q.get_size_slow());
-
-
-  for (int i = 0; i < 6; ++i) {
-    (void) q.dequeue();
-  }
-
-  ASSERT_TRUE(q.empty());
-  ASSERT_EQ(0u, q.get_size_slow());
-}
-
-
-TEST(mClockPriorityQueue, JustStrict)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-
-  q.enqueue_strict(c1, 1, Request(1));
-  q.enqueue_strict(c2, 2, Request(2));
-  q.enqueue_strict(c1, 2, Request(3));
-  q.enqueue_strict(c2, 1, Request(4));
-
-  Request r;
-
-  r = q.dequeue();
-  ASSERT_EQ(2, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(3, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(1, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(4, r.value);
-}
-
-
-TEST(mClockPriorityQueue, StrictPriorities)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-
-  q.enqueue_strict(c1, 1, Request(1));
-  q.enqueue_strict(c2, 2, Request(2));
-  q.enqueue_strict(c1, 3, Request(3));
-  q.enqueue_strict(c2, 4, Request(4));
-
-  Request r;
-
-  r = q.dequeue();
-  ASSERT_EQ(4, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(3, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(2, r.value);
-  r = q.dequeue();
-  ASSERT_EQ(1, r.value);
-}
-
-
-TEST(mClockPriorityQueue, JustNotStrict)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-
-  // non-strict queue ignores priorites, but will divide between
-  // clients evenly and maintain orders between clients
-  q.enqueue(c1, 1, 1u, Request(1));
-  q.enqueue(c1, 2, 1u, Request(2));
-  q.enqueue(c2, 3, 1u, Request(3));
-  q.enqueue(c2, 4, 1u, Request(4));
-
-  Request r1, r2;
-
-  r1 = q.dequeue();
-  ASSERT_TRUE(1 == r1.value || 3 == r1.value);
-
-  r2 = q.dequeue();
-  ASSERT_TRUE(1 == r2.value || 3 == r2.value);
-
-  ASSERT_NE(r1.value, r2.value);
-
-  r1 = q.dequeue();
-  ASSERT_TRUE(2 == r1.value || 4 == r1.value);
-
-  r2 = q.dequeue();
-  ASSERT_TRUE(2 == r2.value || 4 == r2.value);
-
-  ASSERT_NE(r1.value, r2.value);
-}
-
-
-TEST(mClockPriorityQueue, EnqueuFront)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-
-  // non-strict queue ignores priorites, but will divide between
-  // clients evenly and maintain orders between clients
-  q.enqueue(c1, 1, 1u, Request(1));
-  q.enqueue(c1, 2, 1u, Request(2));
-  q.enqueue(c2, 3, 1u, Request(3));
-  q.enqueue(c2, 4, 1u, Request(4));
-  q.enqueue_strict(c2, 6, Request(6));
-  q.enqueue_strict(c1, 7, Request(7));
-
-  std::list<Request> reqs;
-
-  for (uint i = 0; i < 4; ++i) {
-    reqs.emplace_back(q.dequeue());
-  }
-
-  for (uint i = 0; i < 4; ++i) {
-    Request& r = reqs.front();
-    if (r.value > 5) {
-      q.enqueue_strict_front(r.value == 6 ? c2 : 1, r.value, std::move(r));
-    } else {
-      q.enqueue_front(r.value <= 2 ? c1 : c2, r.value, 0, std::move(r));
-    }
-    reqs.pop_front();
-  }
-
-  Request r;
-
-  r = q.dequeue();
-  ASSERT_EQ(7, r.value);
-
-  r = q.dequeue();
-  ASSERT_EQ(6, r.value);
-
-  r = q.dequeue();
-  ASSERT_TRUE(1 == r.value || 3 == r.value);
-
-  r = q.dequeue();
-  ASSERT_TRUE(1 == r.value || 3 == r.value);
-
-  r = q.dequeue();
-  ASSERT_TRUE(2 == r.value || 4 == r.value);
-
-  r = q.dequeue();
-  ASSERT_TRUE(2 == r.value || 4 == r.value);
-}
-
-
-TEST(mClockPriorityQueue, RemoveByClass)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-  Client c3(3);
-
-  q.enqueue(c1, 1, 1u, Request(1));
-  q.enqueue(c2, 1, 1u, Request(2));
-  q.enqueue(c3, 1, 1u, Request(4));
-  q.enqueue_strict(c1, 2, Request(8));
-  q.enqueue_strict(c2, 1, Request(16));
-  q.enqueue_strict(c3, 3, Request(32));
-  q.enqueue(c3, 1, 1u, Request(64));
-  q.enqueue(c2, 1, 1u, Request(128));
-  q.enqueue(c1, 1, 1u, Request(256));
-
-  int out_mask = 2 | 16 | 128;
-  int in_mask = 1 | 8 | 256;
-
-  std::list<Request> out;
-  q.remove_by_class(c2, &out);
-
-  ASSERT_EQ(3u, out.size());
-  while (!out.empty()) {
-    ASSERT_TRUE((out.front().value & out_mask) > 0) <<
-      "had value that was not expected after first removal";
-    out.pop_front();
-  }
-
-  ASSERT_EQ(6u, q.get_size_slow()) << "after removal of three from client c2";
-
-  q.remove_by_class(c3);
-
-  ASSERT_EQ(3u, q.get_size_slow()) << "after removal of three from client c3";
-  while (!q.empty()) {
-    Request r = q.dequeue();
-    ASSERT_TRUE((r.value & in_mask) > 0) <<
-      "had value that was not expected after two removals";
-  }
-}
-
-
-TEST(mClockPriorityQueue, RemoveByFilter)
-{
-  ceph::mClockQueue<Request,Client> q(&client_info_func);
-
-  Client c1(1);
-  Client c2(2);
-  Client c3(3);
-
-  q.enqueue(c1, 1, 1u, Request(1));
-  q.enqueue(c2, 1, 1u, Request(2));
-  q.enqueue(c3, 1, 1u, Request(3));
-  q.enqueue_strict(c1, 2, Request(4));
-  q.enqueue_strict(c2, 1, Request(5));
-  q.enqueue_strict(c3, 3, Request(6));
-  q.enqueue(c3, 1, 1u, Request(7));
-  q.enqueue(c2, 1, 1u, Request(8));
-  q.enqueue(c1, 1, 1u, Request(9));
-
-  std::list<Request> filtered;
-
-  q.remove_by_filter([&](const Request& r) -> bool {
-    if (r.value & 2) {
-      filtered.push_back(r);
-      return true;
-    } else {
-      return false;
-    }
-  });
-
-  ASSERT_EQ(4u, filtered.size()) <<
-    "filter should have removed four elements";
-  while (!filtered.empty()) {
-    ASSERT_TRUE((filtered.front().value & 2) > 0) <<
-      "expect this value to have been filtered out";
-    filtered.pop_front();
-  }
-
-  ASSERT_EQ(5u, q.get_size_slow()) <<
-    "filter should have left five remaining elements";
-  while (!q.empty()) {
-    Request r = q.dequeue();
-    ASSERT_TRUE((r.value & 2) == 0) <<
-      "expect this value to have been left in";
-   }
-}

From 96df279132473f459c692787609702542a32231f Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Fri, 15 Sep 2023 19:24:08 +0530
Subject: [PATCH 0567/2492] common, osd: Apply randomly selected scheduler type
 across all OSD shards

Originally, the choice of 'debug_random' for osd_op_queue resulted in the
selection of a random scheduler type for each OSD shard. A more realistic
scenario for testing would be the selection of the random scheduler type
applied globally for all shards of an OSD. In other words, all OSD shards
would employ the same scheduler type. For e.g., this scenario would be
possible during upgrades when the scheduler type has changed between
releases.

The following changes are made as part of the commit:
 1. Introduce enum class op_queue_type_t within osd_types.h that holds the
    various op queue types supported. This header in included by OpQueue.h.
    Add helper functions osd_types.cc to return the op_queue_type_t as
    enum or a string representing the enum member.
 2. Determine the scheduler type before initializing the OSD shards in
    OSD class constructor.
 3. Pass the determined op_queue_type_t to the OSDShard's make_scheduler()
    method for each shard. This ensures all shards of the OSD are
    initialized with the same scheduler type.
 4. Rename & modify the unused OSDShard::get_scheduler_type() method to
    return op_queue_type_t set for the queue.
 5. Introduce OpScheduler::get_type() and OpQueue::get_type() pure
    virtual functions and define them within the respective queue
    implementation. This returns a value pertaining to the op queue type.
    This is called by OSDShard::get_op_queue_type().
 6. Add OSD::osd_op_queue_type() method for determining the scheduler
    type set on the OSD shards. Since all OSD shards are set to use
    the same scheduler type, the shard with the lowest id is used to
    get the scheduler type using OSDShard::get_op_queue_type().
 7. Improve comment description related to 'osd_op_queue' option in
    common/options/osd.yaml.in.

Call Flow
--------
OSD                     OSDShard                 OpScheduler/OpQueue
---                     --------                 -------------------
osd_op_queue_type() ->
                        get_op_queue_type() ->
                                                 get_type()

Fixes: https://tracker.ceph.com/issues/62171
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/common/OpQueue.h                |  4 +++
 src/common/PrioritizedQueue.h       |  6 +++-
 src/common/WeightedPriorityQueue.h  |  6 +++-
 src/common/options/osd.yaml.in      | 14 ++++-----
 src/osd/OSD.cc                      | 49 ++++++++++++++++++++++++-----
 src/osd/OSD.h                       |  8 +++--
 src/osd/osd_types.cc                | 28 +++++++++++++++++
 src/osd/osd_types.h                 | 14 +++++++++
 src/osd/scheduler/OpScheduler.cc    | 17 +++-------
 src/osd/scheduler/OpScheduler.h     | 11 ++++++-
 src/osd/scheduler/mClockScheduler.h |  7 ++++-
 11 files changed, 132 insertions(+), 32 deletions(-)

diff --git a/src/common/OpQueue.h b/src/common/OpQueue.h
index 0204f4b44039..07104b21f538 100644
--- a/src/common/OpQueue.h
+++ b/src/common/OpQueue.h
@@ -16,6 +16,7 @@
 #define OP_QUEUE_H
 
 #include "include/msgr.h"
+#include "osd/osd_types.h"
 
 #include <list>
 #include <functional>
@@ -66,6 +67,9 @@ class OpQueue {
   // Human readable brief description of queue and relevant parameters
   virtual void print(std::ostream &f) const = 0;
 
+  // Get the type of OpQueue implementation
+  virtual op_queue_type_t get_type() const = 0;
+
   // Don't leak resources on destruction
   virtual ~OpQueue() {};
 };
diff --git a/src/common/PrioritizedQueue.h b/src/common/PrioritizedQueue.h
index 9adf21aafe11..0c006795eb85 100644
--- a/src/common/PrioritizedQueue.h
+++ b/src/common/PrioritizedQueue.h
@@ -345,7 +345,11 @@ class PrioritizedQueue : public OpQueue <T, K> {
   }
 
   void print(std::ostream &ostream) const final {
-    ostream << "PrioritizedQueue";
+    ostream << get_op_queue_type_name(get_type());
+  }
+
+  op_queue_type_t get_type() const final {
+    return op_queue_type_t::PrioritizedQueue;
   }
 };
 
diff --git a/src/common/WeightedPriorityQueue.h b/src/common/WeightedPriorityQueue.h
index cf34709b9794..c8d92b5e05f2 100644
--- a/src/common/WeightedPriorityQueue.h
+++ b/src/common/WeightedPriorityQueue.h
@@ -346,7 +346,11 @@ class WeightedPriorityQueue :  public OpQueue <T, K>
     }
 
     void print(std::ostream &ostream) const final {
-      ostream << "WeightedPriorityQueue";
+      ostream << get_op_queue_type_name(get_type());
+    }
+
+    op_queue_type_t get_type() const final {
+      return op_queue_type_t::WeightedPriorityQueue;
     }
 };
 
diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 6702527f0c07..c2281c00ece4 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -897,13 +897,13 @@ options:
   desc: Do not store full-object checksums if the backend (bluestore) does its own
     checksums.  Only usable with all BlueStore OSDs.
   default: false
-# PrioritzedQueue (prio), Weighted Priority Queue (wpq ; default),
-# mclock_opclass, mclock_client, or debug_random. "mclock_opclass"
-# and "mclock_client" are based on the mClock/dmClock algorithm
-# (Gulati, et al. 2010). "mclock_opclass" prioritizes based on the
-# class the operation belongs to. "mclock_client" does the same but
-# also works to ienforce fairness between clients. "debug_random"
-# chooses among all four with equal probability.
+# Weighted Priority Queue (wpq), mClock Scheduler (mclock_scheduler: default)
+# or debug_random. "mclock_scheduler" is based on the mClock/dmClock
+# algorithm (Gulati, et al. 2010). "mclock_scheduler" prioritizes based on
+# the class the operation belongs to. "wpq" dequeues ops based on their
+# priorities. "debug_random" chooses among the two with equal probability.
+# Note: PrioritzedQueue (prio) implementation is not used for scheduling ops
+# within OSDs and is therefore not listed.
 - name: osd_op_queue
   type: str
   level: advanced
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a0d302081a32..6487eddf0e76 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2378,13 +2378,39 @@ OSD::OSD(CephContext *cct_,
   trace_endpoint.copy_name(ss.str());
 #endif
 
+  // Determine scheduler type for this OSD
+  auto get_op_queue_type = [this, &conf = cct->_conf]() {
+    op_queue_type_t queue_type;
+    if (auto type = conf.get_val<std::string>("osd_op_queue");
+        type != "debug_random") {
+      if (auto qt = get_op_queue_type_by_name(type); qt.has_value()) {
+        queue_type = *qt;
+      } else {
+        // This should never happen
+        dout(0) << "Invalid value passed for 'osd_op_queue': " << type << dendl;
+        ceph_assert(0 == "Unsupported op queue type");
+      }
+    } else {
+      static const std::vector<op_queue_type_t> index_lookup = {
+        op_queue_type_t::mClockScheduler,
+        op_queue_type_t::WeightedPriorityQueue
+      };
+      std::mt19937 random_gen(std::random_device{}());
+      auto which = random_gen() % index_lookup.size();
+      queue_type = index_lookup[which];
+    }
+    return queue_type;
+  };
+  op_queue_type_t op_queue = get_op_queue_type();
+
   // initialize shards
   num_shards = get_num_op_shards();
   for (uint32_t i = 0; i < num_shards; i++) {
     OSDShard *one_shard = new OSDShard(
       i,
       cct,
-      this);
+      this,
+      op_queue);
     shards.push_back(one_shard);
   }
 }
@@ -10162,6 +10188,16 @@ bool OSD::unsupported_objstore_for_qos()
                    store->get_type()) != unsupported_objstores.end();
 }
 
+op_queue_type_t OSD::osd_op_queue_type() const
+{
+  /**
+   * All OSD shards employ the same scheduler type. Therefore, return
+   * the scheduler type set on the OSD shard with lowest id(0).
+   */
+  ceph_assert(shards.size());
+  return shards[0]->get_op_queue_type();
+}
+
 void OSD::update_log_config()
 {
   auto parsed_options = clog->parse_client_options(cct);
@@ -10662,17 +10698,16 @@ void OSDShard::update_scheduler_config()
   scheduler->update_configuration();
 }
 
-std::string OSDShard::get_scheduler_type()
+op_queue_type_t OSDShard::get_op_queue_type() const
 {
-  std::ostringstream scheduler_type;
-  scheduler_type << *scheduler;
-  return scheduler_type.str();
+  return scheduler->get_type();
 }
 
 OSDShard::OSDShard(
   int id,
   CephContext *cct,
-  OSD *osd)
+  OSD *osd,
+  op_queue_type_t osd_op_queue)
   : shard_id(id),
     cct(cct),
     osd(osd),
@@ -10684,7 +10719,7 @@ OSDShard::OSDShard(
     shard_lock{make_mutex(shard_lock_name)},
     scheduler(ceph::osd::scheduler::make_scheduler(
       cct, osd->whoami, osd->num_shards, id, osd->store->is_rotational(),
-      osd->store->get_type(), osd->monc)),
+      osd->store->get_type(), osd_op_queue, osd->monc)),
     context_queue(sdata_wait_lock, sdata_cond)
 {
   dout(0) << "using op scheduler " << *scheduler << dendl;
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index a6b7a3c31ed3..859fdbbbe234 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1032,12 +1032,13 @@ struct OSDShard {
   void register_and_wake_split_child(PG *pg);
   void unprime_split_children(spg_t parent, unsigned old_pg_num);
   void update_scheduler_config();
-  std::string get_scheduler_type();
+  op_queue_type_t get_op_queue_type() const;
 
   OSDShard(
     int id,
     CephContext *cct,
-    OSD *osd);
+    OSD *osd,
+    op_queue_type_t osd_op_queue);
 };
 
 class OSD : public Dispatcher,
@@ -2016,6 +2017,9 @@ class OSD : public Dispatcher,
   OSDService service;
   friend class OSDService;
 
+  /// op queue type set for the OSD
+  op_queue_type_t osd_op_queue_type() const;
+
 private:
   void set_perf_queries(const ConfigPayload &config_payload);
   MetricPayload get_perf_reports();
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 948abeaafc8a..e5b9f487609f 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -7404,3 +7404,31 @@ bool PGLSPlainFilter::filter(const hobject_t& obj,
 {
   return xattr_data.contents_equal(val.c_str(), val.size());
 }
+
+std::string_view get_op_queue_type_name(const op_queue_type_t &q)
+{
+  switch (q) {
+    case op_queue_type_t::WeightedPriorityQueue:
+      return "wpq";
+    case op_queue_type_t::mClockScheduler:
+      return "mclock_scheduler";
+    case op_queue_type_t::PrioritizedQueue:
+      return "PrioritizedQueue";
+    default:
+      return "unknown";
+  }
+}
+
+std::optional<op_queue_type_t> get_op_queue_type_by_name(
+  const std::string_view &s)
+{
+  if (s == "wpq") {
+    return op_queue_type_t::WeightedPriorityQueue;
+  } else if (s == "mclock_scheduler") {
+    return op_queue_type_t::mClockScheduler;
+  } else if (s == "PrioritizedQueue") {
+    return op_queue_type_t::PrioritizedQueue;
+  } else {
+    return std::nullopt;
+  }
+}
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 8b86b0a36356..1945316db816 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -6679,4 +6679,18 @@ using missing_map_t = std::map<hobject_t,
   std::pair<std::optional<uint32_t>,
     std::optional<uint32_t>>>;
 
+/**
+ * op_queue_type_t
+ *
+ * Supported op queue types
+ */
+enum class op_queue_type_t : uint8_t {
+  WeightedPriorityQueue = 0,
+  mClockScheduler,
+  PrioritizedQueue
+};
+std::string_view get_op_queue_type_name(const op_queue_type_t &q);
+std::optional<op_queue_type_t> get_op_queue_type_by_name(
+  const std::string_view &s);
+
 #endif
diff --git a/src/osd/scheduler/OpScheduler.cc b/src/osd/scheduler/OpScheduler.cc
index cb5ef13b6f67..7b89f4be0221 100644
--- a/src/osd/scheduler/OpScheduler.cc
+++ b/src/osd/scheduler/OpScheduler.cc
@@ -23,27 +23,20 @@ namespace ceph::osd::scheduler {
 
 OpSchedulerRef make_scheduler(
   CephContext *cct, int whoami, uint32_t num_shards, int shard_id,
-  bool is_rotational, std::string_view osd_objectstore, MonClient *monc)
+  bool is_rotational, std::string_view osd_objectstore,
+  op_queue_type_t osd_scheduler, MonClient *monc)
 {
-  const std::string *type = &cct->_conf->osd_op_queue;
-  if (*type == "debug_random") {
-    static const std::string index_lookup[] = { "mclock_scheduler",
-						"wpq" };
-    srand(time(NULL));
-    unsigned which = rand() % (sizeof(index_lookup) / sizeof(index_lookup[0]));
-    type = &index_lookup[which];
-  }
-
   // Force the use of 'wpq' scheduler for filestore OSDs.
   // The 'mclock_scheduler' is not supported for filestore OSDs.
-  if (*type == "wpq" || osd_objectstore == "filestore") {
+  if (op_queue_type_t::WeightedPriorityQueue == osd_scheduler ||
+      osd_objectstore == "filestore") {
     return std::make_unique<
       ClassedOpQueueScheduler<WeightedPriorityQueue<OpSchedulerItem, client>>>(
 	cct,
 	cct->_conf->osd_op_pq_max_tokens_per_priority,
 	cct->_conf->osd_op_pq_min_cost
     );
-  } else if (*type == "mclock_scheduler") {
+  } else if (op_queue_type_t::mClockScheduler == osd_scheduler) {
     // default is 'mclock_scheduler'
     return std::make_unique<
       mClockScheduler>(cct, whoami, num_shards, shard_id, is_rotational, monc);
diff --git a/src/osd/scheduler/OpScheduler.h b/src/osd/scheduler/OpScheduler.h
index 1575bcae4f6d..382f48dd40c1 100644
--- a/src/osd/scheduler/OpScheduler.h
+++ b/src/osd/scheduler/OpScheduler.h
@@ -18,6 +18,7 @@
 #include <variant>
 
 #include "common/ceph_context.h"
+#include "common/OpQueue.h"
 #include "mon/MonClient.h"
 #include "osd/scheduler/OpSchedulerItem.h"
 
@@ -54,6 +55,9 @@ class OpScheduler {
   // Apply config changes to the scheduler (if any)
   virtual void update_configuration() = 0;
 
+  // Get the scheduler type set for the queue
+  virtual op_queue_type_t get_type() const = 0;
+
   // Destructor
   virtual ~OpScheduler() {};
 };
@@ -63,7 +67,8 @@ using OpSchedulerRef = std::unique_ptr<OpScheduler>;
 
 OpSchedulerRef make_scheduler(
   CephContext *cct, int whoami, uint32_t num_shards, int shard_id,
-  bool is_rotational, std::string_view osd_objectstore, MonClient *monc);
+  bool is_rotational, std::string_view osd_objectstore,
+  op_queue_type_t osd_scheduler, MonClient *monc);
 
 /**
  * Implements OpScheduler in terms of OpQueue
@@ -143,6 +148,10 @@ class ClassedOpQueueScheduler final : public OpScheduler {
     // no-op
   }
 
+  op_queue_type_t get_type() const final {
+    return queue.get_type();
+  }
+
   ~ClassedOpQueueScheduler() final {};
 };
 
diff --git a/src/osd/scheduler/mClockScheduler.h b/src/osd/scheduler/mClockScheduler.h
index c1aca5da5d14..9f3291882798 100644
--- a/src/osd/scheduler/mClockScheduler.h
+++ b/src/osd/scheduler/mClockScheduler.h
@@ -259,12 +259,17 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
   void dump(ceph::Formatter &f) const final;
 
   void print(std::ostream &ostream) const final {
-    ostream << "mClockScheduler";
+    ostream << get_op_queue_type_name(get_type());
   }
 
   // Update data associated with the modified mclock config key(s)
   void update_configuration() final;
 
+  // Return the scheduler type
+  op_queue_type_t get_type() const final {
+    return op_queue_type_t::mClockScheduler;
+  }
+
   const char** get_tracked_conf_keys() const final;
   void handle_conf_change(const ConfigProxy& conf,
 			  const std::set<std::string> &changed) final;

From fadc0979aadafb19376248519dceb30b8c2ad4be Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Thu, 21 Sep 2023 15:05:04 +0530
Subject: [PATCH 0568/2492] osd/OSD: Query osd op queue type from scheduler
 instead of config subsystem

All OSD shards are guaranteed to use the same scheduler type. Therefore,
OSD::osd_op_queue_type() is used where applicable to determine the
scheduler type. This results in the appropriate setting of other config
options based on the randomly selected scheduler type in case the global
'osd_op_queue' config option is set to 'debug_random' (for e.g., in CI
tests).

Note: If 'osd_op_queue' is set to 'debug_random', the PG specific code
(PGPeering, PrimaryLogPG) would continue to use the existing mechanism of
querying the config option key (osd_op_queue) as before using get_val().

Fixes: https://tracker.ceph.com/issues/62171
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/osd/OSD.cc | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 6487eddf0e76..1bf3f7a5c906 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1696,7 +1696,7 @@ void OSDService::queue_recovery_context(
   epoch_t e = get_osdmap_epoch();
 
   uint64_t cost_for_queue = [this, cost] {
-    if (cct->_conf->osd_op_queue == "mclock_scheduler") {
+    if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) {
       return cost;
     } else {
       /* We retain this legacy behavior for WeightedPriorityQueue. It seems to
@@ -1764,7 +1764,7 @@ int64_t OSDService::get_scrub_cost()
 {
 
   int64_t cost_for_queue = cct->_conf->osd_scrub_cost;
-  if (cct->_conf->osd_op_queue == "mclock_scheduler") {
+  if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) {
     cost_for_queue = cct->_conf->osd_scrub_event_cost *
                      cct->_conf->osd_shallow_scrub_chunk_max;
   }
@@ -2030,7 +2030,7 @@ void OSDService::_queue_for_recovery(
   ceph_assert(ceph_mutex_is_locked_by_me(recovery_lock));
 
   uint64_t cost_for_queue = [this, &reserved_pushes, &p] {
-    if (cct->_conf->osd_op_queue == "mclock_scheduler") {
+    if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) {
       return p.cost_per_object * reserved_pushes;
     } else {
       /* We retain this legacy behavior for WeightedPriorityQueue. It seems to
@@ -9881,7 +9881,7 @@ void OSD::maybe_override_max_osd_capacity_for_qos()
   // If the scheduler enabled is mclock, override the default
   // osd capacity with the value obtained from running the
   // osd bench test. This is later used to setup mclock.
-  if ((cct->_conf.get_val<std::string>("osd_op_queue") == "mclock_scheduler") &&
+  if ((op_queue_type_t::mClockScheduler == osd_op_queue_type()) &&
       (cct->_conf.get_val<bool>("osd_mclock_skip_benchmark") == false) &&
       (!unsupported_objstore_for_qos())) {
     std::string max_capacity_iops_config;
@@ -9981,7 +9981,7 @@ bool OSD::maybe_override_options_for_qos(const std::set<std::string> *changed)
 {
   // Override options only if the scheduler enabled is mclock and the
   // underlying objectstore is supported by mclock
-  if (cct->_conf.get_val<std::string>("osd_op_queue") == "mclock_scheduler" &&
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
       !unsupported_objstore_for_qos()) {
     static const std::map<std::string, uint64_t> recovery_qos_defaults {
       {"osd_recovery_max_active", 0},
@@ -10083,9 +10083,8 @@ void OSD::maybe_override_sleep_options_for_qos()
 {
   // Override options only if the scheduler enabled is mclock and the
   // underlying objectstore is supported by mclock
-  if (cct->_conf.get_val<std::string>("osd_op_queue") == "mclock_scheduler" &&
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
       !unsupported_objstore_for_qos()) {
-
     // Override the various sleep settings
     // Disable recovery sleep
     cct->_conf.set_val("osd_recovery_sleep", std::to_string(0));
@@ -10114,7 +10113,7 @@ void OSD::maybe_override_cost_for_qos()
 {
   // If the scheduler enabled is mclock, override the default PG deletion cost
   // so that mclock can meet the QoS goals.
-  if (cct->_conf.get_val<std::string>("osd_op_queue") == "mclock_scheduler" &&
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
       !unsupported_objstore_for_qos()) {
     uint64_t pg_delete_cost = 15728640;
     cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost));

From bfbc6b65c672d6dc4326d4e29b4a1ee106c9c091 Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Mon, 13 Nov 2023 17:43:40 +0530
Subject: [PATCH 0569/2492] osd: Apply randomly determined IO priority cutoff
 across all OSD shards

Determine the op priority cutoff for an OSD and apply it on all the OSD
shards, which is a more realistic scenario. Previously, the cut off value
was randomized between OSD shards leading to issues in testing. The IO
priority cut off is first determined before initializing the OSD shards.
The cut off value is then passed to the OpScheduler implementations that
are modified accordingly to apply the values during initialization.

Fixes: https://tracker.ceph.com/issues/62171
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/osd/OSD.cc                       | 23 ++++++++++++++++++++---
 src/osd/OSD.h                        |  3 ++-
 src/osd/scheduler/OpScheduler.cc     |  6 ++++--
 src/osd/scheduler/OpScheduler.h      | 17 +++--------------
 src/osd/scheduler/mClockScheduler.cc |  2 ++
 src/osd/scheduler/mClockScheduler.h  | 20 ++++----------------
 src/test/osd/TestMClockScheduler.cc  |  5 ++++-
 7 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1bf3f7a5c906..cc51305a17a3 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2403,6 +2403,21 @@ OSD::OSD(CephContext *cct_,
   };
   op_queue_type_t op_queue = get_op_queue_type();
 
+  // Determine op queue cutoff
+  auto get_op_queue_cut_off = [&conf = cct->_conf]() {
+    if (conf.get_val<std::string>("osd_op_queue_cut_off") == "debug_random") {
+      std::random_device rd;
+      std::mt19937 random_gen(rd());
+      return (random_gen() % 2 < 1) ? CEPH_MSG_PRIO_HIGH : CEPH_MSG_PRIO_LOW;
+    } else if (conf.get_val<std::string>("osd_op_queue_cut_off") == "high") {
+      return CEPH_MSG_PRIO_HIGH;
+    } else {
+      // default / catch-all is 'low'
+      return CEPH_MSG_PRIO_LOW;
+    }
+  };
+  unsigned op_queue_cut_off = get_op_queue_cut_off();
+
   // initialize shards
   num_shards = get_num_op_shards();
   for (uint32_t i = 0; i < num_shards; i++) {
@@ -2410,7 +2425,8 @@ OSD::OSD(CephContext *cct_,
       i,
       cct,
       this,
-      op_queue);
+      op_queue,
+      op_queue_cut_off);
     shards.push_back(one_shard);
   }
 }
@@ -10706,7 +10722,8 @@ OSDShard::OSDShard(
   int id,
   CephContext *cct,
   OSD *osd,
-  op_queue_type_t osd_op_queue)
+  op_queue_type_t osd_op_queue,
+  unsigned osd_op_queue_cut_off)
   : shard_id(id),
     cct(cct),
     osd(osd),
@@ -10718,7 +10735,7 @@ OSDShard::OSDShard(
     shard_lock{make_mutex(shard_lock_name)},
     scheduler(ceph::osd::scheduler::make_scheduler(
       cct, osd->whoami, osd->num_shards, id, osd->store->is_rotational(),
-      osd->store->get_type(), osd_op_queue, osd->monc)),
+      osd->store->get_type(), osd_op_queue, osd_op_queue_cut_off, osd->monc)),
     context_queue(sdata_wait_lock, sdata_cond)
 {
   dout(0) << "using op scheduler " << *scheduler << dendl;
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 859fdbbbe234..0d08466bf1f1 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1038,7 +1038,8 @@ struct OSDShard {
     int id,
     CephContext *cct,
     OSD *osd,
-    op_queue_type_t osd_op_queue);
+    op_queue_type_t osd_op_queue,
+    unsigned osd_op_queue_cut_off);
 };
 
 class OSD : public Dispatcher,
diff --git a/src/osd/scheduler/OpScheduler.cc b/src/osd/scheduler/OpScheduler.cc
index 7b89f4be0221..12e5bdb6c45f 100644
--- a/src/osd/scheduler/OpScheduler.cc
+++ b/src/osd/scheduler/OpScheduler.cc
@@ -24,7 +24,7 @@ namespace ceph::osd::scheduler {
 OpSchedulerRef make_scheduler(
   CephContext *cct, int whoami, uint32_t num_shards, int shard_id,
   bool is_rotational, std::string_view osd_objectstore,
-  op_queue_type_t osd_scheduler, MonClient *monc)
+  op_queue_type_t osd_scheduler, unsigned op_queue_cut_off, MonClient *monc)
 {
   // Force the use of 'wpq' scheduler for filestore OSDs.
   // The 'mclock_scheduler' is not supported for filestore OSDs.
@@ -33,13 +33,15 @@ OpSchedulerRef make_scheduler(
     return std::make_unique<
       ClassedOpQueueScheduler<WeightedPriorityQueue<OpSchedulerItem, client>>>(
 	cct,
+        op_queue_cut_off,
 	cct->_conf->osd_op_pq_max_tokens_per_priority,
 	cct->_conf->osd_op_pq_min_cost
     );
   } else if (op_queue_type_t::mClockScheduler == osd_scheduler) {
     // default is 'mclock_scheduler'
     return std::make_unique<
-      mClockScheduler>(cct, whoami, num_shards, shard_id, is_rotational, monc);
+      mClockScheduler>(cct, whoami, num_shards, shard_id, is_rotational,
+        op_queue_cut_off, monc);
   } else {
     ceph_assert("Invalid choice of wq" == 0);
   }
diff --git a/src/osd/scheduler/OpScheduler.h b/src/osd/scheduler/OpScheduler.h
index 382f48dd40c1..570a2a162900 100644
--- a/src/osd/scheduler/OpScheduler.h
+++ b/src/osd/scheduler/OpScheduler.h
@@ -68,7 +68,7 @@ using OpSchedulerRef = std::unique_ptr<OpScheduler>;
 OpSchedulerRef make_scheduler(
   CephContext *cct, int whoami, uint32_t num_shards, int shard_id,
   bool is_rotational, std::string_view osd_objectstore,
-  op_queue_type_t osd_scheduler, MonClient *monc);
+  op_queue_type_t osd_scheduler, unsigned op_queue_cut_off, MonClient *monc);
 
 /**
  * Implements OpScheduler in terms of OpQueue
@@ -83,21 +83,10 @@ class ClassedOpQueueScheduler final : public OpScheduler {
   unsigned cutoff;
   T queue;
 
-  static unsigned int get_io_prio_cut(CephContext *cct) {
-    if (cct->_conf->osd_op_queue_cut_off == "debug_random") {
-      srand(time(NULL));
-      return (rand() % 2 < 1) ? CEPH_MSG_PRIO_HIGH : CEPH_MSG_PRIO_LOW;
-    } else if (cct->_conf->osd_op_queue_cut_off == "high") {
-      return CEPH_MSG_PRIO_HIGH;
-    } else {
-      // default / catch-all is 'low'
-      return CEPH_MSG_PRIO_LOW;
-    }
-  }
 public:
   template <typename... Args>
-  ClassedOpQueueScheduler(CephContext *cct, Args&&... args) :
-    cutoff(get_io_prio_cut(cct)),
+  ClassedOpQueueScheduler(CephContext *cct, unsigned prio_cut, Args&&... args) :
+    cutoff(prio_cut),
     queue(std::forward<Args>(args)...)
   {}
 
diff --git a/src/osd/scheduler/mClockScheduler.cc b/src/osd/scheduler/mClockScheduler.cc
index 0ea519655d85..f72683d52792 100644
--- a/src/osd/scheduler/mClockScheduler.cc
+++ b/src/osd/scheduler/mClockScheduler.cc
@@ -35,12 +35,14 @@ mClockScheduler::mClockScheduler(CephContext *cct,
   uint32_t num_shards,
   int shard_id,
   bool is_rotational,
+  unsigned cutoff_priority,
   MonClient *monc)
   : cct(cct),
     whoami(whoami),
     num_shards(num_shards),
     shard_id(shard_id),
     is_rotational(is_rotational),
+    cutoff_priority(cutoff_priority),
     monc(monc),
     scheduler(
       std::bind(&mClockScheduler::ClientRegistry::get_info,
diff --git a/src/osd/scheduler/mClockScheduler.h b/src/osd/scheduler/mClockScheduler.h
index 9f3291882798..16e7f911ff95 100644
--- a/src/osd/scheduler/mClockScheduler.h
+++ b/src/osd/scheduler/mClockScheduler.h
@@ -96,6 +96,7 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
   const uint32_t num_shards;
   const int shard_id;
   const bool is_rotational;
+  const unsigned cutoff_priority;
   MonClient *monc;
 
   /**
@@ -198,21 +199,6 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
     };
   }
 
-  static unsigned int get_io_prio_cut(CephContext *cct) {
-    if (cct->_conf->osd_op_queue_cut_off == "debug_random") {
-      std::random_device rd;
-      std::mt19937 random_gen(rd());
-      return (random_gen() % 2 < 1) ? CEPH_MSG_PRIO_HIGH : CEPH_MSG_PRIO_LOW;
-    } else if (cct->_conf->osd_op_queue_cut_off == "high") {
-      return CEPH_MSG_PRIO_HIGH;
-    } else {
-      // default / catch-all is 'low'
-      return CEPH_MSG_PRIO_LOW;
-    }
-  }
-
-  unsigned cutoff_priority = get_io_prio_cut(cct);
-
   /**
    * set_osd_capacity_params_from_config
    *
@@ -232,7 +218,8 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
 
 public: 
   mClockScheduler(CephContext *cct, int whoami, uint32_t num_shards,
-    int shard_id, bool is_rotational, MonClient *monc);
+    int shard_id, bool is_rotational, unsigned cutoff_priority,
+    MonClient *monc);
   ~mClockScheduler() override;
 
   /// Calculate scaled cost per item
@@ -260,6 +247,7 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
 
   void print(std::ostream &ostream) const final {
     ostream << get_op_queue_type_name(get_type());
+    ostream << ", cutoff=" << cutoff_priority;
   }
 
   // Update data associated with the modified mclock config key(s)
diff --git a/src/test/osd/TestMClockScheduler.cc b/src/test/osd/TestMClockScheduler.cc
index e7bac03d2abd..325ebe77e802 100644
--- a/src/test/osd/TestMClockScheduler.cc
+++ b/src/test/osd/TestMClockScheduler.cc
@@ -31,6 +31,7 @@ class mClockSchedulerTest : public testing::Test {
   uint32_t num_shards;
   int shard_id;
   bool is_rotational;
+  unsigned cutoff_priority;
   MonClient *monc;
   mClockScheduler q;
 
@@ -43,8 +44,10 @@ class mClockSchedulerTest : public testing::Test {
     num_shards(1),
     shard_id(0),
     is_rotational(false),
+    cutoff_priority(12),
     monc(nullptr),
-    q(g_ceph_context, whoami, num_shards, shard_id, is_rotational, monc),
+    q(g_ceph_context, whoami, num_shards, shard_id, is_rotational,
+      cutoff_priority, monc),
     client1(1001),
     client2(9999),
     client3(100000001)

From b150e935e8433ce594d9ab25b16094e616d2f489 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Tue, 14 Nov 2023 15:35:50 +0530
Subject: [PATCH 0570/2492] qa: run kernel_untar_build with newer tarball

Replace old tarball link with a url link to linux-6.5.11.tar.xz.

Fixes: https://tracker.ceph.com/issues/57655
Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 qa/workunits/kernel_untar_build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/workunits/kernel_untar_build.sh b/qa/workunits/kernel_untar_build.sh
index 9ee55eac9924..9855f3d31b6f 100755
--- a/qa/workunits/kernel_untar_build.sh
+++ b/qa/workunits/kernel_untar_build.sh
@@ -2,11 +2,11 @@
 
 set -ex
 
-wget -O linux.tar.gz http://download.ceph.com/qa/linux-5.4.tar.gz
+wget -O linux.tar.xz http://download.ceph.com/qa/linux-6.5.11.tar.xz
 
 mkdir t
 cd t
-tar xzf ../linux.tar.gz
+tar xJf ../linux.tar.xz
 cd linux*
 make defconfig
 make -j`grep -c processor /proc/cpuinfo`

From d580e2392f3ae9843967ceccae053b32f6b0f2c3 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 14 Nov 2023 11:30:00 +0000
Subject: [PATCH 0571/2492] qa/suites/rados/verify/validater/valgrind: increase
 op thread timeout

Fixes: https://tracker.ceph.com/issues/62992

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/suites/rados/verify/validater/valgrind.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/suites/rados/verify/validater/valgrind.yaml b/qa/suites/rados/verify/validater/valgrind.yaml
index 03accceaff25..f3ee027b9abb 100644
--- a/qa/suites/rados/verify/validater/valgrind.yaml
+++ b/qa/suites/rados/verify/validater/valgrind.yaml
@@ -9,6 +9,8 @@ overrides:
     conf:
       global:
         osd heartbeat grace: 80
+        # see https://tracker.ceph.com/issues/62992
+        osd op thread timeout: 150
       mon:
         mon osd crush smoke test: false
       osd:

From 9b773eec4a8c343894ba727e05ec89e39e6cf685 Mon Sep 17 00:00:00 2001
From: Suyashd999 <suyashd999@gmail.com>
Date: Tue, 14 Nov 2023 12:59:13 +0530
Subject: [PATCH 0572/2492] qa/suites/rbd: Cleanup of MIRROR_IMAGE_MODE

Fixes: https://tracker.ceph.com/issues/63431
Signed-off-by: Suyash Dongre <suyashd999@gmail.com>
---
 .../rbd-mirror-journal-stress-workunit.yaml         |  2 +-
 ...ror-snapshot-stress-workunit-exclusive-lock.yaml |  2 +-
 ...d-mirror-snapshot-stress-workunit-fast-diff.yaml |  2 +-
 ...rbd-mirror-snapshot-stress-workunit-minimum.yaml |  2 +-
 .../rbd-mirror-journal-bootstrap-workunit.yaml      |  2 +-
 .../rbd-mirror-snapshot-bootstrap-workunit.yaml     |  2 +-
 qa/workunits/rbd/rbd_mirror_helpers.sh              | 13 ++++++-------
 qa/workunits/rbd/rbd_mirror_stress.sh               |  4 ++--
 8 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-stress-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-stress-workunit.yaml
index 9579b70d62c0..4b85b483af46 100644
--- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-stress-workunit.yaml
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-stress-workunit.yaml
@@ -11,5 +11,5 @@ tasks:
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_USE_RBD_MIRROR: '1'
       MIRROR_POOL_MODE: 'pool'
-      MIRROR_IMAGE_MODE: 'journal'
+      RBD_MIRROR_MODE: 'journal'
     timeout: 6h
diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock.yaml
index 87632483d1cb..4b9225bdcba9 100644
--- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock.yaml
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock.yaml
@@ -8,7 +8,7 @@ tasks:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
       MIRROR_POOL_MODE: 'image'
-      MIRROR_IMAGE_MODE: 'snapshot'
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_IMAGE_FEATURES: 'layering,exclusive-lock'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-fast-diff.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-fast-diff.yaml
index fc43b0ec24d7..756fc9c741eb 100644
--- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-fast-diff.yaml
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-fast-diff.yaml
@@ -8,7 +8,7 @@ tasks:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
       MIRROR_POOL_MODE: 'image'
-      MIRROR_IMAGE_MODE: 'snapshot'
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_IMAGE_FEATURES: 'layering,exclusive-lock,object-map,fast-diff'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-minimum.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-minimum.yaml
index af0ea12408bd..cc18601f3d8c 100644
--- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-minimum.yaml
+++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-snapshot-stress-workunit-minimum.yaml
@@ -8,7 +8,7 @@ tasks:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
       MIRROR_POOL_MODE: 'image'
-      MIRROR_IMAGE_MODE: 'snapshot'
+      RBD_MIRROR_MODE: 'snapshot'
       RBD_IMAGE_FEATURES: 'layering'
       RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-journal-bootstrap-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-journal-bootstrap-workunit.yaml
index b9c5562be834..e129374af25e 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-journal-bootstrap-workunit.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-journal-bootstrap-workunit.yaml
@@ -10,4 +10,4 @@ tasks:
       RBD_MIRROR_INSTANCES: '1'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       MIRROR_POOL_MODE: 'pool'
-      MIRROR_IMAGE_MODE: 'journal'
+      RBD_MIRROR_MODE: 'journal'
diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-bootstrap-workunit.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-bootstrap-workunit.yaml
index 5ad78474d3cb..9507c6ddc843 100644
--- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-bootstrap-workunit.yaml
+++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-bootstrap-workunit.yaml
@@ -10,4 +10,4 @@ tasks:
       RBD_MIRROR_INSTANCES: '1'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       MIRROR_POOL_MODE: 'image'
-      MIRROR_IMAGE_MODE: 'snapshot'
+      RBD_MIRROR_MODE: 'snapshot'
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index 7183a2cba258..f7a3e5de0d99 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -94,11 +94,10 @@ TEMPDIR=
 CEPH_ID=${CEPH_ID:-mirror}
 RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling}
 MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.}
+RBD_MIRROR_MODE=${RBD_MIRROR_MODE:-journal}
 MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool}
-MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal}
 if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
   MIRROR_POOL_MODE=image
-  MIRROR_IMAGE_MODE=snapshot
 fi
 
 export CEPH_ARGS="--id ${CEPH_ID}"
@@ -805,9 +804,9 @@ wait_for_replay_complete()
     local pool=$3
     local image=$4
 
-    if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then
+    if [ "${RBD_MIRROR_MODE}" = "journal" ]; then
         wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image}
-    elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then
+    elif [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then
         wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image}
     else
         return 1
@@ -942,7 +941,7 @@ create_image_and_enable_mirror()
     local cluster=$1 ; shift
     local pool=$1 ; shift
     local image=$1 ; shift
-    local mode=${1:-${MIRROR_IMAGE_MODE}}
+    local mode=${1:-${RBD_MIRROR_MODE}}
     if [ -n "$1" ]; then
         shift
     fi
@@ -1064,7 +1063,7 @@ clone_image_and_enable_mirror()
     local clone_image=$6
     shift 6
 
-    local mode=${1:-${MIRROR_IMAGE_MODE}}
+    local mode=${1:-${RBD_MIRROR_MODE}}
     if [ -n "$1" ]; then
         shift
     fi
@@ -1363,7 +1362,7 @@ enable_mirror()
     local cluster=$1
     local pool=$2
     local image=$3
-    local mode=${4:-${MIRROR_IMAGE_MODE}}
+    local mode=${4:-${RBD_MIRROR_MODE}}
 
     rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode}
     # Display image info including the global image id for debugging purpose
diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
index cb79aba7ebc9..5426f495db10 100755
--- a/qa/workunits/rbd/rbd_mirror_stress.sh
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -96,7 +96,7 @@ start_mirrors ${CLUSTER2}
 
 testlog "TEST: add image and test replay after client crashes"
 image=test
-create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '512M'
+create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} '512M'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 clean_snap_name=
@@ -151,7 +151,7 @@ snap_name="snap"
 for i in `seq 1 ${IMAGE_COUNT}`
 do
   image="image_${i}"
-  create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '128M'
+  create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} '128M'
   if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
     write_image ${CLUSTER2} ${POOL} ${image} 100
   else

From 3e04ac48779f62394ebd9587298823bf6b20a570 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 19 Oct 2023 14:57:23 +0300
Subject: [PATCH 0573/2492] os/bluestore: rework vselector calls

We can provide fnode delta to vseector now. Which is a bit more
effective.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueFS.cc    | 67 +++++++++++++------------
 src/os/bluestore/BlueFS.h     | 31 +++++++++---
 src/os/bluestore/BlueStore.cc | 10 +++-
 src/os/bluestore/BlueStore.h  | 93 ++++++++++++++++-------------------
 4 files changed, 109 insertions(+), 92 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 52857a16081d..41f75085ad8c 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -1579,11 +1579,9 @@ int BlueFS::_replay(bool noop, bool to_stdout)
             }
             if (fnode.ino != 1) {
               vselector->sub_usage(f->vselector_hint, f->fnode);
-            }
+	      vselector->add_usage(f->vselector_hint, fnode);
+	    }
             f->fnode = fnode;
-            if (fnode.ino != 1) {
-              vselector->add_usage(f->vselector_hint, f->fnode);
-            }
 
 	    if (fnode.ino > ino_last) {
 	      ino_last = fnode.ino;
@@ -1825,7 +1823,7 @@ int BlueFS::device_migrate_to_existing(
 
       // write entire file
       auto l = _allocate(dev_target, bl.length(), 0,
-        &file_ref->fnode, 0, false);
+        &file_ref->fnode, nullptr, 0, false);
       if (l < 0) {
 	derr << __func__ << " unable to allocate len 0x" << std::hex
 	     << bl.length() << std::dec << " from " << (int)dev_target
@@ -1965,7 +1963,7 @@ int BlueFS::device_migrate_to_new(
 
       // write entire file
       auto l = _allocate(dev_target, bl.length(), 0,
-        &file_ref->fnode, 0, false);
+        &file_ref->fnode, nullptr, 0, false);
       if (l < 0) {
 	derr << __func__ << " unable to allocate len 0x" << std::hex
 	     << bl.length() << std::dec << " from " << (int)dev_target
@@ -2559,7 +2557,6 @@ void BlueFS::_rewrite_log_and_layout_sync_LNF_LD(bool permit_dev_fallback,
   auto t0 = mono_clock::now();
 
   File *log_file = log.writer->file.get();
-  bluefs_fnode_t fnode_tail;
   // log.t.seq is always set to current live seq
   ceph_assert(log.t.seq == log.seq_live);
   // Capturing entire state. Dump anything that has been stored there.
@@ -2614,7 +2611,8 @@ void BlueFS::_rewrite_log_and_layout_sync_LNF_LD(bool permit_dev_fallback,
 
   dout(20) << __func__ << " compacted_meta_need " << compacted_meta_need << dendl;
 
-  int r = _allocate(log_dev, compacted_meta_need, 0, &fnode_tail, 0,
+  bluefs_fnode_t fnode_tail;
+  int r = _allocate(log_dev, compacted_meta_need, 0, &fnode_tail, nullptr, 0,
     permit_dev_fallback);
   ceph_assert(r == 0);
 
@@ -2625,7 +2623,7 @@ void BlueFS::_rewrite_log_and_layout_sync_LNF_LD(bool permit_dev_fallback,
   uint64_t starter_need = _make_initial_transaction(starter_seq, fnode_tail, 0, nullptr);
 
   bluefs_fnode_t fnode_starter(log_file->fnode.ino, 0, mtime);
-  r = _allocate(log_dev, starter_need, 0, &fnode_starter, 0,
+  r = _allocate(log_dev, starter_need, 0, &fnode_starter, nullptr, 0,
     permit_dev_fallback);
   ceph_assert(r == 0);
 
@@ -3126,16 +3124,17 @@ void BlueFS::_extend_log(uint64_t amount) {
   }
   ll.release();
   uint64_t allocated_before_extension = log.writer->file->fnode.get_allocated();
-  vselector->sub_usage(log.writer->file->vselector_hint, log.writer->file->fnode);
   amount = round_up_to(amount, super.block_size);
   int r = _allocate(
       vselector->select_prefer_bdev(log.writer->file->vselector_hint),
       amount,
       0,
-      &log.writer->file->fnode);
+      &log.writer->file->fnode,
+      [&](const bluefs_extent_t& e) {
+        vselector->add_usage(log.writer->file->vselector_hint, e);
+      });
   ceph_assert(r == 0);
   dout(10) << "extended log by 0x" << std::hex << amount << " bytes " << dendl;
-  vselector->add_usage(log.writer->file->vselector_hint, log.writer->file->fnode);
 
   bluefs_transaction_t log_extend_transaction;
   log_extend_transaction.seq = log.t.seq;
@@ -3405,7 +3404,9 @@ int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length)
 
   dout(10) << __func__ << " " << h << " pos 0x" << std::hex << h->pos
 	   << " 0x" << offset << "~" << length << std::dec
-	   << " to " << h->file->fnode << dendl;
+	   << " to " << h->file->fnode
+	   << " hint " << h->file->vselector_hint
+           << dendl;
   if (h->file->deleted) {
     dout(10) << __func__ << "  deleted, no-op" << dendl;
     return 0;
@@ -3426,7 +3427,6 @@ int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length)
   ceph_assert(offset <= h->file->fnode.size);
 
   uint64_t allocated = h->file->fnode.get_allocated();
-  vselector->sub_usage(h->file->vselector_hint, h->file->fnode);
   // do not bother to dirty the file if we are overwriting
   // previously allocated extents.
   if (allocated < offset + length) {
@@ -3435,25 +3435,26 @@ int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length)
     int r = _allocate(vselector->select_prefer_bdev(h->file->vselector_hint),
 		      offset + length - allocated,
                       0,
-		      &h->file->fnode);
+		      &h->file->fnode,
+		      [&](const bluefs_extent_t& e) {
+		        vselector->add_usage(h->file->vselector_hint, e);
+	              });
     if (r < 0) {
       derr << __func__ << " allocated: 0x" << std::hex << allocated
            << " offset: 0x" << offset << " length: 0x" << length << std::dec
            << dendl;
-      vselector->add_usage(h->file->vselector_hint, h->file->fnode); // undo
       ceph_abort_msg("bluefs enospc");
       return r;
     }
     h->file->is_dirty = true;
   }
   if (h->file->fnode.size < offset + length) {
+    vselector->add_usage(h->file->vselector_hint, offset + length - h->file->fnode.size);
     h->file->fnode.size = offset + length;
     h->file->is_dirty = true;
   }
-
   dout(20) << __func__ << " file now, unflushed " << h->file->fnode << dendl;
   int res = _flush_data(h, offset, length, buffered);
-  vselector->add_usage(h->file->vselector_hint, h->file->fnode);
   logger->tinc(l_bluefs_flush_lat, mono_clock::now() - t0);
   return res;
 }
@@ -3710,10 +3711,9 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset)/*_WF_L*/
   _flush_bdev(h);
 
   std::lock_guard ll(log.lock);
-  vselector->sub_usage(h->file->vselector_hint, h->file->fnode.size);
+  vselector->sub_usage(h->file->vselector_hint, h->file->fnode.size - offset);
   h->file->fnode.size = offset;
   h->file->is_dirty = true;
-  vselector->add_usage(h->file->vselector_hint, h->file->fnode.size);
   log.t.op_file_update_inc(h->file->fnode);
   logger->tinc(l_bluefs_truncate_lat, mono_clock::now() - t0);
   return 0;
@@ -3809,6 +3809,7 @@ const char* BlueFS::get_device_name(unsigned id)
 int BlueFS::_allocate(uint8_t id, uint64_t len,
 		      uint64_t alloc_unit,
 		      bluefs_fnode_t* node,
+                      update_fn_t cb,
                       size_t alloc_attempts,
                       bool permit_dev_fallback)
 {
@@ -3892,6 +3893,7 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
                        len,
                        alloc_unit,
                        node,
+		       cb,
                        alloc_attempts,
                        permit_dev_fallback);
     } else if (permit_dev_fallback && id != BDEV_SLOW && alloc[id + 1]) {
@@ -3905,6 +3907,7 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
                        len,
                        0, // back to default alloc unit
                        node,
+		       cb,
                        alloc_attempts,
                        permit_dev_fallback);
     } else {
@@ -3924,9 +3927,12 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
   }
 
   for (auto& p : extents) {
-    node->append_extent(bluefs_extent_t(id, p.offset, p.length));
+    bluefs_extent_t e(id, p.offset, p.length);
+    node->append_extent(e);
+    if (cb) {
+      cb(e);
+    }
   }
-   
   return 0;
 }
 
@@ -3945,12 +3951,13 @@ int BlueFS::preallocate(FileRef f, uint64_t off, uint64_t len)/*_LF*/
   if (off + len > allocated) {
     uint64_t want = off + len - allocated;
 
-    vselector->sub_usage(f->vselector_hint, f->fnode);
     int r = _allocate(vselector->select_prefer_bdev(f->vselector_hint),
       want,
       0,
-      &f->fnode);
-    vselector->add_usage(f->vselector_hint, f->fnode);
+      &f->fnode,
+      [&](const bluefs_extent_t& e) {
+	vselector->add_usage(f->vselector_hint, e);
+      });
     if (r < 0)
       return r;
 
@@ -4007,7 +4014,6 @@ int BlueFS::open_for_write(
   _maybe_check_vselector_LNF();
   FileRef file;
   bool create = false;
-  bool truncate = false;
   mempool::bluefs::vector<bluefs_extent_t> pending_release_extents;
   {
   std::lock_guard ll(log.lock);
@@ -4034,10 +4040,12 @@ int BlueFS::open_for_write(
     }
     file = ceph::make_ref<File>();
     file->fnode.ino = ++ino_last;
+    file->vselector_hint = vselector->get_hint_by_dir(dirname);
     nodes.file_map[ino_last] = file;
     dir->file_map.emplace_hint(q, string{filename}, file);
     ++file->refs;
     create = true;
+    vselector->add_usage(file->vselector_hint, file->fnode.size, true); // update file count
     logger->set(l_bluefs_num_files, nodes.file_map.size());
   } else {
     // overwrite existing file?
@@ -4052,8 +4060,8 @@ int BlueFS::open_for_write(
 	       << " already exists, truncate + overwrite" << dendl;
       vselector->sub_usage(file->vselector_hint, file->fnode);
       file->fnode.size = 0;
+      vselector->add_usage(file->vselector_hint, file->fnode.size, true); // restore file count
       pending_release_extents.swap(file->fnode.extents);
-      truncate = true;
 
       file->fnode.clear_extents();
     }
@@ -4061,11 +4069,6 @@ int BlueFS::open_for_write(
   ceph_assert(file->fnode.ino > 1);
 
   file->fnode.mtime = ceph_clock_now();
-  file->vselector_hint = vselector->get_hint_by_dir(dirname);
-  if (create || truncate) {
-    vselector->add_usage(file->vselector_hint, file->fnode); // update file count
-  }
-
   dout(20) << __func__ << " mapping " << dirname << "/" << filename
 	   << " vsel_hint " << file->vselector_hint
 	   << dendl;
diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index f0493a758dab..38a0bf1a601c 100644
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -91,10 +91,22 @@ class BlueFSVolumeSelector {
   virtual void* get_hint_for_log() const = 0;
   virtual void* get_hint_by_dir(std::string_view dirname) const = 0;
 
-  virtual void add_usage(void* file_hint, const bluefs_fnode_t& fnode) = 0;
-  virtual void sub_usage(void* file_hint, const bluefs_fnode_t& fnode) = 0;
-  virtual void add_usage(void* file_hint, uint64_t fsize) = 0;
-  virtual void sub_usage(void* file_hint, uint64_t fsize) = 0;
+  void add_usage(void* hint, const bluefs_fnode_t& fnode) {
+    for (auto& e : fnode.extents) {
+      add_usage(hint, e);
+    }
+    add_usage(hint, fnode.size, true);
+  }
+  void sub_usage(void* hint, const bluefs_fnode_t& fnode) {
+    for (auto& e : fnode.extents) {
+      sub_usage(hint, e);
+    }
+    sub_usage(hint, fnode.size, true);
+  }
+  virtual void add_usage(void* hint, const bluefs_extent_t& extent) = 0;
+  virtual void sub_usage(void* hint, const bluefs_extent_t& extent) = 0;
+  virtual void add_usage(void* hint, uint64_t fsize, bool upd_files = false) = 0;
+  virtual void sub_usage(void* hint, uint64_t fsize, bool upd_files = false) = 0;
   virtual uint8_t select_prefer_bdev(void* hint) = 0;
   virtual void get_paths(const std::string& base, paths& res) const = 0;
   virtual void dump(std::ostream& sout) = 0;
@@ -439,9 +451,12 @@ class BlueFS {
     return bdev[BDEV_SLOW] ? BDEV_SLOW : BDEV_DB;
   }
   const char* get_device_name(unsigned id);
+
+  typedef std::function<void(const bluefs_extent_t)> update_fn_t;
   int _allocate(uint8_t bdev, uint64_t len,
                 uint64_t alloc_unit,
 		bluefs_fnode_t* node,
+                update_fn_t cb = nullptr,
                 size_t alloc_attempts = 0,
                 bool permit_dev_fallback = true);
 
@@ -716,19 +731,19 @@ class OriginalVolumeSelector : public BlueFSVolumeSelector {
   void* get_hint_for_log() const override;
   void* get_hint_by_dir(std::string_view dirname) const override;
 
-  void add_usage(void* hint, const bluefs_fnode_t& fnode) override {
+  void add_usage(void* hint, const bluefs_extent_t& extent) override {
     // do nothing
     return;
   }
-  void sub_usage(void* hint, const bluefs_fnode_t& fnode) override {
+  void sub_usage(void* hint, const bluefs_extent_t& extent) override {
     // do nothing
     return;
   }
-  void add_usage(void* hint, uint64_t fsize) override {
+  void add_usage(void*, uint64_t, bool) override {
     // do nothing
     return;
   }
-  void sub_usage(void* hint, uint64_t fsize) override {
+  void sub_usage(void*, uint64_t, bool) override {
     // do nothing
     return;
   }
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index b8dca31c0575..0f79fc4f0958 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -7396,7 +7396,7 @@ int BlueStore::_open_bluefs(bool create, bool read_only)
           bluefs->get_block_device_size(BlueFS::BDEV_WAL) * 95 / 100,
           bluefs->get_block_device_size(BlueFS::BDEV_DB) * 95 / 100,
           bluefs->get_block_device_size(BlueFS::BDEV_SLOW) * 95 / 100,
-          1024 * 1024 * 1024, //FIXME: set expected l0 size here
+	  rocks_opts.write_buffer_size * rocks_opts.max_write_buffer_number,
           rocks_opts.max_bytes_for_level_base,
           rocks_opts.max_bytes_for_level_multiplier,
           reserved_factor,
@@ -19266,7 +19266,13 @@ void RocksDBBlueFSVolumeSelector::dump(ostream& sout) {
   auto max_x = per_level_per_dev_usage.get_max_x();
   auto max_y = per_level_per_dev_usage.get_max_y();
 
-  sout << "RocksDBBlueFSVolumeSelector Usage Matrix:" << std::endl;
+  sout << "RocksDBBlueFSVolumeSelector " << std::endl;
+  sout << ">>Settings<<"
+       << " extra=" << byte_u_t(db_avail4slow)
+       << ", l0_size=" << byte_u_t(level0_size)
+       << ", l_base=" << byte_u_t(level_base)
+       << ", l_multi=" << byte_u_t(level_multiplier)
+       << std::endl;
   constexpr std::array<const char*, 8> names{ {
     "DEV/LEV",
     "WAL",
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 095c838608ed..beac82f163a7 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -4240,6 +4240,9 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
 
   uint64_t l_totals[LEVEL_MAX - LEVEL_FIRST];
   uint64_t db_avail4slow = 0;
+  uint64_t level0_size = 0;
+  uint64_t level_base = 0;
+  uint64_t level_multiplier = 0;
   enum {
     OLD_POLICY,
     USE_SOME_EXTRA
@@ -4265,21 +4268,24 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
     if (!new_pol) {
       return;
     }
-
     // Calculating how much extra space is available at DB volume.
     // Depending on the presence of explicit reserved size specification it might be either
     // * DB volume size - reserved
     // or
     // * DB volume size - sum_max_level_size(0, L-1) - max_level_size(L) * reserved_factor
     if (!reserved) {
+      level0_size = _level0_size;
+      level_base = _level_base;
+      level_multiplier = _level_multiplier;
       uint64_t prev_levels = _level0_size;
       uint64_t cur_level = _level_base;
-      uint64_t cur_threshold = 0;
+      uint64_t cur_threshold = prev_levels + cur_level;
       do {
-        uint64_t next_level = cur_level * _level_multiplier;
-        uint64_t next_threshold = prev_levels + cur_level + next_level * reserved_factor;
+	uint64_t next_level = cur_level * _level_multiplier;
+        uint64_t next_threshold = prev_levels + cur_level + next_level;
         if (_db_total <= next_threshold) {
-          db_avail4slow = cur_threshold ? _db_total - cur_threshold : 0;
+	  cur_threshold *= reserved_factor;
+          db_avail4slow = cur_threshold < _db_total ? _db_total - cur_threshold : 0;
           break;
         } else {
           prev_levels += cur_level;
@@ -4288,7 +4294,7 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
         }
       } while (true);
     } else {
-      db_avail4slow = _db_total - reserved;
+      db_avail4slow = reserved < _db_total ? _db_total - reserved : 0;
     }
   }
 
@@ -4297,63 +4303,40 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
   }
   void* get_hint_by_dir(std::string_view dirname) const override;
 
-  void add_usage(void* hint, const bluefs_fnode_t& fnode) override {
+  void add_usage(void* hint, const bluefs_extent_t& extent) override {
     if (hint == nullptr)
       return;
     size_t pos = (size_t)hint - LEVEL_FIRST;
-    for (auto& p : fnode.extents) {
-      auto& cur = per_level_per_dev_usage.at(p.bdev, pos);
-      auto& max = per_level_per_dev_max.at(p.bdev, pos);
-      uint64_t v = cur.fetch_add(p.length) + p.length;
-      while (v > max) {
-	max.exchange(v);
-      }
-      {
-        //update per-device totals
-        auto& cur = per_level_per_dev_usage.at(p.bdev, LEVEL_MAX - LEVEL_FIRST);
-        auto& max = per_level_per_dev_max.at(p.bdev, LEVEL_MAX - LEVEL_FIRST);
-        uint64_t v = cur.fetch_add(p.length) + p.length;
-	while (v > max) {
-	  max.exchange(v);
-	}
-      }
+    auto& cur = per_level_per_dev_usage.at(extent.bdev, pos);
+    auto& max = per_level_per_dev_max.at(extent.bdev, pos);
+    uint64_t v = cur.fetch_add(extent.length) + extent.length;
+    while (v > max) {
+      max.exchange(v);
     }
     {
-      //update per-level actual totals
-      auto& cur = per_level_per_dev_usage.at(BlueFS::MAX_BDEV, pos);
-      auto& max = per_level_per_dev_max.at(BlueFS::MAX_BDEV, pos);
-      uint64_t v = cur.fetch_add(fnode.size) + fnode.size;
+      //update per-device totals
+      auto& cur = per_level_per_dev_usage.at(extent.bdev, LEVEL_MAX - LEVEL_FIRST);
+      auto& max = per_level_per_dev_max.at(extent.bdev, LEVEL_MAX - LEVEL_FIRST);
+      uint64_t v = cur.fetch_add(extent.length) + extent.length;
       while (v > max) {
 	max.exchange(v);
       }
     }
-    ++per_level_files[pos];
-    ++per_level_files[LEVEL_MAX - LEVEL_FIRST];
   }
-  void sub_usage(void* hint, const bluefs_fnode_t& fnode) override {
+  void sub_usage(void* hint, const bluefs_extent_t& extent) override {
     if (hint == nullptr)
       return;
     size_t pos = (size_t)hint - LEVEL_FIRST;
-    for (auto& p : fnode.extents) {
-      auto& cur = per_level_per_dev_usage.at(p.bdev, pos);
-      ceph_assert(cur >= p.length);
-      cur -= p.length;
-
-      //update per-device totals
-      auto& cur2 = per_level_per_dev_usage.at(p.bdev, LEVEL_MAX - LEVEL_FIRST);
-      ceph_assert(cur2 >= p.length);
-      cur2 -= p.length;
-    }
-    //update per-level actual totals
-    auto& cur = per_level_per_dev_usage.at(BlueFS::MAX_BDEV, pos);
-    ceph_assert(cur >= fnode.size);
-    cur -= fnode.size;
-    ceph_assert(per_level_files[pos] > 0);
-    --per_level_files[pos];
-    ceph_assert(per_level_files[LEVEL_MAX - LEVEL_FIRST] > 0);
-    --per_level_files[LEVEL_MAX - LEVEL_FIRST];
+    auto& cur = per_level_per_dev_usage.at(extent.bdev, pos);
+    ceph_assert(cur >= extent.length);
+    cur -= extent.length;
+
+    //update per-device totals
+    auto& cur2 = per_level_per_dev_usage.at(extent.bdev, LEVEL_MAX - LEVEL_FIRST);
+    ceph_assert(cur2 >= extent.length);
+    cur2 -= extent.length;
   }
-  void add_usage(void* hint, uint64_t size_more) override {
+  void add_usage(void* hint, uint64_t size_more, bool upd_files) override {
     if (hint == nullptr)
       return;
     size_t pos = (size_t)hint - LEVEL_FIRST;
@@ -4364,8 +4347,12 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
     while (v > max) {
       max.exchange(v);
     }
+    if (upd_files) {
+      ++per_level_files[pos];
+      ++per_level_files[LEVEL_MAX - LEVEL_FIRST];
+    }
   }
-  void sub_usage(void* hint, uint64_t size_less) override {
+  void sub_usage(void* hint, uint64_t size_less, bool upd_files) override {
     if (hint == nullptr)
       return;
     size_t pos = (size_t)hint - LEVEL_FIRST;
@@ -4373,6 +4360,12 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
     auto& cur = per_level_per_dev_usage.at(BlueFS::MAX_BDEV, pos);
     ceph_assert(cur >= size_less);
     cur -= size_less;
+    if (upd_files) {
+      ceph_assert(per_level_files[pos] > 0);
+      --per_level_files[pos];
+      ceph_assert(per_level_files[LEVEL_MAX - LEVEL_FIRST] > 0);
+      --per_level_files[LEVEL_MAX - LEVEL_FIRST];
+    }
   }
 
   uint8_t select_prefer_bdev(void* h) override;

From 5019495659359bbc19c8ab3988e3d9e12c5f58c1 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 19 Oct 2023 14:59:16 +0300
Subject: [PATCH 0574/2492] test/store_test: refactor spillover tests

Fixes: https://tracker.ceph.com/issues/62401
Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/test/objectstore/store_test.cc | 202 +++++++++++++++++------------
 1 file changed, 118 insertions(+), 84 deletions(-)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 03dc1a87e1fc..c34e7edf3c0d 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -92,7 +92,23 @@ static bool bl_eq(bufferlist& expected, bufferlist& actual)
   return false;
 }
 
+void dump_bluefs_stats()
+{
+  AdminSocket* admin_socket = g_ceph_context->get_admin_socket();
+  ceph_assert(admin_socket);
+
+  ceph::bufferlist in, out;
+  ostringstream err;
 
+  auto r = admin_socket->execute_command(
+    { "{\"prefix\": \"bluefs stats\"}" },
+    in, err, &out);
+  if (r != 0) {
+    cerr << "failure querying: " << cpp_strerror(r) << std::endl;
+  } else {
+    std::cout << std::string(out.c_str(), out.length()) << std::endl;
+  }
+}
 
 template <typename T>
 int queue_transaction(
@@ -10462,24 +10478,11 @@ void doManySetAttr(ObjectStore* store,
 
   std::cout << "done" << std::endl;
   do_check_fn(store);
-  AdminSocket* admin_socket = g_ceph_context->get_admin_socket();
-  ceph_assert(admin_socket);
-
-  ceph::bufferlist in, out;
-  ostringstream err;
-
-  auto r = admin_socket->execute_command(
-    { "{\"prefix\": \"bluefs stats\"}" },
-    in, err, &out);
-  if (r != 0) {
-    cerr << "failure querying: " << cpp_strerror(r) << std::endl;
-  } else {
-    std::cout << std::string(out.c_str(), out.length()) << std::endl;
-  }
+  dump_bluefs_stats();
   test_obj.shutdown();
 }
 
-TEST_P(StoreTestSpecificAUSize, SpilloverTest) {
+TEST_P(StoreTestSpecificAUSize, SpilloverLegacyTest) {
   if (string(GetParam()) != "bluestore")
     return;
   if (smr) {
@@ -10509,24 +10512,12 @@ TEST_P(StoreTestSpecificAUSize, SpilloverTest) {
       ceph_assert(bstore);
       bstore->compact();
       const PerfCounters* logger = bstore->get_bluefs_perf_counters();
-      //experimentally it was discovered that this case results in 400+MB spillover
-      //using lower 300MB threshold just to be safe enough
-      std::cout << "DB used:" << logger->get(l_bluefs_db_used_bytes) << std::endl;
-      std::cout << "SLOW used:" << logger->get(l_bluefs_slow_used_bytes) << std::endl;
-      ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 16 * 1024 * 1024);
-
-      struct store_statfs_t statfs;
-      osd_alert_list_t alerts;
-      int r = store->statfs(&statfs, &alerts);
-      ASSERT_EQ(r, 0);
-      ASSERT_EQ(alerts.count("BLUEFS_SPILLOVER"), 1);
-      std::cout << "spillover_alert:" << alerts.find("BLUEFS_SPILLOVER")->second
-                << std::endl;
+      ASSERT_GT(logger->get(l_bluefs_slow_used_bytes), 0);
     }
   );
 }
 
-TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) {
+TEST_P(StoreTestSpecificAUSize, SpilloverLegacyFixedByFitToFastTest) {
   if (string(GetParam()) != "bluestore")
     return;
   if (smr) {
@@ -10536,8 +10527,15 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) {
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
-  SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra");
-  SetVal(g_conf(), "bluestore_volume_selection_reserved", "1"); // just use non-zero to enable
+  SetVal(g_conf(), "bluestore_volume_selection_policy", "fit_to_fast");
+  // original RocksDB settings used before https://github.com/ceph/ceph/pull/47221/
+  // which enable BlueFS spillover.
+  SetVal(g_conf(), "bluestore_rocksdb_options",
+    "compression=kNoCompression,max_write_buffer_number=4,"
+    "min_write_buffer_number_to_merge=1,recycle_log_file_num=4,"
+    "write_buffer_size=268435456,writable_file_max_buffer_size=0,"
+    "compaction_readahead_size=2097152,max_background_compactions=2,"
+    "max_total_wal_size=1073741824");
 
   g_conf().apply_changes(nullptr);
 
@@ -10549,12 +10547,28 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) {
       ceph_assert(bstore);
       bstore->compact();
       const PerfCounters* logger = bstore->get_bluefs_perf_counters();
-      ASSERT_EQ(0, logger->get(l_bluefs_slow_used_bytes));
+      ASSERT_EQ(logger->get(l_bluefs_slow_used_bytes), 0);
     }
   );
 }
 
-TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) {
+void do_bluefs_write(BlueFS* _fs,
+		     const char* dirname,
+		     const char* filename,
+		     uint64_t to_write)
+{
+  BlueFS::FileWriter* h;
+  ASSERT_EQ(0, _fs->open_for_write(dirname, filename, &h, false));
+  uint64_t buf_size = 1ull << 20;
+  string buf(buf_size, 'a');
+  for (uint64_t w = 0; w < to_write; w += buf_size) {
+    h->append(buf.c_str(), buf_size);
+    _fs->fsync(h);
+  }
+  _fs->close_writer(h);
+}
+
+TEST_P(StoreTestSpecificAUSize, SpilloverTest) {
   if (string(GetParam()) != "bluestore")
     return;
   if (smr) {
@@ -10564,27 +10578,31 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) {
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
-  SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra");
-  //default 2.0 factor results in too high threshold, using less value
-  // that results in less but still present spillover.
-  SetVal(g_conf(), "bluestore_volume_selection_reserved_factor", "0.5");
+  SetVal(g_conf(), "bluestore_volume_selection_policy", "rocksdb_original");
 
   g_conf().apply_changes(nullptr);
 
   StartDeferred(65536);
-  doManySetAttr(store.get(),
-    [&](ObjectStore* _store) {
+  BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
+  ceph_assert(bstore);
+  BlueFS* fs = bstore->get_bluefs();
+  do_bluefs_write(fs, "db", "file1", 1ull << 30); // 1GB
+  do_bluefs_write(fs, "db.slow", "file2", 1ull << 30); // 1 GB
 
-      BlueStore* bstore = dynamic_cast<BlueStore*> (_store);
-      ceph_assert(bstore);
-      bstore->compact();
-      const PerfCounters* logger = bstore->get_bluefs_perf_counters();
-      ASSERT_LE(logger->get(l_bluefs_slow_used_bytes), 300 * 1024 * 1024); // see SpilloverTest for 300MB choice rationale
-    }
-  );
+  dump_bluefs_stats();
+  const PerfCounters* logger = bstore->get_bluefs_perf_counters();
+  ASSERT_EQ(1ull << 30, logger->get(l_bluefs_slow_used_bytes));
+
+  struct store_statfs_t statfs;
+  osd_alert_list_t alerts;
+  int r = store->statfs(&statfs, &alerts);
+  ASSERT_EQ(r, 0);
+  ASSERT_EQ(alerts.count("BLUEFS_SPILLOVER"), 1);
+  std::cout << "spillover_alert:" << alerts.find("BLUEFS_SPILLOVER")->second
+    << std::endl;
 }
 
-TEST_P(StoreTestSpecificAUSize, SpilloverFixed3Test) {
+TEST_P(StoreTestSpecificAUSize, SpilloverFixedCompletelyTest) {
   if (string(GetParam()) != "bluestore")
     return;
   if (smr) {
@@ -10594,21 +10612,60 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixed3Test) {
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
-  SetVal(g_conf(), "bluestore_volume_selection_policy", "fit_to_fast");
+  SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra");
+  SetVal(g_conf(), "bluestore_volume_selection_reserved", "1"); // just use non-zero to enable
 
   g_conf().apply_changes(nullptr);
 
   StartDeferred(65536);
-  doManySetAttr(store.get(),
-    [&](ObjectStore* _store) {
+  BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
+  ceph_assert(bstore);
+  BlueFS* fs = bstore->get_bluefs();
+  do_bluefs_write(fs, "db", "file1", 1ull << 30); // 1GB
+  do_bluefs_write(fs, "db.slow", "file2", 1ull << 30); // 1 GB
 
-      BlueStore* bstore = dynamic_cast<BlueStore*> (_store);
-      ceph_assert(bstore);
-      bstore->compact();
-      const PerfCounters* logger = bstore->get_bluefs_perf_counters();
-      ASSERT_EQ(logger->get(l_bluefs_slow_used_bytes), 0); // reffering to SpilloverFixedTest
-    }
-  );
+  dump_bluefs_stats();
+  const PerfCounters* logger = bstore->get_bluefs_perf_counters();
+  ASSERT_EQ(0, logger->get(l_bluefs_slow_used_bytes));
+}
+
+TEST_P(StoreTestSpecificAUSize, SpilloverFixedPartialTest) {
+  if (string(GetParam()) != "bluestore")
+    return;
+  if (smr) {
+    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
+    return;
+  }
+
+  SetVal(g_conf(), "bluestore_block_db_create", "true");
+  SetVal(g_conf(), "bluestore_block_db_size", stringify(3ull << 30).c_str());
+  SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra");
+  //default 2.0 factor results in too high threshold, using less value
+  // that results in a reduced but existing spillover.
+  //
+  SetVal(g_conf(), "bluestore_volume_selection_reserved_factor", "1");
+
+  g_conf().apply_changes(nullptr);
+
+  StartDeferred(65536);
+  BlueStore* bstore = dynamic_cast<BlueStore*> (store.get());
+  ceph_assert(bstore);
+  BlueFS* fs = bstore->get_bluefs();
+  do_bluefs_write(fs, "db", "file1", 1ull << 30); // 1 GB
+  do_bluefs_write(fs, "db.slow", "file2", 1ull << 30); // 1 GB
+
+  dump_bluefs_stats();
+  const PerfCounters* logger = bstore->get_bluefs_perf_counters();
+  ASSERT_LT(100ull << 20, logger->get(l_bluefs_slow_used_bytes));
+  ASSERT_GT(1ull << 30, logger->get(l_bluefs_slow_used_bytes));
+
+  struct store_statfs_t statfs;
+  osd_alert_list_t alerts;
+  int r = store->statfs(&statfs, &alerts);
+  ASSERT_EQ(r, 0);
+  ASSERT_EQ(alerts.count("BLUEFS_SPILLOVER"), 1);
+  std::cout << "spillover_alert:" << alerts.find("BLUEFS_SPILLOVER")->second
+    << std::endl;
 }
 
 TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) {
@@ -10783,19 +10840,7 @@ TEST_P(StoreTestSpecificAUSize, BluefsWriteInSingleDiskEnvTest) {
   bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul);
   bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul);
 
-  AdminSocket* admin_socket = g_ceph_context->get_admin_socket();
-  ceph_assert(admin_socket);
-
-  ceph::bufferlist in, out;
-  ostringstream err;
-  auto r = admin_socket->execute_command(
-    { "{\"prefix\": \"bluefs stats\"}" },
-    in, err, &out);
-  if (r != 0) {
-    cerr << "failure querying: " << cpp_strerror(r) << std::endl;
-  }  else {
-    std::cout << std::string(out.c_str(), out.length()) << std::endl;
-  }
+  dump_bluefs_stats();
 }
 
 TEST_P(StoreTestSpecificAUSize, BluefsWriteInNoWalDiskEnvTest) {
@@ -10816,20 +10861,7 @@ TEST_P(StoreTestSpecificAUSize, BluefsWriteInNoWalDiskEnvTest) {
   bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul);
   bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul);
 
-  AdminSocket* admin_socket = g_ceph_context->get_admin_socket();
-  ceph_assert(admin_socket);
-
-  ceph::bufferlist in, out;
-  ostringstream err;
-  auto r = admin_socket->execute_command(
-    { "{\"prefix\": \"bluefs stats\"}" },
-    in, err, &out);
-  if (r != 0) {
-    cerr << "failure querying: " << cpp_strerror(r) << std::endl;
-  }
-  else {
-    std::cout << std::string(out.c_str(), out.length()) << std::endl;
-  }
+  dump_bluefs_stats();
 }
 
 TEST_P(StoreTestOmapUpgrade, NoOmapHeader) {
@@ -11005,6 +11037,8 @@ int main(int argc, char **argv) {
   g_ceph_context->_conf.set_val_or_die("bluestore_debug_randomize_serial_transaction",
 				 "10");
 
+  g_ceph_context->_conf.set_val_or_die("bluefs_check_volume_selector_on_umount", "true");
+
   g_ceph_context->_conf.set_val_or_die("bdev_debug_aio", "true");
 
   // specify device size

From 243d21ef1c18d561b35acf309e8e1c13b0781658 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 9 Nov 2023 00:22:28 +0300
Subject: [PATCH 0575/2492] os/bluestore: provide some reference for
 BlueFSVolumeSelector's interface.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueFS.h | 75 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index 38a0bf1a601c..759e71fdf05e 100644
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -88,27 +88,102 @@ class BlueFSVolumeSelector {
 
   virtual ~BlueFSVolumeSelector() {
   }
+  /**
+  *  Method to learn a hint (aka logic level discriminator)  specific for
+  *  BlueFS log
+  *
+  */
   virtual void* get_hint_for_log() const = 0;
+  /**
+  *  Method to learn a hint (aka logic level discriminator) provided directory
+  *  bound to.
+  *
+  */
   virtual void* get_hint_by_dir(std::string_view dirname) const = 0;
 
+  /**
+  *  Increments stats for a given logical level using provided fnode as a delta,
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    fnode: fnode metadata to be used as a complex delta value:
+  *           (+1 file count, +file size, +all the extents)
+  *
+  */
   void add_usage(void* hint, const bluefs_fnode_t& fnode) {
     for (auto& e : fnode.extents) {
       add_usage(hint, e);
     }
     add_usage(hint, fnode.size, true);
   }
+  /**
+  *  Decrements stats for a given logical level using provided fnode as a delta
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    fnode: fnode metadata to be used as a complex delta value:
+  *           (-1 file count, -file size, -all the extents)
+  *
+  */
   void sub_usage(void* hint, const bluefs_fnode_t& fnode) {
     for (auto& e : fnode.extents) {
       sub_usage(hint, e);
     }
     sub_usage(hint, fnode.size, true);
   }
+  /**
+  *  Increments stats for a given logical level using provided extent as a delta,
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    extent: bluefs extent to be used as a complex delta value:
+  *           (.bdev determines physical location, +length)
+  *
+  */
   virtual void add_usage(void* hint, const bluefs_extent_t& extent) = 0;
+  /**
+  *  Decrements stats for a given logical level using provided extent as a delta,
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    extent: bluefs extent to be used as a complex delta value:
+  *           (.bdev determines physical location, -length)
+  *
+  */
   virtual void sub_usage(void* hint, const bluefs_extent_t& extent) = 0;
+  /**
+  *  Increments files count and overall files size for a given logical level
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    fsize: delta value for file size
+  *    upd_files: whether or not to increment file count
+  *
+  */
   virtual void add_usage(void* hint, uint64_t fsize, bool upd_files = false) = 0;
+  /**
+  *  Decrements files count and overall files size for a given logical level
+  *  Parameters:
+  *    hint: logical level discriminator
+  *    fsize: delta value for file size
+  *    upd_files: whether or not to decrement file count
+  *
+  */
   virtual void sub_usage(void* hint, uint64_t fsize, bool upd_files = false) = 0;
+
+  /**
+  *  Determines preferred physical device for the given logical level
+  *  Parameters:
+  *    hint: logical level discriminator
+  *
+  */
   virtual uint8_t select_prefer_bdev(void* hint) = 0;
+  /**
+  *  Builds path set for RocksDB to use
+  *  Parameters:
+  *    base: path's root
+  *
+  */
   virtual void get_paths(const std::string& base, paths& res) const = 0;
+  /**
+  *  Dumps VSelector's state
+  *
+  */
   virtual void dump(std::ostream& sout) = 0;
 
   /* used for sanity checking of vselector */

From 4ea87307e7af088dc056c1c7785aef7ca9be78b1 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 14 Nov 2023 15:28:45 +0000
Subject: [PATCH 0576/2492] doc/rgw/notifications: specify which event types
 are enabled by default

Fixes: https://tracker.ceph.com/issues/63530

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 doc/radosgw/notifications.rst | 2 +-
 doc/radosgw/s3/bucketops.rst  | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
index a38ad9e01bdf..314e0ba6f167 100644
--- a/doc/radosgw/notifications.rst
+++ b/doc/radosgw/notifications.rst
@@ -16,7 +16,7 @@ with buckets it owns.
 
 A notification entity must be created in order to send event notifications for
 a specific bucket. A notification entity can be created either for a subset
-of event types or for all event types (which is the default). The
+of event types or for all "Removed" and "Created" event types (which is the default). The
 notification may also filter out events based on matches of the prefixes and
 suffixes of (1) the keys, (2) the metadata attributes attached to the object,
 or (3) the object tags. Regular-expression matching can also be used on these
diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst
index 17da3a9351d9..984733fff750 100644
--- a/doc/radosgw/s3/bucketops.rst
+++ b/doc/radosgw/s3/bucketops.rst
@@ -537,7 +537,8 @@ Parameters are XML encoded in the body of the request, in the following format:
 | ``Topic``                     | String    | Topic ARN. Topic must be created beforehand                                          | Yes      |
 +-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
 | ``Event``                     | String    | List of supported events see: `S3 Notification Compatibility`_.  Multiple ``Event``  | No       |
-|                               |           | entities can be used. If omitted, all events are handled                             |          |
+|                               |           | entities can be used. If omitted, all "Created" and "Removed" events are handled.    |          |
+|                               |           | "Lifecycle" and "Synced" event types must be specified explicitly.                   |          |
 +-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
 | ``Filter``                    | Container | Holding ``S3Key``, ``S3Metadata`` and ``S3Tags`` entities                            | No       |
 +-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+

From fa8129a7466b3b48b56124eaf1d9ca8e8fd51fbc Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 14 Nov 2023 23:03:28 +1000
Subject: [PATCH 0577/2492] doc/rados: edit "troubleshooting-mon"

Edit the text in the "Initial Troubleshooting" section of
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 71 ++++++++++---------
 1 file changed, 39 insertions(+), 32 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index ad745a3a73de..1170da7c33f6 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -17,59 +17,66 @@ consult the following troubleshooting information.
 Initial Troubleshooting
 =======================
 
-#. **Make sure that the monitors are running.**
+The first steps in the process of troubleshooting Ceph Monitors involve making
+sure that the Monitors are running and that they are able to communicate with
+the network and on the network. Follow the steps in this section to rule out
+the simplest causes of Monitor malfunction.
 
-    First, make sure that the monitor (*mon*) daemon processes (``ceph-mon``)
-    are running. Sometimes Ceph admins either forget to start the mons or
-    forget to restart the mons after an upgrade. Checking for this simple
-    oversight can save hours of painstaking troubleshooting. It is also
-    important to make sure that the manager daemons (``ceph-mgr``) are running.
-    Remember that typical cluster configurations provide one ``ceph-mgr`` for
-    each ``ceph-mon``.
+#. **Make sure that the Monitors are running.**
 
-    .. note:: Rook will not run more than two managers.
+    Make sure that the Monitor (*mon*) daemon processes (``ceph-mon``) are
+    running. It might be the case that the mons have not be restarted after an
+    upgrade. Checking for this simple oversight can save hours of painstaking
+    troubleshooting. 
+    
+    It is also important to make sure that the manager daemons (``ceph-mgr``)
+    are running. Remember that typical cluster configurations provide one
+    Manager (``ceph-mgr``) for each Monitor (``ceph-mon``).
 
-#. **Make sure that you can reach the monitor nodes.**
+    .. note:: In releases prior to v1.12.5, Rook will not run more than two
+       managers.
 
-    In certain rare cases, there may be ``iptables`` rules that block access to
-    monitor nodes or TCP ports. These rules might be left over from earlier
-    stress testing or rule development. To check for the presence of such
-    rules, SSH into the server and then try to connect to the monitor's ports
-    (``tcp/3300`` and ``tcp/6789``) using ``telnet``, ``nc``, or a similar
-    tool.
+#. **Make sure that you can reach the Monitor nodes.**
 
-#. **Make sure that the ``ceph status`` command runs  and receives a reply from the cluster.**
+    In certain rare cases, ``iptables`` rules might be blocking access to
+    Monitor nodes or TCP ports. These rules might be left over from earlier
+    stress testing or rule development. To check for the presence of such
+    rules, SSH into each Monitor node and use ``telnet`` or ``nc`` or a similar
+    tool to attempt to connect to each of the other Monitor nodes on ports
+    ``tcp/3300`` and ``tcp/6789``. 
 
-    If the ``ceph status`` command does receive a reply from the cluster, then
-    the cluster is up and running. The monitors will answer to a ``status``
-    request only if there is a formed quorum. Confirm that one or more ``mgr``
-    daemons are reported as running. Under ideal conditions, all ``mgr``
-    daemons will be reported as running.
+#. **Make sure that the "ceph status" command runs and receives a reply from the cluster.**
 
+    If the ``ceph status`` command receives a reply from the cluster, then the
+    cluster is up and running. Monitors answer to a ``status`` request only if
+    there is a formed quorum. Confirm that one or more ``mgr`` daemons are
+    reported as running. In a cluster with no deficiencies, ``ceph status``
+    will report that all ``mgr`` daemons are running.
 
     If the ``ceph status`` command does not receive a reply from the cluster,
-    then there are probably not enough monitors ``up`` to form a quorum.  The
-    ``ceph -s`` command with no further options specified connects to an
-    arbitrarily selected monitor. In certain cases, however, it might be
-    helpful to connect to a specific monitor (or to several specific monitors
+    then there are probably not enough Monitors ``up`` to form a quorum. If the
+    ``ceph -s`` command is run with no further options specified, it connects
+    to an arbitrarily selected Monitor. In certain cases, however, it might be
+    helpful to connect to a specific Monitor (or to several specific Monitors
     in sequence) by adding the ``-m`` flag to the command: for example, ``ceph
     status -m mymon1``.
 
 #. **None of this worked. What now?**
 
     If the above solutions have not resolved your problems, you might find it
-    helpful to examine each individual monitor in turn. Whether or not a quorum
-    has been formed, it is possible to contact each monitor individually and
+    helpful to examine each individual Monitor in turn. Even if no quorum has
+    been formed, it is possible to contact each Monitor individually and
     request its status by using the ``ceph tell mon.ID mon_status`` command
-    (here ``ID`` is the monitor's identifier).
+    (here ``ID`` is the Monitor's identifier).
 
-    Run the ``ceph tell mon.ID mon_status`` command for each monitor in the
+    Run the ``ceph tell mon.ID mon_status`` command for each Monitor in the
     cluster. For more on this command's output, see :ref:`Understanding
     mon_status
     <rados_troubleshoting_troubleshooting_mon_understanding_mon_status>`.
 
-    There is also an alternative method: SSH into each monitor node and query
-    the daemon's admin socket. See :ref:`Using the Monitor's Admin
+    There is also an alternative method for contacting each individual Monitor:
+    SSH into each Monitor node and query the daemon's admin socket. See
+    :ref:`Using the Monitor's Admin
     Socket<rados_troubleshoting_troubleshooting_mon_using_admin_socket>`.
 
 .. _rados_troubleshoting_troubleshooting_mon_using_admin_socket:

From 3797fb51add7dcfe3ff4cfca1b7af5e3d2952eea Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 6 Nov 2023 13:34:14 -0500
Subject: [PATCH 0578/2492] cephadm: move logrotate config to jinja2 template

This moves both the cluster and cephadm logrotate
configs into jinja2 templates. It looks a bit silly
right now to have the cephadm one in a template given
it has no variables, but it may allow us to implement
custom cephadm log logrotate configs later down
the line

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py                        | 44 ++-----------
 src/cephadm/cephadmlib/logging.py             | 65 ++++++++++++++-----
 .../templates/cephadm.logrotate.config.j2     |  9 +++
 .../templates/cluster.logrotate.config.j2     | 13 ++++
 src/cephadm/cephadmlib/templating.py          |  2 +
 src/cephadm/tests/test_logrotate_config.py    | 57 ++++++++++++++++
 6 files changed, 138 insertions(+), 52 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/templates/cephadm.logrotate.config.j2
 create mode 100644 src/cephadm/cephadmlib/templates/cluster.logrotate.config.j2
 create mode 100644 src/cephadm/tests/test_logrotate_config.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c9bc3a787579..50ed3d1f00fa 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -147,7 +147,12 @@
 from cephadmlib.locking import FileLock
 from cephadmlib.daemon_identity import DaemonIdentity, DaemonSubIdentity
 from cephadmlib.packagers import create_packager, Packager
-from cephadmlib.logging import cephadm_init_logging, Highlight, LogDestination
+from cephadmlib.logging import (
+    cephadm_init_logging,
+    Highlight,
+    LogDestination,
+    write_cluster_logrotate_config,
+)
 from cephadmlib.systemd import check_unit, check_units
 from cephadmlib.container_types import (
     CephContainer,
@@ -3419,42 +3424,7 @@ def install_base_units(ctx, fsid):
     if os.path.exists(ctx.logrotate_dir + f'/ceph-{fsid}'):
         return
 
-    # logrotate for the cluster
-    with write_new(ctx.logrotate_dir + f'/ceph-{fsid}', perms=None) as f:
-        """
-        This is a bit sloppy in that the killall/pkill will touch all ceph daemons
-        in all containers, but I don't see an elegant way to send SIGHUP *just* to
-        the daemons for this cluster.  (1) systemd kill -s will get the signal to
-        podman, but podman will exit.  (2) podman kill will get the signal to the
-        first child (bash), but that isn't the ceph daemon.  This is simpler and
-        should be harmless.
-        """
-        targets: List[str] = [
-            'ceph-mon',
-            'ceph-mgr',
-            'ceph-mds',
-            'ceph-osd',
-            'ceph-fuse',
-            'radosgw',
-            'rbd-mirror',
-            'cephfs-mirror',
-            'tcmu-runner'
-        ]
-
-        f.write("""# created by cephadm
-/var/log/ceph/%s/*.log {
-    rotate 7
-    daily
-    compress
-    sharedscripts
-    postrotate
-        killall -q -1 %s || pkill -1 -x '%s' || true
-    endscript
-    missingok
-    notifempty
-    su root root
-}
-""" % (fsid, ' '.join(targets), '|'.join(targets)))
+    write_cluster_logrotate_config(ctx, fsid)
 
 
 def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
diff --git a/src/cephadm/cephadmlib/logging.py b/src/cephadm/cephadmlib/logging.py
index 5e306484b98d..f5893d3a51d1 100644
--- a/src/cephadm/cephadmlib/logging.py
+++ b/src/cephadm/cephadmlib/logging.py
@@ -12,6 +12,10 @@
 from .context import CephadmContext
 from .constants import QUIET_LOG_LEVEL, LOG_DIR
 
+from cephadmlib.file_utils import write_new
+
+from cephadmlib import templating
+
 
 class _ExcludeErrorsFilter(logging.Filter):
     def filter(self, record: logging.LogRecord) -> bool:
@@ -145,18 +149,6 @@ def format(self, record: Any) -> str:
 }
 
 
-_logrotate_data = """# created by cephadm
-/var/log/ceph/cephadm.log {
-    rotate 7
-    daily
-    compress
-    missingok
-    notifempty
-    su root root
-}
-"""
-
-
 _VERBOSE_HANDLERS = [
     'console',
     'console_stdout',
@@ -222,9 +214,7 @@ def cephadm_init_logging(
 
     logger.setLevel(QUIET_LOG_LEVEL)
 
-    if not os.path.exists(ctx.logrotate_dir + '/cephadm'):
-        with open(ctx.logrotate_dir + '/cephadm', 'w') as f:
-            f.write(_logrotate_data)
+    write_cephadm_logrotate_config(ctx)
 
     for handler in logger.handlers:
         # the following little hack ensures that no matter how cephadm is named
@@ -239,3 +229,48 @@ def cephadm_init_logging(
         if ctx.verbose and handler.name in _VERBOSE_HANDLERS:
             handler.setLevel(QUIET_LOG_LEVEL)
     logger.debug('%s\ncephadm %s' % ('-' * 80, args))
+
+
+def write_cephadm_logrotate_config(ctx: CephadmContext) -> None:
+    if not os.path.exists(ctx.logrotate_dir + '/cephadm'):
+        with open(ctx.logrotate_dir + '/cephadm', 'w') as f:
+            cephadm_logrotate_config = templating.render(
+                ctx, templating.Templates.cephadm_logrotate_config
+            )
+            f.write(cephadm_logrotate_config)
+
+
+def write_cluster_logrotate_config(ctx: CephadmContext, fsid: str) -> None:
+    # logrotate for the cluster
+    with write_new(ctx.logrotate_dir + f'/ceph-{fsid}', perms=None) as f:
+        """
+        See cephadm/cephadmlib/templates/cluster.logrotate.config.j2 to
+        get a better idea what this comment is referring to
+
+        This is a bit sloppy in that the killall/pkill will touch all ceph daemons
+        in all containers, but I don't see an elegant way to send SIGHUP *just* to
+        the daemons for this cluster.  (1) systemd kill -s will get the signal to
+        podman, but podman will exit.  (2) podman kill will get the signal to the
+        first child (bash), but that isn't the ceph daemon.  This is simpler and
+        should be harmless.
+        """
+        targets: List[str] = [
+            'ceph-mon',
+            'ceph-mgr',
+            'ceph-mds',
+            'ceph-osd',
+            'ceph-fuse',
+            'radosgw',
+            'rbd-mirror',
+            'cephfs-mirror',
+            'tcmu-runner',
+        ]
+
+        logrotate_config = templating.render(
+            ctx,
+            templating.Templates.cluster_logrotate_config,
+            fsid=fsid,
+            targets=targets,
+        )
+
+        f.write(logrotate_config)
diff --git a/src/cephadm/cephadmlib/templates/cephadm.logrotate.config.j2 b/src/cephadm/cephadmlib/templates/cephadm.logrotate.config.j2
new file mode 100644
index 000000000000..b18aaff2196e
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/cephadm.logrotate.config.j2
@@ -0,0 +1,9 @@
+# created by cephadm
+/var/log/ceph/cephadm.log {
+    rotate 7
+    daily
+    compress
+    missingok
+    notifempty
+    su root root
+}
diff --git a/src/cephadm/cephadmlib/templates/cluster.logrotate.config.j2 b/src/cephadm/cephadmlib/templates/cluster.logrotate.config.j2
new file mode 100644
index 000000000000..9af2f955d905
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/cluster.logrotate.config.j2
@@ -0,0 +1,13 @@
+# created by cephadm
+/var/log/ceph/{{ fsid }}/*.log {
+    rotate 7
+    daily
+    compress
+    sharedscripts
+    postrotate
+        killall -q -1 {{ targets|join(' ') }} || pkill -1 -x '{{ targets|join('|') }}' || true
+    endscript
+    missingok
+    notifempty
+    su root root
+}
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 3b7c6f9657e0..1f160a07b8e8 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -17,6 +17,8 @@ class Templates(str, enum.Enum):
 
     ceph_service = 'ceph.service.j2'
     agent_service = 'agent.service.j2'
+    cluster_logrotate_config = 'cluster.logrotate.config.j2'
+    cephadm_logrotate_config = 'cephadm.logrotate.config.j2'
 
     def __str__(self) -> str:
         return self.value
diff --git a/src/cephadm/tests/test_logrotate_config.py b/src/cephadm/tests/test_logrotate_config.py
new file mode 100644
index 000000000000..c97f21019d86
--- /dev/null
+++ b/src/cephadm/tests/test_logrotate_config.py
@@ -0,0 +1,57 @@
+from unittest import mock
+
+import pytest
+
+from tests.fixtures import import_cephadm, cephadm_fs
+
+from cephadmlib import logging
+
+
+_cephadm = import_cephadm()
+
+def test_cluster_logrotate_config(cephadm_fs):
+    ctx = _cephadm.CephadmContext()
+    ctx.logrotate_dir = '/my/log/dir'
+    fsid = '5dcc9af0-7cd3-11ee-9e84-525400babd0a'
+
+    cephadm_fs.create_dir(ctx.logrotate_dir)
+
+    expected_cluster_logrotate_file = """# created by cephadm
+/var/log/ceph/5dcc9af0-7cd3-11ee-9e84-525400babd0a/*.log {
+    rotate 7
+    daily
+    compress
+    sharedscripts
+    postrotate
+        killall -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror cephfs-mirror tcmu-runner || pkill -1 -x 'ceph-mon|ceph-mgr|ceph-mds|ceph-osd|ceph-fuse|radosgw|rbd-mirror|cephfs-mirror|tcmu-runner' || true
+    endscript
+    missingok
+    notifempty
+    su root root
+}"""
+
+    logging.write_cluster_logrotate_config(ctx, fsid)
+
+    with open(ctx.logrotate_dir + f'/ceph-{fsid}', 'r') as f:
+        assert f.read() == expected_cluster_logrotate_file
+
+def test_cephadm_logrotate_config(cephadm_fs):
+    ctx = _cephadm.CephadmContext()
+    ctx.logrotate_dir = '/my/log/dir'
+
+    cephadm_fs.create_dir(ctx.logrotate_dir)
+
+    expected_cephadm_logrotate_file = """# created by cephadm
+/var/log/ceph/cephadm.log {
+    rotate 7
+    daily
+    compress
+    missingok
+    notifempty
+    su root root
+}"""
+
+    logging.write_cephadm_logrotate_config(ctx)
+
+    with open(ctx.logrotate_dir + f'/cephadm', 'r') as f:
+        assert f.read() == expected_cephadm_logrotate_file

From f27790b057db667c48b1840472046db3d6d9c5f1 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 3 Oct 2023 19:06:10 -0400
Subject: [PATCH 0579/2492] mgr/cephadm: support for regex based host patterns

For example, with hosts vm-00, vm-01, and vm-02
I was able to provide the placement

service_type: node-exporter
service_name: node-exporter
placement:
  host_pattern:
    pattern: vm-00|vm-02
    pattern_type: regex

and it placed the node-exporter daemons on vm-00
and vm-02 but not vm-01. Obviously there are more
advanced scenarios that justify this than listing
two hosts, but using "|" as an OR like that is an
example of something you can't do with the fnmatch
version of the host pattern

Signed-off-by: Adam King <adking@redhat.com>
---
 .../mgr/cephadm/tests/test_scheduling.py      |  47 +++++-
 src/pybind/mgr/rook/rook_cluster.py           |  11 +-
 .../ceph/deployment/service_spec.py           | 147 ++++++++++++++++--
 .../ceph/tests/test_service_spec.py           |  20 ++-
 4 files changed, 203 insertions(+), 22 deletions(-)

diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py
index 067cd5028a2c..b307cd9d34dd 100644
--- a/src/pybind/mgr/cephadm/tests/test_scheduling.py
+++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py
@@ -6,7 +6,13 @@
 import pytest
 
 from ceph.deployment.hostspec import HostSpec
-from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
+from ceph.deployment.service_spec import (
+    ServiceSpec,
+    PlacementSpec,
+    IngressSpec,
+    PatternType,
+    HostPattern,
+)
 from ceph.deployment.hostspec import SpecValidationError
 
 from cephadm.module import HostAssignment
@@ -1697,3 +1703,42 @@ def test_drain_from_explict_placement(service_type, placement, hosts, maintenanc
     ).place()
     assert sorted([h.hostname for h in to_add]) in expected_add
     assert sorted([h.name() for h in to_remove]) in expected_remove
+
+
+class RegexHostPatternTest(NamedTuple):
+    service_type: str
+    placement: PlacementSpec
+    hosts: List[str]
+    expected_add: List[List[str]]
+
+
+@pytest.mark.parametrize("service_type,placement,hosts,expected_add",
+                         [
+                             RegexHostPatternTest(
+                                 'crash',
+                                 PlacementSpec(host_pattern=HostPattern(pattern='host1|host3', pattern_type=PatternType.regex)),
+                                 'host1 host2 host3 host4'.split(),
+                                 ['host1', 'host3'],
+                             ),
+                             RegexHostPatternTest(
+                                 'crash',
+                                 PlacementSpec(host_pattern=HostPattern(pattern='host[2-4]', pattern_type=PatternType.regex)),
+                                 'host1 host2 host3 host4'.split(),
+                                 ['host2', 'host3', 'host4'],
+                             ),
+                         ])
+def test_placement_regex_host_pattern(service_type, placement, hosts, expected_add):
+    spec = ServiceSpec(service_type=service_type,
+                       service_id='test',
+                       placement=placement)
+
+    host_specs = [HostSpec(h) for h in hosts]
+
+    hosts, to_add, to_remove = HostAssignment(
+        spec=spec,
+        hosts=host_specs,
+        unreachable_hosts=[],
+        draining_hosts=[],
+        daemons=[],
+    ).place()
+    assert sorted([h.hostname for h in to_add]) == expected_add
diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index 5c7c9fc0477f..70581e6051da 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -24,7 +24,14 @@
 
 from ceph.deployment.inventory import Device
 from ceph.deployment.drive_group import DriveGroupSpec
-from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, PlacementSpec, HostPlacementSpec
+from ceph.deployment.service_spec import (
+    ServiceSpec,
+    NFSServiceSpec,
+    RGWSpec,
+    PlacementSpec,
+    HostPlacementSpec,
+    HostPattern,
+)
 from ceph.utils import datetime_now
 from ceph.deployment.drive_selection.matchers import SizeMatcher
 from nfs.cluster import create_ganesha_pool
@@ -1585,7 +1592,7 @@ def node_selector_to_placement_spec(node_selector: ccl.NodeSelectorTermsItem) ->
             res.label = expression.key.split('/')[1]
         elif expression.key == "kubernetes.io/hostname":
             if expression.operator == "Exists":
-                res.host_pattern = "*"
+                res.host_pattern = HostPattern("*")
             elif expression.operator == "In": 
                 res.hosts = [HostPlacementSpec(hostname=value, network='', name='')for value in expression.values]
     return res
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 4181ee2563e4..28f9a51a8a52 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -140,17 +140,120 @@ def validate(self) -> None:
         assert_valid_host(self.hostname)
 
 
+HostPatternType = Union[str, None, Dict[str, Union[str, bool, None]], "HostPattern"]
+
+
+class PatternType(enum.Enum):
+    fnmatch = 'fnmatch'
+    regex = 'regex'
+
+
+class HostPattern():
+    def __init__(self,
+                 pattern: Optional[str] = None,
+                 pattern_type: PatternType = PatternType.fnmatch) -> None:
+        self.pattern: Optional[str] = pattern
+        self.pattern_type: PatternType = pattern_type
+        self.compiled_regex = None
+        if self.pattern_type == PatternType.regex and self.pattern:
+            self.compiled_regex = re.compile(self.pattern)
+
+    def filter_hosts(self, hosts: List[str]) -> List[str]:
+        if not self.pattern:
+            return []
+        if not self.pattern_type or self.pattern_type == PatternType.fnmatch:
+            return fnmatch.filter(hosts, self.pattern)
+        elif self.pattern_type == PatternType.regex:
+            if not self.compiled_regex:
+                self.compiled_regex = re.compile(self.pattern)
+            return [h for h in hosts if re.match(self.compiled_regex, h)]
+        raise SpecValidationError(f'Got unexpected pattern_type: {self.pattern_type}')
+
+    @classmethod
+    def to_host_pattern(cls, arg: HostPatternType) -> "HostPattern":
+        if arg is None:
+            return cls()
+        elif isinstance(arg, str):
+            return cls(arg)
+        elif isinstance(arg, cls):
+            return arg
+        elif isinstance(arg, dict):
+            if 'pattern' not in arg:
+                raise SpecValidationError("Got dict for host pattern "
+                                          f"with no pattern field: {arg}")
+            pattern = arg['pattern']
+            if not pattern:
+                raise SpecValidationError("Got dict for host pattern"
+                                          f"with empty pattern: {arg}")
+            assert isinstance(pattern, str)
+            if 'pattern_type' in arg:
+                pattern_type = arg['pattern_type']
+                if not pattern_type or pattern_type == 'fnmatch':
+                    return cls(pattern, pattern_type=PatternType.fnmatch)
+                elif pattern_type == 'regex':
+                    return cls(pattern, pattern_type=PatternType.regex)
+                else:
+                    raise SpecValidationError("Got dict for host pattern "
+                                              f"with unknown pattern type: {arg}")
+            return cls(pattern)
+        raise SpecValidationError(f"Cannot convert {type(arg)} object to HostPattern")
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            other_hp = self.to_host_pattern(other)
+        except SpecValidationError:
+            return False
+        return self.pattern == other_hp.pattern and self.pattern_type == other_hp.pattern_type
+
+    def pretty_str(self) -> str:
+        # Placement specs must be able to be converted between the Python object
+        # representation and a pretty str both ways. So we need a corresponding
+        # function for HostPattern to convert it to a pretty str that we can
+        # convert back later.
+        res = self.pattern if self.pattern else ''
+        if self.pattern_type == PatternType.regex:
+            res = 'regex:' + res
+        return res
+
+    @classmethod
+    def from_pretty_str(cls, val: str) -> "HostPattern":
+        if 'regex:' in val:
+            return cls(val[6:], pattern_type=PatternType.regex)
+        else:
+            return cls(val)
+
+    def __repr__(self) -> str:
+        return f'HostPattern(pattern=\'{self.pattern}\', pattern_type={str(self.pattern_type)})'
+
+    def to_json(self) -> Union[str, Dict[str, Any], None]:
+        if self.pattern_type and self.pattern_type != PatternType.fnmatch:
+            return {
+                'pattern': self.pattern,
+                'pattern_type': self.pattern_type.name
+            }
+        return self.pattern
+
+    @classmethod
+    def from_json(self, val: Dict[str, Any]) -> "HostPattern":
+        return self.to_host_pattern(val)
+
+    def __bool__(self) -> bool:
+        if self.pattern:
+            return True
+        return False
+
+
 class PlacementSpec(object):
     """
     For APIs that need to specify a host subset
     """
 
     def __init__(self,
-                 label=None,  # type: Optional[str]
-                 hosts=None,  # type: Union[List[str],List[HostPlacementSpec], None]
-                 count=None,  # type: Optional[int]
-                 count_per_host=None,  # type: Optional[int]
-                 host_pattern=None,  # type: Optional[str]
+                 label: Optional[str] = None,
+                 hosts: Union[List[str], List[HostPlacementSpec], None] = None,
+                 count: Optional[int] = None,
+                 count_per_host: Optional[int] = None,
+                 host_pattern: HostPatternType = None,
                  ):
         # type: (...) -> None
         self.label = label
@@ -163,7 +266,7 @@ def __init__(self,
         self.count_per_host = count_per_host   # type: Optional[int]
 
         #: fnmatch patterns to select hosts. Can also be a single host.
-        self.host_pattern = host_pattern  # type: Optional[str]
+        self.host_pattern: HostPattern = HostPattern.to_host_pattern(host_pattern)
 
         self.validate()
 
@@ -206,7 +309,7 @@ def filter_matching_hostspecs(self, hostspecs: Iterable[HostSpec]) -> List[str]:
             return [hs.hostname for hs in hostspecs if self.label in hs.labels]
         all_hosts = [hs.hostname for hs in hostspecs]
         if self.host_pattern:
-            return fnmatch.filter(all_hosts, self.host_pattern)
+            return self.host_pattern.filter_hosts(all_hosts)
         return all_hosts
 
     def get_target_count(self, hostspecs: Iterable[HostSpec]) -> int:
@@ -230,7 +333,7 @@ def pretty_str(self) -> str:
         if self.label:
             kv.append('label:%s' % self.label)
         if self.host_pattern:
-            kv.append(self.host_pattern)
+            kv.append(self.host_pattern.pretty_str())
         return ';'.join(kv)
 
     def __repr__(self) -> str:
@@ -271,7 +374,7 @@ def to_json(self) -> dict:
         if self.count_per_host:
             r['count_per_host'] = self.count_per_host
         if self.host_pattern:
-            r['host_pattern'] = self.host_pattern
+            r['host_pattern'] = self.host_pattern.to_json()
         return r
 
     def validate(self) -> None:
@@ -315,8 +418,9 @@ def validate(self) -> None:
                 "count-per-host cannot be combined explicit placement with names or networks"
             )
         if self.host_pattern:
-            if not isinstance(self.host_pattern, str):
-                raise SpecValidationError('host_pattern must be of type string')
+            # if we got an invalid type for the host_pattern, it would have
+            # triggered a SpecValidationError when attemptying to convert it
+            # to a HostPattern type, so no type checking is needed here.
             if self.hosts:
                 raise SpecValidationError('cannot combine host patterns and hosts')
 
@@ -354,10 +458,17 @@ def from_string(cls, arg):
         >>> PlacementSpec.from_string('3 label:mon')
         PlacementSpec(count=3, label='mon')
 
-        fnmatch is also supported:
+        You can specify a regex to match with `regex:<regex>`
+
+        >>> PlacementSpec.from_string('regex:Foo[0-9]|Bar[0-9]')
+        PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', \
+pattern_type=PatternType.regex))
+
+        fnmatch is the default for a single string if "regex:" is not provided:
 
         >>> PlacementSpec.from_string('data[1-3]')
-        PlacementSpec(host_pattern='data[1-3]')
+        PlacementSpec(host_pattern=HostPattern(pattern='data[1-3]', \
+pattern_type=PatternType.fnmatch))
 
         >>> PlacementSpec.from_string(None)
         PlacementSpec()
@@ -407,7 +518,8 @@ def from_string(cls, arg):
 
         advanced_hostspecs = [h for h in strings if
                               (':' in h or '=' in h or not any(c in '[]?*:=' for c in h)) and
-                              'label:' not in h]
+                              'label:' not in h and
+                              'regex:' not in h]
         for a_h in advanced_hostspecs:
             strings.remove(a_h)
 
@@ -419,15 +531,20 @@ def from_string(cls, arg):
         label = labels[0][6:] if labels else None
 
         host_patterns = strings
+        host_pattern: Optional[HostPattern] = None
         if len(host_patterns) > 1:
             raise SpecValidationError(
                 'more than one host pattern provided: {}'.format(host_patterns))
+        if host_patterns:
+            # host_patterns is a list not > 1, and not empty, so we should
+            # be guaranteed just a single string here
+            host_pattern = HostPattern.from_pretty_str(host_patterns[0])
 
         ps = PlacementSpec(count=count,
                            count_per_host=count_per_host,
                            hosts=advanced_hostspecs,
                            label=label,
-                           host_pattern=host_patterns[0] if host_patterns else None)
+                           host_pattern=host_pattern)
         return ps
 
 
diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py
index 502057f5ca3b..9a55b0a81359 100644
--- a/src/python-common/ceph/tests/test_service_spec.py
+++ b/src/python-common/ceph/tests/test_service_spec.py
@@ -144,11 +144,13 @@ def test_apply_prometheus(spec: PrometheusSpec, raise_exception: bool, msg: str)
         ('2 host1 host2', "PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
         ('label:foo', "PlacementSpec(label='foo')"),
         ('3 label:foo', "PlacementSpec(count=3, label='foo')"),
-        ('*', "PlacementSpec(host_pattern='*')"),
-        ('3 data[1-3]', "PlacementSpec(count=3, host_pattern='data[1-3]')"),
-        ('3 data?', "PlacementSpec(count=3, host_pattern='data?')"),
-        ('3 data*', "PlacementSpec(count=3, host_pattern='data*')"),
+        ('*', "PlacementSpec(host_pattern=HostPattern(pattern='*', pattern_type=PatternType.fnmatch))"),
+        ('3 data[1-3]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data[1-3]', pattern_type=PatternType.fnmatch))"),
+        ('3 data?', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data?', pattern_type=PatternType.fnmatch))"),
+        ('3 data*', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='data*', pattern_type=PatternType.fnmatch))"),
         ("count-per-host:4 label:foo", "PlacementSpec(count_per_host=4, label='foo')"),
+        ('regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"),
+        ('3 regex:Foo[0-9]|Bar[0-9]', "PlacementSpec(count=3, host_pattern=HostPattern(pattern='Foo[0-9]|Bar[0-9]', pattern_type=PatternType.regex))"),
     ])
 def test_parse_placement_specs(test_input, expected):
     ret = PlacementSpec.from_string(test_input)
@@ -161,6 +163,9 @@ def test_parse_placement_specs(test_input, expected):
         ("host=a host*"),
         ("host=a label:wrong"),
         ("host? host*"),
+        ("host? regex:host*"),
+        ("regex:host? host*"),
+        ("regex:host? regex:host*"),
         ('host=a count-per-host:0'),
         ('host=a count-per-host:-10'),
         ('count:2 count-per-host:1'),
@@ -313,6 +318,13 @@ def test_osd_unmanaged():
   host_pattern: '*'
 unmanaged: true
 ---
+service_type: crash
+service_name: crash
+placement:
+  host_pattern:
+    pattern: Foo[0-9]|Bar[0-9]
+    pattern_type: regex
+---
 service_type: rgw
 service_id: default-rgw-realm.eu-central-1.1
 service_name: rgw.default-rgw-realm.eu-central-1.1

From 62bd0b9368d4a53b15d9ecb665444dd943dbcfcf Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Sun, 5 Nov 2023 16:07:53 -0500
Subject: [PATCH 0580/2492] doc/cephadm: document regex based host pattern

Signed-off-by: Adam King <adking@redhat.com>
---
 doc/cephadm/services/index.rst | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst
index 82f83bfac8e7..c1da5d15f895 100644
--- a/doc/cephadm/services/index.rst
+++ b/doc/cephadm/services/index.rst
@@ -357,7 +357,9 @@ Or in YAML:
 Placement by pattern matching
 -----------------------------
 
-Daemons can be placed on hosts as well:
+Daemons can be placed on hosts using a host pattern as well.
+By default, the host pattern is matched using fnmatch which supports
+UNIX shell-style wildcards (see https://docs.python.org/3/library/fnmatch.html):
 
    .. prompt:: bash #
 
@@ -385,6 +387,26 @@ Or in YAML:
     placement:
       host_pattern: "*"
 
+The host pattern also has support for using a regex. To use a regex, you
+must either add "regex: " to the start of the pattern when using the
+command line, or specify a ``pattern_type`` field to be "regex"
+when using YAML.
+
+On the command line:
+
+.. prompt:: bash #
+
+ ceph orch apply prometheus --placement='regex:FOO[0-9]|BAR[0-9]'
+
+In YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      host_pattern:
+        pattern: 'FOO[0-9]|BAR[0-9]'
+        pattern_type: regex
 
 Changing the number of daemons
 ------------------------------

From ad46f1c7f1466801f832e9e9567565305671ab8b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 7 Nov 2023 17:57:43 -0500
Subject: [PATCH 0581/2492] cephadm: update additional container daemon form
 base class methods

Fix the function signature for the previously added stub
customize_container_args.
Add methods for customize_process_args and customize_container_envs.
The "process args" are args passed to the daemon process / entrypoint.
Container args are passed to the container engine.
Add a default_entrypoint method to return the default entrypoint for
a particular daemon-in-container.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadmlib/container_daemon_form.py       | 28 ++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index 4da0624698bc..ae8b6afe3c63 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -55,13 +55,33 @@ def customize_container_mounts(
         """
         pass
 
-    def customize_container_args(self, args: List[str]) -> None:
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
         """Given a list of container arguments this function can update,
         delete, or otherwise mutate the arguments that the container engine
         will use.
         """
         pass
 
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        """Given a list of arguments for the containerized process, this
+        function can update, delete, or otherwise mutate the arguments that the
+        process will use.
+        """
+        pass
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        """Given a list of environment vars this function can update, delete,
+        or otherwise mutate the environment variables that are passed by the
+        container engine to the processes it executes.
+        """
+        pass
+
     def customize_container_endpoints(
         self, endpoints: List[EndPoint], deployment_type: DeploymentType
     ) -> None:
@@ -84,3 +104,9 @@ def osd_fsid(self) -> Optional[str]:
         expected to understand this.
         """
         return None
+
+    def default_entrypoint(self) -> str:
+        """Return the default entrypoint value when running a deamon process
+        in a container.
+        """
+        return ''

From a4022b0738081ca74a5889e100aed68fdd2a0e4b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 7 Nov 2023 17:57:43 -0500
Subject: [PATCH 0582/2492] cephadm: add more ContainerDeamonForm methods to
 the snmp class

Add methods customize_{container,process}_args to the snmp gateway
daemon type class. Use those methods in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c9bc3a787579..8225553c9271 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -558,6 +558,16 @@ def container(self, ctx: CephadmContext) -> CephContainer:
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return self.uid, self.gid
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(f'--env-file={self.conf_file_path}')
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
 
 ##################################
 @register_daemon_form
@@ -2896,11 +2906,9 @@ def get_container(
         binds = get_container_binds(ctx, ident)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == SNMPGateway.daemon_type:
-        sg = SNMPGateway.init(ctx, ident.fsid, ident.daemon_id)
-        container_args.append(
-            f'--env-file={sg.conf_file_path}'
-        )
-        d_args.extend(sg.get_daemon_args())
+        sg = SNMPGateway.create(ctx, ident)
+        sg.customize_container_args(ctx, container_args)
+        sg.customize_process_args(ctx, d_args)
 
     _update_container_args_for_podman(ctx, ident, container_args)
     return CephContainer.for_daemon(

From 3768dba607cd89893e3cda09cc6814ee215fa7eb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 07:18:55 -0500
Subject: [PATCH 0583/2492] cephadm: add more ContainerDeamonForm methods to
 the custom container class

Add methods customize_{process_args,container_envs,container_args} and
default_entrypoint to the custom container daemon type class. Use those
methods in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8225553c9271..2b30e2563ed6 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1981,6 +1981,24 @@ def customize_container_endpoints(
         if deployment_type == DeploymentType.DEFAULT:
             endpoints.extend([EndPoint('0.0.0.0', p) for p in self.ports])
 
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_container_args())
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint or ''
+
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return self.uid, self.gid
 
@@ -2898,11 +2916,11 @@ def get_container(
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
-        entrypoint = cc.entrypoint or ''
+        entrypoint = cc.default_entrypoint()
         host_network = False
-        envs.extend(cc.get_container_envs())
-        container_args.extend(cc.get_container_args())
-        d_args.extend(cc.get_daemon_args())
+        cc.customize_container_envs(ctx, envs)
+        cc.customize_container_args(ctx, container_args)
+        cc.customize_process_args(ctx, d_args)
         binds = get_container_binds(ctx, ident)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == SNMPGateway.daemon_type:

From 4f07a3b678a5a16e97994d242616a5ae088a736b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 07:40:29 -0500
Subject: [PATCH 0584/2492] cephadm: add more ContainerDeamonForm methods to
 the iscsi class

Add method default_entrypoint to the iscsi daemon type class. Use that
method in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 2b30e2563ed6..6e34834535df 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1275,6 +1275,9 @@ def config_and_keyring(
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
         return extract_uid_gid(ctx)
 
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
+
 ##################################
 
 
@@ -2907,8 +2910,8 @@ def get_container(
         binds = get_container_binds(ctx, ident)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephIscsi.daemon_type:
-        entrypoint = CephIscsi.entrypoint
-        name = ident.daemon_name
+        iscsi = CephIscsi.create(ctx, ident)
+        entrypoint = iscsi.default_entrypoint()
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
         privileged = True

From 5bc8b8d80be94e319c70cbb64b2dce9d1f8a660e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 07:45:26 -0500
Subject: [PATCH 0585/2492] cephadm: add more ContainerDeamonForm methods to
 the nvmeof class

Add method customize_container_args to the nvmeof daemon type class. Use
that method in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6e34834535df..ba2dc68bdd0b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1433,6 +1433,13 @@ def config_and_keyring(
     ) -> Tuple[Optional[str], Optional[str]]:
         return get_config_and_keyring(ctx)
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(['--ulimit', 'memlock=-1:-1'])
+        args.extend(['--ulimit', 'nofile=10240'])
+        args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
+
 
 ##################################
 
@@ -2903,10 +2910,8 @@ def get_container(
         container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephNvmeof.daemon_type:
-        name = ident.daemon_name
-        container_args.extend(['--ulimit', 'memlock=-1:-1'])
-        container_args.extend(['--ulimit', 'nofile=10240'])
-        container_args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
+        nvmeof = CephNvmeof.create(ctx, ident)
+        nvmeof.customize_container_args(ctx, container_args)
         binds = get_container_binds(ctx, ident)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephIscsi.daemon_type:

From 3098282c598f82e876718887cfa7a39395b76de0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 18:55:17 -0400
Subject: [PATCH 0586/2492] cephadm: add more ContainerDeamonForm methods to
 the keepalived class

Add methods customize_{container_args,container_envs} to the keepalived
daemon type class. Use those methods in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ba2dc68bdd0b..242f2fa35cab 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1756,6 +1756,16 @@ def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = get_container(ctx, self.identity)
         return to_deployment_container(ctx, ctr)
 
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
+
 
 ##################################
 
@@ -2905,9 +2915,9 @@ def get_container(
         d_args.extend(haproxy.get_daemon_args())
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == Keepalived.daemon_type:
-        name = ident.daemon_name
-        envs.extend(Keepalived.get_container_envs())
-        container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
+        keepalived = Keepalived.create(ctx, ident)
+        keepalived.customize_container_envs(ctx, envs)
+        keepalived.customize_container_args(ctx, container_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephNvmeof.daemon_type:
         nvmeof = CephNvmeof.create(ctx, ident)

From 3d7aa101e4733ca4bc93e83435cc96e8541cbffb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 07:56:29 -0500
Subject: [PATCH 0587/2492] cephadm: add more ContainerDeamonForm methods to
 the haproxy class

Add methods customize_{container_args,process_args} to the haproxy
daemon type class. Use those methods in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 242f2fa35cab..c82e53f0b847 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1637,6 +1637,19 @@ def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = get_container(ctx, self.identity)
         return to_deployment_container(ctx, ctr)
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(
+            ['--user=root']
+        )  # haproxy 2.4 defaults to a different user
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+
 ##################################
 
 
@@ -2909,10 +2922,9 @@ def get_container(
         ceph_exporter = CephExporter.init(ctx, ident.fsid, ident.daemon_id)
         d_args.extend(ceph_exporter.get_daemon_args())
     elif daemon_type == HAproxy.daemon_type:
-        name = ident.daemon_name
-        container_args.extend(['--user=root'])  # haproxy 2.4 defaults to a different user
-        haproxy = HAproxy.init(ctx, ident.fsid, ident.daemon_id)
-        d_args.extend(haproxy.get_daemon_args())
+        haproxy = HAproxy.create(ctx, ident)
+        haproxy.customize_container_args(ctx, container_args)
+        haproxy.customize_process_args(ctx, d_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == Keepalived.daemon_type:
         keepalived = Keepalived.create(ctx, ident)

From a691e95df60872f8b7dd0b672a6f3de6d9f15c4b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 08:01:41 -0500
Subject: [PATCH 0588/2492] cephadm: add more ContainerDeamonForm methods to
 the ceph exporter class

Add methods customize_process_args & default_entrypoint to the ceph
exporter daemon type class. Use those methods in the get_container
function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c82e53f0b847..7e9065ac29f7 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1525,6 +1525,16 @@ def customize_container_mounts(
         cm = Ceph.get_ceph_mounts(ctx, self.identity)
         mounts.update(cm)
 
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        name = 'client.ceph-exporter.%s' % self.identity.daemon_id
+        args.extend(['-n', name, '-f'])
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
+
 
 ##################################
 
@@ -2916,11 +2926,9 @@ def get_container(
         d_args.extend(nfs_ganesha.get_daemon_args())
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephExporter.daemon_type:
-        entrypoint = CephExporter.entrypoint
-        name = 'client.ceph-exporter.%s' % ident.daemon_id
-        ceph_args = ['-n', name, '-f']
-        ceph_exporter = CephExporter.init(ctx, ident.fsid, ident.daemon_id)
-        d_args.extend(ceph_exporter.get_daemon_args())
+        ceph_exporter = CephExporter.create(ctx, ident)
+        entrypoint = ceph_exporter.default_entrypoint()
+        ceph_exporter.customize_process_args(ctx, d_args)
     elif daemon_type == HAproxy.daemon_type:
         haproxy = HAproxy.create(ctx, ident)
         haproxy.customize_container_args(ctx, container_args)

From 9ffe0771f7c719a6a6e8a564adfa573a4fce4d28 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 08:09:10 -0500
Subject: [PATCH 0589/2492] cephadm: add more ContainerDeamonForm methods to
 the nfs class

Add methods customize_{container_envs,process_args} & default_entrypoint
to the nfs daemon type class. Use those methods in the get_container
function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7e9065ac29f7..171d2a1c928e 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1051,6 +1051,19 @@ def config_and_keyring(
     ) -> Tuple[Optional[str], Optional[str]]:
         return get_config_and_keyring(ctx)
 
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
+
 ##################################
 
 
@@ -2919,11 +2932,10 @@ def get_container(
             tracing = Tracing.create(ctx, ident)
             d_args.extend(tracing.get_daemon_args())
     elif daemon_type == NFSGanesha.daemon_type:
-        entrypoint = NFSGanesha.entrypoint
-        name = ident.daemon_name
-        envs.extend(NFSGanesha.get_container_envs())
-        nfs_ganesha = NFSGanesha.init(ctx, ident.fsid, ident.daemon_id)
-        d_args.extend(nfs_ganesha.get_daemon_args())
+        nfs_ganesha = NFSGanesha.create(ctx, ident)
+        entrypoint = nfs_ganesha.default_entrypoint()
+        nfs_ganesha.customize_container_envs(ctx, envs)
+        nfs_ganesha.customize_process_args(ctx, d_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephExporter.daemon_type:
         ceph_exporter = CephExporter.create(ctx, ident)

From c701621ad9a3452bf0548a2d8283e88e9f7d6945 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 11:00:54 -0500
Subject: [PATCH 0590/2492] cephadm: add more ContainerDeamonForm methods to
 the tracing class

Add methods customize_{container_envs,process_args} & default_entrypoint
to the tracing daemon type class. Use those methods in the get_container
function.
This change tries to adapt to, but not fix, the rather strange
set_configuration static method that side-effects the class's(!)
property.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 46 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 171d2a1c928e..6c37a1fb4d21 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1846,6 +1846,18 @@ def set_configuration(config: Dict[str, str], daemon_type: str) -> None:
 
     def __init__(self, ident: DaemonIdentity) -> None:
         self._identity = ident
+        self._configured = False
+
+    def _configure(self, ctx: CephadmContext) -> None:
+        if self._configured:
+            return
+        config = fetch_configs(ctx)
+        # Currently, this method side-effects the class attribute, and that
+        # is unpleasant. In the future it would be nice to move all of
+        # set_configuration into _confiure and only modify each classes data
+        # independently
+        self.set_configuration(config, self.identity.daemon_type)
+        self._configured = True
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Tracing':
@@ -1867,6 +1879,28 @@ def get_daemon_args(self) -> List[str]:
             'daemon_args', []
         )
 
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        self._configure(ctx)
+        # earlier code did an explicit check if the daemon type was jaeger-agent
+        # and would only call get_daemon_args if that was true. However, since
+        # the function only returns a non-empty list in the case of jaeger-agent
+        # that check is unnecessary and is not brought over.
+        args.extend(self.get_daemon_args())
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        self._configure(ctx)
+        envs.extend(
+            self.components[self.identity.daemon_type].get('envs', [])
+        )
+
+    def default_entrypoint(self) -> str:
+        return ''
+
+
 ##################################
 
 
@@ -2923,14 +2957,10 @@ def get_container(
         d_args.extend(monitoring.get_daemon_args())
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type in Tracing.components:
-        entrypoint = ''
-        name = ident.daemon_name
-        config = fetch_configs(ctx)
-        Tracing.set_configuration(config, daemon_type)
-        envs.extend(Tracing.components[daemon_type].get('envs', []))
-        if daemon_type == 'jaeger-agent':
-            tracing = Tracing.create(ctx, ident)
-            d_args.extend(tracing.get_daemon_args())
+        tracing = Tracing.create(ctx, ident)
+        entrypoint = tracing.default_entrypoint()
+        tracing.customize_container_envs(ctx, envs)
+        tracing.customize_process_args(ctx, d_args)
     elif daemon_type == NFSGanesha.daemon_type:
         nfs_ganesha = NFSGanesha.create(ctx, ident)
         entrypoint = nfs_ganesha.default_entrypoint()

From c423850fce5942493c7be5c5e50bfc8bbcb04423 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 11:09:27 -0500
Subject: [PATCH 0591/2492] cephadm: add more ContainerDeamonForm methods to
 the monitoring class

Add methods customize_{container_args,process_args} & default_entrypoint
to the monitoring daemon type class. Use those methods in the
get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 43 +++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6c37a1fb4d21..b704e50c9c49 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -872,6 +872,31 @@ def customize_container_mounts(
         data_dir = self.identity.data_dir(ctx.data_dir)
         mounts.update(self._get_container_mounts(data_dir))
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        uid, _ = self.uid_gid(ctx)
+        monitoring_args = [
+            '--user',
+            str(uid),
+            # FIXME: disable cpu/memory limits for the time being (not supported
+            # by ubuntu 18.04 kernel!)
+        ]
+        args.extend(monitoring_args)
+        if self.identity.daemon_type == 'node-exporter':
+            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
+            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
+            # between the node-exporter container and the host to avoid selinux denials
+            args.extend(['--security-opt', 'label=disable'])
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return ''
+
 ##################################
 
 
@@ -2939,22 +2964,10 @@ def get_container(
         name = ident.daemon_name
         ceph_args = ['-n', name, '-f']
     elif daemon_type in Monitoring.components:
-        entrypoint = ''
-        uid, gid = Monitoring.extract_uid_gid(ctx, daemon_type)
-        monitoring_args = [
-            '--user',
-            str(uid),
-            # FIXME: disable cpu/memory limits for the time being (not supported
-            # by ubuntu 18.04 kernel!)
-        ]
-        container_args.extend(monitoring_args)
-        if daemon_type == 'node-exporter':
-            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
-            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
-            # between the node-exporter container and the host to avoid selinux denials
-            container_args.extend(['--security-opt', 'label=disable'])
         monitoring = Monitoring.create(ctx, ident)
-        d_args.extend(monitoring.get_daemon_args())
+        entrypoint = monitoring.default_entrypoint()
+        monitoring.customize_container_args(ctx, container_args)
+        monitoring.customize_process_args(ctx, d_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type in Tracing.components:
         tracing = Tracing.create(ctx, ident)

From 882073bc46c888289ae76fdc817995ced6169e02 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 11:12:54 -0500
Subject: [PATCH 0592/2492] cephadm: add methods to ceph daemon type class

Add unused methods to the ceph class in order to prepare for moving
the logic for setting up the various ceph deaemon types to be
handled by the class.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b704e50c9c49..3ea41e47c9d7 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -351,6 +351,34 @@ def customize_container_mounts(
         )
         mounts.update(cm)
 
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        ident = self.identity
+        if ident.daemon_type == 'rgw':
+            name = 'client.rgw.%s' % ident.daemon_id
+        elif ident.daemon_type == 'rbd-mirror':
+            name = 'client.rbd-mirror.%s' % ident.daemon_id
+        elif ident.daemon_type == 'cephfs-mirror':
+            name = 'client.cephfs-mirror.%s' % ident.daemon_id
+        elif ident.daemon_type == 'crash':
+            name = 'client.crash.%s' % ident.daemon_id
+        elif ident.daemon_type in ['mon', 'mgr', 'mds', 'osd']:
+            name = ident.daemon_name
+        else:
+            raise ValueError(ident)
+        args.extend(['-n', name, '-f'])
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        ep = {
+            'rgw': '/usr/bin/radosgw',
+            'rbd-mirror': '/usr/bin/rbd-mirror',
+            'cephfs-mirror': '/usr/bin/cephfs-mirror',
+        }
+        daemon_type = self.identity.daemon_type
+        return ep.get(daemon_type) or f'/usr/bin/ceph-{daemon_type}'
+
 ##################################
 
 

From a2bd2a435e858dda39e99442a1bf81e5807c4138 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 20 Oct 2023 20:18:36 -0400
Subject: [PATCH 0593/2492] cephadm: add container envs methods to ceph and
 ceph exporter classes

These, currently unused, methods will be used in a subsequent change to
move logic for setting up containers out of the get_containers function
into the daemon type classes in a common way.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 3ea41e47c9d7..9d27cbaa9901 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -370,6 +370,11 @@ def customize_process_args(
         args.extend(['-n', name, '-f'])
         args.extend(self.get_daemon_args())
 
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+
     def default_entrypoint(self) -> str:
         ep = {
             'rgw': '/usr/bin/radosgw',
@@ -1598,6 +1603,11 @@ def customize_process_args(
         args.extend(['-n', name, '-f'])
         args.extend(self.get_daemon_args())
 
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+
     def default_entrypoint(self) -> str:
         return self.entrypoint
 

From a939d015c1cc49fe694ed58265b76a8658f53759 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 11:44:48 -0500
Subject: [PATCH 0594/2492] cephadm: use class methods when setting up ceph
 based daemon containers

Now that the ceph and ceph exporter classes have all the necessary
methods, rework the get_container function to use those class methods
in a common manner.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 44 ++++++++++++++----------------------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 9d27cbaa9901..37c1cdaf0b62 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -367,7 +367,9 @@ def customize_process_args(
             name = ident.daemon_name
         else:
             raise ValueError(ident)
-        args.extend(['-n', name, '-f'])
+        args.extend(['-n', name])
+        if ident.daemon_type != 'crash':
+            args.append('-f')
         args.extend(self.get_daemon_args())
 
     def customize_container_envs(
@@ -2960,8 +2962,6 @@ def get_container(
     container_args: Optional[List[str]] = None,
 ) -> 'CephContainer':
     entrypoint: str = ''
-    name: str = ''
-    ceph_args: List[str] = []
     d_args: List[str] = []
     envs: List[str] = []
     host_network: bool = True
@@ -2969,39 +2969,21 @@ def get_container(
     mounts: Dict[str, str] = {}
 
     daemon_type = ident.daemon_type
-    if daemon_type in ceph_daemons():
-        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
-        mounts = get_container_mounts(ctx, ident)
     if container_args is None:
         container_args = []
     _update_pids_limit(ctx, daemon_type, container_args)
     if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
-        ceph_daemon = Ceph.create(ctx, ident)
-        d_args.extend(ceph_daemon.get_daemon_args())
+        ceph_daemon = daemon_form_create(ctx, ident)
+        assert isinstance(ceph_daemon, ContainerDaemonForm)
+        entrypoint = ceph_daemon.default_entrypoint()
+        ceph_daemon.customize_container_envs(ctx, envs)
+        ceph_daemon.customize_container_args(ctx, container_args)
+        ceph_daemon.customize_process_args(ctx, d_args)
+        mounts = get_container_mounts(ctx, ident)
     if daemon_type in ['mon', 'osd']:
         # mon and osd need privileged in order for libudev to query devices
         privileged = True
-    if daemon_type == 'rgw':
-        entrypoint = '/usr/bin/radosgw'
-        name = 'client.rgw.%s' % ident.daemon_id
-        ceph_args = ['-n', name, '-f']
-    elif daemon_type == 'rbd-mirror':
-        entrypoint = '/usr/bin/rbd-mirror'
-        name = 'client.rbd-mirror.%s' % ident.daemon_id
-        ceph_args = ['-n', name, '-f']
-    elif daemon_type == 'cephfs-mirror':
-        entrypoint = '/usr/bin/cephfs-mirror'
-        name = 'client.cephfs-mirror.%s' % ident.daemon_id
-        ceph_args = ['-n', name, '-f']
-    elif daemon_type == 'crash':
-        entrypoint = '/usr/bin/ceph-crash'
-        name = 'client.crash.%s' % ident.daemon_id
-        ceph_args = ['-n', name]
-    elif daemon_type in ['mon', 'mgr', 'mds', 'osd']:
-        entrypoint = '/usr/bin/ceph-' + daemon_type
-        name = ident.daemon_name
-        ceph_args = ['-n', name, '-f']
-    elif daemon_type in Monitoring.components:
+    if daemon_type in Monitoring.components:
         monitoring = Monitoring.create(ctx, ident)
         entrypoint = monitoring.default_entrypoint()
         monitoring.customize_container_args(ctx, container_args)
@@ -3021,7 +3003,9 @@ def get_container(
     elif daemon_type == CephExporter.daemon_type:
         ceph_exporter = CephExporter.create(ctx, ident)
         entrypoint = ceph_exporter.default_entrypoint()
+        ceph_exporter.customize_container_envs(ctx, envs)
         ceph_exporter.customize_process_args(ctx, d_args)
+        mounts = get_container_mounts(ctx, ident)
     elif daemon_type == HAproxy.daemon_type:
         haproxy = HAproxy.create(ctx, ident)
         haproxy.customize_container_args(ctx, container_args)
@@ -3064,7 +3048,7 @@ def get_container(
         ctx,
         ident=ident,
         entrypoint=entrypoint,
-        args=ceph_args + d_args,
+        args=d_args,
         container_args=container_args,
         volume_mounts=mounts,
         bind_mounts=binds,

From 57b73442f86eec7f72eebb9704e464bb1f3e7388 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 12:57:45 -0500
Subject: [PATCH 0595/2492] cephadm: eliminate the _update_pids_limit function

Eliminate the _update_pids_limit function by moving the logic into the
various daemon type classes `customize_container_args` functions allow
each class to decide if it needs to run with unlimited-pids or not.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 37c1cdaf0b62..e00dcb0f7a13 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -351,6 +351,11 @@ def customize_container_mounts(
         )
         mounts.update(cm)
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
     def customize_process_args(
         self, ctx: CephadmContext, args: List[str]
     ) -> None:
@@ -1121,6 +1126,11 @@ def customize_process_args(
     ) -> None:
         args.extend(self.get_daemon_args())
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
     def default_entrypoint(self) -> str:
         return self.entrypoint
 
@@ -1351,6 +1361,11 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
     def default_entrypoint(self) -> str:
         return self.entrypoint
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
 ##################################
 
 
@@ -1509,6 +1524,7 @@ def config_and_keyring(
     def customize_container_args(
         self, ctx: CephadmContext, args: List[str]
     ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
         args.extend(['--ulimit', 'memlock=-1:-1'])
         args.extend(['--ulimit', 'nofile=10240'])
         args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
@@ -1605,6 +1621,11 @@ def customize_process_args(
         args.extend(['-n', name, '-f'])
         args.extend(self.get_daemon_args())
 
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
     def customize_container_envs(
         self, ctx: CephadmContext, envs: List[str]
     ) -> None:
@@ -2937,23 +2958,6 @@ def get_ceph_volume_container(ctx: CephadmContext,
     )
 
 
-def _update_pids_limit(ctx: CephadmContext, daemon_type: str, container_args: List[str]) -> None:
-    """Update container_args to contain a pids limit option if the daemon_type
-    is of a suitable match.
-    """
-    # set container's pids-limit to unlimited rather than default (Docker 4096 / Podman 2048)
-    # Useful for daemons like iscsi where the default pids-limit limits the number of luns
-    # per iscsi target or rgw where increasing the rgw_thread_pool_size to a value near
-    # the default pids-limit may cause the container to crash.
-    unlimited_daemons = set(ceph_daemons())
-    unlimited_daemons.add(CephIscsi.daemon_type)
-    unlimited_daemons.add(CephNvmeof.daemon_type)
-    unlimited_daemons.add(NFSGanesha.daemon_type)
-    if daemon_type not in unlimited_daemons:
-        return
-    container_args.append(ctx.container_engine.unlimited_pids_option)
-
-
 def get_container(
     ctx: CephadmContext,
     ident: 'DaemonIdentity',
@@ -2971,7 +2975,6 @@ def get_container(
     daemon_type = ident.daemon_type
     if container_args is None:
         container_args = []
-    _update_pids_limit(ctx, daemon_type, container_args)
     if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
         ceph_daemon = daemon_form_create(ctx, ident)
         assert isinstance(ceph_daemon, ContainerDaemonForm)
@@ -2998,12 +3001,14 @@ def get_container(
         nfs_ganesha = NFSGanesha.create(ctx, ident)
         entrypoint = nfs_ganesha.default_entrypoint()
         nfs_ganesha.customize_container_envs(ctx, envs)
+        nfs_ganesha.customize_container_args(ctx, container_args)
         nfs_ganesha.customize_process_args(ctx, d_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CephExporter.daemon_type:
         ceph_exporter = CephExporter.create(ctx, ident)
         entrypoint = ceph_exporter.default_entrypoint()
         ceph_exporter.customize_container_envs(ctx, envs)
+        ceph_exporter.customize_container_args(ctx, container_args)
         ceph_exporter.customize_process_args(ctx, d_args)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == HAproxy.daemon_type:
@@ -3024,6 +3029,7 @@ def get_container(
     elif daemon_type == CephIscsi.daemon_type:
         iscsi = CephIscsi.create(ctx, ident)
         entrypoint = iscsi.default_entrypoint()
+        iscsi.customize_container_args(ctx, container_args)
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
         privileged = True

From c2f6a770bf0e12296c334d99ac86ff4732ec29b7 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 14 Nov 2023 23:40:42 +1000
Subject: [PATCH 0596/2492] doc/glossary: add "Quorum" to glossary

Add the term "Quorum" to the glossary and link to the part of
architecture.rst concerning Monitors. The sticky header at the top of
the docs.ceph.com website gets in the way of the location linked to in
this commit, but fatigue and disgust prevent me from spending time today
trial-and-erroring my way through the hostile and ill-documented
wilderness of scroll-margin so that the link goes where it should.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 2 ++
 doc/glossary.rst     | 8 +++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index 189977fc7be3..983cec2300a6 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -30,6 +30,8 @@ A Ceph Storage Cluster consists of multiple types of daemons:
 - :term:`Ceph Manager`
 - :term:`Ceph Metadata Server`
 
+.. _arch_monitor:
+
 Ceph Monitors maintain the master copy of the cluster map, which they provide
 to Ceph clients. Provisioning multiple monitors within the Ceph cluster ensures
 availability in the event that one of the monitor daemons or its host fails.
diff --git a/doc/glossary.rst b/doc/glossary.rst
index d5619e464c8b..df7a0d18afa1 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -271,7 +271,7 @@
                 The Ceph manager software, which collects all the state from
                 the whole cluster in one place.
 
-	MON
+	:ref:`MON<arch_monitor>`
 		The Ceph monitor software.
 
 	Node
@@ -337,6 +337,12 @@
                 Firefly (v. 0.80). See :ref:`Primary Affinity
                 <rados_ops_primary_affinity>`.
 
+        Quorum	
+                Quorum is the state that exists when a majority of the
+                :ref:`Monitors<arch_monitor>` in the cluster are ``up``. A
+                minimum of three :ref:`Monitors<arch_monitor>` must exist in
+                the cluster in order for Quorum to be possible.
+
 	RADOS
                 **R**\eliable **A**\utonomic **D**\istributed **O**\bject
                 **S**\tore. RADOS is the object store that provides a scalable

From ea3f155d9008efa6a2d9615159442d0fd99a4b6f Mon Sep 17 00:00:00 2001
From: Alexander Indenbaum <aindenba@redhat.com>
Date: Tue, 14 Nov 2023 10:12:11 +0200
Subject: [PATCH 0597/2492] mgr/cephadm: Expose nvmeof gateway configuration
 parameters through specifications

- min_controller_id, max_controller_id: Enable the specification of minimum and maximum controller IDs utilized by the SPDK.
  Having distinct controller IDs is vital for configuring multipath setups.
- enable_spdk_discovery_controller: Manage whether the SPDK or ceph-nvmeof discovery service is employed. The default value is set to False.

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 .../templates/services/nvmeof/ceph-nvmeof.conf.j2        | 3 +++
 src/pybind/mgr/cephadm/tests/test_services.py            | 3 +++
 src/python-common/ceph/deployment/service_spec.py        | 9 +++++++++
 3 files changed, 15 insertions(+)

diff --git a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
index 72a3e5839edc..4aa0b9093577 100644
--- a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
@@ -7,6 +7,9 @@ port = {{ port }}
 enable_auth = {{ spec.enable_auth }}
 state_update_notify = True
 state_update_interval_sec = 5
+min_controller_id = {{ spec.min_controller_id }}
+max_controller_id = {{ spec.max_controller_id }}
+enable_spdk_discovery_controller = {{ spec.enable_spdk_discovery_controller }}
 
 [ceph]
 pool = {{ spec.pool }}
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index 21c62ffd06fb..dbab022058d4 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -390,6 +390,9 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
 enable_auth = False
 state_update_notify = True
 state_update_interval_sec = 5
+min_controller_id = 1
+max_controller_id = 65519
+enable_spdk_discovery_controller = False
 
 [ceph]
 pool = {pool}
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 4181ee2563e4..6027c292ca6d 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1131,6 +1131,9 @@ def __init__(self,
                  port: Optional[int] = None,
                  pool: Optional[str] = None,
                  enable_auth: bool = False,
+                 min_controller_id: Optional[str] = '1',
+                 max_controller_id: Optional[str] = '65519',
+                 enable_spdk_discovery_controller: Optional[bool] = False,
                  server_key: Optional[str] = None,
                  server_cert: Optional[str] = None,
                  client_key: Optional[str] = None,
@@ -1171,6 +1174,12 @@ def __init__(self,
         self.group = group
         #: ``enable_auth`` enables user authentication on nvmeof gateway
         self.enable_auth = enable_auth
+        #: ``min_controller_id`` minimum controller id used by SPDK, essential for multipath
+        self.min_controller_id = min_controller_id
+        #: ``max_controller_id`` maximum controller id used by SPDK, essential for multipath
+        self.max_controller_id = max_controller_id
+        #: ``enable_spdk_discovery_controller`` SPDK or ceph-nvmeof discovery service
+        self.enable_spdk_discovery_controller = enable_spdk_discovery_controller
         #: ``server_key`` gateway server key
         self.server_key = server_key or './server.key'
         #: ``server_cert`` gateway server certificate

From 68179ae21384b70c284026ac2b3dbf9f318e9af7 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 31 Aug 2023 17:30:44 +0800
Subject: [PATCH 0598/2492] mds: just wait the client flushes the snap and
 dirty buffer

When truncating the inode we will just set the ifile lock state to
LOCK_XLOCKSNAP and then try to revoke the 'Fb' caps, but if the
client couldn't release the 'Fb' cap in time just replies with a
normal cap updating request, the MDS will successfully transfer
the ifile's lock state to LOCK_EXCL, which is stable.

That means the MDS will wake up the truncating request and continue
truncating the objects from Rados without waiting the clients to
flush the diry buffer.

Fixes: commit 9c65920e7f6 ("mds: force client flush snap data before
		truncating objects")
Fixes: https://tracker.ceph.com/issues/62580
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/Locker.cc | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index b4a28fb8379f..4d1a2f498b10 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1237,6 +1237,19 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, MDSCon
 	  send_lock_message(lock, LOCK_AC_SYNC, softdata);
 	}
 	break;
+      case LOCK_XLOCKSNAP:
+	if (lock->get_sm() == &sm_filelock) {
+	  int pending = lock->gcaps_allowed(CAP_ANY) ||
+	                lock->gcaps_allowed(CAP_LONER) ||
+			lock->gcaps_allowed(CAP_XLOCKER);
+	  int revoke = ~pending & (loner_issued | other_issued | xlocker_issued);
+
+	  // wait for 'Fb' to be revoked
+	  if (revoke & CEPH_CAP_GBUFFER) {
+	    return;
+	  }
+	}
+	break;
       }
 
     }

From 540f3962034c30cbc8a09f335ed38a797c4239fb Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Wed, 15 Nov 2023 15:17:19 +0800
Subject: [PATCH 0599/2492] test/libcephfs: move the snapdiff to a dedicated
 binary

The snapdiff test cases will take too much time, sometimes for hours.
It's very inconvenient to run some general tests locally.

Just move it to a dedicated binary.

Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 qa/workunits/libcephfs/test.sh    |  1 +
 src/test/libcephfs/CMakeLists.txt | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/qa/workunits/libcephfs/test.sh b/qa/workunits/libcephfs/test.sh
index c53fe893c13b..9f94c1d2e293 100755
--- a/qa/workunits/libcephfs/test.sh
+++ b/qa/workunits/libcephfs/test.sh
@@ -6,5 +6,6 @@ ceph_test_libcephfs_reclaim
 ceph_test_libcephfs_lazyio
 ceph_test_libcephfs_newops
 ceph_test_libcephfs_suidsgid
+ceph_test_libcephfs_snapdiff
 
 exit 0
diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt
index 09cb7e6dea00..2f3e0c7c2997 100644
--- a/src/test/libcephfs/CMakeLists.txt
+++ b/src/test/libcephfs/CMakeLists.txt
@@ -11,7 +11,6 @@ if(WITH_LIBCEPHFS)
     deleg.cc
     monconfig.cc
     vxattr.cc
-    snapdiff.cc
   )
   target_link_libraries(ceph_test_libcephfs
     ceph-common
@@ -23,6 +22,20 @@ if(WITH_LIBCEPHFS)
   install(TARGETS ceph_test_libcephfs
     DESTINATION ${CMAKE_INSTALL_BINDIR})
 
+  add_executable(ceph_test_libcephfs_snapdiff
+    main.cc
+    snapdiff.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_snapdiff
+    ceph-common
+    cephfs
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs_snapdiff
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
   add_executable(ceph_test_libcephfs_suidsgid
     suidsgid.cc
   )

From fbb7d73e6f2070fb1732ec2d94ae05d23a31e51b Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 9 Nov 2023 09:21:42 -0600
Subject: [PATCH 0600/2492] tests/standalone: fix scrub-related tests following
 command changes

Using
ceph tell $pgid [deep]-scrub
to initiate an 'operator initiated' scrub, and
ceph tell $pgid schedule[-deep]-scrub
for causing a 'periodic scrub' to be scheduled.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/ceph-helpers.sh           | 45 +++++++++++++++++
 qa/standalone/scrub/osd-mapper.sh       |  6 +--
 qa/standalone/scrub/osd-scrub-repair.sh | 65 +++++++++++++------------
 qa/standalone/scrub/osd-scrub-test.sh   | 26 ++++------
 src/ceph.in                             |  2 +-
 5 files changed, 93 insertions(+), 51 deletions(-)

diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh
index bf2c91bc0427..fc4756daf8b1 100755
--- a/qa/standalone/ceph-helpers.sh
+++ b/qa/standalone/ceph-helpers.sh
@@ -1865,6 +1865,9 @@ function test_repair() {
 # **get_last_scrub_stamp** function reports a timestamp different from
 # the one stored before starting the scrub.
 #
+# The scrub is initiated using the "operator initiated" method, and
+# the scrub triggered is not subject to no-scrub flags etc.
+#
 # @param pgid the id of the PG
 # @return 0 on success, 1 on error
 #
@@ -1899,6 +1902,48 @@ function test_pg_scrub() {
 
 #######################################################################
 
+##
+# Trigger a "scheduled" scrub on **pgid** (by mnaually modifying the relevant
+# last-scrub stamp) and wait until it completes. The pg_scrub
+# function will fail if scrubbing does not complete within $TIMEOUT
+# seconds. The pg_scrub is complete whenever the
+# **get_last_scrub_stamp** function reports a timestamp different from
+# the one stored before starting the scrub.
+#
+# @param pgid the id of the PG
+# @return 0 on success, 1 on error
+#
+function pg_schedule_scrub() {
+    local pgid=$1
+    local last_scrub=$(get_last_scrub_stamp $pgid)
+    ceph pg scrub $pgid
+    wait_for_scrub $pgid "$last_scrub"
+}
+
+function pg_schedule_deep_scrub() {
+    local pgid=$1
+    local last_scrub=$(get_last_scrub_stamp $pgid last_deep_scrub_stamp)
+    ceph pg deep-scrub $pgid
+    wait_for_scrub $pgid "$last_scrub" last_deep_scrub_stamp
+}
+
+function test_pg_schedule_scrub() {
+    local dir=$1
+
+    setup $dir || return 1
+    run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1
+    run_mgr $dir x || return 1
+    run_osd $dir 0 || return 1
+    create_rbd_pool || return 1
+    wait_for_clean || return 1
+    pg_schedule_scrub 1.0 || return 1
+    kill_daemons $dir KILL osd || return 1
+    ! TIMEOUT=1 pg_scrub 1.0 || return 1
+    teardown $dir || return 1
+}
+
+#######################################################################
+
 ##
 # Run the *command* and expect it to fail (i.e. return a non zero status).
 # The output (stderr and stdout) is stored in a temporary file in *dir*
diff --git a/qa/standalone/scrub/osd-mapper.sh b/qa/standalone/scrub/osd-mapper.sh
index ed18f94f1af1..bfe57eac03e9 100755
--- a/qa/standalone/scrub/osd-mapper.sh
+++ b/qa/standalone/scrub/osd-mapper.sh
@@ -77,7 +77,7 @@ function TEST_truncated_sna_record() {
     (( extr_dbg >= 1 )) && rados --format json-pretty -p $poolname listsnaps $objname
 
     # scrub the PG
-    ceph pg $pgid deep_scrub || return 1
+    ceph pg $pgid deep-scrub || return 1
 
     # we aren't just waiting for the scrub to terminate, but also for the
     # logs to be published
@@ -149,7 +149,7 @@ function TEST_truncated_sna_record() {
     local cur_prim=`ceph --format=json-pretty osd map $poolname $objname | jq -r '.up[0]'`
     ceph pg dump pgs
     sleep 2
-    ceph pg $pgid deep_scrub || return 1
+    ceph pg $pgid deep-scrub || return 1
     sleep 5
     ceph pg dump pgs
     (( extr_dbg >= 1 )) && grep -a "ERR" $dir/osd.$cur_prim.log
@@ -161,7 +161,7 @@ function TEST_truncated_sna_record() {
     echo "prev count: $prev_err_cnt"
 
     # scrub again. No errors expected this time
-    ceph pg $pgid deep_scrub || return 1
+    ceph pg $pgid deep-scrub || return 1
     sleep 5
     ceph pg dump pgs
     (( extr_dbg >= 1 )) && grep -a "ERR" $dir/osd.$cur_prim.log
diff --git a/qa/standalone/scrub/osd-scrub-repair.sh b/qa/standalone/scrub/osd-scrub-repair.sh
index 13b30360c4e0..008e8ea1959d 100755
--- a/qa/standalone/scrub/osd-scrub-repair.sh
+++ b/qa/standalone/scrub/osd-scrub-repair.sh
@@ -16,6 +16,7 @@
 #
 set -x
 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
+source $CEPH_ROOT/qa/standalone/scrub/scrub-helpers.sh
 
 if [ `uname` = FreeBSD ]; then
     # erasure coding overwrites are only tested on Bluestore
@@ -160,7 +161,7 @@ function scrub_and_not_schedule() {
     #
     local pg=$(get_pg $poolname SOMETHING)
     local last_scrub=$(get_last_scrub_stamp $pg)
-    ceph pg scrub $pg
+    ceph tell $pg schedule-scrub
 
     #
     # 2) Assure the scrub is not scheduled
@@ -329,8 +330,7 @@ function initiate_and_fetch_state() {
     date  --rfc-3339=ns
 
     # note: must initiate a "regular" (periodic) deep scrub - not an operator-initiated one
-    env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) deep_scrub "$pgid"
-    env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) scrub "$pgid"
+    env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) schedule-deep-scrub "$pgid"
 
     # wait for 'scrubbing' to appear
     for ((i=0; i < 80; i++)); do
@@ -436,19 +436,17 @@ function TEST_auto_repair_bluestore_tag() {
 
 function TEST_auto_repair_bluestore_basic() {
     local dir=$1
-    local poolname=testpool
-
-    # Launch a cluster with 5 seconds scrub interval
-    run_mon $dir a || return 1
-    run_mgr $dir x || return 1
-    local ceph_osd_args="--osd-scrub-auto-repair=true \
-            --osd_deep_scrub_randomize_ratio=0 \
-            --osd-scrub-interval-randomize-ratio=0"
-    for id in $(seq 0 2) ; do
-        run_osd $dir $id $ceph_osd_args || return 1
-    done
+    local -A cluster_conf=(
+        ['osds_num']="3" 
+        ['pgs_in_pool']="1"
+        ['pool_name']="testpool"
+        ['extras']=" --osd_scrub_auto_repair=true"
+    )
+    local extr_dbg=3
+    standard_scrub_cluster $dir cluster_conf
+    local poolid=${cluster_conf['pool_id']}
+    local poolname=${cluster_conf['pool_name']}
 
-    create_pool $poolname 1 1 || return 1
     ceph osd pool set $poolname size 2
     wait_for_clean || return 1
 
@@ -460,12 +458,14 @@ function TEST_auto_repair_bluestore_basic() {
     # Remove the object from one shard physically
     # Restarted osd get $ceph_osd_args passed
     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
+    ceph tell osd.* config set osd_scrub_auto_repair true
 
     local pgid=$(get_pg $poolname SOMETHING)
     local primary=$(get_primary $poolname SOMETHING)
     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
-    ceph tell $pgid deep_scrub
-    ceph tell $pgid scrub
+    # note: the scrub initiated must be a "regular" (periodic) deep scrub - not an
+    # operator-initiated one (as there's no 'auto-repair' for the latter)
+    ceph tell $pgid schedule-deep-scrub
 
     # Wait for auto repair
     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
@@ -510,12 +510,16 @@ function TEST_auto_repair_bluestore_scrub() {
     local pgid=$(get_pg $poolname SOMETHING)
     local primary=$(get_primary $poolname SOMETHING)
     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
-    ceph tell $pgid scrub
+    ceph tell $pgid schedule-scrub
 
     # Wait for scrub -> auto repair
     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
     ceph pg dump pgs
     # Actually this causes 2 scrubs, so we better wait a little longer
+    sleep 2
+    ceph pg dump pgs
+    sleep 2
+    ceph pg dump pgs
     sleep 5
     wait_for_clean || return 1
     ceph pg dump pgs
@@ -567,8 +571,7 @@ function TEST_auto_repair_bluestore_failed() {
     local pgid=$(get_pg $poolname obj1)
     local primary=$(get_primary $poolname obj1)
     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
-    ceph tell $pgid deep_scrub
-    ceph tell $pgid scrub
+    ceph tell $pgid schedule-deep-scrub
 
     # Wait for auto repair
     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
@@ -631,12 +634,12 @@ function TEST_auto_repair_bluestore_failed_norecov() {
     # obj2 can't be repaired
     objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
     objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
+    ceph tell osd.* config set osd_scrub_auto_repair true
 
     local pgid=$(get_pg $poolname obj1)
     local primary=$(get_primary $poolname obj1)
     local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
-    ceph tell $pgid deep_scrub
-    ceph tell $pgid scrub
+    ceph tell $pgid schedule-deep-scrub
 
     # Wait for auto repair
     wait_for_scrub $pgid "$last_scrub_stamp" || return 1
@@ -5793,7 +5796,7 @@ function TEST_periodic_scrub_replicated() {
     flush_pg_stats
     local last_scrub=$(get_last_scrub_stamp $pg)
     # Fake a schedule scrub
-    ceph tell $pg scrub || return 1
+    ceph tell $pg schedule-scrub || return 1
     # Wait for schedule regular scrub
     wait_for_scrub $pg "$last_scrub"
 
@@ -5811,7 +5814,7 @@ function TEST_periodic_scrub_replicated() {
     sleep 5
 
     # Fake a schedule scrub
-    ceph tell $pg scrub || return 1
+    ceph tell $pg schedule-scrub || return 1
     # Wait for schedule regular scrub
     # to notice scrub and skip it
     local found=false
@@ -5828,7 +5831,7 @@ function TEST_periodic_scrub_replicated() {
 
     flush_pg_stats
     # Request a regular scrub and it will be done
-    pg_scrub $pg
+    pg_schedule_scrub $pg
     grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
 
     # deep-scrub error is no longer present
@@ -5885,7 +5888,7 @@ function TEST_scrub_warning() {
       else
         overdue_seconds=$conf_overdue_seconds
       fi
-      ceph tell ${i}.0 scrub $(expr ${overdue_seconds} + ${i}00) || return 1
+      ceph tell ${i}.0 schedule-scrub $(expr ${overdue_seconds} + ${i}00) || return 1
     done
     # Fake schedule deep scrubs
     for i in $(seq $(expr $scrubs + 1) $(expr $scrubs + $deep_scrubs))
@@ -5896,7 +5899,7 @@ function TEST_scrub_warning() {
       else
         overdue_seconds=$conf_overdue_seconds
       fi
-      ceph tell ${i}.0 deep_scrub $(expr ${overdue_seconds} + ${i}00) || return 1
+      ceph tell ${i}.0 schedule-deep-scrub $(expr ${overdue_seconds} + ${i}00) || return 1
     done
     flush_pg_stats
 
@@ -5905,7 +5908,7 @@ function TEST_scrub_warning() {
     ceph health | grep -q " pgs not deep-scrubbed in time" || return 1
     ceph health | grep -q " pgs not scrubbed in time" || return 1
 
-    # note that the 'ceph tell pg deep_scrub' command now also sets the regular scrub
+    # note that the 'ceph tell pg deep-scrub' command now also sets the regular scrub
     # time-stamp. I.e. - all 'late for deep scrubbing' pgs are also late for
     # regular scrubbing. For now, we'll allow both responses.
     COUNT=$(ceph health detail | grep "not scrubbed since" | wc -l)
@@ -6222,15 +6225,15 @@ function TEST_request_scrub_priority() {
         otherpgs="${otherpgs}${opg} "
         local other_last_scrub=$(get_last_scrub_stamp $pg)
         # Fake a schedule scrub
-        ceph tell $opg scrub $opg || return 1
+        ceph tell $opg schedule-scrub $opg || return 1
     done
 
     sleep 15
     flush_pg_stats
 
-    # Request a regular scrub and it will be done
+    # Force a shallow scrub and it will be done
     local last_scrub=$(get_last_scrub_stamp $pg)
-    ceph pg scrub $pg
+    ceph tell $pg scrub || return 1
 
     ceph osd unset noscrub || return 1
     ceph osd unset nodeep-scrub || return 1
diff --git a/qa/standalone/scrub/osd-scrub-test.sh b/qa/standalone/scrub/osd-scrub-test.sh
index 354bd22880e3..e11100f8a198 100755
--- a/qa/standalone/scrub/osd-scrub-test.sh
+++ b/qa/standalone/scrub/osd-scrub-test.sh
@@ -231,11 +231,11 @@ function TEST_scrub_extended_sleep() {
     create_pool $poolname 1 1
     wait_for_clean || return 1
 
-    # Trigger a scrub on a PG
+    # Trigger a periodic scrub on a PG (no 'extended sleep' for h.p. scrubs)
     local pgid=$(get_pg $poolname SOMETHING)
     local primary=$(get_primary $poolname SOMETHING)
     local last_scrub=$(get_last_scrub_stamp $pgid)
-    ceph tell $pgid scrub || return 1
+    ceph tell $pgid schedule-scrub || return 1
 
     # Allow scrub to start extended sleep
     PASSED="false"
@@ -330,12 +330,7 @@ function _scrub_abort() {
     local primary=$(get_primary $poolname obj1)
     local pgid="${poolid}.0"
 
-    ceph tell $pgid $type || return 1
-    # deep-scrub won't start without scrub noticing
-    if [ "$type" = "deep_scrub" ];
-    then
-      ceph tell $pgid scrub || return 1
-    fi
+    ceph tell $pgid schedule-$type || return 1
 
     # Wait for scrubbing to start
     set -o pipefail
@@ -359,7 +354,7 @@ function _scrub_abort() {
     fi
 
     ceph osd set $stopscrub
-    if [ "$type" = "deep_scrub" ];
+    if [ "$type" = "deep-scrub" ];
     then
       ceph osd set noscrub
     fi
@@ -390,7 +385,7 @@ function _scrub_abort() {
     ceph config set osd "osd_scrub_sleep" "0.1"
 
     ceph osd unset $stopscrub
-    if [ "$type" = "deep_scrub" ];
+    if [ "$type" = "deep-scrub" ];
     then
       ceph osd unset noscrub
     fi
@@ -405,7 +400,7 @@ function TEST_scrub_abort() {
 
 function TEST_deep_scrub_abort() {
     local dir=$1
-    _scrub_abort $dir deep_scrub
+    _scrub_abort $dir deep-scrub
 }
 
 function TEST_scrub_permit_time() {
@@ -441,7 +436,7 @@ function TEST_scrub_permit_time() {
     # current time to set last_scrub_stamp, it sets the deadline
     # back by osd_max_interval which would cause the time permit checking
     # to be skipped.  Set back 1 day, the default scrub_min_interval.
-    ceph tell $pgid scrub $(( 24 * 60 * 60 )) || return 1
+    ceph tell $pgid schedule-scrub $(( 24 * 60 * 60 )) || return 1
 
     # Scrub should not run
     for ((i=0; i < 30; i++)); do
@@ -495,7 +490,7 @@ function TEST_just_deep_scrubs() {
     local dbg_counter_at_start=${sched_data['query_scrub_seq']}
     echo "test counter @ start: $dbg_counter_at_start"
 
-    ceph pg $pgid deep_scrub
+    ceph tell $pgid schedule-deep-scrub
 
     sleep 5 # 5s is the 'pg dump' interval
     declare -A sc_data_2
@@ -574,8 +569,7 @@ function TEST_dump_scrub_schedule() {
 
     saved_last_stamp=${sched_data['query_last_stamp']}
     ceph tell osd.* config set osd_scrub_sleep "0"
-    ceph pg deep-scrub $pgid
-    ceph pg scrub $pgid
+    ceph tell $pgid deep-scrub
 
     # wait for the 'last duration' entries to change. Note that the 'dump' one will need
     # up to 5 seconds to sync
@@ -602,7 +596,7 @@ function TEST_dump_scrub_schedule() {
     sleep 2
     saved_last_stamp=${sched_data['query_last_stamp']}
 
-    ceph pg $pgid scrub
+    ceph tell $pgid schedule-scrub
     sleep 1
     sched_data=()
     declare -A expct_scrub_peri_sched=( ['query_is_future']="false" )
diff --git a/src/ceph.in b/src/ceph.in
index 2ba2c74768cf..11a76511a8ec 100755
--- a/src/ceph.in
+++ b/src/ceph.in
@@ -1310,7 +1310,7 @@ def main():
         if final_e:
             raise final_e
 
-    # Block until command completion (currently scrub and deep_scrub only)
+    # Block until command completion (currently scrub and deep scrub only)
     if block:
         wait(childargs, waitdata)
 

From 334cdaf28616631ecb9ca1c94a070fcbb7dee729 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 15 Nov 2023 20:51:47 +0530
Subject: [PATCH 0601/2492] qa: correct usage of DEBUGFS_META_DIR in dedent

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/kernel_mount.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 750b6b5335a4..f005152af989 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -257,9 +257,10 @@ def _get_global_id(self):
                 import json
 
                 def get_id_to_dir():
-                    result = {}
+                    meta_dir = "{meta_dir}"
+                    result = dict()
                     for dir in glob.glob("/sys/kernel/debug/ceph/*"):
-                        if os.path.basename(dir) == DEBUGFS_META_DIR:
+                        if os.path.basename(dir) == meta_dir:
                             continue
                         mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines()
                         global_id = mds_sessions_lines[0].split()[1].strip('"')
@@ -267,7 +268,7 @@ def get_id_to_dir():
                         result[client_id] = global_id
                     return result
                 print(json.dumps(get_id_to_dir()))
-            """)
+            """.format(meta_dir=DEBUGFS_META_DIR))
 
             output = self.client_remote.sh([
                 'sudo', 'python3', '-c', pyscript

From 8df2c4642518de18ca2867283d744441b39e283e Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 15 Nov 2023 20:57:02 +0530
Subject: [PATCH 0602/2492] qa: get_global_id() returns a string (global_id)

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/kernel_mount.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index f005152af989..0cd3501637b2 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -340,7 +340,7 @@ def _global_inst(self):
         if self.inst is not None:
             return self.inst
 
-        client_gid = "client%d" % self.get_global_id()
+        client_gid = "client%d" % int(self.get_global_id())
         self.inst = " ".join([client_gid, self._global_addr])
         return self.inst
 

From e5aeade8f2ef3bf2fe7aa9c000c18e7a7e73d4ee Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 25 Oct 2023 13:06:13 +0000
Subject: [PATCH 0603/2492] crimson/osd: remove do_osd_ops_success_func_t and
 do_osd_ops_failure_func_t

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../osd/osd_operations/internal_client_request.cc  | 10 +---------
 src/crimson/osd/pg.cc                              | 14 +++++++++-----
 src/crimson/osd/pg.h                               |  9 ++-------
 3 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc
index c2a371d274f6..ea45e92c148e 100644
--- a/src/crimson/osd/osd_operations/internal_client_request.cc
+++ b/src/crimson/osd/osd_operations/internal_client_request.cc
@@ -93,15 +93,7 @@ seastar::future<> InternalClientRequest::start()
                   std::move(obc),
                   osd_ops,
                   std::as_const(op_info),
-                  get_do_osd_ops_params(),
-                  // success_func
-                  [] {
-                    return PG::do_osd_ops_iertr::now();
-                  },
-                  // failure_func
-                  [] (const std::error_code& e) {
-                    return PG::do_osd_ops_iertr::now();
-                  }
+                  get_do_osd_ops_params()
                 ).safe_then_unpack_interruptible(
                   [](auto submitted, auto all_completed) {
                     return all_completed.handle_error_interruptible(
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 78ae559e8cd5..84a944533b96 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1114,9 +1114,7 @@ PG::do_osd_ops(
   ObjectContextRef obc,
   std::vector<OSDOp>& ops,
   const OpInfo &op_info,
-  const do_osd_ops_params_t &&msg_params,
-  do_osd_ops_success_func_t success_func,
-  do_osd_ops_failure_func_t failure_func)
+  const do_osd_ops_params_t &&msg_params)
 {
   // This overload is generally used for internal client requests,
   // use an empty SnapContext.
@@ -1133,8 +1131,14 @@ PG::do_osd_ops(
         SnapContext{}
       ),
       ops,
-      std::move(success_func),
-      std::move(failure_func));
+      // success_func
+      [] {
+        return do_osd_ops_iertr::now();
+      },
+      // failure_func
+      [] (const std::error_code& e) {
+        return do_osd_ops_iertr::now();
+      });
   });
 }
 
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 6a0231e452f6..e06df874b891 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -571,10 +571,7 @@ class PG : public boost::intrusive_ref_counter<
     ObjectContextRef obc,
     const OpInfo &op_info,
     const SnapContext& snapc);
-  using do_osd_ops_success_func_t =
-    std::function<do_osd_ops_iertr::future<>()>;
-  using do_osd_ops_failure_func_t =
-    std::function<do_osd_ops_iertr::future<>(const std::error_code&)>;
+
   struct do_osd_ops_params_t;
   do_osd_ops_iertr::future<MURef<MOSDOpReply>> log_reply(
     Ref<MOSDOp> m,
@@ -583,9 +580,7 @@ class PG : public boost::intrusive_ref_counter<
     ObjectContextRef obc,
     std::vector<OSDOp>& ops,
     const OpInfo &op_info,
-    const do_osd_ops_params_t &&params,
-    do_osd_ops_success_func_t success_func,
-    do_osd_ops_failure_func_t failure_func);
+    const do_osd_ops_params_t &&params);
   template <class Ret, class SuccessFunc, class FailureFunc>
   do_osd_ops_iertr::future<pg_rep_op_fut_t<Ret>> do_osd_ops_execute(
     seastar::lw_shared_ptr<OpsExecuter> ox,

From 7cd0aa0db0c8f7452d7705a93d765981753b48c4 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 2 Jul 2023 12:34:54 +0000
Subject: [PATCH 0604/2492] crimson/osd/shard_services: add comment to next_tid
 initialization

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/shard_services.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 404f28d7d7f3..c0688f957699 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -47,6 +47,8 @@ PerShardState::PerShardState(
     perf(perf), recoverystate_perf(recoverystate_perf),
     throttler(crimson::common::local_conf()),
     next_tid(
+      // Use shard_id to initialize upper 8 bits of counters to ensure that
+      // ids generated by different shards are disjoint
       static_cast<ceph_tid_t>(seastar::this_shard_id()) <<
       (std::numeric_limits<ceph_tid_t>::digits - 8)),
     startup_time(startup_time)

From bf3845c05c8df3eaa50912af4575acf6daf410a8 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 29 Jun 2023 12:11:23 +0000
Subject: [PATCH 0605/2492] crimson/osd/osd.cc: handle_update_log_missing*
 don't decode payload.

Payload is already decoded in IOHandler::read_message (decode_message).

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 157881ccbe40..8da2d566e6b5 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -1135,7 +1135,6 @@ seastar::future<> OSD::handle_update_log_missing(
   crimson::net::ConnectionRef conn,
   Ref<MOSDPGUpdateLogMissing> m)
 {
-  m->decode_payload();
   return pg_shard_manager.start_pg_operation<LogMissingRequest>(
     std::move(conn),
     std::move(m)).second;
@@ -1145,7 +1144,6 @@ seastar::future<> OSD::handle_update_log_missing_reply(
   crimson::net::ConnectionRef conn,
   Ref<MOSDPGUpdateLogMissingReply> m)
 {
-  m->decode_payload();
   return pg_shard_manager.start_pg_operation<LogMissingRequestReply>(
     std::move(conn),
     std::move(m)).second;

From e3de7c0941803ba8ee318bc2d5bea5d3b1f765a3 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 5 Nov 2023 09:31:05 +0000
Subject: [PATCH 0606/2492] crimson/osd/pg: add logs around submit_error_log()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/client_request.cc | 3 +++
 src/crimson/osd/pg.cc                            | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index d208e2e53d97..2160e4823f72 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -346,10 +346,13 @@ ClientRequest::do_process(
   }
   return pg->do_osd_ops(m, conn, obc, op_info, snapc).safe_then_unpack_interruptible(
     [this, pg, &ihref](auto submitted, auto all_completed) mutable {
+      logger().debug("do_process::{} in submitted", *this);
       return submitted.then_interruptible([this, pg, &ihref] {
+	logger().debug("do_process::{} in enter_stage wait_repop", *this);
 	return ihref.enter_stage<interruptor>(client_pp(*pg).wait_repop, *this);
       }).then_interruptible(
 	[this, pg, all_completed=std::move(all_completed), &ihref]() mutable {
+	  logger().debug("do_process::{} in all_completed", *this);
 	  return all_completed.safe_then_interruptible(
 	    [this, pg, &ihref](MURef<MOSDOpReply> reply) {
 	      return ihref.enter_stage<interruptor>(client_pp(*pg).send_reply, *this
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 84a944533b96..b74da374ad7c 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -923,6 +923,7 @@ PG::do_osd_ops_execute(
     );
   }));
 }
+
 seastar::future<> PG::submit_error_log(
   Ref<MOSDOp> m,
   const OpInfo &op_info,
@@ -931,6 +932,8 @@ seastar::future<> PG::submit_error_log(
   ceph_tid_t rep_tid,
   eversion_t &version)
 {
+  logger().debug("{}: {} rep_tid: {} error: {}",
+                 __func__, *m, rep_tid, e);
   const osd_reqid_t &reqid = m->get_reqid();
   mempool::osd_pglog::list<pg_log_entry_t> log_entries;
   log_entries.push_back(pg_log_entry_t(pg_log_entry_t::ERROR,
@@ -967,10 +970,14 @@ seastar::future<> PG::submit_error_log(
                    rep_tid,
                    peering_state.get_pg_trim_to(),
                    peering_state.get_min_last_complete_ondisk());
+      logger().debug("submit_error_log: sending log"
+        "missing_request (rep_tid: {} entries: {})"
+        " to osd {}", rep_tid, log_entries, peer.osd);
       send_cluster_message(peer.osd, std::move(log_m), get_osdmap_epoch());
       waiting_on.insert(peer);
     }
     waiting_on.insert(pg_whoami);
+    logger().debug("submit_error_log: inserting rep_tid {}", rep_tid);
     log_entry_update_waiting_on.insert(
       std::make_pair(rep_tid, log_update_t{std::move(waiting_on)}));
     return shard_services.get_store().do_transaction(
@@ -1038,6 +1045,7 @@ PG::do_osd_ops(
     // failure_func
     [m, &op_info, obc, this] (const std::error_code& e) {
     return seastar::do_with(eversion_t(), [m, &op_info, obc, e, this](auto &version) {
+      logger().error("do_osd_ops_execute::failure_func {} got error: {}", *m, e);
       auto error_log_fut = seastar::now();
       epoch_t epoch = get_osdmap_epoch();
       ceph_tid_t rep_tid = shard_services.get_tid();

From e1e8d1f3fda37d39d0204ad6fc27db1fa2ed5612 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 2 Nov 2023 10:59:55 +0000
Subject: [PATCH 0607/2492] crimson/osd/pg: cleanup version usage
 submit_error_log

* '!log_entries.empty()' assert instead of if-case.
  log_entries entry is inserted right before.

* 'version != eversion_t()' assert instead of if-case.
  since op_info.may_write() is true, we should have a non-empty version.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index b74da374ad7c..9de30555c156 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -946,10 +946,10 @@ seastar::future<> PG::submit_error_log(
     log_entries.back().set_op_returns(m->ops);
   }
   ceph_assert(is_primary());
-  if (!log_entries.empty()) {
-    ceph_assert(log_entries.rbegin()->version >= projected_last_update);
-    version = projected_last_update = log_entries.rbegin()->version;
-  }
+  ceph_assert(!log_entries.empty());
+  ceph_assert(log_entries.rbegin()->version >= projected_last_update);
+  version = projected_last_update = log_entries.rbegin()->version;
+
   ceph::os::Transaction t;
   peering_state.merge_new_log_entries(
     log_entries, t, peering_state.get_pg_trim_to(),
@@ -1059,6 +1059,7 @@ PG::do_osd_ops(
       return error_log_fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
         auto fut = seastar::now();
         if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
+          ceph_assert(version != eversion_t());
           auto it = log_entry_update_waiting_on.find(rep_tid);
           ceph_assert(it != log_entry_update_waiting_on.end());
           auto it2 = it->second.waiting_on.find(pg_whoami);
@@ -1066,15 +1067,11 @@ PG::do_osd_ops(
           it->second.waiting_on.erase(it2);
           if (it->second.waiting_on.empty()) {
             log_entry_update_waiting_on.erase(it);
-            if (version != eversion_t()) {
-              peering_state.complete_write(version, last_complete);
-            }
+            peering_state.complete_write(version, last_complete);
           } else {
             fut = it->second.all_committed.get_shared_future().then(
               [this, &version, last_complete] {
-              if (version != eversion_t()) {
-                peering_state.complete_write(version, last_complete);
-              }
+              peering_state.complete_write(version, last_complete);
               return seastar::now();
             });
           }

From c5146998b8da1e89debe00477a30c84bec036cd3 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 2 Nov 2023 10:00:06 +0000
Subject: [PATCH 0608/2492] crimson/osd/pg: introduce PG::log_entry_version map

`submit_error_log()` was returning `version` to be used later in
`failure_func` call to `complete_write()`.

Maintain the version returned from `submit_error_log()` in a dedicated map
to avoid handling the lifetime of 'version'.

Note: This change is crucial to the following change that will
      return 'error_fut' separately.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 71 ++++++++++++++++++++-----------------------
 src/crimson/osd/pg.h  |  4 +--
 2 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 9de30555c156..0f6065a83775 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -929,8 +929,7 @@ seastar::future<> PG::submit_error_log(
   const OpInfo &op_info,
   ObjectContextRef obc,
   const std::error_code e,
-  ceph_tid_t rep_tid,
-  eversion_t &version)
+  ceph_tid_t rep_tid)
 {
   logger().debug("{}: {} rep_tid: {} error: {}",
                  __func__, *m, rep_tid, e);
@@ -948,8 +947,7 @@ seastar::future<> PG::submit_error_log(
   ceph_assert(is_primary());
   ceph_assert(!log_entries.empty());
   ceph_assert(log_entries.rbegin()->version >= projected_last_update);
-  version = projected_last_update = log_entries.rbegin()->version;
-
+  log_entry_version[rep_tid] = projected_last_update = log_entries.rbegin()->version;
   ceph::os::Transaction t;
   peering_state.merge_new_log_entries(
     log_entries, t, peering_state.get_pg_trim_to(),
@@ -1044,41 +1042,37 @@ PG::do_osd_ops(
     },
     // failure_func
     [m, &op_info, obc, this] (const std::error_code& e) {
-    return seastar::do_with(eversion_t(), [m, &op_info, obc, e, this](auto &version) {
-      logger().error("do_osd_ops_execute::failure_func {} got error: {}", *m, e);
-      auto error_log_fut = seastar::now();
-      epoch_t epoch = get_osdmap_epoch();
-      ceph_tid_t rep_tid = shard_services.get_tid();
-      auto last_complete = peering_state.get_info().last_complete;
-      if (op_info.may_write()) {
-        // This should be executed as OrderedExclusivePhaseT so that
-        // successive ops will not reorder.
-        // TODO: https://tracker.ceph.com/issues/61651
-        error_log_fut = submit_error_log(m, op_info, obc, e, rep_tid, version);
-      }
-      return error_log_fut.then([m, e, epoch, &op_info, rep_tid, &version, last_complete, this] {
-        auto fut = seastar::now();
-        if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
-          ceph_assert(version != eversion_t());
-          auto it = log_entry_update_waiting_on.find(rep_tid);
-          ceph_assert(it != log_entry_update_waiting_on.end());
-          auto it2 = it->second.waiting_on.find(pg_whoami);
-          ceph_assert(it2 != it->second.waiting_on.end());
-          it->second.waiting_on.erase(it2);
-          if (it->second.waiting_on.empty()) {
-            log_entry_update_waiting_on.erase(it);
-            peering_state.complete_write(version, last_complete);
-          } else {
-            fut = it->second.all_committed.get_shared_future().then(
-              [this, &version, last_complete] {
-              peering_state.complete_write(version, last_complete);
-              return seastar::now();
-            });
-          }
+    logger().error("do_osd_ops_execute::failure_func {} got error: {}", *m, e);
+    auto error_log_fut = seastar::now();
+    epoch_t epoch = get_osdmap_epoch();
+    ceph_tid_t rep_tid = shard_services.get_tid();
+    auto last_complete = peering_state.get_info().last_complete;
+    if (op_info.may_write()) {
+      error_log_fut = submit_error_log(m, op_info, obc, e, rep_tid);
+    }
+    return error_log_fut.then([m, e, epoch, &op_info, rep_tid, last_complete, this] {
+      auto fut = seastar::now();
+      if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
+        ceph_assert(log_entry_version.contains(rep_tid));
+        auto it = log_entry_update_waiting_on.find(rep_tid);
+        ceph_assert(it != log_entry_update_waiting_on.end());
+        auto it2 = it->second.waiting_on.find(pg_whoami);
+        ceph_assert(it2 != it->second.waiting_on.end());
+        it->second.waiting_on.erase(it2);
+        if (it->second.waiting_on.empty()) {
+          log_entry_update_waiting_on.erase(it);
+          peering_state.complete_write(log_entry_version[rep_tid], last_complete);
+          log_entry_version.erase(rep_tid);
+        } else {
+          fut = it->second.all_committed.get_shared_future().then(
+            [this, last_complete, rep_tid] {
+            peering_state.complete_write(log_entry_version[rep_tid], last_complete);
+            return seastar::now();
+          });
         }
-        return fut.then([this, m, e] {
-          return log_reply(m, e);
-        });
+      }
+      return fut.then([this, m, e] {
+        return log_reply(m, e);
       });
     });
   });
@@ -1401,6 +1395,7 @@ PG::interruptible_future<> PG::do_update_log_missing_reply(
       it->second.all_committed.set_value();
       it->second.all_committed = {};
       log_entry_update_waiting_on.erase(it);
+      log_entry_version.erase(m->get_tid());
     }
   } else {
     logger().error("{} : {} got reply {} on unknown tid {}",
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index e06df874b891..93e3ae82ec6e 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -537,8 +537,7 @@ class PG : public boost::intrusive_ref_counter<
     const OpInfo &op_info,
     ObjectContextRef obc,
     const std::error_code e,
-    ceph_tid_t rep_tid,
-    eversion_t &version);
+    ceph_tid_t rep_tid);
 
 private:
 
@@ -769,6 +768,7 @@ class PG : public boost::intrusive_ref_counter<
   };
 
   std::map<ceph_tid_t, log_update_t> log_entry_update_waiting_on;
+  std::map<ceph_tid_t, eversion_t> log_entry_version;
   // snap trimming
   interval_set<snapid_t> snap_trimq;
 };

From 049e071b253555ec89589b8b11c6a8269128d3de Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 5 Nov 2023 11:03:45 +0000
Subject: [PATCH 0609/2492] crimson/osd/pg: add logs and assert around
 log_entry_update_waiting_on

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 0f6065a83775..c09b81615245 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1053,6 +1053,8 @@ PG::do_osd_ops(
     return error_log_fut.then([m, e, epoch, &op_info, rep_tid, last_complete, this] {
       auto fut = seastar::now();
       if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
+        logger().debug("do_osd_ops_execute::failure_func finding rep_tid {}",
+                       rep_tid);
         ceph_assert(log_entry_version.contains(rep_tid));
         auto it = log_entry_update_waiting_on.find(rep_tid);
         ceph_assert(it != log_entry_update_waiting_on.end());
@@ -1063,10 +1065,15 @@ PG::do_osd_ops(
           log_entry_update_waiting_on.erase(it);
           peering_state.complete_write(log_entry_version[rep_tid], last_complete);
           log_entry_version.erase(rep_tid);
+          logger().debug("do_osd_ops_execute::failure_func write complete,"
+                         " erasing rep_tid {}", rep_tid);
+
         } else {
           fut = it->second.all_committed.get_shared_future().then(
             [this, last_complete, rep_tid] {
+            logger().debug("do_osd_ops_execute::failure_func awaited {}", rep_tid);
             peering_state.complete_write(log_entry_version[rep_tid], last_complete);
+            ceph_assert(!log_entry_update_waiting_on.contains(rep_tid));
             return seastar::now();
           });
         }
@@ -1394,6 +1401,8 @@ PG::interruptible_future<> PG::do_update_log_missing_reply(
     if (it->second.waiting_on.empty()) {
       it->second.all_committed.set_value();
       it->second.all_committed = {};
+      logger().debug("{}: erasing rep_tid {}",
+                     __func__, m->get_tid());
       log_entry_update_waiting_on.erase(it);
       log_entry_version.erase(m->get_tid());
     }

From 54fd6760555e28f6d237f7e76a70f64ce3dcc176 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 7 Nov 2023 08:52:03 +0000
Subject: [PATCH 0610/2492] crimson/osd/pg: rep_tid as a faillure_func param

This change is crucial for the next commits,
submit_error_log and failure_func should share the same
rep_tid.

to be shared later with error_log call

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index c09b81615245..407d354134ce 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -892,11 +892,11 @@ PG::do_osd_ops_execute(
         });
       });
     }), OpsExecuter::osd_op_errorator::all_same_way(
-        [rollbacker, failure_func_ptr]
+        [this, rollbacker, failure_func_ptr]
         (const std::error_code& e) mutable {
           return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-          [e, failure_func_ptr] {
-            return (*failure_func_ptr)(e);
+          [this, e, failure_func_ptr] {
+            return (*failure_func_ptr)(e , shard_services.get_tid());
           });
     }));
 
@@ -905,16 +905,17 @@ PG::do_osd_ops_execute(
       std::move(all_completed_fut)
     );
   }, OpsExecuter::osd_op_errorator::all_same_way(
-    [rollbacker, failure_func_ptr]
+    [this, rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
 
     auto submitted_fut = seastar::now();
+    ceph_tid_t rep_tid = shard_services.get_tid();
 
     auto all_completed_fut = e.value() == ENOENT ?
-      (*failure_func_ptr)(e) :
+      (*failure_func_ptr)(e, rep_tid) :
       rollbacker.rollback_obc_if_modified(e).then_interruptible(
-      [e, failure_func_ptr] {
-          return (*failure_func_ptr)(e);
+      [e, failure_func_ptr, rep_tid] {
+          return (*failure_func_ptr)(e, rep_tid);
       });
 
     return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
@@ -1041,11 +1042,10 @@ PG::do_osd_ops(
         std::move(reply));
     },
     // failure_func
-    [m, &op_info, obc, this] (const std::error_code& e) {
+    [m, &op_info, obc, this] (const std::error_code& e, const ceph_tid_t& rep_tid) {
     logger().error("do_osd_ops_execute::failure_func {} got error: {}", *m, e);
     auto error_log_fut = seastar::now();
     epoch_t epoch = get_osdmap_epoch();
-    ceph_tid_t rep_tid = shard_services.get_tid();
     auto last_complete = peering_state.get_info().last_complete;
     if (op_info.may_write()) {
       error_log_fut = submit_error_log(m, op_info, obc, e, rep_tid);
@@ -1142,7 +1142,7 @@ PG::do_osd_ops(
         return do_osd_ops_iertr::now();
       },
       // failure_func
-      [] (const std::error_code& e) {
+      [] (const std::error_code& e, const ceph_tid_t& rep_tid) {
         return do_osd_ops_iertr::now();
       });
   });

From e67dce7120954fff9b7b936348bed0b2ac6c1155 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 15 Nov 2023 16:03:10 +0100
Subject: [PATCH 0611/2492] ceph-volume: fix a regression in `raw list`

'ceph-volume raw list' is broken for a specific use case (rook).

rook copies devices from /dev/ to /mnt for specific/internal needs.

when ceph-volume raw list is passed a device from /mnt then
ceph-volume ignores it and return an empty dict.
That prevent rook from creating OSDs properly.

https://tracker.ceph.com/issues/63545

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../ceph_volume/devices/raw/list.py           | 69 ++++++++++---------
 1 file changed, 38 insertions(+), 31 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/devices/raw/list.py b/src/ceph-volume/ceph_volume/devices/raw/list.py
index dacd980fad60..794bb18c103f 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/list.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/list.py
@@ -5,7 +5,7 @@
 from textwrap import dedent
 from ceph_volume import decorators, process
 from ceph_volume.util import disk
-
+from typing import Any, Dict, List
 
 logger = logging.getLogger(__name__)
 
@@ -66,50 +66,57 @@ class List(object):
     def __init__(self, argv):
         self.argv = argv
 
+    def is_atari_partitions(self, _lsblk: Dict[str, Any]) -> bool:
+        dev = _lsblk['NAME']
+        if _lsblk.get('PKNAME'):
+            parent = _lsblk['PKNAME']
+            try:
+                if disk.has_bluestore_label(parent):
+                    logger.warning(('ignoring child device {} whose parent {} is a BlueStore OSD.'.format(dev, parent),
+                                    'device is likely a phantom Atari partition. device info: {}'.format(_lsblk)))
+                    return True
+            except OSError as e:
+                logger.error(('ignoring child device {} to avoid reporting invalid BlueStore data from phantom Atari partitions.'.format(dev),
+                            'failed to determine if parent device {} is BlueStore. err: {}'.format(parent, e)))
+                return True
+        return False
+
+    def exclude_atari_partitions(self, _lsblk_all: Dict[str, Any]) -> List[Dict[str, Any]]:
+        return [_lsblk for _lsblk in _lsblk_all if not self.is_atari_partitions(_lsblk)]
+
     def generate(self, devs=None):
         logger.debug('Listing block devices via lsblk...')
-        info_devices = disk.lsblk_all(abspath=True)
+        info_devices = []
         if not devs or not any(devs):
             # If no devs are given initially, we want to list ALL devices including children and
             # parents. Parent disks with child partitions may be the appropriate device to return if
             # the parent disk has a bluestore header, but children may be the most appropriate
             # devices to return if the parent disk does not have a bluestore header.
+            info_devices = disk.lsblk_all(abspath=True)
             devs = [device['NAME'] for device in info_devices if device.get('NAME',)]
+        else:
+            for dev in devs:
+                info_devices.append(disk.lsblk(dev, abspath=True))
+
+        # Linux kernels built with CONFIG_ATARI_PARTITION enabled can falsely interpret
+        # bluestore's on-disk format as an Atari partition table. These false Atari partitions
+        # can be interpreted as real OSDs if a bluestore OSD was previously created on the false
+        # partition. See https://tracker.ceph.com/issues/52060 for more info. If a device has a
+        # parent, it is a child. If the parent is a valid bluestore OSD, the child will only
+        # exist if it is a phantom Atari partition, and the child should be ignored. If the
+        # parent isn't bluestore, then the child could be a valid bluestore OSD. If we fail to
+        # determine whether a parent is bluestore, we should err on the side of not reporting
+        # the child so as not to give a false negative.
+        info_devices = self.exclude_atari_partitions(info_devices)
 
         result = {}
         logger.debug('inspecting devices: {}'.format(devs))
-        for dev in devs:
-            # Linux kernels built with CONFIG_ATARI_PARTITION enabled can falsely interpret
-            # bluestore's on-disk format as an Atari partition table. These false Atari partitions
-            # can be interpreted as real OSDs if a bluestore OSD was previously created on the false
-            # partition. See https://tracker.ceph.com/issues/52060 for more info. If a device has a
-            # parent, it is a child. If the parent is a valid bluestore OSD, the child will only
-            # exist if it is a phantom Atari partition, and the child should be ignored. If the
-            # parent isn't bluestore, then the child could be a valid bluestore OSD. If we fail to
-            # determine whether a parent is bluestore, we should err on the side of not reporting
-            # the child so as not to give a false negative.
-            matched_info_devices = [info for info in info_devices if info['NAME'] == dev]
-            if not matched_info_devices:
-                logger.warning('device {} does not exist'.format(dev))
-                continue
-            info_device = matched_info_devices[0]
-            if 'PKNAME' in info_device and info_device['PKNAME'] != "":
-                parent = info_device['PKNAME']
-                try:
-                    if disk.has_bluestore_label(parent):
-                        logger.warning(('ignoring child device {} whose parent {} is a BlueStore OSD.'.format(dev, parent),
-                                        'device is likely a phantom Atari partition. device info: {}'.format(info_device)))
-                        continue
-                except OSError as e:
-                    logger.error(('ignoring child device {} to avoid reporting invalid BlueStore data from phantom Atari partitions.'.format(dev),
-                                'failed to determine if parent device {} is BlueStore. err: {}'.format(parent, e)))
-                    continue
-
-            bs_info = _get_bluestore_info(dev)
+        for info_device in info_devices:
+            bs_info = _get_bluestore_info(info_device['NAME'])
             if bs_info is None:
                 # None is also returned in the rare event that there is an issue reading info from
                 # a BlueStore disk, so be sure to log our assumption that it isn't bluestore
-                logger.info('device {} does not have BlueStore information'.format(dev))
+                logger.info('device {} does not have BlueStore information'.format(info_device['NAME']))
                 continue
             uuid = bs_info['osd_uuid']
             if uuid not in result:

From 4377fbd851bc8614780e61d2e1d22d9a552e2c02 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 13 Nov 2023 19:15:25 -0500
Subject: [PATCH 0612/2492] cephadm: workaround issues running cephadm with
 relative path

Implement a workaround for the jinja2 package loader not correctly
finding a template inside the cephadmlib package when run as
a zipapp. See docstring in the shim class for more details.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 54 ++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 3b7c6f9657e0..e6e8d5e0ea2c 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -1,10 +1,14 @@
 # templating.py - functions to wrap string/file templating libs
 
 import enum
+import os
+import posixpath
+import zipimport
 
-from typing import Any, Optional, IO
+from typing import Any, Optional, IO, Tuple, Callable, cast
 
 import jinja2
+import jinja2.loaders
 
 from .context import CephadmContext
 
@@ -25,6 +29,52 @@ def __repr__(self) -> str:
         return repr(self.value)
 
 
+class _PackageLoader(jinja2.PackageLoader):
+    """Workaround for PackageLoader when using cephadm with relative paths.
+
+    It was found that running the cephadm zipapp from a local dir (like:
+    `./cephadm`) instead of an absolute path (like: `/usr/sbin/cephadm`) caused
+    the PackageLoader to fail to load the template.  After investigation it was
+    found to relate to how the PackageLoader tries to normalize paths and yet
+    the zipimporter type did not have a normalized path (/home/foo/./cephadm
+    and /home/foo/cephadm respectively).  When a full absolute path is passed
+    to zipimporter's get_data method it uses the (non normalized) .archive
+    property to strip the prefix from the argument. When the argument is a
+    normalized path - the prefix fails to match and is not stripped and then
+    the full path fails to match any value in the archive.
+
+    This shim subclass of jinja2.PackageLoader customizes the code path used to
+    load files from the zipimporter so that we try to do the prefix handling
+    all with normalized paths and only path the relative paths to the
+    zipimporter function.
+    """
+
+    def get_source(
+        self, environment: jinja2.Environment, template: str
+    ) -> Tuple[str, str, Optional[Callable[[], bool]]]:
+        if isinstance(self._loader, zipimport.zipimporter):
+            return self._get_archive_source(template)
+        return super().get_source(environment, template)
+
+    def _get_archive_source(self, template: str) -> Tuple[str, str, None]:
+        assert isinstance(self._loader, zipimport.zipimporter)
+        path = arelpath = os.path.normpath(
+            posixpath.join(
+                self._template_root,
+                *jinja2.loaders.split_template_path(template)
+            )
+        )
+        archive_path = os.path.normpath(self._loader.archive)
+        if arelpath.startswith(archive_path + '/'):
+            plen = len(archive_path) + 1
+            arelpath = arelpath[plen:]
+        try:
+            source = cast(bytes, self._loader.get_data(arelpath))
+        except OSError as e:
+            raise jinja2.TemplateNotFound(template) from e
+        return source.decode(self.encoding), path, None
+
+
 class Templater:
     """Cephadm's generic templater class. Based on jinja2."""
 
@@ -44,7 +94,7 @@ def _env(self) -> jinja2.Environment:
     @property
     def _loader(self) -> jinja2.BaseLoader:
         if self._jinja2_loader is None:
-            self._jinja2_loader = jinja2.PackageLoader(self._pkg, self._dir)
+            self._jinja2_loader = _PackageLoader(self._pkg, self._dir)
         return self._jinja2_loader
 
     def render_str(

From 313bfca056ef7f83c0e8c70a3bc4ff9d070b63d9 Mon Sep 17 00:00:00 2001
From: NitzanMordhai <nmordech@redhat.com>
Date: Thu, 16 Nov 2023 07:09:29 +0000
Subject: [PATCH 0613/2492] Tools/rados: Improve Error Messaging for Object
 Name Resolution

The current implementation of 'rados clearomap' exhibits a behavior where
an error message is generated without the associated object name or,
in the case of a non-existent object name, may result in a segmentation fault.

The proposed fix addresses this issue by enhancing the error message.
After applying the fix, error messages will consistently display the correct
object name, providing users with more accurate and actionable information.

Fixes: https://tracker.ceph.com/issues/63541
Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 src/tools/rados/rados.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 3d602e749cde..cd301916c478 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -2971,7 +2971,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
     for (const auto& oid : oids) {
       ret = io_ctx.omap_clear(oid);
       if (ret < 0) {
-        cerr << "error clearing omap keys " << pool_name << "/" << prettify(*obj_name) << "/"
+        cerr << "error clearing omap keys " << pool_name << "/" << prettify(oid) << "/"
              << cpp_strerror(ret) << std::endl;
         return 1;
       }

From d01bd53da20e2c7952ae1a0943b21075b215d28b Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 16 Nov 2023 15:35:05 +0800
Subject: [PATCH 0614/2492] mds: use explicitly sized types for network and
 disk encoding

The size of 'unsigned' type maybe not different from different OSes.
And we should always use explicitly sized type.

Fixes: https://tracker.ceph.com/issues/63552
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/Capability.h          | 2 +-
 src/mds/OpenFileTable.h       | 2 +-
 src/messages/MClientCaps.h    | 2 +-
 src/messages/MClientSession.h | 2 +-
 src/messages/MMDSScrub.h      | 4 ++--
 src/messages/MMDSScrubStats.h | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/mds/Capability.h b/src/mds/Capability.h
index 3fd6d2ce6d4f..ebc626a22949 100644
--- a/src/mds/Capability.h
+++ b/src/mds/Capability.h
@@ -381,7 +381,7 @@ class Capability : public Counter<Capability> {
   ceph_seq_t mseq = 0;
 
   int suppress = 0;
-  unsigned state = 0;
+  uint32_t state = 0;
 
   int lock_cache_allowed = 0;
 };
diff --git a/src/mds/OpenFileTable.h b/src/mds/OpenFileTable.h
index 1f91c202021f..b18395213f56 100644
--- a/src/mds/OpenFileTable.h
+++ b/src/mds/OpenFileTable.h
@@ -113,7 +113,7 @@ class OpenFileTable
 
   version_t omap_version = 0;
 
-  unsigned omap_num_objs = 0;
+  uint32_t omap_num_objs = 0;
   std::vector<unsigned> omap_num_items;
 
   std::map<inodeno_t, OpenedAnchor> anchor_map;
diff --git a/src/messages/MClientCaps.h b/src/messages/MClientCaps.h
index 9d343a4af6b7..96b2cb7d8b81 100644
--- a/src/messages/MClientCaps.h
+++ b/src/messages/MClientCaps.h
@@ -57,7 +57,7 @@ class MClientCaps final : public SafeMessage {
   uint32_t caller_gid = 0;
 
   /* advisory CLIENT_CAPS_* flags to send to mds */
-  unsigned flags = 0;
+  uint32_t flags = 0;
 
   std::vector<uint8_t> fscrypt_auth;
   std::vector<uint8_t> fscrypt_file;
diff --git a/src/messages/MClientSession.h b/src/messages/MClientSession.h
index 3e70370a52d9..37b3fc1d0629 100644
--- a/src/messages/MClientSession.h
+++ b/src/messages/MClientSession.h
@@ -28,7 +28,7 @@ class MClientSession final : public SafeMessage {
   ceph_mds_session_head head;
   static constexpr unsigned SESSION_BLOCKLISTED = (1<<0);
 
-  unsigned flags = 0;
+  uint32_t flags = 0;
   std::map<std::string, std::string> metadata;
   feature_bitset_t supported_features;
   metric_spec_t metric_spec;
diff --git a/src/messages/MMDSScrub.h b/src/messages/MMDSScrub.h
index a1bf2b54f5dd..7714b377608e 100644
--- a/src/messages/MMDSScrub.h
+++ b/src/messages/MMDSScrub.h
@@ -130,11 +130,11 @@ class MMDSScrub : public MMDSOp {
   static constexpr unsigned FLAG_RECURSIVE	= 1<<2;
   static constexpr unsigned FLAG_REPAIR		= 1<<3;
 
-  int op;
+  int32_t op;
   inodeno_t ino;
   fragset_t frags;
   std::string tag;
   inodeno_t origin;
-  unsigned flags = 0;
+  uint32_t flags = 0;
 };
 #endif // CEPH_MMDSSCRUB_H
diff --git a/src/messages/MMDSScrubStats.h b/src/messages/MMDSScrubStats.h
index c60981be5907..2cbb7f2f2116 100644
--- a/src/messages/MMDSScrubStats.h
+++ b/src/messages/MMDSScrubStats.h
@@ -68,7 +68,7 @@ class MMDSScrubStats : public MMDSOp {
   ~MMDSScrubStats() override {}
 
 private:
-  unsigned epoch;
+  uint32_t epoch;
   std::set<std::string> scrubbing_tags;
   bool update_scrubbing = false;
   bool aborting = false;

From 84c5b6c578c1e64807eca5098d51193ee71e6106 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 15 Nov 2023 14:05:47 +0000
Subject: [PATCH 0615/2492] crimson/osd/osd_operations/snaptrim_event: fix
 lifetime on finally()

Sanitized backtrace:
```
DEBUG 2023-11-14 15:23:50,871 [shard 0] osd - snaptrim_event(id=10610, detail=SnapTrimEvent(pgid=16.1a snapid=a needs_pause=0)): interrupted crimson::common::actingset_changed (acting set changed)

    #0 0x5653c613c071 in seastar::shared_mutex::unlock() (/usr/bin/ceph-osd+0x1ed27071)
    #1 0x5653c8670acf in auto seastar::futurize_invoke<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::ExitBarrier<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::BlockingEvent::Trigger<crimson::osd::SnapTrimEvent> >::exit()::{lambda()#1}&>(crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::ExitBarrier<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::BlockingEvent::Trigger<crimson::osd::SnapTrimEvent> >::exit()::{lambda()#1}&) (/usr/bin/ceph-osd+0x2125bacf)
    #2 0x5653c8670e22 in _ZN7seastar20noncopyable_functionIFNS_6futureIvEEvEE17direct_vtable_forIZNS2_4thenIZN7crimson23OrderedConcurrentPhaseTINS7_3osd13SnapTrimEvent9WaitSubopEE11ExitBarrierINSC_13BlockingEvent7TriggerISA_EEE4exitEvEUlvE_S2_EET0_OT_EUlDpOT_E_E4callEPKS4_ (/usr/bin/ceph-osd+0x2125be22)

freed by thread T1 here:
    #0 0x7f10628b73cf in operator delete(void*, unsigned long) (/lib64/libasan.so.6+0xb73cf)
    #1 0x5653c8794bff in crimson::osd::SnapTrimEvent::~SnapTrimEvent() (/usr/bin/ceph-osd+0x2137fbff)

previously allocated by thread T1 here:
    #0 0x7f10628b6367 in operator new(unsigned long) (/lib64/libasan.so.6+0xb6367)

SUMMARY: AddressSanitizer: heap-use-after-free (/usr/bin/ceph-osd+0x1ed27071) in seastar::shared_mutex::unlock()
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index 21fd6e94da94..b0f5a452e7fc 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -89,6 +89,7 @@ SnapTrimEvent::snap_trim_ertr::future<seastar::stop_iteration>
 SnapTrimEvent::start()
 {
   ShardServices &shard_services = pg->get_shard_services();
+  IRef ref = this;
   return interruptor::with_interruption([&shard_services, this] {
     return enter_stage<interruptor>(
       client_pp().wait_for_active
@@ -195,7 +196,7 @@ SnapTrimEvent::start()
   }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
-  }, pg).finally([this] {
+  }, pg).finally([this, ref] {
     logger().debug("{}: exit", *this);
     handle.exit();
   });

From fa6ea9de7cc50e9963723ff85bd924461235ca2b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 11:01:54 -0500
Subject: [PATCH 0616/2492] osd/scrubber: fix ambiguous call to format_to()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

new call to format_to() is ambiguous between std and fmt namespaces:

ceph/src/osd/scrubber/pg_scrubber.cc:654:26:   required from here
ceph/src/osd/scrubber/scrub_job.h:241:21: error: call of overloaded ‘format_to(fmt::v9::basic_format_context<fmt::v9::appender, char>::iterator, const char [60], const double&, double, const double&, const double&, const double&, const double&, const bool&)’ is ambiguous
...
ceph/src/fmt/include/fmt/core.h:3233:17: note: candidate: ‘OutputIt fmt::v9::format_to(OutputIt, format_string<T ...>, T&& ...) [with OutputIt = appender; T = {const double&, double, const double&, const double&, const double&, const double&, const bool&}; typename std::enable_if<detail::is_output_iterator<OutputIt, char>::value, int>::type <anonymous> = 0; format_string<T ...> = basic_format_string<char, const double&, double, const double&, const double&, const double&, const double&, const bool&>]’
 3233 | FMT_INLINE auto format_to(OutputIt out, format_string<T...> fmt, T&&... args)
      |                 ^~~~~~~~~
...
/usr/include/c++/13/format:3824:5: note: candidate: ‘_Out std::format_to(_Out, format_string<_Args ...>, _Args&& ...) [with _Out = fmt::v9::appender; _Args = {const double&, double, const double&, const double&, const double&, const double&, const bool&}; format_string<_Args ...> = basic_format_string<char, const double&, double, const double&, const double&, const double&, const double&, const bool&>]’
 3824 |     format_to(_Out __out, format_string<_Args...> __fmt, _Args&&... __args)
      |     ^~~~~~~~~

gcc (GCC) 13.2.1 20231011 (Red Hat 13.2.1-4)

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/osd/scrubber/scrub_job.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index 57ff1400f9cb..ae29c8ebab44 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -238,7 +238,7 @@ struct formatter<Scrub::sched_conf_t> {
   template <typename FormatContext>
   auto format(const Scrub::sched_conf_t& cf, FormatContext& ctx)
   {
-    return format_to(
+    return fmt::format_to(
 	ctx.out(),
 	"periods: s:{}/{} d:{}/{} iv-ratio:{} deep-rand:{} on-inv:{}",
 	cf.shallow_interval, cf.max_shallow.value_or(-1.0), cf.deep_interval,

From b9c2fb61bd7c53b9003396e59cee1112d3d1de48 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Thu, 9 Nov 2023 16:40:45 -0500
Subject: [PATCH 0617/2492] rgw: fix flight load_bucket call

The interface to the load_bucket call changed when tenant and bucket
name are specified and user is no longer required. This updates the
flight call to load_bucket to reflect those changes.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/rgw_flight.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_flight.cc b/src/rgw/rgw_flight.cc
index 7cc8a460dc4c..955edcced89f 100644
--- a/src/rgw/rgw_flight.cc
+++ b/src/rgw/rgw_flight.cc
@@ -673,6 +673,11 @@ arw::Status FlightServer::DoGet(const flt::ServerCallContext &context,
   ARROW_ASSIGN_OR_RAISE(FlightKey key, TicketToFlightKey(request));
   ARROW_ASSIGN_OR_RAISE(FlightData fd, get_flight_store()->get_flight(key));
 
+#if 0
+  /* load_bucket no longer requires a user parameter. Keep this code
+   * around a bit longer until we fully figure out how permissions
+   * will impact this code.
+   */
   std::unique_ptr<rgw::sal::User> user = driver->get_user(fd.user_id);
   if (user->empty()) {
     INFO << "user is empty" << dendl;
@@ -685,10 +690,11 @@ arw::Status FlightServer::DoGet(const flt::ServerCallContext &context,
     }
     INFO << "user is " << user->get_display_name() << dendl;
   }
+#endif
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
-
-  ret = driver->load_bucket(&dp, &(*user), fd.tenant_name, fd.bucket_name,
+  ret = driver->load_bucket(&dp,
+			    rgw_bucket(fd.tenant_name, fd.bucket_name),
                             &bucket, null_yield);
   if (ret < 0) {
     ERROR << "get_bucket returned " << ret << dendl;

From 44886efc640b94b9a38f7dbfe2ecf473d48c3988 Mon Sep 17 00:00:00 2001
From: Prashant D <pdhange@redhat.com>
Date: Mon, 13 Nov 2023 16:34:59 -0500
Subject: [PATCH 0618/2492] PendingReleaseNotes: Add note for
 POOL_APP_NOT_ENABLED

Adds release notes for the fix added in #47560

Signed-off-by: Prashant D <pdhange@redhat.com>
---
 PendingReleaseNotes | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index b7290fb3938a..eecf724d4638 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -57,6 +57,13 @@
   level storage operator (like Rook) to recreate the missing file system.
   See https://docs.ceph.com/en/latest/cephfs/administration/#file-systems
   docs for more information.
+* RADOS: A POOL_APP_NOT_ENABLED health warning will now be reported if
+  the application is not enabled for the pool irrespective of whether
+  the pool is in use or not. Always tag a pool with an application
+  using ``ceph osd pool application enable`` command to avoid reporting
+  of POOL_APP_NOT_ENABLED health warning for that pool.
+  The user might temporarily mute this warning using
+  ``ceph health mute POOL_APP_NOT_ENABLED``.
 
 >=18.0.0
 

From cacc2eb70fde37475f0d30ca61209d9162fdcc85 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 16 Nov 2023 23:41:18 +0530
Subject: [PATCH 0619/2492] mgr/dashboard: get port from ssl_endpoint

Fixes: https://tracker.ceph.com/issues/63564
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py | 12 +++-
 src/pybind/mgr/dashboard/tests/test_rgw.py  | 73 +++++++++++++++++++--
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 8bfb0f9902d9..9bc1b489edee 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -134,6 +134,16 @@ def list(self) -> List[dict]:
             for service in server['services']:
                 metadata = service['metadata']
 
+                frontend_config = metadata['frontend_config#0']
+                port_match = re.search(r"port=(\d+)", frontend_config)
+                port = None
+                if port_match:
+                    port = port_match.group(1)
+                else:
+                    match_from_endpoint = re.search(r"endpoint=\S+:(\d+)", frontend_config)
+                    if match_from_endpoint:
+                        port = match_from_endpoint.group(1)
+
                 # extract per-daemon service data and health
                 daemon = {
                     'id': metadata['id'],
@@ -144,7 +154,7 @@ def list(self) -> List[dict]:
                     'zonegroup_name': metadata['zonegroup_name'],
                     'zone_name': metadata['zone_name'],
                     'default': instance.daemon.name == metadata['id'],
-                    'port': int(re.findall(r'port=(\d+)', metadata['frontend_config#0'])[0])
+                    'port': int(port) if port else None
                 }
 
                 daemons.append(daemon)
diff --git a/src/pybind/mgr/dashboard/tests/test_rgw.py b/src/pybind/mgr/dashboard/tests/test_rgw.py
index bfb1dbc70355..d9b8593a7d76 100644
--- a/src/pybind/mgr/dashboard/tests/test_rgw.py
+++ b/src/pybind/mgr/dashboard/tests/test_rgw.py
@@ -79,7 +79,13 @@ def test_list(self, send_command):
         RgwStub.get_settings()
         mgr.list_servers.return_value = [{
             'hostname': 'host1',
-            'services': [{'id': '4832', 'type': 'rgw'}, {'id': '5356', 'type': 'rgw'}]
+            'services': [
+                {'id': '4832', 'type': 'rgw'},
+                {'id': '5356', 'type': 'rgw'},
+                {'id': '5357', 'type': 'rgw'},
+                {'id': '5358', 'type': 'rgw'},
+                {'id': '5359', 'type': 'rgw'}
+            ]
         }]
         mgr.get_metadata.side_effect = [
             {
@@ -96,8 +102,34 @@ def test_list(self, send_command):
                 'realm_name': 'realm2',
                 'zonegroup_name': 'zg2',
                 'zone_name': 'zone2',
-                'frontend_config#0': 'beast port=80 ssl_port=443 ssl_certificate=config:/config'
-            }]
+                'frontend_config#0': 'beast ssl_port=443 ssl_certificate=config:/config'
+            },
+            {
+                'ceph_version': 'ceph version master (dev)',
+                'id': 'daemon3',
+                'realm_name': 'realm3',
+                'zonegroup_name': 'zg3',
+                'zone_name': 'zone3',
+                'frontend_config#0':
+                    'beast ssl_endpoint=0.0.0.0:8080 ssl_certificate=config:/config'
+            },
+            {
+                'ceph_version': 'ceph version master (dev)',
+                'id': 'daemon4',
+                'realm_name': 'realm4',
+                'zonegroup_name': 'zg4',
+                'zone_name': 'zone4',
+                'frontend_config#0': 'beast ssl_certificate=config:/config'
+            },
+            {
+                'ceph_version': 'ceph version master (dev)',
+                'id': 'daemon5',
+                'realm_name': 'realm5',
+                'zonegroup_name': 'zg5',
+                'zone_name': 'zone5',
+                'frontend_config#0':
+                    'beast endpoint=0.0.0.0:8445 ssl_certificate=config:/config'
+            }, ]
         self._get('/test/api/rgw/daemon')
         self.assertStatus(200)
         self.assertJsonBody([{
@@ -119,7 +151,40 @@ def test_list(self, send_command):
             'zonegroup_name': 'zg2',
             'zone_name': 'zone2',
             'default': False,
-            'port': 80
+            'port': 443,
+        },
+            {
+            'id': 'daemon3',
+            'service_map_id': '5357',
+            'version': 'ceph version master (dev)',
+            'server_hostname': 'host1',
+            'realm_name': 'realm3',
+            'zonegroup_name': 'zg3',
+            'zone_name': 'zone3',
+            'default': False,
+            'port': 8080,
+        },
+            {
+            'id': 'daemon4',
+            'service_map_id': '5358',
+            'version': 'ceph version master (dev)',
+            'server_hostname': 'host1',
+            'realm_name': 'realm4',
+            'zonegroup_name': 'zg4',
+            'zone_name': 'zone4',
+            'default': False,
+            'port': None,
+        },
+            {
+            'id': 'daemon5',
+            'service_map_id': '5359',
+            'version': 'ceph version master (dev)',
+            'server_hostname': 'host1',
+            'realm_name': 'realm5',
+            'zonegroup_name': 'zg5',
+            'zone_name': 'zone5',
+            'default': False,
+            'port': 8445,
         }])
 
     def test_list_empty(self):

From 4efd9317b83eca48c1a92724e7ebe304650fbc04 Mon Sep 17 00:00:00 2001
From: "yu.wang" <wangyu@wyu.space>
Date: Fri, 17 Nov 2023 01:38:58 +0800
Subject: [PATCH 0620/2492] ceph-menv:fix typo in README

Signed-off-by: yu.wang <wangyu@wyu.space>
---
 ceph-menv/README | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ceph-menv/README b/ceph-menv/README
index badbd3a028f4..91606c48f02e 100644
--- a/ceph-menv/README
+++ b/ceph-menv/README
@@ -1,6 +1,6 @@
 ceph-menv
 
-Environment assistant for use in conjuction with multiple ceph vstart (or more accurately mstart) clusters. Eliminates the need to specify the cluster that is being used with each and every command. Can provide a shell prompt feedback about the currently used cluster.
+Environment assistant for use in conjunction with multiple Ceph vstart (or more accurately mstart) clusters. Eliminates the need to specify the cluster that is being used with each and every command. Can provide a shell prompt feedback about the currently used cluster.
 
 
 Usage:

From b58369e916cfa5067a6aa6cb7783728a0a8a425c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:07:10 -0500
Subject: [PATCH 0621/2492] cephadm: use CustomContainer.create classmethod
 like most other blocks

Use the CustomContainer.create classmethod like most other blocks of
code in the get_container function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 348f581f9e61..5238e33e9c50 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3036,7 +3036,7 @@ def get_container(
         binds = get_container_binds(ctx, ident)
         mounts = get_container_mounts(ctx, ident)
     elif daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.init(ctx, ident.fsid, ident.daemon_id)
+        cc = CustomContainer.create(ctx, ident)
         entrypoint = cc.default_entrypoint()
         host_network = False
         cc.customize_container_envs(ctx, envs)

From 101dbb2f01e220406ef364efaf922fd50301bc1e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:20:26 -0500
Subject: [PATCH 0622/2492] cephadm: handle the case where ctx may not have a
 config attribute

This prevents a test failure in future refactoring changes.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5238e33e9c50..ad3a46faad9b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -344,10 +344,13 @@ def get_ceph_mounts(
     def customize_container_mounts(
         self, ctx: CephadmContext, mounts: Dict[str, str]
     ) -> None:
+        no_config = bool(
+            getattr(ctx, 'config', None) and self.user_supplied_config
+        )
         cm = self.get_ceph_mounts(
             ctx,
             self.identity,
-            no_config=self.ctx.config and self.user_supplied_config,
+            no_config=no_config,
         )
         mounts.update(cm)
 

From fdce99a6df1961a547733fd60cb29b15c082469c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:22:07 -0500
Subject: [PATCH 0623/2492] cephadm: convert calls to
 get_container_{mounts,binds} to method calls

In get_container, convert calls to get_container_binds and
get_container_mounts to direct uses of the ContainerDaemonForm customize
methods, avoiding constructing said ContainerDaemonForm subclasses
redundantly.
This is all done in one pass because the _update_podman_mounts function
is called by get_container_mounts and trying to convert each type at a
time would result in extra calls to that function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ad3a46faad9b..b4d60dde8441 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2985,7 +2985,7 @@ def get_container(
         ceph_daemon.customize_container_envs(ctx, envs)
         ceph_daemon.customize_container_args(ctx, container_args)
         ceph_daemon.customize_process_args(ctx, d_args)
-        mounts = get_container_mounts(ctx, ident)
+        ceph_daemon.customize_container_mounts(ctx, mounts)
     if daemon_type in ['mon', 'osd']:
         # mon and osd need privileged in order for libudev to query devices
         privileged = True
@@ -2994,7 +2994,7 @@ def get_container(
         entrypoint = monitoring.default_entrypoint()
         monitoring.customize_container_args(ctx, container_args)
         monitoring.customize_process_args(ctx, d_args)
-        mounts = get_container_mounts(ctx, ident)
+        monitoring.customize_container_mounts(ctx, mounts)
     elif daemon_type in Tracing.components:
         tracing = Tracing.create(ctx, ident)
         entrypoint = tracing.default_entrypoint()
@@ -3006,29 +3006,29 @@ def get_container(
         nfs_ganesha.customize_container_envs(ctx, envs)
         nfs_ganesha.customize_container_args(ctx, container_args)
         nfs_ganesha.customize_process_args(ctx, d_args)
-        mounts = get_container_mounts(ctx, ident)
+        nfs_ganesha.customize_container_mounts(ctx, mounts)
     elif daemon_type == CephExporter.daemon_type:
         ceph_exporter = CephExporter.create(ctx, ident)
         entrypoint = ceph_exporter.default_entrypoint()
         ceph_exporter.customize_container_envs(ctx, envs)
         ceph_exporter.customize_container_args(ctx, container_args)
         ceph_exporter.customize_process_args(ctx, d_args)
-        mounts = get_container_mounts(ctx, ident)
+        ceph_exporter.customize_container_mounts(ctx, mounts)
     elif daemon_type == HAproxy.daemon_type:
         haproxy = HAproxy.create(ctx, ident)
         haproxy.customize_container_args(ctx, container_args)
         haproxy.customize_process_args(ctx, d_args)
-        mounts = get_container_mounts(ctx, ident)
+        haproxy.customize_container_mounts(ctx, mounts)
     elif daemon_type == Keepalived.daemon_type:
         keepalived = Keepalived.create(ctx, ident)
         keepalived.customize_container_envs(ctx, envs)
         keepalived.customize_container_args(ctx, container_args)
-        mounts = get_container_mounts(ctx, ident)
+        keepalived.customize_container_mounts(ctx, mounts)
     elif daemon_type == CephNvmeof.daemon_type:
         nvmeof = CephNvmeof.create(ctx, ident)
         nvmeof.customize_container_args(ctx, container_args)
-        binds = get_container_binds(ctx, ident)
-        mounts = get_container_mounts(ctx, ident)
+        nvmeof.customize_container_binds(ctx, binds)
+        nvmeof.customize_container_mounts(ctx, mounts)
     elif daemon_type == CephIscsi.daemon_type:
         iscsi = CephIscsi.create(ctx, ident)
         entrypoint = iscsi.default_entrypoint()
@@ -3036,8 +3036,8 @@ def get_container(
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
         privileged = True
-        binds = get_container_binds(ctx, ident)
-        mounts = get_container_mounts(ctx, ident)
+        iscsi.customize_container_binds(ctx, binds)
+        iscsi.customize_container_mounts(ctx, mounts)
     elif daemon_type == CustomContainer.daemon_type:
         cc = CustomContainer.create(ctx, ident)
         entrypoint = cc.default_entrypoint()
@@ -3045,14 +3045,15 @@ def get_container(
         cc.customize_container_envs(ctx, envs)
         cc.customize_container_args(ctx, container_args)
         cc.customize_process_args(ctx, d_args)
-        binds = get_container_binds(ctx, ident)
-        mounts = get_container_mounts(ctx, ident)
+        cc.customize_container_binds(ctx, binds)
+        cc.customize_container_mounts(ctx, mounts)
     elif daemon_type == SNMPGateway.daemon_type:
         sg = SNMPGateway.create(ctx, ident)
         sg.customize_container_args(ctx, container_args)
         sg.customize_process_args(ctx, d_args)
 
     _update_container_args_for_podman(ctx, ident, container_args)
+    _update_podman_mounts(ctx, mounts)
     return CephContainer.for_daemon(
         ctx,
         ident=ident,

From a0e3ff0962c737e6a5321157fcfea0062f60dd1f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:36:41 -0500
Subject: [PATCH 0624/2492] cephadm: add daemon_to_container utility function

Add the function daemon_to_container in the container_daemon_form.py
function. This function is roughly equivalent to `get_container` in
cephadm.py but entirely relies on subclasses implementing
ContainerDaemonForm methods and avoids special casing of daemon
types.

It duplicates the functionality provided by functions in cephadm.py with
regards to special podman specific behaviors. I feel that this is a
minor issue because due to recent refactoring those functions are now
the one- or two-liners that are copied here. I gated them behind
arguments `auto_podman_mounts` and `auto_podman_args` that default to
true in order to make the function more reusable - however I don't need
them that way and if it seems like extra complexity for no win, we can
remove them later.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadmlib/container_daemon_form.py       | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index ae8b6afe3c63..693b0cf8df0b 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -4,6 +4,7 @@
 
 from typing import List, Tuple, Optional, Dict
 
+from .container_engines import Podman
 from .container_types import CephContainer, InitContainer
 from .context import CephadmContext
 from .daemon_form import DaemonForm
@@ -110,3 +111,67 @@ def default_entrypoint(self) -> str:
         in a container.
         """
         return ''
+
+
+def daemon_to_container(
+    ctx: CephadmContext,
+    daemon: ContainerDaemonForm,
+    *,
+    privileged: bool = False,
+    ptrace: bool = False,
+    host_network: bool = True,
+    entrypoint: Optional[str] = None,
+    container_args: Optional[List[str]] = None,
+    container_mounts: Optional[Dict[str, str]] = None,
+    container_binds: Optional[List[List[str]]] = None,
+    envs: Optional[List[str]] = None,
+    args: Optional[List[str]] = None,
+    auto_podman_args: bool = True,
+    auto_podman_mounts: bool = True,
+) -> CephContainer:
+    """daemon_to_container is a utility function that serves to create
+    CephContainer instances from a container daemon form's customize and
+    entrypoint methods.
+    Most of the parameters (like mounts, container_args, etc) can be passed in
+    to "pre customize" the values.
+    The auto_podman_args argument enables adding default arguments expected on
+    all podman daemons (true by default).
+    The auto_podman_mounts argument enables adding mounts expected on all
+    daemons running on podman (true by default).
+    """
+    container_args = container_args if container_args else []
+    container_mounts = container_mounts if container_mounts else {}
+    container_binds = container_binds if container_binds else []
+    envs = envs if envs else []
+    args = args if args else []
+
+    if entrypoint is None:
+        entrypoint = daemon.default_entrypoint()
+    daemon.customize_container_args(ctx, container_args)
+    daemon.customize_container_mounts(ctx, container_mounts)
+    daemon.customize_container_binds(ctx, container_binds)
+    daemon.customize_container_envs(ctx, envs)
+    daemon.customize_process_args(ctx, args)
+
+    _is_podman = isinstance(ctx.container_engine, Podman)
+    if auto_podman_mounts and _is_podman:
+        ctx.container_engine.update_mounts(ctx, container_mounts)
+    if auto_podman_args and _is_podman:
+        service_name = f'{daemon.identity.unit_name}.service'
+        container_args.extend(
+            ctx.container_engine.service_args(ctx, service_name)
+        )
+
+    return CephContainer.for_daemon(
+        ctx,
+        ident=daemon.identity,
+        entrypoint=entrypoint,
+        args=args,
+        container_args=container_args,
+        volume_mounts=container_mounts,
+        bind_mounts=container_binds,
+        envs=envs,
+        privileged=privileged,
+        ptrace=ptrace,
+        host_network=host_network,
+    )

From 32cfb80d81e092a845d1baab0ad39299738a47cb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:41:29 -0500
Subject: [PATCH 0625/2492] cephadm: switch ceph class to use
 daemon_to_container function

Switch the ceph daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b4d60dde8441..fb39a9eb03aa 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -169,7 +169,10 @@
     register as register_daemon_form,
 )
 from cephadmlib.deploy import DeploymentType
-from cephadmlib.container_daemon_form import ContainerDaemonForm
+from cephadmlib.container_daemon_form import (
+    ContainerDaemonForm,
+    daemon_to_container,
+)
 from cephadmlib.sysctl import install_sysctl, migrate_sysctl_dir
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
@@ -246,7 +249,9 @@ def container(self, ctx: CephadmContext) -> CephContainer:
         uid, gid = self.uid_gid(ctx)
         make_var_run(ctx, ctx.fsid, uid, gid)
 
-        ctr = get_container(ctx, self.identity)
+        # mon and osd need privileged in order for libudev to query devices
+        privileged = self.identity.daemon_type in ['mon', 'osd']
+        ctr = daemon_to_container(ctx, self, privileged=privileged)
         ctr = to_deployment_container(ctx, ctr)
         config_json = fetch_configs(ctx)
         if self.identity.daemon_type == 'mon' and config_json is not None:

From 560409a940b67d8d1dc4a86efc01ad0bd8bd8bbb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 13:43:44 -0500
Subject: [PATCH 0626/2492] cephadm: switch snmp class to use
 daemon_to_container function

Switch the snmp daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fb39a9eb03aa..1cde594be101 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -600,7 +600,7 @@ def validate(self) -> None:
             raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:

From 14566e229698dcceb5c054e7d8faedf507035efc Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:16:44 -0500
Subject: [PATCH 0627/2492] cephadm: switch monitoring class to use
 daemon_to_container function

Switch the monitoring daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 1cde594be101..e96d44b34f20 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -774,7 +774,7 @@ def identity(self) -> DaemonIdentity:
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         self._prevalidate(ctx)
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:

From 8176c47e9a50afb7cba0b4e06f3102dad8cb8b6c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:17:10 -0500
Subject: [PATCH 0628/2492] cephadm: switch nfs class to use
 daemon_to_container function

Switch the nfs daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index e96d44b34f20..fd213ffd6c85 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1105,7 +1105,7 @@ def firewall_service_name(self) -> str:
         return 'nfs'
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def customize_container_endpoints(

From 4f567ba2afe5d6227c514f66849487b99fc69c90 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:17:33 -0500
Subject: [PATCH 0629/2492] cephadm: switch iscsi class to use
 daemon_to_container function

Switch the iscsi daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fd213ffd6c85..7d91d90dccbd 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1355,7 +1355,9 @@ def get_tcmu_runner_container(self):
         return tcmu_container
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        # So the container can modprobe iscsi_target_mod and have write perms
+        # to configfs we need to make this a privileged container.
+        ctr = daemon_to_container(ctx, self, privileged=True)
         return to_deployment_container(ctx, ctr)
 
     def config_and_keyring(

From 46ab2e3aa42b511f87fbcd66c4d9d66bb86d787b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:17:47 -0500
Subject: [PATCH 0630/2492] cephadm: switch nvmeof class to use
 daemon_to_container function

Switch the nvmeof daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7d91d90dccbd..524a73a37009 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1520,7 +1520,7 @@ def get_sysctl_settings() -> List[str]:
         ]
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:

From 0c5427064b12e0b7d78451821fd2170dd1d07e18 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:18:08 -0500
Subject: [PATCH 0631/2492] cephadm: switch ceph exporter class to use
 daemon_to_container function

Switch the ceph exporter daemon type class to call daemon_to_container
instead of get_container in the class's container method. This breaks a
dependency loop between the class and the get_container function
hopefully enabling future clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 524a73a37009..4d8a8c9bf950 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1607,7 +1607,7 @@ def validate(self) -> None:
             raise Error(f'Directory does not exist. Got: {self.sock_dir}')
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:

From 20c1e74c2fce164bc2ac7c0577e5116ac63c7f55 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:18:19 -0500
Subject: [PATCH 0632/2492] cephadm: switch haproxy class to use
 daemon_to_container function

Switch the haproxy daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 4d8a8c9bf950..0dda9b8b4fa4 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1753,7 +1753,7 @@ def get_sysctl_settings() -> List[str]:
         ]
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def customize_container_args(

From 20d91cafba61eed3a7bbd46b1de38c692b3acd38 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:18:33 -0500
Subject: [PATCH 0633/2492] cephadm: switch keepalived class to use
 daemon_to_container function

Switch the keepalived daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 0dda9b8b4fa4..fddb63e4a90f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1885,7 +1885,7 @@ def customize_container_mounts(
         mounts.update(self._get_container_mounts(data_dir))
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def customize_container_envs(

From de2547dd26098de862a9e9b08893cd2bb43df191 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:18:49 -0500
Subject: [PATCH 0634/2492] cephadm: switch tracing class to use
 daemon_to_container function

Switch the tracing daemon type class to call daemon_to_container instead of
get_container in the class's container method. This breaks a dependency loop
between the class and the get_container function hopefully enabling future
clean ups and code moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fddb63e4a90f..c5ea9c5bbe08 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1964,7 +1964,7 @@ def identity(self) -> DaemonIdentity:
         return self._identity
 
     def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = get_container(ctx, self.identity)
+        ctr = daemon_to_container(ctx, self)
         return to_deployment_container(ctx, ctr)
 
     def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:

From 66d4a47ca8c6588409e9d0f5975bab36fc37a8e4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:18:57 -0500
Subject: [PATCH 0635/2492] cephadm: switch custom container class to use
 daemon_to_container function

Switch the custom container daemon type class to call
daemon_to_container instead of get_container in the class's container
method. This breaks a dependency loop between the class and the
get_container function hopefully enabling future clean ups and code
moves.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c5ea9c5bbe08..70c36d8c57e4 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2142,9 +2142,10 @@ def customize_container_binds(
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         if self._container is None:
-            ctr = get_container(
+            ctr = daemon_to_container(
                 ctx,
-                self.identity,
+                self,
+                host_network=False,
                 privileged=self.privileged,
                 ptrace=ctx.allow_ptrace,
             )

From 5b8a37b561d117edcd850232dab24d597e9dd836 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 15:22:02 -0500
Subject: [PATCH 0636/2492] cephadm: rework get_container in terms of
 daemon_to_container

With get_container basically equivalent to daemon_to_container except
one creates a ContainerDaemonForm and the other requires one, we
can rework get_container to act as a thin wrapper around
daemon_to_container.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 107 +++--------------------------------------
 1 file changed, 6 insertions(+), 101 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 70c36d8c57e4..8a1357d3fab0 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2972,108 +2972,13 @@ def get_ceph_volume_container(ctx: CephadmContext,
 def get_container(
     ctx: CephadmContext,
     ident: 'DaemonIdentity',
-    privileged: bool = False,
-    ptrace: bool = False,
-    container_args: Optional[List[str]] = None,
 ) -> 'CephContainer':
-    entrypoint: str = ''
-    d_args: List[str] = []
-    envs: List[str] = []
-    host_network: bool = True
-    binds: List[List[str]] = []
-    mounts: Dict[str, str] = {}
-
-    daemon_type = ident.daemon_type
-    if container_args is None:
-        container_args = []
-    if Ceph.for_daemon_type(daemon_type) or OSD.for_daemon_type(daemon_type):
-        ceph_daemon = daemon_form_create(ctx, ident)
-        assert isinstance(ceph_daemon, ContainerDaemonForm)
-        entrypoint = ceph_daemon.default_entrypoint()
-        ceph_daemon.customize_container_envs(ctx, envs)
-        ceph_daemon.customize_container_args(ctx, container_args)
-        ceph_daemon.customize_process_args(ctx, d_args)
-        ceph_daemon.customize_container_mounts(ctx, mounts)
-    if daemon_type in ['mon', 'osd']:
-        # mon and osd need privileged in order for libudev to query devices
-        privileged = True
-    if daemon_type in Monitoring.components:
-        monitoring = Monitoring.create(ctx, ident)
-        entrypoint = monitoring.default_entrypoint()
-        monitoring.customize_container_args(ctx, container_args)
-        monitoring.customize_process_args(ctx, d_args)
-        monitoring.customize_container_mounts(ctx, mounts)
-    elif daemon_type in Tracing.components:
-        tracing = Tracing.create(ctx, ident)
-        entrypoint = tracing.default_entrypoint()
-        tracing.customize_container_envs(ctx, envs)
-        tracing.customize_process_args(ctx, d_args)
-    elif daemon_type == NFSGanesha.daemon_type:
-        nfs_ganesha = NFSGanesha.create(ctx, ident)
-        entrypoint = nfs_ganesha.default_entrypoint()
-        nfs_ganesha.customize_container_envs(ctx, envs)
-        nfs_ganesha.customize_container_args(ctx, container_args)
-        nfs_ganesha.customize_process_args(ctx, d_args)
-        nfs_ganesha.customize_container_mounts(ctx, mounts)
-    elif daemon_type == CephExporter.daemon_type:
-        ceph_exporter = CephExporter.create(ctx, ident)
-        entrypoint = ceph_exporter.default_entrypoint()
-        ceph_exporter.customize_container_envs(ctx, envs)
-        ceph_exporter.customize_container_args(ctx, container_args)
-        ceph_exporter.customize_process_args(ctx, d_args)
-        ceph_exporter.customize_container_mounts(ctx, mounts)
-    elif daemon_type == HAproxy.daemon_type:
-        haproxy = HAproxy.create(ctx, ident)
-        haproxy.customize_container_args(ctx, container_args)
-        haproxy.customize_process_args(ctx, d_args)
-        haproxy.customize_container_mounts(ctx, mounts)
-    elif daemon_type == Keepalived.daemon_type:
-        keepalived = Keepalived.create(ctx, ident)
-        keepalived.customize_container_envs(ctx, envs)
-        keepalived.customize_container_args(ctx, container_args)
-        keepalived.customize_container_mounts(ctx, mounts)
-    elif daemon_type == CephNvmeof.daemon_type:
-        nvmeof = CephNvmeof.create(ctx, ident)
-        nvmeof.customize_container_args(ctx, container_args)
-        nvmeof.customize_container_binds(ctx, binds)
-        nvmeof.customize_container_mounts(ctx, mounts)
-    elif daemon_type == CephIscsi.daemon_type:
-        iscsi = CephIscsi.create(ctx, ident)
-        entrypoint = iscsi.default_entrypoint()
-        iscsi.customize_container_args(ctx, container_args)
-        # So the container can modprobe iscsi_target_mod and have write perms
-        # to configfs we need to make this a privileged container.
-        privileged = True
-        iscsi.customize_container_binds(ctx, binds)
-        iscsi.customize_container_mounts(ctx, mounts)
-    elif daemon_type == CustomContainer.daemon_type:
-        cc = CustomContainer.create(ctx, ident)
-        entrypoint = cc.default_entrypoint()
-        host_network = False
-        cc.customize_container_envs(ctx, envs)
-        cc.customize_container_args(ctx, container_args)
-        cc.customize_process_args(ctx, d_args)
-        cc.customize_container_binds(ctx, binds)
-        cc.customize_container_mounts(ctx, mounts)
-    elif daemon_type == SNMPGateway.daemon_type:
-        sg = SNMPGateway.create(ctx, ident)
-        sg.customize_container_args(ctx, container_args)
-        sg.customize_process_args(ctx, d_args)
-
-    _update_container_args_for_podman(ctx, ident, container_args)
-    _update_podman_mounts(ctx, mounts)
-    return CephContainer.for_daemon(
-        ctx,
-        ident=ident,
-        entrypoint=entrypoint,
-        args=d_args,
-        container_args=container_args,
-        volume_mounts=mounts,
-        bind_mounts=binds,
-        envs=envs,
-        privileged=privileged,
-        ptrace=ptrace,
-        host_network=host_network,
+    daemon = daemon_form_create(ctx, ident)
+    assert isinstance(daemon, ContainerDaemonForm)
+    privileged = ident.daemon_type in {'mon', 'osd', CephIscsi.daemon_type}
+    host_network = ident.daemon_type != CustomContainer.daemon_type
+    return daemon_to_container(
+        ctx, daemon, privileged=privileged, host_network=host_network
     )
 
 

From a4e7f355fc16b2b88e8d72c16cb4d72f14f356fe Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 11:54:42 -0500
Subject: [PATCH 0637/2492] cephadm: remove call to get_container from iscsi
 tcmu runner method

Remove the call to get_container from the iscsi class's
get_tcmu_runner_container method, replacing it with a call to the
daemon_to_container function. While they're roughly equivalent -
we can consider get_container legacy and only for use in cephadm.py
where daemon_to_container is more general.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8a1357d3fab0..959676ba3af1 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1159,15 +1159,12 @@ def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
     def __init__(self,
-                 ctx,
-                 fsid,
-                 daemon_id,
-                 config_json,
-                 image=DEFAULT_IMAGE):
-        # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
+                 ctx: CephadmContext,
+                 ident: DaemonIdentity,
+                 config_json: Dict,
+                 image: str = DEFAULT_IMAGE):
         self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
+        self._identity = ident
         self.image = image
 
         # config-json options
@@ -1177,18 +1174,24 @@ def __init__(self,
         self.validate()
 
     @classmethod
-    def init(cls, ctx, fsid, daemon_id):
-        # type: (CephadmContext, str, Union[int, str]) -> CephIscsi
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
+    def init(cls, ctx: CephadmContext, fsid: str, daemon_id: str) -> 'CephIscsi':
+        return cls.create(ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id))
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephIscsi':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
+        return cls(ctx, ident, fetch_configs(ctx), ctx.image)
 
     @property
     def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+        return self._identity
+
+    @property
+    def fsid(self) -> str:
+        return self._identity.fsid
+
+    @property
+    def daemon_id(self) -> str:
+        return self._identity.daemon_id
 
     @staticmethod
     def _get_container_mounts(data_dir, log_dir):
@@ -1344,8 +1347,9 @@ def get_tcmu_runner_container(self):
         subident = DaemonSubIdentity(
             self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
         )
+        tcmu_dmn = self.create(self.ctx, subident)
         tcmu_container = to_deployment_container(
-            self.ctx, get_container(self.ctx, subident)
+            self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True)
         )
         # TODO: Eventually we don't want to run tcmu-runner through this script.
         # This is intended to be a workaround backported to older releases

From 2921dacf1e7b7ad2aa62f52d6e8914d8ec50a610 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 16 Nov 2023 21:14:13 +1000
Subject: [PATCH 0638/2492] doc/man: edit ceph-osd description

Improve the English grammar and reduce the ambiguity and unclear writing
in the "Description" section of doc/man/8/ceph-osd.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/man/8/ceph-osd.rst | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/doc/man/8/ceph-osd.rst b/doc/man/8/ceph-osd.rst
index 5b631eeff744..4b038437bbd9 100644
--- a/doc/man/8/ceph-osd.rst
+++ b/doc/man/8/ceph-osd.rst
@@ -18,14 +18,16 @@ Synopsis
 Description
 ===========
 
-**ceph-osd** is the object storage daemon for the Ceph distributed file
-system. It is responsible for storing objects on a local file system
-and providing access to them over the network.
-
-The datapath argument should be a directory on a xfs file system
-where the object data resides. The journal is optional, and is only
-useful performance-wise when it resides on a different disk than
-datapath with low latency (ideally, an NVRAM device).
+**ceph-osd** is the **o**\bject **s**\torage **d**\aemon for the Ceph
+distributed file system. It manages data on local storage with redundancy and
+provides access to that data over the network. 
+
+For Filestore-backed clusters, the argument of the ``--osd-data datapath``
+option (which is ``datapath`` in this example) should be a directory on an XFS
+file system where the object data resides. The journal is optional. The journal
+improves performance only when it resides on a different disk than the disk
+specified by ``datapath`` . The storage medium on which the journal is stored
+should be a low-latency medium (ideally, an SSD device).
 
 
 Options

From 77a4e0b2e1c3aea71ebf5001c877e9f2e885ce22 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 12 Nov 2023 05:12:31 -0600
Subject: [PATCH 0639/2492] osd/scrub: fix replica sub-states

The replica states hierarchy was not properly specified.
Fixing it here.
Also - removing the (never yet invoked) call to
replica_handling_done() in the replica state machine.
That call must precede the reply to the Primary.

Fixes: https://tracker.ceph.com/issues/63509

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_machine.cc | 41 ++++++++++++++++++-------------
 src/osd/scrubber/scrub_machine.h  |  6 ++---
 2 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 40b43b6e0770..99286acaa12b 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -674,38 +674,42 @@ ReservedReplica::~ReservedReplica()
 
 ReplicaIdle::ReplicaIdle(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaIdle")
-{
-  dout(10) << "-- state -->> ReplicaIdle" << dendl;
-}
-
-ReplicaIdle::~ReplicaIdle()
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReservedReplica/ReplicaIdle")
 {
+  dout(10) << "-- state -->> ReservedReplica/ReplicaIdle" << dendl;
 }
 
+ReplicaIdle::~ReplicaIdle() = default;
 
 // ----------------------- ReplicaActiveOp --------------------------------
 
 ReplicaActiveOp::ReplicaActiveOp(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActiveOp")
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReservedReplica/ReplicaActiveOp")
 {
-  dout(10) << "-- state -->> ReplicaActiveOp" << dendl;
+  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp" << dendl;
 }
 
-ReplicaActiveOp::~ReplicaActiveOp()
-{
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  scrbr->replica_handling_done();
-}
+/**
+ * \note: here is too late to call replica_handling_done(). See the
+ * comment in build_replica_map_chunk()
+ */
+ReplicaActiveOp::~ReplicaActiveOp() = default;
 
 // ----------------------- ReplicaWaitUpdates --------------------------------
 
 ReplicaWaitUpdates::ReplicaWaitUpdates(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaWaitUpdates")
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReservedReplica/ReplicaActiveOp/ReplicaWaitUpdates")
 {
-  dout(10) << "-- state -->> ReplicaWaitUpdates" << dendl;
+  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp/ReplicaWaitUpdates"
+	   << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   scrbr->on_replica_init();
 }
@@ -732,10 +736,13 @@ sc::result ReplicaWaitUpdates::react(const ReplicaPushesUpd&)
 
 ReplicaBuildingMap::ReplicaBuildingMap(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaBuildingMap")
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReservedReplica/ReplicaActiveOp/ReplicaBuildingMap")
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "-- state -->> ReplicaBuildingMap" << dendl;
+  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp/ReplicaBuildingMap"
+	   << dendl;
   // and as we might have skipped ReplicaWaitUpdates:
   scrbr->on_replica_init();
   post_event(SchedReplica{});
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index cbce07fe183c..a67a85b8e112 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -628,7 +628,7 @@ struct ReplicaIdle : sc::state<ReplicaIdle, ReservedReplica>,
 };
 
 /**
- * ReservedActiveOp
+ * ReplicaActiveOp
  *
  * Lifetime matches handling for a single map request op
  */
@@ -646,7 +646,7 @@ struct ReplicaActiveOp
  * - the details of the Primary's request were internalized by PgScrubber;
  * - 'active' scrubbing is set
  */
-struct ReplicaWaitUpdates : sc::state<ReplicaWaitUpdates, ReservedReplica>,
+struct ReplicaWaitUpdates : sc::state<ReplicaWaitUpdates, ReplicaActiveOp>,
 			    NamedSimply {
   explicit ReplicaWaitUpdates(my_context ctx);
   using reactions = mpl::list<sc::custom_reaction<ReplicaPushesUpd>>;
@@ -655,7 +655,7 @@ struct ReplicaWaitUpdates : sc::state<ReplicaWaitUpdates, ReservedReplica>,
 };
 
 
-struct ReplicaBuildingMap : sc::state<ReplicaBuildingMap, ReservedReplica>
+struct ReplicaBuildingMap : sc::state<ReplicaBuildingMap, ReplicaActiveOp>
 			  , NamedSimply {
   explicit ReplicaBuildingMap(my_context ctx);
   using reactions = mpl::list<sc::custom_reaction<SchedReplica>>;

From 116c72bf1b5afd4eb5e54ce4dd0a8690fdbb03b7 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 12 Nov 2023 09:25:22 -0600
Subject: [PATCH 0640/2492] osd/scrub: maintain a set of remote reservations

... instead of a simple counter.

This - as a preparation for the next commit, which will decouple
the "being reserved" state from the handling of scrub requests.
The planned changes to the scrub state machine will make
it harder to know when to clear the "being reserved" state.
The changes here would allow us to err on the side of caution,
i.e. trying to "un-count" a remote reservation even if it was not
actually reserved or was already deleted.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/scrub/osd-scrub-dump.sh |  2 +-
 src/osd/scrubber/osd_scrub.cc         |  8 ++--
 src/osd/scrubber/osd_scrub.h          |  4 +-
 src/osd/scrubber/pg_scrubber.cc       |  4 +-
 src/osd/scrubber/scrub_resources.cc   | 54 +++++++++++++++++----------
 src/osd/scrubber/scrub_resources.h    | 10 +++--
 6 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/qa/standalone/scrub/osd-scrub-dump.sh b/qa/standalone/scrub/osd-scrub-dump.sh
index 0129114625aa..644f82d80716 100755
--- a/qa/standalone/scrub/osd-scrub-dump.sh
+++ b/qa/standalone/scrub/osd-scrub-dump.sh
@@ -123,7 +123,7 @@ function TEST_recover_unexpected() {
 	for o in $(seq 0 $(expr $OSDS - 1))
 	do
 		CEPH_ARGS='' ceph daemon $(get_asok_path osd.$o) dump_scrub_reservations
-		scrubs=$(CEPH_ARGS='' ceph daemon $(get_asok_path osd.$o) dump_scrub_reservations | jq '.scrubs_local + .scrubs_remote')
+		scrubs=$(CEPH_ARGS='' ceph daemon $(get_asok_path osd.$o) dump_scrub_reservations | jq '.scrubs_local + .granted_reservations')
 		if [ $scrubs -gt $MAX_SCRUBS ]; then
 		    echo "ERROR: More than $MAX_SCRUBS currently reserved"
 		    return 1
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index e3a71e262347..99367170dbac 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -441,14 +441,14 @@ void OsdScrub::dec_scrubs_local()
   m_resource_bookkeeper.dec_scrubs_local();
 }
 
-bool OsdScrub::inc_scrubs_remote()
+bool OsdScrub::inc_scrubs_remote(pg_t pgid)
 {
-  return m_resource_bookkeeper.inc_scrubs_remote();
+  return m_resource_bookkeeper.inc_scrubs_remote(pgid);
 }
 
-void OsdScrub::dec_scrubs_remote()
+void OsdScrub::dec_scrubs_remote(pg_t pgid)
 {
-  m_resource_bookkeeper.dec_scrubs_remote();
+  m_resource_bookkeeper.dec_scrubs_remote(pgid);
 }
 
 void OsdScrub::mark_pg_scrub_blocked(spg_t blocked_pg)
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 570430660ed0..56167df2ee6e 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -67,8 +67,8 @@ class OsdScrub {
   // updating the resource counters
   bool inc_scrubs_local();
   void dec_scrubs_local();
-  bool inc_scrubs_remote();
-  void dec_scrubs_remote();
+  bool inc_scrubs_remote(pg_t pgid);
+  void dec_scrubs_remote(pg_t pgid);
 
   // counting the number of PGs stuck while scrubbing, waiting for objects
   void mark_pg_scrub_blocked(spg_t blocked_pg);
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 4cd861b89c8f..70d314f0d2f6 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -218,7 +218,7 @@ void PgScrubber::initiate_regular_scrub(epoch_t epoch_queued)
 
 void PgScrubber::dec_scrubs_remote()
 {
-  m_osds->get_scrub_services().dec_scrubs_remote();
+  m_osds->get_scrub_services().dec_scrubs_remote(m_pg_id.pgid);
 }
 
 void PgScrubber::advance_token()
@@ -1708,7 +1708,7 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
   if (m_pg->cct->_conf->osd_scrub_during_recovery ||
       !m_osds->is_recovery_active()) {
 
-    granted = m_osds->get_scrub_services().inc_scrubs_remote();
+    granted = m_osds->get_scrub_services().inc_scrubs_remote(m_pg_id.pgid);
     if (granted) {
       m_fsm->process_event(ReplicaGrantReservation{});
     } else {
diff --git a/src/osd/scrubber/scrub_resources.cc b/src/osd/scrubber/scrub_resources.cc
index 179bd5e7e0e6..25dcec2399f1 100644
--- a/src/osd/scrubber/scrub_resources.cc
+++ b/src/osd/scrubber/scrub_resources.cc
@@ -4,10 +4,12 @@
 #include "./scrub_resources.h"
 
 #include <fmt/format.h>
+#include <fmt/ranges.h>
 
 #include "common/debug.h"
 
 #include "include/ceph_assert.h"
+#include "osd/osd_types_fmt.h"
 
 
 using ScrubResources = Scrub::ScrubResources;
@@ -22,25 +24,25 @@ ScrubResources::ScrubResources(
 bool ScrubResources::can_inc_scrubs() const
 {
   std::lock_guard lck{resource_lock};
-  if (scrubs_local + scrubs_remote < conf->osd_max_scrubs) {
+  if (scrubs_local + granted_reservations.size() < conf->osd_max_scrubs) {
     return true;
   }
   log_upwards(fmt::format(
       "{}== false. {} (local) + {} (remote) >= max ({})", __func__,
-      scrubs_local, scrubs_remote, conf->osd_max_scrubs));
+      scrubs_local, granted_reservations.size(), conf->osd_max_scrubs));
   return false;
 }
 
 bool ScrubResources::inc_scrubs_local()
 {
   std::lock_guard lck{resource_lock};
-  if (scrubs_local + scrubs_remote < conf->osd_max_scrubs) {
+  if (scrubs_local + granted_reservations.size() < conf->osd_max_scrubs) {
     ++scrubs_local;
     return true;
   }
   log_upwards(fmt::format(
       "{}: {} (local) + {} (remote) >= max ({})", __func__, scrubs_local,
-      scrubs_remote, conf->osd_max_scrubs));
+      granted_reservations.size(), conf->osd_max_scrubs));
   return false;
 }
 
@@ -49,42 +51,56 @@ void ScrubResources::dec_scrubs_local()
   std::lock_guard lck{resource_lock};
   log_upwards(fmt::format(
       "{}: {} -> {} (max {}, remote {})", __func__, scrubs_local,
-      (scrubs_local - 1), conf->osd_max_scrubs, scrubs_remote));
+      (scrubs_local - 1), conf->osd_max_scrubs, granted_reservations.size()));
   --scrubs_local;
   ceph_assert(scrubs_local >= 0);
 }
 
-bool ScrubResources::inc_scrubs_remote()
+bool ScrubResources::inc_scrubs_remote(pg_t pgid)
 {
   std::lock_guard lck{resource_lock};
-  if (scrubs_local + scrubs_remote < conf->osd_max_scrubs) {
+
+  // if this PG is already reserved - it's probably a benign bug.
+  // report it, but do not fail the reservation.
+  if (granted_reservations.contains(pgid)) {
+    log_upwards(fmt::format("{}: pg[{}] already reserved", __func__, pgid));
+    return true;
+  }
+
+  auto prev = granted_reservations.size();
+  if (scrubs_local + prev < conf->osd_max_scrubs) {
+    granted_reservations.insert(pgid);
     log_upwards(fmt::format(
-	"{}: {} -> {} (max {}, local {})", __func__, scrubs_remote,
-	(scrubs_remote + 1), conf->osd_max_scrubs, scrubs_local));
-    ++scrubs_remote;
+	"{}: pg[{}] {} -> {} (max {}, local {})", __func__, pgid, prev,
+	granted_reservations.size(), conf->osd_max_scrubs, scrubs_local));
     return true;
   }
 
   log_upwards(fmt::format(
-      "{}: {} (local) + {} (remote) >= max ({})", __func__, scrubs_local,
-      scrubs_remote, conf->osd_max_scrubs));
+      "{}: pg[{}] {} (local) + {} (remote) >= max ({})", __func__, pgid,
+      scrubs_local, granted_reservations.size(), conf->osd_max_scrubs));
   return false;
 }
 
-void ScrubResources::dec_scrubs_remote()
+void ScrubResources::dec_scrubs_remote(pg_t pgid)
 {
   std::lock_guard lck{resource_lock};
-  log_upwards(fmt::format(
-      "{}: {} -> {} (max {}, local {})", __func__, scrubs_remote,
-      (scrubs_remote - 1), conf->osd_max_scrubs, scrubs_local));
-  --scrubs_remote;
-  ceph_assert(scrubs_remote >= 0);
+  // we might not have this PG in the set (e.g. if we are concluding a
+  // high priority scrub, one that does not require reservations)
+  auto cnt = granted_reservations.erase(pgid);
+  if (cnt) {
+    log_upwards(fmt::format(
+	"{}: remote reservation for {} removed -> {} (max {}, local {})",
+	__func__, pgid, granted_reservations.size(), conf->osd_max_scrubs,
+	scrubs_local));
+  }
 }
 
 void ScrubResources::dump_scrub_reservations(ceph::Formatter* f) const
 {
   std::lock_guard lck{resource_lock};
   f->dump_int("scrubs_local", scrubs_local);
-  f->dump_int("scrubs_remote", scrubs_remote);
+  f->dump_int("granted_reservations", granted_reservations.size());
+  f->dump_string("PGs being served", fmt::format("{}", granted_reservations));
   f->dump_int("osd_max_scrubs", conf->osd_max_scrubs);
 }
diff --git a/src/osd/scrubber/scrub_resources.h b/src/osd/scrubber/scrub_resources.h
index 890ee5d0e2fa..724e206ee27e 100644
--- a/src/osd/scrubber/scrub_resources.h
+++ b/src/osd/scrubber/scrub_resources.h
@@ -8,6 +8,7 @@
 #include "common/ceph_mutex.h"
 #include "common/config_proxy.h"
 #include "common/Formatter.h"
+#include "osd/osd_types.h"
 
 namespace Scrub {
 
@@ -28,8 +29,9 @@ class ScrubResources {
   /// the number of concurrent scrubs performed by Primaries on this OSD
   int scrubs_local{0};
 
-  /// the number of active scrub reservations granted by replicas
-  int scrubs_remote{0};
+  /// the set of PGs that have active scrub reservations as replicas
+  /// \todo come C++23 - consider std::flat_set<pg_t>
+  std::set<pg_t> granted_reservations;
 
   mutable ceph::mutex resource_lock =
       ceph::make_mutex("ScrubQueue::resource_lock");
@@ -56,10 +58,10 @@ class ScrubResources {
   void dec_scrubs_local();
 
   /// increments the number of scrubs acting as a Replica
-  bool inc_scrubs_remote();
+  bool inc_scrubs_remote(pg_t pgid);
 
   /// decrements the number of scrubs acting as a Replica
-  void dec_scrubs_remote();
+  void dec_scrubs_remote(pg_t pgid);
 
   void dump_scrub_reservations(ceph::Formatter* f) const;
 };

From a78fe85470c2471574aceb723cd304498cde1afb Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 17 Nov 2023 19:24:14 +1000
Subject: [PATCH 0641/2492] doc/start: explain "OSD"

Explain the initialism "OSD" and link to its definition in the glossary.
This PR is raised in response to an anonymous documentation bug that reads

  "Paragraph 2 uses the acronym OSD without any explanation.
   This makes it very difficult to understand this part of
   the documentation as there is no indication of what this
   acronym is until much further into the documentation. Replace
   first occurence of OSD with Object Storage Daemon (OSD) or
   link it to the glossary."
     -- https://pad.ceph.com/p/Report_Documentation_Bugs

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/start/intro.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/start/intro.rst b/doc/start/intro.rst
index 3a50a8733676..9881a7205487 100644
--- a/doc/start/intro.rst
+++ b/doc/start/intro.rst
@@ -9,10 +9,10 @@ System`.  All :term:`Ceph Storage Cluster` deployments begin with setting up
 each :term:`Ceph Node` and then setting up the network. 
 
 A Ceph Storage Cluster requires the following: at least one Ceph Monitor and at
-least one Ceph Manager, and at least as many Ceph OSDs as there are copies of
-an object stored on the Ceph cluster (for example, if three copies of a given
-object are stored on the Ceph cluster, then at least three OSDs must exist in
-that Ceph cluster). 
+least one Ceph Manager, and at least as many :term:`Ceph Object Storage
+Daemon<Ceph OSD>`\s (OSDs) as there are copies of a given object stored in the
+Ceph cluster (for example, if three copies of a given object are stored in the
+Ceph cluster, then at least three OSDs must exist in that Ceph cluster).
 
 The Ceph Metadata Server is necessary to run Ceph File System clients.
 

From e3ececd293c64a76565293616b93b8b87550f8d2 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Sat, 18 Nov 2023 20:14:56 +0100
Subject: [PATCH 0642/2492] rgw: add s3select usage to log usage

Expose bytes processed and bytes returned to log usage.

Fixes: https://tracker.ceph.com/issues/63563
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 qa/tasks/radosgw_admin.py           | 61 ++++++++++++++++++++---------
 src/cls/rgw/cls_rgw_types.cc        | 22 +++++++++++
 src/cls/rgw/cls_rgw_types.h         | 49 +++++++++++++++++++++--
 src/rgw/rgw_common.h                |  1 +
 src/rgw/rgw_log.cc                  |  7 +++-
 src/rgw/rgw_rest_s3.cc              | 11 ++++++
 src/rgw/rgw_s3select.cc             |  2 +
 src/rgw/rgw_usage.cc                | 11 ++++++
 src/tools/ceph-dencoder/rgw_types.h |  1 +
 9 files changed, 140 insertions(+), 25 deletions(-)

diff --git a/qa/tasks/radosgw_admin.py b/qa/tasks/radosgw_admin.py
index 780dae1e15a6..965eb498bc6a 100644
--- a/qa/tasks/radosgw_admin.py
+++ b/qa/tasks/radosgw_admin.py
@@ -58,7 +58,8 @@ def usage_acc_findsum2(summaries, user, add=True):
         return None
     e = {'user': user, 'categories': [],
         'total': {'bytes_received': 0,
-            'bytes_sent': 0, 'ops': 0, 'successful_ops': 0 }}
+            'bytes_sent': 0, 'ops': 0, 'successful_ops': 0,
+            'bytes_processed': 0, 'bytes_returned': 0}}
     summaries.append(e)
     return e
 def usage_acc_update2(x, out, b_in, err):
@@ -78,6 +79,17 @@ def usage_acc_validate_fields(r, x, x2, what):
             return
     if len(q) > 0:
         r.append("incomplete counts in " + what + ": " + ", ".join(q))
+def usage_acc_validate_s3select_fields(r, x, x2, what):
+    q=[]
+    for field in ['bytes_processed', 'bytes_returned']:
+        try:
+            if x2[field] < x[field]:
+                q.append("field %s: %d < %d" % (field, x2[field], x[field]))
+        except Exception as ex:
+            r.append( "missing/bad field " + field + " in " + what + " " + str(ex))
+            return
+    if len(q) > 0:
+        r.append("incomplete counts in " + what + ": " + ", ".join(q))
 class usage_acc:
     def __init__(self):
         self.results = {'entries': [], 'summary': []}
@@ -91,7 +103,9 @@ def e2b(self, e, bucket, add=True):
                 return b
         if not add:
                 return None
-        b = {'bucket': bucket, 'categories': []}
+        b = {'bucket': bucket, 'categories': [], 's3select': {
+            'bytes_processed': 0, 'bytes_returned': 0,
+        }}
         e['buckets'].append(b)
         return b
     def c2x(self, c, cat, add=True):
@@ -145,60 +159,69 @@ def compare_results(self, results):
                 try:
                     b2 = self.e2b(e2, b['bucket'], False)
                     if b2 != None:
-                            c2 = b2['categories']
+                        c2 = b2['categories']
                 except Exception as ex:
                     r.append("malformed entry looking for bucket "
-			+ b['bucket'] + " in user " + e['user'] + " " + str(ex))
+                        + b['bucket'] + " in user " + e['user'] + " " + str(ex))
                     break
                 if b2 == None:
                     r.append("can't find bucket " + b['bucket']
-			+ " in user " + e['user'])
+                        + " in user " + e['user'])
                     continue
                 for x in c:
                     try:
                         x2 = self.c2x(c2, x['category'], False)
                     except Exception as ex:
                         r.append("malformed entry looking for "
-			    + x['category'] + " in bucket " + b['bucket']
-			    + " user " + e['user'] + " " + str(ex))
+                            + x['category'] + " in bucket " + b['bucket']
+                            + " user " + e['user'] + " " + str(ex))
                         break
                     usage_acc_validate_fields(r, x, x2, "entry: category "
-			+ x['category'] + " bucket " + b['bucket']
-			+ " in user " + e['user'])
+                        + x['category'] + " bucket " + b['bucket']
+                        + " in user " + e['user'])
+
+                if 's3select' not in b2:
+                    r.append("missing s3select in bucket "
+                        + b['bucket'] + " in user " + e['user'])
+                    continue
+                usage_acc_validate_s3select_fields(r,
+                    b['s3select'], b2['s3select'],
+                    "entry: s3select in bucket " + b['bucket'] + " in user " + e['user'])
         for s in self.results['summary']:
             c = s['categories']
             try:
                 s2 = usage_acc_findsum2(results['summary'], s['user'], False)
             except Exception as ex:
-                r.append("malformed summary looking for user " + e['user']
-		    + " " + str(ex))
+                r.append("malformed summary looking for user " + s['user']
+                    + " " + str(ex))
                 break
-                if s2 == None:
-                    r.append("missing summary for user " + e['user'] + " " + str(ex))
-                    continue
+            if s2 == None:
+                r.append("missing summary for user " + s['user'])
+                continue
             try:
                 c2 = s2['categories']
             except Exception as ex:
                 r.append("malformed summary missing categories for user "
-		    + e['user'] + " " + str(ex))
+                    + s['user'] + " " + str(ex))
                 break
             for x in c:
                 try:
                     x2 = self.c2x(c2, x['category'], False)
                 except Exception as ex:
                     r.append("malformed summary looking for "
-			+ x['category'] + " user " + e['user'] + " " + str(ex))
+                        + x['category'] + " user " + s['user'] + " " + str(ex))
                     break
                 usage_acc_validate_fields(r, x, x2, "summary: category "
-		    + x['category'] + " in user " + e['user'])
+                    + x['category'] + " in user " + s['user'])
             x = s['total']
             try:
                 x2 = s2['total']
             except Exception as ex:
                 r.append("malformed summary looking for totals for user "
-                         + e['user'] + " " + str(ex))
+                    + s['user'] + " " + str(ex))
                 break
-            usage_acc_validate_fields(r, x, x2, "summary: totals for user" + e['user'])
+            usage_acc_validate_fields(r, x, x2, "summary: totals for user" + s['user'])
+            usage_acc_validate_s3select_fields(r, x, x2, "summary: s3select totals for user" + s['user'])
         return r
 
 def ignore_this_entry(cat, bucket, user, out, b_in, err):
diff --git a/src/cls/rgw/cls_rgw_types.cc b/src/cls/rgw/cls_rgw_types.cc
index 1c232a576bba..05d48d734fc0 100644
--- a/src/cls/rgw/cls_rgw_types.cc
+++ b/src/cls/rgw/cls_rgw_types.cc
@@ -693,6 +693,21 @@ void rgw_bucket_dir::dump(Formatter *f) const
   f->close_section();
 }
 
+void rgw_s3select_usage_data::generate_test_instances(list<rgw_s3select_usage_data*>& o)
+{
+  rgw_s3select_usage_data *s = new rgw_s3select_usage_data;
+  s->bytes_processed = 1024;
+  s->bytes_returned = 512;
+  o.push_back(s);
+  o.push_back(new rgw_s3select_usage_data);
+}
+
+void rgw_s3select_usage_data::dump(Formatter *f) const
+{
+  f->dump_unsigned("bytes_processed", bytes_processed);
+  f->dump_unsigned("bytes_returned", bytes_returned);
+}
+
 void rgw_usage_data::generate_test_instances(list<rgw_usage_data*>& o)
 {
   rgw_usage_data *s = new rgw_usage_data;
@@ -773,12 +788,18 @@ void rgw_usage_log_entry::dump(Formatter *f) const
     }
   }
   f->close_section();
+
+  f->open_object_section("s3select");
+  f->dump_unsigned("bytes_processed", s3select_usage.bytes_processed);
+  f->dump_unsigned("bytes_returned", s3select_usage.bytes_returned);
+  f->close_section();
 }
 
 void rgw_usage_log_entry::generate_test_instances(list<rgw_usage_log_entry *> &o)
 {
   rgw_usage_log_entry *entry = new rgw_usage_log_entry;
   rgw_usage_data usage_data{1024, 2048};
+  rgw_s3select_usage_data s3select_usage_data{8192, 4096};
   entry->owner = rgw_user("owner");
   entry->payer = rgw_user("payer");
   entry->bucket = "bucket";
@@ -788,6 +809,7 @@ void rgw_usage_log_entry::generate_test_instances(list<rgw_usage_log_entry *> &o
   entry->total_usage.ops = usage_data.ops;
   entry->total_usage.successful_ops = usage_data.successful_ops;
   entry->usage_map["get_obj"] = usage_data;
+  entry->s3select_usage = s3select_usage_data;
   o.push_back(entry);
   o.push_back(new rgw_usage_log_entry);
 }
diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 5f94b9918fa4..21918aee5dcb 100644
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -869,6 +869,38 @@ struct rgw_bucket_dir {
 };
 WRITE_CLASS_ENCODER(rgw_bucket_dir)
 
+struct rgw_s3select_usage_data {
+  uint64_t bytes_processed;
+  uint64_t bytes_returned;
+
+  rgw_s3select_usage_data() : bytes_processed(0), bytes_returned(0) {}
+  rgw_s3select_usage_data(uint64_t processed, uint64_t returned)
+    : bytes_processed(processed), bytes_returned(returned) {}
+
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(bytes_processed, bl);
+    encode(bytes_returned, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(bytes_processed, bl);
+    decode(bytes_returned, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void aggregate(const rgw_s3select_usage_data& usage) {
+    bytes_processed += usage.bytes_processed;
+    bytes_returned += usage.bytes_returned;
+  }
+
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<rgw_s3select_usage_data*>& o);
+};
+WRITE_CLASS_ENCODER(rgw_s3select_usage_data)
+
 struct rgw_usage_data {
   uint64_t bytes_sent;
   uint64_t bytes_received;
@@ -915,13 +947,14 @@ struct rgw_usage_log_entry {
   uint64_t epoch;
   rgw_usage_data total_usage; /* this one is kept for backwards compatibility */
   std::map<std::string, rgw_usage_data> usage_map;
+  rgw_s3select_usage_data s3select_usage;
 
   rgw_usage_log_entry() : epoch(0) {}
   rgw_usage_log_entry(std::string& o, std::string& b) : owner(o), bucket(b), epoch(0) {}
   rgw_usage_log_entry(std::string& o, std::string& p, std::string& b) : owner(o), payer(p), bucket(b), epoch(0) {}
 
   void encode(ceph::buffer::list& bl) const {
-    ENCODE_START(3, 1, bl);
+    ENCODE_START(4, 1, bl);
     encode(owner.to_str(), bl);
     encode(bucket, bl);
     encode(epoch, bl);
@@ -931,12 +964,13 @@ struct rgw_usage_log_entry {
     encode(total_usage.successful_ops, bl);
     encode(usage_map, bl);
     encode(payer.to_str(), bl);
+    encode(s3select_usage, bl);
     ENCODE_FINISH(bl);
   }
 
 
    void decode(ceph::buffer::list::const_iterator& bl) {
-    DECODE_START(3, bl);
+    DECODE_START(4, bl);
     std::string s;
     decode(s, bl);
     owner.from_str(s);
@@ -956,6 +990,9 @@ struct rgw_usage_log_entry {
       decode(p, bl);
       payer.from_str(p);
     }
+    if (struct_v >= 4) {
+      decode(s3select_usage, bl);
+    }
     DECODE_FINISH(bl);
   }
 
@@ -970,9 +1007,13 @@ struct rgw_usage_log_entry {
 
     for (auto iter = e.usage_map.begin(); iter != e.usage_map.end(); ++iter) {
       if (!categories || !categories->size() || categories->count(iter->first)) {
-        add(iter->first, iter->second);
+        add_usage(iter->first, iter->second);
       }
     }
+
+    if (!categories || !categories->size() || categories->count("s3select")) {
+      s3select_usage.aggregate(e.s3select_usage);
+    }
   }
 
   void sum(rgw_usage_data& usage,
@@ -985,7 +1026,7 @@ struct rgw_usage_log_entry {
     }
   }
 
-  void add(const std::string& category, const rgw_usage_data& data) {
+  void add_usage(const std::string& category, const rgw_usage_data& data) {
     usage_map[category].aggregate(data);
     total_usage.aggregate(data);
   }
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 5165f9008b9f..a65657a04fa7 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1112,6 +1112,7 @@ struct req_state : DoutPrefixProvider {
   uint64_t obj_size{0};
   bool enable_ops_log;
   bool enable_usage_log;
+  rgw_s3select_usage_data s3select_usage;
   uint8_t defer_to_bucket_acls;
   uint32_t perm_mask{0};
 
diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 9bc27bbe9ffa..79822eb95972 100644
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -238,9 +238,12 @@ static void log_usage(req_state *s, const string& op_name)
   ldpp_dout(s, 30) << "log_usage: bucket_name=" << bucket_name
 	<< " tenant=" << s->bucket_tenant
 	<< ", bytes_sent=" << bytes_sent << ", bytes_received="
-	<< bytes_received << ", success=" << data.successful_ops << dendl;
+	<< bytes_received << ", success=" << data.successful_ops
+	<< ", bytes_processed=" << s->s3select_usage.bytes_processed
+	<< ", bytes_returned=" << s->s3select_usage.bytes_returned << dendl;
 
-  entry.add(op_name, data);
+  entry.add_usage(op_name, data);
+  entry.s3select_usage = s->s3select_usage;
 
   utime_t ts = ceph_clock_now();
 
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6d56766d10eb..04e70daff53f 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1553,7 +1553,16 @@ void RGWGetUsage_ObjStore_S3::send_response()
       utime_t ut(entry.epoch, 0);
       ut.gmtime(formatter->dump_stream("Time"));
       formatter->dump_int("Epoch", entry.epoch);
+
       dump_usage_categories_info(formatter, entry, &categories);
+
+      formatter->open_object_section("s3select");
+      if (categories.empty() || categories.count("s3select")) {
+        encode_json("BytesProcessed", entry.s3select_usage.bytes_processed, formatter);
+        encode_json("BytesReturned", entry.s3select_usage.bytes_returned, formatter);
+      }
+      formatter->close_section(); // s3select
+
       formatter->close_section(); // bucket
     }
 
@@ -1583,6 +1592,8 @@ void RGWGetUsage_ObjStore_S3::send_response()
        encode_json("BytesReceived", total_usage.bytes_received, formatter);
        encode_json("Ops", total_usage.ops, formatter);
        encode_json("SuccessfulOps", total_usage.successful_ops, formatter);
+       encode_json("BytesProcessed", entry.s3select_usage.bytes_processed, formatter);
+       encode_json("BytesReturned", entry.s3select_usage.bytes_returned, formatter);
        formatter->close_section(); // total
        formatter->close_section(); // user
      }
diff --git a/src/rgw/rgw_s3select.cc b/src/rgw/rgw_s3select.cc
index c7eaa69842f3..b939d3827486 100644
--- a/src/rgw/rgw_s3select.cc
+++ b/src/rgw/rgw_s3select.cc
@@ -27,6 +27,7 @@ uint64_t aws_response_handler::get_processed_size()
 void aws_response_handler::update_processed_size(uint64_t value)
 {
   processed_size += value;
+  s->s3select_usage.bytes_processed = processed_size;
 }
 
 uint64_t aws_response_handler::get_total_bytes_returned()
@@ -37,6 +38,7 @@ uint64_t aws_response_handler::get_total_bytes_returned()
 void aws_response_handler::update_total_bytes_returned(uint64_t value)
 {
   total_bytes_returned = value;
+  s->s3select_usage.bytes_returned = total_bytes_returned;
 }
 
 void aws_response_handler::push_header(const char* header_name, const char* header_value)
diff --git a/src/rgw/rgw_usage.cc b/src/rgw/rgw_usage.cc
index 43e56577c24c..1f9cb6f8d5fc 100644
--- a/src/rgw/rgw_usage.cc
+++ b/src/rgw/rgw_usage.cc
@@ -105,7 +105,16 @@ int RGWUsage::show(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
         if (!payer.empty() && payer != owner) {
           formatter->dump_string("payer", payer);
         }
+
         dump_usage_categories_info(formatter, entry, categories);
+
+        formatter->open_object_section("s3select");
+        if (!categories || categories->empty() || categories->count("s3select")) {
+          formatter->dump_unsigned("bytes_processed", entry.s3select_usage.bytes_processed);
+          formatter->dump_unsigned("bytes_returned", entry.s3select_usage.bytes_returned);
+        }
+        formatter->close_section(); // s3select
+
         formatter->close_section(); // bucket
         flusher.flush();
       }
@@ -136,6 +145,8 @@ int RGWUsage::show(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
       encode_json("bytes_received", total_usage.bytes_received, formatter);
       encode_json("ops", total_usage.ops, formatter);
       encode_json("successful_ops", total_usage.successful_ops, formatter);
+      encode_json("bytes_processed", entry.s3select_usage.bytes_processed, formatter);
+      encode_json("bytes_returned", entry.s3select_usage.bytes_returned, formatter);
       formatter->close_section(); // total
 
       formatter->close_section(); // user
diff --git a/src/tools/ceph-dencoder/rgw_types.h b/src/tools/ceph-dencoder/rgw_types.h
index 45d6921c5be2..05484c78fc14 100644
--- a/src/tools/ceph-dencoder/rgw_types.h
+++ b/src/tools/ceph-dencoder/rgw_types.h
@@ -53,6 +53,7 @@ TYPE(rgw_usage_log_entry)
 TYPE(rgw_cls_bi_entry)
 TYPE(rgw_bucket_olh_entry)
 TYPE(rgw_usage_data)
+TYPE(rgw_s3select_usage_data)
 TYPE(rgw_usage_log_info)
 TYPE(rgw_user_bucket)
 TYPE(cls_rgw_lc_entry)

From 4bd7b94f41944af3d02b41e88f5279793aa072e4 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 29 Jun 2023 08:26:48 +0000
Subject: [PATCH 0643/2492] crimson/osd/pg: submit_error_log send messages to
 osd by order

Use chained futurized `send_to_osd()` instead of voided `send_cluster_message()`.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 407d354134ce..b3e6c8480ec7 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -954,10 +954,15 @@ seastar::future<> PG::submit_error_log(
     log_entries, t, peering_state.get_pg_trim_to(),
     peering_state.get_min_last_complete_ondisk());
 
-    set<pg_shard_t> waiting_on;
-    for (auto &i : get_acting_recovery_backfill()) {
+  return seastar::do_with(log_entries, t, set<pg_shard_t>{},
+    [this, rep_tid](auto& log_entries, auto& t,auto& waiting_on) mutable {
+    return seastar::do_for_each(get_acting_recovery_backfill(),
+      [this, log_entries, t=std::move(t), waiting_on, rep_tid]
+      (auto& i) mutable {
       pg_shard_t peer(i);
-      if (peer == pg_whoami) continue;
+      if (peer == pg_whoami) {
+        return seastar::now();
+      }
       ceph_assert(peering_state.get_peer_missing().count(peer));
       ceph_assert(peering_state.has_peer_info(peer));
       auto log_m = crimson::make_message<MOSDPGUpdateLogMissing>(
@@ -969,22 +974,27 @@ seastar::future<> PG::submit_error_log(
                    rep_tid,
                    peering_state.get_pg_trim_to(),
                    peering_state.get_min_last_complete_ondisk());
+      waiting_on.insert(peer);
       logger().debug("submit_error_log: sending log"
         "missing_request (rep_tid: {} entries: {})"
         " to osd {}", rep_tid, log_entries, peer.osd);
-      send_cluster_message(peer.osd, std::move(log_m), get_osdmap_epoch());
-      waiting_on.insert(peer);
-    }
-    waiting_on.insert(pg_whoami);
-    logger().debug("submit_error_log: inserting rep_tid {}", rep_tid);
-    log_entry_update_waiting_on.insert(
-      std::make_pair(rep_tid, log_update_t{std::move(waiting_on)}));
-    return shard_services.get_store().do_transaction(
-      get_collection_ref(), std::move(t))
-      .then([this] {
+      return shard_services.send_to_osd(peer.osd,
+                                        std::move(log_m),
+                                        get_osdmap_epoch());
+    }).then([this, waiting_on, t=std::move(t), rep_tid] () mutable {
+      waiting_on.insert(pg_whoami);
+      logger().debug("submit_error_log: inserting rep_tid {}", rep_tid);
+      log_entry_update_waiting_on.insert(
+        std::make_pair(rep_tid,
+                       log_update_t{std::move(waiting_on)}));
+      return shard_services.get_store().do_transaction(
+        get_collection_ref(), std::move(t)
+      ).then([this] {
         peering_state.update_trim_to();
         return seastar::now();
+      });
     });
+  });
 }
 
 PG::do_osd_ops_iertr::future<PG::pg_rep_op_fut_t<MURef<MOSDOpReply>>>

From 74965cb4dd9a681b16c0aa18abfe49f69d1b0257 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 19 Nov 2023 09:33:26 +0000
Subject: [PATCH 0644/2492] crimson/osd/pg: do_osd_ops_execute assert error
 type handling

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index b3e6c8480ec7..b15e6e056940 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -894,6 +894,10 @@ PG::do_osd_ops_execute(
     }), OpsExecuter::osd_op_errorator::all_same_way(
         [this, rollbacker, failure_func_ptr]
         (const std::error_code& e) mutable {
+          // handle non-fatal errors only
+          ceph_assert(e.value() == EDQUOT ||
+                      e.value() == ENOSPC ||
+                      e.value() == EAGAIN);
           return rollbacker.rollback_obc_if_modified(e).then_interruptible(
           [this, e, failure_func_ptr] {
             return (*failure_func_ptr)(e , shard_services.get_tid());

From 11d2c6675779956166c7b51892459c6841ff16ba Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 19 Nov 2023 09:35:21 +0000
Subject: [PATCH 0645/2492] crimson/osd/pg: add `record_error` bool to
 failure_func

```
submit_error_log records the result of an IO into the pg log so that we can return
the same error code if the client resends the request.
This should only be relevant for logical errors resulting from the target object state
-- for example, EEXIST returned on an exclusive create -- because there is application
logic built to rely on them.
In classic, the only such site is if the return value from do_osd_ops is negative
(or the transaction is empty) -- see PrimaryLogPG::prepare_transaction,
specifically where we set update_log_only to true.

We do not want to record space usage errors or errors specific to conditions on the primary
OSD such as IO errors -- submit_error_log isn't a catch-all error path.
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index b15e6e056940..076cb1b49d97 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -900,7 +900,8 @@ PG::do_osd_ops_execute(
                       e.value() == EAGAIN);
           return rollbacker.rollback_obc_if_modified(e).then_interruptible(
           [this, e, failure_func_ptr] {
-            return (*failure_func_ptr)(e , shard_services.get_tid());
+            // no need to record error log
+            return (*failure_func_ptr)(e , shard_services.get_tid(), false);
           });
     }));
 
@@ -916,10 +917,10 @@ PG::do_osd_ops_execute(
     ceph_tid_t rep_tid = shard_services.get_tid();
 
     auto all_completed_fut = e.value() == ENOENT ?
-      (*failure_func_ptr)(e, rep_tid) :
+      (*failure_func_ptr)(e, rep_tid, true) :
       rollbacker.rollback_obc_if_modified(e).then_interruptible(
       [e, failure_func_ptr, rep_tid] {
-          return (*failure_func_ptr)(e, rep_tid);
+          return (*failure_func_ptr)(e, rep_tid, true);
       });
 
     return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
@@ -1056,8 +1057,10 @@ PG::do_osd_ops(
         std::move(reply));
     },
     // failure_func
-    [m, &op_info, obc, this] (const std::error_code& e, const ceph_tid_t& rep_tid) {
-    logger().error("do_osd_ops_execute::failure_func {} got error: {}", *m, e);
+    [m, &op_info, obc, this]
+    (const std::error_code& e, const ceph_tid_t& rep_tid, bool record_error) {
+    logger().error("do_osd_ops_execute::failure_func {} got error: {} record_error: {}",
+                    *m, e, record_error);
     auto error_log_fut = seastar::now();
     epoch_t epoch = get_osdmap_epoch();
     auto last_complete = peering_state.get_info().last_complete;
@@ -1066,7 +1069,7 @@ PG::do_osd_ops(
     }
     return error_log_fut.then([m, e, epoch, &op_info, rep_tid, last_complete, this] {
       auto fut = seastar::now();
-      if (!peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
+      if (record_error && !peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
         logger().debug("do_osd_ops_execute::failure_func finding rep_tid {}",
                        rep_tid);
         ceph_assert(log_entry_version.contains(rep_tid));
@@ -1156,7 +1159,7 @@ PG::do_osd_ops(
         return do_osd_ops_iertr::now();
       },
       // failure_func
-      [] (const std::error_code& e, const ceph_tid_t& rep_tid) {
+      [] (const std::error_code& e, const ceph_tid_t& rep_tid, bool record_error) {
         return do_osd_ops_iertr::now();
       });
   });

From 63e14893cc7dc6a3aec92c0867ce5e3e67f44b29 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 16 Nov 2023 15:47:06 +0000
Subject: [PATCH 0646/2492] rgw/notifications: cleanup all coroutines after
 sending the notification

this is fixing a regression from: 6b6592f50b6b81fa13a330bcb91273ba7f25c0c9

Fixes: https://tracker.ceph.com/issues/63580

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/rgw_op.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index dc802b4c4528..3accf03e4968 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -7283,7 +7283,6 @@ void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key& o, optional_
     op_ret = 0;
   }
 
-  send_partial_response(o, del_op->result.delete_marker, del_op->result.version_id, op_ret, formatter_flush_cond);
 
   // send request to notification manager
   int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
@@ -7291,6 +7290,8 @@ void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key& o, optional_
     ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
     // too late to rollback operation, hence op_ret is not set here
   }
+  
+  send_partial_response(o, del_op->result.delete_marker, del_op->result.version_id, op_ret, formatter_flush_cond);
 }
 
 void RGWDeleteMultiObj::execute(optional_yield y)

From 1d98e8dab6162aafeede62c3a4be6bef7d5b926d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 7 Nov 2023 10:24:34 +0000
Subject: [PATCH 0647/2492] crimson/osd/pg: move submit_error_log to
 do_osd_ops_execute

Previously, submit_error_log was chained to failure_func
returned future.
Now submit_error_log is called from within do_osd_ops_execute

Fixes: https://tracker.ceph.com/issues/61651

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 112 ++++++++++++++++++++++++------------------
 src/crimson/osd/pg.h  |   3 ++
 2 files changed, 68 insertions(+), 47 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 076cb1b49d97..c61fd541e9d3 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -805,6 +805,9 @@ template <class Ret, class SuccessFunc, class FailureFunc>
 PG::do_osd_ops_iertr::future<PG::pg_rep_op_fut_t<Ret>>
 PG::do_osd_ops_execute(
   seastar::lw_shared_ptr<OpsExecuter> ox,
+  ObjectContextRef obc,
+  const OpInfo &op_info,
+  Ref<MOSDOp> m,
   std::vector<OSDOp>& ops,
   SuccessFunc&& success_func,
   FailureFunc&& failure_func)
@@ -814,6 +817,17 @@ PG::do_osd_ops_execute(
     return obc_loader.reload_obc(obc).handle_error_interruptible(
       load_obc_ertr::assert_all{"can't live with object state messed up"});
   });
+  auto maybe_submit_error_log = [&, op_info, m, obc]
+    (const std::error_code& e, const ceph_tid_t& rep_tid) {
+    // call submit_error_log only for non-internal clients
+    if constexpr (!std::is_same_v<Ret, void>) {
+      if(op_info.may_write()) {
+        return submit_error_log(m, op_info, obc, e, rep_tid);
+      }
+    }
+    return seastar::now();
+  };
+  auto error_func_ptr = seastar::make_lw_shared(std::move(maybe_submit_error_log));
   auto failure_func_ptr = seastar::make_lw_shared(std::move(failure_func));
   return interruptor::do_for_each(ops, [ox](OSDOp& osd_op) {
     logger().debug(
@@ -873,7 +887,7 @@ PG::do_osd_ops_execute(
           std::move(log_entries));
     });
   }).safe_then_unpack_interruptible(
-    [success_func=std::move(success_func), rollbacker, this, failure_func_ptr]
+    [success_func=std::move(success_func), error_func_ptr, rollbacker, this, failure_func_ptr]
     (auto submitted_fut, auto _all_completed_fut) mutable {
 
     auto all_completed_fut = _all_completed_fut.safe_then_interruptible_tuple(
@@ -910,23 +924,27 @@ PG::do_osd_ops_execute(
       std::move(all_completed_fut)
     );
   }, OpsExecuter::osd_op_errorator::all_same_way(
-    [this, rollbacker, failure_func_ptr]
+    [this, error_func_ptr, rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
 
-    auto submitted_fut = seastar::now();
+    PG::interruptible_future<> maybe_rollback_fut = seastar::now();
     ceph_tid_t rep_tid = shard_services.get_tid();
 
-    auto all_completed_fut = e.value() == ENOENT ?
-      (*failure_func_ptr)(e, rep_tid, true) :
-      rollbacker.rollback_obc_if_modified(e).then_interruptible(
-      [e, failure_func_ptr, rep_tid] {
-          return (*failure_func_ptr)(e, rep_tid, true);
-      });
+    if (e.value() == ENOENT) {
+      maybe_rollback_fut = rollbacker.rollback_obc_if_modified(e);
+    }
 
-    return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
-      std::move(submitted_fut),
-      std::move(all_completed_fut)
-    );
+    return maybe_rollback_fut.then_interruptible(
+    [error_func_ptr, e, rep_tid, failure_func_ptr] {
+      // record error log
+      return (*error_func_ptr)(e, rep_tid).then(
+      [failure_func_ptr, e, rep_tid] {
+        return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
+          std::move(seastar::now()),
+          std::move((*failure_func_ptr)(e, rep_tid, true))
+        );
+      });
+    });
   }));
 }
 
@@ -1016,6 +1034,9 @@ PG::do_osd_ops(
   return do_osd_ops_execute<MURef<MOSDOpReply>>(
     seastar::make_lw_shared<OpsExecuter>(
       Ref<PG>{this}, obc, op_info, *m, conn, snapc),
+    obc,
+    op_info,
+    m,
     m->ops,
     // success_func
     [this, m, obc, may_write = op_info.may_write(),
@@ -1061,43 +1082,37 @@ PG::do_osd_ops(
     (const std::error_code& e, const ceph_tid_t& rep_tid, bool record_error) {
     logger().error("do_osd_ops_execute::failure_func {} got error: {} record_error: {}",
                     *m, e, record_error);
-    auto error_log_fut = seastar::now();
     epoch_t epoch = get_osdmap_epoch();
     auto last_complete = peering_state.get_info().last_complete;
-    if (op_info.may_write()) {
-      error_log_fut = submit_error_log(m, op_info, obc, e, rep_tid);
-    }
-    return error_log_fut.then([m, e, epoch, &op_info, rep_tid, last_complete, this] {
-      auto fut = seastar::now();
-      if (record_error && !peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
-        logger().debug("do_osd_ops_execute::failure_func finding rep_tid {}",
-                       rep_tid);
-        ceph_assert(log_entry_version.contains(rep_tid));
-        auto it = log_entry_update_waiting_on.find(rep_tid);
-        ceph_assert(it != log_entry_update_waiting_on.end());
-        auto it2 = it->second.waiting_on.find(pg_whoami);
-        ceph_assert(it2 != it->second.waiting_on.end());
-        it->second.waiting_on.erase(it2);
-        if (it->second.waiting_on.empty()) {
-          log_entry_update_waiting_on.erase(it);
+    auto fut = seastar::now();
+    if (record_error && !peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
+      logger().debug("do_osd_ops_execute::failure_func finding rep_tid {}",
+                      rep_tid);
+      ceph_assert(log_entry_version.contains(rep_tid));
+      auto it = log_entry_update_waiting_on.find(rep_tid);
+      ceph_assert(it != log_entry_update_waiting_on.end());
+      auto it2 = it->second.waiting_on.find(pg_whoami);
+      ceph_assert(it2 != it->second.waiting_on.end());
+      it->second.waiting_on.erase(it2);
+      if (it->second.waiting_on.empty()) {
+        log_entry_update_waiting_on.erase(it);
+        peering_state.complete_write(log_entry_version[rep_tid], last_complete);
+        log_entry_version.erase(rep_tid);
+        logger().debug("do_osd_ops_execute::failure_func write complete,"
+                        " erasing rep_tid {}", rep_tid);
+
+      } else {
+        fut = it->second.all_committed.get_shared_future().then(
+          [this, last_complete, rep_tid] {
+          logger().debug("do_osd_ops_execute::failure_func awaited {}", rep_tid);
           peering_state.complete_write(log_entry_version[rep_tid], last_complete);
-          log_entry_version.erase(rep_tid);
-          logger().debug("do_osd_ops_execute::failure_func write complete,"
-                         " erasing rep_tid {}", rep_tid);
-
-        } else {
-          fut = it->second.all_committed.get_shared_future().then(
-            [this, last_complete, rep_tid] {
-            logger().debug("do_osd_ops_execute::failure_func awaited {}", rep_tid);
-            peering_state.complete_write(log_entry_version[rep_tid], last_complete);
-            ceph_assert(!log_entry_update_waiting_on.contains(rep_tid));
-            return seastar::now();
-          });
-        }
+          ceph_assert(!log_entry_update_waiting_on.contains(rep_tid));
+          return seastar::now();
+        });
       }
-      return fut.then([this, m, e] {
-        return log_reply(m, e);
-      });
+    }
+    return fut.then([this, m, e] {
+      return log_reply(m, e);
     });
   });
 }
@@ -1147,12 +1162,15 @@ PG::do_osd_ops(
     return do_osd_ops_execute<void>(
       seastar::make_lw_shared<OpsExecuter>(
         Ref<PG>{this},
-        std::move(obc),
+        obc,
         op_info,
         msg_params,
         msg_params.get_connection(),
         SnapContext{}
       ),
+      obc,
+      op_info,
+      Ref<MOSDOp>(),
       ops,
       // success_func
       [] {
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 93e3ae82ec6e..711270e4c5e7 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -583,6 +583,9 @@ class PG : public boost::intrusive_ref_counter<
   template <class Ret, class SuccessFunc, class FailureFunc>
   do_osd_ops_iertr::future<pg_rep_op_fut_t<Ret>> do_osd_ops_execute(
     seastar::lw_shared_ptr<OpsExecuter> ox,
+    ObjectContextRef obc,
+    const OpInfo &op_info,
+    Ref<MOSDOp> m,
     std::vector<OSDOp>& ops,
     SuccessFunc&& success_func,
     FailureFunc&& failure_func);

From 45312902f26ec1cda8318486a3258f6000571eac Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 15 Nov 2023 13:45:58 +0000
Subject: [PATCH 0648/2492] crimson/osd/pg: introduce clear_log_entry_maps()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 8 ++++++++
 src/crimson/osd/pg.h  | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index c61fd541e9d3..c8208bedb21e 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -318,6 +318,7 @@ unsigned PG::get_target_pg_log_entries() const
 }
 
 void PG::on_removal(ceph::os::Transaction &t) {
+  clear_log_entry_maps();
   t.register_on_commit(
     new LambdaContext(
       [this](int r) {
@@ -328,6 +329,12 @@ void PG::on_removal(ceph::os::Transaction &t) {
   }));
 }
 
+void PG::clear_log_entry_maps()
+{
+  log_entry_update_waiting_on.clear();
+  log_entry_version.clear();
+}
+
 void PG::on_activate(interval_set<snapid_t> snaps)
 {
   logger().debug("{}: {} snaps={}", *this, __func__, snaps);
@@ -1508,6 +1515,7 @@ seastar::future<> PG::stop()
 
 void PG::on_change(ceph::os::Transaction &t) {
   logger().debug("{} {}:", *this, __func__);
+  clear_log_entry_maps();
   context_registry_on_change();
   obc_loader.notify_on_change(is_primary());
   recovery_backend->on_peering_interval_change(t);
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 711270e4c5e7..056d5e9e1c00 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -332,6 +332,8 @@ class PG : public boost::intrusive_ref_counter<
 
   void on_removal(ceph::os::Transaction &t) final;
 
+  void clear_log_entry_maps();
+
   std::pair<ghobject_t, bool>
   do_delete_work(ceph::os::Transaction &t, ghobject_t _next) final;
 

From 3b162d38b246f4f41e3d9295521a20b685f90bc7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 15 Nov 2023 14:27:05 +0000
Subject: [PATCH 0649/2492] crimson/osd: avoid refcount mutations

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/logmissing_request.cc       | 2 +-
 src/crimson/osd/osd_operations/logmissing_request_reply.cc | 2 +-
 src/crimson/osd/osd_operations/snaptrim_event.cc           | 2 +-
 src/crimson/osd/pg.cc                                      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/crimson/osd/osd_operations/logmissing_request.cc b/src/crimson/osd/osd_operations/logmissing_request.cc
index 797625da629a..5f92ccddfcdc 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request.cc
@@ -88,7 +88,7 @@ seastar::future<> LogMissingRequest::with_pg(
     });
   }, [](std::exception_ptr) {
     return seastar::now();
-  }, pg).finally([this, ref] {
+  }, pg).finally([this, ref=std::move(ref)] {
     logger().debug("{}: exit", *this);
     handle.exit();
   });
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.cc b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
index b2d447d194af..20a9bf7dd00e 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
@@ -74,7 +74,7 @@ seastar::future<> LogMissingRequestReply::with_pg(
     });
   }, [](std::exception_ptr) {
     return seastar::now();
-  }, pg).finally([this, ref] {
+  }, pg).finally([this, ref=std::move(ref)] {
     logger().debug("{}: exit", *this);
     handle.exit();
   });
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index b0f5a452e7fc..f76dd1e36f23 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -196,7 +196,7 @@ SnapTrimEvent::start()
   }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
-  }, pg).finally([this, ref] {
+  }, pg).finally([this, ref=std::move(ref)] {
     logger().debug("{}: exit", *this);
     handle.exit();
   });
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 78ae559e8cd5..c4dce8a8ee30 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -507,7 +507,7 @@ void PG::on_active_actmap()
         logger().debug("{}: trimmed snap={}", *this, trimmed);
       });
     }
-  ).finally([this, pg_ref] {
+  ).finally([this, pg_ref=std::move(pg_ref)] {
     logger().debug("{}: PG::on_active_actmap() finished trimming",
                    *this);
     peering_state.state_clear(PG_STATE_SNAPTRIM);

From abceb1652239629ed11187a5fc670a3b1a3a5bb1 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 09:38:04 +0000
Subject: [PATCH 0650/2492] crimson/osd/osd_operations/snaptrim_event: lifetime
 fixes

```
    // SnapTrimEvent is a background operation,
    // it's lifetime is not guarnteed since the caller
    // returned future is being ignored. We should capture
    // a self reference thourhgout the entire execution
    // progress (not only on finally() continuations).
    // See: PG::on_active_actmap()
```

Sanitized backtrace:
```
DEBUG 2023-11-16 08:42:48,441 [shard 0] osd - snaptrim_event(id=21122, detail=SnapTrimEvent(pgid=3.1 snapid=3cb needs_pause=1)): interrupted crimson::common::actingset_changed (acting set changed

kernel callstack:
    #0 0x55e310e0ace7 in seastar::shared_mutex::unlock() (/usr/bin/ceph-osd+0x1edd0ce7)
    #1 0x55e313325d9c in auto seastar::futurize_invoke<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::ExitBarrier<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::BlockingEvent::Trigger<crimson::osd::SnapTrimEvent> >::exit()::{lambda()#1}&>(crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::ExitBarrier<crimson::OrderedConcurrentPhaseT<crimson::osd::SnapTrimEvent::WaitSubop>::BlockingEvent::Trigger<crimson::osd::SnapTrimEvent> >::exit()::{lambda()#1}&) (/usr/bin/ceph-osd+0x212ebd9c)
    #2 0x55e3133260ef in _ZN7seastar20noncopyable_functionIFNS_6futureIvEEvEE17direct_vtable_forIZNS2_4thenIZN7crimson23OrderedConcurrentPhaseTINS7_3osd13SnapTrimEvent9WaitSubopEE11ExitBarrierINSC_13BlockingEvent7TriggerISA_EEE4exitEvEUlvE_S2_EET0_OT_EUlDpOT_E_E4callEPKS4_ (/usr/bin/ceph-osd+0x212ec0ef)
0x61500013365c is located 92 bytes inside of 472-byte region [0x615000133600,0x6150001337d8)
freed by thread T2 here:
    #0 0x7fb345ab73cf in operator delete(void*, unsigned long) (/lib64/libasan.so.6+0xb73cf)
    #1 0x55e313474863 in crimson::osd::SnapTrimEvent::~SnapTrimEvent() (/usr/bin/ceph-osd+0x2143a863)

previously allocated by thread T2 here:
    #0 0x7fb345ab6367 in operator new(unsigned long) (/lib64/libasan.so.6+0xb6367)
    #1 0x55e31183ac18 in auto crimson::OperationRegistryI::create_operation<crimson::osd::SnapTrimEvent, crimson::osd::PG*, SnapMapper&, snapid_t const&, bool const&>(crimson::osd::PG*&&, SnapMapper&, snapid_t const&, bool const&) (/usr/bin/ceph-osd+0x1f800c18)
SUMMARY: AddressSanitizer: heap-use-after-free (/usr/bin/ceph-osd+0x1edd0ce7) in seastar::shared_mutex::unlock()
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index f76dd1e36f23..8ae36a5483dd 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -90,7 +90,14 @@ SnapTrimEvent::start()
 {
   ShardServices &shard_services = pg->get_shard_services();
   IRef ref = this;
-  return interruptor::with_interruption([&shard_services, this] {
+  return interruptor::with_interruption(
+    // SnapTrimEvent is a background operation,
+    // it's lifetime is not guarnteed since the caller
+    // returned future is being ignored. We should capture
+    // a self reference thourhgout the entire execution
+    // progress (not only on finally() continuations).
+    // See: PG::on_active_actmap()
+    [&shard_services, this, ref] {
     return enter_stage<interruptor>(
       client_pp().wait_for_active
     ).then_interruptible([this] {
@@ -193,7 +200,8 @@ SnapTrimEvent::start()
         });
       });
     });
-  }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
+  }, [this, ref]
+     (std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
   }, pg).finally([this, ref=std::move(ref)] {

From e2e18a572202847195a2a54c0f3bcf646bc77945 Mon Sep 17 00:00:00 2001
From: barakda <barak.davidov@gmail.com>
Date: Sun, 19 Nov 2023 22:24:09 +0200
Subject: [PATCH 0651/2492] nvmeof bump latest version

Signed-off-by: barakda <barak.davidov@gmail.com>
---
 src/cephadm/cephadmlib/constants.py | 2 +-
 src/pybind/mgr/cephadm/module.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index 811aae24c700..e5c03adffb7b 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -12,7 +12,7 @@
 DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.7'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:v0.0.5'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
 DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'
 DEFAULT_ELASTICSEARCH_IMAGE = 'quay.io/omrizeneva/elasticsearch:6.8.23'
 DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29'
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 7f375d6363fc..059b8cdcee8e 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -114,7 +114,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_IMAGE = 'quay.io/ceph/ceph'
 DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.43.0'
 DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.5.0'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:v0.0.5'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'

From f627445806dadcc956fe1256cfc3886cb5b8e1ef Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 20 Nov 2023 06:32:09 +1000
Subject: [PATCH 0652/2492] doc/rados: edit "Using the Monitor's Admin Socket"

Edit the section "Using the Monitor's Admin Socket" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 30 +++++++++++--------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 1170da7c33f6..01d49aae8c28 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -85,23 +85,27 @@ Using the monitor's admin socket
 ================================
 
 A monitor's admin socket allows you to interact directly with a specific daemon
-by using a Unix socket file. This file is found in the monitor's ``run``
-directory. The admin socket's default directory is
-``/var/run/ceph/ceph-mon.ID.asok``, but this can be overridden and the admin
-socket might be elsewhere, especially if your cluster's daemons are deployed in
-containers. If you cannot find it, either check your ``ceph.conf`` for an
-alternative path or run the following command:
+by using a Unix socket file. This socket file is found in the monitor's ``run``
+directory. 
+
+The admin socket's default directory is ``/var/run/ceph/ceph-mon.ID.asok``. It
+is possible to override the admin socket's default location. If the default
+location has been overridden, then the admin socket will be elsewhere. This is
+often the case when a cluster's daemons are deployed in containers. 
+
+To find the directory of the admin socket, check either your ``ceph.conf`` for
+an alternative path or run the following command:
     
 .. prompt:: bash $
 
    ceph-conf --name mon.ID --show-config-value admin_socket
 
-The admin socket is available for use only when the monitor daemon is running.
-Whenever the monitor has been properly shut down, the admin socket is removed.
-However, if the monitor is not running and the admin socket persists, it is
-likely that the monitor has been improperly shut down.  In any case, if the
-monitor is not running, it will be impossible to use the admin socket, and the
-``ceph`` command is likely to return ``Error 111: Connection Refused``.
+The admin socket is available for use only when the Monitor daemon is running.
+Every time the Monitor is properly shut down, the admin socket is removed.  If
+the Monitor is not running and yet the admin socket persists, it is likely that
+the Monitor has been improperly shut down. If the Monitor is not running, it
+will be impossible to use the admin socket, and the ``ceph`` command is likely
+to return ``Error 111: Connection Refused``.
 
 To access the admin socket, run a ``ceph tell`` command of the following form
 (specifying the daemon that you are interested in):
@@ -110,7 +114,7 @@ To access the admin socket, run a ``ceph tell`` command of the following form
 
    ceph tell mon.<id> mon_status
 
-This command passes a ``help`` command to the specific running monitor daemon
+This command passes a ``help`` command to the specified running Monitor daemon
 ``<id>`` via its admin socket. If you know the full path to the admin socket
 file, this can be done more directly by running the following command:
 

From 83ff8f2b67f1a784ebb3ca7d0b7ca2c385f99343 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 20 Nov 2023 00:47:40 +1000
Subject: [PATCH 0653/2492] doc/start: update release names

Update "Quincy" to "Reef" and "Pacific" to "Quincy" in the section
"Viewing Old Ceph Documentation" in /doc/start/documenting-ceph.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/start/documenting-ceph.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/start/documenting-ceph.rst b/doc/start/documenting-ceph.rst
index 02d4dccc44af..d94e87f6d74a 100644
--- a/doc/start/documenting-ceph.rst
+++ b/doc/start/documenting-ceph.rst
@@ -31,14 +31,14 @@ repository. Python Sphinx renders the source into HTML and manpages.
 
 Viewing Old Ceph Documentation
 ==============================
-The https://docs.ceph.com link displays the latest release branch by default
-(for example, if "Quincy" is the most recent release, then by default
-https://docs.ceph.com displays the documentation for Quincy), but you can view
-the documentation for older versions of Ceph (for example, ``pacific``) by
-replacing the version name in the url (for example, ``quincy`` in
-`https://docs.ceph.com/en/pacific <https://docs.ceph.com/en/quincy>`_) with the
-branch name you prefer (for example, ``pacific``, to create a URL that reads
-`https://docs.ceph.com/en/pacific/ <https://docs.ceph.com/en/pacific/>`_).
+The https://docs.ceph.com link displays the documentation for the latest
+release by default (for example, if "Reef" is the most recent release, then by
+default https://docs.ceph.com displays the documentation for Reef), but you can
+view the documentation for older releases of Ceph (for example, ``quincy``) by
+replacing the release name in the url (for example, ``reef`` in
+`https://docs.ceph.com/en/reef/ <https://docs.ceph.com/en/reef>`_) with the
+branch name you prefer (for example, ``quincy``, to create a URL that reads
+`https://docs.ceph.com/en/pacific/ <https://docs.ceph.com/en/quincy/>`_).
 
 .. _making_contributions:
 

From 77e66ad09831e617a21bfa15d42c07041e362cb4 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 24 Oct 2023 15:26:21 +0800
Subject: [PATCH 0654/2492] crimson/common/smp_helpers: generalize
 crosscore_ordering_t

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/smp_helpers.h      | 118 +++++++++++++++++++++++++-
 src/crimson/net/ProtocolV2.cc         |   6 +-
 src/crimson/net/ProtocolV2.h          |   8 +-
 src/crimson/net/io_handler.cc         |  24 +++---
 src/crimson/net/io_handler.h          |  88 +++++--------------
 src/crimson/osd/osd_connection_priv.h |  73 +---------------
 src/crimson/osd/pg_shard_manager.h    |   2 +-
 7 files changed, 163 insertions(+), 156 deletions(-)

diff --git a/src/crimson/common/smp_helpers.h b/src/crimson/common/smp_helpers.h
index c2b7bd9641a7..fad81552d1ff 100644
--- a/src/crimson/common/smp_helpers.h
+++ b/src/crimson/common/smp_helpers.h
@@ -3,10 +3,16 @@
 
 #pragma once
 
+#include <concepts>
 #include <limits>
+#include <optional>
+#include <type_traits>
+#include <vector>
 
+#include <seastar/core/shared_future.hh>
 #include <seastar/core/smp.hh>
 
+#include "common/likely.h"
 #include "crimson/common/errorator.h"
 #include "crimson/common/utility.h"
 
@@ -89,4 +95,114 @@ auto sharded_map_seq(T &t, F &&f) {
     });
 }
 
-}
+enum class crosscore_type_t {
+  ONE,   // from 1 to 1 core
+  ONE_N, // from 1 to n cores
+};
+
+/**
+ * smp_crosscore_ordering_t
+ *
+ * To preserve the event order from source to target core(s).
+ */
+template <crosscore_type_t CTypeValue>
+class smp_crosscore_ordering_t {
+  static constexpr bool IS_ONE = (CTypeValue == crosscore_type_t::ONE);
+  static constexpr bool IS_ONE_N = (CTypeValue == crosscore_type_t::ONE_N);
+  static_assert(IS_ONE || IS_ONE_N);
+
+public:
+  using seq_t = uint64_t;
+
+  smp_crosscore_ordering_t() requires IS_ONE
+    : out_seqs(0) { }
+
+  smp_crosscore_ordering_t() requires IS_ONE_N
+    : out_seqs(seastar::smp::count, 0),
+      in_controls(seastar::smp::count) {}
+
+  ~smp_crosscore_ordering_t() = default;
+
+  /*
+   * Called by the original core to get the ordering sequence
+   */
+
+  seq_t prepare_submit() requires IS_ONE {
+    return do_prepare_submit(out_seqs);
+  }
+
+  seq_t prepare_submit(core_id_t target_core) requires IS_ONE_N {
+    return do_prepare_submit(out_seqs[target_core]);
+  }
+
+  /*
+   * Called by the target core to preserve the ordering
+   */
+
+  seq_t get_in_seq() const requires IS_ONE {
+    return in_controls.seq;
+  }
+
+  seq_t get_in_seq() const requires IS_ONE_N {
+    return in_controls[seastar::this_shard_id()].seq;
+  }
+
+  bool proceed_or_wait(seq_t seq) requires IS_ONE {
+    return in_controls.proceed_or_wait(seq);
+  }
+
+  bool proceed_or_wait(seq_t seq) requires IS_ONE_N {
+    return in_controls[seastar::this_shard_id()].proceed_or_wait(seq);
+  }
+
+  seastar::future<> wait(seq_t seq) requires IS_ONE {
+    return in_controls.wait(seq);
+  }
+
+  seastar::future<> wait(seq_t seq) requires IS_ONE_N {
+    return in_controls[seastar::this_shard_id()].wait(seq);
+  }
+
+private:
+  struct in_control_t {
+    seq_t seq = 0;
+    std::optional<seastar::shared_promise<>> pr_wait;
+
+    bool proceed_or_wait(seq_t in_seq) {
+      if (in_seq == seq + 1) {
+        ++seq;
+        if (unlikely(pr_wait.has_value())) {
+          pr_wait->set_value();
+          pr_wait = std::nullopt;
+        }
+        return true;
+      } else {
+        return false;
+      }
+    }
+
+    seastar::future<> wait(seq_t in_seq) {
+      assert(in_seq != seq + 1);
+      if (!pr_wait.has_value()) {
+        pr_wait = seastar::shared_promise<>();
+      }
+      return pr_wait->get_shared_future();
+    }
+  };
+
+  seq_t do_prepare_submit(seq_t &out_seq) {
+    return ++out_seq;
+  }
+
+  std::conditional_t<
+    IS_ONE,
+    seq_t, std::vector<seq_t>
+  > out_seqs;
+
+  std::conditional_t<
+    IS_ONE,
+    in_control_t, std::vector<in_control_t>
+  > in_controls;
+};
+
+} // namespace crimson
diff --git a/src/crimson/net/ProtocolV2.cc b/src/crimson/net/ProtocolV2.cc
index 55b669384ed3..d4ef3881c408 100644
--- a/src/crimson/net/ProtocolV2.cc
+++ b/src/crimson/net/ProtocolV2.cc
@@ -2073,7 +2073,7 @@ void ProtocolV2::trigger_replacing(bool reconnect,
 // READY state
 
 seastar::future<> ProtocolV2::notify_out_fault(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     const char *where,
     std::exception_ptr eptr,
     io_handler_state _io_states)
@@ -2121,7 +2121,7 @@ void ProtocolV2::execute_standby()
 }
 
 seastar::future<> ProtocolV2::notify_out(
-    crosscore_t::seq_t cc_seq)
+    cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == conn.get_messenger_shard_id());
   if (!crosscore.proceed_or_wait(cc_seq)) {
@@ -2210,7 +2210,7 @@ void ProtocolV2::execute_server_wait()
 // CLOSING state
 
 seastar::future<> ProtocolV2::notify_mark_down(
-    crosscore_t::seq_t cc_seq)
+    cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == conn.get_messenger_shard_id());
   if (!crosscore.proceed_or_wait(cc_seq)) {
diff --git a/src/crimson/net/ProtocolV2.h b/src/crimson/net/ProtocolV2.h
index dd7a1e7039b5..168d079c8e6d 100644
--- a/src/crimson/net/ProtocolV2.h
+++ b/src/crimson/net/ProtocolV2.h
@@ -29,16 +29,16 @@ class ProtocolV2 final : public HandshakeListener {
  */
 private:
   seastar::future<> notify_out(
-      crosscore_t::seq_t cc_seq) final;
+      cc_seq_t cc_seq) final;
 
   seastar::future<> notify_out_fault(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       const char *where,
       std::exception_ptr,
       io_handler_state) final;
 
   seastar::future<> notify_mark_down(
-      crosscore_t::seq_t cc_seq) final;
+      cc_seq_t cc_seq) final;
 
 /*
 * as ProtocolV2 to be called by SocketConnection
@@ -251,7 +251,7 @@ class ProtocolV2 final : public HandshakeListener {
   // asynchronously populated from io_handler
   io_handler_state io_states;
 
-  crosscore_t crosscore;
+  crosscore_ordering_t crosscore;
 
   bool has_socket = false;
 
diff --git a/src/crimson/net/io_handler.cc b/src/crimson/net/io_handler.cc
index c414c48e12f8..e8a868b4d4c7 100644
--- a/src/crimson/net/io_handler.cc
+++ b/src/crimson/net/io_handler.cc
@@ -292,7 +292,7 @@ void IOHandler::assign_frame_assembler(FrameAssemblerV2Ref fa)
 
 void IOHandler::do_set_io_state(
     io_state_t new_state,
-    std::optional<crosscore_t::seq_t> cc_seq,
+    std::optional<cc_seq_t> cc_seq,
     FrameAssemblerV2Ref fa,
     bool set_notify_out)
 {
@@ -363,7 +363,7 @@ void IOHandler::do_set_io_state(
 }
 
 seastar::future<> IOHandler::set_io_state(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     io_state_t new_state,
     FrameAssemblerV2Ref fa,
     bool set_notify_out)
@@ -385,7 +385,7 @@ seastar::future<> IOHandler::set_io_state(
 
 seastar::future<IOHandler::exit_dispatching_ret>
 IOHandler::wait_io_exit_dispatching(
-    crosscore_t::seq_t cc_seq)
+    cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
   if (!crosscore.proceed_or_wait(cc_seq)) {
@@ -429,7 +429,7 @@ IOHandler::wait_io_exit_dispatching(
 }
 
 seastar::future<> IOHandler::reset_session(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     bool full)
 {
   assert(seastar::this_shard_id() == get_shard_id());
@@ -454,7 +454,7 @@ seastar::future<> IOHandler::reset_session(
 }
 
 seastar::future<> IOHandler::reset_peer_state(
-    crosscore_t::seq_t cc_seq)
+    cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
   if (!crosscore.proceed_or_wait(cc_seq)) {
@@ -476,7 +476,7 @@ seastar::future<> IOHandler::reset_peer_state(
 }
 
 seastar::future<> IOHandler::requeue_out_sent(
-    crosscore_t::seq_t cc_seq)
+    cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
   if (!crosscore.proceed_or_wait(cc_seq)) {
@@ -517,7 +517,7 @@ void IOHandler::do_requeue_out_sent()
 }
 
 seastar::future<> IOHandler::requeue_out_sent_up_to(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     seq_num_t msg_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
@@ -583,7 +583,7 @@ void IOHandler::discard_out_sent()
 
 seastar::future<>
 IOHandler::dispatch_accept(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     seastar::shard_id new_sid,
     ConnectionFRef conn_fref,
     bool is_replace)
@@ -593,7 +593,7 @@ IOHandler::dispatch_accept(
 
 seastar::future<>
 IOHandler::dispatch_connect(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     seastar::shard_id new_sid,
     ConnectionFRef conn_fref)
 {
@@ -620,7 +620,7 @@ IOHandler::cleanup_prv_shard(seastar::shard_id prv_sid)
 
 seastar::future<>
 IOHandler::to_new_sid(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     seastar::shard_id new_sid,
     ConnectionFRef conn_fref,
     std::optional<bool> is_replace)
@@ -735,7 +735,7 @@ IOHandler::to_new_sid(
 }
 
 seastar::future<> IOHandler::set_accepted_sid(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     seastar::shard_id sid,
     ConnectionFRef conn_fref)
 {
@@ -1183,7 +1183,7 @@ void IOHandler::do_in_dispatch()
 
 seastar::future<>
 IOHandler::close_io(
-    crosscore_t::seq_t cc_seq,
+    cc_seq_t cc_seq,
     bool is_dispatch_reset,
     bool is_replace)
 {
diff --git a/src/crimson/net/io_handler.h b/src/crimson/net/io_handler.h
index f53c2ba64684..f0f0ba0ae62e 100644
--- a/src/crimson/net/io_handler.h
+++ b/src/crimson/net/io_handler.h
@@ -5,64 +5,16 @@
 
 #include <vector>
 
-#include <seastar/core/shared_future.hh>
 #include <seastar/util/later.hh>
 
 #include "crimson/common/gated.h"
+#include "crimson/common/smp_helpers.h"
 #include "Fwd.h"
 #include "SocketConnection.h"
 #include "FrameAssemblerV2.h"
 
 namespace crimson::net {
 
-/**
- * crosscore_t
- *
- * To preserve the event order across cores.
- */
-class crosscore_t {
-public:
-  using seq_t = uint64_t;
-
-  crosscore_t() = default;
-  ~crosscore_t() = default;
-
-  seq_t get_in_seq() const {
-    return in_seq;
-  }
-
-  seq_t prepare_submit() {
-    ++out_seq;
-    return out_seq;
-  }
-
-  bool proceed_or_wait(seq_t seq) {
-    if (seq == in_seq + 1) {
-      ++in_seq;
-      if (unlikely(in_pr_wait.has_value())) {
-        in_pr_wait->set_value();
-        in_pr_wait = std::nullopt;
-      }
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  seastar::future<> wait(seq_t seq) {
-    assert(seq != in_seq + 1);
-    if (!in_pr_wait.has_value()) {
-      in_pr_wait = seastar::shared_promise<>();
-    }
-    return in_pr_wait->get_shared_future();
-  }
-
-private:
-  seq_t out_seq = 0;
-  seq_t in_seq = 0;
-  std::optional<seastar::shared_promise<>> in_pr_wait;
-};
-
 /**
  * io_handler_state
  *
@@ -118,6 +70,9 @@ struct io_handler_state {
  */
 class HandshakeListener {
 public:
+  using crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
+  using cc_seq_t = crosscore_ordering_t::seq_t;
+
   virtual ~HandshakeListener() = default;
 
   HandshakeListener(const HandshakeListener&) = delete;
@@ -126,16 +81,16 @@ class HandshakeListener {
   HandshakeListener &operator=(HandshakeListener &&) = delete;
 
   virtual seastar::future<> notify_out(
-      crosscore_t::seq_t cc_seq) = 0;
+      cc_seq_t cc_seq) = 0;
 
   virtual seastar::future<> notify_out_fault(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       const char *where,
       std::exception_ptr,
       io_handler_state) = 0;
 
   virtual seastar::future<> notify_mark_down(
-      crosscore_t::seq_t cc_seq) = 0;
+      cc_seq_t cc_seq) = 0;
 
 protected:
   HandshakeListener() = default;
@@ -150,6 +105,9 @@ class HandshakeListener {
  */
 class IOHandler final : public ConnectionHandler {
 public:
+  using crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
+  using cc_seq_t = crosscore_ordering_t::seq_t;
+
   IOHandler(ChainedDispatchers &,
             SocketConnection &);
 
@@ -221,7 +179,7 @@ class IOHandler final : public ConnectionHandler {
   void print_io_stat(std::ostream &out) const;
 
   seastar::future<> set_accepted_sid(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       seastar::shard_id sid,
       ConnectionFRef conn_fref);
 
@@ -230,7 +188,7 @@ class IOHandler final : public ConnectionHandler {
    */
 
   seastar::future<> close_io(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       bool is_dispatch_reset,
       bool is_replace);
 
@@ -251,7 +209,7 @@ class IOHandler final : public ConnectionHandler {
   friend class fmt::formatter<io_state_t>;
 
   seastar::future<> set_io_state(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       io_state_t new_state,
       FrameAssemblerV2Ref fa,
       bool set_notify_out);
@@ -262,30 +220,30 @@ class IOHandler final : public ConnectionHandler {
   };
   seastar::future<exit_dispatching_ret>
   wait_io_exit_dispatching(
-      crosscore_t::seq_t cc_seq);
+      cc_seq_t cc_seq);
 
   seastar::future<> reset_session(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       bool full);
 
   seastar::future<> reset_peer_state(
-      crosscore_t::seq_t cc_seq);
+      cc_seq_t cc_seq);
 
   seastar::future<> requeue_out_sent_up_to(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       seq_num_t msg_seq);
 
   seastar::future<> requeue_out_sent(
-      crosscore_t::seq_t cc_seq);
+      cc_seq_t cc_seq);
 
   seastar::future<> dispatch_accept(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       seastar::shard_id new_sid,
       ConnectionFRef,
       bool is_replace);
 
   seastar::future<> dispatch_connect(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       seastar::shard_id new_sid,
       ConnectionFRef);
 
@@ -426,7 +384,7 @@ class IOHandler final : public ConnectionHandler {
 
   void do_set_io_state(
       io_state_t new_state,
-      std::optional<crosscore_t::seq_t> cc_seq = std::nullopt,
+      std::optional<cc_seq_t> cc_seq = std::nullopt,
       FrameAssemblerV2Ref fa = nullptr,
       bool set_notify_out = false);
 
@@ -449,7 +407,7 @@ class IOHandler final : public ConnectionHandler {
   seastar::future<> do_send_keepalive();
 
   seastar::future<> to_new_sid(
-      crosscore_t::seq_t cc_seq,
+      cc_seq_t cc_seq,
       seastar::shard_id new_sid,
       ConnectionFRef,
       std::optional<bool> is_replace);
@@ -509,7 +467,7 @@ class IOHandler final : public ConnectionHandler {
 private:
   shard_states_ref_t shard_states;
 
-  crosscore_t crosscore;
+  crosscore_ordering_t crosscore;
 
   // drop was happening in the previous sid
   std::optional<seastar::shard_id> maybe_dropped_sid;
diff --git a/src/crimson/osd/osd_connection_priv.h b/src/crimson/osd/osd_connection_priv.h
index 2d2a459017bb..3c7d085c06e9 100644
--- a/src/crimson/osd/osd_connection_priv.h
+++ b/src/crimson/osd/osd_connection_priv.h
@@ -3,8 +3,7 @@
 
 #pragma once
 
-#include <seastar/core/smp.hh>
-
+#include "crimson/common/smp_helpers.h"
 #include "crimson/net/Connection.h"
 #include "crimson/osd/osd_operation.h"
 #include "crimson/osd/osd_operations/client_request.h"
@@ -13,75 +12,9 @@
 
 namespace crimson::osd {
 
-/**
- * crosscore_ordering_t
- *
- * To preserve the event order from 1 source to n target cores.
- */
-class crosscore_ordering_t {
-public:
-  using seq_t = uint64_t;
-
-  crosscore_ordering_t()
-    : out_seqs(seastar::smp::count, 0),
-      in_controls(seastar::smp::count) {}
-
-  ~crosscore_ordering_t() = default;
-
-  // Called by the original core to get the ordering sequence
-  seq_t prepare_submit(core_id_t target_core) {
-    auto &out_seq = out_seqs[target_core];
-    ++out_seq;
-    return out_seq;
-  }
-
-  /*
-   * Called by the target core to preserve the ordering
-   */
-
-  seq_t get_in_seq() const {
-    auto core = seastar::this_shard_id();
-    return in_controls[core].seq;
-  }
-
-  bool proceed_or_wait(seq_t seq) {
-    auto core = seastar::this_shard_id();
-    auto &in_control = in_controls[core];
-    if (seq == in_control.seq + 1) {
-      ++in_control.seq;
-      if (unlikely(in_control.pr_wait.has_value())) {
-        in_control.pr_wait->set_value();
-        in_control.pr_wait = std::nullopt;
-      }
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  seastar::future<> wait(seq_t seq) {
-    auto core = seastar::this_shard_id();
-    auto &in_control = in_controls[core];
-    assert(seq != in_control.seq + 1);
-    if (!in_control.pr_wait.has_value()) {
-      in_control.pr_wait = seastar::shared_promise<>();
-    }
-    return in_control.pr_wait->get_shared_future();
-  }
-
-private:
-  struct in_control_t {
-    seq_t seq = 0;
-    std::optional<seastar::shared_promise<>> pr_wait;
-  };
-
-  // source-side
-  std::vector<seq_t> out_seqs;
-  // target-side
-  std::vector<in_control_t> in_controls;
-};
-
 struct OSDConnectionPriv : public crimson::net::Connection::user_private_t {
+  using crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE_N>;
+
   ConnectionPipeline client_request_conn_pipeline;
   ConnectionPipeline peering_request_conn_pipeline;
   ConnectionPipeline replicated_request_conn_pipeline;
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index cf13cb52bbf7..74154499c8a5 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -151,7 +151,7 @@ class PGShardManager {
 
   template <typename T, typename F>
   auto process_ordered_op_remotely(
-      crosscore_ordering_t::seq_t cc_seq,
+      OSDConnectionPriv::crosscore_ordering_t::seq_t cc_seq,
       ShardServices &target_shard_services,
       typename T::IRef &&op,
       F &&f) {

From 6ebf9cd3671960ceb88d3b95d487c91fd41fc8e6 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 30 Oct 2023 10:00:57 +0800
Subject: [PATCH 0655/2492] crimson/net: preserve the ordering upon the calls
 to Connection::send()/keepalive()

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/smp_helpers.h    |  40 ++++++-
 src/crimson/net/Connection.h        |   8 +-
 src/crimson/net/ProtocolV2.h        |   2 +-
 src/crimson/net/SocketConnection.cc |   5 +-
 src/crimson/net/SocketConnection.h  |   2 +-
 src/crimson/net/io_handler.cc       | 180 ++++++++++++++++++----------
 src/crimson/net/io_handler.h        |  23 ++--
 7 files changed, 169 insertions(+), 91 deletions(-)

diff --git a/src/crimson/common/smp_helpers.h b/src/crimson/common/smp_helpers.h
index fad81552d1ff..429c938229bc 100644
--- a/src/crimson/common/smp_helpers.h
+++ b/src/crimson/common/smp_helpers.h
@@ -98,6 +98,7 @@ auto sharded_map_seq(T &t, F &&f) {
 enum class crosscore_type_t {
   ONE,   // from 1 to 1 core
   ONE_N, // from 1 to n cores
+  N_ONE, // from n to 1 core
 };
 
 /**
@@ -109,7 +110,8 @@ template <crosscore_type_t CTypeValue>
 class smp_crosscore_ordering_t {
   static constexpr bool IS_ONE = (CTypeValue == crosscore_type_t::ONE);
   static constexpr bool IS_ONE_N = (CTypeValue == crosscore_type_t::ONE_N);
-  static_assert(IS_ONE || IS_ONE_N);
+  static constexpr bool IS_N_ONE = (CTypeValue == crosscore_type_t::N_ONE);
+  static_assert(IS_ONE || IS_ONE_N || IS_N_ONE);
 
 public:
   using seq_t = uint64_t;
@@ -117,7 +119,7 @@ class smp_crosscore_ordering_t {
   smp_crosscore_ordering_t() requires IS_ONE
     : out_seqs(0) { }
 
-  smp_crosscore_ordering_t() requires IS_ONE_N
+  smp_crosscore_ordering_t() requires (!IS_ONE)
     : out_seqs(seastar::smp::count, 0),
       in_controls(seastar::smp::count) {}
 
@@ -135,6 +137,10 @@ class smp_crosscore_ordering_t {
     return do_prepare_submit(out_seqs[target_core]);
   }
 
+  seq_t prepare_submit() requires IS_N_ONE {
+    return do_prepare_submit(out_seqs[seastar::this_shard_id()]);
+  }
+
   /*
    * Called by the target core to preserve the ordering
    */
@@ -147,6 +153,10 @@ class smp_crosscore_ordering_t {
     return in_controls[seastar::this_shard_id()].seq;
   }
 
+  seq_t get_in_seq(core_id_t source_core) const requires IS_N_ONE {
+    return in_controls[source_core].seq;
+  }
+
   bool proceed_or_wait(seq_t seq) requires IS_ONE {
     return in_controls.proceed_or_wait(seq);
   }
@@ -155,6 +165,10 @@ class smp_crosscore_ordering_t {
     return in_controls[seastar::this_shard_id()].proceed_or_wait(seq);
   }
 
+  bool proceed_or_wait(seq_t seq, core_id_t source_core) requires IS_N_ONE {
+    return in_controls[source_core].proceed_or_wait(seq);
+  }
+
   seastar::future<> wait(seq_t seq) requires IS_ONE {
     return in_controls.wait(seq);
   }
@@ -163,6 +177,16 @@ class smp_crosscore_ordering_t {
     return in_controls[seastar::this_shard_id()].wait(seq);
   }
 
+  seastar::future<> wait(seq_t seq, core_id_t source_core) requires IS_N_ONE {
+    return in_controls[source_core].wait(seq);
+  }
+
+  void reset_wait() requires IS_N_ONE {
+    for (auto &in_control : in_controls) {
+      in_control.reset_wait();
+    }
+  }
+
 private:
   struct in_control_t {
     seq_t seq = 0;
@@ -171,10 +195,7 @@ class smp_crosscore_ordering_t {
     bool proceed_or_wait(seq_t in_seq) {
       if (in_seq == seq + 1) {
         ++seq;
-        if (unlikely(pr_wait.has_value())) {
-          pr_wait->set_value();
-          pr_wait = std::nullopt;
-        }
+        reset_wait();
         return true;
       } else {
         return false;
@@ -188,6 +209,13 @@ class smp_crosscore_ordering_t {
       }
       return pr_wait->get_shared_future();
     }
+
+    void reset_wait() {
+      if (unlikely(pr_wait.has_value())) {
+        pr_wait->set_value();
+        pr_wait = std::nullopt;
+      }
+    }
   };
 
   seq_t do_prepare_submit(seq_t &out_seq) {
diff --git a/src/crimson/net/Connection.h b/src/crimson/net/Connection.h
index 7141e20f476d..41596987b09f 100644
--- a/src/crimson/net/Connection.h
+++ b/src/crimson/net/Connection.h
@@ -81,8 +81,8 @@ class Connection : public seastar::enable_shared_from_this<Connection> {
    *
    * Send a message over a connection that has completed its handshake.
    *
-   * May be invoked from any core, but that requires to chain the returned
-   * future to preserve ordering.
+   * May be invoked from any core, and the send order will be preserved upon
+   * the call.
    */
   virtual seastar::future<> send(MessageURef msg) = 0;
 
@@ -92,8 +92,8 @@ class Connection : public seastar::enable_shared_from_this<Connection> {
    * Send a keepalive message over a connection that has completed its
    * handshake.
    *
-   * May be invoked from any core, but that requires to chain the returned
-   * future to preserve ordering.
+   * May be invoked from any core, and the send order will be preserved upon
+   * the call.
    */
   virtual seastar::future<> send_keepalive() = 0;
 
diff --git a/src/crimson/net/ProtocolV2.h b/src/crimson/net/ProtocolV2.h
index 168d079c8e6d..4262bbbc70cc 100644
--- a/src/crimson/net/ProtocolV2.h
+++ b/src/crimson/net/ProtocolV2.h
@@ -251,7 +251,7 @@ class ProtocolV2 final : public HandshakeListener {
   // asynchronously populated from io_handler
   io_handler_state io_states;
 
-  crosscore_ordering_t crosscore;
+  proto_crosscore_ordering_t crosscore;
 
   bool has_socket = false;
 
diff --git a/src/crimson/net/SocketConnection.cc b/src/crimson/net/SocketConnection.cc
index 57e5c12c1aed..767192682773 100644
--- a/src/crimson/net/SocketConnection.cc
+++ b/src/crimson/net/SocketConnection.cc
@@ -79,16 +79,13 @@ bool SocketConnection::peer_wins() const
   return (messenger.get_myaddr() > peer_addr || policy.server);
 }
 
-seastar::future<> SocketConnection::send(MessageURef _msg)
+seastar::future<> SocketConnection::send(MessageURef msg)
 {
-  // may be invoked from any core
-  MessageFRef msg = seastar::make_foreign(std::move(_msg));
   return io_handler->send(std::move(msg));
 }
 
 seastar::future<> SocketConnection::send_keepalive()
 {
-  // may be invoked from any core
   return io_handler->send_keepalive();
 }
 
diff --git a/src/crimson/net/SocketConnection.h b/src/crimson/net/SocketConnection.h
index 823d6c574dad..7d20f68867e8 100644
--- a/src/crimson/net/SocketConnection.h
+++ b/src/crimson/net/SocketConnection.h
@@ -54,7 +54,7 @@ class ConnectionHandler {
 
   virtual bool is_connected() const = 0;
 
-  virtual seastar::future<> send(MessageFRef) = 0;
+  virtual seastar::future<> send(MessageURef) = 0;
 
   virtual seastar::future<> send_keepalive() = 0;
 
diff --git a/src/crimson/net/io_handler.cc b/src/crimson/net/io_handler.cc
index e8a868b4d4c7..b9b0339f9448 100644
--- a/src/crimson/net/io_handler.cc
+++ b/src/crimson/net/io_handler.cc
@@ -160,84 +160,132 @@ IOHandler::sweep_out_pending_msgs_to_sent(
 #endif
 }
 
-seastar::future<> IOHandler::send(MessageFRef msg)
+seastar::future<> IOHandler::send(MessageURef _msg)
 {
+  // may be invoked from any core
+  MessageFRef msg = seastar::make_foreign(std::move(_msg));
+  auto cc_seq = io_crosscore.prepare_submit();
+  auto source_core = seastar::this_shard_id();
   // sid may be changed on-the-fly during the submission
-  if (seastar::this_shard_id() == get_shard_id()) {
-    return do_send(std::move(msg));
+  if (source_core == get_shard_id()) {
+    return do_send(cc_seq, source_core, std::move(msg));
   } else {
-    logger().trace("{} send() is directed to {} -- {}",
-                   conn, get_shard_id(), *msg);
+    logger().trace("{} send() {} is directed to core {} -- {}",
+                   conn, cc_seq, get_shard_id(), *msg);
     return seastar::smp::submit_to(
-        get_shard_id(), [this, msg=std::move(msg)]() mutable {
-      return send_redirected(std::move(msg));
+        get_shard_id(),
+        [this, cc_seq, source_core, msg=std::move(msg)]() mutable {
+      return send_recheck_shard(cc_seq, source_core, std::move(msg));
     });
   }
 }
 
-seastar::future<> IOHandler::send_redirected(MessageFRef msg)
+seastar::future<> IOHandler::send_recheck_shard(
+  cc_seq_t cc_seq,
+  core_id_t source_core,
+  MessageFRef msg)
 {
   // sid may be changed on-the-fly during the submission
   if (seastar::this_shard_id() == get_shard_id()) {
-    return do_send(std::move(msg));
+    return do_send(cc_seq, source_core, std::move(msg));
   } else {
-    logger().debug("{} send() is redirected to {} -- {}",
-                   conn, get_shard_id(), *msg);
+    logger().debug("{} send_recheck_shard() {} "
+                   "is redirected from core {} to {} -- {}",
+                   conn, cc_seq, source_core, get_shard_id(), *msg);
     return seastar::smp::submit_to(
-        get_shard_id(), [this, msg=std::move(msg)]() mutable {
-      return send_redirected(std::move(msg));
+        get_shard_id(),
+        [this, cc_seq, source_core, msg=std::move(msg)]() mutable {
+      return send_recheck_shard(cc_seq, source_core, std::move(msg));
     });
   }
 }
 
-seastar::future<> IOHandler::do_send(MessageFRef msg)
+seastar::future<> IOHandler::do_send(
+  cc_seq_t cc_seq,
+  core_id_t source_core,
+  MessageFRef msg)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  logger().trace("{} do_send() got message -- {}", conn, *msg);
-  if (get_io_state() != io_state_t::drop) {
-    out_pending_msgs.push_back(std::move(msg));
-    notify_out_dispatch();
+  if (io_crosscore.proceed_or_wait(cc_seq, source_core)) {
+    logger().trace("{} do_send() got {} from core {}: send message -- {}",
+                   conn, cc_seq, source_core, *msg);
+    if (get_io_state() != io_state_t::drop) {
+      out_pending_msgs.push_back(std::move(msg));
+      notify_out_dispatch();
+    }
+    return seastar::now();
+  } else {
+    logger().debug("{} do_send() got {} from core {}, wait at {} -- {}",
+                   conn, cc_seq, source_core,
+                   io_crosscore.get_in_seq(source_core),
+                   *msg);
+    return io_crosscore.wait(cc_seq, source_core
+    ).then([this, cc_seq, source_core, msg=std::move(msg)]() mutable {
+      return send_recheck_shard(cc_seq, source_core, std::move(msg));
+    });
   }
-  return seastar::now();
 }
 
 seastar::future<> IOHandler::send_keepalive()
 {
+  // may be invoked from any core
+  auto cc_seq = io_crosscore.prepare_submit();
+  auto source_core = seastar::this_shard_id();
   // sid may be changed on-the-fly during the submission
-  if (seastar::this_shard_id() == get_shard_id()) {
-    return do_send_keepalive();
+  if (source_core == get_shard_id()) {
+    return do_send_keepalive(cc_seq, source_core);
   } else {
-    logger().trace("{} send_keepalive() is directed to {}", conn, get_shard_id());
+    logger().trace("{} send_keepalive() {} is directed to core {}",
+                   conn, cc_seq, get_shard_id());
     return seastar::smp::submit_to(
-        get_shard_id(), [this] {
-      return send_keepalive_redirected();
+        get_shard_id(),
+        [this, cc_seq, source_core] {
+      return send_keepalive_recheck_shard(cc_seq, source_core);
     });
   }
 }
 
-seastar::future<> IOHandler::send_keepalive_redirected()
+seastar::future<> IOHandler::send_keepalive_recheck_shard(
+  cc_seq_t cc_seq,
+  core_id_t source_core)
 {
   // sid may be changed on-the-fly during the submission
   if (seastar::this_shard_id() == get_shard_id()) {
-    return do_send_keepalive();
+    return do_send_keepalive(cc_seq, source_core);
   } else {
-    logger().debug("{} send_keepalive() is redirected to {}", conn, get_shard_id());
+    logger().debug("{} send_keepalive_recheck_shard() {} "
+                   "is redirected from core {} to {}",
+                   conn, cc_seq, source_core, get_shard_id());
     return seastar::smp::submit_to(
-        get_shard_id(), [this] {
-      return send_keepalive_redirected();
+        get_shard_id(),
+        [this, cc_seq, source_core] {
+      return send_keepalive_recheck_shard(cc_seq, source_core);
     });
   }
 }
 
-seastar::future<> IOHandler::do_send_keepalive()
+seastar::future<> IOHandler::do_send_keepalive(
+  cc_seq_t cc_seq,
+  core_id_t source_core)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  logger().trace("{} do_send_keeplive(): need_keepalive={}", conn, need_keepalive);
-  if (!need_keepalive) {
-    need_keepalive = true;
-    notify_out_dispatch();
+  if (io_crosscore.proceed_or_wait(cc_seq, source_core)) {
+    logger().trace("{} do_send_keeplive() got {} from core {}: need_keepalive={}",
+                   conn, cc_seq, source_core, need_keepalive);
+    if (!need_keepalive) {
+      need_keepalive = true;
+      notify_out_dispatch();
+    }
+    return seastar::now();
+  } else {
+    logger().debug("{} do_send_keepalive() got {} from core {}, wait at {}",
+                   conn, cc_seq, source_core,
+                   io_crosscore.get_in_seq(source_core));
+    return io_crosscore.wait(cc_seq, source_core
+    ).then([this, cc_seq, source_core] {
+      return send_keepalive_recheck_shard(cc_seq, source_core);
+    });
   }
-  return seastar::now();
 }
 
 void IOHandler::mark_down()
@@ -249,7 +297,7 @@ void IOHandler::mark_down()
     return;
   }
 
-  auto cc_seq = crosscore.prepare_submit();
+  auto cc_seq = proto_crosscore.prepare_submit();
   logger().info("{} mark_down() at {}, send {} notify_mark_down()",
                 conn, io_stat_printer{*this}, cc_seq);
   do_set_io_state(io_state_t::drop);
@@ -369,10 +417,10 @@ seastar::future<> IOHandler::set_io_state(
     bool set_notify_out)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} set_io_state(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq, new_state,
             fa=std::move(fa), set_notify_out]() mutable {
       return set_io_state(cc_seq, new_state, std::move(fa), set_notify_out);
@@ -388,10 +436,10 @@ IOHandler::wait_io_exit_dispatching(
     cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} wait_io_exit_dispatching(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq] {
       return wait_io_exit_dispatching(cc_seq);
     });
@@ -433,10 +481,10 @@ seastar::future<> IOHandler::reset_session(
     bool full)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} reset_session(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq, full] {
       return reset_session(cc_seq, full);
     });
@@ -457,10 +505,10 @@ seastar::future<> IOHandler::reset_peer_state(
     cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} reset_peer_state(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq] {
       return reset_peer_state(cc_seq);
     });
@@ -479,10 +527,10 @@ seastar::future<> IOHandler::requeue_out_sent(
     cc_seq_t cc_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} requeue_out_sent(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq] {
       return requeue_out_sent(cc_seq);
     });
@@ -521,10 +569,10 @@ seastar::future<> IOHandler::requeue_out_sent_up_to(
     seq_num_t msg_seq)
 {
   assert(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} requeue_out_sent_up_to(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq, msg_seq] {
       return requeue_out_sent_up_to(cc_seq, msg_seq);
     });
@@ -626,10 +674,10 @@ IOHandler::to_new_sid(
     std::optional<bool> is_replace)
 {
   ceph_assert_always(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} to_new_sid(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq, new_sid, is_replace,
             conn_fref=std::move(conn_fref)]() mutable {
       return to_new_sid(cc_seq, new_sid, std::move(conn_fref), is_replace);
@@ -685,6 +733,8 @@ IOHandler::to_new_sid(
   shard_states = shard_states_t::create_from_previous(
       *maybe_prv_shard_states, new_sid);
   assert(new_sid == get_shard_id());
+  // broadcast shard change to all the io waiters, atomically.
+  io_crosscore.reset_wait();
 
   return seastar::smp::submit_to(new_sid,
       [this, next_cc_seq, is_dropped, prv_sid, is_replace, conn_fref=std::move(conn_fref)]() mutable {
@@ -699,7 +749,7 @@ IOHandler::to_new_sid(
     ceph_assert_always(seastar::this_shard_id() == get_shard_id());
     ceph_assert_always(get_io_state() != io_state_t::open);
     ceph_assert_always(!maybe_dropped_sid.has_value());
-    ceph_assert_always(crosscore.proceed_or_wait(next_cc_seq));
+    ceph_assert_always(proto_crosscore.proceed_or_wait(next_cc_seq));
 
     if (is_dropped) {
       ceph_assert_always(get_io_state() == io_state_t::drop);
@@ -749,7 +799,7 @@ seastar::future<> IOHandler::set_accepted_sid(
   return seastar::smp::submit_to(sid,
       [this, cc_seq, conn_fref=std::move(conn_fref)]() mutable {
     // must be the first to proceed
-    ceph_assert_always(crosscore.proceed_or_wait(cc_seq));
+    ceph_assert_always(proto_crosscore.proceed_or_wait(cc_seq));
 
     logger().debug("{} set accepted sid", conn);
     ceph_assert_always(seastar::this_shard_id() == get_shard_id());
@@ -875,7 +925,7 @@ IOHandler::do_out_dispatch(shard_states_t &ctx)
     }
 
     if (io_state == io_state_t::open) {
-      auto cc_seq = crosscore.prepare_submit();
+      auto cc_seq = proto_crosscore.prepare_submit();
       logger().info("{} do_out_dispatch(): fault at {}, {}, going to delay -- {}, "
                     "send {} notify_out_fault()",
                     conn, io_state, io_stat_printer{*this}, e.what(), cc_seq);
@@ -922,7 +972,7 @@ void IOHandler::notify_out_dispatch()
   ceph_assert_always(seastar::this_shard_id() == get_shard_id());
   assert(is_out_queued());
   if (need_notify_out) {
-    auto cc_seq = crosscore.prepare_submit();
+    auto cc_seq = proto_crosscore.prepare_submit();
     logger().debug("{} send {} notify_out()",
                    conn, cc_seq);
     shard_states->dispatch_in_background(
@@ -1152,7 +1202,7 @@ void IOHandler::do_in_dispatch()
 
       auto io_state = ctx.get_io_state();
       if (io_state == io_state_t::open) {
-        auto cc_seq = crosscore.prepare_submit();
+        auto cc_seq = proto_crosscore.prepare_submit();
         logger().info("{} do_in_dispatch(): fault at {}, {}, going to delay -- {}, "
                       "send {} notify_out_fault()",
                       conn, io_state, io_stat_printer{*this}, e_what, cc_seq);
@@ -1188,10 +1238,10 @@ IOHandler::close_io(
     bool is_replace)
 {
   ceph_assert_always(seastar::this_shard_id() == get_shard_id());
-  if (!crosscore.proceed_or_wait(cc_seq)) {
+  if (!proto_crosscore.proceed_or_wait(cc_seq)) {
     logger().debug("{} got {} close_io(), wait at {}",
-                   conn, cc_seq, crosscore.get_in_seq());
-    return crosscore.wait(cc_seq
+                   conn, cc_seq, proto_crosscore.get_in_seq());
+    return proto_crosscore.wait(cc_seq
     ).then([this, cc_seq, is_dispatch_reset, is_replace] {
       return close_io(cc_seq, is_dispatch_reset, is_replace);
     });
diff --git a/src/crimson/net/io_handler.h b/src/crimson/net/io_handler.h
index f0f0ba0ae62e..8b88e2f5a254 100644
--- a/src/crimson/net/io_handler.h
+++ b/src/crimson/net/io_handler.h
@@ -70,8 +70,8 @@ struct io_handler_state {
  */
 class HandshakeListener {
 public:
-  using crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
-  using cc_seq_t = crosscore_ordering_t::seq_t;
+  using proto_crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
+  using cc_seq_t = proto_crosscore_ordering_t::seq_t;
 
   virtual ~HandshakeListener() = default;
 
@@ -105,8 +105,9 @@ class HandshakeListener {
  */
 class IOHandler final : public ConnectionHandler {
 public:
-  using crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
-  using cc_seq_t = crosscore_ordering_t::seq_t;
+  using io_crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::N_ONE>;
+  using proto_crosscore_ordering_t = smp_crosscore_ordering_t<crosscore_type_t::ONE>;
+  using cc_seq_t = proto_crosscore_ordering_t::seq_t;
 
   IOHandler(ChainedDispatchers &,
             SocketConnection &);
@@ -131,7 +132,7 @@ class IOHandler final : public ConnectionHandler {
     return protocol_is_connected;
   }
 
-  seastar::future<> send(MessageFRef msg) final;
+  seastar::future<> send(MessageURef msg) final;
 
   seastar::future<> send_keepalive() final;
 
@@ -398,13 +399,13 @@ class IOHandler final : public ConnectionHandler {
 
   void assign_frame_assembler(FrameAssemblerV2Ref);
 
-  seastar::future<> send_redirected(MessageFRef msg);
+  seastar::future<> send_recheck_shard(cc_seq_t, core_id_t, MessageFRef);
 
-  seastar::future<> do_send(MessageFRef msg);
+  seastar::future<> do_send(cc_seq_t, core_id_t, MessageFRef);
 
-  seastar::future<> send_keepalive_redirected();
+  seastar::future<> send_keepalive_recheck_shard(cc_seq_t, core_id_t);
 
-  seastar::future<> do_send_keepalive();
+  seastar::future<> do_send_keepalive(cc_seq_t, core_id_t);
 
   seastar::future<> to_new_sid(
       cc_seq_t cc_seq,
@@ -467,7 +468,9 @@ class IOHandler final : public ConnectionHandler {
 private:
   shard_states_ref_t shard_states;
 
-  crosscore_ordering_t crosscore;
+  proto_crosscore_ordering_t proto_crosscore;
+
+  io_crosscore_ordering_t io_crosscore;
 
   // drop was happening in the previous sid
   std::optional<seastar::shard_id> maybe_dropped_sid;

From 93ef23d90aa71ca3c084c28d53a0acb0bf5fb4f6 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 31 Oct 2023 09:36:23 +0800
Subject: [PATCH 0656/2492] crimson/osd: allow to send messages concurrently

The ordering is now guaranteed upon calling send(), so there is no
reason to couple the crosscore send future with the operation phases --
exclusive phases will limit the send concurrency, potentially causing
OSD starvation.

Decouple the crosscore send futures in the IO pathes, mostly in
ClientRequest and OSDSingletonState::send_to_osd().

Issue-identified-by: Chunmei Liu <chunmei.liu@intel.com>
see PR53934.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/net/Connection.h                  | 27 ++++++++++++++++-
 src/crimson/net/Fwd.h                         |  2 +-
 .../osd/osd_operations/client_request.cc      | 29 +++++++++++--------
 src/crimson/osd/shard_services.cc             |  3 +-
 4 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/src/crimson/net/Connection.h b/src/crimson/net/Connection.h
index 41596987b09f..c19bfb1ff57a 100644
--- a/src/crimson/net/Connection.h
+++ b/src/crimson/net/Connection.h
@@ -83,8 +83,33 @@ class Connection : public seastar::enable_shared_from_this<Connection> {
    *
    * May be invoked from any core, and the send order will be preserved upon
    * the call.
+   *
+   * The returned future will be resolved only after the message is enqueued
+   * remotely.
    */
-  virtual seastar::future<> send(MessageURef msg) = 0;
+  virtual seastar::future<> send(
+      MessageURef msg) = 0;
+
+  /**
+   * send_with_throttling
+   *
+   * Send a message over a connection that has completed its handshake.
+   *
+   * May be invoked from any core, and the send order will be preserved upon
+   * the call.
+   *
+   * TODO:
+   *
+   * The returned future is reserved for throttling.
+   *
+   * Gating is needed for graceful shutdown, to wait until the message is
+   * enqueued remotely.
+   */
+  seastar::future<> send_with_throttling(
+      MessageURef msg /* , seastar::gate & */) {
+    std::ignore = send(std::move(msg));
+    return seastar::now();
+  }
 
   /**
    * send_keepalive
diff --git a/src/crimson/net/Fwd.h b/src/crimson/net/Fwd.h
index 2b159514193c..3a56cf5bb0a6 100644
--- a/src/crimson/net/Fwd.h
+++ b/src/crimson/net/Fwd.h
@@ -21,7 +21,7 @@
 #include <seastar/core/sharded.hh>
 
 #include "msg/Connection.h"
-#include "msg/MessageRef.h"
+#include "msg/Message.h"
 #include "msg/msg_types.h"
 
 #include "crimson/common/errorator.h"
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index d208e2e53d97..c6dbd498c66d 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -204,7 +204,8 @@ ClientRequest::process_pg_op(
   return pg->do_pg_ops(
     m
   ).then_interruptible([this, pg=std::move(pg)](MURef<MOSDOpReply> reply) {
-    return conn->send(std::move(reply));
+    // TODO: gate the crosscore sending
+    return conn->send_with_throttling(std::move(reply));
   });
 }
 
@@ -218,7 +219,8 @@ auto ClientRequest::reply_op_error(const Ref<PG>& pg, int err)
     !m->has_flag(CEPH_OSD_FLAG_RETURNVEC));
   reply->set_reply_versions(eversion_t(), 0);
   reply->set_op_returns(std::vector<pg_log_op_return_item_t>{});
-  return conn->send(std::move(reply));
+  // TODO: gate the crosscore sending
+  return conn->send_with_throttling(std::move(reply));
 }
 
 ClientRequest::interruptible_future<>
@@ -246,7 +248,8 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
           m.get(), completed->err, pg->get_osdmap_epoch(),
           CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK, false);
 	reply->set_reply_versions(completed->version, completed->user_version);
-        return conn->send(std::move(reply));
+        // TODO: gate the crosscore sending
+        return conn->send_with_throttling(std::move(reply));
       } else {
         return ihref.enter_stage<interruptor>(client_pp(*pg).get_obc, *this
 	).then_interruptible(
@@ -319,13 +322,13 @@ ClientRequest::do_process(
 
   SnapContext snapc = get_snapc(pg,obc);
 
-  if ((m->has_flag(CEPH_OSD_FLAG_ORDERSNAP)) &&
-       snapc.seq < obc->ssc->snapset.seq) {
-        DEBUGI("{} ORDERSNAP flag set and snapc seq {}",
-                       " < snapset seq {} on {}",
-                       __func__, snapc.seq, obc->ssc->snapset.seq,
-                       obc->obs.oi.soid);
-     return reply_op_error(pg, -EOLDSNAPC);
+  if (m->has_flag(CEPH_OSD_FLAG_ORDERSNAP) &&
+      snapc.seq < obc->ssc->snapset.seq) {
+    DEBUGI("{} ORDERSNAP flag set and snapc seq {}",
+           " < snapset seq {} on {}",
+           __func__, snapc.seq, obc->ssc->snapset.seq,
+           obc->obs.oi.soid);
+    return reply_op_error(pg, -EOLDSNAPC);
   }
 
   if (!pg->is_primary()) {
@@ -357,8 +360,10 @@ ClientRequest::do_process(
 		[this, reply=std::move(reply)]() mutable {
                   LOG_PREFIX(ClientRequest::do_process);
 		  DEBUGI("{}: sending response", *this);
-		  return conn->send(std::move(reply));
-		});
+		  // TODO: gate the crosscore sending
+		  return conn->send_with_throttling(std::move(reply));
+		}
+	      );
 	    }, crimson::ct_error::eagain::handle([this, pg, &ihref]() mutable {
 	      return process_op(ihref, pg);
 	    }));
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 404f28d7d7f3..ae36d7f6ea00 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -166,7 +166,8 @@ seastar::future<> OSDSingletonState::send_to_osd(
   } else {
     auto conn = cluster_msgr.connect(
         osdmap->get_cluster_addrs(peer).front(), CEPH_ENTITY_TYPE_OSD);
-    return conn->send(std::move(m));
+    // TODO: gate the crosscore sending
+    return conn->send_with_throttling(std::move(m));
   }
 }
 

From 52ca4a61d5d69ccfadeef8408a4711c820b98959 Mon Sep 17 00:00:00 2001
From: Teoman ONAY <tonay@ibm.com>
Date: Tue, 3 Oct 2023 21:47:16 +0200
Subject: [PATCH 0657/2492] ceph-volume: fixes fallback to stat in is_device
 and is_partition

os.stat (or lstat) cannot distinguish a block device from
a partition.

Fixes: https://tracker.ceph.com/issues/58812

Signed-off-by: Teoman ONAY <tonay@ibm.com>
---
 .../ceph_volume/tests/util/test_device.py     |  8 --------
 .../ceph_volume/tests/util/test_disk.py       |  5 +++++
 src/ceph-volume/ceph_volume/util/disk.py      | 20 ++++---------------
 3 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/tests/util/test_device.py b/src/ceph-volume/ceph_volume/tests/util/test_device.py
index e2ea026286f9..69b57c21110e 100644
--- a/src/ceph-volume/ceph_volume/tests/util/test_device.py
+++ b/src/ceph-volume/ceph_volume/tests/util/test_device.py
@@ -152,14 +152,6 @@ def test_disk_is_device(self, fake_call, device_info):
         disk = device.Device("/dev/sda")
         assert disk.is_device is True
 
-    @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
-    def test_is_partition(self, fake_call, device_info):
-        data = {"/dev/sda1": {"foo": "bar"}}
-        lsblk = {"TYPE": "part", "NAME": "sda1", "PKNAME": "sda"}
-        device_info(devices=data, lsblk=lsblk)
-        disk = device.Device("/dev/sda1")
-        assert disk.is_partition
-
     @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False)
     def test_mpath_device_is_device(self, fake_call, device_info):
         data = {"/dev/foo": {"foo": "bar"}}
diff --git a/src/ceph-volume/ceph_volume/tests/util/test_disk.py b/src/ceph-volume/ceph_volume/tests/util/test_disk.py
index ce1f9466fd56..b287530dc17f 100644
--- a/src/ceph-volume/ceph_volume/tests/util/test_disk.py
+++ b/src/ceph-volume/ceph_volume/tests/util/test_disk.py
@@ -33,6 +33,11 @@ def test_is_device_type_mpath(self):
     def test_is_device_type_part(self):
         assert not disk.is_device('/dev/foo1')
 
+    @patch('ceph_volume.util.disk.os.path.exists', MagicMock(return_value=True))
+    @patch('ceph_volume.util.disk.get_partitions', MagicMock(return_value={"sda1": "sda"}))
+    def test_is_partition(self):
+        assert disk.is_partition('sda1')
+
 
 class TestLsblkParser(object):
 
diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index 8f1bf706934c..04104936985a 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -364,30 +364,18 @@ def is_device(dev):
         return TYPE in ['disk', 'mpath']
 
     # fallback to stat
-    return _stat_is_device(os.lstat(dev).st_mode)
+    return _stat_is_device(os.lstat(dev).st_mode) and not is_partition(dev)
 
 
-def is_partition(dev):
+def is_partition(dev: str) -> bool:
     """
     Boolean to determine if a given device is a partition, like /dev/sda1
     """
     if not os.path.exists(dev):
         return False
-    # use lsblk first, fall back to using stat
-    TYPE = lsblk(dev).get('TYPE')
-    if TYPE:
-        return TYPE == 'part'
-
-    # fallback to stat
-    stat_obj = os.stat(dev)
-    if _stat_is_device(stat_obj.st_mode):
-        return False
 
-    major = os.major(stat_obj.st_rdev)
-    minor = os.minor(stat_obj.st_rdev)
-    if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
-        return True
-    return False
+    partitions = get_partitions()
+    return dev.split("/")[-1] in partitions
 
 
 def is_ceph_rbd(dev):

From d0f8426615a7a090b1e0d0eabb4b084def50ce83 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 22 Aug 2023 15:52:42 +0530
Subject: [PATCH 0658/2492] mds/journal: make used_preallocated_ino crash more
 graceful

failure while removing used_preallocated_ino from session's prealloc_inos
set leads to backtrace pointing to interval_set:

interval_set<inodeno_t, std::map>::erase(inodeno_t, inodeno_t, std::function<bool (inodeno_t, inodeno_t)>)
EMetaBlob::replay(MDSRank*, LogSegment*, MDPeerUpdate*)
EUpdate::replay(MDSRank*)
MDLog::_replay_thread()
MDLog::ReplayThread::entry()

which is cumbersome to debug, while the actual root cause is still a WIP, make the crash more graceful
so that it helps while debugging.

Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/mds/journal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index f9eb8a1eecf2..e080b1176105 100644
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1620,8 +1620,9 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, int type, MDPeerUpdate
 	if (used_preallocated_ino) {
 	  if (!session->info.prealloc_inos.empty()) {
 	    inodeno_t ino = session->take_ino(used_preallocated_ino);
-	    session->info.prealloc_inos.erase(ino);
+            dout(5) "received ino " << ino << " from the session" << dendl;
 	    ceph_assert(ino == used_preallocated_ino);
+	    session->info.prealloc_inos.erase(ino);
 	  }
           mds->sessionmap.replay_dirty_session(session);
 	}

From c094f1a90954058a70255099ea8b877fd49db1ff Mon Sep 17 00:00:00 2001
From: Tongliang Deng <dengtongliang@gmail.com>
Date: Thu, 21 Jul 2022 11:54:12 +0000
Subject: [PATCH 0659/2492] rgwlc: lock_lambda overwrites ret val

`lock_lambda` captures `ret` by reference, it will overwrites
returned value of `bucket_lc_process` when `wait_backoff` is called.

Fixes: c069eb7ff09b52003fa00a5cc83b1e52370032f5.

Signed-off-by: Tongliang Deng <dengtongliang@gmail.com>
---
 src/rgw/rgw_lc.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 978f9736b8eb..4ae3a950ade6 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -2133,14 +2133,14 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
 
   utime_t lock_for_s(max_lock_secs, 0);
   const auto& lock_lambda = [&]() {
-    ret = lock->try_lock(this, lock_for_s, null_yield);
+    int ret = lock->try_lock(this, lock_for_s, null_yield);
     if (ret == 0) {
       return true;
     }
     if (ret == -EBUSY || ret == -EEXIST) {
       /* already locked by another lc processor */
       return false;
-      }
+    }
     return false;
   };
 

From 1cf5ecb93fb73a5ed4a82935f635f98efccf7de6 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Mon, 20 Nov 2023 22:04:12 +0800
Subject: [PATCH 0660/2492] osd: add more debug logs for sparse read

This will be very important to get to know what exactly has happened
when client get a very large number of extents.

URL: https://tracker.ceph.com/issues/63586
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/osd/PrimaryLogPG.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 1f493eee26cf..5cf9bbba68d3 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -5933,8 +5933,8 @@ int PrimaryLogPG::do_sparse_read(OpContext *ctx, OSDOp& osd_op) {
     encode(m, osd_op.outdata); // re-encode since it might be modified
     ::encode_destructively(data_bl, osd_op.outdata);
 
-    dout(10) << " sparse_read got " << r << " bytes from object "
-	     << soid << dendl;
+    dout(10) << " sparse_read got " << m.size() << " extents and " << r
+             << " bytes from object " << soid << dendl;
   }
 
   ctx->delta_stats.num_rd_kb += shift_round_up(op.extent.length, 10);

From 08c16aa113dc22dfec3387f575752a5049b7af42 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 21 Nov 2023 02:39:39 +1000
Subject: [PATCH 0661/2492] doc/rados: edit "understanding mon_status"

Edit the section "Understanding mon_status" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 46 +++++++++++--------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 1170da7c33f6..f31c8aa20e97 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -127,10 +127,11 @@ and ``quorum_status``.
 Understanding mon_status
 ========================
 
-The status of the monitor (as reported by the ``ceph tell mon.X mon_status``
-command) can always be obtained via the admin socket. This command outputs a
-great deal of information about the monitor (including the information found in
-the output of the ``quorum_status`` command).
+The status of a Monitor (as reported by the ``ceph tell mon.X mon_status``
+command) can be obtained via the admin socket. The ``ceph tell mon.X
+mon_status``  command outputs a great deal of information about the monitor
+(including the information found in the output of the ``quorum_status``
+command).
 
 To understand this command's output, let us consider the following example, in
 which we see the output of ``ceph tell mon.c mon_status``::
@@ -160,29 +161,34 @@ which we see the output of ``ceph tell mon.c mon_status``::
                 "name": "c",
                 "addr": "127.0.0.1:6795\/0"}]}}
 
-It is clear that there are three monitors in the monmap (*a*, *b*, and *c*),
-the quorum is formed by only two monitors, and *c* is in the quorum as a
-*peon*.
+This output reports that there are three monitors in the monmap (*a*, *b*, and
+*c*), that quorum is formed by only two monitors, and that *c* is in quorum as
+a *peon*.
 
-**Which monitor is out of the quorum?**
+**Which monitor is out of quorum?**
 
-  The answer is **a** (that is, ``mon.a``).
+  The answer is **a** (that is, ``mon.a``). ``mon.a`` is out of quorum.
 
-**Why?**
+**How do we know, in this example, that mon.a is out of quorum?**
 
-  When the ``quorum`` set is examined, there are clearly two monitors in the
-  set: *1* and *2*. But these are not monitor names. They are monitor ranks, as
-  established in the current ``monmap``. The ``quorum`` set does not include
-  the monitor that has rank 0, and according to the ``monmap`` that monitor is
-  ``mon.a``.
+  We know that ``mon.a`` is out of quorum because it has rank 0, and Monitors
+  with rank 0 are by definition out of quorum.
+
+  If we examine the ``quorum`` set, we can see that there are clearly two
+  monitors in the set: *1* and *2*. But these are not monitor names. They are
+  monitor ranks, as established in the current ``monmap``. The ``quorum`` set
+  does not include the monitor that has rank 0, and according to the ``monmap``
+  that monitor is ``mon.a``.
 
 **How are monitor ranks determined?**
 
-  Monitor ranks are calculated (or recalculated) whenever monitors are added or
-  removed. The calculation of ranks follows a simple rule: the **greater** the
-  ``IP:PORT`` combination, the **lower** the rank. In this case, because
-  ``127.0.0.1:6789`` is lower than the other two ``IP:PORT`` combinations,
-  ``mon.a`` has the highest rank: namely, rank 0.
+  Monitor ranks are calculated (or recalculated) whenever monitors are added to
+  or removed from the cluster. The calculation of ranks follows a simple rule:
+  the **greater** the ``IP:PORT`` combination, the **lower** the rank. In this
+  case, because ``127.0.0.1:6789`` (``mon.a``) is numerically less than the
+  other two ``IP:PORT`` combinations (which are ``127.0.0.1:6790`` for "Monitor
+  b" and ``127.0.0.1:6795`` for "Monitor c"), ``mon.a`` has the highest rank:
+  namely, rank 0.
   
 
 Most Common Monitor Issues

From fd4e52b042b5f86350968de610e4e71cba379033 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 20 Nov 2023 12:44:17 -0600
Subject: [PATCH 0662/2492] tools: modify ceph_dedup_tool to maintain Clang 15
 compatibility

Adding 'typename' in two instances, where version 15 of Clang
still requires it. P0634R3, which made those 'typename' redundant,
is only supported starting Clang 16.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/tools/ceph_dedup_tool.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc
index c67368f9318c..91a991c011a3 100644
--- a/src/tools/ceph_dedup_tool.cc
+++ b/src/tools/ceph_dedup_tool.cc
@@ -581,9 +581,9 @@ class SampleDedupWorkerThread : public Thread
       } entry_into = NONE;
 
       /// Valid iterator into map for UNDER|OVER, default for NONE
-      map_t::iterator iter;
+      typename map_t::iterator iter;
 
-      entry_t(entry_into_t entry_into, map_t::iterator iter) :
+      entry_t(entry_into_t entry_into, typename map_t::iterator iter) :
 	entry_into(entry_into), iter(iter) {
 	ceph_assert(entry_into != NONE);
       }

From 7d48e8aa259266dcdc06f11a4ad67a764108b6b1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 20 Nov 2023 16:43:52 -0500
Subject: [PATCH 0663/2492] cephadm: add a custom template not found exception
 with diagnostic info

Add a new exception based on jinja2's template not found exception for
the case where the template was not found in the zip(app). We've been
having sporadic failures with this in CI & testing and hopefully
the additional information will help pinpoint the cause.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 37 +++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index e6e8d5e0ea2c..ceef32ff9fed 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -29,6 +29,34 @@ def __repr__(self) -> str:
         return repr(self.value)
 
 
+class TemplateNotFoundInZipApp(jinja2.TemplateNotFound):
+    def __init__(
+        self,
+        template: str,
+        *,
+        path: str = '',
+        relative_path: str = '',
+        archive_norm_path: str = '',
+        archive_path: str = ''
+    ) -> None:
+        super().__init__(template)
+        self.path = path
+        self.relative_path = relative_path
+        self.archive_norm_path = archive_norm_path
+        self.archive_path = archive_path
+
+    def __str__(self) -> str:
+        msg = self.message
+        msg2 = ''
+        if self.path or self.relative_path:
+            msg2 += f' path [{self.path!r}, rel={self.relative_path!r}] not found'
+        if self.archive_norm_path or self.archive_path:
+            msg2 += f' in [{self.archive_norm_path!r}, orig={self.archive_path!r}]'
+        if msg2:
+            msg2 = ':' + msg2
+        return f'{msg}{msg2}'
+
+
 class _PackageLoader(jinja2.PackageLoader):
     """Workaround for PackageLoader when using cephadm with relative paths.
 
@@ -71,7 +99,14 @@ def _get_archive_source(self, template: str) -> Tuple[str, str, None]:
         try:
             source = cast(bytes, self._loader.get_data(arelpath))
         except OSError as e:
-            raise jinja2.TemplateNotFound(template) from e
+            not_found = TemplateNotFoundInZipApp(
+                template,
+                path=path,
+                relative_path=arelpath,
+                archive_norm_path=archive_path,
+                archive_path=self._loader.archive,
+            )
+            raise not_found from e
         return source.decode(self.encoding), path, None
 
 

From 838489f6b1e1911f71ed2af5c2b9a2eaf5e71bc6 Mon Sep 17 00:00:00 2001
From: Brad Hubbard <bhubbard@redhat.com>
Date: Mon, 20 Nov 2023 14:43:31 +1000
Subject: [PATCH 0664/2492] do_cmake.sh: set python version for Fedora 39

If do_cmake.sh is being exeuted on Fedora 39 set Python version to 3.12.
Remove versions for anything earlier than Fedora 37

Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
---
 do_cmake.sh | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/do_cmake.sh b/do_cmake.sh
index e838bca8b976..90f8a3381df3 100755
--- a/do_cmake.sh
+++ b/do_cmake.sh
@@ -19,16 +19,11 @@ if [ -r /etc/os-release ]; then
   source /etc/os-release
   case "$ID" in
       fedora)
-          if [ "$VERSION_ID" -ge "37" ] ; then
-            PYBUILD="3.11"
-          elif [ "$VERSION_ID" -ge "35" ] ; then
-            PYBUILD="3.10"
-          elif [ "$VERSION_ID" -ge "33" ] ; then
-            PYBUILD="3.9"
-          elif [ "$VERSION_ID" -ge "32" ] ; then
-            PYBUILD="3.8"
+          if [ "$VERSION_ID" -ge "39" ] ; then
+            PYBUILD="3.12"
           else
-            PYBUILD="3.7"
+            # Fedora 37 and above
+            PYBUILD="3.11"
           fi
           ;;
       rocky|rhel|centos)

From 2f931f5114df7166bd2b6453cb23df70970c5fd5 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 16 Nov 2023 02:42:37 -0500
Subject: [PATCH 0665/2492] mds: add debug when trying to merge directory
 fragments

Helps in debugging why a dirfrag is getting merged.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDBalancer.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index cd2e7d9bd002..b0b26132aa2e 100644
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -684,7 +684,12 @@ void MDBalancer::queue_merge(CDir *dir)
       }
       bool all = true;
       for (auto& sib : sibs) {
-        if (!sib->is_auth() || !sib->should_merge()) {
+	auto is_auth = sib->is_auth();
+	auto should_merge = sib->should_merge();
+
+	dout(20) << ": sib=" << *sib << ", is_auth=" << is_auth << ", should_merge="
+		 << should_merge << dendl;
+        if (!is_auth || !should_merge) {
           all = false;
           break;
         }

From 2958a7cd1259cdfb6e04ebc5047cadb86bf90707 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 20 Nov 2023 00:44:24 -0500
Subject: [PATCH 0666/2492] qa: set mds config with `config set` for a
 particular test

The config setting is persisted in ceph.conf after the MDSs are started.
However, the test case fails the file system causing the active MDS to
restart and pick up the new config. When the file system is marked joinable,
then, if the MDS which was standby before the file system was marked failed
takes over as the rank, the updated setting are not used by this MDS.

In the failed test, merging directory fragment is disabled, but since
the config is set in ceph.conf, the (earlier standby) MDS which acquires
a rank uses the default merge size causing the dirfrag to merge and
thereby tripping the test.

Fixes: http://tracker.ceph.com/issues/57087
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_data_scan.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index 63ac60415475..5a7c96996f9e 100644
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -491,10 +491,11 @@ def test_fragmented_injection(self):
 
         file_count = 100
         file_names = ["%s" % n for n in range(0, file_count)]
+        split_size = 100 * file_count
 
         # Make sure and disable dirfrag auto merging and splitting
-        self.fs.set_ceph_conf('mds', 'mds bal merge size', 0)
-        self.fs.set_ceph_conf('mds', 'mds bal split size', 100 * file_count)
+        self.fs.mon_manager.run_cluster_cmd(args='config set mds mds_bal_merge_size 0')
+        self.fs.mon_manager.run_cluster_cmd(args=f'config set mds mds_bal_split_size {split_size}')
 
         # Create a directory of `file_count` files, each named after its
         # decimal number and containing the string of its decimal number

From ebea0f89354882adfafa310aa61c206a8aefa8c9 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 21 Nov 2023 14:59:23 +0530
Subject: [PATCH 0667/2492] Update qa/tasks/cephfs/test_data_scan.py

Co-authored-by: Dhairya Parmar <87528150+dparmar18@users.noreply.github.com>
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_data_scan.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index 5a7c96996f9e..f9f853247d16 100644
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -494,8 +494,8 @@ def test_fragmented_injection(self):
         split_size = 100 * file_count
 
         # Make sure and disable dirfrag auto merging and splitting
-        self.fs.mon_manager.run_cluster_cmd(args='config set mds mds_bal_merge_size 0')
-        self.fs.mon_manager.run_cluster_cmd(args=f'config set mds mds_bal_split_size {split_size}')
+        self.config_set('mds', 'mds_bal_merge_size', 0)
+        self.config_set('mds', 'mds_bal_split_size', split_size)
 
         # Create a directory of `file_count` files, each named after its
         # decimal number and containing the string of its decimal number

From 8d4d5faf69a89810b59aef8cf1cf872e50d9525a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 21 Nov 2023 09:14:52 -0500
Subject: [PATCH 0668/2492] cmake: ceph-dencoder depends on
 ceph-dencoder-modules

building `ceph-dencoder` alone should also build the modules necessary to
run it. otherwise these modules are only built if all targets are built

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/tools/ceph-dencoder/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/ceph-dencoder/CMakeLists.txt b/src/tools/ceph-dencoder/CMakeLists.txt
index 5cb56e136159..86cab179590c 100644
--- a/src/tools/ceph-dencoder/CMakeLists.txt
+++ b/src/tools/ceph-dencoder/CMakeLists.txt
@@ -20,6 +20,7 @@ set_target_properties(ceph-dencoder PROPERTIES
 
 set(denc_plugin_dir ${CEPH_INSTALL_FULL_PKGLIBDIR}/denc)
 add_custom_target(ceph-dencoder-modules)
+add_dependencies(ceph-dencoder ceph-dencoder-modules)
 
 function(add_denc_mod name)
   add_library(${name} SHARED

From 99e58f5854725ed42d91a7bc0d8db9a251701e1b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 25 Oct 2023 21:30:31 -0400
Subject: [PATCH 0669/2492] mds: use const qualifier for MDRequestRef

The ref should be const even if its contents are not.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc   |  34 +++----
 src/mds/Locker.h    |  30 +++---
 src/mds/MDCache.cc  |  60 +++++------
 src/mds/MDCache.h   |  54 +++++-----
 src/mds/MDSRank.cc  |   2 +-
 src/mds/MDSRank.h   |   2 +-
 src/mds/Migrator.cc |   2 +-
 src/mds/Migrator.h  |   2 +-
 src/mds/Server.cc   | 243 ++++++++++++++++++++++----------------------
 src/mds/Server.h    | 192 +++++++++++++++++-----------------
 10 files changed, 311 insertions(+), 310 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index b4a28fb8379f..3b75e16937ee 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -139,7 +139,7 @@ void Locker::send_lock_message(SimpleLock *lock, int msg, const bufferlist &data
   }
 }
 
-bool Locker::try_rdlock_snap_layout(CInode *in, MDRequestRef& mdr,
+bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 				    int n, bool want_layout)
 {
   dout(10) << __func__ << " " << *mdr << " " << *in << dendl;
@@ -205,10 +205,10 @@ bool Locker::try_rdlock_snap_layout(CInode *in, MDRequestRef& mdr,
 }
 
 struct MarkEventOnDestruct {
-  MDRequestRef& mdr;
+  MDRequestRef mdr;
   std::string_view message;
   bool mark_event;
-  MarkEventOnDestruct(MDRequestRef& _mdr, std::string_view _message) :
+  MarkEventOnDestruct(const MDRequestRef& _mdr, std::string_view _message) :
       mdr(_mdr),
       message(_message),
       mark_event(true) {}
@@ -220,7 +220,7 @@ struct MarkEventOnDestruct {
 
 /* If this function returns false, the mdr has been placed
  * on the appropriate wait list */
-bool Locker::acquire_locks(MDRequestRef& mdr,
+bool Locker::acquire_locks(const MDRequestRef& mdr,
 			   MutationImpl::LockOpVec& lov,
 			   CInode *auth_pin_freeze,
 			   bool auth_pin_nonblocking)
@@ -892,7 +892,7 @@ void Locker::invalidate_lock_caches(SimpleLock *lock)
   }
 }
 
-void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri, file_layout_t *dir_layout)
+void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layout_t *dir_layout)
 {
   if (mdr->lock_cache)
     return;
@@ -1039,7 +1039,7 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri, file_layout_t *d
   mdr->lock_cache = lock_cache;
 }
 
-bool Locker::find_and_attach_lock_cache(MDRequestRef& mdr, CInode *diri)
+bool Locker::find_and_attach_lock_cache(const MDRequestRef& mdr, CInode *diri)
 {
   if (mdr->lock_cache)
     return true;
@@ -1569,7 +1569,7 @@ bool Locker::rdlock_try(SimpleLock *lock, client_t client)
   return false;
 }
 
-bool Locker::rdlock_start(SimpleLock *lock, MDRequestRef& mut, bool as_anon)
+bool Locker::rdlock_start(SimpleLock *lock, const MDRequestRef& mut, bool as_anon)
 {
   dout(7) << "rdlock_start  on " << *lock << " on " << *lock->get_parent() << dendl;  
 
@@ -1660,7 +1660,7 @@ void Locker::rdlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
   }
 }
 
-bool Locker::rdlock_try_set(MutationImpl::LockOpVec& lov, MDRequestRef& mdr)
+bool Locker::rdlock_try_set(MutationImpl::LockOpVec& lov, const MDRequestRef& mdr)
 {
   dout(10) << __func__  << dendl;
   for (const auto& p : lov) {
@@ -1747,7 +1747,7 @@ bool Locker::wrlock_try(SimpleLock *lock, const MutationRef& mut, client_t clien
   return false;
 }
 
-bool Locker::wrlock_start(const MutationImpl::LockOp &op, MDRequestRef& mut)
+bool Locker::wrlock_start(const MutationImpl::LockOp &op, const MDRequestRef& mut)
 {
   SimpleLock *lock = op.lock;
   if (lock->get_type() == CEPH_LOCK_IVERSION ||
@@ -1840,7 +1840,7 @@ void Locker::wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
 
 // remote wrlock
 
-void Locker::remote_wrlock_start(SimpleLock *lock, mds_rank_t target, MDRequestRef& mut)
+void Locker::remote_wrlock_start(SimpleLock *lock, mds_rank_t target, const MDRequestRef& mut)
 {
   dout(7) << "remote_wrlock_start mds." << target << " on " << *lock << " on " << *lock->get_parent() << dendl;
 
@@ -1891,7 +1891,7 @@ void Locker::remote_wrlock_finish(const MutationImpl::lock_iterator& it, Mutatio
 // ------------------
 // xlock
 
-bool Locker::xlock_start(SimpleLock *lock, MDRequestRef& mut)
+bool Locker::xlock_start(SimpleLock *lock, const MDRequestRef& mut)
 {
   if (lock->get_type() == CEPH_LOCK_IVERSION ||
       lock->get_type() == CEPH_LOCK_DVERSION)
@@ -2179,7 +2179,7 @@ void Locker::file_update_finish(CInode *in, MutationRef& mut, unsigned flags,
 
 Capability* Locker::issue_new_caps(CInode *in,
 				   int mode,
-				   MDRequestRef& mdr,
+				   const MDRequestRef& mdr,
 				   SnapRealm *realm)
 {
   dout(7) << "issue_new_caps for mode " << mode << " on " << *in << dendl;
@@ -3415,7 +3415,7 @@ class C_Locker_RetryRequestCapRelease : public LockerContext {
   }
 };
 
-void Locker::process_request_cap_release(MDRequestRef& mdr, client_t client, const ceph_mds_request_release& item,
+void Locker::process_request_cap_release(const MDRequestRef& mdr, client_t client, const ceph_mds_request_release& item,
 					 std::string_view dname)
 {
   inodeno_t ino = (uint64_t)item.ino;
@@ -3534,7 +3534,7 @@ void Locker::kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq)
   issue_caps(in, cap);
 }
 
-void Locker::kick_cap_releases(MDRequestRef& mdr)
+void Locker::kick_cap_releases(const MDRequestRef& mdr)
 {
   client_t client = mdr->get_client();
   for (map<vinodeno_t,ceph_seq_t>::iterator p = mdr->cap_releases.begin();
@@ -4253,7 +4253,7 @@ void Locker::handle_client_lease(const cref_t<MClientLease> &m)
 }
 
 
-void Locker::issue_client_lease(CDentry *dn, CInode *in, MDRequestRef &mdr, utime_t now,
+void Locker::issue_client_lease(CDentry *dn, CInode *in, const MDRequestRef& mdr, utime_t now,
                                 bufferlist &bl)
 {
   client_t client = mdr->get_client();
@@ -5329,7 +5329,7 @@ void Locker::local_wrlock_grab(LocalLockC *lock, MutationRef& mut)
   ceph_assert(it->is_wrlock());
 }
 
-bool Locker::local_wrlock_start(LocalLockC *lock, MDRequestRef& mut)
+bool Locker::local_wrlock_start(LocalLockC *lock, const MDRequestRef& mut)
 {
   dout(7) << "local_wrlock_start  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
@@ -5361,7 +5361,7 @@ void Locker::local_wrlock_finish(const MutationImpl::lock_iterator& it, Mutation
   }
 }
 
-bool Locker::local_xlock_start(LocalLockC *lock, MDRequestRef& mut)
+bool Locker::local_xlock_start(LocalLockC *lock, const MDRequestRef& mut)
 {
   dout(7) << "local_xlock_start  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 03772f029cd7..eed421ba0db9 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -51,12 +51,12 @@ class Locker {
 
   void nudge_log(SimpleLock *lock);
 
-  bool acquire_locks(MDRequestRef& mdr,
+  bool acquire_locks(const MDRequestRef& mdr,
 		     MutationImpl::LockOpVec& lov,
 		     CInode *auth_pin_freeze=NULL,
 		     bool auth_pin_nonblocking=false);
 
-  bool try_rdlock_snap_layout(CInode *in, MDRequestRef& mdr,
+  bool try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 			      int n=0, bool want_layout=false);
 
   void notify_freeze_waiter(MDSCacheObject *o);
@@ -68,8 +68,8 @@ class Locker {
   void drop_locks_for_fragment_unfreeze(MutationImpl *mut);
 
   int get_cap_bit_for_lock_cache(int op);
-  void create_lock_cache(MDRequestRef& mdr, CInode *diri, file_layout_t *dir_layout=nullptr);
-  bool find_and_attach_lock_cache(MDRequestRef& mdr, CInode *diri);
+  void create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layout_t *dir_layout=nullptr);
+  bool find_and_attach_lock_cache(const MDRequestRef& mdr, CInode *diri);
   void invalidate_lock_caches(CDir *dir);
   void invalidate_lock_caches(SimpleLock *lock);
   void invalidate_lock_cache(MDLockCache *lock_cache);
@@ -95,20 +95,20 @@ class Locker {
 
   bool _rdlock_kick(SimpleLock *lock, bool as_anon);
   bool rdlock_try(SimpleLock *lock, client_t client);
-  bool rdlock_start(SimpleLock *lock, MDRequestRef& mut, bool as_anon=false);
+  bool rdlock_start(SimpleLock *lock, const MDRequestRef& mut, bool as_anon=false);
   void rdlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut, bool *pneed_issue);
-  bool rdlock_try_set(MutationImpl::LockOpVec& lov, MDRequestRef& mdr);
+  bool rdlock_try_set(MutationImpl::LockOpVec& lov, const MDRequestRef& mdr);
   bool rdlock_try_set(MutationImpl::LockOpVec& lov, MutationRef& mut);
 
   void wrlock_force(SimpleLock *lock, MutationRef& mut);
   bool wrlock_try(SimpleLock *lock, const MutationRef& mut, client_t client=-1);
-  bool wrlock_start(const MutationImpl::LockOp &op, MDRequestRef& mut);
+  bool wrlock_start(const MutationImpl::LockOp &op, const MDRequestRef& mut);
   void wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut, bool *pneed_issue);
 
-  void remote_wrlock_start(SimpleLock *lock, mds_rank_t target, MDRequestRef& mut);
+  void remote_wrlock_start(SimpleLock *lock, mds_rank_t target, const MDRequestRef& mut);
   void remote_wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut);
 
-  bool xlock_start(SimpleLock *lock, MDRequestRef& mut);
+  bool xlock_start(SimpleLock *lock, const MDRequestRef& mut);
   void _finish_xlock(SimpleLock *lock, client_t xlocker, bool *pneed_issue);
   void xlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut, bool *pneed_issue);
 
@@ -137,10 +137,10 @@ class Locker {
   // process_request_cap_release to preserve ordering.
   bool should_defer_client_cap_frozen(CInode *in);
 
-  void process_request_cap_release(MDRequestRef& mdr, client_t client, const ceph_mds_request_release& r,
+  void process_request_cap_release(const MDRequestRef& mdr, client_t client, const ceph_mds_request_release& r,
 				   std::string_view dname);
 
-  void kick_cap_releases(MDRequestRef& mdr);
+  void kick_cap_releases(const MDRequestRef& mdr);
   void kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq);
 
   void remove_client_cap(CInode *in, Capability *cap, bool kill=false);
@@ -162,7 +162,7 @@ class Locker {
 
   // -- file i/o --
   version_t issue_file_data_version(CInode *in);
-  Capability* issue_new_caps(CInode *in, int mode, MDRequestRef& mdr, SnapRealm *conrealm);
+  Capability* issue_new_caps(CInode *in, int mode, const MDRequestRef& mdr, SnapRealm *conrealm);
   int get_allowed_caps(CInode *in, Capability *cap, int &all_allowed,
                        int &loner_allowed, int &xlocker_allowed);
   int issue_caps(CInode *in, Capability *only_cap=0);
@@ -186,7 +186,7 @@ class Locker {
   // -- client leases --
   void handle_client_lease(const cref_t<MClientLease> &m);
 
-  void issue_client_lease(CDentry *dn, CInode *in, MDRequestRef &mdr, utime_t now, bufferlist &bl);
+  void issue_client_lease(CDentry *dn, CInode *in, const MDRequestRef &mdr, utime_t now, bufferlist &bl);
   void revoke_client_leases(SimpleLock *lock);
   static void encode_lease(bufferlist& bl, const session_info_t& info, const LeaseStat& ls);
 
@@ -225,9 +225,9 @@ class Locker {
   void _do_cap_release(client_t client, inodeno_t ino, uint64_t cap_id, ceph_seq_t mseq, ceph_seq_t seq);
   void caps_tick();
 
-  bool local_wrlock_start(LocalLockC *lock, MDRequestRef& mut);
+  bool local_wrlock_start(LocalLockC *lock, const MDRequestRef& mut);
   void local_wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut);
-  bool local_xlock_start(LocalLockC *lock, MDRequestRef& mut);
+  bool local_xlock_start(LocalLockC *lock, const MDRequestRef& mut);
   void local_xlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *mut);
 
   void handle_file_lock(ScatterLock *lock, const cref_t<MLock> &m);
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 35fca081a6be..cffa6e7dad01 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3531,7 +3531,7 @@ MDPeerUpdate* MDCache::get_uncommitted_peer(metareqid_t reqid, mds_rank_t leader
   return su;
 }
 
-void MDCache::finish_rollback(metareqid_t reqid, MDRequestRef& mdr) {
+void MDCache::finish_rollback(metareqid_t reqid, const MDRequestRef& mdr) {
   auto p = resolve_need_rollback.find(reqid);
   ceph_assert(p != resolve_need_rollback.end());
   if (mds->is_resolve()) {
@@ -8255,7 +8255,7 @@ void MDCache::dispatch(const cref_t<Message> &m)
   }
 }
 
-int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
+int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
                            const filepath& path, int flags,
                            vector<CDentry*> *pdnvec, CInode **pin)
 {
@@ -8672,7 +8672,7 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
   return 0;
 }
 
-int MDCache::maybe_request_forward_to_auth(MDRequestRef& mdr, MDSContextFactory& cf,
+int MDCache::maybe_request_forward_to_auth(const MDRequestRef& mdr, MDSContextFactory& cf,
 					   MDSCacheObject *p)
 {
   if (p->is_ambiguous_auth()) {
@@ -8754,7 +8754,7 @@ void MDCache::open_remote_dirfrag(CInode *diri, frag_t approxfg, MDSContext *fin
  * will return inode for primary, or link up/open up remote link's inode as necessary.
  * If it's not available right now, puts mdr on wait list and returns null.
  */
-CInode *MDCache::get_dentry_inode(CDentry *dn, MDRequestRef& mdr, bool projected)
+CInode *MDCache::get_dentry_inode(CDentry *dn, const MDRequestRef& mdr, bool projected)
 {
   CDentry::linkage_t *dnl;
   if (projected)
@@ -9682,7 +9682,7 @@ MDRequestRef MDCache::request_get(metareqid_t rid)
   return p->second;
 }
 
-void MDCache::request_finish(MDRequestRef& mdr)
+void MDCache::request_finish(const MDRequestRef& mdr)
 {
   dout(7) << "request_finish " << *mdr << dendl;
   mdr->mark_event("finishing request");
@@ -9729,7 +9729,7 @@ void MDCache::request_finish(MDRequestRef& mdr)
 }
 
 
-void MDCache::request_forward(MDRequestRef& mdr, mds_rank_t who, int port)
+void MDCache::request_forward(const MDRequestRef& mdr, mds_rank_t who, int port)
 {
   CachedStackStringStream css;
   *css << "forwarding request to mds." << who;
@@ -9754,7 +9754,7 @@ void MDCache::request_forward(MDRequestRef& mdr, mds_rank_t who, int port)
 }
 
 
-void MDCache::dispatch_request(MDRequestRef& mdr)
+void MDCache::dispatch_request(const MDRequestRef& mdr)
 {
   if (mdr->client_request) {
     mds->server->dispatch_client_request(mdr);
@@ -9790,7 +9790,7 @@ void MDCache::dispatch_request(MDRequestRef& mdr)
 }
 
 
-void MDCache::request_drop_foreign_locks(MDRequestRef& mdr)
+void MDCache::request_drop_foreign_locks(const MDRequestRef& mdr)
 {
   if (!mdr->has_more())
     return;
@@ -9845,19 +9845,19 @@ void MDCache::request_drop_foreign_locks(MDRequestRef& mdr)
                                 * this function can get called more than once */
 }
 
-void MDCache::request_drop_non_rdlocks(MDRequestRef& mdr)
+void MDCache::request_drop_non_rdlocks(const MDRequestRef& mdr)
 {
   request_drop_foreign_locks(mdr);
   mds->locker->drop_non_rdlocks(mdr.get());
 }
 
-void MDCache::request_drop_locks(MDRequestRef& mdr)
+void MDCache::request_drop_locks(const MDRequestRef& mdr)
 {
   request_drop_foreign_locks(mdr);
   mds->locker->drop_locks(mdr.get());
 }
 
-void MDCache::request_cleanup(MDRequestRef& mdr)
+void MDCache::request_cleanup(const MDRequestRef& mdr)
 {
   dout(15) << "request_cleanup " << *mdr << dendl;
 
@@ -9899,7 +9899,7 @@ void MDCache::request_cleanup(MDRequestRef& mdr)
   mdr->mark_event("cleaned up request");
 }
 
-void MDCache::request_kill(MDRequestRef& mdr)
+void MDCache::request_kill(const MDRequestRef& mdr)
 {
   // rollback peer requests is tricky. just let the request proceed.
   if (mdr->has_more() &&
@@ -11147,7 +11147,7 @@ void MDCache::decode_remote_dentry_link(CDir *dir, CDentry *dn, bufferlist::cons
   DECODE_FINISH(p);
 }
 
-void MDCache::send_dentry_link(CDentry *dn, MDRequestRef& mdr)
+void MDCache::send_dentry_link(CDentry *dn, const MDRequestRef& mdr)
 {
   dout(7) << __func__ << " " << *dn << dendl;
 
@@ -11217,7 +11217,7 @@ void MDCache::handle_dentry_link(const cref_t<MDentryLink> &m)
 
 // UNLINK
 
-void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequestRef& mdr)
+void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, const MDRequestRef& mdr)
 {
   dout(10) << __func__ << " " << *dn << dendl;
   // share unlink news with replicas
@@ -11537,7 +11537,7 @@ class C_MDC_FragmentFrozen : public MDSInternalContext {
   MDCache *mdcache;
   MDRequestRef mdr;
 public:
-  C_MDC_FragmentFrozen(MDCache *m, MDRequestRef& r) :
+  C_MDC_FragmentFrozen(MDCache *m, const MDRequestRef& r) :
     MDSInternalContext(m->mds), mdcache(m), mdr(r) {}
   void finish(int r) override {
     mdcache->fragment_frozen(mdr, r);
@@ -11695,13 +11695,13 @@ void MDCache::fragment_freeze_dirs(const std::vector<CDir*>& dirs)
 class C_MDC_FragmentMarking : public MDCacheContext {
   MDRequestRef mdr;
 public:
-  C_MDC_FragmentMarking(MDCache *m, MDRequestRef& r) : MDCacheContext(m), mdr(r) {}
+  C_MDC_FragmentMarking(MDCache *m, const MDRequestRef& r) : MDCacheContext(m), mdr(r) {}
   void finish(int r) override {
     mdcache->fragment_mark_and_complete(mdr);
   }
 };
 
-void MDCache::fragment_mark_and_complete(MDRequestRef& mdr)
+void MDCache::fragment_mark_and_complete(const MDRequestRef& mdr)
 {
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
@@ -11883,7 +11883,7 @@ void MDCache::find_stale_fragment_freeze()
 class C_MDC_FragmentPrep : public MDCacheLogContext {
   MDRequestRef mdr;
 public:
-  C_MDC_FragmentPrep(MDCache *m, MDRequestRef& r) : MDCacheLogContext(m),  mdr(r) {}
+  C_MDC_FragmentPrep(MDCache *m, const MDRequestRef& r) : MDCacheLogContext(m),  mdr(r) {}
   void finish(int r) override {
     mdcache->_fragment_logged(mdr);
   }
@@ -11892,7 +11892,7 @@ class C_MDC_FragmentPrep : public MDCacheLogContext {
 class C_MDC_FragmentStore : public MDCacheContext {
   MDRequestRef mdr;
 public:
-  C_MDC_FragmentStore(MDCache *m, MDRequestRef& r) : MDCacheContext(m), mdr(r) {}
+  C_MDC_FragmentStore(MDCache *m, const MDRequestRef& r) : MDCacheContext(m), mdr(r) {}
   void finish(int r) override {
     mdcache->_fragment_stored(mdr);
   }
@@ -11926,7 +11926,7 @@ class C_IO_MDC_FragmentPurgeOld : public MDCacheIOContext {
   }
 };
 
-void MDCache::fragment_frozen(MDRequestRef& mdr, int r)
+void MDCache::fragment_frozen(const MDRequestRef& mdr, int r)
 {
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
@@ -11945,7 +11945,7 @@ void MDCache::fragment_frozen(MDRequestRef& mdr, int r)
   dispatch_fragment_dir(mdr);
 }
 
-void MDCache::dispatch_fragment_dir(MDRequestRef& mdr)
+void MDCache::dispatch_fragment_dir(const MDRequestRef& mdr)
 {
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   map<dirfrag_t,fragment_info_t>::iterator it = fragments.find(basedirfrag);
@@ -12050,7 +12050,7 @@ void MDCache::dispatch_fragment_dir(MDRequestRef& mdr)
   mds->mdlog->flush();
 }
 
-void MDCache::_fragment_logged(MDRequestRef& mdr)
+void MDCache::_fragment_logged(const MDRequestRef& mdr)
 {
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   auto& info = fragments.at(basedirfrag);
@@ -12080,7 +12080,7 @@ void MDCache::_fragment_logged(MDRequestRef& mdr)
   gather.activate();
 }
 
-void MDCache::_fragment_stored(MDRequestRef& mdr)
+void MDCache::_fragment_stored(const MDRequestRef& mdr)
 {
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   fragment_info_t &info = fragments.at(basedirfrag);
@@ -12972,7 +12972,7 @@ void MDCache::enqueue_scrub(
   enqueue_scrub_work(mdr);
 }
 
-void MDCache::enqueue_scrub_work(MDRequestRef& mdr)
+void MDCache::enqueue_scrub_work(const MDRequestRef& mdr)
 {
   CInode *in;
   CF_MDS_RetryRequestFactory cf(this, mdr, true);
@@ -13004,7 +13004,7 @@ void MDCache::enqueue_scrub_work(MDRequestRef& mdr)
 
 struct C_MDC_RespondInternalRequest : public MDCacheLogContext {
   MDRequestRef mdr;
-  C_MDC_RespondInternalRequest(MDCache *c, MDRequestRef& m) :
+  C_MDC_RespondInternalRequest(MDCache *c, const MDRequestRef& m) :
     MDCacheLogContext(c), mdr(m) {}
   void finish(int r) override {
     mdr->apply();
@@ -13036,7 +13036,7 @@ void MDCache::repair_dirfrag_stats(CDir *dir)
   repair_dirfrag_stats_work(mdr);
 }
 
-void MDCache::repair_dirfrag_stats_work(MDRequestRef& mdr)
+void MDCache::repair_dirfrag_stats_work(const MDRequestRef& mdr)
 {
   CDir *dir = static_cast<CDir*>(mdr->internal_op_private);
   dout(10) << __func__ << " " << *dir << dendl;
@@ -13143,7 +13143,7 @@ void MDCache::repair_inode_stats(CInode *diri)
   repair_inode_stats_work(mdr);
 }
 
-void MDCache::repair_inode_stats_work(MDRequestRef& mdr)
+void MDCache::repair_inode_stats_work(const MDRequestRef& mdr)
 {
   CInode *diri = static_cast<CInode*>(mdr->internal_op_private);
   dout(10) << __func__ << " " << *diri << dendl;
@@ -13243,7 +13243,7 @@ void MDCache::rdlock_dirfrags_stats(CInode *diri, MDSInternalContext* fin)
   return rdlock_dirfrags_stats_work(mdr);
 }
 
-void MDCache::rdlock_dirfrags_stats_work(MDRequestRef& mdr)
+void MDCache::rdlock_dirfrags_stats_work(const MDRequestRef& mdr)
 {
   CInode *diri = static_cast<CInode*>(mdr->internal_op_private);
   dout(10) << __func__ << " " << *diri << dendl;
@@ -13289,11 +13289,11 @@ class C_FinishIOMDR : public MDSContext {
   MDRequestRef mdr;
   MDSRank *get_mds() override { return mds; }
 public:
-  C_FinishIOMDR(MDSRank *mds_, MDRequestRef& mdr_) : mds(mds_), mdr(mdr_) {}
+  C_FinishIOMDR(MDSRank *mds_, const MDRequestRef& mdr_) : mds(mds_), mdr(mdr_) {}
   void finish(int r) override { mds->server->respond_to_request(mdr, r); }
 };
 
-void MDCache::flush_dentry_work(MDRequestRef& mdr)
+void MDCache::flush_dentry_work(const MDRequestRef& mdr)
 {
   MutationImpl::LockOpVec lov;
   CInode *in = mds->server->rdlock_path_pin_ref(mdr, true);
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 5c1f34da4df0..94347b8249cc 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -415,16 +415,16 @@ class MDCache {
     return active_requests.count(rid);
   }
   MDRequestRef request_get(metareqid_t rid);
-  void request_pin_ref(MDRequestRef& r, CInode *ref, std::vector<CDentry*>& trace);
-  void request_finish(MDRequestRef& mdr);
-  void request_forward(MDRequestRef& mdr, mds_rank_t mds, int port=0);
-  void dispatch_request(MDRequestRef& mdr);
-  void request_drop_foreign_locks(MDRequestRef& mdr);
-  void request_drop_non_rdlocks(MDRequestRef& r);
-  void request_drop_locks(MDRequestRef& r);
-  void request_cleanup(MDRequestRef& r);
+  void request_pin_ref(const MDRequestRef& r, CInode *ref, std::vector<CDentry*>& trace);
+  void request_finish(const MDRequestRef& mdr);
+  void request_forward(const MDRequestRef& mdr, mds_rank_t mds, int port=0);
+  void dispatch_request(const MDRequestRef& mdr);
+  void request_drop_foreign_locks(const MDRequestRef& mdr);
+  void request_drop_non_rdlocks(const MDRequestRef& r);
+  void request_drop_locks(const MDRequestRef& r);
+  void request_cleanup(const MDRequestRef& r);
   
-  void request_kill(MDRequestRef& r);  // called when session closes
+  void request_kill(const MDRequestRef& r);  // called when session closes
 
   // journal/snap helpers
   CInode *pick_inode_snap(CInode *in, snapid_t follows);
@@ -500,7 +500,7 @@ class MDCache {
   void add_rollback(metareqid_t reqid, mds_rank_t leader) {
     resolve_need_rollback[reqid] = leader;
   }
-  void finish_rollback(metareqid_t reqid, MDRequestRef& mdr);
+  void finish_rollback(metareqid_t reqid, const MDRequestRef& mdr);
 
   // ambiguous imports
   void add_ambiguous_import(dirfrag_t base, const std::vector<dirfrag_t>& bounds);
@@ -845,17 +845,17 @@ class MDCache {
    * If it returns 2 the request has been forwarded, and again the requester
    * should unwind itself and back out.
    */
-  int path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
+  int path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
 		    const filepath& path, int flags,
 		    std::vector<CDentry*> *pdnvec, CInode **pin=nullptr);
 
-  int maybe_request_forward_to_auth(MDRequestRef& mdr, MDSContextFactory& cf,
+  int maybe_request_forward_to_auth(const MDRequestRef& mdr, MDSContextFactory& cf,
 				    MDSCacheObject *p);
 
   CInode *cache_traverse(const filepath& path);
 
   void open_remote_dirfrag(CInode *diri, frag_t fg, MDSContext *fin);
-  CInode *get_dentry_inode(CDentry *dn, MDRequestRef& mdr, bool projected=false);
+  CInode *get_dentry_inode(CDentry *dn, const MDRequestRef& mdr, bool projected=false);
 
   bool parallel_fetch(std::map<inodeno_t,filepath>& pathmap, std::set<inodeno_t>& missing);
   bool parallel_fetch_traverse_dir(inodeno_t ino, filepath& path, 
@@ -913,8 +913,8 @@ class MDCache {
   // -- namespace --
   void encode_remote_dentry_link(CDentry::linkage_t *dnl, bufferlist& bl);
   void decode_remote_dentry_link(CDir *dir, CDentry *dn, bufferlist::const_iterator& p);
-  void send_dentry_link(CDentry *dn, MDRequestRef& mdr);
-  void send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequestRef& mdr);
+  void send_dentry_link(CDentry *dn, const MDRequestRef& mdr);
+  void send_dentry_unlink(CDentry *dn, CDentry *straydn, const MDRequestRef& mdr);
 
   void wait_for_uncommitted_fragment(dirfrag_t dirfrag, MDSContext *c) {
     uncommitted_fragments.at(dirfrag).waiters.push_back(c);
@@ -1160,7 +1160,7 @@ class MDCache {
 
   int dump_cache(std::string_view fn, Formatter *f, double timeout);
 
-  void flush_dentry_work(MDRequestRef& mdr);
+  void flush_dentry_work(const MDRequestRef& mdr);
   /**
    * Resolve path to a dentry and pass it onto the ScrubStack.
    *
@@ -1169,10 +1169,10 @@ class MDCache {
    * this scrub (we won't block them on a whole scrub as it can take a very
    * long time)
    */
-  void enqueue_scrub_work(MDRequestRef& mdr);
-  void repair_inode_stats_work(MDRequestRef& mdr);
-  void repair_dirfrag_stats_work(MDRequestRef& mdr);
-  void rdlock_dirfrags_stats_work(MDRequestRef& mdr);
+  void enqueue_scrub_work(const MDRequestRef& mdr);
+  void repair_inode_stats_work(const MDRequestRef& mdr);
+  void repair_dirfrag_stats_work(const MDRequestRef& mdr);
+  void rdlock_dirfrags_stats_work(const MDRequestRef& mdr);
 
   ceph::unordered_map<inodeno_t,CInode*> inode_map;  // map of head inodes by ino
   std::map<vinodeno_t, CInode*> snap_inode_map;  // map of snap inodes by ino
@@ -1347,14 +1347,14 @@ class MDCache {
 
   bool can_fragment(CInode *diri, const std::vector<CDir*>& dirs);
   void fragment_freeze_dirs(const std::vector<CDir*>& dirs);
-  void fragment_mark_and_complete(MDRequestRef& mdr);
-  void fragment_frozen(MDRequestRef& mdr, int r);
+  void fragment_mark_and_complete(const MDRequestRef& mdr);
+  void fragment_frozen(const MDRequestRef& mdr, int r);
   void fragment_unmark_unfreeze_dirs(const std::vector<CDir*>& dirs);
   void fragment_drop_locks(fragment_info_t &info);
   void fragment_maybe_finish(const fragment_info_iterator& it);
-  void dispatch_fragment_dir(MDRequestRef& mdr);
-  void _fragment_logged(MDRequestRef& mdr);
-  void _fragment_stored(MDRequestRef& mdr);
+  void dispatch_fragment_dir(const MDRequestRef& mdr);
+  void _fragment_logged(const MDRequestRef& mdr);
+  void _fragment_stored(const MDRequestRef& mdr);
   void _fragment_committed(dirfrag_t f, const MDRequestRef& mdr);
   void _fragment_old_purged(dirfrag_t f, int bits, const MDRequestRef& mdr);
 
@@ -1413,14 +1413,14 @@ class C_MDS_RetryRequest : public MDSInternalContext {
   MDCache *cache;
   MDRequestRef mdr;
  public:
-  C_MDS_RetryRequest(MDCache *c, MDRequestRef& r) :
+  C_MDS_RetryRequest(MDCache *c, const MDRequestRef& r) :
     MDSInternalContext(c->mds), cache(c), mdr(r) {}
   void finish(int r) override;
 };
 
 class CF_MDS_RetryRequestFactory : public MDSContextFactory {
 public:
-  CF_MDS_RetryRequestFactory(MDCache *cache, MDRequestRef &mdr, bool dl) :
+  CF_MDS_RetryRequestFactory(MDCache *cache, const MDRequestRef& mdr, bool dl) :
     mdcache(cache), mdr(mdr), drop_locks(dl) {}
   MDSContext *build() override;
 private:
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 20ddf3eec588..7625ac4a9fdc 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1483,7 +1483,7 @@ void MDSRank::send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &
   messenger->send_to_mds(ref_t<Message>(m).detach(), addr);
 }
 
-void MDSRank::forward_message_mds(MDRequestRef& mdr, mds_rank_t mds)
+void MDSRank::forward_message_mds(const MDRequestRef& mdr, mds_rank_t mds)
 {
   ceph_assert(mds != whoami);
 
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 48043df4baf6..fd680863ca97 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -307,7 +307,7 @@ class MDSRank {
 
     void send_message_mds(const ref_t<Message>& m, mds_rank_t mds);
     void send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &addr);
-    void forward_message_mds(MDRequestRef& mdr, mds_rank_t mds);
+    void forward_message_mds(const MDRequestRef& mdr, mds_rank_t mds);
     void send_message_client_counted(const ref_t<Message>& m, client_t client);
     void send_message_client_counted(const ref_t<Message>& m, Session* session);
     void send_message_client_counted(const ref_t<Message>& m, const ConnectionRef& connection);
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 1dd4cb17737e..a3e9d31ded14 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -1003,7 +1003,7 @@ class C_M_ExportDirWait : public MigratorContext {
   }
 };
 
-void Migrator::dispatch_export_dir(MDRequestRef& mdr, int count)
+void Migrator::dispatch_export_dir(const MDRequestRef& mdr, int count)
 {
   CDir *dir = mdr->more()->export_dir;
   dout(7) << *mdr << " " << *dir << dendl;
diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index be24e3111000..53bf99fb639c 100644
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -185,7 +185,7 @@ class Migrator {
 
   // -- import/export --
   // exporter
-  void dispatch_export_dir(MDRequestRef& mdr, int count);
+  void dispatch_export_dir(const MDRequestRef& mdr, int count);
   void export_dir(CDir *dir, mds_rank_t dest);
   void export_empty_import(CDir *dir);
 
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 30861aace0d7..235688251eeb 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -160,7 +160,7 @@ class ServerLogContext : public MDSLogContextBase {
   explicit ServerLogContext(Server *s) : server(s) {
     ceph_assert(server != NULL);
   }
-  explicit ServerLogContext(Server *s, MDRequestRef& r) : server(s), mdr(r) {
+  explicit ServerLogContext(Server *s, const MDRequestRef& r) : server(s), mdr(r) {
     ceph_assert(server != NULL);
   }
 };
@@ -2023,7 +2023,7 @@ void Server::force_clients_readonly()
 /*******
  * some generic stuff for finishing off requests
  */
-void Server::journal_and_reply(MDRequestRef& mdr, CInode *in, CDentry *dn, LogEvent *le, MDSLogContextBase *fin)
+void Server::journal_and_reply(const MDRequestRef& mdr, CInode *in, CDentry *dn, LogEvent *le, MDSLogContextBase *fin)
 {
   dout(10) << "journal_and_reply tracei " << in << " tracedn " << dn << dendl;
   ceph_assert(!mdr->has_completed);
@@ -2057,7 +2057,7 @@ void Server::journal_and_reply(MDRequestRef& mdr, CInode *in, CDentry *dn, LogEv
     mdlog->flush();
 }
 
-void Server::submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin, MDRequestRef& mdr,
+void Server::submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin, const MDRequestRef& mdr,
                                 std::string_view event)
 {
   if (mdr) {
@@ -2071,7 +2071,7 @@ void Server::submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin, MDRequestR
 /*
  * send response built from mdr contents and error code; clean up mdr
  */
-void Server::respond_to_request(MDRequestRef& mdr, int r)
+void Server::respond_to_request(const MDRequestRef& mdr, int r)
 {
   if (mdr->client_request) {
     if (mdr->is_batch_head()) {
@@ -2191,7 +2191,7 @@ void Server::perf_gather_op_latency(const cref_t<MClientRequest> &req, utime_t l
   logger->tinc(code, lat);   
 }
 
-void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
+void Server::early_reply(const MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
 {
   if (!g_conf()->mds_early_reply)
     return;
@@ -2270,7 +2270,7 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
  * include a trace to tracei
  * Clean up mdr
  */
-void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &reply)
+void Server::reply_client_request(const MDRequestRef& mdr, const ref_t<MClientReply> &reply)
 {
   ceph_assert(mdr.get());
   const cref_t<MClientRequest> &req = mdr->client_request;
@@ -2384,7 +2384,7 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
  */
 void Server::set_trace_dist(const ref_t<MClientReply> &reply,
 			    CInode *in, CDentry *dn,
-			    MDRequestRef& mdr)
+			    const MDRequestRef& mdr)
 {
   // skip doing this for debugging purposes?
   if (g_conf()->mds_inject_traceless_reply_probability &&
@@ -2631,7 +2631,7 @@ void Server::handle_osd_map()
     });
 }
 
-void Server::dispatch_client_request(MDRequestRef& mdr)
+void Server::dispatch_client_request(const MDRequestRef& mdr)
 {
   // we shouldn't be waiting on anyone.
   ceph_assert(!mdr->has_more() || mdr->more()->waiting_on_peer.empty());
@@ -2648,7 +2648,8 @@ void Server::dispatch_client_request(MDRequestRef& mdr)
 	mdr->batch_op_map->erase(it);
 	return;
       }
-      mdr = std::move(new_batch_head);
+      mds->finisher->queue(new C_MDS_RetryRequest(mdcache, new_batch_head));
+      return;
     } else {
       return;
     }
@@ -3024,7 +3025,7 @@ void Server::handle_peer_request_reply(const cref_t<MMDSPeerRequest> &m)
   }
 }
 
-void Server::dispatch_peer_request(MDRequestRef& mdr)
+void Server::dispatch_peer_request(const MDRequestRef& mdr)
 {
   dout(7) << "dispatch_peer_request " << *mdr << " " << *mdr->peer_request << dendl;
 
@@ -3138,7 +3139,7 @@ void Server::dispatch_peer_request(MDRequestRef& mdr)
   }
 }
 
-void Server::handle_peer_auth_pin(MDRequestRef& mdr)
+void Server::handle_peer_auth_pin(const MDRequestRef& mdr)
 {
   dout(10) << "handle_peer_auth_pin " << *mdr << dendl;
 
@@ -3264,7 +3265,7 @@ void Server::handle_peer_auth_pin(MDRequestRef& mdr)
   return;
 }
 
-void Server::handle_peer_auth_pin_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
+void Server::handle_peer_auth_pin_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
 {
   dout(10) << "handle_peer_auth_pin_ack on " << *mdr << " " << *ack << dendl;
   mds_rank_t from = mds_rank_t(ack->get_source().num());
@@ -3341,7 +3342,7 @@ void Server::handle_peer_auth_pin_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRe
  * by mask on the given inode, based on the capability in the mdr's
  * session.
  */
-bool Server::check_access(MDRequestRef& mdr, CInode *in, unsigned mask)
+bool Server::check_access(const MDRequestRef& mdr, CInode *in, unsigned mask)
 {
   if (mdr->session) {
     int r = mdr->session->check_access(
@@ -3363,7 +3364,7 @@ bool Server::check_access(MDRequestRef& mdr, CInode *in, unsigned mask)
  * check whether fragment has reached maximum size
  *
  */
-bool Server::check_fragment_space(MDRequestRef &mdr, CDir *dir)
+bool Server::check_fragment_space(const MDRequestRef& mdr, CDir *dir)
 {
   const auto size = dir->get_frag_size();
   const auto max = bal_fragment_size_max;
@@ -3382,7 +3383,7 @@ bool Server::check_fragment_space(MDRequestRef &mdr, CDir *dir)
  * check whether entries in a dir reached maximum size
  *
  */
-bool Server::check_dir_max_entries(MDRequestRef &mdr, CDir *in)
+bool Server::check_dir_max_entries(const MDRequestRef& mdr, CDir *in)
 {
   const uint64_t size = in->inode->get_projected_inode()->dirstat.nfiles +
                    in->inode->get_projected_inode()->dirstat.nsubdirs;
@@ -3395,7 +3396,7 @@ bool Server::check_dir_max_entries(MDRequestRef &mdr, CDir *in)
 }
 
 
-CDentry* Server::prepare_stray_dentry(MDRequestRef& mdr, CInode *in)
+CDentry* Server::prepare_stray_dentry(const MDRequestRef& mdr, CInode *in)
 {
   string straydname;
   in->name_stray_dentry(straydname);
@@ -3437,7 +3438,7 @@ CDentry* Server::prepare_stray_dentry(MDRequestRef& mdr, CInode *in)
  *
  * create a new inode.  set c/m/atime.  hit dir pop.
  */
-CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
+CInode* Server::prepare_new_inode(const MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
 				  const file_layout_t *layout)
 {
   CInode *in = new CInode(mdcache);
@@ -3567,7 +3568,7 @@ CInode* Server::prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino
   return in;
 }
 
-void Server::journal_allocated_inos(MDRequestRef& mdr, EMetaBlob *blob)
+void Server::journal_allocated_inos(const MDRequestRef& mdr, EMetaBlob *blob)
 {
   dout(20) << "journal_allocated_inos sessionmapv " << mds->sessionmap.get_projected()
 	   << " inotablev " << mds->inotable->get_projected_version()
@@ -3580,7 +3581,7 @@ void Server::journal_allocated_inos(MDRequestRef& mdr, EMetaBlob *blob)
 		      mds->inotable->get_projected_version());
 }
 
-void Server::apply_allocated_inos(MDRequestRef& mdr, Session *session)
+void Server::apply_allocated_inos(const MDRequestRef& mdr, Session *session)
 {
   dout(10) << "apply_allocated_inos " << mdr->alloc_ino
 	   << " / " << mdr->prealloc_inos
@@ -3607,14 +3608,14 @@ void Server::apply_allocated_inos(MDRequestRef& mdr, Session *session)
 struct C_MDS_TryOpenInode : public ServerContext {
   MDRequestRef mdr;
   inodeno_t ino;
-  C_MDS_TryOpenInode(Server *s, MDRequestRef& r, inodeno_t i) :
+  C_MDS_TryOpenInode(Server *s, const MDRequestRef& r, inodeno_t i) :
     ServerContext(s), mdr(r), ino(i) {}
   void finish(int r) override {
     server->_try_open_ino(mdr, r, ino);
   }
 };
 
-void Server::_try_open_ino(MDRequestRef& mdr, int r, inodeno_t ino)
+void Server::_try_open_ino(const MDRequestRef& mdr, int r, inodeno_t ino)
 {
   dout(10) << "_try_open_ino " << mdr.get() << " ino " << ino << " r=" << r << dendl;
 
@@ -3639,7 +3640,7 @@ class C_MDS_TryFindInode : public ServerContext {
   MDCache *mdcache;
   inodeno_t ino;
 public:
-  C_MDS_TryFindInode(Server *s, MDRequestRef& r, MDCache *m, inodeno_t i) :
+  C_MDS_TryFindInode(Server *s, const MDRequestRef& r, MDCache *m, inodeno_t i) :
     ServerContext(s), mdr(r), mdcache(m), ino(i) {}
   void finish(int r) override {
     if (r == -CEPHFS_ESTALE) { // :( find_ino_peers failed
@@ -3671,7 +3672,7 @@ class C_MDS_TryFindInode : public ServerContext {
 
 /* If this returns null, the request has been handled
  * as appropriate: forwarded on, or the client's been replied to */
-CInode* Server::rdlock_path_pin_ref(MDRequestRef& mdr,
+CInode* Server::rdlock_path_pin_ref(const MDRequestRef& mdr,
 				    bool want_auth,
 				    bool no_want_auth)
 {
@@ -3749,7 +3750,7 @@ CInode* Server::rdlock_path_pin_ref(MDRequestRef& mdr,
  * to ensure tail dentry and target inode are both auth in one mds. the tail dentry
  * will not be xlocked too if authexist and the target inode exists.
  */
-CDentry* Server::rdlock_path_xlock_dentry(MDRequestRef& mdr,
+CDentry* Server::rdlock_path_xlock_dentry(const MDRequestRef& mdr,
 					  bool create, bool okexist, bool authexist,
 					  bool want_layout)
 {
@@ -3859,7 +3860,7 @@ CDentry* Server::rdlock_path_xlock_dentry(MDRequestRef& mdr,
  *    order.
  */
 std::pair<CDentry*, CDentry*>
-Server::rdlock_two_paths_xlock_destdn(MDRequestRef& mdr, bool xlock_srcdn)
+Server::rdlock_two_paths_xlock_destdn(const MDRequestRef& mdr, bool xlock_srcdn)
 {
 
   const filepath& refpath = mdr->get_filepath();
@@ -4013,7 +4014,7 @@ Server::rdlock_two_paths_xlock_destdn(MDRequestRef& mdr, bool xlock_srcdn)
  * @param mdr request
  * @returns the pointer, or NULL if it had to be delayed (but mdr is taken care of)
  */
-CDir* Server::try_open_auth_dirfrag(CInode *diri, frag_t fg, MDRequestRef& mdr)
+CDir* Server::try_open_auth_dirfrag(CInode *diri, frag_t fg, const MDRequestRef& mdr)
 {
   CDir *dir = diri->get_dirfrag(fg);
 
@@ -4054,7 +4055,7 @@ CDir* Server::try_open_auth_dirfrag(CInode *diri, frag_t fg, MDRequestRef& mdr)
 // ===============================================================================
 // STAT
 
-void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup)
+void Server::handle_client_getattr(const MDRequestRef& mdr, bool is_lookup)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -4193,7 +4194,7 @@ void Server::handle_client_getattr(MDRequestRef& mdr, bool is_lookup)
 
 struct C_MDS_LookupIno2 : public ServerContext {
   MDRequestRef mdr;
-  C_MDS_LookupIno2(Server *s, MDRequestRef& r) : ServerContext(s), mdr(r) {}
+  C_MDS_LookupIno2(Server *s, const MDRequestRef& r) : ServerContext(s), mdr(r) {}
   void finish(int r) override {
     server->_lookup_ino_2(mdr, r);
   }
@@ -4202,7 +4203,7 @@ struct C_MDS_LookupIno2 : public ServerContext {
 /*
  * filepath:  ino
  */
-void Server::handle_client_lookup_ino(MDRequestRef& mdr,
+void Server::handle_client_lookup_ino(const MDRequestRef& mdr,
 				      bool want_parent, bool want_dentry)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
@@ -4305,7 +4306,7 @@ void Server::handle_client_lookup_ino(MDRequestRef& mdr,
   }
 }
 
-void Server::_lookup_snap_ino(MDRequestRef& mdr)
+void Server::_lookup_snap_ino(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -4380,7 +4381,7 @@ void Server::_lookup_snap_ino(MDRequestRef& mdr)
   }
 }
 
-void Server::_lookup_ino_2(MDRequestRef& mdr, int r)
+void Server::_lookup_ino_2(const MDRequestRef& mdr, int r)
 {
   inodeno_t ino = mdr->client_request->get_filepath().get_ino();
   dout(10) << "_lookup_ino_2 " << mdr.get() << " ino " << ino << " r=" << r << dendl;
@@ -4403,7 +4404,7 @@ void Server::_lookup_ino_2(MDRequestRef& mdr, int r)
 
 
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_open(MDRequestRef& mdr)
+void Server::handle_client_open(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   dout(7) << "open on " << req->get_filepath() << dendl;
@@ -4597,7 +4598,7 @@ class C_MDS_openc_finish : public ServerLogContext {
   CDentry *dn;
   CInode *newi;
 public:
-  C_MDS_openc_finish(Server *s, MDRequestRef& r, CDentry *d, CInode *ni) :
+  C_MDS_openc_finish(Server *s, const MDRequestRef& r, CDentry *d, CInode *ni) :
     ServerLogContext(s, r), dn(d), newi(ni) {}
   void finish(int r) override {
     ceph_assert(r == 0);
@@ -4627,7 +4628,7 @@ class C_MDS_openc_finish : public ServerLogContext {
 };
 
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_openc(MDRequestRef& mdr)
+void Server::handle_client_openc(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
@@ -4802,7 +4803,7 @@ void Server::handle_client_openc(MDRequestRef& mdr)
 }
 
 
-void Server::_finalize_readdir(MDRequestRef& mdr,
+void Server::_finalize_readdir(const MDRequestRef& mdr,
                                CInode *diri,
                                CDir* dir,
                                bool start,
@@ -4844,7 +4845,7 @@ void Server::_finalize_readdir(MDRequestRef& mdr,
   respond_to_request(mdr, 0);
 }
 
-void Server::handle_client_readdir(MDRequestRef& mdr)
+void Server::handle_client_readdir(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   Session *session = mds->get_session(req);
@@ -5095,7 +5096,7 @@ class C_MDS_inode_update_finish : public ServerLogContext {
   CInode *in;
   bool truncating_smaller, changed_ranges, adjust_realm;
 public:
-  C_MDS_inode_update_finish(Server *s, MDRequestRef& r, CInode *i,
+  C_MDS_inode_update_finish(Server *s, const MDRequestRef& r, CInode *i,
 			    bool sm=false, bool cr=false, bool ar=false) :
     ServerLogContext(s, r), in(i),
     truncating_smaller(sm), changed_ranges(cr), adjust_realm(ar) { }
@@ -5129,7 +5130,7 @@ class C_MDS_inode_update_finish : public ServerLogContext {
   }
 };
 
-void Server::handle_client_file_setlock(MDRequestRef& mdr)
+void Server::handle_client_file_setlock(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
@@ -5232,7 +5233,7 @@ void Server::handle_client_file_setlock(MDRequestRef& mdr)
   dout(10) << " state after lock change: " << *lock_state << dendl;
 }
 
-void Server::handle_client_file_readlock(MDRequestRef& mdr)
+void Server::handle_client_file_readlock(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
@@ -5285,7 +5286,7 @@ void Server::handle_client_file_readlock(MDRequestRef& mdr)
   respond_to_request(mdr, 0);
 }
 
-void Server::handle_client_setattr(MDRequestRef& mdr)
+void Server::handle_client_setattr(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
@@ -5480,7 +5481,7 @@ void Server::handle_client_setattr(MDRequestRef& mdr)
 }
 
 /* Takes responsibility for mdr */
-void Server::do_open_truncate(MDRequestRef& mdr, int cmode)
+void Server::do_open_truncate(const MDRequestRef& mdr, int cmode)
 {
   CInode *in = mdr->in[0];
   client_t client = mdr->get_client();
@@ -5544,7 +5545,7 @@ void Server::do_open_truncate(MDRequestRef& mdr, int cmode)
 
 
 /* This function cleans up the passed mdr */
-void Server::handle_client_setlayout(MDRequestRef& mdr)
+void Server::handle_client_setlayout(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   CInode *cur = rdlock_path_pin_ref(mdr, true);
@@ -5632,7 +5633,7 @@ void Server::handle_client_setlayout(MDRequestRef& mdr)
   journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(this, mdr, cur));
 }
 
-bool Server::xlock_policylock(MDRequestRef& mdr, CInode *in, bool want_layout, bool xlock_snaplock)
+bool Server::xlock_policylock(const MDRequestRef& mdr, CInode *in, bool want_layout, bool xlock_snaplock)
 {
   if (mdr->locking_state & MutationImpl::ALL_LOCKED)
     return true;
@@ -5659,7 +5660,7 @@ bool Server::xlock_policylock(MDRequestRef& mdr, CInode *in, bool want_layout, b
   return true;
 }
 
-CInode* Server::try_get_auth_inode(MDRequestRef& mdr, inodeno_t ino)
+CInode* Server::try_get_auth_inode(const MDRequestRef& mdr, inodeno_t ino)
 {
   CInode *in = mdcache->get_inode(ino);
   if (!in || in->state_test(CInode::STATE_PURGING)) {
@@ -5674,7 +5675,7 @@ CInode* Server::try_get_auth_inode(MDRequestRef& mdr, inodeno_t ino)
   return in;
 }
 
-void Server::handle_client_setdirlayout(MDRequestRef& mdr)
+void Server::handle_client_setdirlayout(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -5994,7 +5995,7 @@ void Server::create_quota_realm(CInode *in)
  * Return 0 on success, otherwise this function takes
  * responsibility for the passed mdr.
  */
-int Server::check_layout_vxattr(MDRequestRef& mdr,
+int Server::check_layout_vxattr(const MDRequestRef& mdr,
                                 string name,
                                 string value,
                                 file_layout_t *layout)
@@ -6046,7 +6047,7 @@ int Server::check_layout_vxattr(MDRequestRef& mdr,
   return 0;
 }
 
-void Server::handle_set_vxattr(MDRequestRef& mdr, CInode *cur)
+void Server::handle_set_vxattr(const MDRequestRef& mdr, CInode *cur)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
@@ -6349,7 +6350,7 @@ void Server::handle_set_vxattr(MDRequestRef& mdr, CInode *cur)
   return;
 }
 
-void Server::handle_remove_vxattr(MDRequestRef& mdr, CInode *cur)
+void Server::handle_remove_vxattr(const MDRequestRef& mdr, CInode *cur)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   string name(req->get_path2());
@@ -6582,7 +6583,7 @@ void Server::mirror_info_removexattr_handler(CInode *cur, InodeStoreBase::xattr_
   xattr_rm(xattrs, Server::MirrorXattrInfo::FS_ID);
 }
 
-void Server::handle_client_setxattr(MDRequestRef& mdr)
+void Server::handle_client_setxattr(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   string name(req->get_path2());
@@ -6679,7 +6680,7 @@ void Server::handle_client_setxattr(MDRequestRef& mdr)
   journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(this, mdr, cur));
 }
 
-void Server::handle_client_removexattr(MDRequestRef& mdr)
+void Server::handle_client_removexattr(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   std::string name(req->get_path2());
@@ -6748,7 +6749,7 @@ void Server::handle_client_removexattr(MDRequestRef& mdr)
   journal_and_reply(mdr, cur, 0, le, new C_MDS_inode_update_finish(this, mdr, cur));
 }
 
-void Server::handle_client_getvxattr(MDRequestRef& mdr)
+void Server::handle_client_getvxattr(const MDRequestRef& mdr)
 {
   const auto& req = mdr->client_request;
   string xattr_name{req->get_path2()};
@@ -6914,7 +6915,7 @@ class C_MDS_mknod_finish : public ServerLogContext {
   CDentry *dn;
   CInode *newi;
 public:
-  C_MDS_mknod_finish(Server *s, MDRequestRef& r, CDentry *d, CInode *ni) :
+  C_MDS_mknod_finish(Server *s, const MDRequestRef& r, CDentry *d, CInode *ni) :
     ServerLogContext(s, r), dn(d), newi(ni) {}
   void finish(int r) override {
     ceph_assert(r == 0);
@@ -6960,7 +6961,7 @@ class C_MDS_mknod_finish : public ServerLogContext {
 };
 
 
-void Server::handle_client_mknod(MDRequestRef& mdr)
+void Server::handle_client_mknod(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
@@ -7062,7 +7063,7 @@ void Server::handle_client_mknod(MDRequestRef& mdr)
 
 // MKDIR
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_mkdir(MDRequestRef& mdr)
+void Server::handle_client_mkdir(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -7156,7 +7157,7 @@ void Server::handle_client_mkdir(MDRequestRef& mdr)
 
 // SYMLINK
 
-void Server::handle_client_symlink(MDRequestRef& mdr)
+void Server::handle_client_symlink(const MDRequestRef& mdr)
 {
   const auto& req = mdr->client_request;
 
@@ -7223,7 +7224,7 @@ void Server::handle_client_symlink(MDRequestRef& mdr)
 
 // LINK
 
-void Server::handle_client_link(MDRequestRef& mdr)
+void Server::handle_client_link(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -7355,7 +7356,7 @@ class C_MDS_link_local_finish : public ServerLogContext {
   version_t tipv;
   bool adjust_realm;
 public:
-  C_MDS_link_local_finish(Server *s, MDRequestRef& r, CDentry *d, CInode *ti,
+  C_MDS_link_local_finish(Server *s, const MDRequestRef& r, CDentry *d, CInode *ti,
 			  version_t dnpv_, version_t tipv_, bool ar) :
     ServerLogContext(s, r), dn(d), targeti(ti),
     dnpv(dnpv_), tipv(tipv_), adjust_realm(ar) { }
@@ -7366,7 +7367,7 @@ class C_MDS_link_local_finish : public ServerLogContext {
 };
 
 
-void Server::_link_local(MDRequestRef& mdr, CDentry *dn, CInode *targeti, SnapRealm *target_realm)
+void Server::_link_local(const MDRequestRef& mdr, CDentry *dn, CInode *targeti, SnapRealm *target_realm)
 {
   dout(10) << "_link_local " << *dn << " to " << *targeti << dendl;
 
@@ -7408,7 +7409,7 @@ void Server::_link_local(MDRequestRef& mdr, CDentry *dn, CInode *targeti, SnapRe
 		    new C_MDS_link_local_finish(this, mdr, dn, targeti, dnpv, tipv, adjust_realm));
 }
 
-void Server::_link_local_finish(MDRequestRef& mdr, CDentry *dn, CInode *targeti,
+void Server::_link_local_finish(const MDRequestRef& mdr, CDentry *dn, CInode *targeti,
 				version_t dnpv, version_t tipv, bool adjust_realm)
 {
   dout(10) << "_link_local_finish " << *dn << " to " << *targeti << dendl;
@@ -7448,7 +7449,7 @@ class C_MDS_link_remote_finish : public ServerLogContext {
   CInode *targeti;
   version_t dpv;
 public:
-  C_MDS_link_remote_finish(Server *s, MDRequestRef& r, bool i, CDentry *d, CInode *ti) :
+  C_MDS_link_remote_finish(Server *s, const MDRequestRef& r, bool i, CDentry *d, CInode *ti) :
     ServerLogContext(s, r), inc(i), dn(d), targeti(ti),
     dpv(d->get_projected_version()) {}
   void finish(int r) override {
@@ -7457,7 +7458,7 @@ class C_MDS_link_remote_finish : public ServerLogContext {
   }
 };
 
-void Server::_link_remote(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti)
+void Server::_link_remote(const MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti)
 {
   dout(10) << "_link_remote " 
 	   << (inc ? "link ":"unlink ")
@@ -7530,7 +7531,7 @@ void Server::_link_remote(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targ
 		    new C_MDS_link_remote_finish(this, mdr, inc, dn, targeti));
 }
 
-void Server::_link_remote_finish(MDRequestRef& mdr, bool inc,
+void Server::_link_remote_finish(const MDRequestRef& mdr, bool inc,
 				 CDentry *dn, CInode *targeti,
 				 version_t dpv)
 {
@@ -7583,7 +7584,7 @@ class C_MDS_PeerLinkPrep : public ServerLogContext {
   CInode *targeti;
   bool adjust_realm;
 public:
-  C_MDS_PeerLinkPrep(Server *s, MDRequestRef& r, CInode *t, bool ar) :
+  C_MDS_PeerLinkPrep(Server *s, const MDRequestRef& r, CInode *t, bool ar) :
     ServerLogContext(s, r), targeti(t), adjust_realm(ar) { }
   void finish(int r) override {
     ceph_assert(r == 0);
@@ -7595,14 +7596,14 @@ class C_MDS_PeerLinkCommit : public ServerContext {
   MDRequestRef mdr;
   CInode *targeti;
 public:
-  C_MDS_PeerLinkCommit(Server *s, MDRequestRef& r, CInode *t) :
+  C_MDS_PeerLinkCommit(Server *s, const MDRequestRef& r, CInode *t) :
     ServerContext(s), mdr(r), targeti(t) { }
   void finish(int r) override {
     server->_commit_peer_link(mdr, r, targeti);
   }
 };
 
-void Server::handle_peer_link_prep(MDRequestRef& mdr)
+void Server::handle_peer_link_prep(const MDRequestRef& mdr)
 {
   dout(10) << "handle_peer_link_prep " << *mdr
 	   << " on " << mdr->peer_request->get_object_info()
@@ -7704,7 +7705,7 @@ void Server::handle_peer_link_prep(MDRequestRef& mdr)
   mdlog->flush();
 }
 
-void Server::_logged_peer_link(MDRequestRef& mdr, CInode *targeti, bool adjust_realm)
+void Server::_logged_peer_link(const MDRequestRef& mdr, CInode *targeti, bool adjust_realm)
 {
   dout(10) << "_logged_peer_link " << *mdr
 	   << " " << *targeti << dendl;
@@ -7738,13 +7739,13 @@ void Server::_logged_peer_link(MDRequestRef& mdr, CInode *targeti, bool adjust_r
 
 
 struct C_MDS_CommittedPeer : public ServerLogContext {
-  C_MDS_CommittedPeer(Server *s, MDRequestRef& m) : ServerLogContext(s, m) {}
+  C_MDS_CommittedPeer(Server *s, const MDRequestRef& m) : ServerLogContext(s, m) {}
   void finish(int r) override {
     server->_committed_peer(mdr);
   }
 };
 
-void Server::_commit_peer_link(MDRequestRef& mdr, int r, CInode *targeti)
+void Server::_commit_peer_link(const MDRequestRef& mdr, int r, CInode *targeti)
 {  
   dout(10) << "_commit_peer_link " << *mdr
 	   << " r=" << r
@@ -7766,7 +7767,7 @@ void Server::_commit_peer_link(MDRequestRef& mdr, int r, CInode *targeti)
   }
 }
 
-void Server::_committed_peer(MDRequestRef& mdr)
+void Server::_committed_peer(const MDRequestRef& mdr)
 {
   dout(10) << "_committed_peer " << *mdr << dendl;
 
@@ -7782,7 +7783,7 @@ void Server::_committed_peer(MDRequestRef& mdr)
 struct C_MDS_LoggedLinkRollback : public ServerLogContext {
   MutationRef mut;
   map<client_t,ref_t<MClientSnap>> splits;
-  C_MDS_LoggedLinkRollback(Server *s, MutationRef& m, MDRequestRef& r,
+  C_MDS_LoggedLinkRollback(Server *s, MutationRef& m, const MDRequestRef& r,
 			   map<client_t,ref_t<MClientSnap>>&& _splits) :
     ServerLogContext(s, r), mut(m), splits(std::move(_splits)) {
   }
@@ -7791,7 +7792,7 @@ struct C_MDS_LoggedLinkRollback : public ServerLogContext {
   }
 };
 
-void Server::do_link_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr)
+void Server::do_link_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr)
 {
   link_rollback rollback;
   auto p = rbl.cbegin();
@@ -7870,7 +7871,7 @@ void Server::do_link_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef&
   mdlog->flush();
 }
 
-void Server::_link_rollback_finish(MutationRef& mut, MDRequestRef& mdr,
+void Server::_link_rollback_finish(MutationRef& mut, const MDRequestRef& mdr,
 				   map<client_t,ref_t<MClientSnap>>& splits)
 {
   dout(10) << "_link_rollback_finish" << dendl;
@@ -7891,7 +7892,7 @@ void Server::_link_rollback_finish(MutationRef& mut, MDRequestRef& mdr,
 }
 
 
-void Server::handle_peer_link_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m)
+void Server::handle_peer_link_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m)
 {
   dout(10) << "handle_peer_link_prep_ack " << *mdr
 	   << " " << *m << dendl;
@@ -7923,7 +7924,7 @@ void Server::handle_peer_link_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerR
 
 // UNLINK
 
-void Server::handle_client_unlink(MDRequestRef& mdr)
+void Server::handle_client_unlink(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
@@ -8080,7 +8081,7 @@ class C_MDS_unlink_local_finish : public ServerLogContext {
   CDentry *straydn;
   version_t dnpv;  // deleted dentry
 public:
-  C_MDS_unlink_local_finish(Server *s, MDRequestRef& r, CDentry *d, CDentry *sd) :
+  C_MDS_unlink_local_finish(Server *s, const MDRequestRef& r, CDentry *d, CDentry *sd) :
     ServerLogContext(s, r), dn(d), straydn(sd),
     dnpv(d->get_projected_version()) {}
   void finish(int r) override {
@@ -8089,7 +8090,7 @@ class C_MDS_unlink_local_finish : public ServerLogContext {
   }
 };
 
-void Server::_unlink_local(MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
+void Server::_unlink_local(const MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
 {
   dout(10) << "_unlink_local " << *dn << dendl;
 
@@ -8178,7 +8179,7 @@ void Server::_unlink_local(MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
   journal_and_reply(mdr, 0, dn, le, new C_MDS_unlink_local_finish(this, mdr, dn, straydn));
 }
 
-void Server::_unlink_local_finish(MDRequestRef& mdr,
+void Server::_unlink_local_finish(const MDRequestRef& mdr,
 				  CDentry *dn, CDentry *straydn,
 				  version_t dnpv) 
 {
@@ -8240,7 +8241,7 @@ void Server::_unlink_local_finish(MDRequestRef& mdr,
   }
 }
 
-bool Server::_rmdir_prepare_witness(MDRequestRef& mdr, mds_rank_t who, vector<CDentry*>& trace, CDentry *straydn)
+bool Server::_rmdir_prepare_witness(const MDRequestRef& mdr, mds_rank_t who, vector<CDentry*>& trace, CDentry *straydn)
 {
   if (mds->is_cluster_degraded() &&
       !mds->mdsmap->is_clientreplay_or_active_or_stopping(who)) {
@@ -8269,7 +8270,7 @@ bool Server::_rmdir_prepare_witness(MDRequestRef& mdr, mds_rank_t who, vector<CD
 
 struct C_MDS_PeerRmdirPrep : public ServerLogContext {
   CDentry *dn, *straydn;
-  C_MDS_PeerRmdirPrep(Server *s, MDRequestRef& r, CDentry *d, CDentry *st)
+  C_MDS_PeerRmdirPrep(Server *s, const MDRequestRef& r, CDentry *d, CDentry *st)
     : ServerLogContext(s, r), dn(d), straydn(st) {}
   void finish(int r) override {
     server->_logged_peer_rmdir(mdr, dn, straydn);
@@ -8279,14 +8280,14 @@ struct C_MDS_PeerRmdirPrep : public ServerLogContext {
 struct C_MDS_PeerRmdirCommit : public ServerContext {
   MDRequestRef mdr;
   CDentry *straydn;
-  C_MDS_PeerRmdirCommit(Server *s, MDRequestRef& r, CDentry *sd)
+  C_MDS_PeerRmdirCommit(Server *s, const MDRequestRef& r, CDentry *sd)
     : ServerContext(s), mdr(r), straydn(sd) { }
   void finish(int r) override {
     server->_commit_peer_rmdir(mdr, r, straydn);
   }
 };
 
-void Server::handle_peer_rmdir_prep(MDRequestRef& mdr)
+void Server::handle_peer_rmdir_prep(const MDRequestRef& mdr)
 {
   dout(10) << "handle_peer_rmdir_prep " << *mdr
 	   << " " << mdr->peer_request->srcdnpath
@@ -8372,7 +8373,7 @@ void Server::handle_peer_rmdir_prep(MDRequestRef& mdr)
   mdlog->flush();
 }
 
-void Server::_logged_peer_rmdir(MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
+void Server::_logged_peer_rmdir(const MDRequestRef& mdr, CDentry *dn, CDentry *straydn)
 {
   dout(10) << "_logged_peer_rmdir " << *mdr << " on " << *dn << dendl;
   CInode *in = dn->get_linkage()->get_inode();
@@ -8412,7 +8413,7 @@ void Server::_logged_peer_rmdir(MDRequestRef& mdr, CDentry *dn, CDentry *straydn
   }
 }
 
-void Server::handle_peer_rmdir_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
+void Server::handle_peer_rmdir_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
 {
   dout(10) << "handle_peer_rmdir_prep_ack " << *mdr
 	   << " " << *ack << dendl;
@@ -8434,7 +8435,7 @@ void Server::handle_peer_rmdir_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeer
     dout(10) << "still waiting on peers " << mdr->more()->waiting_on_peer << dendl;
 }
 
-void Server::_commit_peer_rmdir(MDRequestRef& mdr, int r, CDentry *straydn)
+void Server::_commit_peer_rmdir(const MDRequestRef& mdr, int r, CDentry *straydn)
 {
   dout(10) << "_commit_peer_rmdir " << *mdr << " r=" << r << dendl;
 
@@ -8467,14 +8468,14 @@ struct C_MDS_LoggedRmdirRollback : public ServerLogContext {
   metareqid_t reqid;
   CDentry *dn;
   CDentry *straydn;
-  C_MDS_LoggedRmdirRollback(Server *s, MDRequestRef& m, metareqid_t mr, CDentry *d, CDentry *st)
+  C_MDS_LoggedRmdirRollback(Server *s, const MDRequestRef& m, metareqid_t mr, CDentry *d, CDentry *st)
     : ServerLogContext(s, m), reqid(mr), dn(d), straydn(st) {}
   void finish(int r) override {
     server->_rmdir_rollback_finish(mdr, reqid, dn, straydn);
   }
 };
 
-void Server::do_rmdir_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr)
+void Server::do_rmdir_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr)
 {
   // unlink the other rollback methods, the rmdir rollback is only
   // needed to record the subtree changes in the journal for inode
@@ -8544,7 +8545,7 @@ void Server::do_rmdir_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef&
   mdlog->flush();
 }
 
-void Server::_rmdir_rollback_finish(MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn)
+void Server::_rmdir_rollback_finish(const MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn)
 {
   dout(10) << "_rmdir_rollback_finish " << reqid << dendl;
 
@@ -8575,7 +8576,7 @@ void Server::_rmdir_rollback_finish(MDRequestRef& mdr, metareqid_t reqid, CDentr
  * the unlocked varient this is a fastpath check.  we can't really be
  * sure until we rdlock the filelock.
  */
-bool Server::_dir_is_nonempty_unlocked(MDRequestRef& mdr, CInode *in)
+bool Server::_dir_is_nonempty_unlocked(const MDRequestRef& mdr, CInode *in)
 {
   dout(10) << "dir_is_nonempty_unlocked " << *in << dendl;
   ceph_assert(in->is_auth());
@@ -8600,7 +8601,7 @@ bool Server::_dir_is_nonempty_unlocked(MDRequestRef& mdr, CInode *in)
   return false;
 }
 
-bool Server::_dir_is_nonempty(MDRequestRef& mdr, CInode *in)
+bool Server::_dir_is_nonempty(const MDRequestRef& mdr, CInode *in)
 {
   dout(10) << "dir_is_nonempty " << *in << dendl;
   ceph_assert(in->is_auth());
@@ -8636,7 +8637,7 @@ class C_MDS_rename_finish : public ServerLogContext {
   CDentry *destdn;
   CDentry *straydn;
 public:
-  C_MDS_rename_finish(Server *s, MDRequestRef& r,
+  C_MDS_rename_finish(Server *s, const MDRequestRef& r,
 		      CDentry *sdn, CDentry *ddn, CDentry *stdn) :
     ServerLogContext(s, r),
     srcdn(sdn), destdn(ddn), straydn(stdn) { }
@@ -8661,7 +8662,7 @@ class C_MDS_rename_finish : public ServerLogContext {
  *
  * This function takes responsibility for the passed mdr.
  */
-void Server::handle_client_rename(MDRequestRef& mdr)
+void Server::handle_client_rename(const MDRequestRef& mdr)
 {
   const auto& req = mdr->client_request;
   dout(7) << "handle_client_rename " << *req << dendl;
@@ -9102,7 +9103,7 @@ void Server::handle_client_rename(MDRequestRef& mdr)
 }
 
 
-void Server::_rename_finish(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+void Server::_rename_finish(const MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
 {
   dout(10) << "_rename_finish " << *mdr << dendl;
 
@@ -9150,7 +9151,7 @@ void Server::_rename_finish(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn,
 
 // helpers
 
-bool Server::_rename_prepare_witness(MDRequestRef& mdr, mds_rank_t who, set<mds_rank_t> &witnesse,
+bool Server::_rename_prepare_witness(const MDRequestRef& mdr, mds_rank_t who, set<mds_rank_t> &witnesse,
 				     vector<CDentry*>& srctrace, vector<CDentry*>& dsttrace, CDentry *straydn)
 {
   const auto& client_req = mdr->client_request;
@@ -9195,7 +9196,7 @@ bool Server::_rename_prepare_witness(MDRequestRef& mdr, mds_rank_t who, set<mds_
   return true;
 }
 
-version_t Server::_rename_prepare_import(MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl)
+version_t Server::_rename_prepare_import(const MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl)
 {
   version_t oldpv = mdr->more()->inode_import_v;
 
@@ -9264,7 +9265,7 @@ bool Server::_need_force_journal(CInode *diri, bool empty)
   return force_journal;
 }
 
-void Server::_rename_prepare(MDRequestRef& mdr,
+void Server::_rename_prepare(const MDRequestRef& mdr,
 			     EMetaBlob *metablob, bufferlist *client_map_bl,
 			     CDentry *srcdn, CDentry *destdn, std::string_view alternate_name,
                              CDentry *straydn)
@@ -9618,7 +9619,7 @@ void Server::_rename_prepare(MDRequestRef& mdr,
 }
 
 
-void Server::_rename_apply(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
+void Server::_rename_apply(const MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn)
 {
   dout(10) << "_rename_apply " << *mdr << " " << *srcdn << " " << *destdn << dendl;
   dout(10) << " pvs " << mdr->more()->pvmap << dendl;
@@ -9822,7 +9823,7 @@ void Server::_rename_apply(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, C
 class C_MDS_PeerRenamePrep : public ServerLogContext {
   CDentry *srcdn, *destdn, *straydn;
 public:
-  C_MDS_PeerRenamePrep(Server *s, MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
+  C_MDS_PeerRenamePrep(Server *s, const MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
     ServerLogContext(s, m), srcdn(sr), destdn(de), straydn(st) {}
   void finish(int r) override {
     server->_logged_peer_rename(mdr, srcdn, destdn, straydn);
@@ -9833,7 +9834,7 @@ class C_MDS_PeerRenameCommit : public ServerContext {
   MDRequestRef mdr;
   CDentry *srcdn, *destdn, *straydn;
 public:
-  C_MDS_PeerRenameCommit(Server *s, MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
+  C_MDS_PeerRenameCommit(Server *s, const MDRequestRef& m, CDentry *sr, CDentry *de, CDentry *st) :
     ServerContext(s), mdr(m), srcdn(sr), destdn(de), straydn(st) {}
   void finish(int r) override {
     server->_commit_peer_rename(mdr, r, srcdn, destdn, straydn);
@@ -9843,14 +9844,14 @@ class C_MDS_PeerRenameCommit : public ServerContext {
 class C_MDS_PeerRenameSessionsFlushed : public ServerContext {
   MDRequestRef mdr;
 public:
-  C_MDS_PeerRenameSessionsFlushed(Server *s, MDRequestRef& r) :
+  C_MDS_PeerRenameSessionsFlushed(Server *s, const MDRequestRef& r) :
     ServerContext(s), mdr(r) {}
   void finish(int r) override {
     server->_peer_rename_sessions_flushed(mdr);
   }
 };
 
-void Server::handle_peer_rename_prep(MDRequestRef& mdr)
+void Server::handle_peer_rename_prep(const MDRequestRef& mdr)
 {
   dout(10) << "handle_peer_rename_prep " << *mdr
 	   << " " << mdr->peer_request->srcdnpath
@@ -10088,7 +10089,7 @@ void Server::handle_peer_rename_prep(MDRequestRef& mdr)
   }
 }
 
-void Server::_logged_peer_rename(MDRequestRef& mdr,
+void Server::_logged_peer_rename(const MDRequestRef& mdr,
 				  CDentry *srcdn, CDentry *destdn, CDentry *straydn)
 {
   dout(10) << "_logged_peer_rename " << *mdr << dendl;
@@ -10166,7 +10167,7 @@ void Server::_logged_peer_rename(MDRequestRef& mdr,
   }
 }
 
-void Server::_commit_peer_rename(MDRequestRef& mdr, int r,
+void Server::_commit_peer_rename(const MDRequestRef& mdr, int r,
 				  CDentry *srcdn, CDentry *destdn, CDentry *straydn)
 {
   dout(10) << "_commit_peer_rename " << *mdr << " r=" << r << dendl;
@@ -10306,7 +10307,7 @@ struct C_MDS_LoggedRenameRollback : public ServerLogContext {
   CDentry *straydn;
   map<client_t,ref_t<MClientSnap>> splits[2];
   bool finish_mdr;
-  C_MDS_LoggedRenameRollback(Server *s, MutationRef& m, MDRequestRef& r,
+  C_MDS_LoggedRenameRollback(Server *s, MutationRef& m, const MDRequestRef& r,
 			     CDentry *sd, version_t pv, CDentry *dd, CDentry *st,
 			     map<client_t,ref_t<MClientSnap>> _splits[2], bool f) :
     ServerLogContext(s, r), mut(m), srcdn(sd), srcdnpv(pv), destdn(dd),
@@ -10320,7 +10321,7 @@ struct C_MDS_LoggedRenameRollback : public ServerLogContext {
   }
 };
 
-void Server::do_rename_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr,
+void Server::do_rename_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr,
 				bool finish_mdr)
 {
   rename_rollback rollback;
@@ -10645,7 +10646,7 @@ void Server::do_rename_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef
   }
 }
 
-void Server::_rename_rollback_finish(MutationRef& mut, MDRequestRef& mdr, CDentry *srcdn,
+void Server::_rename_rollback_finish(MutationRef& mut, const MDRequestRef& mdr, CDentry *srcdn,
 				     version_t srcdnpv, CDentry *destdn, CDentry *straydn,
 				     map<client_t,ref_t<MClientSnap>> splits[2], bool finish_mdr)
 {
@@ -10721,7 +10722,7 @@ void Server::_rename_rollback_finish(MutationRef& mut, MDRequestRef& mdr, CDentr
   mut->cleanup();
 }
 
-void Server::handle_peer_rename_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
+void Server::handle_peer_rename_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
 {
   dout(10) << "handle_peer_rename_prep_ack " << *mdr
 	   << " witnessed by " << ack->get_source()
@@ -10767,7 +10768,7 @@ void Server::handle_peer_rename_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPee
     dout(10) << "still waiting on peers " << mdr->more()->waiting_on_peer << dendl;
 }
 
-void Server::handle_peer_rename_notify_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
+void Server::handle_peer_rename_notify_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack)
 {
   dout(10) << "handle_peer_rename_notify_ack " << *mdr << " from mds."
 	   << ack->get_source() << dendl;
@@ -10786,7 +10787,7 @@ void Server::handle_peer_rename_notify_ack(MDRequestRef& mdr, const cref_t<MMDSP
   }
 }
 
-void Server::_peer_rename_sessions_flushed(MDRequestRef& mdr)
+void Server::_peer_rename_sessions_flushed(const MDRequestRef& mdr)
 {
   dout(10) << "_peer_rename_sessions_flushed " << *mdr << dendl;
 
@@ -10804,7 +10805,7 @@ void Server::_peer_rename_sessions_flushed(MDRequestRef& mdr)
 
 // snaps
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_lssnap(MDRequestRef& mdr)
+void Server::handle_client_lssnap(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -10906,7 +10907,7 @@ void Server::handle_client_lssnap(MDRequestRef& mdr)
 struct C_MDS_mksnap_finish : public ServerLogContext {
   CInode *diri;
   SnapInfo info;
-  C_MDS_mksnap_finish(Server *s, MDRequestRef& r, CInode *di, SnapInfo &i) :
+  C_MDS_mksnap_finish(Server *s, const MDRequestRef& r, CInode *di, SnapInfo &i) :
     ServerLogContext(s, r), diri(di), info(i) {}
   void finish(int r) override {
     server->_mksnap_finish(mdr, diri, info);
@@ -10914,7 +10915,7 @@ struct C_MDS_mksnap_finish : public ServerLogContext {
 };
 
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_mksnap(MDRequestRef& mdr)
+void Server::handle_client_mksnap(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   // make sure we have as new a map as the client
@@ -11072,7 +11073,7 @@ void Server::handle_client_mksnap(MDRequestRef& mdr)
   mdlog->flush();
 }
 
-void Server::_mksnap_finish(MDRequestRef& mdr, CInode *diri, SnapInfo &info)
+void Server::_mksnap_finish(const MDRequestRef& mdr, CInode *diri, SnapInfo &info)
 {
   dout(10) << "_mksnap_finish " << *mdr << " " << info << dendl;
 
@@ -11103,7 +11104,7 @@ void Server::_mksnap_finish(MDRequestRef& mdr, CInode *diri, SnapInfo &info)
 struct C_MDS_rmsnap_finish : public ServerLogContext {
   CInode *diri;
   snapid_t snapid;
-  C_MDS_rmsnap_finish(Server *s, MDRequestRef& r, CInode *di, snapid_t sn) :
+  C_MDS_rmsnap_finish(Server *s, const MDRequestRef& r, CInode *di, snapid_t sn) :
     ServerLogContext(s, r), diri(di), snapid(sn) {}
   void finish(int r) override {
     server->_rmsnap_finish(mdr, diri, snapid);
@@ -11111,7 +11112,7 @@ struct C_MDS_rmsnap_finish : public ServerLogContext {
 };
 
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_rmsnap(MDRequestRef& mdr)
+void Server::handle_client_rmsnap(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
 
@@ -11204,7 +11205,7 @@ void Server::handle_client_rmsnap(MDRequestRef& mdr)
   mdlog->flush();
 }
 
-void Server::_rmsnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid)
+void Server::_rmsnap_finish(const MDRequestRef& mdr, CInode *diri, snapid_t snapid)
 {
   dout(10) << "_rmsnap_finish " << *mdr << " " << snapid << dendl;
   snapid_t stid = mdr->more()->stid;
@@ -11233,7 +11234,7 @@ void Server::_rmsnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid)
 struct C_MDS_renamesnap_finish : public ServerLogContext {
   CInode *diri;
   snapid_t snapid;
-  C_MDS_renamesnap_finish(Server *s, MDRequestRef& r, CInode *di, snapid_t sn) :
+  C_MDS_renamesnap_finish(Server *s, const MDRequestRef& r, CInode *di, snapid_t sn) :
     ServerLogContext(s, r), diri(di), snapid(sn) {}
   void finish(int r) override {
     server->_renamesnap_finish(mdr, diri, snapid);
@@ -11241,7 +11242,7 @@ struct C_MDS_renamesnap_finish : public ServerLogContext {
 };
 
 /* This function takes responsibility for the passed mdr*/
-void Server::handle_client_renamesnap(MDRequestRef& mdr)
+void Server::handle_client_renamesnap(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest> &req = mdr->client_request;
   if (req->get_filepath().get_ino() != req->get_filepath2().get_ino()) {
@@ -11348,7 +11349,7 @@ void Server::handle_client_renamesnap(MDRequestRef& mdr)
   mdlog->flush();
 }
 
-void Server::_renamesnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid)
+void Server::_renamesnap_finish(const MDRequestRef& mdr, CInode *diri, snapid_t snapid)
 {
   dout(10) << "_renamesnap_finish " << *mdr << " " << snapid << dendl;
 
@@ -11370,7 +11371,7 @@ void Server::_renamesnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid
   respond_to_request(mdr, 0);
 }
 
-void Server::handle_client_readdir_snapdiff(MDRequestRef& mdr)
+void Server::handle_client_readdir_snapdiff(const MDRequestRef& mdr)
 {
   const cref_t<MClientRequest>& req = mdr->client_request;
   Session* session = mds->get_session(req);
@@ -11557,7 +11558,7 @@ const bufferlist& Server::get_snap_trace(client_t client, SnapRealm *realm) cons
 
 void Server::_readdir_diff(
   utime_t now,
-  MDRequestRef& mdr,
+  const MDRequestRef& mdr,
   CInode* diri,
   CDir* dir,
   SnapRealm* realm,
@@ -11642,7 +11643,7 @@ void Server::_readdir_diff(
 }
 
 bool Server::build_snap_diff(
-  MDRequestRef& mdr,
+  const MDRequestRef& mdr,
   CDir* dir,
   int bytes_left,
   dentry_key_t* skip_key,
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 4715a0880c83..89cda03155dc 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -162,61 +162,61 @@ class Server {
   void handle_client_request(const cref_t<MClientRequest> &m);
   void handle_client_reply(const cref_t<MClientReply> &m);
 
-  void journal_and_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn,
+  void journal_and_reply(const MDRequestRef& mdr, CInode *tracei, CDentry *tracedn,
 			 LogEvent *le, MDSLogContextBase *fin);
   void submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin,
-                          MDRequestRef& mdr, std::string_view event);
-  void dispatch_client_request(MDRequestRef& mdr);
+                          const MDRequestRef& mdr, std::string_view event);
+  void dispatch_client_request(const MDRequestRef& mdr);
   void perf_gather_op_latency(const cref_t<MClientRequest> &req, utime_t lat);
-  void early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn);
-  void respond_to_request(MDRequestRef& mdr, int r = 0);
+  void early_reply(const MDRequestRef& mdr, CInode *tracei, CDentry *tracedn);
+  void respond_to_request(const MDRequestRef& mdr, int r = 0);
   void set_trace_dist(const ref_t<MClientReply> &reply, CInode *in, CDentry *dn,
-		      MDRequestRef& mdr);
+		      const MDRequestRef& mdr);
 
   void handle_peer_request(const cref_t<MMDSPeerRequest> &m);
   void handle_peer_request_reply(const cref_t<MMDSPeerRequest> &m);
-  void dispatch_peer_request(MDRequestRef& mdr);
-  void handle_peer_auth_pin(MDRequestRef& mdr);
-  void handle_peer_auth_pin_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack);
+  void dispatch_peer_request(const MDRequestRef& mdr);
+  void handle_peer_auth_pin(const MDRequestRef& mdr);
+  void handle_peer_auth_pin_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack);
 
   // some helpers
-  bool check_fragment_space(MDRequestRef& mdr, CDir *in);
-  bool check_dir_max_entries(MDRequestRef& mdr, CDir *in);
-  bool check_access(MDRequestRef& mdr, CInode *in, unsigned mask);
+  bool check_fragment_space(const MDRequestRef& mdr, CDir *in);
+  bool check_dir_max_entries(const MDRequestRef& mdr, CDir *in);
+  bool check_access(const MDRequestRef& mdr, CInode *in, unsigned mask);
   bool _check_access(Session *session, CInode *in, unsigned mask, int caller_uid, int caller_gid, int setattr_uid, int setattr_gid);
-  CDentry *prepare_stray_dentry(MDRequestRef& mdr, CInode *in);
-  CInode* prepare_new_inode(MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
+  CDentry *prepare_stray_dentry(const MDRequestRef& mdr, CInode *in);
+  CInode* prepare_new_inode(const MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
 			    const file_layout_t *layout=nullptr);
-  void journal_allocated_inos(MDRequestRef& mdr, EMetaBlob *blob);
-  void apply_allocated_inos(MDRequestRef& mdr, Session *session);
+  void journal_allocated_inos(const MDRequestRef& mdr, EMetaBlob *blob);
+  void apply_allocated_inos(const MDRequestRef& mdr, Session *session);
 
-  void _try_open_ino(MDRequestRef& mdr, int r, inodeno_t ino);
-  CInode* rdlock_path_pin_ref(MDRequestRef& mdr, bool want_auth,
+  void _try_open_ino(const MDRequestRef& mdr, int r, inodeno_t ino);
+  CInode* rdlock_path_pin_ref(const MDRequestRef& mdr, bool want_auth,
 			      bool no_want_auth=false);
-  CDentry* rdlock_path_xlock_dentry(MDRequestRef& mdr, bool create,
+  CDentry* rdlock_path_xlock_dentry(const MDRequestRef& mdr, bool create,
 				    bool okexist=false, bool authexist=false,
 				    bool want_layout=false);
   std::pair<CDentry*, CDentry*>
-	    rdlock_two_paths_xlock_destdn(MDRequestRef& mdr, bool xlock_srcdn);
+	    rdlock_two_paths_xlock_destdn(const MDRequestRef& mdr, bool xlock_srcdn);
 
-  CDir* try_open_auth_dirfrag(CInode *diri, frag_t fg, MDRequestRef& mdr);
+  CDir* try_open_auth_dirfrag(CInode *diri, frag_t fg, const MDRequestRef& mdr);
 
   // requests on existing inodes.
-  void handle_client_getattr(MDRequestRef& mdr, bool is_lookup);
-  void handle_client_lookup_ino(MDRequestRef& mdr,
+  void handle_client_getattr(const MDRequestRef& mdr, bool is_lookup);
+  void handle_client_lookup_ino(const MDRequestRef& mdr,
 				bool want_parent, bool want_dentry);
-  void _lookup_snap_ino(MDRequestRef& mdr);
-  void _lookup_ino_2(MDRequestRef& mdr, int r);
-  void handle_client_readdir(MDRequestRef& mdr);
-  void handle_client_file_setlock(MDRequestRef& mdr);
-  void handle_client_file_readlock(MDRequestRef& mdr);
+  void _lookup_snap_ino(const MDRequestRef& mdr);
+  void _lookup_ino_2(const MDRequestRef& mdr, int r);
+  void handle_client_readdir(const MDRequestRef& mdr);
+  void handle_client_file_setlock(const MDRequestRef& mdr);
+  void handle_client_file_readlock(const MDRequestRef& mdr);
 
-  bool xlock_policylock(MDRequestRef& mdr, CInode *in,
+  bool xlock_policylock(const MDRequestRef& mdr, CInode *in,
 			bool want_layout=false, bool xlock_snaplock=false);
-  CInode* try_get_auth_inode(MDRequestRef& mdr, inodeno_t ino);
-  void handle_client_setattr(MDRequestRef& mdr);
-  void handle_client_setlayout(MDRequestRef& mdr);
-  void handle_client_setdirlayout(MDRequestRef& mdr);
+  CInode* try_get_auth_inode(const MDRequestRef& mdr, inodeno_t ino);
+  void handle_client_setattr(const MDRequestRef& mdr);
+  void handle_client_setlayout(const MDRequestRef& mdr);
+  void handle_client_setdirlayout(const MDRequestRef& mdr);
 
   int parse_quota_vxattr(std::string name, std::string value, quota_info_t *quota);
   void create_quota_realm(CInode *in);
@@ -226,100 +226,100 @@ class Server {
 				 file_layout_t *layout);
   int parse_layout_vxattr(std::string name, std::string value, const OSDMap& osdmap,
 			  file_layout_t *layout, bool validate=true);
-  int check_layout_vxattr(MDRequestRef& mdr,
+  int check_layout_vxattr(const MDRequestRef& mdr,
                           std::string name,
                           std::string value,
                           file_layout_t *layout);
-  void handle_set_vxattr(MDRequestRef& mdr, CInode *cur);
-  void handle_remove_vxattr(MDRequestRef& mdr, CInode *cur);
-  void handle_client_getvxattr(MDRequestRef& mdr);
-  void handle_client_setxattr(MDRequestRef& mdr);
-  void handle_client_removexattr(MDRequestRef& mdr);
+  void handle_set_vxattr(const MDRequestRef& mdr, CInode *cur);
+  void handle_remove_vxattr(const MDRequestRef& mdr, CInode *cur);
+  void handle_client_getvxattr(const MDRequestRef& mdr);
+  void handle_client_setxattr(const MDRequestRef& mdr);
+  void handle_client_removexattr(const MDRequestRef& mdr);
 
-  void handle_client_fsync(MDRequestRef& mdr);
+  void handle_client_fsync(const MDRequestRef& mdr);
 
   // open
-  void handle_client_open(MDRequestRef& mdr);
-  void handle_client_openc(MDRequestRef& mdr);  // O_CREAT variant.
-  void do_open_truncate(MDRequestRef& mdr, int cmode);  // O_TRUNC variant.
+  void handle_client_open(const MDRequestRef& mdr);
+  void handle_client_openc(const MDRequestRef& mdr);  // O_CREAT variant.
+  void do_open_truncate(const MDRequestRef& mdr, int cmode);  // O_TRUNC variant.
 
   // namespace changes
-  void handle_client_mknod(MDRequestRef& mdr);
-  void handle_client_mkdir(MDRequestRef& mdr);
-  void handle_client_symlink(MDRequestRef& mdr);
+  void handle_client_mknod(const MDRequestRef& mdr);
+  void handle_client_mkdir(const MDRequestRef& mdr);
+  void handle_client_symlink(const MDRequestRef& mdr);
 
   // link
-  void handle_client_link(MDRequestRef& mdr);
-  void _link_local(MDRequestRef& mdr, CDentry *dn, CInode *targeti, SnapRealm *target_realm);
-  void _link_local_finish(MDRequestRef& mdr, CDentry *dn, CInode *targeti,
+  void handle_client_link(const MDRequestRef& mdr);
+  void _link_local(const MDRequestRef& mdr, CDentry *dn, CInode *targeti, SnapRealm *target_realm);
+  void _link_local_finish(const MDRequestRef& mdr, CDentry *dn, CInode *targeti,
 			  version_t, version_t, bool);
 
-  void _link_remote(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti);
-  void _link_remote_finish(MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti,
+  void _link_remote(const MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti);
+  void _link_remote_finish(const MDRequestRef& mdr, bool inc, CDentry *dn, CInode *targeti,
 			   version_t);
 
-  void handle_peer_link_prep(MDRequestRef& mdr);
-  void _logged_peer_link(MDRequestRef& mdr, CInode *targeti, bool adjust_realm);
-  void _commit_peer_link(MDRequestRef& mdr, int r, CInode *targeti);
-  void _committed_peer(MDRequestRef& mdr);  // use for rename, too
-  void handle_peer_link_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
-  void do_link_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr);
-  void _link_rollback_finish(MutationRef& mut, MDRequestRef& mdr,
+  void handle_peer_link_prep(const MDRequestRef& mdr);
+  void _logged_peer_link(const MDRequestRef& mdr, CInode *targeti, bool adjust_realm);
+  void _commit_peer_link(const MDRequestRef& mdr, int r, CInode *targeti);
+  void _committed_peer(const MDRequestRef& mdr);  // use for rename, too
+  void handle_peer_link_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
+  void do_link_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr);
+  void _link_rollback_finish(MutationRef& mut, const MDRequestRef& mdr,
 			     std::map<client_t,ref_t<MClientSnap>>& split);
 
   // unlink
-  void handle_client_unlink(MDRequestRef& mdr);
-  bool _dir_is_nonempty_unlocked(MDRequestRef& mdr, CInode *rmdiri);
-  bool _dir_is_nonempty(MDRequestRef& mdr, CInode *rmdiri);
-  void _unlink_local(MDRequestRef& mdr, CDentry *dn, CDentry *straydn);
-  void _unlink_local_finish(MDRequestRef& mdr,
+  void handle_client_unlink(const MDRequestRef& mdr);
+  bool _dir_is_nonempty_unlocked(const MDRequestRef& mdr, CInode *rmdiri);
+  bool _dir_is_nonempty(const MDRequestRef& mdr, CInode *rmdiri);
+  void _unlink_local(const MDRequestRef& mdr, CDentry *dn, CDentry *straydn);
+  void _unlink_local_finish(const MDRequestRef& mdr,
 			    CDentry *dn, CDentry *straydn,
 			    version_t);
-  bool _rmdir_prepare_witness(MDRequestRef& mdr, mds_rank_t who, std::vector<CDentry*>& trace, CDentry *straydn);
-  void handle_peer_rmdir_prep(MDRequestRef& mdr);
-  void _logged_peer_rmdir(MDRequestRef& mdr, CDentry *srcdn, CDentry *straydn);
-  void _commit_peer_rmdir(MDRequestRef& mdr, int r, CDentry *straydn);
-  void handle_peer_rmdir_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack);
-  void do_rmdir_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr);
-  void _rmdir_rollback_finish(MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn);
+  bool _rmdir_prepare_witness(const MDRequestRef& mdr, mds_rank_t who, std::vector<CDentry*>& trace, CDentry *straydn);
+  void handle_peer_rmdir_prep(const MDRequestRef& mdr);
+  void _logged_peer_rmdir(const MDRequestRef& mdr, CDentry *srcdn, CDentry *straydn);
+  void _commit_peer_rmdir(const MDRequestRef& mdr, int r, CDentry *straydn);
+  void handle_peer_rmdir_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &ack);
+  void do_rmdir_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr);
+  void _rmdir_rollback_finish(const MDRequestRef& mdr, metareqid_t reqid, CDentry *dn, CDentry *straydn);
 
   // rename
-  void handle_client_rename(MDRequestRef& mdr);
-  void _rename_finish(MDRequestRef& mdr,
+  void handle_client_rename(const MDRequestRef& mdr);
+  void _rename_finish(const MDRequestRef& mdr,
 		      CDentry *srcdn, CDentry *destdn, CDentry *straydn);
 
-  void handle_client_lssnap(MDRequestRef& mdr);
-  void handle_client_mksnap(MDRequestRef& mdr);
-  void _mksnap_finish(MDRequestRef& mdr, CInode *diri, SnapInfo &info);
-  void handle_client_rmsnap(MDRequestRef& mdr);
-  void _rmsnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid);
-  void handle_client_renamesnap(MDRequestRef& mdr);
-  void _renamesnap_finish(MDRequestRef& mdr, CInode *diri, snapid_t snapid);
-  void handle_client_readdir_snapdiff(MDRequestRef& mdr);
+  void handle_client_lssnap(const MDRequestRef& mdr);
+  void handle_client_mksnap(const MDRequestRef& mdr);
+  void _mksnap_finish(const MDRequestRef& mdr, CInode *diri, SnapInfo &info);
+  void handle_client_rmsnap(const MDRequestRef& mdr);
+  void _rmsnap_finish(const MDRequestRef& mdr, CInode *diri, snapid_t snapid);
+  void handle_client_renamesnap(const MDRequestRef& mdr);
+  void _renamesnap_finish(const MDRequestRef& mdr, CInode *diri, snapid_t snapid);
+  void handle_client_readdir_snapdiff(const MDRequestRef& mdr);
 
   // helpers
-  bool _rename_prepare_witness(MDRequestRef& mdr, mds_rank_t who, std::set<mds_rank_t> &witnesse,
+  bool _rename_prepare_witness(const MDRequestRef& mdr, mds_rank_t who, std::set<mds_rank_t> &witnesse,
 			       std::vector<CDentry*>& srctrace, std::vector<CDentry*>& dsttrace, CDentry *straydn);
-  version_t _rename_prepare_import(MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl);
+  version_t _rename_prepare_import(const MDRequestRef& mdr, CDentry *srcdn, bufferlist *client_map_bl);
   bool _need_force_journal(CInode *diri, bool empty);
-  void _rename_prepare(MDRequestRef& mdr,
+  void _rename_prepare(const MDRequestRef& mdr,
 		       EMetaBlob *metablob, bufferlist *client_map_bl,
 		       CDentry *srcdn, CDentry *destdn, std::string_view alternate_name,
                        CDentry *straydn);
   /* set not_journaling=true if you're going to discard the results --
    * this bypasses the asserts to make sure we're journaling the right
    * things on the right nodes */
-  void _rename_apply(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  void _rename_apply(const MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
 
   // slaving
-  void handle_peer_rename_prep(MDRequestRef& mdr);
-  void handle_peer_rename_prep_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
-  void handle_peer_rename_notify_ack(MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
-  void _peer_rename_sessions_flushed(MDRequestRef& mdr);
-  void _logged_peer_rename(MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
-  void _commit_peer_rename(MDRequestRef& mdr, int r, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
-  void do_rename_rollback(bufferlist &rbl, mds_rank_t leader, MDRequestRef& mdr, bool finish_mdr=false);
-  void _rename_rollback_finish(MutationRef& mut, MDRequestRef& mdr, CDentry *srcdn, version_t srcdnpv,
+  void handle_peer_rename_prep(const MDRequestRef& mdr);
+  void handle_peer_rename_prep_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
+  void handle_peer_rename_notify_ack(const MDRequestRef& mdr, const cref_t<MMDSPeerRequest> &m);
+  void _peer_rename_sessions_flushed(const MDRequestRef& mdr);
+  void _logged_peer_rename(const MDRequestRef& mdr, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  void _commit_peer_rename(const MDRequestRef& mdr, int r, CDentry *srcdn, CDentry *destdn, CDentry *straydn);
+  void do_rename_rollback(bufferlist &rbl, mds_rank_t leader, const MDRequestRef& mdr, bool finish_mdr=false);
+  void _rename_rollback_finish(MutationRef& mut, const MDRequestRef& mdr, CDentry *srcdn, version_t srcdnpv,
 			       CDentry *destdn, CDentry *staydn, std::map<client_t,ref_t<MClientSnap>> splits[2],
 			       bool finish_mdr);
 
@@ -481,10 +481,10 @@ class Server {
            xattr_name == "ceph.mirror.dirty_snap_id";
   }
 
-  void reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &reply);
+  void reply_client_request(const MDRequestRef& mdr, const ref_t<MClientReply> &reply);
   void flush_session(Session *session, MDSGatherBuilder& gather);
 
-  void _finalize_readdir(MDRequestRef& mdr,
+  void _finalize_readdir(const MDRequestRef& mdr,
                          CInode *diri,
                          CDir* dir,
                          bool start,
@@ -495,7 +495,7 @@ class Server {
                          bufferlist& dnbl);
   void _readdir_diff(
     utime_t now,
-    MDRequestRef& mdr,
+    const MDRequestRef& mdr,
     CInode* diri,
     CDir* dir,
     SnapRealm* realm,
@@ -506,7 +506,7 @@ class Server {
     unsigned req_flags,
     bufferlist& dirbl);
   bool build_snap_diff(
-    MDRequestRef& mdr,
+    const MDRequestRef& mdr,
     CDir* dir,
     int bytes_left,
     dentry_key_t* skip_key,

From 3354b67fbdf5f37bb58ff07743426e179bed8069 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Feb 2023 22:50:37 -0500
Subject: [PATCH 0670/2492] rgw/s3: add part param and response to GetObj

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.h       |  5 +++++
 src/rgw/rgw_rest_s3.cc | 15 +++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 09fe6cf4d7af..dc5bc8be55ec 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -371,6 +371,11 @@ class RGWGetObj : public RGWOp {
   bool get_retention;
   bool get_legal_hold;
 
+  // optional partNumber param for s3
+  std::optional<int> multipart_part_num;
+  // PartsCount response when partNumber is specified
+  std::optional<int> multipart_parts_count;
+
   int init_common();
 public:
   RGWGetObj() {
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6d56766d10eb..5425406b1eb9 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -305,6 +305,18 @@ int RGWGetObj_ObjStore_S3::get_params(optional_yield y)
   dst_zone_trace = s->info.args.get(RGW_SYS_PARAM_PREFIX "if-not-replicated-to");
   get_torrent = s->info.args.exists("torrent");
 
+  // optional part number
+  auto optstr = s->info.args.get_optional("partNumber");
+  if (optstr) {
+    string err;
+    multipart_part_num = strict_strtol(optstr->c_str(), 10, &err);
+    if (!err.empty()) {
+      s->err.message = "Invalid partNumber: " + err;
+      ldpp_dout(s, 10) << "bad part number " << *optstr << ": " << err << dendl;
+      return -ERR_INVALID_PART;
+    }
+  }
+
   return RGWGetObj_ObjStore::get_params(y);
 }
 
@@ -451,6 +463,9 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
       }
     } catch (const buffer::error&) {} // omit x-rgw-replicated-from headers
   }
+  if (multipart_parts_count) {
+    dump_header(s, "x-amz-mp-parts-count", *multipart_parts_count);
+  }
 
   if (! op_ret) {
     if (! lo_etag.empty()) {

From 60dadf3c8d56eea2a6b2074599404fa74345bcc8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 20 Feb 2023 08:06:52 -0500
Subject: [PATCH 0671/2492] rgw/rados: remove get_obj_state() overload for
 follow_olh=true

and just add the follow_olh=true argument to callers

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_putobj_processor.cc |  5 +++--
 src/rgw/driver/rados/rgw_rados.cc            |  4 +++-
 src/rgw/driver/rados/rgw_rados.h             | 10 +++++-----
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 65fbd5791d5e..b6d08e5c7f38 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -595,9 +595,10 @@ int AppendObjectProcessor::process_first_chunk(bufferlist &&data, rgw::sal::Data
 
 int AppendObjectProcessor::prepare(optional_yield y)
 {
-  RGWObjState *astate;
+  RGWObjState *astate = nullptr;
+  constexpr bool follow_olh = true;
   int r = store->get_obj_state(dpp, &obj_ctx, bucket_info, head_obj,
-                               &astate, &cur_manifest, y);
+                               &astate, &cur_manifest, follow_olh, y);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 3632e66af595..efb7fd37f61c 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4658,7 +4658,9 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   RGWObjState *astate = NULL;
   RGWObjManifest *amanifest = nullptr;
 
-  ret = get_obj_state(dpp, &obj_ctx, src_bucket_info, src_obj, &astate, &amanifest, y);
+  constexpr bool follow_olh = true;
+  ret = get_obj_state(dpp, &obj_ctx, src_bucket_info, src_obj,
+                      &astate, &amanifest, follow_olh, y);
   if (ret < 0) {
     return ret;
   }
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 77a08c068a64..cde2fa66bdfa 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -1268,11 +1268,11 @@ class RGWRados
                         optional_yield y,
                         ceph::real_time set_mtime = ceph::real_clock::zero());
 
-  int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest,
-                    bool follow_olh, optional_yield y, bool assume_noent = false);
-  int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest, optional_yield y) {
-    return get_obj_state(dpp, rctx, bucket_info, obj, state, manifest, true, y);
-  }
+  int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
+                    RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                    RGWObjState** pstate, RGWObjManifest** pmanifest,
+                    bool follow_olh, optional_yield y,
+                    bool assume_noent = false);
 
   using iterate_obj_cb = int (*)(const DoutPrefixProvider*, const rgw_raw_obj&, off_t, off_t,
                                  off_t, bool, RGWObjState*, void*);

From a308e3a1d590f1a35337415caf756a550f2617a9 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 20 Feb 2023 08:08:01 -0500
Subject: [PATCH 0672/2492] rgw/rados: add get_obj_state() overload for
 RGWObjStateManifest

add an overload to expose the manifest storage to callers of
get_obj_state(). the existing RGWObjState+RGWObjManifest overload
just calls the RGWObjStateManifest one

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 64 +++++++++++++++++++------------
 src/rgw/driver/rados/rgw_rados.h  | 15 ++++++--
 2 files changed, 51 insertions(+), 28 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index efb7fd37f61c..fc1d7b7a6855 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -5887,48 +5887,38 @@ static bool has_olh_tag(map<string, bufferlist>& attrs)
 int RGWRados::get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx&
 				   obj_ctx, RGWBucketInfo& bucket_info,
 				   const rgw_obj& obj, RGWObjState *olh_state,
-				   RGWObjState **target_state,
-				   RGWObjManifest **target_manifest, optional_yield y)
+				   RGWObjStateManifest **psm, optional_yield y)
 {
   ceph_assert(olh_state->is_olh);
 
   rgw_obj target;
-  int r = RGWRados::follow_olh(dpp, bucket_info, obj_ctx, olh_state, obj, &target, y); /* might return -EAGAIN */
+  int r = RGWRados::follow_olh(dpp, bucket_info, obj_ctx, olh_state,
+                               obj, &target, y); /* might return -EAGAIN */
   if (r < 0) {
     return r;
   }
 
-  r = get_obj_state(dpp, &obj_ctx, bucket_info, target, target_state,
-		    target_manifest, false, y);
-  if (r < 0) {
-    return r;
-  }
-
-  return 0;
+  return get_obj_state(dpp, &obj_ctx, bucket_info, target, psm, false, y);
 }
 
 int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *octx,
-				 RGWBucketInfo& bucket_info, const rgw_obj& obj,
-                                 RGWObjState **state, RGWObjManifest** manifest,
-				 bool follow_olh, optional_yield y, bool assume_noent)
+                                 RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                                 RGWObjStateManifest** psm, bool follow_olh,
+                                 optional_yield y, bool assume_noent)
 {
   if (obj.empty()) {
     return -EINVAL;
   }
 
   bool need_follow_olh = follow_olh && obj.key.instance.empty();
-  *manifest = nullptr;
 
   RGWObjStateManifest *sm = octx->get_state(obj);
   RGWObjState *s = &(sm->state);
   ldpp_dout(dpp, 20) << "get_obj_state: octx=" << (void *)octx << " obj=" << obj << " state=" << (void *)s << " s->prefetch_data=" << s->prefetch_data << dendl;
-  *state = s;
-  if (sm->manifest) {
-    *manifest = &(*sm->manifest);
-  }
+  *psm = sm;
   if (s->has_attrs) {
     if (s->is_olh && need_follow_olh) {
-      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, state, manifest, y);
+      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, psm, y);
     }
     return 0;
   }
@@ -6020,7 +6010,6 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *oc
       ldpp_dout(dpp, 0) << "ERROR: couldn't decode manifest" << dendl;
       return -EIO;
     }
-    *manifest = &(*sm->manifest);
     ldpp_dout(dpp, 10) << "manifest: total_size = " << sm->manifest->get_obj_size() << dendl;
     if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 20>() && \
 	sm->manifest->has_explicit_objs()) {
@@ -6080,7 +6069,7 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *oc
     ldpp_dout(dpp, 20) << __func__ << ": setting s->olh_tag to " << string(s->olh_tag.c_str(), s->olh_tag.length()) << dendl;
 
     if (need_follow_olh) {
-      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, state, manifest, y);
+      return get_olh_target_state(dpp, *octx, bucket_info, obj, s, psm, y);
     } else if (obj.key.have_null_instance() && !sm->manifest) {
       // read null version, and the head object only have olh info
       s->exists = false;
@@ -6091,18 +6080,45 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *oc
   return 0;
 }
 
-int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *octx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest,
-                            bool follow_olh, optional_yield y, bool assume_noent)
+int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *octx,
+                            RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                            RGWObjStateManifest** psm, bool follow_olh,
+                            optional_yield y, bool assume_noent)
 {
   int ret;
 
   do {
-    ret = get_obj_state_impl(dpp, octx, bucket_info, obj, state, manifest, follow_olh, y, assume_noent);
+    ret = get_obj_state_impl(dpp, octx, bucket_info, obj, psm,
+                             follow_olh, y, assume_noent);
   } while (ret == -EAGAIN);
 
   return ret;
 }
 
+int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
+                            RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                            RGWObjState** pstate, RGWObjManifest** pmanifest,
+                            bool follow_olh, optional_yield y, bool assume_noent)
+{
+  RGWObjStateManifest* sm = nullptr;
+  int r = get_obj_state(dpp, rctx, bucket_info, obj, &sm,
+                        follow_olh, y, assume_noent);
+  if (r < 0) {
+    return r;
+  }
+  if (pstate) {
+    *pstate = &sm->state;
+  }
+  if (pmanifest) {
+    if (sm->manifest) {
+      *pmanifest = &(*sm->manifest);
+    } else {
+      *pmanifest = nullptr;
+    }
+  }
+  return 0;
+}
+
 int RGWRados::Object::get_manifest(const DoutPrefixProvider *dpp, RGWObjManifest **pmanifest, optional_yield y)
 {
   RGWObjState *astate;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index cde2fa66bdfa..d8cba7b9b4ed 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -414,10 +414,12 @@ class RGWRados
 
   int get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& rctx,
 			   RGWBucketInfo& bucket_info, const rgw_obj& obj,
-			   RGWObjState *olh_state, RGWObjState **target_state,
-			   RGWObjManifest **target_manifest, optional_yield y);
-  int get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, RGWObjManifest** manifest,
-                         bool follow_olh, optional_yield y, bool assume_noent = false);
+			   RGWObjState *olh_state, RGWObjStateManifest **psm,
+			   optional_yield y);
+  int get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
+                         RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                         RGWObjStateManifest** psm, bool follow_olh,
+                         optional_yield y, bool assume_noent = false);
   int append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx* rctx, RGWBucketInfo& bucket_info, const rgw_obj& obj,
                          librados::ObjectOperation& op, RGWObjState **state,
 			 RGWObjManifest** pmanifest, optional_yield y);
@@ -1268,6 +1270,11 @@ class RGWRados
                         optional_yield y,
                         ceph::real_time set_mtime = ceph::real_clock::zero());
 
+  int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
+                    RGWBucketInfo& bucket_info, const rgw_obj& obj,
+                    RGWObjStateManifest** psm, bool follow_olh,
+                    optional_yield y, bool assume_noent = false);
+
   int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx,
                     RGWBucketInfo& bucket_info, const rgw_obj& obj,
                     RGWObjState** pstate, RGWObjManifest** pmanifest,

From 8ae61ca506473915e25961a44d451ce481cebe59 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 20 Feb 2023 11:17:34 -0500
Subject: [PATCH 0673/2492] rgw/rados: add obj_find_part() to RGWObjManifest

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_obj_manifest.cc | 21 +++++++++++++++++++++
 src/rgw/driver/rados/rgw_obj_manifest.h  | 15 +++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_obj_manifest.cc b/src/rgw/driver/rados/rgw_obj_manifest.cc
index e7fc363907ff..2e9f94350089 100644
--- a/src/rgw/driver/rados/rgw_obj_manifest.cc
+++ b/src/rgw/driver/rados/rgw_obj_manifest.cc
@@ -197,6 +197,27 @@ bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
   return true;
 }
 
+auto RGWObjManifest::obj_find_part(const DoutPrefixProvider *dpp,
+                                   int part_num) const
+    -> obj_iterator
+{
+  const obj_iterator end = obj_end(dpp);
+  if (end.get_cur_part_id() == 0) { // not mulitipart
+    return end;
+  }
+
+  // linear search over parts/stripes
+  for (obj_iterator i = obj_begin(dpp); i != end; ++i) {
+    if (i.get_cur_part_id() == part_num) {
+      return i;
+    }
+    if (i.get_cur_part_id() > part_num) {
+      return end;
+    }
+  }
+  return end;
+}
+
 int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m,
                                             const rgw_placement_rule& head_placement_rule,
                                             const rgw_placement_rule *tail_placement_rule,
diff --git a/src/rgw/driver/rados/rgw_obj_manifest.h b/src/rgw/driver/rados/rgw_obj_manifest.h
index b6b89653372d..af0ce9ce0fa3 100644
--- a/src/rgw/driver/rados/rgw_obj_manifest.h
+++ b/src/rgw/driver/rados/rgw_obj_manifest.h
@@ -20,6 +20,7 @@
 
 #pragma once
 
+#include <optional>
 #include "rgw_zone_types.h"
 #include "rgw_bucket_types.h"
 #include "rgw_obj_types.h"
@@ -57,6 +58,14 @@ class rgw_obj_select {
     }
   }
 
+  std::optional<rgw_obj> get_head_obj() const {
+    if (is_raw) {
+      return std::nullopt;
+    } else {
+      return obj;
+    }
+  }
+
   rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const;
   rgw_raw_obj get_raw_obj(RGWRados* store) const;
 
@@ -547,6 +556,10 @@ class RGWObjManifest {
       return ofs;
     }
 
+    const std::string& get_cur_override_prefix() const {
+      return cur_override_prefix;
+    }
+
     int get_cur_part_id() const {
       return cur_part_id;
     }
@@ -582,6 +595,8 @@ class RGWObjManifest {
   obj_iterator obj_find(const DoutPrefixProvider *dpp, uint64_t ofs) const {
     return obj_iterator{dpp, this, std::min(ofs, obj_size)};
   }
+  // return an iterator to the beginning of the given part number
+  obj_iterator obj_find_part(const DoutPrefixProvider *dpp, int part_num) const;
 
   /*
    * simple object generator. Using a simple single rule manifest.

From 01d8b4c38bdf98c2c2f8777200a1ba45a954a05d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 17 Feb 2023 08:47:05 -0500
Subject: [PATCH 0674/2492] rgw/rados: part support for RGWRados::Object::Read

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc     | 112 +++++++++++++++++++++++++-
 src/rgw/driver/rados/rgw_rados.h      |   2 +
 src/rgw/driver/rados/rgw_sal_rados.cc |   2 +
 src/rgw/rgw_op.cc                     |   5 ++
 src/rgw/rgw_sal.h                     |   6 ++
 5 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index fc1d7b7a6855..c5624ba0171e 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -6527,15 +6527,105 @@ int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBu
   return 0;
 }
 
+static int get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
+                              RGWRados* store, RGWBucketInfo& bucket_info,
+                              RGWObjectCtx* rctx, RGWObjManifest* manifest,
+                              int part_num, int* parts_count, bool prefetch,
+                              RGWObjState** pstate, RGWObjManifest** pmanifest)
+{
+  // navigate to the requested part in the manifest
+  RGWObjManifest::obj_iterator end = manifest->obj_end(dpp);
+  if (end.get_cur_part_id() == 0) { // not multipart
+    ldpp_dout(dpp, 20) << "object does not have a multipart manifest" << dendl;
+    return -ERR_INVALID_PART;
+  }
+  if (parts_count) {
+    *parts_count = end.get_cur_part_id() - 1;
+  }
+  ldpp_dout(dpp, 20) << "seeking to part #" << part_num
+      << " in the object manifest" << dendl;
+  RGWObjManifest::obj_iterator iter = manifest->obj_find_part(dpp, part_num);
+  if (iter == end) { // part number not found
+    ldpp_dout(dpp, 20) << "failed to find part #" << part_num
+        << " in the object manifest" << dendl;
+    return -ERR_INVALID_PART;
+  }
+  auto head_obj = iter.get_location().get_head_obj();
+  if (!head_obj) { // iterator points to a tail object
+    ldpp_dout(dpp, 20) << "object manifest for part #" << part_num
+        << " points to a tail object" << dendl;
+    return -ERR_INVALID_PART;
+  }
+  const auto part_offset = iter.get_ofs();
+
+  // read the part's head object
+  if (prefetch) {
+    rctx->set_prefetch_data(*head_obj);
+  }
+  RGWObjStateManifest* sm = nullptr;
+  constexpr bool follow_olh = false; // parts aren't versioned
+  int r = store->get_obj_state(dpp, rctx, bucket_info, *head_obj,
+                               &sm, follow_olh, y);
+  if (r < 0) {
+    return r;
+  }
+  *pstate = &sm->state;
+
+  // if the part has its own manifest, use it directly
+  if (sm->manifest) {
+    *pmanifest = &*sm->manifest;
+    return 0;
+  }
+
+  // create a new manifest for just this part
+  sm->manifest.emplace();
+  RGWObjManifest& part_manifest = *sm->manifest;
+  part_manifest.set_multipart_part_rule(iter.get_stripe_size(), part_num);
+
+  if (auto& prefix = iter.get_cur_override_prefix(); !prefix.empty()) {
+    // the part was reuploaded with a different prefix
+    part_manifest.set_prefix(prefix);
+  } else {
+    part_manifest.set_prefix(manifest->get_prefix());
+  }
+
+  RGWObjManifest::generator gen;
+  gen.create_begin(store->ctx(), &part_manifest,
+                   manifest->get_head_placement_rule(),
+                   &manifest->get_tail_placement().placement_rule,
+                   head_obj->bucket, *head_obj);
+
+  // copy each of the part's stripes into the new manifest. the final call to
+  // create_next() uses the starting offset of the next part
+  do {
+    ++iter;
+    gen.create_next(iter.get_ofs() - part_offset);
+  } while (iter.get_cur_part_id() == part_num);
+
+  // update the object size
+  sm->state.size = part_manifest.get_obj_size();
+
+  *pmanifest = &part_manifest;
+  return 0;
+}
+
 int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *dpp)
 {
   RGWRados *store = source->get_store();
   CephContext *cct = store->ctx();
+  RGWObjectCtx& obj_ctx = source->get_ctx();
 
   bufferlist etag;
 
   map<string, bufferlist>::iterator iter;
 
+  bool part_prefetch = false;
+  if (params.part_num) {
+    // prefetch from the part's head object instead of the multipart head
+    auto sm = obj_ctx.get_state(source->get_obj());
+    part_prefetch = std::exchange(sm->state.prefetch_data, false);
+  }
+
   RGWObjState *astate;
   RGWObjManifest *manifest = nullptr;
   int r = source->get_state(dpp, &astate, &manifest, true, y);
@@ -6546,7 +6636,27 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *
     return -ENOENT;
   }
 
-  const RGWBucketInfo& bucket_info = source->get_bucket_info();
+  RGWBucketInfo& bucket_info = source->get_bucket_info();
+
+  if (params.part_num) {
+    // use the manifest to redirect to the requested part number
+    if (!manifest) {
+      return -ERR_INVALID_PART;
+    }
+    r = get_part_obj_state(dpp, y, store, bucket_info, &source->get_ctx(),
+                           manifest, *params.part_num, params.parts_count,
+                           part_prefetch, &astate, &manifest);
+    if (r < 0) {
+      ldpp_dout(dpp, 4) << "failed to read part #" << *params.part_num
+          << ": " << cpp_strerror(r) << dendl;
+      return -ERR_INVALID_PART;
+    }
+    if (!astate->exists) {
+      ldpp_dout(dpp, 4) << "part #" << *params.part_num
+          << " does not exist" << dendl;
+      return -ERR_INVALID_PART;
+    }
+  }
 
   state.obj = astate->obj;
   store->obj_to_raw(bucket_info.placement_rule, state.obj, &state.head_obj);
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index d8cba7b9b4ed..89e4576a4788 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -769,6 +769,8 @@ class RGWRados
         std::map<std::string, bufferlist> *attrs;
         rgw_obj *target_obj;
 	uint64_t *epoch;
+        int* part_num = nullptr;
+        int* parts_count = nullptr;
 
         Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr),
 		   target_obj(nullptr), epoch(nullptr)
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index ed6e2525ac2b..e619e9681148 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1946,6 +1946,8 @@ int RadosObject::RadosReadOp::prepare(optional_yield y, const DoutPrefixProvider
   parent_op.conds.if_nomatch = params.if_nomatch;
   parent_op.params.lastmod = params.lastmod;
   parent_op.params.target_obj = params.target_obj;
+  parent_op.params.part_num = params.part_num;
+  parent_op.params.parts_count = params.parts_count;
   parent_op.params.obj_size = &obj_size;
   parent_op.params.attrs = &source->get_attrs();
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 3accf03e4968..1a9cd11c6eff 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2281,6 +2281,11 @@ void RGWGetObj::execute(optional_yield y)
   read_op->params.if_match = if_match;
   read_op->params.if_nomatch = if_nomatch;
   read_op->params.lastmod = &lastmod;
+  if (multipart_part_num) {
+    read_op->params.part_num = &*multipart_part_num;
+    multipart_parts_count.emplace(0);
+    read_op->params.parts_count = &*multipart_parts_count;
+  }
 
   op_ret = read_op->prepare(s->yield, this);
   if (op_ret < 0)
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 9b099be4ac2c..57c7c20d6e15 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -843,6 +843,12 @@ class Object {
         const char* if_nomatch{nullptr};
         ceph::real_time* lastmod{nullptr};
         rgw_obj* target_obj{nullptr}; // XXX dang remove?
+
+        /// If non-null, read data/attributes from the given multipart part.
+        int* part_num{nullptr};
+        /// If part_num is specified, the total number of multipart parts is
+        /// written to this output parameter.
+        int* parts_count{nullptr};
       } params;
 
       virtual ~ReadOp() = default;

From 6fc57159ef7e5b46e0198db98fff5c7ac8f8880c Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 20 Feb 2023 17:27:28 -0500
Subject: [PATCH 0675/2492] rgw/rados: RadosReadOp::prepare only updates object
 instance

when called on a versioned object, prepare() may follow olh and look up
a different object instance

but when called on a multipart part, we should not overwrite the
original object name with the part's object name (of the form
mymultipart.2~_XLFNqOW0NuiALg7q4-Hi_7hdtAkZUH.1)

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index e619e9681148..f536b7609e85 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1955,7 +1955,7 @@ int RadosObject::RadosReadOp::prepare(optional_yield y, const DoutPrefixProvider
   if (ret < 0)
     return ret;
 
-  source->set_key(parent_op.state.obj.key);
+  source->set_instance(parent_op.state.obj.key.instance);
   source->set_obj_size(obj_size);
 
   return ret;

From c02129eb8c31b35a8d283a397070a3ae045955f8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 3 Aug 2023 16:52:43 -0400
Subject: [PATCH 0676/2492] ReleaseNotes: document support for partNumber

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 PendingReleaseNotes | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index eecf724d4638..9acda3d64e14 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -64,6 +64,11 @@
   of POOL_APP_NOT_ENABLED health warning for that pool.
   The user might temporarily mute this warning using
   ``ceph health mute POOL_APP_NOT_ENABLED``.
+CephFS: Disallow delegating preallocated inode ranges to clients. Config
+  `mds_client_delegate_inos_pct` defaults to 0 which disables async dirops
+  in the kclient.
+* S3 Get/HeadObject now support query parameter `partNumber` to read a specific
+  part of a completed multipart upload.
 
 >=18.0.0
 

From 338440ae2c04ed6a14d35825f4503d1222266ed5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 11 Oct 2023 17:45:17 -0400
Subject: [PATCH 0677/2492] rgw: non-multipart uploads serve entire range on
 partNumber=1

and omit the x-amz-mp-parts-count response header

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc     | 22 +++++++++++++++-------
 src/rgw/driver/rados/rgw_rados.h      |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc |  2 +-
 src/rgw/rgw_op.cc                     |  3 +--
 src/rgw/rgw_sal.h                     |  6 +++---
 5 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index c5624ba0171e..3ee43dd91a89 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -6533,6 +6533,9 @@ static int get_part_obj_state(const DoutPrefixProvider* dpp, optional_yield y,
                               int part_num, int* parts_count, bool prefetch,
                               RGWObjState** pstate, RGWObjManifest** pmanifest)
 {
+  if (!manifest) {
+    return -ERR_INVALID_PART;
+  }
   // navigate to the requested part in the manifest
   RGWObjManifest::obj_iterator end = manifest->obj_end(dpp);
   if (end.get_cur_part_id() == 0) { // not multipart
@@ -6639,22 +6642,27 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *
   RGWBucketInfo& bucket_info = source->get_bucket_info();
 
   if (params.part_num) {
+    int parts_count = 0;
     // use the manifest to redirect to the requested part number
-    if (!manifest) {
-      return -ERR_INVALID_PART;
-    }
     r = get_part_obj_state(dpp, y, store, bucket_info, &source->get_ctx(),
-                           manifest, *params.part_num, params.parts_count,
+                           manifest, *params.part_num, &parts_count,
                            part_prefetch, &astate, &manifest);
-    if (r < 0) {
+    if (r == -ERR_INVALID_PART && *params.part_num == 1) {
+      // for non-multipart uploads, treat requests for the first part as a
+      // request for the entire range. this behavior is expected by the java
+      // sdk's TransferManager.download()
+      ldpp_dout(dpp, 4) << "requested part #" << *params.part_num
+          << ": " << cpp_strerror(r) << dendl;
+    } else if (r < 0) {
       ldpp_dout(dpp, 4) << "failed to read part #" << *params.part_num
           << ": " << cpp_strerror(r) << dendl;
       return -ERR_INVALID_PART;
-    }
-    if (!astate->exists) {
+    } else if (!astate->exists) {
       ldpp_dout(dpp, 4) << "part #" << *params.part_num
           << " does not exist" << dendl;
       return -ERR_INVALID_PART;
+    } else {
+      params.parts_count = parts_count;
     }
   }
 
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 89e4576a4788..f61730cb4de3 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -770,7 +770,7 @@ class RGWRados
         rgw_obj *target_obj;
 	uint64_t *epoch;
         int* part_num = nullptr;
-        int* parts_count = nullptr;
+        std::optional<int> parts_count;
 
         Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr),
 		   target_obj(nullptr), epoch(nullptr)
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index f536b7609e85..b60ddd7a4d18 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1947,7 +1947,6 @@ int RadosObject::RadosReadOp::prepare(optional_yield y, const DoutPrefixProvider
   parent_op.params.lastmod = params.lastmod;
   parent_op.params.target_obj = params.target_obj;
   parent_op.params.part_num = params.part_num;
-  parent_op.params.parts_count = params.parts_count;
   parent_op.params.obj_size = &obj_size;
   parent_op.params.attrs = &source->get_attrs();
 
@@ -1957,6 +1956,7 @@ int RadosObject::RadosReadOp::prepare(optional_yield y, const DoutPrefixProvider
 
   source->set_instance(parent_op.state.obj.key.instance);
   source->set_obj_size(obj_size);
+  params.parts_count = parent_op.params.parts_count;
 
   return ret;
 }
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 1a9cd11c6eff..ae89c703376a 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2283,8 +2283,6 @@ void RGWGetObj::execute(optional_yield y)
   read_op->params.lastmod = &lastmod;
   if (multipart_part_num) {
     read_op->params.part_num = &*multipart_part_num;
-    multipart_parts_count.emplace(0);
-    read_op->params.parts_count = &*multipart_parts_count;
   }
 
   op_ret = read_op->prepare(s->yield, this);
@@ -2293,6 +2291,7 @@ void RGWGetObj::execute(optional_yield y)
   version_id = s->object->get_instance();
   s->obj_size = s->object->get_obj_size();
   attrs = s->object->get_attrs();
+  multipart_parts_count = read_op->params.parts_count;
 
   /* STAT ops don't need data, and do no i/o */
   if (get_type() == RGW_OP_STAT_OBJ) {
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 57c7c20d6e15..f0314517bebd 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -846,9 +846,9 @@ class Object {
 
         /// If non-null, read data/attributes from the given multipart part.
         int* part_num{nullptr};
-        /// If part_num is specified, the total number of multipart parts is
-        /// written to this output parameter.
-        int* parts_count{nullptr};
+        /// If part_num is specified and the object is multipart, the total
+        /// number of multipart parts is assigned to this output parameter.
+        std::optional<int> parts_count;
       } params;
 
       virtual ~ReadOp() = default;

From 9cab99bae8f4643c949d6121c14e12a19c2608a3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 21 Nov 2023 13:44:40 -0500
Subject: [PATCH 0678/2492] crush: remove unused variables

[161/715] Building CXX object src/crush/CMakeFiles/crush_objs.dir/CrushTester.cc.o
ceph/src/crush/CrushTester.cc:478:7: warning: variable 'num_devices_active' set but not used [-Wunused-but-set-variable]
  int num_devices_active = 0;
      ^
1 warning generated.
[165/715] Building CXX object src/crush/CMakeFiles/crush_objs.dir/CrushWrapper.cc.o
ceph/src/crush/CrushWrapper.cc:1579:9: warning: variable 'local_changed' set but not used [-Wunused-but-set-variable]
    int local_changed = 0;
        ^

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/crush/CrushTester.cc  | 6 ------
 src/crush/CrushWrapper.cc | 2 --
 2 files changed, 8 deletions(-)

diff --git a/src/crush/CrushTester.cc b/src/crush/CrushTester.cc
index 4f99cffaaa9e..1d659e6a56bf 100644
--- a/src/crush/CrushTester.cc
+++ b/src/crush/CrushTester.cc
@@ -474,12 +474,6 @@ int CrushTester::test(CephContext* cct)
   // make adjustments
   adjust_weights(weight);
 
-
-  int num_devices_active = 0;
-  for (vector<__u32>::iterator p = weight.begin(); p != weight.end(); ++p)
-    if (*p > 0)
-      num_devices_active++;
-
   if (output_choose_tries)
     crush.start_choose_profile();
   
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 530c2413933d..0f40e6875e1b 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -1576,13 +1576,11 @@ int CrushWrapper::adjust_subtree_weight(CephContext *cct, int id, int weight,
   while (!q.empty()) {
     b = q.front();
     q.pop_front();
-    int local_changed = 0;
     for (unsigned i=0; i<b->size; ++i) {
       int n = b->items[i];
       if (n >= 0) {
 	adjust_item_weight_in_bucket(cct, n, weight, b->id, update_weight_sets);
 	++changed;
-	++local_changed;
       } else {
 	crush_bucket *sub = get_bucket(n);
 	if (IS_ERR(sub))

From c94166de2a00406d6e45d61562f6c53be0f7b685 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 21 Nov 2023 14:25:11 -0500
Subject: [PATCH 0679/2492] rgw: fix RGWPeriod encoding after removing
 realm_name

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_zone.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 7542c33451a1..1f837e312c09 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -778,6 +778,8 @@ class RGWPeriod
     encode(master_zonegroup, bl);
     encode(period_config, bl);
     encode(realm_id, bl);
+    std::string realm_name; // removed
+    encode(realm_name, bl);
     ENCODE_FINISH(bl);
   }
 
@@ -793,6 +795,8 @@ class RGWPeriod
     decode(master_zonegroup, bl);
     decode(period_config, bl);
     decode(realm_id, bl);
+    std::string realm_name; // removed
+    decode(realm_name, bl);
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

From 6ccb9f3ca10e814c42effb49c8441c23da911adc Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 22 Nov 2023 03:54:19 +1000
Subject: [PATCH 0680/2492] doc/rados: edit "recovering broken monmap"

Edit the section "Recovering a monitor's broken monmap" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 109 ++++++++++--------
 1 file changed, 58 insertions(+), 51 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index d2095b87286c..e00d391ebef5 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -260,14 +260,15 @@ detail`` returns a message similar to the following::
   Monitors at a wrong address. ``mon_status`` outputs the ``monmap`` that is
   known to the monitor: determine whether the other Monitors' locations as
   specified in the ``monmap`` match the locations of the Monitors in the
-  network. If they do not, see `Recovering a Monitor's Broken monmap`_.
-  If the locations of the Monitors as specified in the ``monmap`` match the
-  locations of the Monitors in the network, then the persistent
-  ``probing`` state could  be related to severe clock skews amongst the monitor
-  nodes.  See `Clock Skews`_.  If the information in `Clock Skews`_ does not
-  bring the Monitor out of the ``probing`` state, then prepare your system logs
-  and ask the Ceph community for help. See `Preparing your logs`_ for
-  information about the proper preparation of logs.
+  network. If they do not, see :ref:`Recovering a Monitor's Broken monmap
+  <rados_troubleshooting_troubleshooting_mon_recovering_broken_monmap>`. If
+  the locations of the Monitors as specified in the ``monmap`` match the
+  locations of the Monitors in the network, then the persistent ``probing``
+  state could  be related to severe clock skews among the monitor nodes.  See
+  `Clock Skews`_.  If the information in `Clock Skews`_ does not bring the
+  Monitor out of the ``probing`` state, then prepare your system logs and ask
+  the Ceph community for help. See `Preparing your logs`_ for information about
+  the proper preparation of logs.
 
 
 **What does it mean when a Monitor's state is ``electing``?**
@@ -324,13 +325,16 @@ detail`` returns a message similar to the following::
   substantiate it. See `Preparing your logs`_ for information about the
   proper preparation of logs.
 
+.. _rados_troubleshooting_troubleshooting_mon_recovering_broken_monmap:
 
-Recovering a Monitor's Broken ``monmap``
-----------------------------------------
+Recovering a Monitor's Broken "monmap"
+--------------------------------------
 
-This is how a ``monmap`` usually looks, depending on the number of
-monitors::
+A monmap can be retrieved by using a command of the form ``ceph tell mon.c
+mon_status``, as described in :ref:`Understanding mon_status
+<rados_troubleshoting_troubleshooting_mon_understanding_mon_status>`.
 
+Here is an example of a ``monmap``::
 
       epoch 3
       fsid 5c4e9d53-e2e1-478a-8061-f543f8be4cf8
@@ -339,61 +343,64 @@ monitors::
       0: 127.0.0.1:6789/0 mon.a
       1: 127.0.0.1:6790/0 mon.b
       2: 127.0.0.1:6795/0 mon.c
-      
-This may not be what you have however. For instance, in some versions of
-early Cuttlefish there was a bug that could cause your ``monmap``
-to be nullified.  Completely filled with zeros. This means that not even
-``monmaptool`` would be able to make sense of cold, hard, inscrutable zeros.
-It's also possible to end up with a monitor with a severely outdated monmap,
-notably if the node has been down for months while you fight with your vendor's
-TAC.  The subject ``ceph-mon`` daemon might be unable to find the surviving
-monitors (e.g., say ``mon.c`` is down; you add a new monitor ``mon.d``,
-then remove ``mon.a``, then add a new monitor ``mon.e`` and remove
-``mon.b``; you will end up with a totally different monmap from the one
-``mon.c`` knows).
 
-In this situation you have two possible solutions:
+This ``monmap`` is in working order, but your ``monmap`` might not be in
+working order. The ``monmap`` in a given node might be outdated because the
+node was down for a long time, during which the cluster's Monitors changed.
 
-Scrap the monitor and redeploy
+There are two ways to update a Monitor's outdated ``monmap``: 
 
-  You should only take this route if you are positive that you won't
-  lose the information kept by that monitor; that you have other monitors
-  and that they are running just fine so that your new monitor is able
-  to synchronize from the remaining monitors. Keep in mind that destroying
-  a monitor, if there are no other copies of its contents, may lead to
-  loss of data.
+A. **Scrap the monitor and redeploy.**
 
-Inject a monmap into the monitor
+    Do this only if you are certain that you will not lose the information kept
+    by the Monitor that you scrap. Make sure that you have other Monitors in
+    good condition, so that the new Monitor will be able to synchronize with
+    the surviving Monitors. Remember that destroying a Monitor can lead to data
+    loss if there are no other copies of the Monitor's contents. 
 
-  These are the basic steps:
+B. **Inject a monmap into the monitor.**
 
-  Retrieve the ``monmap`` from the surviving monitors and inject it into the
-  monitor whose ``monmap`` is corrupted or lost.
+    It is possible to fix a Monitor that has an outdated ``monmap`` by
+    retrieving an up-to-date ``monmap`` from surviving Monitors in the cluster
+    and injecting it into the Monitor that has a corrupted or missing
+    ``monmap``.
 
-  Implement this solution by carrying out the following procedure:
+    Implement this solution by carrying out the following procedure:
 
-  1. Is there a quorum of monitors? If so, retrieve the ``monmap`` from the
-     quorum::
+    #. Retrieve the ``monmap`` in one of the two following ways:
 
-      $ ceph mon getmap -o /tmp/monmap
+       a. **IF THERE IS A QUORUM OF MONITORS:** 
+       
+          Retrieve the ``monmap`` from the quorum:
 
-  2. If there is no quorum, then retrieve the ``monmap`` directly from another
-     monitor that has been stopped (in this example, the other monitor has
-     the ID ``ID-FOO``)::
+             .. prompt:: bash
 
-      $ ceph-mon -i ID-FOO --extract-monmap /tmp/monmap
+                ceph mon getmap -o /tmp/monmap
 
-  3. Stop the monitor you are going to inject the monmap into.
+       b. **IF THERE IS NO QUORUM OF MONITORS:** 
+       
+          Retrieve the ``monmap`` directly from a Monitor that has been stopped
+          :
 
-  4. Inject the monmap::
+             .. prompt:: bash
 
-      $ ceph-mon -i ID --inject-monmap /tmp/monmap
+                ceph-mon -i ID-FOO --extract-monmap /tmp/monmap
 
-  5. Start the monitor
+          In this example, the ID of the stopped Monitor is ``ID-FOO``.
 
-  .. warning:: Injecting ``monmaps`` can cause serious problems because doing
-     so will overwrite the latest existing ``monmap`` stored on the monitor. Be
-     careful!
+    #. Stop the Monitor into which the ``monmap`` will be injected. 
+
+    #. Inject the monmap into the stopped Monitor:
+
+       .. prompt:: bash
+
+          ceph-mon -i ID --inject-monmap /tmp/monmap
+
+    #. Start the Monitor.
+
+       .. warning:: Injecting a ``monmap`` into a Monitor  can cause serious
+          problems. Injecting a ``monmap`` overwrites the latest existing
+          ``monmap`` stored on the monitor.  Be careful!
 
 Clock Skews
 -----------

From e93e70622599614011c307aca7c1c70d8c49a937 Mon Sep 17 00:00:00 2001
From: Dillon Amburgey <dillona@dillona.com>
Date: Wed, 22 Nov 2023 03:02:37 +0000
Subject: [PATCH 0681/2492] ceph-volume: Use safe accessor to get TYPE info

Fixes: https://tracker.ceph.com/issues/63502
Signed-off-by: Dillon Amburgey <dillona@dillona.com>
---
 src/ceph-volume/ceph_volume/util/device.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index bb806292f2c4..c3de2a97fd99 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -460,11 +460,11 @@ def is_mapper(self):
     def device_type(self):
         self.load_blkid_api()
         if 'type' in self.sys_api:
-            return self.sys_api['type']
+            return self.sys_api.get('type')
         elif self.disk_api:
-            return self.disk_api['TYPE']
+            return self.disk_api.get('TYPE')
         elif self.blkid_api:
-            return self.blkid_api['TYPE']
+            return self.blkid_api.get('TYPE')
 
     @property
     def is_mpath(self):
@@ -478,9 +478,9 @@ def is_lv(self):
     def is_partition(self):
         self.load_blkid_api()
         if self.disk_api:
-            return self.disk_api['TYPE'] == 'part'
+            return self.disk_api.get('TYPE') == 'part'
         elif self.blkid_api:
-            return self.blkid_api['TYPE'] == 'part'
+            return self.blkid_api.get('TYPE') == 'part'
         return False
 
     @property

From 38cc750f1fbb06146573e639778835fdb56f0b33 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 6 Sep 2023 11:15:31 +0800
Subject: [PATCH 0682/2492] crimson/osd/osd_operations/client_request: recover
 the head and other necessary objects before proceeding

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/object_context_loader.cc      | 16 +++----
 src/crimson/osd/object_context_loader.h       |  5 +-
 .../osd/osd_operations/client_request.cc      | 17 ++-----
 .../osd/osd_operations/client_request.h       | 10 ++++
 .../osd_operations/client_request_common.cc   | 46 +++++++++++++++++++
 .../osd_operations/client_request_common.h    |  7 +++
 .../osd/osd_operations/common/pg_pipeline.h   |  2 +-
 .../osd_operations/internal_client_request.cc |  2 +-
 src/crimson/osd/pg.cc                         |  4 +-
 src/crimson/osd/pg.h                          |  2 +-
 src/crimson/osd/pg_backend.cc                 |  2 +-
 .../osd/replicated_recovery_backend.cc        |  4 +-
 12 files changed, 84 insertions(+), 33 deletions(-)

diff --git a/src/crimson/osd/object_context_loader.cc b/src/crimson/osd/object_context_loader.cc
index 0a4d74c0d70c..d7d2b6d98c67 100644
--- a/src/crimson/osd/object_context_loader.cc
+++ b/src/crimson/osd/object_context_loader.cc
@@ -22,7 +22,7 @@ using crimson::common::local_conf;
       return get_or_load_obc<State>(obc, existed)
       .safe_then_interruptible(
         [func = std::move(func)](auto obc) {
-        return std::move(func)(std::move(obc));
+        return std::move(func)(obc, obc);
       });
     }).finally([FNAME, this, obc=std::move(obc)] {
       DEBUGDPP("released object {}", dpp, obc->get_oid());
@@ -39,7 +39,7 @@ using crimson::common::local_conf;
     assert(!oid.is_head());
     return with_obc<RWState::RWREAD>(
       oid.get_head(),
-      [FNAME, oid, func=std::move(func), this](auto head) mutable
+      [FNAME, oid, func=std::move(func), this](auto head, auto) mutable
       -> load_obc_iertr::future<> {
       if (!head->obs.exists) {
 	ERRORDPP("head doesn't exist for object {}", dpp, head->obs.oi.soid);
@@ -70,12 +70,12 @@ using crimson::common::local_conf;
     auto [clone, existed] = obc_registry.get_cached_obc(*coid);
     return clone->template with_lock<State, IOInterruptCondition>(
       [existed=existed, clone=std::move(clone),
-       func=std::move(func), head=std::move(head), this]()
+       func=std::move(func), head=std::move(head), this]() mutable
       -> load_obc_iertr::future<> {
       auto loaded = get_or_load_obc<State>(clone, existed);
       return loaded.safe_then_interruptible(
-        [func = std::move(func)](auto clone) {
-        return std::move(func)(std::move(clone));
+        [func = std::move(func), head=std::move(head)](auto clone) mutable {
+        return std::move(func)(std::move(head), std::move(clone));
       });
     });
   }
@@ -84,13 +84,13 @@ using crimson::common::local_conf;
   ObjectContextLoader::load_obc_iertr::future<>
   ObjectContextLoader::with_clone_obc_direct(
     hobject_t oid,
-    with_both_obc_func_t&& func)
+    with_obc_func_t&& func)
   {
     LOG_PREFIX(ObjectContextLoader::with_clone_obc_direct);
     assert(!oid.is_head());
     return with_obc<RWState::RWREAD>(
       oid.get_head(),
-      [FNAME, oid, func=std::move(func), this](auto head) mutable
+      [FNAME, oid, func=std::move(func), this](auto head, auto) mutable
       -> load_obc_iertr::future<> {
       if (!head->obs.exists) {
         ERRORDPP("head doesn't exist for object {}", dpp, head->obs.oi.soid);
@@ -228,5 +228,5 @@ using crimson::common::local_conf;
   template ObjectContextLoader::load_obc_iertr::future<>
   ObjectContextLoader::with_clone_obc_direct<RWState::RWWRITE>(
     hobject_t,
-    with_both_obc_func_t&&);
+    with_obc_func_t&&);
 }
diff --git a/src/crimson/osd/object_context_loader.h b/src/crimson/osd/object_context_loader.h
index 3ab7f6ad80fd..0cd50623abc2 100644
--- a/src/crimson/osd/object_context_loader.h
+++ b/src/crimson/osd/object_context_loader.h
@@ -30,9 +30,6 @@ class ObjectContextLoader {
       load_obc_ertr>;
 
   using with_obc_func_t =
-    std::function<load_obc_iertr::future<> (ObjectContextRef)>;
-
-  using with_both_obc_func_t =
     std::function<load_obc_iertr::future<> (ObjectContextRef, ObjectContextRef)>;
 
   // Use this variant by default
@@ -55,7 +52,7 @@ class ObjectContextLoader {
   template<RWState::State State>
   load_obc_iertr::future<> with_clone_obc_direct(
     hobject_t oid,
-    with_both_obc_func_t&& func);
+    with_obc_func_t&& func);
 
   load_obc_iertr::future<> reload_obc(ObjectContext& obc) const;
 
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index d208e2e53d97..ac441185cdac 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -225,18 +225,9 @@ ClientRequest::interruptible_future<>
 ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
 {
   return ihref.enter_stage<interruptor>(
-    client_pp(*pg).recover_missing,
-    *this
-  ).then_interruptible(
-    [this, pg]() mutable {
-    LOG_PREFIX(ClientRequest::process_op);
-    if (pg->is_primary()) {
-      return do_recover_missing(pg, m->get_hobj());
-    } else {
-      DEBUGI("process_op: Skipping do_recover_missing"
-                     "on non primary pg");
-      return interruptor::now();
-    }
+    client_pp(*pg).recover_missing, *this
+  ).then_interruptible([pg, this]() mutable {
+    return recover_missings(pg, m->get_hobj(), snaps_need_to_recover());
   }).then_interruptible([this, pg, &ihref]() mutable {
     return pg->already_complete(m->get_reqid()).then_interruptible(
       [this, pg, &ihref](auto completed) mutable
@@ -256,7 +247,7 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
           op_info.set_from_op(&*m, *pg->get_osdmap());
           return pg->with_locked_obc(
             m->get_hobj(), op_info,
-            [this, pg, &ihref](auto obc) mutable {
+            [this, pg, &ihref](auto head, auto obc) mutable {
               LOG_PREFIX(ClientRequest::process_op);
               DEBUGI("{}: got obc {}", *this, obc->obs);
               return ihref.enter_stage<interruptor>(
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 3c8c146a4f2f..43535e156c5b 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -160,6 +160,16 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
   }
   auto get_instance_handle() { return instance_handle; }
 
+  std::vector<snapid_t> snaps_need_to_recover() {
+    std::vector<snapid_t> ret;
+    for (auto &op : m->ops) {
+      if (op.op.op == CEPH_OSD_OP_ROLLBACK) {
+	ret.emplace_back((snapid_t)op.op.snap.snapid);
+      }
+    }
+    return ret;
+  }
+
   using ordering_hook_t = boost::intrusive::list_member_hook<>;
   ordering_hook_t ordering_hook;
   class Orderer {
diff --git a/src/crimson/osd/osd_operations/client_request_common.cc b/src/crimson/osd/osd_operations/client_request_common.cc
index cfd22c774e06..903da59a3dce 100644
--- a/src/crimson/osd/osd_operations/client_request_common.cc
+++ b/src/crimson/osd/osd_operations/client_request_common.cc
@@ -11,8 +11,54 @@ namespace {
   }
 }
 
+SET_SUBSYS(osd);
+
 namespace crimson::osd {
 
+InterruptibleOperation::template interruptible_future<>
+CommonClientRequest::recover_missings(
+  Ref<PG> &pg,
+  const hobject_t& soid,
+  std::vector<snapid_t> &&snaps)
+{
+  using interruptor = InterruptibleOperation::interruptor;
+  LOG_PREFIX(CommonClientRequest::recover_missings);
+  auto fut = interruptor::now();
+  if (!pg->is_primary()) {
+    DEBUGI("process_op: Skipping do_recover_missing on non primary pg");
+    return fut;
+  }
+  if (!soid.is_head()) {
+    fut = do_recover_missing(pg, soid.get_head());
+  }
+  return seastar::do_with(
+    std::move(snaps),
+    [pg, soid, fut=std::move(fut)](auto &snaps) mutable {
+    return fut.then_interruptible([&snaps, pg, soid]() mutable {
+      return pg->obc_loader.with_obc<RWState::RWREAD>(
+        soid.get_head(),
+        [&snaps, pg, soid](auto head, auto) mutable {
+        auto oid = resolve_oid(head->get_head_ss(), soid);
+        assert(oid);
+        return do_recover_missing(pg, *oid
+        ).then_interruptible([&snaps, pg, soid, head]() mutable {
+          return InterruptibleOperation::interruptor::do_for_each(
+            snaps,
+            [pg, soid, head](auto &snap) mutable {
+            auto coid = head->obs.oi.soid;
+            coid.snap = snap;
+            auto oid = resolve_oid(head->get_head_ss(), coid);
+            assert(oid);
+            return do_recover_missing(pg, *oid);
+          });
+        });
+      });
+    }).handle_error_interruptible(
+      crimson::ct_error::assert_all("unexpected error")
+    );
+  });
+}
+
 typename InterruptibleOperation::template interruptible_future<>
 CommonClientRequest::do_recover_missing(
   Ref<PG>& pg, const hobject_t& soid)
diff --git a/src/crimson/osd/osd_operations/client_request_common.h b/src/crimson/osd/osd_operations/client_request_common.h
index 6a8a789668c1..46aa038e3432 100644
--- a/src/crimson/osd/osd_operations/client_request_common.h
+++ b/src/crimson/osd/osd_operations/client_request_common.h
@@ -10,6 +10,13 @@
 namespace crimson::osd {
 
 struct CommonClientRequest {
+
+  static InterruptibleOperation::template interruptible_future<>
+  recover_missings(
+    Ref<PG> &pg,
+    const hobject_t& soid,
+    std::vector<snapid_t> &&snaps);
+
   static InterruptibleOperation::template interruptible_future<>
   do_recover_missing(Ref<PG>& pg, const hobject_t& soid);
 
diff --git a/src/crimson/osd/osd_operations/common/pg_pipeline.h b/src/crimson/osd/osd_operations/common/pg_pipeline.h
index 58fa07b8b4d2..1e5d0e511f67 100644
--- a/src/crimson/osd/osd_operations/common/pg_pipeline.h
+++ b/src/crimson/osd/osd_operations/common/pg_pipeline.h
@@ -19,7 +19,7 @@ class CommonPGPipeline {
   } wait_for_active;
   struct RecoverMissing : OrderedExclusivePhaseT<RecoverMissing> {
     static constexpr auto type_name = "CommonPGPipeline::recover_missing";
-  } recover_missing;
+  } recover_missing, recover_missing2;
   struct GetOBC : OrderedExclusivePhaseT<GetOBC> {
     static constexpr auto type_name = "CommonPGPipeline::get_obc";
   } get_obc;
diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc
index c2a371d274f6..bfa09fbe97a2 100644
--- a/src/crimson/osd/osd_operations/internal_client_request.cc
+++ b/src/crimson/osd/osd_operations/internal_client_request.cc
@@ -85,7 +85,7 @@ seastar::future<> InternalClientRequest::start()
               std::as_const(osd_ops), pg->get_pgid().pgid, *pg->get_osdmap());
             assert(ret == 0);
             return pg->with_locked_obc(get_target_oid(), op_info,
-              [&osd_ops, this](auto obc) {
+              [&osd_ops, this](auto, auto obc) {
               return enter_stage<interruptor>(client_pp().process
               ).then_interruptible(
                 [obc=std::move(obc), &osd_ops, this] {
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index c4dce8a8ee30..f2e01fe10341 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1209,9 +1209,9 @@ PG::with_locked_obc(const hobject_t &hobj,
     throw crimson::common::system_shutdown_exception();
   }
   const hobject_t oid = get_oid(hobj);
-  auto wrapper = [f=std::move(f), this](auto obc) {
+  auto wrapper = [f=std::move(f), this](auto head, auto obc) {
     check_blocklisted_obc_watchers(obc);
-    return f(obc);
+    return f(head, obc);
   };
   switch (get_lock_type(op_info)) {
   case RWState::RWREAD:
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 6a0231e452f6..f2b6bb611e42 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -504,7 +504,7 @@ class PG : public boost::intrusive_ref_counter<
 
 public:
   using with_obc_func_t =
-    std::function<load_obc_iertr::future<> (ObjectContextRef)>;
+    std::function<load_obc_iertr::future<> (ObjectContextRef, ObjectContextRef)>;
 
   load_obc_iertr::future<> with_locked_obc(
     const hobject_t &hobj,
diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index 02acb9a55d3f..5e3e7a91c0f7 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -787,7 +787,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
   return obc_loader.with_clone_obc_only<RWState::RWWRITE>(
     head, target_coid,
     [this, &os, &txn, &delta_stats, &osd_op_params]
-    (auto resolved_obc) {
+    (auto, auto resolved_obc) {
     if (resolved_obc->obs.oi.soid.is_head()) {
       // no-op: The resolved oid returned the head object
       logger().debug("PGBackend::rollback: loaded head_obc: {}"
diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index bd301cc2b672..8aaffef104d5 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -34,7 +34,7 @@ ReplicatedRecoveryBackend::recover_object(
   return maybe_pull_missing_obj(soid, need).then_interruptible([this, soid, need] {
     logger().debug("recover_object: loading obc: {}", soid);
     return pg.obc_loader.with_obc<RWState::RWREAD>(soid,
-      [this, soid, need](auto obc) {
+      [this, soid, need](auto, auto obc) {
       logger().debug("recover_object: loaded obc: {}", obc->obs.oi.soid);
       auto& recovery_waiter = get_recovering(soid);
       recovery_waiter.obc = obc;
@@ -689,7 +689,7 @@ ReplicatedRecoveryBackend::_handle_pull_response(
   if (pull_info.recovery_progress.first) {
     prepare_waiter = pg.obc_loader.with_obc<RWState::RWNONE>(
       pull_info.recovery_info.soid,
-      [&pull_info, &recovery_waiter, &push_op](auto obc) {
+      [&pull_info, &recovery_waiter, &push_op](auto, auto obc) {
         pull_info.obc = obc;
         recovery_waiter.obc = obc;
         obc->obs.oi.decode_no_oid(push_op.attrset.at(OI_ATTR), push_op.soid);

From cd5259b5d6a3b3b669fd10689ac44f709c18b309 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 12 Sep 2023 20:37:56 +0800
Subject: [PATCH 0683/2492] crimson/osd/pg_recovery: add the oids of the
 recovered clones to the snapmapper on local recovery finished

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg.h                          |  6 ++
 src/crimson/osd/pg_recovery.cc                | 45 ++++++++++----
 src/crimson/osd/pg_recovery.h                 |  2 +-
 src/crimson/osd/pg_recovery_listener.h        |  2 +
 .../osd/replicated_recovery_backend.cc        | 62 ++++++++++++-------
 5 files changed, 79 insertions(+), 38 deletions(-)

diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index f2b6bb611e42..6e94cf6bbc79 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -644,6 +644,12 @@ class PG : public boost::intrusive_ref_counter<
   std::optional<pg_stat_t> pg_stats;
 
 public:
+  OSDriver &get_osdriver() final {
+    return osdriver;
+  }
+  SnapMapper &get_snap_mapper() final {
+    return snap_mapper;
+  }
   RecoveryBackend* get_recovery_backend() final {
     return recovery_backend.get();
   }
diff --git a/src/crimson/osd/pg_recovery.cc b/src/crimson/osd/pg_recovery.cc
index 09b45779ec87..e96344f207d8 100644
--- a/src/crimson/osd/pg_recovery.cc
+++ b/src/crimson/osd/pg_recovery.cc
@@ -316,7 +316,8 @@ RecoveryBackend::interruptible_future<> PGRecovery::prep_object_replica_pushes(
   );
 }
 
-void PGRecovery::on_local_recover(
+RecoveryBackend::interruptible_future<>
+PGRecovery::on_local_recover(
   const hobject_t& soid,
   const ObjectRecoveryInfo& recovery_info,
   const bool is_delete,
@@ -332,20 +333,38 @@ void PGRecovery::on_local_recover(
       ceph_abort("mark_unfound_lost (LOST_REVERT) is not implemented yet");
     }
   }
-  pg->get_peering_state().recover_got(soid,
-      recovery_info.version, is_delete, t);
-
-  if (pg->is_primary()) {
-    if (!is_delete) {
-      auto& obc = pg->get_recovery_backend()->get_recovering(soid).obc; //TODO: move to pg backend?
-      obc->obs.exists = true;
-      obc->obs.oi = recovery_info.oi;
+
+  return RecoveryBackend::interruptor::async(
+    [soid, &recovery_info, is_delete, &t, this] {
+    if (soid.is_snap()) {
+      OSDriver::OSTransaction _t(pg->get_osdriver().get_transaction(&t));
+      int r = pg->get_snap_mapper().remove_oid(soid, &_t);
+      assert(r == 0 || r == -ENOENT);
+
+      if (!is_delete) {
+	set<snapid_t> snaps;
+	auto p = recovery_info.ss.clone_snaps.find(soid.snap);
+	assert(p != recovery_info.ss.clone_snaps.end());
+	snaps.insert(p->second.begin(), p->second.end());
+	pg->get_snap_mapper().add_oid(recovery_info.soid, snaps, &_t);
+      }
     }
-    if (!pg->is_unreadable_object(soid)) {
-      pg->get_recovery_backend()->get_recovering(soid).set_readable();
+
+    pg->get_peering_state().recover_got(soid,
+	recovery_info.version, is_delete, t);
+
+    if (pg->is_primary()) {
+      if (!is_delete) {
+	auto& obc = pg->get_recovery_backend()->get_recovering(soid).obc; //TODO: move to pg backend?
+	obc->obs.exists = true;
+	obc->obs.oi = recovery_info.oi;
+      }
+      if (!pg->is_unreadable_object(soid)) {
+	pg->get_recovery_backend()->get_recovering(soid).set_readable();
+      }
+      pg->publish_stats_to_osd();
     }
-    pg->publish_stats_to_osd();
-  }
+  });
 }
 
 void PGRecovery::on_global_recover (
diff --git a/src/crimson/osd/pg_recovery.h b/src/crimson/osd/pg_recovery.h
index 719d0ad2d34c..ad2240738579 100644
--- a/src/crimson/osd/pg_recovery.h
+++ b/src/crimson/osd/pg_recovery.h
@@ -64,7 +64,7 @@ class PGRecovery : public crimson::osd::BackfillState::BackfillListener {
     const hobject_t& soid,
     eversion_t need);
 
-  void on_local_recover(
+  RecoveryBackend::interruptible_future<> on_local_recover(
     const hobject_t& soid,
     const ObjectRecoveryInfo& recovery_info,
     bool is_delete,
diff --git a/src/crimson/osd/pg_recovery_listener.h b/src/crimson/osd/pg_recovery_listener.h
index c922b99561a3..a53221c4a69b 100644
--- a/src/crimson/osd/pg_recovery_listener.h
+++ b/src/crimson/osd/pg_recovery_listener.h
@@ -36,4 +36,6 @@ class PGRecoveryListener {
   virtual const pg_info_t& get_info() const= 0;
   virtual seastar::future<> stop() = 0;
   virtual void publish_stats_to_osd() = 0;
+  virtual OSDriver &get_osdriver() = 0;
+  virtual SnapMapper &get_snap_mapper() = 0;
 };
diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 8aaffef104d5..5559304efd23 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -201,15 +201,19 @@ ReplicatedRecoveryBackend::on_local_recover_persist(
   epoch_t epoch_frozen)
 {
   logger().debug("{}", __func__);
-  ceph::os::Transaction t;
-  pg.get_recovery_handler()->on_local_recover(soid, _recovery_info, is_delete, t);
-  logger().debug("ReplicatedRecoveryBackend::on_local_recover_persist: do_transaction...");
-  return interruptor::make_interruptible(
-      shard_services.get_store().do_transaction(coll, std::move(t)))
-  .then_interruptible(
-    [this, epoch_frozen, last_complete = pg.get_info().last_complete] {
-    pg.get_recovery_handler()->_committed_pushed_object(epoch_frozen, last_complete);
-    return seastar::make_ready_future<>();
+  return seastar::do_with(
+    ceph::os::Transaction(),
+    [this, soid, &_recovery_info, is_delete, epoch_frozen](auto &t) {
+    return pg.get_recovery_handler()->on_local_recover(
+      soid, _recovery_info, is_delete, t
+    ).then_interruptible([this, &t] {
+      logger().debug("ReplicatedRecoveryBackend::{}: do_transaction...", __func__);
+      return shard_services.get_store().do_transaction(coll, std::move(t));
+    }).then_interruptible(
+      [this, epoch_frozen, last_complete = pg.get_info().last_complete] {
+      pg.get_recovery_handler()->_committed_pushed_object(epoch_frozen, last_complete);
+      return seastar::make_ready_future<>();
+    });
   });
 }
 
@@ -235,18 +239,24 @@ ReplicatedRecoveryBackend::local_recover_delete(
     }
     return seastar::make_ready_future<>();
   }).safe_then_interruptible([this, soid, epoch_to_freeze, need] {
-    ObjectRecoveryInfo recovery_info;
-    recovery_info.soid = soid;
-    recovery_info.version = need;
-    return on_local_recover_persist(soid, recovery_info,
-	                            true, epoch_to_freeze);
-  }, PGBackend::load_metadata_ertr::all_same_way(
-      [this, soid, epoch_to_freeze, need] (auto e) {
-      ObjectRecoveryInfo recovery_info;
+    return seastar::do_with(
+      ObjectRecoveryInfo(),
+      [soid, need, this, epoch_to_freeze](auto &recovery_info) {
       recovery_info.soid = soid;
       recovery_info.version = need;
       return on_local_recover_persist(soid, recovery_info,
-				      true, epoch_to_freeze);
+                                      true, epoch_to_freeze);
+    });
+  }, PGBackend::load_metadata_ertr::all_same_way(
+      [this, soid, epoch_to_freeze, need] (auto e) {
+      return seastar::do_with(
+        ObjectRecoveryInfo(),
+        [soid, need, this, epoch_to_freeze](auto &recovery_info) {
+        recovery_info.soid = soid;
+        recovery_info.version = need;
+        return on_local_recover_persist(soid, recovery_info,
+                                        true, epoch_to_freeze);
+      });
     })
   );
 }
@@ -724,21 +734,24 @@ ReplicatedRecoveryBackend::_handle_pull_response(
                             push_op.attrset, std::move(push_op.omap_entries), t)
     .then_interruptible(
       [this, response, &pull_info, &push_op, complete,
-        t, bytes_recovered=data.length()] {
+        t, bytes_recovered=data.length()]()
+      -> RecoveryBackend::interruptible_future<bool> {
       pull_info.stat.num_keys_recovered += push_op.omap_entries.size();
       pull_info.stat.num_bytes_recovered += bytes_recovered;
 
       if (complete) {
 	pull_info.stat.num_objects_recovered++;
-	pg.get_recovery_handler()->on_local_recover(
+	return pg.get_recovery_handler()->on_local_recover(
 	    push_op.soid, get_recovering(push_op.soid).pull_info->recovery_info,
-	    false, *t);
-	return true;
+	    false, *t
+        ).then_interruptible([] {
+          return true;
+        });
       } else {
         response->soid = push_op.soid;
         response->recovery_info = pull_info.recovery_info;
         response->recovery_progress = pull_info.recovery_progress;
-        return false;
+        return seastar::make_ready_future<bool>(false);
       }
     });
   });
@@ -834,10 +847,11 @@ ReplicatedRecoveryBackend::_handle_push(
   .then_interruptible(
     [this, complete, &push_op, t] {
     if (complete) {
-      pg.get_recovery_handler()->on_local_recover(
+      return pg.get_recovery_handler()->on_local_recover(
         push_op.recovery_info.soid, push_op.recovery_info,
         false, *t);
     }
+    return RecoveryBackend::interruptor::now();
   });
 }
 

From dc91e78df4cda39b09b80f0cdf7de4f7571a9365 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 12 Sep 2023 21:26:06 +0800
Subject: [PATCH 0684/2492] crimson/osd/replicated_recovery_backend: add heads'
 snapsets to recovery infos of objects that are being recovered

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../osd/replicated_recovery_backend.cc        | 63 ++++++++++++-------
 src/crimson/osd/replicated_recovery_backend.h |  3 +
 2 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 5559304efd23..803d067fd653 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -34,12 +34,12 @@ ReplicatedRecoveryBackend::recover_object(
   return maybe_pull_missing_obj(soid, need).then_interruptible([this, soid, need] {
     logger().debug("recover_object: loading obc: {}", soid);
     return pg.obc_loader.with_obc<RWState::RWREAD>(soid,
-      [this, soid, need](auto, auto obc) {
+      [this, soid, need](auto head, auto obc) {
       logger().debug("recover_object: loaded obc: {}", obc->obs.oi.soid);
       auto& recovery_waiter = get_recovering(soid);
       recovery_waiter.obc = obc;
       recovery_waiter.obc->wait_recovery_read();
-      return maybe_push_shards(soid, need);
+      return maybe_push_shards(head, soid, need);
     }).handle_error_interruptible(
       crimson::osd::PG::load_obc_ertr::all_same_way([soid](auto& code) {
       // TODO: may need eio handling?
@@ -51,16 +51,18 @@ ReplicatedRecoveryBackend::recover_object(
 
 RecoveryBackend::interruptible_future<>
 ReplicatedRecoveryBackend::maybe_push_shards(
+  const crimson::osd::ObjectContextRef &head_obc,
   const hobject_t& soid,
   eversion_t need)
 {
   return seastar::do_with(
     get_shards_to_push(soid),
-    [this, need, soid](auto &shards) {
+    [this, need, soid, head_obc](auto &shards) {
     return interruptor::parallel_for_each(
       shards,
-      [this, need, soid](auto shard) {
-      return prep_push(soid, need, shard).then_interruptible([this, soid, shard](auto push) {
+      [this, need, soid, head_obc](auto shard) {
+      return prep_push(head_obc, soid, need, shard
+      ).then_interruptible([this, soid, shard](auto push) {
         auto msg = crimson::make_message<MOSDPGPush>();
         msg->from = pg.get_pg_whoami();
         msg->pgid = pg.get_pgid();
@@ -113,27 +115,31 @@ ReplicatedRecoveryBackend::maybe_pull_missing_obj(
   if (!local_missing.is_missing(soid)) {
     return seastar::make_ready_future<>();
   }
-  PullOp pull_op;
-  auto& recovery_waiter = get_recovering(soid);
-  recovery_waiter.pull_info =
-    std::make_optional<RecoveryBackend::pull_info_t>();
-  auto& pull_info = *recovery_waiter.pull_info;
-  prepare_pull(pull_op, pull_info, soid, need);
-  auto msg = crimson::make_message<MOSDPGPull>();
-  msg->from = pg.get_pg_whoami();
-  msg->set_priority(pg.get_recovery_op_priority());
-  msg->pgid = pg.get_pgid();
-  msg->map_epoch = pg.get_osdmap_epoch();
-  msg->min_epoch = pg.get_last_peering_reset();
-  msg->set_pulls({std::move(pull_op)});
-  return interruptor::make_interruptible(
-    shard_services.send_to_osd(
+  return pg.obc_loader.with_obc<RWState::RWREAD>(soid.get_head(),
+    [this, soid, need](auto head, auto) {
+    PullOp pull_op;
+    auto& recovery_waiter = get_recovering(soid);
+    recovery_waiter.pull_info =
+      std::make_optional<RecoveryBackend::pull_info_t>();
+    auto& pull_info = *recovery_waiter.pull_info;
+    prepare_pull(head, pull_op, pull_info, soid, need);
+    auto msg = crimson::make_message<MOSDPGPull>();
+    msg->from = pg.get_pg_whoami();
+    msg->set_priority(pg.get_recovery_op_priority());
+    msg->pgid = pg.get_pgid();
+    msg->map_epoch = pg.get_osdmap_epoch();
+    msg->min_epoch = pg.get_last_peering_reset();
+    msg->set_pulls({std::move(pull_op)});
+    return shard_services.send_to_osd(
       pull_info.from.osd,
       std::move(msg),
-      pg.get_osdmap_epoch()
-  )).then_interruptible([&recovery_waiter] {
+      pg.get_osdmap_epoch());
+  }).si_then([this, soid] {
+    auto& recovery_waiter = get_recovering(soid);
     return recovery_waiter.wait_for_pull();
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::assert_all("unexpected error")
+  );
 }
 
 RecoveryBackend::interruptible_future<>
@@ -303,6 +309,7 @@ ReplicatedRecoveryBackend::recover_delete(
 
 RecoveryBackend::interruptible_future<PushOp>
 ReplicatedRecoveryBackend::prep_push(
+  const crimson::osd::ObjectContextRef &head_obc,
   const hobject_t& soid,
   eversion_t need,
   pg_shard_t pg_shard)
@@ -333,6 +340,8 @@ ReplicatedRecoveryBackend::prep_push(
   push_info.recovery_info.copy_subset = data_subset;
   push_info.recovery_info.soid = soid;
   push_info.recovery_info.oi = obc->obs.oi;
+  assert(head_obc->ssc);
+  push_info.recovery_info.ss = head_obc->ssc->snapset;
   push_info.recovery_info.version = obc->obs.oi.version;
   push_info.recovery_info.object_exist =
     missing_iter->second.clean_regions.object_is_exist();
@@ -350,7 +359,9 @@ ReplicatedRecoveryBackend::prep_push(
   });
 }
 
-void ReplicatedRecoveryBackend::prepare_pull(PullOp& pull_op,
+void ReplicatedRecoveryBackend::prepare_pull(
+  const crimson::osd::ObjectContextRef &head_obc,
+  PullOp& pull_op,
   pull_info_t& pull_info,
   const hobject_t& soid,
   eversion_t need) {
@@ -365,6 +376,10 @@ void ReplicatedRecoveryBackend::prepare_pull(PullOp& pull_op,
   pull_op.recovery_info.copy_subset.insert(0, (uint64_t) -1);
   pull_op.recovery_info.copy_subset.intersection_of(
     missing_iter->second.clean_regions.get_dirty_regions());
+  if (soid.is_snap()) {
+    assert(head_obc->ssc);
+    pull_op.recovery_info.ss = head_obc->ssc->snapset;
+  }
   pull_op.recovery_info.size = ((uint64_t) -1);
   pull_op.recovery_info.object_exist =
     missing_iter->second.clean_regions.object_is_exist();
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index b023b7417e5f..16d6369a91f3 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -49,10 +49,12 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
   interruptible_future<> handle_recovery_delete_reply(
     Ref<MOSDPGRecoveryDeleteReply> m);
   interruptible_future<PushOp> prep_push(
+    const crimson::osd::ObjectContextRef &head_obc,
     const hobject_t& soid,
     eversion_t need,
     pg_shard_t pg_shard);
   void prepare_pull(
+    const crimson::osd::ObjectContextRef &head_obc,
     PullOp& pull_op,
     pull_info_t& pull_info,
     const hobject_t& soid,
@@ -124,6 +126,7 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
       load_obc_ertr>;
 
   interruptible_future<> maybe_push_shards(
+    const crimson::osd::ObjectContextRef &head_obc,
     const hobject_t& soid,
     eversion_t need);
 

From c556c7324763332df62f4d382da02314900f1a60 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 6 Nov 2023 19:54:20 +0530
Subject: [PATCH 0685/2492] qa: refactor client upgrade yamls and other minor
 touchups

* start testing new_ops and stress_tests with both the drivers(i.e. fuse and kclient)
therefore moved 0-clients/ from tasks/3-workload/new_ops/ to tasks/ and renamed it to
2-clients/

* since new_ops/ and stress_tests/ now share the common upgrade yaml, moved the
tests yamls(in stress_tests/1-tests) directly under 3-workload/stress_tests/

* renamed 1-client-sanity.yaml in new_ops/ to newops.yaml

Fixes: https://tracker.ceph.com/issues/62953
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 .../fuse-upgrade.yaml}                        |  1 +
 .../0-clients => 2-clients}/kclient.yaml      |  0
 .../upgraded_client/tasks/2-upgrade.yaml      | 63 -------------------
 .../new_ops/0-clients/fuse-upgrade.yaml       | 15 -----
 .../{1-client-sanity.yaml => newops.yaml}     |  0
 .../tasks/3-workload/stress_tests/%           |  0
 .../tasks/3-workload/stress_tests/1-tests/.qa |  1 -
 .../stress_tests/{1-tests => }/blogbench.yaml |  0
 .../stress_tests/{1-tests => }/dbench.yaml    |  0
 .../stress_tests/{1-tests => }/fsstress.yaml  |  0
 .../stress_tests/{1-tests => }/iozone.yaml    |  0
 .../{1-tests => }/kernel_untar_build.yaml     |  0
 12 files changed, 1 insertion(+), 79 deletions(-)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{3-workload/stress_tests/0-client-upgrade.yaml => 2-clients/fuse-upgrade.yaml} (97%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/{3-workload/new_ops/0-clients => 2-clients}/kclient.yaml (100%)
 delete mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
 delete mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/{1-client-sanity.yaml => newops.yaml} (100%)
 delete mode 100644 qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/%
 delete mode 120000 qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/{1-tests => }/blogbench.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/{1-tests => }/dbench.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/{1-tests => }/fsstress.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/{1-tests => }/iozone.yaml (100%)
 rename qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/{1-tests => }/kernel_untar_build.yaml (100%)

diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/2-clients/fuse-upgrade.yaml
similarity index 97%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/2-clients/fuse-upgrade.yaml
index d91a2b90790c..cb8ac4d4f2d1 100644
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/0-client-upgrade.yaml
+++ b/qa/suites/fs/upgrade/upgraded_client/tasks/2-clients/fuse-upgrade.yaml
@@ -19,6 +19,7 @@ tasks:
     client.0:
       branch: []
 - print: "**** done install.upgrade on client.0"
+- ceph.healthy:
 - ceph-fuse:
     client.0:
 - print: "**** done remount client.0"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/kclient.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/2-clients/kclient.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/kclient.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/2-clients/kclient.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
deleted file mode 100644
index 1ff967d4da6b..000000000000
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/2-upgrade.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-teuthology:
-  premerge: |
-            local branch = yaml.teuthology.branch
-            local upgrade_branch = ""
-            local osd_release = ""
-            if tostring(branch) == "nautilus" then
-              upgrade_branch = "pacific"
-              osd_release = "ceph osd require-osd-release pacific"
-            elseif tostring(branch) == "pacific" then
-              upgrade_branch = "reef"
-              osd_release = "ceph osd require-osd-release reef"
-            else
-              error("invalid source branch")
-            end
-            yaml_fragment['tasks'][2]['install.upgrade']['mon.a']['branch'] = upgrade_branch
-            py_attrgetter(yaml_fragment['tasks'][9]['exec']['mon.a']).append(osd_release)
-overrides:
-  ceph:
-    log-ignorelist:
-    - scrub mismatch
-    - ScrubResult
-    - wrongly marked
-    - \(POOL_APP_NOT_ENABLED\)
-    - \(SLOW_OPS\)
-    - overall HEALTH_
-    - \(MON_MSGR2_NOT_ENABLED\)
-    - slow request
-    conf:
-      global:
-        bluestore warn on legacy statfs: false
-        bluestore warn on no per pool omap: false
-      mon:
-        mon warn on osd down out interval zero: false
-
-tasks:
-- mds_pre_upgrade:
-- print: "**** done mds pre-upgrade sequence"
-- install.upgrade:
-    # upgrade the single cluster node, which is running all the mon/mds/osd/mgr daemons
-    mon.a:
-      branch: []
-- print: "**** done install.upgrade the host"
-- ceph.restart:
-    daemons: [mon.*, mgr.*]
-    mon-health-to-clog: false
-    wait-for-healthy: false
-- ceph.healthy:
-- ceph.restart:
-    daemons: [osd.*]
-    wait-for-healthy: false
-    wait-for-osds-up: true
-- ceph.stop: [mds.*]
-- ceph.restart:
-    daemons: [mds.*]
-    wait-for-healthy: false
-    wait-for-osds-up: true
-- exec:
-    mon.a:
-    - ceph osd dump -f json-pretty
-    - ceph versions
-    - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done
-- ceph.healthy:
-- print: "**** done ceph.restart"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml
deleted file mode 100644
index 012969022a41..000000000000
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/0-clients/fuse-upgrade.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-teuthology:
-  postmerge:
-    - if not is_fuse() then reject() end
-tasks:
-- ceph-fuse:
-    client.0:
-      mounted: false
-- print: "**** done unmount client.0"
-- install.upgrade:
-    client.0:
-- print: "**** done install.upgrade on client.0"
-- ceph.healthy:
-- ceph-fuse:
-    client.0:
-- print: "**** done remount client"
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/1-client-sanity.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/newops.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/1-client-sanity.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/new_ops/newops.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/% b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/%
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/blogbench.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/blogbench.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/blogbench.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/blogbench.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/dbench.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/dbench.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/dbench.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/dbench.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/fsstress.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/fsstress.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/fsstress.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/fsstress.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/iozone.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/iozone.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/iozone.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/iozone.yaml
diff --git a/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/kernel_untar_build.yaml b/qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/kernel_untar_build.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/1-tests/kernel_untar_build.yaml
rename to qa/suites/fs/upgrade/upgraded_client/tasks/3-workload/stress_tests/kernel_untar_build.yaml

From 750a9483d169bdc10ab6865492b9e399e40bc20c Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Fri, 18 Aug 2023 12:25:52 +0000
Subject: [PATCH 0686/2492] common: Windows Unicode CLI support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Windows CLI arguments use either ANSI (main()) or UTF-16 (wmain()).
Meanwhile, Ceph libraries expect UTF-8 and raise exceptions when
trying to use Unicode CLI arguments or log Unicode output:

  rbd.exe create test_unicode_șțăâ --size=32M
  terminate called after throwing an instance of 'std::runtime_error'
    what():  invalid utf8

We'll use a Windows application manifest, setting the "activeCodePage"
property [1][2]. This enables the Windows UCRT UTF-8 mode so that
functions that receive char* arguments will expect UTF-8 instead of ANSI,
including main(). One exception is CreateProcess, which will need the
UTF-16 form (CreateProcessW).

Despite the locale being set to utf-8, we'll have to explicitly set
the console output to utf-8 using SetConsoleOutputCP(CP_UTF8).

In order to use the UTF-8 locale, we'll have to switch the mingw-llvm
runtime from msvcrt to ucrt.

This also fixes ceph-dokan crashes that currently occur when non-ANSI
paths are logged.

[1] https://learn.microsoft.com/en-us/windows/win32/sbscs/application-manifests#activecodepage
[2] https://learn.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-code-page

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 mingw_conf.sh                       |  4 ++--
 src/common/win32/SubProcess.cc      |  8 +++++---
 src/common/win32/code_page.manifest |  8 ++++++++
 src/common/win32/code_page.rc       |  2 ++
 src/dokan/CMakeLists.txt            |  3 ++-
 src/dokan/ceph_dokan.cc             |  2 ++
 src/tools/CMakeLists.txt            |  3 +++
 src/tools/rados/rados.cc            |  3 +++
 src/tools/rbd/CMakeLists.txt        |  3 +++
 src/tools/rbd/rbd.cc                |  3 +++
 src/tools/rbd_wnbd/CMakeLists.txt   |  5 ++++-
 src/tools/rbd_wnbd/rbd_wnbd.cc      | 13 ++++++++-----
 win32_deps_build.sh                 |  4 ++--
 13 files changed, 47 insertions(+), 14 deletions(-)
 create mode 100644 src/common/win32/code_page.manifest
 create mode 100644 src/common/win32/code_page.rc

diff --git a/mingw_conf.sh b/mingw_conf.sh
index 6a226da5f040..a03eb95dda3b 100644
--- a/mingw_conf.sh
+++ b/mingw_conf.sh
@@ -129,8 +129,8 @@ EOL
     if [[ -n $USE_MINGW_LLVM ]]; then
         cat >> $MINGW_CMAKE_FILE <<EOL
 add_definitions(-I$mingwX64IncludeDir)
-add_definitions(-march=native)
-add_definitions(-Wno-unknown-attributes)
+add_compile_options(-march=native)
+add_compile_options(-Wno-unknown-attributes)
 EOL
     fi
 fi
diff --git a/src/common/win32/SubProcess.cc b/src/common/win32/SubProcess.cc
index 3ed3b4f54c71..59975b1e6d31 100644
--- a/src/common/win32/SubProcess.cc
+++ b/src/common/win32/SubProcess.cc
@@ -18,6 +18,7 @@
 
 #include "common/SubProcess.h"
 #include "common/errno.h"
+#include "common/win32/wstring.h"
 #include "include/ceph_assert.h"
 #include "include/compat.h"
 
@@ -174,8 +175,9 @@ int SubProcess::spawn() {
   for (auto& arg : cmd_args) {
     cmdline << " " << std::quoted(arg);
   }
+  std::wstring cmdline_w = to_wstring(cmdline.str());
 
-  STARTUPINFO si = {0};
+  STARTUPINFOW si = {0};
   PROCESS_INFORMATION pi = {0};
   SECURITY_ATTRIBUTES sa = {0};
 
@@ -224,8 +226,8 @@ int SubProcess::spawn() {
   // We've transfered ownership from those handles.
   stdin_w = stdout_r = stderr_r = INVALID_HANDLE_VALUE;
 
-  if (!CreateProcess(
-      NULL, const_cast<char*>(cmdline.str().c_str()),
+  if (!CreateProcessW(
+      NULL, const_cast<wchar_t*>(cmdline_w.c_str()),
       NULL, NULL, /* No special security attributes */
       1, /* Inherit handles marked as inheritable */
       0, /* No special flags */
diff --git a/src/common/win32/code_page.manifest b/src/common/win32/code_page.manifest
new file mode 100644
index 000000000000..dab929e1515a
--- /dev/null
+++ b/src/common/win32/code_page.manifest
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>
diff --git a/src/common/win32/code_page.rc b/src/common/win32/code_page.rc
new file mode 100644
index 000000000000..12258c4bd615
--- /dev/null
+++ b/src/common/win32/code_page.rc
@@ -0,0 +1,2 @@
+#include <winuser.h>
+CREATEPROCESS_MANIFEST_RESOURCE_ID RT_MANIFEST "code_page.manifest"
diff --git a/src/dokan/CMakeLists.txt b/src/dokan/CMakeLists.txt
index cc05a0f29f60..2a61d38bb8d0 100644
--- a/src/dokan/CMakeLists.txt
+++ b/src/dokan/CMakeLists.txt
@@ -2,7 +2,8 @@ set(ceph_dokan_srcs
   ceph_dokan.cc
   dbg.cc
   utils.cc
-  options.cc)
+  options.cc
+  ../common/win32/code_page.rc)
 add_executable(ceph-dokan ${ceph_dokan_srcs})
 target_link_libraries(ceph-dokan ${DOKAN_LIBRARIES}
   ${GSSAPI_LIBRARIES}
diff --git a/src/dokan/ceph_dokan.cc b/src/dokan/ceph_dokan.cc
index 9e115222cab2..1ea82e07423b 100644
--- a/src/dokan/ceph_dokan.cc
+++ b/src/dokan/ceph_dokan.cc
@@ -1043,6 +1043,8 @@ boost::intrusive_ptr<CephContext> do_global_init(
 
 int main(int argc, const char** argv)
 {
+  SetConsoleOutputCP(CP_UTF8);
+
   if (!SetConsoleCtrlHandler((PHANDLER_ROUTINE)ConsoleHandler, TRUE)) {
     cerr << "Couldn't initialize console event handler." << std::endl;
     return -EINVAL;
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index d1133798c5c9..993fadb2e2ed 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -6,6 +6,9 @@ set(rados_srcs
   ${PROJECT_SOURCE_DIR}/src/common/util.cc
   ${PROJECT_SOURCE_DIR}/src/common/obj_bencher.cc
   ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc)
+if(WIN32)
+  list(APPEND rados_srcs ../common/win32/code_page.rc)
+endif()
 add_executable(rados ${rados_srcs})
 
 target_link_libraries(rados librados global ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS})
diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 3d602e749cde..b8cf5e4d1dc3 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -4047,6 +4047,9 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
 
 int main(int argc, const char **argv)
 {
+  #ifdef _WIN32
+  SetConsoleOutputCP(CP_UTF8);
+  #endif
   auto args = argv_to_vec(argc, argv);
   if (args.empty()) {
     cerr << argv[0] << ": -h or --help for usage" << std::endl;
diff --git a/src/tools/rbd/CMakeLists.txt b/src/tools/rbd/CMakeLists.txt
index 19b4e806a752..dac1d8babf92 100644
--- a/src/tools/rbd/CMakeLists.txt
+++ b/src/tools/rbd/CMakeLists.txt
@@ -55,6 +55,9 @@ set(rbd_srcs
   action/Ubbd.cc
   action/Watch.cc
   action/Wnbd.cc)
+if(WIN32)
+  list(APPEND rbd_srcs ../../common/win32/code_page.rc)
+endif()
 
 add_executable(rbd ${rbd_srcs}
   $<TARGET_OBJECTS:common_texttable_obj>)
diff --git a/src/tools/rbd/rbd.cc b/src/tools/rbd/rbd.cc
index a8c59d57577b..bdeded4a05b5 100644
--- a/src/tools/rbd/rbd.cc
+++ b/src/tools/rbd/rbd.cc
@@ -5,6 +5,9 @@
 
 int main(int argc, const char **argv)
 {
+  #ifdef _WIN32
+  SetConsoleOutputCP(CP_UTF8);
+  #endif
   rbd::Shell shell;
   return shell.execute(argc, argv);
 }
diff --git a/src/tools/rbd_wnbd/CMakeLists.txt b/src/tools/rbd_wnbd/CMakeLists.txt
index 86c41b2eeb6f..ff09cd80a152 100644
--- a/src/tools/rbd_wnbd/CMakeLists.txt
+++ b/src/tools/rbd_wnbd/CMakeLists.txt
@@ -1,4 +1,7 @@
-add_executable(rbd-wnbd rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc)
+add_executable(
+    rbd-wnbd
+    rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc
+    ../../common/win32/code_page.rc)
 set_target_properties(
     rbd-wnbd PROPERTIES COMPILE_FLAGS
     "-fpermissive -I${WNBD_INCLUDE_DIRS}")
diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc
index d2df88cabb48..1946e83ff967 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.cc
+++ b/src/tools/rbd_wnbd/rbd_wnbd.cc
@@ -331,7 +331,7 @@ int send_map_request(std::string arguments) {
 // which will allow it to communicate the mapping status
 int map_device_using_suprocess(std::string arguments, int timeout_ms)
 {
-  STARTUPINFO si;
+  STARTUPINFOW si;
   PROCESS_INFORMATION pi;
   char ch;
   DWORD err = 0, status = 0;
@@ -407,11 +407,12 @@ int map_device_using_suprocess(std::string arguments, int timeout_ms)
 
   dout(5) << __func__ << ": command line: " << command_line.str() << dendl;
 
-  GetStartupInfo(&si);
+  GetStartupInfoW(&si);
   // Create a detached child
-  if (!CreateProcess(NULL, (char*)command_line.str().c_str(),
-                     NULL, NULL, FALSE, DETACHED_PROCESS,
-                     NULL, NULL, &si, &pi)) {
+  if (!CreateProcessW(
+      NULL, const_cast<wchar_t*>(to_wstring(command_line.str()).c_str()),
+      NULL, NULL, FALSE, DETACHED_PROCESS,
+      NULL, NULL, &si, &pi)) {
     err = GetLastError();
     derr << "CreateProcess failed: " << win32_strerror(err) << dendl;
     exit_code = -ECHILD;
@@ -1904,6 +1905,8 @@ int main(int argc, const char *argv[])
   SetConsoleCtrlHandler(console_handler_routine, true);
   // Avoid the Windows Error Reporting dialog.
   SetErrorMode(GetErrorMode() | SEM_NOGPFAULTERRORBOX);
+  SetConsoleOutputCP(CP_UTF8);
+
   int r = rbd_wnbd(argc, argv);
   if (r < 0) {
     return r;
diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index 6eea81d1b8a2..c61cd7cb78a8 100755
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -40,8 +40,8 @@ dokanTag="v2.0.5.1000"
 dokanSrcDir="${depsSrcDir}/dokany"
 dokanLibDir="${depsToolsetDir}/dokany/lib"
 
-mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-msvcrt-ubuntu-18.04-x86_64.tar.xz"
-mingwLlvmSha256Sum="bc97745e702fb9e8f2a16f7d09dd5061ceeef16554dd12e542f619ce937e8d7a"
+mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-ubuntu-18.04-x86_64.tar.xz"
+mingwLlvmSha256Sum="bc367753dea829d219be32e2e64e2d15d03158ce8e700ae5210ca3d78e6a07ea"
 mingwLlvmDir="${DEPS_DIR}/mingw-llvm"
 
 function _make() {

From 2cef5b3ce4bc988ac93ff128218f4b0dddb0f074 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Mon, 6 Nov 2023 14:58:08 +0000
Subject: [PATCH 0687/2492] test: fix Windows ::_creat

The Windows Universal C Runtime (ucrt) "_creat" function is no
longer POSIX compatible and requires Windows specific mode flags.

We got admin socket test failures after switching from msvcrt to
uscrt.

We'll address the issue with some platform checks.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/global/signal_handler.cc | 4 ++++
 src/test/admin_socket.cc     | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index 055763eee469..d33872678719 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -276,7 +276,11 @@ void generate_crash_dump(char *base,
 	::close(fd);
       }
       snprintf(fn, sizeof(fn)-1, "%s/done", base);
+      #ifdef _WIN32
+      ::creat(fn, _S_IREAD);
+      #else
       ::creat(fn, 0444);
+      #endif
     }
   }
 }
diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc
index a8236271652c..03bbb72b2a9a 100644
--- a/src/test/admin_socket.cc
+++ b/src/test/admin_socket.cc
@@ -239,7 +239,11 @@ TEST(AdminSocketClient, Ping) {
     ASSERT_FALSE(ok);
   }
   // file exists but does not allow connections (no process, wrong type...)
+  #ifdef _WIN32
+  int fd = ::creat(path.c_str(), _S_IREAD | _S_IWRITE);
+  #else
   int fd = ::creat(path.c_str(), 0777);
+  #endif
   ASSERT_TRUE(fd);
   // On Windows, we won't be able to remove the file unless we close it
   // first.
@@ -307,7 +311,11 @@ TEST(AdminSocket, bind_and_listen) {
   {
     int fd = 0;
     string message;
+    #ifdef _WIN32
+    int fd2 = ::creat(path.c_str(), _S_IREAD | _S_IWRITE);
+    #else
     int fd2 = ::creat(path.c_str(), 0777);
+    #endif
     ASSERT_TRUE(fd2);
     // On Windows, we won't be able to remove the file unless we close it
     // first.

From 3904b6754e7defb7e819b4854920896dd0496152 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 22 Nov 2023 11:56:44 +0200
Subject: [PATCH 0688/2492] osd/scrub: allow auto-repair on operator-initiated
 scrubs

Previously, operator-initiated scrubs would not auto-repair, regardless
of the value of the 'osd_scrub_auto_repair' config option.  This was
less confusing to the operator than it could have been, as most
operator commands would in fact cause a regular periodic scrub
to be initiated. However, that quirk is now fixed: operator commands
now trigger 'op-initiated' scrubs. Thus the need for this patch.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc             | 21 ++++++++++++++++++++-
 src/osd/scrubber_common.h |  3 +--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index d2f97a129a24..bda1d206b9dd 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1490,7 +1490,7 @@ std::optional<requested_scrub_t> PG::validate_initiated_scrub(
 
   upd_flags.time_for_deep = time_for_deep;
   upd_flags.deep_scrub_on_error = false;
-  upd_flags.auto_repair = false;  // will only be considered for periodic scrubs
+  upd_flags.auto_repair = false;
 
   if (upd_flags.must_deep_scrub) {
     upd_flags.calculated_to_deep = true;
@@ -1506,6 +1506,25 @@ std::optional<requested_scrub_t> PG::validate_initiated_scrub(
     }
   }
 
+  if (try_to_auto_repair) {
+    // for shallow scrubs: rescrub if errors found
+    // for deep: turn 'auto-repair' on
+    if (upd_flags.calculated_to_deep) {
+      dout(10) << fmt::format(
+                      "{}: performing an auto-repair deep scrub",
+                      __func__)
+               << dendl;
+      upd_flags.auto_repair = true;
+    } else {
+      dout(10) << fmt::format(
+		      "{}: will perform an auto-repair deep scrub if errors "
+		      "are found",
+		      __func__)
+	       << dendl;
+      upd_flags.deep_scrub_on_error = true;
+    }
+  }
+
   return upd_flags;
 }
 
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 16810bba15c6..b9426b9c64f0 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -169,8 +169,7 @@ struct requested_scrub_t {
    * the value of auto_repair is determined in sched_scrub() (once per scrub.
    * previous value is not remembered). Set if
    * - allowed by configuration and backend, and
-   * - must_scrub is not set (i.e. - this is a periodic scrub),
-   * - time_for_deep was just set
+   * - for periodic scrubs: time_for_deep was just set
    */
   bool auto_repair{false};
 

From 9481b7e52dd66ee035adb234104dafdd2878e3a1 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 2 Nov 2023 08:25:53 +0100
Subject: [PATCH 0689/2492] mgr/dashboard: add tags field to bucket edit

Fixes: https://tracker.ceph.com/issues/63412
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   | 13 +++-
 .../bucket-tag-modal.component.html           | 59 +++++++++++++++
 .../bucket-tag-modal.component.scss           |  0
 .../bucket-tag-modal.component.spec.ts        | 27 +++++++
 .../bucket-tag-modal.component.ts             | 75 +++++++++++++++++++
 .../rgw-bucket-details.component.html         | 15 ++++
 .../rgw-bucket-form.component.html            | 60 +++++++++++++++
 .../rgw-bucket-form.component.ts              | 69 ++++++++++++++++-
 .../frontend/src/app/ceph/rgw/rgw.module.ts   |  4 +-
 .../app/shared/api/rgw-bucket.service.spec.ts | 10 ++-
 .../src/app/shared/api/rgw-bucket.service.ts  | 10 ++-
 src/pybind/mgr/dashboard/openapi.yaml         |  4 +
 .../mgr/dashboard/services/rgw_client.py      | 13 ++++
 13 files changed, 347 insertions(+), 12 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.ts

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 8bfb0f9902d9..edd85af7b034 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -280,6 +280,10 @@ def _get_policy(self, bucket: str):
         rgw_client = RgwClient.admin_instance()
         return rgw_client.get_bucket_policy(bucket)
 
+    def _set_tags(self, bucket_name, tags, daemon_name, owner):
+        rgw_client = RgwClient.instance(owner, daemon_name)
+        return rgw_client.set_tags(bucket_name, tags)
+
     @staticmethod
     def strip_tenant_from_bucket_name(bucket_name):
         # type (str) -> str
@@ -345,7 +349,7 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
                lock_enabled='false', lock_mode=None,
                lock_retention_period_days=None,
                lock_retention_period_years=None, encryption_state='false',
-               encryption_type=None, key_id=None, daemon_name=None):
+               encryption_type=None, key_id=None, tags=None, daemon_name=None):
         lock_enabled = str_to_bool(lock_enabled)
         encryption_state = str_to_bool(encryption_state)
         try:
@@ -361,6 +365,9 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
             if encryption_state:
                 self._set_encryption(bucket, encryption_type, key_id, daemon_name, uid)
 
+            if tags:
+                self._set_tags(bucket, tags, daemon_name, uid)
+
             return result
         except RequestException as e:  # pragma: no cover - handling is too obvious
             raise DashboardException(e, http_status_code=500, component='rgw')
@@ -370,7 +377,7 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
             encryption_state='false', encryption_type=None, key_id=None,
             mfa_delete=None, mfa_token_serial=None, mfa_token_pin=None,
             lock_mode=None, lock_retention_period_days=None,
-            lock_retention_period_years=None, daemon_name=None):
+            lock_retention_period_years=None, tags=None, daemon_name=None):
         encryption_state = str_to_bool(encryption_state)
         # When linking a non-tenant-user owned bucket to a tenanted user, we
         # need to prefix bucket name with '/'. e.g. photos -> /photos
@@ -410,6 +417,8 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
             self._set_encryption(bucket_name, encryption_type, key_id, daemon_name, uid)
         if encryption_status['Status'] == 'Enabled' and (not encryption_state):
             self._delete_encryption(bucket_name, daemon_name, uid)
+        if tags:
+            self._set_tags(bucket_name, tags, daemon_name, uid)
         return self._append_bid(result)
 
     def delete(self, bucket, purge_objects='true', daemon_name=None):
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.html
new file mode 100644
index 000000000000..3e732e355689
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.html
@@ -0,0 +1,59 @@
+<cd-modal [modalRef]="activeModal">
+  <span class="modal-title"
+        i18n>{{ getMode() }} Tag</span>
+
+    <ng-container class="modal-content">
+      <form class="form"
+            #formDir="ngForm"
+            [formGroup]="form">
+        <div class="modal-body">
+          <!-- Key -->
+          <div class="form-group row">
+            <label class="cd-col-form-label required"
+                   for="key"
+                   i18n>Key</label>
+            <div class="cd-col-form-input">
+              <input type="text"
+                     class="form-control"
+                     formControlName="key"
+                     id="key">
+              <span class="invalid-feedback"
+                    *ngIf="form.showError('key', formDir, 'required')"
+                    i18n>This field is required.</span>
+              <span class="invalid-feedback"
+                    *ngIf="form.showError('key', formDir, 'unique')"
+                    i18n>This key must be unique.</span>
+              <span class="invalid-feedback"
+                    *ngIf="form.showError('key', formDir, 'maxLength')"
+                    i18n>Length of the key must be maximum of 128 characters</span>
+            </div>
+          </div>
+
+          <!-- Value -->
+          <div class="form-group row">
+            <label class="cd-col-form-label required"
+                   for="value"
+                   i18n>Value</label>
+            <div class="cd-col-form-input">
+              <input id="value"
+                     class="form-control"
+                     type="text"
+                     formControlName="value">
+              <span *ngIf="form.showError('value', formDir, 'required')"
+                    class="invalid-feedback"
+                    i18n>This field is required.</span>
+              <span class="invalid-feedback"
+                    *ngIf="form.showError('value', formDir, 'maxLength')"
+                    i18n>Length of the value must be a maximum of 128 characters</span>
+            </div>
+          </div>
+        </div>
+
+        <div class="modal-footer">
+          <cd-form-button-panel (submitActionEvent)="onSubmit()"
+                                [form]="form"
+                                [submitText]="getMode()"></cd-form-button-panel>
+        </div>
+      </form>
+    </ng-container>
+  </cd-modal>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.spec.ts
new file mode 100644
index 000000000000..a54e7eeee082
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.spec.ts
@@ -0,0 +1,27 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { BucketTagModalComponent } from './bucket-tag-modal.component';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { ReactiveFormsModule } from '@angular/forms';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+
+describe('BucketTagModalComponent', () => {
+  let component: BucketTagModalComponent;
+  let fixture: ComponentFixture<BucketTagModalComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [BucketTagModalComponent],
+      imports: [HttpClientTestingModule, ReactiveFormsModule],
+      providers: [NgbActiveModal]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(BucketTagModalComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.ts
new file mode 100644
index 000000000000..5135539e5d47
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/bucket-tag-modal/bucket-tag-modal.component.ts
@@ -0,0 +1,75 @@
+import { Component, EventEmitter, Output } from '@angular/core';
+import { Validators } from '@angular/forms';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import _ from 'lodash';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { CdFormBuilder } from '~/app/shared/forms/cd-form-builder';
+import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
+import { CdValidators } from '~/app/shared/forms/cd-validators';
+
+@Component({
+  selector: 'cd-bucket-tag-modal',
+  templateUrl: './bucket-tag-modal.component.html',
+  styleUrls: ['./bucket-tag-modal.component.scss']
+})
+export class BucketTagModalComponent {
+  @Output()
+  submitAction = new EventEmitter();
+
+  form: CdFormGroup;
+  editMode = false;
+  currentKeyTags: string[];
+  storedKey: string;
+
+  constructor(
+    private formBuilder: CdFormBuilder,
+    public activeModal: NgbActiveModal,
+    public actionLabels: ActionLabelsI18n
+  ) {
+    this.createForm();
+  }
+
+  private createForm() {
+    this.form = this.formBuilder.group({
+      key: [
+        null,
+        [
+          Validators.required,
+          CdValidators.custom('unique', (value: string) => {
+            if (_.isEmpty(value) && !this.currentKeyTags) {
+              return false;
+            }
+            return this.storedKey !== value && this.currentKeyTags.includes(value);
+          }),
+          CdValidators.custom('maxLength', (value: string) => {
+            if (_.isEmpty(value)) return false;
+            return value.length > 128;
+          })
+        ]
+      ],
+      value: [
+        null,
+        [
+          Validators.required,
+          CdValidators.custom('maxLength', (value: string) => {
+            if (_.isEmpty(value)) return false;
+            return value.length > 128;
+          })
+        ]
+      ]
+    });
+  }
+
+  onSubmit() {
+    this.submitAction.emit(this.form.value);
+    this.activeModal.close();
+  }
+
+  getMode() {
+    return this.editMode ? this.actionLabels.EDIT : this.actionLabels.ADD;
+  }
+
+  fillForm(tag: Record<string, string>) {
+    this.form.setValue(tag);
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
index f2447feab264..e96a89b234f9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
@@ -100,6 +100,21 @@
             </ng-container>
           </tbody>
         </table>
+
+      <!-- Tags -->
+      <ng-container *ngIf="selection.tagset">
+        <legend i18n>Tags</legend>
+        <table class="table table-striped table-bordered">
+          <tbody>
+            <tr *ngFor="let tag of selection.tagset | keyvalue">
+              <td i18n
+                  class="bold w-25">{{tag.key}}</td>
+              <td class="w-75">{{ tag.value }}</td>
+            </tr>
+          </tbody>
+        </table>
+      </ng-container>
+
       </ng-template>
     </ng-container>
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index 761081c37443..a9704c0bdc89 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -385,6 +385,32 @@
           </div>
         </fieldset>
 
+        <!-- Tags -->
+        <legend class="cd-header"
+                i18n>Tags
+          <cd-helper>Tagging gives you a way to categorize storage</cd-helper>
+        </legend>
+        <span *ngFor="let tag of tags; let i=index;">
+          <ng-container *ngTemplateOutlet="tagTpl; context:{index: i, tag: tag}"></ng-container>
+        </span>
+
+        <div class="row">
+          <div class="col-12">
+            <strong *ngIf="tags.length > 19"
+                    class="text-warning"
+                    i18n>Maximum of 20 tags reached</strong>
+            <button type="button"
+                    id="add-tag"
+                    class="btn btn-light float-end my-3"
+                    [disabled]="tags.length > 19"
+                    (click)="showTagModal()">
+              <i [ngClass]="[icons.add]"></i>
+              <ng-container i18n>Add tag</ng-container>
+            </button>
+          </div>
+        </div>
+
+
       </div>
       <div class="card-footer">
         <cd-form-button-panel (submitActionEvent)="submit()"
@@ -395,3 +421,37 @@
     </div>
   </form>
 </div>
+
+<ng-template #tagTpl
+             let-tag="tag"
+             let-index="index">
+  <div class="input-group my-2">
+    <ng-container *ngFor="let config of tagConfig">
+      <input type="text"
+             id="tag-{{config.attribute}}-{{index}}"
+             class="form-control"
+             [ngbTooltip]="config.attribute"
+             [value]="tag[config.attribute]"
+             disabled
+             readonly>
+    </ng-container>
+
+    <!-- Tag actions -->
+    <button type="button"
+            class="btn btn-light"
+            id="tag-edit-{{index}}"
+            i18n-ngbTooltip
+            ngbTooltip="Edit"
+            (click)="showTagModal(index)">
+      <i [ngClass]="[icons.edit]"></i>
+    </button>
+    <button type="button"
+            class="btn btn-light"
+            id="tag-delete-{{index}}"
+            i18n-ngbTooltip
+            ngbTooltip="Delete"
+            (click)="deleteTag(index)">
+      <i [ngClass]="[icons.trash]"></i>
+    </button>
+  </div>
+</ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index de8e0383ac02..6b90b45e16a6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -21,6 +21,7 @@ import { RgwBucketEncryptionModel } from '../models/rgw-bucket-encryption';
 import { RgwBucketMfaDelete } from '../models/rgw-bucket-mfa-delete';
 import { RgwBucketVersioning } from '../models/rgw-bucket-versioning';
 import { RgwConfigModalComponent } from '../rgw-config-modal/rgw-config-modal.component';
+import { BucketTagModalComponent } from '../bucket-tag-modal/bucket-tag-modal.component';
 
 @Component({
   selector: 'cd-rgw-bucket-form',
@@ -42,6 +43,15 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
   icons = Icons;
   kmsVaultConfig = false;
   s3VaultConfig = false;
+  tags: Record<string, string>[] = [];
+  tagConfig = [
+    {
+      attribute: 'key'
+    },
+    {
+      attribute: 'value'
+    }
+  ];
 
   get isVersioningEnabled(): boolean {
     return this.bucketForm.getValue('versioning');
@@ -191,6 +201,11 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           value['versioning'] = bidResp['versioning'] === RgwBucketVersioning.ENABLED;
           value['mfa-delete'] = bidResp['mfa_delete'] === RgwBucketMfaDelete.ENABLED;
           value['encryption_enabled'] = bidResp['encryption'] === 'Enabled';
+          if (bidResp['tagset']) {
+            for (const [key, value] of Object.entries(bidResp['tagset'])) {
+              this.tags.push({ key: key, value: value.toString() });
+            }
+          }
           // Append default values.
           value = _.merge(defaults, value);
           // Update the form.
@@ -224,6 +239,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
       return;
     }
     const values = this.bucketForm.value;
+    const xmlStrTags = this.tagsToXML(this.tags);
     if (this.editing) {
       // Edit
       const versioning = this.getVersioningStatus();
@@ -241,7 +257,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['mfa-token-serial'],
           values['mfa-token-pin'],
           values['lock_mode'],
-          values['lock_retention_period_days']
+          values['lock_retention_period_days'],
+          xmlStrTags
         )
         .subscribe(
           () => {
@@ -269,7 +286,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['lock_retention_period_days'],
           values['encryption_enabled'],
           values['encryption_type'],
-          values['keyId']
+          values['keyId'],
+          xmlStrTags
         )
         .subscribe(
           () => {
@@ -337,4 +355,51 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
       .get('encryptionType')
       .setValue(this.bucketForm.getValue('encryption_type') || 'AES256');
   }
+
+  showTagModal(index?: number) {
+    const modalRef = this.modalService.show(BucketTagModalComponent);
+    const modalComponent = modalRef.componentInstance as BucketTagModalComponent;
+    modalComponent.currentKeyTags = this.tags.map((item) => item.key);
+
+    if (_.isNumber(index)) {
+      modalComponent.editMode = true;
+      modalComponent.fillForm(this.tags[index]);
+      modalComponent.storedKey = this.tags[index]['key'];
+    }
+
+    modalComponent.submitAction.subscribe((tag: Record<string, string>) => {
+      this.setTag(tag, index);
+    });
+  }
+
+  deleteTag(index: number) {
+    this.tags.splice(index, 1);
+  }
+
+  private setTag(tag: Record<string, string>, index?: number) {
+    if (_.isNumber(index)) {
+      this.tags[index] = tag;
+    } else {
+      this.tags.push(tag);
+    }
+    this.bucketForm.markAsDirty();
+    this.bucketForm.updateValueAndValidity();
+  }
+
+  private tagsToXML(tags: Record<string, string>[]): string {
+    let xml = '<Tagging><TagSet>';
+    for (const tag of tags) {
+      xml += '<Tag>';
+      for (const key in tag) {
+        if (key === 'key') {
+          xml += `<Key>${tag[key]}</Key>`;
+        } else if (key === 'value') {
+          xml += `<Value>${tag[key]}</Value>`;
+        }
+      }
+      xml += '</Tag>';
+    }
+    xml += '</TagSet></Tagging>';
+    return xml;
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
index c16c13a81bd7..c8a7cf9884f8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
@@ -44,6 +44,7 @@ import { DashboardV3Module } from '../dashboard-v3/dashboard-v3.module';
 import { RgwSyncPrimaryZoneComponent } from './rgw-sync-primary-zone/rgw-sync-primary-zone.component';
 import { RgwSyncMetadataInfoComponent } from './rgw-sync-metadata-info/rgw-sync-metadata-info.component';
 import { RgwSyncDataInfoComponent } from './rgw-sync-data-info/rgw-sync-data-info.component';
+import { BucketTagModalComponent } from './bucket-tag-modal/bucket-tag-modal.component';
 
 @NgModule({
   imports: [
@@ -100,7 +101,8 @@ import { RgwSyncDataInfoComponent } from './rgw-sync-data-info/rgw-sync-data-inf
     RgwOverviewDashboardComponent,
     RgwSyncPrimaryZoneComponent,
     RgwSyncMetadataInfoComponent,
-    RgwSyncDataInfoComponent
+    RgwSyncDataInfoComponent,
+    BucketTagModalComponent
   ]
 })
 export class RgwModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
index 2c42d8b427c0..15821c3b6265 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
@@ -59,11 +59,12 @@ describe('RgwBucketService', () => {
         '5',
         true,
         'aws:kms',
-        'qwerty1'
+        'qwerty1',
+        null
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&${RgwHelper.DAEMON_QUERY_PARAM}`
+      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&${RgwHelper.DAEMON_QUERY_PARAM}`
     );
     expect(req.request.method).toBe('POST');
   });
@@ -82,11 +83,12 @@ describe('RgwBucketService', () => {
         '1',
         '223344',
         'GOVERNANCE',
-        '10'
+        '10',
+        null
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10`
+      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10&tags=null`
     );
     expect(req.request.method).toBe('PUT');
   });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
index 7207d0b5ca72..87561d92d899 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
@@ -59,7 +59,8 @@ export class RgwBucketService extends ApiClient {
     lock_retention_period_days: string,
     encryption_state: boolean,
     encryption_type: string,
-    key_id: string
+    key_id: string,
+    tags: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       return this.http.post(this.url, null, {
@@ -75,6 +76,7 @@ export class RgwBucketService extends ApiClient {
             encryption_state: String(encryption_state),
             encryption_type,
             key_id,
+            tags: tags,
             daemon_name: params.get('daemon_name')
           }
         })
@@ -94,7 +96,8 @@ export class RgwBucketService extends ApiClient {
     mfaTokenSerial: string,
     mfaTokenPin: string,
     lockMode: 'GOVERNANCE' | 'COMPLIANCE',
-    lockRetentionPeriodDays: string
+    lockRetentionPeriodDays: string,
+    tags: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       params = params.appendAll({
@@ -108,7 +111,8 @@ export class RgwBucketService extends ApiClient {
         mfa_token_serial: mfaTokenSerial,
         mfa_token_pin: mfaTokenPin,
         lock_mode: lockMode,
-        lock_retention_period_days: lockRetentionPeriodDays
+        lock_retention_period_days: lockRetentionPeriodDays,
+        tags: tags
       });
       return this.http.put(`${this.url}/${bucket}`, null, { params: params });
     });
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index d35ea87e15ae..3081d60b16f8 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -8817,6 +8817,8 @@ paths:
                   type: string
                 placement_target:
                   type: string
+                tags:
+                  type: string
                 uid:
                   type: string
                 zonegroup:
@@ -9129,6 +9131,8 @@ paths:
                   type: string
                 mfa_token_serial:
                   type: string
+                tags:
+                  type: string
                 uid:
                   type: string
                 versioning_state:
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index 2d3226bab3e0..c9db37b393e6 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -702,6 +702,19 @@ def set_bucket_encryption(self, bucket_name, key_id,
         except RequestException as e:
             raise DashboardException(msg=str(e), component='rgw')
 
+    @RestClient.api_put('/{bucket_name}?tagging')
+    def set_tags(self, bucket_name, tags, request=None):
+        # pylint: disable=unused-argument
+        try:
+            ET.fromstring(tags)
+        except ET.ParseError:
+            return "Data must be properly formatted"
+        try:
+            result = request(data=tags)  # type: ignore
+        except RequestException as e:
+            raise DashboardException(msg=str(e), component='rgw')
+        return result
+
     @RestClient.api_get('/{bucket_name}?object-lock')
     def get_bucket_locking(self, bucket_name, request=None):
         # type: (str, Optional[object]) -> dict

From 47bae14ad7d373e4177a345f0473cb62d7c89190 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 14:26:50 -0500
Subject: [PATCH 0690/2492] rgw/posix: fix unused variable warning

ceph/src/rgw/driver/posix/rgw_sal_posix.cc:37:19: warning: unused variable 'MP_OBJ_PART_FMT' [-Wunused-const-variable]
const std::string MP_OBJ_PART_FMT = "{:0>5}";
                  ^

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/posix/rgw_sal_posix.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index fc48c4f8fa8c..23f9f83dabc1 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -34,7 +34,6 @@ const std::string ATTR_PREFIX = "user.X-RGW-";
 #define RGW_POSIX_ATTR_OWNER "POSIX-Owner"
 const std::string mp_ns = "multipart";
 const std::string MP_OBJ_PART_PFX = "part-";
-const std::string MP_OBJ_PART_FMT = "{:0>5}";
 const std::string MP_OBJ_HEAD_NAME = MP_OBJ_PART_PFX + "00000";
 
 static inline bool get_attr(Attrs& attrs, const char* name, bufferlist& bl)

From 629cb3232d1e0dc2172d24602286bcfec9ef0323 Mon Sep 17 00:00:00 2001
From: zhangjianwei2 <zhangjianwei2_yewu@cmss.chinamobile.com>
Date: Tue, 14 Nov 2023 16:45:43 +0800
Subject: [PATCH 0691/2492] osd: distinguish between
 osd_pg_stat_report_max_(epoch|seconds)

osd_pg_stat_report_max was previously used as either a max time in seconds
or a max number of epochs.  Instead, separate into two configs and adjust
PeeringState::prepare_stats_for_publish to check both.

Additionally, this commit removes a superfluous check in
PeeringState::Active::react(const AdvMap&) and calls publish_stats_to_osd
unconditionally as with other callers in PeeringState.

Fixes: https://tracker.ceph.com/issues/63520

Signed-off-by: zhangjianwei2 <zhangjianwei2@cmss.chinamobile.com>
---
 qa/standalone/scrub/osd-scrub-snaps.sh | 13 ++++++-----
 qa/standalone/scrub/scrub-helpers.sh   |  3 ++-
 src/common/options/global.yaml.in      | 12 +++++++++-
 src/osd/PeeringState.cc                | 31 +++++++++++++-------------
 4 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/qa/standalone/scrub/osd-scrub-snaps.sh b/qa/standalone/scrub/osd-scrub-snaps.sh
index c543b48a19cc..40bd6a26e74d 100755
--- a/qa/standalone/scrub/osd-scrub-snaps.sh
+++ b/qa/standalone/scrub/osd-scrub-snaps.sh
@@ -209,14 +209,16 @@ function TEST_scrub_snaps() {
     done
     ceph tell osd.* config set osd_shallow_scrub_chunk_max 25
     ceph tell osd.* config set osd_shallow_scrub_chunk_min 5
-    ceph tell osd.* config set osd_pg_stat_report_interval_max 1
+    ceph tell osd.* config set osd_pg_stat_report_interval_max_seconds 1
+    ceph tell osd.* config set osd_pg_stat_report_interval_max_epochs 1
 
 
     wait_for_clean || return 1
 
     ceph tell osd.* config get osd_shallow_scrub_chunk_max
     ceph tell osd.* config get osd_shallow_scrub_chunk_min
-    ceph tell osd.* config get osd_pg_stat_report_interval_max
+    ceph tell osd.* config get osd_pg_stat_report_interval_max_seconds
+    ceph tell osd.* config get osd_pg_stat_report_interval_max_epochs
     ceph tell osd.* config get osd_scrub_chunk_max
     ceph tell osd.* config get osd_scrub_chunk_min
 
@@ -772,7 +774,8 @@ function _scrub_snaps_multi() {
     ceph tell osd.* config set osd_shallow_scrub_chunk_max 3
     ceph tell osd.* config set osd_shallow_scrub_chunk_min 3
     ceph tell osd.* config set osd_scrub_chunk_min 3
-    ceph tell osd.* config set osd_pg_stat_report_interval_max 1
+    ceph tell osd.* config set osd_pg_stat_report_interval_max_seconds 1
+    ceph tell osd.* config set osd_pg_stat_report_interval_max_epochs 1
     wait_for_clean || return 1
 
     local pgid="${poolid}.0"
@@ -1163,7 +1166,7 @@ fi
 function TEST_scrub_snaps_replica() {
     local dir=$1
     ORIG_ARGS=$CEPH_ARGS
-    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1"
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max_seconds=1 --osd_pg_stat_report_interval_max_epochs=1"
     _scrub_snaps_multi $dir replica
     err=$?
     CEPH_ARGS=$ORIG_ARGS
@@ -1173,7 +1176,7 @@ function TEST_scrub_snaps_replica() {
 function TEST_scrub_snaps_primary() {
     local dir=$1
     ORIG_ARGS=$CEPH_ARGS
-    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1"
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max_seconds=1 --osd_pg_stat_report_interval_max_epochs=1"
     _scrub_snaps_multi $dir primary
     err=$?
     CEPH_ARGS=$ORIG_ARGS
diff --git a/qa/standalone/scrub/scrub-helpers.sh b/qa/standalone/scrub/scrub-helpers.sh
index 6816d71de32c..3b0590fdfdef 100644
--- a/qa/standalone/scrub/scrub-helpers.sh
+++ b/qa/standalone/scrub/scrub-helpers.sh
@@ -239,7 +239,8 @@ function standard_scrub_cluster() {
             --osd_scrub_interval_randomize_ratio=0 \
             --osd_scrub_backoff_ratio=0.0 \
             --osd_pool_default_pg_autoscale_mode=off \
-            --osd_pg_stat_report_interval_max=1 \
+            --osd_pg_stat_report_interval_max_seconds=1 \
+            --osd_pg_stat_report_interval_max_epochs=1 \
             $extra_pars"
 
     for osd in $(seq 0 $(expr $OSDS - 1))
diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 48c6788a88b3..087ab80cb4d6 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -2903,9 +2903,19 @@ options:
   default: 5_min
   with_legacy: true
 # report pg stats for any given pg at least this often
-- name: osd_pg_stat_report_interval_max
+- name: osd_pg_stat_report_interval_max_seconds
   type: int
   level: advanced
+  desc: The maximum interval seconds for update pg's reported_epoch,
+        benefit for osdmap trim when osdmap not change frequently.
+  default: 500
+  with_legacy: true
+- name: osd_pg_stat_report_interval_max_epochs
+  type: int
+  level: advanced
+  desc: The maximum interval by which pg's reported_epoch lags behind,
+        otherwise, pg's reported_epoch must be updated,
+        benefit for osdmap trim when osdmap changes frequently.
   default: 500
   with_legacy: true
 # Max number of snap intervals to report to mgr in pg_stat_t
diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 2c41b7b71189..45b593f252c3 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -3890,8 +3890,6 @@ std::optional<pg_stat_t> PeeringState::prepare_stats_for_publish(
 
   pg_stat_t pre_publish = info.stats;
   pre_publish.stats.add(unstable_stats);
-  utime_t cutoff = now;
-  cutoff -= cct->_conf->osd_pg_stat_report_interval_max;
 
   // share (some of) our purged_snaps via the pg_stats. limit # of intervals
   // because we don't want to make the pg_stat_t structures too expensive.
@@ -3906,8 +3904,22 @@ std::optional<pg_stat_t> PeeringState::prepare_stats_for_publish(
   psdout(20) << "reporting purged_snaps "
 	     << pre_publish.purged_snaps << dendl;
 
+  // when there is no change in osdmap,
+  // update info.stats.reported_epoch by the number of time seconds.
+  utime_t cutoff_time = now;
+  cutoff_time -= cct->_conf->osd_pg_stat_report_interval_max_seconds;
+  bool is_time_expired = cutoff_time > info.stats.last_fresh ? true : false;
+
+  // 500 epoch osdmaps are also the minimum number of osdmaps that mon must retain.
+  // if info.stats.reported_epoch less than current osdmap epoch exceeds 500 osdmaps,
+  // it can be considered that the one reported by pgid is too old and needs to be updated.
+  // to facilitate mon trim osdmaps
+  epoch_t cutoff_epoch = info.stats.reported_epoch;
+  cutoff_epoch += cct->_conf->osd_pg_stat_report_interval_max_epochs;
+  bool is_epoch_behind = cutoff_epoch < get_osdmap_epoch() ? true : false;
+
   if (pg_stats_publish && pre_publish == *pg_stats_publish &&
-      info.stats.last_fresh > cutoff) {
+      (!is_epoch_behind && !is_time_expired)) {
     psdout(15) << "publish_stats_to_osd " << pg_stats_publish->reported_epoch
 	       << ": no change since " << info.stats.last_fresh << dendl;
     return std::nullopt;
@@ -5978,14 +5990,12 @@ boost::statechart::result PeeringState::Active::react(const AdvMap& advmap)
     return forward_event();
   }
   psdout(10) << "Active advmap" << dendl;
-  bool need_publish = false;
 
   pl->on_active_advmap(advmap.osdmap);
   if (ps->dirty_big_info) {
     // share updated purged_snaps to mgr/mon so that we (a) stop reporting
     // purged snaps and (b) perhaps share more snaps that we have purged
     // but didn't fit in pg_stat_t.
-    need_publish = true;
     ps->share_pg_info();
   }
 
@@ -6027,18 +6037,9 @@ boost::statechart::result PeeringState::Active::react(const AdvMap& advmap)
       ps->state_set(PG_STATE_UNDERSIZED);
     }
     // degraded changes will be detected by call from publish_stats_to_osd()
-    need_publish = true;
-  }
-
-  // if we haven't reported our PG stats in a long time, do so now.
-  if (ps->info.stats.reported_epoch + ps->cct->_conf->osd_pg_stat_report_interval_max < advmap.osdmap->get_epoch()) {
-    psdout(20) << "reporting stats to osd after " << (advmap.osdmap->get_epoch() - ps->info.stats.reported_epoch)
-		       << " epochs" << dendl;
-    need_publish = true;
   }
 
-  if (need_publish)
-    pl->publish_stats_to_osd();
+  pl->publish_stats_to_osd();
 
   if (ps->check_prior_readable_down_osds(advmap.osdmap)) {
     pl->recheck_readable();

From 111cee4f6cfcced799bd9876684b1d59fb4487bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 25 Aug 2023 21:01:07 +0200
Subject: [PATCH 0692/2492] mempool: switch from thread local storage to cpu
 local storage
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

https://github.com/ceph/ceph/pull/53130#issuecomment-1692353725

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/common/mempool.cc | 20 +++++++++++++++++---
 src/include/mempool.h | 14 ++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/common/mempool.cc b/src/common/mempool.cc
index 79354f708216..a1b83c2e9f86 100644
--- a/src/common/mempool.cc
+++ b/src/common/mempool.cc
@@ -15,9 +15,11 @@
 #include "include/mempool.h"
 #include "include/demangle.h"
 
+#ifndef _GNU_SOURCE
 // Thread local variables should save index, not &shard[index],
 // because shard[] is defined in the class
 static thread_local size_t thread_shard_index = mempool::num_shards;
+#endif
 
 // default to debug_mode off
 bool mempool::debug_mode = false;
@@ -95,9 +97,21 @@ size_t mempool::pool_t::allocated_items() const
 
 void mempool::pool_t::adjust_count(ssize_t items, ssize_t bytes)
 {
-  thread_shard_index = (thread_shard_index == num_shards) ? pick_a_shard_int() : thread_shard_index;
-  shard[thread_shard_index].items += items;
-  shard[thread_shard_index].bytes += bytes;
+#ifndef _GNU_SOURCE
+  // fallback for lack of sched_getcpu()
+  const size_t shard_index = []() {
+    if (thread_shard_index == num_shards) {
+      thread_shard_index = pick_a_shard_int();
+    }
+    return thread_shard_index;
+  }();
+#else
+  // the expected path: we alway pick the shard for a cpu core
+  // a thread is executing on.
+  const size_t shard_index = pick_a_shard_int();
+#endif
+  shard[shard_index].items += items;
+  shard[shard_index].bytes += bytes;
 }
 
 void mempool::pool_t::get_stats(
diff --git a/src/include/mempool.h b/src/include/mempool.h
index 076c62afe191..23d7e80dc9eb 100644
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -26,6 +26,10 @@
 #include <boost/container/flat_set.hpp>
 #include <boost/container/flat_map.hpp>
 
+#ifdef _GNU_SOURCE
+#  include <sched.h>
+#endif
+
 #include "common/Formatter.h"
 #include "common/ceph_atomic.h"
 #include "include/ceph_assert.h"
@@ -265,11 +269,21 @@ class pool_t {
   void adjust_count(ssize_t items, ssize_t bytes);
 
   static size_t pick_a_shard_int() {
+#ifndef _GNU_SOURCE
     // Dirt cheap, see:
     //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
     size_t me = (size_t)pthread_self();
     size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
     return i;
+#else
+    // a thread local storage is actually just an approximation;
+    // what we truly want is a _cpu local storage_.
+    //
+    // on the architectures we care about sched_getcpu() is
+    // a syscall-handled-in-userspace (vdso!). it grabs the cpu
+    // id kernel exposes to a task on context switch.
+    return sched_getcpu() & ((1 << num_shard_bits) - 1);
+#endif
   }
 
   shard_t* pick_a_shard() {

From fd05b5a1bcc9fda4b135e43be472a0ab29413f06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 29 Aug 2023 11:53:46 +0200
Subject: [PATCH 0693/2492] mempool: implement sharding for type_t::items
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It looks this counter was initially thought to be a debug one
but -- because of the `force_register` --  flag it is widely
used now.

Found by Yingxin Cheng <yingxin.cheng@intel.com>

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/common/mempool.cc |  8 +++-
 src/include/mempool.h | 88 +++++++++++++++++++++++--------------------
 src/test/test_c2c.cc  |  2 +-
 3 files changed, 55 insertions(+), 43 deletions(-)

diff --git a/src/common/mempool.cc b/src/common/mempool.cc
index a1b83c2e9f86..5aa1bda4e54a 100644
--- a/src/common/mempool.cc
+++ b/src/common/mempool.cc
@@ -127,8 +127,12 @@ void mempool::pool_t::get_stats(
     for (auto &p : type_map) {
       std::string n = ceph_demangle(p.second.type_name);
       stats_t &s = (*by_type)[n];
-      s.bytes = p.second.items * p.second.item_size;
-      s.items = p.second.items;
+      s.bytes = 0;
+      s.items = 0;
+      for (size_t i = 0 ; i < num_shards; ++i) {
+        s.bytes += p.second.shards[i].items * p.second.item_size;
+        s.items += p.second.shards[i].items;
+      }
     }
   }
 }
diff --git a/src/include/mempool.h b/src/include/mempool.h
index 23d7e80dc9eb..6b633c64d6fc 100644
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -205,6 +205,24 @@ enum {
   num_shards = 1 << num_shard_bits
 };
 
+static size_t pick_a_shard_int() {
+#ifndef _GNU_SOURCE
+  // Dirt cheap, see:
+  //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
+  size_t me = (size_t)pthread_self();
+  size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
+  return i;
+#else
+  // a thread local storage is actually just an approximation;
+  // what we truly want is a _cpu local storage_.
+  //
+  // on the architectures we care about sched_getcpu() is
+  // a syscall-handled-in-userspace (vdso!). it grabs the cpu
+  // id kernel exposes to a task on context switch.
+  return sched_getcpu() & ((1 << num_shard_bits) - 1);
+#endif
+}
+
 //
 // Align shard to a cacheline.
 //
@@ -244,9 +262,16 @@ const char *get_pool_name(pool_index_t ix);
 struct type_t {
   const char *type_name;
   size_t item_size;
-  ceph::atomic<ssize_t> items = {0};  // signed
+  struct type_shard_t {
+    ceph::atomic<ssize_t> items = {0}; // signed
+    char __padding[128 - sizeof(ceph::atomic<ssize_t>)];
+  } __attribute__ ((aligned (128)));
+  type_shard_t shards[num_shards];
 };
 
+static_assert(sizeof(type_t::type_shard_t) == 128,
+	      "type_shard_t should be cacheline-sized");
+
 struct type_info_hash {
   std::size_t operator()(const std::type_info& k) const {
     return k.hash_code();
@@ -259,6 +284,8 @@ class pool_t {
   mutable std::mutex lock;  // only used for types list
   std::unordered_map<const char *, type_t> type_map;
 
+  template<pool_index_t, typename T>
+  friend class pool_allocator;
 public:
   //
   // How much this pool consumes. O(<num_shards>)
@@ -268,29 +295,6 @@ class pool_t {
 
   void adjust_count(ssize_t items, ssize_t bytes);
 
-  static size_t pick_a_shard_int() {
-#ifndef _GNU_SOURCE
-    // Dirt cheap, see:
-    //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
-    size_t me = (size_t)pthread_self();
-    size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
-    return i;
-#else
-    // a thread local storage is actually just an approximation;
-    // what we truly want is a _cpu local storage_.
-    //
-    // on the architectures we care about sched_getcpu() is
-    // a syscall-handled-in-userspace (vdso!). it grabs the cpu
-    // id kernel exposes to a task on context switch.
-    return sched_getcpu() & ((1 << num_shard_bits) - 1);
-#endif
-  }
-
-  shard_t* pick_a_shard() {
-    size_t i = pick_a_shard_int();
-    return &shard[i];
-  }
-
   type_t *get_type(const std::type_info& ti, size_t size) {
     std::lock_guard<std::mutex> l(lock);
     auto p = type_map.find(ti.name());
@@ -353,11 +357,12 @@ class pool_allocator {
 
   T* allocate(size_t n, void *p = nullptr) {
     size_t total = sizeof(T) * n;
-    shard_t *shard = pool->pick_a_shard();
-    shard->bytes += total;
-    shard->items += n;
+    const auto shid = pick_a_shard_int();
+    auto& shard = pool->shard[shid];
+    shard.bytes += total;
+    shard.items += n;
     if (type) {
-      type->items += n;
+      type->shards[shid].items += n;
     }
     T* r = reinterpret_cast<T*>(new char[total]);
     return r;
@@ -365,22 +370,24 @@ class pool_allocator {
 
   void deallocate(T* p, size_t n) {
     size_t total = sizeof(T) * n;
-    shard_t *shard = pool->pick_a_shard();
-    shard->bytes -= total;
-    shard->items -= n;
+    const auto shid = pick_a_shard_int();
+    auto& shard = pool->shard[shid];
+    shard.bytes -= total;
+    shard.items -= n;
     if (type) {
-      type->items -= n;
+      type->shards[shid].items -= n;
     }
     delete[] reinterpret_cast<char*>(p);
   }
 
   T* allocate_aligned(size_t n, size_t align, void *p = nullptr) {
     size_t total = sizeof(T) * n;
-    shard_t *shard = pool->pick_a_shard();
-    shard->bytes += total;
-    shard->items += n;
+    const auto shid = pick_a_shard_int();
+    auto& shard = pool->shard[shid];
+    shard.bytes += total;
+    shard.items += n;
     if (type) {
-      type->items += n;
+      type->shards[shid].items += n;
     }
     char *ptr;
     int rc = ::posix_memalign((void**)(void*)&ptr, align, total);
@@ -392,11 +399,12 @@ class pool_allocator {
 
   void deallocate_aligned(T* p, size_t n) {
     size_t total = sizeof(T) * n;
-    shard_t *shard = pool->pick_a_shard();
-    shard->bytes -= total;
-    shard->items -= n;
+    const auto shid = pick_a_shard_int();
+    auto& shard = pool->shard[shid];
+    shard.bytes -= total;
+    shard.items -= n;
     if (type) {
-      type->items -= n;
+      type->shards[shid].items -= n;
     }
     aligned_free(p);
   }
diff --git a/src/test/test_c2c.cc b/src/test/test_c2c.cc
index 5fe1ac2f1fcd..1569be305e58 100644
--- a/src/test/test_c2c.cc
+++ b/src/test/test_c2c.cc
@@ -70,7 +70,7 @@ int main(int argc, const char **argv)
 	  while(1) {
 	    size_t i;
 	    if (sharding) {
-	      i = mempool::pool_t::pick_a_shard_int();
+	      i = mempool::pick_a_shard_int();
 	    } else {
 	      i = 0;
 	    }

From 8146f835f180f85fdc2cc9803f06a3e2f30744be Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Wed, 13 Sep 2023 14:09:53 +0800
Subject: [PATCH 0694/2492] test/test_mempool: fix to use
 mempool::pick_a_shard_int()

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/test/test_mempool.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/test_mempool.cc b/src/test/test_mempool.cc
index 9dd96682f360..a8e20a41747d 100644
--- a/src/test/test_mempool.cc
+++ b/src/test/test_mempool.cc
@@ -412,7 +412,7 @@ TEST(mempool, check_shard_select)
   for (size_t i = 0; i < samples; i++) {
     workers.push_back(
       std::thread([&](){
-          size_t i = mempool::pool_t::pick_a_shard_int();
+          size_t i = mempool::pick_a_shard_int();
           shards[i]++;
         }));
   }

From 1ac378defc98c4a610a2ce532452cc278248439b Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 23 Oct 2023 15:28:05 +0800
Subject: [PATCH 0695/2492] mempool: only enable cpu local storage in crimson

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/common/mempool.cc | 13 +++++++------
 src/include/mempool.h | 16 ++++++++--------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/common/mempool.cc b/src/common/mempool.cc
index 5aa1bda4e54a..128a3d1a16cc 100644
--- a/src/common/mempool.cc
+++ b/src/common/mempool.cc
@@ -15,7 +15,8 @@
 #include "include/mempool.h"
 #include "include/demangle.h"
 
-#ifndef _GNU_SOURCE
+#if defined(_GNU_SOURCE) && defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
+#else
 // Thread local variables should save index, not &shard[index],
 // because shard[] is defined in the class
 static thread_local size_t thread_shard_index = mempool::num_shards;
@@ -97,7 +98,11 @@ size_t mempool::pool_t::allocated_items() const
 
 void mempool::pool_t::adjust_count(ssize_t items, ssize_t bytes)
 {
-#ifndef _GNU_SOURCE
+#if defined(_GNU_SOURCE) && defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
+  // the expected path: we alway pick the shard for a cpu core
+  // a thread is executing on.
+  const size_t shard_index = pick_a_shard_int();
+#else
   // fallback for lack of sched_getcpu()
   const size_t shard_index = []() {
     if (thread_shard_index == num_shards) {
@@ -105,10 +110,6 @@ void mempool::pool_t::adjust_count(ssize_t items, ssize_t bytes)
     }
     return thread_shard_index;
   }();
-#else
-  // the expected path: we alway pick the shard for a cpu core
-  // a thread is executing on.
-  const size_t shard_index = pick_a_shard_int();
 #endif
   shard[shard_index].items += items;
   shard[shard_index].bytes += bytes;
diff --git a/src/include/mempool.h b/src/include/mempool.h
index 6b633c64d6fc..1091268e8554 100644
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -26,7 +26,7 @@
 #include <boost/container/flat_set.hpp>
 #include <boost/container/flat_map.hpp>
 
-#ifdef _GNU_SOURCE
+#if defined(_GNU_SOURCE) && defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
 #  include <sched.h>
 #endif
 
@@ -206,13 +206,7 @@ enum {
 };
 
 static size_t pick_a_shard_int() {
-#ifndef _GNU_SOURCE
-  // Dirt cheap, see:
-  //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
-  size_t me = (size_t)pthread_self();
-  size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
-  return i;
-#else
+#if defined(_GNU_SOURCE) && defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
   // a thread local storage is actually just an approximation;
   // what we truly want is a _cpu local storage_.
   //
@@ -220,6 +214,12 @@ static size_t pick_a_shard_int() {
   // a syscall-handled-in-userspace (vdso!). it grabs the cpu
   // id kernel exposes to a task on context switch.
   return sched_getcpu() & ((1 << num_shard_bits) - 1);
+#else
+  // Dirt cheap, see:
+  //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
+  size_t me = (size_t)pthread_self();
+  size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
+  return i;
 #endif
 }
 

From 931894f0f6f6003f80d68151d742dc9f9380e249 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 6 Nov 2023 14:53:54 +0800
Subject: [PATCH 0696/2492] mempool: only enable sharding for type_t::items in
 crimson

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/common/mempool.cc |  5 +++++
 src/include/mempool.h | 26 +++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/src/common/mempool.cc b/src/common/mempool.cc
index 128a3d1a16cc..4ecfaf81fc66 100644
--- a/src/common/mempool.cc
+++ b/src/common/mempool.cc
@@ -128,12 +128,17 @@ void mempool::pool_t::get_stats(
     for (auto &p : type_map) {
       std::string n = ceph_demangle(p.second.type_name);
       stats_t &s = (*by_type)[n];
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
       s.bytes = 0;
       s.items = 0;
       for (size_t i = 0 ; i < num_shards; ++i) {
         s.bytes += p.second.shards[i].items * p.second.item_size;
         s.items += p.second.shards[i].items;
       }
+#else
+      s.bytes = p.second.items * p.second.item_size;
+      s.items = p.second.items;
+#endif
     }
   }
 }
diff --git a/src/include/mempool.h b/src/include/mempool.h
index 1091268e8554..a6dca48dd6fe 100644
--- a/src/include/mempool.h
+++ b/src/include/mempool.h
@@ -262,16 +262,20 @@ const char *get_pool_name(pool_index_t ix);
 struct type_t {
   const char *type_name;
   size_t item_size;
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
   struct type_shard_t {
     ceph::atomic<ssize_t> items = {0}; // signed
     char __padding[128 - sizeof(ceph::atomic<ssize_t>)];
   } __attribute__ ((aligned (128)));
+  static_assert(sizeof(type_shard_t) == 128,
+                "type_shard_t should be cacheline-sized");
   type_shard_t shards[num_shards];
+#else
+// XXX: consider dropping this case for classic with perf tests
+  ceph::atomic<ssize_t> items = {0};  // signed
+#endif
 };
 
-static_assert(sizeof(type_t::type_shard_t) == 128,
-	      "type_shard_t should be cacheline-sized");
-
 struct type_info_hash {
   std::size_t operator()(const std::type_info& k) const {
     return k.hash_code();
@@ -362,7 +366,11 @@ class pool_allocator {
     shard.bytes += total;
     shard.items += n;
     if (type) {
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
       type->shards[shid].items += n;
+#else
+      type->items += n;
+#endif
     }
     T* r = reinterpret_cast<T*>(new char[total]);
     return r;
@@ -375,7 +383,11 @@ class pool_allocator {
     shard.bytes -= total;
     shard.items -= n;
     if (type) {
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
       type->shards[shid].items -= n;
+#else
+      type->items -= n;
+#endif
     }
     delete[] reinterpret_cast<char*>(p);
   }
@@ -387,7 +399,11 @@ class pool_allocator {
     shard.bytes += total;
     shard.items += n;
     if (type) {
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
       type->shards[shid].items += n;
+#else
+      type->items += n;
+#endif
     }
     char *ptr;
     int rc = ::posix_memalign((void**)(void*)&ptr, align, total);
@@ -404,7 +420,11 @@ class pool_allocator {
     shard.bytes -= total;
     shard.items -= n;
     if (type) {
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
       type->shards[shid].items -= n;
+#else
+      type->items -= n;
+#endif
     }
     aligned_free(p);
   }

From 8e9871eb962d40d177bee50f304efd92f53cf780 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Wed, 22 Nov 2023 15:59:05 -0500
Subject: [PATCH 0697/2492] unittesting/rgw/lc: [non-functional change] remove
 commented out include statement

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/test/rgw/test_rgw_lc.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 83a4cac676df..5c64afa123ad 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -5,7 +5,6 @@
 #include "rgw_lc.h"
 #include "rgw_lc_s3.h"
 #include <gtest/gtest.h>
-//#include <spawn/spawn.hpp>
 #include <string>
 #include <vector>
 #include <stdexcept>

From ec298091b5e754293e798c90fc8afc3e4504d632 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 23 Nov 2023 14:59:24 +1000
Subject: [PATCH 0698/2492] doc/rados: edit "client can't connect..."

Edit the section "Client can't connect or mount" in the file
doc/rados/troubleshooting/troubleshooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index e00d391ebef5..2859431d11d8 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -481,12 +481,13 @@ Clock Skew Questions and Answers
 Client Can't Connect or Mount
 -----------------------------
 
-Check your IP tables. Some operating-system install utilities add a ``REJECT``
-rule to ``iptables``. ``iptables`` rules will reject all clients other than
-``ssh`` that try to connect to the host. If your monitor host's IP tables have
-a ``REJECT`` rule in place, clients that are connecting from a separate node
-will fail and will raise a timeout error. Any ``iptables`` rules that reject
-clients trying to connect to Ceph daemons must be addressed. For example::
+If a client can't connect to the cluster or mount, check your iptables. Some
+operating-system install utilities add a ``REJECT`` rule to ``iptables``.
+``iptables`` rules will reject all clients other than ``ssh`` that try to
+connect to the host. If your monitor host's iptables have a ``REJECT`` rule in
+place, clients that connect from a separate node will fail, and this will raise
+a timeout error. Look for ``iptables`` rules that reject clients that are
+trying to connect to Ceph daemons. For example::
 
     REJECT all -- anywhere anywhere reject-with icmp-host-prohibited
 

From 2e0b4ae3afc917b7ce0b55879310b41017f0f627 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Sun, 19 Nov 2023 17:12:51 +0530
Subject: [PATCH 0699/2492] mgr/vol: add comments in async_cloner

It's not obvious where the clone creation is performed. It's especially
obfuscated because we call the function/method for cloning indirectly.
Therefore, add comments where necessary and mark it with heavy rating.

Same goes for all other "subvolume snapshot clone" operations.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/pybind/mgr/volumes/fs/async_cloner.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py
index 95f7d64e1b36..af7f9bb8e0ce 100644
--- a/src/pybind/mgr/volumes/fs/async_cloner.py
+++ b/src/pybind/mgr/volumes/fs/async_cloner.py
@@ -224,6 +224,8 @@ def do_clone(fs_client, volspec, volname, groupname, subvolname, should_cancel):
         with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes:
             src_path = clone_volumes[1].snapshot_data_path(clone_volumes[2])
             dst_path = clone_volumes[0].path
+            # XXX: this is where cloning (of subvolume's snapshots) actually
+            # happens.
             bulk_copy(fs_handle, src_path, dst_path, should_cancel)
             set_quota_on_clone(fs_handle, clone_volumes)
 
@@ -287,9 +289,14 @@ def start_clone_sm(fs_client, volspec, volname, index, groupname, subvolname, st
             time.sleep(snapshot_clone_delay)
             log.info("Delayed cloning ({0}, {1}, {2}) -- by {3} seconds".format(volname, groupname, subvolname, snapshot_clone_delay))
         while not finished:
+            # XXX: this is where request operation is mapped to relevant
+            # function.
             handler = state_table.get(current_state, None)
             if not handler:
                 raise VolumeException(-errno.EINVAL, "invalid clone state: \"{0}\"".format(current_state))
+            # XXX: this is where the requested operation for subvolume's
+            # snapshot clone is performed. the function for the request
+            # operation is run through "handler".
             (next_state, finished) = handler(fs_client, volspec, volname, index, groupname, subvolname, should_cancel)
             if next_state:
                 log.debug("({0}, {1}, {2}) transition state [\"{3}\" => \"{4}\"]".format(volname, groupname, subvolname,\

From 2446c0191b70b4e7bdd8fc51b29924366bce6150 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Wed, 22 Nov 2023 16:00:15 -0500
Subject: [PATCH 0700/2492] unittesting/rgw/lc: add testcases for the units
 should_work and schedule_next_start_time

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/test/rgw/test_rgw_lc.cc | 235 ++++++++++++++++++++++++++++++++++++
 1 file changed, 235 insertions(+)

diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 5c64afa123ad..2943094c6d01 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -106,3 +106,238 @@ TEST(TestLCFilterInvalidAnd, XMLDoc3)
   /* check our flags */
   ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
 }
+
+struct LCWorkTimeTests : ::testing::Test
+{
+   CephContext* cct;
+   std::unique_ptr<RGWLC::LCWorker> worker;
+
+   // expects input in the form of "%m/%d/%y %H:%M:%S"; e.g., "01/15/23 23:59:01"
+   utime_t get_utime_by_date_time_string(const std::string& date_time_str)
+   {
+      struct tm tm{};
+      struct timespec ts = {0};
+
+      strptime(date_time_str.c_str(), "%m/%d/%y %H:%M:%S", &tm);
+      ts.tv_sec = mktime(&tm);
+
+      return utime_t(ts);
+   }
+
+   // expects a map from input value (date & time string) to expected result (boolean)
+   void run_should_work_test(const auto& test_values_to_expectations_map) {
+      for (const auto& [date_time_str, expected_value] : test_values_to_expectations_map) {
+         auto ut = get_utime_by_date_time_string(date_time_str);
+         auto should_work = worker->should_work(ut);
+
+         ASSERT_EQ(should_work, expected_value)
+            << "input time: " << ut
+            << " expected: " << expected_value
+            << " should_work: " << should_work
+            << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+      }
+   }
+
+   // expects a map from input value (a tuple of date & time strings) to expected result (seconds)
+   void run_schedule_next_start_time_test(const auto& test_values_to_expectations_map) {
+      for (const auto& [date_time_str_tuple, expected_value] : test_values_to_expectations_map) {
+         auto work_started_at = get_utime_by_date_time_string(std::get<0>(date_time_str_tuple));
+         auto work_completed_at = get_utime_by_date_time_string(std::get<1>(date_time_str_tuple));
+         auto wait_secs_till_next_start = worker->schedule_next_start_time(work_started_at, work_completed_at);
+
+         ASSERT_EQ(wait_secs_till_next_start, expected_value)
+            << "work_started_at: " << work_started_at
+            << " work_completed_at: " << work_completed_at
+            << " expected: " << expected_value
+            << " wait_secs_till_next_start: " << wait_secs_till_next_start
+            << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+      }
+   }
+
+protected:
+
+   void SetUp() override {
+      cct = (new CephContext(CEPH_ENTITY_TYPE_ANY))->get();
+
+      cct->_conf->set_value("rgw_lc_max_wp_worker", 0, 0); // no need to create a real workpool
+      worker = std::make_unique<RGWLC::LCWorker>(nullptr, cct, nullptr, 0);
+   }
+
+   void TearDown() override {
+      worker.reset();
+      cct->put();
+   }
+};
+
+TEST_F(LCWorkTimeTests, ShouldWorkDefaultWorkTime)
+{
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", true},
+      {"01/01/24 00:00:00", true}, // date is not relevant, but only the time-window
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 03:00:00", true},
+      {"01/01/23 05:59:59", true},
+      {"01/01/23 06:00:00", true},
+      {"01/01/23 06:00:59", true}, // seconds don't matter, but only hours and minutes
+      {"01/01/23 06:01:00", false},
+      {"01/01/23 23:59:59", false},
+      {"01/02/23 23:59:59", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 14:00:00", false}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-16:00";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/24 13:59:59", false},
+      {"01/01/23 14:00:00", true},
+      {"01/01/23 16:00:00", true},
+      {"01/01/23 16:00:59", true},
+      {"01/01/23 16:01:00", false},
+      {"01/01/23 17:00:00", false},
+      {"01/01/23 23:59:59", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay24Hours)
+{
+   cct->_conf->rgw_lifecycle_work_time = "00:00-23:59";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 23:59:00", true},
+      {"01/01/23 23:59:59", true},
+      {"01/01/23 00:00:00", true},
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 00:01:00", true},
+      {"01/01/23 01:00:00", true},
+      {"01/01/23 12:00:00", true},
+      {"01/01/23 17:00:00", true},
+      {"01/01/23 23:00:00", true}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-01:00";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 13:59:00", false},
+      {"01/01/23 13:59:59", false},
+      {"01/01/24 14:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 17:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 23:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 00:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 01:00:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 01:01:00", false},
+      {"01/01/23 05:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 13:00:00", false}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay24Hours)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-13:59";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 00:00:01", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 00:01:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 12:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 13:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 13:59:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 13:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 14:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 14:00:01", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 14:01:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 16:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 23:59:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 23:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDayIrregularMins)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 22:14:59", false},
+      {"01/01/23 22:15:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 02:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 03:33:00", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 03:33:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 03:34:00", false},
+      {"01/01/23 04:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 22:00:00", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeStartEndSameHour)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-22:45";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 22:14:59", false},
+      {"01/01/23 22:15:00", true},
+      {"01/01/24 22:44:59", true},
+      {"01/01/24 22:45:59", true},
+      {"01/01/24 22:46:00", false},
+      {"01/01/23 23:00:00", false},
+      {"01/01/23 00:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 21:00:00", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ScheduleNextStartTime)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+   // items of the map: [ (work_started_time, work_completed_time), expected_value (seconds) ]
+   //
+   // expected_value is the difference between configured start time (i.e, 22:15:00) and
+   // the second item of the tuple (i.e., work_completed_time).
+   //
+   // Note that "seconds" of work completion time is taken into account but date is not relevant.
+   // e.g., the first testcase: 75713 == 01:13:07 - 22:15:00 (https://tinyurl.com/ydm86752)
+   std::map<std::tuple<std::string, std::string>, int> test_values_to_expectations = {
+      {{"01/01/23 22:15:05", "01/01/23 01:13:07"}, 75713},
+      {{"01/01/23 22:15:05", "01/02/23 01:13:07"}, 75713},
+      {{"01/01/23 22:15:05", "01/01/23 22:17:07"}, 86273},
+      {{"01/01/23 22:15:05", "01/02/23 22:17:07"}, 86273},
+      {{"01/01/23 22:15:05", "01/01/23 22:14:00"}, 60},
+      {{"01/01/23 22:15:05", "01/02/23 22:14:00"}, 60},
+      {{"01/01/23 22:15:05", "01/01/23 22:15:00"}, 24 * 60 * 60},
+      {{"01/01/23 22:15:05", "01/02/23 22:15:00"}, 24 * 60 * 60},
+      {{"01/01/23 22:15:05", "01/01/23 22:15:01"}, 24 * 60 * 60 - 1},
+      {{"01/01/23 22:15:05", "01/02/23 22:15:01"}, 24 * 60 * 60 - 1},
+   };
+
+   run_schedule_next_start_time_test(test_values_to_expectations);
+}

From b3afae981a389118abf1557ab6eb237be21bc157 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Wed, 22 Nov 2023 15:32:20 -0500
Subject: [PATCH 0701/2492] rgw/lc: [non-functional change] use const expr to
 represent seconds in a day

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_lc.cc | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 4ae3a950ade6..b425f1d19ee0 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -41,6 +41,9 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_rgw
 
+constexpr int32_t hours_in_a_day = 24;
+constexpr int32_t secs_in_a_day = hours_in_a_day * 60 * 60;
+
 using namespace std;
 
 const char* LC_STATUS[] = {
@@ -289,7 +292,7 @@ static bool obj_has_expired(const DoutPrefixProvider *dpp, CephContext *cct, cep
   utime_t base_time;
   if (cct->_conf->rgw_lc_debug_interval <= 0) {
     /* Normal case, run properly */
-    cmp = double(days)*24*60*60;
+    cmp = double(days) * secs_in_a_day;
     base_time = ceph_clock_now().round_to_day();
   } else {
     /* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */
@@ -1944,8 +1947,7 @@ bool RGWLC::expired_session(time_t started)
   }
 
   time_t interval = (cct->_conf->rgw_lc_debug_interval > 0)
-    ? cct->_conf->rgw_lc_debug_interval
-    : 24*60*60;
+    ? cct->_conf->rgw_lc_debug_interval : secs_in_a_day;
 
   auto now = time(nullptr);
 
@@ -1961,8 +1963,7 @@ bool RGWLC::expired_session(time_t started)
 time_t RGWLC::thread_stop_at()
 {
   uint64_t interval = (cct->_conf->rgw_lc_debug_interval > 0)
-    ? cct->_conf->rgw_lc_debug_interval
-    : 24*60*60;
+    ? cct->_conf->rgw_lc_debug_interval : secs_in_a_day;
 
   return time(nullptr) + interval;
 }
@@ -2053,7 +2054,7 @@ static inline bool allow_shard_rollover(CephContext* cct, time_t now, time_t sha
    *    - the current shard has not rolled over in the last 24 hours
    */
   if (((shard_rollover_date < now) &&
-       (now - shard_rollover_date > 24*60*60)) ||
+       (now - shard_rollover_date > secs_in_a_day)) ||
       (! shard_rollover_date /* no rollover date stored */) ||
       (cct->_conf->rgw_lc_debug_interval > 0 /* defaults to -1 == disabled */)) {
     return true;
@@ -2079,7 +2080,7 @@ static inline bool already_run_today(CephContext* cct, time_t start_date)
   bdt.tm_min = 0;
   bdt.tm_sec = 0;
   begin_of_day = mktime(&bdt);
-  if (now - begin_of_day < 24*60*60)
+  if (now - begin_of_day < secs_in_a_day)
     return true;
   else
     return false;
@@ -2465,7 +2466,7 @@ int RGWLC::LCWorker::schedule_next_start_time(utime_t &start, utime_t& now)
   nt = mktime(&bdt);
   secs = nt - tt;
 
-  return secs>0 ? secs : secs+24*60*60;
+  return secs > 0 ? secs : secs + secs_in_a_day;
 }
 
 RGWLC::LCWorker::~LCWorker()
@@ -2756,7 +2757,7 @@ std::string s3_expiration_header(
       if (rule_expiration.has_days()) {
 	rule_expiration_date =
 	  boost::optional<ceph::real_time>(
-	    mtime + make_timespan(double(rule_expiration.get_days())*24*60*60 - ceph::real_clock::to_time_t(mtime)%(24*60*60) + 24*60*60));
+	    mtime + make_timespan(double(rule_expiration.get_days()) * secs_in_a_day - ceph::real_clock::to_time_t(mtime)%(secs_in_a_day) + secs_in_a_day));
       }
     }
 
@@ -2835,7 +2836,7 @@ bool s3_multipart_abort_header(
     std::optional<ceph::real_time> rule_abort_date;
     if (mp_expiration.has_days()) {
       rule_abort_date = std::optional<ceph::real_time>(
-              mtime + make_timespan(mp_expiration.get_days()*24*60*60 - ceph::real_clock::to_time_t(mtime)%(24*60*60) + 24*60*60));
+              mtime + make_timespan(mp_expiration.get_days() * secs_in_a_day - ceph::real_clock::to_time_t(mtime)%(secs_in_a_day) + secs_in_a_day));
     }
 
     // update earliest abort date

From 353360581a0b802435ecf4273ba79f53deb710e6 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 23 Nov 2023 00:12:29 -0500
Subject: [PATCH 0702/2492] cmake/rocksdb: make sure dependencies build before
 rocksdb

some of rocksdb's dependencies may not have built by the time its
ExternalProject starts, so it can fail with missing headers or
libraries. for example, `uring::uring` may itself be an ExternalProject,
and its include directory won't exist until it starts building:

```
[89/1345] Performing configure step for 'rocksdb_ext'
FAILED: src/rocksdb_ext-prefix/src/rocksdb_ext-stamp/rocksdb_ext-configure build/src/rocksdb_ext-prefix/src/rocksdb_ext-stamp/rocksdb_ext-configure
...
CMake Error in CMakeLists.txt:
  Imported target "uring::uring" includes non-existent path

    "build/src/liburing/src/include"

  in its INTERFACE_INCLUDE_DIRECTORIES.  Possible reasons include:

  * The path was deleted, renamed, or moved to another location.

  * An install or uninstall procedure did not complete successfully.

  * The installation package was faulty and references files it does not
  provide.
...
[91/1345] Performing download step (git clone) for 'liburing_ext'
Cloning into 'liburing'...
```

use `add_dependencies(rocksdb_ext)` to make sure all of its dependencies
are available before starting the build

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 cmake/modules/BuildRocksDB.cmake | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmake/modules/BuildRocksDB.cmake b/cmake/modules/BuildRocksDB.cmake
index 97a101edfec3..f9a28274c40d 100644
--- a/cmake/modules/BuildRocksDB.cmake
+++ b/cmake/modules/BuildRocksDB.cmake
@@ -91,6 +91,9 @@ function(build_rocksdb)
     INSTALL_COMMAND ""
     LIST_SEPARATOR !)
 
+  # make sure all the link libraries are built first
+  add_dependencies(rocksdb_ext ${rocksdb_INTERFACE_LINK_LIBRARIES})
+
   add_library(RocksDB::RocksDB STATIC IMPORTED)
   add_dependencies(RocksDB::RocksDB rocksdb_ext)
   set(rocksdb_INCLUDE_DIR "${rocksdb_SOURCE_DIR}/include")

From 48e189fe0ec6c0ace23ddaed09ef3f936e8c2a6f Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Wed, 22 Nov 2023 15:37:00 -0500
Subject: [PATCH 0703/2492] rgw/lc: adjust timing if the configured end
 work-time is less than the start-time

LC work time is given in the form of "HH:MM-HH:MM"; the first part
represents the "start_time" and the next "end_time". "should_work"
function decides, given the current time, whether the LC worker can resume.
It essentially checks whether the current time is within start and end times.

Since there's no "date" (month/day/year) notion taken into account, a work time
whose end_time (i.e., hour field) is less than start_time is not properly
handled by "should_work". For example, "14:00-13:59" would normally mean to
start LC processing at 2PM local time and allowing it to run for 24 hours. So,
given such a work time range, "should_work" must return true for any given
current_time. However, without this adjustment, it always returns false.

The fix simply adds a "next day" adjustment if the end_time is
configured to be less than the configured start_time.

Fixes https://tracker.ceph.com/issues/63613

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_lc.cc           |  6 ++++
 src/test/rgw/test_rgw_lc.cc | 55 +++++++++++++++++++------------------
 2 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index b425f1d19ee0..a6e9b2c4af0d 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -2426,6 +2426,12 @@ bool RGWLC::LCWorker::should_work(utime_t& now)
   time_t tt = now.sec();
   localtime_r(&tt, &bdt);
 
+  // next-day adjustment if the configured end_hour is less than start_hour
+  if (end_hour < start_hour) {
+    bdt.tm_hour = bdt.tm_hour > end_hour ? bdt.tm_hour : bdt.tm_hour + hours_in_a_day;
+    end_hour += hours_in_a_day;
+  }
+
   if (cct->_conf->rgw_lc_debug_interval > 0) {
 	  /* We're debugging, so say we can run */
 	  return true;
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 2943094c6d01..d10b482cbfce 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -235,13 +235,13 @@ TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay)
    std::unordered_map<std::string, bool> test_values_to_expectations = {
       {"01/01/23 13:59:00", false},
       {"01/01/23 13:59:59", false},
-      {"01/01/24 14:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 17:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 23:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 00:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 01:00:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/24 14:00:00", true}, // used-to-fail
+      {"01/01/24 17:00:00", true}, // used-to-fail
+      {"01/01/24 23:59:59", true}, // used-to-fail
+      {"01/01/23 00:00:00", true}, // used-to-fail
+      {"01/01/23 00:59:59", true}, // used-to-fail
+      {"01/01/23 01:00:00", true}, // used-to-fail
+      {"01/01/23 01:00:59", true}, // used-to-fail
       {"01/01/23 01:01:00", false},
       {"01/01/23 05:00:00", false},
       {"01/01/23 12:00:00", false},
@@ -255,21 +255,22 @@ TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay24Hours)
 {
    cct->_conf->rgw_lifecycle_work_time = "14:00-13:59";
 
+   // all of the below cases used-to-fail
    std::unordered_map<std::string, bool> test_values_to_expectations = {
-      {"01/01/23 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 00:00:01", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 00:01:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 12:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 13:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 13:59:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 13:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 14:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 14:00:01", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 14:01:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 16:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 23:59:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 23:59:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 00:00:00", true},
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 00:01:00", true},
+      {"01/01/24 01:00:00", true},
+      {"01/01/24 12:00:00", true},
+      {"01/01/24 13:00:00", true},
+      {"01/01/24 13:59:00", true},
+      {"01/01/24 13:59:59", true},
+      {"01/01/23 14:00:00", true},
+      {"01/01/23 14:00:01", true},
+      {"01/01/23 14:01:00", true},
+      {"01/01/23 16:00:00", true},
+      {"01/01/23 23:59:00", true},
+      {"01/01/23 23:59:59", true},
    };
 
    run_should_work_test(test_values_to_expectations);
@@ -281,12 +282,12 @@ TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDayIrregularMins
 
    std::unordered_map<std::string, bool> test_values_to_expectations = {
       {"01/01/23 22:14:59", false},
-      {"01/01/23 22:15:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 00:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 01:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/24 02:00:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 03:33:00", false}, // should have been true, expected to fail due to tracker issue #63613
-      {"01/01/23 03:33:59", false}, // should have been true, expected to fail due to tracker issue #63613
+      {"01/01/23 22:15:00", true}, // used-to-fail
+      {"01/01/24 00:00:00", true}, // used-to-fail
+      {"01/01/24 01:00:00", true}, // used-to-fail
+      {"01/01/24 02:00:00", true}, // used-to-fail
+      {"01/01/23 03:33:00", true}, // used-to-fail
+      {"01/01/23 03:33:59", true}, // used-to-fail
       {"01/01/23 03:34:00", false},
       {"01/01/23 04:00:00", false},
       {"01/01/23 12:00:00", false},

From ea25085a8743a4696f70220d2d496179b4f425b2 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Wed, 22 Nov 2023 16:12:48 -0500
Subject: [PATCH 0704/2492] rgw/lc/rgw_lifecycle_work_time: add more
 explanation to long_desc

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/common/options/rgw.yaml.in | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index b03e4449ab1b..3f0f691e84e7 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -359,7 +359,11 @@ options:
   type: str
   level: advanced
   desc: Lifecycle allowed work time
-  long_desc: Local time window in which the lifecycle maintenance thread can work.
+  long_desc: Local time window in which the lifecycle maintenance thread can work. It expects
+    24-hour time notation. For example, "00:00-23:59" means starting at midnight lifecycle
+    is allowed to run for the whole day (24 hours). When lifecycle completes, it waits for the
+    next maintenance window. In this example, if it completes at 01:00, it will resume processing
+    23 hours later at the following midnight.
   default: 00:00-06:00
   services:
   - rgw

From 21a1e1cd953248f01a609442293d876deca0afba Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Thu, 23 Nov 2023 15:21:15 +0800
Subject: [PATCH 0705/2492] qa/fscrypt: enable debug log for bluestore

We need to get more debug logs from bluestore to know what exactly
has happened for the extent map.

URL: https://tracker.ceph.com/issues/63586
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 qa/suites/fs/fscrypt/overrides/osd.yaml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 qa/suites/fs/fscrypt/overrides/osd.yaml

diff --git a/qa/suites/fs/fscrypt/overrides/osd.yaml b/qa/suites/fs/fscrypt/overrides/osd.yaml
new file mode 100644
index 000000000000..c78ccec1d622
--- /dev/null
+++ b/qa/suites/fs/fscrypt/overrides/osd.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        debug bluestore: 20

From f0e150a9df0fcae574d22d2e715b323691163daa Mon Sep 17 00:00:00 2001
From: wanwencong <wanwc@chinatelecom.cn>
Date: Thu, 23 Nov 2023 14:31:22 +0800
Subject: [PATCH 0706/2492] mgr: set MMgrBeacon high priority

It's possible mgr beacons can get stuck in the dispatch queue long enough for an
Mgr to be removed from the MgrMap, increase the priority to ensure
processing ahead of monitor commands and other low-priority messages.

Fixes: https://tracker.ceph.com/issues/63615
Signed-off-by: wanwencong <wanwc@chinatelecom.cn>
---
 src/messages/MMgrBeacon.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h
index 1799e2f7168c..ed670d0d8902 100644
--- a/src/messages/MMgrBeacon.h
+++ b/src/messages/MMgrBeacon.h
@@ -53,7 +53,9 @@ class MMgrBeacon final : public PaxosServiceMessage {
   MMgrBeacon()
     : PaxosServiceMessage{MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION},
       gid(0), available(false)
-  {}
+  {
+    set_priority(CEPH_MSG_PRIO_HIGH);
+  }
 
   MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_,
              entity_addrvec_t server_addrs_, bool available_,
@@ -67,6 +69,7 @@ class MMgrBeacon final : public PaxosServiceMessage {
       clients(std::move(clients_)),
       mgr_features(feat)
   {
+    set_priority(CEPH_MSG_PRIO_HIGH);
   }
 
   uint64_t get_gid() const { return gid; }

From 910e5efea19eacd204025fa58f37c27c4c0b2963 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 23 Nov 2023 21:54:54 +1000
Subject: [PATCH 0707/2492] doc/rados: format sections in tshooting-mon.rst

Format definition lists in the section "Do I need to restart a monitor
to adjust debug levels?" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/troubleshooting-mon.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index e00d391ebef5..1550aaccd802 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -673,7 +673,7 @@ Do I need to restart a monitor to adjust debug levels?
 No, restarting a monitor is not necessary. Debug levels may be adjusted by
 using two different methods, depending on whether or not there is a quorum:
 
-There is a quorum
+**If there is a quorum**
 
   Either inject the debug option into the specific monitor that needs to 
   be debugged::
@@ -685,7 +685,7 @@ There is a quorum
         ceph tell mon.* config set debug_mon 10/10
 
 
-There is no quorum
+**If there is no quorum**
 
   Use the admin socket of the specific monitor that needs to be debugged
   and directly adjust the monitor's configuration options::

From 4f2ac201ecad6abbdb7fa0bfd21ed1b86282d218 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 23 Nov 2023 14:29:04 +0000
Subject: [PATCH 0708/2492] rgw/test/kafka: let consumer read events from the
 beginning

in some cases the RGW may publish the notifications before the kafka
consumer started reading the events from the topic

Fixes: https://tracker.ceph.com/issues/62136

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/test/rgw/bucket_notification/test_bn.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 32fe262bdc31..8860df1555ef 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -434,7 +434,8 @@ def __init__(self, topic, security_type):
                 self.consumer = KafkaConsumer(topic, 
                         bootstrap_servers = kafka_server+':'+str(port), 
                         security_protocol=security_type,
-                        consumer_timeout_ms=16000)
+                        consumer_timeout_ms=16000,
+                        auto_offset_reset='earliest')
                 print('Kafka consumer created on topic: '+topic)
                 break
             except Exception as error:
@@ -1534,8 +1535,8 @@ def test_ps_s3_notification_push_kafka_on_master():
         time_diff = time.time() - start_time
         print('average time for creation + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
 
-        print('wait for 5sec for the messages...')
-        time.sleep(5)
+        print('wait for 10sec for the messages...')
+        time.sleep(10)
         keys = list(bucket.list())
         receiver.verify_s3_events(keys, exact_match=True, etags=etags)
 
@@ -1551,8 +1552,8 @@ def test_ps_s3_notification_push_kafka_on_master():
         time_diff = time.time() - start_time
         print('average time for deletion + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
 
-        print('wait for 5sec for the messages...')
-        time.sleep(5)
+        print('wait for 10sec for the messages...')
+        time.sleep(10)
         receiver.verify_s3_events(keys, exact_match=True, deletions=True, etags=etags)
     except Exception as e:
         assert False, str(e)

From 08e655d8baef3e1b2d7101c291e6e9626ecb65c7 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Mon, 23 Oct 2023 13:46:06 +0000
Subject: [PATCH 0709/2492] rgw/lua: print luarocks config during reload

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/rgw_lua.cc | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_lua.cc b/src/rgw/rgw_lua.cc
index 6a5780a3eb1c..5b518d2e27cb 100644
--- a/src/rgw/rgw_lua.cc
+++ b/src/rgw/rgw_lua.cc
@@ -213,11 +213,33 @@ int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
     return rc;
   }
   install_dir.assign(tmp_luarocks_path);
+  
+  {  
+    bp::ipstream is;
+    const auto cmd = p.string() + " config";
+    bp::child c(cmd, bp::std_in.close(), (bp::std_err & bp::std_out) > is);
+    
+    std::string lines = std::string("Lua CMD: ") + cmd;
+    std::string line;
+    
+    do {
+      if (!line.empty()) {
+        lines.append("\n\t");
+        lines.append(line);
+      }
+    } while (c.running() && std::getline(is, line));
+
+    c.wait();
+    line = "exit code: " + std::to_string(c.exit_code());
+    lines.append("\n\t");
+    lines.append(line);
+    ldpp_dout(dpp, 20) << lines << dendl;
+  }
 
   // the lua rocks install dir will be created by luarocks the first time it is called
   for (const auto& package : packages) {
     bp::ipstream is;
-    const auto cmd = p.string() + " install --lua-version " + CEPH_LUA_VERSION + " --tree " + install_dir + " --deps-mode one " + package;
+    const auto cmd = p.string() + " install --no-doc --no-manifest --check-lua-versions --lua-version " + CEPH_LUA_VERSION + " --tree " + install_dir + " --deps-mode one " + package;
     bp::child c(cmd, bp::std_in.close(), (bp::std_err & bp::std_out) > is);
 
     // once package reload is supported, code should yield when reading output

From 46500cace6c07e3848e19aa764b23acc480092dd Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Mon, 3 Jul 2023 07:06:51 +0000
Subject: [PATCH 0710/2492] rgw/test/lua: add lua integration tests suite

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>

Fixes: https://tracker.ceph.com/issues/63616
---
 ceph.spec.in                            |   3 +-
 debian/control                          |   3 +-
 qa/suites/rgw/lua/%                     |   0
 qa/suites/rgw/lua/.qa                   |   1 +
 qa/suites/rgw/lua/beast.yaml            |   1 +
 qa/suites/rgw/lua/bluestore-bitmap.yaml |   1 +
 qa/suites/rgw/lua/fixed-2.yaml          |   1 +
 qa/suites/rgw/lua/overrides.yaml        |   9 +
 qa/suites/rgw/lua/supported-distros     |   1 +
 qa/suites/rgw/lua/tasks/+               |   0
 qa/suites/rgw/lua/tasks/0-install.yaml  |  13 +
 qa/suites/rgw/lua/tasks/test_lua.yaml   |   5 +
 qa/tasks/lua_tests.py                   | 254 +++++++++++++
 src/rgw/rgw_lua.cc                      |  90 +++--
 src/test/rgw/lua/README.rst             |   9 +
 src/test/rgw/lua/__init__.py            |  60 +++
 src/test/rgw/lua/luatests.conf.SAMPLE   |  10 +
 src/test/rgw/lua/pytest.ini             |   5 +
 src/test/rgw/lua/requirements.txt       |   2 +
 src/test/rgw/lua/test_lua.py            | 476 ++++++++++++++++++++++++
 src/test/rgw/lua/tox.ini                |   9 +
 21 files changed, 917 insertions(+), 36 deletions(-)
 create mode 100644 qa/suites/rgw/lua/%
 create mode 120000 qa/suites/rgw/lua/.qa
 create mode 120000 qa/suites/rgw/lua/beast.yaml
 create mode 120000 qa/suites/rgw/lua/bluestore-bitmap.yaml
 create mode 120000 qa/suites/rgw/lua/fixed-2.yaml
 create mode 100644 qa/suites/rgw/lua/overrides.yaml
 create mode 120000 qa/suites/rgw/lua/supported-distros
 create mode 100644 qa/suites/rgw/lua/tasks/+
 create mode 100644 qa/suites/rgw/lua/tasks/0-install.yaml
 create mode 100644 qa/suites/rgw/lua/tasks/test_lua.yaml
 create mode 100644 qa/tasks/lua_tests.py
 create mode 100644 src/test/rgw/lua/README.rst
 create mode 100644 src/test/rgw/lua/__init__.py
 create mode 100644 src/test/rgw/lua/luatests.conf.SAMPLE
 create mode 100644 src/test/rgw/lua/pytest.ini
 create mode 100644 src/test/rgw/lua/requirements.txt
 create mode 100644 src/test/rgw/lua/test_lua.py
 create mode 100644 src/test/rgw/lua/tox.ini

diff --git a/ceph.spec.in b/ceph.spec.in
index aef26acf451d..3807f48280e9 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -283,7 +283,8 @@ BuildRequires:  librabbitmq-devel
 BuildRequires:  librdkafka-devel
 %endif
 %if 0%{with lua_packages}
-BuildRequires:  %{luarocks_package_name}
+Requires:	lua-devel
+Requires:  %{luarocks_package_name}
 %endif
 %if 0%{with make_check}
 BuildRequires:  hostname
diff --git a/debian/control b/debian/control
index 8263e3251835..5ae413ce774a 100644
--- a/debian/control
+++ b/debian/control
@@ -72,7 +72,6 @@ Build-Depends: automake,
                libre2-dev,
                libutf8proc-dev (>= 2.2.0),
                librdkafka-dev,
-               luarocks,
                libthrift-dev (>= 0.13.0),
                libyaml-cpp-dev (>= 0.6),
                libzstd-dev <pkg.ceph.check>,
@@ -919,6 +918,8 @@ Section: libs
 Depends: librados2 (= ${binary:Version}),
          ${misc:Depends},
          ${shlibs:Depends},
+         liblua5.3-dev,
+         luarocks,
 Description: RADOS Gateway client library
  RADOS is a distributed object store used by the Ceph distributed
  storage system.  This package provides a REST gateway to the
diff --git a/qa/suites/rgw/lua/% b/qa/suites/rgw/lua/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/lua/.qa b/qa/suites/rgw/lua/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rgw/lua/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rgw/lua/beast.yaml b/qa/suites/rgw/lua/beast.yaml
new file mode 120000
index 000000000000..09ced62c42aa
--- /dev/null
+++ b/qa/suites/rgw/lua/beast.yaml
@@ -0,0 +1 @@
+.qa/rgw_frontend/beast.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/lua/bluestore-bitmap.yaml b/qa/suites/rgw/lua/bluestore-bitmap.yaml
new file mode 120000
index 000000000000..a59cf5175069
--- /dev/null
+++ b/qa/suites/rgw/lua/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+.qa/objectstore/bluestore-bitmap.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/lua/fixed-2.yaml b/qa/suites/rgw/lua/fixed-2.yaml
new file mode 120000
index 000000000000..230ff0fdab41
--- /dev/null
+++ b/qa/suites/rgw/lua/fixed-2.yaml
@@ -0,0 +1 @@
+.qa/clusters/fixed-2.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/lua/overrides.yaml b/qa/suites/rgw/lua/overrides.yaml
new file mode 100644
index 000000000000..be4b4621542f
--- /dev/null
+++ b/qa/suites/rgw/lua/overrides.yaml
@@ -0,0 +1,9 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        setuser: ceph
+        setgroup: ceph
+        debug rgw: 20
+  rgw:
+    storage classes: LUKEWARM, FROZEN
diff --git a/qa/suites/rgw/lua/supported-distros b/qa/suites/rgw/lua/supported-distros
new file mode 120000
index 000000000000..78f2991b407a
--- /dev/null
+++ b/qa/suites/rgw/lua/supported-distros
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$/
\ No newline at end of file
diff --git a/qa/suites/rgw/lua/tasks/+ b/qa/suites/rgw/lua/tasks/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/lua/tasks/0-install.yaml b/qa/suites/rgw/lua/tasks/0-install.yaml
new file mode 100644
index 000000000000..fa6e279145c3
--- /dev/null
+++ b/qa/suites/rgw/lua/tasks/0-install.yaml
@@ -0,0 +1,13 @@
+tasks:
+- install:
+- ceph:
+- openssl_keys:
+- rgw: [client.0]
+- tox: [client.0]    
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
diff --git a/qa/suites/rgw/lua/tasks/test_lua.yaml b/qa/suites/rgw/lua/tasks/test_lua.yaml
new file mode 100644
index 000000000000..9cfc91c71853
--- /dev/null
+++ b/qa/suites/rgw/lua/tasks/test_lua.yaml
@@ -0,0 +1,5 @@
+tasks:
+- tox: [client.0]
+- lua-tests:
+    client.0:
+      rgw_server: client.0
diff --git a/qa/tasks/lua_tests.py b/qa/tasks/lua_tests.py
new file mode 100644
index 000000000000..b6c0428816d5
--- /dev/null
+++ b/qa/tasks/lua_tests.py
@@ -0,0 +1,254 @@
+"""
+Run a set of lua tests on rgw.
+"""
+from io import BytesIO
+from configobj import ConfigObj
+import base64
+import contextlib
+import logging
+import os
+import random
+import string
+
+from teuthology import misc as teuthology
+from teuthology import contextutil
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def download(ctx, config):
+    assert isinstance(config, dict)
+    log.info('Downloading lua-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    branch = ctx.config.get('suite_branch')
+    repo = ctx.config.get('suite_repo')
+    log.info('Using branch %s from %s for lua tests', branch, repo)
+    for (client, client_config) in config.items():
+
+        ctx.cluster.only(client).run(
+            args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)],
+            )
+
+        sha1 = client_config.get('sha1')
+
+        if sha1 is not None:
+            ctx.cluster.only(client).run(
+                args=[
+                    'cd', '{tdir}/ceph'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    'git', 'reset', '--hard', sha1,
+                    ],
+                )
+
+    try:
+        yield
+    finally:
+        log.info('Removing lua-tests...')
+        testdir = teuthology.get_testdir(ctx)
+        for client in config:
+            ctx.cluster.only(client).run(
+                args=[
+                    'rm',
+                    '-rf',
+                    '{tdir}/ceph'.format(tdir=testdir),
+                    ],
+                )
+
+
+def _config_user(luatests_conf, section, user):
+    """
+    Configure users for this section by stashing away keys, ids, and
+    email addresses.
+    """
+    luatests_conf[section].setdefault('user_id', user)
+    luatests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
+    luatests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
+    luatests_conf[section].setdefault('access_key',
+        ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
+    luatests_conf[section].setdefault('secret_key',
+        base64.b64encode(os.urandom(40)).decode())
+
+
+@contextlib.contextmanager
+def create_users(ctx, config):
+    """
+    Create a main and an alternate s3 user.
+    """
+    assert isinstance(config, dict)
+    log.info('Creating rgw user...')
+    testdir = teuthology.get_testdir(ctx)
+
+    users = {'s3 main': 'foo'}
+    for client in config['clients']:
+        luatests_conf = config['luatests_conf'][client]
+        for section, user in users.items():
+            _config_user(luatests_conf, section, '{user}.{client}'.format(user=user, client=client))
+            log.debug('Creating user {user} on {host}'.format(user=luatests_conf[section]['user_id'], host=client))
+            cluster_name, daemon_type, client_id = teuthology.split_role(client)
+            client_with_id = daemon_type + '.' + client_id
+            ctx.cluster.only(client).run(
+                args=[
+                    'adjust-ulimits',
+                    'ceph-coverage',
+                    '{tdir}/archive/coverage'.format(tdir=testdir),
+                    'radosgw-admin',
+                    '-n', client_with_id,
+                    'user', 'create',
+                    '--uid', luatests_conf[section]['user_id'],
+                    '--display-name', luatests_conf[section]['display_name'],
+                    '--access-key', luatests_conf[section]['access_key'],
+                    '--secret', luatests_conf[section]['secret_key'],
+                    '--cluster', cluster_name,
+                    ],
+                )
+
+    try:
+        yield
+    finally:
+        for client in config['clients']:
+            for user in users.values():
+                uid = '{user}.{client}'.format(user=user, client=client)
+                cluster_name, daemon_type, client_id = teuthology.split_role(client)
+                client_with_id = daemon_type + '.' + client_id
+                ctx.cluster.only(client).run(
+                    args=[
+                        'adjust-ulimits',
+                        'ceph-coverage',
+                        '{tdir}/archive/coverage'.format(tdir=testdir),
+                        'radosgw-admin',
+                        '-n', client_with_id,
+                        'user', 'rm',
+                        '--uid', uid,
+                        '--purge-data',
+                        '--cluster', cluster_name,
+                        ],
+                    )
+
+
+@contextlib.contextmanager
+def configure(ctx, config):
+    assert isinstance(config, dict)
+    log.info('Configuring lua-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    for client, properties in config['clients'].items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        luatests_conf = config['luatests_conf'][client]
+
+        conf_fp = BytesIO()
+        luatests_conf.write(conf_fp)
+        remote.write_file(
+            path='{tdir}/ceph/src/test/rgw/lua/lua-tests.{client}.conf'.format(tdir=testdir, client=client),
+            data=conf_fp.getvalue(),
+            )
+
+    try:
+        yield
+    finally:
+        log.info('Removing lua-tests.conf file...')
+        testdir = teuthology.get_testdir(ctx)
+        for client, properties in config['clients'].items():
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            remote.run(
+                 args=['rm', '-f',
+                       '{tdir}/ceph/src/test/rgw/lua/lua-tests.{client}.conf'.format(tdir=testdir,client=client),
+                 ],
+                 )
+
+
+def get_toxvenv_dir(ctx):
+    return ctx.tox.venv_path
+
+
+def toxvenv_sh(ctx, remote, args, **kwargs):
+    activate = get_toxvenv_dir(ctx) + '/bin/activate'
+    return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)
+
+
+@contextlib.contextmanager
+def run_tests(ctx, config):
+    """
+    Run the lua tests after everything is set up.
+    :param ctx: Context passed to task
+    :param config: specific configuration information
+    """
+    assert isinstance(config, dict)
+    log.info('Running lua-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    for client, client_config in config.items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+
+        # test marks to use by default
+        attr = ['basic_test', 'request_test', 'example_test']
+
+        if 'extra_attr' in client_config:
+            attr = client_config.get('extra_attr')
+
+        args = ['cd', '{tdir}/ceph/src/test/rgw/lua/'.format(tdir=testdir), run.Raw('&&'),
+            'LUATESTS_CONF=./lua-tests.{client}.conf'.format(client=client),
+            'tox', '--', '-v', '-m', ' or '.join(attr)]
+
+        toxvenv_sh(ctx, remote, args, label="lua tests against rgw")
+
+    yield
+
+
+@contextlib.contextmanager
+def task(ctx,config):
+    """
+
+    If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo'
+    parameters in your teuthology-suite command. Example command for this is as follows::
+
+    teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:lua --ceph your_ceph_branch_name -m smithi --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name
+    
+    """
+    assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
+    assert hasattr(ctx, 'tox'), 's3tests must run after the tox task'
+    assert config is None or isinstance(config, list) \
+        or isinstance(config, dict), \
+        "task only supports a list or dictionary for configuration"
+
+    all_clients = ['client.{id}'.format(id=id_)
+                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+    if config is None:
+        config = all_clients
+    if isinstance(config, list):
+        config = dict.fromkeys(config)
+    clients=config.keys()
+
+    log.debug('config is %s', config)
+
+    luatests_conf = {}
+
+    for client in clients:
+        endpoint = ctx.rgw.role_endpoints.get(client)
+        assert endpoint, 'luatests: no rgw endpoint for {}'.format(client)
+
+        luatests_conf[client] = ConfigObj(
+            indent_type='',
+            infile={
+                'DEFAULT':
+                    {
+                    'port':endpoint.port,
+                    'host':endpoint.dns_name,
+                    },
+                's3 main':{}
+            }
+        )
+
+    with contextutil.nested(
+        lambda: download(ctx=ctx, config=config),
+        lambda: create_users(ctx=ctx, config=dict(
+                clients=clients,
+                luatests_conf=luatests_conf,
+                )),
+        lambda: configure(ctx=ctx, config=dict(
+                clients=config,
+                luatests_conf=luatests_conf,
+                )),
+        lambda: run_tests(ctx=ctx, config=config),
+        ):
+        pass
+    yield
+
diff --git a/src/rgw/rgw_lua.cc b/src/rgw/rgw_lua.cc
index 5b518d2e27cb..64643234a4a9 100644
--- a/src/rgw/rgw_lua.cc
+++ b/src/rgw/rgw_lua.cc
@@ -173,6 +173,31 @@ int create_directory_p(const DoutPrefixProvider *dpp, const fs::path& p) {
   return 0;
 }
 
+void get_luarocks_config(const bp::filesystem::path& process,
+    const std::string& luarocks_path,
+    const bp::environment& env, std::string& output) {
+  bp::ipstream is;
+  auto cmd = process.string();
+  cmd.append(" config");
+  output.append("Lua CMD: ");
+  output.append(cmd);
+
+  try {
+    bp::child c(cmd, env, bp::std_in.close(), (bp::std_err & bp::std_out) > is, bp::start_dir(luarocks_path));
+    std::string line;
+    do {
+      if (!line.empty()) {
+        output.append("\n\t").append(line);
+      }
+    } while (c.running() && std::getline(is, line));
+
+    c.wait();
+    output.append("\n\t").append("exit code: ").append(std::to_string(c.exit_code()));
+  } catch (const std::runtime_error& err) {
+    output.append("\n\t").append(err.what());
+  }
+}
+
 int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
                      optional_yield y, const std::string& luarocks_path,
                      packages_t& failed_packages, std::string& install_dir) {
@@ -201,6 +226,7 @@ int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
       luarocks_path << ". error: " << rc << dendl; 
     return rc;
   }
+  
 
   // create a temporary sub-directory to install all luarocks packages
   std::string tmp_path_template = luarocks_path;// fs::temp_directory_path();
@@ -213,51 +239,47 @@ int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
     return rc;
   }
   install_dir.assign(tmp_luarocks_path);
-  
-  {  
-    bp::ipstream is;
-    const auto cmd = p.string() + " config";
-    bp::child c(cmd, bp::std_in.close(), (bp::std_err & bp::std_out) > is);
-    
-    std::string lines = std::string("Lua CMD: ") + cmd;
-    std::string line;
-    
-    do {
-      if (!line.empty()) {
-        lines.append("\n\t");
-        lines.append(line);
-      }
-    } while (c.running() && std::getline(is, line));
 
-    c.wait();
-    line = "exit code: " + std::to_string(c.exit_code());
-    lines.append("\n\t");
-    lines.append(line);
-    ldpp_dout(dpp, 20) << lines << dendl;
+  // get a handle to the current environment
+  auto env = boost::this_process::environment();
+  bp::environment _env = env;
+  _env["HOME"] = luarocks_path;
+
+  if (dpp->get_cct()->_conf->subsys.should_gather<ceph_subsys_rgw, 20>()) {
+    std::string output;
+    get_luarocks_config(p, luarocks_path, _env, output);
+    ldpp_dout(dpp, 20) << output << dendl;
   }
 
   // the lua rocks install dir will be created by luarocks the first time it is called
   for (const auto& package : packages) {
     bp::ipstream is;
-    const auto cmd = p.string() + " install --no-doc --no-manifest --check-lua-versions --lua-version " + CEPH_LUA_VERSION + " --tree " + install_dir + " --deps-mode one " + package;
-    bp::child c(cmd, bp::std_in.close(), (bp::std_err & bp::std_out) > is);
-
-    // once package reload is supported, code should yield when reading output
-    std::string lines = std::string("Lua CMD: ") + cmd;
-    std::string line;
-
-    do {
-      if (!line.empty()) {
-        lines.append("\n\t");
-        lines.append(line);
-      }
-    } while (c.running() && std::getline(is, line));
+    auto cmd = p.string();
+    cmd.append(" install --no-doc --lua-version ").
+      append(CEPH_LUA_VERSION).
+      append(" --tree ").
+      append(install_dir).
+      append(" --deps-mode one ").
+      append(package);
+    bp::child c(cmd, _env, bp::std_in.close(), (bp::std_err & bp::std_out) > is, bp::start_dir(luarocks_path));
+
+    if (dpp->get_cct()->_conf->subsys.should_gather<ceph_subsys_rgw, 20>()) {
+      // TODO: yield when reading output
+      std::string lines = std::string("Lua CMD: ");
+      lines.append(cmd);
+      std::string line;
+      do {
+        if (!line.empty()) {
+          lines.append("\n\t").append(line);
+        }
+      } while (c.running() && std::getline(is, line));
+      ldpp_dout(dpp, 20) << lines << dendl;
+    }
 
     c.wait();
     if (c.exit_code()) {
       failed_packages.insert(package);
     }
-    ldpp_dout(dpp, 20) << lines << dendl;
   }
   
   return 0;
diff --git a/src/test/rgw/lua/README.rst b/src/test/rgw/lua/README.rst
new file mode 100644
index 000000000000..c1c259787a40
--- /dev/null
+++ b/src/test/rgw/lua/README.rst
@@ -0,0 +1,9 @@
+===================
+Lua Scripting Tests
+===================
+
+* Start the cluster using the `vstart.sh` script
+* `luarocks` (package manager for lua) must be installed on the machine
+* Run the test from within the `src/test/rgw/lua` directory: 
+  `LUATESTS_CONF=luatests.conf.SAMPLE tox`
+
diff --git a/src/test/rgw/lua/__init__.py b/src/test/rgw/lua/__init__.py
new file mode 100644
index 000000000000..e4b6b4ef66a1
--- /dev/null
+++ b/src/test/rgw/lua/__init__.py
@@ -0,0 +1,60 @@
+import configparser
+import os
+import pytest
+
+def setup():
+    cfg = configparser.RawConfigParser()
+    try:
+        path = os.environ['LUATESTS_CONF']
+    except KeyError:
+        raise RuntimeError(
+            'To run tests, point environment '
+            + 'variable LUATESTS_CONF to a config file.',
+            )
+    cfg.read(path)
+
+    if not cfg.defaults():
+        raise RuntimeError('Your config file is missing the DEFAULT section!')
+    if not cfg.has_section("s3 main"):
+        raise RuntimeError('Your config file is missing the "s3 main" section!')
+
+    defaults = cfg.defaults()
+
+  	# vars from the DEFAULT section
+    global default_host
+    default_host = defaults.get("host")
+
+    global default_port
+    default_port = int(defaults.get("port"))
+	# vars from the main section
+    global main_access_key
+    main_access_key = cfg.get('s3 main',"access_key")
+
+    global main_secret_key
+    main_secret_key = cfg.get('s3 main',"secret_key")
+
+
+def get_config_host():
+    global default_host
+    return default_host
+
+
+def get_config_port():
+    global default_port
+    return default_port
+
+
+def get_access_key():
+    global main_access_key
+    return main_access_key
+
+
+def get_secret_key():
+    global main_secret_key
+    return main_secret_key
+
+
+@pytest.fixture(autouse=True, scope="package")
+def configfile():
+    setup()
+
diff --git a/src/test/rgw/lua/luatests.conf.SAMPLE b/src/test/rgw/lua/luatests.conf.SAMPLE
new file mode 100644
index 000000000000..eb3291dafa8e
--- /dev/null
+++ b/src/test/rgw/lua/luatests.conf.SAMPLE
@@ -0,0 +1,10 @@
+[DEFAULT]
+port = 8000
+host = localhost
+
+[s3 main]
+access_key = 0555b35654ad1656d804
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+display_name = M. Tester
+user_id = testid
+email = tester@ceph.com
diff --git a/src/test/rgw/lua/pytest.ini b/src/test/rgw/lua/pytest.ini
new file mode 100644
index 000000000000..58cb902516c6
--- /dev/null
+++ b/src/test/rgw/lua/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+markers =
+  basic_test
+  request_test
+  example_test
diff --git a/src/test/rgw/lua/requirements.txt b/src/test/rgw/lua/requirements.txt
new file mode 100644
index 000000000000..c9c9046157c8
--- /dev/null
+++ b/src/test/rgw/lua/requirements.txt
@@ -0,0 +1,2 @@
+boto3 >=1.0.0
+pytest
diff --git a/src/test/rgw/lua/test_lua.py b/src/test/rgw/lua/test_lua.py
new file mode 100644
index 000000000000..f8131956b30f
--- /dev/null
+++ b/src/test/rgw/lua/test_lua.py
@@ -0,0 +1,476 @@
+import logging
+import json
+import tempfile
+import random
+import socket
+import time
+import threading
+import subprocess
+import os
+import stat
+import string
+import pytest
+import boto3
+
+from . import(
+    configfile,
+    get_config_host,
+    get_config_port,
+    get_access_key,
+    get_secret_key
+    )
+
+
+# configure logging for the tests module
+log = logging.getLogger(__name__)
+
+num_buckets = 0
+run_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
+
+test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/../'
+
+def bash(cmd, **kwargs):
+    log.debug('running command: %s', ' '.join(cmd))
+    kwargs['stdout'] = subprocess.PIPE
+    process = subprocess.Popen(cmd, **kwargs)
+    s = process.communicate()[0].decode('utf-8')
+    return (s, process.returncode)
+
+
+def admin(args, **kwargs):
+    """ radosgw-admin command """
+    cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', 'noname'] + args
+    return bash(cmd, **kwargs)
+
+
+def delete_all_objects(conn, bucket_name):
+    objects = []
+    for key in conn.list_objects(Bucket=bucket_name)['Contents']:
+        objects.append({'Key': key['Key']})
+    # delete objects from the bucket
+    response = conn.delete_objects(Bucket=bucket_name,
+            Delete={'Objects': objects})
+
+
+def gen_bucket_name():
+    global num_buckets
+
+    num_buckets += 1
+    return run_prefix + '-' + str(num_buckets)
+
+
+def get_ip():
+    return 'localhost'
+
+
+def get_ip_http():
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        # address should not be reachable
+        s.connect(('10.255.255.255', 1))
+        ip = s.getsockname()[0]
+    finally:
+        s.close()
+    return ip
+
+
+def connection():
+    hostname = get_config_host()
+    port_no = get_config_port()
+    access_key = get_access_key()
+    secret_key = get_secret_key()
+    if port_no == 443 or port_no == 8443:
+        scheme = 'https://'
+    else:
+        scheme = 'http://'
+
+    client = boto3.client('s3',
+            endpoint_url=scheme+hostname+':'+str(port_no),
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key)
+
+    return client
+
+
+def another_user(tenant=None):
+    access_key = str(time.time())
+    secret_key = str(time.time())
+    uid = 'superman' + str(time.time())
+    if tenant:
+        _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+    else:
+        _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+
+    assert result == 0
+    hostname = get_config_host()
+    port_no = get_config_port()
+    if port_no == 443 or port_no == 8443:
+        scheme = 'https://'
+    else:
+        scheme = 'http://'
+
+    client = boto3.client('s3',
+            endpoint_url=scheme+hostname+':'+str(port_no),
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key)
+
+    return client
+
+
+def put_script(script, context, tenant=None):
+    fp = tempfile.NamedTemporaryFile(mode='w+')
+    fp.write(script)
+    fp.flush()
+    if tenant:
+        result = admin(['script', 'put', '--infile', fp.name, '--context', context, '--tenant', tenant])
+    else:
+        result = admin(['script', 'put', '--infile', fp.name, '--context', context])
+
+    fp.close()
+    return result
+
+class UnixSocket:
+    def __init__(self, socket_path):
+        self.socket_path = socket_path
+        self.stop = False
+        self.started = False
+        self.events = []
+        self.t = threading.Thread(target=self.listen_on_socket)
+        self.t.start()
+        while not self.started:
+            print("UnixSocket: waiting for unix socket server to start")
+            time.sleep(1)
+
+    def shutdown(self):
+        self.stop = True
+        self.t.join()
+
+    def get_and_reset_events(self):
+        tmp = self.events
+        self.events = []
+        return tmp
+
+    def listen_on_socket(self):
+        self.started = True
+        # remove the socket file if it already exists
+        try:
+            os.unlink(self.socket_path)
+        except OSError:
+            if os.path.exists(self.socket_path):
+                raise
+
+        # create and bind the Unix socket server
+        server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        server.bind(self.socket_path)
+
+        # give permissions for anyone to write to it
+        os.chmod(self.socket_path, stat.S_IWOTH|stat.S_IWGRP|stat.S_IWUSR)
+
+        # listen for incoming connections
+        server.listen(1)
+        # accept timeout is 30s at the beginning
+        server.settimeout(30)
+        print("UnixSocket '%s' is listening for incoming connections..." % self.socket_path)
+
+        while not self.stop:
+            # accept connections
+            try:
+                connection, _ = server.accept()
+            except Exception as e:
+                print("UnixSocket: accept "+str(e))
+                continue
+            # after we start accept/recv timeouts are 5s
+            server.settimeout(5)
+            connection.settimeout(5)
+
+            try:
+                print("UnixSocket: new connection accepted")
+                # receive data from the client
+                while True:
+                    # recv timeout is 5s
+                    data = connection.recv(1024)
+                    if not data:
+                        break
+                    event = json.loads(data.decode())
+                    self.events.append(event)
+            finally:
+                # close the connection
+                connection.close()
+                print("UnixSocket: connection closed")
+
+        # remove the socket file
+        os.unlink(self.socket_path)
+
+
+#####################
+# lua scripting tests
+#####################
+
+
+@pytest.mark.basic_test
+def test_script_management():
+    contexts = ['prerequest', 'postrequest', 'background', 'getdata', 'putdata']
+    scripts = {}
+    for context in contexts:
+        script = 'print("hello from ' + context + '")'
+        result = put_script(script, context)
+        assert result[1] == 0
+        scripts[context] = script
+    for context in contexts:
+        result = admin(['script', 'get', '--context', context])
+        assert result[1] ==  0
+        assert result[0].strip() == scripts[context]
+    for context in contexts:
+        result = admin(['script', 'rm', '--context', context])
+        assert result[1] == 0
+    for context in contexts:
+        result = admin(['script', 'get', '--context', context])
+        assert result[1] == 0
+        assert result[0].strip() == 'no script exists for context: ' + context
+
+
+@pytest.mark.basic_test
+def test_script_management_with_tenant():
+    tenant = 'mytenant'
+    conn2 = another_user(tenant)
+    contexts = ['prerequest', 'postrequest', 'getdata', 'putdata']
+    scripts = {}
+    for context in contexts:
+        for t in ['', tenant]:
+            script = 'print("hello from ' + context + ' and ' + tenant + '")'
+            result = put_script(script, context, t)
+            assert result[1] ==  0
+            scripts[context+t] = script
+    for context in contexts:
+        result = admin(['script', 'get', '--context', context])
+        assert result[1] == 0
+        assert result[0].strip(), scripts[context]
+        result = admin(['script', 'rm', '--context', context])
+        assert result[1] == 0
+        result = admin(['script', 'get', '--context', context])
+        assert result[1] == 0
+        assert result[0].strip(), 'no script exists for context: ' + context
+        result = admin(['script', 'get', '--context', context, '--tenant', tenant])
+        assert result[1] == 0
+        assert result[0].strip(), scripts[context+tenant]
+        result = admin(['script', 'rm', '--context', context, '--tenant', tenant])
+        assert result[1] == 0
+        result = admin(['script', 'get', '--context', context, '--tenant', tenant])
+        assert result[1] == 0
+        assert result[0].strip(), 'no script exists for context: ' + context + ' in tenant: ' + tenant
+
+
+@pytest.mark.request_test
+def test_put_obj():
+    script = '''
+RGWDebugLog("op was: "..Request.RGWOp)
+if Request.RGWOp == "put_obj" then
+    local object = Request.Object
+    local message = Request.bucket.Name .. "," .. object.Name .. 
+        "," .. object.Id .. "," .. object.Size .. "," .. object.MTime
+    RGWDebugLog("set: x-amz-meta-test to: " .. message)
+    Request.HTTP.Metadata["x-amz-meta-test"] = message
+end
+'''
+    context = "prerequest"
+    result = put_script(script, context)
+    assert result[1] == 0
+	
+    conn = connection()
+    bucket_name = gen_bucket_name()
+    conn.create_bucket(Bucket=bucket_name)
+    key = "hello"
+    conn.put_object(Body="1234567890".encode("ascii"), Bucket=bucket_name, Key=key)
+
+    result = conn.get_object(Bucket=bucket_name, Key=key)
+    message = result['ResponseMetadata']['HTTPHeaders']['x-amz-meta-test']
+    assert message == bucket_name+","+key+","+key+",0,1970-01-01 00:00:00"
+
+    # cleanup
+    conn.delete_object(Bucket=bucket_name, Key=key)
+    conn.delete_bucket(Bucket=bucket_name)
+    contexts = ['prerequest', 'postrequest', 'getdata', 'putdata']
+    for context in contexts:
+        result = admin(['script', 'rm', '--context', context])
+        assert result[1] == 0
+
+
+@pytest.mark.example_test
+def test_copyfrom():
+    script = '''
+function print_object(object)
+    RGWDebugLog("  Name: " .. object.Name)
+    RGWDebugLog("  Instance: " .. object.Instance)
+    RGWDebugLog("  Id: " .. object.Id)
+    RGWDebugLog("  Size: " .. object.Size)
+    RGWDebugLog("  MTime: " .. object.MTime)
+end
+
+if Request.CopyFrom and Request.Object and Request.CopyFrom.Object then
+    RGWDebugLog("copy from object:")
+    print_object(Request.CopyFrom.Object)
+    RGWDebugLog("to object:")
+    print_object(Request.Object)
+end
+RGWDebugLog("op was: "..Request.RGWOp)
+'''
+
+    contexts = ['prerequest', 'postrequest', 'getdata', 'putdata']
+    for context in contexts:
+        footer = '\nRGWDebugLog("context was: '+context+'\\n\\n")'
+        result = put_script(script+footer, context)
+        assert result[1] == 0
+	
+    conn = connection()
+    bucket_name = gen_bucket_name()
+    # create bucket
+    bucket = conn.create_bucket(Bucket=bucket_name)
+    # create objects in the bucket
+    number_of_objects = 5
+    for i in range(number_of_objects):
+        content = str(os.urandom(1024*1024)).encode("ascii")
+        key = str(i)
+        conn.put_object(Body=content, Bucket=bucket_name, Key=key)
+
+    for i in range(number_of_objects):
+        key = str(i)
+        conn.copy_object(Bucket=bucket_name,
+                Key='copyof'+key, 
+                CopySource=bucket_name+'/'+key)
+
+    # cleanup
+    delete_all_objects(conn, bucket_name)
+    conn.delete_bucket(Bucket=bucket_name)
+    contexts = ['prerequest', 'postrequest', 'getdata', 'putdata']
+    for context in contexts:
+        result = admin(['script', 'rm', '--context', context])
+        assert result[1] == 0
+
+
+@pytest.mark.example_test
+def test_entropy():
+    script = '''
+function object_entropy()
+    local byte_hist = {}
+    local byte_hist_size = 256
+    for i = 1,byte_hist_size do
+        byte_hist[i] = 0
+    end
+    local total = 0
+
+    for i, c in pairs(Data)  do
+        local byte = c:byte() + 1
+        byte_hist[byte] = byte_hist[byte] + 1
+        total = total + 1
+    end
+
+    entropy = 0
+
+    for _, count in ipairs(byte_hist) do
+        if count ~= 0 then
+            local p = 1.0 * count / total
+            entropy = entropy - (p * math.log(p)/math.log(byte_hist_size))
+        end
+    end
+
+    return entropy
+end
+
+local full_name = Request.Bucket.Name.."-"..Request.Object.Name
+RGWDebugLog("entropy of chunk of: " .. full_name .. " at offset: " .. tostring(Offset)  ..  " is: " .. tostring(object_entropy()))
+RGWDebugLog("payload size of chunk of: " .. full_name .. " is: " .. #Data)
+'''
+
+    result = put_script(script, "putdata")
+    assert result[1] == 0
+
+    conn = connection()
+    bucket_name = gen_bucket_name()
+    # create bucket
+    bucket = conn.create_bucket(Bucket=bucket_name)
+    # create objects in the bucket (async)
+    number_of_objects = 5
+    for i in range(number_of_objects):
+        content = str(os.urandom(1024*1024*16)).encode("ascii")
+        key = str(i)
+        conn.put_object(Body=content, Bucket=bucket_name, Key=key)
+
+    # cleanup
+    delete_all_objects(conn, bucket_name)
+    conn.delete_bucket(Bucket=bucket_name)
+    contexts = ['prerequest', 'postrequest', 'background', 'getdata', 'putdata']
+    for context in contexts:
+        result = admin(['script', 'rm', '--context', context])
+        assert result[1] == 0
+
+
+@pytest.mark.example_test
+def test_access_log():
+    bucket_name = gen_bucket_name()
+    socket_path = '/tmp/'+bucket_name
+
+    script = '''
+if Request.RGWOp == "get_obj" then
+    local json = require("cjson")
+    local socket = require("socket")
+    local unix = require("socket.unix")
+    local s = unix()
+    E = {{}}
+
+    msg = {{bucket = (Request.Bucket or (Request.CopyFrom or E).Bucket).Name,
+        object = Request.Object.Name,
+        time = Request.Time,
+        operation = Request.RGWOp,
+        http_status = Request.Response.HTTPStatusCode,
+        error_code = Request.Response.HTTPStatus,
+        object_size = Request.Object.Size,
+        trans_id = Request.TransactionId}}
+    assert(s:connect("{}"))
+    s:send(json.encode(msg).."\\n")
+    s:close()
+end
+'''.format(socket_path)
+
+    result = admin(['script-package', 'add', '--package=lua-cjson', '--allow-compilation'])
+    assert result[1] ==  0
+    result = admin(['script-package', 'add', '--package=luasocket', '--allow-compilation'])
+    assert result[1] == 0 
+    result = admin(['script-package', 'reload'])
+    assert result[1] == 0 
+    result = put_script(script, "postrequest")
+    assert result[1] == 0 
+
+    socket_server = UnixSocket(socket_path)
+    try:
+        conn = connection()
+        # create bucket
+        bucket = conn.create_bucket(Bucket=bucket_name)
+        # create objects in the bucket (async)
+        number_of_objects = 5
+        keys = []
+        for i in range(number_of_objects):
+            content = str(os.urandom(1024*1024)).encode("ascii")
+            key = str(i)
+            conn.put_object(Body=content, Bucket=bucket_name, Key=key)
+            keys.append(key)
+
+        for key in conn.list_objects(Bucket=bucket_name)['Contents']:
+            conn.get_object(Bucket=bucket_name, Key=key['Key'])
+
+        time.sleep(5)
+        event_keys = []
+        for event in socket_server.get_and_reset_events():
+            assert event['bucket'] == bucket_name
+            event_keys.append(event['object'])
+
+        assert keys == event_keys
+
+    finally:
+        socket_server.shutdown()
+        delete_all_objects(conn, bucket_name)
+        conn.delete_bucket(Bucket=bucket_name)
+        contexts = ['prerequest', 'postrequest', 'background', 'getdata', 'putdata']
+        for context in contexts:
+            result = admin(['script', 'rm', '--context', context])
+            assert result[1] == 0
+
diff --git a/src/test/rgw/lua/tox.ini b/src/test/rgw/lua/tox.ini
new file mode 100644
index 000000000000..27ffd5a75e97
--- /dev/null
+++ b/src/test/rgw/lua/tox.ini
@@ -0,0 +1,9 @@
+[tox]
+envlist = py
+skipsdist = True
+
+[testenv]
+deps = -rrequirements.txt
+passenv =
+  LUATESTS_CONF
+commands = pytest {posargs}

From 974c4f93b3cc485e9fd340d9e7d5d13948ab25f7 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Fri, 10 Nov 2023 20:23:33 +0530
Subject: [PATCH 0711/2492] qa/cephfs: don't write guest keyring to file
 "keyring"

Writing guest keyring to CWD's file named "keyring" will over-write
build/keyring on developer's machine which will make the cluster
inoperatable and also fail the test.

Fixes: https://tracker.ceph.com/issues/63506
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py | 43 ++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index d40c0d67d541..8f196aeee2b1 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -393,14 +393,16 @@ def _configure_guest_auth(self, guest_mount, authid, key):
         """.format(authid=authid,key=key))
 
         guest_mount.client_id = authid
-        guest_mount.client_remote.write_file(guest_mount.get_keyring_path(),
-                                             keyring_txt, sudo=True)
+        guest_keyring_path = guest_mount.client_remote.mktemp(
+            data=keyring_txt)
         # Add a guest client section to the ceph config file.
         self.config_set("client.{0}".format(authid), "debug client", 20)
         self.config_set("client.{0}".format(authid), "debug objecter", 20)
         self.set_conf("client.{0}".format(authid),
                       "keyring", guest_mount.get_keyring_path())
 
+        return guest_keyring_path
+
     def _auth_metadata_get(self, filedata):
         """
         Return a deserialized JSON object, or None
@@ -1123,11 +1125,12 @@ def test_subvolume_group_quota_mds_path_restriction_to_group_path(self):
         guest_mount.umount_wait()
 
         # configure credentials for guest client
-        self._configure_guest_auth(guest_mount, "guest1", user[0]["key"])
-
+        guest_keyring_path = self._configure_guest_auth(
+            guest_mount, "guest1", user[0]["key"])
         # mount the subvolume
         mount_path = os.path.join("/", subvolpath)
-        guest_mount.mount_wait(cephfs_mntpt=mount_path)
+        guest_mount.mount_wait(cephfs_mntpt=mount_path,
+                               client_keyring_path=guest_keyring_path)
 
         # create 99 files of 1MB
         guest_mount.run_shell_payload("mkdir -p dir1")
@@ -1209,10 +1212,11 @@ def test_subvolume_group_quota_mds_path_restriction_to_subvolume_path(self):
         guest_mount.umount_wait()
 
         # configure credentials for guest client
-        self._configure_guest_auth(guest_mount, "guest1", user[0]["key"])
-
+        guest_keyring_path = self._configure_guest_auth(
+            guest_mount, "guest1", user[0]["key"])
         # mount the subvolume
-        guest_mount.mount_wait(cephfs_mntpt=mount_path)
+        guest_mount.mount_wait(cephfs_mntpt=mount_path,
+                               client_keyring_path=guest_keyring_path)
 
         # create 99 files of 1MB to exceed quota
         guest_mount.run_shell_payload("mkdir -p dir1")
@@ -2602,10 +2606,11 @@ def test_authorize_deauthorize_legacy_subvolume(self):
         self.assertIn("client.{0}".format(authid), existing_ids)
 
         # configure credentials for guest client
-        self._configure_guest_auth(guest_mount, authid, key)
-
+        guest_keyring_path  = self._configure_guest_auth(
+            guest_mount, authid, key)
         # mount the subvolume, and write to it
-        guest_mount.mount_wait(cephfs_mntpt=mount_path)
+        guest_mount.mount_wait(cephfs_mntpt=mount_path,
+                               client_keyring_path=guest_keyring_path)
         guest_mount.write_n_mb("data.bin", 1)
 
         # authorize guest authID read access to subvolume
@@ -2658,10 +2663,11 @@ def test_authorize_deauthorize_subvolume(self):
         self.assertIn("client.{0}".format(authid), existing_ids)
 
         # configure credentials for guest client
-        self._configure_guest_auth(guest_mount, authid, key)
-
+        guest_keyring_path = self._configure_guest_auth(
+            guest_mount, authid, key)
         # mount the subvolume, and write to it
-        guest_mount.mount_wait(cephfs_mntpt=mount_path)
+        guest_mount.mount_wait(cephfs_mntpt=mount_path,
+                               client_keyring_path=guest_keyring_path)
         guest_mount.write_n_mb("data.bin", 1)
 
         # authorize guest authID read access to subvolume
@@ -3272,11 +3278,14 @@ def test_subvolume_evict_client(self):
 
             mount_path = self._fs_cmd("subvolume", "getpath", self.volname, subvolumes[i],
                                       "--group_name", group).rstrip()
-            # configure credentials for guest client
-            self._configure_guest_auth(guest_mounts[i], auth_id, key)
 
+            # configure credentials for guest client
+            guest_keyring_path = self._configure_guest_auth(guest_mounts[i],
+                                                            auth_id, key)
             # mount the subvolume, and write to it
-            guest_mounts[i].mount_wait(cephfs_mntpt=mount_path)
+            guest_mounts[i].mount_wait(
+                cephfs_mntpt=mount_path,
+                client_keyring_path=guest_keyring_path)
             guest_mounts[i].write_n_mb("data.bin", 1)
 
         # Evict client, guest_mounts[0], using auth ID 'guest' and has mounted

From 95dac209a2e0788dce490f64645ff5d5a17fdfb9 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 2 Nov 2023 20:45:27 +0530
Subject: [PATCH 0712/2492] mgr/vol: fix flake8 warnings

Fix warnings printed by the command "flake8 --select=F,E9
--exclude=venv,.tox src/pybind/mgr/volumes/".

While working with mgr/vol, syntax errors are not printed anywhere. Any
attempt to run vstart.sh after such faulty patch causes vstart.sh to go
in a infinite loop. And running "ceph fs volume" command prints "no such
command exists". This doesn't tell the actual issue and causes
confusion.

When flake8 is run, the issue is not immediately apparent due to so many
warnings. Therefore, fix these warnings so that it becomes easier to
spot such critical issues here onwards.

Note: mypy uses comments like "#type: Dict" for type checking and
therefore counts 'from typechecking import Dict' as not unsued. But
flake8 doesn't recognize type hints embedded in comment. Therfore,
switch to actually using type hints instead of adding type hints to
comments.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/pybind/mgr/volumes/fs/async_cloner.py     |  4 ++--
 .../mgr/volumes/fs/operations/access.py       |  2 +-
 src/pybind/mgr/volumes/fs/operations/lock.py  |  4 ++--
 .../mgr/volumes/fs/operations/pin_util.py     |  3 +--
 src/pybind/mgr/volumes/fs/operations/trash.py |  1 -
 .../fs/operations/versions/__init__.py        |  2 +-
 .../operations/versions/metadata_manager.py   |  1 -
 .../volumes/fs/operations/versions/op_sm.py   |  2 +-
 .../fs/operations/versions/subvolume_base.py  |  4 ++--
 .../fs/operations/versions/subvolume_v1.py    | 19 +++++++++----------
 .../fs/operations/versions/subvolume_v2.py    |  7 +++----
 .../mgr/volumes/fs/operations/volume.py       |  4 ++--
 src/pybind/mgr/volumes/fs/volume.py           | 10 ++++------
 13 files changed, 28 insertions(+), 35 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py
index 95f7d64e1b36..bfa34ed03c02 100644
--- a/src/pybind/mgr/volumes/fs/async_cloner.py
+++ b/src/pybind/mgr/volumes/fs/async_cloner.py
@@ -191,7 +191,7 @@ def cptree(src_root_path, dst_root_path):
 def set_quota_on_clone(fs_handle, clone_volumes_pair):
     src_path = clone_volumes_pair[1].snapshot_data_path(clone_volumes_pair[2])
     dst_path = clone_volumes_pair[0].path
-    quota = None # type: Optional[int]
+    quota: Optional[int] = None
     try:
         quota = int(fs_handle.getxattr(src_path, 'ceph.quota.max_bytes').decode('utf-8'))
     except cephfs.NoData:
@@ -205,7 +205,7 @@ def set_quota_on_clone(fs_handle, clone_volumes_pair):
         except cephfs.Error as e:
              raise VolumeException(-e.args[0], e.args[1])
 
-    quota_files = None # type: Optional[int]
+    quota_files: Optional[int] = None
     try:
         quota_files = int(fs_handle.getxattr(src_path, 'ceph.quota.max_files').decode('utf-8'))
     except cephfs.NoData:
diff --git a/src/pybind/mgr/volumes/fs/operations/access.py b/src/pybind/mgr/volumes/fs/operations/access.py
index 9b7b2431605c..7e916e95539e 100644
--- a/src/pybind/mgr/volumes/fs/operations/access.py
+++ b/src/pybind/mgr/volumes/fs/operations/access.py
@@ -4,7 +4,7 @@
 
 
 def prepare_updated_caps_list(existing_caps, mds_cap_str, osd_cap_str, authorize=True):
-    caps_list = []  # type: List[str]
+    caps_list: List[str]  = []
     for k, v in existing_caps['caps'].items():
         if k == 'mds' or k == 'osd':
             continue
diff --git a/src/pybind/mgr/volumes/fs/operations/lock.py b/src/pybind/mgr/volumes/fs/operations/lock.py
index 7ef6923e115c..9588ddec164e 100644
--- a/src/pybind/mgr/volumes/fs/operations/lock.py
+++ b/src/pybind/mgr/volumes/fs/operations/lock.py
@@ -22,10 +22,10 @@ class GlobalLock(object):
 
     See: https://people.eecs.berkeley.edu/~kubitron/courses/cs262a-F14/projects/reports/project6_report.pdf
     """
-    _shared_state = {
+    _shared_state: Dict = {
         'lock' : Lock(),
         'init' : False
-    } # type: Dict
+    }
 
     def __init__(self):
         with self._shared_state['lock']:
diff --git a/src/pybind/mgr/volumes/fs/operations/pin_util.py b/src/pybind/mgr/volumes/fs/operations/pin_util.py
index 9ea79e546e26..a12ab5b4d4b2 100644
--- a/src/pybind/mgr/volumes/fs/operations/pin_util.py
+++ b/src/pybind/mgr/volumes/fs/operations/pin_util.py
@@ -1,4 +1,3 @@
-import os
 import errno
 
 import cephfs
@@ -25,7 +24,7 @@ def pin(fs, path, pin_type, pin_setting):
 
     try:
         pin_setting = _pin_value[pin_type](pin_setting)
-    except ValueError as e:
+    except ValueError:
         raise VolumeException(-errno.EINVAL, f"pin value wrong type: {pin_setting}")
 
     try:
diff --git a/src/pybind/mgr/volumes/fs/operations/trash.py b/src/pybind/mgr/volumes/fs/operations/trash.py
index 66f1d71cf89a..d76d43a43d13 100644
--- a/src/pybind/mgr/volumes/fs/operations/trash.py
+++ b/src/pybind/mgr/volumes/fs/operations/trash.py
@@ -6,7 +6,6 @@
 import cephfs
 
 from .template import GroupTemplate
-from ..fs_util import listdir
 from ..exception import VolumeException
 
 log = logging.getLogger(__name__)
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/__init__.py b/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
index 544afa165f97..6c96c4570199 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
@@ -83,7 +83,7 @@ def upgrade_legacy_subvolume(self, fs, subvolume):
         subvolume_type = SubvolumeTypes.TYPE_NORMAL
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
-        except OpSmException as oe:
+        except OpSmException:
             raise VolumeException(-errno.EINVAL, "subvolume creation failed: internal error")
         qpath = subvolume.base_path.decode('utf-8')
         # legacy is only upgradable to v1
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py b/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
index 718735d91b13..b33a2b48b775 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
@@ -1,7 +1,6 @@
 import os
 import errno
 import logging
-import sys
 import threading
 import configparser
 import re
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/op_sm.py b/src/pybind/mgr/volumes/fs/operations/versions/op_sm.py
index 1142600cbb20..93eafb2bde4c 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/op_sm.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/op_sm.py
@@ -19,7 +19,7 @@ def __neq__(self, other):
         return not(self == other)
 
 class SubvolumeOpSm(object):
-    transition_table = {} # type: Dict
+    transition_table: Dict = {}
 
     @staticmethod
     def is_complete_state(state):
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
index 3bae0707a6a4..8fbe177e5f4b 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
@@ -144,7 +144,7 @@ def load_config(self):
         try:
             self.fs.stat(self.legacy_config_path)
             self.legacy_mode = True
-        except cephfs.Error as e:
+        except cephfs.Error:
             pass
 
         log.debug("loading config "
@@ -160,7 +160,7 @@ def load_config(self):
 
     def get_attrs(self, pathname):
         # get subvolume attributes
-        attrs = {}  # type: Dict[str, Union[int, str, None]]
+        attrs: Dict[str, Union[int, str, None]] = {}
         stx = self.fs.statx(pathname,
                             cephfs.CEPH_STATX_UID | cephfs.CEPH_STATX_GID
                             | cephfs.CEPH_STATX_MODE,
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
index b5a10dd6c7f6..c07f835421e5 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
@@ -55,7 +55,7 @@ def path(self):
         try:
             # no need to stat the path -- open() does that
             return self.metadata_mgr.get_global_option(MetadataManager.GLOBAL_META_KEY_PATH).encode('utf-8')
-        except MetadataMgrException as me:
+        except MetadataMgrException:
             raise VolumeException(-errno.EINVAL, "error fetching subvolume metadata")
 
     @property
@@ -68,7 +68,7 @@ def mark_subvolume(self):
         try:
             # MDS treats this as a noop for already marked subvolume
             self.fs.setxattr(self.path, 'ceph.dir.subvolume', b'1', 0)
-        except cephfs.InvalidValue as e:
+        except cephfs.InvalidValue:
             raise VolumeException(-errno.EINVAL, "invalid value specified for ceph.dir.subvolume")
         except cephfs.Error as e:
             raise VolumeException(-e.args[0], e.args[1])
@@ -89,7 +89,7 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
         subvolume_type = SubvolumeTypes.TYPE_NORMAL
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
-        except OpSmException as oe:
+        except OpSmException:
             raise VolumeException(-errno.EINVAL, "subvolume creation failed: internal error")
 
         subvol_path = os.path.join(self.base_path, str(uuid.uuid4()).encode('utf-8'))
@@ -156,7 +156,7 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
         subvolume_type = SubvolumeTypes.TYPE_CLONE
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
-        except OpSmException as oe:
+        except OpSmException:
             raise VolumeException(-errno.EINVAL, "clone failed: internal error")
 
         subvol_path = os.path.join(self.base_path, str(uuid.uuid4()).encode('utf-8'))
@@ -596,7 +596,7 @@ def authorized_list(self):
         """
         with self.auth_mdata_mgr.subvol_metadata_lock(self.group.groupname, self.subvolname):
             meta = self.auth_mdata_mgr.subvol_metadata_get(self.group.groupname, self.subvolname)
-            auths = [] # type: List[Dict[str,str]]
+            auths: List[Dict[str,str]] = []
             if not meta or not meta['auths']:
                 return auths
 
@@ -669,7 +669,7 @@ def _get_clone_source(self):
                     pass
                 else:
                     raise
-        except MetadataMgrException as me:
+        except MetadataMgrException:
             raise VolumeException(-errno.EINVAL, "error fetching subvolume metadata")
         return clone_source
 
@@ -744,7 +744,7 @@ def has_pending_clones(self, snapname):
             raise
 
     def get_pending_clones(self, snapname):
-        pending_clones_info = {"has_pending_clones": "no"}  # type: Dict[str, Any]
+        pending_clones_info: Dict[str, Any] = {"has_pending_clones": "no"}
         pending_track_id_list = []
         pending_clone_list = []
         index_path = ""
@@ -777,7 +777,6 @@ def get_pending_clones(self, snapname):
                         # If clone is completed between 'list_all_keys_with_specified_values_from_section'
                         # and readlink(track_id_path) call then readlink will fail with error ENOENT (2)
                         # Hence we double check whether track_id is exist in .meta file or not.
-                        value = self.metadata_mgr.get_option('clone snaps', track_id)
                         # Edge case scenario.
                         # If track_id for clone exist but path /volumes/_index/clone/{track_id} not found
                         # then clone is orphan.
@@ -790,7 +789,7 @@ def get_pending_clones(self, snapname):
             path = Path(link_path.decode('utf-8'))
             clone_name = os.path.basename(link_path).decode('utf-8')
             group_name = os.path.basename(path.parent.absolute())
-            details = {"name": clone_name}  # type: Dict[str, str]
+            details = {"name": clone_name}
             if group_name != Group.NO_GROUP_NAME:
                 details["target_group"] = group_name
             pending_clone_list.append(details)
@@ -839,7 +838,7 @@ def snapshot_info(self, snapname):
                 snap_info[key] = self.fs.getxattr(snappath, val)
             pending_clones_info = self.get_pending_clones(snapname)
             info_dict = {'created_at': str(datetime.fromtimestamp(float(snap_info['created_at']))),
-                    'data_pool': snap_info['data_pool'].decode('utf-8')}  # type: Dict[str, Any]
+                    'data_pool': snap_info['data_pool'].decode('utf-8')}
             info_dict.update(pending_clones_info);
             return info_dict
         except cephfs.Error as e:
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
index 03085d049713..17955d3020f8 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
@@ -10,7 +10,6 @@
 from .subvolume_attrs import SubvolumeTypes, SubvolumeStates, SubvolumeFeatures
 from .op_sm import SubvolumeOpSm
 from .subvolume_v1 import SubvolumeV1
-from ..template import SubvolumeTemplate
 from ...exception import OpSmException, VolumeException, MetadataMgrException
 from ...fs_util import listdir, create_base_dir
 from ..template import SubvolumeOpType
@@ -99,7 +98,7 @@ def mark_subvolume(self):
         try:
             # MDS treats this as a noop for already marked subvolume
             self.fs.setxattr(self.base_path, 'ceph.dir.subvolume', b'1', 0)
-        except cephfs.InvalidValue as e:
+        except cephfs.InvalidValue:
             raise VolumeException(-errno.EINVAL, "invalid value specified for ceph.dir.subvolume")
         except cephfs.Error as e:
             raise VolumeException(-e.args[0], e.args[1])
@@ -159,7 +158,7 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
         subvolume_type = SubvolumeTypes.TYPE_NORMAL
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
-        except OpSmException as oe:
+        except OpSmException:
             raise VolumeException(-errno.EINVAL, "subvolume creation failed: internal error")
 
         retained = self.retained
@@ -207,7 +206,7 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
         subvolume_type = SubvolumeTypes.TYPE_CLONE
         try:
             initial_state = SubvolumeOpSm.get_init_state(subvolume_type)
-        except OpSmException as oe:
+        except OpSmException:
             raise VolumeException(-errno.EINVAL, "clone failed: internal error")
 
         retained = self.retained
diff --git a/src/pybind/mgr/volumes/fs/operations/volume.py b/src/pybind/mgr/volumes/fs/operations/volume.py
index 395a3fb4ea07..67fbb891cefb 100644
--- a/src/pybind/mgr/volumes/fs/operations/volume.py
+++ b/src/pybind/mgr/volumes/fs/operations/volume.py
@@ -40,7 +40,7 @@ def get_pool_names(mgr, volname):
     """
     fs_map = mgr.get("fs_map")
     metadata_pool_id = None
-    data_pool_ids = [] # type: List[int]
+    data_pool_ids: List[int] = []
     for f in fs_map['filesystems']:
         if volname == f['mdsmap']['fs_name']:
             metadata_pool_id = f['mdsmap']['metadata_pool']
@@ -61,7 +61,7 @@ def get_pool_ids(mgr, volname):
     """
     fs_map = mgr.get("fs_map")
     metadata_pool_id = None
-    data_pool_ids = [] # type: List[int]
+    data_pool_ids: List[int] = []
     for f in fs_map['filesystems']:
         if volname == f['mdsmap']['fs_name']:
             metadata_pool_id = f['mdsmap']['metadata_pool']
diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 5c6642444b11..e19d1eafb2a6 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -1,7 +1,6 @@
 import json
 import errno
 import logging
-import os
 import mgr_util
 from typing import TYPE_CHECKING
 
@@ -17,7 +16,6 @@
     list_volumes, open_volume, get_pool_names, get_pool_ids, get_pending_subvol_deletions_count
 from .operations.subvolume import open_subvol, create_subvol, remove_subvol, \
     create_clone
-from .operations.trash import Trash
 
 from .vol_spec import VolSpec
 from .exception import VolumeException, ClusterError, ClusterTimeout, EvictionError
@@ -338,7 +336,7 @@ def evict(self, **kwargs):
             with open_volume(self, volname) as fs_handle:
                 with open_group(fs_handle, self.volspec, groupname) as group:
                     with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.EVICT) as subvolume:
-                        key = subvolume.evict(volname, authid)
+                        subvolume.evict(volname, authid)
                         ret = 0, "", ""
         except (VolumeException, ClusterTimeout, ClusterError, EvictionError) as e:
             if isinstance(e, VolumeException):
@@ -695,7 +693,7 @@ def protect_subvolume_snapshot(self, **kwargs):
         try:
             with open_volume(self, volname) as fs_handle:
                 with open_group(fs_handle, self.volspec, groupname) as group:
-                    with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.SNAP_PROTECT) as subvolume:
+                    with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.SNAP_PROTECT):
                         log.warning("snapshot protect call is deprecated and will be removed in a future release")
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
@@ -710,7 +708,7 @@ def unprotect_subvolume_snapshot(self, **kwargs):
         try:
             with open_volume(self, volname) as fs_handle:
                 with open_group(fs_handle, self.volspec, groupname) as group:
-                    with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.SNAP_UNPROTECT) as subvolume:
+                    with open_subvol(self.mgr, fs_handle, self.volspec, group, subvolname, SubvolumeOpType.SNAP_UNPROTECT):
                         log.warning("snapshot unprotect call is deprecated and will be removed in a future release")
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
@@ -962,7 +960,7 @@ def create_subvolume_group_snapshot(self, **kwargs):
 
         try:
             with open_volume(self, volname) as fs_handle:
-                with open_group(fs_handle, self.volspec, groupname) as group:
+                with open_group(fs_handle, self.volspec, groupname):
                     # as subvolumes are marked with the vxattr ceph.dir.subvolume deny snapshots
                     # at the subvolume group (see: https://tracker.ceph.com/issues/46074)
                     # group.create_snapshot(snapname)

From 52bed3747e84e9d5e865f02a57513359c78ac607 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 09:19:37 -0500
Subject: [PATCH 0713/2492] rgw/sal: RGWGetUserStats_CB cleanup

the base class doesn't need member variables. the derived class in
rgw_quota.cc accepts stats by const-ref

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_quota.cc               | 27 ++++++++++++++-------------
 src/rgw/rgw_sal.h                  | 12 ++----------
 src/rgw/services/svc_user_rados.cc | 14 +++++---------
 3 files changed, 21 insertions(+), 32 deletions(-)

diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index c2f300b1fdb8..b245f7a97212 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -86,9 +86,9 @@ class RGWQuotaCache {
                 const DoutPrefixProvider* dpp);
   void adjust_stats(const rgw_user& user, rgw_bucket& bucket, int objs_delta, uint64_t added_bytes, uint64_t removed_bytes);
 
-  void set_stats(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats);
+  void set_stats(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs, const RGWStorageStats& stats);
   int async_refresh(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs);
-  void async_refresh_response(const rgw_user& user, rgw_bucket& bucket, RGWStorageStats& stats);
+  void async_refresh_response(const rgw_user& user, rgw_bucket& bucket, const RGWStorageStats& stats);
   void async_refresh_fail(const rgw_user& user, rgw_bucket& bucket);
 
   class AsyncRefreshHandler {
@@ -140,7 +140,7 @@ void RGWQuotaCache<T>::async_refresh_fail(const rgw_user& user, rgw_bucket& buck
 }
 
 template<class T>
-void RGWQuotaCache<T>::async_refresh_response(const rgw_user& user, rgw_bucket& bucket, RGWStorageStats& stats)
+void RGWQuotaCache<T>::async_refresh_response(const rgw_user& user, rgw_bucket& bucket, const RGWStorageStats& stats)
 {
   ldout(driver->ctx(), 20) << "async stats refresh response for bucket=" << bucket << dendl;
 
@@ -154,7 +154,7 @@ void RGWQuotaCache<T>::async_refresh_response(const rgw_user& user, rgw_bucket&
 }
 
 template<class T>
-void RGWQuotaCache<T>::set_stats(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs, RGWStorageStats& stats)
+void RGWQuotaCache<T>::set_stats(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs, const RGWStorageStats& stats)
 {
   qs.stats = stats;
   qs.expiration = ceph_clock_now();
@@ -380,17 +380,18 @@ class UserAsyncRefreshHandler : public RGWQuotaCache<rgw_user>::AsyncRefreshHand
                                 public RGWGetUserStats_CB {
   const DoutPrefixProvider *dpp;
   rgw_bucket bucket;
-public:
-  UserAsyncRefreshHandler(const DoutPrefixProvider *_dpp, rgw::sal::Driver* _driver, RGWQuotaCache<rgw_user> *_cache,
-                          const rgw_user& _user, const rgw_bucket& _bucket) :
-                          RGWQuotaCache<rgw_user>::AsyncRefreshHandler(_driver, _cache),
-                          RGWGetUserStats_CB(_user),
-                          dpp(_dpp),
-                          bucket(_bucket) {}
+  rgw_user user;
+ public:
+  UserAsyncRefreshHandler(const DoutPrefixProvider *_dpp, rgw::sal::Driver* _driver,
+                          RGWQuotaCache<rgw_user> *_cache,
+                          const rgw_user& _user, const rgw_bucket& _bucket)
+      : RGWQuotaCache<rgw_user>::AsyncRefreshHandler(_driver, _cache),
+        dpp(_dpp), bucket(_bucket), user(_user)
+  {}
 
   void drop_reference() override { put(); }
   int init_fetch() override;
-  void handle_response(int r) override;
+  void handle_response(int r, const RGWStorageStats& stats) override;
 };
 
 int UserAsyncRefreshHandler::init_fetch()
@@ -409,7 +410,7 @@ int UserAsyncRefreshHandler::init_fetch()
   return 0;
 }
 
-void UserAsyncRefreshHandler::handle_response(int r)
+void UserAsyncRefreshHandler::handle_response(int r, const RGWStorageStats& stats)
 {
   if (r < 0) {
     ldout(driver->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 9b099be4ac2c..73ba1e18fd12 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -95,16 +95,8 @@ class RGWGetBucketStats_CB : public RefCountedObject {
 };
 
 class RGWGetUserStats_CB : public RefCountedObject {
-protected:
-  rgw_user user;
-  RGWStorageStats stats;
-public:
-  explicit RGWGetUserStats_CB(const rgw_user& _user) : user(_user) {}
-  ~RGWGetUserStats_CB() override {}
-  virtual void handle_response(int r) = 0;
-  virtual void set_response(RGWStorageStats& _stats) {
-    stats = _stats;
-  }
+ public:
+  virtual void handle_response(int r, const RGWStorageStats& stats) = 0;
 };
 
 struct RGWObjState {
diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc
index c99af935415d..1b23dad53fbd 100644
--- a/src/rgw/services/svc_user_rados.cc
+++ b/src/rgw/services/svc_user_rados.cc
@@ -935,17 +935,13 @@ class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
 
   void handle_response(int r, cls_user_header& header) override {
     const cls_user_stats& hs = header.stats;
-    if (r >= 0) {
-      RGWStorageStats stats;
+    RGWStorageStats stats;
 
-      stats.size = hs.total_bytes;
-      stats.size_rounded = hs.total_bytes_rounded;
-      stats.num_objects = hs.total_entries;
+    stats.size = hs.total_bytes;
+    stats.size_rounded = hs.total_bytes_rounded;
+    stats.num_objects = hs.total_entries;
 
-      cb->set_response(stats);
-    }
-
-    cb->handle_response(r);
+    cb->handle_response(r, stats);
 
     cb->put();
   }

From 3bf735b4ab78f64e434feec7f11b84a2d9aa1a01 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 11:20:26 -0500
Subject: [PATCH 0714/2492] rgw/sal: unify ReadStatsCB for async user/bucket
 stats

`rgw::sal::Bucket::read_stats_async()` had returned stats as
`map<RGWObjCategory, RGWStorageStats>`, but the only caller in
rgw_quota.cc just summed up the categories into a single instance
of `RGWStorageStats`. moving this summation up into rgw_rados'
`RGWGetBucketStatsContext` allows us to unify these sal callbacks
around `RGWStorageStats` alone

the `RGWGetUserStats_CB`/`RGWGetBucketStats_CB` callbacks had inherited
from `RefCountedObject` and required manual reference counting. switch
to `boost::intrusive_ptr` for scope-based shared ownership

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc   |  4 +--
 src/rgw/driver/daos/rgw_sal_daos.h    |  4 +--
 src/rgw/driver/motr/rgw_sal_motr.cc   |  4 +--
 src/rgw/driver/motr/rgw_sal_motr.h    |  4 +--
 src/rgw/driver/posix/rgw_sal_posix.cc |  2 +-
 src/rgw/driver/posix/rgw_sal_posix.h  |  2 +-
 src/rgw/driver/rados/rgw_rados.cc     | 29 +++++++++++-----------
 src/rgw/driver/rados/rgw_rados.h      |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc |  4 +--
 src/rgw/driver/rados/rgw_sal_rados.h  |  4 +--
 src/rgw/driver/rados/rgw_user.h       |  2 --
 src/rgw/rgw_quota.cc                  | 35 ++++++++++++---------------
 src/rgw/rgw_sal.h                     | 34 +++++++++++---------------
 src/rgw/rgw_sal_dbstore.cc            |  4 +--
 src/rgw/rgw_sal_dbstore.h             |  4 +--
 src/rgw/rgw_sal_filter.cc             |  4 +--
 src/rgw/rgw_sal_filter.h              |  4 +--
 src/rgw/rgw_sal_fwd.h                 |  3 +++
 src/rgw/services/svc_user.h           |  5 ++--
 src/rgw/services/svc_user_rados.cc    | 14 +++++------
 src/rgw/services/svc_user_rados.h     |  3 +--
 src/test/rgw/test_rgw_lua.cc          |  2 +-
 22 files changed, 82 insertions(+), 91 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 3687f57b8c89..694320565797 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -179,7 +179,7 @@ int DaosUser::read_stats(const DoutPrefixProvider* dpp, optional_yield y,
 
 /* stats - Not for first pass */
 int DaosUser::read_stats_async(const DoutPrefixProvider* dpp,
-                               RGWGetUserStats_CB* cb) {
+                               boost::intrusive_ptr<ReadStatsCB> cb) {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
 
@@ -495,7 +495,7 @@ int DaosBucket::read_stats(const DoutPrefixProvider* dpp,
 int DaosBucket::read_stats_async(
     const DoutPrefixProvider* dpp,
     const bucket_index_layout_generation& idx_layout, int shard_id,
-    RGWGetBucketStats_CB* ctx) {
+    boost::intrusive_ptr<ReadStatsCB> ctx) {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
 
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 66f570064632..146ba7077a40 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -187,7 +187,7 @@ class DaosUser : public StoreUser {
                          ceph::real_time* last_stats_sync = nullptr,
                          ceph::real_time* last_stats_update = nullptr) override;
   virtual int read_stats_async(const DoutPrefixProvider* dpp,
-                               RGWGetUserStats_CB* cb) override;
+                               boost::intrusive_ptr<ReadStatsCB> cb) override;
   virtual int complete_flush_stats(const DoutPrefixProvider* dpp,
                                    optional_yield y) override;
   virtual int read_usage(
@@ -309,7 +309,7 @@ class DaosBucket : public StoreBucket {
   virtual int read_stats_async(const DoutPrefixProvider* dpp,
                                const bucket_index_layout_generation& idx_layout,
                                int shard_id,
-                               RGWGetBucketStats_CB* ctx) override;
+                               boost::intrusive_ptr<ReadStatsCB> ctx) override;
   virtual int sync_user_stats(const DoutPrefixProvider* dpp,
                               optional_yield y) override;
   virtual int check_bucket_shards(const DoutPrefixProvider* dpp) override;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 8cc8df987d2b..a352e9c9c138 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -289,7 +289,7 @@ int MotrUser::read_stats(const DoutPrefixProvider *dpp,
 }
 
 /* stats - Not for first pass */
-int MotrUser::read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB *cb)
+int MotrUser::read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb)
 {
   return 0;
 }
@@ -803,7 +803,7 @@ int MotrBucket::create_multipart_indices()
 
 int MotrBucket::read_stats_async(const DoutPrefixProvider *dpp,
                                  const bucket_index_layout_generation& idx_layout,
-                                 int shard_id, RGWGetBucketStats_CB *ctx)
+                                 int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx)
 {
   return 0;
 }
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 3a110e36f173..dad8ff8f4bd3 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -241,7 +241,7 @@ class MotrUser : public StoreUser {
         optional_yield y, RGWStorageStats* stats,
         ceph::real_time *last_stats_sync = nullptr,
         ceph::real_time *last_stats_update = nullptr) override;
-    virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) override;
+    virtual int read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb) override;
     virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
     virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
         bool* is_truncated, RGWUsageIter& usage_iter,
@@ -372,7 +372,7 @@ class MotrBucket : public StoreBucket {
         bool *syncstopped = nullptr) override;
     virtual int read_stats_async(const DoutPrefixProvider *dpp,
                                  const bucket_index_layout_generation& idx_layout,
-                                 int shard_id, RGWGetBucketStats_CB* ctx) override;
+                                 int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx) override;
     int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                         RGWBucketEnt* ent) override;
     int check_bucket_shards(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index fc48c4f8fa8c..6afa731d8e3f 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -959,7 +959,7 @@ int POSIXBucket::read_stats(const DoutPrefixProvider *dpp,
 
 int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp,
 				  const bucket_index_layout_generation& idx_layout,
-				  int shard_id, RGWGetBucketStats_CB* ctx)
+				  int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx)
 {
   return 0;
 }
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 1599808beff3..3b76295b0c47 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -195,7 +195,7 @@ class POSIXBucket : public StoreBucket {
 			 bool* syncstopped = nullptr) override;
   virtual int read_stats_async(const DoutPrefixProvider *dpp,
 			       const bucket_index_layout_generation& idx_layout,
-			       int shard_id, RGWGetBucketStats_CB* ctx) override;
+			       int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx) override;
   virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                               RGWBucketEnt* ent) override;
   virtual int check_bucket_shards(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 3632e66af595..06ddda412271 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -8467,34 +8467,36 @@ int RGWRados::get_bucket_stats(const DoutPrefixProvider *dpp,
 }
 
 class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
-  RGWGetBucketStats_CB *cb;
+  boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb;
   uint32_t pendings;
-  map<RGWObjCategory, RGWStorageStats> stats;
+  RGWStorageStats stats;
   int ret_code;
   bool should_cb;
   ceph::mutex lock = ceph::make_mutex("RGWGetBucketStatsContext");
 
 public:
-  RGWGetBucketStatsContext(RGWGetBucketStats_CB *_cb, uint32_t _pendings)
-    : cb(_cb), pendings(_pendings), stats(), ret_code(0), should_cb(true)
+  RGWGetBucketStatsContext(boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb, uint32_t _pendings)
+    : cb(std::move(cb)), pendings(_pendings), stats(), ret_code(0), should_cb(true)
   {}
 
   void handle_response(int r, rgw_bucket_dir_header& header) override {
     std::lock_guard l{lock};
     if (should_cb) {
-      if ( r >= 0) {
-        accumulate_raw_stats(header, stats);
+      if (r >= 0) {
+        for (const auto& [c, s] : header.stats) {
+          stats.size += s.total_size;
+          stats.size_rounded += s.total_size_rounded;
+          stats.size_utilized += s.actual_size;
+          stats.num_objects += s.num_entries;
+        }
       } else {
         ret_code = r;
       }
 
       // Are we all done?
       if (--pendings == 0) {
-        if (!ret_code) {
-          cb->set_response(&stats);
-        }
-        cb->handle_response(ret_code);
-        cb->put();
+        cb->handle_response(ret_code, stats);
+        cb.reset();
       }
     }
   }
@@ -8505,14 +8507,13 @@ class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
   }
 };
 
-int RGWRados::get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB *ctx)
+int RGWRados::get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb)
 {
   int num_aio = 0;
-  RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(ctx, bucket_info.layout.current_index.layout.normal.num_shards ? : 1);
+  RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(std::move(cb), bucket_info.layout.current_index.layout.normal.num_shards ? : 1);
   ceph_assert(get_ctx);
   int r = cls_bucket_head_async(dpp, bucket_info, idx_layout, shard_id, get_ctx, &num_aio);
   if (r < 0) {
-    ctx->put();
     if (num_aio) {
       get_ctx->unset_cb();
     }
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 77a08c068a64..19916992b9d5 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -1392,7 +1392,7 @@ class RGWRados
   int decode_policy(const DoutPrefixProvider *dpp, bufferlist& bl, ACLOwner *owner);
   int get_bucket_stats(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, std::string *bucket_ver, std::string *master_ver,
       std::map<RGWObjCategory, RGWStorageStats>& stats, std::string *max_marker, bool* syncstopped = NULL);
-  int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB *cb);
+  int get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb);
 
   int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, const std::map<std::string, bufferlist> *pattrs, const DoutPrefixProvider *dpp, optional_yield y);
   /* xxx dang obj_ctx -> svc */
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index ed6e2525ac2b..194c2bcd6ae1 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -187,7 +187,7 @@ int RadosUser::read_stats(const DoutPrefixProvider *dpp,
   return store->ctl()->user->read_stats(dpp, get_id(), stats, y, last_stats_sync, last_stats_update);
 }
 
-int RadosUser::read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb)
+int RadosUser::read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb)
 {
   return store->svc()->user->read_stats_async(dpp, get_id(), cb);
 }
@@ -531,7 +531,7 @@ int RadosBucket::read_stats(const DoutPrefixProvider *dpp,
 
 int RadosBucket::read_stats_async(const DoutPrefixProvider *dpp,
 				  const bucket_index_layout_generation& idx_layout,
-				  int shard_id, RGWGetBucketStats_CB* ctx)
+				  int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx)
 {
   return store->getRados()->get_bucket_stats_async(dpp, get_info(), idx_layout, shard_id, ctx);
 }
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 7efd7757c3c4..fbf14410c173 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -271,7 +271,7 @@ class RadosUser : public StoreUser {
                            optional_yield y, RGWStorageStats* stats,
 			   ceph::real_time* last_stats_sync = nullptr,
 			   ceph::real_time* last_stats_update = nullptr) override;
-    virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) override;
+    virtual int read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb) override;
     virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
     virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
 			   bool* is_truncated, RGWUsageIter& usage_iter,
@@ -512,7 +512,7 @@ class RadosBucket : public StoreBucket {
                            bool* syncstopped = nullptr) override;
     virtual int read_stats_async(const DoutPrefixProvider *dpp,
                                  const bucket_index_layout_generation& idx_layout,
-                                 int shard_id, RGWGetBucketStats_CB* ctx) override;
+                                 int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx) override;
     int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                         RGWBucketEnt* ent) override;
     int check_bucket_shards(const DoutPrefixProvider* dpp, uint64_t num_objs,
diff --git a/src/rgw/driver/rados/rgw_user.h b/src/rgw/driver/rados/rgw_user.h
index d5eb167daa3f..a0cd7ed84fee 100644
--- a/src/rgw/driver/rados/rgw_user.h
+++ b/src/rgw/driver/rados/rgw_user.h
@@ -30,8 +30,6 @@ class RGWUserCtl;
 class RGWBucketCtl;
 class RGWUserBuckets;
 
-class RGWGetUserStats_CB;
-
 /**
  * A string wrapper that includes encode/decode functions
  * for easily accessing a UID in all forms
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index b245f7a97212..6c76ce2367b5 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -247,16 +247,19 @@ void RGWQuotaCache<T>::adjust_stats(const rgw_user& user, rgw_bucket& bucket, in
 }
 
 class BucketAsyncRefreshHandler : public RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler,
-                                  public RGWGetBucketStats_CB {
+                                  public rgw::sal::ReadStatsCB {
   rgw_user user;
+  rgw_bucket bucket;
 public:
   BucketAsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<rgw_bucket> *_cache,
-                            const rgw_user& _user, const rgw_bucket& _bucket) :
-                                      RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_driver, _cache),
-                                      RGWGetBucketStats_CB(_bucket), user(_user) {}
+                            const rgw_user& _user, const rgw_bucket& _bucket)
+    : RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_driver, _cache),
+      user(_user), bucket(_bucket) {}
 
-  void drop_reference() override { put(); }
-  void handle_response(int r) override;
+  void drop_reference() override {
+    intrusive_ptr_release(this);
+  }
+  void handle_response(int r, const RGWStorageStats& stats) override;
   int init_fetch() override;
 };
 
@@ -289,7 +292,7 @@ int BucketAsyncRefreshHandler::init_fetch()
   return 0;
 }
 
-void BucketAsyncRefreshHandler::handle_response(const int r)
+void BucketAsyncRefreshHandler::handle_response(const int r, const RGWStorageStats& stats)
 {
   if (r < 0) {
     ldout(driver->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
@@ -297,17 +300,7 @@ void BucketAsyncRefreshHandler::handle_response(const int r)
     return;
   }
 
-  RGWStorageStats bs;
-
-  for (const auto& pair : *stats) {
-    const RGWStorageStats& s = pair.second;
-
-    bs.size += s.size;
-    bs.size_rounded += s.size_rounded;
-    bs.num_objects += s.num_objects;
-  }
-
-  cache->async_refresh_response(user, bucket, bs);
+  cache->async_refresh_response(user, bucket, stats);
 }
 
 class RGWBucketStatsCache : public RGWQuotaCache<rgw_bucket> {
@@ -377,7 +370,7 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_
 }
 
 class UserAsyncRefreshHandler : public RGWQuotaCache<rgw_user>::AsyncRefreshHandler,
-                                public RGWGetUserStats_CB {
+                                public rgw::sal::ReadStatsCB {
   const DoutPrefixProvider *dpp;
   rgw_bucket bucket;
   rgw_user user;
@@ -389,7 +382,9 @@ class UserAsyncRefreshHandler : public RGWQuotaCache<rgw_user>::AsyncRefreshHand
         dpp(_dpp), bucket(_bucket), user(_user)
   {}
 
-  void drop_reference() override { put(); }
+  void drop_reference() override {
+    intrusive_ptr_release(this);
+  }
   int init_fetch() override;
   void handle_response(int r, const RGWStorageStats& stats) override;
 };
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 73ba1e18fd12..38072f2c013e 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -15,6 +15,9 @@
 
 #pragma once
 
+#include <boost/intrusive_ptr.hpp>
+#include <boost/smart_ptr/intrusive_ref_counter.hpp>
+
 #include "common/tracer.h"
 #include "rgw_sal_fwd.h"
 #include "rgw_lua.h"
@@ -81,24 +84,6 @@ struct RGWClusterStat {
   uint64_t num_objects;
 };
 
-class RGWGetBucketStats_CB : public RefCountedObject {
-protected:
-  rgw_bucket bucket;
-  std::map<RGWObjCategory, RGWStorageStats>* stats;
-public:
-  explicit RGWGetBucketStats_CB(const rgw_bucket& _bucket) : bucket(_bucket), stats(NULL) {}
-  ~RGWGetBucketStats_CB() override {}
-  virtual void handle_response(int r) = 0;
-  virtual void set_response(std::map<RGWObjCategory, RGWStorageStats>* _stats) {
-    stats = _stats;
-  }
-};
-
-class RGWGetUserStats_CB : public RefCountedObject {
- public:
-  virtual void handle_response(int r, const RGWStorageStats& stats) = 0;
-};
-
 struct RGWObjState {
   rgw_obj obj;
   bool is_atomic{false};
@@ -441,6 +426,14 @@ class Driver {
     virtual void register_admin_apis(RGWRESTMgr* mgr) = 0;
 };
 
+
+/// \brief Ref-counted callback object for User/Bucket read_stats_async().
+class ReadStatsCB : public boost::intrusive_ref_counter<ReadStatsCB> {
+ public:
+  virtual ~ReadStatsCB() {}
+  virtual void handle_response(int r, const RGWStorageStats& stats) = 0;
+};
+
 /**
  * @brief A list of buckets
  *
@@ -522,7 +515,8 @@ class User {
 			   ceph::real_time* last_stats_sync = nullptr,
 			   ceph::real_time* last_stats_update = nullptr) = 0;
     /** Read the User stats from the backing Store, asynchronous */
-    virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) = 0;
+    virtual int read_stats_async(const DoutPrefixProvider *dpp,
+                                 boost::intrusive_ptr<ReadStatsCB> cb) = 0;
     /** Flush accumulated stat changes for this User to the backing store */
     virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0;
     /** Read detailed usage stats for this User from the backing store */
@@ -675,7 +669,7 @@ class Bucket {
     /** Read the bucket stats from the backing Store, asynchronous */
     virtual int read_stats_async(const DoutPrefixProvider *dpp,
 				 const bucket_index_layout_generation& idx_layout,
-				 int shard_id, RGWGetBucketStats_CB* ctx) = 0;
+				 int shard_id, boost::intrusive_ptr<ReadStatsCB> cb) = 0;
     /** Sync this bucket's stats to the owning user's stats in the backing store */
     virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                                 RGWBucketEnt* optional_ent) = 0;
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 6c8778b87238..2305b217f888 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -90,7 +90,7 @@ namespace rgw::sal {
   }
 
   /* stats - Not for first pass */
-  int DBUser::read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB *cb)
+  int DBUser::read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb)
   {
     return 0;
   }
@@ -221,7 +221,7 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBBucket::read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB *ctx)
+  int DBBucket::read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx)
   {
     return 0;
   }
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 646ee23f8875..bfa424e269fe 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -93,7 +93,7 @@ class DBNotification : public StoreNotification {
           optional_yield y, RGWStorageStats* stats,
           ceph::real_time *last_stats_sync = nullptr,
           ceph::real_time *last_stats_update = nullptr) override;
-      virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) override;
+      virtual int read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb) override;
       virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
       virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
           bool* is_truncated, RGWUsageIter& usage_iter,
@@ -155,7 +155,7 @@ class DBNotification : public StoreNotification {
           std::map<RGWObjCategory, RGWStorageStats>& stats,
           std::string *max_marker = nullptr,
           bool *syncstopped = nullptr) override;
-      virtual int read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override;
+      virtual int read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx) override;
       int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                           RGWBucketEnt* ent) override;
       int check_bucket_shards(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index b49c6887b62e..decd648adb0f 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -508,7 +508,7 @@ int FilterUser::read_stats(const DoutPrefixProvider *dpp,
   return next->read_stats(dpp, y, stats, last_stats_sync, last_stats_update);
 }
 
-int FilterUser::read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb)
+int FilterUser::read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<ReadStatsCB> cb)
 {
   return next->read_stats_async(dpp, cb);
 }
@@ -613,7 +613,7 @@ int FilterBucket::read_stats(const DoutPrefixProvider *dpp,
 
 int FilterBucket::read_stats_async(const DoutPrefixProvider *dpp,
 				   const bucket_index_layout_generation& idx_layout,
-				   int shard_id, RGWGetBucketStats_CB* ctx)
+				   int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx)
 {
   return next->read_stats_async(dpp, idx_layout, shard_id, ctx);
 }
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 239a80a64653..aebbe383714d 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -338,7 +338,7 @@ class FilterUser : public User {
 			 ceph::real_time* last_stats_sync = nullptr,
 			 ceph::real_time* last_stats_update = nullptr) override;
   virtual int read_stats_async(const DoutPrefixProvider *dpp,
-			       RGWGetUserStats_CB* cb) override;
+			       boost::intrusive_ptr<ReadStatsCB> cb) override;
   virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
   virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch,
 			 uint64_t end_epoch, uint32_t max_entries,
@@ -397,7 +397,7 @@ class FilterBucket : public Bucket {
 			 bool* syncstopped = nullptr) override;
   virtual int read_stats_async(const DoutPrefixProvider *dpp,
 			       const bucket_index_layout_generation& idx_layout,
-			       int shard_id, RGWGetBucketStats_CB* ctx) override;
+			       int shard_id, boost::intrusive_ptr<ReadStatsCB> ctx) override;
   int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y,
                       RGWBucketEnt* ent) override;
   int check_bucket_shards(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/rgw_sal_fwd.h b/src/rgw/rgw_sal_fwd.h
index 08866c2bea28..ee7c100080b5 100644
--- a/src/rgw/rgw_sal_fwd.h
+++ b/src/rgw/rgw_sal_fwd.h
@@ -32,6 +32,9 @@ namespace rgw { namespace sal {
   class Zone;
   class LuaManager;
   struct RGWRoleInfo;
+  class DataProcessor;
+  class ObjectProcessor;
+  class ReadStatsCB;
 
   class ConfigStore;
   class RealmWriter;
diff --git a/src/rgw/services/svc_user.h b/src/rgw/services/svc_user.h
index 1cb459d31cb2..f53ec49f2595 100644
--- a/src/rgw/services/svc_user.h
+++ b/src/rgw/services/svc_user.h
@@ -20,9 +20,9 @@
 #include "svc_meta_be.h"
 
 #include "rgw_service.h"
+#include "rgw_sal_fwd.h"
 
 class RGWUserBuckets;
-class RGWGetUserStats_CB;
 
 class RGWSI_User : public RGWServiceInstance
 {
@@ -122,6 +122,7 @@ class RGWSI_User : public RGWServiceInstance
                          optional_yield y) = 0;  /* last time a stats update was done */
 
   virtual int read_stats_async(const DoutPrefixProvider *dpp,
-			       const rgw_user& user, RGWGetUserStats_CB *cb) = 0;
+			       const rgw_user& user,
+			       boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb) = 0;
 };
 
diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc
index 1b23dad53fbd..b9dd8cd15535 100644
--- a/src/rgw/services/svc_user_rados.cc
+++ b/src/rgw/services/svc_user_rados.cc
@@ -927,11 +927,11 @@ int RGWSI_User_RADOS::read_stats(const DoutPrefixProvider *dpp,
 }
 
 class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
-  RGWGetUserStats_CB *cb;
+  boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb;
 
 public:
-  explicit RGWGetUserStatsContext(RGWGetUserStats_CB * const cb)
-    : cb(cb) {}
+  explicit RGWGetUserStatsContext(boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb)
+    : cb(std::move(cb)) {}
 
   void handle_response(int r, cls_user_header& header) override {
     const cls_user_stats& hs = header.stats;
@@ -942,17 +942,17 @@ class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
     stats.num_objects = hs.total_entries;
 
     cb->handle_response(r, stats);
-
-    cb->put();
+    cb.reset();
   }
 };
 
 int RGWSI_User_RADOS::read_stats_async(const DoutPrefixProvider *dpp,
-                                       const rgw_user& user, RGWGetUserStats_CB *_cb)
+                                       const rgw_user& user,
+                                       boost::intrusive_ptr<rgw::sal::ReadStatsCB> _cb)
 {
   string user_str = user.to_str();
 
-  RGWGetUserStatsContext *cb = new RGWGetUserStatsContext(_cb);
+  RGWGetUserStatsContext *cb = new RGWGetUserStatsContext(std::move(_cb));
   int r = cls_user_get_header_async(dpp, user_str, cb);
   if (r < 0) {
     delete cb;
diff --git a/src/rgw/services/svc_user_rados.h b/src/rgw/services/svc_user_rados.h
index 4712d1fdafd4..01d8a8eb1ccb 100644
--- a/src/rgw/services/svc_user_rados.h
+++ b/src/rgw/services/svc_user_rados.h
@@ -34,7 +34,6 @@ class RGWSI_MetaBackend_Handler;
 struct rgw_cache_entry_info;
 
 class RGWGetUserHeader_CB;
-class RGWGetUserStats_CB;
 
 template <class T>
 class RGWChainedCacheImpl;
@@ -207,6 +206,6 @@ class RGWSI_User_RADOS : public RGWSI_User
                  optional_yield y) override;  /* last time a stats update was done */
 
   int read_stats_async(const DoutPrefixProvider *dpp, const rgw_user& user,
-                       RGWGetUserStats_CB *cb) override;
+                       boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb) override;
 };
 
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index be9fa6aed162..dfd2767f5503 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -86,7 +86,7 @@ class TestUser : public sal::StoreUser {
     return 0;
   }
 
-  virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB *cb) override {
+  virtual int read_stats_async(const DoutPrefixProvider *dpp, boost::intrusive_ptr<sal::ReadStatsCB> cb) override {
     return 0;
   }
 

From 131d49a9852d2003b605f92308abcd9abc07e441 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 12:32:14 -0500
Subject: [PATCH 0715/2492] cls/rgw: cls_bucket_head_async uses intrusive_ptr
 for RGWGetDirHeader_CB

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/cls/rgw/cls_rgw_client.cc     | 19 +++++++++----------
 src/cls/rgw/cls_rgw_client.h      | 11 +++++++----
 src/rgw/driver/rados/rgw_rados.cc | 27 ++++++++++++---------------
 src/rgw/driver/rados/rgw_rados.h  |  2 +-
 4 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 5e7fba88f24a..aa576129252a 100644
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -751,12 +751,11 @@ int CLSRGWIssueBucketBILogStop::issue_op(const int shard_id, const string& oid)
 }
 
 class GetDirHeaderCompletion : public ObjectOperationCompletion {
-  RGWGetDirHeader_CB *ret_ctx;
+  boost::intrusive_ptr<RGWGetDirHeader_CB> cb;
 public:
-  explicit GetDirHeaderCompletion(RGWGetDirHeader_CB *_ctx) : ret_ctx(_ctx) {}
-  ~GetDirHeaderCompletion() override {
-    ret_ctx->put();
-  }
+  explicit GetDirHeaderCompletion(boost::intrusive_ptr<RGWGetDirHeader_CB> cb)
+    : cb(std::move(cb)) {}
+
   void handle_completion(int r, bufferlist& outbl) override {
     rgw_cls_list_ret ret;
     try {
@@ -765,20 +764,20 @@ class GetDirHeaderCompletion : public ObjectOperationCompletion {
     } catch (ceph::buffer::error& err) {
       r = -EIO;
     }
-
-    ret_ctx->handle_response(r, ret.dir.header);
+    cb->handle_response(r, ret.dir.header);
   }
 };
 
-int cls_rgw_get_dir_header_async(IoCtx& io_ctx, string& oid, RGWGetDirHeader_CB *ctx)
+int cls_rgw_get_dir_header_async(IoCtx& io_ctx, const string& oid,
+                                 boost::intrusive_ptr<RGWGetDirHeader_CB> cb)
 {
   bufferlist in, out;
   rgw_cls_list_op call;
   call.num_entries = 0;
   encode(call, in);
   ObjectReadOperation op;
-  GetDirHeaderCompletion *cb = new GetDirHeaderCompletion(ctx);
-  op.exec(RGW_CLASS, RGW_BUCKET_LIST, in, cb);
+  op.exec(RGW_CLASS, RGW_BUCKET_LIST, in,
+          new GetDirHeaderCompletion(std::move(cb)));
   AioCompletion *c = librados::Rados::aio_create_completion(nullptr, nullptr);
   int r = io_ctx.aio_operate(oid, c, &op, NULL);
   c->release();
diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 1ae49c877bb4..4ef10245404b 100644
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include <boost/intrusive_ptr.hpp>
+#include <boost/smart_ptr/intrusive_ref_counter.hpp>
 #include "include/str_list.h"
 #include "include/rados/librados.hpp"
 #include "cls_rgw_ops.h"
@@ -151,10 +153,10 @@ class BucketIndexAioManager {
   }
 };
 
-class RGWGetDirHeader_CB : public RefCountedObject {
+class RGWGetDirHeader_CB : public boost::intrusive_ref_counter<RGWGetDirHeader_CB> {
 public:
-  ~RGWGetDirHeader_CB() override {}
-  virtual void handle_response(int r, rgw_bucket_dir_header& header) = 0;
+  virtual ~RGWGetDirHeader_CB() {}
+  virtual void handle_response(int r, const rgw_bucket_dir_header& header) = 0;
 };
 
 class BucketIndexShardsManager {
@@ -572,7 +574,8 @@ class CLSRGWIssueBucketBILogStop : public CLSRGWConcurrentIO {
   virtual ~CLSRGWIssueBucketBILogStop() override {}
 };
 
-int cls_rgw_get_dir_header_async(librados::IoCtx& io_ctx, std::string& oid, RGWGetDirHeader_CB *ctx);
+int cls_rgw_get_dir_header_async(librados::IoCtx& io_ctx, const std::string& oid,
+                                 boost::intrusive_ptr<RGWGetDirHeader_CB> cb);
 
 void cls_rgw_encode_suggestion(char op, rgw_bucket_dir_entry& dirent, ceph::buffer::list& updates);
 
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 06ddda412271..0d11fc10e44e 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -8479,7 +8479,7 @@ class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
     : cb(std::move(cb)), pendings(_pendings), stats(), ret_code(0), should_cb(true)
   {}
 
-  void handle_response(int r, rgw_bucket_dir_header& header) override {
+  void handle_response(int r, const rgw_bucket_dir_header& header) override {
     std::lock_guard l{lock};
     if (should_cb) {
       if (r >= 0) {
@@ -8510,15 +8510,13 @@ class RGWGetBucketStatsContext : public RGWGetDirHeader_CB {
 int RGWRados::get_bucket_stats_async(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, boost::intrusive_ptr<rgw::sal::ReadStatsCB> cb)
 {
   int num_aio = 0;
-  RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(std::move(cb), bucket_info.layout.current_index.layout.normal.num_shards ? : 1);
-  ceph_assert(get_ctx);
-  int r = cls_bucket_head_async(dpp, bucket_info, idx_layout, shard_id, get_ctx, &num_aio);
+  boost::intrusive_ptr headercb = new RGWGetBucketStatsContext(std::move(cb), bucket_info.layout.current_index.layout.normal.num_shards ? : 1);
+  int r = cls_bucket_head_async(dpp, bucket_info, idx_layout, shard_id, headercb, &num_aio);
   if (r < 0) {
     if (num_aio) {
-      get_ctx->unset_cb();
+      headercb->unset_cb();
     }
   }
-  get_ctx->put();
   return r;
 }
 
@@ -10045,7 +10043,9 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo
   return 0;
 }
 
-int RGWRados::cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio)
+int RGWRados::cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info,
+                                    const rgw::bucket_index_layout_generation& idx_layout, int shard_id,
+                                    boost::intrusive_ptr<RGWGetDirHeader_CB> cb, int *num_aio)
 {
   RGWSI_RADOS::Pool index_pool;
   map<int, string> bucket_objs;
@@ -10053,17 +10053,14 @@ int RGWRados::cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBuck
   if (r < 0)
     return r;
 
-  map<int, string>::iterator iter = bucket_objs.begin();
-  for (; iter != bucket_objs.end(); ++iter) {
-    r = cls_rgw_get_dir_header_async(index_pool.ioctx(), iter->second, static_cast<RGWGetDirHeader_CB*>(ctx->get()));
+  for (auto& pair : bucket_objs) {
+    r = cls_rgw_get_dir_header_async(index_pool.ioctx(), pair.second, cb);
     if (r < 0) {
-      ctx->put();
-      break;
-    } else {
-      (*num_aio)++;
+      return r;
     }
+    (*num_aio)++;
   }
-  return r;
+  return 0;
 }
 
 int RGWRados::check_bucket_shards(const RGWBucketInfo& bucket_info,
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 19916992b9d5..38e69048e914 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -1474,7 +1474,7 @@ class RGWRados
   int cls_bucket_head_async(const DoutPrefixProvider *dpp,
 			    const RGWBucketInfo& bucket_info,
 			    const rgw::bucket_index_layout_generation& idx_layout,
-			    int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio);
+			    int shard_id, boost::intrusive_ptr<RGWGetDirHeader_CB> cb, int *num_aio);
   int bi_get_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent, optional_yield y);
   int bi_get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_olh_entry *olh, optional_yield y);
   int bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry, optional_yield y);

From 24de70f59728d4b9b7da1e7aee2975add7a6af1c Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 14:29:23 -0500
Subject: [PATCH 0716/2492] rgw/quota: use intrusive_ptr for
 RefCountedWaitObject

instead of relying on async_refresh_response/fail to manage ref counts
manually, just stash an intrusive_ptr in AsyncRefreshHandler for the
duration of the async operation

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/RefCountedObj.h |  8 +++++++-
 src/rgw/rgw_quota.cc       | 34 +++++++++++++++++-----------------
 2 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h
index ef966463cda9..a26677573996 100644
--- a/src/common/RefCountedObj.h
+++ b/src/common/RefCountedObj.h
@@ -180,6 +180,12 @@ struct RefCountedWaitObject {
   }
 };
 
+static inline void intrusive_ptr_add_ref(RefCountedWaitObject *p) {
+  p->get();
+}
+static inline void intrusive_ptr_release(RefCountedWaitObject *p) {
+  p->put();
+}
 #endif // !defined(WITH_SEASTAR)|| defined(WITH_ALIEN)
 
 static inline void intrusive_ptr_add_ref(const RefCountedObject *p) {
@@ -196,7 +202,7 @@ struct UniquePtrDeleter
     p->put();
   }
 };
-}
+} // namespace TOPNSPC::common
 using RefCountedPtr = ceph::ref_t<TOPNSPC::common::RefCountedObject>;
 
 #endif
diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index 6c76ce2367b5..ff13a6501ccc 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -95,15 +95,19 @@ class RGWQuotaCache {
   protected:
     rgw::sal::Driver* driver;
     RGWQuotaCache<T> *cache;
+    boost::intrusive_ptr<RefCountedWaitObject> waiter;
   public:
-    AsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<T> *_cache) : driver(_driver), cache(_cache) {}
+    AsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<T> *_cache,
+                        boost::intrusive_ptr<RefCountedWaitObject> waiter)
+      : driver(_driver), cache(_cache), waiter(std::move(waiter)) {}
     virtual ~AsyncRefreshHandler() {}
 
     virtual int init_fetch() = 0;
     virtual void drop_reference() = 0;
   };
 
-  virtual AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket) = 0;
+  virtual AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
+                                                        boost::intrusive_ptr<RefCountedWaitObject> waiter) = 0;
 };
 
 template<class T>
@@ -116,14 +120,10 @@ int RGWQuotaCache<T>::async_refresh(const rgw_user& user, const rgw_bucket& buck
     return 0;
   }
 
-  async_refcount->get();
-
-
-  AsyncRefreshHandler *handler = allocate_refresh_handler(user, bucket);
+  AsyncRefreshHandler *handler = allocate_refresh_handler(user, bucket, async_refcount);
 
   int ret = handler->init_fetch();
   if (ret < 0) {
-    async_refcount->put();
     handler->drop_reference();
     return ret;
   }
@@ -135,8 +135,6 @@ template<class T>
 void RGWQuotaCache<T>::async_refresh_fail(const rgw_user& user, rgw_bucket& bucket)
 {
   ldout(driver->ctx(), 20) << "async stats refresh response for bucket=" << bucket << dendl;
-
-  async_refcount->put();
 }
 
 template<class T>
@@ -149,8 +147,6 @@ void RGWQuotaCache<T>::async_refresh_response(const rgw_user& user, rgw_bucket&
   map_find(user, bucket, qs);
 
   set_stats(user, bucket, qs, stats);
-
-  async_refcount->put();
 }
 
 template<class T>
@@ -252,8 +248,9 @@ class BucketAsyncRefreshHandler : public RGWQuotaCache<rgw_bucket>::AsyncRefresh
   rgw_bucket bucket;
 public:
   BucketAsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<rgw_bucket> *_cache,
+                            boost::intrusive_ptr<RefCountedWaitObject> waiter,
                             const rgw_user& _user, const rgw_bucket& _bucket)
-    : RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_driver, _cache),
+    : RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_driver, _cache, std::move(waiter)),
       user(_user), bucket(_bucket) {}
 
   void drop_reference() override {
@@ -323,8 +320,9 @@ class RGWBucketStatsCache : public RGWQuotaCache<rgw_bucket> {
   explicit RGWBucketStatsCache(rgw::sal::Driver* _driver) : RGWQuotaCache<rgw_bucket>(_driver, _driver->ctx()->_conf->rgw_bucket_quota_cache_size) {
   }
 
-  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket) override {
-    return new BucketAsyncRefreshHandler(driver, this, user, bucket);
+  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
+                                                boost::intrusive_ptr<RefCountedWaitObject> waiter) override {
+    return new BucketAsyncRefreshHandler(driver, this, std::move(waiter), user, bucket);
   }
 };
 
@@ -377,8 +375,9 @@ class UserAsyncRefreshHandler : public RGWQuotaCache<rgw_user>::AsyncRefreshHand
  public:
   UserAsyncRefreshHandler(const DoutPrefixProvider *_dpp, rgw::sal::Driver* _driver,
                           RGWQuotaCache<rgw_user> *_cache,
+                          boost::intrusive_ptr<RefCountedWaitObject> waiter,
                           const rgw_user& _user, const rgw_bucket& _bucket)
-      : RGWQuotaCache<rgw_user>::AsyncRefreshHandler(_driver, _cache),
+      : RGWQuotaCache<rgw_user>::AsyncRefreshHandler(_driver, _cache, std::move(waiter)),
         dpp(_dpp), bucket(_bucket), user(_user)
   {}
 
@@ -570,8 +569,9 @@ class RGWUserStatsCache : public RGWQuotaCache<rgw_user> {
     stop();
   }
 
-  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket) override {
-    return new UserAsyncRefreshHandler(dpp, driver, this, user, bucket);
+  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
+                                                boost::intrusive_ptr<RefCountedWaitObject> waiter) override {
+    return new UserAsyncRefreshHandler(dpp, driver, this, std::move(waiter), user, bucket);
   }
 
   bool going_down() {

From 9b1aed0838c415ab79bf7436aefe7c394e929355 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 14:42:07 -0500
Subject: [PATCH 0717/2492] rgw/quota: remove base class AsyncRefreshHandler

instead of using a virtual `allocate_refresh_handler()` to create an
object to call the virtual `AsyncRefreshHandler::init_fetch()` on, just
provide one virtual `init_refresh()` to start the operation. this avoids
the need for `AsyncRefreshHandler` altogether

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_quota.cc | 211 ++++++++++++++++++-------------------------
 1 file changed, 88 insertions(+), 123 deletions(-)

diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc
index ff13a6501ccc..836086b5be29 100644
--- a/src/rgw/rgw_quota.cc
+++ b/src/rgw/rgw_quota.cc
@@ -91,23 +91,10 @@ class RGWQuotaCache {
   void async_refresh_response(const rgw_user& user, rgw_bucket& bucket, const RGWStorageStats& stats);
   void async_refresh_fail(const rgw_user& user, rgw_bucket& bucket);
 
-  class AsyncRefreshHandler {
-  protected:
-    rgw::sal::Driver* driver;
-    RGWQuotaCache<T> *cache;
-    boost::intrusive_ptr<RefCountedWaitObject> waiter;
-  public:
-    AsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<T> *_cache,
-                        boost::intrusive_ptr<RefCountedWaitObject> waiter)
-      : driver(_driver), cache(_cache), waiter(std::move(waiter)) {}
-    virtual ~AsyncRefreshHandler() {}
-
-    virtual int init_fetch() = 0;
-    virtual void drop_reference() = 0;
-  };
-
-  virtual AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
-                                                        boost::intrusive_ptr<RefCountedWaitObject> waiter) = 0;
+  /// start an async refresh that will eventually call async_refresh_response or
+  /// async_refresh_fail. hold a reference to the waiter until completion
+  virtual int init_refresh(const rgw_user& user, const rgw_bucket& bucket,
+                           boost::intrusive_ptr<RefCountedWaitObject> waiter) = 0;
 };
 
 template<class T>
@@ -120,15 +107,7 @@ int RGWQuotaCache<T>::async_refresh(const rgw_user& user, const rgw_bucket& buck
     return 0;
   }
 
-  AsyncRefreshHandler *handler = allocate_refresh_handler(user, bucket, async_refcount);
-
-  int ret = handler->init_fetch();
-  if (ret < 0) {
-    handler->drop_reference();
-    return ret;
-  }
-
-  return 0;
+  return init_refresh(user, bucket, async_refcount);
 }
 
 template<class T>
@@ -242,64 +221,6 @@ void RGWQuotaCache<T>::adjust_stats(const rgw_user& user, rgw_bucket& bucket, in
   data_modified(user, bucket);
 }
 
-class BucketAsyncRefreshHandler : public RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler,
-                                  public rgw::sal::ReadStatsCB {
-  rgw_user user;
-  rgw_bucket bucket;
-public:
-  BucketAsyncRefreshHandler(rgw::sal::Driver* _driver, RGWQuotaCache<rgw_bucket> *_cache,
-                            boost::intrusive_ptr<RefCountedWaitObject> waiter,
-                            const rgw_user& _user, const rgw_bucket& _bucket)
-    : RGWQuotaCache<rgw_bucket>::AsyncRefreshHandler(_driver, _cache, std::move(waiter)),
-      user(_user), bucket(_bucket) {}
-
-  void drop_reference() override {
-    intrusive_ptr_release(this);
-  }
-  void handle_response(int r, const RGWStorageStats& stats) override;
-  int init_fetch() override;
-};
-
-int BucketAsyncRefreshHandler::init_fetch()
-{
-  std::unique_ptr<rgw::sal::Bucket> rbucket;
-
-  const DoutPrefix dp(driver->ctx(), dout_subsys, "rgw bucket async refresh handler: ");
-  int r = driver->load_bucket(&dp, bucket, &rbucket, null_yield);
-  if (r < 0) {
-    ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
-    return r;
-  }
-
-  ldpp_dout(&dp, 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
-
-  const auto& index = rbucket->get_info().get_current_index();
-  if (is_layout_indexless(index)) {
-    return 0;
-  }
-
-  r = rbucket->read_stats_async(&dp, index, RGW_NO_SHARD, this);
-  if (r < 0) {
-    ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
-
-    /* read_stats_async() dropped our reference already */
-    return r;
-  }
-
-  return 0;
-}
-
-void BucketAsyncRefreshHandler::handle_response(const int r, const RGWStorageStats& stats)
-{
-  if (r < 0) {
-    ldout(driver->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
-    cache->async_refresh_fail(user, bucket);
-    return;
-  }
-
-  cache->async_refresh_response(user, bucket, stats);
-}
-
 class RGWBucketStatsCache : public RGWQuotaCache<rgw_bucket> {
 protected:
   bool map_find(const rgw_user& user, const rgw_bucket& bucket, RGWQuotaCacheStats& qs) override {
@@ -320,10 +241,8 @@ class RGWBucketStatsCache : public RGWQuotaCache<rgw_bucket> {
   explicit RGWBucketStatsCache(rgw::sal::Driver* _driver) : RGWQuotaCache<rgw_bucket>(_driver, _driver->ctx()->_conf->rgw_bucket_quota_cache_size) {
   }
 
-  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
-                                                boost::intrusive_ptr<RefCountedWaitObject> waiter) override {
-    return new BucketAsyncRefreshHandler(driver, this, std::move(waiter), user, bucket);
-  }
+  int init_refresh(const rgw_user& user, const rgw_bucket& bucket,
+                   boost::intrusive_ptr<RefCountedWaitObject> waiter) override;
 };
 
 int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_bucket& _b, RGWStorageStats& stats, optional_yield y, const DoutPrefixProvider *dpp)
@@ -367,52 +286,57 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_
   return 0;
 }
 
-class UserAsyncRefreshHandler : public RGWQuotaCache<rgw_user>::AsyncRefreshHandler,
-                                public rgw::sal::ReadStatsCB {
-  const DoutPrefixProvider *dpp;
-  rgw_bucket bucket;
+class BucketAsyncRefreshHandler : public rgw::sal::ReadStatsCB {
+  RGWBucketStatsCache* cache;
+  boost::intrusive_ptr<RefCountedWaitObject> waiter;
   rgw_user user;
- public:
-  UserAsyncRefreshHandler(const DoutPrefixProvider *_dpp, rgw::sal::Driver* _driver,
-                          RGWQuotaCache<rgw_user> *_cache,
-                          boost::intrusive_ptr<RefCountedWaitObject> waiter,
-                          const rgw_user& _user, const rgw_bucket& _bucket)
-      : RGWQuotaCache<rgw_user>::AsyncRefreshHandler(_driver, _cache, std::move(waiter)),
-        dpp(_dpp), bucket(_bucket), user(_user)
-  {}
+  rgw_bucket bucket;
+public:
+  BucketAsyncRefreshHandler(RGWBucketStatsCache* cache,
+                            boost::intrusive_ptr<RefCountedWaitObject> waiter,
+                            const rgw_user& user, const rgw_bucket& bucket)
+    : cache(cache), waiter(std::move(waiter)), user(user), bucket(bucket) {}
 
-  void drop_reference() override {
-    intrusive_ptr_release(this);
+  void handle_response(int r, const RGWStorageStats& stats) override {
+    if (r < 0) {
+      cache->async_refresh_fail(user, bucket);
+      return;
+    }
+
+    cache->async_refresh_response(user, bucket, stats);
   }
-  int init_fetch() override;
-  void handle_response(int r, const RGWStorageStats& stats) override;
 };
 
-int UserAsyncRefreshHandler::init_fetch()
+
+int RGWBucketStatsCache::init_refresh(const rgw_user& user, const rgw_bucket& bucket,
+                                     boost::intrusive_ptr<RefCountedWaitObject> waiter)
 {
-  std::unique_ptr<rgw::sal::User> ruser = driver->get_user(user);
+  std::unique_ptr<rgw::sal::Bucket> rbucket;
 
-  ldpp_dout(dpp, 20) << "initiating async quota refresh for user=" << user << dendl;
-  int r = ruser->read_stats_async(dpp, this);
+  const DoutPrefix dp(driver->ctx(), dout_subsys, "rgw bucket async refresh handler: ");
+  int r = driver->load_bucket(&dp, bucket, &rbucket, null_yield);
   if (r < 0) {
-    ldpp_dout(dpp, 0) << "could not get bucket info for user=" << user << dendl;
-
-    /* get_bucket_stats_async() dropped our reference already */
+    ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
     return r;
   }
 
-  return 0;
-}
+  ldpp_dout(&dp, 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
 
-void UserAsyncRefreshHandler::handle_response(int r, const RGWStorageStats& stats)
-{
+  const auto& index = rbucket->get_info().get_current_index();
+  if (is_layout_indexless(index)) {
+    return 0;
+  }
+
+  boost::intrusive_ptr handler = new BucketAsyncRefreshHandler(
+      this, std::move(waiter), user, bucket);
+
+  r = rbucket->read_stats_async(&dp, index, RGW_NO_SHARD, std::move(handler));
   if (r < 0) {
-    ldout(driver->ctx(), 20) << "AsyncRefreshHandler::handle_response() r=" << r << dendl;
-    cache->async_refresh_fail(user, bucket);
-    return;
+    ldpp_dout(&dp, 0) << "could not get bucket stats for bucket=" << bucket.name << dendl;
+    return r;
   }
 
-  cache->async_refresh_response(user, bucket, stats);
+  return 0;
 }
 
 class RGWUserStatsCache : public RGWQuotaCache<rgw_user> {
@@ -569,10 +493,8 @@ class RGWUserStatsCache : public RGWQuotaCache<rgw_user> {
     stop();
   }
 
-  AsyncRefreshHandler *allocate_refresh_handler(const rgw_user& user, const rgw_bucket& bucket,
-                                                boost::intrusive_ptr<RefCountedWaitObject> waiter) override {
-    return new UserAsyncRefreshHandler(dpp, driver, this, std::move(waiter), user, bucket);
-  }
+  int init_refresh(const rgw_user& user, const rgw_bucket& bucket,
+                   boost::intrusive_ptr<RefCountedWaitObject> waiter) override;
 
   bool going_down() {
     return down_flag;
@@ -588,6 +510,49 @@ class RGWUserStatsCache : public RGWQuotaCache<rgw_user> {
   }
 };
 
+class UserAsyncRefreshHandler : public rgw::sal::ReadStatsCB {
+  RGWUserStatsCache* cache;
+  boost::intrusive_ptr<RefCountedWaitObject> waiter;
+  rgw_bucket bucket;
+  rgw_user user;
+ public:
+  UserAsyncRefreshHandler(RGWUserStatsCache* cache,
+                          boost::intrusive_ptr<RefCountedWaitObject> waiter,
+                          const rgw_user& user, const rgw_bucket& bucket)
+      : cache(cache), waiter(std::move(waiter)), bucket(bucket), user(user)
+  {}
+
+  void handle_response(int r, const RGWStorageStats& stats) override;
+};
+
+int RGWUserStatsCache::init_refresh(const rgw_user& user, const rgw_bucket& bucket,
+                                    boost::intrusive_ptr<RefCountedWaitObject> waiter)
+{
+  boost::intrusive_ptr handler = new UserAsyncRefreshHandler(
+      this, std::move(waiter), user, bucket);
+
+  std::unique_ptr<rgw::sal::User> ruser = driver->get_user(user);
+
+  ldpp_dout(dpp, 20) << "initiating async quota refresh for user=" << user << dendl;
+  int r = ruser->read_stats_async(dpp, std::move(handler));
+  if (r < 0) {
+    ldpp_dout(dpp, 0) << "could not get bucket info for user=" << user << dendl;
+    return r;
+  }
+
+  return 0;
+}
+
+void UserAsyncRefreshHandler::handle_response(int r, const RGWStorageStats& stats)
+{
+  if (r < 0) {
+    cache->async_refresh_fail(user, bucket);
+    return;
+  }
+
+  cache->async_refresh_response(user, bucket, stats);
+}
+
 int RGWUserStatsCache::fetch_stats_from_storage(const rgw_user& _u,
 						const rgw_bucket& _b,
 						RGWStorageStats& stats,

From 68729c951dd3acdbdf86c3ce895a55bbab1aeb89 Mon Sep 17 00:00:00 2001
From: Daniel Parkes <dparkes@redhat.com>
Date: Thu, 23 Nov 2023 22:15:52 +0100
Subject: [PATCH 0718/2492] doc/radosgw: Fix JSON typo in Principal Tag example
 code snippet

Signed-off-by: Daniel Parkes <dparkes@redhat.com>
---
 doc/radosgw/session-tags.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/radosgw/session-tags.rst b/doc/radosgw/session-tags.rst
index 46722c38251a..67a85389593b 100644
--- a/doc/radosgw/session-tags.rst
+++ b/doc/radosgw/session-tags.rst
@@ -104,7 +104,7 @@ An example of a role permission policy that uses aws:PrincipalTag is as follows:
 	    {
 	        "Effect":"Allow",
 	        "Action":["s3:*"],
-            "Resource":["arn:aws:s3::t1tenant:my-test-bucket","arn:aws:s3::t1tenant:my-test-bucket/*],"+
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket","arn:aws:s3::t1tenant:my-test-bucket/*"],
 	        "Condition":{"StringEquals":{"aws:PrincipalTag/Department":"Engineering"}}
 	    }]
 	}

From 96ca98a169d0fd6cee07355d81c11a514eb72d10 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Thu, 30 Mar 2023 01:23:55 -0400
Subject: [PATCH 0719/2492] rgw/swift: preserve dashes/underscores in swift
 user metadata names

Signed-off-by: Juan Zhu <jzhu116@bloomberg.net>
---
 qa/suites/rgw/tempest/tasks/tempest.yaml |  2 -
 src/rgw/rgw_common.cc                    | 73 +++++++++++++++++++-----
 src/rgw/rgw_common.h                     |  2 +
 src/rgw/rgw_rest_swift.cc                |  8 +--
 4 files changed, 66 insertions(+), 19 deletions(-)

diff --git a/qa/suites/rgw/tempest/tasks/tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml
index 988c7515b51f..44afe7efede3 100644
--- a/qa/suites/rgw/tempest/tasks/tempest.yaml
+++ b/qa/suites/rgw/tempest/tasks/tempest.yaml
@@ -38,8 +38,6 @@ tasks:
         - .*test_container_synchronization.*
         - .*test_object_services.PublicObjectTest.test_access_public_container_object_without_using_creds
         - .*test_object_services.ObjectTest.test_create_object_with_transfer_encoding
-        - .*test_container_services.ContainerTest.test_create_container_with_remove_metadata_key
-        - .*test_container_services.ContainerTest.test_create_container_with_remove_metadata_value
         - .*test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time
         - .*test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time
         - .*test_account_services.AccountTest.test_list_no_account_metadata
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index a8a4dea5c8a9..287322634be4 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -420,18 +420,9 @@ void req_info::init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_met
         if (found_bad_meta && strncmp(name, "META_", name_len) == 0)
           *found_bad_meta = true;
 
-        char name_low[meta_prefixes[0].len + name_len + 1];
-        snprintf(name_low, meta_prefixes[0].len - 5 + name_len + 1, "%s%s", meta_prefixes[0].str + 5 /* skip HTTP_ */, name); // normalize meta prefix
-        int j;
-        for (j = 0; name_low[j]; j++) {
-          if (name_low[j] == '_')
-            name_low[j] = '-';
-          else if (name_low[j] == '-')
-            name_low[j] = '_';
-          else
-            name_low[j] = tolower(name_low[j]);
-        }
-        name_low[j] = 0;
+        stringstream ss;
+        ss << meta_prefixes[0].str + 5 /* skip HTTP_ */ << name;
+        string name_low = lowercase_dash_underscore_http_attr(ss.str());
 
         auto it = x_meta_map.find(name_low);
         if (it != x_meta_map.end()) {
@@ -443,7 +434,7 @@ void req_info::init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_met
         } else {
           x_meta_map[name_low] = val;
         }
-        if (strncmp(name_low, "x-amz-server-side-encryption", 20) == 0) {
+        if (strncmp(name_low.c_str(), "x-amz-server-side-encryption", 20) == 0) {
           crypt_attribute_map[name_low] = val;
         }
       }
@@ -2218,6 +2209,31 @@ bool match_policy(std::string_view pattern, std::string_view input,
   }
 }
 
+/*
+ * make attrs look-like_this
+ * converts underscores to dashes, and dashes to underscores
+ */
+string lowercase_dash_underscore_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+    switch (*s) {
+      case '_':
+        buf[i] = '-';
+        break;
+      case '-':
+        buf[i] = '_';
+        break;
+      default:
+        buf[i] = tolower(*s);
+    }
+  }
+  return string(buf);
+}
+
 /*
  * make attrs look-like-this
  * converts underscores to dashes
@@ -2240,6 +2256,37 @@ string lowercase_dash_http_attr(const string& orig)
   return string(buf);
 }
 
+/*
+ * make attrs Look-Like-This or Look_Like_This
+ * converts attrs to camelcase
+ */
+string camelcase_http_attr(const string& orig)
+{
+  const char *s = orig.c_str();
+  char buf[orig.size() + 1];
+  buf[orig.size()] = '\0';
+
+  bool last_sep = true;
+
+  for (size_t i = 0; i < orig.size(); ++i, ++s) {
+    switch (*s) {
+      case '_':
+      case '-':
+        buf[i] = *s;
+        last_sep = true;
+        break;
+      default:
+        if (last_sep) {
+          buf[i] = toupper(*s);
+        } else {
+          buf[i] = tolower(*s);
+        }
+        last_sep = false;
+    }
+  }
+  return string(buf);
+}
+
 /*
  * make attrs Look-Like-This
  * converts underscores to dashes
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 5165f9008b9f..29d4e2870d59 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1771,7 +1771,9 @@ extern bool match_policy(std::string_view pattern, std::string_view input,
                          uint32_t flag);
 
 extern std::string camelcase_dash_http_attr(const std::string& orig);
+extern std::string camelcase_http_attr(const std::string& orig);
 extern std::string lowercase_dash_http_attr(const std::string& orig);
+extern std::string lowercase_dash_underscore_http_attr(const std::string& orig);
 
 void rgw_setup_saved_curl_handles();
 void rgw_release_all_curl_handles();
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 53815e11d1f2..c2c938fb4e56 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -159,7 +159,7 @@ static void dump_account_metadata(req_state * const s,
       dump_header(s, geniter->second, iter->second);
     } else if (strncmp(name, RGW_ATTR_META_PREFIX, PREFIX_LEN) == 0) {
       dump_header_prefixed(s, "X-Account-Meta-",
-                           camelcase_dash_http_attr(name + PREFIX_LEN),
+                           camelcase_http_attr(name + PREFIX_LEN),
                            iter->second);
     }
   }
@@ -499,7 +499,7 @@ static void dump_container_metadata(req_state *s,
         dump_header(s, geniter->second, iter->second);
       } else if (strncmp(name, RGW_ATTR_META_PREFIX, PREFIX_LEN) == 0) {
         dump_header_prefixed(s, "X-Container-Meta-",
-                             camelcase_dash_http_attr(name + PREFIX_LEN),
+                             camelcase_http_attr(name + PREFIX_LEN),
                              iter->second);
       }
     }
@@ -674,7 +674,7 @@ static void get_rmattrs_from_headers(const req_state * const s,
 
     if (prefix_len > 0) {
       string name(RGW_ATTR_META_PREFIX);
-      name.append(lowercase_dash_http_attr(p + prefix_len));
+      name.append(lowercase_dash_underscore_http_attr(p + prefix_len));
       rmattr_names.insert(name);
     }
   }
@@ -1339,7 +1339,7 @@ static void dump_object_metadata(const DoutPrefixProvider* dpp, req_state * cons
 		       sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
       name += sizeof(RGW_ATTR_META_PREFIX) - 1;
       dump_header_prefixed(s, "X-Object-Meta-",
-                           camelcase_dash_http_attr(name), kv.second);
+                           camelcase_http_attr(name), kv.second);
     }
   }
 

From 72065c9c7160cb0e69de751ed4d1c6e9ffd72caf Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Tue, 8 Aug 2023 04:17:47 -0400
Subject: [PATCH 0720/2492] rgw/swift: convert underscores to hyphens in swift
 user metadata names

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/rgw_common.cc     | 73 ++++++---------------------------------
 src/rgw/rgw_common.h      |  8 ++---
 src/rgw/rgw_rest.cc       |  4 +--
 src/rgw/rgw_rest_swift.cc | 13 ++++---
 4 files changed, 25 insertions(+), 73 deletions(-)

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 287322634be4..26557afd8d69 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -400,7 +400,7 @@ struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ_"),
                                    STR_LEN_ENTRY("HTTP_X_ACCOUNT_"),
                                    {NULL, 0} };
 
-void req_info::init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta)
+void req_info::init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta, const int prot_flags)
 {
   x_meta_map.clear();
   crypt_attribute_map.clear();
@@ -420,9 +420,8 @@ void req_info::init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_met
         if (found_bad_meta && strncmp(name, "META_", name_len) == 0)
           *found_bad_meta = true;
 
-        stringstream ss;
-        ss << meta_prefixes[0].str + 5 /* skip HTTP_ */ << name;
-        string name_low = lowercase_dash_underscore_http_attr(ss.str());
+        string name_low = lowercase_dash_http_attr(string(meta_prefixes[0].str + 5) + name,
+                                                   !(prot_flags & RGW_REST_SWIFT));
 
         auto it = x_meta_map.find(name_low);
         if (it != x_meta_map.end()) {
@@ -2209,36 +2208,11 @@ bool match_policy(std::string_view pattern, std::string_view input,
   }
 }
 
-/*
- * make attrs look-like_this
- * converts underscores to dashes, and dashes to underscores
- */
-string lowercase_dash_underscore_http_attr(const string& orig)
-{
-  const char *s = orig.c_str();
-  char buf[orig.size() + 1];
-  buf[orig.size()] = '\0';
-
-  for (size_t i = 0; i < orig.size(); ++i, ++s) {
-    switch (*s) {
-      case '_':
-        buf[i] = '-';
-        break;
-      case '-':
-        buf[i] = '_';
-        break;
-      default:
-        buf[i] = tolower(*s);
-    }
-  }
-  return string(buf);
-}
-
 /*
  * make attrs look-like-this
  * converts underscores to dashes
  */
-string lowercase_dash_http_attr(const string& orig)
+string lowercase_dash_http_attr(const string& orig, bool bidirection)
 {
   const char *s = orig.c_str();
   char buf[orig.size() + 1];
@@ -2249,39 +2223,14 @@ string lowercase_dash_http_attr(const string& orig)
       case '_':
         buf[i] = '-';
         break;
-      default:
-        buf[i] = tolower(*s);
-    }
-  }
-  return string(buf);
-}
-
-/*
- * make attrs Look-Like-This or Look_Like_This
- * converts attrs to camelcase
- */
-string camelcase_http_attr(const string& orig)
-{
-  const char *s = orig.c_str();
-  char buf[orig.size() + 1];
-  buf[orig.size()] = '\0';
-
-  bool last_sep = true;
-
-  for (size_t i = 0; i < orig.size(); ++i, ++s) {
-    switch (*s) {
-      case '_':
       case '-':
-        buf[i] = *s;
-        last_sep = true;
+        if (bidirection)
+          buf[i] = '_';
+        else
+          buf[i] = tolower(*s);
         break;
       default:
-        if (last_sep) {
-          buf[i] = toupper(*s);
-        } else {
-          buf[i] = tolower(*s);
-        }
-        last_sep = false;
+        buf[i] = tolower(*s);
     }
   }
   return string(buf);
@@ -2291,7 +2240,7 @@ string camelcase_http_attr(const string& orig)
  * make attrs Look-Like-This
  * converts underscores to dashes
  */
-string camelcase_dash_http_attr(const string& orig)
+string camelcase_dash_http_attr(const string& orig, bool convert2dash)
 {
   const char *s = orig.c_str();
   char buf[orig.size() + 1];
@@ -2303,7 +2252,7 @@ string camelcase_dash_http_attr(const string& orig)
     switch (*s) {
       case '_':
       case '-':
-        buf[i] = '-';
+        buf[i] = convert2dash ? '-' : *s;
         last_sep = true;
         break;
       default:
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 29d4e2870d59..ee17e2c8d43e 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1074,7 +1074,7 @@ struct req_info {
 
   req_info(CephContext *cct, const RGWEnv *env);
   void rebuild_from(const req_info& src);
-  void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta);
+  void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta, const int prot_flags);
 };
 
 struct req_init_state {
@@ -1770,10 +1770,8 @@ static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
 extern bool match_policy(std::string_view pattern, std::string_view input,
                          uint32_t flag);
 
-extern std::string camelcase_dash_http_attr(const std::string& orig);
-extern std::string camelcase_http_attr(const std::string& orig);
-extern std::string lowercase_dash_http_attr(const std::string& orig);
-extern std::string lowercase_dash_underscore_http_attr(const std::string& orig);
+extern std::string camelcase_dash_http_attr(const std::string& orig, bool convert2dash = true);
+extern std::string lowercase_dash_http_attr(const std::string& orig, bool bidirection = false);
 
 void rgw_setup_saved_curl_handles();
 void rgw_release_all_curl_handles();
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 4c2b9a55b3ee..47272515d348 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -2283,8 +2283,6 @@ int RGWREST::preprocess(req_state *s, rgw::io::BasicClient* cio)
   }
   s->op = op_from_method(info.method);
 
-  info.init_meta_info(s, &s->has_bad_meta);
-
   return 0;
 }
 
@@ -2327,5 +2325,7 @@ RGWHandler_REST* RGWREST::get_handler(
     return nullptr;
   }
 
+  s->info.init_meta_info(s, &s->has_bad_meta, s->prot_flags);
+
   return handler;
 } /* get stream handler */
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index c2c938fb4e56..5f0b1f772226 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -159,7 +159,7 @@ static void dump_account_metadata(req_state * const s,
       dump_header(s, geniter->second, iter->second);
     } else if (strncmp(name, RGW_ATTR_META_PREFIX, PREFIX_LEN) == 0) {
       dump_header_prefixed(s, "X-Account-Meta-",
-                           camelcase_http_attr(name + PREFIX_LEN),
+                           camelcase_dash_http_attr(name + PREFIX_LEN, false),
                            iter->second);
     }
   }
@@ -499,7 +499,7 @@ static void dump_container_metadata(req_state *s,
         dump_header(s, geniter->second, iter->second);
       } else if (strncmp(name, RGW_ATTR_META_PREFIX, PREFIX_LEN) == 0) {
         dump_header_prefixed(s, "X-Container-Meta-",
-                             camelcase_http_attr(name + PREFIX_LEN),
+                             camelcase_dash_http_attr(name + PREFIX_LEN, false),
                              iter->second);
       }
     }
@@ -674,7 +674,12 @@ static void get_rmattrs_from_headers(const req_state * const s,
 
     if (prefix_len > 0) {
       string name(RGW_ATTR_META_PREFIX);
-      name.append(lowercase_dash_underscore_http_attr(p + prefix_len));
+      /* For backward compatibility */
+      name.append(lowercase_dash_http_attr(p + prefix_len, true));
+      rmattr_names.insert(name);
+
+      name = RGW_ATTR_META_PREFIX;
+      name.append(lowercase_dash_http_attr(p + prefix_len, false));
       rmattr_names.insert(name);
     }
   }
@@ -1339,7 +1344,7 @@ static void dump_object_metadata(const DoutPrefixProvider* dpp, req_state * cons
 		       sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
       name += sizeof(RGW_ATTR_META_PREFIX) - 1;
       dump_header_prefixed(s, "X-Object-Meta-",
-                           camelcase_http_attr(name), kv.second);
+                           camelcase_dash_http_attr(name, false), kv.second);
     }
   }
 

From 0540cc65ef984aab63707bd7d59ecef5a9bdf17b Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Wed, 20 Sep 2023 04:12:15 +0000
Subject: [PATCH 0721/2492] rgw: modify ret cond in middle of
 CreateBucket::Execute()

Return prevents swift metadata from being removed
if bucket already exists.

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 src/rgw/rgw_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 3accf03e4968..1f017af5d80d 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3592,7 +3592,7 @@ void RGWCreateBucket::execute(optional_yield y)
    * recover from a partial create by retrying it. */
   ldpp_dout(this, 20) << "Bucket::create() returned ret=" << op_ret << " bucket=" << s->bucket << dendl;
 
-  if (op_ret)
+  if (op_ret < 0 && op_ret != EEXIST && op_ret != ERR_BUCKET_EXISTS)
     return;
 
   const bool existed = s->bucket_exists;

From a4553f035124b53ebd77d3d01911fe523d7df881 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Wed, 1 Nov 2023 12:32:52 -0400
Subject: [PATCH 0722/2492] rgw: fix the ret cond check CreateBucket::Execute()

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/rgw_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 1f017af5d80d..5d56deaf220b 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3592,7 +3592,7 @@ void RGWCreateBucket::execute(optional_yield y)
    * recover from a partial create by retrying it. */
   ldpp_dout(this, 20) << "Bucket::create() returned ret=" << op_ret << " bucket=" << s->bucket << dendl;
 
-  if (op_ret < 0 && op_ret != EEXIST && op_ret != ERR_BUCKET_EXISTS)
+  if (op_ret < 0 && op_ret != -EEXIST && op_ret != -ERR_BUCKET_EXISTS)
     return;
 
   const bool existed = s->bucket_exists;

From b7aa16da4bbb444f6361e4f14be7ef64b0ea3bb1 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 15:29:35 -0500
Subject: [PATCH 0723/2492] encoding: add round_trip_encode()/decode() for
 chrono types

the default encodings for chrono types were made to be
backward-compatible with utime_t, so truncated seconds to 32 bits

adds new functions that encode these chrono types using their underlying
representation, which for ceph::real_time and ceph::timespan is
'nanoseconds as uint64_t'

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/include/encoding.h | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/include/encoding.h b/src/include/encoding.h
index 40ba9d39c76f..aac63a7e6cc9 100644
--- a/src/include/encoding.h
+++ b/src/include/encoding.h
@@ -322,7 +322,9 @@ inline void decode_nohead(int len, bufferlist& s, bufferlist::const_iterator& p)
   p.copy(len, s);
 }
 
-// Time, since the templates are defined in std::chrono
+// Time, since the templates are defined in std::chrono. The default encodings
+// for time_point and duration are backward-compatible with utime_t, but
+// truncate seconds to 32 bits so are not guaranteed to round-trip.
 
 template<typename Clock, typename Duration,
          typename std::enable_if_t<converts_to_timespec_v<Clock>>* = nullptr>
@@ -373,6 +375,40 @@ void decode(std::chrono::duration<Rep, Period>& d,
   d = std::chrono::seconds(s) + std::chrono::nanoseconds(ns);
 }
 
+// Provide encodings for chrono::time_point and duration that use
+// the underlying representation so are guaranteed to round-trip.
+
+template <typename Rep, typename Period,
+          typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+void round_trip_encode(const std::chrono::duration<Rep, Period>& d,
+                       ceph::bufferlist &bl) {
+  const Rep r = d.count();
+  encode(r, bl);
+}
+
+template <typename Rep, typename Period,
+          typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+void round_trip_decode(std::chrono::duration<Rep, Period>& d,
+                       bufferlist::const_iterator& p) {
+  Rep r;
+  decode(r, p);
+  d = std::chrono::duration<Rep, Period>(r);
+}
+
+template <typename Clock, typename Duration>
+void round_trip_encode(const std::chrono::time_point<Clock, Duration>& t,
+                       ceph::bufferlist &bl) {
+  round_trip_encode(t.time_since_epoch(), bl);
+}
+
+template <typename Clock, typename Duration>
+void round_trip_decode(std::chrono::time_point<Clock, Duration>& t,
+                       bufferlist::const_iterator& p) {
+  Duration dur;
+  round_trip_decode(dur, p);
+  t = std::chrono::time_point<Clock, Duration>(dur);
+}
+
 // -----------------------------
 // STL container types
 

From 1fd4309fbbebf0d3e67aa2800d5fb3c7de19dcc7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 16:24:47 -0500
Subject: [PATCH 0724/2492] rgw: object lock uses 64-bit encoding for
 RetainUntilDate

the default encoding of ceph::real_time truncates seconds to uint32_t,
so stores the wrong timestamp for object lock enforcement

Fixes: https://tracker.ceph.com/issues/63537

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 PendingReleaseNotes       | 8 ++++++++
 src/rgw/rgw_object_lock.h | 8 ++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 9acda3d64e14..9b3dfede8bbd 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -69,6 +69,14 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   in the kclient.
 * S3 Get/HeadObject now support query parameter `partNumber` to read a specific
   part of a completed multipart upload.
+* RGW: Fixed a S3 Object Lock bug with PutObjectRetention requests that specify
+  a RetainUntilDate after the year 2106. This date was truncated to 32 bits when
+  stored, so a much earlier date was used for object lock enforcement. This does
+  not effect PutBucketObjectLockConfiguration where a duration is given in Days.
+  The RetainUntilDate encoding is fixed for new PutObjectRetention requests, but
+  cannot repair the dates of existing object locks. Such objects can be identified
+  with a HeadObject request based on the x-amz-object-lock-retain-until-date
+  response header.
 
 >=18.0.0
 
diff --git a/src/rgw/rgw_object_lock.h b/src/rgw/rgw_object_lock.h
index 022aef8d45ab..7c6b9cf612bc 100644
--- a/src/rgw/rgw_object_lock.h
+++ b/src/rgw/rgw_object_lock.h
@@ -174,16 +174,20 @@ class RGWObjectRetention
   }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     encode(mode, bl);
     encode(retain_until_date, bl);
+    ceph::round_trip_encode(retain_until_date, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     decode(mode, bl);
     decode(retain_until_date, bl);
+    if (struct_v >= 2) {
+      ceph::round_trip_decode(retain_until_date, bl);
+    }
     DECODE_FINISH(bl);
   }
 

From b33e08657ecb7949505c6eae920c83e1e84096bb Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 15:28:46 -0500
Subject: [PATCH 0725/2492] encoding: use concepts for chrono encoding

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/ceph_time.h |  3 +++
 src/include/encoding.h | 19 +++++++------------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/common/ceph_time.h b/src/common/ceph_time.h
index 6ada4d8944cd..bae038862cf2 100644
--- a/src/common/ceph_time.h
+++ b/src/common/ceph_time.h
@@ -529,6 +529,9 @@ struct converts_to_timespec<Clock, std::void_t<decltype(
 template <typename Clock>
 constexpr bool converts_to_timespec_v = converts_to_timespec<Clock>::value;
 
+template <typename Clock>
+concept clock_with_timespec = converts_to_timespec_v<Clock>;
+
 template<typename Rep, typename T>
 static Rep to_seconds(T t) {
   return std::chrono::duration_cast<
diff --git a/src/include/encoding.h b/src/include/encoding.h
index aac63a7e6cc9..08c67c33eecf 100644
--- a/src/include/encoding.h
+++ b/src/include/encoding.h
@@ -14,6 +14,7 @@
 #ifndef CEPH_ENCODING_H
 #define CEPH_ENCODING_H
 
+#include <concepts>
 #include <set>
 #include <map>
 #include <deque>
@@ -326,8 +327,7 @@ inline void decode_nohead(int len, bufferlist& s, bufferlist::const_iterator& p)
 // for time_point and duration are backward-compatible with utime_t, but
 // truncate seconds to 32 bits so are not guaranteed to round-trip.
 
-template<typename Clock, typename Duration,
-         typename std::enable_if_t<converts_to_timespec_v<Clock>>* = nullptr>
+template<clock_with_timespec Clock, typename Duration>
 void encode(const std::chrono::time_point<Clock, Duration>& t,
 	    ceph::bufferlist &bl) {
   auto ts = Clock::to_timespec(t);
@@ -338,8 +338,7 @@ void encode(const std::chrono::time_point<Clock, Duration>& t,
   encode(ns, bl);
 }
 
-template<typename Clock, typename Duration,
-         typename std::enable_if_t<converts_to_timespec_v<Clock>>* = nullptr>
+template<clock_with_timespec Clock, typename Duration>
 void decode(std::chrono::time_point<Clock, Duration>& t,
 	    bufferlist::const_iterator& p) {
   uint32_t s;
@@ -353,8 +352,7 @@ void decode(std::chrono::time_point<Clock, Duration>& t,
   t = Clock::from_timespec(ts);
 }
 
-template<typename Rep, typename Period,
-         typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+template<std::integral Rep, typename Period>
 void encode(const std::chrono::duration<Rep, Period>& d,
 	    ceph::bufferlist &bl) {
   using namespace std::chrono;
@@ -364,8 +362,7 @@ void encode(const std::chrono::duration<Rep, Period>& d,
   encode(ns, bl);
 }
 
-template<typename Rep, typename Period,
-         typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+template<std::integral Rep, typename Period>
 void decode(std::chrono::duration<Rep, Period>& d,
 	    bufferlist::const_iterator& p) {
   int32_t s;
@@ -378,16 +375,14 @@ void decode(std::chrono::duration<Rep, Period>& d,
 // Provide encodings for chrono::time_point and duration that use
 // the underlying representation so are guaranteed to round-trip.
 
-template <typename Rep, typename Period,
-          typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+template <std::integral Rep, typename Period>
 void round_trip_encode(const std::chrono::duration<Rep, Period>& d,
                        ceph::bufferlist &bl) {
   const Rep r = d.count();
   encode(r, bl);
 }
 
-template <typename Rep, typename Period,
-          typename std::enable_if_t<std::is_integral_v<Rep>>* = nullptr>
+template <std::integral Rep, typename Period>
 void round_trip_decode(std::chrono::duration<Rep, Period>& d,
                        bufferlist::const_iterator& p) {
   Rep r;

From 54edc7955b0593423d2c03ca308b5e5fcdc528a3 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 23 Nov 2023 20:35:41 +0100
Subject: [PATCH 0726/2492] librados: clarify
 get_pool_is_selfmanaged_snaps_mode() semantics

Refer to the commit message of 8a9769a1d5fd ("librados: add
get_pool_is_selfmanaged_snaps_mode() function").

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/include/rados/librados.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index cb8261af12d2..f2f9e47a2ad7 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -1477,7 +1477,7 @@ inline namespace v14_2_0 {
     int get_pool_stats(std::list<std::string>& v,
                        std::string& category,
 		       std::map<std::string, stats_map>& stats);
-    /// check if pool has selfmanaged snaps
+    /// check if pool has or had selfmanaged snaps
     bool get_pool_is_selfmanaged_snaps_mode(const std::string& poolname);
 
     int cluster_stat(cluster_stat_t& result);

From 08b2255c345b0614cb1e1e4e3435910235c677cd Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Mon, 20 Nov 2023 18:16:31 +0530
Subject: [PATCH 0727/2492] osd/PG: Add helper method to get the average size
 of objects in a PG.

Factor out the code in PG::queue_recovery() that determines the average
object size in a PG to PG::get_average_object_size(). This is used to
determine the cost of a background operation for e.g., recovery,
snaptrim in case mClock scheduler is employed.

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/osd/PG.cc | 10 +---------
 src/osd/PG.h  | 13 +++++++++++++
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index d2f97a129a24..11af6c2b3ca8 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -420,15 +420,7 @@ void PG::queue_recovery()
     dout(10) << "queue_recovery -- queuing" << dendl;
     recovery_queued = true;
     // Let cost per object be the average object size
-    auto num_bytes = static_cast<uint64_t>(
-      std::max<int64_t>(
-	0, // ensure bytes is non-negative
-	info.stats.stats.sum.num_bytes));
-    auto num_objects = static_cast<uint64_t>(
-      std::max<int64_t>(
-	1, // ensure objects is non-negative and non-zero
-	info.stats.stats.sum.num_objects));
-    uint64_t cost_per_object = std::max<uint64_t>(num_bytes / num_objects, 1);
+    uint64_t cost_per_object = get_average_object_size();
     osd->queue_for_recovery(
       this, cost_per_object, recovery_state.get_recovery_op_priority()
     );
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2e82e74ab012..0cfd33d484ee 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1024,6 +1024,19 @@ class PG : public DoutPrefixProvider,
     return num_bytes;
   }
 
+  uint64_t get_average_object_size() {
+    ceph_assert(ceph_mutex_is_locked_by_me(_lock));
+    auto num_bytes = static_cast<uint64_t>(
+      std::max<int64_t>(
+        0, // ensure bytes is non-negative
+        info.stats.stats.sum.num_bytes));
+    auto num_objects = static_cast<uint64_t>(
+      std::max<int64_t>(
+        1, // ensure objects is non-negative and non-zero
+        info.stats.stats.sum.num_objects));
+    return std::max<uint64_t>(num_bytes / num_objects, 1);
+  }
+
 protected:
 
   /*

From ca222e47defb06352041da7980fdef508729c4b2 Mon Sep 17 00:00:00 2001
From: Ilsoo Byun <ilsoobyun@linecorp.com>
Date: Thu, 28 May 2020 17:01:22 +0900
Subject: [PATCH 0728/2492] rgw: add headers to guide cache update in 304
 response to conditional get

Fixes: https://tracker.ceph.com/issues/45736
Signed-off-by: Ilsoo Byun <ilsoobyun@linecorp.com>
---
 src/rgw/driver/rados/rgw_rados.cc |  7 ++++---
 src/rgw/rgw_rest_s3.cc            | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 3ee43dd91a89..f7ce93f773b8 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -6688,6 +6688,10 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *
     }
   }
 
+  if (params.lastmod) {
+    *params.lastmod = astate->mtime;
+  }
+
   /* Convert all times go GMT to make them compatible */
   if (conds.mod_ptr || conds.unmod_ptr) {
     obj_time_weight src_weight;
@@ -6738,9 +6742,6 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider *
   if (params.obj_size) {
     *params.obj_size = astate->size;
   }
-  if (params.lastmod) {
-    *params.lastmod = astate->mtime;
-  }
   if (params.epoch) {
     *params.epoch = astate->epoch;
   }
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 5425406b1eb9..1ebc0e72127e 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -579,6 +579,20 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
   }
 
   if (op_ret == -ERR_NOT_MODIFIED) {
+      dump_last_modified(s, lastmod);
+
+      auto iter = attrs.find(RGW_ATTR_ETAG);
+      if (iter != attrs.end())
+        dump_etag(s, iter->second.to_str());
+
+      iter = attrs.find(RGW_ATTR_CACHE_CONTROL);
+      if (iter != attrs.end())
+        dump_header(s, rgw_to_http_attrs[RGW_ATTR_CACHE_CONTROL], iter->second);
+
+      iter = attrs.find(RGW_ATTR_EXPIRES);
+      if (iter != attrs.end())
+        dump_header(s, rgw_to_http_attrs[RGW_ATTR_EXPIRES], iter->second);
+
       end_header(s, this);
   } else {
       if (!content_type)

From ccd241c565aa898231e4bfa4ea313c3406b55391 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 20 Nov 2023 22:48:55 -0500
Subject: [PATCH 0729/2492] rgw/s3: RGWGetObj still sets attrs on ReadOp error

when `ReadOp::prepare()` fails with `-ERR_NOT_MODIFIED`, it succeeded in
reading the head object for mtime and attrs like etag. make those attrs
available to RGWGetObj so we can still send ETag and other cache-related
response headers

Fixes: https://tracker.ceph.com/issues/45736

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6a7a7313e117..efc0053c8a27 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2286,12 +2286,12 @@ void RGWGetObj::execute(optional_yield y)
   }
 
   op_ret = read_op->prepare(s->yield, this);
-  if (op_ret < 0)
-    goto done_err;
   version_id = s->object->get_instance();
   s->obj_size = s->object->get_obj_size();
   attrs = s->object->get_attrs();
   multipart_parts_count = read_op->params.parts_count;
+  if (op_ret < 0)
+    goto done_err;
 
   /* STAT ops don't need data, and do no i/o */
   if (get_type() == RGW_OP_STAT_OBJ) {

From e0034e5ed5f41d89ce88e8956adc88c48d3ccf8a Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Sun, 19 Nov 2023 16:26:16 +0530
Subject: [PATCH 0730/2492] mgr/vol: re-write for better readability

Instead of writing like this  -

  if abcd1.abcd2(abcd3) and abcd4 == abcd5
	print('efgh6')
  if abcd7.abcd8(abcd9) and abcd4 == abcd5
	print('efgh10')

Write like this because it easier to read, especially in case of the
patch where condition under is really long -

  if abcd4 == abcd5:
    if abcd1.abcd2(abcd3):
	print('abcd4')
    if abcd5.abcd8(abcd9):
	print('abcd5')

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 .../fs/operations/versions/subvolume_v1.py    | 20 +++++++++----------
 .../fs/operations/versions/subvolume_v2.py    | 18 ++++++++++-------
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
index b5a10dd6c7f6..3cd235d39b30 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
@@ -684,16 +684,16 @@ def _get_clone_failure(self):
     def status(self):
         state = SubvolumeStates.from_value(self.metadata_mgr.get_global_option(MetadataManager.GLOBAL_META_KEY_STATE))
         subvolume_type = self.subvol_type
-        subvolume_status = {
-            'state' : state.value
-        }
-        if not SubvolumeOpSm.is_complete_state(state) and subvolume_type == SubvolumeTypes.TYPE_CLONE:
-            subvolume_status["source"] = self._get_clone_source()
-        if SubvolumeOpSm.is_failed_state(state) and subvolume_type == SubvolumeTypes.TYPE_CLONE:
-            try:
-                subvolume_status["failure"] = self._get_clone_failure()
-            except MetadataMgrException:
-                pass
+        subvolume_status = {'state' : state.value}
+
+        if subvolume_type == SubvolumeTypes.TYPE_CLONE:
+            if not SubvolumeOpSm.is_complete_state(state):
+                subvolume_status["source"] = self._get_clone_source()
+            if SubvolumeOpSm.is_failed_state(state):
+                try:
+                    subvolume_status["failure"] = self._get_clone_failure()
+                except MetadataMgrException:
+                    pass
 
         return subvolume_status
 
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
index 03085d049713..3a2f7bf8d43e 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
@@ -308,13 +308,17 @@ def open(self, op_type):
                                       op_type.value, self.subvolname, etype.value))
 
             estate = self.state
-            if op_type not in self.allowed_ops_by_state(estate) and estate == SubvolumeStates.STATE_RETAINED:
-                raise VolumeException(-errno.ENOENT, "subvolume '{0}' is removed and has only snapshots retained".format(
-                                      self.subvolname))
-
-            if op_type not in self.allowed_ops_by_state(estate) and estate != SubvolumeStates.STATE_RETAINED:
-                raise VolumeException(-errno.EAGAIN, "subvolume '{0}' is not ready for operation {1}".format(
-                                      self.subvolname, op_type.value))
+            if op_type not in self.allowed_ops_by_state(estate):
+                if estate == SubvolumeStates.STATE_RETAINED:
+                    raise VolumeException(
+                        -errno.ENOENT,
+                        f'subvolume "{self.subvolname}" is removed and has '
+                        'only snapshots retained')
+                else:
+                    raise VolumeException(
+                        -errno.EAGAIN,
+                        f'subvolume "{self.subvolname}" is not ready for '
+                        f'operation "{op_type.value}"')
 
             if estate != SubvolumeStates.STATE_RETAINED:
                 subvol_path = self.path

From 0a1ce0016871dc2ce29d4e7eff564a199d54c6e3 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 25 Nov 2023 07:33:24 +1000
Subject: [PATCH 0731/2492] doc/rados: edit "monitor store failures"

Edit the section "Monitor Store Failures" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 47 +++++++++----------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index c28a38da5684..4bb9d7b5b860 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -505,9 +505,9 @@ Monitor Store Failures
 Symptoms of store corruption
 ----------------------------
 
-Ceph monitors store the :term:`Cluster Map` in a key-value store.  If key-value
-store corruption causes a monitor to fail, then the monitor log might contain
-one of the following error messages::
+Ceph Monitors maintain the :term:`Cluster Map` in a key-value store. If
+key-value store corruption causes a Monitor to fail, then the Monitor log might
+contain one of the following error messages::
 
   Corruption: error in middle of record
 
@@ -518,10 +518,10 @@ or::
 Recovery using healthy monitor(s)
 ---------------------------------
 
-If there are surviving monitors, we can always :ref:`replace
-<adding-and-removing-monitors>` the corrupted monitor with a new one. After the
-new monitor boots, it will synchronize with a healthy peer. After the new
-monitor is fully synchronized, it will be able to serve clients.
+If the cluster contains surviving Monitors, the corrupted Monitor can be
+:ref:`replaced <adding-and-removing-monitors>` with a new Monitor. After the
+new Monitor boots, it will synchronize with a healthy peer. After the new
+Monitor is fully synchronized, it will be able to serve clients.
 
 .. _mon-store-recovery-using-osds:
 
@@ -529,15 +529,14 @@ Recovery using OSDs
 -------------------
 
 Even if all monitors fail at the same time, it is possible to recover the
-monitor store by using information stored in OSDs. You are encouraged to deploy
-at least three (and preferably five) monitors in a Ceph cluster. In such a
-deployment, complete monitor failure is unlikely. However, unplanned power loss
-in a data center whose disk settings or filesystem settings are improperly
-configured could cause the underlying filesystem to fail and this could kill
-all of the monitors. In such a case, data in the OSDs can be used to recover
-the monitors.  The following is such a script and can be used to recover the
-monitors:
-
+Monitor store by using information that is stored in OSDs. You are encouraged
+to deploy at least three (and preferably five) Monitors in a Ceph cluster. In
+such a deployment, complete Monitor failure is unlikely. However, unplanned
+power loss in a data center whose disk settings or filesystem settings are
+improperly configured could cause the underlying filesystem to fail and this
+could kill all of the monitors. In such a case, data in the OSDs can be used to
+recover the Monitors. The following is a script that can be used in such a case
+to recover the Monitors:
 
 .. code-block:: bash
 
@@ -590,10 +589,10 @@ monitors:
 
 This script performs the following steps:
 
-#. Collects the map from each OSD host.
-#. Rebuilds the store.
-#. Fills the entities in the keyring file with appropriate capabilities.
-#. Replaces the corrupted store on ``mon.foo`` with the recovered copy.
+#. Collect the map from each OSD host.
+#. Rebuild the store.
+#. Fill the entities in the keyring file with appropriate capabilities.
+#. Replace the corrupted store on ``mon.foo`` with the recovered copy.
 
 
 Known limitations
@@ -605,15 +604,15 @@ The above recovery tool is unable to recover the following information:
   auth add`` command are recovered from the OSD's copy, and the
   ``client.admin`` keyring is imported using ``ceph-monstore-tool``. However,
   the MDS keyrings and all other keyrings will be missing in the recovered
-  monitor store. You might need to manually re-add them.
+  Monitor store. It might be necessary to manually re-add them.
 
 - **Creating pools**: If any RADOS pools were in the process of being created,
   that state is lost. The recovery tool operates on the assumption that all
   pools have already been created. If there are PGs that are stuck in the
-  'unknown' state after the recovery for a partially created pool, you can
+  ``unknown`` state after the recovery for a partially created pool, you can
   force creation of the *empty* PG by running the ``ceph osd force-create-pg``
-  command. Note that this will create an *empty* PG, so take this action only
-  if you know the pool is empty.
+  command. This creates an *empty* PG, so take this action only if you are
+  certain that the pool is empty.
 
 - **MDS Maps**: The MDS maps are lost.
 

From adc501c2ceed138dd0b94be5a17ea7489377da57 Mon Sep 17 00:00:00 2001
From: Thomas Lamprecht <t.lamprecht@proxmox.com>
Date: Tue, 5 Sep 2023 13:28:07 +0200
Subject: [PATCH 0732/2492] debian: add missing bcrypt to manager .requires

The `ceph-mgr` package lost its dependency on `python3-bcrypt` when
the dependencies got moved from d/control to a dh_python3 compatible
requires file. Add it again as the bcrypt module is still used there.

Otherwise one gets errors when, e.g., calling `ceph -s` after a fresh
installation:

> 13 mgr modules have failed dependencies
> Module 'balancer' has failed dependency: No module named 'bcrypt'
> Module 'crash' has failed dependency: No module named 'bcrypt'
> Module 'devicehealth' has failed dependency: No module named 'bcrypt'
> Module 'iostat' has failed dependency: No module named 'bcrypt'
> Module 'nfs' has failed dependency: No module named 'bcrypt'
> Module 'orchestrator' has failed dependency: No module named 'bcrypt'
> Module 'pg_autoscaler' has failed dependency: No module named 'bcrypt'
> Module 'progress' has failed dependency: No module named 'bcrypt'
> Module 'rbd_support' has failed dependency: No module named 'bcrypt'
> Module 'restful' has failed dependency: No module named 'bcrypt'
> Module 'status' has failed dependency: No module named 'bcrypt'
> Module 'telemetry' has failed dependency: No module named 'bcrypt'
> Module 'volumes' has failed dependency: No module named 'bcrypt'

Fixes: https://tracker.ceph.com/issues/63637
Fixes: ef19547e83e ("debian: add .requires for specifying python3 deps")
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 debian/ceph-mgr.requires | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/ceph-mgr.requires b/debian/ceph-mgr.requires
index bf334fb9bd29..39336330c939 100644
--- a/debian/ceph-mgr.requires
+++ b/debian/ceph-mgr.requires
@@ -1,3 +1,4 @@
+bcrypt
 pyOpenSSL
 cephfs
 ceph-argparse

From fcff420a580d7f24cdd81aebf6c3b01c5f8785f7 Mon Sep 17 00:00:00 2001
From: lightmelodies <lightmelodies@outlook.com>
Date: Sun, 26 Nov 2023 06:05:54 +0000
Subject: [PATCH 0733/2492] rgw: optimize gc chain size calculation

Reduce time complexity of gc chain size calculation from O(n^2) to O(n).

Signed-off-by: Wei Wang <lightmelodies@outlook.com>
---
 src/rgw/driver/rados/rgw_gc.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_gc.cc b/src/rgw/driver/rados/rgw_gc.cc
index 4705c46fff4b..eac4a1f850b5 100644
--- a/src/rgw/driver/rados/rgw_gc.cc
+++ b/src/rgw/driver/rados/rgw_gc.cc
@@ -71,17 +71,18 @@ std::tuple<int, std::optional<cls_rgw_obj_chain>> RGWGC::send_split_chain(const
 
   if (cct->_conf->rgw_max_chunk_size) {
     cls_rgw_obj_chain broken_chain;
+    cls_rgw_gc_set_entry_op op;
+    op.info.tag = tag;
+    size_t base_encoded_size = op.estimate_encoded_size();
+    size_t total_encoded_size = base_encoded_size;
+
     ldpp_dout(this, 20) << "RGWGC::send_split_chain - rgw_max_chunk_size is: " << cct->_conf->rgw_max_chunk_size << dendl;
 
     for (auto it = chain.objs.begin(); it != chain.objs.end(); it++) {
       ldpp_dout(this, 20) << "RGWGC::send_split_chain - adding obj with name: " << it->key << dendl;
       broken_chain.objs.emplace_back(*it);
-      cls_rgw_gc_obj_info info;
-      info.tag = tag;
-      info.chain = broken_chain;
-      cls_rgw_gc_set_entry_op op;
-      op.info = info;
-      size_t total_encoded_size = op.estimate_encoded_size();
+      total_encoded_size += it->estimate_encoded_size();
+
       ldpp_dout(this, 20) << "RGWGC::send_split_chain - total_encoded_size is: " << total_encoded_size << dendl;
 
       if (total_encoded_size > cct->_conf->rgw_max_chunk_size) { //dont add to chain, and send to gc
@@ -95,6 +96,7 @@ std::tuple<int, std::optional<cls_rgw_obj_chain>> RGWGC::send_split_chain(const
           return {ret, {broken_chain}};
         }
         broken_chain.objs.clear();
+        total_encoded_size = base_encoded_size;
       }
     }
     if (!broken_chain.objs.empty()) { //when the chain is smaller than or equal to rgw_max_chunk_size

From 6b0a9a580f3376a5db208674a24e4599500b712f Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 26 Nov 2023 14:02:41 +0000
Subject: [PATCH 0734/2492] tools/ceph_objectstore_tool:
 action_on_all_objects_in_pg to skip pgmeta

See "Error getting attr on" after creating an empty pool:

```
$ ceph-objectstore-tool --data-path ./dev/osd0 --op list
Error getting attr on : 2.0_head,#2:00000000::::head#, (61) No data available
["2.0",{"oid":"","key":"","snapid":-2,"hash":0,"max":0,"pool":2,"namespace":"","max":0}]
```

_action_on_all_objects_in_pg() should skip getting the (non-existing) attributes of pgmeta hobjects.

Note: pgmeta_oid is created at the PG's constructor and is written to disk in init_pg_ondisk()
      without any OI_ATTR.

Fixes: https://tracker.ceph.com/issues/63640

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/tools/ceph_objectstore_tool.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index 19a445824834..41602ef49462 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -105,7 +105,7 @@ int _action_on_all_objects_in_pg(ObjectStore *store, coll_t coll, action_on_obje
 	 obj != list.end();
 	 ++obj) {
       object_info_t oi;
-      if (coll != coll_t::meta()) {
+      if (coll != coll_t::meta() && !obj->is_pgmeta()) {
         bufferlist attr;
         r = store->getattr(ch, *obj, OI_ATTR, attr);
         if (r < 0) {

From 855098f191ea68c87fb09e6a44e34e7336534895 Mon Sep 17 00:00:00 2001
From: Ali Masarwa <ali.saed.masarwa@gmail.com>
Date: Tue, 31 Oct 2023 15:55:41 +0200
Subject: [PATCH 0735/2492] RGW: make new rados support old RGW 2pc remove

Signed-off-by: Ali Masarwa <ali.saed.masarwa@gmail.com>
---
 src/cls/2pc_queue/cls_2pc_queue.cc           |  28 +++++
 src/cls/2pc_queue/cls_2pc_queue_client.h     |   5 +-
 src/cls/2pc_queue/cls_2pc_queue_ops.h        |  10 +-
 src/cls/queue/cls_queue_client.cc            |  38 ++++--
 src/cls/queue/cls_queue_client.h             |   2 +
 src/cls/queue/cls_queue_ops.h                |   9 +-
 src/cls/queue/cls_queue_src.cc               |  18 ++-
 src/test/cls_2pc_queue/test_cls_2pc_queue.cc | 126 +++++++++++++++++++
 src/test/cls_queue/test_cls_queue.cc         |  45 +++++++
 9 files changed, 261 insertions(+), 20 deletions(-)

diff --git a/src/cls/2pc_queue/cls_2pc_queue.cc b/src/cls/2pc_queue/cls_2pc_queue.cc
index 019f2c96deaf..1e99503943a3 100644
--- a/src/cls/2pc_queue/cls_2pc_queue.cc
+++ b/src/cls/2pc_queue/cls_2pc_queue.cc
@@ -578,6 +578,19 @@ static int cls_2pc_queue_list_entries(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+static int cls_2pc_queue_count_entries(cls_method_context_t hctx, cls_queue_list_op& op, cls_queue_head& head,
+                                       uint32_t& entries_to_remove)
+{
+  cls_queue_list_ret op_ret;
+  auto ret = queue_list_entries(hctx, op, op_ret, head);
+  if (ret < 0) {
+    return ret;
+  }
+
+  entries_to_remove = op_ret.entries.size();
+  return 0;
+}
+
 static int cls_2pc_queue_remove_entries(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
 {
   auto in_iter = in->cbegin();
@@ -594,6 +607,21 @@ static int cls_2pc_queue_remove_entries(cls_method_context_t hctx, bufferlist *i
   if (ret < 0) {
     return ret;
   }
+
+  // Old RGW is running, and it sent cls_queue_remove_op instead of cls_2pc_queue_remove_op
+  if (rem_2pc_op.entries_to_remove == 0) {
+    CLS_LOG(10, "INFO: cls_2pc_queue_remove_entries: incompatible RGW with rados, counting entries to remove...");
+    cls_queue_list_op list_op;
+    list_op.max = std::numeric_limits<uint64_t>::max(); // max length because endmarker is the stopping condition.
+    list_op.end_marker = rem_2pc_op.end_marker;
+    ret = cls_2pc_queue_count_entries(hctx, list_op, head, rem_2pc_op.entries_to_remove);
+    if (ret < 0) {
+      CLS_LOG(1, "ERROR: cls_2pc_queue_count_entries: returned: %d", ret);
+      return ret;
+    }
+    CLS_LOG(10, "INFO: cls_2pc_queue_count_entries: counted: %u", rem_2pc_op.entries_to_remove);
+  }
+
   cls_queue_remove_op rem_op;
   rem_op.end_marker = std::move(rem_2pc_op.end_marker);
   ret = queue_remove_entries(hctx, rem_op, head);
diff --git a/src/cls/2pc_queue/cls_2pc_queue_client.h b/src/cls/2pc_queue/cls_2pc_queue_client.h
index c806d30f59e0..0d55d68e7a05 100644
--- a/src/cls/2pc_queue/cls_2pc_queue_client.h
+++ b/src/cls/2pc_queue/cls_2pc_queue_client.h
@@ -87,5 +87,8 @@ void cls_2pc_queue_expire_reservations(librados::ObjectWriteOperation& op,
         ceph::coarse_real_time stale_time);
 
 // remove all entries up to the given marker
-void cls_2pc_queue_remove_entries(librados::ObjectWriteOperation& op, const std::string& end_marker, uint64_t entries_to_remove);
+// if there is no race condition, providing the number of entries_to_remove is recommended, as it is more efficient.
+// if there is no guarantee against two clienst deleting entries at the same time, you can leave the entries_to_remove unprovided or input zero entries_to_remove
+// the function will count how many entries it needs to removed
+void cls_2pc_queue_remove_entries(librados::ObjectWriteOperation& op, const std::string& end_marker, uint64_t entries_to_remove=0);
 
diff --git a/src/cls/2pc_queue/cls_2pc_queue_ops.h b/src/cls/2pc_queue/cls_2pc_queue_ops.h
index bb61ef341ac1..194fdf6da9d5 100644
--- a/src/cls/2pc_queue/cls_2pc_queue_ops.h
+++ b/src/cls/2pc_queue/cls_2pc_queue_ops.h
@@ -118,21 +118,23 @@ WRITE_CLASS_ENCODER(cls_2pc_queue_reservations_ret)
 
 struct cls_2pc_queue_remove_op {
   std::string end_marker;
-  uint32_t entries_to_remove;
+  uint32_t entries_to_remove = 0;
 
   cls_2pc_queue_remove_op() {}
 
   void encode(ceph::buffer::list& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     encode(end_marker, bl);
     encode(entries_to_remove, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(ceph::buffer::list::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     decode(end_marker, bl);
-    decode(entries_to_remove, bl);
+    if (struct_v > 1) {
+      decode(entries_to_remove, bl);
+    }
     DECODE_FINISH(bl);
   }
 };
diff --git a/src/cls/queue/cls_queue_client.cc b/src/cls/queue/cls_queue_client.cc
index 87d17bb9e315..d3d38a9214e5 100644
--- a/src/cls/queue/cls_queue_client.cc
+++ b/src/cls/queue/cls_queue_client.cc
@@ -48,16 +48,9 @@ void cls_queue_enqueue(ObjectWriteOperation& op, uint32_t expiration_secs, vecto
   op.exec(QUEUE_CLASS, QUEUE_ENQUEUE, in);
 }
 
-int cls_queue_list_entries(IoCtx& io_ctx, const string& oid, const string& marker, uint32_t max,
-                            vector<cls_queue_entry>& entries,
-                            bool *truncated, string& next_marker)
+int cls_queue_list_entries_inner(IoCtx& io_ctx, const string& oid, vector<cls_queue_entry>& entries,
+                                 bool *truncated, string& next_marker, bufferlist& in, bufferlist& out)
 {
-  bufferlist in, out;
-  cls_queue_list_op op;
-  op.start_marker = marker;
-  op.max = max;
-  encode(op, in);
-
   int r = io_ctx.exec(oid, QUEUE_CLASS, QUEUE_LIST_ENTRIES, in, out);
   if (r < 0)
     return r;
@@ -78,6 +71,33 @@ int cls_queue_list_entries(IoCtx& io_ctx, const string& oid, const string& marke
   return 0;
 }
 
+int cls_queue_list_entries(IoCtx& io_ctx, const string& oid, const string& marker, uint32_t max,
+                            vector<cls_queue_entry>& entries,
+                            bool *truncated, string& next_marker)
+{
+  bufferlist in, out;
+  cls_queue_list_op op;
+  op.start_marker = marker;
+  op.max = max;
+  encode(op, in);
+
+  return cls_queue_list_entries_inner(io_ctx, oid, entries, truncated, next_marker, in, out);
+}
+
+int cls_queue_list_entries(IoCtx& io_ctx, const string& oid, const string& marker, const string& end_marker,
+                           vector<cls_queue_entry>& entries,
+                           bool *truncated, string& next_marker)
+{
+  bufferlist in, out;
+  cls_queue_list_op op;
+  op.start_marker = marker;
+  op.max = std::numeric_limits<uint64_t>::max();
+  op.end_marker = end_marker;
+  encode(op, in);
+
+  return cls_queue_list_entries_inner(io_ctx, oid, entries, truncated, next_marker, in, out);
+}
+
 void cls_queue_remove_entries(ObjectWriteOperation& op, const string& end_marker)
 {
   bufferlist in, out;
diff --git a/src/cls/queue/cls_queue_client.h b/src/cls/queue/cls_queue_client.h
index 895a51c11737..903448fd480b 100644
--- a/src/cls/queue/cls_queue_client.h
+++ b/src/cls/queue/cls_queue_client.h
@@ -11,6 +11,8 @@ int cls_queue_get_capacity(librados::IoCtx& io_ctx, const std::string& oid, uint
 void cls_queue_enqueue(librados::ObjectWriteOperation& op, uint32_t expiration_secs, std::vector<bufferlist> bl_data_vec);
 int cls_queue_list_entries(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, uint32_t max,
                     std::vector<cls_queue_entry>& entries, bool *truncated, std::string& next_marker);
+int cls_queue_list_entries(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, const std::string& end_marker,
+                           std::vector<cls_queue_entry>& entries, bool *truncated, std::string& next_marker);
 void cls_queue_remove_entries(librados::ObjectWriteOperation& op, const std::string& end_marker);
 
 #endif
diff --git a/src/cls/queue/cls_queue_ops.h b/src/cls/queue/cls_queue_ops.h
index 8209659bda90..f41572036c51 100644
--- a/src/cls/queue/cls_queue_ops.h
+++ b/src/cls/queue/cls_queue_ops.h
@@ -54,20 +54,25 @@ WRITE_CLASS_ENCODER(cls_queue_enqueue_op)
 struct cls_queue_list_op {
   uint64_t max;
   std::string start_marker;
+  std::string end_marker;
 
   cls_queue_list_op() {}
 
   void encode(ceph::buffer::list& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     encode(max, bl);
     encode(start_marker, bl);
+    encode(end_marker, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(ceph::buffer::list::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     decode(max, bl);
     decode(start_marker, bl);
+    if (struct_v > 1) {
+      decode(end_marker, bl);
+    }
     DECODE_FINISH(bl);
   }
 };
diff --git a/src/cls/queue/cls_queue_src.cc b/src/cls/queue/cls_queue_src.cc
index b34d9929b93a..028b1cb123c9 100644
--- a/src/cls/queue/cls_queue_src.cc
+++ b/src/cls/queue/cls_queue_src.cc
@@ -400,6 +400,10 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c
         CLS_LOG(10, "INFO: queue_list_entries(): not enough data to read data, breaking out!");
         break;
       }
+      if (!op.end_marker.empty() && entry.marker == op.end_marker) {
+        last_marker = entry.marker;
+        break;
+      }
       op_ret.entries.emplace_back(entry);
       // Resetting some values
       offset_populated = false;
@@ -414,11 +418,17 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c
       }
     } while(index < bl_chunk.length());
 
-    CLS_LOG(10, "INFO: num_ops: %lu and op.max is %lu\n", num_ops, op.max);
+    CLS_LOG(10, "INFO: num_ops: %lu and op.max is %lu, last_marker: %s and op.end_marker is %s\n",
+            num_ops, op.max, last_marker.c_str(), op.end_marker.c_str());
 
-    if (num_ops == op.max) {
-      next_marker = cls_queue_marker{(entry_start_offset + index), gen};
-      CLS_LOG(10, "INFO: queue_list_entries(): num_ops is same as op.max, hence breaking out from outer loop with next offset: %lu", next_marker.offset);
+    if (num_ops == op.max || (!op.end_marker.empty() && op.end_marker == last_marker)) {
+      if (!op.end_marker.empty()) {
+        next_marker.from_str(op.end_marker.c_str());
+      } else {
+        next_marker = cls_queue_marker{(entry_start_offset + index), gen};
+      }
+      CLS_LOG(10, "INFO: queue_list_entries(): either num_ops is same as op.max or last_marker is same as op.end_marker, "
+                  "hence breaking out from outer loop with next offset: %lu", next_marker.offset);
       break;
     }
 
diff --git a/src/test/cls_2pc_queue/test_cls_2pc_queue.cc b/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
index 14947244d41f..8f392721f5b7 100644
--- a/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
+++ b/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
@@ -11,6 +11,7 @@
 #include "gtest/gtest.h"
 #include "test/librados/test_cxx.h"
 #include "global/global_context.h"
+#include "cls/2pc_queue/cls_2pc_queue_const.h"
 
 #include <string>
 #include <vector>
@@ -173,6 +174,131 @@ TEST_F(TestCls2PCQueue, Commit)
   ASSERT_EQ(reservations.size(), 0);
 }
 
+TEST_F(TestCls2PCQueue, Stats)
+{
+  const std::string queue_name = __PRETTY_FUNCTION__;
+  const auto max_size = 1024*1024*128;
+  const auto number_of_ops = 200U;
+  const auto number_of_elements = 23U;
+  auto total_committed_elements = 0U;
+  librados::ObjectWriteOperation op;
+  op.create(true);
+  cls_2pc_queue_init(op, queue_name, max_size);
+  ASSERT_EQ(0, ioctx.operate(queue_name, &op));
+
+  for (auto i = 0U; i < number_of_ops; ++i) {
+    const std::string element_prefix("op-" +to_string(i) + "-element-");
+    auto total_size = 0UL;
+    std::vector<bufferlist> data(number_of_elements);
+    // create vector of buffer lists
+    std::generate(data.begin(), data.end(), [j = 0, &element_prefix, &total_size] () mutable {
+      bufferlist bl;
+      bl.append(element_prefix + to_string(j++));
+      total_size += bl.length();
+      return bl;
+    });
+
+    cls_2pc_reservation::id_t res_id;
+    ASSERT_EQ(cls_2pc_queue_reserve(ioctx, queue_name, total_size, number_of_elements, res_id), 0);
+    ASSERT_NE(res_id, cls_2pc_reservation::NO_ID);
+    cls_2pc_queue_commit(op, data, res_id);
+    ASSERT_EQ(0, ioctx.operate(queue_name, &op));
+
+    total_committed_elements += number_of_elements;
+    uint32_t committed_entries;
+    uint64_t size;
+
+    ASSERT_EQ(cls_2pc_queue_get_topic_stats(ioctx, queue_name, committed_entries, size), 0);
+    ASSERT_EQ(committed_entries, total_committed_elements);
+  }
+  cls_2pc_reservations reservations;
+  ASSERT_EQ(0, cls_2pc_queue_list_reservations(ioctx, queue_name, reservations));
+  ASSERT_EQ(reservations.size(), 0);
+}
+
+TEST_F(TestCls2PCQueue, UpgradeFromReef)
+{
+  const std::string queue_name = __PRETTY_FUNCTION__;
+  const auto max_size = 1024*1024*128;
+  const auto number_of_ops = 200U;
+  const auto number_of_elements = 23U;
+  auto total_committed_elements = 0U;
+  librados::ObjectWriteOperation wop;
+  wop.create(true);
+  cls_2pc_queue_init(wop, queue_name, max_size);
+  ASSERT_EQ(0, ioctx.operate(queue_name, &wop));
+
+  for (auto i = 0U; i < number_of_ops; ++i) {
+    const std::string element_prefix("wop-" +to_string(i) + "-element-");
+    auto total_size = 0UL;
+    std::vector<bufferlist> data(number_of_elements);
+    // create vector of buffer lists
+    std::generate(data.begin(), data.end(), [j = 0, &element_prefix, &total_size] () mutable {
+      bufferlist bl;
+      bl.append(element_prefix + to_string(j++));
+      total_size += bl.length();
+      return bl;
+    });
+
+    cls_2pc_reservation::id_t res_id;
+    ASSERT_EQ(cls_2pc_queue_reserve(ioctx, queue_name, total_size, number_of_elements, res_id), 0);
+    ASSERT_NE(res_id, cls_2pc_reservation::NO_ID);
+    cls_2pc_queue_commit(wop, data, res_id);
+    ASSERT_EQ(0, ioctx.operate(queue_name, &wop));
+
+    total_committed_elements += number_of_elements;
+    uint32_t committed_entries;
+    uint64_t size;
+
+    ASSERT_EQ(cls_2pc_queue_get_topic_stats(ioctx, queue_name, committed_entries, size), 0);
+    ASSERT_EQ(committed_entries, total_committed_elements);
+  }
+  cls_2pc_reservations reservations;
+  ASSERT_EQ(0, cls_2pc_queue_list_reservations(ioctx, queue_name, reservations));
+  ASSERT_EQ(reservations.size(), 0);
+
+  constexpr auto max_elements = 42U;
+  std::string marker;
+  std::string end_marker;
+  librados::ObjectReadOperation rop;
+  auto consume_count = 0U;
+  std::vector<cls_queue_entry> entries;
+  bool truncated = true;
+
+  auto simulate_reef_cls_2pc_queue_remove_entries = [](librados::ObjectWriteOperation& wop, const std::string& end_marker) {
+    bufferlist in;
+    cls_queue_remove_op rem_op;
+    rem_op.end_marker = end_marker;
+    encode(rem_op, in);
+    wop.exec(TPC_QUEUE_CLASS, TPC_QUEUE_REMOVE_ENTRIES, in);
+  };
+
+  while (truncated) {
+    bufferlist bl;
+    int rc;
+    cls_2pc_queue_list_entries(rop, marker, max_elements, &bl, &rc);
+    ASSERT_EQ(0, ioctx.operate(queue_name, &rop, nullptr));
+    ASSERT_EQ(rc, 0);
+    ASSERT_EQ(cls_2pc_queue_list_entries_result(bl, entries, &truncated, end_marker), 0);
+
+    consume_count += entries.size();
+    // simulating reef cls_2pc_queue_remove_entries with cls_queue_remove_op
+    simulate_reef_cls_2pc_queue_remove_entries(wop, end_marker);
+    marker = end_marker;
+    total_committed_elements -= entries.size();
+  }
+
+  // execute all delete operations in a batch
+  ASSERT_EQ(0, ioctx.operate(queue_name, &wop));
+  ASSERT_EQ(consume_count, number_of_ops*number_of_elements);
+
+  uint32_t entries_number;
+  uint64_t size;
+  ASSERT_EQ(cls_2pc_queue_get_topic_stats(ioctx, queue_name, entries_number, size), 0);
+  ASSERT_EQ(total_committed_elements, 0);
+  ASSERT_EQ(entries_number, 0);
+}
+
 TEST_F(TestCls2PCQueue, Abort)
 {
   const std::string queue_name = __PRETTY_FUNCTION__;
diff --git a/src/test/cls_queue/test_cls_queue.cc b/src/test/cls_queue/test_cls_queue.cc
index cca615afb0a5..a1f68638c1f4 100644
--- a/src/test/cls_queue/test_cls_queue.cc
+++ b/src/test/cls_queue/test_cls_queue.cc
@@ -137,6 +137,51 @@ TEST_F(TestClsQueue, List)
   ASSERT_EQ(total_elements, number_of_ops*number_of_elements);
 }
 
+TEST_F(TestClsQueue, ListByEndMarker)
+{
+  const std::string queue_name = "my-queue";
+  const uint64_t queue_size = 1024*1024;
+  librados::ObjectWriteOperation op;
+  op.create(true);
+  cls_queue_init(op, queue_name, queue_size);
+  ASSERT_EQ(0, ioctx.operate(queue_name, &op));
+  const auto number_of_ops = 10;
+  const auto number_of_elements = 100;
+
+  // test multiple enqueues
+  test_enqueue(queue_name, number_of_ops, number_of_elements, 0);
+
+  const auto max_elements = 42;
+  std::string marker, end_marker;
+  bool truncated = false;
+  std::string max_op_next_marker;
+  auto total_elements = 0;
+  do {
+    std::vector<cls_queue_entry> entries;
+    auto ret = cls_queue_list_entries(ioctx, queue_name, marker, max_elements, entries, &truncated, max_op_next_marker);
+    ASSERT_EQ(0, ret);
+    end_marker = max_op_next_marker;
+
+    std::vector<cls_queue_entry> end_marker_entries;
+    std::string end_marker_next_marker;
+    bool end_marker_truncated = false;
+    ret = cls_queue_list_entries(ioctx, queue_name, marker, end_marker, end_marker_entries,
+                                 &end_marker_truncated, end_marker_next_marker);
+    ASSERT_EQ(0, ret);
+
+    ASSERT_EQ(end_marker_next_marker, end_marker);
+    ASSERT_EQ(end_marker_entries.size(), entries.size());
+    for (auto i = 0U; i < end_marker_entries.size() && i < entries.size(); ++i) {
+      ASSERT_EQ(end_marker_entries[i].marker, entries[i].marker);
+    }
+
+    marker = max_op_next_marker;
+    total_elements += entries.size();
+  } while (truncated);
+
+  ASSERT_EQ(total_elements, number_of_ops*number_of_elements);
+}
+
 TEST_F(TestClsQueue, Dequeue)
 {
   const std::string queue_name = "my-queue";

From d8cd49259d797304adfbdb8eb46110e8b749256e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 27 Nov 2023 10:17:17 +1000
Subject: [PATCH 0736/2492] doc/rados: edit "Everything Failed! Now What?"

Edit the section "Everything Failed! Now what?" in
doc/rados/troubleshooting/troubleshooting-mon.rst.

Co-authored-by: Anthoy D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../troubleshooting/troubleshooting-mon.rst   | 63 ++++++++++---------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
index 4bb9d7b5b860..443d6c443bee 100644
--- a/doc/rados/troubleshooting/troubleshooting-mon.rst
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -616,7 +616,6 @@ The above recovery tool is unable to recover the following information:
 
 - **MDS Maps**: The MDS maps are lost.
 
-
 Everything Failed! Now What?
 ============================
 
@@ -628,16 +627,20 @@ irc.oftc.net), or at ``dev@ceph.io`` and ``ceph-users@lists.ceph.com``. Make
 sure that you have prepared your logs and that you have them ready upon
 request.
 
-See https://ceph.io/en/community/connect/ for current (as of October 2023)
-information on getting in contact with the upstream Ceph community.
+The upstream Ceph Slack workspace can be joined at this address:
+https://ceph-storage.slack.com/
 
+See https://ceph.io/en/community/connect/ for current (as of December 2023)
+information on getting in contact with the upstream Ceph community.
 
 Preparing your logs
 -------------------
 
-The default location for monitor logs is ``/var/log/ceph/ceph-mon.FOO.log*``.
-However, if they are not there, you can find their current location by running
-the following command:
+The default location for Monitor logs is ``/var/log/ceph/ceph-mon.FOO.log*``.
+It is possible that the location of the Monitor logs has been changed from the
+default. If the location of the Monitor logs has been changed from the default
+location, find the location of the Monitor logs by running the following
+command:
 
 .. prompt:: bash
 
@@ -648,21 +651,21 @@ cluster's configuration files. If Ceph is using the default debug levels, then
 your logs might be missing important information that would help the upstream
 Ceph community address your issue.
 
-To make sure your monitor logs contain relevant information, you can raise
-debug levels. Here we are interested in information from the monitors.  As with
-other components, the monitors have different parts that output their debug
+Raise debug levels to make sure that your Monitor logs contain relevant
+information. Here we are interested in information from the Monitors.  As with
+other components, the Monitors have different parts that output their debug
 information on different subsystems.
 
 If you are an experienced Ceph troubleshooter, we recommend raising the debug
-levels of the most relevant subsystems. Of course, this approach might not be
-easy for beginners. In most cases, however, enough information to address the
-issue will be secured if the following debug levels are entered::
+levels of the most relevant subsystems. This approach might not be easy for
+beginners. In most cases, however, enough information to address the issue will
+be logged if the following debug levels are entered::
 
       debug_mon = 10
       debug_ms = 1
 
 Sometimes these debug levels do not yield enough information. In such cases,
-members of the upstream Ceph community might ask you to make additional changes
+members of the upstream Ceph community will ask you to make additional changes
 to these or to other debug levels. In any case, it is better for us to receive
 at least some useful information than to receive an empty log.
 
@@ -670,10 +673,12 @@ at least some useful information than to receive an empty log.
 Do I need to restart a monitor to adjust debug levels?
 ------------------------------------------------------
 
-No, restarting a monitor is not necessary. Debug levels may be adjusted by
-using two different methods, depending on whether or not there is a quorum:
+No. It is not necessary to restart a Monitor when adjusting its debug levels. 
+
+There are two different methods for adjusting debug levels. One method is used
+when there is quorum. The other is used when there is no quorum. 
 
-**If there is a quorum**
+**Adjusting debug levels when there is a quorum**
 
   Either inject the debug option into the specific monitor that needs to 
   be debugged::
@@ -685,17 +690,19 @@ using two different methods, depending on whether or not there is a quorum:
         ceph tell mon.* config set debug_mon 10/10
 
 
-**If there is no quorum**
+**Adjusting debug levels when there is no quorum**
 
   Use the admin socket of the specific monitor that needs to be debugged
   and directly adjust the monitor's configuration options::
 
       ceph daemon mon.FOO config set debug_mon 10/10
 
+**Returning debug levels to their default values**
 
 To return the debug levels to their default values, run the above commands
-using the debug level ``1/10`` rather than ``10/10``. To check a monitor's
-current values, use the admin socket and run either of the following commands:
+using the debug level ``1/10`` rather than the debug level ``10/10``. To check
+a Monitor's current values, use the admin socket and run either of the
+following commands:
 
   .. prompt:: bash
 
@@ -712,17 +719,17 @@ or:
 I Reproduced the problem with appropriate debug levels. Now what?
 -----------------------------------------------------------------
 
-We prefer that you send us only the portions of your logs that are relevant to
-your monitor problems. Of course, it might not be easy for you to determine
-which portions are relevant so we are willing to accept complete and
-unabridged logs. However, we request that you avoid sending logs containing
-hundreds of thousands of lines with no additional clarifying information. One
-common-sense way of making our task easier is to write down the current time
-and date when you are reproducing the problem and then extract portions of your
+Send the upstream Ceph community only the portions of your logs that are
+relevant to your Monitor problems. Because it might not be easy for you to
+determine which portions are relevant, the upstream Ceph community accepts
+complete and unabridged logs. But don't send logs containing hundreds of
+thousands of lines with no additional clarifying information. One common-sense
+way to help the Ceph community help you is to write down the current time and
+date when you are reproducing the problem and then extract portions of your
 logs based on that information.
 
-Finally, reach out to us on the mailing lists or IRC or Slack, or by filing a
-new issue on the `tracker`_.
+Contact the upstream Ceph community on the mailing lists or IRC or Slack, or by
+filing a new issue on the `tracker`_.
 
 .. _tracker: http://tracker.ceph.com/projects/ceph/issues/new
 

From 1766a354278706eea560cb989796cc97dbef8b4b Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Mon, 27 Nov 2023 15:55:42 +0800
Subject: [PATCH 0737/2492] mds: set the loner to true for LOCK_EXCL_XSYN

For filelock when in LOCK_EXCL_XSYN state the non-loner clients
should be issued empty caps, but since the loner of this state
is set to false and it could make the Locker to issue the Fcb caps
to them, which is incorrect.

This fix will just set the loner to true.

Fixes: https://tracker.ceph.com/issues/63646
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/locks.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/locks.c b/src/mds/locks.c
index dbe3ab8eb00a..f6ff8b9824a1 100644
--- a/src/mds/locks.c
+++ b/src/mds/locks.c
@@ -117,7 +117,7 @@ const struct sm_state_t filelock[LOCK_MAX] = {
     [LOCK_XSYN_EXCL] = { LOCK_EXCL, true,  LOCK_LOCK, AUTH, 0,   XCL, 0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
 
     [LOCK_XSYN]      = { 0,         true,  LOCK_LOCK, AUTH, AUTH,AUTH,XCL, 0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
-    [LOCK_EXCL_XSYN] = { LOCK_XSYN, false, LOCK_LOCK, 0,    0,   XCL, 0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
+    [LOCK_EXCL_XSYN] = { LOCK_XSYN, true,  LOCK_LOCK, 0,    0,   XCL, 0,   0,   0,   0,   0,CEPH_CAP_GCACHE|CEPH_CAP_GBUFFER,0,0 },
 
     [LOCK_PRE_SCAN]  = { LOCK_SCAN, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },
     [LOCK_SCAN]      = { LOCK_LOCK, false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,   0,0,0,0 },

From 2e06ab0b8c730f063e1376b2f330c3b78089af67 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 18 Oct 2023 12:06:47 +0530
Subject: [PATCH 0738/2492] mgr/dashboard: support rgw roles removal

Fixes: https://tracker.ceph.com/issues/63230
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   | 19 +++++++++++++++++--
 .../mgr/dashboard/services/rgw_client.py      |  8 ++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 9bc1b489edee..fd376a53f037 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -718,6 +718,16 @@ def role_create(_, role_name: str = '', role_path: str = '', role_assume_policy_
         return f'Role {role_name} created successfully'
 
 
+    @staticmethod
+    def role_delete(_, role_name: str):
+        assert role_name
+        rgw_client = RgwClient.admin_instance()
+        rgw_client.delete_role(role_name)
+        return f'Role {role_name} deleted successfully'
+
+
+
+
 # pylint: disable=C0301
 assume_role_policy_help = (
     'Paste a json assume role policy document, to find more information on how to get this document, <a '  # noqa: E501
@@ -746,7 +756,8 @@ def role_create(_, role_name: str = '', role_path: str = '', role_assume_policy_
     doc=APIDoc("List of RGW roles", "RGW"),
     actions=[
         TableAction(name='Create', permission='create', icon=Icon.ADD.value,
-                    routerLink='/rgw/roles/create')
+        TableAction(name='Delete', permission='delete', icon=Icon.DESTROY.value,
+                    click='delete', disable=True),
     ],
     forms=[create_role_form],
     permissions=[Scope.CONFIG_OPT],
@@ -756,7 +767,11 @@ def role_create(_, role_name: str = '', role_path: str = '', role_assume_policy_
     ),
     create=CRUDCollectionMethod(
         func=RGWRoleEndpoints.role_create,
-        doc=EndpointDoc("Create Ceph User")
+        doc=EndpointDoc("Create RGW role")
+    ),
+    delete=CRUDCollectionMethod(
+        func=RGWRoleEndpoints.role_delete,
+        doc=EndpointDoc("Delete RGW role")
     ),
     set_column={
         "CreateDate": {'cellTemplate': 'date'},
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index 2d3226bab3e0..fe7b8c66898d 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -851,6 +851,14 @@ def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: st
                    'Looks like the document has a wrong format.'
                    f' For more information about the format look at {link}')
             raise DashboardException(msg=msg, component='rgw')
+    
+    def delete_role(self, role_name: str) -> None:
+        rgw_delete_role_command = ['role', 'delete', '--role-name', role_name]
+        code, _, _err = mgr.send_rgwadmin_command(rgw_delete_role_command,
+                                                  stdout_as_json=False)
+        if code != 0:
+            raise DashboardException(msg=f'Error deleting role with code {code}: {_err}',
+                                     component='rgw')
 
     @RestClient.api_get('/{bucket_name}?policy')
     def get_bucket_policy(self, bucket_name: str, request=None):

From 5c28d78a45d87d2be6b9ea2961be42689673e252 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 18 Oct 2023 12:08:21 +0530
Subject: [PATCH 0739/2492] mgr/dashboard: support rgw roles updating

Right now only the modification of max_session_duration is supported via
the roles update command. To update, we need to use `policy modify`
command which is not added in this PR. That should be done separately

Refer: https://docs.ceph.com/en/latest/radosgw/role/#update-a-role

Fixes: https://tracker.ceph.com/issues/63230
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/_crud.py | 14 ++++
 .../mgr/dashboard/controllers/ceph_users.py   |  3 +-
 src/pybind/mgr/dashboard/controllers/rgw.py   | 55 +++++++++++++-
 .../cypress/e2e/rgw/roles.e2e-spec.ts         | 14 +++-
 .../frontend/cypress/e2e/rgw/roles.po.ts      | 26 ++++++-
 .../frontend/src/app/ceph/rgw/rgw.module.ts   |  7 ++
 .../src/app/core/context/context.component.ts | 12 ++-
 .../crud-table/crud-table.component.ts        |  6 +-
 .../app/shared/datatable/datatable.module.ts  |  6 +-
 .../src/app/shared/forms/crud-form/helpers.ts | 10 ++-
 .../validators/rgw-role-validator.ts          |  9 +++
 .../app/shared/models/crud-table-metadata.ts  |  1 +
 src/pybind/mgr/dashboard/openapi.yaml         | 75 ++++++++++++++++++-
 .../mgr/dashboard/services/rgw_client.py      | 19 ++++-
 14 files changed, 237 insertions(+), 20 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/_crud.py b/src/pybind/mgr/dashboard/controllers/_crud.py
index 240a2b5ab8c4..d65649cadb4b 100644
--- a/src/pybind/mgr/dashboard/controllers/_crud.py
+++ b/src/pybind/mgr/dashboard/controllers/_crud.py
@@ -104,6 +104,7 @@ class Validator(Enum):
     RGW_ROLE_NAME = 'rgwRoleName'
     RGW_ROLE_PATH = 'rgwRolePath'
     FILE = 'file'
+    RGW_ROLE_SESSION_DURATION = 'rgwRoleSessionDuration'
 
 
 class FormField(NamedTuple):
@@ -224,6 +225,10 @@ def to_dict(self, key=''):
                 properties[field.key]['title'] = field.name
                 field_ui_schema['key'] = field_key
                 field_ui_schema['readonly'] = field.readonly
+                if field.readonly:
+                    field_ui_schema['templateOptions'] = {
+                        'disabled': True
+                    }
                 field_ui_schema['help'] = f'{field.help}'
                 field_ui_schema['validators'] = [i.value for i in field.validators]
                 items.append(field_ui_schema)
@@ -307,6 +312,7 @@ def __init__(self):
         self.forms = []
         self.columnKey = ''
         self.detail_columns = []
+        self.resource = ''
 
 
 class CRUDCollectionMethod(NamedTuple):
@@ -330,6 +336,7 @@ def __init__(self, router: APIRouter, doc: APIDoc,
                  actions: Optional[List[TableAction]] = None,
                  permissions: Optional[List[str]] = None, forms: Optional[List[Form]] = None,
                  column_key: Optional[str] = None,
+                 resource: Optional[str] = None,
                  meta: CRUDMeta = CRUDMeta(), get_all: Optional[CRUDCollectionMethod] = None,
                  create: Optional[CRUDCollectionMethod] = None,
                  delete: Optional[CRUDCollectionMethod] = None,
@@ -352,6 +359,7 @@ def __init__(self, router: APIRouter, doc: APIDoc,
         self.detail_columns = detail_columns if detail_columns is not None else []
         self.extra_endpoints = extra_endpoints if extra_endpoints is not None else []
         self.selection_type = selection_type
+        self.resource = resource
 
     def __call__(self, cls: Any):
         self.create_crud_class(cls)
@@ -415,6 +423,7 @@ def _list(self, model_key: str = ''):
             self.generate_forms(model_key)
             self.set_permissions()
             self.set_column_key()
+            self.set_table_resource()
             self.get_detail_columns()
             selection_type = self.__class__.outer_self.selection_type
             self.__class__.outer_self.meta.table.set_selection_type(selection_type)
@@ -468,6 +477,10 @@ def set_column_key(self):
             if self.__class__.outer_self.column_key:
                 self.outer_self.meta.columnKey = self.__class__.outer_self.column_key
 
+        def set_table_resource(self):
+            if self.__class__.outer_self.resource:
+                self.outer_self.meta.resource = self.__class__.outer_self.resource
+
         class_name = self.router.path.replace('/', '')
         meta_class = type(f'{class_name}_CRUDClassMetadata',
                           (RESTController,),
@@ -478,6 +491,7 @@ def set_column_key(self):
                               'generate_forms': generate_forms,
                               'set_permissions': set_permissions,
                               'set_column_key': set_column_key,
+                              'set_table_resource': set_table_resource,
                               'get_detail_columns': get_detail_columns,
                               'outer_self': self,
                           })
diff --git a/src/pybind/mgr/dashboard/controllers/ceph_users.py b/src/pybind/mgr/dashboard/controllers/ceph_users.py
index e1bdc157091c..022f8f36c420 100644
--- a/src/pybind/mgr/dashboard/controllers/ceph_users.py
+++ b/src/pybind/mgr/dashboard/controllers/ceph_users.py
@@ -174,7 +174,7 @@ def model(user_entity: str):
         TableAction(name='Create', permission='create', icon=Icon.ADD.value,
                     routerLink='/cluster/user/create'),
         TableAction(name='Edit', permission='update', icon=Icon.EDIT.value,
-                    click='edit'),
+                    click='edit', routerLink='/cluster/user/edit'),
         TableAction(name='Delete', permission='delete', icon=Icon.DESTROY.value,
                     click='delete', disable=True),
         TableAction(name='Import', permission='create', icon=Icon.IMPORT.value,
@@ -185,6 +185,7 @@ def model(user_entity: str):
     permissions=[Scope.CONFIG_OPT],
     forms=[create_form, edit_form, import_user_form],
     column_key='entity',
+    resource='user',
     get_all=CRUDCollectionMethod(
         func=CephUserEndpoints.user_list,
         doc=EndpointDoc("Get Ceph Users")
diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index fd376a53f037..bf2d95553179 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+# pylint: disable=C0302
 import json
 import logging
 import re
@@ -717,6 +718,15 @@ def role_create(_, role_name: str = '', role_path: str = '', role_assume_policy_
         rgw_client.create_role(role_name, role_path, role_assume_policy_doc)
         return f'Role {role_name} created successfully'
 
+    @staticmethod
+    def role_update(_, role_name: str, max_session_duration: str):
+        assert role_name
+        assert max_session_duration
+        # convert max_session_duration which is in hours to seconds
+        max_session_duration = int(float(max_session_duration) * 3600)
+        rgw_client = RgwClient.admin_instance()
+        rgw_client.update_role(role_name, str(max_session_duration))
+        return f'Role {role_name} updated successfully'
 
     @staticmethod
     def role_delete(_, role_name: str):
@@ -725,7 +735,18 @@ def role_delete(_, role_name: str):
         rgw_client.delete_role(role_name)
         return f'Role {role_name} deleted successfully'
 
+    @staticmethod
+    def model(role_name: str):
+        assert role_name
+        rgw_client = RgwClient.admin_instance()
+        role = rgw_client.get_role(role_name)
+        model = {'role_name': '', 'max_session_duration': ''}
+        model['role_name'] = role['RoleName']
 
+        # convert maxsessionduration which is in seconds to hours
+        if role['MaxSessionDuration']:
+            model['max_session_duration'] = role['MaxSessionDuration'] / 3600
+        return model
 
 
 # pylint: disable=C0301
@@ -735,6 +756,10 @@ def role_delete(_, role_name: str):
     'target="_blank">click here.</a>'
 )
 
+max_session_duration_help = (
+    'The maximum session duration (in hours) that you want to set for the specified role.This setting can have a value from 1 hour to 12 hours.'  # noqa: E501
+)
+
 create_container = VerticalContainer('Create Role', 'create_role', fields=[
     FormField('Role name', 'role_name', validators=[Validator.RGW_ROLE_NAME]),
     FormField('Path', 'role_path', validators=[Validator.RGW_ROLE_PATH]),
@@ -744,23 +769,43 @@ def role_delete(_, role_name: str):
               field_type='textarea',
               validators=[Validator.JSON]),
 ])
-create_role_form = Form(path='/rgw/roles/create',
+
+edit_container = VerticalContainer('Edit Role', 'edit_role', fields=[
+    FormField('Role name', 'role_name', readonly=True),
+    FormField('Max Session Duration', 'max_session_duration',
+              help=max_session_duration_help,
+              validators=[Validator.RGW_ROLE_SESSION_DURATION])
+])
+
+create_role_form = Form(path='/create',
                         root_container=create_container,
                         task_info=FormTaskInfo("IAM RGW Role '{role_name}' created successfully",
                                                ['role_name']),
                         method_type=MethodType.POST.value)
 
+edit_role_form = Form(path='/edit',
+                      root_container=edit_container,
+                      task_info=FormTaskInfo("IAM RGW Role '{role_name}' edited successfully",
+                                             ['role_name']),
+                      method_type=MethodType.PUT.value,
+                      model_callback=RGWRoleEndpoints.model)
+
 
 @CRUDEndpoint(
     router=APIRouter('/rgw/roles', Scope.RGW),
     doc=APIDoc("List of RGW roles", "RGW"),
     actions=[
         TableAction(name='Create', permission='create', icon=Icon.ADD.value,
+                    routerLink='/rgw/roles/create'),
+        TableAction(name='Edit', permission='update', icon=Icon.EDIT.value,
+                    click='edit', routerLink='/rgw/roles/edit'),
         TableAction(name='Delete', permission='delete', icon=Icon.DESTROY.value,
                     click='delete', disable=True),
     ],
-    forms=[create_role_form],
-    permissions=[Scope.CONFIG_OPT],
+    forms=[create_role_form, edit_role_form],
+    column_key='RoleName',
+    resource='Role',
+    permissions=[Scope.RGW],
     get_all=CRUDCollectionMethod(
         func=RGWRoleEndpoints.role_list,
         doc=EndpointDoc("List RGW roles")
@@ -769,6 +814,10 @@ def role_delete(_, role_name: str):
         func=RGWRoleEndpoints.role_create,
         doc=EndpointDoc("Create RGW role")
     ),
+    edit=CRUDCollectionMethod(
+        func=RGWRoleEndpoints.role_update,
+        doc=EndpointDoc("Edit RGW role")
+    ),
     delete=CRUDCollectionMethod(
         func=RGWRoleEndpoints.role_delete,
         doc=EndpointDoc("Delete RGW role")
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.e2e-spec.ts
index 597f7d1be881..80a8b0ec902a 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.e2e-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.e2e-spec.ts
@@ -9,11 +9,21 @@ describe('RGW roles page', () => {
   });
 
   describe('Create, Edit & Delete rgw roles', () => {
+    const roleName = 'testRole';
+
     it('should create rgw roles', () => {
       roles.navigateTo('create');
-      roles.create('testRole', '/', '{}');
+      roles.create(roleName, '/', '{}');
       roles.navigateTo();
-      roles.checkExist('testRole', true);
+      roles.checkExist(roleName, true);
+    });
+
+    it('should edit rgw role', () => {
+      roles.edit(roleName, 3);
+    });
+
+    it('should delete rgw role', () => {
+      roles.delete(roleName);
     });
   });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.po.ts
index b72ca5df9a7f..717655b2f08c 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/roles.po.ts
@@ -11,18 +11,36 @@ export class RolesPageHelper extends PageHelper {
   columnIndex = {
     roleName: 2,
     path: 3,
-    arn: 4
+    arn: 4,
+    createDate: 5,
+    maxSessionDuration: 6
   };
 
   @PageHelper.restrictTo(pages.create.url)
   create(name: string, path: string, policyDocument: string) {
-    cy.get('#formly_3_string_role_name_0').type(name);
-    cy.get('#formly_3_textarea_role_assume_policy_doc_2').type(policyDocument);
-    cy.get('#formly_3_string_role_path_1').type(path);
+    cy.get('[id$="string_role_name_0"]').type(name);
+    cy.get('[id$="role_assume_policy_doc_2"]').type(policyDocument);
+    cy.get('[id$="role_path_1"]').type(path);
     cy.get("[aria-label='Create Role']").should('exist').click();
     cy.get('cd-crud-table').should('exist');
   }
 
+  edit(name: string, maxSessionDuration: number) {
+    this.navigateEdit(name);
+    cy.get('[id$="max_session_duration_1"]').clear().type(maxSessionDuration.toString());
+    cy.get("[aria-label='Edit Role']").should('exist').click();
+    cy.get('cd-crud-table').should('exist');
+
+    this.getTableCell(this.columnIndex.roleName, name)
+      .click()
+      .parent()
+      .find(`datatable-body-cell:nth-child(${this.columnIndex.maxSessionDuration})`)
+      .should(($elements) => {
+        const roleName = $elements.map((_, el) => el.textContent).get();
+        expect(roleName).to.include(`${maxSessionDuration} hours`);
+      });
+  }
+
   @PageHelper.restrictTo(pages.index.url)
   checkExist(name: string, exist: boolean) {
     this.getTableCell(this.columnIndex.roleName, name).should(($elements) => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
index c16c13a81bd7..f2e37af0aa70 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
@@ -156,6 +156,13 @@ const routes: Routes = [
         data: {
           breadcrumbs: ActionLabels.CREATE
         }
+      },
+      {
+        path: URLVerbs.EDIT,
+        component: CrudFormComponent,
+        data: {
+          breadcrumbs: ActionLabels.EDIT
+        }
       }
     ]
   },
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/context/context.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/context/context.component.ts
index e036b754438b..178f230c931f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/context/context.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/context/context.component.ts
@@ -24,12 +24,14 @@ export class ContextComponent implements OnInit, OnDestroy {
   private subs = new Subscription();
   private rgwUrlPrefix = '/rgw';
   private rgwUserUrlPrefix = '/rgw/user';
+  private rgwRoleUrlPrefix = '/rgw/roles';
   private rgwBuckerUrlPrefix = '/rgw/bucket';
   permissions: Permissions;
   featureToggleMap$: FeatureTogglesMap$;
   isRgwRoute =
     document.location.href.includes(this.rgwUserUrlPrefix) ||
-    document.location.href.includes(this.rgwBuckerUrlPrefix);
+    document.location.href.includes(this.rgwBuckerUrlPrefix) ||
+    document.location.href.includes(this.rgwRoleUrlPrefix);
 
   constructor(
     private authStorageService: AuthStorageService,
@@ -48,9 +50,11 @@ export class ContextComponent implements OnInit, OnDestroy {
         .pipe(filter((event: Event) => event instanceof NavigationEnd))
         .subscribe(
           () =>
-            (this.isRgwRoute = [this.rgwBuckerUrlPrefix, this.rgwUserUrlPrefix].some((urlPrefix) =>
-              this.router.url.startsWith(urlPrefix)
-            ))
+            (this.isRgwRoute = [
+              this.rgwBuckerUrlPrefix,
+              this.rgwUserUrlPrefix,
+              this.rgwRoleUrlPrefix
+            ].some((urlPrefix) => this.router.url.startsWith(urlPrefix)))
         )
     );
     // Set daemon list polling only when in RGW route:
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
index 750152161c24..6881e373b588 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
@@ -120,7 +120,7 @@ export class CRUDTableComponent implements OnInit {
   delete() {
     const selectedKey = this.selection.first()[this.meta.columnKey];
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
-      itemDescription: $localize`${this.meta.columnKey}`,
+      itemDescription: $localize`${this.meta.resource}`,
       itemNames: [selectedKey],
       submitAction: () => {
         this.taskWrapper
@@ -153,7 +153,9 @@ export class CRUDTableComponent implements OnInit {
     if (this.selection.hasSelection) {
       key = this.selection.first()[this.meta.columnKey];
     }
-    this.router.navigate(['/cluster/user/edit'], { queryParams: { key: key } });
+
+    const editAction = this.meta.actions.find((action) => action.name === 'Edit');
+    this.router.navigate([editAction.routerLink], { queryParams: { key: key } });
   }
 
   authExport() {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/datatable.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/datatable.module.ts
index 37e94f236be9..76cbbcfb3a20 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/datatable.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/datatable.module.ts
@@ -61,7 +61,11 @@ import { CheckedTableFormComponent } from './checked-table-form/checked-table-fo
             'Role path must start and finish with a slash "/".' +
             ' (pattern: (\u002F)|(\u002F[\u0021-\u007E]+\u002F))'
         },
-        { name: 'file_size', message: 'File size must not exceed 4KiB' }
+        { name: 'file_size', message: 'File size must not exceed 4KiB' },
+        {
+          name: 'rgwRoleSessionDuration',
+          message: 'This field must be a number and should be a value from 1 hour to 12 hour'
+        }
       ],
       wrappers: [{ name: 'input-wrapper', component: FormlyInputWrapperComponent }]
     }),
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/helpers.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/helpers.ts
index 1ea21b71081c..aca9a20af09c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/helpers.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/helpers.ts
@@ -3,7 +3,11 @@ import { FormlyFieldConfig } from '@ngx-formly/core';
 import { forEach } from 'lodash';
 import { formlyAsyncFileValidator } from './validators/file-validator';
 import { formlyAsyncJsonValidator } from './validators/json-validator';
-import { formlyRgwRoleNameValidator, formlyRgwRolePath } from './validators/rgw-role-validator';
+import {
+  formlyFormNumberValidator,
+  formlyRgwRoleNameValidator,
+  formlyRgwRolePath
+} from './validators/rgw-role-validator';
 
 export function getFieldState(field: FormlyFieldConfig, uiSchema: any[] = undefined) {
   const formState: any[] = uiSchema || field.options?.formState;
@@ -34,6 +38,10 @@ export function setupValidators(field: FormlyFieldConfig, uiSchema: any[]) {
         validators.push(formlyAsyncFileValidator);
         break;
       }
+      case 'rgwRoleSessionDuration': {
+        validators.push(formlyFormNumberValidator);
+        break;
+      }
     }
   });
   field.asyncValidators = { validation: validators };
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/validators/rgw-role-validator.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/validators/rgw-role-validator.ts
index a100f278bea0..c994dc964070 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/validators/rgw-role-validator.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/validators/rgw-role-validator.ts
@@ -17,3 +17,12 @@ export function formlyRgwRoleNameValidator(control: AbstractControl): Promise<an
     resolve({ rgwRoleName: true });
   });
 }
+
+export function formlyFormNumberValidator(control: AbstractControl): Promise<any> {
+  return new Promise((resolve, _reject) => {
+    if (control.value.match('^[0-9.]+$')) {
+      if (control.value <= 12 && control.value >= 1) resolve(null);
+    }
+    resolve({ rgwRoleSessionDuration: true });
+  });
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
index 140fa5b5f8ea..dc33e6236ae2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
@@ -14,4 +14,5 @@ export class CrudMetadata {
   actions: CdTableAction[];
   forms: any;
   columnKey: string;
+  resource: string;
 }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index d35ea87e15ae..453352493d45 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -9596,7 +9596,80 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: Create Ceph User
+      summary: Create RGW role
+      tags:
+      - RGW
+    put:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                max_session_duration:
+                  type: string
+                role_name:
+                  type: string
+              required:
+              - role_name
+              - max_session_duration
+              type: object
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource updated.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Edit RGW role
+      tags:
+      - RGW
+  /api/rgw/roles/{role_name}:
+    delete:
+      parameters:
+      - in: path
+        name: role_name
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Delete RGW role
       tags:
       - RGW
   /api/rgw/site:
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index fe7b8c66898d..6358d3979769 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -851,7 +851,24 @@ def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: st
                    'Looks like the document has a wrong format.'
                    f' For more information about the format look at {link}')
             raise DashboardException(msg=msg, component='rgw')
-    
+
+    def get_role(self, role_name: str):
+        rgw_get_role_command = ['role', 'get', '--role-name', role_name]
+        code, role, _err = mgr.send_rgwadmin_command(rgw_get_role_command)
+        if code != 0:
+            raise DashboardException(msg=f'Error getting role with code {code}: {_err}',
+                                     component='rgw')
+        return role
+
+    def update_role(self, role_name: str, max_session_duration: str):
+        rgw_update_role_command = ['role', 'update', '--role-name',
+                                   role_name, '--max_session_duration', max_session_duration]
+        code, _, _err = mgr.send_rgwadmin_command(rgw_update_role_command,
+                                                  stdout_as_json=False)
+        if code != 0:
+            raise DashboardException(msg=f'Error updating role with code {code}: {_err}',
+                                     component='rgw')
+
     def delete_role(self, role_name: str) -> None:
         rgw_delete_role_command = ['role', 'delete', '--role-name', role_name]
         code, _, _err = mgr.send_rgwadmin_command(rgw_delete_role_command,

From f2cc24b94c5e8d8856797bd198dbb41907955ffb Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 24 Nov 2023 14:15:39 +0530
Subject: [PATCH 0740/2492] mgr/dashboard: fs rename only when fs is offline

and refuse_client_session is set

Fixes: https://tracker.ceph.com/issues/63608
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  5 ++-
 .../filesystems/filesystems.e2e-spec.feature  | 19 +++++-----
 .../frontend/src/app/app-routing.module.ts    |  2 +-
 .../cephfs-form/cephfs-form.component.html    | 21 +++++++++++
 .../cephfs-form/cephfs-form.component.spec.ts | 37 +++++++++++++++++++
 .../cephfs-form/cephfs-form.component.ts      | 22 ++++++++++-
 .../cephfs-list/cephfs-list.component.ts      |  2 +-
 7 files changed, 94 insertions(+), 14 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index ed83f91d0c97..a1066cbae0db 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -335,13 +335,16 @@ def fs_status(self, fs_id):
 
         standby_table = self.get_standby_table(fsmap['standbys'], mds_versions)
 
+        flags = mdsmap['flags_state']
+
         return {
             "cephfs": {
                 "id": fs_id,
                 "name": mdsmap['fs_name'],
                 "client_count": client_count,
                 "ranks": rank_table,
-                "pools": pools_table
+                "pools": pools_table,
+                "flags": flags,
             },
             "standbys": standby_table,
             "versions": mds_versions
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/filesystems.e2e-spec.feature b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/filesystems.e2e-spec.feature
index 2c08fb56eff1..54fb1a8139e3 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/filesystems.e2e-spec.feature
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/filesystems.e2e-spec.feature
@@ -12,19 +12,20 @@ Feature: CephFS Management
         And I click on "Create File System" button
         Then I should see a row with "test_cephfs"
 
-    Scenario: Edit CephFS Volume
-        Given I am on the "cephfs" page
-        And I select a row "test_cephfs"
-        And I click on "Edit" button
-        And enter "name" "test_cephfs_edit"
-        And I click on "Edit File System" button
-        Then I should see a row with "test_cephfs_edit"
+    # Should be uncommented once the pre-requisite is fixed
+    # Scenario: Edit CephFS Volume
+    #     Given I am on the "cephfs" page
+    #     And I select a row "test_cephfs"
+    #     And I click on "Edit" button
+    #     And enter "name" "test_cephfs_edit"
+    #     And I click on "Edit File System" button
+    #     Then I should see a row with "test_cephfs_edit"
 
     Scenario: Remove CephFS Volume
         Given I am on the "cephfs" page
-        And I select a row "test_cephfs_edit"
+        And I select a row "test_cephfs"
         And I click on "Remove" button from the table actions
         Then I should see the modal
         And I check the tick box in modal
         And I click on "Remove File System" button
-        Then I should not see a row with "test_cephfs_edit"
+        Then I should not see a row with "test_cephfs"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
index 38ae3a4affd0..2ba634fa25d0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
@@ -361,7 +361,7 @@ const routes: Routes = [
             data: { breadcrumbs: ActionLabels.CREATE }
           },
           {
-            path: `${URLVerbs.EDIT}/:name`,
+            path: `${URLVerbs.EDIT}/:id`,
             component: CephfsVolumeFormComponent,
             data: { breadcrumbs: ActionLabels.EDIT }
           }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
index 05235d16ccd4..18db21f891f5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
@@ -15,6 +15,26 @@
                         i18n
                         *ngIf="!editing">Orchestrator is not configured. Deploy MDS daemons manually after creating the volume.</cd-alert-panel>
       </ng-container>
+
+      <cd-alert-panel type="info"
+                      class="m-3"
+                      spacingClass="mt-3"
+                      i18n
+                      *ngIf="editing && disableRename">
+        <p>The File System can only be renamed if it is shutdown and `refuse_client_session` is set to true.
+           Follow the steps below in the command line and refresh the page:</p>
+        <pre class="d-flex">{{ fsFailCmd }}
+          <cd-copy-2-clipboard-button [source]="fsFailCmd"
+                                      [byId]="false"
+                                      [showIconOnly]="true"></cd-copy-2-clipboard-button>
+        </pre>
+        <pre class="d-flex">{{ fsSetCmd }}
+          <cd-copy-2-clipboard-button [source]="fsSetCmd"
+                                      [byId]="false"
+                                      [showIconOnly]="true"></cd-copy-2-clipboard-button>
+        </pre>
+      </cd-alert-panel>
+
       <div class="card-body">
         <!-- Name -->
         <div class="form-group row">
@@ -98,6 +118,7 @@
         <cd-form-button-panel (submitActionEvent)="submit()"
                               [form]="form"
                               [submitText]="(action | titlecase) + ' ' + (resource | upperFirst)"
+                              [disabled]="editing ? disableRename: false"
                               wrappingClass="text-right"></cd-form-button-panel>
       </div>
     </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
index 461f4bca052d..520f726d5553 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
@@ -78,5 +78,42 @@ describe('CephfsVolumeFormComponent', () => {
       expect(label).toBeNull();
       expect(hosts).toBeNull();
     });
+
+    it('should disable renaming and show info alert if disableRename is true', () => {
+      component.disableRename = true;
+      component.ngOnInit();
+      fixture.detectChanges();
+      const alertPanel = fixture.debugElement.query(By.css('cd-alert-panel'));
+      expect(alertPanel).not.toBeNull();
+    });
+
+    it('should not show the alert if disableRename is false', () => {
+      component.disableRename = false;
+      component.ngOnInit();
+      fixture.detectChanges();
+      const alertPanel = fixture.debugElement.query(By.css('cd-alert-panel'));
+      expect(alertPanel).toBeNull();
+    });
+
+    it('should disable the submit button only if disableRename is true', () => {
+      component.disableRename = true;
+      component.ngOnInit();
+      fixture.detectChanges();
+      const submitButton = fixture.debugElement.query(By.css('button[type=submit]'));
+      expect(submitButton.nativeElement.disabled).toBeTruthy();
+
+      // the submit button should only be disabled when the form is in edit mode
+      component.editing = false;
+      component.ngOnInit();
+      fixture.detectChanges();
+      expect(submitButton.nativeElement.disabled).toBeFalsy();
+
+      // submit button should be enabled if disableRename is false
+      component.editing = true;
+      component.disableRename = false;
+      component.ngOnInit();
+      fixture.detectChanges();
+      expect(submitButton.nativeElement.disabled).toBeFalsy();
+    });
   });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
index 6d84e33c7b61..b0f90979c252 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
@@ -51,6 +51,11 @@ export class CephfsVolumeFormComponent extends CdForm implements OnInit {
   labels: string[];
   hasOrchestrator: boolean;
   currentVolumeName: string;
+  fsId: number;
+  disableRename: boolean = true;
+
+  fsFailCmd: string;
+  fsSetCmd: string;
 
   constructor(
     private router: Router,
@@ -101,9 +106,22 @@ export class CephfsVolumeFormComponent extends CdForm implements OnInit {
 
   ngOnInit() {
     if (this.editing) {
-      this.route.params.subscribe((params: { name: string }) => {
-        this.currentVolumeName = params.name;
+      this.route.params.subscribe((params: { id: string }) => {
+        this.fsId = Number(params.id);
+      });
+
+      this.cephfsService.getCephfs(this.fsId).subscribe((resp: object) => {
+        this.currentVolumeName = resp['cephfs']['name'];
         this.form.get('name').setValue(this.currentVolumeName);
+
+        this.disableRename = !(
+          !resp['cephfs']['flags']['joinable'] && resp['cephfs']['flags']['refuse_client_session']
+        );
+        if (this.disableRename) {
+          this.form.get('name').disable();
+          this.fsFailCmd = `ceph fs fail ${this.currentVolumeName}`;
+          this.fsSetCmd = `ceph fs set ${this.currentVolumeName} refuse_client_session true`;
+        }
       });
     } else {
       const hostContext = new CdTableFetchDataContext(() => undefined);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
index 0d55845ab594..26e79727c12d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
@@ -87,7 +87,7 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
         permission: 'update',
         icon: Icons.edit,
         click: () =>
-          this.router.navigate([this.urlBuilder.getEdit(this.selection.first().mdsmap.fs_name)])
+          this.router.navigate([this.urlBuilder.getEdit(String(this.selection.first().id))])
       },
       {
         permission: 'delete',

From 009832f07abe2bf051965097f550d8695e9cebd4 Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Thu, 26 Oct 2023 13:14:10 +0000
Subject: [PATCH 0741/2492] mgr/pg_autoscaler: add check for norecover flag

The PG autoscaler should not be active when the norecover flag is set.
As each newly split PG will need to be backfilled, it does not make sense to have the autoscaler active if the norecover flag is set.

Fixes: https://tracker.ceph.com/issues/63334
Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 src/pybind/mgr/pg_autoscaler/module.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/pg_autoscaler/module.py b/src/pybind/mgr/pg_autoscaler/module.py
index ea7c4b00b4c6..0ab26387790d 100644
--- a/src/pybind/mgr/pg_autoscaler/module.py
+++ b/src/pybind/mgr/pg_autoscaler/module.py
@@ -260,6 +260,13 @@ def has_noautoscale_flag(self) -> bool:
         else:
             return False
 
+    def has_norecover_flag(self) -> bool:
+        flags = self.get_osdmap().dump().get('flags', '')
+        if 'norecover' in flags:
+            return True
+        else:
+            return False
+
     @CLIWriteCommand("osd pool get noautoscale")
     def get_noautoscale(self) -> Tuple[int, str, str]:
         """
@@ -321,7 +328,7 @@ def set_noautoscale(self) -> Tuple[int, str, str]:
     def serve(self) -> None:
         self.config_notify()
         while not self._shutdown.is_set():
-            if not self.has_noautoscale_flag():
+            if not self.has_noautoscale_flag() and not self.has_norecover_flag():
                 osdmap = self.get_osdmap()
                 pools = osdmap.get_pools_by_name()
                 self._maybe_adjust(osdmap, pools)

From 82ec01e208f1fd8d9e6b188e5c919b6ef4a9cf26 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Thu, 23 Jun 2022 04:11:09 -0400
Subject: [PATCH 0742/2492] rgw: rewrite rgw_admin help text

* The line wrapping was fairly random.
* There were cases where words appeared to be doubled.
  - In fact one was an argument
  - the other was the start of the help text.

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 src/rgw/rgw_admin.cc              | 706 +++++++++++++++---------------
 src/test/cli/radosgw-admin/help.t | 702 +++++++++++++++--------------
 2 files changed, 688 insertions(+), 720 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index d44e66c700cd..04d085daee59 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -131,379 +131,363 @@ void usage()
 {
   cout << "usage: radosgw-admin <cmd> [options...]" << std::endl;
   cout << "commands:\n";
-  cout << "  user create                create a new user\n" ;
-  cout << "  user modify                modify user\n";
-  cout << "  user info                  get user info\n";
-  cout << "  user rename                rename user\n";
-  cout << "  user rm                    remove user\n";
-  cout << "  user suspend               suspend a user\n";
-  cout << "  user enable                re-enable user after suspension\n";
-  cout << "  user check                 check user info\n";
-  cout << "  user stats                 show user stats as accounted by quota subsystem\n";
-  cout << "  user list                  list users\n";
-  cout << "  caps add                   add user capabilities\n";
-  cout << "  caps rm                    remove user capabilities\n";
-  cout << "  subuser create             create a new subuser\n" ;
-  cout << "  subuser modify             modify subuser\n";
-  cout << "  subuser rm                 remove subuser\n";
-  cout << "  key create                 create access key\n";
-  cout << "  key rm                     remove access key\n";
-  cout << "  bucket list                list buckets (specify --allow-unordered for\n";
-  cout << "                             faster, unsorted listing)\n";
-  cout << "  bucket limit check         show bucket sharding stats\n";
-  cout << "  bucket link                link bucket to specified user\n";
-  cout << "  bucket unlink              unlink bucket from specified user\n";
-  cout << "  bucket stats               returns bucket statistics\n";
-  cout << "  bucket rm                  remove bucket\n";
-  cout << "  bucket check               check bucket index by verifying size and object count stats\n";
-  cout << "  bucket check olh           check for olh index entries and objects that are pending removal\n";
-  cout << "  bucket check unlinked      check for object versions that are not visible in a bucket listing \n";
-  cout << "  bucket chown               link bucket to specified user and update its object ACLs\n";
-  cout << "  bucket reshard             reshard bucket\n";
-  cout << "  bucket rewrite             rewrite all objects in the specified bucket\n";
-  cout << "  bucket sync checkpoint     poll a bucket's sync status until it catches up to its remote\n";
-  cout << "  bucket sync disable        disable bucket sync\n";
-  cout << "  bucket sync enable         enable bucket sync\n";
-  cout << "  bucket radoslist           list rados objects backing bucket's objects\n";
-  cout << "  bi get                     retrieve bucket index object entries\n";
-  cout << "  bi put                     store bucket index object entries\n";
-  cout << "  bi list                    list raw bucket index entries\n";
-  cout << "  bi purge                   purge bucket index entries\n";
-  cout << "  object rm                  remove object\n";
-  cout << "  object put                 put object\n";
-  cout << "  object stat                stat an object for its metadata\n";
-  cout << "  object unlink              unlink object from bucket index\n";
-  cout << "  object rewrite             rewrite the specified object\n";
-  cout << "  object reindex             reindex the object(s) indicated by --bucket and either --object or --objects-file\n";
-  cout << "  objects expire             run expired objects cleanup\n";
-  cout << "  objects expire-stale list  list stale expired objects (caused by reshard)\n";
-  cout << "  objects expire-stale rm    remove stale expired objects\n";
-  cout << "  period rm                  remove a period\n";
-  cout << "  period get                 get period info\n";
-  cout << "  period get-current         get current period info\n";
-  cout << "  period pull                pull a period\n";
-  cout << "  period push                push a period\n";
-  cout << "  period list                list all periods\n";
-  cout << "  period update              update the staging period\n";
-  cout << "  period commit              commit the staging period\n";
-  cout << "  quota set                  set quota params\n";
-  cout << "  quota enable               enable quota\n";
-  cout << "  quota disable              disable quota\n";
-  cout << "  ratelimit get              get ratelimit params\n";
-  cout << "  ratelimit set              set ratelimit params\n";
-  cout << "  ratelimit enable           enable ratelimit\n";
-  cout << "  ratelimit disable          disable ratelimit\n";
-  cout << "  global quota get           view global quota params\n";
-  cout << "  global quota set           set global quota params\n";
-  cout << "  global quota enable        enable a global quota\n";
-  cout << "  global quota disable       disable a global quota\n";
-  cout << "  global ratelimit get       view global ratelimit params\n";
-  cout << "  global ratelimit set       set global ratelimit params\n";
-  cout << "  global ratelimit enable    enable a ratelimit quota\n";
-  cout << "  global ratelimit disable   disable a ratelimit quota\n";
-  cout << "  realm create               create a new realm\n";
-  cout << "  realm rm                   remove a realm\n";
-  cout << "  realm get                  show realm info\n";
-  cout << "  realm get-default          get default realm name\n";
-  cout << "  realm list                 list realms\n";
-  cout << "  realm list-periods         list all realm periods\n";
-  cout << "  realm rename               rename a realm\n";
-  cout << "  realm set                  set realm info (requires infile)\n";
-  cout << "  realm default              set realm as default\n";
-  cout << "  realm pull                 pull a realm and its current period\n";
-  cout << "  zonegroup add              add a zone to a zonegroup\n";
-  cout << "  zonegroup create           create a new zone group info\n";
-  cout << "  zonegroup default          set default zone group\n";
-  cout << "  zonegroup delete           delete a zone group info\n";
-  cout << "  zonegroup get              show zone group info\n";
-  cout << "  zonegroup modify           modify an existing zonegroup\n";
-  cout << "  zonegroup set              set zone group info (requires infile)\n";
-  cout << "  zonegroup rm               remove a zone from a zonegroup\n";
-  cout << "  zonegroup rename           rename a zone group\n";
-  cout << "  zonegroup list             list all zone groups set on this cluster\n";
-  cout << "  zonegroup placement list   list zonegroup's placement targets\n";
-  cout << "  zonegroup placement get    get a placement target of a specific zonegroup\n";
-  cout << "  zonegroup placement add    add a placement target id to a zonegroup\n";
-  cout << "  zonegroup placement modify modify a placement target of a specific zonegroup\n";
-  cout << "  zonegroup placement rm     remove a placement target from a zonegroup\n";
-  cout << "  zonegroup placement default  set a zonegroup's default placement target\n";
-  cout << "  zone create                create a new zone\n";
-  cout << "  zone rm                    remove a zone\n";
-  cout << "  zone get                   show zone cluster params\n";
-  cout << "  zone modify                modify an existing zone\n";
-  cout << "  zone set                   set zone cluster params (requires infile)\n";
-  cout << "  zone list                  list all zones set on this cluster\n";
-  cout << "  zone rename                rename a zone\n";
-  cout << "  zone placement list        list zone's placement targets\n";
-  cout << "  zone placement get         get a zone placement target\n";
-  cout << "  zone placement add         add a zone placement target\n";
-  cout << "  zone placement modify      modify a zone placement target\n";
-  cout << "  zone placement rm          remove a zone placement target\n";
-  cout << "  metadata sync status       get metadata sync status\n";
-  cout << "  metadata sync init         init metadata sync\n";
-  cout << "  metadata sync run          run metadata sync\n";
-  cout << "  data sync status           get data sync status of the specified source zone\n";
-  cout << "  data sync init             init data sync for the specified source zone\n";
-  cout << "  data sync run              run data sync for the specified source zone\n";
-  cout << "  pool add                   add an existing pool for data placement\n";
-  cout << "  pool rm                    remove an existing pool from data placement set\n";
-  cout << "  pools list                 list placement active set\n";
-  cout << "  policy                     read bucket/object policy\n";
-  cout << "  log list                   list log objects\n";
-  cout << "  log show                   dump a log from specific object or (bucket + date\n";
-  cout << "                             + bucket-id)\n";
-  cout << "                             (NOTE: required to specify formatting of date\n";
-  cout << "                             to \"YYYY-MM-DD-hh\")\n";
-  cout << "  log rm                     remove log object\n";
-  cout << "  usage show                 show usage (by user, by bucket, date range)\n";
-  cout << "  usage trim                 trim usage (by user, by bucket, date range)\n";
-  cout << "  usage clear                reset all the usage stats for the cluster\n";
-  cout << "  gc list                    dump expired garbage collection objects (specify\n";
-  cout << "                             --include-all to list all entries, including unexpired)\n";
-  cout << "  gc process                 manually process garbage (specify\n";
-  cout << "                             --include-all to process all entries, including unexpired)\n";
-  cout << "  lc list                    list all bucket lifecycle progress\n";
-  cout << "  lc get                     get a lifecycle bucket configuration\n";
-  cout << "  lc process                 manually process lifecycle\n";
-  cout << "  lc reshard fix             fix LC for a resharded bucket\n";
-  cout << "  metadata get               get metadata info\n";
-  cout << "  metadata put               put metadata info\n";
-  cout << "  metadata rm                remove metadata info\n";
-  cout << "  metadata list              list metadata info\n";
-  cout << "  mdlog list                 list metadata log\n";
-  cout << "  mdlog autotrim             auto trim metadata log\n";
-  cout << "  mdlog trim                 trim metadata log (use marker)\n";
-  cout << "  mdlog status               read metadata log status\n";
-  cout << "  bilog list                 list bucket index log\n";
-  cout << "  bilog trim                 trim bucket index log (use start-marker, end-marker)\n";
-  cout << "  bilog status               read bucket index log status\n";
-  cout << "  bilog autotrim             auto trim bucket index log\n";
-  cout << "  datalog list               list data log\n";
-  cout << "  datalog trim               trim data log\n";
-  cout << "  datalog status             read data log status\n";
-  cout << "  datalog type               change datalog type to --log_type={fifo,omap}\n";
-  cout << "  orphans find               deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
-  cout << "  orphans finish             deprecated -- clean up search for leaked rados objects\n";
-  cout << "  orphans list-jobs          deprecated -- list the current job-ids for orphans search\n";
-  cout << "                           * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n";
-  cout << "  role create                create a AWS role for use with STS\n";
-  cout << "  role delete                remove a role\n";
-  cout << "  role get                   get a role\n";
-  cout << "  role list                  list roles with specified path prefix\n";
-  cout << "  role-trust-policy modify   modify the assume role policy of an existing role\n";
-  cout << "  role-policy put            add/update permission policy to role\n";
-  cout << "  role-policy list           list policies attached to a role\n";
-  cout << "  role-policy get            get the specified inline policy document embedded with the given role\n";
-  cout << "  role-policy delete         remove policy attached to a role\n";
-  cout << "  role update                update max_session_duration of a role\n";
-  cout << "  reshard add                schedule a resharding of a bucket\n";
-  cout << "  reshard list               list all bucket resharding or scheduled to be resharded\n";
-  cout << "  reshard status             read bucket resharding status\n";
-  cout << "  reshard process            process of scheduled reshard jobs\n";
-  cout << "  reshard cancel             cancel resharding a bucket\n";
-  cout << "  reshard stale-instances list list stale-instances from bucket resharding\n";
+  cout << "  user create                      create a new user\n" ;
+  cout << "  user modify                      modify user\n";
+  cout << "  user info                        get user info\n";
+  cout << "  user rename                      rename user\n";
+  cout << "  user rm                          remove user\n";
+  cout << "  user suspend                     suspend a user\n";
+  cout << "  user enable                      re-enable user after suspension\n";
+  cout << "  user check                       check user info\n";
+  cout << "  user stats                       show user stats as accounted by quota subsystem\n";
+  cout << "  user list                        list users\n";
+  cout << "  caps add                         add user capabilities\n";
+  cout << "  caps rm                          remove user capabilities\n";
+  cout << "  subuser create                   create a new subuser\n" ;
+  cout << "  subuser modify                   modify subuser\n";
+  cout << "  subuser rm                       remove subuser\n";
+  cout << "  key create                       create access key\n";
+  cout << "  key rm                           remove access key\n";
+  cout << "  bucket list                      list buckets (specify --allow-unordered for faster, unsorted listing)\n";
+  cout << "  bucket limit check               show bucket sharding stats\n";
+  cout << "  bucket link                      link bucket to specified user\n";
+  cout << "  bucket unlink                    unlink bucket from specified user\n";
+  cout << "  bucket stats                     returns bucket statistics\n";
+  cout << "  bucket rm                        remove bucket\n";
+  cout << "  bucket check                     check bucket index by verifying size and object count stats\n";
+  cout << "  bucket check olh                 check for olh index entries and objects that are pending removal\n";
+  cout << "  bucket check unlinked            check for object versions that are not visible in a bucket listing \n";
+  cout << "  bucket chown                     link bucket to specified user and update its object ACLs\n";
+  cout << "  bucket reshard                   reshard bucket\n";
+  cout << "  bucket rewrite                   rewrite all objects in the specified bucket\n";
+  cout << "  bucket sync checkpoint           poll a bucket's sync status until it catches up to its remote\n";
+  cout << "  bucket sync disable              disable bucket sync\n";
+  cout << "  bucket sync enable               enable bucket sync\n";
+  cout << "  bucket radoslist                 list rados objects backing bucket's objects\n";
+  cout << "  bi get                           retrieve bucket index object entries\n";
+  cout << "  bi put                           store bucket index object entries\n";
+  cout << "  bi list                          list raw bucket index entries\n";
+  cout << "  bi purge                         purge bucket index entries\n";
+  cout << "  object rm                        remove object\n";
+  cout << "  object put                       put object\n";
+  cout << "  object stat                      stat an object for its metadata\n";
+  cout << "  object unlink                    unlink object from bucket index\n";
+  cout << "  object rewrite                   rewrite the specified object\n";
+  cout << "  object reindex                   reindex the object(s) indicated by --bucket and either --object or --objects-file\n";
+  cout << "  objects expire                   run expired objects cleanup\n";
+  cout << "  objects expire-stale list        list stale expired objects (caused by reshard)\n";
+  cout << "  objects expire-stale rm          remove stale expired objects\n";
+  cout << "  period rm                        remove a period\n";
+  cout << "  period get                       get period info\n";
+  cout << "  period get-current               get current period info\n";
+  cout << "  period pull                      pull a period\n";
+  cout << "  period push                      push a period\n";
+  cout << "  period list                      list all periods\n";
+  cout << "  period update                    update the staging period\n";
+  cout << "  period commit                    commit the staging period\n";
+  cout << "  quota set                        set quota params\n";
+  cout << "  quota enable                     enable quota\n";
+  cout << "  quota disable                    disable quota\n";
+  cout << "  ratelimit get                    get ratelimit params\n";
+  cout << "  ratelimit set                    set ratelimit params\n";
+  cout << "  ratelimit enable                 enable ratelimit\n";
+  cout << "  ratelimit disable                disable ratelimit\n";
+  cout << "  global quota get                 view global quota params\n";
+  cout << "  global quota set                 set global quota params\n";
+  cout << "  global quota enable              enable a global quota\n";
+  cout << "  global quota disable             disable a global quota\n";
+  cout << "  global ratelimit get             view global ratelimit params\n";
+  cout << "  global ratelimit set             set global ratelimit params\n";
+  cout << "  global ratelimit enable          enable a ratelimit quota\n";
+  cout << "  global ratelimit disable         disable a ratelimit quota\n";
+  cout << "  realm create                     create a new realm\n";
+  cout << "  realm rm                         remove a realm\n";
+  cout << "  realm get                        show realm info\n";
+  cout << "  realm get-default                get default realm name\n";
+  cout << "  realm list                       list realms\n";
+  cout << "  realm list-periods               list all realm periods\n";
+  cout << "  realm rename                     rename a realm\n";
+  cout << "  realm set                        set realm info (requires infile)\n";
+  cout << "  realm default                    set realm as default\n";
+  cout << "  realm pull                       pull a realm and its current period\n";
+  cout << "  zonegroup add                    add a zone to a zonegroup\n";
+  cout << "  zonegroup create                 create a new zone group info\n";
+  cout << "  zonegroup default                set default zone group\n";
+  cout << "  zonegroup delete                 delete a zone group info\n";
+  cout << "  zonegroup get                    show zone group info\n";
+  cout << "  zonegroup modify                 modify an existing zonegroup\n";
+  cout << "  zonegroup set                    set zone group info (requires infile)\n";
+  cout << "  zonegroup rm                     remove a zone from a zonegroup\n";
+  cout << "  zonegroup rename                 rename a zone group\n";
+  cout << "  zonegroup list                   list all zone groups set on this cluster\n";
+  cout << "  zonegroup placement list         list zonegroup's placement targets\n";
+  cout << "  zonegroup placement get          get a placement target of a specific zonegroup\n";
+  cout << "  zonegroup placement add          add a placement target id to a zonegroup\n";
+  cout << "  zonegroup placement modify       modify a placement target of a specific zonegroup\n";
+  cout << "  zonegroup placement rm           remove a placement target from a zonegroup\n";
+  cout << "  zonegroup placement default      set a zonegroup's default placement target\n";
+  cout << "  zone create                      create a new zone\n";
+  cout << "  zone rm                          remove a zone\n";
+  cout << "  zone get                         show zone cluster params\n";
+  cout << "  zone modify                      modify an existing zone\n";
+  cout << "  zone set                         set zone cluster params (requires infile)\n";
+  cout << "  zone list                        list all zones set on this cluster\n";
+  cout << "  zone rename                      rename a zone\n";
+  cout << "  zone placement list              list zone's placement targets\n";
+  cout << "  zone placement get               get a zone placement target\n";
+  cout << "  zone placement add               add a zone placement target\n";
+  cout << "  zone placement modify            modify a zone placement target\n";
+  cout << "  zone placement rm                remove a zone placement target\n";
+  cout << "  metadata sync status             get metadata sync status\n";
+  cout << "  metadata sync init               init metadata sync\n";
+  cout << "  metadata sync run                run metadata sync\n";
+  cout << "  data sync status                 get data sync status of the specified source zone\n";
+  cout << "  data sync init                   init data sync for the specified source zone\n";
+  cout << "  data sync run                    run data sync for the specified source zone\n";
+  cout << "  pool add                         add an existing pool for data placement\n";
+  cout << "  pool rm                          remove an existing pool from data placement set\n";
+  cout << "  pools list                       list placement active set\n";
+  cout << "  policy                           read bucket/object policy\n";
+  cout << "  log list                         list log objects\n";
+  cout << "  log show                         dump a log from specific object or (bucket + date + bucket-id)\n";
+  cout << "                                   (NOTE: required to specify formatting of date to \"YYYY-MM-DD-hh\")\n";
+  cout << "  log rm                           remove log object\n";
+  cout << "  usage show                       show usage (by user, by bucket, date range)\n";
+  cout << "  usage trim                       trim usage (by user, by bucket, date range)\n";
+  cout << "  usage clear                      reset all the usage stats for the cluster\n";
+  cout << "  gc list                          dump expired garbage collection objects (specify\n";
+  cout << "                                   --include-all to list all entries, including unexpired)\n";
+  cout << "  gc process                       manually process garbage (specify\n";
+  cout << "                                   --include-all to process all entries, including unexpired)\n";
+  cout << "  lc list                          list all bucket lifecycle progress\n";
+  cout << "  lc get                           get a lifecycle bucket configuration\n";
+  cout << "  lc process                       manually process lifecycle\n";
+  cout << "  lc reshard fix                   fix LC for a resharded bucket\n";
+  cout << "  metadata get                     get metadata info\n";
+  cout << "  metadata put                     put metadata info\n";
+  cout << "  metadata rm                      remove metadata info\n";
+  cout << "  metadata list                    list metadata info\n";
+  cout << "  mdlog list                       list metadata log\n";
+  cout << "  mdlog autotrim                   auto trim metadata log\n";
+  cout << "  mdlog trim                       trim metadata log (use marker)\n";
+  cout << "  mdlog status                     read metadata log status\n";
+  cout << "  bilog list                       list bucket index log\n";
+  cout << "  bilog trim                       trim bucket index log (use start-marker, end-marker)\n";
+  cout << "  bilog status                     read bucket index log status\n";
+  cout << "  bilog autotrim                   auto trim bucket index log\n";
+  cout << "  datalog list                     list data log\n";
+  cout << "  datalog trim                     trim data log\n";
+  cout << "  datalog status                   read data log status\n";
+  cout << "  datalog type                     change datalog type to --log_type={fifo,omap}\n";
+  cout << "  orphans find                     deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
+  cout << "  orphans finish                   deprecated -- clean up search for leaked rados objects\n";
+  cout << "  orphans list-jobs                deprecated -- list the current job-ids for orphans search\n";
+  cout << "    * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n";
+  cout << "  role create                      create a AWS role for use with STS\n";
+  cout << "  role delete                      remove a role\n";
+  cout << "  role get                         get a role\n";
+  cout << "  role list                        list roles with specified path prefix\n";
+  cout << "  role-trust-policy modify         modify the assume role policy of an existing role\n";
+  cout << "  role-policy put                  add/update permission policy to role\n";
+  cout << "  role-policy list                 list policies attached to a role\n";
+  cout << "  role-policy get                  get the specified inline policy document embedded with the given role\n";
+  cout << "  role-policy delete               remove policy attached to a role\n";
+  cout << "  role update                      update max_session_duration of a role\n";
+  cout << "  reshard add                      schedule a resharding of a bucket\n";
+  cout << "  reshard list                     list all bucket resharding or scheduled to be resharded\n";
+  cout << "  reshard status                   read bucket resharding status\n";
+  cout << "  reshard process                  process of scheduled reshard jobs\n";
+  cout << "  reshard cancel                   cancel resharding a bucket\n";
+  cout << "  reshard stale-instances list     list stale-instances from bucket resharding\n";
   cout << "  reshard stale-instances delete   cleanup stale-instances from bucket resharding\n";
-  cout << "  sync error list            list sync error\n";
-  cout << "  sync error trim            trim sync error\n";
-  cout << "  mfa create                 create a new MFA TOTP token\n";
-  cout << "  mfa list                   list MFA TOTP tokens\n";
-  cout << "  mfa get                    show MFA TOTP token\n";
-  cout << "  mfa remove                 delete MFA TOTP token\n";
-  cout << "  mfa check                  check MFA TOTP token\n";
-  cout << "  mfa resync                 re-sync MFA TOTP token\n";
-  cout << "  topic list                 list bucket notifications topics\n";
-  cout << "  topic get                  get a bucket notifications topic\n";
-  cout << "  topic rm                   remove a bucket notifications topic\n";
-  cout << "  topic stats                get a bucket notifications persistent topic stats (i.e. reservations, entries & size)\n";
-  cout << "  script put                 upload a Lua script to a context\n";
-  cout << "  script get                 get the Lua script of a context\n";
-  cout << "  script rm                  remove the Lua scripts of a context\n";
-  cout << "  script-package add         add a Lua package to the scripts allowlist\n";
-  cout << "  script-package rm          remove a Lua package from the scripts allowlist\n";
-  cout << "  script-package list        get the Lua packages allowlist\n";
-  cout << "  script-package reload      install/remove Lua packages according to allowlist\n";
-  cout << "  notification list          list bucket notifications configuration\n";
-  cout << "  notification get           get a bucket notifications configuration\n";
-  cout << "  notification rm            remove a bucket notifications configuration\n";
+  cout << "  sync error list                  list sync error\n";
+  cout << "  sync error trim                  trim sync error\n";
+  cout << "  mfa create                       create a new MFA TOTP token\n";
+  cout << "  mfa list                         list MFA TOTP tokens\n";
+  cout << "  mfa get                          show MFA TOTP token\n";
+  cout << "  mfa remove                       delete MFA TOTP token\n";
+  cout << "  mfa check                        check MFA TOTP token\n";
+  cout << "  mfa resync                       re-sync MFA TOTP token\n";
+  cout << "  topic list                       list bucket notifications topics\n";
+  cout << "  topic get                        get a bucket notifications topic\n";
+  cout << "  topic rm                         remove a bucket notifications topic\n";
+  cout << "  topic stats                      get a bucket notifications persistent topic stats (i.e. reservations, entries & size)\n";
+  cout << "  script put                       upload a Lua script to a context\n";
+  cout << "  script get                       get the Lua script of a context\n";
+  cout << "  script rm                        remove the Lua scripts of a context\n";
+  cout << "  script-package add               add a Lua package to the scripts allowlist\n";
+  cout << "  script-package rm                remove a Lua package from the scripts allowlist\n";
+  cout << "  script-package list              get the Lua packages allowlist\n";
+  cout << "  script-package reload            install/remove Lua packages according to allowlist\n";
+  cout << "  notification list                list bucket notifications configuration\n";
+  cout << "  notification get                 get a bucket notifications configuration\n";
+  cout << "  notification rm                  remove a bucket notifications configuration\n";
   cout << "options:\n";
-  cout << "   --tenant=<tenant>         tenant name\n";
-  cout << "   --user_ns=<namespace>     namespace of user (oidc in case of users authenticated with oidc provider)\n";
-  cout << "   --uid=<id>                user id\n";
-  cout << "   --new-uid=<id>            new user id\n";
-  cout << "   --subuser=<name>          subuser name\n";
-  cout << "   --access-key=<key>        S3 access key\n";
-  cout << "   --email=<email>           user's email address\n";
-  cout << "   --secret/--secret-key=<key>\n";
-  cout << "                             specify secret key\n";
-  cout << "   --gen-access-key          generate random access key (for S3)\n";
-  cout << "   --gen-secret              generate random secret key\n";
-  cout << "   --key-type=<type>         key type, options are: swift, s3\n";
-  cout << "   --temp-url-key[-2]=<key>  temp url key\n";
-  cout << "   --access=<access>         Set access permissions for sub-user, should be one\n";
-  cout << "                             of read, write, readwrite, full\n";
-  cout << "   --display-name=<name>     user's display name\n";
-  cout << "   --max-buckets             max number of buckets for a user\n";
-  cout << "   --admin                   set the admin flag on the user\n";
-  cout << "   --system                  set the system flag on the user\n";
-  cout << "   --op-mask                 set the op mask on the user\n";
-  cout << "   --bucket=<bucket>         Specify the bucket name. Also used by the quota command.\n";
-  cout << "   --pool=<pool>             Specify the pool name. Also used to scan for leaked rados objects.\n";
-  cout << "   --object=<object>         object name\n";
-  cout << "   --objects-file=<file>     file containing a list of object names to process\n";
-  cout << "   --object-version=<version>         object version\n";
-  cout << "   --date=<date>             date in the format yyyy-mm-dd\n";
-  cout << "   --start-date=<date>       start date in the format yyyy-mm-dd\n";
-  cout << "   --end-date=<date>         end date in the format yyyy-mm-dd\n";
-  cout << "   --bucket-id=<bucket-id>   bucket id\n";
-  cout << "   --bucket-new-name=<bucket>\n";
-  cout << "                             for bucket link: optional new name\n";
-  cout << "   --shard-id=<shard-id>     optional for: \n";
-  cout << "                               mdlog list\n";
-  cout << "                               data sync status\n";
-  cout << "                             required for: \n";
-  cout << "                               mdlog trim\n";
-  cout << "   --gen=<gen-id>            optional for: \n";
-  cout << "                               bilog list\n";
-  cout << "                               bilog trim\n";
-  cout << "                               bilog status\n";
-  cout << "   --max-entries=<entries>   max entries for listing operations\n";
-  cout << "   --metadata-key=<key>      key to retrieve metadata from with metadata get\n";
-  cout << "   --remote=<remote>         zone or zonegroup id of remote gateway\n";
-  cout << "   --period=<id>             period id\n";
-  cout << "   --url=<url>               url for pushing/pulling period/realm\n";
-  cout << "   --epoch=<number>          period epoch\n";
-  cout << "   --commit                  commit the period during 'period update'\n";
-  cout << "   --staging                 get staging period info\n";
-  cout << "   --master                  set as master\n";
-  cout << "   --master-zone=<id>        master zone id\n";
-  cout << "   --rgw-realm=<name>        realm name\n";
-  cout << "   --realm-id=<id>           realm id\n";
-  cout << "   --realm-new-name=<name>   realm new name\n";
-  cout << "   --rgw-zonegroup=<name>    zonegroup name\n";
-  cout << "   --zonegroup-id=<id>       zonegroup id\n";
-  cout << "   --zonegroup-new-name=<name>\n";
-  cout << "                             zonegroup new name\n";
-  cout << "   --rgw-zone=<name>         name of zone in which radosgw is running\n";
-  cout << "   --zone-id=<id>            zone id\n";
-  cout << "   --zone-new-name=<name>    zone new name\n";
-  cout << "   --source-zone             specify the source zone (for data sync)\n";
-  cout << "   --default                 set entity (realm, zonegroup, zone) as default\n";
-  cout << "   --read-only               set zone as read-only (when adding to zonegroup)\n";
-  cout << "   --redirect-zone           specify zone id to redirect when response is 404 (not found)\n";
-  cout << "   --placement-id            placement id for zonegroup placement commands\n";
-  cout << "   --storage-class           storage class for zonegroup placement commands\n";
-  cout << "   --tags=<list>             list of tags for zonegroup placement add and modify commands\n";
-  cout << "   --tags-add=<list>         list of tags to add for zonegroup placement modify command\n";
-  cout << "   --tags-rm=<list>          list of tags to remove for zonegroup placement modify command\n";
-  cout << "   --endpoints=<list>        zone endpoints\n";
-  cout << "   --index-pool=<pool>       placement target index pool\n";
-  cout << "   --data-pool=<pool>        placement target data pool\n";
-  cout << "   --data-extra-pool=<pool>  placement target data extra (non-ec) pool\n";
-  cout << "   --placement-index-type=<type>\n";
-  cout << "                             placement target index type (normal, indexless, or #id)\n";
-  cout << "   --placement-inline-data=<true>\n";
-  cout << "                             set whether the placement target is configured to store a data\n";
-  cout << "                             chunk inline in head objects\n";
-  cout << "   --compression=<type>      placement target compression type (plugin name or empty/none)\n";
-  cout << "   --tier-type=<type>        zone tier type\n";
-  cout << "   --tier-config=<k>=<v>[,...]\n";
-  cout << "                             set zone tier config keys, values\n";
-  cout << "   --tier-config-rm=<k>[,...]\n";
-  cout << "                             unset zone tier config keys\n";
-  cout << "   --sync-from-all[=false]   set/reset whether zone syncs from all zonegroup peers\n";
-  cout << "   --sync-from=[zone-name][,...]\n";
-  cout << "                             set list of zones to sync from\n";
-  cout << "   --sync-from-rm=[zone-name][,...]\n";
-  cout << "                             remove zones from list of zones to sync from\n";
-  cout << "   --bucket-index-max-shards override a zone/zonegroup's default bucket index shard count\n";
-  cout << "   --fix                     besides checking bucket index, will also fix it\n";
-  cout << "   --check-objects           bucket check: rebuilds bucket index according to\n";
-  cout << "                             actual objects state\n";
-  cout << "   --format=<format>         specify output format for certain operations: xml,\n";
-  cout << "                             json\n";
-  cout << "   --purge-data              when specified, user removal will also purge all the\n";
-  cout << "                             user data\n";
-  cout << "   --purge-keys              when specified, subuser removal will also purge all the\n";
-  cout << "                             subuser keys\n";
-  cout << "   --purge-objects           remove a bucket's objects before deleting it\n";
-  cout << "                             (NOTE: required to delete a non-empty bucket)\n";
-  cout << "   --sync-stats              option to 'user stats', update user stats with current\n";
-  cout << "                             stats reported by user's buckets indexes\n";
-  cout << "   --reset-stats             option to 'user stats', reset stats in accordance with user buckets\n";
-  cout << "   --show-config             show configuration\n";
-  cout << "   --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
-  cout << "   --show-log-sum=<flag>     enable/disable dump of log summation on log show\n";
-  cout << "   --skip-zero-entries       log show only dumps entries that don't have zero value\n";
-  cout << "                             in one of the numeric field\n";
-  cout << "   --infile=<file>           file to read in when setting data\n";
-  cout << "   --categories=<list>       comma separated list of categories, used in usage show\n";
-  cout << "   --caps=<caps>             list of caps (e.g., \"usage=read, write; user=read\")\n";
-  cout << "   --op-mask=<op-mask>       permission of user's operations (e.g., \"read, write, delete, *\")\n";
-  cout << "   --yes-i-really-mean-it    required for certain operations\n";
-  cout << "   --warnings-only           when specified with bucket limit check, list\n";
-  cout << "                             only buckets nearing or over the current max\n";
-  cout << "                             objects per shard value\n";
-  cout << "   --bypass-gc               when specified with bucket deletion, triggers\n";
-  cout << "                             object deletions by not involving GC\n";
-  cout << "   --inconsistent-index      when specified with bucket deletion and bypass-gc set to true,\n";
-  cout << "                             ignores bucket index consistency\n";
-  cout << "   --min-rewrite-size        min object size for bucket rewrite (default 4M)\n";
-  cout << "   --max-rewrite-size        max object size for bucket rewrite (default ULLONG_MAX)\n";
-  cout << "   --min-rewrite-stripe-size min stripe size for object rewrite (default 0)\n";
-  cout << "   --trim-delay-ms           time interval in msec to limit the frequency of sync error log entries trimming operations,\n";
-  cout << "                             the trimming process will sleep the specified msec for every 1000 entries trimmed\n";
-  cout << "   --max-concurrent-ios      maximum concurrent ios for bucket operations (default: 32)\n";
-  cout << "   --enable-feature          enable a zone/zonegroup feature\n";
-  cout << "   --disable-feature         disable a zone/zonegroup feature\n";
+  cout << "   --tenant=<tenant>                 tenant name\n";
+  cout << "   --user_ns=<namespace>             namespace of user (oidc in case of users authenticated with oidc provider)\n";
+  cout << "   --uid=<id>                        user id\n";
+  cout << "   --new-uid=<id>                    new user id\n";
+  cout << "   --subuser=<name>                  subuser name\n";
+  cout << "   --access-key=<key>                S3 access key\n";
+  cout << "   --email=<email>                   user's email address\n";
+  cout << "   --secret/--secret-key=<key>       specify secret key\n";
+  cout << "   --gen-access-key                  generate random access key (for S3)\n";
+  cout << "   --gen-secret                      generate random secret key\n";
+  cout << "   --key-type=<type>                 key type, options are: swift, s3\n";
+  cout << "   --temp-url-key[-2]=<key>          temp url key\n";
+  cout << "   --access=<access>                 Set access permissions for sub-user, should be one\n";
+  cout << "                                     of read, write, readwrite, full\n";
+  cout << "   --display-name=<name>             user's display name\n";
+  cout << "   --max-buckets                     max number of buckets for a user\n";
+  cout << "   --admin                           set the admin flag on the user\n";
+  cout << "   --system                          set the system flag on the user\n";
+  cout << "   --op-mask                         set the op mask on the user\n";
+  cout << "   --bucket=<bucket>                 Specify the bucket name. Also used by the quota command.\n";
+  cout << "   --pool=<pool>                     Specify the pool name. Also used to scan for leaked rados objects.\n";
+  cout << "   --object=<object>                 object name\n";
+  cout << "   --objects-file=<file>             file containing a list of object names to process\n";
+  cout << "   --object-version=<version>        object version\n";
+  cout << "   --date=<date>                     date in the format yyyy-mm-dd\n";
+  cout << "   --start-date=<date>               start date in the format yyyy-mm-dd\n";
+  cout << "   --end-date=<date>                 end date in the format yyyy-mm-dd\n";
+  cout << "   --bucket-id=<bucket-id>           bucket id\n";
+  cout << "   --bucket-new-name=<bucket>        for bucket link: optional new name\n";
+  cout << "   --shard-id=<shard-id>             optional for:\n";
+  cout << "                                       mdlog list\n";
+  cout << "                                       data sync status\n";
+  cout << "                                     required for:\n";
+  cout << "                                       mdlog trim\n";
+  cout << "   --gen=<gen-id>                    optional for:\n";
+  cout << "                                       bilog list\n";
+  cout << "                                       bilog trim\n";
+  cout << "                                       bilog status\n";
+  cout << "   --max-entries=<entries>           max entries for listing operations\n";
+  cout << "   --metadata-key=<key>              key to retrieve metadata from with metadata get\n";
+  cout << "   --remote=<remote>                 zone or zonegroup id of remote gateway\n";
+  cout << "   --period=<id>                     period id\n";
+  cout << "   --url=<url>                       url for pushing/pulling period/realm\n";
+  cout << "   --epoch=<number>                  period epoch\n";
+  cout << "   --commit                          commit the period during 'period update'\n";
+  cout << "   --staging                         get staging period info\n";
+  cout << "   --master                          set as master\n";
+  cout << "   --master-zone=<id>                master zone id\n";
+  cout << "   --rgw-realm=<name>                realm name\n";
+  cout << "   --realm-id=<id>                   realm id\n";
+  cout << "   --realm-new-name=<name>           realm new name\n";
+  cout << "   --rgw-zonegroup=<name>            zonegroup name\n";
+  cout << "   --zonegroup-id=<id>               zonegroup id\n";
+  cout << "   --zonegroup-new-name=<name>       zonegroup new name\n";
+  cout << "   --rgw-zone=<name>                 name of zone in which radosgw is running\n";
+  cout << "   --zone-id=<id>                    zone id\n";
+  cout << "   --zone-new-name=<name>            zone new name\n";
+  cout << "   --source-zone                     specify the source zone (for data sync)\n";
+  cout << "   --default                         set entity (realm, zonegroup, zone) as default\n";
+  cout << "   --read-only                       set zone as read-only (when adding to zonegroup)\n";
+  cout << "   --redirect-zone                   specify zone id to redirect when response is 404 (not found)\n";
+  cout << "   --placement-id                    placement id for zonegroup placement commands\n";
+  cout << "   --storage-class                   storage class for zonegroup placement commands\n";
+  cout << "   --tags=<list>                     list of tags for zonegroup placement add and modify commands\n";
+  cout << "   --tags-add=<list>                 list of tags to add for zonegroup placement modify command\n";
+  cout << "   --tags-rm=<list>                  list of tags to remove for zonegroup placement modify command\n";
+  cout << "   --endpoints=<list>                zone endpoints\n";
+  cout << "   --index-pool=<pool>               placement target index pool\n";
+  cout << "   --data-pool=<pool>                placement target data pool\n";
+  cout << "   --data-extra-pool=<pool>          placement target data extra (non-ec) pool\n";
+  cout << "   --placement-index-type=<type>     placement target index type (normal, indexless, or #id)\n";
+  cout << "   --placement-inline-data=<true>    set whether the placement target is configured to store a data\n";
+  cout << "                                     chunk inline in head objects\n";
+  cout << "   --compression=<type>              placement target compression type (plugin name or empty/none)\n";
+  cout << "   --tier-type=<type>                zone tier type\n";
+  cout << "   --tier-config=<k>=<v>[,...]       set zone tier config keys, values\n";
+  cout << "   --tier-config-rm=<k>[,...]        unset zone tier config keys\n";
+  cout << "   --sync-from-all[=false]           set/reset whether zone syncs from all zonegroup peers\n";
+  cout << "   --sync-from=[zone-name][,...]     set list of zones to sync from\n";
+  cout << "   --sync-from-rm=[zone-name][,...]  remove zones from list of zones to sync from\n";
+  cout << "   --bucket-index-max-shards         override a zone/zonegroup's default bucket index shard count\n";
+  cout << "   --fix                             besides checking bucket index, will also fix it\n";
+  cout << "   --check-objects                   bucket check: rebuilds bucket index according to actual objects state\n";
+  cout << "   --format=<format>                 specify output format for certain operations: xml, json\n";
+  cout << "   --purge-data                      when specified, user removal will also purge all the\n";
+  cout << "                                     user data\n";
+  cout << "   --purge-keys                      when specified, subuser removal will also purge all the\n";
+  cout << "                                     subuser keys\n";
+  cout << "   --purge-objects                   remove a bucket's objects before deleting it\n";
+  cout << "                                     (NOTE: required to delete a non-empty bucket)\n";
+  cout << "   --sync-stats                      option to 'user stats', update user stats with current\n";
+  cout << "                                     stats reported by user's buckets indexes\n";
+  cout << "   --reset-stats                     option to 'user stats', reset stats in accordance with user buckets\n";
+  cout << "   --show-config                     show configuration\n";
+  cout << "   --show-log-entries=<flag>         enable/disable dump of log entries on log show\n";
+  cout << "   --show-log-sum=<flag>             enable/disable dump of log summation on log show\n";
+  cout << "   --skip-zero-entries               log show only dumps entries that don't have zero value\n";
+  cout << "                                     in one of the numeric field\n";
+  cout << "   --infile=<file>                   file to read in when setting data\n";
+  cout << "   --categories=<list>               comma separated list of categories, used in usage show\n";
+  cout << "   --caps=<caps>                     list of caps (e.g., \"usage=read, write; user=read\")\n";
+  cout << "   --op-mask=<op-mask>               permission of user's operations (e.g., \"read, write, delete, *\")\n";
+  cout << "   --yes-i-really-mean-it            required for certain operations\n";
+  cout << "   --warnings-only                   when specified with bucket limit check, list\n";
+  cout << "                                     only buckets nearing or over the current max\n";
+  cout << "                                     objects per shard value\n";
+  cout << "   --bypass-gc                       when specified with bucket deletion, triggers\n";
+  cout << "                                     object deletions by not involving GC\n";
+  cout << "   --inconsistent-index              when specified with bucket deletion and bypass-gc set to true,\n";
+  cout << "                                     ignores bucket index consistency\n";
+  cout << "   --min-rewrite-size                min object size for bucket rewrite (default 4M)\n";
+  cout << "   --max-rewrite-size                max object size for bucket rewrite (default ULLONG_MAX)\n";
+  cout << "   --min-rewrite-stripe-size         min stripe size for object rewrite (default 0)\n";
+  cout << "   --trim-delay-ms                   time interval in msec to limit the frequency of sync error log entries trimming operations,\n";
+  cout << "                                     the trimming process will sleep the specified msec for every 1000 entries trimmed\n";
+  cout << "   --max-concurrent-ios              maximum concurrent ios for bucket operations (default: 32)\n";
+  cout << "   --enable-feature                  enable a zone/zonegroup feature\n";
+  cout << "   --disable-feature                 disable a zone/zonegroup feature\n";
   cout << "\n";
   cout << "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
   cout << "\nQuota options:\n";
-  cout << "   --max-objects             specify max objects (negative value to disable)\n";
-  cout << "   --max-size                specify max size (in B/K/M/G/T, negative value to disable)\n";
-  cout << "   --quota-scope             scope of quota (bucket, user)\n";
+  cout << "   --max-objects                 specify max objects (negative value to disable)\n";
+  cout << "   --max-size                    specify max size (in B/K/M/G/T, negative value to disable)\n";
+  cout << "   --quota-scope                 scope of quota (bucket, user)\n";
   cout << "\nRate limiting options:\n";
-  cout << "   --max-read-ops            specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
-  cout << "   --max-read-bytes          specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
-  cout << "   --max-write-ops           specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
-  cout << "   --max-write-bytes         specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
-  cout << "   --ratelimit-scope         scope of rate limiting: bucket, user, anonymous\n";
-  cout << "                             anonymous can be configured only with global rate limit\n";
+  cout << "   --max-read-ops                specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
+  cout << "   --max-read-bytes              specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
+  cout << "   --max-write-ops               specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
+  cout << "   --max-write-bytes             specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
+  cout << "   --ratelimit-scope             scope of rate limiting: bucket, user, anonymous\n";
+  cout << "                                 anonymous can be configured only with global rate limit\n";
   cout << "\nOrphans search options:\n";
-  cout << "   --num-shards              num of shards to use for keeping the temporary scan info\n";
-  cout << "   --orphan-stale-secs       num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
-  cout << "   --job-id                  set the job id (for orphans find)\n";
-  cout << "   --detail                  detailed mode, log and stat head objects as well\n";
+  cout << "   --num-shards                  num of shards to use for keeping the temporary scan info\n";
+  cout << "   --orphan-stale-secs           num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
+  cout << "   --job-id                      set the job id (for orphans find)\n";
+  cout << "   --detail                      detailed mode, log and stat head objects as well\n";
   cout << "\nOrphans list-jobs options:\n";
-  cout << "   --extra-info              provide extra info in job list\n";
+  cout << "   --extra-info                  provide extra info in job list\n";
   cout << "\nRole options:\n";
-  cout << "   --role-name               name of the role to create\n";
-  cout << "   --path                    path to the role\n";
-  cout << "   --assume-role-policy-doc  the trust relationship policy document that grants an entity permission to assume the role\n";
-  cout << "   --policy-name             name of the policy document\n";
-  cout << "   --policy-doc              permission policy document\n";
-  cout << "   --path-prefix             path prefix for filtering roles\n";
+  cout << "   --role-name                   name of the role to create\n";
+  cout << "   --path                        path to the role\n";
+  cout << "   --assume-role-policy-doc      the trust relationship policy document that grants an entity permission to assume the role\n";
+  cout << "   --policy-name                 name of the policy document\n";
+  cout << "   --policy-doc                  permission policy document\n";
+  cout << "   --path-prefix                 path prefix for filtering roles\n";
   cout << "\nMFA options:\n";
-  cout << "   --totp-serial             a string that represents the ID of a TOTP token\n";
-  cout << "   --totp-seed               the secret seed that is used to calculate the TOTP\n";
-  cout << "   --totp-seconds            the time resolution that is being used for TOTP generation\n";
-  cout << "   --totp-window             the number of TOTP tokens that are checked before and after the current token when validating token\n";
-  cout << "   --totp-pin                the valid value of a TOTP token at a certain time\n";
+  cout << "   --totp-serial                 a string that represents the ID of a TOTP token\n";
+  cout << "   --totp-seed                   the secret seed that is used to calculate the TOTP\n";
+  cout << "   --totp-seconds                the time resolution that is being used for TOTP generation\n";
+  cout << "   --totp-window                 the number of TOTP tokens that are checked before and after the current token when validating token\n";
+  cout << "   --totp-pin                    the valid value of a TOTP token at a certain time\n";
   cout << "\nBucket notifications options:\n";
-  cout << "   --topic                   bucket notifications topic name\n";
-  cout << "   --notification-id         bucket notifications id\n";
+  cout << "   --topic                       bucket notifications topic name\n";
+  cout << "   --notification-id             bucket notifications id\n";
   cout << "\nScript options:\n";
-  cout << "   --context                 context in which the script runs. one of: "+LUA_CONTEXT_LIST+"\n";
-  cout << "   --package                 name of the Lua package that should be added/removed to/from the allowlist\n";
-  cout << "   --allow-compilation       package is allowed to compile C code as part of its installation\n";
+  cout << "   --context                     context in which the script runs. one of: "+LUA_CONTEXT_LIST+"\n";
+  cout << "   --package                     name of the Lua package that should be added/removed to/from the allowlist\n";
+  cout << "   --allow-compilation           package is allowed to compile C code as part of its installation\n";
   cout << "\nBucket check olh/unlinked options:\n";
-  cout << "   --min-age-hours           minimum age of unlinked objects to consider for bucket check unlinked (default: 1)\n";
-  cout << "   --dump-keys               when specified, all keys identified as problematic are printed to stdout\n";
-  cout << "   --hide-progress           when specified, per-shard progress details are not printed to stderr\n";
+  cout << "   --min-age-hours               minimum age of unlinked objects to consider for bucket check unlinked (default: 1)\n";
+  cout << "   --dump-keys                   when specified, all keys identified as problematic are printed to stdout\n";
+  cout << "   --hide-progress               when specified, per-shard progress details are not printed to stderr\n";
   cout << "\nradoslist options:\n";
-  cout << "   --rgw-obj-fs              the field separator that will separate the rados\n";
-  cout << "                             object name from the rgw object name;\n";
-  cout << "                             additionally rados objects for incomplete\n";
-  cout << "                             multipart uploads will not be output\n";
+  cout << "   --rgw-obj-fs                  the field separator that will separate the rados object name from the rgw object name;\n";
+  cout << "                                 additionally rados objects for incomplete multipart uploads will not be output\n";
   cout << "\n";
   generic_client_usage();
 }
@@ -2402,7 +2386,7 @@ static void get_data_sync_status(const rgw_zone_id& source_zone, list<string>& s
     push_ss(ss, status, tab) << "data is caught up with source";
   } else if (total_behind > 0) {
     push_ss(ss, status, tab) << "data is behind on " << total_behind << " shards";
-    push_ss(ss, status, tab) << "behind shards: " << "[" << shards_behind_set << "]" ;
+    push_ss(ss, status, tab) << "behind shards: " << "[" << shards_behind_set << "]";
     if (oldest) {
       push_ss(ss, status, tab) << "oldest incremental change not applied: "
           << oldest->second << " [" << oldest->first << ']';
@@ -2611,7 +2595,7 @@ static int bucket_source_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Ra
   }
   if (!shards_behind.empty()) {
     out << indented{width} << "bucket is behind on " << shards_behind.size() << " shards\n";
-    out << indented{width} << "behind shards: [" << shards_behind << "]\n" ;
+    out << indented{width} << "behind shards: [" << shards_behind << "]\n";
   } else {
     out << indented{width} << "bucket is caught up with source\n";
   }
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 179094dc4b54..8f6fc36190cc 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -1,389 +1,373 @@
   $ radosgw-admin --help
   usage: radosgw-admin <cmd> [options...]
   commands:
-    user create                create a new user
-    user modify                modify user
-    user info                  get user info
-    user rename                rename user
-    user rm                    remove user
-    user suspend               suspend a user
-    user enable                re-enable user after suspension
-    user check                 check user info
-    user stats                 show user stats as accounted by quota subsystem
-    user list                  list users
-    caps add                   add user capabilities
-    caps rm                    remove user capabilities
-    subuser create             create a new subuser
-    subuser modify             modify subuser
-    subuser rm                 remove subuser
-    key create                 create access key
-    key rm                     remove access key
-    bucket list                list buckets (specify --allow-unordered for
-                               faster, unsorted listing)
-    bucket limit check         show bucket sharding stats
-    bucket link                link bucket to specified user
-    bucket unlink              unlink bucket from specified user
-    bucket stats               returns bucket statistics
-    bucket rm                  remove bucket
-    bucket check               check bucket index by verifying size and object count stats
-    bucket check olh           check for olh index entries and objects that are pending removal
-    bucket check unlinked      check for object versions that are not visible in a bucket listing 
-    bucket chown               link bucket to specified user and update its object ACLs
-    bucket reshard             reshard bucket
-    bucket rewrite             rewrite all objects in the specified bucket
-    bucket sync checkpoint     poll a bucket's sync status until it catches up to its remote
-    bucket sync disable        disable bucket sync
-    bucket sync enable         enable bucket sync
-    bucket radoslist           list rados objects backing bucket's objects
-    bi get                     retrieve bucket index object entries
-    bi put                     store bucket index object entries
-    bi list                    list raw bucket index entries
-    bi purge                   purge bucket index entries
-    object rm                  remove object
-    object put                 put object
-    object stat                stat an object for its metadata
-    object unlink              unlink object from bucket index
-    object rewrite             rewrite the specified object
-    object reindex             reindex the object(s) indicated by --bucket and either --object or --objects-file
-    objects expire             run expired objects cleanup
-    objects expire-stale list  list stale expired objects (caused by reshard)
-    objects expire-stale rm    remove stale expired objects
-    period rm                  remove a period
-    period get                 get period info
-    period get-current         get current period info
-    period pull                pull a period
-    period push                push a period
-    period list                list all periods
-    period update              update the staging period
-    period commit              commit the staging period
-    quota set                  set quota params
-    quota enable               enable quota
-    quota disable              disable quota
-    ratelimit get              get ratelimit params
-    ratelimit set              set ratelimit params
-    ratelimit enable           enable ratelimit
-    ratelimit disable          disable ratelimit
-    global quota get           view global quota params
-    global quota set           set global quota params
-    global quota enable        enable a global quota
-    global quota disable       disable a global quota
-    global ratelimit get       view global ratelimit params
-    global ratelimit set       set global ratelimit params
-    global ratelimit enable    enable a ratelimit quota
-    global ratelimit disable   disable a ratelimit quota
-    realm create               create a new realm
-    realm rm                   remove a realm
-    realm get                  show realm info
-    realm get-default          get default realm name
-    realm list                 list realms
-    realm list-periods         list all realm periods
-    realm rename               rename a realm
-    realm set                  set realm info (requires infile)
-    realm default              set realm as default
-    realm pull                 pull a realm and its current period
-    zonegroup add              add a zone to a zonegroup
-    zonegroup create           create a new zone group info
-    zonegroup default          set default zone group
-    zonegroup delete           delete a zone group info
-    zonegroup get              show zone group info
-    zonegroup modify           modify an existing zonegroup
-    zonegroup set              set zone group info (requires infile)
-    zonegroup rm               remove a zone from a zonegroup
-    zonegroup rename           rename a zone group
-    zonegroup list             list all zone groups set on this cluster
-    zonegroup placement list   list zonegroup's placement targets
-    zonegroup placement get    get a placement target of a specific zonegroup
-    zonegroup placement add    add a placement target id to a zonegroup
-    zonegroup placement modify modify a placement target of a specific zonegroup
-    zonegroup placement rm     remove a placement target from a zonegroup
-    zonegroup placement default  set a zonegroup's default placement target
-    zone create                create a new zone
-    zone rm                    remove a zone
-    zone get                   show zone cluster params
-    zone modify                modify an existing zone
-    zone set                   set zone cluster params (requires infile)
-    zone list                  list all zones set on this cluster
-    zone rename                rename a zone
-    zone placement list        list zone's placement targets
-    zone placement get         get a zone placement target
-    zone placement add         add a zone placement target
-    zone placement modify      modify a zone placement target
-    zone placement rm          remove a zone placement target
-    metadata sync status       get metadata sync status
-    metadata sync init         init metadata sync
-    metadata sync run          run metadata sync
-    data sync status           get data sync status of the specified source zone
-    data sync init             init data sync for the specified source zone
-    data sync run              run data sync for the specified source zone
-    pool add                   add an existing pool for data placement
-    pool rm                    remove an existing pool from data placement set
-    pools list                 list placement active set
-    policy                     read bucket/object policy
-    log list                   list log objects
-    log show                   dump a log from specific object or (bucket + date
-                               + bucket-id)
-                               (NOTE: required to specify formatting of date
-                               to "YYYY-MM-DD-hh")
-    log rm                     remove log object
-    usage show                 show usage (by user, by bucket, date range)
-    usage trim                 trim usage (by user, by bucket, date range)
-    usage clear                reset all the usage stats for the cluster
-    gc list                    dump expired garbage collection objects (specify
-                               --include-all to list all entries, including unexpired)
-    gc process                 manually process garbage (specify
-                               --include-all to process all entries, including unexpired)
-    lc list                    list all bucket lifecycle progress
-    lc get                     get a lifecycle bucket configuration
-    lc process                 manually process lifecycle
-    lc reshard fix             fix LC for a resharded bucket
-    metadata get               get metadata info
-    metadata put               put metadata info
-    metadata rm                remove metadata info
-    metadata list              list metadata info
-    mdlog list                 list metadata log
-    mdlog autotrim             auto trim metadata log
-    mdlog trim                 trim metadata log (use marker)
-    mdlog status               read metadata log status
-    bilog list                 list bucket index log
-    bilog trim                 trim bucket index log (use start-marker, end-marker)
-    bilog status               read bucket index log status
-    bilog autotrim             auto trim bucket index log
-    datalog list               list data log
-    datalog trim               trim data log
-    datalog status             read data log status
-    datalog type               change datalog type to --log_type={fifo,omap}
-    orphans find               deprecated -- init and run search for leaked rados objects (use job-id, pool)
-    orphans finish             deprecated -- clean up search for leaked rados objects
-    orphans list-jobs          deprecated -- list the current job-ids for orphans search
-                             * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool
-    role create                create a AWS role for use with STS
-    role delete                remove a role
-    role get                   get a role
-    role list                  list roles with specified path prefix
-    role-trust-policy modify   modify the assume role policy of an existing role
-    role-policy put            add/update permission policy to role
-    role-policy list           list policies attached to a role
-    role-policy get            get the specified inline policy document embedded with the given role
-    role-policy delete         remove policy attached to a role
-    role update                update max_session_duration of a role
-    reshard add                schedule a resharding of a bucket
-    reshard list               list all bucket resharding or scheduled to be resharded
-    reshard status             read bucket resharding status
-    reshard process            process of scheduled reshard jobs
-    reshard cancel             cancel resharding a bucket
-    reshard stale-instances list list stale-instances from bucket resharding
+    user create                      create a new user
+    user modify                      modify user
+    user info                        get user info
+    user rename                      rename user
+    user rm                          remove user
+    user suspend                     suspend a user
+    user enable                      re-enable user after suspension
+    user check                       check user info
+    user stats                       show user stats as accounted by quota subsystem
+    user list                        list users
+    caps add                         add user capabilities
+    caps rm                          remove user capabilities
+    subuser create                   create a new subuser
+    subuser modify                   modify subuser
+    subuser rm                       remove subuser
+    key create                       create access key
+    key rm                           remove access key
+    bucket list                      list buckets (specify --allow-unordered for faster, unsorted listing)
+    bucket limit check               show bucket sharding stats
+    bucket link                      link bucket to specified user
+    bucket unlink                    unlink bucket from specified user
+    bucket stats                     returns bucket statistics
+    bucket rm                        remove bucket
+    bucket check                     check bucket index by verifying size and object count stats
+    bucket check olh                 check for olh index entries and objects that are pending removal
+    bucket check unlinked            check for object versions that are not visible in a bucket listing 
+    bucket chown                     link bucket to specified user and update its object ACLs
+    bucket reshard                   reshard bucket
+    bucket rewrite                   rewrite all objects in the specified bucket
+    bucket sync checkpoint           poll a bucket's sync status until it catches up to its remote
+    bucket sync disable              disable bucket sync
+    bucket sync enable               enable bucket sync
+    bucket radoslist                 list rados objects backing bucket's objects
+    bi get                           retrieve bucket index object entries
+    bi put                           store bucket index object entries
+    bi list                          list raw bucket index entries
+    bi purge                         purge bucket index entries
+    object rm                        remove object
+    object put                       put object
+    object stat                      stat an object for its metadata
+    object unlink                    unlink object from bucket index
+    object rewrite                   rewrite the specified object
+    object reindex                   reindex the object(s) indicated by --bucket and either --object or --objects-file
+    objects expire                   run expired objects cleanup
+    objects expire-stale list        list stale expired objects (caused by reshard)
+    objects expire-stale rm          remove stale expired objects
+    period rm                        remove a period
+    period get                       get period info
+    period get-current               get current period info
+    period pull                      pull a period
+    period push                      push a period
+    period list                      list all periods
+    period update                    update the staging period
+    period commit                    commit the staging period
+    quota set                        set quota params
+    quota enable                     enable quota
+    quota disable                    disable quota
+    ratelimit get                    get ratelimit params
+    ratelimit set                    set ratelimit params
+    ratelimit enable                 enable ratelimit
+    ratelimit disable                disable ratelimit
+    global quota get                 view global quota params
+    global quota set                 set global quota params
+    global quota enable              enable a global quota
+    global quota disable             disable a global quota
+    global ratelimit get             view global ratelimit params
+    global ratelimit set             set global ratelimit params
+    global ratelimit enable          enable a ratelimit quota
+    global ratelimit disable         disable a ratelimit quota
+    realm create                     create a new realm
+    realm rm                         remove a realm
+    realm get                        show realm info
+    realm get-default                get default realm name
+    realm list                       list realms
+    realm list-periods               list all realm periods
+    realm rename                     rename a realm
+    realm set                        set realm info (requires infile)
+    realm default                    set realm as default
+    realm pull                       pull a realm and its current period
+    zonegroup add                    add a zone to a zonegroup
+    zonegroup create                 create a new zone group info
+    zonegroup default                set default zone group
+    zonegroup delete                 delete a zone group info
+    zonegroup get                    show zone group info
+    zonegroup modify                 modify an existing zonegroup
+    zonegroup set                    set zone group info (requires infile)
+    zonegroup rm                     remove a zone from a zonegroup
+    zonegroup rename                 rename a zone group
+    zonegroup list                   list all zone groups set on this cluster
+    zonegroup placement list         list zonegroup's placement targets
+    zonegroup placement get          get a placement target of a specific zonegroup
+    zonegroup placement add          add a placement target id to a zonegroup
+    zonegroup placement modify       modify a placement target of a specific zonegroup
+    zonegroup placement rm           remove a placement target from a zonegroup
+    zonegroup placement default      set a zonegroup's default placement target
+    zone create                      create a new zone
+    zone rm                          remove a zone
+    zone get                         show zone cluster params
+    zone modify                      modify an existing zone
+    zone set                         set zone cluster params (requires infile)
+    zone list                        list all zones set on this cluster
+    zone rename                      rename a zone
+    zone placement list              list zone's placement targets
+    zone placement get               get a zone placement target
+    zone placement add               add a zone placement target
+    zone placement modify            modify a zone placement target
+    zone placement rm                remove a zone placement target
+    metadata sync status             get metadata sync status
+    metadata sync init               init metadata sync
+    metadata sync run                run metadata sync
+    data sync status                 get data sync status of the specified source zone
+    data sync init                   init data sync for the specified source zone
+    data sync run                    run data sync for the specified source zone
+    pool add                         add an existing pool for data placement
+    pool rm                          remove an existing pool from data placement set
+    pools list                       list placement active set
+    policy                           read bucket/object policy
+    log list                         list log objects
+    log show                         dump a log from specific object or (bucket + date + bucket-id)
+                                     (NOTE: required to specify formatting of date to "YYYY-MM-DD-hh")
+    log rm                           remove log object
+    usage show                       show usage (by user, by bucket, date range)
+    usage trim                       trim usage (by user, by bucket, date range)
+    usage clear                      reset all the usage stats for the cluster
+    gc list                          dump expired garbage collection objects (specify
+                                     --include-all to list all entries, including unexpired)
+    gc process                       manually process garbage (specify
+                                     --include-all to process all entries, including unexpired)
+    lc list                          list all bucket lifecycle progress
+    lc get                           get a lifecycle bucket configuration
+    lc process                       manually process lifecycle
+    lc reshard fix                   fix LC for a resharded bucket
+    metadata get                     get metadata info
+    metadata put                     put metadata info
+    metadata rm                      remove metadata info
+    metadata list                    list metadata info
+    mdlog list                       list metadata log
+    mdlog autotrim                   auto trim metadata log
+    mdlog trim                       trim metadata log (use marker)
+    mdlog status                     read metadata log status
+    bilog list                       list bucket index log
+    bilog trim                       trim bucket index log (use start-marker, end-marker)
+    bilog status                     read bucket index log status
+    bilog autotrim                   auto trim bucket index log
+    datalog list                     list data log
+    datalog trim                     trim data log
+    datalog status                   read data log status
+    datalog type                     change datalog type to --log_type={fifo,omap}
+    orphans find                     deprecated -- init and run search for leaked rados objects (use job-id, pool)
+    orphans finish                   deprecated -- clean up search for leaked rados objects
+    orphans list-jobs                deprecated -- list the current job-ids for orphans search
+      * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool
+    role create                      create a AWS role for use with STS
+    role delete                      remove a role
+    role get                         get a role
+    role list                        list roles with specified path prefix
+    role-trust-policy modify         modify the assume role policy of an existing role
+    role-policy put                  add/update permission policy to role
+    role-policy list                 list policies attached to a role
+    role-policy get                  get the specified inline policy document embedded with the given role
+    role-policy delete               remove policy attached to a role
+    role update                      update max_session_duration of a role
+    reshard add                      schedule a resharding of a bucket
+    reshard list                     list all bucket resharding or scheduled to be resharded
+    reshard status                   read bucket resharding status
+    reshard process                  process of scheduled reshard jobs
+    reshard cancel                   cancel resharding a bucket
+    reshard stale-instances list     list stale-instances from bucket resharding
     reshard stale-instances delete   cleanup stale-instances from bucket resharding
-    sync error list            list sync error
-    sync error trim            trim sync error
-    mfa create                 create a new MFA TOTP token
-    mfa list                   list MFA TOTP tokens
-    mfa get                    show MFA TOTP token
-    mfa remove                 delete MFA TOTP token
-    mfa check                  check MFA TOTP token
-    mfa resync                 re-sync MFA TOTP token
-    topic list                 list bucket notifications topics
-    topic get                  get a bucket notifications topic
-    topic rm                   remove a bucket notifications topic
-    topic stats                get a bucket notifications persistent topic stats (i.e. reservations, entries & size)
-    script put                 upload a Lua script to a context
-    script get                 get the Lua script of a context
-    script rm                  remove the Lua scripts of a context
-    script-package add         add a Lua package to the scripts allowlist
-    script-package rm          remove a Lua package from the scripts allowlist
-    script-package list        get the Lua packages allowlist
-    script-package reload      install/remove Lua packages according to allowlist
-    notification list          list bucket notifications configuration
-    notification get           get a bucket notifications configuration
-    notification rm            remove a bucket notifications configuration
+    sync error list                  list sync error
+    sync error trim                  trim sync error
+    mfa create                       create a new MFA TOTP token
+    mfa list                         list MFA TOTP tokens
+    mfa get                          show MFA TOTP token
+    mfa remove                       delete MFA TOTP token
+    mfa check                        check MFA TOTP token
+    mfa resync                       re-sync MFA TOTP token
+    topic list                       list bucket notifications topics
+    topic get                        get a bucket notifications topic
+    topic rm                         remove a bucket notifications topic
+    topic stats                      get a bucket notifications persistent topic stats (i.e. reservations, entries & size)
+    script put                       upload a Lua script to a context
+    script get                       get the Lua script of a context
+    script rm                        remove the Lua scripts of a context
+    script-package add               add a Lua package to the scripts allowlist
+    script-package rm                remove a Lua package from the scripts allowlist
+    script-package list              get the Lua packages allowlist
+    script-package reload            install/remove Lua packages according to allowlist
+    notification list                list bucket notifications configuration
+    notification get                 get a bucket notifications configuration
+    notification rm                  remove a bucket notifications configuration
   options:
-     --tenant=<tenant>         tenant name
-     --user_ns=<namespace>     namespace of user (oidc in case of users authenticated with oidc provider)
-     --uid=<id>                user id
-     --new-uid=<id>            new user id
-     --subuser=<name>          subuser name
-     --access-key=<key>        S3 access key
-     --email=<email>           user's email address
-     --secret/--secret-key=<key>
-                               specify secret key
-     --gen-access-key          generate random access key (for S3)
-     --gen-secret              generate random secret key
-     --key-type=<type>         key type, options are: swift, s3
-     --temp-url-key[-2]=<key>  temp url key
-     --access=<access>         Set access permissions for sub-user, should be one
-                               of read, write, readwrite, full
-     --display-name=<name>     user's display name
-     --max-buckets             max number of buckets for a user
-     --admin                   set the admin flag on the user
-     --system                  set the system flag on the user
-     --op-mask                 set the op mask on the user
-     --bucket=<bucket>         Specify the bucket name. Also used by the quota command.
-     --pool=<pool>             Specify the pool name. Also used to scan for leaked rados objects.
-     --object=<object>         object name
-     --objects-file=<file>     file containing a list of object names to process
-     --object-version=<version>         object version
-     --date=<date>             date in the format yyyy-mm-dd
-     --start-date=<date>       start date in the format yyyy-mm-dd
-     --end-date=<date>         end date in the format yyyy-mm-dd
-     --bucket-id=<bucket-id>   bucket id
-     --bucket-new-name=<bucket>
-                               for bucket link: optional new name
-     --shard-id=<shard-id>     optional for: 
-                                 mdlog list
-                                 data sync status
-                               required for: 
-                                 mdlog trim
-     --gen=<gen-id>            optional for: 
-                                 bilog list
-                                 bilog trim
-                                 bilog status
-     --max-entries=<entries>   max entries for listing operations
-     --metadata-key=<key>      key to retrieve metadata from with metadata get
-     --remote=<remote>         zone or zonegroup id of remote gateway
-     --period=<id>             period id
-     --url=<url>               url for pushing/pulling period/realm
-     --epoch=<number>          period epoch
-     --commit                  commit the period during 'period update'
-     --staging                 get staging period info
-     --master                  set as master
-     --master-zone=<id>        master zone id
-     --rgw-realm=<name>        realm name
-     --realm-id=<id>           realm id
-     --realm-new-name=<name>   realm new name
-     --rgw-zonegroup=<name>    zonegroup name
-     --zonegroup-id=<id>       zonegroup id
-     --zonegroup-new-name=<name>
-                               zonegroup new name
-     --rgw-zone=<name>         name of zone in which radosgw is running
-     --zone-id=<id>            zone id
-     --zone-new-name=<name>    zone new name
-     --source-zone             specify the source zone (for data sync)
-     --default                 set entity (realm, zonegroup, zone) as default
-     --read-only               set zone as read-only (when adding to zonegroup)
-     --redirect-zone           specify zone id to redirect when response is 404 (not found)
-     --placement-id            placement id for zonegroup placement commands
-     --storage-class           storage class for zonegroup placement commands
-     --tags=<list>             list of tags for zonegroup placement add and modify commands
-     --tags-add=<list>         list of tags to add for zonegroup placement modify command
-     --tags-rm=<list>          list of tags to remove for zonegroup placement modify command
-     --endpoints=<list>        zone endpoints
-     --index-pool=<pool>       placement target index pool
-     --data-pool=<pool>        placement target data pool
-     --data-extra-pool=<pool>  placement target data extra (non-ec) pool
-     --placement-index-type=<type>
-                               placement target index type (normal, indexless, or #id)
-     --placement-inline-data=<true>
-                               set whether the placement target is configured to store a data
-                               chunk inline in head objects
-     --compression=<type>      placement target compression type (plugin name or empty/none)
-     --tier-type=<type>        zone tier type
-     --tier-config=<k>=<v>[,...]
-                               set zone tier config keys, values
-     --tier-config-rm=<k>[,...]
-                               unset zone tier config keys
-     --sync-from-all[=false]   set/reset whether zone syncs from all zonegroup peers
-     --sync-from=[zone-name][,...]
-                               set list of zones to sync from
-     --sync-from-rm=[zone-name][,...]
-                               remove zones from list of zones to sync from
-     --bucket-index-max-shards override a zone/zonegroup's default bucket index shard count
-     --fix                     besides checking bucket index, will also fix it
-     --check-objects           bucket check: rebuilds bucket index according to
-                               actual objects state
-     --format=<format>         specify output format for certain operations: xml,
-                               json
-     --purge-data              when specified, user removal will also purge all the
-                               user data
-     --purge-keys              when specified, subuser removal will also purge all the
-                               subuser keys
-     --purge-objects           remove a bucket's objects before deleting it
-                               (NOTE: required to delete a non-empty bucket)
-     --sync-stats              option to 'user stats', update user stats with current
-                               stats reported by user's buckets indexes
-     --reset-stats             option to 'user stats', reset stats in accordance with user buckets
-     --show-config             show configuration
-     --show-log-entries=<flag> enable/disable dump of log entries on log show
-     --show-log-sum=<flag>     enable/disable dump of log summation on log show
-     --skip-zero-entries       log show only dumps entries that don't have zero value
-                               in one of the numeric field
-     --infile=<file>           file to read in when setting data
-     --categories=<list>       comma separated list of categories, used in usage show
-     --caps=<caps>             list of caps (e.g., "usage=read, write; user=read")
-     --op-mask=<op-mask>       permission of user's operations (e.g., "read, write, delete, *")
-     --yes-i-really-mean-it    required for certain operations
-     --warnings-only           when specified with bucket limit check, list
-                               only buckets nearing or over the current max
-                               objects per shard value
-     --bypass-gc               when specified with bucket deletion, triggers
-                               object deletions by not involving GC
-     --inconsistent-index      when specified with bucket deletion and bypass-gc set to true,
-                               ignores bucket index consistency
-     --min-rewrite-size        min object size for bucket rewrite (default 4M)
-     --max-rewrite-size        max object size for bucket rewrite (default ULLONG_MAX)
-     --min-rewrite-stripe-size min stripe size for object rewrite (default 0)
-     --trim-delay-ms           time interval in msec to limit the frequency of sync error log entries trimming operations,
-                               the trimming process will sleep the specified msec for every 1000 entries trimmed
-     --max-concurrent-ios      maximum concurrent ios for bucket operations (default: 32)
-     --enable-feature          enable a zone/zonegroup feature
-     --disable-feature         disable a zone/zonegroup feature
+     --tenant=<tenant>                 tenant name
+     --user_ns=<namespace>             namespace of user (oidc in case of users authenticated with oidc provider)
+     --uid=<id>                        user id
+     --new-uid=<id>                    new user id
+     --subuser=<name>                  subuser name
+     --access-key=<key>                S3 access key
+     --email=<email>                   user's email address
+     --secret/--secret-key=<key>       specify secret key
+     --gen-access-key                  generate random access key (for S3)
+     --gen-secret                      generate random secret key
+     --key-type=<type>                 key type, options are: swift, s3
+     --temp-url-key[-2]=<key>          temp url key
+     --access=<access>                 Set access permissions for sub-user, should be one
+                                       of read, write, readwrite, full
+     --display-name=<name>             user's display name
+     --max-buckets                     max number of buckets for a user
+     --admin                           set the admin flag on the user
+     --system                          set the system flag on the user
+     --op-mask                         set the op mask on the user
+     --bucket=<bucket>                 Specify the bucket name. Also used by the quota command.
+     --pool=<pool>                     Specify the pool name. Also used to scan for leaked rados objects.
+     --object=<object>                 object name
+     --objects-file=<file>             file containing a list of object names to process
+     --object-version=<version>        object version
+     --date=<date>                     date in the format yyyy-mm-dd
+     --start-date=<date>               start date in the format yyyy-mm-dd
+     --end-date=<date>                 end date in the format yyyy-mm-dd
+     --bucket-id=<bucket-id>           bucket id
+     --bucket-new-name=<bucket>        for bucket link: optional new name
+     --shard-id=<shard-id>             optional for:
+                                         mdlog list
+                                         data sync status
+                                       required for:
+                                         mdlog trim
+     --gen=<gen-id>                    optional for:
+                                         bilog list
+                                         bilog trim
+                                         bilog status
+     --max-entries=<entries>           max entries for listing operations
+     --metadata-key=<key>              key to retrieve metadata from with metadata get
+     --remote=<remote>                 zone or zonegroup id of remote gateway
+     --period=<id>                     period id
+     --url=<url>                       url for pushing/pulling period/realm
+     --epoch=<number>                  period epoch
+     --commit                          commit the period during 'period update'
+     --staging                         get staging period info
+     --master                          set as master
+     --master-zone=<id>                master zone id
+     --rgw-realm=<name>                realm name
+     --realm-id=<id>                   realm id
+     --realm-new-name=<name>           realm new name
+     --rgw-zonegroup=<name>            zonegroup name
+     --zonegroup-id=<id>               zonegroup id
+     --zonegroup-new-name=<name>       zonegroup new name
+     --rgw-zone=<name>                 name of zone in which radosgw is running
+     --zone-id=<id>                    zone id
+     --zone-new-name=<name>            zone new name
+     --source-zone                     specify the source zone (for data sync)
+     --default                         set entity (realm, zonegroup, zone) as default
+     --read-only                       set zone as read-only (when adding to zonegroup)
+     --redirect-zone                   specify zone id to redirect when response is 404 (not found)
+     --placement-id                    placement id for zonegroup placement commands
+     --storage-class                   storage class for zonegroup placement commands
+     --tags=<list>                     list of tags for zonegroup placement add and modify commands
+     --tags-add=<list>                 list of tags to add for zonegroup placement modify command
+     --tags-rm=<list>                  list of tags to remove for zonegroup placement modify command
+     --endpoints=<list>                zone endpoints
+     --index-pool=<pool>               placement target index pool
+     --data-pool=<pool>                placement target data pool
+     --data-extra-pool=<pool>          placement target data extra (non-ec) pool
+     --placement-index-type=<type>     placement target index type (normal, indexless, or #id)
+     --placement-inline-data=<true>    set whether the placement target is configured to store a data
+                                       chunk inline in head objects
+     --compression=<type>              placement target compression type (plugin name or empty/none)
+     --tier-type=<type>                zone tier type
+     --tier-config=<k>=<v>[,...]       set zone tier config keys, values
+     --tier-config-rm=<k>[,...]        unset zone tier config keys
+     --sync-from-all[=false]           set/reset whether zone syncs from all zonegroup peers
+     --sync-from=[zone-name][,...]     set list of zones to sync from
+     --sync-from-rm=[zone-name][,...]  remove zones from list of zones to sync from
+     --bucket-index-max-shards         override a zone/zonegroup's default bucket index shard count
+     --fix                             besides checking bucket index, will also fix it
+     --check-objects                   bucket check: rebuilds bucket index according to actual objects state
+     --format=<format>                 specify output format for certain operations: xml, json
+     --purge-data                      when specified, user removal will also purge all the
+                                       user data
+     --purge-keys                      when specified, subuser removal will also purge all the
+                                       subuser keys
+     --purge-objects                   remove a bucket's objects before deleting it
+                                       (NOTE: required to delete a non-empty bucket)
+     --sync-stats                      option to 'user stats', update user stats with current
+                                       stats reported by user's buckets indexes
+     --reset-stats                     option to 'user stats', reset stats in accordance with user buckets
+     --show-config                     show configuration
+     --show-log-entries=<flag>         enable/disable dump of log entries on log show
+     --show-log-sum=<flag>             enable/disable dump of log summation on log show
+     --skip-zero-entries               log show only dumps entries that don't have zero value
+                                       in one of the numeric field
+     --infile=<file>                   file to read in when setting data
+     --categories=<list>               comma separated list of categories, used in usage show
+     --caps=<caps>                     list of caps (e.g., "usage=read, write; user=read")
+     --op-mask=<op-mask>               permission of user's operations (e.g., "read, write, delete, *")
+     --yes-i-really-mean-it            required for certain operations
+     --warnings-only                   when specified with bucket limit check, list
+                                       only buckets nearing or over the current max
+                                       objects per shard value
+     --bypass-gc                       when specified with bucket deletion, triggers
+                                       object deletions by not involving GC
+     --inconsistent-index              when specified with bucket deletion and bypass-gc set to true,
+                                       ignores bucket index consistency
+     --min-rewrite-size                min object size for bucket rewrite (default 4M)
+     --max-rewrite-size                max object size for bucket rewrite (default ULLONG_MAX)
+     --min-rewrite-stripe-size         min stripe size for object rewrite (default 0)
+     --trim-delay-ms                   time interval in msec to limit the frequency of sync error log entries trimming operations,
+                                       the trimming process will sleep the specified msec for every 1000 entries trimmed
+     --max-concurrent-ios              maximum concurrent ios for bucket operations (default: 32)
+     --enable-feature                  enable a zone/zonegroup feature
+     --disable-feature                 disable a zone/zonegroup feature
   
   <date> := "YYYY-MM-DD[ hh:mm:ss]"
   
   Quota options:
-     --max-objects             specify max objects (negative value to disable)
-     --max-size                specify max size (in B/K/M/G/T, negative value to disable)
-     --quota-scope             scope of quota (bucket, user)
+     --max-objects                 specify max objects (negative value to disable)
+     --max-size                    specify max size (in B/K/M/G/T, negative value to disable)
+     --quota-scope                 scope of quota (bucket, user)
   
   Rate limiting options:
-     --max-read-ops            specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited
-     --max-read-bytes          specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited
-     --max-write-ops           specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited
-     --max-write-bytes         specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited
-     --ratelimit-scope         scope of rate limiting: bucket, user, anonymous
-                               anonymous can be configured only with global rate limit
+     --max-read-ops                specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited
+     --max-read-bytes              specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited
+     --max-write-ops               specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited
+     --max-write-bytes             specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited
+     --ratelimit-scope             scope of rate limiting: bucket, user, anonymous
+                                   anonymous can be configured only with global rate limit
   
   Orphans search options:
-     --num-shards              num of shards to use for keeping the temporary scan info
-     --orphan-stale-secs       num of seconds to wait before declaring an object to be an orphan (default: 86400)
-     --job-id                  set the job id (for orphans find)
-     --detail                  detailed mode, log and stat head objects as well
+     --num-shards                  num of shards to use for keeping the temporary scan info
+     --orphan-stale-secs           num of seconds to wait before declaring an object to be an orphan (default: 86400)
+     --job-id                      set the job id (for orphans find)
+     --detail                      detailed mode, log and stat head objects as well
   
   Orphans list-jobs options:
-     --extra-info              provide extra info in job list
+     --extra-info                  provide extra info in job list
   
   Role options:
-     --role-name               name of the role to create
-     --path                    path to the role
-     --assume-role-policy-doc  the trust relationship policy document that grants an entity permission to assume the role
-     --policy-name             name of the policy document
-     --policy-doc              permission policy document
-     --path-prefix             path prefix for filtering roles
+     --role-name                   name of the role to create
+     --path                        path to the role
+     --assume-role-policy-doc      the trust relationship policy document that grants an entity permission to assume the role
+     --policy-name                 name of the policy document
+     --policy-doc                  permission policy document
+     --path-prefix                 path prefix for filtering roles
   
   MFA options:
-     --totp-serial             a string that represents the ID of a TOTP token
-     --totp-seed               the secret seed that is used to calculate the TOTP
-     --totp-seconds            the time resolution that is being used for TOTP generation
-     --totp-window             the number of TOTP tokens that are checked before and after the current token when validating token
-     --totp-pin                the valid value of a TOTP token at a certain time
+     --totp-serial                 a string that represents the ID of a TOTP token
+     --totp-seed                   the secret seed that is used to calculate the TOTP
+     --totp-seconds                the time resolution that is being used for TOTP generation
+     --totp-window                 the number of TOTP tokens that are checked before and after the current token when validating token
+     --totp-pin                    the valid value of a TOTP token at a certain time
   
   Bucket notifications options:
-     --topic                   bucket notifications topic name
-     --notification-id         bucket notifications id
+     --topic                       bucket notifications topic name
+     --notification-id             bucket notifications id
   
   Script options:
-     --context                 context in which the script runs. one of: prerequest, postrequest, background, getdata, putdata
-     --package                 name of the Lua package that should be added/removed to/from the allowlist
-     --allow-compilation       package is allowed to compile C code as part of its installation
+     --context                     context in which the script runs. one of: prerequest, postrequest, background, getdata, putdata
+     --package                     name of the Lua package that should be added/removed to/from the allowlist
+     --allow-compilation           package is allowed to compile C code as part of its installation
   
   Bucket check olh/unlinked options:
-     --min-age-hours           minimum age of unlinked objects to consider for bucket check unlinked (default: 1)
-     --dump-keys               when specified, all keys identified as problematic are printed to stdout
-     --hide-progress           when specified, per-shard progress details are not printed to stderr
+     --min-age-hours               minimum age of unlinked objects to consider for bucket check unlinked (default: 1)
+     --dump-keys                   when specified, all keys identified as problematic are printed to stdout
+     --hide-progress               when specified, per-shard progress details are not printed to stderr
   
   radoslist options:
-     --rgw-obj-fs              the field separator that will separate the rados
-                               object name from the rgw object name;
-                               additionally rados objects for incomplete
-                               multipart uploads will not be output
+     --rgw-obj-fs                  the field separator that will separate the rados object name from the rgw object name;
+                                   additionally rados objects for incomplete multipart uploads will not be output
   
     --conf/-c FILE    read configuration from the given configuration file
     --id ID           set ID portion of my name

From ed70d843df4d4077efd053bd8065bdf26635d239 Mon Sep 17 00:00:00 2001
From: Josh Soref <2119212+jsoref@users.noreply.github.com>
Date: Wed, 22 Jun 2022 21:24:12 -0400
Subject: [PATCH 0743/2492] rgw: fix spelling errors

* access
* acquire
* across
* algorithm
* alternatively
* ambiguous
* attributes
* authenticated
* automatically
* based
* between
* beware
* bootstrap
* bucket
* cache
* character
* check
* cleanup
* combinations
* committed
* comparison
* compatibility
* concurrent
* conditions
* conjunction
* connections
* considered
* contains
* correct
* creates
* crypto
* dependencies
* dequeued
* dequeuing
* derivative
* described
* determine
* deterministic
* detritus
* discrepancy
* display
* either way
* elements
* entirely
* entries
* error
* everything
* expected
* explicitly
* extend
* false
* fclose
* feature
* following
* foreign
* format
* forward
* functionality
* functions
* generated
* getting
* gracefully
* happened
* hashicorp
* healthcheck
* hierarchy
* however
* http
* identify
* idleness
* ignored
* incremental
* information
* initialization
* length
* locally
* macros
* malformed
* means
* mechanism
* multipart
* multiple
* must
* namespace
* necessary
* nonexistent
* notifications
* occurred
* official
* outputs
* outputted
* overridden
* ownership
* parallelism
* partition
* passthrough
* pause
* penguin
* perform
* permission
* populated
* position
* preferably
* prefixes
* presumably
* previous
* print
* quadruple
* reference
* regardless
* registry
* renewed
* request
* requires
* reshard
* resource
* result to
* satisfy
* secret
* separate
* separator
* setting
* shouldn't
* skipping
* some
* somehow
* something
* specific
* struct
* succeeded
* success
* successful
* successfully
* syntax
* testing
* truncated
* unambiguously
* unittest
* unmodified
* unnecessary
* valid_action
* verifying
* versioning
* visible
* whatever
* whenever

Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
---
 examples/rgw/golang/object-upload/README.md  |  2 +-
 examples/rgw/java/ceph-s3-upload/README.md   |  2 +-
 qa/tasks/pykmip.py                           |  2 +-
 qa/tasks/s3tests_java.py                     |  2 +-
 qa/tasks/vault.py                            |  2 +-
 qa/workunits/rgw/s3_bucket_quota.pl          |  2 +-
 qa/workunits/rgw/s3_user_quota.pl            |  2 +-
 qa/workunits/rgw/test_librgw_file.sh         |  2 +-
 qa/workunits/rgw/test_rgw_datacache.py       |  2 +-
 src/cls/2pc_queue/cls_2pc_queue.cc           |  2 +-
 src/cls/refcount/cls_refcount_client.h       |  2 +-
 src/cls/rgw/cls_rgw.cc                       | 10 +++---
 src/cls/rgw/cls_rgw_client.cc                |  4 +--
 src/cls/rgw/cls_rgw_client.h                 |  2 +-
 src/cls/rgw/cls_rgw_const.h                  |  2 +-
 src/cls/rgw/cls_rgw_ops.h                    |  4 +--
 src/cls/rgw/cls_rgw_types.h                  |  4 +--
 src/rgw/driver/dbstore/common/dbstore.h      |  8 ++---
 src/rgw/driver/dbstore/sqlite/sqliteDB.cc    | 38 ++++++++++----------
 src/rgw/driver/motr/rgw_sal_motr.cc          | 12 +++----
 src/rgw/driver/motr/rgw_sal_motr.h           |  6 ++--
 src/rgw/driver/rados/rgw_bucket.h            |  2 +-
 src/rgw/driver/rados/rgw_cr_rados.cc         |  2 +-
 src/rgw/driver/rados/rgw_d3n_datacache.cc    |  6 ++--
 src/rgw/driver/rados/rgw_data_sync.cc        |  6 ++--
 src/rgw/driver/rados/rgw_datalog.cc          |  2 +-
 src/rgw/driver/rados/rgw_lc_tier.cc          |  2 +-
 src/rgw/driver/rados/rgw_notify.cc           | 12 +++----
 src/rgw/driver/rados/rgw_notify.h            |  2 +-
 src/rgw/driver/rados/rgw_putobj_processor.cc |  2 +-
 src/rgw/driver/rados/rgw_rados.cc            | 20 +++++------
 src/rgw/driver/rados/rgw_rest_log.h          |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc        |  4 +--
 src/rgw/driver/rados/rgw_sync.cc             |  2 +-
 src/rgw/driver/rados/rgw_sync_module_aws.cc  |  4 +--
 src/rgw/driver/rados/rgw_sync_module_es.cc   |  2 +-
 src/rgw/driver/rados/rgw_zone.h              |  2 +-
 src/rgw/jwt-cpp/jwt.h                        | 10 +++---
 src/rgw/rgw-gap-list                         |  6 ++--
 src/rgw/rgw-orphan-list                      |  2 +-
 src/rgw/rgw_acl_swift.cc                     |  2 +-
 src/rgw/rgw_admin.cc                         |  6 ++--
 src/rgw/rgw_amqp.cc                          |  4 +--
 src/rgw/rgw_asio_client.cc                   |  6 ++--
 src/rgw/rgw_asio_frontend.cc                 |  2 +-
 src/rgw/rgw_auth.h                           |  6 ++--
 src/rgw/rgw_auth_keystone.cc                 |  2 +-
 src/rgw/rgw_basic_types.h                    |  4 +--
 src/rgw/rgw_common.cc                        |  8 ++---
 src/rgw/rgw_common.h                         |  6 ++--
 src/rgw/rgw_crypt.cc                         |  2 +-
 src/rgw/rgw_env.cc                           |  2 +-
 src/rgw/rgw_file.cc                          |  6 ++--
 src/rgw/rgw_iam_policy.cc                    |  8 ++---
 src/rgw/rgw_kafka.cc                         | 12 +++----
 src/rgw/rgw_keystone.cc                      |  6 ++--
 src/rgw/rgw_kmip_client_impl.cc              |  2 +-
 src/rgw/rgw_lc.cc                            |  4 +--
 src/rgw/rgw_lib.cc                           |  2 +-
 src/rgw/rgw_lib.h                            |  2 +-
 src/rgw/rgw_lua_background.h                 |  2 +-
 src/rgw/rgw_lua_utils.cc                     |  2 +-
 src/rgw/rgw_lua_utils.h                      |  2 +-
 src/rgw/rgw_op.cc                            |  8 ++---
 src/rgw/rgw_op.h                             |  2 +-
 src/rgw/rgw_perf_counters.cc                 |  2 +-
 src/rgw/rgw_period_history.cc                |  2 +-
 src/rgw/rgw_pubsub.cc                        |  2 +-
 src/rgw/rgw_realm_reloader.cc                |  2 +-
 src/rgw/rgw_rest.cc                          | 18 +++++-----
 src/rgw/rgw_rest_pubsub.cc                   |  2 +-
 src/rgw/rgw_rest_ratelimit.cc                |  2 +-
 src/rgw/rgw_rest_s3.cc                       |  6 ++--
 src/rgw/rgw_rest_sts.cc                      |  2 +-
 src/rgw/rgw_rest_swift.cc                    |  2 +-
 src/rgw/rgw_s3select.cc                      |  2 +-
 src/rgw/rgw_s3select_private.h               |  2 +-
 src/rgw/rgw_sal.h                            |  2 +-
 src/rgw/rgw_sal_dbstore.cc                   |  6 ++--
 src/rgw/rgw_sts.cc                           |  2 +-
 src/rgw/rgw_swift_auth.cc                    |  2 +-
 src/rgw/rgw_sync_policy.h                    |  2 +-
 src/rgw/rgw_zone_types.h                     |  2 +-
 src/rgw/services/svc_rados.cc                |  2 +-
 src/test/cls_2pc_queue/test_cls_2pc_queue.cc |  8 ++---
 src/test/cls_queue/test_cls_queue.cc         |  6 ++--
 src/test/cls_rgw/test_cls_rgw.cc             |  4 +--
 src/test/cls_rgw/test_cls_rgw_stats.cc       |  2 +-
 src/test/rgw/CMakeLists.txt                  | 10 +++---
 src/test/rgw/bucket_notification/api.py      |  2 +-
 src/test/rgw/bucket_notification/test_bn.py  |  4 +--
 src/test/rgw/rgw_multi/tests_az.py           |  2 +-
 src/test/rgw/test-ceph-diff-sorted.sh        |  2 +-
 src/test/rgw/test_multi.md                   |  2 +-
 src/test/rgw/test_rgw_amqp.cc                |  2 +-
 src/test/rgw/test_rgw_kms.cc                 |  2 +-
 src/test/rgw/test_rgw_lua.cc                 |  4 +--
 src/test/rgw/test_rgw_xml.cc                 |  2 +-
 src/test/test_rgw_admin_log.cc               |  6 ++--
 src/test/test_rgw_admin_meta.cc              |  2 +-
 100 files changed, 216 insertions(+), 216 deletions(-)

diff --git a/examples/rgw/golang/object-upload/README.md b/examples/rgw/golang/object-upload/README.md
index 31ea00006d61..a859667009e3 100644
--- a/examples/rgw/golang/object-upload/README.md
+++ b/examples/rgw/golang/object-upload/README.md
@@ -2,7 +2,7 @@
 This directory contains Golang code examples on how to upload an object to an S3 bucket running on a Ceph RGW cluster.
 
 # Prerequisite
-Linux machine running an RGW Ceph cluster. Preferrably started with the ``OSD=1 MON=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh --debug --new `` command.
+Linux machine running an RGW Ceph cluster. Preferably started with the ``OSD=1 MON=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh --debug --new `` command.
 Go installed on the Linux machine.
 
 ## Workflow Procedure
diff --git a/examples/rgw/java/ceph-s3-upload/README.md b/examples/rgw/java/ceph-s3-upload/README.md
index 235d43666d1c..fbd7d749b02b 100644
--- a/examples/rgw/java/ceph-s3-upload/README.md
+++ b/examples/rgw/java/ceph-s3-upload/README.md
@@ -2,7 +2,7 @@
 This directory contains Java code examples on how to upload an object to an S3 bucket running on a Ceph RGW cluster.
 
 # Prerequisites
-Linux machine running an RGW Ceph cluster. Preferrably started with the ``OSD=1 MON=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh --debug --new `` command.  
+Linux machine running an RGW Ceph cluster. Preferably started with the ``OSD=1 MON=1 MDS=0 MGR=0 RGW=1 ../src/vstart.sh --debug --new `` command.  
 Java and Maven installed on the Linux machine.  
 
 ## Workflow Procedure
diff --git a/qa/tasks/pykmip.py b/qa/tasks/pykmip.py
index 45a5af6891b5..3ab08a301aa0 100644
--- a/qa/tasks/pykmip.py
+++ b/qa/tasks/pykmip.py
@@ -106,7 +106,7 @@ def install_packages(ctx, config):
     passed in to the main task.
     """
     assert isinstance(config, dict)
-    log.info('Installing system dependenies for PyKMIP...')
+    log.info('Installing system dependencies for PyKMIP...')
 
     packages = {}
     for (client, _) in config.items():
diff --git a/qa/tasks/s3tests_java.py b/qa/tasks/s3tests_java.py
index dbe03921c601..78a1edb6f613 100644
--- a/qa/tasks/s3tests_java.py
+++ b/qa/tasks/s3tests_java.py
@@ -182,7 +182,7 @@ def create_users(self):
         Configuration is read from a skelethon config file
         s3tests.teuth.config.yaml in the java-s3tests repository
         and missing information is added from the task.
-        Existing values are NOT overriden unless they are empty!
+        Existing values are NOT overridden unless they are empty!
         """
         log.info("S3 Tests Java: Creating S3 users...")
         testdir = teuthology.get_testdir(self.ctx)
diff --git a/qa/tasks/vault.py b/qa/tasks/vault.py
index 2ff008c4dbef..ae874eb55e35 100644
--- a/qa/tasks/vault.py
+++ b/qa/tasks/vault.py
@@ -38,7 +38,7 @@ def assign_ports(ctx, config, initial_port):
 @contextlib.contextmanager
 def download(ctx, config):
     """
-    Download Vault Release from Hashicopr website.
+    Download Vault Release from Hashicorp website.
     Remove downloaded file upon exit.
     """
     assert isinstance(config, dict)
diff --git a/qa/workunits/rgw/s3_bucket_quota.pl b/qa/workunits/rgw/s3_bucket_quota.pl
index 7f5476ef676c..6f0552ab7018 100755
--- a/qa/workunits/rgw/s3_bucket_quota.pl
+++ b/qa/workunits/rgw/s3_bucket_quota.pl
@@ -16,7 +16,7 @@ =head1 SYNOPSIS
 
 =head1 DESCRIPTION
 
-This script intends to test the rgw bucket quota funcionality using s3 interface 
+This script intends to test the rgw bucket quota functionality using s3 interface 
 and reports the test results
 
 =head1 ARGUMENTS
diff --git a/qa/workunits/rgw/s3_user_quota.pl b/qa/workunits/rgw/s3_user_quota.pl
index 6d5c02a9a015..04546eac8404 100755
--- a/qa/workunits/rgw/s3_user_quota.pl
+++ b/qa/workunits/rgw/s3_user_quota.pl
@@ -16,7 +16,7 @@ =head1 SYNOPSIS
 
 =head1 DESCRIPTION
 
-This script intends to test the rgw user quota funcionality using s3 interface
+This script intends to test the rgw user quota functionality using s3 interface
 and reports the test results
 
 =head1 ARGUMENTS
diff --git a/qa/workunits/rgw/test_librgw_file.sh b/qa/workunits/rgw/test_librgw_file.sh
index 1371ff711075..4b9dfeb41210 100755
--- a/qa/workunits/rgw/test_librgw_file.sh
+++ b/qa/workunits/rgw/test_librgw_file.sh
@@ -19,7 +19,7 @@ fi
 
 # nfsns is the main suite
 
-# create herarchy, and then list it
+# create hierarchy, and then list it
 echo "phase 1.1"
 ceph_test_librgw_file_nfsns ${K} --hier1 --dirs1 --create --rename --verbose
 
diff --git a/qa/workunits/rgw/test_rgw_datacache.py b/qa/workunits/rgw/test_rgw_datacache.py
index f070ec0f1799..2af2a0d3aa3d 100755
--- a/qa/workunits/rgw/test_rgw_datacache.py
+++ b/qa/workunits/rgw/test_rgw_datacache.py
@@ -202,7 +202,7 @@ def main():
     # remove datacache dir
     #cmd = exec_cmd('rm -rf %s' % (cache_dir))
     #log.debug("RGW Datacache dir deleted")
-    #^ commenting for future refrence - the work unit will continue running tests and if the cache_dir is removed
+    #^ commenting for future reference - the work unit will continue running tests and if the cache_dir is removed
     #  all the writes to cache will fail with errno 2 ENOENT No such file or directory.
 
 main()
diff --git a/src/cls/2pc_queue/cls_2pc_queue.cc b/src/cls/2pc_queue/cls_2pc_queue.cc
index 1e99503943a3..759d360b0148 100644
--- a/src/cls/2pc_queue/cls_2pc_queue.cc
+++ b/src/cls/2pc_queue/cls_2pc_queue.cc
@@ -135,7 +135,7 @@ static int cls_2pc_queue_reserve(cls_method_context_t hctx, bufferlist *in, buff
   }
 
   urgent_data.reserved_size += res_op.size + overhead;
-  // note that last id is incremented regadless of failures
+  // note that last id is incremented regardless of failures
   // to avoid "old reservation" issues below
   ++urgent_data.last_id;
   bool result;
diff --git a/src/cls/refcount/cls_refcount_client.h b/src/cls/refcount/cls_refcount_client.h
index 73a23a7ee28d..17af7a78ef3e 100644
--- a/src/cls/refcount/cls_refcount_client.h
+++ b/src/cls/refcount/cls_refcount_client.h
@@ -19,7 +19,7 @@
  * So, the regular usage would be to create an object, to increase the refcount. Then, when
  * wanting to have another reference to it, increase the refcount using a different tag. When
  * removing a reference it is required to drop the refcount (using the same tag that was used
- * for that reference). When the refcount drops to zero, the object is removed automaticfally.
+ * for that reference). When the refcount drops to zero, the object is removed automatically.
  *
  * In order to maintain backwards compatibility with objects that were created without having
  * their refcount increased, the implicit_ref was added. Any object that was created without
diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 75a37bad6342..034660a11ed1 100644
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -518,7 +518,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
   std::string start_after_omap_key;
   encode_list_index_key(hctx, op.start_obj, &start_after_omap_key);
 
-  // this is set whenenver start_after_omap_key is set to keep them in
+  // this is set whenever start_after_omap_key is set to keep them in
   // sync since this will be the returned marker when a marker is
   // returned
   cls_rgw_obj_key start_after_entry_key;
@@ -3181,7 +3181,7 @@ int rgw_bucket_check_index(cls_method_context_t hctx, bufferlist *in, bufferlist
  * Additionally, each of the three segment functions, if successful,
  * is expected to return the number of entries added to the output
  * list as a non-negative value. As per usual, negative return values
- * indicate error condtions.
+ * indicate error conditions.
  */
 static int rgw_bi_list_op(cls_method_context_t hctx,
 			  bufferlist *in,
@@ -4457,7 +4457,7 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis
   try {
     decode(op, in_iter);
   } catch (ceph::buffer::error& err) {
-    CLS_LOG(1, "ERROR: rgw_cls_rehard_list(): failed to decode entry\n");
+    CLS_LOG(1, "ERROR: rgw_cls_reshard_list(): failed to decode entry\n");
     return -EINVAL;
   }
   cls_rgw_reshard_list_ret op_ret;
@@ -4476,7 +4476,7 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis
     try {
       decode(entry, iter);
     } catch (ceph::buffer::error& err) {
-      CLS_LOG(1, "ERROR: rgw_cls_rehard_list(): failed to decode entry\n");
+      CLS_LOG(1, "ERROR: rgw_cls_reshard_list(): failed to decode entry\n");
       return -EIO;
    }
     op_ret.entries.push_back(entry);
@@ -4521,7 +4521,7 @@ static int rgw_reshard_remove(cls_method_context_t hctx, bufferlist *in, bufferl
   try {
     decode(op, in_iter);
   } catch (ceph::buffer::error& err) {
-    CLS_LOG(1, "ERROR: rgw_cls_rehard_remove: failed to decode entry\n");
+    CLS_LOG(1, "ERROR: rgw_cls_reshard_remove: failed to decode entry\n");
     return -EINVAL;
   }
 
diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 5e7fba88f24a..ed2be322da54 100644
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -79,7 +79,7 @@ int CLSRGWConcurrentIO::operator()() {
     cleanup();
   }
   return ret;
-} // CLSRGWConcurrintIO::operator()()
+} // CLSRGWConcurrentIO::operator()()
 
 
 /**
@@ -186,7 +186,7 @@ bool BucketIndexAioManager::wait_for_completions(int valid_ret_code,
   return true;
 }
 
-// note: currently only called by tesing code
+// note: currently only called by testing code
 void cls_rgw_bucket_init_index(ObjectWriteOperation& o)
 {
   bufferlist in;
diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 1ae49c877bb4..3db730651712 100644
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -405,7 +405,7 @@ int cls_rgw_usage_log_trim(librados::IoCtx& io_ctx, const std::string& oid, cons
 /**
  * Std::list the bucket with the starting object and filter prefix.
  * NOTE: this method do listing requests for each bucket index shards identified by
- *       the keys of the *list_results* std::map, which means the std::map should be popludated
+ *       the keys of the *list_results* std::map, which means the std::map should be populated
  *       by the caller to fill with each bucket index object id.
  *
  * io_ctx        - IO context for rados.
diff --git a/src/cls/rgw/cls_rgw_const.h b/src/cls/rgw/cls_rgw_const.h
index 8595db3c9e8b..a3398ebd0164 100644
--- a/src/cls/rgw/cls_rgw_const.h
+++ b/src/cls/rgw/cls_rgw_const.h
@@ -6,7 +6,7 @@
 #define RGW_CLASS "rgw"
 
 /* Special error code returned by cls bucket list operation if it was
- * unable to skip past enough not visibile entries to return any
+ * unable to skip past enough not visible entries to return any
  * entries in the call. */
 constexpr int RGWBIAdvanceAndRetryError = -EFBIG;
 
diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h
index 4d58909a7670..a4d76de108ba 100644
--- a/src/cls/rgw/cls_rgw_ops.h
+++ b/src/cls/rgw/cls_rgw_ops.h
@@ -430,7 +430,7 @@ struct rgw_cls_list_ret {
   // if is_truncated is true, starting marker for next iteration; this
   // is necessary as it's possible after maximum number of tries we
   // still might have zero entries to return, in which case we have to
-  // at least move the ball foward
+  // at least move the ball forward
   cls_rgw_obj_key marker;
 
   // cls_filtered is not transmitted; it is assumed true for versions
@@ -758,7 +758,7 @@ WRITE_CLASS_ENCODER(rgw_cls_bi_put_op)
 
 struct rgw_cls_bi_list_op {
   uint32_t max;
-  std::string name_filter; // limit resultto one object and its instances
+  std::string name_filter; // limit result to one object and its instances
   std::string marker;
 
   rgw_cls_bi_list_op() : max(0) {}
diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 5f94b9918fa4..7b6b46101ccc 100644
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -132,7 +132,7 @@ inline uint64_t cls_rgw_get_rounded_size(uint64_t size) {
  * path that ends with a delimiter and appends a new character to the
  * end such that when a we request bucket-index entries *after* this,
  * we'll get the next object after the "subdirectory". This works
- * because we append a '\xFF' charater, and no valid UTF-8 character
+ * because we append a '\xFF' character, and no valid UTF-8 character
  * can contain that byte, so no valid entries can be skipped.
  */
 inline std::string cls_rgw_after_delim(const std::string& path) {
@@ -181,7 +181,7 @@ enum class RGWObjCategory : uint8_t {
 
   Main      = 1,  // b-i entries for standard objs
 
-  Shadow    = 2,  // presumfably intended for multipart shadow
+  Shadow    = 2,  // presumably intended for multipart shadow
                   // uploads; not currently used in the codebase
 
   MultiMeta = 3,  // b-i entries for multipart upload metadata objs
diff --git a/src/rgw/driver/dbstore/common/dbstore.h b/src/rgw/driver/dbstore/common/dbstore.h
index d3bdf8805381..fc436d55df80 100644
--- a/src/rgw/driver/dbstore/common/dbstore.h
+++ b/src/rgw/driver/dbstore/common/dbstore.h
@@ -156,7 +156,7 @@ struct DBOpParams {
  * Difference with above structure is that all 
  * the fields are strings here to accommodate any
  * style identifiers used by backend db. By default
- * initialized with sqlitedb style, can be overriden
+ * initialized with sqlitedb style, can be overridden
  * using InitPrepareParams()
  *
  * These identifiers are used in prepare and bind statements
@@ -605,7 +605,7 @@ class DBOp {
       REFERENCES '{}' (BucketName) ON DELETE CASCADE ON UPDATE CASCADE \n);";
 
     static constexpr std::string_view CreateObjectViewQ =
-      /* This query creats temporary view with entries from ObjectData table which have
+      /* This query creates temporary view with entries from ObjectData table which have
        * corresponding head object (i.e, with same ObjName, ObjInstance, ObjNS, ObjID)
        * in the Object table.
        *
@@ -711,8 +711,8 @@ class InsertUserOp : virtual public DBOp {
   private:
     /* For existing entires, -
      * (1) INSERT or REPLACE - it will delete previous entry and then
-     * inserts new one. Since it deletes previos enties, it will
-     * trigger all foriegn key cascade deletes or other triggers.
+     * inserts new one. Since it deletes previous entries, it will
+     * trigger all foreign key cascade deletes or other triggers.
      * (2) INSERT or UPDATE - this will set NULL values to unassigned
      * fields.
      * more info: https://code-examples.net/en/q/377728
diff --git a/src/rgw/driver/dbstore/sqlite/sqliteDB.cc b/src/rgw/driver/dbstore/sqlite/sqliteDB.cc
index dc244c07b343..81c716c27f18 100644
--- a/src/rgw/driver/dbstore/sqlite/sqliteDB.cc
+++ b/src/rgw/driver/dbstore/sqlite/sqliteDB.cc
@@ -758,7 +758,7 @@ int SQLiteDB::createUserTable(const DoutPrefixProvider *dpp, DBOpParams *params)
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateUserTable failed" << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateUserTable suceeded" << dendl;
+  ldpp_dout(dpp, 20)<<"CreateUserTable succeeded" << dendl;
 
   return ret;
 }
@@ -774,7 +774,7 @@ int SQLiteDB::createBucketTable(const DoutPrefixProvider *dpp, DBOpParams *param
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateBucketTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateBucketTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateBucketTable succeeded " << dendl;
 
   return ret;
 }
@@ -790,7 +790,7 @@ int SQLiteDB::createObjectTable(const DoutPrefixProvider *dpp, DBOpParams *param
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateObjectTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateObjectTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateObjectTable succeeded " << dendl;
 
   return ret;
 }
@@ -806,7 +806,7 @@ int SQLiteDB::createObjectTableTrigger(const DoutPrefixProvider *dpp, DBOpParams
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateObjectTableTrigger failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateObjectTableTrigger suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateObjectTableTrigger succeeded " << dendl;
 
   return ret;
 }
@@ -822,7 +822,7 @@ int SQLiteDB::createObjectView(const DoutPrefixProvider *dpp, DBOpParams *params
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateObjectView failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateObjectView suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateObjectView succeeded " << dendl;
 
   return ret;
 }
@@ -838,7 +838,7 @@ int SQLiteDB::createQuotaTable(const DoutPrefixProvider *dpp, DBOpParams *params
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateQuotaTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateQuotaTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateQuotaTable succeeded " << dendl;
 
   return ret;
 }
@@ -854,7 +854,7 @@ int SQLiteDB::createObjectDataTable(const DoutPrefixProvider *dpp, DBOpParams *p
   if (ret)
     ldpp_dout(dpp, 0)<<"CreateObjectDataTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"CreateObjectDataTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"CreateObjectDataTable succeeded " << dendl;
 
   return ret;
 }
@@ -875,7 +875,7 @@ int SQLiteDB::createLCTables(const DoutPrefixProvider *dpp)
     ldpp_dout(dpp, 0)<<"CreateLCEntryTable failed" << dendl;
     return ret;
   }
-  ldpp_dout(dpp, 20)<<"CreateLCEntryTable suceeded" << dendl;
+  ldpp_dout(dpp, 20)<<"CreateLCEntryTable succeeded" << dendl;
 
   schema = CreateTableSchema("LCHead", &params);
   ret = exec(dpp, schema.c_str(), NULL);
@@ -883,7 +883,7 @@ int SQLiteDB::createLCTables(const DoutPrefixProvider *dpp)
     ldpp_dout(dpp, 0)<<"CreateLCHeadTable failed" << dendl;
     (void)DeleteLCEntryTable(dpp, &params);
   }
-  ldpp_dout(dpp, 20)<<"CreateLCHeadTable suceeded" << dendl;
+  ldpp_dout(dpp, 20)<<"CreateLCHeadTable succeeded" << dendl;
 
   return ret;
 }
@@ -899,7 +899,7 @@ int SQLiteDB::DeleteUserTable(const DoutPrefixProvider *dpp, DBOpParams *params)
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteUserTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"DeleteUserTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteUserTable succeeded " << dendl;
 
   return ret;
 }
@@ -915,7 +915,7 @@ int SQLiteDB::DeleteBucketTable(const DoutPrefixProvider *dpp, DBOpParams *param
   if (ret)
     ldpp_dout(dpp, 0)<<"DeletebucketTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"DeletebucketTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeletebucketTable succeeded " << dendl;
 
   return ret;
 }
@@ -931,7 +931,7 @@ int SQLiteDB::DeleteObjectTable(const DoutPrefixProvider *dpp, DBOpParams *param
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteObjectTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"DeleteObjectTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteObjectTable succeeded " << dendl;
 
   return ret;
 }
@@ -947,7 +947,7 @@ int SQLiteDB::DeleteObjectDataTable(const DoutPrefixProvider *dpp, DBOpParams *p
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteObjectDataTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"DeleteObjectDataTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteObjectDataTable succeeded " << dendl;
 
   return ret;
 }
@@ -963,7 +963,7 @@ int SQLiteDB::DeleteQuotaTable(const DoutPrefixProvider *dpp, DBOpParams *params
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteQuotaTable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"DeleteQuotaTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteQuotaTable succeeded " << dendl;
 
   return ret;
 }
@@ -977,7 +977,7 @@ int SQLiteDB::DeleteLCEntryTable(const DoutPrefixProvider *dpp, DBOpParams *para
   ret = exec(dpp, schema.c_str(), NULL);
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteLCEntryTable failed " << dendl;
-  ldpp_dout(dpp, 20)<<"DeleteLCEntryTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteLCEntryTable succeeded " << dendl;
 
   return ret;
 }
@@ -991,7 +991,7 @@ int SQLiteDB::DeleteLCHeadTable(const DoutPrefixProvider *dpp, DBOpParams *param
   ret = exec(dpp, schema.c_str(), NULL);
   if (ret)
     ldpp_dout(dpp, 0)<<"DeleteLCHeadTable failed " << dendl;
-  ldpp_dout(dpp, 20)<<"DeleteLCHeadTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"DeleteLCHeadTable succeeded " << dendl;
 
   return ret;
 }
@@ -1006,7 +1006,7 @@ int SQLiteDB::ListAllUsers(const DoutPrefixProvider *dpp, DBOpParams *params)
   if (ret)
     ldpp_dout(dpp, 0)<<"GetUsertable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"GetUserTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"GetUserTable succeeded " << dendl;
 
   return ret;
 }
@@ -1022,7 +1022,7 @@ int SQLiteDB::ListAllBuckets(const DoutPrefixProvider *dpp, DBOpParams *params)
   if (ret)
     ldpp_dout(dpp, 0)<<"Listbuckettable failed " << dendl;
 
-  ldpp_dout(dpp, 20)<<"ListbucketTable suceeded " << dendl;
+  ldpp_dout(dpp, 20)<<"ListbucketTable succeeded " << dendl;
 
   return ret;
 }
@@ -1049,7 +1049,7 @@ int SQLiteDB::ListAllObjects(const DoutPrefixProvider *dpp, DBOpParams *params)
     if (ret)
       ldpp_dout(dpp, 0)<<"ListObjecttable failed " << dendl;
 
-    ldpp_dout(dpp, 20)<<"ListObjectTable suceeded " << dendl;
+    ldpp_dout(dpp, 20)<<"ListObjectTable succeeded " << dendl;
   }
 
   return ret;
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 8cc8df987d2b..3f5a46e4bc9e 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -386,7 +386,7 @@ int MotrUser::store_user(const DoutPrefixProvider* dpp,
   orig_info.user_id = info.user_id;
   // XXX: we open and close motr idx 2 times in this method:
   // 1) on load_user_from_idx() here and 2) on do_idx_op_by_name(PUT) below.
-  // Maybe this can be optimised later somewhow.
+  // Maybe this can be optimised later somehow.
   int rc = load_user_from_idx(dpp, store, orig_info, nullptr, &objv_tr);
   ldpp_dout(dpp, 10) << "Get user: rc = " << rc << dendl;
 
@@ -874,7 +874,7 @@ int MotrBucket::trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch,
 
 int MotrBucket::remove_objs_from_index(const DoutPrefixProvider *dpp, std::list<rgw_obj_index_key>& objs_to_unlink)
 {
-  /* XXX: CHECK: Unlike RadosStore, there is no seperate bucket index table.
+  /* XXX: CHECK: Unlike RadosStore, there is no separate bucket index table.
    * Delete all the object in the list from the object table of this
    * bucket
    */
@@ -1014,7 +1014,7 @@ int MotrBucket::list_multiparts(const DoutPrefixProvider *dpp,
     if (prefix.size() &&
         (0 != ent.key.name.compare(0, prefix.size(), prefix))) {
       ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ <<
-        ": skippping \"" << ent.key <<
+        ": skipping \"" << ent.key <<
         "\" because doesn't match prefix" << dendl;
       continue;
     }
@@ -2338,7 +2338,7 @@ int MotrAtomicWriter::complete(size_t accounted_size, const std::string& etag,
   bufferlist bl;
   rgw_bucket_dir_entry ent;
 
-  // Set rgw_bucet_dir_entry. Some of the member of this structure may not
+  // Set rgw_bucket_dir_entry. Some of the member of this structure may not
   // apply to motr. For example the storage_class.
   //
   // Checkout AtomicObjectProcessor::complete() in rgw_putobj_processor.cc
@@ -2838,7 +2838,7 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp,
   // Update the dir entry and insert it to the bucket index so
   // the object will be seen when listing the bucket.
   bufferlist update_bl;
-  target_obj->get_key().get_index_key(&ent.key);  // Change to offical name :)
+  target_obj->get_key().get_index_key(&ent.key);  // Change to official name :)
   ent.meta.size = off;
   ent.meta.accounted_size = accounted_size;
   ldpp_dout(dpp, 20) << "MotrMultipartUpload::complete(): obj size=" << ent.meta.size
@@ -3667,7 +3667,7 @@ int MotrStore::open_motr_idx(struct m0_uint128 *id, struct m0_idx *idx)
   return 0;
 }
 
-// The following marcos are from dix/fid_convert.h which are not exposed.
+// The following macros are from dix/fid_convert.h which are not exposed.
 enum {
       M0_DIX_FID_DEVICE_ID_OFFSET   = 32,
       M0_DIX_FID_DIX_CONTAINER_MASK = (1ULL << M0_DIX_FID_DEVICE_ID_OFFSET)
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 3a110e36f173..ecfabfc15181 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -70,11 +70,11 @@ class MotrMetaCache
   // of RGW instances under heavy use. If you would like to turn off cache expiry,
   // set this value to zero.
   //
-  // Currently POC hasn't implemented the watch-notify menchanism yet. So the
+  // Currently POC hasn't implemented the watch-notify mechanism yet. So the
   // current implementation is similar to cortx-s3server which is based on expiry
   // time. TODO: see comments on distribute_cache).
   //
-  // Beaware: Motr object data is not cached in current POC as RGW!
+  // Beware: Motr object data is not cached in current POC as RGW!
   // RGW caches the first chunk (4MB by default).
   ObjectCache cache;
 
@@ -877,7 +877,7 @@ class MotrMultipartWriter : public StoreWriter {
 // object part index and for each part an entry is created in extended index.
 // The entry for the object is created in bucket (object list) index. The part
 // index is deleted and an entry removed from bucket_nnn_multipart_index. Like
-// bucket multipart index, bucket part extened metadata index is created during
+// bucket multipart index, bucket part extend metadata index is created during
 // bucket creation.
 //
 // The extended metadata index is used mainly due to fault tolerant
diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h
index 857d530aab50..706683e6bd10 100644
--- a/src/rgw/driver/rados/rgw_bucket.h
+++ b/src/rgw/driver/rados/rgw_bucket.h
@@ -110,7 +110,7 @@ class RGWBucketInstanceMetadataObject : public RGWMetadataObject {
 };
 
 /**
- * store a list of the user's buckets, with associated functinos.
+ * store a list of the user's buckets, with associated functions.
  */
 class RGWUserBuckets {
   std::map<std::string, RGWBucketEnt> buckets;
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index 1baff16ff42e..e7174908dd77 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -795,7 +795,7 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
   } else {
       // r >= 0
       if (bytes_transferred) {
-        // send notification that object was succesfully synced
+        // send notification that object was successfully synced
         std::string user_id = "rgw sync";
         std::string req_id = "0";
         		
diff --git a/src/rgw/driver/rados/rgw_d3n_datacache.cc b/src/rgw/driver/rados/rgw_d3n_datacache.cc
index e07b5f4bd6bf..18d213cf6c34 100644
--- a/src/rgw/driver/rados/rgw_d3n_datacache.cc
+++ b/src/rgw/driver/rados/rgw_d3n_datacache.cc
@@ -143,11 +143,11 @@ int D3nDataCache::d3n_io_write(bufferlist& bl, unsigned int len, std::string oid
 
   // Check whether fclose returned an error
   if (r != 0) {
-    ldout(cct, 0) << "ERROR: D3nDataCache::fclsoe file has return error, errno=" << errno << dendl;
+    ldout(cct, 0) << "ERROR: D3nDataCache::fclose file has return error, errno=" << errno << dendl;
     return -errno;
   }
 
-  { // update cahce_map entries for new chunk in cache
+  { // update cache_map entries for new chunk in cache
     const std::lock_guard l(d3n_cache_lock);
     chunk_info = new D3nChunkDataInfo;
     chunk_info->oid = oid;
@@ -294,7 +294,7 @@ bool D3nDataCache::get(const string& oid, const off_t len)
     struct D3nChunkDataInfo* chdo = iter->second;
     struct stat st;
     int r = stat(location.c_str(), &st);
-    if ( r != -1 && st.st_size == len) { // file exists and containes required data range length
+    if ( r != -1 && st.st_size == len) { // file exists and contains required data range length
       exist = true;
       /*LRU*/
       /*get D3nChunkDataInfo*/
diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index bbd3aaf448da..02965f9073d6 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -3515,7 +3515,7 @@ class CheckBucketShardStatusIsIncremental : public RGWReadBucketPipeSyncStatusCo
 
 class CheckAllBucketShardStatusIsIncremental : public RGWShardCollectCR {
   // start with 1 shard, and only spawn more if we detect an existing shard.
-  // this makes the backward compatilibility check far less expensive in the
+  // this makes the backward compatibility check far less expensive in the
   // general case where no shards exist
   static constexpr int initial_concurrent_shards = 1;
   static constexpr int max_concurrent_shards = 16;
@@ -4320,7 +4320,7 @@ class RGWBucketIncSyncShardMarkerTrack : public RGWSyncShardMarkerTrack<string,
    * create index from key -> <op, marker>, and from marker -> key
    * this is useful so that we can insure that we only have one
    * entry for any key that is used. This is needed when doing
-   * incremenatl sync of data, and we don't want to run multiple
+   * incremental sync of data, and we don't want to run multiple
    * concurrent sync operations for the same bucket shard 
    * Also, we should make sure that we don't run concurrent operations on the same key with
    * different ops.
@@ -5911,7 +5911,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp)
             return set_cr_error(retcode);
           }
           if (bucket_status.state != BucketSyncState::Stopped) {
-            // make sure that state is changed to stopped localy
+            // make sure that state is changed to stopped locally
             bucket_status.state = BucketSyncState::Stopped;
             yield call(new WriteCR(dpp, env->driver, status_obj, bucket_status,
 				   &objv, false));
diff --git a/src/rgw/driver/rados/rgw_datalog.cc b/src/rgw/driver/rados/rgw_datalog.cc
index 167cbcdba4a8..9274dc2ae9be 100644
--- a/src/rgw/driver/rados/rgw_datalog.cc
+++ b/src/rgw/driver/rados/rgw_datalog.cc
@@ -618,7 +618,7 @@ void RGWDataChangesLog::update_renewed(const rgw_bucket_shard& bs,
   auto status = _get_change(bs, gen);
   l.unlock();
 
-  ldout(cct, 20) << "RGWDataChangesLog::update_renewd() bucket_name="
+  ldout(cct, 20) << "RGWDataChangesLog::update_renewed() bucket_name="
 		 << bs.bucket.name << " shard_id=" << bs.shard_id
 		 << " expiration=" << expiration << dendl;
 
diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc
index c52acef654f3..2dd8a6625fcc 100644
--- a/src/rgw/driver/rados/rgw_lc_tier.cc
+++ b/src/rgw/driver/rados/rgw_lc_tier.cc
@@ -454,7 +454,7 @@ int RGWLCStreamRead::init_rest_obj() {
     rest_obj.content_len = m_part_size;
   }
 
-  /* For mulitpart attrs are sent as part of InitMultipartCR itself */
+  /* For multipart attrs are sent as part of InitMultipartCR itself */
   if (multipart) {
     return 0;
   }
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 3935957bc5c3..ecbccc8b220b 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -199,7 +199,7 @@ class Manager : public DoutPrefixProvider {
   std::vector<std::string> entryProcessingResultString = {"Failure", "Successful", "Sleeping", "Expired", "Migrating"};
 
   // processing of a specific entry
-  // return whether processing was successfull (true) or not (false)
+  // return whether processing was successful (true) or not (false)
   EntryProcessingResult process_entry(const ConfigProxy& conf, persistency_tracker& entry_persistency_tracker,
                                       const cls_queue_entry& entry, yield_context yield) {
     event_entry_t event_entry;
@@ -577,7 +577,7 @@ class Manager : public DoutPrefixProvider {
 
       for (const auto& queue_name : queues) {
         // try to lock the queue to check if it is owned by this rgw
-        // or if ownershif needs to be taken
+        // or if ownership needs to be taken
         librados::ObjectWriteOperation op;
         op.assert_exists();
         rados::cls::lock::lock(&op, queue_name+"_lock", 
@@ -612,7 +612,7 @@ class Manager : public DoutPrefixProvider {
           // start processing this queue
           spawn::spawn(io_context, [this, &queue_gc, &queue_gc_lock, queue_name](yield_context yield) {
             process_queue(queue_name, yield);
-            // if queue processing ended, it measn that the queue was removed or not owned anymore
+            // if queue processing ended, it means that the queue was removed or not owned anymore
             // mark it for deletion
             std::lock_guard lock_guard(queue_gc_lock);
             queue_gc.push_back(queue_name);
@@ -790,7 +790,7 @@ int remove_persistent_topic(const std::string& topic_name, optional_yield y) {
   return remove_persistent_topic(s_manager, s_manager->rados_store.getRados()->get_notif_pool_ctx(), topic_name, y);
 }
 
-rgw::sal::Object* get_object_with_atttributes(
+rgw::sal::Object* get_object_with_attributes(
   const reservation_t& res, rgw::sal::Object* obj) {
   // in case of copy obj, the tags and metadata are taken from source
   const auto src_obj = res.src_object ? res.src_object : obj;
@@ -820,7 +820,7 @@ static inline void filter_amz_meta(meta_map_t& dest, const meta_map_t& src) {
 static inline void metadata_from_attributes(
   reservation_t& res, rgw::sal::Object* obj) {
   auto& metadata = res.x_meta_map;
-  const auto src_obj = get_object_with_atttributes(res, obj);
+  const auto src_obj = get_object_with_attributes(res, obj);
   if (!src_obj) {
     return;
   }
@@ -838,7 +838,7 @@ static inline void metadata_from_attributes(
 
 static inline void tags_from_attributes(
   const reservation_t& res, rgw::sal::Object* obj, KeyMultiValueMap& tags) {
-  const auto src_obj = get_object_with_atttributes(res, obj);
+  const auto src_obj = get_object_with_attributes(res, obj);
   if (!src_obj) {
     return;
   }
diff --git a/src/rgw/driver/rados/rgw_notify.h b/src/rgw/driver/rados/rgw_notify.h
index 460a7bacb5da..20f0c17e5bb5 100644
--- a/src/rgw/driver/rados/rgw_notify.h
+++ b/src/rgw/driver/rados/rgw_notify.h
@@ -23,7 +23,7 @@ struct rgw_obj_key;
 namespace rgw::notify {
 
 // initialize the notification manager
-// notification manager is dequeing the 2-phase-commit queues
+// notification manager is dequeuing the 2-phase-commit queues
 // and send the notifications to the endpoints
 bool init(CephContext* cct, rgw::sal::RadosStore* store, const DoutPrefixProvider *dpp);
 
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index b6d08e5c7f38..091adb9830c0 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -189,7 +189,7 @@ RadosWriter::~RadosWriter()
    * Such race condition is caused by the fact that the multipart object is the gatekeeper of a multipart
    * upload, when it is deleted, a second upload would start with the same suffix("2/"), therefore, objects
    * written by the second upload may be deleted by the first upload.
-   * details is describled on #11749
+   * details is described on #11749
    *
    * The above comment still stands, but instead of searching for a specific object in the multipart
    * namespace, we just make sure that we remove the object that is marked as the head object after
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 3ee43dd91a89..2b5f7c4e7a66 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1911,7 +1911,7 @@ int RGWRados::Bucket::List::list_objects_ordered(
       if (cur_end_marker_valid && cur_end_marker <= index_key) {
         truncated = false;
 	ldpp_dout(dpp, 10) << __func__ <<
-	  ": finished due to gitting end marker of \"" << cur_end_marker <<
+	  ": finished due to getting end marker of \"" << cur_end_marker <<
 	  "\" with \"" << entry.key << "\"" << dendl;
         goto done;
       }
@@ -2194,14 +2194,14 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp
 
       if (!params.list_versions && !entry.is_visible()) {
         ldpp_dout(dpp, 20) << __func__ <<
-	  ": skippping \"" << index_key <<
-	  "\" because not listing versions and entry not visibile" << dendl;
+	  ": skipping \"" << index_key <<
+	  "\" because not listing versions and entry not visible" << dendl;
         continue;
       }
 
       if (params.enforce_ns && obj.ns != params.ns) {
         ldpp_dout(dpp, 20) << __func__ <<
-	  ": skippping \"" << index_key <<
+	  ": skipping \"" << index_key <<
 	  "\" because namespace does not match" << dendl;
         continue;
       }
@@ -2210,7 +2210,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp
 	// we're not guaranteed items will come in order, so we have
 	// to loop through all
         ldpp_dout(dpp, 20) << __func__ <<
-	  ": skippping \"" << index_key <<
+	  ": skipping \"" << index_key <<
 	  "\" because after end_marker" << dendl;
 	continue;
       }
@@ -2218,7 +2218,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp
       if (params.access_list_filter &&
 	  !params.access_list_filter->filter(obj.name, index_key.name)) {
         ldpp_dout(dpp, 20) << __func__ <<
-	  ": skippping \"" << index_key <<
+	  ": skipping \"" << index_key <<
 	  "\" because doesn't match filter" << dendl;
         continue;
       }
@@ -2226,7 +2226,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp
       if (params.prefix.size() &&
 	  (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) {
         ldpp_dout(dpp, 20) << __func__ <<
-	  ": skippping \"" << index_key <<
+	  ": skipping \"" << index_key <<
 	  "\" because doesn't match prefix" << dendl;
 	continue;
       }
@@ -3297,7 +3297,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
 
   /* we lost in a race. There are a few options:
    * - existing object was rewritten (ECANCELED)
-   * - non existing object was created (EEXIST)
+   * - nonexistent object was created (EEXIST)
    * - object was removed (ENOENT)
    * should treat it as a success
    */
@@ -4627,7 +4627,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   if (src_attrs.count(RGW_ATTR_CRYPT_MODE)) {
     // Current implementation does not follow S3 spec and even
     // may result in data corruption silently when copying
-    // multipart objects acorss pools. So reject COPY operations
+    // multipart objects across pools. So reject COPY operations
     //on encrypted objects before it is fully functional.
     ldpp_dout(dpp, 0) << "ERROR: copy op for encrypted object " << src_obj
                   << " has not been implemented." << dendl;
@@ -9451,7 +9451,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp,
   for (auto& r : shard_list_results) {
     results_trackers.emplace_back(r.first, r.second, shard_oids[r.first]);
 
-    // if any *one* shard's result is trucated, the entire result is
+    // if any *one* shard's result is truncated, the entire result is
     // truncated
     *is_truncated = *is_truncated || r.second.is_truncated;
 
diff --git a/src/rgw/driver/rados/rgw_rest_log.h b/src/rgw/driver/rados/rgw_rest_log.h
index 02b1d133fc5b..b3a8e49d1b29 100644
--- a/src/rgw/driver/rados/rgw_rest_log.h
+++ b/src/rgw/driver/rados/rgw_rest_log.h
@@ -331,7 +331,7 @@ class RGWRESTMgr_Log : public RGWRESTMgr {
   RGWHandler_REST* get_handler(rgw::sal::Driver* driver,
 			       req_state* const,
                                const rgw::auth::StrategyRegistry& auth_registry,
-                               const std::string& frontend_prefixs) override {
+                               const std::string& frontend_prefixes) override {
     return new RGWHandler_Log(auth_registry);
   }
 };
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index b60ddd7a4d18..478d7a5b7067 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -330,7 +330,7 @@ int RadosBucket::remove(const DoutPrefixProvider* dpp,
   RGWObjVersionTracker ot;
 
   // if we deleted children above we will force delete, as any that
-  // remain is detrius from a prior bug
+  // remain is detritus from a prior bug
   ret = store->getRados()->delete_bucket(info, ot, y, dpp, !delete_children);
   if (ret < 0) {
     ldpp_dout(dpp, -1) << "ERROR: could not remove bucket " <<
@@ -3141,7 +3141,7 @@ int RadosLuaManager::remove_package(const DoutPrefixProvider *dpp, optional_yiel
   librados::ObjectWriteOperation op;
   size_t pos = package_name.find(" ");
   if (pos != package_name.npos) {
-    // remove specfic version of the the package
+    // remove specific version of the the package
     op.omap_rm_keys(std::set<std::string>({package_name}));
     auto ret = rgw_rados_operate(dpp, ioctx,
         PACKAGE_LIST_OBJECT_NAME, &op, y);
diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc
index c973a039efc8..251e9b395e0c 100644
--- a/src/rgw/driver/rados/rgw_sync.cc
+++ b/src/rgw/driver/rados/rgw_sync.cc
@@ -1767,7 +1767,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
                                                          sync_marker, tn));
 
       /*
-       * mdlog_marker: the remote sync marker positiion
+       * mdlog_marker: the remote sync marker position
        * sync_marker: the local sync marker position
        * max_marker: the max mdlog position that we fetched
        * marker: the current position we try to sync
diff --git a/src/rgw/driver/rados/rgw_sync_module_aws.cc b/src/rgw/driver/rados/rgw_sync_module_aws.cc
index cdcd831e9892..46b99968a506 100644
--- a/src/rgw/driver/rados/rgw_sync_module_aws.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_aws.cc
@@ -60,7 +60,7 @@ static string obj_to_aws_path(const rgw_obj& obj)
            
 
       # anything below here is for non trivial configuration 
-      # can be used in conjuction with the above
+      # can be used in conjunction with the above
 
       "default": {
         "connection": {
@@ -107,7 +107,7 @@ static string obj_to_aws_path(const rgw_obj& obj)
 target path optional variables:
 
 (evaluated at init)
-sid: sync instance id, randomly generated by sync process on first sync initalization
+sid: sync instance id, randomly generated by sync process on first sync initialization
 zonegroup: zonegroup name
 zonegroup_id: zonegroup name
 zone: zone name
diff --git a/src/rgw/driver/rados/rgw_sync_module_es.cc b/src/rgw/driver/rados/rgw_sync_module_es.cc
index 4e8eb6201db4..13c2057bf779 100644
--- a/src/rgw/driver/rados/rgw_sync_module_es.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_es.cc
@@ -608,7 +608,7 @@ struct es_obj_metadata {
       f->open_array_section("custom-date");
       for (auto i : custom_date) {
         /*
-         * try to exlicitly parse date field, otherwise elasticsearch could reject the whole doc,
+         * try to explicitly parse date field, otherwise elasticsearch could reject the whole doc,
          * which will end up with failed sync
          */
         real_time t;
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 1f837e312c09..446f1efda91a 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -318,7 +318,7 @@ struct RGWZoneGroup : public RGWSystemMetaObj {
   // TODO: Maybe convert hostnames to a map<std::string,std::list<std::string>> for
   // endpoint_type->hostnames
 /*
-20:05 < _robbat21irssi> maybe I do someting like: if (hostname_map.empty()) { populate all map keys from hostnames; };
+20:05 < _robbat21irssi> maybe I do something like: if (hostname_map.empty()) { populate all map keys from hostnames; };
 20:05 < _robbat21irssi> but that's a later compatability migration planning bit
 20:06 < yehudasa> more like if (!hostnames.empty()) {
 20:06 < yehudasa> for (std::list<std::string>::iterator iter = hostnames.begin(); iter != hostnames.end(); ++iter) {
diff --git a/src/rgw/jwt-cpp/jwt.h b/src/rgw/jwt-cpp/jwt.h
index b86fb57b032f..352b80010b86 100644
--- a/src/rgw/jwt-cpp/jwt.h
+++ b/src/rgw/jwt-cpp/jwt.h
@@ -74,7 +74,7 @@ namespace jwt {
 	namespace helper {
 		inline
 		std::string extract_pubkey_from_cert(const std::string& certstr, const std::string& pw = "") {
-			// TODO: Cannot find the exact version this change happended
+			// TODO: Cannot find the exact version this change happened
 #if OPENSSL_VERSION_NUMBER <= 0x1000114fL
 			std::unique_ptr<BIO, decltype(&BIO_free_all)> certbio(BIO_new_mem_buf(const_cast<char*>(certstr.data()), certstr.size()), BIO_free_all);
 #else
@@ -203,7 +203,7 @@ namespace jwt {
 				return alg_name;
 			}
 		private:
-			/// HMAC secrect
+			/// HMAC secret
 			const std::string secret;
 			/// HMAC hash generator
 			const EVP_MD*(*md)();
@@ -821,7 +821,7 @@ namespace jwt {
 		/**
 		 * Get type of contained object
 		 * \return Type
-		 * \throws std::logic_error An internal error occured
+		 * \throws std::logic_error An internal error occurred
 		 */
 		type get_type() const {
 			if (val.is<picojson::null>()) return type::null;
@@ -1045,7 +1045,7 @@ namespace jwt {
 		std::unordered_map<std::string, claim> header_claims;
 	public:
 		/**
-		 * Check if algortihm is present ("alg")
+		 * Check if algorithm is present ("alg")
 		 * \return true if present, false otherwise
 		 */
 		bool has_algorithm() const noexcept { return has_header_claim("alg"); }
@@ -1119,7 +1119,7 @@ namespace jwt {
 	 */
 	class decoded_jwt : public header, public payload {
 	protected:
-		/// Unmodifed token, as passed to constructor
+		/// Unmodified token, as passed to constructor
 		const std::string token;
 		/// Header part decoded from base64
 		std::string header;
diff --git a/src/rgw/rgw-gap-list b/src/rgw/rgw-gap-list
index 5018cedd7cb6..983e89305779 100755
--- a/src/rgw/rgw-gap-list
+++ b/src/rgw/rgw-gap-list
@@ -2,7 +2,7 @@
 
 # Last revision 2023-01-13
 
-# NOTE: This script based based on rgw-orphan-list but doing the
+# NOTE: This script based on rgw-orphan-list but doing the
 # reverse calculation.
 
 # NOTE: The awk included in this script replaces the 'ceph-diff-sorted'
@@ -276,7 +276,7 @@ for myfile in $rados_out $rgwadmin_out; do
   fi 
 done
 
-# Create an awk script in a file for parsing the two command outoputs.
+# Create an awk script in a file for parsing the two command outputs.
 log "Creating awk script for comparing outputs: ${incremental_grep_awk}"
 
 cat <<"EOF" >$incremental_grep_awk
@@ -292,7 +292,7 @@ cat <<"EOF" >$incremental_grep_awk
 #    indicates a possible deleted tail object and the accompanying
 #    bucket / user object name is output, assuming it had not been
 #    previously identified.
-#    - A map of outputed bucket / user object is maintained in memory
+#    - A map of outputted bucket / user object is maintained in memory
 #  * If a value appears in $rados_out, but not in $rgwadmin_out, the
 #    $rados_out file is iterated until the $rados_out line is equal
 #    or > (alphabetically) the value from the $rgwadmin_out file.
diff --git a/src/rgw/rgw-orphan-list b/src/rgw/rgw-orphan-list
index c8856e8eeec5..70557947e5df 100755
--- a/src/rgw/rgw-orphan-list
+++ b/src/rgw/rgw-orphan-list
@@ -159,7 +159,7 @@ rados_ls() {
         namespace_found=1
     fi
 
-    # check for locators (w/o namespace); we idenitfy them by skipping
+    # check for locators (w/o namespace); we identify them by skipping
     # past the empty namespace (i.e., one TAB), skipping past the oid,
     # then looking for a TAB; note we use egrep to get the '+' character
     # and the $ in front of the ' allows the \t to be interpreted as a TAB
diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index f1ca68d637d7..c6f06bbd37cc 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -384,7 +384,7 @@ boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
   std::vector<std::string> readwrite;
   std::vector<std::string> readonly;
 
-  /* Parition the grant map into three not-overlapping groups. */
+  /* Partition the grant map into three not-overlapping groups. */
   for (const auto& item : get_acl().get_grant_map()) {
     const ACLGrant& grant = item.second;
     const uint32_t perm = grant.get_permission().get_permissions();
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 04d085daee59..c19d44c9e536 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2870,11 +2870,11 @@ static int bucket_sync_status(rgw::sal::Driver* driver, const RGWBucketInfo& inf
 
   for (auto& zone_id : zone_ids) {
     auto z = static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->get_zonegroup().zones.find(zone_id.id);
-    if (z == static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->get_zonegroup().zones.end()) { /* should't happen */
+    if (z == static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone->get_zonegroup().zones.end()) { /* shouldn't happen */
       continue;
     }
     auto c = zone_conn_map.find(zone_id.id);
-    if (c == zone_conn_map.end()) { /* should't happen */
+    if (c == zone_conn_map.end()) { /* shouldn't happen */
       continue;
     }
 
@@ -7641,7 +7641,7 @@ int main(int argc, const char **argv)
 
       do {
         entries.clear();
-	// if object is specified, we use that as a filter to only retrieve some some entries
+	// if object is specified, we use that as a filter to only retrieve some entries
         ret = static_cast<rgw::sal::RadosStore*>(driver)->getRados()->bi_list(bs, object, marker, max_entries, &entries, &is_truncated, null_yield);
         if (ret < 0) {
           ldpp_dout(dpp(), 0) << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl;
diff --git a/src/rgw/rgw_amqp.cc b/src/rgw/rgw_amqp.cc
index 3014edd1db09..67cbbe5c6771 100644
--- a/src/rgw/rgw_amqp.cc
+++ b/src/rgw/rgw_amqp.cc
@@ -301,7 +301,7 @@ std::string to_string(amqp_status_enum s) {
   }
 }
 
-// TODO: add status_to_string on the connection object to prinf full status
+// TODO: add status_to_string on the connection object to print full status
 
 // convert int status to string - including RGW specific values
 std::string status_to_string(int s) {
@@ -688,7 +688,7 @@ class Manager {
               // TODO: add exponential backoff for retries
               conn->next_reconnect = now + reconnect_time;
             } else {
-              ldout(cct, 10) << "AMQP run: connection '" << to_string(conn_id) << "' retry successfull" << dendl;
+              ldout(cct, 10) << "AMQP run: connection '" << to_string(conn_id) << "' retry successful" << dendl;
             }
           }
           INCREMENT_AND_CONTINUE(conn_it);
diff --git a/src/rgw/rgw_asio_client.cc b/src/rgw/rgw_asio_client.cc
index 51d43140ba56..145e16139d7e 100644
--- a/src/rgw/rgw_asio_client.cc
+++ b/src/rgw/rgw_asio_client.cc
@@ -119,9 +119,9 @@ size_t ClientIO::send_status(int status, const char* status_name)
 
 size_t ClientIO::send_100_continue()
 {
-  const char HTTTP_100_CONTINUE[] = "HTTP/1.1 100 CONTINUE\r\n\r\n";
-  const size_t sent = txbuf.sputn(HTTTP_100_CONTINUE,
-                                  sizeof(HTTTP_100_CONTINUE) - 1);
+  const char HTTP_100_CONTINUE[] = "HTTP/1.1 100 CONTINUE\r\n\r\n";
+  const size_t sent = txbuf.sputn(HTTP_100_CONTINUE,
+                                  sizeof(HTTP_100_CONTINUE) - 1);
   flush();
   sent100continue = true;
   return sent;
diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index e7d39d7cece6..23ccfcafeb08 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -513,7 +513,7 @@ tcp::endpoint parse_endpoint(boost::asio::string_view input,
       return endpoint;
     }
     if (addr_end + 1 < input.size()) {
-      // :port must must follow [ipv6]
+      // :port must follow [ipv6]
       if (input[addr_end + 1] != ':') {
         ec = boost::asio::error::invalid_argument;
         return endpoint;
diff --git a/src/rgw/rgw_auth.h b/src/rgw/rgw_auth.h
index 82e0d0c9755a..2668af3a3c1d 100644
--- a/src/rgw/rgw_auth.h
+++ b/src/rgw/rgw_auth.h
@@ -220,7 +220,7 @@ class Engine {
         reason(reason) {
     }
 
-    /* Allow only the reasonable combintations - returning just Completer
+    /* Allow only the reasonable combinations - returning just Completer
      * without accompanying IdentityApplier is strictly prohibited! */
     explicit AuthResult(IdentityApplier::aplptr_t&& applier)
       : result_pair(std::move(applier), nullptr) {
@@ -236,7 +236,7 @@ class Engine {
       /* Engine doesn't grant the access but also doesn't reject it. */
       DENIED,
 
-      /* Engine successfully authenicated requester. */
+      /* Engine successfully authenticated requester. */
       GRANTED,
 
       /* Engine strictly indicates that a request should be rejected
@@ -311,7 +311,7 @@ class TokenExtractor {
 
 /* Abstract class for stacking sub-engines to expose them as a single
  * Engine. It is responsible for ordering its sub-engines and managing
- * fall-backs between them. Derivatee is supposed to encapsulate engine
+ * fall-backs between them. Derivative is supposed to encapsulate engine
  * instances and add them using the add_engine() method in the order it
  * wants to be tried during the call to authenticate().
  *
diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc
index a1d76c3aaf32..552159823bb4 100644
--- a/src/rgw/rgw_auth_keystone.cc
+++ b/src/rgw/rgw_auth_keystone.cc
@@ -662,7 +662,7 @@ rgw::auth::Engine::result_t EC2Engine::authenticate(
   const string_to_sign_t& string_to_sign,
   const signature_factory_t& signature_factory,
   const completer_factory_t& completer_factory,
-  /* Passthorugh only! */
+  /* Passthrough only! */
   const req_state* s,
   optional_yield y) const
 {
diff --git a/src/rgw/rgw_basic_types.h b/src/rgw/rgw_basic_types.h
index 4cffb333288e..a8190aa35ada 100644
--- a/src/rgw/rgw_basic_types.h
+++ b/src/rgw/rgw_basic_types.h
@@ -66,12 +66,12 @@ struct rgw_zone_id {
   rgw_zone_id(std::string&& _id) : id(std::move(_id)) {}
 
   void encode(ceph::buffer::list& bl) const {
-    /* backward compatiblity, not using ENCODE_{START,END} macros */
+    /* backward compatibility, not using ENCODE_{START,END} macros */
     ceph::encode(id, bl);
   }
 
   void decode(ceph::buffer::list::const_iterator& bl) {
-    /* backward compatiblity, not using DECODE_{START,END} macros */
+    /* backward compatibility, not using DECODE_{START,END} macros */
     ceph::decode(id, bl);
   }
 
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 26557afd8d69..6c07e3232435 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -201,10 +201,10 @@ is_err() const
 // S3 authorization and some other processes depending on the requestURI
 // The absoluteURI can start with "http://", "https://", "ws://" or "wss://"
 static string get_abs_path(const string& request_uri) {
-  const static string ABS_PREFIXS[] = {"http://", "https://", "ws://", "wss://"};
+  const static string ABS_PREFIXES[] = {"http://", "https://", "ws://", "wss://"};
   bool isAbs = false;
   for (int i = 0; i < 4; ++i) {
-    if (boost::algorithm::starts_with(request_uri, ABS_PREFIXS[i])) {
+    if (boost::algorithm::starts_with(request_uri, ABS_PREFIXES[i])) {
       isAbs = true;
       break;
     } 
@@ -351,7 +351,7 @@ void set_req_state_err(req_state* s, int err_no, const string& err_msg)
       /* TODO(rzarzynski): there never ever should be a check like this one.
        * It's here only for the sake of the patch's backportability. Further
        * commits will move the logic to a per-RGWHandler replacement of
-       * the end_header() function. Alternativaly, we might consider making
+       * the end_header() function. Alternatively, we might consider making
        * that just for the dump(). Please take a look on @cbodley's comments
        * in PR #10690 (https://github.com/ceph/ceph/pull/10690). */
       s->err.err_code = err_msg;
@@ -2456,7 +2456,7 @@ void RGWBucketEnt::dump(Formatter *f) const
   encode_json("size", size, f);
   encode_json("size_rounded", size_rounded, f);
   utime_t ut(creation_time);
-  encode_json("mtime", ut, f); /* mtime / creation time discrepency needed for backward compatibility */
+  encode_json("mtime", ut, f); /* mtime / creation time discrepancy needed for backward compatibility */
   encode_json("count", count, f);
   encode_json("placement_rule", placement_rule.to_str(), f);
 }
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index ee17e2c8d43e..85d5ab2c155b 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1531,14 +1531,14 @@ struct perm_state_base {
                   const RGWBucketInfo& _bucket_info,
                   int _perm_mask,
                   bool _defer_to_bucket_acls,
-                  boost::optional<PublicAccessBlockConfiguration> _bucket_acess_conf = boost::none) :
+                  boost::optional<PublicAccessBlockConfiguration> _bucket_access_conf = boost::none) :
                                                 cct(_cct),
                                                 env(_env),
                                                 identity(_identity),
                                                 bucket_info(_bucket_info),
                                                 perm_mask(_perm_mask),
                                                 defer_to_bucket_acls(_defer_to_bucket_acls),
-                                                bucket_access_conf(_bucket_acess_conf)
+                                                bucket_access_conf(_bucket_access_conf)
   {}
 
   virtual ~perm_state_base() {}
@@ -1684,7 +1684,7 @@ extern std::string url_decode(const std::string_view& src_str,
 extern void url_encode(const std::string& src, std::string& dst,
                        bool encode_slash = true);
 extern std::string url_encode(const std::string& src, bool encode_slash = true);
-extern std::string url_remove_prefix(const std::string& url); // Removes hhtp, https and www from url
+extern std::string url_remove_prefix(const std::string& url); // Removes http, https and www from url
 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
 extern void calc_hmac_sha1(const char *key, int key_len,
                           const char *msg, int msg_len, char *dest);
diff --git a/src/rgw/rgw_crypt.cc b/src/rgw/rgw_crypt.cc
index 6bc4bb9c1675..085fda0a44be 100644
--- a/src/rgw/rgw_crypt.cc
+++ b/src/rgw/rgw_crypt.cc
@@ -1535,7 +1535,7 @@ int rgw_remove_sse_s3_bucket_key(req_state *s, optional_yield y)
 *	I've left some commented out lines above.  They are there for
 *	a reason, which I will explain.  The "canonical" json constructed
 *	by the code above as a crypto context must take a json object and
-*	turn it into a unique determinstic fixed form.  For most json
+*	turn it into a unique deterministic fixed form.  For most json
 *	types this is easy.  The hardest problem that is handled above is
 *	detailing with unicode strings; they must be turned into
 *	NFC form and sorted in a fixed order.  Numbers, however,
diff --git a/src/rgw/rgw_env.cc b/src/rgw/rgw_env.cc
index d528f0e6d479..2415a93c34fb 100644
--- a/src/rgw/rgw_env.cc
+++ b/src/rgw/rgw_env.cc
@@ -113,7 +113,7 @@ size_t RGWEnv::get_size(const char *name, size_t def_val) const
     sz = stoull(iter->second);
   } catch(...){
     /* it is very unlikely that we'll ever encounter out_of_range, but let's
-       return the default eitherway */
+       return the default either way */
     sz = def_val;
   }
 
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 6130019df032..e1217173feda 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -182,7 +182,7 @@ namespace rgw {
     LookupFHResult fhr{nullptr, 0};
 
     /* XXX the need for two round-trip operations to identify file or
-     * directory leaf objects is unecessary--the current proposed
+     * directory leaf objects is unnecessary--the current proposed
      * mechanism to avoid this is to store leaf object names with an
      * object locator w/o trailing slash */
 
@@ -415,7 +415,7 @@ namespace rgw {
        * leaf object
        */
       if (! rgw_fh) {
-	/* XXX for now, peform a hard lookup to deduce the type of
+	/* XXX for now, perform a hard lookup to deduce the type of
 	 * object to be deleted ("foo" vs. "foo/")--also, ensures
 	 * atomicity at this endpoint */
 	struct rgw_file_handle *fh;
@@ -1510,7 +1510,7 @@ namespace rgw {
     if (factory == nullptr) {
       return false;
     }
-    /* make sure the reclaiming object is the same partiton with newobject factory,
+    /* make sure the reclaiming object is the same partition with newobject factory,
      * then we can recycle the object, and replace with newobject */
     if (!fs->fh_cache.is_same_partition(factory->fhk.fh_hk.object, fh.fh_hk.object)) {
       return false;
diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc
index eb8662d58217..3a118ed78d24 100644
--- a/src/rgw/rgw_iam_policy.cc
+++ b/src/rgw/rgw_iam_policy.cc
@@ -541,7 +541,7 @@ bool ParseState::do_string(CephContext* cct, const char* s, size_t l) {
   auto k = pp->tokens.lookup(s, l);
   Policy& p = pp->policy;
   bool is_action = false;
-  bool is_validaction = false;
+  bool is_valid_action = false;
   Statement* t = p.statements.empty() ? nullptr : &(p.statements.back());
 
   // Top level!
@@ -579,13 +579,13 @@ bool ParseState::do_string(CephContext* cct, const char* s, size_t l) {
 	     (w->id == TokenID::NotAction)) {
     is_action = true;
     if (*s == '*') {
-      is_validaction = true;
+      is_valid_action = true;
       (w->id == TokenID::Action ?
         t->action = allValue : t->notaction = allValue);
     } else {
       for (auto& p : actpairs) {
         if (match_policy({s, l}, p.name, MATCH_POLICY_ACTION)) {
-          is_validaction = true;
+          is_valid_action = true;
           (w->id == TokenID::Action ? t->action[p.bit] = 1 : t->notaction[p.bit] = 1);
         }
         if ((t->action & s3AllValue) == s3AllValue) {
@@ -687,7 +687,7 @@ bool ParseState::do_string(CephContext* cct, const char* s, size_t l) {
     pp->s.pop_back();
   }
 
-  if (is_action && !is_validaction) {
+  if (is_action && !is_valid_action) {
     annotate(fmt::format("`{}` is not a valid action.",
 			 std::string_view{s, l}));
     return false;
diff --git a/src/rgw/rgw_kafka.cc b/src/rgw/rgw_kafka.cc
index 832c4667b94e..eb6b85071a78 100644
--- a/src/rgw/rgw_kafka.cc
+++ b/src/rgw/rgw_kafka.cc
@@ -23,7 +23,7 @@
 // (2) check performance of emptying queue to local list, and go over the list and publish
 // (3) use std::shared_mutex (c++17) or equivalent for the connections lock
 
-// cmparisson operator between topic pointer and name
+// comparison operator between topic pointer and name
 bool operator==(const rd_kafka_topic_t* rkt, const std::string& name) {
     return name == std::string_view(rd_kafka_topic_name(rkt)); 
 }
@@ -69,7 +69,7 @@ struct connection_t {
   CallbackList callbacks;
   const std::string broker;
   const bool use_ssl;
-  const bool verify_ssl; // TODO currently iognored, not supported in librdkafka v0.11.6
+  const bool verify_ssl; // TODO currently ignored, not supported in librdkafka v0.11.6
   const boost::optional<std::string> ca_location;
   const std::string user;
   const std::string password;
@@ -209,7 +209,7 @@ bool new_producer(connection_t* conn) {
     return false;
   }
 
-  // get list of brokers based on the bootsrap broker
+  // get list of brokers based on the bootstrap broker
   if (rd_kafka_conf_set(conn->temp_conf, "bootstrap.servers", conn->broker.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
 
   if (conn->use_ssl) {
@@ -459,7 +459,7 @@ class Manager {
         
         auto& conn = conn_it->second;
 
-        // Checking the connection idlesness
+        // Checking the connection idleness
         if(conn->timestamp.sec() + max_idle_time < ceph_clock_now()) {
           ldout(conn->cct, 20) << "kafka run: deleting a connection due to idle behaviour: " << ceph_clock_now() << dendl;
           std::lock_guard lock(connections_lock);
@@ -478,7 +478,7 @@ class Manager {
             // TODO: add error counter for failed retries
             // TODO: add exponential backoff for retries
           } else {
-            ldout(conn->cct, 10) << "Kafka run: connection (" << broker << ") retry successfull" << dendl;
+            ldout(conn->cct, 10) << "Kafka run: connection (" << broker << ") retry successful" << dendl;
           }
           ++conn_it;
           continue;
@@ -570,7 +570,7 @@ class Manager {
 
     std::lock_guard lock(connections_lock);
     const auto it = connections.find(broker);
-    // note that ssl vs. non-ssl connection to the same host are two separate conenctions
+    // note that ssl vs. non-ssl connection to the same host are two separate connections
     if (it != connections.end()) {
       // connection found - return even if non-ok
       ldout(cct, 20) << "Kafka connect: connection found" << dendl;
diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc
index d0aba2f7832c..7d5264f980ee 100644
--- a/src/rgw/rgw_keystone.cc
+++ b/src/rgw/rgw_keystone.cc
@@ -142,7 +142,7 @@ int Service::get_admin_token(const DoutPrefixProvider *dpp,
                              optional_yield y,
                              std::string& token)
 {
-  /* Let's check whether someone uses the deprecated "admin token" feauture
+  /* Let's check whether someone uses the deprecated "admin token" feature
    * based on a shared secret from keystone.conf file. */
   const auto& admin_token = config.get_admin_token();
   if (! admin_token.empty()) {
@@ -345,7 +345,7 @@ int TokenEnvelope::parse(const DoutPrefixProvider *dpp,
          * speaks in v2 disregarding the promise to go with v3. */
         decode_v3(*token_iter);
 
-        /* Identity v3 conveys the token inforamtion not as a part of JSON but
+        /* Identity v3 conveys the token information not as a part of JSON but
          * in the X-Subject-Token HTTP header we're getting from caller. */
         token.id = token_str;
       } else {
@@ -354,7 +354,7 @@ int TokenEnvelope::parse(const DoutPrefixProvider *dpp,
     } else if (version == rgw::keystone::ApiVersion::VER_3) {
       if (! token_iter.end()) {
         decode_v3(*token_iter);
-        /* v3 suceeded. We have to fill token.id from external input as it
+        /* v3 succeeded. We have to fill token.id from external input as it
          * isn't a part of the JSON response anymore. It has been moved
          * to X-Subject-Token HTTP header instead. */
         token.id = token_str;
diff --git a/src/rgw/rgw_kmip_client_impl.cc b/src/rgw/rgw_kmip_client_impl.cc
index 7f0160748a90..e25ce95a8281 100644
--- a/src/rgw/rgw_kmip_client_impl.cc
+++ b/src/rgw/rgw_kmip_client_impl.cc
@@ -650,7 +650,7 @@ RGWKmipHandles::do_one_entry(RGWKMIPTransceiver &element)
       KeyBlock *kp = static_cast<SymmetricKey *>(pld->object)->key_block;
       ByteString *bp;
       if (kp->key_format_type != KMIP_KEYFORMAT_RAW) {
-	lderr(cct) << "get: expected raw key fromat got  " << kp->key_format_type << dendl;
+	lderr(cct) << "get: expected raw key format got  " << kp->key_format_type << dendl;
 	element.ret = -EINVAL;
 	goto Done;
       }
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 4ae3a950ade6..79b81270457d 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -2146,7 +2146,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
 
   SimpleBackoff shard_lock(5 /* max retries */, 50ms);
   if (! shard_lock.wait_backoff(lock_lambda)) {
-    ldpp_dout(this, 0) << "RGWLC::process(): failed to aquire lock on "
+    ldpp_dout(this, 0) << "RGWLC::process(): failed to acquire lock on "
 		       << lc_shard << " after " << shard_lock.get_retries()
 		       << dendl;
     return 0;
@@ -2314,7 +2314,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
     /* postamble */
     //bucket_lc_post(index, max_lock_secs, entry, ret, worker);
     if (! shard_lock.wait_backoff(lock_lambda)) {
-      ldpp_dout(this, 0) << "RGWLC::process(): failed to aquire lock on "
+      ldpp_dout(this, 0) << "RGWLC::process(): failed to acquire lock on "
 			 << lc_shard << " after " << shard_lock.get_retries()
 			 << dendl;
       return 0;
diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index f449cce21c02..5b0a3c728490 100644
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -114,7 +114,7 @@ namespace rgw {
   void RGWLibProcess::handle_request(const DoutPrefixProvider *dpp, RGWRequest* r)
   {
     /*
-     * invariant: valid requests are derived from RGWLibRequst
+     * invariant: valid requests are derived from RGWLibRequest
      */
     RGWLibRequest* req = static_cast<RGWLibRequest*>(r);
 
diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h
index 1ad54b49b485..30234eebcdd3 100644
--- a/src/rgw/rgw_lib.h
+++ b/src/rgw/rgw_lib.h
@@ -125,7 +125,7 @@ namespace rgw {
   public:
     CephContext* cct;
 
-    /* unambiguiously return req_state */
+    /* unambiguously return req_state */
     inline req_state* get_state() { return this->RGWRequest::s; }
 
     RGWLibRequest(CephContext* _cct, std::unique_ptr<rgw::sal::User> _user)
diff --git a/src/rgw/rgw_lua_background.h b/src/rgw/rgw_lua_background.h
index e2f290213b5c..7b8d12599f4e 100644
--- a/src/rgw/rgw_lua_background.h
+++ b/src/rgw/rgw_lua_background.h
@@ -180,5 +180,5 @@ class Background : public RGWRealmReloader::Pauser {
   void resume(rgw::sal::Driver* _driver) override;
 };
 
-} //namepsace rgw::lua
+} //namespace rgw::lua
 
diff --git a/src/rgw/rgw_lua_utils.cc b/src/rgw/rgw_lua_utils.cc
index 4118bed42e08..35f952974613 100644
--- a/src/rgw/rgw_lua_utils.cc
+++ b/src/rgw/rgw_lua_utils.cc
@@ -9,7 +9,7 @@
 
 namespace rgw::lua {
 
-// TODO - add the folowing generic functions
+// TODO - add the following generic functions
 // lua_push(lua_State* L, const std::string& str)
 // template<typename T> lua_push(lua_State* L, const std::optional<T>& val)
 // lua_push(lua_State* L, const ceph::real_time& tp)
diff --git a/src/rgw/rgw_lua_utils.h b/src/rgw/rgw_lua_utils.h
index 0e8dfb2bf6ba..79ebb09fac07 100644
--- a/src/rgw/rgw_lua_utils.h
+++ b/src/rgw/rgw_lua_utils.h
@@ -222,7 +222,7 @@ void create_metatable(lua_State* L, const std::string_view parent_name, const st
 // following struct may be used as a base class for other MetaTable classes
 // note, however, this is not mandatory to use it as a base
 struct EmptyMetaTable {
-  // by default everythinmg is "readonly"
+  // by default everything is "readonly"
   // to change, overload this function in the derived
   static int NewIndexClosure(lua_State* L) {
     return luaL_error(L, "trying to write to readonly field");
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6a7a7313e117..7363db728a49 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3321,7 +3321,7 @@ static int filter_out_quota_info(std::map<std::string, bufferlist>& add_attrs,
     }
   }
 
-  /* Swift requries checking on raw usage instead of the 4 KiB rounded one. */
+  /* Swift requires checking on raw usage instead of the 4 KiB rounded one. */
   quota.check_on_raw = true;
   quota.enabled = quota.max_size > 0 || quota.max_objects > 0;
 
@@ -7601,7 +7601,7 @@ RGWBulkUploadOp::parse_path(const std::string_view& path)
   const size_t start_pos = path.find_first_not_of('/');
 
   if (std::string_view::npos != start_pos) {
-    /* Seperator is the first slash after the leading ones. */
+    /* Separator is the first slash after the leading ones. */
     const size_t sep_pos = path.substr(start_pos).find('/');
 
     if (std::string_view::npos != sep_pos) {
@@ -8074,10 +8074,10 @@ void RGWBulkUploadOp::execute(optional_yield y)
 
 RGWBulkUploadOp::AlignedStreamGetter::~AlignedStreamGetter()
 {
-  const size_t aligned_legnth = length + (-length % alignment);
+  const size_t aligned_length = length + (-length % alignment);
   ceph::bufferlist junk;
 
-  DecoratedStreamGetter::get_exactly(aligned_legnth - position, junk);
+  DecoratedStreamGetter::get_exactly(aligned_length - position, junk);
 }
 
 ssize_t RGWBulkUploadOp::AlignedStreamGetter::get_at_most(const size_t want,
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index dc5bc8be55ec..be7232ae392d 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -2198,7 +2198,7 @@ inline int rgw_get_request_metadata(const DoutPrefixProvider *dpp,
         return -ENAMETOOLONG;
       }
 
-      /* Similar remarks apply to the check for value size. We're veryfing
+      /* Similar remarks apply to the check for value size. We're verifying
        * it early at the RGW's side as it's being claimed in /info. */
       const auto max_attr_size = cct->_conf->rgw_max_attr_size;
       if (max_attr_size && xattr.length() > max_attr_size) {
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index 8cbda840c5ad..125bace14450 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -56,7 +56,7 @@ void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
   pcb->add_u64(l_rgw_pubsub_push_pending, "pubsub_push_pending", "Pubsub events pending reply from endpoint");
   pcb->add_u64_counter(l_rgw_pubsub_missing_conf, "pubsub_missing_conf", "Pubsub events could not be handled because of missing configuration");
   
-  pcb->add_u64_counter(l_rgw_lua_script_ok, "lua_script_ok", "Successfull executions of Lua scripts");
+  pcb->add_u64_counter(l_rgw_lua_script_ok, "lua_script_ok", "Successful executions of Lua scripts");
   pcb->add_u64_counter(l_rgw_lua_script_fail, "lua_script_fail", "Failed executions of Lua scripts");
   pcb->add_u64(l_rgw_lua_current_vms, "lua_current_vms", "Number of Lua VMs currently being executed");
 }
diff --git a/src/rgw/rgw_period_history.cc b/src/rgw/rgw_period_history.cc
index 40a0daf3b446..66ad7151d0a3 100644
--- a/src/rgw/rgw_period_history.cc
+++ b/src/rgw/rgw_period_history.cc
@@ -106,7 +106,7 @@ class RGWPeriodHistory::Impl final {
   /// and return an iterator to the merged history
   Set::iterator merge(Set::iterator dst, Set::iterator src);
 
-  /// construct a Cursor object using Cursor's private constuctor
+  /// construct a Cursor object using Cursor's private constructor
   Cursor make_cursor(Set::const_iterator history, epoch_t epoch);
 
   CephContext *const cct;
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 14d85abe3b62..7031c2363f0b 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -675,7 +675,7 @@ int RGWPubSub::Bucket::remove_notifications(const DoutPrefixProvider *dpp, optio
     return ret ;
   }
 
-  // remove all auto-genrated topics
+  // remove all auto-generated topics
   for (const auto& topic : bucket_topics.topics) {
     const auto& topic_name = topic.first;
     ret = ps.remove_topic(dpp, topic_name, y);
diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc
index 745dac7fefe4..b3a511becb68 100644
--- a/src/rgw/rgw_realm_reloader.cc
+++ b/src/rgw/rgw_realm_reloader.cc
@@ -22,7 +22,7 @@
 #define dout_prefix (*_dout << "rgw realm reloader: ")
 
 
-// safe callbacks from SafeTimer are unneccessary. reload() can take a long
+// safe callbacks from SafeTimer are unnecessary. reload() can take a long
 // time, so we don't want to hold the mutex and block handle_notify() for the
 // duration
 static constexpr bool USE_SAFE_TIMER_CALLBACKS = false;
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 47272515d348..6b827f983fb6 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -222,13 +222,13 @@ void rgw_rest_init(CephContext *cct, const rgw::sal::ZoneGroup& zone_group)
   hostnames_set.erase(""); // filter out empty hostnames
   ldout(cct, 20) << "RGW hostnames: " << hostnames_set << dendl;
   /* TODO: We should have a sanity check that no hostname matches the end of
-   * any other hostname, otherwise we will get ambigious results from
+   * any other hostname, otherwise we will get ambiguous results from
    * rgw_find_host_in_domains.
    * Eg: 
    * Hostnames: [A, B.A]
    * Inputs: [Z.A, X.B.A]
    * Z.A clearly splits to subdomain=Z, domain=Z
-   * X.B.A ambigously splits to both {X, B.A} and {X.B, A}
+   * X.B.A ambiguously splits to both {X, B.A} and {X.B, A}
    */
 
   zone_group.get_s3website_hostnames(names);
@@ -646,7 +646,7 @@ static void build_redirect_url(req_state *s, const string& redirect_base, string
   
   dest_uri = redirect_base;
   /*
-   * reqest_uri is always start with slash, so we need to remove
+   * request_uri is always start with slash, so we need to remove
    * the unnecessary slash at the end of dest_uri.
    */
   if (dest_uri[dest_uri.size() - 1] == '/') {
@@ -766,11 +766,11 @@ int dump_body(req_state* const s,
               const char* const buf,
               const size_t len)
 {
-  bool healthchk = false;
+  bool healthcheck = false;
   // we dont want to limit health checks
   if(s->op_type == RGW_OP_GET_HEALTH_CHECK)
-    healthchk = true;
-  if(len > 0 && !healthchk) {
+    healthcheck = true;
+  if(len > 0 && !healthcheck) {
     const char *method = s->info.method;
     s->ratelimit_data->decrease_bytes(method, s->ratelimit_user_name, len, &s->user_ratelimit);
     if(!rgw::sal::Bucket::empty(s->bucket.get()))
@@ -803,11 +803,11 @@ int recv_body(req_state* const s,
   } catch (rgw::io::Exception& e) {
     return -e.code().value();
   }
-  bool healthchk = false;
+  bool healthcheck = false;
   // we dont want to limit health checks
   if(s->op_type ==  RGW_OP_GET_HEALTH_CHECK)
-    healthchk = true;
-  if(len > 0 && !healthchk) {
+    healthcheck = true;
+  if(len > 0 && !healthcheck) {
     const char *method = s->info.method;
     s->ratelimit_data->decrease_bytes(method, s->ratelimit_user_name, len, &s->user_ratelimit);
     if(!rgw::sal::Bucket::empty(s->bucket.get()))
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 04f21d9b87d9..843b7eb7d084 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -1035,7 +1035,7 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     
     // create unique topic name. this has 2 reasons:
     // (1) topics cannot be shared between different S3 notifications because they hold the filter information
-    // (2) make topic clneaup easier, when notification is removed
+    // (2) make topic cleanup easier, when notification is removed
     const auto unique_topic_name = topic_to_unique(topic_name, notif_name);
     // generate the internal topic. destination is stored here for the "push-only" case
     // when no subscription exists
diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc
index 3de8ca2618dc..128e44d9a5ae 100644
--- a/src/rgw/rgw_rest_ratelimit.cc
+++ b/src/rgw/rgw_rest_ratelimit.cc
@@ -221,7 +221,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y)
   sval = s->info.args.get("global", &exists);
   if (exists) {
     if (!boost::iequals(sval,"true") && !boost::iequals(sval,"false")) {
-      ldpp_dout(this, 20) << "global is not equal to true or faslse" << dendl;
+      ldpp_dout(this, 20) << "global is not equal to true or false" << dendl;
       op_ret = -EINVAL;
       return;
     }
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 5425406b1eb9..f12d5e733007 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -215,7 +215,7 @@ ldpp_dout(s, 20) << "get_encryption_defaults: found kms_attr " << kms_attr << "
     }
     kms_attr_seen = true;
   } else if (!rest_only && kms_master_key_id != "") {
-ldpp_dout(s, 20) << "get_encryption_defaults: no kms_attr, but kms_master_key_id = " << kms_master_key_id << ", settig kms_attr_seen" << dendl;
+ldpp_dout(s, 20) << "get_encryption_defaults: no kms_attr, but kms_master_key_id = " << kms_master_key_id << ", setting kms_attr_seen" << dendl;
     kms_attr_seen = true;
     rgw_set_amz_meta_header(s->info.crypt_attribute_map, kms_attr, kms_master_key_id, OVERWRITE);
   }
@@ -469,7 +469,7 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
 
   if (! op_ret) {
     if (! lo_etag.empty()) {
-      /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
+      /* Handle etag of Swift API's large objects (DLO/SLO). It's entirely
        * legit to perform GET on them through S3 API. In such situation,
        * a client should receive the composited content with corresponding
        * etag value. */
@@ -3303,7 +3303,7 @@ void RGWPostObj_ObjStore_S3::send_response()
        * What we really would like is to quaily the bucket name, so
        * that the client could simply copy it and paste into next request.
        * Unfortunately, in S3 we cannot know if the client will decide
-       * to come through DNS, with "bucket.tenant" sytanx, or through
+       * to come through DNS, with "bucket.tenant" syntax, or through
        * URL with "tenant\bucket" syntax. Therefore, we provide the
        * tenant separately.
        */
diff --git a/src/rgw/rgw_rest_sts.cc b/src/rgw/rgw_rest_sts.cc
index c5bd8b1b7077..d94181f4e634 100644
--- a/src/rgw/rgw_rest_sts.cc
+++ b/src/rgw/rgw_rest_sts.cc
@@ -577,7 +577,7 @@ int RGWSTSGetSessionToken::verify_permission(optional_yield y)
                               s,
                               rgw::ARN(partition, service, "", s->user->get_tenant(), ""),
                               rgw::IAM::stsGetSessionToken)) {
-    ldpp_dout(this, 0) << "User does not have permssion to perform GetSessionToken" << dendl;
+    ldpp_dout(this, 0) << "User does not have permission to perform GetSessionToken" << dendl;
     return -EACCES;
   }
 
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 5f0b1f772226..25ba61841613 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -1620,7 +1620,7 @@ int RGWBulkDelete_ObjStore_SWIFT::get_data(
     const size_t start_pos = path_str.find_first_not_of('/');
 
     if (string::npos != start_pos) {
-      /* Seperator is the first slash after the leading ones. */
+      /* Separator is the first slash after the leading ones. */
       const size_t sep_pos = path_str.find('/', start_pos);
 
       if (string::npos != sep_pos) {
diff --git a/src/rgw/rgw_s3select.cc b/src/rgw/rgw_s3select.cc
index c7eaa69842f3..cfcba2a63e10 100644
--- a/src/rgw/rgw_s3select.cc
+++ b/src/rgw/rgw_s3select.cc
@@ -370,7 +370,7 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_csv(const char* query, const char*
   uint32_t length_before_processing, length_post_processing;
   csv_object::csv_defintions csv;
   const char* s3select_syntax_error = "s3select-Syntax-Error";
-  const char* s3select_resource_id = "resourcse-id";
+  const char* s3select_resource_id = "resource-id";
   const char* s3select_processTime_error = "s3select-ProcessingTime-Error";
 
   s3select_syntax.parse_query(query);
diff --git a/src/rgw/rgw_s3select_private.h b/src/rgw/rgw_s3select_private.h
index fa595b0da599..f4fd59a29ac8 100644
--- a/src/rgw/rgw_s3select_private.h
+++ b/src/rgw/rgw_s3select_private.h
@@ -201,7 +201,7 @@ class RGWSelectObj_ObjStore_S3 : public RGWGetObj_ObjStore_S3
 #ifdef _ARROW_EXIST
   s3selectEngine::rgw_s3select_api m_rgw_api;
 #endif
-  //a request for range may statisfy by several calls to send_response_date;
+  //a request for range may satisfy by several calls to send_response_date;
   size_t m_request_range;
   std::string requested_buffer;
   std::string range_req_str;
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index f0314517bebd..cc45d5f3a2ce 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -699,7 +699,7 @@ class Bucket {
     virtual const rgw_user& get_owner() const = 0;
     /** Check in the backing store if this bucket is empty */
     virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) = 0;
-    /** Chec k if the given size fits within the quota */
+    /** Check if the given size fits within the quota */
     virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) = 0;
     /** Set the attributes in attrs, leaving any other existing attrs set, and
      * write them to the backing store; a merge operation */
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 6c8778b87238..1a28a7724f4c 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -310,7 +310,7 @@ namespace rgw::sal {
 
   int DBBucket::remove_objs_from_index(const DoutPrefixProvider *dpp, std::list<rgw_obj_index_key>& objs_to_unlink)
   {
-    /* XXX: CHECK: Unlike RadosStore, there is no seperate bucket index table.
+    /* XXX: CHECK: Unlike RadosStore, there is no separate bucket index table.
      * Delete all the object in the list from the object table of this
      * bucket
      */
@@ -1206,7 +1206,7 @@ namespace rgw::sal {
         return 0; /* nothing more to write */
       }
 
-      /* flush watever tail data is present */
+      /* flush whatever tail data is present */
       int ret = parent_op.write_data(dpp, tail_part_data, tail_part_offset);
       if (ret < 0) {
         return ret;
@@ -1360,7 +1360,7 @@ namespace rgw::sal {
         return 0; /* nothing more to write */
       }
 
-      /* flush watever tail data is present */
+      /* flush whatever tail data is present */
       int ret = parent_op.write_data(dpp, tail_part_data, tail_part_offset);
       if (ret < 0) {
         return ret;
diff --git a/src/rgw/rgw_sts.cc b/src/rgw/rgw_sts.cc
index 2b31d5c5a297..557bcf24f2b3 100644
--- a/src/rgw/rgw_sts.cc
+++ b/src/rgw/rgw_sts.cc
@@ -72,7 +72,7 @@ int Credentials::generateCredentials(const DoutPrefixProvider *dpp,
   //Session Token - Encrypt using AES
   auto* cryptohandler = cct->get_crypto_handler(CEPH_CRYPTO_AES);
   if (! cryptohandler) {
-    ldpp_dout(dpp, 0) << "ERROR: No AES cryto handler found !" << dendl;
+    ldpp_dout(dpp, 0) << "ERROR: No AES crypto handler found !" << dendl;
     return -EINVAL;
   }
   string secret_s = cct->_conf->rgw_sts_key;
diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc
index b58f0b45d8e7..765721df190d 100644
--- a/src/rgw/rgw_swift_auth.cc
+++ b/src/rgw/rgw_swift_auth.cc
@@ -88,7 +88,7 @@ void TempURLEngine::get_owner_info(const DoutPrefixProvider* dpp, const req_stat
 
   /* TempURL case is completely different than the Keystone auth - you may
    * get account name only through extraction from URL. In turn, knowledge
-   * about account is neccessary to obtain its bucket tenant. Without that,
+   * about account is necessary to obtain its bucket tenant. Without that,
    * the access would be limited to accounts with empty tenant. */
   string bucket_tenant;
   if (!s->account_name.empty()) {
diff --git a/src/rgw/rgw_sync_policy.h b/src/rgw/rgw_sync_policy.h
index 98d0a4798a5d..ec9d1f2c6234 100644
--- a/src/rgw/rgw_sync_policy.h
+++ b/src/rgw/rgw_sync_policy.h
@@ -591,7 +591,7 @@ WRITE_CLASS_ENCODER(rgw_sync_data_flow_group)
 struct rgw_sync_policy_group {
   std::string id;
 
-  rgw_sync_data_flow_group data_flow; /* override data flow, howver, will not be able to
+  rgw_sync_data_flow_group data_flow; /* override data flow, however, will not be able to
                                                         add new flows that don't exist at higher level */
   std::vector<rgw_sync_bucket_pipes> pipes; /* if not defined then applies to all
                                                               buckets (DR sync) */
diff --git a/src/rgw/rgw_zone_types.h b/src/rgw/rgw_zone_types.h
index c8711a96d6d5..13fce000c412 100644
--- a/src/rgw/rgw_zone_types.h
+++ b/src/rgw/rgw_zone_types.h
@@ -332,7 +332,7 @@ struct RGWZone {
  */
   uint32_t bucket_index_max_shards;
 
-  // pre-shard buckets on creation to enable some write-parallism by default,
+  // pre-shard buckets on creation to enable some write-parallelism by default,
   // delay the need to reshard as the bucket grows, and (in multisite) get some
   // bucket index sharding where dynamic resharding is not supported
   static constexpr uint32_t default_bucket_index_max_shards = 11;
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index 99f400f42b02..d682b87eb2c2 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -424,7 +424,7 @@ bool RGWSI_RADOS::check_secure_mon_conn(const DoutPrefixProvider *dpp) const
   std::vector<uint32_t> modes;
 
   reg.get_supported_methods(CEPH_ENTITY_TYPE_MON, &methods, &modes);
-  ldpp_dout(dpp, 20) << __func__ << "(): auth registy supported: methods=" << methods << " modes=" << modes << dendl;
+  ldpp_dout(dpp, 20) << __func__ << "(): auth registry supported: methods=" << methods << " modes=" << modes << dendl;
 
   for (auto method : methods) {
     if (!reg.is_secure_method(method)) {
diff --git a/src/test/cls_2pc_queue/test_cls_2pc_queue.cc b/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
index 8f392721f5b7..838497924909 100644
--- a/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
+++ b/src/test/cls_2pc_queue/test_cls_2pc_queue.cc
@@ -409,7 +409,7 @@ TEST_F(TestCls2PCQueue, CommitError)
   }
   cls_2pc_reservations reservations;
   ASSERT_EQ(0, cls_2pc_queue_list_reservations(ioctx, queue_name, reservations));
-  // 2 reservations were not comitted
+  // 2 reservations were not committed
   ASSERT_EQ(reservations.size(), 2);
 }
 
@@ -675,7 +675,7 @@ TEST_F(TestCls2PCQueue, ManualCleanup)
   cls_2pc_queue_init(op, queue_name, max_size);
   ASSERT_EQ(0, ioctx.operate(queue_name, &op));
 
-  // anything older than 100ms is cosidered stale
+  // anything older than 100ms is considered stale
   ceph::coarse_real_time stale_time = ceph::coarse_real_clock::now() + std::chrono::milliseconds(100);
 
   std::vector<std::thread> reservers(max_workers);
@@ -749,7 +749,7 @@ TEST_F(TestCls2PCQueue, Cleanup)
   cls_2pc_queue_init(op, queue_name, max_size);
   ASSERT_EQ(0, ioctx.operate(queue_name, &op));
 
-  // anything older than 100ms is cosidered stale
+  // anything older than 100ms is considered stale
   ceph::coarse_real_time stale_time = ceph::coarse_real_clock::now() + std::chrono::milliseconds(100);
 
   std::vector<std::thread> reservers(max_workers);
@@ -989,7 +989,7 @@ TEST_F(TestCls2PCQueue, MultiProducerConsumer)
   std::for_each(producers.begin(), producers.end(), [](auto& p) { p.join(); });
   std::for_each(consumers.begin(), consumers.end(), [](auto& c) { c.join(); });
   if (!retry_happened) {
-      std::cerr << "Queue was never full - all reservations were sucessfull." <<
+      std::cerr << "Queue was never full - all reservations were successful." <<
           "Please decrease the amount of consumer threads" << std::endl;
   }
   // make sure that queue is empty and no reservations remain
diff --git a/src/test/cls_queue/test_cls_queue.cc b/src/test/cls_queue/test_cls_queue.cc
index a1f68638c1f4..8b2f9dc1041f 100644
--- a/src/test/cls_queue/test_cls_queue.cc
+++ b/src/test/cls_queue/test_cls_queue.cc
@@ -85,7 +85,7 @@ TEST_F(TestClsQueue, Enqueue)
   ASSERT_EQ(0, ioctx.operate(queue_name, &op));
 
   // test multiple enqueues
-  // 10 iterations, 100 elelemts each
+  // 10 iterations, 100 elements each
   // expect 0 (OK)
   test_enqueue(queue_name, 10, 100, 0);
 }
@@ -99,10 +99,10 @@ TEST_F(TestClsQueue, QueueFull)
   cls_queue_init(op, queue_name, queue_size);
   ASSERT_EQ(0, ioctx.operate(queue_name, &op));
 
-  // 8 iterations, 5 elelemts each
+  // 8 iterations, 5 elements each
   // expect 0 (OK)
   test_enqueue(queue_name, 8, 5, 0);
-  // 2 iterations, 5 elelemts each
+  // 2 iterations, 5 elements each
   // expect -28 (Q FULL)
   test_enqueue(queue_name, 2, 5, -28);
 }
diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index bf60dfdd0424..a2f2fa66a767 100644
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -456,7 +456,7 @@ TEST_F(cls_rgw, index_list)
     { static_cast<char>(0xCF), static_cast<char>(0x8F) },
     /* treble byte utf8 character */
     { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
-    /* quadruble byte utf8 character */
+    /* quadruple byte utf8 character */
     { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
   };
 
@@ -738,7 +738,7 @@ TEST_F(cls_rgw, bi_list)
       "bi list test with filters should return correct truncation indicator";
   }
 
-  // test whether combined segment count is correcgt
+  // test whether combined segment count is correct
   is_truncated = false;
   entries.clear();
   marker.clear();
diff --git a/src/test/cls_rgw/test_cls_rgw_stats.cc b/src/test/cls_rgw/test_cls_rgw_stats.cc
index 004ccc6d1ce0..80fa88fabf53 100644
--- a/src/test/cls_rgw/test_cls_rgw_stats.cc
+++ b/src/test/cls_rgw/test_cls_rgw_stats.cc
@@ -259,7 +259,7 @@ object_map::iterator simulator::find_or_create(const cls_rgw_obj_key& key)
 
 int simulator::try_start(const cls_rgw_obj_key& key, const std::string& tag)
 {
-  // choose randomly betwen create and delete
+  // choose randomly between create and delete
   const auto type = static_cast<RGWModifyOp>(
       ceph::util::generate_random_number<size_t, size_t>(CLS_RGW_OP_ADD,
                                                          CLS_RGW_OP_DEL));
diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
index d6bb3db1c690..dec7ea1c149c 100644
--- a/src/test/rgw/CMakeLists.txt
+++ b/src/test/rgw/CMakeLists.txt
@@ -68,24 +68,24 @@ add_executable(unittest_rgw_bucket_sync_cache test_rgw_bucket_sync_cache.cc)
 add_ceph_unittest(unittest_rgw_bucket_sync_cache)
 target_link_libraries(unittest_rgw_bucket_sync_cache ${rgw_libs})
 
-#unitttest_rgw_period_history
+#unittest_rgw_period_history
 add_executable(unittest_rgw_period_history test_rgw_period_history.cc)
 add_ceph_unittest(unittest_rgw_period_history)
 target_link_libraries(unittest_rgw_period_history ${rgw_libs})
 
-# unitttest_rgw_compression
+# unittest_rgw_compression
 add_executable(unittest_rgw_compression
   test_rgw_compression.cc
   $<TARGET_OBJECTS:unit-main>)
 add_ceph_unittest(unittest_rgw_compression)
 target_link_libraries(unittest_rgw_compression ${rgw_libs})
 
-# unitttest_http_manager
+# unittest_http_manager
 add_executable(unittest_http_manager test_http_manager.cc)
 add_ceph_unittest(unittest_http_manager)
 target_link_libraries(unittest_http_manager ${rgw_libs})
 
-# unitttest_rgw_reshard_wait
+# unittest_rgw_reshard_wait
 add_executable(unittest_rgw_reshard_wait test_rgw_reshard_wait.cc)
 add_ceph_unittest(unittest_rgw_reshard_wait)
 target_link_libraries(unittest_rgw_reshard_wait ${rgw_libs})
@@ -213,7 +213,7 @@ add_ceph_unittest(unittest_rgw_string)
 target_include_directories(unittest_rgw_string
   SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 
-# unitttest_rgw_dmclock_queue
+# unittest_rgw_dmclock_queue
 add_executable(unittest_rgw_dmclock_scheduler test_rgw_dmclock_scheduler.cc $<TARGET_OBJECTS:unit-main>)
 add_ceph_unittest(unittest_rgw_dmclock_scheduler)
 target_include_directories(unittest_rgw_dmclock_scheduler
diff --git a/src/test/rgw/bucket_notification/api.py b/src/test/rgw/bucket_notification/api.py
index 30fce41a1bb0..0b53d32c7365 100644
--- a/src/test/rgw/bucket_notification/api.py
+++ b/src/test/rgw/bucket_notification/api.py
@@ -26,7 +26,7 @@ def put_object_tagging(conn, bucket_name, key, tags):
 
 def make_request(conn, method, resource, parameters=None, sign_parameters=False, extra_parameters=None):
     """generic request sending to pubsub radogw
-    should cover: topics, notificatios and subscriptions
+    should cover: topics, notifications and subscriptions
     """
     url_params = ''
     if parameters is not None:
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 8860df1555ef..075d47466a23 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -277,7 +277,7 @@ def stop_amqp_receiver(receiver, task):
         receiver.channel.stop_consuming()
         log.info('stopping AMQP receiver')
     except Exception as error:
-        log.info('failed to gracefuly stop AMQP receiver: %s', str(error))
+        log.info('failed to gracefully stop AMQP receiver: %s', str(error))
     task.join(5)
 
 
@@ -487,7 +487,7 @@ def stop_kafka_receiver(receiver, task):
         receiver.consumer.unsubscribe()
         receiver.consumer.close()
     except Exception as error:
-        log.info('failed to gracefuly stop Kafka receiver: %s', str(error))
+        log.info('failed to gracefully stop Kafka receiver: %s', str(error))
 
 
 def get_ip():
diff --git a/src/test/rgw/rgw_multi/tests_az.py b/src/test/rgw/rgw_multi/tests_az.py
index 13ec832a2041..7ad708350dc5 100644
--- a/src/test/rgw/rgw_multi/tests_az.py
+++ b/src/test/rgw/rgw_multi/tests_az.py
@@ -192,7 +192,7 @@ def test_az_create_empty_bucket():
 
 
 def test_az_check_empty_bucket_versioning():
-     """ test bucket vesioning with empty bucket """
+     """ test bucket versioning with empty bucket """
      zones, az_zones = init_env()
      bucket_name = gen_bucket_name()
      # create bucket on the non archive zone
diff --git a/src/test/rgw/test-ceph-diff-sorted.sh b/src/test/rgw/test-ceph-diff-sorted.sh
index dddf4ae1b18f..c164e5547f4b 100755
--- a/src/test/rgw/test-ceph-diff-sorted.sh
+++ b/src/test/rgw/test-ceph-diff-sorted.sh
@@ -37,7 +37,7 @@ cuttlefish
 fox
 llama
 octopus
-penguine
+penguin
 seal
 squid
 whale
diff --git a/src/test/rgw/test_multi.md b/src/test/rgw/test_multi.md
index f2c1285300ec..46bf8e0bfd7c 100644
--- a/src/test/rgw/test_multi.md
+++ b/src/test/rgw/test_multi.md
@@ -14,7 +14,7 @@ Since we use the same entry point file for all tests, running specific tests is
 ```
 $ nosetests test_multi.py:<specific_test_name>
 ```
-To run miltiple tests based on wildcard string, use the following format:
+To run multiple tests based on wildcard string, use the following format:
 ```
 $ nosetests test_multi.py -m "<wildcard string>"
 ```
diff --git a/src/test/rgw/test_rgw_amqp.cc b/src/test/rgw/test_rgw_amqp.cc
index c2297dc2235e..a6c76dd2a2cd 100644
--- a/src/test/rgw/test_rgw_amqp.cc
+++ b/src/test/rgw/test_rgw_amqp.cc
@@ -46,7 +46,7 @@ class TestAMQP : public ::testing::Test {
     amqp::shutdown();
   }
 
-  // wait for at least one new (since last drain) message to be dequeueud
+  // wait for at least one new (since last drain) message to be dequeued
   // and then wait for all pending answers to be received
   void wait_until_drained() {  
     while (amqp::get_dequeued() == current_dequeued) {
diff --git a/src/test/rgw/test_rgw_kms.cc b/src/test/rgw/test_rgw_kms.cc
index aaa4e4174f78..9fc6145d2fe7 100644
--- a/src/test/rgw/test_rgw_kms.cc
+++ b/src/test/rgw/test_rgw_kms.cc
@@ -242,7 +242,7 @@ TEST_F(TestSSEKMS, concat_url)
   // Each test has 3 strings:
   // * the base URL
   // * the path we want to concatenate
-  // * the exepected final URL
+  // * the expected final URL
   std::string tests[9][3] ={
     {"", "", ""},
     {"", "bar", "/bar"},
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index be9fa6aed162..07f8521c2313 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -706,7 +706,7 @@ TEST(TestRGWLua, UseFunction)
 {
 	const std::string script = R"(
 		function print_owner(owner)
-  		print("Owner Dispaly Name: " .. owner.DisplayName)
+  		print("Owner Display Name: " .. owner.DisplayName)
   		print("Owner Id: " .. owner.User.Id)
   		print("Owner Tenanet: " .. owner.User.Tenant)
 		end
@@ -921,7 +921,7 @@ TEST(TestRGWLuaBackground, RequestScript)
 
   pe.lua.background = &lua_background;
 
-  // to make sure test is consistent we have to puase the background
+  // to make sure test is consistent we have to pause the background
   lua_background.pause();
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, request_script);
   ASSERT_EQ(rc, 0);
diff --git a/src/test/rgw/test_rgw_xml.cc b/src/test/rgw/test_rgw_xml.cc
index fa9f21157787..a31be59a3c88 100644
--- a/src/test/rgw/test_rgw_xml.cc
+++ b/src/test/rgw/test_rgw_xml.cc
@@ -376,7 +376,7 @@ TEST(TestDecoder, BasicParsing)
   ASSERT_STREQ(to_string(result).c_str(), expected_output);
 }
 
-TEST(TestDecoder, MalfomedInput)
+TEST(TestDecoder, MalformedInput)
 {
   RGWXMLDecoder::XMLParser parser;
   ASSERT_TRUE(parser.init());
diff --git a/src/test/test_rgw_admin_log.cc b/src/test/test_rgw_admin_log.cc
index fe072ead18a1..7dd7604db1a1 100644
--- a/src/test/test_rgw_admin_log.cc
+++ b/src/test/test_rgw_admin_log.cc
@@ -489,7 +489,7 @@ static int put_bucket_obj(const char *obj_name, char *data, unsigned len) {
   g_test->send_request(string("PUT"), req,
                        read_bucket_object, (void *)data, (size_t)len);
   if (g_test->get_resp_code() != 200U) {
-    cout << "Errror sending object to the bucket, http_code " << g_test->get_resp_code();
+    cout << "Error sending object to the bucket, http_code " << g_test->get_resp_code();
     return -1;
   }
   return 0;
@@ -500,7 +500,7 @@ static int read_bucket_obj(const char *obj_name) {
   req.append(obj_name);
   g_test->send_request(string("GET"), req);
   if (g_test->get_resp_code() != 200U) {
-    cout << "Errror sending object to the bucket, http_code " << g_test->get_resp_code();
+    cout << "Error sending object to the bucket, http_code " << g_test->get_resp_code();
     return -1;
   }
   return 0;
@@ -511,7 +511,7 @@ static int delete_obj(const char *obj_name) {
   req.append(obj_name);
   g_test->send_request(string("DELETE"), req);
   if (g_test->get_resp_code() != 204U) {
-    cout << "Errror deleting object from bucket, http_code " << g_test->get_resp_code();
+    cout << "Error deleting object from bucket, http_code " << g_test->get_resp_code();
     return -1;
   }
   return 0;
diff --git a/src/test/test_rgw_admin_meta.cc b/src/test/test_rgw_admin_meta.cc
index 4699a876dfeb..b1d5fad06009 100644
--- a/src/test/test_rgw_admin_meta.cc
+++ b/src/test/test_rgw_admin_meta.cc
@@ -697,7 +697,7 @@ TEST(TestRGWAdmin, meta_get){
   p2 = RGW_CAP_WRITE;
   EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, p2) != 0);
 
-  /*Version and tag infromation*/
+  /*Version and tag information*/
   EXPECT_TRUE(objv1->ver > objv->ver);
   EXPECT_EQ(objv1->tag, objv->tag);
   

From fbd5c40edccccbb44b66e9d82fc71b14ac1d04ae Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Mon, 20 Nov 2023 18:47:14 +0530
Subject: [PATCH 0744/2492] osd: Update PGSnapTrim op queue item cost to
 reflect average object size

Previously, a static value of snap_trim_cost (1 MiB by default) for
PGSnapTrim item was used (see config option osd_snap_trim_cost). For pools
with significantly different sizes of objects, the static cost doesn't
accurately estimate the amount of IO each snap trim operation requires.
Instead, add a cost_per_object parameter to OSDService::queue_for_snap_trim
and set it to the average object size in the PG being queued by using
PG::get_average_object_size().

In addition, for the mClock scheduler, the cost_per_object is multiplied
by the actual number of object trimmed per iteration. This multiplier is
represented by osd_pg_max_concurrent_snap_trims config option which is
used when the actual work starts (See DoSnapWork).

Note: The above cost calculation is valid for most snap trim
iterations except for:

1. The penultimate iteration which may return only 1 object to be trimmed,
in which case the cost will be off by a factor equivalent to the average
object size.

2. The final iteration (returns -ENOENT), involving clean-ups.

Fixes: https://tracker.ceph.com/issues/63604
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/osd/OSD.cc          | 22 ++++++++++++++++++++--
 src/osd/OSD.h           |  2 +-
 src/osd/PrimaryLogPG.cc |  4 +++-
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a79b388deca4..ba1250d5ccc6 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1719,14 +1719,32 @@ void OSDService::queue_recovery_context(
       e));
 }
 
-void OSDService::queue_for_snap_trim(PG *pg)
+void OSDService::queue_for_snap_trim(PG *pg, uint64_t cost_per_object)
 {
   dout(10) << "queueing " << *pg << " for snaptrim" << dendl;
+  uint64_t cost_for_queue = [this, cost_per_object] {
+    if (cct->_conf->osd_op_queue == "mclock_scheduler") {
+      /* The cost calculation is valid for most snap trim iterations except
+       * for the following cases:
+       * 1) The penultimate iteration which may return 1 object to trim, in
+       *    which case the cost will be off by a factor equivalent to the
+       *    average object size, and,
+       * 2) The final iteration which returns -ENOENT and performs clean-ups.
+       */
+      return cost_per_object * cct->_conf->osd_pg_max_concurrent_snap_trims;
+    } else {
+      /* We retain this legacy behavior for WeightedPriorityQueue.
+       * This branch should be removed after Squid.
+       */
+      return cct->_conf->osd_snap_trim_cost;
+    }
+  }();
+
   enqueue_back(
     OpSchedulerItem(
       unique_ptr<OpSchedulerItem::OpQueueable>(
 	new PGSnapTrim(pg->get_pgid(), pg->get_osdmap_epoch())),
-      cct->_conf->osd_snap_trim_cost,
+      cost_for_queue,
       cct->_conf->osd_snap_trim_priority,
       ceph_clock_now(),
       0,
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index a6b7a3c31ed3..bf611acba4ba 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -499,7 +499,7 @@ class OSDService : public Scrub::ScrubSchedListener {
                               GenContext<ThreadPool::TPHandle&> *c,
                               uint64_t cost,
 			      int priority);
-  void queue_for_snap_trim(PG *pg);
+  void queue_for_snap_trim(PG *pg, uint64_t cost);
   void queue_for_scrub(PG* pg, Scrub::scrub_prio_t with_priority);
 
   void queue_scrub_after_repair(PG* pg, Scrub::scrub_prio_t with_priority);
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 5cf9bbba68d3..7fd8dee072bc 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -15605,8 +15605,10 @@ PrimaryLogPG::AwaitAsyncWork::AwaitAsyncWork(my_context ctx)
     NamedState(nullptr, "Trimming/AwaitAsyncWork")
 {
   auto *pg = context< SnapTrimmer >().pg;
+  // Determine cost in terms of the average object size
+  uint64_t cost_per_object = pg->get_average_object_size();
   context< SnapTrimmer >().log_enter(state_name);
-  context< SnapTrimmer >().pg->osd->queue_for_snap_trim(pg);
+  context< SnapTrimmer >().pg->osd->queue_for_snap_trim(pg, cost_per_object);
   pg->state_set(PG_STATE_SNAPTRIM);
   pg->state_clear(PG_STATE_SNAPTRIM_ERROR);
   pg->publish_stats_to_osd();

From e52b8dc4e87ab17bd454cffcc5ca837e353507a3 Mon Sep 17 00:00:00 2001
From: Suyash Dongre <suyashd999@gmail.com>
Date: Tue, 28 Nov 2023 01:01:06 +0530
Subject: [PATCH 0745/2492] src/cephadm: Added rgw for listing in Daemons

Fixes: https://tracker.ceph.com/issues/46991
Signed-off-by: Suyash Dongre <suyashd999@gmail.com>
---
 src/cephadm/cephadm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 348f581f9e61..1b8ff8dded0a 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3949,7 +3949,7 @@ def _daemon_ls_subset(self) -> Dict[str, Dict[str, Any]]:
         )
         name_id_mapping: Dict[str, str] = self._parse_container_id_name(code, out)
         for i in os.listdir(data_dir):
-            if i in ['mon', 'osd', 'mds', 'mgr']:
+            if i in ['mon', 'osd', 'mds', 'mgr', 'rgw']:
                 daemon_type = i
                 for j in os.listdir(os.path.join(data_dir, i)):
                     if '-' not in j:
@@ -5813,7 +5813,7 @@ def list_daemons(ctx, detail=True, legacy_dir=None):
     # /var/lib/ceph
     if os.path.exists(data_dir):
         for i in os.listdir(data_dir):
-            if i in ['mon', 'osd', 'mds', 'mgr']:
+            if i in ['mon', 'osd', 'mds', 'mgr', 'rgw']:
                 daemon_type = i
                 for j in os.listdir(os.path.join(data_dir, i)):
                     if '-' not in j:

From 686ef1d4d3e09a364396ec668ac97b08844b166b Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Tue, 28 Nov 2023 05:58:15 +0530
Subject: [PATCH 0746/2492] pybind/mgr/mirroring: drop mon_host from peer_list

Drop mon_host from peer_list to make it similar to rbd_mirror peer_list.

Fixes: https://tracker.ceph.com/issues/63614
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 src/pybind/mgr/mirroring/fs/snapshot_mirror.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
index 6fa8d0c4c533..89da37920312 100644
--- a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
+++ b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
@@ -558,14 +558,11 @@ def peer_list(self, filesystem):
                 peers = self.get_filesystem_peers(filesystem)
                 peer_res = {}
                 for peer_uuid, rem in peers.items():
-                    conf = self.config_get(FSSnapshotMirror.peer_config_key(filesystem, peer_uuid))
                     remote = rem['remote']
                     peer_res[peer_uuid] = {'client_name': remote['client_name'],
                                            'site_name': remote['cluster_name'],
                                            'fs_name': remote['fs_name']
                                            }
-                    if 'mon_host' in conf:
-                        peer_res[peer_uuid]['mon_host'] = conf['mon_host']
                 return 0, json.dumps(peer_res), ''
         except MirrorException as me:
             return me.args[0], '', me.args[1]

From 4e085c5d2f0f14275bc73f2936d95fe8b6331bae Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 28 Nov 2023 08:44:29 +0800
Subject: [PATCH 0747/2492] qa/tasks/cephfs/test_misc: fix the incorrect
 comments

https://tracker.ceph.com/issues/63587
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 qa/tasks/cephfs/test_misc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 58c4e379095a..06c7d9defcb0 100644
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -298,8 +298,8 @@ def _test_sync_stuck_for_around_5s(self, dir_path, file_sync=False):
 
     def test_filesystem_sync_stuck_for_around_5s(self):
         """
-        To check whether the fsync will be stuck to wait for the mdlog to be
-        flushed for at most 5 seconds.
+        To check whether the filesystem sync will be stuck to wait for the
+        mdlog to be flushed for at most 5 seconds.
         """
 
         dir_path = "filesystem_sync_do_not_wait_mdlog_testdir"
@@ -307,8 +307,8 @@ def test_filesystem_sync_stuck_for_around_5s(self):
 
     def test_file_sync_stuck_for_around_5s(self):
         """
-        To check whether the filesystem sync will be stuck to wait for the
-        mdlog to be flushed for at most 5 seconds.
+        To check whether the fsync will be stuck to wait for the mdlog to
+        be flushed for at most 5 seconds.
         """
 
         dir_path = "file_sync_do_not_wait_mdlog_testdir"

From f0ebac137bf2ba6ab56c4f923bceceb0499c2ce9 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 28 Nov 2023 09:49:45 +0800
Subject: [PATCH 0748/2492] qa/tasks/cephfs/test_misc: switch duration to
 timeout

The duration is impresise and sometimes will give a false alarm
just in case the shell command itself is issued late.

https://tracker.ceph.com/issues/63587
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 qa/tasks/cephfs/test_misc.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py
index 06c7d9defcb0..72468a81361a 100644
--- a/qa/tasks/cephfs/test_misc.py
+++ b/qa/tasks/cephfs/test_misc.py
@@ -276,23 +276,23 @@ def _test_sync_stuck_for_around_5s(self, dir_path, file_sync=False):
                 self.mount_a.run_shell(["mkdir", os.path.join(dir_path, f"{i}_{j}")])
             start = time.time()
             if file_sync:
-                self.mount_a.run_shell(['python3', '-c', sync_dir_pyscript])
+                self.mount_a.run_shell(['python3', '-c', sync_dir_pyscript], timeout=4)
             else:
-                self.mount_a.run_shell(["sync"])
+                self.mount_a.run_shell(["sync"], timeout=4)
+            # the real duration should be less than the rough one
             duration = time.time() - start
-            log.info(f"sync mkdir i = {i}, duration = {duration}")
-            self.assertLess(duration, 4)
+            log.info(f"sync mkdir i = {i}, rough duration = {duration}")
 
             for j in range(5):
                 self.mount_a.run_shell(["rm", "-rf", os.path.join(dir_path, f"{i}_{j}")])
             start = time.time()
             if file_sync:
-                self.mount_a.run_shell(['python3', '-c', sync_dir_pyscript])
+                self.mount_a.run_shell(['python3', '-c', sync_dir_pyscript], timeout=4)
             else:
-                self.mount_a.run_shell(["sync"])
+                self.mount_a.run_shell(["sync"], timeout=4)
+            # the real duration should be less than the rough one
             duration = time.time() - start
-            log.info(f"sync rmdir i = {i}, duration = {duration}")
-            self.assertLess(duration, 4)
+            log.info(f"sync rmdir i = {i}, rough duration = {duration}")
 
         self.mount_a.run_shell(["rm", "-rf", dir_path])
 

From a44fd94b9681441a3251d0988d1ddd159f3733f8 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Wed, 22 Nov 2023 10:57:41 -0500
Subject: [PATCH 0749/2492] qa: remove vstart runner from radosgw_admin task

Fixes: https://tracker.ceph.com/issues/63583

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 qa/tasks/radosgw_admin.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/qa/tasks/radosgw_admin.py b/qa/tasks/radosgw_admin.py
index 780dae1e15a6..28d58715faca 100644
--- a/qa/tasks/radosgw_admin.py
+++ b/qa/tasks/radosgw_admin.py
@@ -7,8 +7,9 @@
 #   grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //'
 #
 # to run this standalone:
-#	python qa/tasks/radosgw_admin.py [--user=uid] --host=host --port=port
-#
+#   1. uncomment vstart_runner lines to run locally against a vstart cluster
+#   2. run:
+#        $ python qa/tasks/radosgw_admin.py [--user=uid] --host=host --port=port
 
 import json
 import logging
@@ -27,7 +28,7 @@
 
 #import pdb
 
-import tasks.vstart_runner
+#import tasks.vstart_runner
 from tasks.rgw import RGWEndpoint
 from tasks.util.rgw import rgwadmin as tasks_util_rgw_rgwadmin
 from tasks.util.rgw import get_user_summary, get_user_successful_ops
@@ -1107,7 +1108,7 @@ def task(ctx, config):
     (err, out) = rgwadmin(ctx, client, ['zonegroup', 'get'], check_status=True)
 
 from teuthology.config import config
-from teuthology.orchestra import cluster
+from teuthology.orchestra import cluster, remote
 
 import argparse;
 
@@ -1124,7 +1125,9 @@ def main():
     else:
         port = 80
 
-    client0 = tasks.vstart_runner.LocalRemote()
+    client0 = remote.Remote(host)
+    #client0 = tasks.vstart_runner.LocalRemote()
+
     ctx = config
     ctx.cluster=cluster.Cluster(remotes=[(client0,
         [ 'ceph.client.rgw.%s' % (port),  ]),])

From 060de1d35473e0516e502997306cf859460b6c16 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Sun, 19 Nov 2023 16:16:27 +0530
Subject: [PATCH 0750/2492] mgr/vol: rectify variable name

Method "open_clone_subvolume_pair()" returns subvolumes, yet return
value is named as clone_volume. Rectify this mistake.

Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/pybind/mgr/volumes/fs/async_cloner.py | 32 ++++++++++++++---------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py
index af7f9bb8e0ce..e785a5d5e835 100644
--- a/src/pybind/mgr/volumes/fs/async_cloner.py
+++ b/src/pybind/mgr/volumes/fs/async_cloner.py
@@ -221,21 +221,25 @@ def set_quota_on_clone(fs_handle, clone_volumes_pair):
 
 def do_clone(fs_client, volspec, volname, groupname, subvolname, should_cancel):
     with open_volume_lockless(fs_client, volname) as fs_handle:
-        with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes:
-            src_path = clone_volumes[1].snapshot_data_path(clone_volumes[2])
-            dst_path = clone_volumes[0].path
+        with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname,
+                                       groupname, subvolname) \
+            as (subvol0, subvol1, subvol2):
+            src_path = subvol1.snapshot_data_path(subvol2)
+            dst_path = subvol0.path
             # XXX: this is where cloning (of subvolume's snapshots) actually
             # happens.
             bulk_copy(fs_handle, src_path, dst_path, should_cancel)
-            set_quota_on_clone(fs_handle, clone_volumes)
+            set_quota_on_clone(fs_handle, (subvol0, subvol1, subvol2))
 
 def update_clone_failure_status(fs_client, volspec, volname, groupname, subvolname, ve):
     with open_volume_lockless(fs_client, volname) as fs_handle:
-        with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes:
+        with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname,
+                                       groupname, subvolname) \
+            as (subvol0, subvol1, subvol2) :
             if ve.errno == -errno.EINTR:
-                clone_volumes[0].add_clone_failure(-ve.errno, "user interrupted clone operation")
+                subvol0.add_clone_failure(-ve.errno, "user interrupted clone operation")
             else:
-                clone_volumes[0].add_clone_failure(-ve.errno, ve.error_str)
+                subvol0.add_clone_failure(-ve.errno, ve.error_str)
 
 def log_clone_failure(volname, groupname, subvolname, ve):
     if ve.errno == -errno.EINTR:
@@ -263,8 +267,10 @@ def handle_clone_failed(fs_client, volspec, volname, index, groupname, subvolnam
     try:
         with open_volume(fs_client, volname) as fs_handle:
             # detach source but leave the clone section intact for later inspection
-            with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes:
-                clone_volumes[1].detach_snapshot(clone_volumes[2], index)
+            with open_clone_subvolume_pair(fs_client, fs_handle, volspec,
+                                           volname, groupname, subvolname) \
+                as (subvol0, subvol1, subvol2):
+                subvol1.detach_snapshot(subvol2, index)
     except (MetadataMgrException, VolumeException) as e:
         log.error("failed to detach clone from snapshot: {0}".format(e))
     return (None, True)
@@ -272,9 +278,11 @@ def handle_clone_failed(fs_client, volspec, volname, index, groupname, subvolnam
 def handle_clone_complete(fs_client, volspec, volname, index, groupname, subvolname, should_cancel):
     try:
         with open_volume(fs_client, volname) as fs_handle:
-            with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes:
-                clone_volumes[1].detach_snapshot(clone_volumes[2], index)
-                clone_volumes[0].remove_clone_source(flush=True)
+            with open_clone_subvolume_pair(fs_client, fs_handle, volspec,
+                                           volname, groupname, subvolname) \
+                as (subvol0, subvol1, subvol2):
+                subvol1.detach_snapshot(subvol2, index)
+                subvol0.remove_clone_source(flush=True)
     except (MetadataMgrException, VolumeException) as e:
         log.error("failed to detach clone from snapshot: {0}".format(e))
     return (None, True)

From ca819633219b7978ece2201b1ab0d7dc9c0faee8 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 28 Nov 2023 05:29:50 +0100
Subject: [PATCH 0751/2492] doc/rados: ops/pgs: s/power of 2/power of two

Change "power of 2" to "power of two" in
doc/rados/operations/placement-groups.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/placement-groups.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index dda4a017715d..fb21863bb098 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -131,11 +131,11 @@ The output will resemble the following::
   if a ``pg_num`` change is in progress, the current number of PGs that the
   pool is working towards. 
 
-- **NEW PG_NUM** (if present) is the value that the system is recommending the
-  ``pg_num`` of the pool to be changed to. It is always a power of 2, and it is
-  present only if the recommended value varies from the current value by more
-  than the default factor of ``3``. To adjust this factor (in the following
-  example, it is changed to ``2``), run the following command:
+- **NEW PG_NUM** (if present) is the value that the system recommends that the
+  ``pg_num`` of the pool should be. It is always a power of two, and it
+  is present only if the recommended value varies from the current value by
+  more than the default factor of ``3``. To adjust this multiple (in the
+  following example, it is changed to ``2``), run the following command:
 
   .. prompt:: bash #
 

From 7bca5f57cc2c11bdd76dd0edb43c716a1d5ad355 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 28 Nov 2023 06:08:48 +0100
Subject: [PATCH 0752/2492] doc/rados: improve "Ceph Subsystems"

Improve the English in the subsection "Ceph Subsystems" in the section
"Subsystem, Log and Debug Settings" [sic] in
doc/rados/troubleshooting/log-and-debug.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/log-and-debug.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
index 4e2a7283ae74..7c3f848b5461 100644
--- a/doc/rados/troubleshooting/log-and-debug.rst
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -179,10 +179,12 @@ subsystem. Ceph's logging levels operate on a scale of ``1`` to ``20``, where
 logs are not sent to the output log unless one or more of the following
 conditions obtain:
 
-- a fatal signal is raised or
-- an ``assert`` in source code is triggered or
-- upon requested. Please consult `document on admin socket
-  <http://docs.ceph.com/en/latest/man/8/ceph/#daemon>`_ for more details.
+- a fatal signal has been raised or
+- an assertion within Ceph code has been triggered or
+- the sending of in-memory logs to the output log has been manually triggered.
+  Consult `the portion of the "Ceph Administration Tool documentation
+  that provides an example of how to submit admin socket commands
+  <http://docs.ceph.com/en/latest/man/8/ceph/#daemon>`_ for more detail.
 
 .. warning ::
    .. [#f1] In certain rare cases, there are logging levels that can take a value greater than 20. The resulting logs are extremely verbose.

From e5e1d6254a17d269d4d08aec66526ce72dc3103d Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 28 Nov 2023 11:14:19 +0530
Subject: [PATCH 0753/2492] qa: don't fail on gather_mount_info() failure

kernel 5.4 (Ubuntu 20.04) has the following missing commits:

- 5a9e2f5d5590 ceph: add ceph.{cluster_fsid/client_id} vxattrs
- 247b1f19dbeb ceph: add status debugfs file

fs suite relies on these debugfs entries to gather mount information
(client-id, addr/inst) which are required by some tests. In fs suite,
the disto kernel gets overridden by the testing kernel and therefore
even if Ubuntu 20.04 is chosen as the distro, the testing kernel is
installed. However, with smoke suite, the distro kernel is used and
the missing patches causes certain essential information gathering to
fail early on (client-id, etc..) causing the test to not even start
execution. PR #54515 fixes a bug in the client-id fetching path but
isn't complete due to the missing patches - details here:

        https://tracker.ceph.com/issues/63488#note-8

But its essential to have the smoke tests running since those tests
have lately uncovered bugs in the MDS (w/ distro kernels). In order
to benefit from those tests, this change ignores failures when
gathering mount information (which aren't used by the fs relevant
smoke tests). The test (in fs suite) that rely on this piece of
information would fail when run with 20.04 distro kernel (but the
fs suite overrides it with the testing kernel).

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/kernel_mount.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 750b6b5335a4..89f6b66398c9 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -68,7 +68,10 @@ def mount(self, mntopts=None, check_status=True, **kwargs):
                 self.enable_dynamic_debug()
             self.ctx[f'kmount_count.{self.client_remote.hostname}'] = kmount_count + 1
 
-        self.gather_mount_info()
+        try:
+            self.gather_mount_info()
+        except:
+            log.warn('failed to fetch mount info - tests depending on mount addr/inst may fail!')
 
     def gather_mount_info(self):
         self.id = self._get_global_id()

From d2c5c04f36d040d69df0f76679be9802eee95c8e Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 9 Nov 2023 16:51:20 +0000
Subject: [PATCH 0754/2492] crimson/common/shared_lru: introduce
 cached_key_lower_bound()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/common/shared_lru.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/crimson/common/shared_lru.h b/src/crimson/common/shared_lru.h
index 186f02a614c9..92d99d332c44 100644
--- a/src/crimson/common/shared_lru.h
+++ b/src/crimson/common/shared_lru.h
@@ -83,6 +83,7 @@ class SharedLRU {
     cache.clear();
   }
   shared_ptr_t find(const K& key);
+  K cached_key_lower_bound();
   // return the last element that is not greater than key
   shared_ptr_t lower_bound(const K& key);
   // return the first element that is greater than key
@@ -146,6 +147,15 @@ SharedLRU<K,V>::find(const K& key)
   return val;
 }
 
+template<class K, class V>
+K SharedLRU<K,V>::cached_key_lower_bound()
+{
+  if (weak_refs.empty()) {
+    return {};
+  }
+  return weak_refs.begin()->first;
+}
+
 template<class K, class V>
 typename SharedLRU<K,V>::shared_ptr_t
 SharedLRU<K,V>::lower_bound(const K& key)

From 479718e121b36ccce93fd5098d59707fc23a560c Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 12 Nov 2023 08:15:23 +0000
Subject: [PATCH 0755/2492] crimson/osd/osd_meta: add OSDMeta::remove_map

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_meta.cc | 5 +++++
 src/crimson/osd/osd_meta.h  | 1 +
 2 files changed, 6 insertions(+)

diff --git a/src/crimson/osd/osd_meta.cc b/src/crimson/osd/osd_meta.cc
index e40b2b2464be..54785a975f02 100644
--- a/src/crimson/osd/osd_meta.cc
+++ b/src/crimson/osd/osd_meta.cc
@@ -24,6 +24,11 @@ void OSDMeta::store_map(ceph::os::Transaction& t,
   t.write(coll->get_cid(), osdmap_oid(e), 0, m.length(), m);
 }
 
+void OSDMeta::remove_map(ceph::os::Transaction& t, epoch_t e)
+{
+  t.remove(coll->get_cid(), osdmap_oid(e));
+}
+
 seastar::future<bufferlist> OSDMeta::load_map(epoch_t e)
 {
   return store.read(coll,
diff --git a/src/crimson/osd/osd_meta.h b/src/crimson/osd/osd_meta.h
index 652266d9e201..3632a9feef3a 100644
--- a/src/crimson/osd/osd_meta.h
+++ b/src/crimson/osd/osd_meta.h
@@ -40,6 +40,7 @@ class OSDMeta {
 
   void store_map(ceph::os::Transaction& t,
                  epoch_t e, const bufferlist& m);
+  void remove_map(ceph::os::Transaction& t, epoch_t e);
   seastar::future<bufferlist> load_map(epoch_t e);
 
   void store_superblock(ceph::os::Transaction& t,

From 1c3629d6593da326ff97002f680133137d3cab01 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 9 Nov 2023 12:44:42 +0000
Subject: [PATCH 0756/2492] crimson/osd: introduce osdmap trimming

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc             | 16 ++++++++--
 src/crimson/osd/pg_shard_manager.h |  1 +
 src/crimson/osd/shard_services.cc  | 50 ++++++++++++++++++++++++------
 src/crimson/osd/shard_services.h   |  1 +
 4 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 8da2d566e6b5..53004f387b88 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -400,6 +400,9 @@ seastar::future<> OSD::start()
     );
   }).then([this](OSDSuperblock&& sb) {
     superblock = std::move(sb);
+    if (!superblock.cluster_osdmap_trim_lower_bound) {
+      superblock.cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
+    }
     pg_shard_manager.set_superblock(superblock);
     return pg_shard_manager.get_local_map(superblock.current_epoch);
   }).then([this](OSDMapService::local_cached_map_t&& map) {
@@ -934,6 +937,16 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
   logger().info("handle_osd_map epochs [{}..{}], i have {}, src has [{}..{}]",
                 first, last, superblock.get_newest_map(),
                 m->cluster_osdmap_trim_lower_bound, m->newest_map);
+
+  if (superblock.cluster_osdmap_trim_lower_bound <
+      m->cluster_osdmap_trim_lower_bound) {
+    superblock.cluster_osdmap_trim_lower_bound =
+      m->cluster_osdmap_trim_lower_bound;
+    logger().debug("{} superblock cluster_osdmap_trim_lower_bound new epoch is: {}",
+                   __func__, superblock.cluster_osdmap_trim_lower_bound);
+    ceph_assert(
+      superblock.cluster_osdmap_trim_lower_bound >= superblock.get_oldest_map());
+  }
   // make sure there is something new, here, before we bother flushing
   // the queues and such
   if (last <= superblock.get_newest_map()) {
@@ -964,8 +977,7 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
       monc->sub_got("osdmap", last);
 
       if (!superblock.maps.empty()) {
-        // TODO: support osdmap trimming
-        // See: <tracker>
+        pg_shard_manager.trim_maps(t, superblock);
       }
 
       superblock.insert_osdmap_epochs(first, last);
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index cf13cb52bbf7..bb74c18e3714 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -136,6 +136,7 @@ class PGShardManager {
   FORWARD_TO_OSD_SINGLETON(load_map_bl)
   FORWARD_TO_OSD_SINGLETON(load_map_bls)
   FORWARD_TO_OSD_SINGLETON(store_maps)
+  FORWARD_TO_OSD_SINGLETON(trim_maps)
 
   seastar::future<> set_up_epoch(epoch_t e);
 
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index c0688f957699..faa224f99eae 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -457,6 +457,34 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
     });
 }
 
+// Note: store/set_superblock is called in later OSD::handle_osd_map
+//       so we use the OSD's superblock reference meanwhile.
+void OSDSingletonState::trim_maps(ceph::os::Transaction& t,
+                                  OSDSuperblock& superblock)
+{
+  epoch_t min =
+    std::min(superblock.cluster_osdmap_trim_lower_bound,
+             osdmaps.cached_key_lower_bound());
+
+  if (min <= superblock.get_oldest_map()) {
+    return;
+  }
+  logger().debug("{}: min={} oldest_map={}", __func__, min,  superblock.get_oldest_map());
+
+  // Trim from the superblock's oldest_map up to `min`.
+  // Break if we have exceeded the txn target size.
+  while (superblock.get_oldest_map() < min &&
+         t.get_num_ops() < crimson::common::local_conf()->osd_target_transaction_size) {
+    logger().debug("{}: removing old osdmap epoch {}", __func__, superblock.get_oldest_map());
+    meta_coll->remove_map(t, superblock.get_oldest_map());
+    superblock.maps.erase(superblock.get_oldest_map());
+  }
+
+  // we should not trim past osdmaps.cached_key_lower_bound()
+  // as there may still be PGs with those map epochs recorded.
+  ceph_assert(min <= osdmaps.cached_key_lower_bound());
+}
+
 seastar::future<Ref<PG>> ShardServices::make_pg(
   OSDMapService::cached_map_t create_map,
   spg_t pgid,
@@ -715,30 +743,34 @@ seastar::future<> OSDSingletonState::send_incremental_map(
                 "superblock's oldest map: {}",
                 __func__, first, superblock.get_oldest_map());
   if (first >= superblock.get_oldest_map()) {
+    if (first < superblock.cluster_osdmap_trim_lower_bound) {
+      logger().info("{}: cluster osdmap lower bound: {} "
+                " > first {}, starting with full map",
+                __func__, superblock.cluster_osdmap_trim_lower_bound, first);
+      // we don't have the next map the target wants,
+      // so start with a full map.
+      first = superblock.cluster_osdmap_trim_lower_bound;
+    }
     return load_map_bls(
       first, superblock.get_newest_map()
-    ).then([this, &conn, first](auto&& bls) {
+    ).then([this, &conn](auto&& bls) {
       auto m = crimson::make_message<MOSDMap>(
 	monc.get_fsid(),
 	osdmap->get_encoding_features());
-      m->cluster_osdmap_trim_lower_bound = first;
+      m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
       m->newest_map = superblock.get_newest_map();
       m->maps = std::move(bls);
       return conn.send(std::move(m));
     });
   } else {
+    // See OSDService::send_incremental_map
+    // just send latest full map
     return load_map_bl(osdmap->get_epoch()
     ).then([this, &conn](auto&& bl) mutable {
       auto m = crimson::make_message<MOSDMap>(
 	monc.get_fsid(),
 	osdmap->get_encoding_features());
-      /* TODO: once we support the tracking of superblock's
-       *       cluster_osdmap_trim_lower_bound, the MOSDMap should
-       *       be populated with this value instead of the oldest_map.
-       *       See: OSD::handle_osd_map for how classic updates the
-       *       cluster's trim lower bound.
-       */
-      m->cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
+      m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
       m->newest_map = superblock.get_newest_map();
       m->maps.emplace(osdmap->get_epoch(), std::move(bl));
       return conn.send(std::move(m));
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index d71513a6645e..50a4bc633135 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -316,6 +316,7 @@ class OSDSingletonState : public md_config_obs_t {
                     epoch_t e, bufferlist&& bl);
   seastar::future<> store_maps(ceph::os::Transaction& t,
                                epoch_t start, Ref<MOSDMap> m);
+  void trim_maps(ceph::os::Transaction& t, OSDSuperblock& superblock);
 };
 
 /**

From d8981bc7dcf3742fb4f2aeea67a11c412df5ea17 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 23 Nov 2023 14:42:28 +0000
Subject: [PATCH 0757/2492] crimson/osd: update superblock managment

introduce PerShardState::superblock to be updated on
each shard once the OSDSingletonState instance is updated.

Note: PerShardState::superblock is a dependency for
      PG::cluster_osdmap_trim_lower_bound().

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc              | 15 +++++++++------
 src/crimson/osd/pg_shard_manager.cc |  9 +++++++++
 src/crimson/osd/pg_shard_manager.h  |  4 ++--
 src/crimson/osd/shard_services.cc   |  7 +++++++
 src/crimson/osd/shard_services.h    |  7 ++++++-
 5 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 53004f387b88..cf1984d47d9a 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -403,7 +403,8 @@ seastar::future<> OSD::start()
     if (!superblock.cluster_osdmap_trim_lower_bound) {
       superblock.cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
     }
-    pg_shard_manager.set_superblock(superblock);
+    return pg_shard_manager.set_superblock(superblock);
+  }).then([this] {
     return pg_shard_manager.get_local_map(superblock.current_epoch);
   }).then([this](OSDMapService::local_cached_map_t&& map) {
     osdmap = make_local_shared_foreign(OSDMapService::local_cached_map_t(map));
@@ -989,11 +990,13 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
         superblock.clean_thru = last;
       }
       pg_shard_manager.get_meta_coll().store_superblock(t, superblock);
-      pg_shard_manager.set_superblock(superblock);
-      logger().debug("OSD::handle_osd_map: do_transaction...");
-      return store.get_sharded_store().do_transaction(
-	pg_shard_manager.get_meta_coll().collection(),
-	std::move(t));
+      return pg_shard_manager.set_superblock(superblock).then(
+      [this, &t] {
+        logger().debug("OSD::handle_osd_map: do_transaction...");
+        return store.get_sharded_store().do_transaction(
+          pg_shard_manager.get_meta_coll().collection(),
+          std::move(t));
+      });
     });
   }).then([=, this] {
     // TODO: write to superblock and commit the transaction
diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc
index 6061c856be26..1980b36271c2 100644
--- a/src/crimson/osd/pg_shard_manager.cc
+++ b/src/crimson/osd/pg_shard_manager.cc
@@ -105,4 +105,13 @@ seastar::future<> PGShardManager::set_up_epoch(epoch_t e) {
     });
 }
 
+seastar::future<> PGShardManager::set_superblock(OSDSuperblock superblock) {
+  ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
+  get_osd_singleton_state().set_singleton_superblock(superblock);
+  return shard_services.invoke_on_all(
+  [superblock = std::move(superblock)](auto &local_service) {
+    return local_service.local_state.update_shard_superblock(superblock);
+  });
+}
+
 }
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index bb74c18e3714..1d101ce26dc2 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -129,8 +129,6 @@ class PGShardManager {
   FORWARD_TO_OSD_SINGLETON(init_meta_coll)
   FORWARD_TO_OSD_SINGLETON(get_meta_coll)
 
-  FORWARD_TO_OSD_SINGLETON(set_superblock)
-
   // Core OSDMap methods
   FORWARD_TO_OSD_SINGLETON(get_local_map)
   FORWARD_TO_OSD_SINGLETON(load_map_bl)
@@ -140,6 +138,8 @@ class PGShardManager {
 
   seastar::future<> set_up_epoch(epoch_t e);
 
+  seastar::future<> set_superblock(OSDSuperblock superblock);
+
   template <typename F>
   auto with_remote_shard_state(core_id_t core, F &&f) {
     return shard_services.invoke_on(
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index faa224f99eae..208b7fc98df6 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -119,6 +119,13 @@ HeartbeatStampsRef PerShardState::get_hb_stamps(int peer)
   return stamps->second;
 }
 
+seastar::future<> PerShardState::update_shard_superblock(OSDSuperblock superblock)
+{
+  assert_core();
+  per_shard_superblock = std::move(superblock);
+  return seastar::now();
+}
+
 OSDSingletonState::OSDSingletonState(
   int whoami,
   crimson::net::Messenger &cluster_msgr,
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 50a4bc633135..5df425a243cf 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -181,6 +181,8 @@ class PerShardState {
   HeartbeatStampsRef get_hb_stamps(int peer);
   std::map<int, HeartbeatStampsRef> heartbeat_stamps;
 
+  seastar::future<> update_shard_superblock(OSDSuperblock superblock);
+
   // Time state
   const ceph::mono_time startup_time;
   ceph::signedspan get_mnow() const {
@@ -188,6 +190,8 @@ class PerShardState {
     return ceph::mono_clock::now() - startup_time;
   }
 
+  OSDSuperblock per_shard_superblock;
+
 public:
   PerShardState(
     int whoami,
@@ -256,7 +260,7 @@ class OSDSingletonState : public md_config_obs_t {
   }
 
   OSDSuperblock superblock;
-  void set_superblock(OSDSuperblock _superblock) {
+  void set_singleton_superblock(OSDSuperblock _superblock) {
     superblock = std::move(_superblock);
   }
 
@@ -509,6 +513,7 @@ class ShardServices : public OSDMapService {
   FORWARD_TO_OSD_SINGLETON(send_pg_temp)
   FORWARD_TO_LOCAL_CONST(get_mnow)
   FORWARD_TO_LOCAL(get_hb_stamps)
+  FORWARD_TO_LOCAL(update_shard_superblock)
 
   FORWARD(pg_created, pg_created, local_state.pg_map)
 

From 1d96a6c2a0cc439cfb790cbd22dc37af9ef1aefc Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 9 Nov 2023 16:34:44 +0000
Subject: [PATCH 0758/2492] osd/crimson/pg: support
 cluster_osdmap_trim_lower_bound()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.h             | 3 +--
 src/crimson/osd/shard_services.h | 5 +++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 056d5e9e1c00..f56f2dc66273 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -348,8 +348,7 @@ class PG : public boost::intrusive_ref_counter<
   void on_active_advmap(const OSDMapRef &osdmap) final;
 
   epoch_t cluster_osdmap_trim_lower_bound() final {
-    // TODO
-    return 0;
+    return shard_services.get_osdmap_tlb();
   }
 
   void on_backfill_reserved() final {
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 5df425a243cf..51720d706ebe 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -77,6 +77,10 @@ class PerShardState {
   PerfCounters *perf = nullptr;
   PerfCounters *recoverystate_perf = nullptr;
 
+  const epoch_t& get_osdmap_tlb() {
+    return per_shard_superblock.cluster_osdmap_trim_lower_bound;
+  }
+
   // Op Management
   OSDOperationRegistry registry;
   OperationThrottler throttler;
@@ -514,6 +518,7 @@ class ShardServices : public OSDMapService {
   FORWARD_TO_LOCAL_CONST(get_mnow)
   FORWARD_TO_LOCAL(get_hb_stamps)
   FORWARD_TO_LOCAL(update_shard_superblock)
+  FORWARD_TO_LOCAL(get_osdmap_tlb)
 
   FORWARD(pg_created, pg_created, local_state.pg_map)
 

From 8093184f13cf278d1099cb388092e0b3f1dd4b91 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 12 Nov 2023 08:29:50 +0000
Subject: [PATCH 0759/2492] crimson/osd: update TODO comments

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc            | 2 ++
 src/crimson/osd/shard_services.cc | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index cf1984d47d9a..2ff907a79362 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -979,6 +979,8 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
 
       if (!superblock.maps.empty()) {
         pg_shard_manager.trim_maps(t, superblock);
+        // TODO: once we support pg splitting, update pg_num_history here
+        //pg_num_history.prune(superblock.get_oldest_map());
       }
 
       superblock.insert_osdmap_epochs(first, last);
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 208b7fc98df6..c73613dd9888 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -358,7 +358,6 @@ void OSDSingletonState::handle_conf_change(
 seastar::future<OSDSingletonState::local_cached_map_t>
 OSDSingletonState::get_local_map(epoch_t e)
 {
-  // TODO: use LRU cache for managing osdmap, fallback to disk if we have to
   if (auto found = osdmaps.find(e); found) {
     logger().debug("{} osdmap.{} found in cache", __func__, e);
     return seastar::make_ready_future<local_cached_map_t>(std::move(found));
@@ -398,6 +397,9 @@ seastar::future<std::map<epoch_t, bufferlist>> OSDSingletonState::load_map_bls(
   logger().debug("{} loading maps [{},{}]",
                  __func__, first, last);
   ceph_assert(first <= last);
+  // TODO: take osd_map_max into account
+  //int max = cct->_conf->osd_map_message_max;
+  //ssize_t max_bytes = cct->_conf->osd_map_message_max_bytes;
   return seastar::map_reduce(boost::make_counting_iterator<epoch_t>(first),
 			     boost::make_counting_iterator<epoch_t>(last + 1),
 			     [this](epoch_t e) {
@@ -750,6 +752,8 @@ seastar::future<> OSDSingletonState::send_incremental_map(
                 "superblock's oldest map: {}",
                 __func__, first, superblock.get_oldest_map());
   if (first >= superblock.get_oldest_map()) {
+    // TODO: osd_map_share_max_epochs
+    // See OSDService::build_incremental_map_msg
     if (first < superblock.cluster_osdmap_trim_lower_bound) {
       logger().info("{}: cluster osdmap lower bound: {} "
                 " > first {}, starting with full map",

From 633e6b24da1a45a918491fb8f776fcb88fa5d6c3 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 12 Nov 2023 13:31:40 +0000
Subject: [PATCH 0760/2492] crimson/osd: OSDBeacon to report lec

See: https://github.com/ceph/ceph/pull/14504

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/mgr/client.h          |  2 +-
 src/crimson/osd/osd.cc            | 20 +++++++++++++-------
 src/crimson/osd/osd.h             |  5 ++++-
 src/crimson/osd/shard_services.cc |  2 +-
 src/crimson/osd/shard_services.h  |  2 +-
 5 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/crimson/mgr/client.h b/src/crimson/mgr/client.h
index 501949768ddd..b88c60c5e4a4 100644
--- a/src/crimson/mgr/client.h
+++ b/src/crimson/mgr/client.h
@@ -24,7 +24,7 @@ namespace crimson::mgr
 // implement WithStats if you want to report stats to mgr periodically
 class WithStats {
 public:
-  virtual seastar::future<MessageURef> get_stats() const = 0;
+  virtual seastar::future<MessageURef> get_stats() = 0;
   virtual ~WithStats() {}
 };
 
diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 2ff907a79362..1b574abd8fa3 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -883,13 +883,20 @@ void OSD::update_stats()
   });
 }
 
-seastar::future<MessageURef> OSD::get_stats() const
+seastar::future<MessageURef> OSD::get_stats()
 {
   // MPGStats::had_map_for is not used since PGMonitor was removed
   auto m = crimson::make_message<MPGStats>(monc->get_fsid(), osdmap->get_epoch());
   m->osd_stat = osd_stat;
   return pg_shard_manager.get_pg_stats(
-  ).then([m=std::move(m)](auto &&stats) mutable {
+  ).then([this, m=std::move(m)](auto &&stats) mutable {
+    min_last_epoch_clean = osdmap->get_epoch();
+    min_last_epoch_clean_pgs.clear();
+    for (auto [pgid, stat] : stats) {
+      min_last_epoch_clean = std::min(min_last_epoch_clean,
+                                      stat.get_effective_last_epoch_clean());
+      min_last_epoch_clean_pgs.push_back(pgid);
+    }
     m->pg_stat = std::move(stats);
     return seastar::make_ready_future<MessageURef>(std::move(m));
   });
@@ -1283,14 +1290,13 @@ seastar::future<> OSD::send_beacon()
   if (!pg_shard_manager.is_active()) {
     return seastar::now();
   }
-  // FIXME: min lec should be calculated from pg_stat
-  //        and should set m->pgs
-  epoch_t min_last_epoch_clean = osdmap->get_epoch();
-  auto m = crimson::make_message<MOSDBeacon>(osdmap->get_epoch(),
+  auto beacon = crimson::make_message<MOSDBeacon>(osdmap->get_epoch(),
                                     min_last_epoch_clean,
                                     superblock.last_purged_snaps_scrub,
                                     local_conf()->osd_beacon_report_interval);
-  return monc->send_message(std::move(m));
+  beacon->pgs = min_last_epoch_clean_pgs;
+  logger().debug("{} {}", __func__, *beacon);
+  return monc->send_message(std::move(beacon));
 }
 
 seastar::future<> OSD::update_heartbeat_peers()
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 10ff60d47017..8125e89fbaf1 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -106,8 +106,11 @@ class OSD final : public crimson::net::Dispatcher,
   // pg statistics including osd ones
   osd_stat_t osd_stat;
   uint32_t osd_stat_seq = 0;
+  epoch_t min_last_epoch_clean = 0;
+  // which pgs were scanned for min_lec
+  std::vector<pg_t> min_last_epoch_clean_pgs;
   void update_stats();
-  seastar::future<MessageURef> get_stats() const final;
+  seastar::future<MessageURef> get_stats() final;
 
   // AuthHandler methods
   void handle_authentication(const EntityName& name,
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index c73613dd9888..e81c96562ca6 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -72,7 +72,7 @@ seastar::future<> PerShardState::stop_pgs()
     });
 }
 
-std::map<pg_t, pg_stat_t> PerShardState::get_pg_stats() const
+std::map<pg_t, pg_stat_t> PerShardState::get_pg_stats()
 {
   assert_core();
   std::map<pg_t, pg_stat_t> ret;
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 51720d706ebe..37993a4f6795 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -119,7 +119,7 @@ class PerShardState {
   PGMap pg_map;
 
   seastar::future<> stop_pgs();
-  std::map<pg_t, pg_stat_t> get_pg_stats() const;
+  std::map<pg_t, pg_stat_t> get_pg_stats();
   seastar::future<> broadcast_map_to_pgs(
     ShardServices &shard_services,
     epoch_t epoch);

From 9a58b2e63b7f12c54101314abb8d29d4f862b119 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 23 Nov 2023 15:15:58 +0000
Subject: [PATCH 0761/2492] crimson/osd: update osd_beacon_report_interval on
 conf change

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc | 19 +++++++++++++++++++
 src/crimson/osd/osd.h  |  7 ++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 1b574abd8fa3..f3648c6df277 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -868,6 +868,25 @@ void OSD::handle_authentication(const EntityName& name,
   }
 }
 
+const char** OSD::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "osd_beacon_report_interval",
+    nullptr
+  };
+  return KEYS;
+}
+
+void OSD::handle_conf_change(
+  const crimson::common::ConfigProxy& conf,
+  const std::set <std::string> &changed)
+{
+  if (changed.count("osd_beacon_report_interval")) {
+    beacon_timer.rearm_periodic(
+      std::chrono::seconds(conf->osd_beacon_report_interval));
+  }
+}
+
 void OSD::update_stats()
 {
   osd_stat_seq++;
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 8125e89fbaf1..134376ad947e 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -61,7 +61,8 @@ class PG;
 
 class OSD final : public crimson::net::Dispatcher,
 		  private crimson::common::AuthHandler,
-		  private crimson::mgr::WithStats {
+		  private crimson::mgr::WithStats,
+		  public md_config_obs_t {
   const int whoami;
   const uint32_t nonce;
   seastar::abort_source& abort_source;
@@ -126,6 +127,10 @@ class OSD final : public crimson::net::Dispatcher,
   std::unique_ptr<Heartbeat> heartbeat;
   seastar::timer<seastar::lowres_clock> tick_timer;
 
+  const char** get_tracked_conf_keys() const final;
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set<std::string> &changed) final;
+
   // admin-socket
   seastar::lw_shared_ptr<crimson::admin::AdminSocket> asok;
 

From 3e6d5273fc408efab9012883d4d16c0299e62b83 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 28 Nov 2023 17:49:03 +0000
Subject: [PATCH 0762/2492] rgw/kafka/amqp: fix race conditionn in async
 completion handlers

Fixes: https://tracker.ceph.com/issues/63314

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/driver/rados/rgw_pubsub_push.cc | 169 ++++++++----------------
 1 file changed, 55 insertions(+), 114 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_pubsub_push.cc b/src/rgw/driver/rados/rgw_pubsub_push.cc
index bdb24ce9ad10..05dc9e65d0ea 100644
--- a/src/rgw/driver/rados/rgw_pubsub_push.cc
+++ b/src/rgw/driver/rados/rgw_pubsub_push.cc
@@ -115,6 +115,55 @@ class RGWPubSubHTTPEndpoint : public RGWPubSubEndpoint {
   }
 };
 
+namespace {
+// this allows waiting untill "finish()" is called from a different thread
+// waiting could be blocking the waiting thread or yielding, depending
+// with compilation flag support and whether the optional_yield is set
+class Waiter {
+  using Signature = void(boost::system::error_code);
+  using Completion = ceph::async::Completion<Signature>;
+  using CompletionInit = boost::asio::async_completion<yield_context, Signature>;
+  std::unique_ptr<Completion> completion = nullptr;
+  int ret;
+
+  bool done = false;
+  mutable std::mutex lock;
+  mutable std::condition_variable cond;
+
+public:
+  int wait(optional_yield y) {
+    std::unique_lock l{lock};
+    if (done) {
+      return ret;
+    }
+    if (y) {
+      boost::system::error_code ec;
+      auto&& token = y.get_yield_context()[ec];
+      CompletionInit init(token);
+      completion = Completion::create(y.get_io_context().get_executor(),
+          std::move(init.completion_handler));
+      l.unlock();
+      init.result.get();
+      return -ec.value();
+    }
+    cond.wait(l, [this]{return (done==true);});
+    return ret;
+  }
+
+  void finish(int r) {
+    std::unique_lock l{lock};
+    ret = r;
+    done = true;
+    if (completion) {
+      boost::system::error_code ec(-ret, boost::system::system_category());
+      Completion::post(std::move(completion), ec);
+    } else {
+      cond.notify_all();
+    }
+  }
+};
+} // namespace
+
 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
 class RGWPubSubAMQPEndpoint : public RGWPubSubEndpoint {
 private:
@@ -187,71 +236,17 @@ class RGWPubSubAMQPEndpoint : public RGWPubSubEndpoint {
     }
   }
 
-  // this allows waiting untill "finish()" is called from a different thread
-  // waiting could be blocking the waiting thread or yielding, depending
-  // with compilation flag support and whether the optional_yield is set
-  class Waiter {
-    using Signature = void(boost::system::error_code);
-    using Completion = ceph::async::Completion<Signature>;
-    std::unique_ptr<Completion> completion = nullptr;
-    int ret;
-
-    mutable std::atomic<bool> done = false;
-    mutable std::mutex lock;
-    mutable std::condition_variable cond;
-
-    template <typename ExecutionContext, typename CompletionToken>
-    auto async_wait(ExecutionContext& ctx, CompletionToken&& token) {
-      boost::asio::async_completion<CompletionToken, Signature> init(token);
-      auto& handler = init.completion_handler;
-      {
-        std::unique_lock l{lock};
-        completion = Completion::create(ctx.get_executor(), std::move(handler));
-      }
-      return init.result.get();
-    }
-
-  public:
-    int wait(optional_yield y) {
-      if (done) {
-        return ret;
-      }
-      if (y) {
-	auto& io_ctx = y.get_io_context();
-        auto& yield_ctx = y.get_yield_context();
-        boost::system::error_code ec;
-        async_wait(io_ctx, yield_ctx[ec]);
-        return -ec.value();
-      }
-      std::unique_lock l(lock);
-      cond.wait(l, [this]{return (done==true);});
-      return ret;
-    }
-
-    void finish(int r) {
-      std::unique_lock l{lock};
-      ret = r;
-      done = true;
-      if (completion) {
-        boost::system::error_code ec(-ret, boost::system::system_category());
-        Completion::post(std::move(completion), ec);
-      } else {
-        cond.notify_all();
-      }
-    }
-  };
-
   int send_to_completion_async(CephContext* cct, const rgw_pubsub_s3_event& event, optional_yield y) override {
     if (ack_level == ack_level_t::None) {
       return amqp::publish(conn_id, topic, json_format_pubsub_event(event));
     } else {
       // TODO: currently broker and routable are the same - this will require different flags but the same mechanism
-      // note: dynamic allocation of Waiter is needed when this is invoked from a beast coroutine
-      auto w = std::unique_ptr<Waiter>(new Waiter);
+      auto w = std::make_unique<Waiter>();
       const auto rc = amqp::publish_with_confirm(conn_id, 
         topic,
         json_format_pubsub_event(event),
-        std::bind(&Waiter::finish, w.get(), std::placeholders::_1));
+        [wp = w.get()](int r) { wp->finish(r);}
+      );
       if (rc < 0) {
         // failed to publish, does not wait for reply
         return rc;
@@ -314,70 +309,16 @@ class RGWPubSubKafkaEndpoint : public RGWPubSubEndpoint {
     }
   }
 
-  // this allows waiting untill "finish()" is called from a different thread
-  // waiting could be blocking the waiting thread or yielding, depending
-  // with compilation flag support and whether the optional_yield is set
-  class Waiter {
-    using Signature = void(boost::system::error_code);
-    using Completion = ceph::async::Completion<Signature>;
-    std::unique_ptr<Completion> completion = nullptr;
-    int ret;
-
-    mutable std::atomic<bool> done = false;
-    mutable std::mutex lock;
-    mutable std::condition_variable cond;
-
-    template <typename ExecutionContext, typename CompletionToken>
-    auto async_wait(ExecutionContext& ctx, CompletionToken&& token) {
-      boost::asio::async_completion<CompletionToken, Signature> init(token);
-      auto& handler = init.completion_handler;
-      {
-        std::unique_lock l{lock};
-        completion = Completion::create(ctx.get_executor(), std::move(handler));
-      }
-      return init.result.get();
-    }
-
-  public:
-    int wait(optional_yield y) {
-      if (done) {
-        return ret;
-      }
-      if (y) {
-        auto& io_ctx = y.get_io_context();
-        auto& yield_ctx = y.get_yield_context();
-        boost::system::error_code ec;
-        async_wait(io_ctx, yield_ctx[ec]);
-        return -ec.value();
-      }
-      std::unique_lock l(lock);
-      cond.wait(l, [this]{return (done==true);});
-      return ret;
-    }
-
-    void finish(int r) {
-      std::unique_lock l{lock};
-      ret = r;
-      done = true;
-      if (completion) {
-        boost::system::error_code ec(-ret, boost::system::system_category());
-        Completion::post(std::move(completion), ec);
-      } else {
-        cond.notify_all();
-      }
-    }
-  };
-
   int send_to_completion_async(CephContext* cct, const rgw_pubsub_s3_event& event, optional_yield y) override {
     if (ack_level == ack_level_t::None) {
       return kafka::publish(conn_name, topic, json_format_pubsub_event(event));
     } else {
-      // note: dynamic allocation of Waiter is needed when this is invoked from a beast coroutine
-      auto w = std::unique_ptr<Waiter>(new Waiter);
+      auto w = std::make_unique<Waiter>();
       const auto rc = kafka::publish_with_confirm(conn_name, 
         topic,
         json_format_pubsub_event(event),
-        std::bind(&Waiter::finish, w.get(), std::placeholders::_1));
+        [wp = w.get()](int r) { wp->finish(r); }
+      );
       if (rc < 0) {
         // failed to publish, does not wait for reply
         return rc;

From 7f0cf0b7a2d94dd2189de4bef5865b024f3c7d4b Mon Sep 17 00:00:00 2001
From: Neeraj Pratap Singh <neesingh@redhat.com>
Date: Mon, 5 Dec 2022 12:25:15 +0530
Subject: [PATCH 0763/2492] qa: test cases for checking the health status after
 scrub repair

Fixes: https://tracker.ceph.com/issues/54557
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 qa/tasks/cephfs/test_forward_scrub.py | 208 ++++++++++++++++++++++++++
 1 file changed, 208 insertions(+)

diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py
index ab97a6f596ca..5f4f310909dc 100644
--- a/qa/tasks/cephfs/test_forward_scrub.py
+++ b/qa/tasks/cephfs/test_forward_scrub.py
@@ -9,6 +9,7 @@
 """
 import logging
 import json
+import errno
 
 from collections import namedtuple
 from io import BytesIO
@@ -46,6 +47,9 @@ def _get_paths_to_ino(self):
 
         return inos
 
+    def _is_MDS_damage(self):
+        return "MDS_DAMAGE" in self.mds_cluster.mon_manager.get_mon_health()['checks']
+
     def test_apply_tag(self):
         self.mount_a.run_shell(["mkdir", "parentdir"])
         self.mount_a.run_shell(["mkdir", "parentdir/childdir"])
@@ -298,3 +302,207 @@ def test_backtrace_repair(self):
         backtrace = self.fs.read_backtrace(file_ino)
         self.assertEqual(['alpha', 'parent_a'],
                          [a['dname'] for a in backtrace['ancestors']])
+
+    def test_health_status_after_dentry_repair(self):
+        """
+        Test that the damage health status is cleared
+        after the damaged dentry is repaired
+        """
+        # Create a file for checks
+        self.mount_a.run_shell(["mkdir", "subdir/"])
+
+        self.mount_a.run_shell(["touch", "subdir/file_undamaged"])
+        self.mount_a.run_shell(["touch", "subdir/file_to_be_damaged"])
+
+        subdir_ino = self.mount_a.path_to_ino("subdir")
+
+        self.mount_a.umount_wait()
+        for mds_name in self.fs.get_active_names():
+            self.fs.mds_asok(["flush", "journal"], mds_name)
+
+        self.fs.fail()
+
+        # Corrupt a dentry
+        junk = "deadbeef" * 10
+        dirfrag_obj = "{0:x}.00000000".format(subdir_ino)
+        self.fs.radosm(["setomapval", dirfrag_obj, "file_to_be_damaged_head", junk])
+
+        # Start up and try to list it
+        self.fs.set_joinable()
+        self.fs.wait_for_daemons()
+
+        self.mount_a.mount_wait()
+        dentries = self.mount_a.ls("subdir/")
+
+        # The damaged guy should have disappeared
+        self.assertEqual(dentries, ["file_undamaged"])
+
+        # I should get ENOENT if I try and read it normally, because
+        # the dir is considered complete
+        try:
+            self.mount_a.stat("subdir/file_to_be_damaged", wait=True)
+        except CommandFailedError as e:
+            self.assertEqual(e.exitstatus, errno.ENOENT)
+        else:
+            raise AssertionError("Expected ENOENT")
+
+        nfiles = self.mount_a.getfattr("./subdir", "ceph.dir.files")
+        self.assertEqual(nfiles, "2")
+
+        self.mount_a.umount_wait()
+
+        out_json = self.fs.run_scrub(["start", "/subdir", "recursive"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+
+        # Check that an entry for dentry damage is created in the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 1)
+        self.assertEqual(damage[0]['damage_type'], "dentry")
+        self.wait_until_true(lambda: self._is_MDS_damage(), timeout=100)
+
+        out_json = self.fs.run_scrub(["start", "/subdir", "repair,recursive"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+
+        # Check that the entry is cleared from the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 0)
+        self.wait_until_true(lambda: not self._is_MDS_damage(), timeout=100)
+
+        self.mount_a.mount_wait()
+
+        # Check that the file count is now correct
+        nfiles = self.mount_a.getfattr("./subdir", "ceph.dir.files")
+        self.assertEqual(nfiles, "1")
+
+        # Clean up the omap object
+        self.fs.radosm(["setomapval", dirfrag_obj, "file_to_be_damaged_head", junk])
+
+    def test_health_status_after_dirfrag_repair(self):
+        """
+        Test that the damage health status is cleared
+        after the damaged dirfrag is repaired
+        """
+        self.mount_a.run_shell(["mkdir", "dir"])
+        self.mount_a.run_shell(["touch", "dir/file"])
+        self.mount_a.run_shell(["mkdir", "testdir"])
+        self.mount_a.run_shell(["ln", "dir/file", "testdir/hardlink"])
+
+        dir_ino = self.mount_a.path_to_ino("dir")
+
+        # Ensure everything is written to backing store
+        self.mount_a.umount_wait()
+        self.fs.mds_asok(["flush", "journal"])
+
+        # Drop everything from the MDS cache
+        self.fs.fail()
+
+        self.fs.radosm(["rm", "{0:x}.00000000".format(dir_ino)])
+
+        self.fs.journal_tool(['journal', 'reset'], 0)
+        self.fs.set_joinable()
+        self.fs.wait_for_daemons()
+        self.mount_a.mount_wait()
+
+        # Check that touching the hardlink gives EIO
+        ran = self.mount_a.run_shell(["stat", "testdir/hardlink"], wait=False)
+        try:
+            ran.wait()
+        except CommandFailedError:
+            self.assertTrue("Input/output error" in ran.stderr.getvalue())
+
+        out_json = self.fs.run_scrub(["start", "/dir", "recursive"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+
+        # Check that an entry is created in the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 3)
+        damage_types = set()
+        for i in range(0, 3):
+            damage_types.add(damage[i]['damage_type'])
+        self.assertIn("dir_frag", damage_types)
+        self.wait_until_true(lambda: self._is_MDS_damage(), timeout=100)
+
+        out_json = self.fs.run_scrub(["start", "/dir", "recursive,repair"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+
+        # Check that the entry is cleared from the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 1)
+        self.assertNotEqual(damage[0]['damage_type'], "dir_frag")
+
+        self.mount_a.umount_wait()
+        self.fs.mds_asok(["flush", "journal"])
+        self.fs.fail()
+
+        # Run cephfs-data-scan
+        self.fs.data_scan(["scan_extents", self.fs.get_data_pool_name()])
+        self.fs.data_scan(["scan_inodes", self.fs.get_data_pool_name()])
+        self.fs.data_scan(["scan_links"])
+
+        self.fs.set_joinable()
+        self.fs.wait_for_daemons()
+        self.mount_a.mount_wait()
+
+        out_json = self.fs.run_scrub(["start", "/dir", "recursive,repair"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 0)
+        self.wait_until_true(lambda: not self._is_MDS_damage(), timeout=100)
+
+    def test_health_status_after_backtrace_repair(self):
+        """
+        Test that the damage health status is cleared
+        after the damaged backtrace is repaired
+        """
+        # Create a file for checks
+        self.mount_a.run_shell(["mkdir", "dir_test"])
+        self.mount_a.run_shell(["touch", "dir_test/file"])
+        file_ino = self.mount_a.path_to_ino("dir_test/file")
+
+        # That backtrace and layout are written after initial flush
+        self.fs.mds_asok(["flush", "journal"])
+        backtrace = self.fs.read_backtrace(file_ino)
+        self.assertEqual(['file', 'dir_test'],
+                         [a['dname'] for a in backtrace['ancestors']])
+
+        # Corrupt the backtrace
+        self.fs._write_data_xattr(file_ino, "parent",
+                                  "The backtrace is corrupted")
+
+        out_json = self.fs.run_scrub(["start", "/", "recursive"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+        
+        # Check that an entry for backtrace damage is created in the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 1)
+        self.assertEqual(damage[0]['damage_type'], "backtrace")
+        self.wait_until_true(lambda: self._is_MDS_damage(), timeout=100)
+
+        out_json = self.fs.run_scrub(["start", "/", "repair,recursive,force"])
+        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
+
+        # Check that the entry is cleared from the damage table
+        damage = json.loads(
+            self.fs.mon_manager.raw_cluster_cmd(
+                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
+                "damage", "ls", '--format=json-pretty'))
+        self.assertEqual(len(damage), 0)
+        self.wait_until_true(lambda: not self._is_MDS_damage(), timeout=100)

From 61a1a2a526c3fcd937e11a7e32f1e53e4fdf12bc Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Sep 2023 15:05:07 -0700
Subject: [PATCH 0764/2492] PeeringState: introduce optional
 on_replica_activate notification

The OSD will use it to activate the scrubber in replica mode.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/osd/PeeringState.cc | 2 ++
 src/osd/PeeringState.h  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 2c41b7b71189..c81cb958a7a9 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -2967,6 +2967,8 @@ void PeeringState::activate(
 
     state_set(PG_STATE_ACTIVATING);
     pl->on_activate(std::move(to_trim));
+  } else {
+    pl->on_replica_activate();
   }
   if (acting_set_writeable()) {
     PGLog::LogEntryHandlerRef rollbacker{pl->get_log_handler(t)};
diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index b9f8401b05b6..89ef702e0fea 100644
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -389,6 +389,7 @@ class PeeringState : public MissingLoc::MappingInfo {
     virtual void on_role_change() = 0;
     virtual void on_change(ObjectStore::Transaction &t) = 0;
     virtual void on_activate(interval_set<snapid_t> to_trim) = 0;
+    virtual void on_replica_activate() {}
     virtual void on_activate_complete() = 0;
     virtual void on_new_interval() = 0;
     virtual Context *on_clean() = 0;

From 601c4b5a310a687f8e979a41c28dd33affa05479 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 29 Nov 2023 07:16:24 -0600
Subject: [PATCH 0765/2492] osd/scrub: use OpCarryingEvent for scrub events
 carrying common data

Copied from a similar implementation in Crimson.
The payload includes the MOSDScrub* op, and the 'from' id.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>

Split from "osd/scrub: move responsibility for clearing the 'being reserved' state"
---
 src/osd/scrubber/scrub_machine.h | 63 ++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 24 deletions(-)

diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index a67a85b8e112..fcca0d2cc40d 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -48,39 +48,54 @@ namespace mpl = ::boost::mpl;
 void on_event_creation(std::string_view nm);
 void on_event_discard(std::string_view nm);
 
-// reservation grant/reject events carry the peer's response:
 
-/// a replica has granted our reservation request
-struct ReplicaGrant : sc::event<ReplicaGrant> {
-  OpRequestRef m_op;
-  pg_shard_t m_from;
-  ReplicaGrant(OpRequestRef op, pg_shard_t from) : m_op{op}, m_from{from}
+template <typename EV>
+struct OpCarryingEvent : sc::event<EV> {
+  static constexpr const char* event_name = "<>";
+  const OpRequestRef m_op;
+  const pg_shard_t m_from;
+  OpCarryingEvent(OpRequestRef op, pg_shard_t from) : m_op{op}, m_from{from}
   {
-    on_event_creation("ReplicaGrant");
+    on_event_creation(static_cast<EV*>(this)->event_name);
   }
+
+  OpCarryingEvent(const OpCarryingEvent&) = default;
+  OpCarryingEvent(OpCarryingEvent&&) = default;
+  OpCarryingEvent& operator=(const OpCarryingEvent&) = default;
+  OpCarryingEvent& operator=(OpCarryingEvent&&) = default;
+
   void print(std::ostream* out) const
   {
-    *out << fmt::format("ReplicaGrant(from: {})", m_from);
+    *out << fmt::format("{} (from: {})", EV::event_name, m_from);
   }
-  std::string_view print() const { return "ReplicaGrant"; }
-  ~ReplicaGrant() { on_event_discard("ReplicaGrant"); }
+  std::string_view print() const { return EV::event_name; }
+  ~OpCarryingEvent() { on_event_discard(EV::event_name); }
 };
 
-/// a replica has denied our reservation request
-struct ReplicaReject : sc::event<ReplicaReject> {
-  OpRequestRef m_op;
-  pg_shard_t m_from;
-  ReplicaReject(OpRequestRef op, pg_shard_t from) : m_op{op}, m_from{from}
-  {
-    on_event_creation("ReplicaReject");
+#define OP_EV(T)                                                     \
+  struct T : OpCarryingEvent<T> {                                    \
+    static constexpr const char* event_name = #T;                    \
+    template <typename... Args>                                      \
+    T(Args&&... args) : OpCarryingEvent(std::forward<Args>(args)...) \
+    {                                                                \
+    }                                                                \
   }
-  void print(std::ostream* out) const
-  {
-    *out << fmt::format("ReplicaReject(from: {})", m_from);
-  }
-  std::string_view print() const { return "ReplicaReject"; }
-  ~ReplicaReject() { on_event_discard("ReplicaReject"); }
-};
+
+
+// reservation events carry peer's request/response data:
+
+/// a replica has granted our reservation request
+OP_EV(ReplicaGrant);
+
+/// a replica has denied our reservation request
+OP_EV(ReplicaReject);
+
+/// received Primary request for scrub reservation
+OP_EV(ReplicaReserveReq);
+
+/// explicit release request from the Primary
+OP_EV(ReplicaRelease);
+
 
 #define MEV(E)                                          \
   struct E : sc::event<E> {                             \

From ad96e91ee4f6d1bcd47cfd77725d27d64ee06c93 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 23 Nov 2023 16:10:00 +0530
Subject: [PATCH 0766/2492] qa/cephfs: improvements for name generators in
 test_volumes.py

Generate a name that is shorter and easier to remember.

Also, write a simpler, faster & better helper method for generating
unique names. This method will also have shorter and more concise name,
so this will be easier to type and easier to read.

Fixes: https://tracker.ceph.com/issues/63680
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py | 833 ++++++++++++++++----------------
 1 file changed, 406 insertions(+), 427 deletions(-)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index d40c0d67d541..61b21d5b55e8 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -19,11 +19,6 @@
 
 class TestVolumesHelper(CephFSTestCase):
     """Helper class for testing FS volume, subvolume group and subvolume operations."""
-    TEST_VOLUME_PREFIX = "volume"
-    TEST_SUBVOLUME_PREFIX="subvolume"
-    TEST_GROUP_PREFIX="group"
-    TEST_SNAPSHOT_PREFIX="snapshot"
-    TEST_CLONE_PREFIX="clone"
     TEST_FILE_NAME_PREFIX="subvolume_file"
 
     # for filling subvolume with data
@@ -165,35 +160,24 @@ def _verify_clone(self, subvolume, snapshot, clone,
         self._verify_clone_root(path1, path2, clone, clone_group, clone_pool)
         self._verify_clone_attrs(path1, path2)
 
-    def _generate_random_volume_name(self, count=1):
-        n = self.volume_start
-        volumes = [f"{TestVolumes.TEST_VOLUME_PREFIX}_{i:016}" for i in range(n, n+count)]
-        self.volume_start += count
-        return volumes[0] if count == 1 else volumes
-
-    def _generate_random_subvolume_name(self, count=1):
-        n = self.subvolume_start
-        subvolumes = [f"{TestVolumes.TEST_SUBVOLUME_PREFIX}_{i:016}" for i in range(n, n+count)]
-        self.subvolume_start += count
-        return subvolumes[0] if count == 1 else subvolumes
-
-    def _generate_random_group_name(self, count=1):
-        n = self.group_start
-        groups = [f"{TestVolumes.TEST_GROUP_PREFIX}_{i:016}" for i in range(n, n+count)]
-        self.group_start += count
-        return groups[0] if count == 1 else groups
-
-    def _generate_random_snapshot_name(self, count=1):
-        n = self.snapshot_start
-        snaps = [f"{TestVolumes.TEST_SNAPSHOT_PREFIX}_{i:016}" for i in range(n, n+count)]
-        self.snapshot_start += count
-        return snaps[0] if count == 1 else snaps
-
-    def _generate_random_clone_name(self, count=1):
-        n = self.clone_start
-        clones = [f"{TestVolumes.TEST_CLONE_PREFIX}_{i:016}" for i in range(n, n+count)]
-        self.clone_start += count
-        return clones[0] if count == 1 else clones
+    def _gen_name(self, name, n):
+        names = [f'{name}{random.randrange(0, 9999)}{i}' for i in range(n)]
+        return names[0] if n == 1 else names
+
+    def _gen_vol_name(self, n=1):
+        return self._gen_name('vol', n)
+
+    def _gen_subvol_name(self, n=1):
+        return self._gen_name('subvol', n)
+
+    def _gen_subvol_grp_name(self, n=1):
+        return self._gen_name('subvol_grp', n)
+
+    def _gen_subvol_snap_name(self, n=1):
+        return self._gen_name('subvol_snap', n)
+
+    def _gen_subvol_clone_name(self, n=1):
+        return self._gen_name('subvol_clone', n)
 
     def _enable_multi_fs(self):
         self._fs_cmd("flag", "set", "enable_multiple", "true", "--yes-i-really-mean-it")
@@ -202,7 +186,7 @@ def _create_or_reuse_test_volume(self):
         result = json.loads(self._fs_cmd("volume", "ls"))
         if len(result) == 0:
             self.vol_created = True
-            self.volname = self._generate_random_volume_name()
+            self.volname = self._gen_vol_name()
             self._fs_cmd("volume", "create", self.volname)
         else:
             self.volname = result[0]['name']
@@ -418,11 +402,6 @@ def setUp(self):
         self._enable_multi_fs()
         self._create_or_reuse_test_volume()
         self.config_set('mon', 'mon_allow_pool_delete', True)
-        self.volume_start = random.randint(1, (1<<20))
-        self.subvolume_start = random.randint(1, (1<<20))
-        self.group_start = random.randint(1, (1<<20))
-        self.snapshot_start = random.randint(1, (1<<20))
-        self.clone_start = random.randint(1, (1<<20))
 
     def tearDown(self):
         if self.vol_created:
@@ -436,7 +415,7 @@ def test_volume_create(self):
         """
         That the volume can be created and then cleans up
         """
-        volname = self._generate_random_volume_name()
+        volname = self._gen_vol_name()
         self._fs_cmd("volume", "create", volname)
         volumels = json.loads(self._fs_cmd("volume", "ls"))
 
@@ -467,7 +446,7 @@ def test_volume_ls(self):
         volumes = [volume['name'] for volume in vls]
 
         #create new volumes and add it to the existing list of volumes
-        volumenames = self._generate_random_volume_name(2)
+        volumenames = self._gen_vol_name(2)
         for volumename in volumenames:
             self._fs_cmd("volume", "create", volumename)
         volumes.extend(volumenames)
@@ -567,7 +546,7 @@ def test_volume_info(self):
         Tests the 'fs volume info' command
         """
         vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         # get volume metadata
@@ -582,7 +561,7 @@ def test_volume_info_pending_subvol_deletions(self):
         """
         Tests the pending_subvolume_deletions in 'fs volume info' command
         """
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--mode=777")
         # create 3K zero byte files
@@ -617,7 +596,7 @@ def test_volume_info_with_human_readable_flag(self):
         Tests the 'fs volume info --human_readable' command
         """
         vol_fields = ["pools", "used_size", "pending_subvolume_deletions", "mon_addrs"]
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         # get volume metadata
@@ -665,7 +644,7 @@ def test_volume_rename(self):
         for m in self.mounts:
             m.umount_wait()
         oldvolname = self.volname
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
 
         self.run_ceph_cmd(f'fs fail {oldvolname}')
@@ -692,7 +671,7 @@ def test_volume_rename_idempotency(self):
         for m in self.mounts:
             m.umount_wait()
         oldvolname = self.volname
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
         new_data_pool, new_metadata_pool = f"cephfs.{newvolname}.data", f"cephfs.{newvolname}.meta"
 
         self.run_ceph_cmd(f'fs fail {oldvolname}')
@@ -716,7 +695,7 @@ def test_volume_rename_fails_without_confirmation_flag(self):
         """
         That renaming volume fails without --yes-i-really-mean-it flag.
         """
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
 
         self.run_ceph_cmd(f'fs fail {self.volname}')
         self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session true')
@@ -741,7 +720,7 @@ def test_volume_rename_for_more_than_one_data_pool(self):
             m.umount_wait()
         self.fs.add_data_pool('another-data-pool')
         oldvolname = self.volname
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
         self.fs.get_pool_names(refresh=True)
         orig_data_pool_names = list(self.fs.data_pools.values())
         new_metadata_pool = f"cephfs.{newvolname}.meta"
@@ -767,7 +746,7 @@ def test_volume_rename_for_more_than_one_data_pool(self):
     def test_rename_when_fs_is_online(self):
         for m in self.mounts:
             m.umount_wait()
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
 
         self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session true')
         self.negtest_ceph_cmd(
@@ -780,7 +759,7 @@ def test_rename_when_fs_is_online(self):
         self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session false')
 
     def test_rename_when_clients_arent_refused(self):
-        newvolname = self._generate_random_volume_name()
+        newvolname = self._gen_vol_name()
         for m in self.mounts:
             m.umount_wait()
 
@@ -797,7 +776,7 @@ def test_rename_when_clients_arent_refused(self):
 class TestSubvolumeGroups(TestVolumesHelper):
     """Tests for FS subvolume group operations."""
     def test_default_uid_gid_subvolume_group(self):
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         expected_uid = 0
         expected_gid = 0
 
@@ -814,7 +793,7 @@ def test_default_uid_gid_subvolume_group(self):
         self._fs_cmd("subvolumegroup", "rm", self.volname, group)
 
     def test_nonexistent_subvolume_group_create(self):
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         group = "non_existent_group"
 
         # try, creating subvolume in a nonexistent group
@@ -839,7 +818,7 @@ def test_nonexistent_subvolume_group_rm(self):
             raise RuntimeError("expected the 'fs subvolumegroup rm' command to fail")
 
     def test_subvolume_group_create_with_auto_cleanup_on_fail(self):
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         data_pool = "invalid_pool"
         # create group with invalid data pool layout
         with self.assertRaises(CommandFailedError):
@@ -855,7 +834,7 @@ def test_subvolume_group_create_with_auto_cleanup_on_fail(self):
             raise RuntimeError("expected the 'fs subvolumegroup getpath' command to fail")
 
     def test_subvolume_group_create_with_desired_data_pool_layout(self):
-        group1, group2 = self._generate_random_group_name(2)
+        group1, group2 = self._gen_subvol_grp_name(2)
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group1)
@@ -883,7 +862,7 @@ def test_subvolume_group_create_with_desired_data_pool_layout(self):
         self._fs_cmd("subvolumegroup", "rm", self.volname, group2)
 
     def test_subvolume_group_create_with_desired_mode(self):
-        group1, group2 = self._generate_random_group_name(2)
+        group1, group2 = self._gen_subvol_grp_name(2)
         # default mode
         expected_mode1 = "755"
         # desired mode
@@ -917,7 +896,7 @@ def test_subvolume_group_create_with_desired_uid_gid(self):
         gid = 1000
 
         # create subvolume group
-        subvolgroupname = self._generate_random_group_name()
+        subvolgroupname = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, subvolgroupname, "--uid", str(uid), "--gid", str(gid))
 
         # make sure it exists
@@ -934,7 +913,7 @@ def test_subvolume_group_create_with_desired_uid_gid(self):
         self._fs_cmd("subvolumegroup", "rm", self.volname, subvolgroupname)
 
     def test_subvolume_group_create_with_invalid_data_pool_layout(self):
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         data_pool = "invalid_pool"
         # create group with invalid data pool layout
         try:
@@ -947,7 +926,7 @@ def test_subvolume_group_create_with_invalid_data_pool_layout(self):
 
     def test_subvolume_group_create_with_size(self):
         # create group with size -- should set quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group, "1000000000")
 
         # get group metadata
@@ -964,7 +943,7 @@ def test_subvolume_group_info(self):
                      "data_pool", "gid", "mode", "mon_addrs", "mtime", "uid"]
 
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # get group metadata
@@ -993,7 +972,7 @@ def test_subvolume_group_info(self):
 
     def test_subvolume_group_create_idempotence(self):
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # try creating w/ same subvolume group name -- should be idempotent
@@ -1004,7 +983,7 @@ def test_subvolume_group_create_idempotence(self):
 
     def test_subvolume_group_create_idempotence_mode(self):
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # try creating w/ same subvolume group name with mode -- should set mode
@@ -1024,7 +1003,7 @@ def test_subvolume_group_create_idempotence_uid_gid(self):
         desired_gid = 1000
 
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # try creating w/ same subvolume group name with uid/gid -- should set uid/gid
@@ -1043,7 +1022,7 @@ def test_subvolume_group_create_idempotence_uid_gid(self):
 
     def test_subvolume_group_create_idempotence_data_pool(self):
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         group_path = self._get_subvolume_group_path(self.volname, group)
@@ -1068,7 +1047,7 @@ def test_subvolume_group_create_idempotence_data_pool(self):
 
     def test_subvolume_group_create_idempotence_resize(self):
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # try creating w/ same subvolume name with size -- should set quota
@@ -1090,7 +1069,7 @@ def test_subvolume_group_quota_mds_path_restriction_to_group_path(self):
         """
         osize = self.DEFAULT_FILE_SIZE*1024*1024*100
         # create group with 100MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1099,7 +1078,7 @@ def test_subvolume_group_quota_mds_path_restriction_to_group_path(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1174,7 +1153,7 @@ def test_subvolume_group_quota_mds_path_restriction_to_subvolume_path(self):
         """
         osize = self.DEFAULT_FILE_SIZE*1024*1024*100
         # create group with 100MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1183,7 +1162,7 @@ def test_subvolume_group_quota_mds_path_restriction_to_subvolume_path(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1255,7 +1234,7 @@ def test_subvolume_group_quota_exceeded_subvolume_removal(self):
         """
         osize = self.DEFAULT_FILE_SIZE*1024*1024*100
         # create group with 100MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1264,7 +1243,7 @@ def test_subvolume_group_quota_exceeded_subvolume_removal(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1298,9 +1277,9 @@ def test_subvolume_group_quota_exceeded_subvolume_removal_retained_snaps(self):
         """
         Tests retained snapshot subvolume removal if it's group quota is exceeded
         """
-        group = self._generate_random_group_name()
-        subvolname = self._generate_random_subvolume_name()
-        snapshot1, snapshot2 = self._generate_random_snapshot_name(2)
+        group = self._gen_subvol_grp_name()
+        subvolname = self._gen_subvol_name()
+        snapshot1, snapshot2 = self._gen_subvol_snap_name(2)
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*100
         # create group with 100MB quota
@@ -1358,11 +1337,11 @@ def test_subvolume_group_quota_subvolume_removal(self):
         Tests subvolume removal if it's group quota is set.
         """
         # create group with size -- should set quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group, "1000000000")
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--group_name", group)
 
         # remove subvolume
@@ -1381,8 +1360,8 @@ def test_subvolume_group_quota_legacy_subvolume_removal(self):
         """
         Tests legacy subvolume removal if it's group quota is set.
         """
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume -- in a custom group
         createpath1 = os.path.join(".", "volumes", group, subvolume)
@@ -1415,8 +1394,8 @@ def test_subvolume_group_quota_v1_subvolume_removal(self):
         """
         Tests v1 subvolume removal if it's group quota is set.
         """
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a v1 subvolume -- in a custom group
         self._create_v1_subvolume(subvolume, subvol_group=group, has_snapshot=False)
@@ -1443,7 +1422,7 @@ def test_subvolume_group_resize_fail_invalid_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         # create group with 1MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group, "--size", str(osize))
 
         # make sure it exists
@@ -1474,7 +1453,7 @@ def test_subvolume_group_resize_fail_zero_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         # create group with 1MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group, "--size", str(osize))
 
         # make sure it exists
@@ -1506,7 +1485,7 @@ def test_subvolume_group_resize_quota_lt_used_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*20
         # create group with 20MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1515,7 +1494,7 @@ def test_subvolume_group_resize_quota_lt_used_size(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1560,7 +1539,7 @@ def test_subvolume_group_resize_fail_quota_lt_used_size_no_shrink(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*20
         # create group with 20MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1569,7 +1548,7 @@ def test_subvolume_group_resize_fail_quota_lt_used_size_no_shrink(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1615,7 +1594,7 @@ def test_subvolume_group_resize_expand_on_full_subvolume(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*100
         # create group with 100MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1624,7 +1603,7 @@ def test_subvolume_group_resize_expand_on_full_subvolume(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1669,7 +1648,7 @@ def test_subvolume_group_resize_infinite_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize))
 
@@ -1694,7 +1673,7 @@ def test_subvolume_group_resize_infinite_size_future_writes(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*5
         # create group with 5MB quota
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group,
                      "--size", str(osize), "--mode=777")
 
@@ -1703,7 +1682,7 @@ def test_subvolume_group_resize_infinite_size_future_writes(self):
         self.assertNotEqual(grouppath, None)
 
         # create subvolume under the group
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname,
                      "--group_name", group, "--mode=777")
 
@@ -1752,7 +1731,7 @@ def test_subvolume_group_ls(self):
         subvolumegroups = []
 
         #create subvolumegroups
-        subvolumegroups = self._generate_random_group_name(3)
+        subvolumegroups = self._gen_subvol_grp_name(3)
         for groupname in subvolumegroups:
             self._fs_cmd("subvolumegroup", "create", self.volname, groupname)
 
@@ -1770,12 +1749,12 @@ def test_subvolume_group_ls_filter(self):
         subvolumegroups = []
 
         #create subvolumegroup
-        subvolumegroups = self._generate_random_group_name(3)
+        subvolumegroups = self._gen_subvol_grp_name(3)
         for groupname in subvolumegroups:
             self._fs_cmd("subvolumegroup", "create", self.volname, groupname)
 
         # create subvolume and remove. This creates '_deleting' directory.
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         self._fs_cmd("subvolume", "rm", self.volname, subvolume)
 
@@ -1788,10 +1767,10 @@ def test_subvolume_group_ls_filter_internal_directories(self):
         # tests the 'fs subvolumegroup ls' command filters internal directories
         # eg: '_deleting', '_nogroup', '_index', "_legacy"
 
-        subvolumegroups = self._generate_random_group_name(3)
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolumegroups = self._gen_subvol_grp_name(3)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         #create subvolumegroups
         for groupname in subvolumegroups:
@@ -1846,7 +1825,7 @@ def test_subvolumegroup_pin_distributed(self):
         group = "pinme"
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         self._fs_cmd("subvolumegroup", "pin", self.volname, group, "distributed", "True")
-        subvolumes = self._generate_random_subvolume_name(50)
+        subvolumes = self._gen_subvol_name(50)
         for subvolume in subvolumes:
             self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group)
         self._wait_distributed_subtrees(2 * 2, status=status, rank="all")
@@ -1860,7 +1839,7 @@ def test_subvolumegroup_pin_distributed(self):
 
     def test_subvolume_group_rm_force(self):
         # test removing non-existing subvolume group with --force
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         try:
             self._fs_cmd("subvolumegroup", "rm", self.volname, group, "--force")
         except CommandFailedError:
@@ -1869,7 +1848,7 @@ def test_subvolume_group_rm_force(self):
     def test_subvolume_group_exists_with_subvolumegroup_and_no_subvolume(self):
         """Test the presence of any subvolumegroup when only subvolumegroup is present"""
 
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         ret = self._fs_cmd("subvolumegroup", "exist", self.volname)
@@ -1889,8 +1868,8 @@ def test_subvolume_group_exists_with_subvolumegroup_and_subvolume(self):
         """Test the presence of any subvolume when subvolumegroup
             and subvolume both are present"""
 
-        group = self._generate_random_group_name()
-        subvolume = self._generate_random_subvolume_name(2)
+        group = self._gen_subvol_grp_name()
+        subvolume = self._gen_subvol_name(2)
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         # create subvolume in group
@@ -1916,7 +1895,7 @@ def test_subvolume_group_exists_without_subvolumegroup_and_with_subvolume(self):
         """Test the presence of any subvolume when subvolume is present
             but no subvolumegroup is present"""
 
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         ret = self._fs_cmd("subvolumegroup", "exist", self.volname)
@@ -1927,8 +1906,8 @@ def test_subvolume_group_exists_without_subvolumegroup_and_with_subvolume(self):
         self.assertEqual(ret.strip('\n'), "no subvolumegroup exists")
 
     def test_subvolume_group_rm_when_its_not_empty(self):
-        group = self._generate_random_group_name()
-        subvolume = self._generate_random_subvolume_name()
+        group = self._gen_subvol_grp_name()
+        subvolume = self._gen_subvol_name()
 
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -1956,7 +1935,7 @@ def test_subvolume_group_rm_when_its_not_empty(self):
 class TestSubvolumes(TestVolumesHelper):
     """Tests for FS subvolume operations, except snapshot and snapshot clone."""
     def test_async_subvolume_rm(self):
-        subvolumes = self._generate_random_subvolume_name(100)
+        subvolumes = self._gen_subvol_name(100)
 
         # create subvolumes
         for subvolume in subvolumes:
@@ -1975,7 +1954,7 @@ def test_async_subvolume_rm(self):
         self._wait_for_trash_empty(timeout=300)
 
     def test_default_uid_gid_subvolume(self):
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         expected_uid = 0
         expected_gid = 0
 
@@ -2009,7 +1988,7 @@ def test_nonexistent_subvolume_rm(self):
 
     def test_subvolume_create_and_rm(self):
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         # make sure it exists
@@ -2031,8 +2010,8 @@ def test_subvolume_create_and_rm(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_and_rm_in_group(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -2051,7 +2030,7 @@ def test_subvolume_create_and_rm_in_group(self):
 
     def test_subvolume_create_idempotence(self):
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         # try creating w/ same subvolume name -- should be idempotent
@@ -2065,7 +2044,7 @@ def test_subvolume_create_idempotence(self):
 
     def test_subvolume_create_idempotence_resize(self):
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         # try creating w/ same subvolume name with size -- should set quota
@@ -2086,7 +2065,7 @@ def test_subvolume_create_idempotence_mode(self):
         default_mode = "755"
 
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         subvol_path = self._get_subvolume_path(self.volname, subvolume)
@@ -2110,7 +2089,7 @@ def test_subvolume_create_idempotence_mode(self):
     def test_subvolume_create_idempotence_without_passing_mode(self):
         # create subvolume
         desired_mode = "777"
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode", desired_mode)
 
         subvol_path = self._get_subvolume_path(self.volname, subvolume)
@@ -2139,7 +2118,7 @@ def test_subvolume_create_isolated_namespace(self):
         """
 
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--namespace-isolated")
 
         # get subvolume metadata
@@ -2154,7 +2133,7 @@ def test_subvolume_create_isolated_namespace(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_with_auto_cleanup_on_fail(self):
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         data_pool = "invalid_pool"
         # create subvolume with invalid data pool layout fails
         with self.assertRaises(CommandFailedError):
@@ -2172,8 +2151,8 @@ def test_subvolume_create_with_auto_cleanup_on_fail(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_with_desired_data_pool_layout_in_group(self):
-        subvol1, subvol2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvol1, subvol2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         # create group. this also helps set default pool layout for subvolumes
         # created within the group.
@@ -2209,7 +2188,7 @@ def test_subvolume_create_with_desired_data_pool_layout_in_group(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_with_desired_mode(self):
-        subvol1 = self._generate_random_subvolume_name()
+        subvol1 = self._gen_subvol_name()
 
         # default mode
         default_mode = "755"
@@ -2239,9 +2218,9 @@ def test_subvolume_create_with_desired_mode(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_with_desired_mode_in_group(self):
-        subvol1, subvol2, subvol3 = self._generate_random_subvolume_name(3)
+        subvol1, subvol2, subvol3 = self._gen_subvol_name(3)
 
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         # default mode
         expected_mode1 = "755"
         # desired mode
@@ -2285,7 +2264,7 @@ def test_subvolume_create_with_desired_uid_gid(self):
         gid = 1000
 
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--uid", str(uid), "--gid", str(gid))
 
         # make sure it exists
@@ -2305,7 +2284,7 @@ def test_subvolume_create_with_desired_uid_gid(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_create_with_invalid_data_pool_layout(self):
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         data_pool = "invalid_pool"
         # create subvolume with invalid data pool layout
         try:
@@ -2320,7 +2299,7 @@ def test_subvolume_create_with_invalid_data_pool_layout(self):
 
     def test_subvolume_create_with_invalid_size(self):
         # create subvolume with an invalid size -1
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         try:
             self._fs_cmd("subvolume", "create", self.volname, subvolume, "--size", "-1")
         except CommandFailedError as ce:
@@ -2337,7 +2316,7 @@ def test_subvolume_create_and_ls_providing_group_as_nogroup(self):
         permission denied error if option --group=_nogroup is provided.
         """
 
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
 
         # try to create subvolume providing --group_name=_nogroup option
         try:
@@ -2372,7 +2351,7 @@ def test_subvolume_expand(self):
         """
 
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize))
 
@@ -2402,7 +2381,7 @@ def test_subvolume_info(self):
                      "type", "uid", "features", "state"]
 
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         # get subvolume metadata
@@ -2450,7 +2429,7 @@ def test_subvolume_ls(self):
         subvolumes = []
 
         # create subvolumes
-        subvolumes = self._generate_random_subvolume_name(3)
+        subvolumes = self._gen_subvol_name(3)
         for subvolume in subvolumes:
             self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
@@ -2522,7 +2501,7 @@ def test_subvolume_marked(self):
         """
         ensure a subvolume is marked with the ceph.dir.subvolume xattr
         """
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -2558,7 +2537,7 @@ def test_subvolume_pin_export(self):
         self.fs.set_max_mds(2)
         status = self.fs.wait_for_daemons()
 
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         self._fs_cmd("subvolume", "pin", self.volname, subvolume, "export", "1")
         path = self._fs_cmd("subvolume", "getpath", self.volname, subvolume)
@@ -2576,8 +2555,8 @@ def test_subvolume_pin_export(self):
     ### authorize operations
 
     def test_authorize_deauthorize_legacy_subvolume(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
         authid = "alice"
 
         guest_mount = self.mount_b
@@ -2634,8 +2613,8 @@ def test_authorize_deauthorize_legacy_subvolume(self):
         self._fs_cmd("subvolumegroup", "rm", self.volname, group)
 
     def test_authorize_deauthorize_subvolume(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
         authid = "alice"
 
         guest_mount = self.mount_b
@@ -2697,8 +2676,8 @@ def test_multitenant_subvolumes(self):
         subvolumes is stored as a two-way mapping between auth
         IDs and subvolumes that they're authorized to access.
         """
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         guest_mount = self.mount_b
 
@@ -2805,8 +2784,8 @@ def test_multitenant_subvolumes(self):
         self._fs_cmd("subvolumegroup", "rm", self.volname, group)
 
     def test_subvolume_authorized_list(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
         authid1 = "alice"
         authid2 = "guest1"
         authid3 = "guest2"
@@ -2848,8 +2827,8 @@ def test_authorize_auth_id_not_created_by_mgr_volumes(self):
         it's not allowed to authorize the auth-id by default.
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # Create auth_id
         self.run_ceph_cmd(
@@ -2892,8 +2871,8 @@ def test_authorize_allow_existing_id_option(self):
         allowed with option allow_existing_id.
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # Create auth_id
         self.run_ceph_cmd(
@@ -2935,8 +2914,8 @@ def test_deauthorize_auth_id_after_out_of_band_update(self):
         deauthorize. It should only remove caps associated with it.
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         auth_id = "guest1"
         guestclient_1 = {
@@ -2992,8 +2971,8 @@ def test_recover_auth_metadata_during_authorize(self):
 
         guest_mount = self.mount_b
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         auth_id = "guest1"
         guestclient_1 = {
@@ -3044,8 +3023,8 @@ def test_recover_auth_metadata_during_deauthorize(self):
 
         guest_mount = self.mount_b
 
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         guestclient_1 = {
             "auth_id": "guest1",
@@ -3102,8 +3081,8 @@ def test_update_old_style_auth_metadata_to_new_during_authorize(self):
 
         guest_mount = self.mount_b
 
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         auth_id = "guest1"
         guestclient_1 = {
@@ -3177,8 +3156,8 @@ def test_update_old_style_auth_metadata_to_new_during_deauthorize(self):
 
         guest_mount = self.mount_b
 
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         auth_id = "guest1"
         guestclient_1 = {
@@ -3244,8 +3223,8 @@ def test_subvolume_evict_client(self):
         That a subvolume client can be evicted based on the auth ID
         """
 
-        subvolumes = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolumes = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3313,7 +3292,7 @@ def test_subvolume_pin_random(self):
         self.fs.wait_for_daemons()
         self.config_set('mds', 'mds_export_ephemeral_random', True)
 
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         self._fs_cmd("subvolume", "pin", self.volname, subvolume, "random", ".01")
         # no verification
@@ -3331,7 +3310,7 @@ def test_subvolume_resize_fail_invalid_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize))
 
         # make sure it exists
@@ -3364,7 +3343,7 @@ def test_subvolume_resize_fail_zero_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize))
 
         # make sure it exists
@@ -3398,7 +3377,7 @@ def test_subvolume_resize_quota_lt_used_size(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*20
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777")
 
         # make sure it exists
@@ -3445,7 +3424,7 @@ def test_subvolume_resize_fail_quota_lt_used_size_no_shrink(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*20
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777")
 
         # make sure it exists
@@ -3493,7 +3472,7 @@ def test_subvolume_resize_expand_on_full_subvolume(self):
 
         osize = self.DEFAULT_FILE_SIZE*1024*1024*10
         # create subvolume of quota 10MB and make sure it exists
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777")
         subvolpath = self._get_subvolume_path(self.volname, subvolname)
         self.assertNotEqual(subvolpath, None)
@@ -3541,7 +3520,7 @@ def test_subvolume_resize_infinite_size(self):
         """
 
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size",
                      str(self.DEFAULT_FILE_SIZE*1024*1024))
 
@@ -3568,7 +3547,7 @@ def test_subvolume_resize_infinite_size_future_writes(self):
         """
 
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size",
                      str(self.DEFAULT_FILE_SIZE*1024*1024*5), "--mode=777")
 
@@ -3605,7 +3584,7 @@ def test_subvolume_resize_infinite_size_future_writes(self):
 
     def test_subvolume_rm_force(self):
         # test removing non-existing subvolume with --force
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         try:
             self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--force")
         except CommandFailedError:
@@ -3614,8 +3593,8 @@ def test_subvolume_rm_force(self):
     def test_subvolume_exists_with_subvolumegroup_and_subvolume(self):
         """Test the presence of any subvolume by specifying the name of subvolumegroup"""
 
-        group = self._generate_random_group_name()
-        subvolume1 = self._generate_random_subvolume_name()
+        group = self._gen_subvol_grp_name()
+        subvolume1 = self._gen_subvol_name()
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         # create subvolume in group
@@ -3633,7 +3612,7 @@ def test_subvolume_exists_with_subvolumegroup_and_no_subvolume(self):
         """Test the presence of any subvolume specifying the name
             of subvolumegroup and no subvolumes"""
 
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         # create subvolumegroup
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
         ret = self._fs_cmd("subvolume", "exist", self.volname, "--group_name", group)
@@ -3645,7 +3624,7 @@ def test_subvolume_exists_without_subvolumegroup_and_with_subvolume(self):
         """Test the presence of any subvolume without specifying the name
             of subvolumegroup"""
 
-        subvolume1 = self._generate_random_subvolume_name()
+        subvolume1 = self._gen_subvol_name()
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume1)
         ret = self._fs_cmd("subvolume", "exist", self.volname)
@@ -3668,7 +3647,7 @@ def test_subvolume_shrink(self):
         """
 
         # create subvolume
-        subvolname = self._generate_random_subvolume_name()
+        subvolname = self._gen_subvol_name()
         osize = self.DEFAULT_FILE_SIZE*1024*1024
         self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize))
 
@@ -3697,8 +3676,8 @@ def test_subvolume_retain_snapshot_rm_idempotency(self):
         is cleaned up. The subvolume deletion issued while the trash directory is not empty, should pass and should
         not error out with EAGAIN.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -3727,8 +3706,8 @@ def test_subvolume_retain_snapshot_rm_idempotency(self):
 
 
     def test_subvolume_user_metadata_set(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3751,8 +3730,8 @@ def test_subvolume_user_metadata_set(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_set_idempotence(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3781,8 +3760,8 @@ def test_subvolume_user_metadata_set_idempotence(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_get(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3814,8 +3793,8 @@ def test_subvolume_user_metadata_get(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_get_for_nonexisting_key(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3844,8 +3823,8 @@ def test_subvolume_user_metadata_get_for_nonexisting_key(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_get_for_nonexisting_section(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3869,8 +3848,8 @@ def test_subvolume_user_metadata_get_for_nonexisting_section(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_update(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3906,8 +3885,8 @@ def test_subvolume_user_metadata_update(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_list(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3939,8 +3918,8 @@ def test_subvolume_user_metadata_list(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_list_if_no_metadata_set(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -3968,8 +3947,8 @@ def test_subvolume_user_metadata_list_if_no_metadata_set(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_remove(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4003,8 +3982,8 @@ def test_subvolume_user_metadata_remove(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_remove_for_nonexisting_key(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4033,8 +4012,8 @@ def test_subvolume_user_metadata_remove_for_nonexisting_key(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_remove_for_nonexisting_section(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4058,8 +4037,8 @@ def test_subvolume_user_metadata_remove_for_nonexisting_section(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_remove_force(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4093,8 +4072,8 @@ def test_subvolume_user_metadata_remove_force(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_remove_force_for_nonexisting_key(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4134,8 +4113,8 @@ def test_subvolume_user_metadata_remove_force_for_nonexisting_key(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_set_and_get_for_legacy_subvolume(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume in a custom group
         createpath = os.path.join(".", "volumes", group, subvolname)
@@ -4168,8 +4147,8 @@ def test_subvolume_user_metadata_set_and_get_for_legacy_subvolume(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_user_metadata_list_and_remove_for_legacy_subvolume(self):
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume in a custom group
         createpath = os.path.join(".", "volumes", group, subvolname)
@@ -4216,9 +4195,9 @@ class TestSubvolumeGroupSnapshots(TestVolumesHelper):
     """Tests for FS subvolume group snapshot operations."""
     @unittest.skip("skipping subvolumegroup snapshot tests")
     def test_nonexistent_subvolume_group_snapshot_rm(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4252,9 +4231,9 @@ def test_nonexistent_subvolume_group_snapshot_rm(self):
 
     @unittest.skip("skipping subvolumegroup snapshot tests")
     def test_subvolume_group_snapshot_create_and_rm(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4279,9 +4258,9 @@ def test_subvolume_group_snapshot_create_and_rm(self):
 
     @unittest.skip("skipping subvolumegroup snapshot tests")
     def test_subvolume_group_snapshot_idempotence(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4314,11 +4293,11 @@ def test_subvolume_group_snapshot_ls(self):
         snapshots = []
 
         # create group
-        group = self._generate_random_group_name()
+        group = self._gen_subvol_grp_name()
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
 
         # create subvolumegroup snapshots
-        snapshots = self._generate_random_snapshot_name(3)
+        snapshots = self._gen_subvol_snap_name(3)
         for snapshot in snapshots:
             self._fs_cmd("subvolumegroup", "snapshot", "create", self.volname, group, snapshot)
 
@@ -4333,8 +4312,8 @@ def test_subvolume_group_snapshot_ls(self):
     @unittest.skip("skipping subvolumegroup snapshot tests")
     def test_subvolume_group_snapshot_rm_force(self):
         # test removing non-existing subvolume group snapshot with --force
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
         # remove snapshot
         try:
             self._fs_cmd("subvolumegroup", "snapshot", "rm", self.volname, group, snapshot, "--force")
@@ -4342,8 +4321,8 @@ def test_subvolume_group_snapshot_rm_force(self):
             raise RuntimeError("expected the 'fs subvolumegroup snapshot rm --force' command to succeed")
 
     def test_subvolume_group_snapshot_unsupported_status(self):
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4363,8 +4342,8 @@ def test_subvolume_group_snapshot_unsupported_status(self):
 class TestSubvolumeSnapshots(TestVolumesHelper):
     """Tests for FS subvolume snapshot operations."""
     def test_nonexistent_subvolume_snapshot_rm(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4391,8 +4370,8 @@ def test_nonexistent_subvolume_snapshot_rm(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_create_and_rm(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4410,8 +4389,8 @@ def test_subvolume_snapshot_create_and_rm(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_create_idempotence(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4439,8 +4418,8 @@ def test_subvolume_snapshot_info(self):
 
         snap_md = ["created_at", "data_pool", "has_pending_clones"]
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot, snap_missing = self._generate_random_snapshot_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot, snap_missing = self._gen_subvol_snap_name(2)
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -4474,9 +4453,9 @@ def test_subvolume_snapshot_info(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_in_group(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4505,11 +4484,11 @@ def test_subvolume_snapshot_ls(self):
         snapshots = []
 
         # create subvolume
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
 
         # create subvolume snapshots
-        snapshots = self._generate_random_snapshot_name(3)
+        snapshots = self._gen_subvol_snap_name(3)
         for snapshot in snapshots:
             self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot)
 
@@ -4537,8 +4516,8 @@ def test_subvolume_inherited_snapshot_ls(self):
         # at ancestral level
 
         snapshots = []
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
         snap_count = 3
 
         # create group
@@ -4548,7 +4527,7 @@ def test_subvolume_inherited_snapshot_ls(self):
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group)
 
         # create subvolume snapshots
-        snapshots = self._generate_random_snapshot_name(snap_count)
+        snapshots = self._gen_subvol_snap_name(snap_count)
         for snapshot in snapshots:
             self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot, group)
 
@@ -4583,8 +4562,8 @@ def test_subvolume_inherited_snapshot_info(self):
         at ancestral level
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4631,8 +4610,8 @@ def test_subvolume_inherited_snapshot_rm(self):
         at ancestral level
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4679,9 +4658,9 @@ def test_subvolume_subvolumegroup_snapshot_name_conflict(self):
         fail.
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        group_snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        group_snapshot = self._gen_subvol_snap_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -4720,8 +4699,8 @@ def test_subvolume_retain_snapshot_invalid_recreate(self):
         """
         ensure retained subvolume recreate does not leave any incarnations in the subvolume and trash
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4766,8 +4745,8 @@ def test_subvolume_retain_snapshot_recreate_subvolume(self):
         """
         snap_md = ["created_at", "data_pool", "has_pending_clones"]
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot1, snapshot2 = self._generate_random_snapshot_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot1, snapshot2 = self._gen_subvol_snap_name(2)
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4829,8 +4808,8 @@ def test_subvolume_retain_snapshot_with_snapshots(self):
         """
         snap_md = ["created_at", "data_pool", "has_pending_clones"]
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4923,7 +4902,7 @@ def test_subvolume_retain_snapshot_without_snapshots(self):
         """
         ensure retain snapshots based delete of a subvolume with no snapshots, deletes the subbvolume
         """
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4942,8 +4921,8 @@ def test_subvolume_retain_snapshot_trash_busy_recreate(self):
         """
         ensure retained subvolume recreate fails if its trash is not yet purged
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -4981,8 +4960,8 @@ def test_subvolume_retain_snapshot_trash_busy_recreate(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_rm_with_snapshots(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -5013,9 +4992,9 @@ def test_subvolume_snapshot_protect_unprotect_sanity(self):
         Snapshot protect/unprotect commands are deprecated. This test exists to ensure that
         invoking the command does not cause errors, till they are removed from a subsequent release.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -5053,8 +5032,8 @@ def test_subvolume_snapshot_protect_unprotect_sanity(self):
 
     def test_subvolume_snapshot_rm_force(self):
         # test removing non existing subvolume snapshot with --force
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # remove snapshot
         try:
@@ -5066,9 +5045,9 @@ def test_subvolume_snapshot_metadata_set(self):
         """
         Set custom metadata for subvolume snapshot.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5098,9 +5077,9 @@ def test_subvolume_snapshot_metadata_set_idempotence(self):
         """
         Set custom metadata for subvolume snapshot (Idempotency).
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5148,9 +5127,9 @@ def test_subvolume_snapshot_metadata_get(self):
         """
         Get custom metadata for a specified key in subvolume snapshot metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5189,9 +5168,9 @@ def test_subvolume_snapshot_metadata_get_for_nonexisting_key(self):
         """
         Get custom metadata for subvolume snapshot if specified key not exist in metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5227,9 +5206,9 @@ def test_subvolume_snapshot_metadata_get_for_nonexisting_section(self):
         """
         Get custom metadata for subvolume snapshot if metadata is not added for subvolume snapshot.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5260,9 +5239,9 @@ def test_subvolume_snapshot_metadata_update(self):
         """
         Update custom metadata for a specified key in subvolume snapshot metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5305,9 +5284,9 @@ def test_subvolume_snapshot_metadata_list(self):
         """
         List custom metadata for subvolume snapshot.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5344,9 +5323,9 @@ def test_subvolume_snapshot_metadata_list_if_no_metadata_set(self):
         """
         List custom metadata for subvolume snapshot if metadata is not added for subvolume snapshot.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5378,9 +5357,9 @@ def test_subvolume_snapshot_metadata_remove(self):
         """
         Remove custom metadata for a specified key in subvolume snapshot metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5421,9 +5400,9 @@ def test_subvolume_snapshot_metadata_remove_for_nonexisting_key(self):
         """
         Remove custom metadata for subvolume snapshot if specified key not exist in metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5459,9 +5438,9 @@ def test_subvolume_snapshot_metadata_remove_for_nonexisting_section(self):
         """
         Remove custom metadata for subvolume snapshot if metadata is not added for subvolume snapshot.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5492,9 +5471,9 @@ def test_subvolume_snapshot_metadata_remove_force(self):
         """
         Forcefully remove custom metadata for a specified key in subvolume snapshot metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5535,9 +5514,9 @@ def test_subvolume_snapshot_metadata_remove_force_for_nonexisting_key(self):
         """
         Forcefully remove custom metadata for subvolume snapshot if specified key not exist in metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5584,9 +5563,9 @@ def test_subvolume_snapshot_metadata_after_snapshot_remove(self):
         """
         Verify metadata removal of subvolume snapshot after snapshot removal.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5644,9 +5623,9 @@ def test_clean_stale_subvolume_snapshot_metadata(self):
         """
         Validate cleaning of stale subvolume snapshot metadata.
         """
-        subvolname = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolname = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create group.
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5711,9 +5690,9 @@ def test_clone_subvolume_info(self):
                      "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "pool_namespace",
                      "type", "uid"]
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -5755,8 +5734,8 @@ def test_subvolume_snapshot_info_without_snapshot_clone(self):
         If no clone is performed then path /volumes/_index/clone/{track_id}
         will not exist.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume.
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -5783,8 +5762,8 @@ def test_subvolume_snapshot_info_if_no_clone_pending(self):
         """
         Verify subvolume snapshot info output if no clone is in pending state.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
         clone_list =  [f'clone_{i}' for i in range(3)]
 
         # create subvolume.
@@ -5824,8 +5803,8 @@ def test_subvolume_snapshot_info_if_clone_pending_for_no_group(self):
         Clones are not specified for particular target_group. Hence target_group
         should not be in the output as we don't show _nogroup (default group)
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
         clone_list =  [f'clone_{i}' for i in range(3)]
 
         # create subvolume.
@@ -5871,11 +5850,11 @@ def test_subvolume_snapshot_info_if_clone_pending_for_target_group(self):
         Verify subvolume snapshot info output if clones are in pending state.
         Clones are not specified for target_group.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
-        group = self._generate_random_group_name()
-        target_group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
+        group = self._gen_subvol_grp_name()
+        target_group = self._gen_subvol_grp_name()
 
         # create groups
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -5927,8 +5906,8 @@ def test_subvolume_snapshot_info_if_orphan_clone(self):
         Orphan clones should not list under pending clones.
         orphan_clones_count should display correct count of orphan clones'
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
         clone_list =  [f'clone_{i}' for i in range(3)]
 
         # create subvolume.
@@ -5974,7 +5953,7 @@ def test_subvolume_snapshot_info_if_orphan_clone(self):
         self.assertEqual(res['has_pending_clones'], "no")
 
     def test_non_clone_status(self):
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -5994,9 +5973,9 @@ def test_non_clone_status(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_clone_inherit_snapshot_namespace_and_size(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
         osize = self.DEFAULT_FILE_SIZE*1024*1024*12
 
         # create subvolume, in an isolated namespace with a specified size
@@ -6038,9 +6017,9 @@ def test_subvolume_clone_inherit_snapshot_namespace_and_size(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_clone_inherit_quota_attrs(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
         osize = self.DEFAULT_FILE_SIZE*1024*1024*12
 
         # create subvolume with a specified size
@@ -6086,9 +6065,9 @@ def test_subvolume_clone_inherit_quota_attrs(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_clone_in_progress_getpath(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6135,9 +6114,9 @@ def test_subvolume_clone_in_progress_getpath(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_clone_in_progress_snapshot_rm(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6183,9 +6162,9 @@ def test_subvolume_clone_in_progress_snapshot_rm(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_clone_in_progress_source(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6234,9 +6213,9 @@ def test_subvolume_clone_retain_snapshot_with_snapshots(self):
         """
         retain snapshots of a cloned subvolume and check disallowed operations
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot1, snapshot2 = self._generate_random_snapshot_name(2)
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot1, snapshot2 = self._gen_subvol_snap_name(2)
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6308,9 +6287,9 @@ def test_subvolume_retain_snapshot_clone(self):
         """
         clone a snapshot from a snapshot retained subvolume
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6353,9 +6332,9 @@ def test_subvolume_retain_snapshot_clone_from_newer_snapshot(self):
         """
         clone a subvolume from recreated subvolume's latest snapshot
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot1, snapshot2 = self._generate_random_snapshot_name(2)
-        clone = self._generate_random_clone_name(1)
+        subvolume = self._gen_subvol_name()
+        snapshot1, snapshot2 = self._gen_subvol_snap_name(2)
+        clone = self._gen_subvol_clone_name(1)
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6411,8 +6390,8 @@ def test_subvolume_retain_snapshot_recreate(self):
         """
         recreate a subvolume from one of its retained snapshots
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6455,9 +6434,9 @@ def test_subvolume_retain_snapshot_trash_busy_recreate_clone(self):
         """
         ensure retained clone recreate fails if its trash is not yet purged
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
@@ -6509,9 +6488,9 @@ def test_subvolume_retain_snapshot_trash_busy_recreate_clone(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_attr_clone(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6545,9 +6524,9 @@ def test_clone_failure_status_pending_in_progress_complete(self):
         """
         ensure failure status is not shown when clone is not in failed/cancelled state
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1 = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1 = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6611,9 +6590,9 @@ def test_clone_failure_status_failed(self):
         """
         ensure failure status is shown when clone is in failed state and validate the reason
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1 = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1 = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6656,9 +6635,9 @@ def test_clone_failure_status_pending_cancelled(self):
         """
         ensure failure status is shown when clone is cancelled during pending state and validate the reason
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1 = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1 = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6700,9 +6679,9 @@ def test_clone_failure_status_in_progress_cancelled(self):
         """
         ensure failure status is shown when clone is cancelled during in-progress state and validate the reason
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1 = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1 = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6744,9 +6723,9 @@ def test_clone_failure_status_in_progress_cancelled(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6777,9 +6756,9 @@ def test_subvolume_snapshot_clone(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_quota_exceeded(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume with 20MB quota
         osize = self.DEFAULT_FILE_SIZE*1024*1024*20
@@ -6821,9 +6800,9 @@ def test_subvolume_snapshot_in_complete_clone_rm(self):
         'complete|cancelled|failed' states. It fails with EAGAIN in any other states.
         """
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6868,9 +6847,9 @@ def test_subvolume_snapshot_in_complete_clone_rm(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_retain_suid_guid(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6910,9 +6889,9 @@ def test_subvolume_snapshot_clone_retain_suid_guid(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_and_reclone(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1, clone2 = self._generate_random_clone_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1, clone2 = self._gen_subvol_clone_name(2)
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -6963,9 +6942,9 @@ def test_subvolume_snapshot_clone_and_reclone(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_cancel_in_progress(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -7014,9 +6993,9 @@ def test_subvolume_snapshot_clone_cancel_pending(self):
         # yeh, 1gig -- we need the clone to run for sometime
         FILE_SIZE_MB = 1024
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clones = self._generate_random_clone_name(NR_CLONES)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clones = self._gen_subvol_snap_name(NR_CLONES)
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -7066,10 +7045,10 @@ def test_subvolume_snapshot_clone_cancel_pending(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_different_groups(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
-        s_group, c_group = self._generate_random_group_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
+        s_group, c_group = self._gen_subvol_grp_name(2)
 
         # create groups
         self._fs_cmd("subvolumegroup", "create", self.volname, s_group)
@@ -7109,9 +7088,9 @@ def test_subvolume_snapshot_clone_different_groups(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_fail_with_remove(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1, clone2 = self._generate_random_clone_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1, clone2 = self._gen_subvol_clone_name(2)
 
         pool_capacity = 32 * 1024 * 1024
         # number of files required to fill up 99% of the pool
@@ -7172,9 +7151,9 @@ def test_subvolume_snapshot_clone_fail_with_remove(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_on_existing_subvolumes(self):
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create subvolumes
         self._fs_cmd("subvolume", "create", self.volname, subvolume1, "--mode=777")
@@ -7224,9 +7203,9 @@ def test_subvolume_snapshot_clone_on_existing_subvolumes(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_pool_layout(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # add data pool
         new_pool = "new_pool"
@@ -7268,10 +7247,10 @@ def test_subvolume_snapshot_clone_pool_layout(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_under_group(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
-        group = self._generate_random_group_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
+        group = self._gen_subvol_grp_name()
 
         # create subvolume
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
@@ -7308,9 +7287,9 @@ def test_subvolume_snapshot_clone_under_group(self):
         self._wait_for_trash_empty()
 
     def test_subvolume_snapshot_clone_with_attrs(self):
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         mode = "777"
         uid  = "1000"
@@ -7357,9 +7336,9 @@ def test_subvolume_snapshot_clone_with_upgrade(self):
         and verify clone operation.
         further ensure that a legacy volume is not updated to v2, but clone is.
         """
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # emulate a old-fashioned subvolume
         createpath = os.path.join(".", "volumes", "_nogroup", subvolume)
@@ -7473,10 +7452,10 @@ def test_periodic_async_work(self):
         time.sleep(10)
 
     def test_subvolume_under_group_snapshot_clone(self):
-        subvolume = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone = self._generate_random_clone_name()
+        subvolume = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone = self._gen_subvol_clone_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)
@@ -7523,7 +7502,7 @@ def test_connection_expiration(self):
         self.assertLessEqual(len(sessions), 1) # maybe mgr is already mounted
 
         # Get the mgr to definitely mount cephfs
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         sessions = self._session_list()
         self.assertEqual(len(sessions), 1)
@@ -7539,7 +7518,7 @@ def test_mgr_eviction(self):
         self.assertLessEqual(len(sessions), 1) # maybe mgr is already mounted
 
         # Get the mgr to definitely mount cephfs
-        subvolume = self._generate_random_subvolume_name()
+        subvolume = self._gen_subvol_name()
         self._fs_cmd("subvolume", "create", self.volname, subvolume)
         sessions = self._session_list()
         self.assertEqual(len(sessions), 1)
@@ -7643,8 +7622,8 @@ def test_subvolume_upgrade_legacy_to_v1(self):
         accessible.
         further ensure that a legacy volume is not updated to v2.
         """
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume -- one in the default group and
         # the other in a custom group
@@ -7694,9 +7673,9 @@ def test_subvolume_no_upgrade_v1_sanity(self):
                      "type", "uid", "features", "state"]
         snap_md = ["created_at", "data_pool", "has_pending_clones"]
 
-        subvolume = self._generate_random_subvolume_name()
-        snapshot = self._generate_random_snapshot_name()
-        clone1, clone2 = self._generate_random_clone_name(2)
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1, clone2 = self._gen_subvol_clone_name(2)
         mode = "777"
         uid  = "1000"
         gid  = "1000"
@@ -7801,8 +7780,8 @@ def test_subvolume_no_upgrade_v1_to_v2(self):
         poor man's upgrade test -- theme continues...
         ensure v1 to v2 upgrades are not done automatically due to various states of v1
         """
-        subvolume1, subvolume2, subvolume3 = self._generate_random_subvolume_name(3)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2, subvolume3 = self._gen_subvol_name(3)
+        group = self._gen_subvol_grp_name()
 
         # emulate a v1 subvolume -- in the default group
         subvol1_path = self._create_v1_subvolume(subvolume1)
@@ -7859,8 +7838,8 @@ def test_subvolume_upgrade_v1_to_v2(self):
         poor man's upgrade test -- theme continues...
         ensure v1 to v2 upgrades work
         """
-        subvolume1, subvolume2 = self._generate_random_subvolume_name(2)
-        group = self._generate_random_group_name()
+        subvolume1, subvolume2 = self._gen_subvol_name(2)
+        group = self._gen_subvol_grp_name()
 
         # emulate a v1 subvolume -- in the default group
         subvol1_path = self._create_v1_subvolume(subvolume1, has_snapshot=False)
@@ -7892,7 +7871,7 @@ def test_malicious_metafile_on_legacy_to_v1_upgrade(self):
         on legacy subvol upgrade to v1
         poor man's upgrade test -- theme continues...
         """
-        subvol1, subvol2 = self._generate_random_subvolume_name(2)
+        subvol1, subvol2 = self._gen_subvol_name(2)
 
         # emulate a old-fashioned subvolume in the default group
         createpath1 = os.path.join(".", "volumes", "_nogroup", subvol1)
@@ -7945,8 +7924,8 @@ def test_binary_metafile_on_legacy_to_v1_upgrade(self):
         on legacy subvol upgrade to v1
         poor man's upgrade test -- theme continues...
         """
-        subvol = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvol = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume -- in a custom group
         createpath = os.path.join(".", "volumes", group, subvol)
@@ -7988,8 +7967,8 @@ def test_unparseable_metafile_on_legacy_to_v1_upgrade(self):
         on legacy subvol upgrade to v1
         poor man's upgrade test -- theme continues...
         """
-        subvol = self._generate_random_subvolume_name()
-        group = self._generate_random_group_name()
+        subvol = self._gen_subvol_name()
+        group = self._gen_subvol_grp_name()
 
         # emulate a old-fashioned subvolume -- in a custom group
         createpath = os.path.join(".", "volumes", group, subvol)
@@ -8032,8 +8011,8 @@ class TestPerModuleFinsherThread(TestVolumesHelper):
     as four subvolume cmds are run
     """
     def test_volumes_module_finisher_thread(self):
-        subvol1, subvol2, subvol3 = self._generate_random_subvolume_name(3)
-        group = self._generate_random_group_name()
+        subvol1, subvol2, subvol3 = self._gen_subvol_name(3)
+        group = self._gen_subvol_grp_name()
 
         # create group
         self._fs_cmd("subvolumegroup", "create", self.volname, group)

From 750680b2d1889d37c46ec14911a2dd6b3f3338bb Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Thu, 1 Dec 2022 23:25:54 -0500
Subject: [PATCH 0767/2492] rgw: Distentangle `rgw_rados_ref` from
 `RGWSI_RADOS`

`RGWSI_RADOS` has no reason to exist. `rgw_rados_ref`, on the other
hand, is fairly useful. Move `rgw_rados_ref` to
`store/rados/rgw_tools.h` along with a function to create one.

Then clean up the fallout.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_cr_rados.cc          | 16 ++--
 src/rgw/driver/rados/rgw_cr_rados.h           |  7 +-
 .../driver/rados/rgw_object_expirer_core.cc   | 20 +++--
 src/rgw/driver/rados/rgw_putobj_processor.cc  |  4 +-
 src/rgw/driver/rados/rgw_rados.cc             | 86 ++++++++-----------
 src/rgw/driver/rados/rgw_rados.h              |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc         |  2 +-
 src/rgw/driver/rados/rgw_tools.cc             | 18 ++++
 src/rgw/driver/rados/rgw_tools.h              | 28 ++++++
 src/rgw/driver/rados/rgw_trim_bilog.cc        | 20 ++---
 src/rgw/driver/rados/sync_fairness.cc         | 24 +++---
 src/rgw/services/svc_cls.cc                   | 17 ++--
 src/rgw/services/svc_cls.h                    |  2 +
 src/rgw/services/svc_rados.h                  |  2 -
 14 files changed, 143 insertions(+), 105 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index e7174908dd77..d73827bea142 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -152,7 +152,7 @@ int RGWSimpleRadosReadAttrsCR::send_request(const DoutPrefixProvider *dpp)
   }
 
   cn = stack->create_completion_notifier();
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op,
+  return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op,
 				      nullptr);
 }
 
@@ -234,7 +234,7 @@ int RGWAsyncLockSystemObj::_send_request(const DoutPrefixProvider *dpp)
   l.set_cookie(cookie);
   l.set_may_renew(true);
 
-  return l.lock_exclusive(&ref.pool.ioctx(), ref.obj.oid);
+  return l.lock_exclusive(&ref.ioctx, ref.obj.oid);
 }
 
 RGWAsyncLockSystemObj::RGWAsyncLockSystemObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
@@ -260,7 +260,7 @@ int RGWAsyncUnlockSystemObj::_send_request(const DoutPrefixProvider *dpp)
 
   l.set_cookie(cookie);
 
-  return l.unlock(&ref.pool.ioctx(), ref.obj.oid);
+  return l.unlock(&ref.ioctx, ref.obj.oid);
 }
 
 RGWAsyncUnlockSystemObj::RGWAsyncUnlockSystemObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
@@ -303,7 +303,7 @@ int RGWRadosSetOmapKeysCR::send_request(const DoutPrefixProvider *dpp)
   op.omap_set(entries);
 
   cn = stack->create_completion_notifier();
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op);
+  return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
 }
 
 int RGWRadosSetOmapKeysCR::request_complete()
@@ -341,7 +341,7 @@ int RGWRadosGetOmapKeysCR::send_request(const DoutPrefixProvider *dpp) {
   op.omap_get_keys2(marker, max_entries, &result->entries, &result->more, nullptr);
 
   cn = stack->create_completion_notifier(result);
-  return result->ref.pool.ioctx().aio_operate(result->ref.obj.oid, cn->completion(), &op, NULL);
+  return result->ref.ioctx.aio_operate(result->ref.obj.oid, cn->completion(), &op, NULL);
 }
 
 int RGWRadosGetOmapKeysCR::request_complete()
@@ -379,7 +379,7 @@ int RGWRadosGetOmapValsCR::send_request(const DoutPrefixProvider *dpp) {
   op.omap_get_vals2(marker, max_entries, &result->entries, &result->more, nullptr);
 
   cn = stack->create_completion_notifier(result);
-  return result->ref.pool.ioctx().aio_operate(result->ref.obj.oid, cn->completion(), &op, NULL);
+  return result->ref.ioctx.aio_operate(result->ref.obj.oid, cn->completion(), &op, NULL);
 }
 
 int RGWRadosGetOmapValsCR::request_complete()
@@ -414,7 +414,7 @@ int RGWRadosRemoveOmapKeysCR::send_request(const DoutPrefixProvider *dpp) {
   op.omap_rm_keys(keys);
 
   cn = stack->create_completion_notifier();
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op);
+  return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
 }
 
 int RGWRadosRemoveOmapKeysCR::request_complete()
@@ -1131,7 +1131,7 @@ int RGWRadosNotifyCR::send_request(const DoutPrefixProvider *dpp)
   set_status() << "sending request";
 
   cn = stack->create_completion_notifier();
-  return ref.pool.ioctx().aio_notify(ref.obj.oid, cn->completion(), request,
+  return ref.ioctx.aio_notify(ref.obj.oid, cn->completion(), request,
                               timeout_ms, response);
 }
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index 676fd3d3cae2..b9c45f30cf69 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -446,8 +446,7 @@ class RGWSimpleRadosReadCR : public RGWSimpleCoroutine {
     op.read(0, -1, &bl, nullptr);
 
     cn = stack->create_completion_notifier();
-    return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op,
-					nullptr);
+    return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op, nullptr);
   }
 
   int request_complete() {
@@ -557,7 +556,7 @@ class RGWSimpleRadosWriteCR : public RGWSimpleCoroutine {
     op.write_full(bl);
 
     cn = stack->create_completion_notifier();
-    return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op);
+    return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
   }
 
   int request_complete() override {
@@ -624,7 +623,7 @@ class RGWSimpleRadosWriteAttrsCR : public RGWSimpleCoroutine {
       return 0;
     }
 
-    return ref.pool.ioctx().aio_operate(ref.obj.oid, cn->completion(), &op);
+    return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
   }
 
   int request_complete() override {
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc
index 72c21c1d262a..3f20ce26ba8a 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.cc
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc
@@ -163,7 +163,7 @@ static int cls_timeindex_trim_repeat(const DoutPrefixProvider *dpp,
   do {
     librados::ObjectWriteOperation op;
     cls_timeindex_trim(op, from_time, to_time, from_marker, to_marker);
-    int r = rgw_rados_operate(dpp, ref.pool.ioctx(), oid, &op, null_yield);
+    int r = rgw_rados_operate(dpp, ref.ioctx, oid, &op, null_yield);
     if (r == -ENODATA)
       done = true;
     else if (r < 0)
@@ -180,15 +180,17 @@ int RGWObjExpStore::objexp_hint_trim(const DoutPrefixProvider *dpp,
                                const string& from_marker,
                                const string& to_marker, optional_yield y)
 {
-  auto obj = rados_svc->obj(rgw_raw_obj(driver->svc()->zone->get_zone_params().log_pool, oid));
-  int r = obj.open(dpp);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl;
-    return r;
+  rgw_rados_ref ref;
+  auto ret = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(),
+			       {driver->svc()->zone->get_zone_params().log_pool, oid},
+			       &ref);
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open oid="
+		      << oid << " (r=" << ret << ")" << dendl;
+    return ret;
   }
-  auto& ref = obj.get_ref();
-  int ret = cls_timeindex_trim_repeat(dpp, ref, oid, utime_t(start_time), utime_t(end_time),
-          from_marker, to_marker, y);
+  ret = cls_timeindex_trim_repeat(dpp, ref, oid, utime_t(start_time), utime_t(end_time),
+				  from_marker, to_marker, y);
   if ((ret < 0 ) && (ret != -ENOENT)) {
     return ret;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 091adb9830c0..14661935a77c 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -553,7 +553,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
 
   librados::ObjectWriteOperation op;
   cls_rgw_mp_upload_part_info_update(op, p, info);
-  r = rgw_rados_operate(rctx.dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, rctx.y);
+  r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y);
   ldpp_dout(rctx.dpp, 20) << "Update meta: " << meta_obj_ref.obj.oid << " part " << p << " prefix " << info.manifest.get_prefix() << " return " << r << dendl;
 
   if (r == -EOPNOTSUPP) {
@@ -567,7 +567,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
     op = librados::ObjectWriteOperation{};
     op.assert_exists(); // detect races with abort
     op.omap_set(m);
-    r = rgw_rados_operate(rctx.dpp, meta_obj_ref.pool.ioctx(), meta_obj_ref.obj.oid, &op, rctx.y);
+    r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y);
   }
   if (r < 0) {
     return r == -ENOENT ? -ERR_NO_SUCH_UPLOAD : r;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index f9b7dfb65cb6..d12d479be261 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2422,25 +2422,19 @@ int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
                                const rgw_obj& obj,
                                rgw_rados_ref *ref)
 {
-  get_obj_bucket_and_oid_loc(obj, ref->obj.oid, ref->obj.loc);
-
-  rgw_pool pool;
-  if (!get_obj_data_pool(target_placement_rule, obj, &pool)) {
-    ldpp_dout(dpp, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl;
+  rgw_raw_obj raw;
+  get_obj_bucket_and_oid_loc(obj, raw.oid, raw.loc);
+  if (!get_obj_data_pool(target_placement_rule, obj, &raw.pool)) {
+    ldpp_dout(dpp, 0) << "ERROR: cannot get data pool for obj=" << obj
+		      << ", probably misconfiguration" << dendl;
     return -EIO;
   }
-
-  ref->pool = svc.rados->pool(pool);
-
-  int r = ref->pool.open(dpp, RGWSI_RADOS::OpenParams()
-                         .set_mostly_omap(false));
+  auto r = rgw_get_rados_ref(dpp, get_rados_handle(), raw, ref);
   if (r < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: failed opening data pool (pool=" << pool << "); r=" << r << dendl;
+    ldpp_dout(dpp, 0) << "ERROR: failed opening data pool (pool="
+		      << raw.pool << "); r=" << r << dendl;
     return r;
   }
-
-  ref->pool.ioctx().locator_set_key(ref->obj.loc);
-
   return 0;
 }
 
@@ -2452,24 +2446,18 @@ int RGWRados::get_obj_head_ref(const DoutPrefixProvider *dpp,
   return get_obj_head_ref(dpp, bucket_info.placement_rule, obj, ref);
 }
 
-int RGWRados::get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref)
+int RGWRados::get_raw_obj_ref(const DoutPrefixProvider *dpp, rgw_raw_obj obj, rgw_rados_ref* ref)
 {
-  ref->obj = obj;
-
-  if (ref->obj.oid.empty()) {
-    ref->obj.oid = obj.pool.to_str();
-    ref->obj.pool = svc.zone->get_zone_params().domain_root;
+  if (obj.oid.empty()) {
+    obj.oid = obj.pool.to_str();
+    obj.pool = svc.zone->get_zone_params().domain_root;
   }
-  ref->pool = svc.rados->pool(obj.pool);
-  int r = ref->pool.open(dpp, RGWSI_RADOS::OpenParams()
-                         .set_mostly_omap(false));
+  int r = rgw_get_rados_ref(dpp, get_rados_handle(), std::move(obj), ref);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed opening pool (pool=" << obj.pool << "); r=" << r << dendl;
     return r;
   }
 
-  ref->pool.ioctx().locator_set_key(ref->obj.loc);
-
   return 0;
 }
 
@@ -2678,10 +2666,10 @@ int RGWRados::fix_tail_obj_locator(const DoutPrefixProvider *dpp,
 	continue;
       }
 
-      auto& ioctx = ref.pool.ioctx();
+      auto& ioctx = ref.ioctx;
 
       get_obj_bucket_and_oid_loc(loc, oid, locator);
-      ref.pool.ioctx().locator_set_key(locator);
+      ref.ioctx.locator_set_key(locator);
 
       ldpp_dout(dpp, 20) << __func__ << ": key=" << key << " oid=" << oid << " locator=" << locator << dendl;
 
@@ -3215,10 +3203,10 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
       return r;
   }
 
-  auto& ioctx = ref.pool.ioctx();
+  auto& ioctx = ref.ioctx;
 
   tracepoint(rgw_rados, operate_enter, req_id.c_str());
-  r = rgw_rados_operate(rctx.dpp, ref.pool.ioctx(), ref.obj.oid, &op, rctx.y);
+  r = rgw_rados_operate(rctx.dpp, ref.ioctx, ref.obj.oid, &op, rctx.y);
   tracepoint(rgw_rados, operate_exit, req_id.c_str());
   if (r < 0) { /* we can expect to get -ECANCELED if object was replaced under,
                 or -ENOENT if was removed, or -EEXIST if it did not exist
@@ -5740,7 +5728,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
 
   store->remove_rgw_head_obj(op);
 
-  auto& ioctx = ref.pool.ioctx();
+  auto& ioctx = ref.ioctx;
   r = rgw_rados_operate(dpp, ioctx, ref.obj.oid, &op, y);
 
   /* raced with another operation, object state is indeterminate */
@@ -5812,7 +5800,7 @@ int RGWRados::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& o
   ObjectWriteOperation op;
 
   op.remove();
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   if (r < 0)
     return r;
 
@@ -6469,7 +6457,7 @@ int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx* octx, RGWBu
   }
   struct timespec mtime_ts = real_clock::to_timespec(mtime);
   op.mtime2(&mtime_ts);
-  auto& ioctx = ref.pool.ioctx();
+  auto& ioctx = ref.ioctx;
   r = rgw_rados_operate(dpp, ioctx, ref.obj.oid, &op, y);
   if (state) {
     if (r >= 0) {
@@ -7282,7 +7270,7 @@ int RGWRados::obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bu
     return r;
   }
 
-  return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, y);
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, y);
 }
 
 int RGWRados::obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, ObjectReadOperation *op, optional_yield y)
@@ -7295,7 +7283,7 @@ int RGWRados::obj_operate(const DoutPrefixProvider *dpp, const RGWBucketInfo& bu
 
   bufferlist outbl;
 
-  return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, &outbl, y);
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, &outbl, y);
 }
 
 void RGWRados::olh_cancel_modification(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info,
@@ -7320,7 +7308,7 @@ void RGWRados::olh_cancel_modification(const DoutPrefixProvider *dpp, const RGWB
   ObjectWriteOperation op;
   bucket_index_guard_olh_op(dpp, state, op);
   op.rmxattr(attr_name.c_str());
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   if (r < 0) {
     if (r != -ENOENT && r != -ECANCELED) {
       ldpp_dout(dpp, 0) << __func__ << " target_obj=" << olh_obj << " rmxattr rgw_rados_operate() returned " << r << dendl;
@@ -7336,7 +7324,7 @@ void RGWRados::olh_cancel_modification(const DoutPrefixProvider *dpp, const RGWB
     rm_op.cmpxattr(RGW_ATTR_OLH_INFO, CEPH_OSD_CMPXATTR_OP_EQ, bufferlist());
     cls_obj_check_prefix_exist(rm_op, RGW_ATTR_OLH_PENDING_PREFIX, true);
     rm_op.remove();
-    r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &rm_op, y);
+    r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &rm_op, y);
   }
   if (r < 0 && (r != -ENOENT && r != -ECANCELED)) {
     ldpp_dout(dpp, 0) << __func__ << " target_obj=" << olh_obj << " olh rm rgw_rados_operate() returned " << r << dendl;
@@ -7855,7 +7843,7 @@ int RGWRados::repair_olh(const DoutPrefixProvider *dpp, RGWObjState* state, cons
   if (r < 0) {
     return r;
   }
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "repair_olh failed to write olh attributes with "
         << cpp_strerror(r) << dendl;
@@ -8076,7 +8064,7 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
   }
 
   /* update olh object */
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << ": could not apply olh update to oid \"" << ref.obj.oid << "\", r=" << r << dendl;
     return r;
@@ -8154,7 +8142,7 @@ int RGWRados::clear_olh(const DoutPrefixProvider *dpp,
   cls_obj_check_prefix_exist(rm_op, RGW_ATTR_OLH_PENDING_PREFIX, true); /* fail if found one of these, pending modification */
   rm_op.remove();
 
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &rm_op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &rm_op, y);
   if (r == -ECANCELED) {
     return r; /* someone else made a modification in the meantime */
   }
@@ -8442,7 +8430,7 @@ int RGWRados::remove_olh_pending_entries(const DoutPrefixProvider *dpp, const RG
       op.rmxattr(i->first.c_str());
     }
 
-    r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+    r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
     if (r == -ENOENT || r == -ECANCELED) {
       /* raced with some other change, shouldn't sweat about it */
       return 0;
@@ -8538,10 +8526,10 @@ int RGWRados::raw_obj_stat(const DoutPrefixProvider *dpp,
     op.read(0, cct->_conf->rgw_max_chunk_size, first_chunk, NULL);
   }
   bufferlist outbl;
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, &outbl, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, &outbl, y);
 
   if (epoch) {
-    *epoch = ref.pool.ioctx().get_last_version();
+    *epoch = ref.ioctx.get_last_version();
   }
 
   if (r < 0)
@@ -8773,7 +8761,7 @@ int RGWRados::append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size
   librados::Rados *rad = get_rados_handle();
   librados::AioCompletion *completion = rad->aio_create_completion(nullptr, nullptr);
 
-  r = ref.pool.ioctx().aio_append(ref.obj.oid, completion, bl, size);
+  r = ref.ioctx.aio_append(ref.obj.oid, completion, bl, size);
   completion->release();
   return r;
 }
@@ -9847,7 +9835,7 @@ int RGWRados::cls_obj_usage_log_add(const DoutPrefixProvider *dpp, const string&
   ObjectWriteOperation op;
   cls_rgw_usage_log_add(op, info);
 
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   return r;
 }
 
@@ -9866,7 +9854,7 @@ int RGWRados::cls_obj_usage_log_read(const DoutPrefixProvider *dpp, const string
 
   *is_truncated = false;
 
-  r = cls_rgw_usage_log_read(ref.pool.ioctx(), ref.obj.oid, user, bucket, start_epoch, end_epoch,
+  r = cls_rgw_usage_log_read(ref.ioctx, ref.obj.oid, user, bucket, start_epoch, end_epoch,
 			     max_entries, read_iter, usage, is_truncated);
 
   return r;
@@ -9878,7 +9866,7 @@ static int cls_rgw_usage_log_trim_repeat(const DoutPrefixProvider *dpp, rgw_rado
   do {
     librados::ObjectWriteOperation op;
     cls_rgw_usage_log_trim(op, user, bucket, start_epoch, end_epoch);
-    int r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+    int r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
     if (r == -ENODATA)
       done = true;
     else if (r < 0)
@@ -9914,7 +9902,7 @@ int RGWRados::cls_obj_usage_log_clear(const DoutPrefixProvider *dpp, string& oid
   }
   librados::ObjectWriteOperation op;
   cls_rgw_usage_log_clear(op);
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
   return r;
 }
 
@@ -10338,7 +10326,7 @@ int RGWRados::delete_raw_obj_aio(const DoutPrefixProvider *dpp, const rgw_raw_ob
   cls_rgw_remove_obj(op, prefixes);
 
   AioCompletion *c = librados::Rados::aio_create_completion(nullptr, nullptr);
-  ret = ref.pool.ioctx().aio_operate(ref.obj.oid, c, &op);
+  ret = ref.ioctx.aio_operate(ref.obj.oid, c, &op);
   if (ret < 0) {
     ldpp_dout(dpp, -1) << "ERROR: AioOperate failed with ret=" << ret << dendl;
     c->release();
@@ -10378,7 +10366,7 @@ int RGWRados::delete_obj_aio(const DoutPrefixProvider *dpp, const rgw_obj& obj,
   cls_rgw_remove_obj(op, prefixes);
 
   AioCompletion *c = librados::Rados::aio_create_completion(nullptr, nullptr);
-  ret = ref.pool.ioctx().aio_operate(ref.obj.oid, c, &op);
+  ret = ref.ioctx.aio_operate(ref.obj.oid, c, &op);
   if (ret < 0) {
     ldpp_dout(dpp, -1) << "ERROR: AioOperate failed with ret=" << ret << dendl;
     c->release();
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 183a5dbd2729..938bc161a7ef 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -576,7 +576,7 @@ class RGWRados
   }
 
 
-  int get_raw_obj_ref(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, rgw_rados_ref *ref);
+  int get_raw_obj_ref(const DoutPrefixProvider *dpp, rgw_raw_obj obj, rgw_rados_ref *ref);
 
   int list_raw_objects_init(const DoutPrefixProvider *dpp, const rgw_pool& pool, const std::string& marker, RGWListRawObjsCtx *ctx);
   int list_raw_objects_next(const DoutPrefixProvider *dpp, const std::string& prefix_filter, int max,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index c74fcb5b3882..327980f4de6e 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1591,7 +1591,7 @@ int RadosObject::get_torrent_info(const DoutPrefixProvider* dpp,
   librados::ObjectReadOperation op;
   op.omap_get_vals_by_keys(keys, &result, nullptr);
 
-  ret = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, nullptr, y);
+  ret = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, nullptr, y);
   if (ret < 0) {
     return ret;
   }
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 635e5ce887d0..ec373ca6095d 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -95,6 +95,24 @@ int rgw_init_ioctx(const DoutPrefixProvider *dpp,
   return 0;
 }
 
+int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
+		      rgw_raw_obj obj, rgw_rados_ref* ref)
+{
+  ref->obj = std::move(obj);
+
+  int r = rgw_init_ioctx(dpp, rados, ref->obj.pool,
+			 ref->ioctx, true, false);
+  if (r < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: creating ioctx (pool=" << ref->obj.pool
+        << "); r=" << r << dendl;
+    return r;
+  }
+
+  ref->ioctx.locator_set_key(ref->obj.loc);
+  return 0;
+}
+
+
 map<string, bufferlist>* no_change_attrs() {
   static map<string, bufferlist> no_change;
   return &no_change;
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 607b064b1775..32cc029d7581 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -21,6 +21,12 @@ class optional_yield;
 
 struct obj_version;
 
+struct rgw_rados_ref {
+  librados::IoCtx ioctx;
+  rgw_raw_obj obj;
+};
+int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
+		      rgw_raw_obj obj, rgw_rados_ref* ref);
 
 int rgw_init_ioctx(const DoutPrefixProvider *dpp,
                    librados::Rados *rados, const rgw_pool& pool,
@@ -100,9 +106,31 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
 		      int flags = 0);
+inline int rgw_rados_operate(const DoutPrefixProvider *dpp,
+                             rgw_rados_ref& ref,
+                             librados::ObjectReadOperation *op, bufferlist *pbl,
+                             optional_yield y, int flags = 0)
+{
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, pbl, y, flags);
+}
+
+inline int rgw_rados_operate(const DoutPrefixProvider *dpp,
+                             rgw_rados_ref& ref,
+                             librados::ObjectWriteOperation *op,
+                             optional_yield y, int flags = 0)
+{
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, y, flags);
+}
+
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                      bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl,
                      optional_yield y);
+inline int rgw_rados_notify(const DoutPrefixProvider *dpp, rgw_rados_ref& ref,
+			    bufferlist& bl, uint64_t timeout_ms,
+			    bufferlist* pbl, optional_yield y)
+{
+  return rgw_rados_notify(dpp, ref.ioctx, ref.obj.oid, bl, timeout_ms, pbl, y);
+}
 
 int rgw_tools_init(const DoutPrefixProvider *dpp, CephContext *cct);
 void rgw_tools_cleanup();
diff --git a/src/rgw/driver/rados/rgw_trim_bilog.cc b/src/rgw/driver/rados/rgw_trim_bilog.cc
index 16ced4c2ac5c..be6d990d962b 100644
--- a/src/rgw/driver/rados/rgw_trim_bilog.cc
+++ b/src/rgw/driver/rados/rgw_trim_bilog.cc
@@ -270,18 +270,18 @@ class BucketTrimWatcher : public librados::WatchCtx2 {
     }
 
     // register a watch on the realm's control object
-    r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+    r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
     if (r == -ENOENT) {
       constexpr bool exclusive = true;
-      r = ref.pool.ioctx().create(ref.obj.oid, exclusive);
+      r = ref.ioctx.create(ref.obj.oid, exclusive);
       if (r == -EEXIST || r == 0) {
-        r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+        r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
       }
     }
     if (r < 0) {
       ldpp_dout(dpp, -1) << "Failed to watch " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
-      ref.pool.ioctx().close();
+      ref.ioctx.close();
       return r;
     }
 
@@ -290,24 +290,24 @@ class BucketTrimWatcher : public librados::WatchCtx2 {
   }
 
   int restart() {
-    int r = ref.pool.ioctx().unwatch2(handle);
+    int r = ref.ioctx.unwatch2(handle);
     if (r < 0) {
       lderr(store->ctx()) << "Failed to unwatch on " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
     }
-    r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+    r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
     if (r < 0) {
       lderr(store->ctx()) << "Failed to restart watch on " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
-      ref.pool.ioctx().close();
+      ref.ioctx.close();
     }
     return r;
   }
 
   void stop() {
     if (handle) {
-      ref.pool.ioctx().unwatch2(handle);
-      ref.pool.ioctx().close();
+      ref.ioctx.unwatch2(handle);
+      ref.ioctx.close();
     }
   }
 
@@ -332,7 +332,7 @@ class BucketTrimWatcher : public librados::WatchCtx2 {
     } catch (const buffer::error& e) {
       lderr(store->ctx()) << "Failed to decode notification: " << e.what() << dendl;
     }
-    ref.pool.ioctx().notify_ack(ref.obj.oid, notify_id, cookie, reply);
+    ref.ioctx.notify_ack(ref.obj.oid, notify_id, cookie, reply);
   }
 
   /// reestablish the watch if it gets disconnected
diff --git a/src/rgw/driver/rados/sync_fairness.cc b/src/rgw/driver/rados/sync_fairness.cc
index ded1cf56a719..1ac27f4bacc8 100644
--- a/src/rgw/driver/rados/sync_fairness.cc
+++ b/src/rgw/driver/rados/sync_fairness.cc
@@ -144,18 +144,18 @@ class Watcher : public librados::WatchCtx2 {
     }
 
     // register a watch on the control object
-    r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+    r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
     if (r == -ENOENT) {
       constexpr bool exclusive = true;
-      r = ref.pool.ioctx().create(ref.obj.oid, exclusive);
+      r = ref.ioctx.create(ref.obj.oid, exclusive);
       if (r == -EEXIST || r == 0) {
-        r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+        r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
       }
     }
     if (r < 0) {
       ldpp_dout(dpp, -1) << "Failed to watch " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
-      ref.pool.ioctx().close();
+      ref.ioctx.close();
       return r;
     }
 
@@ -165,16 +165,16 @@ class Watcher : public librados::WatchCtx2 {
 
   int restart()
   {
-    int r = ref.pool.ioctx().unwatch2(handle);
+    int r = ref.ioctx.unwatch2(handle);
     if (r < 0) {
       ldpp_dout(dpp, -1) << "Failed to unwatch on " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
     }
-    r = ref.pool.ioctx().watch2(ref.obj.oid, &handle, this);
+    r = ref.ioctx.watch2(ref.obj.oid, &handle, this);
     if (r < 0) {
       ldpp_dout(dpp, -1) << "Failed to restart watch on " << ref.obj
           << " with " << cpp_strerror(-r) << dendl;
-      ref.pool.ioctx().close();
+      ref.ioctx.close();
     }
     return r;
   }
@@ -182,8 +182,8 @@ class Watcher : public librados::WatchCtx2 {
   void stop()
   {
     if (handle) {
-      ref.pool.ioctx().unwatch2(handle);
-      ref.pool.ioctx().close();
+      ref.ioctx.unwatch2(handle);
+      ref.ioctx.close();
     }
   }
 
@@ -210,7 +210,7 @@ class Watcher : public librados::WatchCtx2 {
     bufferlist reply;
     encode(response, reply);
 
-    ref.pool.ioctx().notify_ack(ref.obj.oid, notify_id, cookie, reply);
+    ref.ioctx.notify_ack(ref.obj.oid, notify_id, cookie, reply);
   }
 
   // reestablish the watch if it gets disconnected
@@ -289,7 +289,7 @@ class RadosBidManager : public BidManager, public Server, public DoutPrefix {
     my_bids = this->my_bids;
   }
 
-  bool is_highest_bidder(std::size_t index)
+  bool is_highest_bidder(std::size_t index) override
   {
     auto lock = std::scoped_lock{mutex};
     const bid_value my_bid = my_bids.at(index); // may throw
@@ -303,7 +303,7 @@ class RadosBidManager : public BidManager, public Server, public DoutPrefix {
     return true;
   }
 
-  RGWCoroutine* notify_cr()
+  RGWCoroutine* notify_cr() override
   {
     auto lock = std::scoped_lock{mutex};
     return new NotifyCR(store, this, obj, my_bids);
diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc
index 342146bfefa7..44788940330a 100644
--- a/src/rgw/services/svc_cls.cc
+++ b/src/rgw/services/svc_cls.cc
@@ -47,12 +47,15 @@ int RGWSI_Cls::MFA::get_mfa_obj(const DoutPrefixProvider *dpp, const rgw_user& u
 
 int RGWSI_Cls::MFA::get_mfa_ref(const DoutPrefixProvider *dpp, const rgw_user& user, rgw_rados_ref *ref)
 {
-  std::optional<RGWSI_RADOS::Obj> obj;
-  int r = get_mfa_obj(dpp, user, &obj);
+  string oid = get_mfa_oid(user);
+  rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid);
+
+  auto r = rgw_get_rados_ref(dpp, rados_svc->get_rados_handle(), o, ref);
   if (r < 0) {
+    ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl;
     return r;
   }
-  *ref = obj->get_ref();
+
   return 0;
 }
 
@@ -66,7 +69,7 @@ int RGWSI_Cls::MFA::check_mfa(const DoutPrefixProvider *dpp, const rgw_user& use
 
   rados::cls::otp::otp_check_t result;
 
-  r = rados::cls::otp::OTP::check(cct, ref.pool.ioctx(), ref.obj.oid, otp_id, pin, &result);
+  r = rados::cls::otp::OTP::check(cct, ref.ioctx, ref.obj.oid, otp_id, pin, &result);
   if (r < 0)
     return r;
 
@@ -154,7 +157,7 @@ int RGWSI_Cls::MFA::get_mfa(const DoutPrefixProvider *dpp, const rgw_user& user,
     return r;
   }
 
-  r = rados::cls::otp::OTP::get(nullptr, ref.pool.ioctx(), ref.obj.oid, id, result);
+  r = rados::cls::otp::OTP::get(nullptr, ref.ioctx, ref.obj.oid, id, result);
   if (r < 0) {
     return r;
   }
@@ -172,7 +175,7 @@ int RGWSI_Cls::MFA::list_mfa(const DoutPrefixProvider *dpp, const rgw_user& user
     return r;
   }
 
-  r = rados::cls::otp::OTP::get_all(nullptr, ref.pool.ioctx(), ref.obj.oid, result);
+  r = rados::cls::otp::OTP::get_all(nullptr, ref.ioctx, ref.obj.oid, result);
   if (r < 0) {
     return r;
   }
@@ -190,7 +193,7 @@ int RGWSI_Cls::MFA::otp_get_current_time(const DoutPrefixProvider *dpp, const rg
     return r;
   }
 
-  r = rados::cls::otp::OTP::get_current_time(ref.pool.ioctx(), ref.obj.oid, result);
+  r = rados::cls::otp::OTP::get_current_time(ref.ioctx, ref.obj.oid, result);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h
index d1d1d659be88..08d7cfffec6f 100644
--- a/src/rgw/services/svc_cls.h
+++ b/src/rgw/services/svc_cls.h
@@ -23,6 +23,8 @@
 
 #include "svc_rados.h"
 
+#include "driver/rados/rgw_tools.h"
+
 
 class RGWSI_Cls : public RGWServiceInstance
 {
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index ede029aa897b..4e7cf672c7a7 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -245,8 +245,6 @@ class RGWSI_RADOS : public RGWServiceInstance
   friend Pool::List;
 };
 
-using rgw_rados_ref = RGWSI_RADOS::rados_ref;
-
 inline std::ostream& operator<<(std::ostream& out, const RGWSI_RADOS::Obj& obj) {
   return out << obj.get_raw_obj();
 }

From ac27d7a82492c045eff4d2107865d29924147247 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 00:26:00 -0500
Subject: [PATCH 0768/2492] rgw: Remove RGWSI_RADOS from RGWSI_BI_RADOS

Simply use the RADOS handle and `rgw_rados_ref` directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_cr_rados.cc |  11 +++
 src/rgw/driver/rados/rgw_cr_rados.h  |   4 ++
 src/rgw/driver/rados/rgw_rados.cc    | 103 ++++++++++++++-------------
 src/rgw/driver/rados/rgw_rados.h     |   4 +-
 src/rgw/driver/rados/rgw_service.cc  |  10 +--
 src/rgw/driver/rados/rgw_service.h   |  19 +++--
 src/rgw/driver/rados/rgw_tools.h     |  65 +++++++++--------
 src/rgw/rgw_sal.cc                   |   9 +--
 src/rgw/services/svc_bi_rados.cc     |  66 ++++++++---------
 src/rgw/services/svc_bi_rados.h      |  23 +++---
 src/rgw/services/svc_bilog_rados.cc  |  18 ++---
 11 files changed, 181 insertions(+), 151 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index d73827bea142..cacba3cf44bc 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -497,6 +497,17 @@ RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
   set_description() << "remove dest=" << oid;
 }
 
+RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
+					 rgw_rados_ref&& obj,
+					 RGWObjVersionTracker* objv_tracker)
+  : RGWSimpleCoroutine(store->ctx()),
+    ioctx(std::move(obj.ioctx)),
+    oid(std::move(obj.obj.oid)),
+    objv_tracker(objv_tracker)
+{
+  set_description() << "remove dest=" << oid;
+}
+
 int RGWRadosRemoveOidCR::send_request(const DoutPrefixProvider *dpp)
 {
   librados::ObjectWriteOperation op;
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index b9c45f30cf69..0afb74d93a96 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -760,6 +760,10 @@ class RGWRadosRemoveOidCR : public RGWSimpleCoroutine {
 		      RGWSI_RADOS::Obj&& obj,
 		      RGWObjVersionTracker* objv_tracker = nullptr);
 
+  RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
+		      rgw_rados_ref&& obj,
+		      RGWObjVersionTracker* objv_tracker = nullptr);
+
   int send_request(const DoutPrefixProvider *dpp) override;
   int request_complete() override;
 };
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index d12d479be261..9cf0831ecaa9 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1363,10 +1363,11 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
 int RGWRados::init_svc(bool raw, const DoutPrefixProvider *dpp)
 {
   if (raw) {
-    return svc.init_raw(cct, use_cache, null_yield, dpp);
+    return svc.init_raw(cct, use_cache, get_rados_handle(), null_yield, dpp);
   }
 
-  return svc.init(cct, use_cache, run_sync_thread, null_yield, dpp);
+  return svc.init(cct, use_cache, run_sync_thread, get_rados_handle(),
+		  null_yield, dpp);
 }
 
 int RGWRados::init_ctl(const DoutPrefixProvider *dpp)
@@ -1380,7 +1381,13 @@ int RGWRados::init_ctl(const DoutPrefixProvider *dpp)
  */
 int RGWRados::init_begin(const DoutPrefixProvider *dpp)
 {
-  int ret = init_svc(false, dpp);
+  int ret = init_rados();
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: failed to init rados (ret=" << cpp_strerror(-ret) << ")" << dendl;
+    return ret;
+  }
+
+  ret = init_svc(false, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl;
     return ret;
@@ -1394,7 +1401,7 @@ int RGWRados::init_begin(const DoutPrefixProvider *dpp)
 
   host_id = svc.zone_utils->gen_host_id();
 
-  return init_rados();
+  return 0;
 }
 
 /**
@@ -2730,7 +2737,7 @@ int RGWRados::BucketShard::init(const rgw_bucket& _bucket,
     ldpp_dout(dpp, 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl;
     return ret;
   }
-  ldpp_dout(dpp, 20) << " bucket index object: " << bucket_obj.get_raw_obj() << dendl;
+  ldpp_dout(dpp, 20) << " bucket index object: " << bucket_obj << dendl;
 
   return 0;
 }
@@ -5060,7 +5067,7 @@ int RGWRados::check_bucket_empty(const DoutPrefixProvider *dpp, RGWBucketInfo& b
 int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp, bool check_empty)
 {
   const rgw_bucket& bucket = bucket_info.bucket;
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt, bucket_info.layout.current_index, &index_pool, &bucket_objs, nullptr);
   if (r < 0)
@@ -5115,7 +5122,7 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& ob
     }
 
    /* remove bucket index objects asynchronously by best effort */
-    (void) CLSRGWIssueBucketIndexClean(index_pool.ioctx(),
+    (void) CLSRGWIssueBucketIndexClean(index_pool,
 				       bucket_objs,
 				       cct->_conf->rgw_bucket_index_max_aio)();
   }
@@ -5303,7 +5310,7 @@ int RGWRados::bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& b
 				 map<RGWObjCategory, RGWStorageStats> *existing_stats,
 				 map<RGWObjCategory, RGWStorageStats> *calculated_stats)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
 
   // key - bucket index object id
   // value - bucket index check OP returned result with the given bucket index object (shard)
@@ -5320,7 +5327,7 @@ int RGWRados::bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& b
     bucket_objs_ret.emplace(iter.first, rgw_cls_check_index_ret());
   }
 
-  ret = CLSRGWIssueBucketCheck(index_pool.ioctx(), oids, bucket_objs_ret, cct->_conf->rgw_bucket_index_max_aio)();
+  ret = CLSRGWIssueBucketCheck(index_pool, oids, bucket_objs_ret, cct->_conf->rgw_bucket_index_max_aio)();
   if (ret < 0) {
     return ret;
   }
@@ -5336,7 +5343,7 @@ int RGWRados::bucket_check_index(const DoutPrefixProvider *dpp, RGWBucketInfo& b
 
 int RGWRados::bucket_rebuild_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
 
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt, bucket_info.layout.current_index, &index_pool, &bucket_objs, nullptr);
@@ -5344,7 +5351,7 @@ int RGWRados::bucket_rebuild_index(const DoutPrefixProvider *dpp, RGWBucketInfo&
     return r;
   }
 
-  return CLSRGWIssueBucketRebuild(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
+  return CLSRGWIssueBucketRebuild(index_pool, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
 }
 
 static int resync_encrypted_multipart(const DoutPrefixProvider* dpp,
@@ -5483,7 +5490,7 @@ int RGWRados::bucket_resync_encrypted_multipart(const DoutPrefixProvider* dpp,
 
 int RGWRados::bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const cls_rgw_bucket_instance_entry& entry)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
 
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt, bucket_info.layout.current_index, &index_pool, &bucket_objs, nullptr);
@@ -5494,7 +5501,7 @@ int RGWRados::bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketI
     return r;
   }
 
-  r = CLSRGWIssueSetBucketResharding(index_pool.ioctx(), bucket_objs, entry, cct->_conf->rgw_bucket_index_max_aio)();
+  r = CLSRGWIssueSetBucketResharding(index_pool, bucket_objs, entry, cct->_conf->rgw_bucket_index_max_aio)();
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ <<
       ": unable to issue set bucket resharding, r=" << r << " (" <<
@@ -7544,8 +7551,8 @@ int RGWRados::block_while_resharding(RGWRados::BucketShard *bs,
 
   constexpr int num_retries = 10;
   for (int i = 1; i <= num_retries; i++) { // nb: 1-based for loop
-    auto& ref = bs->bucket_obj.get_ref();
-    ret = cls_rgw_get_bucket_resharding(ref.pool.ioctx(), ref.obj.oid, &entry);
+    auto& ref = bs->bucket_obj;
+    ret = cls_rgw_get_bucket_resharding(ref.ioctx, ref.obj.oid, &entry);
     if (ret == -ENOENT) {
       ret = fetch_new_bucket_info("get_bucket_resharding_failed");
       if (ret < 0) {
@@ -7684,7 +7691,7 @@ int RGWRados::bucket_index_link_olh(const DoutPrefixProvider *dpp, RGWBucketInfo
   r = guard_reshard(dpp, &bs, obj_instance, bucket_info,
 		    [&](BucketShard *bs) -> int {
 		      cls_rgw_obj_key key(obj_instance.key.get_index_key_name(), obj_instance.key.instance);
-		      auto& ref = bs->bucket_obj.get_ref();
+		      auto& ref = bs->bucket_obj;
 		      librados::ObjectWriteOperation op;
 		      op.assert_exists(); // bucket index shard must exist
 		      cls_rgw_guard_bucket_resharding(op, -ERR_BUSY_RESHARDING);
@@ -7692,7 +7699,7 @@ int RGWRados::bucket_index_link_olh(const DoutPrefixProvider *dpp, RGWBucketInfo
                                               delete_marker, op_tag, meta, olh_epoch,
 					      unmod_since, high_precision_time,
 					      svc.zone->need_to_log_data(), zones_trace);
-                      return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+                      return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
                     }, y);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "rgw_rados_operate() after cls_rgw_bucket_link_olh() returned r=" << r << dendl;
@@ -7735,13 +7742,13 @@ int RGWRados::bucket_index_unlink_instance(const DoutPrefixProvider *dpp,
   cls_rgw_obj_key key(obj_instance.key.get_index_key_name(), obj_instance.key.instance);
   r = guard_reshard(dpp, &bs, obj_instance, bucket_info,
 		    [&](BucketShard *bs) -> int {
-		      auto& ref = bs->bucket_obj.get_ref();
+		      auto& ref = bs->bucket_obj;
 		      librados::ObjectWriteOperation op;
 		      op.assert_exists(); // bucket index shard must exist
 		      cls_rgw_guard_bucket_resharding(op, -ERR_BUSY_RESHARDING);
 		      cls_rgw_bucket_unlink_instance(op, key, op_tag,
 						     olh_tag, olh_epoch, svc.zone->need_to_log_data(), zones_trace);
-                      return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+                      return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
                     }, y);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "rgw_rados_operate() after cls_rgw_bucket_link_instance() returned r=" << r << dendl;
@@ -7775,14 +7782,14 @@ int RGWRados::bucket_index_read_olh_log(const DoutPrefixProvider *dpp,
 
   cls_rgw_obj_key key(obj_instance.key.get_index_key_name(), string());
 
-  auto& shard_ref = bs.bucket_obj.get_ref();
+  auto& shard_ref = bs.bucket_obj;
   ObjectReadOperation op;
 
   rgw_cls_read_olh_log_ret log_ret;
   int op_ret = 0;
-  cls_rgw_get_olh_log(op, key, ver_marker, olh_tag, log_ret, op_ret); 
+  cls_rgw_get_olh_log(op, key, ver_marker, olh_tag, log_ret, op_ret);
   bufferlist outbl;
-  r =  rgw_rados_operate(dpp, shard_ref.pool.ioctx(), shard_ref.obj.oid, &op, &outbl, y);
+  r =  rgw_rados_operate(dpp, shard_ref.ioctx, shard_ref.obj.oid, &op, &outbl, y);
   if (r < 0) {
     return r;
   }
@@ -7910,10 +7917,10 @@ int RGWRados::bucket_index_clear_olh(const DoutPrefixProvider *dpp,
 			  [&](BucketShard *pbs) -> int {
 			    ObjectWriteOperation op;
 			    op.assert_exists(); // bucket index shard must exist
-			    auto& ref = pbs->bucket_obj.get_ref();
+			    auto& ref = bs.bucket_obj;
 			    cls_rgw_guard_bucket_resharding(op, -ERR_BUSY_RESHARDING);
 			    cls_rgw_clear_olh(op, key, olh_tag);
-                            return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, y);
+                            return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
                           }, y);
   if (ret < 0) {
     ldpp_dout(dpp, 5) << "rgw_rados_operate() after cls_rgw_clear_olh() returned ret=" << ret << dendl;
@@ -8977,21 +8984,21 @@ int RGWRados::bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_
 
   cls_rgw_obj_key key(obj.key.get_index_key_name(), obj.key.instance);
 
-  auto& ref = bs.bucket_obj.get_ref();
-  
-  return cls_rgw_bi_get(ref.pool.ioctx(), ref.obj.oid, index_type, key, entry);
+  auto& ref = bs.bucket_obj;
+
+  return cls_rgw_bi_get(ref.ioctx, ref.obj.oid, index_type, key, entry);
 }
 
 void RGWRados::bi_put(ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry, optional_yield y)
 {
-  auto& ref = bs.bucket_obj.get_ref();
+  auto& ref = bs.bucket_obj;
   cls_rgw_bi_put(op, ref.obj.oid, entry);
 }
 
 int RGWRados::bi_put(BucketShard& bs, rgw_cls_bi_entry& entry, optional_yield y)
 {
-  auto& ref = bs.bucket_obj.get_ref();
-  int ret = cls_rgw_bi_put(ref.pool.ioctx(), ref.obj.oid, entry);
+  auto& ref = bs.bucket_obj;
+  int ret = cls_rgw_bi_put(ref.ioctx, ref.obj.oid, entry);
   if (ret < 0)
     return ret;
 
@@ -9029,8 +9036,8 @@ int RGWRados::bi_list(const DoutPrefixProvider *dpp, rgw_bucket& bucket,
     return ret;
   }
 
-  auto& ref = bs.bucket_obj.get_ref();
-  ret = cls_rgw_bi_list(ref.pool.ioctx(), ref.obj.oid, obj_name_filter, marker, max, entries, is_truncated);
+  auto& ref = bs.bucket_obj;
+  ret = cls_rgw_bi_list(ref.ioctx, ref.obj.oid, obj_name_filter, marker, max, entries, is_truncated);
   if (ret == -ENOENT) {
     *is_truncated = false;
   }
@@ -9043,8 +9050,8 @@ int RGWRados::bi_list(const DoutPrefixProvider *dpp, rgw_bucket& bucket,
 int RGWRados::bi_list(BucketShard& bs, const string& obj_name_filter, const string& marker, uint32_t max,
 		      list<rgw_cls_bi_entry> *entries, bool *is_truncated, optional_yield y)
 {
-  auto& ref = bs.bucket_obj.get_ref();
-  int ret = cls_rgw_bi_list(ref.pool.ioctx(), ref.obj.oid, obj_name_filter, marker, max, entries, is_truncated);
+  auto& ref = bs.bucket_obj;
+  int ret = cls_rgw_bi_list(ref.ioctx, ref.obj.oid, obj_name_filter, marker, max, entries, is_truncated);
   if (ret < 0)
     return ret;
 
@@ -9069,8 +9076,8 @@ int RGWRados::bi_list(const DoutPrefixProvider *dpp,
 
 int RGWRados::bi_remove(const DoutPrefixProvider *dpp, BucketShard& bs)
 {
-  auto& ref = bs.bucket_obj.get_ref();
-  int ret = ref.pool.ioctx().remove(ref.obj.oid);
+  auto& ref = bs.bucket_obj;
+  int ret = ref.ioctx.remove(ref.obj.oid);
   if (ret == -ENOENT) {
     ret = 0;
   }
@@ -9234,13 +9241,13 @@ int RGWRados::cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj
 
 int RGWRados::cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt, bucket_info.layout.current_index, &index_pool, &bucket_objs, nullptr);
   if (r < 0)
     return r;
 
-  return CLSRGWIssueSetTagTimeout(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio, timeout)();
+  return CLSRGWIssueSetTagTimeout(index_pool, bucket_objs, cct->_conf->rgw_bucket_index_max_aio, timeout)();
 }
 
 
@@ -9320,7 +9327,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp,
 
   m.clear();
 
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   // key   - oid (for different shards if there is any)
   // value - list result for the corresponding oid (shard), it is filled by
   //         the AIO callback
@@ -9368,7 +9375,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp,
     " shard(s) for " << num_entries_per_shard << " entries to get " <<
     num_entries << " total entries" << dendl;
 
-  auto& ioctx = index_pool.ioctx();
+  auto& ioctx = index_pool;
   std::map<int, rgw_cls_list_ret> shard_list_results;
   cls_rgw_obj_key start_after_key(start_after.name, start_after.instance);
   r = CLSRGWIssueBucketList(ioctx, start_after_key, prefix, delimiter,
@@ -9656,7 +9663,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp,
   static MultipartMetaFilter multipart_meta_filter;
 
   *is_truncated = false;
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
 
   std::map<int, std::string> oids;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, nullptr);
@@ -9664,7 +9671,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp,
     return r;
   }
 
-  auto& ioctx = index_pool.ioctx();
+  auto& ioctx = index_pool;
 
   const uint32_t num_shards = oids.size();
 
@@ -9925,7 +9932,7 @@ int RGWRados::remove_objs_from_index(const DoutPrefixProvider *dpp,
   }
   const uint32_t num_shards = current_index.layout.normal.num_shards;
 
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   std::map<int, std::string> index_oids;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt,
 					  bucket_info.layout.current_index,
@@ -9968,7 +9975,7 @@ int RGWRados::remove_objs_from_index(const DoutPrefixProvider *dpp,
       ", shard=" << shard << ", oid=" << oid << ", num_keys=" <<
       removals.second.size() << dendl_bitx;
 
-    r = index_pool.ioctx().omap_rm_keys(oid, removals.second);
+    r = index_pool.omap_rm_keys(oid, removals.second);
     if (r < 0) {
       ldout_bitx(bitx, dpp, 0) << "ERROR: " << __func__ <<
 	": omap_rm_keys returned ret=" << r <<
@@ -10143,7 +10150,7 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp,
 
 int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int shard_id, vector<rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> oids;
   map<int, struct rgw_cls_list_ret> list_results;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, bucket_instance_ids);
@@ -10153,7 +10160,7 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo
     return r;
   }
 
-  r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)();
+  r = CLSRGWIssueGetDirHeader(index_pool, oids, list_results, cct->_conf->rgw_bucket_index_max_aio)();
   if (r < 0) {
     ldpp_dout(dpp, 20) << "cls_bucket_head: CLSRGWIssueGetDirHeader() returned "
                    << r << dendl;
@@ -10171,14 +10178,14 @@ int RGWRados::cls_bucket_head_async(const DoutPrefixProvider *dpp, const RGWBuck
                                     const rgw::bucket_index_layout_generation& idx_layout, int shard_id,
                                     boost::intrusive_ptr<RGWGetDirHeader_CB> cb, int *num_aio)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
   int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &bucket_objs, nullptr);
   if (r < 0)
     return r;
 
   for (auto& pair : bucket_objs) {
-    r = cls_rgw_get_dir_header_async(index_pool.ioctx(), pair.second, cb);
+    r = cls_rgw_get_dir_header_async(index_pool, pair.second, cb);
     if (r < 0) {
       return r;
     }
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 938bc161a7ef..97528735417d 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -651,7 +651,7 @@ class RGWRados
     RGWRados *store;
     rgw_bucket bucket;
     int shard_id;
-    RGWSI_RADOS::Obj bucket_obj;
+    rgw_rados_ref bucket_obj;
 
     explicit BucketShard(RGWRados *_store) : store(_store), shard_id(-1) {}
     int init(const rgw_bucket& _bucket, const rgw_obj& obj,
@@ -664,7 +664,7 @@ class RGWRados
     friend std::ostream& operator<<(std::ostream& out, const BucketShard& bs) {
       out << "BucketShard:{ bucket=" << bs.bucket <<
 	", shard_id=" << bs.shard_id <<
-	", bucket_ojb=" << bs.bucket_obj << "}";
+	", bucket_obj=" << bs.bucket_obj << "}";
       return out;
     }
   };
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 4fcb1ebdef7f..8efc97b2e250 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -51,6 +51,7 @@ int RGWServices_Def::init(CephContext *cct,
 			  bool have_cache,
                           bool raw,
 			  bool run_sync,
+			  librados::Rados* radoshandle,
 			  optional_yield y,
                           const DoutPrefixProvider *dpp)
 {
@@ -85,7 +86,7 @@ int RGWServices_Def::init(CephContext *cct,
   vector<RGWSI_MetaBackend *> meta_bes{meta_be_sobj.get(), meta_be_otp.get()};
 
   finisher->init();
-  bi_rados->init(zone.get(), rados.get(), bilog_rados.get(), datalog_rados.get());
+  bi_rados->init(zone.get(), radoshandle, bilog_rados.get(), datalog_rados.get());
   bilog_rados->init(bi_rados.get());
   bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(),
                     bi_rados.get(), meta.get(), meta_be_sobj.get(),
@@ -305,12 +306,13 @@ void RGWServices_Def::shutdown()
 
 }
 
-
-int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp)
+int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
+			 bool run_sync, librados::Rados* radoshandle,
+			 optional_yield y, const DoutPrefixProvider *dpp)
 {
   cct = _cct;
 
-  int r = _svc.init(cct, have_cache, raw, run_sync, y, dpp);
+  int r = _svc.init(cct, have_cache, raw, run_sync, radoshandle, y, dpp);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 4c0b8d8421f7..46ee16417feb 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -108,7 +108,9 @@ struct RGWServices_Def
   RGWServices_Def();
   ~RGWServices_Def();
 
-  int init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp);
+  int init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync,
+	   librados::Rados* radoshandle, optional_yield y,
+	   const DoutPrefixProvider *dpp);
   void shutdown();
 };
 
@@ -148,14 +150,19 @@ struct RGWServices
   RGWSI_User *user{nullptr};
   RGWSI_Role_RADOS *role{nullptr};
 
-  int do_init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp);
+  int do_init(CephContext *cct, bool have_cache, bool raw_storage,
+	      bool run_sync, librados::Rados* radoshandle, optional_yield y,
+	      const DoutPrefixProvider *dpp);
 
-  int init(CephContext *cct, bool have_cache, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, false, run_sync, y, dpp);
+  int init(CephContext *cct, bool have_cache, bool run_sync,
+	   librados::Rados* radoshandle, optional_yield y,
+	   const DoutPrefixProvider *dpp) {
+    return do_init(cct, have_cache, false, run_sync, radoshandle, y, dpp);
   }
 
-  int init_raw(CephContext *cct, bool have_cache, optional_yield y, const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, true, false, y, dpp);
+  int init_raw(CephContext *cct, bool have_cache, librados::Rados* radoshandle,
+	       optional_yield y, const DoutPrefixProvider *dpp) {
+    return do_init(cct, have_cache, true, false, radoshandle, y, dpp);
   }
   void shutdown() {
     _svc.shutdown();
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 32cc029d7581..afc836a473a4 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -21,13 +21,6 @@ class optional_yield;
 
 struct obj_version;
 
-struct rgw_rados_ref {
-  librados::IoCtx ioctx;
-  rgw_raw_obj obj;
-};
-int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
-		      rgw_raw_obj obj, rgw_rados_ref* ref);
-
 int rgw_init_ioctx(const DoutPrefixProvider *dpp,
                    librados::Rados *rados, const rgw_pool& pool,
                    librados::IoCtx& ioctx,
@@ -106,31 +99,47 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
 		      int flags = 0);
-inline int rgw_rados_operate(const DoutPrefixProvider *dpp,
-                             rgw_rados_ref& ref,
-                             librados::ObjectReadOperation *op, bufferlist *pbl,
-                             optional_yield y, int flags = 0)
-{
-  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, pbl, y, flags);
-}
-
-inline int rgw_rados_operate(const DoutPrefixProvider *dpp,
-                             rgw_rados_ref& ref,
-                             librados::ObjectWriteOperation *op,
-                             optional_yield y, int flags = 0)
-{
-  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, op, y, flags);
-}
-
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                      bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl,
                      optional_yield y);
-inline int rgw_rados_notify(const DoutPrefixProvider *dpp, rgw_rados_ref& ref,
-			    bufferlist& bl, uint64_t timeout_ms,
-			    bufferlist* pbl, optional_yield y)
-{
-  return rgw_rados_notify(dpp, ref.ioctx, ref.obj.oid, bl, timeout_ms, pbl, y);
+
+struct rgw_rados_ref {
+  librados::IoCtx ioctx;
+  rgw_raw_obj obj;
+
+
+  int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op,
+	      bufferlist *pbl, optional_yield y, int flags = 0) {
+    return rgw_rados_operate(dpp, ioctx, obj.oid, op, pbl, y, flags);
+  }
+
+  int operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op,
+	      optional_yield y, int flags = 0) {
+    return rgw_rados_operate(dpp, ioctx, obj.oid, op, y, flags);
+  }
+
+  int aio_operate(librados::AioCompletion *c,
+		  librados::ObjectWriteOperation *op) {
+    return ioctx.aio_operate(obj.oid, c, op);
+  }
+
+  int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
+		  bufferlist *pbl) {
+    return ioctx.aio_operate(obj.oid, c, op, pbl);
+  }
+
+  int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms,
+	     bufferlist* pbl, optional_yield y) {
+    return rgw_rados_notify(dpp, ioctx, obj.oid, bl, timeout_ms, pbl, y);
+  }
+};
+inline std::ostream& operator <<(std::ostream& m, const rgw_rados_ref& ref) {
+  return m << ref.obj;
 }
+int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
+		      rgw_raw_obj obj, rgw_rados_ref* ref);
+
+
 
 int rgw_tools_init(const DoutPrefixProvider *dpp, CephContext *cct);
 void rgw_tools_cleanup();
diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc
index 9aada70899f9..6c0212dae817 100644
--- a/src/rgw/rgw_sal.cc
+++ b/src/rgw/rgw_sal.cc
@@ -270,17 +270,18 @@ rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvi
 
     rados->set_context(cct);
 
-    int ret = rados->init_svc(true, dpp);
-    if (ret < 0) {
-      ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl;
+    if (rados->init_rados() < 0) {
       delete driver;
       return nullptr;
     }
 
-    if (rados->init_rados() < 0) {
+    int ret = rados->init_svc(true, dpp);
+    if (ret < 0) {
+      ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl;
       delete driver;
       return nullptr;
     }
+
     if (driver->initialize(cct, dpp) < 0) {
       delete driver;
       return nullptr;
diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc
index 6002b986f592..20c842c3805d 100644
--- a/src/rgw/services/svc_bi_rados.cc
+++ b/src/rgw/services/svc_bi_rados.cc
@@ -22,29 +22,27 @@ RGWSI_BucketIndex_RADOS::RGWSI_BucketIndex_RADOS(CephContext *cct) : RGWSI_Bucke
 }
 
 void RGWSI_BucketIndex_RADOS::init(RGWSI_Zone *zone_svc,
-                                   RGWSI_RADOS *rados_svc,
-                                   RGWSI_BILog_RADOS *bilog_svc,
-                                   RGWDataChangesLog *datalog_rados_svc)
+				   librados::Rados* rados_,
+				   RGWSI_BILog_RADOS *bilog_svc,
+				   RGWDataChangesLog *datalog_rados_svc)
 {
   svc.zone = zone_svc;
-  svc.rados = rados_svc;
+  rados = rados_;
   svc.bilog = bilog_svc;
   svc.datalog_rados = datalog_rados_svc;
 }
 
 int RGWSI_BucketIndex_RADOS::open_pool(const DoutPrefixProvider *dpp,
                                        const rgw_pool& pool,
-                                       RGWSI_RADOS::Pool *index_pool,
+                                       librados::IoCtx* index_pool,
                                        bool mostly_omap)
 {
-  *index_pool = svc.rados->pool(pool);
-  return index_pool->open(dpp, RGWSI_RADOS::OpenParams()
-                          .set_mostly_omap(mostly_omap));
+  return rgw_init_ioctx(dpp, rados, pool, *index_pool, true, mostly_omap);
 }
 
 int RGWSI_BucketIndex_RADOS::open_bucket_index_pool(const DoutPrefixProvider *dpp,
                                                     const RGWBucketInfo& bucket_info,
-                                                    RGWSI_RADOS::Pool *index_pool)
+                                                    librados::IoCtx* index_pool)
 {
   const rgw_pool& explicit_pool = bucket_info.bucket.explicit_placement.index_pool;
 
@@ -74,7 +72,7 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index_pool(const DoutPrefixProvider *dp
 
 int RGWSI_BucketIndex_RADOS::open_bucket_index_base(const DoutPrefixProvider *dpp,
                                                     const RGWBucketInfo& bucket_info,
-                                                    RGWSI_RADOS::Pool *index_pool,
+                                                    librados::IoCtx* index_pool,
                                                     string *bucket_oid_base)
 {
   const rgw_bucket& bucket = bucket_info.bucket;
@@ -96,7 +94,7 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index_base(const DoutPrefixProvider *dp
 
 int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp,
                                                const RGWBucketInfo& bucket_info,
-                                               RGWSI_RADOS::Pool *index_pool,
+                                               librados::IoCtx* index_pool,
                                                string *bucket_oid)
 {
   const rgw_bucket& bucket = bucket_info.bucket;
@@ -194,7 +192,7 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp,
                                                const RGWBucketInfo& bucket_info,
                                                std::optional<int> _shard_id,
                                                const rgw::bucket_index_layout_generation& idx_layout,
-                                               RGWSI_RADOS::Pool *index_pool,
+                                               librados::IoCtx* index_pool,
                                                map<int, string> *bucket_objs,
                                                map<int, string> *bucket_instance_ids)
 {
@@ -277,32 +275,27 @@ int RGWSI_BucketIndex_RADOS::get_bucket_index_object(
 int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *dpp,
                                                      const RGWBucketInfo& bucket_info,
                                                      const string& obj_key,
-                                                     RGWSI_RADOS::Obj *bucket_obj,
+                                                     rgw_rados_ref* bucket_obj,
                                                      int *shard_id)
 {
   string bucket_oid_base;
 
-  RGWSI_RADOS::Pool pool;
-
-  int ret = open_bucket_index_base(dpp, bucket_info, &pool, &bucket_oid_base);
+  int ret = open_bucket_index_base(dpp, bucket_info, &bucket_obj->ioctx, &bucket_oid_base);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned "
                    << ret << dendl;
     return ret;
   }
 
-  string oid;
-
   const auto& current_index = bucket_info.layout.current_index;
   ret = get_bucket_index_object(bucket_oid_base, current_index.layout.normal,
-                                current_index.gen, obj_key, &oid, shard_id);
+                                current_index.gen, obj_key,
+				&bucket_obj->obj.oid, shard_id);
   if (ret < 0) {
     ldpp_dout(dpp, 10) << "get_bucket_index_object() returned ret=" << ret << dendl;
     return ret;
   }
 
-  *bucket_obj = svc.rados->obj(pool, oid);
-
   return 0;
 }
 
@@ -310,23 +303,19 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *d
                                                      const RGWBucketInfo& bucket_info,
                                                      const rgw::bucket_index_layout_generation& index,
                                                      int shard_id,
-                                                     RGWSI_RADOS::Obj *bucket_obj)
+                                                     rgw_rados_ref* bucket_obj)
 {
-  RGWSI_RADOS::Pool index_pool;
   string bucket_oid_base;
-  int ret = open_bucket_index_base(dpp, bucket_info, &index_pool, &bucket_oid_base);
+  int ret = open_bucket_index_base(dpp, bucket_info, &bucket_obj->ioctx,
+				   &bucket_oid_base);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned "
                    << ret << dendl;
     return ret;
   }
 
-  string oid;
-
   get_bucket_index_object(bucket_oid_base, index.layout.normal,
-                          index.gen, shard_id, &oid);
-
-  *bucket_obj = svc.rados->obj(index_pool, oid);
+                          index.gen, shard_id, &bucket_obj->obj.oid);
 
   return 0;
 }
@@ -339,7 +328,7 @@ int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp,
                                              map<int, string> *bucket_instance_ids,
                                              optional_yield y)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> oids;
   int r = open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, bucket_instance_ids);
   if (r < 0)
@@ -350,7 +339,8 @@ int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp,
     list_results.emplace(iter.first, rgw_cls_list_ret());
   }
 
-  r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)();
+  r = CLSRGWIssueGetDirHeader(index_pool, oids, list_results,
+			      cct->_conf->rgw_bucket_index_max_aio)();
   if (r < 0)
     return r;
 
@@ -361,9 +351,9 @@ int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp,
   return 0;
 }
 
-int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout)
+int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp,RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
 
   string dir_oid = dir_oid_prefix;
   int r = open_bucket_index_pool(dpp, bucket_info, &index_pool);
@@ -376,14 +366,14 @@ int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, RGWBucket
   map<int, string> bucket_objs;
   get_bucket_index_objects(dir_oid, idx_layout.layout.normal.num_shards, idx_layout.gen, &bucket_objs);
 
-  return CLSRGWIssueBucketIndexInit(index_pool.ioctx(),
+  return CLSRGWIssueBucketIndexInit(index_pool,
 				    bucket_objs,
 				    cct->_conf->rgw_bucket_index_max_aio)();
 }
 
 int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
 
   std::string dir_oid = dir_oid_prefix;
   int r = open_bucket_index_pool(dpp, bucket_info, &index_pool);
@@ -397,7 +387,7 @@ int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucke
   get_bucket_index_objects(dir_oid, idx_layout.layout.normal.num_shards,
                            idx_layout.gen, &bucket_objs);
 
-  return CLSRGWIssueBucketIndexClean(index_pool.ioctx(),
+  return CLSRGWIssueBucketIndexClean(index_pool,
 				     bucket_objs,
 				     cct->_conf->rgw_bucket_index_max_aio)();
 }
@@ -440,7 +430,7 @@ int RGWSI_BucketIndex_RADOS::get_reshard_status(const DoutPrefixProvider *dpp, c
 {
   map<int, string> bucket_objs;
 
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
 
   int r = open_bucket_index(dpp, bucket_info,
                             std::nullopt,
@@ -455,7 +445,7 @@ int RGWSI_BucketIndex_RADOS::get_reshard_status(const DoutPrefixProvider *dpp, c
   for (auto i : bucket_objs) {
     cls_rgw_bucket_instance_entry entry;
 
-    int ret = cls_rgw_get_bucket_resharding(index_pool.ioctx(), i.second, &entry);
+    int ret = cls_rgw_get_bucket_resharding(index_pool, i.second, &entry);
     if (ret < 0 && ret != -ENOENT) {
       ldpp_dout(dpp, -1) << "ERROR: " << __func__ << ": cls_rgw_get_bucket_resharding() returned ret=" << ret << dendl;
       return ret;
diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h
index feba0cfcd193..35f5daba2ed2 100644
--- a/src/rgw/services/svc_bi_rados.h
+++ b/src/rgw/services/svc_bi_rados.h
@@ -45,15 +45,15 @@ class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex
 
   int open_pool(const DoutPrefixProvider *dpp,
                 const rgw_pool& pool,
-                RGWSI_RADOS::Pool *index_pool,
+                librados::IoCtx* index_pool,
                 bool mostly_omap);
 
   int open_bucket_index_pool(const DoutPrefixProvider *dpp,
-                            const RGWBucketInfo& bucket_info,
-                            RGWSI_RADOS::Pool *index_pool);
+			     const RGWBucketInfo& bucket_info,
+			     librados::IoCtx* index_pool);
   int open_bucket_index_base(const DoutPrefixProvider *dpp,
                              const RGWBucketInfo& bucket_info,
-                             RGWSI_RADOS::Pool *index_pool,
+                             librados::IoCtx* index_pool,
                              std::string *bucket_oid_base);
 
   // return the index oid for the given shard id
@@ -77,9 +77,10 @@ class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex
 
 public:
 
+  librados::Rados* rados{nullptr};
+
   struct Svc {
     RGWSI_Zone *zone{nullptr};
-    RGWSI_RADOS *rados{nullptr};
     RGWSI_BILog_RADOS *bilog{nullptr};
     RGWDataChangesLog *datalog_rados{nullptr};
   } svc;
@@ -87,7 +88,7 @@ class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex
   RGWSI_BucketIndex_RADOS(CephContext *cct);
 
   void init(RGWSI_Zone *zone_svc,
-            RGWSI_RADOS *rados_svc,
+            librados::Rados* rados_,
             RGWSI_BILog_RADOS *bilog_svc,
             RGWDataChangesLog *datalog_rados_svc);
 
@@ -141,26 +142,24 @@ class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex
   int open_bucket_index_shard(const DoutPrefixProvider *dpp,
                               const RGWBucketInfo& bucket_info,
                               const std::string& obj_key,
-                              RGWSI_RADOS::Obj *bucket_obj,
+                              rgw_rados_ref* bucket_obj,
                               int *shard_id);
 
   int open_bucket_index_shard(const DoutPrefixProvider *dpp,
                               const RGWBucketInfo& bucket_info,
                               const rgw::bucket_index_layout_generation& index,
-                              int shard_id, RGWSI_RADOS::Obj *bucket_obj);
+                              int shard_id, rgw_rados_ref* bucket_obj);
 
   int open_bucket_index(const DoutPrefixProvider *dpp,
                         const RGWBucketInfo& bucket_info,
-                        RGWSI_RADOS::Pool *index_pool,
+                        librados::IoCtx* index_pool,
                         std::string *bucket_oid);
 
   int open_bucket_index(const DoutPrefixProvider *dpp,
                         const RGWBucketInfo& bucket_info,
                         std::optional<int> shard_id,
                         const rgw::bucket_index_layout_generation& idx_layout,
-                        RGWSI_RADOS::Pool *index_pool,
+                        librados::IoCtx* index_pool,
                         std::map<int, std::string> *bucket_objs,
                         std::map<int, std::string> *bucket_instance_ids);
 };
-
-
diff --git a/src/rgw/services/svc_bilog_rados.cc b/src/rgw/services/svc_bilog_rados.cc
index f4bb13ec1f05..a33eb508848a 100644
--- a/src/rgw/services/svc_bilog_rados.cc
+++ b/src/rgw/services/svc_bilog_rados.cc
@@ -26,7 +26,7 @@ int RGWSI_BILog_RADOS::log_trim(const DoutPrefixProvider *dpp,
 				std::string_view start_marker,
 				std::string_view end_marker)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
 
   BucketIndexShardsManager start_marker_mgr;
@@ -48,32 +48,32 @@ int RGWSI_BILog_RADOS::log_trim(const DoutPrefixProvider *dpp,
     return r;
   }
 
-  return CLSRGWIssueBILogTrim(index_pool.ioctx(), start_marker_mgr, end_marker_mgr, bucket_objs,
+  return CLSRGWIssueBILogTrim(index_pool, start_marker_mgr, end_marker_mgr, bucket_objs,
 			      cct->_conf->rgw_bucket_index_max_aio)();
 }
 
 int RGWSI_BILog_RADOS::log_start(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
   const auto& current_index = rgw::log_to_index_layout(log_layout);
   int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &bucket_objs, nullptr);
   if (r < 0)
     return r;
 
-  return CLSRGWIssueResyncBucketBILog(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
+  return CLSRGWIssueResyncBucketBILog(index_pool, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
 }
 
 int RGWSI_BILog_RADOS::log_stop(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id)
 {
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> bucket_objs;
   const auto& current_index = rgw::log_to_index_layout(log_layout);
   int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &bucket_objs, nullptr);
   if (r < 0)
     return r;
 
-  return CLSRGWIssueBucketBILogStop(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
+  return CLSRGWIssueBucketBILogStop(index_pool, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)();
 }
 
 static void build_bucket_index_marker(const string& shard_id_str,
@@ -95,7 +95,7 @@ int RGWSI_BILog_RADOS::log_list(const DoutPrefixProvider *dpp,
   ldpp_dout(dpp, 20) << __func__ << ": " << bucket_info.bucket << " marker " << marker << " shard_id=" << shard_id << " max " << max << dendl;
   result.clear();
 
-  RGWSI_RADOS::Pool index_pool;
+  librados::IoCtx index_pool;
   map<int, string> oids;
   map<int, cls_rgw_bi_log_list_ret> bi_log_lists;
   const auto& current_index = rgw::log_to_index_layout(log_layout);
@@ -112,8 +112,8 @@ int RGWSI_BILog_RADOS::log_list(const DoutPrefixProvider *dpp,
   r = marker_mgr.from_string(marker, shard_id);
   if (r < 0)
     return r;
- 
-  r = CLSRGWIssueBILogList(index_pool.ioctx(), marker_mgr, max, oids, bi_log_lists, cct->_conf->rgw_bucket_index_max_aio)();
+
+  r = CLSRGWIssueBILogList(index_pool, marker_mgr, max, oids, bi_log_lists, cct->_conf->rgw_bucket_index_max_aio)();
   if (r < 0)
     return r;
 

From 23120f7f8b3fc630b91f2cab21109695c41f519d Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 00:55:48 -0500
Subject: [PATCH 0769/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_Cls`

Simply use the RADOS handle and `rgw_rados_ref` directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc |  2 +-
 src/rgw/rgw_mdlog.h                 |  9 +--
 src/rgw/services/svc_cls.cc         | 98 ++++++++++++-----------------
 src/rgw/services/svc_cls.h          | 30 ++++-----
 4 files changed, 59 insertions(+), 80 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 8efc97b2e250..cec7cf22d985 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -95,7 +95,7 @@ int RGWServices_Def::init(CephContext *cct,
                          sysobj.get(),
                          sysobj_cache.get(),
                          bucket_sobj.get());
-  cls->init(zone.get(), rados.get());
+  cls->init(zone.get(), radoshandle);
   config_key_rados->init(rados.get());
   mdlog->init(rados.get(), zone.get(), sysobj.get(), cls.get());
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
diff --git a/src/rgw/rgw_mdlog.h b/src/rgw/rgw_mdlog.h
index 152126890b75..6c78fd384085 100644
--- a/src/rgw/rgw_mdlog.h
+++ b/src/rgw/rgw_mdlog.h
@@ -16,12 +16,13 @@
 
 #pragma once
 
+#include "include/rados/librados.hpp"
+
 #include "common/RWLock.h"
 
 #include "rgw_metadata.h"
 #include "rgw_mdlog_types.h"
-
-#include "services/svc_rados.h"
+#include "rgw_tools.h"
 
 #define META_LOG_OBJ_PREFIX "meta.log."
 
@@ -40,7 +41,7 @@ class RGWMetadataLogInfoCompletion : public RefCountedObject {
   using info_callback_t = std::function<void(int, const cls_log_header&)>;
  private:
   cls_log_header header;
-  RGWSI_RADOS::Obj io_obj;
+  rgw_rados_ref io_obj;
   librados::AioCompletion *completion;
   std::mutex mutex; //< protects callback between cancel/complete
   boost::optional<info_callback_t> callback; //< cleared on cancel
@@ -48,7 +49,7 @@ class RGWMetadataLogInfoCompletion : public RefCountedObject {
   explicit RGWMetadataLogInfoCompletion(info_callback_t callback);
   ~RGWMetadataLogInfoCompletion() override;
 
-  RGWSI_RADOS::Obj& get_io_obj() { return io_obj; }
+  rgw_rados_ref& get_io_obj() { return io_obj; }
   cls_log_header& get_header() { return header; }
   librados::AioCompletion* get_completion() { return completion; }
 
diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc
index 44788940330a..101108981c32 100644
--- a/src/rgw/services/svc_cls.cc
+++ b/src/rgw/services/svc_cls.cc
@@ -30,27 +30,12 @@ int RGWSI_Cls::do_start(optional_yield y, const DoutPrefixProvider *dpp)
   return 0;
 }
 
-int RGWSI_Cls::MFA::get_mfa_obj(const DoutPrefixProvider *dpp, const rgw_user& user, std::optional<RGWSI_RADOS::Obj> *obj)
-{
-  string oid = get_mfa_oid(user);
-  rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid);
-
-  obj->emplace(rados_svc->obj(o));
-  int r = (*obj)->open(dpp);
-  if (r < 0) {
-    ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl;
-    return r;
-  }
-
-  return 0;
-}
-
 int RGWSI_Cls::MFA::get_mfa_ref(const DoutPrefixProvider *dpp, const rgw_user& user, rgw_rados_ref *ref)
 {
   string oid = get_mfa_oid(user);
-  rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid);
+  rgw_raw_obj o(cls->zone_svc->get_zone_params().otp_pool, oid);
 
-  auto r = rgw_get_rados_ref(dpp, rados_svc->get_rados_handle(), o, ref);
+  auto r = rgw_get_rados_ref(dpp, cls->rados, o, ref);
   if (r < 0) {
     ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl;
     return r;
@@ -105,8 +90,8 @@ void RGWSI_Cls::MFA::prepare_mfa_write(librados::ObjectWriteOperation *op,
 int RGWSI_Cls::MFA::create_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const rados::cls::otp::otp_info_t& config,
                          RGWObjVersionTracker *objv_tracker, const ceph::real_time& mtime, optional_yield y)
 {
-  std::optional<RGWSI_RADOS::Obj> obj;
-  int r = get_mfa_obj(dpp, user, &obj);
+  rgw_rados_ref obj;
+  int r = get_mfa_ref(dpp, user, &obj);
   if (r < 0) {
     return r;
   }
@@ -114,7 +99,7 @@ int RGWSI_Cls::MFA::create_mfa(const DoutPrefixProvider *dpp, const rgw_user& us
   librados::ObjectWriteOperation op;
   prepare_mfa_write(&op, objv_tracker, mtime);
   rados::cls::otp::OTP::create(&op, config);
-  r = obj->operate(dpp, &op, y);
+  r = obj.operate(dpp, &op, y);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "OTP create, otp_id=" << config.id << " result=" << (int)r << dendl;
     return r;
@@ -129,8 +114,8 @@ int RGWSI_Cls::MFA::remove_mfa(const DoutPrefixProvider *dpp,
                          const ceph::real_time& mtime,
                          optional_yield y)
 {
-  std::optional<RGWSI_RADOS::Obj> obj;
-  int r = get_mfa_obj(dpp, user, &obj);
+  rgw_rados_ref obj;
+  int r = get_mfa_ref(dpp, user, &obj);
   if (r < 0) {
     return r;
   }
@@ -138,7 +123,7 @@ int RGWSI_Cls::MFA::remove_mfa(const DoutPrefixProvider *dpp,
   librados::ObjectWriteOperation op;
   prepare_mfa_write(&op, objv_tracker, mtime);
   rados::cls::otp::OTP::remove(&op, id);
-  r = obj->operate(dpp, &op, y);
+  r = obj.operate(dpp, &op, y);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "OTP remove, otp_id=" << id << " result=" << (int)r << dendl;
     return r;
@@ -206,11 +191,12 @@ int RGWSI_Cls::MFA::set_mfa(const DoutPrefixProvider *dpp, const string& oid, co
 			    const real_time& mtime,
 			    optional_yield y)
 {
-  rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid);
-  auto obj = rados_svc->obj(o);
-  int r = obj.open(dpp);
+  rgw_rados_ref obj;
+  int r = rgw_get_rados_ref(dpp, cls->rados,
+			    { cls->zone_svc->get_zone_params().otp_pool, oid },
+			    &obj);
   if (r < 0) {
-    ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl;
+    ldpp_dout(dpp, 4) << "failed to open rados context for " << oid << dendl;
     return r;
   }
   librados::ObjectWriteOperation op;
@@ -234,21 +220,21 @@ int RGWSI_Cls::MFA::list_mfa(const DoutPrefixProvider *dpp, const string& oid, l
 			     RGWObjVersionTracker *objv_tracker, ceph::real_time *pmtime,
 			     optional_yield y)
 {
-  rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid);
-  auto obj = rados_svc->obj(o);
-  int r = obj.open(dpp);
+  rgw_rados_ref ref;
+  int r = rgw_get_rados_ref(dpp, cls->rados,
+			    { cls->zone_svc->get_zone_params().otp_pool, oid },
+			    &ref);
   if (r < 0) {
-    ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl;
+    ldpp_dout(dpp, 4) << "failed to open rados context for " << oid << dendl;
     return r;
   }
-  auto& ref = obj.get_ref();
   librados::ObjectReadOperation op;
   struct timespec mtime_ts;
   if (pmtime) {
     op.stat2(nullptr, &mtime_ts, nullptr);
   }
   objv_tracker->prepare_op_for_read(&op);
-  r = rados::cls::otp::OTP::get_all(&op, ref.pool.ioctx(), ref.obj.oid, result);
+  r = rados::cls::otp::OTP::get_all(&op, ref.ioctx, ref.obj.oid, result);
   if (r < 0) {
     return r;
   }
@@ -268,14 +254,13 @@ void RGWSI_Cls::TimeLog::prepare_entry(cls_log_entry& entry,
   cls_log_add_prepare_entry(entry, utime_t(ut), section, key, bl);
 }
 
-int RGWSI_Cls::TimeLog::init_obj(const DoutPrefixProvider *dpp, const string& oid, RGWSI_RADOS::Obj& obj)
+int RGWSI_Cls::TimeLog::init_obj(const DoutPrefixProvider *dpp, const string& oid, rgw_rados_ref& obj)
 {
-  rgw_raw_obj o(zone_svc->get_zone_params().log_pool, oid);
-  obj = rados_svc->obj(o);
-  return obj.open(dpp);
-
+  rgw_raw_obj o(cls->zone_svc->get_zone_params().log_pool, oid);
+  return rgw_get_rados_ref(dpp, cls->rados, o, &obj);
 }
-int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp, 
+
+int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp,
                             const string& oid,
                             const real_time& ut,
                             const string& section,
@@ -283,8 +268,7 @@ int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp,
                             bufferlist& bl,
 			    optional_yield y)
 {
-  RGWSI_RADOS::Obj obj;
-
+  rgw_rados_ref obj;
   int r = init_obj(dpp, oid, obj);
   if (r < 0) {
     return r;
@@ -304,7 +288,7 @@ int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp,
                             bool monotonic_inc,
                             optional_yield y)
 {
-  RGWSI_RADOS::Obj obj;
+  rgw_rados_ref obj;
 
   int r = init_obj(dpp, oid, obj);
   if (r < 0) {
@@ -332,7 +316,7 @@ int RGWSI_Cls::TimeLog::list(const DoutPrefixProvider *dpp,
                              bool *truncated,
                              optional_yield y)
 {
-  RGWSI_RADOS::Obj obj;
+  rgw_rados_ref obj;
 
   int r = init_obj(dpp, oid, obj);
   if (r < 0) {
@@ -361,7 +345,7 @@ int RGWSI_Cls::TimeLog::info(const DoutPrefixProvider *dpp,
                              cls_log_header *header,
                              optional_yield y)
 {
-  RGWSI_RADOS::Obj obj;
+  rgw_rados_ref obj;
 
   int r = init_obj(dpp, oid, obj);
   if (r < 0) {
@@ -382,7 +366,7 @@ int RGWSI_Cls::TimeLog::info(const DoutPrefixProvider *dpp,
 }
 
 int RGWSI_Cls::TimeLog::info_async(const DoutPrefixProvider *dpp,
-                                   RGWSI_RADOS::Obj& obj,
+                                   rgw_rados_ref& obj,
                                    const string& oid,
                                    cls_log_header *header,
                                    librados::AioCompletion *completion)
@@ -412,7 +396,7 @@ int RGWSI_Cls::TimeLog::trim(const DoutPrefixProvider *dpp,
                              librados::AioCompletion *completion,
                              optional_yield y)
 {
-  RGWSI_RADOS::Obj obj;
+  rgw_rados_ref obj;
 
   int r = init_obj(dpp, oid, obj);
   if (r < 0) {
@@ -441,22 +425,23 @@ int RGWSI_Cls::Lock::lock_exclusive(const DoutPrefixProvider *dpp,
                                     string& owner_id,
                                     std::optional<string> lock_name)
 {
-  auto p = rados_svc->pool(pool);
-  int r = p.open(dpp);
+
+  librados::IoCtx p;
+  int r = rgw_init_ioctx(dpp, cls->rados, pool, p, true, false);
   if (r < 0) {
     return r;
   }
 
   uint64_t msec = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
   utime_t ut(msec / 1000, msec % 1000);
-  
+
   rados::cls::lock::Lock l(lock_name.value_or(log_lock_name));
   l.set_duration(ut);
   l.set_cookie(owner_id);
   l.set_tag(zone_id);
   l.set_may_renew(true);
-  
-  return l.lock_exclusive(&p.ioctx(), oid);
+
+  return l.lock_exclusive(&p, oid);
 }
 
 int RGWSI_Cls::Lock::unlock(const DoutPrefixProvider *dpp,
@@ -466,16 +451,15 @@ int RGWSI_Cls::Lock::unlock(const DoutPrefixProvider *dpp,
                             string& owner_id,
                             std::optional<string> lock_name)
 {
-  auto p = rados_svc->pool(pool);
-  int r = p.open(dpp);
+  librados::IoCtx p;
+  int r = rgw_init_ioctx(dpp, cls->rados, pool, p, true, false);
   if (r < 0) {
     return r;
   }
-  
+
   rados::cls::lock::Lock l(lock_name.value_or(log_lock_name));
   l.set_tag(zone_id);
   l.set_cookie(owner_id);
-  
-  return l.unlock(&p.ioctx(), oid);
-}
 
+  return l.unlock(&p, oid);
+}
diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h
index 08d7cfffec6f..c5ee361fc56d 100644
--- a/src/rgw/services/svc_cls.h
+++ b/src/rgw/services/svc_cls.h
@@ -29,19 +29,15 @@
 class RGWSI_Cls : public RGWServiceInstance
 {
   RGWSI_Zone *zone_svc{nullptr};
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados* rados{nullptr};
 
   class ClsSubService : public RGWServiceInstance {
     friend class RGWSI_Cls;
 
-    RGWSI_Cls *cls_svc{nullptr};
-    RGWSI_Zone *zone_svc{nullptr};
-    RGWSI_RADOS *rados_svc{nullptr};
+    RGWSI_Cls *cls{nullptr};
 
-    void init(RGWSI_Cls *_cls_svc, RGWSI_Zone *_zone_svc, RGWSI_RADOS *_rados_svc) {
-      cls_svc = _cls_svc;
-      zone_svc = _cls_svc->zone_svc;
-      rados_svc = _cls_svc->rados_svc;
+    void init(RGWSI_Cls *cls_) {
+      cls = cls_;
     }
 
   public:
@@ -50,7 +46,6 @@ class RGWSI_Cls : public RGWServiceInstance
 
 public:
   class MFA : public ClsSubService {
-    int get_mfa_obj(const DoutPrefixProvider *dpp, const rgw_user& user, std::optional<RGWSI_RADOS::Obj> *obj);
     int get_mfa_ref(const DoutPrefixProvider *dpp, const rgw_user& user, rgw_rados_ref *ref);
 
     void prepare_mfa_write(librados::ObjectWriteOperation *op,
@@ -83,7 +78,7 @@ class RGWSI_Cls : public RGWServiceInstance
   } mfa;
 
   class TimeLog : public ClsSubService {
-    int init_obj(const DoutPrefixProvider *dpp, const std::string& oid, RGWSI_RADOS::Obj& obj);
+    int init_obj(const DoutPrefixProvider *dpp, const std::string& oid, rgw_rados_ref& obj);
   public:
     TimeLog(CephContext *cct): ClsSubService(cct) {}
 
@@ -119,7 +114,7 @@ class RGWSI_Cls : public RGWServiceInstance
              cls_log_header *header,
              optional_yield y);
     int info_async(const DoutPrefixProvider *dpp,
-                   RGWSI_RADOS::Obj& obj,
+                   rgw_rados_ref& obj,
                    const std::string& oid,
                    cls_log_header *header,
                    librados::AioCompletion *completion);
@@ -134,7 +129,7 @@ class RGWSI_Cls : public RGWServiceInstance
   } timelog;
 
   class Lock : public ClsSubService {
-    int init_obj(const std::string& oid, RGWSI_RADOS::Obj& obj);
+    int init_obj(const std::string& oid, rgw_rados_ref& obj);
     public:
     Lock(CephContext *cct): ClsSubService(cct) {}
     int lock_exclusive(const DoutPrefixProvider *dpp,
@@ -154,15 +149,14 @@ class RGWSI_Cls : public RGWServiceInstance
 
   RGWSI_Cls(CephContext *cct): RGWServiceInstance(cct), mfa(cct), timelog(cct), lock(cct) {}
 
-  void init(RGWSI_Zone *_zone_svc, RGWSI_RADOS *_rados_svc) {
-    rados_svc = _rados_svc;
+  void init(RGWSI_Zone *_zone_svc, librados::Rados* rados_) {
+    rados = rados_;
     zone_svc = _zone_svc;
 
-    mfa.init(this, zone_svc, rados_svc);
-    timelog.init(this, zone_svc, rados_svc);
-    lock.init(this, zone_svc, rados_svc);
+    mfa.init(this);
+    timelog.init(this);
+    lock.init(this);
   }
 
   int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
 };
-

From e9c20c81236c10892138cc098fe636eb533cd211 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 01:14:02 -0500
Subject: [PATCH 0770/2492] rgw: Remove `RGWSI_RADOS` from
 `RGWSI_ConfigKey_RADOS`

Simply use the RADOS handle directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc      |  2 +-
 src/rgw/driver/rados/rgw_tools.cc        | 44 ++++++++++++++++++++++++
 src/rgw/driver/rados/rgw_tools.h         |  3 ++
 src/rgw/services/svc_config_key_rados.cc | 21 ++++++-----
 src/rgw/services/svc_config_key_rados.h  | 14 ++------
 5 files changed, 64 insertions(+), 20 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index cec7cf22d985..9dfa5465dfa9 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -96,7 +96,7 @@ int RGWServices_Def::init(CephContext *cct,
                          sysobj_cache.get(),
                          bucket_sobj.get());
   cls->init(zone.get(), radoshandle);
-  config_key_rados->init(rados.get());
+  config_key_rados->init(radoshandle);
   mdlog->init(rados.get(), zone.get(), sysobj.get(), cls.get());
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
   meta_be_sobj->init(sysobj.get(), mdlog.get());
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index ec373ca6095d..090c80aa1f71 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -1,6 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#include "auth/AuthRegistry.h"
+
 #include "common/errno.h"
 #include "librados/librados_asio.h"
 
@@ -453,3 +455,45 @@ void rgw_complete_aio_completion(librados::AioCompletion* c, int r) {
   librados::CB_AioCompleteAndSafe cb(pc);
   cb(r);
 }
+
+bool rgw_check_secure_mon_conn(const DoutPrefixProvider *dpp)
+{
+  AuthRegistry reg(dpp->get_cct());
+
+  reg.refresh_config();
+
+  std::vector<uint32_t> methods;
+  std::vector<uint32_t> modes;
+
+  reg.get_supported_methods(CEPH_ENTITY_TYPE_MON, &methods, &modes);
+  ldpp_dout(dpp, 20) << __func__ << "(): auth registy supported: methods=" << methods << " modes=" << modes << dendl;
+
+  for (auto method : methods) {
+    if (!reg.is_secure_method(method)) {
+      ldpp_dout(dpp, 20) << __func__ << "(): method " << method << " is insecure" << dendl;
+      return false;
+    }
+  }
+
+  for (auto mode : modes) {
+    if (!reg.is_secure_mode(mode)) {
+      ldpp_dout(dpp, 20) << __func__ << "(): mode " << mode << " is insecure" << dendl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+int rgw_clog_warn(librados::Rados* h, const string& msg)
+{
+  string cmd =
+    "{"
+      "\"prefix\": \"log\", "
+      "\"level\": \"warn\", "
+      "\"logtext\": [\"" + msg + "\"]"
+    "}";
+
+  bufferlist inbl;
+  return h->mon_command(cmd, inbl, nullptr, nullptr);
+}
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index afc836a473a4..7fdb3ea3effb 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -311,3 +311,6 @@ void rgw_complete_aio_completion(librados::AioCompletion* c, int r);
 // (Currently providing nullptr will wipe all attributes.)
 
 std::map<std::string, ceph::buffer::list>* no_change_attrs();
+
+bool rgw_check_secure_mon_conn(const DoutPrefixProvider *dpp);
+int rgw_clog_warn(librados::Rados* h, const std::string& msg);
diff --git a/src/rgw/services/svc_config_key_rados.cc b/src/rgw/services/svc_config_key_rados.cc
index 5edb02ea7f34..c17139af2925 100644
--- a/src/rgw/services/svc_config_key_rados.cc
+++ b/src/rgw/services/svc_config_key_rados.cc
@@ -1,14 +1,17 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
 
-#include "svc_rados.h"
 #include "svc_config_key_rados.h"
 
-using namespace std;
+#include "rgw_tools.h"
+
+using std::string;
 
 RGWSI_ConfigKey_RADOS::~RGWSI_ConfigKey_RADOS(){}
 
 int RGWSI_ConfigKey_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp)
 {
-  maybe_insecure_mon_conn = !svc.rados->check_secure_mon_conn(dpp);
+  maybe_insecure_mon_conn = !rgw_check_secure_mon_conn(dpp);
 
   return 0;
 }
@@ -20,14 +23,17 @@ void RGWSI_ConfigKey_RADOS::warn_if_insecure()
     return;
   }
 
-  string s = "rgw is configured to optionally allow insecure connections to the monitors (auth_supported, ms_mon_client_mode), ssl certificates stored at the monitor configuration could leak";
+  string s = ("rgw is configured to optionally allow insecure connections to "
+	      "the monitors (auth_supported, ms_mon_client_mode), ssl "
+	      "certificates stored at the monitor configuration could leak");
 
-  svc.rados->clog_warn(s);
+  rgw_clog_warn(rados, s);
 
   lderr(ctx()) << __func__ << "(): WARNING: " << s << dendl;
 }
 
-int RGWSI_ConfigKey_RADOS::get(const string& key, bool secure, bufferlist *result)
+int RGWSI_ConfigKey_RADOS::get(const string& key, bool secure,
+			       bufferlist *result)
 {
   string cmd =
     "{"
@@ -36,8 +42,7 @@ int RGWSI_ConfigKey_RADOS::get(const string& key, bool secure, bufferlist *resul
     "}";
 
   bufferlist inbl;
-  auto handle = svc.rados->handle();
-  int ret = handle.mon_command(cmd, inbl, result, nullptr);
+  int ret = rados->mon_command(cmd, inbl, result, nullptr);
   if (ret < 0) {
     return ret;
   }
diff --git a/src/rgw/services/svc_config_key_rados.h b/src/rgw/services/svc_config_key_rados.h
index b3b995ac76de..344350278af9 100644
--- a/src/rgw/services/svc_config_key_rados.h
+++ b/src/rgw/services/svc_config_key_rados.h
@@ -1,5 +1,3 @@
-
-
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
@@ -24,8 +22,6 @@
 
 #include "svc_config_key.h"
 
-class RGWSI_RADOS;
-
 class RGWSI_ConfigKey_RADOS : public RGWSI_ConfigKey
 {
   bool maybe_insecure_mon_conn{false};
@@ -36,12 +32,10 @@ class RGWSI_ConfigKey_RADOS : public RGWSI_ConfigKey
   void warn_if_insecure();
 
 public:
-  struct Svc {
-    RGWSI_RADOS *rados{nullptr};
-  } svc;
+  librados::Rados* rados{nullptr};
 
-  void init(RGWSI_RADOS *rados_svc) {
-    svc.rados = rados_svc;
+  void init(librados::Rados* rados_) {
+    rados = rados_;
   }
 
   RGWSI_ConfigKey_RADOS(CephContext *cct) : RGWSI_ConfigKey(cct) {}
@@ -50,5 +44,3 @@ class RGWSI_ConfigKey_RADOS : public RGWSI_ConfigKey
 
   int get(const std::string& key, bool secure, bufferlist *result) override;
 };
-
-

From a493b0710a50786ef0374df52831774fe6ca8d51 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 01:55:47 -0500
Subject: [PATCH 0771/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_MDLog`

Simply use the RADOS handle and `rgw_rados_ref` directly.

Also move `async_processor` out from `RGWSI_RADOS` and into
`RGWServices_Def`. This is as good a place as it for any, for now, as
it's reachable by everyone who needs it and exists through the
lifetime of the process.

Eventually it's going to go away due to coroutinization, anyway.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_data_sync.cc    |  4 +--
 src/rgw/driver/rados/rgw_period.cc       |  2 +-
 src/rgw/driver/rados/rgw_rados.cc        |  6 ++---
 src/rgw/driver/rados/rgw_service.cc      | 10 +++++--
 src/rgw/driver/rados/rgw_service.h       |  3 +++
 src/rgw/driver/rados/rgw_trim_bilog.cc   | 12 ++++-----
 src/rgw/driver/rados/rgw_trim_datalog.cc |  2 +-
 src/rgw/driver/rados/rgw_trim_mdlog.cc   |  6 ++---
 src/rgw/rgw_admin.cc                     | 16 ++++++------
 src/rgw/services/svc_mdlog.cc            | 33 +++++++++++++++---------
 src/rgw/services/svc_mdlog.h             | 10 ++++---
 src/rgw/services/svc_rados.cc            | 13 ----------
 src/rgw/services/svc_rados.h             |  6 -----
 13 files changed, 62 insertions(+), 61 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index 02965f9073d6..7b8c0b7343d9 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -6068,7 +6068,7 @@ int RGWBucketPipeSyncStatusManager::do_init(const DoutPrefixProvider *dpp,
   }
 
   sync_module.reset(new RGWDefaultSyncModuleInstance());
-  auto async_rados = driver->svc()->rados->get_async_processor();
+  auto async_rados = driver->svc()->async_processor;
 
   sync_env.init(this, driver->ctx(), driver,
                 driver->svc(), async_rados, &http_manager,
@@ -6680,7 +6680,7 @@ int rgw_read_bucket_inc_sync_status(const DoutPrefixProvider *dpp,
 
   RGWDataSyncEnv env;
   RGWSyncModuleInstanceRef module; // null sync module
-  env.init(dpp, driver->ctx(), driver, driver->svc(), driver->svc()->rados->get_async_processor(),
+  env.init(dpp, driver->ctx(), driver, driver->svc(), driver->svc()->async_processor,
            nullptr, nullptr, nullptr, module, nullptr);
 
   RGWDataSyncCtx sc;
diff --git a/src/rgw/driver/rados/rgw_period.cc b/src/rgw/driver/rados/rgw_period.cc
index 61602b354e28..4a16faccefb3 100644
--- a/src/rgw/driver/rados/rgw_period.cc
+++ b/src/rgw/driver/rados/rgw_period.cc
@@ -154,7 +154,7 @@ static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Driver* dri
 {
   rgw::sal::RadosStore* rados_store = static_cast<rgw::sal::RadosStore*>(driver);
   // initialize a sync status manager to read the status
-  RGWMetaSyncStatusManager mgr(rados_store, rados_store->svc()->rados->get_async_processor());
+  RGWMetaSyncStatusManager mgr(rados_store, rados_store->svc()->async_processor);
   int r = mgr.init(dpp);
   if (r < 0) {
     return r;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 9cf0831ecaa9..5fd174c3aff0 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1021,7 +1021,7 @@ void RGWRados::finalize()
   /* Before joining any sync threads, drain outstanding requests &
    * mark the async_processor as going_down() */
   if (svc.rados) {
-    svc.rados->stop_processor();
+    svc.async_processor->stop();
   }
 
   if (run_sync_thread) {
@@ -1258,7 +1258,7 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
                       << pt.second.name << " present in zonegroup" << dendl;
       }
     }
-    auto async_processor = svc.rados->get_async_processor();
+    auto async_processor = svc.async_processor;
     std::lock_guard l{meta_sync_thread_lock};
     meta_sync_processor_thread = new RGWMetaSyncProcessorThread(this->driver, async_processor);
     ret = meta_sync_processor_thread->init(dpp);
@@ -1283,7 +1283,7 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
     std::lock_guard dl{data_sync_thread_lock};
     for (auto source_zone : svc.zone->get_data_sync_source_zones()) {
       ldpp_dout(dpp, 5) << "starting data sync thread for zone " << source_zone->name << dendl;
-      auto *thread = new RGWDataSyncProcessorThread(this->driver, svc.rados->get_async_processor(), source_zone);
+      auto *thread = new RGWDataSyncProcessorThread(this->driver, svc.async_processor, source_zone);
       ret = thread->init(dpp);
       if (ret < 0) {
         ldpp_dout(dpp, 0) << "ERROR: failed to initialize data sync thread" << dendl;
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 9dfa5465dfa9..af4043efa7af 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -31,6 +31,7 @@
 #include "common/errno.h"
 
 #include "rgw_bucket.h"
+#include "rgw_cr_rados.h"
 #include "rgw_datalog.h"
 #include "rgw_metadata.h"
 #include "rgw_otp.h"
@@ -78,6 +79,8 @@ int RGWServices_Def::init(CephContext *cct,
   sysobj_core = std::make_unique<RGWSI_SysObj_Core>(cct);
   user_rados = std::make_unique<RGWSI_User_RADOS>(cct);
   role_rados = std::make_unique<RGWSI_Role_RADOS>(cct);
+  async_processor = std::make_unique<RGWAsyncRadosProcessor>(
+    cct, cct->_conf->rgw_num_async_rados_threads);
 
   if (have_cache) {
     sysobj_cache = std::make_unique<RGWSI_SysObj_Cache>(dpp, cct);
@@ -85,6 +88,7 @@ int RGWServices_Def::init(CephContext *cct,
 
   vector<RGWSI_MetaBackend *> meta_bes{meta_be_sobj.get(), meta_be_otp.get()};
 
+  async_processor->start();
   finisher->init();
   bi_rados->init(zone.get(), radoshandle, bilog_rados.get(), datalog_rados.get());
   bilog_rados->init(bi_rados.get());
@@ -97,7 +101,8 @@ int RGWServices_Def::init(CephContext *cct,
                          bucket_sobj.get());
   cls->init(zone.get(), radoshandle);
   config_key_rados->init(radoshandle);
-  mdlog->init(rados.get(), zone.get(), sysobj.get(), cls.get());
+  mdlog->init(radoshandle, zone.get(), sysobj.get(), cls.get(),
+	      async_processor.get());
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
   meta_be_sobj->init(sysobj.get(), mdlog.get());
   meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get());
@@ -300,10 +305,10 @@ void RGWServices_Def::shutdown()
   quota->shutdown();
   zone_utils->shutdown();
   zone->shutdown();
+  async_processor->stop();
   rados->shutdown();
 
   has_shutdown = true;
-
 }
 
 int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
@@ -345,6 +350,7 @@ int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
   core = _svc.sysobj_core.get();
   user = _svc.user_rados.get();
   role = _svc.role_rados.get();
+  async_processor = _svc.async_processor.get();
 
   return 0;
 }
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 46ee16417feb..ad80a6c88266 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -74,6 +74,7 @@ class RGWSI_User;
 class RGWSI_User_RADOS;
 class RGWDataChangesLog;
 class RGWSI_Role_RADOS;
+class RGWAsyncRadosProcessor;
 
 struct RGWServices_Def
 {
@@ -104,6 +105,7 @@ struct RGWServices_Def
   std::unique_ptr<RGWSI_User_RADOS> user_rados;
   std::unique_ptr<RGWDataChangesLog> datalog_rados;
   std::unique_ptr<RGWSI_Role_RADOS> role_rados;
+  std::unique_ptr<RGWAsyncRadosProcessor> async_processor;
 
   RGWServices_Def();
   ~RGWServices_Def();
@@ -149,6 +151,7 @@ struct RGWServices
   RGWSI_SysObj_Core *core{nullptr};
   RGWSI_User *user{nullptr};
   RGWSI_Role_RADOS *role{nullptr};
+  RGWAsyncRadosProcessor* async_processor;
 
   int do_init(CephContext *cct, bool have_cache, bool raw_storage,
 	      bool run_sync, librados::Rados* radoshandle, optional_yield y,
diff --git a/src/rgw/driver/rados/rgw_trim_bilog.cc b/src/rgw/driver/rados/rgw_trim_bilog.cc
index be6d990d962b..db9980470055 100644
--- a/src/rgw/driver/rados/rgw_trim_bilog.cc
+++ b/src/rgw/driver/rados/rgw_trim_bilog.cc
@@ -617,7 +617,7 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
 
     get_policy_params.zone = zone_id;
     get_policy_params.bucket = bucket;
-    yield call(new RGWBucketGetSyncPolicyHandlerCR(store->svc()->rados->get_async_processor(),
+    yield call(new RGWBucketGetSyncPolicyHandlerCR(store->svc()->async_processor,
                                                    store,
                                                    get_policy_params,
                                                    source_policy,
@@ -728,14 +728,14 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
       }
       while (clean_info && retries < MAX_RETRIES) {
 	yield call(new RGWPutBucketInstanceInfoCR(
-		     store->svc()->rados->get_async_processor(),
+		     store->svc()->async_processor,
 		     store, clean_info->first, false, {},
 		     no_change_attrs(), dpp));
 
 	// Raced, try again.
 	if (retcode == -ECANCELED) {
 	  yield call(new RGWGetBucketInstanceInfoCR(
-		       store->svc()->rados->get_async_processor(),
+		       store->svc()->async_processor,
 		       store, clean_info->first.bucket,
 		       &(clean_info->first), nullptr, dpp));
 	  if (retcode < 0) {
@@ -1132,7 +1132,7 @@ int BucketTrimCR::operate(const DoutPrefixProvider *dpp)
           return buckets.size() < config.buckets_per_interval;
         };
 
-        call(new MetadataListCR(cct, store->svc()->rados->get_async_processor(),
+        call(new MetadataListCR(cct, store->svc()->async_processor,
                                 store->ctl()->meta.mgr,
                                 section, status.marker, cb));
       }
@@ -1219,7 +1219,7 @@ int BucketTrimPollCR::operate(const DoutPrefixProvider *dpp)
 
       // prevent others from trimming for our entire wait interval
       set_status("acquiring trim lock");
-      yield call(new RGWSimpleRadosLockCR(store->svc()->rados->get_async_processor(), store,
+      yield call(new RGWSimpleRadosLockCR(store->svc()->async_processor, store,
                                           obj, name, cookie,
                                           config.trim_interval_sec));
       if (retcode < 0) {
@@ -1232,7 +1232,7 @@ int BucketTrimPollCR::operate(const DoutPrefixProvider *dpp)
       if (retcode < 0) {
         // on errors, unlock so other gateways can try
         set_status("unlocking");
-        yield call(new RGWSimpleRadosUnlockCR(store->svc()->rados->get_async_processor(), store,
+        yield call(new RGWSimpleRadosUnlockCR(store->svc()->async_processor, store,
                                               obj, name, cookie));
       }
     }
diff --git a/src/rgw/driver/rados/rgw_trim_datalog.cc b/src/rgw/driver/rados/rgw_trim_datalog.cc
index bac0cda8dd64..5dcddb659e1e 100644
--- a/src/rgw/driver/rados/rgw_trim_datalog.cc
+++ b/src/rgw/driver/rados/rgw_trim_datalog.cc
@@ -224,7 +224,7 @@ int DataLogTrimPollCR::operate(const DoutPrefixProvider *dpp)
       // prevent other gateways from attempting to trim for the duration
       set_status("acquiring trim lock");
 
-      yield call(new RGWSimpleRadosLockCR(store->svc()->rados->get_async_processor(), store,
+      yield call(new RGWSimpleRadosLockCR(store->svc()->async_processor, store,
                                           rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, lock_oid),
                                           "data_trim", lock_cookie,
                                           // interval is a small number and unlikely to overflow
diff --git a/src/rgw/driver/rados/rgw_trim_mdlog.cc b/src/rgw/driver/rados/rgw_trim_mdlog.cc
index 13773ae2877f..be513b6e74df 100644
--- a/src/rgw/driver/rados/rgw_trim_mdlog.cc
+++ b/src/rgw/driver/rados/rgw_trim_mdlog.cc
@@ -565,7 +565,7 @@ class MetaPeerTrimShardCollectCR : public RGWShardCollectCR {
       env(env), mdlog(mdlog), period_id(env.current.get_period().get_id())
   {
     meta_env.init(env.dpp, cct, env.store, env.store->svc()->zone->get_master_conn(),
-                  env.store->svc()->rados->get_async_processor(), env.http, nullptr,
+                  env.store->svc()->async_processor, env.http, nullptr,
                   env.store->getRados()->get_sync_tracer());
   }
 
@@ -669,7 +669,7 @@ int MetaTrimPollCR::operate(const DoutPrefixProvider *dpp)
       // prevent others from trimming for our entire wait interval
       set_status("acquiring trim lock");
 
-      yield call(new RGWSimpleRadosLockCR(store->svc()->rados->get_async_processor(), store,
+      yield call(new RGWSimpleRadosLockCR(store->svc()->async_processor, store,
                                           obj, name, cookie, 
                                           // interval is a small number and unlikely to overflow
                                           // coverity[store_truncates_time_t:SUPPRESS]
@@ -685,7 +685,7 @@ int MetaTrimPollCR::operate(const DoutPrefixProvider *dpp)
       if (retcode < 0) {
         // on errors, unlock so other gateways can try
         set_status("unlocking");
-        yield call(new RGWSimpleRadosUnlockCR(store->svc()->rados->get_async_processor(), store,
+        yield call(new RGWSimpleRadosUnlockCR(store->svc()->async_processor, store,
                                               obj, name, cookie));
       }
     }
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index c19d44c9e536..f4a2d1480bda 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2098,7 +2098,7 @@ stringstream& push_ss(stringstream& ss, list<string>& l, int tab = 0)
 
 static void get_md_sync_status(list<string>& status)
 {
-  RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor());
+  RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor);
 
   int ret = sync.init(dpp());
   if (ret < 0) {
@@ -2254,7 +2254,7 @@ static void get_data_sync_status(const rgw_zone_id& source_zone, list<string>& s
     flush_ss(ss, status);
     return;
   }
-  RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor(), source_zone, nullptr);
+  RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor, source_zone, nullptr);
 
   int ret = sync.init(dpp());
   if (ret < 0) {
@@ -8995,7 +8995,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::METADATA_SYNC_STATUS) {
-    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor());
+    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor);
 
     int ret = sync.init(dpp());
     if (ret < 0) {
@@ -9039,7 +9039,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::METADATA_SYNC_INIT) {
-    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor());
+    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor);
 
     int ret = sync.init(dpp());
     if (ret < 0) {
@@ -9055,7 +9055,7 @@ int main(int argc, const char **argv)
 
 
   if (opt_cmd == OPT::METADATA_SYNC_RUN) {
-    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor());
+    RGWMetaSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor);
 
     int ret = sync.init(dpp());
     if (ret < 0) {
@@ -9075,7 +9075,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: source zone not specified" << std::endl;
       return EINVAL;
     }
-    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor(), source_zone, nullptr);
+    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor, source_zone, nullptr);
 
     int ret = sync.init(dpp());
     if (ret < 0) {
@@ -9145,7 +9145,7 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor(), source_zone, nullptr);
+    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor, source_zone, nullptr);
 
     int ret = sync.init(dpp());
     if (ret < 0) {
@@ -9174,7 +9174,7 @@ int main(int argc, const char **argv)
       return ret;
     }
 
-    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados->get_async_processor(), source_zone, nullptr, sync_module);
+    RGWDataSyncStatusManager sync(static_cast<rgw::sal::RadosStore*>(driver), static_cast<rgw::sal::RadosStore*>(driver)->svc()->async_processor, source_zone, nullptr, sync_module);
 
     ret = sync.init(dpp());
     if (ret < 0) {
diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc
index da723f3e1295..603718dc96e5 100644
--- a/src/rgw/services/svc_mdlog.cc
+++ b/src/rgw/services/svc_mdlog.cc
@@ -30,13 +30,16 @@ RGWSI_MDLog::RGWSI_MDLog(CephContext *cct, bool _run_sync) : RGWServiceInstance(
 RGWSI_MDLog::~RGWSI_MDLog() {
 }
 
-int RGWSI_MDLog::init(RGWSI_RADOS *_rados_svc, RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, RGWSI_Cls *_cls_svc)
+int RGWSI_MDLog::init(librados::Rados* rados_, RGWSI_Zone *_zone_svc,
+		      RGWSI_SysObj *_sysobj_svc, RGWSI_Cls *_cls_svc,
+		      RGWAsyncRadosProcessor* async_processor_)
 {
   svc.zone = _zone_svc;
   svc.sysobj = _sysobj_svc;
   svc.mdlog = this;
-  svc.rados = _rados_svc;
+  rados = rados_;
   svc.cls = _cls_svc;
+  async_processor = async_processor_;
 
   return 0;
 }
@@ -262,11 +265,12 @@ class ReadHistoryCR : public RGWCoroutine {
   ReadHistoryCR(const DoutPrefixProvider *dpp, 
                 const Svc& svc,
                 Cursor *cursor,
-                RGWObjVersionTracker *objv_tracker)
+                RGWObjVersionTracker *objv_tracker,
+		RGWAsyncRadosProcessor* async_processor)
     : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc),
       cursor(cursor),
       objv_tracker(objv_tracker),
-      async_processor(svc.rados->get_async_processor())
+      async_processor(async_processor)
   {}
 
   int operate(const DoutPrefixProvider *dpp) {
@@ -312,10 +316,11 @@ class WriteHistoryCR : public RGWCoroutine {
   WriteHistoryCR(const DoutPrefixProvider *dpp, 
                  Svc& svc,
                  const Cursor& cursor,
-                 RGWObjVersionTracker *objv)
+                 RGWObjVersionTracker *objv,
+		 RGWAsyncRadosProcessor* async_processor)
     : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc),
       cursor(cursor), objv(objv),
-      async_processor(svc.rados->get_async_processor())
+      async_processor(async_processor)
   {}
 
   int operate(const DoutPrefixProvider *dpp) {
@@ -353,18 +358,22 @@ class TrimHistoryCR : public RGWCoroutine {
   RGWObjVersionTracker *objv; //< to prevent racing updates
   Cursor next; //< target cursor for oldest log period
   Cursor existing; //< existing cursor read from disk
+  RGWAsyncRadosProcessor* async_processor;
 
  public:
-  TrimHistoryCR(const DoutPrefixProvider *dpp, const Svc& svc, Cursor cursor, RGWObjVersionTracker *objv)
+  TrimHistoryCR(const DoutPrefixProvider *dpp, const Svc& svc, Cursor cursor,
+		RGWObjVersionTracker *objv,
+		RGWAsyncRadosProcessor* async_processor)
     : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc),
-      cursor(cursor), objv(objv), next(cursor) {
+      cursor(cursor), objv(objv), next(cursor),
+      async_processor(async_processor) {
     next.next(); // advance past cursor
   }
 
   int operate(const DoutPrefixProvider *dpp) {
     reenter(this) {
       // read an existing history, and write the new history if it's newer
-      yield call(new ReadHistoryCR(dpp, svc, &existing, objv));
+      yield call(new ReadHistoryCR(dpp, svc, &existing, objv, async_processor));
       if (retcode < 0) {
         return set_cr_error(retcode);
       }
@@ -375,7 +384,7 @@ class TrimHistoryCR : public RGWCoroutine {
         return set_cr_error(-ECANCELED);
       }
       // overwrite with updated history
-      yield call(new WriteHistoryCR(dpp, svc, next, objv));
+      yield call(new WriteHistoryCR(dpp, svc, next, objv, async_processor));
       if (retcode < 0) {
         return set_cr_error(retcode);
       }
@@ -512,13 +521,13 @@ Cursor RGWSI_MDLog::read_oldest_log_period(optional_yield y, const DoutPrefixPro
 RGWCoroutine* RGWSI_MDLog::read_oldest_log_period_cr(const DoutPrefixProvider *dpp, 
         Cursor *period, RGWObjVersionTracker *objv) const
 {
-  return new mdlog::ReadHistoryCR(dpp, svc, period, objv);
+  return new mdlog::ReadHistoryCR(dpp, svc, period, objv, async_processor);
 }
 
 RGWCoroutine* RGWSI_MDLog::trim_log_period_cr(const DoutPrefixProvider *dpp, 
         Cursor period, RGWObjVersionTracker *objv) const
 {
-  return new mdlog::TrimHistoryCR(dpp, svc, period, objv);
+  return new mdlog::TrimHistoryCR(dpp, svc, period, objv, async_processor);
 }
 
 RGWMetadataLog* RGWSI_MDLog::get_log(const std::string& period)
diff --git a/src/rgw/services/svc_mdlog.h b/src/rgw/services/svc_mdlog.h
index 8b37ba11e565..f169ee88db9b 100644
--- a/src/rgw/services/svc_mdlog.h
+++ b/src/rgw/services/svc_mdlog.h
@@ -29,7 +29,6 @@ class RGWCoroutine;
 
 class RGWSI_Zone;
 class RGWSI_SysObj;
-class RGWSI_RADOS;
 
 namespace mdlog {
   class ReadHistoryCR;
@@ -58,18 +57,21 @@ class RGWSI_MDLog : public RGWServiceInstance
   RGWSI_MDLog(CephContext *cct, bool run_sync);
   virtual ~RGWSI_MDLog();
 
+  librados::Rados* rados{nullptr};
+  RGWAsyncRadosProcessor* async_processor{nullptr};
+
   struct Svc {
-    RGWSI_RADOS *rados{nullptr};
     RGWSI_Zone *zone{nullptr};
     RGWSI_SysObj *sysobj{nullptr};
     RGWSI_MDLog *mdlog{nullptr};
     RGWSI_Cls *cls{nullptr};
   } svc;
 
-  int init(RGWSI_RADOS *_rados_svc,
+  int init(librados::Rados* rados_,
            RGWSI_Zone *_zone_svc,
            RGWSI_SysObj *_sysobj_svc,
-           RGWSI_Cls *_cls_svc);
+           RGWSI_Cls *_cls_svc,
+	   RGWAsyncRadosProcessor* async_processor_);
 
   int do_start(optional_yield y, const DoutPrefixProvider *dpp) override;
 
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index d682b87eb2c2..34de9857e19a 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -34,27 +34,14 @@ int RGWSI_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp)
     return ret;
   }
 
-  async_processor.reset(new RGWAsyncRadosProcessor(cct, cct->_conf->rgw_num_async_rados_threads));
-  async_processor->start();
-
   return 0;
 }
 
 void RGWSI_RADOS::shutdown()
 {
-  if (async_processor) {
-    async_processor->stop();
-  }
   rados.shutdown();
 }
 
-void RGWSI_RADOS::stop_processor()
-{
-  if (async_processor) {
-    async_processor->stop();
-  }
-}
-
 librados::Rados* RGWSI_RADOS::get_rados_handle()
 {
   return &rados;
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index 4e7cf672c7a7..2a3940a34d96 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -28,7 +28,6 @@ struct RGWAccessListFilterPrefix : public RGWAccessListFilter {
 class RGWSI_RADOS : public RGWServiceInstance
 {
   librados::Rados rados;
-  std::unique_ptr<RGWAsyncRadosProcessor> async_processor;
 
   int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
 
@@ -66,16 +65,11 @@ class RGWSI_RADOS : public RGWServiceInstance
 
   void init() {}
   void shutdown() override;
-  void stop_processor();
 
   std::string cluster_fsid();
   uint64_t instance_id();
   bool check_secure_mon_conn(const DoutPrefixProvider *dpp) const;
 
-  RGWAsyncRadosProcessor *get_async_processor() {
-    return async_processor.get();
-  }
-
   int clog_warn(const std::string& msg);
 
   class Handle;

From 897f4a9fce2ca4c037608244f50703157f3cec83 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 02:17:44 -0500
Subject: [PATCH 0772/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_Notify`

Simply use the RADOS handle directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc |  2 +-
 src/rgw/driver/rados/rgw_tools.h    | 33 +++++++++++++++++++++++------
 src/rgw/services/svc_notify.cc      | 29 ++++++++++++-------------
 src/rgw/services/svc_notify.h       | 18 +++++++++-------
 4 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index af4043efa7af..ddd4dfdcd74f 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -106,7 +106,7 @@ int RGWServices_Def::init(CephContext *cct,
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
   meta_be_sobj->init(sysobj.get(), mdlog.get());
   meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get());
-  notify->init(zone.get(), rados.get(), finisher.get());
+  notify->init(zone.get(), radoshandle, finisher.get());
   otp->init(zone.get(), meta.get(), meta_be_otp.get());
   rados->init();
   zone->init(sysobj.get(), rados.get(), sync_modules.get(), bucket_sync_sobj.get());
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 7fdb3ea3effb..a7639acbb24c 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -108,34 +108,53 @@ struct rgw_rados_ref {
   rgw_raw_obj obj;
 
 
-  int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op,
-	      bufferlist *pbl, optional_yield y, int flags = 0) {
+  int operate(const DoutPrefixProvider* dpp, librados::ObjectReadOperation* op,
+	      bufferlist* pbl, optional_yield y, int flags = 0) {
     return rgw_rados_operate(dpp, ioctx, obj.oid, op, pbl, y, flags);
   }
 
-  int operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op,
+  int operate(const DoutPrefixProvider* dpp, librados::ObjectWriteOperation* op,
 	      optional_yield y, int flags = 0) {
     return rgw_rados_operate(dpp, ioctx, obj.oid, op, y, flags);
   }
 
-  int aio_operate(librados::AioCompletion *c,
-		  librados::ObjectWriteOperation *op) {
+  int aio_operate(librados::AioCompletion* c,
+		  librados::ObjectWriteOperation* op) {
     return ioctx.aio_operate(obj.oid, c, op);
   }
 
-  int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
+  int aio_operate(librados::AioCompletion* c, librados::ObjectReadOperation* op,
 		  bufferlist *pbl) {
     return ioctx.aio_operate(obj.oid, c, op, pbl);
   }
 
-  int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms,
+  int watch(uint64_t* handle, librados::WatchCtx2* ctx) {
+    return ioctx.watch2(obj.oid, handle, ctx);
+  }
+
+  int aio_watch(librados::AioCompletion* c, uint64_t* handle,
+		librados::WatchCtx2 *ctx) {
+    return ioctx.aio_watch(obj.oid, c, handle, ctx);
+  }
+
+  int unwatch(uint64_t handle) {
+    return ioctx.unwatch2(handle);
+  }
+
+  int notify(const DoutPrefixProvider* dpp, bufferlist& bl, uint64_t timeout_ms,
 	     bufferlist* pbl, optional_yield y) {
     return rgw_rados_notify(dpp, ioctx, obj.oid, bl, timeout_ms, pbl, y);
   }
+
+  void notify_ack(uint64_t notify_id, uint64_t cookie, bufferlist& bl) {
+    ioctx.notify_ack(obj.oid, notify_id, cookie, bl);
+  }
 };
+
 inline std::ostream& operator <<(std::ostream& m, const rgw_rados_ref& ref) {
   return m << ref.obj;
 }
+
 int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
 		      rgw_raw_obj obj, rgw_rados_ref* ref);
 
diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc
index 43f84ed0a4f7..041354a376f7 100644
--- a/src/rgw/services/svc_notify.cc
+++ b/src/rgw/services/svc_notify.cc
@@ -29,7 +29,7 @@ class RGWWatcher : public DoutPrefixProvider , public librados::WatchCtx2 {
   CephContext *cct;
   RGWSI_Notify *svc;
   int index;
-  RGWSI_RADOS::Obj obj;
+  rgw_rados_ref obj;
   uint64_t watch_handle;
   int register_ret{0};
   bool unregister_done{false};
@@ -51,7 +51,8 @@ class RGWWatcher : public DoutPrefixProvider , public librados::WatchCtx2 {
   }
 
 public:
-  RGWWatcher(CephContext *_cct, RGWSI_Notify *s, int i, RGWSI_RADOS::Obj& o) : cct(_cct), svc(s), index(i), obj(o), watch_handle(0) {}
+  RGWWatcher(CephContext *_cct, RGWSI_Notify *s, int i, rgw_rados_ref& o)
+    : cct(_cct), svc(s), index(i), obj(o), watch_handle(0) {}
   void handle_notify(uint64_t notify_id,
 		     uint64_t cookie,
 		     uint64_t notifier_id,
@@ -174,7 +175,7 @@ string RGWSI_Notify::get_control_oid(int i)
 }
 
 // do not call pick_obj_control before init_watch
-RGWSI_RADOS::Obj RGWSI_Notify::pick_control_obj(const string& key)
+rgw_rados_ref RGWSI_Notify::pick_control_obj(const string& key)
 {
   uint32_t r = ceph_str_hash_linux(key.c_str(), key.size());
 
@@ -206,17 +207,17 @@ int RGWSI_Notify::init_watch(const DoutPrefixProvider *dpp, optional_yield y)
       notify_oid = notify_oid_prefix;
     }
 
-    notify_objs[i] = rados_svc->handle().obj({control_pool, notify_oid});
-    auto& notify_obj = notify_objs[i];
-
-    int r = notify_obj.open(dpp);
+    int r = rgw_get_rados_ref(dpp, rados, { control_pool, notify_oid },
+			      &notify_objs[i]);
     if (r < 0) {
       ldpp_dout(dpp, 0) << "ERROR: notify_obj.open() returned r=" << r << dendl;
       return r;
     }
+    auto& notify_obj = notify_objs[i];
 
     librados::ObjectWriteOperation op;
     op.create(false);
+
     r = notify_obj.operate(dpp, &op, y);
     if (r < 0 && r != -EEXIST) {
       ldpp_dout(dpp, 0) << "ERROR: notify_obj.operate() returned r=" << r << dendl;
@@ -270,10 +271,6 @@ int RGWSI_Notify::do_start(optional_yield y, const DoutPrefixProvider *dpp)
 
   assert(zone_svc->is_started()); /* otherwise there's an ordering problem */
 
-  r = rados_svc->start(y, dpp);
-  if (r < 0) {
-    return r;
-  }
   r = finisher_svc->start(y, dpp);
   if (r < 0) {
     return r;
@@ -315,14 +312,14 @@ void RGWSI_Notify::shutdown()
   finalized = true;
 }
 
-int RGWSI_Notify::unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle)
+int RGWSI_Notify::unwatch(rgw_rados_ref& obj, uint64_t watch_handle)
 {
   int r = obj.unwatch(watch_handle);
   if (r < 0) {
     ldout(cct, 0) << "ERROR: rados->unwatch2() returned r=" << r << dendl;
     return r;
   }
-  r = rados_svc->handle().watch_flush();
+  r = rados->watch_flush();
   if (r < 0) {
     ldout(cct, 0) << "ERROR: rados->watch_flush() returned r=" << r << dendl;
     return r;
@@ -392,9 +389,9 @@ int RGWSI_Notify::distribute(const DoutPrefixProvider *dpp, const string& key,
     which will lead to division by 0 in pick_obj_control (num_watchers is 0).
   */
   if (num_watchers > 0) {
-    RGWSI_RADOS::Obj notify_obj = pick_control_obj(key);
+    auto notify_obj = pick_control_obj(key);
 
-    ldpp_dout(dpp, 10) << "distributing notification oid=" << notify_obj.get_ref().obj
+    ldpp_dout(dpp, 10) << "distributing notification oid=" << notify_obj.obj
 		       << " cni=" << cni << dendl;
     return robust_notify(dpp, notify_obj, cni, y);
   }
@@ -443,7 +440,7 @@ static timeout_vector decode_timeouts(const bufferlist& bl)
 }
 
 int RGWSI_Notify::robust_notify(const DoutPrefixProvider *dpp,
-                                RGWSI_RADOS::Obj& notify_obj,
+                                rgw_rados_ref& notify_obj,
 				const RGWCacheNotifyInfo& cni,
                                 optional_yield y)
 {
diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h
index f7329136ece1..618853d02c65 100644
--- a/src/rgw/services/svc_notify.h
+++ b/src/rgw/services/svc_notify.h
@@ -7,6 +7,8 @@
 
 #include "svc_rados.h"
 
+#include "rgw_tools.h"
+
 
 class Context;
 
@@ -21,14 +23,14 @@ class RGWSI_Notify : public RGWServiceInstance
 {
   friend class RGWWatcher;
   friend class RGWSI_Notify_ShutdownCB;
-  friend class RGWServices_Def;
+  friend struct RGWServices_Def;
 
 public:
   class CB;
 
 private:
   RGWSI_Zone *zone_svc{nullptr};
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados *rados{nullptr};
   RGWSI_Finisher *finisher_svc{nullptr};
 
   ceph::shared_mutex watchers_lock = ceph::make_shared_mutex("watchers_lock");
@@ -37,7 +39,7 @@ class RGWSI_Notify : public RGWServiceInstance
   int num_watchers{0};
   RGWWatcher **watchers{nullptr};
   std::set<int> watchers_set;
-  std::vector<RGWSI_RADOS::Obj> notify_objs;
+  std::vector<rgw_rados_ref> notify_objs;
 
   bool enabled{false};
 
@@ -45,7 +47,7 @@ class RGWSI_Notify : public RGWServiceInstance
   uint64_t max_notify_retries = 10;
 
   std::string get_control_oid(int i);
-  RGWSI_RADOS::Obj pick_control_obj(const std::string& key);
+  rgw_rados_ref pick_control_obj(const std::string& key);
 
   CB *cb{nullptr};
 
@@ -58,16 +60,16 @@ class RGWSI_Notify : public RGWServiceInstance
   void finalize_watch();
 
   void init(RGWSI_Zone *_zone_svc,
-            RGWSI_RADOS *_rados_svc,
+            librados::Rados* rados_,
             RGWSI_Finisher *_finisher_svc) {
     zone_svc = _zone_svc;
-    rados_svc = _rados_svc;
+    rados = rados_;
     finisher_svc = _finisher_svc;
   }
   int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
   void shutdown() override;
 
-  int unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle);
+  int unwatch(rgw_rados_ref& obj, uint64_t watch_handle);
   void add_watcher(int i);
   void remove_watcher(int i);
 
@@ -79,7 +81,7 @@ class RGWSI_Notify : public RGWServiceInstance
   void _set_enabled(bool status);
   void set_enabled(bool status);
 
-  int robust_notify(const DoutPrefixProvider *dpp, RGWSI_RADOS::Obj& notify_obj,
+  int robust_notify(const DoutPrefixProvider *dpp, rgw_rados_ref& notify_obj,
 		    const RGWCacheNotifyInfo& bl, optional_yield y);
 
   void schedule_context(Context *c);

From 5294d668d5c9961fc38fd6d9992175261983deff Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 02:30:39 -0500
Subject: [PATCH 0773/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_Zone`

Simply use the RADOS handle directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc |  2 +-
 src/rgw/services/svc_zone.cc        | 10 +++-------
 src/rgw/services/svc_zone.h         |  5 ++---
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index ddd4dfdcd74f..b9b2d2a8415a 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -109,7 +109,7 @@ int RGWServices_Def::init(CephContext *cct,
   notify->init(zone.get(), radoshandle, finisher.get());
   otp->init(zone.get(), meta.get(), meta_be_otp.get());
   rados->init();
-  zone->init(sysobj.get(), rados.get(), sync_modules.get(), bucket_sync_sobj.get());
+  zone->init(sysobj.get(), radoshandle, sync_modules.get(), bucket_sync_sobj.get());
   zone_utils->init(rados.get(), zone.get());
   quota->init(zone.get());
   sync_modules->init(zone.get());
diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc
index 8bb55e4513d2..a44d53727b16 100644
--- a/src/rgw/services/svc_zone.cc
+++ b/src/rgw/services/svc_zone.cc
@@ -6,6 +6,7 @@
 #include "svc_sys_obj.h"
 #include "svc_sync_modules.h"
 
+#include "rgw_tools.h"
 #include "rgw_zone.h"
 #include "rgw_rest_conn.h"
 #include "rgw_bucket_sync.h"
@@ -23,12 +24,12 @@ RGWSI_Zone::RGWSI_Zone(CephContext *cct) : RGWServiceInstance(cct)
 }
 
 void RGWSI_Zone::init(RGWSI_SysObj *_sysobj_svc,
-                      RGWSI_RADOS * _rados_svc,
+                      librados::Rados* rados_,
                       RGWSI_SyncModules * _sync_modules_svc,
 		      RGWSI_Bucket_Sync *_bucket_sync_svc)
 {
   sysobj_svc = _sysobj_svc;
-  rados_svc = _rados_svc;
+  rados = rados_;
   sync_modules_svc = _sync_modules_svc;
   bucket_sync_svc = _bucket_sync_svc;
 
@@ -134,11 +135,6 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp)
 
   assert(sysobj_svc->is_started()); /* if not then there's ordering issue */
 
-  ret = rados_svc->start(y, dpp);
-  if (ret < 0) {
-    return ret;
-  }
-
   ret = realm->init(dpp, cct, sysobj_svc, y);
   if (ret < 0 && ret != -ENOENT) {
     ldpp_dout(dpp, 0) << "failed reading realm info: ret "<< ret << " " << cpp_strerror(-ret) << dendl;
diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h
index 3891b73ddb4d..c4a3a28f0d7b 100644
--- a/src/rgw/services/svc_zone.h
+++ b/src/rgw/services/svc_zone.h
@@ -6,7 +6,6 @@
 #include "rgw_service.h"
 
 
-class RGWSI_RADOS;
 class RGWSI_SysObj;
 class RGWSI_SyncModules;
 class RGWSI_Bucket_Sync;
@@ -29,7 +28,7 @@ class RGWSI_Zone : public RGWServiceInstance
   friend struct RGWServices_Def;
 
   RGWSI_SysObj *sysobj_svc{nullptr};
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados* rados{nullptr};
   RGWSI_SyncModules *sync_modules_svc{nullptr};
   RGWSI_Bucket_Sync *bucket_sync_svc{nullptr};
 
@@ -58,7 +57,7 @@ class RGWSI_Zone : public RGWServiceInstance
   std::unique_ptr<rgw_sync_policy_info> sync_policy;
 
   void init(RGWSI_SysObj *_sysobj_svc,
-	    RGWSI_RADOS *_rados_svc,
+	    librados::Rados* rados_,
 	    RGWSI_SyncModules *_sync_modules_svc,
 	    RGWSI_Bucket_Sync *_bucket_sync_svc);
   int do_start(optional_yield y, const DoutPrefixProvider *dpp) override;

From a992b36efb15851d8598a50e017b89329f05168b Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 02:49:09 -0500
Subject: [PATCH 0774/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_Zone_Tools`

Simply use the RADOS handle directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc |  2 +-
 src/rgw/services/svc_zone_utils.cc  | 28 +++++++++++++++-------------
 src/rgw/services/svc_zone_utils.h   |  7 +++----
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index b9b2d2a8415a..cf03d7b220de 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -110,7 +110,7 @@ int RGWServices_Def::init(CephContext *cct,
   otp->init(zone.get(), meta.get(), meta_be_otp.get());
   rados->init();
   zone->init(sysobj.get(), radoshandle, sync_modules.get(), bucket_sync_sobj.get());
-  zone_utils->init(rados.get(), zone.get());
+  zone_utils->init(radoshandle, zone.get());
   quota->init(zone.get());
   sync_modules->init(zone.get());
   sysobj_core->core_init(rados.get(), zone.get());
diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc
index 712bb97c9ba3..be9f861f02cc 100644
--- a/src/rgw/services/svc_zone_utils.cc
+++ b/src/rgw/services/svc_zone_utils.cc
@@ -5,6 +5,10 @@
 #include "svc_rados.h"
 #include "svc_zone.h"
 
+#undef FMT_HEADER_ONLY
+#define FMT_HEADER_ONLY 1
+#include <fmt/format.h>
+
 #include "rgw_zone.h"
 
 using namespace std;
@@ -18,26 +22,24 @@ int RGWSI_ZoneUtils::do_start(optional_yield, const DoutPrefixProvider *dpp)
 
 string RGWSI_ZoneUtils::gen_host_id() {
   /* uint64_t needs 16, two '-' separators and a trailing null */
-  const string& zone_name = zone_svc->get_zone().name;
-  const string& zonegroup_name = zone_svc->get_zonegroup().get_name();
-  char charbuf[16 + zone_name.size() + zonegroup_name.size() + 2 + 1];
-  snprintf(charbuf, sizeof(charbuf), "%llx-%s-%s", (unsigned long long)rados_svc->instance_id(), zone_name.c_str(), zonegroup_name.c_str());
-  return string(charbuf);
+  return fmt::format("{}-{}-{}", rados->get_instance_id(),
+		     zone_svc->get_zone().name,
+		     zone_svc->get_zonegroup().get_name());
 }
 
 string RGWSI_ZoneUtils::unique_id(uint64_t unique_num)
 {
-  char buf[32];
-  snprintf(buf, sizeof(buf), ".%llu.%llu", (unsigned long long)rados_svc->instance_id(), (unsigned long long)unique_num);
-  string s = zone_svc->get_zone_params().get_id() + buf;
-  return s;
+  return fmt::format("{}.{}.{}",
+		     zone_svc->get_zone_params().get_id(),
+		     rados->get_instance_id(),
+		     unique_num);
 }
 
 void RGWSI_ZoneUtils::init_unique_trans_id_deps() {
-  char buf[16 + 2 + 1]; /* uint64_t needs 16, 2 hyphens add further 2 */
-
-  snprintf(buf, sizeof(buf), "-%llx-", (unsigned long long)rados_svc->instance_id());
-  url_encode(string(buf) + zone_svc->get_zone().name, trans_id_suffix);
+  url_encode(fmt::format("-{}-{}",
+			 rados->get_instance_id(),
+			 zone_svc->get_zone().name),
+	     trans_id_suffix);
 }
 
 /* In order to preserve compatibility with Swift API, transaction ID
diff --git a/src/rgw/services/svc_zone_utils.h b/src/rgw/services/svc_zone_utils.h
index 43e3fee8d93d..41b9400335b0 100644
--- a/src/rgw/services/svc_zone_utils.h
+++ b/src/rgw/services/svc_zone_utils.h
@@ -6,21 +6,20 @@
 #include "rgw_service.h"
 
 
-class RGWSI_RADOS;
 class RGWSI_Zone;
 
 class RGWSI_ZoneUtils : public RGWServiceInstance
 {
   friend struct RGWServices_Def;
 
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados* rados{nullptr};
   RGWSI_Zone *zone_svc{nullptr};
 
   std::string trans_id_suffix;
 
-  void init(RGWSI_RADOS *_rados_svc,
+  void init(librados::Rados* rados_,
             RGWSI_Zone *_zone_svc) {
-    rados_svc = _rados_svc;
+    rados = rados_;
     zone_svc = _zone_svc;
   }
 

From f2807f02b0da3065accfc05a5ff990148d8a1b08 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 2 Dec 2022 02:59:11 -0500
Subject: [PATCH 0775/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_User_RADOS`

Simply use the RADOS handle and `rgw_rados_ref` directly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_service.cc |  2 +-
 src/rgw/services/svc_user_rados.cc  | 39 +++++++++++++++--------------
 src/rgw/services/svc_user_rados.h   |  6 ++---
 3 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index cf03d7b220de..dfde5c85eab5 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -120,7 +120,7 @@ int RGWServices_Def::init(CephContext *cct,
   } else {
     sysobj->init(rados.get(), sysobj_core.get());
   }
-  user_rados->init(rados.get(), zone.get(), sysobj.get(), sysobj_cache.get(),
+  user_rados->init(radoshandle, zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
   role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
 
diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc
index b9dd8cd15535..dd48bc4c01ef 100644
--- a/src/rgw/services/svc_user_rados.cc
+++ b/src/rgw/services/svc_user_rados.cc
@@ -67,14 +67,14 @@ RGWSI_User_RADOS::RGWSI_User_RADOS(CephContext *cct): RGWSI_User(cct) {
 RGWSI_User_RADOS::~RGWSI_User_RADOS() {
 }
 
-void RGWSI_User_RADOS::init(RGWSI_RADOS *_rados_svc,
+void RGWSI_User_RADOS::init(librados::Rados* rados_,
                             RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc,
                             RGWSI_SysObj_Cache *_cache_svc, RGWSI_Meta *_meta_svc,
                             RGWSI_MetaBackend *_meta_be_svc,
                             RGWSI_SyncModules *_sync_modules_svc)
 {
   svc.user = this;
-  svc.rados = _rados_svc;
+  rados = rados_;
   svc.zone = _zone_svc;
   svc.sysobj = _sysobj_svc;
   svc.cache = _cache_svc;
@@ -602,8 +602,8 @@ int RGWSI_User_RADOS::get_user_info_by_access_key(RGWSI_MetaBackend::Context *ct
 
 int RGWSI_User_RADOS::cls_user_update_buckets(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, list<cls_user_bucket_entry>& entries, bool add, optional_yield y)
 {
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
@@ -628,8 +628,8 @@ int RGWSI_User_RADOS::cls_user_add_bucket(const DoutPrefixProvider *dpp, rgw_raw
 
 int RGWSI_User_RADOS::cls_user_remove_bucket(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, const cls_user_bucket& bucket, optional_yield y)
 {
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
@@ -716,8 +716,8 @@ int RGWSI_User_RADOS::cls_user_list_buckets(const DoutPrefixProvider *dpp,
                                             bool * const truncated,
 					    optional_yield y)
 {
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
@@ -804,12 +804,14 @@ int RGWSI_User_RADOS::reset_bucket_stats(const DoutPrefixProvider *dpp,
 int RGWSI_User_RADOS::cls_user_reset_stats(const DoutPrefixProvider *dpp, const rgw_user& user, optional_yield y)
 {
   rgw_raw_obj obj = get_buckets_obj(user);
-  auto rados_obj = svc.rados->obj(obj);
-  int rval, r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
 
+  int rval;
+
   cls_user_reset_stats2_op call;
   cls_user_reset_stats2_ret ret;
 
@@ -841,11 +843,12 @@ int RGWSI_User_RADOS::complete_flush_stats(const DoutPrefixProvider *dpp,
                                            const rgw_user& user, optional_yield y)
 {
   rgw_raw_obj obj = get_buckets_obj(user);
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
+
   librados::ObjectWriteOperation op;
   ::cls_user_complete_stats_sync(op);
   return rados_obj.operate(dpp, &op, y);
@@ -856,8 +859,8 @@ int RGWSI_User_RADOS::cls_user_get_header(const DoutPrefixProvider *dpp,
 					  optional_yield y)
 {
   rgw_raw_obj obj = get_buckets_obj(user);
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref rados_obj;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
   if (r < 0) {
     return r;
   }
@@ -871,15 +874,13 @@ int RGWSI_User_RADOS::cls_user_get_header(const DoutPrefixProvider *dpp,
 int RGWSI_User_RADOS::cls_user_get_header_async(const DoutPrefixProvider *dpp, const string& user_str, RGWGetUserHeader_CB *cb)
 {
   rgw_raw_obj obj = get_buckets_obj(rgw_user(user_str));
-  auto rados_obj = svc.rados->obj(obj);
-  int r = rados_obj.open(dpp);
+  rgw_rados_ref ref;
+  int r = rgw_get_rados_ref(dpp, rados, obj, &ref);
   if (r < 0) {
     return r;
   }
 
-  auto& ref = rados_obj.get_ref();
-
-  r = ::cls_user_get_header_async(ref.pool.ioctx(), ref.obj.oid, cb);
+  r = ::cls_user_get_header_async(ref.ioctx, ref.obj.oid, cb);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/services/svc_user_rados.h b/src/rgw/services/svc_user_rados.h
index 01d8a8eb1ccb..6912327f0b1b 100644
--- a/src/rgw/services/svc_user_rados.h
+++ b/src/rgw/services/svc_user_rados.h
@@ -23,7 +23,6 @@
 
 #include "driver/rados/rgw_bucket.h" // FIXME: subclass dependency
 
-class RGWSI_RADOS;
 class RGWSI_Zone;
 class RGWSI_SysObj;
 class RGWSI_SysObj_Cache;
@@ -96,9 +95,10 @@ class RGWSI_User_RADOS : public RGWSI_User
 
   int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
 public:
+  librados::Rados* rados{nullptr};
+
   struct Svc {
     RGWSI_User_RADOS *user{nullptr};
-    RGWSI_RADOS *rados{nullptr};
     RGWSI_Zone *zone{nullptr};
     RGWSI_SysObj *sysobj{nullptr};
     RGWSI_SysObj_Cache *cache{nullptr};
@@ -110,7 +110,7 @@ class RGWSI_User_RADOS : public RGWSI_User
   RGWSI_User_RADOS(CephContext *cct);
   ~RGWSI_User_RADOS();
 
-  void init(RGWSI_RADOS *_rados_svc,
+  void init(librados::Rados* rados_,
             RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc,
 	    RGWSI_SysObj_Cache *_cache_svc, RGWSI_Meta *_meta_svc,
             RGWSI_MetaBackend *_meta_be_svc,

From 34b8e9fb2228f7c8db6c63f928e8992dafb2de1b Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Mon, 5 Dec 2022 17:10:51 -0500
Subject: [PATCH 0776/2492] rgw: Remove `RGWSI_RADOS` from `RGWSI_SysObj*`

Simply use the RADOS handle and `rgw_rados_ref` directly.

Also make `rgw::AccessListFilter` a std::function.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/driver/dbstore/common/dbstore.h   |  3 +-
 src/rgw/driver/rados/rgw_rados.cc         | 18 +++---
 src/rgw/driver/rados/rgw_rados.h          |  6 +-
 src/rgw/driver/rados/rgw_sal_rados.cc     |  3 +-
 src/rgw/driver/rados/rgw_service.cc       |  8 +--
 src/rgw/driver/rados/rgw_tools.cc         | 39 ++++++++++++
 src/rgw/driver/rados/rgw_tools.h          | 10 +++
 src/rgw/rgw_lc.cc                         | 15 +++--
 src/rgw/rgw_multipart_meta_filter.cc      |  8 +--
 src/rgw/rgw_op.cc                         |  2 -
 src/rgw/rgw_sal.h                         |  3 +-
 src/rgw/rgw_sal_fwd.h                     | 15 ++++-
 src/rgw/services/svc_rados.cc             | 13 ++--
 src/rgw/services/svc_rados.h              | 24 ++------
 src/rgw/services/svc_sys_obj.h            |  6 +-
 src/rgw/services/svc_sys_obj_cache.cc     |  1 -
 src/rgw/services/svc_sys_obj_cache.h      | 10 +--
 src/rgw/services/svc_sys_obj_core.cc      | 75 ++++++++++-------------
 src/rgw/services/svc_sys_obj_core.h       | 10 +--
 src/rgw/services/svc_sys_obj_core_types.h | 16 ++---
 src/rgw/services/svc_tier_rados.cc        |  6 +-
 src/rgw/services/svc_tier_rados.h         | 23 +++----
 22 files changed, 169 insertions(+), 145 deletions(-)

diff --git a/src/rgw/driver/dbstore/common/dbstore.h b/src/rgw/driver/dbstore/common/dbstore.h
index fc436d55df80..8cf6f70f7515 100644
--- a/src/rgw/driver/dbstore/common/dbstore.h
+++ b/src/rgw/driver/dbstore/common/dbstore.h
@@ -1771,14 +1771,13 @@ class DB {
           rgw_obj_key end_marker;
           std::string ns;
           bool enforce_ns;
-          RGWAccessListFilter* access_list_filter;
+	  rgw::AccessListFilter access_list_filter;
           RGWBucketListNameFilter force_check_filter;
           bool list_versions;
 	  bool allow_unordered;
 
           Params() :
 	        enforce_ns(true),
-	        access_list_filter(nullptr),
 	        list_versions(false),
 	        allow_unordered(false)
 	        {}
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 5fd174c3aff0..66d6b12338d1 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1929,7 +1929,7 @@ int RGWRados::Bucket::List::list_objects_ordered(
       }
 
       if (params.access_list_filter &&
-	  ! params.access_list_filter->filter(obj.name, index_key.name)) {
+	  !params.access_list_filter(obj.name, index_key.name)) {
 	ldpp_dout(dpp, 20) << __func__ <<
 	  ": skipping past namespaced objects, including \"" << entry.key <<
 	  "\"" << dendl;
@@ -2223,7 +2223,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp
       }
 
       if (params.access_list_filter &&
-	  !params.access_list_filter->filter(obj.name, index_key.name)) {
+	  !params.access_list_filter(obj.name, index_key.name)) {
         ldpp_dout(dpp, 20) << __func__ <<
 	  ": skipping \"" << index_key <<
 	  "\" because doesn't match filter" << dendl;
@@ -8824,7 +8824,7 @@ string RGWRados::pool_iterate_get_cursor(RGWPoolIterCtx& ctx)
 
 static int do_pool_iterate(const DoutPrefixProvider *dpp, CephContext* cct, RGWPoolIterCtx& ctx, uint32_t num,
                            vector<rgw_bucket_dir_entry>& objs,
-                           bool *is_truncated, RGWAccessListFilter *filter)
+                           bool *is_truncated, const rgw::AccessListFilter& filter)
 {
   librados::IoCtx& io_ctx = ctx.io_ctx;
   librados::NObjectIterator& iter = ctx.iter;
@@ -8841,7 +8841,7 @@ static int do_pool_iterate(const DoutPrefixProvider *dpp, CephContext* cct, RGWP
     ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
 
     // fill it in with initial values; we may correct later
-    if (filter && !filter->filter(oid, oid))
+    if (filter && !filter(oid, oid))
       continue;
 
     e.key = oid;
@@ -8855,7 +8855,7 @@ static int do_pool_iterate(const DoutPrefixProvider *dpp, CephContext* cct, RGWP
 }
 
 int RGWRados::pool_iterate(const DoutPrefixProvider *dpp, RGWPoolIterCtx& ctx, uint32_t num, vector<rgw_bucket_dir_entry>& objs,
-                           bool *is_truncated, RGWAccessListFilter *filter)
+                           bool *is_truncated, const rgw::AccessListFilter& filter)
 {
   // catch exceptions from NObjectIterator::operator++()
   try {
@@ -8892,9 +8892,9 @@ int RGWRados::list_raw_objects_next(const DoutPrefixProvider *dpp, const string&
   if (!ctx.initialized) {
     return -EINVAL;
   }
-  RGWAccessListFilterPrefix filter(prefix_filter);
+  auto filter = rgw::AccessListFilterPrefix(prefix_filter);
   vector<rgw_bucket_dir_entry> objs;
-  int r = pool_iterate(dpp, ctx.iter_ctx, max, objs, is_truncated, &filter);
+  int r = pool_iterate(dpp, ctx.iter_ctx, max, objs, is_truncated, filter);
   if (r < 0) {
     if(r != -ENOENT)
       ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl;
@@ -9660,7 +9660,6 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp,
   ldout_bitx(bitx, dpp, 25) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl_bitx;
 
   ent_list.clear();
-  static MultipartMetaFilter multipart_meta_filter;
 
   *is_truncated = false;
   librados::IoCtx index_pool;
@@ -10008,9 +10007,8 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp,
 
   rgw_obj obj(bucket_info.bucket, list_state.key);
 
-  MultipartMetaFilter multipart_meta_filter;
   string temp_key;
-  if (multipart_meta_filter.filter(list_state.key.name, temp_key)) {
+  if (MultipartMetaFilter(list_state.key.name, temp_key)) {
     obj.in_extra_data = true;
   }
 
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 97528735417d..f5d9b68b259e 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -40,6 +40,7 @@
 #include "rgw_cache.h"
 #include "rgw_sal_fwd.h"
 #include "rgw_pubsub.h"
+#include "rgw_tools.h"
 
 struct D3nDataCache;
 
@@ -1023,14 +1024,13 @@ class RGWRados
         rgw_obj_key end_marker;
         std::string ns;
         bool enforce_ns;
-        RGWAccessListFilter* access_list_filter;
+	rgw::AccessListFilter access_list_filter;
 	RGWBucketListNameFilter force_check_filter;
         bool list_versions;
 	bool allow_unordered;
 
         Params() :
 	  enforce_ns(true),
-	  access_list_filter(nullptr),
 	  list_versions(false),
 	  allow_unordered(false)
 	{}
@@ -1638,7 +1638,7 @@ class RGWRados
    */
   int pool_iterate(const DoutPrefixProvider *dpp, RGWPoolIterCtx& ctx, uint32_t num,
 		   std::vector<rgw_bucket_dir_entry>& objs,
-                   bool *is_truncated, RGWAccessListFilter *filter);
+                   bool *is_truncated, const rgw::AccessListFilter& filter);
 
   uint64_t next_bucket_id();
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 327980f4de6e..0d7359ea90a7 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -751,13 +751,12 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp,
 {
   rgw::sal::Bucket::ListParams params;
   rgw::sal::Bucket::ListResults results;
-  MultipartMetaFilter mp_filter;
 
   params.prefix = prefix;
   params.delim = delim;
   params.marker = marker;
   params.ns = RGW_OBJ_NS_MULTIPART;
-  params.access_list_filter = &mp_filter;
+  params.access_list_filter = MultipartMetaFilter;
 
   int ret = list(dpp, params, max_uploads, results, y);
 
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index dfde5c85eab5..12e9cd82aec5 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -113,12 +113,12 @@ int RGWServices_Def::init(CephContext *cct,
   zone_utils->init(radoshandle, zone.get());
   quota->init(zone.get());
   sync_modules->init(zone.get());
-  sysobj_core->core_init(rados.get(), zone.get());
+  sysobj_core->core_init(radoshandle, zone.get());
   if (have_cache) {
-    sysobj_cache->init(rados.get(), zone.get(), notify.get());
-    sysobj->init(rados.get(), sysobj_cache.get());
+    sysobj_cache->init(radoshandle, zone.get(), notify.get());
+    sysobj->init(radoshandle, sysobj_cache.get());
   } else {
-    sysobj->init(rados.get(), sysobj_core.get());
+    sysobj->init(radoshandle, sysobj_core.get());
   }
   user_rados->init(radoshandle, zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 090c80aa1f71..aa387cb450d5 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -497,3 +497,42 @@ int rgw_clog_warn(librados::Rados* h, const string& msg)
   bufferlist inbl;
   return h->mon_command(cmd, inbl, nullptr, nullptr);
 }
+
+int rgw_list_pool(const DoutPrefixProvider *dpp,
+		  librados::IoCtx& ioctx,
+		  uint32_t max,
+		  const rgw::AccessListFilter& filter,
+		  std::string& marker,
+		  std::vector<string> *oids,
+		  bool *is_truncated)
+{
+  librados::ObjectCursor oc;
+  if (!oc.from_str(marker)) {
+    ldpp_dout(dpp, 10) << "failed to parse cursor: " << marker << dendl;
+    return -EINVAL;
+  }
+
+  auto iter = ioctx.nobjects_begin(oc);
+  /// Pool_iterate
+  if (iter == ioctx.nobjects_end())
+    return -ENOENT;
+
+  uint32_t i;
+
+  for (i = 0; i < max && iter != ioctx.nobjects_end(); ++i, ++iter) {
+    string oid = iter->get_oid();
+    ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
+
+    // fill it in with initial values; we may correct later
+    if (filter && !filter(oid, oid))
+      continue;
+
+    oids->push_back(oid);
+  }
+
+  marker = iter.get_cursor().to_str();
+  if (is_truncated)
+    *is_truncated = (iter != ioctx.nobjects_end());
+
+  return oids->size();
+}
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index a7639acbb24c..27a8b424ecce 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -3,7 +3,9 @@
 
 #pragma once
 
+#include <functional>
 #include <string>
+#include <string_view>
 
 #include "include/types.h"
 #include "include/ceph_hash.h"
@@ -333,3 +335,11 @@ std::map<std::string, ceph::buffer::list>* no_change_attrs();
 
 bool rgw_check_secure_mon_conn(const DoutPrefixProvider *dpp);
 int rgw_clog_warn(librados::Rados* h, const std::string& msg);
+
+int rgw_list_pool(const DoutPrefixProvider *dpp,
+		  librados::IoCtx& ioctx,
+		  uint32_t max,
+		  const rgw::AccessListFilter& filter,
+		  std::string& marker,
+		  std::vector<std::string> *oids,
+		  bool *is_truncated);
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 79b81270457d..260ad1795bc0 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -814,7 +814,6 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 				       const multimap<string, lc_op>& prefix_map,
 				       LCWorker* worker, time_t stop_at, bool once)
 {
-  MultipartMetaFilter mp_filter;
   int ret;
   rgw::sal::Bucket::ListParams params;
   rgw::sal::Bucket::ListResults results;
@@ -825,7 +824,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
    * operating on one shard at a time */
   params.allow_unordered = true;
   params.ns = RGW_OBJ_NS_MULTIPART;
-  params.access_list_filter = &mp_filter;
+  params.access_list_filter = MultipartMetaFilter;
 
   const auto event_type = rgw::notify::ObjectExpirationAbortMPU;
 
@@ -1108,7 +1107,7 @@ class LCOpAction_CurrentExpiration : public LCOpAction {
     return is_expired;
   }
 
-  int process(lc_op_ctx& oc) {
+  int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r;
     if (o.is_delete_marker()) {
@@ -1172,7 +1171,7 @@ class LCOpAction_NonCurrentExpiration : public LCOpAction {
       pass_object_lock_check(oc.driver, oc.obj.get(), dpp);
   }
 
-  int process(lc_op_ctx& oc) {
+  int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r = remove_expired_obj(oc.dpp, oc, true,
 			       rgw::notify::ObjectExpirationNoncurrent);
@@ -1217,7 +1216,7 @@ class LCOpAction_DMExpiration : public LCOpAction {
     return true;
   }
 
-  int process(lc_op_ctx& oc) {
+  int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r = remove_expired_obj(oc.dpp, oc, true,
 			       rgw::notify::ObjectExpirationDeleteMarker);
@@ -1385,7 +1384,7 @@ class LCOpAction_Transition : public LCOpAction {
     return 0;
   }
 
-  int process(lc_op_ctx& oc) {
+  int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r;
 
@@ -1460,7 +1459,7 @@ class LCOpAction_CurrentTransition : public LCOpAction_Transition {
 public:
   LCOpAction_CurrentTransition(const transition_action& _transition)
     : LCOpAction_Transition(_transition) {}
-    int process(lc_op_ctx& oc) {
+    int process(lc_op_ctx& oc) override {
       int r = LCOpAction_Transition::process(oc);
       if (r == 0) {
         if (perfcounter) {
@@ -1485,7 +1484,7 @@ class LCOpAction_NonCurrentTransition : public LCOpAction_Transition {
 				  const transition_action& _transition)
     : LCOpAction_Transition(_transition)
     {}
-    int process(lc_op_ctx& oc) {
+    int process(lc_op_ctx& oc) override {
       int r = LCOpAction_Transition::process(oc);
       if (r == 0) {
         if (perfcounter) {
diff --git a/src/rgw/rgw_multipart_meta_filter.cc b/src/rgw/rgw_multipart_meta_filter.cc
index c616cd480f75..aeefc731f734 100644
--- a/src/rgw/rgw_multipart_meta_filter.cc
+++ b/src/rgw/rgw_multipart_meta_filter.cc
@@ -3,11 +3,9 @@
 
 #include "svc_tier_rados.h"
 
-using namespace std;
-
 const std::string MP_META_SUFFIX = ".meta";
 
-bool MultipartMetaFilter::filter(const string& name, string& key) {
+bool MultipartMetaFilter(const std::string& name, std::string& key) {
   // the length of the suffix so we can skip past it
   static const size_t MP_META_SUFFIX_LEN = MP_META_SUFFIX.length();
 
@@ -19,11 +17,11 @@ bool MultipartMetaFilter::filter(const string& name, string& key) {
     return false;
 
   size_t pos = name.find(MP_META_SUFFIX, len - MP_META_SUFFIX_LEN);
-  if (pos == string::npos)
+  if (pos == std::string::npos)
     return false;
 
   pos = name.rfind('.', pos - 1);
-  if (pos == string::npos)
+  if (pos == std::string::npos)
     return false;
 
   key = name.substr(0, pos);
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 7363db728a49..a290388efe0c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -102,8 +102,6 @@ static string shadow_ns = RGW_OBJ_NS_SHADOW;
 
 static void forward_req_info(const DoutPrefixProvider *dpp, CephContext *cct, req_info& info, const std::string& bucket_name);
 
-static MultipartMetaFilter mp_filter;
-
 // this probably should belong in the rgw_iam_policy_keywords, I'll get it to it
 // at some point
 static constexpr auto S3_EXISTING_OBJTAG = "s3:ExistingObjectTag";
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 888c5ec20ed1..8771b74c87aa 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -31,7 +31,6 @@
 
 struct RGWBucketEnt;
 class RGWRESTMgr;
-class RGWAccessListFilter;
 class RGWLC;
 struct rgw_user_bucket;
 class RGWUsageBatch;
@@ -582,7 +581,7 @@ class Bucket {
       rgw_obj_key end_marker;
       std::string ns;
       bool enforce_ns{true};
-      RGWAccessListFilter* access_list_filter{nullptr};
+      rgw::AccessListFilter access_list_filter{};
       RGWBucketListNameFilter force_check_filter;
       bool list_versions{false};
       bool allow_unordered{false};
diff --git a/src/rgw/rgw_sal_fwd.h b/src/rgw/rgw_sal_fwd.h
index ee7c100080b5..123d17a5162a 100644
--- a/src/rgw/rgw_sal_fwd.h
+++ b/src/rgw/rgw_sal_fwd.h
@@ -15,8 +15,21 @@
 
 #pragma once
 
+#include <functional>
+#include <string>
 
-namespace rgw { namespace sal {
+namespace rgw {
+using AccessListFilter =
+  std::function<bool(const std::string&, std::string&)>;
+
+inline auto AccessListFilterPrefix(std::string prefix) {
+  return [prefix = std::move(prefix)](const std::string& name,
+				      std::string& key) {
+    return (prefix.compare(key.substr(0, prefix.size())) == 0);
+  };
+}
+
+namespace sal {
 
   class Driver;
   class User;
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index 34de9857e19a..c45853368680 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -71,7 +71,7 @@ int RGWSI_RADOS::pool_iterate(const DoutPrefixProvider *dpp,
                               librados::IoCtx& io_ctx,
                               librados::NObjectIterator& iter,
                               uint32_t num, vector<rgw_bucket_dir_entry>& objs,
-                              RGWAccessListFilter *filter,
+                              const rgw::AccessListFilter& filter,
                               bool *is_truncated)
 {
   if (iter == io_ctx.nobjects_end())
@@ -86,7 +86,7 @@ int RGWSI_RADOS::pool_iterate(const DoutPrefixProvider *dpp,
     ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
 
     // fill it in with initial values; we may correct later
-    if (filter && !filter->filter(oid, oid))
+    if (filter && filter(oid, oid))
       continue;
 
     e.key = oid;
@@ -296,7 +296,7 @@ int RGWSI_RADOS::Pool::open(const DoutPrefixProvider *dpp, const OpenParams& par
   return rados_svc->open_pool_ctx(dpp, pool, state.ioctx, params);
 }
 
-int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& marker, RGWAccessListFilter *filter)
+int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& marker, rgw::AccessListFilter filter)
 {
   if (ctx.initialized) {
     return -EINVAL;
@@ -319,9 +319,6 @@ int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& m
 
   try {
     ctx.iter = ctx.ioctx.nobjects_begin(oc);
-    ctx.filter = filter;
-    ctx.initialized = true;
-    return 0;
   } catch (const std::system_error& e) {
     r = -e.code().value();
     ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what()
@@ -332,6 +329,10 @@ int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& m
        << ", returning -5" << dendl;
     return -EIO;
   }
+  ctx.filter = std::move(filter);
+  ctx.initialized = true;
+
+  return 0;
 }
 
 int RGWSI_RADOS::Pool::List::get_next(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index 2a3940a34d96..bed6522e2ae2 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -8,22 +8,9 @@
 #include "include/rados/librados.hpp"
 #include "common/async/yield_context.h"
 
-class RGWAsyncRadosProcessor;
-
-class RGWAccessListFilter {
-public:
-  virtual ~RGWAccessListFilter() {}
-  virtual bool filter(const std::string& name, std::string& key) = 0;
-};
+#include "rgw_tools.h"
 
-struct RGWAccessListFilterPrefix : public RGWAccessListFilter {
-  std::string prefix;
-
-  explicit RGWAccessListFilterPrefix(const std::string& _prefix) : prefix(_prefix) {}
-  bool filter(const std::string& name, std::string& key) override {
-    return (prefix.compare(key.substr(0, prefix.size())) == 0);
-  }
-};
+class RGWAsyncRadosProcessor;
 
 class RGWSI_RADOS : public RGWServiceInstance
 {
@@ -55,7 +42,7 @@ class RGWSI_RADOS : public RGWServiceInstance
                    librados::IoCtx& ioctx,
                    librados::NObjectIterator& iter,
                    uint32_t num, std::vector<rgw_bucket_dir_entry>& objs,
-                   RGWAccessListFilter *filter,
+                   const rgw::AccessListFilter& filter,
                    bool *is_truncated);
 
 public:
@@ -118,13 +105,14 @@ class RGWSI_RADOS : public RGWServiceInstance
         bool initialized{false};
         librados::IoCtx ioctx;
         librados::NObjectIterator iter;
-        RGWAccessListFilter *filter{nullptr};
+	rgw::AccessListFilter filter;
       } ctx;
 
       List() {}
       List(Pool *_pool) : pool(_pool) {}
 
-      int init(const DoutPrefixProvider *dpp, const std::string& marker, RGWAccessListFilter *filter = nullptr);
+      int init(const DoutPrefixProvider *dpp, const std::string& marker,
+	       rgw::AccessListFilter filter);
       int get_next(const DoutPrefixProvider *dpp, int max,
                    std::vector<std::string> *oids,
                    bool *is_truncated);
diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h
index 292340dcbe32..3739d68d150e 100644
--- a/src/rgw/services/svc_sys_obj.h
+++ b/src/rgw/services/svc_sys_obj.h
@@ -246,12 +246,12 @@ class RGWSI_SysObj : public RGWServiceInstance
   friend class Pool::Op;
 
 protected:
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados* rados{nullptr};
   RGWSI_SysObj_Core *core_svc{nullptr};
 
-  void init(RGWSI_RADOS *_rados_svc,
+  void init(librados::Rados* rados_,
             RGWSI_SysObj_Core *_core_svc) {
-    rados_svc = _rados_svc;
+    rados = rados_;
     core_svc = _core_svc;
   }
 
diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc
index d1b7a3dbb3e7..cd4e2c405abf 100644
--- a/src/rgw/services/svc_sys_obj_cache.cc
+++ b/src/rgw/services/svc_sys_obj_cache.cc
@@ -1,4 +1,3 @@
-
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
diff --git a/src/rgw/services/svc_sys_obj_cache.h b/src/rgw/services/svc_sys_obj_cache.h
index f7950843fa92..8e2f5845dc12 100644
--- a/src/rgw/services/svc_sys_obj_cache.h
+++ b/src/rgw/services/svc_sys_obj_cache.h
@@ -17,7 +17,7 @@ class RGWSI_SysObj_Cache_ASocketHook;
 class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core
 {
   friend class RGWSI_SysObj_Cache_CB;
-  friend class RGWServices_Def;
+  friend RGWServices_Def;
   friend class ASocketHandler;
 
   RGWSI_Notify *notify_svc{nullptr};
@@ -27,10 +27,10 @@ class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core
 
   void normalize_pool_and_obj(const rgw_pool& src_pool, const std::string& src_obj, rgw_pool& dst_pool, std::string& dst_obj);
 protected:
-  void init(RGWSI_RADOS *_rados_svc,
+  void init(librados::Rados* rados_,
             RGWSI_Zone *_zone_svc,
             RGWSI_Notify *_notify_svc) {
-    core_init(_rados_svc, _zone_svc);
+    core_init(rados_, _zone_svc);
     notify_svc = _notify_svc;
   }
 
@@ -80,12 +80,12 @@ class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core
             real_time set_mtime,
             optional_yield y) override;
 
-  int write_data(const DoutPrefixProvider *dpp, 
+  int write_data(const DoutPrefixProvider *dpp,
                  const rgw_raw_obj& obj,
                  const bufferlist& bl,
                  bool exclusive,
                  RGWObjVersionTracker *objv_tracker,
-                 optional_yield y);
+                 optional_yield y) override;
 
   int distribute_cache(const DoutPrefixProvider *dpp, const std::string& normal_name, const rgw_raw_obj& obj,
                        ObjectCacheInfo& obj_info, int op,
diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc
index 30308969131d..68878cdcceb2 100644
--- a/src/rgw/services/svc_sys_obj_core.cc
+++ b/src/rgw/services/svc_sys_obj_core.cc
@@ -12,10 +12,10 @@
 using namespace std;
 
 int RGWSI_SysObj_Core_GetObjState::get_rados_obj(const DoutPrefixProvider *dpp,
-                                                 RGWSI_RADOS *rados_svc,
+                                                 librados::Rados* rados,
                                                  RGWSI_Zone *zone_svc,
                                                  const rgw_raw_obj& obj,
-                                                 RGWSI_RADOS::Obj **pobj)
+                                                 rgw_rados_ref** pobj)
 {
   if (!has_rados_obj) {
     if (obj.oid.empty()) {
@@ -23,8 +23,7 @@ int RGWSI_SysObj_Core_GetObjState::get_rados_obj(const DoutPrefixProvider *dpp,
       return -EINVAL;
     }
 
-    rados_obj = rados_svc->obj(obj);
-    int r = rados_obj.open(dpp);
+    int r = rgw_get_rados_ref(dpp, rados, obj, &rados_obj);
     if (r < 0) {
       return r;
     }
@@ -37,15 +36,14 @@ int RGWSI_SysObj_Core_GetObjState::get_rados_obj(const DoutPrefixProvider *dpp,
 int RGWSI_SysObj_Core::get_rados_obj(const DoutPrefixProvider *dpp,
                                      RGWSI_Zone *zone_svc,
                                      const rgw_raw_obj& obj,
-                                     RGWSI_RADOS::Obj *pobj)
+                                     rgw_rados_ref* pobj)
 {
   if (obj.oid.empty()) {
     ldpp_dout(dpp, 0) << "ERROR: obj.oid is empty" << dendl;
     return -EINVAL;
   }
 
-  *pobj = rados_svc->obj(obj);
-  int r = pobj->open(dpp);
+  int r = rgw_get_rados_ref(dpp, rados, obj, pobj);
   if (r < 0) {
     return r;
   }
@@ -59,7 +57,7 @@ int RGWSI_SysObj_Core::raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj
                                 RGWObjVersionTracker *objv_tracker,
                                 optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     return r;
@@ -172,7 +170,7 @@ int RGWSI_SysObj_Core::read(const DoutPrefixProvider *dpp,
     }
   }
 
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -185,7 +183,7 @@ int RGWSI_SysObj_Core::read(const DoutPrefixProvider *dpp,
   }
   ldpp_dout(dpp, 20) << "rados_obj.operate() r=" << r << " bl.length=" << bl->length() << dendl;
 
-  uint64_t op_ver = rados_obj.get_last_version();
+  uint64_t op_ver = rados_obj.ioctx.get_last_version();
 
   if (read_state.last_ver > 0 &&
       read_state.last_ver != op_ver) {
@@ -218,7 +216,7 @@ int RGWSI_SysObj_Core::get_attr(const DoutPrefixProvider *dpp,
                                 bufferlist *dest,
                                 optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -229,7 +227,7 @@ int RGWSI_SysObj_Core::get_attr(const DoutPrefixProvider *dpp,
 
   int rval;
   op.getxattr(name, dest, &rval);
-  
+
   r = rados_obj.operate(dpp, &op, nullptr, y);
   if (r < 0)
     return r;
@@ -244,7 +242,7 @@ int RGWSI_SysObj_Core::set_attrs(const DoutPrefixProvider *dpp,
                                  RGWObjVersionTracker *objv_tracker,
                                  bool exclusive, optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -301,7 +299,7 @@ int RGWSI_SysObj_Core::omap_get_vals(const DoutPrefixProvider *dpp,
                                      bool *pmore,
                                      optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -341,7 +339,7 @@ int RGWSI_SysObj_Core::omap_get_all(const DoutPrefixProvider *dpp,
                                     std::map<string, bufferlist> *m,
                                     optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -359,7 +357,7 @@ int RGWSI_SysObj_Core::omap_get_all(const DoutPrefixProvider *dpp,
     std::map<string, bufferlist> t;
     int rval;
     op.omap_get_vals2(start_after, count, &t, &more, &rval);
-  
+
     r = rados_obj.operate(dpp, &op, nullptr, y);
     if (r < 0) {
       return r;
@@ -377,7 +375,7 @@ int RGWSI_SysObj_Core::omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj
                                 bufferlist& bl, bool must_exist,
                                 optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -400,7 +398,7 @@ int RGWSI_SysObj_Core::omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj
                                 const std::map<std::string, bufferlist>& m,
                                 bool must_exist, optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -418,7 +416,7 @@ int RGWSI_SysObj_Core::omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj
 int RGWSI_SysObj_Core::omap_del(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const std::string& key,
                                 optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -440,7 +438,7 @@ int RGWSI_SysObj_Core::notify(const DoutPrefixProvider *dpp, const rgw_raw_obj&
                               uint64_t timeout_ms, bufferlist *pbl,
                               optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -456,7 +454,7 @@ int RGWSI_SysObj_Core::remove(const DoutPrefixProvider *dpp,
                               const rgw_raw_obj& obj,
                               optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -487,7 +485,7 @@ int RGWSI_SysObj_Core::write(const DoutPrefixProvider *dpp,
                              real_time set_mtime,
                              optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -552,7 +550,7 @@ int RGWSI_SysObj_Core::write_data(const DoutPrefixProvider *dpp,
                                   RGWObjVersionTracker *objv_tracker,
                                   optional_yield y)
 {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl;
@@ -585,21 +583,17 @@ int RGWSI_SysObj_Core::pool_list_prefixed_objs(const DoutPrefixProvider *dpp,
 {
   bool is_truncated;
 
-  auto rados_pool = rados_svc->pool(pool);
-
-  auto op = rados_pool.op();
+  librados::IoCtx rados_pool;
+  rgw_init_ioctx(dpp, rados, pool, rados_pool, true, false);
 
-  RGWAccessListFilterPrefix filter(prefix);
-
-  int r = op.init(dpp, string(), &filter);
-  if (r < 0) {
-    return r;
-  }
+  auto filter{rgw::AccessListFilterPrefix(prefix)};
+  std::string marker;
 
   do {
     vector<string> oids;
-#define MAX_OBJS_DEFAULT 1000
-    int r = op.get_next(dpp, MAX_OBJS_DEFAULT, &oids, &is_truncated);
+    static constexpr auto MAX_OBJS_DEFAULT = 1000u;
+    int r = rgw_list_pool(dpp, rados_pool, MAX_OBJS_DEFAULT, filter, marker,
+			  &oids, &is_truncated);
     if (r < 0) {
       return r;
     }
@@ -623,12 +617,9 @@ int RGWSI_SysObj_Core::pool_list_objects_init(const DoutPrefixProvider *dpp,
 
   auto& ctx = static_cast<PoolListImplInfo&>(*_ctx->impl);
 
-  ctx.pool = rados_svc->pool(pool);
-  ctx.op = ctx.pool.op();
-
-  int r = ctx.op.init(dpp, marker, &ctx.filter);
+  int r = rgw_init_ioctx(dpp, rados, pool, ctx.pool, true, false);
   if (r < 0) {
-    ldpp_dout(dpp, 10) << "failed to list objects pool_iterate_begin() returned r=" << r << dendl;
+    ldpp_dout(dpp, 10) << "failed to create IoCtx returned r=" << r << dendl;
     return r;
   }
   return 0;
@@ -644,7 +635,8 @@ int RGWSI_SysObj_Core::pool_list_objects_next(const DoutPrefixProvider *dpp,
     return -EINVAL;
   }
   auto& ctx = static_cast<PoolListImplInfo&>(*_ctx.impl);
-  int r = ctx.op.get_next(dpp, max, oids, is_truncated);
+  int r = rgw_list_pool(dpp, ctx.pool, max, ctx.filter, ctx.marker, oids,
+			is_truncated);
   if (r < 0) {
     if(r != -ENOENT)
       ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl;
@@ -662,5 +654,6 @@ int RGWSI_SysObj_Core::pool_list_objects_get_marker(RGWSI_SysObj::Pool::ListCtx&
   }
 
   auto& ctx = static_cast<PoolListImplInfo&>(*_ctx.impl);
-  return ctx.op.get_marker(marker);
+  *marker = ctx.marker;
+  return 0;
 }
diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h
index d02a37eee8af..1b34d7078317 100644
--- a/src/rgw/services/svc_sys_obj_core.h
+++ b/src/rgw/services/svc_sys_obj_core.h
@@ -16,22 +16,22 @@ struct rgw_cache_entry_info;
 
 class RGWSI_SysObj_Core : public RGWServiceInstance
 {
-  friend class RGWServices_Def;
+  friend struct RGWServices_Def;
   friend class RGWSI_SysObj;
 
 protected:
-  RGWSI_RADOS *rados_svc{nullptr};
+  librados::Rados* rados{nullptr};
   RGWSI_Zone *zone_svc{nullptr};
 
   using GetObjState = RGWSI_SysObj_Core_GetObjState;
   using PoolListImplInfo = RGWSI_SysObj_Core_PoolListImplInfo;
 
-  void core_init(RGWSI_RADOS *_rados_svc,
+  void core_init(librados::Rados* rados_,
                  RGWSI_Zone *_zone_svc) {
-    rados_svc = _rados_svc;
+    rados = rados_;
     zone_svc = _zone_svc;
   }
-  int get_rados_obj(const DoutPrefixProvider *dpp, RGWSI_Zone *zone_svc, const rgw_raw_obj& obj, RGWSI_RADOS::Obj *pobj);
+  int get_rados_obj(const DoutPrefixProvider *dpp, RGWSI_Zone *zone_svc, const rgw_raw_obj& obj, rgw_rados_ref* pobj);
 
   virtual int raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj,
                        uint64_t *psize, real_time *pmtime,
diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h
index 74f489d914e5..7bec012f5c29 100644
--- a/src/rgw/services/svc_sys_obj_core_types.h
+++ b/src/rgw/services/svc_sys_obj_core_types.h
@@ -4,6 +4,7 @@
 #pragma once
 
 
+#include "rgw_tools.h"
 #include "rgw_service.h"
 
 #include "svc_rados.h"
@@ -12,23 +13,24 @@
 
 
 struct RGWSI_SysObj_Core_GetObjState : public RGWSI_SysObj_Obj_GetObjState {
-  RGWSI_RADOS::Obj rados_obj;
+  rgw_rados_ref rados_obj;
   bool has_rados_obj{false};
   uint64_t last_ver{0};
 
   RGWSI_SysObj_Core_GetObjState() {}
 
   int get_rados_obj(const DoutPrefixProvider *dpp,
-                    RGWSI_RADOS *rados_svc,
+                    librados::Rados* rados_svc,
                     RGWSI_Zone *zone_svc,
                     const rgw_raw_obj& obj,
-                    RGWSI_RADOS::Obj **pobj);
+                    rgw_rados_ref** pobj);
 };
 
 struct RGWSI_SysObj_Core_PoolListImplInfo : public RGWSI_SysObj_Pool_ListInfo {
-  RGWSI_RADOS::Pool pool;
-  RGWSI_RADOS::Pool::List op;
-  RGWAccessListFilterPrefix filter;
+  librados::IoCtx pool;
+  rgw::AccessListFilter filter;
+  std::string marker;
 
-  RGWSI_SysObj_Core_PoolListImplInfo(const std::string& prefix) : op(pool.op()), filter(prefix) {}
+  RGWSI_SysObj_Core_PoolListImplInfo(const std::string& prefix)
+    : filter(rgw::AccessListFilterPrefix(prefix)) {}
 };
diff --git a/src/rgw/services/svc_tier_rados.cc b/src/rgw/services/svc_tier_rados.cc
index ca87e8aceb90..86ccb5eca56e 100644
--- a/src/rgw/services/svc_tier_rados.cc
+++ b/src/rgw/services/svc_tier_rados.cc
@@ -7,9 +7,7 @@ using namespace std;
 
 const std::string MP_META_SUFFIX = ".meta";
 
-MultipartMetaFilter::~MultipartMetaFilter() {}
-
-bool MultipartMetaFilter::filter(const string& name, string& key) {
+bool MultipartMetaFilter(const string& name, string& key) {
   // the length of the suffix so we can skip past it
   static const size_t MP_META_SUFFIX_LEN = MP_META_SUFFIX.length();
 
@@ -32,5 +30,3 @@ bool MultipartMetaFilter::filter(const string& name, string& key) {
 
   return true;
 }
-
-
diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h
index a2036b933473..f7424b410387 100644
--- a/src/rgw/services/svc_tier_rados.h
+++ b/src/rgw/services/svc_tier_rados.h
@@ -110,21 +110,14 @@ class RGWMPObj {
  * the name provided is such. It will also extract the key used for
  * bucket index shard calculation from the adorned name.
  */
-class MultipartMetaFilter : public RGWAccessListFilter {
-public:
-  MultipartMetaFilter() {}
-
-  virtual ~MultipartMetaFilter() override;
-
-  /**
-   * @param name [in] The object name as it appears in the bucket index.
-   * @param key [out] An output parameter that will contain the bucket
-   *        index key if this entry is in the form of a multipart meta object.
-   * @return true if the name provided is in the form of a multipart meta
-   *         object, false otherwise
-   */
-  bool filter(const std::string& name, std::string& key) override;
-};
+/**
+ * @param name [in] The object name as it appears in the bucket index.
+ * @param key [out] An output parameter that will contain the bucket
+ *        index key if this entry is in the form of a multipart meta object.
+ * @return true if the name provided is in the form of a multipart meta
+ *         object, false otherwise
+ */
+bool MultipartMetaFilter(const std::string& name, std::string& key);
 
 class RGWSI_Tier_RADOS : public RGWServiceInstance
 {

From 0b7b2333984e2ac8a32304336345696f3b6091b5 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Mon, 5 Dec 2022 19:51:25 -0500
Subject: [PATCH 0777/2492] rgw: Remove `RGWSI_RADOS`

And remove the last places that reference it.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/CMakeLists.txt                        |   1 -
 src/rgw/driver/rados/rgw_cr_rados.cc          |  24 +-
 src/rgw/driver/rados/rgw_cr_rados.h           |  10 +-
 src/rgw/driver/rados/rgw_d3n_datacache.h      |  20 +-
 src/rgw/driver/rados/rgw_data_sync.cc         |  25 +-
 .../driver/rados/rgw_object_expirer_core.cc   |  14 +-
 .../driver/rados/rgw_object_expirer_core.h    |   9 +-
 src/rgw/driver/rados/rgw_putobj_processor.cc  |  14 +-
 src/rgw/driver/rados/rgw_putobj_processor.h   |   3 +-
 src/rgw/driver/rados/rgw_rados.cc             |  38 +-
 src/rgw/driver/rados/rgw_rados.h              |   1 -
 src/rgw/driver/rados/rgw_service.cc           |  43 +-
 src/rgw/driver/rados/rgw_service.h            |  15 +-
 src/rgw/driver/rados/rgw_sync_error_repo.cc   |  33 +-
 src/rgw/driver/rados/rgw_sync_error_repo.h    |   5 +-
 src/rgw/services/svc_bi_rados.h               |   1 -
 src/rgw/services/svc_bilog_rados.h            |   6 -
 src/rgw/services/svc_cls.cc                   |   1 -
 src/rgw/services/svc_cls.h                    |   2 -
 src/rgw/services/svc_mdlog.cc                 |   1 -
 src/rgw/services/svc_notify.cc                |   1 -
 src/rgw/services/svc_notify.h                 |   2 -
 src/rgw/services/svc_rados.cc                 | 433 ------------------
 src/rgw/services/svc_rados.h                  | 232 ----------
 src/rgw/services/svc_sys_obj.cc               |   1 -
 src/rgw/services/svc_sys_obj.h                |   1 -
 src/rgw/services/svc_sys_obj_core.cc          |   1 -
 src/rgw/services/svc_sys_obj_core.h           |   1 -
 src/rgw/services/svc_sys_obj_core_types.h     |   1 -
 src/rgw/services/svc_tier_rados.h             |   2 -
 src/rgw/services/svc_zone.cc                  |   1 -
 src/rgw/services/svc_zone_utils.cc            |   1 -
 32 files changed, 110 insertions(+), 833 deletions(-)
 delete mode 100644 src/rgw/services/svc_rados.cc
 delete mode 100644 src/rgw/services/svc_rados.h

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index d54c5b6afa95..00a9e4127e1e 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -48,7 +48,6 @@ set(librgw_common_srcs
   services/svc_otp.cc
   services/svc_quota.cc
   services/svc_sync_modules.cc
-  services/svc_rados.cc
   services/svc_role_rados.cc
   services/svc_sys_obj.cc
   services/svc_sys_obj_cache.cc
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index cacba3cf44bc..f903233cb02c 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -476,29 +476,7 @@ RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
 }
 
 RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-					 RGWSI_RADOS::Obj& obj,
-					 RGWObjVersionTracker* objv_tracker)
-  : RGWSimpleCoroutine(store->ctx()),
-    ioctx(librados::IoCtx(obj.get_ref().pool.ioctx())),
-    oid(obj.get_ref().obj.oid),
-    objv_tracker(objv_tracker)
-{
-  set_description() << "remove dest=" << oid;
-}
-
-RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-					 RGWSI_RADOS::Obj&& obj,
-					 RGWObjVersionTracker* objv_tracker)
-  : RGWSimpleCoroutine(store->ctx()),
-    ioctx(std::move(obj.get_ref().pool.ioctx())),
-    oid(std::move(obj.get_ref().obj.oid)),
-    objv_tracker(objv_tracker)
-{
-  set_description() << "remove dest=" << oid;
-}
-
-RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-					 rgw_rados_ref&& obj,
+					 rgw_rados_ref obj,
 					 RGWObjVersionTracker* objv_tracker)
   : RGWSimpleCoroutine(store->ctx()),
     ioctx(std::move(obj.ioctx)),
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index 0afb74d93a96..2cf99f98cc8e 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -753,15 +753,7 @@ class RGWRadosRemoveOidCR : public RGWSimpleCoroutine {
 		      RGWObjVersionTracker* objv_tracker = nullptr);
 
   RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-		      RGWSI_RADOS::Obj& obj,
-		      RGWObjVersionTracker* objv_tracker = nullptr);
-
-  RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-		      RGWSI_RADOS::Obj&& obj,
-		      RGWObjVersionTracker* objv_tracker = nullptr);
-
-  RGWRadosRemoveOidCR(rgw::sal::RadosStore* store,
-		      rgw_rados_ref&& obj,
+		      rgw_rados_ref obj,
 		      RGWObjVersionTracker* objv_tracker = nullptr);
 
   int send_request(const DoutPrefixProvider *dpp) override;
diff --git a/src/rgw/driver/rados/rgw_d3n_datacache.h b/src/rgw/driver/rados/rgw_d3n_datacache.h
index 196f892329f8..58ac95d093d0 100644
--- a/src/rgw/driver/rados/rgw_d3n_datacache.h
+++ b/src/rgw/driver/rados/rgw_d3n_datacache.h
@@ -195,10 +195,10 @@ int D3nRGWDataCache<T>::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const
         return 0;
     }
 
-    auto obj = d->rgwrados->svc.rados->obj(read_obj);
-    r = obj.open(dpp);
+    rgw_rados_ref ref;
+    r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj, &ref);
     if (r < 0) {
-      lsubdout(g_ceph_context, rgw, 4) << "failed to open rados context for " << read_obj << dendl;
+      ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl;
       return r;
     }
 
@@ -208,8 +208,7 @@ int D3nRGWDataCache<T>::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const
     const uint64_t cost = len;
     const uint64_t id = obj_ofs; // use logical object offset for sorting replies
 
-    auto& ref = obj.get_ref();
-    auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id);
+    auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id);
     return d->flush(std::move(completed));
   } else {
     ldpp_dout(dpp, 20) << "D3nDataCache::" << __func__ << "(): oid=" << read_obj.oid << ", is_head_obj=" << is_head_obj << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << ", len=" << len << dendl;
@@ -221,20 +220,19 @@ int D3nRGWDataCache<T>::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const
     const uint64_t id = obj_ofs; // use logical object offset for sorting replies
     oid = read_obj.oid;
 
-    auto obj = d->rgwrados->svc.rados->obj(read_obj);
-    r = obj.open(dpp);
+    rgw_rados_ref ref;
+    r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj, &ref);
     if (r < 0) {
-      lsubdout(g_ceph_context, rgw, 0) << "D3nDataCache: Error: failed to open rados context for " << read_obj << ", r=" << r << dendl;
+      ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl;
       return r;
     }
-    auto& ref = obj.get_ref();
 
     const bool is_compressed = (astate->attrset.find(RGW_ATTR_COMPRESSION) != astate->attrset.end());
     const bool is_encrypted = (astate->attrset.find(RGW_ATTR_CRYPT_MODE) != astate->attrset.end());
     if (read_ofs != 0 || astate->size != astate->accounted_size || is_compressed || is_encrypted) {
       d->d3n_bypass_cache_write = true;
       lsubdout(g_ceph_context, rgw, 5) << "D3nDataCache: " << __func__ << "(): Note - bypassing datacache: oid=" << read_obj.oid << ", read_ofs!=0 = " << read_ofs << ", size=" << astate->size << " != accounted_size=" << astate->accounted_size << ", is_compressed=" << is_compressed << ", is_encrypted=" << is_encrypted  << dendl;
-      auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id);
+      auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id);
       r = d->flush(std::move(completed));
       return r;
     }
@@ -251,7 +249,7 @@ int D3nRGWDataCache<T>::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const
     } else {
       // Write To Cache
       ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): WRITE TO CACHE: oid=" << read_obj.oid << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << " len=" << len << dendl;
-      auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id);
+      auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id);
       return d->flush(std::move(completed));
     }
   }
diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index 7b8c0b7343d9..b5effb979c9b 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -1457,7 +1457,7 @@ class RGWDataSyncSingleEntryCR : public RGWCoroutine {
         }
         if (complete->timestamp != ceph::real_time{}) {
           tn->log(10, SSTR("writing " << *complete << " to error repo for retry"));
-          yield call(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo,
+          yield call(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                                               rgw::error_repo::encode_key(complete->bs, complete->gen),
                                               complete->timestamp));
           if (retcode < 0) {
@@ -1465,7 +1465,7 @@ class RGWDataSyncSingleEntryCR : public RGWCoroutine {
           }
         }
       } else if (complete->retry) {
-        yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo,
+        yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                                               rgw::error_repo::encode_key(complete->bs, complete->gen),
                                               complete->timestamp));
         if (retcode < 0) {
@@ -1529,7 +1529,7 @@ class RGWDataIncrementalSyncFullObligationCR: public RGWCoroutine {
       if (retcode == -ENOENT) {
         // don't retry if bucket instance does not exist
         tn->log(10, SSTR("bucket instance or log layout does not exist on source for bucket " << source_bs.bucket));
-        yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo,
+        yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                                             error_marker, timestamp));
         return set_cr_done();
       } else if (retcode < 0) {
@@ -1544,7 +1544,7 @@ class RGWDataIncrementalSyncFullObligationCR: public RGWCoroutine {
 	  pool = sync_env->svc->zone->get_zone_params().log_pool;
           error_repo = datalog_oid_for_error_repo(sc, sync_env->driver, pool, source_bs);
           tn->log(10, SSTR("writing shard_id " << sid << " of gen " << each->gen << " to error repo for retry"));
-          yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo,
+          yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                             rgw::error_repo::encode_key(bs, each->gen),
 			    timestamp), sc->lcc.adj_concurrency(cct->_conf->rgw_data_sync_spawn_window),
                             [&](uint64_t stack_id, int ret) {
@@ -1563,7 +1563,7 @@ class RGWDataIncrementalSyncFullObligationCR: public RGWCoroutine {
                  });
 
       // once everything succeeds, remove the full sync obligation from the error repo
-      yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo,
+      yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                                             error_marker, timestamp));
       return set_cr_done();
     }
@@ -1648,7 +1648,7 @@ class RGWDataFullSyncSingleEntryCR : public RGWCoroutine {
       if (retcode < 0) {
         tn->log(10, SSTR("full sync: failed to read remote bucket info. Writing "
                         << source_bs.shard_id << " to error repo for retry"));
-        yield call(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo,
+        yield call(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                                             rgw::error_repo::encode_key(source_bs, std::nullopt),
                                             timestamp));
         if (retcode < 0) {
@@ -1670,7 +1670,7 @@ class RGWDataFullSyncSingleEntryCR : public RGWCoroutine {
           timestamp = timestamp_for_bucket_shard(sync_env->driver, sync_status, source_bs);
           if (retcode < 0) {
             tn->log(10, SSTR("Write " << source_bs.shard_id << " to error repo for retry"));
-            yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo,
+            yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo,
                 rgw::error_repo::encode_key(source_bs, each->gen),
 		timestamp), sc->lcc.adj_concurrency(cct->_conf->rgw_data_sync_spawn_window), std::nullopt);
           } else {
@@ -2016,7 +2016,7 @@ class RGWDataIncSyncShardCR : public RGWDataBaseSyncShardCR {
             }
             if (retcode < 0) {
               tn->log(1, SSTR("failed to parse bucket shard: " << error_marker));
-              spawn(rgw::error_repo::remove_cr(sc->env->driver->svc()->rados,
+              spawn(rgw::error_repo::remove_cr(sc->env->driver->getRados()->get_rados_handle(),
 					       error_repo, error_marker,
 					       entry_timestamp),
 		    false);
@@ -5160,8 +5160,11 @@ int RGWBucketShardIncrementalSyncCR::operate(const DoutPrefixProvider *dpp)
       }
       yield {
         // delete the shard status object
-        auto status_obj = sync_env->svc->rados->obj(marker_tracker.get_obj());
-        retcode = status_obj.open(dpp);
+        rgw_rados_ref status_obj;
+        retcode = rgw_get_rados_ref(dpp,
+				    sync_env->driver->getRados()->get_rados_handle(),
+				    marker_tracker.get_obj(),
+				    &status_obj);
         if (retcode < 0) {
           return set_cr_error(retcode);
         }
@@ -6009,7 +6012,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp)
 	      // use the error repo and sync status timestamp from the datalog shard corresponding to source_bs
               error_repo = datalog_oid_for_error_repo(sc, sc->env->driver,
 			   pool, source_bs);
-              yield call(rgw::error_repo::write_cr(sc->env->driver->svc()->rados, error_repo,
+              yield call(rgw::error_repo::write_cr(sc->env->driver->getRados()->get_rados_handle(), error_repo,
                                               rgw::error_repo::encode_key(source_bs, current_gen),
                                               ceph::real_clock::zero()));
               if (retcode < 0) {
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc
index 3f20ce26ba8a..269043f94df1 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.cc
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc
@@ -32,7 +32,6 @@
 #include "rgw_zone.h"
 #include "rgw_sal_rados.h"
 
-#include "services/svc_rados.h"
 #include "services/svc_zone.h"
 #include "services/svc_sys_obj.h"
 #include "services/svc_bi_rados.h"
@@ -108,8 +107,11 @@ int RGWObjExpStore::objexp_hint_add(const DoutPrefixProvider *dpp,
   cls_timeindex_add(op, utime_t(delete_at), keyext, hebl);
 
   string shard_name = objexp_hint_get_shardname(objexp_key_shard(obj_key, cct->_conf->rgw_objexp_hints_num_shards));
-  auto obj = rados_svc->obj(rgw_raw_obj(driver->svc()->zone->get_zone_params().log_pool, shard_name));
-  int r = obj.open(dpp);
+  rgw_rados_ref obj;
+  int r = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(),
+			    { driver->svc()->zone->get_zone_params().log_pool,
+			      shard_name },
+			    &obj);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl;
     return r;
@@ -131,8 +133,10 @@ int RGWObjExpStore::objexp_hint_list(const DoutPrefixProvider *dpp,
   cls_timeindex_list(op, utime_t(start_time), utime_t(end_time), marker, max_entries, entries,
         out_marker, truncated);
 
-  auto obj = rados_svc->obj(rgw_raw_obj(driver->svc()->zone->get_zone_params().log_pool, oid));
-  int r = obj.open(dpp);
+  rgw_rados_ref obj;
+  int r = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(),
+			    { driver->svc()->zone->get_zone_params().log_pool,
+			      oid }, &obj);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl;
     return r;
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.h b/src/rgw/driver/rados/rgw_object_expirer_core.h
index b616570b82f9..d23457fe75ff 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.h
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.h
@@ -30,19 +30,16 @@
 
 #include "rgw_sal_rados.h"
 
-class RGWSI_RADOS;
 class RGWSI_Zone;
 class RGWBucketInfo;
 class cls_timeindex_entry;
 
 class RGWObjExpStore {
   CephContext *cct;
-  RGWSI_RADOS *rados_svc;
   rgw::sal::RadosStore* driver;
 public:
-  RGWObjExpStore(CephContext *_cct, RGWSI_RADOS *_rados_svc, rgw::sal::RadosStore* _driver) : cct(_cct),
-                                                                                      rados_svc(_rados_svc),
-                                                                                      driver(_driver) {}
+  RGWObjExpStore(CephContext *_cct, rgw::sal::RadosStore* _driver) : cct(_cct),
+								     driver(_driver) {}
 
   int objexp_hint_add(const DoutPrefixProvider *dpp, 
                       const ceph::real_time& delete_at,
@@ -101,7 +98,7 @@ class RGWObjectExpirer {
 public:
   explicit RGWObjectExpirer(rgw::sal::Driver* _driver)
     : driver(_driver),
-      exp_store(_driver->ctx(), static_cast<rgw::sal::RadosStore*>(driver)->svc()->rados, static_cast<rgw::sal::RadosStore*>(driver)),
+      exp_store(_driver->ctx(), static_cast<rgw::sal::RadosStore*>(driver)),
       worker(NULL) {
   }
   ~RGWObjectExpirer() {
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 14661935a77c..67e8a3a998ef 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -126,8 +126,8 @@ void RadosWriter::add_write_hint(librados::ObjectWriteOperation& op) {
 
 int RadosWriter::set_stripe_obj(const rgw_raw_obj& raw_obj)
 {
-  stripe_obj = store->svc.rados->obj(raw_obj);
-  return stripe_obj.open(dpp);
+  return rgw_get_rados_ref(dpp, store->get_rados_handle(), raw_obj,
+			   &stripe_obj);
 }
 
 int RadosWriter::process(bufferlist&& bl, uint64_t offset)
@@ -145,8 +145,9 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset)
     op.write(offset, data);
   }
   constexpr uint64_t id = 0; // unused
-  auto& ref = stripe_obj.get_ref();
-  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+  auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
+						     std::move(op), y),
+		    cost, id);
   return process_completed(c, &written);
 }
 
@@ -160,8 +161,9 @@ int RadosWriter::write_exclusive(const bufferlist& data)
   op.write_full(data);
 
   constexpr uint64_t id = 0; // unused
-  auto& ref = stripe_obj.get_ref();
-  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+  auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
+						     std::move(op), y),
+		    cost, id);
   auto d = aio->drain();
   c.splice(c.end(), d);
   return process_completed(c, &written);
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index 9a21c0c793a3..ce163e36f777 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -18,7 +18,6 @@
 #include <optional>
 
 #include "rgw_putobj.h"
-#include "services/svc_rados.h"
 #include "services/svc_tier_rados.h"
 #include "rgw_sal.h"
 #include "rgw_obj_manifest.h"
@@ -70,7 +69,7 @@ class RadosWriter : public rgw::sal::DataProcessor {
   const RGWBucketInfo& bucket_info;
   RGWObjectCtx& obj_ctx;
   const rgw_obj head_obj;
-  RGWSI_RADOS::Obj stripe_obj; // current stripe object
+  rgw_rados_ref stripe_obj; // current stripe object
   RawObjSet written; // set of written objects for deletion
   const DoutPrefixProvider *dpp;
   optional_yield y;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 66d6b12338d1..592d7a828bc1 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1020,7 +1020,7 @@ void RGWRados::finalize()
 {
   /* Before joining any sync threads, drain outstanding requests &
    * mark the async_processor as going_down() */
-  if (svc.rados) {
+  if (svc.async_processor) {
     svc.async_processor->stop();
   }
 
@@ -1172,7 +1172,7 @@ int RGWRados::update_service_map(const DoutPrefixProvider *dpp, std::map<std::st
   return 0;
 }
 
-/** 
+/**
  * Initialize the RADOS instance and prepare to do other ops
  * Returns 0 on success, -ERR# on failure.
  */
@@ -1180,7 +1180,7 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
 {
   int ret;
 
-  /* 
+  /*
    * create sync module instance even if we don't run sync thread, might need it for radosgw-admin
    */
   sync_module = svc.sync_modules->get_sync_module();
@@ -2394,7 +2394,9 @@ bool RGWRados::obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_ob
 
 std::string RGWRados::get_cluster_fsid(const DoutPrefixProvider *dpp, optional_yield y)
 {
-  return svc.rados->cluster_fsid();
+  std::string s;
+  rados.cluster_fsid(&s);
+  return s;
 }
 
 int RGWRados::get_obj_head_ioctx(const DoutPrefixProvider *dpp,
@@ -4778,8 +4780,10 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
       ref_tag = tag + '\0';
       cls_refcount_get(op, ref_tag, true);
 
-      auto obj = svc.rados->obj(miter.get_location().get_raw_obj(this));
-      ret = obj.open(dpp);
+      rgw_rados_ref obj;
+      ret = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(),
+			      miter.get_location().get_raw_obj(this),
+			      &obj);
       if (ret < 0) {
         ldpp_dout(dpp, 0) << "failed to open rados context for " << obj << dendl;
         goto done_ret;
@@ -4787,8 +4791,9 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
 
       static constexpr uint64_t cost = 1; // 1 throttle unit per request
       static constexpr uint64_t id = 0; // ids unused
-      auto& ref = obj.get_ref();
-      rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+      rgw::AioResultList completed =
+	aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), y),
+		 cost, id);
       ret = rgw::check_for_errors(completed);
       all_results.splice(all_results.end(), completed);
       if (ret < 0) {
@@ -4855,19 +4860,20 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
       if (r.result < 0) {
         continue; // skip errors
       }
-      auto obj = svc.rados->obj(r.obj);
-      ret2 = obj.open(dpp);
+      rgw_rados_ref obj;
+      ret2 = rgw_get_rados_ref(dpp, get_rados_handle(), r.obj, &obj);
       if (ret2 < 0) {
         continue;
       }
-      auto& ref = obj.get_ref();
 
       ObjectWriteOperation op;
       cls_refcount_put(op, ref_tag, true);
 
       static constexpr uint64_t cost = 1; // 1 throttle unit per request
       static constexpr uint64_t id = 0; // ids unused
-      rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+      rgw::AioResultList completed =
+	aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), y),
+		 cost, id);
       ret2 = rgw::check_for_errors(completed);
       if (ret2 < 0) {
         ldpp_dout(dpp, 0) << "ERROR: cleanup after error failed to drop reference on obj=" << r.obj << dendl;
@@ -7156,8 +7162,9 @@ int RGWRados::get_obj_iterate_cb(const DoutPrefixProvider *dpp,
     }
   }
 
-  auto obj = d->rgwrados->svc.rados->obj(read_obj);
-  int r = obj.open(dpp);
+  rgw_rados_ref obj;
+  int r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj,
+			    &obj);
   if (r < 0) {
     ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl;
     return r;
@@ -7169,8 +7176,7 @@ int RGWRados::get_obj_iterate_cb(const DoutPrefixProvider *dpp,
   const uint64_t cost = len;
   const uint64_t id = obj_ofs; // use logical object offset for sorting replies
 
-  auto& ref = obj.get_ref();
-  auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id);
+  auto completed = d->aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), d->yield), cost, id);
 
   return d->flush(std::move(completed));
 }
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index f5d9b68b259e..986dc091163a 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -33,7 +33,6 @@
 #include "rgw_aio.h"
 #include "rgw_d3n_cacherequest.h"
 
-#include "services/svc_rados.h"
 #include "services/svc_bi_rados.h"
 #include "common/Throttle.h"
 #include "common/ceph_mutex.h"
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 12e9cd82aec5..5b78472dfe3b 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -17,7 +17,6 @@
 #include "services/svc_meta_be_otp.h"
 #include "services/svc_notify.h"
 #include "services/svc_otp.h"
-#include "services/svc_rados.h"
 #include "services/svc_zone.h"
 #include "services/svc_zone_utils.h"
 #include "services/svc_quota.h"
@@ -52,7 +51,7 @@ int RGWServices_Def::init(CephContext *cct,
 			  bool have_cache,
                           bool raw,
 			  bool run_sync,
-			  librados::Rados* radoshandle,
+			  librados::Rados* rados,
 			  optional_yield y,
                           const DoutPrefixProvider *dpp)
 {
@@ -70,7 +69,6 @@ int RGWServices_Def::init(CephContext *cct,
   meta_be_otp = std::make_unique<RGWSI_MetaBackend_OTP>(cct);
   notify = std::make_unique<RGWSI_Notify>(cct);
   otp = std::make_unique<RGWSI_OTP>(cct);
-  rados = std::make_unique<RGWSI_RADOS>(cct);
   zone = std::make_unique<RGWSI_Zone>(cct);
   zone_utils = std::make_unique<RGWSI_ZoneUtils>(cct);
   quota = std::make_unique<RGWSI_Quota>(cct);
@@ -90,7 +88,7 @@ int RGWServices_Def::init(CephContext *cct,
 
   async_processor->start();
   finisher->init();
-  bi_rados->init(zone.get(), radoshandle, bilog_rados.get(), datalog_rados.get());
+  bi_rados->init(zone.get(), rados, bilog_rados.get(), datalog_rados.get());
   bilog_rados->init(bi_rados.get());
   bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(),
                     bi_rados.get(), meta.get(), meta_be_sobj.get(),
@@ -99,28 +97,27 @@ int RGWServices_Def::init(CephContext *cct,
                          sysobj.get(),
                          sysobj_cache.get(),
                          bucket_sobj.get());
-  cls->init(zone.get(), radoshandle);
-  config_key_rados->init(radoshandle);
-  mdlog->init(radoshandle, zone.get(), sysobj.get(), cls.get(),
+  cls->init(zone.get(), rados);
+  config_key_rados->init(rados);
+  mdlog->init(rados, zone.get(), sysobj.get(), cls.get(),
 	      async_processor.get());
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
   meta_be_sobj->init(sysobj.get(), mdlog.get());
   meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get());
-  notify->init(zone.get(), radoshandle, finisher.get());
+  notify->init(zone.get(), rados, finisher.get());
   otp->init(zone.get(), meta.get(), meta_be_otp.get());
-  rados->init();
-  zone->init(sysobj.get(), radoshandle, sync_modules.get(), bucket_sync_sobj.get());
-  zone_utils->init(radoshandle, zone.get());
+  zone->init(sysobj.get(), rados, sync_modules.get(), bucket_sync_sobj.get());
+  zone_utils->init(rados, zone.get());
   quota->init(zone.get());
   sync_modules->init(zone.get());
-  sysobj_core->core_init(radoshandle, zone.get());
+  sysobj_core->core_init(rados, zone.get());
   if (have_cache) {
-    sysobj_cache->init(radoshandle, zone.get(), notify.get());
-    sysobj->init(radoshandle, sysobj_cache.get());
+    sysobj_cache->init(rados, zone.get(), notify.get());
+    sysobj->init(rados, sysobj_cache.get());
   } else {
-    sysobj->init(radoshandle, sysobj_core.get());
+    sysobj->init(rados, sysobj_core.get());
   }
-  user_rados->init(radoshandle, zone.get(), sysobj.get(), sysobj_cache.get(),
+  user_rados->init(rados, zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
   role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
 
@@ -140,12 +137,6 @@ int RGWServices_Def::init(CephContext *cct,
     }
   }
 
-  r = rados->start(y, dpp);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to start rados service (" << cpp_strerror(-r) << dendl;
-    return r;
-  }
-
   if (!raw) {
     r = zone->start(y, dpp);
     if (r < 0) {
@@ -155,7 +146,7 @@ int RGWServices_Def::init(CephContext *cct,
 
     r = datalog_rados->start(dpp, &zone->get_zone(),
 			     zone->get_zone_params(),
-			     rados->get_rados_handle());
+			     rados);
     if (r < 0) {
       ldpp_dout(dpp, 0) << "ERROR: failed to start datalog_rados service (" << cpp_strerror(-r) << dendl;
       return r;
@@ -306,18 +297,17 @@ void RGWServices_Def::shutdown()
   zone_utils->shutdown();
   zone->shutdown();
   async_processor->stop();
-  rados->shutdown();
 
   has_shutdown = true;
 }
 
 int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
-			 bool run_sync, librados::Rados* radoshandle,
+			 bool run_sync, librados::Rados* rados,
 			 optional_yield y, const DoutPrefixProvider *dpp)
 {
   cct = _cct;
 
-  int r = _svc.init(cct, have_cache, raw, run_sync, radoshandle, y, dpp);
+  int r = _svc.init(cct, have_cache, raw, run_sync, rados, y, dpp);
   if (r < 0) {
     return r;
   }
@@ -340,7 +330,6 @@ int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
   meta_be_otp = _svc.meta_be_otp.get();
   notify = _svc.notify.get();
   otp = _svc.otp.get();
-  rados = _svc.rados.get();
   zone = _svc.zone.get();
   zone_utils = _svc.zone_utils.get();
   quota = _svc.quota.get();
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index ad80a6c88266..7c05f043a47c 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -62,7 +62,6 @@ class RGWSI_MetaBackend_SObj;
 class RGWSI_MetaBackend_OTP;
 class RGWSI_Notify;
 class RGWSI_OTP;
-class RGWSI_RADOS;
 class RGWSI_Zone;
 class RGWSI_ZoneUtils;
 class RGWSI_Quota;
@@ -94,7 +93,6 @@ struct RGWServices_Def
   std::unique_ptr<RGWSI_MetaBackend_OTP> meta_be_otp;
   std::unique_ptr<RGWSI_Notify> notify;
   std::unique_ptr<RGWSI_OTP> otp;
-  std::unique_ptr<RGWSI_RADOS> rados;
   std::unique_ptr<RGWSI_Zone> zone;
   std::unique_ptr<RGWSI_ZoneUtils> zone_utils;
   std::unique_ptr<RGWSI_Quota> quota;
@@ -111,7 +109,7 @@ struct RGWServices_Def
   ~RGWServices_Def();
 
   int init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync,
-	   librados::Rados* radoshandle, optional_yield y,
+	   librados::Rados* rados, optional_yield y,
 	   const DoutPrefixProvider *dpp);
   void shutdown();
 };
@@ -141,7 +139,6 @@ struct RGWServices
   RGWSI_MetaBackend *meta_be_otp{nullptr};
   RGWSI_Notify *notify{nullptr};
   RGWSI_OTP *otp{nullptr};
-  RGWSI_RADOS *rados{nullptr};
   RGWSI_Zone *zone{nullptr};
   RGWSI_ZoneUtils *zone_utils{nullptr};
   RGWSI_Quota *quota{nullptr};
@@ -154,18 +151,18 @@ struct RGWServices
   RGWAsyncRadosProcessor* async_processor;
 
   int do_init(CephContext *cct, bool have_cache, bool raw_storage,
-	      bool run_sync, librados::Rados* radoshandle, optional_yield y,
+	      bool run_sync, librados::Rados* rados, optional_yield y,
 	      const DoutPrefixProvider *dpp);
 
   int init(CephContext *cct, bool have_cache, bool run_sync,
-	   librados::Rados* radoshandle, optional_yield y,
+	   librados::Rados* rados, optional_yield y,
 	   const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, false, run_sync, radoshandle, y, dpp);
+    return do_init(cct, have_cache, false, run_sync, rados, y, dpp);
   }
 
-  int init_raw(CephContext *cct, bool have_cache, librados::Rados* radoshandle,
+  int init_raw(CephContext *cct, bool have_cache, librados::Rados* rados,
 	       optional_yield y, const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, true, false, radoshandle, y, dpp);
+    return do_init(cct, have_cache, true, false, rados, y, dpp);
   }
   void shutdown() {
     _svc.shutdown();
diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.cc b/src/rgw/driver/rados/rgw_sync_error_repo.cc
index 44305b60b6b2..aca671039ee1 100644
--- a/src/rgw/driver/rados/rgw_sync_error_repo.cc
+++ b/src/rgw/driver/rados/rgw_sync_error_repo.cc
@@ -15,7 +15,6 @@
 #include "rgw_sync_error_repo.h"
 #include "rgw_coroutine.h"
 #include "rgw_sal.h"
-#include "services/svc_rados.h"
 #include "cls/cmpomap/client.h"
 
 namespace rgw::error_repo {
@@ -118,16 +117,17 @@ int remove(librados::ObjectWriteOperation& op,
 }
 
 class RGWErrorRepoWriteCR : public RGWSimpleCoroutine {
-  RGWSI_RADOS::Obj obj;
+  librados::Rados* rados;
+  const rgw_raw_obj& raw_obj;
   std::string key;
   ceph::real_time timestamp;
 
   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
  public:
-  RGWErrorRepoWriteCR(RGWSI_RADOS* rados, const rgw_raw_obj& raw_obj,
+  RGWErrorRepoWriteCR(librados::Rados* rados, const rgw_raw_obj& raw_obj,
                       const std::string& key, ceph::real_time timestamp)
-    : RGWSimpleCoroutine(rados->ctx()),
-      obj(rados->obj(raw_obj)),
+    : RGWSimpleCoroutine(static_cast<CephContext*>(rados->cct())),
+      raw_obj(raw_obj),
       key(key), timestamp(timestamp)
   {}
 
@@ -137,13 +137,14 @@ class RGWErrorRepoWriteCR : public RGWSimpleCoroutine {
     if (r < 0) {
       return r;
     }
-    r = obj.open(dpp);
+    rgw_rados_ref ref;
+    r = rgw_get_rados_ref(dpp, rados, raw_obj, &ref);
     if (r < 0) {
       return r;
     }
 
     cn = stack->create_completion_notifier();
-    return obj.aio_operate(cn->completion(), &op);
+    return ref.aio_operate(cn->completion(), &op);
   }
 
   int request_complete() override {
@@ -151,7 +152,7 @@ class RGWErrorRepoWriteCR : public RGWSimpleCoroutine {
   }
 };
 
-RGWCoroutine* write_cr(RGWSI_RADOS* rados,
+RGWCoroutine* write_cr(librados::Rados* rados,
                        const rgw_raw_obj& obj,
                        const std::string& key,
                        ceph::real_time timestamp)
@@ -161,16 +162,17 @@ RGWCoroutine* write_cr(RGWSI_RADOS* rados,
 
 
 class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine {
-  RGWSI_RADOS::Obj obj;
+  librados::Rados* rados;
+  const rgw_raw_obj& raw_obj;
   std::string key;
   ceph::real_time timestamp;
 
   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
  public:
-  RGWErrorRepoRemoveCR(RGWSI_RADOS* rados, const rgw_raw_obj& raw_obj,
+  RGWErrorRepoRemoveCR(librados::Rados* rados, const rgw_raw_obj& raw_obj,
                        const std::string& key, ceph::real_time timestamp)
-    : RGWSimpleCoroutine(rados->ctx()),
-      obj(rados->obj(raw_obj)),
+    : RGWSimpleCoroutine(static_cast<CephContext*>(rados->cct())),
+      raw_obj(raw_obj),
       key(key), timestamp(timestamp)
   {}
 
@@ -180,13 +182,14 @@ class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine {
     if (r < 0) {
       return r;
     }
-    r = obj.open(dpp);
+    rgw_rados_ref ref;
+    r = rgw_get_rados_ref(dpp, rados, raw_obj, &ref);
     if (r < 0) {
       return r;
     }
 
     cn = stack->create_completion_notifier();
-    return obj.aio_operate(cn->completion(), &op);
+    return ref.aio_operate(cn->completion(), &op);
   }
 
   int request_complete() override {
@@ -194,7 +197,7 @@ class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine {
   }
 };
 
-RGWCoroutine* remove_cr(RGWSI_RADOS* rados,
+RGWCoroutine* remove_cr(librados::Rados* rados,
                         const rgw_raw_obj& obj,
                         const std::string& key,
                         ceph::real_time timestamp)
diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.h b/src/rgw/driver/rados/rgw_sync_error_repo.h
index 60525d281f0f..7760c9fe198d 100644
--- a/src/rgw/driver/rados/rgw_sync_error_repo.h
+++ b/src/rgw/driver/rados/rgw_sync_error_repo.h
@@ -19,7 +19,6 @@
 #include "include/buffer_fwd.h"
 #include "common/ceph_time.h"
 
-class RGWSI_RADOS;
 class RGWCoroutine;
 struct rgw_raw_obj;
 struct rgw_bucket_shard;
@@ -42,7 +41,7 @@ ceph::real_time decode_value(const ceph::bufferlist& bl);
 int write(librados::ObjectWriteOperation& op,
           const std::string& key,
           ceph::real_time timestamp);
-RGWCoroutine* write_cr(RGWSI_RADOS* rados,
+RGWCoroutine* write_cr(librados::Rados* rados,
                        const rgw_raw_obj& obj,
                        const std::string& key,
                        ceph::real_time timestamp);
@@ -51,7 +50,7 @@ RGWCoroutine* write_cr(RGWSI_RADOS* rados,
 int remove(librados::ObjectWriteOperation& op,
            const std::string& key,
            ceph::real_time timestamp);
-RGWCoroutine* remove_cr(RGWSI_RADOS* rados,
+RGWCoroutine* remove_cr(librados::Rados* rados,
                         const rgw_raw_obj& obj,
                         const std::string& key,
                         ceph::real_time timestamp);
diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h
index 35f5daba2ed2..c6c11f8bc00b 100644
--- a/src/rgw/services/svc_bi_rados.h
+++ b/src/rgw/services/svc_bi_rados.h
@@ -21,7 +21,6 @@
 #include "rgw_tools.h"
 
 #include "svc_bi.h"
-#include "svc_rados.h"
 #include "svc_tier_rados.h"
 
 struct rgw_bucket_dir_header;
diff --git a/src/rgw/services/svc_bilog_rados.h b/src/rgw/services/svc_bilog_rados.h
index e9d5dbb5c0e2..e9c948d3fa27 100644
--- a/src/rgw/services/svc_bilog_rados.h
+++ b/src/rgw/services/svc_bilog_rados.h
@@ -1,4 +1,3 @@
-
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
@@ -19,11 +18,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
-
-
-
-
 class RGWSI_BILog_RADOS : public RGWServiceInstance
 {
 public:
diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc
index 101108981c32..740d9ab03072 100644
--- a/src/rgw/services/svc_cls.cc
+++ b/src/rgw/services/svc_cls.cc
@@ -3,7 +3,6 @@
 
 
 #include "svc_cls.h"
-#include "svc_rados.h"
 #include "svc_zone.h"
 
 #include "rgw_zone.h"
diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h
index c5ee361fc56d..6648714dbc8f 100644
--- a/src/rgw/services/svc_cls.h
+++ b/src/rgw/services/svc_cls.h
@@ -21,8 +21,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
-
 #include "driver/rados/rgw_tools.h"
 
 
diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc
index 603718dc96e5..46a158ceb77b 100644
--- a/src/rgw/services/svc_mdlog.cc
+++ b/src/rgw/services/svc_mdlog.cc
@@ -2,7 +2,6 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "svc_mdlog.h"
-#include "svc_rados.h"
 #include "svc_zone.h"
 #include "svc_sys_obj.h"
 
diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc
index 041354a376f7..5593dee9ae29 100644
--- a/src/rgw/services/svc_notify.cc
+++ b/src/rgw/services/svc_notify.cc
@@ -9,7 +9,6 @@
 #include "svc_notify.h"
 #include "svc_finisher.h"
 #include "svc_zone.h"
-#include "svc_rados.h"
 
 #include "rgw_zone.h"
 
diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h
index 618853d02c65..4f7e9d17ee37 100644
--- a/src/rgw/services/svc_notify.h
+++ b/src/rgw/services/svc_notify.h
@@ -5,8 +5,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
-
 #include "rgw_tools.h"
 
 
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
deleted file mode 100644
index c45853368680..000000000000
--- a/src/rgw/services/svc_rados.cc
+++ /dev/null
@@ -1,433 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
-
-#include "svc_rados.h"
-
-#include "include/rados/librados.hpp"
-#include "common/errno.h"
-#include "osd/osd_types.h"
-#include "rgw_tools.h"
-#include "rgw_cr_rados.h"
-
-#include "auth/AuthRegistry.h"
-
-#define dout_subsys ceph_subsys_rgw
-
-using namespace std;
-
-RGWSI_RADOS::RGWSI_RADOS(CephContext *cct) : RGWServiceInstance(cct)
-{
-}
-
-RGWSI_RADOS::~RGWSI_RADOS()
-{
-}
-
-int RGWSI_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp)
-{
-  int ret = rados.init_with_context(cct);
-  if (ret < 0) {
-    return ret;
-  }
-  ret = rados.connect();
-  if (ret < 0) {
-    return ret;
-  }
-
-  return 0;
-}
-
-void RGWSI_RADOS::shutdown()
-{
-  rados.shutdown();
-}
-
-librados::Rados* RGWSI_RADOS::get_rados_handle()
-{
-  return &rados;
-}
-
-std::string RGWSI_RADOS::cluster_fsid()
-{
-  std::string fsid;
-  (void) get_rados_handle()->cluster_fsid(&fsid);
-  return fsid;
-}
-
-uint64_t RGWSI_RADOS::instance_id()
-{
-  return get_rados_handle()->get_instance_id();
-}
-
-int RGWSI_RADOS::open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx,
-                               const OpenParams& params)
-{
-  return rgw_init_ioctx(dpp, get_rados_handle(), pool, io_ctx,
-                        params.create,
-                        params.mostly_omap);
-}
-
-int RGWSI_RADOS::pool_iterate(const DoutPrefixProvider *dpp,
-                              librados::IoCtx& io_ctx,
-                              librados::NObjectIterator& iter,
-                              uint32_t num, vector<rgw_bucket_dir_entry>& objs,
-                              const rgw::AccessListFilter& filter,
-                              bool *is_truncated)
-{
-  if (iter == io_ctx.nobjects_end())
-    return -ENOENT;
-
-  uint32_t i;
-
-  for (i = 0; i < num && iter != io_ctx.nobjects_end(); ++i, ++iter) {
-    rgw_bucket_dir_entry e;
-
-    string oid = iter->get_oid();
-    ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
-
-    // fill it in with initial values; we may correct later
-    if (filter && filter(oid, oid))
-      continue;
-
-    e.key = oid;
-    objs.push_back(e);
-  }
-
-  if (is_truncated)
-    *is_truncated = (iter != io_ctx.nobjects_end());
-
-  return objs.size();
-}
-
-RGWSI_RADOS::Obj::Obj(Pool& pool, const string& oid) : rados_svc(pool.rados_svc)
-{
-  ref.pool = pool;
-  ref.obj = rgw_raw_obj(pool.get_pool(), oid);
-}
-
-void RGWSI_RADOS::Obj::init(const rgw_raw_obj& obj)
-{
-  ref.pool = RGWSI_RADOS::Pool(rados_svc, obj.pool);
-  ref.obj = obj;
-}
-
-int RGWSI_RADOS::Obj::open(const DoutPrefixProvider *dpp)
-{
-  int r = ref.pool.open(dpp);
-  if (r < 0) {
-    return r;
-  }
-
-  ref.pool.ioctx().locator_set_key(ref.obj.loc);
-
-  return 0;
-}
-
-int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op,
-                              optional_yield y, int flags)
-{
-  return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, y, flags);
-}
-
-int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op,
-			      bufferlist *pbl, optional_yield y, int flags)
-{
-  return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, pbl, y, flags);
-}
-
-int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op)
-{
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op);
-}
-
-int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
-                                  bufferlist *pbl)
-{
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op, pbl);
-}
-
-int RGWSI_RADOS::Obj::watch(uint64_t *handle, librados::WatchCtx2 *ctx)
-{
-  return ref.pool.ioctx().watch2(ref.obj.oid, handle, ctx);
-}
-
-int RGWSI_RADOS::Obj::aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx)
-{
-  return ref.pool.ioctx().aio_watch(ref.obj.oid, c, handle, ctx);
-}
-
-int RGWSI_RADOS::Obj::unwatch(uint64_t handle)
-{
-  return ref.pool.ioctx().unwatch2(handle);
-}
-
-int RGWSI_RADOS::Obj::notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms,
-                             bufferlist *pbl, optional_yield y)
-{
-  return rgw_rados_notify(dpp, ref.pool.ioctx(), ref.obj.oid, bl, timeout_ms, pbl, y);
-}
-
-void RGWSI_RADOS::Obj::notify_ack(uint64_t notify_id,
-                                 uint64_t cookie,
-                                 bufferlist& bl)
-{
-  ref.pool.ioctx().notify_ack(ref.obj.oid, notify_id, cookie, bl);
-}
-
-uint64_t RGWSI_RADOS::Obj::get_last_version()
-{
-  return ref.pool.ioctx().get_last_version();
-}
-
-int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp)
-{
-  librados::Rados *rad = rados_svc->get_rados_handle();
-  int r = rad->pool_create(pool.name.c_str());
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "WARNING: pool_create returned " << r << dendl;
-    return r;
-  }
-  librados::IoCtx io_ctx;
-  r = rad->ioctx_create(pool.name.c_str(), io_ctx);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << r << dendl;
-    return r;
-  }
-  r = io_ctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "WARNING: application_enable returned " << r << dendl;
-    return r;
-  }
-  return 0;
-}
-
-int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp, const vector<rgw_pool>& pools, vector<int> *retcodes)
-{
-  vector<librados::PoolAsyncCompletion *> completions;
-  vector<int> rets;
-
-  librados::Rados *rad = rados_svc->get_rados_handle();
-  for (auto iter = pools.begin(); iter != pools.end(); ++iter) {
-    librados::PoolAsyncCompletion *c = librados::Rados::pool_async_create_completion();
-    completions.push_back(c);
-    auto& pool = *iter;
-    int ret = rad->pool_create_async(pool.name.c_str(), c);
-    rets.push_back(ret);
-  }
-
-  vector<int>::iterator riter;
-  vector<librados::PoolAsyncCompletion *>::iterator citer;
-
-  bool error = false;
-  ceph_assert(rets.size() == completions.size());
-  for (riter = rets.begin(), citer = completions.begin(); riter != rets.end(); ++riter, ++citer) {
-    int r = *riter;
-    librados::PoolAsyncCompletion *c = *citer;
-    if (r == 0) {
-      c->wait();
-      r = c->get_return_value();
-      if (r < 0) {
-        ldpp_dout(dpp, 0) << "WARNING: async pool_create returned " << r << dendl;
-        error = true;
-      }
-    }
-    c->release();
-    retcodes->push_back(r);
-  }
-  if (error) {
-    return 0;
-  }
-
-  std::vector<librados::IoCtx> io_ctxs;
-  retcodes->clear();
-  for (auto pool : pools) {
-    io_ctxs.emplace_back();
-    int ret = rad->ioctx_create(pool.name.c_str(), io_ctxs.back());
-    if (ret < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << ret << dendl;
-      error = true;
-    }
-    retcodes->push_back(ret);
-  }
-  if (error) {
-    return 0;
-  }
-
-  completions.clear();
-  for (auto &io_ctx : io_ctxs) {
-    librados::PoolAsyncCompletion *c =
-      librados::Rados::pool_async_create_completion();
-    completions.push_back(c);
-    int ret = io_ctx.application_enable_async(pg_pool_t::APPLICATION_NAME_RGW,
-                                              false, c);
-    ceph_assert(ret == 0);
-  }
-
-  retcodes->clear();
-  for (auto c : completions) {
-    c->wait();
-    int ret = c->get_return_value();
-    if (ret == -EOPNOTSUPP) {
-      ret = 0;
-    } else if (ret < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: async application_enable returned " << ret
-                    << dendl;
-      error = true;
-    }
-    c->release();
-    retcodes->push_back(ret);
-  }
-  return 0;
-}
-
-int RGWSI_RADOS::Pool::lookup()
-{
-  librados::Rados *rad = rados_svc->get_rados_handle();
-  int ret = rad->pool_lookup(pool.name.c_str());
-  if (ret < 0) {
-    return ret;
-  }
-
-  return 0;
-}
-
-int RGWSI_RADOS::Pool::open(const DoutPrefixProvider *dpp, const OpenParams& params)
-{
-  return rados_svc->open_pool_ctx(dpp, pool, state.ioctx, params);
-}
-
-int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& marker, rgw::AccessListFilter filter)
-{
-  if (ctx.initialized) {
-    return -EINVAL;
-  }
-
-  if (!pool) {
-    return -EINVAL;
-  }
-
-  int r = pool->rados_svc->open_pool_ctx(dpp, pool->pool, ctx.ioctx);
-  if (r < 0) {
-    return r;
-  }
-
-  librados::ObjectCursor oc;
-  if (!oc.from_str(marker)) {
-    ldpp_dout(dpp, 10) << "failed to parse cursor: " << marker << dendl;
-    return -EINVAL;
-  }
-
-  try {
-    ctx.iter = ctx.ioctx.nobjects_begin(oc);
-  } catch (const std::system_error& e) {
-    r = -e.code().value();
-    ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what()
-       << ", returning " << r << dendl;
-    return r;
-  } catch (const std::exception& e) {
-    ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what()
-       << ", returning -5" << dendl;
-    return -EIO;
-  }
-  ctx.filter = std::move(filter);
-  ctx.initialized = true;
-
-  return 0;
-}
-
-int RGWSI_RADOS::Pool::List::get_next(const DoutPrefixProvider *dpp,
-                                      int max,
-                                      std::vector<string> *oids,
-                                      bool *is_truncated)
-{
-  if (!ctx.initialized) {
-    return -EINVAL;
-  }
-  vector<rgw_bucket_dir_entry> objs;
-  int r = pool->rados_svc->pool_iterate(dpp, ctx.ioctx, ctx.iter, max, objs, ctx.filter, is_truncated);
-  if (r < 0) {
-    if(r != -ENOENT) {
-      ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl;
-    }
-    return r;
-  }
-
-  for (auto& o : objs) {
-    oids->push_back(o.key.name);
-  }
-
-  return oids->size();
-}
-
-RGWSI_RADOS::Obj RGWSI_RADOS::Handle::obj(const rgw_raw_obj& o)
-{
-  return RGWSI_RADOS::Obj(rados_svc, o);
-}
-int RGWSI_RADOS::Handle::watch_flush()
-{
-  librados::Rados *rad = rados_svc->get_rados_handle();
-  return rad->watch_flush();
-}
-
-int RGWSI_RADOS::Handle::mon_command(std::string cmd,
-                                     const bufferlist& inbl,
-                                     bufferlist *outbl,
-                                     std::string *outs)
-{
-  librados::Rados *rad = rados_svc->get_rados_handle();
-  return rad->mon_command(cmd, inbl, outbl, outs);
-}
-
-int RGWSI_RADOS::Pool::List::get_marker(string *marker)
-{
-  if (!ctx.initialized) {
-    return -EINVAL;
-  }
-
-  *marker = ctx.iter.get_cursor().to_str();
-  return 0;
-}
-
-int RGWSI_RADOS::clog_warn(const string& msg)
-{
-  string cmd =
-    "{"
-      "\"prefix\": \"log\", "
-      "\"level\": \"warn\", "
-      "\"logtext\": [\"" + msg + "\"]"
-    "}";
-
-  bufferlist inbl;
-  auto h = handle();
-  return h.mon_command(cmd, inbl, nullptr, nullptr);
-}
-
-bool RGWSI_RADOS::check_secure_mon_conn(const DoutPrefixProvider *dpp) const
-{
-  AuthRegistry reg(cct);
-
-  reg.refresh_config();
-
-  std::vector<uint32_t> methods;
-  std::vector<uint32_t> modes;
-
-  reg.get_supported_methods(CEPH_ENTITY_TYPE_MON, &methods, &modes);
-  ldpp_dout(dpp, 20) << __func__ << "(): auth registry supported: methods=" << methods << " modes=" << modes << dendl;
-
-  for (auto method : methods) {
-    if (!reg.is_secure_method(method)) {
-      ldpp_dout(dpp, 20) << __func__ << "(): method " << method << " is insecure" << dendl;
-      return false;
-    }
-  }
-
-  for (auto mode : modes) {
-    if (!reg.is_secure_mode(mode)) {
-      ldpp_dout(dpp, 20) << __func__ << "(): mode " << mode << " is insecure" << dendl;
-      return false;
-    }
-  }
-
-  return true;
-}
-
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
deleted file mode 100644
index bed6522e2ae2..000000000000
--- a/src/rgw/services/svc_rados.h
+++ /dev/null
@@ -1,232 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
-
-#pragma once
-
-#include "rgw_service.h"
-
-#include "include/rados/librados.hpp"
-#include "common/async/yield_context.h"
-
-#include "rgw_tools.h"
-
-class RGWAsyncRadosProcessor;
-
-class RGWSI_RADOS : public RGWServiceInstance
-{
-  librados::Rados rados;
-
-  int do_start(optional_yield, const DoutPrefixProvider *dpp) override;
-
-public:
-  struct OpenParams {
-    bool create{true};
-    bool mostly_omap{false};
-
-    OpenParams() {}
-
-    OpenParams& set_create(bool _create) {
-      create = _create;
-      return *this;
-    }
-    OpenParams& set_mostly_omap(bool _mostly_omap) {
-      mostly_omap = _mostly_omap;
-      return *this;
-    }
-  };
-
-private:
-  int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx,
-                    const OpenParams& params = {});
-  int pool_iterate(const DoutPrefixProvider *dpp,
-                   librados::IoCtx& ioctx,
-                   librados::NObjectIterator& iter,
-                   uint32_t num, std::vector<rgw_bucket_dir_entry>& objs,
-                   const rgw::AccessListFilter& filter,
-                   bool *is_truncated);
-
-public:
-  RGWSI_RADOS(CephContext *cct);
-  ~RGWSI_RADOS();
-  librados::Rados* get_rados_handle();
-
-  void init() {}
-  void shutdown() override;
-
-  std::string cluster_fsid();
-  uint64_t instance_id();
-  bool check_secure_mon_conn(const DoutPrefixProvider *dpp) const;
-
-  int clog_warn(const std::string& msg);
-
-  class Handle;
-
-  class Pool {
-    friend class RGWSI_RADOS;
-    friend Handle;
-    friend class Obj;
-
-    RGWSI_RADOS *rados_svc{nullptr};
-    rgw_pool pool;
-
-    struct State {
-      librados::IoCtx ioctx;
-    } state;
-
-    Pool(RGWSI_RADOS *_rados_svc,
-         const rgw_pool& _pool) : rados_svc(_rados_svc),
-                                  pool(_pool) {}
-
-    Pool(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {}
-  public:
-    Pool() {}
-
-    int create(const DoutPrefixProvider *dpp);
-    int create(const DoutPrefixProvider *dpp, const std::vector<rgw_pool>& pools, std::vector<int> *retcodes);
-    int lookup();
-    int open(const DoutPrefixProvider *dpp, const OpenParams& params = {});
-
-    const rgw_pool& get_pool() {
-      return pool;
-    }
-
-    librados::IoCtx& ioctx() & {
-      return state.ioctx;
-    }
-
-    librados::IoCtx&& ioctx() && {
-      return std::move(state.ioctx);
-    }
-
-    struct List {
-      Pool *pool{nullptr};
-
-      struct Ctx {
-        bool initialized{false};
-        librados::IoCtx ioctx;
-        librados::NObjectIterator iter;
-	rgw::AccessListFilter filter;
-      } ctx;
-
-      List() {}
-      List(Pool *_pool) : pool(_pool) {}
-
-      int init(const DoutPrefixProvider *dpp, const std::string& marker,
-	       rgw::AccessListFilter filter);
-      int get_next(const DoutPrefixProvider *dpp, int max,
-                   std::vector<std::string> *oids,
-                   bool *is_truncated);
-
-      int get_marker(std::string *marker);
-    };
-
-    List op() {
-      return List(this);
-    }
-
-    friend List;
-  };
-
-
-  struct rados_ref {
-    RGWSI_RADOS::Pool pool;
-    rgw_raw_obj obj;
-  };
-
-  class Obj {
-    friend class RGWSI_RADOS;
-    friend class Handle;
-
-    RGWSI_RADOS *rados_svc{nullptr};
-    rados_ref ref;
-
-    void init(const rgw_raw_obj& obj);
-
-    Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj)
-      : rados_svc(_rados_svc) {
-      init(_obj);
-    }
-
-    Obj(Pool& pool, const std::string& oid);
-
-  public:
-    Obj() {}
-
-    int open(const DoutPrefixProvider *dpp);
-
-    int operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op, optional_yield y,
-		int flags = 0);
-    int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, bufferlist *pbl,
-                optional_yield y, int flags = 0);
-    int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op);
-    int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
-                    bufferlist *pbl);
-
-    int watch(uint64_t *handle, librados::WatchCtx2 *ctx);
-    int aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx);
-    int unwatch(uint64_t handle);
-    int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms,
-               bufferlist *pbl, optional_yield y);
-    void notify_ack(uint64_t notify_id,
-                    uint64_t cookie,
-                    bufferlist& bl);
-
-    uint64_t get_last_version();
-
-    rados_ref& get_ref() { return ref; }
-    const rados_ref& get_ref() const { return ref; }
-
-    const rgw_raw_obj& get_raw_obj() const {
-      return ref.obj;
-    }
-  };
-
-  class Handle {
-    friend class RGWSI_RADOS;
-
-    RGWSI_RADOS *rados_svc{nullptr};
-
-    Handle(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {}
-  public:
-    Obj obj(const rgw_raw_obj& o);
-
-    Pool pool(const rgw_pool& p) {
-      return Pool(rados_svc, p);
-    }
-
-    int watch_flush();
-
-    int mon_command(std::string cmd,
-                    const bufferlist& inbl,
-                    bufferlist *outbl,
-                    std::string *outs);
-  };
-
-  Handle handle() {
-    return Handle(this);
-  }
-
-  Obj obj(const rgw_raw_obj& o) {
-    return Obj(this, o);
-  }
-
-  Obj obj(Pool& pool, const std::string& oid) {
-    return Obj(pool, oid);
-  }
-
-  Pool pool() {
-    return Pool(this);
-  }
-
-  Pool pool(const rgw_pool& p) {
-    return Pool(this, p);
-  }
-
-  friend Obj;
-  friend Pool;
-  friend Pool::List;
-};
-
-inline std::ostream& operator<<(std::ostream& out, const RGWSI_RADOS::Obj& obj) {
-  return out << obj.get_raw_obj();
-}
diff --git a/src/rgw/services/svc_sys_obj.cc b/src/rgw/services/svc_sys_obj.cc
index 310e60514d2c..38e7acdfe0be 100644
--- a/src/rgw/services/svc_sys_obj.cc
+++ b/src/rgw/services/svc_sys_obj.cc
@@ -3,7 +3,6 @@
 
 #include "svc_sys_obj.h"
 #include "svc_sys_obj_core.h"
-#include "svc_rados.h"
 #include "svc_zone.h"
 
 #include "rgw_zone.h"
diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h
index 3739d68d150e..0bba2fbe01dc 100644
--- a/src/rgw/services/svc_sys_obj.h
+++ b/src/rgw/services/svc_sys_obj.h
@@ -7,7 +7,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
 #include "svc_sys_obj_types.h"
 #include "svc_sys_obj_core_types.h"
 
diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc
index 68878cdcceb2..fb9c7edce238 100644
--- a/src/rgw/services/svc_sys_obj_core.cc
+++ b/src/rgw/services/svc_sys_obj_core.cc
@@ -2,7 +2,6 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "svc_sys_obj_core.h"
-#include "svc_rados.h"
 #include "svc_zone.h"
 
 #include "rgw_tools.h"
diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h
index 1b34d7078317..d9dfc02f01bc 100644
--- a/src/rgw/services/svc_sys_obj_core.h
+++ b/src/rgw/services/svc_sys_obj_core.h
@@ -5,7 +5,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
 #include "svc_sys_obj.h"
 #include "svc_sys_obj_core_types.h"
 
diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h
index 7bec012f5c29..404f0fdde686 100644
--- a/src/rgw/services/svc_sys_obj_core_types.h
+++ b/src/rgw/services/svc_sys_obj_core_types.h
@@ -7,7 +7,6 @@
 #include "rgw_tools.h"
 #include "rgw_service.h"
 
-#include "svc_rados.h"
 #include "svc_sys_obj_types.h"
 
 
diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h
index f7424b410387..0327e4658818 100644
--- a/src/rgw/services/svc_tier_rados.h
+++ b/src/rgw/services/svc_tier_rados.h
@@ -20,8 +20,6 @@
 
 #include "rgw_service.h"
 
-#include "svc_rados.h"
-
 extern const std::string MP_META_SUFFIX;
 
 class RGWMPObj {
diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc
index a44d53727b16..70cf40eb6cb6 100644
--- a/src/rgw/services/svc_zone.cc
+++ b/src/rgw/services/svc_zone.cc
@@ -2,7 +2,6 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "svc_zone.h"
-#include "svc_rados.h"
 #include "svc_sys_obj.h"
 #include "svc_sync_modules.h"
 
diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc
index be9f861f02cc..c809974e6d5f 100644
--- a/src/rgw/services/svc_zone_utils.cc
+++ b/src/rgw/services/svc_zone_utils.cc
@@ -2,7 +2,6 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "svc_zone_utils.h"
-#include "svc_rados.h"
 #include "svc_zone.h"
 
 #undef FMT_HEADER_ONLY

From ea033fe8607c2b31892536afc3f08f3009b24139 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Wed, 29 Nov 2023 11:25:30 -0500
Subject: [PATCH 0778/2492] qa/workunits/rbd/cli_generic.sh: narrow race window

... when checking whether a rbd_support module command fails after
blocklisting the module's client.

In tests that check the recovery of the rbd_support module after its
client is blocklisted, the rbd_support module's client is
blocklisted using the `osd blocklist add` command. Next,
`osd blocklist ls` command is issued to confirm that the client is
blocklisted. A rbd_support module command is then issued and expected
to fail in order to verify that the blocklisting has affected the
rbd_support module's operations. Sometimes it was observed that before
this rbd_support module command reached the ceph-mgr, the rbd_support
module detected the blocklisting, recovered from it, and was able to
serve the command. To reduce the race window that occurs when trying to
verify that the rbd_support module's operation is affected by client
blocklisting, get rid of the `osd blocklist ls` command.

Fixes: https://tracker.ceph.com/issues/63673
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 qa/workunits/rbd/cli_generic.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/qa/workunits/rbd/cli_generic.sh b/qa/workunits/rbd/cli_generic.sh
index 57279d26dcee..c35bbe8f83e6 100755
--- a/qa/workunits/rbd/cli_generic.sh
+++ b/qa/workunits/rbd/cli_generic.sh
@@ -1261,7 +1261,6 @@ test_trash_purge_schedule_recovery() {
 	jq 'select(.name == "rbd_support")' |
 	jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
     ceph osd blocklist add $CLIENT_ADDR
-    ceph osd blocklist ls | grep $CLIENT_ADDR
 
     # Check that you can add a trash purge schedule after a few retries
     expect_fail rbd trash purge schedule add -p rbd3 10m
@@ -1420,7 +1419,6 @@ test_mirror_snapshot_schedule_recovery() {
 	jq 'select(.name == "rbd_support")' |
 	jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
     ceph osd blocklist add $CLIENT_ADDR
-    ceph osd blocklist ls | grep $CLIENT_ADDR
 
     # Check that you can add a mirror snapshot schedule after a few retries
     expect_fail rbd mirror snapshot schedule add -p rbd3/ns1 --image test1 2m
@@ -1529,7 +1527,6 @@ test_perf_image_iostat_recovery() {
 	jq 'select(.name == "rbd_support")' |
 	jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
     ceph osd blocklist add $CLIENT_ADDR
-    ceph osd blocklist ls | grep $CLIENT_ADDR
 
     expect_fail rbd perf image iostat --format json rbd3/ns
     sleep 10
@@ -1661,7 +1658,6 @@ test_tasks_recovery() {
 	jq 'select(.name == "rbd_support")' |
 	jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
     ceph osd blocklist add $CLIENT_ADDR
-    ceph osd blocklist ls | grep $CLIENT_ADDR
 
     expect_fail ceph rbd task add flatten rbd2/clone1
     sleep 10

From d3f1a0e1c0b98b9f1251837ecc8edc367e590dad Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 27 Nov 2023 15:04:42 -0500
Subject: [PATCH 0779/2492] python-common/drive_selection: fix limit with
 existing devices

When devices have already been used for OSDs, they are still
allowed to pass filtering as they are still needed for the
resulting ceph-volume lvm batch command. This was causing an
issue with limit however. Limit adds the devices we've found
that match the filter and existing OSD daemons tied to the spec.
This allows double counting of devices that hae been used for
OSDs, as they're counted in terms of being an existing device
and that they match the filter. To avoid this issue, devices
should only be counted towards the limit if they are not already
part of an OSD.

An additional note: The limit feature is only applied for
data devices, so there is no need to worry about the effect
of this change on selection of db, wal, or journal devices.
Also, we would still want to not count these devices if they
did end up passing the data device filter but had been used
for a db/wal/journal device previously.

Fixes: https://tracker.ceph.com/issues/63525

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/services/osd.py        |  9 +++-
 src/pybind/mgr/cephadm/tests/test_cephadm.py  |  7 +++-
 .../deployment/drive_selection/selector.py    | 24 +++++++----
 .../ceph/tests/test_disk_selector.py          | 41 ++++++++++++++++++-
 4 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py
index 3616d42deee4..75b3fc58c761 100644
--- a/src/pybind/mgr/cephadm/services/osd.py
+++ b/src/pybind/mgr/cephadm/services/osd.py
@@ -319,11 +319,16 @@ def generate_previews(self, osdspecs: List[DriveGroupSpec], for_host: str) -> Li
                             logger.exception('Cannot decode JSON: \'%s\'' % ' '.join(out))
                             concat_out = {}
                         notes = []
-                        if osdspec.data_devices is not None and osdspec.data_devices.limit and len(concat_out) < osdspec.data_devices.limit:
+                        if (
+                            osdspec.data_devices is not None
+                            and osdspec.data_devices.limit
+                            and (len(concat_out) + ds.existing_daemons) < osdspec.data_devices.limit
+                        ):
                             found = len(concat_out)
                             limit = osdspec.data_devices.limit
                             notes.append(
-                                f'NOTE: Did not find enough disks matching filter on host {host} to reach data device limit (Found: {found} | Limit: {limit})')
+                                f'NOTE: Did not find enough disks matching filter on host {host} to reach data device limit\n'
+                                f'(New Devices: {found} | Existing Matching Daemons: {ds.existing_daemons} | Limit: {limit})')
                         ret_all.append({'data': concat_out,
                                         'osdspec': osdspec.service_id,
                                         'host': host,
diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py
index ebdf7eea5aa0..2477de13e00a 100644
--- a/src/pybind/mgr/cephadm/tests/test_cephadm.py
+++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py
@@ -1193,7 +1193,8 @@ def test_create_noncollocated_osd(self, cephadm_module):
     @mock.patch('cephadm.services.osd.OSDService.driveselection_to_ceph_volume')
     @mock.patch('cephadm.services.osd.OsdIdClaims.refresh', lambda _: None)
     @mock.patch('cephadm.services.osd.OsdIdClaims.get', lambda _: {})
-    def test_limit_not_reached(self, d_to_cv, _run_cv_cmd, cephadm_module):
+    @mock.patch('cephadm.inventory.HostCache.get_daemons_by_service')
+    def test_limit_not_reached(self, _get_daemons_by_service, d_to_cv, _run_cv_cmd, cephadm_module):
         with with_host(cephadm_module, 'test'):
             dg = DriveGroupSpec(placement=PlacementSpec(host_pattern='test'),
                                 data_devices=DeviceSelection(limit=5, rotational=1),
@@ -1203,12 +1204,14 @@ def test_limit_not_reached(self, d_to_cv, _run_cv_cmd, cephadm_module):
                 '[{"data": "/dev/vdb", "data_size": "50.00 GB", "encryption": "None"}, {"data": "/dev/vdc", "data_size": "50.00 GB", "encryption": "None"}]']
             d_to_cv.return_value = 'foo'
             _run_cv_cmd.side_effect = async_side_effect((disks_found, '', 0))
+            _get_daemons_by_service.return_value = [DaemonDescription(daemon_type='osd', hostname='test', service_name='not_enough')]
             preview = cephadm_module.osd_service.generate_previews([dg], 'test')
 
             for osd in preview:
                 assert 'notes' in osd
                 assert osd['notes'] == [
-                    'NOTE: Did not find enough disks matching filter on host test to reach data device limit (Found: 2 | Limit: 5)']
+                    ('NOTE: Did not find enough disks matching filter on host test to reach '
+                     'data device limit\n(New Devices: 2 | Existing Matching Daemons: 1 | Limit: 5)')]
 
     @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
     def test_prepare_drivegroup(self, cephadm_module):
diff --git a/src/python-common/ceph/deployment/drive_selection/selector.py b/src/python-common/ceph/deployment/drive_selection/selector.py
index 1b3bfbb4ee3c..31e330432cd1 100644
--- a/src/python-common/ceph/deployment/drive_selection/selector.py
+++ b/src/python-common/ceph/deployment/drive_selection/selector.py
@@ -53,9 +53,12 @@ def journal_devices(self):
         # type: () -> List[Device]
         return self._journal
 
-    def _limit_reached(self, device_filter, len_devices,
-                       disk_path):
-        # type: (DeviceSelection, int, str) -> bool
+    def _limit_reached(
+        self,
+        device_filter: DeviceSelection,
+        devices: List[Device],
+        disk_path: str
+    ) -> bool:
         """ Check for the <limit> property and apply logic
 
         If a limit is set in 'device_attrs' we have to stop adding
@@ -63,14 +66,21 @@ def _limit_reached(self, device_filter, len_devices,
 
         If limit is set (>0) and len(devices) >= limit
 
-        :param int len_devices: Length of the already populated device set/list
+        :param List[Device] devices: Already populated device set/list
         :param str disk_path: The disk identifier (for logging purposes)
         :return: True/False if the device should be added to the list of devices
         :rtype: bool
         """
         limit = device_filter.limit or 0
-
-        if limit > 0 and (len_devices + self.existing_daemons >= limit):
+        # If device A is being used for an OSD already, it can still
+        # match the filter (this is necessary as we still want the
+        # device in the resulting ceph-volume lvm batch command).
+        # If that is the case, we don't want to count the device
+        # towards the limit as it will already be counted through the
+        # existing daemons
+        non_ceph_devices = [d for d in devices if not d.ceph_device]
+
+        if limit > 0 and (len(non_ceph_devices) + self.existing_daemons >= limit):
             logger.debug("Refuse to add {} due to limit policy of <{}>".format(
                 disk_path, limit))
             return True
@@ -147,7 +157,7 @@ def assign_devices(self, device_filter):
                 continue
 
             # break on this condition.
-            if self._limit_reached(device_filter, len(devices), disk.path):
+            if self._limit_reached(device_filter, devices, disk.path):
                 logger.debug("Ignoring disk {}. Limit reached".format(
                     disk.path))
                 break
diff --git a/src/python-common/ceph/tests/test_disk_selector.py b/src/python-common/ceph/tests/test_disk_selector.py
index b08236130e06..03bfcbe16c90 100644
--- a/src/python-common/ceph/tests/test_disk_selector.py
+++ b/src/python-common/ceph/tests/test_disk_selector.py
@@ -557,4 +557,43 @@ def test_disk_selection_raise(self):
         inventory = _mk_inventory(_mk_device(rotational=True)*2)
         m = 'Failed to validate OSD spec "foobar.data_devices": No filters applied'
         with pytest.raises(DriveGroupValidationError, match=m):
-            drive_selection.DriveSelection(spec, inventory)
\ No newline at end of file
+            drive_selection.DriveSelection(spec, inventory)
+
+
+class TestDeviceSelectionLimit:
+
+    def test_limit_existing_devices(self):
+        # Initial setup for this test is meant to be that /dev/sda
+        # is already being used for an OSD, hence it being marked
+        # as a ceph_device. /dev/sdb and /dev/sdc are not being used
+        # for OSDs yet. The limit will be set to 2 and the DriveSelection
+        # is set to have 1 pre-existing device (corresponding to /dev/sda)
+        dev_a = Device('/dev/sda', ceph_device=True, available=False)
+        dev_b = Device('/dev/sdb', ceph_device=False, available=True)
+        dev_c = Device('/dev/sdc', ceph_device=False, available=True)
+        all_devices: List[Device] = [dev_a, dev_b, dev_c]
+        processed_devices: List[Device] = []
+        filter = DeviceSelection(all=True, limit=2)
+        dgs = DriveGroupSpec(data_devices=filter)
+        ds = drive_selection.DriveSelection(dgs, all_devices, existing_daemons=1)
+
+        # Check /dev/sda. It's already being used for an OSD and will
+        # be counted in existing_daemons. This check should return False
+        # as we are not over the limit.
+        assert not ds._limit_reached(filter, processed_devices, '/dev/sda')
+        processed_devices.append(dev_a)
+
+        # We should still not be over the limit here with /dev/sdb since
+        # we will have only one pre-existing daemon /dev/sdb itself. This
+        # case previously failed as devices that contributed to existing_daemons
+        # would be double counted both as devices and daemons.
+        assert not ds._limit_reached(filter, processed_devices, '/dev/sdb')
+        processed_devices.append(dev_b)
+
+        # Now, with /dev/sdb and the pre-existing daemon taking up the 2
+        # slots, /dev/sdc should be rejected for us being over the limit.
+        assert ds._limit_reached(filter, processed_devices, '/dev/sdc')
+
+        # DriveSelection does device assignment on initialization. Let's check
+        # it picked up the expected devices
+        assert ds._data == [dev_a, dev_b]

From d63d96f8bcda08990bc3be28c9a835d6c836f0a3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 29 Nov 2023 20:12:45 -0500
Subject: [PATCH 0780/2492] ceph.spec.in: enable support for cephfs-shell by
 default via EPEL9

The required packages [1,2] are now at the right version and readily available.

[1] https://packages.fedoraproject.org/pkgs/python-cmd2/python3-cmd2/epel-9.html
[2] https://packages.fedoraproject.org/pkgs/python-colorama/python3-colorama/epel-9.html

Fixes: https://tracker.ceph.com/issues/43393
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 ceph.spec.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 3807f48280e9..2420c762b9aa 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -98,7 +98,7 @@
 %else
 %bcond_without jaeger
 %endif
-%if 0%{?fedora} || 0%{?suse_version} >= 1500
+%if 0%{?fedora} || 0%{?suse_version} >= 1500 || 0%{?rhel} >= 9
 # distros that ship cmd2 and/or colorama
 %bcond_without cephfs_shell
 %else

From b12bdfeee3647b495db87be5f3d0abcb3ca77034 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 29 Nov 2023 20:20:54 -0500
Subject: [PATCH 0781/2492] qa: add cephfs-shell to installed rpm packages

For CephFS QA.

Fixes: https://tracker.ceph.com/issues/43393
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/begin/0-install.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/cephfs/begin/0-install.yaml b/qa/cephfs/begin/0-install.yaml
index 413958f1111a..c995efca941d 100644
--- a/qa/cephfs/begin/0-install.yaml
+++ b/qa/cephfs/begin/0-install.yaml
@@ -3,6 +3,7 @@ tasks:
       extra_packages:
         rpm:
         - python3-cephfs
+        - cephfs-shell
         - cephfs-top
         - cephfs-mirror
         deb:

From 215d96c1d1533836d313be54864c1b0525440b4c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 29 Nov 2023 20:21:45 -0500
Subject: [PATCH 0782/2492] qa: test fs:shell on all distros

This is a simple sub-suite that has one job. Always schedule on all supported distros.

Fixes: https://tracker.ceph.com/issues/43393
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/suites/fs/shell/distro                  | 2 +-
 qa/suites/fs/shell/tasks/cephfs-shell.yaml | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/qa/suites/fs/shell/distro b/qa/suites/fs/shell/distro
index 0862b4457b37..23d9e9be86f0 120000
--- a/qa/suites/fs/shell/distro
+++ b/qa/suites/fs/shell/distro
@@ -1 +1 @@
-.qa/distros/supported-random-distro$
\ No newline at end of file
+.qa/distros/supported
\ No newline at end of file
diff --git a/qa/suites/fs/shell/tasks/cephfs-shell.yaml b/qa/suites/fs/shell/tasks/cephfs-shell.yaml
index 9708252e908a..0841ff9e088c 100644
--- a/qa/suites/fs/shell/tasks/cephfs-shell.yaml
+++ b/qa/suites/fs/shell/tasks/cephfs-shell.yaml
@@ -1,7 +1,3 @@
-# Right now, cephfs-shell is only available as a package on Ubuntu
-# This overrides the random distribution that's chosen in the other yaml fragments.
-os_type: ubuntu
-os_version: "20.04"
 tasks:
   - cephfs_test_runner:
       modules:

From 35d168826fd7b96cb73c6d799f7267a92f3453e9 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 30 Nov 2023 08:08:00 +0100
Subject: [PATCH 0783/2492] doc/glossary.rst: add "Monitor Store"

Add the term "Monitor Store" to the glossary.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index df7a0d18afa1..12e40f59c98b 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -274,6 +274,11 @@
 	:ref:`MON<arch_monitor>`
 		The Ceph monitor software.
 
+        Monitor Store
+                The persistent storage that is used by the Monitor. This
+                includes the Monitor's RocksDB and all related files in
+                ``/var/lib/ceph``.
+
 	Node
                 See :term:`Ceph Node`.
 

From 502110b72381d2e5b4bb806c4a87b2a449c2a8d8 Mon Sep 17 00:00:00 2001
From: limingze <limingze_yewu@cmss.chinamobile.com>
Date: Fri, 17 Nov 2023 12:33:48 +0100
Subject: [PATCH 0784/2492] rgw: fix post obj does not return the response etag
 field

When the post obj is successful, no etag field is returned
in the request response, which is inconsistent with aws.

Signed-off-by: limingze <limingze_yewu@cmss.chinamobile.com>
---
 src/rgw/rgw_rest_s3.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6d56766d10eb..734a2bc8e85d 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3368,6 +3368,9 @@ void RGWPostObj_ObjStore_S3::send_response()
   if (op_ret >= 0) {
     dump_content_length(s, s->formatter->get_len());
   }
+  if (op_ret == STATUS_NO_CONTENT) {
+    dump_etag(s, etag);
+  }
   end_header(s, this);
   if (op_ret != STATUS_CREATED)
     return;

From c7f5e2fc74a618dfbdafe3622578d37b8e331ec1 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Mon, 27 Nov 2023 14:42:34 +0100
Subject: [PATCH 0785/2492] mgr/rook: fixing devices listing on rook Fixes:
 https://tracker.ceph.com/issues/63694

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/rook_cluster.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index 5c7c9fc0477f..1a0caa263bfb 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -794,7 +794,12 @@ def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[
                 else:
                     fetcher = DefaultFetcher(sc.metadata.name, self.coreV1_api, self.rook_env)
                 fetcher.fetch()
-                discovered_devices.update(fetcher.devices())
+                nodename_to_devices = fetcher.devices()
+                for node, devices in nodename_to_devices.items():
+                    if node in discovered_devices:
+                        discovered_devices[node].extend(devices)
+                    else:
+                        discovered_devices[node] = devices
 
         return discovered_devices
 

From e65ded632888bd07daf5189b210934a872026116 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 13 Nov 2023 01:09:45 -0600
Subject: [PATCH 0786/2492] osd/scrub: decouple being reserved from handling
 scrub requests

For a replica, following this change:

* 'ReplicaActive' captures the state of the scrubber when
  acting as a replica, from peering to interval change;

* "being reserved" is just a flag maintained by ReplicaActive, and
  is no longer a prerequisite for handling scrub requests.

* each scrub request is now associated with its own 'token' value.

and the following minor simplification:

* the 'should we wait for pushes' decision is now part of the
  code executed on the transition from ReplicaIdle into ReplicaActiveOp.
  StartReplicaNoWait can now be discarded.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/messages/MOSDScrubReserve.h        |   2 +-
 src/osd/PG.cc                          |   5 +
 src/osd/PG.h                           |   2 +
 src/osd/scrubber/pg_scrubber.cc        | 116 ++++-------------
 src/osd/scrubber/pg_scrubber.h         |  30 ++---
 src/osd/scrubber/scrub_machine.cc      | 164 +++++++++++++++++++------
 src/osd/scrubber/scrub_machine.h       | 132 ++++++++++++++------
 src/osd/scrubber/scrub_machine_lstnr.h |   6 -
 src/osd/scrubber_common.h              |   5 +
 9 files changed, 272 insertions(+), 190 deletions(-)

diff --git a/src/messages/MOSDScrubReserve.h b/src/messages/MOSDScrubReserve.h
index f1f76b3e6fe3..c7ab98541175 100644
--- a/src/messages/MOSDScrubReserve.h
+++ b/src/messages/MOSDScrubReserve.h
@@ -24,7 +24,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
 public:
   spg_t pgid;
   epoch_t map_epoch;
-  enum {
+  enum ReserveMsgOp {
     REQUEST = 0,
     GRANT = 1,
     RELEASE = 2,
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index d2f97a129a24..ddef326e2a8a 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1823,6 +1823,11 @@ void PG::on_activate(interval_set<snapid_t> snaps)
   m_scrubber->on_pg_activate(m_planned_scrub);
 }
 
+void PG::on_replica_activate()
+{
+  m_scrubber->on_replica_activate();
+}
+
 void PG::on_active_exit()
 {
   backfill_reserving = false;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2e82e74ab012..e0f070960b4e 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -624,6 +624,8 @@ class PG : public DoutPrefixProvider,
 
   void on_activate(interval_set<snapid_t> snaps) override;
 
+  void on_replica_activate() override;
+
   void on_activate_committed() override;
 
   void on_active_actmap() override;
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 70d314f0d2f6..a88a09aeb07b 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -85,6 +85,13 @@ ostream& operator<<(ostream& out, const requested_scrub_t& sf)
   return out;
 }
 
+void PgScrubber::on_replica_activate()
+{
+  dout(10) << __func__ << dendl;
+  m_fsm->process_event(ReplicaActivate{});
+}
+
+
 /*
  * if the incoming message is from a previous interval, it must mean
  * PrimaryLogPG::on_change() was called when that interval ended. We can safely
@@ -197,7 +204,6 @@ bool PgScrubber::should_abort() const
  *
  * Some of the considerations above are also relevant to the replica-side
  * initiation
- * ('StartReplica' & 'StartReplicaNoWait').
  */
 
 void PgScrubber::initiate_regular_scrub(epoch_t epoch_queued)
@@ -216,11 +222,6 @@ void PgScrubber::initiate_regular_scrub(epoch_t epoch_queued)
   }
 }
 
-void PgScrubber::dec_scrubs_remote()
-{
-  m_osds->get_scrub_services().dec_scrubs_remote(m_pg_id.pgid);
-}
-
 void PgScrubber::advance_token()
 {
   m_current_token++;
@@ -274,13 +275,7 @@ void PgScrubber::send_start_replica(epoch_t epoch_queued,
   }
 
   if (check_interval(epoch_queued) && is_token_current(token)) {
-    // save us some time by not waiting for updates if there are none
-    // to wait for. Affects the transition from NotActive into either
-    // ReplicaWaitUpdates or ActiveReplica.
-    if (pending_active_pushes())
-      m_fsm->process_event(StartReplica{});
-    else
-      m_fsm->process_event(StartReplicaNoWait{});
+    m_fsm->process_event(StartReplica{});
   }
   dout(10) << "scrubber event --<< " << __func__ << dendl;
 }
@@ -452,6 +447,11 @@ unsigned int PgScrubber::scrub_requeue_priority(
  * Responsible for resetting any scrub state and releasing any resources.
  * Any inflight events will be ignored via check_interval/should_drop_message
  * or canceled.
+ * Specifically:
+ * - if Primary and in an active session - the IntervalChanged handler takes
+ *   care of discarding the remote reservations, and transitioning out of
+ *   Session. That resets both the scrubber and the FSM.
+ * - if we are a reserved replica - we need to free ourselves;
  */
 void PgScrubber::on_new_interval()
 {
@@ -461,13 +461,7 @@ void PgScrubber::on_new_interval()
 		  is_scrub_active(), is_queued_or_active())
 	   << dendl;
 
-  // If in active session - the IntervalChanged handler takes care of
-  // discarding the remote reservations, and transitioning out of Session.
-  // That resets both the scrubber and the FSM.
   m_fsm->process_event(IntervalChanged{});
-
-  // The 'FullReset' is only relevant if we are not an active Primary
-  m_fsm->process_event(FullReset{});
   rm_from_osd_scrubbing();
 }
 
@@ -1139,13 +1133,7 @@ void PgScrubber::on_init()
   m_pg->publish_stats_to_osd();
 }
 
-/*
- * Note: as on_replica_init() is likely to be called twice (entering
- * both ReplicaWaitUpdates & ActiveReplica), its operations should be
- * idempotent.
- * Now that it includes some state-changing operations, we need to check
- * m_active against double-activation.
- */
+
 void PgScrubber::on_replica_init()
 {
   dout(10) << __func__ << " called with 'active' "
@@ -1159,6 +1147,7 @@ void PgScrubber::on_replica_init()
   }
 }
 
+
 int PgScrubber::build_primary_map_chunk()
 {
   epoch_t map_building_since = m_pg->get_osdmap_epoch();
@@ -1217,23 +1206,21 @@ int PgScrubber::build_replica_map_chunk()
 
       // the local map has been created. Send it to the primary.
       // Note: once the message reaches the Primary, it may ask us for another
-      // chunk - and we better be done with the current scrub. Thus - the
-      // preparation of the reply message is separate, and we clear the scrub
-      // state before actually sending it.
+      // chunk - and we better be done with the current scrub. The clearing of
+      // state must be complete before we relinquish the PG lock.
 
-      auto reply = prep_replica_map_msg(PreemptionNoted::no_preemption);
-      replica_handling_done();
-      dout(15) << __func__ << " chunk map sent " << dendl;
-      send_replica_map(reply);
-    } break;
+      send_replica_map(prep_replica_map_msg(PreemptionNoted::no_preemption));
+      dout(15) << fmt::format("{}: chunk map sent", __func__) << dendl;
+    }
+    break;
 
     default:
       // negative retval: build_scrub_map_chunk() signalled an error
       // Pre-Pacific code ignored this option, treating it as a success.
       // \todo Add an error flag in the returning message.
+      // \todo: must either abort, send a reply, or return some error message
       dout(1) << "Error! Aborting. ActiveReplica::react(SchedReplica) Ret: "
 	      << ret << dendl;
-      replica_handling_done();
       // only in debug mode for now:
       assert(false && "backend error");
       break;
@@ -1520,6 +1507,7 @@ void PgScrubber::replica_scrub_op(OpRequestRef op)
   replica_scrubmap_pos.reset();	 // needed? RRR
 
   set_queued_or_active();
+  advance_token();
   m_osds->queue_for_rep_scrub(m_pg,
 			      m_replica_request_priority,
 			      m_flags.priority,
@@ -1675,7 +1663,7 @@ void PgScrubber::handle_scrub_reserve_msgs(OpRequestRef op)
   auto m = op->get_req<MOSDScrubReserve>();
   switch (m->type) {
     case MOSDScrubReserve::REQUEST:
-      handle_scrub_reserve_request(op);
+      m_fsm->process_event(ReplicaReserveReq{op, m->from});
       break;
     case MOSDScrubReserve::GRANT:
       m_fsm->process_event(ReplicaGrant{op, m->from});
@@ -1684,65 +1672,12 @@ void PgScrubber::handle_scrub_reserve_msgs(OpRequestRef op)
       m_fsm->process_event(ReplicaReject{op, m->from});
       break;
     case MOSDScrubReserve::RELEASE:
-      handle_scrub_reserve_release(op);
+      m_fsm->process_event(ReplicaRelease{op, m->from});
       break;
   }
 }
 
 
-void PgScrubber::handle_scrub_reserve_request(OpRequestRef op)
-{
-  auto request_ep = op->sent_epoch;
-  dout(20) << fmt::format("{}: request_ep:{} recovery:{}",
-			  __func__,
-			  request_ep,
-			  m_osds->is_recovery_active())
-	   << dendl;
-
-  // The primary may unilaterally restart the scrub process without notifying
-  // replicas. Unconditionally clear any existing state prior to handling
-  // the new reservation.
-  m_fsm->process_event(FullReset{});
-
-  bool granted{false};
-  if (m_pg->cct->_conf->osd_scrub_during_recovery ||
-      !m_osds->is_recovery_active()) {
-
-    granted = m_osds->get_scrub_services().inc_scrubs_remote(m_pg_id.pgid);
-    if (granted) {
-      m_fsm->process_event(ReplicaGrantReservation{});
-    } else {
-      dout(20) << __func__ << ": failed to reserve remotely" << dendl;
-    }
-  } else {
-    dout(10) << __func__ << ": recovery is active; not granting" << dendl;
-  }
-
-  dout(10) << __func__ << " reserved? " << (granted ? "yes" : "no") << dendl;
-
-  Message* reply = new MOSDScrubReserve(
-    spg_t(m_pg->info.pgid.pgid, m_pg->get_primary().shard),
-    request_ep,
-    granted ? MOSDScrubReserve::GRANT : MOSDScrubReserve::REJECT,
-    m_pg_whoami);
-
-  m_osds->send_message_osd_cluster(reply, op->get_req()->get_connection());
-}
-
-void PgScrubber::handle_scrub_reserve_release(OpRequestRef op)
-{
-  dout(10) << __func__ << " " << *op->get_req() << dendl;
-  if (should_drop_message(op)) {
-    // we might have turned into a Primary in the meantime. The interval
-    // change should have been noticed already, and caused us to reset.
-    return;
-  }
-
-  // this specific scrub session has terminated. All incoming events carrying
-  // the old tag will be discarded.
-  m_fsm->process_event(FullReset{});
-}
-
 bool PgScrubber::set_reserving_now() {
   return m_osds->get_scrub_services().set_reserving_now(m_pg_id,
                                                         ceph_clock_now());
@@ -2211,6 +2146,7 @@ void PgScrubber::handle_query_state(ceph::Formatter* f)
 
 PgScrubber::~PgScrubber()
 {
+  m_fsm->process_event(IntervalChanged{});
   if (m_scrub_job) {
     // make sure the OSD won't try to scrub this one just now
     rm_from_osd_scrubbing();
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 9946f9ce5ee9..2553e49b263a 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -258,14 +258,6 @@ class PgScrubber : public ScrubPgIF,
    */
   void handle_scrub_reserve_msgs(OpRequestRef op) final;
 
-  /**
-   *  we are a replica being asked by the Primary to reserve OSD resources for
-   *  scrubbing
-   */
-  void handle_scrub_reserve_request(OpRequestRef op);
-
-  void handle_scrub_reserve_release(OpRequestRef op);
-
   // managing scrub op registration
 
   void update_scrub_job(const requested_scrub_t& request_flags) final;
@@ -334,6 +326,8 @@ class PgScrubber : public ScrubPgIF,
 
   void on_new_interval() final;
 
+  void on_replica_activate() final;
+
   void scrub_clear_state() final;
 
   bool is_queued_or_active() const final;
@@ -476,13 +470,9 @@ class PgScrubber : public ScrubPgIF,
   [[nodiscard]] bool was_epoch_changed() const final;
 
   void set_queued_or_active() final;
-  /// Clears `m_queued_or_active` and restarts snaptrimming
+  /// Clears `m_queued_or_active` and restarts snap-trimming
   void clear_queued_or_active() final;
 
-  void dec_scrubs_remote() final;
-
-  void advance_token() final;
-
   void mark_local_map_ready() final;
 
   [[nodiscard]] bool are_all_maps_available() const final;
@@ -567,6 +557,9 @@ class PgScrubber : public ScrubPgIF,
 
   void requeue_waiting() const { m_pg->requeue_ops(m_pg->waiting_for_scrub); }
 
+  /// Modify the token identifying the current replica scrub operation
+  void advance_token();
+
   /**
    *  mark down some parameters of the initiated scrub:
    *  - the epoch when started;
@@ -675,11 +668,12 @@ class PgScrubber : public ScrubPgIF,
   epoch_t m_epoch_start{0};  ///< the actual epoch when scrubbing started
 
   /**
-   * (replica) a tag identifying a specific scrub "session". Incremented
-   * whenever the Primary releases the replica scrub resources. When the scrub
-   * session is terminated (even if the interval remains unchanged, as might
-   * happen following an asok no-scrub command), stale scrub-resched messages
-   *  triggered by the backend will be discarded.
+   * (replica) a tag identifying a specific replica operation, i.e. the
+   * creation of the replica scrub map for a single chunk.
+   * Incremented immediately before sending a response to the primary,
+   * so that the next request would be identified as such. Also changed
+   * on reservation release.
+   * Used to identify stale scrub-re-sched messages triggered by the backend.
    */
   Scrub::act_token_t m_current_token{1};
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 99286acaa12b..cb10d87236b8 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -654,66 +654,163 @@ ScrubMachine::~ScrubMachine() = default;
 
 // -------- for replicas -----------------------------------------------------
 
-// ----------------------- ReservedReplica --------------------------------
+// ----------------------- ReplicaActive --------------------------------
 
-ReservedReplica::ReservedReplica(my_context ctx)
+ReplicaActive::ReplicaActive(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReservedReplica")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActive")
 {
-  dout(10) << "-- state -->> ReservedReplica" << dendl;
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "-- state -->> ReplicaActive" << dendl;
+  m_pg = scrbr->get_pg();
+  m_osds = m_pg->get_pg_osd(ScrubberPasskey());
+}
+
+ReplicaActive::~ReplicaActive()
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  if (reserved_by_my_primary) {
+    dout(10) << "ReplicaActive::~ReplicaActive(): clearing reservation"
+	     << dendl;
+    clear_reservation_by_remote_primary();
+  }
+}
+
+
+/*
+ * Note: we are expected to be in the initial internal state (Idle) when
+ * receiving any registration request. Our other internal states, the
+ * active ones, have their own handler for this event, and will treat it
+ * as an abort request.
+ *
+ * Process:
+ * - if already reserved: clear existing reservation, then continue
+ * - ask the OSD for the "reservation resource"
+ * - if granted: mark it internally and notify the Primary.
+ * - otherwise: just notify the requesting primary.
+ */
+void ReplicaActive::on_reserve_req(const ReplicaReserveReq& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaActive::on_reserve_req()" << dendl;
+
+  if (reserved_by_my_primary) {
+    dout(10) << "ReplicaActive::on_reserve_req(): already reserved" << dendl;
+    // clear the existing reservation
+    clear_reservation_by_remote_primary();  // clears the flag, too
+  }
+
+  // ask the OSD for the reservation
+  const auto ret = get_remote_reservation();
+  if (ret.granted) {
+    reserved_by_my_primary = true;
+    dout(10) << fmt::format("{}: reserved? yes", __func__) << dendl;
+  } else {
+    dout(10) << fmt::format("{}: reserved? no ({})", __func__, ret.error_msg)
+	     << dendl;
+  }
+
+  Message* reply = new MOSDScrubReserve(
+      spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch, ret.op,
+      m_pg->pg_whoami);
+  m_osds->send_message_osd_cluster(reply, ev.m_op->get_req()->get_connection());
+}
+
+
+void ReplicaActive::on_release(const ReplicaRelease& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  if (!reserved_by_my_primary) {
+    dout(5) << fmt::format(
+		   "ReplicaActive::on_release() from {}: not reserved!",
+		   ev.m_from)
+	    << dendl;
+    return;
+  }
+  dout(10) << fmt::format("ReplicaActive::on_release() from {}", ev.m_from)
+	   << dendl;
+  clear_reservation_by_remote_primary();
+}
+
+
+ReplicaActive::ReservationAttemptRes ReplicaActive::get_remote_reservation()
+{
+  using ReservationAttemptRes = ReplicaActive::ReservationAttemptRes;
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  if (!scrbr->get_pg_cct()->_conf.get_val<bool>("osd_scrub_during_recovery") &&
+      m_osds->is_recovery_active()) {
+    return ReservationAttemptRes{
+	MOSDScrubReserve::REJECT, "recovery is active", false};
+  }
+
+  if (m_osds->get_scrub_services().inc_scrubs_remote(scrbr->get_spgid().pgid)) {
+    return ReservationAttemptRes{MOSDScrubReserve::GRANT, "", true};
+  } else {
+    return ReservationAttemptRes{
+	MOSDScrubReserve::REJECT, "failed to reserve remotely", false};
+  }
 }
 
-ReservedReplica::~ReservedReplica()
+
+void ReplicaActive::clear_reservation_by_remote_primary()
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  scrbr->dec_scrubs_remote();
-  scrbr->advance_token();
+  dout(10) << "ReplicaActive::clear_reservation_by_remote_primary()" << dendl;
+  m_osds->get_scrub_services().dec_scrubs_remote(scrbr->get_spgid().pgid);
+  reserved_by_my_primary = false;
 }
 
-// ----------------------- ReplicaIdle --------------------------------
+
+void ReplicaActive::check_for_updates(const StartReplica& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaActive::check_for_updates()" << dendl;
+  post_event(ReplicaPushesUpd{});
+}
+
+// ---------------- ReplicaActive/ReplicaIdle ---------------------------
 
 ReplicaIdle::ReplicaIdle(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(
-	  context<ScrubMachine>().m_scrbr,
-	  "ReservedReplica/ReplicaIdle")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActive/ReplicaIdle")
 {
-  dout(10) << "-- state -->> ReservedReplica/ReplicaIdle" << dendl;
+  dout(10) << "-- state -->> ReplicaActive/ReplicaIdle" << dendl;
 }
 
-ReplicaIdle::~ReplicaIdle() = default;
 
-// ----------------------- ReplicaActiveOp --------------------------------
+// ------------- ReplicaActive/ReplicaActiveOp --------------------------
 
 ReplicaActiveOp::ReplicaActiveOp(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(
-	  context<ScrubMachine>().m_scrbr,
-	  "ReservedReplica/ReplicaActiveOp")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActiveOp")
 {
-  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp" << dendl;
+  dout(10) << "-- state -->> ReplicaActive/ReplicaActiveOp" << dendl;
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  scrbr->on_replica_init();
+}
+
+
+ReplicaActiveOp::~ReplicaActiveOp()
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << __func__ << dendl;
+  scrbr->replica_handling_done();
 }
 
-/**
- * \note: here is too late to call replica_handling_done(). See the
- * comment in build_replica_map_chunk()
- */
-ReplicaActiveOp::~ReplicaActiveOp() = default;
 
-// ----------------------- ReplicaWaitUpdates --------------------------------
+// ------------- ReplicaActive/ReplicaWaitUpdates ------------------------
 
 ReplicaWaitUpdates::ReplicaWaitUpdates(my_context ctx)
     : my_base(ctx)
     , NamedSimply(
 	  context<ScrubMachine>().m_scrbr,
-	  "ReservedReplica/ReplicaActiveOp/ReplicaWaitUpdates")
+	  "ReplicaActive/ReplicaActiveOp/ReplicaWaitUpdates")
 {
-  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp/ReplicaWaitUpdates"
+  dout(10) << "-- state -->> ReplicaActive/ReplicaActiveOp/ReplicaWaitUpdates"
 	   << dendl;
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  scrbr->on_replica_init();
 }
 
+
 /*
  * Triggered externally, by the entity that had an update re pushes
  */
@@ -724,7 +821,6 @@ sc::result ReplicaWaitUpdates::react(const ReplicaPushesUpd&)
 	   << scrbr->pending_active_pushes() << dendl;
 
   if (scrbr->pending_active_pushes() == 0) {
-
     // done waiting
     return transit<ReplicaBuildingMap>();
   }
@@ -732,22 +828,21 @@ sc::result ReplicaWaitUpdates::react(const ReplicaPushesUpd&)
   return discard_event();
 }
 
+
 // ----------------------- ReplicaBuildingMap -----------------------------------
 
 ReplicaBuildingMap::ReplicaBuildingMap(my_context ctx)
     : my_base(ctx)
     , NamedSimply(
 	  context<ScrubMachine>().m_scrbr,
-	  "ReservedReplica/ReplicaActiveOp/ReplicaBuildingMap")
+	  "ReplicaActive/ReplicaActiveOp/ReplicaBuildingMap")
 {
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "-- state -->> ReservedReplica/ReplicaActiveOp/ReplicaBuildingMap"
+  dout(10) << "-- state -->> ReplicaActive/ReplicaActiveOp/ReplicaBuildingMap"
 	   << dendl;
-  // and as we might have skipped ReplicaWaitUpdates:
-  scrbr->on_replica_init();
   post_event(SchedReplica{});
 }
 
+
 sc::result ReplicaBuildingMap::react(const SchedReplica&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
@@ -758,7 +853,6 @@ sc::result ReplicaBuildingMap::react(const SchedReplica&)
     dout(10) << "replica scrub job preempted" << dendl;
 
     scrbr->send_preempted_replica();
-    scrbr->replica_handling_done();
     return transit<ReplicaIdle>();
   }
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index fcca0d2cc40d..6dbc8b565e58 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -164,15 +164,12 @@ MEV(IntLocalMapDone)
 /// scrub_snapshot_metadata()
 MEV(DigestUpdate)
 
-/// event emitted when the replica grants a reservation to the primary
-MEV(ReplicaGrantReservation)
+/// we are a replica for this PG
+MEV(ReplicaActivate)
 
 /// initiating replica scrub
 MEV(StartReplica)
 
-/// 'start replica' when there are no pending updates
-MEV(StartReplicaNoWait)
-
 MEV(SchedReplica)
 
 /// Update to active_pushes. 'active_pushes' represents recovery
@@ -211,8 +208,11 @@ struct NotActive;	    ///< the quiescent state. No active scrubbing.
 struct Session;            ///< either reserving or actively scrubbing
 struct ReservingReplicas;   ///< securing scrub resources from replicas' OSDs
 struct ActiveScrubbing;	    ///< the active state for a Primary. A sub-machine.
-struct ReplicaIdle;         ///< Initial reserved replica state
-struct ReplicaBuildingMap;	    ///< an active state for a replica.
+// the active states for a replica:
+struct ReplicaActive;    ///< the quiescent state for a replica
+struct ReplicaActiveOp;
+struct ReplicaWaitUpdates;
+struct ReplicaBuildingMap;
 
 
 class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
@@ -370,8 +370,8 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
  *
  *  - a special end-of-recovery Primary scrub event ('AfterRepairScrub').
  *
- *  - (for a replica) 'StartReplica' or 'StartReplicaNoWait', triggered by
- *    an incoming MOSDRepScrub message.
+ *  - (if already in ReplicaActive): an incoming MOSDRepScrub triggers
+ *    'StartReplica'.
  *
  *  note (20.8.21): originally, AfterRepairScrub was triggering a scrub without
  *  waiting for replica resources to be acquired. But once replicas started
@@ -381,11 +381,13 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
 struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
   explicit NotActive(my_context ctx);
 
-  using reactions =
-    mpl::list<sc::custom_reaction<StartScrub>,
-	      // a scrubbing that was initiated at recovery completion:
-	      sc::custom_reaction<AfterRepairScrub>,
-	      sc::transition<ReplicaGrantReservation, ReplicaIdle>>;
+  using reactions = mpl::list<
+      sc::custom_reaction<StartScrub>,
+      // a scrubbing that was initiated at recovery completion:
+      sc::custom_reaction<AfterRepairScrub>,
+      // peering done, and we are a replica
+      sc::transition<ReplicaActivate, ReplicaActive>>;
+
   sc::result react(const StartScrub&);
   sc::result react(const AfterRepairScrub&);
 };
@@ -611,45 +613,95 @@ struct WaitDigestUpdate : sc::state<WaitDigestUpdate, ActiveScrubbing>,
 
 // ----------------------------- the "replica active" states
 
-/**
- * ReservedReplica
+/*
+ *  The replica states:
  *
- * Parent state for replica states,  Controls lifecycle for
- * PgScrubber::m_reservations.
+ *  ReplicaActive - starts after being peered as a replica. Ends on interval.
+ *   - maintain the "I am reserved by a primary" state;
+ *   - handles reservation requests
+ *
+ *     - ReplicaIdle - ready for a new scrub request
+ *          * initial state of ReplicaActive
+ *
+ *     - ReplicaActiveOp - handling a single map request op
+ *          * ReplicaWaitUpdates
+ *  	    * ReplicaBuildingMap
  */
-struct ReservedReplica : sc::state<ReservedReplica, ScrubMachine, ReplicaIdle>,
+
+struct ReplicaIdle;
+
+struct ReplicaActive : sc::state<ReplicaActive, ScrubMachine, ReplicaIdle>,
 			 NamedSimply {
-  explicit ReservedReplica(my_context ctx);
-  ~ReservedReplica();
+  explicit ReplicaActive(my_context ctx);
+  ~ReplicaActive();
 
-  using reactions = mpl::list<sc::transition<FullReset, NotActive>>;
-};
+  /// handle a reservation request from a primary
+  void on_reserve_req(const ReplicaReserveReq&);
 
-struct ReplicaWaitUpdates;
+  /// handle a 'release' from a primary
+  void on_release(const ReplicaRelease&);
 
-/**
- * ReplicaIdle
- *
- * Replica is waiting for a map request.
- */
-struct ReplicaIdle : sc::state<ReplicaIdle, ReservedReplica>,
-		     NamedSimply {
-  explicit ReplicaIdle(my_context ctx);
-  ~ReplicaIdle();
+  void check_for_updates(const StartReplica&);
 
   using reactions = mpl::list<
-    sc::transition<StartReplica, ReplicaWaitUpdates>,
-    sc::transition<StartReplicaNoWait, ReplicaBuildingMap>>;
+      // a reservation request from the primary
+      sc::in_state_reaction<
+	  ReplicaReserveReq,
+	  ReplicaActive,
+	  &ReplicaActive::on_reserve_req>,
+      // an explicit release request from the primary
+      sc::in_state_reaction<
+	  ReplicaRelease,
+	  ReplicaActive,
+	  &ReplicaActive::on_release>,
+      // when the interval ends - we may not be a replica anymore
+      sc::transition<IntervalChanged, NotActive>>;
+
+ private:
+  bool reserved_by_my_primary{false};
+
+  // shortcuts:
+  PG* m_pg;
+  OSDService* m_osds;
+
+  /// a convenience internal result structure
+  struct ReservationAttemptRes {
+    MOSDScrubReserve::ReserveMsgOp op;	// GRANT or REJECT
+    std::string_view error_msg;
+    bool granted;
+  };
+
+  /// request a scrub resource from our local OSD
+  /// (after performing some checks)
+  ReservationAttemptRes get_remote_reservation();
+
+  void clear_reservation_by_remote_primary();
 };
 
+
+struct ReplicaIdle : sc::state<ReplicaIdle, ReplicaActive>, NamedSimply {
+  explicit ReplicaIdle(my_context ctx);
+  ~ReplicaIdle() = default;
+
+  // note the execution of check_for_updates() when transitioning to
+  // ReplicaActiveOp/ReplicaWaitUpdates. That would trigger a ReplicaPushesUpd
+  // event, which will be handled by ReplicaWaitUpdates.
+  using reactions = mpl::list<sc::transition<
+      StartReplica,
+      ReplicaWaitUpdates,
+      ReplicaActive,
+      &ReplicaActive::check_for_updates>>;
+};
+
+
 /**
  * ReplicaActiveOp
  *
- * Lifetime matches handling for a single map request op
+ * Lifetime matches handling for a single map request op.
  */
 struct ReplicaActiveOp
-  : sc::state<ReplicaActiveOp, ReservedReplica, ReplicaWaitUpdates>,
-    NamedSimply {
+    : sc::state<ReplicaActiveOp, ReplicaActive, ReplicaWaitUpdates>,
+      NamedSimply {
   explicit ReplicaActiveOp(my_context ctx);
   ~ReplicaActiveOp();
 };
@@ -670,8 +722,8 @@ struct ReplicaWaitUpdates : sc::state<ReplicaWaitUpdates, ReplicaActiveOp>,
 };
 
 
-struct ReplicaBuildingMap : sc::state<ReplicaBuildingMap, ReplicaActiveOp>
-			  , NamedSimply {
+struct ReplicaBuildingMap : sc::state<ReplicaBuildingMap, ReplicaActiveOp>,
+			    NamedSimply {
   explicit ReplicaBuildingMap(my_context ctx);
   using reactions = mpl::list<sc::custom_reaction<SchedReplica>>;
 
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 4206c789f91a..890a70a8a129 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -196,12 +196,6 @@ struct ScrubMachineListener {
   virtual void set_queued_or_active() = 0;
   virtual void clear_queued_or_active() = 0;
 
-  /// Release remote scrub reservation
-  virtual void dec_scrubs_remote() = 0;
-
-  /// Advance replica token
-  virtual void advance_token() = 0;
-
   /**
    * Our scrubbing is blocked, waiting for an excessive length of time for
    * our target chunk to be unlocked. We will set the corresponding flags,
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 16810bba15c6..d24bb79b801e 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -18,12 +18,14 @@ struct PGPool;
 
 namespace Scrub {
   class ReplicaReservations;
+  struct ReplicaActive;
 }
 
 /// Facilitating scrub-related object access to private PG data
 class ScrubberPasskey {
 private:
   friend class Scrub::ReplicaReservations;
+  friend struct Scrub::ReplicaActive;
   friend class PrimaryLogScrub;
   friend class PgScrubber;
   friend class ScrubBackend;
@@ -310,6 +312,9 @@ struct ScrubPgIF {
   /// the OSD scrub queue
   virtual void on_new_interval() = 0;
 
+  /// we are peered as a replica
+  virtual void on_replica_activate() = 0;
+
   virtual void scrub_clear_state() = 0;
 
   virtual void handle_query_state(ceph::Formatter* f) = 0;

From f610b1e5bed290652c53ae37b36d7c0fb985e3ae Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 14 Nov 2023 06:01:08 -0600
Subject: [PATCH 0787/2492] osd/scrub: skip reserving replicas for
 high-priority scrubs

(The primary side of the required changes)

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc        |  7 ++++++-
 src/osd/scrubber/pg_scrubber.h         |  3 +++
 src/osd/scrubber/scrub_machine_lstnr.h |  3 +++
 src/osd/scrubber/scrub_reservations.cc | 17 +++++++++++------
 4 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index a88a09aeb07b..edc6c2a2dcd4 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -800,7 +800,7 @@ void PgScrubber::cancel_callback(scrubber_callback_cancel_token_t token)
   m_osds->sleep_timer.cancel_event(token);
 }
 
-LogChannelRef &PgScrubber::get_clog() const
+LogChannelRef& PgScrubber::get_clog() const
 {
   return m_osds->clog;
 }
@@ -810,6 +810,11 @@ int PgScrubber::get_whoami() const
   return m_osds->whoami;
 }
 
+[[nodiscard]] bool PgScrubber::is_high_priority() const
+{
+  return m_flags.required;
+}
+
 /*
  * The selected range is set directly into 'm_start' and 'm_end'
  * setting:
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 2553e49b263a..832fa17050cc 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -402,6 +402,9 @@ class PgScrubber : public ScrubPgIF,
     return m_pg->recovery_state.is_primary();
   }
 
+  /// is this scrub more than just regular periodic scrub?
+  [[nodiscard]] bool is_high_priority() const final;
+
   void set_state_name(const char* name) final
   {
     m_fsm_state_name = name;
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 890a70a8a129..34247083f4fd 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -229,4 +229,7 @@ struct ScrubMachineListener {
   // temporary interface (to be discarded in a follow-up PR)
   /// set the 'resources_failure' flag in the scrub-job object
   virtual void flag_reservations_failure() = 0;
+
+  /// is this scrub more than just regular periodic scrub?
+  [[nodiscard]] virtual bool is_high_priority() const = 0;
 };
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 3194c9871d57..449856e7b459 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -49,12 +49,17 @@ ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
       });
 
   m_next_to_request = m_sorted_secondaries.cbegin();
-  // send out the 1'st request (unless we have no replicas)
-  send_next_reservation_or_complete();
-
-  m_slow_response_warn_timeout =
-      m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
-	  "osd_scrub_slow_reservation_response");
+  if (m_scrubber.is_high_priority()) {
+    // for high-priority scrubs (i.e. - user-initiated), no reservations are
+    // needed.
+    dout(10) << "high-priority scrub - no reservations needed" << dendl;
+  } else {
+    // send out the 1'st request (unless we have no replicas)
+    send_next_reservation_or_complete();
+    m_slow_response_warn_timeout =
+	m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
+	    "osd_scrub_slow_reservation_response");
+  }
 }
 
 void ReplicaReservations::release_all()

From 3ce41e3bc1a96cd804b118e1538db90c9927fcce Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 20 Nov 2023 07:31:31 -0600
Subject: [PATCH 0788/2492] osd/scrub: separate counters for primary vs.
 replica scrubs

The OSD limits the number of concurrent scrubs performed on its PGs.
This limit is now enforced separately for primary and replica scrubs.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_resources.cc | 51 +++++++++++++++++++----------
 src/osd/scrubber/scrub_resources.h  |  4 +++
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/src/osd/scrubber/scrub_resources.cc b/src/osd/scrubber/scrub_resources.cc
index 25dcec2399f1..6203a2912aa9 100644
--- a/src/osd/scrubber/scrub_resources.cc
+++ b/src/osd/scrubber/scrub_resources.cc
@@ -21,28 +21,38 @@ ScrubResources::ScrubResources(
     , conf{config}
 {}
 
+// ------------------------- scrubbing as primary on this OSD -----------------
+
+// can we increase the number of concurrent scrubs performed by Primaries
+// on this OSD? note that counted separately from the number of scrubs
+// performed by replicas.
 bool ScrubResources::can_inc_scrubs() const
 {
   std::lock_guard lck{resource_lock};
-  if (scrubs_local + granted_reservations.size() < conf->osd_max_scrubs) {
-    return true;
-  }
-  log_upwards(fmt::format(
-      "{}== false. {} (local) + {} (remote) >= max ({})", __func__,
-      scrubs_local, granted_reservations.size(), conf->osd_max_scrubs));
-  return false;
+  return can_inc_local_scrubs_unlocked();
 }
 
 bool ScrubResources::inc_scrubs_local()
 {
   std::lock_guard lck{resource_lock};
-  if (scrubs_local + granted_reservations.size() < conf->osd_max_scrubs) {
+  if (can_inc_local_scrubs_unlocked()) {
     ++scrubs_local;
+    log_upwards(fmt::format(
+	"{}: {} -> {} (max {}, remote {})", __func__, (scrubs_local - 1),
+	scrubs_local, conf->osd_max_scrubs, granted_reservations.size()));
+    return true;
+  }
+  return false;
+}
+
+bool ScrubResources::can_inc_local_scrubs_unlocked() const
+{
+  if (scrubs_local < conf->osd_max_scrubs) {
     return true;
   }
   log_upwards(fmt::format(
-      "{}: {} (local) + {} (remote) >= max ({})", __func__, scrubs_local,
-      granted_reservations.size(), conf->osd_max_scrubs));
+      "{}: Cannot add local scrubs. Current counter ({}) >= max ({})", __func__,
+      scrubs_local, conf->osd_max_scrubs));
   return false;
 }
 
@@ -50,12 +60,15 @@ void ScrubResources::dec_scrubs_local()
 {
   std::lock_guard lck{resource_lock};
   log_upwards(fmt::format(
-      "{}: {} -> {} (max {}, remote {})", __func__, scrubs_local,
-      (scrubs_local - 1), conf->osd_max_scrubs, granted_reservations.size()));
+      "{}:  {} -> {} (max {}, remote {})",
+      __func__, scrubs_local, (scrubs_local - 1), conf->osd_max_scrubs,
+      granted_reservations.size()));
   --scrubs_local;
   ceph_assert(scrubs_local >= 0);
 }
 
+// ------------------------- scrubbing on this OSD as replicas ----------------
+
 bool ScrubResources::inc_scrubs_remote(pg_t pgid)
 {
   std::lock_guard lck{resource_lock};
@@ -67,18 +80,20 @@ bool ScrubResources::inc_scrubs_remote(pg_t pgid)
     return true;
   }
 
-  auto prev = granted_reservations.size();
-  if (scrubs_local + prev < conf->osd_max_scrubs) {
+  auto pre_op_cnt = granted_reservations.size();
+  if (pre_op_cnt < conf->osd_max_scrubs) {
     granted_reservations.insert(pgid);
     log_upwards(fmt::format(
-	"{}: pg[{}] {} -> {} (max {}, local {})", __func__, pgid, prev,
-	granted_reservations.size(), conf->osd_max_scrubs, scrubs_local));
+	"{}: pg[{}] reserved. Remote scrubs count changed from {} -> {} (max "
+	"{}, local {})",
+	__func__, pgid, pre_op_cnt, granted_reservations.size(),
+	conf->osd_max_scrubs, scrubs_local));
     return true;
   }
 
   log_upwards(fmt::format(
-      "{}: pg[{}] {} (local) + {} (remote) >= max ({})", __func__, pgid,
-      scrubs_local, granted_reservations.size(), conf->osd_max_scrubs));
+      "{}: pg[{}] failed. Too many concurrent replica scrubs ({} >= max ({}))",
+      __func__, pgid, pre_op_cnt, conf->osd_max_scrubs));
   return false;
 }
 
diff --git a/src/osd/scrubber/scrub_resources.h b/src/osd/scrubber/scrub_resources.h
index 724e206ee27e..75807a10f825 100644
--- a/src/osd/scrubber/scrub_resources.h
+++ b/src/osd/scrubber/scrub_resources.h
@@ -40,6 +40,10 @@ class ScrubResources {
 
   const ceph::common::ConfigProxy& conf;
 
+  /// an aux used to check available local scrubs. Must be called with
+  /// the resource lock held.
+  bool can_inc_local_scrubs_unlocked() const;
+
  public:
   explicit ScrubResources(
       log_upwards_t log_access,

From 9802b770f835ea75ea8cae7baaea70d75ce73048 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 30 Nov 2023 00:35:29 -0600
Subject: [PATCH 0789/2492] osd/scrub: handling unexpected scrub requests

if arriving while still handling a previous chunk request,
the handling of of the previous chunk request will be aborted.
No response is sent. The scrubber resets, then handles the new request.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.h    | 16 ++++++++++++----
 src/osd/scrubber/scrub_machine.cc | 16 ++++++++++++++++
 src/osd/scrubber/scrub_machine.h  | 14 ++++++++++++++
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 832fa17050cc..3ac4d6dfdb74 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -673,10 +673,18 @@ class PgScrubber : public ScrubPgIF,
   /**
    * (replica) a tag identifying a specific replica operation, i.e. the
    * creation of the replica scrub map for a single chunk.
-   * Incremented immediately before sending a response to the primary,
-   * so that the next request would be identified as such. Also changed
-   * on reservation release.
-   * Used to identify stale scrub-re-sched messages triggered by the backend.
+   *
+   * Background: the backend is asynchronous, and the specific
+   * operations are size-limited. While the scrubber handles a specific
+   * request, it is continuously triggered to poll the backend for the
+   * full results for the chunk handled.
+   * Once the chunk request becomes obsolete, either following an interval
+   * change or if a new request was received, we must not send the stale
+   * data to the primary. The polling of the obsolete chunk request must
+   * stop, and the stale backend response should be discarded.
+   * In other words - the token should be read as saying "the primary has
+   * lost interest in the results of all operations identified by mismatched
+   * token values".
    */
   Scrub::act_token_t m_current_token{1};
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index cb10d87236b8..2e18ed5cf4e1 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -797,6 +797,22 @@ ReplicaActiveOp::~ReplicaActiveOp()
   scrbr->replica_handling_done();
 }
 
+sc::result ReplicaActiveOp::react(const StartReplica&)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaActiveOp::react(const StartReplica&)" << dendl;
+
+  const auto msg = fmt::format(
+      "osd.{} pg[{}]: new chunk request while still handling the previous one",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  dout(1) << msg << dendl;
+  scrbr->get_clog()->warn() << msg;
+
+  post_event(ReplicaPushesUpd{});
+
+  // exit & re-enter the state
+  return transit<ReplicaActiveOp>();
+}
 
 // ------------- ReplicaActive/ReplicaWaitUpdates ------------------------
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 6dbc8b565e58..7e9c887145ae 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -704,6 +704,20 @@ struct ReplicaActiveOp
       NamedSimply {
   explicit ReplicaActiveOp(my_context ctx);
   ~ReplicaActiveOp();
+
+  using reactions = mpl::list<sc::custom_reaction<StartReplica>>;
+
+  /**
+   * Handling the unexpected (read - caused by a bug) case of receiving a
+   * new chunk request while still handling the previous one.
+   * To note:
+   * - the primary is evidently no longer waiting for the results of the
+   *   previous request. On the other hand
+   * - we must respond to the new request, as the primary would wait for
+   *   it "forever"`,
+   * - and we should log this unexpected scenario clearly in the cluster log.
+   */
+  sc::result react(const StartReplica&);
 };
 
 /*

From bc18beaf8f223b12bcc01b8aed7ddf7e394b77a1 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 29 Nov 2023 13:40:31 +0100
Subject: [PATCH 0790/2492] mgr/rook: fixing inconsistencies on the dashboard
 'services' view Fixes: https://tracker.ceph.com/issues/63678

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py | 59 +++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 9 deletions(-)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index fa75db2cf9cf..ca2d168db636 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -257,6 +257,26 @@ def describe_service(self,
         image_name = cl['spec'].get('cephVersion', {}).get('image', None)
         num_nodes = len(self.rook_cluster.get_node_names())
 
+        def sum_running_pods(service_type: str, service_name: Optional[str] = None) -> int:
+            all_pods = self.rook_cluster.describe_pods(None, None, None)
+            if service_name is None:
+                return sum(pod['phase'] == 'Running' for pod in all_pods if pod['labels']['app'] == f"rook-ceph-{service_type}")
+            else:
+                if service_type == 'mds':
+                    key = 'rook_file_system'
+                elif service_type == 'rgw':
+                    key = 'rook_object_store'
+                elif service_type == 'nfs':
+                    key = 'ceph_nfs'
+                else:
+                    self.log.error(f"Unknow service type {service_type}")
+                    return 0
+
+                return sum(pod['phase'] == 'Running' \
+                           for pod in all_pods \
+                           if pod['labels']['app'] == f"rook-ceph-{service_type}" \
+                           and service_name == pod['labels'][key])
+
         spec = {}
         if service_type == 'mon' or service_type is None:
             spec['mon'] = orchestrator.ServiceDescription(
@@ -269,6 +289,7 @@ def describe_service(self,
                 size=cl['spec'].get('mon', {}).get('count', 1),
                 container_image_name=image_name,
                 last_refresh=now,
+                running=sum_running_pods('mon')
             )
         if service_type == 'mgr' or service_type is None:
             spec['mgr'] = orchestrator.ServiceDescription(
@@ -279,6 +300,7 @@ def describe_service(self,
                 size=1,
                 container_image_name=image_name,
                 last_refresh=now,
+                running=sum_running_pods('mgr')
             )
 
         if (
@@ -293,13 +315,15 @@ def describe_service(self,
                 size=num_nodes,
                 container_image_name=image_name,
                 last_refresh=now,
+                running=sum_running_pods('crashcollector')
             )
 
         if service_type == 'mds' or service_type is None:
             # CephFilesystems
             all_fs = self.rook_cluster.get_resource("cephfilesystems")
             for fs in all_fs:
-                svc = 'mds.' + fs['metadata']['name']
+                fs_name = fs['metadata']['name']
+                svc = 'mds.' + fs_name
                 if svc in spec:
                     continue
                 # FIXME: we are conflating active (+ standby) with count
@@ -316,13 +340,15 @@ def describe_service(self,
                     size=total_mds,
                     container_image_name=image_name,
                     last_refresh=now,
+                    running=sum_running_pods('mds', fs_name)
                 )
 
         if service_type == 'rgw' or service_type is None:
             # CephObjectstores
             all_zones = self.rook_cluster.get_resource("cephobjectstores")
             for zone in all_zones:
-                svc = 'rgw.' + zone['metadata']['name']
+                zone_name = zone['metadata']['name']
+                svc = 'rgw.' + zone_name
                 if svc in spec:
                     continue
                 active = zone['spec']['gateway']['instances'];
@@ -344,6 +370,7 @@ def describe_service(self,
                     size=active,
                     container_image_name=image_name,
                     last_refresh=now,
+                    running=sum_running_pods('rgw', zone_name)
                 )
 
         if service_type == 'nfs' or service_type is None:
@@ -368,7 +395,7 @@ def describe_service(self,
                     ),
                     size=active,
                     last_refresh=now,
-                    running=len([1 for pod in nfs_pods if pod['labels']['ceph_nfs'] == nfs_name]),
+                    running=sum_running_pods('nfs', nfs_name),
                     created=creation_timestamp.astimezone(tz=datetime.timezone.utc)
                 )
         if service_type == 'osd' or service_type is None:
@@ -385,7 +412,7 @@ def describe_service(self,
                 ),
                 size=len(all_osds),
                 last_refresh=now,
-                running=sum(osd.status.phase == 'Running' for osd in all_osds)
+                running=sum_running_pods('osd')
             )
 
             # drivegroups
@@ -396,7 +423,7 @@ def describe_service(self,
                     size=0,
                     running=0,
                 )
-        
+
         if service_type == 'rbd-mirror' or service_type is None:
             # rbd-mirrors
             all_mirrors = self.rook_cluster.get_resource("cephrbdmirrors")
@@ -414,13 +441,13 @@ def describe_service(self,
                     ),
                     size=1,
                     last_refresh=now,
+                    running=sum_running_pods('rbd-mirror', mirror_name)
                 )
-        
+
         for dd in self._list_daemons():
             if dd.service_name() not in spec:
                 continue
             service = spec[dd.service_name()]
-            service.running += 1
             if not service.container_image_id:
                 service.container_image_id = dd.container_image_id
             if not service.container_image_name:
@@ -451,11 +478,25 @@ def _list_daemons(self,
                       daemon_id: Optional[str] = None,
                       host: Optional[str] = None,
                       refresh: bool = False) -> List[orchestrator.DaemonDescription]:
+
+        def _pod_to_servicename(pod: Dict[str, Any]) -> Optional[str]:
+            if 'ceph_daemon_type' not in pod['labels']:
+                return None
+            daemon_type = pod['labels']['ceph_daemon_type']
+            if daemon_type in ['mds', 'rgw', 'nfs', 'rbd-mirror']:
+                if 'app.kubernetes.io/part-of' in pod['labels']:
+                    service_name = f"{daemon_type}.{pod['labels']['app.kubernetes.io/part-of']}"
+                else:
+                    service_name = f"{daemon_type}"
+            else:
+                service_name = f"{daemon_type}"
+            return service_name
+
         pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host)
-        self.log.debug('pods %s' % pods)
         result = []
         for p in pods:
-            sd = orchestrator.DaemonDescription()
+            pod_svc_name = _pod_to_servicename(p)
+            sd = orchestrator.DaemonDescription(service_name=pod_svc_name)
             sd.hostname = p['hostname']
 
             # In Rook environments, the 'ceph-exporter' daemon is named 'exporter' whereas

From 46aac8e2643ec645dcc2c4114df535a488932942 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 3 Nov 2023 16:08:28 -0400
Subject: [PATCH 0791/2492] rgw/acl: remove empty base class ACLGrantee

an unnecessary base class of ACLGrantee_S3 which is just for s3
xml parsing

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_s3.h    | 2 +-
 src/rgw/rgw_acl_types.h | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index c234d722b997..49964c6aa2bc 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -25,7 +25,7 @@ class ACLPermission_S3 : public ACLPermission, public XMLObj
   void to_xml(std::ostream& out);
 };
 
-class ACLGrantee_S3 : public ACLGrantee, public XMLObj
+class ACLGrantee_S3 : public XMLObj
 {
 public:
   ACLGrantee_S3() {}
diff --git a/src/rgw/rgw_acl_types.h b/src/rgw/rgw_acl_types.h
index c76d085e0b55..a257bb4c9e6d 100644
--- a/src/rgw/rgw_acl_types.h
+++ b/src/rgw/rgw_acl_types.h
@@ -204,10 +204,3 @@ class ACLGranteeType
   friend bool operator!=(const ACLGranteeType& lhs, const ACLGranteeType& rhs);
 };
 WRITE_CLASS_ENCODER(ACLGranteeType)
-
-class ACLGrantee
-{
-public:
-  ACLGrantee() {}
-  ~ACLGrantee() {}
-};

From 5791e2222877ad4f2e84c79cc7aa867ee1361976 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Nov 2023 11:43:58 -0500
Subject: [PATCH 0792/2492] rgw: reorder rgw_user members for default
 operator<=>

the default operator<=> does a memberwise comparison the same that
rgw_user::compare() did, except that it compared `ns` before `id`

reorder the rgw_user members so that the default operator<=> can
replace compare() and the related comparison operators

replaces uses of rgw_user::compare() with operator== and !=

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_user.cc | 16 ++++++------
 src/rgw/rgw_acl_s3.cc            |  6 ++---
 src/rgw/rgw_op.cc                |  2 +-
 src/rgw/rgw_user_types.h         | 45 +++++---------------------------
 src/test/test_rgw_admin_meta.cc  |  2 +-
 5 files changed, 20 insertions(+), 51 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index 506ac0acf112..b5569e481c53 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -237,7 +237,7 @@ int RGWAccessKeyPool::init(RGWUserAdminOpState& op_state)
   }
 
   const rgw_user& uid = op_state.get_user_id();
-  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+  if (uid == rgw_user(RGW_USER_ANON_ID)) {
     keys_allowed = false;
     return -EINVAL;
   }
@@ -890,7 +890,7 @@ int RGWSubUserPool::init(RGWUserAdminOpState& op_state)
   }
 
   const rgw_user& uid = op_state.get_user_id();
-  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+  if (uid == rgw_user(RGW_USER_ANON_ID)) {
     subusers_allowed = false;
     return -EACCES;
   }
@@ -1198,7 +1198,7 @@ int RGWUserCapPool::init(RGWUserAdminOpState& op_state)
   }
 
   const rgw_user& uid = op_state.get_user_id();
-  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+  if (uid == rgw_user(RGW_USER_ANON_ID)) {
     caps_allowed = false;
     return -EACCES;
   }
@@ -1373,7 +1373,7 @@ int RGWUser::init(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state,
     }
   }
 
-  if (!user_id.empty() && (user_id.compare(RGW_USER_ANON_ID) != 0)) {
+  if (!user_id.empty() && user_id != rgw_user(RGW_USER_ANON_ID)) {
     user = driver->get_user(user_id);
     found = (user->load_user(dpp, y) >= 0);
     op_state.found_by_uid = found;
@@ -1477,12 +1477,12 @@ int RGWUser::check_op(RGWUserAdminOpState& op_state, std::string *err_msg)
   int ret = 0;
   const rgw_user& uid = op_state.get_user_id();
 
-  if (uid.compare(RGW_USER_ANON_ID) == 0) {
+  if (uid == rgw_user(RGW_USER_ANON_ID)) {
     set_err_msg(err_msg, "unable to perform operations on the anonymous user");
     return -EINVAL;
   }
 
-  if (is_populated() && user_id.compare(uid) != 0) {
+  if (is_populated() && user_id != uid) {
     set_err_msg(err_msg, "user id mismatch, operation id: " + uid.to_str()
             + " does not match: " + user_id.to_str());
 
@@ -1858,7 +1858,7 @@ int RGWUser::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
   }
 
   // ensure that we can modify the user's attributes
-  if (user_id.compare(RGW_USER_ANON_ID) == 0) {
+  if (user_id == rgw_user(RGW_USER_ANON_ID)) {
     set_err_msg(err_msg, "unable to modify anonymous user's info");
     return -EACCES;
   }
@@ -1870,7 +1870,7 @@ int RGWUser::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
     // make sure we are not adding a duplicate email
     if (old_email != op_email) {
       ret = driver->get_user_by_email(dpp, op_email, y, &duplicate_check);
-      if (ret >= 0 && duplicate_check->get_id().compare(user_id) != 0) {
+      if (ret >= 0 && duplicate_check->get_id() != user_id) {
         set_err_msg(err_msg, "cannot add duplicate email");
         return -ERR_EMAIL_EXIST;
       }
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 9f71e3281500..3fc072a33ca6 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -392,11 +392,11 @@ int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_own
     add_grant(&group_grant);
   } else if (canned_acl.compare("bucket-owner-read") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_READ);
-    if (bid.compare(owner.get_id()) != 0)
+    if (bid != owner.get_id())
       add_grant(&bucket_owner_grant);
   } else if (canned_acl.compare("bucket-owner-full-control") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
-    if (bid.compare(owner.get_id()) != 0)
+    if (bid != owner.get_id())
       add_grant(&bucket_owner_grant);
   } else {
     return -EINVAL;
@@ -489,7 +489,7 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
   ACLOwner *requested_owner = static_cast<ACLOwner_S3 *>(find_first("Owner"));
   if (requested_owner) {
     rgw_user& requested_id = requested_owner->get_id();
-    if (!requested_id.empty() && requested_id.compare(owner->get_id()) != 0)
+    if (!requested_id.empty() && requested_id != owner->get_id())
       return -EPERM;
   }
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index a290388efe0c..2193ff104c96 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -457,7 +457,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
       return ret;
     }
     const rgw_user& bucket_owner = bucket_policy.get_owner().get_id();
-    if (bucket_owner.compare(s->user->get_id()) != 0 &&
+    if (bucket_owner != s->user->get_id() &&
         ! s->auth.identity->is_admin_of(bucket_owner)) {
       auto r = eval_identity_or_session_policies(dpp, s->iam_user_policies, s->env,
                                   rgw::IAM::s3ListBucket, ARN(bucket->get_key()));
diff --git a/src/rgw/rgw_user_types.h b/src/rgw/rgw_user_types.h
index c9a1a46ade1e..1aaf4cfa5d34 100644
--- a/src/rgw/rgw_user_types.h
+++ b/src/rgw/rgw_user_types.h
@@ -26,9 +26,10 @@
 #include "common/Formatter.h"
 
 struct rgw_user {
+  // note: order of member variables matches the sort order of operator<=>
   std::string tenant;
-  std::string id;
   std::string ns;
+  std::string id;
 
   rgw_user() {}
   explicit rgw_user(const std::string& s) {
@@ -36,13 +37,13 @@ struct rgw_user {
   }
   rgw_user(const std::string& tenant, const std::string& id, const std::string& ns="")
     : tenant(tenant),
-      id(id),
-      ns(ns) {
+      ns(ns),
+      id(id) {
   }
   rgw_user(std::string&& tenant, std::string&& id, std::string&& ns="")
     : tenant(std::move(tenant)),
-      id(std::move(id)),
-      ns(std::move(ns)) {
+      ns(std::move(ns)),
+      id(std::move(id)) {
   }
 
   void encode(ceph::buffer::list& bl) const {
@@ -118,40 +119,8 @@ struct rgw_user {
     return *this;
   }
 
-  int compare(const rgw_user& u) const {
-    int r = tenant.compare(u.tenant);
-    if (r != 0)
-      return r;
-    r = ns.compare(u.ns);
-    if (r != 0) {
-      return r;
-    }
-    return id.compare(u.id);
-  }
-  int compare(const std::string& str) const {
-    rgw_user u(str);
-    return compare(u);
-  }
+  friend auto operator<=>(const rgw_user&, const rgw_user&) = default;
 
-  bool operator!=(const rgw_user& rhs) const {
-    return (compare(rhs) != 0);
-  }
-  bool operator==(const rgw_user& rhs) const {
-    return (compare(rhs) == 0);
-  }
-  bool operator<(const rgw_user& rhs) const {
-    if (tenant < rhs.tenant) {
-      return true;
-    } else if (tenant > rhs.tenant) {
-      return false;
-    }
-    if (ns < rhs.ns) {
-      return true;
-    } else if (ns > rhs.ns) {
-      return false;
-    }
-    return (id < rhs.id);
-  }
   void dump(ceph::Formatter *f) const;
   static void generate_test_instances(std::list<rgw_user*>& o);
 };
diff --git a/src/test/test_rgw_admin_meta.cc b/src/test/test_rgw_admin_meta.cc
index b1d5fad06009..00c43d10b549 100644
--- a/src/test/test_rgw_admin_meta.cc
+++ b/src/test/test_rgw_admin_meta.cc
@@ -460,7 +460,7 @@ int compare_access_keys(RGWAccessKey& k1, RGWAccessKey& k2) {
 int compare_user_info(RGWUserInfo& i1, RGWUserInfo& i2) {
   int rv;
 
-  if ((rv = i1.user_id.compare(i2.user_id)) != 0)
+  if ((rv = i1.user_id.id.compare(i2.user_id.id)) != 0)
     return rv;
   if ((rv = i1.display_name.compare(i2.display_name)) != 0)
     return rv;

From 4f2ce9b21c665abe668d4bfc41fee59c1ed30ff6 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 13 Nov 2023 13:14:13 -0500
Subject: [PATCH 0793/2492] rgw: ACLOwner as raw struct

this was a class with two protected members, but with a getter/setter
for each. this 'encapsulation' only added unnecessary boilerplate

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/posix/rgw_sal_posix.cc         |  2 +-
 src/rgw/driver/rados/rgw_bucket.cc            |  4 +--
 src/rgw/driver/rados/rgw_cr_rados.cc          |  6 ++--
 src/rgw/driver/rados/rgw_rados.cc             | 22 ++++++-------
 src/rgw/driver/rados/rgw_sal_rados.cc         | 22 +++++++------
 src/rgw/driver/rados/rgw_sync_module_es.cc    |  2 +-
 .../driver/rados/rgw_sync_module_es_rest.cc   |  2 +-
 src/rgw/driver/rados/rgw_tools.cc             |  3 +-
 src/rgw/rgw_acl.cc                            | 17 +++-------
 src/rgw/rgw_acl.h                             | 32 ++++++-------------
 src/rgw/rgw_acl_s3.cc                         | 31 +++++++++---------
 src/rgw/rgw_acl_s3.h                          |  2 +-
 src/rgw/rgw_acl_swift.cc                      | 10 +++---
 src/rgw/rgw_common.cc                         |  2 +-
 src/rgw/rgw_file.cc                           |  2 +-
 src/rgw/rgw_lc.cc                             |  6 ++--
 src/rgw/rgw_lib.cc                            |  4 +--
 src/rgw/rgw_log.cc                            | 10 +++---
 src/rgw/rgw_lua_request.cc                    |  5 ++-
 src/rgw/rgw_op.cc                             | 32 +++++++++----------
 src/rgw/rgw_rest_metadata.cc                  |  4 +--
 src/rgw/rgw_rest_pubsub.cc                    | 32 +++++++++----------
 src/rgw/rgw_rest_s3.cc                        | 14 ++++----
 src/rgw/rgw_sal_dbstore.cc                    | 10 +++---
 src/test/rgw/test_rgw_lua.cc                  | 23 ++++++-------
 25 files changed, 138 insertions(+), 161 deletions(-)

diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 766afd141ca6..5c1e50ca5c93 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -916,7 +916,7 @@ int POSIXBucket::set_acl(const DoutPrefixProvider* dpp,
   acl.encode(aclbl);
 
   attrs[RGW_ATTR_ACL] = aclbl;
-  info.owner = acl.get_owner().get_id();
+  info.owner = acl.get_owner().id;
 
   return write_attrs(dpp, y);
 }
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index bea3e9b09e5a..5ac049af3499 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1068,9 +1068,9 @@ int RGWBucketAdminOp::link(rgw::sal::Driver* driver, RGWBucketAdminOpState& op_s
     return -EIO;
   }
 
-  int r = static_cast<rgw::sal::RadosStore*>(driver)->ctl()->bucket->unlink_bucket(owner.get_id(), old_bucket->get_info().bucket, y, dpp, false);
+  int r = static_cast<rgw::sal::RadosStore*>(driver)->ctl()->bucket->unlink_bucket(owner.id, old_bucket->get_info().bucket, y, dpp, false);
   if (r < 0) {
-    set_err_msg(err, "could not unlink policy from user " + owner.get_id().to_str());
+    set_err_msg(err, "could not unlink policy from user " + owner.id.to_str());
     return r;
   }
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index f903233cb02c..2a4a02f29adc 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -906,7 +906,7 @@ int RGWAsyncRemoveObj::_send_request(const DoutPrefixProvider *dpp)
 
   std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = obj->get_delete_op();
 
-  del_op->params.bucket_owner = bucket->get_info().owner;
+  del_op->params.bucket_owner.id = bucket->get_info().owner;
   del_op->params.obj_owner = policy.get_owner();
   if (del_if_older) {
     del_op->params.unmod_since = timestamp;
@@ -916,8 +916,8 @@ int RGWAsyncRemoveObj::_send_request(const DoutPrefixProvider *dpp)
   }
   del_op->params.olh_epoch = versioned_epoch;
   del_op->params.marker_version_id = marker_version_id;
-  del_op->params.obj_owner.set_id(rgw_user(owner));
-  del_op->params.obj_owner.set_name(owner_display_name);
+  del_op->params.obj_owner.id = rgw_user(owner);
+  del_op->params.obj_owner.display_name = owner_display_name;
   del_op->params.mtime = timestamp;
   del_op->params.high_precision_time = true;
   del_op->params.zones_trace = &zones_trace;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 592d7a828bc1..e85552bc6605 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4327,8 +4327,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     }
 
     ACLOwner new_owner;
-    new_owner.set_id(*override_owner);
-    new_owner.set_name(owner_info.display_name);
+    new_owner.id = *override_owner;
+    new_owner.display_name = owner_info.display_name;
 
     acl.set_owner(new_owner);
 
@@ -5152,7 +5152,7 @@ int RGWRados::set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner, const DoutPr
     return r;
   }
 
-  info.owner = owner.get_id();
+  info.owner = owner.id;
 
   r = put_bucket_instance_info(info, false, real_time(), &attrs, dpp, y);
   if (r < 0) {
@@ -5620,8 +5620,8 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
 
       struct rgw_bucket_dir_entry_meta meta;
 
-      meta.owner = params.obj_owner.get_id().to_str();
-      meta.owner_display_name = params.obj_owner.get_display_name();
+      meta.owner = params.obj_owner.id.to_str();
+      meta.owner_display_name = params.obj_owner.display_name;
 
       if (real_clock::is_zero(params.mtime)) {
         meta.mtime = real_clock::now();
@@ -6892,8 +6892,8 @@ int RGWRados::Bucket::UpdateIndex::complete(const DoutPrefixProvider *dpp, int64
       ldpp_dout(dpp, 0) << "WARNING: could not decode policy ret=" << ret << dendl;
     }
   }
-  ent.meta.owner = owner.get_id().to_str();
-  ent.meta.owner_display_name = owner.get_display_name();
+  ent.meta.owner = owner.id.to_str();
+  ent.meta.owner_display_name = owner.display_name;
   ent.meta.content_type = content_type;
   ent.meta.appendable = appendable;
 
@@ -10109,8 +10109,8 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp,
   object.meta.etag = etag;
   object.meta.content_type = content_type;
   object.meta.storage_class = storage_class;
-  object.meta.owner = owner.get_id().to_str();
-  object.meta.owner_display_name = owner.get_display_name();
+  object.meta.owner = owner.id.to_str();
+  object.meta.owner_display_name = owner.display_name;
   object.meta.appendable = appendable;
 
   // encode suggested updates
@@ -10139,8 +10139,8 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp,
     list_state.tag = astate->obj_tag.c_str();
   }
 
-  list_state.meta.owner = owner.get_id().to_str();
-  list_state.meta.owner_display_name = owner.get_display_name();
+  list_state.meta.owner = owner.id.to_str();
+  list_state.meta.owner_display_name = owner.display_name;
 
   list_state.exists = true;
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 0d7359ea90a7..4e71595736b4 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -684,7 +684,7 @@ int RadosBucket::set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy &
   map<string, bufferlist>& attrs = get_attrs();
 
   attrs[RGW_ATTR_ACL] = aclbl;
-  info.owner = acl.get_owner().get_id();
+  info.owner = acl.get_owner().id;
 
   int r = store->ctl()->bucket->store_bucket_instance_info(info.bucket,
                  info, y, dpp,
@@ -766,8 +766,10 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp,
   if (!results.objs.empty()) {
     for (const rgw_bucket_dir_entry& dentry : results.objs) {
       rgw_obj_key key(dentry.key);
-      ACLOwner owner(rgw_user(dentry.meta.owner));
-      owner.set_name(dentry.meta.owner_display_name);
+      const ACLOwner owner{
+        .id = rgw_user(dentry.meta.owner),
+        .display_name = dentry.meta.owner_display_name
+      };
       uploads.push_back(this->get_multipart_upload(key.name,
 			std::nullopt, std::move(owner), dentry.meta.mtime));
     }
@@ -1663,7 +1665,7 @@ int RadosObject::chown(User& new_user, const DoutPrefixProvider* dpp, optional_y
   RGWAccessControlList& acl = policy.get_acl();
 
   //Remove grant that is set to old owner
-  acl.remove_canon_user_grant(owner.get_id());
+  acl.remove_canon_user_grant(owner.id);
 
   //Create a grant and add grant
   ACLGrant grant;
@@ -1671,8 +1673,8 @@ int RadosObject::chown(User& new_user, const DoutPrefixProvider* dpp, optional_y
   acl.add_grant(&grant);
 
   //Update the ACL owner to the new user
-  owner.set_id(new_user.get_id());
-  owner.set_name(new_user.get_display_name());
+  owner.id = new_user.get_id();
+  owner.display_name = new_user.get_display_name();
   policy.set_owner(owner);
 
   bl.clear();
@@ -1986,7 +1988,7 @@ RadosObject::RadosDeleteOp::RadosDeleteOp(RadosObject *_source) :
 
 int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y)
 {
-  parent_op.params.bucket_owner = params.bucket_owner.get_id();
+  parent_op.params.bucket_owner = params.bucket_owner.id;
   parent_op.params.versioning_status = params.versioning_status;
   parent_op.params.obj_owner = params.obj_owner;
   parent_op.params.olh_epoch = params.olh_epoch;
@@ -2220,7 +2222,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
   }
 
   std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = meta_obj->get_delete_op();
-  del_op->params.bucket_owner = bucket->get_info().owner;
+  del_op->params.bucket_owner.id = bucket->get_info().owner;
   del_op->params.versioning_status = 0;
   if (!remove_objs.empty()) {
     del_op->params.remove_objs = &remove_objs;
@@ -2272,7 +2274,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     RGWRados::Object::Write obj_op(&op_target);
 
     op_target.set_versioning_disabled(true); /* no versioning for multipart meta */
-    obj_op.meta.owner = owner.get_id();
+    obj_op.meta.owner = owner.id;
     obj_op.meta.category = RGWObjCategory::MultiMeta;
     obj_op.meta.flags = PUT_OBJ_CREATE_EXCL;
     obj_op.meta.mtime = &mtime;
@@ -2561,7 +2563,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   obj_op.meta.remove_objs = &remove_objs;
 
   obj_op.meta.ptag = &tag; /* use req_id as operation tag */
-  obj_op.meta.owner = owner.get_id();
+  obj_op.meta.owner = owner.id;
   obj_op.meta.flags = PUT_OBJ_CREATE;
   obj_op.meta.modify_tail = true;
   obj_op.meta.completeMultipart = true;
diff --git a/src/rgw/driver/rados/rgw_sync_module_es.cc b/src/rgw/driver/rados/rgw_sync_module_es.cc
index 13c2057bf779..1030d9dce351 100644
--- a/src/rgw/driver/rados/rgw_sync_module_es.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_es.cc
@@ -501,7 +501,7 @@ struct es_obj_metadata {
 
         const RGWAccessControlList& acl = policy.get_acl();
 
-        permissions.insert(policy.get_owner().get_id().to_str());
+        permissions.insert(policy.get_owner().id.to_str());
         for (auto acliter : acl.get_grant_map()) {
           const ACLGrant& grant = acliter.second;
           if (grant.get_type().get_type() == ACL_TYPE_CANON_USER &&
diff --git a/src/rgw/driver/rados/rgw_sync_module_es_rest.cc b/src/rgw/driver/rados/rgw_sync_module_es_rest.cc
index db9d48adb366..b456f0b4097a 100644
--- a/src/rgw/driver/rados/rgw_sync_module_es_rest.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_es_rest.cc
@@ -347,7 +347,7 @@ class RGWMetadataSearch_ObjStore_S3 : public RGWMetadataSearchOp {
       s->formatter->dump_format("ETag", "\"%s\"", e.meta.etag.c_str());
       s->formatter->dump_string("ContentType", e.meta.content_type.c_str());
       s->formatter->dump_string("StorageClass", e.meta.storage_class.c_str());
-      dump_owner(s, e.owner.get_id(), e.owner.get_display_name());
+      dump_owner(s, e.owner.id, e.owner.display_name);
       s->formatter->open_array_section("CustomMetadata");
       for (auto& m : e.meta.custom_str) {
         s->formatter->open_object_section("Entry");
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index aa387cb450d5..a9454c502dfb 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -355,8 +355,7 @@ int RGWDataAccess::Object::put(bufferlist& data,
   string req_id = driver->zone_unique_id(driver->get_new_req_id());
 
   std::unique_ptr<rgw::sal::Writer> processor;
-  processor = driver->get_atomic_writer(dpp, y, obj.get(),
-				       owner.get_id(),
+  processor = driver->get_atomic_writer(dpp, y, obj.get(), owner.id,
 				       nullptr, olh_epoch, req_id);
 
   int ret = processor->prepare(y);
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index f32a73f26529..7177a5f822ef 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -61,13 +61,6 @@ bool operator!=(const RGWAccessControlList& lhs,
   return !(lhs == rhs);
 }
 
-bool operator==(const ACLOwner& lhs, const ACLOwner& rhs) {
-  return lhs.id == rhs.id && lhs.display_name == rhs.display_name;
-}
-bool operator!=(const ACLOwner& lhs, const ACLOwner& rhs) {
-  return !(lhs == rhs);
-}
-
 bool operator==(const RGWAccessControlPolicy& lhs,
                 const RGWAccessControlPolicy& rhs) {
   return lhs.acl == rhs.acl && lhs.owner == rhs.owner;
@@ -187,7 +180,7 @@ uint32_t RGWAccessControlPolicy::get_perm(const DoutPrefixProvider* dpp,
 
   uint32_t perm = acl.get_perm(dpp, auth_identity, perm_mask);
 
-  if (auth_identity.is_owner_of(owner.get_id())) {
+  if (auth_identity.is_owner_of(owner.id)) {
     perm |= perm_mask & (RGW_PERM_READ_ACP | RGW_PERM_WRITE_ACP);
   }
 
@@ -211,7 +204,7 @@ uint32_t RGWAccessControlPolicy::get_perm(const DoutPrefixProvider* dpp,
   }
 
   ldpp_dout(dpp, 5) << "-- Getting permissions done for identity=" << auth_identity
-                << ", owner=" << owner.get_id()
+                << ", owner=" << owner.id
                 << ", perm=" << perm << dendl;
 
   return perm;
@@ -367,10 +360,8 @@ void RGWAccessControlPolicy::generate_test_instances(list<RGWAccessControlPolicy
     RGWAccessControlList *l = *iter;
     p->acl = *l;
 
-    string name = "radosgw";
-    rgw_user id("rgw");
-    p->owner.set_name(name);
-    p->owner.set_id(id);
+    p->owner.id.id = "rgw";
+    p->owner.display_name = "radosgw";
 
     o.push_back(p);
 
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index c520501583b8..f2dacf285432 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -298,15 +298,9 @@ class RGWAccessControlList
 };
 WRITE_CLASS_ENCODER(RGWAccessControlList)
 
-class ACLOwner
-{
-protected:
+struct ACLOwner {
   rgw_user id;
   std::string display_name;
-public:
-  ACLOwner() {}
-  ACLOwner(const rgw_user& _id) : id(_id) {}
-  ~ACLOwner() {}
 
   void encode(bufferlist& bl) const {
     ENCODE_START(3, 2, bl);
@@ -327,15 +321,8 @@ class ACLOwner
   void dump(Formatter *f) const;
   void decode_json(JSONObj *obj);
   static void generate_test_instances(std::list<ACLOwner*>& o);
-  void set_id(const rgw_user& _id) { id = _id; }
-  void set_name(const std::string& name) { display_name = name; }
-
-  rgw_user& get_id() { return id; }
-  const rgw_user& get_id() const { return id; }
-  std::string& get_display_name() { return display_name; }
-  const std::string& get_display_name() const { return display_name; }
-  friend bool operator==(const ACLOwner& lhs, const ACLOwner& rhs);
-  friend bool operator!=(const ACLOwner& lhs, const ACLOwner& rhs);
+
+  auto operator<=>(const ACLOwner&) const = default;
 };
 WRITE_CLASS_ENCODER(ACLOwner)
 
@@ -388,15 +375,14 @@ class RGWAccessControlPolicy
     DECODE_FINISH(bl);
   }
 
-  void set_owner(ACLOwner& o) { owner = o; }
-  ACLOwner& get_owner() {
-    return owner;
-  }
+  void set_owner(const ACLOwner& o) { owner = o; }
+  const ACLOwner& get_owner() const { return owner; }
+  ACLOwner& get_owner() { return owner; }
 
-  void create_default(const rgw_user& id, std::string& name) {
+  void create_default(const rgw_user& id, const std::string& name) {
     acl.create_default(id, name);
-    owner.set_id(id);
-    owner.set_name(name);
+    owner.id = id;
+    owner.display_name = name;
   }
   RGWAccessControlList& get_acl() {
     return acl;
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 3fc072a33ca6..46a427572be0 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -366,11 +366,11 @@ int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_own
 
   ACLGrant owner_grant;
 
-  rgw_user bid = bucket_owner.get_id();
-  string bname = bucket_owner.get_display_name();
+  const rgw_user& bid = bucket_owner.id;
+  const std::string& bname = bucket_owner.display_name;
 
   /* owner gets full control */
-  owner_grant.set_canon(owner.get_id(), owner.get_display_name(), RGW_PERM_FULL_CONTROL);
+  owner_grant.set_canon(owner.id, owner.display_name, RGW_PERM_FULL_CONTROL);
   add_grant(&owner_grant);
 
   if (canned_acl.size() == 0 || canned_acl.compare("private") == 0) {
@@ -392,12 +392,14 @@ int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_own
     add_grant(&group_grant);
   } else if (canned_acl.compare("bucket-owner-read") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_READ);
-    if (bid != owner.get_id())
+    if (bid != owner.id) {
       add_grant(&bucket_owner_grant);
+    }
   } else if (canned_acl.compare("bucket-owner-full-control") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
-    if (bid != owner.get_id())
+    if (bid != owner.id) {
       add_grant(&bucket_owner_grant);
+    }
   } else {
     return -EINVAL;
   }
@@ -483,28 +485,27 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
 				       rgw::sal::Driver* driver, ACLOwner *owner,
 				       RGWAccessControlPolicy& dest, std::string &err_msg)
 {
-  if (!owner)
+  if (!owner || owner->id.empty()) {
     return -EINVAL;
+  }
 
   ACLOwner *requested_owner = static_cast<ACLOwner_S3 *>(find_first("Owner"));
-  if (requested_owner) {
-    rgw_user& requested_id = requested_owner->get_id();
-    if (!requested_id.empty() && requested_id != owner->get_id())
-      return -EPERM;
+  if (requested_owner && requested_owner->id != owner->id) {
+    return -EPERM;
   }
 
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(owner->get_id());
+  std::unique_ptr<rgw::sal::User> user = driver->get_user(owner->id);
   if (user->load_user(dpp, null_yield) < 0) {
     ldpp_dout(dpp, 10) << "owner info does not exist" << dendl;
     err_msg = "Invalid id";
     return -EINVAL;
   }
   ACLOwner& dest_owner = dest.get_owner();
-  dest_owner.set_id(owner->get_id());
-  dest_owner.set_name(user->get_display_name());
+  dest_owner.id = owner->id;
+  dest_owner.display_name = user->get_display_name();
 
-  ldpp_dout(dpp, 20) << "owner id=" << owner->get_id() << dendl;
-  ldpp_dout(dpp, 20) << "dest owner id=" << dest.get_owner().get_id() << dendl;
+  ldpp_dout(dpp, 20) << "owner id=" << owner->id << dendl;
+  ldpp_dout(dpp, 20) << "dest owner id=" << dest.get_owner().id << dendl;
 
   RGWAccessControlList& dst_acl = dest.get_acl();
 
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index 49964c6aa2bc..c58307600d44 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -89,7 +89,7 @@ class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
 
   virtual int create_canned(ACLOwner& _owner, ACLOwner& bucket_owner, const std::string& canned_acl) {
     RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
-    if (_owner.get_id() == rgw_user("anonymous")) {
+    if (_owner.id == rgw_user("anonymous")) {
       owner = bucket_owner;
     } else {
       owner = _owner;
diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index c6f06bbd37cc..060cfabed83e 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -188,8 +188,8 @@ int RGWAccessControlPolicy_SWIFT::create(const DoutPrefixProvider *dpp,
                                          uint32_t& rw_mask)
 {
   acl.create_default(id, name);
-  owner.set_id(id);
-  owner.set_name(name);
+  owner.id = id;
+  owner.display_name = name;
   rw_mask = 0;
 
   if (read_list) {
@@ -338,8 +338,8 @@ bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
                                               const std::string& acl_str)
 {
   acl.create_default(id, name);
-  owner.set_id(id);
-  owner.set_name(name);
+  owner.id = id;
+  owner.display_name = name;
 
   JSONParser parser;
 
@@ -395,7 +395,7 @@ boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
         continue;
       }
       id = SWIFT_GROUP_ALL_USERS;
-    } else if (owner.get_id() == id) {
+    } else if (owner.id == id) {
       continue;
     }
 
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 6c07e3232435..6ec10f2cc2f4 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -1421,7 +1421,7 @@ int verify_bucket_owner_or_policy(req_state* const s,
       identity_policy_res == Effect::Allow ||
       (e == Effect::Pass &&
        identity_policy_res == Effect::Pass &&
-       s->auth.identity->is_owner_of(s->bucket_owner.get_id()))) {
+       s->auth.identity->is_owner_of(s->bucket_owner.id))) {
     return 0;
   } else {
     return -EACCES;
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index e1217173feda..f0100e37c2bb 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1872,7 +1872,7 @@ namespace rgw {
       }
     }
     processor = get_driver()->get_atomic_writer(this, state->yield, state->object.get(),
-					 state->bucket_owner.get_id(),
+					 state->bucket_owner.id,
 					 &state->dest_placement, 0, state->req_id);
 
     op_ret = processor->prepare(state->yield);
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 260ad1795bc0..2bef2ef31c39 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -546,9 +546,9 @@ static int remove_expired_obj(
     = obj->get_delete_op();
   del_op->params.versioning_status
     = obj->get_bucket()->get_info().versioning_status();
-  del_op->params.obj_owner.set_id(rgw_user {meta.owner});
-  del_op->params.obj_owner.set_name(meta.owner_display_name);
-  del_op->params.bucket_owner.set_id(bucket_info.owner);
+  del_op->params.obj_owner.id = rgw_user{meta.owner};
+  del_op->params.obj_owner.display_name = meta.owner_display_name;
+  del_op->params.bucket_owner.id = bucket_info.owner;
   del_op->params.unmod_since = meta.mtime;
 
   // notification supported only for RADOS driver for now
diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index 5b0a3c728490..8185ec042400 100644
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -601,8 +601,8 @@ namespace rgw {
     s->perm_mask = RGW_PERM_FULL_CONTROL;
 
     // populate the owner info
-    s->owner.set_id(s->user->get_id());
-    s->owner.set_name(s->user->get_display_name());
+    s->owner.id = s->user->get_id();
+    s->owner.display_name = s->user->get_display_name();
 
     return 0;
   } /* RGWHandler_Lib::authorize */
diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 9bc27bbe9ffa..8b8f85ef07ba 100644
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -207,14 +207,14 @@ static void log_usage(req_state *s, const string& op_name)
   bucket_name = s->bucket_name;
 
   if (!bucket_name.empty()) {
-  bucket_name = s->bucket_name;
-    user = s->bucket_owner.get_id();
+    bucket_name = s->bucket_name;
+    user = s->bucket_owner.id;
     if (!rgw::sal::Bucket::empty(s->bucket.get()) &&
 	s->bucket->get_info().requester_pays) {
       payer = s->user->get_id();
     }
   } else {
-      user = s->user->get_id();
+    user = s->user->get_id();
   }
 
   bool error = s->err.is_err();
@@ -648,8 +648,8 @@ int rgw_log_op(RGWREST* const rest, req_state *s, const RGWOp* op, OpsLogSink *o
 
   entry.user = s->user->get_id().to_str();
   if (s->object_acl)
-    entry.object_owner = s->object_acl->get_owner().get_id();
-  entry.bucket_owner = s->bucket_owner.get_id();
+    entry.object_owner = s->object_acl->get_owner().id;
+  entry.bucket_owner = s->bucket_owner.id;
 
   uint64_t bytes_sent = ACCOUNTING_IO(s)->get_bytes_sent();
   uint64_t bytes_received = ACCOUNTING_IO(s)->get_bytes_received();
diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc
index cb819deff14c..afef18be47bf 100644
--- a/src/rgw/rgw_lua_request.cc
+++ b/src/rgw/rgw_lua_request.cc
@@ -260,10 +260,9 @@ struct OwnerMetaTable : public EmptyMetaTable {
     const char* index = luaL_checkstring(L, 2);
 
     if (strcasecmp(index, "DisplayName") == 0) {
-      pushstring(L, owner->get_display_name());
+      pushstring(L, owner->display_name);
     } else if (strcasecmp(index, "User") == 0) {
-      create_metatable<UserMetaTable>(L, name, index, false, 
-          &(owner->get_id()));
+      create_metatable<UserMetaTable>(L, name, index, false, &owner->id);
     } else {
       return error_unknown_field(L, index, name);
     }
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 2193ff104c96..5574162aab04 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -456,7 +456,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
     if (ret < 0) {
       return ret;
     }
-    const rgw_user& bucket_owner = bucket_policy.get_owner().get_id();
+    const rgw_user& bucket_owner = bucket_policy.get_owner().id;
     if (bucket_owner != s->user->get_id() &&
         ! s->auth.identity->is_admin_of(bucket_owner)) {
       auto r = eval_identity_or_session_policies(dpp, s->iam_user_policies, s->env,
@@ -577,7 +577,7 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 			     s->bucket_acl.get(), s->bucket->get_key(), y);
     acct_acl_user = {
       s->bucket->get_info().owner,
-      s->bucket_acl->get_owner().get_display_name(),
+      s->bucket_acl->get_owner().display_name,
     };
 
     s->bucket_owner = s->bucket_acl->get_owner();
@@ -1449,7 +1449,7 @@ int RGWOp::init_quota()
 			driver->get_user(s->bucket->get_info().owner);
   rgw::sal::User* user;
 
-  if (s->user->get_id() == s->bucket_owner.get_id()) {
+  if (s->user->get_id() == s->bucket_owner.id) {
     user = s->user.get();
   } else {
     int r = owner_user->load_user(this, s->yield);
@@ -3527,8 +3527,8 @@ void RGWCreateBucket::execute(optional_yield y)
     }
   }
 
-  s->bucket_owner.set_id(s->user->get_id());
-  s->bucket_owner.set_name(s->user->get_display_name());
+  s->bucket_owner.id = s->user->get_id();
+  s->bucket_owner.display_name = s->user->get_display_name();
   createparams.owner = s->user->get_id();
 
   buffer::list aclbl;
@@ -3860,7 +3860,7 @@ int RGWPutObj::verify_permission(optional_yield y)
     }
 
     /* admin request overrides permission checks */
-    if (! s->auth.identity->is_admin_of(cs_acl.get_owner().get_id())) {
+    if (! s->auth.identity->is_admin_of(cs_acl.get_owner().id)) {
       if (policy || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
         //add source object tags for permission evaluation
         auto [has_s3_existing_tag, has_s3_resource_tag] = rgw_check_policy_condition(this, policy, s->iam_user_policies, s->session_policies);
@@ -4269,7 +4269,7 @@ void RGWPutObj::execute(optional_yield y)
       return;
     }
     processor = driver->get_append_writer(this, s->yield, s->object.get(),
-					 s->bucket_owner.get_id(),
+					 s->bucket_owner.id,
 					 pdest_placement, s->req_id, position,
 					 &cur_accounted_size);
   } else {
@@ -4282,7 +4282,7 @@ void RGWPutObj::execute(optional_yield y)
       }
     }
     processor = driver->get_atomic_writer(this, s->yield, s->object.get(),
-					 s->bucket_owner.get_id(),
+					 s->bucket_owner.id,
 					 pdest_placement, olh_epoch, s->req_id);
   }
 
@@ -4688,7 +4688,7 @@ void RGWPostObj::execute(optional_yield y)
 
     std::unique_ptr<rgw::sal::Writer> processor;
     processor = driver->get_atomic_writer(this, s->yield, obj.get(),
-					 s->bucket_owner.get_id(),
+					 s->bucket_owner.id,
 					 &s->dest_placement, 0, s->req_id);
     op_ret = processor->prepare(s->yield);
     if (op_ret < 0) {
@@ -5512,7 +5512,7 @@ int RGWCopyObj::verify_permission(optional_yield y)
     }
 
     /* admin request overrides permission checks */
-    if (!s->auth.identity->is_admin_of(src_acl.get_owner().get_id())) {
+    if (!s->auth.identity->is_admin_of(src_acl.get_owner().id)) {
       if (src_policy || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
         auto [has_s3_existing_tag, has_s3_resource_tag] = rgw_check_policy_condition(this, src_policy, s->iam_user_policies, s->session_policies);
         if (has_s3_existing_tag || has_s3_resource_tag)
@@ -5596,7 +5596,7 @@ int RGWCopyObj::verify_permission(optional_yield y)
   }
   auto dest_iam_policy = get_iam_policy_from_attr(s->cct, s->bucket->get_attrs(), s->bucket->get_tenant());
   /* admin request overrides permission checks */
-  if (! s->auth.identity->is_admin_of(dest_policy.get_owner().get_id())){
+  if (! s->auth.identity->is_admin_of(dest_policy.get_owner().id)){
     if (dest_iam_policy != boost::none || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
       //Add destination bucket tags for authorization
       auto [has_s3_existing_tag, has_s3_resource_tag] = rgw_check_policy_condition(this, dest_iam_policy, s->iam_user_policies, s->session_policies);
@@ -7445,7 +7445,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
   if (!path.obj_key.empty()) { // object deletion
     ACLOwner bucket_owner;
 
-    bucket_owner.set_id(bucket->get_info().owner);
+    bucket_owner.id = bucket->get_info().owner;
     std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(path.obj_key);
     obj->set_atomic();
 
@@ -7858,7 +7858,7 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
 
   std::unique_ptr<rgw::sal::Writer> processor;
   processor = driver->get_atomic_writer(this, s->yield, obj.get(),
-				       bowner.get_id(),
+				       bowner.id,
 				       &s->dest_placement, 0, s->req_id);
   op_ret = processor->prepare(s->yield);
   if (op_ret < 0) {
@@ -8243,7 +8243,7 @@ void RGWGetObjLayout::execute(optional_yield y)
 
 int RGWConfigBucketMetaSearch::verify_permission(optional_yield y)
 {
-  if (!s->auth.identity->is_owner_of(s->bucket_owner.get_id())) {
+  if (!s->auth.identity->is_owner_of(s->bucket_owner.id)) {
     return -EACCES;
   }
 
@@ -8276,7 +8276,7 @@ void RGWConfigBucketMetaSearch::execute(optional_yield y)
 
 int RGWGetBucketMetaSearch::verify_permission(optional_yield y)
 {
-  if (!s->auth.identity->is_owner_of(s->bucket_owner.get_id())) {
+  if (!s->auth.identity->is_owner_of(s->bucket_owner.id)) {
     return -EACCES;
   }
 
@@ -8290,7 +8290,7 @@ void RGWGetBucketMetaSearch::pre_exec()
 
 int RGWDelBucketMetaSearch::verify_permission(optional_yield y)
 {
-  if (!s->auth.identity->is_owner_of(s->bucket_owner.get_id())) {
+  if (!s->auth.identity->is_owner_of(s->bucket_owner.id)) {
     return -EACCES;
   }
 
diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc
index 23f78819c645..d7881c228909 100644
--- a/src/rgw/rgw_rest_metadata.cc
+++ b/src/rgw/rgw_rest_metadata.cc
@@ -69,9 +69,7 @@ void RGWOp_Metadata_Get::execute(optional_yield y) {
 }
 
 void RGWOp_Metadata_Get_Myself::execute(optional_yield y) {
-  string owner_id;
-
-  owner_id = s->owner.get_id().to_str();
+  const std::string owner_id = s->owner.id.to_str();
   s->info.args.append("key", owner_id);
 
   return RGWOp_Metadata_Get::execute(y);
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 843b7eb7d084..f43a9ef27228 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -72,7 +72,7 @@ std::optional<rgw::IAM::Policy> get_policy_from_text(req_state* const s,
   const auto bl = bufferlist::static_from_string(policy_text);
   try {
     return rgw::IAM::Policy(
-        s->cct, s->owner.get_id().tenant, bl,
+        s->cct, s->owner.id.tenant, bl,
         s->cct->_conf.get_val<bool>("rgw_policy_reject_invalid_principals"));
   } catch (rgw::IAM::PolicyParseException& e) {
     ldout(s->cct, 1) << "failed to parse policy:' " << policy_text
@@ -86,7 +86,7 @@ int verify_topic_owner_or_policy(req_state* const s,
                                  const rgw_pubsub_topic& topic,
                                  const std::string& zonegroup_name,
                                  const uint64_t op) {
-  if (topic.user == s->owner.get_id()) {
+  if (topic.user == s->owner.id) {
     return 0;
   }
   // no policy set.
@@ -187,7 +187,7 @@ class RGWPSCreateTopicOp : public RGWOp {
       return ret;
     }
 
-    const RGWPubSub ps(driver, s->owner.get_id().tenant);
+    const RGWPubSub ps(driver, s->owner.id.tenant);
     rgw_pubsub_topic result;
     ret = ps.get_topic(this, topic_name, result, y);
     if (ret == -ENOENT) {
@@ -195,7 +195,7 @@ class RGWPSCreateTopicOp : public RGWOp {
       return 0;
     }
     if (ret == 0) {
-      if (result.user == s->owner.get_id() ||
+      if (result.user == s->owner.id ||
           !s->cct->_conf->mandatory_topic_permissions) {
         return 0;
       }
@@ -251,9 +251,9 @@ void RGWPSCreateTopicOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
-                           s->owner.get_id(), policy_text, y);
+                           s->owner.id, policy_text, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to create topic '" << topic_name << "', ret=" << op_ret << dendl;
     return;
@@ -306,7 +306,7 @@ class RGWPSListTopicsOp : public RGWOp {
 };
 
 void RGWPSListTopicsOp::execute(optional_yield y) {
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   op_ret = ps.get_topics(this, result, y);
   // if there are no topics it is not considered an error
   op_ret = op_ret == -ENOENT ? 0 : op_ret;
@@ -393,7 +393,7 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -477,7 +477,7 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -605,7 +605,7 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
       return ret;
     }
     rgw_pubsub_topic result;
-    const RGWPubSub ps(driver, s->owner.get_id().tenant);
+    const RGWPubSub ps(driver, s->owner.id.tenant);
     ret = ps.get_topic(this, topic_name, result, y);
     if (ret < 0) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name
@@ -672,7 +672,7 @@ void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
                            topic_owner, policy_text, y);
   if (op_ret < 0) {
@@ -742,7 +742,7 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   rgw_pubsub_topic result;
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret == 0) {
@@ -970,7 +970,7 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     return;
   }
 
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   const RGWPubSub::Bucket b(ps, bucket.get());
 
   if(configurations.list.empty()) {
@@ -1042,7 +1042,7 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     // ARN is cached to make the "GET" method faster
     op_ret = ps.create_topic(this, unique_topic_name, topic_info.dest,
                              topic_info.arn, topic_info.opaque_data,
-                             s->owner.get_id(), topic_info.policy_text, y);
+                             s->owner.id, topic_info.policy_text, y);
     if (op_ret < 0) {
       ldpp_dout(this, 1) << "failed to auto-generate unique topic '" << unique_topic_name << 
         "', ret=" << op_ret << dendl;
@@ -1118,7 +1118,7 @@ void RGWPSDeleteNotifOp::execute(optional_yield y) {
     return;
   }
 
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   const RGWPubSub::Bucket b(ps, bucket.get());
 
   // get all topics on a bucket
@@ -1215,7 +1215,7 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
     return;
   }
 
-  const RGWPubSub ps(driver, s->owner.get_id().tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant);
   const RGWPubSub::Bucket b(ps, bucket.get());
   
   // get all topics on a bucket
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index f12d5e733007..dbc75955940f 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3162,8 +3162,8 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y)
       return -EACCES;
     } else {
       /* Populate the owner info. */
-      s->owner.set_id(s->user->get_id());
-      s->owner.set_name(s->user->get_display_name());
+      s->owner.id = s->user->get_id();
+      s->owner.display_name = s->user->get_display_name();
       ldpp_dout(this, 20) << "Successful Signature Verification!" << dendl;
     }
 
@@ -4107,7 +4107,7 @@ void RGWListMultipart_ObjStore_S3::send_response()
     s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
 
     ACLOwner& owner = policy.get_owner();
-    dump_owner(s, owner.get_id(), owner.get_display_name());
+    dump_owner(s, owner.id, owner.display_name);
 
     for (; iter != upload->get_parts().end(); ++iter) {
       rgw::sal::MultipartPart* part = iter->second.get();
@@ -4170,8 +4170,8 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
       }
       s->formatter->dump_string("UploadId", upload->get_upload_id());
       const ACLOwner& owner = upload->get_owner();
-      dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator");
-      dump_owner(s, owner.get_id(), owner.get_display_name()); // Owner
+      dump_owner(s, owner.id, owner.display_name, "Initiator");
+      dump_owner(s, owner.id, owner.display_name); // Owner
       s->formatter->dump_string("StorageClass", "STANDARD");
       dump_time(s, "Initiated", upload->get_mtime());
       s->formatter->close_section();
@@ -5098,8 +5098,8 @@ int RGW_Auth_S3::authorize(const DoutPrefixProvider *dpp,
   const auto ret = rgw::auth::Strategy::apply(dpp, auth_registry.get_s3_main(), s, y);
   if (ret == 0) {
     /* Populate the owner info. */
-    s->owner.set_id(s->user->get_id());
-    s->owner.set_name(s->user->get_display_name());
+    s->owner.id = s->user->get_id();
+    s->owner.display_name = s->user->get_display_name();
   }
   return ret;
 }
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 49155fce409d..308404c87b88 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -354,7 +354,7 @@ namespace rgw::sal {
     Attrs attrs = get_attrs();
     attrs[RGW_ATTR_ACL] = aclbl;
 
-    ret = store->getDB()->update_bucket(dpp, "attrs", info, false, &(acl.get_owner().get_id()), &attrs, nullptr, nullptr);
+    ret = store->getDB()->update_bucket(dpp, "attrs", info, false, &acl.get_owner().id, &attrs, nullptr, nullptr);
 
     return ret;
   }
@@ -701,7 +701,7 @@ namespace rgw::sal {
 
   int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y)
   {
-    parent_op.params.bucket_owner = params.bucket_owner.get_id();
+    parent_op.params.bucket_owner = params.bucket_owner.id;
     parent_op.params.versioning_status = params.versioning_status;
     parent_op.params.obj_owner = params.obj_owner;
     parent_op.params.olh_epoch = params.olh_epoch;
@@ -793,7 +793,7 @@ namespace rgw::sal {
     int ret;
 
     std::unique_ptr<rgw::sal::Object::DeleteOp> del_op = meta_obj->get_delete_op();
-    del_op->params.bucket_owner = bucket->get_info().owner;
+    del_op->params.bucket_owner.id = bucket->get_info().owner;
     del_op->params.versioning_status = 0;
 
     // Since the data objects are associated with meta obj till
@@ -833,7 +833,7 @@ namespace rgw::sal {
     DB::Object::Write obj_op(&op_target);
 
     /* Create meta object */
-    obj_op.meta.owner = owner.get_id();
+    obj_op.meta.owner = owner.id;
     obj_op.meta.category = RGWObjCategory::MultiMeta;
     obj_op.meta.flags = PUT_OBJ_CREATE_EXCL;
     obj_op.meta.mtime = &mtime;
@@ -1012,7 +1012,7 @@ namespace rgw::sal {
     DB::Object::Write obj_op(&op_target);
     ret = obj_op.prepare(dpp);
 
-    obj_op.meta.owner = owner.get_id();
+    obj_op.meta.owner = owner.id;
     obj_op.meta.flags = PUT_OBJ_CREATE;
     obj_op.meta.category = RGWObjCategory::Main;
     obj_op.meta.modify_tail = true;
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 078d186bab87..09f708309e50 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -662,9 +662,10 @@ TEST(TestRGWLua, Acl)
   )";
 
   DEFINE_REQ_STATE;
-  ACLOwner owner;
-  owner.set_id(rgw_user("jack", "black"));
-  owner.set_name("jack black");
+  const ACLOwner owner{
+    .id = rgw_user("jack", "black"),
+    .display_name = "jack black"
+  };
   s.user_acl.reset(new RGWAccessControlPolicy(g_cct));
   s.user_acl->set_owner(owner);
   ACLGrant grant1, grant2, grant3, grant4, grant5, grant6_1, grant6_2;
@@ -730,17 +731,17 @@ TEST(TestRGWLua, UseFunction)
 	)";
 
   DEFINE_REQ_STATE;
-  s.owner.set_name("user two");
-  s.owner.set_id(rgw_user("tenant2", "user2"));
+  s.owner.display_name = "user two";
+  s.owner.id = rgw_user("tenant2", "user2");
   s.user_acl.reset(new RGWAccessControlPolicy());
-  s.user_acl->get_owner().set_name("user three");
-  s.user_acl->get_owner().set_id(rgw_user("tenant3", "user3"));
+  s.user_acl->get_owner().display_name = "user three";
+  s.user_acl->get_owner().id = rgw_user("tenant3", "user3");
   s.bucket_acl.reset(new RGWAccessControlPolicy());
-  s.bucket_acl->get_owner().set_name("user four");
-  s.bucket_acl->get_owner().set_id(rgw_user("tenant4", "user4"));
+  s.bucket_acl->get_owner().display_name = "user four";
+  s.bucket_acl->get_owner().id = rgw_user("tenant4", "user4");
   s.object_acl.reset(new RGWAccessControlPolicy());
-  s.object_acl->get_owner().set_name("user five");
-  s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5"));
+  s.object_acl->get_owner().display_name = "user five";
+  s.object_acl->get_owner().id = rgw_user("tenant5", "user5");
 
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);

From 323114e7330648f0d96b6c215fa42f0a690a225d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 20:17:54 -0500
Subject: [PATCH 0794/2492] rgw/acl: remove unused virtual compare_group_name()

that was the only virtual function, so remove virtual destructors too

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl.h       |  4 +---
 src/rgw/rgw_acl_s3.cc   | 15 ---------------
 src/rgw/rgw_acl_s3.h    |  1 -
 src/rgw/rgw_acl_swift.h |  2 --
 4 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index f2dacf285432..ff9de6c81ffb 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -283,7 +283,7 @@ class RGWAccessControlList
   ACLGrantMap& get_grant_map() { return grant_map; }
   const ACLGrantMap& get_grant_map() const { return grant_map; }
 
-  void create_default(const rgw_user& id, std::string name) {
+  void create_default(const rgw_user& id, const std::string& name) {
     acl_user_map.clear();
     acl_group_map.clear();
     referer_list.clear();
@@ -336,7 +336,6 @@ class RGWAccessControlPolicy
 public:
   explicit RGWAccessControlPolicy(CephContext *_cct) : cct(_cct), acl(_cct) {}
   RGWAccessControlPolicy() : cct(NULL), acl(NULL) {}
-  virtual ~RGWAccessControlPolicy() {}
 
   void set_ctx(CephContext *ctx) {
     cct = ctx;
@@ -391,7 +390,6 @@ class RGWAccessControlPolicy
     return acl;
   }
 
-  virtual bool compare_group_name(std::string& id, ACLGroupTypeEnum group) { return false; }
   bool is_public(const DoutPrefixProvider *dpp) const;
 
   friend bool operator==(const RGWAccessControlPolicy& lhs, const RGWAccessControlPolicy& rhs);
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 46a427572be0..4700f0996514 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -589,21 +589,6 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
   return 0; 
 }
 
-bool RGWAccessControlPolicy_S3::compare_group_name(string& id, ACLGroupTypeEnum group)
-{
-  switch (group) {
-  case ACL_GROUP_ALL_USERS:
-    return (id.compare(RGW_USER_ANON_ID) == 0);
-  case ACL_GROUP_AUTHENTICATED_USERS:
-    return (id.compare(rgw_uri_auth_users) == 0);
-  default:
-    return id.empty();
-  }
-
-  // shouldn't get here
-  return false;
-}
-
 XMLObj *RGWACLXMLParser_S3::alloc_obj(const char *el)
 {
   XMLObj * obj = NULL;
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index c58307600d44..96480ff8bdd8 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -85,7 +85,6 @@ class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
   void to_xml(std::ostream& out);
   int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
 	      RGWAccessControlPolicy& dest, std::string &err_msg);
-  bool compare_group_name(std::string& id, ACLGroupTypeEnum group) override;
 
   virtual int create_canned(ACLOwner& _owner, ACLOwner& bucket_owner, const std::string& canned_acl) {
     RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 4cb1e4b8f8f5..fee32ae06abf 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -24,7 +24,6 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
   explicit RGWAccessControlPolicy_SWIFT(CephContext* const cct)
     : RGWAccessControlPolicy(cct) {
   }
-  ~RGWAccessControlPolicy_SWIFT() override = default;
 
   int create(const DoutPrefixProvider *dpp,
 	     rgw::sal::Driver* driver,
@@ -43,7 +42,6 @@ class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
   explicit RGWAccessControlPolicy_SWIFTAcct(CephContext * const cct)
     : RGWAccessControlPolicy(cct) {
   }
-  ~RGWAccessControlPolicy_SWIFTAcct() override {}
 
   void add_grants(const DoutPrefixProvider *dpp,
 		  rgw::sal::Driver* driver,

From 09c94ee45d5ddde6c609c2f3d6765cb3c797e29e Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 21:01:27 -0500
Subject: [PATCH 0795/2492] rgw: remove unused policy_bl_to_stream()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc | 11 -----------
 src/rgw/driver/rados/rgw_bucket.h  |  1 -
 2 files changed, 12 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 5ac049af3499..99d602bf0dd4 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -882,17 +882,6 @@ int RGWBucket::sync(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *d
 }
 
 
-int RGWBucket::policy_bl_to_stream(bufferlist& bl, ostream& o)
-{
-  RGWAccessControlPolicy_S3 policy(g_ceph_context);
-  int ret = decode_bl(bl, policy);
-  if (ret < 0) {
-    ldout(driver->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl;
-  }
-  policy.to_xml(o);
-  return 0;
-}
-
 int rgw_object_get_attr(const DoutPrefixProvider *dpp,
 			rgw::sal::Driver* driver, rgw::sal::Object* obj,
 			const char* attr_name, bufferlist& out_bl, optional_yield y)
diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h
index 706683e6bd10..5af10b524de6 100644
--- a/src/rgw/driver/rados/rgw_bucket.h
+++ b/src/rgw/driver/rados/rgw_bucket.h
@@ -355,7 +355,6 @@ class RGWBucket {
   int set_quota(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, optional_yield y, std::string *err_msg = NULL);
 
   int remove_object(const DoutPrefixProvider *dpp, RGWBucketAdminOpState& op_state, optional_yield y, std::string *err_msg = NULL);
-  int policy_bl_to_stream(bufferlist& bl, std::ostream& o);
   int get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolicy& policy, optional_yield y, const DoutPrefixProvider *dpp);
   int sync(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, optional_yield y, std::string *err_msg = NULL);
 

From bd855540a56af86383cb630063f5f96fa4cd0e24 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 15 Nov 2023 21:07:59 -0500
Subject: [PATCH 0796/2492] rgw/acl: remove CephContext members

remove the CephContext member variables and pass in
DoutPrefixProvider for logging where it's necessary

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc          |  6 +--
 src/rgw/driver/rados/rgw_data_sync.cc       |  2 -
 src/rgw/driver/rados/rgw_lc_tier.cc         |  1 -
 src/rgw/driver/rados/rgw_rados.cc           |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc       |  2 +-
 src/rgw/driver/rados/rgw_sync_module_aws.cc |  1 -
 src/rgw/driver/rados/rgw_tools.cc           |  2 +-
 src/rgw/rgw_acl.cc                          | 12 +++---
 src/rgw/rgw_acl.h                           | 19 ---------
 src/rgw/rgw_acl_s3.cc                       | 16 ++++----
 src/rgw/rgw_acl_s3.h                        | 12 ++----
 src/rgw/rgw_acl_swift.cc                    |  5 +--
 src/rgw/rgw_acl_swift.h                     |  8 ----
 src/rgw/rgw_file_int.h                      |  8 ++--
 src/rgw/rgw_op.cc                           | 43 ++++++++++-----------
 src/rgw/rgw_op.h                            | 36 -----------------
 src/rgw/rgw_rest_s3.cc                      | 14 +++----
 src/rgw/rgw_rest_swift.cc                   |  4 +-
 18 files changed, 58 insertions(+), 135 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 99d602bf0dd4..002485c672ff 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -950,7 +950,7 @@ int RGWBucketAdminOp::get_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState
 int RGWBucketAdminOp::get_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState& op_state,
                   RGWFormatterFlusher& flusher, const DoutPrefixProvider *dpp, optional_yield y)
 {
-  RGWAccessControlPolicy policy(driver->ctx());
+  RGWAccessControlPolicy policy;
 
   int ret = get_policy(driver, op_state, policy, dpp, y);
   if (ret < 0)
@@ -972,13 +972,13 @@ int RGWBucketAdminOp::get_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState
 int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState& op_state,
                   ostream& os, const DoutPrefixProvider *dpp, optional_yield y)
 {
-  RGWAccessControlPolicy_S3 policy(driver->ctx());
+  RGWAccessControlPolicy_S3 policy;
 
   int ret = get_policy(driver, op_state, policy, dpp, y);
   if (ret < 0)
     return ret;
 
-  policy.to_xml(os);
+  policy.to_xml(dpp, os);
 
   return 0;
 }
diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index b5effb979c9b..c754e6064521 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -2675,8 +2675,6 @@ class RGWUserPermHandler {
   static int policy_from_attrs(CephContext *cct,
                                const map<string, bufferlist>& attrs,
                                RGWAccessControlPolicy *acl) {
-    acl->set_ctx(cct);
-
     auto aiter = attrs.find(RGW_ATTR_ACL);
     if (aiter == attrs.end()) {
       return -ENOENT;
diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc
index 2dd8a6625fcc..67df02445957 100644
--- a/src/rgw/driver/rados/rgw_lc_tier.cc
+++ b/src/rgw/driver/rados/rgw_lc_tier.cc
@@ -464,7 +464,6 @@ int RGWLCStreamRead::init_rest_obj() {
    */
   init_headers(attrs, rest_obj.attrs);
 
-  rest_obj.acls.set_ctx(cct);
   const auto aiter = attrs.find(RGW_ATTR_ACL);
   if (aiter != attrs.end()) {
     bufferlist& bl = aiter->second;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index e85552bc6605..9ef922701550 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1731,7 +1731,7 @@ int RGWRados::decode_policy(const DoutPrefixProvider *dpp,
 			    ACLOwner *owner)
 {
   auto i = bl.cbegin();
-  RGWAccessControlPolicy policy(cct);
+  RGWAccessControlPolicy policy;
   try {
     policy.decode_owner(i);
   } catch (buffer::error& err) {
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 4e71595736b4..94c6001983ce 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1649,7 +1649,7 @@ int RadosObject::chown(User& new_user, const DoutPrefixProvider* dpp, optional_y
   }
 
   bufferlist& bl = aiter->second;
-  RGWAccessControlPolicy policy(store->ctx());
+  RGWAccessControlPolicy policy;
   ACLOwner owner;
   auto bliter = bl.cbegin();
   try {
diff --git a/src/rgw/driver/rados/rgw_sync_module_aws.cc b/src/rgw/driver/rados/rgw_sync_module_aws.cc
index 46b99968a506..3c269a749498 100644
--- a/src/rgw/driver/rados/rgw_sync_module_aws.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_aws.cc
@@ -705,7 +705,6 @@ static int do_decode_rest_obj(const DoutPrefixProvider *dpp, CephContext *cct, m
     }
   }
 
-  info->acls.set_ctx(cct);
   auto aiter = attrs.find(RGW_ATTR_ACL);
   if (aiter != attrs.end()) {
     bufferlist& bl = aiter->second;
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index a9454c502dfb..8219849af468 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -412,7 +412,7 @@ int RGWDataAccess::Object::put(bufferlist& data,
   }
 
   if (!aclbl) {
-    RGWAccessControlPolicy_S3 policy(cct);
+    RGWAccessControlPolicy_S3 policy;
 
     policy.create_canned(bucket->policy.get_owner(), bucket->policy.get_owner(), string()); /* default private policy */
 
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index 7177a5f822ef..18824f86fe94 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -90,9 +90,7 @@ void RGWAccessControlList::_add_grant(ACLGrant *grant)
   default:
     {
       rgw_user id;
-      if (!grant->get_id(id)) {
-        ldout(cct, 0) << "ERROR: grant->get_id() failed" << dendl;
-      }
+      grant->get_id(id);
       acl_user_map[id.to_str()] |= perm.get_permissions();
     }
   }
@@ -323,7 +321,7 @@ void ACLGranteeType::generate_test_instances(list<ACLGranteeType*>& o)
 
 void RGWAccessControlList::generate_test_instances(list<RGWAccessControlList*>& o)
 {
-  RGWAccessControlList *acl = new RGWAccessControlList(NULL);
+  RGWAccessControlList *acl = new RGWAccessControlList;
 
   list<ACLGrant *> glist;
   list<ACLGrant *>::iterator iter;
@@ -336,7 +334,7 @@ void RGWAccessControlList::generate_test_instances(list<RGWAccessControlList*>&
     delete grant;
   }
   o.push_back(acl);
-  o.push_back(new RGWAccessControlList(NULL));
+  o.push_back(new RGWAccessControlList);
 }
 
 void ACLOwner::generate_test_instances(list<ACLOwner*>& o)
@@ -356,7 +354,7 @@ void RGWAccessControlPolicy::generate_test_instances(list<RGWAccessControlPolicy
     RGWAccessControlList::generate_test_instances(acl_list);
     iter = acl_list.begin();
 
-    RGWAccessControlPolicy *p = new RGWAccessControlPolicy(NULL);
+    RGWAccessControlPolicy *p = new RGWAccessControlPolicy;
     RGWAccessControlList *l = *iter;
     p->acl = *l;
 
@@ -368,7 +366,7 @@ void RGWAccessControlPolicy::generate_test_instances(list<RGWAccessControlPolicy
     delete l;
   }
 
-  o.push_back(new RGWAccessControlPolicy(NULL));
+  o.push_back(new RGWAccessControlPolicy);
 }
 
 void RGWAccessControlList::dump(Formatter *f) const
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index ff9de6c81ffb..13141279e8df 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -219,7 +219,6 @@ using ACLGrantMap = std::multimap<std::string, ACLGrant>;
 class RGWAccessControlList
 {
 protected:
-  CephContext *cct;
   /* FIXME: in the feature we should consider switching to uint32_t also
    * in data structures. */
   std::map<std::string, int> acl_user_map;
@@ -228,15 +227,6 @@ class RGWAccessControlList
   ACLGrantMap grant_map;
   void _add_grant(ACLGrant *grant);
 public:
-  explicit RGWAccessControlList(CephContext *_cct) : cct(_cct) {}
-  RGWAccessControlList() : cct(NULL) {}
-
-  void set_ctx(CephContext *ctx) {
-    cct = ctx;
-  }
-
-  virtual ~RGWAccessControlList() {}
-
   uint32_t get_perm(const DoutPrefixProvider* dpp,
                     const rgw::auth::Identity& auth_identity,
                     uint32_t perm_mask);
@@ -329,19 +319,10 @@ WRITE_CLASS_ENCODER(ACLOwner)
 class RGWAccessControlPolicy
 {
 protected:
-  CephContext *cct;
   RGWAccessControlList acl;
   ACLOwner owner;
 
 public:
-  explicit RGWAccessControlPolicy(CephContext *_cct) : cct(_cct), acl(_cct) {}
-  RGWAccessControlPolicy() : cct(NULL), acl(NULL) {}
-
-  void set_ctx(CephContext *ctx) {
-    cct = ctx;
-    acl.set_ctx(ctx);
-  }
-
   uint32_t get_perm(const DoutPrefixProvider* dpp,
                     const rgw::auth::Identity& auth_identity,
                     uint32_t perm_mask,
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 4700f0996514..2641c19ec178 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -210,7 +210,7 @@ bool ACLGrant_S3::xml_end(const char *el) {
   return true;
 }
 
-void ACLGrant_S3::to_xml(CephContext *cct, ostream& out) {
+void ACLGrant_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
   ACLPermission_S3& perm = static_cast<ACLPermission_S3 &>(permission);
 
   /* only show s3 compatible permissions */
@@ -233,7 +233,7 @@ void ACLGrant_S3::to_xml(CephContext *cct, ostream& out) {
     break;
   case ACL_TYPE_GROUP:
     if (!group_to_uri(group, uri)) {
-      ldout(cct, 0) << "ERROR: group_to_uri failed with group=" << (int)group << dendl;
+      ldpp_dout(dpp, 0) << "ERROR: group_to_uri failed with group=" << (int)group << dendl;
       break;
     }
     out << "<URI>" << uri << "</URI>";
@@ -270,12 +270,12 @@ bool RGWAccessControlList_S3::xml_end(const char *el) {
   return true;
 }
 
-void  RGWAccessControlList_S3::to_xml(ostream& out) {
+void RGWAccessControlList_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
   multimap<string, ACLGrant>::iterator iter;
   out << "<AccessControlList>";
   for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
     ACLGrant_S3& grant = static_cast<ACLGrant_S3 &>(iter->second);
-    grant.to_xml(cct, out);
+    grant.to_xml(dpp, out);
   }
   out << "</AccessControlList>";
 }
@@ -438,12 +438,12 @@ bool RGWAccessControlPolicy_S3::xml_end(const char *el) {
   return true;
 }
 
-void  RGWAccessControlPolicy_S3::to_xml(ostream& out) {
+void RGWAccessControlPolicy_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
   out << "<AccessControlPolicy xmlns=\"" << XMLNS_AWS_S3 << "\">";
   ACLOwner_S3& _owner = static_cast<ACLOwner_S3 &>(owner);
   RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
   _owner.to_xml(out);
-  _acl.to_xml(out);
+  _acl.to_xml(dpp, out);
   out << "</AccessControlPolicy>";
 }
 
@@ -593,11 +593,11 @@ XMLObj *RGWACLXMLParser_S3::alloc_obj(const char *el)
 {
   XMLObj * obj = NULL;
   if (strcmp(el, "AccessControlPolicy") == 0) {
-    obj = new RGWAccessControlPolicy_S3(cct);
+    obj = new RGWAccessControlPolicy_S3();
   } else if (strcmp(el, "Owner") == 0) {
     obj = new ACLOwner_S3();
   } else if (strcmp(el, "AccessControlList") == 0) {
-    obj = new RGWAccessControlList_S3(cct);
+    obj = new RGWAccessControlList_S3();
   } else if (strcmp(el, "ID") == 0) {
     obj = new ACLID_S3();
   } else if (strcmp(el, "DisplayName") == 0) {
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index 96480ff8bdd8..e024f5c42ffb 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -41,7 +41,7 @@ class ACLGrant_S3 : public ACLGrant, public XMLObj
   ACLGrant_S3() {}
   virtual ~ACLGrant_S3() override {}
 
-  void to_xml(CephContext *cct, std::ostream& out);
+  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
   bool xml_end(const char *el) override;
   bool xml_start(const char *el, const char **attr);
 
@@ -52,11 +52,8 @@ class ACLGrant_S3 : public ACLGrant, public XMLObj
 class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
 {
 public:
-  explicit RGWAccessControlList_S3(CephContext *_cct) : RGWAccessControlList(_cct) {}
-  virtual ~RGWAccessControlList_S3() override {}
-
   bool xml_end(const char *el) override;
-  void to_xml(std::ostream& out);
+  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
 
   int create_canned(ACLOwner& owner, ACLOwner& bucket_owner, const std::string& canned_acl);
   int create_from_grants(std::list<ACLGrant>& grants);
@@ -77,12 +74,9 @@ class RGWEnv;
 class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
 {
 public:
-  explicit RGWAccessControlPolicy_S3(CephContext *_cct) : RGWAccessControlPolicy(_cct) {}
-  virtual ~RGWAccessControlPolicy_S3() override {}
-
   bool xml_end(const char *el) override;
 
-  void to_xml(std::ostream& out);
+  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
   int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
 	      RGWAccessControlPolicy& dest, std::string &err_msg);
 
diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 060cfabed83e..e829788c6d5a 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -115,7 +115,6 @@ static boost::optional<ACLGrant> referrer_to_grant(std::string url_spec,
 }
 
 static ACLGrant user_to_grant(const DoutPrefixProvider *dpp,
-			      CephContext* const cct,
                               rgw::sal::Driver* driver,
                               const std::string& uid,
                               const uint32_t perm)
@@ -150,7 +149,7 @@ int RGWAccessControlPolicy_SWIFT::add_grants(const DoutPrefixProvider *dpp,
     const size_t pos = uid.find(':');
     if (std::string::npos == pos) {
       /* No, it don't have -- we've got just a regular user identifier. */
-      grant = user_to_grant(dpp, cct, driver, uid, perm);
+      grant = user_to_grant(dpp, driver, uid, perm);
     } else {
       /* Yes, *potentially* an HTTP referral. */
       auto designator = uid.substr(0, pos);
@@ -161,7 +160,7 @@ int RGWAccessControlPolicy_SWIFT::add_grants(const DoutPrefixProvider *dpp,
       boost::algorithm::trim(designatee);
 
       if (! boost::algorithm::starts_with(designator, ".")) {
-        grant = user_to_grant(dpp, cct, driver, uid, perm);
+        grant = user_to_grant(dpp, driver, uid, perm);
       } else if ((perm & SWIFT_PERM_WRITE) == 0 && is_referrer(designator)) {
         /* HTTP referrer-based ACLs aren't acceptable for writes. */
         grant = referrer_to_grant(designatee, perm);
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index fee32ae06abf..5ef83ab26a7d 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -21,10 +21,6 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
                  uint32_t perm);
 
 public:
-  explicit RGWAccessControlPolicy_SWIFT(CephContext* const cct)
-    : RGWAccessControlPolicy(cct) {
-  }
-
   int create(const DoutPrefixProvider *dpp,
 	     rgw::sal::Driver* driver,
              const rgw_user& id,
@@ -39,10 +35,6 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
 {
 public:
-  explicit RGWAccessControlPolicy_SWIFTAcct(CephContext * const cct)
-    : RGWAccessControlPolicy(cct) {
-  }
-
   void add_grants(const DoutPrefixProvider *dpp,
 		  rgw::sal::Driver* driver,
                   const std::vector<std::string>& uids,
diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h
index 91c858e5b3bd..4e3b269ffd0a 100644
--- a/src/rgw/rgw_file_int.h
+++ b/src/rgw/rgw_file_int.h
@@ -1914,7 +1914,7 @@ class RGWCreateBucketRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy(state->cct);
+    RGWAccessControlPolicy_S3 s3policy;
     /* we don't have (any) headers, so just create canned ACLs */
     int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
     policy = s3policy;
@@ -2030,7 +2030,7 @@ class RGWPutObjRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy(state->cct);
+    RGWAccessControlPolicy_S3 s3policy;
     /* we don't have (any) headers, so just create canned ACLs */
     int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
     policy = s3policy;
@@ -2534,7 +2534,7 @@ class RGWWriteRequest : public RGWLibContinuedReq,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy(state->cct);
+    RGWAccessControlPolicy_S3 s3policy;
     /* we don't have (any) headers, so just create canned ACLs */
     int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
     policy = s3policy;
@@ -2641,7 +2641,7 @@ class RGWCopyObjRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* s = get_state();
-    RGWAccessControlPolicy_S3 s3policy(s->cct);
+    RGWAccessControlPolicy_S3 s3policy;
     /* we don't have (any) headers, so just create canned ACLs */
     int ret = s3policy.create_canned(s->owner, s->bucket_owner, s->canned_acl);
     dest_policy = s3policy;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 5574162aab04..39e12a5ab096 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -231,7 +231,7 @@ static int decode_policy(const DoutPrefixProvider *dpp,
   if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(dpp, 15) << __func__ << " Read AccessControlPolicy";
     RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
-    s3policy->to_xml(*_dout);
+    s3policy->to_xml(dpp, *_dout);
     *_dout << dendl;
   }
   return 0;
@@ -450,7 +450,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
   if (ret == -ENOENT) {
     /* object does not exist checking the bucket's ACL to make sure
        that we send a proper error code */
-    RGWAccessControlPolicy bucket_policy(s->cct);
+    RGWAccessControlPolicy bucket_policy;
     ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info.owner,
                                              bucket_attrs, &bucket_policy, y);
     if (ret < 0) {
@@ -513,17 +513,17 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   }
 
   if(s->dialect.compare("s3") == 0) {
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_S3>(s->cct);
+    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_S3>();
   } else if(s->dialect.compare("swift")  == 0) {
     /* We aren't allocating the account policy for those operations using
      * the Swift's infrastructure that don't really need req_state::user.
      * Typical example here is the implementation of /info. */
     if (!s->user->get_id().empty()) {
-      s->user_acl = std::make_unique<RGWAccessControlPolicy_SWIFTAcct>(s->cct);
+      s->user_acl = std::make_unique<RGWAccessControlPolicy_SWIFTAcct>();
     }
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_SWIFT>(s->cct);
+    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_SWIFT>();
   } else {
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>(s->cct);
+    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
   }
 
   const RGWZoneGroup& zonegroup = s->penv.site->get_zonegroup();
@@ -696,7 +696,7 @@ int rgw_build_object_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
     if (!s->bucket_exists) {
       return -ERR_NO_SUCH_BUCKET;
     }
-    s->object_acl = std::make_unique<RGWAccessControlPolicy>(s->cct);
+    s->object_acl = std::make_unique<RGWAccessControlPolicy>();
 
     s->object->set_atomic();
     if (prefetch_data) {
@@ -1642,7 +1642,7 @@ int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map
   if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
     ldpp_dout(dpp, 15) << __func__ << " Read AccessControlPolicy";
-    s3policy->to_xml(*_dout);
+    s3policy->to_xml(dpp, *_dout);
     *_dout << dendl;
   }
   return 0;
@@ -1667,7 +1667,7 @@ int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
 
   std::unique_ptr<rgw::sal::Object> part = bucket->get_object(ent.key);
 
-  RGWAccessControlPolicy obj_policy(s->cct);
+  RGWAccessControlPolicy obj_policy;
 
   ldpp_dout(this, 20) << "reading obj=" << part << " ofs=" << cur_ofs
       << " end=" << cur_end << dendl;
@@ -1962,7 +1962,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
   const std::string bucket_name = url_decode(prefix_view.substr(0, pos));
   const std::string obj_prefix = url_decode(prefix_view.substr(pos + 1));
 
-  RGWAccessControlPolicy _bucket_acl(s->cct);
+  RGWAccessControlPolicy _bucket_acl;
   RGWAccessControlPolicy *bucket_acl;
   boost::optional<Policy> _bucket_policy;
   boost::optional<Policy>* bucket_policy;
@@ -2100,8 +2100,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
         bucket_policy = piter->second.second.get_ptr();
 	bucket = buckets[bucket_name].get();
       } else {
-	allocated_acls.push_back(RGWAccessControlPolicy(s->cct));
-	RGWAccessControlPolicy& _bucket_acl = allocated_acls.back();
+	RGWAccessControlPolicy& _bucket_acl = allocated_acls.emplace_back();
 
 	std::unique_ptr<rgw::sal::Bucket> tmp_bucket;
 	int r = driver->load_bucket(this, rgw_bucket(s->user->get_tenant(),
@@ -3516,7 +3515,7 @@ void RGWCreateBucket::execute(optional_yield y)
     }
 
     // don't allow changes to the acl policy
-    RGWAccessControlPolicy old_policy(get_cct());
+    RGWAccessControlPolicy old_policy;
     int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
                                                s->bucket->get_attrs(),
                                                &old_policy, y);
@@ -3843,7 +3842,7 @@ int RGWPutObj::verify_permission(optional_yield y)
 {
   if (! copy_source.empty()) {
 
-    RGWAccessControlPolicy cs_acl(s->cct);
+    RGWAccessControlPolicy cs_acl;
     boost::optional<Policy> policy;
     map<string, bufferlist> cs_attrs;
     auto cs_bucket = driver->get_bucket(copy_source_bucket_info);
@@ -5484,7 +5483,7 @@ int RGWCopyObj::init_processing(optional_yield y)
 
 int RGWCopyObj::verify_permission(optional_yield y)
 {
-  RGWAccessControlPolicy src_acl(s->cct);
+  RGWAccessControlPolicy src_acl;
   boost::optional<Policy> src_policy;
 
   /* get buckets info (source and dest) */
@@ -5583,7 +5582,7 @@ int RGWCopyObj::verify_permission(optional_yield y)
     }
   }
 
-  RGWAccessControlPolicy dest_bucket_policy(s->cct);
+  RGWAccessControlPolicy dest_bucket_policy;
 
   s->object->set_atomic();
 
@@ -5898,7 +5897,7 @@ void RGWGetACLs::execute(optional_yield y)
     (!rgw::sal::Object::empty(s->object.get()) ? s->object_acl.get() : s->bucket_acl.get());
   RGWAccessControlPolicy_S3* const s3policy = \
     static_cast<RGWAccessControlPolicy_S3*>(acl);
-  s3policy->to_xml(ss);
+  s3policy->to_xml(this, ss);
   acls = ss.str();
 }
 
@@ -5993,7 +5992,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   RGWAccessControlPolicy_S3 *policy = NULL;
   RGWACLXMLParser_S3 parser(s->cct);
-  RGWAccessControlPolicy_S3 new_policy(s->cct);
+  RGWAccessControlPolicy_S3 new_policy;
   stringstream ss;
 
   op_ret = 0; /* XXX redundant? */
@@ -6085,7 +6084,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "Old AccessControlPolicy";
-    policy->to_xml(*_dout);
+    policy->to_xml(this, *_dout);
     *_dout << dendl;
   }
 
@@ -6095,7 +6094,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "New AccessControlPolicy:";
-    new_policy.to_xml(*_dout);
+    new_policy.to_xml(this, *_dout);
     *_dout << dendl;
   }
 
@@ -7408,7 +7407,7 @@ bool RGWBulkDelete::Deleter::verify_permission(RGWBucketInfo& binfo,
                                                ACLOwner& bucket_owner /* out */,
 					       optional_yield y)
 {
-  RGWAccessControlPolicy bacl(driver->ctx());
+  RGWAccessControlPolicy bacl;
   int ret = read_bucket_policy(dpp, driver, s, binfo, battrs, &bacl, binfo.bucket, y);
   if (ret < 0) {
     return false;
@@ -7747,7 +7746,7 @@ bool RGWBulkUploadOp::handle_file_verify_permission(RGWBucketInfo& binfo,
                                                     ACLOwner& bucket_owner /* out */,
 						    optional_yield y)
 {
-  RGWAccessControlPolicy bacl(driver->ctx());
+  RGWAccessControlPolicy bacl;
   op_ret = read_bucket_policy(this, driver, s, binfo, battrs, &bacl, binfo.bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "cannot read_policy() for bucket" << dendl;
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index be7232ae392d..70266cbfc667 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1098,7 +1098,6 @@ class RGWCreateBucket : public RGWOp {
   void execute(optional_yield y) override;
   void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
     RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
     relaxed_region_enforcement =
 	s->cct->_conf.get_val<bool>("rgw_relaxed_region_enforcement");
   }
@@ -1249,11 +1248,6 @@ class RGWPutObj : public RGWOp {
     delete obj_legal_hold;
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
-
   virtual int init_processing(optional_yield y) override;
 
   void emplace_attr(std::string&& key, buffer::list&& bl) {
@@ -1328,11 +1322,6 @@ class RGWPostObj : public RGWOp {
     attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
-
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
@@ -1367,10 +1356,6 @@ class RGWPutMetadataAccount : public RGWOp {
       has_policy(false) {
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
   int init_processing(optional_yield y) override;
   int verify_permission(optional_yield y) override;
   void pre_exec() override { }
@@ -1406,11 +1391,6 @@ class RGWPutMetadataBucket : public RGWOp {
     attrs.emplace(std::move(key), std::move(bl)); /* key and bl are r-value refs */
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
-
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
@@ -1433,10 +1413,6 @@ class RGWPutMetadataObject : public RGWOp {
     : dlo_manifest(NULL)
   {}
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
@@ -1560,10 +1536,6 @@ class RGWCopyObj : public RGWOp {
     attrs.emplace(std::move(key), std::move(bl));
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    dest_policy.set_ctx(s->cct);
-  }
   int init_processing(optional_yield y) override;
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
@@ -1844,10 +1816,6 @@ class RGWInitMultipart : public RGWOp {
 public:
   RGWInitMultipart() {}
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy.set_ctx(s->cct);
-  }
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
@@ -1919,10 +1887,6 @@ class RGWListMultipart : public RGWOp {
     truncated = false;
   }
 
-  void init(rgw::sal::Driver* driver, req_state *s, RGWHandler *h) override {
-    RGWOp::init(driver, s, h);
-    policy = RGWAccessControlPolicy(s->cct);
-  }
   int verify_permission(optional_yield y) override;
   void pre_exec() override;
   void execute(optional_yield y) override;
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index dbc75955940f..e4c90bc81e1d 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2447,7 +2447,7 @@ class RGWCreateBucketParser : public RGWXMLParser
 
 int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y)
 {
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
 
   int r;
@@ -2589,7 +2589,7 @@ int RGWPutObj_ObjStore_S3::get_params(optional_yield y)
     return ret;
   }
 
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
   ret = create_s3_policy(s, driver, s3policy, s->owner);
   if (ret < 0)
     return ret;
@@ -3215,7 +3215,7 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y)
   string canned_acl;
   part_str(parts, "acl", &canned_acl);
 
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
   ldpp_dout(this, 20) << "canned_acl=" << canned_acl << dendl;
   if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
     err_msg = "Bad canned ACLs";
@@ -3450,7 +3450,7 @@ void RGWDeleteObj_ObjStore_S3::send_response()
 
 int RGWCopyObj_ObjStore_S3::init_dest_policy()
 {
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
 
   /* build a policy for the target object */
   int r = create_s3_policy(s, driver, s3policy, s->owner);
@@ -3625,7 +3625,7 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::Driver* driver,
 						  req_state *s,
 						  stringstream& ss)
 {
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
 
   // bucket-* canned acls do not apply to bucket
   if (rgw::sal::Object::empty(s->object.get())) {
@@ -3637,7 +3637,7 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::Driver* driver,
   if (r < 0)
     return r;
 
-  s3policy.to_xml(ss);
+  s3policy.to_xml(this, ss);
 
   return 0;
 }
@@ -3972,7 +3972,7 @@ int RGWInitMultipart_ObjStore_S3::get_params(optional_yield y)
     return ret;
   }
 
-  RGWAccessControlPolicy_S3 s3policy(s->cct);
+  RGWAccessControlPolicy_S3 s3policy;
   ret = create_s3_policy(s, driver, s3policy, s->owner);
   if (ret < 0)
     return ret;
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 25ba61841613..a92d1f56f810 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -607,7 +607,7 @@ static int get_swift_container_settings(req_state * const s,
   *has_policy = false;
 
   if (read_list || write_list) {
-    RGWAccessControlPolicy_SWIFT swift_policy(s->cct);
+    RGWAccessControlPolicy_SWIFT swift_policy;
     const auto r = swift_policy.create(s, driver,
                                        s->user->get_id(),
                                        s->user->get_display_name(),
@@ -1067,7 +1067,7 @@ static int get_swift_account_settings(req_state * const s,
 
   const char * const acl_attr = s->info.env->get("HTTP_X_ACCOUNT_ACCESS_CONTROL");
   if (acl_attr) {
-    RGWAccessControlPolicy_SWIFTAcct swift_acct_policy(s->cct);
+    RGWAccessControlPolicy_SWIFTAcct swift_acct_policy;
     const bool r = swift_acct_policy.create(s, driver,
                                      s->user->get_id(),
                                      s->user->get_display_name(),

From 9bf24c08259f1164fe158e6028557bd1e1d9193c Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 10:46:35 -0500
Subject: [PATCH 0797/2492] rgw/acl: add_grant() takes const ref

that also required fixing some const-correctness issues

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc |  2 +-
 src/rgw/rgw_acl.cc                    | 34 ++++++++++++---------------
 src/rgw/rgw_acl.h                     | 20 +++++++---------
 src/rgw/rgw_acl_s3.cc                 | 29 +++++++++++------------
 src/rgw/rgw_acl_swift.cc              | 10 ++++----
 src/rgw/rgw_rest_client.cc            | 19 +++++++--------
 src/rgw/rgw_rest_client.h             |  2 +-
 7 files changed, 53 insertions(+), 63 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 94c6001983ce..5ede8d44fa9b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1670,7 +1670,7 @@ int RadosObject::chown(User& new_user, const DoutPrefixProvider* dpp, optional_y
   //Create a grant and add grant
   ACLGrant grant;
   grant.set_canon(new_user.get_id(), new_user.get_display_name(), RGW_PERM_FULL_CONTROL);
-  acl.add_grant(&grant);
+  acl.add_grant(grant);
 
   //Update the ACL owner to the new user
   owner.id = new_user.get_id();
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index 18824f86fe94..6e3aafe762ad 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -70,38 +70,38 @@ bool operator!=(const RGWAccessControlPolicy& lhs,
   return !(lhs == rhs);
 }
 
-void RGWAccessControlList::_add_grant(ACLGrant *grant)
+void RGWAccessControlList::register_grant(const ACLGrant& grant)
 {
-  ACLPermission& perm = grant->get_permission();
-  ACLGranteeType& type = grant->get_type();
+  ACLPermission perm = grant.get_permission();
+  ACLGranteeType type = grant.get_type();
   switch (type.get_type()) {
   case ACL_TYPE_REFERER:
-    referer_list.emplace_back(grant->get_referer(), perm.get_permissions());
+    referer_list.emplace_back(grant.get_referer(), perm.get_permissions());
 
     /* We're specially handling the Swift's .r:* as the S3 API has a similar
      * concept and thus we can have a small portion of compatibility here. */
-     if (grant->get_referer() == RGW_REFERER_WILDCARD) {
+     if (grant.get_referer() == RGW_REFERER_WILDCARD) {
        acl_group_map[ACL_GROUP_ALL_USERS] |= perm.get_permissions();
      }
     break;
   case ACL_TYPE_GROUP:
-    acl_group_map[grant->get_group()] |= perm.get_permissions();
+    acl_group_map[grant.get_group()] |= perm.get_permissions();
     break;
   default:
     {
       rgw_user id;
-      grant->get_id(id);
+      grant.get_id(id);
       acl_user_map[id.to_str()] |= perm.get_permissions();
     }
   }
 }
 
-void RGWAccessControlList::add_grant(ACLGrant *grant)
+void RGWAccessControlList::add_grant(const ACLGrant& grant)
 {
   rgw_user id;
-  grant->get_id(id); // not that this will return false for groups, but that's ok, we won't search groups
-  grant_map.insert(pair<string, ACLGrant>(id.to_str(), *grant));
-  _add_grant(grant);
+  grant.get_id(id); // note that this will return false for groups, but that's ok, we won't search groups
+  grant_map.emplace(id.to_str(), grant);
+  register_grant(grant);
 }
 
 void RGWAccessControlList::remove_canon_user_grant(rgw_user& user_id)
@@ -323,14 +323,10 @@ void RGWAccessControlList::generate_test_instances(list<RGWAccessControlList*>&
 {
   RGWAccessControlList *acl = new RGWAccessControlList;
 
-  list<ACLGrant *> glist;
-  list<ACLGrant *>::iterator iter;
-
-  ACLGrant::generate_test_instances(glist);
-  for (iter = glist.begin(); iter != glist.end(); ++iter) {
-    ACLGrant *grant = *iter;
-    acl->add_grant(grant);
-
+  list<ACLGrant *> grants;
+  ACLGrant::generate_test_instances(grants);
+  for (ACLGrant* grant : grants) {
+    acl->add_grant(*grant);
     delete grant;
   }
   o.push_back(acl);
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index 13141279e8df..f60295d90ab4 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -60,10 +60,8 @@ class ACLGrant
     }
   }
 
-  ACLGranteeType& get_type() { return type; }
-  const ACLGranteeType& get_type() const { return type; }
-  ACLPermission& get_permission() { return permission; }
-  const ACLPermission& get_permission() const { return permission; }
+  ACLGranteeType get_type() const { return type; }
+  ACLPermission get_permission() const { return permission; }
   ACLGroupTypeEnum get_group() const { return group; }
   const std::string& get_referer() const { return url_spec; }
 
@@ -225,7 +223,8 @@ class RGWAccessControlList
   std::map<uint32_t, int> acl_group_map;
   std::list<ACLReferer> referer_list;
   ACLGrantMap grant_map;
-  void _add_grant(ACLGrant *grant);
+  // register a grant in the correspoding acl_user/group_map
+  void register_grant(const ACLGrant& grant);
 public:
   uint32_t get_perm(const DoutPrefixProvider* dpp,
                     const rgw::auth::Identity& auth_identity,
@@ -253,10 +252,9 @@ class RGWAccessControlList
     if (struct_v >= 2) {
       decode(acl_group_map, bl);
     } else if (!maps_initialized) {
-      ACLGrantMap::iterator iter;
-      for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
-        ACLGrant& grant = iter->second;
-        _add_grant(&grant);
+      // register everything in the grant_map
+      for (const auto& [id, grant] : grant_map) {
+        register_grant(grant);
       }
     }
     if (struct_v >= 4) {
@@ -267,7 +265,7 @@ class RGWAccessControlList
   void dump(Formatter *f) const;
   static void generate_test_instances(std::list<RGWAccessControlList*>& o);
 
-  void add_grant(ACLGrant *grant);
+  void add_grant(const ACLGrant& grant);
   void remove_canon_user_grant(rgw_user& user_id);
 
   ACLGrantMap& get_grant_map() { return grant_map; }
@@ -280,7 +278,7 @@ class RGWAccessControlList
 
     ACLGrant grant;
     grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
-    add_grant(&grant);
+    add_grant(grant);
   }
 
   friend bool operator==(const RGWAccessControlList& lhs, const RGWAccessControlList& rhs);
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 2641c19ec178..52eeb44f2430 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -66,7 +66,7 @@ xml_end(const char *el)
 
 class ACLGranteeType_S3 {
 public:
-  static const char *to_string(ACLGranteeType& type) {
+  static const char *to_string(ACLGranteeType type) {
     switch (type.get_type()) {
     case ACL_TYPE_CANON_USER:
       return "CanonicalUser";
@@ -264,7 +264,7 @@ bool RGWAccessControlList_S3::xml_end(const char *el) {
   XMLObjIter iter = find("Grant");
   ACLGrant_S3 *grant = static_cast<ACLGrant_S3 *>(iter.get_next());
   while (grant) {
-    add_grant(grant);
+    add_grant(*grant);
     grant = static_cast<ACLGrant_S3 *>(iter.get_next());
   }
   return true;
@@ -371,7 +371,7 @@ int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_own
 
   /* owner gets full control */
   owner_grant.set_canon(owner.id, owner.display_name, RGW_PERM_FULL_CONTROL);
-  add_grant(&owner_grant);
+  add_grant(owner_grant);
 
   if (canned_acl.size() == 0 || canned_acl.compare("private") == 0) {
     return 0;
@@ -381,24 +381,24 @@ int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_own
   ACLGrant group_grant;
   if (canned_acl.compare("public-read") == 0) {
     group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
-    add_grant(&group_grant);
+    add_grant(group_grant);
   } else if (canned_acl.compare("public-read-write") == 0) {
     group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
-    add_grant(&group_grant);
+    add_grant(group_grant);
     group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_WRITE);
-    add_grant(&group_grant);
+    add_grant(group_grant);
   } else if (canned_acl.compare("authenticated-read") == 0) {
     group_grant.set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_READ);
-    add_grant(&group_grant);
+    add_grant(group_grant);
   } else if (canned_acl.compare("bucket-owner-read") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_READ);
     if (bid != owner.id) {
-      add_grant(&bucket_owner_grant);
+      add_grant(bucket_owner_grant);
     }
   } else if (canned_acl.compare("bucket-owner-full-control") == 0) {
     bucket_owner_grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
     if (bid != owner.id) {
-      add_grant(&bucket_owner_grant);
+      add_grant(bucket_owner_grant);
     }
   } else {
     return -EINVAL;
@@ -415,9 +415,8 @@ int RGWAccessControlList_S3::create_from_grants(std::list<ACLGrant>& grants)
   acl_user_map.clear();
   grant_map.clear();
 
-  for (std::list<ACLGrant>::iterator it = grants.begin(); it != grants.end(); ++it) {
-    ACLGrant g = *it;
-    add_grant(&g);
+  for (const auto& g : grants) {
+    add_grant(g);
   }
 
   return 0;
@@ -513,7 +512,7 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
   multimap<string, ACLGrant>::iterator iter;
   for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
     ACLGrant& src_grant = iter->second;
-    ACLGranteeType& type = src_grant.get_type();
+    ACLGranteeType type = src_grant.get_type();
     ACLGrant new_grant;
     bool grant_ok = false;
     rgw_user uid;
@@ -557,7 +556,7 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
 	    grant_user = user->get_info();
 	  }
         }
-	ACLPermission& perm = src_grant.get_permission();
+	ACLPermission perm = src_grant.get_permission();
 	new_grant.set_canon(uid, grant_user.display_name, perm.get_permissions());
 	grant_ok = true;
 	rgw_user new_id;
@@ -582,7 +581,7 @@ int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
       break;
     }
     if (grant_ok) {
-      dst_acl.add_grant(&new_grant);
+      dst_acl.add_grant(new_grant);
     }
   }
 
diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index e829788c6d5a..70fa793dd9af 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -168,7 +168,7 @@ int RGWAccessControlPolicy_SWIFT::add_grants(const DoutPrefixProvider *dpp,
     }
 
     if (grant) {
-      acl.add_grant(&*grant);
+      acl.add_grant(*grant);
     } else {
       return -EINVAL;
     }
@@ -257,7 +257,7 @@ void RGWAccessControlPolicy_SWIFT::filter_merge(uint32_t rw_mask,
       }
     }
     if (perm & rw_mask) {
-      acl.add_grant(&grant);
+      acl.add_grant(grant);
     }
   }
 }
@@ -313,7 +313,7 @@ void RGWAccessControlPolicy_SWIFTAcct::add_grants(const DoutPrefixProvider *dpp,
 
     if (uid_is_public(uid)) {
       grant.set_group(ACL_GROUP_ALL_USERS, perm);
-      acl.add_grant(&grant);
+      acl.add_grant(grant);
     } else  {
       std::unique_ptr<rgw::sal::User> user = driver->get_user(rgw_user(uid));
 
@@ -321,10 +321,10 @@ void RGWAccessControlPolicy_SWIFTAcct::add_grants(const DoutPrefixProvider *dpp,
         ldpp_dout(dpp, 10) << "grant user does not exist:" << uid << dendl;
         /* skipping silently */
         grant.set_canon(user->get_id(), std::string(), perm);
-        acl.add_grant(&grant);
+        acl.add_grant(grant);
       } else {
         grant.set_canon(user->get_id(), user->get_display_name(), perm);
-        acl.add_grant(&grant);
+        acl.add_grant(grant);
       }
     }
   }
diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index 1ccb813a1098..71ead6a8b892 100644
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -497,7 +497,7 @@ RGWRESTStreamS3PutObj::~RGWRESTStreamS3PutObj()
   delete out_cb;
 }
 
-static void grants_by_type_add_one_grant(map<int, string>& grants_by_type, int perm, ACLGrant& grant)
+static void grants_by_type_add_one_grant(map<int, string>& grants_by_type, int perm, const ACLGrant& grant)
 {
   string& s = grants_by_type[perm];
 
@@ -505,7 +505,7 @@ static void grants_by_type_add_one_grant(map<int, string>& grants_by_type, int p
     s.append(", ");
 
   string id_type_str;
-  ACLGranteeType& type = grant.get_type();
+  ACLGranteeType type = grant.get_type();
   switch (type.get_type()) {
     case ACL_TYPE_GROUP:
       id_type_str = "uri";
@@ -535,7 +535,7 @@ struct grant_type_to_header grants_headers_def[] = {
   { 0, NULL}
 };
 
-static bool grants_by_type_check_perm(map<int, string>& grants_by_type, int perm, ACLGrant& grant, int check_perm)
+static bool grants_by_type_check_perm(map<int, string>& grants_by_type, int perm, const ACLGrant& grant, int check_perm)
 {
   if ((perm & check_perm) == check_perm) {
     grants_by_type_add_one_grant(grants_by_type, check_perm, grant);
@@ -544,7 +544,7 @@ static bool grants_by_type_check_perm(map<int, string>& grants_by_type, int perm
   return false;
 }
 
-static void grants_by_type_add_perm(map<int, string>& grants_by_type, int perm, ACLGrant& grant)
+static void grants_by_type_add_perm(map<int, string>& grants_by_type, int perm, const ACLGrant& grant)
 {
   struct grant_type_to_header *t;
 
@@ -669,16 +669,13 @@ void RGWRESTGenerateHTTPHeaders::set_http_attrs(const map<string, string>& http_
   }
 }
 
-void RGWRESTGenerateHTTPHeaders::set_policy(RGWAccessControlPolicy& policy)
+void RGWRESTGenerateHTTPHeaders::set_policy(const RGWAccessControlPolicy& policy)
 {
   /* update acl headers */
-  RGWAccessControlList& acl = policy.get_acl();
-  multimap<string, ACLGrant>& grant_map = acl.get_grant_map();
-  multimap<string, ACLGrant>::iterator giter;
+  const RGWAccessControlList& acl = policy.get_acl();
   map<int, string> grants_by_type;
-  for (giter = grant_map.begin(); giter != grant_map.end(); ++giter) {
-    ACLGrant& grant = giter->second;
-    ACLPermission& perm = grant.get_permission();
+  for (const auto& [id, grant] : acl.get_grant_map()) {
+    ACLPermission perm = grant.get_permission();
     grants_by_type_add_perm(grants_by_type, perm.get_permissions(), grant);
   }
   add_grants_headers(grants_by_type, *new_env, new_info->x_meta_map);
diff --git a/src/rgw/rgw_rest_client.h b/src/rgw/rgw_rest_client.h
index 6e27576c6768..923f8cc17839 100644
--- a/src/rgw/rgw_rest_client.h
+++ b/src/rgw/rgw_rest_client.h
@@ -94,7 +94,7 @@ class RGWRESTGenerateHTTPHeaders : public DoutPrefix {
   void set_extra_headers(const std::map<std::string, std::string>& extra_headers);
   int set_obj_attrs(const DoutPrefixProvider *dpp, std::map<std::string, bufferlist>& rgw_attrs);
   void set_http_attrs(const std::map<std::string, std::string>& http_attrs);
-  void set_policy(RGWAccessControlPolicy& policy);
+  void set_policy(const RGWAccessControlPolicy& policy);
   int sign(const DoutPrefixProvider *dpp, RGWAccessKey& key, const bufferlist *opt_content);
 
   const std::string& get_url() { return url; }

From 8fbd1efaf01f8b2bc92e96e7d5689765508fc1cd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 12:44:21 -0500
Subject: [PATCH 0798/2492] rgw/acl/swift: parse acl header with ceph::split()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc | 125 ++++++++++++++-------------------------
 src/rgw/rgw_acl_swift.h  |  11 ++--
 2 files changed, 49 insertions(+), 87 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 70fa793dd9af..2a573da72a51 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -8,6 +8,7 @@
 #include <boost/algorithm/string/predicate.hpp>
 
 #include "common/ceph_json.h"
+#include "common/split.h"
 #include "rgw_common.h"
 #include "rgw_user.h"
 #include "rgw_acl_swift.h"
@@ -26,27 +27,6 @@
 
 using namespace std;
 
-static int parse_list(const char* uid_list,
-                      std::vector<std::string>& uids)           /* out */
-{
-  char *s = strdup(uid_list);
-  if (!s) {
-    return -ENOMEM;
-  }
-
-  char *tokctx;
-  const char *p = strtok_r(s, " ,", &tokctx);
-  while (p) {
-    if (*p) {
-      string acl = p;
-      uids.push_back(acl);
-    }
-    p = strtok_r(NULL, " ,", &tokctx);
-  }
-  free(s);
-  return 0;
-}
-
 static bool is_referrer(const std::string& designator)
 {
   return designator.compare(".r") == 0 ||
@@ -135,45 +115,42 @@ static ACLGrant user_to_grant(const DoutPrefixProvider *dpp,
   return grant;
 }
 
-int RGWAccessControlPolicy_SWIFT::add_grants(const DoutPrefixProvider *dpp,
-					     rgw::sal::Driver* driver,
-                                             const std::vector<std::string>& uids,
-                                             const uint32_t perm)
+int RGWAccessControlPolicy_SWIFT::add_grant(const DoutPrefixProvider *dpp,
+					    rgw::sal::Driver* driver,
+                                            const std::string& uid,
+                                            const uint32_t perm)
 {
-  for (const auto& uid : uids) {
-    boost::optional<ACLGrant> grant;
-    ldpp_dout(dpp, 20) << "trying to add grant for ACL uid=" << uid << dendl;
-
-    /* Let's check whether the item has a separator potentially indicating
-     * a special meaning (like an HTTP referral-based grant). */
-    const size_t pos = uid.find(':');
-    if (std::string::npos == pos) {
-      /* No, it don't have -- we've got just a regular user identifier. */
+  boost::optional<ACLGrant> grant;
+  ldpp_dout(dpp, 20) << "trying to add grant for ACL uid=" << uid << dendl;
+
+  /* Let's check whether the item has a separator potentially indicating
+   * a special meaning (like an HTTP referral-based grant). */
+  const size_t pos = uid.find(':');
+  if (std::string::npos == pos) {
+    /* No, it don't have -- we've got just a regular user identifier. */
+    grant = user_to_grant(dpp, driver, uid, perm);
+  } else {
+    /* Yes, *potentially* an HTTP referral. */
+    auto designator = uid.substr(0, pos);
+    auto designatee = uid.substr(pos + 1);
+
+    /* Swift strips whitespaces at both beginning and end. */
+    boost::algorithm::trim(designator);
+    boost::algorithm::trim(designatee);
+
+    if (! boost::algorithm::starts_with(designator, ".")) {
       grant = user_to_grant(dpp, driver, uid, perm);
-    } else {
-      /* Yes, *potentially* an HTTP referral. */
-      auto designator = uid.substr(0, pos);
-      auto designatee = uid.substr(pos + 1);
-
-      /* Swift strips whitespaces at both beginning and end. */
-      boost::algorithm::trim(designator);
-      boost::algorithm::trim(designatee);
-
-      if (! boost::algorithm::starts_with(designator, ".")) {
-        grant = user_to_grant(dpp, driver, uid, perm);
-      } else if ((perm & SWIFT_PERM_WRITE) == 0 && is_referrer(designator)) {
-        /* HTTP referrer-based ACLs aren't acceptable for writes. */
-        grant = referrer_to_grant(designatee, perm);
-      }
+    } else if ((perm & SWIFT_PERM_WRITE) == 0 && is_referrer(designator)) {
+      /* HTTP referrer-based ACLs aren't acceptable for writes. */
+      grant = referrer_to_grant(designatee, perm);
     }
+  }
 
-    if (grant) {
-      acl.add_grant(*grant);
-    } else {
-      return -EINVAL;
-    }
+  if (!grant) {
+    return -EINVAL;
   }
 
+  acl.add_grant(*grant);
   return 0;
 }
 
@@ -192,36 +169,24 @@ int RGWAccessControlPolicy_SWIFT::create(const DoutPrefixProvider *dpp,
   rw_mask = 0;
 
   if (read_list) {
-    std::vector<std::string> uids;
-    int r = parse_list(read_list, uids);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: parse_list for read returned r="
-                    << r << dendl;
-      return r;
-    }
-
-    r = add_grants(dpp, driver, uids, SWIFT_PERM_READ);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: add_grants for read returned r="
-                    << r << dendl;
-      return r;
+    for (std::string_view uid : ceph::split(read_list, " ,")) {
+      int r = add_grant(dpp, driver, std::string{uid}, SWIFT_PERM_READ);
+      if (r < 0) {
+        ldpp_dout(dpp, 0) << "ERROR: add_grants for read returned r="
+                      << r << dendl;
+        return r;
+      }
     }
     rw_mask |= SWIFT_PERM_READ;
   }
   if (write_list) {
-    std::vector<std::string> uids;
-    int r = parse_list(write_list, uids);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: parse_list for write returned r="
-                    << r << dendl;
-      return r;
-    }
-
-    r = add_grants(dpp, driver, uids, SWIFT_PERM_WRITE);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: add_grants for write returned r="
-                    << r << dendl;
-      return r;
+    for (std::string_view uid : ceph::split(write_list, " ,")) {
+      int r = add_grant(dpp, driver, std::string{uid}, SWIFT_PERM_WRITE);
+      if (r < 0) {
+        ldpp_dout(dpp, 0) << "ERROR: add_grants for write returned r="
+                      << r << dendl;
+        return r;
+      }
     }
     rw_mask |= SWIFT_PERM_WRITE;
   }
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 5ef83ab26a7d..db12f6063558 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -16,9 +16,8 @@ class RGWUserCtl;
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 {
-  int add_grants(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
-                 const std::vector<std::string>& uids,
-                 uint32_t perm);
+  int add_grant(const DoutPrefixProvider* dpp, rgw::sal::Driver* driver,
+                const std::string& uid, uint32_t perm);
 
 public:
   int create(const DoutPrefixProvider *dpp,
@@ -34,11 +33,9 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
 {
+  void add_grants(const DoutPrefixProvider* dpp, rgw::sal::Driver* driver,
+                  const std::vector<std::string>& uids, uint32_t perm);
 public:
-  void add_grants(const DoutPrefixProvider *dpp,
-		  rgw::sal::Driver* driver,
-                  const std::vector<std::string>& uids,
-                  uint32_t perm);
   bool create(const DoutPrefixProvider *dpp,
 	      rgw::sal::Driver* driver,
               const rgw_user& id,

From 064b562c2b6438068f4d30d437f991602775834f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:00:25 -0500
Subject: [PATCH 0799/2492] rgw/acl/swift: parse_grant() as static function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc | 79 ++++++++++++++++++++--------------------
 src/rgw/rgw_acl_swift.h  |  3 --
 2 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 2a573da72a51..74d5e9bbaa73 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -3,6 +3,7 @@
 
 #include <string.h>
 
+#include <optional>
 #include <vector>
 
 #include <boost/algorithm/string/predicate.hpp>
@@ -53,8 +54,8 @@ static bool uid_is_public(const string& uid)
   return is_referrer(sub);
 }
 
-static boost::optional<ACLGrant> referrer_to_grant(std::string url_spec,
-                                                   const uint32_t perm)
+static std::optional<ACLGrant> referrer_to_grant(std::string url_spec,
+                                                 const uint32_t perm)
 {
   /* This function takes url_spec as non-ref std::string because of the trim
    * operation that is essential to preserve compliance with Swift. It can't
@@ -79,7 +80,7 @@ static boost::optional<ACLGrant> referrer_to_grant(std::string url_spec,
       }
 
       if (url_spec.empty() || url_spec == ".") {
-        return boost::none;
+        return std::nullopt;
       }
     } else {
       /* Please be aware we're specially handling the .r:* in _add_grant()
@@ -90,7 +91,7 @@ static boost::optional<ACLGrant> referrer_to_grant(std::string url_spec,
     grant.set_referer(url_spec, is_negative ? 0 : perm);
     return grant;
   } catch (const std::out_of_range&) {
-    return boost::none;
+    return std::nullopt;
   }
 }
 
@@ -115,12 +116,14 @@ static ACLGrant user_to_grant(const DoutPrefixProvider *dpp,
   return grant;
 }
 
-int RGWAccessControlPolicy_SWIFT::add_grant(const DoutPrefixProvider *dpp,
-					    rgw::sal::Driver* driver,
-                                            const std::string& uid,
-                                            const uint32_t perm)
+// parse a container acl grant in 'V1' format
+// https://docs.openstack.org/swift/latest/overview_acl.html#container-acls
+static auto parse_grant(const DoutPrefixProvider* dpp,
+                        rgw::sal::Driver* driver,
+                        const std::string& uid,
+                        const uint32_t perm)
+  -> std::optional<ACLGrant>
 {
-  boost::optional<ACLGrant> grant;
   ldpp_dout(dpp, 20) << "trying to add grant for ACL uid=" << uid << dendl;
 
   /* Let's check whether the item has a separator potentially indicating
@@ -128,30 +131,26 @@ int RGWAccessControlPolicy_SWIFT::add_grant(const DoutPrefixProvider *dpp,
   const size_t pos = uid.find(':');
   if (std::string::npos == pos) {
     /* No, it don't have -- we've got just a regular user identifier. */
-    grant = user_to_grant(dpp, driver, uid, perm);
-  } else {
-    /* Yes, *potentially* an HTTP referral. */
-    auto designator = uid.substr(0, pos);
-    auto designatee = uid.substr(pos + 1);
-
-    /* Swift strips whitespaces at both beginning and end. */
-    boost::algorithm::trim(designator);
-    boost::algorithm::trim(designatee);
-
-    if (! boost::algorithm::starts_with(designator, ".")) {
-      grant = user_to_grant(dpp, driver, uid, perm);
-    } else if ((perm & SWIFT_PERM_WRITE) == 0 && is_referrer(designator)) {
-      /* HTTP referrer-based ACLs aren't acceptable for writes. */
-      grant = referrer_to_grant(designatee, perm);
-    }
+    return user_to_grant(dpp, driver, uid, perm);
   }
 
-  if (!grant) {
-    return -EINVAL;
+  /* Yes, *potentially* an HTTP referral. */
+  auto designator = uid.substr(0, pos);
+  auto designatee = uid.substr(pos + 1);
+
+  /* Swift strips whitespaces at both beginning and end. */
+  boost::algorithm::trim(designator);
+  boost::algorithm::trim(designatee);
+
+  if (! boost::algorithm::starts_with(designator, ".")) {
+    return user_to_grant(dpp, driver, uid, perm);
+  }
+  if ((perm & SWIFT_PERM_WRITE) == 0 && is_referrer(designator)) {
+    /* HTTP referrer-based ACLs aren't acceptable for writes. */
+    return referrer_to_grant(designatee, perm);
   }
 
-  acl.add_grant(*grant);
-  return 0;
+  return std::nullopt;
 }
 
 
@@ -170,23 +169,25 @@ int RGWAccessControlPolicy_SWIFT::create(const DoutPrefixProvider *dpp,
 
   if (read_list) {
     for (std::string_view uid : ceph::split(read_list, " ,")) {
-      int r = add_grant(dpp, driver, std::string{uid}, SWIFT_PERM_READ);
-      if (r < 0) {
-        ldpp_dout(dpp, 0) << "ERROR: add_grants for read returned r="
-                      << r << dendl;
-        return r;
+      auto grant = parse_grant(dpp, driver, std::string{uid}, SWIFT_PERM_READ);
+      if (!grant) {
+        ldpp_dout(dpp, 4) << "ERROR: failed to parse read acl grant "
+            << uid << dendl;
+        return -EINVAL;
       }
+      acl.add_grant(*grant);
     }
     rw_mask |= SWIFT_PERM_READ;
   }
   if (write_list) {
     for (std::string_view uid : ceph::split(write_list, " ,")) {
-      int r = add_grant(dpp, driver, std::string{uid}, SWIFT_PERM_WRITE);
-      if (r < 0) {
-        ldpp_dout(dpp, 0) << "ERROR: add_grants for write returned r="
-                      << r << dendl;
-        return r;
+      auto grant = parse_grant(dpp, driver, std::string{uid}, SWIFT_PERM_WRITE);
+      if (!grant) {
+        ldpp_dout(dpp, 4) << "ERROR: failed to parse write acl grant "
+            << uid << dendl;
+        return -EINVAL;
       }
+      acl.add_grant(*grant);
     }
     rw_mask |= SWIFT_PERM_WRITE;
   }
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index db12f6063558..50b020701623 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -16,9 +16,6 @@ class RGWUserCtl;
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 {
-  int add_grant(const DoutPrefixProvider* dpp, rgw::sal::Driver* driver,
-                const std::string& uid, uint32_t perm);
-
 public:
   int create(const DoutPrefixProvider *dpp,
 	     rgw::sal::Driver* driver,

From 33bc63e8d668dca38a57f6bb2cfaa980ee47f8c2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:08:45 -0500
Subject: [PATCH 0800/2492] rgw/acl/swift: create_container_policy() as free
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc  | 26 ++++++++++++++------------
 src/rgw/rgw_acl_swift.h   | 22 +++++++++++++++-------
 src/rgw/rgw_rest_swift.cc | 15 +++++++--------
 3 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 74d5e9bbaa73..a6c23f997202 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -153,19 +153,19 @@ static auto parse_grant(const DoutPrefixProvider* dpp,
   return std::nullopt;
 }
 
-
-int RGWAccessControlPolicy_SWIFT::create(const DoutPrefixProvider *dpp,
-					 rgw::sal::Driver* driver,
-                                         const rgw_user& id,
-                                         const std::string& name,
-                                         const char* read_list,
-                                         const char* write_list,
-                                         uint32_t& rw_mask)
+namespace rgw::swift {
+
+int create_container_policy(const DoutPrefixProvider *dpp,
+                            rgw::sal::Driver* driver,
+                            const rgw_user& id,
+                            const std::string& name,
+                            const char* read_list,
+                            const char* write_list,
+                            uint32_t& rw_mask,
+                            RGWAccessControlPolicy& policy)
 {
-  acl.create_default(id, name);
-  owner.id = id;
-  owner.display_name = name;
-  rw_mask = 0;
+  policy.create_default(id, name);
+  auto& acl = policy.get_acl();
 
   if (read_list) {
     for (std::string_view uid : ceph::split(read_list, " ,")) {
@@ -194,6 +194,8 @@ int RGWAccessControlPolicy_SWIFT::create(const DoutPrefixProvider *dpp,
   return 0;
 }
 
+} // namespace rgw::swift
+
 void RGWAccessControlPolicy_SWIFT::filter_merge(uint32_t rw_mask,
                                                 RGWAccessControlPolicy_SWIFT *old)
 {
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 50b020701623..e1c518849454 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -14,16 +14,24 @@
 
 class RGWUserCtl;
 
+namespace rgw::swift {
+
+/// Create a policy based on swift container acl headers
+/// X-Container-Read/X-Container-Write.
+int create_container_policy(const DoutPrefixProvider *dpp,
+                            rgw::sal::Driver* driver,
+                            const rgw_user& id,
+                            const std::string& name,
+                            const char* read_list,
+                            const char* write_list,
+                            uint32_t& rw_mask,
+                            RGWAccessControlPolicy& policy);
+
+} // namespace rgw::swift
+
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 {
 public:
-  int create(const DoutPrefixProvider *dpp,
-	     rgw::sal::Driver* driver,
-             const rgw_user& id,
-             const std::string& name,
-             const char* read_list,
-             const char* write_list,
-             uint32_t& rw_mask);
   void filter_merge(uint32_t mask, RGWAccessControlPolicy_SWIFT *policy);
   void to_str(std::string& read, std::string& write);
 };
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index a92d1f56f810..b486042e00f8 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -607,18 +607,17 @@ static int get_swift_container_settings(req_state * const s,
   *has_policy = false;
 
   if (read_list || write_list) {
-    RGWAccessControlPolicy_SWIFT swift_policy;
-    const auto r = swift_policy.create(s, driver,
-                                       s->user->get_id(),
-                                       s->user->get_display_name(),
-                                       read_list,
-                                       write_list,
-                                       *rw_mask);
+    int r = rgw::swift::create_container_policy(s, driver,
+                                                s->user->get_id(),
+                                                s->user->get_display_name(),
+                                                read_list,
+                                                write_list,
+                                                *rw_mask,
+                                                *policy);
     if (r < 0) {
       return r;
     }
 
-    *policy = swift_policy;
     *has_policy = true;
   }
 

From 4e1738db99fd76a55589e7b2fedf1a8dc0acaca4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:25:36 -0500
Subject: [PATCH 0801/2492] rgw/acl/swift: merge_policy() as free function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc | 18 ++++++++----------
 src/rgw/rgw_acl_swift.h  |  6 +++++-
 src/rgw/rgw_op.cc        |  6 +-----
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index a6c23f997202..4bf0ba925177 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -194,10 +194,8 @@ int create_container_policy(const DoutPrefixProvider *dpp,
   return 0;
 }
 
-} // namespace rgw::swift
-
-void RGWAccessControlPolicy_SWIFT::filter_merge(uint32_t rw_mask,
-                                                RGWAccessControlPolicy_SWIFT *old)
+void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
+                  RGWAccessControlPolicy& dest)
 {
   /* rw_mask&SWIFT_PERM_READ => setting read acl,
    * rw_mask&SWIFT_PERM_WRITE => setting write acl
@@ -207,15 +205,13 @@ void RGWAccessControlPolicy_SWIFT::filter_merge(uint32_t rw_mask,
     return;
   }
   rw_mask ^= (SWIFT_PERM_READ|SWIFT_PERM_WRITE);
-  for (auto &iter: old->acl.get_grant_map()) {
-    ACLGrant& grant = iter.second;
+  for (const auto &iter: src.get_acl().get_grant_map()) {
+    const ACLGrant& grant = iter.second;
     uint32_t perm = grant.get_permission().get_permissions();
     rgw_user id;
-    string url_spec;
     if (!grant.get_id(id)) {
       if (grant.get_group() != ACL_GROUP_ALL_USERS) {
-        url_spec = grant.get_referer();
-        if (url_spec.empty()) {
+        if (string url_spec = grant.get_referer(); url_spec.empty()) {
           continue;
         }
         if (perm == 0) {
@@ -225,11 +221,13 @@ void RGWAccessControlPolicy_SWIFT::filter_merge(uint32_t rw_mask,
       }
     }
     if (perm & rw_mask) {
-      acl.add_grant(grant);
+      dest.get_acl().add_grant(grant);
     }
   }
 }
 
+} // namespace rgw::swift
+
 void RGWAccessControlPolicy_SWIFT::to_str(string& read, string& write)
 {
   multimap<string, ACLGrant>& m = acl.get_grant_map();
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index e1c518849454..bf983ba2b8c2 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -27,12 +27,16 @@ int create_container_policy(const DoutPrefixProvider *dpp,
                             uint32_t& rw_mask,
                             RGWAccessControlPolicy& policy);
 
+/// Copy grants matching the permission mask (SWIFT_PERM_READ/WRITE) from
+/// one policy to another.
+void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
+                  RGWAccessControlPolicy& dest);
+
 } // namespace rgw::swift
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 {
 public:
-  void filter_merge(uint32_t mask, RGWAccessControlPolicy_SWIFT *policy);
   void to_str(std::string& read, std::string& write);
 };
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 39e12a5ab096..593d1598aefa 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4988,11 +4988,7 @@ void RGWPutMetadataBucket::execute(optional_yield y)
        * contain such keys yet. */
       if (has_policy) {
 	if (s->dialect.compare("swift") == 0) {
-	  auto old_policy =						\
-	    static_cast<RGWAccessControlPolicy_SWIFT*>(s->bucket_acl.get());
-	  auto new_policy = static_cast<RGWAccessControlPolicy_SWIFT*>(&policy);
-	  new_policy->filter_merge(policy_rw_mask, old_policy);
-	  policy = *new_policy;
+	  rgw::swift::merge_policy(policy_rw_mask, *s->bucket_acl, policy);
 	}
 	buffer::list bl;
 	policy.encode(bl);

From ade871e280cdfe0057e8f0b1f6ab0b3d152cc1f2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:33:30 -0500
Subject: [PATCH 0802/2492] rgw/acl/swift: format_container_acls() as free
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc  | 13 +++++--------
 src/rgw/rgw_acl_swift.h   |  6 ++++--
 src/rgw/rgw_rest_swift.cc |  4 +---
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 4bf0ba925177..e948e859162b 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -226,15 +226,10 @@ void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
   }
 }
 
-} // namespace rgw::swift
-
-void RGWAccessControlPolicy_SWIFT::to_str(string& read, string& write)
+void format_container_acls(const RGWAccessControlPolicy& policy,
+                           std::string& read, std::string& write)
 {
-  multimap<string, ACLGrant>& m = acl.get_grant_map();
-  multimap<string, ACLGrant>::iterator iter;
-
-  for (iter = m.begin(); iter != m.end(); ++iter) {
-    ACLGrant& grant = iter->second;
+  for (const auto& [k, grant] : policy.get_acl().get_grant_map()) {
     const uint32_t perm = grant.get_permission().get_permissions();
     rgw_user id;
     string url_spec;
@@ -269,6 +264,8 @@ void RGWAccessControlPolicy_SWIFT::to_str(string& read, string& write)
   }
 }
 
+} // namespace rgw::swift
+
 void RGWAccessControlPolicy_SWIFTAcct::add_grants(const DoutPrefixProvider *dpp,
 						  rgw::sal::Driver* driver,
                                                   const std::vector<std::string>& uids,
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index bf983ba2b8c2..8bc70eac7b0e 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -32,12 +32,14 @@ int create_container_policy(const DoutPrefixProvider *dpp,
 void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
                   RGWAccessControlPolicy& dest);
 
+/// Format the policy in terms of X-Container-Read/X-Container-Write strings.
+void format_container_acls(const RGWAccessControlPolicy& policy,
+                           std::string& read, std::string& write);
+
 } // namespace rgw::swift
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 {
-public:
-  void to_str(std::string& read, std::string& write);
 };
 
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index b486042e00f8..930156d4b512 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -470,10 +470,8 @@ static void dump_container_metadata(req_state *s,
   }
 
   if (rgw::sal::Object::empty(s->object.get())) {
-    auto swift_policy = \
-      static_cast<RGWAccessControlPolicy_SWIFT*>(s->bucket_acl.get());
     std::string read_acl, write_acl;
-    swift_policy->to_str(read_acl, write_acl);
+    rgw::swift::format_container_acls(*s->bucket_acl, read_acl, write_acl);
 
     if (read_acl.size()) {
       dump_header(s, "X-Container-Read", read_acl);

From bdbace0a4cc1c081013f0b88d7509855920f6e15 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:42:21 -0500
Subject: [PATCH 0803/2492] rgw/acl/swift: add_grants() uses user_to_grant()
 helper

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index e948e859162b..8ba3003ec185 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -100,11 +100,9 @@ static ACLGrant user_to_grant(const DoutPrefixProvider *dpp,
                               const std::string& uid,
                               const uint32_t perm)
 {
-  RGWUserInfo grant_user;
   ACLGrant grant;
-  std::unique_ptr<rgw::sal::User> user;
 
-  user = driver->get_user(rgw_user(uid));
+  std::unique_ptr<rgw::sal::User> user = driver->get_user(rgw_user(uid));
   if (user->load_user(dpp, null_yield) < 0) {
     ldpp_dout(dpp, 10) << "grant user does not exist: " << uid << dendl;
     /* skipping silently */
@@ -276,20 +274,10 @@ void RGWAccessControlPolicy_SWIFTAcct::add_grants(const DoutPrefixProvider *dpp,
 
     if (uid_is_public(uid)) {
       grant.set_group(ACL_GROUP_ALL_USERS, perm);
-      acl.add_grant(grant);
     } else  {
-      std::unique_ptr<rgw::sal::User> user = driver->get_user(rgw_user(uid));
-
-      if (user->load_user(dpp, null_yield) < 0) {
-        ldpp_dout(dpp, 10) << "grant user does not exist:" << uid << dendl;
-        /* skipping silently */
-        grant.set_canon(user->get_id(), std::string(), perm);
-        acl.add_grant(grant);
-      } else {
-        grant.set_canon(user->get_id(), user->get_display_name(), perm);
-        acl.add_grant(grant);
-      }
+      grant = user_to_grant(dpp, driver, uid, perm);
     }
+    acl.add_grant(grant);
   }
 }
 

From d2d1a87731f0f89cd3c4b7f15923bf09d8dbf00b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:43:19 -0500
Subject: [PATCH 0804/2492] rgw/acl/swift: add_grants() as static free function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc | 39 +++++++++++++++++++--------------------
 src/rgw/rgw_acl_swift.h  |  2 --
 2 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 8ba3003ec185..55766a336135 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -151,6 +151,22 @@ static auto parse_grant(const DoutPrefixProvider* dpp,
   return std::nullopt;
 }
 
+static void add_grants(const DoutPrefixProvider* dpp,
+                       rgw::sal::Driver* driver,
+                       const std::vector<std::string>& uids,
+                       uint32_t perm, RGWAccessControlList& acl)
+{
+  for (const auto& uid : uids) {
+    ACLGrant grant;
+    if (uid_is_public(uid)) {
+      grant.set_group(ACL_GROUP_ALL_USERS, perm);
+    } else  {
+      grant = user_to_grant(dpp, driver, uid, perm);
+    }
+    acl.add_grant(grant);
+  }
+}
+
 namespace rgw::swift {
 
 int create_container_policy(const DoutPrefixProvider *dpp,
@@ -264,23 +280,6 @@ void format_container_acls(const RGWAccessControlPolicy& policy,
 
 } // namespace rgw::swift
 
-void RGWAccessControlPolicy_SWIFTAcct::add_grants(const DoutPrefixProvider *dpp,
-						  rgw::sal::Driver* driver,
-                                                  const std::vector<std::string>& uids,
-                                                  const uint32_t perm)
-{
-  for (const auto& uid : uids) {
-    ACLGrant grant;
-
-    if (uid_is_public(uid)) {
-      grant.set_group(ACL_GROUP_ALL_USERS, perm);
-    } else  {
-      grant = user_to_grant(dpp, driver, uid, perm);
-    }
-    acl.add_grant(grant);
-  }
-}
-
 bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
 					      rgw::sal::Driver* driver,
                                               const rgw_user& id,
@@ -304,7 +303,7 @@ bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
     decode_json_obj(admin, *iter);
     ldpp_dout(dpp, 0) << "admins: " << admin << dendl;
 
-    add_grants(dpp, driver, admin, SWIFT_PERM_ADMIN);
+    add_grants(dpp, driver, admin, SWIFT_PERM_ADMIN, acl);
   }
 
   iter = parser.find_first("read-write");
@@ -313,7 +312,7 @@ bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
     decode_json_obj(readwrite, *iter);
     ldpp_dout(dpp, 0) << "read-write: " << readwrite << dendl;
 
-    add_grants(dpp, driver, readwrite, SWIFT_PERM_RWRT);
+    add_grants(dpp, driver, readwrite, SWIFT_PERM_RWRT, acl);
   }
 
   iter = parser.find_first("read-only");
@@ -322,7 +321,7 @@ bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
     decode_json_obj(readonly, *iter);
     ldpp_dout(dpp, 0) << "read-only: " << readonly << dendl;
 
-    add_grants(dpp, driver, readonly, SWIFT_PERM_READ);
+    add_grants(dpp, driver, readonly, SWIFT_PERM_READ, acl);
   }
 
   return true;
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 8bc70eac7b0e..68ddf3d95269 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -44,8 +44,6 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
 {
-  void add_grants(const DoutPrefixProvider* dpp, rgw::sal::Driver* driver,
-                  const std::vector<std::string>& uids, uint32_t perm);
 public:
   bool create(const DoutPrefixProvider *dpp,
 	      rgw::sal::Driver* driver,

From 78ba3808fb19950f53043227f2e7ff06809a5a10 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:59:18 -0500
Subject: [PATCH 0805/2492] rgw/acl/swift: create_account_policy() as free
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc  | 25 ++++++++++++-------------
 src/rgw/rgw_acl_swift.h   | 13 ++++++++-----
 src/rgw/rgw_rest_swift.cc | 14 ++++++--------
 3 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 55766a336135..16b03e41ab6c 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -278,23 +278,20 @@ void format_container_acls(const RGWAccessControlPolicy& policy,
   }
 }
 
-} // namespace rgw::swift
-
-bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
-					      rgw::sal::Driver* driver,
-                                              const rgw_user& id,
-                                              const std::string& name,
-                                              const std::string& acl_str)
+int create_account_policy(const DoutPrefixProvider* dpp,
+                          rgw::sal::Driver* driver,
+                          const rgw_user& id,
+                          const std::string& name,
+                          const std::string& acl_str,
+                          RGWAccessControlPolicy& policy)
 {
-  acl.create_default(id, name);
-  owner.id = id;
-  owner.display_name = name;
+  policy.create_default(id, name);
+  auto& acl = policy.get_acl();
 
   JSONParser parser;
-
   if (!parser.parse(acl_str.c_str(), acl_str.length())) {
     ldpp_dout(dpp, 0) << "ERROR: JSONParser::parse returned error=" << dendl;
-    return false;
+    return -EINVAL;
   }
 
   JSONObjIter iter = parser.find_first("admin");
@@ -324,9 +321,11 @@ bool RGWAccessControlPolicy_SWIFTAcct::create(const DoutPrefixProvider *dpp,
     add_grants(dpp, driver, readonly, SWIFT_PERM_READ, acl);
   }
 
-  return true;
+  return 0;
 }
 
+} // namespace rgw::swift
+
 boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
 {
   std::vector<std::string> admin;
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 68ddf3d95269..dbe89efef796 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -36,6 +36,14 @@ void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
 void format_container_acls(const RGWAccessControlPolicy& policy,
                            std::string& read, std::string& write);
 
+/// Create a policy based on swift account acl header X-Account-Access-Control.
+int create_account_policy(const DoutPrefixProvider* dpp,
+                          rgw::sal::Driver* driver,
+                          const rgw_user& id,
+                          const std::string& name,
+                          const std::string& acl_str,
+                          RGWAccessControlPolicy& policy);
+
 } // namespace rgw::swift
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
@@ -45,10 +53,5 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
 {
 public:
-  bool create(const DoutPrefixProvider *dpp,
-	      rgw::sal::Driver* driver,
-              const rgw_user& id,
-              const std::string& name,
-              const std::string& acl_str);
   boost::optional<std::string> to_str() const;
 };
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 930156d4b512..b5ac5a427c3f 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -1064,16 +1064,14 @@ static int get_swift_account_settings(req_state * const s,
 
   const char * const acl_attr = s->info.env->get("HTTP_X_ACCOUNT_ACCESS_CONTROL");
   if (acl_attr) {
-    RGWAccessControlPolicy_SWIFTAcct swift_acct_policy;
-    const bool r = swift_acct_policy.create(s, driver,
-                                     s->user->get_id(),
-                                     s->user->get_display_name(),
-                                     string(acl_attr));
-    if (r != true) {
-      return -EINVAL;
+    int r = rgw::swift::create_account_policy(s, driver,
+                                              s->user->get_id(),
+                                              s->user->get_display_name(),
+                                              acl_attr, *policy);
+    if (r < 0) {
+      return r;
     }
 
-    *policy = swift_acct_policy;
     *has_policy = true;
   }
 

From f646ebf543686f5fc01ae60b9ad1a72324da54ba Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 13:59:57 -0500
Subject: [PATCH 0806/2492] rgw/acl/swift: format_account_acl() as free
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.cc  | 13 ++++++++-----
 src/rgw/rgw_acl_swift.h   |  8 +++++---
 src/rgw/rgw_rest_swift.cc |  4 ++--
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 16b03e41ab6c..048f7b491521 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -324,16 +324,17 @@ int create_account_policy(const DoutPrefixProvider* dpp,
   return 0;
 }
 
-} // namespace rgw::swift
-
-boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
+auto format_account_acl(const RGWAccessControlPolicy& policy)
+  -> std::optional<std::string>
 {
+  const ACLOwner& owner = policy.get_owner();
+
   std::vector<std::string> admin;
   std::vector<std::string> readwrite;
   std::vector<std::string> readonly;
 
   /* Partition the grant map into three not-overlapping groups. */
-  for (const auto& item : get_acl().get_grant_map()) {
+  for (const auto& item : policy.get_acl().get_grant_map()) {
     const ACLGrant& grant = item.second;
     const uint32_t perm = grant.get_permission().get_permissions();
 
@@ -361,7 +362,7 @@ boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
   /* If there is no grant to serialize, let's exit earlier to not return
    * an empty JSON object which brakes the functional tests of Swift. */
   if (admin.empty() && readwrite.empty() && readonly.empty()) {
-    return boost::none;
+    return std::nullopt;
   }
 
   /* Serialize the groups. */
@@ -384,3 +385,5 @@ boost::optional<std::string> RGWAccessControlPolicy_SWIFTAcct::to_str() const
 
   return oss.str();
 }
+
+} // namespace rgw::swift
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index dbe89efef796..9eeb9cb6caea 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -8,7 +8,6 @@
 #include <string>
 #include <include/types.h>
 
-#include <boost/optional.hpp>
 
 #include "rgw_acl.h"
 
@@ -44,6 +43,11 @@ int create_account_policy(const DoutPrefixProvider* dpp,
                           const std::string& acl_str,
                           RGWAccessControlPolicy& policy);
 
+/// Format the policy in terms of the X-Account-Access-Control string. Returns
+/// std::nullopt if there are no admin/read-write/read-only entries.
+auto format_account_acl(const RGWAccessControlPolicy& policy)
+  -> std::optional<std::string>;
+
 } // namespace rgw::swift
 
 class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
@@ -52,6 +56,4 @@ class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
 
 class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
 {
-public:
-  boost::optional<std::string> to_str() const;
 };
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index b5ac5a427c3f..a752e827b12c 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -164,8 +164,8 @@ static void dump_account_metadata(req_state * const s,
     }
   }
 
-  /* Dump account ACLs */
-  auto account_acls = policy.to_str();
+  /* Dump account ACLs, if any */
+  auto account_acls = rgw::swift::format_account_acl(policy);
   if (account_acls) {
     dump_header(s, "X-Account-Access-Control", std::move(*account_acls));
   }

From 2ac0daad1cb7daca8397206d2be693b615d87df2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 14:05:36 -0500
Subject: [PATCH 0807/2492] rgw/acl/swift: remove RGWAccessControlPolicy
 subclasses

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_swift.h   | 19 ++++---------------
 src/rgw/rgw_op.cc         |  4 ++--
 src/rgw/rgw_rest_swift.cc | 17 ++++++-----------
 3 files changed, 12 insertions(+), 28 deletions(-)

diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 9eeb9cb6caea..a16bea894dbe 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -3,15 +3,12 @@
 
 #pragma once
 
-#include <map>
-#include <vector>
 #include <string>
-#include <include/types.h>
+#include "rgw_sal_fwd.h"
+#include "rgw_user_types.h"
 
-
-#include "rgw_acl.h"
-
-class RGWUserCtl;
+class DoutPrefixProvider;
+class RGWAccessControlPolicy;
 
 namespace rgw::swift {
 
@@ -49,11 +46,3 @@ auto format_account_acl(const RGWAccessControlPolicy& policy)
   -> std::optional<std::string>;
 
 } // namespace rgw::swift
-
-class RGWAccessControlPolicy_SWIFT : public RGWAccessControlPolicy
-{
-};
-
-class RGWAccessControlPolicy_SWIFTAcct : public RGWAccessControlPolicy
-{
-};
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 593d1598aefa..cbc13d593f12 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -519,9 +519,9 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
      * the Swift's infrastructure that don't really need req_state::user.
      * Typical example here is the implementation of /info. */
     if (!s->user->get_id().empty()) {
-      s->user_acl = std::make_unique<RGWAccessControlPolicy_SWIFTAcct>();
+      s->user_acl = std::make_unique<RGWAccessControlPolicy>();
     }
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_SWIFT>();
+    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
   } else {
     s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
   }
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index a752e827b12c..55ebac5137c8 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -92,7 +92,7 @@ static void dump_account_metadata(req_state * const s,
                                   /* const */map<string, bufferlist>& attrs,
                                   const RGWQuotaInfo& quota,
                                   int32_t max_buckets,
-                                  const RGWAccessControlPolicy_SWIFTAcct &policy)
+                                  const RGWAccessControlPolicy& policy)
 {
   /* Adding X-Timestamp to keep align with Swift API */
   dump_header(s, "X-Timestamp", ceph_clock_now());
@@ -188,7 +188,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_begin(bool has_buckets)
             s->user->get_attrs(),
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            static_cast<RGWAccessControlPolicy_SWIFTAcct&>(*s->user_acl));
+            *s->user_acl);
     dump_errno(s);
     dump_header(s, "Accept-Ranges", "bytes");
     end_header(s, NULL, NULL, NO_CONTENT_LENGTH, true);
@@ -287,7 +287,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_end()
             s->user->get_attrs(),
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            static_cast<RGWAccessControlPolicy_SWIFTAcct&>(*s->user_acl));
+            *s->user_acl);
     dump_errno(s);
     end_header(s, nullptr, nullptr, s->formatter->get_len(), true);
   }
@@ -565,7 +565,7 @@ void RGWStatAccount_ObjStore_SWIFT::send_response()
             attrs,
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            static_cast<RGWAccessControlPolicy_SWIFTAcct&>(*s->user_acl));
+            *s->user_acl);
   }
 
   set_req_state_err(s, op_ret);
@@ -1057,7 +1057,7 @@ void RGWPutObj_ObjStore_SWIFT::send_response()
 
 static int get_swift_account_settings(req_state * const s,
                                       rgw::sal::Driver*  const driver,
-                                      RGWAccessControlPolicy_SWIFTAcct*  const policy,
+                                      RGWAccessControlPolicy* const policy,
                                       bool * const has_policy)
 {
   *has_policy = false;
@@ -1084,12 +1084,7 @@ int RGWPutMetadataAccount_ObjStore_SWIFT::get_params(optional_yield y)
     return -EINVAL;
   }
 
-  int ret = get_swift_account_settings(s,
-                                       driver,
-                                       // FIXME: we need to carry unique_ptr in generic class
-                                       // and allocate appropriate ACL class in the ctor
-                                       static_cast<RGWAccessControlPolicy_SWIFTAcct *>(&policy),
-                                       &has_policy);
+  int ret = get_swift_account_settings(s, driver, &policy, &has_policy);
   if (ret < 0) {
     return ret;
   }

From 99131fc14bbfcf07f2dbba7a67a6250f9da57583 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 15:11:33 -0500
Subject: [PATCH 0808/2492] rgw/acl: use create_default() instead of empty
 canned acls

create_default() adds the same default acl, without the relation to s3
canned acls

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_tools.cc |  5 +++--
 src/rgw/rgw_file_int.h            | 32 ++++++++++++-------------------
 2 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 8219849af468..bf78b9bb22e2 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -412,9 +412,10 @@ int RGWDataAccess::Object::put(bufferlist& data,
   }
 
   if (!aclbl) {
-    RGWAccessControlPolicy_S3 policy;
+    RGWAccessControlPolicy policy;
 
-    policy.create_canned(bucket->policy.get_owner(), bucket->policy.get_owner(), string()); /* default private policy */
+    const auto& owner = bucket->policy.get_owner();
+    policy.create_default(owner.id, owner.display_name); // default private policy
 
     policy.encode(aclbl.emplace());
   }
diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h
index 4e3b269ffd0a..5658003adb8a 100644
--- a/src/rgw/rgw_file_int.h
+++ b/src/rgw/rgw_file_int.h
@@ -1914,11 +1914,9 @@ class RGWCreateBucketRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy;
-    /* we don't have (any) headers, so just create canned ACLs */
-    int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
-    policy = s3policy;
-    return ret;
+    /* we don't have (any) headers, so just create default ACLs */
+    policy.create_default(state->owner.id, state->owner.display_name);
+    return 0;
   }
 
   void send_response() override {
@@ -2030,11 +2028,9 @@ class RGWPutObjRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy;
-    /* we don't have (any) headers, so just create canned ACLs */
-    int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
-    policy = s3policy;
-    return ret;
+    /* we don't have (any) headers, so just create default ACLs */
+    policy.create_default(state->owner.id, state->owner.display_name);
+    return 0;
   }
 
   int get_data(buffer::list& _bl) override {
@@ -2534,11 +2530,9 @@ class RGWWriteRequest : public RGWLibContinuedReq,
 
   int get_params(optional_yield) override {
     req_state* state = get_state();
-    RGWAccessControlPolicy_S3 s3policy;
-    /* we don't have (any) headers, so just create canned ACLs */
-    int ret = s3policy.create_canned(state->owner, state->bucket_owner, state->canned_acl);
-    policy = s3policy;
-    return ret;
+    /* we don't have (any) headers, so just create default ACLs */
+    policy.create_default(state->owner.id, state->owner.display_name);
+    return 0;
   }
 
   int get_data(buffer::list& _bl) override {
@@ -2641,15 +2635,13 @@ class RGWCopyObjRequest : public RGWLibRequest,
 
   int get_params(optional_yield) override {
     req_state* s = get_state();
-    RGWAccessControlPolicy_S3 s3policy;
-    /* we don't have (any) headers, so just create canned ACLs */
-    int ret = s3policy.create_canned(s->owner, s->bucket_owner, s->canned_acl);
-    dest_policy = s3policy;
+    /* we don't have (any) headers, so just create default ACLs */
+    dest_policy.create_default(s->owner.id, s->owner.display_name);
     /* src_object required before RGWCopyObj::verify_permissions() */
     rgw_obj_key k = rgw_obj_key(src_name);
     s->src_object = s->bucket->get_object(k);
     s->object = s->src_object->clone(); // needed to avoid trap at rgw_op.cc:5150
-    return ret;
+    return 0;
   }
 
   void send_response() override {}

From bce77ab23b57fce2ffd9608fdfb7e8e4a0f33f9f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 15:13:25 -0500
Subject: [PATCH 0809/2492] rgw/acl/s3: create_canned_acl() as free function

adds canned acls to base class RGWAccessControlPolicy

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_s3.cc  | 73 ++++++++++++++++++++++++++----------------
 src/rgw/rgw_acl_s3.h   | 21 ++++++------
 src/rgw/rgw_rest_s3.cc | 12 +++----
 3 files changed, 62 insertions(+), 44 deletions(-)

diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index 52eeb44f2430..f593b5cee528 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -359,46 +359,48 @@ static int parse_acl_header(const DoutPrefixProvider *dpp, rgw::sal::Driver* dri
   return 0;
 }
 
-int RGWAccessControlList_S3::create_canned(ACLOwner& owner, ACLOwner& bucket_owner, const string& canned_acl)
+static int create_canned(const ACLOwner& owner, const ACLOwner& bucket_owner,
+                         const string& canned_acl, RGWAccessControlList& acl)
 {
-  acl_user_map.clear();
-  grant_map.clear();
-
-  ACLGrant owner_grant;
-
   const rgw_user& bid = bucket_owner.id;
   const std::string& bname = bucket_owner.display_name;
 
   /* owner gets full control */
-  owner_grant.set_canon(owner.id, owner.display_name, RGW_PERM_FULL_CONTROL);
-  add_grant(owner_grant);
+  {
+    ACLGrant grant;
+    grant.set_canon(owner.id, owner.display_name, RGW_PERM_FULL_CONTROL);
+    acl.add_grant(grant);
+  }
 
   if (canned_acl.size() == 0 || canned_acl.compare("private") == 0) {
     return 0;
   }
 
-  ACLGrant bucket_owner_grant;
-  ACLGrant group_grant;
-  if (canned_acl.compare("public-read") == 0) {
-    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
-    add_grant(group_grant);
-  } else if (canned_acl.compare("public-read-write") == 0) {
-    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
-    add_grant(group_grant);
-    group_grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_WRITE);
-    add_grant(group_grant);
-  } else if (canned_acl.compare("authenticated-read") == 0) {
-    group_grant.set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_READ);
-    add_grant(group_grant);
-  } else if (canned_acl.compare("bucket-owner-read") == 0) {
-    bucket_owner_grant.set_canon(bid, bname, RGW_PERM_READ);
+  if (canned_acl == "public-read") {
+    ACLGrant grant;
+    grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
+    acl.add_grant(grant);
+  } else if (canned_acl == "public-read-write") {
+    ACLGrant grant;
+    grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_READ);
+    acl.add_grant(grant);
+    grant.set_group(ACL_GROUP_ALL_USERS, RGW_PERM_WRITE);
+    acl.add_grant(grant);
+  } else if (canned_acl == "authenticated-read") {
+    ACLGrant grant;
+    grant.set_group(ACL_GROUP_AUTHENTICATED_USERS, RGW_PERM_READ);
+    acl.add_grant(grant);
+  } else if (canned_acl == "bucket-owner-read") {
     if (bid != owner.id) {
-      add_grant(bucket_owner_grant);
+      ACLGrant grant;
+      grant.set_canon(bid, bname, RGW_PERM_READ);
+      acl.add_grant(grant);
     }
-  } else if (canned_acl.compare("bucket-owner-full-control") == 0) {
-    bucket_owner_grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
+  } else if (canned_acl == "bucket-owner-full-control") {
     if (bid != owner.id) {
-      add_grant(bucket_owner_grant);
+      ACLGrant grant;
+      grant.set_canon(bid, bname, RGW_PERM_FULL_CONTROL);
+      acl.add_grant(grant);
     }
   } else {
     return -EINVAL;
@@ -626,3 +628,20 @@ ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
   return ACL_GROUP_NONE;
 }
 
+
+namespace rgw::s3 {
+
+int create_canned_acl(const ACLOwner& owner,
+                      const ACLOwner& bucket_owner,
+                      const std::string& canned_acl,
+                      RGWAccessControlPolicy& policy)
+{
+  if (owner.id == rgw_user("anonymous")) {
+    policy.set_owner(bucket_owner);
+  } else {
+    policy.set_owner(owner);
+  }
+  return create_canned(owner, bucket_owner, canned_acl, policy.get_acl());
+}
+
+} // namespace rgw::s3
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index e024f5c42ffb..740cbf35f82c 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -55,7 +55,6 @@ class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
   bool xml_end(const char *el) override;
   void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
 
-  int create_canned(ACLOwner& owner, ACLOwner& bucket_owner, const std::string& canned_acl);
   int create_from_grants(std::list<ACLGrant>& grants);
 };
 
@@ -80,16 +79,6 @@ class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
   int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
 	      RGWAccessControlPolicy& dest, std::string &err_msg);
 
-  virtual int create_canned(ACLOwner& _owner, ACLOwner& bucket_owner, const std::string& canned_acl) {
-    RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
-    if (_owner.id == rgw_user("anonymous")) {
-      owner = bucket_owner;
-    } else {
-      owner = _owner;
-    }
-    int ret = _acl.create_canned(owner, bucket_owner, canned_acl);
-    return ret;
-  }
   int create_from_headers(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
 			  const RGWEnv *env, ACLOwner& _owner);
 };
@@ -106,3 +95,13 @@ class RGWACLXMLParser_S3 : public RGWXMLParser
 public:
   explicit RGWACLXMLParser_S3(CephContext *_cct) : cct(_cct) {}
 };
+
+namespace rgw::s3 {
+
+/// Construct a policy from a s3 canned acl string.
+int create_canned_acl(const ACLOwner& owner,
+                      const ACLOwner& bucket_owner,
+                      const std::string& canned_acl,
+                      RGWAccessControlPolicy& policy);
+
+} // namespace rgw::s3
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index e4c90bc81e1d..0e557a13415a 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2393,7 +2393,8 @@ static int create_s3_policy(req_state *s, rgw::sal::Driver* driver,
     return s3policy.create_from_headers(s, driver, s->info.env, owner);
   }
 
-  return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
+  return rgw::s3::create_canned_acl(owner, s->bucket_owner,
+                                    s->canned_acl, s3policy);
 }
 
 class RGWLocationConstraint : public XMLObj
@@ -3215,15 +3216,14 @@ int RGWPostObj_ObjStore_S3::get_policy(optional_yield y)
   string canned_acl;
   part_str(parts, "acl", &canned_acl);
 
-  RGWAccessControlPolicy_S3 s3policy;
   ldpp_dout(this, 20) << "canned_acl=" << canned_acl << dendl;
-  if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
+  int r = rgw::s3::create_canned_acl(s->owner, s->bucket_owner,
+                                     canned_acl, policy);
+  if (r < 0) {
     err_msg = "Bad canned ACLs";
-    return -EINVAL;
+    return r;
   }
 
-  policy = s3policy;
-
   return 0;
 }
 

From 3353abb54972b9bb65aec30ec052fec45bea076b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 16 Nov 2023 16:38:31 -0500
Subject: [PATCH 0810/2492] rgw/acl/s3: create_policy_from_headers() as free
 function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_s3.cc  | 95 +++++++++++++++---------------------------
 src/rgw/rgw_acl_s3.h   | 12 +++---
 src/rgw/rgw_common.cc  |  4 +-
 src/rgw/rgw_common.h   |  4 +-
 src/rgw/rgw_rest_s3.cc |  5 ++-
 5 files changed, 48 insertions(+), 72 deletions(-)

diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index f593b5cee528..f2150dda79bd 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -7,6 +7,7 @@
 #include <map>
 
 #include "include/types.h"
+#include "common/split.h"
 
 #include "rgw_acl_s3.h"
 #include "rgw_user.h"
@@ -285,16 +286,11 @@ struct s3_acl_header {
   const char *http_header;
 };
 
-static const char *get_acl_header(const RGWEnv *env,
-        const struct s3_acl_header *perm)
-{
-  const char *header = perm->http_header;
-
-  return env->get(header, NULL);
-}
-
-static int parse_grantee_str(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, string& grantee_str,
-        const struct s3_acl_header *perm, ACLGrant& grant)
+static int parse_grantee_str(const DoutPrefixProvider* dpp,
+                             rgw::sal::Driver* driver,
+                             const std::string& grantee_str,
+                             const s3_acl_header* perm,
+                             ACLGrant& grant)
 {
   string id_type, id_val_quoted;
   int rgw_perm = perm->rgw_perm;
@@ -333,27 +329,22 @@ static int parse_grantee_str(const DoutPrefixProvider *dpp, rgw::sal::Driver* dr
   return 0;
 }
 
-static int parse_acl_header(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
-			    const RGWEnv *env, const struct s3_acl_header *perm,
-			    std::list<ACLGrant>& _grants)
+static int parse_acl_header(const DoutPrefixProvider* dpp, rgw::sal::Driver* driver,
+                            const RGWEnv& env, const s3_acl_header* perm,
+                            RGWAccessControlList& acl)
 {
-  std::list<string> grantees;
-  std::string hacl_str;
-
-  const char *hacl = get_acl_header(env, perm);
-  if (hacl == NULL)
+  const char* hacl = env.get(perm->http_header, nullptr);
+  if (hacl == nullptr) {
     return 0;
+  }
 
-  hacl_str = hacl;
-  get_str_list(hacl_str, ",", grantees);
-
-  for (list<string>::iterator it = grantees.begin(); it != grantees.end(); ++it) {
+  for (std::string_view grantee : ceph::split(hacl, ",")) {
     ACLGrant grant;
-    int ret = parse_grantee_str(dpp, driver, *it, perm, grant);
+    int ret = parse_grantee_str(dpp, driver, std::string{grantee}, perm, grant);
     if (ret < 0)
       return ret;
 
-    _grants.push_back(grant);
+    acl.add_grant(grant);
   }
 
   return 0;
@@ -409,21 +400,6 @@ static int create_canned(const ACLOwner& owner, const ACLOwner& bucket_owner,
   return 0;
 }
 
-int RGWAccessControlList_S3::create_from_grants(std::list<ACLGrant>& grants)
-{
-  if (grants.empty())
-    return -EINVAL;
-
-  acl_user_map.clear();
-  grant_map.clear();
-
-  for (const auto& g : grants) {
-    add_grant(g);
-  }
-
-  return 0;
-}
-
 bool RGWAccessControlPolicy_S3::xml_end(const char *el) {
   RGWAccessControlList_S3 *s3acl =
       static_cast<RGWAccessControlList_S3 *>(find_first("AccessControlList"));
@@ -457,28 +433,6 @@ static const s3_acl_header acl_header_perms[] = {
   {0, NULL}
 };
 
-int RGWAccessControlPolicy_S3::create_from_headers(const DoutPrefixProvider *dpp,
-						   rgw::sal::Driver* driver,
-						   const RGWEnv *env, ACLOwner& _owner)
-{
-  std::list<ACLGrant> grants;
-  int r = 0;
-
-  for (const struct s3_acl_header *p = acl_header_perms; p->rgw_perm; p++) {
-    r = parse_acl_header(dpp, driver, env, p, grants);
-    if (r < 0) {
-      return r;
-    }
-  }
-
-  RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
-  r = _acl.create_from_grants(grants);
-
-  owner = _owner;
-
-  return r;
-}
-
 /*
   can only be called on object that was parsed
  */
@@ -644,4 +598,23 @@ int create_canned_acl(const ACLOwner& owner,
   return create_canned(owner, bucket_owner, canned_acl, policy.get_acl());
 }
 
+int create_policy_from_headers(const DoutPrefixProvider* dpp,
+                               rgw::sal::Driver* driver,
+                               const ACLOwner& owner,
+                               const RGWEnv& env,
+                               RGWAccessControlPolicy& policy)
+{
+  policy.set_owner(owner);
+  auto& acl = policy.get_acl();
+
+  for (const s3_acl_header* p = acl_header_perms; p->rgw_perm; p++) {
+    int r = parse_acl_header(dpp, driver, env, p, acl);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return 0;
+}
+
 } // namespace rgw::s3
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index 740cbf35f82c..9753dd55b70e 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -54,8 +54,6 @@ class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
 public:
   bool xml_end(const char *el) override;
   void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
-
-  int create_from_grants(std::list<ACLGrant>& grants);
 };
 
 class ACLOwner_S3 : public ACLOwner, public XMLObj
@@ -78,9 +76,6 @@ class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
   void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
   int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
 	      RGWAccessControlPolicy& dest, std::string &err_msg);
-
-  int create_from_headers(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
-			  const RGWEnv *env, ACLOwner& _owner);
 };
 
 /**
@@ -104,4 +99,11 @@ int create_canned_acl(const ACLOwner& owner,
                       const std::string& canned_acl,
                       RGWAccessControlPolicy& policy);
 
+/// Construct a policy from x-amz-grant-* request headers.
+int create_policy_from_headers(const DoutPrefixProvider* dpp,
+                               rgw::sal::Driver* driver,
+                               const ACLOwner& owner,
+                               const RGWEnv& env,
+                               RGWAccessControlPolicy& policy);
+
 } // namespace rgw::s3
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 6ec10f2cc2f4..3ab882d35bbc 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -639,7 +639,7 @@ bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns, bool extended_for
   return true;
 }
 
-int parse_key_value(string& in_str, const char *delim, string& key, string& val)
+int parse_key_value(const string& in_str, const char *delim, string& key, string& val)
 {
   if (delim == NULL)
     return -EINVAL;
@@ -654,7 +654,7 @@ int parse_key_value(string& in_str, const char *delim, string& key, string& val)
   return 0;
 }
 
-int parse_key_value(string& in_str, string& key, string& val)
+int parse_key_value(const string& in_str, string& key, string& val)
 {
   return parse_key_value(in_str, "=", key,val);
 }
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 85d5ab2c155b..15d2e800fdee 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1490,8 +1490,8 @@ bool rgw_set_amz_meta_header(
 
 extern std::string rgw_string_unquote(const std::string& s);
 extern void parse_csv_string(const std::string& ival, std::vector<std::string>& ovals);
-extern int parse_key_value(std::string& in_str, std::string& key, std::string& val);
-extern int parse_key_value(std::string& in_str, const char *delim, std::string& key, std::string& val);
+extern int parse_key_value(const std::string& in_str, std::string& key, std::string& val);
+extern int parse_key_value(const std::string& in_str, const char *delim, std::string& key, std::string& val);
 
 extern boost::optional<std::pair<std::string_view,std::string_view>>
 parse_key_value(const std::string_view& in_str,
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 0e557a13415a..a5956def0d25 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2384,13 +2384,14 @@ void RGWStatBucket_ObjStore_S3::send_response()
 
 static int create_s3_policy(req_state *s, rgw::sal::Driver* driver,
 			    RGWAccessControlPolicy_S3& s3policy,
-			    ACLOwner& owner)
+			    const ACLOwner& owner)
 {
   if (s->has_acl_header) {
     if (!s->canned_acl.empty())
       return -ERR_INVALID_REQUEST;
 
-    return s3policy.create_from_headers(s, driver, s->info.env, owner);
+    return rgw::s3::create_policy_from_headers(s, driver, owner,
+                                               *s->info.env, s3policy);
   }
 
   return rgw::s3::create_canned_acl(owner, s->bucket_owner,

From 4ffa4498d673f7ce9f8382fca1fa78a0c329828b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 17 Nov 2023 15:31:17 -0500
Subject: [PATCH 0811/2492] rgw/acl/s3: write_policy_xml() as free function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc |  2 +-
 src/rgw/rgw_acl.h                  |  2 ++
 src/rgw/rgw_acl_s3.cc              | 58 ++++++++++++++++--------------
 src/rgw/rgw_acl_s3.h               |  9 +++--
 src/rgw/rgw_op.cc                  | 20 +++++------
 src/rgw/rgw_rest_s3.cc             |  2 +-
 6 files changed, 49 insertions(+), 44 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 002485c672ff..6cbcbeae9a2b 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -978,7 +978,7 @@ int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Driver* driver, RGWBucketAdminOpS
   if (ret < 0)
     return ret;
 
-  policy.to_xml(dpp, os);
+  rgw::s3::write_policy_xml(policy, os);
 
   return 0;
 }
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index f60295d90ab4..c69ee88404e3 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -27,6 +27,8 @@ class ACLGrant
   ACLGroupTypeEnum group;
   std::string url_spec;
 
+  friend void to_xml(const ACLGrant& grant, std::ostream& out);
+
 public:
   ACLGrant() : group(ACL_GROUP_NONE) {}
   virtual ~ACLGrant() {}
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index f2150dda79bd..f663e3830921 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -25,8 +25,9 @@ using namespace std;
 static string rgw_uri_all_users = RGW_URI_ALL_USERS;
 static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
 
-void ACLPermission_S3::to_xml(ostream& out)
+void to_xml(ACLPermission perm, std::ostream& out)
 {
+  const uint32_t flags = perm.get_permissions();
   if ((flags & RGW_PERM_FULL_CONTROL) == RGW_PERM_FULL_CONTROL) {
    out << "<Permission>FULL_CONTROL</Permission>";
   } else {
@@ -143,14 +144,15 @@ bool ACLOwner_S3::xml_end(const char *el) {
   return true;
 }
 
-void  ACLOwner_S3::to_xml(ostream& out) {
+void to_xml(const ACLOwner& o, std::ostream& out)
+{
   string s;
-  id.to_str(s);
+  o.id.to_str(s);
   if (s.empty())
     return;
   out << "<Owner>" << "<ID>" << s << "</ID>";
-  if (!display_name.empty())
-    out << "<DisplayName>" << display_name << "</DisplayName>";
+  if (!o.display_name.empty())
+    out << "<DisplayName>" << o.display_name << "</DisplayName>";
   out << "</Owner>";
 }
 
@@ -211,8 +213,9 @@ bool ACLGrant_S3::xml_end(const char *el) {
   return true;
 }
 
-void ACLGrant_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
-  ACLPermission_S3& perm = static_cast<ACLPermission_S3 &>(permission);
+void to_xml(const ACLGrant& grant, ostream& out)
+{
+  const ACLPermission perm = grant.get_permission();
 
   /* only show s3 compatible permissions */
   if (!(perm.get_permissions() & RGW_PERM_ALL_S3))
@@ -221,20 +224,19 @@ void ACLGrant_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
   string uri;
 
   out << "<Grant>" <<
-         "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"" << ACLGranteeType_S3::to_string(type) << "\">";
-  switch (type.get_type()) {
+         "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"" << ACLGranteeType_S3::to_string(grant.type) << "\">";
+  switch (grant.type.get_type()) {
   case ACL_TYPE_CANON_USER:
-    out << "<ID>" << id << "</ID>";
-    if (name.size()) {
-      out << "<DisplayName>" << name << "</DisplayName>";
+    out << "<ID>" << grant.id << "</ID>";
+    if (grant.name.size()) {
+      out << "<DisplayName>" << grant.name << "</DisplayName>";
     }
     break;
   case ACL_TYPE_EMAIL_USER:
-    out << "<EmailAddress>" << email << "</EmailAddress>";
+    out << "<EmailAddress>" << grant.email << "</EmailAddress>";
     break;
   case ACL_TYPE_GROUP:
-    if (!group_to_uri(group, uri)) {
-      ldpp_dout(dpp, 0) << "ERROR: group_to_uri failed with group=" << (int)group << dendl;
+    if (!ACLGrant_S3::group_to_uri(grant.group, uri)) {
       break;
     }
     out << "<URI>" << uri << "</URI>";
@@ -243,7 +245,7 @@ void ACLGrant_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
     break;
   }
   out << "</Grantee>";
-  perm.to_xml(out);
+  to_xml(perm, out);
   out << "</Grant>";
 }
 
@@ -271,12 +273,11 @@ bool RGWAccessControlList_S3::xml_end(const char *el) {
   return true;
 }
 
-void RGWAccessControlList_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
-  multimap<string, ACLGrant>::iterator iter;
+void to_xml(const RGWAccessControlList& acl, std::ostream& out)
+{
   out << "<AccessControlList>";
-  for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
-    ACLGrant_S3& grant = static_cast<ACLGrant_S3 &>(iter->second);
-    grant.to_xml(dpp, out);
+  for (const auto& p : acl.get_grant_map()) {
+    to_xml(p.second, out);
   }
   out << "</AccessControlList>";
 }
@@ -415,12 +416,11 @@ bool RGWAccessControlPolicy_S3::xml_end(const char *el) {
   return true;
 }
 
-void RGWAccessControlPolicy_S3::to_xml(const DoutPrefixProvider* dpp, ostream& out) {
+void to_xml(const RGWAccessControlPolicy& p, std::ostream& out)
+{
   out << "<AccessControlPolicy xmlns=\"" << XMLNS_AWS_S3 << "\">";
-  ACLOwner_S3& _owner = static_cast<ACLOwner_S3 &>(owner);
-  RGWAccessControlList_S3& _acl = static_cast<RGWAccessControlList_S3 &>(acl);
-  _owner.to_xml(out);
-  _acl.to_xml(dpp, out);
+  to_xml(p.get_owner(), out);
+  to_xml(p.get_acl(), out);
   out << "</AccessControlPolicy>";
 }
 
@@ -585,6 +585,12 @@ ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
 
 namespace rgw::s3 {
 
+void write_policy_xml(const RGWAccessControlPolicy& policy,
+                      std::ostream& out)
+{
+  to_xml(policy, out);
+}
+
 int create_canned_acl(const ACLOwner& owner,
                       const ACLOwner& bucket_owner,
                       const std::string& canned_acl,
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index 9753dd55b70e..7b8489299715 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -22,7 +22,6 @@ class ACLPermission_S3 : public ACLPermission, public XMLObj
   virtual ~ACLPermission_S3() override {}
 
   bool xml_end(const char *el) override;
-  void to_xml(std::ostream& out);
 };
 
 class ACLGrantee_S3 : public XMLObj
@@ -41,7 +40,6 @@ class ACLGrant_S3 : public ACLGrant, public XMLObj
   ACLGrant_S3() {}
   virtual ~ACLGrant_S3() override {}
 
-  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
   bool xml_end(const char *el) override;
   bool xml_start(const char *el, const char **attr);
 
@@ -53,7 +51,6 @@ class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
 {
 public:
   bool xml_end(const char *el) override;
-  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
 };
 
 class ACLOwner_S3 : public ACLOwner, public XMLObj
@@ -63,7 +60,6 @@ class ACLOwner_S3 : public ACLOwner, public XMLObj
   virtual ~ACLOwner_S3() override {}
 
   bool xml_end(const char *el) override;
-  void to_xml(std::ostream& out);
 };
 
 class RGWEnv;
@@ -73,7 +69,6 @@ class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
 public:
   bool xml_end(const char *el) override;
 
-  void to_xml(const DoutPrefixProvider* dpp, std::ostream& out);
   int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
 	      RGWAccessControlPolicy& dest, std::string &err_msg);
 };
@@ -93,6 +88,10 @@ class RGWACLXMLParser_S3 : public RGWXMLParser
 
 namespace rgw::s3 {
 
+/// Write an AccessControlPolicy xml document for the given policy.
+void write_policy_xml(const RGWAccessControlPolicy& policy,
+                      std::ostream& out);
+
 /// Construct a policy from a s3 canned acl string.
 int create_canned_acl(const ACLOwner& owner,
                       const ACLOwner& bucket_owner,
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index cbc13d593f12..f822c13932df 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -230,8 +230,7 @@ static int decode_policy(const DoutPrefixProvider *dpp,
   }
   if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(dpp, 15) << __func__ << " Read AccessControlPolicy";
-    RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
-    s3policy->to_xml(dpp, *_dout);
+    rgw::s3::write_policy_xml(*policy, *_dout);
     *_dout << dendl;
   }
   return 0;
@@ -1640,9 +1639,8 @@ int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map
     return -EIO;
   }
   if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
-    RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
     ldpp_dout(dpp, 15) << __func__ << " Read AccessControlPolicy";
-    s3policy->to_xml(dpp, *_dout);
+    rgw::s3::write_policy_xml(*policy, *_dout);
     *_dout << dendl;
   }
   return 0;
@@ -5889,11 +5887,11 @@ void RGWGetACLs::pre_exec()
 void RGWGetACLs::execute(optional_yield y)
 {
   stringstream ss;
-  RGWAccessControlPolicy* const acl = \
-    (!rgw::sal::Object::empty(s->object.get()) ? s->object_acl.get() : s->bucket_acl.get());
-  RGWAccessControlPolicy_S3* const s3policy = \
-    static_cast<RGWAccessControlPolicy_S3*>(acl);
-  s3policy->to_xml(this, ss);
+  if (rgw::sal::Object::empty(s->object.get())) {
+    rgw::s3::write_policy_xml(*s->bucket_acl, ss);
+  } else {
+    rgw::s3::write_policy_xml(*s->object_acl, ss);
+  }
   acls = ss.str();
 }
 
@@ -6080,7 +6078,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "Old AccessControlPolicy";
-    policy->to_xml(this, *_dout);
+    rgw::s3::write_policy_xml(*policy, *_dout);
     *_dout << dendl;
   }
 
@@ -6090,7 +6088,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "New AccessControlPolicy:";
-    new_policy.to_xml(this, *_dout);
+    rgw::s3::write_policy_xml(new_policy, *_dout);
     *_dout << dendl;
   }
 
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index a5956def0d25..73f16e449e99 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3638,7 +3638,7 @@ int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::Driver* driver,
   if (r < 0)
     return r;
 
-  s3policy.to_xml(this, ss);
+  rgw::s3::write_policy_xml(s3policy, ss);
 
   return 0;
 }

From d28c05cb77d0e8ed8a76429ffbb2e6f7a575f588 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 17 Nov 2023 21:29:25 -0500
Subject: [PATCH 0812/2492] rgw/acl/s3: parse_policy() as free function

s3 acl parsing classes no longer inherit from the acl classes
themselves, and are all encapsulated in rgw_acl_s3.cc behind a single
rgw::s3::parse_policy() function

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_bucket.cc |   2 +-
 src/rgw/rgw_acl.cc                 |   4 +-
 src/rgw/rgw_acl.h                  |   2 +-
 src/rgw/rgw_acl_s3.cc              | 339 +++++++++++++++++------------
 src/rgw/rgw_acl_s3.h               |  81 +------
 src/rgw/rgw_op.cc                  |  62 ++----
 src/rgw/rgw_op.h                   |   5 +-
 src/rgw/rgw_rest_s3.cc             |  50 +----
 src/rgw/rgw_rest_s3.h              |   3 +-
 9 files changed, 241 insertions(+), 307 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 6cbcbeae9a2b..64d8e31cc3cc 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -972,7 +972,7 @@ int RGWBucketAdminOp::get_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState
 int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Driver* driver, RGWBucketAdminOpState& op_state,
                   ostream& os, const DoutPrefixProvider *dpp, optional_yield y)
 {
-  RGWAccessControlPolicy_S3 policy;
+  RGWAccessControlPolicy policy;
 
   int ret = get_policy(driver, op_state, policy, dpp, y);
   if (ret < 0)
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index 6e3aafe762ad..316af76ecf05 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -419,9 +419,9 @@ void RGWAccessControlPolicy::dump(Formatter *f) const
   encode_json("owner", owner, f);
 }
 
-ACLGroupTypeEnum ACLGrant::uri_to_group(string& uri)
+ACLGroupTypeEnum ACLGrant::uri_to_group(std::string_view uri)
 {
   // this is required for backward compatibility
-  return ACLGrant_S3::uri_to_group(uri);
+  return rgw::s3::acl_uri_to_group(uri);
 }
 
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index c69ee88404e3..afa63d8cdfb3 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -111,7 +111,7 @@ class ACLGrant
   void dump(Formatter *f) const;
   static void generate_test_instances(std::list<ACLGrant*>& o);
 
-  ACLGroupTypeEnum uri_to_group(std::string& uri);
+  static ACLGroupTypeEnum uri_to_group(std::string_view uri);
 
   void set_canon(const rgw_user& _id, const std::string& _name, const uint32_t perm) {
     type.set(ACL_TYPE_CANON_USER);
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index f663e3830921..f4e4221f2c0b 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -25,6 +25,14 @@ using namespace std;
 static string rgw_uri_all_users = RGW_URI_ALL_USERS;
 static string rgw_uri_auth_users = RGW_URI_AUTH_USERS;
 
+class ACLPermission_S3 : public XMLObj
+{
+public:
+  uint32_t flags = 0;
+
+  bool xml_end(const char *el) override;
+};
+
 void to_xml(ACLPermission perm, std::ostream& out)
 {
   const uint32_t flags = perm.get_permissions();
@@ -42,8 +50,7 @@ void to_xml(ACLPermission perm, std::ostream& out)
   }
 }
 
-bool ACLPermission_S3::
-xml_end(const char *el)
+bool ACLPermission_S3::xml_end(const char *el)
 {
   const char *s = data.c_str();
   if (strcasecmp(s, "READ") == 0) {
@@ -97,6 +104,15 @@ class ACLGranteeType_S3 {
   }
 };
 
+class ACLGrantee_S3 : public XMLObj
+{
+public:
+  ACLGrantee_S3() {}
+  virtual ~ACLGrantee_S3() override {}
+
+  bool xml_start(const char *el, const char **attr);
+};
+
 class ACLID_S3 : public XMLObj
 {
 public:
@@ -126,6 +142,15 @@ class ACLDisplayName_S3 : public XMLObj
  ~ACLDisplayName_S3() override {}
 };
 
+class ACLOwner_S3 : public XMLObj
+{
+public:
+  std::string id;
+  std::string display_name;
+
+  bool xml_end(const char *el) override;
+};
+
 bool ACLOwner_S3::xml_end(const char *el) {
   ACLID_S3 *acl_id = static_cast<ACLID_S3 *>(find_first("ID"));
   ACLID_S3 *acl_name = static_cast<ACLID_S3 *>(find_first("DisplayName"));
@@ -156,12 +181,25 @@ void to_xml(const ACLOwner& o, std::ostream& out)
   out << "</Owner>";
 }
 
+class ACLGrant_S3 : public XMLObj
+{
+public:
+  ACLGranteeType type;
+  std::string id;
+  std::string name;
+  std::string uri;
+  std::string email;
+  ACLPermission_S3* permission = nullptr;
+
+  bool xml_end(const char *el) override;
+  bool xml_start(const char *el, const char **attr);
+};
+
 bool ACLGrant_S3::xml_end(const char *el) {
   ACLGrantee_S3 *acl_grantee;
   ACLID_S3 *acl_id;
   ACLURI_S3 *acl_uri;
   ACLEmail_S3 *acl_email;
-  ACLPermission_S3 *acl_permission;
   ACLDisplayName_S3 *acl_name;
   string uri;
 
@@ -171,17 +209,12 @@ bool ACLGrant_S3::xml_end(const char *el) {
   string type_str;
   if (!acl_grantee->get_attr("xsi:type", type_str))
     return false;
-  ACLGranteeType_S3::set(type_str.c_str(), type);
-  
-  acl_permission = static_cast<ACLPermission_S3 *>(find_first("Permission"));
-  if (!acl_permission)
-    return false;
 
-  permission = *acl_permission;
+  ACLGranteeType_S3::set(type_str.c_str(), type);
 
-  id.clear();
-  name.clear();
-  email.clear();
+  permission = static_cast<ACLPermission_S3*>(find_first("Permission"));
+  if (!permission)
+    return false;
 
   switch (type.get_type()) {
   case ACL_TYPE_CANON_USER:
@@ -198,7 +231,6 @@ bool ACLGrant_S3::xml_end(const char *el) {
     if (!acl_uri)
       return false;
     uri = acl_uri->get_data();
-    group = uri_to_group(uri);
     break;
   case ACL_TYPE_EMAIL_USER:
     acl_email = static_cast<ACLEmail_S3 *>(acl_grantee->find_first("EmailAddress"));
@@ -236,7 +268,7 @@ void to_xml(const ACLGrant& grant, ostream& out)
     out << "<EmailAddress>" << grant.email << "</EmailAddress>";
     break;
   case ACL_TYPE_GROUP:
-    if (!ACLGrant_S3::group_to_uri(grant.group, uri)) {
+    if (!rgw::s3::acl_group_to_uri(grant.group, uri)) {
       break;
     }
     out << "<URI>" << uri << "</URI>";
@@ -249,27 +281,13 @@ void to_xml(const ACLGrant& grant, ostream& out)
   out << "</Grant>";
 }
 
-bool ACLGrant_S3::group_to_uri(ACLGroupTypeEnum group, string& uri)
+class RGWAccessControlList_S3 : public XMLObj
 {
-  switch (group) {
-  case ACL_GROUP_ALL_USERS:
-    uri = rgw_uri_all_users;
-    return true;
-  case ACL_GROUP_AUTHENTICATED_USERS:
-    uri = rgw_uri_auth_users;
-    return true;
-  default:
-    return false;
-  }
-}
+public:
+  bool xml_end(const char *el) override;
+};
 
 bool RGWAccessControlList_S3::xml_end(const char *el) {
-  XMLObjIter iter = find("Grant");
-  ACLGrant_S3 *grant = static_cast<ACLGrant_S3 *>(iter.get_next());
-  while (grant) {
-    add_grant(*grant);
-    grant = static_cast<ACLGrant_S3 *>(iter.get_next());
-  }
   return true;
 }
 
@@ -318,7 +336,7 @@ static int parse_grantee_str(const DoutPrefixProvider* dpp,
 
     grant.set_canon(user->get_id(), user->get_display_name(), rgw_perm);
   } else if (strcasecmp(id_type.c_str(), "uri") == 0) {
-    ACLGroupTypeEnum gid = grant.uri_to_group(id_val);
+    ACLGroupTypeEnum gid = rgw::s3::acl_uri_to_group(id_val);
     if (gid == ACL_GROUP_NONE)
       return -EINVAL;
 
@@ -401,18 +419,21 @@ static int create_canned(const ACLOwner& owner, const ACLOwner& bucket_owner,
   return 0;
 }
 
+class RGWAccessControlPolicy_S3 : public XMLObj
+{
+public:
+  bool xml_end(const char *el) override;
+};
+
 bool RGWAccessControlPolicy_S3::xml_end(const char *el) {
   RGWAccessControlList_S3 *s3acl =
       static_cast<RGWAccessControlList_S3 *>(find_first("AccessControlList"));
   if (!s3acl)
     return false;
 
-  acl = *s3acl;
-
-  ACLOwner *owner_p = static_cast<ACLOwner_S3 *>(find_first("Owner"));
+  ACLOwner_S3 *owner_p = static_cast<ACLOwner_S3 *>(find_first("Owner"));
   if (!owner_p)
     return false;
-  owner = *owner_p;
   return true;
 }
 
@@ -433,117 +454,68 @@ static const s3_acl_header acl_header_perms[] = {
   {0, NULL}
 };
 
-/*
-  can only be called on object that was parsed
- */
-int RGWAccessControlPolicy_S3::rebuild(const DoutPrefixProvider *dpp,
-				       rgw::sal::Driver* driver, ACLOwner *owner,
-				       RGWAccessControlPolicy& dest, std::string &err_msg)
+static int resolve_grant(const DoutPrefixProvider* dpp, optional_yield y,
+                         rgw::sal::Driver* driver, ACLGrant_S3& xml_grant,
+                         ACLGrant& grant, std::string& err_msg)
 {
-  if (!owner || owner->id.empty()) {
-    return -EINVAL;
-  }
+  const uint32_t perm = xml_grant.permission->flags;
 
-  ACLOwner *requested_owner = static_cast<ACLOwner_S3 *>(find_first("Owner"));
-  if (requested_owner && requested_owner->id != owner->id) {
-    return -EPERM;
-  }
+  std::unique_ptr<rgw::sal::User> user;
+  switch (xml_grant.type.get_type()) {
+  case ACL_TYPE_EMAIL_USER:
+    if (xml_grant.email.empty()) {
+      return -EINVAL;
+    }
+    if (driver->get_user_by_email(dpp, xml_grant.email, y, &user) < 0) {
+      ldpp_dout(dpp, 10) << "grant user email not found or other error" << dendl;
+      err_msg = "The e-mail address you provided does not match any account on record.";
+      return -ERR_UNRESOLVABLE_EMAIL;
+    }
+    grant.set_canon(user->get_id(), user->get_display_name(), perm);
+    return 0;
 
-  std::unique_ptr<rgw::sal::User> user = driver->get_user(owner->id);
-  if (user->load_user(dpp, null_yield) < 0) {
-    ldpp_dout(dpp, 10) << "owner info does not exist" << dendl;
-    err_msg = "Invalid id";
-    return -EINVAL;
-  }
-  ACLOwner& dest_owner = dest.get_owner();
-  dest_owner.id = owner->id;
-  dest_owner.display_name = user->get_display_name();
-
-  ldpp_dout(dpp, 20) << "owner id=" << owner->id << dendl;
-  ldpp_dout(dpp, 20) << "dest owner id=" << dest.get_owner().id << dendl;
-
-  RGWAccessControlList& dst_acl = dest.get_acl();
-
-  multimap<string, ACLGrant>& grant_map = acl.get_grant_map();
-  multimap<string, ACLGrant>::iterator iter;
-  for (iter = grant_map.begin(); iter != grant_map.end(); ++iter) {
-    ACLGrant& src_grant = iter->second;
-    ACLGranteeType type = src_grant.get_type();
-    ACLGrant new_grant;
-    bool grant_ok = false;
-    rgw_user uid;
-    RGWUserInfo grant_user;
-    switch (type.get_type()) {
-    case ACL_TYPE_EMAIL_USER:
-      {
-        string email;
-        rgw_user u;
-        if (!src_grant.get_id(u)) {
-          ldpp_dout(dpp, 0) << "ERROR: src_grant.get_id() failed" << dendl;
-          return -EINVAL;
-        }
-        email = u.id;
-        ldpp_dout(dpp, 10) << "grant user email=" << email << dendl;
-	if (driver->get_user_by_email(dpp, email, null_yield, &user) < 0) {
-          ldpp_dout(dpp, 10) << "grant user email not found or other error" << dendl;
-          err_msg = "The e-mail address you provided does not match any account on record.";
-          return -ERR_UNRESOLVABLE_EMAIL;
-        }
-	grant_user = user->get_info();
-        uid = grant_user.user_id;
-      }
-    case ACL_TYPE_CANON_USER:
-      {
-        if (type.get_type() == ACL_TYPE_CANON_USER) {
-          if (!src_grant.get_id(uid)) {
-            ldpp_dout(dpp, 0) << "ERROR: src_grant.get_id() failed" << dendl;
-            err_msg = "Invalid id";
-            return -EINVAL;
-          }
-        }
-    
-        if (grant_user.user_id.empty()) {
-	  user = driver->get_user(uid);
-	  if (user->load_user(dpp, null_yield) < 0) {
-	    ldpp_dout(dpp, 10) << "grant user does not exist:" << uid << dendl;
-	    err_msg = "Invalid id";
-	    return -EINVAL;
-	  } else {
-	    grant_user = user->get_info();
-	  }
-        }
-	ACLPermission perm = src_grant.get_permission();
-	new_grant.set_canon(uid, grant_user.display_name, perm.get_permissions());
-	grant_ok = true;
-	rgw_user new_id;
-	new_grant.get_id(new_id);
-	ldpp_dout(dpp, 10) << "new grant: " << new_id << ":" << grant_user.display_name << dendl;
-      }
-      break;
-    case ACL_TYPE_GROUP:
-      {
-        string uri;
-        if (ACLGrant_S3::group_to_uri(src_grant.get_group(), uri)) {
-          new_grant = src_grant;
-          grant_ok = true;
-          ldpp_dout(dpp, 10) << "new grant: " << uri << dendl;
-        } else {
-          ldpp_dout(dpp, 10) << "bad grant group:" << (int)src_grant.get_group() << dendl;
-          err_msg = "Invalid group uri";
-          return -EINVAL;
-        }
-      }
-    default:
-      break;
+  case ACL_TYPE_CANON_USER:
+    user = driver->get_user(rgw_user{xml_grant.id});
+    if (user->load_user(dpp, y) < 0) {
+      ldpp_dout(dpp, 10) << "grant user does not exist: " << xml_grant.id << dendl;
+      err_msg = "Invalid CanonicalUser id";
+      return -EINVAL;
     }
-    if (grant_ok) {
-      dst_acl.add_grant(new_grant);
+    grant.set_canon(user->get_id(), user->get_display_name(), perm);
+    return 0;
+
+  case ACL_TYPE_GROUP:
+    if (const auto group = rgw::s3::acl_uri_to_group(xml_grant.uri);
+        group != ACL_GROUP_NONE) {
+      grant.set_group(group, perm);
+      return 0;
+    } else {
+      ldpp_dout(dpp, 10) << "bad grant group: " << xml_grant.uri << dendl;
+      err_msg = "Invalid group uri";
+      return -EINVAL;
     }
-  }
 
-  return 0; 
+  case ACL_TYPE_REFERER:
+  case ACL_TYPE_UNKNOWN:
+  default:
+    err_msg = "Invalid Grantee type";
+    return -EINVAL;
+  }
 }
 
+/**
+ * Interfaces with the webserver's XML handling code
+ * to parse it in a way that makes sense for the rgw.
+ */
+class RGWACLXMLParser_S3 : public RGWXMLParser
+{
+  CephContext *cct;
+
+  XMLObj *alloc_obj(const char *el) override;
+public:
+  explicit RGWACLXMLParser_S3(CephContext *_cct) : cct(_cct) {}
+};
+
 XMLObj *RGWACLXMLParser_S3::alloc_obj(const char *el)
 {
   XMLObj * obj = NULL;
@@ -572,18 +544,97 @@ XMLObj *RGWACLXMLParser_S3::alloc_obj(const char *el)
   return obj;
 }
 
-ACLGroupTypeEnum ACLGrant_S3::uri_to_group(string& uri)
+namespace rgw::s3 {
+
+ACLGroupTypeEnum acl_uri_to_group(std::string_view uri)
 {
-  if (uri.compare(rgw_uri_all_users) == 0)
+  if (uri == rgw_uri_all_users)
     return ACL_GROUP_ALL_USERS;
-  else if (uri.compare(rgw_uri_auth_users) == 0)
+  else if (uri == rgw_uri_auth_users)
     return ACL_GROUP_AUTHENTICATED_USERS;
 
   return ACL_GROUP_NONE;
 }
 
+bool acl_group_to_uri(ACLGroupTypeEnum group, std::string& uri)
+{
+  switch (group) {
+  case ACL_GROUP_ALL_USERS:
+    uri = rgw_uri_all_users;
+    return true;
+  case ACL_GROUP_AUTHENTICATED_USERS:
+    uri = rgw_uri_auth_users;
+    return true;
+  default:
+    return false;
+  }
+}
 
-namespace rgw::s3 {
+int parse_policy(const DoutPrefixProvider* dpp, optional_yield y,
+                 rgw::sal::Driver* driver, std::string_view document,
+                 RGWAccessControlPolicy& policy, std::string& err_msg)
+{
+  RGWACLXMLParser_S3 parser(dpp->get_cct());
+  if (!parser.init()) {
+    return -EINVAL;
+  }
+  if (!parser.parse(document.data(), document.size(), 1)) {
+    return -EINVAL;
+  }
+
+  const auto xml_root = static_cast<RGWAccessControlPolicy_S3*>(
+      parser.find_first("AccessControlPolicy"));
+  if (!xml_root) {
+    err_msg = "Missing element AccessControlPolicy";
+    return -EINVAL;
+  }
+
+  const auto xml_owner = static_cast<ACLOwner_S3*>(
+      xml_root->find_first("Owner"));
+  if (!xml_owner) {
+    err_msg = "Missing element Owner";
+    return -EINVAL;
+  }
+
+  // owner must exist
+  std::unique_ptr<rgw::sal::User> user =
+      driver->get_user(rgw_user{xml_owner->id});
+  if (user->load_user(dpp, y) < 0) {
+    ldpp_dout(dpp, 10) << "acl owner does not exist" << dendl;
+    err_msg = "Invalid Owner ID";
+    return -EINVAL;
+  }
+
+  ACLOwner& owner = policy.get_owner();
+  owner.id = xml_owner->id;
+  if (!xml_owner->display_name.empty()) {
+    owner.display_name = xml_owner->display_name;
+  } else {
+    owner.display_name = user->get_display_name();
+  }
+
+  const auto xml_acl = static_cast<ACLOwner_S3*>(
+      xml_root->find_first("AccessControlList"));
+  if (!xml_acl) {
+    err_msg = "Missing element AccessControlList";
+    return -EINVAL;
+  }
+
+  // iterate parsed grants
+  XMLObjIter iter = xml_acl->find("Grant");
+  ACLGrant_S3* xml_grant = static_cast<ACLGrant_S3*>(iter.get_next());
+  while (xml_grant) {
+    ACLGrant grant;
+    int r = resolve_grant(dpp, y, driver, *xml_grant, grant, err_msg);
+    if (r < 0) {
+      return r;
+    }
+    policy.get_acl().add_grant(grant);
+    xml_grant = static_cast<ACLGrant_S3*>(iter.get_next());
+  }
+
+  return 0;
+}
 
 void write_policy_xml(const RGWAccessControlPolicy& policy,
                       std::ostream& out)
diff --git a/src/rgw/rgw_acl_s3.h b/src/rgw/rgw_acl_s3.h
index 7b8489299715..2341461783fd 100644
--- a/src/rgw/rgw_acl_s3.h
+++ b/src/rgw/rgw_acl_s3.h
@@ -8,85 +8,24 @@
 #include <iosfwd>
 #include <include/types.h>
 
-#include "include/str_list.h"
+#include "common/async/yield_context.h"
 #include "rgw_xml.h"
 #include "rgw_acl.h"
 #include "rgw_sal_fwd.h"
 
-class RGWUserCtl;
-
-class ACLPermission_S3 : public ACLPermission, public XMLObj
-{
-public:
-  ACLPermission_S3() {}
-  virtual ~ACLPermission_S3() override {}
-
-  bool xml_end(const char *el) override;
-};
-
-class ACLGrantee_S3 : public XMLObj
-{
-public:
-  ACLGrantee_S3() {}
-  virtual ~ACLGrantee_S3() override {}
-
-  bool xml_start(const char *el, const char **attr);
-};
-
-
-class ACLGrant_S3 : public ACLGrant, public XMLObj
-{
-public:
-  ACLGrant_S3() {}
-  virtual ~ACLGrant_S3() override {}
-
-  bool xml_end(const char *el) override;
-  bool xml_start(const char *el, const char **attr);
-
-  static ACLGroupTypeEnum uri_to_group(std::string& uri);
-  static bool group_to_uri(ACLGroupTypeEnum group, std::string& uri);
-};
-
-class RGWAccessControlList_S3 : public RGWAccessControlList, public XMLObj
-{
-public:
-  bool xml_end(const char *el) override;
-};
-
-class ACLOwner_S3 : public ACLOwner, public XMLObj
-{
-public:
-  ACLOwner_S3() {}
-  virtual ~ACLOwner_S3() override {}
-
-  bool xml_end(const char *el) override;
-};
-
 class RGWEnv;
 
-class RGWAccessControlPolicy_S3 : public RGWAccessControlPolicy, public XMLObj
-{
-public:
-  bool xml_end(const char *el) override;
-
-  int rebuild(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, ACLOwner *owner,
-	      RGWAccessControlPolicy& dest, std::string &err_msg);
-};
-
-/**
- * Interfaces with the webserver's XML handling code
- * to parse it in a way that makes sense for the rgw.
- */
-class RGWACLXMLParser_S3 : public RGWXMLParser
-{
-  CephContext *cct;
+namespace rgw::s3 {
 
-  XMLObj *alloc_obj(const char *el) override;
-public:
-  explicit RGWACLXMLParser_S3(CephContext *_cct) : cct(_cct) {}
-};
+ACLGroupTypeEnum acl_uri_to_group(std::string_view uri);
+bool acl_group_to_uri(ACLGroupTypeEnum group, std::string& uri);
 
-namespace rgw::s3 {
+/// Construct a policy from an AccessControlPolicy xml document. Email grantees
+/// are looked up and converted to a corresponding CanonicalUser grant. All user
+/// ids are verified to exist.
+int parse_policy(const DoutPrefixProvider* dpp, optional_yield y,
+                 rgw::sal::Driver* driver, std::string_view document,
+                 RGWAccessControlPolicy& policy, std::string& err_msg);
 
 /// Write an AccessControlPolicy xml document for the given policy.
 void write_policy_xml(const RGWAccessControlPolicy& policy,
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index f822c13932df..8a0d19526ad5 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -512,7 +512,7 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   }
 
   if(s->dialect.compare("s3") == 0) {
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy_S3>();
+    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
   } else if(s->dialect.compare("swift")  == 0) {
     /* We aren't allocating the account policy for those operations using
      * the Swift's infrastructure that don't really need req_state::user.
@@ -5982,25 +5982,10 @@ void RGWDeleteLC::pre_exec()
 
 void RGWPutACLs::execute(optional_yield y)
 {
-  bufferlist bl;
-
-  RGWAccessControlPolicy_S3 *policy = NULL;
-  RGWACLXMLParser_S3 parser(s->cct);
-  RGWAccessControlPolicy_S3 new_policy;
-  stringstream ss;
-
-  op_ret = 0; /* XXX redundant? */
-
-  if (!parser.init()) {
-    op_ret = -EINVAL;
-    return;
-  }
-
-
   RGWAccessControlPolicy* const existing_policy = \
     (rgw::sal::Object::empty(s->object.get()) ? s->bucket_acl.get() : s->object_acl.get());
 
-  owner = existing_policy->get_owner();
+  const ACLOwner& existing_owner = existing_policy->get_owner();
 
   op_ret = get_params(y);
   if (op_ret < 0) {
@@ -6023,26 +6008,24 @@ void RGWPutACLs::execute(optional_yield y)
     return;
   }
 
+  RGWAccessControlPolicy new_policy;
   if (!s->canned_acl.empty() || s->has_acl_header) {
-    op_ret = get_policy_from_state(driver, s, ss);
-    if (op_ret < 0)
-      return;
-
-    data.clear();
-    data.append(ss.str());
+    op_ret = get_policy_from_state(existing_owner, new_policy);
+  } else {
+    op_ret = rgw::s3::parse_policy(this, y, driver, {data.c_str(), data.length()},
+                                   new_policy, s->err.message);
   }
-
-  if (!parser.parse(data.c_str(), data.length(), 1)) {
-    op_ret = -EINVAL;
+  if (op_ret < 0)
     return;
-  }
-  policy = static_cast<RGWAccessControlPolicy_S3 *>(parser.find_first("AccessControlPolicy"));
-  if (!policy) {
-    op_ret = -EINVAL;
+
+  if (!existing_owner.id.empty() &&
+      existing_owner.id != new_policy.get_owner().id) {
+    s->err.message = "Cannot modify ACL Owner";
+    op_ret = -EPERM;
     return;
   }
 
-  const RGWAccessControlList& req_acl = policy->get_acl();
+  const RGWAccessControlList& req_acl = new_policy.get_acl();
   const multimap<string, ACLGrant>& req_grant_map = req_acl.get_grant_map();
 #define ACL_GRANTS_MAX_NUM      100
   int max_num = s->cct->_conf->rgw_acl_grants_max_num;
@@ -6063,13 +6046,8 @@ void RGWPutACLs::execute(optional_yield y)
 
   // forward bucket acl requests to meta master zone
   if ((rgw::sal::Object::empty(s->object.get()))) {
-    bufferlist in_data;
-    // include acl data unless it was generated from a canned_acl
-    if (s->canned_acl.empty()) {
-      in_data.append(data);
-    }
     op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(),
-                                           &in_data, nullptr, s->info, y);
+                                           &data, nullptr, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
       return;
@@ -6078,15 +6056,9 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "Old AccessControlPolicy";
-    rgw::s3::write_policy_xml(*policy, *_dout);
+    rgw::s3::write_policy_xml(*existing_policy, *_dout);
     *_dout << dendl;
-  }
-
-  op_ret = policy->rebuild(this, driver, &owner, new_policy, s->err.message);
-  if (op_ret < 0)
-    return;
 
-  if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "New AccessControlPolicy:";
     rgw::s3::write_policy_xml(new_policy, *_dout);
     *_dout << dendl;
@@ -6098,6 +6070,8 @@ void RGWPutACLs::execute(optional_yield y)
     op_ret = -EACCES;
     return;
   }
+
+  bufferlist bl;
   new_policy.encode(bl);
   map<string, bufferlist> attrs;
 
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 70266cbfc667..948b68f54d84 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1576,7 +1576,6 @@ class RGWGetACLs : public RGWOp {
 class RGWPutACLs : public RGWOp {
 protected:
   bufferlist data;
-  ACLOwner owner;
 
 public:
   RGWPutACLs() {}
@@ -1586,7 +1585,8 @@ class RGWPutACLs : public RGWOp {
   void pre_exec() override;
   void execute(optional_yield y) override;
 
-  virtual int get_policy_from_state(rgw::sal::Driver* driver, req_state *s, std::stringstream& ss) { return 0; }
+  virtual int get_policy_from_state(const ACLOwner& owner,
+                                    RGWAccessControlPolicy& p) { return 0; }
   virtual int get_params(optional_yield y) = 0;
   void send_response() override = 0;
   const char* name() const override { return "put_acls"; }
@@ -1636,7 +1636,6 @@ class RGWPutLC : public RGWOp {
   void pre_exec() override;
   void execute(optional_yield y) override;
 
-//  virtual int get_policy_from_state(RGWRados* driver, req_state *s, std::stringstream& ss) { return 0; }
   virtual int get_params(optional_yield y) = 0;
   void send_response() override = 0;
   const char* name() const override { return "put_lifecycle"; }
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 73f16e449e99..ed8e1afa7b77 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -2383,7 +2383,7 @@ void RGWStatBucket_ObjStore_S3::send_response()
 }
 
 static int create_s3_policy(req_state *s, rgw::sal::Driver* driver,
-			    RGWAccessControlPolicy_S3& s3policy,
+			    RGWAccessControlPolicy& policy,
 			    const ACLOwner& owner)
 {
   if (s->has_acl_header) {
@@ -2391,11 +2391,11 @@ static int create_s3_policy(req_state *s, rgw::sal::Driver* driver,
       return -ERR_INVALID_REQUEST;
 
     return rgw::s3::create_policy_from_headers(s, driver, owner,
-                                               *s->info.env, s3policy);
+                                               *s->info.env, policy);
   }
 
   return rgw::s3::create_canned_acl(owner, s->bucket_owner,
-                                    s->canned_acl, s3policy);
+                                    s->canned_acl, policy);
 }
 
 class RGWLocationConstraint : public XMLObj
@@ -2449,7 +2449,6 @@ class RGWCreateBucketParser : public RGWXMLParser
 
 int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y)
 {
-  RGWAccessControlPolicy_S3 s3policy;
   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
 
   int r;
@@ -2458,12 +2457,10 @@ int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y)
     if (r) return r;
   }
 
-  r = create_s3_policy(s, driver, s3policy, s->owner);
+  r = create_s3_policy(s, driver, policy, s->owner);
   if (r < 0)
     return r;
 
-  policy = s3policy;
-
   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 
   int op_ret = 0;
@@ -2591,13 +2588,10 @@ int RGWPutObj_ObjStore_S3::get_params(optional_yield y)
     return ret;
   }
 
-  RGWAccessControlPolicy_S3 s3policy;
-  ret = create_s3_policy(s, driver, s3policy, s->owner);
+  ret = create_s3_policy(s, driver, policy, s->owner);
   if (ret < 0)
     return ret;
 
-  policy = s3policy;
-
   if_match = s->info.env->get("HTTP_IF_MATCH");
   if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
 
@@ -3451,16 +3445,8 @@ void RGWDeleteObj_ObjStore_S3::send_response()
 
 int RGWCopyObj_ObjStore_S3::init_dest_policy()
 {
-  RGWAccessControlPolicy_S3 s3policy;
-
   /* build a policy for the target object */
-  int r = create_s3_policy(s, driver, s3policy, s->owner);
-  if (r < 0)
-    return r;
-
-  dest_policy = s3policy;
-
-  return 0;
+  return create_s3_policy(s, driver, dest_policy, s->owner);
 }
 
 int RGWCopyObj_ObjStore_S3::get_params(optional_yield y)
@@ -3622,25 +3608,16 @@ int RGWPutACLs_ObjStore_S3::get_params(optional_yield y)
   return ret;
 }
 
-int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::Driver* driver,
-						  req_state *s,
-						  stringstream& ss)
+int RGWPutACLs_ObjStore_S3::get_policy_from_state(const ACLOwner& owner,
+                                                  RGWAccessControlPolicy& policy)
 {
-  RGWAccessControlPolicy_S3 s3policy;
-
   // bucket-* canned acls do not apply to bucket
   if (rgw::sal::Object::empty(s->object.get())) {
     if (s->canned_acl.find("bucket") != string::npos)
       s->canned_acl.clear();
   }
 
-  int r = create_s3_policy(s, driver, s3policy, owner);
-  if (r < 0)
-    return r;
-
-  rgw::s3::write_policy_xml(s3policy, ss);
-
-  return 0;
+  return create_s3_policy(s, driver, policy, owner);
 }
 
 void RGWPutACLs_ObjStore_S3::send_response()
@@ -3973,14 +3950,7 @@ int RGWInitMultipart_ObjStore_S3::get_params(optional_yield y)
     return ret;
   }
 
-  RGWAccessControlPolicy_S3 s3policy;
-  ret = create_s3_policy(s, driver, s3policy, s->owner);
-  if (ret < 0)
-    return ret;
-
-  policy = s3policy;
-
-  return 0;
+  return create_s3_policy(s, driver, policy, s->owner);
 }
 
 void RGWInitMultipart_ObjStore_S3::send_response()
diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h
index c73971a5fe67..d15ddaba35ae 100644
--- a/src/rgw/rgw_rest_s3.h
+++ b/src/rgw/rgw_rest_s3.h
@@ -355,7 +355,8 @@ class RGWPutACLs_ObjStore_S3 : public RGWPutACLs_ObjStore {
   RGWPutACLs_ObjStore_S3() {}
   ~RGWPutACLs_ObjStore_S3() override {}
 
-  int get_policy_from_state(rgw::sal::Driver* driver, req_state *s, std::stringstream& ss) override;
+  int get_policy_from_state(const ACLOwner& owner,
+                            RGWAccessControlPolicy& p) override;
   void send_response() override;
   int get_params(optional_yield y) override;
 };

From ac4253b403cbcbcb4590fee6d6bd41794b45e763 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Nov 2023 10:22:20 -0500
Subject: [PATCH 0813/2492] rgw/acl: req_state stores ACLs by value instead of
 unique_ptr

we no longer rely on polymorphism for the s3/swift variants of
`RGWAccessControlPolicy`, so `req_state` can store `bucket_acl`,
`object_acl` and `user_acl` by value

most functions now take these acls by const- or mutable reference
instead of pointers since they won't be nullptr

some code paths won't initialize some of these bucket/object/user acl
variables, and we rely on `RGWAccessControlPolicy::verify_permissions()`
to return false for those because we won't match an empty owner or
array of grants

in only one case, `verify_user_permissions()` has to return true when
`user_acl` is uninitialized, because S3 doesn't have user acls so
uninitialized user acls should not deny access

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_data_sync.cc |   8 +-
 src/rgw/rgw_acl.cc                    |   8 +-
 src/rgw/rgw_acl.h                     |   9 +-
 src/rgw/rgw_common.cc                 | 134 ++++++++++------------
 src/rgw/rgw_common.h                  |  42 +++----
 src/rgw/rgw_log.cc                    |   3 +-
 src/rgw/rgw_lua_request.cc            |   6 +-
 src/rgw/rgw_op.cc                     | 155 +++++++++++---------------
 src/rgw/rgw_op.h                      |   4 +-
 src/rgw/rgw_rest_swift.cc             |  22 ++--
 10 files changed, 173 insertions(+), 218 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index c754e6064521..0c74f80603ac 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -2725,8 +2725,8 @@ bool RGWUserPermHandler::Bucket::verify_bucket_permission(int perm)
 {
   return verify_bucket_permission_no_policy(sync_env->dpp,
                                             &(*ps),
-                                            &info->user_acl,
-                                            &bucket_acl,
+                                            info->user_acl,
+                                            bucket_acl,
                                             perm);
 }
 
@@ -2742,8 +2742,8 @@ bool RGWUserPermHandler::Bucket::verify_object_permission(const map<string, buff
 
   return verify_bucket_permission_no_policy(sync_env->dpp,
                                             &(*ps),
-                                            &bucket_acl,
-                                            &obj_acl,
+                                            bucket_acl,
+                                            obj_acl,
                                             perm);
 }
 
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index 316af76ecf05..15b2067f2efe 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -120,7 +120,7 @@ void RGWAccessControlList::remove_canon_user_grant(rgw_user& user_id)
 
 uint32_t RGWAccessControlList::get_perm(const DoutPrefixProvider* dpp, 
                                         const rgw::auth::Identity& auth_identity,
-                                        const uint32_t perm_mask)
+                                        const uint32_t perm_mask) const
 {
   ldpp_dout(dpp, 5) << "Searching permissions for identity=" << auth_identity
                 << " mask=" << perm_mask << dendl;
@@ -147,7 +147,7 @@ uint32_t RGWAccessControlList::get_group_perm(const DoutPrefixProvider *dpp,
 uint32_t RGWAccessControlList::get_referer_perm(const DoutPrefixProvider *dpp,
                                                 const uint32_t current_perm,
                                                 const std::string http_referer,
-                                                const uint32_t perm_mask)
+                                                const uint32_t perm_mask) const
 {
   ldpp_dout(dpp, 5) << "Searching permissions for referer=" << http_referer
                 << " mask=" << perm_mask << dendl;
@@ -171,7 +171,7 @@ uint32_t RGWAccessControlPolicy::get_perm(const DoutPrefixProvider* dpp,
                                           const rgw::auth::Identity& auth_identity,
                                           const uint32_t perm_mask,
                                           const char * const http_referer,
-                                          bool ignore_public_acls)
+                                          bool ignore_public_acls) const
 {
   ldpp_dout(dpp, 20) << "-- Getting permissions begin with perm_mask=" << perm_mask
                  << dendl;
@@ -213,7 +213,7 @@ bool RGWAccessControlPolicy::verify_permission(const DoutPrefixProvider* dpp,
                                                const uint32_t user_perm_mask,
                                                const uint32_t perm,
                                                const char * const http_referer,
-                                               bool ignore_public_acls)
+                                               bool ignore_public_acls) const
 {
   uint32_t test_perm = perm | RGW_PERM_READ_OBJS | RGW_PERM_WRITE_OBJS;
 
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index afa63d8cdfb3..52dfa0b66fda 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -31,7 +31,6 @@ class ACLGrant
 
 public:
   ACLGrant() : group(ACL_GROUP_NONE) {}
-  virtual ~ACLGrant() {}
 
   /* there's an assumption here that email/uri/id encodings are
      different and there can't be any overlap */
@@ -230,11 +229,11 @@ class RGWAccessControlList
 public:
   uint32_t get_perm(const DoutPrefixProvider* dpp,
                     const rgw::auth::Identity& auth_identity,
-                    uint32_t perm_mask);
+                    uint32_t perm_mask) const;
   uint32_t get_group_perm(const DoutPrefixProvider *dpp, ACLGroupTypeEnum group, uint32_t perm_mask) const;
   uint32_t get_referer_perm(const DoutPrefixProvider *dpp, uint32_t current_perm,
                             std::string http_referer,
-                            uint32_t perm_mask);
+                            uint32_t perm_mask) const;
   void encode(bufferlist& bl) const {
     ENCODE_START(4, 3, bl);
     bool maps_initialized = true;
@@ -327,13 +326,13 @@ class RGWAccessControlPolicy
                     const rgw::auth::Identity& auth_identity,
                     uint32_t perm_mask,
                     const char * http_referer,
-                    bool ignore_public_acls=false);
+                    bool ignore_public_acls=false) const;
   bool verify_permission(const DoutPrefixProvider* dpp,
                          const rgw::auth::Identity& auth_identity,
                          uint32_t user_perm_mask,
                          uint32_t perm,
                          const char * http_referer = nullptr,
-                         bool ignore_public_acls=false);
+                         bool ignore_public_acls=false) const;
 
   void encode(bufferlist& bl) const {
     ENCODE_START(2, 2, bl);
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 3ab882d35bbc..b16d9d97fa8e 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -1135,7 +1135,7 @@ Effect eval_identity_or_session_policies(const DoutPrefixProvider* dpp,
 
 bool verify_user_permission(const DoutPrefixProvider* dpp,
                             perm_state_base * const s,
-                            RGWAccessControlPolicy * const user_acl,
+                            const RGWAccessControlPolicy& user_acl,
                             const vector<rgw::IAM::Policy>& user_policies,
                             const vector<rgw::IAM::Policy>& session_policies,
                             const rgw::ARN& res,
@@ -1176,20 +1176,20 @@ bool verify_user_permission(const DoutPrefixProvider* dpp,
 
 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
                                       struct perm_state_base * const s,
-                                      RGWAccessControlPolicy * const user_acl,
+                                      const RGWAccessControlPolicy& user_acl,
                                       const int perm)
 {
   if (s->identity->get_identity_type() == TYPE_ROLE)
     return false;
 
-  /* S3 doesn't support account ACLs. */
-  if (!user_acl)
+  /* S3 doesn't support account ACLs, so user_acl will be uninitialized. */
+  if (user_acl.get_owner().id.empty())
     return true;
 
   if ((perm & (int)s->perm_mask) != perm)
     return false;
 
-  return user_acl->verify_permission(dpp, *s->identity, perm, perm);
+  return user_acl.verify_permission(dpp, *s->identity, perm, perm);
 }
 
 bool verify_user_permission(const DoutPrefixProvider* dpp,
@@ -1199,7 +1199,7 @@ bool verify_user_permission(const DoutPrefixProvider* dpp,
                             bool mandatory_policy)
 {
   perm_state_from_req_state ps(s);
-  return verify_user_permission(dpp, &ps, s->user_acl.get(), s->iam_user_policies, s->session_policies, res, op, mandatory_policy);
+  return verify_user_permission(dpp, &ps, s->user_acl, s->iam_user_policies, s->session_policies, res, op, mandatory_policy);
 }
 
 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp, 
@@ -1207,7 +1207,7 @@ bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
                                       const int perm)
 {
   perm_state_from_req_state ps(s);
-  return verify_user_permission_no_policy(dpp, &ps, s->user_acl.get(), perm);
+  return verify_user_permission_no_policy(dpp, &ps, s->user_acl, perm);
 }
 
 bool verify_requester_payer_permission(struct perm_state_base *s)
@@ -1233,8 +1233,8 @@ bool verify_requester_payer_permission(struct perm_state_base *s)
 bool verify_bucket_permission(const DoutPrefixProvider* dpp,
                               struct perm_state_base * const s,
 			      const rgw_bucket& bucket,
-                              RGWAccessControlPolicy * const user_acl,
-                              RGWAccessControlPolicy * const bucket_acl,
+                              const RGWAccessControlPolicy& user_acl,
+                              const RGWAccessControlPolicy& bucket_acl,
 			      const boost::optional<Policy>& bucket_policy,
                               const vector<Policy>& identity_policies,
                               const vector<Policy>& session_policies,
@@ -1292,8 +1292,8 @@ bool verify_bucket_permission(const DoutPrefixProvider* dpp,
 bool verify_bucket_permission(const DoutPrefixProvider* dpp,
                               req_state * const s,
 			      const rgw_bucket& bucket,
-                              RGWAccessControlPolicy * const user_acl,
-                              RGWAccessControlPolicy * const bucket_acl,
+                              const RGWAccessControlPolicy& user_acl,
+                              const RGWAccessControlPolicy& bucket_acl,
 			      const boost::optional<Policy>& bucket_policy,
                               const vector<Policy>& user_policies,
                               const vector<Policy>& session_policies,
@@ -1307,31 +1307,25 @@ bool verify_bucket_permission(const DoutPrefixProvider* dpp,
 }
 
 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp, struct perm_state_base * const s,
-					RGWAccessControlPolicy * const user_acl,
-					RGWAccessControlPolicy * const bucket_acl,
+					const RGWAccessControlPolicy& user_acl,
+					const RGWAccessControlPolicy& bucket_acl,
 					const int perm)
 {
-  if (!bucket_acl)
-    return false;
-
   if ((perm & (int)s->perm_mask) != perm)
     return false;
 
-  if (bucket_acl->verify_permission(dpp, *s->identity, perm, perm,
-                                    s->get_referer(),
-                                    s->bucket_access_conf &&
-                                    s->bucket_access_conf->ignore_public_acls()))
+  if (bucket_acl.verify_permission(dpp, *s->identity, perm, perm,
+                                   s->get_referer(),
+                                   s->bucket_access_conf &&
+                                   s->bucket_access_conf->ignore_public_acls()))
     return true;
 
-  if (!user_acl)
-    return false;
-
-  return user_acl->verify_permission(dpp, *s->identity, perm, perm);
+  return user_acl.verify_permission(dpp, *s->identity, perm, perm);
 }
 
 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp, req_state * const s,
-					RGWAccessControlPolicy * const user_acl,
-					RGWAccessControlPolicy * const bucket_acl,
+					const RGWAccessControlPolicy& user_acl,
+					const RGWAccessControlPolicy& bucket_acl,
 					const int perm)
 {
   perm_state_from_req_state ps(s);
@@ -1351,8 +1345,8 @@ bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp, req_state
 
   return verify_bucket_permission_no_policy(dpp,
                                             &ps,
-                                            s->user_acl.get(),
-                                            s->bucket_acl.get(),
+                                            s->user_acl,
+                                            s->bucket_acl,
                                             perm);
 }
 
@@ -1368,8 +1362,8 @@ bool verify_bucket_permission(const DoutPrefixProvider* dpp, req_state * const s
   return verify_bucket_permission(dpp, 
                                   &ps,
                                   s->bucket->get_key(),
-                                  s->user_acl.get(),
-                                  s->bucket_acl.get(),
+                                  s->user_acl,
+                                  s->bucket_acl,
                                   s->iam_policy,
                                   s->iam_user_policies,
                                   s->session_policies,
@@ -1432,8 +1426,8 @@ int verify_bucket_owner_or_policy(req_state* const s,
 static inline bool check_deferred_bucket_perms(const DoutPrefixProvider* dpp,
                                                struct perm_state_base * const s,
 					       const rgw_bucket& bucket,
-					       RGWAccessControlPolicy * const user_acl,
-					       RGWAccessControlPolicy * const bucket_acl,
+					       const RGWAccessControlPolicy& user_acl,
+					       const RGWAccessControlPolicy& bucket_acl,
 					       const boost::optional<Policy>& bucket_policy,
                  const vector<Policy>& identity_policies,
                  const vector<Policy>& session_policies,
@@ -1446,8 +1440,8 @@ static inline bool check_deferred_bucket_perms(const DoutPrefixProvider* dpp,
 
 static inline bool check_deferred_bucket_only_acl(const DoutPrefixProvider* dpp,
                                                   struct perm_state_base * const s,
-						  RGWAccessControlPolicy * const user_acl,
-						  RGWAccessControlPolicy * const bucket_acl,
+						  const RGWAccessControlPolicy& user_acl,
+						  const RGWAccessControlPolicy& bucket_acl,
 						  const uint8_t deferred_check,
 						  const int perm)
 {
@@ -1457,9 +1451,9 @@ static inline bool check_deferred_bucket_only_acl(const DoutPrefixProvider* dpp,
 
 bool verify_object_permission(const DoutPrefixProvider* dpp, struct perm_state_base * const s,
 			      const rgw_obj& obj,
-                              RGWAccessControlPolicy * const user_acl,
-                              RGWAccessControlPolicy * const bucket_acl,
-                              RGWAccessControlPolicy * const object_acl,
+                              const RGWAccessControlPolicy& user_acl,
+                              const RGWAccessControlPolicy& bucket_acl,
+                              const RGWAccessControlPolicy& object_acl,
                               const boost::optional<Policy>& bucket_policy,
                               const vector<Policy>& identity_policies,
                               const vector<Policy>& session_policies,
@@ -1512,14 +1506,10 @@ bool verify_object_permission(const DoutPrefixProvider* dpp, struct perm_state_b
     return true;
   }
 
-  if (!object_acl) {
-    return false;
-  }
-
-  bool ret = object_acl->verify_permission(dpp, *s->identity, s->perm_mask, perm,
-					   nullptr, /* http_referrer */
-					   s->bucket_access_conf &&
-					   s->bucket_access_conf->ignore_public_acls());
+  bool ret = object_acl.verify_permission(dpp, *s->identity, s->perm_mask, perm,
+					  nullptr, /* http_referrer */
+					  s->bucket_access_conf &&
+					  s->bucket_access_conf->ignore_public_acls());
   if (ret) {
     return true;
   }
@@ -1541,21 +1531,18 @@ bool verify_object_permission(const DoutPrefixProvider* dpp, struct perm_state_b
 
   /* we already verified the user mask above, so we pass swift_perm as the mask here,
      otherwise the mask might not cover the swift permissions bits */
-  if (bucket_acl->verify_permission(dpp, *s->identity, swift_perm, swift_perm,
-                                    s->get_referer()))
+  if (bucket_acl.verify_permission(dpp, *s->identity, swift_perm, swift_perm,
+                                   s->get_referer()))
     return true;
 
-  if (!user_acl)
-    return false;
-
-  return user_acl->verify_permission(dpp, *s->identity, swift_perm, swift_perm);
+  return user_acl.verify_permission(dpp, *s->identity, swift_perm, swift_perm);
 }
 
 bool verify_object_permission(const DoutPrefixProvider* dpp, req_state * const s,
 			      const rgw_obj& obj,
-                              RGWAccessControlPolicy * const user_acl,
-                              RGWAccessControlPolicy * const bucket_acl,
-                              RGWAccessControlPolicy * const object_acl,
+                              const RGWAccessControlPolicy& user_acl,
+                              const RGWAccessControlPolicy& bucket_acl,
+                              const RGWAccessControlPolicy& object_acl,
                               const boost::optional<Policy>& bucket_policy,
                               const vector<Policy>& identity_policies,
                               const vector<Policy>& session_policies,
@@ -1570,9 +1557,9 @@ bool verify_object_permission(const DoutPrefixProvider* dpp, req_state * const s
 
 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
                                         struct perm_state_base * const s,
-					RGWAccessControlPolicy * const user_acl,
-					RGWAccessControlPolicy * const bucket_acl,
-					RGWAccessControlPolicy * const object_acl,
+					const RGWAccessControlPolicy& user_acl,
+					const RGWAccessControlPolicy& bucket_acl,
+					const RGWAccessControlPolicy& object_acl,
 					const int perm)
 {
   if (check_deferred_bucket_only_acl(dpp, s, user_acl, bucket_acl, RGW_DEFER_TO_BUCKET_ACLS_RECURSE, perm) ||
@@ -1580,14 +1567,10 @@ bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
     return true;
   }
 
-  if (!object_acl) {
-    return false;
-  }
-
-  bool ret = object_acl->verify_permission(dpp, *s->identity, s->perm_mask, perm,
-					   nullptr, /* http referrer */
-					   s->bucket_access_conf &&
-					   s->bucket_access_conf->ignore_public_acls());
+  bool ret = object_acl.verify_permission(dpp, *s->identity, s->perm_mask, perm,
+					  nullptr, /* http referrer */
+					  s->bucket_access_conf &&
+					  s->bucket_access_conf->ignore_public_acls());
   if (ret) {
     return true;
   }
@@ -1609,14 +1592,11 @@ bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
 
   /* we already verified the user mask above, so we pass swift_perm as the mask here,
      otherwise the mask might not cover the swift permissions bits */
-  if (bucket_acl->verify_permission(dpp, *s->identity, swift_perm, swift_perm,
-                                    s->get_referer()))
+  if (bucket_acl.verify_permission(dpp, *s->identity, swift_perm, swift_perm,
+                                   s->get_referer()))
     return true;
 
-  if (!user_acl)
-    return false;
-
-  return user_acl->verify_permission(dpp, *s->identity, swift_perm, swift_perm);
+  return user_acl.verify_permission(dpp, *s->identity, swift_perm, swift_perm);
 }
 
 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, req_state *s, int perm)
@@ -1628,9 +1608,9 @@ bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, req_state
 
   return verify_object_permission_no_policy(dpp,
                                             &ps,
-                                            s->user_acl.get(),
-                                            s->bucket_acl.get(),
-                                            s->object_acl.get(),
+                                            s->user_acl,
+                                            s->bucket_acl,
+                                            s->object_acl,
                                             perm);
 }
 
@@ -1641,9 +1621,9 @@ bool verify_object_permission(const DoutPrefixProvider* dpp, req_state *s, uint6
   return verify_object_permission(dpp,
                                   &ps,
                                   rgw_obj(s->bucket->get_key(), s->object->get_key()),
-                                  s->user_acl.get(),
-                                  s->bucket_acl.get(),
-                                  s->object_acl.get(),
+                                  s->user_acl,
+                                  s->bucket_acl,
+                                  s->object_acl,
                                   s->iam_policy,
                                   s->iam_user_policies,
                                   s->session_policies,
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 15d2e800fdee..e69737669298 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1180,9 +1180,9 @@ struct req_state : DoutPrefixProvider {
     } s3_postobj_creds;
   } auth;
 
-  std::unique_ptr<RGWAccessControlPolicy> user_acl;
-  std::unique_ptr<RGWAccessControlPolicy> bucket_acl;
-  std::unique_ptr<RGWAccessControlPolicy> object_acl;
+  RGWAccessControlPolicy user_acl;
+  RGWAccessControlPolicy bucket_acl;
+  RGWAccessControlPolicy object_acl;
 
   rgw::IAM::Environment env;
   boost::optional<rgw::IAM::Policy> iam_policy;
@@ -1584,20 +1584,20 @@ struct perm_state : public perm_state_base {
 bool verify_bucket_permission_no_policy(
   const DoutPrefixProvider* dpp,
   struct perm_state_base * const s,
-  RGWAccessControlPolicy * const user_acl,
-  RGWAccessControlPolicy * const bucket_acl,
+  const RGWAccessControlPolicy& user_acl,
+  const RGWAccessControlPolicy& bucket_acl,
   const int perm);
 
 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
                                       struct perm_state_base * const s,
-                                      RGWAccessControlPolicy * const user_acl,
+                                      const RGWAccessControlPolicy& user_acl,
                                       const int perm);
 
 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
                                         struct perm_state_base * const s,
-					RGWAccessControlPolicy * const user_acl,
-					RGWAccessControlPolicy * const bucket_acl,
-					RGWAccessControlPolicy * const object_acl,
+					const RGWAccessControlPolicy& user_acl,
+					const RGWAccessControlPolicy& bucket_acl,
+					const RGWAccessControlPolicy& object_acl,
 					const int perm);
 
 /** Check if the req_state's user has the necessary permissions
@@ -1609,7 +1609,7 @@ rgw::IAM::Effect eval_identity_or_session_policies(const DoutPrefixProvider* dpp
                           const rgw::ARN& arn);
 bool verify_user_permission(const DoutPrefixProvider* dpp,
                             req_state * const s,
-                            RGWAccessControlPolicy * const user_acl,
+                            const RGWAccessControlPolicy& user_acl,
                             const std::vector<rgw::IAM::Policy>& user_policies,
                             const std::vector<rgw::IAM::Policy>& session_policies,
                             const rgw::ARN& res,
@@ -1617,7 +1617,7 @@ bool verify_user_permission(const DoutPrefixProvider* dpp,
                             bool mandatory_policy=true);
 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
                                       req_state * const s,
-                                      RGWAccessControlPolicy * const user_acl,
+                                      const RGWAccessControlPolicy& user_acl,
                                       const int perm);
 bool verify_user_permission(const DoutPrefixProvider* dpp,
                             req_state * const s,
@@ -1631,8 +1631,8 @@ bool verify_bucket_permission(
   const DoutPrefixProvider* dpp,
   req_state * const s,
   const rgw_bucket& bucket,
-  RGWAccessControlPolicy * const user_acl,
-  RGWAccessControlPolicy * const bucket_acl,
+  const RGWAccessControlPolicy& user_acl,
+  const RGWAccessControlPolicy& bucket_acl,
   const boost::optional<rgw::IAM::Policy>& bucket_policy,
   const std::vector<rgw::IAM::Policy>& identity_policies,
   const std::vector<rgw::IAM::Policy>& session_policies,
@@ -1641,8 +1641,8 @@ bool verify_bucket_permission(const DoutPrefixProvider* dpp, req_state * const s
 bool verify_bucket_permission_no_policy(
   const DoutPrefixProvider* dpp,
   req_state * const s,
-  RGWAccessControlPolicy * const user_acl,
-  RGWAccessControlPolicy * const bucket_acl,
+  const RGWAccessControlPolicy& user_acl,
+  const RGWAccessControlPolicy& bucket_acl,
   const int perm);
 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp,
                                         req_state * const s,
@@ -1653,9 +1653,9 @@ extern bool verify_object_permission(
   const DoutPrefixProvider* dpp,
   req_state * const s,
   const rgw_obj& obj,
-  RGWAccessControlPolicy * const user_acl,
-  RGWAccessControlPolicy * const bucket_acl,
-  RGWAccessControlPolicy * const object_acl,
+  const RGWAccessControlPolicy& user_acl,
+  const RGWAccessControlPolicy& bucket_acl,
+  const RGWAccessControlPolicy& object_acl,
   const boost::optional<rgw::IAM::Policy>& bucket_policy,
   const std::vector<rgw::IAM::Policy>& identity_policies,
   const std::vector<rgw::IAM::Policy>& session_policies,
@@ -1664,9 +1664,9 @@ extern bool verify_object_permission(const DoutPrefixProvider* dpp, req_state *s
 extern bool verify_object_permission_no_policy(
   const DoutPrefixProvider* dpp,
   req_state * const s,
-  RGWAccessControlPolicy * const user_acl,
-  RGWAccessControlPolicy * const bucket_acl,
-  RGWAccessControlPolicy * const object_acl,
+  const RGWAccessControlPolicy& user_acl,
+  const RGWAccessControlPolicy& bucket_acl,
+  const RGWAccessControlPolicy& object_acl,
   int perm);
 extern bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, req_state *s,
 					       int perm);
diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 8b8f85ef07ba..cc7e085ab726 100644
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -647,8 +647,7 @@ int rgw_log_op(RGWREST* const rest, req_state *s, const RGWOp* op, OpsLogSink *o
   }
 
   entry.user = s->user->get_id().to_str();
-  if (s->object_acl)
-    entry.object_owner = s->object_acl->get_owner().id;
+  entry.object_owner = s->object_acl.get_owner().id;
   entry.bucket_owner = s->bucket_owner.id;
 
   uint64_t bytes_sent = ACCOUNTING_IO(s)->get_bytes_sent();
diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc
index afef18be47bf..1fa44a0329ec 100644
--- a/src/rgw/rgw_lua_request.cc
+++ b/src/rgw/rgw_lua_request.cc
@@ -711,11 +711,11 @@ struct RequestMetaTable : public EmptyMetaTable {
     } else if (strcasecmp(index, "ZoneGroup") == 0) {
       create_metatable<ZoneGroupMetaTable>(L, name, index, false, s);
     } else if (strcasecmp(index, "UserACL") == 0) {
-      create_metatable<ACLMetaTable>(L, name, index, false, s->user_acl);
+      create_metatable<ACLMetaTable>(L, name, index, false, &s->user_acl);
     } else if (strcasecmp(index, "BucketACL") == 0) {
-      create_metatable<ACLMetaTable>(L, name, index, false, s->bucket_acl);
+      create_metatable<ACLMetaTable>(L, name, index, false, &s->bucket_acl);
     } else if (strcasecmp(index, "ObjectACL") == 0) {
-      create_metatable<ACLMetaTable>(L, name, index, false, s->object_acl);
+      create_metatable<ACLMetaTable>(L, name, index, false, &s->object_acl);
     } else if (strcasecmp(index, "Environment") == 0) {
         create_metatable<StringMapMetaTable<rgw::IAM::Environment>>(L, name, index, false, &(s->env));
     } else if (strcasecmp(index, "Policy") == 0) {
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8a0d19526ad5..39767f53989e 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -219,18 +219,18 @@ int RGWGetObj::parse_range(void)
 static int decode_policy(const DoutPrefixProvider *dpp,
                          CephContext *cct,
                          bufferlist& bl,
-                         RGWAccessControlPolicy *policy)
+                         RGWAccessControlPolicy& policy)
 {
   auto iter = bl.cbegin();
   try {
-    policy->decode(iter);
+    policy.decode(iter);
   } catch (buffer::error& err) {
     ldpp_dout(dpp, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
     return -EIO;
   }
   if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(dpp, 15) << __func__ << " Read AccessControlPolicy";
-    rgw::s3::write_policy_xml(*policy, *_dout);
+    rgw::s3::write_policy_xml(policy, *_dout);
     *_dout << dendl;
   }
   return 0;
@@ -242,17 +242,11 @@ static int get_user_policy_from_attr(const DoutPrefixProvider *dpp,
 				     map<string, bufferlist>& attrs,
 				     RGWAccessControlPolicy& policy    /* out */)
 {
-  auto aiter = attrs.find(RGW_ATTR_ACL);
-  if (aiter != attrs.end()) {
-    int ret = decode_policy(dpp, cct, aiter->second, &policy);
-    if (ret < 0) {
-      return ret;
-    }
-  } else {
+  auto i = attrs.find(RGW_ATTR_ACL);
+  if (i == attrs.end()) {
     return -ENOENT;
   }
-
-  return 0;
+  return decode_policy(dpp, cct, i->second, policy);
 }
 
 /**
@@ -267,7 +261,7 @@ int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
 				       rgw::sal::Driver* driver,
 				       const rgw_user& bucket_owner,
 				       map<string, bufferlist>& bucket_attrs,
-				       RGWAccessControlPolicy *policy,
+				       RGWAccessControlPolicy& policy,
 				       optional_yield y)
 {
   map<string, bufferlist>::iterator aiter = bucket_attrs.find(RGW_ATTR_ACL);
@@ -284,7 +278,7 @@ int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
     if (r < 0)
       return r;
 
-    policy->create_default(user->get_id(), user->get_display_name());
+    policy.create_default(user->get_id(), user->get_display_name());
   }
   return 0;
 }
@@ -294,7 +288,7 @@ static int get_obj_policy_from_attr(const DoutPrefixProvider *dpp,
 				    rgw::sal::Driver* driver,
 				    RGWBucketInfo& bucket_info,
 				    map<string, bufferlist>& bucket_attrs,
-				    RGWAccessControlPolicy *policy,
+				    RGWAccessControlPolicy& policy,
                                     string *storage_class,
 				    rgw::sal::Object* obj,
                                     optional_yield y)
@@ -317,7 +311,7 @@ static int get_obj_policy_from_attr(const DoutPrefixProvider *dpp,
     if (ret < 0)
       return ret;
 
-    policy->create_default(bucket_info.owner, user->get_display_name());
+    policy.create_default(bucket_info.owner, user->get_display_name());
   }
 
   if (storage_class) {
@@ -384,7 +378,7 @@ static int read_bucket_policy(const DoutPrefixProvider *dpp,
                               req_state *s,
                               RGWBucketInfo& bucket_info,
                               map<string, bufferlist>& bucket_attrs,
-                              RGWAccessControlPolicy *policy,
+                              RGWAccessControlPolicy& policy,
                               rgw_bucket& bucket,
 			      optional_yield y)
 {
@@ -401,7 +395,7 @@ static int read_bucket_policy(const DoutPrefixProvider *dpp,
   int ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info.owner,
                                                bucket_attrs, policy, y);
   if (ret == -ENOENT) {
-      ret = -ERR_NO_SUCH_BUCKET;
+    ret = -ERR_NO_SUCH_BUCKET;
   }
 
   return ret;
@@ -412,7 +406,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
                            req_state *s,
                            RGWBucketInfo& bucket_info,
                            map<string, bufferlist>& bucket_attrs,
-                           RGWAccessControlPolicy* acl,
+                           RGWAccessControlPolicy& acl,
                            string *storage_class,
                            boost::optional<Policy>& policy,
                            rgw::sal::Bucket* bucket,
@@ -451,7 +445,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
        that we send a proper error code */
     RGWAccessControlPolicy bucket_policy;
     ret = rgw_op_get_bucket_policy_from_attr(dpp, s->cct, driver, bucket_info.owner,
-                                             bucket_attrs, &bucket_policy, y);
+                                             bucket_attrs, bucket_policy, y);
     if (ret < 0) {
       return ret;
     }
@@ -511,20 +505,6 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
     }
   }
 
-  if(s->dialect.compare("s3") == 0) {
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
-  } else if(s->dialect.compare("swift")  == 0) {
-    /* We aren't allocating the account policy for those operations using
-     * the Swift's infrastructure that don't really need req_state::user.
-     * Typical example here is the implementation of /info. */
-    if (!s->user->get_id().empty()) {
-      s->user_acl = std::make_unique<RGWAccessControlPolicy>();
-    }
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
-  } else {
-    s->bucket_acl = std::make_unique<RGWAccessControlPolicy>();
-  }
-
   const RGWZoneGroup& zonegroup = s->penv.site->get_zonegroup();
 
   /* check if copy source is within the current domain */
@@ -573,13 +553,13 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
     s->bucket_attrs = s->bucket->get_attrs();
     ret = read_bucket_policy(dpp, driver, s, s->bucket->get_info(),
 			     s->bucket->get_attrs(),
-			     s->bucket_acl.get(), s->bucket->get_key(), y);
+			     s->bucket_acl, s->bucket->get_key(), y);
     acct_acl_user = {
       s->bucket->get_info().owner,
-      s->bucket_acl->get_owner().display_name,
+      s->bucket_acl.get_owner().display_name,
     };
 
-    s->bucket_owner = s->bucket_acl->get_owner();
+    s->bucket_owner = s->bucket_acl.get_owner();
 
     s->zonegroup_endpoint = rgw::get_zonegroup_endpoint(zonegroup);
     s->zonegroup_name = zonegroup.get_name();
@@ -615,12 +595,12 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
   }
 
   /* handle user ACL only for those APIs which support it */
-  if (s->user_acl) {
+  if (s->dialect == "swift" && !s->user->get_id().empty()) {
     std::unique_ptr<rgw::sal::User> acl_user = driver->get_user(acct_acl_user.uid);
 
     ret = acl_user->read_attrs(dpp, y);
     if (!ret) {
-      ret = get_user_policy_from_attr(dpp, s->cct, acl_user->get_attrs(), *s->user_acl);
+      ret = get_user_policy_from_attr(dpp, s->cct, acl_user->get_attrs(), s->user_acl);
     }
     if (-ENOENT == ret) {
       /* In already existing clusters users won't have ACL. In such case
@@ -630,8 +610,8 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
        *  1. if we try to reach an existing bucket, its owner is considered
        *     as account owner.
        *  2. otherwise account owner is identity stored in s->user->user_id.  */
-      s->user_acl->create_default(acct_acl_user.uid,
-                                  acct_acl_user.display_name);
+      s->user_acl.create_default(acct_acl_user.uid,
+                                 acct_acl_user.display_name);
       ret = 0;
     } else if (ret < 0) {
       ldpp_dout(dpp, 0) << "NOTICE: couldn't get user attrs for handling ACL "
@@ -689,24 +669,21 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 int rgw_build_object_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver,
 			      req_state *s, bool prefetch_data, optional_yield y)
 {
-  int ret = 0;
-
-  if (!rgw::sal::Object::empty(s->object.get())) {
-    if (!s->bucket_exists) {
-      return -ERR_NO_SUCH_BUCKET;
-    }
-    s->object_acl = std::make_unique<RGWAccessControlPolicy>();
+  if (rgw::sal::Object::empty(s->object)) {
+    return 0;
+  }
+  if (!s->bucket_exists) {
+    return -ERR_NO_SUCH_BUCKET;
+  }
 
-    s->object->set_atomic();
-    if (prefetch_data) {
-      s->object->set_prefetch_data();
-    }
-    ret = read_obj_policy(dpp, driver, s, s->bucket->get_info(), s->bucket_attrs,
-			  s->object_acl.get(), nullptr, s->iam_policy, s->bucket.get(),
-                          s->object.get(), y);
+  s->object->set_atomic();
+  if (prefetch_data) {
+    s->object->set_prefetch_data();
   }
 
-  return ret;
+  return read_obj_policy(dpp, driver, s, s->bucket->get_info(), s->bucket_attrs,
+                         s->object_acl, nullptr, s->iam_policy, s->bucket.get(),
+                         s->object.get(), y);
 }
 
 static int rgw_iam_remove_objtags(const DoutPrefixProvider *dpp, req_state* s, rgw::sal::Object* object, bool has_existing_obj_tag, bool has_resource_tag) {
@@ -1648,7 +1625,7 @@ int rgw_policy_from_attrset(const DoutPrefixProvider *dpp, CephContext *cct, map
 
 int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
                                        const rgw_bucket_dir_entry& ent,
-                                       RGWAccessControlPolicy * const bucket_acl,
+                                       const RGWAccessControlPolicy& bucket_acl,
                                        const boost::optional<Policy>& bucket_policy,
                                        const off_t start_ofs,
                                        const off_t end_ofs,
@@ -1724,8 +1701,8 @@ int RGWGetObj::read_user_manifest_part(rgw::sal::Bucket* bucket,
     ldpp_dout(this, 2) << "overriding permissions due to system operation" << dendl;
   } else if (s->auth.identity->is_admin_of(s->user->get_id())) {
     ldpp_dout(this, 2) << "overriding permissions due to admin operation" << dendl;
-  } else if (!verify_object_permission(this, s, part->get_obj(), s->user_acl.get(),
-				       bucket_acl, &obj_policy, bucket_policy,
+  } else if (!verify_object_permission(this, s, part->get_obj(), s->user_acl,
+				       bucket_acl, obj_policy, bucket_policy,
 				       s->iam_user_policies, s->session_policies, action)) {
     return -EPERM;
   }
@@ -1749,14 +1726,14 @@ static int iterate_user_manifest_parts(const DoutPrefixProvider *dpp,
                                        const off_t end,
                                        rgw::sal::Bucket* bucket,
                                        const string& obj_prefix,
-                                       RGWAccessControlPolicy * const bucket_acl,
+                                       const RGWAccessControlPolicy& bucket_acl,
                                        const boost::optional<Policy>& bucket_policy,
                                        uint64_t * const ptotal_len,
                                        uint64_t * const pobj_size,
                                        string * const pobj_sum,
                                        int (*cb)(rgw::sal::Bucket* bucket,
                                                  const rgw_bucket_dir_entry& ent,
-                                                 RGWAccessControlPolicy * const bucket_acl,
+                                                 const RGWAccessControlPolicy& bucket_acl,
                                                  const boost::optional<Policy>& bucket_policy,
                                                  off_t start_ofs,
                                                  off_t end_ofs,
@@ -1858,7 +1835,7 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
                              map<uint64_t, rgw_slo_part>& slo_parts,
                              int (*cb)(rgw::sal::Bucket* bucket,
                                        const rgw_bucket_dir_entry& ent,
-                                       RGWAccessControlPolicy *bucket_acl,
+                                       const RGWAccessControlPolicy& bucket_acl,
                                        const boost::optional<Policy>& bucket_policy,
                                        off_t start_ofs,
                                        off_t end_ofs,
@@ -1917,7 +1894,7 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
                           << dendl;
 
 	// SLO is a Swift thing, and Swift has no knowledge of S3 Policies.
-        int r = cb(part.bucket, ent, part.bucket_acl,
+        int r = cb(part.bucket, ent, *part.bucket_acl,
 		   (part.bucket_policy ?
 		    boost::optional<Policy>(*part.bucket_policy) : none),
 		   start_ofs, end_ofs, cb_param, true /* swift_slo */);
@@ -1934,7 +1911,7 @@ static int iterate_slo_parts(const DoutPrefixProvider *dpp,
 
 static int get_obj_user_manifest_iterate_cb(rgw::sal::Bucket* bucket,
                                             const rgw_bucket_dir_entry& ent,
-                                            RGWAccessControlPolicy * const bucket_acl,
+                                            const RGWAccessControlPolicy& bucket_acl,
                                             const boost::optional<Policy>& bucket_policy,
                                             const off_t start_ofs,
                                             const off_t end_ofs,
@@ -1979,7 +1956,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
       return r;
     }
     bucket_acl = &_bucket_acl;
-    r = read_bucket_policy(this, driver, s, ubucket->get_info(), bucket_attrs, bucket_acl, ubucket->get_key(), y);
+    r = read_bucket_policy(this, driver, s, ubucket->get_info(), bucket_attrs, *bucket_acl, ubucket->get_key(), y);
     if (r < 0) {
       ldpp_dout(this, 0) << "failed to read bucket policy" << dendl;
       return r;
@@ -1989,7 +1966,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
     pbucket = ubucket.get();
   } else {
     pbucket = s->bucket.get();
-    bucket_acl = s->bucket_acl.get();
+    bucket_acl = &s->bucket_acl;
     bucket_policy = &s->iam_policy;
   }
 
@@ -1998,7 +1975,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
    * - overall DLO's content size,
    * - md5 sum of overall DLO's content (for etag of Swift API). */
   r = iterate_user_manifest_parts(this, s->cct, driver, ofs, end,
-        pbucket, obj_prefix, bucket_acl, *bucket_policy,
+        pbucket, obj_prefix, *bucket_acl, *bucket_policy,
         nullptr, &s->obj_size, &lo_etag,
 	nullptr /* cb */, nullptr /* cb arg */, y);
   if (r < 0) {
@@ -2012,7 +1989,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
   }
 
   r = iterate_user_manifest_parts(this, s->cct, driver, ofs, end,
-        pbucket, obj_prefix, bucket_acl, *bucket_policy,
+        pbucket, obj_prefix, *bucket_acl, *bucket_policy,
         &total_len, nullptr, nullptr,
 	nullptr, nullptr, y);
   if (r < 0) {
@@ -2026,7 +2003,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
   }
 
   r = iterate_user_manifest_parts(this, s->cct, driver, ofs, end,
-        pbucket, obj_prefix, bucket_acl, *bucket_policy,
+        pbucket, obj_prefix, *bucket_acl, *bucket_policy,
         nullptr, nullptr, nullptr,
 	get_obj_user_manifest_iterate_cb, (void *)this, y);
   if (r < 0) {
@@ -2111,7 +2088,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
         }
         bucket = tmp_bucket.get();
         bucket_acl = &_bucket_acl;
-        r = read_bucket_policy(this, driver, s, tmp_bucket->get_info(), tmp_bucket->get_attrs(), bucket_acl,
+        r = read_bucket_policy(this, driver, s, tmp_bucket->get_info(), tmp_bucket->get_attrs(), *bucket_acl,
                                tmp_bucket->get_key(), y);
         if (r < 0) {
           ldpp_dout(this, 0) << "failed to read bucket ACL for bucket "
@@ -2126,7 +2103,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
       }
     } else {
       bucket = s->bucket.get();
-      bucket_acl = s->bucket_acl.get();
+      bucket_acl = &s->bucket_acl;
       bucket_policy = s->iam_policy.get_ptr();
     }
 
@@ -3516,7 +3493,7 @@ void RGWCreateBucket::execute(optional_yield y)
     RGWAccessControlPolicy old_policy;
     int r = rgw_op_get_bucket_policy_from_attr(this, s->cct, driver, info.owner,
                                                s->bucket->get_attrs(),
-                                               &old_policy, y);
+                                               old_policy, y);
     if (r >= 0 && old_policy != policy) {
       s->err.message = "Cannot modify existing access control policy";
       op_ret = -EEXIST;
@@ -3850,7 +3827,7 @@ int RGWPutObj::verify_permission(optional_yield y)
     cs_object->set_prefetch_data();
 
     /* check source object permissions */
-    int ret = read_obj_policy(this, driver, s, copy_source_bucket_info, cs_attrs, &cs_acl, nullptr,
+    int ret = read_obj_policy(this, driver, s, copy_source_bucket_info, cs_attrs, cs_acl, nullptr,
                               policy, cs_bucket.get(), cs_object.get(), y, true);
     if (ret < 0) {
       return ret;
@@ -4986,7 +4963,7 @@ void RGWPutMetadataBucket::execute(optional_yield y)
        * contain such keys yet. */
       if (has_policy) {
 	if (s->dialect.compare("swift") == 0) {
-	  rgw::swift::merge_policy(policy_rw_mask, *s->bucket_acl, policy);
+	  rgw::swift::merge_policy(policy_rw_mask, s->bucket_acl, policy);
 	}
 	buffer::list bl;
 	policy.encode(bl);
@@ -5488,7 +5465,7 @@ int RGWCopyObj::verify_permission(optional_yield y)
     rgw_placement_rule src_placement;
 
     /* check source object permissions */
-    op_ret = read_obj_policy(this, driver, s, src_bucket->get_info(), src_bucket->get_attrs(), &src_acl, &src_placement.storage_class,
+    op_ret = read_obj_policy(this, driver, s, src_bucket->get_info(), src_bucket->get_attrs(), src_acl, &src_placement.storage_class,
 			     src_policy, src_bucket.get(), s->src_object.get(), y);
     if (op_ret < 0) {
       return op_ret;
@@ -5583,7 +5560,7 @@ int RGWCopyObj::verify_permission(optional_yield y)
   /* check dest bucket permissions */
   op_ret = read_bucket_policy(this, driver, s, s->bucket->get_info(),
 			      s->bucket->get_attrs(),
-                              &dest_bucket_policy, s->bucket->get_key(), y);
+                              dest_bucket_policy, s->bucket->get_key(), y);
   if (op_ret < 0) {
     return op_ret;
   }
@@ -5888,9 +5865,9 @@ void RGWGetACLs::execute(optional_yield y)
 {
   stringstream ss;
   if (rgw::sal::Object::empty(s->object.get())) {
-    rgw::s3::write_policy_xml(*s->bucket_acl, ss);
+    rgw::s3::write_policy_xml(s->bucket_acl, ss);
   } else {
-    rgw::s3::write_policy_xml(*s->object_acl, ss);
+    rgw::s3::write_policy_xml(s->object_acl, ss);
   }
   acls = ss.str();
 }
@@ -5982,10 +5959,10 @@ void RGWDeleteLC::pre_exec()
 
 void RGWPutACLs::execute(optional_yield y)
 {
-  RGWAccessControlPolicy* const existing_policy = \
-    (rgw::sal::Object::empty(s->object.get()) ? s->bucket_acl.get() : s->object_acl.get());
+  const RGWAccessControlPolicy& existing_policy = \
+    (rgw::sal::Object::empty(s->object.get()) ? s->bucket_acl : s->object_acl);
 
-  const ACLOwner& existing_owner = existing_policy->get_owner();
+  const ACLOwner& existing_owner = existing_policy.get_owner();
 
   op_ret = get_params(y);
   if (op_ret < 0) {
@@ -6056,7 +6033,7 @@ void RGWPutACLs::execute(optional_yield y)
 
   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
     ldpp_dout(this, 15) << "Old AccessControlPolicy";
-    rgw::s3::write_policy_xml(*existing_policy, *_dout);
+    rgw::s3::write_policy_xml(existing_policy, *_dout);
     *_dout << dendl;
 
     ldpp_dout(this, 15) << "New AccessControlPolicy:";
@@ -7376,7 +7353,7 @@ bool RGWBulkDelete::Deleter::verify_permission(RGWBucketInfo& binfo,
 					       optional_yield y)
 {
   RGWAccessControlPolicy bacl;
-  int ret = read_bucket_policy(dpp, driver, s, binfo, battrs, &bacl, binfo.bucket, y);
+  int ret = read_bucket_policy(dpp, driver, s, binfo, battrs, bacl, binfo.bucket, y);
   if (ret < 0) {
     return false;
   }
@@ -7387,8 +7364,8 @@ bool RGWBulkDelete::Deleter::verify_permission(RGWBucketInfo& binfo,
 
   /* We can use global user_acl because each BulkDelete request is allowed
    * to work on entities from a single account only. */
-  return verify_bucket_permission(dpp, s, binfo.bucket, s->user_acl.get(),
-				  &bacl, policy, s->iam_user_policies, s->session_policies, rgw::IAM::s3DeleteBucket);
+  return verify_bucket_permission(dpp, s, binfo.bucket, s->user_acl,
+				  bacl, policy, s->iam_user_policies, s->session_policies, rgw::IAM::s3DeleteBucket);
 }
 
 bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yield y)
@@ -7715,7 +7692,7 @@ bool RGWBulkUploadOp::handle_file_verify_permission(RGWBucketInfo& binfo,
 						    optional_yield y)
 {
   RGWAccessControlPolicy bacl;
-  op_ret = read_bucket_policy(this, driver, s, binfo, battrs, &bacl, binfo.bucket, y);
+  op_ret = read_bucket_policy(this, driver, s, binfo, battrs, bacl, binfo.bucket, y);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "cannot read_policy() for bucket" << dendl;
     return false;
@@ -7768,8 +7745,8 @@ bool RGWBulkUploadOp::handle_file_verify_permission(RGWBucketInfo& binfo,
     }
   }
     
-  return verify_bucket_permission_no_policy(this, s, s->user_acl.get(),
-					    &bacl, RGW_PERM_WRITE);
+  return verify_bucket_permission_no_policy(this, s, s->user_acl,
+					    bacl, RGW_PERM_WRITE);
 }
 
 int RGWBulkUploadOp::handle_file(const std::string_view path,
@@ -8880,7 +8857,7 @@ int RGWGetBucketPolicyStatus::verify_permission(optional_yield y)
 
 void RGWGetBucketPolicyStatus::execute(optional_yield y)
 {
-  isPublic = (s->iam_policy && rgw::IAM::is_public(*s->iam_policy)) || s->bucket_acl->is_public(this);
+  isPublic = (s->iam_policy && rgw::IAM::is_public(*s->iam_policy)) || s->bucket_acl.is_public(this);
 }
 
 int RGWPutBucketPublicAccessBlock::verify_permission(optional_yield y)
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 948b68f54d84..9314d454c791 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -79,7 +79,7 @@ int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider *dpp,
                                        rgw::sal::Driver* driver,
                                        const rgw_user& bucket_owner,
                                        std::map<std::string, bufferlist>& bucket_attrs,
-                                       RGWAccessControlPolicy *policy,
+                                       RGWAccessControlPolicy& policy,
                                        optional_yield y);
 
 class RGWHandler {
@@ -420,7 +420,7 @@ class RGWGetObj : public RGWOp {
   int read_user_manifest_part(
     rgw::sal::Bucket* bucket,
     const rgw_bucket_dir_entry& ent,
-    RGWAccessControlPolicy * const bucket_acl,
+    const RGWAccessControlPolicy& bucket_acl,
     const boost::optional<rgw::IAM::Policy>& bucket_policy,
     const off_t start_ofs,
     const off_t end_ofs,
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 55ebac5137c8..46f02805cd1b 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -188,7 +188,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_begin(bool has_buckets)
             s->user->get_attrs(),
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            *s->user_acl);
+            s->user_acl);
     dump_errno(s);
     dump_header(s, "Accept-Ranges", "bytes");
     end_header(s, NULL, NULL, NO_CONTENT_LENGTH, true);
@@ -287,7 +287,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_end()
             s->user->get_attrs(),
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            *s->user_acl);
+            s->user_acl);
     dump_errno(s);
     end_header(s, nullptr, nullptr, s->formatter->get_len(), true);
   }
@@ -471,7 +471,7 @@ static void dump_container_metadata(req_state *s,
 
   if (rgw::sal::Object::empty(s->object.get())) {
     std::string read_acl, write_acl;
-    rgw::swift::format_container_acls(*s->bucket_acl, read_acl, write_acl);
+    rgw::swift::format_container_acls(s->bucket_acl, read_acl, write_acl);
 
     if (read_acl.size()) {
       dump_header(s, "X-Container-Read", read_acl);
@@ -565,7 +565,7 @@ void RGWStatAccount_ObjStore_SWIFT::send_response()
             attrs,
             s->user->get_info().quota.user_quota,
             s->user->get_max_buckets(),
-            *s->user_acl);
+            s->user_acl);
   }
 
   set_req_state_err(s, op_ret);
@@ -593,7 +593,7 @@ void RGWStatBucket_ObjStore_SWIFT::send_response()
 
 static int get_swift_container_settings(req_state * const s,
                                         rgw::sal::Driver*  const driver,
-                                        RGWAccessControlPolicy * const policy,
+                                        RGWAccessControlPolicy& policy,
                                         bool * const has_policy,
                                         uint32_t * rw_mask,
                                         RGWCORSConfiguration * const cors_config,
@@ -611,7 +611,7 @@ static int get_swift_container_settings(req_state * const s,
                                                 read_list,
                                                 write_list,
                                                 *rw_mask,
-                                                *policy);
+                                                policy);
     if (r < 0) {
       return r;
     }
@@ -713,7 +713,7 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params(optional_yield y)
   bool has_policy;
   uint32_t policy_rw_mask = 0;
 
-  int r = get_swift_container_settings(s, driver, &policy, &has_policy,
+  int r = get_swift_container_settings(s, driver, policy, &has_policy,
 				       &policy_rw_mask, &cors_config, &has_cors);
   if (r < 0) {
     return r;
@@ -1057,7 +1057,7 @@ void RGWPutObj_ObjStore_SWIFT::send_response()
 
 static int get_swift_account_settings(req_state * const s,
                                       rgw::sal::Driver*  const driver,
-                                      RGWAccessControlPolicy* const policy,
+                                      RGWAccessControlPolicy& policy,
                                       bool * const has_policy)
 {
   *has_policy = false;
@@ -1067,7 +1067,7 @@ static int get_swift_account_settings(req_state * const s,
     int r = rgw::swift::create_account_policy(s, driver,
                                               s->user->get_id(),
                                               s->user->get_display_name(),
-                                              acl_attr, *policy);
+                                              acl_attr, policy);
     if (r < 0) {
       return r;
     }
@@ -1084,7 +1084,7 @@ int RGWPutMetadataAccount_ObjStore_SWIFT::get_params(optional_yield y)
     return -EINVAL;
   }
 
-  int ret = get_swift_account_settings(s, driver, &policy, &has_policy);
+  int ret = get_swift_account_settings(s, driver, policy, &has_policy);
   if (ret < 0) {
     return ret;
   }
@@ -1117,7 +1117,7 @@ int RGWPutMetadataBucket_ObjStore_SWIFT::get_params(optional_yield y)
     return -EINVAL;
   }
 
-  int r = get_swift_container_settings(s, driver, &policy, &has_policy,
+  int r = get_swift_container_settings(s, driver, policy, &has_policy,
 				       &policy_rw_mask, &cors_config, &has_cors);
   if (r < 0) {
     return r;

From 7a3eb76761c5b12b139ce0072a9311389b46a5f5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sat, 18 Nov 2023 11:27:50 -0500
Subject: [PATCH 0814/2492] rgw/acl: ACLGrant uses variant for grantee types

use of `ACLGrant::get_id()` was awkward because most grantee types
returned nothing, but emails were returned as `struct rgw_user`. change
the internal representation into a variant, and expose getters for each
grantee type so callers can handle each type specifically. the encoded
format of `ACLGrant` remains unchanged

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/lua-scripting.rst              |   6 +-
 src/rgw/driver/rados/rgw_sync_module_es.cc |  11 +-
 src/rgw/rgw_acl.cc                         |  96 +++++-----
 src/rgw/rgw_acl.h                          | 210 ++++++++++++++-------
 src/rgw/rgw_acl_s3.cc                      |  30 ++-
 src/rgw/rgw_acl_swift.cc                   |  64 ++++---
 src/rgw/rgw_acl_types.h                    |   9 +-
 src/rgw/rgw_lua_request.cc                 |  17 +-
 src/rgw/rgw_rest_client.cc                 |  25 +--
 src/test/rgw/test_rgw_lua.cc               |  40 ++--
 10 files changed, 282 insertions(+), 226 deletions(-)

diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst
index 29cba258efec..8f95aea3c14a 100644
--- a/doc/radosgw/lua-scripting.rst
+++ b/doc/radosgw/lua-scripting.rst
@@ -248,15 +248,15 @@ Request Fields
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
 | ``Request.UserAcl.Grants["<name>"].Type``          | integer  | user ACL grant type                                          | no       | no        | no       |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
-| ``Request.UserAcl.Grants["<name>"].User``          | table    | user ACL grant user                                          | no       | no        | no       |
+| ``Request.UserAcl.Grants["<name>"].User``          | table    | user ACL grant user                                          | no       | no        | yes      |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
 | ``Request.UserAcl.Grants["<name>"].User.Tenant``   | table    | user ACL grant user tenant                                   | no       | no        | no       |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
 | ``Request.UserAcl.Grants["<name>"].User.Id``       | table    | user ACL grant user id                                       | no       | no        | no       |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
-| ``Request.UserAcl.Grants["<name>"].GroupType``     | integer  | user ACL grant group type                                    | no       | no        | no       |
+| ``Request.UserAcl.Grants["<name>"].GroupType``     | integer  | user ACL grant group type                                    | no       | no        | yes      |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
-| ``Request.UserAcl.Grants["<name>"].Referer``       | string   | user ACL grant referer                                       | no       | no        | no       |
+| ``Request.UserAcl.Grants["<name>"].Referer``       | string   | user ACL grant referer                                       | no       | no        | yes      |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
 | ``Request.BucketAcl``                              | table    | bucket ACL. See: ``Request.UserAcl``                         | no       | no        | no       |
 +----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
diff --git a/src/rgw/driver/rados/rgw_sync_module_es.cc b/src/rgw/driver/rados/rgw_sync_module_es.cc
index 1030d9dce351..e3353dc1fc7c 100644
--- a/src/rgw/driver/rados/rgw_sync_module_es.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_es.cc
@@ -502,14 +502,11 @@ struct es_obj_metadata {
         const RGWAccessControlList& acl = policy.get_acl();
 
         permissions.insert(policy.get_owner().id.to_str());
-        for (auto acliter : acl.get_grant_map()) {
+        for (const auto& acliter : acl.get_grant_map()) {
           const ACLGrant& grant = acliter.second;
-          if (grant.get_type().get_type() == ACL_TYPE_CANON_USER &&
-              ((uint32_t)grant.get_permission().get_permissions() & RGW_PERM_READ) != 0) {
-            rgw_user user;
-            if (grant.get_id(user)) {
-              permissions.insert(user.to_str());
-            }
+          const auto* user = grant.get_user();
+          if (user && (grant.get_permission().get_permissions() & RGW_PERM_READ) != 0) {
+            permissions.insert(user->id.to_str());
           }
         }
       } else if (attr_name == RGW_ATTR_TAGS) {
diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc
index 15b2067f2efe..4369bedfc76b 100644
--- a/src/rgw/rgw_acl.cc
+++ b/src/rgw/rgw_acl.cc
@@ -33,10 +33,7 @@ bool operator!=(const ACLGranteeType& lhs, const ACLGranteeType& rhs) {
 }
 
 bool operator==(const ACLGrant& lhs, const ACLGrant& rhs) {
-  return lhs.type == rhs.type && lhs.id == rhs.id
-      && lhs.email == rhs.email && lhs.permission == rhs.permission
-      && lhs.name == rhs.name && lhs.group == rhs.group
-      && lhs.url_spec == rhs.url_spec;
+  return lhs.grantee == rhs.grantee && lhs.permission == rhs.permission;
 }
 bool operator!=(const ACLGrant& lhs, const ACLGrant& rhs) {
   return !(lhs == rhs);
@@ -73,49 +70,41 @@ bool operator!=(const RGWAccessControlPolicy& lhs,
 void RGWAccessControlList::register_grant(const ACLGrant& grant)
 {
   ACLPermission perm = grant.get_permission();
-  ACLGranteeType type = grant.get_type();
-  switch (type.get_type()) {
-  case ACL_TYPE_REFERER:
-    referer_list.emplace_back(grant.get_referer(), perm.get_permissions());
+
+  if (const auto* user = grant.get_user(); user) {
+    acl_user_map[user->id.to_str()] |= perm.get_permissions();
+  } else if (const auto* email = grant.get_email(); email) {
+    acl_user_map[email->address] |= perm.get_permissions();
+  } else if (const auto* group = grant.get_group(); group) {
+    acl_group_map[group->type] |= perm.get_permissions();
+  } else if (const auto* referer = grant.get_referer(); referer) {
+    referer_list.emplace_back(referer->url_spec, perm.get_permissions());
 
     /* We're specially handling the Swift's .r:* as the S3 API has a similar
      * concept and thus we can have a small portion of compatibility here. */
-     if (grant.get_referer() == RGW_REFERER_WILDCARD) {
+     if (referer->url_spec == RGW_REFERER_WILDCARD) {
        acl_group_map[ACL_GROUP_ALL_USERS] |= perm.get_permissions();
      }
-    break;
-  case ACL_TYPE_GROUP:
-    acl_group_map[grant.get_group()] |= perm.get_permissions();
-    break;
-  default:
-    {
-      rgw_user id;
-      grant.get_id(id);
-      acl_user_map[id.to_str()] |= perm.get_permissions();
-    }
   }
 }
 
 void RGWAccessControlList::add_grant(const ACLGrant& grant)
 {
-  rgw_user id;
-  grant.get_id(id); // note that this will return false for groups, but that's ok, we won't search groups
-  grant_map.emplace(id.to_str(), grant);
+  std::string id;
+  if (const auto* user = grant.get_user(); user) {
+    id = user->id.to_str();
+  } else if (const auto* email = grant.get_email(); email) {
+    id = email->address;
+  } // other types share the empty key in the grant multimap
+  grant_map.emplace(id, grant);
   register_grant(grant);
 }
 
-void RGWAccessControlList::remove_canon_user_grant(rgw_user& user_id)
+void RGWAccessControlList::remove_canon_user_grant(const rgw_user& user_id)
 {
-  auto multi_map_iter = grant_map.find(user_id.to_str());
-  if(multi_map_iter != grant_map.end()) {
-    auto grants = grant_map.equal_range(user_id.to_str());
-    grant_map.erase(grants.first, grants.second);
-  }
-
-  auto map_iter = acl_user_map.find(user_id.to_str());
-  if (map_iter != acl_user_map.end()){
-    acl_user_map.erase(map_iter);
-  }
+  const std::string& key = user_id.to_str();
+  grant_map.erase(key);
+  acl_user_map.erase(key);
 }
 
 uint32_t RGWAccessControlList::get_perm(const DoutPrefixProvider* dpp, 
@@ -277,31 +266,36 @@ void ACLGranteeType::dump(Formatter *f) const
 void ACLGrant::dump(Formatter *f) const
 {
   f->open_object_section("type");
-  type.dump(f);
+  get_type().dump(f);
   f->close_section();
 
-  f->dump_string("id", id.to_str());
-  f->dump_string("email", email);
+  struct dump_visitor {
+    Formatter* f;
 
-  f->open_object_section("permission");
-  permission.dump(f);
-  f->close_section();
+    void operator()(const ACLGranteeCanonicalUser& user) {
+      encode_json("id", user.id, f);
+      encode_json("name", user.name, f);
+    }
+    void operator()(const ACLGranteeEmailUser& email) {
+      encode_json("email", email.address, f);
+    }
+    void operator()(const ACLGranteeGroup& group) {
+      encode_json("group", static_cast<int>(group.type), f);
+    }
+    void operator()(const ACLGranteeUnknown&) {}
+    void operator()(const ACLGranteeReferer& r) {
+      encode_json("url_spec", r.url_spec, f);
+    }
+  };
+  std::visit(dump_visitor{f}, grantee);
 
-  f->dump_string("name", name);
-  f->dump_int("group", (int)group);
-  f->dump_string("url_spec", url_spec);
+  encode_json("permission", permission, f);
 }
 
 void ACLGrant::generate_test_instances(list<ACLGrant*>& o)
 {
-  rgw_user id("rgw");
-  string name, email;
-  name = "Mr. RGW";
-  email = "r@gw";
-
   ACLGrant *g1 = new ACLGrant;
-  g1->set_canon(id, name, RGW_PERM_READ);
-  g1->email = email;
+  g1->set_canon(rgw_user{"rgw"}, "Mr. RGW", RGW_PERM_READ);
   o.push_back(g1);
 
   ACLGrant *g2 = new ACLGrant;
@@ -313,9 +307,7 @@ void ACLGrant::generate_test_instances(list<ACLGrant*>& o)
 
 void ACLGranteeType::generate_test_instances(list<ACLGranteeType*>& o)
 {
-  ACLGranteeType *t = new ACLGranteeType;
-  t->set(ACL_TYPE_CANON_USER);
-  o.push_back(t);
+  o.push_back(new ACLGranteeType(ACL_TYPE_CANON_USER));
   o.push_back(new ACLGranteeType);
 }
 
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index 52dfa0b66fda..e2887a7049b6 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -6,6 +6,7 @@
 #include <map>
 #include <string>
 #include <string_view>
+#include <variant>
 #include <include/types.h>
 
 #include <boost/optional.hpp>
@@ -15,95 +16,162 @@
 
 #include "rgw_basic_types.h" //includes rgw_acl_types.h
 
-class ACLGrant
-{
-protected:
-  ACLGranteeType type;
+// acl grantee types
+struct ACLGranteeCanonicalUser {
   rgw_user id;
-  std::string email;
-  mutable rgw_user email_id;
-  ACLPermission permission;
   std::string name;
-  ACLGroupTypeEnum group;
+
+  friend auto operator<=>(const ACLGranteeCanonicalUser&,
+                          const ACLGranteeCanonicalUser&) = default;
+};
+struct ACLGranteeEmailUser {
+  std::string address;
+
+  friend auto operator<=>(const ACLGranteeEmailUser&,
+                          const ACLGranteeEmailUser&) = default;
+};
+struct ACLGranteeGroup {
+  ACLGroupTypeEnum type = ACL_GROUP_NONE;
+
+  friend auto operator<=>(const ACLGranteeGroup&,
+                          const ACLGranteeGroup&) = default;
+};
+struct ACLGranteeUnknown {
+  friend auto operator<=>(const ACLGranteeUnknown&,
+                          const ACLGranteeUnknown&) = default;
+};
+struct ACLGranteeReferer {
   std::string url_spec;
 
-  friend void to_xml(const ACLGrant& grant, std::ostream& out);
+  friend auto operator<=>(const ACLGranteeReferer&,
+                          const ACLGranteeReferer&) = default;
+};
+
+class ACLGrant
+{
+protected:
+  // acl grantee variant, where variant index matches ACLGranteeTypeEnum
+  using ACLGrantee = std::variant<
+    ACLGranteeCanonicalUser,
+    ACLGranteeEmailUser,
+    ACLGranteeGroup,
+    ACLGranteeUnknown,
+    ACLGranteeReferer>;
+
+  ACLGrantee grantee;
+  ACLPermission permission;
 
 public:
-  ACLGrant() : group(ACL_GROUP_NONE) {}
-
-  /* there's an assumption here that email/uri/id encodings are
-     different and there can't be any overlap */
-  bool get_id(rgw_user& _id) const {
-    switch(type.get_type()) {
-    case ACL_TYPE_EMAIL_USER:
-      _id = email; // implies from_str() that parses the 't:u' syntax
-      return true;
-    case ACL_TYPE_GROUP:
-    case ACL_TYPE_REFERER:
-      return false;
-    default:
-      _id = id;
-      return true;
-    }
+  ACLGranteeType get_type() const {
+    return static_cast<ACLGranteeTypeEnum>(grantee.index());
   }
+  ACLPermission get_permission() const { return permission; }
 
-  const rgw_user* get_id() const {
-    switch(type.get_type()) {
-    case ACL_TYPE_EMAIL_USER:
-      email_id.from_str(email);
-      return &email_id;
-    case ACL_TYPE_GROUP:
-    case ACL_TYPE_REFERER:
-      return nullptr;
-    default:
-      return &id;
-    }
+  // return the user grantee, or nullptr
+  const ACLGranteeCanonicalUser* get_user() const {
+    return std::get_if<ACLGranteeCanonicalUser>(&grantee);
+  }
+  // return the email grantee, or nullptr
+  const ACLGranteeEmailUser* get_email() const {
+    return std::get_if<ACLGranteeEmailUser>(&grantee);
+  }
+  // return the group grantee, or nullptr
+  const ACLGranteeGroup* get_group() const {
+    return std::get_if<ACLGranteeGroup>(&grantee);
+  }
+  // return the referer grantee, or nullptr
+  const ACLGranteeReferer* get_referer() const {
+    return std::get_if<ACLGranteeReferer>(&grantee);
   }
-
-  ACLGranteeType get_type() const { return type; }
-  ACLPermission get_permission() const { return permission; }
-  ACLGroupTypeEnum get_group() const { return group; }
-  const std::string& get_referer() const { return url_spec; }
 
   void encode(bufferlist& bl) const {
     ENCODE_START(5, 3, bl);
+    ACLGranteeType type = get_type();
     encode(type, bl);
-    std::string s;
-    id.to_str(s);
-    encode(s, bl);
-    std::string uri;
+
+    if (const ACLGranteeCanonicalUser* user = get_user(); user) {
+      encode(user->id.to_str(), bl);
+    } else {
+      encode(std::string{}, bl); // encode empty id
+    }
+
+    std::string uri; // always empty, v2 converted to 'ACLGroupTypeEnum g' below
     encode(uri, bl);
-    encode(email, bl);
+
+    if (const ACLGranteeEmailUser* email = get_email(); email) {
+      encode(email->address, bl);
+    } else {
+      encode(std::string{}, bl); // encode empty email address
+    }
     encode(permission, bl);
-    encode(name, bl);
-    __u32 g = (__u32)group;
+    if (const ACLGranteeCanonicalUser* user = get_user(); user) {
+      encode(user->name, bl);
+    } else {
+      encode(std::string{}, bl); // encode empty name
+    }
+
+    __u32 g;
+    if (const ACLGranteeGroup* group = get_group(); group) {
+      g = static_cast<__u32>(group->type);
+    } else {
+      g = static_cast<__u32>(ACL_GROUP_NONE);
+    }
     encode(g, bl);
-    encode(url_spec, bl);
+
+    if (const ACLGranteeReferer* referer = get_referer(); referer) {
+      encode(referer->url_spec, bl);
+    } else {
+      encode(std::string{}, bl); // encode empty referer
+    }
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
     DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
+    ACLGranteeType type;
     decode(type, bl);
+
+    ACLGranteeCanonicalUser user;
     std::string s;
     decode(s, bl);
-    id.from_str(s);
+    user.id.from_str(s);
+
     std::string uri;
     decode(uri, bl);
-    decode(email, bl);
+
+    ACLGranteeEmailUser email;
+    decode(email.address, bl);
+
     decode(permission, bl);
-    decode(name, bl);
-    if (struct_v > 1) {
-      __u32 g;
-      decode(g, bl);
-      group = (ACLGroupTypeEnum)g;
-    } else {
-      group = uri_to_group(uri);
-    }
+    decode(user.name, bl);
+
+    ACLGranteeGroup group;
+    __u32 g;
+    decode(g, bl);
+    group.type = static_cast<ACLGroupTypeEnum>(g);
+
+    ACLGranteeReferer referer;
     if (struct_v >= 5) {
-      decode(url_spec, bl);
-    } else {
-      url_spec.clear();
+      decode(referer.url_spec, bl);
+    }
+
+    // construct the grantee type
+    switch (type) {
+      case ACL_TYPE_CANON_USER:
+        grantee = std::move(user);
+        break;
+      case ACL_TYPE_EMAIL_USER:
+        grantee = std::move(email);
+        break;
+      case ACL_TYPE_GROUP:
+        grantee = std::move(group);
+        break;
+      case ACL_TYPE_REFERER:
+        grantee = std::move(referer);
+        break;
+      case ACL_TYPE_UNKNOWN:
+      default:
+        grantee = ACLGranteeUnknown{};
+        break;
     }
     DECODE_FINISH(bl);
   }
@@ -112,20 +180,16 @@ class ACLGrant
 
   static ACLGroupTypeEnum uri_to_group(std::string_view uri);
 
-  void set_canon(const rgw_user& _id, const std::string& _name, const uint32_t perm) {
-    type.set(ACL_TYPE_CANON_USER);
-    id = _id;
-    name = _name;
+  void set_canon(const rgw_user& id, const std::string& name, uint32_t perm) {
+    grantee = ACLGranteeCanonicalUser{id, name};
     permission.set_permissions(perm);
   }
-  void set_group(ACLGroupTypeEnum _group, const uint32_t perm) {
-    type.set(ACL_TYPE_GROUP);
-    group = _group;
+  void set_group(ACLGroupTypeEnum group, uint32_t perm) {
+    grantee = ACLGranteeGroup{group};
     permission.set_permissions(perm);
   }
-  void set_referer(const std::string& _url_spec, const uint32_t perm) {
-    type.set(ACL_TYPE_REFERER);
-    url_spec = _url_spec;
+  void set_referer(const std::string& url_spec, uint32_t perm) {
+    grantee = ACLGranteeReferer{url_spec};
     permission.set_permissions(perm);
   }
 
@@ -267,7 +331,7 @@ class RGWAccessControlList
   static void generate_test_instances(std::list<RGWAccessControlList*>& o);
 
   void add_grant(const ACLGrant& grant);
-  void remove_canon_user_grant(rgw_user& user_id);
+  void remove_canon_user_grant(const rgw_user& user_id);
 
   ACLGrantMap& get_grant_map() { return grant_map; }
   const ACLGrantMap& get_grant_map() const { return grant_map; }
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index f4e4221f2c0b..e45fb552aff6 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -253,28 +253,22 @@ void to_xml(const ACLGrant& grant, ostream& out)
   if (!(perm.get_permissions() & RGW_PERM_ALL_S3))
     return;
 
-  string uri;
+  const std::string type = ACLGranteeType_S3::to_string(grant.get_type());
 
   out << "<Grant>" <<
-         "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"" << ACLGranteeType_S3::to_string(grant.type) << "\">";
-  switch (grant.type.get_type()) {
-  case ACL_TYPE_CANON_USER:
-    out << "<ID>" << grant.id << "</ID>";
-    if (grant.name.size()) {
-      out << "<DisplayName>" << grant.name << "</DisplayName>";
-    }
-    break;
-  case ACL_TYPE_EMAIL_USER:
-    out << "<EmailAddress>" << grant.email << "</EmailAddress>";
-    break;
-  case ACL_TYPE_GROUP:
-    if (!rgw::s3::acl_group_to_uri(grant.group, uri)) {
-      break;
+         "<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"" << type << "\">";
+
+  if (const auto* user = grant.get_user(); user) {
+    out << "<ID>" << user->id << "</ID>";
+    if (user->name.size()) {
+      out << "<DisplayName>" << user->name << "</DisplayName>";
     }
+  } else if (const auto* email = grant.get_email(); email) {
+    out << "<EmailAddress>" << email->address << "</EmailAddress>";
+  } else if (const auto* group = grant.get_group(); group) {
+    std::string uri;
+    rgw::s3::acl_group_to_uri(group->type, uri);
     out << "<URI>" << uri << "</URI>";
-    break;
-  default:
-    break;
   }
   out << "</Grantee>";
   to_xml(perm, out);
diff --git a/src/rgw/rgw_acl_swift.cc b/src/rgw/rgw_acl_swift.cc
index 048f7b491521..260f4530d41f 100644
--- a/src/rgw/rgw_acl_swift.cc
+++ b/src/rgw/rgw_acl_swift.cc
@@ -222,16 +222,13 @@ void merge_policy(uint32_t rw_mask, const RGWAccessControlPolicy& src,
   for (const auto &iter: src.get_acl().get_grant_map()) {
     const ACLGrant& grant = iter.second;
     uint32_t perm = grant.get_permission().get_permissions();
-    rgw_user id;
-    if (!grant.get_id(id)) {
-      if (grant.get_group() != ACL_GROUP_ALL_USERS) {
-        if (string url_spec = grant.get_referer(); url_spec.empty()) {
-          continue;
-        }
-        if (perm == 0) {
-          /* We need to carry also negative, HTTP referrer-based ACLs. */
-          perm = SWIFT_PERM_READ;
-        }
+    if (const auto* referer = grant.get_referer(); referer) {
+      if (referer->url_spec.empty()) {
+        continue;
+      }
+      if (perm == 0) {
+        /* We need to carry also negative, HTTP referrer-based ACLs. */
+        perm = SWIFT_PERM_READ;
       }
     }
     if (perm & rw_mask) {
@@ -245,35 +242,37 @@ void format_container_acls(const RGWAccessControlPolicy& policy,
 {
   for (const auto& [k, grant] : policy.get_acl().get_grant_map()) {
     const uint32_t perm = grant.get_permission().get_permissions();
-    rgw_user id;
-    string url_spec;
-    if (!grant.get_id(id)) {
-      if (grant.get_group() == ACL_GROUP_ALL_USERS) {
+    std::string id;
+    std::string url_spec;
+    if (const auto user = grant.get_user(); user) {
+      id = user->id.to_str();
+    } else if (const auto group = grant.get_group(); group) {
+      if (group->type == ACL_GROUP_ALL_USERS) {
         id = SWIFT_GROUP_ALL_USERS;
-      } else {
-        url_spec = grant.get_referer();
-        if (url_spec.empty()) {
-          continue;
-        }
-        id = (perm != 0) ? ".r:" + url_spec : ".r:-" + url_spec;
       }
+    } else if (const auto referer = grant.get_referer(); referer) {
+      url_spec = referer->url_spec;
+      if (url_spec.empty()) {
+        continue;
+      }
+      id = (perm != 0) ? ".r:" + url_spec : ".r:-" + url_spec;
     }
     if (perm & SWIFT_PERM_READ) {
       if (!read.empty()) {
         read.append(",");
       }
-      read.append(id.to_str());
+      read.append(id);
     } else if (perm & SWIFT_PERM_WRITE) {
       if (!write.empty()) {
         write.append(",");
       }
-      write.append(id.to_str());
+      write.append(id);
     } else if (perm == 0 && !url_spec.empty()) {
       /* only X-Container-Read headers support referers */
       if (!read.empty()) {
         read.append(",");
       }
-      read.append(id.to_str());
+      read.append(id);
     }
   }
 }
@@ -338,22 +337,27 @@ auto format_account_acl(const RGWAccessControlPolicy& policy)
     const ACLGrant& grant = item.second;
     const uint32_t perm = grant.get_permission().get_permissions();
 
-    rgw_user id;
-    if (!grant.get_id(id)) {
-      if (grant.get_group() != ACL_GROUP_ALL_USERS) {
+    std::string id;
+    if (const auto user = grant.get_user(); user) {
+      if (owner.id == user->id) {
+        continue;
+      }
+      id = user->id.to_str();
+    } else if (const auto group = grant.get_group(); group) {
+      if (group->type != ACL_GROUP_ALL_USERS) {
         continue;
       }
       id = SWIFT_GROUP_ALL_USERS;
-    } else if (owner.id == id) {
+    } else {
       continue;
     }
 
     if (SWIFT_PERM_ADMIN == (perm & SWIFT_PERM_ADMIN)) {
-      admin.insert(admin.end(), id.to_str());
+      admin.insert(admin.end(), id);
     } else if (SWIFT_PERM_RWRT == (perm & SWIFT_PERM_RWRT)) {
-      readwrite.insert(readwrite.end(), id.to_str());
+      readwrite.insert(readwrite.end(), id);
     } else if (SWIFT_PERM_READ == (perm & SWIFT_PERM_READ)) {
-      readonly.insert(readonly.end(), id.to_str());
+      readonly.insert(readonly.end(), id);
     } else {
       // FIXME: print a warning
     }
diff --git a/src/rgw/rgw_acl_types.h b/src/rgw/rgw_acl_types.h
index a257bb4c9e6d..11637dd360de 100644
--- a/src/rgw/rgw_acl_types.h
+++ b/src/rgw/rgw_acl_types.h
@@ -181,12 +181,15 @@ class ACLGranteeType
 protected:
   __u32 type;
 public:
-  ACLGranteeType() : type(ACL_TYPE_UNKNOWN) {}
+  ACLGranteeType(ACLGranteeTypeEnum t = ACL_TYPE_UNKNOWN) : type(t) {}
   virtual ~ACLGranteeType() {}
-//  virtual const char *to_string() = 0;
+
   ACLGranteeTypeEnum get_type() const { return (ACLGranteeTypeEnum)type; }
+  operator ACLGranteeTypeEnum() const { return get_type(); }
+
   void set(ACLGranteeTypeEnum t) { type = t; }
-//  virtual void set(const char *s) = 0;
+  ACLGranteeType& operator=(ACLGranteeTypeEnum t) { set(t); return *this; }
+
   void encode(bufferlist& bl) const {
     ENCODE_START(2, 2, bl);
     encode(type, bl);
diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc
index 1fa44a0329ec..a36aad666f46 100644
--- a/src/rgw/rgw_lua_request.cc
+++ b/src/rgw/rgw_lua_request.cc
@@ -364,19 +364,26 @@ struct GrantMetaTable : public EmptyMetaTable {
     if (strcasecmp(index, "Type") == 0) {
       lua_pushinteger(L, grant->get_type().get_type());
     } else if (strcasecmp(index, "User") == 0) {
-      const auto id_ptr = grant->get_id();
-      if (id_ptr) {
+      if (const auto user = grant->get_user(); user) {
         create_metatable<UserMetaTable>(L, name, index, false, 
-            const_cast<rgw_user*>(id_ptr));
+            const_cast<rgw_user*>(&user->id));
       } else {
         lua_pushnil(L);
       }
     } else if (strcasecmp(index, "Permission") == 0) {
       lua_pushinteger(L, grant->get_permission().get_permissions());
     } else if (strcasecmp(index, "GroupType") == 0) {
-      lua_pushinteger(L, grant->get_group());
+      if (const auto group = grant->get_group(); group) {
+        lua_pushinteger(L, group->type);
+      } else {
+        lua_pushnil(L);
+      }
     } else if (strcasecmp(index, "Referer") == 0) {
-      pushstring(L, grant->get_referer());
+      if (const auto referer = grant->get_referer(); referer) {
+        pushstring(L, referer->url_spec);
+      } else {
+        lua_pushnil(L);
+      }
     } else {
       return error_unknown_field(L, index, name);
     }
diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc
index 71ead6a8b892..b1ee1a9340eb 100644
--- a/src/rgw/rgw_rest_client.cc
+++ b/src/rgw/rgw_rest_client.cc
@@ -3,6 +3,7 @@
 
 #include "rgw_common.h"
 #include "rgw_rest_client.h"
+#include "rgw_acl_s3.h"
 #include "rgw_auth_s3.h"
 #include "rgw_http_errors.h"
 
@@ -504,21 +505,15 @@ static void grants_by_type_add_one_grant(map<int, string>& grants_by_type, int p
   if (!s.empty())
     s.append(", ");
 
-  string id_type_str;
-  ACLGranteeType type = grant.get_type();
-  switch (type.get_type()) {
-    case ACL_TYPE_GROUP:
-      id_type_str = "uri";
-      break;
-    case ACL_TYPE_EMAIL_USER:
-      id_type_str = "emailAddress";
-      break;
-    default:
-      id_type_str = "id";
-  }
-  rgw_user id;
-  grant.get_id(id);
-  s.append(id_type_str + "=\"" + id.to_str() + "\"");
+  if (const auto user = grant.get_user(); user) {
+    s.append("id=\"" + user->id.to_str() + "\"");
+  } else if (const auto email = grant.get_email(); email) {
+    s.append("emailAddress=\"" + email->address + "\"");
+  } else if (const auto group = grant.get_group(); group) {
+    std::string uri;
+    rgw::s3::acl_group_to_uri(group->type, uri);
+    s.append("uri=\"" + uri + "\"");
+  }
 }
 
 struct grant_type_to_header {
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 09f708309e50..0485e71ede37 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -635,8 +635,12 @@ TEST(TestRGWLua, Acl)
     function print_grant(k, g)
       print("Grant Key: " .. tostring(k))
       print("Grant Type: " .. g.Type)
-      print("Grant Group Type: " .. g.GroupType)
-      print("Grant Referer: " .. g.Referer)
+      if (g.GroupType) then
+        print("Grant Group Type: " .. g.GroupType)
+      end
+      if (g.Referer) then
+        print("Grant Referer: " .. g.Referer)
+      end
       if (g.User) then
         print("Grant User.Tenant: " .. g.User.Tenant)
         print("Grant User.Id: " .. g.User.Id)
@@ -666,8 +670,7 @@ TEST(TestRGWLua, Acl)
     .id = rgw_user("jack", "black"),
     .display_name = "jack black"
   };
-  s.user_acl.reset(new RGWAccessControlPolicy(g_cct));
-  s.user_acl->set_owner(owner);
+  s.user_acl.set_owner(owner);
   ACLGrant grant1, grant2, grant3, grant4, grant5, grant6_1, grant6_2;
   grant1.set_canon(rgw_user("jane", "doe"), "her grant", 1);
   grant2.set_group(ACL_GROUP_ALL_USERS ,2);
@@ -676,13 +679,13 @@ TEST(TestRGWLua, Acl)
   grant5.set_group(ACL_GROUP_AUTHENTICATED_USERS, 5);
   grant6_1.set_canon(rgw_user("kill", "bill"), "his grant", 6);
   grant6_2.set_canon(rgw_user("kill", "bill"), "her grant", 7);
-  s.user_acl->get_acl().add_grant(&grant1);
-  s.user_acl->get_acl().add_grant(&grant2);
-  s.user_acl->get_acl().add_grant(&grant3);
-  s.user_acl->get_acl().add_grant(&grant4);
-  s.user_acl->get_acl().add_grant(&grant5);
-  s.user_acl->get_acl().add_grant(&grant6_1);
-  s.user_acl->get_acl().add_grant(&grant6_2);
+  s.user_acl.get_acl().add_grant(grant1);
+  s.user_acl.get_acl().add_grant(grant2);
+  s.user_acl.get_acl().add_grant(grant3);
+  s.user_acl.get_acl().add_grant(grant4);
+  s.user_acl.get_acl().add_grant(grant5);
+  s.user_acl.get_acl().add_grant(grant6_1);
+  s.user_acl.get_acl().add_grant(grant6_2);
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
@@ -733,15 +736,12 @@ TEST(TestRGWLua, UseFunction)
   DEFINE_REQ_STATE;
   s.owner.display_name = "user two";
   s.owner.id = rgw_user("tenant2", "user2");
-  s.user_acl.reset(new RGWAccessControlPolicy());
-  s.user_acl->get_owner().display_name = "user three";
-  s.user_acl->get_owner().id = rgw_user("tenant3", "user3");
-  s.bucket_acl.reset(new RGWAccessControlPolicy());
-  s.bucket_acl->get_owner().display_name = "user four";
-  s.bucket_acl->get_owner().id = rgw_user("tenant4", "user4");
-  s.object_acl.reset(new RGWAccessControlPolicy());
-  s.object_acl->get_owner().display_name = "user five";
-  s.object_acl->get_owner().id = rgw_user("tenant5", "user5");
+  s.user_acl.get_owner().display_name = "user three";
+  s.user_acl.get_owner().id = rgw_user("tenant3", "user3");
+  s.bucket_acl.get_owner().display_name = "user four";
+  s.bucket_acl.get_owner().id = rgw_user("tenant4", "user4");
+  s.object_acl.get_owner().display_name = "user five";
+  s.object_acl.get_owner().id = rgw_user("tenant5", "user5");
 
   const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);

From 0654b0b0513c26ca171f738e47295d84eb43203d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 28 Nov 2023 08:51:41 -0500
Subject: [PATCH 0815/2492] rgw/acl: remove virtual destructor of
 ACLGranteeType

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_acl_types.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/rgw/rgw_acl_types.h b/src/rgw/rgw_acl_types.h
index 11637dd360de..b9866e9b289a 100644
--- a/src/rgw/rgw_acl_types.h
+++ b/src/rgw/rgw_acl_types.h
@@ -182,7 +182,6 @@ class ACLGranteeType
   __u32 type;
 public:
   ACLGranteeType(ACLGranteeTypeEnum t = ACL_TYPE_UNKNOWN) : type(t) {}
-  virtual ~ACLGranteeType() {}
 
   ACLGranteeTypeEnum get_type() const { return (ACLGranteeTypeEnum)type; }
   operator ACLGranteeTypeEnum() const { return get_type(); }

From d962dbeb9040a07d0a2c9db8ab9a2cf1fc320cc7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 30 Nov 2023 12:57:09 -0500
Subject: [PATCH 0816/2492] rgw/iam: admin/system users ignore iam policy
 parsing errors

allow admin users to repair broken iam policies that would otherwise
reject access on PutBucketPolicy/DeleteBucketPolicy requests

Fixes: https://tracker.ceph.com/issues/63485

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index a290388efe0c..d828e2ba090b 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -659,18 +659,29 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
       }
     } catch (const std::exception& e) {
       ldpp_dout(dpp, -1) << "Error reading IAM User Policy: " << e.what() << dendl;
-      ret = -EACCES;
+      if (!s->system_request) {
+        ret = -EACCES;
+      }
     }
   }
 
   try {
     s->iam_policy = get_iam_policy_from_attr(s->cct, s->bucket_attrs, s->bucket_tenant);
   } catch (const std::exception& e) {
-    // Really this is a can't happen condition. We parse the policy
-    // when it's given to us, so perhaps we should abort or otherwise
-    // raise bloody murder.
     ldpp_dout(dpp, 0) << "Error reading IAM Policy: " << e.what() << dendl;
-    ret = -EACCES;
+
+    // This really shouldn't happen. We parse the policy when it's given to us,
+    // so a parsing failure here means we broke backward compatibility. The only
+    // sensible thing to do in this case is to deny access, because the policy
+    // may have.
+    //
+    // However, the only way for an administrator to repair such a bucket is to
+    // send a PutBucketPolicy or DeleteBucketPolicy request as an admin/system
+    // user. We can allow such requests, because even if the policy denied
+    // access, admin/system users override that error from verify_permission().
+    if (!s->system_request) {
+      ret = -EACCES;
+    }
   }
 
   bool success = driver->get_zone()->get_redirect_endpoint(&s->redirect_zone_endpoint);

From 3939201f53d6c6eae23eec1ab83a47d987c18987 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 16:03:34 -0500
Subject: [PATCH 0817/2492] cephadm: add a make_run_dir function

This function is roughly the same as make_var_run only it doesn't rely
on shelling out to the install command. Eventually, it will be used
to replace make_var_run in certain locations.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/file_utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/cephadm/cephadmlib/file_utils.py b/src/cephadm/cephadmlib/file_utils.py
index 7c9e6f69e434..1b9f11499a49 100644
--- a/src/cephadm/cephadmlib/file_utils.py
+++ b/src/cephadm/cephadmlib/file_utils.py
@@ -139,3 +139,7 @@ def get_file_timestamp(fn):
         ).strftime(DATEFMT)
     except Exception:
         return None
+
+
+def make_run_dir(fsid: str, uid: int, gid: int) -> None:
+    makedirs(f'/var/run/ceph/{fsid}', uid, gid, 0o770)

From 3aa16497b3ab88c0c88fa60f58d32fbb8156e4d4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 20 Aug 2023 13:50:00 -0400
Subject: [PATCH 0818/2492] cephadm: add a new funkypatch fixture based on
 mock.patch and pytest

This fixture acts like a combination of mock.patch and pytest's
monkeypatch fixture. It has the additional feature of automatically
finding and patching the same object imported in other modules.  If you
have 'from x import y', where y is a function or class, in both a.py and
b.py it will patch both instances (so long as both a and b are already
imported).
This behavior is useful for cephadm because of the heavy use of the
`from x import y` idiom and how cephadm is being actively refactored.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/fixtures.py | 82 ++++++++++++++++++++++++++++++++++-
 1 file changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
index d25dffa9e3b4..572c1f9969d6 100644
--- a/src/cephadm/tests/fixtures.py
+++ b/src/cephadm/tests/fixtures.py
@@ -6,7 +6,7 @@
 from contextlib import contextmanager
 from pyfakefs import fake_filesystem
 
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Any
 
 
 def import_cephadm():
@@ -183,3 +183,83 @@ def with_cephadm_ctx(
         else:
             yield ctx
 
+
+@pytest.fixture()
+def funkypatch(monkeypatch):
+    """Defines the funkypatch fixtures that acts like a mixture between
+    mock.patch and pytest's monkeypatch fixture.
+    """
+    fp = FunkyPatcher(monkeypatch)
+    yield fp
+
+
+class FunkyPatcher:
+    """FunkyPatcher monkeypatches all imported instances of an object.
+
+    Use `patch` to patch the canonical location of an object and FunkyPatcher
+    will automatically replace other imports of that object.
+    """
+
+    def __init__(self, monkeypatcher):
+        self._mp = monkeypatcher
+        # keep track of objects we've already patched. this dictionary
+        # maps a (module-name, object-name) tuple to the original object
+        # before patching. This could be used to determine if a name has
+        # already been patched or compare a patched object to the original.
+        self._originals: Dict[Tuple[str, str], Any] = {}
+
+    def patch(
+        self,
+        mod: str,
+        name: str = '',
+        *,
+        dest: Any = None,
+        force: bool = False,
+    ) -> Any:
+        """Patch an object and all existing imports of that object.
+        Specify mod as `my.mod.name.obj` where obj is name of the object to be
+        patched or as `my.mod.name` and specify `name` as the name of the
+        object to be patched.
+        If the object to be patched is not imported as the same name in `mod`
+        it will *not* be automatically patched. In other words, `from
+        my.mod.name import foo` will work, but `from my.mod.name import foo as
+        _foo` will not.
+        Use the keyword-only argument `dest` to specify the new object to be
+        used. A MagicMock will be created and used if dest is None.
+        Use the keyword-only argument `force` to override checks that a mocked
+        objects are the same across modules. This can be used in the case that
+        some other code already patched an object and you want funkypatch to
+        override that patch (use with caution).
+        Returns the patched object (the MagicMock or supplied dest).
+        """
+        import sys
+        import importlib
+
+        if not name:
+            mod, name = mod.rsplit('.', 1)
+        modname = (mod, name)
+        # We don't strictly need the check but patching already patched objs is
+        # confusing to think about. It's better to block it for now and perhaps
+        # later we can relax these restrictions or be clever in some way.
+        if modname in self._originals:
+            raise KeyError(f'{modname} already patched')
+
+        if dest is None:
+            dest = mock.MagicMock()
+
+        imod = importlib.import_module(mod)
+        self._originals[modname] = getattr(imod, name)
+
+        for mname, imod in sys.modules.items():
+            try:
+                obj = getattr(imod, name)
+            except AttributeError:
+                # no matching name in module
+                continue
+            # make sure that the module imported the same object as the
+            # one we want to patch out, and not just some naming collision.
+            # ensure the original object and the one in the module are the
+            # same object
+            if obj is self._originals[modname] or force:
+                self._mp.setattr(imod, name, dest)
+        return dest

From 40206912a4752ec7eae91a0948797a8cab54b93d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 20 Aug 2023 13:50:31 -0400
Subject: [PATCH 0819/2492] cephadm: update test to use funkypatch fixture

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 8db8edd0c1bf..4bb1fac432f2 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -16,6 +16,7 @@
     with_cephadm_ctx,
     mock_bad_firewalld,
     import_cephadm,
+    funkypatch,
 )
 
 from pyfakefs import fake_filesystem
@@ -2113,16 +2114,12 @@ def test_http_validation(self, _logger, _find_executable, values, cephadm_fs):
 
 
 class TestPull:
-
-    @mock.patch('time.sleep')
-    @mock.patch('cephadm.get_image_info_from_inspect', return_value={})
-    @mock.patch('cephadm.logger')
-    def test_error(self, _logger, _get_image_info_from_inspect, _sleep, monkeypatch):
-        # manually create a mock and use pytest's monkeypatch fixture to set
-        # multiple targets to the *same* mock
-        _call = mock.MagicMock()
-        monkeypatch.setattr('cephadm.call', _call)
-        monkeypatch.setattr('cephadmlib.call_wrappers.call', _call)
+    def test_error(self, funkypatch):
+        funkypatch.patch('time.sleep')
+        funkypatch.patch('cephadm.logger')
+        _giifi = funkypatch.patch('cephadm.get_image_info_from_inspect')
+        _giifi.return_value = {}
+        _call = funkypatch.patch('cephadmlib.call_wrappers.call')
         ctx = _cephadm.CephadmContext()
         ctx.container_engine = mock_podman()
         ctx.insecure = False

From f25048f876b10705077b8c661469558d25be72f3 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 14:31:12 -0500
Subject: [PATCH 0820/2492] cephadm: convert test_mon_crush_location to use
 funkypatch fixture

The test_mon_crush_location test always seems to have me tinkering
with it during refactoring. Re-do the fixtures to use funkpatch instead
of mock.patch and normal monkeypatch. This looks nicer (IMO) and should
avoid having to frequently mess with it when moving functions during future
refactoring.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 44 ++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 4bb1fac432f2..6379ce28a1d2 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -365,25 +365,37 @@ def test_to_deployment_container(self, _get_container, _get_config, _logger):
         assert os.path.join('data', '9b9d7609-f4d5-4aba-94c8-effa764d96c9', 'custom_config_files', 'grafana.host1', 'testing.str') in c.volume_mounts
         assert c.volume_mounts[os.path.join('data', '9b9d7609-f4d5-4aba-94c8-effa764d96c9', 'custom_config_files', 'grafana.host1', 'testing.str')] == '/etc/testing.str'
 
-    @mock.patch('cephadm.logger')
-    @mock.patch('cephadm.FileLock')
-    @mock.patch('cephadm.deploy_daemon')
-    @mock.patch('cephadm.make_var_run')
-    @mock.patch('cephadm.migrate_sysctl_dir')
-    @mock.patch('cephadm.check_unit', lambda *args, **kwargs: (None, 'running', None))
-    @mock.patch('cephadm.get_unit_name', lambda *args, **kwargs: 'mon-unit-name')
-    @mock.patch('cephadm.extract_uid_gid', lambda *args, **kwargs: (0, 0))
-    @mock.patch('cephadm.get_container')
-    @mock.patch('cephadm.apply_deploy_config_to_ctx', lambda d, c: None)
-    def test_mon_crush_location(self, _get_container, _migrate_sysctl, _make_var_run, _deploy_daemon, _file_lock, _logger, monkeypatch):
+    def test_mon_crush_location(self, funkypatch):
         """
         test that crush location for mon is set if it is included in config_json
         """
-        _fetch_configs = mock.MagicMock()
-        monkeypatch.setattr('cephadmlib.context_getters.fetch_configs', _fetch_configs)
-        monkeypatch.setattr('cephadm.fetch_configs', _fetch_configs)
-        monkeypatch.setattr('cephadm.read_configuration_source', lambda c: {})
-        monkeypatch.setattr('cephadm.fetch_custom_config_files', mock.MagicMock())
+        funkypatch.patch('cephadm.logger')
+        funkypatch.patch('cephadm.FileLock')
+        _deploy_daemon = funkypatch.patch('cephadm.deploy_daemon')
+        _make_var_run = funkypatch.patch('cephadm.make_var_run')
+        _migrate_sysctl = funkypatch.patch('cephadm.migrate_sysctl_dir')
+        funkypatch.patch(
+            'cephadm.check_unit',
+            dest=lambda *args, **kwargs: (None, 'running', None),
+        )
+        funkypatch.patch(
+            'cephadm.get_unit_name',
+            dest=lambda *args, **kwargs: 'mon-unit-name',
+        )
+        funkypatch.patch(
+            'cephadm.extract_uid_gid', dest=lambda *args, **kwargs: (0, 0)
+        )
+        _get_container = funkypatch.patch('cephadm.get_container')
+        funkypatch.patch(
+            'cephadm.apply_deploy_config_to_ctx', dest=lambda d, c: None
+        )
+        _fetch_configs = funkypatch.patch(
+            'cephadmlib.context_getters.fetch_configs'
+        )
+        funkypatch.patch(
+            'cephadm.read_configuration_source', dest=lambda c: {}
+        )
+        funkypatch.patch('cephadm.fetch_custom_config_files')
 
         ctx = _cephadm.CephadmContext()
         ctx.name = 'mon.test'

From eca9be6544d86d1ee1a384898dcc86c098405d24 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 5 Nov 2023 16:03:53 -0500
Subject: [PATCH 0821/2492] cephadm: create deployment_utils module

Create a deployment_utils module for deployment related functions that
don't have a better home.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 31 +------------------
 src/cephadm/cephadmlib/deployment_utils.py | 35 ++++++++++++++++++++++
 src/cephadm/tests/test_cephadm.py          | 14 +++++----
 3 files changed, 45 insertions(+), 35 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/deployment_utils.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 959676ba3af1..14b064a95428 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -176,6 +176,7 @@
 from cephadmlib.sysctl import install_sysctl, migrate_sysctl_dir
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
+from cephadmlib.deployment_utils import to_deployment_container
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -5257,36 +5258,6 @@ def command_registry_login(ctx: CephadmContext) -> int:
 ##################################
 
 
-def to_deployment_container(
-    ctx: CephadmContext, ctr: CephContainer
-) -> CephContainer:
-    """Given a standard ceph container instance return a CephContainer
-    prepared for a deployment as a daemon, having the extra args and
-    custom configurations added.
-    NOTE: The `ctr` object is mutated before being returned.
-    """
-    if 'extra_container_args' in ctx and ctx.extra_container_args:
-        ctr.container_args.extend(ctx.extra_container_args)
-    if 'extra_entrypoint_args' in ctx and ctx.extra_entrypoint_args:
-        ctr.args.extend(ctx.extra_entrypoint_args)
-    ccfiles = fetch_custom_config_files(ctx)
-    if ccfiles:
-        mandatory_keys = ['mount_path', 'content']
-        for conf in ccfiles:
-            if all(k in conf for k in mandatory_keys):
-                mount_path = conf['mount_path']
-                assert ctr.identity
-                file_path = os.path.join(
-                    ctx.data_dir,
-                    ctr.identity.fsid,
-                    'custom_config_files',
-                    ctr.identity.daemon_name,
-                    os.path.basename(mount_path)
-                )
-                ctr.volume_mounts[file_path] = mount_path
-    return ctr
-
-
 def get_deployment_type(
     ctx: CephadmContext, ident: 'DaemonIdentity',
 ) -> DeploymentType:
diff --git a/src/cephadm/cephadmlib/deployment_utils.py b/src/cephadm/cephadmlib/deployment_utils.py
new file mode 100644
index 000000000000..908fa979f1a5
--- /dev/null
+++ b/src/cephadm/cephadmlib/deployment_utils.py
@@ -0,0 +1,35 @@
+import os
+
+from .container_types import CephContainer
+from .context import CephadmContext
+from cephadmlib.context_getters import fetch_custom_config_files
+
+
+def to_deployment_container(
+    ctx: CephadmContext, ctr: CephContainer
+) -> CephContainer:
+    """Given a standard ceph container instance return a CephContainer
+    prepared for a deployment as a daemon, having the extra args and
+    custom configurations added.
+    NOTE: The `ctr` object is mutated before being returned.
+    """
+    if 'extra_container_args' in ctx and ctx.extra_container_args:
+        ctr.container_args.extend(ctx.extra_container_args)
+    if 'extra_entrypoint_args' in ctx and ctx.extra_entrypoint_args:
+        ctr.args.extend(ctx.extra_entrypoint_args)
+    ccfiles = fetch_custom_config_files(ctx)
+    if ccfiles:
+        mandatory_keys = ['mount_path', 'content']
+        for conf in ccfiles:
+            if all(k in conf for k in mandatory_keys):
+                mount_path = conf['mount_path']
+                assert ctr.identity
+                file_path = os.path.join(
+                    ctx.data_dir,
+                    ctr.identity.fsid,
+                    'custom_config_files',
+                    ctr.identity.daemon_name,
+                    os.path.basename(mount_path),
+                )
+                ctr.volume_mounts[file_path] = mount_path
+    return ctr
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 6379ce28a1d2..c5d8d19f26d6 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -318,13 +318,17 @@ def test_skip_firewalld(self, _logger, cephadm_fs):
         with pytest.raises(Exception):
             _cephadm.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None)
 
-    @mock.patch('cephadm.logger')
-    @mock.patch('cephadm.fetch_custom_config_files')
-    @mock.patch('cephadm.get_container')
-    def test_to_deployment_container(self, _get_container, _get_config, _logger):
+    def test_to_deployment_container(self, funkypatch):
         """
         test to_deployment_container properly makes use of extra container args and custom conf files
         """
+        from cephadmlib.deployment_utils import to_deployment_container
+
+        funkypatch.patch('cephadm.logger')
+        _get_config = funkypatch.patch(
+            'cephadmlib.deployment_utils.fetch_custom_config_files'
+        )
+        _get_container = funkypatch.patch('cephadm.get_container')
 
         ctx = _cephadm.CephadmContext()
         ctx.config_json = '-'
@@ -358,7 +362,7 @@ def test_to_deployment_container(self, _get_container, _get_config, _logger):
             host_network=True,
         )
         c = _cephadm.get_container(ctx, ident)
-        c = _cephadm.to_deployment_container(ctx, c)
+        c = to_deployment_container(ctx, c)
 
         assert '--pids-limit=12345' in c.container_args
         assert '--something' in c.container_args

From 700ea8d109c71f31079d239dd256a7562e9961f2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 19:21:10 -0500
Subject: [PATCH 0822/2492] cephadm: use funkypatch for setting up common
 patches in deploy tests

Add a shim function and convert to the use of the FunkyPatcher class in
the test_deploy.py test functions. Use a shim as to not have to change
all the tests (yet).

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index c77b243dfa77..9d82b2055b91 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -8,6 +8,7 @@
     import_cephadm,
     mock_podman,
     with_cephadm_ctx,
+    FunkyPatcher,
 )
 
 
@@ -15,25 +16,24 @@
 
 
 def _common_mp(monkeypatch):
+    return _common_patches(FunkyPatcher(monkeypatch))
+
+
+def _common_patches(funkypatch):
     mocks = {}
-    _call = mock.MagicMock(return_value=('', '', 0))
-    monkeypatch.setattr('cephadmlib.container_types.call', _call)
+    _call = funkypatch.patch('cephadmlib.container_types.call')
+    _call.return_value = ('', '', 0)
     mocks['call'] = _call
-    _call_throws = mock.MagicMock(return_value=0)
-    monkeypatch.setattr(
-        'cephadmlib.container_types.call_throws', _call_throws
-    )
+    _call_throws = funkypatch.patch('cephadmlib.container_types.call_throws')
+    _call_throws.return_value = ('', '', 0)
     mocks['call_throws'] = _call_throws
-    _firewalld = mock.MagicMock()
+    _firewalld = funkypatch.patch('cephadm.Firewalld')
     _firewalld().external_ports.get.return_value = []
-    monkeypatch.setattr('cephadm.Firewalld', _firewalld)
     mocks['Firewalld'] = _firewalld
-    _extract_uid_gid = mock.MagicMock()
+    _extract_uid_gid = funkypatch.patch('cephadm.extract_uid_gid', force=True)
     _extract_uid_gid.return_value = (8765, 8765)
-    monkeypatch.setattr('cephadm.extract_uid_gid', _extract_uid_gid)
     mocks['extract_uid_gid'] = _extract_uid_gid
-    _install_sysctl = mock.MagicMock()
-    monkeypatch.setattr('cephadm.install_sysctl', _install_sysctl)
+    _install_sysctl = funkypatch.patch('cephadm.install_sysctl')
     mocks['install_sysctl'] = _install_sysctl
     return mocks
 

From d5ce0b7a8da55f52a473ef4169622357f655fe0d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 09:57:24 -0500
Subject: [PATCH 0823/2492] cephadm: update tests to import dict_get* functions
 from proper module

Update the test that import dict_get and dict_get_join to use the
context_getters module - the module that actually defines the functions.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index c5d8d19f26d6..c8fd3abf7d3e 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -558,25 +558,31 @@ def test_get_image_info_from_inspect(self):
 
 
     def test_dict_get(self):
-        result = _cephadm.dict_get({'a': 1}, 'a', require=True)
+        from cephadmlib.data_utils import dict_get
+
+        result = dict_get({'a': 1}, 'a', require=True)
         assert result == 1
-        result = _cephadm.dict_get({'a': 1}, 'b')
+        result = dict_get({'a': 1}, 'b')
         assert result is None
-        result = _cephadm.dict_get({'a': 1}, 'b', default=2)
+        result = dict_get({'a': 1}, 'b', default=2)
         assert result == 2
 
     def test_dict_get_error(self):
+        from cephadmlib.data_utils import dict_get
+
         with pytest.raises(_cephadm.Error):
-            _cephadm.dict_get({'a': 1}, 'b', require=True)
+            dict_get({'a': 1}, 'b', require=True)
 
     def test_dict_get_join(self):
-        result = _cephadm.dict_get_join({'foo': ['a', 'b']}, 'foo')
+        from cephadmlib.data_utils import dict_get_join
+
+        result = dict_get_join({'foo': ['a', 'b']}, 'foo')
         assert result == 'a\nb'
-        result = _cephadm.dict_get_join({'foo': [1, 2]}, 'foo')
+        result = dict_get_join({'foo': [1, 2]}, 'foo')
         assert result == '1\n2'
-        result = _cephadm.dict_get_join({'bar': 'a'}, 'bar')
+        result = dict_get_join({'bar': 'a'}, 'bar')
         assert result == 'a'
-        result = _cephadm.dict_get_join({'a': 1}, 'a')
+        result = dict_get_join({'a': 1}, 'a')
         assert result == 1
 
     @mock.patch('os.listdir', return_value=[])

From 6276f74ff2c5e5364a3cd6e7e38e9dbda9f0b3a7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:46:04 -0500
Subject: [PATCH 0824/2492] cephamd: update tests to use should_log_to_journald
 from context_getters

Update tests to import should_log_to_journald from context_getters - the
module that actually defines that function. This makes later refactoring
easier.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_cephadm.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index c8fd3abf7d3e..899272cb4143 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -784,24 +784,26 @@ def test_get_container_info(self, _logger, daemon_filter, by_name, daemon_list,
                 assert _cephadm.get_container_info(ctx, daemon_filter, by_name) == output
 
     def test_should_log_to_journald(self):
+        from cephadmlib import context_getters
+
         ctx = _cephadm.CephadmContext()
         # explicit
         ctx.log_to_journald = True
-        assert _cephadm.should_log_to_journald(ctx)
+        assert context_getters.should_log_to_journald(ctx)
 
         ctx.log_to_journald = None
         # enable if podman support --cgroup=split
         ctx.container_engine = mock_podman()
         ctx.container_engine.version = (2, 1, 0)
-        assert _cephadm.should_log_to_journald(ctx)
+        assert context_getters.should_log_to_journald(ctx)
 
         # disable on old podman
         ctx.container_engine.version = (2, 0, 0)
-        assert not _cephadm.should_log_to_journald(ctx)
+        assert not context_getters.should_log_to_journald(ctx)
 
         # disable on docker
         ctx.container_engine = mock_docker()
-        assert not _cephadm.should_log_to_journald(ctx)
+        assert not context_getters.should_log_to_journald(ctx)
 
     def test_normalize_image_digest(self):
         s = 'myhostname:5000/ceph/ceph@sha256:753886ad9049004395ae990fbb9b096923b5a518b819283141ee8716ddf55ad1'

From 5a45aca014571d6a6251cdbd67f3e11fcbf19d4a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 15:45:13 -0500
Subject: [PATCH 0825/2492] cephadm: start a cephadmlib.daemons package

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/__init__.py

diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1

From 17714889f56bed9e09ba1cdfb2a89a321fb87978 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 15:58:36 -0500
Subject: [PATCH 0826/2492] cephadm: move custom container class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 194 +------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   3 +
 src/cephadm/cephadmlib/daemons/custom.py   | 210 +++++++++++++++++++++
 3 files changed, 214 insertions(+), 193 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/custom.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 14b064a95428..fa8e48244f04 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -177,6 +177,7 @@
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
 from cephadmlib.deployment_utils import to_deployment_container
+from cephadmlib.daemons import CustomContainer
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -2005,199 +2006,6 @@ def default_entrypoint(self) -> str:
 ##################################
 
 
-@register_daemon_form
-class CustomContainer(ContainerDaemonForm):
-    """Defines a custom container"""
-    daemon_type = 'container'
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        # config-json options
-        self.entrypoint = dict_get(config_json, 'entrypoint')
-        self.uid = dict_get(config_json, 'uid', 65534)  # nobody
-        self.gid = dict_get(config_json, 'gid', 65534)  # nobody
-        self.volume_mounts = dict_get(config_json, 'volume_mounts', {})
-        self.args = dict_get(config_json, 'args', [])
-        self.envs = dict_get(config_json, 'envs', [])
-        self.privileged = dict_get(config_json, 'privileged', False)
-        self.bind_mounts = dict_get(config_json, 'bind_mounts', [])
-        self.ports = dict_get(config_json, 'ports', [])
-        self.dirs = dict_get(config_json, 'dirs', [])
-        self.files = dict_get(config_json, 'files', {})
-
-    @classmethod
-    def init(cls, ctx: CephadmContext,
-             fsid: str, daemon_id: Union[int, str]) -> 'CustomContainer':
-        return cls(fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CustomContainer':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
-        """
-        Create dirs/files below the container data directory.
-        """
-        logger.info('Creating custom container configuration '
-                    'dirs/files in {} ...'.format(data_dir))
-
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % data_dir)
-
-        for dir_path in self.dirs:
-            logger.info('Creating directory: {}'.format(dir_path))
-            dir_path = os.path.join(data_dir, dir_path.strip('/'))
-            makedirs(dir_path, uid, gid, 0o755)
-
-        for file_path in self.files:
-            logger.info('Creating file: {}'.format(file_path))
-            content = dict_get_join(self.files, file_path)
-            file_path = os.path.join(data_dir, file_path.strip('/'))
-            with write_new(file_path, owner=(uid, gid), encoding='utf-8') as f:
-                f.write(content)
-
-    def get_daemon_args(self) -> List[str]:
-        return []
-
-    def get_container_args(self) -> List[str]:
-        return self.args
-
-    def get_container_envs(self) -> List[str]:
-        return self.envs
-
-    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
-        """
-        Get the volume mounts. Relative source paths will be located below
-        `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
-
-        Example:
-        {
-            /foo/conf: /conf
-            foo/conf: /conf
-        }
-        becomes
-        {
-            /foo/conf: /conf
-            /var/lib/ceph/<cluster-fsid>/<daemon-name>/foo/conf: /conf
-        }
-        """
-        mounts = {}
-        for source, destination in self.volume_mounts.items():
-            source = os.path.join(data_dir, source)
-            mounts[source] = destination
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    def _get_container_binds(self, data_dir: str) -> List[List[str]]:
-        """
-        Get the bind mounts. Relative `source=...` paths will be located below
-        `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
-
-        Example:
-        [
-            'type=bind',
-            'source=lib/modules',
-            'destination=/lib/modules',
-            'ro=true'
-        ]
-        becomes
-        [
-            ...
-            'source=/var/lib/ceph/<cluster-fsid>/<daemon-name>/lib/modules',
-            ...
-        ]
-        """
-        binds = self.bind_mounts.copy()
-        for bind in binds:
-            for index, value in enumerate(bind):
-                match = re.match(r'^source=(.+)$', value)
-                if match:
-                    bind[index] = 'source={}'.format(os.path.join(
-                        data_dir, match.group(1)))
-        return binds
-
-    def customize_container_binds(
-        self, ctx: CephadmContext, binds: List[List[str]]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        binds.extend(self._get_container_binds(data_dir))
-
-    # Cache the container so we don't need to rebuild it again when calling
-    # into init_containers
-    _container: Optional[CephContainer] = None
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        if self._container is None:
-            ctr = daemon_to_container(
-                ctx,
-                self,
-                host_network=False,
-                privileged=self.privileged,
-                ptrace=ctx.allow_ptrace,
-            )
-            self._container = to_deployment_container(ctx, ctr)
-        return self._container
-
-    def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
-        primary = self.container(ctx)
-        init_containers: List[Dict[str, Any]] = getattr(
-            ctx, 'init_containers', []
-        )
-        return [
-            InitContainer.from_primary_and_opts(ctx, primary, ic_opts)
-            for ic_opts in init_containers
-        ]
-
-    def customize_container_endpoints(
-        self, endpoints: List[EndPoint], deployment_type: DeploymentType
-    ) -> None:
-        if deployment_type == DeploymentType.DEFAULT:
-            endpoints.extend([EndPoint('0.0.0.0', p) for p in self.ports])
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        envs.extend(self.get_container_envs())
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_container_args())
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_daemon_args())
-
-    def default_entrypoint(self) -> str:
-        return self.entrypoint or ''
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return self.uid, self.gid
-
-
-##################################
-
-
 def get_supported_daemons():
     # type: () -> List[str]
     supported_daemons = ceph_daemons()
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index e69de29bb2d1..d979ce19a936 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -0,0 +1,3 @@
+from .custom import CustomContainer
+
+__all__ = ['CustomContainer']
diff --git a/src/cephadm/cephadmlib/daemons/custom.py b/src/cephadm/cephadmlib/daemons/custom.py
new file mode 100644
index 000000000000..8e0d59e6f3a9
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/custom.py
@@ -0,0 +1,210 @@
+import logging
+import os
+import re
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, InitContainer
+from ..context import CephadmContext
+from ..context_getters import fetch_configs
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..data_utils import dict_get, dict_get_join
+from ..deploy import DeploymentType
+from ..deployment_utils import to_deployment_container
+from ..file_utils import write_new, makedirs
+from ..net_utils import EndPoint
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class CustomContainer(ContainerDaemonForm):
+    """Defines a custom container"""
+    daemon_type = 'container'
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 fsid: str, daemon_id: Union[int, str],
+                 config_json: Dict, image: str) -> None:
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        # config-json options
+        self.entrypoint = dict_get(config_json, 'entrypoint')
+        self.uid = dict_get(config_json, 'uid', 65534)  # nobody
+        self.gid = dict_get(config_json, 'gid', 65534)  # nobody
+        self.volume_mounts = dict_get(config_json, 'volume_mounts', {})
+        self.args = dict_get(config_json, 'args', [])
+        self.envs = dict_get(config_json, 'envs', [])
+        self.privileged = dict_get(config_json, 'privileged', False)
+        self.bind_mounts = dict_get(config_json, 'bind_mounts', [])
+        self.ports = dict_get(config_json, 'ports', [])
+        self.dirs = dict_get(config_json, 'dirs', [])
+        self.files = dict_get(config_json, 'files', {})
+
+    @classmethod
+    def init(cls, ctx: CephadmContext,
+             fsid: str, daemon_id: Union[int, str]) -> 'CustomContainer':
+        return cls(fsid, daemon_id,
+                   fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CustomContainer':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+        """
+        Create dirs/files below the container data directory.
+        """
+        logger.info('Creating custom container configuration '
+                    'dirs/files in {} ...'.format(data_dir))
+
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % data_dir)
+
+        for dir_path in self.dirs:
+            logger.info('Creating directory: {}'.format(dir_path))
+            dir_path = os.path.join(data_dir, dir_path.strip('/'))
+            makedirs(dir_path, uid, gid, 0o755)
+
+        for file_path in self.files:
+            logger.info('Creating file: {}'.format(file_path))
+            content = dict_get_join(self.files, file_path)
+            file_path = os.path.join(data_dir, file_path.strip('/'))
+            with write_new(file_path, owner=(uid, gid), encoding='utf-8') as f:
+                f.write(content)
+
+    def get_daemon_args(self) -> List[str]:
+        return []
+
+    def get_container_args(self) -> List[str]:
+        return self.args
+
+    def get_container_envs(self) -> List[str]:
+        return self.envs
+
+    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+        """
+        Get the volume mounts. Relative source paths will be located below
+        `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+
+        Example:
+        {
+            /foo/conf: /conf
+            foo/conf: /conf
+        }
+        becomes
+        {
+            /foo/conf: /conf
+            /var/lib/ceph/<cluster-fsid>/<daemon-name>/foo/conf: /conf
+        }
+        """
+        mounts = {}
+        for source, destination in self.volume_mounts.items():
+            source = os.path.join(data_dir, source)
+            mounts[source] = destination
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    def _get_container_binds(self, data_dir: str) -> List[List[str]]:
+        """
+        Get the bind mounts. Relative `source=...` paths will be located below
+        `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+
+        Example:
+        [
+            'type=bind',
+            'source=lib/modules',
+            'destination=/lib/modules',
+            'ro=true'
+        ]
+        becomes
+        [
+            ...
+            'source=/var/lib/ceph/<cluster-fsid>/<daemon-name>/lib/modules',
+            ...
+        ]
+        """
+        binds = self.bind_mounts.copy()
+        for bind in binds:
+            for index, value in enumerate(bind):
+                match = re.match(r'^source=(.+)$', value)
+                if match:
+                    bind[index] = 'source={}'.format(os.path.join(
+                        data_dir, match.group(1)))
+        return binds
+
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        binds.extend(self._get_container_binds(data_dir))
+
+    # Cache the container so we don't need to rebuild it again when calling
+    # into init_containers
+    _container: Optional[CephContainer] = None
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        if self._container is None:
+            ctr = daemon_to_container(
+                ctx,
+                self,
+                host_network=False,
+                privileged=self.privileged,
+                ptrace=ctx.allow_ptrace,
+            )
+            self._container = to_deployment_container(ctx, ctr)
+        return self._container
+
+    def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
+        primary = self.container(ctx)
+        init_containers: List[Dict[str, Any]] = getattr(
+            ctx, 'init_containers', []
+        )
+        return [
+            InitContainer.from_primary_and_opts(ctx, primary, ic_opts)
+            for ic_opts in init_containers
+        ]
+
+    def customize_container_endpoints(
+        self, endpoints: List[EndPoint], deployment_type: DeploymentType
+    ) -> None:
+        if deployment_type == DeploymentType.DEFAULT:
+            endpoints.extend([EndPoint('0.0.0.0', p) for p in self.ports])
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_container_args())
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint or ''
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return self.uid, self.gid

From 8db86d29d646a74f992ea5b05ce7efe63bd9ea6b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 16:04:15 -0500
Subject: [PATCH 0827/2492] cephadm: move tracing class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 104 +------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   3 +-
 src/cephadm/cephadmlib/daemons/tracing.py  | 115 +++++++++++++++++++++
 3 files changed, 118 insertions(+), 104 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/tracing.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index fa8e48244f04..3e46fdba1844 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -35,15 +35,11 @@
 from cephadmlib.constants import (
     # default images
     DEFAULT_ALERT_MANAGER_IMAGE,
-    DEFAULT_ELASTICSEARCH_IMAGE,
     DEFAULT_GRAFANA_IMAGE,
     DEFAULT_HAPROXY_IMAGE,
     DEFAULT_IMAGE,
     DEFAULT_IMAGE_IS_MAIN,
     DEFAULT_IMAGE_RELEASE,
-    DEFAULT_JAEGER_AGENT_IMAGE,
-    DEFAULT_JAEGER_COLLECTOR_IMAGE,
-    DEFAULT_JAEGER_QUERY_IMAGE,
     DEFAULT_KEEPALIVED_IMAGE,
     DEFAULT_LOKI_IMAGE,
     DEFAULT_NODE_EXPORTER_IMAGE,
@@ -177,7 +173,7 @@
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
 from cephadmlib.deployment_utils import to_deployment_container
-from cephadmlib.daemons import CustomContainer
+from cephadmlib.daemons import CustomContainer, Tracing
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -1908,104 +1904,6 @@ def customize_container_args(
 ##################################
 
 
-@register_daemon_form
-class Tracing(ContainerDaemonForm):
-    """Define the configs for the jaeger tracing containers"""
-
-    components: Dict[str, Dict[str, Any]] = {
-        'elasticsearch': {
-            'image': DEFAULT_ELASTICSEARCH_IMAGE,
-            'envs': ['discovery.type=single-node']
-        },
-        'jaeger-agent': {
-            'image': DEFAULT_JAEGER_AGENT_IMAGE,
-        },
-        'jaeger-collector': {
-            'image': DEFAULT_JAEGER_COLLECTOR_IMAGE,
-        },
-        'jaeger-query': {
-            'image': DEFAULT_JAEGER_QUERY_IMAGE,
-        },
-    }  # type: ignore
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return daemon_type in cls.components
-
-    @staticmethod
-    def set_configuration(config: Dict[str, str], daemon_type: str) -> None:
-        if daemon_type in ['jaeger-collector', 'jaeger-query']:
-            assert 'elasticsearch_nodes' in config
-            Tracing.components[daemon_type]['envs'] = [
-                'SPAN_STORAGE_TYPE=elasticsearch',
-                f'ES_SERVER_URLS={config["elasticsearch_nodes"]}']
-        if daemon_type == 'jaeger-agent':
-            assert 'collector_nodes' in config
-            Tracing.components[daemon_type]['daemon_args'] = [
-                f'--reporter.grpc.host-port={config["collector_nodes"]}',
-                '--processor.jaeger-compact.server-host-port=6799'
-            ]
-
-    def __init__(self, ident: DaemonIdentity) -> None:
-        self._identity = ident
-        self._configured = False
-
-    def _configure(self, ctx: CephadmContext) -> None:
-        if self._configured:
-            return
-        config = fetch_configs(ctx)
-        # Currently, this method side-effects the class attribute, and that
-        # is unpleasant. In the future it would be nice to move all of
-        # set_configuration into _confiure and only modify each classes data
-        # independently
-        self.set_configuration(config, self.identity.daemon_type)
-        self._configured = True
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Tracing':
-        return cls(ident)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return self._identity
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return 65534, 65534
-
-    def get_daemon_args(self) -> List[str]:
-        return self.components[self.identity.daemon_type].get(
-            'daemon_args', []
-        )
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        self._configure(ctx)
-        # earlier code did an explicit check if the daemon type was jaeger-agent
-        # and would only call get_daemon_args if that was true. However, since
-        # the function only returns a non-empty list in the case of jaeger-agent
-        # that check is unnecessary and is not brought over.
-        args.extend(self.get_daemon_args())
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        self._configure(ctx)
-        envs.extend(
-            self.components[self.identity.daemon_type].get('envs', [])
-        )
-
-    def default_entrypoint(self) -> str:
-        return ''
-
-
-##################################
-
-
 def get_supported_daemons():
     # type: () -> List[str]
     supported_daemons = ceph_daemons()
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index d979ce19a936..dec915105052 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -1,3 +1,4 @@
 from .custom import CustomContainer
+from .tracing import Tracing
 
-__all__ = ['CustomContainer']
+__all__ = ['CustomContainer', 'Tracing']
diff --git a/src/cephadm/cephadmlib/daemons/tracing.py b/src/cephadm/cephadmlib/daemons/tracing.py
new file mode 100644
index 000000000000..f178bd664086
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/tracing.py
@@ -0,0 +1,115 @@
+import logging
+
+from typing import Any, Dict, List, Tuple
+
+from ..constants import (
+    DEFAULT_ELASTICSEARCH_IMAGE,
+    DEFAULT_JAEGER_AGENT_IMAGE,
+    DEFAULT_JAEGER_COLLECTOR_IMAGE,
+    DEFAULT_JAEGER_QUERY_IMAGE,
+)
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer
+from ..context import CephadmContext
+from ..context_getters import fetch_configs
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..deployment_utils import to_deployment_container
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class Tracing(ContainerDaemonForm):
+    """Define the configs for the jaeger tracing containers"""
+
+    components: Dict[str, Dict[str, Any]] = {
+        'elasticsearch': {
+            'image': DEFAULT_ELASTICSEARCH_IMAGE,
+            'envs': ['discovery.type=single-node']
+        },
+        'jaeger-agent': {
+            'image': DEFAULT_JAEGER_AGENT_IMAGE,
+        },
+        'jaeger-collector': {
+            'image': DEFAULT_JAEGER_COLLECTOR_IMAGE,
+        },
+        'jaeger-query': {
+            'image': DEFAULT_JAEGER_QUERY_IMAGE,
+        },
+    }  # type: ignore
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return daemon_type in cls.components
+
+    @staticmethod
+    def set_configuration(config: Dict[str, str], daemon_type: str) -> None:
+        if daemon_type in ['jaeger-collector', 'jaeger-query']:
+            assert 'elasticsearch_nodes' in config
+            Tracing.components[daemon_type]['envs'] = [
+                'SPAN_STORAGE_TYPE=elasticsearch',
+                f'ES_SERVER_URLS={config["elasticsearch_nodes"]}']
+        if daemon_type == 'jaeger-agent':
+            assert 'collector_nodes' in config
+            Tracing.components[daemon_type]['daemon_args'] = [
+                f'--reporter.grpc.host-port={config["collector_nodes"]}',
+                '--processor.jaeger-compact.server-host-port=6799'
+            ]
+
+    def __init__(self, ident: DaemonIdentity) -> None:
+        self._identity = ident
+        self._configured = False
+
+    def _configure(self, ctx: CephadmContext) -> None:
+        if self._configured:
+            return
+        config = fetch_configs(ctx)
+        # Currently, this method side-effects the class attribute, and that
+        # is unpleasant. In the future it would be nice to move all of
+        # set_configuration into _confiure and only modify each classes data
+        # independently
+        self.set_configuration(config, self.identity.daemon_type)
+        self._configured = True
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Tracing':
+        return cls(ident)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 65534, 65534
+
+    def get_daemon_args(self) -> List[str]:
+        return self.components[self.identity.daemon_type].get(
+            'daemon_args', []
+        )
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        self._configure(ctx)
+        # earlier code did an explicit check if the daemon type was jaeger-agent
+        # and would only call get_daemon_args if that was true. However, since
+        # the function only returns a non-empty list in the case of jaeger-agent
+        # that check is unnecessary and is not brought over.
+        args.extend(self.get_daemon_args())
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        self._configure(ctx)
+        envs.extend(
+            self.components[self.identity.daemon_type].get('envs', [])
+        )
+
+    def default_entrypoint(self) -> str:
+        return ''

From cb960823ba55c1797243384befe48217c54e326a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 19:23:20 -0500
Subject: [PATCH 0828/2492] cephadm: move haproxy and keepalived classes to a
 new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 259 +-------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   3 +-
 src/cephadm/cephadmlib/daemons/ingress.py  | 268 +++++++++++++++++++++
 3 files changed, 271 insertions(+), 259 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/ingress.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 3e46fdba1844..c588d661c211 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -36,11 +36,9 @@
     # default images
     DEFAULT_ALERT_MANAGER_IMAGE,
     DEFAULT_GRAFANA_IMAGE,
-    DEFAULT_HAPROXY_IMAGE,
     DEFAULT_IMAGE,
     DEFAULT_IMAGE_IS_MAIN,
     DEFAULT_IMAGE_RELEASE,
-    DEFAULT_KEEPALIVED_IMAGE,
     DEFAULT_LOKI_IMAGE,
     DEFAULT_NODE_EXPORTER_IMAGE,
     DEFAULT_NVMEOF_IMAGE,
@@ -173,7 +171,7 @@
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
 from cephadmlib.deployment_utils import to_deployment_container
-from cephadmlib.daemons import CustomContainer, Tracing
+from cephadmlib.daemons import CustomContainer, Tracing, HAproxy, Keepalived
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -1646,261 +1644,6 @@ def customize_container_envs(
     def default_entrypoint(self) -> str:
         return self.entrypoint
 
-
-##################################
-
-
-@register_daemon_form
-class HAproxy(ContainerDaemonForm):
-    """Defines an HAproxy container"""
-    daemon_type = 'haproxy'
-    required_files = ['haproxy.cfg']
-    default_image = DEFAULT_HAPROXY_IMAGE
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        # config-json options
-        self.files = dict_get(config_json, 'files', {})
-
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx: CephadmContext,
-             fsid: str, daemon_id: Union[int, str]) -> 'HAproxy':
-        return cls(ctx, fsid, daemon_id, fetch_configs(ctx),
-                   ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'HAproxy':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
-        """Create files under the container data dir"""
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % (data_dir))
-
-        # create additional directories in data dir for HAproxy to use
-        if not os.path.isdir(os.path.join(data_dir, 'haproxy')):
-            makedirs(os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE)
-
-        data_dir = os.path.join(data_dir, 'haproxy')
-        populate_files(data_dir, self.files, uid, gid)
-
-    def get_daemon_args(self) -> List[str]:
-        return ['haproxy', '-f', '/var/lib/haproxy/haproxy.cfg']
-
-    def validate(self):
-        # type: () -> None
-        if not is_fsid(self.fsid):
-            raise Error('not an fsid: %s' % self.fsid)
-        if not self.daemon_id:
-            raise Error('invalid daemon_id: %s' % self.daemon_id)
-        if not self.image:
-            raise Error('invalid image: %s' % self.image)
-
-        # check for the required files
-        if self.required_files:
-            for fname in self.required_files:
-                if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
-
-    def get_daemon_name(self):
-        # type: () -> str
-        return '%s.%s' % (self.daemon_type, self.daemon_id)
-
-    def get_container_name(self, desc=None):
-        # type: (Optional[str]) -> str
-        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
-        if desc:
-            cname = '%s-%s' % (cname, desc)
-        return cname
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        # better directory for this?
-        return extract_uid_gid(self.ctx, file_path='/var/lib')
-
-    @staticmethod
-    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
-        mounts = dict()
-        mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    @staticmethod
-    def get_sysctl_settings() -> List[str]:
-        return [
-            '# IP forwarding and non-local bind',
-            'net.ipv4.ip_forward = 1',
-            'net.ipv4.ip_nonlocal_bind = 1',
-        ]
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(
-            ['--user=root']
-        )  # haproxy 2.4 defaults to a different user
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_daemon_args())
-
-
-##################################
-
-
-@register_daemon_form
-class Keepalived(ContainerDaemonForm):
-    """Defines an Keepalived container"""
-    daemon_type = 'keepalived'
-    required_files = ['keepalived.conf']
-    default_image = DEFAULT_KEEPALIVED_IMAGE
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        # config-json options
-        self.files = dict_get(config_json, 'files', {})
-
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'Keepalived':
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Keepalived':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
-        """Create files under the container data dir"""
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % (data_dir))
-
-        # create additional directories in data dir for keepalived to use
-        if not os.path.isdir(os.path.join(data_dir, 'keepalived')):
-            makedirs(os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE)
-
-        # populate files from the config-json
-        populate_files(data_dir, self.files, uid, gid)
-
-    def validate(self):
-        # type: () -> None
-        if not is_fsid(self.fsid):
-            raise Error('not an fsid: %s' % self.fsid)
-        if not self.daemon_id:
-            raise Error('invalid daemon_id: %s' % self.daemon_id)
-        if not self.image:
-            raise Error('invalid image: %s' % self.image)
-
-        # check for the required files
-        if self.required_files:
-            for fname in self.required_files:
-                if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
-
-    def get_daemon_name(self):
-        # type: () -> str
-        return '%s.%s' % (self.daemon_type, self.daemon_id)
-
-    def get_container_name(self, desc=None):
-        # type: (Optional[str]) -> str
-        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
-        if desc:
-            cname = '%s-%s' % (cname, desc)
-        return cname
-
-    @staticmethod
-    def get_container_envs():
-        # type: () -> List[str]
-        envs = [
-            'KEEPALIVED_AUTOCONF=false',
-            'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf',
-            'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf',
-            'KEEPALIVED_DEBUG=false'
-        ]
-        return envs
-
-    @staticmethod
-    def get_sysctl_settings() -> List[str]:
-        return [
-            '# IP forwarding and non-local bind',
-            'net.ipv4.ip_forward = 1',
-            'net.ipv4.ip_nonlocal_bind = 1',
-        ]
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        # better directory for this?
-        return extract_uid_gid(self.ctx, file_path='/var/lib')
-
-    @staticmethod
-    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
-        mounts = dict()
-        mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        envs.extend(self.get_container_envs())
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
-
-
 ##################################
 
 
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index dec915105052..ec94fc577127 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -1,4 +1,5 @@
 from .custom import CustomContainer
 from .tracing import Tracing
+from .ingress import HAproxy, Keepalived
 
-__all__ = ['CustomContainer', 'Tracing']
+__all__ = ['CustomContainer', 'Tracing', 'HAproxy', 'Keepalived']
diff --git a/src/cephadm/cephadmlib/daemons/ingress.py b/src/cephadm/cephadmlib/daemons/ingress.py
new file mode 100644
index 000000000000..94ee34505ed5
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/ingress.py
@@ -0,0 +1,268 @@
+import os
+
+from typing import Dict, List, Optional, Tuple, Union
+
+from ..constants import (
+    DEFAULT_HAPROXY_IMAGE,
+    DEFAULT_KEEPALIVED_IMAGE,
+    DATA_DIR_MODE,
+)
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context import CephadmContext
+from ..context_getters import fetch_configs
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..data_utils import dict_get, is_fsid
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import makedirs, populate_files
+
+
+@register_daemon_form
+class HAproxy(ContainerDaemonForm):
+    """Defines an HAproxy container"""
+    daemon_type = 'haproxy'
+    required_files = ['haproxy.cfg']
+    default_image = DEFAULT_HAPROXY_IMAGE
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx: CephadmContext,
+                 fsid: str, daemon_id: Union[int, str],
+                 config_json: Dict, image: str) -> None:
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        # config-json options
+        self.files = dict_get(config_json, 'files', {})
+
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx: CephadmContext,
+             fsid: str, daemon_id: Union[int, str]) -> 'HAproxy':
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx),
+                   ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'HAproxy':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        # create additional directories in data dir for HAproxy to use
+        if not os.path.isdir(os.path.join(data_dir, 'haproxy')):
+            makedirs(os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE)
+
+        data_dir = os.path.join(data_dir, 'haproxy')
+        populate_files(data_dir, self.files, uid, gid)
+
+    def get_daemon_args(self) -> List[str]:
+        return ['haproxy', '-f', '/var/lib/haproxy/haproxy.cfg']
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error('required file missing from config-json: %s' % fname)
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        # better directory for this?
+        print('UUUUU', extract_uid_gid)
+        return extract_uid_gid(self.ctx, file_path='/var/lib')
+
+    @staticmethod
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
+        mounts = dict()
+        mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# IP forwarding and non-local bind',
+            'net.ipv4.ip_forward = 1',
+            'net.ipv4.ip_nonlocal_bind = 1',
+        ]
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(
+            ['--user=root']
+        )  # haproxy 2.4 defaults to a different user
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+
+@register_daemon_form
+class Keepalived(ContainerDaemonForm):
+    """Defines an Keepalived container"""
+    daemon_type = 'keepalived'
+    required_files = ['keepalived.conf']
+    default_image = DEFAULT_KEEPALIVED_IMAGE
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx: CephadmContext,
+                 fsid: str, daemon_id: Union[int, str],
+                 config_json: Dict, image: str) -> None:
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        # config-json options
+        self.files = dict_get(config_json, 'files', {})
+
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx: CephadmContext, fsid: str,
+             daemon_id: Union[int, str]) -> 'Keepalived':
+        return cls(ctx, fsid, daemon_id,
+                   fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Keepalived':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        # create additional directories in data dir for keepalived to use
+        if not os.path.isdir(os.path.join(data_dir, 'keepalived')):
+            makedirs(os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE)
+
+        # populate files from the config-json
+        populate_files(data_dir, self.files, uid, gid)
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error('required file missing from config-json: %s' % fname)
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    @staticmethod
+    def get_container_envs():
+        # type: () -> List[str]
+        envs = [
+            'KEEPALIVED_AUTOCONF=false',
+            'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf',
+            'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf',
+            'KEEPALIVED_DEBUG=false'
+        ]
+        return envs
+
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# IP forwarding and non-local bind',
+            'net.ipv4.ip_forward = 1',
+            'net.ipv4.ip_nonlocal_bind = 1',
+        ]
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        # better directory for this?
+        return extract_uid_gid(self.ctx, file_path='/var/lib')
+
+    @staticmethod
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
+        mounts = dict()
+        mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])

From 3b752dde73732a37991ae12e57d6fd8d6ebec9e2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 19:34:58 -0500
Subject: [PATCH 0829/2492] cephadm: move nvmeof class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 173 +-------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   9 +-
 src/cephadm/cephadmlib/daemons/nvmeof.py   | 181 +++++++++++++++++++++
 3 files changed, 196 insertions(+), 167 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/nvmeof.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c588d661c211..8eac20593b08 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -41,7 +41,6 @@
     DEFAULT_IMAGE_RELEASE,
     DEFAULT_LOKI_IMAGE,
     DEFAULT_NODE_EXPORTER_IMAGE,
-    DEFAULT_NVMEOF_IMAGE,
     DEFAULT_PROMETHEUS_IMAGE,
     DEFAULT_PROMTAIL_IMAGE,
     DEFAULT_SNMP_GATEWAY_IMAGE,
@@ -171,7 +170,13 @@
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
 from cephadmlib.deployment_utils import to_deployment_container
-from cephadmlib.daemons import CustomContainer, Tracing, HAproxy, Keepalived
+from cephadmlib.daemons import (
+    CephNvmeof,
+    CustomContainer,
+    HAproxy,
+    Keepalived,
+    Tracing,
+)
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -1379,170 +1384,6 @@ def customize_container_args(
 ##################################
 
 
-@register_daemon_form
-class CephNvmeof(ContainerDaemonForm):
-    """Defines a Ceph-Nvmeof container"""
-
-    daemon_type = 'nvmeof'
-    required_files = ['ceph-nvmeof.conf']
-    default_image = DEFAULT_NVMEOF_IMAGE
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx,
-                 fsid,
-                 daemon_id,
-                 config_json,
-                 image=DEFAULT_NVMEOF_IMAGE):
-        # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        # config-json options
-        self.files = dict_get(config_json, 'files', {})
-
-        # validate the supplied args
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx, fsid, daemon_id):
-        # type: (CephadmContext, str, Union[int, str]) -> CephNvmeof
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephNvmeof':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    @staticmethod
-    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
-        mounts = dict()
-        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
-        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
-        mounts[os.path.join(data_dir, 'ceph-nvmeof.conf')] = '/src/ceph-nvmeof.conf:z'
-        mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
-        mounts['/dev/hugepages'] = '/dev/hugepages'
-        mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    def customize_container_binds(
-        self, ctx: CephadmContext, binds: List[List[str]]
-    ) -> None:
-        lib_modules = [
-            'type=bind',
-            'source=/lib/modules',
-            'destination=/lib/modules',
-            'ro=true',
-        ]
-        binds.append(lib_modules)
-
-    @staticmethod
-    def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
-        out, err, ret = call(ctx,
-                             [ctx.container_engine.path, 'inspect',
-                              '--format', '{{index .Config.Labels "io.ceph.version"}}',
-                              ctx.image])
-        version = None
-        if ret == 0:
-            version = out.strip()
-        return version
-
-    def validate(self):
-        # type: () -> None
-        if not is_fsid(self.fsid):
-            raise Error('not an fsid: %s' % self.fsid)
-        if not self.daemon_id:
-            raise Error('invalid daemon_id: %s' % self.daemon_id)
-        if not self.image:
-            raise Error('invalid image: %s' % self.image)
-
-        # check for the required files
-        if self.required_files:
-            for fname in self.required_files:
-                if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
-
-    def get_daemon_name(self):
-        # type: () -> str
-        return '%s.%s' % (self.daemon_type, self.daemon_id)
-
-    def get_container_name(self, desc=None):
-        # type: (Optional[str]) -> str
-        cname = '%s-%s' % (self.fsid, self.get_daemon_name())
-        if desc:
-            cname = '%s-%s' % (cname, desc)
-        return cname
-
-    def create_daemon_dirs(self, data_dir, uid, gid):
-        # type: (str, int, int) -> None
-        """Create files under the container data dir"""
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % (data_dir))
-
-        logger.info('Creating ceph-nvmeof config...')
-        configfs_dir = os.path.join(data_dir, 'configfs')
-        makedirs(configfs_dir, uid, gid, 0o755)
-
-        # populate files from the config-json
-        populate_files(data_dir, self.files, uid, gid)
-
-    @staticmethod
-    def configfs_mount_umount(data_dir, mount=True):
-        # type: (str, bool) -> List[str]
-        mount_path = os.path.join(data_dir, 'configfs')
-        if mount:
-            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
-                  'mount -t configfs none {0}; fi'.format(mount_path)
-        else:
-            cmd = 'if grep -qs {0} /proc/mounts; then ' \
-                  'umount {0}; fi'.format(mount_path)
-        return cmd.split()
-
-    @staticmethod
-    def get_sysctl_settings() -> List[str]:
-        return [
-            'vm.nr_hugepages = 4096',
-        ]
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return 167, 167  # TODO: need to get properly the uid/gid
-
-    def config_and_keyring(
-        self, ctx: CephadmContext
-    ) -> Tuple[Optional[str], Optional[str]]:
-        return get_config_and_keyring(ctx)
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(ctx.container_engine.unlimited_pids_option)
-        args.extend(['--ulimit', 'memlock=-1:-1'])
-        args.extend(['--ulimit', 'nofile=10240'])
-        args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
-
-
-##################################
-
-
 @register_daemon_form
 class CephExporter(ContainerDaemonForm):
     """Defines a Ceph exporter container"""
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index ec94fc577127..4a48f4da7cac 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -1,5 +1,12 @@
 from .custom import CustomContainer
 from .tracing import Tracing
 from .ingress import HAproxy, Keepalived
+from .nvmeof import CephNvmeof
 
-__all__ = ['CustomContainer', 'Tracing', 'HAproxy', 'Keepalived']
+__all__ = [
+    'CephNvmeof',
+    'CustomContainer',
+    'HAproxy',
+    'Keepalived',
+    'Tracing',
+]
diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
new file mode 100644
index 000000000000..a1a18f5a18ba
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -0,0 +1,181 @@
+import logging
+import os
+
+from typing import Dict, List, Optional, Tuple, Union
+
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer
+from ..context_getters import fetch_configs, get_config_and_keyring
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..constants import DEFAULT_NVMEOF_IMAGE
+from ..context import CephadmContext
+from ..data_utils import dict_get, is_fsid
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import makedirs, populate_files
+from ..call_wrappers import call
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class CephNvmeof(ContainerDaemonForm):
+    """Defines a Ceph-Nvmeof container"""
+
+    daemon_type = 'nvmeof'
+    required_files = ['ceph-nvmeof.conf']
+    default_image = DEFAULT_NVMEOF_IMAGE
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx,
+                 fsid,
+                 daemon_id,
+                 config_json,
+                 image=DEFAULT_NVMEOF_IMAGE):
+        # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        # config-json options
+        self.files = dict_get(config_json, 'files', {})
+
+        # validate the supplied args
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx, fsid, daemon_id):
+        # type: (CephadmContext, str, Union[int, str]) -> CephNvmeof
+        return cls(ctx, fsid, daemon_id,
+                   fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephNvmeof':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    @staticmethod
+    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
+        mounts = dict()
+        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+        mounts[os.path.join(data_dir, 'ceph-nvmeof.conf')] = '/src/ceph-nvmeof.conf:z'
+        mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
+        mounts['/dev/hugepages'] = '/dev/hugepages'
+        mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        lib_modules = [
+            'type=bind',
+            'source=/lib/modules',
+            'destination=/lib/modules',
+            'ro=true',
+        ]
+        binds.append(lib_modules)
+
+    @staticmethod
+    def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
+        out, err, ret = call(ctx,
+                             [ctx.container_engine.path, 'inspect',
+                              '--format', '{{index .Config.Labels "io.ceph.version"}}',
+                              ctx.image])
+        version = None
+        if ret == 0:
+            version = out.strip()
+        return version
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error('required file missing from config-json: %s' % fname)
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = '%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    def create_daemon_dirs(self, data_dir, uid, gid):
+        # type: (str, int, int) -> None
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        logger.info('Creating ceph-nvmeof config...')
+        configfs_dir = os.path.join(data_dir, 'configfs')
+        makedirs(configfs_dir, uid, gid, 0o755)
+
+        # populate files from the config-json
+        populate_files(data_dir, self.files, uid, gid)
+
+    @staticmethod
+    def configfs_mount_umount(data_dir, mount=True):
+        # type: (str, bool) -> List[str]
+        mount_path = os.path.join(data_dir, 'configfs')
+        if mount:
+            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
+                  'mount -t configfs none {0}; fi'.format(mount_path)
+        else:
+            cmd = 'if grep -qs {0} /proc/mounts; then ' \
+                  'umount {0}; fi'.format(mount_path)
+        return cmd.split()
+
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            'vm.nr_hugepages = 4096',
+        ]
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 167, 167  # TODO: need to get properly the uid/gid
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+        args.extend(['--ulimit', 'memlock=-1:-1'])
+        args.extend(['--ulimit', 'nofile=10240'])
+        args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])

From c093a1991d352e636429223f75b599bc4c311dd7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 8 Nov 2023 19:42:27 -0500
Subject: [PATCH 0830/2492] cephadm: move iscsi class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 241 +------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   2 +
 src/cephadm/cephadmlib/daemons/iscsi.py    | 257 +++++++++++++++++++++
 3 files changed, 261 insertions(+), 239 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/iscsi.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8eac20593b08..4ef7957effc0 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -138,7 +138,7 @@
     wrap_ipv6,
 )
 from cephadmlib.locking import FileLock
-from cephadmlib.daemon_identity import DaemonIdentity, DaemonSubIdentity
+from cephadmlib.daemon_identity import DaemonIdentity
 from cephadmlib.packagers import create_packager, Packager
 from cephadmlib.logging import cephadm_init_logging, Highlight, LogDestination
 from cephadmlib.systemd import check_unit, check_units
@@ -171,6 +171,7 @@
 from cephadmlib import templating
 from cephadmlib.deployment_utils import to_deployment_container
 from cephadmlib.daemons import (
+    CephIscsi,
     CephNvmeof,
     CustomContainer,
     HAproxy,
@@ -1146,244 +1147,6 @@ def default_entrypoint(self) -> str:
 ##################################
 
 
-@register_daemon_form
-class CephIscsi(ContainerDaemonForm):
-    """Defines a Ceph-Iscsi container"""
-
-    daemon_type = 'iscsi'
-    entrypoint = '/usr/bin/rbd-target-api'
-
-    required_files = ['iscsi-gateway.cfg']
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx: CephadmContext,
-                 ident: DaemonIdentity,
-                 config_json: Dict,
-                 image: str = DEFAULT_IMAGE):
-        self.ctx = ctx
-        self._identity = ident
-        self.image = image
-
-        # config-json options
-        self.files = dict_get(config_json, 'files', {})
-
-        # validate the supplied args
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str, daemon_id: str) -> 'CephIscsi':
-        return cls.create(ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id))
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephIscsi':
-        return cls(ctx, ident, fetch_configs(ctx), ctx.image)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return self._identity
-
-    @property
-    def fsid(self) -> str:
-        return self._identity.fsid
-
-    @property
-    def daemon_id(self) -> str:
-        return self._identity.daemon_id
-
-    @staticmethod
-    def _get_container_mounts(data_dir, log_dir):
-        # type: (str, str) -> Dict[str, str]
-        mounts = dict()
-        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
-        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
-        mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z'
-        mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
-        mounts[os.path.join(data_dir, 'tcmu-runner-entrypoint.sh')] = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
-        mounts[log_dir] = '/var/log:z'
-        mounts['/dev'] = '/dev'
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        # Removes ending ".tcmu" from data_dir a tcmu-runner uses the same
-        # data_dir as rbd-runner-api
-        if data_dir.endswith('.tcmu'):
-            data_dir = re.sub(r'\.tcmu$', '', data_dir)
-        log_dir = get_log_dir(self.identity.fsid, ctx.log_dir)
-        mounts.update(CephIscsi._get_container_mounts(data_dir, log_dir))
-
-    def customize_container_binds(
-        self, ctx: CephadmContext, binds: List[List[str]]
-    ) -> None:
-        lib_modules = [
-            'type=bind',
-            'source=/lib/modules',
-            'destination=/lib/modules',
-            'ro=true',
-        ]
-        binds.append(lib_modules)
-
-    @staticmethod
-    def get_version(ctx, container_id):
-        # type: (CephadmContext, str) -> Optional[str]
-        version = None
-        out, err, code = call(ctx,
-                              [ctx.container_engine.path, 'exec', container_id,
-                               '/usr/bin/python3', '-c',
-                               "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"],
-                              verbosity=CallVerbosity.QUIET)
-        if code == 0:
-            version = out.strip()
-        return version
-
-    def validate(self):
-        # type: () -> None
-        if not is_fsid(self.fsid):
-            raise Error('not an fsid: %s' % self.fsid)
-        if not self.daemon_id:
-            raise Error('invalid daemon_id: %s' % self.daemon_id)
-        if not self.image:
-            raise Error('invalid image: %s' % self.image)
-
-        # check for the required files
-        if self.required_files:
-            for fname in self.required_files:
-                if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
-
-    def get_daemon_name(self):
-        # type: () -> str
-        return '%s.%s' % (self.daemon_type, self.daemon_id)
-
-    def get_container_name(self, desc=None):
-        # type: (Optional[str]) -> str
-        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
-        if desc:
-            cname = '%s-%s' % (cname, desc)
-        return cname
-
-    def create_daemon_dirs(self, data_dir, uid, gid):
-        # type: (str, int, int) -> None
-        """Create files under the container data dir"""
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % (data_dir))
-
-        logger.info('Creating ceph-iscsi config...')
-        configfs_dir = os.path.join(data_dir, 'configfs')
-        makedirs(configfs_dir, uid, gid, 0o755)
-
-        # set up the tcmu-runner entrypoint script
-        # to be mounted into the container. For more info
-        # on why we need this script, see the
-        # tcmu_runner_entrypoint_script function
-        self.files['tcmu-runner-entrypoint.sh'] = self.tcmu_runner_entrypoint_script()
-
-        # populate files from the config-json
-        populate_files(data_dir, self.files, uid, gid)
-
-        # we want the tcmu runner entrypoint script to be executable
-        # populate_files will give it 0o600 by default
-        os.chmod(os.path.join(data_dir, 'tcmu-runner-entrypoint.sh'), 0o700)
-
-    @staticmethod
-    def configfs_mount_umount(data_dir, mount=True):
-        # type: (str, bool) -> List[str]
-        mount_path = os.path.join(data_dir, 'configfs')
-        if mount:
-            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
-                  'mount -t configfs none {0}; fi'.format(mount_path)
-        else:
-            cmd = 'if grep -qs {0} /proc/mounts; then ' \
-                  'umount {0}; fi'.format(mount_path)
-        return cmd.split()
-
-    @staticmethod
-    def tcmu_runner_entrypoint_script() -> str:
-        # since we are having tcmu-runner be a background
-        # process in its systemd unit (rbd-target-api being
-        # the main process) systemd will not restart it when
-        # it fails. in order to try and get around that for now
-        # we can have a script mounted in the container that
-        # that attempts to do the restarting for us. This script
-        # can then become the entrypoint for the tcmu-runner
-        # container
-
-        # This is intended to be dropped for a better solution
-        # for at least the squid release onward
-        return """#!/bin/bash
-RUN_DIR=/var/run/tcmu-runner
-
-if [ ! -d "${RUN_DIR}" ] ; then
-    mkdir -p "${RUN_DIR}"
-fi
-
-rm -rf "${RUN_DIR}"/*
-
-while true
-do
-    touch "${RUN_DIR}"/start-up-$(date -Ins)
-    /usr/bin/tcmu-runner
-
-    # If we got around 3 kills/segfaults in the last minute,
-    # don't start anymore
-    if [ $(find "${RUN_DIR}" -type f -cmin -1 | wc -l) -ge 3 ] ; then
-        exit 0
-    fi
-
-    sleep 1
-done
-"""
-
-    def get_tcmu_runner_container(self):
-        # type: () -> CephContainer
-        # daemon_id, is used to generated the cid and pid files used by podman but as both tcmu-runner
-        # and rbd-target-api have the same daemon_id, it conflits and prevent the second container from
-        # starting. .tcmu runner is appended to the daemon_id to fix that.
-        subident = DaemonSubIdentity(
-            self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
-        )
-        tcmu_dmn = self.create(self.ctx, subident)
-        tcmu_container = to_deployment_container(
-            self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True)
-        )
-        # TODO: Eventually we don't want to run tcmu-runner through this script.
-        # This is intended to be a workaround backported to older releases
-        # and should eventually be removed in at least squid onward
-        tcmu_container.entrypoint = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
-        tcmu_container.cname = self.get_container_name(desc='tcmu')
-        return tcmu_container
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        # So the container can modprobe iscsi_target_mod and have write perms
-        # to configfs we need to make this a privileged container.
-        ctr = daemon_to_container(ctx, self, privileged=True)
-        return to_deployment_container(ctx, ctr)
-
-    def config_and_keyring(
-        self, ctx: CephadmContext
-    ) -> Tuple[Optional[str], Optional[str]]:
-        return get_config_and_keyring(ctx)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return extract_uid_gid(ctx)
-
-    def default_entrypoint(self) -> str:
-        return self.entrypoint
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(ctx.container_engine.unlimited_pids_option)
-
-##################################
-
-
 @register_daemon_form
 class CephExporter(ContainerDaemonForm):
     """Defines a Ceph exporter container"""
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index 4a48f4da7cac..dbe01783e4c3 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -2,8 +2,10 @@
 from .tracing import Tracing
 from .ingress import HAproxy, Keepalived
 from .nvmeof import CephNvmeof
+from .iscsi import CephIscsi
 
 __all__ = [
+    'CephIscsi',
     'CephNvmeof',
     'CustomContainer',
     'HAproxy',
diff --git a/src/cephadm/cephadmlib/daemons/iscsi.py b/src/cephadm/cephadmlib/daemons/iscsi.py
new file mode 100644
index 000000000000..504db6885583
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/iscsi.py
@@ -0,0 +1,257 @@
+import logging
+import os
+import re
+
+from typing import Dict, List, Optional, Tuple
+
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context_getters import fetch_configs, get_config_and_keyring
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity, DaemonSubIdentity
+from ..constants import DEFAULT_IMAGE
+from ..context import CephadmContext
+from ..data_utils import dict_get, is_fsid
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import makedirs, populate_files
+from ..call_wrappers import call, CallVerbosity
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class CephIscsi(ContainerDaemonForm):
+    """Defines a Ceph-Iscsi container"""
+
+    daemon_type = 'iscsi'
+    entrypoint = '/usr/bin/rbd-target-api'
+
+    required_files = ['iscsi-gateway.cfg']
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx: CephadmContext,
+                 ident: DaemonIdentity,
+                 config_json: Dict,
+                 image: str = DEFAULT_IMAGE):
+        self.ctx = ctx
+        self._identity = ident
+        self.image = image
+
+        # config-json options
+        self.files = dict_get(config_json, 'files', {})
+
+        # validate the supplied args
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx: CephadmContext, fsid: str, daemon_id: str) -> 'CephIscsi':
+        return cls.create(ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id))
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephIscsi':
+        return cls(ctx, ident, fetch_configs(ctx), ctx.image)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    @property
+    def fsid(self) -> str:
+        return self._identity.fsid
+
+    @property
+    def daemon_id(self) -> str:
+        return self._identity.daemon_id
+
+    @staticmethod
+    def _get_container_mounts(data_dir, log_dir):
+        # type: (str, str) -> Dict[str, str]
+        mounts = dict()
+        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+        mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z'
+        mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
+        mounts[os.path.join(data_dir, 'tcmu-runner-entrypoint.sh')] = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
+        mounts[log_dir] = '/var/log:z'
+        mounts['/dev'] = '/dev'
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        # Removes ending ".tcmu" from data_dir a tcmu-runner uses the same
+        # data_dir as rbd-runner-api
+        if data_dir.endswith('.tcmu'):
+            data_dir = re.sub(r'\.tcmu$', '', data_dir)
+        log_dir = os.path.join(ctx.log_dir, self.identity.fsid)
+        mounts.update(CephIscsi._get_container_mounts(data_dir, log_dir))
+
+    def customize_container_binds(
+        self, ctx: CephadmContext, binds: List[List[str]]
+    ) -> None:
+        lib_modules = [
+            'type=bind',
+            'source=/lib/modules',
+            'destination=/lib/modules',
+            'ro=true',
+        ]
+        binds.append(lib_modules)
+
+    @staticmethod
+    def get_version(ctx, container_id):
+        # type: (CephadmContext, str) -> Optional[str]
+        version = None
+        out, err, code = call(ctx,
+                              [ctx.container_engine.path, 'exec', container_id,
+                               '/usr/bin/python3', '-c',
+                               "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"],
+                              verbosity=CallVerbosity.QUIET)
+        if code == 0:
+            version = out.strip()
+        return version
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error('required file missing from config-json: %s' % fname)
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    def create_daemon_dirs(self, data_dir, uid, gid):
+        # type: (str, int, int) -> None
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        logger.info('Creating ceph-iscsi config...')
+        configfs_dir = os.path.join(data_dir, 'configfs')
+        makedirs(configfs_dir, uid, gid, 0o755)
+
+        # set up the tcmu-runner entrypoint script
+        # to be mounted into the container. For more info
+        # on why we need this script, see the
+        # tcmu_runner_entrypoint_script function
+        self.files['tcmu-runner-entrypoint.sh'] = self.tcmu_runner_entrypoint_script()
+
+        # populate files from the config-json
+        populate_files(data_dir, self.files, uid, gid)
+
+        # we want the tcmu runner entrypoint script to be executable
+        # populate_files will give it 0o600 by default
+        os.chmod(os.path.join(data_dir, 'tcmu-runner-entrypoint.sh'), 0o700)
+
+    @staticmethod
+    def configfs_mount_umount(data_dir, mount=True):
+        # type: (str, bool) -> List[str]
+        mount_path = os.path.join(data_dir, 'configfs')
+        if mount:
+            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
+                  'mount -t configfs none {0}; fi'.format(mount_path)
+        else:
+            cmd = 'if grep -qs {0} /proc/mounts; then ' \
+                  'umount {0}; fi'.format(mount_path)
+        return cmd.split()
+
+    @staticmethod
+    def tcmu_runner_entrypoint_script() -> str:
+        # since we are having tcmu-runner be a background
+        # process in its systemd unit (rbd-target-api being
+        # the main process) systemd will not restart it when
+        # it fails. in order to try and get around that for now
+        # we can have a script mounted in the container that
+        # that attempts to do the restarting for us. This script
+        # can then become the entrypoint for the tcmu-runner
+        # container
+
+        # This is intended to be dropped for a better solution
+        # for at least the squid release onward
+        return """#!/bin/bash
+RUN_DIR=/var/run/tcmu-runner
+
+if [ ! -d "${RUN_DIR}" ] ; then
+    mkdir -p "${RUN_DIR}"
+fi
+
+rm -rf "${RUN_DIR}"/*
+
+while true
+do
+    touch "${RUN_DIR}"/start-up-$(date -Ins)
+    /usr/bin/tcmu-runner
+
+    # If we got around 3 kills/segfaults in the last minute,
+    # don't start anymore
+    if [ $(find "${RUN_DIR}" -type f -cmin -1 | wc -l) -ge 3 ] ; then
+        exit 0
+    fi
+
+    sleep 1
+done
+"""
+
+    def get_tcmu_runner_container(self):
+        # type: () -> CephContainer
+        # daemon_id, is used to generated the cid and pid files used by podman but as both tcmu-runner
+        # and rbd-target-api have the same daemon_id, it conflits and prevent the second container from
+        # starting. .tcmu runner is appended to the daemon_id to fix that.
+        subident = DaemonSubIdentity(
+            self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
+        )
+        tcmu_dmn = self.create(self.ctx, subident)
+        tcmu_container = to_deployment_container(
+            self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True)
+        )
+        # TODO: Eventually we don't want to run tcmu-runner through this script.
+        # This is intended to be a workaround backported to older releases
+        # and should eventually be removed in at least squid onward
+        tcmu_container.entrypoint = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
+        tcmu_container.cname = self.get_container_name(desc='tcmu')
+        return tcmu_container
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        # So the container can modprobe iscsi_target_mod and have write perms
+        # to configfs we need to make this a privileged container.
+        ctr = daemon_to_container(ctx, self, privileged=True)
+        return to_deployment_container(ctx, ctr)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return extract_uid_gid(ctx)
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)

From 0a6a1dfa966a560237387629a8be49bf4d61ebf9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 09:57:56 -0500
Subject: [PATCH 0831/2492] cephadm: move nfs class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 200 +------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   2 +
 src/cephadm/cephadmlib/daemons/nfs.py      | 218 +++++++++++++++++++++
 src/cephadm/tests/test_nfs.py              |   6 +-
 4 files changed, 225 insertions(+), 201 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/nfs.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 4ef7957effc0..75fe0104af58 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -100,7 +100,6 @@
     registry_login,
 )
 from cephadmlib.data_utils import (
-    dict_get,
     dict_get_join,
     get_legacy_config_fsid,
     is_fsid,
@@ -113,7 +112,6 @@
     get_file_timestamp,
     makedirs,
     pathify,
-    populate_files,
     read_file,
     recursive_chown,
     touch,
@@ -176,6 +174,7 @@
     CustomContainer,
     HAproxy,
     Keepalived,
+    NFSGanesha,
     Tracing,
 )
 
@@ -950,203 +949,6 @@ def default_entrypoint(self) -> str:
 ##################################
 
 
-@register_daemon_form
-class NFSGanesha(ContainerDaemonForm):
-    """Defines a NFS-Ganesha container"""
-
-    daemon_type = 'nfs'
-    entrypoint = '/usr/bin/ganesha.nfsd'
-    daemon_args = ['-F', '-L', 'STDERR']
-
-    required_files = ['ganesha.conf']
-
-    port_map = {
-        'nfs': 2049,
-    }
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx,
-                 fsid,
-                 daemon_id,
-                 config_json,
-                 image=DEFAULT_IMAGE):
-        # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        # config-json options
-        self.pool = dict_get(config_json, 'pool', require=True)
-        self.namespace = dict_get(config_json, 'namespace')
-        self.userid = dict_get(config_json, 'userid')
-        self.extra_args = dict_get(config_json, 'extra_args', [])
-        self.files = dict_get(config_json, 'files', {})
-        self.rgw = dict_get(config_json, 'rgw', {})
-
-        # validate the supplied args
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx, fsid, daemon_id):
-        # type: (CephadmContext, str, Union[int, str]) -> NFSGanesha
-        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'NFSGanesha':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    def _get_container_mounts(self, data_dir):
-        # type: (str) -> Dict[str, str]
-        mounts = dict()
-        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
-        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
-        mounts[os.path.join(data_dir, 'etc/ganesha')] = '/etc/ganesha:z'
-        if self.rgw:
-            cluster = self.rgw.get('cluster', 'ceph')
-            rgw_user = self.rgw.get('user', 'admin')
-            mounts[os.path.join(data_dir, 'keyring.rgw')] = \
-                '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    @staticmethod
-    def get_container_envs():
-        # type: () -> List[str]
-        envs = [
-            'CEPH_CONF=%s' % (CEPH_DEFAULT_CONF)
-        ]
-        return envs
-
-    @staticmethod
-    def get_version(ctx, container_id):
-        # type: (CephadmContext, str) -> Optional[str]
-        version = None
-        out, err, code = call(ctx,
-                              [ctx.container_engine.path, 'exec', container_id,
-                               NFSGanesha.entrypoint, '-v'],
-                              verbosity=CallVerbosity.QUIET)
-        if code == 0:
-            match = re.search(r'NFS-Ganesha Release\s*=\s*[V]*([\d.]+)', out)
-            if match:
-                version = match.group(1)
-        return version
-
-    def validate(self):
-        # type: () -> None
-        if not is_fsid(self.fsid):
-            raise Error('not an fsid: %s' % self.fsid)
-        if not self.daemon_id:
-            raise Error('invalid daemon_id: %s' % self.daemon_id)
-        if not self.image:
-            raise Error('invalid image: %s' % self.image)
-
-        # check for the required files
-        if self.required_files:
-            for fname in self.required_files:
-                if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
-
-        # check for an RGW config
-        if self.rgw:
-            if not self.rgw.get('keyring'):
-                raise Error('RGW keyring is missing')
-            if not self.rgw.get('user'):
-                raise Error('RGW user is missing')
-
-    def get_daemon_name(self):
-        # type: () -> str
-        return '%s.%s' % (self.daemon_type, self.daemon_id)
-
-    def get_container_name(self, desc=None):
-        # type: (Optional[str]) -> str
-        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
-        if desc:
-            cname = '%s-%s' % (cname, desc)
-        return cname
-
-    def get_daemon_args(self):
-        # type: () -> List[str]
-        return self.daemon_args + self.extra_args
-
-    def create_daemon_dirs(self, data_dir, uid, gid):
-        # type: (str, int, int) -> None
-        """Create files under the container data dir"""
-        if not os.path.isdir(data_dir):
-            raise OSError('data_dir is not a directory: %s' % (data_dir))
-
-        logger.info('Creating ganesha config...')
-
-        # create the ganesha conf dir
-        config_dir = os.path.join(data_dir, 'etc/ganesha')
-        makedirs(config_dir, uid, gid, 0o755)
-
-        # populate files from the config-json
-        populate_files(config_dir, self.files, uid, gid)
-
-        # write the RGW keyring
-        if self.rgw:
-            keyring_path = os.path.join(data_dir, 'keyring.rgw')
-            with write_new(keyring_path, owner=(uid, gid)) as f:
-                f.write(self.rgw.get('keyring', ''))
-
-    def firewall_service_name(self) -> str:
-        return 'nfs'
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def customize_container_endpoints(
-        self, endpoints: List[EndPoint], deployment_type: DeploymentType
-    ) -> None:
-        if deployment_type == DeploymentType.DEFAULT and not endpoints:
-            nfs_ports = list(NFSGanesha.port_map.values())
-            endpoints.extend([EndPoint('0.0.0.0', p) for p in nfs_ports])
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        # TODO: extract ganesha uid/gid (997, 994) ?
-        return extract_uid_gid(ctx)
-
-    def config_and_keyring(
-        self, ctx: CephadmContext
-    ) -> Tuple[Optional[str], Optional[str]]:
-        return get_config_and_keyring(ctx)
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        envs.extend(self.get_container_envs())
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_daemon_args())
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(ctx.container_engine.unlimited_pids_option)
-
-    def default_entrypoint(self) -> str:
-        return self.entrypoint
-
-##################################
-
-
 @register_daemon_form
 class CephExporter(ContainerDaemonForm):
     """Defines a Ceph exporter container"""
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index dbe01783e4c3..96d337b0c660 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -3,6 +3,7 @@
 from .ingress import HAproxy, Keepalived
 from .nvmeof import CephNvmeof
 from .iscsi import CephIscsi
+from .nfs import NFSGanesha
 
 __all__ = [
     'CephIscsi',
@@ -10,5 +11,6 @@
     'CustomContainer',
     'HAproxy',
     'Keepalived',
+    'NFSGanesha',
     'Tracing',
 ]
diff --git a/src/cephadm/cephadmlib/daemons/nfs.py b/src/cephadm/cephadmlib/daemons/nfs.py
new file mode 100644
index 000000000000..48653b775fb0
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/nfs.py
@@ -0,0 +1,218 @@
+import logging
+import os
+import re
+
+from typing import Dict, List, Optional, Tuple, Union
+
+from ..call_wrappers import call, CallVerbosity
+from ..constants import DEFAULT_IMAGE, CEPH_DEFAULT_CONF
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context import CephadmContext
+from ..context_getters import fetch_configs, get_config_and_keyring
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..data_utils import dict_get, is_fsid
+from ..deploy import DeploymentType
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import makedirs, populate_files, write_new
+from ..net_utils import EndPoint
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class NFSGanesha(ContainerDaemonForm):
+    """Defines a NFS-Ganesha container"""
+
+    daemon_type = 'nfs'
+    entrypoint = '/usr/bin/ganesha.nfsd'
+    daemon_args = ['-F', '-L', 'STDERR']
+
+    required_files = ['ganesha.conf']
+
+    port_map = {
+        'nfs': 2049,
+    }
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx,
+                 fsid,
+                 daemon_id,
+                 config_json,
+                 image=DEFAULT_IMAGE):
+        # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        # config-json options
+        self.pool = dict_get(config_json, 'pool', require=True)
+        self.namespace = dict_get(config_json, 'namespace')
+        self.userid = dict_get(config_json, 'userid')
+        self.extra_args = dict_get(config_json, 'extra_args', [])
+        self.files = dict_get(config_json, 'files', {})
+        self.rgw = dict_get(config_json, 'rgw', {})
+
+        # validate the supplied args
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx, fsid, daemon_id):
+        # type: (CephadmContext, str, Union[int, str]) -> NFSGanesha
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'NFSGanesha':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def _get_container_mounts(self, data_dir):
+        # type: (str) -> Dict[str, str]
+        mounts = dict()
+        mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+        mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+        mounts[os.path.join(data_dir, 'etc/ganesha')] = '/etc/ganesha:z'
+        if self.rgw:
+            cluster = self.rgw.get('cluster', 'ceph')
+            rgw_user = self.rgw.get('user', 'admin')
+            mounts[os.path.join(data_dir, 'keyring.rgw')] = \
+                '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    @staticmethod
+    def get_container_envs():
+        # type: () -> List[str]
+        envs = [
+            'CEPH_CONF=%s' % (CEPH_DEFAULT_CONF)
+        ]
+        return envs
+
+    @staticmethod
+    def get_version(ctx, container_id):
+        # type: (CephadmContext, str) -> Optional[str]
+        version = None
+        out, err, code = call(ctx,
+                              [ctx.container_engine.path, 'exec', container_id,
+                               NFSGanesha.entrypoint, '-v'],
+                              verbosity=CallVerbosity.QUIET)
+        if code == 0:
+            match = re.search(r'NFS-Ganesha Release\s*=\s*[V]*([\d.]+)', out)
+            if match:
+                version = match.group(1)
+        return version
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error('required file missing from config-json: %s' % fname)
+
+        # check for an RGW config
+        if self.rgw:
+            if not self.rgw.get('keyring'):
+                raise Error('RGW keyring is missing')
+            if not self.rgw.get('user'):
+                raise Error('RGW user is missing')
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    def get_daemon_args(self):
+        # type: () -> List[str]
+        return self.daemon_args + self.extra_args
+
+    def create_daemon_dirs(self, data_dir, uid, gid):
+        # type: (str, int, int) -> None
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        logger.info('Creating ganesha config...')
+
+        # create the ganesha conf dir
+        config_dir = os.path.join(data_dir, 'etc/ganesha')
+        makedirs(config_dir, uid, gid, 0o755)
+
+        # populate files from the config-json
+        populate_files(config_dir, self.files, uid, gid)
+
+        # write the RGW keyring
+        if self.rgw:
+            keyring_path = os.path.join(data_dir, 'keyring.rgw')
+            with write_new(keyring_path, owner=(uid, gid)) as f:
+                f.write(self.rgw.get('keyring', ''))
+
+    def firewall_service_name(self) -> str:
+        return 'nfs'
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def customize_container_endpoints(
+        self, endpoints: List[EndPoint], deployment_type: DeploymentType
+    ) -> None:
+        if deployment_type == DeploymentType.DEFAULT and not endpoints:
+            nfs_ports = list(NFSGanesha.port_map.values())
+            endpoints.extend([EndPoint('0.0.0.0', p) for p in nfs_ports])
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        # TODO: extract ganesha uid/gid (997, 994) ?
+        return extract_uid_gid(ctx)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.extend(self.get_container_envs())
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
diff --git a/src/cephadm/tests/test_nfs.py b/src/cephadm/tests/test_nfs.py
index 94ab6afcfdf9..aae8113382dc 100644
--- a/src/cephadm/tests/test_nfs.py
+++ b/src/cephadm/tests/test_nfs.py
@@ -155,15 +155,17 @@ def test_nfsganesha_container_envs():
 
 
 def test_nfsganesha_get_version():
+    from cephadmlib.daemons import nfs
+
     with with_cephadm_ctx([]) as ctx:
-        nfsg = _cephadm.NFSGanesha(
+        nfsg = nfs.NFSGanesha(
             ctx,
             SAMPLE_UUID,
             "fred",
             good_nfs_json(),
         )
 
-        with mock.patch("cephadm.call") as _call:
+        with mock.patch("cephadmlib.daemons.nfs.call") as _call:
             _call.return_value = ("NFS-Ganesha Release = V100", "", 0)
             ver = nfsg.get_version(ctx, "fake_version")
             _call.assert_called()

From 643dd5d0f63985715d4b843218990ccca6187bb9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:02:20 -0500
Subject: [PATCH 0832/2492] cephadm: move monitoring class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                       | 336 +-----------------
 src/cephadm/cephadmlib/daemons/__init__.py   |   2 +
 src/cephadm/cephadmlib/daemons/monitoring.py | 350 +++++++++++++++++++
 src/cephadm/tests/test_cephadm.py            |  18 +-
 src/cephadm/tests/test_deploy.py             |   9 +-
 5 files changed, 370 insertions(+), 345 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/monitoring.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 75fe0104af58..ee8182afd144 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -34,15 +34,9 @@
 
 from cephadmlib.constants import (
     # default images
-    DEFAULT_ALERT_MANAGER_IMAGE,
-    DEFAULT_GRAFANA_IMAGE,
     DEFAULT_IMAGE,
     DEFAULT_IMAGE_IS_MAIN,
     DEFAULT_IMAGE_RELEASE,
-    DEFAULT_LOKI_IMAGE,
-    DEFAULT_NODE_EXPORTER_IMAGE,
-    DEFAULT_PROMETHEUS_IMAGE,
-    DEFAULT_PROMTAIL_IMAGE,
     DEFAULT_SNMP_GATEWAY_IMAGE,
     # other constant values
     CEPH_CONF,
@@ -174,6 +168,7 @@
     CustomContainer,
     HAproxy,
     Keepalived,
+    Monitoring,
     NFSGanesha,
     Tracing,
 )
@@ -617,335 +612,6 @@ def customize_process_args(
     ) -> None:
         args.extend(self.get_daemon_args())
 
-
-##################################
-@register_daemon_form
-class Monitoring(ContainerDaemonForm):
-    """Define the configs for the monitoring containers"""
-
-    port_map = {
-        'prometheus': [9095],  # Avoid default 9090, due to conflict with cockpit UI
-        'node-exporter': [9100],
-        'grafana': [3000],
-        'alertmanager': [9093, 9094],
-        'loki': [3100],
-        'promtail': [9080]
-    }
-
-    components = {
-        'prometheus': {
-            'image': DEFAULT_PROMETHEUS_IMAGE,
-            'cpus': '2',
-            'memory': '4GB',
-            'args': [
-                '--config.file=/etc/prometheus/prometheus.yml',
-                '--storage.tsdb.path=/prometheus',
-            ],
-            'config-json-files': [
-                'prometheus.yml',
-            ],
-        },
-        'loki': {
-            'image': DEFAULT_LOKI_IMAGE,
-            'cpus': '1',
-            'memory': '1GB',
-            'args': [
-                '--config.file=/etc/loki/loki.yml',
-            ],
-            'config-json-files': [
-                'loki.yml'
-            ],
-        },
-        'promtail': {
-            'image': DEFAULT_PROMTAIL_IMAGE,
-            'cpus': '1',
-            'memory': '1GB',
-            'args': [
-                '--config.file=/etc/promtail/promtail.yml',
-            ],
-            'config-json-files': [
-                'promtail.yml',
-            ],
-        },
-        'node-exporter': {
-            'image': DEFAULT_NODE_EXPORTER_IMAGE,
-            'cpus': '1',
-            'memory': '1GB',
-            'args': [
-                '--no-collector.timex'
-            ],
-        },
-        'grafana': {
-            'image': DEFAULT_GRAFANA_IMAGE,
-            'cpus': '2',
-            'memory': '4GB',
-            'args': [],
-            'config-json-files': [
-                'grafana.ini',
-                'provisioning/datasources/ceph-dashboard.yml',
-                'certs/cert_file',
-                'certs/cert_key',
-            ],
-        },
-        'alertmanager': {
-            'image': DEFAULT_ALERT_MANAGER_IMAGE,
-            'cpus': '2',
-            'memory': '2GB',
-            'args': [
-                '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]),
-            ],
-            'config-json-files': [
-                'alertmanager.yml',
-            ],
-            'config-json-args': [
-                'peers',
-            ],
-        },
-    }  # type: ignore
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return daemon_type in cls.components
-
-    @staticmethod
-    def get_version(ctx, container_id, daemon_type):
-        # type: (CephadmContext, str, str) -> str
-        """
-        :param: daemon_type Either "prometheus", "alertmanager", "loki", "promtail" or "node-exporter"
-        """
-        assert daemon_type in ('prometheus', 'alertmanager', 'node-exporter', 'loki', 'promtail')
-        cmd = daemon_type.replace('-', '_')
-        code = -1
-        err = ''
-        out = ''
-        version = ''
-        if daemon_type == 'alertmanager':
-            for cmd in ['alertmanager', 'prometheus-alertmanager']:
-                out, err, code = call(ctx, [
-                    ctx.container_engine.path, 'exec', container_id, cmd,
-                    '--version'
-                ], verbosity=CallVerbosity.QUIET)
-                if code == 0:
-                    break
-            cmd = 'alertmanager'  # reset cmd for version extraction
-        else:
-            out, err, code = call(ctx, [
-                ctx.container_engine.path, 'exec', container_id, cmd, '--version'
-            ], verbosity=CallVerbosity.QUIET)
-        if code == 0:
-            if err.startswith('%s, version ' % cmd):
-                version = err.split(' ')[2]
-            elif out.startswith('%s, version ' % cmd):
-                version = out.split(' ')[2]
-        return version
-
-    @staticmethod
-    def extract_uid_gid(
-        ctx: CephadmContext, daemon_type: str
-    ) -> Tuple[int, int]:
-        if daemon_type == 'prometheus':
-            uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus')
-        elif daemon_type == 'node-exporter':
-            uid, gid = 65534, 65534
-        elif daemon_type == 'grafana':
-            uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana')
-        elif daemon_type == 'loki':
-            uid, gid = extract_uid_gid(ctx, file_path='/etc/loki')
-        elif daemon_type == 'promtail':
-            uid, gid = extract_uid_gid(ctx, file_path='/etc/promtail')
-        elif daemon_type == 'alertmanager':
-            uid, gid = extract_uid_gid(
-                ctx, file_path=['/etc/alertmanager', '/etc/prometheus']
-            )
-        else:
-            raise Error('{} not implemented yet'.format(daemon_type))
-        return uid, gid
-
-    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
-        self.ctx = ctx
-        self._identity = ident
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Monitoring':
-        return cls(ctx, ident)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return self._identity
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        self._prevalidate(ctx)
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return self.extract_uid_gid(ctx, self.identity.daemon_type)
-
-    def _prevalidate(self, ctx: CephadmContext) -> None:
-        # before being refactored into a ContainerDaemonForm these checks were
-        # done inside the deploy function. This was the only "family" of daemons
-        # that performed these checks in that location
-        daemon_type = self.identity.daemon_type
-        config = fetch_configs(ctx)  # type: ignore
-        required_files = self.components[daemon_type].get(
-            'config-json-files', list()
-        )
-        required_args = self.components[daemon_type].get(
-            'config-json-args', list()
-        )
-        if required_files:
-            if not config or not all(c in config.get('files', {}).keys() for c in required_files):  # type: ignore
-                raise Error(
-                    '{} deployment requires config-json which must '
-                    'contain file content for {}'.format(
-                        daemon_type.capitalize(), ', '.join(required_files)
-                    )
-                )
-        if required_args:
-            if not config or not all(c in config.keys() for c in required_args):  # type: ignore
-                raise Error(
-                    '{} deployment requires config-json which must '
-                    'contain arg for {}'.format(
-                        daemon_type.capitalize(), ', '.join(required_args)
-                    )
-                )
-
-    def get_daemon_args(self) -> List[str]:
-        ctx = self.ctx
-        daemon_type = self.identity.daemon_type
-        metadata = self.components[daemon_type]
-        r = list(metadata.get('args', []))
-        # set ip and port to bind to for nodeexporter,alertmanager,prometheus
-        if daemon_type not in ['grafana', 'loki', 'promtail']:
-            ip = ''
-            port = self.port_map[daemon_type][0]
-            meta = fetch_meta(ctx)
-            if meta:
-                if 'ip' in meta and meta['ip']:
-                    ip = meta['ip']
-                if 'ports' in meta and meta['ports']:
-                    port = meta['ports'][0]
-            r += [f'--web.listen-address={ip}:{port}']
-            if daemon_type == 'prometheus':
-                config = fetch_configs(ctx)
-                retention_time = config.get('retention_time', '15d')
-                retention_size = config.get('retention_size', '0')  # default to disabled
-                r += [f'--storage.tsdb.retention.time={retention_time}']
-                r += [f'--storage.tsdb.retention.size={retention_size}']
-                scheme = 'http'
-                host = get_fqdn()
-                # in case host is not an fqdn then we use the IP to
-                # avoid producing a broken web.external-url link
-                if '.' not in host:
-                    ipv4_addrs, ipv6_addrs = get_ip_addresses(get_hostname())
-                    # use the first ipv4 (if any) otherwise use the first ipv6
-                    addr = next(iter(ipv4_addrs or ipv6_addrs), None)
-                    host = wrap_ipv6(addr) if addr else host
-                r += [f'--web.external-url={scheme}://{host}:{port}']
-        if daemon_type == 'alertmanager':
-            config = fetch_configs(ctx)
-            peers = config.get('peers', list())  # type: ignore
-            for peer in peers:
-                r += ['--cluster.peer={}'.format(peer)]
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-            # some alertmanager, by default, look elsewhere for a config
-            r += ['--config.file=/etc/alertmanager/alertmanager.yml']
-        if daemon_type == 'promtail':
-            r += ['--config.expand-env']
-        if daemon_type == 'prometheus':
-            config = fetch_configs(ctx)
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-        if daemon_type == 'node-exporter':
-            config = fetch_configs(ctx)
-            try:
-                r += [f'--web.config.file={config["web_config"]}']
-            except KeyError:
-                pass
-            r += ['--path.procfs=/host/proc',
-                  '--path.sysfs=/host/sys',
-                  '--path.rootfs=/rootfs']
-        return r
-
-    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
-        ctx = self.ctx
-        daemon_type = self.identity.daemon_type
-        mounts: Dict[str, str] = {}
-        log_dir = get_log_dir(self.identity.fsid, ctx.log_dir)
-        if daemon_type == 'prometheus':
-            mounts[
-                os.path.join(data_dir, 'etc/prometheus')
-            ] = '/etc/prometheus:Z'
-            mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z'
-        elif daemon_type == 'loki':
-            mounts[os.path.join(data_dir, 'etc/loki')] = '/etc/loki:Z'
-            mounts[os.path.join(data_dir, 'data')] = '/loki:Z'
-        elif daemon_type == 'promtail':
-            mounts[os.path.join(data_dir, 'etc/promtail')] = '/etc/promtail:Z'
-            mounts[log_dir] = '/var/log/ceph:z'
-            mounts[os.path.join(data_dir, 'data')] = '/promtail:Z'
-        elif daemon_type == 'node-exporter':
-            mounts[
-                os.path.join(data_dir, 'etc/node-exporter')
-            ] = '/etc/node-exporter:Z'
-            mounts['/proc'] = '/host/proc:ro'
-            mounts['/sys'] = '/host/sys:ro'
-            mounts['/'] = '/rootfs:ro'
-        elif daemon_type == 'grafana':
-            mounts[
-                os.path.join(data_dir, 'etc/grafana/grafana.ini')
-            ] = '/etc/grafana/grafana.ini:Z'
-            mounts[
-                os.path.join(data_dir, 'etc/grafana/provisioning/datasources')
-            ] = '/etc/grafana/provisioning/datasources:Z'
-            mounts[
-                os.path.join(data_dir, 'etc/grafana/certs')
-            ] = '/etc/grafana/certs:Z'
-            mounts[
-                os.path.join(data_dir, 'data/grafana.db')
-            ] = '/var/lib/grafana/grafana.db:Z'
-        elif daemon_type == 'alertmanager':
-            mounts[
-                os.path.join(data_dir, 'etc/alertmanager')
-            ] = '/etc/alertmanager:Z'
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        uid, _ = self.uid_gid(ctx)
-        monitoring_args = [
-            '--user',
-            str(uid),
-            # FIXME: disable cpu/memory limits for the time being (not supported
-            # by ubuntu 18.04 kernel!)
-        ]
-        args.extend(monitoring_args)
-        if self.identity.daemon_type == 'node-exporter':
-            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
-            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
-            # between the node-exporter container and the host to avoid selinux denials
-            args.extend(['--security-opt', 'label=disable'])
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_daemon_args())
-
-    def default_entrypoint(self) -> str:
-        return ''
-
 ##################################
 
 
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index 96d337b0c660..a163e4a38293 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -4,6 +4,7 @@
 from .nvmeof import CephNvmeof
 from .iscsi import CephIscsi
 from .nfs import NFSGanesha
+from .monitoring import Monitoring
 
 __all__ = [
     'CephIscsi',
@@ -11,6 +12,7 @@
     'CustomContainer',
     'HAproxy',
     'Keepalived',
+    'Monitoring',
     'NFSGanesha',
     'Tracing',
 ]
diff --git a/src/cephadm/cephadmlib/daemons/monitoring.py b/src/cephadm/cephadmlib/daemons/monitoring.py
new file mode 100644
index 000000000000..405dafc6dfcc
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/monitoring.py
@@ -0,0 +1,350 @@
+import os
+
+from typing import Dict, List, Tuple
+
+from ..call_wrappers import call, CallVerbosity
+from ..constants import (
+    DEFAULT_ALERT_MANAGER_IMAGE,
+    DEFAULT_GRAFANA_IMAGE,
+    DEFAULT_LOKI_IMAGE,
+    DEFAULT_NODE_EXPORTER_IMAGE,
+    DEFAULT_PROMETHEUS_IMAGE,
+    DEFAULT_PROMTAIL_IMAGE,
+)
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context import CephadmContext
+from ..context_getters import fetch_configs, fetch_meta
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..net_utils import get_fqdn, get_hostname, get_ip_addresses, wrap_ipv6
+
+
+@register_daemon_form
+class Monitoring(ContainerDaemonForm):
+    """Define the configs for the monitoring containers"""
+
+    port_map = {
+        'prometheus': [9095],  # Avoid default 9090, due to conflict with cockpit UI
+        'node-exporter': [9100],
+        'grafana': [3000],
+        'alertmanager': [9093, 9094],
+        'loki': [3100],
+        'promtail': [9080]
+    }
+
+    components = {
+        'prometheus': {
+            'image': DEFAULT_PROMETHEUS_IMAGE,
+            'cpus': '2',
+            'memory': '4GB',
+            'args': [
+                '--config.file=/etc/prometheus/prometheus.yml',
+                '--storage.tsdb.path=/prometheus',
+            ],
+            'config-json-files': [
+                'prometheus.yml',
+            ],
+        },
+        'loki': {
+            'image': DEFAULT_LOKI_IMAGE,
+            'cpus': '1',
+            'memory': '1GB',
+            'args': [
+                '--config.file=/etc/loki/loki.yml',
+            ],
+            'config-json-files': [
+                'loki.yml'
+            ],
+        },
+        'promtail': {
+            'image': DEFAULT_PROMTAIL_IMAGE,
+            'cpus': '1',
+            'memory': '1GB',
+            'args': [
+                '--config.file=/etc/promtail/promtail.yml',
+            ],
+            'config-json-files': [
+                'promtail.yml',
+            ],
+        },
+        'node-exporter': {
+            'image': DEFAULT_NODE_EXPORTER_IMAGE,
+            'cpus': '1',
+            'memory': '1GB',
+            'args': [
+                '--no-collector.timex'
+            ],
+        },
+        'grafana': {
+            'image': DEFAULT_GRAFANA_IMAGE,
+            'cpus': '2',
+            'memory': '4GB',
+            'args': [],
+            'config-json-files': [
+                'grafana.ini',
+                'provisioning/datasources/ceph-dashboard.yml',
+                'certs/cert_file',
+                'certs/cert_key',
+            ],
+        },
+        'alertmanager': {
+            'image': DEFAULT_ALERT_MANAGER_IMAGE,
+            'cpus': '2',
+            'memory': '2GB',
+            'args': [
+                '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]),
+            ],
+            'config-json-files': [
+                'alertmanager.yml',
+            ],
+            'config-json-args': [
+                'peers',
+            ],
+        },
+    }  # type: ignore
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return daemon_type in cls.components
+
+    @staticmethod
+    def get_version(ctx, container_id, daemon_type):
+        # type: (CephadmContext, str, str) -> str
+        """
+        :param: daemon_type Either "prometheus", "alertmanager", "loki", "promtail" or "node-exporter"
+        """
+        assert daemon_type in ('prometheus', 'alertmanager', 'node-exporter', 'loki', 'promtail')
+        cmd = daemon_type.replace('-', '_')
+        code = -1
+        err = ''
+        out = ''
+        version = ''
+        if daemon_type == 'alertmanager':
+            for cmd in ['alertmanager', 'prometheus-alertmanager']:
+                out, err, code = call(ctx, [
+                    ctx.container_engine.path, 'exec', container_id, cmd,
+                    '--version'
+                ], verbosity=CallVerbosity.QUIET)
+                if code == 0:
+                    break
+            cmd = 'alertmanager'  # reset cmd for version extraction
+        else:
+            out, err, code = call(ctx, [
+                ctx.container_engine.path, 'exec', container_id, cmd, '--version'
+            ], verbosity=CallVerbosity.QUIET)
+        if code == 0:
+            if err.startswith('%s, version ' % cmd):
+                version = err.split(' ')[2]
+            elif out.startswith('%s, version ' % cmd):
+                version = out.split(' ')[2]
+        return version
+
+    @staticmethod
+    def extract_uid_gid(
+        ctx: CephadmContext, daemon_type: str
+    ) -> Tuple[int, int]:
+        if daemon_type == 'prometheus':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus')
+        elif daemon_type == 'node-exporter':
+            uid, gid = 65534, 65534
+        elif daemon_type == 'grafana':
+            uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana')
+        elif daemon_type == 'loki':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/loki')
+        elif daemon_type == 'promtail':
+            uid, gid = extract_uid_gid(ctx, file_path='/etc/promtail')
+        elif daemon_type == 'alertmanager':
+            uid, gid = extract_uid_gid(
+                ctx, file_path=['/etc/alertmanager', '/etc/prometheus']
+            )
+        else:
+            raise Error('{} not implemented yet'.format(daemon_type))
+        return uid, gid
+
+    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
+        self.ctx = ctx
+        self._identity = ident
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Monitoring':
+        return cls(ctx, ident)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        self._prevalidate(ctx)
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return self.extract_uid_gid(ctx, self.identity.daemon_type)
+
+    def _prevalidate(self, ctx: CephadmContext) -> None:
+        # before being refactored into a ContainerDaemonForm these checks were
+        # done inside the deploy function. This was the only "family" of daemons
+        # that performed these checks in that location
+        daemon_type = self.identity.daemon_type
+        config = fetch_configs(ctx)  # type: ignore
+        required_files = self.components[daemon_type].get(
+            'config-json-files', list()
+        )
+        required_args = self.components[daemon_type].get(
+            'config-json-args', list()
+        )
+        if required_files:
+            if not config or not all(c in config.get('files', {}).keys() for c in required_files):  # type: ignore
+                raise Error(
+                    '{} deployment requires config-json which must '
+                    'contain file content for {}'.format(
+                        daemon_type.capitalize(), ', '.join(required_files)
+                    )
+                )
+        if required_args:
+            if not config or not all(c in config.keys() for c in required_args):  # type: ignore
+                raise Error(
+                    '{} deployment requires config-json which must '
+                    'contain arg for {}'.format(
+                        daemon_type.capitalize(), ', '.join(required_args)
+                    )
+                )
+
+    def get_daemon_args(self) -> List[str]:
+        ctx = self.ctx
+        daemon_type = self.identity.daemon_type
+        metadata = self.components[daemon_type]
+        r = list(metadata.get('args', []))
+        # set ip and port to bind to for nodeexporter,alertmanager,prometheus
+        if daemon_type not in ['grafana', 'loki', 'promtail']:
+            ip = ''
+            port = self.port_map[daemon_type][0]
+            meta = fetch_meta(ctx)
+            if meta:
+                if 'ip' in meta and meta['ip']:
+                    ip = meta['ip']
+                if 'ports' in meta and meta['ports']:
+                    port = meta['ports'][0]
+            r += [f'--web.listen-address={ip}:{port}']
+            if daemon_type == 'prometheus':
+                config = fetch_configs(ctx)
+                retention_time = config.get('retention_time', '15d')
+                retention_size = config.get('retention_size', '0')  # default to disabled
+                r += [f'--storage.tsdb.retention.time={retention_time}']
+                r += [f'--storage.tsdb.retention.size={retention_size}']
+                scheme = 'http'
+                host = get_fqdn()
+                # in case host is not an fqdn then we use the IP to
+                # avoid producing a broken web.external-url link
+                if '.' not in host:
+                    ipv4_addrs, ipv6_addrs = get_ip_addresses(get_hostname())
+                    # use the first ipv4 (if any) otherwise use the first ipv6
+                    addr = next(iter(ipv4_addrs or ipv6_addrs), None)
+                    host = wrap_ipv6(addr) if addr else host
+                r += [f'--web.external-url={scheme}://{host}:{port}']
+        if daemon_type == 'alertmanager':
+            config = fetch_configs(ctx)
+            peers = config.get('peers', list())  # type: ignore
+            for peer in peers:
+                r += ['--cluster.peer={}'.format(peer)]
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+            # some alertmanager, by default, look elsewhere for a config
+            r += ['--config.file=/etc/alertmanager/alertmanager.yml']
+        if daemon_type == 'promtail':
+            r += ['--config.expand-env']
+        if daemon_type == 'prometheus':
+            config = fetch_configs(ctx)
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+        if daemon_type == 'node-exporter':
+            config = fetch_configs(ctx)
+            try:
+                r += [f'--web.config.file={config["web_config"]}']
+            except KeyError:
+                pass
+            r += ['--path.procfs=/host/proc',
+                  '--path.sysfs=/host/sys',
+                  '--path.rootfs=/rootfs']
+        return r
+
+    def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+        ctx = self.ctx
+        daemon_type = self.identity.daemon_type
+        mounts: Dict[str, str] = {}
+        log_dir = os.path.join(ctx.log_dir, self.identity.fsid)
+        if daemon_type == 'prometheus':
+            mounts[
+                os.path.join(data_dir, 'etc/prometheus')
+            ] = '/etc/prometheus:Z'
+            mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z'
+        elif daemon_type == 'loki':
+            mounts[os.path.join(data_dir, 'etc/loki')] = '/etc/loki:Z'
+            mounts[os.path.join(data_dir, 'data')] = '/loki:Z'
+        elif daemon_type == 'promtail':
+            mounts[os.path.join(data_dir, 'etc/promtail')] = '/etc/promtail:Z'
+            mounts[log_dir] = '/var/log/ceph:z'
+            mounts[os.path.join(data_dir, 'data')] = '/promtail:Z'
+        elif daemon_type == 'node-exporter':
+            mounts[
+                os.path.join(data_dir, 'etc/node-exporter')
+            ] = '/etc/node-exporter:Z'
+            mounts['/proc'] = '/host/proc:ro'
+            mounts['/sys'] = '/host/sys:ro'
+            mounts['/'] = '/rootfs:ro'
+        elif daemon_type == 'grafana':
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/grafana.ini')
+            ] = '/etc/grafana/grafana.ini:Z'
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/provisioning/datasources')
+            ] = '/etc/grafana/provisioning/datasources:Z'
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/certs')
+            ] = '/etc/grafana/certs:Z'
+            mounts[
+                os.path.join(data_dir, 'data/grafana.db')
+            ] = '/var/lib/grafana/grafana.db:Z'
+        elif daemon_type == 'alertmanager':
+            mounts[
+                os.path.join(data_dir, 'etc/alertmanager')
+            ] = '/etc/alertmanager:Z'
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        mounts.update(self._get_container_mounts(data_dir))
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        uid, _ = self.uid_gid(ctx)
+        monitoring_args = [
+            '--user',
+            str(uid),
+            # FIXME: disable cpu/memory limits for the time being (not supported
+            # by ubuntu 18.04 kernel!)
+        ]
+        args.extend(monitoring_args)
+        if self.identity.daemon_type == 'node-exporter':
+            # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
+            # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
+            # between the node-exporter container and the host to avoid selinux denials
+            args.extend(['--security-opt', 'label=disable'])
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())
+
+    def default_entrypoint(self) -> str:
+        return ''
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 899272cb4143..b2e395fab64e 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1211,15 +1211,17 @@ def test_exit_failure_2(self, _target_exists, _target_state, _logger, _call, _li
 
 
 class TestMonitoring(object):
-    @mock.patch('cephadm.call')
+    @mock.patch('cephadmlib.daemons.monitoring.call')
     def test_get_version_alertmanager(self, _call):
+        from cephadmlib.daemons import monitoring
+
         ctx = _cephadm.CephadmContext()
         ctx.container_engine = mock_podman()
         daemon_type = 'alertmanager'
 
         # binary `prometheus`
         _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0
-        version = _cephadm.Monitoring.get_version(ctx, 'container_id', daemon_type)
+        version = monitoring.Monitoring.get_version(ctx, 'container_id', daemon_type)
         assert version == '0.16.1'
 
         # binary `prometheus-alertmanager`
@@ -1230,13 +1232,15 @@ def test_get_version_alertmanager(self, _call):
         version = _cephadm.Monitoring.get_version(ctx, 'container_id', daemon_type)
         assert version == '0.16.1'
 
-    @mock.patch('cephadm.call')
+    @mock.patch('cephadmlib.daemons.monitoring.call')
     def test_get_version_prometheus(self, _call):
+        from cephadmlib.daemons import monitoring
+
         ctx = _cephadm.CephadmContext()
         ctx.container_engine = mock_podman()
         daemon_type = 'prometheus'
         _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0
-        version = _cephadm.Monitoring.get_version(ctx, 'container_id', daemon_type)
+        version = monitoring.Monitoring.get_version(ctx, 'container_id', daemon_type)
         assert version == '0.16.1'
 
     def test_prometheus_external_url(self):
@@ -1250,13 +1254,15 @@ def test_prometheus_external_url(self):
         ).get_daemon_args()
         assert any([x.startswith('--web.external-url=http://') for x in args])
 
-    @mock.patch('cephadm.call')
+    @mock.patch('cephadmlib.daemons.monitoring.call')
     def test_get_version_node_exporter(self, _call):
+        from cephadmlib.daemons import monitoring
+
         ctx = _cephadm.CephadmContext()
         ctx.container_engine = mock_podman()
         daemon_type = 'node-exporter'
         _call.return_value = '', '{}, version 0.16.1'.format(daemon_type.replace('-', '_')), 0
-        version = _cephadm.Monitoring.get_version(ctx, 'container_id', daemon_type)
+        version = monitoring.Monitoring.get_version(ctx, 'container_id', daemon_type)
         assert version == '0.16.1'
 
     def test_create_daemon_dirs_prometheus(self, cephadm_fs):
diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 9d82b2055b91..5d5b46ad6954 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -9,6 +9,7 @@
     mock_podman,
     with_cephadm_ctx,
     FunkyPatcher,
+    funkypatch,
 )
 
 
@@ -290,11 +291,11 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
         assert (si.st_uid, si.st_gid) == (167, 167)
 
 
-def test_deploy_a_monitoring_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_a_monitoring_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
-    _get_ip_addresses = mock.MagicMock(return_value=(['10.10.10.10'], []))
-    monkeypatch.setattr('cephadm.get_ip_addresses', _get_ip_addresses)
+    _get_ip_addresses = funkypatch.patch('cephadm.get_ip_addresses')
+    _get_ip_addresses.return_value = (['10.10.10.10'], [])
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()

From 63a14afb5c9010bfa29847ac2c560b9551a5b3a2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:11:21 -0500
Subject: [PATCH 0833/2492] cephadm: move snmp class to a new file

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 177 +------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   2 +
 src/cephadm/cephadmlib/daemons/snmp.py     | 189 +++++++++++++++++++++
 3 files changed, 192 insertions(+), 176 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/snmp.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ee8182afd144..61d6633a0a3d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -28,7 +28,6 @@
 from glob import glob
 from io import StringIO
 from threading import Thread, Event
-from urllib.error import HTTPError, URLError
 from urllib.request import urlopen, Request
 from pathlib import Path
 
@@ -37,7 +36,6 @@
     DEFAULT_IMAGE,
     DEFAULT_IMAGE_IS_MAIN,
     DEFAULT_IMAGE_RELEASE,
-    DEFAULT_SNMP_GATEWAY_IMAGE,
     # other constant values
     CEPH_CONF,
     CEPH_CONF_DIR,
@@ -170,6 +168,7 @@
     Keepalived,
     Monitoring,
     NFSGanesha,
+    SNMPGateway,
     Tracing,
 )
 
@@ -438,180 +437,6 @@ def firewall_service_name(self) -> str:
     def osd_fsid(self) -> Optional[str]:
         return self._osd_fsid
 
-
-##################################
-
-
-@register_daemon_form
-class SNMPGateway(ContainerDaemonForm):
-    """Defines an SNMP gateway between Prometheus and SNMP monitoring Frameworks"""
-    daemon_type = 'snmp-gateway'
-    SUPPORTED_VERSIONS = ['V2c', 'V3']
-    default_image = DEFAULT_SNMP_GATEWAY_IMAGE
-    DEFAULT_PORT = 9464
-    env_filename = 'snmp-gateway.conf'
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str,
-                 daemon_id: Union[int, str],
-                 config_json: Dict[str, Any],
-                 image: Optional[str] = None) -> None:
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image or SNMPGateway.default_image
-
-        self.uid = config_json.get('uid', 0)
-        self.gid = config_json.get('gid', 0)
-
-        self.destination = config_json.get('destination', '')
-        self.snmp_version = config_json.get('snmp_version', 'V2c')
-        self.snmp_community = config_json.get('snmp_community', 'public')
-        self.log_level = config_json.get('log_level', 'info')
-        self.snmp_v3_auth_username = config_json.get('snmp_v3_auth_username', '')
-        self.snmp_v3_auth_password = config_json.get('snmp_v3_auth_password', '')
-        self.snmp_v3_auth_protocol = config_json.get('snmp_v3_auth_protocol', '')
-        self.snmp_v3_priv_protocol = config_json.get('snmp_v3_priv_protocol', '')
-        self.snmp_v3_priv_password = config_json.get('snmp_v3_priv_password', '')
-        self.snmp_v3_engine_id = config_json.get('snmp_v3_engine_id', '')
-
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'SNMPGateway':
-        cfgs = fetch_configs(ctx)
-        assert cfgs  # assert some config data was found
-        return cls(ctx, fsid, daemon_id, cfgs, ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'SNMPGateway':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    @staticmethod
-    def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]:
-        """Return the version of the notifier from it's http endpoint"""
-        path = os.path.join(ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta')
-        try:
-            with open(path, 'r') as env:
-                metadata = json.loads(env.read())
-        except (OSError, json.JSONDecodeError):
-            return None
-
-        ports = metadata.get('ports', [])
-        if not ports:
-            return None
-
-        try:
-            with urlopen(f'http://127.0.0.1:{ports[0]}/') as r:
-                html = r.read().decode('utf-8').split('\n')
-        except (HTTPError, URLError):
-            return None
-
-        for h in html:
-            stripped = h.strip()
-            if stripped.startswith(('<pre>', '<PRE>')) and \
-               stripped.endswith(('</pre>', '</PRE>')):
-                # <pre>(version=1.2.1, branch=HEAD, revision=7...
-                return stripped.split(',')[0].split('version=')[1]
-
-        return None
-
-    @property
-    def port(self) -> int:
-        endpoints = fetch_endpoints(self.ctx)
-        if not endpoints:
-            return self.DEFAULT_PORT
-        return endpoints[0].port
-
-    def get_daemon_args(self) -> List[str]:
-        v3_args = []
-        base_args = [
-            f'--web.listen-address=:{self.port}',
-            f'--snmp.destination={self.destination}',
-            f'--snmp.version={self.snmp_version}',
-            f'--log.level={self.log_level}',
-            '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
-        ]
-
-        if self.snmp_version == 'V3':
-            # common auth settings
-            v3_args.extend([
-                '--snmp.authentication-enabled',
-                f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}',
-                f'--snmp.security-engine-id={self.snmp_v3_engine_id}'
-            ])
-            # authPriv setting is applied if we have a privacy protocol setting
-            if self.snmp_v3_priv_protocol:
-                v3_args.extend([
-                    '--snmp.private-enabled',
-                    f'--snmp.private-protocol={self.snmp_v3_priv_protocol}'
-                ])
-
-        return base_args + v3_args
-
-    @property
-    def data_dir(self) -> str:
-        return os.path.join(self.ctx.data_dir, self.ctx.fsid, f'{self.daemon_type}.{self.daemon_id}')
-
-    @property
-    def conf_file_path(self) -> str:
-        return os.path.join(self.data_dir, self.env_filename)
-
-    def create_daemon_conf(self) -> None:
-        """Creates the environment file holding 'secrets' passed to the snmp-notifier daemon"""
-        with write_new(self.conf_file_path) as f:
-            if self.snmp_version == 'V2c':
-                f.write(f'SNMP_NOTIFIER_COMMUNITY={self.snmp_community}\n')
-            else:
-                f.write(f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n')
-                f.write(f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n')
-                if self.snmp_v3_priv_password:
-                    f.write(f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n')
-
-    def validate(self) -> None:
-        """Validate the settings
-
-        Raises:
-            Error: if the fsid doesn't look like an fsid
-            Error: if the snmp version is not supported
-            Error: destination IP and port address missing
-        """
-        if not is_fsid(self.fsid):
-            raise Error(f'not a valid fsid: {self.fsid}')
-
-        if self.snmp_version not in SNMPGateway.SUPPORTED_VERSIONS:
-            raise Error(f'not a valid snmp version: {self.snmp_version}')
-
-        if not self.destination:
-            raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return self.uid, self.gid
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(f'--env-file={self.conf_file_path}')
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.extend(self.get_daemon_args())
-
 ##################################
 
 
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index a163e4a38293..d2c818e927d5 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -5,6 +5,7 @@
 from .iscsi import CephIscsi
 from .nfs import NFSGanesha
 from .monitoring import Monitoring
+from .snmp import SNMPGateway
 
 __all__ = [
     'CephIscsi',
@@ -14,5 +15,6 @@
     'Keepalived',
     'Monitoring',
     'NFSGanesha',
+    'SNMPGateway',
     'Tracing',
 ]
diff --git a/src/cephadm/cephadmlib/daemons/snmp.py b/src/cephadm/cephadmlib/daemons/snmp.py
new file mode 100644
index 000000000000..dc952aa4cb9d
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/snmp.py
@@ -0,0 +1,189 @@
+import json
+import os
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen
+
+from ..constants import DEFAULT_SNMP_GATEWAY_IMAGE
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer
+from ..context import CephadmContext
+from ..context_getters import fetch_configs, fetch_endpoints
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..data_utils import is_fsid
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import write_new
+
+
+@register_daemon_form
+class SNMPGateway(ContainerDaemonForm):
+    """Defines an SNMP gateway between Prometheus and SNMP monitoring Frameworks"""
+    daemon_type = 'snmp-gateway'
+    SUPPORTED_VERSIONS = ['V2c', 'V3']
+    default_image = DEFAULT_SNMP_GATEWAY_IMAGE
+    DEFAULT_PORT = 9464
+    env_filename = 'snmp-gateway.conf'
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx: CephadmContext,
+                 fsid: str,
+                 daemon_id: Union[int, str],
+                 config_json: Dict[str, Any],
+                 image: Optional[str] = None) -> None:
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image or SNMPGateway.default_image
+
+        self.uid = config_json.get('uid', 0)
+        self.gid = config_json.get('gid', 0)
+
+        self.destination = config_json.get('destination', '')
+        self.snmp_version = config_json.get('snmp_version', 'V2c')
+        self.snmp_community = config_json.get('snmp_community', 'public')
+        self.log_level = config_json.get('log_level', 'info')
+        self.snmp_v3_auth_username = config_json.get('snmp_v3_auth_username', '')
+        self.snmp_v3_auth_password = config_json.get('snmp_v3_auth_password', '')
+        self.snmp_v3_auth_protocol = config_json.get('snmp_v3_auth_protocol', '')
+        self.snmp_v3_priv_protocol = config_json.get('snmp_v3_priv_protocol', '')
+        self.snmp_v3_priv_password = config_json.get('snmp_v3_priv_password', '')
+        self.snmp_v3_engine_id = config_json.get('snmp_v3_engine_id', '')
+
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx: CephadmContext, fsid: str,
+             daemon_id: Union[int, str]) -> 'SNMPGateway':
+        cfgs = fetch_configs(ctx)
+        assert cfgs  # assert some config data was found
+        return cls(ctx, fsid, daemon_id, cfgs, ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'SNMPGateway':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    @staticmethod
+    def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]:
+        """Return the version of the notifier from it's http endpoint"""
+        path = os.path.join(ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta')
+        try:
+            with open(path, 'r') as env:
+                metadata = json.loads(env.read())
+        except (OSError, json.JSONDecodeError):
+            return None
+
+        ports = metadata.get('ports', [])
+        if not ports:
+            return None
+
+        try:
+            with urlopen(f'http://127.0.0.1:{ports[0]}/') as r:
+                html = r.read().decode('utf-8').split('\n')
+        except (HTTPError, URLError):
+            return None
+
+        for h in html:
+            stripped = h.strip()
+            if stripped.startswith(('<pre>', '<PRE>')) and \
+               stripped.endswith(('</pre>', '</PRE>')):
+                # <pre>(version=1.2.1, branch=HEAD, revision=7...
+                return stripped.split(',')[0].split('version=')[1]
+
+        return None
+
+    @property
+    def port(self) -> int:
+        endpoints = fetch_endpoints(self.ctx)
+        if not endpoints:
+            return self.DEFAULT_PORT
+        return endpoints[0].port
+
+    def get_daemon_args(self) -> List[str]:
+        v3_args = []
+        base_args = [
+            f'--web.listen-address=:{self.port}',
+            f'--snmp.destination={self.destination}',
+            f'--snmp.version={self.snmp_version}',
+            f'--log.level={self.log_level}',
+            '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
+        ]
+
+        if self.snmp_version == 'V3':
+            # common auth settings
+            v3_args.extend([
+                '--snmp.authentication-enabled',
+                f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}',
+                f'--snmp.security-engine-id={self.snmp_v3_engine_id}'
+            ])
+            # authPriv setting is applied if we have a privacy protocol setting
+            if self.snmp_v3_priv_protocol:
+                v3_args.extend([
+                    '--snmp.private-enabled',
+                    f'--snmp.private-protocol={self.snmp_v3_priv_protocol}'
+                ])
+
+        return base_args + v3_args
+
+    @property
+    def data_dir(self) -> str:
+        return os.path.join(self.ctx.data_dir, self.ctx.fsid, f'{self.daemon_type}.{self.daemon_id}')
+
+    @property
+    def conf_file_path(self) -> str:
+        return os.path.join(self.data_dir, self.env_filename)
+
+    def create_daemon_conf(self) -> None:
+        """Creates the environment file holding 'secrets' passed to the snmp-notifier daemon"""
+        with write_new(self.conf_file_path) as f:
+            if self.snmp_version == 'V2c':
+                f.write(f'SNMP_NOTIFIER_COMMUNITY={self.snmp_community}\n')
+            else:
+                f.write(f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n')
+                f.write(f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n')
+                if self.snmp_v3_priv_password:
+                    f.write(f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n')
+
+    def validate(self) -> None:
+        """Validate the settings
+
+        Raises:
+            Error: if the fsid doesn't look like an fsid
+            Error: if the snmp version is not supported
+            Error: destination IP and port address missing
+        """
+        if not is_fsid(self.fsid):
+            raise Error(f'not a valid fsid: {self.fsid}')
+
+        if self.snmp_version not in SNMPGateway.SUPPORTED_VERSIONS:
+            raise Error(f'not a valid snmp version: {self.snmp_version}')
+
+        if not self.destination:
+            raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return self.uid, self.gid
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(f'--env-file={self.conf_file_path}')
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self.get_daemon_args())

From 31ba507cc7046f68e33683747a1a9704e8588588 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:46:45 -0500
Subject: [PATCH 0834/2492] cephadm: move ceph classes to a new file

Move the ceph classes (Ceph, OSD, CephExporter) along with a few heavily
linked functions to a new ceph.py file under the daemons dir.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     | 419 +-------------------
 src/cephadm/cephadmlib/daemons/__init__.py |   4 +
 src/cephadm/cephadmlib/daemons/ceph.py     | 431 +++++++++++++++++++++
 src/cephadm/tests/test_cephadm.py          |   3 +-
 src/cephadm/tests/test_daemon_form.py      |   3 +-
 src/cephadm/tests/test_deploy.py           |  39 +-
 6 files changed, 460 insertions(+), 439 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/daemons/ceph.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 61d6633a0a3d..07155da5dcf7 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -67,7 +67,6 @@
     get_config_and_keyring,
     get_parm,
     read_configuration_source,
-    should_log_to_journald,
 )
 from cephadmlib.exceptions import (
     ClusterAlreadyExists,
@@ -117,7 +116,6 @@
     check_subnet,
     get_fqdn,
     get_hostname,
-    get_ip_addresses,
     get_short_hostname,
     ip_in_subnets,
     is_ipv6,
@@ -159,8 +157,9 @@
 from cephadmlib.sysctl import install_sysctl, migrate_sysctl_dir
 from cephadmlib.firewalld import Firewalld, update_firewalld
 from cephadmlib import templating
-from cephadmlib.deployment_utils import to_deployment_container
+from cephadmlib.daemons.ceph import get_ceph_mounts_for_type, ceph_daemons
 from cephadmlib.daemons import (
+    Ceph,
     CephIscsi,
     CephNvmeof,
     CustomContainer,
@@ -206,344 +205,6 @@ def __eq__(self, other: Any) -> bool:
 ##################################
 
 
-@register_daemon_form
-class Ceph(ContainerDaemonForm):
-    _daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
-                'crash', 'cephfs-mirror')
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        # TODO: figure out a way to un-special-case osd
-        return daemon_type in cls._daemons and daemon_type != 'osd'
-
-    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
-        self.ctx = ctx
-        self._identity = ident
-        self.user_supplied_config = False
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Ceph':
-        return cls(ctx, ident)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return self._identity
-
-    def firewall_service_name(self) -> str:
-        if self.identity.daemon_type == 'mon':
-            return 'ceph-mon'
-        elif self.identity.daemon_type in ['mgr', 'mds']:
-            return 'ceph'
-        return ''
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        # previous to being a ContainerDaemonForm, this make_var_run
-        # call was hard coded in the deploy path. Eventually, it would be
-        # good to move this somwhere cleaner and avoid needing to know the
-        # uid/gid here.
-        uid, gid = self.uid_gid(ctx)
-        make_var_run(ctx, ctx.fsid, uid, gid)
-
-        # mon and osd need privileged in order for libudev to query devices
-        privileged = self.identity.daemon_type in ['mon', 'osd']
-        ctr = daemon_to_container(ctx, self, privileged=privileged)
-        ctr = to_deployment_container(ctx, ctr)
-        config_json = fetch_configs(ctx)
-        if self.identity.daemon_type == 'mon' and config_json is not None:
-            if 'crush_location' in config_json:
-                c_loc = config_json['crush_location']
-                # was originally "c.args.extend(['--set-crush-location', c_loc])"
-                # but that doesn't seem to persist in the object after it's passed
-                # in further function calls
-                ctr.args = ctr.args + ['--set-crush-location', c_loc]
-        return ctr
-
-    _uid_gid: Optional[Tuple[int, int]] = None
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        if self._uid_gid is None:
-            self._uid_gid = extract_uid_gid(ctx)
-        return self._uid_gid
-
-    def config_and_keyring(
-        self, ctx: CephadmContext
-    ) -> Tuple[Optional[str], Optional[str]]:
-        return get_config_and_keyring(ctx)
-
-    def get_daemon_args(self) -> List[str]:
-        if self.identity.daemon_type == 'crash':
-            return []
-        r = [
-            '--setuser', 'ceph',
-            '--setgroup', 'ceph',
-            '--default-log-to-file=false',
-        ]
-        log_to_journald = should_log_to_journald(self.ctx)
-        if log_to_journald:
-            r += [
-                '--default-log-to-journald=true',
-                '--default-log-to-stderr=false',
-            ]
-        else:
-            r += [
-                '--default-log-to-stderr=true',
-                '--default-log-stderr-prefix=debug ',
-            ]
-        if self.identity.daemon_type == 'mon':
-            r += [
-                '--default-mon-cluster-log-to-file=false',
-            ]
-            if log_to_journald:
-                r += [
-                    '--default-mon-cluster-log-to-journald=true',
-                    '--default-mon-cluster-log-to-stderr=false',
-                ]
-            else:
-                r += ['--default-mon-cluster-log-to-stderr=true']
-        return r
-
-    @staticmethod
-    def get_ceph_mounts(
-        ctx: CephadmContext,
-        ident: DaemonIdentity,
-        no_config: bool = False,
-    ) -> Dict[str, str]:
-        # Warning: This is a hack done for more expedient refactoring
-        mounts = _get_container_mounts_for_type(
-            ctx, ident.fsid, ident.daemon_type
-        )
-        data_dir = ident.data_dir(ctx.data_dir)
-        if ident.daemon_type == 'rgw':
-            cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (
-                ident.daemon_id
-            )
-        else:
-            cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (
-                ident.daemon_type,
-                ident.daemon_id,
-            )
-        if ident.daemon_type != 'crash':
-            mounts[data_dir] = cdata_dir + ':z'
-        if not no_config:
-            mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z'
-        if ident.daemon_type in [
-            'rbd-mirror',
-            'cephfs-mirror',
-            'crash',
-            'ceph-exporter',
-        ]:
-            # these do not search for their keyrings in a data directory
-            mounts[
-                data_dir + '/keyring'
-            ] = '/etc/ceph/ceph.client.%s.%s.keyring' % (
-                ident.daemon_type,
-                ident.daemon_id,
-            )
-        return mounts
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        no_config = bool(
-            getattr(ctx, 'config', None) and self.user_supplied_config
-        )
-        cm = self.get_ceph_mounts(
-            ctx,
-            self.identity,
-            no_config=no_config,
-        )
-        mounts.update(cm)
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(ctx.container_engine.unlimited_pids_option)
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        ident = self.identity
-        if ident.daemon_type == 'rgw':
-            name = 'client.rgw.%s' % ident.daemon_id
-        elif ident.daemon_type == 'rbd-mirror':
-            name = 'client.rbd-mirror.%s' % ident.daemon_id
-        elif ident.daemon_type == 'cephfs-mirror':
-            name = 'client.cephfs-mirror.%s' % ident.daemon_id
-        elif ident.daemon_type == 'crash':
-            name = 'client.crash.%s' % ident.daemon_id
-        elif ident.daemon_type in ['mon', 'mgr', 'mds', 'osd']:
-            name = ident.daemon_name
-        else:
-            raise ValueError(ident)
-        args.extend(['-n', name])
-        if ident.daemon_type != 'crash':
-            args.append('-f')
-        args.extend(self.get_daemon_args())
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
-
-    def default_entrypoint(self) -> str:
-        ep = {
-            'rgw': '/usr/bin/radosgw',
-            'rbd-mirror': '/usr/bin/rbd-mirror',
-            'cephfs-mirror': '/usr/bin/cephfs-mirror',
-        }
-        daemon_type = self.identity.daemon_type
-        return ep.get(daemon_type) or f'/usr/bin/ceph-{daemon_type}'
-
-##################################
-
-
-@register_daemon_form
-class OSD(Ceph):
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        # TODO: figure out a way to un-special-case osd
-        return daemon_type == 'osd'
-
-    def __init__(
-        self,
-        ctx: CephadmContext,
-        ident: DaemonIdentity,
-        osd_fsid: Optional[str] = None,
-    ) -> None:
-        super().__init__(ctx, ident)
-        self._osd_fsid = osd_fsid
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'OSD':
-        osd_fsid = getattr(ctx, 'osd_fsid', None)
-        if osd_fsid is None:
-            logger.info(
-                'Creating an OSD daemon form without an OSD FSID value'
-            )
-        return cls(ctx, ident, osd_fsid)
-
-    @staticmethod
-    def get_sysctl_settings() -> List[str]:
-        return [
-            '# allow a large number of OSDs',
-            'fs.aio-max-nr = 1048576',
-            'kernel.pid_max = 4194304',
-        ]
-
-    def firewall_service_name(self) -> str:
-        return 'ceph'
-
-    @property
-    def osd_fsid(self) -> Optional[str]:
-        return self._osd_fsid
-
-##################################
-
-
-@register_daemon_form
-class CephExporter(ContainerDaemonForm):
-    """Defines a Ceph exporter container"""
-
-    daemon_type = 'ceph-exporter'
-    entrypoint = '/usr/bin/ceph-exporter'
-    DEFAULT_PORT = 9926
-    port_map = {
-        'ceph-exporter': DEFAULT_PORT,
-    }
-
-    @classmethod
-    def for_daemon_type(cls, daemon_type: str) -> bool:
-        return cls.daemon_type == daemon_type
-
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict[str, Any],
-                 image: str = DEFAULT_IMAGE) -> None:
-        self.ctx = ctx
-        self.fsid = fsid
-        self.daemon_id = daemon_id
-        self.image = image
-
-        self.sock_dir = config_json.get('sock-dir', '/var/run/ceph/')
-        ipv4_addrs, _ = get_ip_addresses(get_hostname())
-        addrs = '0.0.0.0' if ipv4_addrs else '::'
-        self.addrs = config_json.get('addrs', addrs)
-        self.port = config_json.get('port', self.DEFAULT_PORT)
-        self.prio_limit = config_json.get('prio-limit', 5)
-        self.stats_period = config_json.get('stats-period', 5)
-
-        self.validate()
-
-    @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'CephExporter':
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
-
-    @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephExporter':
-        return cls.init(ctx, ident.fsid, ident.daemon_id)
-
-    @property
-    def identity(self) -> DaemonIdentity:
-        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
-
-    def get_daemon_args(self) -> List[str]:
-        args = [
-            f'--sock-dir={self.sock_dir}',
-            f'--addrs={self.addrs}',
-            f'--port={self.port}',
-            f'--prio-limit={self.prio_limit}',
-            f'--stats-period={self.stats_period}',
-        ]
-        return args
-
-    def validate(self) -> None:
-        if not os.path.isdir(self.sock_dir):
-            raise Error(f'Directory does not exist. Got: {self.sock_dir}')
-
-    def container(self, ctx: CephadmContext) -> CephContainer:
-        ctr = daemon_to_container(ctx, self)
-        return to_deployment_container(ctx, ctr)
-
-    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
-        return extract_uid_gid(ctx)
-
-    def config_and_keyring(
-        self, ctx: CephadmContext
-    ) -> Tuple[Optional[str], Optional[str]]:
-        return get_config_and_keyring(ctx)
-
-    def customize_container_mounts(
-        self, ctx: CephadmContext, mounts: Dict[str, str]
-    ) -> None:
-        cm = Ceph.get_ceph_mounts(ctx, self.identity)
-        mounts.update(cm)
-
-    def customize_process_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        name = 'client.ceph-exporter.%s' % self.identity.daemon_id
-        args.extend(['-n', name, '-f'])
-        args.extend(self.get_daemon_args())
-
-    def customize_container_args(
-        self, ctx: CephadmContext, args: List[str]
-    ) -> None:
-        args.append(ctx.container_engine.unlimited_pids_option)
-
-    def customize_container_envs(
-        self, ctx: CephadmContext, envs: List[str]
-    ) -> None:
-        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
-
-    def default_entrypoint(self) -> str:
-        return self.entrypoint
-
-##################################
-
-
 def get_supported_daemons():
     # type: () -> List[str]
     supported_daemons = ceph_daemons()
@@ -560,15 +221,6 @@ def get_supported_daemons():
     assert len(supported_daemons) == len(set(supported_daemons))
     return supported_daemons
 
-
-def ceph_daemons() -> List[str]:
-    cds = list(Ceph._daemons)
-    cds.append(CephExporter.daemon_type)
-    return cds
-
-##################################
-
-
 ##################################
 
 
@@ -1191,76 +843,11 @@ def get_container_mounts_for_type(
     """Return a dictionary mapping container-external paths to container-internal
     paths given an fsid and daemon_type.
     """
-    mounts = _get_container_mounts_for_type(ctx, fsid, daemon_type)
+    mounts = get_ceph_mounts_for_type(ctx, fsid, daemon_type)
     _update_podman_mounts(ctx, mounts)
     return mounts
 
 
-def _get_container_mounts_for_type(
-    ctx: CephadmContext, fsid: str, daemon_type: str
-) -> Dict[str, str]:
-    """The main implementation of get_container_mounts_for_type minus the call
-    to _update_podman_mounts so that this can be called from
-    get_container_mounts.
-    """
-    mounts = dict()
-
-    if daemon_type in ceph_daemons():
-        if fsid:
-            run_path = os.path.join('/var/run/ceph', fsid)
-            if os.path.exists(run_path):
-                mounts[run_path] = '/var/run/ceph:z'
-            log_dir = get_log_dir(fsid, ctx.log_dir)
-            mounts[log_dir] = '/var/log/ceph:z'
-            crash_dir = '/var/lib/ceph/%s/crash' % fsid
-            if os.path.exists(crash_dir):
-                mounts[crash_dir] = '/var/lib/ceph/crash:z'
-            if daemon_type != 'crash' and should_log_to_journald(ctx):
-                journald_sock_dir = '/run/systemd/journal'
-                mounts[journald_sock_dir] = journald_sock_dir
-
-    if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']:
-        mounts['/dev'] = '/dev'  # FIXME: narrow this down?
-        mounts['/run/udev'] = '/run/udev'
-    if daemon_type in ['osd', 'clusterless-ceph-volume']:
-        mounts['/sys'] = '/sys'  # for numa.cc, pick_address, cgroups, ...
-        mounts['/run/lvm'] = '/run/lvm'
-        mounts['/run/lock/lvm'] = '/run/lock/lvm'
-    if daemon_type == 'osd':
-        # selinux-policy in the container may not match the host.
-        if HostFacts(ctx).selinux_enabled:
-            cluster_dir = f'{ctx.data_dir}/{fsid}'
-            selinux_folder = f'{cluster_dir}/selinux'
-            if os.path.exists(cluster_dir):
-                if not os.path.exists(selinux_folder):
-                    os.makedirs(selinux_folder, mode=0o755)
-                mounts[selinux_folder] = '/sys/fs/selinux:ro'
-            else:
-                logger.error(f'Cluster direcotry {cluster_dir} does not exist.')
-        mounts['/'] = '/rootfs'
-
-    try:
-        if ctx.shared_ceph_folder:  # make easy manager modules/ceph-volume development
-            ceph_folder = pathify(ctx.shared_ceph_folder)
-            if os.path.exists(ceph_folder):
-                cephadm_binary = ceph_folder + '/src/cephadm/cephadm'
-                if not os.path.exists(pathify(cephadm_binary)):
-                    raise Error("cephadm binary does not exist. Please run './build.sh cephadm' from ceph/src/cephadm/ directory.")
-                mounts[cephadm_binary] = '/usr/sbin/cephadm'
-                mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume'
-                mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr'
-                mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph'
-                mounts[ceph_folder + '/monitoring/ceph-mixin/dashboards_out'] = '/etc/grafana/dashboards/ceph-dashboard'
-                mounts[ceph_folder + '/monitoring/ceph-mixin/prometheus_alerts.yml'] = '/etc/prometheus/ceph/ceph_default_alerts.yml'
-            else:
-                logger.error(
-                    'Ceph shared source folder does not exist.',
-                    extra=Highlight.FAILURE.extra())
-    except AttributeError:
-        pass
-    return mounts
-
-
 def get_container_mounts(
     ctx: CephadmContext, ident: 'DaemonIdentity', no_config: bool = False
 ) -> Dict[str, str]:
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index d2c818e927d5..3931cdf14b61 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -6,8 +6,11 @@
 from .nfs import NFSGanesha
 from .monitoring import Monitoring
 from .snmp import SNMPGateway
+from .ceph import Ceph, OSD, CephExporter
 
 __all__ = [
+    'Ceph',
+    'CephExporter',
     'CephIscsi',
     'CephNvmeof',
     'CustomContainer',
@@ -15,6 +18,7 @@
     'Keepalived',
     'Monitoring',
     'NFSGanesha',
+    'OSD',
     'SNMPGateway',
     'Tracing',
 ]
diff --git a/src/cephadm/cephadmlib/daemons/ceph.py b/src/cephadm/cephadmlib/daemons/ceph.py
new file mode 100644
index 000000000000..ba908ae5d298
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/ceph.py
@@ -0,0 +1,431 @@
+import logging
+import os
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context_getters import (
+    fetch_configs,
+    get_config_and_keyring,
+    should_log_to_journald,
+)
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..constants import DEFAULT_IMAGE
+from ..context import CephadmContext
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import make_run_dir, pathify
+from ..host_facts import HostFacts
+from ..logging import Highlight
+from ..net_utils import get_hostname, get_ip_addresses
+
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class Ceph(ContainerDaemonForm):
+    _daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
+                'crash', 'cephfs-mirror')
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        # TODO: figure out a way to un-special-case osd
+        return daemon_type in cls._daemons and daemon_type != 'osd'
+
+    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
+        self.ctx = ctx
+        self._identity = ident
+        self.user_supplied_config = False
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Ceph':
+        return cls(ctx, ident)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    def firewall_service_name(self) -> str:
+        if self.identity.daemon_type == 'mon':
+            return 'ceph-mon'
+        elif self.identity.daemon_type in ['mgr', 'mds']:
+            return 'ceph'
+        return ''
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        # previous to being a ContainerDaemonForm, this call to create the
+        # var-run directory was hard coded in the deploy path. Eventually, it
+        # would be good to move this somwhere cleaner and avoid needing to know
+        # the uid/gid here.
+        uid, gid = self.uid_gid(ctx)
+        make_run_dir(ctx.fsid, uid, gid)
+
+        # mon and osd need privileged in order for libudev to query devices
+        privileged = self.identity.daemon_type in ['mon', 'osd']
+        ctr = daemon_to_container(ctx, self, privileged=privileged)
+        ctr = to_deployment_container(ctx, ctr)
+        config_json = fetch_configs(ctx)
+        if self.identity.daemon_type == 'mon' and config_json is not None:
+            if 'crush_location' in config_json:
+                c_loc = config_json['crush_location']
+                # was originally "c.args.extend(['--set-crush-location', c_loc])"
+                # but that doesn't seem to persist in the object after it's passed
+                # in further function calls
+                ctr.args = ctr.args + ['--set-crush-location', c_loc]
+        return ctr
+
+    _uid_gid: Optional[Tuple[int, int]] = None
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        if self._uid_gid is None:
+            self._uid_gid = extract_uid_gid(ctx)
+        return self._uid_gid
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def get_daemon_args(self) -> List[str]:
+        if self.identity.daemon_type == 'crash':
+            return []
+        r = [
+            '--setuser', 'ceph',
+            '--setgroup', 'ceph',
+            '--default-log-to-file=false',
+        ]
+        log_to_journald = should_log_to_journald(self.ctx)
+        if log_to_journald:
+            r += [
+                '--default-log-to-journald=true',
+                '--default-log-to-stderr=false',
+            ]
+        else:
+            r += [
+                '--default-log-to-stderr=true',
+                '--default-log-stderr-prefix=debug ',
+            ]
+        if self.identity.daemon_type == 'mon':
+            r += [
+                '--default-mon-cluster-log-to-file=false',
+            ]
+            if log_to_journald:
+                r += [
+                    '--default-mon-cluster-log-to-journald=true',
+                    '--default-mon-cluster-log-to-stderr=false',
+                ]
+            else:
+                r += ['--default-mon-cluster-log-to-stderr=true']
+        return r
+
+    @staticmethod
+    def get_ceph_mounts(
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        no_config: bool = False,
+    ) -> Dict[str, str]:
+        # Warning: This is a hack done for more expedient refactoring
+        mounts = get_ceph_mounts_for_type(
+            ctx, ident.fsid, ident.daemon_type
+        )
+        data_dir = ident.data_dir(ctx.data_dir)
+        if ident.daemon_type == 'rgw':
+            cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (
+                ident.daemon_id
+            )
+        else:
+            cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (
+                ident.daemon_type,
+                ident.daemon_id,
+            )
+        if ident.daemon_type != 'crash':
+            mounts[data_dir] = cdata_dir + ':z'
+        if not no_config:
+            mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z'
+        if ident.daemon_type in [
+            'rbd-mirror',
+            'cephfs-mirror',
+            'crash',
+            'ceph-exporter',
+        ]:
+            # these do not search for their keyrings in a data directory
+            mounts[
+                data_dir + '/keyring'
+            ] = '/etc/ceph/ceph.client.%s.%s.keyring' % (
+                ident.daemon_type,
+                ident.daemon_id,
+            )
+        return mounts
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        no_config = bool(
+            getattr(ctx, 'config', None) and self.user_supplied_config
+        )
+        cm = self.get_ceph_mounts(
+            ctx,
+            self.identity,
+            no_config=no_config,
+        )
+        mounts.update(cm)
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        ident = self.identity
+        if ident.daemon_type == 'rgw':
+            name = 'client.rgw.%s' % ident.daemon_id
+        elif ident.daemon_type == 'rbd-mirror':
+            name = 'client.rbd-mirror.%s' % ident.daemon_id
+        elif ident.daemon_type == 'cephfs-mirror':
+            name = 'client.cephfs-mirror.%s' % ident.daemon_id
+        elif ident.daemon_type == 'crash':
+            name = 'client.crash.%s' % ident.daemon_id
+        elif ident.daemon_type in ['mon', 'mgr', 'mds', 'osd']:
+            name = ident.daemon_name
+        else:
+            raise ValueError(ident)
+        args.extend(['-n', name])
+        if ident.daemon_type != 'crash':
+            args.append('-f')
+        args.extend(self.get_daemon_args())
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+
+    def default_entrypoint(self) -> str:
+        ep = {
+            'rgw': '/usr/bin/radosgw',
+            'rbd-mirror': '/usr/bin/rbd-mirror',
+            'cephfs-mirror': '/usr/bin/cephfs-mirror',
+        }
+        daemon_type = self.identity.daemon_type
+        return ep.get(daemon_type) or f'/usr/bin/ceph-{daemon_type}'
+
+
+@register_daemon_form
+class OSD(Ceph):
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        # TODO: figure out a way to un-special-case osd
+        return daemon_type == 'osd'
+
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        osd_fsid: Optional[str] = None,
+    ) -> None:
+        super().__init__(ctx, ident)
+        self._osd_fsid = osd_fsid
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'OSD':
+        osd_fsid = getattr(ctx, 'osd_fsid', None)
+        if osd_fsid is None:
+            logger.info(
+                'Creating an OSD daemon form without an OSD FSID value'
+            )
+        return cls(ctx, ident, osd_fsid)
+
+    @staticmethod
+    def get_sysctl_settings() -> List[str]:
+        return [
+            '# allow a large number of OSDs',
+            'fs.aio-max-nr = 1048576',
+            'kernel.pid_max = 4194304',
+        ]
+
+    def firewall_service_name(self) -> str:
+        return 'ceph'
+
+    @property
+    def osd_fsid(self) -> Optional[str]:
+        return self._osd_fsid
+
+
+@register_daemon_form
+class CephExporter(ContainerDaemonForm):
+    """Defines a Ceph exporter container"""
+
+    daemon_type = 'ceph-exporter'
+    entrypoint = '/usr/bin/ceph-exporter'
+    DEFAULT_PORT = 9926
+    port_map = {
+        'ceph-exporter': DEFAULT_PORT,
+    }
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self,
+                 ctx: CephadmContext,
+                 fsid: str, daemon_id: Union[int, str],
+                 config_json: Dict[str, Any],
+                 image: str = DEFAULT_IMAGE) -> None:
+        self.ctx = ctx
+        self.fsid = fsid
+        self.daemon_id = daemon_id
+        self.image = image
+
+        self.sock_dir = config_json.get('sock-dir', '/var/run/ceph/')
+        ipv4_addrs, _ = get_ip_addresses(get_hostname())
+        addrs = '0.0.0.0' if ipv4_addrs else '::'
+        self.addrs = config_json.get('addrs', addrs)
+        self.port = config_json.get('port', self.DEFAULT_PORT)
+        self.prio_limit = config_json.get('prio-limit', 5)
+        self.stats_period = config_json.get('stats-period', 5)
+
+        self.validate()
+
+    @classmethod
+    def init(cls, ctx: CephadmContext, fsid: str,
+             daemon_id: Union[int, str]) -> 'CephExporter':
+        return cls(ctx, fsid, daemon_id,
+                   fetch_configs(ctx), ctx.image)
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephExporter':
+        return cls.init(ctx, ident.fsid, ident.daemon_id)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
+
+    def get_daemon_args(self) -> List[str]:
+        args = [
+            f'--sock-dir={self.sock_dir}',
+            f'--addrs={self.addrs}',
+            f'--port={self.port}',
+            f'--prio-limit={self.prio_limit}',
+            f'--stats-period={self.stats_period}',
+        ]
+        return args
+
+    def validate(self) -> None:
+        if not os.path.isdir(self.sock_dir):
+            raise Error(f'Directory does not exist. Got: {self.sock_dir}')
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self)
+        return to_deployment_container(ctx, ctr)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return extract_uid_gid(ctx)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        cm = Ceph.get_ceph_mounts(ctx, self.identity)
+        mounts.update(cm)
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        name = 'client.ceph-exporter.%s' % self.identity.daemon_id
+        args.extend(['-n', name, '-f'])
+        args.extend(self.get_daemon_args())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.append(ctx.container_engine.unlimited_pids_option)
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
+
+
+def get_ceph_mounts_for_type(
+    ctx: CephadmContext, fsid: str, daemon_type: str
+) -> Dict[str, str]:
+    """The main implementation of get_container_mounts_for_type minus the call
+    to _update_podman_mounts so that this can be called from
+    get_container_mounts.
+    """
+    mounts = dict()
+
+    if daemon_type in ceph_daemons():
+        if fsid:
+            run_path = os.path.join('/var/run/ceph', fsid)
+            if os.path.exists(run_path):
+                mounts[run_path] = '/var/run/ceph:z'
+            log_dir = os.path.join(ctx.log_dir, fsid)
+            mounts[log_dir] = '/var/log/ceph:z'
+            crash_dir = '/var/lib/ceph/%s/crash' % fsid
+            if os.path.exists(crash_dir):
+                mounts[crash_dir] = '/var/lib/ceph/crash:z'
+            if daemon_type != 'crash' and should_log_to_journald(ctx):
+                journald_sock_dir = '/run/systemd/journal'
+                mounts[journald_sock_dir] = journald_sock_dir
+
+    if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']:
+        mounts['/dev'] = '/dev'  # FIXME: narrow this down?
+        mounts['/run/udev'] = '/run/udev'
+    if daemon_type in ['osd', 'clusterless-ceph-volume']:
+        mounts['/sys'] = '/sys'  # for numa.cc, pick_address, cgroups, ...
+        mounts['/run/lvm'] = '/run/lvm'
+        mounts['/run/lock/lvm'] = '/run/lock/lvm'
+    if daemon_type == 'osd':
+        # selinux-policy in the container may not match the host.
+        if HostFacts(ctx).selinux_enabled:
+            cluster_dir = f'{ctx.data_dir}/{fsid}'
+            selinux_folder = f'{cluster_dir}/selinux'
+            if os.path.exists(cluster_dir):
+                if not os.path.exists(selinux_folder):
+                    os.makedirs(selinux_folder, mode=0o755)
+                mounts[selinux_folder] = '/sys/fs/selinux:ro'
+            else:
+                logger.error(f'Cluster direcotry {cluster_dir} does not exist.')
+        mounts['/'] = '/rootfs'
+
+    try:
+        if ctx.shared_ceph_folder:  # make easy manager modules/ceph-volume development
+            ceph_folder = pathify(ctx.shared_ceph_folder)
+            if os.path.exists(ceph_folder):
+                cephadm_binary = ceph_folder + '/src/cephadm/cephadm'
+                if not os.path.exists(pathify(cephadm_binary)):
+                    raise Error("cephadm binary does not exist. Please run './build.sh cephadm' from ceph/src/cephadm/ directory.")
+                mounts[cephadm_binary] = '/usr/sbin/cephadm'
+                mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume'
+                mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr'
+                mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph'
+                mounts[ceph_folder + '/monitoring/ceph-mixin/dashboards_out'] = '/etc/grafana/dashboards/ceph-dashboard'
+                mounts[ceph_folder + '/monitoring/ceph-mixin/prometheus_alerts.yml'] = '/etc/prometheus/ceph/ceph_default_alerts.yml'
+            else:
+                logger.error(
+                    'Ceph shared source folder does not exist.',
+                    extra=Highlight.FAILURE.extra())
+    except AttributeError:
+        pass
+    return mounts
+
+
+def ceph_daemons() -> List[str]:
+    """A legacy method that returns a list of all daemon types considered ceph
+    daemons.
+    """
+    cds = list(Ceph._daemons)
+    cds.append(CephExporter.daemon_type)
+    return cds
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index b2e395fab64e..82850ab597d6 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -376,7 +376,8 @@ def test_mon_crush_location(self, funkypatch):
         funkypatch.patch('cephadm.logger')
         funkypatch.patch('cephadm.FileLock')
         _deploy_daemon = funkypatch.patch('cephadm.deploy_daemon')
-        _make_var_run = funkypatch.patch('cephadm.make_var_run')
+        funkypatch.patch('cephadm.make_var_run')
+        funkypatch.patch('cephadmlib.file_utils.make_run_dir')
         _migrate_sysctl = funkypatch.patch('cephadm.migrate_sysctl_dir')
         funkypatch.patch(
             'cephadm.check_unit',
diff --git a/src/cephadm/tests/test_daemon_form.py b/src/cephadm/tests/test_daemon_form.py
index 07896cc58559..a2d1773f1c84 100644
--- a/src/cephadm/tests/test_daemon_form.py
+++ b/src/cephadm/tests/test_daemon_form.py
@@ -6,6 +6,7 @@
 
 from cephadmlib import daemon_form
 from cephadmlib import daemon_identity
+from cephadmlib import daemons
 
 _cephadm = import_cephadm()
 
@@ -22,7 +23,7 @@
         ('mon', _cephadm.Ceph),
         ('nfs', _cephadm.NFSGanesha),
         ('nvmeof', _cephadm.CephNvmeof),
-        ('osd', _cephadm.OSD),
+        ('osd', daemons.OSD),
         ('prometheus', _cephadm.Monitoring),
         ('snmp-gateway', _cephadm.SNMPGateway),
     ],
diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 5d5b46ad6954..94a292dc57b6 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -294,7 +294,7 @@ def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
 def test_deploy_a_monitoring_container(cephadm_fs, funkypatch):
     mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
-    _get_ip_addresses = funkypatch.patch('cephadm.get_ip_addresses')
+    _get_ip_addresses = funkypatch.patch('cephadmlib.net_utils.get_ip_addresses')
     _get_ip_addresses.return_value = (['10.10.10.10'], [])
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
@@ -362,11 +362,10 @@ def test_deploy_a_tracing_container(cephadm_fs, monkeypatch):
     assert not (basedir / 'keyring').exists()
 
 
-def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_ceph_mgr_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
-    _make_var_run = mock.MagicMock()
-    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    _make_run_dir = funkypatch.patch('cephadmlib.file_utils.make_run_dir')
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()
@@ -400,16 +399,15 @@ def test_deploy_ceph_mgr_container(cephadm_fs, monkeypatch):
         assert f.read() == 'XXXXXXX'
     with open(basedir / 'keyring') as f:
         assert f.read() == 'YYYYYY'
-    assert _make_var_run.call_count == 1
-    assert _make_var_run.call_args[0][2] == 8765
-    assert _make_var_run.call_args[0][3] == 8765
+    assert _make_run_dir.call_count == 1
+    assert _make_run_dir.call_args[0][1] == 8765
+    assert _make_run_dir.call_args[0][2] == 8765
 
 
-def test_deploy_ceph_osd_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_ceph_osd_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
-    _make_var_run = mock.MagicMock()
-    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    _make_run_dir = funkypatch.patch('cephadmlib.file_utils.make_run_dir')
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()
@@ -445,18 +443,17 @@ def test_deploy_ceph_osd_container(cephadm_fs, monkeypatch):
         assert f.read() == 'XXXXXXX'
     with open(basedir / 'keyring') as f:
         assert f.read() == 'YYYYYY'
-    assert _make_var_run.call_count == 1
-    assert _make_var_run.call_args[0][2] == 8765
-    assert _make_var_run.call_args[0][3] == 8765
+    assert _make_run_dir.call_count == 1
+    assert _make_run_dir.call_args[0][1] == 8765
+    assert _make_run_dir.call_args[0][2] == 8765
 
 
-def test_deploy_ceph_exporter_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_ceph_exporter_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
-    _get_ip_addresses = mock.MagicMock(return_value=(['10.10.10.10'], []))
-    monkeypatch.setattr('cephadm.get_ip_addresses', _get_ip_addresses)
-    _make_var_run = mock.MagicMock()
-    monkeypatch.setattr('cephadm.make_var_run', _make_var_run)
+    _get_ip_addresses = funkypatch.patch('cephadmlib.net_utils.get_ip_addresses')
+    _get_ip_addresses.return_value = (['10.10.10.10'], [])
+    _make_run_dir = funkypatch.patch('cephadmlib.file_utils.make_run_dir')
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
         ctx.container_engine = mock_podman()

From 13baad75f599c06f12dde335892713ef6a2fa57e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:49:03 -0500
Subject: [PATCH 0835/2492] cephadm: sort imports import daemons/__init__.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index 3931cdf14b61..cf572d487c9b 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -1,12 +1,12 @@
+from .ceph import Ceph, OSD, CephExporter
 from .custom import CustomContainer
-from .tracing import Tracing
 from .ingress import HAproxy, Keepalived
-from .nvmeof import CephNvmeof
 from .iscsi import CephIscsi
-from .nfs import NFSGanesha
 from .monitoring import Monitoring
+from .nfs import NFSGanesha
+from .nvmeof import CephNvmeof
 from .snmp import SNMPGateway
-from .ceph import Ceph, OSD, CephExporter
+from .tracing import Tracing
 
 __all__ = [
     'Ceph',

From ddee9492b4c0f5bbbc84d39aaa56539c88e4c0c2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 14:26:35 -0500
Subject: [PATCH 0836/2492] cephadm: convert all deploy tests to use funkypatch
 fixture

During the refactor of various daemon type classes some of the tests had
been converted to funkypatch in order to deal with imports occuring over
multiple files. However, this conversion was done piece by piece in
order to make clear what was changing. This left the functions in this
file inconsistent. Change all the remaining function to use funkypatch
for consistency.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 94a292dc57b6..dadf3456fd5a 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -16,10 +16,6 @@
 _cephadm = import_cephadm()
 
 
-def _common_mp(monkeypatch):
-    return _common_patches(FunkyPatcher(monkeypatch))
-
-
 def _common_patches(funkypatch):
     mocks = {}
     _call = funkypatch.patch('cephadmlib.container_types.call')
@@ -39,8 +35,8 @@ def _common_patches(funkypatch):
     return mocks
 
 
-def test_deploy_nfs_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_nfs_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
@@ -76,8 +72,8 @@ def test_deploy_nfs_container(cephadm_fs, monkeypatch):
         assert f.read() == 'FAKE'
 
 
-def test_deploy_snmp_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_snmp_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
@@ -108,8 +104,8 @@ def test_deploy_snmp_container(cephadm_fs, monkeypatch):
     assert not (basedir / 'keyring').exists()
 
 
-def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_keepalived_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     _install_sysctl = mocks['install_sysctl']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
@@ -156,8 +152,8 @@ def test_deploy_keepalived_container(cephadm_fs, monkeypatch):
     assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1
 
 
-def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_haproxy_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     _install_sysctl = mocks['install_sysctl']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
@@ -201,8 +197,8 @@ def test_deploy_haproxy_container(cephadm_fs, monkeypatch):
     assert len(_install_sysctl.call_args[0][-1].get_sysctl_settings()) > 1
 
 
-def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_iscsi_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
@@ -245,8 +241,8 @@ def test_deploy_iscsi_container(cephadm_fs, monkeypatch):
         assert (si.st_uid, si.st_gid) == (8765, 8765)
 
 
-def test_deploy_nvmeof_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_nvmeof_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:
@@ -331,8 +327,8 @@ def test_deploy_a_monitoring_container(cephadm_fs, funkypatch):
         assert (si.st_uid, si.st_gid) == (8765, 8765)
 
 
-def test_deploy_a_tracing_container(cephadm_fs, monkeypatch):
-    mocks = _common_mp(monkeypatch)
+def test_deploy_a_tracing_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
     _firewalld = mocks['Firewalld']
     fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
     with with_cephadm_ctx([]) as ctx:

From a7f5b0c20b70c0679019c90007c148b4c23f40ac Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:57:15 -0500
Subject: [PATCH 0837/2492] cephadm: black format daemons/ceph.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/ceph.py | 83 ++++++++++++++++++--------
 1 file changed, 57 insertions(+), 26 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/ceph.py b/src/cephadm/cephadmlib/daemons/ceph.py
index ba908ae5d298..0afb8f734af5 100644
--- a/src/cephadm/cephadmlib/daemons/ceph.py
+++ b/src/cephadm/cephadmlib/daemons/ceph.py
@@ -27,8 +27,16 @@
 
 @register_daemon_form
 class Ceph(ContainerDaemonForm):
-    _daemons = ('mon', 'mgr', 'osd', 'mds', 'rgw', 'rbd-mirror',
-                'crash', 'cephfs-mirror')
+    _daemons = (
+        'mon',
+        'mgr',
+        'osd',
+        'mds',
+        'rgw',
+        'rbd-mirror',
+        'crash',
+        'cephfs-mirror',
+    )
 
     @classmethod
     def for_daemon_type(cls, daemon_type: str) -> bool:
@@ -93,8 +101,10 @@ def get_daemon_args(self) -> List[str]:
         if self.identity.daemon_type == 'crash':
             return []
         r = [
-            '--setuser', 'ceph',
-            '--setgroup', 'ceph',
+            '--setuser',
+            'ceph',
+            '--setgroup',
+            'ceph',
             '--default-log-to-file=false',
         ]
         log_to_journald = should_log_to_journald(self.ctx)
@@ -128,9 +138,7 @@ def get_ceph_mounts(
         no_config: bool = False,
     ) -> Dict[str, str]:
         # Warning: This is a hack done for more expedient refactoring
-        mounts = get_ceph_mounts_for_type(
-            ctx, ident.fsid, ident.daemon_type
-        )
+        mounts = get_ceph_mounts_for_type(ctx, ident.fsid, ident.daemon_type)
         data_dir = ident.data_dir(ctx.data_dir)
         if ident.daemon_type == 'rgw':
             cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (
@@ -270,11 +278,14 @@ class CephExporter(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict[str, Any],
-                 image: str = DEFAULT_IMAGE) -> None:
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        fsid: str,
+        daemon_id: Union[int, str],
+        config_json: Dict[str, Any],
+        image: str = DEFAULT_IMAGE,
+    ) -> None:
         self.ctx = ctx
         self.fsid = fsid
         self.daemon_id = daemon_id
@@ -291,13 +302,15 @@ def __init__(self,
         self.validate()
 
     @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'CephExporter':
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
+    ) -> 'CephExporter':
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephExporter':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'CephExporter':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -397,26 +410,44 @@ def get_ceph_mounts_for_type(
                     os.makedirs(selinux_folder, mode=0o755)
                 mounts[selinux_folder] = '/sys/fs/selinux:ro'
             else:
-                logger.error(f'Cluster direcotry {cluster_dir} does not exist.')
+                logger.error(
+                    f'Cluster direcotry {cluster_dir} does not exist.'
+                )
         mounts['/'] = '/rootfs'
 
     try:
-        if ctx.shared_ceph_folder:  # make easy manager modules/ceph-volume development
+        if (
+            ctx.shared_ceph_folder
+        ):  # make easy manager modules/ceph-volume development
             ceph_folder = pathify(ctx.shared_ceph_folder)
             if os.path.exists(ceph_folder):
                 cephadm_binary = ceph_folder + '/src/cephadm/cephadm'
                 if not os.path.exists(pathify(cephadm_binary)):
-                    raise Error("cephadm binary does not exist. Please run './build.sh cephadm' from ceph/src/cephadm/ directory.")
+                    raise Error(
+                        "cephadm binary does not exist. Please run './build.sh cephadm' from ceph/src/cephadm/ directory."
+                    )
                 mounts[cephadm_binary] = '/usr/sbin/cephadm'
-                mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume'
-                mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr'
-                mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph'
-                mounts[ceph_folder + '/monitoring/ceph-mixin/dashboards_out'] = '/etc/grafana/dashboards/ceph-dashboard'
-                mounts[ceph_folder + '/monitoring/ceph-mixin/prometheus_alerts.yml'] = '/etc/prometheus/ceph/ceph_default_alerts.yml'
+                mounts[
+                    ceph_folder + '/src/ceph-volume/ceph_volume'
+                ] = '/usr/lib/python3.6/site-packages/ceph_volume'
+                mounts[
+                    ceph_folder + '/src/pybind/mgr'
+                ] = '/usr/share/ceph/mgr'
+                mounts[
+                    ceph_folder + '/src/python-common/ceph'
+                ] = '/usr/lib/python3.6/site-packages/ceph'
+                mounts[
+                    ceph_folder + '/monitoring/ceph-mixin/dashboards_out'
+                ] = '/etc/grafana/dashboards/ceph-dashboard'
+                mounts[
+                    ceph_folder
+                    + '/monitoring/ceph-mixin/prometheus_alerts.yml'
+                ] = '/etc/prometheus/ceph/ceph_default_alerts.yml'
             else:
                 logger.error(
                     'Ceph shared source folder does not exist.',
-                    extra=Highlight.FAILURE.extra())
+                    extra=Highlight.FAILURE.extra(),
+                )
     except AttributeError:
         pass
     return mounts

From c7919151647553f39709cc713b06349f315be06d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:57:34 -0500
Subject: [PATCH 0838/2492] cephadm: black format daemons/custom.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/custom.py | 38 ++++++++++++++++--------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/custom.py b/src/cephadm/cephadmlib/daemons/custom.py
index 8e0d59e6f3a9..e833c80c9a5d 100644
--- a/src/cephadm/cephadmlib/daemons/custom.py
+++ b/src/cephadm/cephadmlib/daemons/custom.py
@@ -23,15 +23,20 @@
 @register_daemon_form
 class CustomContainer(ContainerDaemonForm):
     """Defines a custom container"""
+
     daemon_type = 'container'
 
     @classmethod
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
+    def __init__(
+        self,
+        fsid: str,
+        daemon_id: Union[int, str],
+        config_json: Dict,
+        image: str,
+    ) -> None:
         self.fsid = fsid
         self.daemon_id = daemon_id
         self.image = image
@@ -50,13 +55,15 @@ def __init__(self,
         self.files = dict_get(config_json, 'files', {})
 
     @classmethod
-    def init(cls, ctx: CephadmContext,
-             fsid: str, daemon_id: Union[int, str]) -> 'CustomContainer':
-        return cls(fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
+    ) -> 'CustomContainer':
+        return cls(fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CustomContainer':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'CustomContainer':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -67,8 +74,10 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
         """
         Create dirs/files below the container data directory.
         """
-        logger.info('Creating custom container configuration '
-                    'dirs/files in {} ...'.format(data_dir))
+        logger.info(
+            'Creating custom container configuration '
+            'dirs/files in {} ...'.format(data_dir)
+        )
 
         if not os.path.isdir(data_dir):
             raise OSError('data_dir is not a directory: %s' % data_dir)
@@ -82,7 +91,9 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
             logger.info('Creating file: {}'.format(file_path))
             content = dict_get_join(self.files, file_path)
             file_path = os.path.join(data_dir, file_path.strip('/'))
-            with write_new(file_path, owner=(uid, gid), encoding='utf-8') as f:
+            with write_new(
+                file_path, owner=(uid, gid), encoding='utf-8'
+            ) as f:
                 f.write(content)
 
     def get_daemon_args(self) -> List[str]:
@@ -146,8 +157,9 @@ def _get_container_binds(self, data_dir: str) -> List[List[str]]:
             for index, value in enumerate(bind):
                 match = re.match(r'^source=(.+)$', value)
                 if match:
-                    bind[index] = 'source={}'.format(os.path.join(
-                        data_dir, match.group(1)))
+                    bind[index] = 'source={}'.format(
+                        os.path.join(data_dir, match.group(1))
+                    )
         return binds
 
     def customize_container_binds(

From 505624de9fcf46d162914acd858404a4cd02b075 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:57:49 -0500
Subject: [PATCH 0839/2492] cephadm: black format daemons/ingress.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/ingress.py | 68 +++++++++++++++--------
 1 file changed, 45 insertions(+), 23 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/ingress.py b/src/cephadm/cephadmlib/daemons/ingress.py
index 94ee34505ed5..6064cf538fb0 100644
--- a/src/cephadm/cephadmlib/daemons/ingress.py
+++ b/src/cephadm/cephadmlib/daemons/ingress.py
@@ -22,6 +22,7 @@
 @register_daemon_form
 class HAproxy(ContainerDaemonForm):
     """Defines an HAproxy container"""
+
     daemon_type = 'haproxy'
     required_files = ['haproxy.cfg']
     default_image = DEFAULT_HAPROXY_IMAGE
@@ -30,10 +31,14 @@ class HAproxy(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        fsid: str,
+        daemon_id: Union[int, str],
+        config_json: Dict,
+        image: str,
+    ) -> None:
         self.ctx = ctx
         self.fsid = fsid
         self.daemon_id = daemon_id
@@ -45,10 +50,10 @@ def __init__(self,
         self.validate()
 
     @classmethod
-    def init(cls, ctx: CephadmContext,
-             fsid: str, daemon_id: Union[int, str]) -> 'HAproxy':
-        return cls(ctx, fsid, daemon_id, fetch_configs(ctx),
-                   ctx.image)
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
+    ) -> 'HAproxy':
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
     def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'HAproxy':
@@ -65,7 +70,9 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
 
         # create additional directories in data dir for HAproxy to use
         if not os.path.isdir(os.path.join(data_dir, 'haproxy')):
-            makedirs(os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE)
+            makedirs(
+                os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE
+            )
 
         data_dir = os.path.join(data_dir, 'haproxy')
         populate_files(data_dir, self.files, uid, gid)
@@ -86,7 +93,9 @@ def validate(self):
         if self.required_files:
             for fname in self.required_files:
                 if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
 
     def get_daemon_name(self):
         # type: () -> str
@@ -144,6 +153,7 @@ def customize_process_args(
 @register_daemon_form
 class Keepalived(ContainerDaemonForm):
     """Defines an Keepalived container"""
+
     daemon_type = 'keepalived'
     required_files = ['keepalived.conf']
     default_image = DEFAULT_KEEPALIVED_IMAGE
@@ -152,10 +162,14 @@ class Keepalived(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str, daemon_id: Union[int, str],
-                 config_json: Dict, image: str) -> None:
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        fsid: str,
+        daemon_id: Union[int, str],
+        config_json: Dict,
+        image: str,
+    ) -> None:
         self.ctx = ctx
         self.fsid = fsid
         self.daemon_id = daemon_id
@@ -167,13 +181,15 @@ def __init__(self,
         self.validate()
 
     @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'Keepalived':
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
+    ) -> 'Keepalived':
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Keepalived':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'Keepalived':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -187,7 +203,9 @@ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
 
         # create additional directories in data dir for keepalived to use
         if not os.path.isdir(os.path.join(data_dir, 'keepalived')):
-            makedirs(os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE)
+            makedirs(
+                os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE
+            )
 
         # populate files from the config-json
         populate_files(data_dir, self.files, uid, gid)
@@ -205,7 +223,9 @@ def validate(self):
         if self.required_files:
             for fname in self.required_files:
                 if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
 
     def get_daemon_name(self):
         # type: () -> str
@@ -225,7 +245,7 @@ def get_container_envs():
             'KEEPALIVED_AUTOCONF=false',
             'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf',
             'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf',
-            'KEEPALIVED_DEBUG=false'
+            'KEEPALIVED_DEBUG=false',
         ]
         return envs
 
@@ -244,7 +264,9 @@ def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
     @staticmethod
     def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts = dict()
-        mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
+        mounts[
+            os.path.join(data_dir, 'keepalived.conf')
+        ] = '/etc/keepalived/keepalived.conf'
         return mounts
 
     def customize_container_mounts(

From 196ad1855e218177a54b00bf4a1f54f2202675bf Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:58:03 -0500
Subject: [PATCH 0840/2492] cephadm: black format daemons/iscsi.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/iscsi.py | 73 +++++++++++++++++--------
 1 file changed, 51 insertions(+), 22 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/iscsi.py b/src/cephadm/cephadmlib/daemons/iscsi.py
index 504db6885583..1845a37bf4ef 100644
--- a/src/cephadm/cephadmlib/daemons/iscsi.py
+++ b/src/cephadm/cephadmlib/daemons/iscsi.py
@@ -34,11 +34,13 @@ class CephIscsi(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx: CephadmContext,
-                 ident: DaemonIdentity,
-                 config_json: Dict,
-                 image: str = DEFAULT_IMAGE):
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        config_json: Dict,
+        image: str = DEFAULT_IMAGE,
+    ):
         self.ctx = ctx
         self._identity = ident
         self.image = image
@@ -50,11 +52,17 @@ def __init__(self,
         self.validate()
 
     @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str, daemon_id: str) -> 'CephIscsi':
-        return cls.create(ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id))
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: str
+    ) -> 'CephIscsi':
+        return cls.create(
+            ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id)
+        )
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephIscsi':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'CephIscsi':
         return cls(ctx, ident, fetch_configs(ctx), ctx.image)
 
     @property
@@ -75,9 +83,13 @@ def _get_container_mounts(data_dir, log_dir):
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
         mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
-        mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z'
+        mounts[
+            os.path.join(data_dir, 'iscsi-gateway.cfg')
+        ] = '/etc/ceph/iscsi-gateway.cfg:z'
         mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
-        mounts[os.path.join(data_dir, 'tcmu-runner-entrypoint.sh')] = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
+        mounts[
+            os.path.join(data_dir, 'tcmu-runner-entrypoint.sh')
+        ] = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
         mounts[log_dir] = '/var/log:z'
         mounts['/dev'] = '/dev'
         return mounts
@@ -108,11 +120,18 @@ def customize_container_binds(
     def get_version(ctx, container_id):
         # type: (CephadmContext, str) -> Optional[str]
         version = None
-        out, err, code = call(ctx,
-                              [ctx.container_engine.path, 'exec', container_id,
-                               '/usr/bin/python3', '-c',
-                               "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"],
-                              verbosity=CallVerbosity.QUIET)
+        out, err, code = call(
+            ctx,
+            [
+                ctx.container_engine.path,
+                'exec',
+                container_id,
+                '/usr/bin/python3',
+                '-c',
+                "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)",
+            ],
+            verbosity=CallVerbosity.QUIET,
+        )
         if code == 0:
             version = out.strip()
         return version
@@ -130,7 +149,9 @@ def validate(self):
         if self.required_files:
             for fname in self.required_files:
                 if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
 
     def get_daemon_name(self):
         # type: () -> str
@@ -157,7 +178,9 @@ def create_daemon_dirs(self, data_dir, uid, gid):
         # to be mounted into the container. For more info
         # on why we need this script, see the
         # tcmu_runner_entrypoint_script function
-        self.files['tcmu-runner-entrypoint.sh'] = self.tcmu_runner_entrypoint_script()
+        self.files[
+            'tcmu-runner-entrypoint.sh'
+        ] = self.tcmu_runner_entrypoint_script()
 
         # populate files from the config-json
         populate_files(data_dir, self.files, uid, gid)
@@ -171,11 +194,15 @@ def configfs_mount_umount(data_dir, mount=True):
         # type: (str, bool) -> List[str]
         mount_path = os.path.join(data_dir, 'configfs')
         if mount:
-            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
-                  'mount -t configfs none {0}; fi'.format(mount_path)
+            cmd = (
+                'if ! grep -qs {0} /proc/mounts; then '
+                'mount -t configfs none {0}; fi'.format(mount_path)
+            )
         else:
-            cmd = 'if grep -qs {0} /proc/mounts; then ' \
-                  'umount {0}; fi'.format(mount_path)
+            cmd = (
+                'if grep -qs {0} /proc/mounts; then '
+                'umount {0}; fi'.format(mount_path)
+            )
         return cmd.split()
 
     @staticmethod
@@ -230,7 +257,9 @@ def get_tcmu_runner_container(self):
         # TODO: Eventually we don't want to run tcmu-runner through this script.
         # This is intended to be a workaround backported to older releases
         # and should eventually be removed in at least squid onward
-        tcmu_container.entrypoint = '/usr/local/scripts/tcmu-runner-entrypoint.sh'
+        tcmu_container.entrypoint = (
+            '/usr/local/scripts/tcmu-runner-entrypoint.sh'
+        )
         tcmu_container.cname = self.get_container_name(desc='tcmu')
         return tcmu_container
 

From 8535d57b3b5bf778ec8323b31d4b969d8e7dff27 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:58:17 -0500
Subject: [PATCH 0841/2492] cephadm: black format daemons/monitoring.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/monitoring.py | 71 ++++++++++++++------
 1 file changed, 49 insertions(+), 22 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/monitoring.py b/src/cephadm/cephadmlib/daemons/monitoring.py
index 405dafc6dfcc..e0667853dd7b 100644
--- a/src/cephadm/cephadmlib/daemons/monitoring.py
+++ b/src/cephadm/cephadmlib/daemons/monitoring.py
@@ -27,12 +27,14 @@ class Monitoring(ContainerDaemonForm):
     """Define the configs for the monitoring containers"""
 
     port_map = {
-        'prometheus': [9095],  # Avoid default 9090, due to conflict with cockpit UI
+        'prometheus': [
+            9095
+        ],  # Avoid default 9090, due to conflict with cockpit UI
         'node-exporter': [9100],
         'grafana': [3000],
         'alertmanager': [9093, 9094],
         'loki': [3100],
-        'promtail': [9080]
+        'promtail': [9080],
     }
 
     components = {
@@ -55,9 +57,7 @@ class Monitoring(ContainerDaemonForm):
             'args': [
                 '--config.file=/etc/loki/loki.yml',
             ],
-            'config-json-files': [
-                'loki.yml'
-            ],
+            'config-json-files': ['loki.yml'],
         },
         'promtail': {
             'image': DEFAULT_PROMTAIL_IMAGE,
@@ -74,9 +74,7 @@ class Monitoring(ContainerDaemonForm):
             'image': DEFAULT_NODE_EXPORTER_IMAGE,
             'cpus': '1',
             'memory': '1GB',
-            'args': [
-                '--no-collector.timex'
-            ],
+            'args': ['--no-collector.timex'],
         },
         'grafana': {
             'image': DEFAULT_GRAFANA_IMAGE,
@@ -95,7 +93,9 @@ class Monitoring(ContainerDaemonForm):
             'cpus': '2',
             'memory': '2GB',
             'args': [
-                '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]),
+                '--cluster.listen-address=:{}'.format(
+                    port_map['alertmanager'][1]
+                ),
             ],
             'config-json-files': [
                 'alertmanager.yml',
@@ -116,7 +116,13 @@ def get_version(ctx, container_id, daemon_type):
         """
         :param: daemon_type Either "prometheus", "alertmanager", "loki", "promtail" or "node-exporter"
         """
-        assert daemon_type in ('prometheus', 'alertmanager', 'node-exporter', 'loki', 'promtail')
+        assert daemon_type in (
+            'prometheus',
+            'alertmanager',
+            'node-exporter',
+            'loki',
+            'promtail',
+        )
         cmd = daemon_type.replace('-', '_')
         code = -1
         err = ''
@@ -124,17 +130,32 @@ def get_version(ctx, container_id, daemon_type):
         version = ''
         if daemon_type == 'alertmanager':
             for cmd in ['alertmanager', 'prometheus-alertmanager']:
-                out, err, code = call(ctx, [
-                    ctx.container_engine.path, 'exec', container_id, cmd,
-                    '--version'
-                ], verbosity=CallVerbosity.QUIET)
+                out, err, code = call(
+                    ctx,
+                    [
+                        ctx.container_engine.path,
+                        'exec',
+                        container_id,
+                        cmd,
+                        '--version',
+                    ],
+                    verbosity=CallVerbosity.QUIET,
+                )
                 if code == 0:
                     break
             cmd = 'alertmanager'  # reset cmd for version extraction
         else:
-            out, err, code = call(ctx, [
-                ctx.container_engine.path, 'exec', container_id, cmd, '--version'
-            ], verbosity=CallVerbosity.QUIET)
+            out, err, code = call(
+                ctx,
+                [
+                    ctx.container_engine.path,
+                    'exec',
+                    container_id,
+                    cmd,
+                    '--version',
+                ],
+                verbosity=CallVerbosity.QUIET,
+            )
         if code == 0:
             if err.startswith('%s, version ' % cmd):
                 version = err.split(' ')[2]
@@ -169,7 +190,9 @@ def __init__(self, ctx: CephadmContext, ident: DaemonIdentity) -> None:
         self._identity = ident
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'Monitoring':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'Monitoring':
         return cls(ctx, ident)
 
     @property
@@ -232,7 +255,9 @@ def get_daemon_args(self) -> List[str]:
             if daemon_type == 'prometheus':
                 config = fetch_configs(ctx)
                 retention_time = config.get('retention_time', '15d')
-                retention_size = config.get('retention_size', '0')  # default to disabled
+                retention_size = config.get(
+                    'retention_size', '0'
+                )  # default to disabled
                 r += [f'--storage.tsdb.retention.time={retention_time}']
                 r += [f'--storage.tsdb.retention.size={retention_size}']
                 scheme = 'http'
@@ -270,9 +295,11 @@ def get_daemon_args(self) -> List[str]:
                 r += [f'--web.config.file={config["web_config"]}']
             except KeyError:
                 pass
-            r += ['--path.procfs=/host/proc',
-                  '--path.sysfs=/host/sys',
-                  '--path.rootfs=/rootfs']
+            r += [
+                '--path.procfs=/host/proc',
+                '--path.sysfs=/host/sys',
+                '--path.rootfs=/rootfs',
+            ]
         return r
 
     def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:

From 192ff2f6ef34e3ba858bcc93384123cf19293ee1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:58:30 -0500
Subject: [PATCH 0842/2492] cephadm: black format daemons/nfs.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/nfs.py | 41 ++++++++++++++++-----------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/nfs.py b/src/cephadm/cephadmlib/daemons/nfs.py
index 48653b775fb0..6e2f2a945cae 100644
--- a/src/cephadm/cephadmlib/daemons/nfs.py
+++ b/src/cephadm/cephadmlib/daemons/nfs.py
@@ -41,12 +41,9 @@ class NFSGanesha(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx,
-                 fsid,
-                 daemon_id,
-                 config_json,
-                 image=DEFAULT_IMAGE):
+    def __init__(
+        self, ctx, fsid, daemon_id, config_json, image=DEFAULT_IMAGE
+    ):
         # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
         self.ctx = ctx
         self.fsid = fsid
@@ -70,7 +67,9 @@ def init(cls, ctx, fsid, daemon_id):
         return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'NFSGanesha':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'NFSGanesha':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -86,8 +85,9 @@ def _get_container_mounts(self, data_dir):
         if self.rgw:
             cluster = self.rgw.get('cluster', 'ceph')
             rgw_user = self.rgw.get('user', 'admin')
-            mounts[os.path.join(data_dir, 'keyring.rgw')] = \
-                '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
+            mounts[
+                os.path.join(data_dir, 'keyring.rgw')
+            ] = '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
         return mounts
 
     def customize_container_mounts(
@@ -99,19 +99,24 @@ def customize_container_mounts(
     @staticmethod
     def get_container_envs():
         # type: () -> List[str]
-        envs = [
-            'CEPH_CONF=%s' % (CEPH_DEFAULT_CONF)
-        ]
+        envs = ['CEPH_CONF=%s' % (CEPH_DEFAULT_CONF)]
         return envs
 
     @staticmethod
     def get_version(ctx, container_id):
         # type: (CephadmContext, str) -> Optional[str]
         version = None
-        out, err, code = call(ctx,
-                              [ctx.container_engine.path, 'exec', container_id,
-                               NFSGanesha.entrypoint, '-v'],
-                              verbosity=CallVerbosity.QUIET)
+        out, err, code = call(
+            ctx,
+            [
+                ctx.container_engine.path,
+                'exec',
+                container_id,
+                NFSGanesha.entrypoint,
+                '-v',
+            ],
+            verbosity=CallVerbosity.QUIET,
+        )
         if code == 0:
             match = re.search(r'NFS-Ganesha Release\s*=\s*[V]*([\d.]+)', out)
             if match:
@@ -131,7 +136,9 @@ def validate(self):
         if self.required_files:
             for fname in self.required_files:
                 if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
 
         # check for an RGW config
         if self.rgw:

From bc0c9d5422141339b651b6a96cf39f04820e7cc8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:58:41 -0500
Subject: [PATCH 0843/2492] cephadm: black format daemons/nvmeof.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/nvmeof.py | 50 +++++++++++++++---------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
index a1a18f5a18ba..39488406bc8e 100644
--- a/src/cephadm/cephadmlib/daemons/nvmeof.py
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -32,12 +32,9 @@ class CephNvmeof(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx,
-                 fsid,
-                 daemon_id,
-                 config_json,
-                 image=DEFAULT_NVMEOF_IMAGE):
+    def __init__(
+        self, ctx, fsid, daemon_id, config_json, image=DEFAULT_NVMEOF_IMAGE
+    ):
         # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
         self.ctx = ctx
         self.fsid = fsid
@@ -53,11 +50,12 @@ def __init__(self,
     @classmethod
     def init(cls, ctx, fsid, daemon_id):
         # type: (CephadmContext, str, Union[int, str]) -> CephNvmeof
-        return cls(ctx, fsid, daemon_id,
-                   fetch_configs(ctx), ctx.image)
+        return cls(ctx, fsid, daemon_id, fetch_configs(ctx), ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'CephNvmeof':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'CephNvmeof':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -69,7 +67,9 @@ def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
         mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
-        mounts[os.path.join(data_dir, 'ceph-nvmeof.conf')] = '/src/ceph-nvmeof.conf:z'
+        mounts[
+            os.path.join(data_dir, 'ceph-nvmeof.conf')
+        ] = '/src/ceph-nvmeof.conf:z'
         mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
         mounts['/dev/hugepages'] = '/dev/hugepages'
         mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
@@ -94,10 +94,16 @@ def customize_container_binds(
 
     @staticmethod
     def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
-        out, err, ret = call(ctx,
-                             [ctx.container_engine.path, 'inspect',
-                              '--format', '{{index .Config.Labels "io.ceph.version"}}',
-                              ctx.image])
+        out, err, ret = call(
+            ctx,
+            [
+                ctx.container_engine.path,
+                'inspect',
+                '--format',
+                '{{index .Config.Labels "io.ceph.version"}}',
+                ctx.image,
+            ],
+        )
         version = None
         if ret == 0:
             version = out.strip()
@@ -116,7 +122,9 @@ def validate(self):
         if self.required_files:
             for fname in self.required_files:
                 if fname not in self.files:
-                    raise Error('required file missing from config-json: %s' % fname)
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
 
     def get_daemon_name(self):
         # type: () -> str
@@ -147,11 +155,15 @@ def configfs_mount_umount(data_dir, mount=True):
         # type: (str, bool) -> List[str]
         mount_path = os.path.join(data_dir, 'configfs')
         if mount:
-            cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
-                  'mount -t configfs none {0}; fi'.format(mount_path)
+            cmd = (
+                'if ! grep -qs {0} /proc/mounts; then '
+                'mount -t configfs none {0}; fi'.format(mount_path)
+            )
         else:
-            cmd = 'if grep -qs {0} /proc/mounts; then ' \
-                  'umount {0}; fi'.format(mount_path)
+            cmd = (
+                'if grep -qs {0} /proc/mounts; then '
+                'umount {0}; fi'.format(mount_path)
+            )
         return cmd.split()
 
     @staticmethod

From b0f1c7fff3d52e4867c0d74a6124be87865f6953 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:58:53 -0500
Subject: [PATCH 0844/2492] cephadm: black format daemons/snmp.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/snmp.py | 103 +++++++++++++++++--------
 1 file changed, 70 insertions(+), 33 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/snmp.py b/src/cephadm/cephadmlib/daemons/snmp.py
index dc952aa4cb9d..f334e5f76522 100644
--- a/src/cephadm/cephadmlib/daemons/snmp.py
+++ b/src/cephadm/cephadmlib/daemons/snmp.py
@@ -21,6 +21,7 @@
 @register_daemon_form
 class SNMPGateway(ContainerDaemonForm):
     """Defines an SNMP gateway between Prometheus and SNMP monitoring Frameworks"""
+
     daemon_type = 'snmp-gateway'
     SUPPORTED_VERSIONS = ['V2c', 'V3']
     default_image = DEFAULT_SNMP_GATEWAY_IMAGE
@@ -31,12 +32,14 @@ class SNMPGateway(ContainerDaemonForm):
     def for_daemon_type(cls, daemon_type: str) -> bool:
         return cls.daemon_type == daemon_type
 
-    def __init__(self,
-                 ctx: CephadmContext,
-                 fsid: str,
-                 daemon_id: Union[int, str],
-                 config_json: Dict[str, Any],
-                 image: Optional[str] = None) -> None:
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        fsid: str,
+        daemon_id: Union[int, str],
+        config_json: Dict[str, Any],
+        image: Optional[str] = None,
+    ) -> None:
         self.ctx = ctx
         self.fsid = fsid
         self.daemon_id = daemon_id
@@ -49,24 +52,37 @@ def __init__(self,
         self.snmp_version = config_json.get('snmp_version', 'V2c')
         self.snmp_community = config_json.get('snmp_community', 'public')
         self.log_level = config_json.get('log_level', 'info')
-        self.snmp_v3_auth_username = config_json.get('snmp_v3_auth_username', '')
-        self.snmp_v3_auth_password = config_json.get('snmp_v3_auth_password', '')
-        self.snmp_v3_auth_protocol = config_json.get('snmp_v3_auth_protocol', '')
-        self.snmp_v3_priv_protocol = config_json.get('snmp_v3_priv_protocol', '')
-        self.snmp_v3_priv_password = config_json.get('snmp_v3_priv_password', '')
+        self.snmp_v3_auth_username = config_json.get(
+            'snmp_v3_auth_username', ''
+        )
+        self.snmp_v3_auth_password = config_json.get(
+            'snmp_v3_auth_password', ''
+        )
+        self.snmp_v3_auth_protocol = config_json.get(
+            'snmp_v3_auth_protocol', ''
+        )
+        self.snmp_v3_priv_protocol = config_json.get(
+            'snmp_v3_priv_protocol', ''
+        )
+        self.snmp_v3_priv_password = config_json.get(
+            'snmp_v3_priv_password', ''
+        )
         self.snmp_v3_engine_id = config_json.get('snmp_v3_engine_id', '')
 
         self.validate()
 
     @classmethod
-    def init(cls, ctx: CephadmContext, fsid: str,
-             daemon_id: Union[int, str]) -> 'SNMPGateway':
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
+    ) -> 'SNMPGateway':
         cfgs = fetch_configs(ctx)
         assert cfgs  # assert some config data was found
         return cls(ctx, fsid, daemon_id, cfgs, ctx.image)
 
     @classmethod
-    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'SNMPGateway':
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'SNMPGateway':
         return cls.init(ctx, ident.fsid, ident.daemon_id)
 
     @property
@@ -74,9 +90,13 @@ def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
     @staticmethod
-    def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]:
+    def get_version(
+        ctx: CephadmContext, fsid: str, daemon_id: str
+    ) -> Optional[str]:
         """Return the version of the notifier from it's http endpoint"""
-        path = os.path.join(ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta')
+        path = os.path.join(
+            ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta'
+        )
         try:
             with open(path, 'r') as env:
                 metadata = json.loads(env.read())
@@ -95,8 +115,9 @@ def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]
 
         for h in html:
             stripped = h.strip()
-            if stripped.startswith(('<pre>', '<PRE>')) and \
-               stripped.endswith(('</pre>', '</PRE>')):
+            if stripped.startswith(('<pre>', '<PRE>')) and stripped.endswith(
+                ('</pre>', '</PRE>')
+            ):
                 # <pre>(version=1.2.1, branch=HEAD, revision=7...
                 return stripped.split(',')[0].split('version=')[1]
 
@@ -116,28 +137,36 @@ def get_daemon_args(self) -> List[str]:
             f'--snmp.destination={self.destination}',
             f'--snmp.version={self.snmp_version}',
             f'--log.level={self.log_level}',
-            '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
+            '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl',
         ]
 
         if self.snmp_version == 'V3':
             # common auth settings
-            v3_args.extend([
-                '--snmp.authentication-enabled',
-                f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}',
-                f'--snmp.security-engine-id={self.snmp_v3_engine_id}'
-            ])
+            v3_args.extend(
+                [
+                    '--snmp.authentication-enabled',
+                    f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}',
+                    f'--snmp.security-engine-id={self.snmp_v3_engine_id}',
+                ]
+            )
             # authPriv setting is applied if we have a privacy protocol setting
             if self.snmp_v3_priv_protocol:
-                v3_args.extend([
-                    '--snmp.private-enabled',
-                    f'--snmp.private-protocol={self.snmp_v3_priv_protocol}'
-                ])
+                v3_args.extend(
+                    [
+                        '--snmp.private-enabled',
+                        f'--snmp.private-protocol={self.snmp_v3_priv_protocol}',
+                    ]
+                )
 
         return base_args + v3_args
 
     @property
     def data_dir(self) -> str:
-        return os.path.join(self.ctx.data_dir, self.ctx.fsid, f'{self.daemon_type}.{self.daemon_id}')
+        return os.path.join(
+            self.ctx.data_dir,
+            self.ctx.fsid,
+            f'{self.daemon_type}.{self.daemon_id}',
+        )
 
     @property
     def conf_file_path(self) -> str:
@@ -149,10 +178,16 @@ def create_daemon_conf(self) -> None:
             if self.snmp_version == 'V2c':
                 f.write(f'SNMP_NOTIFIER_COMMUNITY={self.snmp_community}\n')
             else:
-                f.write(f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n')
-                f.write(f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n')
+                f.write(
+                    f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n'
+                )
+                f.write(
+                    f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n'
+                )
                 if self.snmp_v3_priv_password:
-                    f.write(f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n')
+                    f.write(
+                        f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n'
+                    )
 
     def validate(self) -> None:
         """Validate the settings
@@ -169,7 +204,9 @@ def validate(self) -> None:
             raise Error(f'not a valid snmp version: {self.snmp_version}')
 
         if not self.destination:
-            raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
+            raise Error(
+                'config is missing destination attribute(<ip>:<port>) of the target SNMP listener'
+            )
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = daemon_to_container(ctx, self)

From ed6f171ba028cb1bdd049300ad9fab007272214e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 9 Nov 2023 13:59:07 -0500
Subject: [PATCH 0845/2492] cephadm: black format daemons/tracing.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/tracing.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/tracing.py b/src/cephadm/cephadmlib/daemons/tracing.py
index f178bd664086..4d4fecacbb0f 100644
--- a/src/cephadm/cephadmlib/daemons/tracing.py
+++ b/src/cephadm/cephadmlib/daemons/tracing.py
@@ -27,7 +27,7 @@ class Tracing(ContainerDaemonForm):
     components: Dict[str, Dict[str, Any]] = {
         'elasticsearch': {
             'image': DEFAULT_ELASTICSEARCH_IMAGE,
-            'envs': ['discovery.type=single-node']
+            'envs': ['discovery.type=single-node'],
         },
         'jaeger-agent': {
             'image': DEFAULT_JAEGER_AGENT_IMAGE,
@@ -50,12 +50,13 @@ def set_configuration(config: Dict[str, str], daemon_type: str) -> None:
             assert 'elasticsearch_nodes' in config
             Tracing.components[daemon_type]['envs'] = [
                 'SPAN_STORAGE_TYPE=elasticsearch',
-                f'ES_SERVER_URLS={config["elasticsearch_nodes"]}']
+                f'ES_SERVER_URLS={config["elasticsearch_nodes"]}',
+            ]
         if daemon_type == 'jaeger-agent':
             assert 'collector_nodes' in config
             Tracing.components[daemon_type]['daemon_args'] = [
                 f'--reporter.grpc.host-port={config["collector_nodes"]}',
-                '--processor.jaeger-compact.server-host-port=6799'
+                '--processor.jaeger-compact.server-host-port=6799',
             ]
 
     def __init__(self, ident: DaemonIdentity) -> None:

From 2c8faf256df6209994df31c93ebb19a41f038efa Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 30 Nov 2023 14:34:41 +0530
Subject: [PATCH 0846/2492] mgr/dashboard: generalized code-block component

Fixes: https://tracker.ceph.com/issues/63608
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../cephfs-form/cephfs-form.component.html    | 12 +-----
 .../code-block/code-block.component.html      | 21 ++++++++++
 .../code-block/code-block.component.scss      |  6 +++
 .../code-block/code-block.component.spec.ts   | 38 +++++++++++++++++++
 .../code-block/code-block.component.ts        | 11 ++++++
 .../shared/components/components.module.ts    |  7 +++-
 .../styles/defaults/_bootstrap-defaults.scss  |  2 +
 7 files changed, 85 insertions(+), 12 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
index 18db21f891f5..f8d0fa803204 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
@@ -23,16 +23,8 @@
                       *ngIf="editing && disableRename">
         <p>The File System can only be renamed if it is shutdown and `refuse_client_session` is set to true.
            Follow the steps below in the command line and refresh the page:</p>
-        <pre class="d-flex">{{ fsFailCmd }}
-          <cd-copy-2-clipboard-button [source]="fsFailCmd"
-                                      [byId]="false"
-                                      [showIconOnly]="true"></cd-copy-2-clipboard-button>
-        </pre>
-        <pre class="d-flex">{{ fsSetCmd }}
-          <cd-copy-2-clipboard-button [source]="fsSetCmd"
-                                      [byId]="false"
-                                      [showIconOnly]="true"></cd-copy-2-clipboard-button>
-        </pre>
+        <cd-code-block [codes]="[fsFailCmd]"></cd-code-block>
+        <cd-code-block [codes]="[fsSetCmd]"></cd-code-block>
       </cd-alert-panel>
 
       <div class="card-body">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
new file mode 100644
index 000000000000..7cf78b8d1fbe
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
@@ -0,0 +1,21 @@
+<ng-container *ngIf="codes.length > 1; else singleCodeBlock">
+  <pre id="bigCodeBlock">
+    <span *ngFor="let code of codes"
+          class="d-flex p-2 align-items-center justify-content-between text-dark">
+      <span>{{code}}</span>
+      <cd-copy-2-clipboard-button
+          [source]="code"
+          [byId]="false"></cd-copy-2-clipboard-button>
+    </span>
+  </pre>
+</ng-container>
+
+<ng-template #singleCodeBlock>
+  <pre class="d-flex p-2 align-items-center justify-content-between text-dark"
+       id="singleCodeBlock">
+    <span>{{codes}}</span>
+    <cd-copy-2-clipboard-button
+      [source]="codes"
+      [byId]="false"></cd-copy-2-clipboard-button>
+  </pre>
+</ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
new file mode 100644
index 000000000000..f601dfe6609f
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
@@ -0,0 +1,6 @@
+@use './src/styles/vendor/variables' as vv;
+
+pre {
+  background-color: vv.$code-block-bg;
+  border-radius: 0.5rem;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.spec.ts
new file mode 100644
index 000000000000..bc5ad428fd8a
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.spec.ts
@@ -0,0 +1,38 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CodeBlockComponent } from './code-block.component';
+import { configureTestBed } from '~/testing/unit-test-helper';
+
+describe('CodeBlockComponent', () => {
+  let component: CodeBlockComponent;
+  let fixture: ComponentFixture<CodeBlockComponent>;
+
+  configureTestBed({
+    declarations: [CodeBlockComponent]
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(CodeBlockComponent);
+    component = fixture.componentInstance;
+    component.codes = [];
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+
+  it('should show single codeblock if there are only one code', () => {
+    component.codes = ['code'];
+    fixture.detectChanges();
+    expect(fixture.nativeElement.querySelector('#singleCodeBlock')).not.toBeNull();
+    expect(fixture.nativeElement.querySelector('#bigCodeBlock')).toBeNull();
+  });
+
+  it('should show single codeblock if there are only one code', () => {
+    component.codes = ['code1', 'code2'];
+    fixture.detectChanges();
+    expect(fixture.nativeElement.querySelector('#bigCodeBlock')).not.toBeNull();
+    expect(fixture.nativeElement.querySelector('#singleCodeBlock')).toBeNull();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts
new file mode 100644
index 000000000000..91d2d991f378
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts
@@ -0,0 +1,11 @@
+import { Component, Input } from '@angular/core';
+
+@Component({
+  selector: 'cd-code-block',
+  templateUrl: './code-block.component.html',
+  styleUrls: ['./code-block.component.scss']
+})
+export class CodeBlockComponent {
+  @Input()
+  codes: string[];
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index 17f418d1e148..327d208ef386 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -51,6 +51,7 @@ import { UsageBarComponent } from './usage-bar/usage-bar.component';
 import { WizardComponent } from './wizard/wizard.component';
 import { CardComponent } from './card/card.component';
 import { CardRowComponent } from './card-row/card-row.component';
+import { CodeBlockComponent } from './code-block/code-block.component';
 
 @NgModule({
   imports: [
@@ -105,7 +106,8 @@ import { CardRowComponent } from './card-row/card-row.component';
     CdLabelComponent,
     ColorClassFromTextPipe,
     CardComponent,
-    CardRowComponent
+    CardRowComponent,
+    CodeBlockComponent
   ],
   providers: [],
   exports: [
@@ -137,7 +139,8 @@ import { CardRowComponent } from './card-row/card-row.component';
     CustomLoginBannerComponent,
     CdLabelComponent,
     CardComponent,
-    CardRowComponent
+    CardRowComponent,
+    CodeBlockComponent
   ]
 })
 export class ComponentsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss b/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
index e9c8a595620a..d69abf12bc8e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
@@ -96,6 +96,8 @@ $chart-color-translucent-blue: #0096dc80 !default;
 $chart-color-border: #00000020 !default;
 $chart-color-translucent-yellow: #ef923472 !default;
 
+$code-block-bg: #f7f7f9 !default;
+
 // Typography
 
 $font-family-sans-serif: 'Helvetica Neue', Helvetica, Arial, 'Noto Sans', sans-serif,

From 36ee8e7ed365933b264321c15f77c0ed1e352d8c Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Fri, 1 Dec 2023 04:32:20 -0500
Subject: [PATCH 0847/2492] mds: encode `bal_rank_mask` using a higher (next)
 version

This is required for a reef client to work with a higher revision
MDS, since essentially, this happens:

reef(client):

if (version >=17) {
    decode(bal_rank_mask, p);
}

and higher-revision MDS (say, upcoming squid):

version = 17
encode(version, bl);
...
...
encode(max_xattr_size, bl);
encode(bal_rank_mask, bl);

The client incorrectly decodes max_xattr_size (type: uint64_t) into
bal_rank_mask (type: string).

This situation ended up due to a couple of reasons:

* the kclient patchset hanlding `max_xattr_size` was merged early on
  and another MDS side change that bumped the MDSMap encoding version
  to 17 got merged in the midst (PR #43284). Details in comment:

        https://github.com/ceph/ceph/pull/46357#issuecomment-1293556227

* The reef backport for PR #46357 got delayed (and, reef branched out).

Which means reef(18.2.0) user-space clients are broken with higher version
MDSs.

Fixes: https://tracker.ceph.com/issues/63713
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDSMap.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 47c823bf7635..013aefcf0480 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -770,7 +770,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
   encode(data_pools, bl);
   encode(cas_pool, bl);
 
-  __u16 ev = 17;
+  __u16 ev = 18;
   encode(ev, bl);
   encode(compat, bl);
   encode(metadata_pool, bl);
@@ -947,6 +947,9 @@ void MDSMap::decode(bufferlist::const_iterator& p)
 
   if (ev >= 17) {
     decode(max_xattr_size, p);
+  }
+
+  if (ev >= 18) {
     decode(bal_rank_mask, p);
   }
 

From e63fa697b72b3c12d98169958b6dd74cc6eb5486 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 30 Nov 2023 09:16:38 +0100
Subject: [PATCH 0848/2492] doc/rados: improve formatting of log-and-debug.rst

Improve the arrangment of information in the section "Ceph Subsystems"
in doc/rados/troubleshooting/log-and-debug.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/log-and-debug.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
index 7c3f848b5461..7d2885522132 100644
--- a/doc/rados/troubleshooting/log-and-debug.rst
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -175,9 +175,12 @@ For each subsystem, there is a logging level for its output logs (a so-called
 "log level") and a logging level for its in-memory logs (a so-called "memory
 level"). Different values may be set for these two logging levels in each
 subsystem. Ceph's logging levels operate on a scale of ``1`` to ``20``, where
-``1`` is terse and ``20`` is verbose [#f1]_.  As a general rule, the in-memory
-logs are not sent to the output log unless one or more of the following
-conditions obtain:
+``1`` is terse and ``20`` is verbose.  In certain rare cases, there are logging
+levels that can take a value greater than 20. The resulting logs are extremely
+verbose.
+
+The in-memory logs are not sent to the output log unless one or more of the
+following conditions are true:
 
 - a fatal signal has been raised or
 - an assertion within Ceph code has been triggered or
@@ -186,9 +189,6 @@ conditions obtain:
   that provides an example of how to submit admin socket commands
   <http://docs.ceph.com/en/latest/man/8/ceph/#daemon>`_ for more detail.
 
-.. warning ::
-   .. [#f1] In certain rare cases, there are logging levels that can take a value greater than 20. The resulting logs are extremely verbose.
-
 Log levels and memory levels can be set either together or separately. If a
 subsystem is assigned a single value, then that value determines both the log
 level and the memory level. For example, ``debug ms = 5`` will give the ``ms``

From b35be54ed9f23b7fc7859f054902e37cb88cefd8 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 18 Oct 2023 23:16:09 +0530
Subject: [PATCH 0849/2492] mgr/dashboard: cephfs subvolume list snapshots

Added a tab for displaying the subvolume snapshots
- this tab will show an info alert when there are no subvolumes present
- if the subvolume is present, then it'll be auto-selected by default

Implemented a filter to search the groups and subvolumes by its name.
Also added a scrollbar when there are too many items in the nav list

Modified the REST APIs to fetch only the names of the resources and
fetch the info when an API call is requesting for it.

Added unit tests

Fixes: https://tracker.ceph.com/issues/63237
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  87 +++++++---
 .../cephfs-subvolume-group.component.ts       |   2 +-
 .../cephfs-subvolume-list.component.html      |  23 +--
 .../cephfs-subvolume-list.component.ts        |  18 ++-
 ...fs-subvolume-snapshots-list.component.html |  36 +++++
 ...fs-subvolume-snapshots-list.component.scss |   0
 ...subvolume-snapshots-list.component.spec.ts |  38 +++++
 ...phfs-subvolume-snapshots-list.component.ts | 148 ++++++++++++++++++
 .../cephfs-tabs/cephfs-tabs.component.html    |   8 +
 .../src/app/ceph/cephfs/cephfs.module.ts      |   4 +-
 .../api/cephfs-subvolume-group.service.ts     |  10 +-
 .../api/cephfs-subvolume.service.spec.ts      |  10 +-
 .../shared/api/cephfs-subvolume.service.ts    |  30 +++-
 .../shared/components/components.module.ts    |   7 +-
 .../vertical-navigation.component.html        |  24 +++
 .../vertical-navigation.component.scss        |   3 +
 .../vertical-navigation.component.spec.ts     |  60 +++++++
 .../vertical-navigation.component.ts          |  37 +++++
 .../models/cephfs-subvolume-group.model.ts    |   2 +-
 .../shared/models/cephfs-subvolume.model.ts   |  10 ++
 .../models/cephfs-subvolumegroup.model.ts     |  13 --
 src/pybind/mgr/dashboard/openapi.yaml         |  86 ++++++++++
 22 files changed, 588 insertions(+), 68 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.ts
 delete mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolumegroup.model.ts

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index a1066cbae0db..eb1f764c472b 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -676,7 +676,7 @@ def ls_dir(self, fs_id, path=None, depth=1):
 @APIDoc('CephFS Subvolume Management API', 'CephFSSubvolume')
 class CephFSSubvolume(RESTController):
 
-    def get(self, vol_name: str, group_name: str = ""):
+    def get(self, vol_name: str, group_name: str = "", info=True):
         params = {'vol_name': vol_name}
         if group_name:
             params['group_name'] = group_name
@@ -687,15 +687,17 @@ def get(self, vol_name: str, group_name: str = ""):
                 f'Failed to list subvolumes for volume {vol_name}: {err}'
             )
         subvolumes = json.loads(out)
-        for subvolume in subvolumes:
-            params['sub_name'] = subvolume['name']
-            error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_info', None,
-                                              params)
-            if error_code != 0:
-                raise DashboardException(
-                    f'Failed to get info for subvolume {subvolume["name"]}: {err}'
-                )
-            subvolume['info'] = json.loads(out)
+
+        if info:
+            for subvolume in subvolumes:
+                params['sub_name'] = subvolume['name']
+                error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_info', None,
+                                                  params)
+                if error_code != 0:
+                    raise DashboardException(
+                        f'Failed to get info for subvolume {subvolume["name"]}: {err}'
+                    )
+                subvolume['info'] = json.loads(out)
         return subvolumes
 
     @RESTController.Resource('GET')
@@ -752,12 +754,27 @@ def delete(self, vol_name: str, subvol_name: str, group_name: str = "",
                 component='cephfs')
         return f'Subvolume {subvol_name} removed successfully'
 
+    @RESTController.Resource('GET')
+    def exists(self, vol_name: str, group_name=''):
+        params = {'vol_name': vol_name}
+        if group_name:
+            params['group_name'] = group_name
+        error_code, out, err = mgr.remote(
+            'volumes', '_cmd_fs_subvolume_exist', None, params)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to check if subvolume exists: {err}'
+            )
+        if out == 'no subvolume exists':
+            return False
+        return True
+
 
 @APIRouter('/cephfs/subvolume/group', Scope.CEPHFS)
 @APIDoc("Cephfs Subvolume Group Management API", "CephfsSubvolumeGroup")
 class CephFSSubvolumeGroups(RESTController):
 
-    def get(self, vol_name):
+    def get(self, vol_name, info=True):
         if not vol_name:
             raise DashboardException(
                 f'Error listing subvolume groups for {vol_name}')
@@ -767,15 +784,17 @@ def get(self, vol_name):
             raise DashboardException(
                 f'Error listing subvolume groups for {vol_name}')
         subvolume_groups = json.loads(out)
-        for group in subvolume_groups:
-            error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolumegroup_info',
-                                              None, {'vol_name': vol_name,
-                                                     'group_name': group['name']})
-            if error_code != 0:
-                raise DashboardException(
-                    f'Failed to get info for subvolume group {group["name"]}: {err}'
-                )
-            group['info'] = json.loads(out)
+
+        if info:
+            for group in subvolume_groups:
+                error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolumegroup_info',
+                                                  None, {'vol_name': vol_name,
+                                                         'group_name': group['name']})
+                if error_code != 0:
+                    raise DashboardException(
+                        f'Failed to get info for subvolume group {group["name"]}: {err}'
+                    )
+                group['info'] = json.loads(out)
         return subvolume_groups
 
     @RESTController.Resource('GET')
@@ -816,3 +835,31 @@ def delete(self, vol_name: str, group_name: str):
                 f'Failed to delete subvolume group {group_name}: {err}'
             )
         return f'Subvolume group {group_name} removed successfully'
+
+
+@APIRouter('/cephfs/subvolume/snapshot', Scope.CEPHFS)
+@APIDoc("Cephfs Subvolume Snapshot Management API", "CephfsSubvolumeSnapshot")
+class CephFSSubvolumeSnapshots(RESTController):
+    def get(self, vol_name: str, subvol_name, group_name: str = '', info=True):
+        params = {'vol_name': vol_name, 'sub_name': subvol_name}
+        if group_name:
+            params['group_name'] = group_name
+        error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_ls', None,
+                                          params)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to list subvolume snapshots for subvolume {subvol_name}: {err}'
+            )
+        snapshots = json.loads(out)
+
+        if info:
+            for snapshot in snapshots:
+                params['snap_name'] = snapshot['name']
+                error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_info',
+                                                  None, params)
+                if error_code != 0:
+                    raise DashboardException(
+                        f'Failed to get info for subvolume snapshot {snapshot["name"]}: {err}'
+                    )
+                snapshot['info'] = json.loads(out)
+        return snapshots
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
index 3807ae61b67c..0e8768c85772 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
@@ -9,7 +9,6 @@ import { CdTableAction } from '~/app/shared/models/cd-table-action';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
 import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
 import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
-import { CephfsSubvolumeGroup } from '~/app/shared/models/cephfs-subvolumegroup.model';
 import { CephfsSubvolumegroupFormComponent } from '../cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component';
 import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
@@ -18,6 +17,7 @@ import { Permissions } from '~/app/shared/models/permissions';
 import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
 import { FinishedTask } from '~/app/shared/models/finished-task';
 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+import { CephfsSubvolumeGroup } from '~/app/shared/models/cephfs-subvolume-group.model';
 
 @Component({
   selector: 'cd-cephfs-subvolume-group',
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.html
index 29731bbbd1b0..f840c8dab116 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.html
@@ -1,21 +1,10 @@
 <div class="row">
-  <div class="col-sm-1">
-    <h3 i18n>Groups</h3>
-    <ng-container *ngIf="subVolumeGroups$ | async as subVolumeGroups">
-      <ul class="nav flex-column nav-pills">
-        <li class="nav-item">
-          <a class="nav-link"
-             [class.active]="!activeGroupName"
-             (click)="selectSubVolumeGroup()">Default</a>
-        </li>
-        <li class="nav-item"
-            *ngFor="let subVolumeGroup of subVolumeGroups">
-          <a class="nav-link text-decoration-none text-break"
-             [class.active]="subVolumeGroup.name === activeGroupName"
-             (click)="selectSubVolumeGroup(subVolumeGroup.name)">{{subVolumeGroup.name}}</a>
-        </li>
-      </ul>
-    </ng-container>
+  <div class="col-sm-1"
+       *ngIf="subVolumeGroups$ | async as subVolumeGroups">
+    <cd-vertical-navigation title="Groups"
+                            [items]="subvolumeGroupList"
+                            inputIdentifier="group-filter"
+                            (emitActiveItem)="selectSubVolumeGroup($event)"></cd-vertical-navigation>
   </div>
   <div class="col-11 vertical-line">
     <cd-table [data]="subVolumes$ | async"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
index 3f679d27b963..2608dfbb3db6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
@@ -1,6 +1,6 @@
 import { Component, Input, OnChanges, OnInit, TemplateRef, ViewChild } from '@angular/core';
 import { Observable, ReplaySubject, of } from 'rxjs';
-import { catchError, shareReplay, switchMap } from 'rxjs/operators';
+import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
 import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
 import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
@@ -22,7 +22,7 @@ import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
 import { CdForm } from '~/app/shared/forms/cd-form';
 import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
 import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
-import { CephfsSubvolumeGroup } from '~/app/shared/models/cephfs-subvolumegroup.model';
+import { CephfsSubvolumeGroup } from '~/app/shared/models/cephfs-subvolume-group.model';
 
 @Component({
   selector: 'cd-cephfs-subvolume-list',
@@ -67,10 +67,12 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
   subject = new ReplaySubject<CephfsSubvolume[]>();
   groupsSubject = new ReplaySubject<CephfsSubvolume[]>();
 
+  subvolumeGroupList: string[] = [];
+
   activeGroupName: string = '';
 
   constructor(
-    private cephfsSubVolume: CephfsSubvolumeService,
+    private cephfsSubVolumeService: CephfsSubvolumeService,
     private actionLabels: ActionLabelsI18n,
     private modalService: ModalService,
     private authStorageService: AuthStorageService,
@@ -150,7 +152,11 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
 
     this.subVolumeGroups$ = this.groupsSubject.pipe(
       switchMap(() =>
-        this.cephfsSubvolumeGroupService.get(this.fsName).pipe(
+        this.cephfsSubvolumeGroupService.get(this.fsName, false).pipe(
+          tap((groups) => {
+            this.subvolumeGroupList = groups.map((group) => group.name);
+            this.subvolumeGroupList.unshift('');
+          }),
           catchError(() => {
             this.context.error();
             return of(null);
@@ -203,7 +209,7 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
         this.taskWrapper
           .wrapTaskAroundCall({
             task: new FinishedTask('cephfs/subvolume/remove', { subVolumeName: this.selectedName }),
-            call: this.cephfsSubVolume.remove(
+            call: this.cephfsSubVolumeService.remove(
               this.fsName,
               this.selectedName,
               this.activeGroupName,
@@ -228,7 +234,7 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
   getSubVolumes(subVolumeGroupName = '') {
     this.subVolumes$ = this.subject.pipe(
       switchMap(() =>
-        this.cephfsSubVolume.get(this.fsName, subVolumeGroupName).pipe(
+        this.cephfsSubVolumeService.get(this.fsName, subVolumeGroupName).pipe(
           catchError(() => {
             this.context.error();
             return of(null);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
new file mode 100644
index 000000000000..de3117236524
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
@@ -0,0 +1,36 @@
+<ng-container *ngIf="isLoading">
+  <cd-loading-panel>
+    <span i18n>Loading snapshots...</span>
+  </cd-loading-panel>
+</ng-container>
+
+<div class="row"
+     *ngIf="isSubVolumesAvailable; else noGroupsTpl">
+  <div class="col-sm-2">
+    <cd-vertical-navigation title="Groups"
+                            [items]="subvolumeGroupList"
+                            inputIdentifier="group-filter"
+                            (emitActiveItem)="selectSubVolumeGroup($event)"></cd-vertical-navigation>
+  </div>
+  <div class="col-sm-2 vertical-line"
+       *ngIf="subVolumes$ | async">
+    <cd-vertical-navigation title="Subvolumes"
+                            [items]="subVolumesList"
+                            (emitActiveItem)="selectSubVolume($event)"
+                            inputIdentifier="subvol-filter"></cd-vertical-navigation>
+  </div>
+  <div class="col-8 vertical-line"
+       *ngIf="isSubVolumesAvailable">
+    <cd-table [data]="snapshots$ | async"
+              columnMode="flex"
+              [columns]="columns"
+              selectionType="single"
+              [hasDetails]="false"
+              (fetchData)="fetchData()"></cd-table>
+  </div>
+</div>
+<ng-template #noGroupsTpl>
+  <cd-alert-panel type="info"
+                  i18n
+                  *ngIf="!isLoading">No subvolumes are present. Please create subvolumes to manage snapshots.</cd-alert-panel>
+</ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
new file mode 100644
index 000000000000..1d03cf2a8bca
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
@@ -0,0 +1,38 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapshots-list.component';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { SharedModule } from '~/app/shared/shared.module';
+
+describe('CephfsSubvolumeSnapshotsListComponent', () => {
+  let component: CephfsSubvolumeSnapshotsListComponent;
+  let fixture: ComponentFixture<CephfsSubvolumeSnapshotsListComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [CephfsSubvolumeSnapshotsListComponent],
+      imports: [HttpClientTestingModule, SharedModule]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(CephfsSubvolumeSnapshotsListComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+
+  it('should show loading when the items are loading', () => {
+    component.isLoading = true;
+    fixture.detectChanges();
+    expect(fixture.nativeElement.querySelector('cd-loading-panel')).toBeTruthy();
+  });
+
+  it('should show the alert panel when there are no subvolumes', () => {
+    component.isLoading = false;
+    component.subvolumeGroupList = [];
+    fixture.detectChanges();
+    expect(fixture.nativeElement.querySelector('cd-alert-panel')).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
new file mode 100644
index 000000000000..ef5c1050513b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
@@ -0,0 +1,148 @@
+import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core';
+import { Observable, ReplaySubject, forkJoin, of } from 'rxjs';
+import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
+import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
+import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
+import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+import { CdTableColumn } from '~/app/shared/models/cd-table-column';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
+import { CephfsSubvolume, SubvolumeSnapshot } from '~/app/shared/models/cephfs-subvolume.model';
+
+@Component({
+  selector: 'cd-cephfs-subvolume-snapshots-list',
+  templateUrl: './cephfs-subvolume-snapshots-list.component.html',
+  styleUrls: ['./cephfs-subvolume-snapshots-list.component.scss']
+})
+export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges {
+  @Input() fsName: string;
+
+  context: CdTableFetchDataContext;
+  columns: CdTableColumn[] = [];
+
+  subVolumes$: Observable<CephfsSubvolume[]>;
+  snapshots$: Observable<any[]>;
+  snapshotSubject = new ReplaySubject<SubvolumeSnapshot[]>();
+  subVolumeSubject = new ReplaySubject<CephfsSubvolume[]>();
+
+  subvolumeGroupList: string[] = [];
+  subVolumesList: string[];
+
+  activeGroupName = '';
+  activeSubVolumeName = '';
+
+  isSubVolumesAvailable = false;
+  isLoading = true;
+
+  observables: any = [];
+
+  constructor(
+    private cephfsSubvolumeGroupService: CephfsSubvolumeGroupService,
+    private cephfsSubvolumeService: CephfsSubvolumeService
+  ) {}
+
+  ngOnInit(): void {
+    this.columns = [
+      {
+        name: $localize`Name`,
+        prop: 'name',
+        flexGrow: 1
+      },
+      {
+        name: $localize`Created`,
+        prop: 'info.created_at',
+        flexGrow: 1,
+        cellTransformation: CellTemplate.timeAgo
+      },
+      {
+        name: $localize`Pending Clones`,
+        prop: 'info.has_pending_clones',
+        flexGrow: 0.5,
+        cellTransformation: CellTemplate.badge,
+        customTemplateConfig: {
+          map: {
+            no: { class: 'badge-success' },
+            yes: { class: 'badge-info' }
+          }
+        }
+      }
+    ];
+
+    this.cephfsSubvolumeGroupService
+      .get(this.fsName)
+      .pipe(
+        switchMap((groups) => {
+          // manually adding the group 'default' to the list.
+          groups.unshift({ name: '' });
+
+          const observables = groups.map((group) =>
+            this.cephfsSubvolumeService.existsInFs(this.fsName, group.name).pipe(
+              switchMap((resp) => {
+                if (resp) {
+                  this.subvolumeGroupList.push(group.name);
+                }
+                return of(resp); // Emit the response
+              })
+            )
+          );
+
+          return forkJoin(observables);
+        })
+      )
+      .subscribe(() => {
+        if (this.subvolumeGroupList.length) {
+          this.isSubVolumesAvailable = true;
+        }
+        this.isLoading = false;
+      });
+  }
+
+  ngOnChanges(changes: SimpleChanges): void {
+    if (changes.fsName) {
+      this.subVolumeSubject.next();
+    }
+  }
+
+  selectSubVolumeGroup(subVolumeGroupName: string) {
+    this.activeGroupName = subVolumeGroupName;
+    this.getSubVolumes();
+  }
+
+  selectSubVolume(subVolumeName: string) {
+    this.activeSubVolumeName = subVolumeName;
+    this.getSubVolumesSnapshot();
+  }
+
+  getSubVolumes() {
+    this.subVolumes$ = this.subVolumeSubject.pipe(
+      switchMap(() =>
+        this.cephfsSubvolumeService.get(this.fsName, this.activeGroupName, false).pipe(
+          tap((resp) => {
+            this.subVolumesList = resp.map((subVolume) => subVolume.name);
+            this.activeSubVolumeName = resp[0].name;
+            this.getSubVolumesSnapshot();
+          })
+        )
+      )
+    );
+  }
+
+  getSubVolumesSnapshot() {
+    this.snapshots$ = this.snapshotSubject.pipe(
+      switchMap(() =>
+        this.cephfsSubvolumeService
+          .getSnapshots(this.fsName, this.activeSubVolumeName, this.activeGroupName)
+          .pipe(
+            catchError(() => {
+              this.context.error();
+              return of(null);
+            })
+          )
+      ),
+      shareReplay(1)
+    );
+  }
+
+  fetchData() {
+    this.snapshotSubject.next();
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
index 0ad69ccf50a3..6a50ad2e0786 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
@@ -29,6 +29,14 @@
         </cd-cephfs-subvolume-group>
       </ng-template>
     </ng-container>
+    <ng-container ngbNavItem="snapshots">
+      <a ngbNavLink
+         i18n>Snapshots</a>
+      <ng-template ngbNavContent>
+        <cd-cephfs-subvolume-snapshots-list [fsName]="selection.mdsmap.fs_name">
+        </cd-cephfs-subvolume-snapshots-list>
+      </ng-template>
+    </ng-container>
     <ng-container ngbNavItem="clients">
       <a ngbNavLink>
         <ng-container i18n>Clients</ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index a83e0f16870f..cbdb2840867a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -19,6 +19,7 @@ import { CephfsSubvolumeListComponent } from './cephfs-subvolume-list/cephfs-sub
 import { CephfsSubvolumeFormComponent } from './cephfs-subvolume-form/cephfs-subvolume-form.component';
 import { CephfsSubvolumeGroupComponent } from './cephfs-subvolume-group/cephfs-subvolume-group.component';
 import { CephfsSubvolumegroupFormComponent } from './cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component';
+import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component';
 
 @NgModule({
   imports: [
@@ -45,7 +46,8 @@ import { CephfsSubvolumegroupFormComponent } from './cephfs-subvolumegroup-form/
     CephfsSubvolumeFormComponent,
     CephfsDirectoriesComponent,
     CephfsSubvolumeGroupComponent,
-    CephfsSubvolumegroupFormComponent
+    CephfsSubvolumegroupFormComponent,
+    CephfsSubvolumeSnapshotsListComponent
   ]
 })
 export class CephfsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume-group.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume-group.service.ts
index db7fcfacd597..49d001f04f09 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume-group.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume-group.service.ts
@@ -1,9 +1,9 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { Observable, of } from 'rxjs';
-import { CephfsSubvolumeGroup } from '../models/cephfs-subvolumegroup.model';
 import _ from 'lodash';
 import { mapTo, catchError } from 'rxjs/operators';
+import { CephfsSubvolumeGroup } from '../models/cephfs-subvolume-group.model';
 
 @Injectable({
   providedIn: 'root'
@@ -13,8 +13,12 @@ export class CephfsSubvolumeGroupService {
 
   constructor(private http: HttpClient) {}
 
-  get(volName: string): Observable<CephfsSubvolumeGroup[]> {
-    return this.http.get<CephfsSubvolumeGroup[]>(`${this.baseURL}/${volName}`);
+  get(volName: string, info = true): Observable<CephfsSubvolumeGroup[]> {
+    return this.http.get<CephfsSubvolumeGroup[]>(`${this.baseURL}/${volName}`, {
+      params: {
+        info: info
+      }
+    });
   }
 
   create(
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
index e40e9a52f3f3..2e8448ff1a22 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
@@ -29,7 +29,7 @@ describe('CephfsSubvolumeService', () => {
 
   it('should call get', () => {
     service.get('testFS').subscribe();
-    const req = httpTesting.expectOne('api/cephfs/subvolume/testFS?group_name=');
+    const req = httpTesting.expectOne('api/cephfs/subvolume/testFS?group_name=&info=true');
     expect(req.request.method).toBe('GET');
   });
 
@@ -40,4 +40,12 @@ describe('CephfsSubvolumeService', () => {
     );
     expect(req.request.method).toBe('DELETE');
   });
+
+  it('should call getSnapshots', () => {
+    service.getSnapshots('testFS', 'testSubvol').subscribe();
+    const req = httpTesting.expectOne(
+      'api/cephfs/subvolume/snapshot/testFS/testSubvol?group_name='
+    );
+    expect(req.request.method).toBe('GET');
+  });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
index 4c167725007e..d76523aafd2a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
@@ -1,6 +1,6 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
-import { CephfsSubvolume } from '../models/cephfs-subvolume.model';
+import { CephfsSubvolume, SubvolumeSnapshot } from '../models/cephfs-subvolume.model';
 import { Observable, of } from 'rxjs';
 import { catchError, mapTo } from 'rxjs/operators';
 import _ from 'lodash';
@@ -13,10 +13,11 @@ export class CephfsSubvolumeService {
 
   constructor(private http: HttpClient) {}
 
-  get(fsName: string, subVolumeGroupName: string = ''): Observable<CephfsSubvolume[]> {
+  get(fsName: string, subVolumeGroupName: string = '', info = true): Observable<CephfsSubvolume[]> {
     return this.http.get<CephfsSubvolume[]>(`${this.baseURL}/${fsName}`, {
       params: {
-        group_name: subVolumeGroupName
+        group_name: subVolumeGroupName,
+        info: info
       }
     });
   }
@@ -86,6 +87,14 @@ export class CephfsSubvolumeService {
     );
   }
 
+  existsInFs(fsName: string, groupName = ''): Observable<boolean> {
+    return this.http.get<boolean>(`${this.baseURL}/${fsName}/exists`, {
+      params: {
+        group_name: groupName
+      }
+    });
+  }
+
   update(fsName: string, subVolumeName: string, size: string, subVolumeGroupName: string = '') {
     return this.http.put(`${this.baseURL}/${fsName}`, {
       subvol_name: subVolumeName,
@@ -93,4 +102,19 @@ export class CephfsSubvolumeService {
       group_name: subVolumeGroupName
     });
   }
+
+  getSnapshots(
+    fsName: string,
+    subVolumeName: string,
+    groupName = ''
+  ): Observable<SubvolumeSnapshot[]> {
+    return this.http.get<SubvolumeSnapshot[]>(
+      `${this.baseURL}/snapshot/${fsName}/${subVolumeName}`,
+      {
+        params: {
+          group_name: groupName
+        }
+      }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index 327d208ef386..1f31374c18bf 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -52,6 +52,7 @@ import { WizardComponent } from './wizard/wizard.component';
 import { CardComponent } from './card/card.component';
 import { CardRowComponent } from './card-row/card-row.component';
 import { CodeBlockComponent } from './code-block/code-block.component';
+import { VerticalNavigationComponent } from './vertical-navigation/vertical-navigation.component';
 
 @NgModule({
   imports: [
@@ -107,7 +108,8 @@ import { CodeBlockComponent } from './code-block/code-block.component';
     ColorClassFromTextPipe,
     CardComponent,
     CardRowComponent,
-    CodeBlockComponent
+    CodeBlockComponent,
+    VerticalNavigationComponent
   ],
   providers: [],
   exports: [
@@ -140,7 +142,8 @@ import { CodeBlockComponent } from './code-block/code-block.component';
     CdLabelComponent,
     CardComponent,
     CardRowComponent,
-    CodeBlockComponent
+    CodeBlockComponent,
+    VerticalNavigationComponent
   ]
 })
 export class ComponentsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
new file mode 100644
index 000000000000..19628f0d1e1e
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
@@ -0,0 +1,24 @@
+<ng-container *ngIf="items.length">
+  <h3 i18n
+      *ngIf="title">{{title}}</h3>
+  <input type="text"
+         placeholder="Filter by name..."
+         (keyup)="updateFilter()"
+         [id]="inputIdentifier"
+         class="form-control text-center mb-2">
+  <div class="overflow-auto">
+    <ul class="nav flex-column nav-pills">
+      <li class="nav-item"
+          *ngFor="let item of filteredItems; trackBy: trackByFn">
+        <a class="nav-link"
+           [class.active]="!activeItem"
+           (click)="selectItem()"
+           *ngIf="item === ''">Default</a>
+        <a class="nav-link text-decoration-none text-break"
+           [class.active]="item === activeItem"
+           (click)="selectItem(item)"
+           *ngIf="item !== ''">{{item}}</a>
+      </li>
+    </ul>
+  </div>
+</ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.scss
new file mode 100644
index 000000000000..569e2d68708a
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.scss
@@ -0,0 +1,3 @@
+.overflow-auto {
+  max-height: 50vh;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.spec.ts
new file mode 100644
index 000000000000..0d45b339a202
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.spec.ts
@@ -0,0 +1,60 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { VerticalNavigationComponent } from './vertical-navigation.component';
+import { By } from '@angular/platform-browser';
+
+describe('VerticalNavigationComponent', () => {
+  let component: VerticalNavigationComponent;
+  let fixture: ComponentFixture<VerticalNavigationComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [VerticalNavigationComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(VerticalNavigationComponent);
+    component = fixture.componentInstance;
+    component.items = ['item1', 'item2', 'item3'];
+    component.inputIdentifier = 'filter';
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+
+  it('should have a title', () => {
+    component.title = 'testTitle';
+    fixture.detectChanges();
+    const title = fixture.debugElement.query(By.css('h3'));
+    expect(title.nativeElement.textContent).toEqual('testTitle');
+  });
+
+  it('should select the first item as active if no item is selected', () => {
+    expect(component.activeItem).toEqual('item1');
+  });
+
+  it('should filter the items by the keyword in filter input', () => {
+    const event = new KeyboardEvent('keyup');
+    const filterInput = fixture.debugElement.query(By.css('#filter'));
+    filterInput.nativeElement.value = 'item1';
+    filterInput.nativeElement.dispatchEvent(event);
+    fixture.detectChanges();
+    expect(component.filteredItems).toEqual(['item1']);
+
+    filterInput.nativeElement.value = 'item2';
+    filterInput.nativeElement.dispatchEvent(event);
+    fixture.detectChanges();
+    expect(component.filteredItems).toEqual(['item2']);
+  });
+
+  it('should select the item when clicked', () => {
+    component.activeItem = '';
+
+    // click on the first item in the nav list
+    const item = fixture.debugElement.query(By.css('.nav-link'));
+    item.nativeElement.click();
+    fixture.detectChanges();
+    expect(component.activeItem).toEqual('item1');
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.ts
new file mode 100644
index 000000000000..a46cc4f6c433
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.ts
@@ -0,0 +1,37 @@
+import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
+
+@Component({
+  selector: 'cd-vertical-navigation',
+  templateUrl: './vertical-navigation.component.html',
+  styleUrls: ['./vertical-navigation.component.scss']
+})
+export class VerticalNavigationComponent implements OnInit {
+  @Input() items: string[];
+  @Input() title: string;
+  @Input() inputIdentifier: string;
+
+  @Output() emitFilteredItems: EventEmitter<string[]> = new EventEmitter();
+  @Output() emitActiveItem: EventEmitter<string> = new EventEmitter();
+
+  activeItem = '';
+  filteredItems: string[];
+
+  ngOnInit(): void {
+    this.filteredItems = this.items;
+    if (!this.activeItem && this.items.length) this.selectItem(this.items[0]);
+  }
+
+  updateFilter() {
+    const filterInput = document.getElementById(this.inputIdentifier) as HTMLInputElement;
+    this.filteredItems = this.items.filter((item) => item.includes(filterInput.value));
+  }
+
+  selectItem(item = '') {
+    this.activeItem = item;
+    this.emitActiveItem.emit(item);
+  }
+
+  trackByFn(item: number) {
+    return item;
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume-group.model.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume-group.model.ts
index fc087ab53d00..246e4543eb9d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume-group.model.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume-group.model.ts
@@ -1,6 +1,6 @@
 export interface CephfsSubvolumeGroup {
   name: string;
-  info: CephfsSubvolumeGroupInfo;
+  info?: CephfsSubvolumeGroupInfo;
 }
 
 export interface CephfsSubvolumeGroupInfo {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume.model.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume.model.ts
index 41858be61304..25a2a5acc7f4 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume.model.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolume.model.ts
@@ -16,3 +16,13 @@ export interface CephfsSubvolumeInfo {
   gid: number;
   pool_namespace: string;
 }
+
+export interface SubvolumeSnapshot {
+  name: string;
+  info: SubvolumeSnapshotInfo;
+}
+
+export interface SubvolumeSnapshotInfo {
+  created_at: string;
+  has_pending_clones: string;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolumegroup.model.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolumegroup.model.ts
deleted file mode 100644
index fc087ab53d00..000000000000
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cephfs-subvolumegroup.model.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-export interface CephfsSubvolumeGroup {
-  name: string;
-  info: CephfsSubvolumeGroupInfo;
-}
-
-export interface CephfsSubvolumeGroupInfo {
-  mode: number;
-  bytes_pcent: number;
-  bytes_quota: number;
-  data_pool: string;
-  state: string;
-  created_at: string;
-}
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 3081d60b16f8..758ed13f9c5d 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1882,6 +1882,11 @@ paths:
         required: true
         schema:
           type: string
+      - default: true
+        in: query
+        name: info
+        schema:
+          type: boolean
       responses:
         '200':
           content:
@@ -1977,6 +1982,48 @@ paths:
       - jwt: []
       tags:
       - CephfsSubvolumeGroup
+  /api/cephfs/subvolume/snapshot/{vol_name}/{subvol_name}:
+    get:
+      parameters:
+      - in: path
+        name: vol_name
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: subvol_name
+        required: true
+        schema:
+          type: string
+      - default: ''
+        in: query
+        name: group_name
+        schema:
+          type: string
+      - default: true
+        in: query
+        name: info
+        schema:
+          type: boolean
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephfsSubvolumeSnapshot
   /api/cephfs/subvolume/{vol_name}:
     delete:
       parameters:
@@ -2036,6 +2083,11 @@ paths:
         name: group_name
         schema:
           type: string
+      - default: true
+        in: query
+        name: info
+        schema:
+          type: boolean
       responses:
         '200':
           content:
@@ -2102,6 +2154,38 @@ paths:
       - jwt: []
       tags:
       - CephFSSubvolume
+  /api/cephfs/subvolume/{vol_name}/exists:
+    get:
+      parameters:
+      - in: path
+        name: vol_name
+        required: true
+        schema:
+          type: string
+      - default: ''
+        in: query
+        name: group_name
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSubvolume
   /api/cephfs/subvolume/{vol_name}/info:
     get:
       parameters:
@@ -12781,6 +12865,8 @@ tags:
   name: Cephfs
 - description: Cephfs Subvolume Group Management API
   name: CephfsSubvolumeGroup
+- description: Cephfs Subvolume Snapshot Management API
+  name: CephfsSubvolumeSnapshot
 - description: Get Cluster Details
   name: Cluster
 - description: Manage Cluster Configurations

From 4689c3d65eb758cecf944da0fe2c5686fa2918ee Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 27 Oct 2023 13:45:44 +0530
Subject: [PATCH 0850/2492] mgr/dashboard: fs snapshots e2e

Fixes: https://tracker.ceph.com/issues/63237
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../e2e/common/table-helper.feature.po.ts     | 13 +++++
 .../filesystems/snapshots.e2e-spec.feature    | 53 +++++++++++++++++++
 .../subvolume-groups.e2e-spec.feature         |  4 +-
 .../cephfs-subvolume-group.component.ts       | 14 ++---
 .../cephfs-subvolume-list.component.ts        | 40 ++++++++------
 ...phfs-subvolume-snapshots-list.component.ts | 12 ++---
 .../vertical-navigation.component.html        |  2 +-
 7 files changed, 107 insertions(+), 31 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/snapshots.e2e-spec.feature

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/common/table-helper.feature.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/common/table-helper.feature.po.ts
index 82a2c7c35cde..330950acd447 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/common/table-helper.feature.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/common/table-helper.feature.po.ts
@@ -54,6 +54,13 @@ Then('I should not see a row with {string}', (row: string) => {
   );
 });
 
+Then('I should see a table in the expanded row', () => {
+  cy.get('.datatable-row-detail').within(() => {
+    cy.get('cd-table').should('exist');
+    cy.get('datatable-scroller, .empty-row');
+  });
+});
+
 Then('I should not see a row with {string} in the expanded row', (row: string) => {
   cy.get('.datatable-row-detail').within(() => {
     cy.get('cd-table .search input').first().clear().type(row);
@@ -133,3 +140,9 @@ And('I should see row {string} have {string} on this tab', (row: string, options
     });
   }
 });
+
+Then('I should see an alert {string} in the expanded row', (alert: string) => {
+  cy.get('.datatable-row-detail').within(() => {
+    cy.get('.alert-panel-text').contains(alert);
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/snapshots.e2e-spec.feature b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/snapshots.e2e-spec.feature
new file mode 100644
index 000000000000..002282172bb0
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/snapshots.e2e-spec.feature
@@ -0,0 +1,53 @@
+Feature: CephFS Snapshot Management
+
+    Goal: To test out the CephFS snapshot management features
+
+    Background: Login
+        Given I am logged in
+
+    Scenario: Create a CephFS Volume
+        Given I am on the "cephfs" page
+        And I click on "Create" button
+        And enter "name" "test_cephfs"
+        And I click on "Create File System" button
+        Then I should see a row with "test_cephfs"
+
+    Scenario: Snapshots tab without a subvolume
+        Given I am on the "cephfs" page
+        When I expand the row "test_cephfs"
+        And I go to the "Snapshots" tab
+        Then I should see an alert "No subvolumes are present" in the expanded row
+
+    Scenario: Create a CephFS Subvolume
+        Given I am on the "cephfs" page
+        When I expand the row "test_cephfs"
+        And I go to the "Subvolumes" tab
+        And I click on "Create" button from the expanded row
+        And enter "subvolumeName" "test_subvolume" in the modal
+        And I click on "Create Subvolume" button
+        Then I should see a row with "test_subvolume" in the expanded row
+
+    Scenario: Show the CephFS Snapshots view
+        Given I am on the "cephfs" page
+        When I expand the row "test_cephfs"
+        And I go to the "Snapshots" tab
+        Then I should see a table in the expanded row
+
+    Scenario: Remove a CephFS Subvolume
+        Given I am on the "cephfs" page
+        When I expand the row "test_cephfs"
+        And I go to the "Subvolumes" tab
+        When I select a row "test_subvolume" in the expanded row
+        And I click on "Remove" button from the table actions in the expanded row
+        And I check the tick box in modal
+        And I click on "Remove Subvolume" button
+        Then I should not see a row with "test_subvolume" in the expanded row
+
+    Scenario: Remove CephFS Volume
+        Given I am on the "cephfs" page
+        And I select a row "test_cephfs"
+        And I click on "Remove" button from the table actions
+        Then I should see the modal
+        And I check the tick box in modal
+        And I click on "Remove File System" button
+        Then I should not see a row with "test_cephfs"
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/subvolume-groups.e2e-spec.feature b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/subvolume-groups.e2e-spec.feature
index 66e3f726a661..e53df64771d5 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/subvolume-groups.e2e-spec.feature
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/filesystems/subvolume-groups.e2e-spec.feature
@@ -21,7 +21,7 @@ Feature: CephFS Subvolume Group management
         And I click on "Create Subvolume group" button
         Then I should see a row with "test_subvolume_group" in the expanded row
 
-    Scenario: Edit a CephFS Subvolume
+    Scenario: Edit a CephFS Subvolume Group
         Given I am on the "cephfs" page
         When I expand the row "test_cephfs"
         And I go to the "Subvolume groups" tab
@@ -31,7 +31,7 @@ Feature: CephFS Subvolume Group management
         And I click on "Edit Subvolume group" button
         Then I should see row "test_subvolume_group" of the expanded row to have a usage bar
 
-    Scenario: Remove a CephFS Subvolume
+    Scenario: Remove a CephFS Subvolume Group
         Given I am on the "cephfs" page
         When I expand the row "test_cephfs"
         And I go to the "Subvolume groups" tab
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
index 0e8768c85772..a91daf8cb930 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-group/cephfs-subvolume-group.component.ts
@@ -1,5 +1,5 @@
-import { Component, Input, OnChanges, OnInit, ViewChild } from '@angular/core';
-import { Observable, ReplaySubject, of } from 'rxjs';
+import { Component, Input, OnChanges, OnInit, SimpleChanges, ViewChild } from '@angular/core';
+import { BehaviorSubject, Observable, of } from 'rxjs';
 import { catchError, shareReplay, switchMap } from 'rxjs/operators';
 
 import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
@@ -52,7 +52,7 @@ export class CephfsSubvolumeGroupComponent implements OnInit, OnChanges {
   permissions: Permissions;
 
   subvolumeGroup$: Observable<CephfsSubvolumeGroup[]>;
-  subject = new ReplaySubject<CephfsSubvolumeGroup[]>();
+  subject = new BehaviorSubject<CephfsSubvolumeGroup[]>([]);
 
   constructor(
     private cephfsSubvolumeGroup: CephfsSubvolumeGroupService,
@@ -138,11 +138,13 @@ export class CephfsSubvolumeGroupComponent implements OnInit, OnChanges {
   }
 
   fetchData() {
-    this.subject.next();
+    this.subject.next([]);
   }
 
-  ngOnChanges() {
-    this.subject.next();
+  ngOnChanges(changes: SimpleChanges) {
+    if (changes.fsName) {
+      this.subject.next([]);
+    }
   }
 
   updateSelection(selection: CdTableSelection) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
index 2608dfbb3db6..4f9cf27db0ff 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
@@ -1,6 +1,14 @@
-import { Component, Input, OnChanges, OnInit, TemplateRef, ViewChild } from '@angular/core';
-import { Observable, ReplaySubject, of } from 'rxjs';
-import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
+import {
+  Component,
+  Input,
+  OnChanges,
+  OnInit,
+  SimpleChanges,
+  TemplateRef,
+  ViewChild
+} from '@angular/core';
+import { BehaviorSubject, Observable, of } from 'rxjs';
+import { catchError, switchMap, tap } from 'rxjs/operators';
 import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
 import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
@@ -64,10 +72,11 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
 
   subVolumes$: Observable<CephfsSubvolume[]>;
   subVolumeGroups$: Observable<CephfsSubvolumeGroup[]>;
-  subject = new ReplaySubject<CephfsSubvolume[]>();
-  groupsSubject = new ReplaySubject<CephfsSubvolume[]>();
+  subject = new BehaviorSubject<CephfsSubvolume[]>([]);
+  groupsSubject = new BehaviorSubject<CephfsSubvolume[]>([]);
 
   subvolumeGroupList: string[] = [];
+  subVolumesList: CephfsSubvolume[] = [];
 
   activeGroupName: string = '';
 
@@ -148,8 +157,6 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
       }
     ];
 
-    this.getSubVolumes();
-
     this.subVolumeGroups$ = this.groupsSubject.pipe(
       switchMap(() =>
         this.cephfsSubvolumeGroupService.get(this.fsName, false).pipe(
@@ -167,12 +174,14 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
   }
 
   fetchData() {
-    this.subject.next();
+    this.subject.next([]);
   }
 
-  ngOnChanges() {
-    this.subject.next();
-    this.groupsSubject.next();
+  ngOnChanges(changes: SimpleChanges) {
+    if (changes.fsName) {
+      this.subject.next([]);
+      this.groupsSubject.next([]);
+    }
   }
 
   updateSelection(selection: CdTableSelection) {
@@ -228,20 +237,19 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
 
   selectSubVolumeGroup(subVolumeGroupName: string) {
     this.activeGroupName = subVolumeGroupName;
-    this.getSubVolumes(subVolumeGroupName);
+    this.getSubVolumes();
   }
 
-  getSubVolumes(subVolumeGroupName = '') {
+  getSubVolumes() {
     this.subVolumes$ = this.subject.pipe(
       switchMap(() =>
-        this.cephfsSubVolumeService.get(this.fsName, subVolumeGroupName).pipe(
+        this.cephfsSubVolumeService.get(this.fsName, this.activeGroupName).pipe(
           catchError(() => {
             this.context.error();
             return of(null);
           })
         )
-      ),
-      shareReplay(1)
+      )
     );
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
index ef5c1050513b..251314c3e86a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
@@ -1,5 +1,5 @@
 import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core';
-import { Observable, ReplaySubject, forkJoin, of } from 'rxjs';
+import { BehaviorSubject, Observable, forkJoin, of } from 'rxjs';
 import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
 import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
 import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
@@ -21,8 +21,8 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
 
   subVolumes$: Observable<CephfsSubvolume[]>;
   snapshots$: Observable<any[]>;
-  snapshotSubject = new ReplaySubject<SubvolumeSnapshot[]>();
-  subVolumeSubject = new ReplaySubject<CephfsSubvolume[]>();
+  snapshotSubject = new BehaviorSubject<SubvolumeSnapshot[]>([]);
+  subVolumeSubject = new BehaviorSubject<CephfsSubvolume[]>([]);
 
   subvolumeGroupList: string[] = [];
   subVolumesList: string[];
@@ -71,7 +71,7 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
       .get(this.fsName)
       .pipe(
         switchMap((groups) => {
-          // manually adding the group 'default' to the list.
+          // manually adding the group '_nogroup' to the list.
           groups.unshift({ name: '' });
 
           const observables = groups.map((group) =>
@@ -98,7 +98,7 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
 
   ngOnChanges(changes: SimpleChanges): void {
     if (changes.fsName) {
-      this.subVolumeSubject.next();
+      this.subVolumeSubject.next([]);
     }
   }
 
@@ -143,6 +143,6 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
   }
 
   fetchData() {
-    this.snapshotSubject.next();
+    this.snapshotSubject.next([]);
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
index 19628f0d1e1e..ae48d7fd7fcc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/vertical-navigation/vertical-navigation.component.html
@@ -13,7 +13,7 @@
         <a class="nav-link"
            [class.active]="!activeItem"
            (click)="selectItem()"
-           *ngIf="item === ''">Default</a>
+           *ngIf="item === ''">_nogroup</a>
         <a class="nav-link text-decoration-none text-break"
            [class.active]="item === activeItem"
            (click)="selectItem(item)"

From 8e92cbac3e4c1d7643b5e02735a1dabb3326658d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 13 Nov 2023 20:05:47 -0500
Subject: [PATCH 0851/2492] common: add versioned encodings for std::variant

adds two encoding strategies for `std::variant<>` under the namespaces
`ceph::versioned_variant` and `ceph::converted_variant`

these versioned encodings allow the variant to be extended with new
types, provided that they're always added to the end without changing
or removing existing types. because of this requirement, no default
encoding is provided for `std::variant`. callers must opt in to one
namespace or the other

the `converted_variant` encoding requires the variant's first type T
to use versioned encoding, and guarantees that the variant's encoding
is backward-compatible with T's

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/versioned_variant.h            | 215 ++++++++++++++
 src/test/common/CMakeLists.txt            |   4 +
 src/test/common/test_versioned_variant.cc | 323 ++++++++++++++++++++++
 3 files changed, 542 insertions(+)
 create mode 100644 src/common/versioned_variant.h
 create mode 100644 src/test/common/test_versioned_variant.cc

diff --git a/src/common/versioned_variant.h b/src/common/versioned_variant.h
new file mode 100644
index 000000000000..9c9c5ada9b01
--- /dev/null
+++ b/src/common/versioned_variant.h
@@ -0,0 +1,215 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <concepts>
+#include <limits>
+#include <variant>
+
+#include <boost/mp11/algorithm.hpp> // for mp_with_index
+#include "include/encoding.h"
+
+/// \file
+/// \brief Contains binary encoding strategies for std::variant.
+
+namespace ceph {
+
+// null encoding for std::monostate
+void encode(const std::monostate&, bufferlist& bl) {}
+void decode(std::monostate&, bufferlist::const_iterator& p) {}
+
+// largest value that can be represented by `__u8 struct_v`
+inline constexpr size_t max_version = std::numeric_limits<__u8>::max();
+
+/// \namespace versioned_variant
+/// \brief A backward-compatible binary encoding for std::variant.
+///
+/// The variant index is encoded in struct_v so the correct decoder can be
+/// selected. This means that existing variant types cannot be changed or
+/// removed without breaking the decode of earlier ceph versions. New types
+/// can only be added to the end of the variant.
+///
+/// In addition to struct_v, the variant index is also encoded in compatv. As
+/// the variant is extended, this means that existing decoders can continue to
+/// decode the types they recognize, but reject the encodings of new types they
+/// don't.
+///
+/// The variant types themselves are free to change their encodings, provided
+/// they manage their own versioning. The types must be default-constructible
+/// so they can be constructed before decode.
+///
+/// The contained encode/decode functions won't be found by argument-dependent
+/// lookup, so you must either qualify the calls with `versioned_variant::` or
+/// add `using namespace versioned_variant` to the calling scope.
+namespace versioned_variant {
+
+// Requirements for the list of types for versioned std::variant encoding.
+template <typename ...Ts>
+concept valid_types = requires {
+    sizeof...(Ts) > 0; // variant cannot be empty
+    sizeof...(Ts) <= max_version; // index must fit in u8
+    requires (std::default_initializable<Ts> && ...); // default-constructible
+  };
+
+/// \brief A versioned_variant encoder.
+///
+/// Example:
+/// \code
+/// struct example {
+///   std::variant<int, bool> value;
+///
+///   void encode(bufferlist& bl) const {
+///     ENCODE_START(0, 0, bl);
+///     ceph::versioned_variant::encode(value, bl);
+///     ...
+/// \endcode
+template <typename ...Ts> requires valid_types<Ts...>
+void encode(const std::variant<Ts...>& v, bufferlist& bl, uint64_t features=0)
+{
+  // encode the variant index in struct_v and compatv
+  const uint8_t ver = static_cast<uint8_t>(v.index());
+  ENCODE_START(ver, ver, bl);
+  // use the variant type's encoder
+  std::visit([&bl] (const auto& value) mutable {
+      encode(value, bl);
+    }, v);
+  ENCODE_FINISH(bl);
+}
+
+/// \brief A versioned_variant decoder.
+///
+/// Example:
+/// \code
+/// struct example {
+///   std::variant<int, bool> value;
+///
+///   void decode(bufferlist::const_iterator& bl) const {
+///     DECODE_START(0, bl);
+///     ceph::versioned_variant::decode(value, bl);
+///     ...
+/// \endcode
+template <typename ...Ts> requires valid_types<Ts...>
+void decode(std::variant<Ts...>& v, bufferlist::const_iterator& p)
+{
+  constexpr uint8_t max_version = sizeof...(Ts) - 1;
+  DECODE_START(max_version, p);
+  // use struct_v as an index into the variant after converting it into a
+  // compile-time index I
+  const uint8_t index = struct_v;
+  boost::mp11::mp_with_index<sizeof...(Ts)>(index, [&v, &p] (auto I) {
+      // default-construct the type at index I and call its decoder
+      decode(v.template emplace<I>(), p);
+    });
+  DECODE_FINISH(p);
+}
+
+} // namespace versioned_variant
+
+
+/// \namespace converted_variant
+/// \brief A std::variant<T, ...> encoding that is backward-compatible with T.
+///
+/// The encoding works the same as versioned_variant, except that a block of
+/// version numbers are reserved for the first type T to allow its encoding
+/// to continue evolving. T must itself use versioned encoding (ie
+/// ENCODE_START/FINISH).
+///
+/// This encoding strategy allows a serialized type T to be transparently
+/// converted into a variant that can represent other types too.
+namespace converted_variant {
+
+// For converted variants, reserve the first 128 versions for the original
+// type. Variant types after the first use the version numbers above this.
+inline constexpr uint8_t converted_max_version = 128;
+
+// Requirements for the list of types for converted std::variant encoding.
+template <typename ...Ts>
+concept valid_types = requires {
+    sizeof...(Ts) > 0; // variant cannot be empty
+    sizeof...(Ts) <= (max_version - converted_max_version); // index must fit in u8
+    requires (std::default_initializable<Ts> && ...); // default-constructible
+  };
+
+/// \brief A converted_variant encoder.
+///
+/// Example:
+/// \code
+/// struct example {
+///   std::variant<int, bool> value; // replaced `int value`
+///
+///   void encode(bufferlist& bl) const {
+///     ENCODE_START(1, 0, bl);
+///     ceph::converted_variant::encode(value, bl);
+///     ...
+/// \endcode
+template <typename ...Ts> requires valid_types<Ts...>
+void encode(const std::variant<Ts...>& v, bufferlist& bl, uint64_t features=0)
+{
+  const uint8_t index = static_cast<uint8_t>(v.index());
+  if (index == 0) {
+    // encode the first type with its own versioning scheme
+    encode(std::get<0>(v), bl);
+    return;
+  }
+
+  // encode the variant index in struct_v and compatv
+  const uint8_t ver = converted_max_version + index;
+  ENCODE_START(ver, ver, bl);
+  // use the variant type's encoder
+  std::visit([&bl] (const auto& value) mutable {
+      encode(value, bl);
+    }, v);
+  ENCODE_FINISH(bl);
+}
+
+/// \brief A converted_variant decoder.
+///
+/// Example:
+/// \code
+/// struct example {
+///   std::variant<int, bool> value; // replaced `int value`
+///
+///   void decode(bufferlist::const_iterator& bl) {
+///     DECODE_START(1, bl);
+///     ceph::converted_variant::decode(value, bl);
+///     ...
+/// \endcode
+template <typename ...Ts> requires valid_types<Ts...>
+void decode(std::variant<Ts...>& v, bufferlist::const_iterator& p)
+{
+  // save the iterator position so the first type can restart decode
+  const bufferlist::const_iterator prev = p;
+
+  constexpr uint8_t max_version = converted_max_version + sizeof...(Ts) - 1;
+  DECODE_START(max_version, p);
+  if (struct_v <= converted_max_version) {
+    p = prev; // rewind and use type 0's DECODE_START/FINISH
+    decode(v.template emplace<0>(), p);
+    return;
+  }
+
+  // use struct_v as an index into the variant after converting it into a
+  // compile-time index I
+  const uint8_t index = struct_v - converted_max_version;
+  boost::mp11::mp_with_index<sizeof...(Ts)>(index, [&v, &p] (auto I) {
+      // default-construct the type at index I and call its decoder
+      decode(v.template emplace<I>(), p);
+    });
+  DECODE_FINISH(p);
+}
+
+} // namespace converted_variant
+
+} // namespace ceph
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index c044daf662ab..b2ed06ee3062 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -390,6 +390,10 @@ target_link_libraries(unittest_blocked_completion Boost::system GTest::GTest)
 add_executable(unittest_allocate_unique test_allocate_unique.cc)
 add_ceph_unittest(unittest_allocate_unique)
 
+add_executable(unittest_versioned_variant test_versioned_variant.cc)
+add_ceph_unittest(unittest_versioned_variant)
+target_link_libraries(unittest_versioned_variant common)
+
 if(WITH_SYSTEMD)
   add_executable(unittest_journald_logger test_journald_logger.cc)
   target_link_libraries(unittest_journald_logger ceph-common)
diff --git a/src/test/common/test_versioned_variant.cc b/src/test/common/test_versioned_variant.cc
new file mode 100644
index 000000000000..b91e24b6f13f
--- /dev/null
+++ b/src/test/common/test_versioned_variant.cc
@@ -0,0 +1,323 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "common/versioned_variant.h"
+#include <string>
+#include <gtest/gtest.h>
+
+namespace {
+
+// type with custom encoding
+struct custom_type {
+  void encode(bufferlist& bl) const {
+    ENCODE_START(0, 0, bl);
+    ENCODE_FINISH(bl);
+  }
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START(0, bl);
+    DECODE_FINISH(bl);
+  }
+};
+WRITE_CLASS_ENCODER(custom_type);
+
+} // anonymous namespace
+
+namespace ceph {
+
+TEST(VersionedVariant, Monostate)
+{
+  using Variant = std::variant<std::monostate>;
+  bufferlist bl;
+  {
+    Variant in;
+    versioned_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    EXPECT_TRUE(std::holds_alternative<std::monostate>(out));
+  }
+}
+
+TEST(VersionedVariant, Custom)
+{
+  using Variant = std::variant<std::monostate, custom_type>;
+  bufferlist bl;
+  {
+    Variant in = custom_type{};
+    versioned_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    EXPECT_TRUE(std::holds_alternative<custom_type>(out));
+  }
+}
+
+TEST(VersionedVariant, DuplicateFirst)
+{
+  using Variant = std::variant<int, int>;
+  bufferlist bl;
+  {
+    Variant in;
+    in.emplace<0>(42);
+    versioned_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_EQ(0, out.index());
+    EXPECT_EQ(42, std::get<0>(out));
+  }
+}
+
+TEST(VersionedVariant, DuplicateSecond)
+{
+  using Variant = std::variant<int, int>;
+  bufferlist bl;
+  {
+    Variant in;
+    in.emplace<1>(42);
+    versioned_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_EQ(1, out.index());
+    EXPECT_EQ(42, std::get<1>(out));
+  }
+}
+
+TEST(VersionedVariant, EncodeOld)
+{
+  using V1 = std::variant<int>;
+  using V2 = std::variant<int, std::string>;
+
+  bufferlist bl;
+  {
+    // use V1 to encode the initial type
+    V1 in = 42;
+    versioned_variant::encode(in, bl);
+  }
+  {
+    // can decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<int>(out));
+    EXPECT_EQ(42, std::get<int>(out));
+  }
+  {
+    // can also decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<int>(out));
+    EXPECT_EQ(42, std::get<int>(out));
+  }
+}
+
+TEST(VersionedVariant, EncodeExisting)
+{
+  using V1 = std::variant<int>;
+  using V2 = std::variant<int, std::string>;
+
+  bufferlist bl;
+  {
+    // use V2 to encode the type shared with V1
+    V2 in = 42;
+    versioned_variant::encode(in, bl);
+  }
+  {
+    // can decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<int>(out));
+    EXPECT_EQ(42, std::get<int>(out));
+  }
+  {
+    // can also decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<int>(out));
+    EXPECT_EQ(42, std::get<int>(out));
+  }
+}
+
+TEST(VersionedVariant, EncodeNew)
+{
+  using V1 = std::variant<int>;
+  using V2 = std::variant<int, std::string>;
+
+  bufferlist bl;
+  {
+    // use V2 to encode the new string type
+    V2 in = "42";
+    versioned_variant::encode(in, bl);
+  }
+  {
+    // can decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(versioned_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<std::string>(out));
+    EXPECT_EQ("42", std::get<std::string>(out));
+  }
+  {
+    // can't decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    EXPECT_THROW(versioned_variant::decode(out, p), buffer::malformed_input);
+  }
+}
+
+
+TEST(ConvertedVariant, Custom)
+{
+  using Variant = std::variant<custom_type>;
+  bufferlist bl;
+  {
+    Variant in = custom_type{};
+    converted_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    EXPECT_TRUE(std::holds_alternative<custom_type>(out));
+  }
+}
+
+TEST(ConvertedVariant, DuplicateFirst)
+{
+  using Variant = std::variant<custom_type, int, int>;
+  bufferlist bl;
+  {
+    Variant in;
+    in.emplace<1>(42);
+    converted_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    ASSERT_EQ(1, out.index());
+    EXPECT_EQ(42, std::get<1>(out));
+  }
+}
+
+TEST(ConvertedVariant, DuplicateSecond)
+{
+  using Variant = std::variant<custom_type, int, int>;
+  bufferlist bl;
+  {
+    Variant in;
+    in.emplace<2>(42);
+    converted_variant::encode(in, bl);
+  }
+  {
+    Variant out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    ASSERT_EQ(2, out.index());
+    EXPECT_EQ(42, std::get<2>(out));
+  }
+}
+
+TEST(ConvertedVariant, EncodeOld)
+{
+  using V1 = custom_type;
+  using V2 = std::variant<custom_type, int>;
+
+  bufferlist bl;
+  {
+    // use V1 to encode the initial type
+    V1 in;
+    encode(in, bl);
+  }
+  {
+    // can decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    EXPECT_NO_THROW(decode(out, p));
+  }
+  {
+    // can also decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    EXPECT_TRUE(std::holds_alternative<custom_type>(out));
+  }
+}
+
+TEST(ConvertedVariant, EncodeExisting)
+{
+  using V1 = custom_type;
+  using V2 = std::variant<custom_type, int>;
+
+  bufferlist bl;
+  {
+    // use V2 to encode the type shared with V1
+    V2 in;
+    converted_variant::encode(in, bl);
+  }
+  {
+    // can decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    EXPECT_TRUE(std::holds_alternative<custom_type>(out));
+  }
+  {
+    // can also decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    EXPECT_NO_THROW(decode(out, p));
+  }
+}
+
+TEST(ConvertedVariant, EncodeNew)
+{
+  using V1 = custom_type;
+  using V2 = std::variant<custom_type, int>;
+
+  bufferlist bl;
+  {
+    // use V2 to encode the new type
+    V2 in = 42;
+    converted_variant::encode(in, bl);
+  }
+  {
+    // can decode as V2
+    V2 out;
+    auto p = bl.cbegin();
+    ASSERT_NO_THROW(converted_variant::decode(out, p));
+    ASSERT_TRUE(std::holds_alternative<int>(out));
+    EXPECT_EQ(42, std::get<int>(out));
+  }
+  {
+    // can't decode as V1
+    V1 out;
+    auto p = bl.cbegin();
+    EXPECT_THROW(decode(out, p), buffer::malformed_input);
+  }
+}
+
+} // namespace ceph

From b7e1c1d59c374f1707d7a7207211a5f8a438ee1d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 14 Nov 2023 14:43:25 -0500
Subject: [PATCH 0852/2492] common: add generate_test_instances() for
 std::variant

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/versioned_variant.h            | 19 +++++++++++++++++++
 src/test/common/test_versioned_variant.cc | 18 ++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/src/common/versioned_variant.h b/src/common/versioned_variant.h
index 9c9c5ada9b01..f7e46b7296e7 100644
--- a/src/common/versioned_variant.h
+++ b/src/common/versioned_variant.h
@@ -16,6 +16,7 @@
 
 #include <concepts>
 #include <limits>
+#include <list>
 #include <variant>
 
 #include <boost/mp11/algorithm.hpp> // for mp_with_index
@@ -212,4 +213,22 @@ void decode(std::variant<Ts...>& v, bufferlist::const_iterator& p)
 
 } // namespace converted_variant
 
+
+/// \brief Generate a list with a default-constructed variant of each type.
+///
+/// This can be used in generate_test_instances() for types that contain
+/// variants to ensure that an encoding of each type is present in the
+/// ceph-object-corpus. This allows the ceph-dencoder tests to catch any
+/// breaking changes to the variant types that are present in encodings.
+template <typename ...Ts>
+void generate_test_instances(std::list<std::variant<Ts...>>& instances)
+{
+  // use an immediately-invoked lambda to get a parameter pack of variant indices
+  [&instances] <std::size_t ...I> (std::index_sequence<I...>) {
+    // use a fold expression to call emplace_back() for each index in the pack
+    // use in_place_index to default-construct a variant of the type at index I
+    (instances.emplace_back(std::in_place_index<I>), ...);
+  } (std::make_index_sequence<sizeof...(Ts)>{});
+}
+
 } // namespace ceph
diff --git a/src/test/common/test_versioned_variant.cc b/src/test/common/test_versioned_variant.cc
index b91e24b6f13f..81f12c23c2b4 100644
--- a/src/test/common/test_versioned_variant.cc
+++ b/src/test/common/test_versioned_variant.cc
@@ -13,6 +13,7 @@
  */
 
 #include "common/versioned_variant.h"
+#include <bitset>
 #include <string>
 #include <gtest/gtest.h>
 
@@ -320,4 +321,21 @@ TEST(ConvertedVariant, EncodeNew)
   }
 }
 
+TEST(Variant, GenerateTestInstances)
+{
+  using Variant = std::variant<int, bool, double>;
+
+  std::bitset<std::variant_size_v<Variant>> bits;
+  ASSERT_TRUE(bits.none());
+
+  std::list<Variant> instances;
+  generate_test_instances(instances);
+
+  for (const auto& v : instances) {
+    bits.set(v.index());
+  }
+
+  EXPECT_TRUE(bits.all());
+}
+
 } // namespace ceph

From 6f1e0e682533a1734e183a71afa6e1953ba83bd7 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Fri, 1 Dec 2023 08:48:49 -0600
Subject: [PATCH 0853/2492] tests/scrub: deactivate osd-scrub-dump stand-alone
 test

as the scrub reservation changes had made it obsolete.
Note - it is not an issue of fixing the test, but rather
that the tested functionality is no longer there.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/scrub/osd-scrub-dump.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/qa/standalone/scrub/osd-scrub-dump.sh b/qa/standalone/scrub/osd-scrub-dump.sh
index 644f82d80716..403ffacd9a65 100755
--- a/qa/standalone/scrub/osd-scrub-dump.sh
+++ b/qa/standalone/scrub/osd-scrub-dump.sh
@@ -15,6 +15,11 @@
 # GNU Library Public License for more details.
 #
 
+
+# 30.11.2023: the test is now disabled, as the reservation mechanism has been
+# thoroughly reworked and the test is no longer valid.  The test is left here
+# as a basis for a new set of primary vs. replicas scrub activation tests.
+
 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
 
 MAX_SCRUBS=4
@@ -22,6 +27,8 @@ SCRUB_SLEEP=3
 POOL_SIZE=3
 
 function run() {
+    echo "This test is disabled"
+    return 0
     local dir=$1
     shift
     local CHUNK_MAX=5

From c53a6e7dd2e7caa6b52a47dbd896a2df7bfc1746 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 1 Dec 2023 13:46:20 -0500
Subject: [PATCH 0854/2492] rgw/sysobj: pool_list_objects_init() initializes
 marker

Fixes: https://tracker.ceph.com/issues/63717

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/services/svc_sys_obj_core.cc      | 2 +-
 src/rgw/services/svc_sys_obj_core_types.h | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc
index fb9c7edce238..397709c5d999 100644
--- a/src/rgw/services/svc_sys_obj_core.cc
+++ b/src/rgw/services/svc_sys_obj_core.cc
@@ -612,7 +612,7 @@ int RGWSI_SysObj_Core::pool_list_objects_init(const DoutPrefixProvider *dpp,
                                               const string& prefix,
                                               RGWSI_SysObj::Pool::ListCtx *_ctx)
 {
-  _ctx->impl.emplace<PoolListImplInfo>(prefix);
+  _ctx->impl.emplace<PoolListImplInfo>(prefix, marker);
 
   auto& ctx = static_cast<PoolListImplInfo&>(*_ctx->impl);
 
diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h
index 404f0fdde686..a7b6aed08429 100644
--- a/src/rgw/services/svc_sys_obj_core_types.h
+++ b/src/rgw/services/svc_sys_obj_core_types.h
@@ -30,6 +30,7 @@ struct RGWSI_SysObj_Core_PoolListImplInfo : public RGWSI_SysObj_Pool_ListInfo {
   rgw::AccessListFilter filter;
   std::string marker;
 
-  RGWSI_SysObj_Core_PoolListImplInfo(const std::string& prefix)
-    : filter(rgw::AccessListFilterPrefix(prefix)) {}
+  RGWSI_SysObj_Core_PoolListImplInfo(const std::string& prefix,
+                                     const std::string& marker)
+    : filter(rgw::AccessListFilterPrefix(prefix)), marker(marker) {}
 };

From 31bffea6948a0b00973b6791673af4cd3d132ec2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 1 Dec 2023 14:27:51 -0500
Subject: [PATCH 0855/2492] rgw/zone: remove check for (very) old pools config

i'm not sure how this was meant to work originally, but now it's
obviously broken - it tries to stat an object from the domain_root pool
before the domain_root pool name gets filled in

https://github.com/ceph/ceph/pull/50359 added a new error message that
makes this clear:

> ERROR: creating ioctx (pool=); r=-22

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_zone.cc | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc
index f41d987ec549..aeb58e2f48fe 100644
--- a/src/rgw/rgw_zone.cc
+++ b/src/rgw/rgw_zone.cc
@@ -33,7 +33,6 @@ std::string zonegroup_names_oid_prefix = "zonegroups_names.";
 std::string RGW_DEFAULT_ZONE_ROOT_POOL = "rgw.root";
 std::string RGW_DEFAULT_ZONEGROUP_ROOT_POOL = "rgw.root";
 std::string RGW_DEFAULT_PERIOD_ROOT_POOL = "rgw.root";
-std::string avail_pools = ".pools.avail";
 std::string default_storage_pool_suffix = "rgw.buckets.data";
 
 }
@@ -412,22 +411,14 @@ int RGWZoneParams::set_as_default(const DoutPrefixProvider *dpp, optional_yield
 
 int RGWZoneParams::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive)
 {
-  /* check for old pools config */
-  rgw_raw_obj obj(domain_root, avail_pools);
-  auto sysobj = sysobj_svc->get_obj(obj);
-  int r = sysobj.rop().stat(y, dpp);
-  if (r < 0) {
-    ldpp_dout(dpp, 10) << "couldn't find old data placement pools config, setting up new ones for the zone" << dendl;
-    /* a new system, let's set new placement info */
-    RGWZonePlacementInfo default_placement;
-    default_placement.index_pool = name + "." + default_bucket_index_pool_suffix;
-    rgw_pool pool = name + "." + default_storage_pool_suffix;
-    default_placement.storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr);
-    default_placement.data_extra_pool = name + "." + default_storage_extra_pool_suffix;
-    placement_pools["default-placement"] = default_placement;
-  }
+  RGWZonePlacementInfo default_placement;
+  default_placement.index_pool = name + "." + default_bucket_index_pool_suffix;
+  rgw_pool pool = name + "." + default_storage_pool_suffix;
+  default_placement.storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr);
+  default_placement.data_extra_pool = name + "." + default_storage_extra_pool_suffix;
+  placement_pools["default-placement"] = default_placement;
 
-  r = fix_pool_names(dpp, y);
+  int r = fix_pool_names(dpp, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: fix_pool_names returned r=" << r << dendl;
     return r;

From 5189c2e457bbbc0c7f430d3fa810b59896c8bb44 Mon Sep 17 00:00:00 2001
From: zhipeng li <qiuxinyidian@gmail.com>
Date: Mon, 27 Nov 2023 15:24:06 +0800
Subject: [PATCH 0856/2492] add checking result for rgw  frontend init fixes:
 https://tracker.ceph.com/issues/63644

Signed-off-by: zhipeng li <qiuxinyidian@gmail.com>
---
 src/rgw/rgw_lib.cc  | 9 ++++++++-
 src/rgw/rgw_main.cc | 7 ++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index f449cce21c02..b105e1dad1f3 100644
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -468,6 +468,7 @@ namespace rgw {
 
   int RGWLib::init(vector<const char*>& args)
   {
+    int r{0};
     /* alternative default for module */
     map<std::string,std::string> defaults = {
       { "debug_rgw", "1/5" },
@@ -524,7 +525,13 @@ namespace rgw {
     register_async_signal_handler(SIGUSR1, rgw::signal::handle_sigterm);
 
     main.init_tracepoints();
-    main.init_frontends2(this /* rgwlib */);
+    r = main.init_frontends2(this /* rgwlib */);
+    if (r != 0) {
+      derr << "ERROR: unable to initialize frontend, r = " << r << dendl;
+      main.shutdown();
+      return r;
+    }
+
     main.init_notification_endpoints();
     main.init_lua();
 
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 27b02f841951..57407b43481d 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -157,7 +157,12 @@ int main(int argc, char *argv[])
   main.init_opslog();
   main.init_tracepoints();
   main.init_lua();
-  main.init_frontends2(nullptr /* RGWLib */);
+  r = main.init_frontends2(nullptr /* RGWLib */);
+  if (r != 0) {
+    derr << "ERROR:  initialize frontend fail, r = " << r << dendl;
+    main.shutdown();
+    return r;
+  }
   main.init_notification_endpoints();
 
 #if defined(HAVE_SYS_PRCTL_H)

From 08ac3114e9fdb4fe7260002639058ded6e47a5ee Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 2 Dec 2023 06:38:28 +0100
Subject: [PATCH 0857/2492] doc/radosgw: fix formatting

Repair the formatting of a string that had a string inside backticks
that itself was inside double asterisks. The presence of the asterisks
around the entire string caused the backticks to appear in the rendered
documentation.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/multisite.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
index c7627371de74..b0bae6c52acf 100644
--- a/doc/radosgw/multisite.rst
+++ b/doc/radosgw/multisite.rst
@@ -464,8 +464,8 @@ For example:
 
 .. important:: The following steps assume a multi-site configuration that uses
    newly installed systems that have not yet begun storing data. **DO NOT
-   DELETE the ``default`` zone or its pools** if you are already using it to
-   store data, or the data will be irretrievably lost.
+   DELETE the** ``default`` **zone or its pools** if you are already using it
+   to store data, or the data will be irretrievably lost.
 
 Delete the default zone if needed:
 

From c0747922040841c371dbe2706354c08c73d8e59f Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 11 Nov 2023 14:15:49 +0100
Subject: [PATCH 0858/2492] librados/snap_set_diff: set end_size only if end
 object exists

Since commit 73f50a13109f ("rbd-mirror: use generalized deep copy for
image sync"), the only user of calc_snap_set_diff() immediately unsets
end_size otherwise.

calc_snap_set_diff() semantics are clearer if end_size is set together
with end_exists and clone_end_snap_id.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librados/snap_set_diff.cc  | 2 +-
 src/librbd/io/ObjectRequest.cc | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/librados/snap_set_diff.cc b/src/librados/snap_set_diff.cc
index 06f76b02345f..f80105b44ae1 100644
--- a/src/librados/snap_set_diff.cc
+++ b/src/librados/snap_set_diff.cc
@@ -76,7 +76,6 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
       saw_start = true;
     }
 
-    *end_size = r->size;
     if (end < a) {
       ldout(cct, 20) << " past end " << end << ", end object does not exist" << dendl;
       *end_exists = false;
@@ -88,6 +87,7 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
     }
     if (end <= b) {
       ldout(cct, 20) << " end" << dendl;
+      *end_size = r->size;
       *end_exists = true;
       *clone_end_snap_id = b;
       break;
diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 827f551d1f79..4bd5562bd874 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -842,8 +842,6 @@ void ObjectListSnapsRequest<I>::handle_list_snaps(int r) {
       diff.insert(0, image_ctx->layout.object_size);
       end_size = image_ctx->layout.object_size;
       clone_end_snap_id = end_snap_id;
-    } else if (!exists) {
-      end_size = 0;
     }
 
     if (exists) {

From bd52297a71100dd35874fa3aeda81fca1d828b38 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sun, 19 Nov 2023 22:44:28 +0100
Subject: [PATCH 0859/2492] test/librbd: make ListSnapsWholeObject actually
 test stuff

Despite being added in commit 66dd53d9c4d9 ("librbd: optionally return
full object extent for any snapshot deltas") ostensibly to test the new
LIST_SNAPS_FLAG_WHOLE_OBJECT code, it surely doesn't do that because
the flag isn't even passed to MockObjectListSnapsRequest::create().

I can only guess, but it looks like snap ID 3 was intended to be
a starting point.  Otherwise, with 0 and CEPH_NOSNAP passed as snap
IDs, the overlap that is set up for the clone wouldn't affect the
computation in any way.

Use snap ID 3 as a starting point and run both with and without
LIST_SNAPS_FLAG_WHOLE_OBJECT on the same snapset to pinpoint the
difference.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/io/test_mock_ObjectRequest.cc | 51 +++++++++++++------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index 0690b7722a0c..ca660470df48 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -1926,7 +1926,7 @@ TEST_F(TestMockIoObjectRequest, ListSnapsWholeObject) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockTestImageCtx mock_image_ctx(*ictx);
-  mock_image_ctx.parent = &mock_image_ctx;
+  mock_image_ctx.snaps = {3};
 
   InSequence seq;
 
@@ -1937,30 +1937,51 @@ TEST_F(TestMockIoObjectRequest, ListSnapsWholeObject) {
   clone_info.cloneid = 3;
   clone_info.snaps = {3};
   clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 1}};
-  clone_info.size = 4194304;
+  clone_info.size = mock_image_ctx.layout.object_size;
   snap_set.clones.push_back(clone_info);
 
   clone_info.cloneid = CEPH_NOSNAP;
   clone_info.snaps = {};
   clone_info.overlap = {};
-  clone_info.size = 4194304;
+  clone_info.size = mock_image_ctx.layout.object_size;
   snap_set.clones.push_back(clone_info);
 
   expect_list_snaps(mock_image_ctx, snap_set, 0);
 
-  SnapshotDelta snapshot_delta;
-  C_SaferCond ctx;
-  auto req = MockObjectListSnapsRequest::create(
-    &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
-    {0, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
+      {3, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
 
-  SnapshotDelta expected_snapshot_delta;
-  expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
-    0, mock_image_ctx.layout.object_size - 1,
-    {SPARSE_EXTENT_STATE_DATA, mock_image_ctx.layout.object_size - 1});
-  ASSERT_EQ(expected_snapshot_delta, snapshot_delta);
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1, mock_image_ctx.layout.object_size - 2,
+      {SPARSE_EXTENT_STATE_DATA, mock_image_ctx.layout.object_size - 2});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
+      {3, CEPH_NOSNAP}, LIST_SNAPS_FLAG_WHOLE_OBJECT, {}, &snapshot_delta,
+      &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      0, mock_image_ctx.layout.object_size - 1,
+      {SPARSE_EXTENT_STATE_DATA, mock_image_ctx.layout.object_size - 1});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
 }
 
 } // namespace io

From 8f86d80614680afecbfe82b2a6e965678a3c6034 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Mon, 27 Nov 2023 10:11:52 +0100
Subject: [PATCH 0860/2492] librbd: fix LIST_SNAPS_FLAG_WHOLE_OBJECT behavior

Bundling read_whole_object and LIST_SNAPS_FLAG_WHOLE_OBJECT cases
together is wrong:

- In read_whole_object case, calc_snap_set_diff() sets just
  read_whole_object.  Everything else is zeroed out and may require
  resetting to fit with the rest of ObjectListSnapsRequest logic.

- In LIST_SNAPS_FLAG_WHOLE_OBJECT case, only the diff should be
  expanded.  Everything else is set by calc_snap_set_diff() and should
  be used as is.  This goes for end_size in particular -- if it's reset
  to object size, bogus zero extents may be returned as the object
  would appear to have grown.

This is a regression introduced in commit 4429ed4f3f4c ("librbd: switch
diff iterate API to use new snaps list dispatch methods") by way of
commit 66dd53d9c4d9 ("librbd: optionally return full object extent for
any snapshot deltas").

Fixes: https://tracker.ceph.com/issues/63654
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/io/ObjectRequest.cc                | 10 ++--
 src/test/librbd/io/test_mock_ObjectRequest.cc | 50 +++++++++++++++++++
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 4bd5562bd874..731b439dd269 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -834,14 +834,16 @@ void ObjectListSnapsRequest<I>::handle_list_snaps(int r) {
                        end_snap_id, &diff, &end_size, &exists,
                        &clone_end_snap_id, &read_whole_object);
 
-    if (read_whole_object ||
-        (!diff.empty() &&
-         ((m_list_snaps_flags & LIST_SNAPS_FLAG_WHOLE_OBJECT) != 0))) {
+    if (read_whole_object) {
       ldout(cct, 1) << "need to read full object" << dendl;
-      diff.clear();
       diff.insert(0, image_ctx->layout.object_size);
       end_size = image_ctx->layout.object_size;
       clone_end_snap_id = end_snap_id;
+    } else if ((m_list_snaps_flags & LIST_SNAPS_FLAG_WHOLE_OBJECT) != 0 &&
+               !diff.empty()) {
+      ldout(cct, 20) << "expanding diff from " << diff << dendl;
+      diff.clear();
+      diff.insert(0, image_ctx->layout.object_size);
     }
 
     if (exists) {
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index ca660470df48..5706abe313bd 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -1984,6 +1984,56 @@ TEST_F(TestMockIoObjectRequest, ListSnapsWholeObject) {
   }
 }
 
+TEST_F(TestMockIoObjectRequest, ListSnapsWholeObjectEndSize) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3};
+
+  InSequence seq;
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 3;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  // smaller than object extent (i.e. the op) to test end_size handling
+  clone_info.size = mock_image_ctx.layout.object_size - 2;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    EXPECT_TRUE(snapshot_delta.empty());
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
+      {4, CEPH_NOSNAP}, LIST_SNAPS_FLAG_WHOLE_OBJECT, {}, &snapshot_delta,
+      &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    EXPECT_TRUE(snapshot_delta.empty());
+  }
+}
+
 } // namespace io
 } // namespace librbd
 

From 0a1f633e0240b4a7cfbcddd96d53fbf4b17f0b28 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Mon, 27 Nov 2023 11:59:26 +0100
Subject: [PATCH 0861/2492] librbd: fix read_whole_object handling in
 ObjectListSnapsRequest

Originally, in commit 2be4840afd4f ("librados/snap_set_diff: don't
assert on empty snapset"), exists was set to true.  This didn't make
ObjectListSnapsRequest, causing the following deep-copy tests to fail
when run against calc_snap_set_diff() rigged to return "whole object"
as described in [1]:

    TestDeepCopy.Snaps
    TestDeepCopy.SnapDiscard
    TestDeepCopy.CloneHideParent
    TestDeepCopy.Snaps_LargerDstObjSize
    TestDeepCopy.Snaps_SmallerDstObjSize

This is a regression introduced in commit cc87a8bd697e ("librbd:
deep-copy object utilizes image-extent IO methods") by way of commit
11923e234efc ("librbd: generic object list snapshot request").

[1] https://github.com/ceph/ceph/pull/20648#issuecomment-369292309

Fixes: https://tracker.ceph.com/issues/63654
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/io/ObjectRequest.cc                |  3 +-
 src/test/librbd/io/test_mock_ObjectRequest.cc | 36 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 731b439dd269..2bf06966c360 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -837,6 +837,7 @@ void ObjectListSnapsRequest<I>::handle_list_snaps(int r) {
     if (read_whole_object) {
       ldout(cct, 1) << "need to read full object" << dendl;
       diff.insert(0, image_ctx->layout.object_size);
+      exists = true;
       end_size = image_ctx->layout.object_size;
       clone_end_snap_id = end_snap_id;
     } else if ((m_list_snaps_flags & LIST_SNAPS_FLAG_WHOLE_OBJECT) != 0 &&
@@ -884,7 +885,7 @@ void ObjectListSnapsRequest<I>::handle_list_snaps(int r) {
                    << "end_size=" << end_size << ", "
                    << "prev_end_size=" << prev_end_size << ", "
                    << "exists=" << exists << ", "
-                   << "whole_object=" << read_whole_object << dendl;
+                   << "read_whole_object=" << read_whole_object << dendl;
 
     // check if object exists prior to start of incremental snap delta so that
     // we don't DNE the object if no additional deltas exist
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index 5706abe313bd..c20c825018bb 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -2034,6 +2034,42 @@ TEST_F(TestMockIoObjectRequest, ListSnapsWholeObjectEndSize) {
   }
 }
 
+TEST_F(TestMockIoObjectRequest, ListSnapsNoSnapsInSnapSet) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3};
+
+  InSequence seq;
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 3;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 0;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  SnapshotDelta snapshot_delta;
+  C_SaferCond ctx;
+  auto req = MockObjectListSnapsRequest::create(
+    &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size - 1}},
+    {0, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
+  SnapshotDelta expected_snapshot_delta;
+  expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+    0, mock_image_ctx.layout.object_size - 1,
+    {SPARSE_EXTENT_STATE_DATA, mock_image_ctx.layout.object_size - 1});
+  EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+}
+
 } // namespace io
 } // namespace librbd
 

From 084fb34971d733ce3f13e4902863539c95660ac7 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 28 Sep 2023 07:04:06 -0500
Subject: [PATCH 0862/2492] osd: fix a missing 'noexcept' on a move ctor

as a non-default, non-noexcept move ctor is ignored by
stl containers.

See clang-tidy's performance-noexcept-move-constructor

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2e82e74ab012..5f071817d3d6 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1446,10 +1446,13 @@ class PG : public DoutPrefixProvider,
  */
 class PGLockWrapper {
  public:
-  explicit PGLockWrapper(PGRef locked_pg) : m_pg{locked_pg} {}
+  template <typename A_PG_REF>
+  explicit PGLockWrapper(A_PG_REF&& locked_pg)
+      : m_pg{std::forward<A_PG_REF>(locked_pg)}
+  {}
   PGRef pg() { return m_pg; }
   ~PGLockWrapper();
-  PGLockWrapper(PGLockWrapper&& rhs) : m_pg(std::move(rhs.m_pg)) {
+  PGLockWrapper(PGLockWrapper&& rhs) noexcept : m_pg(std::move(rhs.m_pg)) {
     rhs.m_pg = nullptr;
   }
   PGLockWrapper(const PGLockWrapper& rhs) = delete;

From c616a9d017b5fcc85bb5c1556bccf4c77cc3899e Mon Sep 17 00:00:00 2001
From: Daniel Persson <mailto.woden@gmail.com>
Date: Wed, 29 Nov 2023 09:39:51 +0000
Subject: [PATCH 0863/2492] mgr/dashboard: Simplify authentication protocol By
 removing the dependency to PyJWT we also remove the dependency to the
 cryptographic library which in the dashboard module will create a crash. In
 newer implementations of the library PyO3 is used to run rust code in order
 to encrypt with Elliptic Curves. This is never used in the dashboard
 communication so a much simpler implementation where we only use the hmac
 sha256 algorithm to create the signed JWT message could be used.

Fixes: https://forum.proxmox.com/threads/ceph-warning-post-upgrade-to-v8.129371
Signed-off-by: Daniel Persson <mailto.woden@gmail.com>
---
 src/pybind/mgr/dashboard/constraints.txt  |  1 -
 src/pybind/mgr/dashboard/exceptions.py    | 12 ++++
 src/pybind/mgr/dashboard/requirements.txt |  1 -
 src/pybind/mgr/dashboard/services/auth.py | 70 ++++++++++++++++++++---
 src/pybind/mgr/dashboard/tox.ini          |  1 +
 5 files changed, 74 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/dashboard/constraints.txt b/src/pybind/mgr/dashboard/constraints.txt
index 55f81c92dec0..fd6141048800 100644
--- a/src/pybind/mgr/dashboard/constraints.txt
+++ b/src/pybind/mgr/dashboard/constraints.txt
@@ -1,6 +1,5 @@
 CherryPy~=13.1
 more-itertools~=8.14
-PyJWT~=2.0
 bcrypt~=3.1
 python3-saml~=1.4
 requests~=2.26
diff --git a/src/pybind/mgr/dashboard/exceptions.py b/src/pybind/mgr/dashboard/exceptions.py
index 96cbc5233561..d396a38d2c3a 100644
--- a/src/pybind/mgr/dashboard/exceptions.py
+++ b/src/pybind/mgr/dashboard/exceptions.py
@@ -121,3 +121,15 @@ class GrafanaError(Exception):
 
 class PasswordPolicyException(Exception):
     pass
+
+
+class ExpiredSignatureError(Exception):
+    pass
+
+
+class InvalidTokenError(Exception):
+    pass
+
+
+class InvalidAlgorithmError(Exception):
+    pass
diff --git a/src/pybind/mgr/dashboard/requirements.txt b/src/pybind/mgr/dashboard/requirements.txt
index 8003d62a5523..292971819c9c 100644
--- a/src/pybind/mgr/dashboard/requirements.txt
+++ b/src/pybind/mgr/dashboard/requirements.txt
@@ -1,7 +1,6 @@
 bcrypt
 CherryPy
 more-itertools
-PyJWT
 pyopenssl
 requests
 Routes
diff --git a/src/pybind/mgr/dashboard/services/auth.py b/src/pybind/mgr/dashboard/services/auth.py
index f13963abffdd..3c6002312524 100644
--- a/src/pybind/mgr/dashboard/services/auth.py
+++ b/src/pybind/mgr/dashboard/services/auth.py
@@ -1,17 +1,19 @@
 # -*- coding: utf-8 -*-
 
+import base64
+import hashlib
+import hmac
 import json
 import logging
 import os
 import threading
 import time
 import uuid
-from base64 import b64encode
 
 import cherrypy
-import jwt
 
 from .. import mgr
+from ..exceptions import ExpiredSignatureError, InvalidAlgorithmError, InvalidTokenError
 from .access_control import LocalAuthenticator, UserDoesNotExist
 
 cherrypy.config.update({
@@ -33,7 +35,7 @@ class JwtManager(object):
     @staticmethod
     def _gen_secret():
         secret = os.urandom(16)
-        return b64encode(secret).decode('utf-8')
+        return base64.b64encode(secret).decode('utf-8')
 
     @classmethod
     def init(cls):
@@ -45,6 +47,54 @@ def init(cls):
             mgr.set_store('jwt_secret', secret)
         cls._secret = secret
 
+    @classmethod
+    def array_to_base64_string(cls, message):
+        jsonstr = json.dumps(message, sort_keys=True).replace(" ", "")
+        string_bytes = base64.urlsafe_b64encode(bytes(jsonstr, 'UTF-8'))
+        return string_bytes.decode('UTF-8').replace("=", "")
+
+    @classmethod
+    def encode(cls, message, secret):
+        header = {"alg": cls.JWT_ALGORITHM, "typ": "JWT"}
+        base64_header = cls.array_to_base64_string(header)
+        base64_message = cls.array_to_base64_string(message)
+        base64_secret = base64.urlsafe_b64encode(hmac.new(
+            bytes(secret, 'UTF-8'),
+            msg=bytes(base64_header + "." + base64_message, 'UTF-8'),
+            digestmod=hashlib.sha256
+        ).digest()).decode('UTF-8').replace("=", "")
+        return base64_header + "." + base64_message + "." + base64_secret
+
+    @classmethod
+    def decode(cls, message, secret):
+        split_message = message.split(".")
+        base64_header = split_message[0]
+        base64_message = split_message[1]
+        base64_secret = split_message[2]
+
+        decoded_header = json.loads(base64.urlsafe_b64decode(base64_header))
+
+        if decoded_header['alg'] != cls.JWT_ALGORITHM:
+            raise InvalidAlgorithmError()
+
+        incoming_secret = base64.urlsafe_b64encode(hmac.new(
+            bytes(secret, 'UTF-8'),
+            msg=bytes(base64_header + "." + base64_message, 'UTF-8'),
+            digestmod=hashlib.sha256
+        ).digest()).decode('UTF-8').replace("=", "")
+
+        if base64_secret != incoming_secret:
+            raise InvalidTokenError()
+
+        # We add ==== as padding to ignore the requirement to have correct padding in
+        # the urlsafe_b64decode method.
+        decoded_message = json.loads(base64.urlsafe_b64decode(base64_message + "===="))
+        now = int(time.time())
+        if decoded_message['exp'] < now:
+            raise ExpiredSignatureError()
+
+        return decoded_message
+
     @classmethod
     def gen_token(cls, username):
         if not cls._secret:
@@ -59,13 +109,13 @@ def gen_token(cls, username):
             'iat': now,
             'username': username
         }
-        return jwt.encode(payload, cls._secret, algorithm=cls.JWT_ALGORITHM)  # type: ignore
+        return cls.encode(payload, cls._secret)  # type: ignore
 
     @classmethod
     def decode_token(cls, token):
         if not cls._secret:
             cls.init()
-        return jwt.decode(token, cls._secret, algorithms=cls.JWT_ALGORITHM)  # type: ignore
+        return cls.decode(token, cls._secret)  # type: ignore
 
     @classmethod
     def get_token_from_header(cls):
@@ -99,8 +149,8 @@ def get_username(cls):
     @classmethod
     def get_user(cls, token):
         try:
-            dtoken = JwtManager.decode_token(token)
-            if not JwtManager.is_blocklisted(dtoken['jti']):
+            dtoken = cls.decode_token(token)
+            if not cls.is_blocklisted(dtoken['jti']):
                 user = AuthManager.get_user(dtoken['username'])
                 if user.last_update <= dtoken['iat']:
                     return user
@@ -110,10 +160,12 @@ def get_user(cls, token):
                 )
             else:
                 cls.logger.debug('Token is block-listed')  # type: ignore
-        except jwt.ExpiredSignatureError:
+        except ExpiredSignatureError:
             cls.logger.debug("Token has expired")  # type: ignore
-        except jwt.InvalidTokenError:
+        except InvalidTokenError:
             cls.logger.debug("Failed to decode token")  # type: ignore
+        except InvalidAlgorithmError:
+            cls.logger.debug("Only the HS256 algorithm is supported.")  # type: ignore
         except UserDoesNotExist:
             cls.logger.debug(  # type: ignore
                 "Invalid token: user %s does not exist", dtoken['username']
diff --git a/src/pybind/mgr/dashboard/tox.ini b/src/pybind/mgr/dashboard/tox.ini
index 47756e946e12..271df286ec5e 100644
--- a/src/pybind/mgr/dashboard/tox.ini
+++ b/src/pybind/mgr/dashboard/tox.ini
@@ -20,6 +20,7 @@ addopts =
 deps =
     -rrequirements.txt
     -cconstraints.txt
+    PyJWT
 
 [base-test]
 deps =

From 28551b41f878e7ad8f43e85bcbc8c9f64c07346c Mon Sep 17 00:00:00 2001
From: PC-Admin <perthserverplus@gmail.com>
Date: Tue, 28 Nov 2023 17:45:52 +0800
Subject: [PATCH 0864/2492] doc/rados: update "stretch mode"

Update stretch mode docs, min_size and max_size are no longer defined in
the CRUSH map and the example rule given will fail to compile.

Specify that the tiebreaker data centre cannot be defined in CRUSH as
this produces an error.

Signed-off-by: Michael Collins <perthserverplus@gmail.com>
---
 doc/rados/operations/stretch-mode.rst | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/doc/rados/operations/stretch-mode.rst b/doc/rados/operations/stretch-mode.rst
index f797b5b91f4c..4adf60e46f6d 100644
--- a/doc/rados/operations/stretch-mode.rst
+++ b/doc/rados/operations/stretch-mode.rst
@@ -121,8 +121,6 @@ your CRUSH map. This procedure shows how to do this.
 
       rule stretch_rule {
              id 1
-             min_size 1
-             max_size 10
              type replicated
              step take site1
              step chooseleaf firstn 2 type host
@@ -141,17 +139,36 @@ your CRUSH map. This procedure shows how to do this.
 
 #. Run the monitors in connectivity mode. See `Changing Monitor Elections`_.
 
+   .. prompt:: bash $
+
+      ceph mon set election_strategy connectivity
+
 #. Command the cluster to enter stretch mode. In this example, ``mon.e`` is the
    tiebreaker monitor and we are splitting across data centers. The tiebreaker
    monitor must be assigned a data center that is neither ``site1`` nor
-   ``site2``. For this purpose you can create another data-center bucket named
-   ``site3`` in your CRUSH and place ``mon.e`` there:
+   ``site2``. This data center **should not** be defined in your CRUSH map, here 
+   we are placing ``mon.e`` in a virtual data center called ``site3``:
 
    .. prompt:: bash $
 
       ceph mon set_location e datacenter=site3
       ceph mon enable_stretch_mode e stretch_rule datacenter
 
+#. Set the replication levels for each pool. Here we are setting the standard
+   replication levels for a stretch mode cluster. Where ``4`` copies will be kept
+   in total, with a minimum of ``2`` in each data center:
+
+   .. prompt:: bash $
+
+      ceph osd pool set ceph_data min_size 2
+      set pool 2 min_size to 2
+      ceph osd pool set ceph_data size 4
+      set pool 2 size to 4
+      ceph osd pool set ceph_metadata min_size 2
+      set pool 3 min_size to 2
+      ceph osd pool set ceph_metadata size 4
+      set pool 3 size to 4
+
 When stretch mode is enabled, PGs will become active only when they peer
 across data centers (or across whichever CRUSH bucket type was specified),
 assuming both are alive. Pools will increase in size from the default ``3`` to

From b73656787d618197c60372d1817eab35879fa478 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 3 Dec 2023 13:17:46 +0100
Subject: [PATCH 0865/2492] doc/rados: repair stretch-mode.rst

Remove a section of doc/rados/operations/stretch-mode.rst that I wrongly
re-included after its removal. The request for this (re)-removal is
here: https://github.com/ceph/ceph/pull/54689#discussion_r1413007655.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/stretch-mode.rst | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/doc/rados/operations/stretch-mode.rst b/doc/rados/operations/stretch-mode.rst
index 4adf60e46f6d..787e8cb4d930 100644
--- a/doc/rados/operations/stretch-mode.rst
+++ b/doc/rados/operations/stretch-mode.rst
@@ -154,21 +154,6 @@ your CRUSH map. This procedure shows how to do this.
       ceph mon set_location e datacenter=site3
       ceph mon enable_stretch_mode e stretch_rule datacenter
 
-#. Set the replication levels for each pool. Here we are setting the standard
-   replication levels for a stretch mode cluster. Where ``4`` copies will be kept
-   in total, with a minimum of ``2`` in each data center:
-
-   .. prompt:: bash $
-
-      ceph osd pool set ceph_data min_size 2
-      set pool 2 min_size to 2
-      ceph osd pool set ceph_data size 4
-      set pool 2 size to 4
-      ceph osd pool set ceph_metadata min_size 2
-      set pool 3 min_size to 2
-      ceph osd pool set ceph_metadata size 4
-      set pool 3 size to 4
-
 When stretch mode is enabled, PGs will become active only when they peer
 across data centers (or across whichever CRUSH bucket type was specified),
 assuming both are alive. Pools will increase in size from the default ``3`` to

From 11114764817b4ee77646cd4e459410626b729301 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 28 Aug 2023 15:59:17 +0530
Subject: [PATCH 0866/2492] mgr/nfs: include pseudo in JSON output when nfs
 export apply -i fails

[
  {
    "msg": "Failed to apply export: export FSAL user_id must be 'nfs.my-nfs.1'",
    "state": "error"
  },
 {
    "msg": "Failed to apply export: export FSAL user_id must be 'nfs.my-nfs.2'",
    "state": "error"
 }
]

it is a bit of hassle to know which export's update/creation failed as the dict contains
no info about the export(like pseudo path), therefore with something like below, it is
much better to figure out which export block needs correction

[
  {
    "pseudo": "cephfs1",
    "msg": "Failed to apply export: export FSAL user_id must be 'nfs.my-nfs.1'",
    "state": "error"
  },
  {
    "pseudo": "cephfs2",
    "msg": "Failed to apply export: export FSAL user_id must be 'nfs.my-nfs.2'",
    "state": "error"
  }
]

Fixes: https://tracker.ceph.com/issues/62626
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/nfs/export.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py
index aaa93c34f6c1..2d07cd6eab60 100644
--- a/src/pybind/mgr/nfs/export.py
+++ b/src/pybind/mgr/nfs/export.py
@@ -583,7 +583,8 @@ def _change_export(self, cluster_id: str, export: Dict) -> Dict[str, Any]:
         except Exception as ex:
             msg = f'Failed to apply export: {ex}'
             log.exception(msg)
-            return {"state": "error", "msg": msg, "exception": ex}
+            return {"state": "error", "msg": msg, "exception": ex,
+                    "pseudo": export['pseudo']}
 
     def _update_user_id(
             self,

From e1ebfc023a929689c2090bafcde38861817f9f37 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 30 Aug 2023 14:24:23 +0530
Subject: [PATCH 0867/2492] qa: test nfs export apply -i failure returns pseudo
 path in output JSON

Fixes: https://tracker.ceph.com/issues/62626
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/cephfs/test_nfs.py | 64 +++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/qa/tasks/cephfs/test_nfs.py b/qa/tasks/cephfs/test_nfs.py
index 94d3182f98b5..2d06cbac7baa 100644
--- a/qa/tasks/cephfs/test_nfs.py
+++ b/qa/tasks/cephfs/test_nfs.py
@@ -1074,3 +1074,67 @@ def test_nfs_export_apply_json_output_states(self):
         finally:
             self._delete_cluster_with_fs(self.fs_name, mnt_pt)
             self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])
+
+    def test_pseudo_path_in_json_response_when_updating_exports_failed(self):
+        """
+        Test that on export update/creation failure while using
+        ceph nfs export apply <nfs_cluster> -i <json/conf>, the failed
+        exports pseudo paths are visible in the JSON response to CLI and the
+        return code is set to EIO.
+        """
+        mnt_pt = self._sys_cmd(['mktemp', '-d']).decode().strip()
+        self._create_cluster_with_fs(self.fs_name, mnt_pt)
+        self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir1'])
+        self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir2'])
+        self._create_export(export_id='1',
+                            extra_cmd=['--pseudo-path', self.pseudo_path])
+
+        ret = self.ctx.cluster.run(args=['ceph', 'nfs', 'export', 'apply',
+                                         self.cluster_id, '-i', '-'],
+                                   check_status=False,
+                                   stdin=json.dumps([
+                                    {
+                                        "export_id": 11,  # change not allowed
+                                        "path": "/testdir1",
+                                        "pseudo": self.pseudo_path,
+                                        "squash": "none",
+                                        "access_type": "rw",
+                                        "protocols": [4],
+                                        "fsal": {
+                                            "name": "CEPH",
+                                            "fs_name": self.fs_name
+                                        }
+                                    },
+                                    {
+                                        "path": "/testdir2",
+                                        "pseudo": self.pseudo_path+'1',
+                                        "squash": "none",
+                                        "access_type": "rw",
+                                        "protocols": [4],
+                                        "fsal": {
+                                            "name": "CEPH",
+                                            "fs_name": "foo"  # invalid fs
+                                        }
+                                    }]),
+                                   stdout=StringIO(), stderr=StringIO())
+
+        try:
+            # EIO since multiple exports failure (first export failed to be
+            # modified while the second one failed to be created)
+            self.assertEqual(ret[0].returncode, errno.EIO)
+            err_info = ret[0].stdout
+            if err_info:
+                update_details = json.loads(err_info.getvalue())
+                self.assertEqual(update_details[0]["pseudo"], self.pseudo_path)
+                self.assertEqual(update_details[1]["pseudo"], self.pseudo_path+'1')
+            else:
+                self.fail("Could not retrieve any export update data")
+
+            # verify second export wasn't created
+            exports = json.loads(self._nfs_cmd('export', 'ls',
+                                               self.cluster_id, '--detailed'))
+            self.assertEqual(len(exports), 1)
+
+        finally:
+            self._delete_cluster_with_fs(self.fs_name, mnt_pt)
+            self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])

From 04a54fbd8297e1027e1a8d0259ff406b41a2ae4c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 4 Dec 2023 08:53:53 -0500
Subject: [PATCH 0868/2492] PendingReleaseNotes: announce cephfs-shell avail.
 on rhel9

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 PendingReleaseNotes | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 9b3dfede8bbd..674b72d464cd 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,5 +1,7 @@
 >=19.0.0
 
+* The cephfs-shell utility is now packaged for RHEL 9 / CentOS 9 as required
+  python dependencies are now available in EPEL9.
 * RGW: S3 multipart uploads using Server-Side Encryption now replicate correctly in
   multi-site. Previously, the replicas of such objects were corrupted on decryption.
   A new tool, ``radosgw-admin bucket resync encrypted multipart``, can be used to

From 42e121a42a51e09f456d2e4e303cae0f114637c2 Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Thu, 26 Oct 2023 13:25:44 +0530
Subject: [PATCH 0869/2492] qa: add rbd/nvmeof test

A basic test for ceph-nvmeof[1] where
nvmeof initiator is created.
It requires use of a new task "nvmeof_gateway_cfg"
under cephadm which shares config information
between two remote hosts.

[1] https://github.com/ceph/ceph-nvmeof/

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 qa/suites/rbd/nvmeof/%                        |  0
 qa/suites/rbd/nvmeof/.qa                      |  1 +
 qa/suites/rbd/nvmeof/base/.qa                 |  1 +
 qa/suites/rbd/nvmeof/base/install.yaml        | 32 ++++++++
 qa/suites/rbd/nvmeof/centos_latest.yaml       |  1 +
 qa/suites/rbd/nvmeof/cluster/+                |  0
 qa/suites/rbd/nvmeof/cluster/.qa              |  1 +
 qa/suites/rbd/nvmeof/cluster/fixed-3.yaml     | 13 +++
 qa/suites/rbd/nvmeof/cluster/openstack.yaml   |  8 ++
 qa/suites/rbd/nvmeof/workloads/.qa            |  1 +
 .../nvmeof/workloads/nvmeof_initiator.yaml    |  6 ++
 qa/tasks/cephadm.py                           | 38 +++++++++
 qa/workunits/rbd/nvmeof_initiator.sh          | 79 +++++++++++++++++++
 13 files changed, 181 insertions(+)
 create mode 100644 qa/suites/rbd/nvmeof/%
 create mode 120000 qa/suites/rbd/nvmeof/.qa
 create mode 120000 qa/suites/rbd/nvmeof/base/.qa
 create mode 100644 qa/suites/rbd/nvmeof/base/install.yaml
 create mode 120000 qa/suites/rbd/nvmeof/centos_latest.yaml
 create mode 100644 qa/suites/rbd/nvmeof/cluster/+
 create mode 120000 qa/suites/rbd/nvmeof/cluster/.qa
 create mode 100644 qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
 create mode 100644 qa/suites/rbd/nvmeof/cluster/openstack.yaml
 create mode 120000 qa/suites/rbd/nvmeof/workloads/.qa
 create mode 100644 qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
 create mode 100755 qa/workunits/rbd/nvmeof_initiator.sh

diff --git a/qa/suites/rbd/nvmeof/% b/qa/suites/rbd/nvmeof/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rbd/nvmeof/.qa b/qa/suites/rbd/nvmeof/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/base/.qa b/qa/suites/rbd/nvmeof/base/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/base/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/base/install.yaml b/qa/suites/rbd/nvmeof/base/install.yaml
new file mode 100644
index 000000000000..5a852f14dbe1
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/base/install.yaml
@@ -0,0 +1,32 @@
+use_shaman: True
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+    host.a:
+    # get state before nvmeof deployment
+    - ceph orch status
+    - ceph orch ps
+    - ceph orch host ls
+    - ceph orch device ls
+    - ceph osd lspools
+    # create pool
+    - ceph osd pool create mypool
+    - rbd pool init mypool
+    # deploy nvmeof
+    ## Uncomment to test specific nvmeof images
+    ## - ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:latest
+    - ceph orch apply nvmeof mypool --placement="1 $(hostname)"
+    - ceph orch ps --refresh
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool
+
+- cephadm.nvmeof_gateway_cfg:
+    source: host.a 
+    target: client.1
+    service: nvmeof.mypool
+
+- exec:
+    client.0:
+      - journalctl -u $(systemctl list-units | grep nvmeof.mypool | awk '{print $1}')
diff --git a/qa/suites/rbd/nvmeof/centos_latest.yaml b/qa/suites/rbd/nvmeof/centos_latest.yaml
new file mode 120000
index 000000000000..bd9854e70298
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/centos_latest.yaml
@@ -0,0 +1 @@
+.qa/distros/supported/centos_latest.yaml
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/cluster/+ b/qa/suites/rbd/nvmeof/cluster/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rbd/nvmeof/cluster/.qa b/qa/suites/rbd/nvmeof/cluster/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/cluster/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml b/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
new file mode 100644
index 000000000000..42e696cd2f14
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
@@ -0,0 +1,13 @@
+roles:
+- - host.a
+  - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - client.0
+- - host.b
+  - mon.b
+  - osd.2
+  - osd.3
+  - osd.4
+  - client.1
diff --git a/qa/suites/rbd/nvmeof/cluster/openstack.yaml b/qa/suites/rbd/nvmeof/cluster/openstack.yaml
new file mode 100644
index 000000000000..40fef4770b9f
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/cluster/openstack.yaml
@@ -0,0 +1,8 @@
+openstack:
+  - machine:
+      disk: 40 # GB
+      ram: 8000 # MB
+      cpus: 1
+    volumes: # attached to each instance
+      count: 4
+      size: 30 # GB
diff --git a/qa/suites/rbd/nvmeof/workloads/.qa b/qa/suites/rbd/nvmeof/workloads/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/workloads/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
new file mode 100644
index 000000000000..4c947c1f787f
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
@@ -0,0 +1,6 @@
+tasks:
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.1:
+        - rbd/nvmeof_initiator.sh
diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 0431aac8533e..1fdf6ee012e3 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -21,6 +21,7 @@
 from teuthology.orchestra import run
 from teuthology.orchestra.daemon import DaemonGroup
 from teuthology.config import config as teuth_config
+from teuthology.exceptions import ConfigError
 from textwrap import dedent
 from tasks.cephfs.filesystem import MDSCluster, Filesystem
 from tasks.util import chacra
@@ -100,6 +101,43 @@ def update_archive_setting(ctx, key, value):
         yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
 
 
+@contextlib.contextmanager
+def nvmeof_gateway_cfg(ctx, config):
+    source_host = config.get('source')
+    target_host = config.get('target')
+    nvmeof_service = config.get('service')
+    if not (source_host and target_host and nvmeof_service):
+        raise ConfigError('nvmeof_gateway_cfg requires "source", "target", and "service"')
+    remote = list(ctx.cluster.only(source_host).remotes.keys())[0]
+    ip_address = remote.ip_address
+    gateway_name = ""
+    r = remote.run(args=[
+        'systemctl', 'list-units',
+        run.Raw('|'), 'grep', nvmeof_service
+    ], stdout=StringIO())
+    output = r.stdout.getvalue()
+    pattern_str = f"{re.escape(nvmeof_service)}(.*?)(?=\.service)"
+    pattern = re.compile(pattern_str)
+    match = pattern.search(output)
+    if match:
+        gateway_name = match.group()
+    conf_data = dedent(f"""
+        NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
+        NVMEOF_GATEWAY_NAME={gateway_name}
+        """)
+    target_remote = list(ctx.cluster.only(target_host).remotes.keys())[0]
+    target_remote.write_file(
+        path='/etc/ceph/nvmeof.env',
+        data=conf_data,
+        sudo=True
+    )
+
+    try:
+        yield
+    finally:
+        pass
+
+
 @contextlib.contextmanager
 def normalize_hostnames(ctx):
     """
diff --git a/qa/workunits/rbd/nvmeof_initiator.sh b/qa/workunits/rbd/nvmeof_initiator.sh
new file mode 100755
index 000000000000..c3f3c1082bfc
--- /dev/null
+++ b/qa/workunits/rbd/nvmeof_initiator.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -ex
+
+sudo modprobe nvme-fabrics
+sudo modprobe nvme-tcp
+sudo dnf install nvme-cli -y
+
+# import NVMEOF_GATEWAY_IP_ADDRESS and NVMEOF_GATEWAY_NAME=nvmeof.poolname.smithiXXX.abcde
+source /etc/ceph/nvmeof.env
+
+HOSTNAME=$(hostname)
+IMAGE="quay.io/ceph/nvmeof-cli:latest"
+RBD_POOL=$(awk -F'.' '{print $2}' <<< "$NVMEOF_GATEWAY_NAME")
+RBD_IMAGE="myimage"
+RBD_SIZE=$((1024*8)) #8GiB
+BDEV="mybdev"
+SERIAL="SPDK00000000000001"
+NQN="nqn.2016-06.io.spdk:cnode1"
+PORT="4420"
+SRPORT="5500"
+DISCOVERY_PORT="8009"
+
+rbd create $RBD_POOL/$RBD_IMAGE --size $RBD_SIZE
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_bdev --pool $RBD_POOL --image $RBD_IMAGE --bdev $BDEV
+sudo podman images
+sudo podman ps
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_subsystem --subnqn $NQN --serial $SERIAL
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT add_namespace --subnqn $NQN --bdev $BDEV
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_listener -n $NQN -g client.$NVMEOF_GATEWAY_NAME -a $NVMEOF_GATEWAY_IP_ADDRESS -s $PORT
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT add_host --subnqn $NQN --host "*"
+sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT get_subsystems
+sudo lsmod | grep nvme
+sudo nvme list
+sudo nvme discover -t tcp -a $NVMEOF_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT
+sudo nvme connect -t tcp --traddr $NVMEOF_GATEWAY_IP_ADDRESS -s $PORT -n $NQN
+sudo nvme list
+
+echo "testing nvmeof initiator..."
+
+nvme_model="SPDK bdev Controller"
+
+echo "Test 1: create initiator - starting"
+if ! sudo nvme list | grep -q "$nvme_model"; then
+  echo "nvmeof initiator not created!"
+  exit 1
+fi
+echo "Test 1: create initiator - passed!"
+
+
+echo "Test 2: device size - starting"
+image_size_in_bytes=$(($RBD_SIZE * 1024 * 1024))
+nvme_size=$(sudo nvme list --output-format=json | \
+        jq -r ".Devices | .[] | select(.ModelNumber == \"$nvme_model\") | .PhysicalSize")
+if [ "$image_size_in_bytes" != "$nvme_size" ]; then
+  echo "block device size do not match!"
+  exit 1
+fi
+echo "Test 2: device size - passed!"
+
+
+echo "Test 3: basic IO - starting"
+nvme_drive=$(sudo nvme list --output-format=json | \
+        jq -r ".Devices | .[] | select(.ModelNumber == \"$nvme_model\") | .DevicePath")
+io_input_file="/tmp/nvmeof_test_input"
+echo "Hello world" > $io_input_file
+truncate -s 2k $io_input_file
+sudo dd if=$io_input_file of=$nvme_drive oflag=direct count=1 bs=2k #write
+io_output_file="/tmp/nvmeof_test_output"
+sudo dd if=$nvme_drive of=$io_output_file iflag=direct count=1 bs=2k #read
+if ! cmp $io_input_file $io_output_file; then
+  echo "nvmeof initiator - io test failed!"
+  exit 1
+fi
+sudo rm -f $io_input_file $io_output_file
+echo "Test 3: basic IO - passed!"
+
+
+echo "nvmeof initiator tests passed!"

From 217c9dbed66730286497e1bbca2a7048d314f391 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 14:34:23 +0000
Subject: [PATCH 0870/2492] doc/dev/release-checklists: uncheck everything!

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 90 +++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 45 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 3a3b2a98cd76..d20bd01ec081 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -15,10 +15,10 @@ that was just released (X-1).2.0.
 Versions and tags
 -----------------
 
-- [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
-- [x] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
-- [x] Update src/ceph_release with the new release name, number, and type ('dev')
-- [x] Initial tag vX.0.0 (so that we can distinguish from (and sort
+- [ ] Update CMakeLists.txt VERSION (right at the top to X.0.0)
+- [ ] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
+- [ ] Update src/ceph_release with the new release name, number, and type ('dev')
+- [ ] Initial tag vX.0.0 (so that we can distinguish from (and sort
       after) the backported (X-1).2.Z versions.
 
 ### Notes on tagging
@@ -33,38 +33,38 @@ Define release names and constants
 
 Make sure X (and, ideally, X+1) is defined:
 
-- [x] src/common/ceph_releases.h (`ceph_release_t`)
-- [x] src/common/ceph_strings.cc (`ceph_release_name()`)
-- [x] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
-- [x] src/include/rbd/librbd.h (`LIBRBD_VER_MINOR` to X)
-- [x] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
-- [x] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
+- [ ] src/common/ceph_releases.h (`ceph_release_t`)
+- [ ] src/common/ceph_strings.cc (`ceph_release_name()`)
+- [ ] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
+- [ ] src/include/rbd/librbd.h (`LIBRBD_VER_MINOR` to X)
+- [ ] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
+- [ ] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
 
 Scripts
 ~~~~~~~
 
-- [x] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
-- [x] src/script/ceph-release-notes (X-1)
+- [ ] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
+- [ ] src/script/ceph-release-notes (X-1)
 - [ ] ceph-build.git scripts/build_utils.sh `release_from_version()`
 
 Misc
 ~~~~
-- [x] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
-- [x] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
-- [x] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
+- [ ] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
+- [ ] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
+- [ ] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
 
 Docs
 ~~~~
 
-- [x] Remove ``doc/releases/*.rst``. This should leave behind ``doc/releases/releases.yml`` which is used for doc building purposes.
-- [x] Cherry-pick 8cf9ad62949516666ad0f2c0bb7726ef68e4d666 ("doc: add releases links to toc"). There will be trivial conflicts.
-- [x] Add redirect for new major release at `RTD <https://readthedocs.org/dashboard/ceph/redirects/>`_.
+- [ ] Remove ``doc/releases/*.rst``. This should leave behind ``doc/releases/releases.yml`` which is used for doc building purposes.
+- [ ] Cherry-pick 8cf9ad62949516666ad0f2c0bb7726ef68e4d666 ("doc: add releases links to toc"). There will be trivial conflicts.
+- [ ] Add redirect for new major release at `RTD <https://readthedocs.org/dashboard/ceph/redirects/>`_.
 
 Feature bits
 ------------
 
-- [x] ensure that `SERVER_X` is defined
-- [x] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
+- [ ] ensure that `SERVER_X` is defined
+- [ ] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
 - [ ] look for features that (1) were present in X-2 and (2) have no
   client dependency and mark them `DEPRECATED` as of X.
 
@@ -72,25 +72,25 @@ Feature bits
 Compatsets
 ----------
 
-- [x] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
-- [x] mon/Monitor.cc (include in `get_supported_features()`)
-- [x] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
-- [x] mon/Monitor.cc (`calc_quorum_requirements()`)
-- [x] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
+- [ ] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
+- [ ] mon/Monitor.cc (include in `get_supported_features()`)
+- [ ] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
+- [ ] mon/Monitor.cc (`calc_quorum_requirements()`)
+- [ ] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
 
 Mon
 ---
 
-- [x] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
-- [x] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
-- [x] mon/MgrMonitor.cc adjust `always_on_modules`
-- [x] common/options/global.yaml.in define `mon_debug_no_require_X`
-- [x] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
-- [x] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
-- [x] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
-- [x] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
-- [x] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
-- [x] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
+- [ ] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
+- [ ] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
+- [ ] mon/MgrMonitor.cc adjust `always_on_modules`
+- [ ] common/options/global.yaml.in define `mon_debug_no_require_X`
+- [ ] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
+- [ ] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
+- [ ] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
+- [ ] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
+- [ ] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
+- [ ] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
 
 
 Code cleanup
@@ -105,20 +105,20 @@ Code cleanup
 QA suite
 --------
 
-- [x] create qa/suites/upgrade/(X-1)-x
-- [x] remove qa/suites/upgrade/(X-3)-x-*
-- [x] create qa/releases/X.yaml
-- [x] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
+- [ ] create qa/suites/upgrade/(X-1)-x
+- [ ] remove qa/suites/upgrade/(X-3)-x-*
+- [ ] create qa/releases/X.yaml
+- [ ] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
 
 
 ceph-build
 ----------
 In the `ceph/ceph-build.git` repo:
 
-- [x] add the version -> X mapping (`release_from_version()` in `scripts/build_utils.sh`)
-- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/build_osc`)
-- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/setup_osc`)
-- [x] grep for previous release and add relevant build targets (e.g. for reef https://github.com/ceph/ceph-build/pull/2076 and https://github.com/ceph/ceph-build/pull/2119)
+- [ ] add the version -> X mapping (`release_from_version()` in `scripts/build_utils.sh`)
+- [ ] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/build_osc`)
+- [ ] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/setup_osc`)
+- [ ] grep for previous release and add relevant build targets (e.g. for reef https://github.com/ceph/ceph-build/pull/2076 and https://github.com/ceph/ceph-build/pull/2119)
 
 
 ceph-container
@@ -138,8 +138,8 @@ See https://github.com/ceph/ceph-container/pull/2109 as an example for what to d
 After dev freeze
 ================
 
-- [x] add release name to redmine (using https://tracker.ceph.com/custom_fields/16/edit)
-- [x] add release name to .github/milestone.yml for github actions to automatically add milestone to backports (this commit must be backported to the release branch)
+- [ ] add release name to redmine (using https://tracker.ceph.com/custom_fields/16/edit)
+- [ ] add release name to .github/milestone.yml for github actions to automatically add milestone to backports (this commit must be backported to the release branch)
 
 First release candidate
 =======================

From 0c0767e243b6a42d4c6cbcc93261ffc71e056d4c Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 14:36:45 +0000
Subject: [PATCH 0871/2492] cmake, doc: bump project version to 19.0.0 for
 squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 CMakeLists.txt                 | 2 +-
 doc/dev/release-checklists.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4feb22c135e1..72d2273db175 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 
 project(ceph
-  VERSION 18.0.0
+  VERSION 19.0.0
   LANGUAGES CXX C ASM)
 
 cmake_policy(SET CMP0028 NEW)
diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index d20bd01ec081..430b253c5f2a 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -15,7 +15,7 @@ that was just released (X-1).2.0.
 Versions and tags
 -----------------
 
-- [ ] Update CMakeLists.txt VERSION (right at the top to X.0.0)
+- [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
 - [ ] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
 - [ ] Update src/ceph_release with the new release name, number, and type ('dev')
 - [ ] Initial tag vX.0.0 (so that we can distinguish from (and sort

From 1b2c8394cbe2b854fd6bd3d8740513bae61c51de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 17:43:39 +0200
Subject: [PATCH 0872/2492] cmake, doc: bump librbd version to 19.0.0 for squid
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 src/librbd/CMakeLists.txt      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 430b253c5f2a..4331e9db85b5 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -16,7 +16,7 @@ Versions and tags
 -----------------
 
 - [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
-- [ ] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
+- [x] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
 - [ ] Update src/ceph_release with the new release name, number, and type ('dev')
 - [ ] Initial tag vX.0.0 (so that we can distinguish from (and sort
       after) the backported (X-1).2.Z versions.
diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
index 2ba8e5d9f72d..3ba46028f0f4 100644
--- a/src/librbd/CMakeLists.txt
+++ b/src/librbd/CMakeLists.txt
@@ -345,7 +345,7 @@ endif()
 if(ENABLE_SHARED)
   set_target_properties(librbd PROPERTIES
     OUTPUT_NAME rbd
-    VERSION 1.18.0
+    VERSION 1.19.0
     SOVERSION 1
     CXX_VISIBILITY_PRESET hidden
     VISIBILITY_INLINES_HIDDEN ON)

From d2639a30b8d1bba54ada104f0f7200f37ce4517b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 17:45:42 +0200
Subject: [PATCH 0873/2492] ceph_release, doc: bump to 19/squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 src/ceph_release               | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 4331e9db85b5..66af04e7d965 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -17,7 +17,7 @@ Versions and tags
 
 - [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
 - [x] Update src/librbd/CMakeLists.txt VERSION (librbd target at the bottom to 1.X.0)
-- [ ] Update src/ceph_release with the new release name, number, and type ('dev')
+- [x] Update src/ceph_release with the new release name, number, and type ('dev')
 - [ ] Initial tag vX.0.0 (so that we can distinguish from (and sort
       after) the backported (X-1).2.Z versions.
 
diff --git a/src/ceph_release b/src/ceph_release
index 5640b4491a0d..67f3c2f5ae34 100644
--- a/src/ceph_release
+++ b/src/ceph_release
@@ -1,3 +1,3 @@
-18
-reef
+19
+squid
 dev

From 61c7b30bbd1f90ee9a1ee506f3a7b0908e0c4de7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 18:05:46 +0200
Subject: [PATCH 0874/2492] common, doc, include, mds, mon: add feature/release
 name squid
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 12 ++++++------
 src/common/ceph_releases.h     |  1 +
 src/common/ceph_strings.cc     |  2 ++
 src/include/rados.h            |  3 ++-
 src/include/rbd/librbd.h       |  2 +-
 src/mds/cephfs_features.h      |  2 +-
 src/mon/mon_types.h            | 14 ++++++++++++--
 7 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 66af04e7d965..938d6e3e343f 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -33,12 +33,12 @@ Define release names and constants
 
 Make sure X (and, ideally, X+1) is defined:
 
-- [ ] src/common/ceph_releases.h (`ceph_release_t`)
-- [ ] src/common/ceph_strings.cc (`ceph_release_name()`)
-- [ ] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
-- [ ] src/include/rbd/librbd.h (`LIBRBD_VER_MINOR` to X)
-- [ ] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
-- [ ] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
+- [x] src/common/ceph_releases.h (`ceph_release_t`)
+- [x] src/common/ceph_strings.cc (`ceph_release_name()`)
+- [x] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
+- [x] src/include/rbd/librbd.h (`LIBRBD_VER_MINOR` to X)
+- [x] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
+- [x] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
 
 Scripts
 ~~~~~~~
diff --git a/src/common/ceph_releases.h b/src/common/ceph_releases.h
index e09e191e5ec7..6d330b5d5b68 100644
--- a/src/common/ceph_releases.h
+++ b/src/common/ceph_releases.h
@@ -30,6 +30,7 @@ enum class ceph_release_t : std::uint8_t {
   pacific,
   quincy,
   reef,
+  squid,
   max,
 };
 
diff --git a/src/common/ceph_strings.cc b/src/common/ceph_strings.cc
index 18dcc701b31d..5103b1a3f83c 100644
--- a/src/common/ceph_strings.cc
+++ b/src/common/ceph_strings.cc
@@ -112,6 +112,8 @@ const char *ceph_release_name(int r)
 		return "quincy";
 	case CEPH_RELEASE_REEF:
 		return "reef";
+	case CEPH_RELEASE_SQUID:
+		return "squid";
 	default:
 		if (r < 0)
 			return "unspecified";
diff --git a/src/include/rados.h b/src/include/rados.h
index eac3a2159bf8..1fb86be947e6 100644
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -207,7 +207,8 @@ extern const char *ceph_osd_state_name(int s);
 #define CEPH_RELEASE_PACIFIC    16
 #define CEPH_RELEASE_QUINCY     17
 #define CEPH_RELEASE_REEF       18
-#define CEPH_RELEASE_MAX        19  /* highest + 1 */
+#define CEPH_RELEASE_SQUID      19
+#define CEPH_RELEASE_MAX        20  /* highest + 1 */
 
 /*
  * The error code to return when an OSD can't handle a write
diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h
index 50901cb8a82d..4a7e108e73f5 100644
--- a/src/include/rbd/librbd.h
+++ b/src/include/rbd/librbd.h
@@ -32,7 +32,7 @@ extern "C" {
 #include "features.h"
 
 #define LIBRBD_VER_MAJOR 1
-#define LIBRBD_VER_MINOR 18
+#define LIBRBD_VER_MINOR 19
 #define LIBRBD_VER_EXTRA 0
 
 #define LIBRBD_VERSION(maj, min, extra) ((maj << 16) + (min << 8) + extra)
diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h
index 3a67e96dba16..b90377779e1c 100644
--- a/src/mds/cephfs_features.h
+++ b/src/mds/cephfs_features.h
@@ -27,7 +27,7 @@ namespace ceph {
 // and update Server::update_required_client_features(). This feature bit
 // is used to indicate that operator only wants clients from that release or
 // later to mount CephFS.
-#define CEPHFS_CURRENT_RELEASE  CEPH_RELEASE_REEF
+#define CEPHFS_CURRENT_RELEASE  CEPH_RELEASE_SQUID
 
 // The first 5 bits are reserved for old ceph releases.
 #define CEPHFS_FEATURE_JEWEL                5
diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index cce9976f3c35..2bf483939a83 100644
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -482,8 +482,9 @@ namespace ceph {
       constexpr mon_feature_t FEATURE_PACIFIC(    (1ULL << 6));
       // elector pinging and CONNECTIVITY mode:
       constexpr mon_feature_t FEATURE_PINGING(    (1ULL << 7));
-      constexpr mon_feature_t FEATURE_QUINCY(    (1ULL << 8));
-      constexpr mon_feature_t FEATURE_REEF(    (1ULL << 9));
+      constexpr mon_feature_t FEATURE_QUINCY(     (1ULL << 8));
+      constexpr mon_feature_t FEATURE_REEF(       (1ULL << 9));
+      constexpr mon_feature_t FEATURE_SQUID(      (1ULL << 10));
 
       constexpr mon_feature_t FEATURE_RESERVED(   (1ULL << 63));
       constexpr mon_feature_t FEATURE_NONE(       (0ULL));
@@ -505,6 +506,7 @@ namespace ceph {
 	  FEATURE_PINGING |
 	  FEATURE_QUINCY |
 	  FEATURE_REEF |
+	  FEATURE_SQUID |
 	  FEATURE_NONE
 	  );
       }
@@ -530,6 +532,7 @@ namespace ceph {
 	  FEATURE_PINGING |
 	  FEATURE_QUINCY |
 	  FEATURE_REEF |
+	  FEATURE_SQUID |
 	  FEATURE_NONE
 	  );
       }
@@ -548,6 +551,9 @@ namespace ceph {
 
 static inline ceph_release_t infer_ceph_release_from_mon_features(mon_feature_t f)
 {
+  if (f.contains_all(ceph::features::mon::FEATURE_SQUID)) {
+    return ceph_release_t::squid;
+  }
   if (f.contains_all(ceph::features::mon::FEATURE_REEF)) {
     return ceph_release_t::reef;
   }
@@ -598,6 +604,8 @@ static inline const char *ceph::features::mon::get_feature_name(uint64_t b) {
     return "quincy";
   } else if (f == FEATURE_REEF) {
     return "reef";
+  } else if (f == FEATURE_SQUID) {
+    return "squid";
   } else if (f == FEATURE_RESERVED) {
     return "reserved";
   }
@@ -626,6 +634,8 @@ inline mon_feature_t ceph::features::mon::get_feature_by_name(const std::string
     return FEATURE_QUINCY;
   } else if (n == "reef") {
     return FEATURE_REEF;
+  } else if (n == "squid") {
+    return FEATURE_SQUID;
   } else if (n == "reserved") {
     return FEATURE_RESERVED;
   }

From 72a2301922eaa5516e7bd4f383eae13977dc21d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 9 May 2023 19:24:07 +0200
Subject: [PATCH 0875/2492] include/ceph_features, doc: define
 FEATURE_SERVER_SQUID
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I've departed from 35d483af5425eb08593bc1c3065cfaec99938244
in that way the _incarnation_ is selected to be the very next
one to the retired bit's one.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 src/include/ceph_features.h    | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 938d6e3e343f..cafbc71987c8 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -63,7 +63,7 @@ Docs
 Feature bits
 ------------
 
-- [ ] ensure that `SERVER_X` is defined
+- [x] ensure that `SERVER_X` is defined
 - [ ] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
 - [ ] look for features that (1) were present in X-2 and (2) have no
   client dependency and mark them `DEPRECATED` as of X.
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 794e10efd7a8..1937eeb4c698 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -157,7 +157,7 @@ DEFINE_CEPH_FEATURE(47, 1, MDS_QUOTA)        // 4.17
 DEFINE_CEPH_FEATURE(48, 1, CRUSH_V4)         // 4.1
 DEFINE_CEPH_FEATURE_RETIRED(49, 1, OSD_MIN_SIZE_RECOVERY, JEWEL, LUMINOUS)
 DEFINE_CEPH_FEATURE_RETIRED(49, 1, OSD_PROXY_FEATURES, JEWEL, LUMINOUS) // overlap
-// available
+DEFINE_CEPH_FEATURE(49, 2, SERVER_SQUID);
 DEFINE_CEPH_FEATURE_RETIRED(50, 1, MON_METADATA, MIMIC, OCTOPUS)
 // available
 DEFINE_CEPH_FEATURE_RETIRED(51, 1, OSD_BITWISE_HOBJ_SORT, MIMIC, OCTOPUS)
@@ -252,6 +252,7 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATUREMASK_SERVER_QUINCY | \
 	 CEPH_FEATURE_RANGE_BLOCKLIST | \
 	 CEPH_FEATUREMASK_SERVER_REEF | \
+	 CEPH_FEATUREMASK_SERVER_SQUID | \
 	 0ULL)
 
 #define CEPH_FEATURES_SUPPORTED_DEFAULT  CEPH_FEATURES_ALL

From adc05c510a387c7711078ed18bf2d060dfed4b5f Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 10 May 2023 09:57:46 +0000
Subject: [PATCH 0876/2492] tools/monmaptool, test/cli, doc: pacific to quincy
 for default min_mon_release

This is modelled after 4210e0a95e267d6297f53903ebd143108b134434.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst                   | 2 +-
 src/test/cli/monmaptool/add-exists.t             | 4 ++--
 src/test/cli/monmaptool/add-many.t               | 4 ++--
 src/test/cli/monmaptool/clobber.t                | 8 ++++----
 src/test/cli/monmaptool/create-print.t           | 6 +++---
 src/test/cli/monmaptool/create-with-add.t        | 4 ++--
 src/test/cli/monmaptool/feature-set-unset-list.t | 2 +-
 src/test/cli/monmaptool/rm-nonexistent.t         | 4 ++--
 src/test/cli/monmaptool/rm.t                     | 4 ++--
 src/tools/monmaptool.cc                          | 2 +-
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index cafbc71987c8..cb6c744f2355 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -50,7 +50,7 @@ Scripts
 Misc
 ~~~~
 - [ ] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
-- [ ] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
+- [x] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
 - [ ] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
 
 Docs
diff --git a/src/test/cli/monmaptool/add-exists.t b/src/test/cli/monmaptool/add-exists.t
index c51d9fb80cb0..28f019aead69 100644
--- a/src/test/cli/monmaptool/add-exists.t
+++ b/src/test/cli/monmaptool/add-exists.t
@@ -1,7 +1,7 @@
   $ monmaptool --create mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (0 monitors)
 
   $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -21,7 +21,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
   0: v1:2.3.4.5:6789/0 mon.foo
 
diff --git a/src/test/cli/monmaptool/add-many.t b/src/test/cli/monmaptool/add-many.t
index 4118d46afd8f..4cd2ae74988f 100644
--- a/src/test/cli/monmaptool/add-many.t
+++ b/src/test/cli/monmaptool/add-many.t
@@ -1,7 +1,7 @@
   $ monmaptool --create mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (0 monitors)
 
   $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -24,7 +24,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
   0: v1:2.3.4.5:6789/0 mon.foo
   1: [v2:172.21.15.68:6791/0,v1:172.21.15.68:6792/0] mon.fiz
diff --git a/src/test/cli/monmaptool/clobber.t b/src/test/cli/monmaptool/clobber.t
index 2fb45ad616df..7b40bfd594ab 100644
--- a/src/test/cli/monmaptool/clobber.t
+++ b/src/test/cli/monmaptool/clobber.t
@@ -1,7 +1,7 @@
   $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (1 monitors)
 
   $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -18,7 +18,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
   0: v1:2.3.4.5:6789/0 mon.foo
 
@@ -28,7 +28,7 @@
   $ monmaptool --create --clobber mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (0 monitors)
 
   $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -40,5 +40,5 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
diff --git a/src/test/cli/monmaptool/create-print.t b/src/test/cli/monmaptool/create-print.t
index 31cca02427d3..88531b0eb18d 100644
--- a/src/test/cli/monmaptool/create-print.t
+++ b/src/test/cli/monmaptool/create-print.t
@@ -1,7 +1,7 @@
   $ monmaptool --create mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (0 monitors)
 
   $ monmaptool --print mymonmap
@@ -10,7 +10,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
 
   $ monmaptool --print -- mymonmap
@@ -19,5 +19,5 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
diff --git a/src/test/cli/monmaptool/create-with-add.t b/src/test/cli/monmaptool/create-with-add.t
index b32a7b35caf8..1e364c318b11 100644
--- a/src/test/cli/monmaptool/create-with-add.t
+++ b/src/test/cli/monmaptool/create-with-add.t
@@ -1,7 +1,7 @@
   $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (1 monitors)
 
   $ monmaptool --print mymonmap
@@ -10,6 +10,6 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
   0: v1:2.3.4.5:6789/0 mon.foo
diff --git a/src/test/cli/monmaptool/feature-set-unset-list.t b/src/test/cli/monmaptool/feature-set-unset-list.t
index 6a0c8a17a6f2..5f9d3859d539 100644
--- a/src/test/cli/monmaptool/feature-set-unset-list.t
+++ b/src/test/cli/monmaptool/feature-set-unset-list.t
@@ -1,7 +1,7 @@
   $ monmaptool --create --add a 10.10.10.10:1234 /tmp/test.monmap.1234
   monmaptool: monmap file /tmp/test.monmap.1234
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ monmaptool --feature-list --feature-list plain --feature-list parseable /tmp/test.monmap.1234
diff --git a/src/test/cli/monmaptool/rm-nonexistent.t b/src/test/cli/monmaptool/rm-nonexistent.t
index 165dbb398bb0..805c91f00a6d 100644
--- a/src/test/cli/monmaptool/rm-nonexistent.t
+++ b/src/test/cli/monmaptool/rm-nonexistent.t
@@ -1,7 +1,7 @@
   $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (1 monitors)
 
   $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -19,7 +19,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
   0: v1:2.3.4.5:6789/0 mon.foo
 
diff --git a/src/test/cli/monmaptool/rm.t b/src/test/cli/monmaptool/rm.t
index 4a693d59aaec..3b2a27518e53 100644
--- a/src/test/cli/monmaptool/rm.t
+++ b/src/test/cli/monmaptool/rm.t
@@ -1,7 +1,7 @@
   $ monmaptool --create --add foo 2.3.4.5:6789 mymonmap
   monmaptool: monmap file mymonmap
   monmaptool: generated fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
-  setting min_mon_release = pacific
+  setting min_mon_release = quincy
   monmaptool: writing epoch 0 to mymonmap (1 monitors)
 
   $ ORIG_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
@@ -17,7 +17,7 @@
   fsid [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} (re)
   last_changed \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
   created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re)
-  min_mon_release 16 (pacific)
+  min_mon_release 17 (quincy)
   election_strategy: 1
 
   $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)"
diff --git a/src/tools/monmaptool.cc b/src/tools/monmaptool.cc
index b858fe5a684a..f1b86e00362c 100644
--- a/src/tools/monmaptool.cc
+++ b/src/tools/monmaptool.cc
@@ -358,7 +358,7 @@ int main(int argc, const char **argv)
     monmap.strategy = static_cast<MonMap::election_strategy>(
 		  g_conf().get_val<uint64_t>("mon_election_default_strategy"));
     if (min_mon_release == ceph_release_t::unknown) {
-      min_mon_release = ceph_release_t::pacific;
+      min_mon_release = ceph_release_t::quincy;
     }
     // TODO: why do we not use build_initial in our normal path here!?!?!
     modified = true;

From b6986466f72ed55016bb6a675f9aa65462252f9d Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 13:51:02 +0000
Subject: [PATCH 0877/2492] cephadm: update DEFAULT_IMAGE_RELEASE to squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst      | 2 +-
 src/cephadm/cephadmlib/constants.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index cb6c744f2355..7462ef2823d5 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -51,7 +51,7 @@ Misc
 ~~~~
 - [ ] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
 - [x] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
-- [ ] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
+- [x] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
 
 Docs
 ~~~~
diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index e5c03adffb7b..0de2dc8ceb0e 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -3,7 +3,7 @@
 # Default container images -----------------------------------------------------
 DEFAULT_IMAGE = 'quay.ceph.io/ceph-ci/ceph:main'
 DEFAULT_IMAGE_IS_MAIN = True
-DEFAULT_IMAGE_RELEASE = 'reef'
+DEFAULT_IMAGE_RELEASE = 'squid'
 DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.43.0'
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'

From da25e58dc1bc268431f1666e8f322fb19ae8d04b Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:00:15 +0000
Subject: [PATCH 0878/2492] doc, qa/standalone/mon/misc: verify that
 len(monmap.features.persistent) == 11

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 qa/standalone/mon/misc.sh      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 7462ef2823d5..65ea864f2dd1 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -82,7 +82,7 @@ Mon
 ---
 
 - [ ] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
-- [ ] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
+- [x] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
 - [ ] mon/MgrMonitor.cc adjust `always_on_modules`
 - [ ] common/options/global.yaml.in define `mon_debug_no_require_X`
 - [ ] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
diff --git a/qa/standalone/mon/misc.sh b/qa/standalone/mon/misc.sh
index c7fc6d441171..6be79c475e6f 100755
--- a/qa/standalone/mon/misc.sh
+++ b/qa/standalone/mon/misc.sh
@@ -265,7 +265,7 @@ function TEST_mon_features() {
     jq_success "$jqinput" "$jqfilter" "pacific" || return 1
     jqfilter='.monmap.features.persistent[]|select(. == "elector-pinging")'
     jq_success "$jqinput" "$jqfilter" "elector-pinging" || return 1
-    jqfilter='.monmap.features.persistent | length == 10'
+    jqfilter='.monmap.features.persistent | length == 11'
     jq_success "$jqinput" "$jqfilter" || return 1
     jqfilter='.monmap.features.persistent[]|select(. == "quincy")'
     jq_success "$jqinput" "$jqfilter" "quincy" || return 1

From 2963ca6eb804e550578d310c15e201869dfefe39 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:08:37 +0000
Subject: [PATCH 0879/2492] doc, mon: add always_on_modules for squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst |  2 +-
 src/mon/MgrMonitor.cc          | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 65ea864f2dd1..6bcb31814ac2 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -83,7 +83,7 @@ Mon
 
 - [ ] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
 - [x] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
-- [ ] mon/MgrMonitor.cc adjust `always_on_modules`
+- [x] mon/MgrMonitor.cc adjust `always_on_modules`
 - [ ] common/options/global.yaml.in define `mon_debug_no_require_X`
 - [ ] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
 - [ ] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index 32739526c9fb..a1a2c256df83 100644
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -127,6 +127,20 @@ const static std::map<uint32_t, std::set<std::string>> always_on_modules = {
       "telemetry",
     }
   },
+  {
+    CEPH_RELEASE_SQUID, {
+      "crash",
+      "status",
+      "progress",
+      "balancer",
+      "devicehealth",
+      "orchestrator",
+      "rbd_support",
+      "volumes",
+      "pg_autoscaler",
+      "telemetry",
+    }
+  },
 };
 
 // Prefix for mon store of active mgr's command descriptions

From 0d18c1f67ff39f82cac43afc08d71be7620a8f26 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:15:02 +0000
Subject: [PATCH 0880/2492] include/ceph_features: retire deprecated bits

Nothing has been deprecated in Mimic, so this commit is just
updates the release checklist.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 6bcb31814ac2..f073c5ac03e1 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -64,7 +64,7 @@ Feature bits
 ------------
 
 - [x] ensure that `SERVER_X` is defined
-- [ ] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
+- [x] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
 - [ ] look for features that (1) were present in X-2 and (2) have no
   client dependency and mark them `DEPRECATED` as of X.
 

From 93a898ea6c45abc6f80f6b6ae5f519376b7ab37c Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:17:42 +0000
Subject: [PATCH 0881/2492] ceph-volume: change to squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst          | 2 +-
 src/ceph-volume/ceph_volume/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index f073c5ac03e1..85851999bdfa 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -49,7 +49,7 @@ Scripts
 
 Misc
 ~~~~
-- [ ] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
+- [x] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
 - [x] update src/tools/monmaptool.cc (`min_mon_release` and corresponding output in `src/test/cli/monmaptool`)
 - [x] update src/cephadm/cephadmlib/constants.py (`DEFAULT_IMAGE_RELEASE` to X)
 
diff --git a/src/ceph-volume/ceph_volume/__init__.py b/src/ceph-volume/ceph_volume/__init__.py
index dad83c95bb7a..500de2237365 100644
--- a/src/ceph-volume/ceph_volume/__init__.py
+++ b/src/ceph-volume/ceph_volume/__init__.py
@@ -19,4 +19,4 @@ def __getattr__(self, *a):
 
 __version__ = "1.0.0"
 
-__release__ = "reef"
+__release__ = "squid"

From b23a134baf2fab0f876b7ddc71a9fb8f78c58a88 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:22:31 +0000
Subject: [PATCH 0882/2492] doc/dev/release-checklists: mark
 backport-resolve-issue is already done

This happened in 3d15f685a88f5d10e3e8c023e9eb15bb4800bed5.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 85851999bdfa..2d6eb5c026f1 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -43,7 +43,7 @@ Make sure X (and, ideally, X+1) is defined:
 Scripts
 ~~~~~~~
 
-- [ ] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
+- [x] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
 - [ ] src/script/ceph-release-notes (X-1)
 - [ ] ceph-build.git scripts/build_utils.sh `release_from_version()`
 

From c9f9174b3274ec5354835e3dac3e186e443d9ed6 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:24:44 +0000
Subject: [PATCH 0883/2492] doc/dev/release-checklists: mark ceph-release-notes
 is already done

This happened in 5d944bb237cb92ea2b13d668a78b939438836574.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 2d6eb5c026f1..dd92c551437e 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -44,7 +44,7 @@ Scripts
 ~~~~~~~
 
 - [x] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
-- [ ] src/script/ceph-release-notes (X-1)
+- [x] src/script/ceph-release-notes (X-1)
 - [ ] ceph-build.git scripts/build_utils.sh `release_from_version()`
 
 Misc

From f44c70236e0009b4886ea6f7031651215687553c Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 21 Aug 2023 14:33:58 +0000
Subject: [PATCH 0884/2492] mon: update for squid ondisk features

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst |  8 ++++----
 src/mon/Monitor.cc             | 11 +++++++++++
 src/mon/Monitor.h              |  1 +
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index dd92c551437e..01d1d0589fea 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -72,10 +72,10 @@ Feature bits
 Compatsets
 ----------
 
-- [ ] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
-- [ ] mon/Monitor.cc (include in `get_supported_features()`)
-- [ ] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
-- [ ] mon/Monitor.cc (`calc_quorum_requirements()`)
+- [x] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
+- [x] mon/Monitor.cc (include in `get_supported_features()`)
+- [x] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
+- [x] mon/Monitor.cc (`calc_quorum_requirements()`)
 - [ ] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
 
 Mon
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 27151e60b220..5543ff316fff 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -534,6 +534,7 @@ CompatSet Monitor::get_supported_features()
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_PACIFIC);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_QUINCY);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_REEF);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SQUID);
   return compat;
 }
 
@@ -2512,6 +2513,13 @@ void Monitor::apply_monmap_to_compatset_features()
     ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_REEF));
     new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_REEF);
   }
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_SQUID)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_SQUID));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, SERVER_SQUID));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SQUID);
+  }
 
   dout(5) << __func__ << dendl;
   _apply_compatset_features(new_features);
@@ -2550,6 +2558,9 @@ void Monitor::calc_quorum_requirements()
   if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_REEF)) {
     required_features |= CEPH_FEATUREMASK_SERVER_REEF;
   }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_SQUID)) {
+    required_features |= CEPH_FEATUREMASK_SERVER_SQUID;
+  }
 
   // monmap
   if (monmap->get_required_features().contains_all(
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 7f9a16a9a36c..ed1e2a4e8a95 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -1116,6 +1116,7 @@ class Monitor : public Dispatcher,
 #define CEPH_MON_FEATURE_INCOMPAT_PACIFIC CompatSet::Feature(13, "pacific ondisk layout")
 #define CEPH_MON_FEATURE_INCOMPAT_QUINCY CompatSet::Feature(14, "quincy ondisk layout")
 #define CEPH_MON_FEATURE_INCOMPAT_REEF CompatSet::Feature(15, "reef ondisk layout")
+#define CEPH_MON_FEATURE_INCOMPAT_SQUID CompatSet::Feature(16, "squid ondisk layout")
 // make sure you add your feature to Monitor::get_supported_features
 
 

From dea8aa67c2f9301fa5643fab4baee750b23fb524 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 22 Aug 2023 10:23:10 +0000
Subject: [PATCH 0885/2492] common, mon, qa: Mon-related updates for squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst    | 16 ++++----
 qa/standalone/mon/misc.sh         | 12 ++++--
 qa/workunits/cephtool/test.sh     |  4 +-
 src/common/options/global.yaml.in |  8 ++--
 src/mon/MonCommands.h             |  2 +-
 src/mon/OSDMonitor.cc             | 62 +++++++++++++++----------------
 6 files changed, 55 insertions(+), 49 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 01d1d0589fea..8eef935948c9 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -81,16 +81,16 @@ Compatsets
 Mon
 ---
 
-- [ ] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
+- [x] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
 - [x] qa/standalone/mon/misc bump up `jqfilter='.monmap.features.persistent | length == N'` to `N+1`
 - [x] mon/MgrMonitor.cc adjust `always_on_modules`
-- [ ] common/options/global.yaml.in define `mon_debug_no_require_X`
-- [ ] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
-- [ ] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
-- [ ] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
-- [ ] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
-- [ ] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
-- [ ] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
+- [x] common/options/global.yaml.in define `mon_debug_no_require_X`
+- [x] common/options/global.yaml.in remove `mon_debug_no_require_X-2`
+- [x] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
+- [x] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
+- [x] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
+- [x] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
+- [x] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
 
 
 Code cleanup
diff --git a/qa/standalone/mon/misc.sh b/qa/standalone/mon/misc.sh
index 6be79c475e6f..4bb80711f60c 100755
--- a/qa/standalone/mon/misc.sh
+++ b/qa/standalone/mon/misc.sh
@@ -171,7 +171,7 @@ function TEST_mon_features() {
     CEPH_ARGS="--fsid=$fsid --auth-supported=none "
     CEPH_ARGS+="--mon-host=$MONA,$MONB,$MONC "
     CEPH_ARGS+="--mon-debug-no-initial-persistent-features "
-    CEPH_ARGS+="--mon-debug-no-require-reef "
+    CEPH_ARGS+="--mon-debug-no-require-squid "
 
     run_mon $dir a --public-addr $MONA || return 1
     run_mon $dir b --public-addr $MONB || return 1
@@ -183,7 +183,7 @@ function TEST_mon_features() {
     # quorum contains two monitors
     jq_success "$jqinput" '.quorum | length == 2' || return 1
     # quorum's monitor features contain kraken, luminous, mimic, nautilus,
-    # octopus, pacific, quincy
+    # octopus, pacific, quincy, reef
     jqfilter='.features.quorum_mon[]|select(. == "kraken")'
     jq_success "$jqinput" "$jqfilter" "kraken" || return 1
     jqfilter='.features.quorum_mon[]|select(. == "luminous")'
@@ -200,6 +200,8 @@ function TEST_mon_features() {
     jq_success "$jqinput" "$jqfilter" "quincy" || return 1
     jqfilter='.features.quorum_mon[]|select(. == "reef")'
     jq_success "$jqinput" "$jqfilter" "reef" || return 1
+    jqfilter='.features.quorum_mon[]|select(. == "squid")'
+    jq_success "$jqinput" "$jqfilter" "squid" || return 1
 
     # monmap must have no persistent features set, because we
     # don't currently have a quorum made out of all the monitors
@@ -214,7 +216,7 @@ function TEST_mon_features() {
     # validate 'mon feature ls'
 
     jqinput="$(ceph mon feature ls --format=json 2>/dev/null)"
-    # k l m n o p q are supported
+    # k l m n o p q r are supported
     jqfilter='.all.supported[] | select(. == "kraken")'
     jq_success "$jqinput" "$jqfilter" "kraken" || return 1
     jqfilter='.all.supported[] | select(. == "luminous")'
@@ -231,6 +233,8 @@ function TEST_mon_features() {
     jq_success "$jqinput" "$jqfilter" "quincy" || return 1
     jqfilter='.all.supported[] | select(. == "reef")'
     jq_success "$jqinput" "$jqfilter" "reef" || return 1
+    jqfilter='.all.supported[] | select(. == "squid")'
+    jq_success "$jqinput" "$jqfilter" "squid" || return 1
 
     # start third monitor
     run_mon $dir c --public-addr $MONC || return 1
@@ -271,6 +275,8 @@ function TEST_mon_features() {
     jq_success "$jqinput" "$jqfilter" "quincy" || return 1
     jqfilter='.monmap.features.persistent[]|select(. == "reef")'
     jq_success "$jqinput" "$jqfilter" "reef" || return 1
+    jqfilter='.monmap.features.persistent[]|select(. == "squid")'
+    jq_success "$jqinput" "$jqfilter" "squid" || return 1
 
     CEPH_ARGS=$CEPH_ARGS_orig
     # that's all folks. thank you for tuning in.
diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index aecfd56a9f53..6dc2ee9513c9 100755
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -1521,10 +1521,10 @@ function test_mon_osd()
 	expect_false ceph osd set $f
 	expect_false ceph osd unset $f
   done
-  ceph osd require-osd-release reef
+  ceph osd require-osd-release squid
   # can't lower
+  expect_false ceph osd require-osd-release reef
   expect_false ceph osd require-osd-release quincy
-  expect_false ceph osd require-osd-release pacific
   # these are no-ops but should succeed.
 
   ceph osd set noup
diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 48c6788a88b3..286a1126a467 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -1961,19 +1961,19 @@ options:
   see_also:
   - mon_debug_dump_transactions
   with_legacy: true
-- name: mon_debug_no_require_quincy
+- name: mon_debug_no_require_reef
   type: bool
   level: dev
-  desc: do not set quincy feature for new mon clusters
+  desc: do not set reef feature for new mon clusters
   default: false
   services:
   - mon
   flags:
   - cluster_create
-- name: mon_debug_no_require_reef
+- name: mon_debug_no_require_squid
   type: bool
   level: dev
-  desc: do not set reef feature for new mon clusters
+  desc: do not set squid feature for new mon clusters
   default: false
   services:
   - mon
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 71a8bec76ac0..7ebd57c8c9ce 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -860,7 +860,7 @@ COMMAND("osd unset "
 	"notieragent|nosnaptrim|noautoscale",
 	"unset <key>", "osd", "rw")
 COMMAND("osd require-osd-release "\
-	"name=release,type=CephChoices,strings=octopus|pacific|quincy|reef "
+	"name=release,type=CephChoices,strings=octopus|pacific|quincy|reef|squid "
         "name=yes_i_really_mean_it,type=CephBool,req=false",
 	"set the minimum allowed OSD release to participate in the cluster",
 	"osd", "rw")
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 79fff068c332..1b513bffd378 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -674,16 +674,16 @@ void OSDMonitor::create_initial()
   if (newmap.nearfull_ratio > 1.0) newmap.nearfull_ratio /= 100;
 
   // new cluster should require latest by default
-  if (g_conf().get_val<bool>("mon_debug_no_require_reef")) {
-    if (g_conf().get_val<bool>("mon_debug_no_require_quincy")) {
-      derr << __func__ << " mon_debug_no_require_reef and quincy=true" << dendl;
-      newmap.require_osd_release = ceph_release_t::pacific;
-    } else {
-      derr << __func__ << " mon_debug_no_require_reef=true" << dendl;
+  if (g_conf().get_val<bool>("mon_debug_no_require_squid")) {
+    if (g_conf().get_val<bool>("mon_debug_no_require_reef")) {
+      derr << __func__ << " mon_debug_no_require_squid and reef=true" << dendl;
       newmap.require_osd_release = ceph_release_t::quincy;
+    } else {
+      derr << __func__ << " mon_debug_no_require_squid=true" << dendl;
+      newmap.require_osd_release = ceph_release_t::reef;
     }
   } else {
-    newmap.require_osd_release = ceph_release_t::reef;
+    newmap.require_osd_release = ceph_release_t::squid;
   }
 
   ceph_release_t r = ceph_release_from_name(g_conf()->mon_osd_initial_require_min_compat_client);
@@ -3486,21 +3486,14 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
   ceph_assert(m->get_orig_source_inst().name.is_osd());
 
   // lower bound of N-2
-  if (!HAVE_FEATURE(m->osd_features, SERVER_PACIFIC)) {
+  if (!HAVE_FEATURE(m->osd_features, SERVER_QUINCY)) {
     mon.clog->info() << "disallowing boot of OSD "
 		     << m->get_orig_source_inst()
-		     << " because the osd lacks CEPH_FEATURE_SERVER_PACIFIC";
+		     << " because the osd lacks CEPH_FEATURE_SERVER_QUINCY";
     goto ignore;
   }
 
   // make sure osd versions do not span more than 3 releases
-  if (HAVE_FEATURE(m->osd_features, SERVER_QUINCY) &&
-      osdmap.require_osd_release < ceph_release_t::octopus) {
-    mon.clog->info() << "disallowing boot of quincy+ OSD "
-		      << m->get_orig_source_inst()
-		      << " because require_osd_release < octopus";
-    goto ignore;
-  }
   if (HAVE_FEATURE(m->osd_features, SERVER_REEF) &&
       osdmap.require_osd_release < ceph_release_t::pacific) {
     mon.clog->info() << "disallowing boot of reef+ OSD "
@@ -3508,6 +3501,13 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
 		      << " because require_osd_release < pacific";
     goto ignore;
   }
+  if (HAVE_FEATURE(m->osd_features, SERVER_SQUID) &&
+      osdmap.require_osd_release < ceph_release_t::quincy) {
+    mon.clog->info() << "disallowing boot of squid+ OSD "
+		      << m->get_orig_source_inst()
+		      << " because require_osd_release < quincy";
+    goto ignore;
+  }
 
   // See crimson/osd/osd.cc: OSD::_send_boot
   if (auto type_iter = m->metadata.find("osd_type");
@@ -11669,7 +11669,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       err = 0;
       goto reply_no_propose;
     }
-    if (osdmap.require_osd_release < ceph_release_t::pacific && !sure) {
+    if (osdmap.require_osd_release < ceph_release_t::quincy && !sure) {
       ss << "Not advisable to continue since current 'require_osd_release' "
          << "refers to a very old Ceph release. Pass "
 	 << "--yes-i-really-mean-it if you really wish to continue.";
@@ -11682,20 +11682,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       err = -EPERM;
       goto reply_no_propose;
     }
-    if (rel == ceph_release_t::pacific) {
-      if (!mon.monmap->get_required_features().contains_all(
-	    ceph::features::mon::FEATURE_PACIFIC)) {
-	ss << "not all mons are pacific";
-	err = -EPERM;
-	goto reply_no_propose;
-      }
-      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_PACIFIC))
-           && !sure) {
-	ss << "not all up OSDs have CEPH_FEATURE_SERVER_PACIFIC feature";
-	err = -EPERM;
-	goto reply_no_propose;
-      }
-    } else if (rel == ceph_release_t::quincy) {
+    if (rel == ceph_release_t::quincy) {
       if (!mon.monmap->get_required_features().contains_all(
 	    ceph::features::mon::FEATURE_QUINCY)) {
 	ss << "not all mons are quincy";
@@ -11721,6 +11708,19 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 	err = -EPERM;
 	goto reply_no_propose;
       }
+    } else if (rel == ceph_release_t::squid) {
+      if (!mon.monmap->get_required_features().contains_all(
+	    ceph::features::mon::FEATURE_SQUID)) {
+	ss << "not all mons are squid";
+	err = -EPERM;
+	goto reply_no_propose;
+      }
+      if ((!HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_SQUID))
+           && !sure) {
+	ss << "not all up OSDs have CEPH_FEATURE_SERVER_SQUID feature";
+	err = -EPERM;
+	goto reply_no_propose;
+      }
     } else {
       ss << "not supported for this release";
       err = -EPERM;

From 081177f6a44972db232f6d180251b7949ac1432b Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 22 Aug 2023 10:29:51 +0000
Subject: [PATCH 0886/2492] qa: stubs and clean up for reef

- remove upgrades from octopus
- stubs for completing upgrade to reef

Still missing the quincy-x upgrade tests.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst                |   6 +-
 qa/releases/squid.yaml                        |   6 +
 .../thrash-old-clients/1-install/reef.yaml    |  35 +++++
 qa/suites/upgrade/pacific-x/.qa               |   1 -
 qa/suites/upgrade/pacific-x/parallel/%        |   0
 qa/suites/upgrade/pacific-x/parallel/.qa      |   1 -
 .../pacific-x/parallel/0-random-distro$       |   1 -
 .../upgrade/pacific-x/parallel/0-start.yaml   |  33 -----
 .../upgrade/pacific-x/parallel/1-tasks.yaml   |  43 ------
 .../upgrade/pacific-x/parallel/mon_election   |   1 -
 .../pacific-x/parallel/upgrade-sequence.yaml  |  16 ---
 .../upgrade/pacific-x/parallel/workload/+     |   0
 .../upgrade/pacific-x/parallel/workload/.qa   |   1 -
 .../parallel/workload/ec-rados-default.yaml   |  25 ----
 .../parallel/workload/rados_api.yaml          |  12 --
 .../parallel/workload/rados_loadgenbig.yaml   |  12 --
 .../parallel/workload/rbd_import_export.yaml  |  14 --
 .../parallel/workload/test_rbd_api.yaml       |  14 --
 .../parallel/workload/test_rbd_python.yaml    |  20 ---
 qa/suites/upgrade/pacific-x/stress-split/%    |   0
 qa/suites/upgrade/pacific-x/stress-split/.qa  |   1 -
 .../upgrade/pacific-x/stress-split/0-distro   |   1 -
 .../pacific-x/stress-split/0-roles.yaml       |  31 -----
 .../pacific-x/stress-split/1-start.yaml       | 122 ------------------
 .../stress-split/2-first-half-tasks/.qa       |   1 -
 .../2-first-half-tasks/radosbench.yaml        |  19 ---
 .../2-first-half-tasks/rbd-cls.yaml           |  10 --
 .../2-first-half-tasks/rbd-import-export.yaml |  12 --
 .../2-first-half-tasks/rbd_api.yaml           |  12 --
 .../2-first-half-tasks/readwrite.yaml         |  16 ---
 .../2-first-half-tasks/snaps-few-objects.yaml |  18 ---
 .../pacific-x/stress-split/3-stress-tasks/+   |   0
 .../pacific-x/stress-split/3-stress-tasks/.qa |   1 -
 .../3-stress-tasks/radosbench.yaml            |  25 ----
 .../stress-split/3-stress-tasks/rbd-cls.yaml  |  10 --
 .../3-stress-tasks/rbd-import-export.yaml     |  12 --
 .../stress-split/3-stress-tasks/rbd_api.yaml  |  12 --
 .../3-stress-tasks/readwrite.yaml             |  16 ---
 .../3-stress-tasks/snaps-few-objects.yaml     |  18 ---
 .../stress-split/4-second-half-tasks/.qa      |   1 -
 .../4-second-half-tasks/radosbench.yaml       |  16 ---
 .../rbd-import-export.yaml                    |  12 --
 .../pacific-x/stress-split/mon_election       |   1 -
 43 files changed, 44 insertions(+), 564 deletions(-)
 create mode 100644 qa/releases/squid.yaml
 create mode 100644 qa/suites/rados/thrash-old-clients/1-install/reef.yaml
 delete mode 120000 qa/suites/upgrade/pacific-x/.qa
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/%
 delete mode 120000 qa/suites/upgrade/pacific-x/parallel/.qa
 delete mode 120000 qa/suites/upgrade/pacific-x/parallel/0-random-distro$
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/0-start.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/1-tasks.yaml
 delete mode 120000 qa/suites/upgrade/pacific-x/parallel/mon_election
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/+
 delete mode 120000 qa/suites/upgrade/pacific-x/parallel/workload/.qa
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/ec-rados-default.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/rados_api.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/rados_loadgenbig.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/rbd_import_export.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_api.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_python.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/%
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/.qa
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/0-distro
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/0-roles.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/.qa
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/radosbench.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-cls.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-import-export.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd_api.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/readwrite.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/snaps-few-objects.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/+
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/.qa
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/radosbench.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-cls.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-import-export.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd_api.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/readwrite.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/snaps-few-objects.yaml
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/.qa
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/radosbench.yaml
 delete mode 100644 qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/rbd-import-export.yaml
 delete mode 120000 qa/suites/upgrade/pacific-x/stress-split/mon_election

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 8eef935948c9..2f83eff7f778 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -106,9 +106,9 @@ QA suite
 --------
 
 - [ ] create qa/suites/upgrade/(X-1)-x
-- [ ] remove qa/suites/upgrade/(X-3)-x-*
-- [ ] create qa/releases/X.yaml
-- [ ] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
+- [x] remove qa/suites/upgrade/(X-3)-x-*
+- [x] create qa/releases/X.yaml
+- [x] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
 
 
 ceph-build
diff --git a/qa/releases/squid.yaml b/qa/releases/squid.yaml
new file mode 100644
index 000000000000..a0474330f188
--- /dev/null
+++ b/qa/releases/squid.yaml
@@ -0,0 +1,6 @@
+tasks:
+- exec:
+    osd.0:
+      - ceph osd require-osd-release squid
+      - ceph osd set-require-min-compat-client squid
+- ceph.healthy:
diff --git a/qa/suites/rados/thrash-old-clients/1-install/reef.yaml b/qa/suites/rados/thrash-old-clients/1-install/reef.yaml
new file mode 100644
index 000000000000..b476eb7c4cd9
--- /dev/null
+++ b/qa/suites/rados/thrash-old-clients/1-install/reef.yaml
@@ -0,0 +1,35 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - \(MON_DOWN\)
+tasks:
+- install:
+    branch: reef
+    exclude_packages:
+      - ceph-mgr-dashboard
+      - ceph-mgr-diskprediction-local
+      - ceph-mgr-rook
+      - ceph-mgr-cephadm
+      - ceph-base-debuginfo
+      - ceph-common-debuginfo
+      - ceph-immutable-object-cache-debuginfo
+      - ceph-radosgw-debuginfo
+      - ceph-test-debuginfo
+      - ceph-base-debuginfo
+      - ceph-mgr-debuginfo
+      - ceph-mds-debuginfo
+      - ceph-mon-debuginfo
+      - ceph-osd-debuginfo
+      - ceph-fuse-debuginfo
+      - librados-devel-debuginfo
+      - libcephfs2-debuginfo
+      - librados2-debuginfo
+      - librbd1-debuginfo
+      - python3-cephfs-debuginfo
+      - python3-rados-debuginfo
+      - python3-rbd-debuginfo
+      - python3-rgw-debuginfo
+      - rbd-fuse-debuginfo
+      - rbd-mirror-debuginfo
+      - rbd-nbd-debuginfo
+      - ceph-volume
diff --git a/qa/suites/upgrade/pacific-x/.qa b/qa/suites/upgrade/pacific-x/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/parallel/% b/qa/suites/upgrade/pacific-x/parallel/%
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/pacific-x/parallel/.qa b/qa/suites/upgrade/pacific-x/parallel/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/parallel/0-random-distro$ b/qa/suites/upgrade/pacific-x/parallel/0-random-distro$
deleted file mode 120000
index 4b341719de40..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/0-random-distro$
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/parallel/0-start.yaml b/qa/suites/upgrade/pacific-x/parallel/0-start.yaml
deleted file mode 100644
index 3814ea3efdb5..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/0-start.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
-roles:
-- - mon.a
-  - mon.c
-  - mgr.y
-  - mds.a
-  - osd.0
-  - osd.1
-  - osd.2
-  - osd.3
-  - client.0
-  - node-exporter.a
-  - alertmanager.a
-- - mon.b
-  - mds.b
-  - mgr.x
-  - osd.4
-  - osd.5
-  - osd.6
-  - osd.7
-  - client.1
-  - prometheus.a
-  - grafana.a
-  - node-exporter.b
-openstack:
-- volumes: # attached to each instance
-    count: 4
-    size: 10 # GB
-overrides:
-  ceph:
-    create_rbd_pool: true
-    conf:
-      osd:
-        osd shutdown pgref assert: true
diff --git a/qa/suites/upgrade/pacific-x/parallel/1-tasks.yaml b/qa/suites/upgrade/pacific-x/parallel/1-tasks.yaml
deleted file mode 100644
index 064d0758d193..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/1-tasks.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-tasks:
-- install:
-    branch: pacific
-    exclude_packages:
-      - ceph-volume
-- print: "**** done install task..."
-- print: "**** done start installing pacific cephadm ..."
-- cephadm:
-    image: quay.ceph.io/ceph-ci/ceph:pacific
-    cephadm_branch: pacific
-    cephadm_git_url: https://github.com/ceph/ceph
-    conf:
-      osd:
-        #set config option for which cls modules are allowed to be loaded / used
-        osd_class_load_list: "*"
-        osd_class_default_list: "*"
-- print: "**** done end installing pacific cephadm ..."
-
-- print: "**** done start cephadm.shell ceph config set mgr..."
-- cephadm.shell:
-    mon.a:
-      - ceph config set mgr mgr/cephadm/use_repo_digest true --force
-- print: "**** done cephadm.shell ceph config set mgr..."
-
-- print: "**** done start telemetry pacific..."
-- workunit:
-    clients:
-      client.0:
-        - test_telemetry_pacific.sh
-- print: "**** done end telemetry pacific..."
-
-- print: "**** done start parallel"
-- parallel:
-    - workload
-    - upgrade-sequence
-- print: "**** done end parallel"
-
-- print: "**** done start telemetry x..."
-- workunit:
-    clients:
-      client.0:
-        - test_telemetry_pacific_x.sh
-- print: "**** done end telemetry x..."
diff --git a/qa/suites/upgrade/pacific-x/parallel/mon_election b/qa/suites/upgrade/pacific-x/parallel/mon_election
deleted file mode 120000
index 3f331e6214c4..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/mon_election
+++ /dev/null
@@ -1 +0,0 @@
-.qa/mon_election
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml b/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
deleted file mode 100644
index a3f0888da1d8..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/upgrade-sequence.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# renamed tasks: to upgrade-sequence:
-upgrade-sequence:
-   sequential:
-   - print: "**** done start upgrade, wait"
-   - cephadm.shell:
-       env: [sha1]
-       mon.a:
-         - ceph config set global log_to_journald false --force
-         - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-         - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
-         - ceph orch ps
-         - ceph versions
-         - ceph versions | jq -e '.overall | length == 1'
-         - ceph versions | jq -e '.overall | keys' | grep $sha1
-   - print: "**** done end upgrade, wait..."
-
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/+ b/qa/suites/upgrade/pacific-x/parallel/workload/+
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/.qa b/qa/suites/upgrade/pacific-x/parallel/workload/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/ec-rados-default.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/ec-rados-default.yaml
deleted file mode 100644
index 67a0f39c5da5..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/ec-rados-default.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-meta:
-- desc: |
-   run run randomized correctness test for rados operations
-   on an erasure-coded pool
-workload:
-  full_sequential:
-  - print: "**** done start ec-rados-default.yaml"
-  - rados:
-      clients: [client.0]
-      ops: 4000
-      objects: 50
-      ec_pool: true
-      write_append_excl: false
-      op_weights:
-        read: 100
-        write: 0
-        append: 100
-        delete: 50
-        snap_create: 50
-        snap_remove: 50
-        rollback: 50
-        copy_from: 50
-        setattr: 25
-        rmattr: 25
-  - print: "**** done end ec-rados-default.yaml"
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/rados_api.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/rados_api.yaml
deleted file mode 100644
index 1380a4016fbb..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/rados_api.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   object class functional tests
-workload:
-  full_sequential:
-    - print: "**** done start rados_api.yaml"
-    - workunit:
-        branch: pacific
-        clients:
-          client.0:
-            - cls
-    - print: "**** done end rados_api.yaml"
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/rados_loadgenbig.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/rados_loadgenbig.yaml
deleted file mode 100644
index f315b1579a0d..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/rados_loadgenbig.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   generate read/write load with rados objects ranging from 1MB to 25MB
-workload:
-  full_sequential:
-    - print: "**** done start rados_loadgenbig.yaml"
-    - workunit:
-        branch: pacific
-        clients:
-          client.0:
-            - rados/load-gen-big.sh
-    - print: "**** done end rados_loadgenbig.yaml"
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/rbd_import_export.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/rbd_import_export.yaml
deleted file mode 100644
index 20e74c176986..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/rbd_import_export.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-meta:
-- desc: |
-   run basic import/export cli tests for rbd
-workload:
-  full_sequential:
-    - print: "**** done start rbd_import_export.yaml"
-    - workunit:
-        branch: pacific
-        clients:
-          client.1:
-              - rbd/import_export.sh
-        env:
-          RBD_CREATE_ARGS: --new-format
-    - print: "**** done end rbd_import_export.yaml"
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_api.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_api.yaml
deleted file mode 100644
index 6a0242b91555..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_api.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-meta:
-- desc: |
-   librbd C and C++ api tests
-workload:
-  full_sequential:
-    - print: "**** done start test_rbd_api.yaml"
-    - workunit:
-        branch: pacific
-        clients:
-          client.0:
-              - rbd/test_librbd.sh
-        env:
-          RBD_FEATURES: "61"
-    - print: "**** done end test_rbd_api.yaml"
diff --git a/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_python.yaml b/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_python.yaml
deleted file mode 100644
index 8d1f0fd0f8bd..000000000000
--- a/qa/suites/upgrade/pacific-x/parallel/workload/test_rbd_python.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-meta:
-- desc: |
-   librbd python api tests
-overrides:
-  install:
-    ceph:
-      extra_system_packages:
-      - python3-pytest
-workload:
-  full_sequential:
-    - print: "**** done start test_rbd_python.yaml"
-    - workunit:
-        branch: pacific
-        clients:
-          client.0:
-            - rbd/test_librbd_python.sh
-        env:
-          RBD_FEATURES: "61"
-    - print: "**** done end test_rbd_python.yaml"
-
diff --git a/qa/suites/upgrade/pacific-x/stress-split/% b/qa/suites/upgrade/pacific-x/stress-split/%
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/pacific-x/stress-split/.qa b/qa/suites/upgrade/pacific-x/stress-split/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/stress-split/0-distro b/qa/suites/upgrade/pacific-x/stress-split/0-distro
deleted file mode 120000
index 4b341719de40..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/0-distro
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/stress-split/0-roles.yaml b/qa/suites/upgrade/pacific-x/stress-split/0-roles.yaml
deleted file mode 100644
index ad3ee43d38e4..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/0-roles.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-roles:
-- - mon.a
-  - mon.c
-  - mgr.y
-  - osd.0
-  - osd.1
-  - osd.2
-  - osd.3
-  - client.0
-  - node-exporter.a
-  - alertmanager.a
-- - mon.b
-  - mgr.x
-  - osd.4
-  - osd.5
-  - osd.6
-  - osd.7
-  - client.1
-  - prometheus.a
-  - grafana.a
-  - node-exporter.b
-openstack:
-- volumes: # attached to each instance
-    count: 4
-    size: 10 # GB
-overrides:
-  ceph:
-    create_rbd_pool: true
-    conf:
-      osd:
-        osd shutdown pgref assert: true
diff --git a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml b/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
deleted file mode 100644
index 9a552df9991f..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/1-start.yaml
+++ /dev/null
@@ -1,122 +0,0 @@
-tasks:
-- install:
-    branch: pacific
-    exclude_packages:
-      - ceph-volume
-
-- cephadm:
-    image: quay.ceph.io/ceph-ci/ceph:pacific
-    cephadm_branch: pacific
-    cephadm_git_url: https://github.com/ceph/ceph
-    conf:
-      osd:
-        #set config option for which cls modules are allowed to be loaded / used
-        osd_class_load_list: "*"
-        osd_class_default_list: "*"
-
-- cephadm.shell:
-    mon.a:
-      - ceph fs volume create foo
-      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
-      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
-
-- ceph.healthy:
-
-- print: "**** upgrading first half of cluster, with stress ****"
-- parallel:
-    - first-half-tasks
-    - first-half-sequence
-- print: "**** done upgrading first half of cluster ****"
-
-- ceph.healthy:
-
-- print: "**** applying stress + thrashing to mixed-version cluster ****"
-
-- parallel:
-    - stress-tasks
-
-- ceph.healthy:
-
-- print: "**** finishing upgrade ****"
-- parallel:
-    - second-half-tasks
-    - second-half-sequence
-
-- ceph.healthy:
-
-
-#################
-
-first-half-sequence:
-- cephadm.shell:
-    env: [sha1]
-    mon.a:
-      - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60
-      - ceph config set global log_to_journald false --force
-
-      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-      - ceph orch ps
-
-      - echo wait for minority of mons to upgrade
-      - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done
-      - ceph orch ps
-      - ceph orch upgrade pause
-      - sleep 60
-      - ceph orch upgrade resume
-
-      - echo wait for majority of mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | egrep ': [23]' ; do sleep 2 ; done"
-      - ceph orch ps
-      - ceph orch upgrade pause
-      - sleep 60
-      - ceph orch upgrade resume
-
-      - echo wait for all mons to upgrade
-      - "while ! ceph mon versions | grep $sha1 | grep ': 3' ; do sleep 2 ; done"
-      - ceph orch ps
-      - ceph orch upgrade pause
-      - sleep 60
-      - ceph orch upgrade resume
-
-      - echo wait for half of osds to upgrade
-      - "while ! ceph osd versions | grep $sha1 | egrep ': [45678]'; do sleep 2 ; done"
-      - ceph orch upgrade pause
-      - ceph orch ps
-
-      - ceph orch ps
-      - ceph versions
-
-
-#################
-
-stress-tasks:
-- thrashosds:
-    timeout: 1200
-    chance_pgnum_grow: 1
-    chance_pgpnum_fix: 1
-    chance_thrash_cluster_full: 0
-    chance_thrash_pg_upmap: 0
-    chance_thrash_pg_upmap_items: 0
-    disable_objectstore_tool_tests: true
-    chance_force_recovery: 0
-    aggressive_pg_num_changes: false
-
-
-#################
-
-second-half-sequence:
-  sequential:
-    - cephadm.shell:
-        env: [sha1]
-        mon.a:
-          - ceph orch upgrade resume
-          - sleep 60
-
-          - echo wait for upgrade to complete
-          - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done
-
-          - echo upgrade complete
-          - ceph orch ps
-          - ceph versions
-          - ceph versions | jq -e '.overall | length == 1'
-          - ceph versions | jq -e '.overall | keys' | grep $sha1
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/.qa b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/radosbench.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/radosbench.yaml
deleted file mode 100644
index 3816ca38c985..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/radosbench.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-meta:
-- desc: |
-   run randomized correctness test for rados operations
-   generate write load with rados bench
-first-half-tasks:
-- full_sequential:
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-- print: "**** done end radosbench.yaml"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-cls.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-cls.yaml
deleted file mode 100644
index 4ebc0931045e..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-cls.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-meta:
-- desc: |
-   run basic cls tests for rbd
-first-half-tasks:
-- workunit:
-    branch: pacific
-    clients:
-      client.0:
-        - cls/test_cls_rbd.sh
-- print: "**** done cls/test_cls_rbd.sh 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-import-export.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-import-export.yaml
deleted file mode 100644
index 6835c9125fe7..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd-import-export.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   run basic import/export cli tests for rbd
-first-half-tasks:
-- workunit:
-    branch: pacific
-    clients:
-      client.0:
-        - rbd/import_export.sh
-    env:
-      RBD_CREATE_ARGS: --new-format
-- print: "**** done rbd/import_export.sh 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd_api.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd_api.yaml
deleted file mode 100644
index a7060c0ac9dd..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/rbd_api.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   librbd C and C++ api tests
-first-half-tasks:
-- workunit:
-     branch: pacific
-     clients:
-        client.0:
-           - rbd/test_librbd.sh
-     env:
-       RBD_FEATURES: "61"
-- print: "**** done rbd/test_librbd.sh 7-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/readwrite.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/readwrite.yaml
deleted file mode 100644
index 21a9f379a9ec..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/readwrite.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-meta:
-- desc: |
-   randomized correctness test for rados operations on a replicated pool,
-   using only reads, writes, and deletes
-first-half-tasks:
-- full_sequential:
-  - rados:
-      clients: [client.0]
-      ops: 4000
-      objects: 500
-      write_append_excl: false
-      op_weights:
-        read: 45
-        write: 45
-        delete: 10
-- print: "**** done rados/readwrite 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/snaps-few-objects.yaml b/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/snaps-few-objects.yaml
deleted file mode 100644
index 6447c2245fb5..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/2-first-half-tasks/snaps-few-objects.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-meta:
-- desc: |
-   randomized correctness test for rados operations on a replicated pool with snapshot operations
-first-half-tasks:
-- full_sequential:
-  - rados:
-      clients: [client.0]
-      ops: 4000
-      objects: 50
-      write_append_excl: false
-      op_weights:
-        read: 100
-        write: 100
-        delete: 50
-        snap_create: 50
-        snap_remove: 50
-        rollback: 50
-- print: "**** done rados/snaps-few-objects 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/+ b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/+
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/.qa b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/radosbench.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/radosbench.yaml
deleted file mode 100644
index 9058bd804a6c..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/radosbench.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-meta:
-- desc: |
-   run randomized correctness test for rados operations
-   generate write load with rados bench
-stress-tasks:
-- full_sequential:
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-- print: "**** done end radosbench.yaml"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-cls.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-cls.yaml
deleted file mode 100644
index e72875c14e06..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-cls.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-meta:
-- desc: |
-   run basic cls tests for rbd
-stress-tasks:
-- workunit:
-    branch: pacific
-    clients:
-      client.0:
-        - cls/test_cls_rbd.sh
-- print: "**** done cls/test_cls_rbd.sh 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-import-export.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-import-export.yaml
deleted file mode 100644
index c3008f3b10b0..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd-import-export.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   run basic import/export cli tests for rbd
-stress-tasks:
-- workunit:
-    branch: pacific
-    clients:
-      client.0:
-        - rbd/import_export.sh
-    env:
-      RBD_CREATE_ARGS: --new-format
-- print: "**** done rbd/import_export.sh 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd_api.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd_api.yaml
deleted file mode 100644
index 8b52658c4527..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/rbd_api.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   librbd C and C++ api tests
-stress-tasks:
-- workunit:
-     branch: pacific
-     clients:
-        client.0:
-           - rbd/test_librbd.sh
-     env:
-       RBD_FEATURES: "61"
-- print: "**** done rbd/test_librbd.sh 7-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/readwrite.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/readwrite.yaml
deleted file mode 100644
index 41e34d6d7d32..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/readwrite.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-meta:
-- desc: |
-   randomized correctness test for rados operations on a replicated pool,
-   using only reads, writes, and deletes
-stress-tasks:
-- full_sequential:
-  - rados:
-      clients: [client.0]
-      ops: 4000
-      objects: 500
-      write_append_excl: false
-      op_weights:
-        read: 45
-        write: 45
-        delete: 10
-- print: "**** done rados/readwrite 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/snaps-few-objects.yaml b/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/snaps-few-objects.yaml
deleted file mode 100644
index f56d0de0f745..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/3-stress-tasks/snaps-few-objects.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-meta:
-- desc: |
-   randomized correctness test for rados operations on a replicated pool with snapshot operations
-stress-tasks:
-- full_sequential:
-  - rados:
-      clients: [client.0]
-      ops: 4000
-      objects: 50
-      write_append_excl: false
-      op_weights:
-        read: 100
-        write: 100
-        delete: 50
-        snap_create: 50
-        snap_remove: 50
-        rollback: 50
-- print: "**** done rados/snaps-few-objects 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/.qa b/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/radosbench.yaml b/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/radosbench.yaml
deleted file mode 100644
index 7268cb170cad..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/radosbench.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-meta:
-- desc: |
-   run randomized correctness test for rados operations
-   generate write load with rados bench
-second-half-tasks:
-- full_sequential:
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-  - radosbench:
-      clients: [client.0]
-      time: 90
-- print: "**** done end radosbench.yaml"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/rbd-import-export.yaml b/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/rbd-import-export.yaml
deleted file mode 100644
index 00cf88d54f16..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/4-second-half-tasks/rbd-import-export.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-meta:
-- desc: |
-   run basic import/export cli tests for rbd
-second-half-tasks:
-- workunit:
-    branch: pacific
-    clients:
-      client.0:
-        - rbd/import_export.sh
-    env:
-      RBD_CREATE_ARGS: --new-format
-- print: "**** done rbd/import_export.sh 5-workload"
diff --git a/qa/suites/upgrade/pacific-x/stress-split/mon_election b/qa/suites/upgrade/pacific-x/stress-split/mon_election
deleted file mode 120000
index 3f331e6214c4..000000000000
--- a/qa/suites/upgrade/pacific-x/stress-split/mon_election
+++ /dev/null
@@ -1 +0,0 @@
-.qa/mon_election
\ No newline at end of file

From 6d25c053127f6aaaf942e674dca30ffd42658b9d Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 22 Aug 2023 10:57:52 +0000
Subject: [PATCH 0887/2492] qa: bring test_telemetry_reef_x.sh for squid

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst        |  1 +
 qa/workunits/test_telemetry_reef_x.sh | 40 +++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100755 qa/workunits/test_telemetry_reef_x.sh

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 2f83eff7f778..bb4777eee636 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -105,6 +105,7 @@ Code cleanup
 QA suite
 --------
 
+- [x] create qa/workunits/test_telemetry_(X-1)_x.sh
 - [ ] create qa/suites/upgrade/(X-1)-x
 - [x] remove qa/suites/upgrade/(X-3)-x-*
 - [x] create qa/releases/X.yaml
diff --git a/qa/workunits/test_telemetry_reef_x.sh b/qa/workunits/test_telemetry_reef_x.sh
new file mode 100755
index 000000000000..4e1a905669b0
--- /dev/null
+++ b/qa/workunits/test_telemetry_reef_x.sh
@@ -0,0 +1,40 @@
+#!/bin/bash -ex
+
+# For quincy, the last_opt_revision remains at 1 since last_opt_revision
+# was phased out for fresh installs of quincy.
+LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision)
+if [ $LAST_OPT_REVISION -ne 1 ]; then
+    echo "last_opt_revision is incorrect"
+    exit 1
+fi
+
+# Check the warning:
+ceph -s
+
+COLLECTIONS=$(ceph telemetry collection ls)
+NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics")
+for col in ${NEW_COLLECTIONS[@]}; do
+    if ! [[ $COLLECTIONS == *$col* ]];
+    then
+        echo "COLLECTIONS does not contain" "'"$col"'."
+	exit 1
+    fi
+done
+
+#Run preview commands
+ceph telemetry preview
+ceph telemetry preview-device
+ceph telemetry preview-all
+
+# Opt in to new collections
+# Currently, no new collections between latest reef and squid (dev)
+
+# Run show commands
+ceph telemetry show
+ceph telemetry show-device
+ceph telemetry show
+
+# Opt out
+ceph telemetry off
+
+echo OK

From b35871b920b45ae004bdc1130b4accb33fca33c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 29 Aug 2023 15:12:00 +0200
Subject: [PATCH 0888/2492] qa: bring test_telemetry_reef.sh for squid
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst      |  1 +
 qa/workunits/test_telemetry_reef.sh | 44 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100755 qa/workunits/test_telemetry_reef.sh

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index bb4777eee636..4a43ea2dc471 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -105,6 +105,7 @@ Code cleanup
 QA suite
 --------
 
+- [x] create qa/workunits/test_telemetry_(X-1).sh
 - [x] create qa/workunits/test_telemetry_(X-1)_x.sh
 - [ ] create qa/suites/upgrade/(X-1)-x
 - [x] remove qa/suites/upgrade/(X-3)-x-*
diff --git a/qa/workunits/test_telemetry_reef.sh b/qa/workunits/test_telemetry_reef.sh
new file mode 100755
index 000000000000..e8b07ec13032
--- /dev/null
+++ b/qa/workunits/test_telemetry_reef.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -ex
+
+# Set up ident details for cluster
+ceph config set mgr mgr/telemetry/channel_ident true
+ceph config set mgr mgr/telemetry/organization 'ceph-qa'
+ceph config set mgr mgr/telemetry/description 'upgrade test cluster'
+
+
+#Run preview commands
+ceph telemetry preview
+ceph telemetry preview-device
+ceph telemetry preview-all
+
+# Assert that new collections are available
+COLLECTIONS=$(ceph telemetry collection ls)
+NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics")
+for col in ${NEW_COLLECTIONS[@]}; do
+    if ! [[ $COLLECTIONS == *$col* ]];
+    then
+        echo "COLLECTIONS does not contain" "'"$col"'."
+	exit 1
+    fi
+done
+
+# Opt-in
+ceph telemetry on --license sharing-1-0
+
+# Enable perf channel
+ceph telemetry enable channel perf
+
+# For quincy, the last_opt_revision remains at 1 since last_opt_revision
+# was phased out for fresh installs of quincy.
+LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision)
+if [ $LAST_OPT_REVISION -ne 1 ]; then
+    echo "last_opt_revision is incorrect"
+    exit 1
+fi
+
+# Run show commands
+ceph telemetry show
+ceph telemetry show-device
+ceph telemetry show-all
+
+echo OK

From 7af05d0944da5b048237a8dcadc8de5cf4f3e2cb Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 22 Aug 2023 10:37:44 +0000
Subject: [PATCH 0889/2492] qa: bring qa/suites/upgrade/reef-x

Bases on quincy-x.

```
$ cp -R qa/suites/upgrade/quincy-x/ qa/suites/upgrade/reef-x
$ git add qa/suites/upgrade/reef-x
$ git mv qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/quincy.yaml qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
$ find qa/suites/upgrade/reef-x/ -type f -exec sed -i 's/quincy/reef/g' {} +
```

A note from rebase: changes from 05e24270a2efe85bcdceade87b0e91efcfca3001
have been pulled in.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst                |  2 +-
 .../upgrade/reef-x/filestore-remove-check/%   |  0
 .../upgrade/reef-x/filestore-remove-check/.qa |  1 +
 .../reef-x/filestore-remove-check/0-cluster/+ |  0
 .../filestore-remove-check/0-cluster/.qa      |  1 +
 .../0-cluster/openstack.yaml                  |  6 ++++
 .../0-cluster/start.yaml                      | 33 +++++++++++++++++++
 .../filestore-remove-check/1-ceph-install/.qa |  1 +
 .../1-ceph-install/reef.yaml                  | 32 ++++++++++++++++++
 .../filestore-remove-check/2 - upgrade.yaml   | 20 +++++++++++
 .../filestore-remove-check/objectstore/.qa    |  1 +
 .../objectstore/filestore-xfs.yaml            | 14 ++++++++
 .../filestore-remove-check/ubuntu_20.04.yaml  |  9 +++++
 .../parallel/workload/test_rbd_api.yaml       |  2 ++
 .../parallel/workload/test_rbd_python.yaml    |  2 ++
 .../2-first-half-tasks/rbd_api.yaml           |  2 ++
 .../stress-split/3-stress-tasks/rbd_api.yaml  |  2 ++
 17 files changed, 127 insertions(+), 1 deletion(-)
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/%
 create mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/.qa
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+
 create mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
 create mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml
 create mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
 create mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 4a43ea2dc471..220c87fe0c11 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -107,7 +107,7 @@ QA suite
 
 - [x] create qa/workunits/test_telemetry_(X-1).sh
 - [x] create qa/workunits/test_telemetry_(X-1)_x.sh
-- [ ] create qa/suites/upgrade/(X-1)-x
+- [x] create qa/suites/upgrade/(X-1)-x
 - [x] remove qa/suites/upgrade/(X-3)-x-*
 - [x] create qa/releases/X.yaml
 - [x] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/% b/qa/suites/upgrade/reef-x/filestore-remove-check/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+ b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
new file mode 100644
index 000000000000..5caffc353f96
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
@@ -0,0 +1,6 @@
+openstack:
+  - machine:
+      disk: 100 # GB
+  - volumes: # attached to each instance
+      count: 4
+      size: 30 # GB
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
new file mode 100644
index 000000000000..57e455ba78d6
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
@@ -0,0 +1,33 @@
+meta:
+- desc: |
+   Run ceph on one nodes,
+   Use xfs beneath the osds. upgrade to reef
+   should fail to start the osds with filestore
+overrides:
+  ceph:
+    mon_bind_msgr2: false
+    mon_bind_addrvec: false
+    mon-health-to-clog: false
+    wait-for-healthy: false
+    wait-for-osds-up: false
+    wait-for-scrub: false
+    skip_stop_pg_num_changes: true
+    fs: xfs
+    log-ignorelist:
+      - overall HEALTH_
+      - \(MON_DOWN\)
+      - \(MGR_DOWN\)
+      - slow request
+      - \(MON_MSGR2_NOT_ENABLED\)
+    conf:
+      global:
+        enable experimental unrecoverable data corrupting features: "*"
+        mon warn on msgr2 not enabled: false
+      mon:
+        mon warn on osd down out interval zero: false
+roles:
+- - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - osd.2
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
new file mode 100644
index 000000000000..d140b823cc7c
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
@@ -0,0 +1,32 @@
+meta:
+- desc: install ceph/reef latest
+tasks:
+- install:
+    exclude_packages:
+      - ceph-mgr-cephadm
+      - cephadm
+      - libcephfs-dev
+    branch: reef
+- print: "**** done install reef"
+- ceph:
+    create_rbd_pool: false
+    conf:
+      global:
+        bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
+        mon pg warn min per osd: 0
+      mon:
+        mon_warn_on_insecure_global_id_reclaim: false
+        mon_warn_on_insecure_global_id_reclaim_allowed: false
+    log-ignorelist:
+      - Not found or unloadable
+      - evicting unresponsive client
+- exec:
+    osd.0:
+      - ceph osd require-osd-release reef
+- print: "**** done ceph"
+overrides:
+  ceph:
+    conf:
+      mon:
+        mon warn on osd down out interval zero: false
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml
new file mode 100644
index 000000000000..6f5d74b3bc84
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml	
@@ -0,0 +1,20 @@
+meta:
+- desc: |
+   install upgrade ceph/-x on cluster
+   restart : mons, osd.*
+tasks:
+- install.upgrade:
+    mon.a:
+- exec:
+    osd.0:
+      - ceph osd require-osd-release reef
+- print: "**** done install.upgrade of nodes"
+- ceph.restart:
+    daemons: [mon.a,mgr.x,osd.0,osd.1,osd.2]
+    mon-health-to-clog: false
+    wait-for-healthy: false
+    wait-for-osds-up: false
+    wait-for-scrub: false
+    skip_stop_pg_num_changes: true
+    expected-failure: "FileStore has been deprecated and is no longer supported"
+- print: "**** done ceph.restart of all mons and osds"
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
new file mode 100644
index 000000000000..b6ef47b06f74
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
@@ -0,0 +1,14 @@
+overrides:
+  ceph:
+    fs: xfs
+    conf:
+      osd:
+        osd objectstore: filestore
+        osd sloppy crc: true
+  ceph-deploy:
+    fs: xfs
+    filestore: True
+    conf:
+      osd:
+        osd objectstore: filestore
+        osd sloppy crc: true
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml
new file mode 100644
index 000000000000..e1374c4105d8
--- /dev/null
+++ b/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml
@@ -0,0 +1,9 @@
+os_type: ubuntu
+os_version: "20.04"
+# the normal ubuntu 20.04 kernel (5.4.0-88-generic currently) have a bug that prevents the nvme_loop
+# from behaving.  I think it is this:
+#   https://lkml.org/lkml/2020/9/21/1456
+# (at least, that is the symptom: nvme nvme1: Connect command failed, error wo/DNR bit: 880)
+overrides:
+  kernel:
+    hwe: true
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_api.yaml b/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_api.yaml
index 5eb0e1d289a1..bb9bedd33530 100644
--- a/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_api.yaml
+++ b/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_api.yaml
@@ -9,4 +9,6 @@ workload:
         clients:
           client.0:
               - rbd/test_librbd.sh
+        env:
+          RBD_FEATURES: "61"
     - print: "**** done end test_rbd_api.yaml"
diff --git a/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_python.yaml b/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_python.yaml
index bfe0cc993e88..2ad33a27950a 100644
--- a/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_python.yaml
+++ b/qa/suites/upgrade/reef-x/parallel/workload/test_rbd_python.yaml
@@ -14,5 +14,7 @@ workload:
         clients:
           client.0:
             - rbd/test_librbd_python.sh
+        env:
+          RBD_FEATURES: "61"
     - print: "**** done end test_rbd_python.yaml"
 
diff --git a/qa/suites/upgrade/reef-x/stress-split/2-first-half-tasks/rbd_api.yaml b/qa/suites/upgrade/reef-x/stress-split/2-first-half-tasks/rbd_api.yaml
index 94451570df52..9d220ec67835 100644
--- a/qa/suites/upgrade/reef-x/stress-split/2-first-half-tasks/rbd_api.yaml
+++ b/qa/suites/upgrade/reef-x/stress-split/2-first-half-tasks/rbd_api.yaml
@@ -7,4 +7,6 @@ first-half-tasks:
      clients:
         client.0:
            - rbd/test_librbd.sh
+     env:
+       RBD_FEATURES: "61"
 - print: "**** done rbd/test_librbd.sh 7-workload"
diff --git a/qa/suites/upgrade/reef-x/stress-split/3-stress-tasks/rbd_api.yaml b/qa/suites/upgrade/reef-x/stress-split/3-stress-tasks/rbd_api.yaml
index 71ef6d961009..7360c5dfcf5e 100644
--- a/qa/suites/upgrade/reef-x/stress-split/3-stress-tasks/rbd_api.yaml
+++ b/qa/suites/upgrade/reef-x/stress-split/3-stress-tasks/rbd_api.yaml
@@ -7,4 +7,6 @@ stress-tasks:
      clients:
         client.0:
            - rbd/test_librbd.sh
+     env:
+       RBD_FEATURES: "61"
 - print: "**** done rbd/test_librbd.sh 7-workload"

From def898c5a22b3d3f6b379ea1f6f4bd70e3f8a801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 29 Aug 2023 15:03:51 +0200
Subject: [PATCH 0890/2492] qa: drop filestore-remove-check from upgrade/reef-x
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

FileStore is removed in Reef.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 .../upgrade/reef-x/filestore-remove-check/%   |  0
 .../upgrade/reef-x/filestore-remove-check/.qa |  1 -
 .../reef-x/filestore-remove-check/0-cluster/+ |  0
 .../filestore-remove-check/0-cluster/.qa      |  1 -
 .../0-cluster/openstack.yaml                  |  6 ----
 .../0-cluster/start.yaml                      | 33 -------------------
 .../filestore-remove-check/1-ceph-install/.qa |  1 -
 .../1-ceph-install/reef.yaml                  | 32 ------------------
 .../filestore-remove-check/2 - upgrade.yaml   | 20 -----------
 .../filestore-remove-check/objectstore/.qa    |  1 -
 .../objectstore/filestore-xfs.yaml            | 14 --------
 .../filestore-remove-check/ubuntu_20.04.yaml  |  9 -----
 12 files changed, 118 deletions(-)
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/%
 delete mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/.qa
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+
 delete mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
 delete mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml
 delete mode 120000 qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
 delete mode 100644 qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml

diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/% b/qa/suites/upgrade/reef-x/filestore-remove-check/%
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/.qa
deleted file mode 120000
index fea2489fdf6d..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+ b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/+
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
deleted file mode 100644
index 5caffc353f96..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/openstack.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-openstack:
-  - machine:
-      disk: 100 # GB
-  - volumes: # attached to each instance
-      count: 4
-      size: 30 # GB
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
deleted file mode 100644
index 57e455ba78d6..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/0-cluster/start.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
-meta:
-- desc: |
-   Run ceph on one nodes,
-   Use xfs beneath the osds. upgrade to reef
-   should fail to start the osds with filestore
-overrides:
-  ceph:
-    mon_bind_msgr2: false
-    mon_bind_addrvec: false
-    mon-health-to-clog: false
-    wait-for-healthy: false
-    wait-for-osds-up: false
-    wait-for-scrub: false
-    skip_stop_pg_num_changes: true
-    fs: xfs
-    log-ignorelist:
-      - overall HEALTH_
-      - \(MON_DOWN\)
-      - \(MGR_DOWN\)
-      - slow request
-      - \(MON_MSGR2_NOT_ENABLED\)
-    conf:
-      global:
-        enable experimental unrecoverable data corrupting features: "*"
-        mon warn on msgr2 not enabled: false
-      mon:
-        mon warn on osd down out interval zero: false
-roles:
-- - mon.a
-  - mgr.x
-  - osd.0
-  - osd.1
-  - osd.2
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
deleted file mode 100644
index d140b823cc7c..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/1-ceph-install/reef.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-meta:
-- desc: install ceph/reef latest
-tasks:
-- install:
-    exclude_packages:
-      - ceph-mgr-cephadm
-      - cephadm
-      - libcephfs-dev
-    branch: reef
-- print: "**** done install reef"
-- ceph:
-    create_rbd_pool: false
-    conf:
-      global:
-        bluestore_warn_on_legacy_statfs: false
-        bluestore warn on no per pool omap: false
-        mon pg warn min per osd: 0
-      mon:
-        mon_warn_on_insecure_global_id_reclaim: false
-        mon_warn_on_insecure_global_id_reclaim_allowed: false
-    log-ignorelist:
-      - Not found or unloadable
-      - evicting unresponsive client
-- exec:
-    osd.0:
-      - ceph osd require-osd-release reef
-- print: "**** done ceph"
-overrides:
-  ceph:
-    conf:
-      mon:
-        mon warn on osd down out interval zero: false
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml
deleted file mode 100644
index 6f5d74b3bc84..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/2 - upgrade.yaml	
+++ /dev/null
@@ -1,20 +0,0 @@
-meta:
-- desc: |
-   install upgrade ceph/-x on cluster
-   restart : mons, osd.*
-tasks:
-- install.upgrade:
-    mon.a:
-- exec:
-    osd.0:
-      - ceph osd require-osd-release reef
-- print: "**** done install.upgrade of nodes"
-- ceph.restart:
-    daemons: [mon.a,mgr.x,osd.0,osd.1,osd.2]
-    mon-health-to-clog: false
-    wait-for-healthy: false
-    wait-for-osds-up: false
-    wait-for-scrub: false
-    skip_stop_pg_num_changes: true
-    expected-failure: "FileStore has been deprecated and is no longer supported"
-- print: "**** done ceph.restart of all mons and osds"
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
deleted file mode 120000
index a602a0353e75..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
deleted file mode 100644
index b6ef47b06f74..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/objectstore/filestore-xfs.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-overrides:
-  ceph:
-    fs: xfs
-    conf:
-      osd:
-        osd objectstore: filestore
-        osd sloppy crc: true
-  ceph-deploy:
-    fs: xfs
-    filestore: True
-    conf:
-      osd:
-        osd objectstore: filestore
-        osd sloppy crc: true
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml b/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml
deleted file mode 100644
index e1374c4105d8..000000000000
--- a/qa/suites/upgrade/reef-x/filestore-remove-check/ubuntu_20.04.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-os_type: ubuntu
-os_version: "20.04"
-# the normal ubuntu 20.04 kernel (5.4.0-88-generic currently) have a bug that prevents the nvme_loop
-# from behaving.  I think it is this:
-#   https://lkml.org/lkml/2020/9/21/1456
-# (at least, that is the symptom: nvme nvme1: Connect command failed, error wo/DNR bit: 880)
-overrides:
-  kernel:
-    hwe: true
\ No newline at end of file

From 7fc77efe2bdfbee5c7d49710095514f36c76c146 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 22 Aug 2023 11:02:04 +0000
Subject: [PATCH 0891/2492] qa: qa/suites/rados/upgrade/parallel points to reef

```
$ git rm qa/suites/rados/upgrade/parallel
$ ln -s ../../upgrade/reef-x/parallel qa/suites/rados/upgrade/parallel
$ git add qa/suites/rados/upgrade/parallel
```

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 qa/suites/rados/upgrade/parallel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/suites/rados/upgrade/parallel b/qa/suites/rados/upgrade/parallel
index 84b63d6a5d88..4a89d75d594a 120000
--- a/qa/suites/rados/upgrade/parallel
+++ b/qa/suites/rados/upgrade/parallel
@@ -1 +1 @@
-../../upgrade/quincy-x/parallel/
\ No newline at end of file
+../../upgrade/reef-x/parallel
\ No newline at end of file

From 4a81193f707cc7551b66803c78bb84145523bcd3 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Fri, 25 Aug 2023 14:17:59 +0000
Subject: [PATCH 0892/2492] test/cli/monmaptool: adjust for the squid monmap
 features

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst                |  2 +-
 .../cli/monmaptool/feature-set-unset-list.t   | 28 +++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 220c87fe0c11..b41a3f00687a 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -76,7 +76,7 @@ Compatsets
 - [x] mon/Monitor.cc (include in `get_supported_features()`)
 - [x] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
 - [x] mon/Monitor.cc (`calc_quorum_requirements()`)
-- [ ] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
+- [x] test/cli/monmaptool/feature-set-unset-list.t (`supported`, `persistent`)
 
 Mon
 ---
diff --git a/src/test/cli/monmaptool/feature-set-unset-list.t b/src/test/cli/monmaptool/feature-set-unset-list.t
index 5f9d3859d539..1e16f57d1678 100644
--- a/src/test/cli/monmaptool/feature-set-unset-list.t
+++ b/src/test/cli/monmaptool/feature-set-unset-list.t
@@ -12,21 +12,21 @@
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
   MONMAP FEATURES:
       persistent: [none]
       optional:   [none]
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
   monmap:persistent:[none]
   monmap:optional:[none]
   monmap:required:[none]
-  available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-  available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+  available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+  available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
 
   $ monmaptool --feature-set foo /tmp/test.monmap.1234
   unknown features name 'foo' or unable to parse value: Expected option value to be integer, got 'foo'
@@ -45,8 +45,8 @@
       required:   [kraken(1),octopus(32),unknown(4096)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
 
   $ monmaptool --feature-unset 32 --optional --feature-list /tmp/test.monmap.1234
   monmaptool: monmap file /tmp/test.monmap.1234
@@ -56,8 +56,8 @@
       required:   [kraken(1),octopus(32),unknown(4096)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ monmaptool --feature-unset 32 --persistent --feature-unset 4096 --optional --feature-list /tmp/test.monmap.1234
@@ -68,8 +68,8 @@
       required:   [kraken(1)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ monmaptool --feature-unset kraken --feature-list /tmp/test.monmap.1234
@@ -80,8 +80,8 @@
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ rm /tmp/test.monmap.1234

From 70b70c5a6b4489cac3405c4ed8b4f3c8de59e3e7 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Sun, 27 Aug 2023 08:04:33 +0000
Subject: [PATCH 0893/2492] release-checklists.rst: mark section for ceph-build
 as complete

This bases on https://github.com/ceph/ceph-build/pull/2160.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index b41a3f00687a..0af98ff3b38c 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -117,10 +117,10 @@ ceph-build
 ----------
 In the `ceph/ceph-build.git` repo:
 
-- [ ] add the version -> X mapping (`release_from_version()` in `scripts/build_utils.sh`)
-- [ ] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/build_osc`)
-- [ ] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/setup_osc`)
-- [ ] grep for previous release and add relevant build targets (e.g. for reef https://github.com/ceph/ceph-build/pull/2076 and https://github.com/ceph/ceph-build/pull/2119)
+- [x] add the version -> X mapping (`release_from_version()` in `scripts/build_utils.sh`)
+- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/build_osc`)
+- [x] add the option for X (`case $RELEASE_BRANCH` in `ceph-dev-build/build/setup_osc`)
+- [x] grep for previous release and add relevant build targets (e.g. for reef https://github.com/ceph/ceph-build/pull/2076 and https://github.com/ceph/ceph-build/pull/2119)
 
 
 ceph-container

From 55b3be5234f1c670b0c7d3f3a1584af2573d9288 Mon Sep 17 00:00:00 2001
From: Mykola Golub <mgolub@suse.com>
Date: Sun, 3 Dec 2023 09:37:02 +0000
Subject: [PATCH 0894/2492] test/pybind/rbd: test callbacks raising exceptions

Signed-off-by: Mykola Golub <mgolub@suse.com>
---
 src/test/pybind/test_rbd.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py
index 7b5f31b577a6..0ce3c0dd90ca 100644
--- a/src/test/pybind/test_rbd.py
+++ b/src/test/pybind/test_rbd.py
@@ -415,6 +415,18 @@ def progress_cb(current, total):
     assert_raises(OperationCanceled, RBD().remove, ioctx, image_name,
                   on_progress=progress_cb)
 
+def test_remove_with_progress_except():
+    create_image()
+    d = {'received_callback': False}
+    def progress_cb(current, total):
+        d['received_callback'] = True
+        raise Exception()
+
+    # exception is logged and ignored with a Cython warning:
+    #   Exception ignored in: 'rbd.progress_callback'
+    RBD().remove(ioctx, image_name, on_progress=progress_cb)
+    eq(True, d['received_callback'])
+
 def test_rename(tmp_image):
     rbd = RBD()
     image_name2 = get_temp_image_name()
@@ -1251,6 +1263,16 @@ def cb(_, buf):
         assert(comp.get_return_value() < 0)
         eq(sys.getrefcount(comp), 2)
 
+        # test3: except case
+        def cbex(_, buf):
+            raise KeyError()
+
+        def test3():
+            comp = self.image.aio_read(IMG_SIZE, 20, cbex)
+            comp.wait_for_complete_and_cb()
+
+        assert_raises(KeyError, test3)
+
     def test_aio_write(self):
         retval = [None]
         def cb(comp):

From e3156050d0ce9b504ee40d30e98f49a860b7dde5 Mon Sep 17 00:00:00 2001
From: Mykola Golub <mgolub@suse.com>
Date: Mon, 4 Dec 2023 09:38:56 +0000
Subject: [PATCH 0895/2492] pybind/rbd: make cdef functions not propagate
 exceptions by default

Setting legacy_implicit_noexcept compiler directive to True will cause
Cython 3.0 to have the same semantics as Cython 0.x.

Fixes: https://tracker.ceph.com/issues/62140
Signed-off-by: Mykola Golub <mgolub@suse.com>
---
 src/pybind/rbd/setup.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/pybind/rbd/setup.py b/src/pybind/rbd/setup.py
index 1f20c3ed42fe..eeb33c73d49b 100755
--- a/src/pybind/rbd/setup.py
+++ b/src/pybind/rbd/setup.py
@@ -14,6 +14,7 @@
 from distutils.ccompiler import new_compiler
 from distutils.errors import CompileError, LinkError
 from itertools import filterfalse, takewhile
+from packaging import version
 import distutils.sysconfig
 
 
@@ -148,11 +149,22 @@ def check_sanity():
     sys.exit(1)
 
 cmdclass = {}
+compiler_directives={'language_level': sys.version_info.major}
 try:
     from Cython.Build import cythonize
     from Cython.Distutils import build_ext
+    from Cython import __version__ as cython_version
 
     cmdclass = {'build_ext': build_ext}
+
+    # Needed for building with Cython 0.x and Cython 3 from the same file,
+    # preserving the same behavior.
+    # When Cython 0.x builds go away, replace this compiler directive with
+    # noexcept on rbd_callback_t and librbd_progress_fn_t (or consider doing
+    # something similar to except? -9000 on rbd_diff_iterate2() callback for
+    # progress callbacks to propagate exceptions).
+    if version.parse(cython_version) >= version.parse('3'):
+        compiler_directives['legacy_implicit_noexcept'] = True
 except ImportError:
     print("WARNING: Cython is not installed.")
 
@@ -197,7 +209,7 @@ def cythonize(x, **kwargs):
                 **ext_args
             )
         ],
-        compiler_directives={'language_level': sys.version_info.major},
+        compiler_directives=compiler_directives,
         build_dir=os.environ.get("CYTHON_BUILD_DIR", None),
         **cythonize_args
     ),

From 0999e63bfbbee46b8e19c3f05881eee64dba8b5e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 22 Nov 2023 14:39:13 +0100
Subject: [PATCH 0896/2492] librados: make querying pools for selfmanaged snaps
 reliable

If get_pool_is_selfmanaged_snaps_mode() is invoked on a fresh RADOS
client instance that still lacks an osdmap, it returns false, same as
for "this pool is not in selfmanaged snaps mode".  The same happens if
the pool in question doesn't exist since the signature doesn't allow to
return an error.

The motivation for this API was to prevent users from running "rados
cppool" on a pool with unmanaged snapshots and deleting the original
thinking that they have a full copy.  Unfortunately, it's exactly
"rados cppool" that fell into this trap, so no warning is printed and
--yes-i-really-mean-it flag isn't enforced.

Fixes: https://tracker.ceph.com/issues/63607
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 PendingReleaseNotes                |  3 ++
 src/include/rados/librados.hpp     |  5 ++-
 src/librados/RadosClient.cc        | 18 ++++++---
 src/librados/RadosClient.h         |  2 +-
 src/librados/librados_cxx.cc       |  9 ++++-
 src/test/librados/snapshots_cxx.cc | 60 ++++++++++++++++++++++++++----
 src/tools/rados/rados.cc           | 14 ++++++-
 7 files changed, 93 insertions(+), 18 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 9b3dfede8bbd..91d29a890f07 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -77,6 +77,9 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   cannot repair the dates of existing object locks. Such objects can be identified
   with a HeadObject request based on the x-amz-object-lock-retain-until-date
   response header.
+* RADOS: `get_pool_is_selfmanaged_snaps_mode` C++ API has been deprecated
+  due to being prone to false negative results.  It's safer replacement is
+  `pool_is_in_selfmanaged_snaps_mode`.
 
 >=18.0.0
 
diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index f2f9e47a2ad7..2cd418627be9 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -1477,8 +1477,11 @@ inline namespace v14_2_0 {
     int get_pool_stats(std::list<std::string>& v,
                        std::string& category,
 		       std::map<std::string, stats_map>& stats);
+
     /// check if pool has or had selfmanaged snaps
-    bool get_pool_is_selfmanaged_snaps_mode(const std::string& poolname);
+    bool get_pool_is_selfmanaged_snaps_mode(const std::string& poolname)
+      __attribute__ ((deprecated));
+    int pool_is_in_selfmanaged_snaps_mode(const std::string& poolname);
 
     int cluster_stat(cluster_stat_t& result);
     int cluster_fsid(std::string *fsid);
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 9abd923f95b0..db9143e2d5b8 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -631,16 +631,22 @@ int librados::RadosClient::get_pool_stats(std::list<string>& pools,
   return 0;
 }
 
-bool librados::RadosClient::get_pool_is_selfmanaged_snaps_mode(
+int librados::RadosClient::pool_is_in_selfmanaged_snaps_mode(
   const std::string& pool)
 {
-  bool ret = false;
-  objecter->with_osdmap([&](const OSDMap& osdmap) {
+  int r = wait_for_osdmap();
+  if (r < 0) {
+    return r;
+  }
+
+  return objecter->with_osdmap([&pool](const OSDMap& osdmap) {
       int64_t poolid = osdmap.lookup_pg_pool_name(pool);
-      if (poolid >= 0)
-	ret = osdmap.get_pg_pool(poolid)->is_unmanaged_snaps_mode();
+      if (poolid < 0) {
+        return -ENOENT;
+      }
+      return static_cast<int>(
+        osdmap.get_pg_pool(poolid)->is_unmanaged_snaps_mode());
     });
-  return ret;
 }
 
 int librados::RadosClient::get_fs_stats(ceph_statfs& stats)
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index 052249a76db3..a93f8185108d 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -131,7 +131,7 @@ class librados::RadosClient : public Dispatcher,
   int get_pool_stats(std::list<std::string>& ls, std::map<std::string,::pool_stat_t> *result,
     bool *per_pool);
   int get_fs_stats(ceph_statfs& result);
-  bool get_pool_is_selfmanaged_snaps_mode(const std::string& pool);
+  int pool_is_in_selfmanaged_snaps_mode(const std::string& pool);
 
   /*
   -1 was set as the default value and monitor will pickup the right crush rule with below order:
diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc
index d20c67556c03..926ddf86dab4 100644
--- a/src/librados/librados_cxx.cc
+++ b/src/librados/librados_cxx.cc
@@ -2719,9 +2719,16 @@ int librados::Rados::get_pool_stats(std::list<string>& v,
   return -EOPNOTSUPP;
 }
 
+// deprecated, use pool_is_in_selfmanaged_snaps_mode() instead
 bool librados::Rados::get_pool_is_selfmanaged_snaps_mode(const std::string& pool)
 {
-  return client->get_pool_is_selfmanaged_snaps_mode(pool);
+  // errors are ignored, prone to false negative results
+  return client->pool_is_in_selfmanaged_snaps_mode(pool) > 0;
+}
+
+int librados::Rados::pool_is_in_selfmanaged_snaps_mode(const std::string& pool)
+{
+  return client->pool_is_in_selfmanaged_snaps_mode(pool);
 }
 
 int librados::Rados::cluster_stat(cluster_stat_t& result)
diff --git a/src/test/librados/snapshots_cxx.cc b/src/test/librados/snapshots_cxx.cc
index 8098b2cb7817..95dbe5da0125 100644
--- a/src/test/librados/snapshots_cxx.cc
+++ b/src/test/librados/snapshots_cxx.cc
@@ -25,9 +25,9 @@ TEST_F(LibRadosSnapshotsPP, SnapListPP) {
   bufferlist bl1;
   bl1.append(buf, sizeof(buf));
   ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
-  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   ASSERT_EQ(0, ioctx.snap_create("snap1"));
-  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   std::vector<snap_t> snaps;
   EXPECT_EQ(0, ioctx.snap_list(&snaps));
   EXPECT_EQ(1U, snaps.size());
@@ -35,7 +35,7 @@ TEST_F(LibRadosSnapshotsPP, SnapListPP) {
   EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
   EXPECT_EQ(rid, snaps[0]);
   EXPECT_EQ(0, ioctx.snap_remove("snap1"));
-  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
 }
 
 TEST_F(LibRadosSnapshotsPP, SnapRemovePP) {
@@ -109,9 +109,9 @@ TEST_F(LibRadosSnapshotsPP, SnapCreateRemovePP) {
 TEST_F(LibRadosSnapshotsSelfManagedPP, SnapPP) {
   std::vector<uint64_t> my_snaps;
   my_snaps.push_back(-2);
-  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
-  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(1, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   ::std::reverse(my_snaps.begin(), my_snaps.end()); 
   ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
   ::std::reverse(my_snaps.begin(), my_snaps.end());
@@ -148,7 +148,7 @@ TEST_F(LibRadosSnapshotsSelfManagedPP, SnapPP) {
   ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
   my_snaps.pop_back();
   ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
-  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(1, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   ASSERT_EQ(0, ioctx.remove("foo"));
 }
 
@@ -509,7 +509,7 @@ TEST_F(LibRadosSnapshotsSelfManagedPP, ReusePurgedSnap) {
   std::vector<uint64_t> my_snaps;
   my_snaps.push_back(-2);
   ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
-  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(1, cluster.pool_is_in_selfmanaged_snaps_mode(pool_name));
   ::std::reverse(my_snaps.begin(), my_snaps.end());
   ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
   ::std::reverse(my_snaps.begin(), my_snaps.end());
@@ -548,6 +548,52 @@ TEST_F(LibRadosSnapshotsSelfManagedPP, ReusePurgedSnap) {
   //sleep(600);
 }
 
+TEST(LibRadosPoolIsInSelfmanagedSnapsMode, NotConnected) {
+  librados::Rados cluster;
+  ASSERT_EQ(0, cluster.init(nullptr));
+
+  EXPECT_EQ(-ENOTCONN, cluster.pool_is_in_selfmanaged_snaps_mode("foo"));
+}
+
+TEST(LibRadosPoolIsInSelfmanagedSnapsMode, FreshInstance) {
+  librados::Rados cluster1;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster1));
+  EXPECT_EQ(0, cluster1.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  {
+    librados::Rados cluster2;
+    ASSERT_EQ("", connect_cluster_pp(cluster2));
+    EXPECT_EQ(0, cluster2.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  }
+
+  librados::IoCtx ioctx;
+  cluster1.ioctx_create(pool_name.c_str(), ioctx);
+  uint64_t snap_id;
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&snap_id));
+  EXPECT_EQ(1, cluster1.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  {
+    librados::Rados cluster2;
+    ASSERT_EQ("", connect_cluster_pp(cluster2));
+    EXPECT_EQ(1, cluster2.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  }
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(snap_id));
+  EXPECT_EQ(1, cluster1.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  {
+    librados::Rados cluster2;
+    ASSERT_EQ("", connect_cluster_pp(cluster2));
+    EXPECT_EQ(1, cluster2.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  }
+
+  ASSERT_EQ(0, cluster1.pool_delete(pool_name.c_str()));
+  EXPECT_EQ(-ENOENT, cluster1.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  {
+    librados::Rados cluster2;
+    ASSERT_EQ("", connect_cluster_pp(cluster2));
+    EXPECT_EQ(-ENOENT, cluster2.pool_is_in_selfmanaged_snaps_mode(pool_name));
+  }
+}
+
 // EC testing
 TEST_F(LibRadosSnapshotsECPP, SnapListPP) {
   SKIP_IF_CRIMSON();
diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index b8cf5e4d1dc3..db3aa543fb56 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -3120,7 +3120,12 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
     cerr << "WARNING: pool copy does not preserve user_version, which some "
 	 << "apps may rely on." << std::endl;
 
-    if (rados.get_pool_is_selfmanaged_snaps_mode(src_pool)) {
+    ret = rados.pool_is_in_selfmanaged_snaps_mode(src_pool);
+    if (ret < 0) {
+      cerr << "failed to query pool " << src_pool << " for selfmanaged snaps: "
+           << cpp_strerror(ret) << std::endl;
+      return 1;
+    } else if (ret > 0) {
       cerr << "WARNING: pool " << src_pool << " has selfmanaged snaps, which are not preserved\n"
 	   << "    by the cppool operation.  This will break any snapshot user."
 	   << std::endl;
@@ -3213,7 +3218,12 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
       return 1;
     }
 
-    if (rados.get_pool_is_selfmanaged_snaps_mode(pool_name)) {
+    ret = rados.pool_is_in_selfmanaged_snaps_mode(pool_name);
+    if (ret < 0) {
+      cerr << "failed to query pool " << pool_name << " for selfmanaged snaps: "
+           << cpp_strerror(ret) << std::endl;
+      return 1;
+    } else if (ret > 0) {
       cerr << "can't create snapshot: pool " << pool_name
            << " is in selfmanaged snaps mode" << std::endl;
       return 1;

From 2b135a2eb60e0dd5ab22d213d3d2435608c6a9d3 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 23 Nov 2023 20:24:24 +0100
Subject: [PATCH 0897/2492] qa: make sure "rados cppool" requires
 --yes-i-really-mean-it for RBD

Safeguards in rados CLI tool isn't really the subject of this test, but
it fits nicely.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 qa/workunits/mon/rbd_snaps_ops.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qa/workunits/mon/rbd_snaps_ops.sh b/qa/workunits/mon/rbd_snaps_ops.sh
index eb88565eab9c..0e5b16b7b80b 100755
--- a/qa/workunits/mon/rbd_snaps_ops.sh
+++ b/qa/workunits/mon/rbd_snaps_ops.sh
@@ -36,6 +36,7 @@ expect 'rbd --pool=test snap ls image' 0
 expect 'rbd --pool=test snap rm image@snapshot' 0
 
 expect 'ceph osd pool mksnap test snapshot' 22
+expect 'rados -p test mksnap snapshot' 1
 
 expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0
 
@@ -52,6 +53,8 @@ expect 'rbd --pool test-foo snap create image@snapshot' 0
 ceph osd pool delete test-bar test-bar --yes-i-really-really-mean-it || true
 expect 'ceph osd pool create test-bar 8' 0
 expect 'ceph osd pool application enable test-bar rbd'
+# "rados cppool" without --yes-i-really-mean-it should fail
+expect 'rados cppool test-foo test-bar' 1
 expect 'rados cppool test-foo test-bar --yes-i-really-mean-it' 0
 expect 'rbd --pool test-bar snap rm image@snapshot' 95
 expect 'ceph osd pool delete test-foo test-foo --yes-i-really-really-mean-it' 0

From de6635582b7615eb54800a616711073a3c7b0dbd Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 24 Nov 2023 19:53:48 +0100
Subject: [PATCH 0898/2492] tools/rados: just return instead of exit(1) in
 "rados cppool" handler

Otherwise an occasional segfault occurs.  This instance was missed in
commit 2c149262888c ("tools/rados: always call rados.shutdown() before
exit()").

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/tools/rados/rados.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index db3aa543fb56..4a38d33abd2d 100644
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -3133,7 +3133,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
 	cerr << "    If you insist on making a broken copy, you can pass\n"
 	     << "    --yes-i-really-mean-it to proceed anyway."
 	     << std::endl;
-	exit(1);
+	return 1;
       }
     }
 

From 334e7fa0879512790e89d1f4d30cd6aaa81f28fa Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 13 Sep 2023 16:51:16 +0300
Subject: [PATCH 0899/2492] os/bluestore: adjust and validate
 bluefs_shared_alloc_size

Make sure it's in-sync (meaning it's higher or equal and properly aligned)
with bluestore_min_alloc_size into account

Fixes: https://tracker.ceph.com/issues/63618
Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueFS.cc | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 52857a16081d..99851bef94f0 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -695,16 +695,24 @@ void BlueFS::_init_alloc()
   }
   logger->set(l_bluefs_wal_alloc_unit, wal_alloc_size);
 
+
+  uint64_t shared_alloc_size = cct->_conf->bluefs_shared_alloc_size;
+  if (shared_alloc && shared_alloc->a) {
+    uint64_t unit = shared_alloc->a->get_block_size();
+    shared_alloc_size = std::max(
+      unit,
+      shared_alloc_size);
+    ceph_assert(0 == p2phase(shared_alloc_size, unit));
+  }
   if (bdev[BDEV_SLOW]) {
     alloc_size[BDEV_DB] = cct->_conf->bluefs_alloc_size;
-    alloc_size[BDEV_SLOW] = cct->_conf->bluefs_shared_alloc_size;
-    logger->set(l_bluefs_db_alloc_unit, cct->_conf->bluefs_alloc_size);
-    logger->set(l_bluefs_main_alloc_unit, cct->_conf->bluefs_shared_alloc_size);
+    alloc_size[BDEV_SLOW] = shared_alloc_size;
   } else {
-    alloc_size[BDEV_DB] = cct->_conf->bluefs_shared_alloc_size;
-    logger->set(l_bluefs_main_alloc_unit, 0);
-    logger->set(l_bluefs_db_alloc_unit, cct->_conf->bluefs_shared_alloc_size);
+    alloc_size[BDEV_DB] = shared_alloc_size;
+    alloc_size[BDEV_SLOW] = 0;
   }
+  logger->set(l_bluefs_db_alloc_unit, alloc_size[BDEV_DB]);
+  logger->set(l_bluefs_main_alloc_unit, alloc_size[BDEV_SLOW]);
   // new wal and db devices are never shared
   if (bdev[BDEV_NEWWAL]) {
     alloc_size[BDEV_NEWWAL] = cct->_conf->bluefs_alloc_size;
@@ -718,13 +726,13 @@ void BlueFS::_init_alloc()
       continue;
     }
     ceph_assert(bdev[id]->get_size());
-    ceph_assert(alloc_size[id]);
     if (is_shared_alloc(id)) {
       dout(1) << __func__ << " shared, id " << id << std::hex
               << ", capacity 0x" << bdev[id]->get_size()
               << ", block size 0x" << alloc_size[id]
               << std::dec << dendl;
     } else {
+      ceph_assert(alloc_size[id]);
       std::string name = "bluefs-";
       const char* devnames[] = { "wal","db","slow" };
       if (id <= BDEV_SLOW)

From b0cb41a761dd7c32bde8e29eb12d44519a2988a5 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 15 Sep 2023 20:35:13 +0300
Subject: [PATCH 0900/2492] os/bluestore: get rid off resulting lba alignment
 in allocators

Fixes: https://tracker.ceph.com/issues/62815

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/AvlAllocator.cc              |   6 +-
 src/os/bluestore/BtreeAllocator.cc            |   6 +-
 src/os/bluestore/StupidAllocator.cc           |  32 +---
 src/os/bluestore/StupidAllocator.h            |   4 -
 src/os/bluestore/fastbmap_allocator_impl.cc   |  16 +-
 src/test/objectstore/Allocator_test.cc        |   3 +-
 .../objectstore/fastbmap_allocator_test.cc    | 181 +++++++++++++-----
 src/test/objectstore/store_test.cc            |   6 +-
 8 files changed, 151 insertions(+), 103 deletions(-)

diff --git a/src/os/bluestore/AvlAllocator.cc b/src/os/bluestore/AvlAllocator.cc
index 1a170d13fdb9..8de249e978cb 100644
--- a/src/os/bluestore/AvlAllocator.cc
+++ b/src/os/bluestore/AvlAllocator.cc
@@ -39,7 +39,7 @@ uint64_t AvlAllocator::_pick_block_after(uint64_t *cursor,
   uint64_t search_bytes = 0;
   auto rs_start = range_tree.lower_bound(range_t{*cursor, size}, compare);
   for (auto rs = rs_start; rs != range_tree.end(); ++rs) {
-    uint64_t offset = p2roundup(rs->start, align);
+    uint64_t offset = rs->start;
     *cursor = offset + size;
     if (offset + size <= rs->end) {
       return offset;
@@ -59,7 +59,7 @@ uint64_t AvlAllocator::_pick_block_after(uint64_t *cursor,
   }
   // If we reached end, start from beginning till cursor.
   for (auto rs = range_tree.begin(); rs != rs_start; ++rs) {
-    uint64_t offset = p2roundup(rs->start, align);
+    uint64_t offset = rs->start;
     *cursor = offset + size;
     if (offset + size <= rs->end) {
       return offset;
@@ -82,7 +82,7 @@ uint64_t AvlAllocator::_pick_block_fits(uint64_t size,
   const auto compare = range_size_tree.key_comp();
   auto rs_start = range_size_tree.lower_bound(range_t{0, size}, compare);
   for (auto rs = rs_start; rs != range_size_tree.end(); ++rs) {
-    uint64_t offset = p2roundup(rs->start, align);
+    uint64_t offset = rs->start;
     if (offset + size <= rs->end) {
       return offset;
     }
diff --git a/src/os/bluestore/BtreeAllocator.cc b/src/os/bluestore/BtreeAllocator.cc
index 2071603a2197..2647b8759920 100644
--- a/src/os/bluestore/BtreeAllocator.cc
+++ b/src/os/bluestore/BtreeAllocator.cc
@@ -25,7 +25,7 @@ uint64_t BtreeAllocator::_pick_block_after(uint64_t *cursor,
 {
   auto rs_start = range_tree.lower_bound(*cursor);
   for (auto rs = rs_start; rs != range_tree.end(); ++rs) {
-    uint64_t offset = p2roundup(rs->first, align);
+    uint64_t offset = rs->first;
     if (offset + size <= rs->second) {
       *cursor = offset + size;
       return offset;
@@ -37,7 +37,7 @@ uint64_t BtreeAllocator::_pick_block_after(uint64_t *cursor,
   }
   // If we reached end, start from beginning till cursor.
   for (auto rs = range_tree.begin(); rs != rs_start; ++rs) {
-    uint64_t offset = p2roundup(rs->first, align);
+    uint64_t offset = rs->first;
     if (offset + size <= rs->second) {
       *cursor = offset + size;
       return offset;
@@ -53,7 +53,7 @@ uint64_t BtreeAllocator::_pick_block_fits(uint64_t size,
   // the needs
   auto rs_start = range_size_tree.lower_bound(range_value_t{0,size});
   for (auto rs = rs_start; rs != range_size_tree.end(); ++rs) {
-    uint64_t offset = p2roundup(rs->start, align);
+    uint64_t offset = rs->start;
     if (offset + size <= rs->start + rs->size) {
       return offset;
     }
diff --git a/src/os/bluestore/StupidAllocator.cc b/src/os/bluestore/StupidAllocator.cc
index 550024e67e77..8f74a499ed31 100644
--- a/src/os/bluestore/StupidAllocator.cc
+++ b/src/os/bluestore/StupidAllocator.cc
@@ -52,20 +52,6 @@ void StupidAllocator::_insert_free(uint64_t off, uint64_t len)
   }
 }
 
-/// return the effective length of the extent if we align to alloc_unit
-uint64_t StupidAllocator::_aligned_len(
-  StupidAllocator::interval_set_t::iterator p,
-  uint64_t alloc_unit)
-{
-  uint64_t skew = p.get_start() % alloc_unit;
-  if (skew)
-    skew = alloc_unit - skew;
-  if (skew > p.get_len())
-    return 0;
-  else
-    return p.get_len() - skew;
-}
-
 int64_t StupidAllocator::allocate_int(
   uint64_t want_size, uint64_t alloc_unit, int64_t hint,
   uint64_t *offset, uint32_t *length)
@@ -89,7 +75,7 @@ int64_t StupidAllocator::allocate_int(
     for (bin = orig_bin; bin < (int)free.size(); ++bin) {
       p = free[bin].lower_bound(hint);
       while (p != free[bin].end()) {
-	if (_aligned_len(p, alloc_unit) >= want_size) {
+	if (p.get_len() >= want_size) {
 	  goto found;
 	}
 	++p;
@@ -102,7 +88,7 @@ int64_t StupidAllocator::allocate_int(
     p = free[bin].begin();
     auto end = hint ? free[bin].lower_bound(hint) : free[bin].end();
     while (p != end) {
-      if (_aligned_len(p, alloc_unit) >= want_size) {
+      if (p.get_len() >= want_size) {
 	goto found;
       }
       ++p;
@@ -114,7 +100,7 @@ int64_t StupidAllocator::allocate_int(
     for (bin = orig_bin; bin >= 0; --bin) {
       p = free[bin].lower_bound(hint);
       while (p != free[bin].end()) {
-	if (_aligned_len(p, alloc_unit) >= alloc_unit) {
+	if (p.get_len() >= alloc_unit) {
 	  goto found;
 	}
 	++p;
@@ -127,7 +113,7 @@ int64_t StupidAllocator::allocate_int(
     p = free[bin].begin();
     auto end = hint ? free[bin].lower_bound(hint) : free[bin].end();
     while (p != end) {
-      if (_aligned_len(p, alloc_unit) >= alloc_unit) {
+      if (p.get_len() >= alloc_unit) {
 	goto found;
       }
       ++p;
@@ -137,11 +123,9 @@ int64_t StupidAllocator::allocate_int(
   return -ENOSPC;
 
  found:
-  uint64_t skew = p.get_start() % alloc_unit;
-  if (skew)
-    skew = alloc_unit - skew;
-  *offset = p.get_start() + skew;
-  *length = std::min(std::max(alloc_unit, want_size), p2align((p.get_len() - skew), alloc_unit));
+  *offset = p.get_start();
+  *length = std::min(std::max(alloc_unit, want_size), p2align(p.get_len(), alloc_unit));
+
   if (cct->_conf->bluestore_debug_small_allocations) {
     uint64_t max =
       alloc_unit * (rand() % cct->_conf->bluestore_debug_small_allocations);
@@ -158,7 +142,7 @@ int64_t StupidAllocator::allocate_int(
 
   free[bin].erase(*offset, *length);
   uint64_t off, len;
-  if (*offset && free[bin].contains(*offset - skew - 1, &off, &len)) {
+  if (*offset && free[bin].contains(*offset - 1, &off, &len)) {
     int newbin = _choose_bin(len);
     if (newbin != bin) {
       ldout(cct, 30) << __func__ << " demoting 0x" << std::hex << off << "~" << len
diff --git a/src/os/bluestore/StupidAllocator.h b/src/os/bluestore/StupidAllocator.h
index 0d50d73f42af..443b091350e1 100644
--- a/src/os/bluestore/StupidAllocator.h
+++ b/src/os/bluestore/StupidAllocator.h
@@ -31,10 +31,6 @@ class StupidAllocator : public Allocator {
   unsigned _choose_bin(uint64_t len);
   void _insert_free(uint64_t offset, uint64_t len);
 
-  uint64_t _aligned_len(
-    interval_set_t::iterator p,
-    uint64_t alloc_unit);
-
 public:
   StupidAllocator(CephContext* cct,
                   int64_t size,
diff --git a/src/os/bluestore/fastbmap_allocator_impl.cc b/src/os/bluestore/fastbmap_allocator_impl.cc
index cea046c3ff22..4833b9d1a7b7 100644
--- a/src/os/bluestore/fastbmap_allocator_impl.cc
+++ b/src/os/bluestore/fastbmap_allocator_impl.cc
@@ -17,19 +17,9 @@ uint64_t AllocatorLevel::l2_allocs = 0;
 
 inline interval_t _align2units(uint64_t offset, uint64_t len, uint64_t min_length)
 {
-  interval_t res;
-  if (len >= min_length) {
-    res.offset = p2roundup(offset, min_length);
-    auto delta_off = res.offset - offset;
-    if (len > delta_off) {
-      res.length = len - delta_off;
-      res.length = p2align<uint64_t>(res.length, min_length);
-      if (res.length) {
-	return res;
-      }
-    }
-  }
-  return interval_t();
+  return len >= min_length ?
+    interval_t(offset, p2align<uint64_t>(len, min_length)) :
+    interval_t();
 }
 
 interval_t AllocatorLevel01Loose::_get_longest_from_l0(uint64_t pos0,
diff --git a/src/test/objectstore/Allocator_test.cc b/src/test/objectstore/Allocator_test.cc
index 8204179b5316..0e76c479002a 100644
--- a/src/test/objectstore/Allocator_test.cc
+++ b/src/test/objectstore/Allocator_test.cc
@@ -587,8 +587,7 @@ TEST_P(AllocTest, test_alloc_47883)
   PExtentVector extents;
   auto need = 0x3f980000;
   auto got = alloc->allocate(need, 0x10000, 0, (int64_t)0, &extents);
-  EXPECT_GT(got, 0);
-  EXPECT_EQ(got, 0x630000);
+  EXPECT_GE(got, 0x630000);
 }
 
 TEST_P(AllocTest, test_alloc_50656_best_fit)
diff --git a/src/test/objectstore/fastbmap_allocator_test.cc b/src/test/objectstore/fastbmap_allocator_test.cc
index c59531985050..710b3798f7a6 100644
--- a/src/test/objectstore/fastbmap_allocator_test.cc
+++ b/src/test/objectstore/fastbmap_allocator_test.cc
@@ -625,6 +625,8 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
     ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
 
     {
+      // Original free space disposition (start chunk, count):
+      // <NC/2, NC/2>
       size_t to_release = 2 * _1m + 0x1000;
       // release 2M + 4K at the beginning
       interval_vector_t r;
@@ -637,6 +639,8 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <0, 513>, <NC / 2, NC / 2>
       // allocate 4K within the deallocated range
       uint64_t allocated4 = 0;
       interval_vector_t a4;
@@ -652,79 +656,91 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
-      // allocate 1M - should go to the second 1M chunk
+      // Original free space disposition (start chunk, count):
+      // <1, 512>, <NC / 2, NC / 2>
+      // allocate 1M - should go to offset 4096
       uint64_t allocated4 = 0;
       interval_vector_t a4;
       al2.allocate_l2(_1m, _1m, &allocated4, &a4);
       ASSERT_EQ(a4.size(), 1u);
       ASSERT_EQ(allocated4, _1m);
-      ASSERT_EQ(a4[0].offset, _1m);
+      ASSERT_EQ(a4[0].offset, 4096);
       ASSERT_EQ(a4[0].length, _1m);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
-      ASSERT_EQ(bins_overall.size(), 3u);
-      ASSERT_EQ(bins_overall[0], 1u);
-      ASSERT_EQ(bins_overall[cbits((_1m - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits(_1m / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <257, 256>, <NC / 2, NC / 2>
       // and allocate yet another 8K within the deallocated range
       uint64_t allocated4 = 0;
       interval_vector_t a4;
       al2.allocate_l2(0x2000, 0x1000, &allocated4, &a4);
       ASSERT_EQ(a4.size(), 1u);
       ASSERT_EQ(allocated4, 0x2000u);
-      ASSERT_EQ(a4[0].offset, 0x1000u);
+      ASSERT_EQ(a4[0].offset, _1m + 0x1000u);
       ASSERT_EQ(a4[0].length, 0x2000u);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
-      ASSERT_EQ(bins_overall[0], 1u);
-      ASSERT_EQ(bins_overall[cbits((_1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
-      // release just allocated 1M
+      // Original free space disposition (start chunk, count):
+      // <259, 254>, <NC / 2, NC / 2>
+      // release 4K~1M
       interval_vector_t r;
-      r.emplace_back(_1m, _1m);
+      r.emplace_back(0x1000, _1m);
       al2.free_l2(r);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
-      ASSERT_EQ(bins_overall.size(), 2u);
-      ASSERT_EQ(bins_overall[cbits((2 * _1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall.size(), 3u);
+      //ASSERT_EQ(bins_overall[cbits((2 * _1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(_1m / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
-      // allocate 3M - should go to the second 1M chunk and @capacity/2
+      // Original free space disposition (start chunk, count):
+      // <1, 257>, <259, 254>, <NC / 2, NC / 2>
+      // allocate 3M - should go to the first 1M chunk and @capacity/2
       uint64_t allocated4 = 0;
       interval_vector_t a4;
       al2.allocate_l2(3 * _1m, _1m, &allocated4, &a4);
       ASSERT_EQ(a4.size(), 2u);
       ASSERT_EQ(allocated4, 3 * _1m);
-      ASSERT_EQ(a4[0].offset, _1m);
+      ASSERT_EQ(a4[0].offset, 0x1000);
       ASSERT_EQ(a4[0].length, _1m);
       ASSERT_EQ(a4[1].offset, capacity / 2);
       ASSERT_EQ(a4[1].length, 2 * _1m);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
-      ASSERT_EQ(bins_overall.size(), 3u);
-      ASSERT_EQ(bins_overall[0], 1u);
-      ASSERT_EQ(bins_overall[cbits((_1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits((num_chunks - 512) / 2) - 1], 1u);
     }
     {
-      // release allocated 1M in the second meg chunk except
+      // Original free space disposition (start chunk, count):
+      // <259, 254>, <NC / 2 - 512, NC / 2 - 512>
+      // release allocated 1M in the first meg chunk except
       // the first 4K chunk
       interval_vector_t r;
-      r.emplace_back(_1m + 0x1000, _1m);
+      r.emplace_back(0x1000, _1m);
       al2.free_l2(r);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
       ASSERT_EQ(bins_overall.size(), 3u);
       ASSERT_EQ(bins_overall[cbits(_1m / 0x1000) - 1], 1u);
-      ASSERT_EQ(bins_overall[cbits((_1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits((num_chunks - 512) / 2) - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <1, 256>, <259, 254>, <NC / 2 - 512, NC / 2 - 512>
       // release 2M @(capacity / 2)
       interval_vector_t r;
       r.emplace_back(capacity / 2, 2 * _1m);
@@ -733,10 +749,12 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       al2.collect_stats(bins_overall);
       ASSERT_EQ(bins_overall.size(), 3u);
       ASSERT_EQ(bins_overall[cbits(_1m / 0x1000) - 1], 1u);
-      ASSERT_EQ(bins_overall[cbits((_1m - 0x3000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits((num_chunks) / 2) - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <1, 256>, <259, 254>, <NC / 2, NC / 2>
       // allocate 4x512K - should go to the second halves of
       // the first and second 1M chunks and @(capacity / 2)
       uint64_t allocated4 = 0;
@@ -744,51 +762,54 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       al2.allocate_l2(2 * _1m, _1m / 2, &allocated4, &a4);
       ASSERT_EQ(a4.size(), 3u);
       ASSERT_EQ(allocated4, 2 * _1m);
-      ASSERT_EQ(a4[0].offset, _1m / 2);
+      ASSERT_EQ(a4[1].offset, 0x1000);
+      ASSERT_EQ(a4[1].length, _1m);
+      ASSERT_EQ(a4[0].offset, _1m + 0x3000);
       ASSERT_EQ(a4[0].length, _1m / 2);
-      ASSERT_EQ(a4[1].offset, _1m + _1m / 2);
-      ASSERT_EQ(a4[1].length, _1m / 2);
       ASSERT_EQ(a4[2].offset, capacity / 2);
-      ASSERT_EQ(a4[2].length, _1m);
+      ASSERT_EQ(a4[2].length, _1m / 2);
 
       bins_overall.clear();
       al2.collect_stats(bins_overall);
-      ASSERT_EQ(bins_overall.size(), 3u);
-      ASSERT_EQ(bins_overall[0], 1u);
-      // below we have 512K - 4K & 512K - 12K chunks which both fit into
-      // the same bin = 6
-      ASSERT_EQ(bins_overall[6], 2u);
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000 - 0x80000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits((num_chunks - 256) / 2) - 1], 1u);
 
     }
     {
-      // cleanup first 2M except except the last 4K chunk
+      // Original free space disposition (start chunk, count):
+      // <387, 126>, <NC / 2 + 128, NC / 2 - 128>
+      // cleanup first 1536K except the last 4K chunk
       interval_vector_t r;
-      r.emplace_back(0, 2 * _1m - 0x1000);
+      r.emplace_back(0, _1m + _1m / 2 - 0x1000);
       al2.free_l2(r);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
 
       ASSERT_EQ(bins_overall.size(), 3u);
-      ASSERT_EQ(bins_overall[0], 1u);
-      ASSERT_EQ(bins_overall[cbits((_2m - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m + _1m / 2 - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000 - 0x80000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits((num_chunks - 256) / 2) - 1], 1u);
     }
     {
-      // release 2M @(capacity / 2)
+      // Original free space disposition (start chunk, count):
+      // <0, 383> <387, 126>, <NC / 2 + 128, NC / 2 - 128>
+      // release 512K @(capacity / 2)
       interval_vector_t r;
-      r.emplace_back(capacity / 2, 2 * _1m);
+      r.emplace_back(capacity / 2, _1m / 2);
       al2.free_l2(r);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
 
       ASSERT_EQ(bins_overall.size(), 3u);
-      ASSERT_EQ(bins_overall[0], 1u);
-      ASSERT_EQ(bins_overall[cbits((_2m - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m + _1m / 2 - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000 - 0x80000) / 0x1000) - 1], 1u);
       ASSERT_EQ(bins_overall[cbits(num_chunks / 2) - 1], 1u);
     }
     {
-      // allocate 132M using 4M granularity should go to (capacity / 2)
+      // Original free space disposition (start chunk, count):
+      // <0, 383> <387, 126>, <NC / 2, NC / 2>
+      // allocate 132M (=33792*4096) = using 4M granularity should go to (capacity / 2)
       uint64_t allocated4 = 0;
       interval_vector_t a4;
       al2.allocate_l2(132 * _1m, 4 * _1m , &allocated4, &a4);
@@ -799,24 +820,40 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       bins_overall.clear();
       al2.collect_stats(bins_overall);
       ASSERT_EQ(bins_overall.size(), 3u);
+      ASSERT_EQ(bins_overall[cbits((_1m + _1m / 2 - 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits((_1m - 0x2000 - 0x80000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2 - 33792)  - 1], 1u);
     }
     {
-      // cleanup left 4K chunk in the first 2M
+      // Original free space disposition (start chunk, count):
+      // <0, 383> <387, 126>, <NC / 2 + 33792, NC / 2 - 33792>
+      // cleanup remaining 4*4K chunks in the first 2M
       interval_vector_t r;
-      r.emplace_back(2 * _1m - 0x1000, 0x1000);
+      r.emplace_back(383 * 4096, 4 * 0x1000);
       al2.free_l2(r);
       bins_overall.clear();
       al2.collect_stats(bins_overall);
 
       ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits((2 * _1m + 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2 - 33792)  - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <0, 513>, <NC / 2 + 33792, NC / 2 - 33792>
       // release 132M @(capacity / 2)
       interval_vector_t r;
       r.emplace_back(capacity / 2, 132 * _1m);
       al2.free_l2(r);
+      bins_overall.clear();
+      al2.collect_stats(bins_overall);
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits((2 * _1m + 0x1000) / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2)  - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      // <0, 513>, <NC / 2, NC / 2>
       // allocate 132M using 2M granularity should go to the first chunk and to
       // (capacity / 2)
       uint64_t allocated4 = 0;
@@ -827,14 +864,31 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       ASSERT_EQ(a4[0].length, 2 * _1m);
       ASSERT_EQ(a4[1].offset, capacity / 2);
       ASSERT_EQ(a4[1].length, 130 * _1m);
+
+      bins_overall.clear();
+      al2.collect_stats(bins_overall);
+
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits(0)], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2 - 33792)  - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      //  <512, 1>, <NC / 2 + 33792, NC / 2 - 33792>
       // release 130M @(capacity / 2)
       interval_vector_t r;
       r.emplace_back(capacity / 2, 132 * _1m);
       al2.free_l2(r);
+      bins_overall.clear();
+      al2.collect_stats(bins_overall);
+
+      ASSERT_EQ(bins_overall.size(), 2u);
+      ASSERT_EQ(bins_overall[cbits(0)], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2)  - 1], 1u);
     }
     {
+      // Original free space disposition (start chunk, count):
+      //  <512,1>, <NC / 2, NC / 2>
       // release 4K~16K
       // release 28K~32K
       // release 68K~24K
@@ -843,21 +897,46 @@ TEST(TestAllocatorLevel01, test_l2_contiguous_alignment)
       r.emplace_back(0x7000, 0x8000);
       r.emplace_back(0x11000, 0x6000);
       al2.free_l2(r);
+
+      bins_overall.clear();
+      al2.collect_stats(bins_overall);
+
+      ASSERT_EQ(bins_overall.size(), 4u);
+      ASSERT_EQ(bins_overall[cbits(0)], 1u);
+      ASSERT_EQ(bins_overall[cbits(0x4000 / 0x1000) - 1], 2u); // accounts both 0x4000 & 0x6000
+      ASSERT_EQ(bins_overall[cbits(0x8000 / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2)  - 1], 1u);
     }
     {
-      // allocate 32K using 16K granularity - should bypass the first
-      // unaligned extent, use the second free extent partially given
-      // the 16K alignment and then fallback to capacity / 2
+      // Original free space disposition (start chunk, count):
+      //  <1, 4>, <7, 8>, <17, 6> <512,1>, <NC / 2, NC / 2>
+      // allocate 80K using 16K granularity
       uint64_t allocated4 = 0;
       interval_vector_t a4;
-      al2.allocate_l2(0x8000, 0x4000, &allocated4, &a4);
-      ASSERT_EQ(a4.size(), 2u);
-      ASSERT_EQ(a4[0].offset, 0x8000u);
-      ASSERT_EQ(a4[0].length, 0x4000u);
-      ASSERT_EQ(a4[1].offset, capacity / 2);
+      al2.allocate_l2(0x14000, 0x4000, &allocated4, &a4);
+
+      ASSERT_EQ(a4.size(), 4);
+      ASSERT_EQ(a4[1].offset, 0x1000u);
       ASSERT_EQ(a4[1].length, 0x4000u);
-    }
+      ASSERT_EQ(a4[0].offset, 0x7000u);
+      ASSERT_EQ(a4[0].length, 0x8000u);
+      ASSERT_EQ(a4[2].offset, 0x11000u);
+      ASSERT_EQ(a4[2].length, 0x4000u);
+      ASSERT_EQ(a4[3].offset, capacity / 2);
+      ASSERT_EQ(a4[3].length, 0x4000u);
+
+      bins_overall.clear();
+      al2.collect_stats(bins_overall);
 
+      ASSERT_EQ(bins_overall.size(), 3u);
+      ASSERT_EQ(bins_overall[cbits(0)], 1u);
+      ASSERT_EQ(bins_overall[cbits(0x2000 / 0x1000) - 1], 1u);
+      ASSERT_EQ(bins_overall[cbits(num_chunks / 2 - 1)  - 1], 1u);
+    }
+    {
+      // Original free space disposition (start chunk, count):
+      //  <21, 2> <512,1>, <NC / 2 + 1, NC / 2 - 1>
+    }
   }
   std::cout << "Done L2 cont aligned" << std::endl;
 }
@@ -913,7 +992,7 @@ TEST(TestAllocatorLevel01, test_4G_alloc_bug2)
     al2.allocate_l2(0x3e000000, _1m, &allocated4, &a4);
     ASSERT_EQ(a4.size(), 2u);
     ASSERT_EQ(allocated4, 0x3e000000u);
-    ASSERT_EQ(a4[0].offset, 0x5fed00000u);
+    ASSERT_EQ(a4[0].offset, 0x5fec30000u);
     ASSERT_EQ(a4[0].length, 0x1300000u);
     ASSERT_EQ(a4[1].offset, 0x628000000u);
     ASSERT_EQ(a4[1].length, 0x3cd00000u);
diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 03dc1a87e1fc..a424e7786ddf 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -9607,9 +9607,9 @@ TEST_P(StoreTestSpecificAUSize, BluestoreRepairSharedBlobTest) {
     string key;
     _key_encode_u64(1, &key);
     bluestore_shared_blob_t sb(1);
-    sb.ref_map.get(0x2000, block_size);
-    sb.ref_map.get(0x4000, block_size);
-    sb.ref_map.get(0x4000, block_size);
+    sb.ref_map.get(0x822000, block_size);
+    sb.ref_map.get(0x824000, block_size);
+    sb.ref_map.get(0x824000, block_size);
     bufferlist bl;
     encode(sb, bl);
     bstore->inject_broken_shared_blob_key(key, bl);

From 47858c5ad24f037616591316a8ad95a304860739 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 4 Dec 2023 13:20:02 -0500
Subject: [PATCH 0901/2492] cephadm: simplify loading jinja2 templates from
 zipapp

The saga of failures in teuthology brought on by the combination of
zipapp and jinja2 template loading continues in this episode. Remove
some of the path handling that was taken from the original jinja2
implementation and try to replace it with something inherently simpler
such that we always construct paths relative to the zip archive and pass
that to the zipimporter. The hope here is that by doing fewer
manipulations of the "path" we are more likely to match the items
tracked inside the zipimporter class.

We do not worry about some of the concerns that regular jinja2 has as we
never expect cephadm to be run on something other than linux and we have
control over what sources we expect to load templates from.  We do take
one precaution and that is to reject any paths that contain "." and ".."
as components. Less to avoid malicious attempts to read files it should
not (but that would be nice) and more to nudge toward simple template
references recorded in our Templates enum.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 37 +++++++++-------------------
 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index ceef32ff9fed..3e852b2d397d 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -34,27 +34,18 @@ def __init__(
         self,
         template: str,
         *,
-        path: str = '',
         relative_path: str = '',
-        archive_norm_path: str = '',
-        archive_path: str = ''
+        archive_path: str = '',
     ) -> None:
         super().__init__(template)
-        self.path = path
         self.relative_path = relative_path
-        self.archive_norm_path = archive_norm_path
         self.archive_path = archive_path
 
     def __str__(self) -> str:
-        msg = self.message
-        msg2 = ''
-        if self.path or self.relative_path:
-            msg2 += f' path [{self.path!r}, rel={self.relative_path!r}] not found'
-        if self.archive_norm_path or self.archive_path:
-            msg2 += f' in [{self.archive_norm_path!r}, orig={self.archive_path!r}]'
-        if msg2:
-            msg2 = ':' + msg2
-        return f'{msg}{msg2}'
+        return (
+            f'{self.message}: path {self.relative_path!r}'
+            f' not found in {self.archive_path!r}'
+        )
 
 
 class _PackageLoader(jinja2.PackageLoader):
@@ -86,27 +77,23 @@ def get_source(
 
     def _get_archive_source(self, template: str) -> Tuple[str, str, None]:
         assert isinstance(self._loader, zipimport.zipimporter)
-        path = arelpath = os.path.normpath(
-            posixpath.join(
-                self._template_root,
-                *jinja2.loaders.split_template_path(template)
-            )
+        arelpath = posixpath.join(
+            self.package_name, self.package_path, template
         )
-        archive_path = os.path.normpath(self._loader.archive)
-        if arelpath.startswith(archive_path + '/'):
-            plen = len(archive_path) + 1
-            arelpath = arelpath[plen:]
+        if any(p == '.' or p == '..' for p in arelpath.split(posixpath.sep)):
+            raise ValueError('template path contains invalid components')
         try:
             source = cast(bytes, self._loader.get_data(arelpath))
         except OSError as e:
             not_found = TemplateNotFoundInZipApp(
                 template,
-                path=path,
                 relative_path=arelpath,
-                archive_norm_path=archive_path,
                 archive_path=self._loader.archive,
             )
             raise not_found from e
+        path = os.path.normpath(
+            posixpath.join(self._loader.archive, arelpath)
+        )
         return source.decode(self.encoding), path, None
 
 

From 269973e919288fe4384f94106c07f9af5b5ff3d4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 29 Nov 2023 15:44:04 -0500
Subject: [PATCH 0902/2492] script/build-integration-branch: add a better error
 when github api access fails

A mysterious `assert(r.ok)` appears after running the script causing me
to read source code. I don't want to read source code every time the
script fails. ;-)

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/build-integration-branch | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/script/build-integration-branch b/src/script/build-integration-branch
index 5d7c37e56fcc..b5f53aa21017 100755
--- a/src/script/build-integration-branch
+++ b/src/script/build-integration-branch
@@ -65,7 +65,11 @@ url = baseurl.format(label=label,
                      repo=repo)
 r = requests.get(url,
                  headers={'Authorization': 'token %s' % token})
-assert(r.ok)
+if not r.ok:
+    print("Failed to access github api")
+    print("(Do you have a valid, unexpired github api token?)")
+    sys.exit(1)
+
 j = json.loads(r.text or r.content)
 print("--- found %d issues tagged with %s" % (len(j), label))
 

From 6e37d99eb14137bf4fa34dd89e92cdb98037875b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 29 Nov 2023 15:45:15 -0500
Subject: [PATCH 0903/2492] script/build-integration-branch: support fetching
 github token from netrc

I constantly forget to update the special place that this script wants
to read the github token from. Other tools I use can read from the .netrc file.
Try reading the token from netrc before falling back to this script's
traditional location.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/build-integration-branch | 33 +++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/script/build-integration-branch b/src/script/build-integration-branch
index b5f53aa21017..c4e1d604f0c8 100755
--- a/src/script/build-integration-branch
+++ b/src/script/build-integration-branch
@@ -7,7 +7,12 @@ Builds integration branches. Something similar to
   >   git pull b
   > done
 
-Requires `~/.github_token`.
+Requires either `~/.github_token` containing ONLY the token
+OR adding an entry like the following to `~/.netrc`:
+  ```
+  machine github.com
+  password ghp_E7ln0tAR34LtoK3nIsw34RyTve2moM3BvK
+  ```
 
 
 Usage:
@@ -24,6 +29,7 @@ import os
 import requests
 import sys
 import time
+import netrc
 
 from subprocess import call, check_output
 from urllib.parse import urljoin
@@ -52,9 +58,28 @@ except ImportError:
     assert len(sys.argv) == 2
     branch = label + postfix
 
-
-with open(os.path.expanduser('~/.github_token')) as myfile:
-    token = myfile.readline().strip()
+token = ''
+try:
+    nrc = netrc.netrc()
+    nrauth = nrc.authenticators("api.github.com")
+    if nrauth:
+        token = nrauth[2]
+    if not token:
+        nrauth = nrc.authenticators("github.com")
+        if nrauth:
+            token = nrauth[2]
+except FileNotFoundError:
+    pass
+if not token:
+    try:
+        with open(os.path.expanduser('~/.github_token')) as myfile:
+            token = myfile.readline().strip()
+    except FileNotFoundError:
+        pass
+if not token:
+    print('No github api access token found')
+    print('  Add a token to .netrc for [api.]github.com')
+    print('  OR add a token to $HOME/.github_token')
 
 # get prs
 baseurl = urljoin('https://api.github.com',

From bc39adceba611b00c04a894039b4c3280434d3fa Mon Sep 17 00:00:00 2001
From: sinashan <sinatak1373@live.com>
Date: Mon, 4 Dec 2023 14:02:54 -0500
Subject: [PATCH 0904/2492] doc: Fixes two typos and grammatical errors.
 Signed-off-by: Sina Ahmadi <sinatak1373@live.com>

---
 doc/start/hardware-recommendations.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index a63b5a457964..c1bff769b289 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -210,7 +210,7 @@ Many "slow OSD" issues (when they are not attributable to hardware failure)
 arise from running an operating system and multiple OSDs on the same drive.
 Also be aware that today's 22TB HDD uses the same SATA interface as a
 3TB HDD from ten years ago: more than seven times the data to squeeze
-through the same same interface.  For this reason, when using HDDs for
+through the same interface.  For this reason, when using HDDs for
 OSDs, drives larger than 8TB may be best suited for storage of large
 files / objects that are not at all performance-sensitive.
 
@@ -262,7 +262,7 @@ enterprise-class drives are best for production use, as they feature power
 loss protection and increased durability compared to client (desktop) SKUs
 that are intended for much lighter and intermittent duty cycles.
 
-SSDs were historically been cost prohibitive for object storage, but
+SSDs have historically been cost prohibitive for object storage, but
 QLC SSDs are closing the gap, offering greater density with lower power
 consumption and less power spent on cooling. Also, HDD OSDs may see a
 significant write latency improvement by offloading WAL+DB onto an SSD.

From 86b855141afa0fd6d82b9929ee0bb10f70ae2d50 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 4 Dec 2023 16:34:12 -0500
Subject: [PATCH 0905/2492] rgw/rados: rgw_list_pool() loops until 'max'
 matches

instead of looping up to 'max' oids, continue looping until the filter
actually matches 'max' entries

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_tools.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index bf78b9bb22e2..269c790e401e 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -517,9 +517,7 @@ int rgw_list_pool(const DoutPrefixProvider *dpp,
   if (iter == ioctx.nobjects_end())
     return -ENOENT;
 
-  uint32_t i;
-
-  for (i = 0; i < max && iter != ioctx.nobjects_end(); ++i, ++iter) {
+  for (; oids->size() < max && iter != ioctx.nobjects_end(); ++iter) {
     string oid = iter->get_oid();
     ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl;
 

From 4bd2d6bcf19265863a72a40396300d55137f7d09 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 5 Dec 2023 11:13:16 +0530
Subject: [PATCH 0906/2492] cephfs: add a release note for PR #53899

With merging of PR #53899 it is mandatory now for CephFS to be offline
and refuse_client_session to be set before an attempt to rename the
CephFS is made. Add a release note for this.

Fixes: https://github.com/ceph/ceph/pull/53899
Fixes: https://tracker.ceph.com/issues/63154
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 PendingReleaseNotes | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 9b3dfede8bbd..c5ad5e0649f8 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -57,6 +57,9 @@
   level storage operator (like Rook) to recreate the missing file system.
   See https://docs.ceph.com/en/latest/cephfs/administration/#file-systems
   docs for more information.
+* CephFS: Before running the command "ceph fs rename", the CephFS to be
+  renamed must be taken offline and "refuse_client_session" must be set for
+  it.
 * RADOS: A POOL_APP_NOT_ENABLED health warning will now be reported if
   the application is not enabled for the pool irrespective of whether
   the pool is in use or not. Always tag a pool with an application

From f5e3f263724d04be181225d9e24cbe2f4669e8bc Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 29 Nov 2023 12:28:51 +0100
Subject: [PATCH 0907/2492] test/librbd: add
 DiffIterateTest.DiffIterateDeterministic{,PP}

scribble()-based DiffIterate tests are too weak: at least two
regressions that should been caught by DiffIterate.DiffIterate or
DiffIterate.DiffIterateStress were missed [1][2].  Aside from the
randomness which can be both a good and a bad thing, asserts there
ensure only that the returned diff covers all changes that were made.
If the returned diff is too excessive or otherwise bogus, this isn't
detected [3].

Add a deterministic test to systematically cover the most common cases
that don't involve discards.  A similar test for discards will be added
with the fix for [4].

Comment out debug log in vector_iterate_cb() like it's done in
iterate_cb().

[1] https://tracker.ceph.com/issues/50787
[2] https://tracker.ceph.com/issues/63654
[3] https://tracker.ceph.com/issues/63719
[4] https://tracker.ceph.com/issues/53897

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 263 ++++++++++++++++++++++++++++++++-
 1 file changed, 262 insertions(+), 1 deletion(-)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index f4bb74fe397d..372340c7b8ef 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7501,12 +7501,273 @@ ostream& operator<<(ostream & o, const diff_extent& e) {
 
 int vector_iterate_cb(uint64_t off, size_t len, int exists, void *arg)
 {
-  cout << "iterate_cb " << off << "~" << len << std::endl;
+  //cout << "iterate_cb " << off << "~" << len << std::endl;
   vector<diff_extent> *diff = static_cast<vector<diff_extent> *>(arg);
   diff->push_back(diff_extent(off, len, exists, 0));
   return 0;
 }
 
+TYPED_TEST(DiffIterateTest, DiffIterateDeterministic)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_STRIPINGV2));
+
+  rados_ioctx_t ioctx;
+  ASSERT_EQ(0, rados_ioctx_create(this->_cluster, this->m_pool_name.c_str(),
+                                  &ioctx));
+
+  rbd_image_t image;
+  int order = 22;
+  std::string name = this->get_temp_image_name();
+  uint64_t size = 20 << 20;
+
+  ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
+
+  uint64_t object_size = 0;
+  if (this->whole_object) {
+    object_size = 1 << order;
+  }
+
+  std::vector<diff_extent> extents;
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
+
+  std::string buf(256, '1');
+  ASSERT_EQ(256, rbd_write(image, 0, 256, buf.data()));
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
+
+  ASSERT_EQ(256, rbd_write(image, 1 << order, 256, buf.data()));
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  ASSERT_EQ(0, rbd_snap_create(image, "snap3"));
+
+  // 1. beginning of time -> HEAD
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 2. snap1 -> HEAD
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap1", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 3. snap2 -> HEAD
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap2", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  // 4. snap3 -> HEAD
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap3", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, rbd_snap_set(image, "snap3"));
+
+  // 5. beginning of time -> snap3
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 6. snap1 -> snap3
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap1", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 7. snap2 -> snap3
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap2", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, rbd_snap_set(image, "snap2"));
+
+  // 8. beginning of time -> snap2
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  // 9. snap1 -> snap2
+  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap1", 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, rbd_snap_set(image, "snap1"));
+
+  // 10. beginning of time -> snap1
+  ASSERT_EQ(0, rbd_diff_iterate2(image, NULL, 0, size, true, this->whole_object,
+                                 vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_PASSED(this->validate_object_map, image);
+}
+
+TYPED_TEST(DiffIterateTest, DiffIterateDeterministicPP)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_STRIPINGV2));
+
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
+
+  librbd::RBD rbd;
+  librbd::Image image;
+  int order = 22;
+  std::string name = this->get_temp_image_name();
+  uint64_t size = 20 << 20;
+
+  ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+
+  uint64_t object_size = 0;
+  if (this->whole_object) {
+    object_size = 1 << order;
+  }
+
+  std::vector<diff_extent> extents;
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_EQ(0, image.snap_create("snap1"));
+
+  ceph::bufferlist bl;
+  bl.append(std::string(256, '1'));
+  ASSERT_EQ(256, image.write(0, 256, bl));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(0, image.snap_create("snap2"));
+
+  ASSERT_EQ(256, image.write(1 << order, 256, bl));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  ASSERT_EQ(0, image.snap_create("snap3"));
+
+  // 1. beginning of time -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 2. snap1 -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 3. snap2 -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  // 4. snap3 -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2("snap3", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, image.snap_set("snap3"));
+
+  // 5. beginning of time -> snap3
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 6. snap1 -> snap3
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 7. snap2 -> snap3
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, image.snap_set("snap2"));
+
+  // 8. beginning of time -> snap2
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  // 9. snap1 -> snap2
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, image.snap_set("snap1"));
+
+  // 10. beginning of time -> snap1
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_PASSED(this->validate_object_map, image);
+}
+
 TYPED_TEST(DiffIterateTest, DiffIterateDiscard)
 {
   librados::IoCtx ioctx;

From 356ac6a5477061389e673053f178d1ec6a9b9427 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 1 Dec 2023 18:54:19 +0100
Subject: [PATCH 0908/2492] test/librbd: drop TestLibRBD.SnapDiff

This was added to integration test [1], separate from the fix which
went in only with unit test adjustments.  It's duplicated by several
cases in DiffIterateTest.DiffIterateDeterministic now.  Specifically,
the issue could be reproduced by any of:

    (3) snap2 -> HEAD
    (4) snap3 -> HEAD
    (7) snap2 -> snap3

[1] https://tracker.ceph.com/issues/50787

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 55 ----------------------------------
 1 file changed, 55 deletions(-)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index 372340c7b8ef..59ef7aa19b9f 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7358,61 +7358,6 @@ interval_set<uint64_t> round_diff_interval(const interval_set<uint64_t>& diff,
   return rounded_diff;
 }
 
-TEST_F(TestLibRBD, SnapDiff)
-{
-  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
-
-  rados_ioctx_t ioctx;
-  rados_ioctx_create(_cluster, m_pool_name.c_str(), &ioctx);
-
-  rbd_image_t image;
-  int order = 0;
-  std::string image_name = get_temp_image_name();
-  uint64_t size = 100 << 20;
-  ASSERT_EQ(0, create_image(ioctx, image_name.c_str(), size, &order));
-  ASSERT_EQ(0, rbd_open(ioctx, image_name.c_str(), &image, nullptr));
-
-  char test_data[TEST_IO_SIZE + 1];
-  for (size_t i = 0; i < TEST_IO_SIZE; ++i) {
-    test_data[i] = (char) (rand() % (126 - 33) + 33);
-  }
-  test_data[TEST_IO_SIZE] = '\0';
-
-  ASSERT_PASSED(write_test_data, image, test_data, 0,
-                TEST_IO_SIZE, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
-
-  interval_set<uint64_t> diff;
-  ASSERT_EQ(0, rbd_diff_iterate2(image, nullptr, 0, size, true, true,
-                                 iterate_cb, &diff));
-  EXPECT_EQ(1 << order, diff.size());
-
-  ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
-  ASSERT_EQ(0, rbd_snap_create(image, "snap2"));
-
-  diff.clear();
-  ASSERT_EQ(0, rbd_diff_iterate2(image, nullptr, 0, size, true, true,
-                                 iterate_cb, &diff));
-  EXPECT_EQ(1 << order, diff.size());
-
-  diff.clear();
-  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap1", 0, size, true, true,
-                                 iterate_cb, &diff));
-  EXPECT_EQ(0, diff.size());
-
-  diff.clear();
-  ASSERT_EQ(0, rbd_diff_iterate2(image, "snap2", 0, size, true, true,
-                                 iterate_cb, &diff));
-  EXPECT_EQ(0, diff.size());
-
-  ASSERT_EQ(0, rbd_snap_remove(image, "snap1"));
-  ASSERT_EQ(0, rbd_snap_remove(image, "snap2"));
-
-  ASSERT_EQ(0, rbd_close(image));
-  ASSERT_EQ(0, rbd_remove(ioctx, image_name.c_str()));
-
-  rados_ioctx_destroy(ioctx);
-}
-
 template <typename T>
 class DiffIterateTest : public TestLibRBD {
 public:

From 93ff7fe6e4b1232a97f669f3dce6653cc08638a8 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 1 Dec 2023 18:29:12 +0100
Subject: [PATCH 0909/2492] test/librbd: drop
 DiffIterateTest.DiffIterateRegression6926

This was added to test [1].  It's duplicated by several cases in
DiffIterateTest.DiffIterateDeterministicPP now.  Specifically, the
issue could be reproduced by any of:

    (8) beginning of time -> snap2
    (9) snap1 -> snap2
    (10) beginning of time -> snap1

[1] https://tracker.ceph.com/issues/6926

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 44 ----------------------------------
 1 file changed, 44 deletions(-)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index 59ef7aa19b9f..d9e43b5e7364 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7861,50 +7861,6 @@ TYPED_TEST(DiffIterateTest, DiffIterateStress)
   ASSERT_PASSED(this->validate_object_map, image);
 }
 
-TYPED_TEST(DiffIterateTest, DiffIterateRegression6926)
-{
-  librados::IoCtx ioctx;
-  ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
-
-  librbd::RBD rbd;
-  librbd::Image image;
-  int order = 0;
-  std::string name = this->get_temp_image_name();
-  uint64_t size = 20 << 20;
-
-  ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
-  ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
-
-  uint64_t object_size = 0;
-  if (this->whole_object) {
-    object_size = 1 << order;
-  }
-  vector<diff_extent> extents;
-  ceph::bufferlist bl;
-
-  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(0u, extents.size());
-
-  ASSERT_EQ(0, image.snap_create("snap1"));
-  char data[256];
-  memset(data, 1, sizeof(data));
-  bl.append(data, 256);
-  ASSERT_EQ(256, image.write(0, 256, bl));
-
-  extents.clear();
-  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(1u, extents.size());
-  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
-
-  ASSERT_EQ(0, image.snap_set("snap1"));
-  extents.clear();
-  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(static_cast<size_t>(0), extents.size());
-}
-
 TYPED_TEST(DiffIterateTest, DiffIterateParent)
 {
   REQUIRE_FEATURE(RBD_FEATURE_LAYERING);

From c68b5af0fb639fccc89d26606c7924c6834bf606 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 5 Dec 2023 17:58:07 +0100
Subject: [PATCH 0910/2492] python-common: fix osdspec_affinity check

When no `service_id` is provided to service spec (osd) it results in
OSDs created with "osdspec_affinity" attribute set to a string
containing "None".

The DriveSelection class relies on the comparison of the actual
value of this attribute with the value of the service_id which has
the python type `None` in that case.

If any existing deployments were created without the service_id
attribute, we now have to support this case and make sure the check
won't filter out devices unexpectedly.

Fixes: https://tracker.ceph.com/issues/63729

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/python-common/ceph/deployment/drive_selection/selector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/python-common/ceph/deployment/drive_selection/selector.py b/src/python-common/ceph/deployment/drive_selection/selector.py
index 1b3bfbb4ee3c..86fd4616fdac 100644
--- a/src/python-common/ceph/deployment/drive_selection/selector.py
+++ b/src/python-common/ceph/deployment/drive_selection/selector.py
@@ -132,7 +132,7 @@ def assign_devices(self, device_filter):
                 other_osdspec_affinity = ''
                 for lv in disk.lvs:
                     if 'osdspec_affinity' in lv.keys():
-                        if lv['osdspec_affinity'] != self.spec.service_id:
+                        if lv['osdspec_affinity'] != str(self.spec.service_id):
                             other_osdspec_affinity = lv['osdspec_affinity']
                             break
                 if other_osdspec_affinity:

From a6dc3b6e69ee591b08943fd00aef4d3e2fad9964 Mon Sep 17 00:00:00 2001
From: Philipp Hufnagl <p.hufnagl@proxmox.com>
Date: Wed, 22 Nov 2023 12:31:41 +0100
Subject: [PATCH 0911/2492] osd: adding 'reef' to pending_require_osd_release

It appears adding 'reef' to pending_require_osd_release has been
forgotten. This PR adds it

Signed-off-by: Philipp Hufangl <p.hufnagl@proxmox.com>
(cherry picked from commit e443ef3562f9718b224acb550206497b5fdfe82e)
---
 src/osd/OSDMap.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 8f60b2f3c83c..a1f22c591de5 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -7410,6 +7410,10 @@ unsigned OSDMap::get_device_class_flags(int id) const
 
 std::optional<std::string> OSDMap::pending_require_osd_release() const
 {
+  if (HAVE_FEATURE(get_up_osd_features(), SERVER_REEF) &&
+      require_osd_release < ceph_release_t::reef) {
+    return "reef";
+  }
   if (HAVE_FEATURE(get_up_osd_features(), SERVER_QUINCY) &&
       require_osd_release < ceph_release_t::quincy) {
     return "quincy";

From 0179a8ea2f2a90f70d10aade9369e9c6a2a08cb0 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 5 Dec 2023 12:21:18 -0500
Subject: [PATCH 0912/2492] common: use inline for monostate dencoders

fix a 'multiple definition' error when included by multiple sources:

src/common/versioned_variant.h:31: multiple definition of `ceph::encode(std::monostate const&, ceph::buffer::v15_2_0::list&)';
rgw_main.cc.o:src/common/versioned_variant.h:31: first defined here

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/versioned_variant.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common/versioned_variant.h b/src/common/versioned_variant.h
index f7e46b7296e7..124c58839169 100644
--- a/src/common/versioned_variant.h
+++ b/src/common/versioned_variant.h
@@ -28,8 +28,8 @@
 namespace ceph {
 
 // null encoding for std::monostate
-void encode(const std::monostate&, bufferlist& bl) {}
-void decode(std::monostate&, bufferlist::const_iterator& p) {}
+inline void encode(const std::monostate&, bufferlist& bl) {}
+inline void decode(std::monostate&, bufferlist::const_iterator& p) {}
 
 // largest value that can be represented by `__u8 struct_v`
 inline constexpr size_t max_version = std::numeric_limits<__u8>::max();

From 2eb298c76acfe33a5431d07b25f7cfe767ef77ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 5 Dec 2023 18:26:33 +0100
Subject: [PATCH 0913/2492] doc/dev/release-checklist: enlist
 OSDMap::pending_require_osd_release()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 0af98ff3b38c..b2cd47258a51 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -92,6 +92,10 @@ Mon
 - [x] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
 - [x] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
 
+OSDMap
+------
+
+- [ ] src/osd/OSDMap.cc add release name mapping for `SERVER_X` in `pending_require_osd_release()`
 
 Code cleanup
 ------------

From 4d420e1f3036e912744eb9381d29f41e70b5768a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 5 Dec 2023 18:38:46 +0100
Subject: [PATCH 0914/2492] osd, doc: make
 OSDMap::pending_require_osd_release() aware about squid
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 2 +-
 src/osd/OSDMap.cc              | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index b2cd47258a51..dfef01156414 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -95,7 +95,7 @@ Mon
 OSDMap
 ------
 
-- [ ] src/osd/OSDMap.cc add release name mapping for `SERVER_X` in `pending_require_osd_release()`
+- [x] src/osd/OSDMap.cc add release name mapping for `SERVER_X` in `pending_require_osd_release()`
 
 Code cleanup
 ------------
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index a1f22c591de5..07da2ed6bde6 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -7410,6 +7410,10 @@ unsigned OSDMap::get_device_class_flags(int id) const
 
 std::optional<std::string> OSDMap::pending_require_osd_release() const
 {
+  if (HAVE_FEATURE(get_up_osd_features(), SERVER_SQUID) &&
+      require_osd_release < ceph_release_t::squid) {
+    return "squid";
+  }
   if (HAVE_FEATURE(get_up_osd_features(), SERVER_REEF) &&
       require_osd_release < ceph_release_t::reef) {
     return "reef";

From 66e9815d0c4a50e93e0aae5da9754d061f6e35a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 5 Dec 2023 19:07:47 +0100
Subject: [PATCH 0915/2492] doc/dev/release-checklist: mark ceph-container
 changes done
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Per https://github.com/ceph/ceph-container/pull/2174.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 doc/dev/release-checklists.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index dfef01156414..65952cba4505 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -131,12 +131,12 @@ ceph-container
 --------------
 In the `ceph/ceph-container.git` repo:
 
-- [ ] Add the release name to `Makefile`
-- [ ] Update `ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__` with the with the supported nfs-ganesha version
-- [ ] Update `contrib/build-push-ceph-container-imgs.sh` with the new release
-- [ ] Update `contrib/ceph-build-config.sh` with the release name
-- [ ] Update `contrib/common.sh` with supported version numbers
-- [ ] Update `maint-lib/ceph_version.sh` with the release name
+- [x] Add the release name to `Makefile`
+- [x] Update `ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__` with the with the supported nfs-ganesha version
+- [x] Update `contrib/build-push-ceph-container-imgs.sh` with the new release
+- [x] Update `contrib/ceph-build-config.sh` with the release name
+- [x] Update `contrib/common.sh` with supported version numbers
+- [x] Update `maint-lib/ceph_version.sh` with the release name
 
 See https://github.com/ceph/ceph-container/pull/2109 as an example for what to do.
 

From e80d560d97bd44f9a308916a099bf9391813d95b Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 5 Dec 2023 20:46:26 +0100
Subject: [PATCH 0916/2492] doc/radosgw: update link in rgw-cache.rst

Update link in doc/radosgw/rgw-cache.rst. The link updated here is a
link to all the Nginx configuration files. The old link was broken. This
update comes to us from an anonymous report on
https://pad.ceph.com/p/Report_Documentation_Bugs.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/rgw-cache.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/radosgw/rgw-cache.rst b/doc/radosgw/rgw-cache.rst
index 116db8ed4efe..fb6486405e85 100644
--- a/doc/radosgw/rgw-cache.rst
+++ b/doc/radosgw/rgw-cache.rst
@@ -90,7 +90,8 @@ $ sudo ln -sf /usr/local/openresty/bin/openresty /usr/bin/nginx
 
 Put in-place your Nginx configuration files and edit them according to your environment:
 
-All Nginx conf files are under: https://github.com/ceph/ceph/tree/main/examples/rgw/rgw-cache
+All Nginx conf files are under:
+https://github.com/ceph/ceph/tree/main/examples/rgw/rgw-cache
 
 `nginx.conf` should go to `/etc/nginx/nginx.conf`
 

From 07a4a9bdfc408be027d4bf78d63909efeff38a44 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 5 Dec 2023 16:12:56 -0500
Subject: [PATCH 0917/2492] osd/scrubber: fix signed comparison warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[681/1140] Building CXX object src/osd/CMakeFiles/osd.dir/scrubber/scrub_resources.cc.o
src/osd/scrubber/scrub_resources.cc: In member function ‘bool Scrub::ScrubResources::inc_scrubs_remote(pg_t)’:
src/osd/scrubber/scrub_resources.cc:84:18: warning: comparison of integer expressions of different signedness: ‘long unsigned int’ and ‘const int64_t’ {aka ‘const long int’} [-Wsign-compare]
   84 |   if (pre_op_cnt < conf->osd_max_scrubs) {
      |       ~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/osd/scrubber/scrub_resources.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/scrubber/scrub_resources.cc b/src/osd/scrubber/scrub_resources.cc
index 6203a2912aa9..dd9d31a15732 100644
--- a/src/osd/scrubber/scrub_resources.cc
+++ b/src/osd/scrubber/scrub_resources.cc
@@ -81,7 +81,7 @@ bool ScrubResources::inc_scrubs_remote(pg_t pgid)
   }
 
   auto pre_op_cnt = granted_reservations.size();
-  if (pre_op_cnt < conf->osd_max_scrubs) {
+  if (std::cmp_less(pre_op_cnt, conf->osd_max_scrubs)) {
     granted_reservations.insert(pgid);
     log_upwards(fmt::format(
 	"{}: pg[{}] reserved. Remote scrubs count changed from {} -> {} (max "

From e8d54e3c9faeddedc2890294556cd66095b83be4 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Thu, 9 Nov 2023 09:43:19 -0700
Subject: [PATCH 0918/2492] test/librbd: Add a stress test that reproduces a
 crash during discard journaling

See the comments in DiscardWithPruneWriteOverlap for details.

Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/test/librbd/CMakeLists.txt         |   3 +-
 src/test/librbd/journal/test_Stress.cc | 101 +++++++++++++++++++++++++
 src/test/librbd/test_main.cc           |   2 +
 3 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 src/test/librbd/journal/test_Stress.cc

diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt
index 0ae29b8bf185..c3f0edbea5d3 100644
--- a/src/test/librbd/CMakeLists.txt
+++ b/src/test/librbd/CMakeLists.txt
@@ -19,7 +19,8 @@ set(librbd_test
   test_Operations.cc
   test_Trash.cc
   journal/test_Entries.cc
-  journal/test_Replay.cc)
+  journal/test_Replay.cc
+  journal/test_Stress.cc)
 add_library(rbd_test STATIC ${librbd_test})
 target_link_libraries(rbd_test PRIVATE
   rbd_test_support
diff --git a/src/test/librbd/journal/test_Stress.cc b/src/test/librbd/journal/test_Stress.cc
new file mode 100644
index 000000000000..752ecf01f05e
--- /dev/null
+++ b/src/test/librbd/journal/test_Stress.cc
@@ -0,0 +1,101 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/test_fixture.h"
+#include "test/librbd/test_support.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "cls/journal/cls_journal_types.h"
+#include "cls/journal/cls_journal_client.h"
+#include "journal/Journaler.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/ImageWatcher.h"
+#include "librbd/internal.h"
+#include "librbd/Journal.h"
+#include "librbd/Operations.h"
+#include "librbd/api/Io.h"
+#include "librbd/api/Snapshot.h"
+#include "librbd/io/AioCompletion.h"
+#include "librbd/io/ImageDispatchSpec.h"
+#include "librbd/io/ImageRequest.h"
+#include "librbd/io/ReadResult.h"
+#include "librbd/journal/Types.h"
+
+void register_test_journal_stress() {
+}
+
+namespace librbd {
+namespace journal {
+
+class TestJournalStress : public TestFixture {
+};
+
+TEST_F(TestJournalStress, DiscardWithPruneWriteOverlap) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  // Overlap discards and writes while discard pruning is occurring. This tests
+  // the conditions under which https://tracker.ceph.com/issues/63422 occurred.
+
+  // Create an image that is multiple objects so that we can force multiple
+  // image extents on the discard path.
+  CephContext* cct = reinterpret_cast<CephContext*>(_rados.cct());
+  auto object_size = 1ull << cct->_conf.get_val<uint64_t>("rbd_default_order");
+  auto image_size = 4 * object_size;
+
+  // Write-around cache required for overlapping I/O delays.
+  cct->_conf.set_val_or_die("rbd_cache_writethrough_until_flush", "false");
+  cct->_conf.set_val_or_die("rbd_cache_policy", "writearound");
+
+  auto image_name = get_temp_image_name();
+  ASSERT_EQ(0, create_image_pp(m_rbd, m_ioctx, image_name, image_size));
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(image_name, &ictx));
+
+  std::thread write_thread(
+    [ictx, object_size]() {
+      std::string payload(object_size, '1');
+
+      for (auto i = 0; i < 200; i++) {
+        // Alternate overlaps with the two objects that the discard below
+        // touches.
+        for (auto offset = object_size;
+             offset < object_size * 3;
+             offset += object_size) {
+          bufferlist payload_bl;
+          payload_bl.append(payload);
+          auto aio_comp = new librbd::io::AioCompletion();
+          api::Io<>::aio_write(*ictx, aio_comp, 0, payload.size(),
+                               std::move(payload_bl), 0, true);
+          ASSERT_EQ(0, aio_comp->wait_for_complete());
+          aio_comp->release();
+        }
+      }
+    }
+  );
+
+  auto discard_exit = false;
+  std::thread discard_thread(
+    [ictx, object_size, &discard_exit]() {
+      while (!discard_exit) {
+        // We offset the discard by -4096 bytes and set discard granularity to
+        // 8192; this should cause two image extents to be formed in
+        // AbstractImageWriteRequest<I>::send_request() on objects 1 and 2,
+        // overlapping with the writes above.
+        auto aio_comp = new librbd::io::AioCompletion();
+        api::Io<>::aio_discard(*ictx, aio_comp, object_size - 4096,
+                               2 * object_size, 8192, true);
+        ASSERT_EQ(0, aio_comp->wait_for_complete());
+        aio_comp->release();
+      }
+    }
+  );
+
+  write_thread.join();
+  discard_exit = true;
+  discard_thread.join();
+}
+
+} // namespace journal
+} // namespace librbd
diff --git a/src/test/librbd/test_main.cc b/src/test/librbd/test_main.cc
index 2ff9f69dea97..82b72b1ef7e8 100644
--- a/src/test/librbd/test_main.cc
+++ b/src/test/librbd/test_main.cc
@@ -17,6 +17,7 @@ extern void register_test_image_watcher();
 extern void register_test_internal();
 extern void register_test_journal_entries();
 extern void register_test_journal_replay();
+extern void register_test_journal_stress();
 extern void register_test_migration();
 extern void register_test_mirroring();
 extern void register_test_mirroring_watcher();
@@ -37,6 +38,7 @@ int main(int argc, char **argv)
   register_test_internal();
   register_test_journal_entries();
   register_test_journal_replay();
+  register_test_journal_stress();
   register_test_migration();
   register_test_mirroring();
   register_test_mirroring_watcher();

From a0ccd8bb3ffbecb3313c0cdd7d3e6091fc2a9721 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Thu, 9 Nov 2023 09:43:20 -0700
Subject: [PATCH 0919/2492] librbd: Eliminate unused m_synchronous from
 AbstractImageWriteRequest

This has been unused since its introduction years ago and so isn't worth
keeping.

Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/librbd/io/ImageRequest.cc | 19 +++++++++----------
 src/librbd/io/ImageRequest.h  | 21 ++++++---------------
 2 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/src/librbd/io/ImageRequest.cc b/src/librbd/io/ImageRequest.cc
index e4c41c22976a..95bac7b245cb 100644
--- a/src/librbd/io/ImageRequest.cc
+++ b/src/librbd/io/ImageRequest.cc
@@ -473,7 +473,7 @@ void AbstractImageWriteRequest<I>::send_request() {
     if (journaling) {
       // in-flight ops are flushed prior to closing the journal
       ceph_assert(image_ctx.journal != NULL);
-      journal_tid = append_journal_event(m_synchronous);
+      journal_tid = append_journal_event();
     }
 
     // it's very important that IOContext is captured here instead of
@@ -518,7 +518,7 @@ void ImageWriteRequest<I>::assemble_extent(
 }
 
 template <typename I>
-uint64_t ImageWriteRequest<I>::append_journal_event(bool synchronous) {
+uint64_t ImageWriteRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
   uint64_t tid = 0;
@@ -530,7 +530,7 @@ uint64_t ImageWriteRequest<I>::append_journal_event(bool synchronous) {
     buffer_offset += extent.second;
 
     tid = image_ctx.journal->append_write_event(extent.first, extent.second,
-                                                sub_bl, synchronous);
+                                                sub_bl, false);
   }
 
   return tid;
@@ -566,7 +566,7 @@ void ImageWriteRequest<I>::update_stats(size_t length) {
 }
 
 template <typename I>
-uint64_t ImageDiscardRequest<I>::append_journal_event(bool synchronous) {
+uint64_t ImageDiscardRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
   uint64_t tid = 0;
@@ -578,7 +578,7 @@ uint64_t ImageDiscardRequest<I>::append_journal_event(bool synchronous) {
                                this->m_discard_granularity_bytes));
     tid = image_ctx.journal->append_io_event(std::move(event_entry),
                                              extent.first, extent.second,
-                                             synchronous, 0);
+                                             false, 0);
   }
 
   return tid;
@@ -717,7 +717,7 @@ void ImageFlushRequest<I>::send_request() {
 }
 
 template <typename I>
-uint64_t ImageWriteSameRequest<I>::append_journal_event(bool synchronous) {
+uint64_t ImageWriteSameRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
   uint64_t tid = 0;
@@ -728,7 +728,7 @@ uint64_t ImageWriteSameRequest<I>::append_journal_event(bool synchronous) {
                                                                m_data_bl));
     tid = image_ctx.journal->append_io_event(std::move(event_entry),
                                              extent.first, extent.second,
-                                             synchronous, 0);
+                                             false, 0);
   }
 
   return tid;
@@ -768,8 +768,7 @@ void ImageWriteSameRequest<I>::update_stats(size_t length) {
 }
 
 template <typename I>
-uint64_t ImageCompareAndWriteRequest<I>::append_journal_event(
-    bool synchronous) {
+uint64_t ImageCompareAndWriteRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
   uint64_t tid = 0;
@@ -779,7 +778,7 @@ uint64_t ImageCompareAndWriteRequest<I>::append_journal_event(
                                                           extent.second,
                                                           m_cmp_bl,
                                                           m_bl,
-                                                          synchronous);
+                                                          false);
 
   return tid;
 }
diff --git a/src/librbd/io/ImageRequest.h b/src/librbd/io/ImageRequest.h
index 2668c1acb2cd..996c90a11f21 100644
--- a/src/librbd/io/ImageRequest.h
+++ b/src/librbd/io/ImageRequest.h
@@ -114,11 +114,6 @@ class ImageReadRequest : public ImageRequest<ImageCtxT> {
 
 template <typename ImageCtxT = ImageCtx>
 class AbstractImageWriteRequest : public ImageRequest<ImageCtxT> {
-public:
-  inline void flag_synchronous() {
-    m_synchronous = true;
-  }
-
 protected:
   using typename ImageRequest<ImageCtxT>::ObjectRequests;
 
@@ -127,8 +122,7 @@ class AbstractImageWriteRequest : public ImageRequest<ImageCtxT> {
                             const char *trace_name,
 			    const ZTracer::Trace &parent_trace)
     : ImageRequest<ImageCtxT>(image_ctx, aio_comp, std::move(image_extents),
-                              area, trace_name, parent_trace),
-      m_synchronous(false) {
+                              area, trace_name, parent_trace) {
   }
 
   void send_request() override;
@@ -144,11 +138,8 @@ class AbstractImageWriteRequest : public ImageRequest<ImageCtxT> {
       const LightweightObjectExtent &object_extent, IOContext io_context,
       uint64_t journal_tid, bool single_extent, Context *on_finish) = 0;
 
-  virtual uint64_t append_journal_event(bool synchronous) = 0;
+  virtual uint64_t append_journal_event() = 0;
   virtual void update_stats(size_t length) = 0;
-
-private:
-  bool m_synchronous;
 };
 
 template <typename ImageCtxT = ImageCtx>
@@ -180,7 +171,7 @@ class ImageWriteRequest : public AbstractImageWriteRequest<ImageCtxT> {
       const LightweightObjectExtent &object_extent, IOContext io_context,
       uint64_t journal_tid, bool single_extent, Context *on_finish) override;
 
-  uint64_t append_journal_event(bool synchronous) override;
+  uint64_t append_journal_event() override;
   void update_stats(size_t length) override;
 
 private:
@@ -215,7 +206,7 @@ class ImageDiscardRequest : public AbstractImageWriteRequest<ImageCtxT> {
       const LightweightObjectExtent &object_extent, IOContext io_context,
       uint64_t journal_tid, bool single_extent, Context *on_finish) override;
 
-  uint64_t append_journal_event(bool synchronous) override;
+  uint64_t append_journal_event() override;
   void update_stats(size_t length) override;
 
   int prune_object_extents(
@@ -283,7 +274,7 @@ class ImageWriteSameRequest : public AbstractImageWriteRequest<ImageCtxT> {
       const LightweightObjectExtent &object_extent, IOContext io_context,
       uint64_t journal_tid, bool single_extent, Context *on_finish) override;
 
-  uint64_t append_journal_event(bool synchronous) override;
+  uint64_t append_journal_event() override;
   void update_stats(size_t length) override;
 private:
   bufferlist m_data_bl;
@@ -315,7 +306,7 @@ class ImageCompareAndWriteRequest : public AbstractImageWriteRequest<ImageCtxT>
       const LightweightObjectExtent &object_extent, IOContext io_context,
       uint64_t journal_tid, bool single_extent, Context *on_finish) override;
 
-  uint64_t append_journal_event(bool synchronous) override;
+  uint64_t append_journal_event() override;
   void update_stats(size_t length) override;
 
   aio_type_t get_aio_type() const override {

From 9fae091ea22c627cb18cc68055afa7acf634c1b7 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Thu, 9 Nov 2023 09:43:21 -0700
Subject: [PATCH 0920/2492] librbd: Change append_io_events() to take Extents

An upcoming commits will use this to change how multi-extent image
requests are appended to the journal.

Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/librbd/Journal.cc | 19 ++++++++++---------
 src/librbd/Journal.h  | 11 +++++++----
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index 8ddce2e8f7d4..121701c70d25 100644
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -39,6 +39,7 @@ using util::create_async_context_callback;
 using util::create_context_callback;
 using journal::util::C_DecodeTag;
 using journal::util::C_DecodeTags;
+using io::Extents;
 
 namespace {
 
@@ -788,8 +789,8 @@ uint64_t Journal<I>::append_write_event(uint64_t offset, size_t length,
     bytes_remaining -= event_length;
   } while (bytes_remaining > 0);
 
-  return append_io_events(journal::EVENT_TYPE_AIO_WRITE, bufferlists, offset,
-                          length, flush_entry, 0);
+  return append_io_events(journal::EVENT_TYPE_AIO_WRITE, bufferlists,
+                          {{offset, length}}, flush_entry, 0);
 }
 
 template <typename I>
@@ -832,7 +833,8 @@ uint64_t Journal<I>::append_compare_and_write_event(uint64_t offset,
   } while (bytes_remaining > 0);
 
   return append_io_events(journal::EVENT_TYPE_AIO_COMPARE_AND_WRITE,
-                          bufferlists, offset, length, flush_entry, -EILSEQ);
+                          bufferlists, {{offset, length}}, flush_entry,
+                          -EILSEQ);
 }
 
 template <typename I>
@@ -842,14 +844,14 @@ uint64_t Journal<I>::append_io_event(journal::EventEntry &&event_entry,
   bufferlist bl;
   event_entry.timestamp = ceph_clock_now();
   encode(event_entry, bl);
-  return append_io_events(event_entry.get_event_type(), {bl}, offset, length,
-                          flush_entry, filter_ret_val);
+  return append_io_events(event_entry.get_event_type(), {bl},
+                          {{offset, length}}, flush_entry, filter_ret_val);
 }
 
 template <typename I>
 uint64_t Journal<I>::append_io_events(journal::EventType event_type,
                                       const Bufferlists &bufferlists,
-                                      uint64_t offset, size_t length,
+                                      const Extents &image_extents,
                                       bool flush_entry, int filter_ret_val) {
   ceph_assert(!bufferlists.empty());
 
@@ -870,14 +872,13 @@ uint64_t Journal<I>::append_io_events(journal::EventType event_type,
 
   {
     std::lock_guard event_locker{m_event_lock};
-    m_events[tid] = Event(futures, offset, length, filter_ret_val);
+    m_events[tid] = Event(futures, image_extents, filter_ret_val);
   }
 
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 20) << this << " " << __func__ << ": "
                  << "event=" << event_type << ", "
-                 << "offset=" << offset << ", "
-                 << "length=" << length << ", "
+                 << "image_extents=" << image_extents << ", "
                  << "flush=" << flush_entry << ", tid=" << tid << dendl;
 
   Context *on_safe = create_async_context_callback(
diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h
index 1ef9ffa8830f..e33256bd575d 100644
--- a/src/librbd/Journal.h
+++ b/src/librbd/Journal.h
@@ -18,6 +18,7 @@
 #include "journal/ReplayHandler.h"
 #include "librbd/Utils.h"
 #include "librbd/asio/ContextWQ.h"
+#include "librbd/io/Types.h"
 #include "librbd/journal/Types.h"
 #include "librbd/journal/TypeTraits.h"
 
@@ -200,11 +201,13 @@ class Journal : public RefCountedObject {
 
     Event() {
     }
-    Event(const Futures &_futures, uint64_t offset, size_t length,
+    Event(const Futures &_futures, const io::Extents &image_extents,
           int filter_ret_val)
       : futures(_futures), filter_ret_val(filter_ret_val) {
-      if (length > 0) {
-        pending_extents.insert(offset, length);
+      for (auto &extent : image_extents) {
+        if (extent.second > 0) {
+          pending_extents.insert(extent.first, extent.second);
+        }
       }
     }
   };
@@ -324,7 +327,7 @@ class Journal : public RefCountedObject {
 
   uint64_t append_io_events(journal::EventType event_type,
                             const Bufferlists &bufferlists,
-                            uint64_t offset, size_t length, bool flush_entry,
+                            const io::Extents &extents, bool flush_entry,
                             int filter_ret_val);
   Future wait_event(ceph::mutex &lock, uint64_t tid, Context *on_safe);
 

From 4a8fa2da72fe64109073fddca0d4cfd99aeb9c77 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Thu, 9 Nov 2023 09:43:22 -0700
Subject: [PATCH 0921/2492] librbd: Append one journal event per image request

In the case where an image request is split across multiple object
extents and journaling is enabled, multiple journal events are appended.
Prior to this change, all object requests would wait for the last
journal event to complete, since journal events complete in order and
thus the last one completing implies that all prior journal events were
safe at that point.

The issue with this is that there's nothing stopping that last journal
event from being cleaned up before all object requests have stopped
referring to it. Thus, it's entirely possible for the following sequence
to occur:
1. An image request gets split into two image extents and two object
   requests. Journal events are appended (one per image extent).
2. The first object request gets delayed due to an overlap, but the
   second object request gets submitted and starts waiting on the last
   journal event (which also causes a C_CommitIOEvent to be instantiated
   against that journal event).
3. Journaling completes, and the C_CommitIOEvent fires. The
   C_CommitIOEvent covers the entire range of data that was journaled in
   this event, and so the event is cleaned up.
4. The first object request from above is allowed to make progress; it
   tries to wait for the journal event that was just cleaned up which
   causes the assert in wait_event() to fire.

As far as I can tell, this is only possible on the discard path today,
and only recently. Up until 21a26a752843295ff946d1543c2f5f9fac764593
(librbd: Fix local rbd mirror journals growing forever), m_image_extents
always contained a single extent for all I/O types; this commit changed
the discard path so that if discard granularity changed the discard
request, m_image_extents would be repopulated, and if the request
happened to cross objects then there would be multiple m_image_extents.

It appears that the intent here was that there should be one journal
event per image request and the pending_extents kept track of what had
completed thus far. This commit restores that 1:1 relationship.

Fixes: https://tracker.ceph.com/issues/63422
Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/librbd/Journal.cc                        | 69 +++++++++++++++++---
 src/librbd/Journal.h                         | 12 +++-
 src/librbd/io/ImageRequest.cc                | 41 ++----------
 src/test/librbd/io/test_mock_ImageRequest.cc | 31 +++++----
 src/test/librbd/journal/test_Entries.cc      | 63 ++++++++++++++++++
 src/test/librbd/test_mock_Journal.cc         |  2 +-
 6 files changed, 158 insertions(+), 60 deletions(-)

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index 121701c70d25..1b37a30c17c0 100644
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -761,36 +761,87 @@ void Journal<I>::user_flushed() {
 }
 
 template <typename I>
-uint64_t Journal<I>::append_write_event(uint64_t offset, size_t length,
-                                        const bufferlist &bl,
-                                        bool flush_entry) {
+void Journal<I>::add_write_event_entries(uint64_t offset, size_t length,
+                                         const bufferlist &bl,
+                                         uint64_t buffer_offset,
+                                         Bufferlists *bufferlists) {
   ceph_assert(m_max_append_size > journal::AioWriteEvent::get_fixed_size());
-  uint64_t max_write_data_size =
+  const uint64_t max_write_data_size =
     m_max_append_size - journal::AioWriteEvent::get_fixed_size();
 
   // ensure that the write event fits within the journal entry
-  Bufferlists bufferlists;
   uint64_t bytes_remaining = length;
   uint64_t event_offset = 0;
   do {
     uint64_t event_length = std::min(bytes_remaining, max_write_data_size);
 
     bufferlist event_bl;
-    event_bl.substr_of(bl, event_offset, event_length);
+    event_bl.substr_of(bl, buffer_offset + event_offset, event_length);
     journal::EventEntry event_entry(journal::AioWriteEvent(offset + event_offset,
                                                            event_length,
                                                            event_bl),
                                     ceph_clock_now());
 
-    bufferlists.emplace_back();
-    encode(event_entry, bufferlists.back());
+    bufferlists->emplace_back();
+    encode(event_entry, bufferlists->back());
 
     event_offset += event_length;
     bytes_remaining -= event_length;
   } while (bytes_remaining > 0);
+}
+
+template <typename I>
+uint64_t Journal<I>::append_write_event(const Extents &image_extents,
+                                        const bufferlist &bl,
+                                        bool flush_entry) {
+  Bufferlists bufferlists;
+  uint64_t buffer_offset = 0;
+  for (auto &extent : image_extents) {
+    add_write_event_entries(extent.first, extent.second, bl, buffer_offset,
+                            &bufferlists);
+
+    buffer_offset += extent.second;
+  }
 
   return append_io_events(journal::EVENT_TYPE_AIO_WRITE, bufferlists,
-                          {{offset, length}}, flush_entry, 0);
+                          image_extents, flush_entry, 0);
+}
+
+template <typename I>
+uint64_t Journal<I>::append_write_same_event(const Extents &image_extents,
+                                             const bufferlist &bl,
+                                             bool flush_entry) {
+  Bufferlists bufferlists;
+  for (auto &extent : image_extents) {
+    journal::EventEntry event_entry(
+      journal::AioWriteSameEvent(extent.first, extent.second, bl),
+      ceph_clock_now());
+
+    bufferlists.emplace_back();
+    encode(event_entry, bufferlists.back());
+  }
+
+  return append_io_events(journal::EVENT_TYPE_AIO_WRITESAME, bufferlists,
+                          image_extents, flush_entry, 0);
+}
+
+template <typename I>
+uint64_t Journal<I>::append_discard_event(const Extents &image_extents,
+                                          uint32_t discard_granularity_bytes,
+                                          bool flush_entry) {
+  Bufferlists bufferlists;
+  for (auto &extent : image_extents) {
+    journal::EventEntry event_entry(
+      journal::AioDiscardEvent(extent.first, extent.second,
+                               discard_granularity_bytes),
+      ceph_clock_now());
+
+    bufferlists.emplace_back();
+    encode(event_entry, bufferlists.back());
+  }
+
+  return append_io_events(journal::EVENT_TYPE_AIO_DISCARD, bufferlists,
+                          image_extents, flush_entry, 0);
 }
 
 template <typename I>
diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h
index e33256bd575d..5327adac7192 100644
--- a/src/librbd/Journal.h
+++ b/src/librbd/Journal.h
@@ -134,14 +134,20 @@ class Journal : public RefCountedObject {
 
   void user_flushed();
 
-  uint64_t append_write_event(uint64_t offset, size_t length,
+  uint64_t append_write_event(const io::Extents &image_extents,
                               const bufferlist &bl,
                               bool flush_entry);
+  uint64_t append_write_same_event(const io::Extents &image_extents,
+                                   const bufferlist &bl,
+                                   bool flush_entry);
   uint64_t append_compare_and_write_event(uint64_t offset,
                                           size_t length,
                                           const bufferlist &cmp_bl,
                                           const bufferlist &write_bl,
                                           bool flush_entry);
+  uint64_t append_discard_event(const io::Extents &image_extents,
+                                uint32_t discard_granularity_bytes,
+                                bool flush_entry);
   uint64_t append_io_event(journal::EventEntry &&event_entry,
                            uint64_t offset, size_t length,
                            bool flush_entry, int filter_ret_val);
@@ -325,6 +331,10 @@ class Journal : public RefCountedObject {
   bool is_journal_replaying(const ceph::mutex &) const;
   bool is_tag_owner(const ceph::mutex &) const;
 
+  void add_write_event_entries(uint64_t offset, size_t length,
+                               const bufferlist &bl,
+                               uint64_t buffer_offset,
+                               Bufferlists *bufferlists);
   uint64_t append_io_events(journal::EventType event_type,
                             const Bufferlists &bufferlists,
                             const io::Extents &extents, bool flush_entry,
diff --git a/src/librbd/io/ImageRequest.cc b/src/librbd/io/ImageRequest.cc
index 95bac7b245cb..fb9f8944ed84 100644
--- a/src/librbd/io/ImageRequest.cc
+++ b/src/librbd/io/ImageRequest.cc
@@ -521,19 +521,9 @@ template <typename I>
 uint64_t ImageWriteRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
-  uint64_t tid = 0;
-  uint64_t buffer_offset = 0;
   ceph_assert(!this->m_image_extents.empty());
-  for (auto &extent : this->m_image_extents) {
-    bufferlist sub_bl;
-    sub_bl.substr_of(m_bl, buffer_offset, extent.second);
-    buffer_offset += extent.second;
-
-    tid = image_ctx.journal->append_write_event(extent.first, extent.second,
-                                                sub_bl, false);
-  }
-
-  return tid;
+  return image_ctx.journal->append_write_event(
+    this->m_image_extents, m_bl, false);
 }
 
 template <typename I>
@@ -569,19 +559,9 @@ template <typename I>
 uint64_t ImageDiscardRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
-  uint64_t tid = 0;
   ceph_assert(!this->m_image_extents.empty());
-  for (auto &extent : this->m_image_extents) {
-    journal::EventEntry event_entry(
-      journal::AioDiscardEvent(extent.first,
-                               extent.second,
-                               this->m_discard_granularity_bytes));
-    tid = image_ctx.journal->append_io_event(std::move(event_entry),
-                                             extent.first, extent.second,
-                                             false, 0);
-  }
-
-  return tid;
+  return image_ctx.journal->append_discard_event(
+    this->m_image_extents, m_discard_granularity_bytes, false);
 }
 
 template <typename I>
@@ -720,18 +700,9 @@ template <typename I>
 uint64_t ImageWriteSameRequest<I>::append_journal_event() {
   I &image_ctx = this->m_image_ctx;
 
-  uint64_t tid = 0;
   ceph_assert(!this->m_image_extents.empty());
-  for (auto &extent : this->m_image_extents) {
-    journal::EventEntry event_entry(journal::AioWriteSameEvent(extent.first,
-                                                               extent.second,
-                                                               m_data_bl));
-    tid = image_ctx.journal->append_io_event(std::move(event_entry),
-                                             extent.first, extent.second,
-                                             false, 0);
-  }
-
-  return tid;
+  return image_ctx.journal->append_write_same_event(
+    this->m_image_extents, m_data_bl, false);
 }
 
 template <typename I>
diff --git a/src/test/librbd/io/test_mock_ImageRequest.cc b/src/test/librbd/io/test_mock_ImageRequest.cc
index 9d6423d66c4b..6ee67fe5f1c3 100644
--- a/src/test/librbd/io/test_mock_ImageRequest.cc
+++ b/src/test/librbd/io/test_mock_ImageRequest.cc
@@ -16,12 +16,15 @@ namespace {
 struct MockTestImageCtx;
 
 struct MockTestJournal : public MockJournal {
-  MOCK_METHOD4(append_write_event, uint64_t(uint64_t, size_t,
+  MOCK_METHOD3(append_write_event, uint64_t(const io::Extents&,
                                             const bufferlist &, bool));
+  MOCK_METHOD3(append_write_same_event, uint64_t(const io::Extents&,
+                                                 const bufferlist &, bool));
   MOCK_METHOD5(append_compare_and_write_event, uint64_t(uint64_t, size_t,
                                                         const bufferlist &,
                                                         const bufferlist &,
                                                         bool));
+  MOCK_METHOD3(append_discard_event, uint64_t(const io::Extents&, uint32_t, bool));
   MOCK_METHOD5(append_io_event_mock, uint64_t(const journal::EventEntry&,
                                               uint64_t, size_t, bool, int));
   uint64_t append_io_event(journal::EventEntry &&event_entry,
@@ -119,9 +122,10 @@ struct TestMockIoImageRequest : public TestMockFixture {
     }
   }
 
-  void expect_journal_append_io_event(MockTestJournal &mock_journal, uint64_t journal_tid,
-                                      uint64_t offset, size_t length) {
-    EXPECT_CALL(mock_journal, append_io_event_mock(_, offset, length, _, _))
+  void expect_journal_append_discard_event(MockTestJournal &mock_journal,
+                                           uint64_t journal_tid,
+                                           const io::Extents& extents) {
+    EXPECT_CALL(mock_journal, append_discard_event(extents, _, _))
       .WillOnce(Return(journal_tid));
   }
 
@@ -386,8 +390,8 @@ TEST_F(TestMockIoImageRequest, PartialDiscardJournalAppendEnabled) {
   InSequence seq;
   expect_get_modify_timestamp(mock_image_ctx, false);
   expect_is_journal_appending(mock_journal, true);
-  expect_journal_append_io_event(mock_journal, 0, 16, 63);
-  expect_journal_append_io_event(mock_journal, 1, 84, 100);
+  expect_journal_append_discard_event(mock_journal, 0,
+                                      {{16, 63}, {84, 100}});
   expect_object_discard_request(mock_image_ctx, 0, 16, 63, 0);
   expect_object_discard_request(mock_image_ctx, 0, 84, 100, 0);
 
@@ -419,8 +423,8 @@ TEST_F(TestMockIoImageRequest, TailDiscardJournalAppendEnabled) {
   InSequence seq;
   expect_get_modify_timestamp(mock_image_ctx, false);
   expect_is_journal_appending(mock_journal, true);
-  expect_journal_append_io_event(
-    mock_journal, 0, ictx->layout.object_size - 1024, 1024);
+  expect_journal_append_discard_event(
+    mock_journal, 0, {{ictx->layout.object_size - 1024, 1024}});
   expect_object_discard_request(
     mock_image_ctx, 0, ictx->layout.object_size - 1024, 1024, 0);
 
@@ -452,7 +456,7 @@ TEST_F(TestMockIoImageRequest, PruneRequiredDiscardJournalAppendEnabled) {
   InSequence seq;
   expect_get_modify_timestamp(mock_image_ctx, false);
   expect_is_journal_appending(mock_journal, true);
-  EXPECT_CALL(mock_journal, append_io_event_mock(_, _, _, _, _)).Times(0);
+  EXPECT_CALL(mock_journal, append_discard_event(_, _, _)).Times(0);
   EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, send(_)).Times(0);
 
   C_SaferCond aio_comp_ctx;
@@ -482,7 +486,7 @@ TEST_F(TestMockIoImageRequest, LengthModifiedDiscardJournalAppendEnabled) {
   InSequence seq;
   expect_get_modify_timestamp(mock_image_ctx, false);
   expect_is_journal_appending(mock_journal, true);
-  expect_journal_append_io_event(mock_journal, 0, 32, 32);
+  expect_journal_append_discard_event(mock_journal, 0, {{32, 32}});
   expect_object_discard_request(mock_image_ctx, 0, 32, 32, 0);
 
   C_SaferCond aio_comp_ctx;
@@ -513,10 +517,9 @@ TEST_F(TestMockIoImageRequest, DiscardGranularityJournalAppendEnabled) {
   InSequence seq;
   expect_get_modify_timestamp(mock_image_ctx, false);
   expect_is_journal_appending(mock_journal, true);
-  expect_journal_append_io_event(mock_journal, 0, 32, 32);
-  expect_journal_append_io_event(mock_journal, 1, 96, 64);
-  expect_journal_append_io_event(
-    mock_journal, 2, ictx->layout.object_size - 32, 32);
+  expect_journal_append_discard_event(
+    mock_journal, 0,
+    {{32, 32}, {96, 64}, {ictx->layout.object_size - 32, 32}});
   expect_object_discard_request(mock_image_ctx, 0, 32, 32, 0);
   expect_object_discard_request(mock_image_ctx, 0, 96, 64, 0);
   expect_object_discard_request(
diff --git a/src/test/librbd/journal/test_Entries.cc b/src/test/librbd/journal/test_Entries.cc
index c392fb9f88a8..bb4b06c0368a 100644
--- a/src/test/librbd/journal/test_Entries.cc
+++ b/src/test/librbd/journal/test_Entries.cc
@@ -196,6 +196,69 @@ TEST_F(TestJournalEntries, AioDiscard) {
   ASSERT_EQ(234U, aio_discard_event.length);
 }
 
+TEST_F(TestJournalEntries, AioDiscardWithPrune) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  // The discard path can create multiple image extents (ImageRequest.cc) in the
+  // case where the discard request needs to be pruned and multiple objects are
+  // involved in the request. This test ensures that journal event entries are
+  // queued up for each image extent.
+
+  // Create an image that is multiple objects so that we can force multiple
+  // image extents on the discard path.
+  CephContext* cct = reinterpret_cast<CephContext*>(_rados.cct());
+  auto object_size = 1ull << cct->_conf.get_val<uint64_t>("rbd_default_order");
+  auto image_size = 4 * object_size;
+
+  auto image_name = get_temp_image_name();
+  ASSERT_EQ(0, create_image_pp(m_rbd, m_ioctx, image_name, image_size));
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(image_name, &ictx));
+
+  ::journal::Journaler *journaler = create_journaler(ictx);
+  ASSERT_TRUE(journaler != NULL);
+
+  C_SaferCond cond_ctx;
+  auto c = librbd::io::AioCompletion::create(&cond_ctx);
+  c->get();
+  // We offset the discard by -4096 bytes and set discard granularity to 8192;
+  // this should cause two image extents to be formed in
+  // AbstractImageWriteRequest<I>::send_request().
+  api::Io<>::aio_discard(*ictx, c, object_size - 4096, 2 * object_size, 8192,
+                         true);
+  ASSERT_EQ(0, c->wait_for_complete());
+  c->put();
+
+  for (uint64_t chunk = 0; chunk < 2; chunk++) {
+    auto offset = object_size;
+    auto size = object_size;
+    if (chunk == 1) {
+      offset = object_size * 2;
+      size = object_size - 8192;
+    }
+
+    ::journal::ReplayEntry replay_entry;
+    if (!journaler->try_pop_front(&replay_entry)) {
+      ASSERT_TRUE(wait_for_entries_available(ictx));
+      ASSERT_TRUE(journaler->try_pop_front(&replay_entry));
+    }
+
+    librbd::journal::EventEntry event_entry;
+    ASSERT_TRUE(get_event_entry(replay_entry, &event_entry));
+
+    ASSERT_EQ(librbd::journal::EVENT_TYPE_AIO_DISCARD,
+              event_entry.get_event_type());
+
+    librbd::journal::AioDiscardEvent aio_discard_event =
+      boost::get<librbd::journal::AioDiscardEvent>(event_entry.event);
+    ASSERT_EQ(offset, aio_discard_event.offset);
+    ASSERT_EQ(size, aio_discard_event.length);
+
+    journaler->committed(replay_entry);
+  }
+}
+
 TEST_F(TestJournalEntries, AioFlush) {
   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 
diff --git a/src/test/librbd/test_mock_Journal.cc b/src/test/librbd/test_mock_Journal.cc
index 2fe74d2fe466..589695c50b39 100644
--- a/src/test/librbd/test_mock_Journal.cc
+++ b/src/test/librbd/test_mock_Journal.cc
@@ -460,7 +460,7 @@ class TestMockJournal : public TestMockFixture {
     bl.append_zero(length);
 
     std::shared_lock owner_locker{mock_image_ctx.owner_lock};
-    return mock_journal->append_write_event(0, length, bl, false);
+    return mock_journal->append_write_event({{0, length}}, bl, false);
   }
 
   uint64_t when_append_compare_and_write_event(

From 7436b4c99034dd19b2ff1e759603779061cbfdb6 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Wed, 29 Nov 2023 08:47:01 -0700
Subject: [PATCH 0922/2492] test/librbd: Add workaround for a journaling
 deadlock that this test exposes

Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/test/librbd/journal/test_Stress.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/test/librbd/journal/test_Stress.cc b/src/test/librbd/journal/test_Stress.cc
index 752ecf01f05e..d2b9353fb5f8 100644
--- a/src/test/librbd/journal/test_Stress.cc
+++ b/src/test/librbd/journal/test_Stress.cc
@@ -46,6 +46,10 @@ TEST_F(TestJournalStress, DiscardWithPruneWriteOverlap) {
   // Write-around cache required for overlapping I/O delays.
   cct->_conf.set_val_or_die("rbd_cache_writethrough_until_flush", "false");
   cct->_conf.set_val_or_die("rbd_cache_policy", "writearound");
+  // XXX: Work around https://tracker.ceph.com/issues/63681, which this test
+  // exposes when run under Valgrind.
+  cct->_conf.set_val_or_die("librados_thread_count", "15");
+  cct->_conf.apply_changes(nullptr);
 
   auto image_name = get_temp_image_name();
   ASSERT_EQ(0, create_image_pp(m_rbd, m_ioctx, image_name, image_size));

From 5fb2f98cbc1f4797f5d1c15bc8cd6d2b5ce35ecd Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Tue, 10 Oct 2023 05:04:01 +0530
Subject: [PATCH 0923/2492] mds: fix `dump dir` indicate dir is not cached

Example output:
$ ceph daemon mds.a dump dir /test-dir
[
    {
        "value/bits": "0/0",
        "status": "dirfrag not in cache"
    }
]

Fixes: https://tracker.ceph.com/issues/63093
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 src/mds/MDSRank.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 20ddf3eec588..64485c9413e2 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3393,8 +3393,14 @@ void MDSRank::command_dump_dir(Formatter *f, const cmdmap_t &cmdmap, std::ostrea
   in->dirfragtree.get_leaves_under(frag_t(), leaves);
   for (const auto& leaf : leaves) {
     CDir *dir = in->get_dirfrag(leaf);
-    if (dir)
+    if (dir) {
       mdcache->dump_dir(f, dir, dentry_dump);
+    } else {
+      f->open_object_section("frag");
+      f->dump_stream("frag") << leaf;
+      f->dump_string("status", "dirfrag not in cache");
+      f->close_section();
+    }
   }
   f->close_section();
 }

From 889b94d510356ac67983190d7fc02276057cbe14 Mon Sep 17 00:00:00 2001
From: zhangjianwei <zhangjianwei2_yewu@cmss.chinamobile.com>
Date: Fri, 2 Jun 2023 05:37:56 +0000
Subject: [PATCH 0924/2492] common: LogClient do not output meaningless logs by
 default

the default output log to syslog/graylog is meaningless
it will consume a lot of log resources

Fixes: https://tracker.ceph.com/issues/63727
Signed-off-by: zhangjianwei <zhangjianwei2@cmss.chinamobile.com>
---
 src/common/LogClient.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/common/LogClient.cc b/src/common/LogClient.cc
index 1ba363da790e..d5ae6b753216 100644
--- a/src/common/LogClient.cc
+++ b/src/common/LogClient.cc
@@ -186,13 +186,11 @@ void LogChannel::do_log(clog_type prio, const std::string& s)
 
   // log to syslog?
   if (do_log_to_syslog()) {
-    ldout(cct,0) << __func__ << " log to syslog"  << dendl;
     e.log_to_syslog(get_log_prio(), get_syslog_facility());
   }
 
   // log to graylog?
   if (do_log_to_graylog()) {
-    ldout(cct,0) << __func__ << " log to graylog"  << dendl;
     graylog->log_log_entry(&e);
   }
 }

From dd0a5aac965197c986588b885d0bb4311ae4b0fb Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 6 Dec 2023 11:56:44 +0530
Subject: [PATCH 0925/2492] monitoring: upgrade grafana container to 9.4.12

Fixes the CVEs mentioned here: https://grafana.com/blog/2023/06/06/grafana-security-release-new-grafana-versions-with-security-fixes-for-cve-2023-2183-and-cve-2023-2801/

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 monitoring/grafana/build/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monitoring/grafana/build/Makefile b/monitoring/grafana/build/Makefile
index 163931fbbaae..684e6fa1c816 100755
--- a/monitoring/grafana/build/Makefile
+++ b/monitoring/grafana/build/Makefile
@@ -1,5 +1,5 @@
 
-GRAFANA_VERSION ?= 9.4.7-1
+GRAFANA_VERSION ?= 9.4.12-1
 PIECHART_VERSION ?= "1.6.2"
 STATUS_PANEL_VERSION ?= "1.0.11"
 DASHBOARD_DIR := "../../ceph-mixin/dashboards_out"

From ec7c515490c2ade44d886e423a6601c7ef0cf5e8 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 2 Dec 2023 06:32:26 +0100
Subject: [PATCH 0926/2492] doc/radosgw: add gateway starting command

Add a command that properly starts (or restarts) the RADOS gateway after
RGW settings have been changed. This commit has been added in response
to an issue reported anonymously on
https://pad.ceph.com/p/Report_Documentation_Bugs.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/multisite.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
index b0bae6c52acf..cbd664971e71 100644
--- a/doc/radosgw/multisite.rst
+++ b/doc/radosgw/multisite.rst
@@ -528,6 +528,17 @@ running the following commands on the object gateway host:
    systemctl start ceph-radosgw@rgw.`hostname -s`
    systemctl enable ceph-radosgw@rgw.`hostname -s`
 
+If the ``cephadm`` command was used to deploy the cluster, you will not be able
+to use ``systemctl`` to start the gateway because no services will exist on
+which ``systemctl`` could operate. This is due to the containerized nature of
+the ``cephadm``-deployed Ceph cluster. If you have used the ``cephadm`` command
+and you have a containerized cluster, you must run a command of the following
+form to start the gateway:
+
+.. prompt:: bash #
+
+   ceph orch apply rgw <name> --realm=<realm> --zone=<zone> --placement --port
+
 Checking Synchronization Status
 -------------------------------
 

From dd258d8da00c400fdca752c47366b6bd453e27ed Mon Sep 17 00:00:00 2001
From: Mark Kogan <mkogan@ibm.com>
Date: Tue, 28 Nov 2023 14:34:31 +0200
Subject: [PATCH 0927/2492] rgw: d3n: fix valgrind reported leak related to
 libaio worker threads

which sporadically reproduces on teuthology ubuntu instances

happens because a race between RGW shutdown occurring before
the libaio worker threads had terminated

to fix, reduced the libaio threads inactivity shutdown time

ref:
man aio_init
...
aio_idle_time
This field specifies the amount of time in seconds that a worker thread
should wait for further requests before terminating, after having
completed a previous request.   The
default value is 1.
...

Fixes: https://tracker.ceph.com/issues/63445

Signed-off-by: Mark Kogan <mkogan@ibm.com>
---
 src/rgw/driver/rados/rgw_d3n_datacache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_d3n_datacache.cc b/src/rgw/driver/rados/rgw_d3n_datacache.cc
index 18d213cf6c34..c81954fce1c7 100644
--- a/src/rgw/driver/rados/rgw_d3n_datacache.cc
+++ b/src/rgw/driver/rados/rgw_d3n_datacache.cc
@@ -104,7 +104,7 @@ void D3nDataCache::init(CephContext *_cct) {
   struct aioinit ainit{0};
   ainit.aio_threads = cct->_conf.get_val<int64_t>("rgw_d3n_libaio_aio_threads");
   ainit.aio_num = cct->_conf.get_val<int64_t>("rgw_d3n_libaio_aio_num");
-  ainit.aio_idle_time = 10;
+  ainit.aio_idle_time = 5;
   aio_init(&ainit);
 #endif
 }

From 3d44e5dfdc83e33167ee2b63caf7c4fb2d34f6ba Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Wed, 6 Dec 2023 18:07:12 +0530
Subject: [PATCH 0928/2492] doc: add supported file types in
 cephfs-mirroring.rst

Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 doc/cephfs/cephfs-mirroring.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/cephfs/cephfs-mirroring.rst b/doc/cephfs/cephfs-mirroring.rst
index fd00a1eef2e3..973a2affa1f5 100644
--- a/doc/cephfs/cephfs-mirroring.rst
+++ b/doc/cephfs/cephfs-mirroring.rst
@@ -93,6 +93,15 @@ providing high-availability.
 .. note:: Deploying a single mirror daemon is recommended. Running multiple
    daemons is untested.
 
+The following file types are supported by the mirroring:
+
+- Regular files (-)
+- Directory files (d)
+- Symbolic link file (l)
+
+The other file types are ignored by the mirroring. So they won't be
+available on a successfully synchronized peer.
+
 The mirroring module is disabled by default. To enable the mirroring module,
 run the following command:
 

From 993ce910f41c69a84b5b3058da534ed0e8841a03 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 6 Dec 2023 10:58:03 -0500
Subject: [PATCH 0929/2492] mgr/orchestrator: block OSD specs with no service
 id

This is a copy of a code comment from this commit, but
it explains the change, so putting it here as well

There is a general "osd" service with no service id, but we use
that to dump osds created individually with "ceph orch daemon add osd"
and those made with "ceph orch apply osd --all-available-devices"
For actual user created OSD specs, we should promote users having a
service id so it doesn't get mixed in with those other OSDs. This
check is being done in this spot in particular as this is the only
place we can 100% differentiate between an actual user created OSD
spec and a spec we made ourselves to cover the all-available-devices case

Relates to: https://tracker.ceph.com/issues/63729

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/orchestrator/module.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index d6f36e81b718..5f36005b0b1a 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -1351,6 +1351,21 @@ def apply_misc(self,
                         except KeyError:
                             raise SpecValidationError(f'Invalid config option {k} in spec')
 
+                # There is a general "osd" service with no service id, but we use
+                # that to dump osds created individually with "ceph orch daemon add osd"
+                # and those made with "ceph orch apply osd --all-available-devices"
+                # For actual user created OSD specs, we should promote users having a
+                # service id so it doesn't get mixed in with those other OSDs. This
+                # check is being done in this spot in particular as this is the only
+                # place we can 100% differentiate between an actual user created OSD
+                # spec and a spec we made ourselves to cover the all-available-devices case
+                if (
+                    isinstance(spec, DriveGroupSpec)
+                    and spec.service_type == 'osd'
+                    and not spec.service_id
+                ):
+                    raise SpecValidationError('Please provide the service_id field in your OSD spec')
+
                 if dry_run and not isinstance(spec, HostSpec):
                     spec.preview_only = dry_run
 

From d5520f291400bbb767394471a8abdf5ca6508bd0 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 6 Dec 2023 12:05:38 -0500
Subject: [PATCH 0930/2492] doc/cephadm: document requiring service ids for
 advanced OSD specs

Signed-off-by: Adam King <adking@redhat.com>
---
 doc/cephadm/services/osd.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst
index 96d05d5dc0bd..d9c30249b41e 100644
--- a/doc/cephadm/services/osd.rst
+++ b/doc/cephadm/services/osd.rst
@@ -454,6 +454,15 @@ configurations, without knowing the specifics of device names and paths.
 Service specifications make it possible to define a yaml or json file that can
 be used to reduce the amount of manual work involved in creating OSDs.
 
+.. note::
+    It is recommended that advanced OSD specs include the ``service_id`` field
+    set. The plain ``osd`` service with no service id is where OSDs created
+    using ``ceph orch daemon add`` or ``ceph orch apply osd --all-available-devices``
+    are placed. Not including a ``service_id`` in your OSD spec would mix
+    the OSDs from your spec with those OSDs and potentially overwrite services
+    specs created by cephadm to track them. Newer versions of cephadm will even
+    block creation of advanced OSD specs without the service_id present
+
 For example, instead of running the following command:
 
 .. prompt:: bash [monitor.1]#

From dc604f8424269db389983f6c7df786e9c64a5e02 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 6 Dec 2023 22:56:53 +0530
Subject: [PATCH 0931/2492] mgr/cephadm: bump grafana to 9.4.12

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/cephadm/cephadmlib/constants.py | 2 +-
 src/pybind/mgr/cephadm/module.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index e5c03adffb7b..b09301f7f87a 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -9,7 +9,7 @@
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.5.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'
-DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.7'
+DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.12'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
 DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 059b8cdcee8e..de0f49fb2b42 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -118,7 +118,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'
-DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.7'
+DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.12'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
 DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'

From b19e15cb238b41d38c81fd00063906baf7816fdc Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 28 Nov 2023 12:28:41 -0500
Subject: [PATCH 0932/2492] neorados: Remove broken out duplicates of calls

Make everything use IOContext rather than duplicating everything with
versions taking a separate pool, namespace, and key.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp     | 153 -----------------------
 src/neorados/RADOS.cc              | 190 -----------------------------
 src/test/neorados/list_pool.cc     |   4 +-
 src/test/neorados/test_neorados.cc |   2 +-
 src/tools/neorados.cc              |  11 +-
 5 files changed, 9 insertions(+), 351 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index b4f48d12ffc1..258a2ead5ffd 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -575,36 +575,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto execute(const Object& o, std::int64_t pool,
-	       ReadOp&& op,
-	       ceph::buffer::list* bl,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {},
-	       uint64_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, pool, std::move(op), bl,
-	    ReadOp::Completion::create(get_executor(),
-				       std::move(init.completion_handler)),
-	    ns, key, objver);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto execute(const Object& o, std::int64_t pool, WriteOp&& op,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {},
-	       uint64_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, pool, std::move(op),
-	    Op::Completion::create(get_executor(),
-				   std::move(init.completion_handler)),
-	    ns, key, objver);
-    return init.result.get();
-  }
-
   boost::uuids::uuid get_fsid() const noexcept;
 
   using LookupPoolSig = void(boost::system::error_code,
@@ -756,20 +726,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto watch(const Object& o, std::int64_t pool,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, CompletionToken&& token,
-	     std::optional<std::string_view> ns = {},
-	     std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    watch(o, pool, timeout, std::move(cb),
-	  WatchComp::create(get_executor(),
-			    std::move(init.completion_handler)),
-	  ns, key);
-    return init.result.get();
-  }
-
   template<typename CompletionToken>
   auto notify_ack(const Object& o,
 		  const IOContext& ioc,
@@ -784,23 +740,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto notify_ack(const Object& o,
-		  std::int64_t pool,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  CompletionToken&& token,
-		  std::optional<std::string_view> ns = {},
-		  std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    notify_ack(o, pool, notify_id, cookie, std::move(bl),
-	       SimpleOpComp::create(get_executor(),
-				    std::move(init.completion_handler)),
-	       ns, key);
-    return init.result.get();
-  }
-
   template<typename CompletionToken>
   auto unwatch(uint64_t cookie, const IOContext& ioc,
 	       CompletionToken&& token) {
@@ -811,19 +750,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto unwatch(uint64_t cookie, std::int64_t pool,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    unwatch(cookie, pool,
-	    SimpleOpComp::create(get_executor(),
-				 std::move(init.completion_handler)),
-	    ns, key);
-    return init.result.get();
-  }
-
   // This is one of those places where having to force everything into
   // a .cc file is really infuriating. If we had modules, that would
   // let us separate out the implementation details without
@@ -852,21 +778,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto notify(const Object& oid, std::int64_t pool, ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      CompletionToken&& token,
-	      std::optional<std::string_view> ns = {},
-	      std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, NotifySig> init(token);
-    notify(oid, pool, bl, timeout,
-	   NotifyComp::create(get_executor(),
-			      std::move(init.completion_handler)),
-	   ns, key);
-
-    return init.result.get();
-  }
-
   // The versions with pointers are fine for coroutines, but
   // extraordinarily unappealing for callback-oriented programming.
   using EnumerateSig = void(boost::system::error_code,
@@ -885,21 +796,6 @@ class RADOS final
     return init.result.get();
   }
 
-  template<typename CompletionToken>
-  auto enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 CompletionToken&& token,
-			 std::optional<std::string_view> ns = {},
-			 std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, EnumerateSig> init(token);
-    enumerate_objects(pool, begin, end, max, filter,
-		      EnumerateComp::create(get_executor(),
-					    std::move(init.completion_handler)),
-		      ns, key);
-    return init.result.get();
-  }
-
   using CommandSig = void(boost::system::error_code,
 			  std::string, ceph::buffer::list);
   using CommandComp = ceph::async::Completion<CommandSig>;
@@ -984,18 +880,6 @@ class RADOS final
 	       std::unique_ptr<Op::Completion> c, uint64_t* objver,
 	       const blkin_trace_info* trace_info);
 
-  void execute(const Object& o, std::int64_t pool, ReadOp&& op,
-	       ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key,
-	       uint64_t* objver);
-
-  void execute(const Object& o, std::int64_t pool, WriteOp&& op,
-	       std::unique_ptr<Op::Completion> c,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key,
-	       uint64_t* objver);
-
   void lookup_pool(std::string_view name, std::unique_ptr<LookupPoolComp> c);
   void list_pools(std::unique_ptr<LSPoolsComp> c);
   void create_pool_snap(int64_t pool, std::string_view snapName,
@@ -1019,11 +903,6 @@ class RADOS final
   void watch(const Object& o, const IOContext& ioc,
 	     std::optional<std::chrono::seconds> timeout,
 	     WatchCB&& cb, std::unique_ptr<WatchComp> c);
-  void watch(const Object& o, std::int64_t pool,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, std::unique_ptr<WatchComp> c,
-	     std::optional<std::string_view> ns,
-	     std::optional<std::string_view> key);
   tl::expected<ceph::timespan, boost::system::error_code>
   watch_check(uint64_t cookie);
   void notify_ack(const Object& o,
@@ -1032,30 +911,12 @@ class RADOS final
 		  uint64_t cookie,
 		  ceph::buffer::list&& bl,
 		  std::unique_ptr<SimpleOpComp>);
-  void notify_ack(const Object& o,
-		  std::int64_t pool,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  std::unique_ptr<SimpleOpComp>,
-		  std::optional<std::string_view> ns,
-		  std::optional<std::string_view> key);
   void unwatch(uint64_t cookie, const IOContext& ioc,
 	       std::unique_ptr<SimpleOpComp>);
-  void unwatch(uint64_t cookie, std::int64_t pool,
-	       std::unique_ptr<SimpleOpComp>,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key);
   void notify(const Object& oid, const IOContext& ioctx,
 	      ceph::buffer::list&& bl,
 	      std::optional<std::chrono::milliseconds> timeout,
 	      std::unique_ptr<NotifyComp> c);
-  void notify(const Object& oid, std::int64_t pool,
-	      ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      std::unique_ptr<NotifyComp> c,
-	      std::optional<std::string_view> ns,
-	      std::optional<std::string_view> key);
   void flush_watch(std::unique_ptr<VoidOpComp>);
 
   void enumerate_objects(const IOContext& ioc, const Cursor& begin,
@@ -1064,24 +925,10 @@ class RADOS final
 			 std::vector<Entry>* ls,
 			 Cursor* cursor,
 			 std::unique_ptr<SimpleOpComp> c);
-  void enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 std::unique_ptr<SimpleOpComp> c,
-			 std::optional<std::string_view> ns,
-			 std::optional<std::string_view> key);
   void enumerate_objects(const IOContext& ioc, const Cursor& begin,
 			 const Cursor& end, const std::uint32_t max,
 			 const ceph::buffer::list& filter,
 			 std::unique_ptr<EnumerateComp> c);
-  void enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::unique_ptr<EnumerateComp> c,
-			 std::optional<std::string_view> ns,
-			 std::optional<std::string_view> key);
   void osd_command(int osd, std::vector<std::string>&& cmd,
 		   ceph::buffer::list&& in, std::unique_ptr<CommandComp> c);
   void pg_command(PG pg, std::vector<std::string>&& cmd,
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 93f02b1ae702..667127d11253 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -849,54 +849,6 @@ void RADOS::execute(const Object& o, const IOContext& _ioc, WriteOp&& _op,
   trace.event("submitted");
 }
 
-void RADOS::execute(const Object& o, std::int64_t pool, ReadOp&& _op,
-		    cb::list* bl,
-		    std::unique_ptr<ReadOp::Completion> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key,
-		    version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = op->op.flags;
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  impl->objecter->read(
-    *oid, oloc, std::move(op->op), CEPH_NOSNAP, bl, flags,
-    std::move(c), objver);
-}
-
-void RADOS::execute(const Object& o, std::int64_t pool, WriteOp&& _op,
-		    std::unique_ptr<WriteOp::Completion> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key,
-		    version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = op->op.flags;
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ceph::real_time mtime;
-  if (op->mtime)
-    mtime = *op->mtime;
-  else
-    mtime = ceph::real_clock::now();
-
-  impl->objecter->mutate(
-    *oid, oloc, std::move(op->op), {},
-    mtime, flags,
-    std::move(c), objver);
-}
-
 boost::uuids::uuid RADOS::get_fsid() const noexcept {
   return impl->monclient.get_fsid().uuid;
 }
@@ -1130,35 +1082,6 @@ void RADOS::watch(const Object& o, const IOContext& _ioc,
       }), nullptr);
 }
 
-void RADOS::watch(const Object& o, std::int64_t pool,
-		  std::optional<std::chrono::seconds> timeout, WatchCB&& cb,
-		  std::unique_ptr<WatchComp> c,
-		  std::optional<std::string_view> ns,
-		  std::optional<std::string_view> key) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ObjectOperation op;
-
-  Objecter::LingerOp *linger_op = impl->objecter->linger_register(*oid, oloc, 0);
-  uint64_t cookie = linger_op->get_cookie();
-  linger_op->handle = std::move(cb);
-  op.watch(cookie, CEPH_OSD_WATCH_OP_WATCH, timeout.value_or(0s).count());
-  bufferlist bl;
-  impl->objecter->linger_watch(
-    linger_op, op, {}, ceph::real_clock::now(), bl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c), cookie](bs::error_code e, bufferlist) mutable {
-	ca::dispatch(std::move(c), e, cookie);
-      }), nullptr);
-}
-
 void RADOS::notify_ack(const Object& o,
 		       const IOContext& _ioc,
 		       uint64_t notify_id,
@@ -1176,28 +1099,6 @@ void RADOS::notify_ack(const Object& o,
 		       nullptr, ioc->extra_op_flags, std::move(c));
 }
 
-void RADOS::notify_ack(const Object& o,
-		       std::int64_t pool,
-		       uint64_t notify_id,
-		       uint64_t cookie,
-		       bufferlist&& bl,
-		       std::unique_ptr<SimpleOpComp> c,
-		       std::optional<std::string_view> ns,
-		       std::optional<std::string_view> key) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ObjectOperation op;
-  op.notify_ack(notify_id, cookie, bl);
-  impl->objecter->read(*oid, oloc, std::move(op), CEPH_NOSNAP, nullptr, 0,
-		       std::move(c));
-}
-
 tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check(uint64_t cookie)
 {
   Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
@@ -1225,34 +1126,6 @@ void RADOS::unwatch(uint64_t cookie, const IOContext& _ioc,
 			   }));
 }
 
-void RADOS::unwatch(uint64_t cookie, std::int64_t pool,
-		    std::unique_ptr<SimpleOpComp> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key)
-{
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-
-  ObjectOperation op;
-  op.watch(cookie, CEPH_OSD_WATCH_OP_UNWATCH);
-  impl->objecter->mutate(linger_op->target.base_oid, oloc, std::move(op),
-			 {}, ceph::real_clock::now(), 0,
-			 Objecter::Op::OpComp::create(
-			   get_executor(),
-			   [objecter = impl->objecter,
-			    linger_op, c = std::move(c)]
-			   (bs::error_code ec) mutable {
-			     objecter->linger_cancel(linger_op);
-			     ca::dispatch(std::move(c), ec);
-			   }));
-}
-
 void RADOS::flush_watch(std::unique_ptr<VoidOpComp> c)
 {
   impl->objecter->linger_callback_flush([c = std::move(c)]() mutable {
@@ -1346,45 +1219,6 @@ void RADOS::notify(const Object& o, const IOContext& _ioc, bufferlist&& bl,
       }), nullptr);
 }
 
-void RADOS::notify(const Object& o, std::int64_t pool, bufferlist&& bl,
-		   std::optional<std::chrono::milliseconds> timeout,
-		   std::unique_ptr<NotifyComp> c,
-		   std::optional<std::string_view> ns,
-		   std::optional<std::string_view> key)
-{
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-  auto linger_op = impl->objecter->linger_register(*oid, oloc, 0);
-
-  auto cb = std::make_shared<NotifyHandler>(impl->ioctx, impl->objecter,
-                                            linger_op, std::move(c));
-  linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, ceph::bufferlist&& bl) mutable {
-	(*cb)(ec, std::move(bl));
-      });
-  ObjectOperation rd;
-  bufferlist inbl;
-  rd.notify(
-    linger_op->get_cookie(), 1,
-    timeout ? timeout->count() : impl->cct->_conf->client_notify_timeout,
-    bl, &inbl);
-
-  impl->objecter->linger_notify(
-    linger_op, rd, CEPH_NOSNAP, inbl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, bufferlist&& bl) mutable {
-	cb->handle_ack(ec, std::move(bl));
-      }), nullptr);
-}
-
 // Enumeration
 
 Cursor::Cursor() {
@@ -1509,30 +1343,6 @@ void RADOS::enumerate_objects(const IOContext& _ioc,
     });
 }
 
-void RADOS::enumerate_objects(std::int64_t pool,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
-			      std::unique_ptr<EnumerateComp> c,
-			      std::optional<std::string_view> ns,
-			      std::optional<std::string_view> key) {
-  impl->objecter->enumerate_objects<Entry>(
-    pool,
-    ns ? *ns : std::string_view{},
-    *reinterpret_cast<const hobject_t*>(&begin.impl),
-    *reinterpret_cast<const hobject_t*>(&end.impl),
-    max,
-    filter,
-    [c = std::move(c)]
-    (bs::error_code ec, std::vector<Entry>&& v,
-     hobject_t&& n) mutable {
-      ca::dispatch(std::move(c), ec, std::move(v),
-		   Cursor(static_cast<void*>(&n)));
-    });
-}
-
-
 void RADOS::osd_command(int osd, std::vector<std::string>&& cmd,
 			ceph::bufferlist&& in, std::unique_ptr<CommandComp> c) {
   impl->objecter->osd_command(osd, std::move(cmd), std::move(in), nullptr,
diff --git a/src/test/neorados/list_pool.cc b/src/test/neorados/list_pool.cc
index ae36c36e6552..4a1dc9cb7b1b 100644
--- a/src/test/neorados/list_pool.cc
+++ b/src/test/neorados/list_pool.cc
@@ -83,9 +83,9 @@ bs::error_code noisy_list(R::RADOS& r, int64_t p)
 
   std::cout << "begin = " << b.to_str() << std::endl;
   std::cout << "end = " << e.to_str() << std::endl;
+  R::IOContext pool{p, R::all_nspaces};
   try {
-    auto [v, next] = r.enumerate_objects(p, b, e, 1000, {}, ca::use_blocked,
-					 R::all_nspaces);
+    auto [v, next] = r.enumerate_objects(pool, b, e, 1000, {}, ca::use_blocked);
 
     std::cout << "Got " << v.size() << " entries." << std::endl;
     std::cout << "next cursor = " << next.to_str() << std::endl;
diff --git a/src/test/neorados/test_neorados.cc b/src/test/neorados/test_neorados.cc
index 953e772e115b..84bb5d96d4ac 100644
--- a/src/test/neorados/test_neorados.cc
+++ b/src/test/neorados/test_neorados.cc
@@ -29,7 +29,7 @@ TEST_F(TestNeoRADOS, MakeWithLibRADOS) {
 
   // provide pool that doesn't exists -- just testing round-trip
   ASSERT_THROW(
-    rados.execute({"dummy-obj"}, std::numeric_limits<int64_t>::max(),
+    rados.execute({"dummy-obj"}, IOContext{std::numeric_limits<int64_t>::max()},
                   std::move(op), nullptr, ceph::async::use_blocked),
     boost::system::system_error);
 }
diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
index 4c5f8475d394..097dedb6e499 100644
--- a/src/tools/neorados.cc
+++ b/src/tools/neorados.cc
@@ -67,7 +67,7 @@ void printseq(const V& v, std::ostream& m, F&& f)
 		});
 }
 
-std::int64_t lookup_pool(R::RADOS& r, const std::string& pname,
+R::IOContext lookup_pool(R::RADOS& r, const std::string& pname,
 			 s::yield_context y)
 {
   bs::error_code ec;
@@ -75,7 +75,7 @@ std::int64_t lookup_pool(R::RADOS& r, const std::string& pname,
   if (ec)
     throw bs::system_error(
       ec, fmt::format("when looking up '{}'", pname));
-  return p;
+  return R::IOContext{p};
 }
 
 
@@ -92,14 +92,15 @@ void lspools(R::RADOS& r, const std::vector<std::string>&,
 void ls(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 {
   const auto& pname = p[0];
-  const auto pool = lookup_pool(r, pname, y);
+  auto pool = lookup_pool(r, pname, y);
+  pool.ns(R::all_nspaces);
 
   std::vector<R::Entry> ls;
   R::Cursor next = R::Cursor::begin();
   bs::error_code ec;
   do {
     std::tie(ls, next) = r.enumerate_objects(pool, next, R::Cursor::end(),
-					     1000, {}, y[ec], R::all_nspaces);
+					     1000, {}, y[ec]);
     if (ec)
       throw bs::system_error(ec, fmt::format("when listing {}", pname));
     printseq(ls, std::cout);
@@ -212,7 +213,7 @@ void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
       throw bs::system_error(
 	ec,
 	fmt::format("when reading from object '{}' in pool '{}'",
-		    obj, pool));
+		    obj, pool.pool()));
 
     off += bl.length();
     bl.write_stream(std::cout);

From 20aee5bbbcb50e8eebe63348f7371bf0070832d9 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 28 Nov 2023 15:12:29 -0500
Subject: [PATCH 0933/2492] neorados: Make IOContext getters/setters less weird

Name them get and set for one thing. In practice a lot of existing
Ceph code is written using special values to indicate nullity, so
optional doesn't add anything.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                |  61 +++++----
 src/librbd/ImageCtx.cc                        |   6 +-
 .../cache/ObjectCacherObjectDispatch.cc       |   6 +-
 src/librbd/cache/ObjectCacherWriteback.cc     |   4 +-
 src/librbd/cache/ParentCacheObjectDispatch.cc |   4 +-
 src/librbd/cache/WriteLogImageDispatch.cc     |   2 +-
 src/librbd/crypto/CryptoObjectDispatch.cc     |   2 +-
 src/librbd/deep_copy/ObjectCopyRequest.cc     |   2 +-
 src/librbd/io/CopyupRequest.cc                |   2 +-
 src/librbd/io/ObjectRequest.cc                |  16 +--
 src/librbd/migration/ImageDispatch.cc         |   2 +-
 src/neorados/RADOS.cc                         | 127 +++++++++---------
 .../librados_test_stub/NeoradosTestStub.cc    |  81 ++++++-----
 .../deep_copy/test_mock_ObjectCopyRequest.cc  |   2 +-
 src/test/librbd/io/test_mock_ObjectRequest.cc |   2 +-
 src/test/librbd/mock/MockImageCtx.cc          |   4 +-
 src/test/neorados/list_pool.cc                |   4 +-
 src/tools/neorados.cc                         |   5 +-
 18 files changed, 181 insertions(+), 151 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 258a2ead5ffd..915c3c4d1332 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -24,6 +24,9 @@
 #include <type_traits>
 #include <variant>
 
+#include <fmt/format.h>
+#include <fmt/ostream.h>
+
 #include <boost/asio/async_result.hpp>
 #include <boost/asio/io_context.hpp>
 
@@ -129,8 +132,7 @@ class IOContext final {
 
   IOContext();
   explicit IOContext(std::int64_t pool);
-  IOContext(std::int64_t _pool, std::string_view _ns);
-  IOContext(std::int64_t _pool, std::string&& _ns);
+  IOContext(std::int64_t pool, std::string ns, std::string key = {});
   ~IOContext();
 
   IOContext(const IOContext& rhs);
@@ -139,36 +141,41 @@ class IOContext final {
   IOContext(IOContext&& rhs);
   IOContext& operator =(IOContext&& rhs);
 
-  std::int64_t pool() const;
-  void pool(std::int64_t _pool);
+  std::int64_t get_pool() const;
+  void set_pool(std::int64_t _pool) &;
+  IOContext&& set_pool(std::int64_t _pool) &&;
 
-  std::string_view ns() const;
-  void ns(std::string_view _ns);
-  void ns(std::string&& _ns);
+  std::string_view get_ns() const;
+  void set_ns(std::string ns) &;
+  IOContext&& set_ns(std::string ns) &&;
 
-  std::optional<std::string_view> key() const;
-  void key(std::string_view _key);
-  void key(std::string&& _key);
-  void clear_key();
+  std::string_view get_key() const;
+  void set_key(std::string key) &;
+  IOContext&& set_key(std::string key) &&;
 
-  std::optional<std::int64_t> hash() const;
-  void hash(std::int64_t _hash);
-  void clear_hash();
+  std::int64_t get_hash() const;
+  void set_hash(std::int64_t hash) &;
+  IOContext&& set_hash(std::int64_t hash) &&;
 
-  std::optional<std::uint64_t> read_snap() const;
-  void read_snap(std::optional<std::uint64_t> _snapid);
+  std::uint64_t get_read_snap() const;
+  void set_read_snap(std::uint64_t snapid) &;
+  IOContext&& set_read_snap(std::uint64_t snapid) &&;
 
   // I can't actually move-construct here since snapid_t is its own
   // separate class type, not an alias.
   std::optional<
     std::pair<std::uint64_t,
-	      std::vector<std::uint64_t>>> write_snap_context() const;
-  void write_snap_context(std::optional<
-			  std::pair<std::uint64_t,
-			              std::vector<std::uint64_t>>> snapc);
-
-  bool full_try() const;
-  void full_try(bool _full_try);
+	      std::vector<std::uint64_t>>> get_write_snap_context() const;
+  void set_write_snap_context(
+    std::optional<std::pair<std::uint64_t,
+                            std::vector<std::uint64_t>>> snapc) &;
+  IOContext&& set_write_snap_context(
+    std::optional<std::pair<std::uint64_t,
+                            std::vector<std::uint64_t>>> snapc) &&;
+
+  bool get_full_try() const;
+  void set_full_try(bool full_try) &;
+  IOContext&& set_full_try(bool full_try) &&;
 
   friend std::ostream& operator <<(std::ostream& m, const IOContext& o);
   friend bool operator <(const IOContext& lhs, const IOContext& rhs);
@@ -185,7 +192,7 @@ class IOContext final {
   std::aligned_storage_t<impl_size> impl;
 };
 
-inline constexpr std::string_view all_nspaces("\001");
+inline const std::string all_nspaces("\001");
 
 enum class cmpxattr_op : std::uint8_t {
   eq  = 1,
@@ -995,4 +1002,10 @@ struct hash<neorados::IOContext> {
 };
 } // namespace std
 
+#if FMT_VERSION >= 90000
+template<> struct fmt::formatter<neorados::Object> : fmt::ostream_formatter {};
+template<> struct fmt::formatter<neorados::IOContext>
+  : fmt::ostream_formatter {};
+#endif // FMT_VERSION
+
 #endif // NEORADOS_RADOS_HPP
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index f4cd7e5f75d5..df256ba6c321 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -1003,14 +1003,14 @@ librados::IoCtx duplicate_io_ctx(librados::IoCtx& io_ctx) {
     auto ctx = std::make_shared<neorados::IOContext>(
       data_ctx.get_id(), data_ctx.get_namespace());
     if (snap_id != CEPH_NOSNAP) {
-      ctx->read_snap(snap_id);
+      ctx->set_read_snap(snap_id);
     }
     if (!snapc.snaps.empty()) {
-      ctx->write_snap_context(
+      ctx->set_write_snap_context(
         {{snapc.seq, {snapc.snaps.begin(), snapc.snaps.end()}}});
     }
     if (data_ctx.get_pool_full_try()) {
-      ctx->full_try(true);
+      ctx->set_full_try(true);
     }
 
     // atomically reset the data IOContext to new version
diff --git a/src/librbd/cache/ObjectCacherObjectDispatch.cc b/src/librbd/cache/ObjectCacherObjectDispatch.cc
index baa8685513da..cd081d58d0d6 100644
--- a/src/librbd/cache/ObjectCacherObjectDispatch.cc
+++ b/src/librbd/cache/ObjectCacherObjectDispatch.cc
@@ -224,7 +224,7 @@ bool ObjectCacherObjectDispatch<I>::read(
 
   m_image_ctx->image_lock.lock_shared();
   auto rd = m_object_cacher->prepare_read(
-    io_context->read_snap().value_or(CEPH_NOSNAP), bl, op_flags);
+    io_context->get_read_snap(), bl, op_flags);
   m_image_ctx->image_lock.unlock_shared();
 
   uint64_t off = 0;
@@ -324,8 +324,8 @@ bool ObjectCacherObjectDispatch<I>::write(
   }
 
   SnapContext snapc;
-  if (io_context->write_snap_context()) {
-    auto write_snap_context = *io_context->write_snap_context();
+  if (io_context->get_write_snap_context()) {
+    auto write_snap_context = *io_context->get_write_snap_context();
     snapc = SnapContext(write_snap_context.first,
                         {write_snap_context.second.begin(),
                          write_snap_context.second.end()});
diff --git a/src/librbd/cache/ObjectCacherWriteback.cc b/src/librbd/cache/ObjectCacherWriteback.cc
index 97f2d46ba853..9f2c26b2a106 100644
--- a/src/librbd/cache/ObjectCacherWriteback.cc
+++ b/src/librbd/cache/ObjectCacherWriteback.cc
@@ -142,7 +142,7 @@ void ObjectCacherWriteback::read(const object_t& oid, uint64_t object_no,
 
   auto io_context = m_ictx->duplicate_data_io_context();
   if (snapid != CEPH_NOSNAP) {
-    io_context->read_snap(snapid);
+    io_context->set_read_snap(snapid);
   }
 
   // extract the embedded RBD read flags from the op_flags
@@ -208,7 +208,7 @@ ceph_tid_t ObjectCacherWriteback::write(const object_t& oid,
 
   auto io_context = m_ictx->duplicate_data_io_context();
   if (!snapc.empty()) {
-    io_context->write_snap_context(
+    io_context->set_write_snap_context(
       {{snapc.seq, {snapc.snaps.begin(), snapc.snaps.end()}}});
   }
 
diff --git a/src/librbd/cache/ParentCacheObjectDispatch.cc b/src/librbd/cache/ParentCacheObjectDispatch.cc
index d5ef373abcf1..7d561098b78d 100644
--- a/src/librbd/cache/ParentCacheObjectDispatch.cc
+++ b/src/librbd/cache/ParentCacheObjectDispatch.cc
@@ -102,7 +102,7 @@ bool ParentCacheObjectDispatch<I>::read(
 
   m_cache_client->lookup_object(m_image_ctx->data_ctx.get_namespace(),
                                 m_image_ctx->data_ctx.get_id(),
-                                io_context->read_snap().value_or(CEPH_NOSNAP),
+                                io_context->get_read_snap(),
                                 m_image_ctx->layout.object_size,
                                 oid, std::move(ctx));
   return true;
@@ -141,7 +141,7 @@ void ParentCacheObjectDispatch<I>::handle_read_cache(
         on_dispatched->complete(r);
       });
     m_plugin_api.read_parent(m_image_ctx, object_no, extents,
-                             io_context->read_snap().value_or(CEPH_NOSNAP),
+                             io_context->get_read_snap(),
                              parent_trace, ctx);
     return;
   }
diff --git a/src/librbd/cache/WriteLogImageDispatch.cc b/src/librbd/cache/WriteLogImageDispatch.cc
index 7e0040fe7da8..bab082addf8a 100644
--- a/src/librbd/cache/WriteLogImageDispatch.cc
+++ b/src/librbd/cache/WriteLogImageDispatch.cc
@@ -49,7 +49,7 @@ bool WriteLogImageDispatch<I>::read(
   auto cct = m_image_ctx->cct;
   ldout(cct, 20) << "image_extents=" << image_extents << dendl;
 
-  if (io_context->read_snap().value_or(CEPH_NOSNAP) != CEPH_NOSNAP) {
+  if (io_context->get_read_snap() != CEPH_NOSNAP) {
     return false;
   }
 
diff --git a/src/librbd/crypto/CryptoObjectDispatch.cc b/src/librbd/crypto/CryptoObjectDispatch.cc
index 6ba449099410..8f33f259a2b0 100644
--- a/src/librbd/crypto/CryptoObjectDispatch.cc
+++ b/src/librbd/crypto/CryptoObjectDispatch.cc
@@ -102,7 +102,7 @@ struct C_AlignedObjectReadRequest : public Context {
       if (r == -ENOENT && !disable_read_from_parent) {
         io::util::read_parent<I>(
                 image_ctx, object_no, extents,
-                io_context->read_snap().value_or(CEPH_NOSNAP),
+                io_context->get_read_snap(),
                 parent_trace, this);
       } else {
         complete(r);
diff --git a/src/librbd/deep_copy/ObjectCopyRequest.cc b/src/librbd/deep_copy/ObjectCopyRequest.cc
index 3c1b9460140a..0bf9855cbc1e 100644
--- a/src/librbd/deep_copy/ObjectCopyRequest.cc
+++ b/src/librbd/deep_copy/ObjectCopyRequest.cc
@@ -150,7 +150,7 @@ void ObjectCopyRequest<I>::send_read() {
   }
 
   auto io_context = m_src_image_ctx->duplicate_data_io_context();
-  io_context->read_snap(index.second);
+  io_context->set_read_snap(index.second);
 
   io::Extents image_extents{read_op.image_interval.begin(),
                             read_op.image_interval.end()};
diff --git a/src/librbd/io/CopyupRequest.cc b/src/librbd/io/CopyupRequest.cc
index 228f95980177..855e60523c03 100644
--- a/src/librbd/io/CopyupRequest.cc
+++ b/src/librbd/io/CopyupRequest.cc
@@ -470,7 +470,7 @@ void CopyupRequest<I>::copyup() {
     ldout(cct, 20) << "copyup with empty snapshot context" << dendl;
 
     auto copyup_io_context = *io_context;
-    copyup_io_context.write_snap_context({});
+    copyup_io_context.set_write_snap_context({});
 
     m_image_ctx->rados_api.execute(
       object, copyup_io_context, std::move(copyup_op),
diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 2bf06966c360..86650869616d 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -44,7 +44,7 @@ template <typename I>
 inline bool is_copy_on_read(I *ictx, const IOContext& io_context) {
   std::shared_lock image_locker{ictx->image_lock};
   return (ictx->clone_copy_on_read && !ictx->read_only &&
-          io_context->read_snap().value_or(CEPH_NOSNAP) == CEPH_NOSNAP &&
+          io_context->get_read_snap() == CEPH_NOSNAP &&
           (ictx->exclusive_lock == nullptr ||
            ictx->exclusive_lock->is_lock_owner()));
 }
@@ -155,7 +155,7 @@ bool ObjectRequest<I>::compute_parent_extents(Extents *parent_extents,
 
   uint64_t raw_overlap;
   int r = m_ictx->get_parent_overlap(
-      m_io_context->read_snap().value_or(CEPH_NOSNAP), &raw_overlap);
+      m_io_context->get_read_snap(), &raw_overlap);
   if (r < 0) {
     // NOTE: it's possible for a snapshot to be deleted while we are
     // still reading from it
@@ -222,7 +222,7 @@ void ObjectReadRequest<I>::read_object() {
   I *image_ctx = this->m_ictx;
 
   std::shared_lock image_locker{image_ctx->image_lock};
-  auto read_snap_id = this->m_io_context->read_snap().value_or(CEPH_NOSNAP);
+  auto read_snap_id = this->m_io_context->get_read_snap();
   if (read_snap_id == image_ctx->snap_id &&
       image_ctx->object_map != nullptr &&
       !image_ctx->object_map->object_may_exist(this->m_object_no)) {
@@ -289,7 +289,7 @@ void ObjectReadRequest<I>::read_parent() {
 
   io::util::read_parent<I>(
     image_ctx, this->m_object_no, this->m_extents,
-    this->m_io_context->read_snap().value_or(CEPH_NOSNAP), this->m_trace,
+    this->m_io_context->get_read_snap(), this->m_trace,
     ctx);
 }
 
@@ -389,7 +389,7 @@ void AbstractObjectWriteRequest<I>::compute_parent_info() {
 
   if (!this->has_parent() ||
       (m_full_object &&
-       !this->m_io_context->write_snap_context() &&
+       !this->m_io_context->get_write_snap_context() &&
        !is_post_copyup_write_required())) {
     m_copyup_enabled = false;
   }
@@ -491,8 +491,8 @@ void AbstractObjectWriteRequest<I>::write_object() {
   neorados::WriteOp write_op;
   if (m_copyup_enabled) {
     if (m_guarding_migration_write) {
-      auto snap_seq = (this->m_io_context->write_snap_context() ?
-          this->m_io_context->write_snap_context()->first : 0);
+      auto snap_seq = (this->m_io_context->get_write_snap_context() ?
+          this->m_io_context->get_write_snap_context()->first : 0);
       ldout(image_ctx->cct, 20) << "guarding write: snap_seq=" << snap_seq
                                 << dendl;
 
@@ -738,7 +738,7 @@ ObjectListSnapsRequest<I>::ObjectListSnapsRequest(
     m_object_extents(std::move(object_extents)),
     m_snap_ids(std::move(snap_ids)), m_list_snaps_flags(list_snaps_flags),
     m_snapshot_delta(snapshot_delta) {
-  this->m_io_context->read_snap(CEPH_SNAPDIR);
+  this->m_io_context->set_read_snap(CEPH_SNAPDIR);
 }
 
 template <typename I>
diff --git a/src/librbd/migration/ImageDispatch.cc b/src/librbd/migration/ImageDispatch.cc
index 1ae143d78615..ad9fdb45166f 100644
--- a/src/librbd/migration/ImageDispatch.cc
+++ b/src/librbd/migration/ImageDispatch.cc
@@ -44,7 +44,7 @@ bool ImageDispatch<I>::read(
   ldout(cct, 20) << dendl;
 
   *dispatch_result = io::DISPATCH_RESULT_COMPLETE;
-  return m_format->read(aio_comp, io_context->read_snap().value_or(CEPH_NOSNAP),
+  return m_format->read(aio_comp, io_context->get_read_snap(),
                         std::move(image_extents), std::move(read_result),
                         op_flags, read_flags, parent_trace);
 }
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 667127d11253..3b3259b8aea6 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -143,20 +143,15 @@ IOContext::IOContext() {
   new (&impl) IOContextImpl();
 }
 
-IOContext::IOContext(std::int64_t _pool) : IOContext() {
-  pool(_pool);
+IOContext::IOContext(std::int64_t pool) : IOContext() {
+  set_pool(pool);
 }
 
-IOContext::IOContext(std::int64_t _pool, std::string_view _ns)
+IOContext::IOContext(std::int64_t pool, std::string ns, std::string key)
   : IOContext() {
-  pool(_pool);
-  ns(_ns);
-}
-
-IOContext::IOContext(std::int64_t _pool, std::string&& _ns)
-  : IOContext() {
-  pool(_pool);
-  ns(std::move(_ns));
+  set_pool(pool);
+  set_ns(std::move(ns));
+  set_key(std::move(key));
 }
 
 IOContext::~IOContext() {
@@ -186,88 +181,77 @@ IOContext& IOContext::operator =(IOContext&& rhs) {
   return *this;
 }
 
-std::int64_t IOContext::pool() const {
+std::int64_t IOContext::get_pool() const {
   return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.pool;
 }
 
-void IOContext::pool(std::int64_t _pool) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.pool = _pool;
+void IOContext::set_pool(std::int64_t pool) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->oloc.pool = pool;
 }
 
-std::string_view IOContext::ns() const {
-  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.nspace;
+IOContext&& IOContext::set_pool(std::int64_t pool) && {
+  set_pool(pool);
+  return std::move(*this);
 }
 
-void IOContext::ns(std::string_view _ns) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = _ns;
+std::string_view IOContext::get_ns() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.nspace;
 }
 
-void IOContext::ns(std::string&& _ns) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = std::move(_ns);
+void IOContext::set_ns(std::string ns) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = std::move(ns);
 }
 
-std::optional<std::string_view> IOContext::key() const {
-  auto& oloc = reinterpret_cast<const IOContextImpl*>(&impl)->oloc;
-  if (oloc.key.empty())
-    return std::nullopt;
-  else
-    return std::string_view(oloc.key);
+IOContext&& IOContext::set_ns(std::string ns) && {
+  set_ns(std::move(ns));
+  return std::move(*this);
 }
 
-void IOContext::key(std::string_view _key) {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = -1;
-  oloc.key = _key;
+std::string_view IOContext::get_key() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.key;
 }
 
-void IOContext::key(std::string&&_key) {
+void IOContext::set_key(std::string key) & {
   auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
   oloc.hash = -1;
-  oloc.key = std::move(_key);
+  oloc.key = std::move(key);
 }
 
-void IOContext::clear_key() {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = -1;
-  oloc.key.clear();
+IOContext&& IOContext::set_key(std::string key) && {
+  set_key(std::move(key));
+  return std::move(*this);
 }
 
-std::optional<std::int64_t> IOContext::hash() const {
-  auto& oloc = reinterpret_cast<const IOContextImpl*>(&impl)->oloc;
-  if (oloc.hash < 0)
-    return std::nullopt;
-  else
-    return oloc.hash;
+std::int64_t IOContext::get_hash() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.hash;
 }
 
-void IOContext::hash(std::int64_t _hash) {
+void IOContext::set_hash(std::int64_t hash) & {
   auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = _hash;
+  oloc.hash = hash;
   oloc.key.clear();
 }
 
-void IOContext::clear_hash() {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = -1;
-  oloc.key.clear();
+IOContext&& IOContext::set_hash(std::int64_t hash) && {
+  set_hash(hash);
+  return std::move(*this);
 }
 
+std::uint64_t IOContext::get_read_snap() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->snap_seq;
+}
 
-std::optional<std::uint64_t> IOContext::read_snap() const {
-  auto& snap_seq = reinterpret_cast<const IOContextImpl*>(&impl)->snap_seq;
-  if (snap_seq == CEPH_NOSNAP)
-    return std::nullopt;
-  else
-    return snap_seq;
+void IOContext::set_read_snap(std::uint64_t snapid) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->snap_seq = snapid;
 }
-void IOContext::read_snap(std::optional<std::uint64_t> _snapid) {
-  auto& snap_seq = reinterpret_cast<IOContextImpl*>(&impl)->snap_seq;
-  snap_seq = _snapid.value_or(CEPH_NOSNAP);
+
+IOContext&& IOContext::set_read_snap(std::uint64_t snapid) && {
+  set_read_snap(snapid);
+  return std::move(*this);
 }
 
-std::optional<
-  std::pair<std::uint64_t,
-	    std::vector<std::uint64_t>>> IOContext::write_snap_context() const {
+std::optional<std::pair<std::uint64_t, std::vector<std::uint64_t>>>
+IOContext::get_write_snap_context() const {
   auto& snapc = reinterpret_cast<const IOContextImpl*>(&impl)->snapc;
   if (snapc.empty()) {
     return std::nullopt;
@@ -277,8 +261,9 @@ std::optional<
   }
 }
 
-void IOContext::write_snap_context(
-  std::optional<std::pair<std::uint64_t, std::vector<std::uint64_t>>> _snapc) {
+void IOContext::set_write_snap_context(
+  std::optional<std::pair<std::uint64_t,
+                          std::vector<std::uint64_t>>> _snapc) & {
   auto& snapc = reinterpret_cast<IOContextImpl*>(&impl)->snapc;
   if (!_snapc) {
     snapc.clear();
@@ -295,20 +280,32 @@ void IOContext::write_snap_context(
   }
 }
 
-bool IOContext::full_try() const {
+IOContext&& IOContext::set_write_snap_context(
+  std::optional<std::pair<std::uint64_t,
+                          std::vector<std::uint64_t>>> snapc) && {
+  set_write_snap_context(std::move(snapc));
+  return std::move(*this);
+}
+
+bool IOContext::get_full_try() const {
   const auto ioc = reinterpret_cast<const IOContextImpl*>(&impl);
   return (ioc->extra_op_flags & CEPH_OSD_FLAG_FULL_TRY) != 0;
 }
 
-void IOContext::full_try(bool _full_try) {
+void IOContext::set_full_try(bool full_try) & {
   auto ioc = reinterpret_cast<IOContextImpl*>(&impl);
-  if (_full_try) {
+  if (full_try) {
     ioc->extra_op_flags |= CEPH_OSD_FLAG_FULL_TRY;
   } else {
     ioc->extra_op_flags &= ~CEPH_OSD_FLAG_FULL_TRY;
   }
 }
 
+IOContext&& IOContext::set_full_try(bool full_try) && {
+  set_full_try(full_try);
+  return std::move(*this);
+}
+
 bool operator <(const IOContext& lhs, const IOContext& rhs) {
   const auto l = reinterpret_cast<const IOContextImpl*>(&lhs.impl);
   const auto r = reinterpret_cast<const IOContextImpl*>(&rhs.impl);
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index 0de2cd9029d6..6d3bd0de7527 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -49,8 +49,8 @@ class Client {
   }
 
   librados::TestIoCtxImpl* get_io_ctx(const IOContext& ioc) {
-    int64_t pool_id = ioc.pool();
-    std::string ns = std::string{ioc.ns()};
+    int64_t pool_id = ioc.get_pool();
+    std::string ns = std::string{ioc.get_ns()};
 
     auto lock = std::scoped_lock{mutex};
     auto key = make_pair(pool_id, ns);
@@ -157,47 +157,72 @@ IOContext::IOContext(const IOContext& rhs) {
   new (&impl) IOContextImpl(*reinterpret_cast<const IOContextImpl*>(&rhs.impl));
 }
 
-IOContext::IOContext(int64_t _pool, std::string&& _ns)
+IOContext::IOContext(int64_t pool, std::string ns, std::string key)
   : IOContext() {
-  pool(_pool);
-  ns(std::move(_ns));
+  set_pool(pool);
+  set_ns(std::move(ns));
+  set_key(std::move(key));
 }
 
 IOContext::~IOContext() {
   reinterpret_cast<IOContextImpl*>(&impl)->~IOContextImpl();
 }
 
-std::int64_t IOContext::pool() const {
+std::int64_t IOContext::get_pool() const {
   return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.pool;
 }
 
-void IOContext::pool(std::int64_t _pool) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.pool = _pool;
+void IOContext::set_pool(std::int64_t pool) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->oloc.pool = pool;
 }
 
-std::string_view IOContext::ns() const {
+IOContext&& IOContext::set_pool(std::int64_t pool) && {
+  set_pool(pool);
+  return std::move(*this);
+}
+
+std::string_view IOContext::get_ns() const {
   return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.nspace;
 }
 
-void IOContext::ns(std::string&& _ns) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = std::move(_ns);
+void IOContext::set_ns(std::string ns) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = std::move(ns);
 }
 
-std::optional<std::uint64_t> IOContext::read_snap() const {
-  auto& snap_seq = reinterpret_cast<const IOContextImpl*>(&impl)->snap_seq;
-  if (snap_seq == CEPH_NOSNAP)
-    return std::nullopt;
-  else
-    return snap_seq;
+IOContext&& IOContext::set_ns(std::string ns) && {
+  set_ns(std::move(ns));
+  return std::move(*this);
+}
+
+std::string_view IOContext::get_key() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.key;
+}
+
+void IOContext::set_key(std::string key) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->oloc.key = std::move(key);
 }
-void IOContext::read_snap(std::optional<std::uint64_t> _snapid) {
-  auto& snap_seq = reinterpret_cast<IOContextImpl*>(&impl)->snap_seq;
-  snap_seq = _snapid.value_or(CEPH_NOSNAP);
+
+IOContext&& IOContext::set_key(std::string key) && {
+  set_key(std::move(key));
+  return std::move(*this);
 }
 
+std::uint64_t IOContext::get_read_snap() const {
+  return reinterpret_cast<const IOContextImpl*>(&impl)->snap_seq;
+}
+
+void IOContext::set_read_snap(std::uint64_t snapid) & {
+  reinterpret_cast<IOContextImpl*>(&impl)->snap_seq = snapid;
+}
+IOContext&& IOContext::set_read_snap(std::uint64_t snapid) && {
+  set_read_snap(snapid);
+  return std::move(*this);
+}
+
+
 std::optional<
   std::pair<std::uint64_t,
-            std::vector<std::uint64_t>>> IOContext::write_snap_context() const {
+            std::vector<std::uint64_t>>> IOContext::get_write_snap_context() const {
   auto& snapc = reinterpret_cast<const IOContextImpl*>(&impl)->snapc;
   if (snapc.empty()) {
     return std::nullopt;
@@ -207,8 +232,8 @@ std::optional<
   }
 }
 
-void IOContext::write_snap_context(
-  std::optional<std::pair<std::uint64_t, std::vector<std::uint64_t>>> _snapc) {
+void IOContext::set_write_snap_context(
+  std::optional<std::pair<std::uint64_t, std::vector<std::uint64_t>>> _snapc) & {
   auto& snapc = reinterpret_cast<IOContextImpl*>(&impl)->snapc;
   if (!_snapc) {
     snapc.clear();
@@ -224,7 +249,7 @@ void IOContext::write_snap_context(
   }
 }
 
-void IOContext::full_try(bool _full_try) {
+void IOContext::set_full_try(bool _full_try) & {
   // no-op
 }
 
@@ -519,11 +544,7 @@ void RADOS::execute(const Object& o, const IOContext& ioc, ReadOp&& op,
 
   auto ops = *reinterpret_cast<librados::TestObjectOperationImpl**>(&op.impl);
 
-  auto snap_id = CEPH_NOSNAP;
-  auto opt_snap_id = ioc.read_snap();
-  if (opt_snap_id) {
-    snap_id = *opt_snap_id;
-  }
+  auto snap_id = ioc.get_read_snap();
 
   auto completion = create_aio_completion(std::move(c));
   auto r = io_ctx->aio_operate_read(std::string{o}, *ops, completion, 0U, bl,
@@ -543,7 +564,7 @@ void RADOS::execute(const Object& o, const IOContext& ioc, WriteOp&& op,
   auto ops = *reinterpret_cast<librados::TestObjectOperationImpl**>(&op.impl);
 
   SnapContext snapc;
-  auto opt_snapc = ioc.write_snap_context();
+  auto opt_snapc = ioc.get_write_snap_context();
   if (opt_snapc) {
     snapc.seq = opt_snapc->first;
     snapc.snaps.assign(opt_snapc->second.begin(), opt_snapc->second.end());
diff --git a/src/test/librbd/deep_copy/test_mock_ObjectCopyRequest.cc b/src/test/librbd/deep_copy/test_mock_ObjectCopyRequest.cc
index d813a5a33bcf..5fbb4d6ce5c0 100644
--- a/src/test/librbd/deep_copy/test_mock_ObjectCopyRequest.cc
+++ b/src/test/librbd/deep_copy/test_mock_ObjectCopyRequest.cc
@@ -126,7 +126,7 @@ MATCHER(IsListSnaps, "") {
 MATCHER_P2(IsRead, snap_id, image_interval, "") {
   auto req = boost::get<io::ImageDispatchSpec::Read>(&arg->request);
   if (req == nullptr ||
-      arg->io_context->read_snap().value_or(CEPH_NOSNAP) != snap_id) {
+      arg->io_context->get_read_snap() != snap_id) {
     return false;
   }
 
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index c20c825018bb..d7f9156d7eda 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -395,7 +395,7 @@ struct TestMockIoObjectRequest : public TestMockFixture {
   void expect_list_snaps(MockTestImageCtx &mock_image_ctx,
                         const librados::snap_set_t& snap_set, int r) {
     auto io_context = *mock_image_ctx.get_data_io_context();
-    io_context.read_snap(CEPH_SNAPDIR);
+    io_context.set_read_snap(CEPH_SNAPDIR);
     auto& mock_io_ctx = librados::get_mock_io_ctx(mock_image_ctx.rados_api,
                                                   io_context);
     EXPECT_CALL(mock_io_ctx, list_snaps(_, _))
diff --git a/src/test/librbd/mock/MockImageCtx.cc b/src/test/librbd/mock/MockImageCtx.cc
index 52e8e2b43491..0826f32061ca 100644
--- a/src/test/librbd/mock/MockImageCtx.cc
+++ b/src/test/librbd/mock/MockImageCtx.cc
@@ -133,10 +133,10 @@ IOContext MockImageCtx::get_data_io_context() {
   auto ctx = std::make_shared<neorados::IOContext>(
     data_ctx.get_id(), data_ctx.get_namespace());
   if (snap_id != CEPH_NOSNAP) {
-    ctx->read_snap(snap_id);
+    ctx->set_read_snap(snap_id);
   }
   if (!snapc.snaps.empty()) {
-    ctx->write_snap_context(
+    ctx->set_write_snap_context(
       {{snapc.seq, {snapc.snaps.begin(), snapc.snaps.end()}}});
   }
   return ctx;
diff --git a/src/test/neorados/list_pool.cc b/src/test/neorados/list_pool.cc
index 4a1dc9cb7b1b..95e05bd2ab73 100644
--- a/src/test/neorados/list_pool.cc
+++ b/src/test/neorados/list_pool.cc
@@ -83,9 +83,9 @@ bs::error_code noisy_list(R::RADOS& r, int64_t p)
 
   std::cout << "begin = " << b.to_str() << std::endl;
   std::cout << "end = " << e.to_str() << std::endl;
-  R::IOContext pool{p, R::all_nspaces};
   try {
-    auto [v, next] = r.enumerate_objects(pool, b, e, 1000, {}, ca::use_blocked);
+    auto [v, next] = r.enumerate_objects({p, R::all_nspaces}, b, e, 1000, {},
+					 ca::use_blocked);
 
     std::cout << "Got " << v.size() << " entries." << std::endl;
     std::cout << "next cursor = " << next.to_str() << std::endl;
diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
index 097dedb6e499..c59cf739fafb 100644
--- a/src/tools/neorados.cc
+++ b/src/tools/neorados.cc
@@ -92,8 +92,7 @@ void lspools(R::RADOS& r, const std::vector<std::string>&,
 void ls(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 {
   const auto& pname = p[0];
-  auto pool = lookup_pool(r, pname, y);
-  pool.ns(R::all_nspaces);
+  const auto pool = lookup_pool(r, pname, y).set_ns(R::all_nspaces);
 
   std::vector<R::Entry> ls;
   R::Cursor next = R::Cursor::begin();
@@ -213,7 +212,7 @@ void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
       throw bs::system_error(
 	ec,
 	fmt::format("when reading from object '{}' in pool '{}'",
-		    obj, pool.pool()));
+		    obj, pool));
 
     off += bl.length();
     bl.write_stream(std::cout);

From af645b15b5f4723590bc5f750de52c4057cbff2b Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Tue, 8 Mar 2022 19:14:59 -0500
Subject: [PATCH 0934/2492] neorados: Switch to async_initiate

This form of completion handling is compatible with C++20 Completions
and generally more flexible stuff.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp | 304 ++++++++++++++++++---------------
 1 file changed, 165 insertions(+), 139 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 915c3c4d1332..737d5da7e28c 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -474,6 +474,13 @@ class Cursor final {
   std::aligned_storage_t<impl_size> impl;
 };
 
+// Clang reports a spurious warning that a captured `this` is unused
+// in the public 'wrapper' functions that construct the completion
+// handler and pass it to the actual worker member functions. The `this` is
+// used to call the member functions, and even doing so explicitly
+// (e.g. `this->execute`) doesn't silence it.
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-lambda-capture"
 class RADOS final
 {
 public:
@@ -517,11 +524,11 @@ class RADOS final
 
     template<typename CompletionToken>
     auto build(boost::asio::io_context& ioctx, CompletionToken&& token) {
-      boost::asio::async_completion<CompletionToken, BuildSig> init(token);
-      build(ioctx,
-	    BuildComp::create(ioctx.get_executor(),
-			      std::move(init.completion_handler)));
-      return init.result.get();
+      return boost::asio::async_initiate<CompletionToken, BuildSig>(
+	[&, this](auto&& handler) {
+	  build(ioctx, BuildComp::create(ioctx.get_executor(),
+					 std::move(handler)));
+	}, token);
     }
 
   private:
@@ -534,11 +541,12 @@ class RADOS final
   static auto make_with_cct(CephContext* cct,
 			    boost::asio::io_context& ioctx,
 			    CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, BuildSig> init(token);
-    make_with_cct(cct, ioctx,
-		  BuildComp::create(ioctx.get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, BuildSig>(
+      [&](auto&& handler) {
+	make_with_cct(cct, ioctx,
+		      BuildComp::create(ioctx.get_executor(),
+					std::move(handler)));
+      }, token);
   }
 
   static RADOS make_with_librados(librados::Rados& rados);
@@ -562,24 +570,26 @@ class RADOS final
 	       ceph::buffer::list* bl,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, ioc, std::move(op), bl,
-	    ReadOp::Completion::create(get_executor(),
-				       std::move(init.completion_handler)),
-	    objver, trace_info);
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, Op::Signature>(
+      [&, this](auto&& handler) {
+	execute(o, ioc, std::move(op), bl,
+		ReadOp::Completion::create(get_executor(),
+					   std::move(handler)),
+		objver, trace_info);
+      }, token);
   }
 
   template<typename CompletionToken>
   auto execute(const Object& o, const IOContext& ioc, WriteOp&& op,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, ioc, std::move(op),
-	    Op::Completion::create(get_executor(),
-				   std::move(init.completion_handler)),
-	    objver, trace_info);
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, Op::Signature>(
+      [&, this](auto&& handler) {
+	execute(o, ioc, std::move(op),
+		WriteOp::Completion::create(get_executor(),
+					    std::move(handler)),
+		objver, trace_info);
+      }, token);
   }
 
   boost::uuids::uuid get_fsid() const noexcept;
@@ -590,11 +600,11 @@ class RADOS final
   template<typename CompletionToken>
   auto lookup_pool(std::string_view name,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, LookupPoolSig> init(token);
-    lookup_pool(name,
-		LookupPoolComp::create(get_executor(),
-				       std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, LookupPoolSig>(
+      [&, this](auto&& handler) {
+	lookup_pool(name, LookupPoolComp::create(get_executor(),
+						 std::move(handler)));
+      }, token);
   }
 
   std::optional<uint64_t> get_pool_alignment(int64_t pool_id);
@@ -603,24 +613,24 @@ class RADOS final
   using LSPoolsComp = ceph::async::Completion<LSPoolsSig>;
   template<typename CompletionToken>
   auto list_pools(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, LSPoolsSig> init(token);
-    list_pools(LSPoolsComp::create(get_executor(),
-				   std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, LSPoolsSig>(
+      [this](auto&& handler) {
+	list_pools(LSPoolsComp::create(get_executor(),
+				       std::move(handler)));
+      }, token);
   }
 
-
-
   using SimpleOpSig = void(boost::system::error_code);
   using SimpleOpComp = ceph::async::Completion<SimpleOpSig>;
   template<typename CompletionToken>
   auto create_pool_snap(int64_t pool, std::string_view snapName,
 			CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    create_pool_snap(pool, snapName,
-		     SimpleOpComp::create(get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	create_pool_snap(pool, snapName,
+			 SimpleOpComp::create(get_executor(),
+					      std::move(handler)));
+      }, token);
   }
 
   using SMSnapSig = void(boost::system::error_code, std::uint64_t);
@@ -628,63 +638,67 @@ class RADOS final
   template<typename CompletionToken>
   auto allocate_selfmanaged_snap(int64_t pool,
 				 CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SMSnapSig> init(token);
-    allocate_selfmanaged_snap(pool,
-			      SMSnapComp::create(
-				get_executor(),
-				std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SMSnapSig>(
+      [&, this](auto&& handler) {
+	allocage_selfmanaged_snap(pool,
+				  SMSnapComp::create(get_executor(),
+						     std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto delete_pool_snap(int64_t pool, std::string_view snapName,
 			CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool_snap(pool, snapName,
-		     SimpleOpComp::create(get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	delete_pool_snap(pool, snapName,
+			 SimpleOpComp::create(get_executor(),
+					      std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto delete_selfmanaged_snap(int64_t pool, std::string_view snapName,
 			       CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_selfmanaged_snap(pool, snapName,
-			    SimpleOpComp::create(
-			      get_executor(),
-			      std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	delete_selfmanaged_snap(pool, snapName,
+				SimpleOpComp::create(get_executor(),
+						     std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto create_pool(std::string_view name, std::optional<int> crush_rule,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    create_pool(name, crush_rule,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	create_pool(name, crush_rule,
+		    SimpleOpComp::create(get_executor(),
+					 std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto delete_pool(std::string_view name,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool(name,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	delete_pool(name,
+		    SimpleOpComp::create(get_executor(),
+					 std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto delete_pool(int64_t pool,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool(pool,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	delete_pool(pool,
+		    SimpleOpComp::create(get_executor(),
+					 std::move(handler)));
+      }, token);
   }
 
   using PoolStatSig = void(boost::system::error_code,
@@ -694,11 +708,12 @@ class RADOS final
   template<typename CompletionToken>
   auto stat_pools(const std::vector<std::string>& pools,
 		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, PoolStatSig> init(token);
-    stat_pools(pools,
-	       PoolStatComp::create(get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, PoolStatSig>(
+      [&, this](auto&& handler) {
+	stat_pools(pools,
+		   PoolStatComp::create(get_executor(),
+					std::move(handler)));
+      }, token);
   }
 
   using StatFSSig = void(boost::system::error_code,
@@ -707,10 +722,11 @@ class RADOS final
   template<typename CompletionToken>
   auto statfs(std::optional<int64_t> pool,
 	      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, StatFSSig> init(token);
-    ceph_statfs(pool, StatFSComp::create(get_executor(),
-					 std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, StatFSSig>(
+      [&, this](auto&& handler) {
+	statfs(pool, StatFSComp::create(get_executor(),
+					std::move(handler)));
+      }, token);
   }
 
   using WatchCB = fu2::unique_function<void(boost::system::error_code,
@@ -726,11 +742,11 @@ class RADOS final
   auto watch(const Object& o, const IOContext& ioc,
 	     std::optional<std::chrono::seconds> timeout,
 	     WatchCB&& cb, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    watch(o, ioc, timeout, std::move(cb),
-	  WatchComp::create(get_executor(),
-			    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, WatchSig>(
+      [&, this](auto&& handler) {
+	watch(o, ioc, timeout, cb, WatchComp::create(get_executor(),
+						     std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
@@ -740,21 +756,22 @@ class RADOS final
 		  uint64_t cookie,
 		  ceph::buffer::list&& bl,
 		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    notify_ack(o, ioc, notify_id, cookie, std::move(bl),
-	       SimpleOpComp::create(get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	notify_ack(o, ioc, notify_id, cookie, std::move(bl),
+		   SimpleOpComp::create(get_executor(),
+					std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto unwatch(uint64_t cookie, const IOContext& ioc,
 	       CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    unwatch(cookie, ioc,
-	    SimpleOpComp::create(get_executor(),
-				 std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	unwatch(cookie, ioc, SimpleOpComp::create(get_executor(),
+						  std::move(handler)));
+      }, token);
   }
 
   // This is one of those places where having to force everything into
@@ -765,10 +782,11 @@ class RADOS final
   using VoidOpComp = ceph::async::Completion<VoidOpSig>;
   template<typename CompletionToken>
   auto flush_watch(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, VoidOpSig> init(token);
-    flush_watch(VoidOpComp::create(get_executor(),
-				   std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, VoidOpSig>(
+      [this](auto&& handler) {
+	flush_watch(VoidOpComp::create(get_executor(),
+				       std::move(handler)));
+      }, token);
   }
 
   using NotifySig = void(boost::system::error_code, ceph::buffer::list);
@@ -777,12 +795,12 @@ class RADOS final
   auto notify(const Object& oid, const IOContext& ioc, ceph::buffer::list&& bl,
 	      std::optional<std::chrono::milliseconds> timeout,
 	      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, NotifySig> init(token);
-    notify(oid, ioc, std::move(bl), timeout,
-	   NotifyComp::create(get_executor(),
-			      std::move(init.completion_handler)));
-
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, NotifySig>(
+      [&, this](auto&& handler) {
+	notify(oid, ioc, std::move(bl), timeout,
+	       NotifyComp::create(get_executor(),
+				  std::move(handler)));
+      }, token);
   }
 
   // The versions with pointers are fine for coroutines, but
@@ -796,11 +814,12 @@ class RADOS final
 			 const Cursor& end, const std::uint32_t max,
 			 const ceph::buffer::list& filter,
 			 CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, EnumerateSig> init(token);
-    enumerate_objects(ioc, begin, end, max, filter,
-		      EnumerateComp::create(get_executor(),
-					    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, EnumerateSig>(
+      [&, this](auto&& handler) {
+	enumerate_objects(ioc, begin, end, max, filter,
+			  EnumerateComp::create(get_executor(),
+					     std::move(handler)));
+      }, token);
   }
 
   using CommandSig = void(boost::system::error_code,
@@ -809,61 +828,67 @@ class RADOS final
   template<typename CompletionToken>
   auto osd_command(int osd, std::vector<std::string>&& cmd,
 		   ceph::buffer::list&& in, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    osd_command(osd, std::move(cmd), std::move(in),
-		CommandComp::create(get_executor(),
-				      std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, CommandSig>(
+      [&, this](auto&& handler) {
+	osd_command(osd, std::move(cmd), std::move(in),
+		    CommandComp::create(get_executor(),
+					std::move(handler)));
+      }, token);
   }
   template<typename CompletionToken>
   auto pg_command(PG pg, std::vector<std::string>&& cmd,
 		  ceph::buffer::list&& in, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    pg_command(pg, std::move(cmd), std::move(in),
-	       CommandComp::create(get_executor(),
-				      std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, CommandSig>(
+      [&, this](auto&& handler) {
+	pg_command(pg, std::move(cmd), std::move(in),
+		   CommandComp::create(get_executor(),
+				       std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
-  auto mon_command(std::vector<std::string> command,
+  auto mon_command(const std::vector<std::string>& command,
 		   const ceph::buffer::list& bl,
 		   std::string* outs, ceph::buffer::list* outbl,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    mon_command(command, bl, outs, outbl,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	mon_command(command, bl, outs, outbl,
+		    SimpleOpComp::create(get_executor(),
+					 std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto enable_application(std::string_view pool, std::string_view app_name,
 			  bool force, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    enable_application(pool, app_name, force,
-		       SimpleOpComp::create(get_executor(),
-					    std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	enable_application(pool, app_name, force,
+			   SimpleOpComp::create(get_executor(),
+						std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto blocklist_add(std::string_view client_address,
                      std::optional<std::chrono::seconds> expire,
                      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    blocklist_add(client_address, expire,
-                  SimpleOpComp::create(get_executor(),
-                                       std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [&, this](auto&& handler) {
+	blocklist_add(client_address, expire,
+		      SimpleOpComp::create(get_executor(),
+					   std::move(handler)));
+      }, token);
   }
 
   template<typename CompletionToken>
   auto wait_for_latest_osd_map(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    wait_for_latest_osd_map(
-      SimpleOpComp::create(get_executor(), std::move(init.completion_handler)));
-    return init.result.get();
+    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+      [this](auto&& handler) {
+	wait_for_latest_osd_map(SimpleOpComp::create(get_executor(),
+						     std::move(handler)));
+      }, token);
   }
 
   uint64_t instance_id() const;
@@ -958,6 +983,7 @@ class RADOS final
   // Proxy object to provide access to low-level RADOS messaging clients
   std::unique_ptr<detail::Client> impl;
 };
+#pragma clang diagnostic pop
 
 enum class errc {
   pool_dne = 1,

From 06addeedffc239f7dca3497ff18f86bf111a59c1 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Thu, 17 Mar 2022 12:30:19 -0400
Subject: [PATCH 0935/2492] neorados: Don't call dispatch inside with_osdmap

The lock will continue to be held over the 'dispatch' with C++20
coroutines.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/neorados/RADOS.cc | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 3b3259b8aea6..c238ff450dc5 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -900,13 +900,15 @@ std::optional<uint64_t> RADOS::get_pool_alignment(int64_t pool_id)
 }
 
 void RADOS::list_pools(std::unique_ptr<LSPoolsComp> c) {
-  impl->objecter->with_osdmap(
-    [&](OSDMap& o) {
-      std::vector<std::pair<std::int64_t, std::string>> v;
-      for (auto p : o.get_pools())
-	v.push_back(std::make_pair(p.first, o.get_pool_name(p.first)));
-      ca::dispatch(std::move(c), std::move(v));
-    });
+  ca::dispatch(std::move(c),
+	       impl->objecter->with_osdmap(
+		 [&](OSDMap& o) {
+		   std::vector<std::pair<std::int64_t, std::string>> v;
+		   for (auto p : o.get_pools())
+		     v.push_back(std::make_pair(p.first,
+						o.get_pool_name(p.first)));
+		   return v;
+		 }));
 }
 
 void RADOS::create_pool_snap(std::int64_t pool,

From bab36e143389de00b5c70fbd80ff3617aee79c45 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 29 Nov 2023 01:03:07 -0500
Subject: [PATCH 0936/2492] neorados: No dangling references in arguments

`async_initiate` may call its lambda after the containing function
returns. As such, reference arguments may no longer point to anything by
the time we use them.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                | 228 +++++++++---------
 src/neorados/RADOS.cc                         |  66 +++--
 .../librados_test_stub/NeoradosTestStub.cc    |  32 ++-
 3 files changed, 168 insertions(+), 158 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 737d5da7e28c..1b0443f30ccb 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -525,7 +525,7 @@ class RADOS final
     template<typename CompletionToken>
     auto build(boost::asio::io_context& ioctx, CompletionToken&& token) {
       return boost::asio::async_initiate<CompletionToken, BuildSig>(
-	[&, this](auto&& handler) {
+	[&ioctx, this](auto&& handler) {
 	  build(ioctx, BuildComp::create(ioctx.get_executor(),
 					 std::move(handler)));
 	}, token);
@@ -542,7 +542,7 @@ class RADOS final
 			    boost::asio::io_context& ioctx,
 			    CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, BuildSig>(
-      [&](auto&& handler) {
+      [cct, &ioctx](auto&& handler) {
 	make_with_cct(cct, ioctx,
 		      BuildComp::create(ioctx.get_executor(),
 					std::move(handler)));
@@ -566,13 +566,14 @@ class RADOS final
   boost::asio::io_context& get_io_context();
 
   template<typename CompletionToken>
-  auto execute(const Object& o, const IOContext& ioc, ReadOp&& op,
+  auto execute(Object o, IOContext ioc, ReadOp op,
 	       ceph::buffer::list* bl,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
     return boost::asio::async_initiate<CompletionToken, Op::Signature>(
-      [&, this](auto&& handler) {
-	execute(o, ioc, std::move(op), bl,
+      [o = std::move(o), ioc = std::move(ioc), op = std::move(op),
+       bl, objver, trace_info, this](auto&& handler) mutable {
+	execute(std::move(o), std::move(ioc), std::move(op), bl,
 		ReadOp::Completion::create(get_executor(),
 					   std::move(handler)),
 		objver, trace_info);
@@ -580,12 +581,13 @@ class RADOS final
   }
 
   template<typename CompletionToken>
-  auto execute(const Object& o, const IOContext& ioc, WriteOp&& op,
+  auto execute(Object o, IOContext ioc, WriteOp op,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
     return boost::asio::async_initiate<CompletionToken, Op::Signature>(
-      [&, this](auto&& handler) {
-	execute(o, ioc, std::move(op),
+      [o = std::move(o), ioc = std::move(ioc), op = std::move(op),
+       objver, trace_info, this](auto&& handler) mutable {
+	execute(std::move(o), std::move(ioc), std::move(op),
 		WriteOp::Completion::create(get_executor(),
 					    std::move(handler)),
 		objver, trace_info);
@@ -598,12 +600,11 @@ class RADOS final
 			     std::int64_t);
   using LookupPoolComp = ceph::async::Completion<LookupPoolSig>;
   template<typename CompletionToken>
-  auto lookup_pool(std::string_view name,
-		   CompletionToken&& token) {
+  auto lookup_pool(std::string name, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, LookupPoolSig>(
-      [&, this](auto&& handler) {
-	lookup_pool(name, LookupPoolComp::create(get_executor(),
-						 std::move(handler)));
+      [name = std::move(name), this](auto&& handler) mutable {
+	lookup_pool(std::move(name),
+		    LookupPoolComp::create(get_executor(), std::move(handler)));
       }, token);
   }
 
@@ -623,11 +624,11 @@ class RADOS final
   using SimpleOpSig = void(boost::system::error_code);
   using SimpleOpComp = ceph::async::Completion<SimpleOpSig>;
   template<typename CompletionToken>
-  auto create_pool_snap(int64_t pool, std::string_view snapName,
+  auto create_pool_snap(int64_t pool, std::string snap_name,
 			CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	create_pool_snap(pool, snapName,
+      [snap_name = std::move(snap_name), pool, this](auto&& handler) mutable {
+	create_pool_snap(pool, std::move(snap_name),
 			 SimpleOpComp::create(get_executor(),
 					      std::move(handler)));
       }, token);
@@ -636,10 +637,9 @@ class RADOS final
   using SMSnapSig = void(boost::system::error_code, std::uint64_t);
   using SMSnapComp = ceph::async::Completion<SMSnapSig>;
   template<typename CompletionToken>
-  auto allocate_selfmanaged_snap(int64_t pool,
-				 CompletionToken&& token) {
+  auto allocate_selfmanaged_snap(int64_t pool, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SMSnapSig>(
-      [&, this](auto&& handler) {
+      [pool, this](auto&& handler) {
 	allocage_selfmanaged_snap(pool,
 				  SMSnapComp::create(get_executor(),
 						     std::move(handler)));
@@ -647,54 +647,52 @@ class RADOS final
   }
 
   template<typename CompletionToken>
-  auto delete_pool_snap(int64_t pool, std::string_view snapName,
+  auto delete_pool_snap(int64_t pool, std::string snap_name,
 			CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	delete_pool_snap(pool, snapName,
+      [snap_name = std::move(snap_name), pool, this](auto&& handler) mutable {
+	delete_pool_snap(pool, std::move(snap_name),
 			 SimpleOpComp::create(get_executor(),
 					      std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto delete_selfmanaged_snap(int64_t pool, std::string_view snapName,
+  auto delete_selfmanaged_snap(int64_t pool, std::uint64_t snap,
 			       CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	delete_selfmanaged_snap(pool, snapName,
+      [snap, pool, this](auto&& handler) {
+	delete_selfmanaged_snap(pool, std::move(snap),
 				SimpleOpComp::create(get_executor(),
 						     std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto create_pool(std::string_view name, std::optional<int> crush_rule,
+  auto create_pool(std::string name, std::optional<int> crush_rule,
 		   CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	create_pool(name, crush_rule,
+      [name = std::move(name), crush_rule, this](auto&& handler) mutable {
+	create_pool(std::move(name), crush_rule,
 		    SimpleOpComp::create(get_executor(),
 					 std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto delete_pool(std::string_view name,
-		   CompletionToken&& token) {
+  auto delete_pool(std::string name, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	delete_pool(name,
+      [name = std::move(name), this](auto&& handler) mutable {
+	delete_pool(std::move(name),
 		    SimpleOpComp::create(get_executor(),
 					 std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto delete_pool(int64_t pool,
-		   CompletionToken&& token) {
+  auto delete_pool(int64_t pool, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
+      [pool, this](auto&& handler) mutable {
 	delete_pool(pool,
 		    SimpleOpComp::create(get_executor(),
 					 std::move(handler)));
@@ -706,11 +704,10 @@ class RADOS final
 			                              PoolStats>, bool);
   using PoolStatComp = ceph::async::Completion<PoolStatSig>;
   template<typename CompletionToken>
-  auto stat_pools(const std::vector<std::string>& pools,
-		  CompletionToken&& token) {
+  auto stat_pools(std::vector<std::string> pools, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, PoolStatSig>(
-      [&, this](auto&& handler) {
-	stat_pools(pools,
+      [pools = std::move(pools), this](auto&& handler) mutable {
+	stat_pools(std::move(pools),
 		   PoolStatComp::create(get_executor(),
 					std::move(handler)));
       }, token);
@@ -720,10 +717,9 @@ class RADOS final
 			 FSStats);
   using StatFSComp = ceph::async::Completion<StatFSSig>;
   template<typename CompletionToken>
-  auto statfs(std::optional<int64_t> pool,
-	      CompletionToken&& token) {
+  auto statfs(std::optional<int64_t> pool, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, StatFSSig>(
-      [&, this](auto&& handler) {
+      [pool, this](auto&& handler) {
 	statfs(pool, StatFSComp::create(get_executor(),
 					std::move(handler)));
       }, token);
@@ -739,38 +735,40 @@ class RADOS final
 			uint64_t cookie);
   using WatchComp = ceph::async::Completion<WatchSig>;
   template<typename CompletionToken>
-  auto watch(const Object& o, const IOContext& ioc,
+  auto watch(Object o, IOContext ioc,
 	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, CompletionToken&& token) {
+	     WatchCB cb, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, WatchSig>(
-      [&, this](auto&& handler) {
-	watch(o, ioc, timeout, cb, WatchComp::create(get_executor(),
-						     std::move(handler)));
+      [o = std::move(o), ioc = std::move(ioc), timeout,
+       cb = std::move(cb), this](auto&& handler) mutable {
+	watch(std::move(o), std::move(ioc), timeout, std::move(cb),
+	      WatchComp::create(get_executor(), std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto notify_ack(const Object& o,
-		  const IOContext& ioc,
+  auto notify_ack(Object o, IOContext ioc,
 		  uint64_t notify_id,
 		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
+		  ceph::buffer::list bl,
 		  CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	notify_ack(o, ioc, notify_id, cookie, std::move(bl),
+      [o = std::move(o), ioc = std::move(ioc), notify_id,
+       cookie, bl = std::move(bl), this](auto&& handler) mutable {
+	notify_ack(std::move(o), std::move(ioc), notify_id, cookie,
+		   std::move(bl),
 		   SimpleOpComp::create(get_executor(),
 					std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto unwatch(uint64_t cookie, const IOContext& ioc,
+  auto unwatch(uint64_t cookie, IOContext ioc,
 	       CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	unwatch(cookie, ioc, SimpleOpComp::create(get_executor(),
-						  std::move(handler)));
+      [cookie, ioc = std::move(ioc), this](auto&& handler) mutable {
+	unwatch(cookie, std::move(ioc),
+		SimpleOpComp::create(get_executor(), std::move(handler)));
       }, token);
   }
 
@@ -792,12 +790,13 @@ class RADOS final
   using NotifySig = void(boost::system::error_code, ceph::buffer::list);
   using NotifyComp = ceph::async::Completion<NotifySig>;
   template<typename CompletionToken>
-  auto notify(const Object& oid, const IOContext& ioc, ceph::buffer::list&& bl,
+  auto notify(Object oid, IOContext ioc, ceph::buffer::list bl,
 	      std::optional<std::chrono::milliseconds> timeout,
 	      CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, NotifySig>(
-      [&, this](auto&& handler) {
-	notify(oid, ioc, std::move(bl), timeout,
+      [oid = std::move(oid), ioc = std::move(ioc),
+       bl = std::move(bl), timeout, this](auto&& handler) mutable {
+	notify(std::move(oid), std::move(ioc), std::move(bl), timeout,
 	       NotifyComp::create(get_executor(),
 				  std::move(handler)));
       }, token);
@@ -810,15 +809,17 @@ class RADOS final
 			    Cursor);
   using EnumerateComp = ceph::async::Completion<EnumerateSig>;
   template<typename CompletionToken>
-  auto enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
+  auto enumerate_objects(IOContext ioc, Cursor begin,
+			 Cursor end, std::uint32_t max,
+			 ceph::buffer::list filter,
 			 CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, EnumerateSig>(
-      [&, this](auto&& handler) {
-	enumerate_objects(ioc, begin, end, max, filter,
+      [ioc = std::move(ioc), begin = std::move(begin), end = std::move(end),
+       max, filter = std::move(filter), this](auto&& handler) mutable {
+	enumerate_objects(std::move(ioc), std::move(begin), std::move(end),
+			  max, std::move(filter),
 			  EnumerateComp::create(get_executor(),
-					     std::move(handler)));
+						std::move(handler)));
       }, token);
   }
 
@@ -826,20 +827,22 @@ class RADOS final
 			  std::string, ceph::buffer::list);
   using CommandComp = ceph::async::Completion<CommandSig>;
   template<typename CompletionToken>
-  auto osd_command(int osd, std::vector<std::string>&& cmd,
-		   ceph::buffer::list&& in, CompletionToken&& token) {
+  auto osd_command(int osd, std::vector<std::string> cmd,
+		   ceph::buffer::list in, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, CommandSig>(
-      [&, this](auto&& handler) {
+      [osd, cmd = std::move(cmd), in = std::move(in),
+       this](auto&& handler) mutable {
 	osd_command(osd, std::move(cmd), std::move(in),
 		    CommandComp::create(get_executor(),
 					std::move(handler)));
       }, token);
   }
   template<typename CompletionToken>
-  auto pg_command(PG pg, std::vector<std::string>&& cmd,
-		  ceph::buffer::list&& in, CompletionToken&& token) {
+  auto pg_command(PG pg, std::vector<std::string> cmd,
+		  ceph::buffer::list in, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, CommandSig>(
-      [&, this](auto&& handler) {
+      [pg, cmd = std::move(cmd), in = std::move(in),
+       this](auto&& handler) mutable {
 	pg_command(pg, std::move(cmd), std::move(in),
 		   CommandComp::create(get_executor(),
 				       std::move(handler)));
@@ -847,36 +850,39 @@ class RADOS final
   }
 
   template<typename CompletionToken>
-  auto mon_command(const std::vector<std::string>& command,
-		   const ceph::buffer::list& bl,
+  auto mon_command(std::vector<std::string> command,
+		   ceph::buffer::list bl,
 		   std::string* outs, ceph::buffer::list* outbl,
 		   CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	mon_command(command, bl, outs, outbl,
+      [command = std::move(command), bl = std::move(bl), outs,
+       outbl, this](auto&& handler) mutable {
+	mon_command(std::move(command), std::move(bl), outs, outbl,
 		    SimpleOpComp::create(get_executor(),
 					 std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto enable_application(std::string_view pool, std::string_view app_name,
+  auto enable_application(std::string pool, std::string app_name,
 			  bool force, CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	enable_application(pool, app_name, force,
+      [pool = std::move(pool), app_name = std::move(app_name),
+       force, this](auto&& handler) mutable {
+	enable_application(std::move(pool), std::move(app_name), force,
 			   SimpleOpComp::create(get_executor(),
 						std::move(handler)));
       }, token);
   }
 
   template<typename CompletionToken>
-  auto blocklist_add(std::string_view client_address,
+  auto blocklist_add(std::string client_address,
                      std::optional<std::chrono::seconds> expire,
                      CompletionToken&& token) {
     return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [&, this](auto&& handler) {
-	blocklist_add(client_address, expire,
+      [client_address = std::move(client_address), expire,
+       this](auto&& handler) mutable {
+	blocklist_add(std::move(client_address), expire,
 		      SimpleOpComp::create(get_executor(),
 					   std::move(handler)));
       }, token);
@@ -904,77 +910,69 @@ class RADOS final
 			    boost::asio::io_context& ioctx,
 		    std::unique_ptr<BuildComp> c);
 
-  void execute(const Object& o, const IOContext& ioc, ReadOp&& op,
+  void execute(Object o, IOContext ioc, ReadOp op,
 	       ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
 	       uint64_t* objver, const blkin_trace_info* trace_info);
 
-  void execute(const Object& o, const IOContext& ioc, WriteOp&& op,
+  void execute(Object o, IOContext ioc, WriteOp op,
 	       std::unique_ptr<Op::Completion> c, uint64_t* objver,
 	       const blkin_trace_info* trace_info);
 
-  void lookup_pool(std::string_view name, std::unique_ptr<LookupPoolComp> c);
+  void lookup_pool(std::string name, std::unique_ptr<LookupPoolComp> c);
   void list_pools(std::unique_ptr<LSPoolsComp> c);
-  void create_pool_snap(int64_t pool, std::string_view snapName,
+  void create_pool_snap(int64_t pool, std::string snap_name,
 			std::unique_ptr<SimpleOpComp> c);
   void allocate_selfmanaged_snap(int64_t pool, std::unique_ptr<SMSnapComp> c);
-  void delete_pool_snap(int64_t pool, std::string_view snapName,
+  void delete_pool_snap(int64_t pool, std::string snap_name,
 			std::unique_ptr<SimpleOpComp> c);
   void delete_selfmanaged_snap(int64_t pool, std::uint64_t snap,
 			       std::unique_ptr<SimpleOpComp> c);
-  void create_pool(std::string_view name, std::optional<int> crush_rule,
-		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(std::string_view name,
+  void create_pool(std::string name, std::optional<int> crush_rule,
 		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(int64_t pool,
+  void delete_pool(std::string name,
 		   std::unique_ptr<SimpleOpComp> c);
-  void stat_pools(const std::vector<std::string>& pools,
+  void delete_pool(int64_t pool, std::unique_ptr<SimpleOpComp> c);
+  void stat_pools(std::vector<std::string> pools,
 		  std::unique_ptr<PoolStatComp> c);
   void stat_fs(std::optional<std::int64_t> pool,
 	       std::unique_ptr<StatFSComp> c);
 
-  void watch(const Object& o, const IOContext& ioc,
+  void watch(Object o, IOContext ioc,
 	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, std::unique_ptr<WatchComp> c);
+	     WatchCB cb, std::unique_ptr<WatchComp> c);
   tl::expected<ceph::timespan, boost::system::error_code>
   watch_check(uint64_t cookie);
-  void notify_ack(const Object& o,
-		  const IOContext& _ioc,
+  void notify_ack(Object o, IOContext _ioc,
 		  uint64_t notify_id,
 		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
+		  ceph::buffer::list bl,
 		  std::unique_ptr<SimpleOpComp>);
-  void unwatch(uint64_t cookie, const IOContext& ioc,
+  void unwatch(uint64_t cookie, IOContext ioc,
 	       std::unique_ptr<SimpleOpComp>);
-  void notify(const Object& oid, const IOContext& ioctx,
-	      ceph::buffer::list&& bl,
+  void notify(Object oid, IOContext ioctx,
+	      ceph::buffer::list bl,
 	      std::optional<std::chrono::milliseconds> timeout,
 	      std::unique_ptr<NotifyComp> c);
   void flush_watch(std::unique_ptr<VoidOpComp>);
 
-  void enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 std::unique_ptr<SimpleOpComp> c);
-  void enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
+  void enumerate_objects(IOContext ioc, Cursor begin,
+			 Cursor end, std::uint32_t max,
+			 ceph::buffer::list filter,
 			 std::unique_ptr<EnumerateComp> c);
-  void osd_command(int osd, std::vector<std::string>&& cmd,
-		   ceph::buffer::list&& in, std::unique_ptr<CommandComp> c);
-  void pg_command(PG pg, std::vector<std::string>&& cmd,
-		  ceph::buffer::list&& in, std::unique_ptr<CommandComp> c);
+  void osd_command(int osd, std::vector<std::string> cmd,
+		   ceph::buffer::list in, std::unique_ptr<CommandComp> c);
+  void pg_command(PG pg, std::vector<std::string> cmd,
+		  ceph::buffer::list in, std::unique_ptr<CommandComp> c);
 
   void mon_command(std::vector<std::string> command,
-		   const ceph::buffer::list& bl,
+		   ceph::buffer::list bl,
 		   std::string* outs, ceph::buffer::list* outbl,
 		   std::unique_ptr<SimpleOpComp> c);
 
-  void enable_application(std::string_view pool, std::string_view app_name,
+  void enable_application(std::string pool, std::string app_name,
 			  bool force, std::unique_ptr<SimpleOpComp> c);
 
-  void blocklist_add(std::string_view client_address,
+  void blocklist_add(std::string client_address,
                      std::optional<std::chrono::seconds> expire,
                      std::unique_ptr<SimpleOpComp> c);
 
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index c238ff450dc5..15b74f8e78d1 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -796,7 +796,7 @@ boost::asio::io_context& RADOS::get_io_context() {
   return impl->ioctx;
 }
 
-void RADOS::execute(const Object& o, const IOContext& _ioc, ReadOp&& _op,
+void RADOS::execute(Object o, IOContext _ioc, ReadOp _op,
 		    cb::list* bl,
 		    std::unique_ptr<ReadOp::Completion> c, version_t* objver,
 		    const blkin_trace_info *trace_info) {
@@ -819,7 +819,7 @@ void RADOS::execute(const Object& o, const IOContext& _ioc, ReadOp&& _op,
   trace.event("submitted");
 }
 
-void RADOS::execute(const Object& o, const IOContext& _ioc, WriteOp&& _op,
+void RADOS::execute(Object o, IOContext _ioc, WriteOp _op,
 		    std::unique_ptr<WriteOp::Completion> c, version_t* objver,
 		    const blkin_trace_info *trace_info) {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
@@ -851,8 +851,7 @@ boost::uuids::uuid RADOS::get_fsid() const noexcept {
 }
 
 
-void RADOS::lookup_pool(std::string_view name,
-			std::unique_ptr<LookupPoolComp> c)
+void RADOS::lookup_pool(std::string name, std::unique_ptr<LookupPoolComp> c)
 {
   // I kind of want to make lookup_pg_pool return
   // std::optional<int64_t> since it can only return one error code.
@@ -861,9 +860,8 @@ void RADOS::lookup_pool(std::string_view name,
     name);
   if (ret < 0) {
     impl->objecter->wait_for_latest_osdmap(
-      [name = std::string(name), c = std::move(c),
-       objecter = impl->objecter]
-      (bs::error_code ec) mutable {
+      [name = std::move(name), c = std::move(c),
+       objecter = impl->objecter](bs::error_code ec) mutable {
 	int64_t ret =
 	  objecter->with_osdmap([&](const OSDMap &osdmap) {
 	    return osdmap.lookup_pg_pool_name(name);
@@ -874,9 +872,6 @@ void RADOS::lookup_pool(std::string_view name,
 	else
 	  ca::dispatch(std::move(c), bs::error_code{}, ret);
       });
-  } else if (ret < 0) {
-    ca::post(std::move(c), osdc_errc::pool_dne,
-		 std::int64_t(0));
   } else {
     ca::post(std::move(c), bs::error_code{}, ret);
   }
@@ -912,11 +907,11 @@ void RADOS::list_pools(std::unique_ptr<LSPoolsComp> c) {
 }
 
 void RADOS::create_pool_snap(std::int64_t pool,
-			     std::string_view snapName,
+			     std::string snap_name,
 			     std::unique_ptr<SimpleOpComp> c)
 {
   impl->objecter->create_pool_snap(
-    pool, snapName,
+    pool, snap_name,
     Objecter::PoolOp::OpComp::create(
       get_executor(),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
@@ -936,11 +931,11 @@ void RADOS::allocate_selfmanaged_snap(int64_t pool,
 }
 
 void RADOS::delete_pool_snap(std::int64_t pool,
-			     std::string_view snapName,
+			     std::string snap_name,
 			     std::unique_ptr<SimpleOpComp> c)
 {
   impl->objecter->delete_pool_snap(
-    pool, snapName,
+    pool, snap_name,
     Objecter::PoolOp::OpComp::create(
       get_executor(),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
@@ -961,7 +956,7 @@ void RADOS::delete_selfmanaged_snap(std::int64_t pool,
       }));
 }
 
-void RADOS::create_pool(std::string_view name,
+void RADOS::create_pool(std::string name,
 			std::optional<int> crush_rule,
 			std::unique_ptr<SimpleOpComp> c)
 {
@@ -975,8 +970,7 @@ void RADOS::create_pool(std::string_view name,
       crush_rule.value_or(-1));
 }
 
-void RADOS::delete_pool(std::string_view name,
-			std::unique_ptr<SimpleOpComp> c)
+void RADOS::delete_pool(std::string name, std::unique_ptr<SimpleOpComp> c)
 {
   impl->objecter->delete_pool(
     name,
@@ -999,7 +993,7 @@ void RADOS::delete_pool(std::int64_t pool,
       }));
 }
 
-void RADOS::stat_pools(const std::vector<std::string>& pools,
+void RADOS::stat_pools(std::vector<std::string> pools,
 		       std::unique_ptr<PoolStatComp> c) {
   impl->objecter->get_pool_stats(
     pools,
@@ -1058,8 +1052,8 @@ void RADOS::stat_fs(std::optional<std::int64_t> _pool,
 
 // --- Watch/Notify
 
-void RADOS::watch(const Object& o, const IOContext& _ioc,
-		  std::optional<std::chrono::seconds> timeout, WatchCB&& cb,
+void RADOS::watch(Object o, IOContext _ioc,
+		  std::optional<std::chrono::seconds> timeout, WatchCB cb,
 		  std::unique_ptr<WatchComp> c) {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
@@ -1081,11 +1075,10 @@ void RADOS::watch(const Object& o, const IOContext& _ioc,
       }), nullptr);
 }
 
-void RADOS::notify_ack(const Object& o,
-		       const IOContext& _ioc,
+void RADOS::notify_ack(Object o, IOContext _ioc,
 		       uint64_t notify_id,
 		       uint64_t cookie,
-		       bufferlist&& bl,
+		       bufferlist bl,
 		       std::unique_ptr<SimpleOpComp> c)
 {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
@@ -1104,7 +1097,7 @@ tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check(uint64_t cookie)
   return impl->objecter->linger_check(linger_op);
 }
 
-void RADOS::unwatch(uint64_t cookie, const IOContext& _ioc,
+void RADOS::unwatch(uint64_t cookie, IOContext _ioc,
 		    std::unique_ptr<SimpleOpComp> c)
 {
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
@@ -1185,7 +1178,7 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
   }
 };
 
-void RADOS::notify(const Object& o, const IOContext& _ioc, bufferlist&& bl,
+void RADOS::notify(Object o, IOContext _ioc, bufferlist bl,
 		   std::optional<std::chrono::milliseconds> timeout,
 		   std::unique_ptr<NotifyComp> c)
 {
@@ -1319,14 +1312,10 @@ Cursor::from_str(const std::string& s) {
   return e;
 }
 
-void RADOS::enumerate_objects(const IOContext& _ioc,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
+void RADOS::enumerate_objects(IOContext _ioc, Cursor begin, Cursor end,
+			      std::uint32_t max, ceph::buffer::list filter,
 			      std::unique_ptr<EnumerateComp> c) {
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
   impl->objecter->enumerate_objects<Entry>(
     ioc->oloc.pool,
     ioc->oloc.nspace,
@@ -1342,8 +1331,8 @@ void RADOS::enumerate_objects(const IOContext& _ioc,
     });
 }
 
-void RADOS::osd_command(int osd, std::vector<std::string>&& cmd,
-			ceph::bufferlist&& in, std::unique_ptr<CommandComp> c) {
+void RADOS::osd_command(int osd, std::vector<std::string> cmd,
+			ceph::buffer::list in, std::unique_ptr<CommandComp> c) {
   impl->objecter->osd_command(osd, std::move(cmd), std::move(in), nullptr,
 			      [c = std::move(c)]
 			      (bs::error_code ec,
@@ -1354,8 +1343,8 @@ void RADOS::osd_command(int osd, std::vector<std::string>&& cmd,
 					     std::move(b));
 			      });
 }
-void RADOS::pg_command(PG pg, std::vector<std::string>&& cmd,
-		       ceph::bufferlist&& in, std::unique_ptr<CommandComp> c) {
+void RADOS::pg_command(PG pg, std::vector<std::string> cmd,
+		       ceph::buffer::list in, std::unique_ptr<CommandComp> c) {
   impl->objecter->pg_command(pg_t{pg.seed, pg.pool}, std::move(cmd), std::move(in), nullptr,
 			     [c = std::move(c)]
 			     (bs::error_code ec,
@@ -1367,7 +1356,7 @@ void RADOS::pg_command(PG pg, std::vector<std::string>&& cmd,
 			     });
 }
 
-void RADOS::enable_application(std::string_view pool, std::string_view app_name,
+void RADOS::enable_application(std::string pool, std::string app_name,
 			       bool force, std::unique_ptr<SimpleOpComp> c) {
   // pre-Luminous clusters will return -EINVAL and application won't be
   // preserved until Luminous is configured as minimum version.
@@ -1387,7 +1376,7 @@ void RADOS::enable_application(std::string_view pool, std::string_view app_name,
   }
 }
 
-void RADOS::blocklist_add(std::string_view client_address,
+void RADOS::blocklist_add(std::string client_address,
                           std::optional<std::chrono::seconds> expire,
                           std::unique_ptr<SimpleOpComp> c) {
   auto expire_arg = (expire ?
@@ -1425,8 +1414,7 @@ void RADOS::wait_for_latest_osd_map(std::unique_ptr<SimpleOpComp> c) {
 }
 
 void RADOS::mon_command(std::vector<std::string> command,
-			const cb::list& bl,
-			std::string* outs, cb::list* outbl,
+			cb::list bl, std::string* outs, cb::list* outbl,
 			std::unique_ptr<SimpleOpComp> c) {
 
   impl->monclient.start_mon_command(
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index 6d3bd0de7527..4b6866669fb1 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -133,6 +133,16 @@ Object::Object(std::string&& s) {
   new (&impl) object_t(std::move(s));
 }
 
+Object::Object(const Object& rhs) {
+  static_assert(impl_size >= sizeof(object_t));
+  new (&impl) object_t(*reinterpret_cast<const object_t*>(&rhs.impl));
+}
+
+Object::Object(Object&& rhs) {
+  static_assert(impl_size >= sizeof(object_t));
+  new (&impl) object_t(*std::move(reinterpret_cast<object_t*>(&rhs.impl)));
+}
+
 Object::~Object() {
   reinterpret_cast<object_t*>(&impl)->~object_t();
 }
@@ -157,6 +167,11 @@ IOContext::IOContext(const IOContext& rhs) {
   new (&impl) IOContextImpl(*reinterpret_cast<const IOContextImpl*>(&rhs.impl));
 }
 
+IOContext::IOContext(IOContext&& rhs) {
+  static_assert(impl_size >= sizeof(IOContextImpl));
+  new (&impl) IOContextImpl(std::move(*reinterpret_cast<const IOContextImpl*>(&rhs.impl)));
+}
+
 IOContext::IOContext(int64_t pool, std::string ns, std::string key)
   : IOContext() {
   set_pool(pool);
@@ -273,6 +288,15 @@ Op::Op() {
   o->get();
 }
 
+Op::Op(Op&& rhs) {
+  static_assert(Op::impl_size >= sizeof(librados::TestObjectOperationImpl*));
+  auto& o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
+  auto& p = *reinterpret_cast<librados::TestObjectOperationImpl**>(&rhs.impl);
+  o = p;
+  p = new librados::TestObjectOperationImpl();
+  p->get();
+}
+
 Op::~Op() {
   auto& o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   if (o != nullptr) {
@@ -533,7 +557,7 @@ boost::asio::io_context::executor_type neorados::RADOS::get_executor() const {
   return impl->io_context.get_executor();
 }
 
-void RADOS::execute(const Object& o, const IOContext& ioc, ReadOp&& op,
+void RADOS::execute(Object o, IOContext ioc, ReadOp op,
                     ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
                     uint64_t* objver, const blkin_trace_info* trace_info) {
   auto io_ctx = impl->get_io_ctx(ioc);
@@ -552,7 +576,7 @@ void RADOS::execute(const Object& o, const IOContext& ioc, ReadOp&& op,
   ceph_assert(r == 0);
 }
 
-void RADOS::execute(const Object& o, const IOContext& ioc, WriteOp&& op,
+void RADOS::execute(Object o, IOContext ioc, WriteOp op,
                     std::unique_ptr<Op::Completion> c, uint64_t* objver,
                     const blkin_trace_info* trace_info) {
   auto io_ctx = impl->get_io_ctx(ioc);
@@ -576,7 +600,7 @@ void RADOS::execute(const Object& o, const IOContext& ioc, WriteOp&& op,
 }
 
 void RADOS::mon_command(std::vector<std::string> command,
-                        const bufferlist& bl,
+                        bufferlist bl,
                         std::string* outs, bufferlist* outbl,
                         std::unique_ptr<Op::Completion> c) {
   auto r = impl->test_rados_client->mon_command(command, bl, outbl, outs);
@@ -584,7 +608,7 @@ void RADOS::mon_command(std::vector<std::string> command,
           (r < 0 ? bs::error_code(-r, osd_category()) : bs::error_code()));
 }
 
-void RADOS::blocklist_add(std::string_view client_address,
+void RADOS::blocklist_add(std::string client_address,
                           std::optional<std::chrono::seconds> expire,
                           std::unique_ptr<SimpleOpComp> c) {
   auto r = impl->test_rados_client->blocklist_add(

From 79832dd08f1a396f6fc4c6169ea004f06e2dca67 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Thu, 15 Sep 2022 14:46:47 -0400
Subject: [PATCH 0937/2492] neorados: Fix logic error in exerciser

Use `min` rather than `max` when deciding how much we need to read in
`read()`.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/tools/neorados.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
index c59cf739fafb..be5dd1a2541d 100644
--- a/src/tools/neorados.cc
+++ b/src/tools/neorados.cc
@@ -204,7 +204,7 @@ void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 
   std::size_t off = 0;
   ceph::buffer::list bl;
-  while (auto toread = std::max(len - off, io_size)) {
+  while (auto toread = std::min(len - off, io_size)) {
     R::ReadOp op;
     op.read(off, toread, &bl);
     r.execute(obj, pool, std::move(op), nullptr, y[ec]);

From e3dc51050aa0986a7b72f38b40d9f0e90fe28599 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Thu, 15 Sep 2022 16:42:51 -0400
Subject: [PATCH 0938/2492] neorados: Use C++20 Coroutines in exerciser/demo

Just as a demonstration to see how well they work and how to put
things together with them.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/tools/CMakeLists.txt |   2 +-
 src/tools/neorados.cc    | 102 ++++++++++++++++++++++-----------------
 2 files changed, 59 insertions(+), 45 deletions(-)

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index 993fadb2e2ed..56bb0414bcd6 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -23,7 +23,7 @@ if(NOT WIN32)
   set(neorados_srcs
       neorados.cc)
   add_executable(neorados ${neorados_srcs})
-  target_link_libraries(neorados libneorados spawn ${FMT_LIB} ${CMAKE_DL_LIBS})
+  target_link_libraries(neorados libneorados ${FMT_LIB} ${CMAKE_DL_LIBS})
   #install(TARGETS neorados DESTINATION bin)
 endif()
 
diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
index be5dd1a2541d..731ffdd2d091 100644
--- a/src/tools/neorados.cc
+++ b/src/tools/neorados.cc
@@ -13,16 +13,20 @@
  *
  */
 
-#define BOOST_COROUTINES_NO_DEPRECATION_WARNING
-
 #include <algorithm>
 #include <cassert>
+#include <coroutine>
 #include <iostream>
 #include <string>
 #include <string_view>
 #include <tuple>
 #include <vector>
 
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/redirect_error.hpp>
+#include <boost/asio/use_awaitable.hpp>
+#include <boost/asio/co_spawn.hpp>
+
 #include <boost/io/ios_state.hpp>
 #include <boost/program_options.hpp>
 #include <boost/system/system_error.hpp>
@@ -30,8 +34,6 @@
 #include <fmt/format.h>
 #include <fmt/ostream.h>
 
-#include <spawn/spawn.hpp>
-
 #include "include/buffer.h" // :(
 
 #include "include/neorados/RADOS.hpp"
@@ -41,7 +43,6 @@ using namespace std::literals;
 namespace ba = boost::asio;
 namespace bs = boost::system;
 namespace R = neorados;
-namespace s = spawn;
 
 std::string verstr(const std::tuple<uint32_t, uint32_t, uint32_t>& v)
 {
@@ -67,91 +68,94 @@ void printseq(const V& v, std::ostream& m, F&& f)
 		});
 }
 
-R::IOContext lookup_pool(R::RADOS& r, const std::string& pname,
-			 s::yield_context y)
+ba::awaitable<R::IOContext> lookup_pool(R::RADOS& r, const std::string& pname)
 {
   bs::error_code ec;
-  auto p = r.lookup_pool(pname, y[ec]);
+  auto p = co_await r.lookup_pool(pname,
+				  ba::redirect_error(ba::use_awaitable, ec));
   if (ec)
     throw bs::system_error(
       ec, fmt::format("when looking up '{}'", pname));
-  return R::IOContext{p};
+  co_return R::IOContext(p);
 }
 
 
-void lspools(R::RADOS& r, const std::vector<std::string>&,
-	     s::yield_context y)
+ba::awaitable<void> lspools(R::RADOS& r, const std::vector<std::string>&)
 {
-  const auto l = r.list_pools(y);
+  const auto l = co_await r.list_pools(ba::use_awaitable);
   printseq(l, std::cout, [](const auto& p) -> const std::string& {
 			   return p.second;
 			 });
+  co_return;
 }
 
 
-void ls(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
+ba::awaitable<void> ls(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
-  const auto pool = lookup_pool(r, pname, y).set_ns(R::all_nspaces);
-
+  const auto pool = (co_await lookup_pool(r, pname)).set_ns(R::all_nspaces);
   std::vector<R::Entry> ls;
   R::Cursor next = R::Cursor::begin();
   bs::error_code ec;
   do {
-    std::tie(ls, next) = r.enumerate_objects(pool, next, R::Cursor::end(),
-					     1000, {}, y[ec]);
+    std::tie(ls, next) =
+      co_await r.enumerate_objects(pool, next, R::Cursor::end(), 1000, {},
+				   ba::redirect_error(ba::use_awaitable, ec));
     if (ec)
       throw bs::system_error(ec, fmt::format("when listing {}", pname));
     printseq(ls, std::cout);
     ls.clear();
   } while (next != R::Cursor::end());
+  co_return;
 }
 
-void mkpool(R::RADOS& r, const std::vector<std::string>& p,
-	    s::yield_context y)
+ba::awaitable<void> mkpool(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   bs::error_code ec;
-  r.create_pool(pname, std::nullopt, y[ec]);
+  co_await r.create_pool(pname, std::nullopt,
+			 ba::redirect_error(ba::use_awaitable, ec));
   if (ec)
     throw bs::system_error(ec, fmt::format("when creating pool '{}'", pname));
+  co_return;
 }
 
-void rmpool(R::RADOS& r, const std::vector<std::string>& p,
-	    s::yield_context y)
+ba::awaitable<void> rmpool(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   bs::error_code ec;
-  r.delete_pool(pname, y[ec]);
+  co_await r.delete_pool(pname, ba::redirect_error(ba::use_awaitable, ec));
   if (ec)
     throw bs::system_error(ec, fmt::format("when removing pool '{}'", pname));
+  co_return;
 }
 
-void create(R::RADOS& r, const std::vector<std::string>& p,
-	    s::yield_context y)
+ba::awaitable<void> create(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   const R::Object obj = p[1];
-  const auto pool = lookup_pool(r, pname, y);
+  const auto pool = co_await lookup_pool(r, pname);
 
   bs::error_code ec;
   R::WriteOp op;
   op.create(true);
-  r.execute(obj, pool, std::move(op), y[ec]);
+  co_await r.execute(obj, pool, std::move(op),
+		     ba::redirect_error(ba::use_awaitable, ec));
   if (ec)
     throw bs::system_error(ec,
 			   fmt::format(
 			     "when creating object '{}' in pool '{}'",
 			     obj, pname));
+  co_return;
 }
 
 inline constexpr std::size_t io_size = 4 << 20;
 
-void write(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
+ba::awaitable<void> write(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   const R::Object obj(p[1]);
-  const auto pool = lookup_pool(r, pname, y);
+  const auto pool = co_await lookup_pool(r, pname);
 
   bs::error_code ec;
   std::unique_ptr<char[]> buf = std::make_unique<char[]>(io_size);
@@ -173,28 +177,30 @@ void write(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
     bl.append(buffer::create_static(len, buf.get()));
     R::WriteOp op;
     op.write(curoff, std::move(bl));
-    r.execute(obj, pool, std::move(op), y[ec]);
+    co_await r.execute(obj, pool, std::move(op),
+		       ba::redirect_error(ba::use_awaitable, ec));
 
     if (ec)
       throw bs::system_error(ec, fmt::format(
 			       "when writing object '{}' in pool '{}'",
 			       obj, pname));
   }
+  co_return;
 }
 
-void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
+ba::awaitable<void> read(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   const R::Object obj(p[1]);
-  const auto pool = lookup_pool(r, pname, y);
+  const auto pool = co_await lookup_pool(r, pname);
 
   bs::error_code ec;
   std::uint64_t len;
   {
     R::ReadOp op;
     op.stat(&len, nullptr);
-    r.execute(obj, pool, std::move(op),
-	      nullptr, y[ec]);
+    co_await r.execute(obj, pool, std::move(op),
+		       nullptr, ba::redirect_error(ba::use_awaitable, ec));
     if (ec)
       throw bs::system_error(
 	ec,
@@ -207,7 +213,8 @@ void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
   while (auto toread = std::min(len - off, io_size)) {
     R::ReadOp op;
     op.read(off, toread, &bl);
-    r.execute(obj, pool, std::move(op), nullptr, y[ec]);
+    co_await r.execute(obj, pool, std::move(op), nullptr,
+		       ba::redirect_error(ba::use_awaitable, ec));
     if (ec)
       throw bs::system_error(
 	ec,
@@ -218,28 +225,31 @@ void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
     bl.write_stream(std::cout);
     bl.clear();
   }
+  co_return;
 }
 
-void rm(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
+ba::awaitable<void> rm(R::RADOS& r, const std::vector<std::string>& p)
 {
   const auto& pname = p[0];
   const R::Object obj = p[1];
-  const auto pool = lookup_pool(r, pname, y);
+  const auto pool = co_await lookup_pool(r, pname);
 
   bs::error_code ec;
   R::WriteOp op;
   op.remove();
-  r.execute(obj, pool, std::move(op), y[ec]);
+  co_await r.execute(obj, pool, std::move(op),
+		     ba::redirect_error(ba::use_awaitable, ec));
   if (ec)
     throw bs::system_error(ec, fmt::format(
 			     "when removing object '{}' in pool '{}'",
 			     obj, pname));
+  co_return;
 }
 
 static constexpr auto version = std::make_tuple(0ul, 0ul, 1ul);
 
-using cmdfunc = void (*)(R::RADOS& r, const std::vector<std::string>& p,
-			 s::yield_context);
+using cmdfunc =
+  ba::awaitable<void> (*)(R::RADOS& r, const std::vector<std::string>& p);
 
 struct cmdesc {
   std::string_view name;
@@ -370,10 +380,14 @@ int main(int argc, char* argv[])
 		   prog, command, ci->name, ci->usage);
 	return 1;
       }
-      s::spawn(c, [&](s::yield_context y) {
-		    auto r = R::RADOS::Builder{}.build(c, y);
-		    ci->f(r, parameters, y);
-		  });
+      ba::co_spawn(c,
+		   [&]() -> ba::awaitable<void> {
+		     auto r = co_await R::RADOS::Builder{}.build(
+		       c, ba::use_awaitable);
+		     co_await ci->f(r, parameters);
+		   }, [](std::exception_ptr e) {
+		     if (e) std::rethrow_exception(e);
+		   });
     } else {
       fmt::print(std::cerr, "{}: {}: unknown command\n", prog, command);
       return 1;

From bf8447afda47745c89e7ba1291a556e1734cab28 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Mon, 23 Jan 2023 22:50:58 -0500
Subject: [PATCH 0939/2492] common/async: Don't get allocator from moved-from
 handler

In the `destroy_` functions of `CompletionImpl` we were getting the
associated allocator after moving out of the handler into the call to
`bind_and_forward`. This was triggering a crash on null-pointer access
in operations made with `co_composed`.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/common/async/completion.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/common/async/completion.h b/src/common/async/completion.h
index 384c85f0fe38..a7dd7370f024 100644
--- a/src/common/async/completion.h
+++ b/src/common/async/completion.h
@@ -189,24 +189,24 @@ class CompletionImpl final : public Completion<void(Args...), T> {
 
   void destroy_defer(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
+    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
     w.second.get_executor().defer(std::move(f), alloc2);
   }
   void destroy_dispatch(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
+    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
     w.second.get_executor().dispatch(std::move(f), alloc2);
   }
   void destroy_post(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
+    auto f = bind_and_forward(std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
     w.second.get_executor().post(std::move(f), alloc2);

From 35231f7251ed70d8d817ee7e727d9763669d101f Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 20 Jan 2023 21:35:41 -0500
Subject: [PATCH 0940/2492] test/neorados: Harness and convenience for Neorados
 tests

Google Test does not support C++ coroutines, so kludge together a test
harness that supports coroutines reasonably well.

Also add a couple utility functions.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/test/neorados/CMakeLists.txt  |   2 +-
 src/test/neorados/common_tests.cc |   6 +-
 src/test/neorados/common_tests.h  | 360 ++++++++++++++++++++++++++++--
 3 files changed, 346 insertions(+), 22 deletions(-)

diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index ba96fb479d29..023f6580397f 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -21,7 +21,7 @@ target_link_libraries(ceph_test_neorados_op_speed
 
 add_library(neoradostest-support STATIC common_tests.cc)
 target_link_libraries(neoradostest-support
-  libneorados ${FMT_LIB})
+  libneorados ${FMT_LIB} GTest::GTest)
 
 add_executable(ceph_test_neorados_list_pool list_pool.cc)
 target_link_libraries(ceph_test_neorados_list_pool
diff --git a/src/test/neorados/common_tests.cc b/src/test/neorados/common_tests.cc
index 4e4b6c0af14c..6f274551d95e 100644
--- a/src/test/neorados/common_tests.cc
+++ b/src/test/neorados/common_tests.cc
@@ -12,7 +12,6 @@
  * Foundation.  See file COPYING.
  */
 
-#include <cstring>
 #include <string>
 #include <string_view>
 
@@ -23,12 +22,11 @@
 #include "common_tests.h"
 #include "include/neorados/RADOS.hpp"
 
-namespace ba = boost::asio;
-namespace R = neorados;
+namespace asio = boost::asio;
 
 std::string get_temp_pool_name(std::string_view prefix)
 {
-  static auto hostname = ba::ip::host_name();
+  static auto hostname = asio::ip::host_name();
   static auto num = 1ull;
   return fmt::format("{}{}-{}-{}", prefix, hostname, getpid(), num++);
 }
diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index ca3d7bf7f26d..4ac4741f2e4b 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -12,30 +12,356 @@
  * Foundation.  See file COPYING.
  */
 
+#pragma once
+
+#include <coroutine>
+#include <cstddef>
+#include <exception>
+#include <initializer_list>
+#include <optional>
 #include <string>
 #include <string_view>
+#include <utility>
+
+#include <boost/asio/async_result.hpp>
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/co_spawn.hpp>
+#include <boost/asio/deferred.hpp>
+#include <boost/asio/detached.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/asio/experimental/co_composed.hpp>
+
+#include <boost/system/error_code.hpp>
+#include <boost/system/system_error.hpp>
 
 #include "include/neorados/RADOS.hpp"
 
+#include "common/dout.h"
+
+#include "gtest/gtest.h"
+
+/// \file test/neorados/common_tests.h
+///
+/// \brief Tools for testing neorados code
+///
+/// This is a set of utilities for testing code using the neorados
+/// library, as well as for tests using C++20 Coroutines more generally.
+
+/// \brief Get a random, unique pool name
+///
+/// Return a uniquified pool name specific to the host on which we are running.
+///
+/// \param prefix A prefix for the returned pool name
+///
+/// \return A unique pool name
 std::string get_temp_pool_name(std::string_view prefix = {});
 
-template<typename CompletionToken>
-auto create_pool(neorados::RADOS& r, std::string_view pname,
+/// \brief Create a RADOS pool
+///
+/// Create a RADOS pool, returning its ID on success.
+///
+/// \param r RADOS handle
+/// \param pname Pool name
+/// \param token Boost.Asio completion token
+///
+/// \return The ID of the newly created pool
+template<boost::asio::completion_token_for<
+	   void(boost::system::error_code, int64_t)> CompletionToken>
+auto create_pool(neorados::RADOS& r,
+		 std::string pname,
 		 CompletionToken&& token)
 {
-  boost::asio::async_completion<CompletionToken,
-				void(boost::system::error_code,
-				     std::int64_t)> init(token);
-  r.create_pool(pname, std::nullopt,
-		[&r, pname = std::string(pname),
-		 h = std::move(init.completion_handler)]
-		(boost::system::error_code ec) mutable {
-		  r.lookup_pool(
-		    pname,
-		    [h = std::move(h)]
-		    (boost::system::error_code ec, std::int64_t pool) mutable {
-		      std::move(h)(ec, pool);
-		    });
-		});
-  return init.result.get();
+  namespace asio = boost::asio;
+  using boost::system::error_code;
+  using boost::system::system_error;
+
+  return asio::async_initiate<CompletionToken, void(error_code, int64_t)>
+    (asio::experimental::co_composed<void(error_code, int64_t)>
+     ([](auto state, neorados::RADOS& r, std::string pname) -> void {
+       try {
+	 co_await r.create_pool(pname, std::nullopt, asio::deferred);
+	 auto pool = co_await r.lookup_pool(pname, asio::deferred);
+	 co_return {error_code{}, pool};
+       } catch (const system_error& e) {
+	 co_return {e.code(), int64_t{}};
+       }
+     }, r.get_executor()),
+     token, std::ref(r), std::move(pname));
+}
+
+/// \brief Create a new, empty RADOS object
+///
+/// \param r RADOS handle
+/// \param oid Object name
+/// \param ioc Locator
+/// \param token Boost.Asio completion token
+template<boost::asio::completion_token_for<
+  void(boost::system::error_code)> CompletionToken>
+auto create_obj(neorados::RADOS& r, std::string_view oid,
+		const neorados::IOContext& ioc,
+		CompletionToken&& token)
+{
+  neorados::WriteOp op;
+  op.create(true);
+  return r.execute(oid, ioc, std::move(op),
+		   std::forward<CompletionToken>(token));
 }
+
+/// \brief Expect one of several errors from a coroutine
+///
+/// \param coro Awaitable coroutine
+/// \param ec Valid errors
+boost::asio::awaitable<void>
+expect_error_code(auto&& coro, auto ...ecs) {
+  bool failed = false;
+  try {
+    co_await std::move(coro);
+  } catch (const boost::system::system_error& e) {
+    failed = true;
+    auto h = [c = e.code()](auto t) -> bool { return t == c; };
+    EXPECT_TRUE((h(ecs) || ...))
+      << "Got unexpected error code " << e.code().message() << ".";
+  }
+  EXPECT_TRUE(failed) << "Operation did not error at all.";
+  co_return;
+}
+
+/// \brief Test harness for C++20 Coroutines
+///
+/// C++20 coroutines are better than what we had before, but don't
+/// play well with RAII. There's no good way to run a coroutine from a
+/// destructor, especially in a single-threaded, non-blocking
+/// program.
+///
+/// To be fair to C++20, this is difficult and even rust doesn't have
+/// async drop yet.
+///
+/// GTest has explicit SetUp and TearDown methods, however they're
+/// just regular functions. So we get Coroutine analogues of SetUp and
+/// TearDown that we then call from our custom TestBody. The user
+/// writes their tests in CoTestBody.
+class CoroTest : public testing::Test {
+private:
+  std::exception_ptr eptr;
+protected:
+  boost::asio::io_context asio_context; ///< The context on which the
+					///  coroutine runs.
+public:
+  /// Final override that does nothing. Actual setup code should go in CoSetUp
+  void SetUp() override final { };
+  /// Final override that does nothing. Actual teardown code should go
+  /// in CotearDown.
+  void TearDown() override final { };
+
+  /// \brief SetUp coroutine
+  ///
+  /// Called before the test body. Indicate failure by throwing
+  /// an exception. If an exception is thrown, neither the test body
+  /// nor teardown code are run.
+  virtual boost::asio::awaitable<void> CoSetUp() {
+    co_return;
+  }
+
+  /// \brief TearDown coroutine
+  ///
+  /// Called after the test body exits.
+  ///
+  /// \note This function is not run if CoSetup fails
+  virtual boost::asio::awaitable<void> CoTearDown() {
+    co_return;
+  }
+
+  /// \brief TestBody coroutine
+  ///
+  /// Run after setup.
+  virtual boost::asio::awaitable<void> CoTestBody() = 0;
+
+  /// \brief Run our coroutines
+  ///
+  /// This is marked final, since the actual test body belongs in
+  /// CoTestBody.
+  ///
+  /// Run CoSetUp and, if CoSetUp succeeded, CoTestBody and
+  /// CoTearDown.
+  ///
+  /// Error reporting of failures in CoSetUp and CoTearDown leaves
+  /// something to be desired as GTest thinks everything is the test
+  /// proper.
+  void TestBody() override final {
+    boost::asio::co_spawn(
+      asio_context,
+      [](CoroTest* t) -> boost::asio::awaitable<void> {
+	co_await t->CoSetUp();
+	try {
+	  co_await t->CoTestBody();
+	} catch (...) {
+	  t->eptr = std::current_exception();
+	}
+	co_await t->CoTearDown();
+	if (t->eptr) {
+	  std::rethrow_exception(t->eptr);
+	}
+	co_return;
+      }(this),
+      [](std::exception_ptr e) {
+	if (e) std::rethrow_exception(e);
+      });
+    asio_context.run();
+  }
+};
+
+/// \brief C++20 coroutine test harness for NeoRados
+///
+/// CoTestBody has access to `rados`, a `neorados::RADOS` handle, and
+/// `pool`, a `neorados::IOContext` representing a pool that will be
+/// destroyed when the test exits.
+class NeoRadosTest : public CoroTest {
+private:
+  const std::string prefix_{std::string{"test framework "} +
+			    testing::UnitTest::GetInstance()->
+			    current_test_info()->name() +
+			    std::string{": "}};
+
+  std::optional<neorados::RADOS> rados_;
+  const std::string pool_name_ = get_temp_pool_name(
+    testing::UnitTest::GetInstance()->current_test_info()->name());
+  neorados::IOContext pool_;
+  std::unique_ptr<DoutPrefix> dpp_;
+
+protected:
+
+  /// \brief Return reference to RADOS
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  neorados::RADOS& rados() noexcept { return *rados_; }
+
+  /// \brief Return name of created pool
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  const std::string& pool_name() const noexcept { return pool_name_; }
+
+  /// \brief Return reference to pool
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  const neorados::IOContext& pool() const noexcept { return pool_; }
+
+  /// \brief Return prefix for this test run
+  std::string_view prefix() const noexcept { return prefix_; }
+
+  /// \brief Return DoutPrefixProvider*
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  const DoutPrefixProvider* dpp() const noexcept { return dpp_.get(); }
+
+  auto execute(std::string_view oid, neorados::WriteOp&& op,
+	       std::uint64_t* ver = nullptr) {
+    return rados().execute(oid, pool(), std::move(op),
+			   boost::asio::use_awaitable, ver);
+  }
+
+  auto execute(std::string_view oid, neorados::ReadOp&& op,
+	       std::uint64_t* ver = nullptr) {
+    return rados().execute(oid, pool(), std::move(op), nullptr,
+			   boost::asio::use_awaitable, ver);
+  }
+
+  boost::asio::awaitable<ceph::buffer::list>
+  read(std::string_view oid, std::uint64_t off = 0, std::uint64_t len = 0) {
+    ceph::buffer::list bl;
+    neorados::ReadOp op;
+    op.read(off, len, &bl);
+    co_await rados().execute(oid, pool(), std::move(op),
+			     nullptr, boost::asio::use_awaitable);
+    co_return bl;
+  }
+
+public:
+
+  /// \brief Create RADOS handle and pool for the test
+  boost::asio::awaitable<void> CoSetUp() override {
+    rados_ = co_await neorados::RADOS::Builder{}
+      .build(asio_context, boost::asio::use_awaitable);
+    dpp_ = std::make_unique<DoutPrefix>(rados().cct(), 0, prefix().data());
+    pool_.set_pool(co_await create_pool(rados(), pool_name(),
+				    boost::asio::use_awaitable));
+    co_return;
+  }
+
+  ~NeoRadosTest() override = default;
+
+  /// \brief Delete pool used for testing
+  boost::asio::awaitable<void> CoTearDown() override {
+    co_await rados().delete_pool(pool().get_pool(),
+				 boost::asio::use_awaitable);
+    co_return;
+  }
+};
+
+/// \brief Helper macro for defining coroutine tests with a fixture
+///
+/// Defines a test using a coroutine fixture for
+/// SetUp/TearDown. Fixtures must be descendants of `CoroTest`.
+///
+/// \note Uses more of GTest's internals that I would like.
+///
+/// \warning Use `EXPECT_*` only, not `ASSERT_*`. `ASSERT_` macros
+/// return from the calling function and will not work in a
+/// coroutine.
+///
+/// \param test_suite_name Name of the test suite
+/// \param test_name Name of the test
+/// \param fixture Fixture class to use (descendent of CoroTest)
+#define CORO_TEST_F(test_suite_name, test_name, fixture)                       \
+  static_assert(sizeof(GTEST_STRINGIFY_(test_suite_name)) > 1,                 \
+		"test_suite_name must not be empty");                          \
+  static_assert(sizeof(GTEST_STRINGIFY_(test_name)) > 1,                       \
+		"test_name must not be empty");                                \
+  class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) : public fixture {  \
+  public:                                                                      \
+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() = default;            \
+    ~GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() override = default;  \
+    GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name,    \
+							   test_name));        \
+    GTEST_DISALLOW_MOVE_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name,    \
+							   test_name));        \
+									       \
+  private:                                                                     \
+    boost::asio::awaitable<void> CoTestBody() override;                        \
+    static ::testing::TestInfo *const test_info_ GTEST_ATTRIBUTE_UNUSED_;      \
+  };                                                                           \
+									       \
+  ::testing::TestInfo *const GTEST_TEST_CLASS_NAME_(test_suite_name,           \
+						    test_name)::test_info_ =   \
+      ::testing::internal::MakeAndRegisterTestInfo(                            \
+	  #test_suite_name, #test_name, nullptr, nullptr,                      \
+	  ::testing::internal::CodeLocation(__FILE__, __LINE__),               \
+	  (::testing::internal::GetTypeId<fixture>()),                         \
+	  ::testing::internal::SuiteApiResolver<fixture>::GetSetUpCaseOrSuite( \
+	      __FILE__, __LINE__),                                             \
+	  ::testing::internal::SuiteApiResolver<                               \
+	      fixture>::GetTearDownCaseOrSuite(__FILE__, __LINE__),            \
+	  new ::testing::internal::TestFactoryImpl<GTEST_TEST_CLASS_NAME_(     \
+	      test_suite_name, test_name)>);                                   \
+  boost::asio::awaitable<void> GTEST_TEST_CLASS_NAME_(test_suite_name,         \
+						      test_name)::CoTestBody()
+
+/// \brief Helper macro for defining coroutine tests
+///
+/// Tests created this way are direct descendants of `CoroTest`.
+///
+/// The Boost.Asio IO Context is `io_context`.
+///
+/// \warning Use `EXPECT_*` only, not `ASSERT_*`. `ASSERT_` macros
+/// return from the calling function and will not work in a
+/// coroutine.
+///
+/// \param test_suite_name Name of the test suite
+/// \param test_name Name of the test
+#define CORO_TEST(test_suite_name, test_name)                                  \
+  CORO_TEST_F(test_suite_name, test_name, CoroTest)

From af192bf800006fb484cca388aa909193c016f803 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Thu, 2 Feb 2023 01:14:35 -0500
Subject: [PATCH 0941/2492] osdc: Catch exceptions thrown in CLS client
 decoders

And return them to the client by setting the error code and result in
the vector and returning an error from the operation as a whole.

Pass OSD failure to subsequent handlers so that, for example, in the
event of a cancellation, we don't try to decode data that isn't there.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/osdc/Objecter.cc               | 51 +++++++++++++++++++++---
 src/osdc/error_code.cc             |  7 ++++
 src/osdc/error_code.h              |  3 +-
 src/test/neorados/CMakeLists.txt   | 17 +++++++-
 src/test/neorados/handler_error.cc | 62 ++++++++++++++++++++++++++++++
 5 files changed, 133 insertions(+), 7 deletions(-)
 create mode 100644 src/test/neorados/handler_error.cc

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index f006597e8273..6bc713fcbf5e 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -3537,11 +3537,23 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
   ceph_assert(op->out_bl.size() == op->out_rval.size());
   ceph_assert(op->out_bl.size() == op->out_handler.size());
   auto p = out_ops.begin();
+  // Propagates handler error to Op::completion. In the event of
+  // multiple handler errors, the most recent wins.
+  bs::error_code handler_error;
+  // Holds OSD error code, so handlers downstream of a failing op are
+  // made aware of it.
+  bs::error_code first_osd_error;
   for (unsigned i = 0;
        p != out_ops.end() && pb != op->out_bl.end();
        ++i, ++p, ++pb, ++pr, ++pe, ++ph) {
     ldout(cct, 10) << " op " << i << " rval " << p->rval
 		   << " len " << p->outdata.length() << dendl;
+    // Track when we get an OSD error and supply it to subsequent
+    // handlers so they won't attempt to operate on data that isn't
+    // there.
+    if (!first_osd_error && (p->rval < 0)) {
+      first_osd_error = bs::error_code(-p->rval, osd_category());
+    }
     if (*pb)
       **pb = p->outdata;
     // set rval before running handlers so that handlers
@@ -3552,10 +3564,35 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
       **pe = p->rval < 0 ? bs::error_code(-p->rval, osd_category()) :
 	bs::error_code();
     if (*ph) {
-      std::move((*ph))(p->rval < 0 ?
-		       bs::error_code(-p->rval, osd_category()) :
-		       bs::error_code(),
-		       p->rval, p->outdata);
+      try {
+	bs::error_code e;
+	if (first_osd_error) {
+	  e = first_osd_error;
+	} else if (p->rval < 0) {
+	  e = bs::error_code(-p->rval, osd_category());
+	}
+	std::move((*ph))(e, p->rval, p->outdata);
+      } catch (const bs::system_error& e) {
+	ldout(cct, 10) << "ERROR: tid " << op->tid << ": handler function threw "
+		       << e.what() << dendl;
+	handler_error = e.code();
+	if (*pe) {
+	  **pe = e.code();
+	}
+	if (*pr) {
+	  **pr = ceph::from_error_code(e.code());
+	}
+      } catch (const std::exception& e) {
+	ldout(cct, 0) << "ERROR: tid " << op->tid << ": handler function threw "
+		      << e.what() << dendl;
+	handler_error = osdc_errc::handler_failed;
+	if (*pe) {
+	  **pe = osdc_errc::handler_failed;
+	}
+	if (*pr) {
+	  **pr = -EIO;
+	}
+      }
     }
   }
 
@@ -3587,7 +3624,11 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
 
   // do callbacks
   if (Op::has_completion(onfinish)) {
-    Op::complete(std::move(onfinish), osdcode(rc), rc);
+    if (rc == 0 && handler_error) {
+      Op::complete(std::move(onfinish), handler_error, -EIO);
+    } else {
+      Op::complete(std::move(onfinish), osdcode(rc), rc);
+    }
   }
   if (completion_lock.mutex()) {
     completion_lock.unlock();
diff --git a/src/osdc/error_code.cc b/src/osdc/error_code.cc
index 7823e8b088c4..d60faa587bcc 100644
--- a/src/osdc/error_code.cc
+++ b/src/osdc/error_code.cc
@@ -73,6 +73,9 @@ const char* osdc_error_category::message(int ev, char*,
 
   case osdc_errc::pool_eio:
     return "Pool EIO flag set";
+
+  case osdc_errc::handler_failed:
+    return "Handler function threw unknown exception";
   }
 
   return "Unknown error";
@@ -101,6 +104,8 @@ osdc_error_category::default_error_condition(int ev) const noexcept {
     return bs::errc::timed_out;
   case osdc_errc::pool_eio:
     return bs::errc::io_error;
+  case osdc_errc::handler_failed:
+    return bs::errc::io_error;
   }
 
   return { ev, *this };
@@ -156,6 +161,8 @@ int osdc_error_category::from_code(int ev) const noexcept {
     return -ETIMEDOUT;
   case osdc_errc::pool_eio:
     return -EIO;
+  case osdc_errc::handler_failed:
+    return -EIO;
   }
   return -EDOM;
 }
diff --git a/src/osdc/error_code.h b/src/osdc/error_code.h
index eb78a5110b01..88d6f080a8a4 100644
--- a/src/osdc/error_code.h
+++ b/src/osdc/error_code.h
@@ -30,7 +30,8 @@ enum class osdc_errc {
   snapshot_exists,
   snapshot_dne,
   timed_out,
-  pool_eio
+  pool_eio,
+  handler_failed
 };
 
 namespace boost::system {
diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index 023f6580397f..c625133d50a5 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -1,4 +1,3 @@
-
 add_executable(ceph_test_neorados test_neorados.cc)
 target_link_libraries(ceph_test_neorados global libneorados
   ${unittest_libs}
@@ -26,3 +25,19 @@ target_link_libraries(neoradostest-support
 add_executable(ceph_test_neorados_list_pool list_pool.cc)
 target_link_libraries(ceph_test_neorados_list_pool
   libneorados neoradostest-support global ${FMT_LIB} ${unittest_libs})
+
+add_executable(ceph_test_neorados_handler_error
+  handler_error.cc
+  )
+target_link_libraries(ceph_test_neorados_handler_error
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_handler_error
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/test/neorados/handler_error.cc b/src/test/neorados/handler_error.cc
new file mode 100644
index 000000000000..26d468bc0246
--- /dev/null
+++ b/src/test/neorados/handler_error.cc
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+#include <boost/asio/use_awaitable.hpp>
+#include <boost/asio/redirect_error.hpp>
+
+#include <boost/system/errc.hpp>
+#include <boost/system/error_code.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "cls/version/cls_version_types.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace sys = boost::system;
+namespace buffer = ceph::buffer;
+
+CORO_TEST_F(neocls_handler_error, test_handler_error, NeoRadosTest)
+{
+  std::string_view oid = "obj";
+  co_await create_obj(rados(), oid, pool(), asio::use_awaitable);
+
+  {
+    neorados::ReadOp op;
+    op.exec("version", "read", {},
+	    [](sys::error_code ec, const buffer::list& bl) {
+	      throw buffer::end_of_buffer{};
+	    });
+    co_await expect_error_code(rados().execute(oid, pool(), std::move(op),
+					       nullptr, asio::use_awaitable),
+			       buffer::errc::end_of_buffer);
+  }
+
+  {
+    neorados::ReadOp op;
+    op.exec("version", "read", {},
+	    [](sys::error_code ec, const buffer::list& bl) {
+	      throw std::exception();
+	    });
+    co_await expect_error_code(rados().execute(oid, pool(), std::move(op),
+					       nullptr, asio::use_awaitable),
+			       sys::errc::io_error);
+  }
+  co_return;
+}

From 37ed6c41a9b92381fa13a8f6b43e8d1674039448 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 14:46:27 -0500
Subject: [PATCH 0942/2492] osdc: replace asio::io_context::strand with
 asio::strand<>

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/librados/RadosClient.h | 3 ++-
 src/mon/MonClient.h        | 5 +++--
 src/osdc/Objecter.h        | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index 052249a76db3..e7144b2e3f9d 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -96,7 +96,8 @@ class librados::RadosClient : public Dispatcher,
   int wait_for_osdmap();
 
 public:
-  boost::asio::io_context::strand finish_strand{poolctx.get_io_context()};
+  boost::asio::strand<boost::asio::io_context::executor_type>
+      finish_strand{poolctx.get_executor()};
 
   explicit RadosClient(CephContext *cct);
   ~RadosClient() override;
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 081893ae9c4a..803c74eb7f62 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -23,8 +23,8 @@
 #include <vector>
 
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/io_context_strand.hpp>
 #include <boost/asio/steady_timer.hpp>
+#include <boost/asio/strand.hpp>
 
 #include "msg/Messenger.h"
 
@@ -301,7 +301,8 @@ class MonClient : public Dispatcher,
   mutable ceph::mutex monc_lock = ceph::make_mutex("MonClient::monc_lock");
   SafeTimer timer;
   boost::asio::io_context& service;
-  boost::asio::io_context::strand finish_strand{service};
+  boost::asio::strand<boost::asio::io_context::executor_type>
+      finish_strand{service.get_executor()};
 
   bool initialized;
   bool stopping = false;
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index a0d9d0b32e76..9ca09b476476 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1643,7 +1643,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   boost::asio::io_context& service;
   // The guaranteed sequenced, one-at-a-time execution and apparently
   // people sometimes depend on this.
-  boost::asio::io_context::strand finish_strand{service};
+  boost::asio::strand<boost::asio::io_context::executor_type>
+      finish_strand{service.get_executor()};
   ZTracer::Endpoint trace_endpoint{"0.0.0.0", 0, "Objecter"};
 private:
   std::unique_ptr<OSDMap> osdmap{std::make_unique<OSDMap>()};

From 83e1ba13b00de15e8c225a1814523b60f456d450 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 16:41:53 -0500
Subject: [PATCH 0943/2492] common/async: completion uses free functions for
 defer/dispatch/post

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/async/completion.h | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/common/async/completion.h b/src/common/async/completion.h
index a7dd7370f024..d8065934e016 100644
--- a/src/common/async/completion.h
+++ b/src/common/async/completion.h
@@ -17,7 +17,11 @@
 
 #include <memory>
 
+#include <boost/asio/bind_executor.hpp>
+#include <boost/asio/defer.hpp>
+#include <boost/asio/dispatch.hpp>
 #include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/post.hpp>
 
 #include "bind_handler.h"
 #include "forward_handler.h"
@@ -183,33 +187,38 @@ class CompletionImpl final : public Completion<void(Args...), T> {
     RebindTraits2::deallocate(alloc2, static_cast<CompletionImpl*>(p), 1);
   }
 
-  static auto bind_and_forward(Handler&& h, std::tuple<Args...>&& args) {
-    return forward_handler(CompletionHandler{std::move(h), std::move(args)});
+  static auto bind_and_forward(const Executor2& ex, Handler&& h,
+                               std::tuple<Args...>&& args) {
+    return forward_handler(CompletionHandler{
+        boost::asio::bind_executor(ex, std::move(h)), std::move(args)});
   }
 
   void destroy_defer(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
+    auto ex2 = w.second.get_executor();
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
+    auto f = bind_and_forward(ex2, std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
-    w.second.get_executor().defer(std::move(f), alloc2);
+    boost::asio::defer(boost::asio::bind_executor(ex2, std::move(f)));
   }
   void destroy_dispatch(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
+    auto ex2 = w.second.get_executor();
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
+    auto f = bind_and_forward(ex2, std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
-    w.second.get_executor().dispatch(std::move(f), alloc2);
+    boost::asio::dispatch(std::move(f));
   }
   void destroy_post(std::tuple<Args...>&& args) override {
     auto w = std::move(work);
+    auto ex2 = w.second.get_executor();
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);
-    auto f = bind_and_forward(std::move(handler), std::move(args));
+    auto f = bind_and_forward(ex2, std::move(handler), std::move(args));
     RebindTraits2::destroy(alloc2, this);
     RebindTraits2::deallocate(alloc2, this, 1);
-    w.second.get_executor().post(std::move(f), alloc2);
+    boost::asio::post(std::move(f));
   }
   void destroy() override {
     RebindAlloc2 alloc2 = boost::asio::get_associated_allocator(handler);

From 7c30f36554d191d4e686c6bebcfe19ee09862694 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 18:58:21 -0500
Subject: [PATCH 0944/2492] common/async: SharedMutex uses free function post

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/async/detail/shared_mutex.h | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/common/async/detail/shared_mutex.h b/src/common/async/detail/shared_mutex.h
index 8e5436350cfd..5bfd0d58b1cc 100644
--- a/src/common/async/detail/shared_mutex.h
+++ b/src/common/async/detail/shared_mutex.h
@@ -135,10 +135,9 @@ auto SharedMutexImpl::async_lock(Mutex& mtx, CompletionToken&& token)
 
       // post a successful completion
       auto ex2 = boost::asio::get_associated_executor(handler, ex1);
-      auto alloc2 = boost::asio::get_associated_allocator(handler);
-      auto b = bind_handler(std::move(handler), ec,
-                            std::unique_lock{mtx, std::adopt_lock});
-      ex2.post(forward_handler(std::move(b)), alloc2);
+      auto h = boost::asio::bind_executor(ex2, std::move(handler));
+      boost::asio::post(bind_handler(std::move(h), ec,
+                                     std::unique_lock{mtx, std::adopt_lock}));
     } else {
       // create a request and add it to the exclusive list
       using LockCompletion = typename Request::LockCompletion;
@@ -227,10 +226,9 @@ auto SharedMutexImpl::async_lock_shared(Mutex& mtx, CompletionToken&& token)
       state++;
 
       auto ex2 = boost::asio::get_associated_executor(handler, ex1);
-      auto alloc2 = boost::asio::get_associated_allocator(handler);
-      auto b = bind_handler(std::move(handler), ec,
-                            std::shared_lock{mtx, std::adopt_lock});
-      ex2.post(forward_handler(std::move(b)), alloc2);
+      auto h = boost::asio::bind_executor(ex2, std::move(handler));
+      boost::asio::post(bind_handler(std::move(h), ec,
+                                     std::shared_lock{mtx, std::adopt_lock}));
     } else {
       using LockCompletion = typename Request::LockCompletion;
       auto request = LockCompletion::create(ex1, std::move(handler), mtx);

From 521211fe2fd53061e41eadc17561f9fb183e6f78 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 10 Feb 2023 10:36:22 -0500
Subject: [PATCH 0945/2492] neorados: use strand template and make_strand()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/neorados/RADOS.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 15b74f8e78d1..fabf2d5b3570 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -1127,7 +1127,7 @@ void RADOS::flush_watch(std::unique_ptr<VoidOpComp> c)
 
 struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
   boost::asio::io_context& ioc;
-  boost::asio::io_context::strand strand;
+  boost::asio::strand<boost::asio::io_context::executor_type> strand;
   Objecter* objecter;
   Objecter::LingerOp* op;
   std::unique_ptr<RADOS::NotifyComp> c;
@@ -1141,7 +1141,8 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
 		Objecter* objecter,
 		Objecter::LingerOp* op,
 		std::unique_ptr<RADOS::NotifyComp> c)
-    : ioc(ioc), strand(ioc), objecter(objecter), op(op), c(std::move(c)) {}
+    : ioc(ioc), strand(boost::asio::make_strand(ioc)),
+      objecter(objecter), op(op), c(std::move(c)) {}
 
   // Use bind or a lambda to pass this in.
   void handle_ack(bs::error_code ec,

From 51e9c2e59dc0637e6e3a206aa157f8faa4d4b468 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 10 Feb 2023 10:36:57 -0500
Subject: [PATCH 0946/2492] librbd: use strand template and make_strand()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/librbd/AsioEngine.cc           |  4 ++--
 src/librbd/AsioEngine.h            |  6 +++---
 src/librbd/asio/ContextWQ.cc       |  3 ++-
 src/librbd/asio/ContextWQ.h        |  5 +++--
 src/librbd/migration/QCOWFormat.cc | 12 +++++++-----
 src/librbd/migration/QCOWFormat.h  |  5 +++--
 6 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/src/librbd/AsioEngine.cc b/src/librbd/AsioEngine.cc
index 8e2beb49cfd9..ad0051efa8e5 100644
--- a/src/librbd/AsioEngine.cc
+++ b/src/librbd/AsioEngine.cc
@@ -20,8 +20,8 @@ AsioEngine::AsioEngine(std::shared_ptr<librados::Rados> rados)
       neorados::RADOS::make_with_librados(*rados))),
     m_cct(m_rados_api->cct()),
     m_io_context(m_rados_api->get_io_context()),
-    m_api_strand(std::make_unique<boost::asio::io_context::strand>(
-      m_io_context)),
+    m_api_strand(std::make_unique<boost::asio::strand<executor_type>>(
+      boost::asio::make_strand(m_io_context))),
     m_context_wq(std::make_unique<asio::ContextWQ>(m_cct, m_io_context)) {
   ldout(m_cct, 20) << dendl;
 
diff --git a/src/librbd/AsioEngine.h b/src/librbd/AsioEngine.h
index 0f476d80b2a4..6f2f22413c17 100644
--- a/src/librbd/AsioEngine.h
+++ b/src/librbd/AsioEngine.h
@@ -9,7 +9,7 @@
 #include <memory>
 #include <boost/asio/dispatch.hpp>
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/asio/post.hpp>
 
 struct Context;
@@ -45,7 +45,7 @@ class AsioEngine {
     return m_io_context.get_executor();
   }
 
-  inline boost::asio::io_context::strand& get_api_strand() {
+  inline boost::asio::strand<executor_type>& get_api_strand() {
     // API client callbacks should never fire concurrently
     return *m_api_strand;
   }
@@ -71,7 +71,7 @@ class AsioEngine {
   CephContext* m_cct;
 
   boost::asio::io_context& m_io_context;
-  std::unique_ptr<boost::asio::io_context::strand> m_api_strand;
+  std::unique_ptr<boost::asio::strand<executor_type>> m_api_strand;
   std::unique_ptr<asio::ContextWQ> m_context_wq;
 };
 
diff --git a/src/librbd/asio/ContextWQ.cc b/src/librbd/asio/ContextWQ.cc
index 4f6c72770806..80c650935d4c 100644
--- a/src/librbd/asio/ContextWQ.cc
+++ b/src/librbd/asio/ContextWQ.cc
@@ -16,7 +16,8 @@ namespace asio {
 
 ContextWQ::ContextWQ(CephContext* cct, boost::asio::io_context& io_context)
   : m_cct(cct), m_io_context(io_context),
-    m_strand(std::make_unique<boost::asio::io_context::strand>(io_context)),
+    m_strand(std::make_unique<boost::asio::strand<executor_type>>(
+      boost::asio::make_strand(io_context))),
     m_queued_ops(0) {
   ldout(m_cct, 20) << dendl;
 }
diff --git a/src/librbd/asio/ContextWQ.h b/src/librbd/asio/ContextWQ.h
index 85c254161213..3db5008d40a9 100644
--- a/src/librbd/asio/ContextWQ.h
+++ b/src/librbd/asio/ContextWQ.h
@@ -9,7 +9,7 @@
 #include <atomic>
 #include <memory>
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/asio/post.hpp>
 
 namespace librbd {
@@ -38,7 +38,8 @@ class ContextWQ {
 private:
   CephContext* m_cct;
   boost::asio::io_context& m_io_context;
-  std::unique_ptr<boost::asio::io_context::strand> m_strand;
+  using executor_type = boost::asio::io_context::executor_type;
+  std::unique_ptr<boost::asio::strand<executor_type>> m_strand;
 
   std::atomic<uint64_t> m_queued_ops;
 
diff --git a/src/librbd/migration/QCOWFormat.cc b/src/librbd/migration/QCOWFormat.cc
index 300bb712335e..b2e277ad3762 100644
--- a/src/librbd/migration/QCOWFormat.cc
+++ b/src/librbd/migration/QCOWFormat.cc
@@ -125,7 +125,8 @@ class QCOWFormat<I>::ClusterCache {
 public:
   ClusterCache(QCOWFormat* qcow_format)
     : qcow_format(qcow_format),
-      m_strand(*qcow_format->m_image_ctx->asio_engine) {
+      m_strand(boost::asio::make_strand(
+        *qcow_format->m_image_ctx->asio_engine)) {
   }
 
   void get_cluster(uint64_t cluster_offset, uint64_t cluster_length,
@@ -149,7 +150,7 @@ class QCOWFormat<I>::ClusterCache {
   typedef std::list<Completion> Completions;
 
   QCOWFormat* qcow_format;
-  boost::asio::io_context::strand m_strand;
+  boost::asio::strand<boost::asio::io_context::executor_type> m_strand;
 
   std::shared_ptr<Cluster> cluster;
   std::unordered_map<uint64_t, Completions> cluster_completions;
@@ -256,7 +257,8 @@ class QCOWFormat<I>::L2TableCache {
 public:
   L2TableCache(QCOWFormat* qcow_format)
     : qcow_format(qcow_format),
-      m_strand(*qcow_format->m_image_ctx->asio_engine),
+      m_strand(boost::asio::make_strand(
+        *qcow_format->m_image_ctx->asio_engine)),
       l2_cache_entries(QCOW_L2_CACHE_SIZE) {
   }
 
@@ -316,7 +318,7 @@ class QCOWFormat<I>::L2TableCache {
 private:
   QCOWFormat* qcow_format;
 
-  boost::asio::io_context::strand m_strand;
+  boost::asio::strand<boost::asio::io_context::executor_type> m_strand;
 
   struct Request {
     const LookupTable* l1_table;
@@ -832,7 +834,7 @@ QCOWFormat<I>::QCOWFormat(
     const SourceSpecBuilder<I>* source_spec_builder)
   : m_image_ctx(image_ctx), m_json_object(json_object),
     m_source_spec_builder(source_spec_builder),
-    m_strand(*image_ctx->asio_engine) {
+    m_strand(boost::asio::make_strand(*image_ctx->asio_engine)) {
 }
 
 template <typename I>
diff --git a/src/librbd/migration/QCOWFormat.h b/src/librbd/migration/QCOWFormat.h
index b36506716502..3b355628cd2c 100644
--- a/src/librbd/migration/QCOWFormat.h
+++ b/src/librbd/migration/QCOWFormat.h
@@ -10,7 +10,8 @@
 #include "librbd/migration/QCOW.h"
 #include "acconfig.h"
 #include "json_spirit/json_spirit.h"
-#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/iostreams/filter/zlib.hpp>
 #include <deque>
 #include <vector>
@@ -142,7 +143,7 @@ class QCOWFormat : public FormatInterface {
   json_spirit::mObject m_json_object;
   const SourceSpecBuilder<ImageCtxT>* m_source_spec_builder;
 
-  boost::asio::io_context::strand m_strand;
+  boost::asio::strand<boost::asio::io_context::executor_type> m_strand;
   std::shared_ptr<StreamInterface> m_stream;
 
   bufferlist m_bl;

From d875d242b1eb6b95bddac5804833eeaa752c45fe Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 10 Feb 2023 14:54:51 -0500
Subject: [PATCH 0947/2492] test/common: test_blocked_completion uses
 boost::asio::post()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/common/test_blocked_completion.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/test/common/test_blocked_completion.cc b/src/test/common/test_blocked_completion.cc
index 9a694de79e28..14c91e4fbe0d 100644
--- a/src/test/common/test_blocked_completion.cc
+++ b/src/test/common/test_blocked_completion.cc
@@ -72,11 +72,9 @@ auto id(const Executor& executor, CompletionToken&& token,
 	Args&& ...args)
 {
   ba::async_completion<CompletionToken, void(Args...)> init(token);
-  auto a = ba::get_associated_allocator(init.completion_handler);
-  executor.post(ca::forward_handler(
+  boost::asio::post(ca::forward_handler(
 		  ca::bind_handler(std::move(init.completion_handler),
-				   std::forward<Args>(args)...)),
-		a);
+				   std::forward<Args>(args)...)));
   return init.result.get();
 }
 

From bd0f18b082b4fc0eaa95ace78ac5e73520bb39dd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 14 Feb 2023 10:28:51 -0500
Subject: [PATCH 0948/2492] submodule: update spawn submodule for
 any_io_executor

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/spawn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/spawn b/src/spawn
index 9ee6d12f35ab..03d9e1b949b3 160000
--- a/src/spawn
+++ b/src/spawn
@@ -1 +1 @@
-Subproject commit 9ee6d12f35ab2fa48b469f13b4830a5e5cfde45e
+Subproject commit 03d9e1b949b3f7f3c8036ba0e8b35a49c305a03e

From 71bb05d8979d3ac2c5f52001cb449f8814b2ff5a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 18:43:10 -0500
Subject: [PATCH 0949/2492] cmake: drop BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT

this define was added for backward compatibility before boost 1.74

https://www.boost.org/doc/libs/1_74_0/doc/html/boost_asio/std_executors.html

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/CMakeLists.txt | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 5b1300bb84a3..2f34a6ceb289 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -78,10 +78,8 @@ add_definitions(
   -D_THREAD_SAFE
   -D__STDC_FORMAT_MACROS
   -D_FILE_OFFSET_BITS=64
-  -DBOOST_ASIO_DISABLE_THREAD_KEYWORD_EXTENSION)
-if(Boost_VERSION VERSION_GREATER_EQUAL 1.74)
-  add_definitions(-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT)
-endif()
+  -DBOOST_ASIO_DISABLE_THREAD_KEYWORD_EXTENSION
+  -DBOOST_ASIO_NO_TS_EXECUTORS)
 
 if(LINUX)
   add_definitions("-D_GNU_SOURCE")

From ea67f3dee2a3f8fcdcbb0bc0e80e38ec70378f05 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Wed, 29 Mar 2023 01:35:22 -0400
Subject: [PATCH 0950/2492] neorados: Use `asio::any_completion_handler`

As we'd like to reduce (and eliminate) internal Ceph dependencies to
the extent possible, now that Boost.Asio has a type-erased handler
type, let's use it.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                | 572 ++++++++++--------
 src/librados/IoCtxImpl.cc                     |  14 +-
 src/mds/Server.cc                             |   7 +-
 src/neorados/RADOS.cc                         | 352 ++++++-----
 src/osdc/Objecter.cc                          | 170 +++---
 src/osdc/Objecter.h                           | 440 +++++++++-----
 .../librados_test_stub/NeoradosTestStub.cc    |  66 +-
 7 files changed, 954 insertions(+), 667 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 1b0443f30ccb..9bbe10b6670f 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -27,7 +27,11 @@
 #include <fmt/format.h>
 #include <fmt/ostream.h>
 
+#include <boost/asio/any_completion_handler.hpp>
+#include <boost/asio/associated_executor.hpp>
 #include <boost/asio/async_result.hpp>
+#include <boost/asio/consign.hpp>
+#include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/io_context.hpp>
 
 #include <boost/container/flat_map.hpp>
@@ -48,11 +52,6 @@
 
 #include "include/neorados/RADOS_Decodable.hpp"
 
-// Needed for type erasure and template support. We can't really avoid
-// it.
-
-#include "common/async/completion.h"
-
 // These are needed for RGW, but in general as a 'shiny new interface'
 // we should try to use forward declarations and provide standard alternatives.
 
@@ -279,7 +278,7 @@ class Op {
 
   std::size_t size() const;
   using Signature = void(boost::system::error_code);
-  using Completion = ceph::async::Completion<Signature>;
+  using Completion = boost::asio::any_completion_handler<Signature>;
 
   friend std::ostream& operator <<(std::ostream& m, const Op& o);
 protected:
@@ -489,7 +488,7 @@ class RADOS final
   }
 
   using BuildSig = void(boost::system::error_code, RADOS);
-  using BuildComp = ceph::async::Completion<BuildSig>;
+  using BuildComp = boost::asio::any_completion_handler<BuildSig>;
   class Builder {
     std::optional<std::string> conf_files;
     std::optional<std::string> cluster;
@@ -522,31 +521,34 @@ class RADOS final
       return *this;
     }
 
-    template<typename CompletionToken>
+    template<boost::asio::completion_token_for<BuildSig> CompletionToken>
     auto build(boost::asio::io_context& ioctx, CompletionToken&& token) {
-      return boost::asio::async_initiate<CompletionToken, BuildSig>(
-	[&ioctx, this](auto&& handler) {
-	  build(ioctx, BuildComp::create(ioctx.get_executor(),
-					 std::move(handler)));
-	}, token);
+      auto consigned = boost::asio::consign(
+	std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	  boost::asio::get_associated_executor(token, ioctx.get_executor())));
+      return boost::asio::async_initiate<decltype(consigned), BuildSig>(
+	[&ioctx, this](auto handler) {
+	  build_(ioctx, std::move(handler));
+	}, consigned);
     }
 
   private:
-    void build(boost::asio::io_context& ioctx,
-	       std::unique_ptr<BuildComp> c);
+    void build_(boost::asio::io_context& ioctx,
+		BuildComp c);
   };
 
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<BuildSig> CompletionToken>
   static auto make_with_cct(CephContext* cct,
 			    boost::asio::io_context& ioctx,
 			    CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, BuildSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, ioctx.get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), BuildSig>(
       [cct, &ioctx](auto&& handler) {
-	make_with_cct(cct, ioctx,
-		      BuildComp::create(ioctx.get_executor(),
-					std::move(handler)));
-      }, token);
+	make_with_cct_(cct, ioctx, std::move(handler));
+      }, consigned);
   }
 
   static RADOS make_with_librados(librados::Rados& rados);
@@ -565,164 +567,183 @@ class RADOS final
   executor_type get_executor() const;
   boost::asio::io_context& get_io_context();
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<Op::Signature> CompletionToken>
   auto execute(Object o, IOContext ioc, ReadOp op,
 	       ceph::buffer::list* bl,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
-    return boost::asio::async_initiate<CompletionToken, Op::Signature>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), Op::Signature>(
       [o = std::move(o), ioc = std::move(ioc), op = std::move(op),
        bl, objver, trace_info, this](auto&& handler) mutable {
-	execute(std::move(o), std::move(ioc), std::move(op), bl,
-		ReadOp::Completion::create(get_executor(),
-					   std::move(handler)),
-		objver, trace_info);
-      }, token);
+	execute_(std::move(o), std::move(ioc), std::move(op), bl,
+		 std::move(handler), objver, trace_info);
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<Op::Signature> CompletionToken>
   auto execute(Object o, IOContext ioc, WriteOp op,
 	       CompletionToken&& token, uint64_t* objver = nullptr,
 	       const blkin_trace_info* trace_info = nullptr) {
-    return boost::asio::async_initiate<CompletionToken, Op::Signature>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), Op::Signature>(
       [o = std::move(o), ioc = std::move(ioc), op = std::move(op),
        objver, trace_info, this](auto&& handler) mutable {
-	execute(std::move(o), std::move(ioc), std::move(op),
-		WriteOp::Completion::create(get_executor(),
-					    std::move(handler)),
-		objver, trace_info);
-      }, token);
+	execute_(std::move(o), std::move(ioc), std::move(op),
+		 std::move(handler), objver, trace_info);
+      }, consigned);
   }
 
   boost::uuids::uuid get_fsid() const noexcept;
 
   using LookupPoolSig = void(boost::system::error_code,
 			     std::int64_t);
-  using LookupPoolComp = ceph::async::Completion<LookupPoolSig>;
-  template<typename CompletionToken>
+  using LookupPoolComp = boost::asio::any_completion_handler<LookupPoolSig>;
+  template<boost::asio::completion_token_for<LookupPoolSig> CompletionToken>
   auto lookup_pool(std::string name, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, LookupPoolSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), LookupPoolSig>(
       [name = std::move(name), this](auto&& handler) mutable {
-	lookup_pool(std::move(name),
-		    LookupPoolComp::create(get_executor(), std::move(handler)));
-      }, token);
+	lookup_pool_(std::move(name), std::move(handler));
+      }, consigned);
   }
 
   std::optional<uint64_t> get_pool_alignment(int64_t pool_id);
 
   using LSPoolsSig = void(std::vector<std::pair<std::int64_t, std::string>>);
-  using LSPoolsComp = ceph::async::Completion<LSPoolsSig>;
-  template<typename CompletionToken>
+  using LSPoolsComp = boost::asio::any_completion_handler<LSPoolsSig>;
+  template<boost::asio::completion_token_for<LSPoolsSig> CompletionToken>
   auto list_pools(CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, LSPoolsSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), LSPoolsSig>(
       [this](auto&& handler) {
-	list_pools(LSPoolsComp::create(get_executor(),
-				       std::move(handler)));
-      }, token);
+	list_pools_(std::move(handler));
+      }, consigned);
   }
 
   using SimpleOpSig = void(boost::system::error_code);
-  using SimpleOpComp = ceph::async::Completion<SimpleOpSig>;
-  template<typename CompletionToken>
+  using SimpleOpComp = boost::asio::any_completion_handler<SimpleOpSig>;
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto create_pool_snap(int64_t pool, std::string snap_name,
 			CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [snap_name = std::move(snap_name), pool, this](auto&& handler) mutable {
-	create_pool_snap(pool, std::move(snap_name),
-			 SimpleOpComp::create(get_executor(),
-					      std::move(handler)));
-      }, token);
+	create_pool_snap_(pool, std::move(snap_name),
+			  std::move(handler));
+      }, consigned);
   }
 
   using SMSnapSig = void(boost::system::error_code, std::uint64_t);
-  using SMSnapComp = ceph::async::Completion<SMSnapSig>;
-  template<typename CompletionToken>
+  using SMSnapComp = boost::asio::any_completion_handler<SMSnapSig>;
+  template<boost::asio::completion_token_for<SMSnapSig> CompletionToken>
   auto allocate_selfmanaged_snap(int64_t pool, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SMSnapSig>(
-      [pool, this](auto&& handler) {
-	allocage_selfmanaged_snap(pool,
-				  SMSnapComp::create(get_executor(),
-						     std::move(handler)));
-      }, token);
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SMSnapSig>(
+      [pool, this](auto&& handler) mutable {
+	allocage_selfmanaged_snap_(pool, std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto delete_pool_snap(int64_t pool, std::string snap_name,
 			CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [snap_name = std::move(snap_name), pool, this](auto&& handler) mutable {
-	delete_pool_snap(pool, std::move(snap_name),
-			 SimpleOpComp::create(get_executor(),
-					      std::move(handler)));
-      }, token);
+	delete_pool_snap_(pool, std::move(snap_name),
+			  std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto delete_selfmanaged_snap(int64_t pool, std::uint64_t snap,
 			       CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [snap, pool, this](auto&& handler) {
-	delete_selfmanaged_snap(pool, std::move(snap),
-				SimpleOpComp::create(get_executor(),
-						     std::move(handler)));
-      }, token);
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
+      [pool, snap, this](auto&& handler) mutable {
+	delete_selfmanaged_snap_(pool, snap, std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto create_pool(std::string name, std::optional<int> crush_rule,
 		   CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [name = std::move(name), crush_rule, this](auto&& handler) mutable {
-	create_pool(std::move(name), crush_rule,
-		    SimpleOpComp::create(get_executor(),
-					 std::move(handler)));
-      }, token);
+	create_pool_(std::move(name), crush_rule,
+		     std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto delete_pool(std::string name, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [name = std::move(name), this](auto&& handler) mutable {
-	delete_pool(std::move(name),
-		    SimpleOpComp::create(get_executor(),
-					 std::move(handler)));
-      }, token);
+	delete_pool_(std::move(name), std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto delete_pool(int64_t pool, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [pool, this](auto&& handler) mutable {
-	delete_pool(pool,
-		    SimpleOpComp::create(get_executor(),
-					 std::move(handler)));
-      }, token);
+	delete_pool_(pool, std::move(handler));
+      }, consigned);
   }
 
   using PoolStatSig = void(boost::system::error_code,
 			   boost::container::flat_map<std::string,
-			                              PoolStats>, bool);
-  using PoolStatComp = ceph::async::Completion<PoolStatSig>;
-  template<typename CompletionToken>
+						      PoolStats>, bool);
+  using PoolStatComp = boost::asio::any_completion_handler<PoolStatSig>;
+  template<boost::asio::completion_token_for<PoolStatSig> CompletionToken>
   auto stat_pools(std::vector<std::string> pools, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, PoolStatSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), PoolStatSig>(
       [pools = std::move(pools), this](auto&& handler) mutable {
-	stat_pools(std::move(pools),
-		   PoolStatComp::create(get_executor(),
-					std::move(handler)));
-      }, token);
+	stat_pools_(std::move(pools), std::move(handler));
+      }, consigned);
   }
 
   using StatFSSig = void(boost::system::error_code,
 			 FSStats);
-  using StatFSComp = ceph::async::Completion<StatFSSig>;
-  template<typename CompletionToken>
+  using StatFSComp = boost::asio::any_completion_handler<StatFSSig>;
+  template<boost::asio::completion_token_for<StatFSSig> CompletionToken>
   auto statfs(std::optional<int64_t> pool, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, StatFSSig>(
-      [pool, this](auto&& handler) {
-	statfs(pool, StatFSComp::create(get_executor(),
-					std::move(handler)));
-      }, token);
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), StatFSSig>(
+      [pool, this](auto&& handler) mutable {
+	statfs_(pool, std::move(handler));
+      }, consigned);
   }
 
   using WatchCB = fu2::unique_function<void(boost::system::error_code,
@@ -733,43 +754,48 @@ class RADOS final
 
   using WatchSig = void(boost::system::error_code ec,
 			uint64_t cookie);
-  using WatchComp = ceph::async::Completion<WatchSig>;
-  template<typename CompletionToken>
+  using WatchComp = boost::asio::any_completion_handler<WatchSig>;
+  template<boost::asio::completion_token_for<WatchSig> CompletionToken>
   auto watch(Object o, IOContext ioc,
 	     std::optional<std::chrono::seconds> timeout,
 	     WatchCB cb, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, WatchSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), WatchSig>(
       [o = std::move(o), ioc = std::move(ioc), timeout,
        cb = std::move(cb), this](auto&& handler) mutable {
-	watch(std::move(o), std::move(ioc), timeout, std::move(cb),
-	      WatchComp::create(get_executor(), std::move(handler)));
-      }, token);
+	watch_(std::move(o), std::move(ioc), timeout, std::move(cb),
+	       std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto notify_ack(Object o, IOContext ioc,
-		  uint64_t notify_id,
-		  uint64_t cookie,
+		  uint64_t notify_id, uint64_t cookie,
 		  ceph::buffer::list bl,
 		  CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [o = std::move(o), ioc = std::move(ioc), notify_id,
        cookie, bl = std::move(bl), this](auto&& handler) mutable {
-	notify_ack(std::move(o), std::move(ioc), notify_id, cookie,
-		   std::move(bl),
-		   SimpleOpComp::create(get_executor(),
-					std::move(handler)));
-      }, token);
+	notify_ack_(std::move(o), std::move(ioc), std::move(notify_id),
+		    std::move(cookie), std::move(bl), std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
-  auto unwatch(uint64_t cookie, IOContext ioc,
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
+  auto unwatch(std::uint64_t cookie, IOContext ioc,
 	       CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [cookie, ioc = std::move(ioc), this](auto&& handler) mutable {
-	unwatch(cookie, std::move(ioc),
-		SimpleOpComp::create(get_executor(), std::move(handler)));
-      }, token);
+	unwatch_(cookie, std::move(ioc), std::move(handler));
+      }, consigned);
   }
 
   // This is one of those places where having to force everything into
@@ -777,29 +803,33 @@ class RADOS final
   // let us separate out the implementation details without
   // sacrificing all the benefits of templates.
   using VoidOpSig = void();
-  using VoidOpComp = ceph::async::Completion<VoidOpSig>;
-  template<typename CompletionToken>
+  using VoidOpComp = boost::asio::any_completion_handler<VoidOpSig>;
+  template<boost::asio::completion_token_for<VoidOpSig> CompletionToken>
   auto flush_watch(CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, VoidOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), VoidOpSig>(
       [this](auto&& handler) {
-	flush_watch(VoidOpComp::create(get_executor(),
-				       std::move(handler)));
-      }, token);
+	flush_watch_(std::move(handler));
+      }, consigned);
   }
 
   using NotifySig = void(boost::system::error_code, ceph::buffer::list);
-  using NotifyComp = ceph::async::Completion<NotifySig>;
-  template<typename CompletionToken>
-  auto notify(Object oid, IOContext ioc, ceph::buffer::list bl,
+  using NotifyComp = boost::asio::any_completion_handler<NotifySig>;
+  template<boost::asio::completion_token_for<NotifySig> CompletionToken>
+  auto notify(Object o, IOContext ioc, ceph::buffer::list bl,
 	      std::optional<std::chrono::milliseconds> timeout,
 	      CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, NotifySig>(
-      [oid = std::move(oid), ioc = std::move(ioc),
-       bl = std::move(bl), timeout, this](auto&& handler) mutable {
-	notify(std::move(oid), std::move(ioc), std::move(bl), timeout,
-	       NotifyComp::create(get_executor(),
-				  std::move(handler)));
-      }, token);
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), NotifySig>(
+      [o = std::move(o), ioc = std::move(ioc), bl = std::move(bl), timeout,
+       this](auto&& handler) mutable {
+	notify_(std::move(o), std::move(ioc), std::move(bl), timeout,
+		std::move(handler));
+      }, consigned);
   }
 
   // The versions with pointers are fine for coroutines, but
@@ -807,94 +837,108 @@ class RADOS final
   using EnumerateSig = void(boost::system::error_code,
 			    std::vector<Entry>,
 			    Cursor);
-  using EnumerateComp = ceph::async::Completion<EnumerateSig>;
-  template<typename CompletionToken>
+  using EnumerateComp = boost::asio::any_completion_handler<EnumerateSig>;
+  template<boost::asio::completion_token_for<EnumerateSig> CompletionToken>
   auto enumerate_objects(IOContext ioc, Cursor begin,
 			 Cursor end, std::uint32_t max,
 			 ceph::buffer::list filter,
 			 CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, EnumerateSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), EnumerateSig>(
       [ioc = std::move(ioc), begin = std::move(begin), end = std::move(end),
        max, filter = std::move(filter), this](auto&& handler) mutable {
-	enumerate_objects(std::move(ioc), std::move(begin), std::move(end),
-			  max, std::move(filter),
-			  EnumerateComp::create(get_executor(),
-						std::move(handler)));
-      }, token);
+	enumerate_objects_(std::move(ioc), std::move(begin), std::move(end),
+			   std::move(max), std::move(filter),
+			   std::move(handler));
+      }, consigned);
   }
 
   using CommandSig = void(boost::system::error_code,
 			  std::string, ceph::buffer::list);
-  using CommandComp = ceph::async::Completion<CommandSig>;
-  template<typename CompletionToken>
+  using CommandComp = boost::asio::any_completion_handler<CommandSig>;
+  template<boost::asio::completion_token_for<CommandSig> CompletionToken>
   auto osd_command(int osd, std::vector<std::string> cmd,
 		   ceph::buffer::list in, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, CommandSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), CommandSig>(
       [osd, cmd = std::move(cmd), in = std::move(in),
        this](auto&& handler) mutable {
-	osd_command(osd, std::move(cmd), std::move(in),
-		    CommandComp::create(get_executor(),
-					std::move(handler)));
-      }, token);
+	osd_command_(osd, std::move(cmd), std::move(in),
+		     std::move(handler));
+      }, consigned);
   }
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<CommandSig> CompletionToken>
   auto pg_command(PG pg, std::vector<std::string> cmd,
 		  ceph::buffer::list in, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, CommandSig>(
-      [pg, cmd = std::move(cmd), in = std::move(in),
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), CommandSig>(
+      [pg = std::move(pg), cmd = std::move(cmd), in = std::move(in),
        this](auto&& handler) mutable {
-	pg_command(pg, std::move(cmd), std::move(in),
-		   CommandComp::create(get_executor(),
-				       std::move(handler)));
-      }, token);
+	pg_command_(std::move(pg), std::move(cmd), std::move(in),
+		    std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto mon_command(std::vector<std::string> command,
 		   ceph::buffer::list bl,
 		   std::string* outs, ceph::buffer::list* outbl,
 		   CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
-      [command = std::move(command), bl = std::move(bl), outs,
-       outbl, this](auto&& handler) mutable {
-	mon_command(std::move(command), std::move(bl), outs, outbl,
-		    SimpleOpComp::create(get_executor(),
-					 std::move(handler)));
-      }, token);
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
+      [command = std::move(command), bl = std::move(bl), outs, outbl,
+       this](auto&& handler) mutable {
+	mon_command_(std::move(command), std::move(bl), outs, outbl,
+		     std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto enable_application(std::string pool, std::string app_name,
 			  bool force, CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [pool = std::move(pool), app_name = std::move(app_name),
        force, this](auto&& handler) mutable {
-	enable_application(std::move(pool), std::move(app_name), force,
-			   SimpleOpComp::create(get_executor(),
-						std::move(handler)));
-      }, token);
+	enable_application_(std::move(pool), std::move(app_name), force,
+			    std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto blocklist_add(std::string client_address,
                      std::optional<std::chrono::seconds> expire,
                      CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [client_address = std::move(client_address), expire,
        this](auto&& handler) mutable {
-	blocklist_add(std::move(client_address), expire,
-		      SimpleOpComp::create(get_executor(),
-					   std::move(handler)));
-      }, token);
+	blocklist_add_(std::move(client_address), expire,
+		       std::move(handler));
+      }, consigned);
   }
 
-  template<typename CompletionToken>
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto wait_for_latest_osd_map(CompletionToken&& token) {
-    return boost::asio::async_initiate<CompletionToken, SimpleOpSig>(
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	boost::asio::get_associated_executor(token, get_executor())));
+    return boost::asio::async_initiate<decltype(consigned), SimpleOpSig>(
       [this](auto&& handler) {
-	wait_for_latest_osd_map(SimpleOpComp::create(get_executor(),
-						     std::move(handler)));
-      }, token);
+	wait_for_latest_osd_map_(std::move(handler));
+      }, consigned);
   }
 
   uint64_t instance_id() const;
@@ -906,77 +950,83 @@ class RADOS final
   friend Builder;
 
   RADOS(std::unique_ptr<detail::Client> impl);
-  static void make_with_cct(CephContext* cct,
-			    boost::asio::io_context& ioctx,
-		    std::unique_ptr<BuildComp> c);
-
-  void execute(Object o, IOContext ioc, ReadOp op,
-	       ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
-	       uint64_t* objver, const blkin_trace_info* trace_info);
-
-  void execute(Object o, IOContext ioc, WriteOp op,
-	       std::unique_ptr<Op::Completion> c, uint64_t* objver,
-	       const blkin_trace_info* trace_info);
-
-  void lookup_pool(std::string name, std::unique_ptr<LookupPoolComp> c);
-  void list_pools(std::unique_ptr<LSPoolsComp> c);
-  void create_pool_snap(int64_t pool, std::string snap_name,
-			std::unique_ptr<SimpleOpComp> c);
-  void allocate_selfmanaged_snap(int64_t pool, std::unique_ptr<SMSnapComp> c);
-  void delete_pool_snap(int64_t pool, std::string snap_name,
-			std::unique_ptr<SimpleOpComp> c);
-  void delete_selfmanaged_snap(int64_t pool, std::uint64_t snap,
-			       std::unique_ptr<SimpleOpComp> c);
-  void create_pool(std::string name, std::optional<int> crush_rule,
-		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(std::string name,
-		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(int64_t pool, std::unique_ptr<SimpleOpComp> c);
-  void stat_pools(std::vector<std::string> pools,
-		  std::unique_ptr<PoolStatComp> c);
-  void stat_fs(std::optional<std::int64_t> pool,
-	       std::unique_ptr<StatFSComp> c);
-
-  void watch(Object o, IOContext ioc,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB cb, std::unique_ptr<WatchComp> c);
+  static void make_with_cct_(CephContext* cct,
+			     boost::asio::io_context& ioctx,
+			     BuildComp c);
+
+  void execute_(Object o, IOContext ioc, ReadOp op,
+		ceph::buffer::list* bl, Op::Completion c,
+		uint64_t* objver, const blkin_trace_info* trace_info);
+
+  void execute_(Object o, IOContext ioc, WriteOp op,
+		Op::Completion c, uint64_t* objver,
+		const blkin_trace_info* trace_info);
+
+  void lookup_pool_(std::string name, LookupPoolComp c);
+  void list_pools_(LSPoolsComp c);
+  void create_pool_snap_(int64_t pool, std::string snap_name,
+			 SimpleOpComp c);
+  void allocate_selfmanaged_snap_(int64_t pool, SMSnapComp c);
+  void delete_pool_snap_(int64_t pool, std::string snap_name,
+			 SimpleOpComp c);
+  void delete_selfmanaged_snap_(int64_t pool, std::uint64_t snap,
+				SimpleOpComp c);
+  void create_pool_(std::string name, std::optional<int> crush_rule,
+		    SimpleOpComp c);
+  void delete_pool_(std::string name,
+		    SimpleOpComp c);
+  void delete_pool_(int64_t pool,
+		    SimpleOpComp c);
+  void stat_pools_(std::vector<std::string> pools,
+		   PoolStatComp c);
+  void stat_fs_(std::optional<std::int64_t> pool,
+		StatFSComp c);
+  void watch_(Object o, IOContext ioc,
+	      std::optional<std::chrono::seconds> timeout,
+	      WatchCB cb, WatchComp c);
   tl::expected<ceph::timespan, boost::system::error_code>
-  watch_check(uint64_t cookie);
-  void notify_ack(Object o, IOContext _ioc,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list bl,
-		  std::unique_ptr<SimpleOpComp>);
-  void unwatch(uint64_t cookie, IOContext ioc,
-	       std::unique_ptr<SimpleOpComp>);
-  void notify(Object oid, IOContext ioctx,
-	      ceph::buffer::list bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      std::unique_ptr<NotifyComp> c);
-  void flush_watch(std::unique_ptr<VoidOpComp>);
-
-  void enumerate_objects(IOContext ioc, Cursor begin,
-			 Cursor end, std::uint32_t max,
-			 ceph::buffer::list filter,
-			 std::unique_ptr<EnumerateComp> c);
-  void osd_command(int osd, std::vector<std::string> cmd,
-		   ceph::buffer::list in, std::unique_ptr<CommandComp> c);
-  void pg_command(PG pg, std::vector<std::string> cmd,
-		  ceph::buffer::list in, std::unique_ptr<CommandComp> c);
-
-  void mon_command(std::vector<std::string> command,
+  watch_check_(uint64_t cookie);
+  void notify_ack_(Object o, IOContext _ioc,
+		   uint64_t notify_id,
+		   uint64_t cookie,
 		   ceph::buffer::list bl,
-		   std::string* outs, ceph::buffer::list* outbl,
-		   std::unique_ptr<SimpleOpComp> c);
-
-  void enable_application(std::string pool, std::string app_name,
-			  bool force, std::unique_ptr<SimpleOpComp> c);
-
-  void blocklist_add(std::string client_address,
-                     std::optional<std::chrono::seconds> expire,
-                     std::unique_ptr<SimpleOpComp> c);
-
-  void wait_for_latest_osd_map(std::unique_ptr<SimpleOpComp> c);
+		   SimpleOpComp);
+  void unwatch_(uint64_t cookie, IOContext ioc,
+		SimpleOpComp);
+  void notify_(Object oid, IOContext ioctx,
+	       ceph::buffer::list bl,
+	       std::optional<std::chrono::milliseconds> timeout,
+	       NotifyComp c);
+  void flush_watch_(VoidOpComp);
+
+  void enumerate_objects_(IOContext ioc, Cursor begin,
+			  Cursor end, std::uint32_t max,
+			  ceph::buffer::list filter,
+			  std::vector<Entry>* ls,
+			  Cursor* cursor,
+			  SimpleOpComp c);
+  void enumerate_objects_(IOContext ioc, Cursor begin,
+			  Cursor end, std::uint32_t max,
+			  ceph::buffer::list filter,
+			  EnumerateComp c);
+  void osd_command_(int osd, std::vector<std::string> cmd,
+		    ceph::buffer::list in, CommandComp c);
+  void pg_command_(PG pg, std::vector<std::string> cmd,
+		   ceph::buffer::list in, CommandComp c);
+
+  void mon_command_(std::vector<std::string> command,
+		    ceph::buffer::list bl,
+		    std::string* outs, ceph::buffer::list* outbl,
+		    SimpleOpComp c);
+
+  void enable_application_(std::string pool, std::string app_name,
+			   bool force, SimpleOpComp c);
+
+  void blocklist_add_(std::string client_address,
+		      std::optional<std::chrono::seconds> expire,
+		      SimpleOpComp c);
+
+  void wait_for_latest_osd_map_(SimpleOpComp c);
 
   // Proxy object to provide access to low-level RADOS messaging clients
   std::unique_ptr<detail::Client> impl;
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index e1d38fd014a4..d66b56560f9c 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -1788,9 +1788,12 @@ int librados::IoCtxImpl::notify(const object_t& oid, bufferlist& bl,
                                                             extra_op_flags);
 
   C_SaferCond notify_finish_cond;
+  auto e = boost::asio::prefer(
+    objecter->service.get_executor(),
+    boost::asio::execution::outstanding_work.tracked);
   linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      objecter->service.get_executor(),
+    boost::asio::bind_executor(
+      std::move(e),
       CB_notify_Finish(client->cct, &notify_finish_cond,
                        objecter, linger_op, preply_bl,
                        preply_buf, preply_buf_len));
@@ -1844,9 +1847,12 @@ int librados::IoCtxImpl::aio_notify(const object_t& oid, AioCompletionImpl *c,
   c->io = this;
 
   C_aio_notify_Complete *oncomplete = new C_aio_notify_Complete(c, linger_op);
+  auto e = boost::asio::prefer(
+    objecter->service.get_executor(),
+    boost::asio::execution::outstanding_work.tracked);
   linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      objecter->service.get_executor(),
+    boost::asio::bind_executor(
+      std::move(e),
       CB_notify_Finish(client->cct, oncomplete,
                        objecter, linger_op,
                        preply_bl, preply_buf,
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 30861aace0d7..60d922d825f4 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -6027,8 +6027,11 @@ int Server::check_layout_vxattr(MDRequestRef& mdr,
       // latest map. One day if COMPACT_VERSION of MClientRequest >=3,
       // we can remove those code.
       mdr->waited_for_osdmap = true;
-      mds->objecter->wait_for_latest_osdmap(std::ref(*new C_IO_Wrapper(
-        mds, new C_MDS_RetryRequest(mdcache, mdr))));
+      mds->objecter->wait_for_latest_osdmap(
+	[c = new C_IO_Wrapper(mds, new C_MDS_RetryRequest(mdcache, mdr))]
+	(boost::system::error_code ec) {
+	  c->complete(ceph::from_error_code(ec));
+	});
       return r;
     }
   }
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index fabf2d5b3570..1a7c5cd37db3 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -39,9 +39,9 @@
 
 using namespace std::literals;
 
+namespace asio = boost::asio;
 namespace bc = boost::container;
 namespace bs = boost::system;
-namespace ca = ceph::async;
 namespace cb = ceph::buffer;
 
 namespace neorados {
@@ -703,8 +703,8 @@ RADOS::Builder& RADOS::Builder::add_conf_file(std::string_view f) {
   return *this;
 }
 
-void RADOS::Builder::build(boost::asio::io_context& ioctx,
-			   std::unique_ptr<BuildComp> c) {
+void RADOS::Builder::build_(asio::io_context& ioctx,
+			    BuildComp c) {
   constexpr auto env = CODE_ENVIRONMENT_LIBRARY;
   CephInitParameters ci(env);
   if (name)
@@ -732,7 +732,9 @@ void RADOS::Builder::build(boost::asio::io_context& ioctx,
     auto r = cct->_conf.parse_config_files(conf_files ? conf_files->data() : nullptr,
 					   &ss, flags);
     if (r < 0)
-      c->post(std::move(c), ceph::to_error_code(r), RADOS{nullptr});
+      asio::post(ioctx.get_executor(),
+		 asio::append(std::move(c), ceph::to_error_code(r),
+			      RADOS{nullptr}));
   }
 
   cct->_conf.parse_env(cct->get_module_type());
@@ -741,7 +743,9 @@ void RADOS::Builder::build(boost::asio::io_context& ioctx,
     std::stringstream ss;
     auto r = cct->_conf.set_val(n, v, &ss);
     if (r < 0)
-      c->post(std::move(c), ceph::to_error_code(-EINVAL), RADOS{nullptr});
+      asio::post(ioctx.get_executor(),
+		 asio::append(std::move(c), ceph::to_error_code(-EINVAL),
+			      RADOS{nullptr}));
   }
 
   if (!no_mon_conf) {
@@ -749,7 +753,9 @@ void RADOS::Builder::build(boost::asio::io_context& ioctx,
     // TODO This function should return an error code.
     auto err = mc_bootstrap.get_monmap_and_config();
     if (err < 0)
-      c->post(std::move(c), ceph::to_error_code(err), RADOS{nullptr});
+      asio::post(ioctx.get_executor(),
+		 asio::append(std::move(c), ceph::to_error_code(err),
+			      RADOS{nullptr}));
   }
   if (!cct->_log->is_started()) {
     cct->_log->start();
@@ -759,18 +765,19 @@ void RADOS::Builder::build(boost::asio::io_context& ioctx,
   RADOS::make_with_cct(cct, ioctx, std::move(c));
 }
 
-void RADOS::make_with_cct(CephContext* cct,
-			  boost::asio::io_context& ioctx,
-			  std::unique_ptr<BuildComp> c) {
+void RADOS::make_with_cct_(CephContext* cct,
+			   asio::io_context& ioctx,
+			   BuildComp c) {
   try {
     auto r = new detail::NeoClient{std::make_unique<detail::RADOS>(ioctx, cct)};
     r->objecter->wait_for_osd_map(
       [c = std::move(c), r = std::unique_ptr<detail::Client>(r)]() mutable {
-	c->dispatch(std::move(c), bs::error_code{},
-		    RADOS{std::move(r)});
+	asio::dispatch(asio::append(std::move(c), bs::error_code{},
+				    RADOS{std::move(r)}));
       });
   } catch (const bs::system_error& err) {
-    c->post(std::move(c), err.code(), RADOS{nullptr});
+    asio::post(ioctx.get_executor(),
+	       asio::append(std::move(c), err.code(), RADOS{nullptr}));
   }
 }
 
@@ -792,14 +799,14 @@ RADOS::executor_type RADOS::get_executor() const {
   return impl->ioctx.get_executor();
 }
 
-boost::asio::io_context& RADOS::get_io_context() {
+asio::io_context& RADOS::get_io_context() {
   return impl->ioctx;
 }
 
-void RADOS::execute(Object o, IOContext _ioc, ReadOp _op,
-		    cb::list* bl,
-		    std::unique_ptr<ReadOp::Completion> c, version_t* objver,
-		    const blkin_trace_info *trace_info) {
+void RADOS::execute_(Object o, IOContext _ioc, ReadOp _op,
+		     cb::list* bl,
+		     ReadOp::Completion c, version_t* objver,
+		     const blkin_trace_info *trace_info) {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
   auto op = reinterpret_cast<OpImpl*>(&_op.impl);
@@ -819,9 +826,9 @@ void RADOS::execute(Object o, IOContext _ioc, ReadOp _op,
   trace.event("submitted");
 }
 
-void RADOS::execute(Object o, IOContext _ioc, WriteOp _op,
-		    std::unique_ptr<WriteOp::Completion> c, version_t* objver,
-		    const blkin_trace_info *trace_info) {
+void RADOS::execute_(Object o, IOContext _ioc, WriteOp _op,
+		     WriteOp::Completion c, version_t* objver,
+		     const blkin_trace_info *trace_info) {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
   auto op = reinterpret_cast<OpImpl*>(&_op.impl);
@@ -850,8 +857,7 @@ boost::uuids::uuid RADOS::get_fsid() const noexcept {
   return impl->monclient.get_fsid().uuid;
 }
 
-
-void RADOS::lookup_pool(std::string name, std::unique_ptr<LookupPoolComp> c)
+void RADOS::lookup_pool_(std::string name, LookupPoolComp c)
 {
   // I kind of want to make lookup_pg_pool return
   // std::optional<int64_t> since it can only return one error code.
@@ -867,13 +873,14 @@ void RADOS::lookup_pool(std::string name, std::unique_ptr<LookupPoolComp> c)
 	    return osdmap.lookup_pg_pool_name(name);
 	  });
 	if (ret < 0)
-	  ca::dispatch(std::move(c), osdc_errc::pool_dne,
-		       std::int64_t(0));
+	  asio::dispatch(asio::append(std::move(c), osdc_errc::pool_dne,
+				      std::int64_t(0)));
 	else
-	  ca::dispatch(std::move(c), bs::error_code{}, ret);
+	  asio::dispatch(asio::append(std::move(c), bs::error_code{}, ret));
       });
   } else {
-    ca::post(std::move(c), bs::error_code{}, ret);
+    asio::post(get_executor(),
+	       asio::append(std::move(c), bs::error_code{}, ret));
   }
 }
 
@@ -894,107 +901,124 @@ std::optional<uint64_t> RADOS::get_pool_alignment(int64_t pool_id)
     });
 }
 
-void RADOS::list_pools(std::unique_ptr<LSPoolsComp> c) {
-  ca::dispatch(std::move(c),
-	       impl->objecter->with_osdmap(
-		 [&](OSDMap& o) {
-		   std::vector<std::pair<std::int64_t, std::string>> v;
-		   for (auto p : o.get_pools())
-		     v.push_back(std::make_pair(p.first,
-						o.get_pool_name(p.first)));
-		   return v;
-		 }));
+void RADOS::list_pools_(LSPoolsComp c) {
+  asio::dispatch(asio::append(std::move(c),
+			      impl->objecter->with_osdmap(
+				[&](OSDMap& o) {
+				  std::vector<std::pair<std::int64_t, std::string>> v;
+				  for (auto p : o.get_pools())
+				    v.push_back(std::make_pair(p.first,
+							       o.get_pool_name(p.first)));
+				  return v;
+				})));
 }
 
-void RADOS::create_pool_snap(std::int64_t pool,
-			     std::string snap_name,
-			     std::unique_ptr<SimpleOpComp> c)
+void RADOS::create_pool_snap_(std::int64_t pool,
+			      std::string snap_name,
+			      SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->create_pool_snap(
     pool, snap_name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }));
 }
 
-void RADOS::allocate_selfmanaged_snap(int64_t pool,
-				      std::unique_ptr<SMSnapComp> c) {
+void RADOS::allocate_selfmanaged_snap_(int64_t pool,
+				       SMSnapComp c) {
+  auto e = asio::prefer(
+    get_executor(),
+    asio::execution::outstanding_work.tracked);
+
   impl->objecter->allocate_selfmanaged_snap(
     pool,
-    ca::Completion<void(bs::error_code, snapid_t)>::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, snapid_t snap) mutable {
-	ca::dispatch(std::move(c), e, snap);
+	asio::dispatch(asio::append(std::move(c), e, snap));
       }));
 }
 
-void RADOS::delete_pool_snap(std::int64_t pool,
-			     std::string snap_name,
-			     std::unique_ptr<SimpleOpComp> c)
+void RADOS::delete_pool_snap_(std::int64_t pool,
+			      std::string snap_name,
+			      SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->delete_pool_snap(
     pool, snap_name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }));
 }
 
-void RADOS::delete_selfmanaged_snap(std::int64_t pool,
-				    std::uint64_t snap,
-				    std::unique_ptr<SimpleOpComp> c)
+void RADOS::delete_selfmanaged_snap_(std::int64_t pool,
+				     std::uint64_t snap,
+				     SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->delete_selfmanaged_snap(
     pool, snap,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }));
 }
 
-void RADOS::create_pool(std::string name,
-			std::optional<int> crush_rule,
-			std::unique_ptr<SimpleOpComp> c)
+void RADOS::create_pool_(std::string name,
+			 std::optional<int> crush_rule,
+			 SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
+
   impl->objecter->create_pool(
     name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }),
       crush_rule.value_or(-1));
 }
 
-void RADOS::delete_pool(std::string name, std::unique_ptr<SimpleOpComp> c)
+void RADOS::delete_pool_(std::string name, SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->delete_pool(
     name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }));
 }
 
-void RADOS::delete_pool(std::int64_t pool,
-			std::unique_ptr<SimpleOpComp> c)
+void RADOS::delete_pool_(std::int64_t pool,
+			 SimpleOpComp c)
 {
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->delete_pool(
     pool,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
+	asio::dispatch(asio::append(std::move(c), e));
       }));
 }
 
-void RADOS::stat_pools(std::vector<std::string> pools,
-		       std::unique_ptr<PoolStatComp> c) {
+void RADOS::stat_pools_(std::vector<std::string> pools,
+			PoolStatComp c) {
   impl->objecter->get_pool_stats(
     pools,
     [c = std::move(c)]
@@ -1033,12 +1057,13 @@ void RADOS::stat_pools(std::vector<std::string> pools,
 	pv.compressed_bytes_alloc = statfs.data_compressed_allocated;
       }
 
-      ca::dispatch(std::move(c), ec, std::move(result), per_pool);
+      asio::dispatch(asio::append(std::move(c), ec, std::move(result),
+				  per_pool));
     });
 }
 
-void RADOS::stat_fs(std::optional<std::int64_t> _pool,
-		    std::unique_ptr<StatFSComp> c) {
+void RADOS::stat_fs_(std::optional<std::int64_t> _pool,
+		     StatFSComp c) {
   std::optional<int64_t> pool;
   if (_pool)
     pool = *pool;
@@ -1046,15 +1071,15 @@ void RADOS::stat_fs(std::optional<std::int64_t> _pool,
     pool,
     [c = std::move(c)](bs::error_code ec, const struct ceph_statfs s) mutable {
       FSStats fso{s.kb, s.kb_used, s.kb_avail, s.num_objects};
-      c->dispatch(std::move(c), ec, std::move(fso));
+      asio::dispatch(asio::append(std::move(c), ec, std::move(fso)));
     });
 }
 
 // --- Watch/Notify
 
-void RADOS::watch(Object o, IOContext _ioc,
-		  std::optional<std::chrono::seconds> timeout, WatchCB cb,
-		  std::unique_ptr<WatchComp> c) {
+void RADOS::watch_(Object o, IOContext _ioc,
+		   std::optional<std::chrono::seconds> timeout, WatchCB cb,
+		   WatchComp c) {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
 
@@ -1066,20 +1091,22 @@ void RADOS::watch(Object o, IOContext _ioc,
   linger_op->handle = std::move(cb);
   op.watch(cookie, CEPH_OSD_WATCH_OP_WATCH, timeout.value_or(0s).count());
   bufferlist bl;
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->linger_watch(
     linger_op, op, ioc->snapc, ceph::real_clock::now(), bl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      std::move(e),
       [c = std::move(c), cookie](bs::error_code e, cb::list) mutable {
-	ca::dispatch(std::move(c), e, cookie);
+	asio::dispatch(asio::append(std::move(c), e, cookie));
       }), nullptr);
 }
 
-void RADOS::notify_ack(Object o, IOContext _ioc,
-		       uint64_t notify_id,
-		       uint64_t cookie,
-		       bufferlist bl,
-		       std::unique_ptr<SimpleOpComp> c)
+void RADOS::notify_ack_(Object o, IOContext _ioc,
+			uint64_t notify_id,
+			uint64_t cookie,
+			bufferlist bl,
+			SimpleOpComp c)
 {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
@@ -1091,14 +1118,14 @@ void RADOS::notify_ack(Object o, IOContext _ioc,
 		       nullptr, ioc->extra_op_flags, std::move(c));
 }
 
-tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check(uint64_t cookie)
+tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check_(uint64_t cookie)
 {
   Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
   return impl->objecter->linger_check(linger_op);
 }
 
-void RADOS::unwatch(uint64_t cookie, IOContext _ioc,
-		    std::unique_ptr<SimpleOpComp> c)
+void RADOS::unwatch_(uint64_t cookie, IOContext _ioc,
+		     SimpleOpComp c)
 {
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
 
@@ -1106,48 +1133,50 @@ void RADOS::unwatch(uint64_t cookie, IOContext _ioc,
 
   ObjectOperation op;
   op.watch(cookie, CEPH_OSD_WATCH_OP_UNWATCH);
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   impl->objecter->mutate(linger_op->target.base_oid, ioc->oloc, std::move(op),
 			 ioc->snapc, ceph::real_clock::now(), ioc->extra_op_flags,
-			 Objecter::Op::OpComp::create(
-			   get_executor(),
+			 asio::bind_executor(
+			   std::move(e),
 			   [objecter = impl->objecter,
 			    linger_op, c = std::move(c)]
 			   (bs::error_code ec) mutable {
 			     objecter->linger_cancel(linger_op);
-			     ca::dispatch(std::move(c), ec);
+			     asio::dispatch(asio::append(std::move(c), ec));
 			   }));
 }
 
-void RADOS::flush_watch(std::unique_ptr<VoidOpComp> c)
+void RADOS::flush_watch_(VoidOpComp c)
 {
   impl->objecter->linger_callback_flush([c = std::move(c)]() mutable {
-					  ca::post(std::move(c));
+					  asio::dispatch(std::move(c));
 					});
 }
 
 struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
-  boost::asio::io_context& ioc;
-  boost::asio::strand<boost::asio::io_context::executor_type> strand;
+  asio::io_context& ioc;
+  asio::strand<asio::io_context::executor_type> strand;
   Objecter* objecter;
   Objecter::LingerOp* op;
-  std::unique_ptr<RADOS::NotifyComp> c;
+  RADOS::NotifyComp c;
 
   bool acked = false;
   bool finished = false;
   bs::error_code res;
   bufferlist rbl;
 
-  NotifyHandler(boost::asio::io_context& ioc,
+  NotifyHandler(asio::io_context& ioc,
 		Objecter* objecter,
 		Objecter::LingerOp* op,
-		std::unique_ptr<RADOS::NotifyComp> c)
-    : ioc(ioc), strand(boost::asio::make_strand(ioc)),
+		RADOS::NotifyComp c)
+    : ioc(ioc), strand(asio::make_strand(ioc)),
       objecter(objecter), op(op), c(std::move(c)) {}
 
   // Use bind or a lambda to pass this in.
   void handle_ack(bs::error_code ec,
 		  bufferlist&&) {
-    boost::asio::post(
+    asio::post(
       strand,
       [this, ec, p = shared_from_this()]() mutable {
 	acked = true;
@@ -1159,7 +1188,7 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
 
   void operator()(bs::error_code ec,
 		  bufferlist&& bl) {
-    boost::asio::post(
+    asio::post(
       strand,
       [this, ec, p = shared_from_this()]() mutable {
 	finished = true;
@@ -1174,14 +1203,14 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
     if ((acked && finished) || res) {
       objecter->linger_cancel(op);
       ceph_assert(c);
-      ca::dispatch(std::move(c), res, std::move(rbl));
+      asio::dispatch(asio::append(std::move(c), res, std::move(rbl)));
     }
   }
 };
 
-void RADOS::notify(Object o, IOContext _ioc, bufferlist bl,
-		   std::optional<std::chrono::milliseconds> timeout,
-		   std::unique_ptr<NotifyComp> c)
+void RADOS::notify_(Object o, IOContext _ioc, bufferlist bl,
+		    std::optional<std::chrono::milliseconds> timeout,
+		    NotifyComp c)
 {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
@@ -1190,9 +1219,11 @@ void RADOS::notify(Object o, IOContext _ioc, bufferlist bl,
 
   auto cb = std::make_shared<NotifyHandler>(impl->ioctx, impl->objecter,
                                             linger_op, std::move(c));
+  auto e = asio::prefer(get_executor(),
+			asio::execution::outstanding_work.tracked);
   linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      e,
       [cb](bs::error_code ec, ceph::bufferlist bl) mutable {
 	(*cb)(ec, std::move(bl));
       });
@@ -1205,8 +1236,8 @@ void RADOS::notify(Object o, IOContext _ioc, bufferlist bl,
 
   impl->objecter->linger_notify(
     linger_op, rd, ioc->snap_seq, inbl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
+    asio::bind_executor(
+      e,
       [cb](bs::error_code ec, ceph::bufferlist bl) mutable {
 	cb->handle_ack(ec, std::move(bl));
       }), nullptr);
@@ -1313,9 +1344,10 @@ Cursor::from_str(const std::string& s) {
   return e;
 }
 
-void RADOS::enumerate_objects(IOContext _ioc, Cursor begin, Cursor end,
-			      std::uint32_t max, ceph::buffer::list filter,
-			      std::unique_ptr<EnumerateComp> c) {
+void RADOS::enumerate_objects_(IOContext _ioc, Cursor begin, Cursor end,
+			       const std::uint32_t max,
+			       bufferlist filter,
+			       EnumerateComp c) {
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
   impl->objecter->enumerate_objects<Entry>(
     ioc->oloc.pool,
@@ -1327,43 +1359,42 @@ void RADOS::enumerate_objects(IOContext _ioc, Cursor begin, Cursor end,
     [c = std::move(c)]
     (bs::error_code ec, std::vector<Entry>&& v,
      hobject_t&& n) mutable {
-      ca::dispatch(std::move(c), ec, std::move(v),
-		   Cursor(static_cast<void*>(&n)));
+      asio::dispatch(asio::append(std::move(c), ec, std::move(v),
+				  Cursor(static_cast<void*>(&n))));
+    });
+}
+
+void RADOS::osd_command_(int osd, std::vector<std::string> cmd,
+			 ceph::bufferlist in, CommandComp c) {
+  impl->objecter->osd_command(
+    osd, std::move(cmd), std::move(in), nullptr,
+    [c = std::move(c)]
+    (bs::error_code ec, std::string&& s, ceph::bufferlist&& b) mutable {
+      asio::dispatch(asio::append(std::move(c), ec, std::move(s),
+				  std::move(b)));
     });
 }
 
-void RADOS::osd_command(int osd, std::vector<std::string> cmd,
-			ceph::buffer::list in, std::unique_ptr<CommandComp> c) {
-  impl->objecter->osd_command(osd, std::move(cmd), std::move(in), nullptr,
-			      [c = std::move(c)]
-			      (bs::error_code ec,
-			       std::string&& s,
-			       ceph::bufferlist&& b) mutable {
-				ca::dispatch(std::move(c), ec,
-					     std::move(s),
-					     std::move(b));
-			      });
-}
-void RADOS::pg_command(PG pg, std::vector<std::string> cmd,
-		       ceph::buffer::list in, std::unique_ptr<CommandComp> c) {
-  impl->objecter->pg_command(pg_t{pg.seed, pg.pool}, std::move(cmd), std::move(in), nullptr,
-			     [c = std::move(c)]
-			     (bs::error_code ec,
-			      std::string&& s,
-			      ceph::bufferlist&& b) mutable {
-			       ca::dispatch(std::move(c), ec,
-					    std::move(s),
-					    std::move(b));
-			     });
-}
-
-void RADOS::enable_application(std::string pool, std::string app_name,
-			       bool force, std::unique_ptr<SimpleOpComp> c) {
+void RADOS::pg_command_(PG pg, std::vector<std::string> cmd,
+			ceph::bufferlist in, CommandComp c) {
+  impl->objecter->pg_command(
+    pg_t{pg.seed, pg.pool}, std::move(cmd), std::move(in), nullptr,
+    [c = std::move(c)]
+    (bs::error_code ec, std::string&& s,
+     ceph::bufferlist&& b) mutable {
+      asio::dispatch(asio::append(std::move(c), ec, std::move(s),
+				  std::move(b)));
+    });
+}
+
+void RADOS::enable_application_(std::string pool, std::string app_name,
+				bool force, SimpleOpComp c) {
   // pre-Luminous clusters will return -EINVAL and application won't be
   // preserved until Luminous is configured as minimum version.
   if (!impl->get_required_monitor_features().contains_all(
 	ceph::features::mon::FEATURE_LUMINOUS)) {
-    ca::post(std::move(c), ceph::to_error_code(-EOPNOTSUPP));
+    asio::post(get_executor(),
+	       asio::append(std::move(c), ceph::to_error_code(-EOPNOTSUPP)));
   } else {
     impl->monclient.start_mon_command(
       { fmt::format("{{ \"prefix\": \"osd pool application enable\","
@@ -1372,14 +1403,14 @@ void RADOS::enable_application(std::string pool, std::string app_name,
 		    force ? " ,\"yes_i_really_mean_it\": true" : "")},
       {}, [c = std::move(c)](bs::error_code e,
 			     std::string, cb::list) mutable {
-	    ca::post(std::move(c), e);
-	  });
+	asio::dispatch(asio::append(std::move(c), e));
+      });
   }
 }
 
-void RADOS::blocklist_add(std::string client_address,
-                          std::optional<std::chrono::seconds> expire,
-                          std::unique_ptr<SimpleOpComp> c) {
+void RADOS::blocklist_add_(std::string client_address,
+			   std::optional<std::chrono::seconds> expire,
+			   SimpleOpComp c) {
   auto expire_arg = (expire ?
     fmt::format(", \"expire\": \"{}.0\"", expire->count()) : std::string{});
   impl->monclient.start_mon_command(
@@ -1392,7 +1423,8 @@ void RADOS::blocklist_add(std::string client_address,
     [this, client_address = std::string(client_address), expire_arg,
      c = std::move(c)](bs::error_code ec, std::string, cb::list) mutable {
       if (ec != bs::errc::invalid_argument) {
-        ca::post(std::move(c), ec);
+        asio::post(get_executor(),
+		   asio::append(std::move(c), ec));
         return;
       }
 
@@ -1405,18 +1437,18 @@ void RADOS::blocklist_add(std::string client_address,
                       client_address, expire_arg) },
         {},
         [c = std::move(c)](bs::error_code ec, std::string, cb::list) mutable {
-          ca::post(std::move(c), ec);
+          asio::dispatch(asio::append(std::move(c), ec));
         });
     });
 }
 
-void RADOS::wait_for_latest_osd_map(std::unique_ptr<SimpleOpComp> c) {
+void RADOS::wait_for_latest_osd_map_(SimpleOpComp c) {
   impl->objecter->wait_for_latest_osdmap(std::move(c));
 }
 
-void RADOS::mon_command(std::vector<std::string> command,
-			cb::list bl, std::string* outs, cb::list* outbl,
-			std::unique_ptr<SimpleOpComp> c) {
+void RADOS::mon_command_(std::vector<std::string> command,
+			 cb::list bl, std::string* outs, cb::list* outbl,
+			 SimpleOpComp c) {
 
   impl->monclient.start_mon_command(
     command, bl,
@@ -1426,7 +1458,7 @@ void RADOS::mon_command(std::vector<std::string> command,
 	*outs = std::move(s);
       if (outbl)
 	*outbl = std::move(bl);
-      ca::post(std::move(c), e);
+      asio::dispatch(asio::append(std::move(c), e));
     });
 }
 
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 6bc713fcbf5e..a74d21b0f15f 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -95,6 +95,7 @@ namespace bc = boost::container;
 namespace bs = boost::system;
 namespace ca = ceph::async;
 namespace cb = ceph::buffer;
+namespace asio = boost::asio;
 
 #define dout_subsys ceph_subsys_objecter
 #undef dout_prefix
@@ -604,14 +605,14 @@ void Objecter::_linger_commit(LingerOp *info, bs::error_code ec,
   std::unique_lock wl(info->watch_lock);
   ldout(cct, 10) << "_linger_commit " << info->linger_id << dendl;
   if (info->on_reg_commit) {
-    info->on_reg_commit->defer(std::move(info->on_reg_commit),
-			       ec, cb::list{});
-    info->on_reg_commit.reset();
+    asio::defer(service.get_executor(),
+		asio::append(std::move(info->on_reg_commit),
+			     ec, cb::list{}));
   }
   if (ec && info->on_notify_finish) {
-    info->on_notify_finish->defer(std::move(info->on_notify_finish),
-				  ec, cb::list{});
-    info->on_notify_finish.reset();
+    asio::defer(service.get_executor(),
+		asio::append(std::move(info->on_notify_finish),
+			     ec, cb::list{}));
   }
 
   // only tell the user the first time we do this
@@ -673,7 +674,7 @@ void Objecter::_linger_reconnect(LingerOp *info, bs::error_code ec)
     if (!info->last_error) {
       ec = _normalize_watch_error(ec);
       if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
+	asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
       }
     }
   }
@@ -708,7 +709,7 @@ void Objecter::_send_linger_ping(LingerOp *info)
 
   Op *o = new Op(info->target.base_oid, info->target.base_oloc,
 		 std::move(opv), info->target.flags | CEPH_OSD_FLAG_READ,
-		 CB_Linger_Ping(this, info, now),
+		 fu2::unique_function<Op::OpSig>{CB_Linger_Ping(this, info, now)},
 		 nullptr, nullptr);
   o->target = info->target;
   o->should_resend = false;
@@ -736,7 +737,7 @@ void Objecter::_linger_ping(LingerOp *info, bs::error_code ec, ceph::coarse_mono
       ec = _normalize_watch_error(ec);
       info->last_error = ec;
       if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
+	asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
       }
     }
   } else {
@@ -924,7 +925,7 @@ void Objecter::handle_watch_notify(MWatchNotify *m)
     if (!info->last_error) {
       info->last_error = bs::error_code(ENOTCONN, osd_category());
       if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info,
+	asio::defer(finish_strand, CB_DoWatchError(this, info,
 							  info->last_error));
       }
     }
@@ -937,16 +938,16 @@ void Objecter::handle_watch_notify(MWatchNotify *m)
       ldout(cct, 10) << __func__ << " reply notify " << m->notify_id
 		     << " != " << info->notify_id << ", ignoring" << dendl;
     } else if (info->on_notify_finish) {
-      info->on_notify_finish->defer(
-	std::move(info->on_notify_finish),
-	osdcode(m->return_code), std::move(m->get_data()));
-
+      asio::defer(service.get_executor(),
+		  asio::append(std::move(info->on_notify_finish),
+			       osdcode(m->return_code),
+			       std::move(m->get_data())));
       // if we race with reconnect we might get a second notify; only
       // notify the caller once!
       info->on_notify_finish = nullptr;
     }
   } else {
-    boost::asio::defer(finish_strand, CB_DoWatchNotify(this, info, m));
+    asio::defer(finish_strand, CB_DoWatchNotify(this, info, m));
   }
 }
 
@@ -1379,7 +1380,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
 	 p->first <= osdmap->get_epoch()) {
     //go through the list and call the onfinish methods
     for (auto& [c, ec] : p->second) {
-      ca::post(std::move(c), ec);
+      asio::post(service.get_executor(), asio::append(std::move(c), ec));
     }
     waiting_for_map.erase(p++);
   }
@@ -1568,7 +1569,7 @@ void Objecter::_check_op_pool_dne(Op *op, std::unique_lock<std::shared_mutex> *s
 		     << " dne" << dendl;
       if (op->has_completion()) {
 	num_in_flight--;
-	op->complete(osdc_errc::pool_dne, -ENOENT);
+	op->complete(osdc_errc::pool_dne, -ENOENT, service.get_executor());
       }
 
       OSDSession *s = op->session;
@@ -1603,7 +1604,7 @@ void Objecter::_check_op_pool_eio(Op *op, std::unique_lock<std::shared_mutex> *s
 		 << " has eio" << dendl;
   if (op->has_completion()) {
     num_in_flight--;
-    op->complete(osdc_errc::pool_eio, -EIO);
+    op->complete(osdc_errc::pool_eio, -EIO, service.get_executor());
   }
 
   OSDSession *s = op->session;
@@ -1701,13 +1702,15 @@ void Objecter::_check_linger_pool_dne(LingerOp *op, bool *need_unregister)
     if (osdmap->get_epoch() >= op->map_dne_bound) {
       std::unique_lock wl{op->watch_lock};
       if (op->on_reg_commit) {
-	op->on_reg_commit->defer(std::move(op->on_reg_commit),
-				 osdc_errc::pool_dne, cb::list{});
+	asio::defer(service.get_executor(),
+		    asio::append(std::move(op->on_reg_commit),
+				 osdc_errc::pool_dne, cb::list{}));
 	op->on_reg_commit = nullptr;
       }
       if (op->on_notify_finish) {
-	op->on_notify_finish->defer(std::move(op->on_notify_finish),
-				    osdc_errc::pool_dne, cb::list{});
+	asio::defer(service.get_executor(),
+		    asio::append(std::move(op->on_notify_finish),
+				 osdc_errc::pool_dne, cb::list{}));
         op->on_notify_finish = nullptr;
       }
       *need_unregister = true;
@@ -1723,14 +1726,14 @@ void Objecter::_check_linger_pool_eio(LingerOp *op)
 
   std::unique_lock wl{op->watch_lock};
   if (op->on_reg_commit) {
-    op->on_reg_commit->defer(std::move(op->on_reg_commit),
-			     osdc_errc::pool_dne, cb::list{});
-    op->on_reg_commit = nullptr;
+    asio::defer(service.get_executor(),
+		asio::append(std::move(op->on_reg_commit),
+			     osdc_errc::pool_dne, cb::list{}));
   }
   if (op->on_notify_finish) {
-    op->on_notify_finish->defer(std::move(op->on_notify_finish),
-				osdc_errc::pool_dne, cb::list{});
-    op->on_notify_finish = nullptr;
+    asio::defer(service.get_executor(),
+		asio::append(std::move(op->on_notify_finish),
+			     osdc_errc::pool_dne, cb::list{}));
   }
 }
 
@@ -1984,7 +1987,10 @@ void Objecter::wait_for_osd_map(epoch_t e)
   }
 
   ca::waiter<bs::error_code> w;
-  waiting_for_map[e].emplace_back(OpCompletion::create(
+  auto ex = boost::asio::prefer(
+    service.get_executor(),
+    boost::asio::execution::outstanding_work.tracked);
+  waiting_for_map[e].emplace_back(asio::bind_executor(
 				    service.get_executor(),
 				    w.ref()),
 				  bs::error_code{});
@@ -1993,14 +1999,15 @@ void Objecter::wait_for_osd_map(epoch_t e)
 }
 
 void Objecter::_get_latest_version(epoch_t oldest, epoch_t newest,
-				   std::unique_ptr<OpCompletion> fin,
+				   OpCompletion fin,
 				   std::unique_lock<ceph::shared_mutex>&& l)
 {
   ceph_assert(fin);
   if (osdmap->get_epoch() >= newest) {
     ldout(cct, 10) << __func__ << " latest " << newest << ", have it" << dendl;
     l.unlock();
-    ca::defer(std::move(fin), bs::error_code{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(fin), bs::error_code{}));
   } else {
     ldout(cct, 10) << __func__ << " latest " << newest << ", waiting" << dendl;
     _wait_for_new_map(std::move(fin), newest, bs::error_code{});
@@ -2034,7 +2041,7 @@ void Objecter::_maybe_request_map()
   }
 }
 
-void Objecter::_wait_for_new_map(std::unique_ptr<OpCompletion> c, epoch_t epoch,
+void Objecter::_wait_for_new_map(OpCompletion c, epoch_t epoch,
 				 bs::error_code ec)
 {
   // rwlock is locked unique
@@ -2399,7 +2406,7 @@ void Objecter::_op_submit(Op *op, shunique_lock<ceph::shared_mutex>& sul, ceph_t
     break;
   case RECALC_OP_TARGET_POOL_EIO:
     if (op->has_completion()) {
-      op->complete(osdc_errc::pool_eio, -EIO);
+      op->complete(osdc_errc::pool_eio, -EIO, service.get_executor());
     }
     return;
   }
@@ -2510,7 +2517,7 @@ int Objecter::op_cancel(OSDSession *s, ceph_tid_t tid, int r)
   Op *op = p->second;
   if (op->has_completion()) {
     num_in_flight--;
-    op->complete(osdcode(r), r);
+    op->complete(osdcode(r), r, service.get_executor());
   }
   _op_cancel_map_check(op);
   _finish_op(op, r);
@@ -3625,9 +3632,9 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
   // do callbacks
   if (Op::has_completion(onfinish)) {
     if (rc == 0 && handler_error) {
-      Op::complete(std::move(onfinish), handler_error, -EIO);
+      Op::complete(std::move(onfinish), handler_error, -EIO, service.get_executor());
     } else {
-      Op::complete(std::move(onfinish), osdcode(rc), rc);
+      Op::complete(std::move(onfinish), osdcode(rc), rc, service.get_executor());
     }
   }
   if (completion_lock.mutex()) {
@@ -3928,12 +3935,14 @@ void Objecter::create_pool_snap(int64_t pool, std::string_view snap_name,
 
   const pg_pool_t *p = osdmap->get_pg_pool(pool);
   if (!p) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::pool_dne, cb::list{}));
     return;
   }
   if (p->snap_exists(snap_name)) {
-    onfinish->defer(std::move(onfinish), osdc_errc::snapshot_exists,
-		    cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::snapshot_exists,
+			     cb::list{}));
     return;
   }
 
@@ -3949,7 +3958,7 @@ void Objecter::create_pool_snap(int64_t pool, std::string_view snap_name,
 }
 
 struct CB_SelfmanagedSnap {
-  std::unique_ptr<ca::Completion<void(bs::error_code, snapid_t)>> fin;
+  asio::any_completion_handler<void(bs::error_code, snapid_t)> fin;
   CB_SelfmanagedSnap(decltype(fin)&& fin)
     : fin(std::move(fin)) {}
   void operator()(bs::error_code ec, const cb::list& bl) {
@@ -3962,22 +3971,23 @@ struct CB_SelfmanagedSnap {
         ec = e.code();
       }
     }
-    fin->defer(std::move(fin), ec, snapid);
+    asio::dispatch(asio::append(std::move(fin), ec, snapid));
   }
 };
 
 void Objecter::allocate_selfmanaged_snap(
   int64_t pool,
-  std::unique_ptr<ca::Completion<void(bs::error_code, snapid_t)>> onfinish)
+  asio::any_completion_handler<void(bs::error_code, snapid_t)> onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "allocate_selfmanaged_snap; pool: " << pool << dendl;
   auto op = new PoolOp;
   op->tid = ++last_tid;
   op->pool = pool;
-  op->onfinish = PoolOp::OpComp::create(
+  auto e = boost::asio::prefer(
     service.get_executor(),
-    CB_SelfmanagedSnap(std::move(onfinish)));
+    boost::asio::execution::outstanding_work.tracked);
+  op->onfinish = asio::bind_executor(e, CB_SelfmanagedSnap(std::move(onfinish)));
   op->pool_op = POOL_OP_CREATE_UNMANAGED_SNAP;
   pool_ops[op->tid] = op;
 
@@ -3994,12 +4004,15 @@ void Objecter::delete_pool_snap(
 
   const pg_pool_t *p = osdmap->get_pg_pool(pool);
   if (!p) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::pool_dne,
+			     cb::list{}));
     return;
   }
 
   if (!p->snap_exists(snap_name)) {
-    onfinish->defer(std::move(onfinish), osdc_errc::snapshot_dne, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::snapshot_dne, cb::list{}));
     return;
   }
 
@@ -4039,7 +4052,9 @@ void Objecter::create_pool(std::string_view name,
   ldout(cct, 10) << "create_pool name=" << name << dendl;
 
   if (osdmap->lookup_pg_pool_name(name) >= 0) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_exists, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::pool_exists,
+			     cb::list{}));
     return;
   }
 
@@ -4062,7 +4077,9 @@ void Objecter::delete_pool(int64_t pool,
   ldout(cct, 10) << "delete_pool " << pool << dendl;
 
   if (!osdmap->have_pg_pool(pool))
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::pool_dne,
+			     cb::list{}));
   else
     _do_delete_pool(pool, std::move(onfinish));
 }
@@ -4076,7 +4093,9 @@ void Objecter::delete_pool(std::string_view pool_name,
   int64_t pool = osdmap->lookup_pg_pool_name(pool_name);
   if (pool < 0)
     // This only returns one error: -ENOENT.
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(onfinish), osdc_errc::pool_dne,
+			     cb::list{}));
   else
     _do_delete_pool(pool, std::move(onfinish));
 }
@@ -4162,12 +4181,16 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
       if (osdmap->get_epoch() < m->epoch) {
 	ldout(cct, 20) << "waiting for client to reach epoch " << m->epoch
 		       << " before calling back" << dendl;
-	_wait_for_new_map(OpCompletion::create(
-			    service.get_executor(),
+	auto e = boost::asio::prefer(
+	  service.get_executor(),
+	  boost::asio::execution::outstanding_work.tracked);
+	_wait_for_new_map(asio::bind_executor(
+			    e,
 			    [o = std::move(op->onfinish),
-			     bl = std::move(bl)](
+			     bl = std::move(bl),
+			     e = service.get_executor()](
 			      bs::error_code ec) mutable {
-			      o->defer(std::move(o), ec, bl);
+			      asio::defer(e, asio::append(std::move(o), ec, bl));
 			    }),
 			  m->epoch,
 			  ec);
@@ -4176,11 +4199,11 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
 	// sneaked in. Do caller-specified callback now or else
 	// we lose it forever.
 	ceph_assert(op->onfinish);
-	op->onfinish->defer(std::move(op->onfinish), ec, std::move(bl));
+	asio::defer(service.get_executor(), asio::append(std::move(op->onfinish), ec, std::move(bl)));
       }
     } else {
       ceph_assert(op->onfinish);
-      op->onfinish->defer(std::move(op->onfinish), ec, std::move(bl));
+      asio::defer(service.get_executor(), asio::append(std::move(op->onfinish), ec, std::move(bl)));
     }
     op->onfinish = nullptr;
     if (!sul.owns_lock()) {
@@ -4219,7 +4242,8 @@ int Objecter::pool_op_cancel(ceph_tid_t tid, int r)
 
   PoolOp *op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r), cb::list{});
+    asio::defer(service.get_executor(), asio::append(std::move(op->onfinish),
+						     osdcode(r), cb::list{}));
 
   _finish_pool_op(op, r);
   return 0;
@@ -4240,7 +4264,7 @@ void Objecter::_finish_pool_op(PoolOp *op, int r)
 
 // pool stats
 
-void Objecter::get_pool_stats(
+void Objecter::get_pool_stats_(
   const std::vector<std::string>& pools,
   decltype(PoolStatOp::onfinish)&& onfinish)
 {
@@ -4297,8 +4321,9 @@ void Objecter::handle_get_pool_stats_reply(MGetPoolStatsReply *m)
     if (m->version > last_seen_pgmap_version) {
       last_seen_pgmap_version = m->version;
     }
-    op->onfinish->defer(std::move(op->onfinish), bs::error_code{},
-			std::move(m->pool_stats), m->per_pool);
+    asio::defer(service.get_executor(),
+		asio::append(std::move(op->onfinish), bs::error_code{},
+			     std::move(m->pool_stats), m->per_pool));
     _finish_pool_stat_op(op, 0);
   } else {
     ldout(cct, 10) << "unknown request " << tid << dendl;
@@ -4323,8 +4348,9 @@ int Objecter::pool_stat_op_cancel(ceph_tid_t tid, int r)
 
   auto op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r),
-			bc::flat_map<std::string, pool_stat_t>{}, false);
+    asio::defer(service.get_executor(),
+		asio::append(std::move(op->onfinish), osdcode(r),
+			     bc::flat_map<std::string, pool_stat_t>{}, false));
   _finish_pool_stat_op(op, r);
   return 0;
 }
@@ -4342,8 +4368,8 @@ void Objecter::_finish_pool_stat_op(PoolStatOp *op, int r)
   delete op;
 }
 
-void Objecter::get_fs_stats(std::optional<int64_t> poolid,
-			    decltype(StatfsOp::onfinish)&& onfinish)
+void Objecter::get_fs_stats_(std::optional<int64_t> poolid,
+			     decltype(StatfsOp::onfinish)&& onfinish)
 {
   ldout(cct, 10) << "get_fs_stats" << dendl;
   unique_lock l(rwlock);
@@ -4396,7 +4422,8 @@ void Objecter::handle_fs_stats_reply(MStatfsReply *m)
     ldout(cct, 10) << "have request " << tid << " at " << op << dendl;
     if (m->h.version > last_seen_pgmap_version)
       last_seen_pgmap_version = m->h.version;
-    op->onfinish->defer(std::move(op->onfinish), bs::error_code{}, m->h.st);
+    asio::defer(service.get_executor(), asio::append(std::move(op->onfinish),
+						     bs::error_code{}, m->h.st));
     _finish_statfs_op(op, 0);
   } else {
     ldout(cct, 10) << "unknown request " << tid << dendl;
@@ -4421,7 +4448,9 @@ int Objecter::statfs_op_cancel(ceph_tid_t tid, int r)
 
   auto op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r), ceph_statfs{});
+    asio::defer(service.get_executor(),
+		asio::append(std::move(op->onfinish),
+			     osdcode(r), ceph_statfs{}));
   _finish_statfs_op(op, r);
   return 0;
 }
@@ -5022,7 +5051,9 @@ void Objecter::_finish_command(CommandOp *c, bs::error_code ec,
 		 << rs << dendl;
 
   if (c->onfinish)
-    c->onfinish->defer(std::move(c->onfinish), ec, std::move(rs), std::move(bl));
+    asio::defer(service.get_executor(),
+		asio::append(std::move(c->onfinish), ec, std::move(rs),
+			     std::move(bl)));
 
   if (c->ontimeout && ec != bs::errc::timed_out)
     timer.cancel_event(c->ontimeout);
@@ -5045,7 +5076,7 @@ Objecter::OSDSession::~OSDSession()
 
 Objecter::Objecter(CephContext *cct,
 		   Messenger *m, MonClient *mc,
-		   boost::asio::io_context& service) :
+		   asio::io_context& service) :
   Dispatcher(cct), messenger(m), monc(mc), service(service)
 {
   mon_timeout = cct->_conf.get_val<std::chrono::seconds>("rados_mon_op_timeout");
@@ -5249,9 +5280,12 @@ void Objecter::_issue_enumerate(hobject_t start,
   auto pbl = &on_ack->bl;
 
   // Issue.  See you later in _enumerate_reply
+  auto e = boost::asio::prefer(
+    service.get_executor(),
+    boost::asio::execution::outstanding_work.tracked);
   pg_read(start.get_hash(),
 	  c->oloc, op, pbl, 0,
-	  Op::OpComp::create(service.get_executor(),
+	  asio::bind_executor(e,
 			     [c = std::move(on_ack)]
 			     (bs::error_code ec) mutable {
 			       (*c)(ec);
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 9ca09b476476..870083a29b6c 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -27,7 +27,10 @@
 #include <variant>
 
 #include <boost/container/small_vector.hpp>
+#include <boost/asio/any_completion_handler.hpp>
+#include <boost/asio/append.hpp>
 #include <boost/asio/async_result.hpp>
+#include <boost/asio/consign.hpp>
 #include <boost/asio/defer.hpp>
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/io_context_strand.hpp>
@@ -45,8 +48,8 @@
 #include "include/function2.hpp"
 #include "include/neorados/RADOS_Decodable.hpp"
 
-#include "common/admin_socket.h"
 #include "common/async/completion.h"
+#include "common/admin_socket.h"
 #include "common/ceph_time.h"
 #include "common/ceph_mutex.h"
 #include "common/ceph_timer.h"
@@ -1630,7 +1633,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   using MOSDOp = _mosdop::MOSDOp<osdc_opvec>;
 public:
   using OpSignature = void(boost::system::error_code);
-  using OpCompletion = ceph::async::Completion<OpSignature>;
+  using OpCompletion = boost::asio::any_completion_handler<OpSignature>;
 
   // config observer bits
   const char** get_tracked_conf_keys() const override;
@@ -1847,55 +1850,91 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     void dump(ceph::Formatter *f) const;
   };
 
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code)>>
+  boost::asio::any_completion_handler<void(boost::system::error_code)>
   OpContextVert(Context* c) {
-    if (c)
-      return ceph::async::Completion<void(boost::system::error_code)>::create(
+    if (c) {
+      auto e = boost::asio::prefer(
 	service.get_executor(),
+	boost::asio::execution::outstanding_work.tracked);
+
+      return boost::asio::bind_executor(
+	std::move(e),
 	[c = std::unique_ptr<Context>(c)]
 	(boost::system::error_code e) mutable {
 	  c.release()->complete(e);
 	});
+    }
     else
       return nullptr;
   }
 
   template<typename T>
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code, T)>>
+  boost::asio::any_completion_handler<void(boost::system::error_code, T)>
   OpContextVert(Context* c, T* p) {
 
-    if (c || p)
+    if (c || p) {
+      auto e = boost::asio::prefer(
+	service.get_executor(),
+	boost::asio::execution::outstanding_work.tracked);
       return
-	ceph::async::Completion<void(boost::system::error_code, T)>::create(
-	  service.get_executor(),
+	boost::asio::bind_executor(
+	  e,
 	  [c = std::unique_ptr<Context>(c), p]
 	  (boost::system::error_code e, T r) mutable {
 	      if (p)
 		*p = std::move(r);
 	      if (c)
 		c.release()->complete(ceph::from_error_code(e));
-	    });
-    else
+	  });
+    } else {
       return nullptr;
+    }
   }
 
   template<typename T>
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code, T)>>
+  boost::asio::any_completion_handler<void(boost::system::error_code, T)>
   OpContextVert(Context* c, T& p) {
+    if (c) {
+      auto e = boost::asio::prefer(
+	service.get_executor(),
+	boost::asio::execution::outstanding_work.tracked);
+      return boost::asio::bind_executor(
+	e,
+	[c = std::unique_ptr<Context>(c), &p]
+	(boost::system::error_code e, T r) mutable {
+	  p = std::move(r);
+	  if (c)
+	    c.release()->complete(ceph::from_error_code(e));
+	});
+    } else {
+      return nullptr;
+    }
+  }
+
+  boost::asio::any_completion_handler<void(boost::system::error_code)>
+  OpCompletionVert(std::unique_ptr<ceph::async::Completion<
+		     void(boost::system::error_code)>> c) {
     if (c)
-      return ceph::async::Completion<
-	void(boost::system::error_code, T)>::create(
-	  service.get_executor(),
-	  [c = std::unique_ptr<Context>(c), &p]
-	  (boost::system::error_code e, T r) mutable {
-	    p = std::move(r);
-	    if (c)
-	      c.release()->complete(ceph::from_error_code(e));
-	  });
+      return [c = std::move(c)](boost::system::error_code ec) mutable {
+	c->dispatch(std::move(c), ec);
+      };
     else
       return nullptr;
   }
 
+  template<typename T>
+  boost::asio::any_completion_handler<void(boost::system::error_code, T)>
+  OpCompletionVert(std::unique_ptr<ceph::async::Completion<
+		     void(boost::system::error_code, T)>> c) {
+    if (c) {
+      return [c = std::move(c)](boost::system::error_code ec, T t) mutable {
+	c->dispatch(std::move(c), ec, std::move(t));
+      };
+    } else {
+      return nullptr;
+    }
+  }
+
   struct Op : public RefCountedObject {
     OSDSession *session = nullptr;
     int incarnation = 0;
@@ -1923,7 +1962,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
     int priority = 0;
     using OpSig = void(boost::system::error_code);
-    using OpComp = ceph::async::Completion<OpSig>;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
     // Due to an irregularity of cmpxattr, we actualy need the 'int'
     // value for onfinish for legacy librados users. As such just
     // preserve the Context* in this one case. That way we can have
@@ -1933,7 +1972,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     //
     // Add a function for the linger case, where we want better
     // semantics than Context, but still need to be under the completion_lock.
-    std::variant<std::unique_ptr<OpComp>, fu2::unique_function<OpSig>,
+    std::variant<OpComp, fu2::unique_function<OpSig>,
 		 Context*> onfinish;
     uint64_t ontimeout = 0;
 
@@ -1971,8 +2010,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     }
 
     static void complete(decltype(onfinish)&& f, boost::system::error_code ec,
-			 int r) {
-      std::visit([ec, r](auto&& arg) {
+			 int r, boost::asio::io_context::executor_type e) {
+      std::visit([ec, r, e](auto&& arg) {
 		   if constexpr (std::is_same_v<std::decay_t<decltype(arg)>,
 				 Context*>) {
 		     arg->complete(r);
@@ -1980,17 +2019,18 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 			      fu2::unique_function<OpSig>>) {
 		     std::move(arg)(ec);
                    } else {
-		     arg->defer(std::move(arg), ec);
+		     boost::asio::defer(e,
+					boost::asio::append(std::move(arg), ec));
 		   }
 		 }, std::move(f));
     }
-    void complete(boost::system::error_code ec, int r) {
-      complete(std::move(onfinish), ec, r);
+    void complete(boost::system::error_code ec, int r,
+		  boost::asio::io_context::executor_type e) {
+      complete(std::move(onfinish), ec, r, e);
     }
 
     Op(const object_t& o, const object_locator_t& ol,  osdc_opvec&& _ops,
-       int f, std::unique_ptr<OpComp>&& fin,
-       version_t *ov, int *offset = nullptr,
+       int f, OpComp&& fin, version_t *ov, int *offset = nullptr,
        ZTracer::Trace *parent_trace = nullptr) :
       target(o, ol, f),
       ops(std::move(_ops)),
@@ -2172,8 +2212,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     using OpSig = void(boost::system::error_code,
 		       boost::container::flat_map<std::string, pool_stat_t>,
 		       bool);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
+    OpComp onfinish;
     std::uint64_t ontimeout;
     ceph::coarse_mono_time last_submit;
   };
@@ -2183,9 +2223,9 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     std::optional<int64_t> data_pool;
     using OpSig = void(boost::system::error_code,
 		       const struct ceph_statfs);
-    using OpComp = ceph::async::Completion<OpSig>;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
 
-    std::unique_ptr<OpComp> onfinish;
+    OpComp onfinish;
     uint64_t ontimeout;
 
     ceph::coarse_mono_time last_submit;
@@ -2196,8 +2236,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     int64_t pool = 0;
     std::string name;
     using OpSig = void(boost::system::error_code, ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
+    OpComp onfinish;
     uint64_t ontimeout = 0;
     int pool_op = 0;
     int16_t crush_rule = 0;
@@ -2226,8 +2266,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
     using OpSig = void(boost::system::error_code, std::string,
 		       ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
+    OpComp onfinish;
 
     uint64_t ontimeout = 0;
     ceph::coarse_mono_time last_submit;
@@ -2293,9 +2333,9 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     bool registered{false};
     bool canceled{false};
     using OpSig = void(boost::system::error_code, ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> on_reg_commit;
-    std::unique_ptr<OpComp> on_notify_finish;
+    using OpComp = boost::asio::any_completion_handler<OpSig>;
+    OpComp on_reg_commit;
+    OpComp on_notify_finish;
     uint64_t notify_id{0};
 
     fu2::unique_function<void(boost::system::error_code,
@@ -2453,7 +2493,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   std::map<ceph_tid_t, CommandOp*> check_latest_map_commands;
 
   std::map<epoch_t,
-	   std::vector<std::pair<std::unique_ptr<OpCompletion>,
+	   std::vector<std::pair<OpCompletion,
 				 boost::system::error_code>>> waiting_for_map;
 
   ceph::timespan mon_timeout;
@@ -2520,9 +2560,13 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 public:
   template<typename CT>
   auto linger_callback_flush(CT&& ct) {
-    boost::asio::async_completion<CT, void(void)> init(ct);
-    boost::asio::defer(finish_strand, std::move(init.completion_handler));
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CT>(ct), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), void()>(
+      [this](auto handler) {
+	boost::asio::defer(finish_strand, std::move(handler));
+      }, consigned);
   }
 
 private:
@@ -2673,22 +2717,28 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   template<typename CompletionToken>
   auto wait_for_osd_map(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, void()> init(token);
-    std::unique_lock l(rwlock);
-    if (osdmap->get_epoch()) {
-      l.unlock();
-      boost::asio::post(std::move(init.completion_handler));
-    } else {
-      waiting_for_map[0].emplace_back(
-	OpCompletion::create(
-	  service.get_executor(),
-	  [c = std::move(init.completion_handler)]
-	  (boost::system::error_code) mutable {
-	    std::move(c)();
-	  }), boost::system::error_code{});
-      l.unlock();
-    }
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), void()>(
+      [this](auto handler) {
+	std::unique_lock l(rwlock);
+	if (osdmap->get_epoch()) {
+	  l.unlock();
+	  boost::asio::post(std::move(handler));
+	} else {
+	  auto e = boost::asio::get_associated_executor(
+	    handler, service.get_executor());
+	  waiting_for_map[0].emplace_back(
+	    boost::asio::bind_executor(
+	      e, [c = std::move(handler)]
+	      (boost::system::error_code) mutable {
+		boost::asio::dispatch(std::move(c));
+	      }),
+	    boost::system::error_code{});
+	  l.unlock();
+	}
+      }, consigned);
   }
 
 
@@ -2755,9 +2805,9 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   struct CB_Objecter_GetVersion {
     Objecter *objecter;
-    std::unique_ptr<OpCompletion> fin;
+    OpCompletion fin;
 
-    CB_Objecter_GetVersion(Objecter *o, std::unique_ptr<OpCompletion> c)
+    CB_Objecter_GetVersion(Objecter *o, OpCompletion c)
       : objecter(o), fin(std::move(c)) {}
     void operator()(boost::system::error_code ec, version_t newest,
 		    version_t oldest) {
@@ -2765,7 +2815,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 	// try again as instructed
 	objecter->_wait_for_latest_osdmap(std::move(*this));
       } else if (ec) {
-	ceph::async::post(std::move(fin), ec);
+	boost::asio::post(objecter->service.get_executor(),
+			  boost::asio::append(std::move(fin), ec));
       } else {
 	auto l = std::unique_lock(objecter->rwlock);
 	objecter->_get_latest_version(oldest, newest, std::move(fin),
@@ -2776,24 +2827,23 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   template<typename CompletionToken>
   auto wait_for_map(epoch_t epoch, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-
-    if (osdmap->get_epoch() >= epoch) {
-      boost::asio::post(service,
-			ceph::async::bind_handler(
-			  std::move(init.completion_handler),
-			  boost::system::error_code()));
-    } else {
-      monc->get_version("osdmap",
-			CB_Objecter_GetVersion(
-			  this,
-			  OpCompletion::create(service.get_executor(),
-					       std::move(init.completion_handler))));
-    }
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), OpSignature>(
+      [epoch, this](auto handler) {
+	if (osdmap->get_epoch() >= epoch) {
+	  boost::asio::post(boost::asio::append(
+			      std::move(handler),
+			      boost::system::error_code{}));
+	} else {
+	  monc->get_version(
+	    "osdmap",
+	    CB_Objecter_GetVersion(this, std::move(handler)));
+	}
+      }, consigned);
   }
-
-  void _wait_for_new_map(std::unique_ptr<OpCompletion>, epoch_t epoch,
+  void _wait_for_new_map(OpCompletion, epoch_t epoch,
 			 boost::system::error_code = {});
 
 private:
@@ -2805,38 +2855,40 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   template<typename CompletionToken>
   auto wait_for_latest_osdmap(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-
-    monc->get_version("osdmap",
-		      CB_Objecter_GetVersion(
-			this,
-			OpCompletion::create(service.get_executor(),
-					     std::move(init.completion_handler))));
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    boost::asio::async_initiate<decltype(consigned), OpSignature>(
+      [this](auto handler) {
+	monc->get_version("osdmap",
+			  CB_Objecter_GetVersion(
+			    this,
+			    std::move(handler)));
+      }, consigned);
   }
 
-  void wait_for_latest_osdmap(std::unique_ptr<OpCompletion> c) {
-    monc->get_version("osdmap",
-		      CB_Objecter_GetVersion(this, std::move(c)));
+  auto wait_for_latest_osdmap(std::unique_ptr<ceph::async::Completion<OpSignature>> c) {
+    wait_for_latest_osdmap([c = std::move(c)](boost::system::error_code e) mutable {
+      c->dispatch(std::move(c), e);
+    });
   }
 
   template<typename CompletionToken>
   auto get_latest_version(epoch_t oldest, epoch_t newest,
 			  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-    {
-      std::unique_lock wl(rwlock);
-      _get_latest_version(oldest, newest,
-			  OpCompletion::create(
-			    service.get_executor(),
-			    std::move(init.completion_handler)),
-			  std::move(wl));
-    }
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), OpSignature>(
+      [oldest, newest, this](auto handler) {
+	std::unique_lock wl(rwlock);
+	_get_latest_version(oldest, newest,
+			    std::move(handler), std::move(wl));
+      }, consigned);
   }
 
   void _get_latest_version(epoch_t oldest, epoch_t neweset,
-			   std::unique_ptr<OpCompletion> fin,
+			   OpCompletion fin,
 			   std::unique_lock<ceph::shared_mutex>&& ul);
 
   /** Get the current set of global op flags */
@@ -2869,7 +2921,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   epoch_t op_cancel_writes(int r, int64_t pool=-1);
 
   // commands
-  void osd_command(int osd, std::vector<std::string> cmd,
+  void osd_command_(int osd, std::vector<std::string> cmd,
 		   ceph::buffer::list inbl, ceph_tid_t *ptid,
 		   decltype(CommandOp::onfinish)&& onfinish) {
     ceph_assert(osd >= 0);
@@ -2884,17 +2936,20 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   auto osd_command(int osd, std::vector<std::string> cmd,
 		   ceph::buffer::list inbl, ceph_tid_t *ptid,
 		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  CommandOp::OpSig> init(token);
-    osd_command(osd, std::move(cmd), std::move(inbl), ptid,
-		CommandOp::OpComp::create(service.get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  void pg_command(pg_t pgid, std::vector<std::string> cmd,
-		  ceph::buffer::list inbl, ceph_tid_t *ptid,
-		  decltype(CommandOp::onfinish)&& onfinish) {
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), CommandOp::OpSig>(
+      [osd, cmd = std::move(cmd), inbl = std::move(inbl), ptid, this]
+      (auto handler) {
+	osd_command_(osd, std::move(cmd), std::move(inbl), ptid,
+		     std::move(handler));
+      }, consigned);
+  }
+
+  void pg_command_(pg_t pgid, std::vector<std::string> cmd,
+		   ceph::buffer::list inbl, ceph_tid_t *ptid,
+		   decltype(CommandOp::onfinish)&& onfinish) {
     auto *c = new CommandOp(
       pgid,
       std::move(cmd),
@@ -2907,12 +2962,14 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   auto pg_command(pg_t pgid, std::vector<std::string> cmd,
 		  ceph::buffer::list inbl, ceph_tid_t *ptid,
 		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  CommandOp::OpSig> init(token);
-    pg_command(pgid, std::move(cmd), std::move(inbl), ptid,
-	       CommandOp::OpComp::create(service.get_executor(),
-					 std::move(init.completion_handler)));
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(service.get_executor()));
+    return async_initiate<decltype(consigned), CommandOp::OpSig> (
+      [pgid, cmd = std::move(cmd), inbl = std::move(inbl), ptid, this]
+      (auto handler) {
+	pg_command_(pgid, std::move(cmd), std::move(inbl), ptid,
+		    std::move(handler));
+      }, consigned);
   }
 
   // mid-level helpers
@@ -2953,7 +3010,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   void mutate(const object_t& oid, const object_locator_t& oloc,
 	      ObjectOperation&& op, const SnapContext& snapc,
 	      ceph::real_time mtime, int flags,
-	      std::unique_ptr<Op::OpComp>&& oncommit,
+	      Op::OpComp oncommit,
 	      version_t *objver = NULL, osd_reqid_t reqid = osd_reqid_t(),
 	      ZTracer::Trace *parent_trace = nullptr) {
     Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
@@ -2971,6 +3028,18 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     op_submit(o);
   }
 
+  void mutate(const object_t& oid, const object_locator_t& oloc,
+	      ObjectOperation&& op, const SnapContext& snapc,
+	      ceph::real_time mtime, int flags,
+	      std::unique_ptr<ceph::async::Completion<Op::OpSig>> oncommit,
+	      version_t *objver = NULL, osd_reqid_t reqid = osd_reqid_t(),
+	      ZTracer::Trace *parent_trace = nullptr) {
+    mutate(oid, oloc, std::move(op), snapc, mtime, flags,
+	   [c = std::move(oncommit)](boost::system::error_code ec) mutable {
+	     c->dispatch(std::move(c), ec);
+	   }, objver, reqid, parent_trace);
+  }
+
   Op *prepare_read_op(
     const object_t& oid, const object_locator_t& oloc,
     ObjectOperation& op,
@@ -3012,7 +3081,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   void read(const object_t& oid, const object_locator_t& oloc,
 	    ObjectOperation&& op, snapid_t snapid, ceph::buffer::list *pbl,
-	    int flags, std::unique_ptr<Op::OpComp>&& onack,
+	    int flags, Op::OpComp onack,
 	    version_t *objver = nullptr, int *data_offset = nullptr,
 	    uint64_t features = 0, ZTracer::Trace *parent_trace = nullptr) {
     Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
@@ -3035,6 +3104,17 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     op_submit(o);
   }
 
+  void read(const object_t& oid, const object_locator_t& oloc,
+	    ObjectOperation&& op, snapid_t snapid, ceph::buffer::list *pbl,
+	    int flags, std::unique_ptr<ceph::async::Completion<Op::OpSig>> onack,
+	    version_t *objver = nullptr, int *data_offset = nullptr,
+	    uint64_t features = 0, ZTracer::Trace *parent_trace = nullptr) {
+    read(oid, oloc, std::move(op), snapid, pbl, flags,
+	 [c = std::move(onack)](boost::system::error_code e) mutable {
+	   c->dispatch(std::move(c), e);
+	 }, objver, data_offset, features, parent_trace);
+  }
+
 
   Op *prepare_pg_read_op(
     uint32_t hash, object_locator_t oloc,
@@ -3078,7 +3158,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   ceph_tid_t pg_read(
     uint32_t hash, object_locator_t oloc,
     ObjectOperation& op, ceph::buffer::list *pbl, int flags,
-    std::unique_ptr<Op::OpComp>&& onack, epoch_t *reply_epoch, int *ctx_budget) {
+    Op::OpComp onack, epoch_t *reply_epoch, int *ctx_budget) {
     ceph_tid_t tid;
     Op *o = new Op(object_t(), oloc,
 		   std::move(op.ops),
@@ -3122,6 +3202,18 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     return linger_watch(info, op, snapc, mtime, inbl,
 			OpContextVert<ceph::buffer::list>(onfinish, nullptr), objver);
   }
+  ceph_tid_t linger_watch(LingerOp *info,
+			  ObjectOperation& op,
+			  const SnapContext& snapc, ceph::real_time mtime,
+			  ceph::buffer::list& inbl,
+			  std::unique_ptr<ceph::async::Completion<
+			    void(boost::system::error_code,
+			         ceph::buffer::list)>> onfinish,
+			  version_t *objver) {
+    return linger_watch(info, op, snapc, mtime, inbl,
+			OpCompletionVert<ceph::buffer::list>(
+			  std::move(onfinish)), objver);
+  }
   ceph_tid_t linger_notify(LingerOp *info,
 			   ObjectOperation& op,
 			   snapid_t snap, ceph::buffer::list& inbl,
@@ -3137,6 +3229,17 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 			 OpContextVert(onack, poutbl),
 			 objver);
   }
+  ceph_tid_t linger_notify(LingerOp *info,
+			   ObjectOperation& op,
+			   snapid_t snap, ceph::buffer::list& inbl,
+			   std::unique_ptr<ceph::async::Completion<
+			     void(boost::system::error_code,
+			          ceph::buffer::list)>> onack,
+			   version_t *objver) {
+    return linger_notify(info, op, snap, inbl,
+			 OpCompletionVert<ceph::buffer::list>(
+			   std::move(onack)), objver);
+  }
   tl::expected<ceph::timespan,
 	       boost::system::error_code> linger_check(LingerOp *info);
   void linger_cancel(LingerOp *info);  // releases a reference
@@ -3717,15 +3820,27 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     create_pool_snap(pool, snapName,
 		     OpContextVert<ceph::buffer::list>(c, nullptr));
   }
+  void create_pool_snap(
+    int64_t pool, std::string_view snapName,
+    std::unique_ptr<ceph::async::Completion<PoolOp::OpSig>> c) {
+    create_pool_snap(pool, snapName,
+		     OpCompletionVert<ceph::buffer::list>(std::move(c)));
+  }
   void allocate_selfmanaged_snap(int64_t pool,
-				 std::unique_ptr<ceph::async::Completion<
+				 boost::asio::any_completion_handler<
 				 void(boost::system::error_code,
-				      snapid_t)>> onfinish);
+				      snapid_t)> onfinish);
   void allocate_selfmanaged_snap(int64_t pool, snapid_t* psnapid,
 				 Context* c) {
     allocate_selfmanaged_snap(pool,
 			      OpContextVert(c, psnapid));
   }
+  void allocate_selfmanaged_snap(int64_t pool,
+				 std::unique_ptr<ceph::async::Completion<void(
+				   boost::system::error_code, snapid_t)>> c) {
+    allocate_selfmanaged_snap(pool,
+			      OpCompletionVert<snapid_t>(std::move(c)));
+  }
   void delete_pool_snap(int64_t pool, std::string_view snapName,
 			decltype(PoolOp::onfinish)&& onfinish);
   void delete_pool_snap(int64_t pool, std::string_view snapName,
@@ -3733,6 +3848,12 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     delete_pool_snap(pool, snapName,
 		     OpContextVert<ceph::buffer::list>(c, nullptr));
   }
+  void delete_pool_snap(int64_t pool, std::string_view snapName,
+			std::unique_ptr<ceph::async::Completion<void(
+                          boost::system::error_code, ceph::buffer::list)>> c) {
+    delete_pool_snap(pool, snapName,
+		     OpCompletionVert<ceph::buffer::list>(std::move(c)));
+  }
 
   void delete_selfmanaged_snap(int64_t pool, snapid_t snap,
 			       decltype(PoolOp::onfinish)&& onfinish);
@@ -3741,6 +3862,12 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     delete_selfmanaged_snap(pool, snap,
 			    OpContextVert<ceph::buffer::list>(c, nullptr));
   }
+  void delete_selfmanaged_snap(int64_t pool, snapid_t snap,
+			       std::unique_ptr<ceph::async::Completion<void(
+                                 boost::system::error_code, ceph::buffer::list)>> c) {
+    delete_selfmanaged_snap(pool, snap,
+			    OpCompletionVert<ceph::buffer::list>(std::move(c)));
+  }
 
 
   void create_pool(std::string_view name,
@@ -3752,12 +3879,25 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 		OpContextVert<ceph::buffer::list>(onfinish, nullptr),
 		crush_rule);
   }
+  void create_pool(std::string_view name,
+		   std::unique_ptr<ceph::async::Completion<void(
+                     boost::system::error_code, ceph::buffer::list)>> c,
+		   int crush_rule=-1) {
+    create_pool(name,
+		OpCompletionVert<ceph::buffer::list>(std::move(c)),
+		crush_rule);
+  }
   void delete_pool(int64_t pool,
 		   decltype(PoolOp::onfinish)&& onfinish);
   void delete_pool(int64_t pool,
 		   Context* onfinish) {
     delete_pool(pool, OpContextVert<ceph::buffer::list>(onfinish, nullptr));
   }
+  void delete_pool(int64_t pool,
+		   std::unique_ptr<ceph::async::Completion<void(
+                    boost::system::error_code, ceph::buffer::list)>> c) {
+    delete_pool(pool, OpCompletionVert<ceph::buffer::list>(std::move(c)));
+  }
 
   void delete_pool(std::string_view name,
 		   decltype(PoolOp::onfinish)&& onfinish);
@@ -3766,6 +3906,11 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 		   Context* onfinish) {
     delete_pool(name, OpContextVert<ceph::buffer::list>(onfinish, nullptr));
   }
+  void delete_pool(std::string_view name,
+		   std::unique_ptr<ceph::async::Completion<void(
+                     boost::system::error_code, ceph::buffer::list)>> c) {
+    delete_pool(name, OpCompletionVert<ceph::buffer::list>(std::move(c)));
+  }
 
   void handle_pool_op_reply(MPoolOpReply *m);
   int pool_op_cancel(ceph_tid_t tid, int r);
@@ -3776,18 +3921,18 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   void _poolstat_submit(PoolStatOp *op);
 public:
   void handle_get_pool_stats_reply(MGetPoolStatsReply *m);
-  void get_pool_stats(const std::vector<std::string>& pools,
-		      decltype(PoolStatOp::onfinish)&& onfinish);
+  void get_pool_stats_(const std::vector<std::string>& pools,
+		       decltype(PoolStatOp::onfinish)&& onfinish);
   template<typename CompletionToken>
-  auto get_pool_stats(const std::vector<std::string>& pools,
+  auto get_pool_stats(std::vector<std::string> pools,
 		      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  PoolStatOp::OpSig> init(token);
-    get_pool_stats(pools,
-		   PoolStatOp::OpComp::create(
-		     service.get_executor(),
-		     std::move(init.completion_handler)));
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), PoolStatOp::OpSig>(
+      [pools = std::move(pools), this](auto handler) {
+	get_pool_stats_(pools, std::move(handler));
+      }, consigned);
   }
   int pool_stat_op_cancel(ceph_tid_t tid, int r);
   void _finish_pool_stat_op(PoolStatOp *op, int r);
@@ -3798,20 +3943,27 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   void _fs_stats_submit(StatfsOp *op);
 public:
   void handle_fs_stats_reply(MStatfsReply *m);
-  void get_fs_stats(std::optional<int64_t> poolid,
-		    decltype(StatfsOp::onfinish)&& onfinish);
+  void get_fs_stats_(std::optional<int64_t> poolid,
+		     decltype(StatfsOp::onfinish)&& onfinish);
   template<typename CompletionToken>
   auto get_fs_stats(std::optional<int64_t> poolid,
 		    CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, StatfsOp::OpSig> init(token);
-    get_fs_stats(poolid,
-		 StatfsOp::OpComp::create(service.get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
+    auto consigned = boost::asio::consign(
+      std::forward<CompletionToken>(token), boost::asio::make_work_guard(
+	service.get_executor()));
+    return boost::asio::async_initiate<decltype(consigned), StatfsOp::OpSig>(
+      [poolid, this](auto handler) {
+	get_fs_stats_(poolid, std::move(handler));
+      }, consigned);
   }
   void get_fs_stats(struct ceph_statfs& result, std::optional<int64_t> poolid,
 		    Context *onfinish) {
-    get_fs_stats(poolid, OpContextVert(onfinish, result));
+    get_fs_stats_(poolid, OpContextVert(onfinish, result));
+  }
+  void get_fs_stats(std::optional<int64_t> poolid,
+		    std::unique_ptr<ceph::async::Completion<void(
+                      boost::system::error_code, struct ceph_statfs)>> c) {
+    get_fs_stats_(poolid, OpCompletionVert<struct ceph_statfs>(std::move(c)));
   }
   int statfs_op_cancel(ceph_tid_t tid, int r);
   void _finish_statfs_op(StatfsOp *op, int r);
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index 4b6866669fb1..e6a87c1617b5 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -14,14 +14,20 @@
 #include "test/librados_test_stub/TestClassHandler.h"
 #include "test/librados_test_stub/TestIoCtxImpl.h"
 #include "test/librados_test_stub/TestRadosClient.h"
+
 #include <map>
 #include <memory>
 #include <optional>
 #include <string>
 #include <functional>
+
+#include <boost/asio/append.hpp>
+#include <boost/asio/dispatch.hpp>
+#include <boost/asio/post.hpp>
 #include <boost/system/system_error.hpp>
 
 namespace bs = boost::system;
+namespace asio = boost::asio;
 using namespace std::literals;
 using namespace std::placeholders;
 
@@ -82,7 +88,7 @@ class Client {
 namespace {
 
 struct CompletionPayload {
-  std::unique_ptr<Op::Completion> c;
+  Op::Completion c;
 };
 
 void completion_callback_adapter(rados_completion_t c, void *arg) {
@@ -91,14 +97,14 @@ void completion_callback_adapter(rados_completion_t c, void *arg) {
   impl->release();
 
   auto payload = reinterpret_cast<CompletionPayload*>(arg);
-  payload->c->defer(std::move(payload->c),
-                    (r < 0) ? bs::error_code(-r, osd_category()) :
-                              bs::error_code());
+  asio::dispatch(asio::append(std::move(payload->c),
+			      (r < 0) ? bs::error_code(-r, osd_category()) :
+			      bs::error_code()));
   delete payload;
 }
 
 librados::AioCompletionImpl* create_aio_completion(
-    std::unique_ptr<Op::Completion>&& c) {
+  Op::Completion&& c) {
   auto payload = new CompletionPayload{std::move(c)};
 
   auto impl = new librados::AioCompletionImpl();
@@ -557,12 +563,12 @@ boost::asio::io_context::executor_type neorados::RADOS::get_executor() const {
   return impl->io_context.get_executor();
 }
 
-void RADOS::execute(Object o, IOContext ioc, ReadOp op,
-                    ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
-                    uint64_t* objver, const blkin_trace_info* trace_info) {
+void RADOS::execute_(Object o, IOContext ioc, ReadOp op,
+		     ceph::buffer::list* bl, Op::Completion c,
+		     uint64_t* objver, const blkin_trace_info* trace_info) {
   auto io_ctx = impl->get_io_ctx(ioc);
   if (io_ctx == nullptr) {
-    c->dispatch(std::move(c), osdc_errc::pool_dne);
+    asio::dispatch(asio::append(std::move(c), osdc_errc::pool_dne));
     return;
   }
 
@@ -576,12 +582,12 @@ void RADOS::execute(Object o, IOContext ioc, ReadOp op,
   ceph_assert(r == 0);
 }
 
-void RADOS::execute(Object o, IOContext ioc, WriteOp op,
-                    std::unique_ptr<Op::Completion> c, uint64_t* objver,
-                    const blkin_trace_info* trace_info) {
+void RADOS::execute_(Object o, IOContext ioc, WriteOp op,
+		     Op::Completion c, uint64_t* objver,
+		     const blkin_trace_info* trace_info) {
   auto io_ctx = impl->get_io_ctx(ioc);
   if (io_ctx == nullptr) {
-    c->dispatch(std::move(c), osdc_errc::pool_dne);
+    asio::dispatch(asio::append(std::move(c), osdc_errc::pool_dne));
     return;
   }
 
@@ -599,29 +605,33 @@ void RADOS::execute(Object o, IOContext ioc, WriteOp op,
   ceph_assert(r == 0);
 }
 
-void RADOS::mon_command(std::vector<std::string> command,
-                        bufferlist bl,
-                        std::string* outs, bufferlist* outbl,
-                        std::unique_ptr<Op::Completion> c) {
+void RADOS::mon_command_(std::vector<std::string> command,
+			 bufferlist bl,
+			 std::string* outs, bufferlist* outbl,
+			 Op::Completion c) {
   auto r = impl->test_rados_client->mon_command(command, bl, outbl, outs);
-  c->post(std::move(c),
-          (r < 0 ? bs::error_code(-r, osd_category()) : bs::error_code()));
+  asio::post(get_executor(),
+	     asio::append(std::move(c),
+			  (r < 0 ? bs::error_code(-r, osd_category()) :
+			   bs::error_code())));
 }
 
-void RADOS::blocklist_add(std::string client_address,
-                          std::optional<std::chrono::seconds> expire,
-                          std::unique_ptr<SimpleOpComp> c) {
+void RADOS::blocklist_add_(std::string client_address,
+			   std::optional<std::chrono::seconds> expire,
+			   SimpleOpComp c) {
   auto r = impl->test_rados_client->blocklist_add(
     std::string(client_address), expire.value_or(0s).count());
-  c->post(std::move(c),
-          (r < 0 ? bs::error_code(-r, mon_category()) : bs::error_code()));
+  asio::post(get_executor(),
+	     asio::append(std::move(c),
+			  (r < 0 ? bs::error_code(-r, mon_category()) :
+			   bs::error_code())));
 }
 
-void RADOS::wait_for_latest_osd_map(std::unique_ptr<Op::Completion> c) {
+void RADOS::wait_for_latest_osd_map_(Op::Completion c) {
   auto r = impl->test_rados_client->wait_for_latest_osd_map();
-  c->dispatch(std::move(c),
-              (r < 0 ? bs::error_code(-r, osd_category()) :
-                       bs::error_code()));
+  asio::dispatch(asio::append(std::move(c),
+			      (r < 0 ? bs::error_code(-r, osd_category()) :
+			       bs::error_code())));
 }
 
 } // namespace neorados

From 129330058faa76aab929e92efd3478c71bbadf7d Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Mon, 20 Mar 2023 12:20:38 -0400
Subject: [PATCH 0951/2492] neorados: Op builder functions should chain

Make every operation function return the (Read/Write)Op, for chaining.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                | 881 ++++++++++++++++--
 src/neorados/RADOS.cc                         | 135 +--
 .../librados_test_stub/NeoradosTestStub.cc    |  49 +-
 3 files changed, 931 insertions(+), 134 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 9bbe10b6670f..67a01b43148e 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -16,6 +16,7 @@
 #ifndef NEORADOS_RADOS_HPP
 #define NEORADOS_RADOS_HPP
 
+#include <concepts>
 #include <cstddef>
 #include <memory>
 #include <tuple>
@@ -217,6 +218,71 @@ enum alloc_hint_t {
 };
 }
 
+class Op;
+class ReadOp;
+class WriteOp;
+
+template<std::invocable<Op&> F>
+class ClsOp {
+  F f;
+public:
+  ClsOp(F&& f) : f(std::move(f)) {}
+
+  ReadOp& operator()(ReadOp& op) {
+    std::move(f)(op);
+    return op;
+  }
+
+  ReadOp&& operator()(ReadOp&& op) {
+    std::move(f)(op);
+    return std::move(op);
+  }
+
+  WriteOp& operator()(WriteOp& op) {
+    std::move(f)(op);
+    return op;
+  }
+
+  WriteOp&& operator()(WriteOp&& op) {
+    std::move(f)(op);
+    return std::move(op);
+  }
+};
+
+template<std::invocable<ReadOp&> F>
+class ClsReadOp {
+  F f;
+public:
+  ClsReadOp(F&& f) : f(std::move(f)) {}
+
+  ReadOp& operator()(ReadOp& op) {
+    std::move(f)(op);
+    return op;
+  }
+
+  ReadOp&& operator()(ReadOp&& op) {
+    std::move(f)(op);
+    return std::move(op);
+  }
+};
+
+template<std::invocable<WriteOp&> F>
+class ClsWriteOp {
+  F f;
+public:
+  ClsWriteOp(F&& f) : f(std::move(f)) {}
+
+  WriteOp& operator()(WriteOp& op) {
+    std::move(f)(op);
+    return op;
+  }
+
+  WriteOp&& operator()(WriteOp&& op) {
+    std::move(f)(op);
+    return std::move(op);
+  }
+};
+
 class Op {
   friend RADOS;
 
@@ -242,9 +308,9 @@ class Op {
   void cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val);
   void assert_version(uint64_t ver);
   void assert_exists();
-  void cmp_omap(const boost::container::flat_map<
-		  std::string,
-		  std::pair<ceph::buffer::list, int>>& assertions);
+  void cmp_omap(const boost::container::flat_map<std::string,
+		                                 std::pair<ceph::buffer::list,
+		                                           int>>& assertions);
 
   void exec(std::string_view cls, std::string_view method,
 	    const ceph::buffer::list& inbl,
@@ -302,51 +368,398 @@ class ReadOp final : public Op {
   ReadOp& operator =(const ReadOp&) = delete;
   ReadOp& operator =(ReadOp&&) = default;
 
-  void read(size_t off, uint64_t len, ceph::buffer::list* out,
-	    boost::system::error_code* ec = nullptr);
-  void get_xattr(std::string_view name, ceph::buffer::list* out,
-		 boost::system::error_code* ec = nullptr);
-  void get_omap_header(ceph::buffer::list*,
-		       boost::system::error_code* ec = nullptr);
+  ReadOp& read(size_t off, uint64_t len, ceph::buffer::list* out,
+	       boost::system::error_code* ec = nullptr) &;
+  ReadOp&& read(size_t off, uint64_t len, ceph::buffer::list* out,
+		boost::system::error_code* ec = nullptr) && {
+    return std::move(read(off, len, out, ec));
+  }
+  ReadOp& get_xattr(std::string_view name, ceph::buffer::list* out,
+		    boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_xattr(std::string_view name, ceph::buffer::list* out,
+		     boost::system::error_code* ec = nullptr) && {
+    return std::move(get_xattr(name, out, ec));
+  }
+  ReadOp& get_omap_header(ceph::buffer::list* bl,
+			  boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_omap_header(ceph::buffer::list* bl,
+			   boost::system::error_code* ec = nullptr) && {
+    return std::move(get_omap_header(bl, ec));
+  }
+  ReadOp& sparse_read(uint64_t off, uint64_t len,
+		      ceph::buffer::list* out,
+		      std::vector<std::pair<std::uint64_t,
+		                            std::uint64_t>>* extents,
+		      boost::system::error_code* ec = nullptr) &;
+  ReadOp&& sparse_read(uint64_t off, uint64_t len,
+		       ceph::buffer::list* out,
+		       std::vector<std::pair<std::uint64_t,
+		                             std::uint64_t>>* extents,
+		       boost::system::error_code* ec = nullptr) && {
+    return std::move(sparse_read(off, len, out, extents, ec));
+  }
 
-  void sparse_read(uint64_t off, uint64_t len,
-		   ceph::buffer::list* out,
-		   std::vector<std::pair<std::uint64_t, std::uint64_t>>* extents,
-		   boost::system::error_code* ec = nullptr);
+  ReadOp& stat(std::uint64_t* size, ceph::real_time* mtime,
+	       boost::system::error_code* ec = nullptr) &;
+  ReadOp&& stat(std::uint64_t* size, ceph::real_time* mtime,
+		boost::system::error_code* ec = nullptr) && {
+    return std::move(stat(size, mtime, ec));
+  }
 
-  void stat(std::uint64_t* size, ceph::real_time* mtime,
-	    boost::system::error_code* ec = nullptr);
+  ReadOp& get_omap_keys(std::optional<std::string_view> start_after,
+			std::uint64_t max_return,
+			boost::container::flat_set<std::string>* keys,
+			bool* truncated,
+			boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_omap_keys(std::optional<std::string_view> start_after,
+			 std::uint64_t max_return,
+			 boost::container::flat_set<std::string>* keys,
+			 bool* truncated,
+			 boost::system::error_code* ec = nullptr) && {
+    return std::move(get_omap_keys(start_after, max_return, keys, truncated, ec));
+  }
 
-  void get_omap_keys(std::optional<std::string_view> start_after,
-		     std::uint64_t max_return,
-		     boost::container::flat_set<std::string>* keys,
-		     bool* truncated,
-		     boost::system::error_code* ec = nullptr);
 
+  ReadOp& get_xattrs(boost::container::flat_map<std::string,
+		                               ceph::buffer::list>* kv,
+		     boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_xattrs(boost::container::flat_map<std::string,
+		                                ceph::buffer::list>* kv,
+		      boost::system::error_code* ec = nullptr) && {
+    return std::move(get_xattrs(kv, ec));
+  }
 
-  void get_xattrs(boost::container::flat_map<std::string,
-		                             ceph::buffer::list>* kv,
-		     boost::system::error_code* ec = nullptr);
+  ReadOp& get_omap_vals(std::optional<std::string_view> start_after,
+			std::optional<std::string_view> filter_prefix,
+			uint64_t max_return,
+			boost::container::flat_map<std::string,
+		                                   ceph::buffer::list>* kv,
+			bool* truncated,
+			boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_omap_vals(std::optional<std::string_view> start_after,
+			 std::optional<std::string_view> filter_prefix,
+			 uint64_t max_return,
+			 boost::container::flat_map<std::string,
+		                                    ceph::buffer::list>* kv,
+			 bool* truncated,
+			 boost::system::error_code* ec = nullptr) && {
+    return std::move(get_omap_vals(start_after, filter_prefix, max_return, kv,
+				   truncated, ec));
+  }
 
-  void get_omap_vals(std::optional<std::string_view> start_after,
-		     std::optional<std::string_view> filter_prefix,
-		     uint64_t max_return,
-		     boost::container::flat_map<std::string,
-		                                ceph::buffer::list>* kv,
-		     bool* truncated,
-		     boost::system::error_code* ec = nullptr);
+  ReadOp& get_omap_vals_by_keys(
+    const boost::container::flat_set<std::string>& keys,
+    boost::container::flat_map<std::string, ceph::buffer::list>* kv,
+    boost::system::error_code* ec = nullptr) &;
+  ReadOp&& get_omap_vals_by_keys(
+    const boost::container::flat_set<std::string>& keys,
+    boost::container::flat_map<std::string, ceph::buffer::list>* kv,
+    boost::system::error_code* ec = nullptr) && {
+    return std::move(get_omap_vals_by_keys(keys, kv, ec));
+  }
 
+  ReadOp& list_watchers(std::vector<struct ObjWatcher>* watchers,
+			boost::system::error_code* ec = nullptr) &;
+  ReadOp&& list_watchers(std::vector<struct ObjWatcher>* watchers,
+			 boost::system::error_code* ec = nullptr) && {
+    return std::move(list_watchers(watchers, ec));
+  }
 
-  void get_omap_vals_by_keys(const boost::container::flat_set<std::string>& keys,
-			     boost::container::flat_map<std::string,
-			                                ceph::buffer::list>* kv,
-			     boost::system::error_code* ec = nullptr);
+  ReadOp& list_snaps(struct SnapSet* snaps,
+		     boost::system::error_code* ec = nullptr) &;
+  ReadOp&& list_snaps(struct SnapSet* snaps,
+		      boost::system::error_code* ec = nullptr) && {
+    return std::move(list_snaps(snaps, ec));
+  }
 
-  void list_watchers(std::vector<struct ObjWatcher>* watchers,
-		     boost::system::error_code* ec = nullptr);
+  // Chaining versions of functions from Op
+  ReadOp& set_excl() & {
+    Op::set_excl();
+    return *this;
+  }
+  ReadOp&& set_excl() && {
+    Op::set_excl();
+    return std::move(*this);
+  }
+
+  ReadOp& set_failok() & {
+    Op::set_failok();
+    return *this;
+  }
+  ReadOp&& set_failok() && {
+    Op::set_failok();
+    return std::move(*this);
+  }
+
+  ReadOp& set_fadvise_random() & {
+    Op::set_fadvise_random();
+    return *this;
+  }
+  ReadOp&& set_fadvise_random() && {
+    Op::set_fadvise_random();
+    return std::move(*this);
+  }
+
+  ReadOp& set_fadvise_sequential() & {
+    Op::set_fadvise_sequential();
+    return *this;
+  }
+  ReadOp&& set_fadvise_sequential() && {
+    Op::set_fadvise_sequential();
+    return std::move(*this);
+  }
+
+  ReadOp& set_fadvise_willneed() & {
+    Op::set_fadvise_willneed();
+    return *this;
+  }
+  ReadOp&& set_fadvise_willneed() && {
+    Op::set_fadvise_willneed();
+    return std::move(*this);
+  }
 
-  void list_snaps(struct SnapSet* snaps,
-		  boost::system::error_code* ec = nullptr);
+  ReadOp& set_fadvise_dontneed() & {
+    Op::set_fadvise_dontneed();
+    return *this;
+  }
+  ReadOp&& set_fadvise_dontneed() && {
+    Op::set_fadvise_dontneed();
+    return std::move(*this);
+  }
+
+  ReadOp& set_fadvise_nocache() & {
+    Op::set_fadvise_nocache();
+    return *this;
+  }
+  ReadOp&& set_fadvise_nocache() && {
+    Op::set_fadvise_nocache();
+    return std::move(*this);
+  }
+
+  ReadOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) & {
+    Op::cmpext(off, std::move(cmp_bl), s);
+    return *this;
+  }
+  ReadOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) && {
+    Op::cmpext(off, std::move(cmp_bl), s);
+    return std::move(*this);
+  }
+
+  ReadOp& cmpxattr(std::string_view name, cmpxattr_op op,
+		   const ceph::buffer::list& val) & {
+    Op::cmpxattr(name, op, val);
+    return *this;
+  }
+  ReadOp&& cmpxattr(std::string_view name, cmpxattr_op op,
+		    const ceph::buffer::list& val) && {
+    Op::cmpxattr(name, op, val);
+    return std::move(*this);
+  }
+
+  ReadOp& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) & {
+    Op::cmpxattr(name, op, val);
+    return *this;
+  }
+  ReadOp&& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) && {
+    Op::cmpxattr(name, op, val);
+    return std::move(*this);
+  }
+
+  ReadOp& assert_version(uint64_t ver) & {
+    Op::assert_version(ver);
+    return *this;
+  }
+  ReadOp&& assert_version(uint64_t ver) && {
+    Op::assert_version(ver);
+    return std::move(*this);
+  }
+
+  ReadOp& assert_exists() & {
+    Op::assert_exists();
+    return *this;
+  }
+  ReadOp&& assert_exists() && {
+    Op::assert_exists();
+    return std::move(*this);
+  }
+
+  ReadOp& cmp_omap(
+    const boost::container::flat_map<
+      std::string, std::pair<ceph::buffer::list, int>>& assertions) & {
+    Op::cmp_omap(assertions);
+    return *this;
+  }
+  ReadOp&& cmp_omap(
+    const boost::container::flat_map<
+      std::string, std::pair<ceph::buffer::list, int>>& assertions) && {
+    Op::cmp_omap(assertions);
+    return std::move(*this);
+  }
+
+  ReadOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       ceph::buffer::list* out,
+	       boost::system::error_code* ec = nullptr) & {
+    Op::exec(cls, method, inbl, out, ec);
+    return *this;
+  }
+  ReadOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+		ceph::buffer::list* out,
+		boost::system::error_code* ec = nullptr) && {
+    Op::exec(cls, method, inbl, out, ec);
+    return std::move(*this);
+  }
+
+  ReadOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       fu2::unique_function<void(boost::system::error_code,
+	                            const ceph::buffer::list&) &&> f) & {
+    Op::exec(cls, method, inbl, std::move(f));
+    return *this;
+  }
+  ReadOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+	        fu2::unique_function<void(boost::system::error_code,
+	                             const ceph::buffer::list&) &&> f) && {
+    Op::exec(cls, method, inbl, std::move(f));
+    return std::move(*this);
+  }
+
+  ReadOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       fu2::unique_function<void(boost::system::error_code, int,
+	                                 const ceph::buffer::list&) &&> f) & {
+    Op::exec(cls, method, inbl, std::move(f));
+    return *this;
+  }
+  ReadOp&& exec(std::string_view cls, std::string_view method,
+	        const ceph::buffer::list& inbl,
+	        fu2::unique_function<void(boost::system::error_code, int,
+	                                  const ceph::buffer::list&) &&> f) && {
+    Op::exec(cls, method, inbl, std::move(f));
+    return std::move(*this);
+  }
+
+  ReadOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       boost::system::error_code* ec = nullptr) & {
+    Op::exec(cls, method, inbl, ec);
+    return *this;
+  }
+  ReadOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+		boost::system::error_code* ec = nullptr) && {
+    Op::exec(cls, method, inbl, ec);
+    return std::move(*this);
+  }
+
+  template<typename F>
+  ReadOp& exec(ClsOp<F>&& clsop) & {
+    return clsop(*this);
+  }
+  template<typename F>
+  ReadOp&& exec(ClsOp<F>&& clsop) && {
+    return std::move(clsop(*this));
+  }
+  template<typename F>
+  ReadOp& exec(ClsReadOp<F>&& clsop) & {
+    return clsop(*this);
+  }
+  template<typename F>
+  ReadOp&& exec(ClsReadOp<F>&& clsop) && {
+    return std::move(clsop(*this));
+  }
+
+  // Flags that apply to all ops in the operation vector
+  ReadOp& balance_reads() & {
+    Op::balance_reads();
+    return *this;
+  }
+  ReadOp&& balance_reads() && {
+    Op::balance_reads();
+    return std::move(*this);
+  }
+  ReadOp& localize_reads() & {
+    Op::localize_reads();
+    return *this;
+  }
+  ReadOp&& localize_reads() && {
+    Op::localize_reads();
+    return std::move(*this);
+  }
+  ReadOp& order_reads_writes() & {
+    Op::order_reads_writes();
+    return *this;
+  }
+  ReadOp&& order_reads_writes() && {
+    Op::order_reads_writes();
+    return std::move(*this);
+  }
+  ReadOp& ignore_cache() & {
+    Op::ignore_cache();
+    return *this;
+  }
+  ReadOp&& ignore_cache() && {
+    Op::ignore_cache();
+    return std::move(*this);
+  }
+  ReadOp& skiprwlocks() & {
+    Op::skiprwlocks();
+    return *this;
+  }
+  ReadOp&& skiprwlocks() && {
+    Op::skiprwlocks();
+    return std::move(*this);
+  }
+  ReadOp& ignore_overlay() & {
+    Op::ignore_overlay();
+    return *this;
+  }
+  ReadOp&& ignore_overlay() && {
+    Op::ignore_overlay();
+    return std::move(*this);
+  }
+  ReadOp& full_try() & {
+    Op::full_try();
+    return *this;
+  }
+  ReadOp&& full_try() && {
+    Op::full_try();
+    return std::move(*this);
+  }
+  ReadOp& full_force() & {
+    Op::full_force();
+    return *this;
+  }
+  ReadOp&& full_force() && {
+    Op::full_force();
+    return std::move(*this);
+  }
+  ReadOp& ignore_redirect() & {
+    Op::ignore_redirect();
+    return *this;
+  }
+  ReadOp&& ignore_redirect() && {
+    Op::ignore_redirect();
+    return std::move(*this);
+  }
+  ReadOp& ordersnap() & {
+    Op::ordersnap();
+    return *this;
+  }
+  ReadOp&& ordersnap() && {
+    Op::ordersnap();
+    return std::move(*this);
+  }
+  ReadOp& returnvec() & {
+    Op::returnvec();
+    return *this;
+  }
+  ReadOp&& returnvec() && {
+    Op::returnvec();
+    return std::move(*this);
+  }
 };
 
 class WriteOp final : public Op {
@@ -360,28 +773,376 @@ class WriteOp final : public Op {
   WriteOp& operator =(const WriteOp&) = delete;
   WriteOp& operator =(WriteOp&&) = default;
 
-  void set_mtime(ceph::real_time t);
-  void create(bool exclusive);
-  void write(uint64_t off, ceph::buffer::list&& bl);
-  void write_full(ceph::buffer::list&& bl);
-  void writesame(std::uint64_t off, std::uint64_t write_len,
-		 ceph::buffer::list&& bl);
-  void append(ceph::buffer::list&& bl);
-  void remove();
-  void truncate(uint64_t off);
-  void zero(uint64_t off, uint64_t len);
-  void rmxattr(std::string_view name);
-  void setxattr(std::string_view name,
-		ceph::buffer::list&& bl);
-  void rollback(uint64_t snapid);
-  void set_omap(const boost::container::flat_map<std::string,
-		                                 ceph::buffer::list>& map);
-  void set_omap_header(ceph::buffer::list&& bl);
-  void clear_omap();
-  void rm_omap_keys(const boost::container::flat_set<std::string>& to_rm);
-  void set_alloc_hint(uint64_t expected_object_size,
-		      uint64_t expected_write_size,
-		      alloc_hint::alloc_hint_t flags);
+  WriteOp& set_mtime(ceph::real_time t) &;
+  WriteOp&& set_mtime(ceph::real_time t) && {
+    return std::move(set_mtime(t));
+  }
+  WriteOp& create(bool exclusive) &;
+  WriteOp&& create(bool exclusive) && {
+    return std::move(create(exclusive));
+  }
+  WriteOp& write(uint64_t off, ceph::buffer::list&& bl) &;
+  WriteOp&& write(uint64_t off, ceph::buffer::list&& bl) && {
+    return std::move(write(off, std::move(bl)));
+  }
+  WriteOp& write_full(ceph::buffer::list&& bl) &;
+  WriteOp&& write_full(ceph::buffer::list&& bl) && {
+    return std::move(write_full(std::move(bl)));
+  }
+  WriteOp& writesame(std::uint64_t off, std::uint64_t write_len,
+		     ceph::buffer::list&& bl) &;
+  WriteOp&& writesame(std::uint64_t off, std::uint64_t write_len,
+		      ceph::buffer::list&& bl) && {
+    return std::move(writesame(off, write_len, std::move(bl)));
+  }
+  WriteOp& append(ceph::buffer::list&& bl) &;
+  WriteOp&& append(ceph::buffer::list&& bl) && {
+    return std::move(append(std::move(bl)));
+  }
+  WriteOp& remove() &;
+  WriteOp&& remove() && {
+    return std::move(remove());
+  }
+  WriteOp& truncate(uint64_t off) &;
+  WriteOp&& truncate(uint64_t off) && {
+    return std::move(truncate(off));
+  }
+  WriteOp& zero(uint64_t off, uint64_t len) &;
+  WriteOp&& zero(uint64_t off, uint64_t len) && {
+    return std::move(zero(off, len));
+  }
+  WriteOp& rmxattr(std::string_view name) &;
+  WriteOp&& rmxattr(std::string_view name) && {
+    return std::move(rmxattr(name));
+  }
+  WriteOp& setxattr(std::string_view name,
+		    ceph::buffer::list&& bl) &;
+  WriteOp&& setxattr(std::string_view name,
+		     ceph::buffer::list&& bl) && {
+    return std::move(setxattr(name, std::move(bl)));
+  }
+  WriteOp& rollback(uint64_t snapid) &;
+  WriteOp&& rollback(uint64_t snapid) && {
+    return std::move(rollback(snapid));
+  }
+  WriteOp& set_omap(
+    const boost::container::flat_map<std::string, ceph::buffer::list>& map) &;
+  WriteOp&& set_omap(
+    const boost::container::flat_map<std::string, ceph::buffer::list>& map) && {
+    return std::move(set_omap(map));
+  }
+  WriteOp& set_omap_header(ceph::buffer::list&& bl) &;
+  WriteOp&& set_omap_header(ceph::buffer::list&& bl) && {
+    return std::move(set_omap_header(std::move(bl)));
+  }
+  WriteOp& clear_omap() &;
+  WriteOp&& clear_omap() && {
+    return std::move(clear_omap());
+  }
+  WriteOp& rm_omap_keys(const boost::container::flat_set<std::string>& to_rm) &;
+  WriteOp&& rm_omap_keys(const boost::container::flat_set<std::string>& to_rm) && {
+    return std::move(rm_omap_keys(to_rm));
+  }
+  WriteOp& set_alloc_hint(uint64_t expected_object_size,
+			  uint64_t expected_write_size,
+			  alloc_hint::alloc_hint_t flags) &;
+  WriteOp&& set_alloc_hint(uint64_t expected_object_size,
+			   uint64_t expected_write_size,
+			   alloc_hint::alloc_hint_t flags) && {
+    return std::move(set_alloc_hint(expected_object_size,
+				    expected_write_size,
+				    flags));
+  }
+
+  // Chaining versions of functions from Op
+  WriteOp& set_excl() & {
+    Op::set_excl();
+    return *this;
+  }
+  WriteOp&& set_excl() && {
+    Op::set_excl();
+    return std::move(*this);
+  }
+
+  WriteOp& set_failok() & {
+    Op::set_failok();
+    return *this;
+  }
+  WriteOp&& set_failok() && {
+    Op::set_failok();
+    return std::move(*this);
+  }
+
+  WriteOp& set_fadvise_random() & {
+    Op::set_fadvise_random();
+    return *this;
+  }
+  WriteOp&& set_fadvise_random() && {
+    Op::set_fadvise_random();
+    return std::move(*this);
+  }
+
+  WriteOp& set_fadvise_sequential() & {
+    Op::set_fadvise_sequential();
+    return *this;
+  }
+  WriteOp&& set_fadvise_sequential() && {
+    Op::set_fadvise_sequential();
+    return std::move(*this);
+  }
+
+  WriteOp& set_fadvise_willneed() & {
+    Op::set_fadvise_willneed();
+    return *this;
+  }
+  WriteOp&& set_fadvise_willneed() && {
+    Op::set_fadvise_willneed();
+    return std::move(*this);
+  }
+
+  WriteOp& set_fadvise_dontneed() & {
+    Op::set_fadvise_dontneed();
+    return *this;
+  }
+  WriteOp&& set_fadvise_dontneed() && {
+    Op::set_fadvise_dontneed();
+    return std::move(*this);
+  }
+
+  WriteOp& set_fadvise_nocache() & {
+    Op::set_fadvise_nocache();
+    return *this;
+  }
+  WriteOp&& set_fadvise_nocache() && {
+    Op::set_fadvise_nocache();
+    return std::move(*this);
+  }
+
+  WriteOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) & {
+    Op::cmpext(off, std::move(cmp_bl), s);
+    return *this;
+  }
+  WriteOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) && {
+    Op::cmpext(off, std::move(cmp_bl), s);
+    return std::move(*this);
+  }
+
+  WriteOp& cmpxattr(std::string_view name, cmpxattr_op op,
+		   const ceph::buffer::list& val) & {
+    Op::cmpxattr(name, op, val);
+    return *this;
+  }
+  WriteOp&& cmpxattr(std::string_view name, cmpxattr_op op,
+		    const ceph::buffer::list& val) && {
+    Op::cmpxattr(name, op, val);
+    return std::move(*this);
+  }
+
+  WriteOp& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) & {
+    Op::cmpxattr(name, op, val);
+    return *this;
+  }
+  WriteOp&& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) && {
+    Op::cmpxattr(name, op, val);
+    return std::move(*this);
+  }
+
+  WriteOp& assert_version(uint64_t ver) & {
+    Op::assert_version(ver);
+    return *this;
+  }
+  WriteOp&& assert_version(uint64_t ver) && {
+    Op::assert_version(ver);
+    return std::move(*this);
+  }
+
+  WriteOp& assert_exists() & {
+    Op::assert_exists();
+    return *this;
+  }
+  WriteOp&& assert_exists() && {
+    Op::assert_exists();
+    return std::move(*this);
+  }
+
+  WriteOp& cmp_omap(
+    const boost::container::flat_map<
+      std::string, std::pair<ceph::buffer::list, int>>& assertions) & {
+    Op::cmp_omap(assertions);
+    return *this;
+  }
+  WriteOp&& cmp_omap(
+    const boost::container::flat_map<
+      std::string, std::pair<ceph::buffer::list, int>>& assertions) && {
+    Op::cmp_omap(assertions);
+    return std::move(*this);
+  }
+
+  WriteOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       ceph::buffer::list* out,
+	       boost::system::error_code* ec = nullptr) & {
+    Op::exec(cls, method, inbl, out, ec);
+    return *this;
+  }
+  WriteOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+		ceph::buffer::list* out,
+		boost::system::error_code* ec = nullptr) && {
+    Op::exec(cls, method, inbl, out, ec);
+    return std::move(*this);
+  }
+
+  WriteOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       fu2::unique_function<void(boost::system::error_code,
+	                            const ceph::buffer::list&) &&> f) & {
+    Op::exec(cls, method, inbl, std::move(f));
+    return *this;
+  }
+  WriteOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+	        fu2::unique_function<void(boost::system::error_code,
+	                             const ceph::buffer::list&) &&> f) && {
+    Op::exec(cls, method, inbl, std::move(f));
+    return std::move(*this);
+  }
+
+  WriteOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       fu2::unique_function<void(boost::system::error_code, int,
+	                                 const ceph::buffer::list&) &&> f) & {
+    Op::exec(cls, method, inbl, std::move(f));
+    return *this;
+  }
+  WriteOp&& exec(std::string_view cls, std::string_view method,
+	        const ceph::buffer::list& inbl,
+	        fu2::unique_function<void(boost::system::error_code, int,
+	                                  const ceph::buffer::list&) &&> f) && {
+    Op::exec(cls, method, inbl, std::move(f));
+    return std::move(*this);
+  }
+
+  WriteOp& exec(std::string_view cls, std::string_view method,
+	       const ceph::buffer::list& inbl,
+	       boost::system::error_code* ec = nullptr) & {
+    Op::exec(cls, method, inbl, ec);
+    return *this;
+  }
+  WriteOp&& exec(std::string_view cls, std::string_view method,
+		const ceph::buffer::list& inbl,
+		boost::system::error_code* ec = nullptr) && {
+    Op::exec(cls, method, inbl, ec);
+    return std::move(*this);
+  }
+
+  template<typename F>
+  WriteOp& exec(ClsOp<F>&& clsop) & {
+    return clsop(*this);
+  }
+  template<typename F>
+  WriteOp&& exec(ClsOp<F>&& clsop) && {
+    return std::move(clsop(*this));
+  }
+  template<typename F>
+  WriteOp& exec(ClsWriteOp<F>&& clsop) & {
+    return clsop(*this);
+  }
+  template<typename F>
+  WriteOp&& exec(ClsWriteOp<F>&& clsop) && {
+    return std::move(clsop(*this));
+  }
+
+
+  // Flags that apply to all ops in the operation vector
+  WriteOp& balance_reads() & {
+    Op::balance_reads();
+    return *this;
+  }
+  WriteOp&& balance_reads() && {
+    Op::balance_reads();
+    return std::move(*this);
+  }
+  WriteOp& localize_reads() & {
+    Op::localize_reads();
+    return *this;
+  }
+  WriteOp&& localize_reads() && {
+    Op::localize_reads();
+    return std::move(*this);
+  }
+  WriteOp& order_reads_writes() & {
+    Op::order_reads_writes();
+    return *this;
+  }
+  WriteOp&& order_reads_writes() && {
+    Op::order_reads_writes();
+    return std::move(*this);
+  }
+  WriteOp& ignore_cache() & {
+    Op::ignore_cache();
+    return *this;
+  }
+  WriteOp&& ignore_cache() && {
+    Op::ignore_cache();
+    return std::move(*this);
+  }
+  WriteOp& skiprwlocks() & {
+    Op::skiprwlocks();
+    return *this;
+  }
+  WriteOp&& skiprwlocks() && {
+    Op::skiprwlocks();
+    return std::move(*this);
+  }
+  WriteOp& ignore_overlay() & {
+    Op::ignore_overlay();
+    return *this;
+  }
+  WriteOp&& ignore_overlay() && {
+    Op::ignore_overlay();
+    return std::move(*this);
+  }
+  WriteOp& full_try() & {
+    Op::full_try();
+    return *this;
+  }
+  WriteOp&& full_try() && {
+    Op::full_try();
+    return std::move(*this);
+  }
+  WriteOp& full_force() & {
+    Op::full_force();
+    return *this;
+  }
+  WriteOp&& full_force() && {
+    Op::full_force();
+    return std::move(*this);
+  }
+  WriteOp& ignore_redirect() & {
+    Op::ignore_redirect();
+    return *this;
+  }
+  WriteOp&& ignore_redirect() && {
+    Op::ignore_redirect();
+    return std::move(*this);
+  }
+  WriteOp& ordersnap() & {
+    Op::ordersnap();
+    return *this;
+  }
+  WriteOp&& ordersnap() && {
+    Op::ordersnap();
+    return std::move(*this);
+  }
+  WriteOp& returnvec() & {
+    Op::returnvec();
+    return *this;
+  }
+  WriteOp&& returnvec() && {
+    Op::returnvec();
+    return std::move(*this);
+  }
 };
 
 
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 1a7c5cd37db3..026f77043dc9 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -523,149 +523,173 @@ std::ostream& operator <<(std::ostream& m, const Op& o) {
 
 // ReadOp / WriteOp
 
-void ReadOp::read(size_t off, uint64_t len, cb::list* out,
-		  bs::error_code* ec) {
+ReadOp& ReadOp::read(size_t off, uint64_t len, cb::list* out,
+		      bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.read(off, len, ec, out);
+  return *this;
 }
 
-void ReadOp::get_xattr(std::string_view name, cb::list* out,
-		       bs::error_code* ec) {
+ReadOp& ReadOp::get_xattr(std::string_view name, cb::list* out,
+			  bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.getxattr(name, ec, out);
+  return *this;
 }
 
-void ReadOp::get_omap_header(cb::list* out,
-			     bs::error_code* ec) {
+ReadOp& ReadOp::get_omap_header(cb::list* out,
+				bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_get_header(ec, out);
+  return *this;
 }
 
-void ReadOp::sparse_read(uint64_t off, uint64_t len, cb::list* out,
-			 std::vector<std::pair<std::uint64_t,
-			 std::uint64_t>>* extents,
-			 bs::error_code* ec) {
+ReadOp& ReadOp::sparse_read(uint64_t off, uint64_t len, cb::list* out,
+			    std::vector<std::pair<std::uint64_t,
+			    std::uint64_t>>* extents,
+			    bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.sparse_read(off, len, ec, extents, out);
+  return *this;
 }
 
-void ReadOp::stat(std::uint64_t* size, ceph::real_time* mtime,
-		  bs::error_code* ec) {
+ReadOp& ReadOp::stat(std::uint64_t* size, ceph::real_time* mtime,
+		     bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.stat(size, mtime, ec);
+  return *this;
 }
 
-void ReadOp::get_omap_keys(std::optional<std::string_view> start_after,
-			   std::uint64_t max_return,
-			   bc::flat_set<std::string>* keys,
-			   bool* done,
-			   bs::error_code* ec) {
+ReadOp& ReadOp::get_omap_keys(std::optional<std::string_view> start_after,
+			      std::uint64_t max_return,
+			      bc::flat_set<std::string>* keys,
+			      bool* done,
+			      bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_get_keys(start_after, max_return,
 						     ec, keys, done);
+  return *this;
 }
 
-void ReadOp::get_xattrs(bc::flat_map<std::string,
-			cb::list>* kv,
-			bs::error_code* ec) {
+ReadOp& ReadOp::get_xattrs(bc::flat_map<std::string, cb::list>* kv,
+			   bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.getxattrs(ec, kv);
+  return *this;
 }
 
-void ReadOp::get_omap_vals(std::optional<std::string_view> start_after,
-			   std::optional<std::string_view> filter_prefix,
-			   uint64_t max_return,
-			   bc::flat_map<std::string,
-			   cb::list>* kv,
-			   bool* done,
-			   bs::error_code* ec) {
+ReadOp& ReadOp::get_omap_vals(std::optional<std::string_view> start_after,
+			      std::optional<std::string_view> filter_prefix,
+			      uint64_t max_return,
+			      bc::flat_map<std::string, cb::list>* kv,
+			      bool* done,
+			      bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_get_vals(start_after, filter_prefix,
 						     max_return, ec, kv, done);
+  return *this;
 }
 
-void ReadOp::get_omap_vals_by_keys(
+ReadOp& ReadOp::get_omap_vals_by_keys(
   const bc::flat_set<std::string>& keys,
   bc::flat_map<std::string, cb::list>* kv,
-  bs::error_code* ec) {
+  bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_get_vals_by_keys(keys, ec, kv);
+  return *this;
 }
 
-void ReadOp::list_watchers(std::vector<ObjWatcher>* watchers,
-			   bs::error_code* ec) {
+ReadOp& ReadOp::list_watchers(std::vector<ObjWatcher>* watchers,
+			      bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)-> op.list_watchers(watchers, ec);
+  return *this;
 }
 
-void ReadOp::list_snaps(SnapSet* snaps,
-			bs::error_code* ec) {
+ReadOp& ReadOp::list_snaps(SnapSet* snaps,
+			   bs::error_code* ec) & {
   reinterpret_cast<OpImpl*>(&impl)->op.list_snaps(snaps, nullptr, ec);
+  return *this;
 }
 
 // WriteOp
 
-void WriteOp::set_mtime(ceph::real_time t) {
+WriteOp& WriteOp::set_mtime(ceph::real_time t) & {
   auto o = reinterpret_cast<OpImpl*>(&impl);
   o->mtime = t;
+  return *this;
 }
 
-void WriteOp::create(bool exclusive) {
+WriteOp& WriteOp::create(bool exclusive) & {
   reinterpret_cast<OpImpl*>(&impl)->op.create(exclusive);
+  return *this;
 }
 
-void WriteOp::write(uint64_t off, bufferlist&& bl) {
+WriteOp& WriteOp::write(uint64_t off, bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.write(off, bl);
+  return *this;
 }
 
-void WriteOp::write_full(bufferlist&& bl) {
+WriteOp& WriteOp::write_full(bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.write_full(bl);
+  return *this;
 }
 
-void WriteOp::writesame(uint64_t off, uint64_t write_len, bufferlist&& bl) {
+WriteOp& WriteOp::writesame(uint64_t off, uint64_t write_len, bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.writesame(off, write_len, bl);
+  return *this;
 }
 
-void WriteOp::append(bufferlist&& bl) {
+WriteOp& WriteOp::append(bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.append(bl);
+  return *this;
 }
 
-void WriteOp::remove() {
+WriteOp& WriteOp::remove() & {
   reinterpret_cast<OpImpl*>(&impl)->op.remove();
+  return *this;
 }
 
-void WriteOp::truncate(uint64_t off) {
+WriteOp& WriteOp::truncate(uint64_t off) & {
   reinterpret_cast<OpImpl*>(&impl)->op.truncate(off);
+  return *this;
 }
 
-void WriteOp::zero(uint64_t off, uint64_t len) {
+WriteOp& WriteOp::zero(uint64_t off, uint64_t len) & {
   reinterpret_cast<OpImpl*>(&impl)->op.zero(off, len);
+  return *this;
 }
 
-void WriteOp::rmxattr(std::string_view name) {
+WriteOp& WriteOp::rmxattr(std::string_view name) & {
   reinterpret_cast<OpImpl*>(&impl)->op.rmxattr(name);
+  return *this;
 }
 
-void WriteOp::setxattr(std::string_view name,
-                       bufferlist&& bl) {
+WriteOp& WriteOp::setxattr(std::string_view name,
+			   bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.setxattr(name, bl);
+  return *this;
 }
 
-void WriteOp::rollback(uint64_t snapid) {
+WriteOp& WriteOp::rollback(uint64_t snapid) & {
   reinterpret_cast<OpImpl*>(&impl)->op.rollback(snapid);
+  return *this;
 }
 
-void WriteOp::set_omap(
-  const bc::flat_map<std::string, cb::list>& map) {
+WriteOp& WriteOp::set_omap(
+  const bc::flat_map<std::string, cb::list>& map) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_set(map);
+  return *this;
 }
 
-void WriteOp::set_omap_header(bufferlist&& bl) {
+WriteOp& WriteOp::set_omap_header(bufferlist&& bl) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_set_header(bl);
+  return *this;
 }
 
-void WriteOp::clear_omap() {
+WriteOp& WriteOp::clear_omap() & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_clear();
+  return *this;
 }
 
-void WriteOp::rm_omap_keys(
-  const bc::flat_set<std::string>& to_rm) {
+WriteOp& WriteOp::rm_omap_keys(const bc::flat_set<std::string>& to_rm) & {
   reinterpret_cast<OpImpl*>(&impl)->op.omap_rm_keys(to_rm);
+  return *this;
 }
 
-void WriteOp::set_alloc_hint(uint64_t expected_object_size,
-			     uint64_t expected_write_size,
-			     alloc_hint::alloc_hint_t flags) {
+WriteOp& WriteOp::set_alloc_hint(uint64_t expected_object_size,
+				 uint64_t expected_write_size,
+				 alloc_hint::alloc_hint_t flags) & {
   using namespace alloc_hint;
   static_assert(sequential_write ==
 		static_cast<int>(CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE));
@@ -691,6 +715,7 @@ void WriteOp::set_alloc_hint(uint64_t expected_object_size,
   reinterpret_cast<OpImpl*>(&impl)->op.set_alloc_hint(expected_object_size,
 						      expected_write_size,
 						      flags);
+  return *this;
 }
 
 // RADOS
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index e6a87c1617b5..8e2a864f9b89 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -411,8 +411,8 @@ void Op::exec(std::string_view cls, std::string_view method,
   o->ops.push_back(op);
 }
 
-void ReadOp::read(size_t off, uint64_t len, ceph::buffer::list* out,
-	          boost::system::error_code* ec) {
+ReadOp& ReadOp::read(size_t off, uint64_t len, ceph::buffer::list* out,
+		     boost::system::error_code* ec) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   librados::ObjectOperationTestImpl op;
   if (out != nullptr) {
@@ -428,13 +428,14 @@ void ReadOp::read(size_t off, uint64_t len, ceph::buffer::list* out,
       save_operation_ec, std::bind(op, _1, _2, _3, _4, _5, _6), ec);
   }
   o->ops.push_back(op);
+  return *this;
 }
 
-void ReadOp::sparse_read(uint64_t off, uint64_t len,
-		         ceph::buffer::list* out,
-		         std::vector<std::pair<std::uint64_t,
-                                               std::uint64_t>>* extents,
-		         boost::system::error_code* ec) {
+ReadOp& ReadOp::sparse_read(uint64_t off, uint64_t len,
+			    ceph::buffer::list* out,
+			    std::vector<std::pair<std::uint64_t,
+                                                  std::uint64_t>>* extents,
+			    boost::system::error_code* ec) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   librados::ObjectOperationTestImpl op =
     [off, len, out, extents]
@@ -454,9 +455,10 @@ void ReadOp::sparse_read(uint64_t off, uint64_t len,
                      std::bind(op, _1, _2, _3, _4, _5, _6), ec);
   }
   o->ops.push_back(op);
+  return *this;
 }
 
-void ReadOp::list_snaps(SnapSet* snaps, bs::error_code* ec) {
+ReadOp& ReadOp::list_snaps(SnapSet* snaps, bs::error_code* ec) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   librados::ObjectOperationTestImpl op =
     [snaps]
@@ -484,55 +486,64 @@ void ReadOp::list_snaps(SnapSet* snaps, bs::error_code* ec) {
                    std::bind(op, _1, _2, _3, _4, _5, _6), ec);
   }
   o->ops.push_back(op);
+  return *this;
 }
 
-void WriteOp::create(bool exclusive) {
+WriteOp& WriteOp::create(bool exclusive) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::create, _1, _2, exclusive, _5));
+  return *this;
 }
 
-void WriteOp::write(uint64_t off, ceph::buffer::list&& bl) {
+WriteOp& WriteOp::write(uint64_t off, ceph::buffer::list&& bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::write, _1, _2, bl, bl.length(), off, _5));
+  return *this;
 }
 
-void WriteOp::write_full(ceph::buffer::list&& bl) {
+WriteOp& WriteOp::write_full(ceph::buffer::list&& bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::write_full, _1, _2, bl, _5));
+  return *this;
 }
 
-void WriteOp::remove() {
+WriteOp& WriteOp::remove() & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::remove, _1, _2, _5));
+  return *this;
 }
 
-void WriteOp::truncate(uint64_t off) {
+WriteOp& WriteOp::truncate(uint64_t off) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::truncate, _1, _2, off, _5));
+  return *this;
 }
 
-void WriteOp::zero(uint64_t off, uint64_t len) {
+WriteOp& WriteOp::zero(uint64_t off, uint64_t len) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::zero, _1, _2, off, len, _5));
+  return *this;
 }
 
-void WriteOp::writesame(std::uint64_t off, std::uint64_t write_len,
-                        ceph::buffer::list&& bl) {
+WriteOp& WriteOp::writesame(std::uint64_t off, std::uint64_t write_len,
+			    ceph::buffer::list&& bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::writesame, _1, _2, bl, write_len, off, _5));
+  return *this;
 }
 
-void WriteOp::set_alloc_hint(uint64_t expected_object_size,
-		             uint64_t expected_write_size,
-		             alloc_hint::alloc_hint_t flags) {
+WriteOp& WriteOp::set_alloc_hint(uint64_t expected_object_size,
+				 uint64_t expected_write_size,
+				 alloc_hint::alloc_hint_t flags) & {
   // no-op
+  return *this;
 }
 
 RADOS::RADOS() = default;

From 153145df82ee9c086c4688c7c3cd07c98c42f026 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 7 Aug 2023 15:36:27 -0400
Subject: [PATCH 0952/2492] common/ceph_json: `encode_json()` for `std::array`

No decode, though, since JSON arrays can vary in length and
`std::array` cannot.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/common/ceph_json.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h
index 08e8d9e46623..4a12c037d8ad 100644
--- a/src/common/ceph_json.h
+++ b/src/common/ceph_json.h
@@ -582,6 +582,17 @@ static void encode_json(const char *name, const std::vector<T>& l, ceph::Formatt
   f->close_section();
 }
 
+template<class T, std::size_t N>
+static void encode_json(const char *name, const std::array<T, N>& l,
+                        ceph::Formatter *f)
+{
+  f->open_array_section(name);
+  for (auto iter = l.cbegin(); iter != l.cend(); ++iter) {
+    encode_json("obj", *iter, f);
+  }
+  f->close_section();
+}
+
 template<class K, class V, class C = std::less<K>>
 static void encode_json(const char *name, const std::map<K, V, C>& m, ceph::Formatter *f)
 {

From 19509ce650367de8dfb979d3c6a40d5752c822f2 Mon Sep 17 00:00:00 2001
From: Shachar Sharon <ssharon@redhat.com>
Date: Thu, 30 Nov 2023 13:29:30 +0200
Subject: [PATCH 0953/2492] client/fuse: handle case of renameat2 with non-zero
 flags

When user issues renameat(2) with non-zero flags (RENAME_EXCHANGE or
RENAME_NOREPALCE) the current code ignores those flags and treat the
call as ordinary rename. This, in turn, may yield successful rename with
wrong semantics then those expected by the caller.

Follow the same semantics as kernel's cephfs client: return -EINVAL when
having non-zero flags to renameat2 (see 'ceph_rename' at fs/ceph/dir.c).

Fixes: https://tracker.ceph.com/issues/63722

Signed-off-by: Shachar Sharon <ssharon@redhat.com>
---
 src/client/fuse_ll.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc
index 7f92dd668ba3..0e2e200644de 100644
--- a/src/client/fuse_ll.cc
+++ b/src/client/fuse_ll.cc
@@ -753,6 +753,15 @@ static void fuse_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name,
 #endif
                            )
 {
+#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+  // cephfs does not support renameat2 flavors; follow same logic as done in
+  // kclient's ceph_rename()
+  if (flags) {
+    fuse_reply_err(req, get_sys_errno(CEPHFS_EINVAL));
+    return;
+  }
+#endif
+
   CephFuse::Handle *cfuse = fuse_ll_req_prepare(req);
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   UserPerm perm(ctx->uid, ctx->gid);

From f25fd90619a065b6ea728f3f64b69c4530ceb910 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 6 Dec 2023 16:08:05 +0530
Subject: [PATCH 0954/2492] src/test: test sync call providing nullptr as ctx
 to async api

Fixes: https://tracker.ceph.com/issues/63633
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 59 ++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index d5d5e7a5626e..787e022d4922 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -149,3 +149,62 @@ TEST_F(TestClient, LlreadvLlwritev) {
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
 }
 
+TEST_F(TestClient, LlreadvLlwritevNullContext) {
+  /* Test that if Client::ll_preadv_pwritev is called with nullptr context
+  then it performs a sync call. */
+
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevnullcontextfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+				  O_RDWR | O_CREAT | O_TRUNC,
+				  &file, &fh, &stx, 0, 0, myperm));
+
+  char out0[] = "hello ";
+  char out1[] = "world\n";  
+  struct iovec iov_out[2] = {
+	  {out0, sizeof(out0)},
+	  {out1, sizeof(out1)}
+  };
+
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+	  {in0, sizeof(in0)},
+	  {in1, sizeof(in1)}
+  };
+
+  ssize_t bytes_to_write = iov_out[0].iov_len + iov_out[1].iov_len;
+
+  int64_t rc;
+  bufferlist bl;
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, nullptr, nullptr);
+  ASSERT_EQ(rc, bytes_to_write);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, nullptr, &bl);
+  ASSERT_EQ(rc, bytes_to_write);
+
+  copy_bufferlist_to_iovec(iov_in, 2, &bl, rc);
+  ASSERT_EQ(0, strncmp((const char*)iov_in[0].iov_base,
+                       (const char*)iov_out[0].iov_base,
+                       iov_out[0].iov_len));
+  ASSERT_EQ(0, strncmp((const char*)iov_in[1].iov_base,
+                       (const char*)iov_out[1].iov_base, 
+                       iov_out[1].iov_len));
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+}

From f78549da806e12474d04c446d2b3628f7136b9f9 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 7 Dec 2023 18:38:35 +0530
Subject: [PATCH 0955/2492] client: fix copying bufferlist to iovec structures
 in Client::_read

Fixes: https://tracker.ceph.com/issues/63633
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 195738efc87d..8966bf60adb0 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -11269,7 +11269,7 @@ int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
         }
 
         client_lock.unlock();
-        copy_bufferlist_to_iovec(iov, iovcnt, &bl, r);
+        copy_bufferlist_to_iovec(iov, iovcnt, blp ? blp : &bl, r);
         client_lock.lock();
         return r;
     }

From 826e18552412a99550d8186d3cc763e1e60ef645 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 8 Aug 2023 20:38:24 -0400
Subject: [PATCH 0956/2492] neorados: Fix `Op::cmpext` error and unfound
 behavior

A failure will produce the output `osd_errc::cmpext_mismatch`. On
failure, the `unmatch` out parameter, if provided, will be set to the
index of the first nonmatching character; On success, it will be set
to `-1`.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                | 23 ++++++----
 src/librbd/io/ObjectRequest.cc                |  9 ++--
 src/librbd/io/ObjectRequest.h                 |  1 +
 src/neorados/RADOS.cc                         |  4 +-
 src/osd/error_code.cc                         |  9 +++-
 src/osd/error_code.h                          |  5 ++-
 src/osdc/Objecter.cc                          |  6 ++-
 src/osdc/Objecter.h                           | 43 ++++++++++++++-----
 .../librados_test_stub/NeoradosTestStub.cc    | 18 +++++---
 9 files changed, 84 insertions(+), 34 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 67a01b43148e..e890d7a93619 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -302,7 +302,8 @@ class Op {
   void set_fadvise_dontneed();
   void set_fadvise_nocache();
 
-  void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s);
+  void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+	      uint64_t* unmatch = nullptr);
   void cmpxattr(std::string_view name, cmpxattr_op op,
 		const ceph::buffer::list& val);
   void cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val);
@@ -536,12 +537,14 @@ class ReadOp final : public Op {
     return std::move(*this);
   }
 
-  ReadOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) & {
-    Op::cmpext(off, std::move(cmp_bl), s);
+  ReadOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+		 uint64_t* unmatch = nullptr) & {
+    Op::cmpext(off, std::move(cmp_bl), unmatch);
     return *this;
   }
-  ReadOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) && {
-    Op::cmpext(off, std::move(cmp_bl), s);
+  ReadOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+		  uint64_t* unmatch = nullptr) && {
+    Op::cmpext(off, std::move(cmp_bl), unmatch);
     return std::move(*this);
   }
 
@@ -918,12 +921,14 @@ class WriteOp final : public Op {
     return std::move(*this);
   }
 
-  WriteOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) & {
-    Op::cmpext(off, std::move(cmp_bl), s);
+  WriteOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+		  uint64_t* unmatch = nullptr) & {
+    Op::cmpext(off, std::move(cmp_bl), unmatch);
     return *this;
   }
-  WriteOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) && {
-    Op::cmpext(off, std::move(cmp_bl), s);
+  WriteOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+		   uint64_t* unmatch = nullptr) && {
+    Op::cmpext(off, std::move(cmp_bl), unmatch);
     return std::move(*this);
   }
 
diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 86650869616d..428f0fbdf584 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -698,7 +698,7 @@ void ObjectWriteSameRequest<I>::add_write_ops(neorados::WriteOp* wr) {
 
 template <typename I>
 void ObjectCompareAndWriteRequest<I>::add_write_ops(neorados::WriteOp* wr) {
-  wr->cmpext(this->m_object_off, bufferlist{m_cmp_bl}, nullptr);
+  wr->cmpext(this->m_object_off, bufferlist{m_cmp_bl}, &m_mismatch_object_offset);
 
   if (this->m_full_object) {
     wr->write_full(bufferlist{m_write_bl});
@@ -710,13 +710,14 @@ void ObjectCompareAndWriteRequest<I>::add_write_ops(neorados::WriteOp* wr) {
 
 template <typename I>
 int ObjectCompareAndWriteRequest<I>::filter_write_result(int r) const {
-  if (r <= -MAX_ERRNO) {
+  // Error code value for cmpext mismatch. Works for both neorados and
+  // mock image, which seems to be short-circuiting on nonexistence.
+  if (r == -MAX_ERRNO) {
     I *image_ctx = this->m_ictx;
 
     // object extent compare mismatch
-    uint64_t offset = -MAX_ERRNO - r;
     auto [image_extents, _] = io::util::object_to_area_extents(
-        image_ctx, this->m_object_no, {{offset, this->m_object_len}});
+        image_ctx, this->m_object_no, {{m_mismatch_object_offset, this->m_object_len}});
     ceph_assert(image_extents.size() == 1);
 
     if (m_mismatch_offset) {
diff --git a/src/librbd/io/ObjectRequest.h b/src/librbd/io/ObjectRequest.h
index caf644023be7..d9815860f78f 100644
--- a/src/librbd/io/ObjectRequest.h
+++ b/src/librbd/io/ObjectRequest.h
@@ -440,6 +440,7 @@ class ObjectCompareAndWriteRequest : public AbstractObjectWriteRequest<ImageCtxT
   ceph::bufferlist m_cmp_bl;
   ceph::bufferlist m_write_bl;
   uint64_t *m_mismatch_offset;
+  uint64_t m_mismatch_object_offset;
   int m_op_flags;
 };
 
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 026f77043dc9..47b1c0d48824 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -420,9 +420,9 @@ void Op::set_fadvise_nocache() {
     CEPH_OSD_OP_FLAG_FADVISE_NOCACHE);
 }
 
-void Op::cmpext(uint64_t off, bufferlist&& cmp_bl, std::size_t* s) {
+void Op::cmpext(uint64_t off, bufferlist&& cmp_bl, uint64_t* unmatch) {
   reinterpret_cast<OpImpl*>(&impl)->op.cmpext(off, std::move(cmp_bl), nullptr,
-					      s);
+					      unmatch);
 }
 void Op::cmpxattr(std::string_view name, cmpxattr_op op, const bufferlist& val) {
   reinterpret_cast<OpImpl*>(&impl)->
diff --git a/src/osd/error_code.cc b/src/osd/error_code.cc
index 97f0012fdba1..927e4592b123 100644
--- a/src/osd/error_code.cc
+++ b/src/osd/error_code.cc
@@ -53,6 +53,8 @@ const char* osd_error_category::message(int ev, char* buf,
     return "ORDERSNAP flag set; writer has old snapc";
   case osd_errc::blocklisted:
     return "Blocklisted";
+  case osd_errc::cmpext_mismatch:
+    return "CmpExt mismatch";
   }
 
   if (len) {
@@ -72,6 +74,8 @@ std::string osd_error_category::message(int ev) const {
     return "ORDERSNAP flag set; writer has old snapc";
   case osd_errc::blocklisted:
     return "Blocklisted";
+  case osd_errc::cmpext_mismatch:
+    return "CmpExt mismatch";
   }
 
   return cpp_strerror(ev);
@@ -79,7 +83,8 @@ std::string osd_error_category::message(int ev) const {
 
 boost::system::error_condition osd_error_category::default_error_condition(int ev) const noexcept {
   if (ev == static_cast<int>(osd_errc::old_snapc) ||
-      ev == static_cast<int>(osd_errc::blocklisted))
+      ev == static_cast<int>(osd_errc::blocklisted) ||
+      ev == static_cast<int>(osd_errc::cmpext_mismatch))
     return { ev, *this };
   else
     return { ev, boost::system::generic_category() };
@@ -91,6 +96,8 @@ bool osd_error_category::equivalent(int ev, const boost::system::error_condition
       return c == boost::system::errc::invalid_argument;
   case osd_errc::blocklisted:
       return c == boost::system::errc::operation_not_permitted;
+  case osd_errc::cmpext_mismatch:
+      return c == boost::system::errc::operation_canceled;
   }
   return default_error_condition(ev) == c;
 }
diff --git a/src/osd/error_code.h b/src/osd/error_code.h
index d36e79db4a8b..762e03a4643b 100644
--- a/src/osd/error_code.h
+++ b/src/osd/error_code.h
@@ -19,6 +19,8 @@
 
 #include "include/rados.h"
 
+#include "include/err.h"
+
 const boost::system::error_category& osd_category() noexcept;
 
 // Since the OSD mostly uses POSIX error codes plus a couple
@@ -27,7 +29,8 @@ const boost::system::error_category& osd_category() noexcept;
 
 enum class osd_errc {
   old_snapc = 85,  /* ORDERSNAP flag set; writer has old snapc*/
-  blocklisted = 108 /* blocklisted */
+  blocklisted = 108, /* blocklisted */
+  cmpext_mismatch = MAX_ERRNO /* cmpext failed */
 };
 
 namespace boost::system {
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index a74d21b0f15f..9c2daed3ba6e 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -3586,7 +3586,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
 	if (*pe) {
 	  **pe = e.code();
 	}
-	if (*pr) {
+	if (*pr && **pr == 0) {
 	  **pr = ceph::from_error_code(e.code());
 	}
       } catch (const std::exception& e) {
@@ -3596,7 +3596,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
 	if (*pe) {
 	  **pe = osdc_errc::handler_failed;
 	}
-	if (*pr) {
+	if (*pr && **pr == 0) {
 	  **pr = -EIO;
 	}
       }
@@ -3633,6 +3633,8 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
   if (Op::has_completion(onfinish)) {
     if (rc == 0 && handler_error) {
       Op::complete(std::move(onfinish), handler_error, -EIO, service.get_executor());
+    } else if (handler_error) {
+      Op::complete(std::move(onfinish), handler_error, rc, service.get_executor());
     } else {
       Op::complete(std::move(onfinish), osdcode(rc), rc, service.get_executor());
     }
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 870083a29b6c..512a4435eccb 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -64,6 +64,7 @@
 #include "msg/Dispatcher.h"
 
 #include "osd/OSDMap.h"
+#include "osd/error_code.h"
 
 class Context;
 class Messenger;
@@ -434,19 +435,41 @@ struct ObjectOperation {
   struct CB_ObjectOperation_cmpext {
     int* prval = nullptr;
     boost::system::error_code* ec = nullptr;
-    std::size_t* s = nullptr;
+    uint64_t* mismatch_offset = nullptr;
     explicit CB_ObjectOperation_cmpext(int *prval)
       : prval(prval) {}
-    CB_ObjectOperation_cmpext(boost::system::error_code* ec, std::size_t* s)
-      : ec(ec), s(s) {}
+    CB_ObjectOperation_cmpext(boost::system::error_code* ec,
+			      uint64_t* mismatch_offset)
+      : ec(ec), mismatch_offset(mismatch_offset) {}
 
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list&) {
+    void operator()(boost::system::error_code ec, int r,
+		    const ceph::buffer::list&) {
       if (prval)
         *prval = r;
-      if (this->ec)
-	*this->ec = ec;
-      if (s)
-	*s = static_cast<std::size_t>(-(MAX_ERRNO - r));
+
+      if (r <= -MAX_ERRNO) {
+	if (this->ec) {
+	  *this->ec = make_error_code(osd_errc::cmpext_mismatch);
+	}
+	if (mismatch_offset) {
+	  *mismatch_offset = -MAX_ERRNO - r;
+	}
+	throw boost::system::system_error(osd_errc::cmpext_mismatch);
+      } else if (r < 0) {
+	if (this->ec) {
+	  *this->ec = ec;
+	}
+	if (mismatch_offset) {
+	  *mismatch_offset = -1;
+	}
+      } else {
+	if (this->ec) {
+	  this->ec->clear();
+	}
+	if (mismatch_offset) {
+	  *mismatch_offset = -1;
+	}
+      }
     }
   };
 
@@ -457,9 +480,9 @@ struct ObjectOperation {
   }
 
   void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, boost::system::error_code* ec,
-	      std::size_t* s) {
+	      uint64_t* mismatch_offset) {
     add_data(CEPH_OSD_OP_CMPEXT, off, cmp_bl.length(), cmp_bl);
-    set_handler(CB_ObjectOperation_cmpext(ec, s));
+    set_handler(CB_ObjectOperation_cmpext(ec, mismatch_offset));
     out_ec.back() = ec;
   }
 
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index 8e2a864f9b89..1eb0d627ebee 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -113,11 +113,19 @@ librados::AioCompletionImpl* create_aio_completion(
   return impl;
 }
 
-int save_operation_size(int result, size_t* pval) {
-  if (pval != NULL) {
-    *pval = result;
+int save_operation_size(int result, uint64_t* pval) {
+  int our_r = result;
+  if (result <= -MAX_ERRNO) {
+    if (pval != NULL) {
+      *pval = -MAX_ERRNO - result;
+    }
+    our_r = -MAX_ERRNO;
+  } else {
+    if (pval != NULL) {
+      *pval = -1;
+    }
   }
-  return result;
+  return our_r;
 }
 
 int save_operation_ec(int result, boost::system::error_code* ec) {
@@ -323,7 +331,7 @@ void Op::assert_version(uint64_t ver) {
           &librados::TestIoCtxImpl::assert_version, _1, _2, ver));
 }
 
-void Op::cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s) {
+void Op::cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, uint64_t* s) {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   librados::ObjectOperationTestImpl op = std::bind(
     &librados::TestIoCtxImpl::cmpext, _1, _2, off, cmp_bl, _4);

From 529674aa029bc78ff3555412490182542cdcad72 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 8 Aug 2023 23:19:24 -0400
Subject: [PATCH 0957/2492] neorados: Change `cmpxattr_op` to `cmp_op` and use
 in `Op::cmp_omap`

Omap and Xattr comparison use the same values, but in the Xattr case
it's eight bits wide.

Rename the enum to `cmp_op` to reflect this, and open-code encoding
the assertions so we don't have to allocate an intermediate structure.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp | 32 ++++++++++++++++----------------
 src/neorados/RADOS.cc          | 16 ++++++++++++----
 src/osdc/Objecter.h            | 18 +++++++++---------
 3 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index e890d7a93619..6268f0219f33 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -194,7 +194,7 @@ class IOContext final {
 
 inline const std::string all_nspaces("\001");
 
-enum class cmpxattr_op : std::uint8_t {
+enum class cmp_op : std::uint8_t {
   eq  = 1,
   ne  = 2,
   gt  = 3,
@@ -304,14 +304,14 @@ class Op {
 
   void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
 	      uint64_t* unmatch = nullptr);
-  void cmpxattr(std::string_view name, cmpxattr_op op,
+  void cmpxattr(std::string_view name, cmp_op op,
 		const ceph::buffer::list& val);
-  void cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val);
+  void cmpxattr(std::string_view name, cmp_op op, std::uint64_t val);
   void assert_version(uint64_t ver);
   void assert_exists();
   void cmp_omap(const boost::container::flat_map<std::string,
 		                                 std::pair<ceph::buffer::list,
-		                                           int>>& assertions);
+		                                           cmp_op>>& assertions);
 
   void exec(std::string_view cls, std::string_view method,
 	    const ceph::buffer::list& inbl,
@@ -548,22 +548,22 @@ class ReadOp final : public Op {
     return std::move(*this);
   }
 
-  ReadOp& cmpxattr(std::string_view name, cmpxattr_op op,
+  ReadOp& cmpxattr(std::string_view name, cmp_op op,
 		   const ceph::buffer::list& val) & {
     Op::cmpxattr(name, op, val);
     return *this;
   }
-  ReadOp&& cmpxattr(std::string_view name, cmpxattr_op op,
+  ReadOp&& cmpxattr(std::string_view name, cmp_op op,
 		    const ceph::buffer::list& val) && {
     Op::cmpxattr(name, op, val);
     return std::move(*this);
   }
 
-  ReadOp& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) & {
+  ReadOp& cmpxattr(std::string_view name, cmp_op op, std::uint64_t val) & {
     Op::cmpxattr(name, op, val);
     return *this;
   }
-  ReadOp&& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) && {
+  ReadOp&& cmpxattr(std::string_view name, cmp_op op, std::uint64_t val) && {
     Op::cmpxattr(name, op, val);
     return std::move(*this);
   }
@@ -588,13 +588,13 @@ class ReadOp final : public Op {
 
   ReadOp& cmp_omap(
     const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, int>>& assertions) & {
+      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) & {
     Op::cmp_omap(assertions);
     return *this;
   }
   ReadOp&& cmp_omap(
     const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, int>>& assertions) && {
+      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) && {
     Op::cmp_omap(assertions);
     return std::move(*this);
   }
@@ -932,22 +932,22 @@ class WriteOp final : public Op {
     return std::move(*this);
   }
 
-  WriteOp& cmpxattr(std::string_view name, cmpxattr_op op,
+  WriteOp& cmpxattr(std::string_view name, cmp_op op,
 		   const ceph::buffer::list& val) & {
     Op::cmpxattr(name, op, val);
     return *this;
   }
-  WriteOp&& cmpxattr(std::string_view name, cmpxattr_op op,
+  WriteOp&& cmpxattr(std::string_view name, cmp_op op,
 		    const ceph::buffer::list& val) && {
     Op::cmpxattr(name, op, val);
     return std::move(*this);
   }
 
-  WriteOp& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) & {
+  WriteOp& cmpxattr(std::string_view name, cmp_op op, std::uint64_t val) & {
     Op::cmpxattr(name, op, val);
     return *this;
   }
-  WriteOp&& cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) && {
+  WriteOp&& cmpxattr(std::string_view name, cmp_op op, std::uint64_t val) && {
     Op::cmpxattr(name, op, val);
     return std::move(*this);
   }
@@ -972,13 +972,13 @@ class WriteOp final : public Op {
 
   WriteOp& cmp_omap(
     const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, int>>& assertions) & {
+      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) & {
     Op::cmp_omap(assertions);
     return *this;
   }
   WriteOp&& cmp_omap(
     const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, int>>& assertions) && {
+      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) && {
     Op::cmp_omap(assertions);
     return std::move(*this);
   }
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 47b1c0d48824..84cc200929f5 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -424,11 +424,11 @@ void Op::cmpext(uint64_t off, bufferlist&& cmp_bl, uint64_t* unmatch) {
   reinterpret_cast<OpImpl*>(&impl)->op.cmpext(off, std::move(cmp_bl), nullptr,
 					      unmatch);
 }
-void Op::cmpxattr(std::string_view name, cmpxattr_op op, const bufferlist& val) {
+void Op::cmpxattr(std::string_view name, cmp_op op, const bufferlist& val) {
   reinterpret_cast<OpImpl*>(&impl)->
     op.cmpxattr(name, std::uint8_t(op), CEPH_OSD_CMPXATTR_MODE_STRING, val);
 }
-void Op::cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) {
+void Op::cmpxattr(std::string_view name, cmp_op op, std::uint64_t val) {
   bufferlist bl;
   encode(val, bl);
   reinterpret_cast<OpImpl*>(&impl)->
@@ -446,8 +446,16 @@ void Op::assert_exists() {
 }
 void Op::cmp_omap(const bc::flat_map<
 		  std::string, std::pair<cb::list,
-		  int>>& assertions) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_cmp(assertions, nullptr);
+		  cmp_op>>& assertions) {
+  buffer::list bl;
+  encode(uint32_t(assertions.size()), bl);
+  for (const auto& [key, assertion] : assertions) {
+    const auto& [value, op] = assertion;
+    encode(key, bl);
+    encode(value, bl);
+    encode(int(op), bl);
+  }
+  reinterpret_cast<OpImpl*>(&impl)->op.omap_cmp(std::move(bl), nullptr);
 }
 
 void Op::exec(std::string_view cls, std::string_view method,
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 512a4435eccb..0dcdca7982c0 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -1046,18 +1046,18 @@ struct ObjectOperation {
     }
   }
 
-  void omap_cmp(const boost::container::flat_map<
-		std::string, std::pair<ceph::buffer::list, int>>& assertions,
-		boost::system::error_code *ec) {
+  void omap_cmp(ceph::buffer::list&& assertions,
+		int *prval) {
+    using ceph::encode;
     OSDOp &op = add_op(CEPH_OSD_OP_OMAP_CMP);
-    ceph::buffer::list bl;
-    encode(assertions, bl);
     op.op.extent.offset = 0;
-    op.op.extent.length = bl.length();
-    op.indata.claim_append(bl);
-    out_ec.back() = ec;
+    op.op.extent.length = assertions.length();
+    op.indata.claim_append(assertions);
+    if (prval) {
+      unsigned p = ops.size() - 1;
+      out_rval[p] = prval;
+    }
   }
-
   struct C_ObjectOperation_copyget : public Context {
     ceph::buffer::list bl;
     object_copy_cursor_t *cursor;

From 6fab4159ec6c4586d88bd26979a442692a541caf Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 9 Aug 2023 23:59:34 -0400
Subject: [PATCH 0958/2492] neorados: Improve `cmp_omap` argument structure

A map from strings to value/op pairs is a bit much and a bit annoying
for what's essentially a list of assertions. Just use a vector.

Also put op in the middle so it matches `cmpxattr`.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp | 26 +++++++++++---------------
 src/neorados/RADOS.cc          |  7 ++-----
 2 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 6268f0219f33..60e57aa5fc90 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -203,6 +203,12 @@ enum class cmp_op : std::uint8_t {
   lte = 6
 };
 
+struct cmp_assertion {
+  std::string attr;
+  cmp_op op;
+  ceph::buffer::list bl;
+};
+
 namespace alloc_hint {
 enum alloc_hint_t {
   sequential_write = 1,
@@ -309,9 +315,7 @@ class Op {
   void cmpxattr(std::string_view name, cmp_op op, std::uint64_t val);
   void assert_version(uint64_t ver);
   void assert_exists();
-  void cmp_omap(const boost::container::flat_map<std::string,
-		                                 std::pair<ceph::buffer::list,
-		                                           cmp_op>>& assertions);
+  void cmp_omap(const std::vector<cmp_assertion>& assertions);
 
   void exec(std::string_view cls, std::string_view method,
 	    const ceph::buffer::list& inbl,
@@ -586,15 +590,11 @@ class ReadOp final : public Op {
     return std::move(*this);
   }
 
-  ReadOp& cmp_omap(
-    const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) & {
+  ReadOp& cmp_omap(const std::vector<cmp_assertion>& assertions) & {
     Op::cmp_omap(assertions);
     return *this;
   }
-  ReadOp&& cmp_omap(
-    const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) && {
+  ReadOp&& cmp_omap(const std::vector<cmp_assertion>& assertions) && {
     Op::cmp_omap(assertions);
     return std::move(*this);
   }
@@ -970,15 +970,11 @@ class WriteOp final : public Op {
     return std::move(*this);
   }
 
-  WriteOp& cmp_omap(
-    const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) & {
+  WriteOp& cmp_omap(const std::vector<cmp_assertion>& assertions) & {
     Op::cmp_omap(assertions);
     return *this;
   }
-  WriteOp&& cmp_omap(
-    const boost::container::flat_map<
-      std::string, std::pair<ceph::buffer::list, cmp_op>>& assertions) && {
+  WriteOp&& cmp_omap(const std::vector<cmp_assertion>& assertions) && {
     Op::cmp_omap(assertions);
     return std::move(*this);
   }
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 84cc200929f5..0c12b861c870 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -444,13 +444,10 @@ void Op::assert_exists() {
     static_cast<ceph::real_time*>(nullptr),
     static_cast<bs::error_code*>(nullptr));
 }
-void Op::cmp_omap(const bc::flat_map<
-		  std::string, std::pair<cb::list,
-		  cmp_op>>& assertions) {
+void Op::cmp_omap(const std::vector<cmp_assertion>& assertions) {
   buffer::list bl;
   encode(uint32_t(assertions.size()), bl);
-  for (const auto& [key, assertion] : assertions) {
-    const auto& [value, op] = assertion;
+  for (const auto& [key, op, value] : assertions) {
     encode(key, bl);
     encode(value, bl);
     encode(int(op), bl);

From a668ddbef69919731bba3737bdc384d6974cf7d2 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 9 Aug 2023 21:57:41 -0400
Subject: [PATCH 0959/2492] test/neorados: common_tests gets functions for
 bufferlists

Mostly for getting things into and out of them conveniently.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/neorados/common_tests.h | 81 ++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index 4ac4741f2e4b..ae150b7e9430 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -14,13 +14,16 @@
 
 #pragma once
 
+#include <concepts>
 #include <coroutine>
 #include <cstddef>
 #include <exception>
 #include <initializer_list>
 #include <optional>
+#include <span>
 #include <string>
 #include <string_view>
+#include <type_traits>
 #include <utility>
 
 #include <boost/asio/async_result.hpp>
@@ -365,3 +368,81 @@ class NeoRadosTest : public CoroTest {
 /// \param test_name Name of the test
 #define CORO_TEST(test_suite_name, test_name)                                  \
   CORO_TEST_F(test_suite_name, test_name, CoroTest)
+
+/// \brief Generate buffer::list filled with repeating byte
+///
+/// \param c Byte with which to fill
+/// \param s Number of bites
+///
+/// \return A buffer::list filled with `s` copies of `c`
+inline auto filled_buffer_list(char c, std::size_t s) {
+  ceph::buffer::ptr bp{buffer::create(s)};
+  std::memset(bp.c_str(), c, bp.length());
+  ceph::buffer::list bl;
+  bl.push_back(std::move(bp));
+  return bl;
+};
+
+/// \brief Create buffer::list with specified bytes
+///
+/// \param cs Bytes the buffer::list should contain
+///
+/// \return A buffer::list containing the bytes in `cs`
+inline auto to_buffer_list(std::initializer_list<unsigned char> cs) {
+  ceph::buffer::ptr bp{buffer::create(cs.size())};
+  auto ci = cs.begin();
+  for (auto i = 0; i < std::ssize(cs); ++i, ++ci) {
+    bp[i] = *ci;
+  }
+  ceph::buffer::list bl;
+  bl.push_back(std::move(bp));
+  return bl;
+};
+
+/// \brief Create buffer::list with the content of a string_view
+///
+/// \param s View with data to copy
+///
+/// \return A buffer::list containing a copy of `s`.
+inline auto to_buffer_list(std::string_view s) {
+  ceph::buffer::list bl;
+  bl.append(s);
+  return bl;
+};
+
+/// \brief Create buffer::list with the content of a span
+///
+/// \param s Span with data to copy
+///
+/// \return A buffer::list containing a copy of `s`.
+inline auto to_buffer_list(std::span<char> s) {
+  ceph::buffer::list bl;
+  bl.append(s.data(), s.size());
+  return bl;
+};
+
+/// \brief Create buffer::list containing integer
+///
+/// \param n Integer with which to fill the list
+///
+/// \return A buffer::list containing the encoded `n`
+inline auto to_buffer_list(std::integral auto n) {
+  ceph::buffer::list bl;
+  encode(n, bl);
+  return bl;
+};
+
+/// \brief Return value contained by buffer::list
+///
+/// \param bl List with encoded value
+///
+/// \return The value encoded in `bl`.
+template<std::default_initializable T>
+inline auto from_buffer_list(const ceph::buffer::list& bl)
+{
+  using ceph::decode;
+  T t;
+  auto bi = bl.begin();
+  decode(t, bi);
+  return t;
+}

From 3775d7e3ee65330538d2e0d970a9edce91891c0f Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 9 Aug 2023 22:00:24 -0400
Subject: [PATCH 0960/2492] neorados: Implement `ReadOp::checksum`

Rather than using `buffer::list`s for initial values and hash results,
serialize to/from the expected types.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp | 43 ++++++++++++++++++++++++++
 src/neorados/RADOS.cc          | 56 ++++++++++++++++++++++++++++++++++
 src/osdc/Objecter.h            | 17 +++++++++++
 3 files changed, 116 insertions(+)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index 60e57aa5fc90..aaa1fea005cb 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -19,6 +19,7 @@
 #include <concepts>
 #include <cstddef>
 #include <memory>
+#include <optional>
 #include <tuple>
 #include <string>
 #include <string_view>
@@ -224,6 +225,33 @@ enum alloc_hint_t {
 };
 }
 
+namespace hash_alg {
+struct xxhash32_t {
+  using init_value = std::uint32_t;
+  using hash_value = std::uint32_t;
+};
+struct xxhash64_t {
+  using init_value = std::uint64_t;
+  using hash_value = std::uint64_t;
+};
+struct crc32c_t {
+  using init_value = std::uint32_t;
+  using hash_value = std::uint32_t;
+};
+
+inline constexpr xxhash32_t xxhash32;
+inline constexpr xxhash64_t xxhash64;
+inline constexpr crc32c_t crc32c;
+};
+
+template<typename T>
+concept HashAlg = requires {
+  // Just enumerate, what's supported is what's on the OSD.
+  (std::is_same_v<hash_alg::xxhash32_t, T> ||
+   std::is_same_v<hash_alg::xxhash64_t, T> ||
+   std::is_same_v<hash_alg::crc32c_t, T>);
+};
+
 class Op;
 class ReadOp;
 class WriteOp;
@@ -477,6 +505,21 @@ class ReadOp final : public Op {
     return std::move(list_snaps(snaps, ec));
   }
 
+  template<HashAlg T>
+  ReadOp& checksum(T, const typename T::init_value& iv,
+		   std::uint64_t off, std::uint64_t len,
+		   std::uint64_t chunk_size,
+		   std::vector<typename T::hash_value>* out,
+		   boost::system::error_code* ec = nullptr) &;
+  template<HashAlg T>
+  ReadOp&& checksum(T t, const typename T::init_value& iv,
+		    std::uint64_t off, std::uint64_t len,
+		    std::uint64_t chunk_size,
+		    std::vector<typename T::hash_value>* out,
+		    boost::system::error_code* ec = nullptr) && {
+    return std::move(checksum(t, iv, off, len, chunk_size, out, ec));
+  }
+
   // Chaining versions of functions from Op
   ReadOp& set_excl() & {
     Op::set_excl();
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 0c12b861c870..b04aa6c2b9be 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -607,6 +607,62 @@ ReadOp& ReadOp::list_snaps(SnapSet* snaps,
   return *this;
 }
 
+inline uint8_t checksum_op_type(hash_alg::xxhash32_t) {
+  return CEPH_OSD_CHECKSUM_OP_TYPE_XXHASH32;
+}
+inline uint8_t checksum_op_type(hash_alg::xxhash64_t) {
+  return CEPH_OSD_CHECKSUM_OP_TYPE_XXHASH64;
+}
+inline uint8_t checksum_op_type(hash_alg::crc32c_t) {
+    return CEPH_OSD_CHECKSUM_OP_TYPE_CRC32C;
+}
+
+template<HashAlg T>
+ReadOp& ReadOp::checksum(T t, const typename T::init_value& iv,
+			 std::uint64_t off, std::uint64_t len,
+			 std::uint64_t chunk_size,
+			 std::vector<typename T::hash_value>* out,
+			 boost::system::error_code* ec) & {
+  using ceph::encode;
+  buffer::list init_bl;
+  encode(iv, init_bl);
+  // If this isn't the case we have a programming error
+  assert(init_bl.length() == sizeof(typename T::init_value));
+  reinterpret_cast<OpImpl*>(&impl)->op.
+    checksum(checksum_op_type(t), std::move(init_bl),
+	     off, len, chunk_size,
+	     [out](bs::error_code ec, int, const buffer::list& bl) {
+	       if (!ec) {
+		 std::vector<typename T::hash_value> v;
+		 auto bi = bl.begin();
+		 decode(v, bi);
+		 if (out) {
+		   *out = std::move(v);
+		 };
+	       }
+	     }, ec);
+  return *this;
+}
+
+template
+ReadOp& ReadOp::checksum<hash_alg::xxhash32_t>(
+  hash_alg::xxhash32_t, const typename hash_alg::xxhash32_t::init_value&,
+  std::uint64_t off, std::uint64_t len, std::uint64_t chunk_size,
+  std::vector<typename hash_alg::xxhash32_t::hash_value>* out,
+  boost::system::error_code* ec) &;
+template
+ReadOp& ReadOp::checksum<hash_alg::xxhash64_t>(
+  hash_alg::xxhash64_t, const typename hash_alg::xxhash64_t::init_value&,
+  std::uint64_t off, std::uint64_t len, std::uint64_t chunk_size,
+  std::vector<typename hash_alg::xxhash64_t::hash_value>* out,
+  boost::system::error_code* ec) &;
+template
+ReadOp& ReadOp::checksum<hash_alg::crc32c_t>(
+  hash_alg::crc32c_t, const typename hash_alg::crc32c_t::init_value&,
+  std::uint64_t off, std::uint64_t len, std::uint64_t chunk_size,
+  std::vector<typename hash_alg::crc32c_t::hash_value>* out,
+  boost::system::error_code* ec) &;
+
 // WriteOp
 
 WriteOp& WriteOp::set_mtime(ceph::real_time t) & {
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 0dcdca7982c0..2fd8bf5fb46b 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -625,6 +625,23 @@ struct ObjectOperation {
     set_handler(ctx);
   }
 
+  void checksum(uint8_t type, ceph::buffer::list&& init_value,
+		uint64_t off, uint64_t len, size_t chunk_size,
+		fu2::unique_function<void(boost::system::error_code, int,
+					  const ceph::buffer::list&) &&> f,
+		boost::system::error_code* ec) {
+    OSDOp& osd_op = add_op(CEPH_OSD_OP_CHECKSUM);
+    osd_op.op.checksum.offset = off;
+    osd_op.op.checksum.length = len;
+    osd_op.op.checksum.type = type;
+    osd_op.op.checksum.chunk_size = chunk_size;
+    osd_op.indata.append(std::move(init_value));
+
+    unsigned p = ops.size() - 1;
+    out_ec[p] = ec;
+    set_handler(std::move(f));
+  }
+
   // object attrs
   void getxattr(const char *name, ceph::buffer::list *pbl, int *prval) {
     ceph::buffer::list bl;

From ef49105d3db5bc62739834e81896766074487488 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 7 Aug 2023 13:21:10 -0400
Subject: [PATCH 0961/2492] test/neorados: Rounding out test coverage, part 1

This includes cls, cmd, and read_operations.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/neorados/CMakeLists.txt     |  48 ++
 src/test/neorados/cls.cc             |  97 ++++
 src/test/neorados/cmd.cc             | 116 +++++
 src/test/neorados/read_operations.cc | 748 +++++++++++++++++++++++++++
 4 files changed, 1009 insertions(+)
 create mode 100644 src/test/neorados/cls.cc
 create mode 100644 src/test/neorados/cmd.cc
 create mode 100644 src/test/neorados/read_operations.cc

diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index c625133d50a5..bfb201506aae 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -41,3 +41,51 @@ target_link_libraries(ceph_test_neorados_handler_error
 install(TARGETS
   ceph_test_neorados_handler_error
   DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_cls
+  cls.cc
+  )
+target_link_libraries(ceph_test_neorados_cls
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_cls
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_cmd
+  cmd.cc
+  )
+target_link_libraries(ceph_test_neorados_cmd
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_cmd
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_read_operations
+  read_operations.cc
+  )
+target_link_libraries(ceph_test_neorados_read_operations
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_read_operations
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/test/neorados/cls.cc b/src/test/neorados/cls.cc
new file mode 100644
index 000000000000..c3578da78064
--- /dev/null
+++ b/src/test/neorados/cls.cc
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <array>
+#include <coroutine>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "common/ceph_json.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+CORO_TEST_F(NeoRadosCls, DNE, NeoRadosTest)
+{
+  std::string_view oid = "obj";
+  co_await execute(oid, WriteOp{}.create(true));
+  // Call a bogus class
+  co_await expect_error_code(
+    execute(oid, ReadOp{}.exec("doesnotexistasdfasdf", "method", {})),
+    sys::errc::operation_not_supported);
+
+  // Call a bogus method on an existent class
+  co_await expect_error_code(
+    execute(oid, ReadOp{}.exec("lock", "doesnotexistasdfasdfasdf", {})),
+    sys::errc::operation_not_supported);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosCls, RemoteReads, NeoRadosTest)
+{
+  static constexpr std::size_t object_size = 4096;
+  static constexpr std::array oids{"src_object.1"sv, "src_object.2"sv,
+				   "src_object.3"sv};
+
+  std::array<char, object_size> buf;
+  buf.fill(1);
+
+  for (const auto& oid : oids) {
+    buffer::list in;
+    in.append(buf.data(), buf.size());
+    co_await execute(oid, WriteOp{}.write_full(std::move(in)));
+  }
+
+  // Construct JSON request passed to "test_gather" method, and in
+  // turn, to "test_read" method
+  buffer::list in;
+  {
+    auto formatter = std::make_unique<JSONFormatter>(true);
+    formatter->open_object_section("foo");
+    encode_json("src_objects", oids, formatter.get());
+    encode_json("cls", "test_remote_reads", formatter.get());
+    encode_json("method", "test_read", formatter.get());
+    encode_json("pool", pool_name(), formatter.get());
+    formatter->close_section();
+    formatter->flush(in);
+  }
+
+  static const auto target = "tgt_object"s;
+
+  // Create target object by combining data gathered from source
+  // objects using "test_read" method
+  co_await execute(target,
+		   WriteOp{}.exec("test_remote_reads", "test_gather", in));
+
+
+  // Read target object and check its size.
+  buffer::list out;
+  co_await execute(target, ReadOp{}.read(0, 0, &out));
+  EXPECT_EQ(3 * object_size, out.length());
+
+  co_return;
+}
diff --git a/src/test/neorados/cmd.cc b/src/test/neorados/cmd.cc
new file mode 100644
index 000000000000..f27df529eacb
--- /dev/null
+++ b/src/test/neorados/cmd.cc
@@ -0,0 +1,116 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <array>
+#include <coroutine>
+
+#include <fmt/format.h>
+
+#include <boost/asio/as_tuple.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "include/stringify.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace sys = boost::system;
+namespace buffer = ceph::buffer;
+
+using neorados::ReadOp;
+
+using namespace std::literals;
+
+CORO_TEST_F(NeoRadosCmd, MonDescribe, NeoRadosTest) {
+  std::string outs;
+  buffer::list outbl;
+  std::vector arg({R"({"prefix": "get_command_descriptions"})"s});
+  co_await rados().mon_command(std::move(arg), {}, &outs, &outbl,
+			       asio::use_awaitable);
+  EXPECT_LT(0u, outbl.length());
+  EXPECT_LE(0u, outs.length());
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosCmd, OSDCmd, NeoRadosTest) {
+  {
+    std::vector arg({R"(asdfasdf)"s});
+    co_await expect_error_code(
+      rados().osd_command(0, std::move(arg),
+			  {}, asio::use_awaitable),
+      sys::errc::invalid_argument, sys::errc::no_such_device_or_address);
+  }
+
+  {
+    std::vector arg({R"(version)"s});
+    co_await expect_error_code(
+      rados().osd_command(0, std::move(arg),
+			  {}, asio::use_awaitable),
+      sys::errc::invalid_argument, sys::errc::no_such_device_or_address);
+  }
+
+  {
+    std::vector arg({R"({"prefix":"version"})"s});
+    auto [ec, outs, outbl] = co_await
+      rados().osd_command(0, std::move(arg), {},
+			  asio::as_tuple(asio::use_awaitable));
+
+    EXPECT_TRUE((!ec && outbl.length() > 0) ||
+		(ec == sys::errc::no_such_device_or_address &&
+		 outbl.length() == 0));
+
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosCmd, PGCmd, NeoRadosTest) {
+  const neorados::PG pgid{uint64_t(pool().get_pool()), 0};
+
+  {
+    std::vector arg({R"(asdfasdf)"s});
+    // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+    co_await expect_error_code(
+      rados().pg_command(pgid, std::move(arg),
+			 {}, asio::use_awaitable),
+      sys::errc::invalid_argument, sys::errc::no_such_device_or_address);
+  }
+
+  // make sure the pg exists on the osd before we query it
+  for (auto i = 0; i < 100; ++i) {
+    co_await expect_error_code(
+      rados().execute(fmt::format("obj{}", i), pool(),
+		      ReadOp{}.assert_exists(), nullptr,
+		      asio::use_awaitable),
+      sys::errc::no_such_file_or_directory);
+  }
+
+  {
+    std::vector arg({fmt::format(R"({{"prefix":"pg", "cmd":"query", "pgid":"{}.{}"}})",
+				 pgid.pool, pgid.seed)});
+    // Working around a bug in GCC.
+    auto coro = rados().pg_command(pgid, std::move(arg),
+				   {}, asio::as_tuple(asio::use_awaitable));
+    auto [ec, outs, outbl] = co_await std::move(coro);
+
+    EXPECT_TRUE(!ec || ec == sys::errc::no_such_file_or_directory ||
+		ec == sys::errc::no_such_device_or_address);
+
+    EXPECT_LT(0u, outbl.length());
+  }
+
+  co_return;
+}
diff --git a/src/test/neorados/read_operations.cc b/src/test/neorados/read_operations.cc
new file mode 100644
index 000000000000..adf5f34ae5e5
--- /dev/null
+++ b/src/test/neorados/read_operations.cc
@@ -0,0 +1,748 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstring>
+#include <cstdint>
+#include <initializer_list>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_map.hpp>
+
+#include <boost/system/error_code.hpp>
+#include <boost/system/errc.hpp>
+
+#include <xxHash/xxhash.h>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "osd/error_code.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace ctnr = boost::container;
+namespace hash_alg = neorados::hash_alg;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::cmp_op;
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+class ReadOpTest : public NeoRadosTest {
+protected:
+  static constexpr auto oid = "testobj"sv;
+  static constexpr auto data = "testdata"sv;
+  static constexpr std::size_t datalen = 16;
+
+  auto write_object(std::string_view data, uint64_t* objver = nullptr) {
+    return execute(oid, WriteOp{}.write_full(to_buffer_list(data)), objver);
+  }
+
+  auto remove_object() {
+    return execute(oid, WriteOp{}.remove());
+  }
+
+  asio::awaitable<void> CoSetUp() override {
+    co_await NeoRadosTest::CoSetUp();
+    co_await write_object(data);
+  }
+
+  asio::awaitable<void> CoTearDown() override {
+    co_await remove_object();
+    co_await NeoRadosTest::CoTearDown();
+  }
+
+  auto assert_version(uint64_t v) {
+    return execute(oid, ReadOp{}.assert_version(v));
+  }
+
+  auto setxattr(std::string_view xattr, buffer::list bl) {
+    return execute(oid, WriteOp{}.setxattr(xattr, std::move(bl)));
+  }
+
+  auto cmpxattr(std::string_view xattr, cmp_op op, buffer::list bl) {
+    return execute(oid, ReadOp{}.cmpxattr(xattr, op, std::move(bl)));
+  }
+};
+
+CORO_TEST_F(NeoRadosReadOps, SetOpFlags, ReadOpTest) {
+  sys::error_code ec;
+  co_await execute(oid, ReadOp{}
+		   .exec("rbd"sv, "get_id"sv, {}, nullptr, &ec)
+                   .set_failok());
+  EXPECT_EQ(sys::errc::io_error, ec);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, AssertExists, ReadOpTest) {
+  co_await expect_error_code(execute("nonexistent"sv, ReadOp{}.assert_exists()),
+			     sys::errc::no_such_file_or_directory);
+  co_await execute(oid, ReadOp{}.assert_exists());
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, AssertVersion, ReadOpTest) {
+  std::uint64_t v = 0;
+  // Write to the object a second time to guarantee that its
+  // version number is greater than 0
+  co_await write_object(data, &v);
+
+  co_await expect_error_code(assert_version(v + 1),
+			     sys::errc::value_too_large);
+  co_await assert_version(v);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, CmpXattr, ReadOpTest) {
+  using enum cmp_op;
+  using sys::errc::operation_canceled;
+
+  static constexpr auto xattr = "test"sv;
+
+  co_await setxattr(xattr, filled_buffer_list(0xcc, datalen));
+
+  // Equal value
+  co_await cmpxattr(xattr, eq, filled_buffer_list(0xcc, datalen));
+  co_await expect_error_code(cmpxattr(xattr, ne,
+				      filled_buffer_list(0xcc, datalen)),
+			     operation_canceled);
+  co_await expect_error_code(cmpxattr(xattr, gt,
+				      filled_buffer_list(0xcc, datalen)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, gte, filled_buffer_list(0xcc, datalen));
+  co_await expect_error_code(cmpxattr(xattr, lt,
+				      filled_buffer_list(0xcc, datalen)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, lte, filled_buffer_list(0xcc, datalen));
+
+  // < value
+  co_await expect_error_code(cmpxattr(xattr, eq,
+				      filled_buffer_list(0xcb, datalen)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, ne, filled_buffer_list(0xcb, datalen));
+  co_await expect_error_code(cmpxattr(xattr, gt,
+				      filled_buffer_list(0xcb, datalen)),
+			     operation_canceled);
+  co_await expect_error_code(cmpxattr(xattr, gte,
+				      filled_buffer_list(0xcb, datalen)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, lt, filled_buffer_list(0xcb, datalen));
+  co_await cmpxattr(xattr, lte, filled_buffer_list(0xcb, datalen));
+
+  // > value
+  co_await expect_error_code(cmpxattr(xattr, eq,
+				      filled_buffer_list(0xcd, datalen)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, ne, filled_buffer_list(0xcd, datalen));
+  co_await cmpxattr(xattr, gt, filled_buffer_list(0xcd, datalen));
+  co_await cmpxattr(xattr, gte, filled_buffer_list(0xcd, datalen));
+  co_await expect_error_code(cmpxattr(xattr, lt,
+				      filled_buffer_list(0xcd, datalen)),
+			     operation_canceled);
+  co_await expect_error_code(cmpxattr(xattr, lte,
+				      filled_buffer_list(0xcd, datalen)),
+			     operation_canceled);
+
+  // check that null bytes are compared correctly
+
+  co_await setxattr(xattr, to_buffer_list("\x00\x00"sv));
+  co_await expect_error_code(cmpxattr(xattr, eq,
+				      to_buffer_list("\x00\xcc"sv)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, ne, to_buffer_list("\x00\xcc"sv));
+  co_await cmpxattr(xattr, gt, to_buffer_list("\x00\xcc"sv));
+  co_await cmpxattr(xattr, gte, to_buffer_list("\x00\xcc"sv));
+  co_await expect_error_code(cmpxattr(xattr, lt,
+				      to_buffer_list("\x00\xcc"sv)),
+			     operation_canceled);
+  co_await expect_error_code(cmpxattr(xattr, lte,
+				      to_buffer_list("\x00\xcc"sv)),
+			     operation_canceled);
+
+  co_await cmpxattr(xattr, eq, to_buffer_list("\x00\x00"sv));
+  co_await expect_error_code(cmpxattr(xattr, ne,
+				      to_buffer_list("\x00\x00"sv)),
+			     operation_canceled);
+  co_await expect_error_code(cmpxattr(xattr, gt,
+				      to_buffer_list("\x00\x00"sv)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, gte, to_buffer_list("\x00\x00"sv));
+  co_await expect_error_code(cmpxattr(xattr, lt,
+				      to_buffer_list("\x00\x00"sv)),
+			     operation_canceled);
+  co_await cmpxattr(xattr, lte, to_buffer_list("\x00\x00"sv));
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, Read, ReadOpTest) {
+  // Check that using read_ops returns the same data with
+  // or without ec out params
+  {
+    buffer::list bl;
+    co_await execute(oid, ReadOp{}.read(0, 0, &bl));
+    EXPECT_TRUE((data.length() == bl.length()) &&
+		(0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+  }
+  {
+    buffer::list bl;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}.read(0, 0, &bl, &ec));
+    EXPECT_TRUE((data.length() == bl.length()) &&
+		(0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+    EXPECT_FALSE(ec);
+  }
+
+  {
+    buffer::list bl;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}
+		     .read(0, 0, &bl, &ec)
+		     .set_fadvise_dontneed());
+    EXPECT_TRUE((data.length() == bl.length()) &&
+		(0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+    EXPECT_FALSE(ec);
+  }
+  co_return;
+}
+
+inline std::uint32_t crc32c(uint32_t seed, std::string_view v) {
+  return ceph_crc32c(
+    seed, reinterpret_cast<const uint8_t*>(v.data()),
+    uint32_t(v.size()));
+}
+
+CORO_TEST_F(NeoRadosReadOps, Checksum, ReadOpTest) {
+  {
+    static constexpr uint64_t seed = -1;
+    std::vector<uint64_t> hash;
+
+    co_await execute(oid, ReadOp{}
+		     .checksum(hash_alg::xxhash64, seed, 0, 0, 0, &hash));
+    EXPECT_EQ(1u, hash.size());
+    EXPECT_EQ(XXH64(data.data(), data.size(), seed), hash[0]);
+  }
+  {
+    static constexpr uint32_t seed = -1;
+    std::vector<uint32_t> crc;
+    co_await execute(oid, ReadOp{}
+		     .checksum(hash_alg::crc32c, seed, 0, 0, 0, &crc));
+    EXPECT_EQ(crc32c(seed, data), crc[0]);
+  }
+  {
+    static constexpr uint32_t seed = -1;
+    std::vector<uint32_t> hash;
+    co_await execute(oid, ReadOp{}
+		     .checksum(hash_alg::xxhash32, seed, 0, 0, 0, &hash));
+    EXPECT_EQ(XXH32(data.data(), data.size(), seed), hash[0]);
+  }
+
+  {
+    static constexpr uint32_t seed = -1;
+    std::vector<uint32_t> crc;
+    co_await execute(oid, ReadOp{}.checksum(hash_alg::crc32c, seed, 0,
+					    data.length(), 4, &crc));
+    EXPECT_EQ(crc32c(seed, data.substr(0, 4)), crc[0]);
+    EXPECT_EQ(crc32c(seed, data.substr(4, 4)), crc[1]);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, RWOrderedRead, ReadOpTest) {
+  buffer::list bl;
+  sys::error_code ec;
+  ReadOp op;
+  op.read(0, 0, &bl, &ec);
+  op.set_fadvise_dontneed();
+  op.order_reads_writes();
+  co_await execute(oid, std::move(op));
+
+  EXPECT_FALSE(ec);
+  EXPECT_TRUE((data.length() == bl.length()) &&
+	      (0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, ShortRead, ReadOpTest) {
+  constexpr auto read_len = data.size() * 2;
+  buffer::list bl;
+  // check that using read_ops returns the same data with
+  // or without ec out params
+  co_await execute(oid, ReadOp{}.read(0, read_len, &bl));
+  EXPECT_TRUE((data.length() == bl.length()) &&
+	      (0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+
+  sys::error_code ec;
+  bl.clear();
+  co_await execute(oid, ReadOp{}.read(0, read_len, &bl, &ec));
+  EXPECT_FALSE(ec);
+  EXPECT_TRUE((data.length() == bl.length()) &&
+	      (0 == std::memcmp(data.data(), bl.c_str(), data.length())));
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, Exec, ReadOpTest) {
+  buffer::list bl;
+  sys::error_code ec;
+  co_await execute(oid,
+		   ReadOp{}.exec("rbd"sv, "get_all_features"sv, {}, &bl, &ec));
+  EXPECT_FALSE(ec);
+  std::uint64_t features;
+  EXPECT_EQ(sizeof(features), bl.length());
+  auto it = bl.cbegin();
+  ceph::decode(features, it);
+  EXPECT_EQ(RBD_FEATURES_ALL, features);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, Stat, ReadOpTest) {
+  std::uint64_t size = 1;
+  sys::error_code ec;
+  co_await expect_error_code(execute("nonexistent"sv,
+				     ReadOp{}.stat(&size, nullptr, &ec)),
+			     sys::errc::no_such_file_or_directory);
+  EXPECT_EQ(sys::errc::io_error, ec);
+  EXPECT_EQ(1u, size);
+
+  const ceph::real_time ts{1'457'129'052 * 1s};
+  auto bl = to_buffer_list(data);
+  co_await execute(oid, WriteOp{}.write(0, std::move(bl)).set_mtime(ts));
+
+  ceph::real_time ts2;
+  ec.clear();
+  co_await execute(oid, ReadOp{}.stat(&size, &ts2, &ec));
+  EXPECT_FALSE(ec);
+  EXPECT_EQ(data.size(), size);
+  EXPECT_EQ(ts, ts2);
+
+  co_await execute(oid, ReadOp{}.stat(nullptr, nullptr));
+
+  co_await expect_error_code(execute("nonexistent"sv,
+				     ReadOp{}.stat(nullptr, nullptr)),
+			     sys::errc::no_such_file_or_directory);
+
+  co_return;
+}
+
+
+CORO_TEST_F(NeoRadosReadOps, Omap, ReadOpTest) {
+  const ctnr::flat_map<std::string, buffer::list> omap{
+    {"bar"s, {}},
+    {"foo"s, to_buffer_list("\0"sv)},
+    {"test1"s, to_buffer_list("abc"sv)},
+    {"test2"s, to_buffer_list("va\0lue"sv)}
+  };
+
+  co_await expect_error_code(
+    execute("nonexistent"sv,
+	    ReadOp{}.get_omap_vals({}, {}, 10, nullptr, nullptr)),
+    sys::errc::no_such_file_or_directory);
+
+  {
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    bool truncated;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}.get_omap_vals({}, {}, 10, &omap2,
+						 &truncated, &ec));
+    EXPECT_FALSE(ec);
+    EXPECT_TRUE(omap2.empty());
+    EXPECT_FALSE(truncated);
+  }
+
+  co_await execute(oid, WriteOp{}.set_omap(omap));
+
+  // Check for readability
+  {
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    ctnr::flat_set<std::string> keys;
+    bool truncated, truncated2;
+    sys::error_code ec, ec2;
+
+    co_await execute(oid, ReadOp{}
+		     .get_omap_vals({}, {}, 10, &omap2, &truncated, &ec)
+		     .get_omap_keys({}, 10, &keys, &truncated2, &ec2));
+    EXPECT_FALSE(ec);
+    EXPECT_FALSE(ec2);
+    EXPECT_FALSE(truncated2);
+    EXPECT_EQ(omap, omap2);
+    EXPECT_FALSE(truncated);
+    EXPECT_EQ(omap.size(), keys.size());
+    EXPECT_TRUE(std::all_of(keys.begin(), keys.end(),
+			    [&](const auto& s) {
+			      return omap.contains(s);
+			    }));
+    EXPECT_TRUE(std::all_of(omap.begin(), omap.end(),
+			    [&](const auto& kv) {
+			      return keys.contains(kv.first);
+			    }));
+  }
+
+  // Check iteration and truncation
+  {
+    std::unordered_set<std::string> keys;
+    for (const auto& [key, value] : omap) {
+      keys.insert(key);
+    }
+    bool truncated = true;
+    std::optional<std::string> lastkey;
+    while (truncated) {
+      ctnr::flat_set<std::string> keys2;
+      ctnr::flat_map<std::string, buffer::list> omap2;
+      bool truncated2;
+      ReadOp op;
+      op.get_omap_vals(lastkey, {}, 1, &omap2, &truncated);
+      op.get_omap_keys(lastkey, 1, &keys2, &truncated2);
+      co_await execute(oid, std::move(op));
+      EXPECT_EQ(1, std::ssize(keys2));
+      EXPECT_EQ(1, std::ssize(omap2));
+      EXPECT_EQ(truncated, truncated2);
+
+      const auto& key = *keys2.begin();
+      EXPECT_EQ(omap2.begin()->first, key);
+      EXPECT_TRUE(keys.contains(key));
+      EXPECT_EQ(omap.at(key), omap2[key]);
+      keys.erase(key);
+      lastkey = key;
+    }
+    EXPECT_TRUE(keys.empty());
+  }
+
+  // check omap_cmp finds all expected values
+  {
+    ReadOp op;
+    for (const auto& [key, value] : omap) {
+      op.cmp_omap({{key, cmp_op::eq, value}});
+    }
+    co_await execute(oid, std::move(op));
+  }
+  {
+    std::vector<neorados::cmp_assertion> cmps;
+    for (const auto& [key, value] : omap) {
+      cmps.push_back({key,  cmp_op::eq, value});
+    }
+    co_await execute(oid, ReadOp{}.cmp_omap(cmps));
+  }
+
+  // try to remove keys with a guard that should fail
+  {
+    WriteOp op;
+    auto key = (omap.begin() + 2)->first;
+    op.cmp_omap({{key, cmp_op::lt,omap.at(key)}});
+    op.rm_omap_keys({omap.begin()->first, (omap.begin() + 1)->first});
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       sys::errc::operation_canceled);
+  }
+  // Verify the keys are still there, and then remove them
+  {
+    WriteOp op;
+    op.cmp_omap({{omap.begin()->first, cmp_op::eq, omap.begin()->second}});
+    op.cmp_omap({{(omap.begin() + 1)->first, cmp_op::eq,
+		  {(omap.begin() + 1)->second}}});
+    op.rm_omap_keys({omap.begin()->first, (omap.begin() + 1)->first});
+    co_await execute(oid, std::move(op));
+
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    const ctnr::flat_map omapcmp{omap.begin() + 2, omap.end()};
+    bool trunc;
+    co_await execute(oid, ReadOp{}.get_omap_vals({}, {}, 10, &omap2, &trunc));
+    EXPECT_FALSE(trunc);
+    EXPECT_EQ(omapcmp, omap2);
+  }
+
+  // clear the rest and check there are none left
+  {
+    co_await execute(oid, WriteOp{}.clear_omap());
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    bool trunc;
+    co_await execute(oid, ReadOp{}.get_omap_vals({}, {}, 10, &omap2, &trunc));
+    EXPECT_FALSE(trunc);
+    EXPECT_TRUE(omap2.empty());
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, OmapNuls, ReadOpTest) {
+  const ctnr::flat_map<std::string, buffer::list> omap{
+    {"1\0bar"s, to_buffer_list("_\0var"sv)},
+    {"2baar\0"s, to_buffer_list("_vaar\0"sv)},
+    {"3baa\0rr"s, to_buffer_list("__vaa\0rr"sv)}
+  };
+
+  co_await expect_error_code(
+    execute("nonexistent"sv, ReadOp{}.get_omap_vals({}, {}, 10, nullptr, nullptr)),
+    sys::errc::no_such_file_or_directory);
+  {
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    bool truncated;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}
+		     .get_omap_vals({}, {}, 10, &omap2, &truncated, &ec));
+    EXPECT_FALSE(ec);
+    EXPECT_TRUE(omap2.empty());
+    EXPECT_FALSE(truncated);
+  }
+
+  co_await execute(oid, WriteOp{}.set_omap(omap));
+
+  // Check for readability
+  {
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    ctnr::flat_set<std::string> keys;
+    bool truncated, truncated2;
+    sys::error_code ec, ec2;
+    ReadOp op;
+    op.get_omap_vals({}, {}, 10, &omap2, &truncated, &ec);
+    op.get_omap_keys({}, 10, &keys, &truncated2, &ec2);
+    co_await execute(oid, std::move(op));
+    EXPECT_FALSE(ec);
+    EXPECT_FALSE(ec2);
+    EXPECT_FALSE(truncated2);
+    EXPECT_EQ(omap, omap2);
+    EXPECT_FALSE(truncated);
+    EXPECT_EQ(omap.size(), keys.size());
+    EXPECT_TRUE(std::all_of(keys.begin(), keys.end(),
+			    [&](const auto& s) {
+			      return omap.contains(s);
+			    }));
+    EXPECT_TRUE(std::all_of(omap.begin(), omap.end(),
+			    [&](const auto& kv) {
+			      return keys.contains(kv.first);
+			    }));
+  }
+
+  // Check iteration and truncation
+  {
+    std::unordered_set<std::string> keys;
+    for (const auto& [key, value] : omap) {
+      keys.insert(key);
+    }
+    bool truncated = true;
+    std::optional<std::string> lastkey;
+    while (truncated) {
+      ctnr::flat_set<std::string> keys2;
+      ctnr::flat_map<std::string, buffer::list> omap2;
+      bool truncated2;
+      ReadOp op;
+      op.get_omap_vals(lastkey, {}, 1, &omap2, &truncated);
+      op.get_omap_keys(lastkey, 1, &keys2, &truncated2);
+      co_await execute(oid, std::move(op));
+      EXPECT_EQ(1, std::ssize(keys2));
+      EXPECT_EQ(1, std::ssize(omap2));
+      EXPECT_EQ(truncated, truncated2);
+
+      const auto& key = *keys2.begin();
+      EXPECT_EQ(omap2.begin()->first, key);
+      EXPECT_TRUE(keys.contains(key));
+      EXPECT_EQ(omap.at(key), omap2[key]);
+      keys.erase(key);
+      lastkey = key;
+    }
+    EXPECT_TRUE(keys.empty());
+  }
+
+  // check omap_cmp finds all expected values
+  {
+    ReadOp op;
+    for (const auto& [key, value] : omap) {
+      op.cmp_omap({{key, cmp_op::eq, value}});
+    }
+    co_await execute(oid, std::move(op));
+  }
+  {
+    std::vector<neorados::cmp_assertion> cmps;
+    for (const auto& [key, value] : omap) {
+      cmps.push_back({key, cmp_op::eq, value});
+    }
+    co_await execute(oid, ReadOp{}.cmp_omap(cmps));
+  }
+
+  // try to remove keys with a guard that should fail
+  {
+    WriteOp op;
+    auto key = (omap.begin() + 2)->first;
+    op.cmp_omap({{key, cmp_op::lt, omap.at(key)}});
+    op.rm_omap_keys({omap.begin()->first, (omap.begin() + 1)->first});
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       sys::errc::operation_canceled);
+  }
+  // Verify the keys are still there, and then remove them
+  {
+    WriteOp op;
+    op.cmp_omap({{omap.begin()->first, cmp_op::eq, omap.begin()->second}});
+    op.cmp_omap({{(omap.begin() + 1)->first, cmp_op::eq,
+		  (omap.begin() + 1)->second}});
+    op.rm_omap_keys({omap.begin()->first, (omap.begin() + 1)->first});
+    co_await execute(oid, std::move(op));
+
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    const ctnr::flat_map omapcmp{omap.begin() + 2, omap.end()};
+    bool trunc;
+    co_await execute(oid, ReadOp{}.get_omap_vals({}, {}, 10, &omap2, &trunc));
+    EXPECT_FALSE(trunc);
+    EXPECT_EQ(omapcmp, omap2);
+  }
+
+  // clear the rest and check there are none left
+  {
+    co_await execute(oid, WriteOp{}.clear_omap());
+    ctnr::flat_map<std::string, buffer::list> omap2;
+    bool trunc;
+    co_await execute(oid, ReadOp{}.get_omap_vals({}, {}, 10, &omap2, &trunc));
+    EXPECT_FALSE(trunc);
+    EXPECT_TRUE(omap2.empty());
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, GetXattrs, ReadOpTest) {
+  const ctnr::flat_map<std::string, buffer::list> xattrs{
+    {"bar"s, {}},
+    {"foo"s, to_buffer_list("\0"sv)},
+    {"test1"s, to_buffer_list("abc"sv)},
+    {"test2"s, to_buffer_list("va\0lue"sv)}
+  };
+
+  {
+    ctnr::flat_map<std::string, buffer::list> xattrs2;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}.get_xattrs(&xattrs2, &ec));
+    EXPECT_FALSE(ec);
+    EXPECT_TRUE(xattrs2.empty());
+  }
+
+  {
+    WriteOp op;
+    for (const auto& [key, value] : xattrs) {
+      op.setxattr(key, buffer::list{value});
+    }
+    co_await execute(oid, std::move(op));
+  }
+
+  {
+    ctnr::flat_map<std::string, buffer::list> xattrs2;
+    sys::error_code ec;
+    co_await execute(oid, ReadOp{}.get_xattrs(&xattrs2, &ec));
+    EXPECT_FALSE(ec);
+    EXPECT_EQ(xattrs, xattrs2);
+  }
+
+  {
+    ReadOp op;
+    std::vector<buffer::list> bls;
+    std::vector<sys::error_code> ecs;
+    bls.reserve(xattrs.size());
+    ecs.reserve(xattrs.size());
+    for (const auto& [key, value] : xattrs) {
+      bls.push_back({});
+      ecs.push_back({});
+      op.get_xattr(key, &bls.back(), &ecs.back());
+    }
+
+    co_await execute(oid, std::move(op));
+
+    EXPECT_EQ(xattrs.size(), ecs.size());
+    EXPECT_EQ(xattrs.size(), bls.size());
+    for (auto i = 0; i < std::ssize(xattrs); ++i) {
+      const auto& key = (xattrs.begin() + i)->first;
+      EXPECT_FALSE(ecs[i]);
+      EXPECT_EQ(xattrs.at(key), bls[i]);
+    }
+  }
+
+  {
+    ReadOp op;
+    for (const auto& [key, value] : xattrs) {
+      op.cmpxattr(key, cmp_op::eq, value);
+    }
+    co_await execute(oid, std::move(op));
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosReadOps, CmpExt, ReadOpTest) {
+  co_await execute(oid, WriteOp{}.write_full(to_buffer_list("\x01\x02\x03"sv)));
+  uint64_t unmatch = 0;
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x01\x02\x03"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await execute(oid, std::move(op));
+    EXPECT_EQ(-1 , unmatch);
+    EXPECT_EQ(to_buffer_list("\x01\x02\x03"sv), bl);
+  }
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x00\x02\x03"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       osd_errc::cmpext_mismatch);
+    EXPECT_EQ(0 , unmatch);
+    EXPECT_EQ(0, bl.length());
+  }
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x01\x00\x03"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       osd_errc::cmpext_mismatch);
+    EXPECT_EQ(1, unmatch);
+    EXPECT_EQ(0, bl.length());
+  }
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x01\x02\x00"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       osd_errc::cmpext_mismatch);
+    EXPECT_EQ(2, unmatch);
+    EXPECT_EQ(0, bl.length());
+  }
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x01\x02\x03\x04"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await expect_error_code(execute(oid, std::move(op)),
+			       osd_errc::cmpext_mismatch);
+    EXPECT_EQ(3, unmatch);
+    EXPECT_EQ(0, bl.length());
+  }
+  // Make sure other error codes work properly
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.cmpext(0, to_buffer_list("\x01\x02\x03"sv), &unmatch);
+    op.read(0, 0, &bl);
+    co_await expect_error_code(execute("nonexistent"sv, std::move(op)),
+			       sys::errc::no_such_file_or_directory);
+    EXPECT_EQ(-1, unmatch);
+    EXPECT_EQ(0, bl.length());
+  }
+  co_return;
+}

From 39ede8b1009bd6dd1c19a277343c68e9121a3738 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 14 Aug 2023 19:11:27 -0400
Subject: [PATCH 0962/2492] neorados: Take `claim_append`ed `buffer::list`s by
 value

Since both const lvalue and rvalue make sense, instead of duplicating
everything, just take a value and move from it.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp                | 34 +++++++++----------
 src/neorados/RADOS.cc                         | 26 +++++++-------
 src/osdc/Objecter.h                           | 18 +++++++++-
 .../librados_test_stub/NeoradosTestStub.cc    |  8 ++---
 4 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index aaa1fea005cb..e89c7d02e41a 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -336,7 +336,7 @@ class Op {
   void set_fadvise_dontneed();
   void set_fadvise_nocache();
 
-  void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+  void cmpext(uint64_t off, ceph::buffer::list cmp_bl,
 	      uint64_t* unmatch = nullptr);
   void cmpxattr(std::string_view name, cmp_op op,
 		const ceph::buffer::list& val);
@@ -584,12 +584,12 @@ class ReadOp final : public Op {
     return std::move(*this);
   }
 
-  ReadOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+  ReadOp& cmpext(uint64_t off, ceph::buffer::list cmp_bl,
 		 uint64_t* unmatch = nullptr) & {
     Op::cmpext(off, std::move(cmp_bl), unmatch);
     return *this;
   }
-  ReadOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+  ReadOp&& cmpext(uint64_t off, ceph::buffer::list cmp_bl,
 		  uint64_t* unmatch = nullptr) && {
     Op::cmpext(off, std::move(cmp_bl), unmatch);
     return std::move(*this);
@@ -827,22 +827,22 @@ class WriteOp final : public Op {
   WriteOp&& create(bool exclusive) && {
     return std::move(create(exclusive));
   }
-  WriteOp& write(uint64_t off, ceph::buffer::list&& bl) &;
-  WriteOp&& write(uint64_t off, ceph::buffer::list&& bl) && {
+  WriteOp& write(uint64_t off, ceph::buffer::list bl) &;
+  WriteOp&& write(uint64_t off, ceph::buffer::list bl) && {
     return std::move(write(off, std::move(bl)));
   }
-  WriteOp& write_full(ceph::buffer::list&& bl) &;
-  WriteOp&& write_full(ceph::buffer::list&& bl) && {
+  WriteOp& write_full(ceph::buffer::list bl) &;
+  WriteOp&& write_full(ceph::buffer::list bl) && {
     return std::move(write_full(std::move(bl)));
   }
   WriteOp& writesame(std::uint64_t off, std::uint64_t write_len,
-		     ceph::buffer::list&& bl) &;
+		     ceph::buffer::list bl) &;
   WriteOp&& writesame(std::uint64_t off, std::uint64_t write_len,
-		      ceph::buffer::list&& bl) && {
+		      ceph::buffer::list bl) && {
     return std::move(writesame(off, write_len, std::move(bl)));
   }
-  WriteOp& append(ceph::buffer::list&& bl) &;
-  WriteOp&& append(ceph::buffer::list&& bl) && {
+  WriteOp& append(ceph::buffer::list bl) &;
+  WriteOp&& append(ceph::buffer::list bl) && {
     return std::move(append(std::move(bl)));
   }
   WriteOp& remove() &;
@@ -862,9 +862,9 @@ class WriteOp final : public Op {
     return std::move(rmxattr(name));
   }
   WriteOp& setxattr(std::string_view name,
-		    ceph::buffer::list&& bl) &;
+		    ceph::buffer::list bl) &;
   WriteOp&& setxattr(std::string_view name,
-		     ceph::buffer::list&& bl) && {
+		     ceph::buffer::list bl) && {
     return std::move(setxattr(name, std::move(bl)));
   }
   WriteOp& rollback(uint64_t snapid) &;
@@ -877,8 +877,8 @@ class WriteOp final : public Op {
     const boost::container::flat_map<std::string, ceph::buffer::list>& map) && {
     return std::move(set_omap(map));
   }
-  WriteOp& set_omap_header(ceph::buffer::list&& bl) &;
-  WriteOp&& set_omap_header(ceph::buffer::list&& bl) && {
+  WriteOp& set_omap_header(ceph::buffer::list bl) &;
+  WriteOp&& set_omap_header(ceph::buffer::list bl) && {
     return std::move(set_omap_header(std::move(bl)));
   }
   WriteOp& clear_omap() &;
@@ -964,12 +964,12 @@ class WriteOp final : public Op {
     return std::move(*this);
   }
 
-  WriteOp& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+  WriteOp& cmpext(uint64_t off, ceph::buffer::list cmp_bl,
 		  uint64_t* unmatch = nullptr) & {
     Op::cmpext(off, std::move(cmp_bl), unmatch);
     return *this;
   }
-  WriteOp&& cmpext(uint64_t off, ceph::buffer::list&& cmp_bl,
+  WriteOp&& cmpext(uint64_t off, ceph::buffer::list cmp_bl,
 		   uint64_t* unmatch = nullptr) && {
     Op::cmpext(off, std::move(cmp_bl), unmatch);
     return std::move(*this);
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index b04aa6c2b9be..91c55ba5f849 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -420,7 +420,7 @@ void Op::set_fadvise_nocache() {
     CEPH_OSD_OP_FLAG_FADVISE_NOCACHE);
 }
 
-void Op::cmpext(uint64_t off, bufferlist&& cmp_bl, uint64_t* unmatch) {
+void Op::cmpext(uint64_t off, bufferlist cmp_bl, uint64_t* unmatch) {
   reinterpret_cast<OpImpl*>(&impl)->op.cmpext(off, std::move(cmp_bl), nullptr,
 					      unmatch);
 }
@@ -676,23 +676,23 @@ WriteOp& WriteOp::create(bool exclusive) & {
   return *this;
 }
 
-WriteOp& WriteOp::write(uint64_t off, bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.write(off, bl);
+WriteOp& WriteOp::write(uint64_t off, bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.write(off, std::move(bl));
   return *this;
 }
 
-WriteOp& WriteOp::write_full(bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.write_full(bl);
+WriteOp& WriteOp::write_full(bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.write_full(std::move(bl));
   return *this;
 }
 
-WriteOp& WriteOp::writesame(uint64_t off, uint64_t write_len, bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.writesame(off, write_len, bl);
+WriteOp& WriteOp::writesame(uint64_t off, uint64_t write_len, bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.writesame(off, write_len, std::move(bl));
   return *this;
 }
 
-WriteOp& WriteOp::append(bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.append(bl);
+WriteOp& WriteOp::append(bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.append(std::move(bl));
   return *this;
 }
 
@@ -717,8 +717,8 @@ WriteOp& WriteOp::rmxattr(std::string_view name) & {
 }
 
 WriteOp& WriteOp::setxattr(std::string_view name,
-			   bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.setxattr(name, bl);
+			   bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.setxattr(name, std::move(bl));
   return *this;
 }
 
@@ -733,8 +733,8 @@ WriteOp& WriteOp::set_omap(
   return *this;
 }
 
-WriteOp& WriteOp::set_omap_header(bufferlist&& bl) & {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_set_header(bl);
+WriteOp& WriteOp::set_omap_header(bufferlist bl) & {
+  reinterpret_cast<OpImpl*>(&impl)->op.omap_set_header(std::move(bl));
   return *this;
 }
 
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 2fd8bf5fb46b..c54d87bace43 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -579,15 +579,27 @@ struct ObjectOperation {
   void write(uint64_t off, ceph::buffer::list& bl) {
     write(off, bl, 0, 0);
   }
+  void write(uint64_t off, ceph::buffer::list&& bl) {
+    write(off, bl, 0, 0);
+  }
   void write_full(ceph::buffer::list& bl) {
     add_data(CEPH_OSD_OP_WRITEFULL, 0, bl.length(), bl);
   }
+  void write_full(ceph::buffer::list&& bl) {
+    add_data(CEPH_OSD_OP_WRITEFULL, 0, bl.length(), bl);
+  }
   void writesame(uint64_t off, uint64_t write_len, ceph::buffer::list& bl) {
     add_writesame(CEPH_OSD_OP_WRITESAME, off, write_len, bl);
   }
+  void writesame(uint64_t off, uint64_t write_len, ceph::buffer::list&& bl) {
+    add_writesame(CEPH_OSD_OP_WRITESAME, off, write_len, bl);
+  }
   void append(ceph::buffer::list& bl) {
     add_data(CEPH_OSD_OP_APPEND, 0, bl.length(), bl);
   }
+  void append(ceph::buffer::list&& bl) {
+    add_data(CEPH_OSD_OP_APPEND, 0, bl.length(), bl);
+  }
   void zero(uint64_t off, uint64_t len) {
     ceph::buffer::list bl;
     add_data(CEPH_OSD_OP_ZERO, off, len, bl);
@@ -1357,7 +1369,11 @@ struct ObjectOperation {
     add_data(CEPH_OSD_OP_OMAPSETVALS, 0, bl.length(), bl);
   }
 
-  void omap_set_header(ceph::buffer::list &bl) {
+  void omap_set_header(ceph::buffer::list& bl) {
+    add_data(CEPH_OSD_OP_OMAPSETHEADER, 0, bl.length(), bl);
+  }
+
+  void omap_set_header(ceph::buffer::list&& bl) {
     add_data(CEPH_OSD_OP_OMAPSETHEADER, 0, bl.length(), bl);
   }
 
diff --git a/src/test/librados_test_stub/NeoradosTestStub.cc b/src/test/librados_test_stub/NeoradosTestStub.cc
index 1eb0d627ebee..a1b61f2459b3 100644
--- a/src/test/librados_test_stub/NeoradosTestStub.cc
+++ b/src/test/librados_test_stub/NeoradosTestStub.cc
@@ -331,7 +331,7 @@ void Op::assert_version(uint64_t ver) {
           &librados::TestIoCtxImpl::assert_version, _1, _2, ver));
 }
 
-void Op::cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, uint64_t* s) {
+void Op::cmpext(uint64_t off, ceph::buffer::list cmp_bl, uint64_t* s) {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   librados::ObjectOperationTestImpl op = std::bind(
     &librados::TestIoCtxImpl::cmpext, _1, _2, off, cmp_bl, _4);
@@ -504,14 +504,14 @@ WriteOp& WriteOp::create(bool exclusive) & {
   return *this;
 }
 
-WriteOp& WriteOp::write(uint64_t off, ceph::buffer::list&& bl) & {
+WriteOp& WriteOp::write(uint64_t off, ceph::buffer::list bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::write, _1, _2, bl, bl.length(), off, _5));
   return *this;
 }
 
-WriteOp& WriteOp::write_full(ceph::buffer::list&& bl) & {
+WriteOp& WriteOp::write_full(ceph::buffer::list bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::write_full, _1, _2, bl, _5));
@@ -540,7 +540,7 @@ WriteOp& WriteOp::zero(uint64_t off, uint64_t len) & {
 }
 
 WriteOp& WriteOp::writesame(std::uint64_t off, std::uint64_t write_len,
-			    ceph::buffer::list&& bl) & {
+			    ceph::buffer::list bl) & {
   auto o = *reinterpret_cast<librados::TestObjectOperationImpl**>(&impl);
   o->ops.push_back(std::bind(
     &librados::TestIoCtxImpl::writesame, _1, _2, bl, write_len, off, _5));

From 29ad5be02ca8483fb96965ff93094bdbd4d773dd Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 17 Aug 2023 14:39:15 -0400
Subject: [PATCH 0963/2492] neorados/test: Support erasure coded pools

The test framework in common test can now support erasure-coded pools
and be more easily extended generally.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/neorados/common_tests.cc | 42 ++++++++++++++++++++++++++++++
 src/test/neorados/common_tests.h  | 43 ++++++++++++++++++++++++++-----
 2 files changed, 78 insertions(+), 7 deletions(-)

diff --git a/src/test/neorados/common_tests.cc b/src/test/neorados/common_tests.cc
index 6f274551d95e..fa16f05ec877 100644
--- a/src/test/neorados/common_tests.cc
+++ b/src/test/neorados/common_tests.cc
@@ -30,3 +30,45 @@ std::string get_temp_pool_name(std::string_view prefix)
   static auto num = 1ull;
   return fmt::format("{}{}-{}-{}", prefix, hostname, getpid(), num++);
 }
+
+
+asio::awaitable<uint64_t> NeoRadosECTest::create_pool() {
+  // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110913
+  std::vector<std::string> profile_set = {
+    fmt::format(
+      R"({{"prefix": "osd erasure-code-profile set", "name": "testprofile-{}", )"
+      R"( "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}})",
+      pool_name())
+  };
+  co_await rados().mon_command(std::move(profile_set), {}, nullptr, nullptr,
+			       asio::use_awaitable);
+  std::vector<std::string> pool_create = {
+    fmt::format(
+      R"({{"prefix": "osd pool create", "pool": "{}", "pool_type":"erasure", )"
+      R"("pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-{}"}})",
+      pool_name(), pool_name())
+  };
+  auto c = rados().mon_command(std::move(pool_create), {}, nullptr, nullptr,
+			       asio::use_awaitable);
+  co_await std::move(c);
+
+  co_return co_await rados().lookup_pool(pool_name(), asio::use_awaitable);
+}
+
+asio::awaitable<void> NeoRadosECTest::clean_pool() {
+  co_await rados().delete_pool(pool().get_pool(), asio::use_awaitable);
+  std::vector<std::string> profile_rm = {
+    fmt::format(
+      R"({{"prefix": "osd erasure-code-profile rm", "name": "testprofile-{}"}})",
+      pool_name())
+  };
+  co_await rados().mon_command(std::move(profile_rm), {}, nullptr, nullptr, asio::use_awaitable);
+  std::vector<std::string> rule_rm = {
+    fmt::format(
+      R"({{"prefix": "osd crush rule rm", "name":"{}"}})",
+      pool_name())
+  };
+  co_await rados().mon_command(std::move(rule_rm), {}, nullptr, nullptr,
+			       asio::use_awaitable);
+  co_return;
+}
diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index ae150b7e9430..c92bcd230b9f 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -220,7 +220,9 @@ class CoroTest : public testing::Test {
 /// CoTestBody has access to `rados`, a `neorados::RADOS` handle, and
 /// `pool`, a `neorados::IOContext` representing a pool that will be
 /// destroyed when the test exits.
-class NeoRadosTest : public CoroTest {
+///
+/// Derived classes must define `create_pool()` and `clean_pool()`.
+class NeoRadosTestBase : public CoroTest {
 private:
   const std::string prefix_{std::string{"test framework "} +
 			    testing::UnitTest::GetInstance()->
@@ -228,11 +230,14 @@ class NeoRadosTest : public CoroTest {
 			    std::string{": "}};
 
   std::optional<neorados::RADOS> rados_;
+  neorados::IOContext pool_;
   const std::string pool_name_ = get_temp_pool_name(
     testing::UnitTest::GetInstance()->current_test_info()->name());
-  neorados::IOContext pool_;
   std::unique_ptr<DoutPrefix> dpp_;
 
+  virtual boost::asio::awaitable<uint64_t> create_pool() = 0;
+  virtual boost::asio::awaitable<void> clean_pool() = 0;
+
 protected:
 
   /// \brief Return reference to RADOS
@@ -291,21 +296,45 @@ class NeoRadosTest : public CoroTest {
     rados_ = co_await neorados::RADOS::Builder{}
       .build(asio_context, boost::asio::use_awaitable);
     dpp_ = std::make_unique<DoutPrefix>(rados().cct(), 0, prefix().data());
-    pool_.set_pool(co_await create_pool(rados(), pool_name(),
-				    boost::asio::use_awaitable));
+    pool_.set_pool(co_await create_pool());
     co_return;
   }
 
-  ~NeoRadosTest() override = default;
+  ~NeoRadosTestBase() override = default;
 
   /// \brief Delete pool used for testing
   boost::asio::awaitable<void> CoTearDown() override {
-    co_await rados().delete_pool(pool().get_pool(),
-				 boost::asio::use_awaitable);
+    co_await clean_pool();
     co_return;
   }
 };
 
+/// \brief C++20 coroutine test harness for NeoRados on normal pools
+///
+/// The supplied pool is not erasure coded.
+class NeoRadosTest : public NeoRadosTestBase {
+private:
+  boost::asio::awaitable<uint64_t> create_pool() override {
+    co_return co_await ::create_pool(rados(), pool_name(),
+				     boost::asio::use_awaitable);
+  }
+
+  boost::asio::awaitable<void> clean_pool() override {
+    co_await rados().delete_pool(pool().get_pool(),
+				boost::asio::use_awaitable);
+  }
+};
+
+/// \brief C++20 coroutine test harness for NeoRados on erasure-coded
+/// pools
+///
+/// The supplied pool is erasure coded
+class NeoRadosECTest : public NeoRadosTestBase {
+private:
+  boost::asio::awaitable<uint64_t> create_pool() override;
+  boost::asio::awaitable<void> clean_pool() override;
+};
+
 /// \brief Helper macro for defining coroutine tests with a fixture
 ///
 /// Defines a test using a coroutine fixture for

From b8a192380201121a3fd9cf49f8ee570606d63218 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Fri, 18 Aug 2023 18:07:22 -0400
Subject: [PATCH 0964/2492] neorados: Make neorados::Entry aggregate
 initializable

Now that C++20 has designated initializers, this is the least
confusing way to handle objects like this.

Also switch to using the spaceship operator, while we're at it.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS_Decodable.hpp | 28 +-----------------------
 1 file changed, 1 insertion(+), 27 deletions(-)

diff --git a/src/include/neorados/RADOS_Decodable.hpp b/src/include/neorados/RADOS_Decodable.hpp
index 5e3a0d36eabb..777a2525c8e0 100644
--- a/src/include/neorados/RADOS_Decodable.hpp
+++ b/src/include/neorados/RADOS_Decodable.hpp
@@ -35,34 +35,8 @@ struct Entry {
   std::string oid;
   std::string locator;
 
-  Entry() {}
-  Entry(std::string nspace, std::string oid, std::string locator) :
-    nspace(std::move(nspace)), oid(std::move(oid)), locator(locator) {}
+  friend auto operator <=>(const Entry&, const Entry&) = default;
 };
-inline bool operator ==(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) ==
-    std::tie(r.nspace, r.oid, r.locator);
-}
-inline bool operator !=(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) !=
-    std::tie(r.nspace, r.oid, r.locator);
-}
-inline bool operator <(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) <
-    std::tie(r.nspace, r.oid, r.locator);
-}
-inline bool operator <=(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) <=
-    std::tie(r.nspace, r.oid, r.locator);
-}
-inline bool operator >=(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) >=
-    std::tie(r.nspace, r.oid, r.locator);
-}
-inline bool operator >(const Entry& l, const Entry r) {
-  return std::tie(l.nspace, l.oid, l.locator) >
-    std::tie(r.nspace, r.oid, r.locator);
-}
 
 inline std::ostream& operator <<(std::ostream& out, const Entry& entry) {
   if (!entry.nspace.empty())

From cd9964944ed202f86c8aaef3f8135ba3dda70a11 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 21 Aug 2023 17:28:05 -0400
Subject: [PATCH 0965/2492] neorados: Moved-from operations are equivalent to
 new

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/neorados/RADOS.cc | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 91c55ba5f849..b8d2303a38dc 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -369,10 +369,22 @@ struct OpImpl {
   OpImpl() = default;
 
   OpImpl(const OpImpl& rhs) = delete;
-  OpImpl(OpImpl&& rhs) = default;
+  OpImpl(OpImpl&& rhs)
+    : op(std::move(rhs.op)), mtime(std::move(rhs.mtime)) {
+    rhs.op = ObjectOperation{};
+    rhs.mtime.reset();
+  }
 
   OpImpl& operator =(const OpImpl& rhs) = delete;
-  OpImpl& operator =(OpImpl&& rhs) = default;
+  OpImpl& operator =(OpImpl&& rhs) {
+    op = std::move(rhs.op);
+    mtime = std::move(rhs.mtime);
+
+    rhs.op = ObjectOperation{};
+    rhs.mtime.reset();
+
+    return *this;
+  }
 };
 
 Op::Op() {

From 1076a098a24c1dc68204c94721ed01fd1f9c9980 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 14 Aug 2023 16:01:02 -0400
Subject: [PATCH 0966/2492] test/neorados: Rounding out test coverage, part 2

This includes write_operations, io, misc, and pool.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/neorados/CMakeLists.txt      |  80 +++++
 src/test/neorados/common_tests.h      |  35 ++
 src/test/neorados/io.cc               | 471 ++++++++++++++++++++++++++
 src/test/neorados/list.cc             | 247 ++++++++++++++
 src/test/neorados/misc.cc             | 293 ++++++++++++++++
 src/test/neorados/pool.cc             | 173 ++++++++++
 src/test/neorados/write_operations.cc | 193 +++++++++++
 7 files changed, 1492 insertions(+)
 create mode 100644 src/test/neorados/io.cc
 create mode 100644 src/test/neorados/list.cc
 create mode 100644 src/test/neorados/misc.cc
 create mode 100644 src/test/neorados/pool.cc
 create mode 100644 src/test/neorados/write_operations.cc

diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index bfb201506aae..ceb64c85bedf 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -89,3 +89,83 @@ target_link_libraries(ceph_test_neorados_read_operations
 install(TARGETS
   ceph_test_neorados_read_operations
   DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_write_operations
+  write_operations.cc
+  )
+target_link_libraries(ceph_test_neorados_write_operations
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_write_operations
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_io
+  io.cc
+  )
+target_link_libraries(ceph_test_neorados_io
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_io
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_list
+  list.cc
+  )
+target_link_libraries(ceph_test_neorados_list
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_list
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_misc
+  misc.cc
+  )
+target_link_libraries(ceph_test_neorados_misc
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_misc
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_pool
+  pool.cc
+  )
+target_link_libraries(ceph_test_neorados_pool
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_pool
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index c92bcd230b9f..eecebe20f9ef 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -279,6 +279,18 @@ class NeoRadosTestBase : public CoroTest {
 			   boost::asio::use_awaitable, ver);
   }
 
+  auto execute(std::string_view oid, neorados::WriteOp&& op,
+	       neorados::IOContext ioc, std::uint64_t* ver = nullptr) {
+    return rados().execute(oid, std::move(ioc), std::move(op),
+			   boost::asio::use_awaitable, ver);
+  }
+
+  auto execute(std::string_view oid, neorados::ReadOp&& op,
+	       neorados::IOContext ioc, std::uint64_t* ver = nullptr) {
+    return rados().execute(oid, std::move(ioc), std::move(op), nullptr,
+			   boost::asio::use_awaitable, ver);
+  }
+
   boost::asio::awaitable<ceph::buffer::list>
   read(std::string_view oid, std::uint64_t off = 0, std::uint64_t len = 0) {
     ceph::buffer::list bl;
@@ -289,6 +301,17 @@ class NeoRadosTestBase : public CoroTest {
     co_return bl;
   }
 
+  boost::asio::awaitable<ceph::buffer::list>
+  read(std::string_view oid, neorados::IOContext ioc, std::uint64_t off = 0,
+       std::uint64_t len = 0) {
+    ceph::buffer::list bl;
+    neorados::ReadOp op;
+    op.read(off, len, &bl);
+    co_await rados().execute(oid, std::move(ioc), std::move(op),
+			     nullptr, boost::asio::use_awaitable);
+    co_return bl;
+  }
+
 public:
 
   /// \brief Create RADOS handle and pool for the test
@@ -475,3 +498,15 @@ inline auto from_buffer_list(const ceph::buffer::list& bl)
   decode(t, bi);
   return t;
 }
+
+inline bool is_crimson_cluster() {
+  return getenv("CRIMSON_COMPAT") != nullptr;
+}
+
+// Yet more nonsense caused by Google's ridiculous except-o-phobia.
+
+#define SKIP_IF_CRIMSON()                                                      \
+  if (is_crimson_cluster()) {                                                  \
+    std::cerr << "Not supported by crimson yet. Skipped" << std::endl;         \
+    co_return;                                                                 \
+  }
diff --git a/src/test/neorados/io.cc b/src/test/neorados/io.cc
new file mode 100644
index 000000000000..3cc2d4d4c993
--- /dev/null
+++ b/src/test/neorados/io.cc
@@ -0,0 +1,471 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <array>
+#include <coroutine>
+#include <cstdint>
+#include <limits>
+#include <utility>
+
+#include <fmt/format.h>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_map.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "include/buffer.h"
+#include "include/stringify.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace buffer = ceph::buffer;
+namespace container = boost::container;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+static constexpr auto oid = "oid"sv;
+
+CORO_TEST_F(NeoRadosIo, Limits, NeoRadosTest) {
+  co_await expect_error_code(
+    execute(oid, WriteOp{}
+	    .write(std::numeric_limits<std::uint64_t>::max(), {})),
+    sys::errc::file_too_large);
+  co_await expect_error_code(
+    execute(oid, WriteOp{}
+	    .writesame(0, std::numeric_limits<std::uint64_t>::max(), {})),
+    sys::errc::invalid_argument);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, SimpleWrite, NeoRadosTest) {
+  static constexpr auto nspace = "nspace";
+  auto pool2 = pool();
+  const auto bl = filled_buffer_list(0xcc, 128);
+
+  pool2.set_ns(nspace);
+  EXPECT_EQ(nspace, pool2.get_ns());
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl));
+    auto resbl = co_await read(oid);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl), pool2);
+    auto resbl = co_await read(oid, pool2);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, ReadOp, NeoRadosTest) {
+  const auto refbl = filled_buffer_list(0xcc, 128);
+
+  co_await execute(oid, WriteOp{}.write_full(refbl));
+  {
+    buffer::list op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+
+  {
+    buffer::list read_bl, read_bl2, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(), ReadOp{}
+			     .read(0, 0, &read_bl)
+			     .read(0, 0, &read_bl2),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, read_bl2);
+    buffer::list bl2;
+    bl2.append(refbl);
+    bl2.append(refbl);
+    EXPECT_EQ(bl2, op_bl);
+  }
+  {
+    // Read into buffer with a cached crc
+    auto op_bl = filled_buffer_list('z', refbl.length());
+    EXPECT_NE(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+    EXPECT_EQ(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+  }
+
+  co_return;
+}
+
+void expect_eq_sparse(
+    const buffer::list& expected,
+    const std::vector<std::pair<std::uint64_t, std::uint64_t>>& extents,
+    const buffer::list& actual)
+{
+  auto i = expected.begin();
+  auto p = actual.begin();
+  uint64_t pos = 0;
+  for (auto extent : extents) {
+    const uint64_t start = extent.first;
+    const uint64_t end = start + extent.second;
+    for (; pos < end; ++i, ++pos) {
+      EXPECT_FALSE(i.end());
+      if (pos < start) {
+        // check the hole
+        EXPECT_EQ('\0', *i);
+      } else {
+        // then the extent
+        EXPECT_EQ(*i, *p);
+        ++p;
+      }
+    }
+  }
+  EXPECT_EQ(expected.length(), pos);
+}
+
+
+CORO_TEST_F(NeoRadosIo, SparseRead, NeoRadosTest) {
+  {
+    const auto refbl = filled_buffer_list(0xcc, 4'096);
+    co_await execute(oid, WriteOp{}.write_full(refbl));
+
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> extents;
+    buffer::list readbl;
+    co_await execute(oid, ReadOp{}.sparse_read(0, refbl.length(), &readbl, &extents));
+    expect_eq_sparse(refbl, extents, readbl);
+    EXPECT_EQ(refbl, readbl);
+  }
+  {
+    buffer::list refbl;
+    refbl.append(filled_buffer_list(0xcc, 4'096));
+    refbl.append(filled_buffer_list(0x00, 4'096));
+    refbl.append(filled_buffer_list(0xdd, 4'096));
+    refbl.append(filled_buffer_list(0x00, 4'096));
+    refbl.append(filled_buffer_list(0xee, 4'096));
+    co_await execute(oid, WriteOp{}.write_full(refbl));
+
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> extents;
+    buffer::list readbl;
+    co_await execute(oid, ReadOp{}
+		     .sparse_read(0, refbl.length(), &readbl, &extents));
+    expect_eq_sparse(refbl, extents, readbl);
+  }
+}
+
+CORO_TEST_F(NeoRadosIo, RoundTrip, NeoRadosTest) {
+  const auto refbl = filled_buffer_list(0xcc, 128);
+  co_await execute(oid, WriteOp{}.write_full(refbl));
+  {
+    auto bl = co_await read(oid);
+    EXPECT_EQ(refbl, bl);
+  }
+  {
+    buffer::list bl;
+    ReadOp op;
+    op.read(0, 0, & bl)
+      .set_fadvise_nocache()
+      .set_fadvise_random();
+    co_await execute(oid, std::move(op));
+    EXPECT_EQ(refbl, bl);
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, ReadIntoBuufferlist, NeoRadosTest) {
+  auto refbl = filled_buffer_list(0xcc, 128);
+  co_await execute(oid, WriteOp{}.write_full(refbl));
+  {
+    // here we test reading into a non-empty bufferlist referencing existing
+    // buffers
+    std::array<char, 128> buf;
+    buf.fill(0xbb);
+    buffer::list bl2;
+    bl2.append(buffer::create_static(buf.size(), buf.data()));
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &bl2, asio::use_awaitable);
+    EXPECT_EQ(refbl, bl2);
+    EXPECT_EQ(0, memcmp(refbl.c_str(), buf.data(), buf.size()));
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, OverlappingWriteRoundTrip, NeoRadosTest) {
+  const auto buf1 = filled_buffer_list(0xcc, 128);
+  const auto buf2 = filled_buffer_list(0xdd, 64);
+  co_await execute(oid, WriteOp{}.write(0, buf1));
+  co_await execute(oid, WriteOp{}.write(0, buf2));
+
+  buffer::list buf3 = buf2;
+  buf3.append(filled_buffer_list(0xcc, 64));
+  auto resbl = co_await read(oid);
+  EXPECT_EQ(buf3, resbl);
+  co_return;
+}
+
+
+CORO_TEST_F(NeoRadosIo, WriteFullRoundTrip, NeoRadosTest) {
+  {
+    const auto buf1 = filled_buffer_list(0xcc, 128);
+    const auto buf2 = filled_buffer_list(0xdd, 64);
+    co_await execute(oid, WriteOp{}.write_full(buf1));
+    co_await execute(oid, WriteOp{}.write_full(buf2));
+    auto resbl = co_await read(oid);
+    EXPECT_EQ(buf2, resbl);
+  }
+  {
+    const auto bl = to_buffer_list("ceph");
+    co_await execute(oid, WriteOp()
+		     .write_full(bl)
+		     .set_fadvise_nocache());
+
+    buffer::list resbl;
+    co_await execute(oid, ReadOp()
+		     .read(0, 0, &resbl).balance_reads()
+		     .set_fadvise_dontneed()
+		     .set_fadvise_random());
+    EXPECT_EQ(bl, resbl);
+  }
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, AppendRoundTrip, NeoRadosTest) {
+  const auto buf1 = filled_buffer_list(0xde, 64);
+  const auto buf2 = filled_buffer_list(0xad, 64);
+  co_await execute(oid, WriteOp{}.append(buf1));
+  co_await execute(oid, WriteOp{}.append(buf2));
+  auto resbl = co_await read(oid);
+  auto cmpbl = buf1;
+  cmpbl.append(buf2);
+  EXPECT_EQ(cmpbl, resbl);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, Trunc, NeoRadosTest) {
+  const auto buf = filled_buffer_list(0xaa, 128);
+  co_await execute(oid, WriteOp{}.append(buf));
+  co_await execute(oid, WriteOp{}.truncate(buf.length() / 2));
+  const auto resbl = co_await read(oid);
+  buffer::list cmpbl;
+  cmpbl.substr_of(buf, 0, buf.length() / 2);
+  EXPECT_EQ(cmpbl, resbl);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, Remove, NeoRadosTest) {
+  co_await execute(oid, WriteOp{}.create(true));
+  co_await execute(oid, ReadOp{}.stat(nullptr, nullptr));
+  co_await execute(oid, WriteOp{}.remove());
+  co_await expect_error_code(execute(oid, WriteOp{}.remove()),
+			     sys::errc::no_such_file_or_directory);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, XattrsRoundTrip, NeoRadosTest) {
+  const auto obj_buf = filled_buffer_list(0xaa, 128);
+  const auto attrkey = "attr1"sv;
+  const auto attrval = to_buffer_list("foo bar baz");
+  co_await execute(oid, WriteOp{}.append(obj_buf));
+  buffer::list attrval_res;
+
+  co_await expect_error_code(execute(oid,
+				     ReadOp{}.get_xattr(attrkey, &attrval_res)),
+			     sys::errc::no_message_available);
+  EXPECT_EQ(0, attrval_res.length());
+
+  co_await execute(oid, WriteOp{}.setxattr(attrkey, attrval));
+  co_await execute(oid, ReadOp{}.get_xattr(attrkey, &attrval_res));
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, RmXattr, NeoRadosTest) {
+  const auto objbl= filled_buffer_list(0xaa, 128);
+  const auto attrkey = "attr1"sv;
+  const auto attrval = to_buffer_list("foo bar baz");
+
+  co_await execute(oid, WriteOp{}.append(objbl));
+
+  co_await expect_error_code(execute(oid,
+				     ReadOp{}.get_xattr(attrkey, nullptr)),
+			     sys::errc::no_message_available);
+  co_await execute(oid, WriteOp{}.setxattr(attrkey, attrval));
+  co_await execute(oid, ReadOp{}.get_xattr(attrkey, nullptr));
+
+  co_await execute(oid, WriteOp{}.rmxattr(attrkey));
+  co_await expect_error_code(execute(oid,
+				     ReadOp{}.get_xattr(attrkey, nullptr)),
+			     sys::errc::no_message_available);
+
+  // Test rmxattr of a removed object
+  co_await execute(oid, WriteOp{}.remove());
+  co_await expect_error_code(execute(oid,
+				     WriteOp{}.rmxattr(attrkey)),
+			     sys::errc::no_such_file_or_directory);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosIo, GetXattrs, NeoRadosTest) {
+  const auto objbl= filled_buffer_list(0xaa, 128);
+  const auto attrkey1 = "attr1"s;
+  const auto attrval1 = to_buffer_list("foo bar baz");
+  const auto attrkey2 = "attr2"s;
+  std::array<char, 256> attrbuf2;
+  for (auto i = 0u; i < attrbuf2.size(); ++i) {
+    attrbuf2[i] = i % 0xff;
+  }
+  buffer::list attrval2;
+  attrval2.append(attrbuf2.data(), attrbuf2.size());
+
+  co_await execute(oid, WriteOp{}
+		   .append(objbl)
+		   .setxattr(attrkey1, attrval1)
+		   .setxattr(attrkey2, attrval2));
+
+  container::flat_map<std::string, buffer::list> attrset;
+  co_await execute(oid, ReadOp{}.get_xattrs(&attrset));
+  EXPECT_EQ(2, attrset.size());
+  EXPECT_EQ(attrval1, attrset[attrkey1]);
+  EXPECT_EQ(attrval2, attrset[attrkey2]);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosECIo, SimpleWrite, NeoRadosECTest) {
+  SKIP_IF_CRIMSON();
+  static constexpr auto nspace = "nspace";
+  auto pool2 = pool();
+  const auto bl = filled_buffer_list(0xcc, 128);
+
+  pool2.set_ns(nspace);
+  EXPECT_EQ(nspace, pool2.get_ns());
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl));
+    auto resbl = co_await read(oid);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl), pool2);
+    auto resbl = co_await read(oid, pool2);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosECIo, ReadOp, NeoRadosECTest) {
+  SKIP_IF_CRIMSON();
+  const auto refbl = filled_buffer_list(0xcc, 128);
+
+  co_await execute(oid, WriteOp{}.write_full(refbl));
+  {
+    buffer::list op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+
+  {
+    buffer::list read_bl, read_bl2, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(), ReadOp{}
+			     .read(0, 0, &read_bl)
+			     .read(0, 0, &read_bl2),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, read_bl2);
+    buffer::list bl2;
+    bl2.append(refbl);
+    bl2.append(refbl);
+    EXPECT_EQ(bl2, op_bl);
+  }
+  {
+    // Read into buffer with a cached crc
+    auto op_bl = filled_buffer_list('z', refbl.length());
+    EXPECT_NE(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+    EXPECT_EQ(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+  }
+
+  co_return;
+}
diff --git a/src/test/neorados/list.cc b/src/test/neorados/list.cc
new file mode 100644
index 000000000000..56cc042a2b88
--- /dev/null
+++ b/src/test/neorados/list.cc
@@ -0,0 +1,247 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstdint>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_set.hpp>
+
+#include <fmt/format.h>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace container = boost::container;
+
+using namespace std::literals;
+
+using neorados::Cursor;
+using neorados::IOContext;
+using neorados::WriteOp;
+
+using Entries = std::vector<neorados::Entry>;
+using REntries = container::flat_set<neorados::Entry>;
+
+CORO_TEST_F(NeoradosList, ListObjects, NeoRadosTest) {
+  static constexpr auto oid = "foo";
+  co_await execute(oid, WriteOp{}.create(true));
+  auto [entries, cursor] = co_await
+    rados().enumerate_objects(pool(), Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  EXPECT_EQ(1, entries.size());
+  EXPECT_EQ(oid, entries.front().oid);
+  co_return;
+}
+
+
+asio::awaitable<void> populate(neorados::RADOS& rados, const IOContext& pool,
+			       const REntries& entries) {
+  for (const auto& entry : entries) {
+    co_await ::create_obj(rados, entry.oid, pool, asio::use_awaitable);
+  }
+  co_return;
+};
+
+void compare(const REntries& ref, const Entries& res) {
+  EXPECT_EQ(ref.size(), res.size());
+  for (const auto& e : res) {
+    EXPECT_TRUE(ref.contains(e));
+  }
+  return;
+};
+
+CORO_TEST_F(NeoradosList, ListObjectsNS, NeoRadosTest) {
+  auto pdef = pool();
+  IOContext p1{pool().get_pool(), "ns1"};
+  IOContext p2{pool().get_pool(), "ns2"};
+  IOContext pall{pool().get_pool(), neorados::all_nspaces};
+
+  neorados::Entry meow{.oid="foo1"s};
+  REntries def{
+    {.oid = "foo1"s},
+    {.oid = "foo2"s},
+    {.oid = "foo3"s}
+  };
+  REntries ns1{
+    {.nspace = "ns1"s, .oid = "foo1"s},
+    {.nspace = "ns1"s, .oid = "foo4"s},
+    {.nspace = "ns1"s, .oid = "foo5"s},
+    {.nspace = "ns1"s, .oid = "foo6"s},
+    {.nspace = "ns1"s, .oid = "foo7"s}
+  };
+  REntries ns2{
+    {.nspace = "ns2"s, .oid = "foo6"s},
+    {.nspace = "ns2"s, .oid = "foo7"s}
+  };
+  REntries all{def};
+  all.insert(ns1.begin(), ns1.end());
+  all.insert(ns2.begin(), ns2.end());
+
+  co_await populate(rados(), pdef, def);
+  co_await populate(rados(), p1, ns1);
+  co_await populate(rados(), p2, ns2);
+
+  auto [resdef, cdef] = co_await
+    rados().enumerate_objects(pdef, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res1, c1] = co_await
+    rados().enumerate_objects(p1, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res2, c2] = co_await
+    rados().enumerate_objects(p2, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [resall, call] = co_await
+    rados().enumerate_objects(pall, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  compare(def, resdef);
+  compare(ns1, res1);
+  compare(ns2, res2);
+  compare(all, resall);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoradosList, ListObjectsMany, NeoRadosTest) {
+  REntries ref;
+  for (auto i = 0u; i < 512; ++i) {
+    ref.insert({.oid = fmt::format("{:0>3}", i)});
+  }
+  co_await populate(rados(), pool(), ref);
+  REntries res;
+  {
+    Cursor c;
+    Entries e;
+    static constexpr auto per = 10;
+    e.reserve(per);
+    while (c != Cursor::end()) {
+      std::tie(e, c) = co_await
+	rados().enumerate_objects(pool(), c, Cursor::end(), per, {},
+				  asio::use_awaitable);
+      for (auto&& n : e) {
+	res.insert(std::move(n));
+      }
+      e.clear();
+    }
+  }
+  EXPECT_EQ(ref, res);
+
+  co_return;
+}
+
+// Sadly I don't think there's a good way to templatize testcases over
+// fixture.
+
+
+CORO_TEST_F(NeoradosECList, ListObjects, NeoRadosECTest) {
+  SKIP_IF_CRIMSON();
+  static constexpr auto oid = "foo";
+  co_await execute(oid, WriteOp{}.create(true));
+  auto [entries, cursor] = co_await
+    rados().enumerate_objects(pool(), Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  EXPECT_EQ(1, entries.size());
+  EXPECT_EQ(oid, entries.front().oid);
+  co_return;
+}
+
+CORO_TEST_F(NeoradosECList, ListObjectsNS, NeoRadosECTest) {
+  SKIP_IF_CRIMSON();
+  auto pdef = pool();
+  IOContext p1{pool().get_pool(), "ns1"};
+  IOContext p2{pool().get_pool(), "ns2"};
+  IOContext pall{pool().get_pool(), neorados::all_nspaces};
+
+  neorados::Entry meow{.oid="foo1"s};
+  REntries def{
+    {.oid = "foo1"s},
+    {.oid = "foo2"s},
+    {.oid = "foo3"s}
+  };
+  REntries ns1{
+    {.nspace = "ns1"s, .oid = "foo1"s},
+    {.nspace = "ns1"s, .oid = "foo4"s},
+    {.nspace = "ns1"s, .oid = "foo5"s},
+    {.nspace = "ns1"s, .oid = "foo6"s},
+    {.nspace = "ns1"s, .oid = "foo7"s}
+  };
+  REntries ns2{
+    {.nspace = "ns2"s, .oid = "foo6"s},
+    {.nspace = "ns2"s, .oid = "foo7"s}
+  };
+  REntries all{def};
+  all.insert(ns1.begin(), ns1.end());
+  all.insert(ns2.begin(), ns2.end());
+
+  co_await populate(rados(), pdef, def);
+  co_await populate(rados(), p1, ns1);
+  co_await populate(rados(), p2, ns2);
+
+  auto [resdef, cdef] = co_await
+    rados().enumerate_objects(pdef, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res1, c1] = co_await
+    rados().enumerate_objects(p1, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res2, c2] = co_await
+    rados().enumerate_objects(p2, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [resall, call] = co_await
+    rados().enumerate_objects(pall, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  compare(def, resdef);
+  compare(ns1, res1);
+  compare(ns2, res2);
+  compare(all, resall);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoradosECList, ListObjectsMany, NeoRadosECTest) {
+  SKIP_IF_CRIMSON();
+  REntries ref;
+  for (auto i = 0u; i < 512; ++i) {
+    ref.insert({.oid = fmt::format("{:0>3}", i)});
+  }
+  co_await populate(rados(), pool(), ref);
+  REntries res;
+  {
+    Cursor c;
+    Entries e;
+    static constexpr auto per = 10;
+    e.reserve(per);
+    while (c != Cursor::end()) {
+      std::tie(e, c) = co_await
+	rados().enumerate_objects(pool(), c, Cursor::end(), per, {},
+				  asio::use_awaitable);
+      for (auto&& n : e) {
+	res.insert(std::move(n));
+      }
+      e.clear();
+    }
+  }
+  EXPECT_EQ(ref, res);
+
+  co_return;
+}
diff --git a/src/test/neorados/misc.cc b/src/test/neorados/misc.cc
new file mode 100644
index 000000000000..d9c007b47e2c
--- /dev/null
+++ b/src/test/neorados/misc.cc
@@ -0,0 +1,293 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstdint>
+#include <utility>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "include/buffer.h"
+
+#include "common/ceph_context.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace buffer = ceph::buffer;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+static constexpr auto oid = "oid"sv;
+
+CORO_TEST_F(NeoRadosMisc, Version, NeoRadosTest) {
+  [[maybe_unused]] auto [major, minor, point] = neorados::RADOS::version();
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, WaitOSDMap, NeoRadosTest) {
+  co_await rados().wait_for_latest_osd_map(asio::use_awaitable);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, LongName, NeoRadosTest) {
+  const auto maxlen = rados().cct()->_conf->osd_max_object_name_len;
+  const auto bl = to_buffer_list("content"sv);
+  co_await execute(std::string(maxlen / 2, 'a'),
+                   WriteOp{}.write_full(bl));
+  co_await execute(std::string(maxlen - 1, 'a'),
+                   WriteOp{}.write_full(bl));
+  co_await execute(std::string(maxlen, 'a'),
+                   WriteOp{}.write_full(bl));
+
+  co_await expect_error_code(execute(std::string(maxlen + 1, 'a'),
+                                     WriteOp{}.write_full(bl)),
+                             sys::errc::filename_too_long);
+  co_await expect_error_code(execute(std::string(maxlen * 2, 'a'),
+                                     WriteOp{}.write_full(bl)),
+                             sys::errc::filename_too_long);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, LongLocator, NeoRadosTest) {
+  const auto maxlen = rados().cct()->_conf->osd_max_object_name_len;
+  const auto bl = to_buffer_list("content"sv);
+  {
+    auto p = pool();
+    p.set_key(std::string(maxlen / 2, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_key(std::string(maxlen - 1, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_key(std::string(maxlen, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_key(std::string(maxlen + 1, 'a'));
+    co_await expect_error_code(execute(oid,
+				       WriteOp{}.write_full(bl), p),
+			       sys::errc::filename_too_long);
+  }
+  {
+    auto p = pool();
+    p.set_key(std::string(maxlen * 2, 'a'));
+    co_await expect_error_code(execute(oid,
+				       WriteOp{}.write_full(bl), p),
+			       sys::errc::filename_too_long);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, LongNamespace, NeoRadosTest) {
+  const auto maxlen = rados().cct()->_conf->osd_max_object_namespace_len;
+  const auto bl = to_buffer_list("content"sv);
+  {
+    auto p = pool();
+    p.set_ns(std::string(maxlen / 2, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_ns(std::string(maxlen - 1, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_ns(std::string(maxlen, 'a'));
+    co_await execute(oid,
+                     WriteOp{}.write_full(bl), p);
+  }
+  {
+    auto p = pool();
+    p.set_ns(std::string(maxlen + 1, 'a'));
+    co_await expect_error_code(execute(oid,
+				       WriteOp{}.write_full(bl), p),
+			       sys::errc::filename_too_long);
+  }
+  {
+    auto p = pool();
+    p.set_ns(std::string(maxlen * 2, 'a'));
+    co_await expect_error_code(execute(oid,
+				       WriteOp{}.write_full(bl), p),
+			       sys::errc::filename_too_long);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, LongAttrName, NeoRadosTest) {
+  const auto maxlen = rados().cct()->_conf->osd_max_attr_name_len;
+  const auto bl = to_buffer_list("content"sv);
+
+  co_await execute(oid, WriteOp{}.setxattr(std::string(maxlen / 2, 'a'), bl));
+  co_await execute(oid, WriteOp{}.setxattr(std::string(maxlen - 1, 'a'), bl));
+  co_await execute(oid, WriteOp{}.setxattr(std::string(maxlen, 'a'), bl));
+
+  co_await expect_error_code(
+    execute(oid, WriteOp{}.setxattr(std::string(maxlen + 1, 'a'), bl)),
+    sys::errc::filename_too_long);
+  co_await expect_error_code(
+    execute(oid, WriteOp{}.setxattr(std::string(maxlen * 2, 'a'), bl)),
+    sys::errc::filename_too_long);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, Exec, NeoRadosTest) {
+  buffer::list out;
+  co_await execute(oid, WriteOp{}.create(true));
+  co_await execute(oid,
+		   ReadOp{}.exec("rbd"sv, "get_all_features"sv, {}, &out));
+  auto features = from_buffer_list<std::uint64_t>(out);
+  // make sure *some* features are specified; don't care which ones
+  EXPECT_NE(0, features);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, Operate1, NeoRadosTest) {
+  static constexpr auto key1 = "key1"sv;
+  const auto val1 = to_buffer_list("val1\0"sv);
+  {
+    WriteOp op;
+    op.write(0, {})
+      .setxattr(key1, val1)
+      // Should not affect xattr
+      .clear_omap();
+    co_await execute(oid, std::move(op));
+  }
+
+  // Op is empty now
+  co_await execute(oid, WriteOp{});
+  {
+    buffer::list bl;
+    co_await execute(oid, ReadOp{}.get_xattr(key1, &bl));
+    EXPECT_EQ(val1, bl);
+  }
+  const auto notval1 = to_buffer_list("val1"sv);
+  co_await expect_error_code(execute(oid, WriteOp{}
+				     .cmpxattr(key1, neorados::cmp_op::eq, val1)
+				     .rmxattr(key1)),
+			     sys::errc::operation_canceled);
+  co_await expect_error_code(
+    execute(oid, WriteOp{} .cmpxattr(key1, neorados::cmp_op::eq, val1)),
+    sys::errc::operation_canceled);
+
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, Operate2, NeoRadosTest) {
+  static constexpr auto key1 = "key1"sv;
+  const auto val1 = to_buffer_list("val1\0"sv);
+  WriteOp op;
+  op.write(0, to_buffer_list("abcdefg"sv))
+    .setxattr(key1, val1)
+    .truncate(0);
+  co_await execute(oid, std::move(op));
+  std::uint64_t size;
+  co_await execute(oid, ReadOp{}.stat(&size, nullptr));
+  EXPECT_EQ(0, size);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, BigObject, NeoRadosTest) {
+  const auto data = to_buffer_list("abcdefg"sv);
+  co_await execute(oid, WriteOp{}.write(0, data));
+
+  co_await expect_error_code(execute(oid, WriteOp{}.truncate(500000000000ull)),
+			     sys::errc::file_too_large);
+  co_await expect_error_code(execute(oid, WriteOp{}.zero(500000000000ull, 1)),
+			     sys::errc::file_too_large);
+  co_await expect_error_code(execute(oid, WriteOp{}.zero(1, 500000000000ull)),
+			     sys::errc::file_too_large);
+  co_await expect_error_code(execute(oid, WriteOp{}.zero(500000000000ull,
+							 500000000000ull)),
+			     sys::errc::file_too_large);
+#ifdef __LP64__
+  co_await expect_error_code(execute(oid, WriteOp{}.write(500000000000ull, data)),
+			     sys::errc::file_too_large);
+#endif // __LP64__
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, BigAttr, NeoRadosTest) {
+  const auto maxlen = rados().cct()->_conf->osd_max_attr_size;
+  if (maxlen > 0) {
+    buffer::list attrval;
+    attrval.append(buffer::create(maxlen));
+    co_await execute(oid, WriteOp{}.setxattr("one"sv, attrval));
+
+    attrval.clear();
+    attrval.append(buffer::create(maxlen + 1));
+    co_await expect_error_code(execute(oid, WriteOp()
+				       .setxattr("one"sv, attrval)),
+			       sys::errc::file_too_large);
+  } else {
+    SUCCEED() << "osd_max_attr_size == 0, skipping test." << std::endl;
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosMisc, WriteSame, NeoRadosTest) {
+  static constexpr auto patlen = 128u;
+  static constexpr auto samelen = patlen * 4;
+  static constexpr char fill = 0xcc;
+  const auto patbl = filled_buffer_list(fill, patlen);
+  const auto refbl = filled_buffer_list(fill, samelen);
+
+  // Zero the full range before using `writesame`
+  co_await execute(oid, WriteOp{}.zero(0, patlen));
+
+  // Write the same pattern four times
+  co_await execute(oid, WriteOp{}.writesame(0, samelen, patbl));
+  auto resbl = co_await read(oid);
+  EXPECT_EQ(refbl, resbl);
+
+  // Write length must be a multiple of the pattern length
+  co_await expect_error_code(execute(oid, WriteOp{}
+				     .writesame(0, samelen - 1, patbl)),
+			     sys::errc::invalid_argument);
+
+  // Write length is the same as pattern length (same as write)
+  co_await execute(oid, WriteOp{}.truncate(0));
+  co_await execute(oid, WriteOp{}.writesame(0, patbl.length(), patbl));
+  resbl = co_await read(oid);
+  EXPECT_EQ(patbl, resbl);
+
+
+  co_return;
+}
+
+// We already have tests for cmpext and checksum. The rest uses
+// currently unimplemented functionality.
diff --git a/src/test/neorados/pool.cc b/src/test/neorados/pool.cc
new file mode 100644
index 000000000000..9ca5b95c504f
--- /dev/null
+++ b/src/test/neorados/pool.cc
@@ -0,0 +1,173 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <boost/system/detail/errc.hpp>
+#include <coroutine>
+#include <cstdint>
+#include <unordered_set>
+#include <utility>
+
+#include <fmt/format.h>
+
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "common/error_code.h"
+
+#include "osdc/error_code.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+// We want to be able to test pool functionality, to clean up after
+// ourselves, and not create a footgun where someone wipes out their
+// entire Ceph cluster by running a test against it. So track all
+// pools we create during a test and remove them after.
+class NeoRadosPool : public CoroTest {
+private:
+  std::optional<neorados::RADOS> rados_;
+  std::unique_ptr<DoutPrefix> dpp_;
+  const std::string prefix_{
+    fmt::format("Test framework: {}: ",
+		testing::UnitTest::GetInstance()->
+		current_test_info()->name())};
+
+  std::unordered_set<std::string> created_pools;
+
+protected:
+
+  /// \brief Return reference to RADOS
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  neorados::RADOS& rados() noexcept { return *rados_; }
+
+  /// \brief Return DoutPrefixProvider*
+  ///
+  /// \warning This function should only be called from test bodies
+  /// (i.e. after `CoSetUp()`)
+  const DoutPrefixProvider* dpp() const noexcept { return dpp_.get(); }
+
+  /// \brief Return prefix for this test run
+  std::string_view prefix() const noexcept { return prefix_; }
+
+  auto lookup_pool(std::string pname) {
+    return rados().lookup_pool(pname, asio::use_awaitable);
+  }
+
+  // Create a pool and track it
+  asio::awaitable<std::int64_t>
+  create_pool(std::string pname, std::optional<int> crush_rule = std::nullopt) {
+    co_await rados().create_pool(pname, crush_rule, asio::use_awaitable);
+    created_pools.insert(pname);
+    co_return co_await lookup_pool(pname);
+  }
+
+  auto delete_pool(std::string pname) {
+    return rados().delete_pool(pname, asio::use_awaitable);
+  }
+
+  auto delete_pool(std::int64_t pid) {
+    return rados().delete_pool(pid, asio::use_awaitable);
+  }
+
+public:
+
+  /// \brief Create RADOS handle for the test
+  boost::asio::awaitable<void> CoSetUp() override {
+    rados_ = co_await neorados::RADOS::Builder{}
+      .build(asio_context, boost::asio::use_awaitable);
+    dpp_ = std::make_unique<DoutPrefix>(rados().cct(), 0, "NeoRadosPoolTest");
+    co_return;
+  }
+
+  ~NeoRadosPool() override = default;
+
+  /// \brief Delete pool used for testing
+  boost::asio::awaitable<void> CoTearDown() override {
+    for (const auto& name : created_pools) try {
+	co_await delete_pool(name);
+      } catch (const sys::system_error& e) {
+	if (e.code() != osdc_errc::pool_dne) {
+	  throw;
+	}
+      }
+    co_return;
+  }
+};
+
+CORO_TEST_F(NeoRadosPools, PoolList, NeoRadosPool) {
+  const auto pname = get_temp_pool_name();
+  co_await create_pool(pname);
+  auto pools = co_await rados().list_pools(asio::use_awaitable);
+  EXPECT_FALSE(pools.empty());
+  EXPECT_TRUE(
+    std::find_if(pools.begin(),
+		 pools.end(),
+		 [&pname](const auto& kv) {return kv.second == pname;})
+    != pools.end());
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosPools, PoolLookup, NeoRadosPool) {
+  const auto pname = get_temp_pool_name();
+  const auto refpid = co_await create_pool(pname);
+  auto respid = co_await lookup_pool(pname);
+  EXPECT_EQ(refpid, respid);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosPools, PoolLookupOtherInstance, NeoRadosPool) {
+  auto rados2 = co_await neorados::RADOS::Builder{}
+    .build(asio_context, asio::use_awaitable);
+  const auto pname = get_temp_pool_name();
+  const auto refpid = co_await create_pool(pname);
+  auto respid = co_await rados2.lookup_pool(pname, asio::use_awaitable);
+  EXPECT_EQ(refpid, respid);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosPools, PoolDelete, NeoRadosPool) {
+  const auto pname = get_temp_pool_name();
+  co_await create_pool(pname);
+  co_await delete_pool(pname);
+  co_await expect_error_code(lookup_pool(pname),
+			     sys::errc::no_such_file_or_directory);
+  co_await create_pool(pname);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosPools, PoolCreateDelete, NeoRadosPool) {
+  const auto pname = get_temp_pool_name();
+  co_await create_pool(pname);
+  co_await expect_error_code(create_pool(pname), ceph::errc::exists);
+  co_await delete_pool(pname);
+  co_await expect_error_code(delete_pool(pname), ceph::errc::does_not_exist);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosPools, PoolCreateWithCrushRule, NeoRadosPool) {
+  const auto pname = get_temp_pool_name();
+  co_await create_pool(pname, 0);
+  co_await delete_pool(pname);
+  co_return;
+}
diff --git a/src/test/neorados/write_operations.cc b/src/test/neorados/write_operations.cc
new file mode 100644
index 000000000000..7dd720295d89
--- /dev/null
+++ b/src/test/neorados/write_operations.cc
@@ -0,0 +1,193 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstring>
+#include <cstdint>
+#include <memory>
+#include <string_view>
+#include <utility>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/system/error_code.hpp>
+#include <boost/system/errc.hpp>
+
+#include <fmt/format.h>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "osd/error_code.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::cmp_op;
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+constexpr auto oid = "test"sv;
+
+CORO_TEST_F(NeoRadosWriteOps, AssertExists, NeoRadosTest) {
+  co_await expect_error_code(execute(oid, WriteOp{}.assert_exists()),
+			     sys::errc::no_such_file_or_directory);
+  co_await execute(oid, WriteOp{}.create(true));
+  co_await execute(oid, WriteOp{}.assert_exists());
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, AssertVersion, NeoRadosTest) {
+  co_await execute(oid, WriteOp{}.create(true));
+  std::uint64_t v;
+  // Write to the object a second time to guarantee that its
+  // version number is greater than 0
+  co_await execute(oid, WriteOp{}.write_full(to_buffer_list("hi")), &v);
+
+  co_await expect_error_code(execute(oid, WriteOp{}.assert_version(v + 1)),
+			     sys::errc::value_too_large);
+  co_await expect_error_code(execute(oid, WriteOp{}.assert_version(v - 1)),
+			     sys::errc::result_out_of_range);
+  co_await execute(oid, WriteOp{}.assert_version(v));
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, Xattrs, NeoRadosTest) {
+  // Hey, the compiler won't check that I don't have typos in my strings…
+  static constexpr auto key = "key"sv;
+  const auto value = to_buffer_list("value");
+  // Create an object with an xattr
+  co_await execute(oid, WriteOp{}
+		   .create(true)
+		   .setxattr(key, value));
+  // Check that xattr exists, if it does, delete it.
+  co_await execute(oid, WriteOp{}
+		   .cmpxattr("key", cmp_op::eq, value)
+		   .rmxattr(key));
+
+  // Check the xattr exits, if it does, add it again (will fail)
+  co_await expect_error_code(execute(oid, WriteOp{}
+				     .cmpxattr(key, cmp_op::eq, value)
+				     .setxattr(key, value)),
+			     sys::errc::operation_canceled);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, Write, NeoRadosTest) {
+  // Create an object, write and write full to it
+  {
+    const auto value = to_buffer_list("hi");
+    co_await execute(oid, WriteOp{}
+		     .write(0, to_buffer_list("four"))
+		     .write_full(value));
+    auto bl = co_await read(oid);
+    EXPECT_EQ(value, bl);
+  }
+  // Create write op with I/O hint
+  {
+    const auto value = to_buffer_list("ceph");
+    co_await execute(oid, WriteOp{}
+		     .write_full(value)
+		     .set_fadvise_nocache());
+    auto bl = co_await read(oid);
+    EXPECT_EQ(value, bl);
+  }
+  // Truncate and append
+  {
+    co_await execute(oid, WriteOp{}
+		     .truncate(1)
+		     .append(to_buffer_list("hi")));
+    auto bl = co_await read(oid);
+    EXPECT_EQ(to_buffer_list("chi"), bl);
+  }
+  // Zero and remove
+  {
+    co_await execute(oid, WriteOp{}
+		     .zero(0, 3)
+		     .remove());
+    co_await expect_error_code(execute(oid, ReadOp{}.read(0, 0, nullptr)),
+			       sys::errc::no_such_file_or_directory);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, Exec, NeoRadosTest) {
+  co_await execute(oid, WriteOp{}
+		   .exec("hello"sv, "record_hello"sv,
+			 to_buffer_list("test")));
+  const auto bl = co_await read(oid);
+  EXPECT_EQ(to_buffer_list("Hello, test!"), bl);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, WriteSame, NeoRadosTest) {
+  co_await execute(oid, WriteOp{}
+		   .writesame(0, 4 * 4, // Total bytes, not total copies
+			      to_buffer_list("four")));
+  const auto bl = co_await read(oid);
+  EXPECT_EQ(to_buffer_list("fourfourfourfour"), bl);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWriteOps, CmpExt, NeoRadosTest) {
+  static const auto four = to_buffer_list("four");
+  static const auto five = to_buffer_list("five");
+  static const auto six = to_buffer_list("six");
+
+  // Create an object, write to it
+  {
+    co_await execute(oid, WriteOp{}
+		     .create(true)
+		     .write_full(four));
+    const auto bl = co_await read(oid);
+    EXPECT_EQ(four, bl);
+  }
+  // Compare and overwrite on (expected) match
+  {
+    uint64_t unmatch = 0;
+    co_await execute(oid, WriteOp{}
+		     .cmpext(0, four, &unmatch)
+		     .write(0, five));
+    const auto bl = co_await read(oid);
+    EXPECT_EQ(five, bl);
+    EXPECT_EQ(-1, unmatch);
+  }
+  // check offset return error value
+  {
+    uint64_t unmatch = -2;
+    co_await expect_error_code(execute(oid, WriteOp()
+				       .cmpext(0, four, &unmatch)
+				       .write(0, six)
+				       .returnvec()),
+			       osd_errc::cmpext_mismatch);
+    // 'four' mistmatches 'five' on character 1.
+    EXPECT_EQ(1, unmatch);
+  }
+  // Compare and bail before write due to mismatch. Do it a thousand
+  // times to make sure we are hitting some socket injection
+  for (auto i = 0; i < 1000; ++i) {
+    uint64_t unmatch = -2;
+    co_await expect_error_code(execute(fmt::format("test_{}", i), WriteOp()
+				       .cmpext(0, four, &unmatch)
+				       .write(0, six)
+				       .returnvec()),
+			       osd_errc::cmpext_mismatch);
+    EXPECT_EQ(0, unmatch);
+    EXPECT_EQ(0, unmatch);
+  }
+  co_return;
+}

From ecd9411dee235cea0b69baa2a825b7f85d69f63f Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Fri, 8 Sep 2023 12:48:39 -0400
Subject: [PATCH 0967/2492] neorados: `execute` returns success on empty
 operations

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/neorados/RADOS.cc     |  8 ++++++++
 src/test/neorados/misc.cc | 13 +++++++------
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index b8d2303a38dc..d8d1abbe2973 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -905,6 +905,10 @@ void RADOS::execute_(Object o, IOContext _ioc, ReadOp _op,
 		     cb::list* bl,
 		     ReadOp::Completion c, version_t* objver,
 		     const blkin_trace_info *trace_info) {
+  if (_op.size() == 0) {
+    asio::dispatch(asio::append(std::move(c), bs::error_code{}));
+    return;
+  }
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
   auto op = reinterpret_cast<OpImpl*>(&_op.impl);
@@ -927,6 +931,10 @@ void RADOS::execute_(Object o, IOContext _ioc, ReadOp _op,
 void RADOS::execute_(Object o, IOContext _ioc, WriteOp _op,
 		     WriteOp::Completion c, version_t* objver,
 		     const blkin_trace_info *trace_info) {
+  if (_op.size() == 0) {
+    asio::dispatch(asio::append(std::move(c), bs::error_code{}));
+    return;
+  }
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
   auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
   auto op = reinterpret_cast<OpImpl*>(&_op.impl);
diff --git a/src/test/neorados/misc.cc b/src/test/neorados/misc.cc
index d9c007b47e2c..2c761bd7c7dc 100644
--- a/src/test/neorados/misc.cc
+++ b/src/test/neorados/misc.cc
@@ -193,15 +193,16 @@ CORO_TEST_F(NeoRadosMisc, Operate1, NeoRadosTest) {
     co_await execute(oid, ReadOp{}.get_xattr(key1, &bl));
     EXPECT_EQ(val1, bl);
   }
+  // Comparisons differing in NUL termination.
   const auto notval1 = to_buffer_list("val1"sv);
-  co_await expect_error_code(execute(oid, WriteOp{}
-				     .cmpxattr(key1, neorados::cmp_op::eq, val1)
-				     .rmxattr(key1)),
-			     sys::errc::operation_canceled);
   co_await expect_error_code(
-    execute(oid, WriteOp{} .cmpxattr(key1, neorados::cmp_op::eq, val1)),
+    execute(oid, WriteOp{}
+	    .cmpxattr(key1, neorados::cmp_op::eq, notval1)
+	    .rmxattr(key1)),
+    sys::errc::operation_canceled);
+  co_await expect_error_code(
+    execute(oid, WriteOp{}.cmpxattr(key1, neorados::cmp_op::eq, notval1)),
     sys::errc::operation_canceled);
-
 
   co_return;
 }

From 0dd6486ca6e05a50f72321733685e74e62509db6 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 29 Aug 2023 18:13:35 -0400
Subject: [PATCH 0968/2492] osdc: with_osdmap should be const

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/osdc/Objecter.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index c54d87bace43..201b7b62d08f 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -2710,7 +2710,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   // here or you will have great woe and misery.
 
   template<typename Callback, typename...Args>
-  decltype(auto) with_osdmap(Callback&& cb, Args&&... args) {
+  decltype(auto) with_osdmap(Callback&& cb, Args&&... args) const {
     std::shared_lock l(rwlock);
     return std::forward<Callback>(cb)(*osdmap, std::forward<Args>(args)...);
   }

From ccc1b89aa98ec0ad6f9e92e238c120870c3195bf Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 29 Aug 2023 18:13:55 -0400
Subject: [PATCH 0969/2492] neorados: More snapshot support

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp |  46 +++++++++-
 src/neorados/RADOS.cc          | 163 ++++++++++++++++++++++++++++++++-
 2 files changed, 207 insertions(+), 2 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index e89c7d02e41a..ab95bdb17993 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -120,6 +120,9 @@ class Object final {
   std::aligned_storage_t<impl_size> impl;
 };
 
+inline constexpr std::uint64_t snap_dir = -1;
+inline constexpr std::uint64_t snap_head = -2;
+
 // Not the same as the librados::IoCtx, but it does gather together
 // some of the same metadata. Since we're likely to do multiple
 // operations in the same pool or namespace, it doesn't make sense to
@@ -1449,6 +1452,13 @@ class RADOS final
       }, consigned);
   }
 
+  template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
+  auto create_pool_snap(const IOContext& pool, std::string snap_name,
+			CompletionToken&& token) {
+    return create_pool_snap(pool.get_pool(), std::move(snap_name),
+			    std::forward<CompletionToken>(token));
+  }
+
   using SMSnapSig = void(boost::system::error_code, std::uint64_t);
   using SMSnapComp = boost::asio::any_completion_handler<SMSnapSig>;
   template<boost::asio::completion_token_for<SMSnapSig> CompletionToken>
@@ -1458,7 +1468,7 @@ class RADOS final
 	boost::asio::get_associated_executor(token, get_executor())));
     return boost::asio::async_initiate<decltype(consigned), SMSnapSig>(
       [pool, this](auto&& handler) mutable {
-	allocage_selfmanaged_snap_(pool, std::move(handler));
+	allocate_selfmanaged_snap_(pool, std::move(handler));
       }, consigned);
   }
 
@@ -1487,6 +1497,39 @@ class RADOS final
       }, consigned);
   }
 
+  bool get_self_managed_snaps_mode(std::int64_t pool) const;
+  bool get_self_managed_snaps_mode(std::string_view pool) const;
+  bool get_self_managed_snaps_mode(const IOContext& pool) const {
+    return get_self_managed_snaps_mode(pool.get_pool());
+  }
+
+  std::vector<std::uint64_t> list_snaps(std::int64_t pool) const;
+  std::vector<std::uint64_t> list_snaps(std::string_view pool) const;
+  std::vector<std::uint64_t> list_snaps(const IOContext& pool) const {
+    return list_snaps(pool.get_pool());
+  }
+
+  std::uint64_t lookup_snap(std::int64_t pool, std::string_view snap) const;
+  std::uint64_t lookup_snap(std::string_view pool, std::string_view snap) const;
+  std::uint64_t lookup_snap(const IOContext& pool, std::string_view snap) const {
+    return lookup_snap(pool.get_pool(), snap);
+  }
+
+  std::string get_snap_name(std::int64_t pool, std::uint64_t snap) const;
+  std::string get_snap_name(std::string_view pool, std::uint64_t snap) const;
+  std::string get_snap_name(const IOContext& pool, std::uint64_t snap) const {
+    return get_snap_name(pool.get_pool(), snap);
+  }
+
+  ceph::real_time get_snap_timestamp(std::int64_t pool,
+				     std::uint64_t snap) const;
+  ceph::real_time get_snap_timestamp(std::string_view pool,
+				     std::uint64_t snap) const;
+  ceph::real_time get_snap_timestamp(const IOContext& pool,
+				     std::uint64_t snap) const {
+    return get_snap_timestamp(pool.get_pool(), snap);
+  }
+
   template<boost::asio::completion_token_for<SimpleOpSig> CompletionToken>
   auto create_pool(std::string name, std::optional<int> crush_rule,
 		   CompletionToken&& token) {
@@ -1840,6 +1883,7 @@ class RADOS final
 
 enum class errc {
   pool_dne = 1,
+  snap_dne,
   invalid_snapcontext
 };
 
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index d8d1abbe2973..9a71b1269a16 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -1079,6 +1079,157 @@ void RADOS::delete_selfmanaged_snap_(std::int64_t pool,
       }));
 }
 
+bool RADOS::get_self_managed_snaps_mode(std::int64_t pool) const {
+  return impl->objecter->with_osdmap([pool](const OSDMap& osdmap) {
+    const auto pgpool = osdmap.get_pg_pool(pool);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    return pgpool->is_unmanaged_snaps_mode();
+  });
+}
+
+bool RADOS::get_self_managed_snaps_mode(std::string_view pool) const {
+  return impl->objecter->with_osdmap([pool](const OSDMap& osdmap) {
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool);
+    if (poolid < 0) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    const auto pgpool = osdmap.get_pg_pool(poolid);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    return pgpool->is_unmanaged_snaps_mode();
+  });
+}
+
+std::vector<std::uint64_t> RADOS::list_snaps(std::int64_t pool) const {
+  return impl->objecter->with_osdmap([pool](const OSDMap& osdmap) {
+    const auto pgpool = osdmap.get_pg_pool(pool);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    std::vector<std::uint64_t> snaps;
+    for (const auto& [snapid, snapinfo] : pgpool->snaps) {
+      snaps.push_back(snapid);
+    }
+    return snaps;
+  });
+}
+
+std::vector<std::uint64_t> RADOS::list_snaps(std::string_view pool) const {
+  return impl->objecter->with_osdmap([pool](const OSDMap& osdmap) {
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool);
+    if (poolid < 0) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    const auto pgpool = osdmap.get_pg_pool(poolid);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    std::vector<std::uint64_t> snaps;
+    for (const auto& [snapid, snapinfo] : pgpool->snaps) {
+      snaps.push_back(snapid);
+    }
+    return snaps;
+  });
+}
+
+std::uint64_t RADOS::lookup_snap(std::int64_t pool, std::string_view snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    const auto pgpool = osdmap.get_pg_pool(pool);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    for (const auto& [id, snapinfo] : pgpool->snaps) {
+      if (snapinfo.name == snap) return id;
+    }
+    throw bs::system_error(bs::error_code(errc::snap_dne));
+  });
+}
+
+std::uint64_t RADOS::lookup_snap(std::string_view pool, std::string_view snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool);
+    if (poolid < 0) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    const auto pgpool = osdmap.get_pg_pool(poolid);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    for (const auto& [id, snapinfo] : pgpool->snaps) {
+      if (snapinfo.name == snap) return id;
+    }
+    throw bs::system_error(bs::error_code(errc::snap_dne));
+  });
+}
+
+std::string RADOS::get_snap_name(std::int64_t pool, std::uint64_t snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    const auto pgpool = osdmap.get_pg_pool(pool);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    if (auto i = pgpool->snaps.find(snap); i == pgpool->snaps.cend()) {
+      throw bs::system_error(bs::error_code(errc::snap_dne));
+    } else {
+      return i->second.name;
+    }
+  });
+}
+std::string RADOS::get_snap_name(std::string_view pool,
+				 std::uint64_t snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool);
+    if (poolid < 0) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    const auto pgpool = osdmap.get_pg_pool(poolid);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    if (auto i = pgpool->snaps.find(snap); i == pgpool->snaps.cend()) {
+      throw bs::system_error(bs::error_code(errc::snap_dne));
+    } else {
+      return i->second.name;
+    }
+  });
+}
+
+ceph::real_time RADOS::get_snap_timestamp(std::int64_t pool,
+					  std::uint64_t snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    const auto pgpool = osdmap.get_pg_pool(pool);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    if (auto i = pgpool->snaps.find(snap); i == pgpool->snaps.cend()) {
+      throw bs::system_error(bs::error_code(errc::snap_dne));
+    } else {
+      return i->second.stamp.to_real_time();
+    }
+  });
+}
+ceph::real_time RADOS::get_snap_timestamp(std::string_view pool,
+					  std::uint64_t snap) const {
+  return impl->objecter->with_osdmap([pool, snap](const OSDMap& osdmap) {
+    int64_t poolid = osdmap.lookup_pg_pool_name(pool);
+    if (poolid < 0) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    const auto pgpool = osdmap.get_pg_pool(poolid);
+    if (!pgpool) {
+      throw bs::system_error(bs::error_code(errc::pool_dne));
+    }
+    if (auto i = pgpool->snaps.find(snap); i == pgpool->snaps.cend()) {
+      throw bs::system_error(bs::error_code(errc::snap_dne));
+    } else {
+      return i->second.stamp.to_real_time();
+    }
+  });
+}
+
 void RADOS::create_pool_(std::string name,
 			 std::optional<int> crush_rule,
 			 SimpleOpComp c)
@@ -1604,7 +1755,8 @@ const char* category::message(int ev, char*,
   switch (static_cast<errc>(ev)) {
   case errc::pool_dne:
     return "Pool does not exist";
-
+  case errc::snap_dne:
+    return "Snapshot does not exist";
   case errc::invalid_snapcontext:
     return "Invalid snapcontext";
   }
@@ -1620,6 +1772,8 @@ bs::error_condition category::default_error_condition(int ev) const noexcept {
   switch (static_cast<errc>(ev)) {
   case errc::pool_dne:
     return ceph::errc::does_not_exist;
+  case errc::snap_dne:
+    return ceph::errc::does_not_exist;
   case errc::invalid_snapcontext:
     return bs::errc::invalid_argument;
   }
@@ -1633,6 +1787,11 @@ bool category::equivalent(int ev, const bs::error_condition& c) const noexcept {
       return true;
     }
   }
+  if (static_cast<errc>(ev) == errc::snap_dne) {
+    if (c == bs::errc::no_such_file_or_directory) {
+      return true;
+    }
+  }
 
   return default_error_condition(ev) == c;
 }
@@ -1641,6 +1800,8 @@ int category::from_code(int ev) const noexcept {
   switch (static_cast<errc>(ev)) {
   case errc::pool_dne:
     return -ENOENT;
+  case errc::snap_dne:
+    return -ENOENT;
   case errc::invalid_snapcontext:
     return -EINVAL;
   }

From 7dc969e489f0262c1655e78f0bc3df0871e9e300 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 7 Sep 2023 17:35:40 -0400
Subject: [PATCH 0970/2492] neorados: Validate pointers from watch cookies

Since Objecter has a set used to validate pointers converted from
watch cookies before dereferencing them, expose a function to do so.

Also make the set a `std::unordered_set` since we use it for only this
purpose.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp |  9 +++++----
 src/neorados/RADOS.cc          | 10 +++++++---
 src/osdc/Objecter.h            |  8 +++++++-
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index ab95bdb17993..dbc4aff3c5ac 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -494,9 +494,9 @@ class ReadOp final : public Op {
     return std::move(get_omap_vals_by_keys(keys, kv, ec));
   }
 
-  ReadOp& list_watchers(std::vector<struct ObjWatcher>* watchers,
+  ReadOp& list_watchers(std::vector<ObjWatcher>* watchers,
 			boost::system::error_code* ec = nullptr) &;
-  ReadOp&& list_watchers(std::vector<struct ObjWatcher>* watchers,
+  ReadOp&& list_watchers(std::vector<ObjWatcher>* watchers,
 			 boost::system::error_code* ec = nullptr) && {
     return std::move(list_watchers(watchers, ec));
   }
@@ -1663,6 +1663,9 @@ class RADOS final
       }, consigned);
   }
 
+  tl::expected<ceph::timespan, boost::system::error_code>
+  check_watch(uint64_t cookie);
+
   using NotifySig = void(boost::system::error_code, ceph::buffer::list);
   using NotifyComp = boost::asio::any_completion_handler<NotifySig>;
   template<boost::asio::completion_token_for<NotifySig> CompletionToken>
@@ -1832,8 +1835,6 @@ class RADOS final
   void watch_(Object o, IOContext ioc,
 	      std::optional<std::chrono::seconds> timeout,
 	      WatchCB cb, WatchComp c);
-  tl::expected<ceph::timespan, boost::system::error_code>
-  watch_check_(uint64_t cookie);
   void notify_ack_(Object o, IOContext _ioc,
 		   uint64_t notify_id,
 		   uint64_t cookie,
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 9a71b1269a16..9b743f6e9862 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -1375,10 +1375,14 @@ void RADOS::notify_ack_(Object o, IOContext _ioc,
 		       nullptr, ioc->extra_op_flags, std::move(c));
 }
 
-tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check_(uint64_t cookie)
+tl::expected<ceph::timespan, bs::error_code> RADOS::check_watch(uint64_t cookie)
 {
-  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-  return impl->objecter->linger_check(linger_op);
+  auto linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
+  if (impl->objecter->is_valid_watch(linger_op)) {
+    return impl->objecter->linger_check(linger_op);
+  } else {
+    return tl::unexpected(bs::error_code(ENOTCONN, bs::generic_category()));
+  }
 }
 
 void RADOS::unwatch_(uint64_t cookie, IOContext _ioc,
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 201b7b62d08f..6daf57b928d3 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -2532,7 +2532,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
   std::map<uint64_t, LingerOp*> linger_ops;
   // we use this just to confirm a cookie is valid before dereferencing the ptr
-  std::set<LingerOp*> linger_ops_set;
+  std::unordered_set<LingerOp*> linger_ops_set;
 
   std::map<ceph_tid_t,PoolStatOp*> poolstat_ops;
   std::map<ceph_tid_t,StatfsOp*> statfs_ops;
@@ -2614,6 +2614,12 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   friend class CB_Objecter_GetVersion;
   friend class CB_DoWatchError;
 public:
+
+  bool is_valid_watch(LingerOp* op) {
+    std::shared_lock l(rwlock);
+    return linger_ops_set.contains(op);
+  }
+
   template<typename CT>
   auto linger_callback_flush(CT&& ct) {
     auto consigned = boost::asio::consign(

From 1d6b9c8052db95311ccff9b6c0cebbefc1bcd114 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Fri, 8 Sep 2023 16:50:46 -0400
Subject: [PATCH 0971/2492] neorados: Notify fixes

1. Objecter expects a count of seconds, not milliseconds
2. Actually return result

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/neorados/RADOS.hpp | 4 ++--
 src/neorados/RADOS.cc          | 9 ++++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/include/neorados/RADOS.hpp b/src/include/neorados/RADOS.hpp
index dbc4aff3c5ac..a9c9cf1462db 100644
--- a/src/include/neorados/RADOS.hpp
+++ b/src/include/neorados/RADOS.hpp
@@ -1670,7 +1670,7 @@ class RADOS final
   using NotifyComp = boost::asio::any_completion_handler<NotifySig>;
   template<boost::asio::completion_token_for<NotifySig> CompletionToken>
   auto notify(Object o, IOContext ioc, ceph::buffer::list bl,
-	      std::optional<std::chrono::milliseconds> timeout,
+	      std::optional<std::chrono::seconds> timeout,
 	      CompletionToken&& token) {
     auto consigned = boost::asio::consign(
       std::forward<CompletionToken>(token), boost::asio::make_work_guard(
@@ -1844,7 +1844,7 @@ class RADOS final
 		SimpleOpComp);
   void notify_(Object oid, IOContext ioctx,
 	       ceph::buffer::list bl,
-	       std::optional<std::chrono::milliseconds> timeout,
+	       std::optional<std::chrono::seconds> timeout,
 	       NotifyComp c);
   void flush_watch_(VoidOpComp);
 
diff --git a/src/neorados/RADOS.cc b/src/neorados/RADOS.cc
index 9b743f6e9862..3d97ead1ff7c 100644
--- a/src/neorados/RADOS.cc
+++ b/src/neorados/RADOS.cc
@@ -1451,8 +1451,9 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
 		  bufferlist&& bl) {
     asio::post(
       strand,
-      [this, ec, p = shared_from_this()]() mutable {
+      [this, ec, bl = std::move(bl), p = shared_from_this()]() mutable {
 	finished = true;
+	rbl = std::move(bl);
 	maybe_cleanup(ec);
       });
   }
@@ -1470,7 +1471,7 @@ struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
 };
 
 void RADOS::notify_(Object o, IOContext _ioc, bufferlist bl,
-		    std::optional<std::chrono::milliseconds> timeout,
+		    std::optional<std::chrono::seconds> timeout,
 		    NotifyComp c)
 {
   auto oid = reinterpret_cast<const object_t*>(&o.impl);
@@ -1490,9 +1491,11 @@ void RADOS::notify_(Object o, IOContext _ioc, bufferlist bl,
       });
   ObjectOperation rd;
   bufferlist inbl;
+  // 30s is the default in librados. Use that rather than borrowing from CephFS.
+  // TODO add a config option later.
   rd.notify(
     linger_op->get_cookie(), 1,
-    timeout ? timeout->count() : impl->cct->_conf->client_notify_timeout,
+    timeout.value_or(30s).count(),
     bl, &inbl);
 
   impl->objecter->linger_notify(

From dce23263d642640bae1f656d0266d53e3146dac0 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Tue, 29 Aug 2023 18:14:26 -0400
Subject: [PATCH 0972/2492] test/neorados: Rounding out test coverage, part 3

This includes snapshots and watch_notify

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/neorados/CMakeLists.txt   |  32 +++
 src/test/neorados/common_tests.h   |  20 ++
 src/test/neorados/handler_error.cc |   2 +-
 src/test/neorados/snapshots.cc     | 431 +++++++++++++++++++++++++++++
 src/test/neorados/watch_notify.cc  | 168 +++++++++++
 5 files changed, 652 insertions(+), 1 deletion(-)
 create mode 100644 src/test/neorados/snapshots.cc
 create mode 100644 src/test/neorados/watch_notify.cc

diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index ceb64c85bedf..97b99a04e5d7 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -169,3 +169,35 @@ target_link_libraries(ceph_test_neorados_pool
 install(TARGETS
   ceph_test_neorados_pool
   DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_snapshots
+  snapshots.cc
+  )
+target_link_libraries(ceph_test_neorados_snapshots
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_snapshots
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+add_executable(ceph_test_neorados_watch_notify
+  watch_notify.cc
+  )
+target_link_libraries(ceph_test_neorados_watch_notify
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_watch_notify
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index eecebe20f9ef..9610f08f643f 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <chrono>
 #include <concepts>
 #include <coroutine>
 #include <cstddef>
@@ -32,6 +33,7 @@
 #include <boost/asio/deferred.hpp>
 #include <boost/asio/detached.hpp>
 #include <boost/asio/io_context.hpp>
+#include <boost/asio/steady_timer.hpp>
 #include <boost/asio/use_awaitable.hpp>
 
 #include <boost/asio/experimental/co_composed.hpp>
@@ -312,6 +314,14 @@ class NeoRadosTestBase : public CoroTest {
     co_return bl;
   }
 
+  boost::asio::awaitable<void>
+  create_obj(std::string_view oid) {
+    neorados::WriteOp op;
+    op.create(true);
+    co_return co_await rados().execute(oid, pool(), std::move(op),
+				       boost::asio::use_awaitable);
+  }
+
 public:
 
   /// \brief Create RADOS handle and pool for the test
@@ -510,3 +520,13 @@ inline bool is_crimson_cluster() {
     std::cerr << "Not supported by crimson yet. Skipped" << std::endl;         \
     co_return;                                                                 \
   }
+
+/// \brief Wait for a specified time
+///
+/// \param dur Time to wait.
+template<typename Rep, typename Period>
+boost::asio::awaitable<void> wait_for(std::chrono::duration<Rep, Period> dur)
+{
+  boost::asio::steady_timer t(co_await boost::asio::this_coro::executor, dur);
+  co_return co_await t.async_wait(boost::asio::use_awaitable);
+}
diff --git a/src/test/neorados/handler_error.cc b/src/test/neorados/handler_error.cc
index 26d468bc0246..7f09c4e7d754 100644
--- a/src/test/neorados/handler_error.cc
+++ b/src/test/neorados/handler_error.cc
@@ -35,7 +35,7 @@ namespace buffer = ceph::buffer;
 CORO_TEST_F(neocls_handler_error, test_handler_error, NeoRadosTest)
 {
   std::string_view oid = "obj";
-  co_await create_obj(rados(), oid, pool(), asio::use_awaitable);
+  co_await create_obj(oid);
 
   {
     neorados::ReadOp op;
diff --git a/src/test/neorados/snapshots.cc b/src/test/neorados/snapshots.cc
new file mode 100644
index 000000000000..d1e6c96d6e83
--- /dev/null
+++ b/src/test/neorados/snapshots.cc
@@ -0,0 +1,431 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstdint>
+#include <cstring>
+#include <iostream>
+#include <utility>
+#include <vector>
+
+#include <boost/asio/use_awaitable.hpp>
+#include <boost/asio/awaitable.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "osd/error_code.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+using std::uint64_t;
+
+namespace asio = boost::asio;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+inline asio::awaitable<void> new_selfmanaged_snap(neorados::RADOS& rados,
+						  std::vector<uint64_t>& snaps,
+						  neorados::IOContext& ioc) {
+  snaps.push_back(co_await rados.allocate_selfmanaged_snap(
+		    ioc.get_pool(), asio::use_awaitable));
+  std::reverse(snaps.begin(), snaps.end());
+  ioc.set_write_snap_context({{snaps[0], snaps}});
+  std::reverse(snaps.begin(), snaps.end());
+  co_return;
+}
+
+inline asio::awaitable<void> rm_selfmanaged_snaps(neorados::RADOS& rados,
+						  std::vector<uint64_t>& snaps,
+						  neorados::IOContext& ioc) {
+  std::reverse(snaps.begin(), snaps.end());
+  for (auto snapid : snaps) {
+    co_await rados.delete_selfmanaged_snap(ioc.get_pool(), snapid,
+					   asio::use_awaitable);
+  }
+  snaps.clear();
+}
+
+static constexpr auto oid = "oid"sv;
+
+CORO_TEST_F(NeoRadosSnapshots, SnapList, NeoRadosTest) {
+  static const auto snap1 = "snap1"s;
+  co_await create_obj(oid);
+  EXPECT_FALSE(rados().get_self_managed_snaps_mode(pool()));
+  co_await rados().create_pool_snap(pool(), snap1,
+                                    asio::use_awaitable);
+  EXPECT_FALSE(rados().get_self_managed_snaps_mode(pool()));
+
+  auto snaps = rados().list_snaps(pool());
+  EXPECT_EQ(1u, snaps.size());
+  auto rid = rados().lookup_snap(pool(), snap1);
+  EXPECT_EQ(rid, snaps[0]);
+  co_await rados().delete_pool_snap(pool().get_pool(), snap1, asio::use_awaitable);
+  EXPECT_FALSE(rados().get_self_managed_snaps_mode(pool()));
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSnapshots, SnapRemove, NeoRadosTest) {
+  static const auto snap1 = "snap1"s;
+  co_await create_obj(oid);
+  co_await rados().create_pool_snap(pool(), snap1,
+                                    asio::use_awaitable);
+  rados().lookup_snap(pool(), snap1);
+  co_await rados().delete_pool_snap(pool().get_pool(), snap1, asio::use_awaitable);
+  EXPECT_THROW(rados().lookup_snap(pool(), snap1);,
+	       sys::system_error);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSnapshots, Rollback, NeoRadosTest) {
+  static const auto snap1 = "snap1"s;
+  const auto bl1 = filled_buffer_list(0xcc, 128);
+  const auto bl2 = filled_buffer_list(0xdd, 128);
+
+  co_await execute(oid, WriteOp{}.write(0, bl1));
+  co_await rados().create_pool_snap(pool(), snap1, asio::use_awaitable);
+  co_await execute(oid, WriteOp{}.write_full(bl2));
+
+  auto resbl = co_await read(oid);
+  EXPECT_EQ(bl2, resbl);
+
+  co_await execute(oid, WriteOp{}.rollback(rados().lookup_snap(pool(), snap1)));
+
+  resbl = co_await read(oid);
+  EXPECT_EQ(bl1, resbl);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSnapshots, SnapGetName, NeoRadosTest) {
+  static const auto snapfoo = "snapfoo"s;
+  static const auto snapbar = "snapbar"s;
+  co_await create_obj(oid);
+  co_await rados().create_pool_snap(pool(), snapfoo, asio::use_awaitable);
+  auto rid = rados().lookup_snap(pool(), snapfoo);
+  EXPECT_EQ(snapfoo, rados().get_snap_name(pool(), rid));
+  rados().get_snap_timestamp(pool(), rid);
+  co_await rados().delete_pool_snap(pool().get_pool(), snapfoo, asio::use_awaitable);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSnapshots, SnapCreateRemove, NeoRadosTest) {
+  // reproduces http://tracker.ceph.com/issues/10262
+  static const auto snapfoo = "snapfoo"s;
+  static const auto snapbar = "snapbar"s;
+  const auto bl = to_buffer_list("foo"sv);
+  co_await execute(oid, WriteOp{}.write_full(bl));
+  co_await rados().create_pool_snap(pool(), snapfoo, asio::use_awaitable);
+  co_await execute(oid, WriteOp{}.remove());
+  co_await rados().create_pool_snap(pool(), snapbar, asio::use_awaitable);
+
+  WriteOp op;
+  op.create(false);
+  op.remove();
+  co_await execute(oid, std::move(op));
+  co_await rados().delete_pool_snap(pool().get_pool(), snapfoo,
+				    asio::use_awaitable);
+  co_await rados().delete_pool_snap(pool().get_pool(), snapbar,
+				    asio::use_awaitable);
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, Snap, NeoRadosTest) {
+  std::vector<uint64_t> my_snaps;
+  EXPECT_FALSE(rados().get_self_managed_snaps_mode(pool()));
+  auto ioc = pool();
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  EXPECT_TRUE(rados().get_self_managed_snaps_mode(pool()));
+
+
+  const auto bl1 = filled_buffer_list(0xcc, 128);
+  co_await execute(oid, WriteOp{}.write(0, bl1), ioc);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  const auto bl2 = filled_buffer_list(0xdd, 128);
+  co_await execute(oid, WriteOp{}.write(0, bl2), ioc);
+
+  ioc.set_read_snap(my_snaps[1]);
+  auto resbl = co_await read(oid, ioc);
+  EXPECT_EQ(bl1, resbl);
+
+  co_await rados().delete_selfmanaged_snap(ioc.get_pool(), my_snaps.back(),
+					   asio::use_awaitable);
+  my_snaps.pop_back();
+  ioc.set_read_snap(neorados::snap_head);
+  EXPECT_TRUE(rados().get_self_managed_snaps_mode(pool()));
+  co_await execute(oid, WriteOp{}.remove());
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, Rollback, NeoRadosTest) {
+  SKIP_IF_CRIMSON();
+  static constexpr auto len = 128u;
+  std::vector<uint64_t> my_snaps;
+
+  auto ioc = pool();
+  auto readioc = pool();
+  readioc.set_read_snap(neorados::snap_dir);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  const auto bl1 = filled_buffer_list(0xcc, len);
+  co_await execute(oid, WriteOp{}.write(0, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 2, bl1), ioc);
+
+  neorados::SnapSet ss;
+  co_await execute(oid, ReadOp{}.list_snaps(&ss), readioc);
+  EXPECT_EQ(1u, ss.clones.size());
+  EXPECT_EQ(neorados::snap_head, ss.clones[0].cloneid);
+  EXPECT_EQ(0u, ss.clones[0].snaps.size());
+  EXPECT_EQ(0u, ss.clones[0].overlap.size());
+  EXPECT_EQ(len * 3, ss.clones[0].size);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  const auto bl2 = filled_buffer_list(0xdd, 128);
+  // Once in the middle
+  co_await execute(oid, WriteOp{}.write(len, bl2), ioc);
+  // Once after the end
+  co_await execute(oid, WriteOp{}.write(len * 3, bl1), ioc);
+
+
+  co_await expect_error_code(execute(oid, ReadOp{}.list_snaps(&ss), ioc),
+			     sys::errc::invalid_argument);
+  co_await execute(oid, ReadOp{}.list_snaps(&ss), readioc);
+  EXPECT_EQ(2u, ss.clones.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  EXPECT_EQ(1u, ss.clones[0].snaps.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  EXPECT_EQ(2u, ss.clones[0].overlap.size());
+  EXPECT_EQ(0u, ss.clones[0].overlap[0].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[0].second);
+  EXPECT_EQ(len * 2, ss.clones[0].overlap[1].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[1].second);
+  EXPECT_EQ(len * 3, ss.clones[0].size);
+  EXPECT_EQ(neorados::snap_head, ss.clones[1].cloneid);
+  EXPECT_EQ(0u, ss.clones[1].snaps.size());
+  EXPECT_EQ(0u, ss.clones[1].overlap.size());
+  EXPECT_EQ(len * 4, ss.clones[1].size);
+
+  co_await execute(oid, WriteOp{}.rollback(my_snaps[1]), ioc);
+
+  auto resbl = co_await read(oid, 0, len);
+  EXPECT_EQ(len, resbl.length());
+  EXPECT_EQ(bl1, resbl);
+  resbl = co_await read(oid, len, len);
+  EXPECT_EQ(len, resbl.length());
+  EXPECT_EQ(bl1, resbl);
+
+  resbl = co_await read(oid, len * 2, len);
+  EXPECT_EQ(len, resbl.length());
+  EXPECT_EQ(bl1, resbl);
+
+  resbl = co_await read(oid, len * 3, len);
+  EXPECT_EQ(0u, resbl.length());
+
+  co_await rm_selfmanaged_snaps(rados(), my_snaps, ioc);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, SnapOverlap, NeoRadosTest) {
+  // WIP https://tracker.ceph.com/issues/58263
+  SKIP_IF_CRIMSON();
+  static constexpr auto len = 128u;
+  std::vector<uint64_t> my_snaps;
+  auto ioc = pool();
+  auto readioc = pool();
+  readioc.set_read_snap(neorados::snap_dir);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  const auto bl1 = filled_buffer_list(0xcc, len);
+  co_await execute(oid, WriteOp{}.write(0, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 2, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 4, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 6, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 8, bl1), ioc);
+
+  neorados::SnapSet ss;
+  co_await execute(oid, ReadOp{}.list_snaps(&ss), readioc);
+  EXPECT_EQ(1u, ss.clones.size());
+  EXPECT_EQ(neorados::snap_head, ss.clones[0].cloneid);
+  EXPECT_EQ(0u, ss.clones[0].snaps.size());
+  EXPECT_EQ(0u, ss.clones[0].overlap.size());
+  EXPECT_EQ(9u * len, ss.clones[0].size);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  const auto bl2 = filled_buffer_list(0xdd, len);
+  co_await execute(oid, WriteOp{}.write(len * 1, bl2), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 3, bl2), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 5, bl2), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 7, bl2), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 9, bl2), ioc);
+
+  co_await execute(oid, ReadOp{}.list_snaps(&ss), readioc);
+  EXPECT_EQ(2u, ss.clones.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  EXPECT_EQ(1u, ss.clones[0].snaps.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  EXPECT_EQ(5u, ss.clones[0].overlap.size());
+  EXPECT_EQ(0u, ss.clones[0].overlap[0].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[0].second);
+  EXPECT_EQ(len * 2, ss.clones[0].overlap[1].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[1].second);
+  EXPECT_EQ(len * 4, ss.clones[0].overlap[2].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[2].second);
+  EXPECT_EQ(len * 6, ss.clones[0].overlap[3].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[3].second);
+  EXPECT_EQ(len * 8, ss.clones[0].overlap[4].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[4].second);
+  EXPECT_EQ(len * 9, ss.clones[0].size);
+  EXPECT_EQ(neorados::snap_head, ss.clones[1].cloneid);
+  EXPECT_EQ(0u, ss.clones[1].snaps.size());
+  EXPECT_EQ(0u, ss.clones[1].overlap.size());
+  EXPECT_EQ(len * 10, ss.clones[1].size);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+
+  const auto bl3 = filled_buffer_list(0xee, len);
+  co_await execute(oid, WriteOp{}.write(len * 1, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 4, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 5, bl1), ioc);
+  co_await execute(oid, WriteOp{}.write(len * 8, bl1), ioc);
+
+  co_await execute(oid, ReadOp{}.list_snaps(&ss), readioc);
+
+  EXPECT_EQ(3u, ss.clones.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  EXPECT_EQ(1u, ss.clones[0].snaps.size());
+  EXPECT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  EXPECT_EQ(5u, ss.clones[0].overlap.size());
+  EXPECT_EQ(0u, ss.clones[0].overlap[0].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[0].second);
+  EXPECT_EQ(len * 2, ss.clones[0].overlap[1].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[1].second);
+  EXPECT_EQ(len * 4, ss.clones[0].overlap[2].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[2].second);
+  EXPECT_EQ(len * 6, ss.clones[0].overlap[3].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[3].second);
+  EXPECT_EQ(len * 8, ss.clones[0].overlap[4].first);
+  EXPECT_EQ(len, ss.clones[0].overlap[4].second);
+  EXPECT_EQ(len * 9, ss.clones[0].size);
+
+  EXPECT_EQ(my_snaps[2], ss.clones[1].cloneid);
+  EXPECT_EQ(1u, ss.clones[1].snaps.size());
+  EXPECT_EQ(my_snaps[2], ss.clones[1].snaps[0]);
+  EXPECT_EQ(4u, ss.clones[1].overlap.size());
+  EXPECT_EQ(0u, ss.clones[1].overlap[0].first);
+  EXPECT_EQ(len, ss.clones[1].overlap[0].second);
+  EXPECT_EQ(len * 2, ss.clones[1].overlap[1].first);
+  EXPECT_EQ(len * 2, ss.clones[1].overlap[1].second);
+  EXPECT_EQ(len * 6, ss.clones[1].overlap[2].first);
+  EXPECT_EQ(len * 2, ss.clones[1].overlap[2].second);
+  EXPECT_EQ(len * 9, ss.clones[1].overlap[3].first);
+  EXPECT_EQ(len, ss.clones[1].overlap[3].second);
+  EXPECT_EQ(len * 10, ss.clones[1].size);
+
+  EXPECT_EQ(neorados::snap_head, ss.clones[2].cloneid);
+  EXPECT_EQ(0u, ss.clones[2].snaps.size());
+  EXPECT_EQ(0u, ss.clones[2].overlap.size());
+  EXPECT_EQ(len * 10, ss.clones[2].size);
+
+  co_await rm_selfmanaged_snaps(rados(), my_snaps, ioc);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, Bug11677, NeoRadosTest) {
+  std::vector<uint64_t> my_snaps;
+  auto ioc = pool();
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+
+  static constexpr auto len = 1 << 20; // 1 MiB
+  auto buf = std::make_unique<char[]>(len);
+  std::memset(buf.get(), 0xcc, len);
+
+  buffer::list bl1;
+  bl1.append(buf.get(), len);
+  co_await execute(oid, WriteOp{}.write(0, bl1), ioc);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+
+  WriteOp op;
+  op.assert_exists()
+    .remove();
+  co_await execute(oid, std::move(op), ioc);
+
+  co_await rm_selfmanaged_snaps(rados(), my_snaps, ioc);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, OrderSnap, NeoRadosTest) {
+  static constexpr auto len = 128u;
+  std::vector<uint64_t> my_snaps;
+  auto ioc = pool();
+  const auto bl = filled_buffer_list(0xcc, len);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  co_await execute(oid, WriteOp{}.write(0, bl).ordersnap(), ioc);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  co_await execute(oid, WriteOp{}.write(0, bl).ordersnap(), ioc);
+
+  my_snaps.pop_back();
+  std::reverse(my_snaps.begin(), my_snaps.end());
+  ioc.set_write_snap_context({{my_snaps[0], my_snaps}});
+  std::reverse(my_snaps.begin(), my_snaps.end());
+
+  co_await expect_error_code(execute(oid, WriteOp()
+				     .write(0, bl).ordersnap(), ioc),
+			     osd_errc::old_snapc);
+
+  co_await execute(oid, WriteOp{}.write(0, bl), ioc);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosSelfManagedSnaps, ReusePurgedSnap, NeoRadosTest) {
+  static constexpr auto len = 128u;
+  std::vector<uint64_t> my_snaps;
+  auto ioc = pool();
+  const auto bl = filled_buffer_list(0xcc, len);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  EXPECT_TRUE(rados().get_self_managed_snaps_mode(pool()));
+  co_await execute(oid, WriteOp{}.write(0, bl), ioc);
+
+  co_await new_selfmanaged_snap(rados(), my_snaps, ioc);
+  std::cout << "Deleting snap " << my_snaps.back() << " in pool "
+	    << pool_name() << "." << std::endl;
+  co_await rados().delete_selfmanaged_snap(ioc.get_pool(), my_snaps.back(),
+					   asio::use_awaitable);
+  std::cout << "Waiting for snaps to purge." << std::endl;
+  co_await wait_for(15s);
+  std::reverse(my_snaps.begin(), my_snaps.end());
+  ioc.set_write_snap_context({{my_snaps[0], my_snaps}});
+  std::reverse(my_snaps.begin(), my_snaps.end());
+
+  co_await execute(oid, WriteOp()
+		   .write(0, filled_buffer_list(0xdd, len)));
+
+
+  co_return;
+}
diff --git a/src/test/neorados/watch_notify.cc b/src/test/neorados/watch_notify.cc
new file mode 100644
index 000000000000..284b9436edc7
--- /dev/null
+++ b/src/test/neorados/watch_notify.cc
@@ -0,0 +1,168 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstdint>
+#include <iostream>
+#include <utility>
+#include <vector>
+
+#include <boost/asio/as_tuple.hpp>
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/co_spawn.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_set.hpp>
+
+#include <boost/system/errc.hpp>
+#include <boost/system/error_code.hpp>
+#include <boost/system/system_error.hpp>
+
+#include "include/neorados/RADOS.hpp"
+#include "include/buffer.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+using std::uint64_t;
+
+namespace asio = boost::asio;
+namespace buffer = ceph::buffer;
+namespace container = boost::container;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+using std::uint64_t;
+
+class NeoRadosWatchNotifyTest : public NeoRadosTest {
+protected:
+  buffer::list notify_bl;
+  container::flat_set<uint64_t> notify_cookies;
+  const std::string notify_oid = "foo"s;
+  sys::error_code notify_err;
+  ceph::timespan notify_sleep = 0s;
+
+  asio::awaitable<void> handle_notify(uint64_t notify_id, uint64_t cookie,
+                                      uint64_t notifier_gid, buffer::list&& bl) {
+    std::cout << __func__ << " cookie " << cookie << " notify_id " << notify_id
+	      << " notifier_gid " << notifier_gid << std::endl;
+    notify_bl = std::move(bl);
+    notify_cookies.insert(cookie);
+    if (notify_sleep > 0s) {
+      std::cout << "Waiting for " << notify_sleep << std::endl;
+      co_await wait_for(notify_sleep);
+    }
+    co_await rados().notify_ack(notify_oid, pool(), notify_id, cookie,
+                                to_buffer_list("reply"sv), asio::use_awaitable);
+  }
+
+  asio::awaitable<void> handle_error(sys::error_code ec, uint64_t cookie) {
+    std::cout << __func__ << " cookie " << cookie
+              << " err " << ec.message() << std::endl;
+    ceph_assert(cookie > 1000);
+    co_await rados().unwatch(cookie, pool(), asio::use_awaitable);
+    notify_cookies.erase(cookie);
+    notify_err = ec;
+    try {
+      auto watchcookie
+        = co_await rados().watch(notify_oid, pool(), std::nullopt,
+                                 std::ref(*this), asio::use_awaitable);
+      notify_cookies.insert(watchcookie);
+    } catch (const sys::system_error& e) {
+      std::cout << "reconnect error: " << e.what() << std::endl;
+    }
+  }
+
+public:
+  void operator ()(sys::error_code ec, uint64_t notify_id, uint64_t cookie,
+                   uint64_t notifier_id, buffer::list&& bl) {
+    asio::co_spawn(
+      asio_context,
+      [](NeoRadosWatchNotifyTest* t, sys::error_code ec, uint64_t notify_id,
+         uint64_t cookie, uint64_t notifier_id, buffer::list bl)
+      -> asio::awaitable<void> {
+        if (ec) {
+          co_await t->handle_error(ec, cookie);
+        } else {
+          co_await t->handle_notify(notify_id, cookie, notifier_id,
+                                    std::move(bl));
+        }
+	co_return;
+      }(this, ec, notify_id, cookie, notifier_id, std::move(bl)),
+      [](std::exception_ptr e) {
+	if (e) std::rethrow_exception(e);
+      });
+  }
+};
+
+CORO_TEST_F(NeoRadosWatchNotify, WatchNotify, NeoRadosWatchNotifyTest) {
+  co_await create_obj(notify_oid);
+  auto handle = co_await rados().watch(notify_oid, pool(), std::nullopt,
+                                       std::ref(*this),
+                                       asio::use_awaitable);
+  EXPECT_TRUE(rados().check_watch(handle));
+  std::vector<neorados::ObjWatcher> watchers;
+  co_await execute(notify_oid, ReadOp{}.list_watchers(&watchers));
+  EXPECT_EQ(1u, watchers.size());
+  auto reply = co_await rados().notify(notify_oid, pool(), {}, {},
+                                       asio::use_awaitable);
+  std::map<std::pair<uint64_t, uint64_t>, buffer::list> reply_map;
+  std::set<std::pair<uint64_t, uint64_t>> missed_set;
+  auto p = reply.cbegin();
+  decode(reply_map, p);
+  decode(missed_set, p);
+  EXPECT_EQ(1u, notify_cookies.size());
+  EXPECT_EQ(1u, notify_cookies.count(handle));
+  EXPECT_EQ(1u, reply_map.size());
+  EXPECT_EQ(5u, reply_map.begin()->second.length());
+  EXPECT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  EXPECT_EQ(0u, missed_set.size());
+  EXPECT_TRUE(rados().check_watch(handle));
+  co_await rados().unwatch(handle, pool(), asio::use_awaitable);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosWatchNotify, WatchNotifyTimeout, NeoRadosWatchNotifyTest) {
+  co_await create_obj(notify_oid);
+  auto handle = co_await rados().watch(notify_oid, pool(), std::nullopt,
+                                       std::ref(*this),
+                                       asio::use_awaitable);
+  EXPECT_TRUE(rados().check_watch(handle));
+  std::vector<neorados::ObjWatcher> watchers;
+  co_await execute(notify_oid, ReadOp{}.list_watchers(&watchers));
+  EXPECT_EQ(1u, watchers.size());
+
+  notify_sleep = 3s;
+
+  std::cout << "Trying..." << std::endl;
+  co_await expect_error_code(rados().notify(notify_oid, pool(), {}, 1s,
+					    asio::use_awaitable),
+			     sys::errc::timed_out);
+  std::cout << "Timed out." << std::endl;
+
+  EXPECT_TRUE(rados().check_watch(handle));
+  co_await rados().unwatch(handle, pool(), asio::use_awaitable);
+
+  std::cout << "Flushing..." << std::endl;
+  co_await rados().flush_watch(asio::use_awaitable);
+  std::cout << "Flushed..." << std::endl;
+
+  // Give time for notify_ack to fire before pool gets deleted.
+  co_await wait_for(notify_sleep);
+
+  co_return;
+}

From 5d38a2f883b95c9e01cced4418069f614ea4c369 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 7 Sep 2023 22:47:04 -0400
Subject: [PATCH 0973/2492] build: Make `tests` depend on `neorados`

To make sure it's built by `make check`.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/tools/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index 56bb0414bcd6..4e7b2f4adc5c 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -24,6 +24,7 @@ if(NOT WIN32)
       neorados.cc)
   add_executable(neorados ${neorados_srcs})
   target_link_libraries(neorados libneorados ${FMT_LIB} ${CMAKE_DL_LIBS})
+  add_dependencies(tests neorados)
   #install(TARGETS neorados DESTINATION bin)
 endif()
 

From 8a07fdb2eb7f8f19b41566c8d773e7a3e5023110 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Mon, 18 Sep 2023 13:27:51 -0400
Subject: [PATCH 0974/2492] qa/rados: Add neorados tests to run

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 qa/workunits/rados/test.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
index a88826cf619c..8d5f40088950 100755
--- a/qa/workunits/rados/test.sh
+++ b/qa/workunits/rados/test.sh
@@ -48,6 +48,22 @@ do
     fi
 done
 
+for f in \
+    cls cmd handler_error io list misc pool read_operations snapshots \
+    watch_notify write_operations
+do
+    if [ $parallel -eq 1 ]; then
+	r=`printf '%25s' $f`
+	ff=`echo $f | awk '{print $1}'`
+	bash -o pipefail -exc "ceph_test_neorados_$f $color 2>&1 | tee ceph_test_neorados_$ff.log | sed \"s/^/$r: /\"" &
+	pid=$!
+	echo "test $f on pid $pid"
+	pids[$f]=$pid
+    else
+	ceph_test_neorados_$f
+    fi
+done
+
 ret=0
 if [ $parallel -eq 1 ]; then
 for t in "${!pids[@]}"

From e64d609156c53c6486a0b8902cc3608365c7d00b Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 28 Sep 2023 13:00:59 -0400
Subject: [PATCH 0975/2492] libcephfs: Fix order of destruction of context pool

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/libcephfs.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libcephfs.cc b/src/libcephfs.cc
index bafd0e6e4b4b..7eea6665f614 100644
--- a/src/libcephfs.cc
+++ b/src/libcephfs.cc
@@ -232,7 +232,6 @@ struct ceph_mount_info
       delete messenger;
       messenger = nullptr;
     }
-    icp.reset();
     if (monclient) {
       delete monclient;
       monclient = nullptr;
@@ -241,6 +240,7 @@ struct ceph_mount_info
       delete client;
       client = nullptr;
     }
+    icp.reset();
   }
 
   bool is_initialized() const

From eb275076ed724ec79b18fc3769b575d2aeb391fa Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 19 Oct 2023 20:10:05 -0400
Subject: [PATCH 0976/2492] test/os: Add missing include to
 Fragmentation_simulator.cc

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/test/objectstore/Fragmentation_simulator.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/objectstore/Fragmentation_simulator.cc b/src/test/objectstore/Fragmentation_simulator.cc
index 8e99dc7036d2..02a2991cd0ce 100644
--- a/src/test/objectstore/Fragmentation_simulator.cc
+++ b/src/test/objectstore/Fragmentation_simulator.cc
@@ -17,6 +17,7 @@
 #include "include/buffer_fwd.h"
 #include "os/ObjectStore.h"
 #include "test/objectstore/ObjectStoreImitator.h"
+#include <fstream>
 #include <boost/random/uniform_int.hpp>
 #include <fmt/core.h>
 #include <mutex>

From f50693f62387e48c5c07ba913f2d384de0fe2d5c Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Tue, 28 Nov 2023 11:21:38 +0800
Subject: [PATCH 0977/2492] mds: allow lock state to be LOCK_MIX_SYNC in
 replica for filelock

When the auth MDS is transmitting state from LOCK_MIX to LOCK_SYNC
it will transmit the state to a intermidiate state LOCK_MIX_SYNC.
Then notifies the replicate MDSs and waits for the acks. But just
in case for some reasons the replicate MDSs couldn't successfully
transmit the state from LOCK_MIX to LOCK_MIX_SYNC and then
LOCK_MIX_SYNC2. And just the auth MDS failover or restart and
then resend the notification.

Fixes: https://tracker.ceph.com/issues/54833
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 src/mds/Locker.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index b4a28fb8379f..de5b920efaf4 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -5765,6 +5765,7 @@ void Locker::handle_file_lock(ScatterLock *lock, const cref_t<MLock> &m)
   case LOCK_AC_SYNC:
     ceph_assert(lock->get_state() == LOCK_LOCK ||
 	   lock->get_state() == LOCK_MIX ||
+	   lock->get_state() == LOCK_MIX_SYNC ||
 	   lock->get_state() == LOCK_MIX_SYNC2);
     
     if (lock->get_state() == LOCK_MIX) {

From 3071316fe3c135a38a1be8631d5b0cbeac4e8985 Mon Sep 17 00:00:00 2001
From: Zhiwei Huang <ai.william@outlook.com>
Date: Fri, 8 Dec 2023 11:37:24 +0800
Subject: [PATCH 0978/2492] cephadm: Fix `cephadm shell` command help message

The meaning of the `--volume` option is to bind mount a volume.

Signed-off-by: Zhiwei Huang <ai.william@outlook.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 98a2585c760c..ce100e1823d2 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -5206,7 +5206,7 @@ def _get_parser():
         '--volume', '-v',
         action='append',
         default=[],
-        help='set environment variable')
+        help='mount a volume')
     parser_shell.add_argument(
         'command', nargs=argparse.REMAINDER,
         help='command (optional)')

From 6f581fb588672f1da884e43a7c711c0f1f2a80e7 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Tue, 5 Dec 2023 11:19:07 +0530
Subject: [PATCH 0979/2492] cephfs: move release note for PR #41779 to right
 spot

Release note for PR #41779 applies releases after Reef, not  Reef
onwards. Move the release note accordingly.

Fixes: https://github.com/ceph/ceph/pull/41779
Fixes: https://tracker.ceph.com/issues/47264
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 PendingReleaseNotes | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index c5ad5e0649f8..3670f78512c4 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -50,6 +50,11 @@
   recommend that users with versioned buckets, especially those that existed
   on prior releases, use these new tools to check whether their buckets are
   affected and to clean them up accordingly.
+* CephFS: Running the command "ceph fs authorize" for an existing entity now
+  upgrades the entity's capabilities instead of printing an error. It can now
+  also change read/write permissions in a capability that the entity already
+  holds. If the capability passed by user is same as one of the capabilities
+  that the entity already holds, idempotency is maintained.
 * CephFS: Two FS names can now be swapped, optionally along with their IDs,
   using "ceph fs swap" command. The function of this API is to facilitate
   file system swaps for disaster recovery. In particular, it avoids situations
@@ -57,9 +62,10 @@
   level storage operator (like Rook) to recreate the missing file system.
   See https://docs.ceph.com/en/latest/cephfs/administration/#file-systems
   docs for more information.
-* CephFS: Before running the command "ceph fs rename", the CephFS to be
-  renamed must be taken offline and "refuse_client_session" must be set for
-  it.
+* CephFS: Before running the command "ceph fs rename", the filesystem to be
+  renamed must be offline and the config "refuse_client_session" must be set
+  for it. The config "refuse_client_session" can be removed/unset and
+  filesystem can be online after the rename operation is complete.
 * RADOS: A POOL_APP_NOT_ENABLED health warning will now be reported if
   the application is not enabled for the pool irrespective of whether
   the pool is in use or not. Always tag a pool with an application
@@ -256,11 +262,6 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   than the number mentioned against the config tunable `mds_max_snaps_per_dir`
   so that a new snapshot can be created and retained during the next schedule
   run.
-* cephfs: Running the command "ceph fs authorize" for an existing entity now
-  upgrades the entity's capabilities instead of printing an error. It can now
-  also change read/write permissions in a capability that the entity already
-  holds. If the capability passed by user is same as one of the capabilities
-  that the entity already holds, idempotency is maintained.
 * `ceph config dump --format <json|xml>` output will display the localized
   option names instead of its normalized version. For e.g.,
   "mgr/prometheus/x/server_port" will be displayed instead of

From 1c241e49e5a97361b42408ec5bb5ec72eadcc54b Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Fri, 8 Dec 2023 15:18:24 +0530
Subject: [PATCH 0980/2492] qa: drop mon_host check from
 test_cephfs_mirror_peer_bootstrap

Fixes: https://tracker.ceph.com/issues/63614
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 qa/tasks/cephfs/test_mirroring.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py
index 24dbd61b17fb..afd49f6b2ffc 100644
--- a/qa/tasks/cephfs/test_mirroring.py
+++ b/qa/tasks/cephfs/test_mirroring.py
@@ -798,7 +798,6 @@ def test_cephfs_mirror_peer_bootstrap(self):
         peer_uuid = self.get_peer_uuid("client.mirror_peer_bootstrap@site-remote")
         res = json.loads(self.get_ceph_cmd_stdout("fs", "snapshot", "mirror", "peer_list", self.primary_fs_name))
         self.assertTrue(peer_uuid in res)
-        self.assertTrue('mon_host' in res[peer_uuid] and res[peer_uuid]['mon_host'] != '')
 
         # remove peer
         self.peer_remove(self.primary_fs_name, self.primary_fs_id, "client.mirror_peer_bootstrap@site-remote")

From 8f5d61df634e839b5b2c8ab223c5c5fa2c0c1dda Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 25 Oct 2023 16:15:37 +0300
Subject: [PATCH 0981/2492] os: introduce ObjectStore::refresh_perf_counters()
 method.

This is a common method to refresh Object Store performance counters.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/ObjectStore.h                       | 7 +++++++
 src/os/bluestore/BlueStore.cc              | 6 ++++--
 src/os/bluestore/BlueStore.h               | 3 ++-
 src/os/kstore/KStore.h                     | 2 ++
 src/os/memstore/MemStore.h                 | 3 ++-
 src/test/objectstore/ObjectStoreImitator.h | 1 +
 6 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 4c837b84d010..302be387fae6 100644
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -111,6 +111,13 @@ class ObjectStore {
    * This appears to be called with nothing locked.
    */
   virtual objectstore_perf_stat_t get_cur_stats() = 0;
+  /**
+   * Propagate Object Store performance counters with the actual values
+   *
+   *
+   * Intended primarily for testing purposes
+   */
+  virtual void refresh_perf_counters() = 0;
 
   /**
    * Fetch Object Store performance counters.
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index ec03fcde14ae..e8fcdce3bfe4 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -1310,6 +1310,7 @@ struct LruBufferCacheShard : public BlueStore::BufferCacheShard {
                  uint64_t *blobs,
                  uint64_t *buffers,
                  uint64_t *bytes) override {
+    std::lock_guard l(lock);
     *extents += num_extents;
     *blobs += num_blobs;
     *buffers += num;
@@ -1615,6 +1616,7 @@ struct TwoQBufferCacheShard : public BlueStore::BufferCacheShard {
                  uint64_t *blobs,
                  uint64_t *buffers,
                  uint64_t *bytes) override {
+    std::lock_guard l(lock);
     *extents += num_extents;
     *blobs += num_blobs;
     *buffers += num;
@@ -5358,7 +5360,7 @@ void *BlueStore::MempoolThread::entry()
     _resize_shards(interval_stats_trim);
     interval_stats_trim = false;
 
-    store->_update_logger();
+    store->refresh_perf_counters();
     auto wait = ceph::make_timespan(
       store->cct->_conf->bluestore_cache_trim_interval);
     cond.wait_for(l, wait);
@@ -11780,7 +11782,7 @@ void BlueStore::_reap_collections()
   }
 }
 
-void BlueStore::_update_logger()
+void BlueStore::refresh_perf_counters()
 {
   uint64_t num_onodes = 0;
   uint64_t num_pinned_onodes = 0;
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 095c838608ed..8079f239c71c 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2777,7 +2777,6 @@ class BlueStore : public ObjectStore,
   CollectionRef _get_collection_by_oid(const ghobject_t& oid);
   void _queue_reap_collection(CollectionRef& c);
   void _reap_collections();
-  void _update_logger();
 
   void _assign_nid(TransContext *txc, OnodeRef& o);
   uint64_t _assign_blobid(TransContext *txc);
@@ -3328,6 +3327,8 @@ class BlueStore : public ObjectStore,
   const PerfCounters* get_perf_counters() const override {
     return logger;
   }
+  void refresh_perf_counters() override;
+
   const PerfCounters* get_bluefs_perf_counters() const {
     return bluefs->get_perf_counters();
   }
diff --git a/src/os/kstore/KStore.h b/src/os/kstore/KStore.h
index 9e3c7acd73b4..96bbdfaa424d 100644
--- a/src/os/kstore/KStore.h
+++ b/src/os/kstore/KStore.h
@@ -562,6 +562,8 @@ class KStore : public ObjectStore {
   objectstore_perf_stat_t get_cur_stats() override {
     return objectstore_perf_stat_t();
   }
+  void refresh_perf_counters() override {
+  }
   const PerfCounters* get_perf_counters() const override {
     return logger;
   }
diff --git a/src/os/memstore/MemStore.h b/src/os/memstore/MemStore.h
index 858379ed9c7c..2abe552891fd 100644
--- a/src/os/memstore/MemStore.h
+++ b/src/os/memstore/MemStore.h
@@ -395,7 +395,8 @@ class MemStore : public ObjectStore {
   }
 
   objectstore_perf_stat_t get_cur_stats() override;
-
+  void refresh_perf_counters() override {
+  }
   const PerfCounters* get_perf_counters() const override {
     return nullptr;
   }
diff --git a/src/test/objectstore/ObjectStoreImitator.h b/src/test/objectstore/ObjectStoreImitator.h
index c01e77897460..d71d7f2fe58b 100644
--- a/src/test/objectstore/ObjectStoreImitator.h
+++ b/src/test/objectstore/ObjectStoreImitator.h
@@ -354,4 +354,5 @@ class ObjectStoreImitator : public ObjectStore {
   }
   objectstore_perf_stat_t get_cur_stats() override { return {}; }
   const PerfCounters *get_perf_counters() const override { return nullptr; };
+  void refresh_perf_counters() override {}
 };

From 73f7c94f479ab7b2e244f05a7f3ca0d13b2fec00 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 25 Oct 2023 16:17:58 +0300
Subject: [PATCH 0982/2492] test/store_test: refactor DeferredOnBitOverwrite*
 tests

Fixes: https://tracker.ceph.com/issues/63106

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/test/objectstore/store_test.cc | 105 +++++++++++++++++++++++------
 1 file changed, 85 insertions(+), 20 deletions(-)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 03dc1a87e1fc..6652c1a55f1b 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -7844,7 +7844,7 @@ TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionBigOverwrite) {
   }
 }
 
-TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
+TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite1) {
 
   if (string(GetParam()) != "bluestore")
     return;
@@ -7961,7 +7961,7 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     bufferlist bl;
 
     bl.append(std::string(block_size, 'e'));
-    t.write(cid, hoid2, block_size , bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE);
+    t.write(cid, hoid2, block_size, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE);
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
@@ -8018,17 +8018,50 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 5);
     ASSERT_LE(statfs.allocated, (unsigned)block_size * 5);
   }
+  store->refresh_perf_counters();
   ASSERT_EQ(logger->get(l_bluestore_blobs), 2u);
   ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
-
   {
     ObjectStore::Transaction t;
     t.remove(cid, hoid);
     t.remove(cid, hoid2);
+    t.remove_collection(cid);
+    cerr << "Cleaning" << std::endl;
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
+}
+
+TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) {
+
+  if (string(GetParam()) != "bluestore")
+    return;
+  if (smr) {
+    cout << "SKIP: no deferred" << std::endl;
+    return;
+  }
 
+  size_t block_size = 4096;
+  StartDeferred(block_size);
+  SetVal(g_conf(), "bluestore_max_blob_size", "131072");
+  SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536");
+
+  g_conf().apply_changes(nullptr);
+
+  int r;
+  coll_t cid;
+  ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, ""));
+  ghobject_t hoid2(hobject_t("test2", "", CEPH_NOSNAP, 0, -1, ""));
+
+  PerfCounters* logger = const_cast<PerfCounters*>(store->get_perf_counters());
+
+  auto ch = store->create_new_collection(cid);
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid, 0);
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
   {
     ObjectStore::Transaction t;
     bufferlist bl;
@@ -8038,8 +8071,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 6u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 3u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 1u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 0u);
 
   {
     ObjectStore::Transaction t;
@@ -8061,7 +8094,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     expected.append(string(block_size * 2 - 100, 'f'));
     ASSERT_TRUE(bl_eq(expected, bl));
   }
-  sleep(2);
   {
     struct store_statfs_t statfs;
     int r = store->statfs(&statfs);
@@ -8069,6 +8101,7 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 2 - 100);
     ASSERT_LE(statfs.allocated, (unsigned)block_size * 2);
   }
+  store->refresh_perf_counters();
   ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
   ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
 
@@ -8081,8 +8114,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 7u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 2u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u);
   {
     bufferlist bl, expected;
     r = store->read(ch, hoid, 0, block_size, bl);
@@ -8105,6 +8138,7 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 2);
     ASSERT_LE(statfs.allocated, (unsigned)block_size * 2);
   }
+  store->refresh_perf_counters();
   ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
   ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
 
@@ -8118,8 +8152,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 8u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 3u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u);
 
   {
     bufferlist bl, expected;
@@ -8141,10 +8175,43 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     ObjectStore::Transaction t;
     t.remove(cid, hoid);
     t.remove(cid, hoid2);
+    t.remove_collection(cid);
+    cerr << "Cleaning" << std::endl;
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
+}
+
+TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) {
+
+  if (string(GetParam()) != "bluestore")
+    return;
+  if (smr) {
+    cout << "SKIP: no deferred" << std::endl;
+    return;
+  }
+
+  size_t block_size = 4096;
+  StartDeferred(block_size);
+  SetVal(g_conf(), "bluestore_max_blob_size", "131072");
+  SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536");
+
+  g_conf().apply_changes(nullptr);
+
+  int r;
+  coll_t cid;
+  ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, ""));
+  ghobject_t hoid2(hobject_t("test2", "", CEPH_NOSNAP, 0, -1, ""));
 
+  PerfCounters* logger = const_cast<PerfCounters*>(store->get_perf_counters());
+
+  auto ch = store->create_new_collection(cid);
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid, 0);
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
   {
     ObjectStore::Transaction t;
     bufferlist bl;
@@ -8156,8 +8223,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 10u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 2u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 0u);
 
   // check whether overwrite (less than prefer_deferred_size) partially overlapping two adjacent blobs goes
   // deferred
@@ -8170,8 +8237,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 11u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 6u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 3u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 2u);
 
   {
     bufferlist bl, expected;
@@ -8213,9 +8280,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  sleep(2);
-  ASSERT_EQ(logger->get(l_bluestore_write_big), 12u);
-  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 8u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big), 4u);
+  ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u);
 
   {
     bufferlist bl, expected;
@@ -8258,7 +8324,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
     r = queue_transaction(store, ch, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  sleep(2);
   ASSERT_EQ(logger->get(l_bluestore_write_big), 1u);
   ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u);
   ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 1u);
@@ -8283,7 +8348,7 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) {
   }
 }
 
-TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) {
+TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite4) {
 
   if (string(GetParam()) != "bluestore")
     return;
@@ -8359,7 +8424,7 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) {
   }
 }
 
-TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) {
+TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite5) {
 
   if (string(GetParam()) != "bluestore")
     return;

From b50755e3a1bccdc45768bf42c603b765b0eaeee8 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 25 Oct 2023 18:46:51 +0300
Subject: [PATCH 0983/2492] test/store_test: make use of
 refresh_perf_counters() call. And get rid off sleep() ones.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/test/objectstore/store_test.cc | 46 +++++++++++-------------------
 1 file changed, 17 insertions(+), 29 deletions(-)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 6652c1a55f1b..c663eedb5776 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -7375,14 +7375,13 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, 0, block_size, bl);
     ASSERT_EQ(r, (int)block_size);
     expected.append(string(block_size, 'b'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
   }
@@ -7400,14 +7399,13 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, 0, block_size, bl);
     ASSERT_EQ(r, (int)block_size);
     expected.append(string(block_size, 'b'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
   }
@@ -7423,11 +7421,6 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) {
     ASSERT_EQ(r, 0);
   }
   {
-    // we need to wait some time for mempool
-    // thread to update stats to be able to check blob/extent numbers from
-    // perf counters.
-    sleep(1);
-
     bufferlist bl, expected;
     r = store->read(ch, hoid, 0, block_size, bl);
     ASSERT_EQ(r, (int)block_size);
@@ -7470,6 +7463,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) {
     expected.append(string(block_size * 2, 'e'));
     ASSERT_TRUE(bl_eq(expected, bl));
   }
+  store->refresh_perf_counters();
   ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
   ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
 
@@ -8653,15 +8647,14 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, block_size * 9, block_size * 2, bl);
     ASSERT_EQ(r, (int)block_size * 2);
     expected.append(string(block_size, 'b'));
     expected.append(string(block_size, 'a'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
   }
@@ -8680,9 +8673,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, block_size * 7, block_size * 3, bl);
     ASSERT_EQ(r, (int)block_size * 3);
@@ -8690,6 +8681,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
     expected.append(string(block_size, 0));
     expected.append(string(block_size, 'b'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
   }
@@ -8707,9 +8699,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, block_size * 11, block_size * 3, bl);
     ASSERT_EQ(r, (int)block_size * 3);
@@ -8717,6 +8707,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
     expected.append(string(block_size, 0));
     expected.append(string(block_size, 'd'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 3u);
   }
@@ -8736,9 +8727,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, block_size * 17, block_size * 3, bl);
     ASSERT_EQ(r, (int)block_size * 3);
@@ -8746,6 +8735,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
     expected.append(string(block_size, 0));
     expected.append(string(block_size, 'e'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 2u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 5u);
   }
@@ -8764,9 +8754,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, block_size * 16, block_size * 4, bl);
     ASSERT_EQ(r, (int)block_size * 4);
@@ -8775,6 +8763,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
     expected.append(string(block_size, 'f'));
     expected.append(string(block_size, 'e'));
     ASSERT_TRUE(bl_eq(expected, bl));
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 2u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 4u);
   }
@@ -8839,9 +8828,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) {
   }
   {
     // We need to issue a read to trigger cache stat update that refresh
-    // perf counters. additionally we need to wait some time for mempool
-    // thread to update stats.
-    sleep(1);
+    // perf counters.
     bufferlist bl, expected;
     r = store->read(ch, hoid, 0, block_size * 3, bl);
     ASSERT_EQ(r, (int)block_size * 3);
@@ -8852,6 +8839,7 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) {
     expected.append(string(block_size, 'a'));
     ASSERT_TRUE(bl_eq(expected, bl));
 
+    store->refresh_perf_counters();
     ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
     ASSERT_EQ(logger->get(l_bluestore_extents), 3u);
   }

From 260644a7639c63d98b4a66e0aea0d14b0478cbb6 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 7 Dec 2023 09:11:12 -0500
Subject: [PATCH 0984/2492] vstart: add --rgw_store option for
 rados|dbstore|posix

enables dbstore for rgw_backend_store and rgw_config_store, allowing
vstart to run without any mons or osds. database files are put under
the dev subdirectory

when rgw_store=posix, the posix filter is added on top of dbstore

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/vstart.sh | 48 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/src/vstart.sh b/src/vstart.sh
index 634fb74ac874..1187e86b9ed8 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -168,6 +168,7 @@ fi
 ceph_osd=ceph-osd
 rgw_frontend="beast"
 rgw_compression=""
+rgw_store="rados"
 lockdep=${LOCKDEP:-1}
 spdk_enabled=0 # disable SPDK by default
 pmem_enabled=0
@@ -229,6 +230,7 @@ options:
 	--rgw_frontend specify the rgw frontend configuration
 	--rgw_arrow_flight start arrow flight frontend
 	--rgw_compression specify the rgw compression plugin
+	--rgw_store storage backend: rados|dbstore|posix
 	--seastore use seastore as crimson osd backend
 	-b, --bluestore use bluestore as the osd objectstore backend (default)
 	-K, --kstore use kstore as the osd objectstore backend
@@ -461,6 +463,10 @@ case $1 in
         rgw_compression=$2
         shift
         ;;
+    --rgw_store)
+        rgw_store=$2
+        shift
+        ;;
     --kstore_path)
         kstore_path=$2
         shift
@@ -692,6 +698,22 @@ done
 
 }
 
+do_rgw_dbstore_conf() {
+    if [ $CEPH_NUM_RGW -gt 1 ]; then
+        echo "dbstore is not distributed so only works with CEPH_NUM_RGW=1"
+        exit 1
+    fi
+
+    prun mkdir -p "$CEPH_DEV_DIR/rgw/dbstore"
+    wconf <<EOF
+        rgw backend store = dbstore
+        rgw config store = dbstore
+        dbstore db dir = $CEPH_DEV_DIR/rgw/dbstore
+        dbstore_config_uri = file://$CEPH_DEV_DIR/rgw/dbstore/config.db
+
+EOF
+}
+
 format_conf() {
     local opts=$1
     local indent="        "
@@ -861,6 +883,20 @@ $CCLIENTDEBUG
         ; rgw lc debug interval = 10
         $(format_conf "${extra_conf}")
 EOF
+    if [ "$rgw_store" == "dbstore" ] ; then
+        do_rgw_dbstore_conf
+    elif [ "$rgw_store" == "posix" ] ; then
+        # use dbstore as the backend and posix as the filter
+        do_rgw_dbstore_conf
+        posix_dir="$CEPH_DEV_DIR/rgw/posix"
+        prun mkdir -p $posix_dir/root $posix_dir/lmdb
+        wconf <<EOF
+        rgw filter = posix
+        rgw posix base path = $posix_dir/root
+        rgw posix database root = $posix_dir/lmdb
+
+EOF
+    fi
 	do_rgw_conf
 	wconf << EOF
 [mds]
@@ -1785,11 +1821,13 @@ do_rgw()
     for n in $(seq 1 $CEPH_NUM_RGW); do
         rgw_name="client.rgw.${current_port}"
 
-        ceph_adm auth get-or-create $rgw_name \
-            mon 'allow rw' \
-            osd 'allow rwx' \
-            mgr 'allow rw' \
-            >> "$keyring_fn"
+        if [ "$CEPH_NUM_MON" -gt 0 ]; then
+            ceph_adm auth get-or-create $rgw_name \
+                mon 'allow rw' \
+                osd 'allow rwx' \
+                mgr 'allow rw' \
+                >> "$keyring_fn"
+        fi
 
         debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port}
         run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \

From 4304ca95d3aa2ff009498e792ee09f4d84566ad3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 8 Dec 2023 11:24:54 -0500
Subject: [PATCH 0985/2492] librbd: include asio strand headers in librbd.cc

Fixes: https://tracker.ceph.com/issues/63682

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/librbd/librbd.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc
index 1272d95dd7e5..132a0084a9f6 100644
--- a/src/librbd/librbd.cc
+++ b/src/librbd/librbd.cc
@@ -15,6 +15,12 @@
 
 #include <errno.h>
 
+// these strand headers declare static variables that need to be shared between
+// librbd.so and librados.so. referencing them here causes librbd.so to link
+// their symbols as 'global unique'. see https://tracker.ceph.com/issues/63682
+#include <boost/asio/strand.hpp>
+#include <boost/asio/io_context_strand.hpp>
+
 #include "common/deleter.h"
 #include "common/dout.h"
 #include "common/errno.h"

From 5df7b44727ebe1ae070b6af7a81bb5bf07681863 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 8 Dec 2023 13:07:39 -0500
Subject: [PATCH 0986/2492] qa/rgw/crypt: disable failing kmip testing

kmip tests have been failing all year and are not being maintained:

https://tracker.ceph.com/issues/58751
https://tracker.ceph.com/issues/58523

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/crypt/2-kms/kmip.yaml | 37 -----------------------------
 1 file changed, 37 deletions(-)
 delete mode 100644 qa/suites/rgw/crypt/2-kms/kmip.yaml

diff --git a/qa/suites/rgw/crypt/2-kms/kmip.yaml b/qa/suites/rgw/crypt/2-kms/kmip.yaml
deleted file mode 100644
index 0057d954e320..000000000000
--- a/qa/suites/rgw/crypt/2-kms/kmip.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-overrides:
-  ceph:
-    conf:
-      client:
-        rgw crypt s3 kms backend: kmip
-        rgw crypt kmip ca path: /etc/ceph/kmiproot.crt
-        rgw crypt kmip client cert: /etc/ceph/kmip-client.crt
-        rgw crypt kmip client key: /etc/ceph/kmip-client.key
-        rgw crypt kmip kms key template: pykmip-$keyid
-  rgw:
-    client.0:
-      use-pykmip-role: client.0
-
-tasks:
-- openssl_keys:
-    kmiproot:
-      client: client.0
-      cn: kmiproot
-      key-type: rsa:4096
-    kmip-server:
-      client: client.0
-      ca: kmiproot
-    kmip-client:
-      client: client.0
-      ca: kmiproot
-      cn: rgw-client
-- exec:
-    client.0:
-      - chmod 644 /home/ubuntu/cephtest/ca/kmip-client.key
-- pykmip:
-    client.0:
-      clientca: kmiproot
-      servercert: kmip-server
-      clientcert: kmip-client
-      secrets:
-      - name: pykmip-my-key-1
-      - name: pykmip-my-key-2

From 1c0901cf8503ca8486dee8875ce85adaa464b127 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 9 Dec 2023 04:46:00 +0100
Subject: [PATCH 0987/2492] doc/radosgw: format POST statements

Format the POST methods so that they appear in the rendered text as
examples of POST API calls and not as plain old unformatted text, which
is how they looked before this commit. The content of these API calls
remains to be tested and confirmed to work, but this is a first step.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/role.rst | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
index e974498722b5..3ef78d8d209c 100644
--- a/doc/radosgw/role.rst
+++ b/doc/radosgw/role.rst
@@ -31,7 +31,7 @@ Request Parameters
 :Description: The trust relationship policy document that grants an entity permission to assume the role.
 :Type: String
 
-For example:: 	
+For example::
 	
   radosgw-admin role create --role-name=S3Access1 --path=/application_abc/component_xyz/ --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}
   
@@ -334,6 +334,7 @@ Create a Role
 -------------
 
 Example::
+
   POST "<hostname>?Action=CreateRole&RoleName=S3Access&Path=/application_abc/component_xyz/&AssumeRolePolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
 
 .. code-block:: XML
@@ -353,6 +354,7 @@ Delete a Role
 -------------
 
 Example::
+
   POST "<hostname>?Action=DeleteRole&RoleName=S3Access"
 
 Note: A role can be deleted only when it doesn't have any permission policy attached to it.
@@ -361,6 +363,7 @@ Get a Role
 ----------
 
 Example::
+
   POST "<hostname>?Action=GetRole&RoleName=S3Access"
 
 .. code-block:: XML
@@ -380,6 +383,7 @@ List Roles
 ----------
 
 Example::
+
   POST "<hostname>?Action=ListRoles&RoleName=S3Access&PathPrefix=/application"
 
 .. code-block:: XML
@@ -399,18 +403,21 @@ Update Assume Role Policy Document
 ----------------------------------
 
 Example::
+
   POST "<hostname>?Action=UpdateAssumeRolePolicy&RoleName=S3Access&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER2\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
 
 Add/ Update a Policy attached to a Role
 ---------------------------------------
 
 Example::
+
   POST "<hostname>?Action=PutRolePolicy&RoleName=S3Access&PolicyName=Policy1&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:CreateBucket\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\}"
 
 List Permission Policy Names attached to a Role
 -----------------------------------------------
 
 Example::
+
   POST "<hostname>?Action=ListRolePolicies&RoleName=S3Access"
 
 .. code-block:: XML
@@ -424,6 +431,7 @@ Get Permission Policy attached to a Role
 ----------------------------------------
 
 Example::
+
   POST "<hostname>?Action=GetRolePolicy&RoleName=S3Access&PolicyName=Policy1"
 
 .. code-block:: XML
@@ -439,6 +447,7 @@ Delete Policy attached to a Role
 --------------------------------
 
 Example::
+
   POST "<hostname>?Action=DeleteRolePolicy&RoleName=S3Access&PolicyName=Policy1"
 
 Tag a role
@@ -447,6 +456,7 @@ A role can have multivalued tags attached to it. These tags can be passed in as
 AWS does not support multi-valued role tags.
 
 Example::
+
   POST "<hostname>?Action=TagRole&RoleName=S3Access&Tags.member.1.Key=Department&Tags.member.1.Value=Engineering"
 
 .. code-block:: XML
@@ -463,6 +473,7 @@ List role tags
 Lists the tags attached to a role.
 
 Example::
+
   POST "<hostname>?Action=ListRoleTags&RoleName=S3Access"
 
 .. code-block:: XML
@@ -486,6 +497,7 @@ Delete role tags
 Delete a tag/ tags attached to a role.
 
 Example::
+
   POST "<hostname>?Action=UntagRoles&RoleName=S3Access&TagKeys.member.1=Department"
 
 .. code-block:: XML
@@ -500,6 +512,7 @@ Update Role
 -----------
 
 Example::
+
   POST "<hostname>?Action=UpdateRole&RoleName=S3Access&MaxSessionDuration=43200"
 
 .. code-block:: XML
@@ -565,6 +578,3 @@ The following is sample code for adding tags to role, listing tags and untagging
             'Department',
         ]
     )
-
-
-

From de397f7588897fb0a3f15dcddf660c8e569b1e4e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 9 Dec 2023 21:00:42 +0100
Subject: [PATCH 0988/2492] test/librbd: avoid config-related crashes in
 DiscardWithPruneWriteOverlap

For reasons that I think no longer apply today, set_val() and
set_val_or_die() refuse to set "type: str" config options that aren't
marked as "can be changed at runtime" -- set_val() returns an error and
set_val_or_die() terminates the process.  What is and isn't marked as
"can be changed at runtime" seems to be pretty much random both within
and outside of RBD, so let's just refactor how config is set here.

While at it, I realized that reproducer config is underspecified:

- for rbd_cache_policy and rbd_cache_writethrough_until_flush settings
  to matter, rbd_cache must be set to true and rbd_cache_max_dirty must
  be set to a positive number

- order should be set explicitly, because rbd_default_order can be as
  low as 12 (for 4096-byte objects), interfering with the logic of the
  test

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/journal/test_Stress.cc | 34 +++++++++++++++++++-------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/test/librbd/journal/test_Stress.cc b/src/test/librbd/journal/test_Stress.cc
index d2b9353fb5f8..5abdc8bae807 100644
--- a/src/test/librbd/journal/test_Stress.cc
+++ b/src/test/librbd/journal/test_Stress.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include "test/librados/test_cxx.h"
 #include "test/librbd/test_fixture.h"
 #include "test/librbd/test_support.h"
 #include "cls/rbd/cls_rbd_types.h"
@@ -21,6 +22,7 @@
 #include "librbd/io/ImageRequest.h"
 #include "librbd/io/ReadResult.h"
 #include "librbd/journal/Types.h"
+#include <boost/scope_exit.hpp>
 
 void register_test_journal_stress() {
 }
@@ -39,23 +41,37 @@ TEST_F(TestJournalStress, DiscardWithPruneWriteOverlap) {
 
   // Create an image that is multiple objects so that we can force multiple
   // image extents on the discard path.
-  CephContext* cct = reinterpret_cast<CephContext*>(_rados.cct());
-  auto object_size = 1ull << cct->_conf.get_val<uint64_t>("rbd_default_order");
+  int order = 22;
+  auto object_size = uint64_t{1} << order;
   auto image_size = 4 * object_size;
 
   // Write-around cache required for overlapping I/O delays.
-  cct->_conf.set_val_or_die("rbd_cache_writethrough_until_flush", "false");
-  cct->_conf.set_val_or_die("rbd_cache_policy", "writearound");
+  std::map<std::string, std::string> config;
+  config["rbd_cache"] = "true";
+  config["rbd_cache_policy"] = "writearound";
+  config["rbd_cache_max_dirty"] = std::to_string(image_size);
+  config["rbd_cache_writethrough_until_flush"] = "false";
   // XXX: Work around https://tracker.ceph.com/issues/63681, which this test
   // exposes when run under Valgrind.
-  cct->_conf.set_val_or_die("librados_thread_count", "15");
-  cct->_conf.apply_changes(nullptr);
+  config["librados_thread_count"] = "15";
 
+  librados::Rados rados;
+  ASSERT_EQ("", connect_cluster_pp(rados, config));
+
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, rados.ioctx_create(_pool_name.c_str(), ioctx));
+
+  uint64_t features;
+  ASSERT_TRUE(::get_features(&features));
   auto image_name = get_temp_image_name();
-  ASSERT_EQ(0, create_image_pp(m_rbd, m_ioctx, image_name, image_size));
+  ASSERT_EQ(0, create_image_full_pp(m_rbd, ioctx, image_name, image_size,
+                                    features, false, &order));
 
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(image_name, &ictx));
+  auto ictx = new librbd::ImageCtx(image_name, "", nullptr, ioctx, false);
+  ASSERT_EQ(0, ictx->state->open(0));
+  BOOST_SCOPE_EXIT(ictx) {
+    ictx->state->close();
+  } BOOST_SCOPE_EXIT_END;
 
   std::thread write_thread(
     [ictx, object_size]() {

From d790b57b4bd3868cf12941c565b6906534e19f91 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 9 Dec 2023 21:00:51 +0100
Subject: [PATCH 0989/2492] test/librbd: actually alternate overlaps in
 DiscardWithPruneWriteOverlap

Make sense of the inner loop in write_thread.  The crash on
"it != m_events.end()" assert reproduces even faster this way.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/journal/test_Stress.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/librbd/journal/test_Stress.cc b/src/test/librbd/journal/test_Stress.cc
index 5abdc8bae807..d3df9147ae6a 100644
--- a/src/test/librbd/journal/test_Stress.cc
+++ b/src/test/librbd/journal/test_Stress.cc
@@ -86,7 +86,7 @@ TEST_F(TestJournalStress, DiscardWithPruneWriteOverlap) {
           bufferlist payload_bl;
           payload_bl.append(payload);
           auto aio_comp = new librbd::io::AioCompletion();
-          api::Io<>::aio_write(*ictx, aio_comp, 0, payload.size(),
+          api::Io<>::aio_write(*ictx, aio_comp, offset, payload.size(),
                                std::move(payload_bl), 0, true);
           ASSERT_EQ(0, aio_comp->wait_for_complete());
           aio_comp->release();

From be40bbbb12823721f8fb1cac4db681f5c22825ea Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 9 Dec 2023 16:26:25 +0100
Subject: [PATCH 0990/2492] test/librbd: close image in
 DiffIterateDeterministic

... to avoid valgrind reporting a memory leak on ImageCtx.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index d9e43b5e7364..9d17c4d85cfd 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7580,6 +7580,9 @@ TYPED_TEST(DiffIterateTest, DiffIterateDeterministic)
   ASSERT_EQ(0u, extents.size());
 
   ASSERT_PASSED(this->validate_object_map, image);
+
+  ASSERT_EQ(0, rbd_close(image));
+  rados_ioctx_destroy(ioctx);
 }
 
 TYPED_TEST(DiffIterateTest, DiffIterateDeterministicPP)

From b0a77a53ab86b9b24a2cfb46049957d40a8c80aa Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 5 Dec 2023 18:36:20 +0000
Subject: [PATCH 0991/2492] build/rgw/lua: remove lua-devel as a runtime
 dependency

this is due to the fact that lua-devel
cannot be installed in teuthology for rhel8

this would fail the build of some luarocks packages

Fixes: https://tracker.ceph.com/issues/63672

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 ceph.spec.in | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 3807f48280e9..047e06c24f77 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -283,7 +283,6 @@ BuildRequires:  librabbitmq-devel
 BuildRequires:  librdkafka-devel
 %endif
 %if 0%{with lua_packages}
-Requires:	lua-devel
 Requires:  %{luarocks_package_name}
 %endif
 %if 0%{with make_check}

From 66b7f3245c3fb56a5500ac009ce2ac70aa6a1fc8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 16:44:14 -0400
Subject: [PATCH 0992/2492] cephadm: add shellquote filter to template

This is to support generating command lines or shell scripts using
Jinja2 templates in the future.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templating.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index a431ba669999..5f4dda056086 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -3,6 +3,7 @@
 import enum
 import os
 import posixpath
+import shlex
 import zipimport
 
 from typing import Any, Optional, IO, Tuple, Callable, cast
@@ -113,6 +114,7 @@ class Templater:
     def _env(self) -> jinja2.Environment:
         if self._jinja2_env is None:
             self._jinja2_env = jinja2.Environment(loader=self._loader)
+            self._jinja2_env.filters['shellquote'] = shlex.quote
         return self._jinja2_env
 
     @property

From d4013a1d11c532438f92080c70f670d163f2c437 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 17:01:11 -0400
Subject: [PATCH 0993/2492] cephadm: update container engine test imports

Update container engine tests file to use the canonical module for
container engines rather than importing them indirectly from cephadm.py.
This future proofs the test in case the imports in cephadm.py change.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_container_engine.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/tests/test_container_engine.py b/src/cephadm/tests/test_container_engine.py
index 7c5ef5131271..49f9f9a2e50a 100644
--- a/src/cephadm/tests/test_container_engine.py
+++ b/src/cephadm/tests/test_container_engine.py
@@ -4,7 +4,7 @@
 
 from tests.fixtures import with_cephadm_ctx, import_cephadm
 
-_cephadm = import_cephadm()
+from cephadmlib import container_engines
 
 
 _find_program_loc = 'cephadmlib.container_engine_base.find_program'
@@ -29,7 +29,7 @@ class PhonyContainerEngine(ContainerEngine):
 def test_podman():
     with mock.patch(_find_program_loc) as find_program:
         find_program.return_value = "/usr/bin/podman"
-        pm = _cephadm.Podman()
+        pm = container_engines.Podman()
         find_program.assert_called()
         with pytest.raises(RuntimeError):
             pm.version
@@ -44,7 +44,7 @@ def test_podman():
 def test_podman_badversion():
     with mock.patch(_find_program_loc) as find_program:
         find_program.return_value = "/usr/bin/podman"
-        pm = _cephadm.Podman()
+        pm = container_engines.Podman()
         find_program.assert_called()
         with mock.patch(_call_throws_loc) as call_throws:
             call_throws.return_value = ("4.10.beta2", None, None)
@@ -56,5 +56,5 @@ def test_podman_badversion():
 def test_docker():
     with mock.patch(_find_program_loc) as find_program:
         find_program.return_value = "/usr/bin/docker"
-        docker = _cephadm.Docker()
+        docker = container_engines.Docker()
         assert str(docker) == "docker (/usr/bin/docker)"

From 676a6da7107d2ae3fb8b8939476e35cd7098376b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 17:02:52 -0400
Subject: [PATCH 0994/2492] cephadm: create systemd_unit.py for systemd unit
 related functions

Continue to modularize cephadm by moving the functions related to
generating systemd unit files to a new module.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                 | 66 +----------------------
 src/cephadm/cephadmlib/systemd_unit.py | 73 ++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 65 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/systemd_unit.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 242c85997477..9d65a97352a7 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -83,7 +83,6 @@
     concurrent_tasks,
 )
 from cephadmlib.container_engines import (
-    Docker,
     Podman,
     check_container_engine,
     find_container_engine,
@@ -132,9 +131,9 @@
     cephadm_init_logging,
     Highlight,
     LogDestination,
-    write_cluster_logrotate_config,
 )
 from cephadmlib.systemd import check_unit, check_units
+from cephadmlib.systemd_unit import get_unit_file, install_base_units
 from cephadmlib.container_types import (
     CephContainer,
     InitContainer,
@@ -1381,69 +1380,6 @@ def _write_iscsi_unit_poststop_commands(
     f.write('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
     f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
 
-
-def install_base_units(ctx, fsid):
-    # type: (CephadmContext, str) -> None
-    """
-    Set up ceph.target and ceph-$fsid.target units.
-    """
-    # global unit
-    existed = os.path.exists(ctx.unit_dir + '/ceph.target')
-    with write_new(ctx.unit_dir + '/ceph.target', perms=None) as f:
-        f.write('[Unit]\n'
-                'Description=All Ceph clusters and services\n'
-                '\n'
-                '[Install]\n'
-                'WantedBy=multi-user.target\n')
-    if not existed:
-        # we disable before enable in case a different ceph.target
-        # (from the traditional package) is present; while newer
-        # systemd is smart enough to disable the old
-        # (/lib/systemd/...) and enable the new (/etc/systemd/...),
-        # some older versions of systemd error out with EEXIST.
-        call_throws(ctx, ['systemctl', 'disable', 'ceph.target'])
-        call_throws(ctx, ['systemctl', 'enable', 'ceph.target'])
-        call_throws(ctx, ['systemctl', 'start', 'ceph.target'])
-
-    # cluster unit
-    existed = os.path.exists(ctx.unit_dir + '/ceph-%s.target' % fsid)
-    with write_new(ctx.unit_dir + f'/ceph-{fsid}.target', perms=None) as f:
-        f.write(
-            '[Unit]\n'
-            'Description=Ceph cluster {fsid}\n'
-            'PartOf=ceph.target\n'
-            'Before=ceph.target\n'
-            '\n'
-            '[Install]\n'
-            'WantedBy=multi-user.target ceph.target\n'.format(
-                fsid=fsid)
-        )
-    if not existed:
-        call_throws(ctx, ['systemctl', 'enable', 'ceph-%s.target' % fsid])
-        call_throws(ctx, ['systemctl', 'start', 'ceph-%s.target' % fsid])
-
-    # don't overwrite file in order to allow users to manipulate it
-    if os.path.exists(ctx.logrotate_dir + f'/ceph-{fsid}'):
-        return
-
-    write_cluster_logrotate_config(ctx, fsid)
-
-
-def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
-    has_docker_engine = isinstance(ctx.container_engine, Docker)
-    has_podman_engine = isinstance(ctx.container_engine, Podman)
-    has_podman_split_version = (
-        has_podman_engine and ctx.container_engine.supports_split_cgroups
-    )
-    return templating.render(
-        ctx,
-        templating.Templates.ceph_service,
-        fsid=fsid,
-        has_docker_engine=has_docker_engine,
-        has_podman_engine=has_podman_engine,
-        has_podman_split_version=has_podman_split_version,
-    )
-
 ##################################
 
 
diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
new file mode 100644
index 000000000000..849833804c25
--- /dev/null
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -0,0 +1,73 @@
+# systemd_unit.py - creating/managing systemd unit files
+
+import os
+
+from . import templating
+from .call_wrappers import call_throws
+from .container_engines import Docker, Podman
+from .context import CephadmContext
+from .file_utils import write_new
+from .logging import write_cluster_logrotate_config
+
+
+def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
+    has_docker_engine = isinstance(ctx.container_engine, Docker)
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    has_podman_split_version = (
+        has_podman_engine and ctx.container_engine.supports_split_cgroups
+    )
+    return templating.render(
+        ctx,
+        templating.Templates.ceph_service,
+        fsid=fsid,
+        has_docker_engine=has_docker_engine,
+        has_podman_engine=has_podman_engine,
+        has_podman_split_version=has_podman_split_version,
+    )
+
+
+def install_base_units(ctx, fsid):
+    # type: (CephadmContext, str) -> None
+    """
+    Set up ceph.target and ceph-$fsid.target units.
+    """
+    # global unit
+    existed = os.path.exists(ctx.unit_dir + '/ceph.target')
+    with write_new(ctx.unit_dir + '/ceph.target', perms=None) as f:
+        f.write('[Unit]\n'
+                'Description=All Ceph clusters and services\n'
+                '\n'
+                '[Install]\n'
+                'WantedBy=multi-user.target\n')
+    if not existed:
+        # we disable before enable in case a different ceph.target
+        # (from the traditional package) is present; while newer
+        # systemd is smart enough to disable the old
+        # (/lib/systemd/...) and enable the new (/etc/systemd/...),
+        # some older versions of systemd error out with EEXIST.
+        call_throws(ctx, ['systemctl', 'disable', 'ceph.target'])
+        call_throws(ctx, ['systemctl', 'enable', 'ceph.target'])
+        call_throws(ctx, ['systemctl', 'start', 'ceph.target'])
+
+    # cluster unit
+    existed = os.path.exists(ctx.unit_dir + '/ceph-%s.target' % fsid)
+    with write_new(ctx.unit_dir + f'/ceph-{fsid}.target', perms=None) as f:
+        f.write(
+            '[Unit]\n'
+            'Description=Ceph cluster {fsid}\n'
+            'PartOf=ceph.target\n'
+            'Before=ceph.target\n'
+            '\n'
+            '[Install]\n'
+            'WantedBy=multi-user.target ceph.target\n'.format(
+                fsid=fsid)
+        )
+    if not existed:
+        call_throws(ctx, ['systemctl', 'enable', 'ceph-%s.target' % fsid])
+        call_throws(ctx, ['systemctl', 'start', 'ceph-%s.target' % fsid])
+
+    # don't overwrite file in order to allow users to manipulate it
+    if os.path.exists(ctx.logrotate_dir + f'/ceph-{fsid}'):
+        return
+
+    write_cluster_logrotate_config(ctx, fsid)

From 160b768b4925a93bf0962bacf9a97176c638684b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 17:17:03 -0400
Subject: [PATCH 0995/2492] cephadm: update unit file test imports

Update systemd unit file tests file to use the canonical module for
systemd units functions rather than importing them indirectly from
cephadm.py.
This future proofs the test in case the imports in cephadm.py
change.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_unit_file.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
index 0ebd84be3e84..ab24aab4e57b 100644
--- a/src/cephadm/tests/test_unit_file.py
+++ b/src/cephadm/tests/test_unit_file.py
@@ -16,24 +16,26 @@
     with_cephadm_ctx,
 )
 
+from cephadmlib import context
+from cephadmlib import systemd_unit
 from cephadmlib.constants import CGROUPS_SPLIT_PODMAN_VERSION
 
 _cephadm = import_cephadm()
 
 
 def _get_unit_file(ctx, fsid):
-    return str(_cephadm.get_unit_file(ctx, fsid))
+    return str(systemd_unit.get_unit_file(ctx, fsid))
 
 
 def test_docker_engine_requires_docker():
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_docker()
     r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Requires=docker.service' in r
 
 
 def test_podman_engine_does_not_req_docker():
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_podman()
     r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Requires=docker.service' not in r
@@ -42,7 +44,7 @@ def test_podman_engine_does_not_req_docker():
 def test_podman_engine_forking_service():
     # verity that the podman service uses the forking service type
     # and related parameters
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_podman()
     r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert 'Type=forking' in r
@@ -52,7 +54,7 @@ def test_podman_engine_forking_service():
 
 
 def test_podman_with_split_cgroups_sets_delegate():
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_podman()
     ctx.container_engine.version = CGROUPS_SPLIT_PODMAN_VERSION
     r = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
@@ -65,7 +67,7 @@ def _ignore_blank_lines(value):
 
 
 def test_new_docker():
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_docker()
     ru = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
     assert _ignore_blank_lines(ru) == [
@@ -104,7 +106,7 @@ def test_new_docker():
 
 
 def test_new_podman():
-    ctx = _cephadm.CephadmContext()
+    ctx = context.CephadmContext()
     ctx.container_engine = mock_podman()
     ctx.container_engine.version = CGROUPS_SPLIT_PODMAN_VERSION
     ru = _get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')

From eaa6cbd39fb0e8c1259c0aa7e8a73cc8d65f3a29 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 28 Sep 2023 17:18:18 -0400
Subject: [PATCH 0996/2492] cephadm: add a higher-level function for managing
 systemd units

Add the function update_files to systemd_unit.py to encapsulate and
abstract the details regarding the generation of system unit files.
This will make it simpler in the future to add more advanced systemd
configurations include managing customized unit files and systemd
unit drop-in files.

Some additional work was needed to update the recently added
command_unit_install function. Because the new systemd_unit.update_files
function requires a full daemon identity. The command_unit_install
function now requires a daemon name. In addition, while testing this
change it was found that the function could not have worked as it was
because it required the fsid but neither used the infer_fsid decorator
nor provided a `--fsid` argument. Both were added.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                 | 32 ++++++++++++--------------
 src/cephadm/cephadmlib/systemd_unit.py | 14 ++++++++---
 src/cephadm/tests/test_unit_file.py    |  2 +-
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 9d65a97352a7..d53bb38cdf6b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -133,7 +133,7 @@
     LogDestination,
 )
 from cephadmlib.systemd import check_unit, check_units
-from cephadmlib.systemd_unit import get_unit_file, install_base_units
+from cephadmlib import systemd_unit
 from cephadmlib.container_types import (
     CephContainer,
     InitContainer,
@@ -1254,11 +1254,7 @@ def deploy_daemon_units(
     install_sysctl(ctx, fsid, daemon_form_create(ctx, ident))
 
     # systemd
-    install_base_units(ctx, fsid)
-    unit = get_unit_file(ctx, fsid)
-    unit_file = 'ceph-%s@.service' % (fsid)
-    with write_new(ctx.unit_dir + '/' + unit_file, perms=None) as f:
-        f.write(unit)
+    systemd_unit.update_files(ctx, ident)
     call_throws(ctx, ['systemctl', 'daemon-reload'])
 
     unit_name = get_unit_name(fsid, daemon_type, daemon_id)
@@ -3431,21 +3427,16 @@ def command_ceph_volume(ctx):
 ##################################
 
 
+@infer_fsid
 def command_unit_install(ctx):
     # type: (CephadmContext) -> int
-    if not ctx.fsid:
+    if not getattr(ctx, 'fsid', None):
         raise Error('must pass --fsid to specify cluster')
-
-    fsid = ctx.fsid
-    install_base_units(ctx, fsid)
-    unit = get_unit_file(ctx, fsid)
-    unit_file = 'ceph-%s@.service' % (fsid)
-    with open(ctx.unit_dir + '/' + unit_file + '.new', 'w') as f:
-        f.write(unit)
-        os.rename(ctx.unit_dir + '/' + unit_file + '.new',
-                  ctx.unit_dir + '/' + unit_file)
+    if not getattr(ctx, 'name', None):
+        raise Error('daemon name required')
+    ident = DaemonIdentity.from_context(ctx)
+    systemd_unit.update_files(ctx, ident)
     call_throws(ctx, ['systemctl', 'daemon-reload'])
-
     return 0
 
 
@@ -5191,6 +5182,13 @@ def _get_parser():
     parser_unit_install = subparsers.add_parser(
         'unit-install', help="Install the daemon's systemd unit")
     parser_unit_install.set_defaults(func=command_unit_install)
+    parser_unit_install.add_argument(
+        '--fsid',
+        help='cluster FSID')
+    parser_unit_install.add_argument(
+        '--name', '-n',
+        required=True,
+        help='daemon name (type.id)')
 
     parser_logs = subparsers.add_parser(
         'logs', help='print journald logs for a daemon container')
diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index 849833804c25..0c392c77bfe3 100644
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -6,11 +6,12 @@
 from .call_wrappers import call_throws
 from .container_engines import Docker, Podman
 from .context import CephadmContext
+from .daemon_identity import DaemonIdentity
 from .file_utils import write_new
 from .logging import write_cluster_logrotate_config
 
 
-def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
+def _get_unit_file(ctx: CephadmContext, fsid: str) -> str:
     has_docker_engine = isinstance(ctx.container_engine, Docker)
     has_podman_engine = isinstance(ctx.container_engine, Podman)
     has_podman_split_version = (
@@ -26,8 +27,7 @@ def get_unit_file(ctx: CephadmContext, fsid: str) -> str:
     )
 
 
-def install_base_units(ctx, fsid):
-    # type: (CephadmContext, str) -> None
+def _install_base_units(ctx: CephadmContext, fsid: str) -> None:
     """
     Set up ceph.target and ceph-$fsid.target units.
     """
@@ -71,3 +71,11 @@ def install_base_units(ctx, fsid):
         return
 
     write_cluster_logrotate_config(ctx, fsid)
+
+
+def update_files(ctx: CephadmContext, ident: DaemonIdentity) -> None:
+    _install_base_units(ctx, ident.fsid)
+    unit = _get_unit_file(ctx, ident.fsid)
+    unit_file = 'ceph-%s@.service' % (ident.fsid)
+    with write_new(ctx.unit_dir + '/' + unit_file, perms=None) as f:
+        f.write(unit)
diff --git a/src/cephadm/tests/test_unit_file.py b/src/cephadm/tests/test_unit_file.py
index ab24aab4e57b..a1d49c93c201 100644
--- a/src/cephadm/tests/test_unit_file.py
+++ b/src/cephadm/tests/test_unit_file.py
@@ -24,7 +24,7 @@
 
 
 def _get_unit_file(ctx, fsid):
-    return str(systemd_unit.get_unit_file(ctx, fsid))
+    return str(systemd_unit._get_unit_file(ctx, fsid))
 
 
 def test_docker_engine_requires_docker():

From 49245dabeb2b9caf7d8df79a27f3647aa2c86674 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 15 Nov 2023 16:39:07 -0500
Subject: [PATCH 0997/2492] cephadm: move abstract script handling functions to
 runscripts.py

Add a new file runscripts.py for the lower-level management of scripts
and related files that are invoked by systemd units. This patch ended up
uglier than I desired because there was a bunch of daemon specific logic
that remains in cephadm.py and those functions all needed to be updated
to avoid calling functions that write to the scripts directly.
Now customizations are done by passing a list of commands: these
commands can be either a string that will be literally added to the
scipt, a list that will be quoted and then added to the script, or
a ContainerCommand which is basically a wrapper around the arguments to
_write_container_cmd_to bash.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py               | 251 +++++++--------------------
 src/cephadm/cephadmlib/runscripts.py | 235 +++++++++++++++++++++++++
 2 files changed, 294 insertions(+), 192 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/runscripts.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index d53bb38cdf6b..ec90db6305b7 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -18,7 +18,7 @@
 import time
 import errno
 import ssl
-from typing import Dict, List, Tuple, Optional, Union, Any, Callable, IO, Sequence, TypeVar, cast, Iterable, TextIO
+from typing import Dict, List, Tuple, Optional, Union, Any, Callable, Sequence, TypeVar, cast, Iterable
 
 import re
 import uuid
@@ -134,6 +134,7 @@
 )
 from cephadmlib.systemd import check_unit, check_units
 from cephadmlib import systemd_unit
+from cephadmlib import runscripts
 from cephadmlib.container_types import (
     CephContainer,
     InitContainer,
@@ -1055,97 +1056,6 @@ def deploy_daemon(
         call_throws(ctx, ['systemctl', 'restart', ident.unit_name])
 
 
-def _bash_cmd(
-    fh: IO[str],
-    cmd: List[str],
-    check: bool = True,
-    background: bool = False,
-    stderr: bool = True,
-) -> None:
-    line = ' '.join(shlex.quote(arg) for arg in cmd)
-    if not check:
-        line = f'! {line}'
-    if not stderr:
-        line = f'{line} 2> /dev/null'
-    if background:
-        line = f'{line} &'
-    fh.write(line)
-    fh.write('\n')
-
-
-def _write_container_cmd_to_bash(
-    ctx: CephadmContext,
-    file_obj: IO[str],
-    container: 'CephContainer',
-    comment: Optional[str] = None,
-    background: Optional[bool] = False,
-) -> None:
-    if comment:
-        # Sometimes adding a comment, especially if there are multiple containers in one
-        # unit file, makes it easier to read and grok.
-        assert '\n' not in comment
-        file_obj.write(f'# {comment}\n')
-    # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually
-    _bash_cmd(
-        file_obj, container.rm_cmd(old_cname=True), check=False, stderr=False
-    )
-    _bash_cmd(file_obj, container.rm_cmd(), check=False, stderr=False)
-
-    # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
-    if isinstance(ctx.container_engine, Podman):
-        _bash_cmd(
-            file_obj,
-            container.rm_cmd(storage=True),
-            check=False,
-            stderr=False,
-        )
-        _bash_cmd(
-            file_obj,
-            container.rm_cmd(old_cname=True, storage=True),
-            check=False,
-            stderr=False,
-        )
-
-    # container run command
-    _bash_cmd(file_obj, container.run_cmd(), background=bool(background))
-
-
-def _write_init_container_cmds(
-    ctx: CephadmContext,
-    file_obj: IO[str],
-    index: int,
-    init_container: 'InitContainer',
-) -> None:
-    file_obj.write(f'# init container {index}: {init_container.cname}\n')
-    _bash_cmd(file_obj, init_container.run_cmd())
-    _write_init_container_cmds_clean(ctx, file_obj, init_container, comment='')
-
-
-def _write_init_container_cmds_clean(
-    ctx: CephadmContext,
-    file_obj: IO[str],
-    init_container: 'InitContainer',
-    comment: str = 'init container cleanup',
-) -> None:
-    if comment:
-        assert '\n' not in comment
-        file_obj.write(f'# {comment}\n')
-    _bash_cmd(
-        file_obj,
-        init_container.rm_cmd(),
-        check=False,
-        stderr=False,
-    )
-    # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
-    if isinstance(ctx.container_engine, Podman):
-        _bash_cmd(
-            file_obj,
-            init_container.rm_cmd(storage=True),
-            check=False,
-            stderr=False,
-        )
-
-
 def clean_cgroup(ctx: CephadmContext, fsid: str, unit_name: str) -> None:
     # systemd may fail to cleanup cgroups from previous stopped unit, which will cause next "systemctl start" to fail.
     # see https://tracker.ceph.com/issues/50998
@@ -1183,81 +1093,44 @@ def deploy_daemon_units(
     endpoints: Optional[List[EndPoint]] = None,
     init_containers: Optional[List['InitContainer']] = None,
 ) -> None:
-    # cmd
-
-    # unpack values from ident because they're used very frequently
-    fsid = ident.fsid
-    daemon_type = ident.daemon_type
-    daemon_id = ident.daemon_id
-
     data_dir = ident.data_dir(ctx.data_dir)
-    run_file_path = data_dir + '/unit.run'
-    meta_file_path = data_dir + '/unit.meta'
-    with write_new(run_file_path) as f, write_new(meta_file_path) as metaf:
-
-        f.write('set -e\n')
-
-        if daemon_type in ceph_daemons():
-            install_path = find_program('install')
-            f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid))
+    pre_start_commands: List[runscripts.Command] = []
+    post_stop_commands: List[runscripts.Command] = []
+
+    if ident.daemon_type in ceph_daemons():
+        install_path = find_program('install')
+        pre_start_commands.append('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=ident.fsid, uid=uid, gid=gid))
+    if ident.daemon_type == 'osd':
+        assert osd_fsid
+        pre_start_commands.extend(_osd_unit_run_commands(
+            ctx, ident, osd_fsid, data_dir, uid, gid
+        ))
+        post_stop_commands.extend(
+            _osd_unit_poststop_commands(ctx, ident, osd_fsid)
+        )
+    if ident.daemon_type == CephIscsi.daemon_type:
+        pre_start_commands.extend(_iscsi_unit_run_commands(ctx, ident, data_dir))
+        post_stop_commands.extend(_iscsi_unit_poststop_commands(ctx, ident, data_dir))
 
-        # pre-start cmd(s)
-        if daemon_type == 'osd':
-            assert osd_fsid
-            _write_osd_unit_run_commands(
-                ctx, f, ident, osd_fsid, data_dir, uid, gid
-            )
-        elif daemon_type == CephIscsi.daemon_type:
-            _write_iscsi_unit_run_commands(ctx, f, ident, data_dir)
-        init_containers = init_containers or []
-        if init_containers:
-            _write_init_container_cmds_clean(ctx, f, init_containers[0])
-        for idx, ic in enumerate(init_containers):
-            _write_init_container_cmds(ctx, f, idx, ic)
-
-        _write_container_cmd_to_bash(ctx, f, container, '%s.%s' % (daemon_type, str(daemon_id)))
-
-        # some metadata about the deploy
-        meta: Dict[str, Any] = fetch_meta(ctx)
-        meta.update({
-            'memory_request': int(ctx.memory_request) if ctx.memory_request else None,
-            'memory_limit': int(ctx.memory_limit) if ctx.memory_limit else None,
-        })
-        if not meta.get('ports'):
-            if endpoints:
-                meta['ports'] = [e.port for e in endpoints]
-            else:
-                meta['ports'] = []
-        metaf.write(json.dumps(meta, indent=4) + '\n')
-
-    timeout = 30 if daemon_type == 'osd' else None
-    # post-stop command(s)
-    with write_new(data_dir + '/unit.poststop') as f:
-        # this is a fallback to eventually stop any underlying container that was not stopped properly by unit.stop,
-        # this could happen in very slow setups as described in the issue https://tracker.ceph.com/issues/58242.
-        _write_stop_actions(ctx, cast(TextIO, f), container, timeout)
-        if daemon_type == 'osd':
-            assert osd_fsid
-            _write_osd_unit_poststop_commands(ctx, f, ident, osd_fsid)
-        elif daemon_type == CephIscsi.daemon_type:
-            _write_iscsi_unit_poststop_commands(ctx, f, ident, data_dir)
-
-    # post-stop command(s)
-    with write_new(data_dir + '/unit.stop') as f:
-        _write_stop_actions(ctx, cast(TextIO, f), container, timeout)
-
-    if container:
-        with write_new(data_dir + '/unit.image') as f:
-            f.write(container.image + '\n')
+    runscripts.write_service_scripts(
+        ctx,
+        ident,
+        container=container,
+        init_containers=init_containers,
+        endpoints=endpoints,
+        pre_start_commands=pre_start_commands,
+        post_stop_commands=post_stop_commands,
+        timeout=30 if ident.daemon_type == 'osd' else None,
+    )
 
     # sysctl
-    install_sysctl(ctx, fsid, daemon_form_create(ctx, ident))
+    install_sysctl(ctx, ident.fsid, daemon_form_create(ctx, ident))
 
     # systemd
     systemd_unit.update_files(ctx, ident)
     call_throws(ctx, ['systemctl', 'daemon-reload'])
 
-    unit_name = get_unit_name(fsid, daemon_type, daemon_id)
+    unit_name = get_unit_name(ident.fsid, ident.daemon_type, ident.daemon_id)
     call(ctx, ['systemctl', 'stop', unit_name],
          verbosity=CallVerbosity.DEBUG)
     call(ctx, ['systemctl', 'reset-failed', unit_name],
@@ -1265,38 +1138,27 @@ def deploy_daemon_units(
     if enable:
         call_throws(ctx, ['systemctl', 'enable', unit_name])
     if start:
-        clean_cgroup(ctx, fsid, unit_name)
+        clean_cgroup(ctx, ident.fsid, unit_name)
         call_throws(ctx, ['systemctl', 'start', unit_name])
 
 
-def _write_stop_actions(
-    ctx: CephadmContext, f: TextIO, container: 'CephContainer', timeout: Optional[int]
-) -> None:
-    # following generated script basically checks if the container exists
-    # before stopping it. Exit code will be success either if it doesn't
-    # exist or if it exists and is stopped successfully.
-    container_exists = f'{ctx.container_engine.path} inspect %s &>/dev/null'
-    f.write(f'! {container_exists % container.old_cname} || {" ".join(container.stop_cmd(old_cname=True, timeout=timeout))} \n')
-    f.write(f'! {container_exists % container.cname} || {" ".join(container.stop_cmd(timeout=timeout))} \n')
-
-
-def _write_osd_unit_run_commands(
+def _osd_unit_run_commands(
     ctx: CephadmContext,
-    f: IO,
     ident: 'DaemonIdentity',
     osd_fsid: str,
     data_dir: str,
     uid: int,
     gid: int,
-) -> None:
+) -> List[runscripts.Command]:
+    cmds: List[runscripts.Command] = []
     # osds have a pre-start step
     simple_fn = os.path.join('/etc/ceph/osd',
                              '%s-%s.json.adopted-by-cephadm' % (ident.daemon_id, osd_fsid))
     if os.path.exists(simple_fn):
-        f.write('# Simple OSDs need chown on startup:\n')
+        cmds.append('# Simple OSDs need chown on startup:\n')
         for n in ['block', 'block.db', 'block.wal']:
             p = os.path.join(data_dir, n)
-            f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid))
+            cmds.append('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid))
     else:
         # if ceph-volume does not support 'ceph-volume activate', we must
         # do 'ceph-volume lvm activate'.
@@ -1336,21 +1198,24 @@ def _write_osd_unit_run_commands(
             bind_mounts=get_container_binds(ctx, ident),
             cname='ceph-%s-%s.%s-activate' % (fsid, daemon_type, daemon_id),
         )
-        _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate')
+        cmds.append(runscripts.ContainerCommand(prestart, comment='LVM OSDs use ceph-volume lvm activate'))
+    return cmds
 
 
-def _write_iscsi_unit_run_commands(
-    ctx: CephadmContext, f: IO, ident: 'DaemonIdentity', data_dir: str
-) -> None:
-    f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
+def _iscsi_unit_run_commands(
+    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
+) -> List[runscripts.Command]:
+    cmds: List[runscripts.Command] = []
+    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
     ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
     tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runner container', background=True)
+    cmds.append(runscripts.ContainerCommand(tcmu_container, comment='iscsi tcmu-runner container', background=True))
+    return cmds
 
 
-def _write_osd_unit_poststop_commands(
-    ctx: CephadmContext, f: IO, ident: 'DaemonIdentity', osd_fsid: str
-) -> None:
+def _osd_unit_poststop_commands(
+    ctx: CephadmContext, ident: 'DaemonIdentity', osd_fsid: str
+) -> List[runscripts.Command]:
     poststop = get_ceph_volume_container(
         ctx,
         args=[
@@ -1361,20 +1226,22 @@ def _write_osd_unit_poststop_commands(
         bind_mounts=get_container_binds(ctx, ident),
         cname='ceph-%s-%s.%s-deactivate' % (ident.fsid, ident.daemon_type, ident.daemon_id),
     )
-    _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd')
+    return [runscripts.ContainerCommand(poststop, comment='deactivate osd')]
 
 
-def _write_iscsi_unit_poststop_commands(
-    ctx: CephadmContext, f: IO, ident: 'DaemonIdentity', data_dir: str
-) -> None:
+def _iscsi_unit_poststop_commands(
+    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
+) -> List[runscripts.Command]:
     # make sure we also stop the tcmu container
+    cmds: List[runscripts.Command] = []
     runtime_dir = '/run'
     ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
     tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    f.write('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n')
-    f.write('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    f.write('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
+    cmds.append('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n')
+    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
+    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
+    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
+    return cmds
 
 ##################################
 
diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
new file mode 100644
index 000000000000..a5ac1a805c6b
--- /dev/null
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -0,0 +1,235 @@
+import contextlib
+import json
+import pathlib
+import shlex
+
+from typing import Any, Dict, Union, List, IO, TextIO, Optional, cast
+
+from .container_engines import Podman
+from .container_types import CephContainer, InitContainer
+from .context import CephadmContext
+from .context_getters import fetch_meta
+from .daemon_identity import DaemonIdentity
+from .file_utils import write_new
+from .net_utils import EndPoint
+
+
+# Ideally, all ContainerCommands would be converted to init containers. Until
+# that is done one can wrap a CephContainer in a ContainerCommand object and
+# pass that as a pre- or post- command to run arbitrary container based
+# commands in the script.
+class ContainerCommand:
+    def __init__(
+        self,
+        container: CephContainer,
+        comment: str = '',
+        background: bool = False,
+    ):
+        self.container = container
+        self.comment = comment
+        self.background = background
+
+
+Command = Union[List[str], str, ContainerCommand]
+
+
+def write_service_scripts(
+    ctx: CephadmContext,
+    ident: DaemonIdentity,
+    *,
+    container: CephContainer,
+    init_containers: Optional[List[InitContainer]] = None,
+    endpoints: Optional[List[EndPoint]] = None,
+    pre_start_commands: Optional[List[Command]] = None,
+    post_stop_commands: Optional[List[Command]] = None,
+    timeout: Optional[int] = None,
+) -> None:
+    """Write the scripts that systemd services will call in order to
+    start/stop/etc components of a cephadm managed daemon. Also writes some
+    metadata about the service getting deployed.
+    """
+    data_dir = pathlib.Path(ident.data_dir(ctx.data_dir))
+    run_file_path = data_dir / 'unit.run'
+    meta_file_path = data_dir / 'unit.meta'
+    post_stop_file_path = data_dir / 'unit.poststop'
+    stop_file_path = data_dir / 'unit.stop'
+    image_file_path = data_dir / 'unit.image'
+    # use an ExitStack to make writing the files an all-or-nothing affair. If
+    # any file fails to write then the write_new'd file will not get renamed
+    # into place
+    with contextlib.ExitStack() as estack:
+        # write out the main file to run (start) a service
+        runf = estack.enter_context(write_new(run_file_path))
+        runf.write('set -e\n')
+        for command in pre_start_commands or []:
+            _write_command(ctx, runf, command)
+        init_containers = init_containers or []
+        if init_containers:
+            _write_init_container_cmds_clean(ctx, runf, init_containers[0])
+        for idx, ic in enumerate(init_containers):
+            _write_init_container_cmds(ctx, runf, idx, ic)
+        _write_container_cmd_to_bash(ctx, runf, container, ident.daemon_name)
+
+        # some metadata about the deploy
+        metaf = estack.enter_context(write_new(meta_file_path))
+        meta: Dict[str, Any] = fetch_meta(ctx)
+        meta.update(
+            {
+                'memory_request': int(ctx.memory_request)
+                if ctx.memory_request
+                else None,
+                'memory_limit': int(ctx.memory_limit)
+                if ctx.memory_limit
+                else None,
+            }
+        )
+        if not meta.get('ports'):
+            if endpoints:
+                meta['ports'] = [e.port for e in endpoints]
+            else:
+                meta['ports'] = []
+        metaf.write(json.dumps(meta, indent=4) + '\n')
+
+        # post-stop command(s)
+        pstopf = estack.enter_context(write_new(post_stop_file_path))
+        # this is a fallback to eventually stop any underlying container that
+        # was not stopped properly by unit.stop, this could happen in very slow
+        # setups as described in the issue
+        # https://tracker.ceph.com/issues/58242.
+        _write_stop_actions(ctx, cast(TextIO, pstopf), container, timeout)
+        for command in post_stop_commands or []:
+            _write_command(ctx, pstopf, command)
+
+        # stop command(s)
+        stopf = estack.enter_context(write_new(stop_file_path))
+        _write_stop_actions(ctx, cast(TextIO, stopf), container, timeout)
+
+        if container:
+            imgf = estack.enter_context(write_new(image_file_path))
+            imgf.write(container.image + '\n')
+
+
+def _write_container_cmd_to_bash(
+    ctx: CephadmContext,
+    file_obj: IO[str],
+    container: 'CephContainer',
+    comment: Optional[str] = None,
+    background: Optional[bool] = False,
+) -> None:
+    if comment:
+        # Sometimes adding a comment, especially if there are multiple containers in one
+        # unit file, makes it easier to read and grok.
+        assert '\n' not in comment
+        file_obj.write(f'# {comment}\n')
+    # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually
+    _bash_cmd(
+        file_obj, container.rm_cmd(old_cname=True), check=False, stderr=False
+    )
+    _bash_cmd(file_obj, container.rm_cmd(), check=False, stderr=False)
+
+    # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
+    if isinstance(ctx.container_engine, Podman):
+        _bash_cmd(
+            file_obj,
+            container.rm_cmd(storage=True),
+            check=False,
+            stderr=False,
+        )
+        _bash_cmd(
+            file_obj,
+            container.rm_cmd(old_cname=True, storage=True),
+            check=False,
+            stderr=False,
+        )
+
+    # container run command
+    _bash_cmd(file_obj, container.run_cmd(), background=bool(background))
+
+
+def _write_init_container_cmds(
+    ctx: CephadmContext,
+    file_obj: IO[str],
+    index: int,
+    init_container: 'InitContainer',
+) -> None:
+    file_obj.write(f'# init container {index}: {init_container.cname}\n')
+    _bash_cmd(file_obj, init_container.run_cmd())
+    _write_init_container_cmds_clean(ctx, file_obj, init_container, comment='')
+
+
+def _write_init_container_cmds_clean(
+    ctx: CephadmContext,
+    file_obj: IO[str],
+    init_container: 'InitContainer',
+    comment: str = 'init container cleanup',
+) -> None:
+    if comment:
+        assert '\n' not in comment
+        file_obj.write(f'# {comment}\n')
+    _bash_cmd(
+        file_obj,
+        init_container.rm_cmd(),
+        check=False,
+        stderr=False,
+    )
+    # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
+    if isinstance(ctx.container_engine, Podman):
+        _bash_cmd(
+            file_obj,
+            init_container.rm_cmd(storage=True),
+            check=False,
+            stderr=False,
+        )
+
+
+def _write_stop_actions(
+    ctx: CephadmContext, f: TextIO, container: 'CephContainer', timeout: Optional[int]
+) -> None:
+    # following generated script basically checks if the container exists
+    # before stopping it. Exit code will be success either if it doesn't
+    # exist or if it exists and is stopped successfully.
+    container_exists = f'{ctx.container_engine.path} inspect %s &>/dev/null'
+    f.write(f'! {container_exists % container.old_cname} || {" ".join(container.stop_cmd(old_cname=True, timeout=timeout))} \n')
+    f.write(f'! {container_exists % container.cname} || {" ".join(container.stop_cmd(timeout=timeout))} \n')
+
+
+def _bash_cmd(
+    fh: IO[str],
+    cmd: List[str],
+    check: bool = True,
+    background: bool = False,
+    stderr: bool = True,
+) -> None:
+    line = ' '.join(shlex.quote(arg) for arg in cmd)
+    if not check:
+        line = f'! {line}'
+    if not stderr:
+        line = f'{line} 2> /dev/null'
+    if background:
+        line = f'{line} &'
+    fh.write(line)
+    fh.write('\n')
+
+
+def _write_command(
+    ctx: CephadmContext,
+    fh: IO[str],
+    cmd: Command,
+) -> None:
+    """Wrapper func for turning a command list or string into something suitable
+    for appending to a run script.
+    """
+    if isinstance(cmd, list):
+        _bash_cmd(fh, cmd)
+    elif isinstance(cmd, ContainerCommand):
+        _write_container_cmd_to_bash(
+            ctx,
+            fh,
+            cmd.container,
+            comment=cmd.comment,
+            background=cmd.background,
+        )
+    else:
+        fh.write(cmd)
+        if not cmd.endswith('\n'):
+            fh.write('\n')

From 4e37eb7356c14220672c6b27e369f9ecc298c5ee Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 16 Nov 2023 16:38:38 -0500
Subject: [PATCH 0998/2492] cephadm: black format runscripts.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/runscripts.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
index a5ac1a805c6b..7772248f362d 100644
--- a/src/cephadm/cephadmlib/runscripts.py
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -154,7 +154,9 @@ def _write_init_container_cmds(
 ) -> None:
     file_obj.write(f'# init container {index}: {init_container.cname}\n')
     _bash_cmd(file_obj, init_container.run_cmd())
-    _write_init_container_cmds_clean(ctx, file_obj, init_container, comment='')
+    _write_init_container_cmds_clean(
+        ctx, file_obj, init_container, comment=''
+    )
 
 
 def _write_init_container_cmds_clean(
@@ -183,14 +185,21 @@ def _write_init_container_cmds_clean(
 
 
 def _write_stop_actions(
-    ctx: CephadmContext, f: TextIO, container: 'CephContainer', timeout: Optional[int]
+    ctx: CephadmContext,
+    f: TextIO,
+    container: 'CephContainer',
+    timeout: Optional[int],
 ) -> None:
     # following generated script basically checks if the container exists
     # before stopping it. Exit code will be success either if it doesn't
     # exist or if it exists and is stopped successfully.
     container_exists = f'{ctx.container_engine.path} inspect %s &>/dev/null'
-    f.write(f'! {container_exists % container.old_cname} || {" ".join(container.stop_cmd(old_cname=True, timeout=timeout))} \n')
-    f.write(f'! {container_exists % container.cname} || {" ".join(container.stop_cmd(timeout=timeout))} \n')
+    f.write(
+        f'! {container_exists % container.old_cname} || {" ".join(container.stop_cmd(old_cname=True, timeout=timeout))} \n'
+    )
+    f.write(
+        f'! {container_exists % container.cname} || {" ".join(container.stop_cmd(timeout=timeout))} \n'
+    )
 
 
 def _bash_cmd(

From 9d10361961e01c4d60f600f6e7ed106d6203e554 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 8 Dec 2023 20:45:43 +0100
Subject: [PATCH 0999/2492] doc/radosgw: edit front matter - role.rst

Clarify the front matter in doc/radosgw/role.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/role.rst | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
index 3ef78d8d209c..029d8b778d63 100644
--- a/doc/radosgw/role.rst
+++ b/doc/radosgw/role.rst
@@ -2,9 +2,15 @@
  Role
 ======
 
-A role is similar to a user and has permission policies attached to it, that determine what a role can or can not do. A role can be assumed by any identity that needs it. If a user assumes a role, a set of dynamically created temporary credentials are returned to the user. A role can be used to delegate access to users, applications, services that do not have permissions to access some s3 resources.
-
-The following radosgw-admin commands can be used to create/ delete/ update a role and permissions associated with a role.
+A role is similar to a user. It has permission policies attached to it that
+determine what it can do and what it cannot do. A role can be assumed by any
+identity that needs it. When a user assumes a role, a set of
+dynamically-created temporary credentials are provided to the user. A role can
+be used to delegate access to users, to applications, and to services that do
+not have permissions to access certain S3 resources.
+
+The following ``radosgw-admin`` commands can be used to create or delete or
+update a role and the permissions associated with it.
 
 Create a Role
 -------------

From 0e8663b93a236d932c0d3f56f291677e38356e84 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 24 Oct 2023 19:55:28 -0700
Subject: [PATCH 1000/2492] crimson/osd: explicitly instantiate
 interrupt_cond<IOInterruptCondition>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Solves an FTBFS when using with_interruption from pg_commands.cc:

In file included from ../src/crimson/admin/pg_commands.cc:14:
../src/crimson/common/interruptible_future.h:135:46: error: redefinition of ‘bool __tls_guard’
  135 | thread_local interrupt_cond_t<InterruptCond> interrupt_cond;
      |                                              ^~~~~~~~~~~~~~
In file included from ../src/common/TrackedOp.h:18,
                 from ../src/os/ObjectStore.h:25,
                 from ../src/osd/PGLog.h:23,
                 from ../src/osd/PeeringState.h:19,
                 from ../src/crimson/osd/shard_services.h:17,
                 from ../src/crimson/osd/pg_shard_manager.h:10,
                 from ../src/crimson/osd/osd.h:23,
                 from ../src/crimson/admin/pg_commands.cc:16:
../src/common/StackStringStream.h:188:36: note: ‘bool __tls_guard’ previously declared here
  188 |   inline static thread_local Cache cache;

This matches what we already do with
interrupt_cond_t<crimson::os::seastore::TransactionConflictCondition>.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h          | 7 +++++++
 src/crimson/osd/pg_interval_interrupt_condition.cc | 5 +++++
 2 files changed, 12 insertions(+)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 6ed11088e1a7..857704dab378 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -53,6 +53,10 @@ namespace crimson::os::seastore {
   class TransactionConflictCondition;
 }
 
+namespace crimson::osd {
+  class IOInterruptCondition;
+}
+
 // GCC tries to instantiate
 // seastar::lw_shared_ptr<crimson::os::seastore::TransactionConflictCondition>.
 // but we *may* not have the definition of TransactionConflictCondition at this moment,
@@ -136,6 +140,9 @@ struct interrupt_cond_t {
 template <typename InterruptCond>
 thread_local interrupt_cond_t<InterruptCond> interrupt_cond;
 
+extern template thread_local interrupt_cond_t<crimson::osd::IOInterruptCondition>
+interrupt_cond<crimson::osd::IOInterruptCondition>;
+
 extern template thread_local interrupt_cond_t<crimson::os::seastore::TransactionConflictCondition>
 interrupt_cond<crimson::os::seastore::TransactionConflictCondition>;
 
diff --git a/src/crimson/osd/pg_interval_interrupt_condition.cc b/src/crimson/osd/pg_interval_interrupt_condition.cc
index 36243b825752..b72044d4e57e 100644
--- a/src/crimson/osd/pg_interval_interrupt_condition.cc
+++ b/src/crimson/osd/pg_interval_interrupt_condition.cc
@@ -8,6 +8,11 @@
 
 SET_SUBSYS(osd);
 
+namespace crimson::interruptible {
+template thread_local interrupt_cond_t<crimson::osd::IOInterruptCondition>
+interrupt_cond<crimson::osd::IOInterruptCondition>;
+}
+
 namespace crimson::osd {
 
 IOInterruptCondition::IOInterruptCondition(Ref<PG>& pg)

From dcc09b00f3c879ea2b7dbecefdaa348c2e54a38f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 24 Oct 2023 14:50:17 -0700
Subject: [PATCH 1001/2492] crimson/.../snaptrim_event.h: add TODO about
 delta_stats bug

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 5afba7d7a47e..12f244512ac3 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -142,6 +142,8 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
   CommonPGPipeline& client_pp();
 
 private:
+  /* TODO: we don't actually update the PG's stats
+   * https://tracker.ceph.com/issues/63307 */
   object_stat_sum_t delta_stats;
 
   remove_or_update_iertr::future<> remove_clone(

From 8efb26a8e7b25049c0a7a8cc965d7600ef975e05 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 23 Oct 2023 16:55:10 -0700
Subject: [PATCH 1002/2492] crimson/.../client_request: make logging uniform,
 clarify

- Switch to DEBUGDPPU to add pg prefix to each line
- Pass this_instance_id through for uniformity
- Clarify log lines particularly entering/leaving stages

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 .../osd/osd_operations/client_request.cc      | 241 +++++++++---------
 .../osd/osd_operations/client_request.h       |   8 +-
 src/crimson/osd/pg.cc                         |   2 +-
 3 files changed, 133 insertions(+), 118 deletions(-)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 3fba73a0c235..62eb82a281a9 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -5,6 +5,7 @@
 #include "messages/MOSDOpReply.h"
 
 #include "crimson/common/exception.h"
+#include "crimson/common/log.h"
 #include "crimson/osd/pg.h"
 #include "crimson/osd/osd.h"
 #include "common/Formatter.h"
@@ -13,12 +14,6 @@
 #include "crimson/osd/osd_connection_priv.h"
 #include "osd/object_state_fmt.h"
 
-namespace {
-  seastar::logger& logger() {
-    return crimson::get_logger(ceph_subsys_osd);
-  }
-}
-
 SET_SUBSYS(osd);
 
 namespace crimson::osd {
@@ -29,19 +24,17 @@ void ClientRequest::Orderer::requeue(
 {
   LOG_PREFIX(ClientRequest::Orderer::requeue);
   for (auto &req: list) {
-    DEBUGI("{}: {} requeueing {}", __func__, *pg, req);
+    DEBUGDPP("requeueing {}", *pg, req);
     req.reset_instance_handle();
     std::ignore = req.with_pg_int(shard_services, pg);
   }
 }
 
-void ClientRequest::Orderer::clear_and_cancel()
+void ClientRequest::Orderer::clear_and_cancel(PG &pg)
 {
   LOG_PREFIX(ClientRequest::Orderer::clear_and_cancel);
   for (auto i = list.begin(); i != list.end(); ) {
-    DEBUGI(
-      "ClientRequest::Orderer::clear_and_cancel: {}",
-      *i);
+    DEBUGDPP("{}", pg, *i);
     i->complete_request();
     remove_request(*(i++));
   }
@@ -64,8 +57,8 @@ ClientRequest::ClientRequest(
 
 ClientRequest::~ClientRequest()
 {
-  LOG_PREFIX(ClientRequest::~ClientRequest);
-  DEBUGI("{}: destroying", *this);
+  LOG_PREFIX(~ClientRequest);
+  DEBUG("{}: destroying", *this);
 }
 
 void ClientRequest::print(std::ostream &lhs) const
@@ -76,7 +69,7 @@ void ClientRequest::print(std::ostream &lhs) const
 void ClientRequest::dump_detail(Formatter *f) const
 {
   LOG_PREFIX(ClientRequest::dump_detail);
-  DEBUGI("{}: dumping", *this);
+  TRACE("{}: dumping", *this);
   std::apply([f] (auto... event) {
     (..., event.dump(f));
   }, tracking_events);
@@ -110,7 +103,7 @@ seastar::future<> ClientRequest::with_pg_int(
 {
   LOG_PREFIX(ClientRequest::with_pg_int);
   epoch_t same_interval_since = pgref->get_interval_start_epoch();
-  DEBUGI("{} same_interval_since: {}", *this, same_interval_since);
+  DEBUGDPP("{}: same_interval_since: {}", *pgref, *this, same_interval_since);
   if (m->finish_decode()) {
     m->clear_payload();
   }
@@ -119,70 +112,70 @@ seastar::future<> ClientRequest::with_pg_int(
   auto instance_handle = get_instance_handle();
   auto &ihref = *instance_handle;
   return interruptor::with_interruption(
-    [this, pgref, this_instance_id, &ihref, &shard_services]() mutable {
-      LOG_PREFIX(ClientRequest::with_pg_int);
-      DEBUGI("{} start", *this);
+    [FNAME, this, pgref, this_instance_id, &ihref, &shard_services]() mutable {
+      DEBUGDPP("{} start", *pgref, *this);
       PG &pg = *pgref;
       if (pg.can_discard_op(*m)) {
 	return shard_services.send_incremental_map(
 	  std::ref(*conn), m->get_map_epoch()
-	).then([this, this_instance_id, pgref] {
-          LOG_PREFIX(ClientRequest::with_pg_int);
-	  DEBUGI("{}.{}: discarding", *this, this_instance_id);
+	).then([FNAME, this, this_instance_id, pgref] {
+	  DEBUGDPP("{}: discarding {}", *pgref, *this, this_instance_id);
 	  pgref->client_request_orderer.remove_request(*this);
 	  complete_request();
 	  return interruptor::now();
 	});
       }
+      DEBUGDPP("{}.{}: entering await_map stage",
+	       *pgref, *this, this_instance_id);
       return ihref.enter_stage<interruptor>(client_pp(pg).await_map, *this
-      ).then_interruptible([this, this_instance_id, &pg, &ihref] {
-        LOG_PREFIX(ClientRequest::with_pg_int);
-	DEBUGI("{}.{}: after await_map stage", *this, this_instance_id);
+      ).then_interruptible([FNAME, this, this_instance_id, &pg, &ihref] {
+	DEBUGDPP("{}.{}: entered await_map stage, waiting for map",
+		 pg, *this, this_instance_id);
 	return ihref.enter_blocker(
 	  *this, pg.osdmap_gate, &decltype(pg.osdmap_gate)::wait_for_map,
 	  m->get_min_epoch(), nullptr);
-      }).then_interruptible([this, this_instance_id, &pg, &ihref](auto map) {
-        LOG_PREFIX(ClientRequest::with_pg_int);
-	DEBUGI("{}.{}: after wait_for_map", *this, this_instance_id);
+      }).then_interruptible(
+	[FNAME, this, this_instance_id, &pg, &ihref](auto map_epoch) {
+	DEBUGDPP("{}.{}: map epoch got {}, entering wait_for_active",
+		 pg, *this, this_instance_id, map_epoch);
 	return ihref.enter_stage<interruptor>(client_pp(pg).wait_for_active, *this);
-      }).then_interruptible([this, this_instance_id, &pg, &ihref]() {
-        LOG_PREFIX(ClientRequest::with_pg_int);
-	DEBUGI("{}.{}: after wait_for_active stage", *this, this_instance_id);
+      }).then_interruptible([FNAME, this, this_instance_id, &pg, &ihref]() {
+	DEBUGDPP("{}.{}: entered wait_for_active stage, waiting for active",
+		 pg, *this, this_instance_id);
 	return ihref.enter_blocker(
 	  *this,
 	  pg.wait_for_active_blocker,
 	  &decltype(pg.wait_for_active_blocker)::wait);
-      }).then_interruptible([this, pgref, this_instance_id, &ihref]() mutable
-			    -> interruptible_future<> {
-        LOG_PREFIX(ClientRequest::with_pg_int);
-	DEBUGI("{}.{}: after wait_for_active", *this, this_instance_id);
+	}).then_interruptible(
+	  [FNAME, this, pgref, this_instance_id, &ihref]() mutable
+	  -> interruptible_future<> {
+	DEBUGDPP("{}.{}: pg active, entering process[_pg]_op",
+		 *pgref, *this, this_instance_id);
 	if (is_pg_op()) {
 	  return process_pg_op(pgref);
 	} else {
-	  return process_op(ihref, pgref);
+	  return process_op(ihref, pgref, this_instance_id);
 	}
-      }).then_interruptible([this, this_instance_id, &ihref] {
-        logger().debug("{}.{}: complete", *this, this_instance_id);
+      }).then_interruptible([FNAME, this, this_instance_id, pgref, &ihref] {
+	DEBUGDPP("{}.{}: process[_pg]_op complete, completing handle",
+		 *pgref, *this, this_instance_id);
         return ihref.handle.complete();
-      }).then_interruptible([this, this_instance_id, pgref] {
-        LOG_PREFIX(ClientRequest::with_pg_int);
-	DEBUGI("{}.{}: after process*", *this, this_instance_id);
+      }).then_interruptible([FNAME, this, this_instance_id, pgref] {
+	DEBUGDPP("{}.{}: process[_pg]_op complete,"
+		 "removing request from orderer",
+		 *pgref, *this, this_instance_id);
 	pgref->client_request_orderer.remove_request(*this);
 	complete_request();
       });
-    }, [this, this_instance_id, pgref](std::exception_ptr eptr) {
-      LOG_PREFIX(ClientRequest::with_pg_int);
-      // TODO: better debug output
-      DEBUGI("{}.{}: interrupted {}", *this, this_instance_id, eptr);
-    },
-    pgref
-  ).finally(
-    [opref=std::move(opref), pgref,
-     instance_handle=std::move(instance_handle), &ihref,
-     this_instance_id, this] {
-    logger().debug("{}.{}: exit", *this, this_instance_id);
-    ihref.handle.exit();
-  });
+    }, [FNAME, this, this_instance_id, pgref](std::exception_ptr eptr) {
+      DEBUGDPP("{}.{}: interrupted due to {}",
+	       *pgref, *this, this_instance_id, eptr);
+    }, pgref).finally(
+      [this, FNAME, opref=std::move(opref), pgref=std::move(pgref),
+       this_instance_id, instance_handle=std::move(instance_handle), &ihref] {
+	DEBUGDPP("{}.{}: exit", *pgref, *this, this_instance_id);
+	ihref.handle.exit();
+    });
 }
 
 seastar::future<> ClientRequest::with_pg(
@@ -212,7 +205,7 @@ ClientRequest::process_pg_op(
 auto ClientRequest::reply_op_error(const Ref<PG>& pg, int err)
 {
   LOG_PREFIX(ClientRequest::reply_op_error);
-  DEBUGI("{}: replying with error {}", *this, err);
+  DEBUGDPP("{}: replying with error {}", *pg, *this, err);
   auto reply = crimson::make_message<MOSDOpReply>(
     m.get(), err, pg->get_osdmap_epoch(),
     m->get_flags() & (CEPH_OSD_FLAG_ACK|CEPH_OSD_FLAG_ONDISK),
@@ -224,17 +217,23 @@ auto ClientRequest::reply_op_error(const Ref<PG>& pg, int err)
 }
 
 ClientRequest::interruptible_future<>
-ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
+ClientRequest::process_op(
+  instance_handle_t &ihref, Ref<PG> &pg, unsigned this_instance_id)
 {
+  LOG_PREFIX(ClientRequest::process_op);
   return ihref.enter_stage<interruptor>(
     client_pp(*pg).recover_missing, *this
   ).then_interruptible([pg, this]() mutable {
     return recover_missings(pg, m->get_hobj(), snaps_need_to_recover());
-  }).then_interruptible([this, pg, &ihref]() mutable {
+  }).then_interruptible([FNAME, this, pg, this_instance_id, &ihref]() mutable {
+    DEBUGDPP("{}.{}: checking already_complete",
+	     *pg, *this, this_instance_id);
     return pg->already_complete(m->get_reqid()).then_interruptible(
-      [this, pg, &ihref](auto completed) mutable
+      [FNAME, this, pg, this_instance_id, &ihref](auto completed) mutable
       -> PG::load_obc_iertr::future<> {
       if (completed) {
+	DEBUGDPP("{}.{}: already completed, sending reply",
+		 *pg, *this, this_instance_id);
         auto reply = crimson::make_message<MOSDOpReply>(
           m.get(), completed->err, pg->get_osdmap_epoch(),
           CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK, false);
@@ -242,30 +241,38 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
         // TODO: gate the crosscore sending
         return conn->send_with_throttling(std::move(reply));
       } else {
+	DEBUGDPP("{}.{}: not completed, entering get_obc stage",
+		 *pg, *this, this_instance_id);
         return ihref.enter_stage<interruptor>(client_pp(*pg).get_obc, *this
 	).then_interruptible(
-          [this, pg, &ihref]() mutable -> PG::load_obc_iertr::future<> {
-          LOG_PREFIX(ClientRequest::process_op);
-          DEBUGI("{}: in get_obc stage", *this);
+          [FNAME, this, pg, this_instance_id, &ihref]() mutable
+	  -> PG::load_obc_iertr::future<> {
+	  DEBUGDPP("{}.{}: entered get_obc stage, about to wait_scrub",
+		   *pg, *this, this_instance_id);
           op_info.set_from_op(&*m, *pg->get_osdmap());
-          return pg->with_locked_obc(
-            m->get_hobj(), op_info,
-            [this, pg, &ihref](auto head, auto obc) mutable {
-              LOG_PREFIX(ClientRequest::process_op);
-              DEBUGI("{}: got obc {}", *this, obc->obs);
-              return ihref.enter_stage<interruptor>(
-                client_pp(*pg).process, *this
-              ).then_interruptible([this, pg, obc, &ihref]() mutable {
-                return do_process(ihref, pg, obc);
-              });
-            });
+	  return pg->with_locked_obc(
+	    m->get_hobj(), op_info,
+	    [FNAME, this, pg, this_instance_id, &ihref](
+	      auto head, auto obc) mutable {
+	      DEBUGDPP("{}.{}: got obc {}, entering process stage",
+		       *pg, *this, this_instance_id, obc->obs);
+	      return ihref.enter_stage<interruptor>(
+		client_pp(*pg).process, *this
+	      ).then_interruptible(
+		[FNAME, this, pg, this_instance_id, obc, &ihref]() mutable {
+		  DEBUGDPP("{}.{}: in process stage, calling do_process",
+			   *pg, *this, this_instance_id);
+		  return do_process(ihref, pg, obc, this_instance_id);
+		});
+	    });
         });
       }
     });
   }).handle_error_interruptible(
-    PG::load_obc_ertr::all_same_way([this, pg=std::move(pg)](const auto &code) {
-      LOG_PREFIX(ClientRequest::process_op);
-      ERRORI("ClientRequest saw error code {}", code);
+    PG::load_obc_ertr::all_same_way(
+      [FNAME, this, pg=std::move(pg), this_instance_id](const auto &code) {
+      DEBUGDPP("{}.{}: saw error code {}",
+	       *pg, *this, this_instance_id, code);
       assert(code.value() > 0);
       return reply_op_error(pg, -code.value());
   }));
@@ -274,7 +281,8 @@ ClientRequest::process_op(instance_handle_t &ihref, Ref<PG> &pg)
 ClientRequest::interruptible_future<>
 ClientRequest::do_process(
   instance_handle_t &ihref,
-  Ref<PG>& pg, crimson::osd::ObjectContextRef obc)
+  Ref<PG>& pg, crimson::osd::ObjectContextRef obc,
+  unsigned this_instance_id)
 {
   LOG_PREFIX(ClientRequest::do_process);
   if (m->has_flag(CEPH_OSD_FLAG_PARALLELEXEC)) {
@@ -284,10 +292,12 @@ ClientRequest::do_process(
   if (pool.has_flag(pg_pool_t::FLAG_EIO)) {
     // drop op on the floor; the client will handle returning EIO
     if (m->has_flag(CEPH_OSD_FLAG_SUPPORTSPOOLEIO)) {
-      DEBUGI("discarding op due to pool EIO flag");
+      DEBUGDPP("{}.{}: discarding op due to pool EIO flag",
+	       *pg, *this, this_instance_id);
       return seastar::now();
     } else {
-      DEBUGI("replying EIO due to pool EIO flag");
+      DEBUGDPP("{}.{}: replying EIO due to pool EIO flag",
+	       *pg, *this, this_instance_id);
       return reply_op_error(pg, -EIO);
     }
   }
@@ -303,7 +313,8 @@ ClientRequest::do_process(
   } else if (m->get_hobj().oid.name.empty()) {
     return reply_op_error(pg, -EINVAL);
   } else if (pg->get_osdmap()->is_blocklisted(conn->get_peer_addr())) {
-    logger().info("{} is blocklisted", conn->get_peer_addr());
+    DEBUGDPP("{}.{}: {} is blocklisted",
+	     *pg, *this, this_instance_id, conn->get_peer_addr());
     return reply_op_error(pg, -EBLOCKLISTED);
   }
 
@@ -313,58 +324,61 @@ ClientRequest::do_process(
 
   SnapContext snapc = get_snapc(pg,obc);
 
-  if (m->has_flag(CEPH_OSD_FLAG_ORDERSNAP) &&
-      snapc.seq < obc->ssc->snapset.seq) {
-    DEBUGI("{} ORDERSNAP flag set and snapc seq {}",
-           " < snapset seq {} on {}",
-           __func__, snapc.seq, obc->ssc->snapset.seq,
-           obc->obs.oi.soid);
+  if ((m->has_flag(CEPH_OSD_FLAG_ORDERSNAP)) &&
+       snapc.seq < obc->ssc->snapset.seq) {
+    DEBUGDPP("{}.{}: ORDERSNAP flag set "
+	     "and snapc seq {} < snapset seq {} on {}",
+	     *pg, *this, this_instance_id,
+	     snapc.seq, obc->ssc->snapset.seq,
+	     obc->obs.oi.soid);
     return reply_op_error(pg, -EOLDSNAPC);
   }
 
   if (!pg->is_primary()) {
     // primary can handle both normal ops and balanced reads
     if (is_misdirected(*pg)) {
-      TRACEI("do_process: dropping misdirected op");
+      DEBUGDPP("{}.{}: dropping misdirected op",
+	       *pg, *this, this_instance_id);
       return seastar::now();
     } else if (const hobject_t& hoid = m->get_hobj();
                !pg->get_peering_state().can_serve_replica_read(hoid)) {
-      DEBUGI("{}: unstable write on replica, "
-	             "bouncing to primary",
-                     __func__);
+      DEBUGDPP("{}.{}: unstable write on replica, bouncing to primary",
+	       *pg, *this, this_instance_id);
       return reply_op_error(pg, -EAGAIN);
     } else {
-      DEBUGI("{}: serving replica read on oid {}",
-                     __func__, m->get_hobj());
+      DEBUGDPP("{}.{}: serving replica read on oid {}",
+	       *pg, *this, this_instance_id, m->get_hobj());
     }
   }
   return pg->do_osd_ops(m, conn, obc, op_info, snapc).safe_then_unpack_interruptible(
-    [this, pg, &ihref](auto submitted, auto all_completed) mutable {
-      logger().debug("do_process::{} in submitted", *this);
-      return submitted.then_interruptible([this, pg, &ihref] {
-	logger().debug("do_process::{} in enter_stage wait_repop", *this);
+    [FNAME, this, pg, this_instance_id, &ihref](
+      auto submitted, auto all_completed) mutable {
+      return submitted.then_interruptible(
+	[FNAME, this, pg, this_instance_id, &ihref] {
 	return ihref.enter_stage<interruptor>(client_pp(*pg).wait_repop, *this);
       }).then_interruptible(
-	[this, pg, all_completed=std::move(all_completed), &ihref]() mutable {
-	  logger().debug("do_process::{} in all_completed", *this);
+	[FNAME, this, pg, this_instance_id,
+	 all_completed=std::move(all_completed), &ihref]() mutable {
 	  return all_completed.safe_then_interruptible(
-	    [this, pg, &ihref](MURef<MOSDOpReply> reply) {
+	    [FNAME, this, pg, this_instance_id, &ihref](
+	      MURef<MOSDOpReply> reply) {
 	      return ihref.enter_stage<interruptor>(client_pp(*pg).send_reply, *this
 	      ).then_interruptible(
-		[this, reply=std::move(reply)]() mutable {
-                  LOG_PREFIX(ClientRequest::do_process);
-		  DEBUGI("{}: sending response", *this);
-		  // TODO: gate the crosscore sending
-		  return conn->send_with_throttling(std::move(reply));
-		}
-	      );
-	    }, crimson::ct_error::eagain::handle([this, pg, &ihref]() mutable {
-	      return process_op(ihref, pg);
+		[FNAME, this, pg, this_instance_id,
+		 reply=std::move(reply)]() mutable {
+		  DEBUGDPP("{}.{}: sending response",
+			   *pg, *this, this_instance_id);
+		  return conn->send(std::move(reply));
+		});
+	    }, crimson::ct_error::eagain::handle(
+	      [FNAME, this, pg, this_instance_id, &ihref]() mutable {
+		return process_op(ihref, pg, this_instance_id);
 	    }));
 	});
-    }, crimson::ct_error::eagain::handle([this, pg, &ihref]() mutable {
-      return process_op(ihref, pg);
-    }));
+    }, crimson::ct_error::eagain::handle(
+      [FNAME, this, pg, this_instance_id, &ihref]() mutable {
+	return process_op(ihref, pg, this_instance_id);
+      }));
 }
 
 bool ClientRequest::is_misdirected(const PG& pg) const
@@ -405,15 +419,14 @@ const SnapContext ClientRequest::get_snapc(
     if (pg->get_pgpool().info.is_pool_snaps_mode()) {
       // use pool's snapc
       snapc = pg->get_pgpool().snapc;
-      DEBUGI("{} using pool's snapc snaps={}",
-                     __func__, snapc.snaps);
-
+      DEBUGDPP("{} using pool's snapc snaps={}",
+	       *pg, *this, snapc.snaps);
     } else {
       // client specified snapc
       snapc.seq = m->get_snap_seq();
       snapc.snaps = m->get_snaps();
-      DEBUGI("{} client specified snapc seq={} snaps={}",
-                     __func__, snapc.seq, snapc.snaps);
+      DEBUGDPP("{}: client specified snapc seq={} snaps={}",
+	       *pg, *this, snapc.seq, snapc.snaps);
     }
   }
   return snapc;
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 43535e156c5b..6599e4dbc6ab 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -194,7 +194,7 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
       intrusive_ptr_release(&request);
     }
     void requeue(ShardServices &shard_services, Ref<PG> pg);
-    void clear_and_cancel();
+    void clear_and_cancel(PG &pg);
   };
   void complete_request();
 
@@ -252,14 +252,16 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
   interruptible_future<> do_process(
     instance_handle_t &ihref,
     Ref<PG>& pg,
-    crimson::osd::ObjectContextRef obc);
+    crimson::osd::ObjectContextRef obc,
+    unsigned this_instance_id);
   ::crimson::interruptible::interruptible_future<
     ::crimson::osd::IOInterruptCondition> process_pg_op(
     Ref<PG> &pg);
   ::crimson::interruptible::interruptible_future<
     ::crimson::osd::IOInterruptCondition> process_op(
       instance_handle_t &ihref,
-      Ref<PG> &pg);
+      Ref<PG> &pg,
+      unsigned this_instance_id);
   bool is_pg_op() const;
 
   PGPipeline &client_pp(PG &pg);
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 00592eb8347e..e303ecf9a45c 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1530,7 +1530,7 @@ void PG::on_change(ceph::os::Transaction &t) {
     client_request_orderer.requeue(shard_services, this);
   } else {
     logger().debug("{} {}: dropping requests", *this, __func__);
-    client_request_orderer.clear_and_cancel();
+    client_request_orderer.clear_and_cancel(*this);
   }
 }
 

From 6397b1cae91fb1eaf1585bfd2c0c2780b98eb900 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 17 Oct 2023 02:38:17 +0000
Subject: [PATCH 1003/2492] qa/.../crimson-rados/.../thrashers/default.yaml:
 enable wait-for-scrub

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/suites/crimson-rados/thrash/thrashers/default.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qa/suites/crimson-rados/thrash/thrashers/default.yaml b/qa/suites/crimson-rados/thrash/thrashers/default.yaml
index 5ffbcbd7ffa7..25bbee02fbe4 100644
--- a/qa/suites/crimson-rados/thrash/thrashers/default.yaml
+++ b/qa/suites/crimson-rados/thrash/thrashers/default.yaml
@@ -1,6 +1,5 @@
 overrides:
   ceph:
-    wait-for-scrub: false
     log-ignorelist:
     - but it is still running
     - objects unfound and apparently lost

From 41f930ae26034407d948f6be5ed35097de1db063 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 17 Oct 2023 14:58:51 +0000
Subject: [PATCH 1004/2492] qa/.../crimson-rados/perf/.../fixed-2.yaml: fix
 ignorelist

It's common during cluster setup for there to be periods with
degraded/recovering PGs.  Ignore those errors.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/suites/crimson-rados/perf/clusters/fixed-2.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
index 8ab2f228ebd5..3f322f371496 100644
--- a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
@@ -2,6 +2,11 @@ roles:
 - [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0]
 overrides:
   ceph:
+    log-ignorelist:
+      - \(PG_
+      - \(OSD_
+      - \(OBJECT_
+      - overall HEALTH
     conf:
       osd:
         osd shutdown pgref assert: true

From 070fa433282fa9592380cd3972e7294f49aa8b84 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 17 Oct 2023 17:34:38 -0700
Subject: [PATCH 1005/2492] qa/.../crimson-rados/.../thrashers/default.yaml:
 disable reset_purged_snaps for crimson

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/suites/crimson-rados/thrash/thrashers/default.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/suites/crimson-rados/thrash/thrashers/default.yaml b/qa/suites/crimson-rados/thrash/thrashers/default.yaml
index 25bbee02fbe4..bbb33324706b 100644
--- a/qa/suites/crimson-rados/thrash/thrashers/default.yaml
+++ b/qa/suites/crimson-rados/thrash/thrashers/default.yaml
@@ -32,3 +32,4 @@ tasks:
     ceph_objectstore_tool: false
     chance_inject_pause_short: 0
     chance_thrash_cluster_full: 0
+    chance_reset_purged_snaps_last: 0

From 50e5b37f26b6744fb497177c73d6fed122fc9602 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 18 Sep 2023 13:57:21 -0700
Subject: [PATCH 1006/2492] crimson/os/alienstore/alien_store.cc: don't
 pre-size array to max

Scrub wants to be able to pass numeric_limits::max() as limit
since we've already selected bounds based on the number of objects.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/os/alienstore/alien_store.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 61f23de97711..593716df3519 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -202,7 +202,6 @@ AlienStore::list_objects(CollectionRef ch,
   assert(tp);
   return do_with_op_gate(std::vector<ghobject_t>(), ghobject_t(),
                          [=, this] (auto &objects, auto &next) {
-    objects.reserve(limit);
     return tp->submit(ch->get_cid().hash_to_shard(tp->size()),
       [=, this, &objects, &next] {
       auto c = static_cast<AlienCollection*>(ch.get());

From f52580d786dc74779c412bc3acab11295f71d97b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 16 Oct 2023 21:33:08 -0700
Subject: [PATCH 1007/2492] os/bluestore/BlueStore: fix crimson-only
 omap_get_values start handling

seek_to_first() if start is nullopt, upper_bound() otherwise.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index b8dca31c0575..0345d9ec2cd3 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -13277,7 +13277,11 @@ int BlueStore::omap_get_values(
       r = -ENOENT;
       goto out;
     }
-    iter->upper_bound(*start_after);
+    if (start_after) {
+      iter->upper_bound(*start_after);
+    } else {
+      iter->seek_to_first();
+    }
     for (; iter->valid(); iter->next()) {
       output->insert(make_pair(iter->key(), iter->value()));
     }

From b581d02d94ccf93724f170cff4de80ed10846a43 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 16 Oct 2023 20:47:17 -0700
Subject: [PATCH 1008/2492] crimson/osd/pg_backend: maintain num_objects_omap
 correctly

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/pg_backend.cc | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index 5e3e7a91c0f7..630efa586c78 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -990,6 +990,12 @@ PGBackend::remove(ObjectState& os, ceph::os::Transaction& txn,
   txn.remove(coll->get_cid(),
 	     ghobject_t{os.oi.soid, ghobject_t::NO_GEN, shard});
   delta_stats.num_bytes -= os.oi.size;
+
+  if (os.oi.is_omap()) {
+    os.oi.clear_flag(object_info_t::FLAG_OMAP);
+    delta_stats.num_objects_omap--;
+  }
+
   os.oi.size = 0;
   os.oi.new_object();
 
@@ -1599,7 +1605,10 @@ PGBackend::omap_set_vals(
   osd_op_params.clean_regions.mark_omap_dirty();
   delta_stats.num_wr++;
   delta_stats.num_wr_kb += shift_round_up(to_set_bl.length(), 10);
-  os.oi.set_flag(object_info_t::FLAG_OMAP);
+  if (!os.oi.is_omap()) {
+    os.oi.set_flag(object_info_t::FLAG_OMAP);
+    delta_stats.num_objects_omap++;
+  }
   os.oi.clear_omap_digest();
   return seastar::now();
 }
@@ -1616,7 +1625,10 @@ PGBackend::omap_set_header(
   txn.omap_setheader(coll->get_cid(), ghobject_t{os.oi.soid}, osd_op.indata);
   osd_op_params.clean_regions.mark_omap_dirty();
   delta_stats.num_wr++;
-  os.oi.set_flag(object_info_t::FLAG_OMAP);
+  if (!os.oi.is_omap()) {
+    os.oi.set_flag(object_info_t::FLAG_OMAP);
+    delta_stats.num_objects_omap++;
+  }
   os.oi.clear_omap_digest();
   return seastar::now();
 }

From 77fe84c0956d6cee40cc083a39824741c0b5cb0a Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 13 Oct 2023 00:44:40 +0000
Subject: [PATCH 1009/2492] qa/tasks/ceph: use tell <pgid> deep_scrub in
 osd_scrub_pgs

This is the more modern variant.  Crimson doesn't currently
support the pg <pgid> deep_scrub variant, so let's just use
this one generally.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/tasks/ceph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index a7b3ad4c5eec..e478fd40af2c 100644
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -1303,7 +1303,7 @@ def osd_scrub_pgs(ctx, config):
                     # request was missed.  do not do it every time because
                     # the scrub may be in progress or not reported yet and
                     # we will starve progress.
-                    manager.raw_cluster_cmd('pg', 'deep-scrub', pgid)
+                    manager.raw_cluster_cmd('tell', pgid, 'deep_scrub')
             if gap_cnt > retries:
                 raise RuntimeError('Exiting scrub checking -- not all pgs scrubbed.')
         if loop:

From f28000550acb4c1f11336905dc30255cb9dfc756 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 23 Oct 2023 16:45:58 -0700
Subject: [PATCH 1010/2492] common/hobject: introduce get_max_object_boundary

Scrub doesn't want to split snapshot sets across chunks.  The range,
however, is exclusive on the right.  get_max_object_boundary() gives
us a way to always include head and all snapshots in the same chunk
as it will sort greater than any legal object with the same head.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/hobject.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/common/hobject.h b/src/common/hobject.h
index 34191ccf5ec8..e35e2b0732f6 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -166,6 +166,7 @@ struct hobject_t {
     return ret;
   }
 
+  /// @return min hobject_t ret s.t. ret.get_head() == get_head()
   hobject_t get_object_boundary() const {
     if (is_max())
       return *this;
@@ -174,6 +175,15 @@ struct hobject_t {
     return ret;
   }
 
+  /// @return max hobject_t ret s.t. ret.get_head() == get_head()
+  hobject_t get_max_object_boundary() const {
+    if (is_max())
+      return *this;
+    // CEPH_SNAPDIR happens to sort above HEAD and MAX_SNAP and is no longer used
+    // for actual objects
+    return get_snapdir();
+  }
+
   /// @return head version of this hobject_t
   hobject_t get_head() const {
     hobject_t ret(*this);

From 6e8229b2260c301fe74708f13b61d155de555152 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Sep 2023 15:06:59 -0700
Subject: [PATCH 1011/2492] common/fmt_common.h: move optional<T> formatter
 from pg.cc

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/fmt_common.h | 15 +++++++++++++++
 src/crimson/osd/pg.cc   | 11 -----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/common/fmt_common.h b/src/common/fmt_common.h
index d68d6457dcb8..27e488aedceb 100644
--- a/src/common/fmt_common.h
+++ b/src/common/fmt_common.h
@@ -2,6 +2,8 @@
 // vim: ts=8 sw=2 smarttab
 #pragma once
 
+#include <optional>
+
 /**
  * \file default fmtlib formatters for specifically-tagged types
  */
@@ -61,4 +63,17 @@ struct formatter<T> {
   }
   bool verbose{true};
 };
+
+template <typename T>
+struct formatter<std::optional<T>> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+  template <typename FormatContext>
+  auto format(const std::optional<T> &v, FormatContext& ctx) const {
+    if (v.has_value()) {
+      return fmt::format_to(ctx.out(), "{}", *v);
+    }
+    return fmt::format_to(ctx.out(), "<null>");
+  }
+};
+
 }  // namespace fmt
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index e303ecf9a45c..013f564b5f12 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -64,17 +64,6 @@ std::ostream& operator<<(std::ostream& out, const signedspan& d)
 }
 }
 
-template <typename T>
-struct fmt::formatter<std::optional<T>> : fmt::formatter<T> {
-  template <typename FormatContext>
-  auto format(const std::optional<T>& v, FormatContext& ctx) const {
-    if (v.has_value()) {
-      return fmt::formatter<T>::format(*v, ctx);
-    }
-    return fmt::format_to(ctx.out(), "<null>");
-  }
-};
-
 namespace crimson::osd {
 
 using crimson::common::local_conf;

From 3a1640b12d26d18f975240bdfc5a1418603f6e4a Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 11 Oct 2023 22:13:10 -0700
Subject: [PATCH 1012/2492] common/fmt_common: add has_fmt_print_ctx concept
 and formatter

fmt_print_ctx avoids creating a string.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/fmt_common.h | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/common/fmt_common.h b/src/common/fmt_common.h
index 27e488aedceb..5db8bac43cd7 100644
--- a/src/common/fmt_common.h
+++ b/src/common/fmt_common.h
@@ -22,6 +22,8 @@
  * *or*
  * std::string alt_fmt_print(bool short_format) const
  * as public member functions.
+ * *or*
+ * auto fmt_print_ctx(auto &ctx) -> decltype(ctx.out());
  */
 template<class T>
 concept has_fmt_print = requires(T t) {
@@ -31,6 +33,11 @@ template<class T>
 concept has_alt_fmt_print = requires(T t) {
   { t.alt_fmt_print(bool{}) } -> std::same_as<std::string>;
 };
+template<class T>
+concept has_fmt_print_ctx = requires(
+  T t, fmt::buffer_context<char> &ctx) {
+  { t.fmt_print_ctx(ctx) } -> std::same_as<decltype(ctx.out())>;
+};
 
 namespace fmt {
 
@@ -64,6 +71,16 @@ struct formatter<T> {
   bool verbose{true};
 };
 
+template <has_fmt_print_ctx T>
+struct formatter<T> {
+  template <typename ParseContext>
+  constexpr auto parse(ParseContext& ctx) { return ctx.begin(); }
+  template <typename FormatContext>
+  auto format(const T& k, FormatContext& ctx) const {
+    return k.fmt_print_ctx(ctx);
+  }
+};
+
 template <typename T>
 struct formatter<std::optional<T>> {
   constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }

From 83cb1c730a8ad08796b6f36a10673faf3b8bcad2 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 14 Oct 2023 13:27:51 -0700
Subject: [PATCH 1013/2492] common/fmt_common: add has_formatter concept

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/fmt_common.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/common/fmt_common.h b/src/common/fmt_common.h
index 5db8bac43cd7..1a2a6eac9357 100644
--- a/src/common/fmt_common.h
+++ b/src/common/fmt_common.h
@@ -15,6 +15,10 @@
  * has a begin()/end() method pair. This is a problem because we have
  * such classes in Crimson.
  */
+
+template <typename T>
+concept has_formatter = fmt::has_formatter<T, fmt::format_context>::value;
+
 /**
  * Tagging classes that provide support for default fmtlib formatting,
  * by having either

From 74d606549785c4073e6b6e2c599f4bfeb2002de7 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 18 Sep 2023 11:49:29 -0700
Subject: [PATCH 1014/2492] crimson/common/interruptible_future: pass lambdas
 to seastar::repeat as mutable, don't move action

The lambda passed to seastar::repeat will be invoked multiple times,
don't pass an rvalue ref to call_with_interruption.  Also, declare
lambda as mutable in case action needs to be mutable.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 857704dab378..5f8323a66d14 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -1212,7 +1212,7 @@ struct interruptor
 	    (typename Iterator::reference x) mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action),
+		      action,
 		      std::forward<decltype(*begin)>(x)).to_future();
 	  })
       );
@@ -1224,7 +1224,7 @@ struct interruptor
 	    (typename Iterator::reference x) mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action),
+		      action,
 		      std::forward<decltype(*begin)>(x)).to_future();
 	  })
       );
@@ -1243,7 +1243,7 @@ struct interruptor
 	    (typename Iterator::reference x) mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action),
+		      action,
 		      std::forward<decltype(*begin)>(x));
 	  })
       );
@@ -1255,7 +1255,7 @@ struct interruptor
 	    (typename Iterator::reference x) mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action),
+		      action,
 		      std::forward<decltype(*begin)>(x));
 	  })
       );
@@ -1270,10 +1270,10 @@ struct interruptor
       return make_interruptible(
 	  ::seastar::repeat(
 	    [action=std::move(action),
-	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond] {
+	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond]() mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action)).to_future();
+		      action).to_future();
 	  })
       );
     } else {
@@ -1283,7 +1283,7 @@ struct interruptor
 	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond]() mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action)).to_future();
+		      action).to_future();
 	  })
       );
     }
@@ -1296,20 +1296,20 @@ struct interruptor
       return make_interruptible(
 	  ::seastar::repeat(
 	    [action=std::move(action),
-	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond] {
+	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond]() mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action));
+		      action);
 	  })
       );
     } else {
       return make_interruptible(
 	  ::crimson::repeat(
 	    [action=std::move(action),
-	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond] {
+	    interrupt_condition=interrupt_cond<InterruptCond>.interrupt_cond]() mutable {
 	    return call_with_interruption(
 		      interrupt_condition,
-		      std::move(action));
+		      action);
 	  })
       );
     }

From 1ea79f1e34305f5a2e7266ec790ad84c4953108c Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 19 Jul 2023 21:01:04 -0700
Subject: [PATCH 1015/2492] include/rados/rados_types.hpp: add
 err_t::has_errors()

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/include/rados/rados_types.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/include/rados/rados_types.hpp b/src/include/rados/rados_types.hpp
index 84023579b3ec..e19e52127646 100644
--- a/src/include/rados/rados_types.hpp
+++ b/src/include/rados/rados_types.hpp
@@ -137,6 +137,9 @@ struct err_t {
   bool has_snapset_corrupted() const {
     return errors & SNAPSET_CORRUPTED;
   }
+  bool has_errors() const {
+    return errors;
+  }
   bool has_shallow_errors() const {
     return errors & SHALLOW_ERRORS;
   }

From 3be59463cc3325344a1c871450831c07b45098ca Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 9 Oct 2023 14:08:13 -0700
Subject: [PATCH 1016/2492] osd/osd_types_fmt.h: add object_stat_sum_t
 formatter and operator<<

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/osd/osd_types_fmt.h | 58 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h
index ed0e48ae082a..d6d746d295f7 100644
--- a/src/osd/osd_types_fmt.h
+++ b/src/osd/osd_types_fmt.h
@@ -328,6 +328,64 @@ struct fmt::formatter<ScrubMap> {
   bool debug_log{false};
 };
 
+template <>
+struct fmt::formatter<object_stat_sum_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const object_stat_sum_t &stats, FormatContext& ctx) const
+  {
+#define FORMAT(FIELD) fmt::format_to(ctx.out(), #FIELD"={}, ", stats.FIELD);
+    fmt::format_to(ctx.out(), "object_stat_sum_t(");
+    FORMAT(num_bytes);
+    FORMAT(num_objects);
+    FORMAT(num_object_clones);
+    FORMAT(num_object_copies);
+    FORMAT(num_objects_missing_on_primary);
+    FORMAT(num_objects_missing);
+    FORMAT(num_objects_degraded);
+    FORMAT(num_objects_misplaced);
+    FORMAT(num_objects_unfound);
+    FORMAT(num_rd);
+    FORMAT(num_rd_kb);
+    FORMAT(num_wr);
+    FORMAT(num_wr_kb);
+    FORMAT(num_large_omap_objects);
+    FORMAT(num_objects_manifest);
+    FORMAT(num_omap_bytes);
+    FORMAT(num_omap_keys);
+    FORMAT(num_shallow_scrub_errors);
+    FORMAT(num_deep_scrub_errors);
+    FORMAT(num_scrub_errors);
+    FORMAT(num_objects_recovered);
+    FORMAT(num_bytes_recovered);
+    FORMAT(num_keys_recovered);
+    FORMAT(num_objects_dirty);
+    FORMAT(num_whiteouts);
+    FORMAT(num_objects_omap);
+    FORMAT(num_objects_hit_set_archive);
+    FORMAT(num_bytes_hit_set_archive);
+    FORMAT(num_flush);
+    FORMAT(num_flush_kb);
+    FORMAT(num_evict);
+    FORMAT(num_evict_kb);
+    FORMAT(num_promote);
+    FORMAT(num_flush_mode_high);
+    FORMAT(num_flush_mode_low);
+    FORMAT(num_evict_mode_some);
+    FORMAT(num_evict_mode_full);
+    FORMAT(num_objects_pinned);
+    FORMAT(num_legacy_snapsets);
+    return fmt::format_to(
+      ctx.out(), "num_objects_repaired={})",
+      stats.num_objects_repaired);
+#undef FORMAT
+  }
+};
+inline std::ostream &operator<<(std::ostream &lhs, const object_stat_sum_t &sum) {
+  return lhs << fmt::format("{}", sum);
+}
+
 #if FMT_VERSION >= 90000
 template <bool TrackChanges> struct fmt::formatter<pg_missing_set<TrackChanges>> : fmt::ostream_formatter {};
 #endif

From 652da7a71a0a30e21c6f647fb340f69b795f3d11 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Sep 2023 23:09:15 +0000
Subject: [PATCH 1017/2492] common/scrub_types.h: add formatters for several
 scrub types

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/scrub_types.h | 195 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)

diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h
index 0394eddd7e6b..150ba87af8af 100644
--- a/src/common/scrub_types.h
+++ b/src/common/scrub_types.h
@@ -4,6 +4,8 @@
 #ifndef CEPH_SCRUB_TYPES_H
 #define CEPH_SCRUB_TYPES_H
 
+#include <fmt/ranges.h>
+
 #include "osd/osd_types.h"
 
 // wrappers around scrub types to offer the necessary bits other than
@@ -207,4 +209,197 @@ struct scrub_ls_result_t {
 
 WRITE_CLASS_ENCODER(scrub_ls_result_t);
 
+template <>
+struct fmt::formatter<librados::object_id_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &oid, FormatContext& ctx) const
+  {
+    return format_to(ctx.out(), "{}/{}/{}", oid.locator, oid.nspace, oid.name);
+  }
+};
+
+template <>
+struct fmt::formatter<librados::err_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &err, FormatContext& ctx) const
+  {
+    bool first = true;
+#define F(FLAG_NAME)					\
+    if (err.errors & librados::err_t::FLAG_NAME) {	\
+      if (!first) {					\
+	fmt::format_to(ctx.out(), "|");			\
+      } else {						\
+	first = false;					\
+      }							\
+      fmt::format_to(ctx.out(), #FLAG_NAME);		\
+    }
+    F(SHARD_MISSING);
+    F(SHARD_STAT_ERR);
+    F(SHARD_READ_ERR);
+    F(DATA_DIGEST_MISMATCH_INFO);
+    F(OMAP_DIGEST_MISMATCH_INFO);
+    F(SIZE_MISMATCH_INFO);
+    F(SHARD_EC_HASH_MISMATCH);
+    F(SHARD_EC_SIZE_MISMATCH);
+    F(INFO_MISSING);
+    F(INFO_CORRUPTED);
+    F(SNAPSET_MISSING);
+    F(SNAPSET_CORRUPTED);
+    F(OBJ_SIZE_INFO_MISMATCH);
+    F(HINFO_MISSING);
+    F(HINFO_CORRUPTED);
+#undef F
+    return ctx.out();
+  }
+};
+
+template <>
+struct fmt::formatter<librados::shard_info_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &err, FormatContext& ctx) const
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "shard_info_t(error: {}, "
+      "size: {}, "
+      "omap_digest_present: {}, "
+      "omap_digest: {}, "
+      "data_digest_present: {}, "
+      "data_digest: {}, "
+      "selected_io: {}, "
+      "primary: {})",
+      static_cast<librados::err_t>(err),
+      err.size,
+      err.omap_digest_present,
+      err.omap_digest,
+      err.data_digest_present,
+      err.data_digest,
+      err.selected_oi,
+      err.primary);
+  }
+};
+
+template <>
+struct fmt::formatter<shard_info_wrapper> :
+  fmt::formatter<librados::shard_info_t> {};
+
+template <>
+struct fmt::formatter<librados::obj_err_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &err, FormatContext& ctx) const
+  {
+    bool first = true;
+#define F(FLAG_NAME)					\
+    if (err.errors & librados::obj_err_t::FLAG_NAME) {	\
+      if (!first) {					\
+	fmt::format_to(ctx.out(), "|");			\
+      } else {						\
+	first = false;					\
+      }							\
+      fmt::format_to(ctx.out(), #FLAG_NAME);		\
+    }
+    F(OBJECT_INFO_INCONSISTENCY);
+    F(DATA_DIGEST_MISMATCH);
+    F(OMAP_DIGEST_MISMATCH);
+    F(SIZE_MISMATCH);
+    F(ATTR_VALUE_MISMATCH);
+    F(ATTR_NAME_MISMATCH);
+    F(SNAPSET_INCONSISTENCY);
+    F(HINFO_INCONSISTENCY);
+    F(SIZE_TOO_LARGE);
+#undef F
+    return ctx.out();
+  }
+};
+
+template <>
+struct fmt::formatter<librados::osd_shard_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &shard, FormatContext& ctx) const
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "osd_shard_t(osd: {}, shard: {})",
+      shard.osd, shard.shard);
+  }
+};
+
+template <>
+struct fmt::formatter<librados::inconsistent_obj_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &err, FormatContext& ctx) const
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "inconsistent_obj_t(error: {}, "
+      "object: {}, "
+      "version: {}, "
+      "shards: {}, "
+      "union_shards: {})",
+      static_cast<librados::obj_err_t>(err),
+      err.object,
+      err.version,
+      err.shards,
+      err.union_shards);
+  }
+};
+
+template <>
+struct fmt::formatter<inconsistent_obj_wrapper> :
+  fmt::formatter<librados::inconsistent_obj_t> {};
+
+template <>
+struct fmt::formatter<librados::inconsistent_snapset_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &err, FormatContext& ctx) const
+  {
+    fmt::format_to(ctx.out(), "inconsistent_snapset_t(errors: ");
+    bool first = true;
+#define F(FLAG_NAME)							\
+    if (err.errors & librados::inconsistent_snapset_t::FLAG_NAME) {	\
+      if (!first) {							\
+	fmt::format_to(ctx.out(), "|");					\
+      } else {								\
+	first = false;							\
+      }									\
+      fmt::format_to(ctx.out(), #FLAG_NAME);				\
+    }
+    F(SNAPSET_MISSING);
+    F(SNAPSET_CORRUPTED);
+    F(CLONE_MISSING);
+    F(SNAP_ERROR);
+    F(HEAD_MISMATCH);
+    F(HEADLESS_CLONE);
+    F(SIZE_MISMATCH);
+    F(OI_MISSING);
+    F(INFO_MISSING);
+    F(OI_CORRUPTED);
+    F(INFO_CORRUPTED);
+    F(EXTRA_CLONES);
+#undef F
+    return fmt::format_to(
+      ctx.out(),
+      ", object: {}, clones: {}, missing: {}",
+      err.object, err.clones, err.missing);
+  }
+};
+
+template <>
+struct fmt::formatter<inconsistent_snapset_wrapper> :
+  fmt::formatter<librados::inconsistent_snapset_t> {};
+
 #endif

From 20d15d4bd2130439c59071c9b84c907b988a8a37 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 1 Jul 2023 21:06:17 +0000
Subject: [PATCH 1018/2492] common/scrub_types.h: add
 inconsistent_obj_wrapper::merge

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/scrub_types.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h
index 150ba87af8af..972d0993193f 100644
--- a/src/common/scrub_types.h
+++ b/src/common/scrub_types.h
@@ -113,6 +113,10 @@ namespace librados {
 struct inconsistent_obj_wrapper : librados::inconsistent_obj_t {
   explicit inconsistent_obj_wrapper(const hobject_t& hoid);
 
+  void merge(obj_err_t other) {
+    errors |= other.errors;
+  }
+
   void set_object_info_inconsistency() {
     errors |= obj_err_t::OBJECT_INFO_INCONSISTENCY;
   }

From 6678e5df2ad1afa0eb09322ecdfa786fde14ae2f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 25 Aug 2023 16:01:05 -0700
Subject: [PATCH 1019/2492] crimson/osd/osd_operations/peering_event: make
 PGPeeringPipeline a struct

There will ultimately be many Operation types that need to use the
PGPeeringPipeline, let's not enumerate them as friends.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd_operations/peering_event.h | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/crimson/osd/osd_operations/peering_event.h b/src/crimson/osd/osd_operations/peering_event.h
index 6bbfe6c91174..dad1076a93bb 100644
--- a/src/crimson/osd/osd_operations/peering_event.h
+++ b/src/crimson/osd/osd_operations/peering_event.h
@@ -23,19 +23,13 @@ class ShardServices;
 class PG;
 class BackfillRecovery;
 
-  class PGPeeringPipeline {
+  struct PGPeeringPipeline {
     struct AwaitMap : OrderedExclusivePhaseT<AwaitMap> {
       static constexpr auto type_name = "PeeringEvent::PGPipeline::await_map";
     } await_map;
     struct Process : OrderedExclusivePhaseT<Process> {
       static constexpr auto type_name = "PeeringEvent::PGPipeline::process";
     } process;
-    template <class T>
-    friend class PeeringEvent;
-    friend class LocalPeeringEvent;
-    friend class RemotePeeringEvent;
-    friend class PGAdvanceMap;
-    friend class BackfillRecovery;
   };
 
 template <class T>

From 1bda4f5dfb0aea1cd02baa592f8cd4b09153374f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 2 Sep 2023 03:22:45 +0000
Subject: [PATCH 1020/2492] crimson/osd: generalize PG::snaptrim_mutex to
 background_process_lock

We don't want scrub and snaptrimming running at the same time,
use the same lock.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 10 +++++-----
 src/crimson/osd/osd_operations/snaptrim_event.h  |  4 ++--
 src/crimson/osd/pg.cc                            |  6 ++++++
 src/crimson/osd/pg.h                             | 13 +++++++------
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index 20f7439f52f9..e90c7a213faf 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -31,9 +31,9 @@ namespace crimson {
 namespace crimson::osd {
 
 PG::interruptible_future<>
-PG::SnapTrimMutex::lock(SnapTrimEvent &st_event) noexcept
+PG::BackgroundProcessLock::lock_with_op(SnapTrimEvent &st_event) noexcept
 {
-  return st_event.enter_stage<interruptor>(wait_pg
+  return st_event.enter_stage<interruptor>(wait
   ).then_interruptible([this] {
     return mutex.lock();
   });
@@ -115,7 +115,7 @@ SnapTrimEvent::start()
       return enter_stage<interruptor>(
         client_pp().get_obc);
     }).then_interruptible([this] {
-      return pg->snaptrim_mutex.lock(*this);
+      return pg->background_process_lock.lock_with_op(*this);
     }).then_interruptible([this] {
       return enter_stage<interruptor>(
         client_pp().process);
@@ -147,7 +147,7 @@ SnapTrimEvent::start()
         if (to_trim.empty()) {
           // the legit ENOENT -> done
           logger().debug("{}: to_trim is empty! Stopping iteration", *this);
-	  pg->snaptrim_mutex.unlock();
+	  pg->background_process_lock.unlock();
           return snap_trim_iertr::make_ready_future<seastar::stop_iteration>(
             seastar::stop_iteration::yes);
         }
@@ -171,7 +171,7 @@ SnapTrimEvent::start()
           logger().debug("{}: awaiting completion", *this);
           return subop_blocker.wait_completion();
         }).finally([this] {
-	  pg->snaptrim_mutex.unlock();
+	  pg->background_process_lock.unlock();
 	}).si_then([this] {
           if (!needs_pause) {
             return interruptor::now();
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 12f244512ac3..39038c71ef91 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -104,12 +104,12 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
     CommonPGPipeline::GetOBC::BlockingEvent,
     CommonPGPipeline::Process::BlockingEvent,
     WaitSubop::BlockingEvent,
-    PG::SnapTrimMutex::WaitPG::BlockingEvent,
+    PG::BackgroundProcessLock::Wait::BlockingEvent,
     WaitTrimTimer::BlockingEvent,
     CompletionEvent
   > tracking_events;
 
-  friend class PG::SnapTrimMutex;
+  friend class PG::BackgroundProcessLock;
 };
 
 // remove single object. a SnapTrimEvent can create multiple subrequests.
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 013f564b5f12..5b7f1fd25781 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -801,6 +801,12 @@ PG::interruptible_future<> PG::repair_object(
   return std::move(fut);
 }
 
+PG::interruptible_future<>
+PG::BackgroundProcessLock::lock() noexcept
+{
+  return interruptor::make_interruptible(mutex.lock());
+}
+
 template <class Ret, class SuccessFunc, class FailureFunc>
 PG::do_osd_ops_iertr::future<PG::pg_rep_op_fut_t<Ret>>
 PG::do_osd_ops_execute(
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 4628b09b509a..7f734ecd5e77 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -542,18 +542,19 @@ class PG : public boost::intrusive_ref_counter<
 
 private:
 
-  struct SnapTrimMutex {
-    struct WaitPG : OrderedConcurrentPhaseT<WaitPG> {
-      static constexpr auto type_name = "SnapTrimEvent::wait_pg";
-    } wait_pg;
+  struct BackgroundProcessLock {
+    struct Wait : OrderedConcurrentPhaseT<Wait> {
+      static constexpr auto type_name = "PG::BackgroundProcessLock::wait";
+    } wait;
     seastar::shared_mutex mutex;
 
-    interruptible_future<> lock(SnapTrimEvent &st_event) noexcept;
+    interruptible_future<> lock_with_op(SnapTrimEvent &st_event) noexcept;
+    interruptible_future<> lock() noexcept;
 
     void unlock() noexcept {
       mutex.unlock();
     }
-  } snaptrim_mutex;
+  } background_process_lock;
 
   using do_osd_ops_ertr = crimson::errorator<
    crimson::ct_error::eagain>;

From b1455202b177e94bc4dd97104c1d5e351b3aa537 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 11 Aug 2023 14:03:48 -0700
Subject: [PATCH 1021/2492] crimson/osd/scrub: add scrub_machine and
 scrub_validator along with tests

scrub_validator.h/cc contain the logic from translating a set of ScrubMaps
into per-chunk stats and errors.

scrub_machine.h/cc contain the logic for the overall scrub lifecycle and
reservation management.

ScrubContext (scrub_machine.h) is the interface through which the above
logic performs reservations and scans.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/CMakeLists.txt               |    1 +
 src/crimson/osd/CMakeLists.txt           |    3 +
 src/crimson/osd/scrub/scrub_machine.cc   |   76 ++
 src/crimson/osd/scrub/scrub_machine.h    |  613 ++++++++++
 src/crimson/osd/scrub/scrub_validator.cc |  502 ++++++++
 src/crimson/osd/scrub/scrub_validator.h  |  180 +++
 src/test/crimson/CMakeLists.txt          |   10 +
 src/test/crimson/test_crimson_scrub.cc   | 1347 ++++++++++++++++++++++
 8 files changed, 2732 insertions(+)
 create mode 100644 src/crimson/osd/scrub/scrub_machine.cc
 create mode 100644 src/crimson/osd/scrub/scrub_machine.h
 create mode 100644 src/crimson/osd/scrub/scrub_validator.cc
 create mode 100644 src/crimson/osd/scrub/scrub_validator.h
 create mode 100644 src/test/crimson/test_crimson_scrub.cc

diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt
index 9e751fcebc91..510ffbd9df99 100644
--- a/src/crimson/CMakeLists.txt
+++ b/src/crimson/CMakeLists.txt
@@ -121,6 +121,7 @@ add_library(crimson-common STATIC
   ${PROJECT_SOURCE_DIR}/src/osd/HitSet.cc
   ${PROJECT_SOURCE_DIR}/src/osd/OSDMap.cc
   ${PROJECT_SOURCE_DIR}/src/osd/PGPeeringEvent.cc
+  ${PROJECT_SOURCE_DIR}/src/common/scrub_types.cc
   ${PROJECT_SOURCE_DIR}/src/xxHash/xxhash.c
   ${crimson_common_srcs}
   $<TARGET_OBJECTS:common_mountcephfs_objs>
diff --git a/src/crimson/osd/CMakeLists.txt b/src/crimson/osd/CMakeLists.txt
index 817027c18ffc..e3ab3cf4d735 100644
--- a/src/crimson/osd/CMakeLists.txt
+++ b/src/crimson/osd/CMakeLists.txt
@@ -33,6 +33,8 @@ add_executable(crimson-osd
   replicated_recovery_backend.cc
   scheduler/scheduler.cc
   scheduler/mclock_scheduler.cc
+  scrub/scrub_machine.cc
+  scrub/scrub_validator.cc
   osdmap_gate.cc
   pg_activation_blocker.cc
   pg_map.cc
@@ -40,6 +42,7 @@ add_executable(crimson-osd
   objclass.cc
   ${PROJECT_SOURCE_DIR}/src/objclass/class_api.cc
   ${PROJECT_SOURCE_DIR}/src/osd/ClassHandler.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc
   ${PROJECT_SOURCE_DIR}/src/osd/osd_op_util.cc
   ${PROJECT_SOURCE_DIR}/src/osd/OSDCap.cc
   ${PROJECT_SOURCE_DIR}/src/osd/PeeringState.cc
diff --git a/src/crimson/osd/scrub/scrub_machine.cc b/src/crimson/osd/scrub/scrub_machine.cc
new file mode 100644
index 000000000000..7d674feffeb7
--- /dev/null
+++ b/src/crimson/osd/scrub/scrub_machine.cc
@@ -0,0 +1,76 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/ceph_assert.h"
+
+#include "crimson/osd/scrub/scrub_machine.h"
+
+namespace crimson::osd::scrub {
+
+WaitUpdate::WaitUpdate(my_context ctx) : ScrubState(ctx)
+{
+  auto &cs = context<ChunkState>();
+  cs.range_reserved = true;
+  assert(cs.range);
+  get_scrub_context().reserve_range(cs.range->start, cs.range->end);
+}
+
+ScanRange::ScanRange(my_context ctx) : ScrubState(ctx)
+{
+  ceph_assert(context<ChunkState>().range);
+  const auto &cs = context<ChunkState>();
+  const auto &range = cs.range.value();
+  get_scrub_context(
+  ).foreach_id_to_scrub([this, &range, &cs](const auto &id) {
+    get_scrub_context().scan_range(
+      id, cs.version,
+      context<Scrubbing>().deep,
+      range.start, range.end);
+    waiting_on++;
+  });
+}
+
+sc::result ScanRange::react(const ScrubContext::scan_range_complete_t &event)
+{
+  auto [_, inserted] = maps.insert(event.value.to_pair());
+  ceph_assert(inserted);
+  ceph_assert(waiting_on > 0);
+  --waiting_on;
+
+  if (waiting_on > 0) {
+    return discard_event();
+  } else {
+    ceph_assert(context<ChunkState>().range);
+    {
+      auto results = validate_chunk(
+	get_scrub_context().get_dpp(),
+	context<Scrubbing>().policy,
+	maps);
+      context<Scrubbing>().stats.add(results.stats);
+      get_scrub_context().emit_chunk_result(
+	*(context<ChunkState>().range),
+	std::move(results));
+    }
+    if (context<ChunkState>().range->end.is_max()) {
+      get_scrub_context().emit_scrub_result(
+	context<Scrubbing>().deep,
+	context<Scrubbing>().stats);
+      return transit<PrimaryActive>();
+    } else {
+      context<Scrubbing>().advance_current(
+	context<ChunkState>().range->end);
+      return transit<ChunkState>();
+    }
+  }
+}
+
+ReplicaScanChunk::ReplicaScanChunk(my_context ctx) : ScrubState(ctx)
+{
+  auto &to_scan = context<ReplicaChunkState>().to_scan;
+  get_scrub_context().generate_and_submit_chunk_result(
+    to_scan.start,
+    to_scan.end,
+    to_scan.deep);
+}
+
+};
diff --git a/src/crimson/osd/scrub/scrub_machine.h b/src/crimson/osd/scrub/scrub_machine.h
new file mode 100644
index 000000000000..d2d127adc0d9
--- /dev/null
+++ b/src/crimson/osd/scrub/scrub_machine.h
@@ -0,0 +1,613 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <string>
+#include <ranges>
+
+#include <boost/statechart/custom_reaction.hpp>
+#include <boost/statechart/deferral.hpp>
+#include <boost/statechart/event.hpp>
+#include <boost/statechart/event_base.hpp>
+#include <boost/statechart/in_state_reaction.hpp>
+#include <boost/statechart/simple_state.hpp>
+#include <boost/statechart/state.hpp>
+#include <boost/statechart/state_machine.hpp>
+#include <boost/statechart/transition.hpp>
+
+#include "common/fmt_common.h"
+#include "common/hobject.h"
+#include "common/hobject_fmt.h"
+#include "crimson/common/log.h"
+#include "osd/osd_types_fmt.h"
+#include "scrub_validator.h"
+
+namespace crimson::osd::scrub {
+
+/* Development Notes
+ *
+ * Notes:
+ * - We're leaving out all of the throttle waits.  We actually want to handle
+ *   that using crimson's operation throttler machinery.
+ *
+ * TODOs:
+ * - Leaving SnapMapper validation to later work
+ *   - Note, each replica should validate and repair locally as the SnapMapper
+ *     is meant to be a local index of the local object contents
+ * - Leaving preemption for later
+ * - Leaving scheduling for later, for now the only way to trigger a scrub
+ *   is via the ceph tell <pgid> [deep_]scrub command
+ */
+
+namespace sc = boost::statechart;
+
+template <typename T>
+struct simple_event_t : sc::event<T> {
+  template <typename FormatContext>
+  auto fmt_print_ctx(FormatContext & ctx) const {
+    return fmt::format_to(ctx.out(), "{}", T::event_name);
+  }
+};
+
+template <typename T, has_formatter V>
+struct value_event_t : sc::event<T> {
+  const V value;
+
+  template <typename... Args>
+  value_event_t(Args&&... args) : value(std::forward<Args>(args)...) {}
+
+  value_event_t(const value_event_t &) = default;
+  value_event_t(value_event_t &&) = default;
+  value_event_t &operator=(const value_event_t&) = default;
+  value_event_t &operator=(value_event_t&&) = default;
+
+  template <typename FormatContext>
+  auto fmt_print_ctx(FormatContext & ctx) const {
+    return fmt::format_to(ctx.out(), "{}", T::event_name);
+  }
+};
+
+
+#define SIMPLE_EVENT(T) struct T : simple_event_t<T> {			\
+    static constexpr const char * event_name = #T;			\
+  };
+
+#define VALUE_EVENT(T, V) struct T : value_event_t<T, V> {		\
+    static constexpr const char * event_name = #T;			\
+									\
+    template <typename... Args>						\
+    T(Args&&... args) : value_event_t(					\
+      std::forward<Args>(args)...) {}					\
+  };
+
+/**
+ * ScrubContext
+ *
+ * Interface to external PG/OSD/IO machinery.
+ *
+ * Methods which may take time return immediately and define an event which
+ * will be asynchronously delivered to the state machine with the result.  This
+ * is a bit clumsy to use, but should render this component highly testable.
+ *
+ * Events sent as a completion to a ScrubContext interface method are defined
+ * within ScrubContext.  Other events are defined within ScrubMachine.
+ */
+struct ScrubContext {
+  /// return ids to scrub
+  virtual const std::set<pg_shard_t> &get_ids_to_scrub() const = 0;
+
+  /// iterates over each pg_shard_t to scrub
+  template <typename F>
+  void foreach_id_to_scrub(F &&f) {
+    for (const auto &id : get_ids_to_scrub()) {
+      std::invoke(f, id);
+    }
+  }
+
+  /// return struct defining chunk validation rules
+  virtual chunk_validation_policy_t get_policy() const = 0;
+
+  /// notifies implementation of scrub start
+  virtual void notify_scrub_start(bool deep) = 0;
+
+  /// notifies implementation of scrub end
+  virtual void notify_scrub_end(bool deep) = 0;
+
+  /// requests range to scrub starting at start
+  struct request_range_result_t {
+    hobject_t start;
+    hobject_t end;
+
+    request_range_result_t(
+      const hobject_t &start,
+      const hobject_t &end) : start(start), end(end) {}
+
+    auto fmt_print_ctx(auto &ctx) const -> decltype(ctx.out()) {
+      return fmt::format_to(ctx.out(), "start: {}, end: {}", start, end);
+    }
+  };
+  VALUE_EVENT(request_range_complete_t, request_range_result_t);
+  virtual void request_range(
+    const hobject_t &start) = 0;
+
+  /// reserves range [start, end)
+  VALUE_EVENT(reserve_range_complete_t, eversion_t);
+  virtual void reserve_range(
+    const hobject_t &start,
+    const hobject_t &end) = 0;
+
+  /// waits until implementation has committed up to version
+  SIMPLE_EVENT(await_update_complete_t);
+  virtual bool await_update(
+    const eversion_t &version) = 0;
+
+  /// cancel in progress or currently reserved range
+  virtual void release_range() = 0;
+
+  /// scans [begin, end) on target as of version
+  struct scan_range_value_t {
+    pg_shard_t from;
+    ScrubMap map;
+
+    template <typename Map>
+    scan_range_value_t(
+      pg_shard_t from,
+      Map &&map) : from(from), map(std::forward<Map>(map)) {}
+
+    auto to_pair() const { return std::make_pair(from, map); }
+    auto fmt_print_ctx(auto &ctx) const -> decltype(ctx.out()) {
+      return fmt::format_to(ctx.out(), "from: {}", from);
+    }
+  };
+  VALUE_EVENT(scan_range_complete_t, scan_range_value_t);
+  virtual void scan_range(
+    pg_shard_t target,
+    eversion_t version,
+    bool deep,
+    const hobject_t &start,
+    const hobject_t &end) = 0;
+
+  /// instructs implmentatino to scan [begin, end) and emit result to primary
+  SIMPLE_EVENT(generate_and_submit_chunk_result_complete_t);
+  virtual void generate_and_submit_chunk_result(
+    const hobject_t &begin,
+    const hobject_t &end,
+    bool deep) = 0;
+
+  /// notifies implementation of chunk scrub results
+  virtual void emit_chunk_result(
+    const request_range_result_t &range,
+    chunk_result_t &&result) = 0;
+
+  /// notifies implementation of full scrub results
+  virtual void emit_scrub_result(
+    bool deep,
+    object_stat_sum_t scrub_stats) = 0;
+
+  /// get dpp instance for logging
+  virtual DoutPrefixProvider &get_dpp() = 0;
+};
+
+struct Crash;
+struct Inactive;
+
+namespace events {
+/// reset ScrubMachine
+SIMPLE_EVENT(reset_t);
+
+/// start (deep) scrub
+struct start_scrub_event_t {
+  bool deep = false;
+
+  start_scrub_event_t(bool deep) : deep(deep) {}
+
+  auto fmt_print_ctx(auto &ctx) const -> decltype(ctx.out()) {
+    return fmt::format_to(ctx.out(), "deep: {}", deep);
+  }
+};
+VALUE_EVENT(start_scrub_t, start_scrub_event_t);
+
+/// notifies ScrubMachine about a write on oid resulting in delta_stats
+struct op_stat_event_t {
+  hobject_t oid;
+  object_stat_sum_t delta_stats;
+
+  op_stat_event_t(
+    hobject_t oid,
+    object_stat_sum_t delta_stats) : oid(oid), delta_stats(delta_stats) {}
+
+  auto fmt_print_ctx(auto &ctx) const -> decltype(ctx.out()) {
+    return fmt::format_to(ctx.out(), "oid: {}", oid);
+  }
+};
+VALUE_EVENT(op_stats_t, op_stat_event_t);
+
+/// Prepares statemachine for primary events
+SIMPLE_EVENT(primary_activate_t);
+
+/// Prepares statemachine for replica events
+SIMPLE_EVENT(replica_activate_t);
+
+/// Instructs replica to (deep) scrub [start, end) as of version version
+struct replica_scan_event_t {
+  hobject_t start;
+  hobject_t end;
+  eversion_t version;
+  bool deep = false;
+
+  replica_scan_event_t() = default;
+
+  replica_scan_event_t(
+    hobject_t start,
+    hobject_t end,
+    eversion_t version,
+    bool deep) : start(start), end(end), version(version), deep(deep) {}
+
+  auto fmt_print_ctx(auto &ctx) const -> decltype(ctx.out()) {
+    return fmt::format_to(
+      ctx.out(), "start: {}, end: {}, version: {}, deep: {}",
+      start, end, version, deep);
+  }
+};
+VALUE_EVENT(replica_scan_t, replica_scan_event_t);
+
+}
+
+
+/**
+ * ScrubMachine
+ *
+ * Manages orchestration of rados's distributed scrub process.
+ *
+ * There are two general ways in which ScrubMachine may need to release
+ * resources:
+ * - interval_change_t -- represents case where PG as a whole undergoes
+ *   a distributed mapping change.  Distributed resources are released
+ *   implicitly as remote PG instances receive the new map.  Local
+ *   resources are still released by ScrubMachine via ScrubContext methods
+ *   generally via state destructors
+ * - otherwise, ScrubMachine is responsible for notifying remote PG
+ *   instances via the appropriate ScrubContext methods again generally
+ *   from state destructors.
+ *
+ * TODO: interval_change_t will be added with remote reservations.
+ */
+class ScrubMachine
+  : public sc::state_machine<ScrubMachine, Inactive> {
+public:
+  static constexpr std::string_view full_name = "ScrubMachine";
+
+  ScrubContext &context;
+  ScrubMachine(ScrubContext &context) : context(context) {}
+};
+
+/**
+ * ScrubState
+ *
+ * Template defining machinery/state common to all scrub state machine
+ * states.
+ */
+template <typename S, typename P, typename... T>
+struct ScrubState : sc::state<S, P, T...> {
+  using sc_base = sc::state<S, P, T...>;
+  DoutPrefixProvider &dpp;
+
+  /* machinery for populating a full_name member for each ScrubState with
+   * ScrubMachine/.../ParentState/ChildState full_name */
+  template <std::string_view const &PN, typename PI,
+	    std::string_view const &CN, typename CI>
+  struct concat;
+
+  template <std::string_view const &PN, std::size_t... PI,
+	    std::string_view const &CN, std::size_t... CI>
+  struct concat<PN, std::index_sequence<PI...>, CN, std::index_sequence<CI...>> {
+    static constexpr size_t value_size = PN.size() + CN.size() + 1;
+    static constexpr const char value[value_size]{PN[PI]..., '/', CN[CI]...};
+  };
+
+  template <std::string_view const &PN, std::string_view const &CN>
+  struct join {
+    using conc = concat<
+      PN, std::make_index_sequence<PN.size()>,
+      CN, std::make_index_sequence<CN.size()>>;
+    static constexpr std::string_view value{
+      conc::value,
+      conc::value_size
+    };
+  };
+
+  /// Populated with ScrubMachine/.../Parent/Child for each state Child
+  static constexpr std::string_view full_name =
+    join<P::full_name, S::state_name>::value;
+
+  template <typename C>
+  explicit ScrubState(C ctx) : sc_base(ctx), dpp(get_scrub_context().get_dpp()) {
+    LOG_PREFIX(ScrubState::ScrubState);
+    SUBDEBUGDPP(osd, "entering state {}", dpp, full_name);
+  }
+
+  ~ScrubState() {
+    LOG_PREFIX(ScrubState::~ScrubState);
+    SUBDEBUGDPP(osd, "exiting state {}", dpp, full_name);
+  }
+
+  auto &get_scrub_context() {
+    return sc_base::template context<ScrubMachine>().context;
+  }
+};
+
+struct Crash : ScrubState<Crash, ScrubMachine> {
+  static constexpr std::string_view state_name = "Crash";
+  explicit Crash(my_context ctx) : ScrubState(ctx) {
+    ceph_abort("Crash state impossible");
+  }
+
+};
+
+struct PrimaryActive;
+struct ReplicaActive;
+struct Inactive : ScrubState<Inactive, ScrubMachine> {
+  static constexpr std::string_view state_name = "Inactive";
+  explicit Inactive(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::transition<events::primary_activate_t, PrimaryActive>,
+    sc::transition<events::replica_activate_t, ReplicaActive>,
+    sc::custom_reaction<events::reset_t>,
+    sc::custom_reaction<events::start_scrub_t>,
+    sc::custom_reaction<events::op_stats_t>,
+    sc::transition< boost::statechart::event_base, Crash >
+    >;
+
+  sc::result react(const events::reset_t &) {
+    return discard_event();
+  }
+  sc::result react(const events::start_scrub_t &) {
+    return discard_event();
+  }
+  sc::result react(const events::op_stats_t &) {
+    return discard_event();
+  }
+};
+
+struct AwaitScrub;
+struct PrimaryActive : ScrubState<PrimaryActive, ScrubMachine, AwaitScrub> {
+  static constexpr std::string_view state_name = "PrimaryActive";
+  explicit PrimaryActive(my_context ctx) : ScrubState(ctx) {}
+
+  bool local_reservation_held = false;
+  std::set<pg_shard_t> remote_reservations_held;
+
+  using reactions = boost::mpl::list<
+    sc::transition<events::reset_t, Inactive>,
+    sc::custom_reaction<events::start_scrub_t>,
+    sc::custom_reaction<events::op_stats_t>,
+    sc::transition< boost::statechart::event_base, Crash >
+    >;
+
+  sc::result react(const events::start_scrub_t &event) {
+    return discard_event();
+  }
+
+  sc::result react(const events::op_stats_t &) {
+    return discard_event();
+  }
+};
+
+namespace internal_events {
+VALUE_EVENT(set_deep_t, bool);
+}
+
+struct Scrubbing;
+struct AwaitScrub : ScrubState<AwaitScrub, PrimaryActive> {
+  static constexpr std::string_view state_name = "AwaitScrub";
+  explicit AwaitScrub(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<events::start_scrub_t>
+    >;
+
+  sc::result react(const events::start_scrub_t &event) {
+    post_event(internal_events::set_deep_t{event.value.deep});
+    return transit<Scrubbing>();
+  }
+};
+
+struct ChunkState;
+struct Scrubbing : ScrubState<Scrubbing, PrimaryActive, ChunkState> {
+  static constexpr std::string_view state_name = "Scrubbing";
+  explicit Scrubbing(my_context ctx)
+    : ScrubState(ctx), policy(get_scrub_context().get_policy()) {}
+
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<internal_events::set_deep_t>,
+    sc::custom_reaction<events::op_stats_t>
+    >;
+
+  chunk_validation_policy_t policy;
+
+  /// hobjects < current have been scrubbed
+  hobject_t current;
+
+  /// true for deep scrub
+  bool deep = false;
+
+  /// stats for objects < current, maintained via events::op_stats_t
+  object_stat_sum_t stats;
+
+  void advance_current(const hobject_t &next) {
+    current = next;
+  }
+
+  sc::result react(const internal_events::set_deep_t &event) {
+    deep = event.value;
+    get_scrub_context().notify_scrub_start(deep);
+    return discard_event();
+  }
+
+  void exit() {
+    get_scrub_context().notify_scrub_end(deep);
+  }
+
+  sc::result react(const events::op_stats_t &event) {
+    if (event.value.oid < current) {
+      stats.add(event.value.delta_stats);
+    }
+    return discard_event();
+  }
+};
+
+struct GetRange;
+struct ChunkState : ScrubState<ChunkState, Scrubbing, GetRange> {
+  static constexpr std::string_view state_name = "ChunkState";
+  explicit ChunkState(my_context ctx) : ScrubState(ctx) {}
+
+  /// Current chunk includes objects in [range_start, range_end)
+  boost::optional<ScrubContext::request_range_result_t> range;
+
+  /// true once we have requested that the range be reserved
+  bool range_reserved = false;
+
+  /// version of last update for the reserved chunk
+  eversion_t version;
+
+  void exit() {
+    if (range_reserved) {
+      get_scrub_context().release_range();
+    }
+  }
+};
+
+struct WaitUpdate;
+struct GetRange : ScrubState<GetRange, ChunkState> {
+  static constexpr std::string_view state_name = "GetRange";
+  explicit GetRange(my_context ctx) : ScrubState(ctx) {
+    get_scrub_context().request_range(context<Scrubbing>().current);
+  }
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<ScrubContext::request_range_complete_t>
+    >;
+
+  sc::result react(const ScrubContext::request_range_complete_t &event) {
+    context<ChunkState>().range = event.value;
+    return transit<WaitUpdate>();
+  }
+};
+
+struct ScanRange;
+struct WaitUpdate : ScrubState<WaitUpdate, ChunkState> {
+  static constexpr std::string_view state_name = "WaitUpdate";
+  explicit WaitUpdate(my_context ctx);
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<ScrubContext::reserve_range_complete_t>
+    >;
+
+  sc::result react(const ScrubContext::reserve_range_complete_t &e) {
+    context<ChunkState>().version = e.value;
+    return transit<ScanRange>();
+  }
+};
+
+struct ScanRange : ScrubState<ScanRange, ChunkState> {
+  static constexpr std::string_view state_name = "ScanRange";
+  explicit ScanRange(my_context ctx);
+
+  scrub_map_set_t maps;
+  unsigned waiting_on = 0;
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<ScrubContext::scan_range_complete_t>
+    >;
+
+  sc::result react(const ScrubContext::scan_range_complete_t &);
+};
+
+struct ReplicaIdle;
+struct ReplicaActive :
+    ScrubState<ReplicaActive, ScrubMachine, ReplicaIdle> {
+  static constexpr std::string_view state_name = "ReplicaActive";
+  explicit ReplicaActive(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::transition<events::reset_t, Inactive>,
+    sc::custom_reaction<events::start_scrub_t>,
+    sc::custom_reaction<events::op_stats_t>,
+    sc::transition< boost::statechart::event_base, Crash >
+    >;
+
+  sc::result react(const events::start_scrub_t &) {
+    return discard_event();
+  }
+
+  sc::result react(const events::op_stats_t &) {
+    return discard_event();
+  }
+};
+
+struct ReplicaChunkState;
+struct ReplicaIdle : ScrubState<ReplicaIdle, ReplicaActive> {
+  static constexpr std::string_view state_name = "ReplicaIdle";
+  explicit ReplicaIdle(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<events::replica_scan_t>
+    >;
+
+  sc::result react(const events::replica_scan_t &event) {
+    LOG_PREFIX(ScrubState::ReplicaIdle::react(events::replica_scan_t));
+    SUBDEBUGDPP(osd, "event.value: {}", get_scrub_context().get_dpp(), event.value);
+    post_event(event);
+    return transit<ReplicaChunkState>();
+  }
+};
+
+struct ReplicaWaitUpdate;
+struct ReplicaChunkState : ScrubState<ReplicaChunkState, ReplicaActive, ReplicaWaitUpdate> {
+  static constexpr std::string_view state_name = "ReplicaChunkState";
+  explicit ReplicaChunkState(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::custom_reaction<events::replica_scan_t>
+    >;
+
+  events::replica_scan_event_t to_scan;
+
+  sc::result react(const events::replica_scan_t &event) {
+    LOG_PREFIX(ScrubState::ReplicaWaitUpdate::react(events::replica_scan_t));
+    SUBDEBUGDPP(osd, "event.value: {}", get_scrub_context().get_dpp(), event.value);
+    to_scan = event.value;
+    if (get_scrub_context().await_update(event.value.version)) {
+      post_event(ScrubContext::await_update_complete_t{});
+    }
+    return discard_event();
+  }
+};
+
+struct ReplicaScanChunk;
+struct ReplicaWaitUpdate : ScrubState<ReplicaWaitUpdate, ReplicaChunkState> {
+  static constexpr std::string_view state_name = "ReplicaWaitUpdate";
+  explicit ReplicaWaitUpdate(my_context ctx) : ScrubState(ctx) {}
+
+  using reactions = boost::mpl::list<
+    sc::transition<ScrubContext::await_update_complete_t, ReplicaScanChunk>
+    >;
+};
+
+struct ReplicaScanChunk : ScrubState<ReplicaScanChunk, ReplicaChunkState> {
+  static constexpr std::string_view state_name = "ReplicaScanChunk";
+  explicit ReplicaScanChunk(my_context ctx);
+
+  using reactions = boost::mpl::list<
+    sc::transition<ScrubContext::generate_and_submit_chunk_result_complete_t,
+		   ReplicaIdle>
+    >;
+};
+
+#undef SIMPLE_EVENT
+#undef VALUE_EVENT
+
+}
diff --git a/src/crimson/osd/scrub/scrub_validator.cc b/src/crimson/osd/scrub/scrub_validator.cc
new file mode 100644
index 000000000000..b7dcc46a35e7
--- /dev/null
+++ b/src/crimson/osd/scrub/scrub_validator.cc
@@ -0,0 +1,502 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <ranges>
+
+#include "osd/osd_types_fmt.h"
+
+#include "crimson/common/log.h"
+#include "crimson/osd/scrub/scrub_validator.h"
+#include "osd/ECUtil.h"
+
+SET_SUBSYS(osd);
+
+namespace crimson::osd::scrub {
+
+using object_set_t = std::set<hobject_t>;
+object_set_t get_object_set(const scrub_map_set_t &in)
+{
+  object_set_t ret;
+  for (const auto& [from, map] : in) {
+    std::transform(map.objects.begin(), map.objects.end(),
+                   std::inserter(ret, ret.end()),
+                   [](const auto& i) { return i.first; });
+  }
+  return ret;
+}
+
+struct shard_evaluation_t {
+  pg_shard_t source;
+  shard_info_wrapper shard_info;
+
+  std::optional<object_info_t> object_info;
+  std::optional<SnapSet> snapset;
+  std::optional<ECUtil::HashInfo> hinfo;
+
+  size_t omap_keys{0};
+  size_t omap_bytes{0};
+
+  bool has_errors() const {
+    return shard_info.has_errors();
+  }
+
+  bool is_primary() const {
+    return shard_info.primary;
+  }
+
+  std::weak_ordering operator<=>(const shard_evaluation_t &rhs) const {
+    return std::make_tuple(!has_errors(), is_primary()) <=>
+      std::make_tuple(!rhs.has_errors(), rhs.is_primary());
+  }
+};
+shard_evaluation_t evaluate_object_shard(
+  const chunk_validation_policy_t &policy,
+  const hobject_t &oid,
+  pg_shard_t from,
+  const ScrubMap::object *maybe_obj)
+{
+  shard_evaluation_t ret;
+  ret.source = from;
+  if (from == policy.primary) {
+    ret.shard_info.primary = true;
+  }
+  if (!maybe_obj || maybe_obj->negative) {
+    // impossible since chunky scrub was introduced
+    ceph_assert(!maybe_obj->negative);
+    ret.shard_info.set_missing();
+    return ret;
+  }
+
+  auto &obj = *maybe_obj;
+  /* We are ignoring ScrubMap::object::large_omap_object*, object_omap_* is all the
+   * info we need */
+  ret.omap_keys = obj.object_omap_keys;
+  ret.omap_bytes = obj.object_omap_bytes;
+
+  ret.shard_info.set_object(obj);
+
+  if (obj.ec_hash_mismatch) {
+    ret.shard_info.set_ec_hash_mismatch();
+  }
+
+  if (obj.ec_size_mismatch) {
+    ret.shard_info.set_ec_size_mismatch();
+  }
+
+  if (obj.read_error) {
+    ret.shard_info.set_read_error();
+  }
+
+  if (obj.stat_error) {
+    ret.shard_info.set_stat_error();
+  }
+
+  {
+    auto xiter = obj.attrs.find(OI_ATTR);
+    if (xiter == obj.attrs.end()) {
+      ret.shard_info.set_info_missing();
+    } else {
+      bufferlist bl;
+      bl.push_back(xiter->second);
+      ret.object_info = object_info_t{};
+      try {
+	auto bliter = bl.cbegin();
+	::decode(*(ret.object_info), bliter);
+      } catch (...) {
+	ret.shard_info.set_info_corrupted();
+	ret.object_info = std::nullopt;
+      }
+    }
+  }
+
+  ret.shard_info.size = obj.size;
+  if (ret.object_info &&
+      obj.size != policy.logical_to_ondisk_size(ret.object_info->size)) {
+    ret.shard_info.set_size_mismatch_info();
+  }
+
+  if (oid.is_head()) {
+    auto xiter = obj.attrs.find(SS_ATTR);
+    if (xiter == obj.attrs.end()) {
+      ret.shard_info.set_snapset_missing();
+    } else {
+      bufferlist bl;
+      bl.push_back(xiter->second);
+      ret.snapset = SnapSet{};
+      try {
+	auto bliter = bl.cbegin();
+	::decode(*(ret.snapset), bliter);
+      } catch (...) {
+	ret.shard_info.set_snapset_corrupted();
+	ret.snapset = std::nullopt;
+      }
+    }
+  }
+
+  if (policy.is_ec()) {
+    auto xiter = obj.attrs.find(ECUtil::get_hinfo_key());
+    if (xiter == obj.attrs.end()) {
+      ret.shard_info.set_hinfo_missing();
+    } else {
+      bufferlist bl;
+      bl.push_back(xiter->second);
+      ret.hinfo = ECUtil::HashInfo{};
+      try {
+	auto bliter = bl.cbegin();
+	decode(*(ret.hinfo), bliter);
+      } catch (...) {
+	ret.shard_info.set_hinfo_corrupted();
+	ret.hinfo = std::nullopt;
+      }
+    }
+  }
+
+  if (ret.object_info) {
+    if (ret.shard_info.data_digest_present &&
+	ret.object_info->is_data_digest() &&
+	(ret.object_info->data_digest != ret.shard_info.data_digest)) {
+      ret.shard_info.set_data_digest_mismatch_info();
+    }
+    if (ret.shard_info.omap_digest_present &&
+	ret.object_info->is_omap_digest() &&
+	(ret.object_info->omap_digest != ret.shard_info.omap_digest)) {
+      ret.shard_info.set_omap_digest_mismatch_info();
+    }
+  }
+
+  return ret;
+}
+
+librados::obj_err_t compare_candidate_to_authoritative(
+  const chunk_validation_policy_t &policy,
+  const hobject_t &oid,
+  const shard_evaluation_t &auth,
+  const shard_evaluation_t &cand)
+{
+  using namespace librados;
+  obj_err_t ret;
+
+  if (cand.shard_info.has_shard_missing()) {
+    return ret;
+  }
+
+  const auto &auth_si = auth.shard_info;
+  const auto &cand_si = cand.shard_info;
+
+  if (auth_si.data_digest != cand_si.data_digest) {
+    ret.errors |= obj_err_t::DATA_DIGEST_MISMATCH;
+  }
+
+  if (auth_si.omap_digest != cand_si.omap_digest) {
+    ret.errors |= obj_err_t::OMAP_DIGEST_MISMATCH;
+  }
+
+  {
+    auto aiter = auth_si.attrs.find(OI_ATTR);
+    ceph_assert(aiter != auth_si.attrs.end());
+
+    auto citer = cand_si.attrs.find(OI_ATTR);
+    if (citer == cand_si.attrs.end() ||
+	!aiter->second.contents_equal(citer->second)) {
+      ret.errors |= obj_err_t::OBJECT_INFO_INCONSISTENCY;
+    }
+  }
+
+  if (oid.is_head()) {
+    auto aiter = auth_si.attrs.find(SS_ATTR);
+    ceph_assert(aiter != auth_si.attrs.end());
+
+    auto citer = cand_si.attrs.find(SS_ATTR);
+    if (citer == cand_si.attrs.end() ||
+	!aiter->second.contents_equal(citer->second)) {
+      ret.errors |= obj_err_t::SNAPSET_INCONSISTENCY;
+    }
+  }
+
+  if (policy.is_ec()) {
+    auto aiter = auth_si.attrs.find(ECUtil::get_hinfo_key());
+    ceph_assert(aiter != auth_si.attrs.end());
+
+    auto citer = cand_si.attrs.find(ECUtil::get_hinfo_key());
+    if (citer == cand_si.attrs.end() ||
+	!aiter->second.contents_equal(citer->second)) {
+      ret.errors |= obj_err_t::HINFO_INCONSISTENCY;
+    }
+  }
+
+  if (auth_si.size != cand_si.size) {
+    ret.errors |= obj_err_t::SIZE_MISMATCH;
+  }
+
+  auto is_sys_attr = [&policy](const auto &str) {
+    return str == OI_ATTR || str == SS_ATTR ||
+      (policy.is_ec() && str == ECUtil::get_hinfo_key());
+  };
+  for (auto aiter = auth_si.attrs.begin(); aiter != auth_si.attrs.end(); ++aiter) {
+    if (is_sys_attr(aiter->first)) continue;
+
+    auto citer = cand_si.attrs.find(aiter->first);
+    if (citer == cand_si.attrs.end()) {
+      ret.errors |= obj_err_t::ATTR_NAME_MISMATCH;
+    } else if (!aiter->second.contents_equal(citer->second)) {
+      ret.errors |= obj_err_t::ATTR_VALUE_MISMATCH;
+    }
+  }
+  if (std::any_of(
+	cand_si.attrs.begin(), cand_si.attrs.end(),
+	[&is_sys_attr, &auth_si](auto &p) {
+	  return !is_sys_attr(p.first) &&
+	    auth_si.attrs.find(p.first) == auth_si.attrs.end();
+	})) {
+    ret.errors |= obj_err_t::ATTR_NAME_MISMATCH;
+  }
+
+  return ret;
+}
+
+struct object_evaluation_t {
+  std::optional<inconsistent_obj_wrapper> inconsistency;
+  std::optional<object_info_t> object_info;
+  std::optional<SnapSet> snapset;
+
+  size_t omap_keys{0};
+  size_t omap_bytes{0};
+};
+object_evaluation_t evaluate_object(
+  const chunk_validation_policy_t &policy,
+  const hobject_t &hoid,
+  const scrub_map_set_t &maps)
+{
+  ceph_assert(maps.size() > 0);
+  using evaluation_vec_t = std::vector<shard_evaluation_t>;
+  evaluation_vec_t shards;
+  std::transform(
+    maps.begin(), maps.end(),
+    std::inserter(shards, shards.end()),
+    [&hoid, &policy](const auto &item) -> evaluation_vec_t::value_type {
+      const auto &[shard, scrub_map] = item;
+      auto miter = scrub_map.objects.find(hoid);
+      auto maybe_shard = miter == scrub_map.objects.end() ?
+	nullptr : &(miter->second);
+      return evaluate_object_shard(policy, hoid, shard, maybe_shard);
+    });
+
+  std::sort(shards.begin(), shards.end());
+
+  auto &auth_eval = shards.back();
+
+  object_evaluation_t ret;
+  inconsistent_obj_wrapper iow{hoid};
+  if (!auth_eval.has_errors()) {
+    ret.object_info = auth_eval.object_info;
+    ret.omap_keys = auth_eval.omap_keys;
+    ret.omap_bytes = auth_eval.omap_bytes;
+    ret.snapset = auth_eval.snapset;
+    if (auth_eval.object_info->size > policy.max_object_size) {
+      iow.set_size_too_large();
+    }
+    auth_eval.shard_info.selected_oi = true;
+    std::for_each(
+      shards.begin(), shards.end() - 1,
+      [&policy, &hoid, &auth_eval, &iow](auto &cand_eval) {
+	auto err = compare_candidate_to_authoritative(
+	  policy, hoid, auth_eval, cand_eval);
+	iow.merge(err);
+      });
+  }
+
+  if (iow.errors ||
+      std::any_of(shards.begin(), shards.end(),
+		  [](auto &cand) { return cand.has_errors(); })) {
+    for (auto &eval : shards) {
+      iow.shards.emplace(
+	librados::osd_shard_t{eval.source.osd, eval.source.shard},
+	eval.shard_info);
+      iow.union_shards.errors |= eval.shard_info.errors;
+    }
+    if (auth_eval.object_info) {
+      iow.version = auth_eval.object_info->version.version;
+    }
+    ret.inconsistency = iow;
+  }
+  return ret;
+}
+
+using clone_meta_list_t = std::list<std::pair<hobject_t, object_info_t>>;
+std::optional<inconsistent_snapset_wrapper> evaluate_snapset(
+  DoutPrefixProvider &dpp,
+  const hobject_t &hoid,
+  const std::optional<SnapSet> &maybe_snapset,
+  const clone_meta_list_t &clones)
+{
+  LOG_PREFIX(evaluate_snapset);
+  /* inconsistent_snapset_t has several error codes that seem to pertain to
+   * specific objects rather than to the snapset specifically.  I'm choosing
+   * to ignore those for now */
+  inconsistent_snapset_wrapper ret{hoid};
+  if (!maybe_snapset) {
+    ret.set_headless();
+    return ret;
+  }
+  const auto &snapset = *maybe_snapset;
+
+  auto clone_iter = clones.begin();
+  for (auto ss_clone_id : snapset.clones) {
+    for (; clone_iter != clones.end() &&
+	   clone_iter->first.snap < ss_clone_id;
+	 ++clone_iter) {
+      ret.set_clone(clone_iter->first.snap);
+    }
+
+    if (clone_iter != clones.end() &&
+	clone_iter->first.snap == ss_clone_id) {
+      auto ss_clone_size_iter = snapset.clone_size.find(ss_clone_id);
+      if (ss_clone_size_iter == snapset.clone_size.end() ||
+	  ss_clone_size_iter->second != clone_iter->second.size) {
+	ret.set_size_mismatch();
+      }
+      ++clone_iter;
+    } else {
+      ret.set_clone_missing(ss_clone_id);
+    }
+  }
+
+  for (; clone_iter != clones.end(); ++clone_iter) {
+    ret.set_clone(clone_iter->first.snap);
+  }
+
+  if (ret.errors) {
+    DEBUGDPP(
+      "snapset {}, clones {}",
+      dpp, snapset, clones);
+    return ret;
+  } else {
+    return std::nullopt;
+  }
+}
+
+void add_object_to_stats(
+  const chunk_validation_policy_t &policy,
+  const object_evaluation_t &eval,
+  object_stat_sum_t *out)
+{
+  auto &ss = eval.snapset;
+  if (!eval.object_info) {
+    return;
+  }
+  auto &oi = *eval.object_info;
+  ceph_assert(out);
+  out->num_objects++;
+  if (ss) {
+    out->num_bytes += oi.size;
+    for (auto clone : ss->clones) {
+      out->num_bytes += ss->get_clone_bytes(clone);
+      out->num_object_clones++;
+    }
+    if (oi.is_whiteout()) {
+      out->num_whiteouts++;
+    }
+  }
+  if (oi.is_dirty()) {
+    out->num_objects_dirty++;
+  }
+  if (oi.is_cache_pinned()) {
+    out->num_objects_pinned++;
+  }
+  if (oi.has_manifest()) {
+    out->num_objects_manifest++;
+  }
+
+  if (eval.omap_keys > 0) {
+    out->num_objects_omap++;
+  }
+  out->num_omap_keys += eval.omap_keys;
+  out->num_omap_bytes += eval.omap_bytes;
+
+  if (oi.soid.nspace == policy.hitset_namespace) {
+    out->num_objects_hit_set_archive++;
+    out->num_bytes_hit_set_archive += oi.size;
+  }
+
+  if (eval.omap_keys > policy.omap_key_limit ||
+      eval.omap_bytes > policy.omap_bytes_limit) {
+    out->num_large_omap_objects++;
+  }
+}
+
+chunk_result_t validate_chunk(
+  DoutPrefixProvider &dpp,
+  const chunk_validation_policy_t &policy, const scrub_map_set_t &in)
+{
+  chunk_result_t ret;
+
+  const std::set<hobject_t> object_set = get_object_set(in);
+
+  std::list<std::pair<hobject_t, SnapSet>> heads;
+  clone_meta_list_t clones;
+  for (const auto &oid: object_set) {
+    object_evaluation_t eval = evaluate_object(policy, oid, in);
+    add_object_to_stats(policy, eval, &ret.stats);
+    if (eval.inconsistency) {
+      ret.object_errors.push_back(*eval.inconsistency);
+    }
+    if (oid.is_head()) {
+      /* We're only going to consider the head object as "existing" if
+       * evaluate_object was able to find a sensible, authoritative copy
+       * complete with snapset */
+      if (eval.snapset) {
+	heads.emplace_back(oid, *eval.snapset);
+      }
+    } else {
+      /* We're only going to consider the clone object as "existing" if
+       * evaluate_object was able to find a sensible, authoritative copy
+       * complete with an object_info */
+      if (eval.object_info) {
+	clones.emplace_back(oid, *eval.object_info);
+      }
+    }
+  }
+
+  const hobject_t max_oid = hobject_t::get_max();
+  while (heads.size() || clones.size()) {
+    const hobject_t &next_head = heads.size() ? heads.front().first : max_oid;
+    const hobject_t &next_clone = clones.size() ? clones.front().first : max_oid;
+    hobject_t head_to_process = std::min(next_head, next_clone).get_head();
+
+    clone_meta_list_t clones_to_process;
+    auto clone_iter = clones.begin();
+    while (clone_iter != clones.end() && clone_iter->first < head_to_process)
+      ++clone_iter;
+    clones_to_process.splice(
+      clones_to_process.end(), clones, clones.begin(), clone_iter);
+
+    const auto head_meta = [&]() -> std::optional<SnapSet> {
+      if (head_to_process == next_head) {
+	auto ret = std::move(heads.front().second);
+	heads.pop_front();
+	return ret;
+      } else {
+	return std::nullopt;
+      }
+    }();
+
+    if (auto result = evaluate_snapset(
+	  dpp, head_to_process, head_meta, clones_to_process); result) {
+      ret.snapset_errors.push_back(*result);
+    }
+  }
+
+  for (const auto &i: ret.object_errors) {
+    ret.stats.num_shallow_scrub_errors +=
+      (i.has_shallow_errors() || i.union_shards.has_shallow_errors());
+    ret.stats.num_deep_scrub_errors +=
+      (i.has_deep_errors() || i.union_shards.has_deep_errors());
+  }
+  ret.stats.num_shallow_scrub_errors += ret.snapset_errors.size();
+  ret.stats.num_scrub_errors = ret.stats.num_shallow_scrub_errors +
+    ret.stats.num_deep_scrub_errors;
+
+  return ret;
+}
+
+}
diff --git a/src/crimson/osd/scrub/scrub_validator.h b/src/crimson/osd/scrub/scrub_validator.h
new file mode 100644
index 000000000000..32f5933d0db7
--- /dev/null
+++ b/src/crimson/osd/scrub/scrub_validator.h
@@ -0,0 +1,180 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <string>
+#include <map>
+
+#include "common/config_proxy.h"
+#include "common/scrub_types.h"
+#include "crimson/common/log.h"
+#include "osd/ECUtil.h"
+#include "osd/osd_types.h"
+
+namespace crimson::osd::scrub {
+
+struct chunk_validation_policy_t {
+  pg_shard_t primary;
+  std::optional<ECUtil::stripe_info_t> stripe_info;
+
+  // osd_max_object_size
+  size_t max_object_size;
+
+  // osd_hit_set_namespace
+  std::string hitset_namespace;
+
+  // osd_deep_scrub_large_omap_object_key_threshold
+  // osd_deep_scrub_large_omap_object_value_sum_threshold
+  uint64_t omap_key_limit;
+  size_t omap_bytes_limit;
+
+
+  bool is_ec() const {
+    return !!stripe_info;
+  }
+
+  size_t logical_to_ondisk_size(size_t size) const {
+    return stripe_info ? stripe_info->logical_to_next_chunk_offset(size) : size;
+  }
+};
+
+using scrub_map_set_t = std::map<pg_shard_t, ScrubMap>;
+
+struct chunk_result_t {
+  /* Scrub interacts with stats in two ways:
+   * 1. scrub accumulates a subset of object_stat_sum_t members to
+   *    to ultimately compare to the object_stat_sum_t value maintained
+   *    by the OSD. These members will be referred to as
+   *    *scrub_checked_stats*.
+   *    See iterate_scrub_checked_stats() for the relevant members.
+   * 2. scrub also updates some members that can't be maintained online
+   *    (like num_omap_*, num_large_omap_objects) or that pertain
+   *    specifically to scrub (like num_shallow_scrub_errors).
+   *    Let these by referred to as *scrub_maintained_stats*.
+   *    See iterate_scrub_maintained_stats() for the relevant members.
+   *
+   * The following stats member contains both, but the two sets are
+   * disjoint and treated seperately.
+   */
+  object_stat_sum_t stats;
+
+  // detected errors
+  std::vector<inconsistent_snapset_wrapper> snapset_errors;
+  std::vector<inconsistent_obj_wrapper> object_errors;
+
+  bool has_errors() const {
+    return !snapset_errors.empty() || !object_errors.empty();
+  }
+};
+
+/**
+ * validate_chunk
+ *
+ * Compares shard chunks and based on policy and returns a chunk_result_t
+ * containing the results.  See chunk_result_t for details.
+ */
+chunk_result_t validate_chunk(
+  DoutPrefixProvider &dpp,
+  const chunk_validation_policy_t &policy, const scrub_map_set_t &in);
+
+/**
+ * iterate_scrub_checked_stats
+ *
+ * For each scrub_checked_stat member of object_stat_sum_t, invokes
+ * op with three arguments:
+ * - name of member (string_view)
+ * - pointer to member (T object_stat_sum_t::*)
+ * - function to corresponding pg_stat_t invalid member
+ *   (bool func(const pg_stat_t &))
+ *
+ * Should be used to perform operations on all scrub_checked_stat members
+ * such as checking the accumlated scrub stats against the maintained
+ * pg stats.
+ */
+template <typename Func>
+void foreach_scrub_checked_stat(Func &&op) {
+  using namespace std::string_view_literals;
+  op("num_objects"sv,
+     &object_stat_sum_t::num_objects,
+     [](const pg_stat_t &in) { return false; });
+  op("num_bytes"sv,
+     &object_stat_sum_t::num_bytes,
+     [](const pg_stat_t &in) { return false; });
+  op("num_object_clones"sv,
+     &object_stat_sum_t::num_object_clones,
+     [](const pg_stat_t &in) { return false; });
+  op("num_whiteouts"sv,
+     &object_stat_sum_t::num_whiteouts,
+     [](const pg_stat_t &in) { return false; });
+  op("num_objects_dirty"sv,
+     &object_stat_sum_t::num_objects_dirty,
+     [](const pg_stat_t &in) { return in.dirty_stats_invalid; });
+  op("num_objects_omap"sv,
+     &object_stat_sum_t::num_objects_omap,
+     [](const pg_stat_t &in) { return in.omap_stats_invalid; });
+  op("num_objects_pinned"sv,
+     &object_stat_sum_t::num_objects_pinned,
+     [](const pg_stat_t &in) { return in.pin_stats_invalid; });
+  op("num_objects_hit_set_archive"sv,
+     &object_stat_sum_t::num_objects_hit_set_archive,
+     [](const pg_stat_t &in) { return in.hitset_stats_invalid; });
+  op("num_bytes_hit_set_archive"sv,
+     &object_stat_sum_t::num_bytes_hit_set_archive,
+     [](const pg_stat_t &in) { return in.hitset_bytes_stats_invalid; });
+  op("num_objects_manifest"sv,
+     &object_stat_sum_t::num_objects_manifest,
+     [](const pg_stat_t &in) { return in.manifest_stats_invalid; });
+}
+
+/**
+ * iterate_scrub_maintained_stats
+ *
+ * For each scrub_maintained_stat member of object_stat_sum_t, invokes
+ * op with three arguments:
+ * - name of member (string_view)
+ * - pointer to member (T object_stat_sum_t::*)
+ * - skip for shallow (bool)
+ *
+ * Should be used to perform operations on all scrub_maintained_stat members
+ * such as updating the pg maintained instance once scrub is complete.
+ */
+template <typename Func>
+void foreach_scrub_maintained_stat(Func &&op) {
+  using namespace std::string_view_literals;
+  op("num_scrub_errors"sv, &object_stat_sum_t::num_scrub_errors, false);
+  op("num_shallow_scrub_errors"sv,
+     &object_stat_sum_t::num_shallow_scrub_errors,
+     false);
+  op("num_deep_scrub_errors"sv, &object_stat_sum_t::num_deep_scrub_errors, true);
+  op("num_omap_bytes"sv, &object_stat_sum_t::num_omap_bytes, true);
+  op("num_omap_keys"sv, &object_stat_sum_t::num_omap_keys, true);
+  op("num_large_omap_objects"sv,
+     &object_stat_sum_t::num_large_omap_objects,
+     true);
+}
+
+}
+
+template <>
+struct fmt::formatter<crimson::osd::scrub::chunk_result_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(
+    const crimson::osd::scrub::chunk_result_t &result, FormatContext& ctx) const
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "chunk_result_t("
+      "num_scrub_errors: {}, "
+      "num_deep_scrub_errors: {}, "
+      "snapset_errors: [{}], "
+      "object_errors: [{}])",
+      result.stats.num_scrub_errors,
+      result.stats.num_deep_scrub_errors,
+      result.snapset_errors,
+      result.object_errors
+    );
+  }
+};
diff --git a/src/test/crimson/CMakeLists.txt b/src/test/crimson/CMakeLists.txt
index b1851cca2c7e..c943ff885464 100644
--- a/src/test/crimson/CMakeLists.txt
+++ b/src/test/crimson/CMakeLists.txt
@@ -103,3 +103,13 @@ target_link_libraries(
   crimson::gtest)
 add_ceph_unittest(unittest-seastar-errorator
   --memory 256M --smp 1)
+
+add_executable(unittest-crimson-scrub
+  test_crimson_scrub.cc
+  ${PROJECT_SOURCE_DIR}/src/crimson/osd/scrub/scrub_machine.cc
+  ${PROJECT_SOURCE_DIR}/src/crimson/osd/scrub/scrub_validator.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc)
+target_link_libraries(
+  unittest-crimson-scrub
+  crimson-common
+  crimson::gtest)
diff --git a/src/test/crimson/test_crimson_scrub.cc b/src/test/crimson/test_crimson_scrub.cc
new file mode 100644
index 000000000000..65b1f3152539
--- /dev/null
+++ b/src/test/crimson/test_crimson_scrub.cc
@@ -0,0 +1,1347 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <boost/iterator/transform_iterator.hpp>
+
+#include <fmt/ranges.h>
+
+#include <seastar/core/sleep.hh>
+
+#include "test/crimson/gtest_seastar.h"
+
+#include "include/rados/rados_types.hpp"
+#include "common/scrub_types.h"
+#include "crimson/common/interruptible_future.h"
+#include "crimson/osd/scrub/scrub_machine.h"
+#include "crimson/osd/scrub/scrub_validator.h"
+
+#include "osd/osd_types_fmt.h"
+
+constexpr static size_t TEST_MAX_OBJECT_SIZE = 128<<20;
+constexpr static std::string_view TEST_INTERNAL_NAMESPACE = ".internal";
+constexpr static uint64_t TEST_OMAP_KEY_LIMIT = 200000;
+constexpr static size_t TEST_OMAP_BYTES_LIMIT = 1<<30;
+
+void so_set_attr_len(ScrubMap::object &obj, const std::string &name, size_t len)
+{
+  obj.attrs[name] = buffer::ptr(len);
+}
+
+void so_set_attr(ScrubMap::object &obj, const std::string &name, bufferlist bl)
+{
+  bl.rebuild();
+  obj.attrs[name] = bl.front();
+}
+
+std::optional<bufferlist> so_get_attr(
+  ScrubMap::object &obj, const std::string &name)
+{
+  if (obj.attrs.count(name)) {
+    bufferlist bl;
+    bl.push_back(obj.attrs[name]);
+    return bl;
+  } else {
+    return std::nullopt;
+  }
+}
+
+template <typename T>
+void so_set_attr_type(
+  ScrubMap::object &obj, const std::string &name,
+  const std::optional<T> &v)
+{
+  if (v) {
+    bufferlist bl;
+    encode(*v, bl, CEPH_FEATURES_ALL);
+    so_set_attr(obj, name, std::move(bl));
+  } else {
+    obj.attrs.erase(name);
+  }
+}
+
+template <typename T>
+std::optional<T> so_get_attr_type(ScrubMap::object &obj, const std::string &name)
+{
+  auto maybe_bl = so_get_attr(obj, name);
+  if (!maybe_bl) {
+    return std::nullopt;
+  }
+  auto bl = std::move(*maybe_bl);
+  try {
+    T ret;
+    auto bliter = bl.cbegin();
+    decode(ret, bliter);
+    return ret;
+  } catch (...) {
+    return std::nullopt;
+  }
+}
+
+void so_set_oi(ScrubMap::object &obj, const std::optional<object_info_t> &oi)
+{
+  return so_set_attr_type<object_info_t>(obj, OI_ATTR, oi);
+}
+
+std::optional<object_info_t> so_get_oi(ScrubMap::object &obj)
+{
+  return so_get_attr_type<object_info_t>(obj, OI_ATTR);
+}
+
+template <typename F>
+void so_mut_oi(ScrubMap::object &obj, F &&f) {
+  so_set_oi(obj, std::invoke(std::forward<F>(f), so_get_oi(obj)));
+}
+
+void so_set_ss(ScrubMap::object &obj, const std::optional<SnapSet> &ss)
+{
+  return so_set_attr_type<SnapSet>(obj, SS_ATTR, ss);
+}
+
+std::optional<SnapSet> so_get_ss(ScrubMap::object &obj)
+{
+  return so_get_attr_type<SnapSet>(obj, SS_ATTR);
+}
+
+template <typename F>
+void so_mut_ss(ScrubMap::object &obj, F &&f) {
+  so_set_ss(obj, std::invoke(std::forward<F>(f), so_get_ss(obj)));
+}
+
+void so_set_hinfo(
+  ScrubMap::object &obj, const std::optional<ECUtil::HashInfo> &hinfo)
+{
+  return so_set_attr_type<ECUtil::HashInfo>(obj, ECUtil::get_hinfo_key(), hinfo);
+}
+
+std::optional<ECUtil::HashInfo> so_get_hinfo(ScrubMap::object &obj)
+{
+  return so_get_attr_type<ECUtil::HashInfo>(obj, ECUtil::get_hinfo_key());
+}
+
+template <typename F>
+void so_mut_hinfo(ScrubMap::object &obj, F &&f) {
+  auto maybe_hinfo = so_get_hinfo(obj);
+  auto new_maybe_hinfo = std::invoke(std::forward<F>(f), std::move(maybe_hinfo));
+  so_set_hinfo(obj, new_maybe_hinfo);
+}
+
+/**
+ * so_builder_t
+ *
+ * Utility class for constructing test objects.
+ */
+struct so_builder_t {
+  ScrubMap::object so;
+
+  void set_defaults() {
+    so.size = 0;
+    so_mut_oi(so, [](auto maybe_oi) {
+      if (maybe_oi) {
+	maybe_oi->size = 0;
+      }
+      return maybe_oi;
+    });
+  }
+
+  static hobject_t make_hoid(std::string name, snapid_t cloneid=CEPH_NOSNAP) {
+    auto oid = object_t(name);
+    return hobject_t{
+      oid,
+      "",
+      cloneid,
+      static_cast<uint32_t>(std::hash<object_t>()(oid)),
+      1,
+      ""
+    };
+  }
+
+  static so_builder_t make_head(std::string name) {
+    auto hoid = make_hoid(name);
+    so_builder_t ret;
+    so_set_oi(ret.so, object_info_t{hoid});
+    so_set_ss(ret.so, SnapSet{});
+    ret.set_defaults();
+    return ret;
+  }
+
+  static so_builder_t make_clone(
+    std::string name,
+    snapid_t cloneid = 4
+  ) {
+    auto hoid = make_hoid(name, cloneid);
+    so_builder_t ret;
+    so_set_oi(ret.so, object_info_t{hoid});
+    ret.set_defaults();
+    return ret;
+  }
+
+  static so_builder_t make_ec_head(std::string name) {
+    auto ret = make_head(name);
+    so_set_hinfo(ret.so, ECUtil::HashInfo{});
+    return ret;
+  }
+
+  static so_builder_t make_ec_clone(
+    std::string name,
+    snapid_t cloneid = 4
+  ) {
+    auto ret = make_clone(name, cloneid);
+    so_set_hinfo(ret.so, ECUtil::HashInfo{});
+    return ret;
+  }
+
+  so_builder_t &set_size(
+    size_t size,
+    const std::optional<ECUtil::stripe_info_t> stripe_info = std::nullopt) {
+    if (stripe_info) {
+      so.size = stripe_info->logical_to_next_chunk_offset(size);
+    } else {
+      so.size = size;
+    }
+
+    so_mut_oi(so, [size](auto maybe_oi) {
+      if (maybe_oi) {
+	maybe_oi->size = size;
+      }
+      return maybe_oi;
+    });
+    so_mut_hinfo(so, [size, &stripe_info](auto maybe_hinfo) {
+      if (maybe_hinfo) {
+	ceph_assert(stripe_info);
+	maybe_hinfo->set_total_chunk_size_clear_hash(
+	  stripe_info->logical_to_next_chunk_offset(size));
+      }
+      return maybe_hinfo;
+    });
+    return *this;
+  }
+
+  so_builder_t &add_attr(const std::string &name, size_t len) {
+    so_set_attr_len(so, name, len);
+    return *this;
+  }
+
+  ScrubMap::object get() const {
+    return so;
+  }
+};
+
+/**
+ * test_obj_t
+ *
+ * test param combining an so_builder_t with human readable description with
+ * a stripe_info.
+ */
+struct test_obj_t : so_builder_t {
+  std::optional<ECUtil::stripe_info_t> stripe_info;
+  std::string desc;
+  hobject_t hoid;
+
+  test_obj_t(
+    so_builder_t _builder,
+    std::optional<ECUtil::stripe_info_t> _stripe_info,
+    std::string _desc,
+    hobject_t _hoid) :
+    so_builder_t(std::move(_builder)),
+    stripe_info(std::move(_stripe_info)),
+    desc(std::move(_desc)),
+    hoid(std::move(_hoid)) {
+    ceph_assert(!desc.empty());
+  }
+
+  static test_obj_t make(
+    const std::string &desc,
+    std::optional<ECUtil::stripe_info_t> stripe_info,
+    so_builder_t builder) {
+    hobject_t hoid = so_get_oi(builder.so)->soid;
+    return test_obj_t{
+      std::move(builder),
+      stripe_info,
+      desc,
+      std::move(hoid)};
+  }
+
+  template <typename... Args>
+  static test_obj_t make_head(const std::string &desc, Args&&... args) {
+    return make(
+      desc,
+      std::nullopt,
+      so_builder_t::make_head(std::forward<Args>(args)...));
+  }
+
+  template <typename... Args>
+  static test_obj_t make_clone(const std::string &desc, Args&&... args) {
+    return make(
+      desc,
+      std::nullopt,
+      so_builder_t::make_clone(std::forward<Args>(args)...));
+  }
+
+  template <typename... Args>
+  static test_obj_t make_ec_head(const std::string &desc, Args&&... args) {
+    return make(
+      desc,
+      ECUtil::stripe_info_t{4, 1<<20},
+      so_builder_t::make_ec_head(std::forward<Args>(args)...));
+  }
+
+  template <typename... Args>
+  static test_obj_t make_ec_clone(const std::string &desc, Args&&... args) {
+    return make(
+      desc,
+      ECUtil::stripe_info_t{4, 1<<20},
+      so_builder_t::make_ec_clone(std::forward<Args>(args)...));
+  }
+
+  test_obj_t &set_size(
+    size_t size) {
+    so_builder_t::set_size(size, stripe_info);
+    return *this;
+  }
+
+  test_obj_t &add_attr(const std::string &name, size_t len) {
+    so_builder_t::add_attr(name, len);
+    return *this;
+  }
+
+  ScrubMap::object get() const {
+    return so_builder_t::get();
+  }
+};
+
+/**
+ * Interface for a test case on a single object.
+ */
+struct SingleErrorTestCase {
+  /// Describes limitations on test preconditions
+  enum class restriction_t {
+    NONE,         /// No limitations
+    REPLICA_ONLY, /// Only works if injected on replica
+    EC_ONLY,      /// Only valid for ec objects
+    HEAD_ONLY     /// Only valid for head objects
+  };
+
+  /// returns human-readable string describing the test for debugging
+  virtual std::string_view get_description() const = 0;
+
+  /// returns test_obj_t with error injected
+  virtual test_obj_t adjust_base_object(test_obj_t ret) const {
+    return ret;
+  }
+
+  /// returns test_obj_t with error injected
+  virtual test_obj_t inject_error(test_obj_t) const = 0;
+
+  /// returns expected shard error
+  virtual librados::err_t get_shard_error_sig() const = 0;
+
+  /// returns expected object error
+  virtual librados::obj_err_t get_object_error_sig() const = 0;
+
+  /// returns true if test should be run with passed restriction
+  virtual bool valid_for_restriction(restriction_t restriction) const = 0;
+
+  virtual ~SingleErrorTestCase() = default;
+};
+
+/// Utility template for implementing SimpleErrorTestCase
+template <typename T>
+struct SingleErrorTestCaseT : SingleErrorTestCase {
+  /// Defaults for REQUIRE_EC and REQUIRES_HEAD
+  constexpr static bool REQUIRES_EC = false;
+  constexpr static bool REQUIRES_HEAD = false;
+
+  /* Every implementor must define:
+  constexpr static librados::err_t shard_error_sig{
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+  };
+  */
+
+  librados::err_t get_shard_error_sig() const final {
+    return T::shard_error_sig;
+  }
+  librados::obj_err_t get_object_error_sig() const final {
+    return T::object_error_sig;
+  }
+
+  constexpr static bool requires_ec() {
+    return T::REQUIRES_EC;
+  }
+  constexpr static bool requires_head() {
+    return T::REQUIRES_HEAD;
+  }
+  constexpr static bool requires_replica() {
+    /* If there are no shard_errors, we'll take primary to be authoritative. */
+    return T::shard_error_sig.errors == 0;
+  }
+
+  bool valid_for_restriction(restriction_t restriction) const final {
+    // There aren't currently any tests with two restrictions, if this
+    // changes, the suite instantiations will need to change as well.
+    static_assert(
+      (requires_ec() + requires_head() + requires_replica()) <= 1);
+    return [] {
+      if constexpr (requires_replica()) {
+	return restriction_t::REPLICA_ONLY;
+      } else if constexpr (requires_head()) {
+	return restriction_t::HEAD_ONLY;
+      } else if constexpr (requires_ec()) {
+	return restriction_t::EC_ONLY;
+      } else {
+	return restriction_t::NONE;
+      }
+    }() == restriction;
+  }
+  virtual ~SingleErrorTestCaseT() = default;
+};
+
+/* The following classes exercise each possible error code detected
+ * by evaluate_object_shard and compare_candidate_to_authoritative
+ * in crimson/osd/scrub/scrub_validator.*
+ *
+ * Note, any newly added cases must also be added to the test_cases
+ * array below.
+ */
+
+struct ECHashMismatch : SingleErrorTestCaseT<ECHashMismatch> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SHARD_EC_HASH_MISMATCH
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+  };
+
+  std::string_view get_description() const {
+    return "ECHashMismatch";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.ec_hash_mismatch = true;
+    return obj;
+  }
+};
+
+struct ECSizeMismatch : SingleErrorTestCaseT<ECSizeMismatch> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SHARD_EC_SIZE_MISMATCH
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+  };
+
+  std::string_view get_description() const {
+    return "ECSizeMismatch";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.ec_size_mismatch = true;
+    return obj;
+  }
+};
+
+struct ReadError : SingleErrorTestCaseT<ReadError> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SHARD_READ_ERR
+  };
+  constexpr static librados::obj_err_t object_error_sig{};
+
+  std::string_view get_description() const {
+    return "ReadError";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.read_error = true;
+    return obj;
+  }
+};
+
+struct StatError : SingleErrorTestCaseT<StatError> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SHARD_STAT_ERR
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+  };
+
+  std::string_view get_description() const {
+    return "StatError";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.stat_error = true;
+    return obj;
+  }
+};
+
+struct MissingOI : SingleErrorTestCaseT<MissingOI> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::INFO_MISSING
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::OBJECT_INFO_INCONSISTENCY
+  };
+
+  std::string_view get_description() const {
+    return "MissingOI";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    so_mut_oi(obj.so, [](auto) { return std::nullopt; });
+    return obj;
+  }
+};
+
+struct CorruptOI: SingleErrorTestCaseT<CorruptOI> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::INFO_CORRUPTED
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::OBJECT_INFO_INCONSISTENCY
+  };
+
+  std::string_view get_description() const {
+    return "CorruptOI";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    so_set_attr_len(obj.so, OI_ATTR, 10);
+    return obj;
+  }
+};
+
+struct CorruptOndiskSize : SingleErrorTestCaseT<CorruptOndiskSize> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SIZE_MISMATCH_INFO
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::SIZE_MISMATCH
+  };
+
+  std::string_view get_description() const {
+    return "CorruptOndiskSize";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.size += 2;
+    return obj;
+  }
+};
+
+struct MissingSS : SingleErrorTestCaseT<MissingSS> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SNAPSET_MISSING
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::SNAPSET_INCONSISTENCY
+  };
+  constexpr static bool REQUIRES_HEAD = true;
+
+  std::string_view get_description() const {
+    return "MissingSS";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(obj.hoid.is_head());
+    so_mut_ss(obj.so, [](auto) { return std::nullopt; });
+    return obj;
+  }
+};
+
+struct CorruptSS : SingleErrorTestCaseT<CorruptSS> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::SNAPSET_CORRUPTED
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::SNAPSET_INCONSISTENCY
+  };
+  constexpr static bool REQUIRES_HEAD = true;
+
+  std::string_view get_description() const {
+    return "CorruptSS";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(obj.hoid.is_head());
+    so_set_attr_len(obj.so, SS_ATTR, 10);
+    return obj;
+  }
+};
+
+struct MissingHinfo : SingleErrorTestCaseT<MissingHinfo> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::HINFO_MISSING
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::HINFO_INCONSISTENCY
+  };
+  constexpr static bool REQUIRES_EC = true;
+
+  std::string_view get_description() const {
+    return "MissingHinfo";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(obj.stripe_info);
+    so_mut_hinfo(obj.so, [](auto) { return std::nullopt; });
+    return obj;
+  }
+};
+
+struct CorruptHinfo : SingleErrorTestCaseT<CorruptHinfo> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::HINFO_CORRUPTED
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::HINFO_INCONSISTENCY
+  };
+  constexpr static bool REQUIRES_EC = true;
+
+  std::string_view get_description() const {
+    return "CorruptHinfo";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(obj.stripe_info);
+    so_set_attr_len(obj.so, ECUtil::get_hinfo_key(), 10);
+    return obj;
+  }
+};
+
+struct DataDigestMismatch : SingleErrorTestCaseT<DataDigestMismatch> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::DATA_DIGEST_MISMATCH_INFO
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::DATA_DIGEST_MISMATCH
+  };
+
+  std::string_view get_description() const {
+    return "DataDigestMismatch";
+  };
+  test_obj_t adjust_base_object(test_obj_t obj) const {
+    so_mut_oi(obj.so, [](auto maybe_oi) {
+      ceph_assert(maybe_oi);
+      maybe_oi->set_data_digest(1);
+      return maybe_oi;
+    });
+    obj.so.digest_present = true;
+    obj.so.digest = 1;
+    return obj;
+  }
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(so_get_oi(obj.so)->is_data_digest());
+    obj.so.digest = 2;
+    return obj;
+  }
+};
+
+struct OmapDigestMismatch : SingleErrorTestCaseT<OmapDigestMismatch> {
+  constexpr static librados::err_t shard_error_sig{
+    librados::err_t::OMAP_DIGEST_MISMATCH_INFO
+  };
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::OMAP_DIGEST_MISMATCH
+  };
+
+  std::string_view get_description() const {
+    return "OmapDigestMismatch";
+  };
+  test_obj_t adjust_base_object(test_obj_t obj) const {
+    so_mut_oi(obj.so, [](auto maybe_oi) {
+      ceph_assert(maybe_oi);
+      maybe_oi->set_omap_digest(1);
+      return maybe_oi;
+    });
+    obj.so.omap_digest_present = true;
+    obj.so.omap_digest = 1;
+    return obj;
+  }
+  test_obj_t inject_error(test_obj_t obj) const {
+    ceph_assert(so_get_oi(obj.so)->is_omap_digest());
+    obj.so.omap_digest = 2;
+    return obj;
+  }
+};
+
+struct ExtraAttribute : SingleErrorTestCaseT<ExtraAttribute> {
+  constexpr static librados::err_t shard_error_sig{};
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::ATTR_NAME_MISMATCH
+  };
+
+  std::string_view get_description() const {
+    return "ExtraAttribute";
+  };
+  test_obj_t inject_error(test_obj_t obj) const {
+    so_set_attr_len(obj.so, "attr_added_erroneously", 10);
+    return obj;
+  }
+};
+
+struct MissingAttribute : SingleErrorTestCaseT<MissingAttribute> {
+  constexpr static librados::err_t shard_error_sig{};
+  constexpr static librados::obj_err_t object_error_sig{
+    librados::obj_err_t::ATTR_NAME_MISMATCH
+  };
+
+  std::string_view get_description() const {
+    return "MissingAttribute";
+  };
+  test_obj_t adjust_base_object(test_obj_t obj) const {
+    so_set_attr_len(obj.so, "attr_to_be_missing", 10);
+    return obj;
+  }
+  test_obj_t inject_error(test_obj_t obj) const {
+    obj.so.attrs.erase("attr_to_be_missing");
+    return obj;
+  }
+};
+
+template <>
+struct fmt::formatter<SingleErrorTestCase> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &test_case, FormatContext& ctx) const
+  {
+    return fmt::format_to(
+      ctx.out(), "{}",
+      test_case.get_description());
+  }
+};
+
+std::unique_ptr<SingleErrorTestCase> test_cases[] = {
+  std::make_unique<ECHashMismatch>(),
+  std::make_unique<ECSizeMismatch>(),
+  std::make_unique<ReadError>(),
+  std::make_unique<StatError>(),
+  std::make_unique<MissingOI>(),
+  std::make_unique<CorruptOI>(),
+  std::make_unique<CorruptOndiskSize>(),
+  std::make_unique<MissingSS>(),
+  std::make_unique<CorruptSS>(),
+  std::make_unique<MissingHinfo>(),
+  std::make_unique<CorruptHinfo>(),
+  std::make_unique<DataDigestMismatch>(),
+  std::make_unique<OmapDigestMismatch>(),
+  std::make_unique<ExtraAttribute>(),
+  std::make_unique<MissingAttribute>()
+};
+const SingleErrorTestCase *to_ptr(
+  const std::unique_ptr<SingleErrorTestCase> &tc) {
+  return tc.get();
+}
+// iterator over the above set as pointers
+using test_case_ptr_iter_t = boost::transform_iterator<
+  std::function<decltype(to_ptr)>, decltype(std::begin(test_cases))>;
+template <SingleErrorTestCase::restriction_t restriction>
+struct test_case_filter_t {
+  bool operator()(const SingleErrorTestCase *tc) const {
+    return tc->valid_for_restriction(restriction);
+  }
+};
+template <SingleErrorTestCase::restriction_t restriction>
+// iterator over the above set filtered by restriction
+using test_case_filter_iter_t = boost::filter_iterator<
+  test_case_filter_t<restriction>,
+  test_case_ptr_iter_t>;
+template <SingleErrorTestCase::restriction_t restriction>
+// begin and end, used below to instantiate test suites
+auto test_cases_begin() {
+  return test_case_filter_iter_t<restriction>(
+    test_case_filter_t<restriction>(),
+    test_case_ptr_iter_t(std::begin(test_cases), to_ptr),
+    test_case_ptr_iter_t(std::end(test_cases), to_ptr));
+}
+template <SingleErrorTestCase::restriction_t restriction>
+auto test_cases_end() {
+  return test_case_filter_iter_t<restriction>(
+    test_case_filter_t<restriction>(),
+    test_case_ptr_iter_t(std::end(test_cases), to_ptr),
+    test_case_ptr_iter_t(std::end(test_cases), to_ptr));
+}
+
+/// tuple defining each generated test case
+using single_error_test_param_t = std::tuple<
+  test_obj_t,                /// initial test object
+  bool,                      /// inject on primary?
+  const SingleErrorTestCase* /// test case
+  >;
+template <>
+struct fmt::formatter<single_error_test_param_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &param, FormatContext& ctx) const
+  {
+    const auto &[obj, is_primary, test_case] = param;
+    return fmt::format_to(
+      ctx.out(), "{}{}{}",
+      obj.desc,
+      is_primary ? "Primary" : "Replica",
+      test_case->get_description());
+  }
+};
+std::ostream &operator<<(std::ostream &out, const single_error_test_param_t &p)
+{
+  return out << fmt::format("{}", p);
+}
+
+class TestSingleError :
+  public testing::TestWithParam<single_error_test_param_t> {
+};
+
+/**
+ * compare_error_signatures
+ *
+ * Generic helper for comparing err_t, obj_err_t, and
+ * inconsistent_snapset_t with descriptive output.
+ */
+auto compare_error_signatures(const auto &lh, const auto &rh)
+{
+  if (lh.errors == rh.errors) {
+    return ::testing::AssertionSuccess() << fmt::format(
+      "Signature match: {}", lh);
+  } else {
+    return ::testing::AssertionFailure() << fmt::format(
+      "Signature mismatch: {} should be {}",
+      lh, rh);
+  }
+}
+
+TEST_P(TestSingleError, SingleError) {
+  const auto &[_obj, is_primary, test_case] = GetParam();
+  auto obj = test_case->adjust_base_object(_obj);
+
+  const pg_shard_t primary(0, shard_id_t::NO_SHARD);
+  const pg_shard_t replica(1, shard_id_t::NO_SHARD);
+  crimson::osd::scrub::chunk_validation_policy_t policy {
+    primary,
+    obj.stripe_info,
+    TEST_MAX_OBJECT_SIZE,
+    std::string{TEST_INTERNAL_NAMESPACE},
+    TEST_OMAP_KEY_LIMIT,
+    TEST_OMAP_BYTES_LIMIT
+  };
+  const pg_shard_t &target = is_primary ? primary : replica;
+  const std::vector<pg_shard_t> shards = {
+    primary, replica
+  };
+
+  auto with_error = test_case->inject_error(obj);
+  crimson::osd::scrub::scrub_map_set_t maps;
+  for (const auto &osd : shards) {
+    if (osd == target) {
+      maps[osd].objects[obj.hoid] = with_error.get();
+    } else {
+      maps[osd].objects[obj.hoid] = obj.get();
+    }
+  }
+
+  DoutPrefix dpp(nullptr, ceph_subsys_test, "test_crimson_scrub");
+  const auto ret = crimson::osd::scrub::validate_chunk(
+    dpp, policy, maps);
+  const auto &object_errors = ret.object_errors;
+
+  ASSERT_EQ(object_errors.size(), 1) << fmt::format(
+    "{}: generated an incorrect number of errors: {}\n",
+    *test_case, object_errors);
+
+  auto &obj_error = object_errors.front();
+
+  EXPECT_EQ(
+    ret.stats.num_shallow_scrub_errors,
+    (obj_error.has_shallow_errors() ||
+     obj_error.union_shards.has_shallow_errors()) +
+    ret.snapset_errors.size());
+  EXPECT_EQ(
+    ret.stats.num_deep_scrub_errors,
+    (obj_error.has_deep_errors() ||
+     obj_error.union_shards.has_deep_errors()));
+
+  EXPECT_TRUE(compare_error_signatures(
+    static_cast<const librados::obj_err_t&>(obj_error),
+    test_case->get_object_error_sig()));
+
+  EXPECT_EQ(obj_error.shards.size(), shards.size());
+  bool found_selected_oi = false;
+  for (const auto &shard : shards) {
+    auto siter = obj_error.shards.find(
+      librados::osd_shard_t(shard.osd, shard.shard)
+    );
+    if (siter == obj_error.shards.end()) {
+      EXPECT_NE(siter, obj_error.shards.end());
+      continue;
+    }
+    if (shard == target) {
+      EXPECT_TRUE(compare_error_signatures(
+	static_cast<const librados::err_t&>(siter->second),
+	test_case->get_shard_error_sig()));
+    } else {
+      EXPECT_FALSE(siter->second.has_errors());
+      if (siter->second.selected_oi) found_selected_oi = true;
+    }
+    if (shard == primary) {
+      EXPECT_TRUE(siter->second.primary);
+    }
+  }
+  EXPECT_TRUE(found_selected_oi);
+}
+
+/* Tests that don't have restrictions */
+INSTANTIATE_TEST_SUITE_P(
+  SingleErrorGeneral,
+  TestSingleError,
+  ::testing::Combine(
+    ::testing::Values(
+      test_obj_t::make_head("Small", "foo").set_size(64),
+      test_obj_t::make_clone("EmptyWithAttr", "foo2").add_attr("extra_attr", 64),
+      test_obj_t::make_head("ReplicatedRBD", "foo2").set_size(4<<20),
+      test_obj_t::make_ec_head("ECHead", "foo").set_size(4<<20),
+      test_obj_t::make_ec_clone("LargeECClone", "foo").set_size(16<<20)
+    ),
+    ::testing::Bool(),
+    ::testing::ValuesIn(
+      test_cases_begin<SingleErrorTestCase::restriction_t::NONE>(),
+      test_cases_end<SingleErrorTestCase::restriction_t::NONE>())
+  ),
+  [](const auto &info) {
+    return fmt::format("{}", info.param);
+  }
+);
+
+/* Some tests don't trigger shard errors, so we can't actually tell which
+ * replica is wrong.  Such tests are written for the error to be injected
+ * on the replica. */
+INSTANTIATE_TEST_SUITE_P(
+  SingleErrorPrimaryOnly,
+  TestSingleError,
+  ::testing::Combine(
+    ::testing::Values(
+      test_obj_t::make_head("Small", "foo").set_size(64),
+      test_obj_t::make_clone("EmptyWithAttr", "foo2").add_attr("extra_attr", 64),
+      test_obj_t::make_head("ReplicatedRBD", "foo2").set_size(4<<20),
+      test_obj_t::make_ec_head("ECHead", "foo").set_size(4<<20),
+      test_obj_t::make_ec_clone("LargeECClone", "foo").set_size(16<<20)
+    ),
+    ::testing::Values(false), // replica only
+    ::testing::ValuesIn(
+      test_cases_begin<SingleErrorTestCase::restriction_t::REPLICA_ONLY>(),
+      test_cases_end<SingleErrorTestCase::restriction_t::REPLICA_ONLY>())
+  ),
+  [](const auto &info) {
+    return fmt::format("{}", info.param);
+  }
+);
+
+/* Some tests only make sense on ec objects. */
+INSTANTIATE_TEST_SUITE_P(
+  SingleErrorOnly,
+  TestSingleError,
+  ::testing::Combine(
+    ::testing::Values(
+      test_obj_t::make_ec_head("ECHead", "foo").set_size(4<<20),
+      test_obj_t::make_ec_clone("LargeECClone", "foo").set_size(16<<20)
+    ),
+    ::testing::Bool(),
+    ::testing::ValuesIn(
+      test_cases_begin<SingleErrorTestCase::restriction_t::EC_ONLY>(),
+      test_cases_end<SingleErrorTestCase::restriction_t::EC_ONLY>())
+  ),
+  [](const auto &info) {
+    return fmt::format("{}", info.param);
+  }
+);
+
+/* Some tests only make sense on head objects. */
+INSTANTIATE_TEST_SUITE_P(
+  SingleErrorHEAD,
+  TestSingleError,
+  ::testing::Combine(
+    ::testing::Values(
+      test_obj_t::make_head("Small", "foo").set_size(64),
+      test_obj_t::make_head("ReplicatedRBD", "foo2").set_size(4<<20),
+      test_obj_t::make_ec_head("ECHead", "foo").set_size(4<<20)
+    ),
+    ::testing::Bool(),
+    ::testing::ValuesIn(
+      test_cases_begin<SingleErrorTestCase::restriction_t::HEAD_ONLY>(),
+      test_cases_end<SingleErrorTestCase::restriction_t::HEAD_ONLY>())
+  ),
+  [](const auto &info) {
+    return fmt::format("{}", info.param);
+  }
+);
+
+using test_clone_spec_t = std::pair<
+  snapid_t, // clone id
+  size_t    // clone size
+  >;
+
+/// descending order of clone id
+using test_clone_list_t = std::vector<test_clone_spec_t>;
+
+/**
+ * snapset_test_case_t
+ *
+ * This descriptor can express 3 types of error
+ * - missing clone
+ * - extra clone
+ * - clone size mismatch
+ * in 4 positions using one bit for each pair.
+ */
+class snapset_test_case_t {
+  uint32_t signature;
+
+  snapset_test_case_t(uint32_t signature) : signature(signature) {}
+
+  constexpr static uint32_t POSITION_BITS = 4;
+  constexpr static uint32_t position_mask[] = {
+    0x1, 0x2, 0x4, 0x8
+  };
+  constexpr static unsigned MAX_POS = std::size(position_mask);
+
+  constexpr static uint32_t MIN_VALID = 0;
+  constexpr static uint32_t MAX_VALID = 0xFFF;
+  enum type_t {
+    MISSING = 0,
+    EXTRA,
+    SIZE
+  };
+
+  bool should_inject(type_t type, unsigned position) const {
+    ceph_assert(position < MAX_POS);
+    return (signature >> (type * POSITION_BITS)) & position_mask[position];
+  }
+  static snapset_test_case_t make(type_t type, unsigned position) {
+    ceph_assert(position < std::size(position_mask));
+    return snapset_test_case_t{
+      position_mask[position] << (type * POSITION_BITS)
+    };
+  }
+  static auto generate_single_errors(type_t type) {
+    std::vector<snapset_test_case_t> ret;
+    ret.reserve(std::size(position_mask));
+    for (unsigned i = 0; i < MAX_POS; ++i) {
+      ret.push_back(make(type, i));
+    }
+    return ret;
+  }
+
+public:
+  constexpr static unsigned get_max_pos() { return MAX_POS; }
+
+  bool should_inject_missing(unsigned position) const {
+    return should_inject(MISSING, position);
+  }
+  bool should_inject_extra(unsigned position) const {
+    return should_inject(EXTRA, position);
+  }
+  bool should_inject_size(unsigned position) const {
+    return should_inject(SIZE, position);
+  }
+
+  static auto generate_single_missing_errors() {
+    return generate_single_errors(MISSING);
+  }
+  static auto generate_single_extra_errors() {
+    return generate_single_errors(EXTRA);
+  }
+  static auto generate_single_size_errors() {
+    return generate_single_errors(SIZE);
+  }
+  static auto generate_random_errors(size_t num, int seed = 0) {
+    std::default_random_engine e1(seed);
+    std::uniform_int_distribution<uint32_t> uniform_dist(1, MAX_VALID);
+
+    std::vector<snapset_test_case_t> ret;
+    ret.reserve(num);
+    for (unsigned i = 0; i < num; ++i) {
+      ret.push_back(snapset_test_case_t{uniform_dist(e1)});
+    }
+    return ret;
+  }
+  friend std::ostream &operator<<(std::ostream &out, snapset_test_case_t rhs);
+};
+std::ostream &operator<<(std::ostream &out, snapset_test_case_t rhs) {
+  for (auto &[s, type] :
+	 std::vector<std::pair<std::string, snapset_test_case_t::type_t>>(
+	   {{"M", snapset_test_case_t::MISSING},
+	    {"E", snapset_test_case_t::EXTRA},
+	    {"S", snapset_test_case_t::SIZE}})) {
+    out << s;
+    for (unsigned i = 0;
+	 i < snapset_test_case_t::MAX_POS; ++i) {
+      if (rhs.should_inject(type, i)) {
+	out << i;
+      }
+    }
+  }
+  return out;
+}
+
+class TestSnapSetCloneError :
+  public testing::TestWithParam<snapset_test_case_t> {
+};
+
+
+SnapSet make_snapset(const test_clone_list_t &clone_list)
+{
+  SnapSet ss;
+  for (const auto &[cloneid, size] : clone_list) {
+    ss.clones.push_back(cloneid);
+    ss.clone_size[cloneid] = size;
+    ss.clone_overlap[cloneid];
+    ss.clone_snaps[cloneid].push_back(cloneid);
+  }
+  return ss;
+}
+
+std::pair<hobject_t, ScrubMap::object> make_clone(
+  std::string name, std::pair<snapid_t, size_t> in)
+{
+  ScrubMap ret;
+  auto [cloneid, size] = in;
+  hobject_t hoid = so_builder_t::make_hoid(name, in.first);
+  auto so = so_builder_t::make_clone(
+    name, cloneid);
+  so.set_size(size);
+  return std::make_pair(hoid, so.get());
+}
+
+TEST_P(TestSnapSetCloneError, CloneError) {
+  const pg_shard_t primary(0, shard_id_t::NO_SHARD);
+  crimson::osd::scrub::chunk_validation_policy_t policy {
+    primary,
+    std::nullopt,
+    TEST_MAX_OBJECT_SIZE,
+    std::string{TEST_INTERNAL_NAMESPACE},
+    TEST_OMAP_KEY_LIMIT,
+    TEST_OMAP_BYTES_LIMIT
+  };
+
+  crimson::osd::scrub::scrub_map_set_t maps;
+  const std::string name = "test_obj";
+  auto &map = maps[primary];
+  inconsistent_snapset_wrapper expected_error;
+
+  test_clone_list_t should_exist = {
+    { 10, 32 }, { 25,  64 }, { 50,  32 }, { 100,  64 }
+  };
+  test_clone_list_t extra = {
+    { 9, 64 }, { 11, 32 }, { 99, 64 }, { 101, 32 }
+  };
+
+  for (unsigned i = 0; i < snapset_test_case_t::get_max_pos(); ++i) {
+    hobject_t hoid = so_builder_t::make_hoid(name, should_exist[i].first);
+    if (!GetParam().should_inject_missing(i)) {
+      auto to_insert = make_clone(name, should_exist[i]);
+      if (GetParam().should_inject_size(i)) {
+	expected_error.set_size_mismatch();
+	to_insert.second = so_builder_t(to_insert.second).set_size(
+	  so_get_oi(to_insert.second)->size + 1).get();
+      }
+      map.objects.insert(to_insert);
+    } else {
+      expected_error.set_clone_missing(should_exist[i].first);
+    }
+    if (GetParam().should_inject_extra(i)) {
+      map.objects.insert(make_clone(name, extra[i]));
+      expected_error.set_clone(extra[i].first);
+    }
+  }
+
+  hobject_t hoid = so_builder_t::make_hoid(name);
+  map.objects[hoid] = so_builder_t::make_head(name).get();
+
+  so_set_ss(map.objects[hoid], make_snapset(should_exist));
+
+  DoutPrefix dpp(nullptr, ceph_subsys_test, "test_crimson_scrub");
+  const auto ret = crimson::osd::scrub::validate_chunk(
+    dpp, policy, maps);
+  EXPECT_EQ(ret.object_errors.size(), 0);
+  ASSERT_EQ(ret.snapset_errors.size(), 1) << fmt::format(
+    "Got snapset_errors: {}", ret.snapset_errors);
+
+  EXPECT_TRUE(compare_error_signatures(
+    ret.snapset_errors.front(),
+    expected_error));
+
+}
+
+INSTANTIATE_TEST_SUITE_P(
+  SingleMissing,
+  TestSnapSetCloneError,
+  ::testing::ValuesIn(snapset_test_case_t::generate_single_missing_errors())
+);
+
+INSTANTIATE_TEST_SUITE_P(
+  SingleExtra,
+  TestSnapSetCloneError,
+  ::testing::ValuesIn(snapset_test_case_t::generate_single_extra_errors())
+);
+
+INSTANTIATE_TEST_SUITE_P(
+  SingleSize,
+  TestSnapSetCloneError,
+  ::testing::ValuesIn(snapset_test_case_t::generate_single_size_errors())
+);
+
+INSTANTIATE_TEST_SUITE_P(
+  MultipleRandom,
+  TestSnapSetCloneError,
+  ::testing::ValuesIn(snapset_test_case_t::generate_random_errors(100))
+);
+
+TEST(TestSnapSet, MissingHead) {
+  const pg_shard_t primary(0, shard_id_t::NO_SHARD);
+  crimson::osd::scrub::chunk_validation_policy_t policy {
+    primary,
+    std::nullopt,
+    TEST_MAX_OBJECT_SIZE,
+    std::string{TEST_INTERNAL_NAMESPACE},
+    TEST_OMAP_KEY_LIMIT,
+    TEST_OMAP_BYTES_LIMIT
+  };
+
+  crimson::osd::scrub::scrub_map_set_t maps;
+  inconsistent_snapset_wrapper expected_error;
+
+  test_clone_list_t clones = {
+    { 10, 64 }, { 25, 32 }, { 50, 64 }, { 100, 32 }
+  };
+  for (const auto &desc : test_clone_list_t{clones}) {
+    maps[primary].objects.emplace(make_clone("test_object", desc));
+  }
+  expected_error.set_headless();
+
+
+  DoutPrefix dpp(nullptr, ceph_subsys_test, "test_crimson_scrub");
+  const auto ret = crimson::osd::scrub::validate_chunk(
+    dpp, policy, maps);
+  EXPECT_EQ(ret.object_errors.size(), 0);
+  ASSERT_EQ(ret.snapset_errors.size(), 1) << fmt::format(
+    "Got snapset_errors: {}", ret.snapset_errors);
+
+  EXPECT_TRUE(compare_error_signatures(
+    ret.snapset_errors.front(),
+    expected_error));
+
+}
+
+TEST(TestSnapSet, Stats) {
+  const pg_shard_t primary(0, shard_id_t::NO_SHARD);
+  crimson::osd::scrub::chunk_validation_policy_t policy {
+    primary,
+    std::nullopt,
+    TEST_MAX_OBJECT_SIZE,
+    std::string{TEST_INTERNAL_NAMESPACE},
+    TEST_OMAP_KEY_LIMIT,
+    TEST_OMAP_BYTES_LIMIT
+  };
+
+
+  object_stat_sum_t expected_stats;
+  crimson::osd::scrub::scrub_map_set_t maps;
+  auto &objs = maps[primary].objects;
+
+  unsigned num = 0;
+  auto add_simple_head = [&](size_t size, auto &&f)
+    -> ScrubMap::object & {
+    auto name = fmt::format("obj-{}", ++num);
+    auto hoid = so_builder_t::make_hoid(name);
+    auto obj = so_builder_t::make_head(name).set_size(size).get();
+    so_mut_oi(obj, std::forward<decltype(f)>(f));
+    expected_stats.num_bytes += size;
+    expected_stats.num_objects++;
+    return objs[hoid] = obj;
+  };
+
+  add_simple_head(64, [&expected_stats](auto maybe_oi) {
+    ceph_assert(maybe_oi);
+    maybe_oi->set_flag(object_info_t::FLAG_DIRTY);
+    expected_stats.num_objects_dirty++;
+    return maybe_oi;
+  });
+
+  add_simple_head(128, [&expected_stats](auto maybe_oi) {
+    ceph_assert(maybe_oi);
+    maybe_oi->set_flag(object_info_t::FLAG_MANIFEST);
+    expected_stats.num_objects_manifest++;
+    return maybe_oi;
+  });
+
+  add_simple_head(0, [&expected_stats](auto maybe_oi) {
+    ceph_assert(maybe_oi);
+    maybe_oi->set_flag(object_info_t::FLAG_WHITEOUT);
+    expected_stats.num_whiteouts++;
+    return maybe_oi;
+  });
+
+  {
+    auto &so = add_simple_head(32, [](auto ret) { return ret; });
+    expected_stats.num_omap_keys += (so.object_omap_keys = 10);
+    expected_stats.num_omap_bytes += (so.object_omap_bytes = 100);
+    expected_stats.num_objects_omap++;
+  }
+
+  {
+    auto &so = add_simple_head(64, [](auto ret) { return ret; });
+    expected_stats.num_omap_keys +=
+      (so.object_omap_keys = (TEST_OMAP_KEY_LIMIT + 1));
+    expected_stats.num_omap_bytes +=
+      (so.object_omap_bytes = so.object_omap_keys);
+    expected_stats.num_objects_omap++;
+    expected_stats.num_large_omap_objects++;
+  }
+
+  {
+    auto &so = add_simple_head(64, [](auto ret) { return ret; });
+    expected_stats.num_omap_keys += (so.object_omap_keys = 1);
+    expected_stats.num_omap_bytes +=
+      (so.object_omap_bytes = (TEST_OMAP_BYTES_LIMIT + 1));
+    expected_stats.num_objects_omap++;
+    expected_stats.num_large_omap_objects++;
+  }
+
+  {
+    auto name = fmt::format("obj-{}", ++num);
+
+    std::map<snapid_t, interval_set<uint64_t>> clone_overlap;
+    test_clone_list_t clones;
+    auto add_clone = [&](std::pair<snapid_t, size_t> clone_desc,
+			 interval_set<uint64_t> overlap) -> ScrubMap::object & {
+      auto hoid = so_builder_t::make_hoid(name, clone_desc.first);
+      clones.push_back(clone_desc);
+      auto [_, obj] = make_clone(name, clone_desc);
+      expected_stats.num_object_clones++;
+      expected_stats.num_objects++;
+
+      expected_stats.num_bytes += clone_desc.second - overlap.size();
+      clone_overlap[clone_desc.first] = std::move(overlap);
+
+      return objs[hoid] = obj;
+    };
+
+    auto make_is = [](uint64_t off, uint64_t len) {
+      interval_set<uint64_t> ret;
+      ret.insert(off, len);
+      return ret;
+    };
+
+    add_clone({99, 32}, {});
+    add_clone({100, 64}, make_is(31, 33));
+
+    {
+      auto hoid = so_builder_t::make_hoid(name);
+      size_t size = 64;
+      auto obj = so_builder_t::make_head(name).set_size(size).get();
+      expected_stats.num_bytes += size;
+      expected_stats.num_objects++;
+
+      SnapSet ss = make_snapset(clones);
+      ss.clone_overlap = std::move(clone_overlap);
+      so_mut_ss(obj, [ss=std::move(ss)](auto) mutable {
+	return std::move(ss);
+      });
+
+      objs[hoid] = obj;
+    }
+  }
+
+  DoutPrefix dpp(nullptr, ceph_subsys_test, "test_crimson_scrub");
+  const auto ret = crimson::osd::scrub::validate_chunk(
+    dpp, policy, maps);
+  EXPECT_EQ(ret.object_errors.size(), 0);
+  ASSERT_EQ(ret.snapset_errors.size(), 0) << fmt::format(
+    "Got snapset_errors: {}", ret.snapset_errors);
+
+  EXPECT_EQ(ret.stats, expected_stats);
+}

From d75ea50d5a6463b5ddf36bbcd472481cae5daf9c Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 12 Aug 2023 00:38:40 +0000
Subject: [PATCH 1022/2492] crimson/osd: scrub integration with crimson

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/admin/pg_commands.cc              |  44 ++
 src/crimson/admin/pg_commands.h               |   2 +
 src/crimson/osd/CMakeLists.txt                |   2 +
 src/crimson/osd/ops_executer.cc               |   1 +
 src/crimson/osd/osd.cc                        |  34 +-
 src/crimson/osd/osd.h                         |   6 +-
 src/crimson/osd/osd_operation.h               |  10 +
 .../osd/osd_operation_external_tracking.h     |  13 +
 .../osd/osd_operations/client_request.cc      |  42 +-
 .../osd/osd_operations/client_request.h       |   2 +
 .../osd/osd_operations/scrub_events.cc        | 396 ++++++++++++++++++
 src/crimson/osd/osd_operations/scrub_events.h | 290 +++++++++++++
 src/crimson/osd/pg.cc                         |  14 +-
 src/crimson/osd/pg.h                          |  16 +-
 src/crimson/osd/scrub/pg_scrubber.cc          | 309 ++++++++++++++
 src/crimson/osd/scrub/pg_scrubber.h           | 152 +++++++
 16 files changed, 1302 insertions(+), 31 deletions(-)
 create mode 100644 src/crimson/osd/osd_operations/scrub_events.cc
 create mode 100644 src/crimson/osd/osd_operations/scrub_events.h
 create mode 100644 src/crimson/osd/scrub/pg_scrubber.cc
 create mode 100644 src/crimson/osd/scrub/pg_scrubber.h

diff --git a/src/crimson/admin/pg_commands.cc b/src/crimson/admin/pg_commands.cc
index f2c84b254db2..c71bd429e78a 100644
--- a/src/crimson/admin/pg_commands.cc
+++ b/src/crimson/admin/pg_commands.cc
@@ -11,9 +11,11 @@
 #include <seastar/core/future.hh>
 
 #include "crimson/admin/admin_socket.h"
+#include "crimson/common/log.h"
 #include "crimson/osd/osd.h"
 #include "crimson/osd/pg.h"
 
+SET_SUBSYS(osd);
 
 using crimson::osd::OSD;
 using crimson::osd::PG;
@@ -148,6 +150,43 @@ class MarkUnfoundLostCommand final : public PGCommand {
   }
 };
 
+template <bool deep>
+class ScrubCommand : public PGCommand {
+public:
+  explicit ScrubCommand(crimson::osd::OSD& osd) :
+    PGCommand{
+      osd,
+      deep ? "deep_scrub" : "scrub",
+      "",
+      deep ? "deep scrub pg" : "scrub pg"}
+  {}
+
+  seastar::future<tell_result_t>
+  do_command(Ref<PG> pg,
+	     const cmdmap_t& cmdmap,
+	     std::string_view format,
+	     ceph::bufferlist&&) const final
+  {
+    LOG_PREFIX(ScrubCommand::do_command);
+    DEBUGDPP("deep: {}", *pg, deep);
+    return PG::interruptor::with_interruption([pg] {
+      pg->scrubber.handle_scrub_requested(deep);
+      return PG::interruptor::now();
+    }, [FNAME, pg](std::exception_ptr ep) {
+      DEBUGDPP("interrupted with {}", *pg, ep);
+    }, pg).then([format] {
+      std::unique_ptr<Formatter> f{
+	Formatter::create(format, "json-pretty", "json-pretty")
+      };
+      f->open_object_section("scrub");
+      f->dump_bool("deep", deep);
+      f->dump_stream("stamp") << ceph_clock_now();
+      f->close_section();
+      return seastar::make_ready_future<tell_result_t>(std::move(f));
+    });
+  }
+};
+
 } // namespace crimson::admin::pg
 
 namespace crimson::admin {
@@ -164,4 +203,9 @@ make_asok_hook<crimson::admin::pg::QueryCommand>(crimson::osd::OSD& osd);
 template std::unique_ptr<AdminSocketHook>
 make_asok_hook<crimson::admin::pg::MarkUnfoundLostCommand>(crimson::osd::OSD& osd);
 
+template std::unique_ptr<AdminSocketHook>
+make_asok_hook<crimson::admin::pg::ScrubCommand<true>>(crimson::osd::OSD& osd);
+template std::unique_ptr<AdminSocketHook>
+make_asok_hook<crimson::admin::pg::ScrubCommand<false>>(crimson::osd::OSD& osd);
+
 } // namespace crimson::admin
diff --git a/src/crimson/admin/pg_commands.h b/src/crimson/admin/pg_commands.h
index 873b3c923aaf..eb7912e7aa42 100644
--- a/src/crimson/admin/pg_commands.h
+++ b/src/crimson/admin/pg_commands.h
@@ -6,5 +6,7 @@ namespace crimson::admin::pg {
 
 class QueryCommand;
 class MarkUnfoundLostCommand;
+template <bool deep>
+class ScrubCommand;
 
 }  // namespace crimson::admin::pg
diff --git a/src/crimson/osd/CMakeLists.txt b/src/crimson/osd/CMakeLists.txt
index e3ab3cf4d735..65fb7201f766 100644
--- a/src/crimson/osd/CMakeLists.txt
+++ b/src/crimson/osd/CMakeLists.txt
@@ -28,6 +28,7 @@ add_executable(crimson-osd
   osd_operations/background_recovery.cc
   osd_operations/recovery_subrequest.cc
   osd_operations/snaptrim_event.cc
+  osd_operations/scrub_events.cc
   pg_recovery.cc
   recovery_backend.cc
   replicated_recovery_backend.cc
@@ -35,6 +36,7 @@ add_executable(crimson-osd
   scheduler/mclock_scheduler.cc
   scrub/scrub_machine.cc
   scrub/scrub_validator.cc
+  scrub/pg_scrubber.cc
   osdmap_gate.cc
   pg_activation_blocker.cc
   pg_map.cc
diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 09b1a492576f..7cdc7d9027b0 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -1040,6 +1040,7 @@ std::pair<object_info_t, ObjectContextRef> OpsExecuter::prepare_clone(
 void OpsExecuter::apply_stats()
 {
   pg->get_peering_state().apply_op_stats(get_target(), delta_stats);
+  pg->scrubber.handle_op_stats(get_target(), delta_stats);
   pg->publish_stats_to_osd();
 }
 
diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index f3648c6df277..98f5f9d7ea79 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -54,6 +54,7 @@
 #include "crimson/osd/osd_operations/pg_advance_map.h"
 #include "crimson/osd/osd_operations/recovery_subrequest.h"
 #include "crimson/osd/osd_operations/replicated_request.h"
+#include "crimson/osd/osd_operations/scrub_events.h"
 #include "crimson/osd/osd_operation_external_tracking.h"
 #include "crimson/crush/CrushLocation.h"
 
@@ -644,6 +645,8 @@ seastar::future<> OSD::start_asok_admin()
     // PG commands
     asok->register_command(make_asok_hook<pg::QueryCommand>(*this));
     asok->register_command(make_asok_hook<pg::MarkUnfoundLostCommand>(*this));
+    asok->register_command(make_asok_hook<pg::ScrubCommand<true>>(*this));
+    asok->register_command(make_asok_hook<pg::ScrubCommand<false>>(*this));
     // ops commands
     asok->register_command(
       make_asok_hook<DumpInFlightOpsHook>(
@@ -819,7 +822,13 @@ OSD::do_ms_dispatch(
   case MSG_OSD_REPOPREPLY:
     return handle_rep_op_reply(conn, boost::static_pointer_cast<MOSDRepOpReply>(m));
   case MSG_OSD_SCRUB2:
-    return handle_scrub(conn, boost::static_pointer_cast<MOSDScrub2>(m));
+    return handle_scrub_command(
+      conn, boost::static_pointer_cast<MOSDScrub2>(m));
+  case MSG_OSD_REP_SCRUB:
+  case MSG_OSD_REP_SCRUBMAP:
+    return handle_scrub_message(
+      conn,
+      boost::static_pointer_cast<MOSDFastDispatchOp>(m));
   case MSG_OSD_PG_UPDATE_LOG_MISSING:
     return handle_update_log_missing(conn, boost::static_pointer_cast<
       MOSDPGUpdateLogMissing>(m));
@@ -1220,7 +1229,7 @@ seastar::future<> OSD::handle_rep_op_reply(
     });
 }
 
-seastar::future<> OSD::handle_scrub(
+seastar::future<> OSD::handle_scrub_command(
   crimson::net::ConnectionRef conn,
   Ref<MOSDScrub2> m)
 {
@@ -1230,17 +1239,22 @@ seastar::future<> OSD::handle_scrub(
   }
   return seastar::parallel_for_each(std::move(m->scrub_pgs),
     [m, conn, this](spg_t pgid) {
-    pg_shard_t from_shard{static_cast<int>(m->get_source().num()),
-                          pgid.shard};
-    PeeringState::RequestScrub scrub_request{m->deep, m->repair};
-    return pg_shard_manager.start_pg_operation<RemotePeeringEvent>(
-      conn,
-      from_shard,
-      pgid,
-      PGPeeringEvent{m->epoch, m->epoch, scrub_request}).second;
+    return pg_shard_manager.start_pg_operation<
+      crimson::osd::ScrubRequested
+      >(m->deep, conn, m->epoch, pgid).second;
   });
 }
 
+seastar::future<> OSD::handle_scrub_message(
+  crimson::net::ConnectionRef conn,
+  Ref<MOSDFastDispatchOp> m)
+{
+  ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
+  return pg_shard_manager.start_pg_operation<
+    crimson::osd::ScrubMessage
+    >(m, conn, m->get_min_epoch(), m->get_spg()).second;
+}
+
 seastar::future<> OSD::handle_mark_me_down(
   crimson::net::ConnectionRef conn,
   Ref<MOSDMarkMeDown> m)
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 134376ad947e..db30ad0ec737 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -206,8 +206,10 @@ class OSD final : public crimson::net::Dispatcher,
                                       Ref<MOSDPeeringOp> m);
   seastar::future<> handle_recovery_subreq(crimson::net::ConnectionRef conn,
                                            Ref<MOSDFastDispatchOp> m);
-  seastar::future<> handle_scrub(crimson::net::ConnectionRef conn,
-                                 Ref<MOSDScrub2> m);
+  seastar::future<> handle_scrub_command(crimson::net::ConnectionRef conn,
+					 Ref<MOSDScrub2> m);
+  seastar::future<> handle_scrub_message(crimson::net::ConnectionRef conn,
+					 Ref<MOSDFastDispatchOp> m);
   seastar::future<> handle_mark_me_down(crimson::net::ConnectionRef conn,
                                         Ref<MOSDMarkMeDown> m);
 
diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h
index 7174143fe01e..b379d4515c1d 100644
--- a/src/crimson/osd/osd_operation.h
+++ b/src/crimson/osd/osd_operation.h
@@ -54,6 +54,11 @@ enum class OperationTypeCode {
   logmissing_request_reply,
   snaptrim_event,
   snaptrimobj_subevent,
+  scrub_requested,
+  scrub_message,
+  scrub_find_range,
+  scrub_reserve_range,
+  scrub_scan,
   last_op
 };
 
@@ -71,6 +76,11 @@ static constexpr const char* const OP_NAMES[] = {
   "logmissing_request_reply",
   "snaptrim_event",
   "snaptrimobj_subevent",
+  "scrub_requested",
+  "scrub_message",
+  "scrub_find_range",
+  "scrub_reserve_range",
+  "scrub_scan",
 };
 
 // prevent the addition of OperationTypeCode-s with no matching OP_NAMES entry:
diff --git a/src/crimson/osd/osd_operation_external_tracking.h b/src/crimson/osd/osd_operation_external_tracking.h
index d5e2ed453284..5bc457c78096 100644
--- a/src/crimson/osd/osd_operation_external_tracking.h
+++ b/src/crimson/osd/osd_operation_external_tracking.h
@@ -14,6 +14,7 @@
 #include "crimson/osd/osd_operations/snaptrim_event.h"
 #include "crimson/osd/pg_activation_blocker.h"
 #include "crimson/osd/pg_map.h"
+#include "crimson/osd/scrub/pg_scrubber.h"
 
 namespace crimson::osd {
 
@@ -30,6 +31,7 @@ struct LttngBackend
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent::Backend,
     ClientRequest::PGPipeline::WaitForActive::BlockingEvent::Backend,
     PGActivationBlocker::BlockingEvent::Backend,
+    scrub::PGScrubber::BlockingEvent::Backend,
     ClientRequest::PGPipeline::RecoverMissing::BlockingEvent::Backend,
     ClientRequest::PGPipeline::GetOBC::BlockingEvent::Backend,
     ClientRequest::PGPipeline::Process::BlockingEvent::Backend,
@@ -91,6 +93,11 @@ struct LttngBackend
               const PGActivationBlocker& blocker) override {
   }
 
+  void handle(scrub::PGScrubber::BlockingEvent& ev,
+              const Operation& op,
+              const scrub::PGScrubber& blocker) override {
+  }
+
   void handle(ClientRequest::PGPipeline::RecoverMissing::BlockingEvent& ev,
               const Operation& op,
               const ClientRequest::PGPipeline::RecoverMissing& blocker) override {
@@ -136,6 +143,7 @@ struct HistoricBackend
     PG_OSDMapGate::OSDMapBlocker::BlockingEvent::Backend,
     ClientRequest::PGPipeline::WaitForActive::BlockingEvent::Backend,
     PGActivationBlocker::BlockingEvent::Backend,
+    scrub::PGScrubber::BlockingEvent::Backend,
     ClientRequest::PGPipeline::RecoverMissing::BlockingEvent::Backend,
     ClientRequest::PGPipeline::GetOBC::BlockingEvent::Backend,
     ClientRequest::PGPipeline::Process::BlockingEvent::Backend,
@@ -197,6 +205,11 @@ struct HistoricBackend
               const PGActivationBlocker& blocker) override {
   }
 
+  void handle(scrub::PGScrubber::BlockingEvent& ev,
+              const Operation& op,
+              const scrub::PGScrubber& blocker) override {
+  }
+
   void handle(ClientRequest::PGPipeline::RecoverMissing::BlockingEvent& ev,
               const Operation& op,
               const ClientRequest::PGPipeline::RecoverMissing& blocker) override {
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 62eb82a281a9..120b92ae0949 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -250,21 +250,31 @@ ClientRequest::process_op(
 	  DEBUGDPP("{}.{}: entered get_obc stage, about to wait_scrub",
 		   *pg, *this, this_instance_id);
           op_info.set_from_op(&*m, *pg->get_osdmap());
-	  return pg->with_locked_obc(
-	    m->get_hobj(), op_info,
-	    [FNAME, this, pg, this_instance_id, &ihref](
-	      auto head, auto obc) mutable {
-	      DEBUGDPP("{}.{}: got obc {}, entering process stage",
-		       *pg, *this, this_instance_id, obc->obs);
-	      return ihref.enter_stage<interruptor>(
-		client_pp(*pg).process, *this
-	      ).then_interruptible(
-		[FNAME, this, pg, this_instance_id, obc, &ihref]() mutable {
-		  DEBUGDPP("{}.{}: in process stage, calling do_process",
-			   *pg, *this, this_instance_id);
+	  return ihref.enter_blocker(
+	    *this,
+	    pg->scrubber,
+	    &decltype(pg->scrubber)::wait_scrub,
+	    m->get_hobj()
+	  ).then_interruptible(
+	    [FNAME, this, pg, this_instance_id, &ihref]() mutable {
+	      DEBUGDPP("{}.{}: past scrub blocker, getting obc",
+		       *pg, *this, this_instance_id);
+	    return pg->with_locked_obc(
+	      m->get_hobj(), op_info,
+	      [FNAME, this, pg, this_instance_id, &ihref](
+		auto head, auto obc) mutable {
+		DEBUGDPP("{}.{}: got obc {}, entering process stage",
+			 *pg, *this, this_instance_id, obc->obs);
+		return ihref.enter_stage<interruptor>(
+		  client_pp(*pg).process, *this
+		).then_interruptible(
+		  [FNAME, this, pg, this_instance_id, obc, &ihref]() mutable {
+		    DEBUGDPP("{}.{}: in process stage, calling do_process",
+			     *pg, *this, this_instance_id);
 		  return do_process(ihref, pg, obc, this_instance_id);
 		});
-	    });
+	      });
+	  });
         });
       }
     });
@@ -354,7 +364,7 @@ ClientRequest::do_process(
     [FNAME, this, pg, this_instance_id, &ihref](
       auto submitted, auto all_completed) mutable {
       return submitted.then_interruptible(
-	[FNAME, this, pg, this_instance_id, &ihref] {
+	[this, pg, &ihref] {
 	return ihref.enter_stage<interruptor>(client_pp(*pg).wait_repop, *this);
       }).then_interruptible(
 	[FNAME, this, pg, this_instance_id,
@@ -371,12 +381,12 @@ ClientRequest::do_process(
 		  return conn->send(std::move(reply));
 		});
 	    }, crimson::ct_error::eagain::handle(
-	      [FNAME, this, pg, this_instance_id, &ihref]() mutable {
+	      [this, pg, this_instance_id, &ihref]() mutable {
 		return process_op(ihref, pg, this_instance_id);
 	    }));
 	});
     }, crimson::ct_error::eagain::handle(
-      [FNAME, this, pg, this_instance_id, &ihref]() mutable {
+      [this, pg, this_instance_id, &ihref]() mutable {
 	return process_op(ihref, pg, this_instance_id);
       }));
 }
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 6599e4dbc6ab..ac4c46981c3c 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -17,6 +17,7 @@
 #include "crimson/osd/osd_operations/common/pg_pipeline.h"
 #include "crimson/osd/pg_activation_blocker.h"
 #include "crimson/osd/pg_map.h"
+#include "crimson/osd/scrub/pg_scrubber.h"
 #include "crimson/common/type_helpers.h"
 #include "crimson/common/utility.h"
 #include "messages/MOSDOp.h"
@@ -103,6 +104,7 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
       PGPipeline::WaitForActive::BlockingEvent,
       PGActivationBlocker::BlockingEvent,
       PGPipeline::RecoverMissing::BlockingEvent,
+      scrub::PGScrubber::BlockingEvent,
       PGPipeline::GetOBC::BlockingEvent,
       PGPipeline::Process::BlockingEvent,
       PGPipeline::WaitRepop::BlockingEvent,
diff --git a/src/crimson/osd/osd_operations/scrub_events.cc b/src/crimson/osd/osd_operations/scrub_events.cc
new file mode 100644
index 000000000000..4f54cf0b274f
--- /dev/null
+++ b/src/crimson/osd/osd_operations/scrub_events.cc
@@ -0,0 +1,396 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "crimson/common/log.h"
+#include "crimson/osd/pg.h"
+#include "crimson/osd/osd_connection_priv.h"
+#include "messages/MOSDRepScrubMap.h"
+#include "scrub_events.h"
+
+SET_SUBSYS(osd);
+
+namespace crimson::osd {
+
+template <class T>
+PGPeeringPipeline &RemoteScrubEventBaseT<T>::get_peering_pipeline(PG &pg)
+{
+  return pg.peering_request_pg_pipeline;
+}
+
+template <class T>
+ConnectionPipeline &RemoteScrubEventBaseT<T>::get_connection_pipeline()
+{
+  return get_osd_priv(conn.get()).peering_request_conn_pipeline;
+}
+
+template <class T>
+PerShardPipeline &RemoteScrubEventBaseT<T>::get_pershard_pipeline(
+  ShardServices &shard_services)
+{
+  return shard_services.get_client_request_pipeline();
+}
+
+template <class T>
+seastar::future<> RemoteScrubEventBaseT<T>::with_pg(
+  ShardServices &shard_services, Ref<PG> pg)
+{
+  LOG_PREFIX(RemoteEventBaseT::with_pg);
+  return interruptor::with_interruption([FNAME, this, pg] {
+    DEBUGDPP("{} pg present", *pg, *that());
+    return this->template enter_stage<interruptor>(
+      get_peering_pipeline(*pg).await_map
+    ).then_interruptible([this, pg] {
+      return this->template with_blocking_event<
+	PG_OSDMapGate::OSDMapBlocker::BlockingEvent
+	>([this, pg](auto &&trigger) {
+	  return pg->osdmap_gate.wait_for_map(
+	    std::move(trigger), get_epoch());
+	});
+    }).then_interruptible([this, pg](auto) {
+      return this->template enter_stage<interruptor>(
+	get_peering_pipeline(*pg).process);
+    }).then_interruptible([this, pg] {
+      return handle_event(*pg);
+    });
+  }, [FNAME, pg, this](std::exception_ptr ep) {
+    DEBUGDPP("{} interrupted with {}", *pg, *that(), ep);
+  }, pg);
+}
+
+ScrubRequested::ifut<> ScrubRequested::handle_event(PG &pg)
+{
+  pg.scrubber.handle_scrub_requested(deep);
+  return seastar::now();
+}
+
+ScrubMessage::ifut<> ScrubMessage::handle_event(PG &pg)
+{
+  pg.scrubber.handle_scrub_message(*m);
+  return seastar::now();
+}
+
+template class RemoteScrubEventBaseT<ScrubRequested>;
+template class RemoteScrubEventBaseT<ScrubMessage>;
+
+template <typename T>
+ScrubAsyncOpT<T>::ScrubAsyncOpT(Ref<PG> pg) : pg(pg) {}
+
+template <typename T>
+typename ScrubAsyncOpT<T>::template ifut<> ScrubAsyncOpT<T>::start()
+{
+  LOG_PREFIX(ScrubAsyncOpT::start);
+  DEBUGDPP("{} starting", *pg, *this);
+  return run(*pg);
+}
+
+ScrubFindRange::ifut<> ScrubFindRange::run(PG &pg)
+{
+  LOG_PREFIX(ScrubFindRange::run);
+  using crimson::common::local_conf;
+  return interruptor::make_interruptible(
+    pg.shard_services.get_store().list_objects(
+      pg.get_collection_ref(),
+      ghobject_t(begin, ghobject_t::NO_GEN, pg.get_pgid().shard),
+      ghobject_t::get_max(),
+      local_conf().get_val<int64_t>("osd_scrub_chunk_max")
+    )
+  ).then_interruptible([FNAME, this, &pg](auto ret) {
+    auto &[_, next] = ret;
+
+    // We rely on seeing an entire set of snapshots in a single chunk
+    auto end = next.hobj.get_max_object_boundary();
+
+    DEBUGDPP("got next.hobj: {}, returning begin, end: {}, {}",
+	     pg, next.hobj, begin, end);
+    pg.scrubber.machine.process_event(
+      scrub::ScrubContext::request_range_complete_t{begin, end});
+  });
+}
+
+template class ScrubAsyncOpT<ScrubFindRange>;
+
+ScrubReserveRange::ifut<> ScrubReserveRange::run(PG &pg)
+{
+  LOG_PREFIX(ScrubReserveRange::run);
+  DEBUGDPP("", pg);
+  return pg.background_process_lock.lock(
+  ).then_interruptible([FNAME, this, &pg] {
+    DEBUGDPP("pg_background_io_mutex locked", pg);
+    auto &scrubber = pg.scrubber;
+    ceph_assert(!scrubber.blocked);
+    scrubber.blocked = scrub::blocked_range_t{begin, end};
+    blocked_set = true;
+    auto& log = pg.peering_state.get_pg_log().get_log().log;
+    auto p = find_if(
+      log.crbegin(), log.crend(),
+      [this](const auto& e) -> bool {
+	return e.soid >= begin && e.soid < end;
+      });
+
+    if (p == log.crend()) {
+      return scrubber.machine.process_event(
+	scrub::ScrubContext::reserve_range_complete_t{eversion_t{}});
+    } else {
+      return scrubber.machine.process_event(
+	scrub::ScrubContext::reserve_range_complete_t{p->version});
+    }
+  }).finally([&pg, this] {
+    if (!blocked_set) {
+      pg.background_process_lock.unlock();
+    }
+  });
+}
+
+template class ScrubAsyncOpT<ScrubReserveRange>;
+
+ScrubScan::ifut<> ScrubScan::run(PG &pg)
+{
+  LOG_PREFIX(ScrubScan::start);
+  // legacy value, unused
+  ret.valid_through = pg.get_info().last_update;
+
+  DEBUGDPP("begin: {}, end: {}", pg, begin, end);
+  return interruptor::make_interruptible(
+    pg.shard_services.get_store().list_objects(
+      pg.get_collection_ref(),
+      ghobject_t(begin, ghobject_t::NO_GEN, pg.get_pgid().shard),
+      ghobject_t(end, ghobject_t::NO_GEN, pg.get_pgid().shard),
+      std::numeric_limits<uint64_t>::max())
+  ).then_interruptible([FNAME, this, &pg](auto &&result) {
+    DEBUGDPP("listed {} objects", pg, std::get<0>(result).size());
+    return seastar::do_with(
+      std::move(std::get<0>(result)),
+      [this, &pg](auto &objects) {
+	return interruptor::do_for_each(
+	  objects,
+	  [this, &pg](auto &obj) {
+	    if (obj.is_pgmeta() || obj.hobj.is_temp()) {
+	      return interruptor::now();
+	    } else {
+	      return scan_object(pg, obj);
+	    }
+	  });
+      });
+  }).then_interruptible([FNAME, this, &pg] {
+    if (local) {
+      DEBUGDPP("complete, submitting local event", pg);
+      pg.scrubber.handle_event(
+	scrub::ScrubContext::scan_range_complete_t(
+	  pg.get_pg_whoami(),
+	  std::move(ret)));
+      return seastar::now();
+    } else {
+      DEBUGDPP("complete, sending response to primary", pg);
+      auto m = crimson::make_message<MOSDRepScrubMap>(
+	spg_t(pg.get_pgid().pgid, pg.get_primary().shard),
+	pg.get_osdmap_epoch(),
+	pg.get_pg_whoami());
+      encode(ret, m->get_data());
+      pg.scrubber.handle_event(
+	scrub::ScrubContext::generate_and_submit_chunk_result_complete_t{});
+      return pg.shard_services.send_to_osd(
+	pg.get_primary().osd,
+	std::move(m),
+	pg.get_osdmap_epoch());
+    }
+  });
+}
+
+ScrubScan::ifut<> ScrubScan::scan_object(
+  PG &pg,
+  const ghobject_t &obj)
+{
+  LOG_PREFIX(ScrubScan::scan_object);
+  DEBUGDPP("obj: {}", pg, obj);
+  auto &entry = ret.objects[obj.hobj];
+  return interruptor::make_interruptible(
+    pg.shard_services.get_store().stat(
+      pg.get_collection_ref(),
+      obj)
+  ).then_interruptible([FNAME, &pg, &obj, &entry](struct stat obj_stat) {
+    DEBUGDPP("obj: {}, stat complete, size {}", pg, obj, obj_stat.st_size);
+    entry.size = obj_stat.st_size;
+    return pg.shard_services.get_store().get_attrs(
+      pg.get_collection_ref(),
+      obj);
+  }).safe_then_interruptible([FNAME, &pg, &obj, &entry](auto &&attrs) {
+    DEBUGDPP("obj: {}, got {} attrs", pg, obj, attrs.size());
+    for (auto &i : attrs) {
+      i.second.rebuild();
+      if (i.second.length() == 0) {
+	entry.attrs[i.first];
+      } else {
+	entry.attrs.emplace(i.first, i.second.front());
+      }
+    }
+  }).handle_error_interruptible(
+    ct_error::all_same_way([FNAME, &pg, &obj, &entry](auto e) {
+      DEBUGDPP("obj: {} stat error", pg, obj);
+      entry.stat_error = true;
+    })
+  ).then_interruptible([FNAME, this, &pg, &obj] {
+    if (deep) {
+      DEBUGDPP("obj: {} doing deep scan", pg, obj);
+      return deep_scan_object(pg, obj);
+    } else {
+      return interruptor::now();
+    }
+  });
+
+}
+
+struct obj_scrub_progress_t {
+  // nullopt once complete
+  std::optional<uint64_t> offset = 0;
+  ceph::buffer::hash data_hash{std::numeric_limits<uint32_t>::max()};
+
+  bool header_done = false;
+  std::optional<std::string> next_key;
+  bool keys_done = false;
+  ceph::buffer::hash omap_hash{std::numeric_limits<uint32_t>::max()};
+};
+ScrubScan::ifut<> ScrubScan::deep_scan_object(
+  PG &pg,
+  const ghobject_t &obj)
+{
+  LOG_PREFIX(ScrubScan::deep_scan_object);
+  DEBUGDPP("obj: {}", pg, obj);
+  using crimson::common::local_conf;
+  auto &entry = ret.objects[obj.hobj];
+  return interruptor::repeat(
+    [FNAME, this, progress = obj_scrub_progress_t(),
+     &obj, &entry, &pg]() mutable
+    -> interruptible_future<seastar::stop_iteration> {
+      if (progress.offset) {
+	DEBUGDPP("op: {}, obj: {}, progress: {} scanning data",
+		 pg, *this, obj, progress);
+	const auto stride = local_conf().get_val<Option::size_t>(
+	  "osd_deep_scrub_stride");
+	return pg.shard_services.get_store().read(
+	  pg.get_collection_ref(),
+	  obj,
+	  *(progress.offset),
+	  stride
+	).safe_then([stride, &progress, &entry](auto bl) {
+	  progress.data_hash << bl;
+	  if (bl.length() < stride) {
+	    progress.offset = std::nullopt;
+	    entry.digest = progress.data_hash.digest();
+	    entry.digest_present = true;
+	  } else {
+	    ceph_assert(stride == bl.length());
+	    *(progress.offset) += stride;
+	  }
+	}).handle_error(
+	  ct_error::all_same_way([&progress, &entry](auto e) {
+	    entry.read_error = true;
+	    progress.offset = std::nullopt;
+	  })
+	).then([] {
+	  return interruptor::make_interruptible(
+	    seastar::make_ready_future<seastar::stop_iteration>(
+	      seastar::stop_iteration::no));
+	});
+      } else if (!progress.header_done) {
+	DEBUGDPP("op: {}, obj: {}, progress: {} scanning omap header",
+		 pg, *this, obj, progress);
+	return pg.shard_services.get_store().omap_get_header(
+	  pg.get_collection_ref(),
+	  obj
+	).safe_then([&progress](auto bl) {
+	  progress.omap_hash << bl;
+	}).handle_error(
+	  ct_error::enodata::handle([] {}),
+	  ct_error::all_same_way([&entry](auto e) {
+	    entry.read_error = true;
+	  })
+	).then([&progress] {
+	  progress.header_done = true;
+	  return interruptor::make_interruptible(
+	    seastar::make_ready_future<seastar::stop_iteration>(
+	      seastar::stop_iteration::no));
+	});
+      } else if (!progress.keys_done) {
+	DEBUGDPP("op: {}, obj: {}, progress: {} scanning omap keys",
+		 pg, *this, obj, progress);
+	return pg.shard_services.get_store().omap_get_values(
+	  pg.get_collection_ref(),
+	  obj,
+	  progress.next_key
+	).safe_then([FNAME, this, &obj, &progress, &entry, &pg](auto result) {
+	  const auto &[done, omap] = result;
+	  DEBUGDPP("op: {}, obj: {}, progress: {} got {} keys",
+		   pg, *this, obj, progress, omap.size());
+	  for (const auto &p : omap) {
+	    bufferlist bl;
+	    encode(p.first, bl);
+	    encode(p.second, bl);
+	    progress.omap_hash << bl;
+	    entry.object_omap_keys++;
+	    entry.object_omap_bytes += p.second.length();
+	  }
+	  if (done) {
+	    DEBUGDPP("op: {}, obj: {}, progress: {} omap done",
+		     pg, *this, obj, progress);
+	    progress.keys_done = true;
+	    entry.omap_digest = progress.omap_hash.digest();
+	    entry.omap_digest_present = true;
+
+	    if ((entry.object_omap_keys >
+		 local_conf().get_val<uint64_t>(
+		   "osd_deep_scrub_large_omap_object_key_threshold")) ||
+		(entry.object_omap_bytes >
+		 local_conf().get_val<Option::size_t>(
+		   "osd_deep_scrub_large_omap_object_value_sum_threshold"))) {
+	      entry.large_omap_object_found = true;
+	      entry.large_omap_object_key_count = entry.object_omap_keys;
+	      ret.has_large_omap_object_errors = true;
+	    }
+	  } else {
+	    ceph_assert(!omap.empty()); // omap_get_values invariant
+	    DEBUGDPP("op: {}, obj: {}, progress: {} omap not done, next {}",
+		     pg, *this, obj, progress, omap.crbegin()->first);
+	    progress.next_key = omap.crbegin()->first;
+	  }
+	}).handle_error(
+	  ct_error::all_same_way([FNAME, this, &obj, &progress, &entry, &pg]
+				 (auto e) {
+	    DEBUGDPP("op: {}, obj: {}, progress: {} error reading omap {}",
+		     pg, *this, obj, progress, e);
+	    progress.keys_done = true;
+	    entry.read_error = true;
+	  })
+	).then([] {
+	  return interruptor::make_interruptible(
+	    seastar::make_ready_future<seastar::stop_iteration>(
+	      seastar::stop_iteration::no));
+	});
+      } else {
+	DEBUGDPP("op: {}, obj: {}, progress: {} done",
+		 pg, *this, obj, progress);
+	return interruptor::make_interruptible(
+	  seastar::make_ready_future<seastar::stop_iteration>(
+	    seastar::stop_iteration::yes));
+      }
+    });
+}
+
+template class ScrubAsyncOpT<ScrubScan>;
+
+}
+
+template <>
+struct fmt::formatter<crimson::osd::obj_scrub_progress_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+  template <typename FormatContext>
+  auto format(const crimson::osd::obj_scrub_progress_t &progress,
+	      FormatContext& ctx)
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "obj_scrub_progress_t(offset: {}, "
+      "header_done: {}, next_key: {}, keys_done: {})",
+      progress.offset, progress.header_done,
+      progress.next_key, progress.keys_done);
+  }
+};
diff --git a/src/crimson/osd/osd_operations/scrub_events.h b/src/crimson/osd/osd_operations/scrub_events.h
new file mode 100644
index 000000000000..0793983d8c66
--- /dev/null
+++ b/src/crimson/osd/osd_operations/scrub_events.h
@@ -0,0 +1,290 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include "common/Formatter.h"
+#include "crimson/osd/osd_operation.h"
+#include "crimson/osd/scrub/pg_scrubber.h"
+#include "osd/osd_types.h"
+#include "peering_event.h"
+
+namespace crimson::osd {
+
+class PG;
+
+template <typename T>
+class RemoteScrubEventBaseT : public PhasedOperationT<T> {
+  T* that() {
+    return static_cast<T*>(this);
+  }
+  const T* that() const {
+    return static_cast<const T*>(this);
+  }
+
+  PipelineHandle handle;
+
+  crimson::net::ConnectionRef conn;
+  epoch_t epoch;
+  spg_t pgid;
+
+protected:
+  using interruptor = InterruptibleOperation::interruptor;
+
+  template <typename U=void>
+  using ifut = InterruptibleOperation::interruptible_future<U>;
+
+  virtual ifut<> handle_event(PG &pg) = 0;
+public:
+  RemoteScrubEventBaseT(
+    crimson::net::ConnectionRef conn, epoch_t epoch, spg_t pgid)
+    : conn(conn), epoch(epoch), pgid(pgid) {}
+
+  PGPeeringPipeline &get_peering_pipeline(PG &pg);
+  ConnectionPipeline &get_connection_pipeline();
+  PerShardPipeline &get_pershard_pipeline(ShardServices &);
+
+  crimson::net::Connection &get_connection() {
+    assert(conn);
+    return *conn;
+  };
+
+  static constexpr bool can_create() { return false; }
+
+  spg_t get_pgid() const {
+    return pgid;
+  }
+
+  PipelineHandle &get_handle() { return handle; }
+  epoch_t get_epoch() const { return epoch; }
+
+  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
+    assert(conn);
+    return conn.get_foreign(
+    ).then([this](auto f_conn) {
+      conn.reset();
+      return f_conn;
+    });
+  }
+  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
+    assert(!conn);
+    conn = make_local_shared_foreign(std::move(_conn));
+  }
+
+  seastar::future<> with_pg(
+    ShardServices &shard_services, Ref<PG> pg);
+
+  std::tuple<
+    class TrackableOperationT<T>::StartEvent,
+    ConnectionPipeline::AwaitActive::BlockingEvent,
+    ConnectionPipeline::AwaitMap::BlockingEvent,
+    OSD_OSDMapGate::OSDMapBlocker::BlockingEvent,
+    ConnectionPipeline::GetPGMapping::BlockingEvent,
+    PerShardPipeline::CreateOrWaitPG::BlockingEvent,
+    PGMap::PGCreationBlockingEvent,
+    PGPeeringPipeline::AwaitMap::BlockingEvent,
+    PG_OSDMapGate::OSDMapBlocker::BlockingEvent,
+    PGPeeringPipeline::Process::BlockingEvent,
+    class TrackableOperationT<T>::CompletionEvent
+  > tracking_events;
+
+  virtual ~RemoteScrubEventBaseT() = default;
+};
+
+class ScrubRequested final : public RemoteScrubEventBaseT<ScrubRequested> {
+  bool deep = false;
+protected:
+  ifut<> handle_event(PG &pg) final;
+
+public:
+  static constexpr OperationTypeCode type = OperationTypeCode::scrub_requested;
+
+  template <typename... Args>
+  ScrubRequested(bool deep, Args&&... base_args)
+    : RemoteScrubEventBaseT<ScrubRequested>(std::forward<Args>(base_args)...),
+      deep(deep) {}
+
+  void print(std::ostream &out) const final {
+    out << "(deep=" << deep << ")";
+  }
+  void dump_detail(ceph::Formatter *f) const final {
+    f->dump_bool("deep", deep);
+  }
+
+};
+
+class ScrubMessage final : public RemoteScrubEventBaseT<ScrubMessage> {
+  MessageRef m;
+protected:
+  ifut<> handle_event(PG &pg) final;
+
+public:
+  static constexpr OperationTypeCode type = OperationTypeCode::scrub_message;
+
+  template <typename... Args>
+  ScrubMessage(MessageRef m, Args&&... base_args)
+    : RemoteScrubEventBaseT<ScrubMessage>(std::forward<Args>(base_args)...),
+      m(m) {
+    ceph_assert(scrub::PGScrubber::is_scrub_message(*m));
+  }
+
+  void print(std::ostream &out) const final {
+    out << "(m=" << *m << ")";
+  }
+  void dump_detail(ceph::Formatter *f) const final {
+    f->dump_stream("m") << *m;
+  }
+
+};
+
+template <typename T>
+class ScrubAsyncOpT : public TrackableOperationT<T> {
+  Ref<PG> pg;
+
+public:
+  using interruptor = InterruptibleOperation::interruptor;
+  template <typename U=void>
+  using ifut = InterruptibleOperation::interruptible_future<U>;
+
+  ScrubAsyncOpT(Ref<PG> pg);
+
+  ifut<> start();
+
+  virtual ~ScrubAsyncOpT() = default;
+
+protected:
+  virtual ifut<> run(PG &pg) = 0;
+};
+
+class ScrubFindRange : public ScrubAsyncOpT<ScrubFindRange> {
+  hobject_t begin;
+public:
+  static constexpr OperationTypeCode type = OperationTypeCode::scrub_find_range;
+
+  template <typename... Args>
+  ScrubFindRange(const hobject_t &begin, Args&&... args)
+    : ScrubAsyncOpT(std::forward<Args>(args)...), begin(begin) {}
+
+  void print(std::ostream &out) const final {
+    out << "(begin=" << begin << ")";
+  }
+  void dump_detail(ceph::Formatter *f) const final {
+    f->dump_stream("begin") << begin;
+  }
+
+
+protected:
+  ifut<> run(PG &pg) final;
+};
+
+class ScrubReserveRange : public ScrubAsyncOpT<ScrubReserveRange> {
+  hobject_t begin;
+  hobject_t end;
+
+  /// see run(), used to unlock background_io_mutex on interval change
+  bool blocked_set = false;
+public:
+  static constexpr OperationTypeCode type =
+    OperationTypeCode::scrub_reserve_range;
+
+  template <typename... Args>
+  ScrubReserveRange(const hobject_t &begin, const hobject_t &end, Args&&... args)
+    : ScrubAsyncOpT(std::forward<Args>(args)...), begin(begin), end(end) {}
+
+  void print(std::ostream &out) const final {
+    out << "(begin=" << begin << ", end=" << end << ")";
+  }
+  void dump_detail(ceph::Formatter *f) const final {
+    f->dump_stream("begin") << begin;
+    f->dump_stream("end") << end;
+  }
+
+
+protected:
+  ifut<> run(PG &pg) final;
+};
+
+class ScrubScan : public ScrubAsyncOpT<ScrubScan> {
+  /// deep or shallow scrub
+  const bool deep;
+
+  /// true: send event locally, false: send result to primary
+  const bool local;
+
+  /// object range to scan: [begin, end)
+  const hobject_t begin;
+  const hobject_t end;
+
+  /// result, see local
+  ScrubMap ret;
+
+  ifut<> scan_object(PG &pg, const ghobject_t &obj);
+  ifut<> deep_scan_object(PG &pg, const ghobject_t &obj);
+
+public:
+  static constexpr OperationTypeCode type = OperationTypeCode::scrub_scan;
+
+  void print(std::ostream &out) const final {
+    out << "(deep=" << deep
+	<< ", local=" << local
+	<< ", begin=" << begin
+	<< ", end=" << end
+	<< ")";
+  }
+  void dump_detail(ceph::Formatter *f) const final {
+    f->dump_bool("deep", deep);
+    f->dump_bool("local", local);
+    f->dump_stream("begin") << begin;
+    f->dump_stream("end") << end;
+  }
+
+  ScrubScan(
+    Ref<PG> pg, bool deep, bool local,
+    const hobject_t &begin, const hobject_t &end)
+    : ScrubAsyncOpT(pg), deep(deep), local(local), begin(begin), end(end) {}
+
+protected:
+  ifut<> run(PG &pg) final;
+};
+
+}
+
+namespace crimson {
+
+template <>
+struct EventBackendRegistry<osd::ScrubRequested> {
+  static std::tuple<> get_backends() {
+    return {};
+  }
+};
+
+template <>
+struct EventBackendRegistry<osd::ScrubMessage> {
+  static std::tuple<> get_backends() {
+    return {};
+  }
+};
+
+}
+
+#if FMT_VERSION >= 90000
+template <> struct fmt::formatter<crimson::osd::ScrubRequested>
+  : fmt::ostream_formatter {};
+
+template <> struct fmt::formatter<crimson::osd::ScrubMessage>
+  : fmt::ostream_formatter {};
+
+template <typename T>
+struct fmt::formatter<crimson::osd::ScrubAsyncOpT<T>>
+  : fmt::ostream_formatter {};
+
+template <> struct fmt::formatter<crimson::osd::ScrubFindRange>
+  : fmt::ostream_formatter {};
+
+template <> struct fmt::formatter<crimson::osd::ScrubReserveRange>
+  : fmt::ostream_formatter {};
+
+template <> struct fmt::formatter<crimson::osd::ScrubScan>
+  : fmt::ostream_formatter {};
+
+#endif
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 5b7f1fd25781..6537eed2b7d5 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -125,6 +125,7 @@ PG::PG(
       osdmap,
       this,
       this),
+    scrubber(*this),
     obc_registry{
       local_conf()},
     obc_loader{
@@ -144,6 +145,7 @@ PG::PG(
       pgid.shard),
     wait_for_active_blocker(this)
 {
+  scrubber.initiate();
   peering_state.set_backend_predicates(
     new ReadablePredicate(pg_whoami),
     new RecoverablePredicate());
@@ -331,6 +333,12 @@ void PG::on_activate(interval_set<snapid_t> snaps)
   projected_last_update = peering_state.get_info().last_update;
 }
 
+void PG::on_replica_activate()
+{
+  logger().debug("{}: {}", *this, __func__);
+  scrubber.on_replica_activate();
+}
+
 void PG::on_activate_complete()
 {
   wait_for_active_blocker.unblock();
@@ -458,7 +466,7 @@ PG::do_delete_work(ceph::os::Transaction &t, ghobject_t _next)
 
 Context *PG::on_clean()
 {
-  // Not needed yet (will be needed for IO unblocking)
+  scrubber.on_primary_active_clean();
   return nullptr;
 }
 
@@ -1347,6 +1355,9 @@ void PG::log_operation(
   if (!is_primary()) { // && !is_ec_pg()
     replica_clear_repop_obc(logv);
   }
+  if (!logv.empty()) {
+    scrubber.on_log_update(logv.rbegin()->version);
+  }
   peering_state.append_log(std::move(logv),
                            trim_to,
                            roll_forward_to,
@@ -1527,6 +1538,7 @@ void PG::on_change(ceph::os::Transaction &t) {
     logger().debug("{} {}: dropping requests", *this, __func__);
     client_request_orderer.clear_and_cancel(*this);
   }
+  scrubber.on_interval_change();
 }
 
 void PG::context_registry_on_change() {
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 7f734ecd5e77..80a181f24a23 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -39,6 +39,7 @@
 #include "crimson/osd/pg_recovery_listener.h"
 #include "crimson/osd/recovery_backend.h"
 #include "crimson/osd/object_context_loader.h"
+#include "crimson/osd/scrub/pg_scrubber.h"
 
 class MQuery;
 class OSDMap;
@@ -160,8 +161,6 @@ class PG : public boost::intrusive_ref_counter<
     bool need_write_epoch,
     ceph::os::Transaction &t) final;
 
-  void scrub_requested(scrub_level_t scrub_level, scrub_type_t scrub_type) final;
-
   uint64_t get_snap_trimq_size() const final {
     return std::size(snap_trimq);
   }
@@ -318,6 +317,7 @@ class PG : public boost::intrusive_ref_counter<
   }
   void on_change(ceph::os::Transaction &t) final;
   void on_activate(interval_set<snapid_t> to_trim) final;
+  void on_replica_activate() final;
   void on_activate_complete() final;
   void on_new_interval() final {
     // Not needed yet
@@ -627,6 +627,18 @@ class PG : public boost::intrusive_ref_counter<
   eversion_t projected_last_update;
 
 public:
+  // scrub state
+
+  friend class ScrubScan;
+  friend class ScrubFindRange;
+  friend class ScrubReserveRange;
+  friend class scrub::PGScrubber;
+  template <typename T> friend class RemoteScrubEventBaseT;
+
+  scrub::PGScrubber scrubber;
+
+  void scrub_requested(scrub_level_t scrub_level, scrub_type_t scrub_type) final;
+
   ObjectContextRegistry obc_registry;
   ObjectContextLoader obc_loader;
 
diff --git a/src/crimson/osd/scrub/pg_scrubber.cc b/src/crimson/osd/scrub/pg_scrubber.cc
new file mode 100644
index 000000000000..3ac2c910db2e
--- /dev/null
+++ b/src/crimson/osd/scrub/pg_scrubber.cc
@@ -0,0 +1,309 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 smarttab expandtab
+
+#include <fmt/ranges.h>
+
+#include "crimson/common/log.h"
+#include "crimson/osd/pg.h"
+#include "crimson/osd/osd_operations/scrub_events.h"
+#include "messages/MOSDRepScrub.h"
+#include "messages/MOSDRepScrubMap.h"
+#include "pg_scrubber.h"
+
+SET_SUBSYS(osd);
+
+namespace crimson::osd::scrub {
+
+void PGScrubber::dump_detail(Formatter *f) const
+{
+  f->dump_stream("pgid") << pg.get_pgid();
+}
+
+PGScrubber::PGScrubber(PG &pg) : pg(pg), dpp(pg), machine(*this) {}
+
+void PGScrubber::on_primary_active_clean()
+{
+  LOG_PREFIX(PGScrubber::on_primary_active_clean);
+  DEBUGDPP("", pg);
+  handle_event(events::primary_activate_t{});
+}
+
+void PGScrubber::on_replica_activate()
+{
+  LOG_PREFIX(PGScrubber::on_replica_activate);
+  DEBUGDPP("", pg);
+  handle_event(events::replica_activate_t{});
+}
+
+void PGScrubber::on_interval_change()
+{
+  LOG_PREFIX(PGScrubber::on_interval_change);
+  DEBUGDPP("", pg);
+  /* Once reservations and scheduling are introduced, we'll need an
+   * IntervalChange event to drop remote resources (they'll be automatically
+   * released on the other side) */
+  handle_event(events::reset_t{});
+  waiting_for_update = std::nullopt;
+  ceph_assert(!blocked);
+}
+
+void PGScrubber::on_log_update(eversion_t v)
+{
+  LOG_PREFIX(PGScrubber::on_interval_change);
+  if (waiting_for_update && v >= *waiting_for_update) {
+    DEBUGDPP("waiting_for_update: {}, v: {}", pg, *waiting_for_update, v);
+    handle_event(await_update_complete_t{});
+    waiting_for_update = std::nullopt;
+  }
+}
+
+void PGScrubber::handle_scrub_requested(bool deep)
+{
+  LOG_PREFIX(PGScrubber::handle_scrub_requested);
+  DEBUGDPP("deep: {}", pg, deep);
+  handle_event(events::start_scrub_t{deep});
+}
+
+void PGScrubber::handle_scrub_message(Message &_m)
+{
+  LOG_PREFIX(PGScrubber::handle_scrub_requested);
+  switch (_m.get_type()) {
+  case MSG_OSD_REP_SCRUB: {
+    MOSDRepScrub &m = *static_cast<MOSDRepScrub*>(&_m);
+    DEBUGDPP("MOSDRepScrub: {}", pg, m);
+    handle_event(events::replica_scan_t{
+	m.start, m.end, m.scrub_from, m.deep
+      });
+    break;
+  }
+  case MSG_OSD_REP_SCRUBMAP: {
+    MOSDRepScrubMap &m = *static_cast<MOSDRepScrubMap*>(&_m);
+    DEBUGDPP("MOSDRepScrubMap: {}", pg, m);
+    ScrubMap map;
+    auto iter = m.get_data().cbegin();
+    ::decode(map, iter);
+    handle_event(scan_range_complete_t{
+	m.from, std::move(map)
+      });
+    break;
+  }
+  default:
+    DEBUGDPP("invalid message: {}", pg, _m);
+    ceph_assert(is_scrub_message(_m));
+  }
+}
+
+void PGScrubber::handle_op_stats(
+  const hobject_t &on_object,
+  object_stat_sum_t delta_stats) {
+  handle_event(events::op_stats_t{on_object, delta_stats});
+}
+
+PGScrubber::ifut<> PGScrubber::wait_scrub(
+  PGScrubber::BlockingEvent::TriggerI&& trigger,
+  const hobject_t &hoid)
+{
+  LOG_PREFIX(PGScrubber::wait_scrub);
+  if (blocked && (hoid >= blocked->begin) && (hoid < blocked->end)) {
+    DEBUGDPP("blocked: {}, hoid: {}", pg, *blocked, hoid);
+    return trigger.maybe_record_blocking(
+      blocked->p.get_shared_future(),
+      *this);
+  } else {
+    return seastar::now();
+  }
+}
+
+void PGScrubber::notify_scrub_start(bool deep)
+{
+  LOG_PREFIX(PGScrubber::notify_scrub_start);
+  DEBUGDPP("deep: {}", pg, deep);
+  pg.peering_state.state_set(PG_STATE_SCRUBBING);
+  if (deep) {
+    pg.peering_state.state_set(PG_STATE_DEEP_SCRUB);
+  }
+  pg.publish_stats_to_osd();
+}
+
+void PGScrubber::notify_scrub_end(bool deep)
+{
+  LOG_PREFIX(PGScrubber::notify_scrub_end);
+  DEBUGDPP("deep: {}", pg, deep);
+  pg.peering_state.state_clear(PG_STATE_SCRUBBING);
+  if (deep) {
+    pg.peering_state.state_clear(PG_STATE_DEEP_SCRUB);
+  }
+  pg.publish_stats_to_osd();
+}
+
+const std::set<pg_shard_t> &PGScrubber::get_ids_to_scrub() const
+{
+  return pg.peering_state.get_actingset();
+}
+
+chunk_validation_policy_t PGScrubber::get_policy() const
+{
+  return chunk_validation_policy_t{
+    pg.get_primary(),
+    std::nullopt /* stripe_info, populate when EC is implemented */,
+    crimson::common::local_conf().get_val<Option::size_t>(
+      "osd_max_object_size"),
+    crimson::common::local_conf().get_val<std::string>(
+      "osd_hit_set_namespace"),
+    crimson::common::local_conf().get_val<Option::size_t>(
+      "osd_deep_scrub_large_omap_object_value_sum_threshold"),
+    crimson::common::local_conf().get_val<uint64_t>(
+      "osd_deep_scrub_large_omap_object_key_threshold")
+  };
+}
+
+void PGScrubber::request_range(const hobject_t &start)
+{
+  LOG_PREFIX(PGScrubber::request_range);
+  DEBUGDPP("start: {}", pg, start);
+  std::ignore = pg.shard_services.start_operation_may_interrupt<
+    interruptor, ScrubFindRange
+    >(start, &pg);
+}
+
+/* TODO: This isn't actually enough.  Here, classic would
+ * hold the pg lock from the wait_scrub through to IO submission.
+ * ClientRequest, however, isn't in the processing ExclusivePhase
+ * bit yet, and so this check may miss ops between the wait_scrub
+ * check and adding the IO to the log. */
+
+void PGScrubber::reserve_range(const hobject_t &start, const hobject_t &end)
+{
+  LOG_PREFIX(PGScrubber::reserve_range);
+  DEBUGDPP("start: {}, end: {}", pg, start, end);
+  std::ignore = pg.shard_services.start_operation_may_interrupt<
+    interruptor, ScrubReserveRange
+    >(start, end, &pg);
+}
+
+void PGScrubber::release_range()
+{
+  LOG_PREFIX(PGScrubber::release_range);
+  ceph_assert(blocked);
+  DEBUGDPP("blocked: {}", pg, *blocked);
+  pg.background_process_lock.unlock();
+  blocked->p.set_value();
+  blocked = std::nullopt;
+}
+
+void PGScrubber::scan_range(
+  pg_shard_t target,
+  eversion_t version,
+  bool deep,
+  const hobject_t &start,
+  const hobject_t &end)
+{
+  LOG_PREFIX(PGScrubber::scan_range);
+  DEBUGDPP("target: {}, version: {}, deep: {}, start: {}, end: {}",
+	   pg, target, version, deep, start, end);
+  if (target == pg.get_pg_whoami()) {
+    std::ignore = pg.shard_services.start_operation_may_interrupt<
+      interruptor, ScrubScan
+      >(&pg, deep, true /* local */, start, end);
+  } else {
+    std::ignore = pg.shard_services.send_to_osd(
+      target.osd,
+      crimson::make_message<MOSDRepScrub>(
+	spg_t(pg.get_pgid().pgid, target.shard),
+	version,
+	pg.get_osdmap_epoch(),
+	pg.get_osdmap_epoch(),
+	start,
+	end,
+	deep,
+	false /* allow preemption -- irrelevant for replicas TODO */,
+	64 /* priority, TODO */,
+	false /* high_priority TODO */),
+      pg.get_osdmap_epoch());
+  }
+}
+
+bool PGScrubber::await_update(const eversion_t &version)
+{
+  LOG_PREFIX(PGScrubber::await_update);
+  DEBUGDPP("version: {}", pg, version);
+  ceph_assert(!waiting_for_update);
+  auto& log = pg.peering_state.get_pg_log().get_log().log;
+  eversion_t current = log.empty() ? eversion_t() : log.rbegin()->version;
+  if (version <= current) {
+    return true;
+  } else {
+    waiting_for_update = version;
+    return false;
+  }
+}
+
+void PGScrubber::generate_and_submit_chunk_result(
+  const hobject_t &begin,
+  const hobject_t &end,
+  bool deep)
+{
+  LOG_PREFIX(PGScrubber::generate_and_submit_chunk_result);
+  DEBUGDPP("begin: {}, end: {}, deep: {}", pg, begin, end, deep);
+  std::ignore = pg.shard_services.start_operation_may_interrupt<
+    interruptor, ScrubScan
+    >(&pg, deep, false /* local */, begin, end);
+}
+
+#define LOG_SCRUB_ERROR(MSG, ...) {					\
+    auto errorstr = fmt::format(MSG, __VA_ARGS__);			\
+    ERRORDPP("{}", pg, errorstr);					\
+    pg.get_clog_error() << "pg " << pg.get_pgid() << ": " << errorstr;	\
+  }
+
+void PGScrubber::emit_chunk_result(
+  const request_range_result_t &range,
+  chunk_result_t &&result)
+{
+  LOG_PREFIX(PGScrubber::emit_chunk_result);
+  if (result.has_errors()) {
+    LOG_SCRUB_ERROR(
+      "Scrub errors found. range: {}, result: {}",
+      range, result);
+  } else {
+    DEBUGDPP("Chunk complete. range: {}", pg, range);
+  }
+}
+
+void PGScrubber::emit_scrub_result(
+  bool deep,
+  object_stat_sum_t in_stats)
+{
+  LOG_PREFIX(PGScrubber::emit_scrub_result);
+  DEBUGDPP("", pg);
+  pg.peering_state.update_stats(
+    [this, FNAME, deep, &in_stats](auto &history, auto &pg_stats) {
+      foreach_scrub_maintained_stat(
+	[deep, &pg_stats, &in_stats](
+	  const auto &name, auto statptr, bool skip_for_shallow) {
+	  if (deep && !skip_for_shallow) {
+	    pg_stats.stats.sum.*statptr = in_stats.*statptr;
+	  }
+	});
+      foreach_scrub_checked_stat(
+	[this, FNAME, &pg_stats, &in_stats](
+	  const auto &name, auto statptr, const auto &invalid_predicate) {
+	  if (!invalid_predicate(pg_stats) &&
+	      (in_stats.*statptr != pg_stats.stats.sum.*statptr)) {
+	    LOG_SCRUB_ERROR(
+	      "stat mismatch for {}: scrubbed value: {}, stored pg value: {}",
+	      name, in_stats.*statptr, pg_stats.stats.sum.*statptr);
+	    ++pg_stats.stats.sum.num_shallow_scrub_errors;
+	  }
+	});
+      history.last_scrub = pg.peering_state.get_info().last_update;
+      auto now = ceph_clock_now();
+      history.last_scrub_stamp = now;
+      if (deep) {
+	history.last_deep_scrub_stamp = now;
+      }
+      return false; // notify_scrub_end will flush stats to osd
+    });
+}
+
+}
diff --git a/src/crimson/osd/scrub/pg_scrubber.h b/src/crimson/osd/scrub/pg_scrubber.h
new file mode 100644
index 000000000000..2d528e04d4bd
--- /dev/null
+++ b/src/crimson/osd/scrub/pg_scrubber.h
@@ -0,0 +1,152 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 smarttab expandtab
+
+#pragma once
+
+#include "crimson/osd/pg_interval_interrupt_condition.h"
+#include "scrub_machine.h"
+
+namespace crimson::osd {
+class PG;
+class ScrubScan;
+class ScrubFindRange;
+class ScrubReserveRange;
+}
+
+namespace crimson::osd::scrub {
+
+struct blocked_range_t {
+  hobject_t begin;
+  hobject_t end;
+  seastar::shared_promise<> p;
+};
+
+class PGScrubber : public crimson::BlockerT<PGScrubber>, ScrubContext {
+  friend class ::crimson::osd::ScrubScan;
+  friend class ::crimson::osd::ScrubFindRange;
+  friend class ::crimson::osd::ScrubReserveRange;
+
+  using interruptor = ::crimson::interruptible::interruptor<
+    ::crimson::osd::IOInterruptCondition>;
+  template <typename T = void>
+  using ifut =
+    ::crimson::interruptible::interruptible_future<
+      ::crimson::osd::IOInterruptCondition, T>;
+
+  PG &pg;
+
+  /// PG alias for logging in header functions
+  DoutPrefixProvider &dpp;
+
+  ScrubMachine machine;
+
+  std::optional<blocked_range_t> blocked;
+
+  std::optional<eversion_t> waiting_for_update;
+
+  template <typename E>
+  void handle_event(E &&e)
+  {
+    LOG_PREFIX(PGScrubber::handle_event);
+    SUBDEBUGDPP(osd, "handle_event: {}", dpp, e);
+    machine.process_event(std::forward<E>(e));
+  }
+
+public:
+  static constexpr const char *type_name = "PGScrubber";
+  using Blocker = PGScrubber;
+  void dump_detail(Formatter *f) const;
+
+  static inline bool is_scrub_message(Message &m) {
+    switch (m.get_type()) {
+    case MSG_OSD_REP_SCRUB:
+    case MSG_OSD_REP_SCRUBMAP:
+      return true;
+    default:
+      return false;
+    }
+    return false;
+  }
+
+  PGScrubber(PG &pg);
+
+  /// setup scrub machine state
+  void initiate() { machine.initiate(); }
+
+  /// notify machine on primary that PG is active+clean
+  void on_primary_active_clean();
+
+  /// notify machine on replica that PG is active
+  void on_replica_activate();
+
+  /// notify machine of interval change
+  void on_interval_change();
+
+  /// notify machine that PG has committed up to versino v
+  void on_log_update(eversion_t v);
+
+  /// handle scrub request
+  void handle_scrub_requested(bool deep);
+
+
+  /// handle other scrub message
+  void handle_scrub_message(Message &m);
+
+  /// notify machine of a mutation of on_object resulting in delta_stats
+  void handle_op_stats(
+    const hobject_t &on_object,
+    object_stat_sum_t delta_stats);
+
+  /// maybe block an op trying to mutate hoid until chunk is complete
+  ifut<> wait_scrub(
+    PGScrubber::BlockingEvent::TriggerI&& trigger,
+    const hobject_t &hoid);
+
+private:
+  DoutPrefixProvider &get_dpp() final { return dpp; }
+
+  void notify_scrub_start(bool deep) final;
+  void notify_scrub_end(bool deep) final;
+
+  const std::set<pg_shard_t> &get_ids_to_scrub() const final;
+
+  chunk_validation_policy_t get_policy() const final;
+
+  void request_range(const hobject_t &start) final;
+  void reserve_range(const hobject_t &start, const hobject_t &end) final;
+  void release_range() final;
+  void scan_range(
+    pg_shard_t target,
+    eversion_t version,
+    bool deep,
+    const hobject_t &start,
+    const hobject_t &end) final;
+  bool await_update(const eversion_t &version) final;
+  void generate_and_submit_chunk_result(
+    const hobject_t &begin,
+    const hobject_t &end,
+    bool deep) final;
+  void emit_chunk_result(
+    const request_range_result_t &range,
+    chunk_result_t &&result) final;
+  void emit_scrub_result(
+    bool deep,
+    object_stat_sum_t scrub_stats) final;
+};
+
+};
+
+template <>
+struct fmt::formatter<crimson::osd::scrub::blocked_range_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const auto &range, FormatContext& ctx)
+  {
+    return fmt::format_to(
+      ctx.out(),
+      "{}~{}",
+      range.begin,
+      range.end);
+  }
+};

From 12e5f2289d3445f7f39320a5e63342e09a779b99 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 9 Oct 2023 15:12:29 -0700
Subject: [PATCH 1023/2492] crimson/osd/pg: PG::scrub_requested isn't actually
 used in crimson

Crimson routes the scrub request directly to the PGScrubber rather
than going through PeeringState.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/pg.cc | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 6537eed2b7d5..5e3b0d429414 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -547,19 +547,10 @@ void PG::on_active_advmap(const OSDMapRef &osdmap)
 
 void PG::scrub_requested(scrub_level_t scrub_level, scrub_type_t scrub_type)
 {
-  // TODO: should update the stats upon finishing the scrub
-  peering_state.update_stats([scrub_level, this](auto& history, auto& stats) {
-    const utime_t now = ceph_clock_now();
-    history.last_scrub = peering_state.get_info().last_update;
-    history.last_scrub_stamp = now;
-    history.last_clean_scrub_stamp = now;
-    if (scrub_level == scrub_level_t::deep) {
-      history.last_deep_scrub = history.last_scrub;
-      history.last_deep_scrub_stamp = now;
-    }
-    // yes, please publish the stats
-    return true;
-  });
+  /* We don't actually route the scrub request message into the state machine.
+   * Instead, we handle it directly in PGScrubber::handle_scrub_requested).
+   */
+  ceph_assert(0 == "impossible in crimson");
 }
 
 void PG::log_state_enter(const char *state) {

From 8a5553597ca6a428cb8ffc9fc5bebde048fbd068 Mon Sep 17 00:00:00 2001
From: Nico Wang <wangn4@rpi.edu>
Date: Tue, 10 Oct 2023 21:00:29 +0000
Subject: [PATCH 1024/2492] mgr/balancer: Improved 'ceph balancer status'
 output

We have recently introduced a detailed command to check the status of our balancer.
This command displays the location of pg_up_map items where they are being transferred
when the balancer runs. The primary objective of this command is to obtain a better
understanding of the balancer's behavior in terms of transferring pgs. This will help
us identify when the balancing is taking place in the effort to improve the balancing
process to evenly distribute pgs in the balancing process.

Signed-off-by: Christopher Poon <poon3@rpi.edu>
Signed-off-by: Zuriel Aviles <avilez@rpi.edu>
Signed-off-by: Nico Wang <wangn4@rpi.edu>
---
 src/pybind/mgr/balancer/module.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 1c40425115cf..590f24653f0c 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -327,6 +327,9 @@ class Module(MgrModule):
     no_optimization_needed = False
     success_string = 'Optimization plan created successfully'
     in_progress_string = 'in progress'
+    last_pg_upmap: List[Dict[str, Any]] = []
+    added_pg_upmap_items: List[Dict[str, Any]] = []
+    removed_pg_upmap_items: List[Dict[str, Any]] = []
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super(Module, self).__init__(*args, **kwargs)
@@ -348,6 +351,24 @@ def show_status(self) -> Tuple[int, str, str]:
         }
         return (0, json.dumps(s, indent=4, sort_keys=True), '')
 
+    @CLIReadCommand('balancer status detail')
+    def show_status_detail(self) -> Tuple[int, str, str]:
+        """
+        Show balancer status (detailed)
+        """
+        s = {
+            'plans': list(self.plans.keys()),
+            'active': self.active,
+            'last_optimize_started': self.last_optimize_started,
+            'last_optimize_duration': self.last_optimize_duration,
+            'optimize_result': self.optimize_result,
+            'no_optimization_needed': self.no_optimization_needed,
+            'mode': self.get_module_option('mode'),
+            'added_pg_upmap_items': self.added_pg_upmap_items,
+            'removed_pg_upmap_items': self.removed_pg_upmap_items,
+        }
+        return (0, json.dumps(s, indent=4), '')
+
     @CLICommand('balancer mode')
     def set_mode(self, mode: Mode) -> Tuple[int, str, str]:
         """
@@ -693,6 +714,9 @@ def serve(self) -> None:
                 start = time.time()
                 r, detail = self.optimize(plan)
                 end = time.time()
+                self.added_pg_upmap_items = [pg for pg in osdmap.dump().get('pg_upmap_items', '') if pg not in self.last_pg_upmap]
+                self.removed_pg_upmap_items = [pg for pg in self.last_pg_upmap if pg not in osdmap.dump().get('pg_upmap_items', '')]
+                self.last_pg_upmap = osdmap.dump().get('pg_upmap_items', '')
                 self.last_optimize_duration = str(datetime.timedelta(seconds=(end - start)))
                 if r == 0:
                     self.optimize_result = self.success_string

From ea15ee08c3c7a4ed591ee74e8943017d079bc060 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 7 Dec 2023 03:37:57 -0600
Subject: [PATCH 1025/2492] osd/scrub: expose h.p. scrub jobs in the queue

- a high-priority bit is added to the scrub job
- a scrub scheduling attempt will be performed even if only
  high-priority
  jobs should be allowed to run

Note: some of the changes in this PR are temporary, in the sense that
they would be made obsolete when the scrub scheduler is refactored.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc       | 100 +++++++++++++---------------
 src/osd/scrubber/osd_scrub.h        |  11 ++-
 src/osd/scrubber/osd_scrub_sched.cc |  21 +++---
 src/osd/scrubber/scrub_job.h        |   8 +++
 src/osd/scrubber_common.h           |  11 ++-
 5 files changed, 78 insertions(+), 73 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index 99367170dbac..536c4479b1d3 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -69,10 +69,11 @@ bool OsdScrub::scrub_random_backoff() const
 
 void OsdScrub::initiate_scrub(bool is_recovery_active)
 {
-  if (scrub_random_backoff()) {
-    // dice-roll says we should not scrub now
-    return;
-  }
+  const utime_t scrub_time = ceph_clock_now();
+  dout(10) << fmt::format(
+		  "time now:{}, recover is active?:{}", scrub_time,
+		  is_recovery_active)
+	   << dendl;
 
   if (auto blocked_pgs = get_blocked_pgs_count(); blocked_pgs > 0) {
     // some PGs managed by this OSD were blocked by a locked object during
@@ -84,35 +85,14 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
 	<< dendl;
   }
 
-  // fail fast if no resources are available
-  if (!m_resource_bookkeeper.can_inc_scrubs()) {
-    dout(20) << "too many scrubs already running on this OSD" << dendl;
-    return;
-  }
-
-  // if there is a PG that is just now trying to reserve scrub replica resources -
-  // we should wait and not initiate a new scrub
-  if (m_queue.is_reserving_now()) {
-    dout(10) << "scrub resources reservation in progress" << dendl;
-    return;
-  }
-
-  utime_t scrub_time = ceph_clock_now();
-  dout(10) << fmt::format(
-		  "time now:{}, recover is active?:{}", scrub_time,
-		  is_recovery_active)
-	   << dendl;
-
   // check the OSD-wide environment conditions (scrub resources, time, etc.).
   // These may restrict the type of scrubs we are allowed to start, or just
-  // prevent us from starting any scrub at all.
+  // prevent us from starting any non-operator-initiated scrub at all.
   auto env_restrictions =
       restrictions_on_scrubbing(is_recovery_active, scrub_time);
-  if (!env_restrictions) {
-    return;
-  }
 
-  if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
+  if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>() &&
+      !env_restrictions.high_priority_only) {
     dout(20) << "scrub scheduling (@tick) starts" << dendl;
     auto all_jobs = m_queue.list_registered_jobs();
     for (const auto& sj : all_jobs) {
@@ -124,7 +104,7 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
   // queue interface used here: we ask for a list of
   // eligible targets (based on the known restrictions).
   // We try all elements of this list until a (possibly temporary) success.
-  auto candidates = m_queue.ready_to_scrub(*env_restrictions, scrub_time);
+  auto candidates = m_queue.ready_to_scrub(env_restrictions, scrub_time);
   if (candidates.empty()) {
     dout(20) << "no PGs are ready for scrubbing" << dendl;
     return;
@@ -137,7 +117,7 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
     // scrub. For some failures - we can continue with the next candidate. For
     // others - we should stop trying to scrub at this tick.
     auto res = initiate_a_scrub(
-	candidate, env_restrictions->allow_requested_repair_only);
+	candidate, env_restrictions.allow_requested_repair_only);
 
     if (res == schedule_result_t::target_specific_failure) {
       // continue with the next job.
@@ -157,39 +137,51 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
 }
 
 
-std::optional<Scrub::OSDRestrictions> OsdScrub::restrictions_on_scrubbing(
+Scrub::OSDRestrictions OsdScrub::restrictions_on_scrubbing(
     bool is_recovery_active,
     utime_t scrub_clock_now) const
 {
-  // our local OSD may already be running too many scrubs
+  Scrub::OSDRestrictions env_conditions;
+
+  // some environmental conditions prevent all but high priority scrubs
+
   if (!m_resource_bookkeeper.can_inc_scrubs()) {
-    dout(10) << "OSD cannot inc scrubs" << dendl;
-    return std::nullopt;
-  }
+    // our local OSD is already running too many scrubs
+    dout(15) << "OSD cannot inc scrubs" << dendl;
+    env_conditions.high_priority_only = true;
 
-  // if there is a PG that is just now trying to reserve scrub replica resources
-  // - we should wait and not initiate a new scrub
-  if (m_queue.is_reserving_now()) {
+  } else if (scrub_random_backoff()) {
+    // dice-roll says we should not scrub now
+      dout(15) << "Lost in dice. Only high priority scrubs allowed."
+	       << dendl;
+      env_conditions.high_priority_only = true;
+
+  } else if (m_queue.is_reserving_now()) {
+    // if there is a PG that is just now trying to reserve scrub replica
+    // resources - we should wait and not initiate a new scrub
     dout(10) << "scrub resources reservation in progress" << dendl;
-    return std::nullopt;
-  }
+    env_conditions.high_priority_only = true;
 
-  Scrub::OSDRestrictions env_conditions;
-  env_conditions.time_permit = scrub_time_permit(scrub_clock_now);
-  env_conditions.load_is_low = m_load_tracker.scrub_load_below_threshold();
-  env_conditions.only_deadlined =
-      !env_conditions.time_permit || !env_conditions.load_is_low;
-
-  if (is_recovery_active && !conf->osd_scrub_during_recovery) {
-    if (!conf->osd_repair_during_recovery) {
-      dout(15) << "not scheduling scrubs due to active recovery" << dendl;
-      return std::nullopt;
+  } else if (is_recovery_active && !conf->osd_scrub_during_recovery) {
+    if (conf->osd_repair_during_recovery) {
+      dout(15)
+	  << "will only schedule explicitly requested repair due to active "
+	     "recovery"
+	  << dendl;
+      env_conditions.allow_requested_repair_only = true;
+
+    } else {
+      dout(15) << "recovery in progress. Only high priority scrubs allowed."
+	       << dendl;
+      env_conditions.high_priority_only = true;
     }
+  } else {
 
-    dout(10) << "will only schedule explicitly requested repair due to active "
-		"recovery"
-	     << dendl;
-    env_conditions.allow_requested_repair_only = true;
+    // regular, i.e. non-high-priority scrubs are allowed
+    env_conditions.time_permit = scrub_time_permit(scrub_clock_now);
+    env_conditions.load_is_low = m_load_tracker.scrub_load_below_threshold();
+    env_conditions.only_deadlined =
+	!env_conditions.time_permit || !env_conditions.load_is_low;
   }
 
   return env_conditions;
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 56167df2ee6e..fcc4fd3fe9c5 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -168,20 +168,17 @@ class OsdScrub {
 
   /**
    * check the OSD-wide environment conditions (scrub resources, time, etc.).
-   * These may restrict the type of scrubs we are allowed to start, or just
-   * prevent us from starting any scrub at all.
+   * These may restrict the type of scrubs we are allowed to start, maybe
+   * down to allowing only high-priority scrubs
    *
    * Specifically:
-   * a nullopt is returned if we are not allowed to scrub at all, for either of
+   * 'only high priority' flag is set for either of
    * the following reasons: no local resources (too many scrubs on this OSD);
    * a dice roll says we will not scrub in this tick;
    * a recovery is in progress, and we are not allowed to scrub while recovery;
    * a PG is trying to acquire replica resources.
-   *
-   * If we are allowed to scrub, the returned value specifies whether the only
-   * high priority scrubs or only overdue ones are allowed to go on.
    */
-  std::optional<Scrub::OSDRestrictions> restrictions_on_scrubbing(
+  Scrub::OSDRestrictions restrictions_on_scrubbing(
       bool is_recovery_active,
       utime_t scrub_clock_now) const;
 
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index 324899f29ab8..1b3506a35e50 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -149,6 +149,7 @@ void ScrubQueue::update_job(Scrub::ScrubJobRef scrub_job,
   // adjust the suggested scrub time according to OSD-wide status
   auto adjusted = adjust_target_time(suggested);
   scrub_job->update_schedule(adjusted);
+  scrub_job->high_priority = suggested.is_must == must_scrub_t::mandatory;
 }
 
 sched_params_t ScrubQueue::determine_scrub_time(
@@ -299,11 +300,13 @@ void ScrubQueue::rm_unregistered_jobs(ScrubQContainer& group)
 }
 
 namespace {
-struct cmp_sched_time_t {
-  bool operator()(const Scrub::ScrubJobRef& lhs,
-		  const Scrub::ScrubJobRef& rhs) const
+struct cmp_time_n_priority_t {
+  bool operator()(const Scrub::ScrubJobRef& lhs, const Scrub::ScrubJobRef& rhs)
+      const
   {
-    return lhs->schedule.scheduled_at < rhs->schedule.scheduled_at;
+    return lhs->is_high_priority() > rhs->is_high_priority() ||
+	   (lhs->is_high_priority() == rhs->is_high_priority() &&
+	    lhs->schedule.scheduled_at < rhs->schedule.scheduled_at);
   }
 };
 }  // namespace
@@ -314,11 +317,11 @@ ScrubQContainer ScrubQueue::collect_ripe_jobs(
     OSDRestrictions restrictions,
     utime_t time_now)
 {
-  auto filtr = [time_now, restrictions](const auto& jobref) -> bool {
+  auto filtr = [time_now, rst = restrictions](const auto& jobref) -> bool {
     return jobref->schedule.scheduled_at <= time_now &&
-	   (!restrictions.only_deadlined ||
-	    (!jobref->schedule.deadline.is_zero() &&
-	     jobref->schedule.deadline <= time_now));
+	   (!rst.high_priority_only || jobref->high_priority) &&
+	   (!rst.only_deadlined || (!jobref->schedule.deadline.is_zero() &&
+				    jobref->schedule.deadline <= time_now));
   };
 
   rm_unregistered_jobs(group);
@@ -327,7 +330,7 @@ ScrubQContainer ScrubQueue::collect_ripe_jobs(
   ripes.reserve(group.size());
 
   std::copy_if(group.begin(), group.end(), std::back_inserter(ripes), filtr);
-  std::sort(ripes.begin(), ripes.end(), cmp_sched_time_t{});
+  std::sort(ripes.begin(), ripes.end(), cmp_time_n_priority_t{});
 
   if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
     for (const auto& jobref : group) {
diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index ae29c8ebab44..11e7388f6362 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -93,6 +93,8 @@ class ScrubJob final : public RefCountedObject {
 
   CephContext* cct;
 
+  bool high_priority{false};
+
   ScrubJob(CephContext* cct, const spg_t& pg, int node_id);
 
   utime_t get_sched_time() const { return schedule.scheduled_at; }
@@ -130,6 +132,12 @@ class ScrubJob final : public RefCountedObject {
    */
   bool is_state_registered() const { return state == qu_state_t::registered; }
 
+  /**
+   * is this a high priority scrub job?
+   * High priority - (usually) a scrub that was initiated by the operator
+   */
+  bool is_high_priority() const { return high_priority; }
+
   /**
    * a text description of the "scheduling intentions" of this PG:
    * are we already scheduled for a scrub/deep scrub? when?
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index d24bb79b801e..f39907f3f6b3 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -49,12 +49,16 @@ enum class scrub_prio_t : bool { low_priority = false, high_priority = true };
 using act_token_t = uint32_t;
 
 /// "environment" preconditions affecting which PGs are eligible for scrubbing
+/// (note: struct size should be kept small, as it is copied around)
 struct OSDRestrictions {
+  /// high local OSD concurrency. Thus - only high priority scrubs are allowed
+  bool high_priority_only{false};
   bool allow_requested_repair_only{false};
-  bool load_is_low{true};
-  bool time_permit{true};
   bool only_deadlined{false};
+  bool load_is_low:1{true};
+  bool time_permit:1{true};
 };
+static_assert(sizeof(Scrub::OSDRestrictions) <= sizeof(uint32_t));
 
 }  // namespace Scrub
 
@@ -68,7 +72,8 @@ struct formatter<Scrub::OSDRestrictions> {
   {
     return fmt::format_to(
       ctx.out(),
-      "overdue-only:{} load:{} time:{} repair-only:{}",
+      "priority-only:{} overdue-only:{} load:{} time:{} repair-only:{}",
+        conds.high_priority_only,
         conds.only_deadlined,
         conds.load_is_low ? "ok" : "high",
         conds.time_permit ? "ok" : "no",

From fcfab283c0a2b5dcf22ef17b5469d2266f6887c6 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 7 Dec 2023 08:55:30 -0600
Subject: [PATCH 1026/2492] osd/scrub: fix scrub eligibility tests

By:
- removing duplicate checks;
- moving most checks "down" to the PG;
- allowing high-priority scrubs to override most limiting conditions.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                 | 50 +++++++++++++++++++++++------------
 src/osd/PG.h                  |  9 +++++--
 src/osd/scrubber/osd_scrub.cc | 26 ++++--------------
 src/osd/scrubber/osd_scrub.h  |  2 +-
 4 files changed, 46 insertions(+), 41 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ddef326e2a8a..5ff1246427b7 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1331,30 +1331,44 @@ unsigned int PG::scrub_requeue_priority(Scrub::scrub_prio_t with_priority, unsig
 // ==========================================================================================
 // SCRUB
 
+
 /*
  *  implementation note:
- *  PG::sched_scrub() is called only once per a specific scrub session.
+ *  PG::start_scrubbing() is called only once per a specific scrub session.
  *  That call commits us to the whatever choices are made (deep/shallow, etc').
  *  Unless failing to start scrubbing, the 'planned scrub' flag-set is 'frozen' into
  *  PgScrubber's m_flags, then cleared.
  */
-Scrub::schedule_result_t PG::sched_scrub()
+Scrub::schedule_result_t PG::start_scrubbing(
+    Scrub::OSDRestrictions osd_restrictions)
 {
   using Scrub::schedule_result_t;
-  dout(15) << __func__ << " pg(" << info.pgid
-	  << (is_active() ? ") <active>" : ") <not-active>")
-	  << (is_clean() ? " <clean>" : " <not-clean>") << dendl;
+  dout(10) << fmt::format(
+		  "{}: {}+{} (env restrictions:{})", __func__,
+		  (is_active() ? "<active>" : "<not-active>"),
+		  (is_clean() ? "<clean>" : "<not-clean>"), osd_restrictions)
+	   << dendl;
   ceph_assert(ceph_mutex_is_locked(_lock));
-  ceph_assert(m_scrubber);
-
-  if (is_scrub_queued_or_active()) {
-     dout(10) << __func__ << ": already scrubbing" << dendl;
-     return schedule_result_t::target_specific_failure;
-  }
 
   if (!is_primary() || !is_active() || !is_clean()) {
     dout(10) << __func__ << ": cannot scrub (not a clean and active primary)"
-      << dendl;
+	     << dendl;
+    return schedule_result_t::target_specific_failure;
+  }
+
+  ceph_assert(m_scrubber);
+  if (is_scrub_queued_or_active()) {
+    dout(10) << __func__ << ": scrub already in progress" << dendl;
+    return schedule_result_t::target_specific_failure;
+  }
+  // if only explicitly requested repairing is allowed - skip other types
+  // of scrubbing
+  if (osd_restrictions.allow_requested_repair_only &&
+      !get_planned_scrub().must_repair) {
+    dout(10) << __func__
+	     << ": skipping this PG as repairing was not explicitly "
+		"requested for it"
+	     << dendl;
     return schedule_result_t::target_specific_failure;
   }
 
@@ -1366,9 +1380,9 @@ Scrub::schedule_result_t PG::sched_scrub()
     return schedule_result_t::target_specific_failure;
   }
 
-  // analyse the combination of the requested scrub flags, the osd/pool configuration
-  // and the PG status to determine whether we should scrub now, and what type of scrub
-  // should that be.
+  // analyze the combination of the requested scrub flags, the osd/pool
+  // configuration and the PG status to determine whether we should scrub
+  // now, and what type of scrub should that be.
   auto updated_flags = validate_scrub_mode();
   if (!updated_flags) {
     // the stars do not align for starting a scrub for this PG at this time
@@ -1391,15 +1405,17 @@ Scrub::schedule_result_t PG::sched_scrub()
   // An interrupted recovery repair could leave this set.
   state_clear(PG_STATE_REPAIR);
 
-  // Pass control to the scrubber. It is the scrubber that handles the replicas'
-  // resources reservations.
+  // Pass control to the scrubber. It is the scrubber that handles the
+  // replicas' resources reservations.
   m_scrubber->set_op_parameters(m_planned_scrub);
 
+  // using the OSD queue, as to not execute the scrub code as part of the tick.
   dout(10) << __func__ << ": queueing" << dendl;
   osd->queue_for_scrub(this, Scrub::scrub_prio_t::low_priority);
   return schedule_result_t::scrub_initiated;
 }
 
+
 double PG::next_deepscrub_interval() const
 {
   double deep_scrub_interval =
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 8713b1c8ae88..d9acfafd0328 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -709,11 +709,16 @@ class PG : public DoutPrefixProvider,
   virtual void on_shutdown() = 0;
 
   bool get_must_scrub() const;
-  Scrub::schedule_result_t sched_scrub();
 
-  unsigned int scrub_requeue_priority(Scrub::scrub_prio_t with_priority, unsigned int suggested_priority) const;
+  Scrub::schedule_result_t start_scrubbing(
+      Scrub::OSDRestrictions osd_restrictions);
+
+  unsigned int scrub_requeue_priority(
+      Scrub::scrub_prio_t with_priority,
+      unsigned int suggested_priority) const;
   /// the version that refers to flags_.priority
   unsigned int scrub_requeue_priority(Scrub::scrub_prio_t with_priority) const;
+
 private:
   // auxiliaries used by sched_scrub():
   double next_deepscrub_interval() const;
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index 536c4479b1d3..fb21e0e1c5ee 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -71,7 +71,7 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
 {
   const utime_t scrub_time = ceph_clock_now();
   dout(10) << fmt::format(
-		  "time now:{}, recover is active?:{}", scrub_time,
+		  "time now:{:s}, recovery is active?:{}", scrub_time,
 		  is_recovery_active)
 	   << dendl;
 
@@ -116,8 +116,7 @@ void OsdScrub::initiate_scrub(bool is_recovery_active)
     // we have a candidate to scrub. But we may fail when trying to initiate that
     // scrub. For some failures - we can continue with the next candidate. For
     // others - we should stop trying to scrub at this tick.
-    auto res = initiate_a_scrub(
-	candidate, env_restrictions.allow_requested_repair_only);
+    auto res = initiate_a_scrub(candidate, env_restrictions);
 
     if (res == schedule_result_t::target_specific_failure) {
       // continue with the next job.
@@ -190,7 +189,7 @@ Scrub::OSDRestrictions OsdScrub::restrictions_on_scrubbing(
 
 Scrub::schedule_result_t OsdScrub::initiate_a_scrub(
     spg_t pgid,
-    bool allow_requested_repair_only)
+    Scrub::OSDRestrictions restrictions)
 {
   dout(20) << fmt::format("trying pg[{}]", pgid) << dendl;
 
@@ -205,23 +204,8 @@ Scrub::schedule_result_t OsdScrub::initiate_a_scrub(
     return Scrub::schedule_result_t::target_specific_failure;
   }
 
-  // This one is already scrubbing, so go on to the next scrub job
-  if (locked_pg->pg()->is_scrub_queued_or_active()) {
-    dout(10) << fmt::format("pg[{}]: scrub already in progress", pgid) << dendl;
-    return Scrub::schedule_result_t::target_specific_failure;
-  }
-  // Skip other kinds of scrubbing if only explicitly requested repairing is allowed
-  if (allow_requested_repair_only &&
-      !locked_pg->pg()->get_planned_scrub().must_repair) {
-    dout(10) << fmt::format(
-		    "skipping pg[{}] as repairing was not explicitly "
-		    "requested for that pg",
-		    pgid)
-	     << dendl;
-    return Scrub::schedule_result_t::target_specific_failure;
-  }
-
-  return locked_pg->pg()->sched_scrub();
+  // later on, here is where the scrub target would be dequeued
+  return locked_pg->pg()->start_scrubbing(restrictions);
 }
 
 void OsdScrub::on_config_change()
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index fcc4fd3fe9c5..774076711e2d 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -193,7 +193,7 @@ class OsdScrub {
    */
   Scrub::schedule_result_t initiate_a_scrub(
       spg_t pgid,
-      bool allow_requested_repair_only);
+      Scrub::OSDRestrictions restrictions);
 
   /// resource reservation management
   Scrub::ScrubResources m_resource_bookkeeper;

From e1d91b24e4e63b08928b03f61e9bc68f6147507f Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 7 Dec 2023 10:11:34 -0600
Subject: [PATCH 1027/2492] osd/scrub: don't block high-priority scrubs on a
 too-high scrubs count

A scrub can only be initiated if the number of active scrubs
performed on the OSD when acting as a primary is below a certain
configured limit. After this change - high priority scrubs are
not blocked by this limit (although they are still counted
towards it).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.h                        |  1 -
 src/osd/scrubber/osd_scrub.cc       |  5 +++--
 src/osd/scrubber/osd_scrub.h        |  3 ++-
 src/osd/scrubber/pg_scrubber.cc     | 33 ++++++-----------------------
 src/osd/scrubber/pg_scrubber.h      | 22 ++++++-------------
 src/osd/scrubber/scrub_resources.cc | 23 ++++++++++++++++----
 src/osd/scrubber/scrub_resources.h  | 30 ++++++++++++++++++++++++--
 7 files changed, 65 insertions(+), 52 deletions(-)

diff --git a/src/osd/PG.h b/src/osd/PG.h
index d9acfafd0328..6e81af436e7d 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -67,7 +67,6 @@ class ScrubBackend;
 namespace Scrub {
   class Store;
   class ReplicaReservations;
-  class LocalReservation;
   class ReservedByRemotePrimary;
   enum class schedule_result_t;
 }
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index fb21e0e1c5ee..f11ddd2737ca 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -407,9 +407,10 @@ void OsdScrub::remove_from_osd_queue(Scrub::ScrubJobRef sjob)
   m_queue.remove_from_osd_queue(sjob);
 }
 
-bool OsdScrub::inc_scrubs_local()
+std::unique_ptr<Scrub::LocalResourceWrapper> OsdScrub::inc_scrubs_local(
+    bool is_high_priority)
 {
-  return m_resource_bookkeeper.inc_scrubs_local();
+  return m_resource_bookkeeper.inc_scrubs_local(is_high_priority);
 }
 
 void OsdScrub::dec_scrubs_local()
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 774076711e2d..dae11f860011 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -65,7 +65,8 @@ class OsdScrub {
   // ---------------------------------------------------------------
 
   // updating the resource counters
-  bool inc_scrubs_local();
+  std::unique_ptr<Scrub::LocalResourceWrapper> inc_scrubs_local(
+      bool is_high_priority);
   void dec_scrubs_local();
   bool inc_scrubs_remote(pg_t pgid);
   void dec_scrubs_remote(pg_t pgid);
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index edc6c2a2dcd4..11112bc48d96 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -587,16 +587,17 @@ void PgScrubber::request_rescrubbing(requested_scrub_t& request_flags)
 bool PgScrubber::reserve_local()
 {
   // try to create the reservation object (which translates into asking the
-  // OSD for the local scrub resource). If failing - undo it immediately
-
-  m_local_osd_resource.emplace(m_osds);
-  if (m_local_osd_resource->is_reserved()) {
+  // OSD for a local scrub resource). The object returned is a
+  // a wrapper around the actual reservation, and that object releases
+  // the local resource automatically when reset.
+  m_local_osd_resource = m_osds->get_scrub_services().inc_scrubs_local(
+      m_scrub_job->is_high_priority());
+  if (m_local_osd_resource) {
     dout(15) << __func__ << ": local resources reserved" << dendl;
     return true;
   }
 
-  dout(10) << __func__ << ": failed to reserve local scrub resources" << dendl;
-  m_local_osd_resource.reset();
+  dout(15) << __func__ << ": failed to reserve local scrub resources" << dendl;
   return false;
 }
 
@@ -2445,28 +2446,8 @@ void PgScrubber::preemption_data_t::reset()
   m_size_divisor = 1;
 }
 
-
-// ///////////////////// LocalReservation //////////////////////////////////
-
 namespace Scrub {
 
-// note: no dout()s in LocalReservation functions. Client logs interactions.
-LocalReservation::LocalReservation(OSDService* osds) : m_osds{osds}
-{
-  if (m_osds->get_scrub_services().inc_scrubs_local()) {
-    // a failure is signalled by not having m_holding_local_reservation set
-    m_holding_local_reservation = true;
-  }
-}
-
-LocalReservation::~LocalReservation()
-{
-  if (m_holding_local_reservation) {
-    m_holding_local_reservation = false;
-    m_osds->get_scrub_services().dec_scrubs_local();
-  }
-}
-
 // ///////////////////// MapsCollectionStatus ////////////////////////////////
 
 auto MapsCollectionStatus::mark_arriving_map(pg_shard_t from)
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 3ac4d6dfdb74..9f71f2d83230 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -87,21 +87,9 @@ Main Scrubber interfaces:
 namespace Scrub {
 class ScrubMachine;
 struct BuildMap;
+class LocalResourceWrapper;
 
 
-/**
- *  wraps the local OSD scrub resource reservation in an RAII wrapper
- */
-class LocalReservation {
-  OSDService* m_osds;
-  bool m_holding_local_reservation{false};
-
- public:
-  explicit LocalReservation(OSDService* osds);
-  ~LocalReservation();
-  bool is_reserved() const { return m_holding_local_reservation; }
-};
-
 /**
  * Once all replicas' scrub maps are received, we go on to compare the maps.
  * That is - unless we we have not yet completed building our own scrub map.
@@ -628,9 +616,11 @@ class PgScrubber : public ScrubPgIF,
 
   epoch_t m_last_aborted{};  // last time we've noticed a request to abort
 
-  // 'optional', as 'LocalReservation' is
-  // 'RAII-designed' to guarantee un-reserving when deleted.
-  std::optional<Scrub::LocalReservation> m_local_osd_resource;
+  /**
+   * once we acquire the local OSD resource, this is set to a wrapper that
+   * guarantees that the resource will be released when the scrub is done
+   */
+  std::unique_ptr<Scrub::LocalResourceWrapper> m_local_osd_resource;
 
   void cleanup_on_finish();  // scrub_clear_state() as called for a Primary when
 			     // Active->NotActive
diff --git a/src/osd/scrubber/scrub_resources.cc b/src/osd/scrubber/scrub_resources.cc
index dd9d31a15732..a69c6f06aca9 100644
--- a/src/osd/scrubber/scrub_resources.cc
+++ b/src/osd/scrubber/scrub_resources.cc
@@ -13,6 +13,7 @@
 
 
 using ScrubResources = Scrub::ScrubResources;
+using LocalResourceWrapper = Scrub::LocalResourceWrapper;
 
 ScrubResources::ScrubResources(
     log_upwards_t log_access,
@@ -32,17 +33,18 @@ bool ScrubResources::can_inc_scrubs() const
   return can_inc_local_scrubs_unlocked();
 }
 
-bool ScrubResources::inc_scrubs_local()
+std::unique_ptr<LocalResourceWrapper> ScrubResources::inc_scrubs_local(
+    bool is_high_priority)
 {
   std::lock_guard lck{resource_lock};
-  if (can_inc_local_scrubs_unlocked()) {
+  if (is_high_priority || can_inc_local_scrubs_unlocked()) {
     ++scrubs_local;
     log_upwards(fmt::format(
 	"{}: {} -> {} (max {}, remote {})", __func__, (scrubs_local - 1),
 	scrubs_local, conf->osd_max_scrubs, granted_reservations.size()));
-    return true;
+    return std::make_unique<LocalResourceWrapper>(*this);
   }
-  return false;
+  return nullptr;
 }
 
 bool ScrubResources::can_inc_local_scrubs_unlocked() const
@@ -119,3 +121,16 @@ void ScrubResources::dump_scrub_reservations(ceph::Formatter* f) const
   f->dump_string("PGs being served", fmt::format("{}", granted_reservations));
   f->dump_int("osd_max_scrubs", conf->osd_max_scrubs);
 }
+
+// --------------- LocalResourceWrapper
+
+Scrub::LocalResourceWrapper::LocalResourceWrapper(
+    ScrubResources& resource_bookkeeper)
+    : m_resource_bookkeeper{resource_bookkeeper}
+{}
+
+Scrub::LocalResourceWrapper::~LocalResourceWrapper()
+{
+  m_resource_bookkeeper.dec_scrubs_local();
+}
+
diff --git a/src/osd/scrubber/scrub_resources.h b/src/osd/scrubber/scrub_resources.h
index 75807a10f825..ff9587bb60b0 100644
--- a/src/osd/scrubber/scrub_resources.h
+++ b/src/osd/scrubber/scrub_resources.h
@@ -18,6 +18,7 @@ namespace Scrub {
  * (prefix func, OSD id, etc.)
  */
 using log_upwards_t = std::function<void(std::string msg)>;
+class LocalResourceWrapper;
 
 /**
  * The number of concurrent scrub operations performed on an OSD is limited
@@ -26,7 +27,15 @@ using log_upwards_t = std::function<void(std::string msg)>;
  * acting as primary and acting as a replica, and for enforcing the limit.
  */
 class ScrubResources {
-  /// the number of concurrent scrubs performed by Primaries on this OSD
+  friend class LocalResourceWrapper;
+
+  /**
+   * the number of concurrent scrubs performed by Primaries on this OSD.
+   *
+   * Note that, as high priority scrubs are always allowed to proceed, this
+   * counter may exceed the configured limit. When in this state - no new
+   * regular scrubs will be allowed to start.
+   */
   int scrubs_local{0};
 
   /// the set of PGs that have active scrub reservations as replicas
@@ -56,7 +65,7 @@ class ScrubResources {
   bool can_inc_scrubs() const;
 
   /// increments the number of scrubs acting as a Primary
-  bool inc_scrubs_local();
+  std::unique_ptr<LocalResourceWrapper> inc_scrubs_local(bool is_high_priority);
 
   /// decrements the number of scrubs acting as a Primary
   void dec_scrubs_local();
@@ -69,4 +78,21 @@ class ScrubResources {
 
   void dump_scrub_reservations(ceph::Formatter* f) const;
 };
+
+
+/**
+ * a wrapper around a "local scrub resource". The resources bookkeeper
+ * is handing these out to the PGs that acquired the local OSD's scrub
+ * resources. The PGs use these to release the resources when they are
+ * done scrubbing.
+ */
+class LocalResourceWrapper {
+  ScrubResources& m_resource_bookkeeper;
+
+ public:
+  LocalResourceWrapper(
+      ScrubResources& resource_bookkeeper);
+  ~LocalResourceWrapper();
+};
+
 }  // namespace Scrub

From eb5a5e990adda4eedbd5ab859b3860340a58cce8 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 11 Dec 2023 01:30:06 -0600
Subject: [PATCH 1028/2492] test/scrub: fix osd-recovery-scrub.sh to look for
 correct log message

... matching the code changes in the previous commit.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/scrub/osd-recovery-scrub.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/standalone/scrub/osd-recovery-scrub.sh b/qa/standalone/scrub/osd-recovery-scrub.sh
index 14c98c5e8f62..3d3121fe8d80 100755
--- a/qa/standalone/scrub/osd-recovery-scrub.sh
+++ b/qa/standalone/scrub/osd-recovery-scrub.sh
@@ -33,7 +33,7 @@ function run() {
     done
 }
 
-# Simple test for "not scheduling scrubs due to active recovery"
+# Simple test for "recovery in progress. Only high priority scrubs allowed."
 # OSD::sched_scrub() called on all OSDs during ticks
 function TEST_recovery_scrub_1() {
     local dir=$1
@@ -99,11 +99,11 @@ function TEST_recovery_scrub_1() {
     kill_daemons $dir #|| return 1
 
     declare -a err_strings
-    err_strings[0]="not scheduling scrubs due to active recovery"
+    err_strings[0]="recovery in progress. Only high priority scrubs allowed."
 
     for osd in $(seq 0 $(expr $OSDS - 1))
     do
-        grep "not scheduling scrubs" $dir/osd.${osd}.log
+        grep "recovery in progress. Only high priority scrubs allowed." $dir/osd.${osd}.log
     done
     for err_string in "${err_strings[@]}"
     do

From 3d67d7ad7ebab7cb47c07257945f60c156307243 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 11 Dec 2023 09:12:24 -0500
Subject: [PATCH 1029/2492] mgr/rgwam: period doesn't have a realm_name anymore

Fixes: https://tracker.ceph.com/issues/63783

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/python-common/ceph/rgw/types.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/python-common/ceph/rgw/types.py b/src/python-common/ceph/rgw/types.py
index 3f65f9da00e0..ddff7e15ea3d 100644
--- a/src/python-common/ceph/rgw/types.py
+++ b/src/python-common/ceph/rgw/types.py
@@ -117,7 +117,6 @@ def __init__(self, period_dict):
         self.epoch = period_dict['epoch']
         self.master_zone = period_dict['master_zone']
         self.master_zonegroup = period_dict['master_zonegroup']
-        self.realm_name = period_dict['realm_name']
         self.realm_id = period_dict['realm_id']
         pm = period_dict['period_map']
         self.zonegroups_by_id = {}

From 89b9fa299c0fe80b3e8619dd9dd8cb77e5b08016 Mon Sep 17 00:00:00 2001
From: sp98 <sapillai@redhat.com>
Date: Thu, 7 Dec 2023 16:18:05 +0530
Subject: [PATCH 1030/2492] mds: change priority of mds rss perf counter to
 useful

mds_cache_mem_rss perf counter can be a useful metric to detect
the usage of mds cache and detect mds cache oversize before it happens.
The PR changes the priority of the mds_rss metric from Debug to useful

Fixes: https://tracker.ceph.com/issues/63765
Signed-off-by: sp98 <sapillai@redhat.com>
---
 src/mds/MDSRank.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 20ddf3eec588..c02f8f07f61e 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3553,6 +3553,9 @@ void MDSRank::create_logger()
                     PerfCountersBuilder::PRIO_INTERESTING);
     mdm_plb.add_u64(l_mdm_dn, "dn", "Dentries", "dn",
                     PerfCountersBuilder::PRIO_INTERESTING);
+    // mds rss metric is set to PRIO_USEFUL as it can be useful to detect mds cache oversizing
+    mdm_plb.add_u64(l_mdm_rss, "rss", "RSS", "rss",
+                    PerfCountersBuilder::PRIO_USEFUL);
 
     mdm_plb.set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
     mdm_plb.add_u64_counter(l_mdm_inoa, "ino+", "Inodes opened");
@@ -3567,9 +3570,6 @@ void MDSRank::create_logger()
     mdm_plb.add_u64_counter(l_mdm_caps, "cap-", "Capabilities removed");
     mdm_plb.add_u64(l_mdm_heap, "heap", "Heap size");
 
-    mdm_plb.set_prio_default(PerfCountersBuilder::PRIO_DEBUGONLY);
-    mdm_plb.add_u64(l_mdm_rss, "rss", "RSS");
-
     mlogger = mdm_plb.create_perf_counters();
     g_ceph_context->get_perfcounters_collection()->add(mlogger);
   }

From acaad3555414aa1f1b4fa732af70612d50c5e883 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Sat, 25 Nov 2023 19:10:35 +0000
Subject: [PATCH 1031/2492] mgr/dashboard: added snap-schedule api and ui list

Fixes: https://tracker.ceph.com/issues/63767
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       | 30 ++++++
 ...ephfs-snapshotschedule-list.component.html | 58 +++++++++++
 ...ephfs-snapshotschedule-list.component.scss |  0
 ...fs-snapshotschedule-list.component.spec.ts | 30 ++++++
 .../cephfs-snapshotschedule-list.component.ts | 99 +++++++++++++++++++
 .../cephfs-tabs/cephfs-tabs.component.html    | 87 ++++++++++------
 .../src/app/ceph/cephfs/cephfs.module.ts      |  8 +-
 .../cephfs-snapshot-schedule.service.spec.ts  | 22 +++++
 .../api/cephfs-snapshot-schedule.service.ts   | 39 ++++++++
 .../app/shared/models/snapshot-schedule.ts    | 17 ++++
 src/pybind/mgr/dashboard/openapi.yaml         | 39 ++++++++
 11 files changed, 395 insertions(+), 34 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index eb1f764c472b..61b31eb809fd 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -863,3 +863,33 @@ def get(self, vol_name: str, subvol_name, group_name: str = '', info=True):
                     )
                 snapshot['info'] = json.loads(out)
         return snapshots
+
+
+@APIRouter('/cephfs/snaphost/schedule', Scope.CEPHFS)
+@APIDoc("Cephfs Snapshot Scheduling API", "CephFSSnapshotSchedule")
+class CephFSSnapshotSchedule(RESTController):
+
+    def list(self, fs: str, path: str = '/', recursive: bool = True):
+        error_code, out, err = mgr.remote('snap_schedule', 'snap_schedule_list',
+                                          path, recursive, fs, 'plain')
+
+        if len(out) == 0:
+            return []
+
+        snapshot_schedule_list = out.split('\n')
+        output = []
+
+        for snap in snapshot_schedule_list:
+            current_path = snap.strip().split(' ')[0]
+            error_code, status_out, err = mgr.remote('snap_schedule', 'snap_schedule_get',
+                                                     current_path, fs, 'plain')
+            output.append(json.loads(status_out))
+
+        output_json = json.dumps(output)
+
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to get list of snapshot schedules for path {path}: {err}'
+            )
+
+        return json.loads(output_json)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
new file mode 100644
index 000000000000..2e270057d579
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
@@ -0,0 +1,58 @@
+<ng-container *ngIf="isLoading$ | async">
+  <cd-loading-panel>
+    <span i18n>Loading snapshot schedules...</span>
+  </cd-loading-panel>
+</ng-container>
+
+<ng-template #pathTpl
+             let-row="row">
+  <span
+    class="fw-bold"
+    [ngbTooltip]="fullpathTpl"
+    triggers="click:blur">{{row.path | path}}</span>
+
+  <span *ngIf="row.active; else inactiveStatusTpl">
+    <i [ngClass]="[icons.success, icons.large]"
+       ngbTooltip="{{row.path}} is active"
+       class="text-success"></i>
+  </span>
+
+  <ng-template #inactiveStatusTpl>
+    <i [ngClass]="[icons.warning, icons.large]"
+       class="text-warning"
+       ngbTooltip="{{row.path}} has been deactivated"></i>
+  </ng-template>
+
+  <ng-template #fullpathTpl>
+  <span data-toggle="tooltip"
+        [title]="row.path"
+        class="font-monospace">{{ row.path }}
+    <cd-copy-2-clipboard-button *ngIf="row.path"
+                                [source]="row.path"
+                                [byId]="false"
+                                [showIconOnly]="true">
+    </cd-copy-2-clipboard-button>
+  </span>
+</ng-template>
+
+</ng-template>
+
+<cd-table
+  [data]="snapshotSchedules$ | async"
+  columnMode="flex"
+  [columns]="columns"
+  selectionType="single"
+  [hasDetails]="false"
+  (fetchData)="fetchData()"
+  (updateSelection)="updateSelection($event)"
+>
+  <div class="table-actions btn-toolbar">
+    <cd-table-actions
+      [permission]="permissions.cephfs"
+      [selection]="selection"
+      class="btn-group"
+      [tableActions]="tableActions"
+    >
+    </cd-table-actions>
+  </div>
+</cd-table>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.spec.ts
new file mode 100644
index 000000000000..a20972f1cf85
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.spec.ts
@@ -0,0 +1,30 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsSnapshotscheduleListComponent } from './cephfs-snapshotschedule-list.component';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { SharedModule } from '~/app/shared/shared.module';
+import { ToastrModule } from 'ngx-toastr';
+import { RouterTestingModule } from '@angular/router/testing';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { configureTestBed } from '~/testing/unit-test-helper';
+
+describe('CephfsSnapshotscheduleListComponent', () => {
+  let component: CephfsSnapshotscheduleListComponent;
+  let fixture: ComponentFixture<CephfsSnapshotscheduleListComponent>;
+
+  configureTestBed({
+    declarations: [CephfsSnapshotscheduleListComponent],
+    imports: [HttpClientTestingModule, SharedModule, ToastrModule.forRoot(), RouterTestingModule],
+    providers: [NgbActiveModal]
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(CephfsSnapshotscheduleListComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
new file mode 100644
index 000000000000..d5f24e897284
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -0,0 +1,99 @@
+import { Component, Input, OnChanges, OnInit, SimpleChanges, ViewChild } from '@angular/core';
+import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
+import { BehaviorSubject, Observable } from 'rxjs';
+import { finalize, shareReplay, switchMap } from 'rxjs/operators';
+import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
+import { CdForm } from '~/app/shared/forms/cd-form';
+import { CdTableAction } from '~/app/shared/models/cd-table-action';
+import { CdTableColumn } from '~/app/shared/models/cd-table-column';
+import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
+import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
+import { Permissions } from '~/app/shared/models/permissions';
+import { SnapshotSchedule } from '~/app/shared/models/snapshot-schedule';
+import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
+import { ModalService } from '~/app/shared/services/modal.service';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+
+@Component({
+  selector: 'cd-cephfs-snapshotschedule-list',
+  templateUrl: './cephfs-snapshotschedule-list.component.html',
+  styleUrls: ['./cephfs-snapshotschedule-list.component.scss']
+})
+export class CephfsSnapshotscheduleListComponent extends CdForm implements OnInit, OnChanges {
+  @Input() fsName!: string;
+
+  @ViewChild('pathTpl', { static: true })
+  pathTpl: any;
+
+  snapshotSchedules$!: Observable<SnapshotSchedule[]>;
+  subject$ = new BehaviorSubject<SnapshotSchedule[]>([]);
+  isLoading$ = new BehaviorSubject<boolean>(true);
+  columns: CdTableColumn[] = [];
+  tableActions: CdTableAction[] = [];
+  context!: CdTableFetchDataContext;
+  selection = new CdTableSelection();
+  permissions!: Permissions;
+  modalRef!: NgbModalRef;
+  errorMessage: string = '';
+  selectedName: string = '';
+  icons = Icons;
+
+  constructor(
+    private snapshotScheduleService: CephfsSnapshotScheduleService,
+    private authStorageService: AuthStorageService,
+    private modalService: ModalService
+  ) {
+    super();
+    this.permissions = this.authStorageService.getPermissions();
+  }
+
+  ngOnChanges(changes: SimpleChanges): void {
+    if (changes.fsName) {
+      this.subject$.next([]);
+    }
+  }
+
+  ngOnInit(): void {
+    this.snapshotSchedules$ = this.subject$.pipe(
+      switchMap(() =>
+        this.snapshotScheduleService
+          .getSnapshotScheduleList('/', this.fsName)
+          .pipe(finalize(() => this.isLoading$.next(false)))
+      ),
+      shareReplay(1)
+    );
+
+    this.columns = [
+      { prop: 'path', name: $localize`Path`, flexGrow: 3, cellTemplate: this.pathTpl },
+      { prop: 'subvol', name: $localize`Subvolume` },
+      { prop: 'schedule', name: $localize`Repeat interval` },
+      { prop: 'retention', name: $localize`Retention policy` },
+      { prop: 'created_count', name: $localize`Created Count` },
+      { prop: 'pruned_count', name: $localize`Deleted Count` },
+      { prop: 'start', name: $localize`Start time`, cellTransformation: CellTemplate.timeAgo },
+      { prop: 'created', name: $localize`Created`, cellTransformation: CellTemplate.timeAgo }
+    ];
+
+    this.tableActions = [];
+  }
+
+  fetchData() {
+    this.subject$.next([]);
+  }
+
+  updateSelection(selection: CdTableSelection) {
+    this.selection = selection;
+  }
+
+  openModal(edit = false) {
+    this.modalService.show(
+      {},
+      {
+        fsName: 'fs1',
+        isEdit: edit
+      },
+      { size: 'lg' }
+    );
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
index 6a50ad2e0786..87268858be50 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
@@ -1,71 +1,94 @@
 <ng-container *ngIf="selection">
-  <nav ngbNav
-       #nav="ngbNav"
-       (navChange)="softRefresh()"
-       class="nav-tabs"
-       cdStatefulTab="cephfs-tabs">
+  <nav
+    ngbNav
+    #nav="ngbNav"
+    (navChange)="softRefresh()"
+    class="nav-tabs"
+    cdStatefulTab="cephfs-tabs"
+  >
     <ng-container ngbNavItem="details">
-      <a ngbNavLink
-         i18n>Details</a>
+      <a
+        ngbNavLink
+        i18n>Details</a>
       <ng-template ngbNavContent>
-        <cd-cephfs-detail [data]="details">
-        </cd-cephfs-detail>
+        <cd-cephfs-detail [data]="details"> </cd-cephfs-detail>
       </ng-template>
     </ng-container>
     <ng-container ngbNavItem="subvolumes">
-      <a ngbNavLink
-         i18n>Subvolumes</a>
+      <a
+      ngbNavLink
+      i18n>Subvolumes</a>
       <ng-template ngbNavContent>
-        <cd-cephfs-subvolume-list [fsName]="selection.mdsmap.fs_name"
-                                  [pools]="details.pools"></cd-cephfs-subvolume-list>
+        <cd-cephfs-subvolume-list
+          [fsName]="selection.mdsmap.fs_name"
+          [pools]="details.pools"
+        ></cd-cephfs-subvolume-list>
       </ng-template>
     </ng-container>
     <ng-container ngbNavItem="subvolume-groups">
-      <a ngbNavLink
-         i18n>Subvolume groups</a>
+      <a
+      ngbNavLink
+      i18n>Subvolume groups</a>
       <ng-template ngbNavContent>
-        <cd-cephfs-subvolume-group [fsName]="selection.mdsmap.fs_name"
-                                   [pools]="details.pools">
+        <cd-cephfs-subvolume-group
+        [fsName]="selection.mdsmap.fs_name"
+        [pools]="details.pools">
         </cd-cephfs-subvolume-group>
       </ng-template>
     </ng-container>
     <ng-container ngbNavItem="snapshots">
-      <a ngbNavLink
-         i18n>Snapshots</a>
+      <a
+      ngbNavLink
+      i18n>Snapshots</a>
       <ng-template ngbNavContent>
         <cd-cephfs-subvolume-snapshots-list [fsName]="selection.mdsmap.fs_name">
         </cd-cephfs-subvolume-snapshots-list>
       </ng-template>
     </ng-container>
+    <ng-container ngbNavItem="snapshot-schedules">
+      <a
+      ngbNavLink
+      i18n>Snapshot schedules</a>
+      <ng-template ngbNavContent>
+        <cd-cephfs-snapshotschedule-list
+          [fsName]="selection.mdsmap.fs_name"
+        ></cd-cephfs-snapshotschedule-list>
+      </ng-template>
+    </ng-container>
     <ng-container ngbNavItem="clients">
       <a ngbNavLink>
         <ng-container i18n>Clients</ng-container>
         <span class="badge badge-pill badge-tab ms-1">{{ clients.data.length }}</span>
       </a>
       <ng-template ngbNavContent>
-        <cd-cephfs-clients [id]="id"
-                           [clients]="clients"
-                           (triggerApiUpdate)="refresh()">
+        <cd-cephfs-clients
+        [id]="id"
+        [clients]="clients"
+        (triggerApiUpdate)="refresh()">
         </cd-cephfs-clients>
       </ng-template>
     </ng-container>
     <ng-container ngbNavItem="directories">
-      <a ngbNavLink
-         i18n>Directories</a>
+      <a
+      ngbNavLink
+      i18n>Directories</a>
       <ng-template ngbNavContent>
         <cd-cephfs-directories [id]="id"></cd-cephfs-directories>
       </ng-template>
     </ng-container>
     <ng-container ngbNavItem="performance-details">
-      <a ngbNavLink
-         i18n>Performance Details</a>
+      <a
+      ngbNavLink
+      i18n>Performance Details</a>
       <ng-template ngbNavContent>
-        <cd-grafana i18n-title
-                    title="CephFS MDS performance"
-                    [grafanaPath]="'mds-performance?var-mds_servers=mds.' + grafanaId"
-                    [type]="'metrics'"
-                    uid="tbO9LAiZz"
-                    grafanaStyle="one">
+        <cd-grafana
+          i18n-title
+          title="CephFS MDS performance"
+          [grafanaPath]="'mds-performance?var-mds_servers=mds.' + grafanaId"
+          [type]="'metrics'"
+          uid="tbO9LAiZz"
+          grafanaStyle="one"
+        >
         </cd-grafana>
       </ng-template>
     </ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index cbdb2840867a..53544ccd9ed4 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -20,6 +20,8 @@ import { CephfsSubvolumeFormComponent } from './cephfs-subvolume-form/cephfs-sub
 import { CephfsSubvolumeGroupComponent } from './cephfs-subvolume-group/cephfs-subvolume-group.component';
 import { CephfsSubvolumegroupFormComponent } from './cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component';
 import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component';
+import { CephfsSnapshotscheduleListComponent } from './cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component';
+import { DataTableModule } from '../../shared/datatable/datatable.module';
 
 @NgModule({
   imports: [
@@ -32,7 +34,8 @@ import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapsh
     FormsModule,
     ReactiveFormsModule,
     NgbTypeaheadModule,
-    NgbTooltipModule
+    NgbTooltipModule,
+    DataTableModule
   ],
   declarations: [
     CephfsDetailComponent,
@@ -47,7 +50,8 @@ import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapsh
     CephfsDirectoriesComponent,
     CephfsSubvolumeGroupComponent,
     CephfsSubvolumegroupFormComponent,
-    CephfsSubvolumeSnapshotsListComponent
+    CephfsSubvolumeSnapshotsListComponent,
+    CephfsSnapshotscheduleListComponent
   ]
 })
 export class CephfsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.spec.ts
new file mode 100644
index 000000000000..766b8f385b24
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.spec.ts
@@ -0,0 +1,22 @@
+import { TestBed } from '@angular/core/testing';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+
+import { configureTestBed } from '~/testing/unit-test-helper';
+import { CephfsSnapshotScheduleService } from './cephfs-snapshot-schedule.service';
+
+describe('CephfsSnapshotScheduleService', () => {
+  let service: CephfsSnapshotScheduleService;
+
+  configureTestBed({
+    providers: [CephfsSnapshotScheduleService],
+    imports: [HttpClientTestingModule]
+  });
+
+  beforeEach(() => {
+    service = TestBed.inject(CephfsSnapshotScheduleService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
new file mode 100644
index 000000000000..ec9f58c0feec
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -0,0 +1,39 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { Observable } from 'rxjs/internal/Observable';
+import { SnapshotSchedule } from '../models/snapshot-schedule';
+import { map } from 'rxjs/operators';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class CephfsSnapshotScheduleService {
+  baseURL = 'api/cephfs';
+
+  constructor(private http: HttpClient) {}
+
+  getSnapshotScheduleList(
+    path: string,
+    fs: string,
+    recursive = true
+  ): Observable<SnapshotSchedule[]> {
+    return this.http
+      .get<SnapshotSchedule[]>(
+        `${this.baseURL}/snaphost/schedule?path=${path}&fs=${fs}&recursive=${recursive}`
+      )
+      .pipe(
+        map((snapList: SnapshotSchedule[]) =>
+          snapList.map((snapItem: SnapshotSchedule) => ({
+            ...snapItem,
+            status: snapItem.active ? 'Active' : 'Inactive',
+            subvol: snapItem?.subvol || ' - ',
+            retention: Object.values(snapItem.retention)?.length
+              ? Object.entries(snapItem.retention)
+                  ?.map?.(([frequency, interval]) => `${interval}${frequency.toLocaleUpperCase()}`)
+                  .join(' ')
+              : '-'
+          }))
+        )
+      );
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts
new file mode 100644
index 000000000000..b1cea7466f67
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts
@@ -0,0 +1,17 @@
+export interface SnapshotSchedule {
+  fs?: string;
+  subvol?: string;
+  path: string;
+  rel_path?: string;
+  schedule: string;
+  retention?: Record<string, number> | string;
+  start: Date;
+  created: Date;
+  first?: string;
+  last?: string;
+  last_pruned?: string;
+  created_count?: number;
+  pruned_count?: number;
+  active: boolean;
+  status: 'Active' | 'Inactive';
+}
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 758ed13f9c5d..144c29731ddf 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1758,6 +1758,43 @@ paths:
       summary: Rename CephFS Volume
       tags:
       - Cephfs
+  /api/cephfs/snaphost/schedule:
+    get:
+      parameters:
+      - in: query
+        name: fs
+        required: true
+        schema:
+          type: string
+      - default: /
+        in: query
+        name: path
+        schema:
+          type: string
+      - default: true
+        in: query
+        name: recursive
+        schema:
+          type: boolean
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
   /api/cephfs/subvolume:
     post:
       parameters: []
@@ -12859,6 +12896,8 @@ servers:
 tags:
 - description: Initiate a session with Ceph
   name: Auth
+- description: Cephfs Snapshot Scheduling API
+  name: CephFSSnapshotSchedule
 - description: CephFS Subvolume Management API
   name: CephFSSubvolume
 - description: Cephfs Management API

From be507aaed15fb7a193a90c5f88eda61b9be2af1b Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 9 Nov 2023 20:44:18 +0100
Subject: [PATCH 1032/2492] librbd: diff-iterate shouldn't ever report "new
 hole" against a hole

If an object doesn't exist in both start and end versions but there is
an intermediate snapshot which contains it (i.e. the object is written
to and captured at some point but then discarded prior to or in the end
version), diff-iterate reports "new hole" -- callback is invoked with
exists=false.  This occurs both on the slow list_snaps path and in
fast-diff mode.

Despite going all the way back to the introduction of diff-iterate in
commit 0296c7cdae91 ("librbd: implement diff_iterate"), this behavior
is wrong and contradicts diff-iterate API documentation added in commit
a69532e86450 ("librbd: document diff_iterate in header") in the same
series:

    If the source snapshot name is NULL, we interpret that as
    the beginning of time and return all allocated regions of the
    image.

It also triggered an assert added in commit c680531e070a ("librbd:
change diff_iterate interface to be more C-friendly") in the same
series.  Unfortunately, commit f1f6407221a0 ("test_librbd: add
diff_iterate test including discard"), also part of the same series,
added a test which expected the wrong behavior.  Very confusing!

A year later, a different manifestation of this bug was fixed in commit
9a1ab95176fe ("rbd: Fix rbd diff for non-existent objects"), but the
fix only covered the case where calc_snap_set_diff() goes past the end
snap ID while processing clones.  The case where it runs out of clones
to process before reaching the end snap ID remained mishandled.

A year after that, commit 3ccc3bb4bd35 ("librbd: diff_iterate needs to
handle holes in parent images") dropped the assert mentioned above and
this bug got enshrined in the newly introduced fast-diff mode.

Finally, a few years later, deep-copy actually started relying on this
bug in commit e5a21e904142 ("librbd: deep-copy image copy state machine
skips clean objects").  This necessitates bifurcation in DiffRequest
because deep-copy wants the "has this object been touched" semantics,
which is different from diff-iterate (and also potentially much more
expensive to produce!).

This commit brings a minimal update to TestMockObjectMapDiffRequest
tests and DiffIterateTest.DiffIterateDiscard.  Coverage is expanded in
the following commits.

Fixes: https://tracker.ceph.com/issues/53897
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librados/snap_set_diff.cc                 | 25 +++++---
 src/librbd/api/DiffIterate.cc                 |  2 +-
 src/librbd/deep_copy/ImageCopyRequest.cc      |  7 ++-
 src/librbd/object_map/DiffRequest.cc          | 61 +++++++++++++++----
 src/librbd/object_map/DiffRequest.h           | 13 ++--
 src/librbd/object_map/Types.h                 | 15 +++--
 src/test/cli-integration/rbd/snap-diff.t      |  4 ++
 .../deep_copy/test_mock_ImageCopyRequest.cc   |  1 +
 .../object_map/test_mock_DiffRequest.cc       | 45 +++++++++-----
 src/test/librbd/test_librbd.cc                |  4 ++
 10 files changed, 127 insertions(+), 50 deletions(-)

diff --git a/src/librados/snap_set_diff.cc b/src/librados/snap_set_diff.cc
index f80105b44ae1..0029bcd64780 100644
--- a/src/librados/snap_set_diff.cc
+++ b/src/librados/snap_set_diff.cc
@@ -31,9 +31,8 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
   *clone_end_snap_id = 0;
   *whole_object = false;
 
-  for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
-       r != snap_set.clones.end();
-       ) {
+  auto r = snap_set.clones.begin();
+  while (r != snap_set.clones.end()) {
     // make an interval, and hide the fact that the HEAD doesn't
     // include itself in the snaps list
     librados::snap_t a, b;
@@ -77,12 +76,6 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
     }
 
     if (end < a) {
-      ldout(cct, 20) << " past end " << end << ", end object does not exist" << dendl;
-      *end_exists = false;
-      diff->clear();
-      if (start_size) {
-	diff->insert(0, start_size);
-      }
       break;
     }
     if (end <= b) {
@@ -90,7 +83,7 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
       *end_size = r->size;
       *end_exists = true;
       *clone_end_snap_id = b;
-      break;
+      return;
     }
 
     // start with the max(this size, next size), and subtract off any
@@ -114,4 +107,16 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
     diff->union_of(diff_to_next);
     ldout(cct, 20) << "  diff now " << *diff << dendl;
   }
+
+  if (r != snap_set.clones.end()) {
+    ldout(cct, 20) << " past end " << end
+                   << ", end object does not exist" << dendl;
+  } else {
+    ldout(cct, 20) << " ran out of clones before reaching end " << end
+                   << ", end object does not exist" << dendl;
+  }
+  diff->clear();
+  if (start_size) {
+    diff->insert(0, start_size);
+  }
 }
diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index b400b5d5a406..1b6b3f72148d 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -249,7 +249,7 @@ int DiffIterate<I>::execute() {
   if (m_whole_object) {
     C_SaferCond ctx;
     auto req = object_map::DiffRequest<I>::create(&m_image_ctx, from_snap_id,
-                                                  end_snap_id,
+                                                  end_snap_id, true,
                                                   &object_diff_state, &ctx);
     req->send();
 
diff --git a/src/librbd/deep_copy/ImageCopyRequest.cc b/src/librbd/deep_copy/ImageCopyRequest.cc
index 9cce590ec183..0ba9bc82f18d 100644
--- a/src/librbd/deep_copy/ImageCopyRequest.cc
+++ b/src/librbd/deep_copy/ImageCopyRequest.cc
@@ -108,9 +108,10 @@ void ImageCopyRequest<I>::compute_diff() {
 
   auto ctx = create_context_callback<
     ImageCopyRequest<I>, &ImageCopyRequest<I>::handle_compute_diff>(this);
-  auto req = object_map::DiffRequest<I>::create(m_src_image_ctx, m_src_snap_id_start,
-                                                m_src_snap_id_end, &m_object_diff_state,
-                                                ctx);
+  auto req = object_map::DiffRequest<I>::create(m_src_image_ctx,
+                                                m_src_snap_id_start,
+                                                m_src_snap_id_end, false,
+                                                &m_object_diff_state, ctx);
   req->send();
 }
 
diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 606d48bbf33c..8820aacc87bb 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -195,16 +195,34 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   for (; it != overlap_end_it; ++it, ++diff_it, ++i) {
     uint8_t object_map_state = *it;
     uint8_t prev_object_diff_state = *diff_it;
-    if (object_map_state == OBJECT_EXISTS ||
-        object_map_state == OBJECT_PENDING ||
-        (object_map_state == OBJECT_EXISTS_CLEAN &&
-         prev_object_diff_state != DIFF_STATE_DATA &&
-         prev_object_diff_state != DIFF_STATE_DATA_UPDATED)) {
-      *diff_it = DIFF_STATE_DATA_UPDATED;
-    } else if (object_map_state == OBJECT_NONEXISTENT &&
-               prev_object_diff_state != DIFF_STATE_HOLE &&
-               prev_object_diff_state != DIFF_STATE_HOLE_UPDATED) {
-      *diff_it = DIFF_STATE_HOLE_UPDATED;
+    switch (prev_object_diff_state) {
+    case DIFF_STATE_HOLE:
+      if (object_map_state != OBJECT_NONEXISTENT) {
+        // stay in HOLE on intermediate snapshots for diff-iterate
+        if (!m_diff_iterate_range || m_current_snap_id == m_snap_id_end) {
+          *diff_it = DIFF_STATE_DATA_UPDATED;
+        }
+      }
+      break;
+    case DIFF_STATE_DATA:
+      if (object_map_state == OBJECT_NONEXISTENT) {
+        *diff_it = DIFF_STATE_HOLE_UPDATED;
+      } else if (object_map_state != OBJECT_EXISTS_CLEAN) {
+        *diff_it = DIFF_STATE_DATA_UPDATED;
+      }
+      break;
+    case DIFF_STATE_HOLE_UPDATED:
+      if (object_map_state != OBJECT_NONEXISTENT) {
+        *diff_it = DIFF_STATE_DATA_UPDATED;
+      }
+      break;
+    case DIFF_STATE_DATA_UPDATED:
+      if (object_map_state == OBJECT_NONEXISTENT) {
+        *diff_it = DIFF_STATE_HOLE_UPDATED;
+      }
+      break;
+    default:
+      ceph_abort();
     }
 
     ldout(cct, 20) << "object state: " << i << " "
@@ -225,8 +243,29 @@ void DiffRequest<I>::handle_load_object_map(int r) {
       } else if (diff_from_start ||
                  (m_object_diff_state_valid &&
                   object_map_state != OBJECT_EXISTS_CLEAN)) {
-        *diff_it = DIFF_STATE_DATA_UPDATED;
+        // diffing against the beginning of time or image was grown
+        // (implicit) starting state is HOLE, this is the first object
+        // map after
+        if (m_diff_iterate_range) {
+          // for diff-iterate, if the object is discarded prior to or
+          // in the end version, result should be HOLE
+          // since DATA_UPDATED can transition only to HOLE_UPDATED,
+          // stay in HOLE on intermediate snapshots -- another way to
+          // put this is that when starting with a hole, intermediate
+          // snapshots can be ignored as the result depends only on the
+          // end version
+          if (m_current_snap_id == m_snap_id_end) {
+            *diff_it = DIFF_STATE_DATA_UPDATED;
+          } else {
+            *diff_it = DIFF_STATE_HOLE;
+          }
+        } else {
+          // for deep-copy, if the object is discarded prior to or
+          // in the end version, result should be HOLE_UPDATED
+          *diff_it = DIFF_STATE_DATA_UPDATED;
+        }
       } else {
+        // diffing against a snapshot, this is its object map
         *diff_it = DIFF_STATE_DATA;
       }
 
diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
index e83a1629e623..b02ef34ba347 100644
--- a/src/librbd/object_map/DiffRequest.h
+++ b/src/librbd/object_map/DiffRequest.h
@@ -22,19 +22,19 @@ template <typename ImageCtxT>
 class DiffRequest {
 public:
   static DiffRequest* create(ImageCtxT* image_ctx, uint64_t snap_id_start,
-                             uint64_t snap_id_end,
+                             uint64_t snap_id_end, bool diff_iterate_range,
                              BitVector<2>* object_diff_state,
                              Context* on_finish) {
     return new DiffRequest(image_ctx, snap_id_start, snap_id_end,
-                           object_diff_state, on_finish);
+                           diff_iterate_range, object_diff_state, on_finish);
   }
 
   DiffRequest(ImageCtxT* image_ctx, uint64_t snap_id_start,
-              uint64_t snap_id_end, BitVector<2>* object_diff_state,
-              Context* on_finish)
+              uint64_t snap_id_end, bool diff_iterate_range,
+              BitVector<2>* object_diff_state, Context* on_finish)
     : m_image_ctx(image_ctx), m_snap_id_start(snap_id_start),
-      m_snap_id_end(snap_id_end), m_object_diff_state(object_diff_state),
-      m_on_finish(on_finish) {
+      m_snap_id_end(snap_id_end), m_diff_iterate_range(diff_iterate_range),
+      m_object_diff_state(object_diff_state), m_on_finish(on_finish) {
   }
 
   void send();
@@ -58,6 +58,7 @@ class DiffRequest {
   ImageCtxT* m_image_ctx;
   uint64_t m_snap_id_start;
   uint64_t m_snap_id_end;
+  bool m_diff_iterate_range;
   BitVector<2>* m_object_diff_state;
   Context* m_on_finish;
 
diff --git a/src/librbd/object_map/Types.h b/src/librbd/object_map/Types.h
index 0ce91bd96a1c..576ea0e4b6b7 100644
--- a/src/librbd/object_map/Types.h
+++ b/src/librbd/object_map/Types.h
@@ -8,10 +8,17 @@ namespace librbd {
 namespace object_map {
 
 enum DiffState {
-  DIFF_STATE_HOLE         = 0, /* unchanged hole */
-  DIFF_STATE_DATA         = 1, /* unchanged data */
-  DIFF_STATE_HOLE_UPDATED = 2, /* new hole */
-  DIFF_STATE_DATA_UPDATED = 3  /* new data */
+  // diff-iterate: hole with or without data captured in intermediate snapshot
+  // deep-copy: hole without data captured in intermediate snapshot
+  DIFF_STATE_HOLE         = 0,
+  // diff-iterate, deep-copy: unchanged data
+  DIFF_STATE_DATA         = 1,
+  // diff-iterate: new hole (data -> hole)
+  // deep-copy: new hole (data -> hole) or hole with data captured in
+  //            intermediate snapshot
+  DIFF_STATE_HOLE_UPDATED = 2,
+  // diff-iterate, deep-copy: new data (hole -> data) or changed data
+  DIFF_STATE_DATA_UPDATED = 3
 };
 
 } // namespace object_map
diff --git a/src/test/cli-integration/rbd/snap-diff.t b/src/test/cli-integration/rbd/snap-diff.t
index 1ca2fb04ddd9..fa564891a4b9 100644
--- a/src/test/cli-integration/rbd/snap-diff.t
+++ b/src/test/cli-integration/rbd/snap-diff.t
@@ -39,10 +39,14 @@
   $ rbd diff --from-snap=snap1 xrbddiff1/xtestdiff1 --format json
   []
   $ rbd snap rollback xrbddiff1/xtestdiff1@snap1 --no-progress
+  $ rbd diff --from-snap=allzeroes xrbddiff1/xtestdiff1 --format json
+  [{"offset":0,"length":1048576,"exists":"true"}]
   $ rbd diff --from-snap=snap1 xrbddiff1/xtestdiff1 --format json
   []
   $ rbd snap rollback xrbddiff1/xtestdiff1@allzeroes --no-progress
   $ rbd diff --from-snap=allzeroes xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd diff --from-snap=snap1 xrbddiff1/xtestdiff1 --format json
   [{"offset":0,"length":1048576,"exists":"false"}]
   $ ceph osd pool rm xrbddiff1 xrbddiff1 --yes-i-really-really-mean-it
   pool 'xrbddiff1' removed
diff --git a/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc b/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
index e38ffffdbe49..634cabfa79e3 100644
--- a/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
+++ b/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
@@ -92,6 +92,7 @@ struct DiffRequest<MockTestImageCtx> {
   static DiffRequest* s_instance;
   static DiffRequest* create(MockTestImageCtx *image_ctx,
                              uint64_t snap_id_start, uint64_t snap_id_end,
+                             bool diff_iterate_range,
                              BitVector<2>* object_diff_state,
                              Context* on_finish) {
     ceph_assert(s_instance != nullptr);
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index c25ae4a95c5e..85fe456d929e 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -42,6 +42,10 @@ class TestMockObjectMapDiffRequest : public TestMockFixture {
     ASSERT_EQ(0, open_image(m_image_name, &m_image_ctx));
   }
 
+  bool is_diff_iterate() const {
+    return true;
+  }
+
   void expect_get_flags(MockTestImageCtx& mock_image_ctx, uint64_t snap_id,
                         int32_t flags, int r) {
     EXPECT_CALL(mock_image_ctx, get_flags(snap_id, _))
@@ -87,7 +91,8 @@ TEST_F(TestMockObjectMapDiffRequest, InvalidStartSnap) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, CEPH_NOSNAP, 0,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -98,7 +103,7 @@ TEST_F(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
   InSequence seq;
 
   C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 1,
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, 1, is_diff_iterate(),
                                  &m_object_diff_state, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -115,7 +120,8 @@ TEST_F(TestMockObjectMapDiffRequest, FastDiffDisabled) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -133,7 +139,8 @@ TEST_F(TestMockObjectMapDiffRequest, FastDiffInvalid) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -180,7 +187,8 @@ TEST_F(TestMockObjectMapDiffRequest, FullDelta) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
@@ -188,7 +196,7 @@ TEST_F(TestMockObjectMapDiffRequest, FullDelta) {
   expected_diff_state.resize(object_count);
   expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
   expected_diff_state[2] = DIFF_STATE_DATA_UPDATED;
-  expected_diff_state[3] = DIFF_STATE_HOLE_UPDATED;
+  expected_diff_state[3] = DIFF_STATE_HOLE;
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
@@ -226,7 +234,7 @@ TEST_F(TestMockObjectMapDiffRequest, IntermediateDelta) {
   expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
 
   C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2,
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2, is_diff_iterate(),
                                  &m_object_diff_state, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -274,7 +282,8 @@ TEST_F(TestMockObjectMapDiffRequest, EndDelta) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 2, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
@@ -302,7 +311,8 @@ TEST_F(TestMockObjectMapDiffRequest, StartSnapDNE) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
@@ -328,7 +338,7 @@ TEST_F(TestMockObjectMapDiffRequest, EndSnapDNE) {
   expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
 
   C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2,
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2, is_diff_iterate(),
                                  &m_object_diff_state, &ctx);
   req->send();
   ASSERT_EQ(-ENOENT, ctx.wait());
@@ -367,7 +377,8 @@ TEST_F(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
@@ -394,7 +405,8 @@ TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
@@ -427,7 +439,8 @@ TEST_F(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
@@ -458,7 +471,8 @@ TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapError) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-EPERM, ctx.wait());
 }
@@ -484,7 +498,8 @@ TEST_F(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
 
   C_SaferCond ctx;
   auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 &m_object_diff_state, &ctx);
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index d9e43b5e7364..e90569d68c6f 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7780,6 +7780,10 @@ TYPED_TEST(DiffIterateTest, DiffIterateDiscard)
   extents.clear();
   ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
       			           vector_iterate_cb, (void *) &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, (void *) &extents));
   ASSERT_EQ(1u, extents.size());
   ASSERT_EQ(diff_extent(0, 256, false, object_size), extents[0]);
   ASSERT_PASSED(this->validate_object_map, image);

From bcb107afcd2b0e34b1eefa4767691c5d8d109a82 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 10 Nov 2023 11:14:42 +0100
Subject: [PATCH 1033/2492] librbd: resurrect "exists" assert in
 simple_diff_cb()

This effectively reverts commit 3ccc3bb4bd35 ("librbd: diff_iterate
needs to handle holes in parent images") which just dropped the assert
instead of addressing the root cause of reported crashes.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/api/DiffIterate.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index 1b6b3f72148d..7679bf77759d 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -150,11 +150,12 @@ class C_DiffObject : public Context {
 };
 
 int simple_diff_cb(uint64_t off, size_t len, int exists, void *arg) {
-  // it's possible for a discard to create a hole in the parent image -- ignore
-  if (exists) {
-    interval_set<uint64_t> *diff = static_cast<interval_set<uint64_t> *>(arg);
-    diff->insert(off, len);
-  }
+  // This reads the existing extents in a parent from the beginning
+  // of time.  Since images are thin-provisioned, the extents will
+  // always represent data, not holes.
+  ceph_assert(exists);
+  auto diff = static_cast<interval_set<uint64_t>*>(arg);
+  diff->insert(off, len);
   return 0;
 }
 

From 6b45a2d920aad2edf17544807f7032e2b11efef3 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 30 Nov 2023 17:30:33 +0100
Subject: [PATCH 1034/2492] test/librbd: expand
 DiffIterateTest.DiffIterateDiscard

Similar to DiffIterateTest.DiffIterateDeterministic, systematically
cover the most common cases involving full-object discards.  With this
in place, issue [1] can be reproduced by any of:

    (preparatory) before snap3 is taken
    (1) beginning of time -> HEAD
    (2) snap1 -> HEAD
    (5) beginning of time -> snap3
    (6) snap1 -> snap3

Sub-object discards aren't covered here because of further issues
[2][3].

[1] https://tracker.ceph.com/issues/53897
[2] https://tracker.ceph.com/issues/63770
[3] https://tracker.ceph.com/issues/63771

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 120 ++++++++++++++++++++++++++-------
 1 file changed, 94 insertions(+), 26 deletions(-)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index e90569d68c6f..77808b9721a8 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7715,12 +7715,14 @@ TYPED_TEST(DiffIterateTest, DiffIterateDeterministicPP)
 
 TYPED_TEST(DiffIterateTest, DiffIterateDiscard)
 {
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_STRIPINGV2));
+
   librados::IoCtx ioctx;
   ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
 
   librbd::RBD rbd;
   librbd::Image image;
-  int order = 0;
+  int order = 22;
   std::string name = this->get_temp_image_name();
   uint64_t size = 20 << 20;
 
@@ -7731,61 +7733,127 @@ TYPED_TEST(DiffIterateTest, DiffIterateDiscard)
   if (this->whole_object) {
     object_size = 1 << order;
   }
-  vector<diff_extent> extents;
-  ceph::bufferlist bl;
 
+  std::vector<diff_extent> extents;
   ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
+                                   vector_iterate_cb, &extents));
   ASSERT_EQ(0u, extents.size());
 
-  char data[256];
-  memset(data, 1, sizeof(data));
-  bl.append(data, 256);
+  ceph::bufferlist bl;
+  bl.append(std::string(256, '1'));
   ASSERT_EQ(256, image.write(0, 256, bl));
+  ASSERT_EQ(256, image.write(1 << order, 256, bl));
   ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(1u, extents.size());
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
   ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
-
-  int obj_ofs = 256;
-  ASSERT_EQ(1 << order, image.discard(0, 1 << order));
-
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
   extents.clear();
+
+  ASSERT_EQ(size, image.discard(0, size));
   ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
+                                   vector_iterate_cb, &extents));
   ASSERT_EQ(0u, extents.size());
 
   ASSERT_EQ(0, image.snap_create("snap1"));
+
   ASSERT_EQ(256, image.write(0, 256, bl));
+  ASSERT_EQ(256, image.write(1 << order, 256, bl));
   ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(1u, extents.size());
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
   ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
   ASSERT_EQ(0, image.snap_create("snap2"));
 
-  ASSERT_EQ(obj_ofs, image.discard(0, obj_ofs));
+  ASSERT_EQ(1 << order, image.discard(0, 1 << order));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(0, image.snap_create("snap3"));
 
+  // 1. beginning of time -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
   extents.clear();
-  ASSERT_EQ(0, image.snap_set("snap2"));
+
+  // 2. snap1 -> HEAD
   ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
+                                   vector_iterate_cb, &extents));
   ASSERT_EQ(1u, extents.size());
-  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
 
-  ASSERT_EQ(0, image.snap_set(NULL));
-  ASSERT_EQ(1 << order, image.discard(0, 1 << order));
-  ASSERT_EQ(0, image.snap_create("snap3"));
+  // 3. snap2 -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, false, object_size), extents[0]);
+  extents.clear();
+
+  // 4. snap3 -> HEAD
+  ASSERT_EQ(0, image.diff_iterate2("snap3", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_PASSED(this->validate_object_map, image);
   ASSERT_EQ(0, image.snap_set("snap3"));
 
+  // 5. beginning of time -> snap3
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
   extents.clear();
+
+  // 6. snap1 -> snap3
   ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
-      			           vector_iterate_cb, (void *) &extents));
-  ASSERT_EQ(0u, extents.size());
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[0]);
+  extents.clear();
 
+  // 7. snap2 -> snap3
   ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
-                                   vector_iterate_cb, (void *) &extents));
+                                   vector_iterate_cb, &extents));
   ASSERT_EQ(1u, extents.size());
   ASSERT_EQ(diff_extent(0, 256, false, object_size), extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, image.snap_set("snap2"));
+
+  // 8. beginning of time -> snap2
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  // 9. snap1 -> snap2
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(2u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+  ASSERT_EQ(diff_extent(1 << order, 256, true, object_size), extents[1]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+  ASSERT_EQ(0, image.snap_set("snap1"));
+
+  // 10. beginning of time -> snap1
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
   ASSERT_PASSED(this->validate_object_map, image);
 }
 

From 70c1991f225c137abf769111086b6c61abc0ece3 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 8 Dec 2023 15:19:02 +0100
Subject: [PATCH 1035/2492] librbd: OBJECT_PENDING should always be treated as
 dirty

OBJECT_PENDING is a transition state which normally isn't encountered
in (snapshot) object maps.  In case it's encountered, for example when
a snapshot is taken after losing power at the time a discard was being
handled, the object should be treated as dirty and produce a diff as
a result.

Assuming an object is marked OBJECT_PENDING, theoretically there are
four cases with respect to object's state in the next snapshot:

    1. OBJECT_NONEXISTENT
    2. OBJECT_EXISTS
    3. OBJECT_PENDING
    4. OBJECT_EXISTS_CLEAN

Prior to commit b81cd2460de7 ("librbd/object_map: diff state machine
should track object existence"), (3) was handled incorrectly (diff set
to DIFF_STATE_NONE instead of DIFF_STATE_UPDATED).

Post commit 399a45e11332 ("librbd/object_map: rbd diff between two
snapshots lists entire image content"), (4) is handled incorrectly
(diff set to DIFF_STATE_DATA instead of DIFF_STATE_DATA_UPDATED).

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 8820aacc87bb..5b785aeaf3c7 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -266,7 +266,11 @@ void DiffRequest<I>::handle_load_object_map(int r) {
         }
       } else {
         // diffing against a snapshot, this is its object map
-        *diff_it = DIFF_STATE_DATA;
+        if (object_map_state != OBJECT_PENDING) {
+          *diff_it = DIFF_STATE_DATA;
+        } else {
+          *diff_it = DIFF_STATE_DATA_UPDATED;
+        }
       }
 
       ldout(cct, 20) << "object state: " << i << " "

From f47f7f87d05da661786415dd8fbb0ac67c530e18 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sun, 3 Dec 2023 14:49:43 +0100
Subject: [PATCH 1036/2492] test/librbd: parametrize
 TestMockObjectMapDiffRequest tests

Exercise both diff-iterate and deep-copy modes of operation.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../object_map/test_mock_DiffRequest.cc       | 36 ++++++++++---------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 85fe456d929e..440040e9a835 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -32,7 +32,8 @@ using ::testing::WithArg;
 namespace librbd {
 namespace object_map {
 
-class TestMockObjectMapDiffRequest : public TestMockFixture {
+class TestMockObjectMapDiffRequest : public TestMockFixture,
+                                     public ::testing::WithParamInterface<bool> {
 public:
   typedef DiffRequest<MockTestImageCtx> MockDiffRequest;
 
@@ -43,7 +44,7 @@ class TestMockObjectMapDiffRequest : public TestMockFixture {
   }
 
   bool is_diff_iterate() const {
-    return true;
+    return GetParam();
   }
 
   void expect_get_flags(MockTestImageCtx& mock_image_ctx, uint64_t snap_id,
@@ -84,7 +85,7 @@ class TestMockObjectMapDiffRequest : public TestMockFixture {
   BitVector<2> m_object_diff_state;
 };
 
-TEST_F(TestMockObjectMapDiffRequest, InvalidStartSnap) {
+TEST_P(TestMockObjectMapDiffRequest, InvalidStartSnap) {
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
 
   InSequence seq;
@@ -97,7 +98,7 @@ TEST_F(TestMockObjectMapDiffRequest, InvalidStartSnap) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
+TEST_P(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
 
   InSequence seq;
@@ -110,7 +111,7 @@ TEST_F(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
   ASSERT_EQ(0U, m_object_diff_state.size());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, FastDiffDisabled) {
+TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   // negative test -- object-map implicitly enables fast-diff
   REQUIRE(!is_feature_enabled(RBD_FEATURE_OBJECT_MAP));
 
@@ -126,7 +127,7 @@ TEST_F(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, FastDiffInvalid) {
+TEST_P(TestMockObjectMapDiffRequest, FastDiffInvalid) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
@@ -145,7 +146,7 @@ TEST_F(TestMockObjectMapDiffRequest, FastDiffInvalid) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, FullDelta) {
+TEST_P(TestMockObjectMapDiffRequest, FullDelta) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -200,7 +201,7 @@ TEST_F(TestMockObjectMapDiffRequest, FullDelta) {
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_F(TestMockObjectMapDiffRequest, IntermediateDelta) {
+TEST_P(TestMockObjectMapDiffRequest, IntermediateDelta) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -247,7 +248,7 @@ TEST_F(TestMockObjectMapDiffRequest, IntermediateDelta) {
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_F(TestMockObjectMapDiffRequest, EndDelta) {
+TEST_P(TestMockObjectMapDiffRequest, EndDelta) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -295,7 +296,7 @@ TEST_F(TestMockObjectMapDiffRequest, EndDelta) {
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_F(TestMockObjectMapDiffRequest, StartSnapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -317,7 +318,7 @@ TEST_F(TestMockObjectMapDiffRequest, StartSnapDNE) {
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, EndSnapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -344,7 +345,7 @@ TEST_F(TestMockObjectMapDiffRequest, EndSnapDNE) {
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -388,7 +389,7 @@ TEST_F(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -411,7 +412,7 @@ TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -450,7 +451,7 @@ TEST_F(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapError) {
+TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapError) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -477,7 +478,7 @@ TEST_F(TestMockObjectMapDiffRequest, LoadObjectMapError) {
   ASSERT_EQ(-EPERM, ctx.wait());
 }
 
-TEST_F(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
+TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -504,5 +505,8 @@ TEST_F(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
+INSTANTIATE_TEST_SUITE_P(MockObjectMapDiffRequestTests,
+                         TestMockObjectMapDiffRequest, ::testing::Bool());
+
 } // namespace object_map
 } // librbd

From c3ad41ba3588329b935dc67d14ee8d1d52424e1a Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Sun, 10 Dec 2023 15:10:19 -0500
Subject: [PATCH 1037/2492] rgw/common: add rgw lifecycle specific debug log
 subsystem

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 doc/rados/troubleshooting/log-and-debug.rst          | 2 ++
 qa/suites/crimson-rados/perf/settings/optimized.yaml | 1 +
 qa/suites/perf-basic/settings/optimized.yaml         | 1 +
 qa/suites/rados/perf/settings/optimized.yaml         | 1 +
 qa/suites/rgw/multisite/overrides.yaml               | 1 +
 src/common/subsys.h                                  | 1 +
 src/rgw/rgw_lc.cc                                    | 3 ++-
 src/rgw/rgw_lc_s3.cc                                 | 2 +-
 8 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
index 7d2885522132..3c44d120656c 100644
--- a/doc/rados/troubleshooting/log-and-debug.rst
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -300,6 +300,8 @@ to their default level or to a level suitable for normal operations.
 +--------------------------+-----------+--------------+
 | ``rgw dbstore``          |     1     |      5       |
 +--------------------------+-----------+--------------+
+| ``rgw lifecycle``        |     1     |      5       |
++--------------------------+-----------+--------------+
 | ``javaclient``           |     1     |      5       |
 +--------------------------+-----------+--------------+
 | ``asok``                 |     1     |      5       |
diff --git a/qa/suites/crimson-rados/perf/settings/optimized.yaml b/qa/suites/crimson-rados/perf/settings/optimized.yaml
index dc4dcbb969e5..deefc61d3d83 100644
--- a/qa/suites/crimson-rados/perf/settings/optimized.yaml
+++ b/qa/suites/crimson-rados/perf/settings/optimized.yaml
@@ -54,6 +54,7 @@ overrides:
         debug perfcounter: "0/0"
         debug rgw: "0/0"
         debug rgw sync: "0/0"
+        debug rgw lifecycle: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/perf-basic/settings/optimized.yaml b/qa/suites/perf-basic/settings/optimized.yaml
index 31d8cb3dc9f7..3983ed9d26e6 100644
--- a/qa/suites/perf-basic/settings/optimized.yaml
+++ b/qa/suites/perf-basic/settings/optimized.yaml
@@ -58,6 +58,7 @@ overrides:
         debug perfcounter: "0/0"
         debug rgw: "0/0"
         debug rgw sync: "0/0"
+        debug rgw lifecycle: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/rados/perf/settings/optimized.yaml b/qa/suites/rados/perf/settings/optimized.yaml
index 3eb6a506062c..8f05f59163ac 100644
--- a/qa/suites/rados/perf/settings/optimized.yaml
+++ b/qa/suites/rados/perf/settings/optimized.yaml
@@ -54,6 +54,7 @@ overrides:
         debug perfcounter: "0/0"
         debug rgw: "0/0"
         debug rgw sync: "0/0"
+        debug rgw lifecycle: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/rgw/multisite/overrides.yaml b/qa/suites/rgw/multisite/overrides.yaml
index a1559e1fe1aa..8f0d376f7d82 100644
--- a/qa/suites/rgw/multisite/overrides.yaml
+++ b/qa/suites/rgw/multisite/overrides.yaml
@@ -6,6 +6,7 @@ overrides:
         setgroup: ceph
         debug rgw: 20
         debug rgw sync: 20
+        debug rgw lifecycle: 20
         rgw crypt s3 kms backend: testing
         rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=
         rgw crypt require ssl: false
diff --git a/src/common/subsys.h b/src/common/subsys.h
index 4892d202cf56..d52018c880dd 100644
--- a/src/common/subsys.h
+++ b/src/common/subsys.h
@@ -64,6 +64,7 @@ SUBSYS(rgw_datacache, 1, 5)
 SUBSYS(rgw_access, 1, 5)
 SUBSYS(rgw_dbstore, 1, 5)
 SUBSYS(rgw_flight, 1, 5)
+SUBSYS(rgw_lifecycle, 1, 5)
 SUBSYS(javaclient, 1, 5)
 SUBSYS(asok, 1, 5)
 SUBSYS(throttle, 1, 1)
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 2bef2ef31c39..09f6a4b86e63 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -39,7 +39,8 @@
 #include "services/svc_tier_rados.h"
 
 #define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_rgw
+#define dout_subsys ceph_subsys_rgw_lifecycle
+
 
 using namespace std;
 
diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index cf152b84a84c..1e1e16b7147f 100644
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -12,7 +12,7 @@
 #include "rgw_lc_s3.h"
 
 
-#define dout_subsys ceph_subsys_rgw
+#define dout_subsys ceph_subsys_rgw_lifecycle
 
 using namespace std;
 

From 5939cebdd97ae4832fdf87db413d656c0cec60e5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 12 Nov 2021 12:59:37 -0500
Subject: [PATCH 1038/2492] mds: dump sr objects in debug output

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CInode.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 621d8e93e407..3a12eba1714b 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -3128,8 +3128,8 @@ void CInode::open_snaprealm(bool nosplit)
     SnapRealm *parent = find_snaprealm();
     snaprealm = new SnapRealm(mdcache, this);
     if (parent) {
-      dout(10) << __func__ << " " << snaprealm
-	       << " parent is " << parent
+      dout(10) << __func__ << " " << *snaprealm
+	       << " parent is " << *parent
 	       << dendl;
       dout(30) << " siblings are " << parent->open_children << dendl;
       snaprealm->parent = parent;

From 4d8c6519b374d14c8c367cebb9228387cbdeaa80 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Tue, 12 Dec 2023 10:33:19 +0800
Subject: [PATCH 1039/2492] crimson: add set_val and rm_val to global conf.

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/crimson/common/config_proxy.h |  7 +++++++
 src/global/global_context.cc      | 29 +++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/src/crimson/common/config_proxy.h b/src/crimson/common/config_proxy.h
index 822db34f61a4..b04fbee2e8a7 100644
--- a/src/crimson/common/config_proxy.h
+++ b/src/crimson/common/config_proxy.h
@@ -14,6 +14,11 @@ namespace ceph {
 class Formatter;
 }
 
+namespace ceph::global {
+int g_conf_set_val(const std::string& key, const std::string& s);
+int g_conf_rm_val(const std::string& key);
+}
+
 namespace crimson::common {
 
 // a facade for managing config. each shard has its own copy of ConfigProxy.
@@ -128,6 +133,7 @@ class ConfigProxy : public seastar::peering_sharded_service<ConfigProxy>
     obs_mgr.remove_observer(obs);
   }
   seastar::future<> rm_val(const std::string& key) {
+    ceph::global::g_conf_rm_val(key);
     return do_change([key, this](ConfigValues& values) {
       auto ret = get_config().rm_val(values, key);
       if (ret < 0) {
@@ -137,6 +143,7 @@ class ConfigProxy : public seastar::peering_sharded_service<ConfigProxy>
   }
   seastar::future<> set_val(const std::string& key,
 			    const std::string& val) {
+    ceph::global::g_conf_set_val(key, val);
     return do_change([key, val, this](ConfigValues& values) {
       std::stringstream err;
       auto ret = get_config().set_val(values, obs_mgr, key, val, &err);
diff --git a/src/global/global_context.cc b/src/global/global_context.cc
index b1e37bfbe595..0fea21d558ae 100644
--- a/src/global/global_context.cc
+++ b/src/global/global_context.cc
@@ -20,6 +20,17 @@
 #include "crimson/common/config_proxy.h"
 #endif
 
+#if defined(WITH_SEASTAR) && !defined(WITH_ALIEN)
+namespace ceph::global {
+int __attribute__((weak)) g_conf_set_val(const std::string& key, const std::string& s) {
+  return 0;
+}
+
+int __attribute__((weak)) g_conf_rm_val(const std::string& key) {
+  return 0;
+}
+}
+#endif
 
 /*
  * Global variables for use from process context.
@@ -34,6 +45,24 @@ ConfigProxy& g_conf() {
 #endif
 }
 
+#ifdef WITH_ALIEN
+int g_conf_set_val(const std::string& key, const std::string& s)
+{
+  if (g_ceph_context != NULL)
+    return g_ceph_context->_conf.set_val(key, s);
+
+  return 0;
+}
+
+int g_conf_rm_val(const std::string& key)
+{
+  if (g_ceph_context != NULL)
+    return g_ceph_context->_conf.rm_val(key);
+
+  return 0;
+}
+#endif
+
 const char *g_assert_file = 0;
 int g_assert_line = 0;
 const char *g_assert_func = 0;

From fa6ad94f3afc6d11c3ed243d66c96a666e973bde Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 12 Dec 2023 22:09:13 +1000
Subject: [PATCH 1040/2492] doc/radosgw: format "Create a Role"

Properly format the section "Create a Role" in the file
doc/radosgw/role.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/role.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
index 029d8b778d63..1e009df21ab0 100644
--- a/doc/radosgw/role.rst
+++ b/doc/radosgw/role.rst
@@ -15,7 +15,7 @@ update a role and the permissions associated with it.
 Create a Role
 -------------
 
-To create a role, execute the following::
+To create a role, run a command of the following form::
 
 	radosgw-admin role create --role-name={role-name} [--path=="{path to the role}"] [--assume-role-policy-doc={trust-policy-document}]
 
@@ -29,12 +29,13 @@ Request Parameters
 
 ``path``
 
-:Description: Path to the role. The default value is a slash(/).
+:Description: Path to the role. The default value is a slash(``/``).
 :Type: String
 
 ``assume-role-policy-doc``
 
-:Description: The trust relationship policy document that grants an entity permission to assume the role.
+:Description: The trust relationship policy document that grants an entity
+              permission to assume the role.
 :Type: String
 
 For example::

From fd3be76ecc2c6b1f436cda19ba961f8f45014b02 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Tue, 12 Dec 2023 16:01:11 +0300
Subject: [PATCH 1041/2492] test/store_test: fix deferred writing test cases

When bitmap allocator is in use different space fragmentation and
allocation strategy cause blob fragmentation. Which in turn impacts the
decision on deferred write applicability.

Fixing by using standalone DB volume to reduce main device space fragmentation.

Fixes: https://tracker.ceph.com/issues/63795
Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/test/objectstore/store_test.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 03dc1a87e1fc..8f85c1a4f1a2 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -8293,6 +8293,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) {
   }
 
   size_t block_size = 4096;
+  SetVal(g_conf(), "bluestore_block_db_create", "true");
+  SetVal(g_conf(), "bluestore_block_db_size", stringify(1 << 30).c_str());
   StartDeferred(block_size);
   SetVal(g_conf(), "bluestore_max_blob_size", "65536");
   SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536");
@@ -8369,6 +8371,8 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) {
   }
 
   size_t block_size = 4096;
+  SetVal(g_conf(), "bluestore_block_db_create", "true");
+  SetVal(g_conf(), "bluestore_block_db_size", stringify(1 << 30).c_str());
   StartDeferred(block_size);
   SetVal(g_conf(), "bluestore_max_blob_size", "65536");
   SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536");
@@ -8447,6 +8451,10 @@ TEST_P(StoreTestSpecificAUSize, DeferredDifferentChunks) {
   size_t alloc_size = 4096;
   size_t large_object_size = 1 * 1024 * 1024;
   size_t prefer_deferred_size = 65536;
+ 
+  SetVal(g_conf(), "bluestore_block_db_create", "true");
+  SetVal(g_conf(), "bluestore_block_db_size", stringify(1 << 30).c_str());
+
   StartDeferred(alloc_size);
   SetVal(g_conf(), "bluestore_max_blob_size", "131072");
   SetVal(g_conf(), "bluestore_prefer_deferred_size",

From f3cfd02db3eab956f2cb8e69736c30b3f90ec1b4 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 12 Dec 2023 15:06:01 +0000
Subject: [PATCH 1042/2492] rgw: split RGWDataAccess from rgw_tools.cc

that class is not rados specific, while the rest of the code
in rgw_tools.cc is

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/CMakeLists.txt            |   1 +
 src/rgw/driver/rados/rgw_tools.cc | 170 -----------------------
 src/rgw/driver/rados/rgw_tools.h  | 155 ---------------------
 src/rgw/rgw_admin.cc              |   1 +
 src/rgw/rgw_data_access.cc        | 222 ++++++++++++++++++++++++++++++
 src/rgw/rgw_data_access.h         | 124 +++++++++++++++++
 6 files changed, 348 insertions(+), 325 deletions(-)
 create mode 100644 src/rgw/rgw_data_access.cc
 create mode 100644 src/rgw/rgw_data_access.h

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 00a9e4127e1e..a308c833ddce 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -147,6 +147,7 @@ set(librgw_common_srcs
   rgw_bucket_encryption.cc
   rgw_tracer.cc
   rgw_lua_background.cc
+  rgw_data_access.cc
   driver/rados/cls_fifo_legacy.cc
   driver/rados/rgw_bucket.cc
   driver/rados/rgw_bucket_sync.cc
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index bf78b9bb22e2..20de47d2cf98 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -280,176 +280,6 @@ void rgw_filter_attrset(map<string, bufferlist>& unfiltered_attrset, const strin
   }
 }
 
-RGWDataAccess::RGWDataAccess(rgw::sal::Driver* _driver) : driver(_driver)
-{
-}
-
-
-int RGWDataAccess::Bucket::finish_init()
-{
-  auto iter = attrs.find(RGW_ATTR_ACL);
-  if (iter == attrs.end()) {
-    return 0;
-  }
-
-  bufferlist::const_iterator bliter = iter->second.begin();
-  try {
-    policy.decode(bliter);
-  } catch (buffer::error& err) {
-    return -EIO;
-  }
-
-  return 0;
-}
-
-int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y)
-{
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = sd->driver->load_bucket(dpp, rgw_bucket(tenant, name), &bucket, y);
-  if (ret < 0) {
-    return ret;
-  }
-
-  bucket_info = bucket->get_info();
-  mtime = bucket->get_modification_time();
-  attrs = bucket->get_attrs();
-
-  return finish_init();
-}
-
-int RGWDataAccess::Bucket::init(const RGWBucketInfo& _bucket_info,
-				const map<string, bufferlist>& _attrs)
-{
-  bucket_info = _bucket_info;
-  attrs = _attrs;
-
-  return finish_init();
-}
-
-int RGWDataAccess::Bucket::get_object(const rgw_obj_key& key,
-				      ObjectRef *obj) {
-  obj->reset(new Object(sd, shared_from_this(), key));
-  return 0;
-}
-
-int RGWDataAccess::Object::put(bufferlist& data,
-			       map<string, bufferlist>& attrs,
-                               const DoutPrefixProvider *dpp,
-                               optional_yield y)
-{
-  rgw::sal::Driver* driver = sd->driver;
-  CephContext *cct = driver->ctx();
-
-  string tag;
-  append_rand_alpha(cct, tag, tag, 32);
-
-  RGWBucketInfo& bucket_info = bucket->bucket_info;
-
-  rgw::BlockingAioThrottle aio(driver->ctx()->_conf->rgw_put_obj_min_window_size);
-
-  std::unique_ptr<rgw::sal::Bucket> b = driver->get_bucket(bucket_info);
-  std::unique_ptr<rgw::sal::Object> obj = b->get_object(key);
-
-  auto& owner = bucket->policy.get_owner();
-
-  string req_id = driver->zone_unique_id(driver->get_new_req_id());
-
-  std::unique_ptr<rgw::sal::Writer> processor;
-  processor = driver->get_atomic_writer(dpp, y, obj.get(), owner.id,
-				       nullptr, olh_epoch, req_id);
-
-  int ret = processor->prepare(y);
-  if (ret < 0)
-    return ret;
-
-  rgw::sal::DataProcessor *filter = processor.get();
-
-  CompressorRef plugin;
-  boost::optional<RGWPutObj_Compress> compressor;
-
-  const auto& compression_type = driver->get_compression_type(bucket_info.placement_rule);
-  if (compression_type != "none") {
-    plugin = Compressor::create(driver->ctx(), compression_type);
-    if (!plugin) {
-      ldpp_dout(dpp, 1) << "Cannot load plugin for compression type "
-        << compression_type << dendl;
-    } else {
-      compressor.emplace(driver->ctx(), plugin, filter);
-      filter = &*compressor;
-    }
-  }
-
-  off_t ofs = 0;
-  auto obj_size = data.length();
-
-  RGWMD5Etag etag_calc;
-
-  do {
-    size_t read_len = std::min(data.length(), (unsigned int)cct->_conf->rgw_max_chunk_size);
-
-    bufferlist bl;
-
-    data.splice(0, read_len, &bl);
-    etag_calc.update(bl);
-
-    ret = filter->process(std::move(bl), ofs);
-    if (ret < 0)
-      return ret;
-
-    ofs += read_len;
-  } while (data.length() > 0);
-
-  ret = filter->process({}, ofs);
-  if (ret < 0) {
-    return ret;
-  }
-  bool has_etag_attr = false;
-  auto iter = attrs.find(RGW_ATTR_ETAG);
-  if (iter != attrs.end()) {
-    bufferlist& bl = iter->second;
-    etag = bl.to_str();
-    has_etag_attr = true;
-  }
-
-  if (!aclbl) {
-    RGWAccessControlPolicy policy;
-
-    const auto& owner = bucket->policy.get_owner();
-    policy.create_default(owner.id, owner.display_name); // default private policy
-
-    policy.encode(aclbl.emplace());
-  }
-
-  if (etag.empty()) {
-    etag_calc.finish(&etag);
-  }
-
-  if (!has_etag_attr) {
-    bufferlist etagbl;
-    etagbl.append(etag);
-    attrs[RGW_ATTR_ETAG] = etagbl;
-  }
-  attrs[RGW_ATTR_ACL] = *aclbl;
-
-  string *puser_data = nullptr;
-  if (user_data) {
-    puser_data = &(*user_data);
-  }
-
-  const req_context rctx{dpp, y, nullptr};
-  return processor->complete(obj_size, etag,
-			    &mtime, mtime,
-			    attrs, delete_at,
-                            nullptr, nullptr,
-                            puser_data,
-                            nullptr, nullptr, rctx);
-}
-
-void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy)
-{
-  policy.encode(aclbl.emplace());
-}
-
 void rgw_complete_aio_completion(librados::AioCompletion* c, int r) {
   auto pc = c->pc;
   librados::CB_AioCompleteAndSafe cb(pc);
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 27a8b424ecce..27bc6f0c4daa 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -165,161 +165,6 @@ int rgw_get_rados_ref(const DoutPrefixProvider* dpp, librados::Rados* rados,
 int rgw_tools_init(const DoutPrefixProvider *dpp, CephContext *cct);
 void rgw_tools_cleanup();
 
-template<class H, size_t S>
-class RGWEtag
-{
-  H hash;
-
-public:
-  RGWEtag() {
-    if constexpr (std::is_same_v<H, MD5>) {
-      // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
-      hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
-    }
-  }
-
-  void update(const char *buf, size_t len) {
-    hash.Update((const unsigned char *)buf, len);
-  }
-
-  void update(bufferlist& bl) {
-    if (bl.length() > 0) {
-      update(bl.c_str(), bl.length());
-    }
-  }
-
-  void update(const std::string& s) {
-    if (!s.empty()) {
-      update(s.c_str(), s.size());
-    }
-  }
-  void finish(std::string *etag) {
-    char etag_buf[S];
-    char etag_buf_str[S * 2 + 16];
-
-    hash.Final((unsigned char *)etag_buf);
-    buf_to_hex((const unsigned char *)etag_buf, S,
-	       etag_buf_str);
-
-    *etag = etag_buf_str;
-  }
-};
-
-using RGWMD5Etag = RGWEtag<MD5, CEPH_CRYPTO_MD5_DIGESTSIZE>;
-
-class RGWDataAccess
-{
-  rgw::sal::Driver* driver;
-
-public:
-  RGWDataAccess(rgw::sal::Driver* _driver);
-
-  class Object;
-  class Bucket;
-
-  using BucketRef = std::shared_ptr<Bucket>;
-  using ObjectRef = std::shared_ptr<Object>;
-
-  class Bucket : public std::enable_shared_from_this<Bucket> {
-    friend class RGWDataAccess;
-    friend class Object;
-
-    RGWDataAccess *sd{nullptr};
-    RGWBucketInfo bucket_info;
-    std::string tenant;
-    std::string name;
-    std::string bucket_id;
-    ceph::real_time mtime;
-    std::map<std::string, bufferlist> attrs;
-
-    RGWAccessControlPolicy policy;
-    int finish_init();
-    
-    Bucket(RGWDataAccess *_sd,
-	   const std::string& _tenant,
-	   const std::string& _name,
-	   const std::string& _bucket_id) : sd(_sd),
-                                       tenant(_tenant),
-                                       name(_name),
-				       bucket_id(_bucket_id) {}
-    Bucket(RGWDataAccess *_sd) : sd(_sd) {}
-    int init(const DoutPrefixProvider *dpp, optional_yield y);
-    int init(const RGWBucketInfo& _bucket_info, const std::map<std::string, bufferlist>& _attrs);
-  public:
-    int get_object(const rgw_obj_key& key,
-		   ObjectRef *obj);
-
-  };
-
-
-  class Object {
-    RGWDataAccess *sd{nullptr};
-    BucketRef bucket;
-    rgw_obj_key key;
-
-    ceph::real_time mtime;
-    std::string etag;
-    uint64_t olh_epoch{0};
-    ceph::real_time delete_at;
-    std::optional<std::string> user_data;
-
-    std::optional<bufferlist> aclbl;
-
-    Object(RGWDataAccess *_sd,
-           BucketRef&& _bucket,
-           const rgw_obj_key& _key) : sd(_sd),
-                                      bucket(_bucket),
-                                      key(_key) {}
-  public:
-    int put(bufferlist& data, std::map<std::string, bufferlist>& attrs, const DoutPrefixProvider *dpp, optional_yield y); /* might modify attrs */
-
-    void set_mtime(const ceph::real_time& _mtime) {
-      mtime = _mtime;
-    }
-
-    void set_etag(const std::string& _etag) {
-      etag = _etag;
-    }
-
-    void set_olh_epoch(uint64_t epoch) {
-      olh_epoch = epoch;
-    }
-
-    void set_delete_at(ceph::real_time _delete_at) {
-      delete_at = _delete_at;
-    }
-
-    void set_user_data(const std::string& _user_data) {
-      user_data = _user_data;
-    }
-
-    void set_policy(const RGWAccessControlPolicy& policy);
-
-    friend class Bucket;
-  };
-
-  int get_bucket(const DoutPrefixProvider *dpp, 
-                 const std::string& tenant,
-		 const std::string name,
-		 const std::string bucket_id,
-		 BucketRef *bucket,
-		 optional_yield y) {
-    bucket->reset(new Bucket(this, tenant, name, bucket_id));
-    return (*bucket)->init(dpp, y);
-  }
-
-  int get_bucket(const RGWBucketInfo& bucket_info,
-		 const std::map<std::string, bufferlist>& attrs,
-		 BucketRef *bucket) {
-    bucket->reset(new Bucket(this));
-    return (*bucket)->init(bucket_info, attrs);
-  }
-  friend class Bucket;
-  friend class Object;
-};
-
-using RGWDataAccessRef = std::shared_ptr<RGWDataAccess>;
-
 /// Complete an AioCompletion. To return error values or otherwise
 /// satisfy the caller. Useful for making complicated asynchronous
 /// calls and error handling.
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index f4a2d1480bda..2288399d4d85 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -62,6 +62,7 @@ extern "C" {
 #include "rgw_lua.h"
 #include "rgw_sal.h"
 #include "rgw_sal_config.h"
+#include "rgw_data_access.h"
 
 #include "services/svc_sync_modules.h"
 #include "services/svc_cls.h"
diff --git a/src/rgw/rgw_data_access.cc b/src/rgw/rgw_data_access.cc
new file mode 100644
index 000000000000..07bf12e37460
--- /dev/null
+++ b/src/rgw/rgw_data_access.cc
@@ -0,0 +1,222 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "rgw_data_access.h"
+#include "rgw_acl_s3.h"
+#include "rgw_aio_throttle.h"
+#include "rgw_compression.h"
+#include "common/BackTrace.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+template<class H, size_t S>
+class RGWEtag
+{
+  H hash;
+
+public:
+  RGWEtag() {
+    if constexpr (std::is_same_v<H, MD5>) {
+      // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
+      hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+    }
+  }
+
+  void update(const char *buf, size_t len) {
+    hash.Update((const unsigned char *)buf, len);
+  }
+
+  void update(bufferlist& bl) {
+    if (bl.length() > 0) {
+      update(bl.c_str(), bl.length());
+    }
+  }
+
+  void update(const std::string& s) {
+    if (!s.empty()) {
+      update(s.c_str(), s.size());
+    }
+  }
+  void finish(std::string *etag) {
+    char etag_buf[S];
+    char etag_buf_str[S * 2 + 16];
+
+    hash.Final((unsigned char *)etag_buf);
+    buf_to_hex((const unsigned char *)etag_buf, S,
+	       etag_buf_str);
+
+    *etag = etag_buf_str;
+  }
+};
+
+using RGWMD5Etag = RGWEtag<MD5, CEPH_CRYPTO_MD5_DIGESTSIZE>;
+
+RGWDataAccess::RGWDataAccess(rgw::sal::Driver* _driver) : driver(_driver)
+{
+}
+
+int RGWDataAccess::Bucket::finish_init()
+{
+  auto iter = attrs.find(RGW_ATTR_ACL);
+  if (iter == attrs.end()) {
+    return 0;
+  }
+
+  bufferlist::const_iterator bliter = iter->second.begin();
+  try {
+    policy.decode(bliter);
+  } catch (buffer::error& err) {
+    return -EIO;
+  }
+
+  return 0;
+}
+
+int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y)
+{
+  std::unique_ptr<rgw::sal::Bucket> bucket;
+  int ret = sd->driver->load_bucket(dpp, rgw_bucket(tenant, name), &bucket, y);
+  if (ret < 0) {
+    return ret;
+  }
+
+  bucket_info = bucket->get_info();
+  mtime = bucket->get_modification_time();
+  attrs = bucket->get_attrs();
+
+  return finish_init();
+}
+
+int RGWDataAccess::Bucket::init(const RGWBucketInfo& _bucket_info,
+				const std::map<std::string, bufferlist>& _attrs)
+{
+  bucket_info = _bucket_info;
+  attrs = _attrs;
+
+  return finish_init();
+}
+
+int RGWDataAccess::Bucket::get_object(const rgw_obj_key& key,
+				      ObjectRef *obj) {
+  obj->reset(new Object(sd, shared_from_this(), key));
+  return 0;
+}
+
+int RGWDataAccess::Object::put(bufferlist& data,
+			       std::map<std::string, bufferlist>& attrs,
+                               const DoutPrefixProvider *dpp,
+                               optional_yield y)
+{
+  rgw::sal::Driver* driver = sd->driver;
+  CephContext *cct = driver->ctx();
+
+  std::string tag;
+  append_rand_alpha(cct, tag, tag, 32);
+
+  RGWBucketInfo& bucket_info = bucket->bucket_info;
+
+  rgw::BlockingAioThrottle aio(driver->ctx()->_conf->rgw_put_obj_min_window_size);
+
+  std::unique_ptr<rgw::sal::Bucket> b = driver->get_bucket(bucket_info);
+  std::unique_ptr<rgw::sal::Object> obj = b->get_object(key);
+
+  auto& owner = bucket->policy.get_owner();
+
+  std::string req_id = driver->zone_unique_id(driver->get_new_req_id());
+
+  std::unique_ptr<rgw::sal::Writer> processor;
+  processor = driver->get_atomic_writer(dpp, y, obj.get(), owner.id,
+				       nullptr, olh_epoch, req_id);
+
+  int ret = processor->prepare(y);
+  if (ret < 0)
+    return ret;
+
+  rgw::sal::DataProcessor *filter = processor.get();
+
+  CompressorRef plugin;
+  boost::optional<RGWPutObj_Compress> compressor;
+
+  const auto& compression_type = driver->get_compression_type(bucket_info.placement_rule);
+  if (compression_type != "none") {
+    plugin = Compressor::create(driver->ctx(), compression_type);
+    if (!plugin) {
+      ldpp_dout(dpp, 1) << "Cannot load plugin for compression type "
+        << compression_type << dendl;
+    } else {
+      compressor.emplace(driver->ctx(), plugin, filter);
+      filter = &*compressor;
+    }
+  }
+
+  off_t ofs = 0;
+  auto obj_size = data.length();
+
+  RGWMD5Etag etag_calc;
+
+  do {
+    size_t read_len = std::min(data.length(), (unsigned int)cct->_conf->rgw_max_chunk_size);
+
+    bufferlist bl;
+
+    data.splice(0, read_len, &bl);
+    etag_calc.update(bl);
+
+    ret = filter->process(std::move(bl), ofs);
+    if (ret < 0)
+      return ret;
+
+    ofs += read_len;
+  } while (data.length() > 0);
+
+  ret = filter->process({}, ofs);
+  if (ret < 0) {
+    return ret;
+  }
+  bool has_etag_attr = false;
+  auto iter = attrs.find(RGW_ATTR_ETAG);
+  if (iter != attrs.end()) {
+    bufferlist& bl = iter->second;
+    etag = bl.to_str();
+    has_etag_attr = true;
+  }
+
+  if (!aclbl) {
+    RGWAccessControlPolicy policy;
+
+    const auto& owner = bucket->policy.get_owner();
+    policy.create_default(owner.id, owner.display_name); // default private policy
+
+    policy.encode(aclbl.emplace());
+  }
+
+  if (etag.empty()) {
+    etag_calc.finish(&etag);
+  }
+
+  if (!has_etag_attr) {
+    bufferlist etagbl;
+    etagbl.append(etag);
+    attrs[RGW_ATTR_ETAG] = etagbl;
+  }
+  attrs[RGW_ATTR_ACL] = *aclbl;
+
+  std::string *puser_data = nullptr;
+  if (user_data) {
+    puser_data = &(*user_data);
+  }
+
+  const req_context rctx{dpp, y, nullptr};
+  return processor->complete(obj_size, etag,
+			    &mtime, mtime,
+			    attrs, delete_at,
+                            nullptr, nullptr,
+                            puser_data,
+                            nullptr, nullptr, rctx);
+}
+
+void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy)
+{
+  policy.encode(aclbl.emplace());
+}
+
diff --git a/src/rgw/rgw_data_access.h b/src/rgw/rgw_data_access.h
new file mode 100644
index 000000000000..df921a67f4de
--- /dev/null
+++ b/src/rgw/rgw_data_access.h
@@ -0,0 +1,124 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#pragma once
+
+#include <string>
+#include "include/types.h"
+#include "common/ceph_time.h"
+#include "rgw_common.h"
+#include "rgw_sal_fwd.h"
+
+class RGWDataAccess
+{
+  rgw::sal::Driver* driver;
+
+public:
+  RGWDataAccess(rgw::sal::Driver* _driver);
+
+  class Object;
+  class Bucket;
+
+  using BucketRef = std::shared_ptr<Bucket>;
+  using ObjectRef = std::shared_ptr<Object>;
+
+  class Bucket : public std::enable_shared_from_this<Bucket> {
+    friend class RGWDataAccess;
+    friend class Object;
+
+    RGWDataAccess *sd{nullptr};
+    RGWBucketInfo bucket_info;
+    std::string tenant;
+    std::string name;
+    std::string bucket_id;
+    ceph::real_time mtime;
+    std::map<std::string, bufferlist> attrs;
+
+    RGWAccessControlPolicy policy;
+    int finish_init();
+    
+    Bucket(RGWDataAccess *_sd,
+	   const std::string& _tenant,
+	   const std::string& _name,
+	   const std::string& _bucket_id) : sd(_sd),
+                                       tenant(_tenant),
+                                       name(_name),
+				       bucket_id(_bucket_id) {}
+    Bucket(RGWDataAccess *_sd) : sd(_sd) {}
+    int init(const DoutPrefixProvider *dpp, optional_yield y);
+    int init(const RGWBucketInfo& _bucket_info, const std::map<std::string, bufferlist>& _attrs);
+  public:
+    int get_object(const rgw_obj_key& key,
+		   ObjectRef *obj);
+
+  };
+
+
+  class Object {
+    RGWDataAccess *sd{nullptr};
+    BucketRef bucket;
+    rgw_obj_key key;
+
+    ceph::real_time mtime;
+    std::string etag;
+    uint64_t olh_epoch{0};
+    ceph::real_time delete_at;
+    std::optional<std::string> user_data;
+
+    std::optional<bufferlist> aclbl;
+
+    Object(RGWDataAccess *_sd,
+           BucketRef&& _bucket,
+           const rgw_obj_key& _key) : sd(_sd),
+                                      bucket(_bucket),
+                                      key(_key) {}
+  public:
+    int put(bufferlist& data, std::map<std::string, bufferlist>& attrs, const DoutPrefixProvider *dpp, optional_yield y); /* might modify attrs */
+
+    void set_mtime(const ceph::real_time& _mtime) {
+      mtime = _mtime;
+    }
+
+    void set_etag(const std::string& _etag) {
+      etag = _etag;
+    }
+
+    void set_olh_epoch(uint64_t epoch) {
+      olh_epoch = epoch;
+    }
+
+    void set_delete_at(ceph::real_time _delete_at) {
+      delete_at = _delete_at;
+    }
+
+    void set_user_data(const std::string& _user_data) {
+      user_data = _user_data;
+    }
+
+    void set_policy(const RGWAccessControlPolicy& policy);
+
+    friend class Bucket;
+  };
+
+  int get_bucket(const DoutPrefixProvider *dpp, 
+                 const std::string& tenant,
+		 const std::string name,
+		 const std::string bucket_id,
+		 BucketRef *bucket,
+		 optional_yield y) {
+    bucket->reset(new Bucket(this, tenant, name, bucket_id));
+    return (*bucket)->init(dpp, y);
+  }
+
+  int get_bucket(const RGWBucketInfo& bucket_info,
+		 const std::map<std::string, bufferlist>& attrs,
+		 BucketRef *bucket) {
+    bucket->reset(new Bucket(this));
+    return (*bucket)->init(bucket_info, attrs);
+  }
+  friend class Bucket;
+  friend class Object;
+};
+
+using RGWDataAccessRef = std::shared_ptr<RGWDataAccess>;
+

From 47342a8b3ce1ee998ac70687468253135e29f46b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 23 Jun 2023 17:01:00 -0400
Subject: [PATCH 1043/2492] mds: add balance_automate fs setting

To turn off the automatic ("default") balancer in multiple MDS clusters. The
new default is "off" as the balancer  is a constant source of problems and
surprise for administrators trying multiple actives. Instead, it should be a
deliberate decision to turn it on and usually with customization like the
"bal_rank_mask" setting or pinning.

Fixes: https://tracker.ceph.com/issues/61378
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 PendingReleaseNotes   |  4 ++++
 src/include/ceph_fs.h |  1 +
 src/mds/MDBalancer.cc |  4 +++-
 src/mds/MDSMap.cc     |  3 +++
 src/mds/MDSMap.h      | 12 +++++++++++-
 src/mon/FSCommands.cc | 15 +++++++++++++++
 src/mon/MonCommands.h | 25 ++++++++++++++++++++-----
 7 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index c44851a5eafb..8e344dba79b9 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -95,6 +95,10 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   mirroring policies between RGW and AWS, you may wish to set
   "rgw policy reject invalid principals" to "false". This affects only newly set
   policies, not policies that are already in place.
+* The CephFS automatic metadata load (sometimes called "default") balancer is
+  now disabled by default. The new file system flag `balance_automate`
+  can be used to toggle it on or off. It can be enabled or disabled via
+  `ceph fs set <fs_name> balance_automate <bool>`.
 * RGW's default backend for `rgw_enable_ops_log` changed from RADOS to file.
   The default value of `rgw_ops_log_rados` is now false, and `rgw_ops_log_file_path`
   defaults to "/var/log/ceph/ops-log-$cluster-$name.log".
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index f567a26f4110..245421680265 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -292,6 +292,7 @@ struct ceph_mon_subscribe_ack {
                                                             request */
 #define CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS (1<<7) /* fs is forbidden to use standby
                                                             for another fs */
+#define CEPH_MDSMAP_BALANCE_AUTOMATE             (1<<8)  /* automate metadata balancing */
 #define CEPH_MDSMAP_DEFAULTS (CEPH_MDSMAP_ALLOW_SNAPS | \
 			      CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS)
 
diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index cd2e7d9bd002..b48c59f2606a 100644
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -230,6 +230,7 @@ void MDBalancer::handle_export_pins(void)
 void MDBalancer::tick()
 {
   static int num_bal_times = g_conf()->mds_bal_max;
+  bool balance_automate = mds->mdsmap->allows_balance_automate();
   auto bal_interval = g_conf().get_val<int64_t>("mds_bal_interval");
   auto bal_max_until = g_conf().get_val<int64_t>("mds_bal_max_until");
   time now = clock::now();
@@ -248,7 +249,8 @@ void MDBalancer::tick()
   // We can use duration_cast below, although the result is an int,
   // because the values from g_conf are also integers.
   // balance?
-  if (mds->get_nodeid() == 0
+  if (balance_automate
+      && mds->get_nodeid() == 0
       && mds->is_active()
       && bal_interval > 0
       && chrono::duration_cast<chrono::seconds>(now - last_heartbeat).count() >= bal_interval
diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 47c823bf7635..699765ebe917 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -239,6 +239,7 @@ void MDSMap::dump_flags_state(Formatter *f) const
     f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY), allows_standby_replay());
     f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION), test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION));
     f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS), test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS));
+    f->dump_bool(flag_display.at(CEPH_MDSMAP_BALANCE_AUTOMATE), test_flag(CEPH_MDSMAP_BALANCE_AUTOMATE));
     f->close_section();
 }
 
@@ -383,6 +384,8 @@ void MDSMap::print_flags(std::ostream& out) const {
     out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION);
   if (test_flag(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS))
     out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS);
+  if (test_flag(CEPH_MDSMAP_BALANCE_AUTOMATE))
+    out << " " << flag_display.at(CEPH_MDSMAP_BALANCE_AUTOMATE);
 }
 
 void MDSMap::get_health(list<pair<health_status_t,string> >& summary,
diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 9057f05a8ced..746ae8597151 100644
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -244,6 +244,15 @@ class MDSMap {
   bool allows_standby_replay() const { return test_flag(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY); }
   bool was_standby_replay_ever_allowed() const { return ever_allowed_features & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY; }
 
+  void set_balance_automate() {
+    set_flag(CEPH_MDSMAP_BALANCE_AUTOMATE);
+    ever_allowed_features |= CEPH_MDSMAP_BALANCE_AUTOMATE;
+    explicitly_allowed_features |= CEPH_MDSMAP_BALANCE_AUTOMATE;
+  }
+  void clear_balance_automate() { clear_flag(CEPH_MDSMAP_BALANCE_AUTOMATE); }
+  bool allows_balance_automate() const { return test_flag(CEPH_MDSMAP_BALANCE_AUTOMATE); }
+  bool was_balance_automate_ever_allowed() const { return ever_allowed_features & CEPH_MDSMAP_BALANCE_AUTOMATE; }
+
   void set_multimds_snaps_allowed() {
     set_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS);
     ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS;
@@ -676,7 +685,8 @@ class MDSMap {
     {CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS, "allow_multimds_snaps"},
     {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"},
     {CEPH_MDSMAP_REFUSE_CLIENT_SESSION, "refuse_client_session"},
-    {CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS, "refuse_standby_for_another_fs"}
+    {CEPH_MDSMAP_REFUSE_STANDBY_FOR_ANOTHER_FS, "refuse_standby_for_another_fs"},
+    {CEPH_MDSMAP_BALANCE_AUTOMATE, "balance_automate"}
   };
 };
 WRITE_CLASS_ENCODER_FEATURES(MDSMap::mds_info_t)
diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 2faba87f73de..260619969078 100644
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -665,6 +665,21 @@ class SetHandler : public FileSystemCommandHandler
         }
       };
       fsmap.modify_filesystem(fsp->get_fscid(), std::move(f));
+    } else if (var == "balance_automate") {
+      bool allow = false;
+      int r = parse_bool(val, &allow, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      auto f = [allow](auto&& fs) {
+        if (allow) {
+          fs.get_mds_map().set_balance_automate();
+        } else {
+          fs.get_mds_map().clear_balance_automate();
+        }
+      };
+      fsmap.modify_filesystem(fsp->get_fscid(), std::move(f));
     } else if (var == "min_compat_client") {
       auto vno = ceph_release_from_name(val.c_str());
       if (!vno) {
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 71a8bec76ac0..6f8f25e050af 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -374,11 +374,26 @@ COMMAND("fs get name=fs_name,type=CephString",
 	"fs", "r")
 COMMAND("fs set "
 	"name=fs_name,type=CephString "
-	"name=var,type=CephChoices,strings=max_mds|max_file_size"
-        "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer"
-        "|standby_count_wanted|session_timeout|session_autoclose"
-        "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask"
-	"|refuse_client_session|max_xattr_size|refuse_standby_for_another_fs "
+	"name=var,type=CephChoices,strings=max_mds"
+          "|allow_dirfrags"
+          "|allow_new_snaps"
+          "|allow_standby_replay"
+          "|bal_rank_mask"
+          "|balance_automate"
+          "|balancer"
+          "|cluster_down"
+          "|down"
+          "|inline_data"
+          "|joinable"
+          "|max_file_size"
+          "|max_xattr_size"
+          "|min_compat_client"
+          "|refuse_client_session"
+          "|refuse_standby_for_another_fs"
+          "|session_autoclose"
+          "|session_timeout"
+          "|standby_count_wanted"
+          " "
 	"name=val,type=CephString "
 	"name=yes_i_really_mean_it,type=CephBool,req=false "
 	"name=yes_i_really_really_mean_it,type=CephBool,req=false",

From fc5c358f74f495eedca09abcde7ea2716a893314 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 12 Dec 2023 11:44:43 -0500
Subject: [PATCH 1044/2492] doc/cephfs: use bash prompt for shell code

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 doc/cephfs/multimds.rst | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/doc/cephfs/multimds.rst b/doc/cephfs/multimds.rst
index e50a5148ec2d..02564db45a88 100644
--- a/doc/cephfs/multimds.rst
+++ b/doc/cephfs/multimds.rst
@@ -116,7 +116,7 @@ The mechanism provided for this purpose is called an ``export pin``, an
 extended attribute of directories. The name of this extended attribute is
 ``ceph.dir.pin``.  Users can set this attribute using standard commands:
 
-::
+.. prompt:: bash #
 
     setfattr -n ceph.dir.pin -v 2 path/to/dir
 
@@ -128,7 +128,7 @@ pin.  In this way, setting the export pin on a directory affects all of its
 children. However, the parents pin can be overridden by setting the child
 directory's export pin. For example:
 
-::
+.. prompt:: bash #
 
     mkdir -p a/b
     # "a" and "a/b" both start without an export pin set
@@ -173,7 +173,7 @@ immediate children across a range of MDS ranks.  The canonical example use-case
 would be the ``/home`` directory: we want every user's home directory to be
 spread across the entire MDS cluster. This can be set via:
 
-::
+.. prompt:: bash #
 
     setfattr -n ceph.dir.pin.distributed -v 1 /cephfs/home
 
@@ -183,7 +183,7 @@ may be ephemerally pinned. This is set through the extended attribute
 ``ceph.dir.pin.random`` with the value set to the percentage of directories
 that should be pinned. For example:
 
-::
+.. prompt:: bash #
 
     setfattr -n ceph.dir.pin.random -v 0.5 /cephfs/tmp
 
@@ -205,7 +205,7 @@ Ephemeral pins may override parent export pins and vice versa. What determines
 which policy is followed is the rule of the closest parent: if a closer parent
 directory has a conflicting policy, use that one instead. For example:
 
-::
+.. prompt:: bash #
 
     mkdir -p foo/bar1/baz foo/bar2
     setfattr -n ceph.dir.pin -v 0 foo
@@ -217,7 +217,7 @@ directory will obey the pin on ``foo`` normally.
 
 For the reverse situation:
 
-::
+.. prompt:: bash #
 
     mkdir -p home/{patrick,john}
     setfattr -n ceph.dir.pin.distributed -v 1 home
@@ -229,7 +229,8 @@ because its export pin overrides the policy on ``home``.
 To remove a partitioning policy, remove the respective extended attribute
 or set the value to 0.
 
-.. code::bash
+.. prompt:: bash #
+
    $ setfattr -n ceph.dir.pin.distributed -v 0 home
    # or
    $ setfattr -x ceph.dir.pin.distributed home
@@ -237,7 +238,8 @@ or set the value to 0.
 For export pins, remove the extended attribute or set the extended attribute
 value to `-1`.
 
-.. code::bash
+.. prompt:: bash #
+
    $ setfattr -n ceph.dir.pin -v -1 home
 
 
@@ -260,27 +262,27 @@ static pinned subtrees.
 
 This option can be configured with the ``ceph fs set`` command. For example:
 
-::
+.. prompt:: bash #
 
     ceph fs set <fs_name> bal_rank_mask <hex> 
 
 Each bitfield of the ``<hex>`` number represents a dedicated rank. If the ``<hex>`` is
 set to ``0x3``, the balancer runs on active ``0`` and ``1`` ranks. For example:
 
-::
+.. prompt:: bash #
 
     ceph fs set <fs_name> bal_rank_mask 0x3
 
 If the ``bal_rank_mask`` is set to ``-1`` or ``all``, all active ranks are masked
 and utilized by the balancer. As an example:
 
-::
+.. prompt:: bash #
 
     ceph fs set <fs_name> bal_rank_mask -1
 
 On the other hand, if the balancer needs to be disabled,
 the ``bal_rank_mask`` should be set to ``0x0``. For example:
 
-::
+.. prompt:: bash #
 
     ceph fs set <fs_name> bal_rank_mask 0x0

From 08a342485c9ffc957e959ad3f5d4abc3b092bfb0 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 12 Dec 2023 11:45:05 -0500
Subject: [PATCH 1045/2492] doc/cephfs: add docs for balance_automate

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 doc/cephfs/multimds.rst | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/doc/cephfs/multimds.rst b/doc/cephfs/multimds.rst
index 02564db45a88..d105c74ad62c 100644
--- a/doc/cephfs/multimds.rst
+++ b/doc/cephfs/multimds.rst
@@ -243,6 +243,31 @@ value to `-1`.
    $ setfattr -n ceph.dir.pin -v -1 home
 
 
+Dynamic Subtree Partitioning
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CephFS has long had a dynamic metadata blanacer (sometimes called the "default
+balancer") which can split or merge subtrees while placing them on "colder" MDS
+ranks. Moving the metadata around can improve overall file system throughput
+and cache size.
+
+However, the balancer has suffered from problem with efficiency and performance
+so it is by default turned off. This is to avoid an administrator "turning on
+multimds" by increasing the ``max_mds`` setting and then finding the balancer
+has made a mess of the cluster performance (reverting is straightforward but
+can take time).
+
+The setting to turn on the balancer is:
+
+.. prompt:: bash #
+
+   ceph fs set <fs_name> balance_automate true
+
+Turning on the balancer should only be done with appropriate configuration,
+such as with the ``bal_rank_mask`` setting (described below). Careful
+monitoring of the file system performance and MDS is advised.
+
+
 Dynamic subtree partitioning with Balancer on specific ranks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 7a2f59562c855348bc9bebc1c1f15c98fd4680f2 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 18 Jul 2023 14:08:22 -0400
Subject: [PATCH 1046/2492] qa: add automate_balance to dashboard qa schema

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/mgr/dashboard/test_health.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index 8f9d5ee16762..0b7b7a3b449c 100644
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -59,7 +59,8 @@ class HealthTest(DashboardTestCase):
             'allow_multimds_snaps': bool,
             'allow_standby_replay': bool,
             'refuse_client_session': bool,
-            'refuse_standby_for_another_fs': bool
+            'refuse_standby_for_another_fs': bool,
+            'balance_automate': bool,
         }),
         'ever_allowed_features': int,
         'root': int

From e4f14d449daefa359fd6bcc17610424790b256ca Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 12 Jul 2023 12:44:49 -0400
Subject: [PATCH 1047/2492] qa: do not rely on default max_mds value

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/suites/fs/workload/ranks/1.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qa/suites/fs/workload/ranks/1.yaml b/qa/suites/fs/workload/ranks/1.yaml
index e69de29bb2d1..f9e95daa9230 100644
--- a/qa/suites/fs/workload/ranks/1.yaml
+++ b/qa/suites/fs/workload/ranks/1.yaml
@@ -0,0 +1,4 @@
+overrides:
+  ceph:
+    cephfs:
+      max_mds: 1

From e555ab774bbe0bfae606091f723a31b7961fcf23 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 17 Jul 2023 11:07:14 -0400
Subject: [PATCH 1048/2492] qa: create qa subvolumes in named subvolumegroup

So we can twiddle the subvolumegroup settings (like pins).

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/filesystem.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 3516bf4b86cd..943b9cb32471 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -773,17 +773,30 @@ def create(self, recover=False, metadata_overlay=False):
                 assert(isinstance(subvols['create'], int))
                 assert(subvols['create'] > 0)
 
+                self.run_ceph_cmd('fs', 'subvolumegroup', 'create', self.name, 'qa')
+                subvol_options = self.fs_config.get('subvol_options', '')
+
                 for sv in range(0, subvols['create']):
                     sv_name = f'sv_{sv}'
-                    self.run_ceph_cmd('fs', 'subvolume', 'create', self.name,
-                                      sv_name,
-                                      self.fs_config.get('subvol_options', ''))
+                    cmd = [
+                      'fs',
+                      'subvolume',
+                      'create',
+                      self.name,
+                      sv_name,
+                      '--group_name', 'qa',
+                    ]
+                    if subvol_options:
+                        cmd.append(subvol_options)
+                    self.run_ceph_cmd(*cmd)
 
                     if self.name not in self._ctx.created_subvols:
                         self._ctx.created_subvols[self.name] = []
                     
                     subvol_path = self.get_ceph_cmd_stdout(
-                        'fs', 'subvolume', 'getpath', self.name, sv_name)
+                        'fs', 'subvolume', 'getpath', self.name,
+                        '--group_name', 'qa',
+                        sv_name)
                     subvol_path = subvol_path.strip()
                     self._ctx.created_subvols[self.name].append(subvol_path)
             else:

From 5f51fc31471b1426c3046ec14b57e4a6bfe9f09b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 12 Jul 2023 12:46:35 -0400
Subject: [PATCH 1049/2492] qa: configure balancer for multi-mds workloads

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../fs/workload/ranks/multi/balancer/automatic.yaml    |  4 ++++
 .../ranks/multi/balancer/distributed.yaml.disabled     |  6 ++++++
 qa/suites/fs/workload/ranks/multi/balancer/random.yaml | 10 ++++++++++
 .../fs/workload/tasks/0-subvolume/no-subvolume.yaml    |  0
 4 files changed, 20 insertions(+)
 create mode 100644 qa/suites/fs/workload/ranks/multi/balancer/automatic.yaml
 create mode 100644 qa/suites/fs/workload/ranks/multi/balancer/distributed.yaml.disabled
 create mode 100644 qa/suites/fs/workload/ranks/multi/balancer/random.yaml
 delete mode 100644 qa/suites/fs/workload/tasks/0-subvolume/no-subvolume.yaml

diff --git a/qa/suites/fs/workload/ranks/multi/balancer/automatic.yaml b/qa/suites/fs/workload/ranks/multi/balancer/automatic.yaml
new file mode 100644
index 000000000000..020eaa4bf927
--- /dev/null
+++ b/qa/suites/fs/workload/ranks/multi/balancer/automatic.yaml
@@ -0,0 +1,4 @@
+tasks:
+- exec:
+    mon.a:
+      - ceph fs set cephfs balance_automate true
diff --git a/qa/suites/fs/workload/ranks/multi/balancer/distributed.yaml.disabled b/qa/suites/fs/workload/ranks/multi/balancer/distributed.yaml.disabled
new file mode 100644
index 000000000000..be06d518611b
--- /dev/null
+++ b/qa/suites/fs/workload/ranks/multi/balancer/distributed.yaml.disabled
@@ -0,0 +1,6 @@
+# distributed pins would be interesting if we had workloads on multiple clients. We do not yet. So it's disabled.
+tasks:
+- exec:
+    mon.a:
+      - ceph fs set cephfs balance_automate false
+      - ceph fs subvolumegroup pin cephfs qa distributed 1
diff --git a/qa/suites/fs/workload/ranks/multi/balancer/random.yaml b/qa/suites/fs/workload/ranks/multi/balancer/random.yaml
new file mode 100644
index 000000000000..977e83fc23c0
--- /dev/null
+++ b/qa/suites/fs/workload/ranks/multi/balancer/random.yaml
@@ -0,0 +1,10 @@
+overrides:
+  ceph:
+    conf:
+      mds:
+        mds_export_ephemeral_random_max: 0.10
+tasks:
+- exec:
+    mon.a:
+      - ceph fs set cephfs balance_automate false
+      - ceph fs subvolumegroup pin cephfs qa random 0.10
diff --git a/qa/suites/fs/workload/tasks/0-subvolume/no-subvolume.yaml b/qa/suites/fs/workload/tasks/0-subvolume/no-subvolume.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000

From 66b2e34d368d0bc9d03e6a5dc666159815d800bf Mon Sep 17 00:00:00 2001
From: chenxuqiang <chenxuqiang3@hisilicon.com>
Date: Thu, 18 May 2023 14:34:19 +0800
Subject: [PATCH 1050/2492] erasure-code/isa/xor_op: add neon-based region_xor
 implementation

The load instruction of NEON can load 128 bits. Generally, the CPU has
two load channels. Therefore, the 32-byte Region_xor can be implemented.
According to the test by ceph_erasure_code_benchmark, the performance
is improved by more than 20% ~ 50% on average.

loop = 10000

    (k, m, size)   |   base(s)  |   neon(s)
 ------------------------------------------
   (4, 1, 16384)   |   0.018    |   0.015
 ------------------------------------------
   (4, 1, 65536)   |   0.043    |   0.037
 ------------------------------------------
   (4, 1, 102400)  |   0.058    |   0.049
 ------------------------------------------
   (8, 1, 32768)   |   0.034    |   0.029
 ------------------------------------------
   (8, 1, 65536)   |   0.052    |   0.045
 ------------------------------------------
   (8, 1, 102400)  |   0.068    |   0.061
 ------------------------------------------
   (8, 1, 524288)  |   0.631    |   0.420
 ------------------------------------------
   (8, 1, 1048576) |   1.561    |   0.931
 ------------------------------------------
   (8, 1, 8388608) |   16.70    |   8.244
 ------------------------------------------

Signed-off-by: chenxuqiang <chenxuqiang3@hisilicon.com>
---
 src/erasure-code/isa/xor_op.cc | 54 ++++++++++++++++++++++++++++++++++
 src/erasure-code/isa/xor_op.h  | 11 ++++++-
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/src/erasure-code/isa/xor_op.cc b/src/erasure-code/isa/xor_op.cc
index 2b56e977c7fd..4f507c9b2625 100644
--- a/src/erasure-code/isa/xor_op.cc
+++ b/src/erasure-code/isa/xor_op.cc
@@ -15,6 +15,11 @@
 #include <stdio.h>
 #include <string.h>
 #include "arch/intel.h"
+#include "arch/arm.h"
+
+#if defined(__aarch64__) && defined(__ARM_NEON)
+  #include <arm_neon.h>
+#endif
 
 #include "include/ceph_assert.h"
 
@@ -101,6 +106,16 @@ region_xor(unsigned char** src,
       // 64-byte region xor
       region_sse2_xor((char**) src, (char*) parity, src_size, region_size);
     } else
+#elif defined (__aarch64__) && defined(__ARM_NEON)
+    if (ceph_arch_neon) {
+      // -----------------------------
+      // use NEON region xor function
+      // -----------------------------
+      unsigned region_size = 
+        (size / EC_ISA_VECTOR_NEON_WORDSIZE) * EC_ISA_VECTOR_NEON_WORDSIZE;
+      size_left -= region_size;
+      region_neon_xor((char**) src, (char *) parity, src_size, region_size);
+    } else
 #endif
     {
       // --------------------------------------------
@@ -181,3 +196,42 @@ region_sse2_xor(char** src,
 #endif // __x86_64__
   return;
 }
+
+void
+// -----------------------------------------------------------------------------
+region_neon_xor(char **src,
+                char *parity,
+                int src_size,
+                unsigned size)
+// -----------------------------------------------------------------------------
+{
+#if defined(__aarch64__) && defined(__ARM_NEON)
+  ceph_assert(!(size % EC_ISA_VECTOR_NEON_WORDSIZE));
+  unsigned char *p = (unsigned char *)parity;
+  unsigned char *vbuf[256] = { NULL };
+  for (int v = 0; v < src_size; v++) {
+    vbuf[v] = (unsigned char *)src[v];
+  }
+
+  // ----------------------------------------------------------------------------------------
+  // NEON load instructions can load 128bits of data each time, and there are 2 load channels
+  // ----------------------------------------------------------------------------------------
+  for (unsigned i = 0; i < size; i += EC_ISA_VECTOR_NEON_WORDSIZE) {
+    uint64x2_t d0_1 = vld1q_u64((uint64_t *)(&(vbuf[0][i])));
+    uint64x2_t d0_2 = vld1q_u64((uint64_t *)(&(vbuf[0][i + 16])));
+
+    for (int d = 1; d < src_size; d++) {
+      uint64x2_t di_1 = vld1q_u64((uint64_t *)(&(vbuf[d][i])));
+      uint64x2_t di_2 = vld1q_u64((uint64_t *)(&(vbuf[d][i + 16])));
+
+      d0_1 = veorq_u64(d0_1, di_1);
+      d0_2 = veorq_u64(d0_2, di_2);
+    }
+
+    vst1q_u64((uint64_t *)p, d0_1);
+    vst1q_u64((uint64_t *)(p + 16), d0_2);
+    p += EC_ISA_VECTOR_NEON_WORDSIZE;
+  }
+#endif // __aarch64__ && __ARM_NEON
+  return;
+}
diff --git a/src/erasure-code/isa/xor_op.h b/src/erasure-code/isa/xor_op.h
index 978b9a953582..46304eee1cce 100644
--- a/src/erasure-code/isa/xor_op.h
+++ b/src/erasure-code/isa/xor_op.h
@@ -27,7 +27,7 @@
 
 #define EC_ISA_ADDRESS_ALIGNMENT 32u
 #define EC_ISA_VECTOR_SSE2_WORDSIZE 64u
-
+#define EC_ISA_VECTOR_NEON_WORDSIZE 32u
 #if __GNUC__ > 4 || \
   ( (__GNUC__ == 4) && (__GNUC_MINOR__ >= 4) ) ||\
   (__clang__ == 1 )
@@ -83,5 +83,14 @@ region_sse2_xor(char** src /* array of 64-byte aligned source pointer to xor */,
                 int src_size /* size of the source pointer array */,
                 unsigned size /* size of the region to xor */);
 
+// -------------------------------------------------------------------------
+// compute region XOR like parity = src[0] ^ src[1] ... ^ src[src_size-1]
+// using NEON 32-byte operations
+// -------------------------------------------------------------------------
+void
+region_neon_xor(char** src    /* array of 64-byte aligned source pointer to xor */,
+                char* parity  /* 32-byte aligned output pointer containing the parity */,
+                int src_size  /* size of the source pointer array */,
+                unsigned size /* size of the region to xor */);
 
 #endif // EC_ISA_XOR_OP_H

From 4f616e87b403d6988b555e4454247c339d82f363 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Wed, 13 Dec 2023 11:30:11 +0800
Subject: [PATCH 1051/2492] os/memstore: deep copy on partial bufferptrs on
 setattrs

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/os/memstore/MemStore.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/os/memstore/MemStore.cc b/src/os/memstore/MemStore.cc
index 99e99dcba041..35711517708c 100644
--- a/src/os/memstore/MemStore.cc
+++ b/src/os/memstore/MemStore.cc
@@ -1131,8 +1131,14 @@ int MemStore::_setattrs(const coll_t& cid, const ghobject_t& oid,
   if (!o)
     return -ENOENT;
   std::lock_guard lock{o->xattr_mutex};
-  for (auto p = aset.begin(); p != aset.end(); ++p)
-    o->xattr[p->first] = p->second;
+  for (auto p = aset.begin(); p != aset.end(); ++p) {
+    if (p->second.is_partial()) {
+      o->xattr[p->first] = bufferptr(p->second.c_str(), p->second.length());
+    } else {
+      o->xattr[p->first] = p->second;
+    }
+  }
+
   return 0;
 }
 

From 7f2c6108fceeebefd0724788683d6ea67bfed766 Mon Sep 17 00:00:00 2001
From: chenxuqiang <chenxuqiang3@hisilicon.com>
Date: Wed, 17 May 2023 17:43:05 +0800
Subject: [PATCH 1052/2492] test/common: add mem_is_zero ut

Signed-off-by: chenxuqiang <chenxuqiang3@hisilicon.com>
---
 src/test/common/CMakeLists.txt |  7 ++++
 src/test/common/test_memory.cc | 73 ++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 src/test/common/test_memory.cc

diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index b2ed06ee3062..e3b6e5497677 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -294,6 +294,13 @@ add_executable(unittest_perf_histogram
 add_ceph_unittest(unittest_perf_histogram)
 target_link_libraries(unittest_perf_histogram ceph-common)
 
+# unittest_memory
+add_executable(unittest_memory
+  test_memory.cc
+  )
+add_ceph_unittest(unittest_memory)
+target_link_libraries(unittest_memory ceph-common)
+
 # unittest_perf_cache_key
 add_executable(unittest_perf_counters_key test_perf_counters_key.cc)
 add_ceph_unittest(unittest_perf_counters_key)
diff --git a/src/test/common/test_memory.cc b/src/test/common/test_memory.cc
new file mode 100644
index 000000000000..cf075f01e5d3
--- /dev/null
+++ b/src/test/common/test_memory.cc
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <iostream>
+#include <vector>
+#include <string.h>
+
+#include "include/inline_memory.h"
+#include "include/utime.h"
+#include "common/Clock.h"
+#include "gtest/gtest.h"
+
+class MemoryIsZeroBigTest : public ::testing::TestWithParam<size_t> {};
+class MemoryIsZeroSmallTest : public ::testing::TestWithParam<size_t> {};
+class MemoryIsZeroPerformance : public ::testing::TestWithParam<size_t> {};
+
+TEST_P(MemoryIsZeroBigTest, MemoryIsZeroTestBig) {
+  size_t size = GetParam();
+  char *data = (char *)malloc(sizeof(char) * size);
+  memset(data, 0, sizeof(char) * size);
+  EXPECT_TRUE(mem_is_zero(data, size));
+
+  size_t pos = rand() % size;
+  data[pos] = 'a';
+  EXPECT_FALSE(mem_is_zero(data, size));
+
+  free(data);
+}
+
+TEST_P(MemoryIsZeroSmallTest, MemoryIsZeroTestSmall) {
+  size_t size = GetParam();
+  for (size_t i = 0; i < size; i++) {
+    char data[size] = { 0 };
+    EXPECT_TRUE(mem_is_zero(data, size));
+
+    data[i] = 'a';
+    EXPECT_FALSE(mem_is_zero(data, size));
+  }
+}
+
+TEST_P(MemoryIsZeroPerformance, MemoryIsZeroPerformanceTest) {
+  constexpr size_t ITER = 1000000;
+  utime_t start;
+  utime_t end;
+
+  size_t size = GetParam();
+  char *data = (char *)malloc(size);
+  memset(data, 0, size);
+  
+  bool res = false;
+  start = ceph_clock_now();
+  for (size_t i = 0; i < ITER; i++) {
+    res = mem_is_zero(data, size);
+  }
+  end = ceph_clock_now();
+
+  std::cout << "iterators=" << ITER 
+            << " size= " << size 
+            << " time=" << (double)(end - start)
+            << std::endl;
+
+  ASSERT_TRUE(res);
+  free(data);
+}
+
+INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroSmallTest,
+                        ::testing::Values(1, 4, 7, 8, 12, 28, 60, 64));
+
+INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroBigTest,
+                        ::testing::Values(1024, 4096, 8192, 64 * 1024));
+
+INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroPerformance,
+                        ::testing::Values(1024, 2048, 4096, 8192, 64 * 1024));

From c7749f09bb7759d8696279882ce6852b60a2fae3 Mon Sep 17 00:00:00 2001
From: chenxuqiang <chenxuqiang3@hisilicon.com>
Date: Wed, 17 May 2023 17:34:50 +0800
Subject: [PATCH 1053/2492] inline_memory: optimized mem_is_zero for aarch64 by
 neon intrinsic

The Neon instruction can be used to optimize mem_is_Zero. The Neon
    instruction can load 128-bit data. Most ARM CPU architectures have 2
    LD loading channels. Therefore, two 128-bit data can be loaded at the
    same time, which is good for block data.

    We performed a set of performance tests on TSV110 to compare the no-x64
    implementation, loop: 1,000,000, used time:

        size    |    no-x64      |     neon
     --------------------------------------------
        1024    |    0.091       |     0.044
     --------------------------------------------
        2048    |    0.177       |     0.092
     --------------------------------------------
        4096    |    0.348       |     0.175
     --------------------------------------------
        8192    |    0.689       |     0.345
     --------------------------------------------
        65536   |    5.471       |     2.746

Signed-off-by: chenxuqiang <chenxuqiang3@hisilicon.com>
---
 src/include/inline_memory.h | 43 +++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/src/include/inline_memory.h b/src/include/inline_memory.h
index 48d889763f72..6a1fe34800ab 100644
--- a/src/include/inline_memory.h
+++ b/src/include/inline_memory.h
@@ -123,6 +123,49 @@ bool mem_is_zero(const char *data, size_t len)
   return true;
 }
 
+#elif defined(__GNUC__) && defined(__aarch64__) && defined(__ARM_NEON) // gcc and aarch64 neon
+
+#include <arm_neon.h>
+
+static inline bool mem_is_zero(const char *data, size_t len) {
+  const char *end = data + len;
+  const char *end256 = data + (len / sizeof(uint64x2x2_t)) * sizeof(uint64x2x2_t);
+  while (data < end256) {
+    uint64x2x2_t value = vld1q_u64_x2((uint64_t *)data);
+    if (value.val[0][0] != 0 || value.val[0][1] != 0 ||
+        value.val[1][0] != 0 || value.val[1][1] != 0) {
+      return false;
+    }
+    data += sizeof(uint64x2x2_t);
+  }
+
+  const char *end128 = data + sizeof(uint64x2_t);
+  if (end128 < end) {
+    uint64x2_t value = vld1q_u64((uint64_t *)data);
+    if (value[0] != 0 || value[1] != 0) {
+      return false;
+    }
+    data += sizeof(uint64x2_t);
+  }
+
+  const char *end64 = data + sizeof(uint64_t);
+  if (end64 < end) {
+    if(*(uint64_t *)data != 0) {
+      return false;
+    }
+    data += sizeof(uint64_t);
+  }
+
+  while (data < end) {
+    if (*data != 0) {
+      return false;
+    }
+    ++data;
+  }
+
+  return true;
+}
+
 #else  // gcc and x86_64
 
 static inline bool mem_is_zero(const char *data, size_t len) {

From c1ea66fe12f86e7a63681cba860fb91b1ea86e12 Mon Sep 17 00:00:00 2001
From: Daniel Persson <mailto.woden@gmail.com>
Date: Sun, 3 Dec 2023 08:03:47 +0000
Subject: [PATCH 1054/2492] mgr/dashboard: Changes suggested after review by
 @epuertat.

Move the JWT requirement to the test requirements file. Also remove JWT from ceph specification and debian build.

Signed-off-by: Daniel Persson <mailto.woden@gmail.com>
---
 ceph.spec.in                                   | 4 ----
 debian/control                                 | 1 -
 src/pybind/mgr/dashboard/requirements-test.txt | 1 +
 src/pybind/mgr/dashboard/tox.ini               | 1 -
 4 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 3807f48280e9..9aa2432dcd48 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -414,7 +414,6 @@ BuildRequires:	xmlsec1-nss
 BuildRequires:	xmlsec1-openssl
 BuildRequires:	xmlsec1-openssl-devel
 BuildRequires:	python%{python3_pkgversion}-cherrypy
-BuildRequires:	python%{python3_pkgversion}-jwt
 BuildRequires:	python%{python3_pkgversion}-routes
 BuildRequires:	python%{python3_pkgversion}-scipy
 BuildRequires:	python%{python3_pkgversion}-werkzeug
@@ -427,7 +426,6 @@ BuildRequires:	libxmlsec1-1
 BuildRequires:	libxmlsec1-nss1
 BuildRequires:	libxmlsec1-openssl1
 BuildRequires:	python%{python3_pkgversion}-CherryPy
-BuildRequires:	python%{python3_pkgversion}-PyJWT
 BuildRequires:	python%{python3_pkgversion}-Routes
 BuildRequires:	python%{python3_pkgversion}-Werkzeug
 BuildRequires:	python%{python3_pkgversion}-numpy-devel
@@ -629,7 +627,6 @@ Requires:       ceph-prometheus-alerts = %{_epoch_prefix}%{version}-%{release}
 Requires:       python%{python3_pkgversion}-setuptools
 %if 0%{?fedora} || 0%{?rhel} || 0%{?openEuler}
 Requires:       python%{python3_pkgversion}-cherrypy
-Requires:       python%{python3_pkgversion}-jwt
 Requires:       python%{python3_pkgversion}-routes
 Requires:       python%{python3_pkgversion}-werkzeug
 %if 0%{?weak_deps}
@@ -638,7 +635,6 @@ Recommends:     python%{python3_pkgversion}-saml
 %endif
 %if 0%{?suse_version}
 Requires:       python%{python3_pkgversion}-CherryPy
-Requires:       python%{python3_pkgversion}-PyJWT
 Requires:       python%{python3_pkgversion}-Routes
 Requires:       python%{python3_pkgversion}-Werkzeug
 Recommends:     python%{python3_pkgversion}-python3-saml
diff --git a/debian/control b/debian/control
index 5ae413ce774a..f1bdec95a2ef 100644
--- a/debian/control
+++ b/debian/control
@@ -91,7 +91,6 @@ Build-Depends: automake,
                python3-all-dev,
                python3-cherrypy3,
                python3-natsort,
-               python3-jwt <pkg.ceph.check>,
                python3-pecan <pkg.ceph.check>,
                python3-bcrypt <pkg.ceph.check>,
                tox <pkg.ceph.check>,
diff --git a/src/pybind/mgr/dashboard/requirements-test.txt b/src/pybind/mgr/dashboard/requirements-test.txt
index da283d0b64aa..aa80b3336b54 100644
--- a/src/pybind/mgr/dashboard/requirements-test.txt
+++ b/src/pybind/mgr/dashboard/requirements-test.txt
@@ -2,3 +2,4 @@ pytest-cov
 pytest-instafail
 pyfakefs==4.5.0
 jsonschema~=4.0
+PyJWT~=2.0
diff --git a/src/pybind/mgr/dashboard/tox.ini b/src/pybind/mgr/dashboard/tox.ini
index 271df286ec5e..47756e946e12 100644
--- a/src/pybind/mgr/dashboard/tox.ini
+++ b/src/pybind/mgr/dashboard/tox.ini
@@ -20,7 +20,6 @@ addopts =
 deps =
     -rrequirements.txt
     -cconstraints.txt
-    PyJWT
 
 [base-test]
 deps =

From 06765e648acb1676d5d563c631b8d8fc08b5323c Mon Sep 17 00:00:00 2001
From: Daniel Persson <mailto.woden@gmail.com>
Date: Sun, 3 Dec 2023 09:46:56 +0000
Subject: [PATCH 1055/2492] mgr/dashboard: Updated test dependencies

Seemed that the test dependencies was separated in two different requirements files
one for the testing and one for linting. Added the JWT dependency in the linting file
as well.

Signed-off-by: Daniel Persson <mailto.woden@gmail.com>
---
 src/pybind/mgr/dashboard/requirements-lint.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pybind/mgr/dashboard/requirements-lint.txt b/src/pybind/mgr/dashboard/requirements-lint.txt
index 57e519157408..571c92a4ebfb 100644
--- a/src/pybind/mgr/dashboard/requirements-lint.txt
+++ b/src/pybind/mgr/dashboard/requirements-lint.txt
@@ -9,3 +9,4 @@ autopep8==1.5.7
 pyfakefs==4.5.0
 isort==5.5.3
 jsonschema~=4.0
+PyJWT~=2.0

From 6b04749215de73b74007e6496f0ca68af78c13b1 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Dec 2023 09:30:35 -0500
Subject: [PATCH 1056/2492] rgw/multisite: forwarded requests always pass a
 bufferlist

d2dbe7550296da6db885b5344c71f77f9acbfd8f added a rgw_forward_request_to_master()
that took the input bufferlist by pointer instead of reference so it
could be optional; however, RGWRESTSimpleRequest::forward_request()
omits the Content-Length header when the data is nullptr. this was an
unintended change and broke the forwarding of some requests

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 750ad7cb7739..d4162ec2e3d3 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -132,6 +132,12 @@ int rgw_forward_request_to_master(const DoutPrefixProvider* dpp,
   }
   const RGWAccessKey& creds = site.get_zone_params().system_key;
 
+  bufferlist data;
+  if (indata == nullptr) {
+    // forward() needs an input bufferlist to set the content-length
+    indata = &data;
+  }
+
   // use the master zone's endpoints
   auto conn = RGWRESTConn{dpp->get_cct(), z->second.id, z->second.endpoints,
                           creds, zg->second.id, zg->second.api_name};

From 1950cc0ea764bb03f3598dc2c4576110d0d51d9d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 14:57:15 -0500
Subject: [PATCH 1057/2492] cephadm: add InstallSpec type to build.py

Add a new InstallSpec type to build.py and rework the requirements lists
in terms of the InstallSpec. The new type will be able to encode more
per-dependency behavior than the previous approach.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 74 +++++++++++++++++++++++++++++++-------------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 1634eeac5931..54de52a1fa8a 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -31,12 +31,18 @@
 
 
 PY36_REQUIREMENTS = [
-    'MarkupSafe >= 2.0.1, <2.2',
-    'Jinja2 >= 3.0.2, <3.2',
+    {
+        'package_spec': 'MarkupSafe >= 2.0.1, <2.2',
+        'unique': True,
+    },
+    {
+        'package_spec': 'Jinja2 >= 3.0.2, <3.2',
+        'unique': True,
+    },
 ]
 PY_REQUIREMENTS = [
-    'MarkupSafe >= 2.1.3, <2.2',
-    'Jinja2 >= 3.1.2, <3.2',
+    {'package_spec': 'MarkupSafe >= 2.1.3, <2.2'},
+    {'package_spec': 'Jinja2 >= 3.1.2, <3.2'},
 ]
 # IMPORTANT to be fully compatible with all the distros ceph is built for we
 # need to work around various old versions of python/pip. As such it's easier
@@ -53,6 +59,33 @@
 ]
 
 
+class InstallSpec:
+    def __init__(
+        self, package_spec, custom_pip_args=None, unique=False, **kwargs
+    ):
+        self.package_spec = package_spec
+        self.name = package_spec.split()[0]
+        self.custom_pip_args = custom_pip_args or []
+        self.unique = unique
+        self.extra = kwargs
+
+    @property
+    def pip_args(self):
+        return self.custom_pip_args
+
+    @property
+    def pip_args_and_package(self):
+        return self.pip_args + [self.package_spec]
+
+    def compatible(self, other):
+        return (
+            other
+            and not self.unique
+            and not other.unique
+            and self.pip_args == other.pip_args
+        )
+
+
 class PipEnv(enum.Enum):
     never = enum.auto()
     auto = enum.auto()
@@ -84,15 +117,13 @@ def __init__(self, cli_args):
 
     def _setup_pip(self):
         if self._maj_min == (3, 6):
-            self.pip_split = True
-            self.requirements = PY36_REQUIREMENTS
+            self.requirements = [InstallSpec(**v) for v in PY36_REQUIREMENTS]
         else:
-            self.pip_split = False
-            self.requirements = PY_REQUIREMENTS
+            self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS]
         self.pip_venv = PipEnv[self.cli_args.pip_use_venv]
 
     def _setup_rpm(self):
-        self.requirements = [s.split()[0] for s in PY_REQUIREMENTS]
+        self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS]
 
 
 class DependencyInfo:
@@ -101,7 +132,7 @@ class DependencyInfo:
     def __init__(self, config):
         self._config = config
         self._deps = []
-        self._reqs = {s.split()[0]: s for s in self._config.requirements}
+        self._reqs = {s.name: s.package_spec for s in self._config.requirements}
 
     @property
     def requirements(self):
@@ -259,15 +290,16 @@ def _install_pip_deps(tempdir, config):
         env['PYTHONPATH'] = env['PYTHONPATH'] + f':{tempdir}'
     else:
         env['PYTHONPATH'] = f'{tempdir}'
-    if config.pip_split:
-        # a list of single item lists; so that pip run once for each
-        # requirement
-        req_batches = [[r] for r in config.requirements]
-    else:
-        # a list containing another list of the requirements, so we only
-        # need to run pip once
-        req_batches = [list(config.requirements)]
-    for batch in req_batches:
+
+    pip_args = []
+    prev = None
+    for ispec in config.requirements:
+        if ispec.compatible(prev) and pip_args:
+            pip_args[0].append(ispec.package_spec)
+        else:
+            pip_args.append(ispec.pip_args_and_package)
+        prev = ispec
+    for batch in pip_args:
         _run(
             [
                 executable,
@@ -321,8 +353,8 @@ def _install_rpm_deps(tempdir, config):
     log.info("Installing dependencies using RPMs")
     dinfo = DependencyInfo(config)
     for pkg in config.requirements:
-        log.info(f"Looking for rpm package for: {pkg!r}")
-        _deps_from_rpm(tempdir, config, dinfo, pkg)
+        log.info(f"Looking for rpm package for: {pkg.name!r}")
+        _deps_from_rpm(tempdir, config, dinfo, pkg.name)
     return dinfo
 
 

From adb28150e45b0ec3b054d3bcd4a0b6a5a2f7156f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 15:13:01 -0500
Subject: [PATCH 1058/2492] cephadm: make the pip-install-from-source approach
 for deps explicit

Previously, the code assumed that all dependencies can be cleanly
installed from source with native-code compilers disabled. This worked
ok for MarkupSafe because it falls back to pure-python if it can't
compile the C extension code. Unfortunately, not all packages that can
fall back to pure python do so cleanly in the build. As a first step
to resolving that, make the current behavior an explicitly selected mode
and permit installing from wheels in the future.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 54de52a1fa8a..06dd376be2f0 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -33,16 +33,18 @@
 PY36_REQUIREMENTS = [
     {
         'package_spec': 'MarkupSafe >= 2.0.1, <2.2',
+        'from_source': True,
         'unique': True,
     },
     {
         'package_spec': 'Jinja2 >= 3.0.2, <3.2',
+        'from_source': True,
         'unique': True,
     },
 ]
 PY_REQUIREMENTS = [
-    {'package_spec': 'MarkupSafe >= 2.1.3, <2.2'},
-    {'package_spec': 'Jinja2 >= 3.1.2, <3.2'},
+    {'package_spec': 'MarkupSafe >= 2.1.3, <2.2', 'from_source': True},
+    {'package_spec': 'Jinja2 >= 3.1.2, <3.2', 'from_source': True},
 ]
 # IMPORTANT to be fully compatible with all the distros ceph is built for we
 # need to work around various old versions of python/pip. As such it's easier
@@ -61,17 +63,27 @@
 
 class InstallSpec:
     def __init__(
-        self, package_spec, custom_pip_args=None, unique=False, **kwargs
+        self,
+        package_spec,
+        custom_pip_args=None,
+        unique=False,
+        from_source=False,
+        **kwargs,
     ):
         self.package_spec = package_spec
         self.name = package_spec.split()[0]
         self.custom_pip_args = custom_pip_args or []
         self.unique = unique
+        self.from_source = from_source
         self.extra = kwargs
 
     @property
     def pip_args(self):
-        return self.custom_pip_args
+        args = []
+        if self.from_source:
+            args.append("--no-binary")
+            args.append(":all:")
+        return args + self.custom_pip_args
 
     @property
     def pip_args_and_package(self):
@@ -306,8 +318,6 @@ def _install_pip_deps(tempdir, config):
                 "-m",
                 "pip",
                 "install",
-                "--no-binary",
-                ":all:",
                 "--target",
                 tempdir,
             ]

From 68735aa0f558c410c107f70bab754a1c4b11e5b9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 16:37:45 -0500
Subject: [PATCH 1059/2492] cephadm: add a pyyaml zipapp dependency

Add a pyyaml zipapp dependency based on installing from wheels but
stripping those wheels of compiled (shared object) extensions. This is
the only way I could get it to work because the pyproject.toml file in
pyyaml requires Cython and having Cython present enables the extension
in setup.py even though some setup.py options exist to disable building
the C/Cython based extension.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 56 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 06dd376be2f0..3ca8cdd4e262 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -8,6 +8,7 @@
 import argparse
 import compileall
 import enum
+import functools
 import json
 import logging
 import os
@@ -41,10 +42,26 @@
         'from_source': True,
         'unique': True,
     },
+    {
+        'package_spec': 'PyYAML >= 6.0, <6.1',
+        # do not include the stub package for compatibility with
+        # old versions of the extension module. We are going out of our
+        # way to avoid the binary extension module for our zipapp, no
+        # point in pulling this unnecessary module for wrapping it.
+        'ignore_exact': ['_yaml'],
+    },
 ]
 PY_REQUIREMENTS = [
     {'package_spec': 'MarkupSafe >= 2.1.3, <2.2', 'from_source': True},
     {'package_spec': 'Jinja2 >= 3.1.2, <3.2', 'from_source': True},
+    {
+        'package_spec': 'PyYAML >= 6.0, <6.1',
+        # do not include the stub package for compatibility with
+        # old versions of the extension module. We are going out of our
+        # way to avoid the binary extension module for our zipapp, no
+        # point in pulling this unnecessary module for wrapping it.
+        'ignore_exact': ['_yaml'],
+    },
 ]
 # IMPORTANT to be fully compatible with all the distros ceph is built for we
 # need to work around various old versions of python/pip. As such it's easier
@@ -68,6 +85,8 @@ def __init__(
         custom_pip_args=None,
         unique=False,
         from_source=False,
+        ignore_suffixes=None,
+        ignore_exact=None,
         **kwargs,
     ):
         self.package_spec = package_spec
@@ -75,6 +94,8 @@ def __init__(
         self.custom_pip_args = custom_pip_args or []
         self.unique = unique
         self.from_source = from_source
+        self.ignore_suffixes = ignore_suffixes or []
+        self.ignore_exact = ignore_exact or []
         self.extra = kwargs
 
     @property
@@ -197,18 +218,31 @@ def _build(dest, src, config):
     tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
     log.debug("working in %s", tempdir)
     dinfo = None
+    appdir = tempdir / "app"
     try:
         if config.install_dependencies:
-            dinfo = _install_deps(tempdir, config)
+            depsdir = tempdir / "deps"
+            dinfo = _install_deps(depsdir, config)
+            ignore_suffixes = []
+            ignore_exact = []
+            for ispec in config.requirements:
+                ignore_suffixes.extend(ispec.ignore_suffixes)
+                ignore_exact.extend(ispec.ignore_exact)
+            ignorefn = functools.partial(
+                _ignore_cephadmlib,
+                ignore_suffixes=ignore_suffixes,
+                ignore_exact=ignore_exact,
+            )
+            shutil.copytree(depsdir, appdir, ignore=ignorefn)
         log.info("Copying contents")
         # cephadmlib is cephadm's private library of modules
         shutil.copytree(
-            "cephadmlib", tempdir / "cephadmlib", ignore=_ignore_cephadmlib
+            "cephadmlib", appdir / "cephadmlib", ignore=_ignore_cephadmlib
         )
         # cephadm.py is cephadm's main script for the "binary"
         # this must be renamed to __main__.py for the zipapp
-        shutil.copy("cephadm.py", tempdir / "__main__.py")
-        mdir = tempdir / "_cephadmmeta"
+        shutil.copy("cephadm.py", appdir / "__main__.py")
+        mdir = appdir / "_cephadmmeta"
         mdir.mkdir(parents=True, exist_ok=True)
         (mdir / "__init__.py").touch(exist_ok=True)
         versioning_vars = config.cli_args.version_vars
@@ -216,19 +250,23 @@ def _build(dest, src, config):
             generate_version_file(versioning_vars, mdir / "version.py")
         if dinfo:
             dinfo.save(mdir / "deps.json")
-        _compile(dest, tempdir)
+        _compile(dest, appdir)
     finally:
         shutil.rmtree(tempdir)
 
 
-def _ignore_cephadmlib(source_dir, names):
+def _ignore_cephadmlib(source_dir, names, ignore_suffixes=None, ignore_exact=None):
     # shutil.copytree callback: return the list of names *to ignore*
+    suffixes = ["~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__"]
+    exact = []
+    if ignore_suffixes:
+        suffixes += ignore_suffixes
+    if ignore_exact:
+        exact += ignore_exact
     return [
         name
         for name in names
-        if name.endswith(
-            ("~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__")
-        )
+        if name.endswith(tuple(suffixes)) or name in exact
     ]
 
 

From d4f73f2d813406fcc8dd1994899e12378ff9b2f2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 16:51:47 -0500
Subject: [PATCH 1060/2492] cephadm: update test cases for build.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/build/test_cephadm_build.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/tests/build/test_cephadm_build.py b/src/cephadm/tests/build/test_cephadm_build.py
index 7cf51961ffca..1465c2c5efea 100644
--- a/src/cephadm/tests/build/test_cephadm_build.py
+++ b/src/cephadm/tests/build/test_cephadm_build.py
@@ -25,12 +25,12 @@
     'centos-8-plusdeps': {
         'name': 'cephadm-build-test:centos8-py36-deps',
         'base_image': 'quay.io/centos/centos:stream8',
-        'script': 'dnf install -y python36 python3-jinja2',
+        'script': 'dnf install -y python36 python3-jinja2 python3-pyyaml',
     },
     'centos-9-plusdeps': {
         'name': 'cephadm-build-test:centos9-py3-deps',
         'base_image': 'quay.io/centos/centos:stream9',
-        'script': 'dnf install -y python3 python3-jinja2',
+        'script': 'dnf install -y python3 python3-jinja2 python3-pyyaml',
     },
     'ubuntu-20.04': {
         'name': 'cephadm-build-test:ubuntu-20-04-py3',
@@ -122,7 +122,7 @@ def test_cephadm_build(env, source_dir, tmp_path):
     assert 'bundled_packages' in data
     assert all(v['package_source'] == 'pip' for v in data['bundled_packages'])
     assert all(
-        v['name'] in ('Jinja2', 'MarkupSafe')
+        v['name'] in ('Jinja2', 'MarkupSafe', 'PyYAML')
         for v in data['bundled_packages']
     )
     assert all('requirements_entry' in v for v in data['bundled_packages'])
@@ -178,7 +178,7 @@ def test_cephadm_build_from_rpms(env, source_dir, tmp_path):
     assert 'bundled_packages' in data
     assert all(v['package_source'] == 'rpm' for v in data['bundled_packages'])
     assert all(
-        v['name'] in ('Jinja2', 'MarkupSafe')
+        v['name'] in ('Jinja2', 'MarkupSafe', 'PyYAML')
         for v in data['bundled_packages']
     )
     assert all('requirements_entry' in v for v in data['bundled_packages'])

From 7b53a2e34fdafbfa07be47469795041599226174 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 17:15:05 -0500
Subject: [PATCH 1061/2492] cepadm: black reformat build.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 3ca8cdd4e262..da237174ad0d 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -165,7 +165,9 @@ class DependencyInfo:
     def __init__(self, config):
         self._config = config
         self._deps = []
-        self._reqs = {s.name: s.package_spec for s in self._config.requirements}
+        self._reqs = {
+            s.name: s.package_spec for s in self._config.requirements
+        }
 
     @property
     def requirements(self):
@@ -186,7 +188,6 @@ def save(self, path):
             json.dump(self._deps, fh)
 
 
-
 def _run(command, *args, **kwargs):
     log.info(
         'Running cmd: %s', ' '.join(shlex.quote(str(c)) for c in command)
@@ -255,7 +256,9 @@ def _build(dest, src, config):
         shutil.rmtree(tempdir)
 
 
-def _ignore_cephadmlib(source_dir, names, ignore_suffixes=None, ignore_exact=None):
+def _ignore_cephadmlib(
+    source_dir, names, ignore_suffixes=None, ignore_exact=None
+):
     # shutil.copytree callback: return the list of names *to ignore*
     suffixes = ["~", ".old", ".swp", ".pyc", ".pyo", ".so", "__pycache__"]
     exact = []

From 1ced6bc8894f720ffafb7a2e10d169054ff9fa60 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 12 Dec 2023 17:14:44 -0500
Subject: [PATCH 1062/2492] cephadm: add comments documenting install spec
 options to build.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/build.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index da237174ad0d..ef2bfef78fdf 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -31,6 +31,23 @@
 log = logging.getLogger(__name__)
 
 
+# Fill in the package requirements for the zipapp build below. The PY36_REQUIREMENTS
+# list applies *only* to python 3.6. The PY_REQUIREMENTS list applies to all other
+# python versions. Python lower than 3.6 is not supported by this script.
+#
+# Each item must be a dict with the following fields:
+# - package_spec (REQUIRED, str): A python package requirement in the same style as
+#   requirements.txt and pip.
+# - from_source (bool): Try to force a clean no-binaries build using source packages.
+# - unique (bool): If true, this requirement should not be combined with any other
+#   on the pip command line.
+# - ignore_suffixes (list of str): A list of file and directory suffixes to EXCLUDE
+#   from the final zipapp.
+# - ignore_exact (list of str): A list of exact file and directory names to EXCLUDE
+#   from the final zipapp.
+# - custom_pip_args (list of str): A list of additional custom arguments to pass
+#   to pip when installing this dependency.
+#
 PY36_REQUIREMENTS = [
     {
         'package_spec': 'MarkupSafe >= 2.0.1, <2.2',
@@ -54,6 +71,11 @@
 PY_REQUIREMENTS = [
     {'package_spec': 'MarkupSafe >= 2.1.3, <2.2', 'from_source': True},
     {'package_spec': 'Jinja2 >= 3.1.2, <3.2', 'from_source': True},
+    # We can not install PyYAML using sources. Unlike MarkupSafe it requires
+    # Cython to build and Cython must be compiled and there's not clear way past
+    # the requirement in pyyaml's pyproject.toml. Instead, rely on fetching
+    # a platform specific pyyaml wheel and then stripping of the binary shared
+    # object.
     {
         'package_spec': 'PyYAML >= 6.0, <6.1',
         # do not include the stub package for compatibility with

From 8ddd5b4dc5e2afd9e7e2b8d1a4c660e1b029e720 Mon Sep 17 00:00:00 2001
From: HuangWei <huangwei.a180285@gmail.com>
Date: Sat, 25 Nov 2023 16:52:21 +0800
Subject: [PATCH 1063/2492] Update rbd import-diff command help message

Signed-off-by: HuangWei <huangwei.a180285@gmail.com>
---
 src/test/cli/rbd/help.t        | 5 +++--
 src/tools/rbd/action/Import.cc | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index a80c0fb491ed..2d1c901462c7 100644
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -63,7 +63,8 @@
                                         associated.
       image-meta set                    Image metadata set key with value.
       import                            Import image from file.
-      import-diff                       Import an incremental diff.
+      import-diff                       Apply an incremental diff to image HEAD,
+                                        then create a snapshot.
       info                              Show information about image size,
                                         striping, etc.
       journal client disconnect         Flag image journal client as disconnected.
@@ -1254,7 +1255,7 @@
                          [--sparse-size <sparse-size>] [--no-progress] 
                          <path-name> <image-spec> 
   
-  Import an incremental diff.
+  Apply an incremental diff to image HEAD, then create a snapshot.
   
   Positional arguments
     <path-name>          import file (or '-' for stdin)
diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc
index 3358c5bc6855..f6b3c4c3ccd8 100644
--- a/src/tools/rbd/action/Import.cc
+++ b/src/tools/rbd/action/Import.cc
@@ -526,8 +526,9 @@ int execute_diff(const po::variables_map &vm,
 }
 
 Shell::Action action_diff(
-  {"import-diff"}, {}, "Import an incremental diff.", "", &get_arguments_diff,
-  &execute_diff);
+  {"import-diff"}, {},
+  "Apply an incremental diff to image HEAD, then create a snapshot.", "",
+  &get_arguments_diff, &execute_diff);
 
 class C_Import : public Context {
 public:

From 15479752096d7d927c1e0051cebc6144246d7b2e Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Wed, 13 Dec 2023 16:59:16 +0000
Subject: [PATCH 1064/2492] src/mon/OSDMonitor.cc: more descriptive loggings
 for crc mismatch

Problem:
When mons are running different versions, e.g.,
during upgrades, we sometimes will encounter a crc
mismatch between the crc that is generated
from the leader MON (version n+1) and
the peon MON (version). Which will
later then gets resolved when all the MONs
are at version n+1.

Solution:
Provide a more informative loggings when
we encounter a CRC mismatch and also
output the mon versions so it's easier
for us to detect which if the crc mismatch
comes from OSDMap running mixed versions or
not.

Fixes: https://tracker.ceph.com/issues/63389

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/mon/OSDMonitor.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 79fff068c332..8deaba4f9107 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -847,6 +847,7 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
 
     bufferlist orig_full_bl;
     get_version_full(osdmap.epoch, orig_full_bl);
+    dout(20) << __func__ << " mon is running version: " << ceph_version_to_str() << dendl;
     if (orig_full_bl.length()) {
       // the primary provided the full map
       ceph_assert(inc.have_crc);
@@ -857,8 +858,12 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
 	// sync with the primary for this and all future maps.  OSDs
 	// will also be brought back into sync when they discover the
 	// crc mismatch and request a full map from a mon.
-	derr << __func__ << " full map CRC mismatch, resetting to canonical"
-	     << dendl;
+	derr << __func__ << " full map CRC mismatch,"
+    << " might be because mons are running mixed versions ..."
+    << " resetting to canonical" << dendl;
+
+  dout(20) << __func__ << " canonical crc: " << inc.full_crc
+    << " my crc: " << osdmap.crc << dendl;
 
 	dout(20) << __func__ << " my (bad) full osdmap:\n";
 	JSONFormatter jf(true);
@@ -2024,6 +2029,8 @@ void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t)
   bufferlist bl;
   encode(pending_inc, bl, features | CEPH_FEATURE_RESERVED);
 
+  dout(20) << __func__ << " mon is running version: "
+    << ceph_version_to_str() << dendl;
   dout(20) << " full_crc " << tmp.get_crc()
 	   << " inc_crc " << pending_inc.inc_crc << dendl;
 

From ce6905f9e852c8fa4373002413f9bf563cd0c152 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 8 Dec 2023 15:26:21 -0500
Subject: [PATCH 1065/2492] rgw/multisite: error repo coroutines store raw_obj
 by value

RGWErrorRepoWriteCR/RGWErrorRepoRemoveCR need to copy their rgw_raw_obj
arguments to extend the lifetimes until send_request() is called

potential regression from https://github.com/ceph/ceph/pull/50359

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sync_error_repo.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.cc b/src/rgw/driver/rados/rgw_sync_error_repo.cc
index aca671039ee1..c180497bd73d 100644
--- a/src/rgw/driver/rados/rgw_sync_error_repo.cc
+++ b/src/rgw/driver/rados/rgw_sync_error_repo.cc
@@ -118,7 +118,7 @@ int remove(librados::ObjectWriteOperation& op,
 
 class RGWErrorRepoWriteCR : public RGWSimpleCoroutine {
   librados::Rados* rados;
-  const rgw_raw_obj& raw_obj;
+  rgw_raw_obj raw_obj;
   std::string key;
   ceph::real_time timestamp;
 
@@ -163,7 +163,7 @@ RGWCoroutine* write_cr(librados::Rados* rados,
 
 class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine {
   librados::Rados* rados;
-  const rgw_raw_obj& raw_obj;
+  rgw_raw_obj raw_obj;
   std::string key;
   ceph::real_time timestamp;
 

From b4592f3588d2cff5766949508d181ef603128ff4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Dec 2023 15:32:37 -0500
Subject: [PATCH 1066/2492] rgw/multisite: error repo coroutines initialize
 rados pointer

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sync_error_repo.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.cc b/src/rgw/driver/rados/rgw_sync_error_repo.cc
index c180497bd73d..b07037d655d3 100644
--- a/src/rgw/driver/rados/rgw_sync_error_repo.cc
+++ b/src/rgw/driver/rados/rgw_sync_error_repo.cc
@@ -127,7 +127,7 @@ class RGWErrorRepoWriteCR : public RGWSimpleCoroutine {
   RGWErrorRepoWriteCR(librados::Rados* rados, const rgw_raw_obj& raw_obj,
                       const std::string& key, ceph::real_time timestamp)
     : RGWSimpleCoroutine(static_cast<CephContext*>(rados->cct())),
-      raw_obj(raw_obj),
+      rados(rados), raw_obj(raw_obj),
       key(key), timestamp(timestamp)
   {}
 
@@ -172,7 +172,7 @@ class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine {
   RGWErrorRepoRemoveCR(librados::Rados* rados, const rgw_raw_obj& raw_obj,
                        const std::string& key, ceph::real_time timestamp)
     : RGWSimpleCoroutine(static_cast<CephContext*>(rados->cct())),
-      raw_obj(raw_obj),
+      rados(rados), raw_obj(raw_obj),
       key(key), timestamp(timestamp)
   {}
 

From 40337a8e472579d05a4d1cc44f7248dd1447a982 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 16 Oct 2023 21:33:08 -0700
Subject: [PATCH 1067/2492] os/bluestore/BlueStore: fix crimson-only
 omap_get_values start handling

seek_to_first() if start is nullopt, upper_bound() otherwise.

Introduced: 9cf183ed46ad
Fixes: https://tracker.ceph.com/issues/63819
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 0f79fc4f0958..edc8e7ac2644 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -13277,7 +13277,11 @@ int BlueStore::omap_get_values(
       r = -ENOENT;
       goto out;
     }
-    iter->upper_bound(*start_after);
+    if (start_after) {
+      iter->upper_bound(*start_after);
+    } else {
+      iter->seek_to_first();
+    }
     for (; iter->valid(); iter->next()) {
       output->insert(make_pair(iter->key(), iter->value()));
     }

From 999845c25121cf8d15fdce6b23ee5b3c4b584a9b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 13 Dec 2023 12:43:35 -0800
Subject: [PATCH 1068/2492] crimson/osd/pg: rollback obc unconditionally before
 error log

If we're recording an error to the log, the write definitionally did not
happen.  Rollback the obc regardless of the specific error code.

Introduced: 1d98e8da
Fixes: https://tracker.ceph.com/issues/63820
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/pg.cc | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 00592eb8347e..a0b74ff9b21e 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -938,14 +938,8 @@ PG::do_osd_ops_execute(
     [this, error_func_ptr, rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
 
-    PG::interruptible_future<> maybe_rollback_fut = seastar::now();
     ceph_tid_t rep_tid = shard_services.get_tid();
-
-    if (e.value() == ENOENT) {
-      maybe_rollback_fut = rollbacker.rollback_obc_if_modified(e);
-    }
-
-    return maybe_rollback_fut.then_interruptible(
+    return rollbacker.rollback_obc_if_modified(e).then_interruptible(
     [error_func_ptr, e, rep_tid, failure_func_ptr] {
       // record error log
       return (*error_func_ptr)(e, rep_tid).then(

From f053cdd00d6ac13d0954d8821dce301bd3eb5150 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Wed, 13 Dec 2023 17:03:00 +0800
Subject: [PATCH 1069/2492] test/common: switch to INSTANTIATE_TEST_SUITE_P

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/test/common/test_memory.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/test/common/test_memory.cc b/src/test/common/test_memory.cc
index cf075f01e5d3..05992e5897c6 100644
--- a/src/test/common/test_memory.cc
+++ b/src/test/common/test_memory.cc
@@ -30,7 +30,7 @@ TEST_P(MemoryIsZeroBigTest, MemoryIsZeroTestBig) {
 TEST_P(MemoryIsZeroSmallTest, MemoryIsZeroTestSmall) {
   size_t size = GetParam();
   for (size_t i = 0; i < size; i++) {
-    char data[size] = { 0 };
+    char* data = new char[size]();
     EXPECT_TRUE(mem_is_zero(data, size));
 
     data[i] = 'a';
@@ -63,11 +63,11 @@ TEST_P(MemoryIsZeroPerformance, MemoryIsZeroPerformanceTest) {
   free(data);
 }
 
-INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroSmallTest,
+INSTANTIATE_TEST_SUITE_P(MemoryIsZeroSmallTests, MemoryIsZeroSmallTest,
                         ::testing::Values(1, 4, 7, 8, 12, 28, 60, 64));
 
-INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroBigTest,
+INSTANTIATE_TEST_SUITE_P(MemoryIsZeroBigTests, MemoryIsZeroBigTest,
                         ::testing::Values(1024, 4096, 8192, 64 * 1024));
 
-INSTANTIATE_TEST_CASE_P(Default, MemoryIsZeroPerformance,
+INSTANTIATE_TEST_SUITE_P(MemoryIsZeroPerformanceTests, MemoryIsZeroPerformance,
                         ::testing::Values(1024, 2048, 4096, 8192, 64 * 1024));

From 0e43e691fbf84ba140f7aca68f475c46bb96f4df Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Thu, 14 Dec 2023 09:38:08 +0000
Subject: [PATCH 1070/2492] os/bluestore: Fix Blob::copy_extents function

The function had an error, that it assumed its smallest operational block is allocation unit.
In reality, the smallest block is min_release_size, that is a max(allocation size, checksum size).
In some wierd cases single min_release_size block could be composed of *disjointed*
allocation on the disk. This is something that bitmap allocator does often.

Blob::copy_extents could not handle it properly, but luckily, it asserted when faced with that data.

New fixed and improved version is able to handle this case too.

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
---
 src/os/bluestore/BlueStore.cc | 75 +++++++++++++++++++++++++----------
 1 file changed, 53 insertions(+), 22 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index b8dca31c0575..a69bae5a1fc9 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2534,21 +2534,60 @@ void BlueStore::Blob::copy_extents(
   CephContext* cct, const Blob& from, uint32_t start,
   uint32_t pre_len, uint32_t main_len, uint32_t post_len)
 {
-  constexpr uint64_t invalid = bluestore_pextent_t::INVALID_OFFSET;
-  auto at = [&](const PExtentVector& e, uint32_t pos, uint32_t len) -> uint64_t {
-    auto it = e.begin();
-    while (it != e.end() && pos >= it->length) {
-      pos -= it->length;
-      ++it;
+  // There are 2 valid states:
+  // 1) `to` is not defined on [pos~len] range
+  //    (need to copy this region - return true)
+  // 2) `from` and `to` are exact on [pos~len] range
+  //    (no need to copy region - return false)
+  // Otherwise just assert.
+  auto check_sane_need_copy = [&](
+    const PExtentVector& from,
+    const PExtentVector& to,
+    uint32_t pos, uint32_t len) -> bool
+  {
+    uint32_t pto = pos;
+    auto ito = to.begin();
+    while (ito != to.end() && pto >= ito->length) {
+      pto -= ito->length;
+      ++ito;
     }
-    if (it == e.end()) {
-      return invalid;
+    if (ito == to.end()) return true; // case 1 - obviously empty
+    if (!ito->is_valid()) {
+      // now sanity check that all the rest is invalid too
+      pto += len;
+      while (ito != to.end() && pto >= ito->length) {
+        ceph_assert(!ito->is_valid());
+        pto -= ito->length;
+        ++ito;
+      }
+      return true;
     }
-    if (!it->is_valid()) {
-      return invalid;
+    uint32_t pfrom = pos;
+    auto ifrom = from.begin();
+    while (ifrom != from.end() && pfrom >= ifrom->length) {
+      pfrom -= ifrom->length;
+      ++ifrom;
+    }
+    ceph_assert(ifrom != from.end());
+    ceph_assert(ifrom->is_valid());
+    // here we require from and to be the same
+    while (len > 0) {
+      ceph_assert(ifrom->offset + pfrom == ito->offset + pto);
+      uint32_t jump = std::min(len, ifrom->length - pfrom);
+      jump = std::min(jump, ito->length - pto);
+      pfrom += jump;
+      if (pfrom == ifrom->length) {
+        pfrom = 0;
+        ++ifrom;
+      }
+      pto += jump;
+      if (pto == ito->length) {
+        pto = 0;
+        ++ito;
+      }
+      len -= jump;
     }
-    ceph_assert(pos + len <= it->length); // post_len should be single au, and we do not split
-    return it->offset + pos;
+    return false;
   };
   const PExtentVector& exfrom = from.blob.get_extents();
   PExtentVector& exto = blob.dirty_extents();
@@ -2557,24 +2596,16 @@ void BlueStore::Blob::copy_extents(
 
   // the extents that cover same area must be the same
   if (pre_len > 0) {
-    uint64_t au_from = at(exfrom, start, pre_len);
-    ceph_assert(au_from != bluestore_pextent_t::INVALID_OFFSET);
-    uint64_t au_to = at(exto, start, pre_len);
-    if (au_to == bluestore_pextent_t::INVALID_OFFSET) {
+    if (check_sane_need_copy(exfrom, exto, start, pre_len)) {
       main_len += pre_len; // also copy pre_len
     } else {
-      ceph_assert(au_from == au_to);
       start += pre_len; // skip, already there
     }
   }
   if (post_len > 0) {
-    uint64_t au_from = at(exfrom, start + main_len, post_len);
-    ceph_assert(au_from != bluestore_pextent_t::INVALID_OFFSET);
-    uint64_t au_to = at(exto, start + main_len, post_len);
-    if (au_to == bluestore_pextent_t::INVALID_OFFSET) {
+    if (check_sane_need_copy(exfrom, exto, start + main_len, post_len)) {
       main_len += post_len; // also copy post_len
     } else {
-      ceph_assert(au_from == au_to);
       // skip, already there
     }
   }

From 18c202c979aefe7c34de3688036586a070addff8 Mon Sep 17 00:00:00 2001
From: Ali Masarwa <ali.saed.masarwa@gmail.com>
Date: Sun, 19 Nov 2023 15:29:28 +0200
Subject: [PATCH 1071/2492] RGW:pubsub publish commit with etag populated

Signed-off-by: Ali Masarwa <ali.saed.masarwa@gmail.com>
---
 src/rgw/driver/daos/rgw_sal_daos.cc         |  2 +-
 src/rgw/driver/motr/rgw_sal_motr.cc         |  2 +-
 src/rgw/driver/posix/rgw_sal_posix.cc       |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc       |  2 +-
 src/rgw/rgw_op.cc                           | 63 +++++++++++++-------
 src/rgw/rgw_op.h                            |  5 ++
 src/rgw/rgw_sal_dbstore.cc                  |  2 +-
 src/test/rgw/bucket_notification/test_bn.py | 64 +++++++++++++++++++++
 8 files changed, 116 insertions(+), 26 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 694320565797..21252c906daa 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -1694,7 +1694,7 @@ int DaosMultipartUpload::complete(
   int marker = 0;
   uint64_t min_part_size = cct->_conf->rgw_multipart_min_part_size;
   auto etags_iter = part_etags.begin();
-  rgw::sal::Attrs attrs = target_obj->get_attrs();
+  rgw::sal::Attrs& attrs = target_obj->get_attrs();
 
   do {
     ldpp_dout(dpp, 20) << "DaosMultipartUpload::complete(): list_parts()"
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 08053b9b90c1..6a97ef2f01e8 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -2683,7 +2683,7 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp,
   int marker = 0;
   uint64_t min_part_size = cct->_conf->rgw_multipart_min_part_size;
   auto etags_iter = part_etags.begin();
-  rgw::sal::Attrs attrs = target_obj->get_attrs();
+  rgw::sal::Attrs& attrs = target_obj->get_attrs();
 
   do {
     ldpp_dout(dpp, 20) << "MotrMultipartUpload::complete(): list_parts()" << dendl;
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 5c1e50ca5c93..f5b79f9b0933 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -2612,7 +2612,7 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp,
   int marker = 0;
   uint64_t min_part_size = cct->_conf->rgw_multipart_min_part_size;
   auto etags_iter = part_etags.begin();
-  rgw::sal::Attrs attrs = target_obj->get_attrs();
+  rgw::sal::Attrs& attrs = target_obj->get_attrs();
 
   do {
     ret = list_parts(dpp, cct, max_parts, marker, &marker, &truncated, y);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 5ede8d44fa9b..171978c68629 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2422,7 +2422,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   int marker = 0;
   uint64_t min_part_size = cct->_conf->rgw_multipart_min_part_size;
   auto etags_iter = part_etags.begin();
-  rgw::sal::Attrs attrs = target_obj->get_attrs();
+  rgw::sal::Attrs& attrs = target_obj->get_attrs();
 
   do {
     ret = list_parts(dpp, cct, max_parts, marker, &marker, &truncated, y);
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 750ad7cb7739..57a21cbb55bf 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6591,9 +6591,6 @@ void RGWCompleteMultipart::execute(optional_yield y)
   RGWMultiCompleteUpload *parts;
   RGWMultiXMLParser parser;
   std::unique_ptr<rgw::sal::MultipartUpload> upload;
-  off_t ofs = 0;
-  std::unique_ptr<rgw::sal::Object> meta_obj;
-  std::unique_ptr<rgw::sal::Object> target_obj;
   uint64_t olh_epoch = 0;
 
   op_ret = get_params(y);
@@ -6682,8 +6679,8 @@ void RGWCompleteMultipart::execute(optional_yield y)
   
 
   // make reservation for notification if needed
-  std::unique_ptr<rgw::sal::Notification> res
-    = driver->get_notification(meta_obj.get(), nullptr, s, rgw::notify::ObjectCreatedCompleteMultipartUpload, y, &s->object->get_name());
+  res = driver->get_notification(meta_obj.get(), nullptr, s, rgw::notify::ObjectCreatedCompleteMultipartUpload, y,
+                                 &s->object->get_name());
   op_ret = res->publish_reserve(this);
   if (op_ret < 0) {
     return;
@@ -6706,21 +6703,10 @@ void RGWCompleteMultipart::execute(optional_yield y)
     return;
   }
 
-  // remove the upload meta object ; the meta object is not versioned
-  // when the bucket is, as that would add an unneeded delete marker
-  int r = meta_obj->delete_object(this, y, true /* prevent versioning */);
-  if (r >= 0)  {
-    /* serializer's exclusive lock is released */
-    serializer->clear_locked();
-  } else {
-    ldpp_dout(this, 0) << "WARNING: failed to remove object " << meta_obj << dendl;
-  }
-
-  // send request to notification manager
-  int ret = res->publish_commit(this, ofs, upload->get_mtime(), etag, target_obj->get_instance());
-  if (ret < 0) {
-    ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
-    // too late to rollback operation, hence op_ret is not set here
+  upload_time = upload->get_mtime();
+  int r = serializer->unlock();
+  if (r < 0) {
+    ldpp_dout(this, 0) << "WARNING: failed to unlock " << *serializer.get() << dendl;
   }
 } // RGWCompleteMultipart::execute
 
@@ -6773,7 +6759,42 @@ void RGWCompleteMultipart::complete()
     }
   }
 
-  etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str();
+  if (op_ret >= 0 && target_obj.get() != nullptr) {
+    s->object->set_attrs(target_obj->get_attrs());
+    etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str();
+    // send request to notification manager
+    if (res.get() != nullptr) {
+      int ret = res->publish_commit(this, ofs, upload_time, etag, target_obj->get_instance());
+      if (ret < 0) {
+        ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
+        // too late to rollback operation, hence op_ret is not set here
+      }
+    } else {
+      ldpp_dout(this, 1) << "ERROR: reservation is null" << dendl;
+    }
+  } else {
+    ldpp_dout(this, 1) << "ERROR: either op_ret is negative (execute failed) or target_obj is null, op_ret: "
+                       << op_ret << dendl;
+  }
+
+  // remove the upload meta object ; the meta object is not versioned
+  // when the bucket is, as that would add an unneeded delete marker
+  // moved to complete to prevent segmentation fault in publish commit
+  if (meta_obj.get() != nullptr) {
+    int ret = meta_obj->delete_object(this, null_yield, true /* prevent versioning */);
+    if (ret >= 0) {
+      /* serializer's exclusive lock is released */
+      serializer->clear_locked();
+    } else {
+      ldpp_dout(this, 0) << "WARNING: failed to remove object " << meta_obj << ", ret: " << ret << dendl;
+    }
+  } else {
+    ldpp_dout(this, 0) << "WARNING: meta_obj is null" << dendl;
+  }
+
+  res.reset();
+  meta_obj.reset();
+  target_obj.reset();
 
   send_response();
 }
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 9314d454c791..fcfb24786e8f 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1835,6 +1835,11 @@ class RGWCompleteMultipart : public RGWOp {
   bufferlist data;
   std::unique_ptr<rgw::sal::MPSerializer> serializer;
   jspan_ptr multipart_trace;
+  ceph::real_time upload_time;
+  std::unique_ptr<rgw::sal::Object> target_obj;
+  std::unique_ptr<rgw::sal::Notification> res;
+  std::unique_ptr<rgw::sal::Object> meta_obj;
+  off_t ofs = 0;
 
 public:
   RGWCompleteMultipart() {}
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 308404c87b88..4edb30166dbd 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -932,7 +932,7 @@ namespace rgw::sal {
     int marker = 0;
     uint64_t min_part_size = cct->_conf->rgw_multipart_min_part_size;
     auto etags_iter = part_etags.begin();
-    rgw::sal::Attrs attrs = target_obj->get_attrs();
+    rgw::sal::Attrs& attrs = target_obj->get_attrs();
 
     ofs = 0;
     accounted_size = 0;
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 075d47466a23..d493a57e004e 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -2397,6 +2397,70 @@ def test_http_post_object_upload():
     conn1.delete_bucket(Bucket=bucket_name)
 
 
+@attr('mpu_test')
+def test_ps_s3_multipart_on_master_http():
+    """ test http multipart object upload on master"""
+    conn = connection()
+    zonegroup = 'default'
+
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(host, port, num_workers=10)
+
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = conn.create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    opaque_data = 'http://1.2.3.4:8888'
+    topic_conf = PSTopicS3(conn, topic_name, zonegroup, endpoint_args=endpoint_args, opaque_data=opaque_data)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': []
+                        }]
+    s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    content = str(os.urandom(20*1024*1024))
+    key = bucket.new_key('obj')
+    thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+    thr.start()
+    client_threads.append(thr)
+    [thr.join() for thr in client_threads]
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check http receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    events = http_server.get_and_reset_events()
+    for event in events:
+        assert_equal(event['Records'][0]['opaqueData'], opaque_data)
+        assert_true(event['Records'][0]['s3']['object']['eTag'] != '')
+
+    # cleanup
+    for key in keys:
+        key.delete()
+    [thr.join() for thr in client_threads]
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    conn.delete_bucket(bucket_name)
+    http_server.close()
+
+
 @attr('amqp_test')
 def test_ps_s3_multipart_on_master():
     """ test multipart object upload on master"""

From 6b3f7423abe43dcbdfe06c53dd686bdde1a2b5f0 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 30 Nov 2023 17:39:34 -0500
Subject: [PATCH 1072/2492] qa/rgw/upgrade: stop testing upgrade from pacific

squid won't support upgrades from pacific. stop all testing against
centos8 and ubuntu20

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/upgrade/1-install/pacific/%                    | 0
 qa/suites/rgw/upgrade/1-install/pacific/.qa                  | 1 -
 qa/suites/rgw/upgrade/1-install/pacific/distro$/.qa          | 1 -
 .../upgrade/1-install/pacific/distro$/centos_8.stream.yaml   | 1 -
 .../rgw/upgrade/1-install/pacific/distro$/ubuntu_20.04.yaml  | 1 -
 qa/suites/rgw/upgrade/1-install/pacific/install.yaml         | 5 -----
 qa/suites/rgw/upgrade/1-install/pacific/overrides.yaml       | 3 ---
 7 files changed, 12 deletions(-)
 delete mode 100644 qa/suites/rgw/upgrade/1-install/pacific/%
 delete mode 120000 qa/suites/rgw/upgrade/1-install/pacific/.qa
 delete mode 120000 qa/suites/rgw/upgrade/1-install/pacific/distro$/.qa
 delete mode 120000 qa/suites/rgw/upgrade/1-install/pacific/distro$/centos_8.stream.yaml
 delete mode 120000 qa/suites/rgw/upgrade/1-install/pacific/distro$/ubuntu_20.04.yaml
 delete mode 100644 qa/suites/rgw/upgrade/1-install/pacific/install.yaml
 delete mode 100644 qa/suites/rgw/upgrade/1-install/pacific/overrides.yaml

diff --git a/qa/suites/rgw/upgrade/1-install/pacific/% b/qa/suites/rgw/upgrade/1-install/pacific/%
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/.qa b/qa/suites/rgw/upgrade/1-install/pacific/.qa
deleted file mode 120000
index fea2489fdf6d..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/distro$/.qa b/qa/suites/rgw/upgrade/1-install/pacific/distro$/.qa
deleted file mode 120000
index fea2489fdf6d..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/distro$/.qa
+++ /dev/null
@@ -1 +0,0 @@
-../.qa
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/distro$/centos_8.stream.yaml b/qa/suites/rgw/upgrade/1-install/pacific/distro$/centos_8.stream.yaml
deleted file mode 120000
index e787191b5a27..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/distro$/centos_8.stream.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/all/centos_8.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/distro$/ubuntu_20.04.yaml b/qa/suites/rgw/upgrade/1-install/pacific/distro$/ubuntu_20.04.yaml
deleted file mode 120000
index 162964882d38..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/distro$/ubuntu_20.04.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/all/ubuntu_20.04.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/install.yaml b/qa/suites/rgw/upgrade/1-install/pacific/install.yaml
deleted file mode 100644
index ec89e7921030..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/install.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-tasks:
-- install:
-    branch: pacific
-    exclude_packages:
-      - ceph-volume
diff --git a/qa/suites/rgw/upgrade/1-install/pacific/overrides.yaml b/qa/suites/rgw/upgrade/1-install/pacific/overrides.yaml
deleted file mode 100644
index ab06356f6b36..000000000000
--- a/qa/suites/rgw/upgrade/1-install/pacific/overrides.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-overrides:
-  ragweed:
-    default-branch: ceph-master # ceph-pacific doesn't have tox, but tests are the same

From d785add11c38411a0b2f5933cc9581c00cc51ac0 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 14 Dec 2023 13:38:22 -0500
Subject: [PATCH 1073/2492] radosgw-admin: 'zone set' won't overwrite existing
 default-placement

Fixes: https://tracker.ceph.com/issues/63829

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_zone.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index 11b57e427207..e6cc54d69d89 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -1035,10 +1035,12 @@ int create_zone(const DoutPrefixProvider* dpp, optional_yield y,
   }
 
   // add default placement with empty pool name
+  RGWZonePlacementInfo placement;
   rgw_pool pool;
-  auto& placement = info.placement_pools["default-placement"];
   placement.storage_classes.set_storage_class(
       RGW_STORAGE_CLASS_STANDARD, &pool, nullptr);
+  // don't overwrite if it already exists
+  info.placement_pools.emplace("default-placement", std::move(placement));
 
   // build a set of all pool names used by other zones
   std::set<rgw_pool> pools;

From 31d72b8ecc1b75b7996027418614c6e2e6a1d0e7 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 14 Dec 2023 16:14:29 +1000
Subject: [PATCH 1074/2492] doc/radosgw: format commands in role.rst

Format approximately one-hundred lines of doc/radosgw/role.rst to
include proper command prompts. I also made one small English usage
improvement.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/role.rst | 47 +++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
index 1e009df21ab0..514f17d5caec 100644
--- a/doc/radosgw/role.rst
+++ b/doc/radosgw/role.rst
@@ -58,9 +58,11 @@ For example::
 Delete a Role
 -------------
 
-To delete a role, execute the following::
+To delete a role, run a command of the following form:
 
-	radosgw-admin role delete --role-name={role-name}
+.. prompt:: bash
+
+   radosgw-admin role delete --role-name={role-name}
 
 Request Parameters
 ~~~~~~~~~~~~~~~~~~
@@ -70,18 +72,23 @@ Request Parameters
 :Description: Name of the role.
 :Type: String
 
-For example:: 	
+For example:
+
+.. prompt:: bash
 	
-  radosgw-admin role delete --role-name=S3Access1
+   radosgw-admin role delete --role-name=S3Access1
 
-Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+Note: A role can be deleted only when it has no permission policy attached to
+it.
 
 Get a Role
 ----------
 
-To get information about a role, execute the following::
+To get information about a role, run a command of the following form:
 
-	radosgw-admin role get --role-name={role-name}
+.. prompt:: bash
+
+   radosgw-admin role get --role-name={role-name}
 
 Request Parameters
 ~~~~~~~~~~~~~~~~~~
@@ -91,9 +98,11 @@ Request Parameters
 :Description: Name of the role.
 :Type: String
 
-For example:: 	
+For example:
+
+.. prompt:: bash
 	
-  radosgw-admin role get --role-name=S3Access1
+   radosgw-admin role get --role-name=S3Access1
   
 .. code-block:: javascript
   
@@ -111,21 +120,26 @@ For example::
 List Roles
 ----------
 
-To list roles with a specified path prefix, execute the following::
+To list roles with a specified path prefix, run a command of the following form:
 
-	radosgw-admin role list [--path-prefix ={path prefix}]
+.. prompt:: bash
+
+   radosgw-admin role list [--path-prefix ={path prefix}]
 
 Request Parameters
 ~~~~~~~~~~~~~~~~~~
 
 ``path-prefix``
 
-:Description: Path prefix for filtering roles. If this is not specified, all roles are listed.
+:Description: Path prefix for filtering roles. If this is not specified, all
+              roles are listed.
 :Type: String
 
-For example:: 	
+For example:
+
+.. prompt:: bash
 	
-  radosgw-admin role list --path-prefix="/application"
+   radosgw-admin role list --path-prefix="/application"
   
 .. code-block:: javascript
   
@@ -141,7 +155,6 @@ For example::
     }
   ]
 
-
 Update Assume Role Policy Document of a role
 --------------------------------------------
 
@@ -364,7 +377,9 @@ Example::
 
   POST "<hostname>?Action=DeleteRole&RoleName=S3Access"
 
-Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+Note: A role can be deleted only when it doesn't have any permission policy
+attached to it. If you intend to delete a role, you must first delete any
+policies attached to it.
 
 Get a Role
 ----------

From 07c8087e59d53820543d95bb3752582b6c10045d Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Mon, 11 Dec 2023 03:51:50 +0000
Subject: [PATCH 1075/2492] osd: fix Incremental decode for
 new/old_pg_upmap_primary

Should have been included in commit e9921ee76c060122d64e61a2fe483318a49bc0a9.
The impact is not major; the effect is that the osdmap
is not properly updated whenever the Incremental is reused.
Since the current read balancer offline tool uses a new Incremental
every time it's run, the balancing works as expected. This fix will be
necessary, however, for the mgr module integration in https://github.com/ceph/ceph/pull/53988
since we will need to reuse the Incremental.

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/osd/OSDMap.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 8f60b2f3c83c..434146d9647e 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -934,6 +934,10 @@ void OSDMap::Incremental::decode(ceph::buffer::list::const_iterator& bl)
       decode(new_last_up_change, bl);
       decode(new_last_in_change, bl);
     }
+    if (struct_v >= 9) {
+      decode(new_pg_upmap_primary, bl);
+      decode(old_pg_upmap_primary, bl);
+    }
     DECODE_FINISH(bl); // client-usable data
   }
 

From 6ffc70a1b7f3dbe9237c2379c477d7dc4898024f Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Tue, 2 May 2023 14:17:07 +0000
Subject: [PATCH 1076/2492] mon/ConnectionTracker.cc: disregard connection
 scores from mon_rank = -1

There are certain situations where we would
come across rank -1 in our MON connection scores;

- New MON sends probe message to existing MON,
existing MON handle probe message by adding -1
to existing peer_scores.

This is not good because we want to implement
a connection scores check mechanism where we
should not have to take into account the possibility
of having rank -1 in our score.

Fixes: https://tracker.ceph.com/issues/59564

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/mon/ConnectionTracker.cc | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/mon/ConnectionTracker.cc b/src/mon/ConnectionTracker.cc
index 272ad40c2746..c87d614f6420 100644
--- a/src/mon/ConnectionTracker.cc
+++ b/src/mon/ConnectionTracker.cc
@@ -62,7 +62,9 @@ void ConnectionTracker::receive_peer_report(const ConnectionTracker& o)
   ldout(cct, 30) << __func__ << dendl;
   for (auto& i : o.peer_reports) {
     const ConnectionReport& report = i.second;
-    if (i.first == rank) continue;
+    if (i.first == rank || i.first < 0) {
+      continue;
+    }
     ConnectionReport& existing = *reports(i.first);
     if (report.epoch > existing.epoch ||
 	(report.epoch == existing.epoch &&
@@ -79,26 +81,32 @@ void ConnectionTracker::receive_peer_report(const ConnectionTracker& o)
 bool ConnectionTracker::increase_epoch(epoch_t e)
 {
   ldout(cct, 30) << __func__ << " to " << e << dendl;
-  if (e > epoch) {
+  if (e > epoch && rank >= 0) {
     my_reports.epoch_version = version = 0;
     my_reports.epoch = epoch = e;
     peer_reports[rank] = my_reports;
     encoding.clear();
     return true;
   }
+  ldout(cct, 10) << "Either got a report from a rank -1 or our epoch is >= to "
+    << e << " not increasing our epoch!" << dendl;
   return false;
 }
 
 void ConnectionTracker::increase_version()
 {
   ldout(cct, 30) << __func__ << " to " << version+1 << dendl;
-  encoding.clear();
-  ++version;
-  my_reports.epoch_version = version;
-  peer_reports[rank] = my_reports;
-  if ((version % persist_interval) == 0 ) {
-    ldout(cct, 30) << version << " % " << persist_interval << " == 0" << dendl;
-    owner->persist_connectivity_scores();
+  if (rank >= 0) {
+    encoding.clear();
+    ++version;
+    my_reports.epoch_version = version;
+    peer_reports[rank] = my_reports;
+    if ((version % persist_interval) == 0 ) {
+      ldout(cct, 30) << version << " % " << persist_interval << " == 0" << dendl;
+      owner->persist_connectivity_scores();
+    }
+  } else {
+      ldout(cct, 10) << "Got a report from a rank -1, not increasing our version!" << dendl;
   }
 }
 
@@ -110,6 +118,10 @@ void ConnectionTracker::report_live_connection(int peer_rank, double units_alive
     lderr(cct) << "Got a report from my own rank, hopefully this is startup weirdness, dropping" << dendl;
     return;
   }
+  if (peer_rank < 0) {
+    ldout(cct, 10) << "Got a report from a rank -1, not adding that to our report!" << dendl;
+    return;
+  }  
   // we need to "auto-initialize" to 1, do shenanigans
   auto i = my_reports.history.find(peer_rank);
   if (i == my_reports.history.end()) {
@@ -138,6 +150,10 @@ void ConnectionTracker::report_dead_connection(int peer_rank, double units_dead)
     lderr(cct) << "Got a report from my own rank, hopefully this is startup weirdness, dropping" << dendl;
     return;
   }
+  if (peer_rank < 0) {
+    ldout(cct, 10) << "Got a report from a rank -1, not adding that to our report!" << dendl;
+    return;
+  }
   // we need to "auto-initialize" to 1, do shenanigans
   auto i = my_reports.history.find(peer_rank);
   if (i == my_reports.history.end()) {

From 223848a08f58fc20ee4afc008fabbec2d448be68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B6=E6=B5=B7=E4=B8=B0?= <769358362@qq.com>
Date: Fri, 15 Dec 2023 17:05:38 +0800
Subject: [PATCH 1077/2492] doc/cephfs/client-auth.rst: correct ``fs authorize
 cephfs1 /dir1 client.x rw``

---
 doc/cephfs/client-auth.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/cephfs/client-auth.rst b/doc/cephfs/client-auth.rst
index 5a642e4f8ae9..946211bf682e 100644
--- a/doc/cephfs/client-auth.rst
+++ b/doc/cephfs/client-auth.rst
@@ -316,8 +316,8 @@ Changing rw permissions in caps
 It's not possible to modify caps by running ``fs authorize`` except for the
 case when read/write permissions have to be changed. This so because the
 ``fs authorize`` becomes ambiguous. For example, user runs ``fs authorize
-cephfs1 /dir1 client.x rw`` to create a client and then runs ``fs authorize
-cephfs1 /dir2 client.x rw`` (notice ``/dir1`` is changed to ``/dir2``).
+cephfs1 client.x /dir1 rw`` to create a client and then runs ``fs authorize
+cephfs1 client.x /dir2 rw`` (notice ``/dir1`` is changed to ``/dir2``).
 Running second command can be interpreted as changing ``/dir1`` to ``/dir2``
 in current cap or can also be interpreted as authorizing the client with a
 new cap for path ``/dir2``. As seen in previous sections, second

From 34dc9dda1faaa0489f6d28d30babc542c6385c3b Mon Sep 17 00:00:00 2001
From: zhangjianwei2 <zhangjianwei2_yewu@cmss.chinamobile.com>
Date: Fri, 15 Dec 2023 16:42:07 +0800
Subject: [PATCH 1078/2492] OSD: oldest_map and newest_map should be displayed
 directly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

9215996586ba79302c07e5cf586ec2add1ac159e delete these,
ceph daemon osd.x status
when osd status is displayed, only superblock.maps (begin ~ len) can be seen.
this need calculate maps to get oldest_map/newest_map held by the current osd.
moreover, it cannot be simply calculate. need pay attention to the boundaries。
this is not very friendly.

Fixes: https://tracker.ceph.com/issues/63836
Signed-off-by: zhangjianwei2 <zhangjianwei2@cmss.chinamobile.com>
---
 src/crimson/osd/osd.cc | 2 ++
 src/osd/OSD.cc         | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index f3648c6df277..2c58b7c638c9 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -713,6 +713,8 @@ void OSD::dump_status(Formatter* f) const
   f->dump_unsigned("whoami", superblock.whoami);
   f->dump_string("state", pg_shard_manager.get_osd_state_string());
   f->dump_stream("maps") << superblock.maps;
+  f->dump_stream("oldest_map") << superblock.get_oldest_map();
+  f->dump_stream("newest_map") << superblock.get_newest_map();
   f->dump_unsigned("cluster_osdmap_trim_lower_bound",
                    superblock.cluster_osdmap_trim_lower_bound);
   f->dump_unsigned("num_pgs", pg_shard_manager.get_num_pgs());
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a79b388deca4..f74686537134 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2696,6 +2696,8 @@ void OSD::asok_command(
     f->dump_unsigned("whoami", superblock.whoami);
     f->dump_string("state", get_state_name(get_state()));
     f->dump_stream("maps") << superblock.maps;
+    f->dump_stream("oldest_map") << superblock.get_oldest_map();
+    f->dump_stream("newest_map") << superblock.get_newest_map();
     f->dump_unsigned("cluster_osdmap_trim_lower_bound",
                      superblock.cluster_osdmap_trim_lower_bound);
     f->dump_unsigned("num_pgs", num_pgs);

From c24a4c19848fefcfecbc9f706cccacf2e93b3f14 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 18:04:53 +0800
Subject: [PATCH 1079/2492] cmake: only apply workaround for boost < 1.84

https://github.com/boostorg/phoenix/issues/111 was fixed in master.
see the fix which addresses the issue:
https://github.com/boostorg/phoenix/commit/8913607a3788cb82d48ed461ea59c919b7bad3df

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4feb22c135e1..1364426067ce 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -703,7 +703,7 @@ else()
 endif()
 include_directories(BEFORE SYSTEM ${Boost_INCLUDE_DIRS})
 
-if(Boost_VERSION VERSION_EQUAL 1.81 OR Boost_VERSION VERSION_EQUAL 1.82)
+if(Boost_VERSION VERSION_LESS 1.84)
   # This is a workaround for https://github.com/boostorg/phoenix/issues/111
   add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-DBOOST_PHOENIX_STL_TUPLE_H_>)
 endif()

From 7aff35c987b9f0e9f0d4198d032737da0ce12b36 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sun, 3 Dec 2023 16:39:39 +0100
Subject: [PATCH 1080/2492] test/librbd: redo
 TestMockObjectMapDiffRequest.*Delta tests

Existing *Delta tests cover:

- beginning of time -> HEAD, through intermediate snap
- snap -> snap, directly
- snap -> HEAD, directly

But coverage is too weak: none of the weird OBJECT_PENDING cases and
only a single diff-iterate vs deep-copy case is tested, for example.

Coverage is missing completely for:

- beginning of time -> HEAD, directly
- beginning of time -> snap, directly
- beginning of time -> snap, through intermediate snap
- snap -> snap, through intermediate snap
- snap -> HEAD, through intermediate snap

This adds the following tests:

- FromBeginningToHead
- FromBeginningToHeadIntermediateSnap (expands FullDelta)
- FromBeginningToSnap
- FromBeginningToSnapIntermediateSnap
- FromSnapToSnap (expands IntermediateDelta)
- FromSnapToSnapIntermediateSnap
- FromSnapToHead (expands EndDelta)
- FromSnapToHeadIntermediateSnap

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../object_map/test_mock_DiffRequest.cc       | 420 +++++++++++++++---
 1 file changed, 370 insertions(+), 50 deletions(-)

diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 440040e9a835..aefebf4a05bb 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -32,6 +32,122 @@ using ::testing::WithArg;
 namespace librbd {
 namespace object_map {
 
+static constexpr uint8_t from_beginning_table[][2] = {
+  //        to                expected
+  { OBJECT_NONEXISTENT,   DIFF_STATE_HOLE },
+  { OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED }
+};
+
+static constexpr uint8_t from_beginning_intermediate_table[][4] = {
+  //   intermediate               to             diff-iterate expected       deep-copy expected
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED }
+};
+
+static constexpr uint8_t from_snap_table[][3] = {
+  //       from                   to                expected
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA }
+};
+
+static constexpr uint8_t from_snap_intermediate_table[][5] = {
+  //       from              intermediate               to             diff-iterate expected       deep-copy expected
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE },
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE,          DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA,          DIFF_STATE_DATA },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  OBJECT_NONEXISTENT,   DIFF_STATE_HOLE_UPDATED,  DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS,        DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  OBJECT_PENDING,       DIFF_STATE_DATA_UPDATED,  DIFF_STATE_DATA_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA,          DIFF_STATE_DATA }
+};
+
 class TestMockObjectMapDiffRequest : public TestMockFixture,
                                      public ::testing::WithParamInterface<bool> {
 public:
@@ -146,44 +262,108 @@ TEST_P(TestMockObjectMapDiffRequest, FastDiffInvalid) {
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
 
-TEST_P(TestMockObjectMapDiffRequest, FullDelta) {
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
-  uint32_t object_count = 5;
+  uint32_t object_count = std::size(from_beginning_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
 
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
   mock_image_ctx.snap_info = {
     {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
-          {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
           {}, {}, {}}}
   };
 
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
   InSequence seq;
 
   expect_get_flags(mock_image_ctx, 1U, 0, 0);
+  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
+
+  C_SaferCond ctx;
+  auto req = new MockDiffRequest(&mock_image_ctx, 0, 1, is_diff_iterate(),
+                                 &m_object_diff_state, &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_beginning_intermediate_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  MockTestImageCtx mock_image_ctx(*m_image_ctx);
+  mock_image_ctx.snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}}
+  };
 
   BitVector<2> object_map_1;
   object_map_1.resize(object_count);
-  object_map_1[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_2[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+
+  InSequence seq;
+
+  expect_get_flags(mock_image_ctx, 1U, 0, 0);
   expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
 
   expect_get_flags(mock_image_ctx, 2U, 0, 0);
-
-  BitVector<2> object_map_2;
-  object_map_2.resize(object_count);
-  object_map_2[1] = OBJECT_EXISTS_CLEAN;
-  object_map_2[2] = OBJECT_EXISTS;
-  object_map_2[3] = OBJECT_EXISTS;
   expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
 
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+  C_SaferCond ctx;
+  auto req = new MockDiffRequest(&mock_image_ctx, 0, 2, is_diff_iterate(),
+                                 &m_object_diff_state, &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_beginning_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  MockTestImageCtx mock_image_ctx(*m_image_ctx);
 
   BitVector<2> object_map_head;
   object_map_head.resize(object_count);
-  object_map_head[1] = OBJECT_EXISTS_CLEAN;
-  object_map_head[2] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_head[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  InSequence seq;
+
+  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
   expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
 
   C_SaferCond ctx;
@@ -193,18 +373,59 @@ TEST_P(TestMockObjectMapDiffRequest, FullDelta) {
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
+  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_beginning_intermediate_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  MockTestImageCtx mock_image_ctx(*m_image_ctx);
+  mock_image_ctx.snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
   BitVector<2> expected_diff_state;
   expected_diff_state.resize(object_count);
-  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
-  expected_diff_state[2] = DIFF_STATE_DATA_UPDATED;
-  expected_diff_state[3] = DIFF_STATE_HOLE;
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_head[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+
+  InSequence seq;
+
+  expect_get_flags(mock_image_ctx, 1U, 0, 0);
+  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
+
+  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+
+  C_SaferCond ctx;
+  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_P(TestMockObjectMapDiffRequest, IntermediateDelta) {
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
-  uint32_t object_count = 5;
+  uint32_t object_count = std::size(from_snap_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
 
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
@@ -215,23 +436,24 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateDelta) {
           {}, {}, {}}}
   };
 
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_2[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+
   InSequence seq;
 
   expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
-  BitVector<2> object_map_1;
-  object_map_1.resize(object_count);
-  object_map_1[1] = OBJECT_EXISTS;
-  object_map_1[2] = OBJECT_EXISTS_CLEAN;
   expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
 
   expect_get_flags(mock_image_ctx, 2U, 0, 0);
-
-  BitVector<2> object_map_2;
-  object_map_2.resize(object_count);
-  object_map_2[1] = OBJECT_EXISTS_CLEAN;
-  object_map_2[2] = OBJECT_EXISTS;
-  object_map_2[3] = OBJECT_EXISTS;
   expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
 
   C_SaferCond ctx;
@@ -240,18 +462,13 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateDelta) {
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
-  BitVector<2> expected_diff_state;
-  expected_diff_state.resize(object_count);
-  expected_diff_state[1] = DIFF_STATE_DATA;
-  expected_diff_state[2] = DIFF_STATE_DATA_UPDATED;
-  expected_diff_state[3] = DIFF_STATE_DATA_UPDATED;
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 
-TEST_P(TestMockObjectMapDiffRequest, EndDelta) {
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
-  uint32_t object_count = 5;
+  uint32_t object_count = std::size(from_snap_intermediate_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
 
   MockTestImageCtx mock_image_ctx(*m_image_ctx);
@@ -259,40 +476,143 @@ TEST_P(TestMockObjectMapDiffRequest, EndDelta) {
     {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
           {}, {}, {}}},
     {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}},
+    {3U, {"snap3", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
           {}, {}, {}}}
   };
 
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  BitVector<2> object_map_3;
+  object_map_3.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_snap_intermediate_table[i][0];
+    object_map_2[i] = from_snap_intermediate_table[i][1];
+    object_map_3[i] = from_snap_intermediate_table[i][2];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_snap_intermediate_table[i][3];
+    } else {
+      expected_diff_state[i] = from_snap_intermediate_table[i][4];
+    }
+  }
+
   InSequence seq;
 
-  expect_get_flags(mock_image_ctx, 2U, 0, 0);
+  expect_get_flags(mock_image_ctx, 1U, 0, 0);
+  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
 
-  BitVector<2> object_map_2;
-  object_map_2.resize(object_count);
-  object_map_2[1] = OBJECT_EXISTS_CLEAN;
-  object_map_2[2] = OBJECT_EXISTS;
-  object_map_2[3] = OBJECT_EXISTS;
+  expect_get_flags(mock_image_ctx, 2U, 0, 0);
   expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
 
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+  expect_get_flags(mock_image_ctx, 3U, 0, 0);
+  expect_load_map(mock_image_ctx, 3U, object_map_3, 0);
 
+  C_SaferCond ctx;
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, 3, is_diff_iterate(),
+                                 &m_object_diff_state, &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
+  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_snap_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  MockTestImageCtx mock_image_ctx(*m_image_ctx);
+  mock_image_ctx.snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
   BitVector<2> object_map_head;
   object_map_head.resize(object_count);
-  object_map_head[1] = OBJECT_EXISTS_CLEAN;
-  object_map_head[2] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_head[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+
+  InSequence seq;
+
+  expect_get_flags(mock_image_ctx, 1U, 0, 0);
+  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
+
+  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
   expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
 
   C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 2, CEPH_NOSNAP,
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
                                  is_diff_iterate(), &m_object_diff_state,
                                  &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 
+  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_snap_intermediate_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  MockTestImageCtx mock_image_ctx(*m_image_ctx);
+  mock_image_ctx.snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
   BitVector<2> expected_diff_state;
   expected_diff_state.resize(object_count);
-  expected_diff_state[1] = DIFF_STATE_DATA;
-  expected_diff_state[2] = DIFF_STATE_DATA;
-  expected_diff_state[3] = DIFF_STATE_HOLE_UPDATED;
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_snap_intermediate_table[i][0];
+    object_map_2[i] = from_snap_intermediate_table[i][1];
+    object_map_head[i] = from_snap_intermediate_table[i][2];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_snap_intermediate_table[i][3];
+    } else {
+      expected_diff_state[i] = from_snap_intermediate_table[i][4];
+    }
+  }
+
+  InSequence seq;
+
+  expect_get_flags(mock_image_ctx, 1U, 0, 0);
+  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
+
+  expect_get_flags(mock_image_ctx, 2U, 0, 0);
+  expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
+
+  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+
+  C_SaferCond ctx;
+  auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
+                                 is_diff_iterate(), &m_object_diff_state,
+                                 &ctx);
+  req->send();
+  ASSERT_EQ(0, ctx.wait());
+
   ASSERT_EQ(expected_diff_state, m_object_diff_state);
 }
 

From f8ced6d1fe667864bee224104c23b62a2c1c0bfc Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sun, 10 Dec 2023 17:01:24 +0100
Subject: [PATCH 1081/2492] test/pybind/rbd: don't ignore from_snapshot in
 check_diff()

Despite the test in test_diff_iterate() being correct, it started
failing:

    >       check_diff(self.image, 0, IMG_SIZE, 'snap1', [(0, 512, False)])
    ...
    a = [], b = [(0, 512, False)]
    ...
    >       assert a == b
    E       AssertionError

This is because check_diff() drops 'snap1' argument on the floor and
passes None to image.diff_iterate() instead.  This goes back to 2013,
see commit e88fe3cbbc8f ("rbd.py: add some missing functions").

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/pybind/test_rbd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py
index 0ce3c0dd90ca..df47b0d2976b 100644
--- a/src/test/pybind/test_rbd.py
+++ b/src/test/pybind/test_rbd.py
@@ -1471,7 +1471,7 @@ def check_diff(image, offset, length, from_snapshot, expected):
     extents = []
     def cb(offset, length, exists):
         extents.append((offset, length, exists))
-    image.diff_iterate(0, IMG_SIZE, None, cb)
+    image.diff_iterate(0, IMG_SIZE, from_snapshot, cb)
     eq(extents, expected)
 
 class TestClone(object):

From 620b68a348f07145c49c12668576a89dee8198cb Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 19:01:46 +0800
Subject: [PATCH 1082/2492] cmake/modules/BuildRocksDB.cmake: use string(APPEND
 ..) when appropriate

less repeatings this way.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/BuildRocksDB.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/BuildRocksDB.cmake b/cmake/modules/BuildRocksDB.cmake
index f9a28274c40d..f81f5248506d 100644
--- a/cmake/modules/BuildRocksDB.cmake
+++ b/cmake/modules/BuildRocksDB.cmake
@@ -60,11 +60,11 @@ function(build_rocksdb)
   include(CheckCXXCompilerFlag)
   check_cxx_compiler_flag("-Wno-deprecated-copy" HAS_WARNING_DEPRECATED_COPY)
   if(HAS_WARNING_DEPRECATED_COPY)
-    set(rocksdb_CXX_FLAGS -Wno-deprecated-copy)
+    string(APPEND rocksdb_CXX_FLAGS " -Wno-deprecated-copy")
   endif()
   check_cxx_compiler_flag("-Wno-pessimizing-move" HAS_WARNING_PESSIMIZING_MOVE)
   if(HAS_WARNING_PESSIMIZING_MOVE)
-    set(rocksdb_CXX_FLAGS "${rocksdb_CXX_FLAGS} -Wno-pessimizing-move")
+    string(APPEND rocksdb_CXX_FLAGS " -Wno-pessimizing-move")
   endif()
   if(rocksdb_CXX_FLAGS)
     list(APPEND rocksdb_CMAKE_ARGS -DCMAKE_CXX_FLAGS='${rocksdb_CXX_FLAGS}')

From beb1a624921d7589db63dea066935b3aa9ce2698 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 19:03:57 +0800
Subject: [PATCH 1083/2492] cmake/modules/BuildRocksDB.cmake: inherit parent's
 CMAKE_CXX_FLAGS

CMake allows us to customize `CMAKE_CXX_FLAGS` by setting CXXFLAGS
environmental variable. and Debian's debhelper also sets CXXFLAGS
when it builds cmake projects for customizing the building flags.

but we fail to populate this setting down when building external
projects. this is important when it comes to the projects which
is critical to the performance. RocksDB is one of them.

in this change, we pass the `CMAKE_CXX_FLAGS` down in
`BuildRocksDB.cmake` so that its `CMAKE_CXX_FLAGS` contains
the same set of `CMAKE_CXX_FLAGS` used by its parent project.

this should help with the performance in the bluestore, where
RocksDB is used.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/BuildRocksDB.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/modules/BuildRocksDB.cmake b/cmake/modules/BuildRocksDB.cmake
index f81f5248506d..e0208f6545b7 100644
--- a/cmake/modules/BuildRocksDB.cmake
+++ b/cmake/modules/BuildRocksDB.cmake
@@ -59,6 +59,7 @@ function(build_rocksdb)
   endif()
   include(CheckCXXCompilerFlag)
   check_cxx_compiler_flag("-Wno-deprecated-copy" HAS_WARNING_DEPRECATED_COPY)
+  set(rocksdb_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
   if(HAS_WARNING_DEPRECATED_COPY)
     string(APPEND rocksdb_CXX_FLAGS " -Wno-deprecated-copy")
   endif()

From f61f651fb3693e34d94e2a1750d689dc497d604e Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 17:28:37 +0800
Subject: [PATCH 1084/2492] cmake: add 1.82 and 1.83 to known versions

this change should silence the warning when building Ceph with Boost 1.82:
```
CMake Warning at cmake/modules/FindBoost.cmake:1384 (message):
  New Boost version may have incorrect or missing dependencies and imported
  targets
Call Stack (most recent call first):
  cmake/modules/FindBoost.cmake:1507 (_Boost_COMPONENT_DEPENDENCIES)
  cmake/modules/FindBoost.cmake:2125 (_Boost_MISSING_DEPENDENCIES)
  CMakeLists.txt:687 (find_package)
```

despite that 244c5ebb added boost to `_Boost_KNOWN_VERSIONS`, it was
not enough. would be better to sync up the CMake upstream for an updated
version of this file like we did before.

in this change

* syncs this file with
  https://github.com/Kitware/CMake/blob/897a149067b84375d094700632255c0b54f5e44c/Modules/FindBoost.cmak
* re-apply the change of 06824bc1ecbe68b6809c1f47f0f5d636842746b5

Signed-off-by: Kefu Chai <tchaikov@gmail.com>

cmake: adapt FindBoost.cmake to our needs

the vanilla FindBoost.cmake pulled from cmake has couple assumptions
which do not hold in our environment. so address them case by case.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/FindBoost.cmake | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/cmake/modules/FindBoost.cmake b/cmake/modules/FindBoost.cmake
index d47c4862dd08..ba63e24ddba5 100644
--- a/cmake/modules/FindBoost.cmake
+++ b/cmake/modules/FindBoost.cmake
@@ -1365,7 +1365,7 @@ function(_Boost_COMPONENT_DEPENDENCIES component _ret)
       set(_Boost_TIMER_DEPENDENCIES chrono)
       set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic)
       set(_Boost_WSERIALIZATION_DEPENDENCIES serialization)
-    else()
+    elseif(Boost_VERSION_STRING VERSION_LESS 1.83.0)
       set(_Boost_CONTRACT_DEPENDENCIES thread chrono)
       set(_Boost_COROUTINE_DEPENDENCIES context)
       set(_Boost_FIBER_DEPENDENCIES context)
@@ -1380,7 +1380,21 @@ function(_Boost_COMPONENT_DEPENDENCIES component _ret)
       set(_Boost_TIMER_DEPENDENCIES chrono)
       set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic)
       set(_Boost_WSERIALIZATION_DEPENDENCIES serialization)
-      if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.81.0 AND NOT Boost_NO_WARN_NEW_VERSIONS)
+    else()
+      set(_Boost_CONTRACT_DEPENDENCIES thread chrono)
+      set(_Boost_COROUTINE_DEPENDENCIES context)
+      set(_Boost_FIBER_DEPENDENCIES context)
+      set(_Boost_IOSTREAMS_DEPENDENCIES regex)
+      set(_Boost_JSON_DEPENDENCIES container)
+      set(_Boost_LOG_DEPENDENCIES log_setup filesystem thread regex chrono atomic)
+      set(_Boost_MATH_DEPENDENCIES math_c99 math_c99f math_c99l math_tr1 math_tr1f math_tr1l)
+      set(_Boost_MPI_DEPENDENCIES serialization)
+      set(_Boost_MPI_PYTHON_DEPENDENCIES python${component_python_version} mpi serialization)
+      set(_Boost_NUMPY_DEPENDENCIES python${component_python_version})
+      set(_Boost_THREAD_DEPENDENCIES chrono atomic)
+      set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono atomic)
+      set(_Boost_WSERIALIZATION_DEPENDENCIES serialization)
+      if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.84.0 AND NOT Boost_NO_WARN_NEW_VERSIONS)
         message(WARNING "New Boost version may have incorrect or missing dependencies and imported targets")
       endif()
     endif()
@@ -1445,6 +1459,7 @@ function(_Boost_COMPONENT_HEADERS component _hdrs)
   set(_Boost_MATH_TR1L_HEADERS           "boost/math/tr1.hpp")
   set(_Boost_MPI_HEADERS                 "boost/mpi.hpp")
   set(_Boost_MPI_PYTHON_HEADERS          "boost/mpi/python/config.hpp")
+  set(_Boost_MYSQL_HEADERS               "boost/mysql.hpp")
   set(_Boost_NUMPY_HEADERS               "boost/python/numpy.hpp")
   set(_Boost_NOWIDE_HEADERS              "boost/nowide/cstdlib.hpp")
   set(_Boost_PRG_EXEC_MONITOR_HEADERS    "boost/test/prg_exec_monitor.hpp")
@@ -1466,6 +1481,7 @@ function(_Boost_COMPONENT_HEADERS component _hdrs)
   set(_Boost_TIMER_HEADERS               "boost/timer.hpp")
   set(_Boost_TYPE_ERASURE_HEADERS        "boost/type_erasure/config.hpp")
   set(_Boost_UNIT_TEST_FRAMEWORK_HEADERS "boost/test/framework.hpp")
+  set(_Boost_URL_HEADERS                 "boost/url.hpp")
   set(_Boost_WAVE_HEADERS                "boost/wave.hpp")
   set(_Boost_WSERIALIZATION_HEADERS      "boost/archive/text_wiarchive.hpp")
   set(_Boost_BZIP2_HEADERS               "boost/iostreams/filter/bzip2.hpp")
@@ -1653,7 +1669,7 @@ else()
   # _Boost_COMPONENT_HEADERS.  See the instructions at the top of
   # _Boost_COMPONENT_DEPENDENCIES.
   set(_Boost_KNOWN_VERSIONS ${Boost_ADDITIONAL_VERSIONS}
-    "1.82.0" "1.82" "1.81.0" "1.81" "1.80.0" "1.80" "1.79.0" "1.79"
+    "1.83.0" "1.83" "1.82.0" "1.82" "1.81.0" "1.81" "1.80.0" "1.80" "1.79.0" "1.79"
     "1.78.0" "1.78" "1.77.0" "1.77" "1.76.0" "1.76" "1.75.0" "1.75" "1.74.0" "1.74"
     "1.73.0" "1.73" "1.72.0" "1.72" "1.71.0" "1.71" "1.70.0" "1.70" "1.69.0" "1.69"
     "1.68.0" "1.68" "1.67.0" "1.67" "1.66.0" "1.66" "1.65.1" "1.65.0" "1.65"

From 186540c112df9c1a2e467fe99b36d0cc13256535 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 20:20:46 +0800
Subject: [PATCH 1085/2492] cls/queue: always set member variables in ctor

this should address the test failures like

```
/tmp/typ-WWFeFl6yK /tmp/typ-sMfwoaGMU differ: byte 24, line 1
**** cls_2pc_reservation test 2 binary reencode check failed ****
   ceph-dencoder type cls_2pc_reservation select_test 2 encode export /tmp/typ-WWFeFl6yK
   ceph-dencoder type cls_2pc_reservation select_test 2 encode decode encode export /tmp/typ-sMfwoaGMU
2c2
< 00000010  00 00 00 00 00 00 c0 c6  92 10                    |..........|
---
> 00000010  00 00 00 00 00 00 c0 e6  cd 53                    |.........S|
```

where we
1. encode the 2nd sample created by `generate_test_instances()`,
2. encode, decode, and encode again, the same sample

and compare the encoded blobs.

but if we fail to set any of the fields in `cls_2pc_reservation`,
we are at the mercy of the random bits on stack/heap.

in this change, all bits are initialized.

the flaky test was introduced by 1d7cabf3

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/cls/2pc_queue/cls_2pc_queue_types.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cls/2pc_queue/cls_2pc_queue_types.h b/src/cls/2pc_queue/cls_2pc_queue_types.h
index b270c9d6e790..093b69cb193d 100644
--- a/src/cls/2pc_queue/cls_2pc_queue_types.h
+++ b/src/cls/2pc_queue/cls_2pc_queue_types.h
@@ -8,9 +8,9 @@ struct cls_2pc_reservation
 {
   using id_t = uint32_t;
   inline static const id_t NO_ID{0};
-  uint64_t size;                     // how much size to reserve (bytes)
+  uint64_t size = 0;                 // how much size to reserve (bytes)
   ceph::coarse_real_time timestamp;  // when the reservation was done (used for cleaning stale reservations)
-  uint32_t entries;                  // how many entries are reserved
+  uint32_t entries = 0;              // how many entries are reserved
 
   cls_2pc_reservation(uint64_t _size, ceph::coarse_real_time _timestamp, uint32_t _entries) :
       size(_size), timestamp(_timestamp), entries(_entries) {}

From 60bf5e5967ba3d654c1a7d044f1584322cbf9039 Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Tue, 14 Nov 2023 13:14:46 -0800
Subject: [PATCH 1086/2492] doc: Add Reef 18.2.1 release notes

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 doc/releases/index.rst    |   1 +
 doc/releases/reef.rst     | 543 ++++++++++++++++++++++++++++++++++++--
 doc/releases/releases.yml |   2 +
 3 files changed, 521 insertions(+), 25 deletions(-)

diff --git a/doc/releases/index.rst b/doc/releases/index.rst
index a078e5e47ce4..6b0d16f384f9 100644
--- a/doc/releases/index.rst
+++ b/doc/releases/index.rst
@@ -64,6 +64,7 @@ Release timeline
 
 .. _Reef: reef
 .. _18.2.0: reef#v18-2-0-reef
+.. _18.2.1: reef#v18-2-1-reef
 
 .. _Quincy: quincy
 .. _17.2.0: quincy#v17-2-0-quincy
diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
index 6f4f827830b7..89a4d16b546f 100644
--- a/doc/releases/reef.rst
+++ b/doc/releases/reef.rst
@@ -5,6 +5,499 @@ Reef
 Reef is the 18th stable release of Ceph. It is named after the reef squid
 (Sepioteuthis).
 
+v18.2.1 Reef
+============
+
+This is the first backport release in the Reef series, and the first with Debian packages,
+for Debian Bookworm. We recommend that all users update to this release.
+
+Notable Changes
+---------------
+
+* RGW: S3 multipart uploads using Server-Side Encryption now replicate correctly in
+  a multi-site deployment. Previously, the replicas of such objects were corrupted on
+  decryption. A new command, ``radosgw-admin bucket resync encrypted multipart``, can be
+  used to identify these original multipart uploads. The ``LastModified`` timestamp of
+  any identified object is incremented by 1ns to cause peer zones to replicate it again.
+  For multi-site deployments that make any use of Server-Side Encryption, we
+  recommended running this command against every bucket in every zone after all
+  zones have upgraded.
+
+* CEPHFS: MDS now evicts clients which are not advancing their request tids (transaction IDs),
+  which causes a large buildup of session metadata, resulting in the MDS going read-only due to
+  the RADOS operation exceeding the size threshold. `mds_session_metadata_threshold`
+  config controls the maximum size that an (encoded) session metadata can grow.
+
+* RGW: New tools have been added to ``radosgw-admin`` for identifying and
+  correcting issues with versioned bucket indexes. Historical bugs with the
+  versioned bucket index transaction workflow made it possible for the index
+  to accumulate extraneous "book-keeping" olh (object logical head) entries
+  and plain placeholder entries. In some specific scenarios where clients made
+  concurrent requests referencing the same object key, it was likely that a lot
+  of extra index entries would accumulate. When a significant number of these entries are
+  present in a single bucket index shard, they can cause high bucket listing
+  latencies and lifecycle processing failures. To check whether a versioned
+  bucket has unnecessary olh entries, users can now run ``radosgw-admin
+  bucket check olh``. If the ``--fix`` flag is used, the extra entries will
+  be safely removed. A distinct issue from the one described thus far, it is
+  also possible that some versioned buckets are maintaining extra unlinked
+  objects that are not listable from the S3/ Swift APIs. These extra objects
+  are typically a result of PUT requests that exited abnormally, in the middle
+  of a bucket index transaction - so the client would not have received a
+  successful response. Bugs in prior releases made these unlinked objects easy
+  to reproduce with any PUT request that was made on a bucket that was actively
+  resharding. Besides the extra space that these hidden, unlinked objects
+  consume, there can be another side effect in certain scenarios, caused by
+  the nature of the failure mode that produced them, where a client of a bucket
+  that was a victim of this bug may find the object associated with the key to
+  be in an inconsistent state. To check whether a versioned bucket has unlinked
+  entries, users can now run ``radosgw-admin bucket check unlinked``. If the
+  ``--fix`` flag is used, the unlinked objects will be safely removed. Finally,
+  a third issue made it possible for versioned bucket index stats to be
+  accounted inaccurately. The tooling for recalculating versioned bucket stats
+  also had a bug, and was not previously capable of fixing these inaccuracies.
+  This release resolves those issues and users can now expect that the existing
+  ``radosgw-admin bucket check`` command will produce correct results. We
+  recommend that users with versioned buckets, especially those that existed
+  on prior releases, use these new tools to check whether their buckets are
+  affected and to clean them up accordingly.
+
+* mgr/snap-schedule: For clusters with multiple CephFS file systems, all the
+  snap-schedule commands now expect the '--fs' argument.
+
+* RADOS: A POOL_APP_NOT_ENABLED health warning will now be reported if
+  the application is not enabled for the pool irrespective of whether
+  the pool is in use or not. Always tag a pool with an application
+  using ``ceph osd pool application enable`` command to avoid reporting
+  of POOL_APP_NOT_ENABLED health warning for that pool.
+  The user might temporarily mute this warning using
+  ``ceph health mute POOL_APP_NOT_ENABLED``.
+
+* An overview page for rgw to show the overall status of rgw components
+
+* Added management support for RGW Multi-site and Ceph FS Subvolumes and groups
+
+* Fixed few bugs and issues around the new dashboard page like the broken layout,
+  some metrics giving wrong values and introduced a popover to display details
+  when there are HEALTH_WARN or HEALTH_ERR
+
+* Fixed several issues in ceph dashboard on rook backed clusters,
+  and improved the user experience on the rook environment
+
+Changelog
+---------
+
+* .github: Clarify checklist details (`pr#54130 <https://github.com/ceph/ceph/pull/54130>`_, Anthony D'Atri)
+* [CVE-2023-43040] rgw: Fix bucket validation against POST policies (`pr#53756 <https://github.com/ceph/ceph/pull/53756>`_, Joshua Baergen)
+* Adding rollback mechanism to handle bootstrap failures (`pr#53864 <https://github.com/ceph/ceph/pull/53864>`_, Adam King, Redouane Kachach)
+* backport of rook orchestrator fixes and e2e automated testing (`pr#54224 <https://github.com/ceph/ceph/pull/54224>`_, Redouane Kachach)
+* Bluestore: fix bluestore collection_list latency perf counter (`pr#52950 <https://github.com/ceph/ceph/pull/52950>`_, Wangwenjuan)
+* build: Remove ceph-libboost\* packages in install-deps (`pr#52769 <https://github.com/ceph/ceph/pull/52769>`_, Adam Emerson)
+* ceph-volume/cephadm: support lv devices in inventory (`pr#53286 <https://github.com/ceph/ceph/pull/53286>`_, Guillaume Abrioux)
+* ceph-volume: add --osd-id option to raw prepare (`pr#52927 <https://github.com/ceph/ceph/pull/52927>`_, Guillaume Abrioux)
+* ceph-volume: fix a regression in `raw list` (`pr#54521 <https://github.com/ceph/ceph/pull/54521>`_, Guillaume Abrioux)
+* ceph-volume: fix mpath device support (`pr#53539 <https://github.com/ceph/ceph/pull/53539>`_, Guillaume Abrioux)
+* ceph-volume: fix raw list for lvm devices (`pr#52619 <https://github.com/ceph/ceph/pull/52619>`_, Guillaume Abrioux)
+* ceph-volume: fix raw list for lvm devices (`pr#52980 <https://github.com/ceph/ceph/pull/52980>`_, Guillaume Abrioux)
+* ceph-volume: Revert "ceph-volume: fix raw list for lvm devices" (`pr#54429 <https://github.com/ceph/ceph/pull/54429>`_, Matthew Booth, Guillaume Abrioux)
+* ceph: allow xlock state to be LOCK_PREXLOCK when putting it (`pr#53661 <https://github.com/ceph/ceph/pull/53661>`_, Xiubo Li)
+* ceph_fs.h: add separate owner\_{u,g}id fields (`pr#53138 <https://github.com/ceph/ceph/pull/53138>`_, Alexander Mikhalitsyn)
+* ceph_volume: support encrypted volumes for lvm new-db/new-wal/migrate commands (`pr#52875 <https://github.com/ceph/ceph/pull/52875>`_, Igor Fedotov)
+* cephadm batch backport Aug 23 (`pr#53124 <https://github.com/ceph/ceph/pull/53124>`_, Adam King, Luis Domingues, John Mulligan, Redouane Kachach)
+* cephadm: add a --dry-run option to cephadm shell (`pr#54220 <https://github.com/ceph/ceph/pull/54220>`_, John Mulligan)
+* cephadm: add tcmu-runner to logrotate config (`pr#53122 <https://github.com/ceph/ceph/pull/53122>`_, Adam King)
+* cephadm: Adding support to configure public_network cfg section (`pr#53110 <https://github.com/ceph/ceph/pull/53110>`_, Redouane Kachach)
+* cephadm: delete /tmp/cephadm-<fsid> when removing the cluster (`pr#53109 <https://github.com/ceph/ceph/pull/53109>`_, Redouane Kachach)
+* cephadm: Fix extra_container_args for iSCSI (`pr#53010 <https://github.com/ceph/ceph/pull/53010>`_, Raimund Sacherer)
+* cephadm: fix haproxy version with certain containers (`pr#53751 <https://github.com/ceph/ceph/pull/53751>`_, Adam King)
+* cephadm: make custom_configs work for tcmu-runner container (`pr#53404 <https://github.com/ceph/ceph/pull/53404>`_, Adam King)
+* cephadm: run tcmu-runner through script to do restart on failure (`pr#53866 <https://github.com/ceph/ceph/pull/53866>`_, Adam King)
+* cephadm: support for CA signed keys (`pr#53121 <https://github.com/ceph/ceph/pull/53121>`_, Adam King)
+* cephfs-journal-tool: disambiguate usage of all keyword (in tool help) (`pr#53646 <https://github.com/ceph/ceph/pull/53646>`_, Manish M Yathnalli)
+* cephfs-mirror: do not run concurrent C_RestartMirroring context (`issue#62072 <http://tracker.ceph.com/issues/62072>`_, `pr#53638 <https://github.com/ceph/ceph/pull/53638>`_, Venky Shankar)
+* cephfs: implement snapdiff (`pr#53229 <https://github.com/ceph/ceph/pull/53229>`_, Igor Fedotov, Lucian Petrut, Denis Barahtanov)
+* cephfs_mirror: correctly set top level dir permissions (`pr#53271 <https://github.com/ceph/ceph/pull/53271>`_, Milind Changire)
+* client: always refresh mds feature bits on session open (`issue#63188 <http://tracker.ceph.com/issues/63188>`_, `pr#54146 <https://github.com/ceph/ceph/pull/54146>`_, Venky Shankar)
+* client: correct quota check in Client::_rename() (`pr#52578 <https://github.com/ceph/ceph/pull/52578>`_, Rishabh Dave)
+* client: do not send metrics until the MDS rank is ready (`pr#52501 <https://github.com/ceph/ceph/pull/52501>`_, Xiubo Li)
+* client: force sending cap revoke ack always (`pr#52507 <https://github.com/ceph/ceph/pull/52507>`_, Xiubo Li)
+* client: issue a cap release immediately if no cap exists (`pr#52850 <https://github.com/ceph/ceph/pull/52850>`_, Xiubo Li)
+* client: move the Inode to new auth mds session when changing auth cap (`pr#53666 <https://github.com/ceph/ceph/pull/53666>`_, Xiubo Li)
+* client: trigger to flush the buffer when making snapshot (`pr#52497 <https://github.com/ceph/ceph/pull/52497>`_, Xiubo Li)
+* client: wait rename to finish (`pr#52504 <https://github.com/ceph/ceph/pull/52504>`_, Xiubo Li)
+* cmake: ensure fmtlib is at least 8.1.1 (`pr#52970 <https://github.com/ceph/ceph/pull/52970>`_, Abhishek Lekshmanan)
+* Consider setting "bulk" autoscale pool flag when automatically creating a data pool for CephFS (`pr#52899 <https://github.com/ceph/ceph/pull/52899>`_, Leonid Usov)
+* crimson/admin/admin_socket: remove path file if it exists (`pr#53964 <https://github.com/ceph/ceph/pull/53964>`_, Matan Breizman)
+* crimson/ertr: assert on invocability of func provided to safe_then() (`pr#53958 <https://github.com/ceph/ceph/pull/53958>`_, Radosław Zarzyński)
+* crimson/mgr: Fix config show command (`pr#53954 <https://github.com/ceph/ceph/pull/53954>`_, Aishwarya Mathuria)
+* crimson/net: consolidate messenger implementations and enable multi-shard UTs (`pr#54095 <https://github.com/ceph/ceph/pull/54095>`_, Yingxin Cheng)
+* crimson/net: set TCP_NODELAY according to ms_tcp_nodelay (`pr#54063 <https://github.com/ceph/ceph/pull/54063>`_, Xuehan Xu)
+* crimson/net: support connections in multiple shards (`pr#53949 <https://github.com/ceph/ceph/pull/53949>`_, Yingxin Cheng)
+* crimson/os/object_data_handler: splitting right side doesn't mean splitting only one extent (`pr#54061 <https://github.com/ceph/ceph/pull/54061>`_, Xuehan Xu)
+* crimson/os/seastore/backref_manager: scan backref entries by journal seq (`pr#53939 <https://github.com/ceph/ceph/pull/53939>`_, Zhang Song)
+* crimson/os/seastore/btree: should add left's size when merging levels… (`pr#53946 <https://github.com/ceph/ceph/pull/53946>`_, Xuehan Xu)
+* crimson/os/seastore/cache: don't add EXIST_CLEAN extents to lru (`pr#54098 <https://github.com/ceph/ceph/pull/54098>`_, Xuehan Xu)
+* crimson/os/seastore/cached_extent: add prepare_commit interface (`pr#53941 <https://github.com/ceph/ceph/pull/53941>`_, Xuehan Xu)
+* crimson/os/seastore/cbj: fix a potential overflow bug on segment_seq (`pr#53968 <https://github.com/ceph/ceph/pull/53968>`_, Myoungwon Oh)
+* crimson/os/seastore/collection_manager: fill CollectionNode::decoded on clean reads (`pr#53956 <https://github.com/ceph/ceph/pull/53956>`_, Xuehan Xu)
+* crimson/os/seastore/journal/cbj: generalize scan_valid_records() (`pr#53961 <https://github.com/ceph/ceph/pull/53961>`_, Myoungwon Oh, Yingxin Cheng)
+* crimson/os/seastore/omap_manager: correct editor settings (`pr#53947 <https://github.com/ceph/ceph/pull/53947>`_, Zhang Song)
+* crimson/os/seastore/omap_manager: fix the entry leak issue in BtreeOMapManager::omap_list() (`pr#53962 <https://github.com/ceph/ceph/pull/53962>`_, Xuehan Xu)
+* crimson/os/seastore/onode_manager: populate value recorders of onodes to be erased (`pr#53966 <https://github.com/ceph/ceph/pull/53966>`_, Xuehan Xu)
+* crimson/os/seastore/rbm: make rbm support multiple shards (`pr#53952 <https://github.com/ceph/ceph/pull/53952>`_, Myoungwon Oh)
+* crimson/os/seastore/transaction_manager: data loss issues (`pr#53955 <https://github.com/ceph/ceph/pull/53955>`_, Xuehan Xu)
+* crimson/os/seastore/transaction_manager: move intermediate_key by "remap_offset" when remapping the "back" half of the original pin (`pr#54140 <https://github.com/ceph/ceph/pull/54140>`_, Xuehan Xu)
+* crimson/os/seastore/zbd: zbdsegmentmanager write path fixes (`pr#54062 <https://github.com/ceph/ceph/pull/54062>`_, Aravind Ramesh)
+* crimson/os/seastore: add metrics about total invalidated transactions (`pr#53953 <https://github.com/ceph/ceph/pull/53953>`_, Zhang Song)
+* crimson/os/seastore: create page aligned bufferptr in copy ctor of CachedExtent (`pr#54097 <https://github.com/ceph/ceph/pull/54097>`_, Zhang Song)
+* crimson/os/seastore: enable SMR HDD (`pr#53935 <https://github.com/ceph/ceph/pull/53935>`_, Aravind Ramesh)
+* crimson/os/seastore: fix ceph_assert in segment_manager.h (`pr#53938 <https://github.com/ceph/ceph/pull/53938>`_, Aravind Ramesh)
+* crimson/os/seastore: fix daggling reference of oid in SeaStore::Shard::stat() (`pr#53960 <https://github.com/ceph/ceph/pull/53960>`_, Xuehan Xu)
+* crimson/os/seastore: fix in check_node (`pr#53945 <https://github.com/ceph/ceph/pull/53945>`_, Xinyu Huang)
+* crimson/os/seastore: OP_CLONE in seastore (`pr#54092 <https://github.com/ceph/ceph/pull/54092>`_, xuxuehan, Xuehan Xu)
+* crimson/os/seastore: realize lazy read in split overwrite with overwrite refactor (`pr#53951 <https://github.com/ceph/ceph/pull/53951>`_, Xinyu Huang)
+* crimson/os/seastore: retire_extent_addr clean up (`pr#53959 <https://github.com/ceph/ceph/pull/53959>`_, Xinyu Huang)
+* crimson/osd/heartbeat: Improve maybe_share_osdmap behavior (`pr#53940 <https://github.com/ceph/ceph/pull/53940>`_, Samuel Just)
+* crimson/osd/lsan_suppressions.cc: Add MallocExtension::Initialize() (`pr#54057 <https://github.com/ceph/ceph/pull/54057>`_, Mark Nelson, Matan Breizman)
+* crimson/osd/lsan_suppressions: add MallocExtension::Register (`pr#54139 <https://github.com/ceph/ceph/pull/54139>`_, Matan Breizman)
+* crimson/osd/object_context: consider clones found as long as they're in SnapSet::clones (`pr#53965 <https://github.com/ceph/ceph/pull/53965>`_, Xuehan Xu)
+* crimson/osd/osd_operations: add pipeline to LogMissingRequest to sync it (`pr#53957 <https://github.com/ceph/ceph/pull/53957>`_, Xuehan Xu)
+* crimson/osd/osd_operations: consistent naming to pipeline users (`pr#54060 <https://github.com/ceph/ceph/pull/54060>`_, Matan Breizman)
+* crimson/osd/pg: check if backfill_state exists when judging objects' (`pr#53963 <https://github.com/ceph/ceph/pull/53963>`_, Xuehan Xu)
+* crimson/osd/watch: Add logs around Watch/Notify (`pr#53950 <https://github.com/ceph/ceph/pull/53950>`_, Matan Breizman)
+* crimson/osd: add embedded suppression ruleset for LSan (`pr#53937 <https://github.com/ceph/ceph/pull/53937>`_, Radoslaw Zarzynski)
+* crimson/osd: cleanup and drop OSD::ShardDispatcher (`pr#54138 <https://github.com/ceph/ceph/pull/54138>`_, Yingxin Cheng)
+* Crimson/osd: Disable concurrent MOSDMap handling (`pr#53944 <https://github.com/ceph/ceph/pull/53944>`_, Matan Breizman)
+* crimson/osd: don't ignore start_pg_operation returned future (`pr#53948 <https://github.com/ceph/ceph/pull/53948>`_, Matan Breizman)
+* crimson/osd: fix ENOENT on accessing RadosGW user's index of buckets (`pr#53942 <https://github.com/ceph/ceph/pull/53942>`_, Radoslaw Zarzynski)
+* crimson/osd: fix Notify life-time mismanagement in Watch::notify_ack (`pr#53943 <https://github.com/ceph/ceph/pull/53943>`_, Radoslaw Zarzynski)
+* crimson/osd: fixes and cleanups around multi-core OSD (`pr#54091 <https://github.com/ceph/ceph/pull/54091>`_, Yingxin Cheng)
+* Crimson/osd: support multicore osd (`pr#54058 <https://github.com/ceph/ceph/pull/54058>`_, chunmei)
+* crimson/tools/perf_crimson_msgr: integrate multi-core msgr with various improvements (`pr#54059 <https://github.com/ceph/ceph/pull/54059>`_, Yingxin Cheng)
+* crimson/tools/perf_crimson_msgr: randomize client nonce (`pr#54093 <https://github.com/ceph/ceph/pull/54093>`_, Yingxin Cheng)
+* crimson/tools/perf_staged_fltree: fix compile error (`pr#54096 <https://github.com/ceph/ceph/pull/54096>`_, Myoungwon Oh)
+* crimson/vstart: default seastore_device_size will be out of space f… (`pr#53969 <https://github.com/ceph/ceph/pull/53969>`_, chunmei)
+* crimson: Enable tcmalloc when using seastar (`pr#54105 <https://github.com/ceph/ceph/pull/54105>`_, Mark Nelson, Matan Breizman)
+* debian/control: add docker-ce as recommends for cephadm package (`pr#52908 <https://github.com/ceph/ceph/pull/52908>`_, Adam King)
+* Debian: update to dh compat 12, fix more serious packaging errors, correct copyright syntax (`pr#53654 <https://github.com/ceph/ceph/pull/53654>`_, Matthew Vernon)
+* doc/architecture.rst - edit a sentence (`pr#53372 <https://github.com/ceph/ceph/pull/53372>`_, Zac Dover)
+* doc/architecture.rst - edit up to "Cluster Map" (`pr#53366 <https://github.com/ceph/ceph/pull/53366>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (`pr#53619 <https://github.com/ceph/ceph/pull/53619>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (one of several) (`pr#53585 <https://github.com/ceph/ceph/pull/53585>`_, Zac Dover)
+* doc/architecture: "Edit HA Auth" (one of several) (`pr#53491 <https://github.com/ceph/ceph/pull/53491>`_, Zac Dover)
+* doc/architecture: edit "Calculating PG IDs" (`pr#53748 <https://github.com/ceph/ceph/pull/53748>`_, Zac Dover)
+* doc/architecture: edit "Cluster Map" (`pr#53434 <https://github.com/ceph/ceph/pull/53434>`_, Zac Dover)
+* doc/architecture: edit "Data Scrubbing" (`pr#53730 <https://github.com/ceph/ceph/pull/53730>`_, Zac Dover)
+* doc/architecture: Edit "HA Auth" (`pr#53488 <https://github.com/ceph/ceph/pull/53488>`_, Zac Dover)
+* doc/architecture: edit "HA Authentication" (`pr#53632 <https://github.com/ceph/ceph/pull/53632>`_, Zac Dover)
+* doc/architecture: edit "High Avail. Monitors" (`pr#53451 <https://github.com/ceph/ceph/pull/53451>`_, Zac Dover)
+* doc/architecture: edit "OSD Membership and Status" (`pr#53727 <https://github.com/ceph/ceph/pull/53727>`_, Zac Dover)
+* doc/architecture: edit "OSDs service clients directly" (`pr#53686 <https://github.com/ceph/ceph/pull/53686>`_, Zac Dover)
+* doc/architecture: edit "Peering and Sets" (`pr#53871 <https://github.com/ceph/ceph/pull/53871>`_, Zac Dover)
+* doc/architecture: edit "Replication" (`pr#53738 <https://github.com/ceph/ceph/pull/53738>`_, Zac Dover)
+* doc/architecture: edit "SDEH" (`pr#53659 <https://github.com/ceph/ceph/pull/53659>`_, Zac Dover)
+* doc/architecture: edit several sections (`pr#53742 <https://github.com/ceph/ceph/pull/53742>`_, Zac Dover)
+* doc/architecture: repair RBD sentence (`pr#53877 <https://github.com/ceph/ceph/pull/53877>`_, Zac Dover)
+* doc/ceph-volume: explain idempotence (`pr#54233 <https://github.com/ceph/ceph/pull/54233>`_, Zac Dover)
+* doc/ceph-volume: improve front matter (`pr#54235 <https://github.com/ceph/ceph/pull/54235>`_, Zac Dover)
+* doc/cephadm/services: remove excess rendered indentation in osd.rst (`pr#54323 <https://github.com/ceph/ceph/pull/54323>`_, Ville Ojamo)
+* doc/cephadm: add ssh note to install.rst (`pr#53199 <https://github.com/ceph/ceph/pull/53199>`_, Zac Dover)
+* doc/cephadm: edit "Adding Hosts" in install.rst (`pr#53224 <https://github.com/ceph/ceph/pull/53224>`_, Zac Dover)
+* doc/cephadm: edit sentence in mgr.rst (`pr#53164 <https://github.com/ceph/ceph/pull/53164>`_, Zac Dover)
+* doc/cephadm: edit troubleshooting.rst (1 of x) (`pr#54283 <https://github.com/ceph/ceph/pull/54283>`_, Zac Dover)
+* doc/cephadm: edit troubleshooting.rst (2 of x) (`pr#54320 <https://github.com/ceph/ceph/pull/54320>`_, Zac Dover)
+* doc/cephadm: fix typo in cephadm initial crush location section (`pr#52887 <https://github.com/ceph/ceph/pull/52887>`_, John Mulligan)
+* doc/cephadm: fix typo in set ssh key command (`pr#54388 <https://github.com/ceph/ceph/pull/54388>`_, Piotr Parczewski)
+* doc/cephadm: update cephadm reef version (`pr#53162 <https://github.com/ceph/ceph/pull/53162>`_, Rongqi Sun)
+* doc/cephfs: edit mount-using-fuse.rst (`pr#54353 <https://github.com/ceph/ceph/pull/54353>`_, Jaanus Torp)
+* doc/cephfs: write cephfs commands fully in docs (`pr#53402 <https://github.com/ceph/ceph/pull/53402>`_, Rishabh Dave)
+* doc/config: edit "ceph-conf.rst" (`pr#54463 <https://github.com/ceph/ceph/pull/54463>`_, Zac Dover)
+* doc/configuration: edit "bg" in mon-config-ref.rst (`pr#53347 <https://github.com/ceph/ceph/pull/53347>`_, Zac Dover)
+* doc/dev/release-checklist: check telemetry validation (`pr#52805 <https://github.com/ceph/ceph/pull/52805>`_, Yaarit Hatuka)
+* doc/dev: Fix typos in files cephfs-mirroring.rst and deduplication.rst (`pr#53519 <https://github.com/ceph/ceph/pull/53519>`_, Daniel Parkes)
+* doc/dev: remove cache-pool (`pr#54007 <https://github.com/ceph/ceph/pull/54007>`_, Zac Dover)
+* doc/glossary: add "primary affinity" to glossary (`pr#53427 <https://github.com/ceph/ceph/pull/53427>`_, Zac Dover)
+* doc/glossary: add "Quorum" to glossary (`pr#54509 <https://github.com/ceph/ceph/pull/54509>`_, Zac Dover)
+* doc/glossary: improve "BlueStore" entry (`pr#54265 <https://github.com/ceph/ceph/pull/54265>`_, Zac Dover)
+* doc/man/8/ceph-monstore-tool: add documentation (`pr#52872 <https://github.com/ceph/ceph/pull/52872>`_, Matan Breizman)
+* doc/man/8: improve radosgw-admin.rst (`pr#53267 <https://github.com/ceph/ceph/pull/53267>`_, Anthony D'Atri)
+* doc/man: edit ceph-monstore-tool.rst (`pr#53476 <https://github.com/ceph/ceph/pull/53476>`_, Zac Dover)
+* doc/man: radosgw-admin.rst typo (`pr#53315 <https://github.com/ceph/ceph/pull/53315>`_, Zac Dover)
+* doc/man: remove docs about support for unix domain sockets (`pr#53312 <https://github.com/ceph/ceph/pull/53312>`_, Zac Dover)
+* doc/man: s/kvstore-tool/monstore-tool/ (`pr#53536 <https://github.com/ceph/ceph/pull/53536>`_, Zac Dover)
+* doc/rados/configuration: Avoid repeating "support" in msgr2.rst (`pr#52998 <https://github.com/ceph/ceph/pull/52998>`_, Ville Ojamo)
+* doc/rados: add bulk flag to pools.rst (`pr#53317 <https://github.com/ceph/ceph/pull/53317>`_, Zac Dover)
+* doc/rados: edit "troubleshooting-mon" (`pr#54502 <https://github.com/ceph/ceph/pull/54502>`_, Zac Dover)
+* doc/rados: edit memory-profiling.rst (`pr#53932 <https://github.com/ceph/ceph/pull/53932>`_, Zac Dover)
+* doc/rados: edit operations/add-or-rm-mons (1 of x) (`pr#52889 <https://github.com/ceph/ceph/pull/52889>`_, Zac Dover)
+* doc/rados: edit operations/add-or-rm-mons (2 of x) (`pr#52825 <https://github.com/ceph/ceph/pull/52825>`_, Zac Dover)
+* doc/rados: edit ops/control.rst (1 of x) (`pr#53811 <https://github.com/ceph/ceph/pull/53811>`_, zdover23, Zac Dover)
+* doc/rados: edit ops/control.rst (2 of x) (`pr#53815 <https://github.com/ceph/ceph/pull/53815>`_, Zac Dover)
+* doc/rados: edit t-mon "common issues" (1 of x) (`pr#54418 <https://github.com/ceph/ceph/pull/54418>`_, Zac Dover)
+* doc/rados: edit t-mon "common issues" (2 of x) (`pr#54421 <https://github.com/ceph/ceph/pull/54421>`_, Zac Dover)
+* doc/rados: edit t-mon "common issues" (3 of x) (`pr#54438 <https://github.com/ceph/ceph/pull/54438>`_, Zac Dover)
+* doc/rados: edit t-mon "common issues" (4 of x) (`pr#54443 <https://github.com/ceph/ceph/pull/54443>`_, Zac Dover)
+* doc/rados: edit t-mon "common issues" (5 of x) (`pr#54455 <https://github.com/ceph/ceph/pull/54455>`_, Zac Dover)
+* doc/rados: edit t-mon.rst text (`pr#54349 <https://github.com/ceph/ceph/pull/54349>`_, Zac Dover)
+* doc/rados: edit t-shooting-mon.rst (`pr#54427 <https://github.com/ceph/ceph/pull/54427>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (2 of x) (`pr#52839 <https://github.com/ceph/ceph/pull/52839>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (3 of x) (`pr#53879 <https://github.com/ceph/ceph/pull/53879>`_, Zac Dover)
+* doc/rados: edit troubleshooting-mon.rst (4 of x) (`pr#53897 <https://github.com/ceph/ceph/pull/53897>`_, Zac Dover)
+* doc/rados: edit troubleshooting-osd (1 of x) (`pr#53982 <https://github.com/ceph/ceph/pull/53982>`_, Zac Dover)
+* doc/rados: Edit troubleshooting-osd (2 of x) (`pr#54000 <https://github.com/ceph/ceph/pull/54000>`_, Zac Dover)
+* doc/rados: Edit troubleshooting-osd (3 of x) (`pr#54026 <https://github.com/ceph/ceph/pull/54026>`_, Zac Dover)
+* doc/rados: edit troubleshooting-pg (2 of x) (`pr#54114 <https://github.com/ceph/ceph/pull/54114>`_, Zac Dover)
+* doc/rados: edit troubleshooting-pg.rst (`pr#54228 <https://github.com/ceph/ceph/pull/54228>`_, Zac Dover)
+* doc/rados: edit troubleshooting-pg.rst (1 of x) (`pr#54073 <https://github.com/ceph/ceph/pull/54073>`_, Zac Dover)
+* doc/rados: edit troubleshooting.rst (`pr#53837 <https://github.com/ceph/ceph/pull/53837>`_, Zac Dover)
+* doc/rados: edit troubleshooting/community.rst (`pr#53881 <https://github.com/ceph/ceph/pull/53881>`_, Zac Dover)
+* doc/rados: format "initial troubleshooting" (`pr#54477 <https://github.com/ceph/ceph/pull/54477>`_, Zac Dover)
+* doc/rados: format Q&A list in t-mon.rst (`pr#54345 <https://github.com/ceph/ceph/pull/54345>`_, Zac Dover)
+* doc/rados: format Q&A list in tshooting-mon.rst (`pr#54366 <https://github.com/ceph/ceph/pull/54366>`_, Zac Dover)
+* doc/rados: improve "scrubbing" explanation (`pr#54270 <https://github.com/ceph/ceph/pull/54270>`_, Zac Dover)
+* doc/rados: parallelize t-mon headings (`pr#54461 <https://github.com/ceph/ceph/pull/54461>`_, Zac Dover)
+* doc/rados: remove cache-tiering-related keys (`pr#54227 <https://github.com/ceph/ceph/pull/54227>`_, Zac Dover)
+* doc/rados: remove FileStore material (in Reef) (`pr#54008 <https://github.com/ceph/ceph/pull/54008>`_, Zac Dover)
+* doc/rados: remove HitSet-related key information (`pr#54217 <https://github.com/ceph/ceph/pull/54217>`_, Zac Dover)
+* doc/rados: update monitoring-osd-pg.rst (`pr#52958 <https://github.com/ceph/ceph/pull/52958>`_, Zac Dover)
+* doc/radosgw: Improve dynamicresharding.rst (`pr#54368 <https://github.com/ceph/ceph/pull/54368>`_, Anthony D'Atri)
+* doc/radosgw: Improve language and formatting in config-ref.rst (`pr#52835 <https://github.com/ceph/ceph/pull/52835>`_, Ville Ojamo)
+* doc/radosgw: multisite - edit "migrating a single-site" (`pr#53261 <https://github.com/ceph/ceph/pull/53261>`_, Qi Tao)
+* doc/radosgw: update rate limit management (`pr#52910 <https://github.com/ceph/ceph/pull/52910>`_, Zac Dover)
+* doc/README.md - edit "Building Ceph" (`pr#53057 <https://github.com/ceph/ceph/pull/53057>`_, Zac Dover)
+* doc/README.md - improve "Running a test cluster" (`pr#53258 <https://github.com/ceph/ceph/pull/53258>`_, Zac Dover)
+* doc/rgw: correct statement about default zone features (`pr#52833 <https://github.com/ceph/ceph/pull/52833>`_, Casey Bodley)
+* doc/rgw: pubsub capabilities reference was removed from docs (`pr#54137 <https://github.com/ceph/ceph/pull/54137>`_, Yuval Lifshitz)
+* doc/rgw: several response headers are supported (`pr#52803 <https://github.com/ceph/ceph/pull/52803>`_, Casey Bodley)
+* doc/start: correct ABC test chart (`pr#53256 <https://github.com/ceph/ceph/pull/53256>`_, Dmitry Kvashnin)
+* doc/start: edit os-recommendations.rst (`pr#53179 <https://github.com/ceph/ceph/pull/53179>`_, Zac Dover)
+* doc/start: fix typo in hardware-recommendations.rst (`pr#54480 <https://github.com/ceph/ceph/pull/54480>`_, Anthony D'Atri)
+* doc/start: Modernize and clarify hardware-recommendations.rst (`pr#54071 <https://github.com/ceph/ceph/pull/54071>`_, Anthony D'Atri)
+* doc/start: refactor ABC test chart (`pr#53094 <https://github.com/ceph/ceph/pull/53094>`_, Zac Dover)
+* doc/start: update "platforms" table (`pr#53075 <https://github.com/ceph/ceph/pull/53075>`_, Zac Dover)
+* doc/start: update linking conventions (`pr#52912 <https://github.com/ceph/ceph/pull/52912>`_, Zac Dover)
+* doc/start: update linking conventions (`pr#52841 <https://github.com/ceph/ceph/pull/52841>`_, Zac Dover)
+* doc/troubleshooting: edit cpu-profiling.rst (`pr#53059 <https://github.com/ceph/ceph/pull/53059>`_, Zac Dover)
+* doc: Add a note on possible deadlock on volume deletion (`pr#52946 <https://github.com/ceph/ceph/pull/52946>`_, Kotresh HR)
+* doc: add note for removing (automatic) partitioning policy (`pr#53569 <https://github.com/ceph/ceph/pull/53569>`_, Venky Shankar)
+* doc: Add Reef 18.2.0 release notes (`pr#52905 <https://github.com/ceph/ceph/pull/52905>`_, Zac Dover)
+* doc: Add warning on manual CRUSH rule removal (`pr#53420 <https://github.com/ceph/ceph/pull/53420>`_, Alvin Owyong)
+* doc: clarify upmap balancer documentation (`pr#53004 <https://github.com/ceph/ceph/pull/53004>`_, Laura Flores)
+* doc: correct option name (`pr#53128 <https://github.com/ceph/ceph/pull/53128>`_, Patrick Donnelly)
+* doc: do not recommend pulling cephadm from git (`pr#52997 <https://github.com/ceph/ceph/pull/52997>`_, John Mulligan)
+* doc: Documentation about main Ceph metrics (`pr#54111 <https://github.com/ceph/ceph/pull/54111>`_, Juan Miguel Olmo Martínez)
+* doc: edit README.md - contributing code (`pr#53049 <https://github.com/ceph/ceph/pull/53049>`_, Zac Dover)
+* doc: expand and consolidate mds placement (`pr#53146 <https://github.com/ceph/ceph/pull/53146>`_, Patrick Donnelly)
+* doc: Fix doc for mds cap acquisition throttle (`pr#53024 <https://github.com/ceph/ceph/pull/53024>`_, Kotresh HR)
+* doc: improve submodule update command - README.md (`pr#53000 <https://github.com/ceph/ceph/pull/53000>`_, Zac Dover)
+* doc: make instructions to get an updated cephadm common (`pr#53260 <https://github.com/ceph/ceph/pull/53260>`_, John Mulligan)
+* doc: remove egg fragment from dev/developer_guide/running-tests-locally (`pr#53853 <https://github.com/ceph/ceph/pull/53853>`_, Dhairya Parmar)
+* doc: Update dynamicresharding.rst (`pr#54329 <https://github.com/ceph/ceph/pull/54329>`_, Aliaksei Makarau)
+* doc: Update mClock QOS documentation to discard osd_mclock_cost_per\_\* (`pr#54079 <https://github.com/ceph/ceph/pull/54079>`_, tanchangzhi)
+* doc: update rados.cc (`pr#52967 <https://github.com/ceph/ceph/pull/52967>`_, Zac Dover)
+* doc: update test cluster commands in README.md (`pr#53349 <https://github.com/ceph/ceph/pull/53349>`_, Zac Dover)
+* exporter: add ceph_daemon labels to labeled counters as well (`pr#53695 <https://github.com/ceph/ceph/pull/53695>`_, avanthakkar)
+* exposed the open api and telemetry links in details card (`pr#53142 <https://github.com/ceph/ceph/pull/53142>`_, cloudbehl, dpandit)
+* libcephsqlite: fill 0s in unread portion of buffer (`pr#53101 <https://github.com/ceph/ceph/pull/53101>`_, Patrick Donnelly)
+* librbd: kick ExclusiveLock state machine on client being blocklisted when waiting for lock (`pr#53293 <https://github.com/ceph/ceph/pull/53293>`_, Ramana Raja)
+* librbd: kick ExclusiveLock state machine stalled waiting for lock from reacquire_lock() (`pr#53919 <https://github.com/ceph/ceph/pull/53919>`_, Ramana Raja)
+* librbd: make CreatePrimaryRequest remove any unlinked mirror snapshots (`pr#53276 <https://github.com/ceph/ceph/pull/53276>`_, Ilya Dryomov)
+* MClientRequest: properly handle ceph_mds_request_head_legacy for ext_num_retry, ext_num_fwd, owner_uid, owner_gid (`pr#54407 <https://github.com/ceph/ceph/pull/54407>`_, Alexander Mikhalitsyn)
+* MDS imported_inodes metric is not updated (`pr#51698 <https://github.com/ceph/ceph/pull/51698>`_, Yongseok Oh)
+* mds/FSMap: allow upgrades if no up mds (`pr#53851 <https://github.com/ceph/ceph/pull/53851>`_, Patrick Donnelly)
+* mds/Server: mark a cap acquisition throttle event in the request (`pr#53168 <https://github.com/ceph/ceph/pull/53168>`_, Leonid Usov)
+* mds: acquire inode snaplock in open (`pr#53183 <https://github.com/ceph/ceph/pull/53183>`_, Patrick Donnelly)
+* mds: add event for batching getattr/lookup (`pr#53558 <https://github.com/ceph/ceph/pull/53558>`_, Patrick Donnelly)
+* mds: adjust pre_segments_size for MDLog when trimming segments for st… (`issue#59833 <http://tracker.ceph.com/issues/59833>`_, `pr#54035 <https://github.com/ceph/ceph/pull/54035>`_, Venky Shankar)
+* mds: blocklist clients with "bloated" session metadata (`issue#62873 <http://tracker.ceph.com/issues/62873>`_, `issue#61947 <http://tracker.ceph.com/issues/61947>`_, `pr#53329 <https://github.com/ceph/ceph/pull/53329>`_, Venky Shankar)
+* mds: do not send split_realms for CEPH_SNAP_OP_UPDATE msg (`pr#52847 <https://github.com/ceph/ceph/pull/52847>`_, Xiubo Li)
+* mds: drop locks and retry when lock set changes (`pr#53241 <https://github.com/ceph/ceph/pull/53241>`_, Patrick Donnelly)
+* mds: dump locks when printing mutation ops (`pr#52975 <https://github.com/ceph/ceph/pull/52975>`_, Patrick Donnelly)
+* mds: fix deadlock between unlinking and linkmerge (`pr#53497 <https://github.com/ceph/ceph/pull/53497>`_, Xiubo Li)
+* mds: fix stray evaluation using scrub and introduce new option (`pr#50813 <https://github.com/ceph/ceph/pull/50813>`_, Dhairya Parmar)
+* mds: Fix the linkmerge assert check (`pr#52724 <https://github.com/ceph/ceph/pull/52724>`_, Kotresh HR)
+* mds: log message when exiting due to asok command (`pr#53548 <https://github.com/ceph/ceph/pull/53548>`_, Patrick Donnelly)
+* mds: MDLog::_recovery_thread: handle the errors gracefully (`pr#52512 <https://github.com/ceph/ceph/pull/52512>`_, Jos Collin)
+* mds: session ls command appears twice in command listing (`pr#52515 <https://github.com/ceph/ceph/pull/52515>`_, Neeraj Pratap Singh)
+* mds: skip forwarding request if the session were removed (`pr#52846 <https://github.com/ceph/ceph/pull/52846>`_, Xiubo Li)
+* mds: update mdlog perf counters during replay (`pr#52681 <https://github.com/ceph/ceph/pull/52681>`_, Patrick Donnelly)
+* mds: use variable g_ceph_context directly in MDSAuthCaps (`pr#52819 <https://github.com/ceph/ceph/pull/52819>`_, Rishabh Dave)
+* mgr/cephadm: Add "networks" parameter to orch apply rgw (`pr#53120 <https://github.com/ceph/ceph/pull/53120>`_, Teoman ONAY)
+* mgr/cephadm: add ability to zap OSDs' devices while draining host (`pr#53869 <https://github.com/ceph/ceph/pull/53869>`_, Adam King)
+* mgr/cephadm: add is_host\_<status> functions to HostCache (`pr#53118 <https://github.com/ceph/ceph/pull/53118>`_, Adam King)
+* mgr/cephadm: Adding sort-by support for ceph orch ps (`pr#53867 <https://github.com/ceph/ceph/pull/53867>`_, Redouane Kachach)
+* mgr/cephadm: allow draining host without removing conf/keyring files (`pr#53123 <https://github.com/ceph/ceph/pull/53123>`_, Adam King)
+* mgr/cephadm: also don't write client files/tuned profiles to maintenance hosts (`pr#53111 <https://github.com/ceph/ceph/pull/53111>`_, Adam King)
+* mgr/cephadm: ceph orch add fails when ipv6 address is surrounded by square brackets (`pr#53870 <https://github.com/ceph/ceph/pull/53870>`_, Teoman ONAY)
+* mgr/cephadm: don't use image tag in orch upgrade ls (`pr#53865 <https://github.com/ceph/ceph/pull/53865>`_, Adam King)
+* mgr/cephadm: fix default image base in reef (`pr#53922 <https://github.com/ceph/ceph/pull/53922>`_, Adam King)
+* mgr/cephadm: fix REFRESHED column of orch ps being unpopulated (`pr#53741 <https://github.com/ceph/ceph/pull/53741>`_, Adam King)
+* mgr/cephadm: fix upgrades with nvmeof (`pr#53924 <https://github.com/ceph/ceph/pull/53924>`_, Adam King)
+* mgr/cephadm: removing double quotes from the generated nvmeof config (`pr#53868 <https://github.com/ceph/ceph/pull/53868>`_, Redouane Kachach)
+* mgr/cephadm: show meaningful messages when failing to execute cmds (`pr#53106 <https://github.com/ceph/ceph/pull/53106>`_, Redouane Kachach)
+* mgr/cephadm: storing prometheus/alertmanager credentials in monstore (`pr#53119 <https://github.com/ceph/ceph/pull/53119>`_, Redouane Kachach)
+* mgr/cephadm: validate host label before removing (`pr#53112 <https://github.com/ceph/ceph/pull/53112>`_, Redouane Kachach)
+* mgr/dashboard: add e2e tests for cephfs management (`pr#53190 <https://github.com/ceph/ceph/pull/53190>`_, Nizamudeen A)
+* mgr/dashboard: Add more decimals in latency graph (`pr#52727 <https://github.com/ceph/ceph/pull/52727>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: add port and zone endpoints to import realm token form in rgw multisite (`pr#54118 <https://github.com/ceph/ceph/pull/54118>`_, Aashish Sharma)
+* mgr/dashboard: add validator for size field in the forms (`pr#53378 <https://github.com/ceph/ceph/pull/53378>`_, Nizamudeen A)
+* mgr/dashboard: align charts of landing page (`pr#53543 <https://github.com/ceph/ceph/pull/53543>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: allow PUT in CORS (`pr#52705 <https://github.com/ceph/ceph/pull/52705>`_, Nizamudeen A)
+* mgr/dashboard: allow tls 1.2 with a config option (`pr#53780 <https://github.com/ceph/ceph/pull/53780>`_, Nizamudeen A)
+* mgr/dashboard: Block Ui fails in angular with target es2022 (`pr#54260 <https://github.com/ceph/ceph/pull/54260>`_, Aashish Sharma)
+* mgr/dashboard: cephfs volume and subvolume management (`pr#53017 <https://github.com/ceph/ceph/pull/53017>`_, Pedro Gonzalez Gomez, Nizamudeen A, Pere Diaz Bou)
+* mgr/dashboard: cephfs volume rm and rename (`pr#53026 <https://github.com/ceph/ceph/pull/53026>`_, avanthakkar)
+* mgr/dashboard: cleanup rbd-mirror process in dashboard e2e (`pr#53220 <https://github.com/ceph/ceph/pull/53220>`_, Nizamudeen A)
+* mgr/dashboard: cluster upgrade management (batch backport) (`pr#53016 <https://github.com/ceph/ceph/pull/53016>`_, avanthakkar, Nizamudeen A)
+* mgr/dashboard: Dashboard RGW multisite configuration (`pr#52922 <https://github.com/ceph/ceph/pull/52922>`_, Aashish Sharma, Pedro Gonzalez Gomez, Avan Thakkar, avanthakkar)
+* mgr/dashboard: disable hosts field while editing the filesystem (`pr#54069 <https://github.com/ceph/ceph/pull/54069>`_, Nizamudeen A)
+* mgr/dashboard: disable promote on mirroring not enabled (`pr#52536 <https://github.com/ceph/ceph/pull/52536>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: disable protect if layering is not enabled on the image (`pr#53173 <https://github.com/ceph/ceph/pull/53173>`_, avanthakkar)
+* mgr/dashboard: display the groups in cephfs subvolume tab (`pr#53394 <https://github.com/ceph/ceph/pull/53394>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: empty grafana panels for performance of daemons (`pr#52774 <https://github.com/ceph/ceph/pull/52774>`_, Avan Thakkar, avanthakkar)
+* mgr/dashboard: enable protect option if layering enabled (`pr#53795 <https://github.com/ceph/ceph/pull/53795>`_, avanthakkar)
+* mgr/dashboard: fix cephfs create form validator (`pr#53219 <https://github.com/ceph/ceph/pull/53219>`_, Nizamudeen A)
+* mgr/dashboard: fix cephfs form validator (`pr#53778 <https://github.com/ceph/ceph/pull/53778>`_, Nizamudeen A)
+* mgr/dashboard: fix cephfs forms validations (`pr#53831 <https://github.com/ceph/ceph/pull/53831>`_, Nizamudeen A)
+* mgr/dashboard: fix image columns naming (`pr#53254 <https://github.com/ceph/ceph/pull/53254>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: fix progress bar color visibility (`pr#53209 <https://github.com/ceph/ceph/pull/53209>`_, Nizamudeen A)
+* mgr/dashboard: fix prometheus queries subscriptions (`pr#53669 <https://github.com/ceph/ceph/pull/53669>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: fix rgw multi-site import form helper (`pr#54395 <https://github.com/ceph/ceph/pull/54395>`_, Aashish Sharma)
+* mgr/dashboard: fix rgw multisite error when no rgw entity is present (`pr#54261 <https://github.com/ceph/ceph/pull/54261>`_, Aashish Sharma)
+* mgr/dashboard: fix rgw page issues when hostname not resolvable (`pr#53214 <https://github.com/ceph/ceph/pull/53214>`_, Nizamudeen A)
+* mgr/dashboard: fix rgw port manipulation error in dashboard (`pr#53392 <https://github.com/ceph/ceph/pull/53392>`_, Nizamudeen A)
+* mgr/dashboard: fix the landing page layout issues (`issue#62961 <http://tracker.ceph.com/issues/62961>`_, `pr#53835 <https://github.com/ceph/ceph/pull/53835>`_, Nizamudeen A)
+* mgr/dashboard: Fix user/bucket count in rgw overview dashboard (`pr#53818 <https://github.com/ceph/ceph/pull/53818>`_, Aashish Sharma)
+* mgr/dashboard: fixed edit user quota form error (`pr#54223 <https://github.com/ceph/ceph/pull/54223>`_, Ivo Almeida)
+* mgr/dashboard: images -> edit -> disable checkboxes for layering and deef-flatten (`pr#53388 <https://github.com/ceph/ceph/pull/53388>`_, avanthakkar)
+* mgr/dashboard: minor usability improvements (`pr#53143 <https://github.com/ceph/ceph/pull/53143>`_, cloudbehl)
+* mgr/dashboard: n/a entries behind primary snapshot mode (`pr#53223 <https://github.com/ceph/ceph/pull/53223>`_, Pere Diaz Bou)
+* mgr/dashboard: Object gateway inventory card incorrect Buckets and user count (`pr#53382 <https://github.com/ceph/ceph/pull/53382>`_, Aashish Sharma)
+* mgr/dashboard: Object gateway sync status cards keeps loading when multisite is not configured (`pr#53381 <https://github.com/ceph/ceph/pull/53381>`_, Aashish Sharma)
+* mgr/dashboard: paginate hosts (`pr#52918 <https://github.com/ceph/ceph/pull/52918>`_, Pere Diaz Bou)
+* mgr/dashboard: rbd image hide usage bar when disk usage is not provided (`pr#53810 <https://github.com/ceph/ceph/pull/53810>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: remove empty popover when there are no health warns (`pr#53652 <https://github.com/ceph/ceph/pull/53652>`_, Nizamudeen A)
+* mgr/dashboard: remove green tick on old password field (`pr#53386 <https://github.com/ceph/ceph/pull/53386>`_, Nizamudeen A)
+* mgr/dashboard: remove unnecessary failing hosts e2e (`pr#53458 <https://github.com/ceph/ceph/pull/53458>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: remove used and total used columns in favor of usage bar (`pr#53304 <https://github.com/ceph/ceph/pull/53304>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: replace sync progress bar with last synced timestamp in rgw multisite sync status card (`pr#53379 <https://github.com/ceph/ceph/pull/53379>`_, Aashish Sharma)
+* mgr/dashboard: RGW Details card cleanup (`pr#53020 <https://github.com/ceph/ceph/pull/53020>`_, Nizamudeen A, cloudbehl)
+* mgr/dashboard: Rgw Multi-site naming improvements (`pr#53806 <https://github.com/ceph/ceph/pull/53806>`_, Aashish Sharma)
+* mgr/dashboard: rgw multisite topology view shows blank table for multisite entities (`pr#53380 <https://github.com/ceph/ceph/pull/53380>`_, Aashish Sharma)
+* mgr/dashboard: set CORS header for unauthorized access (`pr#53201 <https://github.com/ceph/ceph/pull/53201>`_, Nizamudeen A)
+* mgr/dashboard: show a message to restart the rgw daemons after moving from single-site to multi-site (`pr#53805 <https://github.com/ceph/ceph/pull/53805>`_, Aashish Sharma)
+* mgr/dashboard: subvolume rm with snapshots (`pr#53233 <https://github.com/ceph/ceph/pull/53233>`_, Pedro Gonzalez Gomez)
+* mgr/dashboard: update rgw multisite import form helper info (`pr#54253 <https://github.com/ceph/ceph/pull/54253>`_, Aashish Sharma)
+* mgr/dashboard: upgrade angular v14 and v15 (`pr#52662 <https://github.com/ceph/ceph/pull/52662>`_, Nizamudeen A)
+* mgr/rbd_support: fix recursive locking on CreateSnapshotRequests lock (`pr#54289 <https://github.com/ceph/ceph/pull/54289>`_, Ramana Raja)
+* mgr/snap_schedule: allow retention spec 'n' to be user defined (`pr#52748 <https://github.com/ceph/ceph/pull/52748>`_, Milind Changire, Jakob Haufe)
+* mgr/snap_schedule: make fs argument mandatory if more than one filesystem exists (`pr#54094 <https://github.com/ceph/ceph/pull/54094>`_, Milind Changire)
+* mgr/volumes: Fix pending_subvolume_deletions in volume info (`pr#53572 <https://github.com/ceph/ceph/pull/53572>`_, Kotresh HR)
+* mgr: register OSDs in ms_handle_accept (`pr#53187 <https://github.com/ceph/ceph/pull/53187>`_, Patrick Donnelly)
+* mon, qa: issue pool application warning even if pool is empty (`pr#53041 <https://github.com/ceph/ceph/pull/53041>`_, Prashant D)
+* mon/ConfigMonitor: update crush_location from osd entity (`pr#52466 <https://github.com/ceph/ceph/pull/52466>`_, Didier Gazen)
+* mon/MDSMonitor: plug paxos when maybe manipulating osdmap (`pr#52246 <https://github.com/ceph/ceph/pull/52246>`_, Patrick Donnelly)
+* mon/MonClient: resurrect original client_mount_timeout handling (`pr#52535 <https://github.com/ceph/ceph/pull/52535>`_, Ilya Dryomov)
+* mon/OSDMonitor: do not propose on error in prepare_update (`pr#53186 <https://github.com/ceph/ceph/pull/53186>`_, Patrick Donnelly)
+* mon: fix iterator mishandling in PGMap::apply_incremental (`pr#52554 <https://github.com/ceph/ceph/pull/52554>`_, Oliver Schmidt)
+* msgr: AsyncMessenger add faulted connections metrics (`pr#53033 <https://github.com/ceph/ceph/pull/53033>`_, Pere Diaz Bou)
+* os/bluestore: don't require bluestore_db_block_size when attaching new (`pr#52942 <https://github.com/ceph/ceph/pull/52942>`_, Igor Fedotov)
+* os/bluestore: get rid off resulting lba alignment in allocators (`pr#54772 <https://github.com/ceph/ceph/pull/54772>`_, Igor Fedotov)
+* osd/OpRequest: Add detail description for delayed op in osd log file (`pr#53688 <https://github.com/ceph/ceph/pull/53688>`_, Yite Gu)
+* osd/OSDMap: Check for uneven weights & != 2 buckets post stretch mode (`pr#52457 <https://github.com/ceph/ceph/pull/52457>`_, Kamoltat)
+* osd/scheduler/mClockScheduler: Use same profile and client ids for all clients to ensure allocated QoS limit consumption (`pr#53093 <https://github.com/ceph/ceph/pull/53093>`_, Sridhar Seshasayee)
+* osd: fix logic in check_pg_upmaps (`pr#54276 <https://github.com/ceph/ceph/pull/54276>`_, Laura Flores)
+* osd: fix read balancer logic to avoid redundant primary assignment (`pr#53820 <https://github.com/ceph/ceph/pull/53820>`_, Laura Flores)
+* osd: fix use-after-move in build_incremental_map_msg() (`pr#54267 <https://github.com/ceph/ceph/pull/54267>`_, Ronen Friedman)
+* osd: fix: slow scheduling when item_cost is large (`pr#53861 <https://github.com/ceph/ceph/pull/53861>`_, Jrchyang Yu)
+* Overview graph improvements (`pr#53090 <https://github.com/ceph/ceph/pull/53090>`_, cloudbehl)
+* pybind/mgr/devicehealth: do not crash if db not ready (`pr#52213 <https://github.com/ceph/ceph/pull/52213>`_, Patrick Donnelly)
+* pybind/mgr/pg_autoscaler: Cut back osdmap.get_pools calls (`pr#52767 <https://github.com/ceph/ceph/pull/52767>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: fix warn when not too few pgs (`pr#53674 <https://github.com/ceph/ceph/pull/53674>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: noautoscale flag retains individual pool configs (`pr#53658 <https://github.com/ceph/ceph/pull/53658>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: Reorderd if statement for the func: _maybe_adjust (`pr#53429 <https://github.com/ceph/ceph/pull/53429>`_, Kamoltat)
+* pybind/mgr/pg_autoscaler: Use bytes_used for actual_raw_used (`pr#53534 <https://github.com/ceph/ceph/pull/53534>`_, Kamoltat)
+* pybind/mgr/volumes: log mutex locks to help debug deadlocks (`pr#53918 <https://github.com/ceph/ceph/pull/53918>`_, Kotresh HR)
+* pybind/mgr: reopen database handle on blocklist (`pr#52460 <https://github.com/ceph/ceph/pull/52460>`_, Patrick Donnelly)
+* pybind/rbd: don't produce info on errors in aio_mirror_image_get_info() (`pr#54055 <https://github.com/ceph/ceph/pull/54055>`_, Ilya Dryomov)
+* python-common/drive_group: handle fields outside of 'spec' even when 'spec' is provided (`pr#53115 <https://github.com/ceph/ceph/pull/53115>`_, Adam King)
+* python-common/drive_selection: lower log level of limit policy message (`pr#53114 <https://github.com/ceph/ceph/pull/53114>`_, Adam King)
+* python-common: drive_selection: fix KeyError when osdspec_affinity is not set (`pr#53159 <https://github.com/ceph/ceph/pull/53159>`_, Guillaume Abrioux)
+* qa/cephfs: fix build failure for mdtest project (`pr#53827 <https://github.com/ceph/ceph/pull/53827>`_, Rishabh Dave)
+* qa/cephfs: fix ior project build failure (`pr#53825 <https://github.com/ceph/ceph/pull/53825>`_, Rishabh Dave)
+* qa/cephfs: switch to python3 for centos stream 9 (`pr#53624 <https://github.com/ceph/ceph/pull/53624>`_, Xiubo Li)
+* qa/rgw: add new POOL_APP_NOT_ENABLED failures to log-ignorelist (`pr#53896 <https://github.com/ceph/ceph/pull/53896>`_, Casey Bodley)
+* qa/smoke,orch,perf-basic: add POOL_APP_NOT_ENABLED to ignorelist (`pr#54376 <https://github.com/ceph/ceph/pull/54376>`_, Prashant D)
+* qa/standalone/osd/divergent-prior.sh: Divergent test 3 with pg_autoscale_mode on pick divergent osd (`pr#52721 <https://github.com/ceph/ceph/pull/52721>`_, Nitzan Mordechai)
+* qa/suites/crimson-rados: add centos9 to supported distros (`pr#54020 <https://github.com/ceph/ceph/pull/54020>`_, Matan Breizman)
+* qa/suites/crimson-rados: bring backfill testing (`pr#54021 <https://github.com/ceph/ceph/pull/54021>`_, Radoslaw Zarzynski, Matan Breizman)
+* qa/suites/crimson-rados: Use centos8 for testing (`pr#54019 <https://github.com/ceph/ceph/pull/54019>`_, Matan Breizman)
+* qa/suites/krbd: stress test for recovering from watch errors (`pr#53786 <https://github.com/ceph/ceph/pull/53786>`_, Ilya Dryomov)
+* qa/suites/rbd: add test to check rbd_support module recovery (`pr#54291 <https://github.com/ceph/ceph/pull/54291>`_, Ramana Raja)
+* qa/suites/rbd: drop cache tiering workload tests (`pr#53996 <https://github.com/ceph/ceph/pull/53996>`_, Ilya Dryomov)
+* qa/suites/upgrade: enable default RBD image features (`pr#53352 <https://github.com/ceph/ceph/pull/53352>`_, Ilya Dryomov)
+* qa/suites/upgrade: fix env indentation in stress-split upgrade tests (`pr#53921 <https://github.com/ceph/ceph/pull/53921>`_, Laura Flores)
+* qa/suites/{rbd,krbd}: disable POOL_APP_NOT_ENABLED health check (`pr#53599 <https://github.com/ceph/ceph/pull/53599>`_, Ilya Dryomov)
+* qa/tests: added - \(POOL_APP_NOT_ENABLED\) to the ignore list (`pr#54436 <https://github.com/ceph/ceph/pull/54436>`_, Yuri Weinstein)
+* qa: add POOL_APP_NOT_ENABLED to ignorelist for cephfs tests (`issue#62482 <http://tracker.ceph.com/issues/62482>`_, `issue#62508 <http://tracker.ceph.com/issues/62508>`_, `pr#54380 <https://github.com/ceph/ceph/pull/54380>`_, Venky Shankar, Patrick Donnelly)
+* qa: assign file system affinity for replaced MDS (`issue#61764 <http://tracker.ceph.com/issues/61764>`_, `pr#54037 <https://github.com/ceph/ceph/pull/54037>`_, Venky Shankar)
+* qa: descrease pgbench scale factor to 32 for postgresql database test (`pr#53627 <https://github.com/ceph/ceph/pull/53627>`_, Xiubo Li)
+* qa: fix cephfs-mirror unwinding and 'fs volume create/rm' order (`pr#52656 <https://github.com/ceph/ceph/pull/52656>`_, Jos Collin)
+* qa: fix keystone in rgw/crypt/barbican.yaml (`pr#53412 <https://github.com/ceph/ceph/pull/53412>`_, Ali Maredia)
+* qa: ignore expected cluster warning from damage tests (`pr#53484 <https://github.com/ceph/ceph/pull/53484>`_, Patrick Donnelly)
+* qa: lengthen shutdown timeout for thrashed MDS (`pr#53553 <https://github.com/ceph/ceph/pull/53553>`_, Patrick Donnelly)
+* qa: move nfs (mgr/nfs) related tests to fs suite (`pr#53906 <https://github.com/ceph/ceph/pull/53906>`_, Dhairya Parmar, Venky Shankar)
+* qa: wait for file to have correct size (`pr#52742 <https://github.com/ceph/ceph/pull/52742>`_, Patrick Donnelly)
+* qa: wait for MDSMonitor tick to replace daemons (`pr#52235 <https://github.com/ceph/ceph/pull/52235>`_, Patrick Donnelly)
+* RadosGW API: incorrect bucket quota in response to HEAD /{bucket}/?usage (`pr#53437 <https://github.com/ceph/ceph/pull/53437>`_, shreyanshjain7174)
+* rbd-mirror: fix image replayer shut down description on force promote (`pr#52880 <https://github.com/ceph/ceph/pull/52880>`_, Prasanna Kumar Kalever)
+* rbd-mirror: fix race preventing local image deletion (`pr#52627 <https://github.com/ceph/ceph/pull/52627>`_, N Balachandran)
+* rbd-nbd: fix stuck with disable request (`pr#54254 <https://github.com/ceph/ceph/pull/54254>`_, Prasanna Kumar Kalever)
+* read balancer documentation (`pr#52777 <https://github.com/ceph/ceph/pull/52777>`_, Laura Flores)
+* Rgw overview dashboard backport (`pr#53065 <https://github.com/ceph/ceph/pull/53065>`_, Aashish Sharma)
+* rgw/amqp: remove possible race conditions with the amqp connections (`pr#53516 <https://github.com/ceph/ceph/pull/53516>`_, Yuval Lifshitz)
+* rgw/amqp: skip idleness tests since it needs to sleep longer than 30s (`pr#53506 <https://github.com/ceph/ceph/pull/53506>`_, Yuval Lifshitz)
+* rgw/crypt: apply rgw_crypt_default_encryption_key by default (`pr#52796 <https://github.com/ceph/ceph/pull/52796>`_, Casey Bodley)
+* rgw/crypt: don't deref null manifest_bl (`pr#53590 <https://github.com/ceph/ceph/pull/53590>`_, Casey Bodley)
+* rgw/kafka: failed to reconnect to broker after idle timeout (`pr#53513 <https://github.com/ceph/ceph/pull/53513>`_, Yuval Lifshitz)
+* rgw/kafka: make sure that destroy is called after connection is removed (`pr#53515 <https://github.com/ceph/ceph/pull/53515>`_, Yuval Lifshitz)
+* rgw/keystone: EC2Engine uses reject() for ERR_SIGNATURE_NO_MATCH (`pr#53762 <https://github.com/ceph/ceph/pull/53762>`_, Casey Bodley)
+* rgw/multisite[archive zone]: fix storing of bucket instance info in the new bucket entrypoint (`pr#53466 <https://github.com/ceph/ceph/pull/53466>`_, Shilpa Jagannath)
+* rgw/notification: pass in bytes_transferred to populate object_size in sync notification (`pr#53377 <https://github.com/ceph/ceph/pull/53377>`_, Juan Zhu)
+* rgw/notification: remove non x-amz-meta-\* attributes from bucket notifications (`pr#53375 <https://github.com/ceph/ceph/pull/53375>`_, Juan Zhu)
+* rgw/notifications: allow cross tenant notification management (`pr#53510 <https://github.com/ceph/ceph/pull/53510>`_, Yuval Lifshitz)
+* rgw/s3: ListObjectsV2 returns correct object owners (`pr#54161 <https://github.com/ceph/ceph/pull/54161>`_, Casey Bodley)
+* rgw/s3select: fix per QE defect (`pr#54163 <https://github.com/ceph/ceph/pull/54163>`_, galsalomon66)
+* rgw/s3select: s3select fixes related to Trino/TPCDS benchmark and QE tests (`pr#53034 <https://github.com/ceph/ceph/pull/53034>`_, galsalomon66)
+* rgw/sal: get_placement_target_names() returns void (`pr#53584 <https://github.com/ceph/ceph/pull/53584>`_, Casey Bodley)
+* rgw/sync-policy: Correct "sync status" & "sync group" commands (`pr#53395 <https://github.com/ceph/ceph/pull/53395>`_, Soumya Koduri)
+* rgw/upgrade: point upgrade suites to ragweed ceph-reef branch (`pr#53797 <https://github.com/ceph/ceph/pull/53797>`_, Shilpa Jagannath)
+* RGW: add admin interfaces to get and delete notifications by bucket (`pr#53509 <https://github.com/ceph/ceph/pull/53509>`_, Ali Masarwa)
+* rgw: add radosgw-admin bucket check olh/unlinked commands (`pr#53823 <https://github.com/ceph/ceph/pull/53823>`_, Cory Snyder)
+* rgw: add versioning info to radosgw-admin bucket stats output (`pr#54191 <https://github.com/ceph/ceph/pull/54191>`_, Cory Snyder)
+* RGW: bucket notification - hide auto generated topics when listing topics (`pr#53507 <https://github.com/ceph/ceph/pull/53507>`_, Ali Masarwa)
+* rgw: don't dereference nullopt in DeleteMultiObj (`pr#54124 <https://github.com/ceph/ceph/pull/54124>`_, Casey Bodley)
+* rgw: fetch_remote_obj() preserves original part lengths for BlockDecrypt (`pr#52816 <https://github.com/ceph/ceph/pull/52816>`_, Casey Bodley)
+* rgw: fetch_remote_obj() uses uncompressed size for encrypted objects (`pr#54371 <https://github.com/ceph/ceph/pull/54371>`_, Casey Bodley)
+* rgw: fix 2 null versionID after convert_plain_entry_to_versioned (`pr#53398 <https://github.com/ceph/ceph/pull/53398>`_, rui ma, zhuo li)
+* rgw: fix multipart upload object leaks due to re-upload (`pr#52615 <https://github.com/ceph/ceph/pull/52615>`_, J. Eric Ivancich)
+* rgw: fix rgw rate limiting RGWRateLimitInfo class decode_json max_rea… (`pr#53765 <https://github.com/ceph/ceph/pull/53765>`_, xiangrui meng)
+* rgw: fix SignatureDoesNotMatch when extra headers start with 'x-amz' (`pr#53770 <https://github.com/ceph/ceph/pull/53770>`_, rui ma)
+* rgw: fix unwatch crash at radosgw startup (`pr#53760 <https://github.com/ceph/ceph/pull/53760>`_, lichaochao)
+* rgw: handle http options CORS with v4 auth (`pr#53413 <https://github.com/ceph/ceph/pull/53413>`_, Tobias Urdin)
+* rgw: improve buffer list utilization in the chunkupload scenario (`pr#53773 <https://github.com/ceph/ceph/pull/53773>`_, liubingrun)
+* rgw: pick http_date in case of http_x_amz_date absence (`pr#53440 <https://github.com/ceph/ceph/pull/53440>`_, Seena Fallah, Mohamed Awnallah)
+* rgw: retry metadata cache notifications with INVALIDATE_OBJ (`pr#52798 <https://github.com/ceph/ceph/pull/52798>`_, Casey Bodley)
+* rgw: s3 object lock avoids overflow in retention date (`pr#52604 <https://github.com/ceph/ceph/pull/52604>`_, Casey Bodley)
+* rgw: s3website doesn't prefetch for web_dir() check (`pr#53767 <https://github.com/ceph/ceph/pull/53767>`_, Casey Bodley)
+* RGW: Solving the issue of not populating etag in Multipart upload result (`pr#51447 <https://github.com/ceph/ceph/pull/51447>`_, Ali Masarwa)
+* RGW:notifications: persistent topics are not deleted via radosgw-admin (`pr#53514 <https://github.com/ceph/ceph/pull/53514>`_, Ali Masarwa)
+* src/mon/Monitor: Fix set_elector_disallowed_leaders (`pr#54003 <https://github.com/ceph/ceph/pull/54003>`_, Kamoltat)
+* test/crimson/seastore/rbm: add sub-tests regarding RBM to the existing tests (`pr#53967 <https://github.com/ceph/ceph/pull/53967>`_, Myoungwon Oh)
+* test/TestOSDMap: don't use the deprecated std::random_shuffle method (`pr#52737 <https://github.com/ceph/ceph/pull/52737>`_, Leonid Usov)
+* valgrind: UninitCondition under __run_exit_handlers suppression (`pr#53681 <https://github.com/ceph/ceph/pull/53681>`_, Mark Kogan)
+* xfstests_dev: install extra packages from powertools repo for xfsprogs (`pr#52843 <https://github.com/ceph/ceph/pull/52843>`_, Xiubo Li)
+
 v18.2.0 Reef
 ============
 
@@ -63,13 +556,13 @@ Dashboard
 
 * Cephx Auth Management: There is a new section dedicated to listing and
   managing Ceph cluster users.
-  
+
 * RGW Server Side Encryption: The SSE-S3 and KMS encryption of rgw buckets can
   now be configured at the time of bucket creation.
 
 * RBD Snapshot mirroring: Snapshot mirroring can now be configured through UI.
   Snapshots can now be scheduled.
-  
+
 * 1-Click OSD Creation Wizard: OSD creation has been broken into 3 options:
 
   #. Cost/Capacity Optimized: Use all HDDs
@@ -112,7 +605,7 @@ MGR
 * The ``ceph mgr dump`` command now outputs ``last_failure_osd_epoch`` and
   ``active_clients`` fields at the top level. Previously, these fields were
   output under the ``always_on_modules`` field.
-  
+
 RADOS
 ~~~~~
 
@@ -145,10 +638,10 @@ RADOS
     recovery I/O. Users can then choose either the ``high_client_ops`` profile
     to prioritize client I/O or the ``high_recovery_ops`` profile to prioritize
     recovery I/O.
-  * QoS parameters including ``reservation`` and ``limit`` are now specified in 
+  * QoS parameters including ``reservation`` and ``limit`` are now specified in
     terms of a fraction (range: 0.0 to 1.0) of the OSD's IOPS capacity.
   * The cost parameters (``osd_mclock_cost_per_io_usec_*`` and
-    ``osd_mclock_cost_per_byte_usec_*``) have been removed. The cost of an 
+    ``osd_mclock_cost_per_byte_usec_*``) have been removed. The cost of an
     operation is now a function of the random IOPS and maximum sequential
     bandwidth capability of the OSD's underlying device.
   * Degraded object recovery is given higher priority than misplaced
@@ -157,7 +650,7 @@ RADOS
     backfilling operations with the ``balanced`` and ``high_client_ops`` mClock
     profiles might progress more slowly than in the past, when backfilling
     operations used the 'WeightedPriorityQueue' (WPQ) scheduler.
-  * The QoS allocations in all the mClock profiles are optimized in 
+  * The QoS allocations in all the mClock profiles are optimized in
     accordance with the above fixes and enhancements.
   * For more details, see:
     https://docs.ceph.com/en/reef/rados/configuration/mclock-config-ref/
@@ -172,7 +665,7 @@ RADOS
   has a ``name`` field that shows the name of the manager module that
   registered a RADOS client. Previously, the ``active_clients`` array showed
   the address of a module's RADOS client, but not the name of the module.
-* The ``perf dump`` and ``perf schema`` commands have been deprecated in 
+* The ``perf dump`` and ``perf schema`` commands have been deprecated in
   favor of the new ``counter dump`` and ``counter schema`` commands. These new
   commands add support for labeled perf counters and also emit existing
   unlabeled perf counters. Some unlabeled perf counters became labeled in this
@@ -243,7 +736,7 @@ RGW
   without enclosing quotation marks so that the fields can be decoded as
   boolean values in JSON. The same is true of the ``is_truncated`` field
   returned by ``/subscriptions/<sub-name>``.
-* RGW's response of ``Action=GetTopicAttributes&TopicArn=<topic-arn>`` REST 
+* RGW's response of ``Action=GetTopicAttributes&TopicArn=<topic-arn>`` REST
   API now returns ``HasStoredSecret`` and ``Persistent`` as boolean in the JSON
   string that is encoded in ``Attributes/EndPoint``.
 * All boolean fields that were previously rendered as strings by the
@@ -285,13 +778,13 @@ RGW
   has been removed. As a result, the pubsub zone should not be used anymore.
   The following have also been removed: the REST operations, ``radosgw-admin``
   commands for manipulating subscriptions, fetching the notifications, and
-  acking the notifications. 
+  acking the notifications.
 
   If the endpoint to which the notifications are sent is down or disconnected,
   we recommend that you use persistent notifications to guarantee their
   delivery. If the system that consumes the notifications has to pull them
   (instead of the notifications being pushed to the system), use an external
-  message bus (for example, RabbitMQ or Kafka) for that purpose. 
+  message bus (for example, RabbitMQ or Kafka) for that purpose.
 * The serialized format of notification and topics has changed. This means
   that new and updated topics will be unreadable by old RGWs. We recommend
   completing the RGW upgrades before creating or modifying any notification
@@ -303,7 +796,7 @@ RGW
   `compress-encrypted` zonegroup feature: see
   https://docs.ceph.com/en/reef/radosgw/multisite/#zone-features and note the
   security considerations.
-  
+
 Telemetry
 ~~~~~~~~~
 
@@ -343,7 +836,7 @@ The same process is used to upgrade to future minor releases.
 Upgrade progress can be monitored with
 
   .. prompt:: bash #
-    
+
     ceph orch upgrade status
 
 Upgrade progress can also be monitored with `ceph -s` (which provides a simple progress bar) or more verbosely with
@@ -437,7 +930,7 @@ Upgrading non-cephadm clusters
 #. Upgrade all OSDs by installing the new packages and restarting the ceph-osd daemons on all OSD hosts
 
    .. prompt:: bash #
-   
+
       systemctl restart ceph-osd.target
 
 #. Upgrade all CephFS MDS daemons. For each CephFS file system,
@@ -445,19 +938,19 @@ Upgrading non-cephadm clusters
    #. Disable standby_replay:
 
          .. prompt:: bash #
-         
+
             ceph fs set <fs_name> allow_standby_replay false
 
    #. If upgrading from Pacific <=16.2.5:
 
          .. prompt:: bash #
-   
+
             ceph config set mon mon_mds_skip_sanity true
 
    #. Reduce the number of ranks to 1. (Make note of the original number of MDS daemons first if you plan to restore it later.)
-   
-      .. prompt:: bash #   
-   
+
+      .. prompt:: bash #
+
          ceph status # ceph fs set <fs_name> max_mds 1
 
    #. Wait for the cluster to deactivate any non-zero ranks by periodically checking the status
@@ -469,7 +962,7 @@ Upgrading non-cephadm clusters
    #. Take all standby MDS daemons offline on the appropriate hosts with
 
       .. prompt:: bash #
-   
+
          systemctl stop ceph-mds@<daemon_name>
 
    #. Confirm that only one MDS is online and is rank 0 for your FS
@@ -479,7 +972,7 @@ Upgrading non-cephadm clusters
          ceph status
 
    #. Upgrade the last remaining MDS daemon by installing the new packages and restarting the daemon
-      
+
       .. prompt:: bash #
 
          systemctl restart ceph-mds.target
@@ -509,9 +1002,9 @@ Upgrading non-cephadm clusters
       systemctl restart ceph-radosgw.target
 
 #. Complete the upgrade by disallowing pre-Reef OSDs and enabling all new Reef-only functionality
-   
+
    .. prompt:: bash #
-   
+
       ceph osd require-osd-release reef
 
 #. If you set `noout` at the beginning, be sure to clear it with
@@ -538,9 +1031,9 @@ Post-upgrade
       ceph telemetry preview-all
 
    If you are comfortable with the data that is reported, you can opt-in to automatically report the high-level cluster metadata with
-   
-   .. prompt:: bash #   
-   
+
+   .. prompt:: bash #
+
       ceph telemetry on
 
    The public dashboard that aggregates Ceph telemetry can be found at https://telemetry-public.ceph.com/.
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
index ff60a3e7b607..2793ae351006 100644
--- a/doc/releases/releases.yml
+++ b/doc/releases/releases.yml
@@ -17,6 +17,8 @@ releases:
     releases:
       - version: 18.2.0
         released: 2023-08-07
+      - version: 18.2.1
+        released: 2023-12-18
   quincy:
     target_eol: 2024-06-01
     releases:

From b8bd69df337491524f324be3aabdb870b29b6ade Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 13 Dec 2023 22:41:00 -0800
Subject: [PATCH 1087/2492] crimson/.../client_request_common: simplify
 recover_missings

- Modify ClientRequest::snaps_need_to_recover() to return all
  relevant snaps including the operation target.
- Update ClientRequest::snaps_need_to_recover() and
  CommonClientRequest::recover_missings to use set::set
  for snaps.
- Remove special handling for soid from
  CommonClientRequest::recover_missings
- Simplify CommonClientRequest::recover_missings.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 .../osd/osd_operations/client_request.h       | 10 +++-
 .../osd_operations/client_request_common.cc   | 59 ++++++++-----------
 .../osd_operations/client_request_common.h    |  2 +-
 3 files changed, 34 insertions(+), 37 deletions(-)

diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 43535e156c5b..27d0c0ef65dd 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -160,11 +160,15 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
   }
   auto get_instance_handle() { return instance_handle; }
 
-  std::vector<snapid_t> snaps_need_to_recover() {
-    std::vector<snapid_t> ret;
+  std::set<snapid_t> snaps_need_to_recover() {
+    std::set<snapid_t> ret;
+    auto target = m->get_hobj();
+    if (!target.is_head()) {
+      ret.insert(target.snap);
+    }
     for (auto &op : m->ops) {
       if (op.op.op == CEPH_OSD_OP_ROLLBACK) {
-	ret.emplace_back((snapid_t)op.op.snap.snapid);
+	ret.insert((snapid_t)op.op.snap.snapid);
       }
     }
     return ret;
diff --git a/src/crimson/osd/osd_operations/client_request_common.cc b/src/crimson/osd/osd_operations/client_request_common.cc
index 903da59a3dce..c406a6c8330b 100644
--- a/src/crimson/osd/osd_operations/client_request_common.cc
+++ b/src/crimson/osd/osd_operations/client_request_common.cc
@@ -19,44 +19,37 @@ InterruptibleOperation::template interruptible_future<>
 CommonClientRequest::recover_missings(
   Ref<PG> &pg,
   const hobject_t& soid,
-  std::vector<snapid_t> &&snaps)
+  std::set<snapid_t> &&snaps)
 {
-  using interruptor = InterruptibleOperation::interruptor;
   LOG_PREFIX(CommonClientRequest::recover_missings);
-  auto fut = interruptor::now();
   if (!pg->is_primary()) {
-    DEBUGI("process_op: Skipping do_recover_missing on non primary pg");
-    return fut;
-  }
-  if (!soid.is_head()) {
-    fut = do_recover_missing(pg, soid.get_head());
+    DEBUGDPP(
+      "Skipping recover_missings on non primary pg for soid {}", *pg, soid);
+    return seastar::now();
   }
-  return seastar::do_with(
-    std::move(snaps),
-    [pg, soid, fut=std::move(fut)](auto &snaps) mutable {
-    return fut.then_interruptible([&snaps, pg, soid]() mutable {
-      return pg->obc_loader.with_obc<RWState::RWREAD>(
-        soid.get_head(),
-        [&snaps, pg, soid](auto head, auto) mutable {
-        auto oid = resolve_oid(head->get_head_ss(), soid);
-        assert(oid);
-        return do_recover_missing(pg, *oid
-        ).then_interruptible([&snaps, pg, soid, head]() mutable {
-          return InterruptibleOperation::interruptor::do_for_each(
-            snaps,
-            [pg, soid, head](auto &snap) mutable {
-            auto coid = head->obs.oi.soid;
-            coid.snap = snap;
-            auto oid = resolve_oid(head->get_head_ss(), coid);
-            assert(oid);
-            return do_recover_missing(pg, *oid);
-          });
-        });
+  return do_recover_missing(
+    pg, soid.get_head()
+  ).then_interruptible([snaps=std::move(snaps), pg, soid]() mutable {
+    return pg->obc_loader.with_obc<RWState::RWREAD>(
+      soid.get_head(),
+      [snaps=std::move(snaps), pg, soid](auto head, auto) mutable {
+      return seastar::do_with(
+	std::move(snaps),
+	[pg, soid, head](auto &snaps) mutable {
+	return InterruptibleOperation::interruptor::do_for_each(
+	  snaps,
+	  [pg, soid, head](auto &snap) mutable {
+	  auto coid = head->obs.oi.soid;
+	  coid.snap = snap;
+	  auto oid = resolve_oid(head->get_head_ss(), coid);
+	  assert(oid);
+	  return do_recover_missing(pg, *oid);
+	});
       });
-    }).handle_error_interruptible(
-      crimson::ct_error::assert_all("unexpected error")
-    );
-  });
+    });
+  }).handle_error_interruptible(
+    crimson::ct_error::assert_all("unexpected error")
+  );
 }
 
 typename InterruptibleOperation::template interruptible_future<>
diff --git a/src/crimson/osd/osd_operations/client_request_common.h b/src/crimson/osd/osd_operations/client_request_common.h
index 46aa038e3432..98ee1ef123bd 100644
--- a/src/crimson/osd/osd_operations/client_request_common.h
+++ b/src/crimson/osd/osd_operations/client_request_common.h
@@ -15,7 +15,7 @@ struct CommonClientRequest {
   recover_missings(
     Ref<PG> &pg,
     const hobject_t& soid,
-    std::vector<snapid_t> &&snaps);
+    std::set<snapid_t> &&snaps);
 
   static InterruptibleOperation::template interruptible_future<>
   do_recover_missing(Ref<PG>& pg, const hobject_t& soid);

From decd662b8c1c41ff194bdefb611eee90a060cfae Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 16 Dec 2023 00:26:30 +0000
Subject: [PATCH 1088/2492] crimson/.../client_request_common: skip
 non-existent oids in recover_missings

See comment.

Introduced: 38cc750f
Fixes: https://tracker.ceph.com/issues/63821
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 .../osd/osd_operations/client_request_common.cc  | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/osd_operations/client_request_common.cc b/src/crimson/osd/osd_operations/client_request_common.cc
index c406a6c8330b..fb919a1b07cf 100644
--- a/src/crimson/osd/osd_operations/client_request_common.cc
+++ b/src/crimson/osd/osd_operations/client_request_common.cc
@@ -38,12 +38,22 @@ CommonClientRequest::recover_missings(
 	[pg, soid, head](auto &snaps) mutable {
 	return InterruptibleOperation::interruptor::do_for_each(
 	  snaps,
-	  [pg, soid, head](auto &snap) mutable {
+	  [pg, soid, head](auto &snap) mutable ->
+	  InterruptibleOperation::template interruptible_future<> {
 	  auto coid = head->obs.oi.soid;
 	  coid.snap = snap;
 	  auto oid = resolve_oid(head->get_head_ss(), coid);
-	  assert(oid);
-	  return do_recover_missing(pg, *oid);
+	  /* Rollback targets may legitimately not exist if, for instance,
+	   * the object is an rbd block which happened to be sparse and
+	   * therefore non-existent at the time of the specified snapshot.
+	   * In such a case, rollback will simply delete the object.  Here,
+	   * we skip the oid as there is no corresponding clone to recover.
+	   * See https://tracker.ceph.com/issues/63821 */
+	  if (oid) {
+	    return do_recover_missing(pg, *oid);
+	  } else {
+	    return seastar::now();
+	  }
 	});
       });
     });

From 0ca294ecc52f4f5ad70ed26e1df7f63091a58892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B6=E6=B5=B7=E4=B8=B0?= <769358362@qq.com>
Date: Sat, 16 Dec 2023 17:31:47 +0800
Subject: [PATCH 1089/2492] doc/cephfs/client-auth.rst: correct `This so
 because` to  `This is because`

---
 doc/cephfs/client-auth.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephfs/client-auth.rst b/doc/cephfs/client-auth.rst
index 946211bf682e..75528f91eaeb 100644
--- a/doc/cephfs/client-auth.rst
+++ b/doc/cephfs/client-auth.rst
@@ -314,7 +314,7 @@ Changing rw permissions in caps
 -------------------------------
 
 It's not possible to modify caps by running ``fs authorize`` except for the
-case when read/write permissions have to be changed. This so because the
+case when read/write permissions have to be changed. This is because the
 ``fs authorize`` becomes ambiguous. For example, user runs ``fs authorize
 cephfs1 client.x /dir1 rw`` to create a client and then runs ``fs authorize
 cephfs1 client.x /dir2 rw`` (notice ``/dir1`` is changed to ``/dir2``).

From 76df4d8243a8cc3c39539423e4db16a61601ff41 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 12 Dec 2023 08:11:57 +0000
Subject: [PATCH 1090/2492] build/rgw/lua: return lua-devel runtime dependency

reverting: b0a77a53ab86b9b24a2cfb46049957d40a8c80aa
after sepia lab issue is resolved

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 ceph.spec.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index ac707ac702de..f67c675029c2 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -283,6 +283,7 @@ BuildRequires:  librabbitmq-devel
 BuildRequires:  librdkafka-devel
 %endif
 %if 0%{with lua_packages}
+Requires:  lua-devel
 Requires:  %{luarocks_package_name}
 %endif
 %if 0%{with make_check}

From d57b799ff05709b435af56d29158ceece62e57df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20Gr=C3=BCninger?= <foss@grueninger.de>
Date: Sun, 17 Dec 2023 22:18:43 +0100
Subject: [PATCH 1091/2492] cmake: Do not set CMake policy to new that are set
 anyway
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CMP0097 and below are all implicitly set to new because
the minimum required CMake version is 3.16 and these
policies are older.

Signed-off-by: Christoph Grüninger <foss@grueninger.de>
---
 CMakeLists.txt | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4feb22c135e1..6e810d105895 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,17 +4,6 @@ project(ceph
   VERSION 18.0.0
   LANGUAGES CXX C ASM)
 
-cmake_policy(SET CMP0028 NEW)
-cmake_policy(SET CMP0046 NEW)
-cmake_policy(SET CMP0048 NEW)
-cmake_policy(SET CMP0051 NEW)
-cmake_policy(SET CMP0054 NEW)
-cmake_policy(SET CMP0056 NEW)
-cmake_policy(SET CMP0065 NEW)
-cmake_policy(SET CMP0074 NEW)
-cmake_policy(SET CMP0075 NEW)
-cmake_policy(SET CMP0093 NEW)
-cmake_policy(SET CMP0094 NEW)
 foreach(policy CMP0127 CMP0135)
   if(POLICY ${policy})
     cmake_policy(SET ${policy} NEW)

From ee2ee31d3f72a2ee63f9d28bbaf6045657f43ee6 Mon Sep 17 00:00:00 2001
From: Daniel Clavijo Coca <dclavijo@opennebula.io>
Date: Fri, 15 Dec 2023 09:54:02 -0600
Subject: [PATCH 1092/2492] docs: Add information about OpenNebula integration

- Exclude doc build output from git
- Fix missing doc build dependency
- Also includes some involuntary automatically persistent linting by vscode

Co-authored-by: Ilya Dryomov <idryomov@redhat.com>
Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Co-authored-by: Zac Dover <zac.dover@proton.me>
Signed-off-by: Daniel Clavijo <dclavijo@opennebula.io>
---
 .gitignore                             |  14 ++
 doc/architecture.rst                   | 229 +++++++++++++------------
 doc/install/index.rst                  |   6 +-
 doc/rbd/index.rst                      |   7 +-
 doc/rbd/libvirt.rst                    | 122 ++++++-------
 doc/rbd/rbd-snapshot.rst               |  31 ++--
 doc/start/documenting-ceph.rst         | 205 +++++++++++-----------
 doc/start/hardware-recommendations.rst |  38 ++--
 8 files changed, 337 insertions(+), 315 deletions(-)

diff --git a/.gitignore b/.gitignore
index b01aef839bef..c74ad2efd69b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,3 +83,17 @@ GTAGS
 # Python building things where it shouldn't
 /src/python-common/build/
 .cache
+
+# Doc build output
+src/pybind/cephfs/build/
+src/pybind/cephfs/cephfs.c
+src/pybind/cephfs/cephfs.egg-info/
+src/pybind/rados/build/
+src/pybind/rados/rados.c
+src/pybind/rados/rados.egg-info/
+src/pybind/rbd/build/
+src/pybind/rbd/rbd.c
+src/pybind/rbd/rbd.egg-info/
+src/pybind/rgw/build/
+src/pybind/rgw/rgw.c
+src/pybind/rgw/rgw.egg-info/
diff --git a/doc/architecture.rst b/doc/architecture.rst
index 983cec2300a6..1be58f682925 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -37,7 +37,7 @@ to Ceph clients. Provisioning multiple monitors within the Ceph cluster ensures
 availability in the event that one of the monitor daemons or its host fails.
 The Ceph monitor provides copies of the cluster map to storage cluster clients.
 
-A Ceph OSD Daemon checks its own state and the state of other OSDs and reports 
+A Ceph OSD Daemon checks its own state and the state of other OSDs and reports
 back to monitors.
 
 A Ceph Manager serves as an endpoint for monitoring, orchestration, and plug-in
@@ -61,7 +61,7 @@ comes through a :term:`Ceph Block Device`, :term:`Ceph Object Storage`, the
 ``librados``. The data received by the Ceph Storage Cluster is stored as RADOS
 objects. Each object is stored on an :term:`Object Storage Device` (this is
 also called an "OSD"). Ceph OSDs control read, write, and replication
-operations on storage drives. The default BlueStore back end stores objects 
+operations on storage drives. The default BlueStore back end stores objects
 in a monolithic, database-like fashion.
 
 .. ditaa::
@@ -69,7 +69,7 @@ in a monolithic, database-like fashion.
            /------\       +-----+       +-----+
            | obj  |------>| {d} |------>| {s} |
            \------/       +-----+       +-----+
-   
+
             Object         OSD          Drive
 
 Ceph OSD Daemons store data as objects in a flat namespace. This means that
@@ -85,10 +85,10 @@ created date, and the last modified date.
            /------+------------------------------+----------------\
            | ID   | Binary Data                  | Metadata       |
            +------+------------------------------+----------------+
-           | 1234 | 0101010101010100110101010010 | name1 = value1 | 
+           | 1234 | 0101010101010100110101010010 | name1 = value1 |
            |      | 0101100001010100110101010010 | name2 = value2 |
            |      | 0101100001010100110101010010 | nameN = valueN |
-           \------+------------------------------+----------------/    
+           \------+------------------------------+----------------/
 
 .. note:: An object ID is unique across the entire cluster, not just the local
    filesystem.
@@ -147,14 +147,14 @@ five maps that constitute the cluster map are:
    the address, and the TCP port of each monitor. The monitor map specifies the
    current epoch, the time of the monitor map's creation, and the time of the
    monitor map's last modification.  To view a monitor map, run ``ceph mon
-   dump``.   
-   
+   dump``.
+
 #. **The OSD Map:** Contains the cluster ``fsid``, the time of the OSD map's
    creation, the time of the OSD map's last modification, a list of pools, a
    list of replica sizes, a list of PG numbers, and a list of OSDs and their
    statuses (for example, ``up``, ``in``). To view an OSD map, run ``ceph
-   osd dump``. 
-   
+   osd dump``.
+
 #. **The PG Map:** Contains the PG version, its time stamp, the last OSD map
    epoch, the full ratios, and the details of each placement group. This
    includes the PG ID, the `Up Set`, the `Acting Set`, the state of the PG (for
@@ -168,8 +168,8 @@ five maps that constitute the cluster map are:
    {decomp-crushmap-filename}``. Use a text editor or ``cat`` to view the
    decompiled map.
 
-#. **The MDS Map:** Contains the current MDS map epoch, when the map was 
-   created, and the last time it changed. It also contains the pool for 
+#. **The MDS Map:** Contains the current MDS map epoch, when the map was
+   created, and the last time it changed. It also contains the pool for
    storing metadata, a list of metadata servers, and which metadata servers
    are ``up`` and ``in``. To view an MDS map, execute ``ceph fs dump``.
 
@@ -212,13 +212,13 @@ High Availability Authentication
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The ``cephx`` authentication system is used by Ceph to authenticate users and
-daemons and to protect against man-in-the-middle attacks. 
+daemons and to protect against man-in-the-middle attacks.
 
-.. note:: The ``cephx`` protocol does not address data encryption in transport 
+.. note:: The ``cephx`` protocol does not address data encryption in transport
    (for example, SSL/TLS) or encryption at rest.
 
 ``cephx`` uses shared secret keys for authentication. This means that both the
-client and the monitor cluster keep a copy of the client's secret key. 
+client and the monitor cluster keep a copy of the client's secret key.
 
 The ``cephx`` protocol makes it possible for each party to prove to the other
 that it has a copy of the key without revealing it. This provides mutual
@@ -235,7 +235,7 @@ Direct interactions between Ceph clients and OSDs require authenticated
 connections. The ``cephx`` authentication system establishes and sustains these
 authenticated connections.
 
-The ``cephx`` protocol operates in a manner similar to `Kerberos`_. 
+The ``cephx`` protocol operates in a manner similar to `Kerberos`_.
 
 A user invokes a Ceph client to contact a monitor. Unlike Kerberos, each
 monitor can authenticate users and distribute keys, which means that there is
@@ -248,7 +248,7 @@ Monitors. The client then uses the session key to request services from the
 monitors, and the monitors provide the client with a ticket that authenticates
 the client against the OSDs that actually handle data. Ceph Monitors and OSDs
 share a secret, which means that the clients can use the ticket provided by the
-monitors to authenticate against any OSD or metadata server in the cluster. 
+monitors to authenticate against any OSD or metadata server in the cluster.
 
 Like Kerberos tickets, ``cephx`` tickets expire. An attacker cannot use an
 expired ticket or session key that has been obtained surreptitiously. This form
@@ -264,8 +264,8 @@ subsystem generates the username and key, stores a copy on the monitor(s), and
 transmits the user's secret back to the ``client.admin`` user. This means that
 the client and the monitor share a secret key.
 
-.. note:: The ``client.admin`` user must provide the user ID and 
-   secret key to the user in a secure manner. 
+.. note:: The ``client.admin`` user must provide the user ID and
+   secret key to the user in a secure manner.
 
 .. ditaa::
 
@@ -275,7 +275,7 @@ the client and the monitor share a secret key.
                 |  request to   |
                 | create a user |
                 |-------------->|----------+ create user
-                |               |          | and                 
+                |               |          | and
                 |<--------------|<---------+ store key
                 | transmit key  |
                 |               |
@@ -298,25 +298,25 @@ and uses it to sign requests to OSDs and to metadata servers in the cluster.
            +---------+     +---------+
                 |  authenticate |
                 |-------------->|----------+ generate and
-                |               |          | encrypt                
+                |               |          | encrypt
                 |<--------------|<---------+ session key
                 | transmit      |
                 | encrypted     |
                 | session key   |
-                |               |             
+                |               |
                 |-----+ decrypt |
-                |     | session | 
-                |<----+ key     |              
+                |     | session |
+                |<----+ key     |
                 |               |
                 |  req. ticket  |
                 |-------------->|----------+ generate and
-                |               |          | encrypt                
+                |               |          | encrypt
                 |<--------------|<---------+ ticket
                 | recv. ticket  |
-                |               |             
+                |               |
                 |-----+ decrypt |
-                |     | ticket  | 
-                |<----+         |              
+                |     | ticket  |
+                |<----+         |
 
 
 The ``cephx`` protocol authenticates ongoing communications between the clients
@@ -331,7 +331,7 @@ between the client and the daemon.
            |  Client |     | Monitor |     |  MDS  |     |  OSD  |
            +---------+     +---------+     +-------+     +-------+
                 |  request to   |              |             |
-                | create a user |              |             |               
+                | create a user |              |             |
                 |-------------->| mon and      |             |
                 |<--------------| client share |             |
                 |    receive    | a secret.    |             |
@@ -339,7 +339,7 @@ between the client and the daemon.
                 |               |<------------>|             |
                 |               |<-------------+------------>|
                 |               | mon, mds,    |             |
-                | authenticate  | and osd      |             |  
+                | authenticate  | and osd      |             |
                 |-------------->| share        |             |
                 |<--------------| a secret     |             |
                 |  session key  |              |             |
@@ -355,7 +355,7 @@ between the client and the daemon.
                 | receive response (CephFS only)             |
                 |                                            |
                 |                make request                |
-                |------------------------------------------->|  
+                |------------------------------------------->|
                 |<-------------------------------------------|
                                receive response
 
@@ -364,7 +364,7 @@ daemons. The authentication is not extended beyond the Ceph client. If a user
 accesses the Ceph client from a remote host, cephx authentication will not be
 applied to the connection between the user's host and the client host.
 
-See `Cephx Config Guide`_ for more on configuration details. 
+See `Cephx Config Guide`_ for more on configuration details.
 
 See `User Management`_ for more on user management.
 
@@ -418,7 +418,7 @@ the greater cluster provides several benefits:
    Monitors receive no such message after a configurable period of time,
    then they mark the OSD ``down``. This mechanism is a failsafe, however.
    Normally, Ceph OSD Daemons determine if a neighboring OSD is ``down`` and
-   report it to the Ceph Monitors. This contributes to making Ceph Monitors 
+   report it to the Ceph Monitors. This contributes to making Ceph Monitors
    lightweight processes. See `Monitoring OSDs`_ and `Heartbeats`_ for
    additional details.
 
@@ -465,7 +465,7 @@ the greater cluster provides several benefits:
     Write (2) |  |   |  |  Write (3)
        +------+  |   |  +------+
        |  +------+   +------+  |
-       |  | Ack (4)  Ack (5)|  | 
+       |  | Ack (4)  Ack (5)|  |
        v  *                 *  v
  +---------------+   +---------------+
  | Secondary OSD |   | Tertiary OSD  |
@@ -492,7 +492,7 @@ About Pools
 
 The Ceph storage system supports the notion of 'Pools', which are logical
 partitions for storing objects.
-   
+
 Ceph Clients retrieve a `Cluster Map`_ from a Ceph Monitor, and write RADOS
 objects to pools. The way that Ceph places the data in the pools is determined
 by the pool's ``size`` or number of replicas, the CRUSH rule, and the number of
@@ -513,12 +513,12 @@ placement groups in the pool.
             +--------+           +---------------+
             |  Pool  |---------->|  CRUSH Rule   |
             +--------+  Selects  +---------------+
-                 
+
 
 Pools set at least the following parameters:
 
 - Ownership/Access to Objects
-- The Number of Placement Groups, and 
+- The Number of Placement Groups, and
 - The CRUSH Rule to Use.
 
 See `Set Pool Values`_ for details.
@@ -531,12 +531,12 @@ Mapping PGs to OSDs
 
 Each pool has a number of placement groups (PGs) within it. CRUSH dynamically
 maps PGs to OSDs. When a Ceph Client stores objects, CRUSH maps each RADOS
-object to a PG. 
+object to a PG.
 
 This mapping of RADOS objects to PGs implements an abstraction and indirection
 layer between Ceph OSD Daemons and Ceph Clients. The Ceph Storage Cluster must
 be able to grow (or shrink) and redistribute data adaptively when the internal
-topology changes. 
+topology changes.
 
 If the Ceph Client "knew" which Ceph OSD Daemons were storing which objects, a
 tight coupling would exist between the Ceph Client and the Ceph OSD Daemon.
@@ -565,11 +565,11 @@ placement groups, and how it maps placement groups to OSDs.
         +------+------+-------------+             |
         |             |             |             |
         v             v             v             v
-   /----------\  /----------\  /----------\  /----------\ 
+   /----------\  /----------\  /----------\  /----------\
    |          |  |          |  |          |  |          |
    |  OSD #1  |  |  OSD #2  |  |  OSD #3  |  |  OSD #4  |
    |          |  |          |  |          |  |          |
-   \----------/  \----------/  \----------/  \----------/  
+   \----------/  \----------/  \----------/  \----------/
 
 The client uses its copy of the cluster map and the CRUSH algorithm to compute
 precisely which OSD it will use when reading or writing a particular object.
@@ -583,7 +583,7 @@ When a Ceph Client binds to a Ceph Monitor, it retrieves the latest version of
 the `Cluster Map`_. When a client has been equipped with a copy of the cluster
 map, it is aware of all the monitors, OSDs, and metadata servers in the
 cluster. **However, even equipped with a copy of the latest version of the
-cluster map, the client doesn't know anything about object locations.** 
+cluster map, the client doesn't know anything about object locations.**
 
 **Object locations must be computed.**
 
@@ -626,7 +626,7 @@ persists, you may need to refer to the `Troubleshooting Peering Failure`_
 section.
 
 .. Note:: PGs that agree on the state of the cluster do not necessarily have
-   the current data yet. 
+   the current data yet.
 
 The Ceph Storage Cluster was designed to store at least two copies of an object
 (that is, ``size = 2``), which is the minimum requirement for data safety. For
@@ -656,7 +656,7 @@ epoch.
 The Ceph OSD daemons that are part of an *Acting Set* might not always be
 ``up``. When an OSD in the *Acting Set* is ``up``, it is part of the *Up Set*.
 The *Up Set* is an important distinction, because Ceph can remap PGs to other
-Ceph OSD Daemons when an OSD fails. 
+Ceph OSD Daemons when an OSD fails.
 
 .. note:: Consider a hypothetical *Acting Set* for a PG that contains
    ``osd.25``, ``osd.32`` and ``osd.61``. The first OSD (``osd.25``), is the
@@ -676,7 +676,7 @@ process (albeit rather crudely, since it is substantially less impactful with
 large clusters) where some, but not all of the PGs migrate from existing OSDs
 (OSD 1, and OSD 2) to the new OSD (OSD 3). Even when rebalancing, CRUSH is
 stable. Many of the placement groups remain in their original configuration,
-and each OSD gets some added capacity, so there are no load spikes on the 
+and each OSD gets some added capacity, so there are no load spikes on the
 new OSD after rebalancing is complete.
 
 
@@ -823,7 +823,7 @@ account.
 	            |              |               |  |
 	            |              +-------+-------+  |
 	            |                      ^          |
-	            |                      |          | 
+	            |                      |          |
 	            |                      |          |
 	         +--+---+   +------+   +---+--+   +---+--+
 	   name  | NYAN |   | NYAN |   | NYAN |   | NYAN |
@@ -876,7 +876,7 @@ version 1).
 .. ditaa::
 
      Primary OSD
-    
+
    +-------------+
    |    OSD 1    |             +-------------+
    |         log |  Write Full |             |
@@ -921,7 +921,7 @@ as ``D2v2`` ) while others are acknowledged and persisted to storage drives
 .. ditaa::
 
      Primary OSD
-    
+
    +-------------+
    |    OSD 1    |
    |         log |
@@ -930,11 +930,11 @@ as ``D2v2`` ) while others are acknowledged and persisted to storage drives
    |  +----+     +<------------+ Ceph Client |
    |             |      v2     |             |
    |  +----+     |             +-------------+
-   |  |D1v1| 1,1 |           
-   |  +----+     |           
-   +------+------+           
-          |                  
-          |                  
+   |  |D1v1| 1,1 |
+   |  +----+     |
+   +------+------+
+          |
+          |
           |           +------+------+
           |           |    OSD 2    |
           |  +------+ |         log |
@@ -962,7 +962,7 @@ the logs' ``last_complete`` pointer can move from ``1,1`` to ``1,2``.
 .. ditaa::
 
      Primary OSD
-    
+
    +-------------+
    |    OSD 1    |
    |         log |
@@ -971,10 +971,10 @@ the logs' ``last_complete`` pointer can move from ``1,1`` to ``1,2``.
    |  +----+     +<------------+ Ceph Client |
    |             |      v2     |             |
    |  +----+     |             +-------------+
-   |  |D1v1| 1,1 |           
-   |  +----+     |           
-   +------+------+           
-          |                  
+   |  |D1v1| 1,1 |
+   |  +----+     |
+   +------+------+
+          |
           |           +-------------+
           |           |    OSD 2    |
           |           |         log |
@@ -986,7 +986,7 @@ the logs' ``last_complete`` pointer can move from ``1,1`` to ``1,2``.
           |           |  |D2v1| 1,1 |
           |           |  +----+     |
           |           +-------------+
-          |                  
+          |
           |           +-------------+
           |           |    OSD 3    |
           |           |         log |
@@ -1007,7 +1007,7 @@ on **OSD 3**.
 .. ditaa::
 
      Primary OSD
-    
+
    +-------------+
    |    OSD 1    |
    |         log |
@@ -1050,7 +1050,7 @@ will be the head of the new authoritative log.
    |   (down)    |
    | c333        |
    +------+------+
-          |                  
+          |
           |           +-------------+
           |           |    OSD 2    |
           |           |         log |
@@ -1059,7 +1059,7 @@ will be the head of the new authoritative log.
           |           |  +----+     |
           |           |             |
           |           +-------------+
-          |                  
+          |
           |           +-------------+
           |           |    OSD 3    |
           |           |         log |
@@ -1079,20 +1079,20 @@ will be the head of the new authoritative log.
    |         1,1 |
    |             |
    +------+------+
-          
+
 
 
 The log entry 1,2 found on **OSD 3** is divergent from the new authoritative log
 provided by **OSD 4**: it is discarded and the file containing the ``C1v2``
 chunk is removed. The ``D1v1`` chunk is rebuilt with the ``decode`` function of
-the erasure coding library during scrubbing and stored on the new primary 
+the erasure coding library during scrubbing and stored on the new primary
 **OSD 4**.
 
 
 .. ditaa::
 
      Primary OSD
-    
+
    +-------------+
    |    OSD 4    |
    |         log |
@@ -1140,7 +1140,7 @@ configured to act as a cache tier, and a backing pool of either erasure-coded
 or relatively slower/cheaper devices configured to act as an economical storage
 tier. The Ceph objecter handles where to place the objects and the tiering
 agent determines when to flush objects from the cache to the backing storage
-tier. So the cache tier and the backing storage tier are completely transparent 
+tier. So the cache tier and the backing storage tier are completely transparent
 to Ceph clients.
 
 
@@ -1150,14 +1150,14 @@ to Ceph clients.
            | Ceph Client |
            +------+------+
                   ^
-     Tiering is   |  
+     Tiering is   |
     Transparent   |              Faster I/O
         to Ceph   |           +---------------+
-     Client Ops   |           |               |   
+     Client Ops   |           |               |
                   |    +----->+   Cache Tier  |
                   |    |      |               |
                   |    |      +-----+---+-----+
-                  |    |            |   ^ 
+                  |    |            |   ^
                   v    v            |   |   Active Data in Cache Tier
            +------+----+--+         |   |
            |   Objecter   |         |   |
@@ -1198,11 +1198,11 @@ operations on the outbound data and return the data to the client.
 
    A Ceph class for a content management system that presents pictures of a
    particular size and aspect ratio could take an inbound bitmap image, crop it
-   to a particular aspect ratio, resize it and embed an invisible copyright or 
-   watermark to help protect the intellectual property; then, save the 
+   to a particular aspect ratio, resize it and embed an invisible copyright or
+   watermark to help protect the intellectual property; then, save the
    resulting bitmap image to the object store.
 
-See ``src/objclass/objclass.h``, ``src/fooclass.cc`` and ``src/barclass`` for 
+See ``src/objclass/objclass.h``, ``src/fooclass.cc`` and ``src/barclass`` for
 exemplary implementations.
 
 
@@ -1279,7 +1279,7 @@ synchronization/communication channel.
            +----------+     +----------+     +----------+     +---------------+
                  |                |                |                  |
                  |                |                |                  |
-                 |                |  Watch Object  |                  |               
+                 |                |  Watch Object  |                  |
                  |--------------------------------------------------->|
                  |                |                |                  |
                  |<---------------------------------------------------|
@@ -1295,7 +1295,7 @@ synchronization/communication channel.
                  |                |                |                  |
                  |                |                |<-----------------|
                  |                |                |    Ack/Commit    |
-                 |                |     Notify     |                  |               
+                 |                |     Notify     |                  |
                  |--------------------------------------------------->|
                  |                |                |                  |
                  |<---------------------------------------------------|
@@ -1305,7 +1305,7 @@ synchronization/communication channel.
                  |                |     Notify     |                  |
                  |                |                |<-----------------|
                  |                |                |      Notify      |
-                 |                |       Ack      |                  |               
+                 |                |       Ack      |                  |
                  |----------------+---------------------------------->|
                  |                |                |                  |
                  |                |       Ack      |                  |
@@ -1313,7 +1313,7 @@ synchronization/communication channel.
                  |                |                |                  |
                  |                |                |        Ack       |
                  |                |                |----------------->|
-                 |                |                |                  | 
+                 |                |                |                  |
                  |<---------------+----------------+------------------|
                  |                     Complete
 
@@ -1331,13 +1331,13 @@ volume'. Ceph's striping offers the throughput of RAID 0 striping, the
 reliability of n-way RAID mirroring and faster recovery.
 
 Ceph provides three types of clients: Ceph Block Device, Ceph File System, and
-Ceph Object Storage. A Ceph Client converts its data from the representation 
+Ceph Object Storage. A Ceph Client converts its data from the representation
 format it provides to its users (a block device image, RESTful objects, CephFS
-filesystem directories) into objects for storage in the Ceph Storage Cluster. 
+filesystem directories) into objects for storage in the Ceph Storage Cluster.
 
-.. tip:: The objects Ceph stores in the Ceph Storage Cluster are not striped. 
-   Ceph Object Storage, Ceph Block Device, and the Ceph File System stripe their 
-   data over multiple Ceph Storage Cluster objects. Ceph Clients that write 
+.. tip:: The objects Ceph stores in the Ceph Storage Cluster are not striped.
+   Ceph Object Storage, Ceph Block Device, and the Ceph File System stripe their
+   data over multiple Ceph Storage Cluster objects. Ceph Clients that write
    directly to the Ceph Storage Cluster via ``librados`` must perform the
    striping (and parallel I/O) for themselves to obtain these benefits.
 
@@ -1380,7 +1380,7 @@ diagram depicts the simplest form of striping:
                  | End cCCC  |    | End cCCC  |
                  | Object 0  |    | Object 1  |
                  \-----------/    \-----------/
-   
+
 
 If you anticipate large images sizes, large S3 or Swift objects (e.g., video),
 or large CephFS directories, you may see considerable read/write performance
@@ -1420,16 +1420,16 @@ stripe (``stripe unit 16``) in the first object in the new object set (``object
        +-----------------+--------+--------+-----------------+
        |                 |                 |                 |     +--\
        v                 v                 v                 v        |
- /-----------\     /-----------\     /-----------\     /-----------\  |   
+ /-----------\     /-----------\     /-----------\     /-----------\  |
  | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
  | Object 0  |     | Object  1 |     | Object  2 |     | Object  3 |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
  |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
  |  unit 0   |     |  unit 1   |     |  unit 2   |     |  unit 3   |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
- |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\
  |  unit 4   |     |  unit 5   |     |  unit 6   |     |  unit 7   |    | Object
- +-----------+     +-----------+     +-----------+     +-----------+    +- Set 
+ +-----------+     +-----------+     +-----------+     +-----------+    +- Set
  |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   1
  |  unit 8   |     |  unit 9   |     |  unit 10  |     |  unit 11  |  +-/
  +-----------+     +-----------+     +-----------+     +-----------+  |
@@ -1437,36 +1437,36 @@ stripe (``stripe unit 16``) in the first object in the new object set (``object
  |  unit 12  |     |  unit 13  |     |  unit 14  |     |  unit 15  |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
  | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
- | Object 0  |     | Object 1  |     | Object 2  |     | Object 3  |  |  
+ | Object 0  |     | Object 1  |     | Object 2  |     | Object 3  |  |
  \-----------/     \-----------/     \-----------/     \-----------/  |
                                                                       |
                                                                    +--/
-  
+
                                                                    +--\
                                                                       |
- /-----------\     /-----------\     /-----------\     /-----------\  |   
+ /-----------\     /-----------\     /-----------\     /-----------\  |
  | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
- | Object  4 |     | Object  5 |     | Object  6 |     | Object  7 |  |  
+ | Object  4 |     | Object  5 |     | Object  6 |     | Object  7 |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
  |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
  |  unit 16  |     |  unit 17  |     |  unit 18  |     |  unit 19  |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
- |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\
  |  unit 20  |     |  unit 21  |     |  unit 22  |     |  unit 23  |    | Object
  +-----------+     +-----------+     +-----------+     +-----------+    +- Set
- |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   2 
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   2
  |  unit 24  |     |  unit 25  |     |  unit 26  |     |  unit 27  |  +-/
  +-----------+     +-----------+     +-----------+     +-----------+  |
  |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
  |  unit 28  |     |  unit 29  |     |  unit 30  |     |  unit 31  |  |
  +-----------+     +-----------+     +-----------+     +-----------+  |
  | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
- | Object 4  |     | Object 5  |     | Object 6  |     | Object 7  |  |  
+ | Object 4  |     | Object 5  |     | Object 6  |     | Object 7  |  |
  \-----------/     \-----------/     \-----------/     \-----------/  |
                                                                       |
                                                                    +--/
 
-Three important variables determine how Ceph stripes data: 
+Three important variables determine how Ceph stripes data:
 
 - **Object Size:** Objects in the Ceph Storage Cluster have a maximum
   configurable size (e.g., 2MB, 4MB, etc.). The object size should be large
@@ -1474,24 +1474,24 @@ Three important variables determine how Ceph stripes data:
   the stripe unit.
 
 - **Stripe Width:** Stripes have a configurable unit size (e.g., 64kb).
-  The Ceph Client divides the data it will write to objects into equally 
-  sized stripe units, except for the last stripe unit. A stripe width, 
-  should be a fraction of the Object Size so that an object may contain 
+  The Ceph Client divides the data it will write to objects into equally
+  sized stripe units, except for the last stripe unit. A stripe width,
+  should be a fraction of the Object Size so that an object may contain
   many stripe units.
 
 - **Stripe Count:** The Ceph Client writes a sequence of stripe units
-  over a series of objects determined by the stripe count. The series 
-  of objects is called an object set. After the Ceph Client writes to 
+  over a series of objects determined by the stripe count. The series
+  of objects is called an object set. After the Ceph Client writes to
   the last object in the object set, it returns to the first object in
   the object set.
-  
+
 .. important:: Test the performance of your striping configuration before
    putting your cluster into production. You CANNOT change these striping
    parameters after you stripe the data and write it to objects.
 
 Once the Ceph Client has striped data to stripe units and mapped the stripe
 units to objects, Ceph's CRUSH algorithm maps the objects to placement groups,
-and the placement groups to Ceph OSD Daemons before the objects are stored as 
+and the placement groups to Ceph OSD Daemons before the objects are stored as
 files on a storage drive.
 
 .. note:: Since a client writes to a single pool, all data striped into objects
@@ -1515,23 +1515,23 @@ Ceph Clients include a number of service interfaces. These include:
   that uses ``librbd`` directly--avoiding the kernel object overhead for
   virtualized systems.
 
-- **Object Storage:** The :term:`Ceph Object Storage` (a.k.a., RGW) service 
+- **Object Storage:** The :term:`Ceph Object Storage` (a.k.a., RGW) service
   provides RESTful APIs with interfaces that are compatible with Amazon S3
-  and OpenStack Swift. 
-  
-- **Filesystem**: The :term:`Ceph File System` (CephFS) service provides 
-  a POSIX compliant filesystem usable with ``mount`` or as 
+  and OpenStack Swift.
+
+- **Filesystem**: The :term:`Ceph File System` (CephFS) service provides
+  a POSIX compliant filesystem usable with ``mount`` or as
   a filesystem in user space (FUSE).
 
 Ceph can run additional instances of OSDs, MDSs, and monitors for scalability
 and high availability. The following diagram depicts the high-level
-architecture. 
+architecture.
 
 .. ditaa::
 
             +--------------+  +----------------+  +-------------+
             | Block Device |  | Object Storage |  |   CephFS    |
-            +--------------+  +----------------+  +-------------+            
+            +--------------+  +----------------+  +-------------+
 
             +--------------+  +----------------+  +-------------+
             |    librbd    |  |     librgw     |  |  libcephfs  |
@@ -1563,10 +1563,10 @@ another application.
 .. topic:: S3/Swift Objects and Store Cluster Objects Compared
 
    Ceph's Object Storage uses the term *object* to describe the data it stores.
-   S3 and Swift objects are not the same as the objects that Ceph writes to the 
+   S3 and Swift objects are not the same as the objects that Ceph writes to the
    Ceph Storage Cluster. Ceph Object Storage objects are mapped to Ceph Storage
-   Cluster objects. The S3 and Swift objects do not necessarily 
-   correspond in a 1:1 manner with an object stored in the storage cluster. It 
+   Cluster objects. The S3 and Swift objects do not necessarily
+   correspond in a 1:1 manner with an object stored in the storage cluster. It
    is possible for an S3 or Swift object to map to multiple Ceph objects.
 
 See `Ceph Object Storage`_ for details.
@@ -1582,7 +1582,7 @@ Ceph Storage Cluster, where each object gets mapped to a placement group and
 distributed, and the placement groups are spread across separate ``ceph-osd``
 daemons throughout the cluster.
 
-.. important:: Striping allows RBD block devices to perform better than a single 
+.. important:: Striping allows RBD block devices to perform better than a single
    server could!
 
 Thin-provisioned snapshottable Ceph Block Devices are an attractive option for
@@ -1591,7 +1591,8 @@ typically deploy a Ceph Block Device with the ``rbd`` network storage driver in
 QEMU/KVM, where the host machine uses ``librbd`` to provide a block device
 service to the guest. Many cloud computing stacks use ``libvirt`` to integrate
 with hypervisors. You can use thin-provisioned Ceph Block Devices with QEMU and
-``libvirt`` to support OpenStack and CloudStack among other solutions.
+``libvirt`` to support OpenStack, OpenNebula and CloudStack
+among other solutions.
 
 While we do not provide ``librbd`` support with other hypervisors at this time,
 you may also use Ceph Block Device kernel objects to provide a block device to a
@@ -1616,7 +1617,7 @@ a Filesystem in User Space (FUSE).
 
             +-----------------------+  +------------------------+
             | CephFS Kernel Object  |  |      CephFS FUSE       |
-            +-----------------------+  +------------------------+            
+            +-----------------------+  +------------------------+
 
             +---------------------------------------------------+
             |            CephFS Library (libcephfs)             |
@@ -1645,9 +1646,9 @@ CephFS separates the metadata from the data, storing the metadata in the MDS,
 and storing the file data in one or more objects in the Ceph Storage Cluster.
 The Ceph filesystem aims for POSIX compatibility. ``ceph-mds`` can run as a
 single process, or it can be distributed out to multiple physical machines,
-either for high availability or for scalability. 
+either for high availability or for scalability.
 
-- **High Availability**: The extra ``ceph-mds`` instances can be `standby`, 
+- **High Availability**: The extra ``ceph-mds`` instances can be `standby`,
   ready to take over the duties of any failed ``ceph-mds`` that was
   `active`. This is easy because all the data, including the journal, is
   stored on RADOS. The transition is triggered automatically by ``ceph-mon``.
diff --git a/doc/install/index.rst b/doc/install/index.rst
index d8e9ca3a63e9..82585edd8b8c 100644
--- a/doc/install/index.rst
+++ b/doc/install/index.rst
@@ -4,13 +4,13 @@
 Installing Ceph
 ===============
 
-There are multiple ways to install Ceph.  
+There are multiple ways to install Ceph.
 
 Recommended methods
 ~~~~~~~~~~~~~~~~~~~
 
 :ref:`Cephadm <cephadm_deploying_new_cluster>` is a tool that can be used to
-install and manage a Ceph cluster. 
+install and manage a Ceph cluster.
 
 * cephadm supports only Octopus and newer releases.
 * cephadm is fully integrated with the orchestration API and fully supports the
@@ -59,6 +59,8 @@ tool that can be used to quickly deploy clusters. It is deprecated.
 
 `github.com/openstack/puppet-ceph <https://github.com/openstack/puppet-ceph>`_  installs Ceph via Puppet.
 
+`OpenNebula HCI clusters <https://docs.opennebula.io/stable/provision_clusters/hci_clusters/overview.html>`_ deploys Ceph on various cloud platforms.
+
 Ceph can also be :ref:`installed manually <install-manual>`.
 
 
diff --git a/doc/rbd/index.rst b/doc/rbd/index.rst
index 4a8029bbaeef..96f1e1389788 100644
--- a/doc/rbd/index.rst
+++ b/doc/rbd/index.rst
@@ -32,9 +32,9 @@ the ``librbd`` library.
 
 Ceph's block devices deliver high performance with vast scalability to
 `kernel modules`_, or to :abbr:`KVMs (kernel virtual machines)` such as `QEMU`_, and
-cloud-based computing systems like `OpenStack`_ and `CloudStack`_ that rely on
-libvirt and QEMU to integrate with Ceph block devices. You can use the same cluster
-to operate the :ref:`Ceph RADOS Gateway <object-gateway>`, the
+cloud-based computing systems like `OpenStack`_, `OpenNebula`_ and `CloudStack`_
+that rely on libvirt and QEMU to integrate with Ceph block devices. You can use
+the same cluster to operate the :ref:`Ceph RADOS Gateway <object-gateway>`, the
 :ref:`Ceph File System <ceph-file-system>`, and Ceph block devices simultaneously.
 
 .. important:: To use Ceph Block Devices, you must have access to a running
@@ -69,4 +69,5 @@ to operate the :ref:`Ceph RADOS Gateway <object-gateway>`, the
 .. _kernel modules: ./rbd-ko/
 .. _QEMU: ./qemu-rbd/
 .. _OpenStack: ./rbd-openstack
+.. _OpenNebula: https://docs.opennebula.io/stable/open_cluster_deployment/storage_setup/ceph_ds.html
 .. _CloudStack: ./rbd-cloudstack
diff --git a/doc/rbd/libvirt.rst b/doc/rbd/libvirt.rst
index e3523f8a8005..a55a4f95b799 100644
--- a/doc/rbd/libvirt.rst
+++ b/doc/rbd/libvirt.rst
@@ -4,11 +4,11 @@
 
 .. index:: Ceph Block Device; livirt
 
-The ``libvirt`` library creates a virtual machine abstraction layer between 
-hypervisor interfaces and the software applications that use them. With 
-``libvirt``, developers and system administrators can focus on a common 
+The ``libvirt`` library creates a virtual machine abstraction layer between
+hypervisor interfaces and the software applications that use them. With
+``libvirt``, developers and system administrators can focus on a common
 management framework, common API, and common shell interface (i.e., ``virsh``)
-to many different hypervisors, including: 
+to many different hypervisors, including:
 
 - QEMU/KVM
 - XEN
@@ -18,7 +18,7 @@ to many different hypervisors, including:
 
 Ceph block devices support QEMU/KVM. You can use Ceph block devices with
 software that interfaces with ``libvirt``. The following stack diagram
-illustrates how ``libvirt`` and QEMU use Ceph block devices via ``librbd``. 
+illustrates how ``libvirt`` and QEMU use Ceph block devices via ``librbd``.
 
 
 .. ditaa::
@@ -41,10 +41,11 @@ illustrates how ``libvirt`` and QEMU use Ceph block devices via ``librbd``.
 
 
 The most common ``libvirt`` use case involves providing Ceph block devices to
-cloud solutions like OpenStack or CloudStack. The cloud solution uses
+cloud solutions like OpenStack, OpenNebula or CloudStack. The cloud solution uses
 ``libvirt`` to  interact with QEMU/KVM, and QEMU/KVM interacts with Ceph block
-devices via  ``librbd``. See `Block Devices and OpenStack`_ and `Block Devices
-and CloudStack`_ for details. See `Installation`_ for installation details.
+devices via  ``librbd``. See `Block Devices and OpenStack`_,
+`Block Devices and OpenNebula`_ and `Block Devices and CloudStack`_ for details.
+See `Installation`_ for installation details.
 
 You can also use Ceph block devices with ``libvirt``, ``virsh`` and the
 ``libvirt`` API. See `libvirt Virtualization API`_ for details.
@@ -62,12 +63,12 @@ Configuring Ceph
 
 To configure Ceph for use with ``libvirt``, perform the following steps:
 
-#. `Create a pool`_. The following example uses the 
+#. `Create a pool`_. The following example uses the
    pool name ``libvirt-pool``.::
 
 	ceph osd pool create libvirt-pool
 
-   Verify the pool exists. :: 
+   Verify the pool exists. ::
 
 	ceph osd lspools
 
@@ -80,23 +81,23 @@ To configure Ceph for use with ``libvirt``, perform the following steps:
    and references ``libvirt-pool``. ::
 
 	ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=libvirt-pool'
-	
-   Verify the name exists. :: 
-   
+
+   Verify the name exists. ::
+
 	ceph auth ls
 
-   **NOTE**: ``libvirt`` will access Ceph using the ID ``libvirt``, 
-   not the Ceph name ``client.libvirt``. See `User Management - User`_ and 
-   `User Management - CLI`_ for a detailed explanation of the difference 
-   between ID and name.	
+   **NOTE**: ``libvirt`` will access Ceph using the ID ``libvirt``,
+   not the Ceph name ``client.libvirt``. See `User Management - User`_ and
+   `User Management - CLI`_ for a detailed explanation of the difference
+   between ID and name.
 
-#. Use QEMU to `create an image`_ in your RBD pool. 
+#. Use QEMU to `create an image`_ in your RBD pool.
    The following example uses the image name ``new-libvirt-image``
    and references ``libvirt-pool``. ::
 
 	qemu-img create -f rbd rbd:libvirt-pool/new-libvirt-image 2G
 
-   Verify the image exists. :: 
+   Verify the image exists. ::
 
 	rbd -p libvirt-pool ls
 
@@ -111,7 +112,7 @@ To configure Ceph for use with ``libvirt``, perform the following steps:
 	admin socket = /var/run/ceph/$cluster-$type.$id.$pid.$cctid.asok
 
    The ``client.libvirt`` section name should match the cephx user you created
-   above.  
+   above.
    If SELinux or AppArmor is enabled, note that this could prevent the client
    process (qemu via libvirt) from doing some operations, such as writing logs
    or operate the images or admin socket to the destination locations (``/var/
@@ -123,7 +124,7 @@ Preparing the VM Manager
 ========================
 
 You may use ``libvirt`` without a VM manager, but you may find it simpler to
-create your first domain with ``virt-manager``. 
+create your first domain with ``virt-manager``.
 
 #. Install a virtual machine manager. See `KVM/VirtManager`_ for details. ::
 
@@ -131,7 +132,7 @@ create your first domain with ``virt-manager``.
 
 #. Download an OS image (if necessary).
 
-#. Launch the virtual machine manager. :: 
+#. Launch the virtual machine manager. ::
 
 	sudo virt-manager
 
@@ -142,12 +143,12 @@ Creating a VM
 
 To create a VM with ``virt-manager``, perform the following steps:
 
-#. Press the **Create New Virtual Machine** button. 
+#. Press the **Create New Virtual Machine** button.
 
 #. Name the new virtual machine domain. In the exemplary embodiment, we
    use the name ``libvirt-virtual-machine``. You may use any name you wish,
-   but ensure you replace ``libvirt-virtual-machine`` with the name you 
-   choose in subsequent commandline and configuration examples. :: 
+   but ensure you replace ``libvirt-virtual-machine`` with the name you
+   choose in subsequent commandline and configuration examples. ::
 
 	libvirt-virtual-machine
 
@@ -155,9 +156,9 @@ To create a VM with ``virt-manager``, perform the following steps:
 
 	/path/to/image/recent-linux.img
 
-   **NOTE:** Import a recent image. Some older images may not rescan for 
+   **NOTE:** Import a recent image. Some older images may not rescan for
    virtual devices properly.
-   
+
 #. Configure and start the VM.
 
 #. You may use ``virsh list`` to verify the VM domain exists. ::
@@ -179,11 +180,11 @@ you that root privileges are required. For a reference of ``virsh``
 commands, refer to `Virsh Command Reference`_.
 
 
-#. Open the configuration file with ``virsh edit``. :: 
+#. Open the configuration file with ``virsh edit``. ::
 
 	sudo virsh edit {vm-domain-name}
 
-   Under ``<devices>`` there should be a ``<disk>`` entry. :: 
+   Under ``<devices>`` there should be a ``<disk>`` entry. ::
 
 	<devices>
 		<emulator>/usr/bin/kvm</emulator>
@@ -196,18 +197,18 @@ commands, refer to `Virsh Command Reference`_.
 
 
    Replace ``/path/to/image/recent-linux.img`` with the path to the OS image.
-   The minimum kernel for using the faster ``virtio`` bus is 2.6.25. See 
+   The minimum kernel for using the faster ``virtio`` bus is 2.6.25. See
    `Virtio`_ for details.
 
-   **IMPORTANT:** Use ``sudo virsh edit`` instead of a text editor. If you edit 
-   the configuration file under ``/etc/libvirt/qemu`` with a text editor, 
-   ``libvirt`` may not recognize the change. If there is a discrepancy between 
-   the contents of the XML file under ``/etc/libvirt/qemu`` and the result of 
-   ``sudo virsh dumpxml {vm-domain-name}``, then your VM may not work 
+   **IMPORTANT:** Use ``sudo virsh edit`` instead of a text editor. If you edit
+   the configuration file under ``/etc/libvirt/qemu`` with a text editor,
+   ``libvirt`` may not recognize the change. If there is a discrepancy between
+   the contents of the XML file under ``/etc/libvirt/qemu`` and the result of
+   ``sudo virsh dumpxml {vm-domain-name}``, then your VM may not work
    properly.
-   
 
-#. Add the Ceph RBD image you created as a ``<disk>`` entry. :: 
+
+#. Add the Ceph RBD image you created as a ``<disk>`` entry. ::
 
 	<disk type='network' device='disk'>
 		<source protocol='rbd' name='libvirt-pool/new-libvirt-image'>
@@ -216,21 +217,21 @@ commands, refer to `Virsh Command Reference`_.
 		<target dev='vdb' bus='virtio'/>
 	</disk>
 
-   Replace ``{monitor-host}`` with the name of your host, and replace the 
-   pool and/or image name as necessary. You may add multiple ``<host>`` 
+   Replace ``{monitor-host}`` with the name of your host, and replace the
+   pool and/or image name as necessary. You may add multiple ``<host>``
    entries for your Ceph monitors. The ``dev`` attribute is the logical
-   device name that will appear under the ``/dev`` directory of your 
-   VM. The optional ``bus`` attribute indicates the type of disk device to 
-   emulate. The valid settings are driver specific (e.g., "ide", "scsi", 
+   device name that will appear under the ``/dev`` directory of your
+   VM. The optional ``bus`` attribute indicates the type of disk device to
+   emulate. The valid settings are driver specific (e.g., "ide", "scsi",
    "virtio", "xen", "usb" or "sata").
-   
+
    See `Disks`_ for details of the ``<disk>`` element, and its child elements
    and attributes.
-	
+
 #. Save the file.
 
-#. If your Ceph Storage Cluster has `Ceph Authentication`_ enabled (it does by 
-   default), you must generate a secret. :: 
+#. If your Ceph Storage Cluster has `Ceph Authentication`_ enabled (it does by
+   default), you must generate a secret. ::
 
 	cat > secret.xml <<EOF
 	<secret ephemeral='no' private='no'>
@@ -249,11 +250,11 @@ commands, refer to `Virsh Command Reference`_.
 
 	ceph auth get-key client.libvirt | sudo tee client.libvirt.key
 
-#. Set the UUID of the secret. :: 
+#. Set the UUID of the secret. ::
 
 	sudo virsh secret-set-value --secret {uuid of secret} --base64 $(cat client.libvirt.key) && rm client.libvirt.key secret.xml
 
-   You must also set the secret manually by adding the following ``<auth>`` 
+   You must also set the secret manually by adding the following ``<auth>``
    entry to the ``<disk>`` element you entered earlier (replacing the
    ``uuid`` value with the result from the command line example above). ::
 
@@ -266,14 +267,14 @@ commands, refer to `Virsh Command Reference`_.
 	<auth username='libvirt'>
 		<secret type='ceph' uuid='{uuid of secret}'/>
 	</auth>
-	<target ... 
+	<target ...
 
 
-   **NOTE:** The exemplary ID is ``libvirt``, not the Ceph name 
-   ``client.libvirt`` as generated at step 2 of `Configuring Ceph`_. Ensure 
-   you use the ID component of the Ceph name you generated. If for some reason 
-   you need to regenerate the secret, you will have to execute 
-   ``sudo virsh secret-undefine {uuid}`` before executing 
+   **NOTE:** The exemplary ID is ``libvirt``, not the Ceph name
+   ``client.libvirt`` as generated at step 2 of `Configuring Ceph`_. Ensure
+   you use the ID component of the Ceph name you generated. If for some reason
+   you need to regenerate the secret, you will have to execute
+   ``sudo virsh secret-undefine {uuid}`` before executing
    ``sudo virsh secret-set-value`` again.
 
 
@@ -285,30 +286,31 @@ To verify that the VM and Ceph are communicating, you may perform the
 following procedures.
 
 
-#. Check to see if Ceph is running:: 
+#. Check to see if Ceph is running::
 
 	ceph health
 
-#. Check to see if the VM is running. :: 
+#. Check to see if the VM is running. ::
 
 	sudo virsh list
 
-#. Check to see if the VM is communicating with Ceph. Replace 
-   ``{vm-domain-name}`` with the name of your VM domain:: 
+#. Check to see if the VM is communicating with Ceph. Replace
+   ``{vm-domain-name}`` with the name of your VM domain::
 
 	sudo virsh qemu-monitor-command --hmp {vm-domain-name} 'info block'
 
 #. Check to see if the device from ``<target dev='vdb' bus='virtio'/>`` exists::
-   
+
        virsh domblklist {vm-domain-name} --details
 
-If everything looks okay, you may begin using the Ceph block device 
+If everything looks okay, you may begin using the Ceph block device
 within your VM.
 
 
 .. _Installation: ../../install
 .. _libvirt Virtualization API: http://www.libvirt.org
 .. _Block Devices and OpenStack: ../rbd-openstack
+.. _Block Devices and OpenNebula: https://docs.opennebula.io/stable/open_cluster_deployment/storage_setup/ceph_ds.html#datastore-internals
 .. _Block Devices and CloudStack: ../rbd-cloudstack
 .. _Create a pool: ../../rados/operations/pools#create-a-pool
 .. _Create a Ceph User: ../../rados/operations/user-management#add-a-user
diff --git a/doc/rbd/rbd-snapshot.rst b/doc/rbd/rbd-snapshot.rst
index 120dd8ec1256..4a4309f8e7dd 100644
--- a/doc/rbd/rbd-snapshot.rst
+++ b/doc/rbd/rbd-snapshot.rst
@@ -10,7 +10,7 @@ you can create snapshots of images to retain point-in-time state history.  Ceph
 also supports snapshot layering, which allows you to clone images (for example,
 VM images) quickly and easily. Ceph block device snapshots are managed using
 the ``rbd`` command and several higher-level interfaces, including `QEMU`_,
-`libvirt`_, `OpenStack`_, and `CloudStack`_.
+`libvirt`_, `OpenStack`_, `OpenNebula`_ and `CloudStack`_.
 
 .. important:: To use RBD snapshots, you must have a running Ceph cluster.
 
@@ -18,14 +18,14 @@ the ``rbd`` command and several higher-level interfaces, including `QEMU`_,
 .. note:: Because RBD is unaware of any file system within an image (volume),
    snapshots are merely `crash-consistent` unless they are coordinated within
    the mounting (attaching) operating system. We therefore recommend that you
-   pause or stop I/O before taking a snapshot.  
-   
+   pause or stop I/O before taking a snapshot.
+
    If the volume contains a file system, the file system should be in an
    internally consistent state before a snapshot is taken. Snapshots taken
    without write quiescing could need an `fsck` pass before they are mounted
    again. To quiesce I/O you can use `fsfreeze` command. See the `fsfreeze(8)`
-   man page for more details. 
-   
+   man page for more details.
+
    For virtual machines, `qemu-guest-agent` can be used to automatically freeze
    file systems when creating a snapshot.
 
@@ -44,7 +44,7 @@ Cephx Notes
 
 When `cephx`_ authentication is enabled (it is by default), you must specify a
 user name or ID and a path to the keyring containing the corresponding key. See
-:ref:`User Management <user-management>` for details. 
+:ref:`User Management <user-management>` for details.
 
 .. prompt:: bash $
 
@@ -83,7 +83,7 @@ For example:
 .. prompt:: bash $
 
    rbd snap create rbd/foo@snapname
-	
+
 
 List Snapshots
 --------------
@@ -135,7 +135,7 @@ name, the image name, and the snap name:
 .. prompt:: bash $
 
    rbd snap rm {pool-name}/{image-name}@{snap-name}
-	
+
 For example:
 
 .. prompt:: bash $
@@ -186,20 +186,20 @@ snapshot simplifies semantics, making it possible to create clones rapidly.
            |             |  to Parent   |             |
            | (read only) |              | (writable)  |
            +-------------+              +-------------+
-           
+
                Parent                        Child
 
 .. note:: The terms "parent" and "child" refer to a Ceph block device snapshot
    (parent) and the corresponding image cloned from the snapshot (child).
    These terms are important for the command line usage below.
-   
+
 Each cloned image (child) stores a reference to its parent image, which enables
 the cloned image to open the parent snapshot and read it.
 
 A copy-on-write clone of a snapshot behaves exactly like any other Ceph
 block device image. You can read to, write from, clone, and resize cloned
 images. There are no special restrictions with cloned images. However, the
-copy-on-write clone of a snapshot depends on the snapshot, so you must 
+copy-on-write clone of a snapshot depends on the snapshot, so you must
 protect the snapshot before you clone it. The diagram below depicts this
 process.
 
@@ -222,7 +222,7 @@ have performed these steps, you can begin cloning the snapshot.
            |                            |        |                             |
            +----------------------------+        +-----------------------------+
                                                                 |
-                         +--------------------------------------+ 
+                         +--------------------------------------+
                          |
                          v
            +----------------------------+        +-----------------------------+
@@ -265,7 +265,7 @@ Protecting a Snapshot
 ---------------------
 
 Clones access the parent snapshots. All clones would break if a user
-inadvertently deleted the parent snapshot. To prevent data loss, you must 
+inadvertently deleted the parent snapshot. To prevent data loss, you must
 protect the snapshot before you can clone it:
 
 .. prompt:: bash $
@@ -290,13 +290,13 @@ protect the snapshot before you can clone it:
 .. prompt:: bash $
 
    rbd clone {pool-name}/{parent-image-name}@{snap-name} {pool-name}/{child-image-name}
-	
+
 For example:
 
 .. prompt:: bash $
 
    rbd clone rbd/foo@snapname rbd/bar
-	
+
 
 .. note:: You may clone a snapshot from one pool to an image in another pool.
    For example, you may maintain read-only images and snapshots as templates in
@@ -364,5 +364,6 @@ For example:
 .. _cephx: ../../rados/configuration/auth-config-ref/
 .. _QEMU: ../qemu-rbd/
 .. _OpenStack: ../rbd-openstack/
+.. _OpenNebula: https://docs.opennebula.io/stable/management_and_operations/vm_management/vm_instances.html?highlight=ceph#managing-disk-snapshots
 .. _CloudStack: ../rbd-cloudstack/
 .. _libvirt: ../libvirt/
diff --git a/doc/start/documenting-ceph.rst b/doc/start/documenting-ceph.rst
index d94e87f6d74a..fef870f00861 100644
--- a/doc/start/documenting-ceph.rst
+++ b/doc/start/documenting-ceph.rst
@@ -5,7 +5,7 @@
 ==================
 
 You can help the Ceph project by contributing to the documentation.  Even
-small contributions help the Ceph project. 
+small contributions help the Ceph project.
 
 The easiest way to suggest a correction to the documentation is to send an
 email to `ceph-users@ceph.io`. Include the string "ATTN: DOCS" or
@@ -27,7 +27,7 @@ Location of the Documentation in the Repository
 ===============================================
 
 The Ceph documentation source is in the ``ceph/doc`` directory of the Ceph
-repository. Python Sphinx renders the source into HTML and manpages. 
+repository. Python Sphinx renders the source into HTML and manpages.
 
 Viewing Old Ceph Documentation
 ==============================
@@ -113,27 +113,27 @@ this, you must:
 
 The Ceph documentation is organized by component:
 
-- **Ceph Storage Cluster:** The Ceph Storage Cluster documentation is 
+- **Ceph Storage Cluster:** The Ceph Storage Cluster documentation is
   in the ``doc/rados`` directory.
-  
-- **Ceph Block Device:** The Ceph Block Device documentation is in 
+
+- **Ceph Block Device:** The Ceph Block Device documentation is in
   the ``doc/rbd`` directory.
-  
-- **Ceph Object Storage:** The Ceph Object Storage documentation is in 
+
+- **Ceph Object Storage:** The Ceph Object Storage documentation is in
   the ``doc/radosgw`` directory.
 
-- **Ceph File System:** The Ceph File System documentation is in the 
+- **Ceph File System:** The Ceph File System documentation is in the
   ``doc/cephfs`` directory.
-  
+
 - **Installation (Quick):** Quick start documentation is in the
   ``doc/start`` directory.
-  
+
 - **Installation (Manual):** Documentaton concerning the manual installation of
   Ceph is in the ``doc/install`` directory.
-  
+
 - **Manpage:** Manpage source is in the ``doc/man`` directory.
 
-- **Developer:** Developer documentation is in the ``doc/dev`` 
+- **Developer:** Developer documentation is in the ``doc/dev``
   directory.
 
 - **Images:** Images including JPEG and PNG files are stored in the
@@ -152,7 +152,7 @@ are in the current release. ``main`` is the most commonly used branch. :
 
 	git checkout main
 
-When you make changes to documentation that affect an upcoming release, use 
+When you make changes to documentation that affect an upcoming release, use
 the ``next`` branch. ``next`` is the second most commonly used branch. :
 
 .. prompt:: bash $
@@ -206,8 +206,8 @@ or a table of contents entry. The ``index.rst`` file of a top-level directory
 usually contains a TOC, where you can add the new file name. All documents must
 have a title. See `Headings`_ for details.
 
-Your new document doesn't get tracked by ``git`` automatically. When you want 
-to add the document to the repository,  you must use ``git add 
+Your new document doesn't get tracked by ``git`` automatically. When you want
+to add the document to the repository,  you must use ``git add
 {path-to-filename}``. For example, from the top level  directory of the
 repository, adding an ``example.rst`` file to the ``rados`` subdirectory would
 look like this:
@@ -307,6 +307,7 @@ the following packages are required:
 - graphviz
 - ant
 - ditaa
+- cython3
 
 .. raw:: html
 
@@ -354,7 +355,7 @@ distributions, execute the following:
 .. prompt:: bash $
 
 	sudo apt-get install gcc python-dev python3-pip libxml2-dev libxslt-dev doxygen graphviz ant ditaa
-	sudo apt-get install python3-sphinx python3-venv
+	sudo apt-get install python3-sphinx python3-venv cython3
 
 For Fedora distributions, execute the following:
 
@@ -436,39 +437,39 @@ Ceph documentation commits are simple, but follow a strict convention:
 - A commit MUST have a comment.
 - A commit comment MUST be prepended with ``doc:``. (strict)
 - The comment summary MUST be one line only. (strict)
-- Additional comments MAY follow a blank line after the summary, 
+- Additional comments MAY follow a blank line after the summary,
   but should be terse.
 - A commit MAY include ``Fixes: https://tracker.ceph.com/issues/{bug number}``.
 - Commits MUST include ``Signed-off-by: Firstname Lastname <email>``. (strict)
 
-.. tip:: Follow the foregoing convention particularly where it says 
-   ``(strict)`` or you will be asked to modify your commit to comply with 
+.. tip:: Follow the foregoing convention particularly where it says
+   ``(strict)`` or you will be asked to modify your commit to comply with
    this convention.
 
-The following is a common commit comment (preferred):: 
+The following is a common commit comment (preferred)::
 
 	doc: Fixes a spelling error and a broken hyperlink.
-	
+
 	Signed-off-by: John Doe <john.doe@gmail.com>
 
 
-The following comment includes a reference to a bug. :: 
+The following comment includes a reference to a bug. ::
 
 	doc: Fixes a spelling error and a broken hyperlink.
 
 	Fixes: https://tracker.ceph.com/issues/1234
-	
+
 	Signed-off-by: John Doe <john.doe@gmail.com>
 
 
 The following comment includes a terse sentence following the comment summary.
-There is a carriage return between the summary line and the description:: 
+There is a carriage return between the summary line and the description::
 
 	doc: Added mon setting to monitor config reference
-	
+
 	Describes 'mon setting', which is a new setting added
 	to config_opts.h.
-	
+
 	Signed-off-by: John Doe <john.doe@gmail.com>
 
 
@@ -477,7 +478,7 @@ To commit changes, execute the following:
 .. prompt:: bash $
 
 	git commit -a
-	
+
 
 An easy way to manage your documentation commits is to use visual tools for
 ``git``. For example, ``gitk`` provides a graphical interface for viewing the
@@ -504,7 +505,7 @@ Then, execute:
 
 	cd {git-ceph-repo-path}
 	gitk
-	
+
 Finally, select **File->Start git gui** to activate the graphical user interface.
 
 
@@ -546,15 +547,15 @@ commits will be squashed into a single commit.
 #. Make the commits that you will later squash.
 
    #. Make the first commit.
-   
+
       ::
-   
+
          doc/glossary: improve "CephX" entry
-   
+
          Improve the glossary entry for "CephX".
-   
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-   
+
          # Please enter the commit message for your changes. Lines starting
          # with '#' will be ignored, and an empty message aborts the commit.
          #
@@ -562,18 +563,18 @@ commits will be squashed into a single commit.
          # Changes to be committed:
          #       modified:   glossary.rst
          #
-   
+
    #. Make the second commit.
-   
+
       ::
-   
+
          doc/glossary: add link to architecture doc
-         
+
          Add a link to a section in the architecture document, which link
          will be used in the process of improving the "CephX" glossary entry.
-         
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-      
+
             # Please enter the commit message for your changes. Lines starting
             # with '#' will be ignored, and an empty message aborts the commit.
             #
@@ -582,18 +583,18 @@ commits will be squashed into a single commit.
             #
             # Changes to be committed:
             #       modified:   architecture.rst
-      
+
    #. Make the third commit.
-   
+
       ::
-      
+
          doc/glossary: link to Arch doc in "CephX" glossary
-         
+
          Link to the Architecture document from the "CephX" entry in the
          Glossary.
-         
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-         
+
          # Please enter the commit message for your changes. Lines starting
          # with '#' will be ignored, and an empty message aborts the commit.
          #
@@ -604,24 +605,24 @@ commits will be squashed into a single commit.
          #       modified:   glossary.rst
 
 #. There are now three commits in the feature branch. We will now begin the
-   process of squashing them into a single commit. 
-   
-   #. Run the command ``git rebase -i main``, which rebases the current branch 
+   process of squashing them into a single commit.
+
+   #. Run the command ``git rebase -i main``, which rebases the current branch
       (the feature branch) against the ``main`` branch:
 
       .. prompt:: bash
-   
+
          git rebase -i main
-   
+
    #. A list of the commits that have been made to the feature branch now
       appear, and looks like this:
 
       ::
-      
+
          pick d395e500883 doc/glossary: improve "CephX" entry
          pick b34986e2922 doc/glossary: add link to architecture doc
          pick 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
-         
+
          # Rebase 0793495b9d1..74d0719735c onto 0793495b9d1 (3 commands)
          #
          # Commands:
@@ -650,7 +651,7 @@ commits will be squashed into a single commit.
          #
          # If you remove a line here THAT COMMIT WILL BE LOST.
 
-      Find the part of the screen that says "pick". This is the part that you will 
+      Find the part of the screen that says "pick". This is the part that you will
       alter. There are three commits that are currently labeled "pick". We will
       choose one of them to remain labeled "pick", and we will label the other two
       commits "squash".
@@ -662,7 +663,7 @@ commits will be squashed into a single commit.
       pick d395e500883 doc/glossary: improve "CephX" entry
       squash b34986e2922 doc/glossary: add link to architecture doc
       squash 74d0719735c doc/glossary: link to Arch doc in "CephX" glossary
-      
+
       # Rebase 0793495b9d1..74d0719735c onto 0793495b9d1 (3 commands)
       #
       # Commands:
@@ -699,34 +700,34 @@ commits will be squashed into a single commit.
       like this:
 
       ::
-      
+
          # This is a combination of 3 commits.
          # This is the 1st commit message:
-      
+
          doc/glossary: improve "CephX" entry
-      
+
          Improve the glossary entry for "CephX".
-      
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-      
+
          # This is the commit message #2:
-      
+
          doc/glossary: add link to architecture doc
-      
+
          Add a link to a section in the architecture document, which link
          will be used in the process of improving the "CephX" glossary entry.
-      
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-      
+
          # This is the commit message #3:
-      
+
          doc/glossary: link to Arch doc in "CephX" glossary
-      
+
          Link to the Architecture document from the "CephX" entry in the
          Glossary.
-      
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-      
+
          # Please enter the commit message for your changes. Lines starting
          # with '#' will be ignored, and an empty message aborts the commit.
          #
@@ -742,17 +743,17 @@ commits will be squashed into a single commit.
          # Changes to be committed:
          #       modified:   doc/architecture.rst
          #       modified:   doc/glossary.rst
-      
-   #. The commit messages have been revised into the simpler form presented here:   
-            
+
+   #. The commit messages have been revised into the simpler form presented here:
+
       ::
-      
+
          doc/glossary: improve "CephX" entry
-      
+
          Improve the glossary entry for "CephX".
-      
+
          Signed-off-by: Zac Dover <zac.dover@proton.me>
-      
+
          # Please enter the commit message for your changes. Lines starting
          # with '#' will be ignored, and an empty message aborts the commit.
          #
@@ -771,13 +772,13 @@ commits will be squashed into a single commit.
 
 #. Force push the squashed commit from your local working copy to the remote
    upstream branch. The force push is necessary because the newly squashed commit
-   does not have an ancestor in the remote. If that confuses you, just run this 
+   does not have an ancestor in the remote. If that confuses you, just run this
    command and don't think too much about it:
 
-   .. prompt:: bash $  
+   .. prompt:: bash $
 
       git push -f
-   
+
    ::
 
       Enumerating objects: 9, done.
@@ -821,17 +822,17 @@ Review the following style guides to maintain this consistency.
 Headings
 --------
 
-#. **Document Titles:** Document titles use the ``=`` character overline and 
-   underline with a leading and trailing space on the title text line. 
+#. **Document Titles:** Document titles use the ``=`` character overline and
+   underline with a leading and trailing space on the title text line.
    See `Document Title`_ for details.
 
 #. **Section Titles:** Section tiles use the ``=`` character underline with no
-   leading or trailing spaces for text. Two carriage returns should precede a 
+   leading or trailing spaces for text. Two carriage returns should precede a
    section title (unless an inline reference precedes it). See `Sections`_ for
    details.
 
-#. **Subsection Titles:** Subsection titles use the ``_`` character underline 
-   with no leading or trailing spaces for text.  Two carriage returns should 
+#. **Subsection Titles:** Subsection titles use the ``_`` character underline
+   with no leading or trailing spaces for text.  Two carriage returns should
    precede a subsection title (unless an inline reference precedes it).
 
 
@@ -843,18 +844,18 @@ a command line interface without leading or trailing white space. Where
 possible, we prefer to maintain this convention with text, lists, literal text
 (exceptions allowed), tables, and ``ditaa`` graphics.
 
-#. **Paragraphs**: Paragraphs have a leading and a trailing carriage return, 
-   and should be 80 characters wide or less so that the documentation can be 
+#. **Paragraphs**: Paragraphs have a leading and a trailing carriage return,
+   and should be 80 characters wide or less so that the documentation can be
    read in native format in a command line terminal.
 
 #. **Literal Text:** To create an example of literal text (e.g., command line
    usage), terminate the preceding paragraph with ``::`` or enter a carriage
    return to create an empty line after the preceding paragraph; then, enter
    ``::`` on a separate line followed by another empty line. Then, begin the
-   literal text with tab indentation (preferred) or space indentation of 3 
+   literal text with tab indentation (preferred) or space indentation of 3
    characters.
 
-#. **Indented Text:** Indented text such as bullet points 
+#. **Indented Text:** Indented text such as bullet points
    (e.g., ``- some text``) may span multiple lines. The text of subsequent
    lines should begin at the same character position as the text of the
    indented text (less numbers, bullets, etc.).
@@ -867,13 +868,13 @@ possible, we prefer to maintain this convention with text, lists, literal text
 
 #. **Numbered Lists:** Numbered lists should use autonumbering by starting
    a numbered indent with ``#.`` instead of the actual number so that
-   numbered paragraphs can be repositioned without requiring manual 
+   numbered paragraphs can be repositioned without requiring manual
    renumbering.
 
-#. **Code Examples:** Ceph supports the use of the 
-   ``.. code-block::<language>`` role, so that you can add highlighting to 
-   source examples. This is preferred for source code. However, use of this 
-   tag will cause autonumbering to restart at 1 if it is used as an example 
+#. **Code Examples:** Ceph supports the use of the
+   ``.. code-block::<language>`` role, so that you can add highlighting to
+   source examples. This is preferred for source code. However, use of this
+   tag will cause autonumbering to restart at 1 if it is used as an example
    within a numbered list. See `Showing code examples`_ for details.
 
 
@@ -894,12 +895,12 @@ The Ceph project uses `paragraph level markup`_ to highlight points.
 #. **Version Added:** Use the ``.. versionadded::`` directive for new features
    or configuration settings so that users know the minimum release for using
    a feature.
-   
+
 #. **Version Changed:** Use the ``.. versionchanged::`` directive for changes
    in usage or configuration settings.
 
-#. **Deprecated:** Use the ``.. deprecated::`` directive when CLI usage, 
-   a feature or a configuration setting is no longer preferred or will be 
+#. **Deprecated:** Use the ``.. deprecated::`` directive when CLI usage,
+   a feature or a configuration setting is no longer preferred or will be
    discontinued.
 
 #. **Topic:** Use the ``.. topic::`` directive to encapsulate text that is
@@ -917,7 +918,7 @@ Every document (every ``.rst`` file) in the Sphinx-controlled Ceph
 documentation suite must be linked either (1) from another document in the
 documentation suite or (2) from a table of contents (TOC). If any document in
 the documentation suite is not linked in this way, the ``build-doc`` script
-generates warnings when it tries to build the documentation. 
+generates warnings when it tries to build the documentation.
 
 The Ceph project uses the ``.. toctree::`` directive. See `The TOC tree`_ for
 details. When rendering a table of contents (TOC), specify the ``:maxdepth:``
@@ -943,16 +944,16 @@ to refer explicitly to the title of the section being linked to.
 
 For example, RST that links to the Sphinx Python Document Generator homepage
 and generates a sentence reading "Click here to learn more about Python
-Sphinx." looks like this: 
+Sphinx." looks like this:
 
 ::
 
     ``Click `here <https://www.sphinx-doc.org>`_ to learn more about Python
-    Sphinx.`` 
+    Sphinx.``
 
 And here it is, rendered:
 
-Click `here <https://www.sphinx-doc.org>`_ to learn more about Python Sphinx. 
+Click `here <https://www.sphinx-doc.org>`_ to learn more about Python Sphinx.
 
 Pay special attention to the underscore after the backtick. If you forget to
 include it and this is your first day working with RST, there's a chance that
@@ -998,8 +999,8 @@ addresses external to the Ceph documentation:
    `inline text <http:www.foo.com>`_
 
 .. note:: Do not fail to include the space between the inline text and the
-   less-than sign. 
-   
+   less-than sign.
+
    Do not fail to include the underscore after the final backtick.
 
    To link to addresses that are external to the Ceph documentation, include a
@@ -1041,7 +1042,7 @@ Link to target with inline text::
 
    :ref:`inline text<target>`
 
-.. note:: 
+.. note::
 
    There is no space between "inline text" and the angle bracket that
    immediately follows it. This is precisely the opposite of :ref:`the
@@ -1053,7 +1054,7 @@ Escaping Bold Characters within Words
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 This section explains how to make certain letters within a word bold while
-leaving the other letters in the word regular (non-bold). 
+leaving the other letters in the word regular (non-bold).
 
 The following single-line paragraph provides an example of this:
 
diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index c1bff769b289..e41c2eaa45e1 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -5,17 +5,17 @@
 ==========================
 
 Ceph is designed to run on commodity hardware, which makes building and
-maintaining petabyte-scale data clusters flexible and economically feasible. 
-When planning your cluster's hardware, you will need to balance a number 
+maintaining petabyte-scale data clusters flexible and economically feasible.
+When planning your cluster's hardware, you will need to balance a number
 of considerations, including failure domains, cost, and performance.
-Hardware planning should include distributing Ceph daemons and 
-other processes that use Ceph across many hosts. Generally, we recommend 
-running Ceph daemons of a specific type on a host configured for that type 
-of daemon. We recommend using separate hosts for processes that utilize your 
-data cluster (e.g., OpenStack, CloudStack, Kubernetes, etc).
+Hardware planning should include distributing Ceph daemons and
+other processes that use Ceph across many hosts. Generally, we recommend
+running Ceph daemons of a specific type on a host configured for that type
+of daemon. We recommend using separate hosts for processes that utilize your
+data cluster (e.g., OpenStack, OpenNebula, CloudStack, Kubernetes, etc).
 
 The requirements of one Ceph cluster are not the same as the requirements of
-another, but below are some general guidelines. 
+another, but below are some general guidelines.
 
 .. tip:: check out the `ceph blog`_ too.
 
@@ -106,7 +106,7 @@ that the OSD attempts to consume by changing the :confval:`osd_memory_target`
 configuration option.
 
 - Setting the :confval:`osd_memory_target` below 2GB is not
-  recommended. Ceph may fail to keep the memory consumption under 2GB and 
+  recommended. Ceph may fail to keep the memory consumption under 2GB and
   extremely slow performance is likely.
 
 - Setting the memory target between 2GB and 4GB typically works but may result
@@ -118,7 +118,7 @@ configuration option.
   OSD performance.
 
 - Setting the :confval:`osd_memory_target` higher than 4GB can improve
-  performance when there many (small) objects or when large (256GB/OSD 
+  performance when there many (small) objects or when large (256GB/OSD
   or more) data sets are processed.  This is especially true with fast
   NVMe OSDs.
 
@@ -130,7 +130,7 @@ configuration option.
    fragmented huge pages. Modern versions of Ceph disable transparent huge
    pages at the application level to avoid this, but that does not
    guarantee that the kernel will immediately reclaim unmapped memory. The OSD
-   may still at times exceed its memory target. We recommend budgeting 
+   may still at times exceed its memory target. We recommend budgeting
    at least 20% extra memory on your system to prevent OSDs from going OOM
    (**O**\ut **O**\f **M**\emory) during temporary spikes or due to delay in
    the kernel reclaiming freed pages. That 20% value might be more or less than
@@ -193,11 +193,11 @@ per gigabyte (i.e., $150 / 3072 = 0.0488). In the foregoing example, using the
 .. tip:: Hosting multiple OSDs on a single SAS / SATA HDD
    is **NOT** a good idea.
 
-.. tip:: Hosting an OSD with monitor, manager, or MDS data on a single 
+.. tip:: Hosting an OSD with monitor, manager, or MDS data on a single
    drive is also **NOT** a good idea.
 
 .. tip:: With spinning disks, the SATA and SAS interface increasingly
-   becomes a bottleneck at larger capacities. See also the `Storage Networking 
+   becomes a bottleneck at larger capacities. See also the `Storage Networking
    Industry Association's Total Cost of Ownership calculator`_.
 
 
@@ -219,7 +219,7 @@ Solid State Drives
 ------------------
 
 Ceph performance is much improved when using solid-state drives (SSDs). This
-reduces random access time and reduces latency while increasing throughput. 
+reduces random access time and reduces latency while increasing throughput.
 
 SSDs cost more per gigabyte than do HDDs but SSDs often offer
 access times that are, at a minimum, 100 times faster than HDDs.
@@ -236,10 +236,10 @@ to many of the limitations of HDDs.  SSDs do have significant
 limitations though. When evaluating SSDs, it is important to consider the
 performance of sequential and random reads and writes.
 
-.. important:: We recommend exploring the use of SSDs to improve performance. 
+.. important:: We recommend exploring the use of SSDs to improve performance.
    However, before making a significant investment in SSDs, we **strongly
    recommend** reviewing the performance metrics of an SSD and testing the
-   SSD in a test configuration in order to gauge performance. 
+   SSD in a test configuration in order to gauge performance.
 
 Relatively inexpensive SSDs may appeal to your sense of economy. Use caution.
 Acceptable IOPS are not the only factor to consider when selecting SSDs for
@@ -317,7 +317,7 @@ An HBA-free system may also cost hundreds of US dollars less every year if one
 purchases an annual maintenance contract or extended warranty.
 
 .. tip:: The `Ceph blog`_ is often an excellent source of information on Ceph
-   performance issues. See `Ceph Write Throughput 1`_ and `Ceph Write 
+   performance issues. See `Ceph Write Throughput 1`_ and `Ceph Write
    Throughput 2`_ for additional details.
 
 
@@ -490,7 +490,7 @@ The faster that a placement group (PG) can recover from a degraded state to
 an ``active + clean`` state, the better. Notably, fast recovery minimizes
 the likelihood of multiple, overlapping failures that can cause data to become
 temporarily unavailable or even lost. Of course, when provisioning your
-network, you will have to balance price against performance. 
+network, you will have to balance price against performance.
 
 Some deployment tools employ VLANs to make hardware and network cabling more
 manageable. VLANs that use the 802.1q protocol require VLAN-capable NICs and
@@ -520,7 +520,7 @@ carefully consider before deploying a large scale data cluster.
 Additionally BMCs as of 2023 rarely sport network connections faster than 1 Gb/s,
 so dedicated and inexpensive 1 Gb/s switches for BMC administrative traffic
 may reduce costs by wasting fewer expenive ports on faster host switches.
- 
+
 
 Failure Domains
 ===============

From 834556fa332f3b543ecb2737dfbd12ec7de86a19 Mon Sep 17 00:00:00 2001
From: "daegon.yang" <daegon.yang@samsung.com>
Date: Mon, 18 Dec 2023 06:12:24 +0000
Subject: [PATCH 1093/2492] vstart: Pick only CIDR-formatted routes when
 cephadm enabled

When cephadm is enabled, the script for determining the public_network
value from 'ip route list' output was previously capturing all routes
associated with the specified IP. This included non-CIDR formatted
entries such as specific IP routes (e.g., 8.8.8.8) and gateway
addresses, leading to the selection of multiple, potentially
irrelevant entries. This behavior resulted in an issue where vstart
could not correctly identify the network, causing it to terminate
unexpectedly.

This commit adds a grep command to ensure only CIDR network formats are
identified.

Signed-off-by: daegon.yang <daegon.yang@samsung.com>
---
 src/vstart.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vstart.sh b/src/vstart.sh
index 634fb74ac874..46c1b85a8d20 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1587,7 +1587,7 @@ EOF
     fi
     if [ "$cephadm" -gt 0 ]; then
         debug echo Setting mon public_network ...
-        public_network=$(ip route list | grep -w "$IP" | grep -v default | awk '{print $1}')
+        public_network=$(ip route list | grep -w "$IP" | grep -v default | grep -E "/[0-9]+" | awk '{print $1}')
         ceph_adm config set mon public_network $public_network
     fi
 fi

From 36dd26d0f62b4c3ca48cd80898c793c9509f5e4c Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 18 Dec 2023 15:16:38 +0800
Subject: [PATCH 1094/2492] cmake: add deps to vstart and vstart-base with loop

so we don't have to repeat the settings enabling a given target
when adding the deps to vstart and vstart-base

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/CMakeLists.txt | 59 ++++++++++++++++++----------------------------
 1 file changed, 23 insertions(+), 36 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 5b1300bb84a3..0454d05dc179 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -947,44 +947,31 @@ add_custom_target(vstart-base DEPENDS
     monmaptool
     crushtool
     rados)
-if(NOT WIN32)
-  # WIN32 port does not build python bindings
-  # TODO: introduce an option for enabling python binding
-  add_dependencies(vstart-base
-    cython_rados)
-endif()
-
-if (WITH_MGR)
-  add_dependencies(vstart-base ceph-mgr)
-  add_dependencies(vstart-base ceph-exporter)
-endif()
+foreach(dep
+    cython_rados
+    ceph-mgr
+    ceph-exporter)
+  if(TARGET ${dep})
+    add_dependencies(vstart-base ${dep})
+  endif()
+endforeach()
 
 add_custom_target(vstart DEPENDS vstart-base)
-if(WITH_RBD AND NOT WIN32)
-  add_dependencies(vstart cython_rbd)
-endif()
-if (WITH_CEPHFS)
-  add_dependencies(vstart ceph-mds cephfs cython_cephfs)
-endif()
-if(WITH_RADOSGW)
-  add_dependencies(vstart radosgw radosgw-admin)
-endif()
-
-if(WITH_LTTNG)
-  add_dependencies(vstart tracepoint_libraries)
-endif(WITH_LTTNG)
-
-if(WITH_MGR AND WITH_MGR_DASHBOARD_FRONTEND AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64")
-  add_dependencies(vstart mgr-dashboard-frontend-build)
-endif()
-
-if(WITH_MGR)
-  add_dependencies(vstart ceph-volume-venv-setup)
-endif()
-
-if(WITH_MGR)
-  add_dependencies(vstart cephadm)
-endif()
+foreach(dep
+    cython_rbd
+    cephfs
+    cython_cephfs
+    ceph-mds
+    mgr-dashboard-frontend-build
+    radosgw
+    radosgw-admin
+    tracepoint_libraries
+    ceph-volume-venv-setup
+    cephadm)
+  if(TARGET ${dep})
+    add_dependencies(vstart ${dep})
+  endif()
+endforeach()
 
 # Everything you need to run CephFS tests
 add_custom_target(cephfs_testing DEPENDS

From 476f904cbc47f107f92522c01525a3568fd147c0 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Fri, 15 Dec 2023 18:18:01 +0800
Subject: [PATCH 1095/2492] cmake: differentiate WITH_CEPHFS from
 WITH_LIBCEPHFS

WITH_CEPHFS is used for enabling the server side of cephfs, i.e., mds,
while WITH_LIBCEPHFS is for the client side of it, namely libcephfs
and its python bindings. since we have these two different options,
in theory, user is allowed to build with WITH_CEPHFS=ON and
WITH_LIBCEPHFS=OFF, and in that case, `vstart` should not depend on
the non-existent target of cephfs and cython_cephfs. the same applies to
the build of cython_cephfs. this is an unusual combination, but
it is a valid one.

in this change, we

* build the python binding only if WITH_LIBCEPHFS is ON
* add cephfs and cython_cephfs as dependencies of vstart only if
  WITH_LIBCEPHFS is ON

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/pybind/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/CMakeLists.txt b/src/pybind/CMakeLists.txt
index b01c49b6215a..fb357700a470 100644
--- a/src/pybind/CMakeLists.txt
+++ b/src/pybind/CMakeLists.txt
@@ -12,7 +12,7 @@ if(WITH_RBD)
   add_subdirectory(rbd)
   add_dependencies(cython_modules cython_rbd)
 endif()
-if(WITH_CEPHFS)
+if(WITH_LIBCEPHFS)
   add_subdirectory(cephfs)
   add_dependencies(cython_modules cython_cephfs)
 endif()

From 171da5b5cb75022a27af6e77a74c752f9247ed9c Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 30 Aug 2023 08:57:18 +0000
Subject: [PATCH 1096/2492] osd/OSD: introduce trim_stale_maps

```
  /**
   * trim_stale_maps
   *
   * trim_maps had a possible (rare) leak which resulted in stale osdmaps.
   * This method will cleanup any existing osdmaps from the store
   * with an epoch earlier than the superblock's oldest_map epoch.
   * See: https://tracker.ceph.com/issues/61962
   */
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/osd/OSD.h  | 12 +++++++++
 2 files changed, 82 insertions(+)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a79b388deca4..a14d70605f8a 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3078,6 +3078,19 @@ will start to track new ops received afterwards.";
       f, false, logger, counter);
   }
 
+  else if (prefix == "trim stale osdmaps") {
+    // osd_lock is not taken since trimming won't go past
+    // the superblock's oldest_map. The superblock won't
+    // be updated. Only the stored stale (no longer referenced)
+    // osdmaps are removed.
+    int ret = trim_stale_maps();
+    if (ret < 0) {
+     ss << " Error trimming stale osdmaps: " << cpp_strerror(ret);
+     goto out;
+    }
+    dout(20) << fmt::format("Trimmed {} osdmaps", ret) << dendl;
+  }
+
   else if (prefix == "cache drop") {
     lock_guard l(osd_lock);
     dout(20) << "clearing all caches" << dendl;
@@ -4303,6 +4316,12 @@ void OSD::final_init()
     asok_hook,
     "reset pg recovery statistics");
   ceph_assert(r == 0);
+  r = admin_socket->register_command(
+    "trim stale osdmaps",
+    asok_hook,
+    "cleanup any existing osdmap from the store "
+    "in the range of 0 up to the superblock's oldest_map.");
+  ceph_assert(r == 0);
   r = admin_socket->register_command(
     "cache drop",
     asok_hook,
@@ -7877,6 +7896,57 @@ void OSD::trim_maps(epoch_t oldest)
   ceph_assert(min <= service.map_cache.cached_key_lower_bound());
 }
 
+std::optional<epoch_t> OSD::get_epoch_from_osdmap_object(const ghobject_t& osdmap) {
+  const auto& name = osdmap.hobj.oid.name;
+  string osdmap_prefix = "osdmap.";
+  auto osdmap_pos = name.find(osdmap_prefix);
+  if (osdmap_pos == string::npos) {
+    return std::nullopt;
+  }
+  auto osdmap_string = name.substr(osdmap_pos + osdmap_prefix.size());
+  return stoul(osdmap_string);
+}
+
+int OSD::trim_stale_maps()
+{
+  int num_removed = 0;
+  vector<ghobject_t> objects;
+  int r = store->collection_list(
+      service.meta_ch,
+      ghobject_t{},
+      ghobject_t::get_max(),
+      INT_MAX,
+      &objects,
+      NULL);
+  if (r != 0) {
+    derr << __func__ << " list collection " << service.meta_ch
+         << " got: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  ObjectStore::Transaction t;
+  for (const auto& osdmap_obj : objects) {
+    if (auto epoch = get_epoch_from_osdmap_object(osdmap_obj);
+        epoch.has_value() && epoch < superblock.get_oldest_map()) {
+      dout(20) << __func__ << " removing stale osdmap epoch "
+               << epoch << dendl;
+      t.remove(coll_t::meta(), osdmap_obj);
+    }
+    if (t.get_num_ops() > cct->_conf->osd_target_transaction_size) {
+      num_removed += t.get_num_ops();
+      int tr = store->queue_transaction(service.meta_ch, t.claim_and_reset(), nullptr);
+      ceph_assert(tr == 0);
+    }
+  }
+
+  if (t.get_num_ops() > 0) {
+    num_removed += t.get_num_ops();
+    int tr = store->queue_transaction(service.meta_ch, t.claim_and_reset(), nullptr);
+    ceph_assert(tr == 0);
+  }
+
+  return num_removed;
+}
+
 void OSD::handle_osd_map(MOSDMap *m)
 {
   // wait for pgs to catch up
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index a6b7a3c31ed3..83aed8a410bf 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1672,6 +1672,18 @@ class OSD : public Dispatcher,
   void note_up_osd(int osd);
   friend struct C_OnMapCommit;
 
+  std::optional<epoch_t> get_epoch_from_osdmap_object(const ghobject_t& osdmap);
+  /**
+   * trim_stale_maps
+   *
+   * trim_maps had a possible (rare) leak which resulted in stale osdmaps.
+   * This method will cleanup any existing osdmap from the store
+   * in the range of 0 up to the superblock's oldest_map.
+   * @return number of stale osdmaps which were removed.
+   * See: https://tracker.ceph.com/issues/61962
+   */
+  int trim_stale_maps();
+
   bool advance_pg(
     epoch_t advance_to,
     PG *pg,

From 7ea40f0ee611eee917b331ad4ac0645bd25c2618 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 26 Oct 2023 10:18:59 +0000
Subject: [PATCH 1097/2492] qa/tasks/ceph_manager: thrash - add trim stale
 osdmaps

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/tasks/ceph_manager.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 86b57028ee27..0b6170ff5b0e 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -235,6 +235,7 @@ def __init__(self, manager, config, name, logger):
         self.random_eio = self.config.get('random_eio')
         self.chance_force_recovery = self.config.get('chance_force_recovery', 0.3)
         self.chance_reset_purged_snaps_last = self.config.get('chance_reset_purged_snaps_last', 0.3)
+        self.chance_trim_stale_osdmaps = self.config.get('chance_trim_stale_osdmaps', 0.3)
 
         num_osds = self.in_osds + self.out_osds
         self.max_pgs = self.config.get("max_pgs_per_pool_osd", 1200) * len(num_osds)
@@ -793,6 +794,19 @@ def reset_purged_snaps_last(self):
             except CommandFailedError:
                 self.log('Failed to reset_purged_snaps_last, ignoring')
 
+    def trim_stale_osdmaps(self):
+       """
+       Trim stale osdmaps
+       """
+       self.log('trim_stale_osdmaps')
+       for osd in self.in_osds:
+           try:
+               self.ceph_manager.raw_cluster_cmd(
+               'tell', "osd.%s" % (str(osd)),
+               'trim stale osdmaps')
+           except CommandFailedError:
+               self.log('Failed to trim stale osdmaps, ignoring')
+
     def all_up(self):
         """
         Make sure all osds are up and not out.
@@ -1245,6 +1259,8 @@ def choose_action(self):
             actions.append((self.force_cancel_recovery, self.chance_force_recovery))
         if self.chance_reset_purged_snaps_last > 0:
             actions.append((self.reset_purged_snaps_last, self.chance_reset_purged_snaps_last))
+        if self.chance_trim_stale_osdmaps > 0:
+            actions.append((self.trim_stale_osdmaps, self.chance_trim_stale_osdmaps))
 
         for key in ['heartbeat_inject_failure', 'filestore_inject_stall']:
             for scenario in [

From 66dceb24d6d0779f87fd4ac3c1226534cc91e216 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 14 Nov 2023 14:36:58 +0000
Subject: [PATCH 1098/2492] osd/OSD: cleanup send_incremental_map

See: 9fba69a11aa940ed36339bb24b05cb92165db516

build_incremental_map_msg() no longer returns NULL on failures.
while(!m) if-case is removed.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a79b388deca4..d62695ae1c62 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1457,22 +1457,19 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
            << " to " << con << " " << con->get_peer_addr() << dendl;
 
   MOSDMap *m = NULL;
-  while (!m) {
-    OSDSuperblock sblock(get_superblock());
-    if (since < sblock.get_oldest_map()) {
-      // just send latest full map
-      MOSDMap *m = new MOSDMap(monc->get_fsid(),
-			       osdmap->get_encoding_features());
-      m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
-      m->newest_map = sblock.get_newest_map();
-      get_map_bl(to, m->maps[to]);
-      send_map(m, con);
-      return;
-    }
-
+  OSDSuperblock sblock(get_superblock());
+  if (since < sblock.get_oldest_map()) {
+    // just send latest full map
+    m = new MOSDMap(monc->get_fsid(),
+		       osdmap->get_encoding_features());
+    m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
+    m->newest_map = sblock.get_newest_map();
+    get_map_bl(to, m->maps[to]);
+  } else {
     if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {
-      dout(10) << "  " << (to - since) << " > max " << cct->_conf->osd_map_share_max_epochs
-	       << ", only sending most recent" << dendl;
+      dout(10) << "  " << (to - since) << " > max "
+               << cct->_conf->osd_map_share_max_epochs
+               << ", only sending most recent" << dendl;
       since = to - cct->_conf->osd_map_share_max_epochs;
     }
 

From bd4d481307bfb1befbf4568a5a0a64300140ff6e Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 10:19:51 +0000
Subject: [PATCH 1099/2492] osd/OSD: optimize send_incremental_map

Previosuly, if `since` was earlier than the oldest_map,
only the lastest full osdmap would be sent.
However, we could actually send inc/full maps from oldest_map
up to the current epoch.

Moreover, build_incremental_map_msg is suited to handle this case as well.
if `since` is earlier than `cluster_osdmap_trim_lower_bound` it
will be adjusted accordingly.
At any point, the cluster_osdmap_trim_lower_bound is later (or equal) to
the superblock's oldest_map. See: past_intervals.rst (OSDSuperblock::maps)
Hence, if since < oldest_map, and oldest_map <= clutser_lower_bound
then since < cluster_lower_bound.

Note: oldest_map stands as a mark to indicate up until which epoch
      the current OSD trimmed its osdmaps. See OSD::trim_maps().
      Meaning, the OSDMaps in the range of [oldest_map, trim_lower_bound]
      may be at hand to be shared. Nevertheless, we should actually use
      the cluster_trim_lower_bound to set the correct range.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index d62695ae1c62..136e51cccc81 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1458,23 +1458,13 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
 
   MOSDMap *m = NULL;
   OSDSuperblock sblock(get_superblock());
-  if (since < sblock.get_oldest_map()) {
-    // just send latest full map
-    m = new MOSDMap(monc->get_fsid(),
-		       osdmap->get_encoding_features());
-    m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
-    m->newest_map = sblock.get_newest_map();
-    get_map_bl(to, m->maps[to]);
-  } else {
-    if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {
-      dout(10) << "  " << (to - since) << " > max "
-               << cct->_conf->osd_map_share_max_epochs
-               << ", only sending most recent" << dendl;
-      since = to - cct->_conf->osd_map_share_max_epochs;
-    }
-
-    m = build_incremental_map_msg(since, to, sblock);
+  if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {
+    dout(10) << "  " << (to - since) << " > max "
+             << cct->_conf->osd_map_share_max_epochs
+             << ", only sending most recent" << dendl;
+    since = to - cct->_conf->osd_map_share_max_epochs;
   }
+  m = build_incremental_map_msg(since, to, sblock);
   send_map(m, con);
 }
 

From 3c8226058db5000543b9b065f3a23d334abba30d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 10:24:29 +0000
Subject: [PATCH 1100/2492] osd/OSD: further cleanup send_incremental_map()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 136e51cccc81..04e9b4b6d545 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1456,7 +1456,6 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
   dout(10) << "send_incremental_map " << since << " -> " << to
            << " to " << con << " " << con->get_peer_addr() << dendl;
 
-  MOSDMap *m = NULL;
   OSDSuperblock sblock(get_superblock());
   if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {
     dout(10) << "  " << (to - since) << " > max "
@@ -1464,8 +1463,7 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
              << ", only sending most recent" << dendl;
     since = to - cct->_conf->osd_map_share_max_epochs;
   }
-  m = build_incremental_map_msg(since, to, sblock);
-  send_map(m, con);
+  con->send_message(build_incremental_map_msg(since, to, sblock));
 }
 
 bool OSDService::_get_map_bl(epoch_t e, bufferlist& bl)

From 88dfd657d8026f692974776ea1c309e7da8ceb30 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 10:24:51 +0000
Subject: [PATCH 1101/2492] osd/OSD: remove unused send_map()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 5 -----
 src/osd/OSD.h  | 1 -
 2 files changed, 6 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 04e9b4b6d545..d4b1fa670992 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1444,11 +1444,6 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
   return m;
 }
 
-void OSDService::send_map(MOSDMap *m, Connection *con)
-{
-  con->send_message(m);
-}
-
 void OSDService::send_incremental_map(epoch_t since, Connection *con,
                                       const OSDMapRef& osdmap)
 {
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index a6b7a3c31ed3..ef7c2b0ab659 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -209,7 +209,6 @@ class OSDService : public Scrub::ScrubSchedListener {
 		       const OSDMapRef& osdmap,
 		       epoch_t peer_epoch_lb=0);
 
-  void send_map(class MOSDMap *m, Connection *con);
   void send_incremental_map(epoch_t since, Connection *con,
 			    const OSDMapRef& osdmap);
   MOSDMap *build_incremental_map_msg(epoch_t from, epoch_t to,

From a88f039d9bc6baf0f71a0c409b18e13694c9173a Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 11:06:56 +0000
Subject: [PATCH 1102/2492] osd/OSD: send_incremental_map pass correct
 send_from

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index d4b1fa670992..d4629630a41b 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1402,7 +1402,7 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
     max_bytes -= bl.length();
     m->maps[since] = std::move(bl);
   }
-  for (epoch_t e = since + 1; e <= to; ++e) {
+  for (epoch_t e = since; e <= to; ++e) {
     bufferlist bl;
     if (get_inc_map_bl(e, bl)) {
       m->incremental_maps[e] = bl;
@@ -7372,8 +7372,9 @@ void OSDService::maybe_share_map(
                << session->projected_epoch << dendl;
       return;
     }
-
-    send_from = session->projected_epoch;
+    // send incremental maps in the range of:
+    // (projected_epoch, osdmap]
+    send_from = session->projected_epoch + 1;
     dout(10) << __func__ << ": con " << con->get_peer_addr()
              << " map epoch " << session->projected_epoch
              << " -> " << osdmap->get_epoch()

From 54505d524410a3562186c3c3b31ddd96afe4ccc3 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 16 Nov 2023 11:38:51 +0000
Subject: [PATCH 1103/2492] osd/OSD: send_incremental_map fix log message

from:
```
send_incremental_map 88 -> 88 to 0xab1c880 v2:172.21.5.151:6820/252144826
send_incremental_map 89 -> 89 to 0xab1a900 v2:172.21.5.151:6812/1097023676
```

to:
```
send_incremental_map epoch range: (75, 75] to osd.1 v2:172.21.5.151:6812/3634942927
send_incremental_map epoch range: (76, 77] to osd.1 v2:172.21.5.151:6812/3634942927
send_incremental_map epoch range: (78, 78] to osd.1 v2:172.21.5.151:6812/3634942927
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index d4629630a41b..c3d03d862336 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1448,8 +1448,10 @@ void OSDService::send_incremental_map(epoch_t since, Connection *con,
                                       const OSDMapRef& osdmap)
 {
   epoch_t to = osdmap->get_epoch();
-  dout(10) << "send_incremental_map " << since << " -> " << to
-           << " to " << con << " " << con->get_peer_addr() << dendl;
+  dout(10) << fmt::format("{} epoch range: ({}, {}] to {} {}",
+                          __func__, since, to,
+                          con->get_peer_entity_name().to_str(),
+                          con->get_peer_addr()) << dendl;
 
   OSDSuperblock sblock(get_superblock());
   if (to > since && (int64_t)(to - since) > cct->_conf->osd_map_share_max_epochs) {

From 58dc60e790212e61b9880fd68163c78fde764c69 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 19 Nov 2023 15:19:29 +0000
Subject: [PATCH 1104/2492] osd/OSD: rewrite build_incremental_map_msg()

* use lambda to maintain size limits.
* avoid copying by std::move the bl into the message.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/OSD.cc | 49 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index c3d03d862336..7d3bbe085a7e 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1377,6 +1377,19 @@ void OSDService::got_stop_ack()
 MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
                                                OSDSuperblock& sblock)
 {
+  auto get_map_and_adjust_counters = [] (
+    bufferlist& bl,
+    int& max,
+    ssize_t& max_bytes,
+    std::function<bool()> map_getter) {
+      if (!map_getter()) {
+        return false;
+      }
+      max--;
+      max_bytes -= bl.length();
+      return true;
+  };
+
   MOSDMap *m = new MOSDMap(monc->get_fsid(),
 			   osdmap->get_encoding_features());
   m->cluster_osdmap_trim_lower_bound = sblock.cluster_osdmap_trim_lower_bound;
@@ -1388,36 +1401,34 @@ MOSDMap *OSDService::build_incremental_map_msg(epoch_t since, epoch_t to,
   if (since < m->cluster_osdmap_trim_lower_bound) {
     // we don't have the next map the target wants, so start with a
     // full map.
-    bufferlist bl;
     dout(10) << __func__ << " cluster osdmap lower bound "
              << sblock.cluster_osdmap_trim_lower_bound
              << " > since " << since << ", starting with full map"
              << dendl;
     since = m->cluster_osdmap_trim_lower_bound;
-    if (!get_map_bl(since, bl)) {
-      derr << __func__ << " missing full map " << since << dendl;
+    if (bufferlist bl;
+        get_map_and_adjust_counters(bl, max, max_bytes, [&] { return get_map_bl(since, bl);})) {
+      m->maps[since] = std::move(bl);
+      ++since;
+    } else {
+      derr << __func__ << " missing full map after map gap " << since << dendl;
       goto panic;
     }
-    max--;
-    max_bytes -= bl.length();
-    m->maps[since] = std::move(bl);
   }
-  for (epoch_t e = since; e <= to; ++e) {
-    bufferlist bl;
-    if (get_inc_map_bl(e, bl)) {
-      m->incremental_maps[e] = bl;
+
+  for (epoch_t e = since; e <= to && max > 0 && max_bytes > 0; ++e) {
+    if (bufferlist bl;
+        get_map_and_adjust_counters(bl, max, max_bytes, [&] { return get_inc_map_bl(e, bl);})) {
+        m->incremental_maps[e] = std::move(bl);
     } else {
       dout(10) << __func__ << " missing incremental map " << e << dendl;
-      if (!get_map_bl(e, bl)) {
-	derr << __func__ << " also missing full map " << e << dendl;
-	goto panic;
+      if (bufferlist bl;
+          get_map_and_adjust_counters(bl, max, max_bytes, [&] { return get_map_bl(e, bl);})) {
+        m->maps[e] = std::move(bl);
+      } else {
+        derr << __func__ << " also missing full map " << e << dendl;
+        goto panic;
       }
-      m->maps[e] = bl;
-    }
-    max--;
-    max_bytes -= bl.length();
-    if (max <= 0 || max_bytes <= 0) {
-      break;
     }
   }
   return m;

From 10b1580ecfdafff38c02581eb45a469b2b2b556d Mon Sep 17 00:00:00 2001
From: Michael Fritch <mfritch@suse.com>
Date: Fri, 8 Dec 2023 11:17:29 -0700
Subject: [PATCH 1105/2492] cephadm: add runtime python minor version to RPM
 query macro

when multiple versions of python are installed, the `python3dist` macro
produces a multiline string causing later rpm query commands to fail.

Signed-off-by: Michael Fritch <mfritch@suse.com>
---
 src/cephadm/build.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/build.py b/src/cephadm/build.py
index 1634eeac5931..63e70d91d234 100755
--- a/src/cephadm/build.py
+++ b/src/cephadm/build.py
@@ -328,7 +328,7 @@ def _install_rpm_deps(tempdir, config):
 
 def _deps_from_rpm(tempdir, config, dinfo, pkg):
     # first, figure out what rpm provides a particular python lib
-    dist = f'python3dist({pkg})'.lower()
+    dist = f'python3.{sys.version_info.minor}dist({pkg})'.lower()
     try:
         res = subprocess.run(
             ['rpm', '-q', '--whatprovides', dist],

From 75d86c64b3315cea028d193d1d3c553cba11e528 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 22 Nov 2023 13:16:36 +0800
Subject: [PATCH 1106/2492] crimson/osd/pg_backend: whiteout objects if they
 will have clones when rolling back

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/ops_executer.cc | 10 +++++-----
 src/crimson/osd/pg_backend.cc   |  8 +++++---
 src/crimson/osd/pg_backend.h    |  4 +++-
 src/osd/osd_types.h             |  7 ++++++-
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 09b1a492576f..eb8d3b601c1b 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -638,8 +638,10 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
   case CEPH_OSD_OP_ROLLBACK:
     return do_write_op([this, &head=obc,
                         &osd_op](auto& backend, auto& os, auto& txn) {
-      return backend.rollback(os, osd_op, txn, *osd_op_params, delta_stats,
-                              head, pg->obc_loader);
+      ceph_assert(obc->ssc);
+      return backend.rollback(os, obc->ssc->snapset,
+			      osd_op, txn, *osd_op_params, delta_stats,
+                              head, pg->obc_loader, snapc);
     });
   case CEPH_OSD_OP_APPEND:
     return do_write_op([this, &osd_op](auto& backend, auto& os, auto& txn) {
@@ -666,9 +668,7 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
   case CEPH_OSD_OP_DELETE:
   {
     bool whiteout = false;
-    if (!obc->ssc->snapset.clones.empty() ||
-        (snapc.snaps.size() &&                      // there are snaps
-        snapc.snaps[0] > obc->ssc->snapset.seq)) {  // existing obj is old
+    if (should_whiteout(obc->ssc->snapset, snapc)) {  // existing obj is old
       logger().debug("{} has or will have clones, will whiteout {}",
                      __func__, obc->obs.oi.soid);
       whiteout = true;
diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index 5e3e7a91c0f7..981bfca26d8e 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -770,12 +770,14 @@ PGBackend::write_iertr::future<> PGBackend::writefull(
 
 PGBackend::rollback_iertr::future<> PGBackend::rollback(
   ObjectState& os,
+  const SnapSet &ss,
   const OSDOp& osd_op,
   ceph::os::Transaction& txn,
   osd_op_params_t& osd_op_params,
   object_stat_sum_t& delta_stats,
   crimson::osd::ObjectContextRef head,
-  crimson::osd::ObjectContextLoader& obc_loader)
+  crimson::osd::ObjectContextLoader& obc_loader,
+  const SnapContext &snapc)
 {
   const ceph_osd_op& op = osd_op.op;
   snapid_t snapid = (uint64_t)op.snap.snapid;
@@ -833,12 +835,12 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
     // if there's no snapshot, we delete the object;
     // otherwise, do nothing.
     crimson::ct_error::enoent::handle(
-    [this, &os, &snapid, &txn, &delta_stats] {
+    [this, &os, &snapid, &txn, &delta_stats, &snapc, &ss] {
       logger().debug("PGBackend::rollback: deleting head on {}"
                      " with snap_id of {}"
                      " because got ENOENT|whiteout on obc lookup",
                      os.oi.soid, snapid);
-      return remove(os, txn, delta_stats, false);
+      return remove(os, txn, delta_stats, should_whiteout(ss, snapc));
     }),
     rollback_ertr::pass_further{},
     crimson::ct_error::assert_all{"unexpected error in rollback"}
diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h
index fbad37d4c71c..aa26b2cd2c1e 100644
--- a/src/crimson/osd/pg_backend.h
+++ b/src/crimson/osd/pg_backend.h
@@ -197,12 +197,14 @@ class PGBackend
       rollback_ertr>;
   rollback_iertr::future<> rollback(
     ObjectState& os,
+    const SnapSet &ss,
     const OSDOp& osd_op,
     ceph::os::Transaction& txn,
     osd_op_params_t& osd_op_params,
     object_stat_sum_t& delta_stats,
     crimson::osd::ObjectContextRef head,
-    crimson::osd::ObjectContextLoader& obc_loader);
+    crimson::osd::ObjectContextLoader& obc_loader,
+    const SnapContext &snapc);
   write_iertr::future<> truncate(
     ObjectState& os,
     const OSDOp& osd_op,
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 8b86b0a36356..2d697773e492 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -5572,7 +5572,12 @@ WRITE_CLASS_ENCODER(SnapSet)
 
 std::ostream& operator<<(std::ostream& out, const SnapSet& cs);
 
-
+inline static const bool should_whiteout(
+  const SnapSet &ss,
+  const SnapContext &client_snapc) {
+  return !ss.clones.empty() ||
+    (!client_snapc.snaps.empty() && client_snapc.snaps[0] > ss.seq);
+}
 
 #define OI_ATTR "_"
 #define SS_ATTR "snapset"

From 31fb87e32649adf2d51f37ff562248ed0e6f5a3e Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 19 Dec 2023 00:19:39 -0500
Subject: [PATCH 1107/2492] mds: func name fixup, plus, and additional log
 message

... when tid isn't found in pending_notifies.

Found this by code reading when trying to RCA

        https://tracker.ceph.com/issues/54741

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDSTableServer.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDSTableServer.cc b/src/mds/MDSTableServer.cc
index 5514f1e78478..c5cb0b817e02 100644
--- a/src/mds/MDSTableServer.cc
+++ b/src/mds/MDSTableServer.cc
@@ -71,7 +71,7 @@ void MDSTableServer::handle_prepare(const cref_t<MMDSTableRequest> &req)
 
 void MDSTableServer::_prepare_logged(const cref_t<MMDSTableRequest> &req, version_t tid)
 {
-  dout(7) << "_create_logged " << *req << " tid " << tid << dendl;
+  dout(7) << __func__ << ": req=" << *req << " tid " << tid << dendl;
   mds_rank_t from = mds_rank_t(req->get_source().num());
 
   ceph_assert(g_conf()->mds_kill_mdstable_at != 2);
@@ -114,6 +114,8 @@ void MDSTableServer::handle_notify_ack(const cref_t<MMDSTableRequest> &m)
       dout(0) << "got unexpected notify ack for tid " <<  tid << " from mds." << from << dendl;
     }
   } else {
+    dout(0) << __func__ << ": tid=" << tid << " from mds." << from
+	    << " not tracked in pending notifies" << dendl;
   }
 }
 

From 8178d440ec94b7c9505de6de1c5825fd44033bcb Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Tue, 19 Dec 2023 15:15:02 +0800
Subject: [PATCH 1108/2492] erasure-code/isa/xor_op: avoid empty function if
 not ARM.

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/erasure-code/isa/xor_op.cc | 4 ++--
 src/erasure-code/isa/xor_op.h  | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/erasure-code/isa/xor_op.cc b/src/erasure-code/isa/xor_op.cc
index 4f507c9b2625..33f133539d5f 100644
--- a/src/erasure-code/isa/xor_op.cc
+++ b/src/erasure-code/isa/xor_op.cc
@@ -197,6 +197,7 @@ region_sse2_xor(char** src,
   return;
 }
 
+#if defined(__aarch64__) && defined(__ARM_NEON)
 void
 // -----------------------------------------------------------------------------
 region_neon_xor(char **src,
@@ -205,7 +206,6 @@ region_neon_xor(char **src,
                 unsigned size)
 // -----------------------------------------------------------------------------
 {
-#if defined(__aarch64__) && defined(__ARM_NEON)
   ceph_assert(!(size % EC_ISA_VECTOR_NEON_WORDSIZE));
   unsigned char *p = (unsigned char *)parity;
   unsigned char *vbuf[256] = { NULL };
@@ -232,6 +232,6 @@ region_neon_xor(char **src,
     vst1q_u64((uint64_t *)(p + 16), d0_2);
     p += EC_ISA_VECTOR_NEON_WORDSIZE;
   }
-#endif // __aarch64__ && __ARM_NEON
   return;
 }
+#endif // __aarch64__ && __ARM_NEON
diff --git a/src/erasure-code/isa/xor_op.h b/src/erasure-code/isa/xor_op.h
index 46304eee1cce..86b1645b616e 100644
--- a/src/erasure-code/isa/xor_op.h
+++ b/src/erasure-code/isa/xor_op.h
@@ -83,6 +83,7 @@ region_sse2_xor(char** src /* array of 64-byte aligned source pointer to xor */,
                 int src_size /* size of the source pointer array */,
                 unsigned size /* size of the region to xor */);
 
+#if defined(__aarch64__) && defined(__ARM_NEON)
 // -------------------------------------------------------------------------
 // compute region XOR like parity = src[0] ^ src[1] ... ^ src[src_size-1]
 // using NEON 32-byte operations
@@ -92,5 +93,5 @@ region_neon_xor(char** src    /* array of 64-byte aligned source pointer to xor
                 char* parity  /* 32-byte aligned output pointer containing the parity */,
                 int src_size  /* size of the source pointer array */,
                 unsigned size /* size of the region to xor */);
-
+#endif // __aarch64__ && __ARM_NEON
 #endif // EC_ISA_XOR_OP_H

From 4eb42ff15db14680c0798ceb752fa6578b76fb92 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 7 Dec 2023 22:58:52 +0530
Subject: [PATCH 1109/2492] mgr/dashboard: small fixes to directories view

Increased the precedence for Directories
Expand the node by default and select them as well

Fixes: https://tracker.ceph.com/issues/63754
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../cephfs-directories.component.html             |  2 +-
 .../cephfs-directories.component.scss             |  4 ++++
 .../cephfs-directories.component.ts               | 11 +++++++++++
 .../cephfs/cephfs-tabs/cephfs-tabs.component.html | 15 +++++++--------
 4 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html
index ce6cc71c5ea4..c4c4728bab5d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.html
@@ -10,7 +10,7 @@
              [class.fa-spin]="loadingIndicator"></i>
         </button>
       </div>
-      <div class="card-body">
+      <div class="card-body card-tree">
         <tree-root *ngIf="nodes"
                    [nodes]="nodes"
                    [options]="treeOptions">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.scss
index 3334f06182d6..5228f35426e0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.scss
@@ -15,3 +15,7 @@
     color: vv.$gray-900;
   }
 }
+
+.card-tree {
+  height: 50vh;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts
index 812176717a1d..55cf3b8ae75b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-directories/cephfs-directories.component.ts
@@ -289,6 +289,10 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges {
         this.updateTree();
         resolve(this.getChildren(path));
         this.setLoadingIndicator(path, false);
+
+        if (path === '/' && this.treeComponent.treeModel.activeNodes?.length === 0) {
+          this.selectNode(this.getNode('/'));
+        }
       });
     });
   }
@@ -314,6 +318,13 @@ export class CephfsDirectoriesComponent implements OnInit, OnChanges {
     if (!subTree) {
       this.getSubTree(dir.parent);
     }
+
+    if (dir.path === '/volumes') {
+      const innerNode = this.treeComponent.treeModel.getNodeById('/volumes');
+      if (innerNode) {
+        innerNode.expand();
+      }
+    }
     return {
       name: dir.name,
       id: dir.path,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
index 87268858be50..d21d47034f80 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
@@ -14,6 +14,13 @@
         <cd-cephfs-detail [data]="details"> </cd-cephfs-detail>
       </ng-template>
     </ng-container>
+    <ng-container ngbNavItem="directories">
+      <a ngbNavLink
+         i18n>Directories</a>
+      <ng-template ngbNavContent>
+        <cd-cephfs-directories [id]="id"></cd-cephfs-directories>
+      </ng-template>
+    </ng-container>
     <ng-container ngbNavItem="subvolumes">
       <a
       ngbNavLink
@@ -68,14 +75,6 @@
         </cd-cephfs-clients>
       </ng-template>
     </ng-container>
-    <ng-container ngbNavItem="directories">
-      <a
-      ngbNavLink
-      i18n>Directories</a>
-      <ng-template ngbNavContent>
-        <cd-cephfs-directories [id]="id"></cd-cephfs-directories>
-      </ng-template>
-    </ng-container>
     <ng-container ngbNavItem="performance-details">
       <a
       ngbNavLink

From f5c87693abcfc9a6a3869a3f52549384807fba9d Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 19 Dec 2023 19:15:57 +1000
Subject: [PATCH 1110/2492] doc/install: update "update submodules"

Remove misleading material that would give readers the wrong idea about
when stale submodules are present. This commit is made in response to
information given to me by Ilya Dryomov here: https://github.com/ceph/ceph/pull/54929#issuecomment-1859237986.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/install/clone-source.rst | 74 ++----------------------------------
 1 file changed, 4 insertions(+), 70 deletions(-)

diff --git a/doc/install/clone-source.rst b/doc/install/clone-source.rst
index 8e783fed8e83..453d544b6856 100644
--- a/doc/install/clone-source.rst
+++ b/doc/install/clone-source.rst
@@ -98,59 +98,7 @@ repository.
 Updating Submodules
 -------------------
 
-#. Determine whether your submodules are out of date:
-
-   .. prompt:: bash $
-
-      git status
-
-   A. If your submodules are up to date 
-         If your submodules are up to date, the following console output will
-         appear: 
-
-         ::
-   
-           On branch main
-           Your branch is up to date with 'origin/main'.
-           
-           nothing to commit, working tree clean
-   
-         If you see this console output, then your submodules are up to date.
-         You do not need this procedure.
-
-
-   B. If your submodules are not up to date 
-         If your submodules are not up to date, you will see a message that
-         includes a list of "untracked files". The example here shows such a
-         list, which was generated from a real situation in which the
-         submodules were no longer current. Your list of files will not be the
-         same as this list of files, but this list is provided as an example.
-         If in your case any untracked files are listed, then you should
-         continue to the next step of this procedure.
-
-         ::
-
-            On branch main
-            Your branch is up to date with 'origin/main'.
-            
-            Untracked files:
-              (use "git add <file>..." to include in what will be committed)
-            src/pybind/cephfs/build/
-            src/pybind/cephfs/cephfs.c
-            src/pybind/cephfs/cephfs.egg-info/
-            src/pybind/rados/build/
-            src/pybind/rados/rados.c
-            src/pybind/rados/rados.egg-info/
-            src/pybind/rbd/build/
-            src/pybind/rbd/rbd.c
-            src/pybind/rbd/rbd.egg-info/
-            src/pybind/rgw/build/
-            src/pybind/rgw/rgw.c
-            src/pybind/rgw/rgw.egg-info/
-            
-            nothing added to commit but untracked files present (use "git add" to track)
-
-#. If your submodules are out of date, run the following commands:
+If your submodules are out of date, run the following commands:
 
    .. prompt:: bash $
 
@@ -158,24 +106,10 @@ Updating Submodules
       git clean -fdx
       git submodule foreach git clean -fdx
 
-   If you still have problems with a submodule directory, use ``rm -rf
-   [directory name]`` to remove the directory. Then run ``git submodule update
-   --init --recursive --progress`` again.
-
-#. Run ``git status`` again:
-
-   .. prompt:: bash $
-
-      git status
-   
-   Your submodules are up to date if you see the following message:
-
-   ::
+If you still have problems with a submodule directory, use ``rm -rf [directory
+name]`` to remove the directory. Then run ``git submodule update --init
+--recursive --progress`` again.
 
-     On branch main
-     Your branch is up to date with 'origin/main'.
-     
-     nothing to commit, working tree clean
 
 Choose a Branch
 ===============

From 8615731637a116f7b9299c6122a0e058d43a4f6d Mon Sep 17 00:00:00 2001
From: Dominique Leuenberger <dimstar@opensuse.org>
Date: Tue, 19 Dec 2023 13:28:29 +0100
Subject: [PATCH 1111/2492] cmake: Ensure git exists before executing it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CMake 3.28 has turned stricter when executing string(REPLACE …) and
expects four or more commands. In case of distro package builds from
tarball, it happens that git is not present. CTags.cmake tries to
catch that by veriying the exit status of the command, but as there
is in fact git | awk, awk returns 0 even when git does not exist.

Ensure that the variable submodules has been defined before trying
to replace substrings in this variable.

Signed-off-by: Dominique Leuenberger <dimstar@opensuse.org>
---
 cmake/modules/CTags.cmake | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/CTags.cmake b/cmake/modules/CTags.cmake
index c3e1b3799b02..772f647f947d 100644
--- a/cmake/modules/CTags.cmake
+++ b/cmake/modules/CTags.cmake
@@ -3,13 +3,14 @@ find_program(CTAGS_EXECUTABLE ctags)
 function(add_tags name)
   cmake_parse_arguments(TAGS "" "SRC_DIR;TAG_FILE" "EXCLUDE_OPTS;EXCLUDES" ${ARGN})
   set(excludes ${TAGS_EXCLUDES})
+  find_package(Git)
   if(TAGS_EXCLUDE_OPTS)
     # always respect EXCLUDES_OPTS
     list(APPEND excludes ${TAGS_EXCLUDE_OPTS})
-  else()
+  elseif(Git_FOUND)
     # exclude the submodules under SRC_DIR by default
     execute_process(
-      COMMAND git config --file .gitmodules --get-regexp path
+      COMMAND ${GIT_EXECUTABLE} config --file .gitmodules --get-regexp path
       COMMAND awk "/${TAGS_SRC_DIR}/ { print $2 }"
       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
       RESULT_VARIABLE result_code

From b550e1c7970a9200706de753573066528d5d7179 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 16 Nov 2023 16:22:34 +0800
Subject: [PATCH 1112/2492] tools/ceph_objectstore_tool: add op "expand-log"

There are circumstances in which, although we can't find enough OSDs to
form a valid ec acting set, we can enough OSDs that are fully backfilled
and it's only because their PGLog are lagged behind that they can't be
added the acting set. In this case, right now, we can only drop the
entire pg's data by recreating it.

This commit add the ability to fill in the pg log gap between the
backfilled OSDs and the auth log shard, so the pg can be brought to
active losing only part of its data and those uncorruped data can be
recovered.

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/tools/ceph_objectstore_tool.cc | 100 ++++++++++++++++++++++++++++-
 1 file changed, 98 insertions(+), 2 deletions(-)

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index 19a445824834..9e99f63a8639 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -1057,6 +1057,84 @@ int get_osdmap(ObjectStore *store, epoch_t e, OSDMap &osdmap, bufferlist& bl)
   return 0;
 }
 
+int expand_log(
+  CephContext *cct,
+  ObjectStore *fs,
+  spg_t pgid,
+  pg_info_t &info,
+  eversion_t target_version)
+{
+  try {
+    bufferlist bl;
+    OSDMap osdmap;
+    int ret = get_osdmap(fs, info.last_update.epoch, osdmap, bl);
+    if (ret < 0) {
+      std::cerr << "Can't find latest local OSDMap " << info.last_update.epoch << std::endl;
+      return ret;
+    }
+    ceph_assert(osdmap.have_pg_pool(info.pgid.pool()));
+    auto pool_info = osdmap.get_pg_pool(info.pgid.pool());
+    if (!pool_info->is_erasure()) {
+      std::cerr << "extend-log-with-fake-entries can only apply to pgs of ec pools" << std::endl;
+      return -EINVAL;
+    }
+
+    PGLog log(cct);
+    pg_missing_t missing;
+    auto ch = fs->open_collection(coll_t(pgid));
+    if (!ch) {
+      return -ENOENT;
+    }
+    ostringstream oss;
+    log.read_log_and_missing(
+      fs, ch,
+      pgid.make_pgmeta_oid(),
+      info,
+      oss,
+      cct->_conf->osd_ignore_stale_divergent_priors,
+      cct->_conf->osd_debug_verify_missing_on_start);
+    if (debug && oss.str().size())
+      cerr << oss.str() << std::endl;
+
+    auto e = target_version;
+    e.version = log.get_head().version + 1;
+    auto entry = *log.get_log().log.rbegin();
+    for (; e <= target_version; e.version++) {
+      entry.version = e;
+      std::cout << "adding " << e << std::endl;
+      log.add(entry, true);
+    }
+    info.last_complete = target_version;
+    info.last_update = target_version;
+    info.last_user_version = target_version.version + 1;
+
+    std::map<string, bufferlist> km;
+    ObjectStore::Transaction t;
+
+    pg_fast_info_t fast;
+    fast.populate_from(info);
+    encode(fast, km[string(fastinfo_key)]);
+    encode(info, km[string(info_key)]);
+    log.write_log_and_missing(
+      t,
+      &km,
+      coll_t(pgid),
+      pgid.make_pgmeta_oid(),
+      pool_info->require_rollback());
+
+    for (auto &ent : km) {
+      std::cout << "km key: " << ent.first << std::endl;
+    }
+
+    t.omap_setkeys(coll_t(pgid), pgid.make_pgmeta_oid(), km);
+    fs->queue_transaction(ch, std::move(t));
+    return 0;
+  } catch (const buffer::error &e) {
+    cerr << "read_log_and_missing threw exception error " << e.what() << std::endl;
+    return -EFAULT;
+  }
+}
+
 int get_pg_num_history(ObjectStore *store, pool_pg_num_history_t *h)
 {
   ObjectStore::CollectionHandle ch = store->open_collection(coll_t::meta());
@@ -3350,7 +3428,7 @@ bool ends_with(const string& check, const string& ending)
 int main(int argc, char **argv)
 {
   string dpath, jpath, pgidstr, op, file, mountpoint, mon_store_path, object;
-  string target_data_path, fsid;
+  string target_data_path, fsid, target_version_str;
   string objcmd, arg1, arg2, type, format, argnspace, pool, rmtypestr, dump_data_dir;
   boost::optional<std::string> nspace;
   spg_t pgid;
@@ -3368,6 +3446,8 @@ int main(int argc, char **argv)
      "Arg is one of [bluestore (default), memstore]")
     ("data-path", po::value<string>(&dpath),
      "path to object store, mandatory")
+    ("target-version", po::value<string>(&target_version_str),
+     "the target version that log is expected to be expanded to")
     ("journal-path", po::value<string>(&jpath),
      "path to journal, use if tool can't find it")
     ("pgid", po::value<string>(&pgidstr),
@@ -3623,6 +3703,18 @@ int main(int argc, char **argv)
     return 1;
   }
 
+  eversion_t target_version;
+  if (op == "extend-log-with-fake-entries") {
+    if (target_version_str.empty()) {
+      std::cerr << "target-version needed" << std::endl;
+      return 1;
+    }
+    std::string epoch_str = target_version_str.substr(0, target_version_str.find("."));
+    std::string version_str = target_version_str.substr(target_version_str.find(".") + 1);
+    target_version.epoch = std::stoi(epoch_str);
+    target_version.version = std::stoll(version_str);
+  }
+
   std::unique_ptr<ObjectStore> fs = ObjectStore::create(g_ceph_context, type, dpath, jpath, flags);
   if (!fs) {
     cerr << "Unable to create store of type " << type << std::endl;
@@ -4110,7 +4202,7 @@ int main(int argc, char **argv)
 
   // If not an object command nor any of the ops handled below, then output this usage
   // before complaining about a bad pgid
-  if (!vm.count("objcmd") && op != "export" && op != "export-remove" && op != "info" && op != "log" && op != "mark-complete" && op != "trim-pg-log" && op != "trim-pg-log-dups" && op != "pg-log-inject-dups") {
+  if (!vm.count("objcmd") && op != "export" && op != "export-remove" && op != "info" && op != "log" && op != "mark-complete" && op != "trim-pg-log" && op != "trim-pg-log-dups" && op != "pg-log-inject-dups" && op != "extend-log-with-fake-entries") {
     cerr << "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, dump-super, meta-list, "
       "get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, dump-export, trim-pg-log, trim-pg-log-dups statfs)"
 	 << std::endl;
@@ -4426,6 +4518,10 @@ int main(int argc, char **argv)
           goto out;
 
       dump_log(formatter, cout, log, missing);
+    } else if (op == "extend-log-with-fake-entries") {
+      ret = expand_log(cct.get(), fs.get(), pgid, info, target_version);
+      if (ret < 0)
+	goto out;
     } else if (op == "mark-complete") {
       ObjectStore::Transaction tran;
       ObjectStore::Transaction *t = &tran;

From 37dc16f232b3e10c571f4fdb0452669b1940ba2e Mon Sep 17 00:00:00 2001
From: "yite.gu" <yitegu0@gmail.com>
Date: Tue, 29 Aug 2023 23:27:23 +0800
Subject: [PATCH 1113/2492] os/bluestore: add perfcount for bluestore/bluefs
 allocator

Allocator performance is the performance limiting factor
on performance storage. This performance count can help
us intuitively observe the reasons for changes in bluestore
performance.

Signed-off-by: Yite Gu <yitegu0@gamil.com>
---
 src/os/bluestore/BlueFS.cc    | 59 +++++++++++++++++++++++++++++++++--
 src/os/bluestore/BlueFS.h     | 11 ++++++-
 src/os/bluestore/BlueStore.cc |  9 ++++++
 src/os/bluestore/BlueStore.h  |  1 +
 4 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 54d0bbc38e57..f9e8dae1282d 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -285,9 +285,9 @@ void BlueFS::_init_logger()
 		    "mxwb",
 		    PerfCountersBuilder::PRIO_INTERESTING,
 		    unit_t(UNIT_BYTES));
-  b.add_u64_counter(l_bluefs_main_alloc_unit, "alloc_unit_main",
+  b.add_u64_counter(l_bluefs_slow_alloc_unit, "alloc_unit_slow",
 		    "Allocation unit size (in bytes) for primary/shared device",
-		    "aumb",
+		    "ausb",
 		    PerfCountersBuilder::PRIO_CRITICAL,
 		    unit_t(UNIT_BYTES));
   b.add_u64_counter(l_bluefs_db_alloc_unit, "alloc_unit_db",
@@ -427,6 +427,30 @@ void BlueFS::_init_logger()
 	    "How many times bluefs read found page with all 0s");
   b.add_u64(l_bluefs_read_zeros_errors, "read_zeros_errors",
 	    "How many times bluefs read found transient page with all 0s");
+  b.add_time_avg(l_bluefs_wal_alloc_lat, "wal_alloc_lat",
+                "Average bluefs wal allocate latency",
+                "bwal",
+                PerfCountersBuilder::PRIO_USEFUL);
+  b.add_time_avg(l_bluefs_db_alloc_lat, "db_alloc_lat",
+                "Average bluefs db allocate latency",
+                "bdal",
+                PerfCountersBuilder::PRIO_USEFUL);
+  b.add_time_avg(l_bluefs_slow_alloc_lat, "slow_alloc_lat",
+                "Average allocation latency for primary/shared device",
+                "bsal",
+                PerfCountersBuilder::PRIO_USEFUL);
+  b.add_time(l_bluefs_wal_alloc_max_lat, "alloc_wal_max_lat",
+             "Max allocation latency for wal device",
+             "awxt",
+             PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time(l_bluefs_db_alloc_max_lat, "alloc_db_max_lat",
+             "Max allocation latency for db device",
+             "adxt",
+             PerfCountersBuilder::PRIO_INTERESTING);
+  b.add_time(l_bluefs_slow_alloc_max_lat, "alloc_slow_max_lat",
+             "Max allocation latency for primary/shared device",
+             "asxt",
+             PerfCountersBuilder::PRIO_INTERESTING);
 
   logger = b.create_perf_counters();
   cct->get_perfcounters_collection()->add(logger);
@@ -712,7 +736,7 @@ void BlueFS::_init_alloc()
     alloc_size[BDEV_SLOW] = 0;
   }
   logger->set(l_bluefs_db_alloc_unit, alloc_size[BDEV_DB]);
-  logger->set(l_bluefs_main_alloc_unit, alloc_size[BDEV_SLOW]);
+  logger->set(l_bluefs_slow_alloc_unit, alloc_size[BDEV_SLOW]);
   // new wal and db devices are never shared
   if (bdev[BDEV_NEWWAL]) {
     alloc_size[BDEV_NEWWAL] = cct->_conf->bluefs_alloc_size;
@@ -3814,6 +3838,33 @@ const char* BlueFS::get_device_name(unsigned id)
   return names[id];
 }
 
+void BlueFS::_update_allocate_stats(uint8_t id, const ceph::timespan& d)
+{
+  switch(id) {
+    case BDEV_SLOW:
+      logger->tinc(l_bluefs_slow_alloc_lat, d);
+      if (d > max_alloc_lat[id]) {
+        logger->tset(l_bluefs_slow_alloc_max_lat, utime_t(d));
+        max_alloc_lat[id] = d;
+      }
+      break;
+    case BDEV_DB:
+      logger->tinc(l_bluefs_db_alloc_lat, d);
+      if (d > max_alloc_lat[id]) {
+        logger->tset(l_bluefs_db_alloc_max_lat, utime_t(d));
+        max_alloc_lat[id] = d;
+      }
+      break;
+    case BDEV_WAL:
+      logger->tinc(l_bluefs_wal_alloc_lat, d);
+      if (d > max_alloc_lat[id]) {
+        logger->tset(l_bluefs_wal_alloc_max_lat, utime_t(d));
+        max_alloc_lat[id] = d;
+      }
+      break;
+  }
+}
+
 int BlueFS::_allocate(uint8_t id, uint64_t len,
 		      uint64_t alloc_unit,
 		      bluefs_fnode_t* node,
@@ -3859,7 +3910,9 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
     }   
     ++alloc_attempts;
     extents.reserve(4);  // 4 should be (more than) enough for most allocations
+    auto t0 = mono_clock::now();
     alloc_len = alloc[id]->allocate(need, alloc_unit, hint, &extents);
+    _update_allocate_stats(id, mono_clock::now() - t0);
   }
   if (alloc_len < 0 || alloc_len < need) {
     if (alloc[id]) {
diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index 759e71fdf05e..86fb3cc882e1 100644
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -43,7 +43,7 @@ enum {
   l_bluefs_max_bytes_wal,
   l_bluefs_max_bytes_db,
   l_bluefs_max_bytes_slow,
-  l_bluefs_main_alloc_unit,
+  l_bluefs_slow_alloc_unit,
   l_bluefs_db_alloc_unit,
   l_bluefs_wal_alloc_unit,
   l_bluefs_read_random_lat,
@@ -79,6 +79,12 @@ enum {
   l_bluefs_alloc_shared_size_fallbacks,
   l_bluefs_read_zeros_candidate,
   l_bluefs_read_zeros_errors,
+  l_bluefs_wal_alloc_lat,
+  l_bluefs_db_alloc_lat,
+  l_bluefs_slow_alloc_lat,
+  l_bluefs_wal_alloc_max_lat,
+  l_bluefs_db_alloc_max_lat,
+  l_bluefs_slow_alloc_max_lat,
   l_bluefs_last,
 };
 
@@ -440,6 +446,8 @@ class BlueFS {
     l_bluefs_max_bytes_db,
   };
 
+  ceph::timespan max_alloc_lat[MAX_BDEV] = {ceph::make_timespan(0)};
+
   // cache
   struct {
     ceph::mutex lock = ceph::make_mutex("BlueFS::nodes.lock");
@@ -528,6 +536,7 @@ class BlueFS {
   const char* get_device_name(unsigned id);
 
   typedef std::function<void(const bluefs_extent_t)> update_fn_t;
+  void _update_allocate_stats(uint8_t id, const ceph::timespan& d);
   int _allocate(uint8_t bdev, uint64_t len,
                 uint64_t alloc_unit,
 		bluefs_fnode_t* node,
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index d2cc85300215..b2feb54c0ae3 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6456,6 +6456,10 @@ void BlueStore::_init_logger()
     l_bluestore_allocate_hist, "allocate_histogram",
     alloc_hist_x_axis_config, alloc_hist_y_axis_config,
     "Histogram of requested block allocations vs. given ones");
+  b.add_time_avg(l_bluestore_allocator_lat, "allocator_lat",
+    "Average bluestore allocator latency",
+    "bsal",
+    PerfCountersBuilder::PRIO_USEFUL);
 
   logger = b.create_perf_counters();
   cct->get_perfcounters_collection()->add(logger);
@@ -16799,9 +16803,14 @@ int BlueStore::_do_alloc_write(
   PExtentVector prealloc;
   prealloc.reserve(2 * wctx->writes.size());
   int64_t prealloc_left = 0;
+  auto start = mono_clock::now();
   prealloc_left = alloc->allocate(
     need, min_alloc_size, need,
     0, &prealloc);
+  log_latency("allocator@_do_alloc_write",
+    l_bluestore_allocator_lat,
+    mono_clock::now() - start,
+    cct->_conf->bluestore_log_op_age);
   if (prealloc_left < 0 || prealloc_left < (int64_t)need) {
     derr << __func__ << " failed to allocate 0x" << std::hex << need
          << " allocated 0x " << (prealloc_left < 0 ? 0 : prealloc_left)
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index beac82f163a7..f6c85b1d2190 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -211,6 +211,7 @@ enum {
   // allocation stats
   //****************************************
   l_bluestore_allocate_hist,
+  l_bluestore_allocator_lat,
   //****************************************
 
   // slow op counter

From d5ab720dc54e19904267fbde5ff54260eff9bd60 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Mon, 18 Dec 2023 15:09:43 -0600
Subject: [PATCH 1114/2492] doc/releases: specify dashboard improvements

...and address other nits

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 doc/releases/reef.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
index 89a4d16b546f..378c64beb11b 100644
--- a/doc/releases/reef.rst
+++ b/doc/releases/reef.rst
@@ -73,16 +73,16 @@ Notable Changes
   The user might temporarily mute this warning using
   ``ceph health mute POOL_APP_NOT_ENABLED``.
 
-* An overview page for rgw to show the overall status of rgw components
+* Dashboard: An overview page for RGW to show the overall status of RGW components.
 
-* Added management support for RGW Multi-site and Ceph FS Subvolumes and groups
+* Dashboard: Added management support for RGW Multi-site and CephFS Subvolumes and groups.
 
-* Fixed few bugs and issues around the new dashboard page like the broken layout,
+* Dashboard: Fixed few bugs and issues around the new dashboard page including the broken layout,
   some metrics giving wrong values and introduced a popover to display details
-  when there are HEALTH_WARN or HEALTH_ERR
+  when there are HEALTH_WARN or HEALTH_ERR.
 
-* Fixed several issues in ceph dashboard on rook backed clusters,
-  and improved the user experience on the rook environment
+* Dashboard: Fixed several issues in Ceph dashboard on Rook-backed clusters,
+  and improved the user experience on the Rook environment.
 
 Changelog
 ---------

From 899276a5aa9f07a2faf4e7b659c0ea5c90dbf3b9 Mon Sep 17 00:00:00 2001
From: NitzanMordhai <nmordech@redhat.com>
Date: Wed, 28 Jun 2023 09:57:11 +0000
Subject: [PATCH 1115/2492] ceph-dencoder: osd - Add missing types

Currently, ceph-dencoder lacks certain osd types, preventing us from accurately checking the ceph corpus for encode-decode mismatches.
This pull request aims to address this issue by adding the missing types to ceph-dencoder.

To successfully incorporate these types into ceph-dencoder, we need to introduce the necessary `dump` and `generate_test_instances`
functions that was missing in some types. These functions are essential for proper encode and decode of the added types.

This PR will enhance the functionality of ceph-dencoder by including the missing types, enabling a comprehensive analysis of encode-decode consistency.
With the addition of these types, we can ensure the robustness and correctness of the ceph corpus.

This update will significantly contribute to improving the overall reliability and accuracy of ceph-dencoder.
It allows for a more comprehensive assessment of the encode-decode behavior, leading to enhanced data integrity and stability within the ceph ecosystem.

Fixes: https://tracker.ceph.com/issues/61788
Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 src/cls/fifo/cls_fifo.cc                |  9 +++
 src/include/cephfs/types.h              |  8 +++
 src/mgr/DaemonHealthMetric.h            | 12 +++-
 src/mgr/MetricTypes.h                   |  8 +++
 src/mgr/OSDPerfMetricTypes.cc           |  2 +-
 src/mgr/OSDPerfMetricTypes.h            | 76 ++++++++++++++++++++++-
 src/mon/CreatingPGs.h                   | 14 ++++-
 src/mon/MgrMap.h                        | 53 ++++++++++++++--
 src/mon/MonCommand.h                    | 19 ++++++
 src/mon/MonMap.cc                       | 21 +++++++
 src/mon/MonMap.h                        |  2 +
 src/mon/PGMap.h                         | 12 ++++
 src/mon/health_check.h                  |  1 +
 src/mon/mon_types.h                     | 40 +++++++++++-
 src/os/Transaction.h                    |  4 +-
 src/os/bluestore/bluefs_types.cc        |  9 +++
 src/os/bluestore/bluefs_types.h         |  1 +
 src/os/bluestore/bluestore_types.h      | 11 +++-
 src/osd/OSDMap.h                        |  6 ++
 src/osd/SnapMapper.cc                   | 16 +++++
 src/osd/SnapMapper.h                    | 12 ++++
 src/osd/osd_types.cc                    | 24 ++++++--
 src/osd/osd_types.h                     | 82 ++++++++++++++++++++++++-
 src/tools/ceph-dencoder/denc_plugin.h   |  1 +
 src/tools/ceph-dencoder/denc_registry.h |  5 +-
 src/tools/ceph-dencoder/osd_types.h     | 63 ++++++++++++++++++-
 26 files changed, 488 insertions(+), 23 deletions(-)

diff --git a/src/cls/fifo/cls_fifo.cc b/src/cls/fifo/cls_fifo.cc
index 85022eeb061a..4b02903916dc 100644
--- a/src/cls/fifo/cls_fifo.cc
+++ b/src/cls/fifo/cls_fifo.cc
@@ -14,6 +14,7 @@
 
 #include <fmt/format.h>
 
+#include "common/Formatter.h"
 #include "include/buffer.h"
 #include "include/types.h"
 
@@ -53,6 +54,14 @@ struct entry_header {
     decode(mtime, bl);
     DECODE_FINISH(bl);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("mtime") << mtime;
+  }
+  static void generate_test_instances(std::list<entry_header*>& ls) {
+    ls.push_back(new entry_header);
+    ls.push_back(new entry_header);
+    ls.back()->mtime = ceph::real_clock::now();
+  }
 };
 WRITE_CLASS_ENCODER(entry_header)
 
diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 2abfe29b08cf..068c9ef81995 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -199,6 +199,14 @@ struct vinodeno_t {
     decode(ino, p);
     decode(snapid, p);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("ino", ino);
+    f->dump_unsigned("snapid", snapid);
+  }
+  static void generate_test_instances(std::list<vinodeno_t*>& ls) {
+    ls.push_back(new vinodeno_t);
+    ls.push_back(new vinodeno_t(1, 2));
+  }
 
   inodeno_t ino;
   snapid_t snapid;
diff --git a/src/mgr/DaemonHealthMetric.h b/src/mgr/DaemonHealthMetric.h
index ce0dad2c87e3..a6fbce29910b 100644
--- a/src/mgr/DaemonHealthMetric.h
+++ b/src/mgr/DaemonHealthMetric.h
@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <ostream>
+#include "common/Formatter.h"
 #include "include/denc.h"
 
 enum class daemon_metric : uint8_t {
@@ -66,7 +67,16 @@ class DaemonHealthMetric
     denc(v.value.n, p);
     DENC_FINISH(p);
   }
-
+  void dump(Formatter *f) const {
+    f->dump_string("type", get_type_name());
+    f->dump_int("n", get_n());
+    f->dump_int("n1", get_n1());
+    f->dump_int("n2", get_n2());
+  }
+  static void generate_test_instances(std::list<DaemonHealthMetric*>& o) {
+    o.push_back(new DaemonHealthMetric(daemon_metric::SLOW_OPS, 1));
+    o.push_back(new DaemonHealthMetric(daemon_metric::PENDING_CREATING_PGS, 1, 2));
+  }
   std::string get_type_name() const {
     return daemon_metric_name(get_type());
   }
diff --git a/src/mgr/MetricTypes.h b/src/mgr/MetricTypes.h
index 586c470ca569..762564f37ee1 100644
--- a/src/mgr/MetricTypes.h
+++ b/src/mgr/MetricTypes.h
@@ -30,6 +30,14 @@ struct OSDMetricPayload {
     denc(v.report, p);
     DENC_FINISH(p);
   }
+
+  void dump(ceph::Formatter *f) const {
+    encode_json("report", report, f);
+  }
+
+  static void generate_test_instances(std::list<OSDMetricPayload*>& ls) {
+    ls.push_back(new OSDMetricPayload);
+  }
 };
 
 struct MDSMetricPayload {
diff --git a/src/mgr/OSDPerfMetricTypes.cc b/src/mgr/OSDPerfMetricTypes.cc
index bce95e0ae388..e21318cc7443 100644
--- a/src/mgr/OSDPerfMetricTypes.cc
+++ b/src/mgr/OSDPerfMetricTypes.cc
@@ -1,8 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include "common/Formatter.h"
 #include "mgr/OSDPerfMetricTypes.h"
-
 #include <ostream>
 
 using ceph::bufferlist;
diff --git a/src/mgr/OSDPerfMetricTypes.h b/src/mgr/OSDPerfMetricTypes.h
index 1b5904e13ae7..8dd0afd8cb45 100644
--- a/src/mgr/OSDPerfMetricTypes.h
+++ b/src/mgr/OSDPerfMetricTypes.h
@@ -4,9 +4,9 @@
 #ifndef OSD_PERF_METRIC_H_
 #define OSD_PERF_METRIC_H_
 
+#include "common/ceph_json.h"
 #include "include/denc.h"
 #include "include/stringify.h"
-
 #include "mgr/Types.h"
 
 #include <regex>
@@ -70,6 +70,23 @@ struct OSDPerfMetricSubKeyDescriptor {
     denc(v.regex_str, p);
     DENC_FINISH(p);
   }
+
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("type", static_cast<uint8_t>(type));
+    f->dump_string("regex", regex_str);
+  }
+
+  static void generate_test_instances(std::list<OSDPerfMetricSubKeyDescriptor*>& o) {
+    o.push_back(new OSDPerfMetricSubKeyDescriptor());
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::CLIENT_ID, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::CLIENT_ADDRESS, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::POOL_ID, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::NAMESPACE, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::OSD_ID, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::PG_ID, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::OBJECT_NAME, ".*"));
+    o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::SNAP_ID, ".*"));
+  }
 };
 WRITE_CLASS_DENC(OSDPerfMetricSubKeyDescriptor)
 
@@ -184,6 +201,23 @@ struct PerformanceCounterDescriptor {
     DENC_FINISH(p);
   }
 
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("type", static_cast<uint8_t>(type));
+  }
+
+  static void generate_test_instances(std::list<PerformanceCounterDescriptor*>& o) {
+    o.push_back(new PerformanceCounterDescriptor());
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::OPS));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_OPS));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_OPS));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::BYTES));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_BYTES));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_BYTES));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::LATENCY));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_LATENCY));
+    o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_LATENCY));
+  }
+
   void pack_counter(const PerformanceCounter &c, ceph::buffer::list *bl) const;
   void unpack_counter(ceph::buffer::list::const_iterator& bl,
                       PerformanceCounter *c) const;
@@ -306,6 +340,28 @@ struct OSDPerfMetricQuery {
     DENC_FINISH(p);
   }
 
+  void dump(ceph::Formatter *f) const {
+    encode_json("key_descriptor", key_descriptor, f);
+    encode_json("performance_counter_descriptors",
+                performance_counter_descriptors, f);
+  }
+
+  static void generate_test_instances(std::list<OSDPerfMetricQuery*> &o) {
+    o.push_back(new OSDPerfMetricQuery());
+    o.push_back(new OSDPerfMetricQuery(OSDPerfMetricKeyDescriptor(),
+                                       PerformanceCounterDescriptors()));
+    o.push_back(new OSDPerfMetricQuery(OSDPerfMetricKeyDescriptor(),
+                                       PerformanceCounterDescriptors{
+                                         PerformanceCounterType::WRITE_OPS,
+                                         PerformanceCounterType::READ_OPS,
+                                         PerformanceCounterType::BYTES,
+                                         PerformanceCounterType::WRITE_BYTES,
+                                         PerformanceCounterType::READ_BYTES,
+                                         PerformanceCounterType::LATENCY,
+                                         PerformanceCounterType::WRITE_LATENCY,
+                                         PerformanceCounterType::READ_LATENCY}));
+  }
+
   void get_performance_counter_descriptors(
       PerformanceCounterDescriptors *descriptors) const {
     *descriptors = performance_counter_descriptors;
@@ -353,6 +409,24 @@ struct OSDPerfMetricReport {
     denc(v.group_packed_performance_counters, p);
     DENC_FINISH(p);
   }
+
+  void dump(ceph::Formatter *f) const {
+    encode_json("performance_counter_descriptors",
+                performance_counter_descriptors, f);
+    encode_json("group_packed_performance_counters",
+                group_packed_performance_counters, f);
+  }
+
+  static void generate_test_instances(std::list<OSDPerfMetricReport *> &o) {
+    o.push_back(new OSDPerfMetricReport);
+    o.push_back(new OSDPerfMetricReport);
+    o.back()->performance_counter_descriptors.push_back(
+        PerformanceCounterDescriptor(PerformanceCounterType::OPS));
+    o.back()->performance_counter_descriptors.push_back(
+        PerformanceCounterDescriptor(PerformanceCounterType::WRITE_OPS));
+    o.back()->performance_counter_descriptors.push_back(
+        PerformanceCounterDescriptor(PerformanceCounterType::READ_OPS));
+  }
 };
 WRITE_CLASS_DENC(OSDPerfMetricReport)
 
diff --git a/src/mon/CreatingPGs.h b/src/mon/CreatingPGs.h
index 0075f81e7bbb..808a22b1567a 100644
--- a/src/mon/CreatingPGs.h
+++ b/src/mon/CreatingPGs.h
@@ -83,8 +83,20 @@ struct creating_pgs_t {
       f->dump_object("pg_history", history);
       f->dump_object("past_intervals", past_intervals);
     }
+    static void generate_test_instances(std::list<pg_create_info*>& o) {
+      o.push_back(new pg_create_info);
+      o.back()->create_epoch = 10;
+      o.push_back(new pg_create_info);
+      o.back()->create_epoch = 1;
+      o.back()->create_stamp = utime_t(2, 3);
+      o.back()->up = {1, 2, 3};
+      o.back()->up_primary = 1;
+      o.back()->acting = {1, 2, 3};
+      o.back()->acting_primary = 1;
+    }
 
-    pg_create_info() {}
+    pg_create_info() 
+      : create_epoch(0) {}
     pg_create_info(epoch_t e, utime_t t)
       : create_epoch(e),
 	create_stamp(t) {
diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index f37ed97fd16c..0a906cd039fc 100644
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -20,6 +20,7 @@
 #include "msg/msg_types.h"
 #include "include/encoding.h"
 #include "include/utime.h"
+#include "common/ceph_json.h"
 #include "common/Formatter.h"
 #include "common/ceph_releases.h"
 #include "common/version.h"
@@ -74,7 +75,8 @@ class MgrMap
       decode(see_also, p);
       DECODE_FINISH(p);
     }
-    void dump(ceph::Formatter *f) const {
+    void dump(ceph::Formatter *f) const
+    {
       f->dump_string("name", name);
       f->dump_string("type", Option::type_to_str(
 		       static_cast<Option::type_t>(type)));
@@ -102,6 +104,23 @@ class MgrMap
       }
       f->close_section();
     }
+    static void generate_test_instances(std::list<ModuleOption*>& ls)
+    {
+      ls.push_back(new ModuleOption);
+      ls.push_back(new ModuleOption);
+      ls.back()->name = "name";
+      ls.back()->type = Option::TYPE_STR;
+      ls.back()->level = Option::LEVEL_ADVANCED;
+      ls.back()->flags = Option::FLAG_RUNTIME;
+      ls.back()->default_value = "default_value";
+      ls.back()->min = "min";
+      ls.back()->max = "max";
+      ls.back()->enum_allowed.insert("enum_allowed");
+      ls.back()->desc = "desc";
+      ls.back()->long_desc = "long_desc";
+      ls.back()->tags.insert("tag");
+      ls.back()->see_also.insert("see_also");
+    }
   };
 
   class ModuleInfo
@@ -139,7 +158,8 @@ class MgrMap
       return (name == rhs.name) && (can_run == rhs.can_run);
     }
 
-    void dump(ceph::Formatter *f) const {
+    void dump(ceph::Formatter *f) const 
+    {
       f->open_object_section("module");
       f->dump_string("name", name);
       f->dump_bool("can_run", can_run);
@@ -151,6 +171,16 @@ class MgrMap
       f->close_section();
       f->close_section();
     }
+
+    static void generate_test_instances(std::list<ModuleInfo*>& ls)
+    {
+      ls.push_back(new ModuleInfo);
+      ls.push_back(new ModuleInfo);
+      ls.back()->name = "name";
+      ls.back()->can_run = true;
+      ls.back()->error_string = "error_string";
+      ls.back()->module_options["module_option"] = ModuleOption();
+    }
   };
 
   class StandbyInfo
@@ -209,6 +239,19 @@ class MgrMap
       }
       DECODE_FINISH(p);
     }
+    void dump(ceph::Formatter *f) const
+    {
+      f->dump_unsigned("gid", gid);
+      f->dump_string("name", name);
+      encode_json("available_modules", available_modules, f);
+      f->dump_unsigned("mgr_features", mgr_features);
+    }
+    static void generate_test_instances(std::list<StandbyInfo*>& ls)
+    {
+      ls.push_back(new StandbyInfo(1, "a", {}, 0));
+      ls.push_back(new StandbyInfo(2, "b", {}, 0));
+      ls.push_back(new StandbyInfo(3, "c", {}, 0));
+    }
 
     bool have_module(const std::string &module_name) const
     {
@@ -501,7 +544,8 @@ class MgrMap
     DECODE_FINISH(p);
   }
 
-  void dump(ceph::Formatter *f) const {
+  void dump(ceph::Formatter *f) const
+  {
     f->dump_int("epoch", epoch);
     f->dump_int("active_gid", get_active_gid());
     f->dump_string("active_name", get_active_name());
@@ -561,7 +605,8 @@ class MgrMap
     f->close_section(); // active_clients
   }
 
-  static void generate_test_instances(std::list<MgrMap*> &l) {
+  static void generate_test_instances(std::list<MgrMap*> &l)
+  {
     l.push_back(new MgrMap);
   }
 
diff --git a/src/mon/MonCommand.h b/src/mon/MonCommand.h
index cb60d3d17065..5caebfc6f134 100644
--- a/src/mon/MonCommand.h
+++ b/src/mon/MonCommand.h
@@ -14,6 +14,7 @@
 #pragma once
 
 #include <string>
+#include "common/Formatter.h"
 #include "include/encoding.h"
 
 struct MonCommand {
@@ -53,6 +54,24 @@ struct MonCommand {
     DECODE_FINISH(bl);
   }
 
+  void dump(ceph::Formatter *f) const {
+    f->dump_string("cmdstring", cmdstring);
+    f->dump_string("helpstring", helpstring);
+    f->dump_string("module", module);
+    f->dump_string("req_perms", req_perms);
+    f->dump_unsigned("flags", flags);
+  }
+
+  static void generate_test_instances(std::list<MonCommand*>& ls) {
+    ls.push_back(new MonCommand);
+    ls.push_back(new MonCommand);
+    ls.back()->cmdstring = "foo";
+    ls.back()->helpstring = "bar";
+    ls.back()->module = "baz";
+    ls.back()->req_perms = "quux";
+    ls.back()->flags = FLAG_NOFORWARD;
+  }
+
   /**
    * Unversioned encoding for use within encode_array.
    */
diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
index bb8a4b19455d..6eb37df171a9 100644
--- a/src/mon/MonMap.cc
+++ b/src/mon/MonMap.cc
@@ -21,6 +21,7 @@
 #include "include/ceph_features.h"
 #include "include/addr_parsing.h"
 #include "common/ceph_argparse.h"
+#include "common/ceph_json.h"
 #include "common/dns_resolve.h"
 #include "common/errno.h"
 #include "common/dout.h"
@@ -110,6 +111,26 @@ void mon_info_t::print(ostream& out) const
       << " crush location " << crush_loc;
 }
 
+void mon_info_t::dump(ceph::Formatter *f) const
+{
+  f->dump_string("name", name);
+  f->dump_stream("addr") << public_addrs;
+  f->dump_int("priority", priority);
+  f->dump_float("weight", weight);
+  encode_json("crush_location", crush_loc, f);
+}
+
+void mon_info_t::generate_test_instances(list<mon_info_t*>& ls)
+{
+  ls.push_back(new mon_info_t);
+  ls.push_back(new mon_info_t);
+  ls.back()->name = "noname";
+  ls.back()->public_addrs.parse("v1:1.2.3.4:567/890");
+  ls.back()->priority = 1;
+  ls.back()->weight = 1.0;
+  ls.back()->crush_loc.emplace("root", "default");
+  ls.back()->crush_loc.emplace("host", "foo");
+}
 namespace {
   struct rank_cmp {
     bool operator()(const mon_info_t &a, const mon_info_t &b) const {
diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h
index 7fedd2c1128b..5bd72b1d917f 100644
--- a/src/mon/MonMap.h
+++ b/src/mon/MonMap.h
@@ -85,6 +85,8 @@ struct mon_info_t {
   void encode(ceph::buffer::list& bl, uint64_t features) const;
   void decode(ceph::buffer::list::const_iterator& p);
   void print(std::ostream& out) const;
+  void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<mon_info_t*>& ls);
 };
 WRITE_CLASS_ENCODER_FEATURES(mon_info_t)
 
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index dbbfce9d52aa..2a127b802a86 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -66,6 +66,18 @@ class PGMapDigest {
       decode(up_not_acting, p);
       decode(primary, p);
     }
+    void dump(ceph::Formatter *f) const {
+      f->dump_int("acting", acting);
+      f->dump_int("up_not_acting", up_not_acting);
+      f->dump_int("primary", primary);
+    }
+    static void generate_test_instances(std::list<pg_count*>& o) {
+      o.push_back(new pg_count);
+      o.push_back(new pg_count);
+      o.back()->acting = 1;
+      o.back()->up_not_acting = 2;
+      o.back()->primary = 3;
+    }
   };
   mempool::pgmap::unordered_map<int32_t,pg_count> num_pg_by_osd;
 
diff --git a/src/mon/health_check.h b/src/mon/health_check.h
index 4e74637f9e53..e6d1d7595510 100644
--- a/src/mon/health_check.h
+++ b/src/mon/health_check.h
@@ -60,6 +60,7 @@ struct health_check_t {
 
   static void generate_test_instances(std::list<health_check_t*>& ls) {
     ls.push_back(new health_check_t);
+    ls.back()->severity = HEALTH_WARN;
     ls.push_back(new health_check_t);
     ls.back()->severity = HEALTH_ERR;
     ls.back()->summary = "summarization";
diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index cce9976f3c35..d23d3ba3ae4b 100644
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -110,6 +110,15 @@ struct FeatureMap {
       f->close_section();
     }
   }
+
+  static void generate_test_instances(std::list<FeatureMap*>& ls) {
+    ls.push_back(new FeatureMap);
+    ls.push_back(new FeatureMap);
+    ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_UID);
+    ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_NOSRCADDR);
+    ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_PGID64);
+    ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_INCSUBOSDMAP);
+  }
 };
 WRITE_CLASS_ENCODER(FeatureMap)
 
@@ -190,6 +199,20 @@ struct DataStats {
     store_stats.dump(f);
     f->close_section();
   }
+  static void generate_test_instances(std::list<DataStats*>& ls) {
+    ls.push_back(new DataStats);
+    ls.push_back(new DataStats);
+    ls.back()->fs_stats.byte_total = 1024*1024;
+    ls.back()->fs_stats.byte_used = 512*1024;
+    ls.back()->fs_stats.byte_avail = 256*1024;
+    ls.back()->fs_stats.avail_percent = 50;
+    ls.back()->last_update = utime_t();
+    ls.back()->store_stats.bytes_total = 1024*1024;
+    ls.back()->store_stats.bytes_sst = 512*1024;
+    ls.back()->store_stats.bytes_log = 256*1024;
+    ls.back()->store_stats.bytes_misc = 256*1024;
+    ls.back()->store_stats.last_update = utime_t();
+  }
 
   void encode(ceph::buffer::list &bl) const {
     ENCODE_START(3, 1, bl);
@@ -467,6 +490,14 @@ class mon_feature_t {
     decode(features, p);
     DECODE_FINISH(p);
   }
+
+  static void generate_test_instances(std::list<mon_feature_t*>& ls) {
+    ls.push_back(new mon_feature_t);
+    ls.push_back(new mon_feature_t);
+    ls.back()->features = 1;
+    ls.push_back(new mon_feature_t);
+    ls.back()->features = 2;
+  }
 };
 WRITE_CLASS_ENCODER(mon_feature_t)
 
@@ -642,7 +673,7 @@ inline std::ostream& operator<<(std::ostream& out, const mon_feature_t& f) {
 
 struct ProgressEvent {
   std::string message;                  ///< event description
-  float progress;                  ///< [0..1]
+  float progress = 0.0f;                  ///< [0..1]
   bool add_to_ceph_s;
   void encode(ceph::buffer::list& bl) const {
     ENCODE_START(2, 1, bl);
@@ -669,6 +700,13 @@ struct ProgressEvent {
     f->dump_float("progress", progress);
     f->dump_bool("add_to_ceph_s", add_to_ceph_s);
   }
+  static void generate_test_instances(std::list<ProgressEvent*>& o) {
+    o.push_back(new ProgressEvent);
+    o.push_back(new ProgressEvent);
+    o.back()->message = "test message";
+    o.back()->progress = 0.5;
+    o.back()->add_to_ceph_s = true;
+  }
 };
 WRITE_CLASS_ENCODER(ProgressEvent)
 
diff --git a/src/os/Transaction.h b/src/os/Transaction.h
index 119d676702e0..f349a29c9f12 100644
--- a/src/os/Transaction.h
+++ b/src/os/Transaction.h
@@ -1299,8 +1299,8 @@ class Transaction {
   void dump(ceph::Formatter *f);
   static void generate_test_instances(std::list<Transaction*>& o);
 };
-WRITE_CLASS_ENCODER(Transaction)
-WRITE_CLASS_ENCODER(Transaction::TransactionData)
+WRITE_CLASS_ENCODER(ceph::os::Transaction)
+WRITE_CLASS_ENCODER(ceph::os::Transaction::TransactionData)
 
 std::ostream& operator<<(std::ostream& out, const Transaction& tx);
 
diff --git a/src/os/bluestore/bluefs_types.cc b/src/os/bluestore/bluefs_types.cc
index 70c8a4fbf1c5..ff9c96ab660f 100644
--- a/src/os/bluestore/bluefs_types.cc
+++ b/src/os/bluestore/bluefs_types.cc
@@ -64,6 +64,15 @@ void bluefs_layout_t::dump(Formatter *f) const
   f->dump_stream("dedicated_wal") << dedicated_wal;
 }
 
+void bluefs_layout_t::generate_test_instances(list<bluefs_layout_t*>& ls)
+{
+  ls.push_back(new bluefs_layout_t);
+  ls.push_back(new bluefs_layout_t);
+  ls.back()->shared_bdev = 1;
+  ls.back()->dedicated_db = true;
+  ls.back()->dedicated_wal = true;
+}
+
 // bluefs_super_t
 
 void bluefs_super_t::encode(bufferlist& bl) const
diff --git a/src/os/bluestore/bluefs_types.h b/src/os/bluestore/bluefs_types.h
index b0ce7c5c9d38..6516f404e12b 100644
--- a/src/os/bluestore/bluefs_types.h
+++ b/src/os/bluestore/bluefs_types.h
@@ -205,6 +205,7 @@ struct bluefs_layout_t {
   void encode(ceph::buffer::list& bl) const;
   void decode(ceph::buffer::list::const_iterator& p);
   void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<bluefs_layout_t*>& ls);
 };
 WRITE_CLASS_ENCODER(bluefs_layout_t)
 
diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h
index 66d4189ab6d1..500a78a143f1 100644
--- a/src/os/bluestore/bluestore_types.h
+++ b/src/os/bluestore/bluestore_types.h
@@ -158,6 +158,14 @@ struct bluestore_extent_ref_map_t {
       denc_varint_lowz(v.length, p);
       denc_varint(v.refs, p);
     }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("length", length);
+      f->dump_unsigned("refs", refs);
+    }
+    static void generate_test_instances(std::list<record_t*>& o) {
+      o.push_back(new record_t);
+      o.push_back(new record_t(123, 456));
+    }
   };
 
   typedef mempool::bluestore_cache_other::map<uint64_t,record_t> map_t;
@@ -229,7 +237,7 @@ struct bluestore_extent_ref_map_t {
   static void generate_test_instances(std::list<bluestore_extent_ref_map_t*>& o);
 };
 WRITE_CLASS_DENC(bluestore_extent_ref_map_t)
-
+WRITE_CLASS_DENC(bluestore_extent_ref_map_t::record_t)
 
 std::ostream& operator<<(std::ostream& out, const bluestore_extent_ref_map_t& rm);
 static inline bool operator==(const bluestore_extent_ref_map_t::record_t& l,
@@ -973,6 +981,7 @@ struct bluestore_shared_blob_t {
   uint64_t sbid;                       ///> shared blob id
   bluestore_extent_ref_map_t ref_map;  ///< shared blob extents
 
+  bluestore_shared_blob_t() : sbid(0) {}
   bluestore_shared_blob_t(uint64_t _sbid) : sbid(_sbid) {}
   bluestore_shared_blob_t(uint64_t _sbid,
 			  bluestore_extent_ref_map_t&& _ref_map ) 
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 963039d0213f..e37aeafc4312 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -342,6 +342,12 @@ struct PGTempMap {
       f->close_section();
     }
   }
+  static void generate_test_instances(std::list<PGTempMap*>& o) {
+    o.push_back(new PGTempMap);
+    o.push_back(new PGTempMap);
+    o.back()->set(pg_t(1, 2), { 3, 4 });
+    o.back()->set(pg_t(2, 3), { 4, 5 });
+  }
 };
 WRITE_CLASS_ENCODER(PGTempMap)
 
diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index e41cd1df08e3..e79bc5d450a2 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -280,6 +280,22 @@ void SnapMapper::object_snaps::decode(ceph::buffer::list::const_iterator &bl)
   DECODE_FINISH(bl);
 }
 
+void SnapMapper::object_snaps::dump(ceph::Formatter *f) const
+{
+  f->dump_stream("oid") << oid;
+  f->dump_stream("snaps") << snaps;
+}
+
+void SnapMapper::object_snaps::generate_test_instances(
+  std::list<object_snaps *> &o)
+{
+  o.push_back(new object_snaps);
+  o.push_back(new object_snaps);
+  o.back()->oid = hobject_t(sobject_t("name", CEPH_NOSNAP));
+  o.back()->snaps.insert(1);
+  o.back()->snaps.insert(2);
+}
+
 bool SnapMapper::check(const hobject_t &hoid) const
 {
   if (hoid.match(mask_bits, match)) {
diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index a28b25970fb4..a0a4cd3221b9 100644
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -132,6 +132,8 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     object_snaps() {}
     void encode(ceph::buffer::list &bl) const;
     void decode(ceph::buffer::list::const_iterator &bp);
+    void dump(ceph::Formatter *f) const;
+    static void generate_test_instances(std::list<object_snaps*>& o);
   };
 
   struct Mapping {
@@ -152,6 +154,16 @@ class SnapMapper : public Scrub::SnapMapReaderI {
       decode(hoid, bl);
       DECODE_FINISH(bl);
     }
+    void dump(ceph::Formatter *f) const {
+      f->dump_unsigned("snap", snap);
+      f->dump_stream("hoid") << hoid;
+    }
+    static void generate_test_instances(std::list<Mapping*>& o) {
+      o.push_back(new Mapping);
+      o.push_back(new Mapping);
+      o.back()->snap = 1;
+      o.back()->hoid = hobject_t(object_t("objname"), "key", 123, 456, 0, "");
+    }
   };
 
   static const std::string LEGACY_MAPPING_PREFIX;
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 948abeaafc8a..acb9777ffe9c 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -1528,6 +1528,11 @@ void pool_opts_t::decode(ceph::buffer::list::const_iterator& bl)
   DECODE_FINISH(bl);
 }
 
+void pool_opts_t::generate_test_instances(std::list<pool_opts_t*>& o)
+{
+  o.push_back(new pool_opts_t);
+}
+
 ostream& operator<<(ostream& out, const pool_opts_t& opts)
 {
   for (auto i = opt_mapping.begin(); i != opt_mapping.end(); ++i) {
@@ -3614,6 +3619,7 @@ void pg_info_t::decode(ceph::buffer::list::const_iterator &bl)
 void pg_info_t::dump(Formatter *f) const
 {
   f->dump_stream("pgid") << pgid;
+  f->dump_stream("shared") << pgid.shard;
   f->dump_stream("last_update") << last_update;
   f->dump_stream("last_complete") << last_complete;
   f->dump_stream("log_tail") << log_tail;
@@ -3714,10 +3720,11 @@ void pg_notify_t::dump(Formatter *f) const
 
 void pg_notify_t::generate_test_instances(list<pg_notify_t*>& o)
 {
+  o.push_back(new pg_notify_t);
   o.push_back(new pg_notify_t(shard_id_t(3), shard_id_t::NO_SHARD, 1, 1,
-			      pg_info_t(), PastIntervals()));
-  o.push_back(new pg_notify_t(shard_id_t(0), shard_id_t(0), 3, 10,
-			      pg_info_t(), PastIntervals()));
+            pg_info_t(spg_t(pg_t(0,10), shard_id_t(-1))), PastIntervals()));
+  o.push_back(new pg_notify_t(shard_id_t(0), shard_id_t(2), 3, 10,
+            pg_info_t(spg_t(pg_t(10,10), shard_id_t(2))), PastIntervals()));
 }
 
 ostream &operator<<(ostream &lhs, const pg_notify_t &notify)
@@ -3812,9 +3819,6 @@ void PastIntervals::pg_interval_t::generate_test_instances(list<pg_interval_t*>&
   o.back()->maybe_went_rw = true;
 }
 
-WRITE_CLASS_ENCODER(PastIntervals::pg_interval_t)
-
-
 /**
  * pi_compact_rep
  *
@@ -6059,6 +6063,14 @@ void chunk_info_t::dump(Formatter *f) const
   f->dump_unsigned("flags", flags);
 }
 
+void chunk_info_t::generate_test_instances(std::list<chunk_info_t*>& o)
+{
+  o.push_back(new chunk_info_t);
+  o.push_back(new chunk_info_t);
+  o.back()->length = 123;
+  o.back()->oid = hobject_t(object_t("foo"), "", 123, 456, -1, "");
+  o.back()->flags = cflag_t::FLAG_DIRTY;
+}
 
 bool chunk_info_t::operator==(const chunk_info_t& cit) const
 {
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 8b86b0a36356..11c8e7a3b9b6 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -196,6 +196,11 @@ struct pg_shard_t {
       f->dump_unsigned("shard", shard);
     }
   }
+  static void generate_test_instances(std::list<pg_shard_t*>& o) {
+    o.push_back(new pg_shard_t);
+    o.push_back(new pg_shard_t(1));
+    o.push_back(new pg_shard_t(1, shard_id_t(2)));
+  }
   auto operator<=>(const pg_shard_t&) const = default;
 };
 WRITE_CLASS_ENCODER(pg_shard_t)
@@ -596,7 +601,14 @@ struct spg_t {
     decode(shard, bl);
     DECODE_FINISH(bl);
   }
-
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("pgid") << pgid;
+    f->dump_unsigned("shard", shard);
+  }
+  static void generate_test_instances(std::list<spg_t*>& o) {
+    o.push_back(new spg_t);
+    o.push_back(new spg_t(pg_t(1, 2), shard_id_t(3)));
+  }
   ghobject_t make_temp_ghobject(const std::string& name) const {
     return ghobject_t(
       hobject_t(object_t(name), "", CEPH_NOSNAP,
@@ -922,6 +934,14 @@ class eversion_t {
     auto p = std::cbegin(bl);
     decode(p);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_unsigned("version", version);
+    f->dump_unsigned("epoch", epoch);
+  }
+  static void generate_test_instances(std::list<eversion_t*>& o) {
+    o.push_back(new eversion_t);
+    o.push_back(new eversion_t(1, 2));
+  }
 };
 WRITE_CLASS_ENCODER(eversion_t)
 
@@ -1143,6 +1163,7 @@ class pool_opts_t {
   void dump(ceph::Formatter *f) const;
   void encode(ceph::buffer::list &bl, uint64_t features) const;
   void decode(ceph::buffer::list::const_iterator &bl);
+  static void generate_test_instances(std::list<pool_opts_t*>& o);
 
 private:
   typedef std::map<key_t, value_t> opts_t;
@@ -1188,6 +1209,16 @@ struct pg_merge_meta_t {
     f->dump_stream("source_version") << source_version;
     f->dump_stream("target_version") << target_version;
   }
+  static void generate_test_instances(std::list<pg_merge_meta_t*>& o) {
+    o.push_back(new pg_merge_meta_t);
+    o.push_back(new pg_merge_meta_t);
+    o.back()->source_pgid = pg_t(1,2);
+    o.back()->ready_epoch = 1;
+    o.back()->last_epoch_started = 2;
+    o.back()->last_epoch_clean = 3;
+    o.back()->source_version = eversion_t(4,5);
+    o.back()->target_version = eversion_t(6,7);
+  }
 };
 WRITE_CLASS_ENCODER(pg_merge_meta_t)
 
@@ -3187,6 +3218,46 @@ struct pg_fast_info_t {
     decode(stats.stats.sum.num_objects_dirty, p);
     DECODE_FINISH(p);
   }
+  void dump(ceph::Formatter *f) const {
+    f->dump_stream("last_update") << last_update;
+    f->dump_stream("last_complete") << last_complete;
+    f->dump_stream("last_user_version") << last_user_version;
+    f->open_object_section("stats");
+    f->dump_stream("version") << stats.version;
+    f->dump_unsigned("reported_seq", stats.reported_seq);
+    f->dump_stream("last_fresh") << stats.last_fresh;
+    f->dump_stream("last_active") << stats.last_active;
+    f->dump_stream("last_peered") << stats.last_peered;
+    f->dump_stream("last_clean") << stats.last_clean;
+    f->dump_stream("last_unstale") << stats.last_unstale;
+    f->dump_stream("last_undegraded") << stats.last_undegraded;
+    f->dump_stream("last_fullsized") << stats.last_fullsized;
+    f->dump_unsigned("log_size", stats.log_size);
+    f->dump_unsigned("ondisk_log_size", stats.log_size);
+    f->dump_unsigned("num_bytes", stats.stats.sum.num_bytes);
+    f->dump_unsigned("num_objects", stats.stats.sum.num_objects);
+    f->dump_unsigned("num_object_copies", stats.stats.sum.num_object_copies);
+    f->dump_unsigned("num_rd", stats.stats.sum.num_rd);
+    f->dump_unsigned("num_rd_kb", stats.stats.sum.num_rd_kb);
+    f->dump_unsigned("num_wr", stats.stats.sum.num_wr);
+    f->dump_unsigned("num_wr_kb", stats.stats.sum.num_wr_kb);
+    f->dump_unsigned("num_objects_dirty", stats.stats.sum.num_objects_dirty);
+    f->close_section();
+  }
+  static void generate_test_instances(std::list<pg_fast_info_t*>& o) {
+    o.push_back(new pg_fast_info_t);
+    o.push_back(new pg_fast_info_t);
+    o.back()->last_update = eversion_t(1, 2);
+    o.back()->last_complete = eversion_t(3, 4);
+    o.back()->last_user_version = version_t(5);
+    o.back()->stats.version = eversion_t(7, 8);
+    o.back()->stats.reported_seq = 9;
+    o.back()->stats.last_fresh = utime_t(10, 0);
+    o.back()->stats.last_active = utime_t(11, 0);
+    o.back()->stats.last_peered = utime_t(12, 0);
+    o.back()->stats.last_clean = utime_t(13, 0);
+    o.back()->stats.last_unstale = utime_t(14, 0);
+  }
 };
 WRITE_CLASS_ENCODER(pg_fast_info_t)
 
@@ -3548,6 +3619,7 @@ class PastIntervals {
   }
 };
 WRITE_CLASS_ENCODER(PastIntervals)
+WRITE_CLASS_ENCODER(PastIntervals::pg_interval_t)
 
 std::ostream& operator<<(std::ostream& out, const PastIntervals::pg_interval_t& i);
 std::ostream& operator<<(std::ostream& out, const PastIntervals &i);
@@ -4209,6 +4281,13 @@ struct pg_log_op_return_item_t {
     f->dump_int("rval", rval);
     f->dump_unsigned("bl_length", bl.length());
   }
+  static void generate_test_instances(std::list<pg_log_op_return_item_t*>& o) {
+    o.push_back(new pg_log_op_return_item_t);
+    o.back()->rval = 0;
+    o.push_back(new pg_log_op_return_item_t);
+    o.back()->rval = 1;
+    o.back()->bl.append("asdf");
+  }
   friend bool operator==(const pg_log_op_return_item_t& lhs,
 			 const pg_log_op_return_item_t& rhs) {
     return lhs.rval == rhs.rval &&
@@ -5727,6 +5806,7 @@ struct chunk_info_t {
   void encode(ceph::buffer::list &bl) const;
   void decode(ceph::buffer::list::const_iterator &bl);
   void dump(ceph::Formatter *f) const;
+  static void generate_test_instances(std::list<chunk_info_t*>& ls);
   friend std::ostream& operator<<(std::ostream& out, const chunk_info_t& ci);
   bool operator==(const chunk_info_t& cit) const;
   bool operator!=(const chunk_info_t& cit) const {
diff --git a/src/tools/ceph-dencoder/denc_plugin.h b/src/tools/ceph-dencoder/denc_plugin.h
index a203551eadc5..e43ad26e174f 100644
--- a/src/tools/ceph-dencoder/denc_plugin.h
+++ b/src/tools/ceph-dencoder/denc_plugin.h
@@ -66,6 +66,7 @@ class DencoderPlugin {
 };
 
 #define TYPE(t) plugin->emplace<DencoderImplNoFeature<t>>(#t, false, false);
+#define TYPE_VARARGS(t, ...) plugin->emplace<DencoderImplNoFeature<t>>(#t, false, false, ##__VA_ARGS__);
 #define TYPE_STRAYDATA(t) plugin->emplace<DencoderImplNoFeature<t>>(#t, true, false);
 #define TYPE_NONDETERMINISTIC(t) plugin->emplace<DencoderImplNoFeature<t>>(#t, false, true);
 #define TYPE_FEATUREFUL(t) plugin->emplace<DencoderImplFeatureful<t>>(#t, false, false);
diff --git a/src/tools/ceph-dencoder/denc_registry.h b/src/tools/ceph-dencoder/denc_registry.h
index aad52cbf7105..61acc833b90e 100644
--- a/src/tools/ceph-dencoder/denc_registry.h
+++ b/src/tools/ceph-dencoder/denc_registry.h
@@ -47,8 +47,9 @@ class DencoderBase : public Dencoder {
   bool nondeterministic;
 
 public:
-  DencoderBase(bool stray_okay, bool nondeterministic)
-    : m_object(new T),
+  template<typename... Args>
+  DencoderBase(bool stray_okay, bool nondeterministic, Args&&... args)
+    : m_object(new T(std::forward<Args>(args)...)),
       stray_okay(stray_okay),
       nondeterministic(nondeterministic) {}
   ~DencoderBase() override {
diff --git a/src/tools/ceph-dencoder/osd_types.h b/src/tools/ceph-dencoder/osd_types.h
index 2dcaeb92827d..749233e3b7d1 100644
--- a/src/tools/ceph-dencoder/osd_types.h
+++ b/src/tools/ceph-dencoder/osd_types.h
@@ -3,6 +3,7 @@ TYPE(osd_info_t)
 TYPE_FEATUREFUL(osd_xinfo_t)
 TYPE_FEATUREFUL_NOCOPY(OSDMap)
 TYPE_FEATUREFUL_STRAYDATA(OSDMap::Incremental)
+TYPE(PGTempMap)
 
 #include "osd/osd_types.h"
 TYPE(osd_reqid_t)
@@ -24,12 +25,12 @@ TYPE(pg_hit_set_history_t)
 TYPE(pg_history_t)
 TYPE(pg_info_t)
 TYPE(PastIntervals)
+TYPE(PastIntervals::pg_interval_t)
 TYPE_FEATUREFUL(pg_query_t)
 TYPE(ObjectModDesc)
 TYPE(pg_log_entry_t)
 TYPE(pg_log_dup_t)
 TYPE(pg_log_t)
-TYPE_FEATUREFUL(pg_missing_item)
 TYPE_FEATUREFUL(pg_missing_t)
 TYPE(pg_nls_response_t)
 TYPE(pg_ls_response_t)
@@ -54,6 +55,22 @@ TYPE_FEATUREFUL(obj_list_watch_response_t)
 TYPE(clone_info)
 TYPE(obj_list_snap_response_t)
 TYPE(pool_pg_num_history_t)
+TYPE(store_statfs_t)
+TYPE(spg_t)
+TYPE(ObjectCleanRegions)
+TYPE(pg_notify_t) 
+TYPE(pg_merge_meta_t)
+TYPE(pg_shard_t)
+TYPE(pg_fast_info_t)
+TYPE(pg_lease_ack_t)
+TYPE(pg_lease_t)
+TYPE(pg_log_op_return_item_t)
+TYPE(chunk_info_t)
+TYPE_FEATUREFUL(pool_opts_t)
+TYPE_FEATUREFUL(pg_missing_item)
+TYPE(eversion_t)
+//TYPE(compact_interval_t) declared in .cc
+//TYPE(pg_missing_t::item)
 
 #include "osd/ECUtil.h"
 // TYPE(stripe_info_t) non-standard encoding/decoding functions
@@ -72,6 +89,13 @@ TYPE(BloomHitSet)
 TYPE_NONDETERMINISTIC(HitSet)   // because some subclasses are
 TYPE(HitSet::Params)
 
+#include "osd/SnapMapper.h"
+TYPE(SnapMapper::Mapping)
+TYPE(SnapMapper::object_snaps)
+
+#include "os/Transaction.h"
+TYPE(ceph::os::Transaction)
+
 #include "os/ObjectStore.h"
 TYPE(ObjectStore::Transaction)
 
@@ -84,6 +108,7 @@ TYPE(bluestore_bdev_label_t)
 TYPE(bluestore_cnode_t)
 TYPE(bluestore_compression_header_t)
 TYPE(bluestore_extent_ref_map_t)
+TYPE_FEATUREFUL(bluestore_extent_ref_map_t::record_t)
 TYPE(bluestore_pextent_t)
 TYPE(bluestore_blob_use_tracker_t)
 // TODO: bluestore_blob_t repurposes the "feature" param of encode() for its
@@ -92,8 +117,11 @@ TYPE(bluestore_blob_use_tracker_t)
 // BlueStore::ExtentMap::encode_some(). maybe we can test it using another
 // approach.
 // TYPE_FEATUREFUL(bluestore_blob_t)
-// TYPE(bluestore_shared_blob_t) there is no encode here
+TYPE(bluestore_shared_blob_t)
 TYPE(bluestore_onode_t)
+TYPE(bluestore_onode_t::shard_info)
+using shard_info = bluestore_onode_t::shard_info;
+TYPE(shard_info)
 TYPE(bluestore_deferred_op_t)
 TYPE(bluestore_deferred_transaction_t)
 // TYPE(bluestore_compression_header_t) there is no encode here
@@ -103,6 +131,7 @@ TYPE(bluefs_extent_t)
 TYPE(bluefs_fnode_t)
 TYPE(bluefs_super_t)
 TYPE(bluefs_transaction_t)
+TYPE(bluefs_layout_t)
 #endif
 
 #include "mon/AuthMonitor.h"
@@ -110,7 +139,9 @@ TYPE_FEATUREFUL(AuthMonitor::Incremental)
 
 #include "mon/PGMap.h"
 TYPE_FEATUREFUL_NONDETERMINISTIC(PGMapDigest)
+TYPE(PGMapDigest::pg_count)
 TYPE_FEATUREFUL_NONDETERMINISTIC(PGMap)
+//TYPE(PGMap::Incremental)
 
 #include "mon/MonitorDBStore.h"
 TYPE(MonitorDBStore::Transaction)
@@ -118,29 +149,57 @@ TYPE(MonitorDBStore::Op)
 
 #include "mon/MonMap.h"
 TYPE_FEATUREFUL(MonMap)
+TYPE_FEATUREFUL(mon_info_t)
+
+#include "mon/MonCommand.h"
+TYPE(MonCommand)
 
 #include "mon/MonCap.h"
 TYPE(MonCap)
 
 #include "mon/MgrMap.h"
 TYPE_FEATUREFUL(MgrMap)
+TYPE(MgrMap::ModuleOption)
+TYPE(MgrMap::ModuleInfo)
+TYPE(MgrMap::StandbyInfo)
 
 #include "mon/mon_types.h"
 TYPE(MonitorDBStoreStats)
 TYPE(ScrubResult)
+TYPE(mon_feature_t)
+TYPE_FEATUREFUL(DataStats)
+TYPE_FEATUREFUL(ProgressEvent)
+TYPE(FeatureMap)
 
 #include "mon/CreatingPGs.h"
 TYPE_FEATUREFUL(creating_pgs_t)
+TYPE_FEATUREFUL(creating_pgs_t::pg_create_info)
 
 #include "mgr/ServiceMap.h"
 TYPE_FEATUREFUL(ServiceMap)
 TYPE_FEATUREFUL(ServiceMap::Service)
 TYPE_FEATUREFUL(ServiceMap::Daemon)
 
+#include "mgr/DaemonHealthMetric.h"
+TYPE(DaemonHealthMetric)
+
+#include "mgr/MetricTypes.h"
+TYPE(OSDMetricPayload)
+
+#include "mgr/OSDPerfMetricTypes.h"
+TYPE(OSDPerfMetricQuery)
+TYPE(OSDPerfMetricSubKeyDescriptor)
+TYPE(PerformanceCounterDescriptor)
+TYPE(OSDPerfMetricReport)
+
 #include "mon/ConnectionTracker.h"
 TYPE(ConnectionReport);
 TYPE(ConnectionTracker);
 
+#include "mon/health_check.h"
+TYPE(health_check_t)
+TYPE(health_check_map_t)
+
 #include "os/DBObjectMap.h"
 TYPE(DBObjectMap::_Header)
 TYPE(DBObjectMap::State)

From d674330ec573901bb46c544952c9a63275d5adc9 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 20 Dec 2023 15:37:00 +0000
Subject: [PATCH 1116/2492] osd/osd/PrimaryLogPG: use should_whiteout helper in
 _delete_oid()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/PrimaryLogPG.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 5cf9bbba68d3..0d36fe65f52b 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -8186,8 +8186,7 @@ inline int PrimaryLogPG::_delete_oid(
   // in luminous or later, we can't delete the head if there are
   // clones. we trust the caller passing no_whiteout has already
   // verified they don't exist.
-  if (!snapset.clones.empty() ||
-      (!ctx->snapc.snaps.empty() && ctx->snapc.snaps[0] > snapset.seq)) {
+  if (should_whiteout(snapset, ctx->snapc)) {
     if (no_whiteout) {
       dout(20) << __func__ << " has or will have clones but no_whiteout=1"
 	       << dendl;

From ff20d130e3397ee69e5f849c05a367465af61f31 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 14 Dec 2023 08:27:13 -0600
Subject: [PATCH 1117/2492] osd/scrub: add a basic set of performance counters

Add a labeled set of performance counters, with the labels selecting
one of four copies of the counters - one per each scrub level and
pool type combination.

Inside the Scrubber, the relevant set of counters is selected
when the scrub is initialized.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_perf_counters.cc  | 44 ++++++++++++++++++++++++++++++++
 src/osd/osd_perf_counters.h   | 41 ++++++++++++++++++++++++++++++
 src/osd/scrubber/osd_scrub.cc | 40 ++++++++++++++++++++++++++++-
 src/osd/scrubber/osd_scrub.h  | 48 ++++++++++++++++++++++++++++++++++-
 4 files changed, 171 insertions(+), 2 deletions(-)

diff --git a/src/osd/osd_perf_counters.cc b/src/osd/osd_perf_counters.cc
index 3e14244329eb..af8c62fb3042 100644
--- a/src/osd/osd_perf_counters.cc
+++ b/src/osd/osd_perf_counters.cc
@@ -319,6 +319,25 @@ PerfCounters *build_osd_logger(CephContext *cct) {
   osd_plb.add_u64_counter(
     l_osd_pg_biginfo, "osd_pg_biginfo", "PG updated its biginfo attr");
 
+  /// scrub's replicas reservation time/#replicas histogram
+  PerfHistogramCommon::axis_config_d rsrv_hist_x_axis_config{
+      "number of replicas",
+      PerfHistogramCommon::SCALE_LINEAR,
+      0,   ///< Start at 0
+      1,   ///< Quantization unit is 1
+      8,   ///< 9 OSDs in the active set
+  };
+  PerfHistogramCommon::axis_config_d rsrv_hist_y_axis_config{
+      "duration",
+      PerfHistogramCommon::SCALE_LOG2,	///< Request size in logarithmic scale
+      0,				///< Start at 0
+      250'000,				///< 250us granularity
+      10,				///< should be enough
+  };
+  osd_plb.add_u64_counter_histogram(
+      l_osd_scrub_reservation_dur_hist, "scrub_resrv_repnum_vs_duration",
+      rsrv_hist_x_axis_config, rsrv_hist_y_axis_config, "Histogram of scrub replicas reservation duration");
+
   return osd_plb.create_perf_counters();
 }
  
@@ -360,3 +379,28 @@ PerfCounters *build_recoverystate_perf(CephContext *cct) {
 
   return rs_perf.create_perf_counters();
 }
+
+
+PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label)
+{
+  // the labels matrix is:
+  //   <shallow/deep>  X  <replicated/EC>  // maybe later we'll add <periodic/operator>
+  PerfCountersBuilder scrub_perf(cct, label, scrbcnt_first, scrbcnt_last);
+
+  scrub_perf.set_prio_default(PerfCountersBuilder::PRIO_INTERESTING);
+
+  scrub_perf.add_u64_counter(scrbcnt_started, "num_scrubs_started", "scrubs attempted count");
+  scrub_perf.add_u64_counter(scrbcnt_active_started, "num_scrubs_past_reservation", "scrubs count");
+  scrub_perf.add_u64_counter(scrbcnt_failed, "failed_scrubs", "failed scrubs count");
+  scrub_perf.add_u64_counter(scrbcnt_successful, "successful_scrubs", "successful scrubs count");
+  scrub_perf.add_time_avg(scrbcnt_failed_elapsed, "failed_scrubs_elapsed", "time to scrub failure");
+  scrub_perf.add_time_avg(scrbcnt_successful_elapsed, "successful_scrubs_elapsed", "time to scrub completion");
+
+  scrub_perf.add_u64_counter(scrbcnt_preempted, "preemptions", "preemptions on scrubs");
+  scrub_perf.add_u64_counter(scrbcnt_chunks_selected, "chunk_selected", "chunk selection during scrubs");
+  scrub_perf.add_u64_counter(scrbcnt_chunks_busy, "chunk_busy", "chunk busy during scrubs");
+  scrub_perf.add_u64_counter(scrbcnt_blocked, "locked_object", "waiting on locked object events");
+  scrub_perf.add_u64_counter(scrbcnt_write_blocked, "write_blocked_by_scrub", "write blocked by scrub");
+
+  return scrub_perf.create_perf_counters();
+}
diff --git a/src/osd/osd_perf_counters.h b/src/osd/osd_perf_counters.h
index 2445a9dc2c38..1f65e953d337 100644
--- a/src/osd/osd_perf_counters.h
+++ b/src/osd/osd_perf_counters.h
@@ -5,6 +5,7 @@
 
 #include "include/common_fwd.h"
 #include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
 
 enum {
   l_osd_first = 10000,
@@ -131,6 +132,10 @@ enum {
   l_osd_pg_fastinfo,
   l_osd_pg_biginfo,
 
+  // scrubber related. Here, as the rest of the scrub counters
+  // are labeled, and histograms do not fully support labels.
+  l_osd_scrub_reservation_dur_hist,
+
   l_osd_last,
 };
 
@@ -174,3 +179,39 @@ enum {
 };
 
 PerfCounters *build_recoverystate_perf(CephContext *cct);
+
+// Scrubber perf counters. There are four sets (shallow vs. deep,
+// EC vs. replicated) of these counters:
+enum {
+  scrbcnt_first = 20500,
+
+  // -- basic statistics --
+  /// The number of times we started a scrub
+  scrbcnt_started,
+  /// # scrubs that got past replicas reservation
+  scrbcnt_active_started,
+  /// # successful scrubs
+  scrbcnt_successful,
+  /// time to complete a successful scrub
+  scrbcnt_successful_elapsed,
+  /// # failed scrubs
+  scrbcnt_failed,
+  /// time for a scrub to fail
+  scrbcnt_failed_elapsed,
+
+  // -- interruptions of various types
+  /// # preemptions
+  scrbcnt_preempted,
+  /// # chunks selection performed
+  scrbcnt_chunks_selected,
+  /// # busy chunks
+  scrbcnt_chunks_busy,
+  /// # waiting on object events
+  scrbcnt_blocked,
+  /// # write blocked by the scrub
+  scrbcnt_write_blocked,
+
+  scrbcnt_last,
+};
+
+PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label);
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index f11ddd2737ca..5c4eebba2953 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -4,6 +4,7 @@
 #include "./osd_scrub.h"
 
 #include "osd/OSD.h"
+#include "osd/osd_perf_counters.h"
 #include "osdc/Objecter.h"
 
 #include "pg_scrubber.h"
@@ -37,7 +38,14 @@ OsdScrub::OsdScrub(
     , m_queue{cct, m_osd_svc}
     , m_log_prefix{fmt::format("osd.{} osd-scrub:", m_osd_svc.get_nodeid())}
     , m_load_tracker{cct, conf, m_osd_svc.get_nodeid()}
-{}
+{
+  create_scrub_perf_counters();
+}
+
+OsdScrub::~OsdScrub()
+{
+  destroy_scrub_perf_counters();
+}
 
 std::ostream& OsdScrub::gen_prefix(std::ostream& out, std::string_view fn) const
 {
@@ -377,6 +385,36 @@ std::chrono::milliseconds OsdScrub::scrub_sleep_time(
   return std::max(extended_sleep, regular_sleep_period);
 }
 
+
+// ////////////////////////////////////////////////////////////////////////// //
+// scrub-related performance counters
+
+void OsdScrub::create_scrub_perf_counters()
+{
+  auto idx = perf_counters_indices.begin();
+  // create a separate set for each pool type & scrub level
+  for (const auto& label : perf_labels) {
+    PerfCounters* counters = build_scrub_labeled_perf(cct, label);
+    cct->get_perfcounters_collection()->add(counters);
+    m_perf_counters[*idx++] = counters;
+  }
+}
+
+void OsdScrub::destroy_scrub_perf_counters()
+{
+  for (const auto& [label, counters] : m_perf_counters) {
+    std::ignore = label;
+    cct->get_perfcounters_collection()->remove(counters);
+    delete counters;
+  }
+  m_perf_counters.clear();
+}
+
+PerfCounters* OsdScrub::get_perf_counters(int pool_type, scrub_level_t level)
+{
+  return m_perf_counters[pc_index_t{level, pool_type}];
+}
+
 // ////////////////////////////////////////////////////////////////////////// //
 // forwarders to the queue
 
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index dae11f860011..ce7b8524d69c 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -5,6 +5,7 @@
 #include <string_view>
 
 #include "osd/osd_types_fmt.h"
+#include "osd/osd_perf_counters.h"
 #include "osd/scrubber/osd_scrub_sched.h"
 #include "osd/scrubber/scrub_resources.h"
 #include "osd/scrubber_common.h"
@@ -26,7 +27,7 @@ class OsdScrub {
       Scrub::ScrubSchedListener& osd_svc,
       const ceph::common::ConfigProxy& config);
 
-  ~OsdScrub() = default;
+  ~OsdScrub();
 
   // note: public, as accessed by the dout macros
   std::ostream& gen_prefix(std::ostream& out, std::string_view fn) const;
@@ -162,6 +163,10 @@ class OsdScrub {
    */
   std::optional<double> update_load_average();
 
+   // the scrub performance counters collections
+   // ---------------------------------------------------------------
+  PerfCounters* get_perf_counters(int pool_type, scrub_level_t level);
+
  private:
   CephContext* cct;
   Scrub::ScrubSchedListener& m_osd_svc;
@@ -238,4 +243,45 @@ class OsdScrub {
     std::ostream& gen_prefix(std::ostream& out, std::string_view fn) const;
   };
   LoadTracker m_load_tracker;
+
+  // the scrub performance counters collections
+  // ---------------------------------------------------------------
+
+  // indexed by scrub level & pool type
+
+  using pc_index_t = std::pair<scrub_level_t, int /*pool type*/>;
+  // easy way to loop over the counter sets. Order must match the
+  // perf_labels vector
+  static inline std::array<pc_index_t, 4> perf_counters_indices = {
+      pc_index_t{scrub_level_t::shallow, pg_pool_t::TYPE_REPLICATED},
+      pc_index_t{scrub_level_t::deep, pg_pool_t::TYPE_REPLICATED},
+      pc_index_t{scrub_level_t::shallow, pg_pool_t::TYPE_ERASURE},
+      pc_index_t{scrub_level_t::deep, pg_pool_t::TYPE_ERASURE}};
+
+  std::map<pc_index_t, ceph::common::PerfCounters*> m_perf_counters;
+
+  // the labels matrix is: <shallow/deep>  X  <replicated/EC>
+  static inline std::vector<std::string> perf_labels = {
+      ceph::perf_counters::key_create(
+	  "osd_scrub_sh_repl",
+	  {{"level", "shallow"}, {"pooltype", "replicated"}}),
+      ceph::perf_counters::key_create(
+	  "osd_scrub_dp_repl",
+	  {{"level", "deep"}, {"pooltype", "replicated"}}),
+      ceph::perf_counters::key_create(
+	  "osd_scrub_sh_ec",
+	  {{"level", "shallow"}, {"pooltype", "ec"}}),
+      ceph::perf_counters::key_create(
+	  "osd_scrub_dp_ec",
+	  {{"level", "deep"}, {"pooltype", "ec"}})};
+
+  /**
+   * create 4 sets of performance counters (for shallow vs. deep,
+   * replicated vs. erasure pools). Add them to the cct, but also maintain
+   * a separate map of the counters, indexed by the pool type and scrub level.
+   */
+  void create_scrub_perf_counters();
+
+  // 'remove' the counters from the cct, and delete them
+  void destroy_scrub_perf_counters();
 };

From ddefb777a440b1a1ccd674a5d0e9fda4648ce734 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 16 Dec 2023 04:02:54 -0600
Subject: [PATCH 1118/2492] osd/scrub: provide selected counters set to
 Scrubber & FSM

also - move most time keeping from the Scrubber to the FSM.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc          |  3 +-
 src/osd/scrubber/pg_scrubber.cc        | 51 +++++++++++++---------
 src/osd/scrubber/pg_scrubber.h         |  8 ++--
 src/osd/scrubber/scrub_machine.cc      | 59 +++++++++++++++++++++++---
 src/osd/scrubber/scrub_machine.h       | 20 ++++++---
 src/osd/scrubber/scrub_machine_lstnr.h | 13 ++++--
 src/osd/scrubber_common.h              |  3 ++
 7 files changed, 116 insertions(+), 41 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index 5c4eebba2953..e5abe03feaad 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -395,8 +395,9 @@ void OsdScrub::create_scrub_perf_counters()
   // create a separate set for each pool type & scrub level
   for (const auto& label : perf_labels) {
     PerfCounters* counters = build_scrub_labeled_perf(cct, label);
+    ceph_assert(counters);
     cct->get_perfcounters_collection()->add(counters);
-    m_perf_counters[*idx++] = counters;
+    m_perf_counters[*(idx++)] = counters;
   }
 }
 
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 11112bc48d96..06dff82b3612 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -926,6 +926,8 @@ bool PgScrubber::select_range()
 
 void PgScrubber::select_range_n_notify()
 {
+  get_counters_set().inc(scrbcnt_chunks_selected);
+
   if (select_range()) {
     // the next chunk to handle is not blocked
     dout(20) << __func__ << ": selection OK" << dendl;
@@ -935,6 +937,7 @@ void PgScrubber::select_range_n_notify()
     // we will wait for the objects range to become available for scrubbing
     dout(10) << __func__ << ": selected chunk is busy" << dendl;
     m_osds->queue_scrub_chunk_busy(m_pg, Scrub::scrub_prio_t::low_priority);
+    get_counters_set().inc(scrbcnt_chunks_busy);
   }
 }
 
@@ -944,6 +947,7 @@ bool PgScrubber::write_blocked_by_scrub(const hobject_t& soid)
     return false;
   }
 
+  get_counters_set().inc(scrbcnt_write_blocked);
   dout(20) << __func__ << " " << soid << " can preempt? "
 	   << preemption_data.is_preemptable() << " already preempted? "
 	   << preemption_data.was_preempted() << dendl;
@@ -1062,6 +1066,11 @@ void PgScrubber::update_op_mode_text()
 	   << ". Displayed: " << m_mode_desc << dendl;
 }
 
+std::string_view PgScrubber::get_op_mode_text() const
+{
+  return m_mode_desc;
+}
+
 void PgScrubber::_request_scrub_map(pg_shard_t replica,
 				    eversion_t version,
 				    hobject_t start,
@@ -2077,14 +2086,15 @@ pg_scrubbing_status_t PgScrubber::get_schedule() const
 	false};
 
     } else {
-      int32_t duration = (utime_t{now_is} - scrub_begin_stamp).sec();
+      int32_t dur_seconds =
+	  duration_cast<seconds>(m_fsm->get_time_scrubbing()).count();
       return pg_scrubbing_status_t{
-	utime_t{},
-	duration,
-	pg_scrub_sched_status_t::active,
-	true,  // active
-	(m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow),
-	false /* is periodic? unknown, actually */};
+	  utime_t{},
+	  dur_seconds,
+	  pg_scrub_sched_status_t::active,
+	  true,	 // active
+	  (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow),
+	  false /* is periodic? unknown, actually */};
     }
   }
   if (m_scrub_job->state != Scrub::qu_state_t::registered) {
@@ -2175,26 +2185,25 @@ PgScrubber::PgScrubber(PG* pg)
       m_osds->cct, m_pg->pg_id, m_osds->get_nodeid());
 }
 
-void PgScrubber::set_scrub_begin_time()
+void PgScrubber::set_scrub_duration(std::chrono::milliseconds duration)
 {
-  scrub_begin_stamp = ceph_clock_now();
-  m_osds->clog->debug() << fmt::format(
-    "{} {} starts",
-    m_pg->info.pgid.pgid,
-    m_mode_desc);
-}
-
-void PgScrubber::set_scrub_duration()
-{
-  utime_t stamp = ceph_clock_now();
-  utime_t duration = stamp - scrub_begin_stamp;
+  dout(20) << fmt::format("{}: to {}", __func__, duration) << dendl;
+  double dur_ms = double(duration.count());
   m_pg->recovery_state.update_stats([=](auto& history, auto& stats) {
-    stats.last_scrub_duration = ceill(duration.to_msec() / 1000.0);
-    stats.scrub_duration = double(duration);
+    stats.last_scrub_duration = ceill(dur_ms / 1000.0);
+    stats.scrub_duration = dur_ms;
     return true;
   });
 }
 
+PerfCounters& PgScrubber::get_counters_set() const
+{
+  return *m_osds->get_scrub_services().get_perf_counters(
+      (m_pg->pool.info.is_replicated() ? pg_pool_t::TYPE_REPLICATED
+				       : pg_pool_t::TYPE_ERASURE),
+      (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow));
+}
+
 void PgScrubber::cleanup_on_finish()
 {
   dout(10) << __func__ << dendl;
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 9f71f2d83230..c6be51b2cae6 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -363,6 +363,7 @@ class PgScrubber : public ScrubPgIF,
   int get_whoami() const final;
   spg_t get_spgid() const final { return m_pg->get_pgid(); }
   PG* get_pg() const final { return m_pg; }
+  PerfCounters& get_counters_set() const final;
 
   // temporary interface (to be discarded in a follow-up PR)
   /// set the 'resources_failure' flag in the scrub-job object
@@ -470,11 +471,8 @@ class PgScrubber : public ScrubPgIF,
 
   std::string dump_awaited_maps() const final;
 
-  void set_scrub_begin_time() final;
+  void set_scrub_duration(std::chrono::milliseconds duration) final;
 
-  void set_scrub_duration() final;
-
-  utime_t scrub_begin_stamp;
   std::ostream& gen_prefix(std::ostream& out) const final;
 
   /// facilitate scrub-backend access to SnapMapper mappings
@@ -768,6 +766,8 @@ class PgScrubber : public ScrubPgIF,
 
   void update_op_mode_text();
 
+  std::string_view get_op_mode_text() const final;
+
  private:
   /**
    * initiate a deep-scrub after the current scrub ended with errors.
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 2e18ed5cf4e1..3a0cd9866d70 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -80,6 +80,27 @@ std::ostream& ScrubMachine::gen_prefix(std::ostream& out) const
   return m_scrbr->gen_prefix(out) << "FSM: ";
 }
 
+ceph::timespan ScrubMachine::get_time_scrubbing() const
+{
+  // note: the state_cast does not work in the Session ctor
+  auto session = state_cast<const Session*>();
+  if (!session) {
+    dout(20) << fmt::format("{}: not in session", __func__) << dendl;
+    return ceph::timespan{};
+  }
+
+  if (session && session->m_session_started_at != ScrubTimePoint{}) {
+    dout(20) << fmt::format(
+		    "{}: session_started_at: {} d:{}", __func__,
+		    session->m_session_started_at,
+		    ScrubClock::now() - session->m_session_started_at)
+	     << dendl;
+    return ScrubClock::now() - session->m_session_started_at;
+  }
+  dout(30) << fmt::format("{}: no session_start time", __func__) << dendl;
+  return ceph::timespan{};
+}
+
 // ////////////// the actual actions
 
 // ----------------------- NotActive -----------------------------------------
@@ -97,7 +118,6 @@ sc::result NotActive::react(const StartScrub&)
 {
   dout(10) << "NotActive::react(const StartScrub&)" << dendl;
   DECLARE_LOCALS;
-  scrbr->set_scrub_begin_time();
   return transit<ReservingReplicas>();
 }
 
@@ -105,7 +125,6 @@ sc::result NotActive::react(const AfterRepairScrub&)
 {
   dout(10) << "NotActive::react(const AfterRepairScrub&)" << dendl;
   DECLARE_LOCALS;
-  scrbr->set_scrub_begin_time();
   return transit<ReservingReplicas>();
 }
 
@@ -126,6 +145,9 @@ Session::Session(my_context ctx)
   // occasions will cause no harm.
   // We choose ignorance.
   std::ignore = scrbr->set_reserving_now();
+
+  m_perf_set = &scrbr->get_counters_set();
+  m_perf_set->inc(scrbcnt_started);
 }
 
 Session::~Session()
@@ -246,6 +268,12 @@ ActiveScrubbing::ActiveScrubbing(my_context ctx)
 {
   dout(10) << "-- state -->> ActiveScrubbing" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  auto& session = context<Session>();
+
+  session.m_perf_set->inc(scrbcnt_active_started);
+  scrbr->get_clog()->debug() << fmt::format(
+    "{} {} starts", machine.m_pg_id, scrbr->get_op_mode_text());
+
   scrbr->on_init();
 }
 
@@ -255,7 +283,16 @@ ActiveScrubbing::ActiveScrubbing(my_context ctx)
 ActiveScrubbing::~ActiveScrubbing()
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  auto& session = context<Session>();
   dout(15) << __func__ << dendl;
+
+  // if the begin-time stamp was not set 'off' (as done if the scrubbing
+  // completed successfully), we use it now to set the 'failed scrub' duration.
+  if (session.m_session_started_at != ScrubTimePoint{}) {
+    auto logged_duration = ScrubClock::now() - session.m_session_started_at;
+    session.m_perf_set->tinc(scrbcnt_failed_elapsed, logged_duration);
+    session.m_perf_set->inc(scrbcnt_failed);
+  }
 }
 
 /*
@@ -306,13 +343,14 @@ RangeBlocked::RangeBlocked(my_context ctx)
     m_timeout_token = machine.schedule_timer_event_after<RangeBlockedAlarm>(
       grace);
   }
+  context<Session>().m_perf_set->inc(scrbcnt_blocked);
 }
 
 sc::result RangeBlocked::react(const RangeBlockedAlarm&)
 {
   DECLARE_LOCALS;
   char buf[50];
-  std::time_t now_c = ceph::coarse_real_clock::to_time_t(entered_at);
+  std::time_t now_c = ScrubClock::to_time_t(entered_at);
   strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", std::localtime(&now_c));
   dout(10)
     << "PgScrubber: " << scrbr->get_spgid()
@@ -358,7 +396,7 @@ sc::result PendingTimer::react(const SleepComplete&)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "PendingTimer::react(const SleepComplete&)" << dendl;
 
-  auto slept_for = ceph::coarse_real_clock::now() - entered_at;
+  auto slept_for = ScrubClock::now() - entered_at;
   dout(20) << "PgScrubber: " << scrbr->get_spgid()
 	   << " slept for " << slept_for << dendl;
 
@@ -478,6 +516,7 @@ BuildMap::BuildMap(my_context ctx)
 {
   dout(10) << " -- state -->> Session/Act/BuildMap" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  auto& session = context<Session>();
 
   // no need to check for an epoch change, as all possible flows that brought
   // us here have a check_interval() verification of their final event.
@@ -488,6 +527,7 @@ BuildMap::BuildMap(my_context ctx)
     dout(10) << __func__ << " preempted!!!" << dendl;
     scrbr->mark_local_map_ready();
     post_event(IntBmPreempted{});
+    session.m_perf_set->inc(scrbcnt_preempted);
 
   } else {
 
@@ -640,7 +680,16 @@ sc::result WaitDigestUpdate::react(const ScrubFinished&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "WaitDigestUpdate::react(const ScrubFinished&)" << dendl;
-  scrbr->set_scrub_duration();
+  auto& session = context<Session>();
+
+  session.m_perf_set->inc(scrbcnt_successful);
+
+  // set the 'scrub duration'
+  auto duration = machine.get_time_scrubbing();
+  session.m_perf_set->tinc(scrbcnt_successful_elapsed, duration);
+  scrbr->set_scrub_duration(duration_cast<milliseconds>(duration));
+  session.m_session_started_at = ScrubTimePoint{};
+
   scrbr->scrub_finish();
   return transit<NotActive>();
 }
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 7e9c887145ae..42cf1b20f748 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -231,6 +231,8 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
   [[nodiscard]] bool is_reserving() const;
   [[nodiscard]] bool is_accepting_updates() const;
 
+  // elapsed time for the currently active scrub.session
+  ceph::timespan get_time_scrubbing() const;
 
 // ///////////////// aux declarations & functions //////////////////////// //
 
@@ -420,6 +422,13 @@ struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>,
   /// managing the scrub session's reservations (optional, as
   /// it's an RAII wrapper around the state of 'holding reservations')
   std::optional<ReplicaReservations> m_reservations{std::nullopt};
+
+  /// the relevant set of performance counters for this session
+  /// (relevant, i.e. for this pool type X scrub level)
+  PerfCounters* m_perf_set{nullptr};
+
+  /// the time when the session was initiated
+  ScrubTimePoint m_session_started_at{ScrubClock::now()};
 };
 
 struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
@@ -431,8 +440,7 @@ struct ReservingReplicas : sc::state<ReservingReplicas, Session>,
 			      sc::transition<RemotesReserved, ActiveScrubbing>,
 			      sc::custom_reaction<ReservationTimeout>>;
 
-  ceph::coarse_real_clock::time_point entered_at =
-    ceph::coarse_real_clock::now();
+  ScrubTimePoint entered_at = ScrubClock::now();
   ScrubMachine::timer_event_token_t m_timeout_token;
 
   /// a "raw" event carrying a peer's grant response
@@ -485,10 +493,9 @@ struct RangeBlocked : sc::state<RangeBlocked, ActiveScrubbing>, NamedSimply {
     sc::custom_reaction<RangeBlockedAlarm>,
     sc::transition<Unblocked, PendingTimer>>;
 
-  ceph::coarse_real_clock::time_point entered_at =
-    ceph::coarse_real_clock::now();
+  ScrubTimePoint entered_at = ScrubClock::now();
   ScrubMachine::timer_event_token_t m_timeout_token;
-  sc::result react(const RangeBlockedAlarm &);
+  sc::result react(const RangeBlockedAlarm&);
 };
 
 /**
@@ -506,8 +513,7 @@ struct PendingTimer : sc::state<PendingTimer, ActiveScrubbing>, NamedSimply {
     sc::transition<InternalSchedScrub, NewChunk>,
     sc::custom_reaction<SleepComplete>>;
 
-  ceph::coarse_real_clock::time_point entered_at =
-    ceph::coarse_real_clock::now();
+  ScrubTimePoint entered_at = ScrubClock::now();
   ScrubMachine::timer_event_token_t m_sleep_timer;
   sc::result react(const SleepComplete&);
 };
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 34247083f4fd..8b110fe62719 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -56,6 +56,12 @@ struct ScrubMachineListener {
   virtual spg_t get_spgid() const = 0;
   virtual PG* get_pg() const = 0;
 
+  /**
+   * access the set of performance counters relevant to this scrub
+   * (one of the four sets of counters maintained by the OSD)
+   */
+  virtual PerfCounters& get_counters_set() const = 0;
+
   using scrubber_callback_t = std::function<void(void)>;
   using scrubber_callback_cancel_token_t = Context*;
 
@@ -96,6 +102,9 @@ struct ScrubMachineListener {
   /// state.
   virtual void set_state_name(const char* name) = 0;
 
+  /// access the text specifying scrub level and whether it is a repair
+  virtual std::string_view get_op_mode_text() const = 0;
+
   [[nodiscard]] virtual bool is_primary() const = 0;
 
   virtual void select_range_n_notify() = 0;
@@ -174,9 +183,7 @@ struct ScrubMachineListener {
    */
   virtual void maps_compare_n_cleanup() = 0;
 
-  virtual void set_scrub_begin_time() = 0;
-
-  virtual void set_scrub_duration() = 0;
+  virtual void set_scrub_duration(std::chrono::milliseconds duration) = 0;
 
   /**
    * No new scrub session will start while a scrub was initiate on a PG,
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 63333177bd39..30d305036f44 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -4,6 +4,7 @@
 
 #include <fmt/ranges.h>
 
+#include "common/ceph_time.h"
 #include "common/scrub_types.h"
 #include "include/types.h"
 #include "os/ObjectStore.h"
@@ -15,6 +16,8 @@ class Formatter;
 }
 
 struct PGPool;
+using ScrubClock = ceph::coarse_real_clock;
+using ScrubTimePoint = ScrubClock::time_point;
 
 namespace Scrub {
   class ReplicaReservations;

From 5c0ae99be50c25c46fb31d29cfc3030094a7d7b0 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 16 Dec 2023 04:10:22 -0600
Subject: [PATCH 1119/2492] test/osd: log scrub performance counters in
 osd-scrub-test.sh

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/standalone/scrub/osd-scrub-test.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/qa/standalone/scrub/osd-scrub-test.sh b/qa/standalone/scrub/osd-scrub-test.sh
index e11100f8a198..171961676abd 100755
--- a/qa/standalone/scrub/osd-scrub-test.sh
+++ b/qa/standalone/scrub/osd-scrub-test.sh
@@ -39,6 +39,15 @@ function run() {
     done
 }
 
+function perf_counters() {
+    local dir=$1
+    local OSDS=$2
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      ceph tell osd.$osd counter dump | jq 'with_entries(select(.key | startswith("osd_scrub")))'
+    done
+}
+
 function TEST_scrub_test() {
     local dir=$1
     local poolname=test
@@ -115,6 +124,7 @@ function TEST_scrub_test() {
     test "$(ceph pg $pgid query | jq '.peer_info[0].stats.stat_sum.num_scrub_errors')" = "0" || return 1
     test "$(ceph pg $pgid query | jq '.peer_info[1].stats.stat_sum.num_scrub_errors')" = "0" || return 1
     ceph pg dump pgs | grep ^${pgid} | grep -vq -- +inconsistent || return 1
+    perf_counters $dir $OSDS
 }
 
 # Grab year-month-day
@@ -192,6 +202,7 @@ function TEST_interval_changes() {
     ceph osd pool set $poolname scrub_max_interval $(expr $week \* 3)
     sleep $WAIT_FOR_UPDATE
     check_dump_scrubs $primary "3 days" "3 week" || return 1
+    perf_counters $dir $OSDS
 }
 
 function TEST_scrub_extended_sleep() {
@@ -391,6 +402,7 @@ function _scrub_abort() {
     fi
     TIMEOUT=$(($objects / 2))
     wait_for_scrub $pgid "$last_scrub" || return 1
+    perf_counters $dir $OSDS
 }
 
 function TEST_scrub_abort() {
@@ -445,6 +457,7 @@ function TEST_scrub_permit_time() {
         fi
         sleep 1
     done
+    perf_counters $dir $OSDS
 }
 
 #  a test to recreate the problem described in bug #52901 - setting 'noscrub'
@@ -506,6 +519,7 @@ function TEST_just_deep_scrubs() {
     sc_data_2=()
     echo "test counter @ should be higher than before the unset: " ${sc_data_2['query_scrub_seq']}
     wait_any_cond $pgid 10 $saved_last_stamp expct_qry_duration "WaitingAfterScrub " sc_data_2 || return 1
+    perf_counters $dir ${cluster_conf['osds_num']}
 }
 
 function TEST_dump_scrub_schedule() {
@@ -624,6 +638,7 @@ function TEST_dump_scrub_schedule() {
     declare -A cond_active_dmp=( ['dmp_state_has_scrubbing']="true" ['query_active']="false" )
     sched_data=()
     wait_any_cond $pgid 10 $saved_last_stamp cond_active_dmp "WaitingActive " sched_data || return 1
+    perf_counters $dir $OSDS
 }
 
 function TEST_pg_dump_objects_scrubbed() {
@@ -659,6 +674,7 @@ function TEST_pg_dump_objects_scrubbed() {
     #Trigger a scrub on a PG
     pg_scrub $pgid || return 1
     test "$(ceph pg $pgid query | jq '.info.stats.objects_scrubbed')" '=' $objects || return 1
+    perf_counters $dir $OSDS
 
     teardown $dir || return 1
 }

From 35603069833065b531f4486994c2614d1cfd8e38 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 17 Dec 2023 12:14:53 -0600
Subject: [PATCH 1120/2492] osd/scrub: collect replica reservations performance
 counters

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_perf_counters.cc           | 12 +++++++-
 src/osd/osd_perf_counters.h            | 18 +++++++++++
 src/osd/scrubber/scrub_machine.cc      | 12 +++++---
 src/osd/scrubber/scrub_reservations.cc | 41 +++++++++++++++++++++++---
 src/osd/scrubber/scrub_reservations.h  | 27 ++++++++++++++---
 5 files changed, 97 insertions(+), 13 deletions(-)

diff --git a/src/osd/osd_perf_counters.cc b/src/osd/osd_perf_counters.cc
index af8c62fb3042..30f0ba531665 100644
--- a/src/osd/osd_perf_counters.cc
+++ b/src/osd/osd_perf_counters.cc
@@ -340,7 +340,7 @@ PerfCounters *build_osd_logger(CephContext *cct) {
 
   return osd_plb.create_perf_counters();
 }
- 
+
 
 PerfCounters *build_recoverystate_perf(CephContext *cct) {
   PerfCountersBuilder rs_perf(cct, "recoverystate_perf", rs_first, rs_last);
@@ -402,5 +402,15 @@ PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label)
   scrub_perf.add_u64_counter(scrbcnt_blocked, "locked_object", "waiting on locked object events");
   scrub_perf.add_u64_counter(scrbcnt_write_blocked, "write_blocked_by_scrub", "write blocked by scrub");
 
+  // the replica reservation process
+  scrub_perf.add_u64_counter(scrbcnt_resrv_success, "scrub_reservations_completed", "successfully completed reservation processes");
+  scrub_perf.add_time_avg(scrbcnt_resrv_successful_elapsed, "successful_reservations_elapsed", "time to scrub reservation completion");
+  scrub_perf.add_u64_counter(scrbcnt_resrv_aborted, "reservation_process_aborted", "scrub reservation was aborted");
+  scrub_perf.add_u64_counter(scrbcnt_resrv_timed_out, "reservation_process_timed_out", "scrub reservation timed out");
+  scrub_perf.add_u64_counter(scrbcnt_resrv_rejected, "reservation_process_failure", "scrub reservation failed due to replica denial");
+  scrub_perf.add_u64_counter(scrbcnt_resrv_skipped, "reservation_process_skipped", "scrub reservation skipped for high priority scrub");
+  scrub_perf.add_time_avg(scrbcnt_resrv_failed_elapsed, "failed_reservations_elapsed", "time for scrub reservation to fail");
+  scrub_perf.add_u64(scrbcnt_resrv_replicas_num, "replicas_in_reservation", "number of replicas in reservation");
+
   return scrub_perf.create_perf_counters();
 }
diff --git a/src/osd/osd_perf_counters.h b/src/osd/osd_perf_counters.h
index 1f65e953d337..00127dd7ff5d 100644
--- a/src/osd/osd_perf_counters.h
+++ b/src/osd/osd_perf_counters.h
@@ -211,6 +211,24 @@ enum {
   /// # write blocked by the scrub
   scrbcnt_write_blocked,
 
+  // -- replicas reservation
+  /// # successfully completed reservation steps
+  scrbcnt_resrv_success,
+  /// time to complete a successful replicas reservation
+  scrbcnt_resrv_successful_elapsed,
+  /// # failed attempt to reserve replicas due to an abort
+  scrbcnt_resrv_aborted,
+  /// # reservation process timed out
+  scrbcnt_resrv_timed_out,
+  /// # reservation failed due to a 'rejected' response
+  scrbcnt_resrv_rejected,
+  /// # reservation skipped for high-priority scrubs
+  scrbcnt_resrv_skipped,
+  /// time for a replicas reservation process to fail
+  scrbcnt_resrv_failed_elapsed,
+  /// # number of replicas
+  scrbcnt_resrv_replicas_num,
+
   scrbcnt_last,
 };
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 3a0cd9866d70..c68a4a1119c1 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -180,12 +180,13 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
     , NamedSimply(context<ScrubMachine>().m_scrbr, "Session/ReservingReplicas")
 {
   dout(10) << "-- state -->> ReservingReplicas" << dendl;
+  auto& session = context<Session>();
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   // initiate the reservation process
-  context<Session>().m_reservations.emplace(*scrbr);
+  session.m_reservations.emplace(*scrbr, *session.m_perf_set);
 
-  if (context<Session>().m_reservations->get_last_sent()) {
+  if (session.m_reservations->get_last_sent()) {
     // the 1'st reservation request was sent
 
     auto timeout = scrbr->get_pg_cct()->_conf.get_val<milliseconds>(
@@ -228,12 +229,13 @@ sc::result ReservingReplicas::react(const ReplicaGrant& ev)
 sc::result ReservingReplicas::react(const ReplicaReject& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  auto& session = context<Session>();
   dout(10) << "ReservingReplicas::react(const ReplicaReject&)" << dendl;
+  session.m_reservations->log_failure_and_duration(scrbcnt_resrv_rejected);
 
   // manipulate the 'next to reserve' iterator to exclude
   // the rejecting replica from the set of replicas requiring release
-  context<Session>().m_reservations->verify_rejections_source(
-      ev.m_op, ev.m_from);
+  session.m_reservations->verify_rejections_source(ev.m_op, ev.m_from);
 
   // set 'reservation failure' as the scrub termination cause (affecting
   // the rescheduling of this PG)
@@ -246,7 +248,9 @@ sc::result ReservingReplicas::react(const ReplicaReject& ev)
 sc::result ReservingReplicas::react(const ReservationTimeout&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  auto& session = context<Session>();
   dout(10) << "ReservingReplicas::react(const ReservationTimeout&)" << dendl;
+  session.m_reservations->log_failure_and_duration(scrbcnt_resrv_timed_out);
 
   const auto msg = fmt::format(
       "osd.{} PgScrubber: {} timeout on reserving replicas (since {})",
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 449856e7b459..3faafe9cb0a1 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -29,11 +29,14 @@ static std::ostream& _prefix_fn(std::ostream* _dout, T* t, std::string fn = "")
 
 namespace Scrub {
 
-ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
+ReplicaReservations::ReplicaReservations(
+    ScrubMachineListener& scrbr,
+    PerfCounters& pc)
     : m_scrubber{scrbr}
     , m_pg{m_scrubber.get_pg()}
     , m_pgid{m_scrubber.get_spgid().pgid}
     , m_osds{m_pg->get_pg_osd(ScrubberPasskey())}
+    , m_perf_set{pc}
 {
   // the acting set is sorted by pg_shard_t. The reservations are to be issued
   // in this order, so that the OSDs will receive the requests in a consistent
@@ -47,13 +50,17 @@ ReplicaReservations::ReplicaReservations(ScrubMachineListener& scrbr)
       [whoami = m_pg->pg_whoami](const pg_shard_t& shard) {
 	return shard != whoami;
       });
+  m_perf_set.set(scrbcnt_resrv_replicas_num, m_sorted_secondaries.size());
 
   m_next_to_request = m_sorted_secondaries.cbegin();
   if (m_scrubber.is_high_priority()) {
     // for high-priority scrubs (i.e. - user-initiated), no reservations are
-    // needed.
+    // needed. Note: not perf-counted as either success or failure.
     dout(10) << "high-priority scrub - no reservations needed" << dendl;
+    m_perf_set.inc(scrbcnt_resrv_skipped);
   } else {
+    m_process_started_at = ScrubClock::now();
+
     // send out the 1'st request (unless we have no replicas)
     send_next_reservation_or_complete();
     m_slow_response_warn_timeout =
@@ -88,9 +95,34 @@ void ReplicaReservations::discard_remote_reservations()
   m_next_to_request = m_sorted_secondaries.cbegin();
 }
 
+void ReplicaReservations::log_success_and_duration()
+{
+  auto logged_duration = ScrubClock::now() - m_process_started_at.value();
+  m_perf_set.tinc(scrbcnt_resrv_successful_elapsed, logged_duration);
+  m_perf_set.inc(scrbcnt_resrv_success);
+  m_osds->logger->hinc(
+      l_osd_scrub_reservation_dur_hist, m_sorted_secondaries.size(),
+      logged_duration.count());
+  m_process_started_at.reset();
+}
+
+void ReplicaReservations::log_failure_and_duration(int failure_cause_counter)
+{
+  if (!m_process_started_at.has_value()) {
+    // outcome (success/failure) already logged
+    return;
+  }
+  auto logged_duration = ScrubClock::now() - m_process_started_at.value();
+  m_perf_set.tinc(scrbcnt_resrv_failed_elapsed, logged_duration);
+  m_process_started_at.reset();
+  // note: not counted into l_osd_scrub_reservation_dur_hist
+  m_perf_set.inc(failure_cause_counter);
+}
+
 ReplicaReservations::~ReplicaReservations()
 {
   release_all();
+  log_failure_and_duration(scrbcnt_resrv_aborted);
 }
 
 bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
@@ -106,7 +138,7 @@ bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
     return false;
   }
 
-  auto elapsed = clock::now() - m_last_request_sent_at;
+  auto elapsed = ScrubClock::now() - m_last_request_sent_at;
 
   // log a warning if the response was slow to arrive
   if ((m_slow_response_warn_timeout > 0ms) &&
@@ -131,6 +163,7 @@ bool ReplicaReservations::send_next_reservation_or_complete()
   if (m_next_to_request == m_sorted_secondaries.cend()) {
     // granted by all replicas
     dout(10) << "remote reservation complete" << dendl;
+    log_success_and_duration();
     return true;  // done
   }
 
@@ -141,7 +174,7 @@ bool ReplicaReservations::send_next_reservation_or_complete()
       spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
       m_pg->pg_whoami);
   m_pg->send_cluster_message(peer.osd, m, epoch, false);
-  m_last_request_sent_at = clock::now();
+  m_last_request_sent_at = ScrubClock::now();
   dout(10) << fmt::format(
 		  "reserving {} (the {} of {} replicas)", *m_next_to_request,
 		  active_requests_cnt() + 1, m_sorted_secondaries.size())
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index 1f2628579dcc..9d59033bac87 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -46,8 +46,6 @@ namespace Scrub {
  *  lost - either due to a bug or due to a network issue.)
  */
 class ReplicaReservations {
-  using clock = ceph::coarse_real_clock;
-
   ScrubMachineListener& m_scrubber;
   PG* m_pg;
 
@@ -64,14 +62,22 @@ class ReplicaReservations {
   std::vector<pg_shard_t>::const_iterator m_next_to_request;
 
   /// for logs, and for detecting slow peers
-  clock::time_point m_last_request_sent_at;
+  ScrubTimePoint m_last_request_sent_at;
 
   /// the 'slow response' timeout (in milliseconds) - as configured.
   /// Doubles as a 'do once' flag for the warning.
   std::chrono::milliseconds m_slow_response_warn_timeout;
 
+  /// access to the performance counters container relevant to this scrub
+  /// parameters
+  PerfCounters& m_perf_set;
+
+  /// used only for the 'duration of the reservation process' perf counter.
+  /// discarded once the success or failure are recorded
+  std::optional<ScrubTimePoint> m_process_started_at;
+
  public:
-  ReplicaReservations(ScrubMachineListener& scrubber);
+  ReplicaReservations(ScrubMachineListener& scrubber, PerfCounters& pc);
 
   ~ReplicaReservations();
 
@@ -112,6 +118,12 @@ class ReplicaReservations {
   /// the only replica we are expecting a reply from
   std::optional<pg_shard_t> get_last_sent() const;
 
+  /**
+   * if the start time is still set, i.e. we have not yet marked
+   * this as a success or a failure - log its duration as that of a failure.
+   */
+  void log_failure_and_duration(int failure_cause_counter);
+
   // note: 'public', as accessed via the 'standard' dout_prefix() macro
   std::ostream& gen_prefix(std::ostream& out, std::string fn) const;
 
@@ -127,6 +139,13 @@ class ReplicaReservations {
    * - if there are no more replicas to send requests to, return true
    */
   bool send_next_reservation_or_complete();
+
+  // ---   perf counters helpers
+
+  /**
+   * log the duration of the reservation process as that of a success.
+   */
+  void log_success_and_duration();
 };
 
 } // namespace Scrub

From 037d7aab1de970a998111c07d82464d569800c1f Mon Sep 17 00:00:00 2001
From: Alex Wojno <awojno@bloomberg.net>
Date: Wed, 6 Dec 2023 11:10:17 -0500
Subject: [PATCH 1121/2492] rgw/iam: fix role deletion replication

Signed-off-by: Alex Wojno <awojno@bloomberg.net>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 14 ++++++++------
 src/test/rgw/rgw_multi/tests.py       | 24 ++++++++++++++++++++++++
 src/test/rgw/rgw_multi/zone_cloud.py  |  6 ++++++
 src/test/rgw/rgw_multi/zone_es.py     |  6 ++++++
 src/test/rgw/rgw_multi/zone_rados.py  | 13 +++++++++++++
 5 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 5ede8d44fa9b..e550efb86841 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -3677,16 +3677,18 @@ int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y)
     return -ERR_DELETE_CONFLICT;
   }
 
-  // Delete id
-  std::string oid = get_info_oid_prefix() + info.id;
-  ret = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, oid, nullptr, y);
+  // Delete id & insert MD Log
+  RGWSI_MBSObj_RemoveParams params;
+  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(store->svc()->role->svc.meta_be->alloc_ctx());
+  ctx->init(store->svc()->role->get_be_handler());
+  ret = store->svc()->role->svc.meta_be->remove(ctx.get(), info.id, params, &info.objv_tracker, y, dpp);
   if (ret < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: deleting role id from Role pool: "
-                  << info.id << ": " << cpp_strerror(-ret) << dendl;
+    ldpp_dout(dpp, 0) << "ERROR: deleting role id: " << info.id << " failed with code: " << cpp_strerror(-ret) << dendl;
+    return ret;
   }
 
   // Delete name
-  oid = info.tenant + get_names_oid_prefix() + info.name;
+  std::string oid = info.tenant + get_names_oid_prefix() + info.name;
   ret = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, oid, nullptr, y);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: deleting role name from Role pool: "
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 156fac12e7f3..c720423e9239 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -1705,6 +1705,30 @@ def test_role_sync():
 
             check_role_eq(source_conn, target_conn, role)
 
+def test_role_delete_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    role_name = gen_role_name()
+    log.info('create role zone=%s name=%s', zonegroup_conns.master_zone.name, role_name)
+    zonegroup_conns.master_zone.create_role("", role_name, None, "")
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        log.info(f'checking if zone: {zone.name} has role: {role_name}')
+        assert(zone.has_role(role_name))
+        log.info(f'success, zone: {zone.name} has role: {role_name}')
+
+    log.info(f"deleting role: {role_name}")
+    zonegroup_conns.master_zone.delete_role(role_name)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    for zone in zonegroup_conns.zones:
+        log.info(f'checking if zone: {zone.name} does not have role: {role_name}')
+        assert(not zone.has_role(role_name))
+        log.info(f'success, zone: {zone.name} does not have role: {role_name}')
+
+
 @attr('data_sync_init')
 def test_bucket_full_sync_after_data_sync_init():
     zonegroup = realm.master_zonegroup()
diff --git a/src/test/rgw/rgw_multi/zone_cloud.py b/src/test/rgw/rgw_multi/zone_cloud.py
index dd5640cf2716..7c94aaa8a60f 100644
--- a/src/test/rgw/rgw_multi/zone_cloud.py
+++ b/src/test/rgw/rgw_multi/zone_cloud.py
@@ -304,6 +304,12 @@ def check_bucket_eq(self, zone_conn, bucket_name):
         def create_role(self, path, rolename, policy_document, tag_list):
             assert False
 
+        def delete_role(self, role_name):
+            assert False
+
+        def has_role(self, role_name):
+            assert False
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 
diff --git a/src/test/rgw/rgw_multi/zone_es.py b/src/test/rgw/rgw_multi/zone_es.py
index e98b3fdd8fa6..84628b775d1c 100644
--- a/src/test/rgw/rgw_multi/zone_es.py
+++ b/src/test/rgw/rgw_multi/zone_es.py
@@ -246,6 +246,12 @@ def check_bucket_eq(self, zone_conn, bucket_name):
         def create_role(self, path, rolename, policy_document, tag_list):
             assert False
 
+        def delete_role(self, role_name):
+            assert False
+
+        def has_role(self, role_name):
+            assert False
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 
diff --git a/src/test/rgw/rgw_multi/zone_rados.py b/src/test/rgw/rgw_multi/zone_rados.py
index ac4edd004d6e..7b7fe5228cbf 100644
--- a/src/test/rgw/rgw_multi/zone_rados.py
+++ b/src/test/rgw/rgw_multi/zone_rados.py
@@ -1,5 +1,6 @@
 import logging
 from boto.s3.deletemarker import DeleteMarker
+from boto.exception import BotoServerError
 
 from itertools import zip_longest  # type: ignore
 
@@ -127,8 +128,20 @@ def check_role_eq(self, zone_conn, role_name):
             return True
 
         def create_role(self, path, rolename, policy_document, tag_list):
+            if policy_document is None:
+                policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
             return self.iam_conn.create_role(rolename, policy_document, path)
 
+        def delete_role(self, role_name):
+            return self.iam_conn.delete_role(role_name)
+
+        def has_role(self, role_name):
+            try:
+                self.get_role(role_name)
+            except BotoServerError:
+                return False
+            return True
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 

From 5bc735fb6ffbdcacffc3e678b7682f91fe7593c2 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Wed, 20 Dec 2023 18:47:51 -0500
Subject: [PATCH 1122/2492] orchestrator: Fix representation of threads in host
 ls

This patch fixes the calculation when determining the
number of threads for hosts when using the --detail
parameter.

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 src/pybind/mgr/orchestrator/module.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index d6f36e81b718..d14c84ec1838 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -95,8 +95,9 @@ def _build(self) -> None:
 
         if self._facts:
             self.server = f"{self._facts.get('vendor', '').strip()} {self._facts.get('model', '').strip()}"
-            _cores = self._facts.get('cpu_cores', 0) * self._facts.get('cpu_count', 0)
-            _threads = self._facts.get('cpu_threads', 0) * _cores
+            _cpu_count = self._facts.get('cpu_count', 1)
+            _cores = self._facts.get('cpu_cores', 0) * _cpu_count
+            _threads = self._facts.get('cpu_threads', 0) * _cpu_count
             self.os = self._facts.get('operating_system', 'N/A')
             self.cpu_summary = f"{_cores}C/{_threads}T" if _cores > 0 else 'N/A'
 

From ae7ee5ebab9a728e1f648dfbae1e83f9827cee0b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 09:09:51 -0500
Subject: [PATCH 1123/2492] Revert "spawn: use explicit strand executor"

This reverts commit 9d9258e06b78bb47fd0156d9bd7bb00b52a726b0.

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/common/async/yield_context.h           | 12 +++---------
 src/rgw/driver/rados/rgw_bucket.cc         |  4 ++--
 src/rgw/driver/rados/rgw_notify.cc         | 18 +++++++++---------
 src/rgw/driver/rados/rgw_pubsub_push.cc    |  2 +-
 src/rgw/rgw_aio.cc                         |  4 ++--
 src/rgw/rgw_aio_throttle.h                 |  4 ++--
 src/rgw/rgw_asio_frontend.cc               | 10 +++++-----
 src/rgw/rgw_d3n_cacherequest.h             |  4 ++--
 src/rgw/rgw_op.cc                          |  2 +-
 src/rgw/rgw_sync_checkpoint.cc             |  4 ++--
 src/test/rgw/test_rgw_dmclock_scheduler.cc |  2 +-
 src/test/rgw/test_rgw_reshard_wait.cc      |  6 +++---
 src/test/rgw/test_rgw_throttle.cc          |  4 ++--
 13 files changed, 35 insertions(+), 41 deletions(-)

diff --git a/src/common/async/yield_context.h b/src/common/async/yield_context.h
index 05e6ca6140c5..baa028fa1b4a 100644
--- a/src/common/async/yield_context.h
+++ b/src/common/async/yield_context.h
@@ -22,23 +22,17 @@
 
 #include <spawn/spawn.hpp>
 
-// use explicit executor types instead of the type-erased boost::asio::executor.
-// coroutines wrap the default io_context executor with a strand executor
-using yield_context = spawn::basic_yield_context<
-    boost::asio::executor_binder<void(*)(),
-        boost::asio::strand<boost::asio::io_context::executor_type>>>;
-
 /// optional-like wrapper for a spawn::yield_context and its associated
 /// boost::asio::io_context. operations that take an optional_yield argument
 /// will, when passed a non-empty yield context, suspend this coroutine instead
 /// of the blocking the thread of execution
 class optional_yield {
   boost::asio::io_context *c = nullptr;
-  yield_context *y = nullptr;
+  spawn::yield_context *y = nullptr;
  public:
   /// construct with a valid io and yield_context
   explicit optional_yield(boost::asio::io_context& c,
-                          yield_context& y) noexcept
+                          spawn::yield_context& y) noexcept
     : c(&c), y(&y) {}
 
   /// type tag to construct an empty object
@@ -52,7 +46,7 @@ class optional_yield {
   boost::asio::io_context& get_io_context() const noexcept { return *c; }
 
   /// return a reference to the yield_context. only valid if non-empty
-  yield_context& get_yield_context() const noexcept { return *y; }
+  spawn::yield_context& get_yield_context() const noexcept { return *y; }
 };
 
 // type tag object to construct an empty optional_yield
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 64d8e31cc3cc..aac80909634f 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -584,7 +584,7 @@ int RGWBucket::check_index_olh(rgw::sal::RadosStore* const rados_store,
   const int max_aio = std::max(1, op_state.get_max_aio());
 
   for (int i=0; i<max_aio; i++) {
-    spawn::spawn(context, [&](yield_context yield) {
+    spawn::spawn(context, [&](spawn::yield_context yield) {
       while (true) {
         int shard = next_shard;
         next_shard += 1;
@@ -793,7 +793,7 @@ int RGWBucket::check_index_unlinked(rgw::sal::RadosStore* const rados_store,
   int next_shard = 0;
   boost::asio::io_context context;
   for (int i=0; i<max_aio; i++) {
-    spawn::spawn(context, [&](yield_context yield) {
+    spawn::spawn(context, [&](spawn::yield_context yield) {
       while (true) {
         int shard = next_shard;
         next_shard += 1;
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index ecbccc8b220b..6711f310c268 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -178,7 +178,7 @@ class Manager : public DoutPrefixProvider {
       pending_tokens(0),
       timer(io_context) {}  
  
-    void async_wait(yield_context yield) {
+    void async_wait(spawn::yield_context yield) { 
       if (pending_tokens == 0) {
         return;
       }
@@ -201,7 +201,7 @@ class Manager : public DoutPrefixProvider {
   // processing of a specific entry
   // return whether processing was successful (true) or not (false)
   EntryProcessingResult process_entry(const ConfigProxy& conf, persistency_tracker& entry_persistency_tracker,
-                                      const cls_queue_entry& entry, yield_context yield) {
+                                      const cls_queue_entry& entry, spawn::yield_context yield) {
     event_entry_t event_entry;
     auto iter = entry.data.cbegin();
     try {
@@ -269,7 +269,7 @@ class Manager : public DoutPrefixProvider {
   }
 
   // clean stale reservation from queue
-  void cleanup_queue(const std::string& queue_name, yield_context yield) {
+  void cleanup_queue(const std::string& queue_name, spawn::yield_context yield) {
     while (true) {
       ldpp_dout(this, 20) << "INFO: trying to perform stale reservation cleanup for queue: " << queue_name << dendl;
       const auto now = ceph::coarse_real_time::clock::now();
@@ -305,13 +305,13 @@ class Manager : public DoutPrefixProvider {
   }
 
   // processing of a specific queue
-  void process_queue(const std::string& queue_name, yield_context yield) {
+  void process_queue(const std::string& queue_name, spawn::yield_context yield) {
     constexpr auto max_elements = 1024;
     auto is_idle = false;
     const std::string start_marker;
 
     // start a the cleanup coroutine for the queue
-    spawn::spawn(io_context, [this, queue_name](yield_context yield) {
+    spawn::spawn(io_context, [this, queue_name](spawn::yield_context yield) {
             cleanup_queue(queue_name, yield);
             }, make_stack_allocator());
 
@@ -392,7 +392,7 @@ class Manager : public DoutPrefixProvider {
 
         entries_persistency_tracker& notifs_persistency_tracker = topics_persistency_tracker[queue_name];
         spawn::spawn(yield, [this, &notifs_persistency_tracker, &queue_name, entry_idx, total_entries, &end_marker,
-                             &remove_entries, &has_error, &waiter, &entry, &needs_migration_vector](yield_context yield) {
+                             &remove_entries, &has_error, &waiter, &entry, &needs_migration_vector](spawn::yield_context yield) {
             const auto token = waiter.make_token();
             auto& persistency_tracker = notifs_persistency_tracker[entry.marker];
             auto result = process_entry(this->get_cct()->_conf, persistency_tracker, entry, yield);
@@ -543,7 +543,7 @@ class Manager : public DoutPrefixProvider {
 
   // process all queues
   // find which of the queues is owned by this daemon and process it
-  void process_queues(yield_context yield) {
+  void process_queues(spawn::yield_context yield) {
     auto has_error = false;
     owned_queues_t owned_queues;
 
@@ -610,7 +610,7 @@ class Manager : public DoutPrefixProvider {
         if (owned_queues.insert(queue_name).second) {
           ldpp_dout(this, 10) << "INFO: queue: " << queue_name << " now owned (locked) by this daemon" << dendl;
           // start processing this queue
-          spawn::spawn(io_context, [this, &queue_gc, &queue_gc_lock, queue_name](yield_context yield) {
+          spawn::spawn(io_context, [this, &queue_gc, &queue_gc_lock, queue_name](spawn::yield_context yield) {
             process_queue(queue_name, yield);
             // if queue processing ended, it means that the queue was removed or not owned anymore
             // mark it for deletion
@@ -661,7 +661,7 @@ class Manager : public DoutPrefixProvider {
     reservations_cleanup_period_s(_reservations_cleanup_period_s),
     rados_store(*store)
     {
-      spawn::spawn(io_context, [this] (yield_context yield) {
+      spawn::spawn(io_context, [this](spawn::yield_context yield) {
             process_queues(yield);
           }, make_stack_allocator());
 
diff --git a/src/rgw/driver/rados/rgw_pubsub_push.cc b/src/rgw/driver/rados/rgw_pubsub_push.cc
index 05dc9e65d0ea..52bee2a16d31 100644
--- a/src/rgw/driver/rados/rgw_pubsub_push.cc
+++ b/src/rgw/driver/rados/rgw_pubsub_push.cc
@@ -122,7 +122,7 @@ namespace {
 class Waiter {
   using Signature = void(boost::system::error_code);
   using Completion = ceph::async::Completion<Signature>;
-  using CompletionInit = boost::asio::async_completion<yield_context, Signature>;
+  using CompletionInit = boost::asio::async_completion<spawn::yield_context, Signature>;
   std::unique_ptr<Completion> completion = nullptr;
   int ret;
 
diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index cd85ea6d7fab..c70acae79e95 100644
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -89,12 +89,12 @@ struct Handler {
 template <typename Op>
 Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
                          boost::asio::io_context& context,
-                         yield_context yield) {
+                         spawn::yield_context yield) {
   return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
       // arrange for the completion Handler to run on the yield_context's strand
       // executor so it can safely call back into Aio without locking
       using namespace boost::asio;
-      async_completion<yield_context, void()> init(yield);
+      async_completion<spawn::yield_context, void()> init(yield);
       auto ex = get_associated_executor(init.completion_handler);
 
       librados::async_operate(context, ctx, r.obj.oid, &op, 0,
diff --git a/src/rgw/rgw_aio_throttle.h b/src/rgw/rgw_aio_throttle.h
index 89f9c0eef649..c0656ef225e6 100644
--- a/src/rgw/rgw_aio_throttle.h
+++ b/src/rgw/rgw_aio_throttle.h
@@ -81,7 +81,7 @@ class BlockingAioThrottle final : public Aio, private Throttle {
 // functions must be called within the coroutine strand
 class YieldingAioThrottle final : public Aio, private Throttle {
   boost::asio::io_context& context;
-  yield_context yield;
+  spawn::yield_context yield;
   struct Handler;
 
   // completion callback associated with the waiter
@@ -95,7 +95,7 @@ class YieldingAioThrottle final : public Aio, private Throttle {
 
  public:
   YieldingAioThrottle(uint64_t window, boost::asio::io_context& context,
-                      yield_context yield)
+                      spawn::yield_context yield)
     : Throttle(window), context(context), yield(yield)
   {}
 
diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index 23ccfcafeb08..226fbe6936d7 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -76,12 +76,12 @@ class StreamIO : public rgw::asio::ClientIO {
   CephContext* const cct;
   Stream& stream;
   timeout_timer& timeout;
-  yield_context yield;
+  spawn::yield_context yield;
   parse_buffer& buffer;
   boost::system::error_code fatal_ec;
  public:
   StreamIO(CephContext *cct, Stream& stream, timeout_timer& timeout,
-           rgw::asio::parser_type& parser, yield_context yield,
+           rgw::asio::parser_type& parser, spawn::yield_context yield,
            parse_buffer& buffer, bool is_ssl,
            const tcp::endpoint& local_endpoint,
            const tcp::endpoint& remote_endpoint)
@@ -207,7 +207,7 @@ void handle_connection(boost::asio::io_context& context,
                        rgw::dmclock::Scheduler *scheduler,
                        const std::string& uri_prefix,
                        boost::system::error_code& ec,
-                       yield_context yield)
+                       spawn::yield_context yield)
 {
   // don't impose a limit on the body, since we read it in pieces
   static constexpr size_t body_limit = std::numeric_limits<size_t>::max();
@@ -1029,7 +1029,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
 #ifdef WITH_RADOSGW_BEAST_OPENSSL
   if (l.use_ssl) {
     spawn::spawn(context,
-      [this, s=std::move(stream)] (yield_context yield) mutable {
+      [this, s=std::move(stream)] (spawn::yield_context yield) mutable {
         auto conn = boost::intrusive_ptr{new Connection(std::move(s))};
         auto c = connections.add(*conn);
         // wrap the tcp stream in an ssl stream
@@ -1060,7 +1060,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
   {
 #endif // WITH_RADOSGW_BEAST_OPENSSL
     spawn::spawn(context,
-      [this, s=std::move(stream)] (yield_context yield) mutable {
+      [this, s=std::move(stream)] (spawn::yield_context yield) mutable {
         auto conn = boost::intrusive_ptr{new Connection(std::move(s))};
         auto c = connections.add(*conn);
         auto timeout = timeout_timer{context.get_executor(), request_timeout, conn};
diff --git a/src/rgw/rgw_d3n_cacherequest.h b/src/rgw/rgw_d3n_cacherequest.h
index 40fc758e2eb4..2ae4e16396de 100644
--- a/src/rgw/rgw_d3n_cacherequest.h
+++ b/src/rgw/rgw_d3n_cacherequest.h
@@ -131,11 +131,11 @@ struct D3nL1CacheRequest {
     }
   };
 
-  void file_aio_read_abstract(const DoutPrefixProvider *dpp, boost::asio::io_context& context, yield_context yield,
+  void file_aio_read_abstract(const DoutPrefixProvider *dpp, boost::asio::io_context& context, spawn::yield_context yield,
                               std::string& cache_location, off_t read_ofs, off_t read_len,
                               rgw::Aio* aio, rgw::AioResult& r) {
     using namespace boost::asio;
-    async_completion<yield_context, void()> init(yield);
+    async_completion<spawn::yield_context, void()> init(yield);
     auto ex = get_associated_executor(init.completion_handler);
 
     ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): oid=" << r.obj.oid << dendl;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 0a1c3b1cf459..91aa27620284 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -7356,7 +7356,7 @@ void RGWDeleteMultiObj::execute(optional_yield y)
         return aio_count < max_aio;
       });
       aio_count++;
-      spawn::spawn(y.get_yield_context(), [this, &y, &aio_count, obj_key, &formatter_flush_cond] (yield_context yield) {
+      spawn::spawn(y.get_yield_context(), [this, &y, &aio_count, obj_key, &formatter_flush_cond] (spawn::yield_context yield) {
         handle_individual_object(obj_key, optional_yield { y.get_io_context(), yield }, &*formatter_flush_cond); 
         aio_count--;
       }); 
diff --git a/src/rgw/rgw_sync_checkpoint.cc b/src/rgw/rgw_sync_checkpoint.cc
index 7ef6c3fc02b4..1394a712a94f 100644
--- a/src/rgw/rgw_sync_checkpoint.cc
+++ b/src/rgw/rgw_sync_checkpoint.cc
@@ -226,7 +226,7 @@ int rgw_bucket_sync_checkpoint(const DoutPrefixProvider* dpp,
     entry.pipe = pipe;
 
     // fetch remote markers
-    spawn::spawn(ioctx, [&] (yield_context yield) {
+    spawn::spawn(ioctx, [&] (spawn::yield_context yield) {
       auto y = optional_yield{ioctx, yield};
       rgw_bucket_index_marker_info info;
       int r = source_bilog_info(dpp, store->svc()->zone, entry.pipe,
@@ -239,7 +239,7 @@ int rgw_bucket_sync_checkpoint(const DoutPrefixProvider* dpp,
       entry.latest_gen = info.latest_gen;
     });
     // fetch source bucket info
-    spawn::spawn(ioctx, [&] (yield_context yield) {
+    spawn::spawn(ioctx, [&] (spawn::yield_context yield) {
       auto y = optional_yield{ioctx, yield};
       int r = store->getRados()->get_bucket_instance_info(
           *entry.pipe.source.bucket, entry.source_bucket_info,
diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc
index 92800767c99a..36652d22f038 100644
--- a/src/test/rgw/test_rgw_dmclock_scheduler.cc
+++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc
@@ -400,7 +400,7 @@ TEST(Queue, SpawnAsyncRequest)
 {
   boost::asio::io_context context;
 
-  spawn::spawn(context, [&] (yield_context yield) {
+  spawn::spawn(context, [&] (spawn::yield_context yield) {
     ClientCounters counters(g_ceph_context);
     AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr,
                     [] (client_id client) -> ClientInfo* {
diff --git a/src/test/rgw/test_rgw_reshard_wait.cc b/src/test/rgw/test_rgw_reshard_wait.cc
index 06caae34adef..98b2aa235b95 100644
--- a/src/test/rgw/test_rgw_reshard_wait.cc
+++ b/src/test/rgw/test_rgw_reshard_wait.cc
@@ -64,7 +64,7 @@ TEST(ReshardWait, wait_yield)
   RGWReshardWait waiter(wait_duration);
 
   boost::asio::io_context context;
-  spawn::spawn(context, [&] (yield_context yield) {
+  spawn::spawn(context, [&] (spawn::yield_context yield) {
       EXPECT_EQ(0, waiter.wait(optional_yield{context, yield}));
     });
 
@@ -90,7 +90,7 @@ TEST(ReshardWait, stop_yield)
 
   boost::asio::io_context context;
   spawn::spawn(context,
-    [&] (yield_context yield) {
+    [&] (spawn::yield_context yield) {
       EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield}));
     });
 
@@ -133,7 +133,7 @@ TEST(ReshardWait, stop_multiple)
   // spawn 4 coroutines
   boost::asio::io_context context;
   {
-    auto async_waiter = [&] (yield_context yield) {
+    auto async_waiter = [&] (spawn::yield_context yield) {
         EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield}));
       };
     spawn::spawn(context, async_waiter);
diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc
index 5e18fc3c1854..e5df9f84efa1 100644
--- a/src/test/rgw/test_rgw_throttle.cc
+++ b/src/test/rgw/test_rgw_throttle.cc
@@ -144,7 +144,7 @@ TEST(Aio_Throttle, YieldCostOverWindow)
 
   boost::asio::io_context context;
   spawn::spawn(context,
-    [&] (yield_context yield) {
+    [&] (spawn::yield_context yield) {
       YieldingAioThrottle throttle(4, context, yield);
       scoped_completion op;
       auto c = throttle.get(obj, wait_on(op), 8, 0);
@@ -167,7 +167,7 @@ TEST(Aio_Throttle, YieldingThrottleOverMax)
 
   boost::asio::io_context context;
   spawn::spawn(context,
-    [&] (yield_context yield) {
+    [&] (spawn::yield_context yield) {
       YieldingAioThrottle throttle(window, context, yield);
       for (uint64_t i = 0; i < total; i++) {
         using namespace std::chrono_literals;

From 0d6689f11bfacc2c6fef81cf87e57e36e0d2a77f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 9 Feb 2023 10:37:29 -0500
Subject: [PATCH 1124/2492] Revert "rgw/beast: use explicit executor type for
 tcp socket and stream"

This reverts commit 0bee64d8757f679bd532f251ccbcb16c3b5199da.

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_asio_frontend.cc | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index 226fbe6936d7..1e443a84f90d 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -52,14 +52,8 @@ namespace ssl = boost::asio::ssl;
 
 struct Connection;
 
-// use explicit executor types instead of the type-erased boost::asio::executor
-using executor_type = boost::asio::io_context::executor_type;
-
-using tcp_socket = boost::asio::basic_stream_socket<tcp, executor_type>;
-using tcp_stream = boost::beast::basic_stream<tcp, executor_type>;
-
 using timeout_timer = rgw::basic_timeout_timer<ceph::coarse_mono_clock,
-      executor_type, Connection>;
+      boost::asio::any_io_executor, Connection>;
 
 static constexpr size_t parse_buffer_size = 65536;
 using parse_buffer = boost::beast::flat_static_buffer<parse_buffer_size>;
@@ -102,7 +96,7 @@ class StreamIO : public rgw::asio::ClientIO {
       ldout(cct, 4) << "write_data failed: " << ec.message() << dendl;
       if (ec == boost::asio::error::broken_pipe) {
         boost::system::error_code ec_ignored;
-        stream.lowest_layer().shutdown(tcp_socket::shutdown_both, ec_ignored);
+        stream.lowest_layer().shutdown(tcp::socket::shutdown_both, ec_ignored);
       }
       if (!fatal_ec) {
         fatal_ec = ec;
@@ -196,7 +190,7 @@ std::ostream& operator<<(std::ostream& out, const log_apache_time& a) {
       << std::put_time(local, " %z");
 };
 
-using SharedMutex = ceph::async::SharedMutex<boost::asio::io_context::executor_type>;
+using SharedMutex = ceph::async::SharedMutex<boost::asio::any_io_executor>;
 
 template <typename Stream>
 void handle_connection(boost::asio::io_context& context,
@@ -358,17 +352,17 @@ void handle_connection(boost::asio::io_context& context,
 struct Connection : boost::intrusive::list_base_hook<>,
                     boost::intrusive_ref_counter<Connection>
 {
-  tcp_socket socket;
+  tcp::socket socket;
   parse_buffer buffer;
 
-  explicit Connection(tcp_socket&& socket) noexcept
+  explicit Connection(tcp::socket&& socket) noexcept
       : socket(std::move(socket)) {}
 
   void close(boost::system::error_code& ec) {
     socket.close(ec);
   }
 
-  tcp_socket& get_socket() { return socket; }
+  tcp::socket& get_socket() { return socket; }
 };
 
 class ConnectionList {
@@ -427,7 +421,7 @@ class AsioFrontend {
   struct Listener {
     tcp::endpoint endpoint;
     tcp::acceptor acceptor;
-    tcp_socket socket;
+    tcp::socket socket;
     bool use_ssl = false;
     bool use_nodelay = false;
 
@@ -1033,7 +1027,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
         auto conn = boost::intrusive_ptr{new Connection(std::move(s))};
         auto c = connections.add(*conn);
         // wrap the tcp stream in an ssl stream
-        boost::asio::ssl::stream<tcp_socket&> stream{conn->socket, *ssl_context};
+        boost::asio::ssl::stream<tcp::socket&> stream{conn->socket, *ssl_context};
         auto timeout = timeout_timer{context.get_executor(), request_timeout, conn};
         // do ssl handshake
         boost::system::error_code ec;
@@ -1068,7 +1062,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
         handle_connection(context, env, conn->socket, timeout, header_limit,
                           conn->buffer, false, pause_mutex, scheduler.get(),
                           uri_prefix, ec, yield);
-        conn->socket.shutdown(tcp_socket::shutdown_both, ec);
+        conn->socket.shutdown(tcp::socket::shutdown_both, ec);
       }, make_stack_allocator());
   }
 }

From 247ace17086ddddd6b7bda44a067d4b1eaa238fd Mon Sep 17 00:00:00 2001
From: NitzanMordhai <nmordech@redhat.com>
Date: Thu, 23 Nov 2023 12:01:03 +0000
Subject: [PATCH 1125/2492] mgr/BaseMgrModule: Optimize CPython Call in Finish
 Function

Remove CPython overhead packing tuple during the 'finish' function to
improve memory consumption when we deal with long-string outputs.
When modules like Restful return large amounts of output the use
of PyObject_CallFunction without createing PyObject will reduce the
time the memory held by the mgr.

Fixes: https://tracker.ceph.com/issues/59580
Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 src/mgr/BaseMgrModule.cc | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc
index ab64ac39fbed..6cb3a6bce245 100644
--- a/src/mgr/BaseMgrModule.cc
+++ b/src/mgr/BaseMgrModule.cc
@@ -92,22 +92,13 @@ class MonCommandCompletion : public Context
 
       auto set_fn = PyObject_GetAttrString(python_completion, "complete");
       ceph_assert(set_fn != nullptr);
-
-      auto pyR = PyLong_FromLong(r);
-      auto pyOutBl = PyUnicode_FromString(outbl.to_str().c_str());
-      auto pyOutS = PyUnicode_FromString(outs.c_str());
-      auto args = PyTuple_Pack(3, pyR, pyOutBl, pyOutS);
-      Py_DECREF(pyR);
-      Py_DECREF(pyOutBl);
-      Py_DECREF(pyOutS);
-
-      auto rtn = PyObject_CallObject(set_fn, args);
-      if (rtn != nullptr) {
-	Py_DECREF(rtn);
+      auto rtn = PyObject_CallFunction(set_fn, "(iss)", r, outbl.to_str().c_str(), outs.c_str());
+      if (rtn == nullptr) {
+        PyErr_Print();
+      } else {
+        Py_DECREF(rtn);
       }
-      Py_DECREF(args);
       Py_DECREF(set_fn);
-
       Py_DECREF(python_completion);
       python_completion = nullptr;
     }

From 84c5d2c828c2fbd70bdeadedd341ca42ddb1c20c Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 20 Dec 2023 15:00:38 +1000
Subject: [PATCH 1126/2492] doc/radosgw: edit compression.rst

Improve the grammar and simplify the sentence structure of
doc/radosgw/compression.rst. This commit is made in anticipation of a
near-future commit that will list the compression algorithms available
to users of Ceph.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/compression.rst | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/doc/radosgw/compression.rst b/doc/radosgw/compression.rst
index fba0681da905..0fa1f57185c9 100644
--- a/doc/radosgw/compression.rst
+++ b/doc/radosgw/compression.rst
@@ -4,8 +4,8 @@ Compression
 
 .. versionadded:: Kraken
 
-The Ceph Object Gateway supports server-side compression of uploaded objects,
-using any of Ceph's existing compression plugins.
+The Ceph Object Gateway supports server-side compression of uploaded objects.
+using any of the existing compression plugins.
 
 .. note:: The Reef release added a :ref:`feature_compress_encrypted` zonegroup
    feature to enable compression with `Server-Side Encryption`_.
@@ -18,14 +18,15 @@ Compression can be enabled on a storage class in the Zone's placement target
 by providing the ``--compression=<type>`` option to the command
 ``radosgw-admin zone placement modify``.
 
-The compression ``type`` refers to the name of the compression plugin to use
-when writing new object data. Each compressed object remembers which plugin
-was used, so changing this setting does not hinder the ability to decompress
-existing objects, nor does it force existing objects to be recompressed.
+The compression ``type`` refers to the name of the compression plugin that will
+be used when writing new object data. Each compressed object remembers which
+plugin was used, so any change to this setting will neither affect Ceph's
+ability to decompress existing objects nor require existing objects to be
+recompressed.
 
-This compression setting applies to all new objects uploaded to buckets using
-this placement target. Compression can be disabled by setting the ``type`` to
-an empty string or ``none``.
+Compression settings apply to all new objects uploaded to buckets using this
+placement target. Compression can be disabled by setting the ``type`` to an
+empty string or ``none``.
 
 For example::
 
@@ -62,11 +63,15 @@ For example::
 Statistics
 ==========
 
-While all existing commands and APIs continue to report object and bucket
-sizes based their uncompressed data, compression statistics for a given bucket
-are included in its ``bucket stats``::
+Run the ``radosgw-admin bucket stats`` command to see compression statistics
+for a given bucket:
+
+.. prompt:: bash
+
+   radosgw-admin bucket stats --bucket=<name>
+
+::
 
-  $ radosgw-admin bucket stats --bucket=<name>
   {
   ...
       "usage": {
@@ -83,6 +88,9 @@ are included in its ``bucket stats``::
   ...
   }
 
+Other commands and APIs will report object and bucket sizes based on their
+uncompressed data. 
+
 The ``size_utilized`` and ``size_kb_utilized`` fields represent the total
 size of compressed data, in bytes and kilobytes respectively.
 

From 94fa9bd2d90857ca2c6a956a79406318ab2fa485 Mon Sep 17 00:00:00 2001
From: zhipeng li <qiuxinyidian@gmail.com>
Date: Sat, 26 Aug 2023 00:54:38 +0800
Subject: [PATCH 1127/2492] rgw: modify string match_wildcards with fnmatch
 Fixes: https://tracker.ceph.com/issues/62292

Signed-off-by: zhipeng li <qiuxinyidian@gmail.com>
---
 src/rgw/rgw_common.cc               |  2 +-
 src/rgw/rgw_common.h                |  2 +-
 src/rgw/rgw_iam_policy.cc           |  2 +-
 src/rgw/rgw_string.cc               | 45 +++++++----------------------
 src/rgw/rgw_string.h                |  4 +--
 src/test/rgw/test_rgw_iam_policy.cc |  5 ++--
 6 files changed, 18 insertions(+), 42 deletions(-)

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 5868268c5e1b..3372845a0042 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -2162,7 +2162,7 @@ int rgw_parse_op_type_list(const string& str, uint32_t *perm)
   return rgw_parse_list_of_flags(op_type_mapping, str, perm);
 }
 
-bool match_policy(std::string_view pattern, std::string_view input,
+bool match_policy(const std::string& pattern, const std::string& input,
                   uint32_t flag)
 {
   const uint32_t flag2 = flag & (MATCH_POLICY_ACTION|MATCH_POLICY_ARN) ?
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 67333be99885..6c4ed55db932 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1756,7 +1756,7 @@ static constexpr uint32_t MATCH_POLICY_RESOURCE = 0x02;
 static constexpr uint32_t MATCH_POLICY_ARN = 0x04;
 static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
 
-extern bool match_policy(std::string_view pattern, std::string_view input,
+extern bool match_policy(const std::string& pattern, const std::string& input,
                          uint32_t flag);
 
 extern std::string camelcase_dash_http_attr(const std::string& orig);
diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc
index 35aeb15fcdc5..cf56f0f55728 100644
--- a/src/rgw/rgw_iam_policy.cc
+++ b/src/rgw/rgw_iam_policy.cc
@@ -578,7 +578,7 @@ bool ParseState::do_string(CephContext* cct, const char* s, size_t l) {
         t->action = allValue : t->notaction = allValue);
     } else {
       for (auto& p : actpairs) {
-        if (match_policy({s, l}, p.name, MATCH_POLICY_ACTION)) {
+        if (match_policy(string(s, l), p.name, MATCH_POLICY_ACTION)) {
           is_validaction = true;
           (w->id == TokenID::Action ? t->action[p.bit] = 1 : t->notaction[p.bit] = 1);
         }
diff --git a/src/rgw/rgw_string.cc b/src/rgw/rgw_string.cc
index 7be82f854a84..420db96c4f2e 100644
--- a/src/rgw/rgw_string.cc
+++ b/src/rgw/rgw_string.cc
@@ -2,44 +2,21 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "rgw_string.h"
+#include <fnmatch.h>
 
-static bool char_eq(char c1, char c2)
-{
-  return c1 == c2;
-}
-
-static bool ci_char_eq(char c1, char c2)
-{
-  return tolower(c1) == tolower(c2);
-}
-
-bool match_wildcards(std::string_view pattern, std::string_view input,
+bool match_wildcards(const std::string& pattern, const std::string& input,
                      uint32_t flags)
 {
-  const auto eq = (flags & MATCH_CASE_INSENSITIVE) ? &ci_char_eq : &char_eq;
+  bool case_insensive = flags & MATCH_CASE_INSENSITIVE;
+  uint32_t  flag = 0;
+
+  if (case_insensive) {
+    flag = FNM_CASEFOLD;
+  }
 
-  auto it1 = pattern.begin();
-  auto it2 = input.begin();
-  while (true) {
-    if (it1 == pattern.end())
-      return it2 == input.end();
-    if (*it1 == '*') {
-      if (it1 + 1 == pattern.end())
-        return true;
-      if (it2 == input.end() || eq(*(it1 + 1), *it2))
-        ++it1;
-      else
-        ++it2;
-      continue;
-    }
-    if (it2 == input.end())
-      return false;
-    if (*it1 == '?' || eq(*it1, *it2)) {
-      ++it1;
-      ++it2;
-      continue;
-    }
+  if (fnmatch(pattern.data(), input.data(), flag) == 0) {
+    return true;
+  } else {
     return false;
   }
-  return false;
 }
diff --git a/src/rgw/rgw_string.h b/src/rgw/rgw_string.h
index e58a356f4715..c6de42e38414 100644
--- a/src/rgw/rgw_string.h
+++ b/src/rgw/rgw_string.h
@@ -230,6 +230,6 @@ static constexpr uint32_t MATCH_CASE_INSENSITIVE = 0x01;
 
 /// attempt to match the given input string with the pattern, which may contain
 /// the wildcard characters * and ?
-extern bool match_wildcards(std::string_view pattern,
-                            std::string_view input,
+extern bool match_wildcards(const std::string& pattern,
+                            const std::string& input,
                             uint32_t flags = 0);
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index f4c3c6aff6f1..f1ef29a00f22 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -1258,9 +1258,8 @@ TEST(MatchWildcards, Asterisk)
                               "http://example.com/index.html"));
   EXPECT_TRUE(match_wildcards("http://example.com/*/*.jpg",
                               "http://example.com/fun/smiley.jpg"));
-  // note: parsing of * is not greedy, so * does not match 'bc' here
-  EXPECT_FALSE(match_wildcards("a*c", "abcc"));
-  EXPECT_FALSE(match_wildcards("a*c", "abcc", MATCH_CASE_INSENSITIVE));
+  EXPECT_TRUE(match_wildcards("a*c", "abcc"));
+  EXPECT_TRUE(match_wildcards("a*c", "abcc", MATCH_CASE_INSENSITIVE));
 }
 
 TEST(MatchPolicy, Action)

From 8fa844383f9c22e758f39ecdda74f70de054ad68 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 21 Dec 2023 14:54:08 -0500
Subject: [PATCH 1128/2492] rgw: Don't hold mutex over yield in LazyFIFO

If the FIFO doesn't exist, let clients race to create it, then stash
and use whoever wins.

Fixes: https://tracker.ceph.com/issues/63373
Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_log_backing.h | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_log_backing.h b/src/rgw/driver/rados/rgw_log_backing.h
index 3dfdb8ee4ef1..6cda9a4ca978 100644
--- a/src/rgw/driver/rados/rgw_log_backing.h
+++ b/src/rgw/driver/rados/rgw_log_backing.h
@@ -262,12 +262,24 @@ class LazyFIFO {
 
   int lazy_init(const DoutPrefixProvider *dpp, optional_yield y) {
     std::unique_lock l(m);
-    if (fifo) return 0;
-    auto r = rgw::cls::fifo::FIFO::create(dpp, ioctx, oid, &fifo, y);
-    if (r) {
-      fifo.reset();
+    if (fifo) {
+      return 0;
+    } else {
+      l.unlock();
+      // FIFO supports multiple clients by design, so it's safe to
+      // race to create them.
+      std::unique_ptr<rgw::cls::fifo::FIFO> fifo_tmp;
+      auto r = rgw::cls::fifo::FIFO::create(dpp, ioctx, oid, &fifo, y);
+      if (r) {
+	return r;
+      }
+      l.lock();
+      if (!fifo) {
+	// We won the race
+	fifo = std::move(fifo_tmp);
+      }
     }
-    return r;
+    return 0;
   }
 
 public:

From b5fe51379ddd368a2237bdbe1c23c88cf28321fd Mon Sep 17 00:00:00 2001
From: imtzw <tongzhiwei_yewu@cmss.chinamobile.com>
Date: Fri, 8 Dec 2023 06:11:13 +0000
Subject: [PATCH 1129/2492] rgw: fix list obj v1 nextmarker

when encoding-type=url given, aws sdk will try to decode
next marker returned in a list obj response.
if it is not actually encoded and the marker obj just
have an encoding symbol in its name, aws sdk may get
error when trying to decode it.

Signed-off-by: imtzw <tongzhiwei_yewu@cmss.chinamobile.com>
---
 src/rgw/rgw_rest.h     | 10 ++++++
 src/rgw/rgw_rest_s3.cc | 76 +++++++-----------------------------------
 2 files changed, 22 insertions(+), 64 deletions(-)

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 434de99e9c2e..0b6cf62ed2c7 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -706,6 +706,16 @@ extern void dump_start(req_state *s);
 extern void list_all_buckets_start(req_state *s);
 extern void dump_owner(req_state *s, const rgw_user& id,
                        const std::string& name, const char *section = NULL);
+inline void dump_urlsafe(req_state *s, bool encode_key, const char* key, const std::string& val, bool encode_slash = true) {
+  if (encode_key) {
+    std::string _val;
+    url_encode(val, _val, encode_slash);
+    s->formatter->dump_string(key, _val);
+  }
+  else {
+    s->formatter->dump_string(key, val);
+  }
+}
 extern void dump_header(req_state* s,
                         const std::string_view& name,
                         const std::string_view& val);
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6c3a8c8adc16..511adb990ad1 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1707,11 +1707,7 @@ void RGWListBucket_ObjStore_S3::send_common_versioned_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
 
       s->formatter->close_section();
       }
@@ -1729,7 +1725,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
   s->formatter->dump_string("KeyMarker", marker.name);
   s->formatter->dump_string("VersionIdMarker", marker.instance);
   if (is_truncated && !next_marker.empty()) {
-    s->formatter->dump_string("NextKeyMarker", next_marker.name);
+    dump_urlsafe(s ,encode_key, "NextKeyMarker", next_marker.name);
     if (next_marker.instance.empty()) {
       s->formatter->dump_string("NextVersionIdMarker", "null");
     }
@@ -1752,14 +1748,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
         s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
       }
       rgw_obj_key key(iter->key);
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s ,encode_key, "Key", key.name);
       string version_id = key.instance;
       if (version_id.empty()) {
         version_id = "null";
@@ -1807,11 +1796,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
   s->formatter->dump_string("Prefix", prefix);
   s->formatter->dump_int("MaxKeys", max);
   if (!delimiter.empty()) {
-    if (encode_key) {
-      s->formatter->dump_string("Delimiter", url_encode(delimiter, false));
-    } else {
-      s->formatter->dump_string("Delimiter", delimiter);
-    }
+    dump_urlsafe(s, encode_key, "Delimiter", delimiter, false);
   }
   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
               : "false"));
@@ -1821,11 +1806,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
       s->formatter->close_section();
       }
     }
@@ -1866,13 +1847,6 @@ void RGWListBucket_ObjStore_S3::send_response()
     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 
       rgw_obj_key key(iter->key);
-      std::string key_name;
-
-      if (encode_key) {
-	url_encode(key.name, key_name);
-      } else {
-	key_name = key.name;
-      }
       /* conditionally format JSON in the obvious way--I'm unsure if
        * AWS actually does this */
       if (s->format == RGWFormat::XML) {
@@ -1881,7 +1855,7 @@ void RGWListBucket_ObjStore_S3::send_response()
 	// json
 	s->formatter->open_object_section("dummy");
       }
-      s->formatter->dump_string("Key", key_name);
+      dump_urlsafe(s ,encode_key, "Key", key.name);
       dump_time(s, "LastModified", iter->meta.mtime);
       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
       s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -1905,7 +1879,7 @@ void RGWListBucket_ObjStore_S3::send_response()
   }
   s->formatter->dump_string("Marker", marker.name);
   if (is_truncated && !next_marker.empty()) {
-    s->formatter->dump_string("NextMarker", next_marker.name);
+    dump_urlsafe(s, encode_key, "NextMarker", next_marker.name);
   }
   s->formatter->close_section();
   rgw_flush_formatter_and_reset(s, s->formatter);
@@ -1941,14 +1915,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
         s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker());
       }
       rgw_obj_key key(iter->key);
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s, encode_key, "Key", key.name);
       string version_id = key.instance;
       if (version_id.empty()) {
         version_id = "null";
@@ -1986,11 +1953,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
 
       s->formatter->dump_int("KeyCount",objs.size());
       if (start_after_exist) {
@@ -2036,14 +1999,7 @@ void RGWListBucket_ObjStore_S3v2::send_response()
     for (iter = objs.begin(); iter != objs.end(); ++iter) {
       rgw_obj_key key(iter->key);
       s->formatter->open_array_section("Contents");
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s, encode_key, "Key", key.name);
       dump_time(s, "LastModified", iter->meta.mtime);
       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
       s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -4147,11 +4103,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
     for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
       rgw::sal::MultipartUpload* upload = iter->get();
       s->formatter->open_array_section("Upload");
-      if (encode_url) {
-        s->formatter->dump_string("Key", url_encode(upload->get_key(), false));
-      } else {
-        s->formatter->dump_string("Key", upload->get_key());
-      }
+      dump_urlsafe(s, encode_url, "Key", upload->get_key(), false);
       s->formatter->dump_string("UploadId", upload->get_upload_id());
       const ACLOwner& owner = upload->get_owner();
       dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator");
@@ -4163,11 +4115,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
     if (!common_prefixes.empty()) {
       s->formatter->open_array_section("CommonPrefixes");
       for (const auto& kv : common_prefixes) {
-        if (encode_url) {
-          s->formatter->dump_string("Prefix", url_encode(kv.first, false));
-        } else {
-          s->formatter->dump_string("Prefix", kv.first);
-        }
+        dump_urlsafe(s, encode_url, "Prefix", kv.first, false);
       }
       s->formatter->close_section();
     }

From 2573426f54aab237ed855a4e88d5b89ddbd76acd Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Wed, 4 Oct 2023 14:37:42 +0530
Subject: [PATCH 1130/2492] mgr/dashboard: upgrade from old 'graph' type panels
 to the new 'timeseries' panel

The graph panel type is deprecated, and disappears after Grafana v9.1 (current version is 10.0) to prevent more old type panels being created. These should be migrated to the timeseries panel type, to avoid potential problems with future Grafana versions.

Fixes: https://tracker.ceph.com/issues/61720

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../ceph-mixin/dashboards/osd.libsonnet       |   6 +-
 .../ceph-mixin/dashboards/rbd.libsonnet       |   4 +-
 .../ceph-mixin/dashboards/rgw.libsonnet       |   6 +-
 .../ceph-mixin/dashboards/utils.libsonnet     |   2 +-
 .../dashboards_out/ceph-cluster.json          |  55 ++++++++-
 .../dashboards_out/cephfs-overview.json       |  22 +++-
 .../dashboards_out/host-details.json          |  99 ++++++++++++++--
 .../dashboards_out/hosts-overview.json        |  22 +++-
 .../dashboards_out/osd-device-details.json    |  77 ++++++++++--
 .../dashboards_out/osds-overview.json         |  44 ++++++-
 .../dashboards_out/pool-detail.json           |  44 ++++++-
 .../dashboards_out/pool-overview.json         |  33 +++++-
 .../dashboards_out/radosgw-detail.json        |  33 +++++-
 .../dashboards_out/radosgw-overview.json      | 110 ++++++++++++++++--
 .../dashboards_out/radosgw-sync-overview.json |  44 ++++++-
 .../dashboards_out/rbd-details.json           |  39 ++++++-
 .../dashboards_out/rbd-overview.json          |  39 ++++++-
 17 files changed, 605 insertions(+), 74 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards/osd.libsonnet b/monitoring/ceph-mixin/dashboards/osd.libsonnet
index 0015c7f398bd..259bb00d76f5 100644
--- a/monitoring/ceph-mixin/dashboards/osd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/osd.libsonnet
@@ -257,7 +257,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                        nullPointMode='null')
       .addTarget($.addTargetSchema(
         'ceph_osd_numpg{%(matchers)s}' % $.matchers(), 'PGs per OSD', 'time_series', 1, true
-      )) + { gridPos: { x: 12, y: 8, w: 8, h: 8 } },
+      )) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: 'short', custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: 12, y: 8, w: 8, h: 8 } },
       $.gaugeSingleStatPanel(
         'percentunit',
         'OSD onode Hits Ratio',
@@ -357,7 +357,7 @@ local g = import 'grafonnet/grafana.libsonnet';
                             legendFormat1),
           $.addTargetSchema(expr2, legendFormat2),
         ]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'OSD device details',
@@ -613,6 +613,6 @@ local g = import 'grafonnet/grafana.libsonnet';
           )
         ||| % $.matchers(),
         '{{device}} on {{instance}}'
-      )) + { gridPos: { x: 18, y: 11, w: 6, h: 9 } },
+      )) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: 'percentunit', custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: 18, y: 11, w: 6, h: 9 } },
     ]),
 }
diff --git a/monitoring/ceph-mixin/dashboards/rbd.libsonnet b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
index 8ae9d272e50d..f95bcef3e672 100644
--- a/monitoring/ceph-mixin/dashboards/rbd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rbd.libsonnet
@@ -32,7 +32,7 @@ local info_rbd_stats = std.join(
                             '{{pool}} Write'),
           $.addTargetSchema(expr2, '{{pool}} Read'),
         ]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'RBD Details',
@@ -163,7 +163,7 @@ local info_rbd_stats = std.join(
           $.addTargetSchema(expr2,
                             legendFormat2),
         ]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'RBD Overview',
diff --git a/monitoring/ceph-mixin/dashboards/rgw.libsonnet b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
index 49dcf9156884..932eb2122ca7 100644
--- a/monitoring/ceph-mixin/dashboards/rgw.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
@@ -24,7 +24,7 @@ local u = import 'utils.libsonnet';
             '{{source_zone}}'
           ),
         ]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'RGW Sync Overview',
@@ -158,7 +158,7 @@ local u = import 'utils.libsonnet';
       )
       .addTargets(
         [$.addTargetSchema(expr1, legendFormat1)]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'RGW Overview',
@@ -669,7 +669,7 @@ local u = import 'utils.libsonnet';
                          '$datasource')
       .addTargets(
         [$.addTargetSchema(expr1, legendFormat1), $.addTargetSchema(expr2, legendFormat2)]
-      ) + { gridPos: { x: x, y: y, w: w, h: h } };
+      ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } };
 
     $.dashboardSchema(
       'RGW Instance Detail',
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index 257b4a44f660..16ba771cb3a0 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -314,7 +314,7 @@ local timeSeries = import 'timeseries_panel.libsonnet';
                        '$datasource')
     .addTargets(
       [$.addTargetSchema(expr, legendFormat)]
-    ) + { gridPos: { x: x, y: y, w: w, h: h } },
+    ) + { type: 'timeseries' } + { fieldConfig: { defaults: { unit: formatY1, custom: { fillOpacity: 8, showPoints: 'never' } } } } + { gridPos: { x: x, y: y, w: w, h: h } },
 
   simpleSingleStatPanel(format,
                         title,
diff --git a/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json b/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
index 240c17677e3d..95ff659e3977 100644
--- a/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
+++ b/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
@@ -603,6 +603,15 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+           "custom": {
+              "fillOpacity": 8,
+              "showPoints": "never"
+           },
+           "unit": "short"
+        }
+      },
       "fill": 0,
       "gridPos": {
         "h": 6,
@@ -726,7 +735,7 @@
         "sort": 0,
         "value_type": "individual"
       },
-      "type": "graph",
+      "type": "timeseries",
       "xaxis": {
         "buckets": null,
         "mode": "time",
@@ -759,6 +768,15 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+           "custom": {
+              "fillOpacity": 8,
+              "showPoints": "never"
+           },
+           "unit": "ms"
+        }
+      },
       "fill": 0,
       "gridPos": {
         "h": 6,
@@ -832,7 +850,7 @@
         "sort": 0,
         "value_type": "individual"
       },
-      "type": "graph",
+      "type": "timeseries",
       "xaxis": {
         "buckets": null,
         "mode": "time",
@@ -865,6 +883,15 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+           "custom": {
+              "fillOpacity": 8,
+              "showPoints": "never"
+           },
+           "unit": "Bps"
+        }
+      },
       "fill": 1,
       "gridPos": {
         "h": 9,
@@ -924,7 +951,7 @@
         "sort": 0,
         "value_type": "individual"
       },
-      "type": "graph",
+      "type": "timeseries",
       "xaxis": {
         "buckets": null,
         "mode": "time",
@@ -957,6 +984,15 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+           "custom": {
+              "fillOpacity": 8,
+              "showPoints": "never"
+           },
+           "unit": "Bps"
+        }
+      },
       "fill": 1,
       "gridPos": {
         "h": 9,
@@ -1003,7 +1039,7 @@
         "sort": 0,
         "value_type": "individual"
       },
-      "type": "graph",
+      "type": "timeseries",
       "xaxis": {
         "buckets": null,
         "mode": "time",
@@ -1159,6 +1195,15 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+           "custom": {
+              "fillOpacity": 8,
+              "showPoints": "never"
+           },
+           "unit": "ops"
+        }
+      },
       "fill": 0,
       "gridPos": {
         "h": 9,
@@ -1206,7 +1251,7 @@
         "sort": 0,
         "value_type": "individual"
       },
-      "type": "graph",
+      "type": "timeseries",
       "xaxis": {
         "buckets": null,
         "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/cephfs-overview.json b/monitoring/ceph-mixin/dashboards_out/cephfs-overview.json
index 3e7aeef45785..50c850ab197c 100644
--- a/monitoring/ceph-mixin/dashboards_out/cephfs-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/cephfs-overview.json
@@ -63,6 +63,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "none"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -127,7 +136,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -161,6 +170,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "none"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -213,7 +231,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/host-details.json b/monitoring/ceph-mixin/dashboards_out/host-details.json
index 7a5ac25917a0..4971a716eeb9 100644
--- a/monitoring/ceph-mixin/dashboards_out/host-details.json
+++ b/monitoring/ceph-mixin/dashboards_out/host-details.json
@@ -156,6 +156,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Shows the CPU breakdown. When multiple servers are selected, only the first host's cpu data is shown",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "percent"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -208,7 +217,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -249,6 +258,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -330,7 +348,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -364,6 +382,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Show the network load (rx,tx) across all interfaces (excluding loopback 'lo')",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "decbytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -428,7 +455,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -462,6 +489,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "pps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -526,7 +562,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -642,6 +678,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "pps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -706,7 +751,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -759,6 +804,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "For any OSD devices on the host, this chart shows the iops per physical device. Each device is shown by it's name and corresponding OSD id value",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ops"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -823,7 +877,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -857,6 +911,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "For OSD hosts, this chart shows the disk bandwidth (read bytes/sec + write bytes/sec) of the physical OSD device. Each device is shown by device name, and corresponding OSD id",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -921,7 +984,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -955,6 +1018,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "For OSD hosts, this chart shows the latency at the physical drive. Each drive is shown by device name, with it's corresponding OSD id",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1007,7 +1079,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -1041,6 +1113,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Show disk utilization % (util) of any OSD devices on the host by the physical device name and associated OSD id.",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "percent"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1093,7 +1174,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/hosts-overview.json b/monitoring/ceph-mixin/dashboards_out/hosts-overview.json
index f1cd4c49935b..ef8f12abe93f 100644
--- a/monitoring/ceph-mixin/dashboards_out/hosts-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/hosts-overview.json
@@ -547,6 +547,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Show the top 10 busiest hosts by cpu",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "percent"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -599,7 +608,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -633,6 +642,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Top 10 hosts by network load",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -685,7 +703,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/osd-device-details.json b/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
index 811e6d57ef2e..359f5522cd0c 100644
--- a/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
+++ b/monitoring/ceph-mixin/dashboards_out/osd-device-details.json
@@ -63,6 +63,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -127,7 +136,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -161,6 +170,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -225,7 +243,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -259,6 +277,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -323,7 +350,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -376,6 +403,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -440,7 +476,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -474,6 +510,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -538,7 +583,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -572,6 +617,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -636,7 +690,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -670,6 +724,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "percentunit"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -722,7 +785,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/osds-overview.json b/monitoring/ceph-mixin/dashboards_out/osds-overview.json
index b34c6642263d..79210f3cf2eb 100644
--- a/monitoring/ceph-mixin/dashboards_out/osds-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/osds-overview.json
@@ -58,6 +58,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ms"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -124,7 +133,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -245,6 +254,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ms"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -311,7 +329,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -587,6 +605,15 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -640,7 +667,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": 20,
             "mode": "histogram",
@@ -775,6 +802,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Show the read/write workload profile overtime",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -834,7 +870,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/pool-detail.json b/monitoring/ceph-mixin/dashboards_out/pool-detail.json
index dc8b4152a494..b43e38ccb4af 100644
--- a/monitoring/ceph-mixin/dashboards_out/pool-detail.json
+++ b/monitoring/ceph-mixin/dashboards_out/pool-detail.json
@@ -216,6 +216,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ops"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -268,7 +277,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -305,6 +314,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "iops"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -369,7 +387,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -406,6 +424,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -470,7 +497,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -507,6 +534,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -559,7 +595,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/pool-overview.json b/monitoring/ceph-mixin/dashboards_out/pool-overview.json
index 7f042aa5b0d8..d155609046c9 100644
--- a/monitoring/ceph-mixin/dashboards_out/pool-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/pool-overview.json
@@ -1161,6 +1161,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "This chart shows the sum of read and write IOPS from all clients by pool",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1220,7 +1229,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -1254,6 +1263,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "The chart shows the sum of read and write bytes from all clients, by pool",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1306,7 +1324,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -1340,6 +1358,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Historical view of capacity usage, to help identify growth and trends in pool consumption",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1392,7 +1419,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
index 4568f9a4d854..18ba369460a3 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
@@ -69,6 +69,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -128,7 +137,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -162,6 +171,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -221,7 +239,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -261,6 +279,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -334,7 +361,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
index a8256c1f5e16..92e416764e7c 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
@@ -63,6 +63,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -122,7 +131,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -156,6 +165,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "none"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -208,7 +226,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -242,6 +260,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Latencies are shown stacked, without a yaxis to provide a visual indication of GET latency imbalance across RGW hosts",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -294,7 +321,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -328,6 +355,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Total bytes transferred in/out of all radosgw instances within the cluster",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -387,7 +423,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -421,6 +457,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Total bytes transferred in/out through get/put operations, by radosgw instance",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "bytes"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -473,7 +518,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -507,6 +552,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "Latencies are shown stacked, without a yaxis to provide a visual indication of PUT latency imbalance across RGW hosts",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "s"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -559,7 +613,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -612,6 +666,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -696,7 +759,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -730,6 +793,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -835,7 +907,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -869,6 +941,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -942,7 +1023,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -976,6 +1057,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -1056,7 +1146,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-sync-overview.json b/monitoring/ceph-mixin/dashboards_out/radosgw-sync-overview.json
index e0c3037d50a3..09227b958300 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-sync-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-sync-overview.json
@@ -44,6 +44,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -96,7 +105,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -130,6 +139,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -182,7 +200,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -216,6 +234,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ms"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -268,7 +295,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -302,6 +329,15 @@
          "dashes": false,
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -354,7 +390,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/rbd-details.json b/monitoring/ceph-mixin/dashboards_out/rbd-details.json
index 2d5d4ecac4a5..22d7e0ea8baa 100644
--- a/monitoring/ceph-mixin/dashboards_out/rbd-details.json
+++ b/monitoring/ceph-mixin/dashboards_out/rbd-details.json
@@ -43,7 +43,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "iops"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -103,7 +112,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -136,7 +145,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -196,7 +214,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -229,7 +247,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ns"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -289,7 +316,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
diff --git a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
index c367bd16fa38..2cd881345c89 100644
--- a/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/rbd-overview.json
@@ -55,7 +55,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "short"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -115,7 +124,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -148,7 +157,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "Bps"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -208,7 +226,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",
@@ -241,7 +259,16 @@
          "dashLength": 10,
          "dashes": false,
          "datasource": "$datasource",
-         "description": "RBD per-image IO statistics are disabled by default.\n\nPlease refer to https://docs.ceph.com/en/latest/mgr/prometheus/#rbd-io-statistics for information about how to enable those optionally.",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 8,
+                  "showPoints": "never"
+               },
+               "unit": "ns"
+            }
+         },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
@@ -301,7 +328,7 @@
             "sort": 0,
             "value_type": "individual"
          },
-         "type": "graph",
+         "type": "timeseries",
          "xaxis": {
             "buckets": null,
             "mode": "time",

From 64b0eb7d050755aab58a4ec7bcb218f9ced19921 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 22 Dec 2023 17:39:20 +1000
Subject: [PATCH 1131/2492] doc/radosgw: list supported plugins-compression.rst

List supported compression plugins in doc/radosgw/compression.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/compression.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/radosgw/compression.rst b/doc/radosgw/compression.rst
index 0fa1f57185c9..429dde91c73d 100644
--- a/doc/radosgw/compression.rst
+++ b/doc/radosgw/compression.rst
@@ -10,6 +10,12 @@ using any of the existing compression plugins.
 .. note:: The Reef release added a :ref:`feature_compress_encrypted` zonegroup
    feature to enable compression with `Server-Side Encryption`_.
 
+Supported compression plugins include the following:
+
+* lz4
+* snappy
+* zlib
+* zstd
 
 Configuration
 =============

From 273f359380bab87ae46b3325ea45940912533fe5 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 22 Dec 2023 23:20:27 +0300
Subject: [PATCH 1132/2492] os/bluestore: report effective
 bluestore_allocation_from_file in osd metadata.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 25a8520bce0f..7b89f0e46fdc 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -11521,6 +11521,7 @@ void BlueStore::collect_metadata(map<string,string> *pm)
     }
   }
   (*pm)["bluestore_min_alloc_size"] = stringify(min_alloc_size);
+  (*pm)["bluestore_allocation_from_file"] = stringify(fm && fm->is_null_manager());
 }
 
 int BlueStore::get_numa_node(

From f3061c41e420de9d626442079b2affc57da64b5a Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Sat, 23 Dec 2023 00:05:37 +0300
Subject: [PATCH 1133/2492] os/bluestore: fix free space update after
 bdev-expand in NCB mode.

Initially this was omitted due to read-only mode.

Fixes: https://tracker.ceph.com/issues/63858

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 7b89f0e46fdc..b154e0b52213 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -8772,17 +8772,16 @@ int BlueStore::expand_devices(ostream& out)
           << std::endl;
       }
     }
+    _close_db_and_around();
 
+    // mount in read/write to sync expansion changes
+    r = _mount();
+    ceph_assert(r == 0);
     if (fm && fm->is_null_manager()) {
       // we grow the allocation range, must reflect it in the allocation file
       alloc->init_add_free(size0, size - size0);
       need_to_destage_allocation_file = true;
     }
-    _close_db_and_around();
-
-    // mount in read/write to sync expansion changes
-    r = _mount();
-    ceph_assert(r == 0);
     umount();
   } else {
     _close_db_and_around();

From bf5b1cd07a29edf7feafa26aa5ae38470453b149 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@redhat.com>
Date: Fri, 22 Dec 2023 17:43:17 -0600
Subject: [PATCH 1134/2492] .github/workflows: upgrade stale bot

Signed-off-by: Laura Flores <lflores@redhat.com>
---
 .github/workflows/stale.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 31478e8e8f65..510a6bebd4e2 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -9,7 +9,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           # PAT for GitHub API authentication
           repo-token: "${{ secrets.GITHUB_TOKEN }}"

From 491745a1c53e3ff1d8b792946ad556974a232651 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 28 Nov 2023 09:46:01 -0600
Subject: [PATCH 1135/2492] osd/scrub: unhandled scrub backend errors should
 cause an abort

... as we do not have any mechanism to handle them.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc   | 11 ++++++-----
 src/osd/scrubber/scrub_machine.cc | 23 +++--------------------
 src/osd/scrubber/scrub_machine.h  | 13 ++++---------
 3 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 06dff82b3612..9dd35d61871c 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -1230,14 +1230,15 @@ int PgScrubber::build_replica_map_chunk()
     break;
 
     default:
-      // negative retval: build_scrub_map_chunk() signalled an error
+      // build_scrub_map_chunk() signalled an error
       // Pre-Pacific code ignored this option, treating it as a success.
-      // \todo Add an error flag in the returning message.
-      // \todo: must either abort, send a reply, or return some error message
+      // Now: "regular" I/O errors were already handled by the backend (by
+      // setting the error flag in the scrub-map). We are left with the
+      // "unknown" error case - and we have no mechanism to handle it.
+      // Thus - we must abort.
       dout(1) << "Error! Aborting. ActiveReplica::react(SchedReplica) Ret: "
 	      << ret << dendl;
-      // only in debug mode for now:
-      assert(false && "backend error");
+      ceph_abort_msg("backend error");
       break;
   };
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index c68a4a1119c1..e0da61dde0f8 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -299,18 +299,6 @@ ActiveScrubbing::~ActiveScrubbing()
   }
 }
 
-/*
- * The only source of an InternalError event as of now is the BuildMap state,
- * when encountering a backend error.
- * We kill the scrub and reset the FSM.
- */
-sc::result ActiveScrubbing::react(const InternalError&)
-{
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << __func__ << dendl;
-  return transit<NotActive>();
-}
-
 // ----------------------- RangeBlocked -----------------------------------
 
 /*
@@ -535,18 +523,13 @@ BuildMap::BuildMap(my_context ctx)
 
   } else {
 
-    auto ret = scrbr->build_primary_map_chunk();
-
-    if (ret == -EINPROGRESS) {
+    // note that build_primary_map_chunk() may return -EINPROGRESS, but no
+    // other error value (as those errors would cause it to crash the OSD).
+    if (scrbr->build_primary_map_chunk() == -EINPROGRESS) {
       // must wait for the backend to finish. No specific event provided.
       // build_primary_map_chunk() has already requeued us.
       dout(20) << "waiting for the backend..." << dendl;
 
-    } else if (ret < 0) {
-
-      dout(10) << "BuildMap::BuildMap() Error! Aborting. Ret: " << ret << dendl;
-      post_event(InternalError{});
-
     } else {
 
       // the local map was created
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 42cf1b20f748..df9a4c45e95f 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -156,8 +156,6 @@ MEV(GotReplicas)
 /// internal - BuildMap preempted. Required, as detected within the ctor
 MEV(IntBmPreempted)
 
-MEV(InternalError)
-
 MEV(IntLocalMapDone)
 
 /// external. called upon success of a MODIFY op. See
@@ -481,10 +479,6 @@ struct ActiveScrubbing
 
   explicit ActiveScrubbing(my_context ctx);
   ~ActiveScrubbing();
-
-  using reactions = mpl::list<sc::custom_reaction<InternalError>>;
-
-  sc::result react(const InternalError&);
 };
 
 struct RangeBlocked : sc::state<RangeBlocked, ActiveScrubbing>, NamedSimply {
@@ -566,12 +560,13 @@ struct BuildMap : sc::state<BuildMap, ActiveScrubbing>, NamedSimply {
   explicit BuildMap(my_context ctx);
 
   // possible error scenarios:
-  // - an error reported by the backend will trigger an 'InternalError' event,
-  //   handled by our parent state;
+  // - an error reported by the backend will cause the scrubber to
+  //   ceph_abort() the OSD. No need to handle it here.
   // - if preempted, we switch to DrainReplMaps, where we will wait for all
   //   replicas to send their maps before acknowledging the preemption;
   // - an interval change will be handled by the relevant 'send-event'
-  //   functions, and will translated into a 'FullReset' event.
+  //   functions, translated into an IntervalChanged event (handled by
+  //   the 'Session' state).
   using reactions = mpl::list<sc::transition<IntBmPreempted, DrainReplMaps>,
 			      // looping, waiting for the backend to finish:
 			      sc::transition<InternalSchedScrub, BuildMap>,

From 304db57b36d871143273620fa3b4481456b4a8d8 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 23 Dec 2023 12:19:06 -0600
Subject: [PATCH 1136/2492] osd/scrub: remove unused PGScrubGotLocalMap message

The relevant event (IntLocalMapDone) is processed directly within the
scrubber, instead of being queued to the OSD events queue.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc                       | 6 ------
 src/osd/OSD.h                        | 3 ---
 src/osd/PG.h                         | 5 -----
 src/osd/scheduler/OpSchedulerItem.cc | 9 ---------
 src/osd/scheduler/OpSchedulerItem.h  | 8 --------
 5 files changed, 31 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0f9c4f1acc79..39e408c30354 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1840,12 +1840,6 @@ void OSDService::queue_scrub_digest_update(PG* pg, Scrub::scrub_prio_t with_prio
   queue_scrub_event_msg<PGScrubDigestUpdate>(pg, with_priority);
 }
 
-void OSDService::queue_scrub_got_local_map(PG* pg, Scrub::scrub_prio_t with_priority)
-{
-  // Resulting scrub event: 'IntLocalMapDone'
-  queue_scrub_event_msg<PGScrubGotLocalMap>(pg, with_priority);
-}
-
 void OSDService::queue_scrub_got_repl_maps(PG* pg, Scrub::scrub_prio_t with_priority)
 {
   // Resulting scrub event: 'GotReplicas'
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 2eff838a2f93..2c806fd356e6 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -526,9 +526,6 @@ class OSDService : public Scrub::ScrubSchedListener {
   /// Signals that all write OPs are done
   void queue_scrub_digest_update(PG* pg, Scrub::scrub_prio_t with_priority);
 
-  /// Signals that the the local (Primary's) scrub map is ready
-  void queue_scrub_got_local_map(PG* pg, Scrub::scrub_prio_t with_priority);
-
   /// Signals that we (the Primary) got all waited-for scrub-maps from our replicas
   void queue_scrub_got_repl_maps(PG* pg, Scrub::scrub_prio_t with_priority);
 
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 6e81af436e7d..a5f01aecc27d 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -475,11 +475,6 @@ class PG : public DoutPrefixProvider,
     forward_scrub_event(&ScrubPgIF::digest_update_notification, queued, "DigestUpdate");
   }
 
-  void scrub_send_local_map_ready(epoch_t queued, ThreadPool::TPHandle& handle)
-  {
-    forward_scrub_event(&ScrubPgIF::send_local_map_done, queued, "IntLocalMapDone");
-  }
-
   void scrub_send_replmaps_ready(epoch_t queued, ThreadPool::TPHandle& handle)
   {
     forward_scrub_event(&ScrubPgIF::send_replica_maps_ready, queued, "GotReplicas");
diff --git a/src/osd/scheduler/OpSchedulerItem.cc b/src/osd/scheduler/OpSchedulerItem.cc
index 750fc2a4f58e..ecad656bab9a 100644
--- a/src/osd/scheduler/OpSchedulerItem.cc
+++ b/src/osd/scheduler/OpSchedulerItem.cc
@@ -113,15 +113,6 @@ void PGScrubDigestUpdate::run(OSD* osd,
   pg->unlock();
 }
 
-void PGScrubGotLocalMap::run(OSD* osd,
-			     OSDShard* sdata,
-			     PGRef& pg,
-			     ThreadPool::TPHandle& handle)
-{
-  pg->scrub_send_local_map_ready(epoch_queued, handle);
-  pg->unlock();
-}
-
 void PGScrubGotReplMaps::run(OSD* osd,
 			     OSDShard* sdata,
 			     PGRef& pg,
diff --git a/src/osd/scheduler/OpSchedulerItem.h b/src/osd/scheduler/OpSchedulerItem.h
index 7fb7125a1416..0b28e64158e4 100644
--- a/src/osd/scheduler/OpSchedulerItem.h
+++ b/src/osd/scheduler/OpSchedulerItem.h
@@ -423,14 +423,6 @@ class PGScrubDigestUpdate : public PGScrubItem {
   void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
 };
 
-class PGScrubGotLocalMap : public PGScrubItem {
- public:
-  PGScrubGotLocalMap(spg_t pg, epoch_t epoch_queued)
-    : PGScrubItem{pg, epoch_queued, "PGScrubGotLocalMap"}
-  {}
-  void run(OSD* osd, OSDShard* sdata, PGRef& pg, ThreadPool::TPHandle& handle) final;
-};
-
 class PGScrubGotReplMaps : public PGScrubItem {
  public:
   PGScrubGotReplMaps(spg_t pg, epoch_t epoch_queued)

From 819b2974e64548fcf66423cca94a785887fd7d07 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 24 Dec 2023 20:24:26 +1000
Subject: [PATCH 1137/2492] doc/radosgw: edit admin.rst 1 of x

Edit doc/radosgw/admin.rst to the end of the section "User Management".
Correct grammar (mostly awkward prepositions) and tidy up weird clauses.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index e228f22dd239..2ca02e593d14 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -2,25 +2,23 @@
  Admin Guide
 =============
 
-Once you have your Ceph Object Storage service up and running, you may
-administer the service with user management, access controls, quotas 
-and usage tracking among other features.
-
+After the Ceph Object Storage service is up and running, it can be administered
+with user management, access controls, quotas, and usage tracking.
 
 User Management
 ===============
 
-Ceph Object Storage user management refers to users of the Ceph Object Storage
-service (i.e., not the Ceph Object Gateway as a user of the Ceph Storage
-Cluster). You must create a user, access key and secret to enable end users to
-interact with Ceph Object Gateway services.
+Ceph Object Storage user management refers only to users of the Ceph Object
+Storage service and not to the Ceph Object Gateway as a user of the Ceph
+Storage Cluster. Create a user, access key, and secret key to enable end users
+to interact with Ceph Object Gateway services.
 
-There are two user types: 
+There are two types of user: 
 
-- **User:** The term 'user' reflects a user of the S3 interface.
+- **User:** The term "user" refers to  user of the S3 interface.
 
-- **Subuser:** The term 'subuser' reflects a user of the Swift interface. A subuser
-  is associated to a user .
+- **Subuser:** The term "subuser" refers to a user of the Swift interface. A
+  subuser is associated with a user. 
   
 .. ditaa::
            +---------+
@@ -31,12 +29,14 @@ There are two user types:
                 +-----+  Subuser  |
                       +-----------+
 
-You can create, modify, view, suspend and remove users and subusers. In addition
-to user and subuser IDs, you may add a display name and an email address for a
-user.  You can specify a key and secret, or generate a key and secret
-automatically. When generating or specifying keys, note that user IDs correspond
-to an S3 key type and subuser IDs correspond to a swift key type. Swift keys
-also have access levels of ``read``, ``write``, ``readwrite`` and ``full``.
+Users and subusers can be created, modified, viewed, suspended and removed.
+you may add a Display names and an email addresses can be added to user
+profiles. Keys and secrets can either be specified or generated automatically.
+When generating or specifying keys, remember that user IDs correspond to S3 key
+types and subuser IDs correspond to Swift key types. 
+
+Swift keys have access levels of ``read``, ``write``, ``readwrite`` and
+``full``.
 
 
 Create a User

From 46139c8fa9d4ae0e493bca17ba1199e3a67c39f8 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 26 Dec 2023 14:32:35 +1000
Subject: [PATCH 1138/2492] doc/radosgw: admin.rst - edit "Create a User"

Edit the section "Create a User" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 2ca02e593d14..bbcf9e218965 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -42,13 +42,17 @@ Swift keys have access levels of ``read``, ``write``, ``readwrite`` and
 Create a User
 -------------
 
-To create a user (S3 interface), execute the following::
+To create a user (S3 interface), run a command of the following form:
 
-	radosgw-admin user create --uid={username} --display-name="{display-name}" [--email={email}]
+.. prompt:: bash
 
-For example:: 	
+   radosgw-admin user create --uid={username} --display-name="{display-name}" [--email={email}]
+
+For example:
+
+.. prompt:: bash
 	
-  radosgw-admin user create --uid=johndoe --display-name="John Doe" --email=john@example.com
+   radosgw-admin user create --uid=johndoe --display-name="John Doe" --email=john@example.com
   
 .. code-block:: javascript
   
@@ -75,16 +79,15 @@ For example::
         "max_objects": -1},
     "temp_url_keys": []}
 
-Creating a user also creates an ``access_key`` and ``secret_key`` entry for use
-with any S3 API-compatible client.  
+The creation of a user entails the creation of an ``access_key`` and a
+``secret_key`` entry, which can be used with any S3 API-compatible client.  
 
-.. important:: Check the key output. Sometimes ``radosgw-admin``
-   generates a JSON escape (``\``) character, and some clients
-   do not know how to handle JSON escape characters. Remedies include 
-   removing the JSON escape character (``\``), encapsulating the string
-   in quotes, regenerating the key and ensuring that it 
-   does not have a JSON escape character or specify the key and secret 
-   manually.
+.. important:: Check the key output. Sometimes ``radosgw-admin`` generates a
+   JSON escape (``\``) character, and some clients do not know how to handle
+   JSON escape characters. Remedies include removing the JSON escape character
+   (``\``), encapsulating the string in quotes, regenerating the key and
+   ensuring that it does not have a JSON escape character, or specifying the
+   key and secret manually.
 
 
 Create a Subuser

From 3c7a6c41df540f935b0b0b2c4fdc7375327a7662 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 27 Dec 2023 14:28:51 +1000
Subject: [PATCH 1139/2492] doc/radosgw: admin.rst - edit "Create a Subuser"

Edit the section "Create a Subuser" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index bbcf9e218965..7cd11552660c 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -93,17 +93,23 @@ The creation of a user entails the creation of an ``access_key`` and a
 Create a Subuser
 ----------------
 
-To create a subuser (Swift interface) for the user, you must specify the user ID
-(``--uid={username}``), a subuser ID and the access level for the subuser. ::
+To create a subuser (a user of the Swift interface) for the user, specify the
+user ID (``--uid={username}``), a subuser ID, and the subuser's access level:
 
-  radosgw-admin subuser create --uid={uid} --subuser={uid} --access=[ read | write | readwrite | full ]
+.. prompt:: bash
 
-For example::
+   radosgw-admin subuser create --uid={uid} --subuser={uid} --access=[ read | write | readwrite | full ]
+
+For example:
+
+.. prompt:: bash
 
-  radosgw-admin subuser create --uid=johndoe --subuser=johndoe:swift --access=full
+   radosgw-admin subuser create --uid=johndoe --subuser=johndoe:swift --access=full
 
 
-.. note:: ``full`` is not ``readwrite``, as it also includes the access control policy.
+.. note:: ``full`` is not the same as ``readwrite``. The ``full`` access level
+   includes ``read`` and ``write``, but it also includes the access control
+   policy.
 
 .. code-block:: javascript
 

From 390fbf7fc0638b0c30a2dfb518cdbc8d4310a978 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 08:20:35 +0000
Subject: [PATCH 1140/2492] crimson/osd/heartbeat: maybe_share_osdmap() fix
 `send_from`

send_from should be the projected_epoch + 1,
See: OSDSingletonState::load_map_bls() which loads the
following range of osdmaps: [send_from, last].

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/heartbeat.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc
index 266e56533c3b..49e356249479 100644
--- a/src/crimson/osd/heartbeat.cc
+++ b/src/crimson/osd/heartbeat.cc
@@ -333,9 +333,8 @@ seastar::future<> Heartbeat::maybe_share_osdmap(
     return seastar::now();
   }
 
-  const epoch_t send_from = peer.get_projected_epoch();
-  logger().debug("{} sending peer {} peer maps from projected epoch {} through "
-		 "local osdmap epoch {}",
+  const epoch_t send_from = peer.get_projected_epoch() + 1;
+  logger().debug("{} sending peer {} peer maps ({}, {}]",
 		 __func__,
 		 from,
 		 send_from,

From ae940291c4dd43dce3aeb142fef095492fba1f93 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 08:26:50 +0000
Subject: [PATCH 1141/2492] crimson/osd/shard_services: load_map_bl() cache
 when loading from disc

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/shard_services.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index d73a85fb5f85..5e50e740fd3e 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -387,7 +387,10 @@ seastar::future<bufferlist> OSDSingletonState::load_map_bl(
     return seastar::make_ready_future<bufferlist>(*found);
   } else {
     logger().debug("{} loading osdmap.{} from disk", __func__, e);
-    return meta_coll->load_map(e);
+    return meta_coll->load_map(e).then([this, e](auto&& bl) {
+      map_bl_cache.insert(e, bl);
+      return seastar::make_ready_future<bufferlist>(std::move(bl));
+    });
   }
 }
 

From 42a6221e4d7f4b2882fbda6fa8eddb107adcdd11 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 08:31:05 +0000
Subject: [PATCH 1142/2492] crimson/osd: incremental maps groundwork

Note: * no actual users are added yet.
      * load_inc_map returns erroartor since we should handle
        inc maps reading errors (not-fatal).

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_meta.cc       | 25 ++++++++++++++++++++++++-
 src/crimson/osd/osd_meta.h        |  8 ++++++++
 src/crimson/osd/shard_services.cc | 23 +++++++++++++++++++++++
 src/crimson/osd/shard_services.h  |  5 +++++
 4 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_meta.cc b/src/crimson/osd/osd_meta.cc
index 06d6932bc54f..551229a4c6eb 100644
--- a/src/crimson/osd/osd_meta.cc
+++ b/src/crimson/osd/osd_meta.cc
@@ -12,7 +12,6 @@
 #include "osd/OSDMap.h"
 
 using std::string;
-using read_errorator = crimson::os::FuturizedStore::Shard::read_errorator;
 
 void OSDMeta::create(ceph::os::Transaction& t)
 {
@@ -25,11 +24,22 @@ void OSDMeta::store_map(ceph::os::Transaction& t,
   t.write(coll->get_cid(), osdmap_oid(e), 0, m.length(), m);
 }
 
+void OSDMeta::store_inc_map(ceph::os::Transaction& t,
+                        epoch_t e, const bufferlist& m)
+{
+  t.write(coll->get_cid(), inc_osdmap_oid(e), 0, m.length(), m);
+}
+
 void OSDMeta::remove_map(ceph::os::Transaction& t, epoch_t e)
 {
   t.remove(coll->get_cid(), osdmap_oid(e));
 }
 
+void OSDMeta::remove_inc_map(ceph::os::Transaction& t, epoch_t e)
+{
+  t.remove(coll->get_cid(), inc_osdmap_oid(e));
+}
+
 seastar::future<bufferlist> OSDMeta::load_map(epoch_t e)
 {
   return store.read(coll,
@@ -41,6 +51,13 @@ seastar::future<bufferlist> OSDMeta::load_map(epoch_t e)
     }));
 }
 
+read_errorator::future<ceph::bufferlist> OSDMeta::load_inc_map(epoch_t e)
+{
+  return store.read(coll,
+                    osdmap_oid(e), 0, 0,
+                    CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
+}
+
 void OSDMeta::store_superblock(ceph::os::Transaction& t,
                                const OSDSuperblock& superblock)
 {
@@ -122,6 +139,12 @@ ghobject_t OSDMeta::osdmap_oid(epoch_t epoch)
   return ghobject_t(hobject_t(sobject_t(object_t(name), 0)));
 }
 
+ghobject_t OSDMeta::inc_osdmap_oid(epoch_t epoch)
+{
+  string name = fmt::format("inc_osdmap.{}", epoch);
+  return ghobject_t(hobject_t(sobject_t(object_t(name), 0)));
+}
+
 ghobject_t OSDMeta::final_pool_info_oid(int64_t pool)
 {
   string name = fmt::format("final_pool_{}", pool);
diff --git a/src/crimson/osd/osd_meta.h b/src/crimson/osd/osd_meta.h
index 2363671e330d..506007e397d6 100644
--- a/src/crimson/osd/osd_meta.h
+++ b/src/crimson/osd/osd_meta.h
@@ -19,6 +19,8 @@ namespace crimson::os {
   class FuturizedStore;
 }
 
+using read_errorator = crimson::os::FuturizedStore::Shard::read_errorator;
+
 /// metadata shared across PGs, or put in another way,
 /// metadata not specific to certain PGs.
 class OSDMeta {
@@ -40,8 +42,13 @@ class OSDMeta {
 
   void store_map(ceph::os::Transaction& t,
                  epoch_t e, const bufferlist& m);
+  void store_inc_map(ceph::os::Transaction& t,
+                 epoch_t e, const bufferlist& m);
   void remove_map(ceph::os::Transaction& t, epoch_t e);
+  void remove_inc_map(ceph::os::Transaction& t, epoch_t e);
+
   seastar::future<bufferlist> load_map(epoch_t e);
+  read_errorator::future<ceph::bufferlist> load_inc_map(epoch_t e);
 
   void store_superblock(ceph::os::Transaction& t,
                         const OSDSuperblock& sb);
@@ -60,6 +67,7 @@ class OSDMeta {
     std::map<epoch_t, OSDMap*>&);
 private:
   static ghobject_t osdmap_oid(epoch_t epoch);
+  static ghobject_t inc_osdmap_oid(epoch_t epoch);
   static ghobject_t final_pool_info_oid(int64_t pool);
   static ghobject_t superblock_oid();
 };
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 5e50e740fd3e..9d9efb8945a6 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -379,6 +379,14 @@ void OSDSingletonState::store_map_bl(
   map_bl_cache.insert(e, std::move(bl));
 }
 
+void OSDSingletonState::store_inc_map_bl(
+  ceph::os::Transaction& t,
+  epoch_t e, bufferlist&& bl)
+{
+  meta_coll->store_inc_map(t, e, bl);
+  inc_map_bl_cache.insert(e, std::move(bl));
+}
+
 seastar::future<bufferlist> OSDSingletonState::load_map_bl(
   epoch_t e)
 {
@@ -394,6 +402,21 @@ seastar::future<bufferlist> OSDSingletonState::load_map_bl(
   }
 }
 
+read_errorator::future<ceph::bufferlist> OSDSingletonState::load_inc_map_bl(
+  epoch_t e)
+{
+  if (std::optional<bufferlist> found = inc_map_bl_cache.find(e); found) {
+    logger().debug("{} inc map.{} found in cache", __func__, e);
+    return read_errorator::make_ready_future<bufferlist>(*found);
+  } else {
+    logger().debug("{} loading inc map.{} from disk", __func__, e);
+    return meta_coll->load_inc_map(e).safe_then([this, e](auto&& bl) {
+      inc_map_bl_cache.insert(e, bl);
+      return seastar::make_ready_future<bufferlist>(std::move(bl));
+    }, read_errorator::pass_further{});
+  }
+}
+
 seastar::future<std::map<epoch_t, bufferlist>> OSDSingletonState::load_map_bls(
   epoch_t first,
   epoch_t last)
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 37993a4f6795..62f0080e2a7b 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -218,6 +218,7 @@ class OSDSingletonState : public md_config_obs_t {
   friend class OSD;
   using cached_map_t = OSDMapService::cached_map_t;
   using local_cached_map_t = OSDMapService::local_cached_map_t;
+  using read_errorator = crimson::os::FuturizedStore::Shard::read_errorator;
 
 public:
   OSDSingletonState(
@@ -236,6 +237,7 @@ class OSDSingletonState : public md_config_obs_t {
 
   SharedLRU<epoch_t, OSDMap> osdmaps;
   SimpleLRU<epoch_t, bufferlist, false> map_bl_cache;
+  SimpleLRU<epoch_t, bufferlist, false> inc_map_bl_cache;
 
   cached_map_t osdmap;
   cached_map_t &get_osdmap() { return osdmap; }
@@ -319,9 +321,12 @@ class OSDSingletonState : public md_config_obs_t {
   seastar::future<std::unique_ptr<OSDMap>> load_map(epoch_t e);
   seastar::future<bufferlist> load_map_bl(epoch_t e);
   seastar::future<std::map<epoch_t, bufferlist>>
+  read_errorator::future<ceph::bufferlist> load_inc_map_bl(epoch_t e);
   load_map_bls(epoch_t first, epoch_t last);
   void store_map_bl(ceph::os::Transaction& t,
                     epoch_t e, bufferlist&& bl);
+  void store_inc_map_bl(ceph::os::Transaction& t,
+                    epoch_t e, bufferlist&& bl);
   seastar::future<> store_maps(ceph::os::Transaction& t,
                                epoch_t start, Ref<MOSDMap> m);
   void trim_maps(ceph::os::Transaction& t, OSDSuperblock& superblock);

From 21ebd2e855a4dd2d2d1244bb3688db0c6613dc30 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 27 Dec 2023 22:14:59 +1000
Subject: [PATCH 1143/2492] doc/radosgw: admin.rst - edit sections

Edit the following sections in doc/radosgw/admin.rst:

- Get User Info
- Modify User Info
- User Suspend/Enable (I split this into two sections)

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 51 ++++++++++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index bbcf9e218965..e9d5697acc27 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -136,41 +136,56 @@ For example::
 Get User Info
 -------------
 
-To get information about a user, you must specify ``user info`` and the user ID
-(``--uid={username}``) . :: 
+To get information about a user, specify ``user info`` and the user ID
+(``--uid={username}``). Use a command of the following form: 
 
-	radosgw-admin user info --uid=johndoe
+.. prompt:: bash
 
+   radosgw-admin user info --uid=johndoe
 
 
 Modify User Info
 ----------------
 
-To modify information about a user, you must specify the user ID (``--uid={username}``)
-and the attributes you want to modify. Typical modifications are to keys and secrets,
-email addresses, display names and access levels. For example:: 
+To modify information about a user, specify the user ID (``--uid={username}``)
+and the attributes that you want to modify. Typical modifications are made to
+keys and secrets, email addresses, display names, and access levels. Use a
+command of the following form: 
+
+.. prompt:: bash
 
-	radosgw-admin user modify --uid=johndoe --display-name="John E. Doe"
+   radosgw-admin user modify --uid=johndoe --display-name="John E. Doe"
 
-To modify subuser values, specify ``subuser modify``, user ID and the subuser ID. For example::
+To modify subuser values, specify ``subuser modify``, user ID and the subuser
+ID. Use a command of the following form:
 
-	radosgw-admin subuser modify --uid=johndoe --subuser=johndoe:swift --access=full
+.. prompt:: bash
 
+   radosgw-admin subuser modify --uid=johndoe --subuser=johndoe:swift --access=full
 
-User Enable/Suspend
--------------------
 
-When you create a user, the user is enabled by default. However, you may suspend
-user  privileges and re-enable them at a later time. To suspend a user, specify
-``user suspend`` and the user ID. ::
+User Suspend
+------------
 
-	radosgw-admin user suspend --uid=johndoe
+When a user is created, the user is enabled by default. However, it is possible
+to suspend user privileges and to re-enable them at a later time. To suspend a
+user, specify ``user suspend`` and the user ID in a command of the following
+form:
 
-To re-enable a suspended user, specify ``user enable`` and the user ID. :: 
+.. prompt:: bash
+
+   radosgw-admin user suspend --uid=johndoe
+
+User Enable
+-----------
+To re-enable a suspended user, provide ``user enable`` and specify the user ID
+in a command of the following form:
+
+.. prompt:: bash
 
-	radosgw-admin user enable --uid=johndoe
+   radosgw-admin user enable --uid=johndoe
 	
-.. note:: Disabling the user disables the subuser.
+.. note:: Disabling the user also disables any subusers.
 
 
 Remove a User

From 7480bae4b30ec1633f5c8f31855f0556bcbfc136 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 08:33:46 +0000
Subject: [PATCH 1144/2492] crimson/osd: rewrite build_incremental_map_msg

* load_map_bls() to load inc maps first (if possible).

* build_incremental_map_msg() to load full map in case of a map gap.

* Conf options are now maintained:
	* osd_map_share_max_epochs
        * osd_map_message_max
	* osd_map_message_max_bytes

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osdmap_service.h  |   7 ++
 src/crimson/osd/shard_services.cc | 126 ++++++++++++++++++++----------
 src/crimson/osd/shard_services.h  |   7 +-
 3 files changed, 96 insertions(+), 44 deletions(-)

diff --git a/src/crimson/osd/osdmap_service.h b/src/crimson/osd/osdmap_service.h
index 017303536dc0..b70f6635d809 100644
--- a/src/crimson/osd/osdmap_service.h
+++ b/src/crimson/osd/osdmap_service.h
@@ -12,6 +12,13 @@ class OSDMapService {
 public:
   using cached_map_t = OSDMapRef;
   using local_cached_map_t = LocalOSDMapRef;
+  enum class encoded_osdmap_type_t {
+    FULLMAP,
+    INCMAP
+  };
+  using bls_pair = std::pair<encoded_osdmap_type_t, bufferlist>;
+  using bls_map_pair_t = std::pair<epoch_t, bls_pair>;
+  using bls_map_t = std::map<epoch_t, bls_pair>;
 
   virtual ~OSDMapService() = default;
   virtual seastar::future<cached_map_t> get_map(epoch_t e) = 0;
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 9d9efb8945a6..60faaac047bd 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -417,25 +417,34 @@ read_errorator::future<ceph::bufferlist> OSDSingletonState::load_inc_map_bl(
   }
 }
 
-seastar::future<std::map<epoch_t, bufferlist>> OSDSingletonState::load_map_bls(
+seastar::future<OSDMapService::bls_map_t> OSDSingletonState::load_map_bls(
   epoch_t first,
   epoch_t last)
 {
   logger().debug("{} loading maps [{},{}]",
                  __func__, first, last);
   ceph_assert(first <= last);
-  // TODO: take osd_map_max into account
-  //int max = cct->_conf->osd_map_message_max;
-  //ssize_t max_bytes = cct->_conf->osd_map_message_max_bytes;
   return seastar::map_reduce(boost::make_counting_iterator<epoch_t>(first),
 			     boost::make_counting_iterator<epoch_t>(last + 1),
 			     [this](epoch_t e) {
-    return load_map_bl(e).then([e](auto&& bl) {
-      return seastar::make_ready_future<std::pair<epoch_t, bufferlist>>(
-	std::make_pair(e, std::move(bl)));
+    return load_inc_map_bl(e).safe_then([](auto&& bl) {
+      return seastar::make_ready_future<OSDMapService::bls_pair>(
+        std::make_pair(OSDMapService::encoded_osdmap_type_t::INCMAP,
+                       std::move(bl)));
+    }, read_errorator::all_same_way([this, e] {
+      logger().debug("load_map_bls: can't load inc map {}, attempting full map instread",
+                      e);
+      return load_map_bl(e).then([](auto&& bl) {
+        return seastar::make_ready_future<OSDMapService::bls_pair>(
+          std::make_pair(OSDMapService::encoded_osdmap_type_t::FULLMAP,
+                         std::move(bl)));
+      });
+    })).then([e] (auto&& loaded_map) {
+      return seastar::make_ready_future<OSDMapService::bls_map_pair_t>(
+        std::make_pair(e, std::move(loaded_map)));
     });
   },
-  std::map<epoch_t, bufferlist>{},
+  OSDMapService::bls_map_t{},
   [](auto&& bls, auto&& epoch_bl) {
     bls.emplace(std::move(epoch_bl));
     return std::move(bls);
@@ -479,7 +488,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	              "loading osdmap.{}", e, e - 1);
 	ceph_assert(std::cmp_greater(e, 0u));
 	return load_map(e - 1).then(
-	  [&added_maps, e, bl=p->second, &t, this](auto o) {
+	  [&added_maps, e, bl=p->second, &t, this](auto o) mutable {
 	  OSDMap::Incremental inc;
 	  auto i = bl.cbegin();
 	  inc.decode(i);
@@ -783,49 +792,80 @@ seastar::future<> ShardServices::dispatch_context(
   });
 }
 
-seastar::future<> OSDSingletonState::send_incremental_map(
-  crimson::net::Connection &conn,
-  epoch_t first)
+seastar::future<MURef<MOSDMap>> OSDSingletonState::build_incremental_map_msg(
+  epoch_t first,
+  epoch_t last)
 {
-  logger().info("{}: first osdmap: {} "
-                "superblock's oldest map: {}",
-                __func__, first, superblock.get_oldest_map());
-  if (first >= superblock.get_oldest_map()) {
-    // TODO: osd_map_share_max_epochs
-    // See OSDService::build_incremental_map_msg
+  return seastar::do_with(crimson::common::local_conf()->osd_map_message_max,
+                          crimson::make_message<MOSDMap>(
+                            monc.get_fsid(),
+                            osdmap->get_encoding_features()),
+                          [this, &first, last](unsigned int map_message_max,
+                                                      auto& m) {
+    m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
+    m->newest_map = superblock.get_newest_map();
+    auto maybe_handle_mapgap = seastar::now();
     if (first < superblock.cluster_osdmap_trim_lower_bound) {
       logger().info("{}: cluster osdmap lower bound: {} "
-                " > first {}, starting with full map",
-                __func__, superblock.cluster_osdmap_trim_lower_bound, first);
+                    " > first {}, starting with full map",
+                    __func__, superblock.cluster_osdmap_trim_lower_bound, first);
       // we don't have the next map the target wants,
       // so start with a full map.
       first = superblock.cluster_osdmap_trim_lower_bound;
+      maybe_handle_mapgap = load_map_bl(first).then(
+      [&first, &map_message_max, &m](auto&& bl) {
+        m->maps[first] = std::move(bl);
+        --map_message_max;
+        ++first;
+      });
     }
-    return load_map_bls(
-      first, superblock.get_newest_map()
-    ).then([this, &conn](auto&& bls) {
-      auto m = crimson::make_message<MOSDMap>(
-	monc.get_fsid(),
-	osdmap->get_encoding_features());
-      m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
-      m->newest_map = superblock.get_newest_map();
-      m->maps = std::move(bls);
-      return conn.send(std::move(m));
-    });
-  } else {
-    // See OSDService::send_incremental_map
-    // just send latest full map
-    return load_map_bl(osdmap->get_epoch()
-    ).then([this, &conn](auto&& bl) mutable {
-      auto m = crimson::make_message<MOSDMap>(
-	monc.get_fsid(),
-	osdmap->get_encoding_features());
-      m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
-      m->newest_map = superblock.get_newest_map();
-      m->maps.emplace(osdmap->get_epoch(), std::move(bl));
-      return conn.send(std::move(m));
+    return maybe_handle_mapgap.then([this, first, last, &map_message_max, &m] {
+      if (first > last) {
+        // first may be later than last in the case of map gap
+        ceph_assert(!m->maps.empty());
+        return seastar::make_ready_future<MURef<MOSDMap>>(std::move(m));
+      }
+      return load_map_bls(
+        first,
+        ((last - first) > map_message_max) ? (first + map_message_max) : last
+      ).then([&m](auto&& bls) {
+        ssize_t map_message_max_bytes = crimson::common::local_conf()->osd_map_message_max_bytes;
+        for (auto const& [e, val] : bls) {
+          map_message_max_bytes -= val.second.length();
+          if (map_message_max_bytes < 0) {
+            break;
+          }
+          if (val.first == OSDMapService::encoded_osdmap_type_t::FULLMAP) {
+            m->maps.emplace(e, std::move(val.second));
+          } else if (val.first == OSDMapService::encoded_osdmap_type_t::INCMAP) {
+            m->incremental_maps.emplace(e, std::move(val.second));
+          } else {
+            ceph_abort();
+          }
+        }
+        return seastar::make_ready_future<MURef<MOSDMap>>(std::move(m));
+      });
     });
+  });
+}
+
+seastar::future<> OSDSingletonState::send_incremental_map(
+  crimson::net::Connection &conn,
+  epoch_t first)
+{
+  epoch_t to = osdmap->get_epoch();
+  logger().info("{}: first osdmap: {} "
+                "superblock's oldest map: {}, "
+                "to {}",
+                __func__, first, superblock.get_oldest_map(), to);
+  if (to > first && (int64_t)(to - first) > crimson::common::local_conf()->osd_map_share_max_epochs) {
+    logger().debug("{} {} > max epochs to send of {}, only sending most recent,",
+                   __func__, (to - first), crimson::common::local_conf()->osd_map_share_max_epochs);
+    first = to - crimson::common::local_conf()->osd_map_share_max_epochs;
   }
+  return build_incremental_map_msg(first, to).then([&conn](auto&& m) {
+    return conn.send(std::move(m));
+  });
 }
 
 seastar::future<> OSDSingletonState::send_incremental_map_to_osd(
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 62f0080e2a7b..fbfe0e55c001 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -270,6 +270,10 @@ class OSDSingletonState : public md_config_obs_t {
     superblock = std::move(_superblock);
   }
 
+  seastar::future<MURef<MOSDMap>> build_incremental_map_msg(
+    epoch_t first,
+    epoch_t last);
+
   seastar::future<> send_incremental_map(
     crimson::net::Connection &conn,
     epoch_t first);
@@ -320,8 +324,8 @@ class OSDSingletonState : public md_config_obs_t {
   seastar::future<local_cached_map_t> get_local_map(epoch_t e);
   seastar::future<std::unique_ptr<OSDMap>> load_map(epoch_t e);
   seastar::future<bufferlist> load_map_bl(epoch_t e);
-  seastar::future<std::map<epoch_t, bufferlist>>
   read_errorator::future<ceph::bufferlist> load_inc_map_bl(epoch_t e);
+  seastar::future<OSDMapService::bls_map_t>
   load_map_bls(epoch_t first, epoch_t last);
   void store_map_bl(ceph::os::Transaction& t,
                     epoch_t e, bufferlist&& bl);
@@ -510,6 +514,7 @@ class ShardServices : public OSDMapService {
   FORWARD_TO_OSD_SINGLETON(get_pool_info)
   FORWARD(with_throttle_while, with_throttle_while, local_state.throttler)
 
+  FORWARD_TO_OSD_SINGLETON(build_incremental_map_msg)
   FORWARD_TO_OSD_SINGLETON(send_incremental_map)
   FORWARD_TO_OSD_SINGLETON(send_incremental_map_to_osd)
 

From df8666abdefaba67d9dfc692e15f218817e1439b Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 09:18:53 +0000
Subject: [PATCH 1145/2492] crimson/osd/shard_services: record inc maps

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/shard_services.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 60faaac047bd..3c3f2a126f10 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -493,6 +493,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	  auto i = bl.cbegin();
 	  inc.decode(i);
 	  o->apply_incremental(inc);
+	  store_inc_map_bl(t, e, std::move(bl));
 	  bufferlist fbl;
 	  o->encode(fbl, inc.encode_features | CEPH_FEATURE_RESERVED);
 	  logger().info("store_maps storing osdmap.{}", o->get_epoch());

From 57030a06a688ad2b45c89c7be3da2a0aa67f7bfd Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 22 Nov 2023 08:34:10 +0000
Subject: [PATCH 1146/2492] crimson/osd/shard_services: trim_maps to trim
 inc_maps

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/shard_services.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 3c3f2a126f10..8da4ccd8262c 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -535,6 +535,7 @@ void OSDSingletonState::trim_maps(ceph::os::Transaction& t,
          t.get_num_ops() < crimson::common::local_conf()->osd_target_transaction_size) {
     logger().debug("{}: removing old osdmap epoch {}", __func__, superblock.get_oldest_map());
     meta_coll->remove_map(t, superblock.get_oldest_map());
+    meta_coll->remove_inc_map(t, superblock.get_oldest_map());
     superblock.maps.erase(superblock.get_oldest_map());
   }
 

From 592f504db1203124d6fdf4ac3bf04e0f2a0bcbdb Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Thu, 16 Nov 2023 13:55:18 +0900
Subject: [PATCH 1147/2492] crimson/os/seastore: move overwritten delta extents
 to delta_overwrite_list if needed during rewrite_dirty

In RBM, Data extents like OBJECT_DATA_BLOCK can be overwritten.
So, this commit overwrites the (delta) extents in Journal to the corresponding  blocks in RBM
if they are Data extents.

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cached_extent.h       |  5 +++
 .../os/seastore/extent_placement_manager.h    | 27 ++++++++++++++++
 src/crimson/os/seastore/seastore_types.cc     |  4 +++
 src/crimson/os/seastore/seastore_types.h      |  2 ++
 src/crimson/os/seastore/transaction.h         | 32 ++++++++++++++-----
 .../os/seastore/transaction_manager.cc        |  6 ++++
 6 files changed, 68 insertions(+), 8 deletions(-)

diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 5fcb8142527d..8bd4a2693851 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -587,6 +587,11 @@ class CachedExtent
     rewrite_generation = gen;
   }
 
+  void set_inplace_rewrite_generation() {
+    user_hint = placement_hint_t::REWRITE;
+    rewrite_generation = OOL_GENERATION;
+  }
+
   bool is_inline() const {
     return poffset.is_relative();
   }
diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h
index b94c03ec34ad..371b49e07414 100644
--- a/src/crimson/os/seastore/extent_placement_manager.h
+++ b/src/crimson/os/seastore/extent_placement_manager.h
@@ -43,6 +43,9 @@ class ExtentOolWriter {
 
   using close_ertr = base_ertr;
   virtual close_ertr::future<> close() = 0;
+
+  virtual bool can_inplace_rewrite(Transaction& t,
+    CachedExtentRef extent) = 0;
 };
 using ExtentOolWriterRef = std::unique_ptr<ExtentOolWriter>;
 
@@ -79,6 +82,11 @@ class SegmentedOolWriter : public ExtentOolWriter {
     return make_delayed_temp_paddr(0);
   }
 
+  bool can_inplace_rewrite(Transaction& t,
+    CachedExtentRef extent) final {
+    return false;
+  }
+
 private:
   alloc_write_iertr::future<> do_write(
     Transaction& t,
@@ -122,6 +130,17 @@ class RandomBlockOolWriter : public ExtentOolWriter {
     return rb_cleaner->alloc_paddr(length);
   }
 
+  bool can_inplace_rewrite(Transaction& t,
+    CachedExtentRef extent) final {
+    if (!extent->is_dirty()) {
+      return false;
+    }
+    assert(t.get_src() == transaction_type_t::TRIM_DIRTY);
+    ceph_assert_always(extent->get_type() == extent_types_t::ROOT ||
+	extent->get_paddr().is_absolute());
+    return crimson::os::seastore::can_inplace_rewrite(extent->get_type());
+  }
+
 private:
   alloc_write_iertr::future<> do_write(
     Transaction& t,
@@ -199,6 +218,14 @@ class ExtentPlacementManager {
     background_process.set_extent_callback(cb);
   }
 
+  bool can_inplace_rewrite(Transaction& t, CachedExtentRef extent) {
+    auto writer = get_writer(placement_hint_t::REWRITE,
+      get_extent_category(extent->get_type()),
+      OOL_GENERATION);
+    ceph_assert(writer);
+    return writer->can_inplace_rewrite(t, extent);
+  }
+
   journal_type_t get_journal_type() const {
     return background_process.get_journal_type();
   }
diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc
index 0acfdb74ebb5..93818ba02586 100644
--- a/src/crimson/os/seastore/seastore_types.cc
+++ b/src/crimson/os/seastore/seastore_types.cc
@@ -286,6 +286,10 @@ std::ostream &operator<<(std::ostream &out, data_category_t c)
   }
 }
 
+bool can_inplace_rewrite(extent_types_t type) {
+  return get_extent_category(type) == data_category_t::DATA;
+}
+
 std::ostream &operator<<(std::ostream &out, sea_time_point_printer_t tp)
 {
   if (tp.tp == NULL_TIME) {
diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h
index 0b4ad853687f..f0e40fed082e 100644
--- a/src/crimson/os/seastore/seastore_types.h
+++ b/src/crimson/os/seastore/seastore_types.h
@@ -1307,6 +1307,8 @@ constexpr data_category_t get_extent_category(extent_types_t type) {
   }
 }
 
+bool can_inplace_rewrite(extent_types_t type);
+
 // type for extent modification time, milliseconds since the epoch
 using sea_time_point = seastar::lowres_system_clock::time_point;
 using sea_duration = seastar::lowres_system_clock::duration;
diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h
index d423196feba7..b43fb0d9cd8e 100644
--- a/src/crimson/os/seastore/transaction.h
+++ b/src/crimson/os/seastore/transaction.h
@@ -217,6 +217,14 @@ class Transaction {
     written_ool_block_list.push_back(ref);
   }
 
+  void add_inplace_rewrite_extent(CachedExtentRef ref) {
+   ceph_assert(!is_weak());
+   ceph_assert(ref);
+   ceph_assert(ref->get_paddr().is_absolute());
+   assert(ref->state == CachedExtent::extent_state_t::DIRTY);
+   pre_inplace_rewrite_list.emplace_back(ref->cast<LogicalCachedExtent>());
+  }
+
   void add_mutated_extent(CachedExtentRef ref) {
     ceph_assert(!is_weak());
     assert(ref->is_exist_mutation_pending() ||
@@ -278,6 +286,11 @@ class Transaction {
 	++num_allocated_invalid_extents;
       }
     }
+    for (auto& extent : pre_inplace_rewrite_list) {
+      if (extent->is_valid()) {
+	ret.push_back(extent);
+      } 
+    }
     return ret;
   }
 
@@ -388,6 +401,7 @@ class Transaction {
     inline_block_list.clear();
     written_ool_block_list.clear();
     pre_alloc_list.clear();
+    pre_inplace_rewrite_list.clear();
     retired_set.clear();
     existing_block_list.clear();
     existing_block_stats = {};
@@ -529,16 +543,18 @@ class Transaction {
   io_stat_t fresh_block_stats;
   uint64_t num_delayed_invalid_extents = 0;
   uint64_t num_allocated_invalid_extents = 0;
-  /// blocks that will be committed with journal record inline
-  std::list<CachedExtentRef> inline_block_list;
-  /// blocks that will be committed with out-of-line record
-  std::list<CachedExtentRef> written_ool_block_list;
-  /// blocks with delayed allocation, may become inline or ool above
+  /// fresh blocks with delayed allocation, may become inline or ool below
   std::list<LogicalCachedExtentRef> delayed_alloc_list;
-
-  /// Extents with pre-allocated addresses,
-  /// will be added to written_ool_block_list after write
+  /// fresh blocks with pre-allocated addresses with RBM,
+  /// should be released upon conflicts, will be added to ool below
   std::list<LogicalCachedExtentRef> pre_alloc_list;
+  /// dirty blocks for inplace rewrite with RBM, will be added to inplace ool below
+  std::list<LogicalCachedExtentRef> pre_inplace_rewrite_list;
+
+  /// fresh blocks that will be committed with inline journal record
+  std::list<CachedExtentRef> inline_block_list;
+  /// fresh blocks that will be committed with out-of-line record
+  std::list<CachedExtentRef> written_ool_block_list;
 
   /// list of mutated blocks, holds refcounts, subset of write_set
   std::list<CachedExtentRef> mutated_block_list;
diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index 7261c84d2df7..e45224412057 100644
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -487,6 +487,12 @@ TransactionManager::rewrite_extent_ret TransactionManager::rewrite_extent(
 
   assert(extent->is_valid() && !extent->is_initial_pending());
   if (extent->is_dirty()) {
+    if (epm->can_inplace_rewrite(t, extent)) {
+      DEBUGT("delta overwriting extent -- {}", t, *extent);
+      t.add_inplace_rewrite_extent(extent);
+      extent->set_inplace_rewrite_generation();
+      return rewrite_extent_iertr::now();
+    }
     extent->set_target_rewrite_generation(INIT_GENERATION);
   } else {
     extent->set_target_rewrite_generation(target_generation);

From 52bf8278121def862f32ad9e7d047f535f1af660 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Thu, 16 Nov 2023 14:39:38 +0900
Subject: [PATCH 1148/2492] crimson/os/seastore: perform a
 inplace-partial-overwrite on the corresponding block using ool

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc              | 21 ++++++++++++++++++-
 src/crimson/os/seastore/cached_extent.h       |  2 ++
 .../os/seastore/extent_placement_manager.cc   |  9 +++++++-
 src/crimson/os/seastore/object_data_handler.h |  4 ++++
 src/crimson/os/seastore/transaction.h         |  9 ++++++++
 src/test/crimson/seastore/test_block.h        |  4 ++++
 6 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 4d1dc929607d..f0c7a0a34f24 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1261,6 +1261,24 @@ record_t Cache::prepare_record(
     }
   }
 
+  for (auto &i: t.written_inplace_ool_block_list) {
+    if (!i->is_valid()) {
+      continue;
+    }
+    assert(i->state == CachedExtent::extent_state_t::DIRTY);
+    assert(i->version > 0);
+    remove_from_dirty(i);
+    // set the version to zero because the extent state is now clean
+    // in order to handle this transparently
+    i->version = 0;
+    i->dirty_from_or_retired_at = JOURNAL_SEQ_MIN;
+    i->state = CachedExtent::extent_state_t::CLEAN;
+    assert(i->is_logical());
+    i->cast<LogicalCachedExtent>()->clear_delta();
+    touch_extent(*i);
+    DEBUGT("inplace rewrite ool block is commmitted -- {}", t, *i);
+  }
+
   for (auto &i: t.existing_block_list) {
     if (i->is_valid()) {
       alloc_delta.alloc_blk_ranges.emplace_back(
@@ -1330,7 +1348,8 @@ record_t Cache::prepare_record(
 	      t.num_allocated_invalid_extents);
 
   auto& ool_stats = t.get_ool_write_stats();
-  ceph_assert(ool_stats.extents.num == t.written_ool_block_list.size());
+  ceph_assert(ool_stats.extents.num == t.written_ool_block_list.size() +
+    t.written_inplace_ool_block_list.size());
 
   if (record.is_empty()) {
     SUBINFOT(seastore_t,
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 8bd4a2693851..18c102ed3e17 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1242,6 +1242,8 @@ class LogicalCachedExtent : public ChildableCachedExtent {
 
   void on_replace_prior(Transaction &t) final;
 
+  virtual void clear_delta() {}
+
   virtual ~LogicalCachedExtent();
 protected:
 
diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc
index b7aabefc6441..f53b9f5be916 100644
--- a/src/crimson/os/seastore/extent_placement_manager.cc
+++ b/src/crimson/os/seastore/extent_placement_manager.cc
@@ -799,7 +799,14 @@ RandomBlockOolWriter::do_write(
     ).safe_then([&t, &ex, paddr, FNAME]() {
       TRACET("ool extent written at {} -- {}",
 	     t, paddr, *ex);
-      t.mark_allocated_extent_ool(ex);
+      if (ex->is_initial_pending()) {
+	t.mark_allocated_extent_ool(ex);
+      } else if (ex->is_dirty()) {
+	assert(t.get_src() == transaction_type_t::TRIM_DIRTY);
+	t.mark_inplace_rewrite_extent_ool(ex);
+      } else {
+	ceph_assert("impossible");
+      }
       return alloc_write_iertr::now();
     });
   });
diff --git a/src/crimson/os/seastore/object_data_handler.h b/src/crimson/os/seastore/object_data_handler.h
index eaa05da8d545..76d69eb73f9b 100644
--- a/src/crimson/os/seastore/object_data_handler.h
+++ b/src/crimson/os/seastore/object_data_handler.h
@@ -60,6 +60,10 @@ struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
   ceph::bufferlist get_delta() final;
 
   void apply_delta(const ceph::bufferlist &bl) final;
+
+  void clear_delta() final {
+    delta.clear();
+  }
 };
 using ObjectDataBlockRef = TCachedExtentRef<ObjectDataBlock>;
 
diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h
index b43fb0d9cd8e..a283982f699e 100644
--- a/src/crimson/os/seastore/transaction.h
+++ b/src/crimson/os/seastore/transaction.h
@@ -217,6 +217,12 @@ class Transaction {
     written_ool_block_list.push_back(ref);
   }
 
+  void mark_inplace_rewrite_extent_ool(LogicalCachedExtentRef& ref) {
+    assert(ref->get_paddr().is_absolute());
+    assert(!ref->is_inline());
+    written_inplace_ool_block_list.push_back(ref);
+  }
+
   void add_inplace_rewrite_extent(CachedExtentRef ref) {
    ceph_assert(!is_weak());
    ceph_assert(ref);
@@ -400,6 +406,7 @@ class Transaction {
     delayed_alloc_list.clear();
     inline_block_list.clear();
     written_ool_block_list.clear();
+    written_inplace_ool_block_list.clear();
     pre_alloc_list.clear();
     pre_inplace_rewrite_list.clear();
     retired_set.clear();
@@ -555,6 +562,8 @@ class Transaction {
   std::list<CachedExtentRef> inline_block_list;
   /// fresh blocks that will be committed with out-of-line record
   std::list<CachedExtentRef> written_ool_block_list;
+  /// dirty blocks that will be committed out-of-line with inplace rewrite
+  std::list<LogicalCachedExtentRef> written_inplace_ool_block_list;
 
   /// list of mutated blocks, holds refcounts, subset of write_set
   std::list<CachedExtentRef> mutated_block_list;
diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h
index ccdafb7843fe..bfb50670420c 100644
--- a/src/test/crimson/seastore/test_block.h
+++ b/src/test/crimson/seastore/test_block.h
@@ -79,6 +79,10 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
   }
 
   void apply_delta(const ceph::bufferlist &bl) final;
+
+  void clear_delta() final {
+    delta.clear();
+  }
 };
 using TestBlockRef = TCachedExtentRef<TestBlock>;
 

From 9d86be6af2e29ad8815c59ee4b86b5fc832f8200 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Thu, 16 Nov 2023 21:10:55 +0900
Subject: [PATCH 1149/2492] crimson/os/seastore: replay inplace-rewritten
 extents correctly at mount time

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index f0c7a0a34f24..8645a3071062 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1824,11 +1824,23 @@ Cache::replay_delta(
       DEBUG("replay extent delta at {} {} ... -- {}, prv_extent={}",
             journal_seq, record_base, delta, *extent);
 
-      assert(extent->last_committed_crc == delta.prev_crc);
-      assert(extent->version == delta.pversion);
-      extent->apply_delta_and_adjust_crc(record_base, delta.bl);
-      extent->set_modify_time(modify_time);
-      assert(extent->last_committed_crc == delta.final_crc);
+      if (delta.paddr.get_addr_type() == paddr_types_t::SEGMENT ||
+	  !can_inplace_rewrite(delta.type)) {
+	assert(extent->last_committed_crc == delta.prev_crc);
+	assert(extent->version == delta.pversion);
+	extent->apply_delta_and_adjust_crc(record_base, delta.bl);
+	extent->set_modify_time(modify_time);
+	assert(extent->last_committed_crc == delta.final_crc);
+      } else {
+	assert(delta.paddr.get_addr_type() == paddr_types_t::RANDOM_BLOCK);
+	extent->apply_delta_and_adjust_crc(record_base, delta.bl);
+	extent->set_modify_time(modify_time);
+	// Since rewrite_dirty can conflict with other transaction after 
+	// inplace rewrite is complete, crc may not be matched
+	if (delta.final_crc == extent->last_committed_crc) {
+	  assert(extent->version == delta.pversion);
+	}
+      }
 
       extent->version++;
       if (extent->version == 1) {

From c31b795210d104c27c10d21384cd0b2fdbd3aa89 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 23 Dec 2023 06:40:06 -0600
Subject: [PATCH 1150/2492] osd/scrub: downgrade one periodic log message to
 l.25

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 9dd35d61871c..253962f0fd49 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -2416,7 +2416,7 @@ void PgScrubber::update_scrub_stats(ceph::coarse_real_clock::time_point now_is)
   /// \todo use the date library (either the one included in Arrow or directly)
   /// to get the formatting of the time_points.
 
-  if (g_conf()->subsys.should_gather<ceph_subsys_osd, 20>()) {
+  if (g_conf()->subsys.should_gather<ceph_subsys_osd, 25>()) {
     // will only create the debug strings if required
     char buf[50];
     auto printable_last = fmt::localtime(clock::to_time_t(m_last_stat_upd));

From bde52ebef19bc0d3fd3382926df6e202fbb79a53 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 21 Dec 2023 11:29:09 -0600
Subject: [PATCH 1151/2492] osd/scrub: add a "clean primary" base state

... to the scrubber state machine.

Similar to ReplicaActive, this state is entered after the peering
is concluded and the PG is set to be Primary, active & clean.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                     |  10 +++
 src/osd/PG.h                      |   8 +-
 src/osd/scrubber/pg_scrubber.cc   |  17 +++-
 src/osd/scrubber/pg_scrubber.h    |   2 +
 src/osd/scrubber/scrub_machine.cc |  64 +++++++++++---
 src/osd/scrubber/scrub_machine.h  | 136 ++++++++++++++++++++++++------
 src/osd/scrubber_common.h         |   4 +
 7 files changed, 193 insertions(+), 48 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 8420fd432a3f..3138b8c32f9c 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1861,6 +1861,16 @@ void PG::on_active_exit()
   agent_stop();
 }
 
+Context* PG::on_clean()
+{
+  if (is_active()) {
+    kick_snap_trim();
+  }
+  m_scrubber->on_primary_active_clean();
+  requeue_ops(waiting_for_clean_to_primary_repair);
+  return finish_recovery();
+}
+
 void PG::on_active_advmap(const OSDMapRef &osdmap)
 {
   const auto& new_removed_snaps = osdmap->get_new_removed_snaps();
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 9272000eb34d..2a823f675969 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -608,13 +608,7 @@ class PG : public DoutPrefixProvider,
 
   void on_active_exit() override;
 
-  Context *on_clean() override {
-    if (is_active()) {
-      kick_snap_trim();
-    }
-    requeue_ops(waiting_for_clean_to_primary_repair);
-    return finish_recovery();
-  }
+  Context *on_clean() override;
 
   void on_activate(interval_set<snapid_t> snaps) override;
 
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 253962f0fd49..dd279f720add 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -152,7 +152,7 @@ bool PgScrubber::verify_against_abort(epoch_t epoch_to_verify)
 
   // if we were not aware of the abort before - kill the scrub.
   if (epoch_to_verify >= m_last_aborted) {
-    scrub_clear_state();
+    m_fsm->process_event(FullReset{});
     m_last_aborted = std::max(epoch_to_verify, m_epoch_start);
   }
   return false;
@@ -407,7 +407,7 @@ void PgScrubber::reset_epoch(epoch_t epoch_queued)
 {
   dout(10) << __func__ << " state deep? " << state_test(PG_STATE_DEEP_SCRUB)
 	   << dendl;
-  m_fsm->assert_not_active();
+  m_fsm->assert_not_in_session();
 
   m_epoch_start = epoch_queued;
   ceph_assert(m_is_deep == state_test(PG_STATE_DEEP_SCRUB));
@@ -462,6 +462,11 @@ void PgScrubber::on_new_interval()
 	   << dendl;
 
   m_fsm->process_event(IntervalChanged{});
+  // the following asserts were added due to a past bug, where PG flags were
+  // left set in some scenarios.
+  ceph_assert(!is_queued_or_active());
+  ceph_assert(!state_test(PG_STATE_SCRUBBING));
+  ceph_assert(!state_test(PG_STATE_DEEP_SCRUB));
   rm_from_osd_scrubbing();
 }
 
@@ -517,6 +522,14 @@ void PgScrubber::on_pg_activate(const requested_scrub_t& request_flags)
 	   << dendl;
 }
 
+void PgScrubber::on_primary_active_clean()
+{
+  dout(10) << fmt::format(
+		  "{}: reg state: {}", __func__, m_scrub_job->state_desc())
+	   << dendl;
+  m_fsm->process_event(PrimaryActivate{});
+}
+
 /*
  * A note re the call to publish_stats_to_osd() below:
  * - we are called from either request_rescrubbing() or scrub_requested().
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index c6be51b2cae6..af667f732143 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -314,6 +314,8 @@ class PgScrubber : public ScrubPgIF,
 
   void on_new_interval() final;
 
+  void on_primary_active_clean() final;
+
   void on_replica_activate() final;
 
   void scrub_clear_state() final;
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index e0da61dde0f8..1928eed7d0cd 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -47,9 +47,9 @@ void on_event_discard(std::string_view nm)
   dout(20) << " event: --^^^^---- " << nm << dendl;
 }
 
-void ScrubMachine::assert_not_active() const
+void ScrubMachine::assert_not_in_session() const
 {
-  ceph_assert(state_cast<const NotActive*>());
+  ceph_assert(!state_cast<const Session*>());
 }
 
 bool ScrubMachine::is_reserving() const
@@ -114,27 +114,64 @@ NotActive::NotActive(my_context ctx)
   scrbr->clear_queued_or_active();
 }
 
-sc::result NotActive::react(const StartScrub&)
+
+// ----------------------- PrimaryActive --------------------------------
+
+PrimaryActive::PrimaryActive(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "PrimaryActive")
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "-- state -->> PrimaryActive" << dendl;
+}
+
+PrimaryActive::~PrimaryActive()
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  // we may have set some PG state flags without reaching Session.
+  // And we may be holding a 'local resource'.
+  scrbr->clear_pgscrub_state();
+  scrbr->rm_from_osd_scrubbing();
+}
+
+
+// ---------------- PrimaryActive/PrimaryIdle ---------------------------
+
+PrimaryIdle::PrimaryIdle(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "PrimaryActive/PrimaryIdle")
+{
+  dout(10) << "-- state -->> PrimaryActive/PrimaryIdle" << dendl;
+}
+
+sc::result PrimaryIdle::react(const StartScrub&)
 {
-  dout(10) << "NotActive::react(const StartScrub&)" << dendl;
+  dout(10) << "PrimaryIdle::react(const StartScrub&)" << dendl;
   DECLARE_LOCALS;
   return transit<ReservingReplicas>();
 }
 
-sc::result NotActive::react(const AfterRepairScrub&)
+sc::result PrimaryIdle::react(const AfterRepairScrub&)
 {
-  dout(10) << "NotActive::react(const AfterRepairScrub&)" << dendl;
+  dout(10) << "PrimaryIdle::react(const AfterRepairScrub&)" << dendl;
   DECLARE_LOCALS;
   return transit<ReservingReplicas>();
 }
 
+void PrimaryIdle::clear_state(const FullReset&) {
+  dout(10) << "PrimaryIdle::react(const FullReset&): clearing state flags"
+           << dendl;
+  DECLARE_LOCALS;
+  scrbr->clear_pgscrub_state();
+}
+
 // ----------------------- Session -----------------------------------------
 
 Session::Session(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "Session")
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "PrimaryActive/Session")
 {
-  dout(10) << "-- state -->> Session" << dendl;
+  dout(10) << "-- state -->> PrimaryActive/Session" << dendl;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   // while we've checked the 'someone is reserving' flag before queueing
@@ -242,7 +279,7 @@ sc::result ReservingReplicas::react(const ReplicaReject& ev)
   scrbr->flag_reservations_failure();
 
   // 'Session' state dtor stops the scrubber
-  return transit<NotActive>();
+  return transit<PrimaryIdle>();
 }
 
 sc::result ReservingReplicas::react(const ReservationTimeout&)
@@ -261,7 +298,7 @@ sc::result ReservingReplicas::react(const ReservationTimeout&)
   // cause the scrubber to stop the scrub session, marking 'reservation
   // failure' as the cause (affecting future scheduling)
   scrbr->flag_reservations_failure();
-  return transit<NotActive>();
+  return transit<PrimaryIdle>();
 }
 
 // ----------------------- ActiveScrubbing -----------------------------------
@@ -678,7 +715,7 @@ sc::result WaitDigestUpdate::react(const ScrubFinished&)
   session.m_session_started_at = ScrubTimePoint{};
 
   scrbr->scrub_finish();
-  return transit<NotActive>();
+  return transit<PrimaryIdle>();
 }
 
 ScrubMachine::ScrubMachine(PG* pg, ScrubMachineListener* pg_scrub)
@@ -813,6 +850,11 @@ ReplicaIdle::ReplicaIdle(my_context ctx)
   dout(10) << "-- state -->> ReplicaActive/ReplicaIdle" << dendl;
 }
 
+void ReplicaIdle::reset_ignored(const FullReset&)
+{
+  dout(10) << "ReplicaIdle::react(const FullReset&): FullReset ignored"
+	   << dendl;
+}
 
 // ------------- ReplicaActive/ReplicaActiveOp --------------------------
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index df9a4c45e95f..fbd8f8b3a359 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -162,6 +162,9 @@ MEV(IntLocalMapDone)
 /// scrub_snapshot_metadata()
 MEV(DigestUpdate)
 
+/// peered as Primary - and clean
+MEV(PrimaryActivate)
+
 /// we are a replica for this PG
 MEV(ReplicaActivate)
 
@@ -176,7 +179,9 @@ MEV(ReplicaPushesUpd)
 
 /**
  * IntervalChanged
+ * The only path from PrimaryActive or ReplicaActive down to NotActive.
  *
+ * Note re reserved replicas:
  * This event notifies the ScrubMachine that it is no longer responsible for
  * releasing replica state.  It will generally be submitted upon a PG interval
  * change.
@@ -189,7 +194,12 @@ MEV(ReplicaPushesUpd)
  */
 MEV(IntervalChanged)
 
-/// guarantee that the FSM is in the quiescent state (i.e. NotActive)
+/**
+ * stops the scrubbing session, and resets the scrubber.
+ * For a replica - aborts the handling of the current request.
+ * In both cases - a transition to the peering mode quiescent state (i.e.
+ * PrimaryIdle or ReplicaIdle).
+ */
 MEV(FullReset)
 
 /// finished handling this chunk. Go get the next one
@@ -203,9 +213,15 @@ MEV(ScrubFinished)
 //
 
 struct NotActive;	    ///< the quiescent state. No active scrubbing.
-struct Session;            ///< either reserving or actively scrubbing
 struct ReservingReplicas;   ///< securing scrub resources from replicas' OSDs
 struct ActiveScrubbing;	    ///< the active state for a Primary. A sub-machine.
+
+// the states for a Primary:
+// note: PrimaryActive <==> in the OSD scrub queue
+struct PrimaryActive;	   ///< base state for a Primary
+struct PrimaryIdle;	   ///< ready for a new scrub request
+struct Session;            ///< either reserving or actively scrubbing
+
 // the active states for a replica:
 struct ReplicaActive;    ///< the quiescent state for a replica
 struct ReplicaActiveOp;
@@ -225,7 +241,7 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
   ScrubMachineListener* m_scrbr;
   std::ostream& gen_prefix(std::ostream& out) const;
 
-  void assert_not_active() const;
+  void assert_not_in_session() const;
   [[nodiscard]] bool is_reserving() const;
   [[nodiscard]] bool is_accepting_updates() const;
 
@@ -359,40 +375,94 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
 
 // ///////////////// the states //////////////////////// //
 
+/*
+ * When not scrubbing, the FSM is in one of three states:
+ *
+ * <> PrimaryActive - we are a Primary and active. The PG
+ * is queued for some future scrubs in the OSD's scrub queue.
+ *
+ * <> ReplicaActive - we are a replica. In this state, we are
+ * expecting either a replica reservation request from the Primary, or a
+ * scrubbing request for a specific chunk.
+ *
+ * <> NotActive - the quiescent state. No active scrubbing.
+ * We are neither an active Primary nor a replica.
+ */
+struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
+  explicit NotActive(my_context ctx);
+
+  using reactions = mpl::list<
+      // peering done, and we are a replica
+      sc::transition<ReplicaActivate, ReplicaActive>,
+      // peering done, and we are a Primary
+      sc::transition<PrimaryActivate, PrimaryActive>>;
+};
+
+// ----------------------- when Primary --------------------------------------
+// ---------------------------------------------------------------------------
+
+
+/*
+ *  The primary states:
+ *
+ *  PrimaryActive - starts when peering ends with us as a primary,
+ *     and we are active and clean.
+ *   - when in this state - we (our scrub targets) are queued in the
+ *     OSD's scrub queue.
+ *
+ *  Sub-states:
+ *     - PrimaryIdle - ready for a new scrub request
+ *          * initial state of PrimaryActive
+ *
+ *     - Session - handling a single scrub session
+ */
+
+struct PrimaryIdle;
 
 /**
- *  The Scrubber's base (quiescent) state.
- *  Scrubbing is triggered by one of the following events:
+ *  PrimaryActive
+ *
+ *  The basic state for an active Primary. Ready to accept a new scrub request.
+ *  State managed here: being in the OSD's scrub queue (unless when scrubbing).
  *
+ *  Scrubbing is triggered by one of the following events:
  *  - (standard scenario for a Primary): 'StartScrub'. Initiates the OSDs
  *    resources reservation process. Will be issued by PG::scrub(), following a
  *    queued "PGScrub" op.
- *
  *  - a special end-of-recovery Primary scrub event ('AfterRepairScrub').
- *
- *  - (if already in ReplicaActive): an incoming MOSDRepScrub triggers
- *    'StartReplica'.
- *
- *  note (20.8.21): originally, AfterRepairScrub was triggering a scrub without
- *  waiting for replica resources to be acquired. But once replicas started
- *  using the resource-request to identify and tag the scrub session, this
- *  bypass cannot be supported anymore.
  */
-struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
-  explicit NotActive(my_context ctx);
+struct PrimaryActive : sc::state<PrimaryActive, ScrubMachine, PrimaryIdle>,
+			 NamedSimply {
+  explicit PrimaryActive(my_context ctx);
+  ~PrimaryActive();
+
+  using reactions = mpl::list<
+      // when the interval ends - we may not be a primary anymore
+      sc::transition<IntervalChanged, NotActive>>;
+};
+
+/**
+ * \ATTN: set_op_parameters() is called while we are still in this state (waiting
+ * for a queued OSD message to trigger the transition into Session). Thus,
+ * even in this 'idle' state - there is some state we must take care to reset.
+ * Specifically - the PG state flags we were playing with in set_op_parameters().
+ */
+struct PrimaryIdle : sc::state<PrimaryIdle, PrimaryActive>, NamedSimply {
+  explicit PrimaryIdle(my_context ctx);
+  ~PrimaryIdle() = default;
+  void clear_state(const FullReset&);
 
   using reactions = mpl::list<
       sc::custom_reaction<StartScrub>,
       // a scrubbing that was initiated at recovery completion:
       sc::custom_reaction<AfterRepairScrub>,
-      // peering done, and we are a replica
-      sc::transition<ReplicaActivate, ReplicaActive>>;
+      // undoing set_op_params(), if aborted before starting the scrub:
+      sc::in_state_reaction<FullReset, PrimaryIdle, &PrimaryIdle::clear_state>>;
 
   sc::result react(const StartScrub&);
   sc::result react(const AfterRepairScrub&);
 };
 
-
 /**
  *  Session
  *
@@ -407,12 +477,12 @@ struct NotActive : sc::state<NotActive, ScrubMachine>, NamedSimply {
  *  reservations are released. This is because we know that the replicas are
  *  also resetting their reservations.
  */
-struct Session : sc::state<Session, ScrubMachine, ReservingReplicas>,
+struct Session : sc::state<Session, PrimaryActive, ReservingReplicas>,
                  NamedSimply {
   explicit Session(my_context ctx);
   ~Session();
 
-  using reactions = mpl::list<sc::transition<FullReset, NotActive>,
+  using reactions = mpl::list<sc::transition<FullReset, PrimaryIdle>,
                               sc::custom_reaction<IntervalChanged>>;
 
   sc::result react(const IntervalChanged&);
@@ -612,7 +682,9 @@ struct WaitDigestUpdate : sc::state<WaitDigestUpdate, ActiveScrubbing>,
   sc::result react(const ScrubFinished&);
 };
 
-// ----------------------------- the "replica active" states
+
+// ---------------------------------------------------------------------------
+// ----------------------------- the "replica active" states -----------------
 
 /*
  *  The replica states:
@@ -683,15 +755,21 @@ struct ReplicaActive : sc::state<ReplicaActive, ScrubMachine, ReplicaIdle>,
 struct ReplicaIdle : sc::state<ReplicaIdle, ReplicaActive>, NamedSimply {
   explicit ReplicaIdle(my_context ctx);
   ~ReplicaIdle() = default;
+  void reset_ignored(const FullReset&);
 
   // note the execution of check_for_updates() when transitioning to
   // ReplicaActiveOp/ReplicaWaitUpdates. That would trigger a ReplicaPushesUpd
   // event, which will be handled by ReplicaWaitUpdates.
-  using reactions = mpl::list<sc::transition<
-      StartReplica,
-      ReplicaWaitUpdates,
-      ReplicaActive,
-      &ReplicaActive::check_for_updates>>;
+  using reactions = mpl::list<
+      sc::transition<
+	  StartReplica,
+	  ReplicaWaitUpdates,
+	  ReplicaActive,
+	  &ReplicaActive::check_for_updates>,
+      sc::in_state_reaction<
+	  FullReset,
+	  ReplicaIdle,
+	  &ReplicaIdle::reset_ignored>>;
 };
 
 
@@ -706,7 +784,9 @@ struct ReplicaActiveOp
   explicit ReplicaActiveOp(my_context ctx);
   ~ReplicaActiveOp();
 
-  using reactions = mpl::list<sc::custom_reaction<StartReplica>>;
+  using reactions = mpl::list<
+      sc::custom_reaction<StartReplica>,
+      sc::transition<FullReset, ReplicaIdle>>;
 
   /**
    * Handling the unexpected (read - caused by a bug) case of receiving a
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 30d305036f44..fbbef578ae69 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -319,6 +319,10 @@ struct ScrubPgIF {
   /// the OSD scrub queue
   virtual void on_new_interval() = 0;
 
+  /// we are peered as primary, and the PG is active and clean
+  /// Scrubber's internal FSM should be ActivePrimary
+  virtual void on_primary_active_clean() = 0;
+
   /// we are peered as a replica
   virtual void on_replica_activate() = 0;
 

From 8631dc58f7144b32febadd9072c6691456ac8f28 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 29 Dec 2023 18:32:04 +1000
Subject: [PATCH 1152/2492] doc/radosgw: edit sections

Edit the following sections in doc/radosgw/admin.rst:

* Remove a User
* Remove a Subuser

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 8f25add73dfb..f0904a9ee362 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -197,41 +197,45 @@ in a command of the following form:
 Remove a User
 -------------
 
-When you remove a user, the user and subuser are removed from the system.
-However, you may remove just the subuser if you wish. To remove a user (and
-subuser), specify ``user rm`` and the user ID. ::
+The act of removing a user removes both the user and any subusers associated
+with the user from the system. Remember that it is possible to remove a subuser
+without removing the user with which it is associated.   
 
-	radosgw-admin user rm --uid=johndoe
+To remove a user and any subusers associated with it, use the ``user rm``
+command and provide the user ID of the user to be removed. Use a command of the
+following form: 
 
-To remove the subuser only, specify ``subuser rm`` and the subuser ID. ::
-
-	radosgw-admin subuser rm --subuser=johndoe:swift
+.. prompt:: bash
 
+   radosgw-admin user rm --uid=johndoe
 
 Options include:
 
 - **Purge Data:** The ``--purge-data`` option purges all data associated 
-  to the UID.
+  with the UID.
   
 - **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
-  to the UID.
+  with the UID.
 
 
 Remove a Subuser
 ----------------
 
-When you remove a sub user, you are removing access to the Swift interface. 
-The user will remain in the system. To remove the subuser, specify 
-``subuser rm`` and the subuser ID. ::
+Removing a subuser removes access to the Swift interface or to S3. The user
+associated with the removed subuser remains in the system after the subuser's
+removal. 
 
-	radosgw-admin subuser rm --subuser=johndoe:swift
+To remove the subuser, use the command ``subuser rm`` and provide the subuser
+ID of the subuser to be removed. Use a command of the following form: 
 
+.. prompt:: bash
 
+   radosgw-admin subuser rm --subuser=johndoe:swift
 
 Options include:
   
 - **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
-  to the UID.
+  with the UID.
 
 
 Add / Remove a Key

From 8fc3c488861887490f6ec6bac8f49770664b49a6 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 31 Dec 2023 16:22:33 +1000
Subject: [PATCH 1153/2492] doc/radosgw: edit "remove a subuser"

Edit the English language in the section "Remove a Subuser" in
doc/radosgw/admin.rst. This commit is made in response to Matt
Benjamin's request for improvement of this section
(https://github.com/ceph/ceph/pull/55028#discussion_r1438599833).

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index f0904a9ee362..f108d58d7120 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -197,9 +197,10 @@ in a command of the following form:
 Remove a User
 -------------
 
-The act of removing a user removes both the user and any subusers associated
-with the user from the system. Remember that it is possible to remove a subuser
-without removing the user with which it is associated.   
+When you remove a user, you also remove any subusers associated with the user.
+
+It is possible to remove a subuser without removing its associated user. This
+is covered in the section called :ref:`Remove a Subuser <radosgw-admin-remove-a-subuser>`.
 
 To remove a user and any subusers associated with it, use the ``user rm``
 command and provide the user ID of the user to be removed. Use a command of the
@@ -217,6 +218,7 @@ Options include:
 - **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
   with the UID.
 
+.. _radosgw-admin-remove-a-subuser:
 
 Remove a Subuser
 ----------------

From 03656c02a70d4ab719cbcf605517f8a2e9e81117 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 1 Jan 2024 10:10:42 +1000
Subject: [PATCH 1154/2492] doc/mgr: document lack of MSWin NFS 4.x support

Document that no version of Microsoft Windows supports mounting an NFS
v4.x export natively, as of the time of this commit.

This commit has been made in response to an anonymous note left on
https://pad.ceph.com/p/Report_Documentation_Bugs.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/mgr/nfs.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst
index 7e6637684faa..746ab4247f39 100644
--- a/doc/mgr/nfs.rst
+++ b/doc/mgr/nfs.rst
@@ -568,6 +568,9 @@ If the NFS service is running on a non-standard port number:
 
 .. note:: Only NFS v4.0+ is supported.
 
+.. note:: As of this writing (01 Jan 2024), no version of Microsoft Windows
+   supports mouting an NFS v4.x export natively.
+
 Troubleshooting
 ===============
 

From 3a8bff857e40256b6b3a7fd933265cfe409a46bc Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 1 Jan 2024 09:11:41 +0200
Subject: [PATCH 1155/2492] test/crush: removed unused code

to fix a compiler warning re unused variable.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/crush/crush.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 2d87958b383c..1f53084a70af 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -219,11 +219,6 @@ TEST_F(CRUSHTest, indep_out_progressive) {
       c->do_rule(0, x, out, 7, weight, 0);
       cout << "(" << i << "/" << weight.size() << " out) "
 	   << x << " -> " << out << std::endl;
-      int num_none = 0;
-      for (unsigned k=0; k<out.size(); ++k) {
-	if (out[k] == CRUSH_ITEM_NONE)
-	  num_none++;
-      }
       ASSERT_EQ(0, get_num_dups(out));
 
       // make sure nothing moved

From 1f2a5e38cf85446f508a4c2a5d15a8366b7f51e1 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@ibm.com>
Date: Thu, 28 Dec 2023 18:53:14 +0000
Subject: [PATCH 1156/2492] rgw/kafka: make the connection idle and sleep
 timeouts and configurable

Fixes: https://tracker.ceph.com/issues/63901

Signed-off-by: Yuval Lifshitz <ylifshit@ibm.com>
---
 src/common/options/rgw.yaml.in | 24 ++++++++++++++++++++++++
 src/rgw/rgw_kafka.cc           | 18 +++++++-----------
 2 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 3f0f691e84e7..a37079775225 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3965,3 +3965,27 @@ options:
   see_also:
   - rgw_bucket_counters_cache
   with_legacy: true
+- name: rgw_kafka_connection_idle
+  type: uint 
+  level: advanced
+  desc: Time in seconds to delete idle kafka connections
+  long_desc: A conection will be considered "idle" if no messages
+    are sent to it for more than the time defined.
+    Note that the connection will not be considered idle, even if it is down,
+    as long as there are attempts to send messages to it.
+  default: 30
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_kafka_sleep_timeout
+  type: uint 
+  level: advanced
+  desc: Time in milliseconds to sleep while polling for kafka replies
+  long_desc: This will be used to prevent busy waiting for the kafka replies
+    As well as for the cases where the broker is down and we try to reconnect.
+    The same values times 3 will be used to sleep if there were no messages
+    sent or received across all kafka connections
+  default: 10
+  services:
+  - rgw
+  with_legacy: true
diff --git a/src/rgw/rgw_kafka.cc b/src/rgw/rgw_kafka.cc
index 4593a4de67b0..0d6e77375014 100644
--- a/src/rgw/rgw_kafka.cc
+++ b/src/rgw/rgw_kafka.cc
@@ -326,7 +326,6 @@ class Manager {
   const size_t max_connections;
   const size_t max_inflight;
   const size_t max_queue;
-  const size_t max_idle_time;
 private:
   std::atomic<size_t> connection_count;
   bool stopped;
@@ -457,13 +456,15 @@ class Manager {
         conn_it = connections.begin();
         end_it = connections.end();
       }
+
+      const auto read_timeout = cct->_conf->rgw_kafka_sleep_timeout;
       // loop over all connections to read acks
       for (;conn_it != end_it;) {
         
         auto& conn = conn_it->second;
 
         // Checking the connection idleness
-        if(conn->timestamp.sec() + max_idle_time < ceph_clock_now()) {
+        if(conn->timestamp.sec() + conn->cct->_conf->rgw_kafka_connection_idle < ceph_clock_now()) {
           ldout(conn->cct, 20) << "kafka run: deleting a connection due to idle behaviour: " << ceph_clock_now() << dendl;
           std::lock_guard lock(connections_lock);
           conn->status = STATUS_CONNECTION_IDLE;
@@ -488,15 +489,14 @@ class Manager {
           continue;
         }
 
-        reply_count += rd_kafka_poll(conn->producer, read_timeout_ms);
+        reply_count += rd_kafka_poll(conn->producer, read_timeout);
 
         // just increment the iterator
         ++conn_it;
       }
-      // if no messages were received or published
-      // across all connection, sleep for 100ms
+      // sleep if no messages were received or published across all connection
       if (send_count == 0 && reply_count == 0) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        std::this_thread::sleep_for(std::chrono::milliseconds(read_timeout*3));
       }
     }
   }
@@ -510,15 +510,12 @@ class Manager {
   Manager(size_t _max_connections,
       size_t _max_inflight,
       size_t _max_queue, 
-      int _read_timeout_ms,
       CephContext* _cct) : 
     max_connections(_max_connections),
     max_inflight(_max_inflight),
     max_queue(_max_queue),
-    max_idle_time(30),
     connection_count(0),
     stopped(false),
-    read_timeout_ms(_read_timeout_ms),
     connections(_max_connections),
     messages(max_queue),
     queued(0),
@@ -673,14 +670,13 @@ static Manager* s_manager = nullptr;
 static const size_t MAX_CONNECTIONS_DEFAULT = 256;
 static const size_t MAX_INFLIGHT_DEFAULT = 8192; 
 static const size_t MAX_QUEUE_DEFAULT = 8192;
-static const int READ_TIMEOUT_MS_DEFAULT = 500;
 
 bool init(CephContext* cct) {
   if (s_manager) {
     return false;
   }
   // TODO: take conf from CephContext
-  s_manager = new Manager(MAX_CONNECTIONS_DEFAULT, MAX_INFLIGHT_DEFAULT, MAX_QUEUE_DEFAULT, READ_TIMEOUT_MS_DEFAULT, cct);
+  s_manager = new Manager(MAX_CONNECTIONS_DEFAULT, MAX_INFLIGHT_DEFAULT, MAX_QUEUE_DEFAULT, cct);
   return true;
 }
 

From 6601e7f38c0b2c1326d40f3bb0599069e38421ff Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 1 Jan 2024 17:13:12 +0530
Subject: [PATCH 1157/2492] doc/cephfs: Update disaster-recovery-experts.rst to
 mention Slack

Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 doc/cephfs/disaster-recovery-experts.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephfs/disaster-recovery-experts.rst b/doc/cephfs/disaster-recovery-experts.rst
index c881c24239b6..9a196c88e234 100644
--- a/doc/cephfs/disaster-recovery-experts.rst
+++ b/doc/cephfs/disaster-recovery-experts.rst
@@ -15,7 +15,7 @@ Advanced: Metadata repair tools
     file system before attempting to repair it.
 
     If you do not have access to professional support for your cluster,
-    consult the ceph-users mailing list or the #ceph IRC channel.
+    consult the ceph-users mailing list or the #ceph IRC/Slack channel.
 
 
 Journal export

From 46eba3333a2a89335d3cf507ab7244bb72243c40 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 11 Oct 2023 06:10:18 -0400
Subject: [PATCH 1158/2492] qa: run scrub before mounting client and
 validations

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_data_scan.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index 63ac60415475..d99dc7780c74 100644
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -435,13 +435,13 @@ def get_state(mds_id):
         self.fs.wait_for_daemons()
         log.info(str(self.mds_cluster.status()))
 
-        # Mount a client
-        self.mount_a.mount_wait()
-
         # run scrub as it is recommended post recovery for most
         # (if not all) recovery mechanisms.
         workload.scrub()
 
+        # Mount a client
+        self.mount_a.mount_wait()
+
         # See that the files are present and correct
         errors = workload.validate()
         if errors:

From 79ba10e807f07c606338655d6b79d5f670850951 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 11 Oct 2023 06:31:39 -0400
Subject: [PATCH 1159/2492] qa: add test to mangle lost+found directory object
 and ensure safety

Earlier ceph versions didn't allow the lost+found directory to be removed
and nor the entries inside it. Users are recommended to fail the filesystem
and remove the directory object using rados cli commands. Therefore, include
this step as part of our testing.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_data_scan.py | 32 +++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index d99dc7780c74..acc4fdc6f364 100644
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -101,6 +101,16 @@ def scrub(self):
         self.assert_equal(out_json["return_code"], 0)
         self.assert_equal(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
 
+    def mangle(self):
+        """
+        Gives an opportunity to fiddle with metadata objects before bringing back
+        the MDSs online. This is used in testing the lost+found case (when recovering
+        a file without a backtrace) to verify if lost+found directory object can be
+        removed via RADOS operation and the file system can be continued to be used
+        as expected.
+        """
+        pass
+
 class SimpleWorkload(Workload):
     """
     Single file, single directory, check that it gets recovered and so does its size
@@ -190,6 +200,23 @@ def validate(self):
 
         return self._errors
 
+class BacktracelessFileRemoveLostAndFoundDirectory(Workload):
+    def write(self):
+        self._mount.run_shell(["mkdir", "subdir"])
+        self._mount.write_n_mb("subdir/sixmegs", 6)
+        self._initial_state = self._mount.stat("subdir/sixmegs")
+
+    def flush(self):
+        # Never flush metadata, so backtrace won't be written
+        pass
+
+    def mangle(self):
+        self._filesystem.rados(["-p", self._filesystem.get_metadata_pool_name(), "rm", "4.00000000"])
+        self._filesystem.rados(["-p", self._filesystem.get_metadata_pool_name(), "rmomapkey", "1.00000000", "lost+found_head"])
+
+    def validate(self):
+        # The dir should be gone since we manually removed it
+        self.assert_not_equal(self._mount.ls(sudo=True), ["lost+found"])
 
 class StripedStashedLayout(Workload):
     def __init__(self, fs, m, pool=None):
@@ -427,6 +454,8 @@ def get_state(mds_id):
         self.fs.data_scan(["scan_inodes"], worker_count=workers)
         self.fs.data_scan(["scan_links"])
 
+        workload.mangle()
+
         # Mark the MDS repaired
         self.run_ceph_cmd('mds', 'repaired', '0')
 
@@ -465,6 +494,9 @@ def test_rebuild_moved_file(self):
     def test_rebuild_backtraceless(self):
         self._rebuild_metadata(BacktracelessFile(self.fs, self.mount_a))
 
+    def test_rebuild_backtraceless_with_lf_dir_removed(self):
+        self._rebuild_metadata(BacktracelessFileRemoveLostAndFoundDirectory(self.fs, self.mount_a))
+
     def test_rebuild_moved_dir(self):
         self._rebuild_metadata(MovedDir(self.fs, self.mount_a))
 

From 69c57c3b55b7e8d5e1870b0e045a6815126934e7 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 2 Jan 2024 22:21:08 +1000
Subject: [PATCH 1160/2492] doc/rados: improve subsection title in
 log-and-debug.rst

Change the title of "Logging Settings" to "Logging and Debugging
Settings" in doc/rados/troubleshooting/log-and-debug.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/troubleshooting/log-and-debug.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
index 7d2885522132..f6462a8598ce 100644
--- a/doc/rados/troubleshooting/log-and-debug.rst
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -380,8 +380,8 @@ to their default level or to a level suitable for normal operations.
 +--------------------------+-----------+--------------+
 
 
-Logging Settings
-----------------
+Logging and Debugging Settings
+------------------------------
 
 It is not necessary to specify logging and debugging settings in the Ceph
 configuration file, but you may override default settings when needed. Ceph

From 14d2f0a108b0b8a93b895ce518583a9584a752ac Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 21 Sep 2023 16:36:45 +0200
Subject: [PATCH 1161/2492] mgr/dashboard: chartjs and ng2-charts version
 pugrade

Fixes:https://tracker.ceph.com/issues/63160
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../mgr/dashboard/frontend/angular.json       |   3 -
 .../mgr/dashboard/frontend/package-lock.json  |  83 ++++------
 .../mgr/dashboard/frontend/package.json       |   5 +-
 .../rbd-configuration-list.component.spec.ts  |   4 +-
 .../cephfs-chart/cephfs-chart.component.html  |   2 +-
 .../cephfs-chart.component.spec.ts            |   8 +-
 .../cephfs-chart/cephfs-chart.component.ts    | 117 +++++++-------
 .../src/app/ceph/cephfs/cephfs.module.ts      |   4 +-
 .../osd/osd-list/osd-list.component.spec.ts   |   4 +
 .../dashboard-area-chart.component.html       |   2 +-
 .../dashboard-area-chart.component.ts         | 147 +++++++++---------
 .../dashboard-pie.component.html              |   3 +-
 .../dashboard-pie/dashboard-pie.component.ts  | 118 +++++++-------
 .../ceph/dashboard-v3/dashboard-v3.module.ts  |   4 +-
 .../app/ceph/dashboard/dashboard.module.ts    |   4 +-
 .../health-pie/health-pie.component.html      |   3 +-
 .../health-pie/health-pie.component.ts        |  76 +++++----
 .../ceph/dashboard/health/health.component.ts |   6 +-
 .../shared/components/components.module.ts    |   4 +-
 .../sparkline/sparkline.component.html        |   6 +-
 .../sparkline/sparkline.component.spec.ts     |  18 ++-
 .../sparkline/sparkline.component.ts          | 101 ++++++------
 .../src/app/shared/models/chart-tooltip.ts    |   5 +-
 23 files changed, 355 insertions(+), 372 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/angular.json b/src/pybind/mgr/dashboard/frontend/angular.json
index e1cb4c29fc32..9d26d5876b9f 100644
--- a/src/pybind/mgr/dashboard/frontend/angular.json
+++ b/src/pybind/mgr/dashboard/frontend/angular.json
@@ -99,9 +99,6 @@
               "node_modules/ngx-toastr/toastr.css",
               "src/styles.scss"
             ],
-            "scripts": [
-              "node_modules/chart.js/dist/Chart.bundle.js"
-            ],
             "stylePreprocessorOptions": {
               "includePaths": [
                 "src"
diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json
index 15aaff89db92..7ce746060f9a 100644
--- a/src/pybind/mgr/dashboard/frontend/package-lock.json
+++ b/src/pybind/mgr/dashboard/frontend/package-lock.json
@@ -28,7 +28,8 @@
         "@types/file-saver": "2.0.1",
         "async-mutex": "0.2.4",
         "bootstrap": "5.2.3",
-        "chart.js": "2.9.4",
+        "chart.js": "4.4.0",
+        "chartjs-adapter-moment": "1.0.1",
         "detect-browser": "5.2.0",
         "file-saver": "2.0.2",
         "fork-awesome": "1.1.7",
@@ -36,7 +37,7 @@
         "moment": "2.29.4",
         "ng-block-ui": "3.0.2",
         "ng-click-outside": "7.0.0",
-        "ng2-charts": "2.4.2",
+        "ng2-charts": "4.1.1",
         "ngx-pipe-function": "1.0.0",
         "ngx-toastr": "17.0.2",
         "rxjs": "6.6.3",
@@ -4950,6 +4951,11 @@
       "resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.4.0.tgz",
       "integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA=="
     },
+    "node_modules/@kurkle/color": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz",
+      "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw=="
+    },
     "node_modules/@leichtgewicht/ip-codec": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz",
@@ -6568,14 +6574,6 @@
       "integrity": "sha512-SaU/Kgp6z40CiF9JxlsrSrBEa+8YIry9IiCPhhYSNekeEhIAkY7iyu9aZ+5dSQIdo7mf86MUVvxWYm5GAzB/0g==",
       "dev": true
     },
-    "node_modules/@types/chart.js": {
-      "version": "2.9.38",
-      "resolved": "https://registry.npmjs.org/@types/chart.js/-/chart.js-2.9.38.tgz",
-      "integrity": "sha512-rLoHHprkVEDpAXqke/xHalyXR+5Nv+3tfViwT/UnJZ41Wp/XPaSRlJKw2PU3S3tTCqKKyjkYai+VpeHoti79XQ==",
-      "dependencies": {
-        "moment": "^2.10.2"
-      }
-    },
     "node_modules/@types/connect": {
       "version": "3.4.36",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz",
@@ -9504,44 +9502,25 @@
       }
     },
     "node_modules/chart.js": {
-      "version": "2.9.4",
-      "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz",
-      "integrity": "sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A==",
-      "dependencies": {
-        "chartjs-color": "^2.1.0",
-        "moment": "^2.10.2"
-      }
-    },
-    "node_modules/chartjs-color": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz",
-      "integrity": "sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==",
-      "dependencies": {
-        "chartjs-color-string": "^0.6.0",
-        "color-convert": "^1.9.3"
-      }
-    },
-    "node_modules/chartjs-color-string": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz",
-      "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.0.tgz",
+      "integrity": "sha512-vQEj6d+z0dcsKLlQvbKIMYFHd3t8W/7L2vfJIbYcfyPcRx92CsHqECpueN8qVGNlKyDcr5wBrYAYKnfu/9Q1hQ==",
       "dependencies": {
-        "color-name": "^1.0.0"
+        "@kurkle/color": "^0.3.0"
+      },
+      "engines": {
+        "pnpm": ">=7"
       }
     },
-    "node_modules/chartjs-color/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-      "dependencies": {
-        "color-name": "1.1.3"
+    "node_modules/chartjs-adapter-moment": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/chartjs-adapter-moment/-/chartjs-adapter-moment-1.0.1.tgz",
+      "integrity": "sha512-Uz+nTX/GxocuqXpGylxK19YG4R3OSVf8326D+HwSTsNw1LgzyIGRo+Qujwro1wy6X+soNSnfj5t2vZ+r6EaDmA==",
+      "peerDependencies": {
+        "chart.js": ">=3.0.0",
+        "moment": "^2.10.2"
       }
     },
-    "node_modules/chartjs-color/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
-    },
     "node_modules/check-error": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz",
@@ -21161,19 +21140,19 @@
       }
     },
     "node_modules/ng2-charts": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/ng2-charts/-/ng2-charts-2.4.2.tgz",
-      "integrity": "sha512-mY3C2uKCaApHCQizS2YxEOqQ7sSZZLxdV6N1uM9u/VvUgVtYvlPtdcXbKpN52ak93ZE22I73DiLWVDnDNG4/AQ==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/ng2-charts/-/ng2-charts-4.1.1.tgz",
+      "integrity": "sha512-iHwXDbmX86lfeH8VRcsaW2tJATsuAZo4kvvC/Yk2l35zOHjevja1qBvO6BAibiDazi9r9aS6ZRJOqWPsz1pP2w==",
       "dependencies": {
-        "@types/chart.js": "^2.9.24",
         "lodash-es": "^4.17.15",
-        "tslib": "^2.0.0"
+        "tslib": "^2.3.0"
       },
       "peerDependencies": {
-        "@angular/common": ">=7.2.0",
-        "@angular/core": ">=7.2.0",
-        "chart.js": "^2.9.3",
-        "rxjs": "^6.3.3"
+        "@angular/cdk": ">=14.0.0",
+        "@angular/common": ">=14.0.0",
+        "@angular/core": ">=14.0.0",
+        "chart.js": "^3.4.0 || ^4.0.0",
+        "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/ngx-pipe-function": {
diff --git a/src/pybind/mgr/dashboard/frontend/package.json b/src/pybind/mgr/dashboard/frontend/package.json
index 3205888f5708..6e19fff3aaa8 100644
--- a/src/pybind/mgr/dashboard/frontend/package.json
+++ b/src/pybind/mgr/dashboard/frontend/package.json
@@ -62,7 +62,8 @@
     "@types/file-saver": "2.0.1",
     "async-mutex": "0.2.4",
     "bootstrap": "5.2.3",
-    "chart.js": "2.9.4",
+    "chart.js": "4.4.0",
+    "chartjs-adapter-moment": "1.0.1",
     "detect-browser": "5.2.0",
     "file-saver": "2.0.2",
     "fork-awesome": "1.1.7",
@@ -70,7 +71,7 @@
     "moment": "2.29.4",
     "ng-block-ui": "3.0.2",
     "ng-click-outside": "7.0.0",
-    "ng2-charts": "2.4.2",
+    "ng2-charts": "4.1.1",
     "ngx-pipe-function": "1.0.0",
     "ngx-toastr": "17.0.2",
     "rxjs": "6.6.3",
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-configuration-list/rbd-configuration-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-configuration-list/rbd-configuration-list.component.spec.ts
index f54ad02720c5..03c40a9e03ed 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-configuration-list/rbd-configuration-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/block/rbd-configuration-list/rbd-configuration-list.component.spec.ts
@@ -5,7 +5,7 @@ import { RouterTestingModule } from '@angular/router/testing';
 
 import { NgbDropdownModule, NgbTooltipModule } from '@ng-bootstrap/ng-bootstrap';
 import { NgxDatatableModule } from '@swimlane/ngx-datatable';
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 
 import { ComponentsModule } from '~/app/shared/components/components.module';
 import { RbdConfigurationEntry } from '~/app/shared/models/configuration';
@@ -27,7 +27,7 @@ describe('RbdConfigurationListComponent', () => {
       RouterTestingModule,
       ComponentsModule,
       NgbDropdownModule,
-      ChartsModule,
+      NgChartsModule,
       SharedModule,
       NgbTooltipModule
     ],
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.html
index b81bc20ba1cf..e7a106d55075 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.html
@@ -3,7 +3,7 @@
           #chartCanvas
           [datasets]="chart.datasets"
           [options]="chart.options"
-          [chartType]="chart.chartType">
+          [type]="chart.chartType">
   </canvas>
   <div class="chartjs-tooltip"
        #chartTooltip>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.spec.ts
index 4ba20fa89663..070f8ef98e82 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.spec.ts
@@ -1,9 +1,10 @@
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 
 import { configureTestBed } from '~/testing/unit-test-helper';
 import { CephfsChartComponent } from './cephfs-chart.component';
+import { ResizeObserver as ResizeObserverPolyfill } from '@juggle/resize-observer';
 
 describe('CephfsChartComponent', () => {
   let component: CephfsChartComponent;
@@ -17,7 +18,7 @@ describe('CephfsChartComponent', () => {
   ];
 
   configureTestBed({
-    imports: [ChartsModule],
+    imports: [NgChartsModule],
     declarations: [CephfsChartComponent]
   });
 
@@ -29,6 +30,9 @@ describe('CephfsChartComponent', () => {
       'mds_mem.ino': counter,
       name: 'a'
     };
+    if (typeof window !== 'undefined') {
+      window.ResizeObserver = window.ResizeObserver || ResizeObserverPolyfill;
+    }
     fixture.detectChanges();
   });
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.ts
index 7f3c9437d47d..7a161f076842 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-chart/cephfs-chart.component.ts
@@ -1,8 +1,8 @@
 import { Component, ElementRef, Input, OnChanges, OnInit, ViewChild } from '@angular/core';
 
-import { ChartDataSets, ChartOptions, ChartPoint, ChartType } from 'chart.js';
 import _ from 'lodash';
 import moment from 'moment';
+import 'chartjs-adapter-moment';
 
 import { ChartTooltip } from '~/app/shared/models/chart-tooltip';
 
@@ -24,78 +24,78 @@ export class CephfsChartComponent implements OnChanges, OnInit {
   rhsCounter = 'mds_server.handle_client_request';
 
   chart: {
-    datasets: ChartDataSets[];
-    options: ChartOptions;
-    chartType: ChartType;
+    datasets: any[];
+    options: any;
+    chartType: any;
   } = {
     datasets: [
       {
         label: this.lhsCounter,
         yAxisID: 'LHS',
         data: [],
-        lineTension: 0.1
+        tension: 0.1,
+        fill: {
+          target: 'origin'
+        }
       },
       {
         label: this.rhsCounter,
         yAxisID: 'RHS',
         data: [],
-        lineTension: 0.1
+        tension: 0.1,
+        fill: {
+          target: 'origin'
+        }
       }
     ],
     options: {
-      title: {
-        text: '',
-        display: true
+      plugins: {
+        title: {
+          text: '',
+          display: true
+        },
+        tooltip: {
+          enabled: false,
+          mode: 'index',
+          intersect: false,
+          position: 'nearest',
+          callbacks: {
+            // Pick the Unix timestamp of the first tooltip item.
+            title: (context: any): string => {
+              let ts = '';
+              if (context.length > 0) {
+                ts = context[0].label;
+              }
+              return moment(ts).format('LTS');
+            }
+          }
+        },
+        legend: {
+          position: 'top'
+        }
       },
       responsive: true,
       maintainAspectRatio: false,
-      legend: {
-        position: 'top'
-      },
       scales: {
-        xAxes: [
-          {
-            position: 'top',
-            type: 'time',
-            time: {
-              displayFormats: {
-                quarter: 'MMM YYYY'
-              }
-            },
-            ticks: {
-              maxRotation: 0
+        x: {
+          position: 'top',
+          type: 'time',
+          time: {
+            displayFormats: {
+              quarter: 'MMM YYYY'
             }
-          }
-        ],
-        yAxes: [
-          {
-            id: 'LHS',
-            type: 'linear',
-            position: 'left'
           },
-          {
-            id: 'RHS',
-            type: 'linear',
-            position: 'right'
-          }
-        ]
-      },
-      tooltips: {
-        enabled: false,
-        mode: 'index',
-        intersect: false,
-        position: 'nearest',
-        callbacks: {
-          // Pick the Unix timestamp of the first tooltip item.
-          title: (tooltipItems, data): string => {
-            let ts = 0;
-            if (tooltipItems.length > 0) {
-              const item = tooltipItems[0];
-              const point = data.datasets[item.datasetIndex].data[item.index] as ChartPoint;
-              ts = point.x as number;
-            }
-            return ts.toString();
+          ticks: {
+            maxRotation: 0
           }
+        },
+        LHS: {
+          type: 'linear',
+          position: 'left'
+        },
+        RHS: {
+          type: 'linear',
+          position: 'right'
         }
       }
     },
@@ -124,21 +124,20 @@ export class CephfsChartComponent implements OnChanges, OnInit {
       (tooltip: any) => tooltip.caretX + 'px',
       (tooltip: any) => tooltip.caretY - tooltip.height - 23 + 'px'
     );
-    chartTooltip.getTitle = (ts) => moment(ts, 'x').format('LTS');
     chartTooltip.checkOffset = true;
-    const chartOptions: ChartOptions = {
+    const chartOptions: any = {
       title: {
         text: this.mdsCounter.name
       },
-      tooltips: {
-        custom: (tooltip) => chartTooltip.customTooltips(tooltip)
+      tooltip: {
+        external: (context: any) => chartTooltip.customTooltips(context)
       }
     };
-    _.merge(this.chart, { options: chartOptions });
+    _.merge(this.chart, { options: { plugins: chartOptions } });
   }
 
   private updateChart() {
-    const chartDataSets: ChartDataSets[] = [
+    const chartDataset: any[] = [
       {
         data: this.convertTimeSeries(this.mdsCounter[this.lhsCounter])
       },
@@ -147,7 +146,7 @@ export class CephfsChartComponent implements OnChanges, OnInit {
       }
     ];
     _.merge(this.chart, {
-      datasets: chartDataSets
+      datasets: chartDataset
     });
     this.chart.datasets = [...this.chart.datasets]; // Force angular to update
   }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index a83e0f16870f..3dca9993c61e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -4,7 +4,7 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 
 import { TreeModule } from '@circlon/angular-tree-component';
 import { NgbNavModule, NgbTooltipModule, NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap';
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 
 import { AppRoutingModule } from '~/app/app-routing.module';
 import { SharedModule } from '~/app/shared/shared.module';
@@ -25,7 +25,7 @@ import { CephfsSubvolumegroupFormComponent } from './cephfs-subvolumegroup-form/
     CommonModule,
     SharedModule,
     AppRoutingModule,
-    ChartsModule,
+    NgChartsModule,
     TreeModule,
     NgbNavModule,
     FormsModule,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-list/osd-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-list/osd-list.component.spec.ts
index d6f865471481..83d00665025f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-list/osd-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-list/osd-list.component.spec.ts
@@ -33,6 +33,7 @@ import {
 } from '~/testing/unit-test-helper';
 import { OsdReweightModalComponent } from '../osd-reweight-modal/osd-reweight-modal.component';
 import { OsdListComponent } from './osd-list.component';
+import { ResizeObserver as ResizeObserverPolyfill } from '@juggle/resize-observer';
 
 describe('OsdListComponent', () => {
   let component: OsdListComponent;
@@ -121,6 +122,9 @@ describe('OsdListComponent', () => {
       close: jest.fn()
     });
     orchService = TestBed.inject(OrchestratorService);
+    if (typeof window !== 'undefined') {
+      window.ResizeObserver = window.ResizeObserver || ResizeObserverPolyfill;
+    }
   });
 
   it('should create', () => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
index cb8b9dadb283..cebfcf903781 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
@@ -31,7 +31,7 @@
       <canvas baseChart
               [datasets]="chartData.dataset"
               [options]="options"
-              [chartType]="'line'"
+              [type]="'line'"
               [plugins]="chartAreaBorderPlugin">
       </canvas>
     </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index 70157ab6fa5c..cbf97691d27d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -4,9 +4,10 @@ import { CssHelper } from '~/app/shared/classes/css-helper';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 import { DimlessBinaryPerSecondPipe } from '~/app/shared/pipes/dimless-binary-per-second.pipe';
 import { FormatterService } from '~/app/shared/services/formatter.service';
-import { BaseChartDirective, PluginServiceGlobalRegistrationAndOptions } from 'ng2-charts';
+import { BaseChartDirective } from 'ng2-charts';
 import { DimlessPipe } from '~/app/shared/pipes/dimless.pipe';
 import { NumberFormatterService } from '~/app/shared/services/number-formatter.service';
+import 'chartjs-adapter-moment';
 
 @Component({
   selector: 'cd-dashboard-area-chart',
@@ -41,25 +42,25 @@ export class DashboardAreaChartComponent implements OnChanges {
   maxConvertedValueUnits?: string;
 
   chartDataUnits: string;
-  chartData: any = {};
-  options: any = {};
+  chartData: any;
+  options: any;
 
-  public chartAreaBorderPlugin: PluginServiceGlobalRegistrationAndOptions[] = [
+  public chartAreaBorderPlugin: any[] = [
     {
-      beforeDraw(chart: Chart) {
+      beforeDraw(chart: any) {
         if (!chart.options.plugins.borderArea) {
           return;
         }
         const {
           ctx,
-          chartArea: { left, top, right, bottom }
+          chartArea: { left, top, width, height }
         } = chart;
         ctx.save();
         ctx.strokeStyle = chart.options.plugins.chartAreaBorder.borderColor;
         ctx.lineWidth = chart.options.plugins.chartAreaBorder.borderWidth;
         ctx.setLineDash(chart.options.plugins.chartAreaBorder.borderDash || []);
         ctx.lineDashOffset = chart.options.plugins.chartAreaBorder.borderDashOffset;
-        ctx.strokeRect(left, top, right - left - 1, bottom);
+        ctx.strokeRect(left, top, width, height);
         ctx.restore();
       }
     }
@@ -82,7 +83,10 @@ export class DashboardAreaChartComponent implements OnChanges {
           pointBackgroundColor: this.cssHelper.propertyValue('chart-color-strong-blue'),
           backgroundColor: this.cssHelper.propertyValue('chart-color-translucent-blue'),
           borderColor: this.cssHelper.propertyValue('chart-color-strong-blue'),
-          borderWidth: 1
+          borderWidth: 1,
+          fill: {
+            target: 'origin'
+          }
         },
         {
           label: '',
@@ -91,12 +95,50 @@ export class DashboardAreaChartComponent implements OnChanges {
           pointBackgroundColor: this.cssHelper.propertyValue('chart-color-orange'),
           backgroundColor: this.cssHelper.propertyValue('chart-color-translucent-yellow'),
           borderColor: this.cssHelper.propertyValue('chart-color-orange'),
-          borderWidth: 1
+          borderWidth: 1,
+          fill: {
+            target: 'origin'
+          }
         }
       ]
     };
 
     this.options = {
+      plugins: {
+        legend: {
+          display: false
+        },
+        tooltip: {
+          mode: 'index',
+          external: function (tooltipModel: any) {
+            tooltipModel.tooltip.x = 10;
+            tooltipModel.tooltip.y = 0;
+          }.bind(this),
+          intersect: false,
+          displayColors: true,
+          backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
+          callbacks: {
+            title: function (tooltipItem: any): any {
+              return tooltipItem[0].xLabel;
+            },
+            label: (context: any) => {
+              return (
+                ' ' +
+                context.dataset.label +
+                ' - ' +
+                context.formattedValue +
+                ' ' +
+                this.chartDataUnits
+              );
+            }
+          }
+        },
+        borderArea: true,
+        chartAreaBorder: {
+          borderColor: this.cssHelper.propertyValue('chart-color-slight-dark-gray'),
+          borderWidth: 1
+        }
+      },
       responsive: true,
       maintainAspectRatio: false,
       animation: false,
@@ -105,74 +147,29 @@ export class DashboardAreaChartComponent implements OnChanges {
           radius: 0
         }
       },
-      legend: {
-        display: false
-      },
-      tooltips: {
-        mode: 'index',
-        custom: function (tooltipModel: { x: number; y: number }) {
-          tooltipModel.x = 10;
-          tooltipModel.y = 0;
-        }.bind(this),
-        intersect: false,
-        displayColors: true,
-        backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
-        callbacks: {
-          title: function (tooltipItem: any): any {
-            return tooltipItem[0].xLabel;
-          },
-          label: (tooltipItems: any, data: any) => {
-            return (
-              ' ' +
-              data.datasets[tooltipItems.datasetIndex].label +
-              ' - ' +
-              tooltipItems.value +
-              ' ' +
-              this.chartDataUnits
-            );
-          }
-        }
-      },
       hover: {
         intersect: false
       },
       scales: {
-        xAxes: [
-          {
-            display: false,
-            type: 'time',
-            gridLines: {
-              display: false
-            },
-            time: {
-              tooltipFormat: 'DD/MM/YYYY - HH:mm:ss'
-            }
+        x: {
+          display: false,
+          type: 'time',
+          grid: {
+            display: false
+          },
+          time: {
+            tooltipFormat: 'DD/MM/YYYY - HH:mm:ss'
           }
-        ],
-        yAxes: [
-          {
-            afterFit: (scaleInstance: any) => (scaleInstance.width = 100),
-            gridLines: {
-              display: false
-            },
-            ticks: {
-              beginAtZero: true,
-              maxTicksLimit: 4,
-              callback: (value: any) => {
-                if (value === 0) {
-                  return null;
-                }
-                return this.convertUnits(value);
-              }
-            }
+        },
+        y: {
+          afterFit: (scaleInstance: any) => (scaleInstance.width = 100),
+          grid: {
+            display: false
+          },
+          beginAtZero: true,
+          ticks: {
+            maxTicksLimit: 4
           }
-        ]
-      },
-      plugins: {
-        borderArea: true,
-        chartAreaBorder: {
-          borderColor: this.cssHelper.propertyValue('chart-color-slight-dark-gray'),
-          borderWidth: 1
         }
       }
     };
@@ -269,12 +266,12 @@ export class DashboardAreaChartComponent implements OnChanges {
 
   private setChartTicks() {
     if (!this.chart) {
+      this.chartDataUnits = '';
       return;
     }
 
     let maxValue = 0;
     let maxValueDataUnits = '';
-    let extraRoom = 1.2;
 
     if (this.data) {
       let maxValueData = Math.max(...this.data.map((values: any) => values[1]));
@@ -287,10 +284,8 @@ export class DashboardAreaChartComponent implements OnChanges {
       [maxValue, maxValueDataUnits] = this.convertUnits(maxValue).split(' ');
     }
 
-    const yAxesTicks = this.chart.chart.options.scales.yAxes[0].ticks;
-    yAxesTicks.suggestedMax = maxValue * extraRoom;
-    yAxesTicks.suggestedMin = 0;
-    yAxesTicks.callback = (value: any) => {
+    const yAxesTicks = this.chart.chart.options.scales.y;
+    yAxesTicks.ticks.callback = (value: any) => {
       if (value === 0) {
         return null;
       }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.html
index c013ab5404b4..25473cb05313 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.html
@@ -2,10 +2,9 @@
   <canvas baseChart
           #chartCanvas
           [datasets]="chartConfig.dataset"
-          [chartType]="chartConfig.chartType"
+          [type]="chartConfig.chartType"
           [options]="chartConfig.options"
           [labels]="chartConfig.labels"
-          [colors]="chartConfig.colors"
           [plugins]="doughnutChartPlugins"
           class="chart-canvas">
   </canvas>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
index 716ca3500ba0..4680fb850a14 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
@@ -2,7 +2,6 @@ import { Component, Input, OnChanges, OnInit } from '@angular/core';
 
 import * as Chart from 'chart.js';
 import _ from 'lodash';
-import { PluginServiceGlobalRegistrationAndOptions } from 'ng2-charts';
 
 import { CssHelper } from '~/app/shared/classes/css-helper';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
@@ -22,15 +21,15 @@ export class DashboardPieComponent implements OnChanges, OnInit {
 
   color: string;
 
-  chartConfig: any = {};
+  chartConfig: any;
 
-  public doughnutChartPlugins: PluginServiceGlobalRegistrationAndOptions[] = [
+  public doughnutChartPlugins: any[] = [
     {
       id: 'center_text',
-      beforeDraw(chart: Chart) {
+      beforeDraw(chart: any) {
         const cssHelper = new CssHelper();
         const defaultFontFamily = 'Helvetica Neue, Helvetica, Arial, sans-serif';
-        Chart.defaults.global.defaultFontFamily = defaultFontFamily;
+        Chart.defaults.font.family = defaultFontFamily;
         const ctx = chart.ctx;
         if (!chart.options.plugins.center_text || !chart.data.datasets[0].label) {
           return;
@@ -80,66 +79,67 @@ export class DashboardPieComponent implements OnChanges, OnInit {
         }
       ],
       options: {
-        cutoutPercentage: 70,
+        cutout: '70%',
         events: ['click', 'mouseout', 'touchstart'],
-        legend: {
-          display: true,
-          position: 'right',
-          labels: {
-            boxWidth: 10,
-            usePointStyle: false,
-            generateLabels: (chart: any) => {
-              const labels = { 0: {}, 1: {}, 2: {} };
-              labels[0] = {
-                text: $localize`Used: ${chart.data.datasets[1].data[2]}`,
-                fillStyle: chart.data.datasets[1].backgroundColor[0],
-                strokeStyle: chart.data.datasets[1].backgroundColor[0]
-              };
-              labels[1] = {
-                text: $localize`Warning: ${chart.data.datasets[0].data[0]}%`,
-                fillStyle: chart.data.datasets[0].backgroundColor[1],
-                strokeStyle: chart.data.datasets[0].backgroundColor[1]
-              };
-              labels[2] = {
-                text: $localize`Danger: ${
-                  chart.data.datasets[0].data[0] + chart.data.datasets[0].data[1]
-                }%`,
-                fillStyle: chart.data.datasets[0].backgroundColor[2],
-                strokeStyle: chart.data.datasets[0].backgroundColor[2]
-              };
-
-              return labels;
-            }
-          }
-        },
+        aspectRatio: 2,
         plugins: {
-          center_text: true
-        },
-        tooltips: {
-          enabled: true,
-          displayColors: false,
-          backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
-          cornerRadius: 0,
-          bodyFontSize: 14,
-          bodyFontStyle: '600',
-          position: 'nearest',
-          xPadding: 12,
-          yPadding: 12,
-          filter: (tooltipItem: any) => {
-            return tooltipItem.datasetIndex === 1;
+          center_text: true,
+          legend: {
+            display: true,
+            position: 'right',
+            labels: {
+              boxWidth: 10,
+              usePointStyle: false,
+              generateLabels: (chart: any) => {
+                let labels = chart.data.labels.slice(0, this.chartConfig.labels.length);
+                labels[0] = {
+                  text: $localize`Used: ${chart.data.datasets[1].data[2]}`,
+                  fillStyle: chart.data.datasets[1].backgroundColor[0],
+                  strokeStyle: chart.data.datasets[1].backgroundColor[0]
+                };
+                labels[1] = {
+                  text: $localize`Warning: ${chart.data.datasets[0].data[0]}%`,
+                  fillStyle: chart.data.datasets[0].backgroundColor[1],
+                  strokeStyle: chart.data.datasets[0].backgroundColor[1]
+                };
+                labels[2] = {
+                  text: $localize`Danger: ${
+                    chart.data.datasets[0].data[0] + chart.data.datasets[0].data[1]
+                  }%`,
+                  fillStyle: chart.data.datasets[0].backgroundColor[2],
+                  strokeStyle: chart.data.datasets[0].backgroundColor[2]
+                };
+
+                return labels;
+              }
+            }
           },
-          callbacks: {
-            label: (item: Record<string, any>, data: Record<string, any>) => {
-              let text = data.labels[item.index];
-              if (!text.includes('%')) {
-                text = `${text} (${data.datasets[item.datasetIndex].data[item.index]}%)`;
+          tooltip: {
+            enabled: true,
+            displayColors: false,
+            backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
+            cornerRadius: 0,
+            bodyFontSize: 14,
+            bodyFontStyle: '600',
+            position: 'nearest',
+            xPadding: 12,
+            yPadding: 12,
+            filter: (tooltipItem: any) => {
+              return tooltipItem.datasetIndex === 1;
+            },
+            callbacks: {
+              label: (item: Record<string, any>, data: Record<string, any>) => {
+                let text = data.labels[item.index];
+                if (!text.includes('%')) {
+                  text = `${text} (${data.datasets[item.datasetIndex].data[item.index]}%)`;
+                }
+                return text;
               }
-              return text;
             }
+          },
+          title: {
+            display: false
           }
-        },
-        title: {
-          display: false
         }
       }
     };
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
index 50db430906e2..73b4f9fa840f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
@@ -4,7 +4,7 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { RouterModule } from '@angular/router';
 
 import { NgbNavModule, NgbPopoverModule, NgbTooltipModule } from '@ng-bootstrap/ng-bootstrap';
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 import { SimplebarAngularModule } from 'simplebar-angular';
 
 import { SharedModule } from '~/app/shared/shared.module';
@@ -21,7 +21,7 @@ import { PgSummaryPipe } from './pg-summary.pipe';
     CommonModule,
     NgbNavModule,
     SharedModule,
-    ChartsModule,
+    NgChartsModule,
     RouterModule,
     NgbPopoverModule,
     NgbTooltipModule,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/dashboard.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/dashboard.module.ts
index 81164d15b9d9..c779feb3156f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/dashboard.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/dashboard.module.ts
@@ -4,7 +4,7 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { RouterModule } from '@angular/router';
 
 import { NgbNavModule, NgbPopoverModule } from '@ng-bootstrap/ng-bootstrap';
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 
 import { SharedModule } from '~/app/shared/shared.module';
 import { DashboardV3Module } from '../dashboard-v3/dashboard-v3.module';
@@ -26,7 +26,7 @@ import { OsdSummaryPipe } from './osd-summary.pipe';
     CommonModule,
     NgbNavModule,
     SharedModule,
-    ChartsModule,
+    NgChartsModule,
     RouterModule,
     NgbPopoverModule,
     FormsModule,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.html
index 0a2535fc9142..a159dddc29e9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.html
@@ -2,10 +2,9 @@
   <canvas baseChart
           #chartCanvas
           [datasets]="chartConfig.dataset"
-          [chartType]="chartConfig.chartType"
+          [type]="chartConfig.chartType"
           [options]="chartConfig.options"
           [labels]="chartConfig.labels"
-          [colors]="chartConfig.colors"
           [plugins]="doughnutChartPlugins"
           class="chart-canvas">
   </canvas>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.ts
index 3b04714c55bd..63e15f5776dd 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health-pie/health-pie.component.ts
@@ -11,7 +11,6 @@ import {
 
 import * as Chart from 'chart.js';
 import _ from 'lodash';
-import { PluginServiceGlobalRegistrationAndOptions } from 'ng2-charts';
 
 import { CssHelper } from '~/app/shared/classes/css-helper';
 import { ChartTooltip } from '~/app/shared/models/chart-tooltip';
@@ -42,15 +41,15 @@ export class HealthPieComponent implements OnChanges, OnInit {
   @Output()
   prepareFn = new EventEmitter();
 
-  chartConfig: any = {};
+  chartConfig: any;
 
-  public doughnutChartPlugins: PluginServiceGlobalRegistrationAndOptions[] = [
+  public doughnutChartPlugins: any[] = [
     {
       id: 'center_text',
-      beforeDraw(chart: Chart) {
+      beforeDraw(chart: any) {
         const cssHelper = new CssHelper();
         const defaultFontFamily = 'Helvetica Neue, Helvetica, Arial, sans-serif';
-        Chart.defaults.global.defaultFontFamily = defaultFontFamily;
+        Chart.defaults.font.family = defaultFontFamily;
         const ctx = chart.ctx;
         if (!chart.options.plugins.center_text || !chart.data.datasets[0].label) {
           return;
@@ -88,11 +87,7 @@ export class HealthPieComponent implements OnChanges, OnInit {
       dataset: [
         {
           label: null,
-          borderWidth: 0
-        }
-      ],
-      colors: [
-        {
+          borderWidth: 0,
           backgroundColor: [
             this.cssHelper.propertyValue('chart-color-green'),
             this.cssHelper.propertyValue('chart-color-yellow'),
@@ -103,41 +98,42 @@ export class HealthPieComponent implements OnChanges, OnInit {
         }
       ],
       options: {
-        cutoutPercentage: 90,
+        cutout: '90%',
         events: ['click', 'mouseout', 'touchstart'],
-        legend: {
-          display: true,
-          position: 'right',
-          labels: {
-            boxWidth: 10,
-            usePointStyle: false
-          }
-        },
+        aspectRatio: 2,
         plugins: {
-          center_text: true
-        },
-        tooltips: {
-          enabled: true,
-          displayColors: false,
-          backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
-          cornerRadius: 0,
-          bodyFontSize: 14,
-          bodyFontStyle: '600',
-          position: 'nearest',
-          xPadding: 12,
-          yPadding: 12,
-          callbacks: {
-            label: (item: Record<string, any>, data: Record<string, any>) => {
-              let text = data.labels[item.index];
-              if (!text.includes('%')) {
-                text = `${text} (${data.datasets[item.datasetIndex].data[item.index]}%)`;
+          center_text: true,
+          legend: {
+            display: true,
+            position: 'right',
+            labels: {
+              boxWidth: 10,
+              usePointStyle: false
+            }
+          },
+          tooltips: {
+            enabled: true,
+            displayColors: false,
+            backgroundColor: this.cssHelper.propertyValue('chart-color-tooltip-background'),
+            cornerRadius: 0,
+            bodyFontSize: 14,
+            bodyFontStyle: '600',
+            position: 'nearest',
+            xPadding: 12,
+            yPadding: 12,
+            callbacks: {
+              label: (item: Record<string, any>, data: Record<string, any>) => {
+                let text = data.labels[item.index];
+                if (!text.includes('%')) {
+                  text = `${text} (${data.datasets[item.datasetIndex].data[item.index]}%)`;
+                }
+                return text;
               }
-              return text;
             }
+          },
+          title: {
+            display: false
           }
-        },
-        title: {
-          display: false
         }
       }
     };
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
index b11d12e49690..8210a4c81037 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
@@ -60,7 +60,7 @@ export class HealthComponent implements OnInit, OnDestroy {
 
   ngOnInit() {
     this.clientStatsConfig = {
-      colors: [
+      dataset: [
         {
           backgroundColor: [
             this.cssHelper.propertyValue('chart-color-cyan'),
@@ -71,7 +71,7 @@ export class HealthComponent implements OnInit, OnDestroy {
     };
 
     this.rawCapacityChartConfig = {
-      colors: [
+      dataset: [
         {
           backgroundColor: [
             this.cssHelper.propertyValue('chart-color-blue'),
@@ -171,7 +171,7 @@ export class HealthComponent implements OnInit, OnDestroy {
     } else {
       this.color = 'chart-color-blue';
     }
-    this.rawCapacityChartConfig.colors[0].backgroundColor[0] = this.cssHelper.propertyValue(
+    this.rawCapacityChartConfig.dataset[0].backgroundColor[0] = this.cssHelper.propertyValue(
       this.color
     );
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index 17f418d1e148..f4880e655905 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -13,7 +13,7 @@ import {
   NgbTooltipModule
 } from '@ng-bootstrap/ng-bootstrap';
 import { ClickOutsideModule } from 'ng-click-outside';
-import { ChartsModule } from 'ng2-charts';
+import { NgChartsModule } from 'ng2-charts';
 import { SimplebarAngularModule } from 'simplebar-angular';
 
 import { MotdComponent } from '~/app/shared/components/motd/motd.component';
@@ -61,7 +61,7 @@ import { CardRowComponent } from './card-row/card-row.component';
     NgbPopoverModule,
     NgbProgressbarModule,
     NgbTooltipModule,
-    ChartsModule,
+    NgChartsModule,
     ReactiveFormsModule,
     PipesModule,
     DirectivesModule,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.html
index c823605d12a9..d66efd2d006b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.html
@@ -2,11 +2,9 @@
      [ngStyle]="style">
   <canvas baseChart
           #sparkCanvas
-          [labels]="labels"
-          [datasets]="datasets"
+          [data]="chartData"
           [options]="options"
-          [colors]="colors"
-          [chartType]="'line'">
+          [type]="'line'">
   </canvas>
   <div class="chartjs-tooltip"
        #sparkTooltip>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.spec.ts
index b8e731d6e24f..27d170190de5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.spec.ts
@@ -5,6 +5,7 @@ import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 import { FormatterService } from '~/app/shared/services/formatter.service';
 import { configureTestBed } from '~/testing/unit-test-helper';
 import { SparklineComponent } from './sparkline.component';
+import { ResizeObserver as ResizeObserverPolyfill } from '@juggle/resize-observer';
 
 describe('SparklineComponent', () => {
   let component: SparklineComponent;
@@ -19,34 +20,37 @@ describe('SparklineComponent', () => {
   beforeEach(() => {
     fixture = TestBed.createComponent(SparklineComponent);
     component = fixture.componentInstance;
+    if (typeof window !== 'undefined') {
+      window.ResizeObserver = window.ResizeObserver || ResizeObserverPolyfill;
+    }
     fixture.detectChanges();
   });
 
   it('should create', () => {
     expect(component).toBeTruthy();
-    expect(component.options.tooltips.custom).toBeDefined();
+    expect(component.options.plugins.tooltip.external).toBeDefined();
   });
 
   it('should update', () => {
-    expect(component.datasets).toEqual([{ data: [] }]);
-    expect(component.labels.length).toBe(0);
+    expect(component.datasets[0].data).toEqual([]);
+    expect(component.chartData.labels.length).toBe(0);
 
     component.data = [11, 22, 33];
     component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
 
-    expect(component.datasets).toEqual([{ data: [11, 22, 33] }]);
-    expect(component.labels.length).toBe(3);
+    expect(component.datasets[0].data).toEqual([11, 22, 33]);
+    expect(component.chartData.labels.length).toBe(3);
   });
 
   it('should not transform the label, if not isBinary', () => {
     component.isBinary = false;
-    const result = component.options.tooltips.callbacks.label({ yLabel: 1024 });
+    const result = component.options.plugins.tooltip.callbacks.label({ parsed: { y: 1024 } });
     expect(result).toBe(1024);
   });
 
   it('should transform the label, if isBinary', () => {
     component.isBinary = true;
-    const result = component.options.tooltips.callbacks.label({ yLabel: 1024 });
+    const result = component.options.plugins.tooltip.callbacks.label({ parsed: { y: 1024 } });
     expect(result).toBe('1 KiB');
   });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.ts
index e2f5af5e0f96..7b791af0ecfa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/sparkline/sparkline.component.ts
@@ -8,6 +8,7 @@ import {
   ViewChild
 } from '@angular/core';
 
+import { BaseChartDirective } from 'ng2-charts';
 import { ChartTooltip } from '~/app/shared/models/chart-tooltip';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 
@@ -21,6 +22,7 @@ export class SparklineComponent implements OnInit, OnChanges {
   chartCanvasRef: ElementRef;
   @ViewChild('sparkTooltip', { static: true })
   chartTooltipRef: ElementRef;
+  @ViewChild(BaseChartDirective) chart: BaseChartDirective;
 
   @Input()
   data: any;
@@ -32,74 +34,76 @@ export class SparklineComponent implements OnInit, OnChanges {
   @Input()
   isBinary: boolean;
 
-  public colors: Array<any> = [
-    {
-      backgroundColor: 'rgba(40,140,234,0.2)',
-      borderColor: 'rgba(40,140,234,1)',
-      pointBackgroundColor: 'rgba(40,140,234,1)',
-      pointBorderColor: '#fff',
-      pointHoverBackgroundColor: '#fff',
-      pointHoverBorderColor: 'rgba(40,140,234,0.8)'
-    }
-  ];
-
   options: Record<string, any> = {
+    plugins: {
+      legend: {
+        display: false
+      },
+      tooltip: {
+        enabled: false,
+        mode: 'index',
+        intersect: false,
+        custom: undefined,
+        callbacks: {
+          label: (tooltipItem: any) => {
+            if (!tooltipItem.parsed) return;
+            if (this.isBinary) {
+              return this.dimlessBinaryPipe.transform(tooltipItem.parsed.y);
+            } else {
+              return tooltipItem.parsed.y;
+            }
+          },
+          title: () => ''
+        }
+      }
+    },
     animation: {
       duration: 0
     },
     responsive: true,
     maintainAspectRatio: false,
-    legend: {
-      display: false
-    },
     elements: {
       line: {
         borderWidth: 1
       }
     },
-    tooltips: {
-      enabled: false,
-      mode: 'index',
-      intersect: false,
-      custom: undefined,
-      callbacks: {
-        label: (tooltipItem: any) => {
-          if (this.isBinary) {
-            return this.dimlessBinaryPipe.transform(tooltipItem.yLabel);
-          } else {
-            return tooltipItem.yLabel;
-          }
-        },
-        title: () => ''
-      }
-    },
     scales: {
-      yAxes: [
-        {
-          display: false
-        }
-      ],
-      xAxes: [
-        {
-          display: false
-        }
-      ]
+      y: {
+        display: false
+      },
+      x: {
+        display: false
+      }
     }
   };
 
   public datasets: Array<any> = [
     {
-      data: []
+      data: [],
+      backgroundColor: 'rgba(40,140,234,0.2)',
+      borderColor: 'rgba(40,140,234,1)',
+      pointBackgroundColor: 'rgba(40,140,234,1)',
+      pointBorderColor: '#fff',
+      pointHoverBackgroundColor: '#fff',
+      pointHoverBorderColor: 'rgba(40,140,234,0.8)'
     }
   ];
 
   public labels: Array<any> = [];
 
+  chartData: {
+    datasets: any[];
+    labels: any[];
+  } = {
+    datasets: this.datasets,
+    labels: this.labels
+  };
+
   constructor(private dimlessBinaryPipe: DimlessBinaryPipe) {}
 
   ngOnInit() {
     const getStyleTop = (tooltip: any) => {
-      return tooltip.caretY - tooltip.height - tooltip.yPadding - 5 + 'px';
+      return tooltip.caretY - tooltip.height - 6 - 5 + 'px';
     };
 
     const getStyleLeft = (tooltip: any, positionX: number) => {
@@ -114,17 +118,20 @@ export class SparklineComponent implements OnInit, OnChanges {
     );
 
     chartTooltip.customColors = {
-      backgroundColor: this.colors[0].pointBackgroundColor,
-      borderColor: this.colors[0].pointBorderColor
+      backgroundColor: this.datasets[0].pointBackgroundColor,
+      borderColor: this.datasets[0].pointBorderColor
     };
 
-    this.options.tooltips.custom = (tooltip: any) => {
+    this.options.plugins.tooltip.external = (tooltip: any) => {
       chartTooltip.customTooltips(tooltip);
     };
   }
 
   ngOnChanges(changes: SimpleChanges) {
-    this.datasets[0].data = changes['data'].currentValue;
-    this.labels = [...Array(changes['data'].currentValue.length)];
+    this.chartData.datasets[0].data = changes['data'].currentValue;
+    this.chartData.labels = [...Array(changes['data'].currentValue.length).fill('')];
+    if (this.chart) {
+      this.chart.chart.update();
+    }
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/chart-tooltip.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/chart-tooltip.ts
index 93a259e79d62..c1312fa3ae6d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/chart-tooltip.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/chart-tooltip.ts
@@ -37,7 +37,8 @@ export class ChartTooltip {
    * @param {any} tooltip
    * @memberof ChartTooltip
    */
-  customTooltips(tooltip: any) {
+  customTooltips(context: any) {
+    const tooltip = context.tooltip;
     // Hide if no tooltip
     if (tooltip.opacity === 0) {
       this.tooltipEl.style.opacity = 0;
@@ -102,7 +103,7 @@ export class ChartTooltip {
     this.tooltipEl.style.fontFamily = tooltip._fontFamily;
     this.tooltipEl.style.fontSize = tooltip.fontSize;
     this.tooltipEl.style.fontStyle = tooltip._fontStyle;
-    this.tooltipEl.style.padding = tooltip.yPadding + 'px ' + tooltip.xPadding + 'px';
+    this.tooltipEl.style.padding = 6 + 'px ' + 6 + 'px';
   }
 
   getBody(body: string) {

From d7f6f632d03438de24ea267d93db9b38089e4026 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 14:47:10 -0400
Subject: [PATCH 1162/2492] cephadm: add systemd name generation to daemon
 identity

There are a bunch of places in the code that wants the full systemd
service name but the code generally is just doing string concatenation.
Add a method to avoid stringing it all the time. This will help as there
will be more systemd work in the future.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 45 ++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index 7fc4af1cb771..cd3c06f2ffea 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -1,5 +1,6 @@
 # deamon_identity.py - classes for identifying daemons & services
 
+import enum
 import os
 import pathlib
 import re
@@ -9,6 +10,14 @@
 from .context import CephadmContext
 
 
+class Categories(str, enum.Enum):
+    SIDECAR = 'sidecar'
+    INIT = 'init'
+
+    def __str__(self) -> str:
+        return self.value
+
+
 class DaemonIdentity:
     def __init__(
         self,
@@ -48,9 +57,35 @@ def container_name(self) -> str:
         name = f'ceph-{self.fsid}-{self.daemon_type}-{self.daemon_id}'
         return name.replace('.', '-')
 
+    def _systemd_name(
+        self,
+        *,
+        framework: str = 'ceph',
+        category: str = '',
+        suffix: str = '',
+        extension: str = '',
+    ) -> str:
+        if category:
+            # validate the category value
+            category = Categories(category)
+        template_terms = [framework, self.fsid, category]
+        instance_terms = [self.daemon_type]
+        instance_terms.append(
+            f'{self.daemon_id}:{suffix}' if suffix else self.daemon_id
+        )
+        instance_terms.append(extension)
+        # use a comprehension to filter out terms that are blank
+        base = '-'.join(v for v in template_terms if v)
+        svc = '.'.join(v for v in instance_terms if v)
+        return f'{base}@{svc}'
+
     @property
     def unit_name(self) -> str:
-        return f'ceph-{self.fsid}@{self.daemon_type}.{self.daemon_id}'
+        return self._systemd_name()
+
+    @property
+    def service_name(self) -> str:
+        return self._systemd_name(extension='service')
 
     def data_dir(self, base_data_dir: Union[str, os.PathLike]) -> str:
         return str(pathlib.Path(base_data_dir) / self.fsid / self.daemon_name)
@@ -99,7 +134,13 @@ def unit_name(self) -> str:
         # of the same unit as the primary. However, to fix a bug with iscsi
         # this is a quick and dirty workaround for distinguishing the two types
         # when generating --cidfile and --conmon-pidfile values.
-        return f'ceph-{self.fsid}@{self.daemon_type}.{self.daemon_id}.{self.subcomponent}'
+        return self._systemd_name(suffix=self.subcomponent)
+
+    @property
+    def service_name(self) -> str:
+        return self._systemd_name(
+            suffix=self.subcomponent, extension='service'
+        )
 
     @property
     def legacy_container_name(self) -> str:

From 8d97f74e933261f08ed6b0c6e102cfb98580c345 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 11:23:47 -0500
Subject: [PATCH 1163/2492] cephadm: use service_name method of daemon identity

Use the new service_name method of the DaemonIdentity type in some basic
cases where string concatenation was being used.
I also converted the name of the agent's helper method to match and made
it private to avoid future confusion.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                          | 15 +++++++--------
 src/cephadm/cephadmlib/container_daemon_form.py |  3 +--
 src/cephadm/tests/test_agent.py                 |  9 +++++----
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 79f5bab5f246..dcb62c9517e4 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -930,9 +930,8 @@ def _update_container_args_for_podman(
 ) -> None:
     if not isinstance(ctx.container_engine, Podman):
         return
-    service_name = f'{ident.unit_name}.service'
     container_args.extend(
-        ctx.container_engine.service_args(ctx, service_name)
+        ctx.container_engine.service_args(ctx, ident.service_name)
     )
 
 
@@ -1394,19 +1393,19 @@ def deploy_daemon_unit(self, config: Dict[str, str] = {}) -> None:
         with write_new(meta_file_path) as f:
             f.write(json.dumps(meta, indent=4) + '\n')
 
-        unit_file_path = os.path.join(self.ctx.unit_dir, self.unit_name())
+        unit_file_path = os.path.join(self.ctx.unit_dir, self._service_name())
         with write_new(unit_file_path) as f:
             f.write(self.unit_file())
 
         call_throws(self.ctx, ['systemctl', 'daemon-reload'])
-        call(self.ctx, ['systemctl', 'stop', self.unit_name()],
+        call(self.ctx, ['systemctl', 'stop', self._service_name()],
              verbosity=CallVerbosity.DEBUG)
-        call(self.ctx, ['systemctl', 'reset-failed', self.unit_name()],
+        call(self.ctx, ['systemctl', 'reset-failed', self._service_name()],
              verbosity=CallVerbosity.DEBUG)
-        call_throws(self.ctx, ['systemctl', 'enable', '--now', self.unit_name()])
+        call_throws(self.ctx, ['systemctl', 'enable', '--now', self._service_name()])
 
-    def unit_name(self) -> str:
-        return '{}.service'.format(get_unit_name(self.fsid, self.daemon_type, self.daemon_id))
+    def _service_name(self) -> str:
+        return self.identity.service_name
 
     def unit_run(self) -> str:
         py3 = shutil.which('python3')
diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index 693b0cf8df0b..1fc6960d6520 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -157,9 +157,8 @@ def daemon_to_container(
     if auto_podman_mounts and _is_podman:
         ctx.container_engine.update_mounts(ctx, container_mounts)
     if auto_podman_args and _is_podman:
-        service_name = f'{daemon.identity.unit_name}.service'
         container_args.extend(
-            ctx.container_engine.service_args(ctx, service_name)
+            ctx.container_engine.service_args(ctx, daemon.identity.service_name)
         )
 
     return CephContainer.for_daemon(
diff --git a/src/cephadm/tests/test_agent.py b/src/cephadm/tests/test_agent.py
index 38c35e355830..4904cb4f61fc 100644
--- a/src/cephadm/tests/test_agent.py
+++ b/src/cephadm/tests/test_agent.py
@@ -69,17 +69,18 @@ def test_agent_deploy_daemon_unit(_call_throws, cephadm_fs):
         _check_file(f'{AGENT_DIR}/unit.meta', json.dumps({'meta': 'data'}, indent=4) + '\n')
 
         # check unit file was created correctly
-        _check_file(f'{ctx.unit_dir}/{agent.unit_name()}', agent.unit_file())
+        svcname = agent._service_name()
+        _check_file(f'{ctx.unit_dir}/{svcname}', agent.unit_file())
 
         expected_call_throws_calls = [
             mock.call(ctx, ['systemctl', 'daemon-reload']),
-            mock.call(ctx, ['systemctl', 'enable', '--now', agent.unit_name()]),
+            mock.call(ctx, ['systemctl', 'enable', '--now', svcname]),
         ]
         _call_throws.assert_has_calls(expected_call_throws_calls)
 
         expected_call_calls = [
-            mock.call(ctx, ['systemctl', 'stop', agent.unit_name()], verbosity=_cephadm.CallVerbosity.DEBUG),
-            mock.call(ctx, ['systemctl', 'reset-failed', agent.unit_name()], verbosity=_cephadm.CallVerbosity.DEBUG),
+            mock.call(ctx, ['systemctl', 'stop', svcname], verbosity=_cephadm.CallVerbosity.DEBUG),
+            mock.call(ctx, ['systemctl', 'reset-failed', svcname], verbosity=_cephadm.CallVerbosity.DEBUG),
         ]
         _cephadm.call.assert_has_calls(expected_call_calls)
 

From 0fae380e655ba75c47cc140eb867539800531e97 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 14:47:10 -0400
Subject: [PATCH 1164/2492] cephadm: add daemon identity methods for upcoming
 sidecars/init services

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index cd3c06f2ffea..4d83ec8f9bfa 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -87,6 +87,11 @@ def unit_name(self) -> str:
     def service_name(self) -> str:
         return self._systemd_name(extension='service')
 
+    @property
+    def init_service_name(self) -> str:
+        # all init contaienrs are run as a single systemd service
+        return self._systemd_name(category='init', extension='service')
+
     def data_dir(self, base_data_dir: Union[str, os.PathLike]) -> str:
         return str(pathlib.Path(base_data_dir) / self.fsid / self.daemon_name)
 
@@ -142,6 +147,12 @@ def service_name(self) -> str:
             suffix=self.subcomponent, extension='service'
         )
 
+    @property
+    def sidecar_service_name(self) -> str:
+        return self._systemd_name(
+            category='sidecar', suffix=self.subcomponent, extension='service'
+        )
+
     @property
     def legacy_container_name(self) -> str:
         raise ValueError(

From 517d8fb2c919aabd4b4b4ec1cb4b3dd69accbcd7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 29 Sep 2023 15:34:28 -0400
Subject: [PATCH 1165/2492] cephadm: add a container copying method

Add a method for duplicating the container properties into a new object.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py | 28 +++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 7ec323acf314..7bddecb6516c 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -1,5 +1,6 @@
 # container_types.py - container instance wrapper types
 
+import copy
 import os
 
 from typing import Dict, List, Optional, Any, Union, Tuple
@@ -179,6 +180,33 @@ def build_stop_cmd(
         cmd.append(cname or self.cname)
         return cmd
 
+    @classmethod
+    def from_container(
+        cls,
+        other: 'BasicContainer',
+        *,
+        ident: Optional[DaemonIdentity] = None
+    ) -> 'BasicContainer':
+        return cls(
+            other.ctx,
+            image=other.image,
+            entrypoint=other.entrypoint,
+            identity=(ident or other.identity),
+            args=other.args,
+            container_args=copy.copy(other.container_args),
+            envs=copy.copy(other.envs),
+            volume_mounts=copy.copy(other.volume_mounts),
+            bind_mounts=copy.copy(other.bind_mounts),
+            network=other.network,
+            ipc=other.ipc,
+            init=other.init,
+            ptrace=other.ptrace,
+            privileged=other.privileged,
+            remove=other.remove,
+            memory_request=other.memory_request,
+            memory_limit=other.memory_limit,
+        )
+
 
 class CephContainer(BasicContainer):
     def __init__(

From ba1a1fe83b360f2f1a4a31e274e85876e4f4373a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 30 Sep 2023 10:10:37 -0400
Subject: [PATCH 1166/2492] cephadm: add SidecarContainer type

Add a new type representing sidecar containers.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py | 69 +++++++++++++++++++++--
 1 file changed, 64 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 7bddecb6516c..0bdbb5c80bbe 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -3,7 +3,7 @@
 import copy
 import os
 
-from typing import Dict, List, Optional, Any, Union, Tuple
+from typing import Dict, List, Optional, Any, Union, Tuple, cast
 
 from .call_wrappers import call, call_throws, CallVerbosity
 from .constants import DEFAULT_TIMEOUT
@@ -71,9 +71,8 @@ def cname(self) -> str:
         assert self.identity
         return self.identity.container_name
 
-    def build_run_cmd(self) -> List[str]:
-        cmd_args: List[str] = [self._container_engine]
-        cmd_args.append('run')
+    def build_engine_run_args(self) -> List[str]:
+        cmd_args: List[str] = []
         if self.remove:
             cmd_args.append('--rm')
         if self.ipc:
@@ -154,8 +153,14 @@ def build_run_cmd(self) -> List[str]:
             + envs
             + vols
             + binds
+        )
+
+    def build_run_cmd(self) -> List[str]:
+        return (
+            [self._container_engine, 'run']
+            + self.build_engine_run_args()
             + [self.image]
-            + self.args
+            + list(self.args)
         )
 
     def build_rm_cmd(
@@ -488,6 +493,60 @@ def rm_cmd(self, storage: bool = False) -> List[str]:
         return self.build_rm_cmd(storage=storage)
 
 
+class SidecarContainer(BasicContainer):
+    @classmethod
+    def from_primary_and_values(
+        cls,
+        ctx: CephadmContext,
+        primary: BasicContainer,
+        sidecar_name: str,
+        *,
+        image: str = '',
+        entrypoint: str = '',
+        args: Optional[List[str]] = None,
+        init: Optional[bool] = None,
+    ) -> 'SidecarContainer':
+        assert primary.identity
+        identity = DaemonSubIdentity.from_parent(
+            primary.identity, sidecar_name
+        )
+        ctr = cast(
+            SidecarContainer, cls.from_container(primary, ident=identity)
+        )
+        ctr.remove = True
+        if image:
+            ctr.image = image
+        if entrypoint:
+            ctr.entrypoint = entrypoint
+        if args:
+            ctr.args = args
+        if init is not None:
+            ctr.init = init
+        return ctr
+
+    def build_engine_run_args(self) -> List[str]:
+        assert isinstance(self.identity, DaemonSubIdentity)
+        cmd_args = super().build_engine_run_args()
+        if self._using_podman:
+            # sidecar containers are always services, otherwise they
+            # would not be sidecars
+            cmd_args += self.ctx.container_engine.service_args(
+                self.ctx, self.identity.sidecar_service_name
+            )
+        return cmd_args
+
+    def run_cmd(self) -> List[str]:
+        if not (self.envs and self.envs[0].startswith('NODE_NAME=')):
+            self.envs.insert(0, 'NODE_NAME=%s' % get_hostname())
+        return self.build_run_cmd()
+
+    def rm_cmd(self, storage: bool = False) -> List[str]:
+        return self.build_rm_cmd(storage=storage)
+
+    def stop_cmd(self, timeout: Optional[int] = None) -> List[str]:
+        return self.build_stop_cmd(timeout=timeout)
+
+
 def is_container_running(ctx: CephadmContext, c: 'CephContainer') -> bool:
     if ctx.name.split('.', 1)[0] in ['agent', 'cephadm-exporter']:
         # these are non-containerized daemon types

From 4fd8f045c48db258335072975a85818ec9b26a91 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 1 Oct 2023 12:49:59 -0400
Subject: [PATCH 1167/2492] cephadm: add a utilty method for type checking
 DaemonSubIdentity types

Add a `must` method that ensures a DaemonIdentity is a DaemonSubIdentity
in a single call.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index 4d83ec8f9bfa..aa47f2815d49 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -5,7 +5,7 @@
 import pathlib
 import re
 
-from typing import Union
+from typing import Union, Optional
 
 from .context import CephadmContext
 
@@ -169,3 +169,12 @@ def from_parent(
             parent.daemon_id,
             subcomponent,
         )
+
+    @classmethod
+    def must(cls, value: Optional[DaemonIdentity]) -> 'DaemonSubIdentity':
+        """Helper to assert value is of the correct type.  Mostly to make mypy
+        happy.
+        """
+        if not isinstance(value, cls):
+            raise TypeError(f'{value!r} is not a {cls}')
+        return value

From 5b6b179d40937032c3d823eb79c10293ae698457 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 14:33:11 -0500
Subject: [PATCH 1168/2492] cephadm: add a daemon identity method for getting
 sidecar script path

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index aa47f2815d49..f13473c2a239 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -153,6 +153,10 @@ def sidecar_service_name(self) -> str:
             category='sidecar', suffix=self.subcomponent, extension='service'
         )
 
+    def sidecar_script(self, base_data_dir: Union[str, os.PathLike]) -> str:
+        sname = f'sidecar-{ self.subcomponent }.run'
+        return str(pathlib.Path(self.data_dir(base_data_dir)) / sname)
+
     @property
     def legacy_container_name(self) -> str:
         raise ValueError(

From 1224a512a267003417252e398b297f20c48995b8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 15:45:55 -0500
Subject: [PATCH 1169/2492] cepahdm: avoid subclass related issues for data_dir

Do not have data dir depend on the daemon_name property. This property
is overriden by the subclass(es) but we want DaemonIdentities and
DaemonSubIdentities to share the same data_dir.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index f13473c2a239..4114cdf1b597 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -93,7 +93,9 @@ def init_service_name(self) -> str:
         return self._systemd_name(category='init', extension='service')
 
     def data_dir(self, base_data_dir: Union[str, os.PathLike]) -> str:
-        return str(pathlib.Path(base_data_dir) / self.fsid / self.daemon_name)
+        # do not use self.daemon_name as that may be overridden in subclasses
+        dn = f'{self.daemon_type}.{self.daemon_id}'
+        return str(pathlib.Path(base_data_dir) / self.fsid / dn)
 
     @classmethod
     def from_name(cls, fsid: str, name: str) -> 'DaemonIdentity':

From 53e1c7d47349daae1dd6981ab8f00b9e17cf60ae Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 14:33:32 -0500
Subject: [PATCH 1170/2492] cephadm: add templates for new systemd unit files

Add templates that will be used for advanced systemd services: sidecars,
init containers, and drop-in files to glue them all together.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadmlib/templates/dropin.service.j2    |  9 ++++
 .../cephadmlib/templates/init_ctr.service.j2  | 33 +++++++++++++++
 .../cephadmlib/templates/sidecar.service.j2   | 41 +++++++++++++++++++
 src/cephadm/cephadmlib/templating.py          |  3 ++
 4 files changed, 86 insertions(+)
 create mode 100644 src/cephadm/cephadmlib/templates/dropin.service.j2
 create mode 100644 src/cephadm/cephadmlib/templates/init_ctr.service.j2
 create mode 100644 src/cephadm/cephadmlib/templates/sidecar.service.j2

diff --git a/src/cephadm/cephadmlib/templates/dropin.service.j2 b/src/cephadm/cephadmlib/templates/dropin.service.j2
new file mode 100644
index 000000000000..6e261f33beaf
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/dropin.service.j2
@@ -0,0 +1,9 @@
+# generated by cephadm
+[Unit]
+{%- if enable_init_containers %}
+Wants={{ identity.init_service_name }}
+{%- endif %}
+{%- for sidecar in sidecar_ids %}
+Wants={{ sidecar.sidecar_service_name }}
+{%- endfor %}
+
diff --git a/src/cephadm/cephadmlib/templates/init_ctr.service.j2 b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
new file mode 100644
index 000000000000..1562a5b18183
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
@@ -0,0 +1,33 @@
+# generated by cephadm
+[Unit]
+Description=Ceph Init Containers for %i on {{ identity.fsid }}
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+{%- if has_docker_engine %}
+After=docker.service
+Requires=docker.service
+{%- endif %}
+Before=ceph-{{ identity.fsid }}@%i.service
+
+PartOf=ceph-{{ identity.fsid }}.target
+Before=ceph-{{ identity.fsid }}.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+EnvironmentFile=-/etc/environment
+ExecStart=/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run
+Restart=on-failure
+RestartSec=10s
+TimeoutStopSec=120
+StartLimitInterval=30min
+StartLimitBurst=5
+Type=oneshot
+RemainAfterExit=yes
+{%- if has_podman_split_version %}
+Delegate=yes
+{%- endif %}
+
+[Install]
+WantedBy=ceph-{{ identity.fsid }}.target
+
diff --git a/src/cephadm/cephadmlib/templates/sidecar.service.j2 b/src/cephadm/cephadmlib/templates/sidecar.service.j2
new file mode 100644
index 000000000000..74e92cf6333c
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/sidecar.service.j2
@@ -0,0 +1,41 @@
+# generated by cephadm
+[Unit]
+Description=Ceph sidecar %i for {{ sidecar.fsid }}
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+{%- if has_docker_engine %}
+After=docker.service
+Requires=docker.service
+{%- endif %}
+After={{ primary.service_name }}
+
+PartOf=ceph-{{ sidecar.fsid }}.target
+Before=ceph-{{ sidecar.fsid }}.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+EnvironmentFile=-/etc/environment
+ExecStart=/bin/bash {{ sidecar_script }} start
+ExecStop=/bin/bash {{ sidecar_script }} stop
+ExecStopPost=-/bin/bash {{ sidecar_script }} poststop
+KillMode=none
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=200
+TimeoutStopSec=120
+StartLimitInterval=30min
+StartLimitBurst=5
+{%- if has_podman_engine %}
+ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
+ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid
+Type=forking
+PIDFile=%t/%n-pid
+{%- if has_podman_split_version %}
+Delegate=yes
+{%- endif %}
+{%- endif %}
+
+[Install]
+WantedBy=ceph-{{ sidecar.fsid }}.target
+
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 5f4dda056086..2f944fd328f8 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -22,6 +22,9 @@ class Templates(str, enum.Enum):
 
     ceph_service = 'ceph.service.j2'
     agent_service = 'agent.service.j2'
+    dropin_service = 'dropin.service.j2'
+    init_ctr_service = 'init_ctr.service.j2'
+    sidecar_service = 'sidecar.service.j2'
     cluster_logrotate_config = 'cluster.logrotate.config.j2'
     cephadm_logrotate_config = 'cephadm.logrotate.config.j2'
 

From 02f02fd6c8cf235629014b8646980fd353800211 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 14:34:24 -0500
Subject: [PATCH 1171/2492] cephadm: add support for systemd unit files for
 init/sidecar ctrs

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/systemd_unit.py | 139 ++++++++++++++++++++++++-
 1 file changed, 135 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index 0c392c77bfe3..208bfba2706a 100644
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -1,16 +1,138 @@
 # systemd_unit.py - creating/managing systemd unit files
 
+import contextlib
 import os
+import pathlib
+
+from typing import IO, List, Optional, Union
 
 from . import templating
 from .call_wrappers import call_throws
 from .container_engines import Docker, Podman
 from .context import CephadmContext
-from .daemon_identity import DaemonIdentity
+from .daemon_identity import DaemonIdentity, DaemonSubIdentity
 from .file_utils import write_new
 from .logging import write_cluster_logrotate_config
 
 
+_DROP_IN_FILENAME = '99-cephadm.conf'
+
+
+class PathInfo:
+    """Utility class to map basic service identities, to the paths used by
+    their corresponding systemd unit files.
+    """
+
+    def __init__(
+        self,
+        unit_dir: Union[str, pathlib.Path],
+        identity: DaemonIdentity,
+        sidecar_ids: Optional[List[DaemonSubIdentity]] = None,
+    ) -> None:
+        self.identity = identity
+        self.sidecar_ids = sidecar_ids or []
+
+        unit_dir = pathlib.Path(unit_dir)
+        self.default_unit_file = unit_dir / f'ceph-{identity.fsid}@.service'
+        self.init_ctr_unit_file = unit_dir / identity.init_service_name
+        self.sidecar_unit_files = {
+            si: unit_dir / si.sidecar_service_name for si in self.sidecar_ids
+        }
+        dname = f'{identity.service_name}.d'
+        self.drop_in_file = unit_dir / dname / _DROP_IN_FILENAME
+
+
+def _write_drop_in(
+    dest: IO,
+    ctx: CephadmContext,
+    identity: DaemonIdentity,
+    enable_init_containers: bool,
+    sidecar_ids: List[DaemonSubIdentity],
+) -> None:
+    templating.render_to_file(
+        dest,
+        ctx,
+        templating.Templates.dropin_service,
+        identity=identity,
+        enable_init_containers=enable_init_containers,
+        sidecar_ids=sidecar_ids,
+    )
+
+
+def _write_init_containers_unit_file(
+    dest: IO, ctx: CephadmContext, identity: DaemonIdentity
+) -> None:
+    has_docker_engine = isinstance(ctx.container_engine, Docker)
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    templating.render_to_file(
+        dest,
+        ctx,
+        templating.Templates.init_ctr_service,
+        identity=identity,
+        has_docker_engine=has_docker_engine,
+        has_podman_engine=has_podman_engine,
+        has_podman_split_version=(
+            has_podman_engine and ctx.container_engine.supports_split_cgroups
+        ),
+    )
+
+
+def _write_sidecar_unit_file(
+    dest: IO, ctx: CephadmContext, primary: DaemonIdentity, sidecar: DaemonSubIdentity
+) -> None:
+    has_docker_engine = isinstance(ctx.container_engine, Docker)
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    templating.render_to_file(
+        dest,
+        ctx,
+        templating.Templates.sidecar_service,
+        primary=primary,
+        sidecar=sidecar,
+        sidecar_script=sidecar.sidecar_script(ctx.data_dir),
+        has_docker_engine=has_docker_engine,
+        has_podman_engine=has_podman_engine,
+        has_podman_split_version=(
+            has_podman_engine and ctx.container_engine.supports_split_cgroups
+        ),
+    )
+
+
+def _install_extended_systemd_services(
+    ctx: CephadmContext,
+    pinfo: PathInfo,
+    identity: DaemonIdentity,
+    enable_init_containers: bool = False,
+) -> None:
+    """Install the systemd unit files needed for more complex services
+    that have init containers and/or sidecars.
+    """
+    with contextlib.ExitStack() as estack:
+        # install the unit file to handle running init containers
+        if enable_init_containers:
+            icfh = estack.enter_context(
+                write_new(pinfo.init_ctr_unit_file, perms=None)
+            )
+            _write_init_containers_unit_file(icfh, ctx, identity)
+
+        # install the unit files to handle running sidecars
+        sids = []
+        for si, sup in pinfo.sidecar_unit_files.items():
+            sufh = estack.enter_context(write_new(sup, perms=None))
+            _write_sidecar_unit_file(sufh, ctx, identity, si)
+            sids.append(si)
+
+        # create a drop-in to create a relationship between the primary
+        # service and the init- and sidecar-based services.
+        if enable_init_containers or sids:
+            pinfo.drop_in_file.parent.mkdir(parents=True, exist_ok=True)
+            difh = estack.enter_context(
+                write_new(pinfo.drop_in_file, perms=None)
+            )
+            _write_drop_in(
+                difh, ctx, identity, enable_init_containers, sids
+            )
+
+
 def _get_unit_file(ctx: CephadmContext, fsid: str) -> str:
     has_docker_engine = isinstance(ctx.container_engine, Docker)
     has_podman_engine = isinstance(ctx.container_engine, Podman)
@@ -73,9 +195,18 @@ def _install_base_units(ctx: CephadmContext, fsid: str) -> None:
     write_cluster_logrotate_config(ctx, fsid)
 
 
-def update_files(ctx: CephadmContext, ident: DaemonIdentity) -> None:
+def update_files(
+    ctx: CephadmContext,
+    ident: DaemonIdentity,
+    *,
+    init_container_ids: Optional[List[DaemonSubIdentity]] = None,
+    sidecar_ids: Optional[List[DaemonSubIdentity]] = None,
+) -> None:
     _install_base_units(ctx, ident.fsid)
     unit = _get_unit_file(ctx, ident.fsid)
-    unit_file = 'ceph-%s@.service' % (ident.fsid)
-    with write_new(ctx.unit_dir + '/' + unit_file, perms=None) as f:
+    pathinfo = PathInfo(ctx.unit_dir, ident, sidecar_ids=sidecar_ids)
+    with write_new(pathinfo.default_unit_file, perms=None) as f:
         f.write(unit)
+    _install_extended_systemd_services(
+        ctx, pathinfo, ident, bool(init_container_ids)
+    )

From 246451215feabb1a22dcdca511df96c511a0be86 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 14:34:52 -0500
Subject: [PATCH 1172/2492] cephadm: black format systemd_unit.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/systemd_unit.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index 208bfba2706a..f44fd2b8fcd5 100644
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -156,11 +156,13 @@ def _install_base_units(ctx: CephadmContext, fsid: str) -> None:
     # global unit
     existed = os.path.exists(ctx.unit_dir + '/ceph.target')
     with write_new(ctx.unit_dir + '/ceph.target', perms=None) as f:
-        f.write('[Unit]\n'
-                'Description=All Ceph clusters and services\n'
-                '\n'
-                '[Install]\n'
-                'WantedBy=multi-user.target\n')
+        f.write(
+            '[Unit]\n'
+            'Description=All Ceph clusters and services\n'
+            '\n'
+            '[Install]\n'
+            'WantedBy=multi-user.target\n'
+        )
     if not existed:
         # we disable before enable in case a different ceph.target
         # (from the traditional package) is present; while newer
@@ -181,8 +183,7 @@ def _install_base_units(ctx: CephadmContext, fsid: str) -> None:
             'Before=ceph.target\n'
             '\n'
             '[Install]\n'
-            'WantedBy=multi-user.target ceph.target\n'.format(
-                fsid=fsid)
+            'WantedBy=multi-user.target ceph.target\n'.format(fsid=fsid)
         )
     if not existed:
         call_throws(ctx, ['systemctl', 'enable', 'ceph-%s.target' % fsid])

From d653bec59cd2d3b60811339670e32cdd3b9a7f72 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 15:08:24 -0500
Subject: [PATCH 1173/2492] cephadm: switch to new style init containers

Convert the call that writes out files for a daemon to use the new
dedicated systemd-unit support for init containers.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                     |  7 ++++--
 src/cephadm/cephadmlib/runscripts.py       | 14 +++++++----
 src/cephadm/tests/test_custom_container.py | 27 ++++++++++++----------
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index dcb62c9517e4..f9df7e0fd00b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -125,7 +125,7 @@
     wrap_ipv6,
 )
 from cephadmlib.locking import FileLock
-from cephadmlib.daemon_identity import DaemonIdentity
+from cephadmlib.daemon_identity import DaemonIdentity, DaemonSubIdentity
 from cephadmlib.packagers import create_packager, Packager
 from cephadmlib.logging import (
     cephadm_init_logging,
@@ -1126,7 +1126,10 @@ def deploy_daemon_units(
     install_sysctl(ctx, ident.fsid, daemon_form_create(ctx, ident))
 
     # systemd
-    systemd_unit.update_files(ctx, ident)
+    ic_ids = [
+        DaemonSubIdentity.must(ic.identity) for ic in init_containers or []
+    ]
+    systemd_unit.update_files(ctx, ident, init_container_ids=ic_ids)
     call_throws(ctx, ['systemctl', 'daemon-reload'])
 
     unit_name = get_unit_name(ident.fsid, ident.daemon_type, ident.daemon_id)
diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
index 7772248f362d..6305b51bac5e 100644
--- a/src/cephadm/cephadmlib/runscripts.py
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -54,6 +54,7 @@ def write_service_scripts(
     post_stop_file_path = data_dir / 'unit.poststop'
     stop_file_path = data_dir / 'unit.stop'
     image_file_path = data_dir / 'unit.image'
+    initctr_file_path = data_dir / 'init_containers.run'
     # use an ExitStack to make writing the files an all-or-nothing affair. If
     # any file fails to write then the write_new'd file will not get renamed
     # into place
@@ -63,11 +64,6 @@ def write_service_scripts(
         runf.write('set -e\n')
         for command in pre_start_commands or []:
             _write_command(ctx, runf, command)
-        init_containers = init_containers or []
-        if init_containers:
-            _write_init_container_cmds_clean(ctx, runf, init_containers[0])
-        for idx, ic in enumerate(init_containers):
-            _write_init_container_cmds(ctx, runf, idx, ic)
         _write_container_cmd_to_bash(ctx, runf, container, ident.daemon_name)
 
         # some metadata about the deploy
@@ -90,6 +86,14 @@ def write_service_scripts(
                 meta['ports'] = []
         metaf.write(json.dumps(meta, indent=4) + '\n')
 
+        # init-container commands
+        if init_containers:
+            initf = estack.enter_context(write_new(initctr_file_path))
+            _write_init_container_cmds_clean(ctx, initf, init_containers[0])
+            for idx, ic in enumerate(init_containers):
+                _write_init_container_cmds(ctx, initf, idx, ic)
+            initf.write('exit 0\n')
+
         # post-stop command(s)
         pstopf = estack.enter_context(write_new(post_stop_file_path))
         # this is a fallback to eventually stop any underlying container that
diff --git a/src/cephadm/tests/test_custom_container.py b/src/cephadm/tests/test_custom_container.py
index 2bcfa0aae6c4..fec435594a73 100644
--- a/src/cephadm/tests/test_custom_container.py
+++ b/src/cephadm/tests/test_custom_container.py
@@ -221,14 +221,17 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
             if not l.startswith(('#', 'set', '/usr/bin/podman run'))
         ]), 'remaining commands should be "rms"'
 
-        idx = runfile_lines.index('# init container cleanup')
-        assert idx > 0
-        assert runfile_lines[idx + 1].startswith('! /usr/bin/podman rm')
-        assert runfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
+        with open(f'/var/lib/ceph/{fsid}/container.tdccai/init_containers.run') as f:
+            icfile_lines = f.read().splitlines()
+
+        idx = icfile_lines.index('# init container cleanup')
+        assert idx >= 0
+        assert icfile_lines[idx + 1].startswith('! /usr/bin/podman rm')
+        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
 
-        idx = runfile_lines.index('# init container 0: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
+        idx = icfile_lines.index('# init container 0: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
         assert idx > 0
-        assert runfile_lines[idx + 1] == (
+        assert icfile_lines[idx + 1] == (
             '/usr/bin/podman run'
             ' --stop-signal=SIGTERM'
             ' --entrypoint /usr/local/bin/prepare.sh'
@@ -237,12 +240,12 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
             ' -v /var/lib/ceph/b01dbeef-701d-9abe-0000-e1e5a47004a7/container.tdccai/data1:/var/lib/myapp'
             ' quay.io/foobar/quux:latest'
         )
-        assert runfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
-        assert runfile_lines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
+        assert icfile_lines[idx + 3].startswith('! /usr/bin/podman rm')
 
-        idx = runfile_lines.index('# init container 1: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
+        idx = icfile_lines.index('# init container 1: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
         assert idx > 0
-        assert runfile_lines[idx + 1] == (
+        assert icfile_lines[idx + 1] == (
             '/usr/bin/podman run'
             ' --stop-signal=SIGTERM'
             ' --entrypoint /usr/local/bin/populate.sh'
@@ -253,5 +256,5 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
             ' quay.io/foobar/quux:latest'
             ' --source=https://my.cool.example.com/samples/geo.1.txt'
         )
-        assert runfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
-        assert runfile_lines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
+        assert icfile_lines[idx + 3].startswith('! /usr/bin/podman rm')

From 31689a134ec7d0b9cf21d94b6fd13d0f0acfed50 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 15:20:29 -0500
Subject: [PATCH 1174/2492] cephadm: add template for sidecar run script

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templates/sidecar.run.j2 | 17 +++++++++++++++++
 src/cephadm/cephadmlib/templating.py            |  1 +
 2 files changed, 18 insertions(+)
 create mode 100644 src/cephadm/cephadmlib/templates/sidecar.run.j2

diff --git a/src/cephadm/cephadmlib/templates/sidecar.run.j2 b/src/cephadm/cephadmlib/templates/sidecar.run.j2
new file mode 100644
index 000000000000..b3e4ecdaba3b
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/sidecar.run.j2
@@ -0,0 +1,17 @@
+#!/bin/sh
+# {{ comment }}
+
+set -e
+if [ "$1" = stop ] || [ "$1" = poststop ]; then
+    ! {{ sidecar.stop_cmd() | map('shellquote') | join(' ') }}
+    ! {{ ctx.container_engine.path }} inspect {{ sidecar.cname | shellquote }} &>/dev/null
+    exit $?
+fi
+
+! {{ sidecar.rm_cmd() | map('shellquote') | join(' ') }} 2> /dev/null
+{%- if has_podman_engine %}
+! {{ sidecar.rm_cmd(storage=True) | map('shellquote') | join(' ') }} 2> /dev/null
+{%- endif %}
+
+exec {{ sidecar.run_cmd() | map('shellquote') | join(' ') }}
+
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 2f944fd328f8..4604d8c14c74 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -27,6 +27,7 @@ class Templates(str, enum.Enum):
     sidecar_service = 'sidecar.service.j2'
     cluster_logrotate_config = 'cluster.logrotate.config.j2'
     cephadm_logrotate_config = 'cephadm.logrotate.config.j2'
+    sidecar_run = 'sidecar.run.j2'
 
     def __str__(self) -> str:
         return self.value

From 99a2c306d74053386b095af087c79d2bd1fd5a24 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 15:31:03 -0500
Subject: [PATCH 1175/2492] cephadm: add function for writing sidecar script

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/runscripts.py | 35 ++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
index 6305b51bac5e..bfa73edbd7d1 100644
--- a/src/cephadm/cephadmlib/runscripts.py
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -5,11 +5,12 @@
 
 from typing import Any, Dict, Union, List, IO, TextIO, Optional, cast
 
+from . import templating
 from .container_engines import Podman
-from .container_types import CephContainer, InitContainer
+from .container_types import CephContainer, InitContainer, SidecarContainer
 from .context import CephadmContext
 from .context_getters import fetch_meta
-from .daemon_identity import DaemonIdentity
+from .daemon_identity import DaemonIdentity, DaemonSubIdentity
 from .file_utils import write_new
 from .net_utils import EndPoint
 
@@ -39,6 +40,7 @@ def write_service_scripts(
     *,
     container: CephContainer,
     init_containers: Optional[List[InitContainer]] = None,
+    sidecars: Optional[List[SidecarContainer]] = None,
     endpoints: Optional[List[EndPoint]] = None,
     pre_start_commands: Optional[List[Command]] = None,
     post_stop_commands: Optional[List[Command]] = None,
@@ -94,6 +96,18 @@ def write_service_scripts(
                 _write_init_container_cmds(ctx, initf, idx, ic)
             initf.write('exit 0\n')
 
+        # sidecar container scripts
+        for sidecar in sidecars or []:
+            assert isinstance(sidecar.identity, DaemonSubIdentity)
+            script_path = sidecar.identity.sidecar_script(ctx.data_dir)
+            scsf = estack.enter_context(write_new(script_path))
+            _write_sidecar_script(
+                ctx,
+                scsf,
+                sidecar,
+                f'sidecar: {sidecar.identity.subcomponent}',
+            )
+
         # post-stop command(s)
         pstopf = estack.enter_context(write_new(post_stop_file_path))
         # this is a fallback to eventually stop any underlying container that
@@ -206,6 +220,23 @@ def _write_stop_actions(
     )
 
 
+def _write_sidecar_script(
+    ctx: CephadmContext,
+    file_obj: IO[str],
+    sidecar: SidecarContainer,
+    comment: str = '',
+) -> None:
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    templating.render_to_file(
+        file_obj,
+        ctx,
+        templating.Templates.sidecar_run,
+        sidecar=sidecar,
+        comment=comment,
+        has_podman_engine=has_podman_engine,
+    )
+
+
 def _bash_cmd(
     fh: IO[str],
     cmd: List[str],

From e0c8b6d85dc11b22f21aabae22e74ad344c9b33d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 15 Oct 2023 12:57:42 -0400
Subject: [PATCH 1176/2492] cepahdm: add support for sidecars to container
 daemon forms

Add support for producing sidecar containers to the container daemon
form type and pass the sidecars through to the unit/script management
function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                         | 18 +++++++++++++++---
 .../cephadmlib/container_daemon_form.py        |  8 +++++++-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index f9df7e0fd00b..b50a03854f8f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -138,8 +138,9 @@
 from cephadmlib.container_types import (
     CephContainer,
     InitContainer,
-    is_container_running,
+    SidecarContainer,
     extract_uid_gid,
+    is_container_running,
 )
 from cephadmlib.decorators import (
     deprecated_command,
@@ -947,6 +948,7 @@ def deploy_daemon(
     deployment_type: DeploymentType = DeploymentType.DEFAULT,
     endpoints: Optional[List[EndPoint]] = None,
     init_containers: Optional[List['InitContainer']] = None,
+    sidecars: Optional[List[SidecarContainer]] = None,
 ) -> None:
     endpoints = endpoints or []
     daemon_type = ident.daemon_type
@@ -1026,6 +1028,7 @@ def deploy_daemon(
                     osd_fsid=osd_fsid,
                     endpoints=endpoints,
                     init_containers=init_containers,
+                    sidecars=sidecars,
                 )
             else:
                 raise RuntimeError('attempting to deploy a daemon without a container image')
@@ -1090,7 +1093,8 @@ def deploy_daemon_units(
     start: bool = True,
     osd_fsid: Optional[str] = None,
     endpoints: Optional[List[EndPoint]] = None,
-    init_containers: Optional[List['InitContainer']] = None,
+    init_containers: Optional[List[InitContainer]] = None,
+    sidecars: Optional[List[SidecarContainer]] = None,
 ) -> None:
     data_dir = ident.data_dir(ctx.data_dir)
     pre_start_commands: List[runscripts.Command] = []
@@ -1116,6 +1120,7 @@ def deploy_daemon_units(
         ident,
         container=container,
         init_containers=init_containers,
+        sidecars=sidecars,
         endpoints=endpoints,
         pre_start_commands=pre_start_commands,
         post_stop_commands=post_stop_commands,
@@ -1129,7 +1134,12 @@ def deploy_daemon_units(
     ic_ids = [
         DaemonSubIdentity.must(ic.identity) for ic in init_containers or []
     ]
-    systemd_unit.update_files(ctx, ident, init_container_ids=ic_ids)
+    sc_ids = [
+        DaemonSubIdentity.must(sc.identity) for sc in sidecars or []
+    ]
+    systemd_unit.update_files(
+        ctx, ident, init_container_ids=ic_ids, sidecar_ids=sc_ids
+    )
     call_throws(ctx, ['systemctl', 'daemon-reload'])
 
     unit_name = get_unit_name(ident.fsid, ident.daemon_type, ident.daemon_id)
@@ -3080,6 +3090,7 @@ def _deploy_daemon_container(
     daemon.customize_container_endpoints(daemon_endpoints, deployment_type)
     ctr = daemon.container(ctx)
     ics = daemon.init_containers(ctx)
+    sccs = daemon.sidecar_containers(ctx)
     config, keyring = daemon.config_and_keyring(ctx)
     uid, gid = daemon.uid_gid(ctx)
     deploy_daemon(
@@ -3094,6 +3105,7 @@ def _deploy_daemon_container(
         endpoints=daemon_endpoints,
         osd_fsid=daemon.osd_fsid,
         init_containers=ics,
+        sidecars=sccs,
     )
 
 ##################################
diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index 1fc6960d6520..1b65194aae7a 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -5,7 +5,7 @@
 from typing import List, Tuple, Optional, Dict
 
 from .container_engines import Podman
-from .container_types import CephContainer, InitContainer
+from .container_types import CephContainer, InitContainer, SidecarContainer
 from .context import CephadmContext
 from .daemon_form import DaemonForm
 from .deploy import DeploymentType
@@ -40,6 +40,12 @@ def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
         """
         return []
 
+    def sidecar_containers(self, ctx: CephadmContext) -> List[SidecarContainer]:
+        """Returns a list of sidecar containers that should be executed along
+        with the primary service container.
+        """
+        return []
+
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]
     ) -> None:

From 794652e268b8301ff97b895680a9e71070d7477b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 15:41:17 -0500
Subject: [PATCH 1177/2492] cephadm: convert iscsi daemon form to use sidecars

Convert the iscsi container daemon form class to produce a sidecar
container for the tcmu runner and reduce the amount of special casing
for iscsi.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py                  | 33 ++++--------------
 src/cephadm/cephadmlib/daemons/iscsi.py | 45 +++++++++++--------------
 src/cephadm/tests/test_cephadm.py       | 40 ++++++++++++++--------
 3 files changed, 51 insertions(+), 67 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b50a03854f8f..800484b4195f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1112,8 +1112,12 @@ def deploy_daemon_units(
             _osd_unit_poststop_commands(ctx, ident, osd_fsid)
         )
     if ident.daemon_type == CephIscsi.daemon_type:
-        pre_start_commands.extend(_iscsi_unit_run_commands(ctx, ident, data_dir))
-        post_stop_commands.extend(_iscsi_unit_poststop_commands(ctx, ident, data_dir))
+        pre_start_commands.append(
+            CephIscsi.configfs_mount_umount(data_dir, mount=True)
+        )
+        post_stop_commands.append(
+            CephIscsi.configfs_mount_umount(data_dir, mount=False)
+        )
 
     runscripts.write_service_scripts(
         ctx,
@@ -1214,17 +1218,6 @@ def _osd_unit_run_commands(
     return cmds
 
 
-def _iscsi_unit_run_commands(
-    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
-) -> List[runscripts.Command]:
-    cmds: List[runscripts.Command] = []
-    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
-    ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
-    tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    cmds.append(runscripts.ContainerCommand(tcmu_container, comment='iscsi tcmu-runner container', background=True))
-    return cmds
-
-
 def _osd_unit_poststop_commands(
     ctx: CephadmContext, ident: 'DaemonIdentity', osd_fsid: str
 ) -> List[runscripts.Command]:
@@ -1241,20 +1234,6 @@ def _osd_unit_poststop_commands(
     return [runscripts.ContainerCommand(poststop, comment='deactivate osd')]
 
 
-def _iscsi_unit_poststop_commands(
-    ctx: CephadmContext, ident: 'DaemonIdentity', data_dir: str
-) -> List[runscripts.Command]:
-    # make sure we also stop the tcmu container
-    cmds: List[runscripts.Command] = []
-    runtime_dir = '/run'
-    ceph_iscsi = CephIscsi.init(ctx, ident.fsid, ident.daemon_id)
-    tcmu_container = ceph_iscsi.get_tcmu_runner_container()
-    cmds.append('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n')
-    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    cmds.append('! ' + 'rm ' + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (ident.fsid, ident.daemon_type, ident.daemon_id + '.tcmu') + '\n')
-    cmds.append(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
-    return cmds
-
 ##################################
 
 
diff --git a/src/cephadm/cephadmlib/daemons/iscsi.py b/src/cephadm/cephadmlib/daemons/iscsi.py
index 1845a37bf4ef..ade88a90af0f 100644
--- a/src/cephadm/cephadmlib/daemons/iscsi.py
+++ b/src/cephadm/cephadmlib/daemons/iscsi.py
@@ -5,10 +5,10 @@
 from typing import Dict, List, Optional, Tuple
 
 from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
-from ..container_types import CephContainer, extract_uid_gid
+from ..container_types import CephContainer, SidecarContainer, extract_uid_gid
 from ..context_getters import fetch_configs, get_config_and_keyring
 from ..daemon_form import register as register_daemon_form
-from ..daemon_identity import DaemonIdentity, DaemonSubIdentity
+from ..daemon_identity import DaemonIdentity
 from ..constants import DEFAULT_IMAGE
 from ..context import CephadmContext
 from ..data_utils import dict_get, is_fsid
@@ -190,8 +190,7 @@ def create_daemon_dirs(self, data_dir, uid, gid):
         os.chmod(os.path.join(data_dir, 'tcmu-runner-entrypoint.sh'), 0o700)
 
     @staticmethod
-    def configfs_mount_umount(data_dir, mount=True):
-        # type: (str, bool) -> List[str]
+    def configfs_mount_umount(data_dir: str, mount: bool = True) -> str:
         mount_path = os.path.join(data_dir, 'configfs')
         if mount:
             cmd = (
@@ -203,7 +202,7 @@ def configfs_mount_umount(data_dir, mount=True):
                 'if grep -qs {0} /proc/mounts; then '
                 'umount {0}; fi'.format(mount_path)
             )
-        return cmd.split()
+        return cmd
 
     @staticmethod
     def tcmu_runner_entrypoint_script() -> str:
@@ -242,27 +241,6 @@ def tcmu_runner_entrypoint_script() -> str:
 done
 """
 
-    def get_tcmu_runner_container(self):
-        # type: () -> CephContainer
-        # daemon_id, is used to generated the cid and pid files used by podman but as both tcmu-runner
-        # and rbd-target-api have the same daemon_id, it conflits and prevent the second container from
-        # starting. .tcmu runner is appended to the daemon_id to fix that.
-        subident = DaemonSubIdentity(
-            self.fsid, self.daemon_type, self.daemon_id, 'tcmu'
-        )
-        tcmu_dmn = self.create(self.ctx, subident)
-        tcmu_container = to_deployment_container(
-            self.ctx, daemon_to_container(self.ctx, tcmu_dmn, privileged=True)
-        )
-        # TODO: Eventually we don't want to run tcmu-runner through this script.
-        # This is intended to be a workaround backported to older releases
-        # and should eventually be removed in at least squid onward
-        tcmu_container.entrypoint = (
-            '/usr/local/scripts/tcmu-runner-entrypoint.sh'
-        )
-        tcmu_container.cname = self.get_container_name(desc='tcmu')
-        return tcmu_container
-
     def container(self, ctx: CephadmContext) -> CephContainer:
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
@@ -284,3 +262,18 @@ def customize_container_args(
         self, ctx: CephadmContext, args: List[str]
     ) -> None:
         args.append(ctx.container_engine.unlimited_pids_option)
+
+    def sidecar_containers(
+        self, ctx: CephadmContext
+    ) -> List[SidecarContainer]:
+        tcmu_sidecar = SidecarContainer.from_primary_and_values(
+            ctx,
+            self.container(ctx),
+            'tcmu',
+            # TODO: Eventually we don't want to run tcmu-runner through this
+            # script.  This is intended to be a workaround backported to older
+            # releases and should eventually be removed in at least squid
+            # onward
+            entrypoint='/usr/local/scripts/tcmu-runner-entrypoint.sh',
+        )
+        return [tcmu_sidecar]
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 82850ab597d6..7bca8b9cbf3c 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1760,7 +1760,11 @@ def test_keyring(self, cephadm_fs):
 
 
 class TestIscsi:
-    def test_unit_run(self, cephadm_fs):
+    def test_unit_run(self, cephadm_fs, funkypatch):
+        funkypatch.patch(
+            'cephadmlib.daemons.iscsi.extract_uid_gid'
+        ).return_value = (123, 123)
+
         fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
         config_json = {
                 'files': {'iscsi-gateway.cfg': ''}
@@ -1773,27 +1777,35 @@ def test_unit_run(self, cephadm_fs):
             _cephadm.get_parm.return_value = config_json
 
             ident = _cephadm.DaemonIdentity(fsid, 'iscsi', 'daemon_id')
-            c = _cephadm.get_container(ctx, ident)
-            _cephadm.make_data_dir(ctx, ident)
-            _cephadm.deploy_daemon_units(
-                ctx,
-                ident,
-                0, 0,
-                c,
-                True, True
+
+            _cephadm._deploy_daemon_container(
+                ctx, ident, [], _cephadm.DeploymentType.DEFAULT
             )
 
             with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f:
-                assert f.read() == """set -e
+                contents = f.read()
+                assert contents == """set -e
 if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi
-# iscsi tcmu-runner container
-! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null
-! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null
-/usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph &
 # iscsi.daemon_id
 ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null
 ! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null
 /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph
+"""
+            with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/sidecar-tcmu.run') as f:
+                contents = f.read()
+            assert contents == """#!/bin/sh
+# sidecar: tcmu
+
+set -e
+if [ "$1" = stop ] || [ "$1" = poststop ]; then
+    ! /usr/bin/docker stop ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu
+    ! /usr/bin/docker inspect ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu &>/dev/null
+    exit $?
+fi
+
+! /usr/bin/docker rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null
+
+exec /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --ulimit nofile=1048576 --net=host --entrypoint /usr/local/scripts/tcmu-runner-entrypoint.sh --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu --pids-limit=0 -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/tcmu-runner-entrypoint.sh:/usr/local/scripts/tcmu-runner-entrypoint.sh -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph
 """
 
     def test_get_container(self):

From 185182b0a9861e6720b50835baf55250c8baf74f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 17 Nov 2023 14:43:06 -0500
Subject: [PATCH 1178/2492] cephadm: avoid errors when calling sub-ident method

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index 4114cdf1b597..cbefa15041a0 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -145,9 +145,10 @@ def unit_name(self) -> str:
 
     @property
     def service_name(self) -> str:
-        return self._systemd_name(
-            suffix=self.subcomponent, extension='service'
-        )
+        # use the parent's service_name to get the service. sub-identities
+        # must use other specific methods (like sidecar_service_name) for
+        # sub-identity based services
+        raise ValueError('called service_name on DaemonSubIdentity')
 
     @property
     def sidecar_service_name(self) -> str:

From 77cbbd614872bbad2e84efe9f87ecc6de6bb4d39 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 11:35:36 -0500
Subject: [PATCH 1179/2492] cephadm: rename
 {get,lookup}_unit_name_by_daemon_name

Rename get_unit_name_by_daemon_name to lookup__unit_name_by_daemon_name
to emphasize that the function does a full lookup (via the daemon list)
to determine the systemd service name for the given values and that
there's a potential performance implication to doing that.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 800484b4195f..82e29d941eea 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -659,7 +659,7 @@ def get_unit_name(
     return DaemonIdentity(fsid, daemon_type, daemon_id).unit_name
 
 
-def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> str:
+def lookup_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> str:
     daemon = get_daemon_description(ctx, fsid, name)
     try:
         return daemon['systemd_unit']
@@ -3306,7 +3306,7 @@ def command_unit(ctx):
     if not ctx.fsid:
         raise Error('must pass --fsid to specify cluster')
 
-    unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+    unit_name = lookup_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
 
     _, _, code = call(
         ctx,
@@ -3325,7 +3325,7 @@ def command_logs(ctx):
     if not ctx.fsid:
         raise Error('must pass --fsid to specify cluster')
 
-    unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+    unit_name = lookup_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
 
     cmd = [find_program('journalctl')]
     cmd.extend(['-u', unit_name])
@@ -4137,7 +4137,7 @@ def command_rm_daemon(ctx):
     lock.acquire()
 
     (daemon_type, daemon_id) = ctx.name.split('.', 1)
-    unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+    unit_name = lookup_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
 
     if daemon_type in ['mon', 'osd'] and not ctx.force:
         raise Error('must pass --force to proceed: '

From d942a1b44a61580feef2d0add90f8b763019ee9f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 13:57:48 -0500
Subject: [PATCH 1180/2492] cephadm: add unlink_file function

Add the unlink_file function to file_utils. This fills in a gap between
python 3.6 and features provided in pathlib.Path in later versions of
python. Adds an option to ignore all errors for good measure.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/file_utils.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/cephadm/cephadmlib/file_utils.py b/src/cephadm/cephadmlib/file_utils.py
index 1b9f11499a49..9c1ca50b9255 100644
--- a/src/cephadm/cephadmlib/file_utils.py
+++ b/src/cephadm/cephadmlib/file_utils.py
@@ -143,3 +143,21 @@ def get_file_timestamp(fn):
 
 def make_run_dir(fsid: str, uid: int, gid: int) -> None:
     makedirs(f'/var/run/ceph/{fsid}', uid, gid, 0o770)
+
+
+def unlink_file(
+    path: Union[str, Path],
+    missing_ok: bool = False,
+    ignore_errors: bool = False,
+) -> None:
+    """Wrapper around unlink that can either ignore missing files or all
+    errors.
+    """
+    try:
+        Path(path).unlink()
+    except FileNotFoundError:
+        if not missing_ok and not ignore_errors:
+            raise
+    except Exception:
+        if not ignore_errors:
+            raise

From 9504801b02cd16e8abced2bc678e1f3b94826adc Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 13:58:33 -0500
Subject: [PATCH 1181/2492] cephadm: add method for parsing a sub identity from
 a service name

This will become handy later when we need DaemonSubIdentity objects and
are not on the deploy path.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemon_identity.py | 46 ++++++++++++++++-
 src/cephadm/tests/test_util_funcs.py      | 62 +++++++++++++++++++++++
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/daemon_identity.py b/src/cephadm/cephadmlib/daemon_identity.py
index cbefa15041a0..760401a3e540 100644
--- a/src/cephadm/cephadmlib/daemon_identity.py
+++ b/src/cephadm/cephadmlib/daemon_identity.py
@@ -5,7 +5,7 @@
 import pathlib
 import re
 
-from typing import Union, Optional
+from typing import Union, Optional, Tuple
 
 from .context import CephadmContext
 
@@ -177,6 +177,50 @@ def from_parent(
             subcomponent,
         )
 
+    @classmethod
+    def from_service_name(
+        cls, service_name: str
+    ) -> Tuple['DaemonSubIdentity', str]:
+        """Return a DaemonSubIdentity and category value by parsing the
+        contents of a systemd service name for a sidecar container.
+        """
+        # ceph services always have the template@instance form
+        tpart, ipart = service_name.split('@', 1)
+        # drop the .service if it exists
+        if ipart.endswith('.service'):
+            ipart = ipart[:-8]
+        # verify the service name starts with 'ceph' -- our framework
+        framework, tpart = tpart.split('-', 1)
+        if framework != 'ceph':
+            raise ValueError(f'Invalid framework value: {service_name}')
+        # we're parsing only services for subcomponents. it must take the
+        # form <FSID>-<CATEGORY>. Where categories are sidecar or init.
+        fsid, category = tpart.rsplit('-', 1)
+        try:
+            Categories(category)
+        except ValueError:
+            raise ValueError(f'Invalid service category: {service_name}')
+        # if it is a sidecar it will have a subcomponent name following a colon
+        svcparts = ipart.split(':')
+        if len(svcparts) == 1:
+            subc = ''
+        elif len(svcparts) == 2:
+            subc = svcparts[1]
+        else:
+            raise ValueError(f'Unexpected instance value: {ipart}')
+        # only services based on sidecars currently have named subcomponents
+        # init subcomponents are all "hidden" within a single init service
+        if subc and not category == Categories.SIDECAR:
+            raise ValueError(
+                f'Unexpected subcomponent {subc!r} for category {category}'
+            )
+        elif not subc:
+            # because we return a DaemonSubIdentity we need some value for
+            # the subcomponent on init services. Just repeat the category
+            subc = str(category)
+        daemon_type, daemon_id = svcparts[0].split('.', 1)
+        return cls(fsid, daemon_type, daemon_id, subc), category
+
     @classmethod
     def must(cls, value: Optional[DaemonIdentity]) -> 'DaemonSubIdentity':
         """Helper to assert value is of the correct type.  Mostly to make mypy
diff --git a/src/cephadm/tests/test_util_funcs.py b/src/cephadm/tests/test_util_funcs.py
index ffcf3909c4ee..c52852db679f 100644
--- a/src/cephadm/tests/test_util_funcs.py
+++ b/src/cephadm/tests/test_util_funcs.py
@@ -810,3 +810,65 @@ def test_apply_deploy_config_to_ctx(cc, monkeypatch):
     ctx = FakeContext()
     _cephadm.apply_deploy_config_to_ctx(cc.cfg_data, ctx)
     cc.check(ctx)
+
+
+def test_daemon_sub_identity_from_sidecar_service():
+    from cephadmlib.daemon_identity import DaemonSubIdentity
+
+    dsi = DaemonSubIdentity(
+        '244c9842-866b-11ee-80ad-3497f6318048', 'iscsi', 'rab.oof', 'tcmu'
+    )
+    service_name = dsi.sidecar_service_name
+    assert (
+        service_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048-sidecar@iscsi.rab.oof:tcmu.service'
+    )
+    d2, category = DaemonSubIdentity.from_service_name(service_name)
+    assert category == 'sidecar'
+    assert d2.fsid == '244c9842-866b-11ee-80ad-3497f6318048'
+    assert d2.daemon_type == 'iscsi'
+    assert d2.daemon_id == 'rab.oof'
+    assert d2.subcomponent == 'tcmu'
+
+
+def test_daemon_sub_identity_from_init_service():
+    from cephadmlib.daemon_identity import DaemonIdentity, DaemonSubIdentity
+
+    di = DaemonIdentity(
+        '244c9842-866b-11ee-80ad-3497f6318048', 'putrats', 'wow',
+    )
+    service_name = di.init_service_name
+    assert (
+        service_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048-init@putrats.wow.service'
+    )
+    d2, category = DaemonSubIdentity.from_service_name(service_name)
+    assert category == 'init'
+    assert d2.fsid == '244c9842-866b-11ee-80ad-3497f6318048'
+    assert d2.daemon_type == 'putrats'
+    assert d2.daemon_id == 'wow'
+    assert d2.subcomponent == 'init'
+
+
+def test_daemon_sub_identity_from_service_invalid():
+    from cephadmlib.daemon_identity import DaemonSubIdentity
+
+    service_name = 'ceph-244c9842-866b-11ee-80ad-3497f6318048-morbo@iscsi.rab.oof.tcmu.service'
+    with pytest.raises(ValueError):
+        DaemonSubIdentity.from_service_name(service_name)
+
+    service_name = 'ceph-244c9842-866b-11ee-80ad-3497f6318048@iscsi.rab.oof.service'
+    with pytest.raises(ValueError):
+        DaemonSubIdentity.from_service_name(service_name)
+
+    service_name = 'ceph-244c9842-866b-11ee-80ad-3497f6318048-sidecar@foo.bar.baz:acolon:toomany.service'
+    with pytest.raises(ValueError):
+        DaemonSubIdentity.from_service_name(service_name)
+
+    service_name = 'ceph-244c9842-866b-11ee-80ad-3497f6318048-init@foo.bar.baz:woops.service'
+    with pytest.raises(ValueError):
+        DaemonSubIdentity.from_service_name(service_name)
+
+    service_name = 'random-task@elsewise.service'
+    with pytest.raises(ValueError):
+        DaemonSubIdentity.from_service_name(service_name)

From 1991cd6de845c65d8ab3b887a154f1e671d9b9f2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 13:59:19 -0500
Subject: [PATCH 1182/2492] cephadm: add sidecars_from_dropin function

Add a sidecars_from_dropin function that takes a minimal PathInfo object
and returns a PathInfo populated with sidecars information based on
reading the content of the dropin file. This will be useful when we need
information on configured sidecars but are not on the deploy path.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/systemd_unit.py | 27 ++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index f44fd2b8fcd5..b0f62b9cbfac 100644
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -211,3 +211,30 @@ def update_files(
     _install_extended_systemd_services(
         ctx, pathinfo, ident, bool(init_container_ids)
     )
+
+
+def sidecars_from_dropin(
+    pathinfo: PathInfo, missing_ok: bool = False
+) -> PathInfo:
+    """Read the list of sidecars for a service from the service's drop in file."""
+    # This is useful in the cases where the sidecars would be determined from
+    # input data (deployment) but we lack the original deployment data (rm
+    # daemon).
+    sidecars = []
+    try:
+        with open(pathinfo.drop_in_file) as fh:
+            lines = fh.readlines()
+    except FileNotFoundError:
+        if missing_ok:
+            return pathinfo
+        raise
+    for line in lines:
+        if not line.startswith('Wants='):
+            continue
+        for item in line[6:].strip().split():
+            si, category = DaemonSubIdentity.from_service_name(item)
+            if category == 'sidecar':
+                sidecars.append(si)
+    return PathInfo(
+        pathinfo.default_unit_file.parent, pathinfo.identity, sidecars
+    )

From 87e6dc6c5fac5f3ad2641015ffebd4206bc9a0b0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 19:00:38 -0500
Subject: [PATCH 1183/2492] cephadm: add a terminate_service systemd function

Add a function that encapsulates the common 3-step process found in
cephadm of stopping, resetting errors, and disabling a service.
This will be used to handle new services to stop as well as cleaning
up some places where the three steps are performed inline.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/systemd.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/cephadm/cephadmlib/systemd.py b/src/cephadm/cephadmlib/systemd.py
index 69fc8b740868..a07757eccadc 100644
--- a/src/cephadm/cephadmlib/systemd.py
+++ b/src/cephadm/cephadmlib/systemd.py
@@ -68,3 +68,21 @@ def check_units(ctx, units, enabler=None):
                 logger.info('Enabling unit %s' % u)
                 enabler.enable_service(u)
     return False
+
+
+def terminate_service(ctx: CephadmContext, service_name: str) -> None:
+    call(
+        ctx,
+        ['systemctl', 'stop', service_name],
+        verbosity=CallVerbosity.DEBUG,
+    )
+    call(
+        ctx,
+        ['systemctl', 'reset-failed', service_name],
+        verbosity=CallVerbosity.DEBUG,
+    )
+    call(
+        ctx,
+        ['systemctl', 'disable', service_name],
+        verbosity=CallVerbosity.DEBUG,
+    )

From 2d8045d96f1a305b57a34ff96c130959821c393f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 11:50:29 -0500
Subject: [PATCH 1184/2492] cephadm: minor reorgs to command_rm_daemon

Make a few small chanes to command_rm_daemon to clean it up in
preparation for future changes. Move the construction of a
DaemonIdentity earlier and use the service name derived directly from it
if possible. I didn't remove the old approach of looking up a service
name in the listing because I don't know if doing so would break
old/adopted services and there's no unit test coverage for that.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 82e29d941eea..c44dfbecbde3 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4136,10 +4136,19 @@ def command_rm_daemon(ctx):
     lock = FileLock(ctx, ctx.fsid)
     lock.acquire()
 
-    (daemon_type, daemon_id) = ctx.name.split('.', 1)
-    unit_name = lookup_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
-
-    if daemon_type in ['mon', 'osd'] and not ctx.force:
+    ident = DaemonIdentity.from_context(ctx)
+    try:
+        # attempt a fast-path conversion that maps the fsid+name to
+        # the systemd service name, verifying that there is such a service
+        call_throws(ctx, ['systemctl', 'status', ident.service_name])
+        unit_name = ident.service_name
+    except RuntimeError:
+        # fall back to looking up all possible services that might match
+        # (JJM) Preserved this operation in case theres some backwards compat
+        # issues where the DaemonIdentity derived name is not correct.
+        unit_name = lookup_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+
+    if ident.daemon_type in ['mon', 'osd'] and not ctx.force:
         raise Error('must pass --force to proceed: '
                     'this command may destroy precious data!')
 
@@ -4151,21 +4160,21 @@ def command_rm_daemon(ctx):
          verbosity=CallVerbosity.DEBUG)
 
     # force remove rgw admin socket file if leftover
-    if daemon_type in ['rgw']:
+    if ident.daemon_type in ['rgw']:
         rgw_asok_path = f'/var/run/ceph/{ctx.fsid}/ceph-client.{ctx.name}.*.asok'
         call(ctx, ['rm', '-rf', rgw_asok_path],
              verbosity=CallVerbosity.DEBUG)
 
-    ident = DaemonIdentity(ctx.fsid, daemon_type, daemon_id)
     data_dir = ident.data_dir(ctx.data_dir)
-    if daemon_type in ['mon', 'osd', 'prometheus'] and \
+    if ident.daemon_type in ['mon', 'osd', 'prometheus'] and \
        not ctx.force_delete_data:
         # rename it out of the way -- do not delete
         backup_dir = os.path.join(ctx.data_dir, ctx.fsid, 'removed')
         if not os.path.exists(backup_dir):
             makedirs(backup_dir, 0, 0, DATA_DIR_MODE)
-        dirname = '%s.%s_%s' % (daemon_type, daemon_id,
-                                datetime.datetime.utcnow().strftime(DATEFMT))
+        dirname = '%s_%s' % (
+            ident.daemon_name, datetime.datetime.utcnow().strftime(DATEFMT)
+        )
         os.rename(data_dir,
                   os.path.join(backup_dir, dirname))
     else:

From 50928d56dd83b60526124abc37de8d7adf9cfe94 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 13:07:57 -0500
Subject: [PATCH 1185/2492] cephadm: replace some calls to rm with fs function
 calls

This should be a little cleaner, perhaps a tiny performance improvement,
and most importantly work better with the virtual-fs provided for the
unit tests and make this function more testable in the future.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c44dfbecbde3..99791a2de679 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4178,7 +4178,7 @@ def command_rm_daemon(ctx):
         os.rename(data_dir,
                   os.path.join(backup_dir, dirname))
     else:
-        call_throws(ctx, ['rm', '-rf', data_dir])
+        shutil.rmtree(data_dir, ignore_errors=True)
 
     endpoints = fetch_endpoints(ctx)
     ports: List[int] = [e.port for e in endpoints]

From ef857dce2daed842818645ba1a8dd195fb228b4a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 14:03:34 -0500
Subject: [PATCH 1186/2492] cephadm: add support to remove sidecar/init-ctr
 systemd units

Add support to the command_rm_daemon function to remove systemd services
and corresponding unit files for any sidecars or init-containers
associated with a primary daemon.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 99791a2de679..09a373dd27ed 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -105,6 +105,7 @@
     read_file,
     recursive_chown,
     touch,
+    unlink_file,
     write_new,
     write_tmp,
 )
@@ -132,7 +133,7 @@
     Highlight,
     LogDestination,
 )
-from cephadmlib.systemd import check_unit, check_units
+from cephadmlib.systemd import check_unit, check_units, terminate_service
 from cephadmlib import systemd_unit
 from cephadmlib import runscripts
 from cephadmlib.container_types import (
@@ -4152,12 +4153,22 @@ def command_rm_daemon(ctx):
         raise Error('must pass --force to proceed: '
                     'this command may destroy precious data!')
 
-    call(ctx, ['systemctl', 'stop', unit_name],
-         verbosity=CallVerbosity.DEBUG)
-    call(ctx, ['systemctl', 'reset-failed', unit_name],
-         verbosity=CallVerbosity.DEBUG)
-    call(ctx, ['systemctl', 'disable', unit_name],
-         verbosity=CallVerbosity.DEBUG)
+    terminate_service(ctx, unit_name)
+
+    # clean up any extra systemd unit files
+    sd_path_info = systemd_unit.sidecars_from_dropin(
+        systemd_unit.PathInfo(ctx.unit_dir, ident), missing_ok=True
+    )
+    for sc_unit in sd_path_info.sidecar_unit_files.values():
+        terminate_service(ctx, sc_unit.name)
+        unlink_file(sc_unit, missing_ok=True)
+    terminate_service(ctx, sd_path_info.init_ctr_unit_file.name)
+    unlink_file(sd_path_info.init_ctr_unit_file, missing_ok=True)
+    unlink_file(sd_path_info.drop_in_file, missing_ok=True)
+    try:
+        sd_path_info.drop_in_file.parent.rmdir()
+    except OSError:
+        pass
 
     # force remove rgw admin socket file if leftover
     if ident.daemon_type in ['rgw']:

From 0f7a48cc4dc620e388f87a083504664552705758 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 18 Nov 2023 19:09:29 -0500
Subject: [PATCH 1187/2492] cephadm: update _rm_cluster to call
 terminate_service

The terminate_service function was added to encapsulate the act of
terminating a systemd service. The rm-deamon handler was updated to use
this function previously; this commit updates rm-cluster function(s)
to do the same. This needed a bunch of test updates do to how the tests
were mocking commands.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py            | 14 +++-----------
 src/cephadm/tests/test_cephadm.py | 27 ++++++++++++++++++++-------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 09a373dd27ed..7b9ca0fd0ab6 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4282,14 +4282,6 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
     if not ctx.fsid:
         raise Error('must select the cluster to delete by passing --fsid to proceed')
 
-    def disable_systemd_service(unit_name: str) -> None:
-        call(ctx, ['systemctl', 'stop', unit_name],
-             verbosity=CallVerbosity.DEBUG)
-        call(ctx, ['systemctl', 'reset-failed', unit_name],
-             verbosity=CallVerbosity.DEBUG)
-        call(ctx, ['systemctl', 'disable', unit_name],
-             verbosity=CallVerbosity.DEBUG)
-
     logger.info(f'Deleting cluster with fsid: {ctx.fsid}')
 
     # stop + disable individual daemon units
@@ -4298,11 +4290,11 @@ def disable_systemd_service(unit_name: str) -> None:
             continue
         if d['style'] != 'cephadm:v1':
             continue
-        disable_systemd_service('ceph-%s@%s' % (ctx.fsid, d['name']))
+        terminate_service(ctx, 'ceph-%s@%s' % (ctx.fsid, d['name']))
 
     # cluster units
     for unit_name in ['ceph-%s.target' % ctx.fsid]:
-        disable_systemd_service(unit_name)
+        terminate_service(ctx, unit_name)
 
     slice_name = 'system-ceph\\x2d{}.slice'.format(ctx.fsid.replace('-', '\\x2d'))
     call(ctx, ['systemctl', 'stop', slice_name],
@@ -4333,7 +4325,7 @@ def disable_systemd_service(unit_name: str) -> None:
 
     # if last cluster on host remove shared files
     if get_ceph_cluster_count(ctx) == 0:
-        disable_systemd_service('ceph.target')
+        terminate_service(ctx, 'ceph.target')
 
         # rm shared ceph target files
         call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/multi-user.target.wants/ceph.target'])
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 7bca8b9cbf3c..9c6caf0092b7 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1345,7 +1345,9 @@ def _get_cmd(*args):
 
 ###############################################3
 
-    def test_config(self, cephadm_fs):
+    def test_config(self, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         conf_file = 'foo'
         cmd = self._get_cmd(
             '--mon-ip', '192.168.1.1',
@@ -1363,14 +1365,17 @@ def test_config(self, cephadm_fs):
             retval = _cephadm.command_bootstrap(ctx)
             assert retval == 0
 
-    def test_no_mon_addr(self, cephadm_fs):
+    def test_no_mon_addr(self, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         cmd = self._get_cmd()
         with with_cephadm_ctx(cmd) as ctx:
             msg = r'must specify --mon-ip or --mon-addrv'
             with pytest.raises(_cephadm.Error, match=msg):
                 _cephadm.command_bootstrap(ctx)
 
-    def test_skip_mon_network(self, cephadm_fs):
+    def test_skip_mon_network(self, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
         cmd = self._get_cmd('--mon-ip', '192.168.1.1')
 
         with with_cephadm_ctx(cmd, list_networks={}) as ctx:
@@ -1453,7 +1458,9 @@ def test_skip_mon_network(self, cephadm_fs):
                 True,
             ),
         ])
-    def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs):
+    def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         cmd = self._get_cmd('--mon-ip', mon_ip)
         if not result:
             with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
@@ -1515,7 +1522,9 @@ def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs):
                 None,
             ),
         ])
-    def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs):
+    def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         cmd = self._get_cmd('--mon-addrv', mon_addrv)
         if err:
             with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
@@ -1526,7 +1535,9 @@ def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs):
                 retval = _cephadm.command_bootstrap(ctx)
                 assert retval == 0
 
-    def test_allow_fqdn_hostname(self, cephadm_fs):
+    def test_allow_fqdn_hostname(self, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         hostname = 'foo.bar'
         cmd = self._get_cmd(
             '--mon-ip', '192.168.1.1',
@@ -1549,7 +1560,9 @@ def test_allow_fqdn_hostname(self, cephadm_fs):
             ('00000000-0000-0000-0000-0000deadbeef', None),
             ('00000000-0000-0000-0000-0000deadbeez', 'not an fsid'),
         ])
-    def test_fsid(self, fsid, err, cephadm_fs):
+    def test_fsid(self, fsid, err, cephadm_fs, funkypatch):
+        funkypatch.patch('cephadmlib.systemd.call')
+
         cmd = self._get_cmd(
             '--mon-ip', '192.168.1.1',
             '--skip-mon-network',

From 3d00ecf27a5a04aff1cee813efcf1f66ce1b45a7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 19 Nov 2023 07:56:46 -0500
Subject: [PATCH 1188/2492] cephadm: update rm cluster to remove
 sidecar/init-ctr services

A previous patch updated the rm-daemon functionality to remove sidecar
and init-container systemd services. This patch does the same for the
rm-cluster codepath.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 7b9ca0fd0ab6..5d5c2a72eb5a 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4285,12 +4285,22 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
     logger.info(f'Deleting cluster with fsid: {ctx.fsid}')
 
     # stop + disable individual daemon units
+    sd_paths = []
     for d in list_daemons(ctx, detail=False):
         if d['fsid'] != ctx.fsid:
             continue
         if d['style'] != 'cephadm:v1':
             continue
         terminate_service(ctx, 'ceph-%s@%s' % (ctx.fsid, d['name']))
+        # terminate sidecar & other supplemental services
+        ident = DaemonIdentity.from_name(ctx.fsid, d['name'])
+        sd_path_info = systemd_unit.sidecars_from_dropin(
+            systemd_unit.PathInfo(ctx.unit_dir, ident), missing_ok=True
+        )
+        for sc_unit in sd_path_info.sidecar_unit_files.values():
+            terminate_service(ctx, sc_unit.name)
+        terminate_service(ctx, sd_path_info.init_ctr_unit_file.name)
+        sd_paths.append(sd_path_info)
 
     # cluster units
     for unit_name in ['ceph-%s.target' % ctx.fsid]:
@@ -4305,12 +4315,16 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
         _zap_osds(ctx)
 
     # rm units
-    call_throws(ctx, ['rm', '-f', ctx.unit_dir
-                      + '/ceph-%s@.service' % ctx.fsid])
-    call_throws(ctx, ['rm', '-f', ctx.unit_dir
-                      + '/ceph-%s.target' % ctx.fsid])
-    call_throws(ctx, ['rm', '-rf',
-                      ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid])
+    for sd_path_info in sd_paths:
+        for sc_unit in sd_path_info.sidecar_unit_files.values():
+            unlink_file(sc_unit, missing_ok=True)
+        unlink_file(sd_path_info.init_ctr_unit_file, missing_ok=True)
+        shutil.rmtree(sd_path_info.drop_in_file.parent, ignore_errors=True)
+    unit_dir = Path(ctx.unit_dir)
+    unlink_file(unit_dir / f'ceph-{ctx.fsid}@.service', missing_ok=True)
+    unlink_file(unit_dir / f'ceph-{ctx.fsid}.target', missing_ok=True)
+    shutil.rmtree(unit_dir / f'ceph-{ctx.fsid}.target.wants', ignore_errors=True)
+
     # rm data
     call_throws(ctx, ['rm', '-rf', ctx.data_dir + '/' + ctx.fsid])
 

From 43fd3deb1d8b19e1ac14aeecfbafcb3ab7d424c7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 19 Nov 2023 15:39:52 -0500
Subject: [PATCH 1189/2492] cephadm: add a unit test to check sidecar removal

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 44 ++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index dadf3456fd5a..cfde3fbce0aa 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -488,3 +488,47 @@ def test_deploy_ceph_exporter_container(cephadm_fs, funkypatch):
         assert f.read() == 'XXXXXXX'
     with open(basedir / 'keyring') as f:
         assert f.read() == 'YYYYYY'
+
+
+def test_deploy_and_rm_iscsi(cephadm_fs, funkypatch):
+    # Test that the deploy and remove paths for iscsi (which has sidecar container)
+    # create and remove the correct unit files.
+    mocks = _common_patches(funkypatch)
+    _firewalld = mocks['Firewalld']
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'iscsi.wuzzy'
+        ctx.image = 'quay.io/ayeaye/iscsi:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'config': 'XXXXXXX',
+            'keyring': 'YYYYYY',
+            'files': {
+                'iscsi-gateway.cfg': 'portal',
+            },
+        }
+        _cephadm._common_deploy(ctx)
+
+    unit_dir = pathlib.Path('/etc/systemd/system')
+    assert unit_dir.is_dir()
+    assert (unit_dir / f'ceph-{fsid}@.service').exists()
+    drop_in = unit_dir / f'ceph-{fsid}@iscsi.wuzzy.service.d/99-cephadm.conf'
+    assert drop_in.parent.is_dir()
+    assert drop_in.exists()
+    assert 'tcmu' in drop_in.read_text()
+    tcmu_sidecar = unit_dir / f'ceph-{fsid}-sidecar@iscsi.wuzzy:tcmu.service'
+    assert tcmu_sidecar.exists()
+    assert 'sidecar-tcmu.run' in tcmu_sidecar.read_text()
+
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'iscsi.wuzzy'
+        ctx.image = 'quay.io/ayeaye/iscsi:latest'
+        _cephadm.command_rm_daemon(ctx)
+
+    assert not drop_in.exists()
+    assert not drop_in.parent.exists()
+    assert not tcmu_sidecar.exists()

From 133b23f13e434cf01f59d04d6b83cb03df05b43b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 19 Nov 2023 16:06:38 -0500
Subject: [PATCH 1190/2492] cephadm: remove a non-functional line in
 _rm_cluster

Remove a line from _rm_cluster that has no effect. The line uses
call_throws to execute an `rm -rf` command. The argument contains
asterisk chars that indicate that the file(s)/dir(s) to be created are
expected to match the given pattern. However, globs do not work in this
context in contemporary versions of cephadm.

To double check I added the following temporary unit test:
```
def test_does_it_glob(tmp_path):
    from cephadmlib.call_wrappers import call_throws

    d1 = (tmp_path / 'dir1')
    d1.mkdir()
    fns = ['f1.txt', 'f2.txt', 'f3.txt', 'f4']
    for fn in fns:
        with open(d1/fn, "w") as fh:
            fh.write("xxx")
    assert d1.exists()
    for fn in fns:
        assert (d1 / fn).exists()

    ctx = FakeContext()
    call_throws(ctx, ['rm', '-rf', f'{d1}/*.txt'])
    print('files:', os.listdir(d1))
    assert d1.exists()
    for fn in fns:
        if fn.endswith('.txt'):
            assert not (d1 / fn).exists()
        else:
            assert (d1 / fn).exists()
```

If globs worked in this context this test would have passed. It does
not. I confirmed that the current implementation of call/call_wrapper
does not execute the command in a shell context.

I wondered if it was possible that an earlier version of cephadm did
execute this command in a shell context and some changes along the way
changed the behavior. I tracked the origin of the line back to
16ebc620349f6e7c9afa6b992c85900f56fcfca3 the first change to implement
rm-cluster. In this commit the code uses subprocess.check_output
directly. I am familiar with check_output and unless `shell=True` is
passed this function doesn't execute the args in a shell context.  The
`shell=True` argument is not passed to check_output. This means that the
very first implementation of this line suffered from the same issue
- it would have no effect on any files except one named with actual
  asterisk (`*`) characters.

While I'm sure there was a good intention behind this line, the fact
that it persisted in the code so long in a non-functional state and no
one noticed in both production and qa testing makes me feel that it can
be safely removed with no negative effect. Removing the line simplifies
the code and avoids needing to spend effort unit-testing or manually
checking a fix.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5d5c2a72eb5a..23352651f27d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4331,8 +4331,6 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
     if not keep_logs:
         # rm logs
         call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid])
-        call_throws(ctx, ['rm', '-rf', ctx.log_dir
-                          + '/*.wants/ceph-%s@*' % ctx.fsid])
 
     # rm logrotate config
     call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid])

From 08e7a56ead5d3a241cd626d0436dff24867d3445 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 19 Nov 2023 16:35:06 -0500
Subject: [PATCH 1191/2492] cephadm: call functions instead of executing rm in
 rm_cluster

Convert a bunch of invocations of rm via a subprocess to function calls.
This should make it easier (or possible?) to test the function
in the unit test framework as well as possibly saving a few resources.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 23352651f27d..636c21615474 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4326,25 +4326,30 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
     shutil.rmtree(unit_dir / f'ceph-{ctx.fsid}.target.wants', ignore_errors=True)
 
     # rm data
-    call_throws(ctx, ['rm', '-rf', ctx.data_dir + '/' + ctx.fsid])
+    shutil.rmtree(Path(ctx.data_dir) / ctx.fsid, ignore_errors=True)
 
     if not keep_logs:
         # rm logs
-        call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid])
+        shutil.rmtree(Path(ctx.log_dir) / ctx.fsid, ignore_errors=True)
 
     # rm logrotate config
-    call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid])
+    unlink_file(
+        Path(ctx.logrotate_dir) / ('ceph-%s' % ctx.fsid), ignore_errors=True
+    )
 
     # if last cluster on host remove shared files
     if get_ceph_cluster_count(ctx) == 0:
         terminate_service(ctx, 'ceph.target')
 
         # rm shared ceph target files
-        call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/multi-user.target.wants/ceph.target'])
-        call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/ceph.target'])
+        unlink_file(
+            Path(ctx.unit_dir) / 'multi-user.target.wants/ceph.target',
+            ignore_errors=True
+        )
+        unlink_file(Path(ctx.unit_dir) / 'ceph.target', ignore_errors=True)
 
         # rm cephadm logrotate config
-        call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm'])
+        unlink_file(Path(ctx.logrotate_dir) / 'cephadm', ignore_errors=True)
 
         if not keep_logs:
             # remove all cephadm logs

From 57540a28c5c83140c2429f1e94fbaeee06d80f64 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 5 Dec 2023 16:20:57 -0500
Subject: [PATCH 1192/2492] cephadm: move init container script generation to a
 function

Move the generation of the init container run script to a small function
fixing a missing `set -e` along the way. This isolates the logic of
generating this run script.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/runscripts.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
index bfa73edbd7d1..52694b16b9c0 100644
--- a/src/cephadm/cephadmlib/runscripts.py
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -91,10 +91,7 @@ def write_service_scripts(
         # init-container commands
         if init_containers:
             initf = estack.enter_context(write_new(initctr_file_path))
-            _write_init_container_cmds_clean(ctx, initf, init_containers[0])
-            for idx, ic in enumerate(init_containers):
-                _write_init_container_cmds(ctx, initf, idx, ic)
-            initf.write('exit 0\n')
+            _write_init_containers_script(ctx, initf, init_containers)
 
         # sidecar container scripts
         for sidecar in sidecars or []:
@@ -220,6 +217,18 @@ def _write_stop_actions(
     )
 
 
+def _write_init_containers_script(
+    ctx: CephadmContext,
+    file_obj: IO[str],
+    init_containers: List[InitContainer],
+) -> None:
+    file_obj.write('set -e\n')
+    _write_init_container_cmds_clean(ctx, file_obj, init_containers[0])
+    for idx, ic in enumerate(init_containers):
+        _write_init_container_cmds(ctx, file_obj, idx, ic)
+    file_obj.write('exit 0\n')
+
+
 def _write_sidecar_script(
     ctx: CephadmContext,
     file_obj: IO[str],

From 11ccb59e8e1254305143c82efe8a3b2bce070ffe Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 11 Dec 2023 14:45:46 -0500
Subject: [PATCH 1193/2492] cephadm: convert init containers script to use a
 template

Convert the init containers run script to be based on a template, like
the sidecar run scripts are. The new script is loosely based on the
sidecars run script but only does actions in batches - logically
iterating over each init container configured.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py     |  3 +
 src/cephadm/cephadmlib/runscripts.py          | 53 ++++-------------
 .../templates/init_containers.run.j2          | 57 +++++++++++++++++++
 src/cephadm/cephadmlib/templating.py          |  1 +
 src/cephadm/tests/test_custom_container.py    | 23 ++++----
 5 files changed, 84 insertions(+), 53 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/templates/init_containers.run.j2

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 0bdbb5c80bbe..2af16eb47504 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -492,6 +492,9 @@ def run_cmd(self) -> List[str]:
     def rm_cmd(self, storage: bool = False) -> List[str]:
         return self.build_rm_cmd(storage=storage)
 
+    def stop_cmd(self, timeout: Optional[int] = None) -> List[str]:
+        return self.build_stop_cmd(timeout=timeout)
+
 
 class SidecarContainer(BasicContainer):
     @classmethod
diff --git a/src/cephadm/cephadmlib/runscripts.py b/src/cephadm/cephadmlib/runscripts.py
index 52694b16b9c0..b4f83ab3077f 100644
--- a/src/cephadm/cephadmlib/runscripts.py
+++ b/src/cephadm/cephadmlib/runscripts.py
@@ -161,44 +161,6 @@ def _write_container_cmd_to_bash(
     _bash_cmd(file_obj, container.run_cmd(), background=bool(background))
 
 
-def _write_init_container_cmds(
-    ctx: CephadmContext,
-    file_obj: IO[str],
-    index: int,
-    init_container: 'InitContainer',
-) -> None:
-    file_obj.write(f'# init container {index}: {init_container.cname}\n')
-    _bash_cmd(file_obj, init_container.run_cmd())
-    _write_init_container_cmds_clean(
-        ctx, file_obj, init_container, comment=''
-    )
-
-
-def _write_init_container_cmds_clean(
-    ctx: CephadmContext,
-    file_obj: IO[str],
-    init_container: 'InitContainer',
-    comment: str = 'init container cleanup',
-) -> None:
-    if comment:
-        assert '\n' not in comment
-        file_obj.write(f'# {comment}\n')
-    _bash_cmd(
-        file_obj,
-        init_container.rm_cmd(),
-        check=False,
-        stderr=False,
-    )
-    # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
-    if isinstance(ctx.container_engine, Podman):
-        _bash_cmd(
-            file_obj,
-            init_container.rm_cmd(storage=True),
-            check=False,
-            stderr=False,
-        )
-
-
 def _write_stop_actions(
     ctx: CephadmContext,
     f: TextIO,
@@ -221,12 +183,17 @@ def _write_init_containers_script(
     ctx: CephadmContext,
     file_obj: IO[str],
     init_containers: List[InitContainer],
+    comment: str = 'start and stop init containers',
 ) -> None:
-    file_obj.write('set -e\n')
-    _write_init_container_cmds_clean(ctx, file_obj, init_containers[0])
-    for idx, ic in enumerate(init_containers):
-        _write_init_container_cmds(ctx, file_obj, idx, ic)
-    file_obj.write('exit 0\n')
+    has_podman_engine = isinstance(ctx.container_engine, Podman)
+    templating.render_to_file(
+        file_obj,
+        ctx,
+        templating.Templates.init_ctr_run,
+        init_containers=init_containers,
+        comment=comment,
+        has_podman_engine=has_podman_engine,
+    )
 
 
 def _write_sidecar_script(
diff --git a/src/cephadm/cephadmlib/templates/init_containers.run.j2 b/src/cephadm/cephadmlib/templates/init_containers.run.j2
new file mode 100644
index 000000000000..b93b7ac1aef5
--- /dev/null
+++ b/src/cephadm/cephadmlib/templates/init_containers.run.j2
@@ -0,0 +1,57 @@
+#!/bin/sh
+# {{ comment }}
+
+set -e
+
+stop_all_init_containers() {
+    {%- for ic in init_containers %}
+    # stop init container {{ loop.index0 }}: {{ ic.cname }}
+    ! {{ ic.stop_cmd() | map('shellquote') | join(' ') }}
+    {%- endfor %}
+    return 0
+}
+
+rm_all_init_containers() {
+    {%- for ic in init_containers %}
+    # remove init container {{ loop.index0 }}: {{ ic.cname }}
+    ! {{ ic.rm_cmd() | map('shellquote') | join(' ') }} 2> /dev/null
+    {%- if has_podman_engine %}
+    ! {{ ic.rm_cmd(storage=True) | map('shellquote') | join(' ') }} 2> /dev/null
+    {%- endif %}
+    {%- endfor %}
+    return 0
+}
+
+has_running_init_container() {
+    {%- for ic in init_containers %}
+    if {{ ctx.container_engine.path }} inspect {{ ic.cname | shellquote }} &>/dev/null; then return 0; fi
+    {%- endfor %}
+    return 1
+}
+
+run_init_containers() {
+    {%- for ic in init_containers %}
+    # run init container {{ loop.index0 }}: {{ ic.cname }}
+    {{ ic.run_cmd() | map('shellquote') | join(' ') }}
+    # clean up {{ ic.cname }}
+    ! {{ ic.rm_cmd() | map('shellquote') | join(' ') }} 2> /dev/null
+    {%- if has_podman_engine %}
+    ! {{ ic.rm_cmd(storage=True) | map('shellquote') | join(' ') }} 2> /dev/null
+    {%- endif %}
+    {%- endfor %}
+    return 0
+}
+
+if [ "$1" = stop ] || [ "$1" = poststop ]; then
+    stop_all_init_containers
+    if has_running_init_container; then
+        exit 1
+    fi
+    exit 0
+fi
+
+# init container cleanup
+rm_all_init_containers
+
+run_init_containers
+exit 0
diff --git a/src/cephadm/cephadmlib/templating.py b/src/cephadm/cephadmlib/templating.py
index 4604d8c14c74..8c28cde57c27 100644
--- a/src/cephadm/cephadmlib/templating.py
+++ b/src/cephadm/cephadmlib/templating.py
@@ -28,6 +28,7 @@ class Templates(str, enum.Enum):
     cluster_logrotate_config = 'cluster.logrotate.config.j2'
     cephadm_logrotate_config = 'cephadm.logrotate.config.j2'
     sidecar_run = 'sidecar.run.j2'
+    init_ctr_run = 'init_containers.run.j2'
 
     def __str__(self) -> str:
         return self.value
diff --git a/src/cephadm/tests/test_custom_container.py b/src/cephadm/tests/test_custom_container.py
index fec435594a73..c185b0908df6 100644
--- a/src/cephadm/tests/test_custom_container.py
+++ b/src/cephadm/tests/test_custom_container.py
@@ -226,12 +226,15 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
 
         idx = icfile_lines.index('# init container cleanup')
         assert idx >= 0
-        assert icfile_lines[idx + 1].startswith('! /usr/bin/podman rm')
-        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
+        assert any(
+            l.strip().startswith('! /usr/bin/podman rm')
+            for l in icfile_lines
+        )
 
-        idx = icfile_lines.index('# init container 0: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
+        slines = [l.strip() for l in icfile_lines]
+        idx = slines.index('# run init container 0: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
         assert idx > 0
-        assert icfile_lines[idx + 1] == (
+        assert slines[idx + 1] == (
             '/usr/bin/podman run'
             ' --stop-signal=SIGTERM'
             ' --entrypoint /usr/local/bin/prepare.sh'
@@ -240,12 +243,12 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
             ' -v /var/lib/ceph/b01dbeef-701d-9abe-0000-e1e5a47004a7/container.tdccai/data1:/var/lib/myapp'
             ' quay.io/foobar/quux:latest'
         )
-        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
-        assert icfile_lines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert slines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert slines[idx + 4].startswith('! /usr/bin/podman rm')
 
-        idx = icfile_lines.index('# init container 1: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
+        idx = slines.index('# run init container 1: ceph-b01dbeef-701d-9abe-0000-e1e5a47004a7-container-tdccai-init')
         assert idx > 0
-        assert icfile_lines[idx + 1] == (
+        assert slines[idx + 1] == (
             '/usr/bin/podman run'
             ' --stop-signal=SIGTERM'
             ' --entrypoint /usr/local/bin/populate.sh'
@@ -256,5 +259,5 @@ def test_deploy_custom_container_and_inits(cephadm_fs):
             ' quay.io/foobar/quux:latest'
             ' --source=https://my.cool.example.com/samples/geo.1.txt'
         )
-        assert icfile_lines[idx + 2].startswith('! /usr/bin/podman rm')
-        assert icfile_lines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert slines[idx + 3].startswith('! /usr/bin/podman rm')
+        assert slines[idx + 4].startswith('! /usr/bin/podman rm')

From e31ef9a379c63e62c7b6887228f41dbf1f49a204 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 11 Dec 2023 15:44:41 -0500
Subject: [PATCH 1194/2492] cephadm: add stop and poststop commands to init
 container service

Now that there is a more robust init_containers.run script, we can use
the new stop and poststop subcommands from the systemd unit.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templates/init_ctr.service.j2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/cephadmlib/templates/init_ctr.service.j2 b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
index 1562a5b18183..730d5043dece 100644
--- a/src/cephadm/cephadmlib/templates/init_ctr.service.j2
+++ b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
@@ -17,6 +17,8 @@ LimitNOFILE=1048576
 LimitNPROC=1048576
 EnvironmentFile=-/etc/environment
 ExecStart=/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run
+ExecStop=/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run stop
+ExecStopPost=-/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run  poststop
 Restart=on-failure
 RestartSec=10s
 TimeoutStopSec=120

From aaffa661681f7156b325a894f2549416f265d679 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 19 Dec 2023 14:10:39 -0500
Subject: [PATCH 1195/2492] cehphadm: disable restarting init containers
 service

Disable the Restart= line in the init containers systemd template.
It is commented out as there are probably more appropriate workarounds
to use in the future but this should fix testing on centos8 based
systems.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/templates/init_ctr.service.j2 | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/templates/init_ctr.service.j2 b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
index 730d5043dece..e23a72be40f5 100644
--- a/src/cephadm/cephadmlib/templates/init_ctr.service.j2
+++ b/src/cephadm/cephadmlib/templates/init_ctr.service.j2
@@ -19,8 +19,12 @@ EnvironmentFile=-/etc/environment
 ExecStart=/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run
 ExecStop=/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run stop
 ExecStopPost=-/bin/bash {{ ctx.data_dir }}/{{ identity.fsid }}/%i/init_containers.run  poststop
-Restart=on-failure
-RestartSec=10s
+# FIXME: Disable Restart on oneshot service. systemd versions before v224
+# did not allow Restart=on-failure with a oneshot service. Having it set
+# prevents the service from starting on centos8. Disable it for now and
+# revisit this at a later time.
+#Restart=on-failure
+#RestartSec=10s
 TimeoutStopSec=120
 StartLimitInterval=30min
 StartLimitBurst=5

From dfb4b26a715bf2cc20e116c9bee4dad507383f3b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 22 Nov 2023 03:12:12 +0000
Subject: [PATCH 1196/2492] PendingReleaseNotes: add release note for 62338

See https://tracker.ceph.com/issues/62338 and
2fc5486e.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 PendingReleaseNotes | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 3ae291f0ecb7..8041ab81ab75 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -91,6 +91,14 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
 * RADOS: `get_pool_is_selfmanaged_snaps_mode` C++ API has been deprecated
   due to being prone to false negative results.  It's safer replacement is
   `pool_is_in_selfmanaged_snaps_mode`.
+* RADOS: For bug 62338 (https://tracker.ceph.com/issues/62338), we did not choose
+  to condition the fix on a server flag in order to simplify backporting.  As
+  a result, in rare cases it may be possible for a PG to flip between two acting
+  sets while an upgrade to a version with the fix is in progress.  If you observe
+  this behavior, you should be able to work around it by completing the upgrade or
+  by disabling async recovery by setting osd_async_recovery_min_cost to a very
+  large value on all OSDs until the upgrade is complete:
+  ``ceph config set osd osd_async_recovery_min_cost 1099511627776``
 
 >=18.0.0
 

From 8028e2c5fce1cf000647b5214c05b32f38ae91b7 Mon Sep 17 00:00:00 2001
From: cuiming_yewu <cuiming_yewu@cmss.chinamobile.com>
Date: Fri, 20 Oct 2023 12:59:05 +0800
Subject: [PATCH 1197/2492] mon&test: resolve warning about inconsistent
 variable types

resolve a warning issue about inconsistent variable types with
Building CXX object src/mon/CMakeFiles/mon.dir/Elector.cc.o
../src/mon/Elector.cc: In member function 'void Elector::
notify_rank_removed(int)':
../src/mon/Elector.cc:765:15: warning: comparison of integer
expressions of different signedness: 'int' and 'unsigned int'
[-Wsign-compare] if (rank_removed < paxos_size()) {
warning:comparison of integer expressions of different
signedness: 'unsigned int' and 'const int'
[-Wsign-compare] if (total < NR_ROUNDS)

Fixes: https://tracker.ceph.com/issues/63917

Signed-off-by: cuiming <cuiming_yewu@cmss.chinamobile.com>
---
 src/mon/Elector.cc                 | 2 +-
 src/test/common/test_fair_mutex.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc
index a7221ebfb196..5ad30ff225c8 100644
--- a/src/mon/Elector.cc
+++ b/src/mon/Elector.cc
@@ -762,7 +762,7 @@ void Elector::notify_rank_removed(unsigned rank_removed, unsigned new_rank)
      In the case where we are removing the highest rank,
      we erase the removed rank from all sets.
    */
-  if (rank_removed < paxos_size()) {
+  if (std::cmp_less(rank_removed, paxos_size())) {
     for (unsigned i = rank_removed + 1; i <= paxos_size() ; ++i) {
       if (live_pinging.count(i)) {
         dead_pinging.erase(i-1);
diff --git a/src/test/common/test_fair_mutex.cc b/src/test/common/test_fair_mutex.cc
index 10ba835a2ddb..4b9997706ba7 100644
--- a/src/test/common/test_fair_mutex.cc
+++ b/src/test/common/test_fair_mutex.cc
@@ -47,7 +47,7 @@ TEST(FairMutex, fair)
                                        scoreboard.end(),
                                        0);
       for (unsigned score : scoreboard) {
-        if (total < NR_ROUNDS) {
+        if (std::cmp_less(total, NR_ROUNDS)) {
           // not quite statistically significant. to reduce the false positive,
           // just consider it fair
           continue;

From dd4589258a2c7e2061d9a096bf754c23bf848bb9 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 19 Dec 2023 17:33:38 +0000
Subject: [PATCH 1198/2492] rgw/notifications/lc: handle publish_commit()
 failures as warnings

Fixes: https://tracker.ceph.com/issues/63859

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/rgw_lc.cc | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index b857cc7a82f1..f117aaa81f84 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -574,13 +574,12 @@ static int remove_expired_obj(
       fmt::format("ERROR: {} failed, with error: {}", __func__, ret) << dendl;
   } else {
     // send request to notification manager
-    ret = notify->publish_commit(dpp, obj_state->size,
+    int publish_ret = notify->publish_commit(dpp, obj_state->size,
 				 ceph::real_clock::now(),
 				 obj_state->attrset[RGW_ATTR_ETAG].to_str(),
 				 version_id);
-    if (ret < 0) {
-      ldpp_dout(dpp, 1) << "ERROR: notify publish_commit failed, with error: "
-												<< ret << dendl;
+    if (publish_ret < 0) {
+      ldpp_dout(dpp, 5) << "WARNING: notify publish_commit failed, with error: " << publish_ret << dendl;
     }
   }
 
@@ -860,13 +859,13 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 
       ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
-        ret = notify->publish_commit(
+        int publish_ret = notify->publish_commit(
             this, sal_obj->get_obj_size(), ceph::real_clock::now(),
             sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 						version_id);
-        if (ret < 0) {
-          ldpp_dout(wk->get_lc(), 1)
-              << "ERROR: notify publish_commit failed, with error: " << ret
+        if (publish_ret < 0) {
+          ldpp_dout(wk->get_lc(), 5)
+              << "WARNING: notify publish_commit failed, with error: " << publish_ret
               << dendl;
         }
         if (perfcounter) {
@@ -1366,13 +1365,13 @@ class LCOpAction_Transition : public LCOpAction {
       return ret;
     } else {
       // send request to notification manager
-      ret =  notify->publish_commit(oc.dpp, obj->get_obj_size(),
+      int publish_ret = notify->publish_commit(oc.dpp, obj->get_obj_size(),
 				    ceph::real_clock::now(),
 				    obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
 				    version_id);
-      if (ret < 0) {
-	ldpp_dout(oc.dpp, 1) <<
-	  "ERROR: notify publish_commit failed, with error: " << ret << dendl;
+      if (publish_ret < 0) {
+	ldpp_dout(oc.dpp, 5) <<
+	  "WARNING: notify publish_commit failed, with error: " << publish_ret << dendl;
       }
     }
 

From 4df348c17ba0c72d2ecf08d3607b70b3ae253176 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 7 Jun 2023 10:33:13 -0400
Subject: [PATCH 1199/2492] mgr/cephadm: Also catch
 concurrent.futures.TimeoutError for timeouts

On python 3.6 which Ceph currently uses for its
container builds (which are based on centos 8 stream builds
hence the python version) the exception raised by a timeout
from a concurrent.futures.Future is successfully caught by
looking for asyncio.TimeoutError. However, in builds with
later python versions, e.g. 3.9.16, the timeout is no
longer caught. This results in situations like

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/utils.py", line 79, in do_work
    return f(*arg)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 241, in refresh
    r = self._refresh_host_devices(host)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 352, in _refresh_host_devices
    devices = self.mgr.wait_async(self._run_cephadm_json(
  File "/usr/share/ceph/mgr/cephadm/module.py", line 635, in wait_async
    return self.event_loop.get_result(coro, timeout)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 63, in get_result
    return future.result(timeout)
  File "/lib64/python3.9/concurrent/futures/_base.py", line 448, in result
    raise TimeoutError()
concurrent.futures._base.TimeoutError

which causes the cephadm module to crash whenever one of these
command timeouts happen. This patch is to also catch the
newer exception type so it works on later python versions as well

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 3 ++-
 src/pybind/mgr/cephadm/ssh.py    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index de0f49fb2b42..46848add150d 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1,4 +1,5 @@
 import asyncio
+import concurrent
 import json
 import errno
 import ipaddress
@@ -724,7 +725,7 @@ def async_timeout_handler(self, host: Optional[str] = '',
         # are provided, that will be included in the OrchestratorError's message
         try:
             yield
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             err_str: str = ''
             if cmd:
                 err_str = f'Command "{cmd}" timed out '
diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index d17cc0fcc198..7460fc159d7b 100644
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import asyncio
+import concurrent
 from tempfile import NamedTemporaryFile
 from threading import Thread
 from contextlib import contextmanager
@@ -61,7 +62,7 @@ def get_result(self, coro: Awaitable[T], timeout: Optional[int] = None) -> T:
         future = asyncio.run_coroutine_threadsafe(coro, self._loop)
         try:
             return future.result(timeout)
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             # try to cancel the task before raising the exception further up
             future.cancel()
             raise

From efb69ee350d1c20bdbfc88db8a01ec88761bd99f Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Sat, 4 Nov 2023 18:45:17 -0400
Subject: [PATCH 1200/2492] qa/cephadm: add test for cephadm asyncio based
 timeout

Adds a test that will set the default cephadm command
timeout and then force a timeout to occur by holding
the cephadm lock and triggering a device refresh.
This works because cephadm ceph-volume commands
require the cephadm lock to run, so the command will
timeout waiting for the lock to become available.

Signed-off-by: Adam King <adking@redhat.com>
---
 .../workunits/task/test_cephadm_timeout.yaml  |  13 ++
 qa/workunits/cephadm/test_cephadm_timeout.py  | 179 ++++++++++++++++++
 2 files changed, 192 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml
 create mode 100755 qa/workunits/cephadm/test_cephadm_timeout.py

diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml b/qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml
new file mode 100644
index 000000000000..24b53d029123
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml
@@ -0,0 +1,13 @@
+roles:
+- - host.a
+  - mon.a
+  - mgr.a
+  - osd.0
+  - client.0
+tasks:
+- install:
+- cephadm:
+- workunit:
+    clients:
+      client.0:
+        - cephadm/test_cephadm_timeout.py
\ No newline at end of file
diff --git a/qa/workunits/cephadm/test_cephadm_timeout.py b/qa/workunits/cephadm/test_cephadm_timeout.py
new file mode 100755
index 000000000000..67b43a2dfc64
--- /dev/null
+++ b/qa/workunits/cephadm/test_cephadm_timeout.py
@@ -0,0 +1,179 @@
+#!/usr/bin/python3 -s
+
+import time
+import os
+import fcntl
+import subprocess
+import uuid
+import sys
+
+from typing import Optional, Any
+
+LOCK_DIR = '/run/cephadm'
+DATA_DIR = '/var/lib/ceph'
+
+class _Acquire_ReturnProxy(object):
+    def __init__(self, lock: 'FileLock') -> None:
+        self.lock = lock
+        return None
+
+    def __enter__(self) -> 'FileLock':
+        return self.lock
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.lock.release()
+        return None
+
+class FileLock(object):
+    def __init__(self, name: str, timeout: int = -1) -> None:
+        if not os.path.exists(LOCK_DIR):
+            os.mkdir(LOCK_DIR, 0o700)
+        self._lock_file = os.path.join(LOCK_DIR, name + '.lock')
+
+        self._lock_file_fd: Optional[int] = None
+        self.timeout = timeout
+        self._lock_counter = 0
+        return None
+
+    @property
+    def is_locked(self) -> bool:
+        return self._lock_file_fd is not None
+
+    def acquire(self, timeout: Optional[int] = None, poll_intervall: float = 0.05) -> _Acquire_ReturnProxy:
+        # Use the default timeout, if no timeout is provided.
+        if timeout is None:
+            timeout = self.timeout
+
+        # Increment the number right at the beginning.
+        # We can still undo it, if something fails.
+        self._lock_counter += 1
+
+        start_time = time.time()
+        try:
+            while True:
+                if not self.is_locked:
+                    self._acquire()
+
+                if self.is_locked:
+                    break
+                elif timeout >= 0 and time.time() - start_time > timeout:
+                    raise Exception(self._lock_file)
+                else:
+                    time.sleep(poll_intervall)
+        except Exception:
+            # Something did go wrong, so decrement the counter.
+            self._lock_counter = max(0, self._lock_counter - 1)
+
+            raise
+        return _Acquire_ReturnProxy(lock=self)
+
+    def release(self, force: bool = False) -> None:
+        if self.is_locked:
+            self._lock_counter -= 1
+
+            if self._lock_counter == 0 or force:
+                self._release()
+                self._lock_counter = 0
+
+        return None
+
+    def __enter__(self) -> 'FileLock':
+        self.acquire()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.release()
+        return None
+
+    def __del__(self) -> None:
+        self.release(force=True)
+        return None
+
+    def _acquire(self) -> None:
+        open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
+        fd = os.open(self._lock_file, open_mode)
+
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except (IOError, OSError):
+            os.close(fd)
+        else:
+            self._lock_file_fd = fd
+        return None
+
+    def _release(self) -> None:
+        fd = self._lock_file_fd
+        self._lock_file_fd = None
+        fcntl.flock(fd, fcntl.LOCK_UN)  # type: ignore
+        os.close(fd)  # type: ignore
+        return None
+
+def _is_fsid(s):
+    try:
+        uuid.UUID(s)
+    except ValueError:
+        return False
+    return True
+
+def find_fsid():
+    if not os.path.exists(DATA_DIR):
+        raise Exception(f'{DATA_DIR} does not exist. Aborting...')
+
+    for d in os.listdir(DATA_DIR):
+        # assume the first thing we find that is an fsid
+        # is what we want. Not expecting multiple clusters
+        # to have been installed here.
+        if _is_fsid(d):
+            return d
+    raise Exception(f'No fsid dir found in {DATA_DIR} does not exist. Aborting...')
+
+def main():
+    print('Looking for cluster fsid...')
+    fsid = find_fsid()
+    print(f'Found fsid {fsid}')
+
+    print('Setting cephadm command timeout to 120...')
+    subprocess.run(['cephadm', 'shell', '--', 'ceph', 'config', 'set',
+                    'mgr', 'mgr/cephadm/default_cephadm_command_timeout', '120'],
+                    check=True)
+
+    print('Taking hold of cephadm lock for 300 seconds...')
+    lock = FileLock(fsid, 300)
+    lock.acquire()
+
+    print('Triggering cephadm device refresh...')
+    subprocess.run(['cephadm', 'shell', '--', 'ceph', 'orch', 'device', 'ls', '--refresh'],
+                    check=True)
+
+    print('Sleeping 150 seconds to allow for timeout to occur...')
+    time.sleep(150)
+
+    print('Checking ceph health detail...')
+    # directing stdout to res.stdout via "capture_stdout" option
+    # (and same for stderr) seems to have been added in python 3.7.
+    # Using files so this works with 3.6 as well
+    with open('/tmp/ceph-health-detail-stdout', 'w') as f_stdout:
+        with open('/tmp/ceph-health-detail-stderr', 'w') as f_stderr:
+            subprocess.run(['cephadm', 'shell', '--', 'ceph', 'health', 'detail'],
+                           check=True, stdout=f_stdout, stderr=f_stderr)
+
+    res_stdout = open('/tmp/ceph-health-detail-stdout', 'r').read()
+    res_stderr = open('/tmp/ceph-health-detail-stderr', 'r').read()
+    print(f'"cephadm shell -- ceph health detail" stdout:\n{res_stdout}')
+    print(f'"cephadm shell -- ceph health detail" stderr:\n{res_stderr}')
+
+    print('Checking for correct health warning in health detail...')
+    if 'CEPHADM_REFRESH_FAILED' not in res_stdout:
+        raise Exception('No health warning caused by timeout was raised')
+    if 'Command "cephadm ceph-volume -- inventory" timed out' not in res_stdout:
+        raise Exception('Health warnings did not contain message about time out')
+
+    print('Health warnings found succesfully. Exiting.')
+    return 0
+
+    
+if __name__ == '__main__':
+    if os.getuid() != 0:
+        print('Trying to run myself with sudo...')
+        os.execvp('sudo', [sys.executable] + list(sys.argv))
+    main()

From e917d66838b2ea52538f4460d89d633ce7786067 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 3 Jan 2024 08:35:05 -0500
Subject: [PATCH 1201/2492] mgr/cephadm: make default command timeout field an
 int

When the fields was "secs" instead, we could hit

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 1380, in _run_cephadm_json
    out, err, code = await self._run_cephadm(
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 1525, in _run_cephadm
    raise OrchestratorError(
orchestrator._interface.OrchestratorError: cephadm exited with an error code: 2, stderr: usage: cephadm
       [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR]
       [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR]
       [--sysctl-dir SYSCTL_DIR] [--unit-dir UNIT_DIR] [--verbose]
       [--timeout TIMEOUT] [--retry RETRY] [--env ENV] [--no-container-init]
       [--no-cgroups-split]
       {version,pull,inspect-image, . . .
       ...
cephadm: error: argument --timeout: invalid int value: '295.0'

where the value ends up as a floating point value
after converting to a string (which is necessary to actually
pass it to the binary). By setting the field to be an
int, we should be able to avoid this.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 46848add150d..b8f9d648ed4f 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -475,7 +475,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
         ),
         Option(
             'default_cephadm_command_timeout',
-            type='secs',
+            type='int',
             default=15 * 60,
             desc='Default timeout applied to cephadm commands run directly on '
             'the host (in seconds)'

From d02178fdbd5707c6469fdd678f42a4575a3c10b4 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 4 Jan 2024 00:20:51 +1000
Subject: [PATCH 1202/2492] doc/releases: edit reef.rst

Make minor corrections to doc/releases/reef.rst. These corrections were
suggested by Anthony D'Atri in https://github.com/ceph/ceph/pull/55049.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/releases/reef.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
index 378c64beb11b..4eff836fd3eb 100644
--- a/doc/releases/reef.rst
+++ b/doc/releases/reef.rst
@@ -65,13 +65,12 @@ Notable Changes
 * mgr/snap-schedule: For clusters with multiple CephFS file systems, all the
   snap-schedule commands now expect the '--fs' argument.
 
-* RADOS: A POOL_APP_NOT_ENABLED health warning will now be reported if
-  the application is not enabled for the pool irrespective of whether
-  the pool is in use or not. Always tag a pool with an application
-  using ``ceph osd pool application enable`` command to avoid reporting
-  of POOL_APP_NOT_ENABLED health warning for that pool.
-  The user might temporarily mute this warning using
-  ``ceph health mute POOL_APP_NOT_ENABLED``.
+* RADOS: A ``POOL_APP_NOT_ENABLED`` health warning will now be reported if the
+  application is not enabled for the pool whether the pool is in use or not.
+  Always tag a pool with an application using ``ceph osd pool application
+  enable`` command to avoid reporting ``POOL_APP_NOT_ENABLED`` for that pool.
+  The user might temporarily mute this warning using ``ceph health mute
+  POOL_APP_NOT_ENABLED``.
 
 * Dashboard: An overview page for RGW to show the overall status of RGW components.
 

From f62e93cbe73cd8f624a6c99497051c1a0aaf3ab6 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 3 Jan 2024 18:41:51 +1000
Subject: [PATCH 1203/2492] doc/radosgw: edit "Add/Remove a Key"

Edit the section "Add/Remove a Key" in doc/radosgw/admin.rst. Each
operation (e.g. "Adding an S3 key pair for a user", "Removing an S3 key
pair for a user") now has its own subsection. This increased granularity
should make it easier in the future to link to each of these specific
operations, if needed.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 70 ++++++++++++++++++++++++++++++-------------
 1 file changed, 49 insertions(+), 21 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index f108d58d7120..f922ad2b8df6 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -240,25 +240,30 @@ Options include:
   with the UID.
 
 
-Add / Remove a Key
-------------------------
+Add or  Remove a Key
+--------------------
 
-Both users and subusers require the key to access the S3 or Swift interface. To
-use S3, the user needs a key pair which is composed of an access key and a 
-secret key. On the other hand, to use Swift, the user typically needs a secret 
-key (password), and use it together with the associated user ID. You may create
-a key and either specify or generate the access key and/or secret key. You may 
-also remove a key. Options include:
+Both users and subusers require a key to access the S3 or Swift interface. To
+use S3, the user needs a key pair which is composed of an access key and a
+secret key. To use Swift, the user needs a secret key (password), which is used
+together with its associated user ID. You can create a key and either specify
+or generate the access key or secret key. You can also remove a key. Options
+include:
 
-- ``--key-type=<type>`` specifies the key type. The options are: s3, swift
+- ``--key-type=<type>`` specifies the key type. The options are: ``s3``, ``swift``
 - ``--access-key=<key>`` manually specifies an S3 access key.
 - ``--secret-key=<key>`` manually specifies a S3 secret key or a Swift secret key.
 - ``--gen-access-key`` automatically generates a random S3 access key.
 - ``--gen-secret`` automatically generates a random S3 secret key or a random Swift secret key.
 
-An example how to add a specified S3 key pair for a user. ::
+Adding S3 keys
+~~~~~~~~~~~~~~
+
+To add a specific S3 key pair for a user, run a command of the following form:
+
+.. prompt:: bash
 
-	radosgw-admin key create --uid=foo --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey
+   radosgw-admin key create --uid=foo --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey
 
 .. code-block:: javascript
 
@@ -273,11 +278,17 @@ An example how to add a specified S3 key pair for a user. ::
         "secret_key": "fooSecretKey"}],
   }
 
-Note that you may create multiple S3 key pairs for a user.
+.. note:: You can create multiple S3 key pairs for a user.
 
-To attach a specified swift secret key for a subuser. ::
+Adding Swift secret keys
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-	radosgw-admin key create --subuser=foo:bar --key-type=swift --secret-key barSecret
+To attach a specific Swift secret key for a subuser, run a command of the
+following form:
+
+.. prompt:: bash
+
+   radosgw-admin key create --subuser=foo:bar --key-type=swift --secret-key barSecret
 
 .. code-block:: javascript
 
@@ -293,11 +304,18 @@ To attach a specified swift secret key for a subuser. ::
       { "user": "foo:bar",
         "secret_key": "asfghjghghmgm"}]}
 
-Note that a subuser can have only one swift secret key.
+.. note:: A subuser can have only one Swift secret key.
+
+Associating subusers with S3 key pairs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Subusers can also be used with S3 APIs if the subuser is associated with a S3 key pair. ::	
+Subusers can also be used with S3 APIs if the subuser is associated with a S3
+key pair. To associate a subuser with an S3 key pair, run a command of the
+following form:
 
-	radosgw-admin key create --subuser=foo:bar --key-type=s3 --access-key barAccessKey --secret-key barSecretKey
+.. prompt:: bash
+
+   radosgw-admin key create --subuser=foo:bar --key-type=s3 --access-key barAccessKey --secret-key barSecretKey
 	
 .. code-block:: javascript
 
@@ -316,13 +334,23 @@ Subusers can also be used with S3 APIs if the subuser is associated with a S3 ke
   }
 
 
-To remove a S3 key pair, specify the access key. :: 
+Removing S3 key pairs
+~~~~~~~~~~~~~~~~~~~~~
+
+To remove a S3 key pair, specify the access key to be removed. Run a command of the following form: 
+
+.. prompt:: bash
+
+   radosgw-admin key rm --uid=foo --key-type=s3 --access-key=fooAccessKey 
 
-	radosgw-admin key rm --uid=foo --key-type=s3 --access-key=fooAccessKey 
+Removing Swift secret keys
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To remove the swift secret key. ::
+To remove a Swift secret key, run a command of the following form: 
+
+.. prompt:: bash
 
-	radosgw-admin key rm --subuser=foo:bar --key-type=swift
+   radosgw-admin key rm --subuser=foo:bar --key-type=swift
 
 
 Add / Remove Admin Capabilities

From 6e0af1d00df841101b032959fdbf6e6c6194870e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 3 Jan 2024 09:47:47 -0500
Subject: [PATCH 1204/2492] cepahdm: add some specific tests for daemon id
 systemd name generation

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_util_funcs.py | 34 ++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/src/cephadm/tests/test_util_funcs.py b/src/cephadm/tests/test_util_funcs.py
index c52852db679f..6b5380711f33 100644
--- a/src/cephadm/tests/test_util_funcs.py
+++ b/src/cephadm/tests/test_util_funcs.py
@@ -872,3 +872,37 @@ def test_daemon_sub_identity_from_service_invalid():
     service_name = 'random-task@elsewise.service'
     with pytest.raises(ValueError):
         DaemonSubIdentity.from_service_name(service_name)
+
+
+def test_daemon_id_systemd_names():
+    from cephadmlib.daemon_identity import DaemonIdentity
+
+    di = DaemonIdentity(
+        '244c9842-866b-11ee-80ad-3497f6318048', 'test', 'foo.bar'
+    )
+    assert (
+        di.unit_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048@test.foo.bar'
+    )
+    assert (
+        di.service_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048@test.foo.bar.service'
+    )
+    assert (
+        di.init_service_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048-init@test.foo.bar.service'
+    )
+
+
+def test_daemon_sub_id_systemd_names():
+    from cephadmlib.daemon_identity import DaemonSubIdentity
+
+    dsi = DaemonSubIdentity(
+        '244c9842-866b-11ee-80ad-3497f6318048', 'test', 'foo.bar', 'quux',
+    )
+    assert (
+        dsi.sidecar_service_name
+        == 'ceph-244c9842-866b-11ee-80ad-3497f6318048-sidecar@test.foo.bar:quux.service'
+    )
+    with pytest.raises(ValueError):
+        dsi.service_name

From 169bd8553ed5c0985e46c5da366a07965bc362e6 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Wed, 3 Jan 2024 16:32:06 +0100
Subject: [PATCH 1205/2492] os/bluestore: remove zoned namespace support

Lately we've been adding a lot of commits that could've interfered with
smr support but since no one is actively reviewing/supporting smr in
bluestore, it doesn't make sense for us to mantain it.

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/common/options/global.yaml.in             |   2 -
 src/os/CMakeLists.txt                         |   6 -
 src/os/bluestore/Allocator.cc                 |   7 -
 src/os/bluestore/Allocator.h                  |   2 -
 src/os/bluestore/BitmapFreelistManager.cc     |   1 -
 src/os/bluestore/BitmapFreelistManager.h      |   1 -
 src/os/bluestore/BlueFS.cc                    |   1 -
 src/os/bluestore/BlueStore.cc                 | 966 ++----------------
 src/os/bluestore/BlueStore.h                  |  66 --
 src/os/bluestore/FreelistManager.cc           |  11 -
 src/os/bluestore/FreelistManager.h            |   1 -
 src/os/bluestore/ZonedAllocator.cc            | 240 -----
 src/os/bluestore/ZonedAllocator.h             | 120 ---
 src/os/bluestore/ZonedFreelistManager.cc      | 372 -------
 src/os/bluestore/ZonedFreelistManager.h       | 113 --
 src/os/bluestore/zoned_types.h                |  66 --
 .../objectstore/run_smr_bluestore_test.sh     |  48 -
 src/test/objectstore/store_test.cc            | 174 ----
 src/vstart.sh                                 |   5 -
 19 files changed, 76 insertions(+), 2126 deletions(-)
 delete mode 100644 src/os/bluestore/ZonedAllocator.cc
 delete mode 100644 src/os/bluestore/ZonedAllocator.h
 delete mode 100644 src/os/bluestore/ZonedFreelistManager.cc
 delete mode 100644 src/os/bluestore/ZonedFreelistManager.h
 delete mode 100644 src/os/bluestore/zoned_types.h
 delete mode 100644 src/test/objectstore/run_smr_bluestore_test.sh

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 286a1126a467..f4a92295e48a 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4897,7 +4897,6 @@ options:
   - stupid
   - avl
   - hybrid
-  - zoned
   with_legacy: true
 - name: bluestore_freelist_blocks_per_key
   type: size
@@ -6326,7 +6325,6 @@ options:
   - aio
   - spdk
   - pmem
-  - hm_smr
 - name: bluestore_cleaner_sleep_interval
   type: float
   level: advanced
diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt
index 55415fb37228..1cd85d3b213c 100644
--- a/src/os/CMakeLists.txt
+++ b/src/os/CMakeLists.txt
@@ -27,12 +27,6 @@ if(WITH_BLUESTORE)
   )
 endif(WITH_BLUESTORE)
 
-if(WITH_ZBD)
-  list(APPEND libos_srcs
-    bluestore/ZonedFreelistManager.cc
-    bluestore/ZonedAllocator.cc)
-endif()
-
 if(WITH_FUSE)
   list(APPEND libos_srcs
     FuseStore.cc)
diff --git a/src/os/bluestore/Allocator.cc b/src/os/bluestore/Allocator.cc
index 5c5b8db70ecc..7029420b53d0 100644
--- a/src/os/bluestore/Allocator.cc
+++ b/src/os/bluestore/Allocator.cc
@@ -173,8 +173,6 @@ Allocator *Allocator::create(
   std::string_view type,
   int64_t size,
   int64_t block_size,
-  int64_t zone_size,
-  int64_t first_sequential_zone,
   std::string_view name)
 {
   Allocator* alloc = nullptr;
@@ -190,11 +188,6 @@ Allocator *Allocator::create(
     return new HybridAllocator(cct, size, block_size,
       cct->_conf.get_val<uint64_t>("bluestore_hybrid_alloc_mem_cap"),
       name);
-#ifdef HAVE_LIBZBD
-  } else if (type == "zoned") {
-    return new ZonedAllocator(cct, size, block_size, zone_size, first_sequential_zone,
-			      name);
-#endif
   }
   if (alloc == nullptr) {
     lderr(cct) << "Allocator::" << __func__ << " unknown alloc type "
diff --git a/src/os/bluestore/Allocator.h b/src/os/bluestore/Allocator.h
index f136c98b2926..f5a128fda876 100644
--- a/src/os/bluestore/Allocator.h
+++ b/src/os/bluestore/Allocator.h
@@ -72,8 +72,6 @@ class Allocator {
     std::string_view type,
     int64_t size,
     int64_t block_size,
-    int64_t zone_size = 0,
-    int64_t firs_sequential_zone = 0,
     const std::string_view name = ""
     );
 
diff --git a/src/os/bluestore/BitmapFreelistManager.cc b/src/os/bluestore/BitmapFreelistManager.cc
index bec6ace868b1..f1f4831d5671 100644
--- a/src/os/bluestore/BitmapFreelistManager.cc
+++ b/src/os/bluestore/BitmapFreelistManager.cc
@@ -67,7 +67,6 @@ BitmapFreelistManager::BitmapFreelistManager(CephContext* cct,
 }
 
 int BitmapFreelistManager::create(uint64_t new_size, uint64_t granularity,
-				  uint64_t zone_size, uint64_t first_sequential_zone,
 				  KeyValueDB::Transaction txn)
 {
   bytes_per_block = granularity;
diff --git a/src/os/bluestore/BitmapFreelistManager.h b/src/os/bluestore/BitmapFreelistManager.h
index 8e4ea8fd385c..5b04e8fd28cc 100644
--- a/src/os/bluestore/BitmapFreelistManager.h
+++ b/src/os/bluestore/BitmapFreelistManager.h
@@ -63,7 +63,6 @@ class BitmapFreelistManager : public FreelistManager {
   static void setup_merge_operator(KeyValueDB *db, std::string prefix);
 
   int create(uint64_t size, uint64_t granularity,
-	     uint64_t zone_size, uint64_t first_sequential_zone,
 	     KeyValueDB::Transaction txn) override;
 
   int init(KeyValueDB *kvdb, bool db_in_read_only,
diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 54d0bbc38e57..04b680d950d4 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -749,7 +749,6 @@ void BlueFS::_init_alloc()
       alloc[id] = Allocator::create(cct, cct->_conf->bluefs_allocator,
 				    bdev[id]->get_size(),
 				    alloc_size[id],
-				    0, 0,
 				    name);
       alloc[id]->init_add_free(
         block_reserved[id],
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 4f449c3d1c08..0c0cb5c00cf5 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -53,11 +53,6 @@
 #include "common/WorkQueue.h"
 #include "kv/KeyValueHistogram.h"
 
-#ifdef HAVE_LIBZBD
-#include "ZonedAllocator.h"
-#include "ZonedFreelistManager.h"
-#endif
-
 #if defined(WITH_LTTNG)
 #define TRACEPOINT_DEFINE
 #define TRACEPOINT_PROBE_DYNAMIC_LINKAGE
@@ -134,12 +129,6 @@ const string PREFIX_ALLOC = "B";       // u64 offset -> u64 length (freelist)
 const string PREFIX_ALLOC_BITMAP = "b";// (see BitmapFreelistManager)
 const string PREFIX_SHARED_BLOB = "X"; // u64 SB id -> shared_blob_t
 
-#ifdef HAVE_LIBZBD
-const string PREFIX_ZONED_FM_META = "Z";  // (see ZonedFreelistManager)
-const string PREFIX_ZONED_FM_INFO = "z";  // (see ZonedFreelistManager)
-const string PREFIX_ZONED_CL_INFO = "G";  // (per-zone cleaner metadata)
-#endif
-
 const string BLUESTORE_GLOBAL_STATFS_KEY = "bluestore_statfs";
 
 #define OBJECT_MAX_SIZE 0xffffffff // 32 bits
@@ -569,37 +558,6 @@ static int get_key_pool_stat(const string& key, uint64_t* pool_id)
   return 0;
 }
 
-#ifdef HAVE_LIBZBD
-static void get_zone_offset_object_key(
-  uint32_t zone,
-  uint64_t offset,
-  ghobject_t oid,
-  std::string *key)
-{
-  key->clear();
-  _key_encode_u32(zone, key);
-  _key_encode_u64(offset, key);
-  _get_object_key(oid, key);
-}
-
-static int get_key_zone_offset_object(
-  const string& key,
-  uint32_t *zone,
-  uint64_t *offset,
-  ghobject_t *oid)
-{
-  const char *p = key.c_str();
-  if (key.length() < sizeof(uint64_t) + sizeof(uint32_t) + ENCODED_KEY_PREFIX_LEN + 1)
-    return -1;
-  p = _key_decode_u32(p, zone);
-  p = _key_decode_u64(p, offset);
-  int r = _get_key_object(p, oid);
-  if (r < 0) {
-    return r;
-  }
-  return 0;
-}
-#endif
 
 template <int LogLevelV>
 void _dump_extent_map(CephContext *cct, const BlueStore::ExtentMap &em)
@@ -5690,9 +5648,6 @@ BlueStore::BlueStore(CephContext *cct,
     finisher(cct, "commit_finisher", "cfin"),
     kv_sync_thread(this),
     kv_finalize_thread(this),
-#ifdef HAVE_LIBZBD
-    zoned_cleaner_thread(this),
-#endif
     min_alloc_size(_min_alloc_size),
     min_alloc_size_order(std::countr_zero(_min_alloc_size)),
     mempool_thread(this)
@@ -6628,12 +6583,6 @@ void BlueStore::_set_alloc_sizes(void)
 {
   max_alloc_size = cct->_conf->bluestore_max_alloc_size;
 
-#ifdef HAVE_LIBZBD
-  ceph_assert(bdev);
-  if (bdev->is_smr()) {
-    prefer_deferred_size = 0;
-  } else
-#endif
   if (cct->_conf->bluestore_prefer_deferred_size) {
     prefer_deferred_size = cct->_conf->bluestore_prefer_deferred_size;
   } else {
@@ -6741,8 +6690,7 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t,
   bool can_have_null_fm = !is_db_rotational() &&
                           !read_only &&
                           db_avail &&
-                          cct->_conf->bluestore_allocation_from_file &&
-                          !bdev->is_smr();
+                          cct->_conf->bluestore_allocation_from_file;
 
   // When allocation-info is stored in a single file we set freelist_type to "null"
   if (can_have_null_fm) {
@@ -6764,20 +6712,13 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t,
     ceph_assert(cct->_conf->bdev_block_size <= min_alloc_size);
 
     uint64_t alloc_size = min_alloc_size;
-    if (bdev->is_smr() && freelist_type != "zoned") {
-      derr << "SMR device but freelist_type = " << freelist_type << " (not zoned)"
-           << dendl;
-      return -EINVAL;
-    }
     if (!bdev->is_smr() && freelist_type == "zoned") {
       derr << "non-SMR device (or SMR support not built-in) but freelist_type = zoned"
 	   << dendl;
       return -EINVAL;
     }
 
-    fm->create(bdev->get_size(), alloc_size,
-	       zone_size, first_sequential_zone,
-	       t);
+    fm->create(bdev->get_size(), alloc_size, t);
 
     // allocate superblock reserved space.  note that we do not mark
     // bluefs space as allocated in the freelist; we instead rely on
@@ -6902,18 +6843,10 @@ int BlueStore::_create_alloc()
 
   std::string allocator_type = cct->_conf->bluestore_allocator;
 
-#ifdef HAVE_LIBZBD
-  if (freelist_type == "zoned") {
-    allocator_type = "zoned";
-  }
-#endif
-
   alloc = Allocator::create(
     cct, allocator_type,
     bdev->get_size(),
     alloc_size,
-    zone_size,
-    first_sequential_zone,
     "block");
   if (!alloc) {
     lderr(cct) << __func__ << " failed to create " << allocator_type << " allocator"
@@ -6921,27 +6854,8 @@ int BlueStore::_create_alloc()
     return -EINVAL;
   }
 
-#ifdef HAVE_LIBZBD
-  if (freelist_type == "zoned") {
-    Allocator *a = Allocator::create(
-      cct, cct->_conf->bluestore_allocator,
-      bdev->get_conventional_region_size(),
-      alloc_size,
-      zone_size, 0,
-      "zoned_block");
-    if (!a) {
-      lderr(cct) << __func__ << " failed to create " << cct->_conf->bluestore_allocator
-		 << " allocator" << dendl;
-      delete alloc;
-      return -EINVAL;
-    }
-    shared_alloc.set(a, alloc_size);
-  } else
-#endif
-  {
-    // BlueFS will share the same allocator
-    shared_alloc.set(alloc, alloc_size);
-  }
+  // BlueFS will share the same allocator
+  shared_alloc.set(alloc, alloc_size);
 
   return 0;
 }
@@ -6954,66 +6868,6 @@ int BlueStore::_init_alloc(std::map<uint64_t, uint64_t> *zone_adjustments)
   }
   ceph_assert(alloc != NULL);
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    auto a = dynamic_cast<ZonedAllocator*>(alloc);
-    ceph_assert(a);
-    auto f = dynamic_cast<ZonedFreelistManager*>(fm);
-    ceph_assert(f);
-    vector<uint64_t> wp = bdev->get_zones();
-    vector<zone_state_t> zones = f->get_zone_states(db);
-    ceph_assert(wp.size() == zones.size());
-
-    // reconcile zone state
-    auto num_zones = bdev->get_size() / zone_size;
-    for (unsigned i = first_sequential_zone; i < num_zones; ++i) {
-      ceph_assert(wp[i] >= i * zone_size);
-      ceph_assert(wp[i] <= (i + 1) * zone_size); // pos might be at start of next zone
-      uint64_t p = wp[i] - i * zone_size;
-      if (zones[i].write_pointer > p) {
-	derr << __func__ << " zone 0x" << std::hex << i
-	     << " bluestore write pointer 0x" << zones[i].write_pointer
-	     << " > device write pointer 0x" << p
-	     << std::dec << " -- VERY SUSPICIOUS!" << dendl;
-      } else if (zones[i].write_pointer < p) {
-	// this is "normal" in that it can happen after any crash (if we have a
-	// write in flight but did not manage to commit the transaction)
-	auto delta = p - zones[i].write_pointer;
-	dout(1) << __func__ << " zone 0x" << std::hex << i
-		 << " device write pointer 0x" << p
-		 << " > bluestore pointer 0x" << zones[i].write_pointer
-		 << ", advancing 0x" << delta << std::dec << dendl;
-	(*zone_adjustments)[zones[i].write_pointer] = delta;
-	zones[i].num_dead_bytes += delta;
-	zones[i].write_pointer = p;
-      }
-    }
-
-    // start with conventional zone "free" (bluefs may adjust this when it starts up)
-    auto reserved = _get_ondisk_reserved();
-    // for now we require a conventional zone
-    ceph_assert(bdev->get_conventional_region_size());
-    ceph_assert(shared_alloc.a != alloc);  // zoned allocator doesn't use conventional region
-    shared_alloc.a->init_add_free(
-      reserved,
-      p2align(bdev->get_conventional_region_size(), min_alloc_size) - reserved);
-
-    // init sequential zone based on the device's write pointers
-    a->init_from_zone_pointers(std::move(zones));
-    dout(1) << __func__
-	    << " loaded zone pointers: "
-	    << std::hex
-	    << ", allocator type " << alloc->get_type()
-	    << ", capacity 0x" << alloc->get_capacity()
-	    << ", block size 0x" << alloc->get_block_size()
-	    << ", free 0x" << alloc->get_free()
-	    << ", fragmentation " << alloc->get_fragmentation()
-	    << std::dec << dendl;
-
-    return 0;
-  }
-#endif
-
   uint64_t num = 0, bytes = 0;
   utime_t start_time = ceph_clock_now();
   if (!fm->is_null_manager()) {
@@ -7070,24 +6924,6 @@ int BlueStore::_init_alloc(std::map<uint64_t, uint64_t> *zone_adjustments)
 void BlueStore::_post_init_alloc(const std::map<uint64_t, uint64_t>& zone_adjustments)
 {
   int r = 0;
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    if (zone_adjustments.empty()) {
-      return;
-    }
-    dout(1) << __func__ << " adjusting freelist based on device write pointers" << dendl;
-    auto f = dynamic_cast<ZonedFreelistManager*>(fm);
-    ceph_assert(f);
-    KeyValueDB::Transaction t = db->get_transaction();
-    for (auto& i : zone_adjustments) {
-      // allocate AND release since this gap is now dead space
-      // note that the offset is imprecise, but only need to select the zone
-      f->allocate(i.first, i.second, t);
-      f->release(i.first, i.second, t);
-    }
-    r = db->submit_transaction_sync(t);
-  } else
-#endif
   if (fm->is_null_manager()) {
     // Now that we load the allocation map we need to invalidate the file as new allocation won't be reflected
     // Changes to the allocation map (alloc/release) are not updated inline and will only be stored on umount()
@@ -7584,11 +7420,7 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair)
   }
 
   // when function is called in repair mode (to_repair=true) we skip db->open()/create()
-  if (!is_db_rotational() && !read_only && !to_repair && cct->_conf->bluestore_allocation_from_file
-#ifdef HAVE_LIBZBD
-      && !bdev->is_smr()
-#endif
-    ) {
+  if (!is_db_rotational() && !read_only && !to_repair && cct->_conf->bluestore_allocation_from_file) {
     dout(5) << __func__ << "::NCB::Commit to Null-Manager" << dendl;
     commit_to_null_manager();
     need_to_destage_allocation_file = true;
@@ -8271,18 +8103,7 @@ int BlueStore::mkfs()
   if (r < 0)
     goto out_close_fsid;
 
-  // choose freelist manager
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    freelist_type = "zoned";
-    zone_size = bdev->get_zone_size();
-    first_sequential_zone = bdev->get_conventional_region_size() / zone_size;
-    bdev->reset_all_zones();
-  } else
-#endif
-  {
-    freelist_type = "bitmap";
-  }
+  freelist_type = "bitmap";
   dout(10) << " freelist_type " << freelist_type << dendl;
 
   // choose min_alloc_size
@@ -8333,13 +8154,6 @@ int BlueStore::mkfs()
   reserved = _get_ondisk_reserved();
   alloc->init_add_free(reserved,
     p2align(bdev->get_size(), min_alloc_size) - reserved);
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr() && alloc != shared_alloc.a) {
-    shared_alloc.a->init_add_free(reserved,
-				  p2align(bdev->get_conventional_region_size(),
-					  min_alloc_size) - reserved);
-  }
-#endif
 
   r = _open_db(true);
   if (r < 0)
@@ -8372,21 +8186,6 @@ int BlueStore::mkfs()
       t->set(PREFIX_SUPER, "per_pool_omap", bl);
     }
 
-#ifdef HAVE_LIBZBD
-    if (bdev->is_smr()) {
-      {
-	bufferlist bl;
-	encode((uint64_t)zone_size, bl);
-	t->set(PREFIX_SUPER, "zone_size", bl);
-      }
-      {
-	bufferlist bl;
-	encode((uint64_t)first_sequential_zone, bl);
-	t->set(PREFIX_SUPER, "first_sequential_zone", bl);
-      }
-    }
-#endif
-    
     ondisk_format = latest_ondisk_format;
     _prepare_ondisk_format_super(t);
     db->submit_transaction_sync(t);
@@ -8900,12 +8699,6 @@ int BlueStore::_mount()
     return r;
   }
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    _zoned_cleaner_start();
-  }
-#endif
-
   mempool_thread.init();
 
   if ((!per_pool_stat_collection || per_pool_omap != OMAP_PER_PG) &&
@@ -8937,12 +8730,6 @@ int BlueStore::umount()
 
   if (!_kv_only) {
     mempool_thread.shutdown();
-#ifdef HAVE_LIBZBD
-    if (bdev->is_smr()) {
-      dout(20) << __func__ << " stopping zone cleaner thread" << dendl;
-      _zoned_cleaner_stop();
-    }
-#endif
     dout(20) << __func__ << " stopping kv thread" << dendl;
     _kv_stop();
     // skip cache cleanup step on fast shutdown
@@ -9339,7 +9126,6 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow(
     &ctx.expected_pool_statfs[pool_id] :
     &ctx.expected_store_statfs;
 
-  map<uint32_t, uint64_t> zone_first_offsets;  // for zoned/smr devices
 
   dout(10) << __func__ << "  " << oid << dendl;
   OnodeRef o;
@@ -9396,22 +9182,6 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow(
     ceph_assert(l.blob);
     const bluestore_blob_t& blob = l.blob->get_blob();
 
-#ifdef HAVE_LIBZBD
-    if (bdev->is_smr() && depth != FSCK_SHALLOW) {
-      for (auto& e : blob.get_extents()) {
-	if (e.is_valid()) {
-	  uint32_t zone = e.offset / zone_size;
-	  uint64_t offset = e.offset % zone_size;
-	  auto p = zone_first_offsets.find(zone);
-	  if (p == zone_first_offsets.end() || p->second > offset) {
-	    // FIXME: use interator for guided insert?
-	    zone_first_offsets[zone] = offset;
-	  }
-	}
-      }
-    }
-#endif
-
     auto& ref = ref_map[l.blob];
     if (ref.is_empty()) {
       uint32_t min_release_size = blob.get_release_size(min_alloc_size);
@@ -9545,33 +9315,6 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow(
       }
     }
 
-#ifdef HAVE_LIBZBD
-    if (bdev->is_smr() && depth != FSCK_SHALLOW) {
-      for (auto& [zone, first_offset] : zone_first_offsets) {
-	auto p = (*ctx.zone_refs)[zone].find(oid);
-	if (p != (*ctx.zone_refs)[zone].end()) {
-	  if (first_offset < p->second) {
-	    dout(20) << " slightly wonky zone ref 0x" << std::hex << zone
-		 << " offset 0x" << p->second
-		 << " but first offset is 0x" << first_offset
-		 << "; this can happen due to clone_range"
-		 << dendl;
-	  } else {
-	    dout(20) << " good zone ref 0x" << std::hex << zone << " offset 0x" << p->second
-		     << " <= first offset 0x" << first_offset
-		     << std::dec << dendl;
-	  }
-	  (*ctx.zone_refs)[zone].erase(p);
-	} else {
-	  derr << "fsck error: " << oid << " references zone 0x" << std::hex << zone
-	       << " but there is no zone ref" << std::dec << dendl;
-	  // FIXME: add repair
-	  ++errors;
-	}
-      }
-    }
-#endif
-
     if (broken) {
       derr << "fsck error: " << oid << " - " << broken
            << " zombie spanning blob(s) found, the first one: "
@@ -10449,69 +10192,6 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair)
     goto out_scan;
   }
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    auto a = dynamic_cast<ZonedAllocator*>(alloc);
-    ceph_assert(a);
-    auto f = dynamic_cast<ZonedFreelistManager*>(fm);
-    ceph_assert(f);
-    vector<uint64_t> wp = bdev->get_zones();
-    vector<zone_state_t> zones = f->get_zone_states(db);
-    ceph_assert(wp.size() == zones.size());
-    auto num_zones = bdev->get_size() / zone_size;
-    for (unsigned i = first_sequential_zone; i < num_zones; ++i) {
-      uint64_t p = wp[i] == (i + 1) * zone_size ? zone_size : wp[i] % zone_size;
-      if (zones[i].write_pointer > p &&
-	  zones[i].num_dead_bytes < zones[i].write_pointer) {
-	derr << "fsck error: zone 0x" << std::hex << i
-	     << " bluestore write pointer 0x" << zones[i].write_pointer
-	     << " > device write pointer 0x" << p
-	     << " (with only 0x" << zones[i].num_dead_bytes << " dead bytes)"
-	     << std::dec << dendl;
-	++errors;
-      }
-    }
-
-    if (depth != FSCK_SHALLOW) {
-      // load zone refs
-      zone_refs.resize(bdev->get_size() / zone_size);
-      it = db->get_iterator(PREFIX_ZONED_CL_INFO, KeyValueDB::ITERATOR_NOCACHE);
-      if (it) {
-	for (it->lower_bound(string());
-	     it->valid();
-	     it->next()) {
-	  uint32_t zone = 0;
-	  uint64_t offset = 0;
-	  ghobject_t oid;
-	  string key = it->key();
-	  int r = get_key_zone_offset_object(key, &zone, &offset, &oid);
-	  if (r < 0) {
-	    derr << "fsck error: invalid zone ref key " << pretty_binary_string(key)
-		 << dendl;
-	    if (repair) {
-	      repairer.remove_key(db, PREFIX_ZONED_CL_INFO, key);
-	    }
-	    ++errors;
-	    continue;
-	  }
-	  dout(30) << " zone ref 0x" << std::hex << zone << " offset 0x" << offset
-		   << " -> " << std::dec << oid << dendl;
-	  if (zone_refs[zone].count(oid)) {
-	    derr << "fsck error: second zone ref in zone 0x" << std::hex << zone
-		 << " offset 0x" << offset << std::dec << " for " << oid << dendl;
-	    if (repair) {
-	      repairer.remove_key(db, PREFIX_ZONED_CL_INFO, key);
-	    }
-	    ++errors;
-	    continue;
-	  }
-	  zone_refs[zone][oid] = offset;
-	}
-      }
-    }
-  }
-#endif
-
   dout(1) << __func__ << " checking shared_blobs (phase 1)" << dendl;
   it = db->get_iterator(PREFIX_SHARED_BLOB, KeyValueDB::ITERATOR_NOCACHE);
   if (it) {
@@ -10576,20 +10256,6 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair)
     _fsck_check_objects(depth, ctx);
   }
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr() && depth != FSCK_SHALLOW) {
-    dout(1) << __func__ << " checking for leaked zone refs" << dendl;
-    for (uint32_t zone = 0; zone < zone_refs.size(); ++zone) {
-      for (auto& [oid, offset] : zone_refs[zone]) {
-	derr << "fsck error: stray zone ref 0x" << std::hex << zone
-	     << " offset 0x" << offset << " -> " << std::dec << oid << dendl;
-	// FIXME: add repair
-	++errors;
-      }
-    }
-  }
-#endif
-
   sb_ref_mismatches = sb_ref_counts.count_non_zero();
   if (sb_ref_mismatches != 0) {
     derr << "fsck error:" << "*" << sb_ref_mismatches
@@ -11035,150 +10701,76 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair)
     // skip freelist vs allocated compare when we have Null fm
     if (!fm->is_null_manager()) {
       dout(1) << __func__ << " checking freelist vs allocated" << dendl;
-#ifdef HAVE_LIBZBD
-      if (freelist_type == "zoned") {
-	// verify per-zone state
-	//  - verify no allocations beyond write pointer
-	//  - verify num_dead_bytes count (neither allocated nor
-	//    free space past the write pointer)
-	auto a = dynamic_cast<ZonedAllocator*>(alloc);
-	auto num_zones = bdev->get_size() / zone_size;
-
-	// mark the free space past the write pointer
-	for (uint32_t zone = first_sequential_zone; zone < num_zones; ++zone) {
-	  auto wp = a->get_write_pointer(zone);
-	  uint64_t offset = zone_size * zone + wp;
-	  uint64_t length = zone_size - wp;
-	  if (!length) {
-	    continue;
-	  }
-	  bool intersects = false;
-	  dout(10) << "  marking zone 0x" << std::hex << zone
-		   << " region after wp 0x" << offset << "~" << length
-		   << std::dec << dendl;
-	  apply_for_bitset_range(
-	    offset, length, alloc_size, used_blocks,
-	    [&](uint64_t pos, mempool_dynamic_bitset &bs) {
-	      if (bs.test(pos)) {
-		derr << "fsck error: zone 0x" << std::hex << zone
-		     << " has used space at 0x" << pos * alloc_size
-		     << " beyond write pointer 0x" << wp
-		     << std::dec << dendl;
-		intersects = true;
-	      } else {
-		bs.set(pos);
-	      }
-	    }
-	    );
-	  if (intersects) {
-	    ++errors;
-	  }
-	}
-
-	used_blocks.flip();
-
-	// skip conventional zones
-	uint64_t pos = (first_sequential_zone * zone_size) / min_alloc_size - 1;
-	pos = used_blocks.find_next(pos);
-
-	uint64_t zone_dead = 0;
-	for (uint32_t zone = first_sequential_zone;
-	     zone < num_zones;
-	     ++zone, zone_dead = 0) {
-	  while (pos != decltype(used_blocks)::npos &&
-		 (pos * min_alloc_size) / zone_size == zone) {
-	    dout(40) << " zone 0x" << std::hex << zone
-		     << " dead 0x" << (pos * min_alloc_size) << "~" << min_alloc_size
-		     << std::dec << dendl;
-	    zone_dead += min_alloc_size;
-	    pos = used_blocks.find_next(pos);
-	  }
-	  dout(20) << " zone 0x" << std::hex << zone << " dead is 0x" << zone_dead
-		   << std::dec << dendl;
-	  // cross-check dead bytes against zone state
-	  if (a->get_dead_bytes(zone) != zone_dead) {
-	    derr << "fsck error: zone 0x" << std::hex << zone << " has 0x" << zone_dead
-		 << " dead bytes but freelist says 0x" << a->get_dead_bytes(zone)
-		 << dendl;
-	    ++errors;
-	    // TODO: repair
-	  }
-	}
-	used_blocks.flip();
-      } else
-#endif
-      {
-	fm->enumerate_reset();
-	uint64_t offset, length;
-	while (fm->enumerate_next(db, &offset, &length)) {
-	  bool intersects = false;
-	  apply_for_bitset_range(
-	    offset, length, alloc_size, used_blocks,
-	    [&](uint64_t pos, mempool_dynamic_bitset &bs) {
-	      ceph_assert(pos < bs.size());
-	      if (bs.test(pos) && !bluefs_used_blocks.test(pos)) {
-		if (offset == DB_SUPER_RESERVED &&
-		    length == min_alloc_size - DB_SUPER_RESERVED) {
-		  // this is due to the change just after luminous to min_alloc_size
-		  // granularity allocations, and our baked in assumption at the top
-		  // of _fsck that 0~round_up_to(DB_SUPER_RESERVED,min_alloc_size) is used
-		  // (vs luminous's round_up_to(DB_SUPER_RESERVED,block_size)).  harmless,
-		  // since we will never allocate this region below min_alloc_size.
-		  dout(10) << __func__ << " ignoring free extent between DB_SUPER_RESERVED"
-			   << " and min_alloc_size, 0x" << std::hex << offset << "~"
-			   << length << std::dec << dendl;
-		} else {
-		  intersects = true;
-		  if (repair) {
-		    repairer.fix_false_free(db, fm,
-					    pos * min_alloc_size,
-					    min_alloc_size);
-		  }
-		}
-	      } else {
-		bs.set(pos);
-	      }
-	    }
-	    );
-	  if (intersects) {
-	    derr << "fsck error: free extent 0x" << std::hex << offset
-		 << "~" << length << std::dec
-		 << " intersects allocated blocks" << dendl;
-	    ++errors;
-	  }
-	}
-	fm->enumerate_reset();
-
-	// check for leaked extents
-	size_t count = used_blocks.count();
-	if (used_blocks.size() != count) {
-	  ceph_assert(used_blocks.size() > count);
-	  used_blocks.flip();
-	  size_t start = used_blocks.find_first();
-	  while (start != decltype(used_blocks)::npos) {
-	    size_t cur = start;
-	    while (true) {
-	      size_t next = used_blocks.find_next(cur);
-	      if (next != cur + 1) {
-		++errors;
-		derr << "fsck error: leaked extent 0x" << std::hex
-		     << ((uint64_t)start * fm->get_alloc_size()) << "~"
-		     << ((cur + 1 - start) * fm->get_alloc_size()) << std::dec
-		     << dendl;
-		if (repair) {
-		  repairer.fix_leaked(db,
-				      fm,
-				      start * min_alloc_size,
-				      (cur + 1 - start) * min_alloc_size);
-		}
-		start = next;
-		break;
-	      }
-	      cur = next;
-	    }
-	  }
-	  used_blocks.flip();
-	}
+      fm->enumerate_reset();
+      uint64_t offset, length;
+      while (fm->enumerate_next(db, &offset, &length)) {
+        bool intersects = false;
+        apply_for_bitset_range(
+          offset, length, alloc_size, used_blocks,
+          [&](uint64_t pos, mempool_dynamic_bitset &bs) {
+            ceph_assert(pos < bs.size());
+            if (bs.test(pos) && !bluefs_used_blocks.test(pos)) {
+              if (offset == DB_SUPER_RESERVED &&
+                  length == min_alloc_size - DB_SUPER_RESERVED) {
+                // this is due to the change just after luminous to min_alloc_size
+                // granularity allocations, and our baked in assumption at the top
+                // of _fsck that 0~round_up_to(DB_SUPER_RESERVED,min_alloc_size) is used
+                // (vs luminous's round_up_to(DB_SUPER_RESERVED,block_size)).  harmless,
+                // since we will never allocate this region below min_alloc_size.
+                dout(10) << __func__ << " ignoring free extent between DB_SUPER_RESERVED"
+                         << " and min_alloc_size, 0x" << std::hex << offset << "~"
+                         << length << std::dec << dendl;
+              } else {
+                intersects = true;
+                if (repair) {
+                  repairer.fix_false_free(db, fm,
+                                          pos * min_alloc_size,
+                                          min_alloc_size);
+                }
+              }
+            } else {
+              bs.set(pos);
+            }
+          }
+          );
+        if (intersects) {
+          derr << "fsck error: free extent 0x" << std::hex << offset
+               << "~" << length << std::dec
+               << " intersects allocated blocks" << dendl;
+          ++errors;
+        }
+      }
+      fm->enumerate_reset();
+
+      // check for leaked extents
+      size_t count = used_blocks.count();
+      if (used_blocks.size() != count) {
+        ceph_assert(used_blocks.size() > count);
+        used_blocks.flip();
+        size_t start = used_blocks.find_first();
+        while (start != decltype(used_blocks)::npos) {
+          size_t cur = start;
+          while (true) {
+            size_t next = used_blocks.find_next(cur);
+            if (next != cur + 1) {
+              ++errors;
+              derr << "fsck error: leaked extent 0x" << std::hex
+                   << ((uint64_t)start * fm->get_alloc_size()) << "~"
+                   << ((cur + 1 - start) * fm->get_alloc_size()) << std::dec
+                   << dendl;
+              if (repair) {
+                repairer.fix_leaked(db,
+                                    fm,
+                                    start * min_alloc_size,
+                                    (cur + 1 - start) * min_alloc_size);
+              }
+              start = next;
+              break;
+            }
+            cur = next;
+          }
+        }
+        used_blocks.flip();
       }
     }
   }
@@ -13545,33 +13137,6 @@ int BlueStore::_open_super_meta()
     logger->set(l_bluestore_alloc_unit, min_alloc_size);
   }
 
-  // smr fields
-  {
-    bufferlist bl;
-    int r = db->get(PREFIX_SUPER, "zone_size", &bl);
-    if (r >= 0) {
-      auto p = bl.cbegin();
-      decode(zone_size, p);
-      dout(1) << __func__ << " zone_size 0x" << std::hex << zone_size << std::dec << dendl;
-      ceph_assert(bdev->is_smr());
-    } else {
-      ceph_assert(!bdev->is_smr());
-    }
-  }
-  {
-    bufferlist bl;
-    int r = db->get(PREFIX_SUPER, "first_sequential_zone", &bl);
-    if (r >= 0) {
-      auto p = bl.cbegin();
-      decode(first_sequential_zone, p);
-      dout(1) << __func__ << " first_sequential_zone 0x" << std::hex
-	      << first_sequential_zone << std::dec << dendl;
-      ceph_assert(bdev->is_smr());
-    } else {
-      ceph_assert(!bdev->is_smr());
-    }
-  }
-
   _set_per_pool_omap();
 
   _open_statfs();
@@ -14003,29 +13568,6 @@ void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t)
     }
   }
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    for (auto& i : txc->old_zone_offset_refs) {
-      dout(20) << __func__ << " rm ref zone 0x" << std::hex << i.first.second
-	       << " offset 0x" << i.second << std::dec
-	       << " -> " << i.first.first->oid << dendl;
-      string key;
-      get_zone_offset_object_key(i.first.second, i.second, i.first.first->oid, &key);
-      txc->t->rmkey(PREFIX_ZONED_CL_INFO, key);
-    }
-    for (auto& i : txc->new_zone_offset_refs) {
-      // (zone, offset) -> oid
-      dout(20) << __func__ << " add ref zone 0x" << std::hex << i.first.second
-	       << " offset 0x" << i.second << std::dec
-	       << " -> " << i.first.first->oid << dendl;
-      string key;
-      get_zone_offset_object_key(i.first.second, i.second, i.first.first->oid, &key);
-      bufferlist v;
-      txc->t->set(PREFIX_ZONED_CL_INFO, key, v);
-    }
-  }
-#endif
-
   _txc_update_store_statfs(txc);
 }
 
@@ -14747,210 +14289,6 @@ void BlueStore::_kv_finalize_thread()
   kv_finalize_started = false;
 }
 
-#ifdef HAVE_LIBZBD
-void BlueStore::_zoned_cleaner_start()
-{
-  dout(10) << __func__ << dendl;
-  zoned_cleaner_thread.create("bstore_zcleaner");
-}
-
-void BlueStore::_zoned_cleaner_stop()
-{
-  dout(10) << __func__ << dendl;
-  {
-    std::unique_lock l{zoned_cleaner_lock};
-    while (!zoned_cleaner_started) {
-      zoned_cleaner_cond.wait(l);
-    }
-    zoned_cleaner_stop = true;
-    zoned_cleaner_cond.notify_all();
-  }
-  zoned_cleaner_thread.join();
-  {
-    std::lock_guard l{zoned_cleaner_lock};
-    zoned_cleaner_stop = false;
-  }
-  dout(10) << __func__ << " done" << dendl;
-}
-
-void BlueStore::_zoned_cleaner_thread()
-{
-  dout(10) << __func__ << " start" << dendl;
-  std::unique_lock l{zoned_cleaner_lock};
-  ceph_assert(!zoned_cleaner_started);
-  zoned_cleaner_started = true;
-  zoned_cleaner_cond.notify_all();
-  auto a = dynamic_cast<ZonedAllocator*>(alloc);
-  ceph_assert(a);
-  auto f = dynamic_cast<ZonedFreelistManager*>(fm);
-  ceph_assert(f);
-  while (true) {
-    // thresholds to trigger cleaning
-    // FIXME
-    float min_score = .05;                // score: bytes saved / bytes moved
-    uint64_t min_saved = zone_size / 32;  // min bytes saved to consider cleaning
-    auto zone_to_clean = a->pick_zone_to_clean(min_score, min_saved);
-    if (zone_to_clean < 0) {
-      if (zoned_cleaner_stop) {
-	break;
-      }
-      auto period = ceph::make_timespan(cct->_conf->bluestore_cleaner_sleep_interval);
-      dout(20) << __func__ << " sleep for " << period << dendl;
-      zoned_cleaner_cond.wait_for(l, period);
-      dout(20) << __func__ << " wake" << dendl;
-    } else {
-      l.unlock();
-      a->set_cleaning_zone(zone_to_clean);
-      _zoned_clean_zone(zone_to_clean, a, f);
-      a->clear_cleaning_zone(zone_to_clean);
-      l.lock();
-    }
-  }
-  dout(10) << __func__ << " finish" << dendl;
-  zoned_cleaner_started = false;
-}
-
-void BlueStore::_zoned_clean_zone(
-  uint64_t zone,
-  ZonedAllocator *a,
-  ZonedFreelistManager *f
-  )
-{
-  dout(10) << __func__ << " cleaning zone 0x" << std::hex << zone << std::dec << dendl;
-
-  KeyValueDB::Iterator it = db->get_iterator(PREFIX_ZONED_CL_INFO);
-  std::string zone_start;
-  get_zone_offset_object_key(zone, 0, ghobject_t(), &zone_start);
-  for (it->lower_bound(zone_start); it->valid(); it->next()) {
-    uint32_t z;
-    uint64_t offset;
-    ghobject_t oid;
-    string k = it->key();
-    int r = get_key_zone_offset_object(k, &z, &offset, &oid);
-    if (r < 0) {
-      derr << __func__ << " failed to decode zone ref " << pretty_binary_string(k)
-	   << dendl;
-      continue;
-    }
-    if (zone != z) {
-      dout(10) << __func__ << " reached end of zone refs" << dendl;
-      break;
-    }
-    dout(10) << __func__ << " zone 0x" << std::hex << zone << " offset 0x" << offset
-	     << std::dec << " " << oid << dendl;
-    _clean_some(oid, zone);
-  }
-
-  if (a->get_live_bytes(zone) > 0) {
-    derr << "zone 0x" << std::hex << zone << " still has 0x" << a->get_live_bytes(zone)
-	 << " live bytes" << std::dec << dendl;
-    // should we do something else here to avoid a live-lock in the event of a problem?
-    return;
-  }
-
-  // make sure transactions flush/drain/commit (and data is all rewritten
-  // safely elsewhere) before we blow away the cleaned zone
-  _osr_drain_all();
-
-  // reset the device zone
-  dout(10) << __func__ << " resetting zone 0x" << std::hex << zone << std::dec << dendl;
-  bdev->reset_zone(zone);
-
-  // record that we can now write there
-  f->mark_zone_to_clean_free(zone, db);
-  bdev->flush();
-
-  // then allow ourselves to start allocating there
-  dout(10) << __func__ << " done cleaning zone 0x" << std::hex << zone << std::dec
-	   << dendl;
-  a->reset_zone(zone);
-}
-
-void BlueStore::_clean_some(ghobject_t oid, uint32_t zone)
-{
-  dout(10) << __func__ << " " << oid << " from zone 0x" << std::hex << zone << std::dec
-	   << dendl;
-
-  CollectionRef cref = _get_collection_by_oid(oid);
-  if (!cref) {
-    dout(10) << __func__ << " can't find collection for " << oid << dendl;
-    return;
-  }
-  Collection *c = cref.get();
-
-  // serialize io dispatch vs other transactions
-  std::lock_guard l(atomic_alloc_and_submit_lock);
-  std::unique_lock l2(c->lock);
-
-  auto o = c->get_onode(oid, false);
-  if (!o) {
-    dout(10) << __func__ << " can't find " << oid << dendl;
-    return;
-  }
-
-  o->extent_map.fault_range(db, 0, OBJECT_MAX_SIZE);
-  _dump_onode<30>(cct, *o);
-
-  // NOTE: This is a naive rewrite strategy.  If any blobs are
-  // shared, they will be duplicated for each object that references
-  // them.  That means any cloned/snapshotted objects will explode
-  // their utilization.  This won't matter for RGW workloads, but
-  // for RBD and CephFS it is completely unacceptable, and it's
-  // entirely reasonable to have "archival" data workloads on SMR
-  // for CephFS and (possibly/probably) RBD.
-  //
-  // At some point we need to replace this with something more
-  // sophisticated that ensures that a shared blob gets moved once
-  // and all referencing objects get updated to point to the new
-  // location.
-
-  map<uint32_t, uint32_t> to_move;
-  for (auto& e : o->extent_map.extent_map) {
-    bool touches_zone = false;
-    for (auto& be : e.blob->get_blob().get_extents()) {
-      if (be.is_valid()) {
-	uint32_t z = be.offset / zone_size;
-	if (z == zone) {
-	  touches_zone = true;
-	  break;
-	}
-      }
-    }
-    if (touches_zone) {
-      to_move[e.logical_offset] = e.length;
-    }
-  }
-  if (to_move.empty()) {
-    dout(10) << __func__ << " no references to zone 0x" << std::hex << zone
-	     << std::dec << " from " << oid << dendl;
-    return;
-  }
-
-  dout(10) << __func__ << " rewriting object extents 0x" << std::hex << to_move
-	   << std::dec << dendl;
-  OpSequencer *osr = c->osr.get();
-  TransContext *txc = _txc_create(c, osr, nullptr);
-
-  spg_t pgid;
-  if (c->cid.is_pg(&pgid)) {
-    txc->osd_pool_id = pgid.pool();
-  }
-
-  for (auto& [offset, length] : to_move) {
-    bufferlist bl;
-    int r = _do_read(c, o, offset, length, bl, 0);
-    ceph_assert(r == (int)length);
-
-    r = _do_write(txc, cref, o, offset, length, bl, 0);
-    ceph_assert(r >= 0);
-  }
-  txc->write_onode(o);
-
-  _txc_write_nodes(txc, txc->t);
-  _txc_finalize_kv(txc, txc->t);
-  _txc_state_proc(txc);
-}
-#endif
 
 bluestore_deferred_op_t *BlueStore::_get_deferred_op(
   TransContext *txc, uint64_t len)
@@ -15316,16 +14654,6 @@ int BlueStore::queue_transactions(
   OpSequencer *osr = c->osr.get();
   dout(10) << __func__ << " ch " << c << " " << c->cid << dendl;
 
-  // With HM-SMR drives (and ZNS SSDs) we want the I/O allocation and I/O
-  // submission to happen atomically because if I/O submission happens in a
-  // different order than I/O allocation, we end up issuing non-sequential
-  // writes to the drive.  This is a temporary solution until ZONE APPEND
-  // support matures in the kernel.  For more information please see:
-  // https://www.usenix.org/conference/vault20/presentation/bjorling
-  if (bdev->is_smr()) {
-    atomic_alloc_and_submit_lock.lock();
-  }
-
   // prepare
   TransContext *txc = _txc_create(static_cast<Collection*>(ch.get()), osr,
 				  &on_commit, op);
@@ -15391,10 +14719,6 @@ int BlueStore::queue_transactions(
   // execute (start)
   _txc_state_proc(txc);
 
-  if (bdev->is_smr()) {
-    atomic_alloc_and_submit_lock.unlock();
-  }
-
   // we're immediately readable (unlike FileStore)
   for (auto c : on_applied_sync) {
     c->complete(0);
@@ -15913,36 +15237,6 @@ void BlueStore::_do_write_small(
   // than 'offset' only).
   o->extent_map.fault_range(db, min_off, offset + max_bsize - min_off);
 
-#ifdef HAVE_LIBZBD
-  // On zoned devices, the first goal is to support non-overwrite workloads,
-  // such as RGW, with large, aligned objects.  Therefore, for user writes
-  // _do_write_small should not trigger.  OSDs, however, write and update a tiny
-  // amount of metadata, such as OSD maps, to disk.  For those cases, we
-  // temporarily just pad them to min_alloc_size and write them to a new place
-  // on every update.
-  if (bdev->is_smr()) {
-    uint64_t b_off = p2phase<uint64_t>(offset, alloc_len);
-    uint64_t b_off0 = b_off;
-    o->extent_map.punch_hole(c, offset, length, &wctx->old_extents);
-
-    // Zero detection -- small block
-    if (!cct->_conf->bluestore_zero_block_detection || !bl.is_zero()) {
-      BlobRef b = c->new_blob();
-      _pad_zeros(&bl, &b_off0, min_alloc_size);
-      wctx->write(offset, b, alloc_len, b_off0, bl, b_off, length, false, true);
-    } else { // if (bl.is_zero())
-      dout(20) << __func__ << " skip small zero block " << std::hex
-        << " (0x" << b_off0 << "~" << bl.length() << ")"
-        << " (0x" << b_off << "~" << length << ")"
-        << std::dec << dendl;
-      logger->inc(l_bluestore_write_small_skipped);
-      logger->inc(l_bluestore_write_small_skipped_bytes, length);
-    }
-
-    return;
-  }
-#endif
-
   // Look for an existing mutable blob we can use.
   auto begin = o->extent_map.extent_map.begin();
   auto end = o->extent_map.extent_map.end();
@@ -16972,27 +16266,6 @@ void BlueStore::_wctx_finish(
   WriteContext *wctx,
   set<SharedBlob*> *maybe_unshared_blobs)
 {
-#ifdef HAVE_LIBZBD
-  bool is_smr = bdev && bdev->is_smr();
-  if (is_smr) {
-    for (auto& w : wctx->writes) {
-      for (auto& e : w.b->get_blob().get_extents()) {
-	if (!e.is_valid()) {
-	  continue;
-	}
-	uint32_t zone = e.offset / zone_size;
-	if (!o->onode.zone_offset_refs.count(zone)) {
-	  uint64_t zoff = e.offset % zone_size;
-	  dout(20) << __func__ << " add ref zone 0x" << std::hex << zone
-		   << " offset 0x" << zoff << std::dec << dendl;
-	  txc->note_write_zone_offset(o, zone, zoff);
-	}
-      }
-    }
-  }
-  set<uint32_t> zones_with_releases;
-#endif
-
   auto oep = wctx->old_extents.begin();
   while (oep != wctx->old_extents.end()) {
     auto &lo = *oep;
@@ -17020,12 +16293,6 @@ void BlueStore::_wctx_finish(
 	  b->shared_blob->put_ref(
 	    e.offset, e.length, &final,
 	    unshare_ptr);
-#ifdef HAVE_LIBZBD
-	  // we also drop zone ref for shared blob extents
-	  if (is_smr && e.is_valid()) {
-	    zones_with_releases.insert(e.offset / zone_size);
-	  }
-#endif
 	}
 	if (unshare) {
 	  ceph_assert(maybe_unshared_blobs);
@@ -17052,11 +16319,6 @@ void BlueStore::_wctx_finish(
       if (blob.is_compressed()) {
         txc->statfs_delta.compressed_allocated() -= e.length;
       }
-#ifdef HAVE_LIBZBD
-      if (is_smr && e.is_valid()) {
-	zones_with_releases.insert(e.offset / zone_size);
-      }
-#endif
     }
 
     if (b->is_spanning() && !b->is_referenced() && lo.blob_empty) {
@@ -17066,29 +16328,6 @@ void BlueStore::_wctx_finish(
     }
     delete &lo;
   }
-
-#ifdef HAVE_LIBZBD
-  if (!zones_with_releases.empty()) {
-    // we need to fault the entire extent range in here to determinte if we've dropped
-    // all refs to a zone.
-    o->extent_map.fault_range(db, 0, OBJECT_MAX_SIZE);
-    for (auto& b : o->extent_map.extent_map) {
-      for (auto& e : b.blob->get_blob().get_extents()) {
-	if (e.is_valid()) {
-	  zones_with_releases.erase(e.offset / zone_size);
-	}
-      }
-    }
-    for (auto zone : zones_with_releases) {
-      auto p = o->onode.zone_offset_refs.find(zone);
-      if (p != o->onode.zone_offset_refs.end()) {
-	dout(20) << __func__ << " rm ref zone 0x" << std::hex << zone
-		 << " offset 0x" << p->second << std::dec << dendl;
-	txc->note_release_zone_offset(o, zone, p->second);
-      }
-    }
-  }
-#endif
 }
 
 void BlueStore::_do_write_data(
@@ -18092,37 +17331,6 @@ int BlueStore::_do_clone_range(
     oldo->extent_map.dup(this, txc, c, oldo, newo, srcoff, length, dstoff);
   }
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    // duplicate the refs for the shared region.
-    Extent dummy(dstoff);
-    for (auto e = newo->extent_map.extent_map.lower_bound(dummy);
-	 e != newo->extent_map.extent_map.end();
-	 ++e) {
-      if (e->logical_offset >= dstoff + length) {
-	break;
-      }
-      for (auto& ex : e->blob->get_blob().get_extents()) {
-	// note that we may introduce a new extent reference that is
-	// earlier than the first zone ref.  we allow this since it is
-	// a lot of work to avoid and has marginal impact on cleaning
-	// performance.
-	if (!ex.is_valid()) {
-	  continue;
-	}
-	uint32_t zone = ex.offset / zone_size;
-	if (!newo->onode.zone_offset_refs.count(zone)) {
-	  uint64_t zoff = ex.offset % zone_size;
-	  dout(20) << __func__ << " add ref zone 0x" << std::hex << zone
-		   << " offset 0x" << zoff << std::dec
-		   << " -> " << newo->oid << dendl;
-	  txc->note_write_zone_offset(newo, zone, zoff);
-	}
-      }
-    }
-  }
-#endif
-
   _dump_onode<30>(cct, *oldo);
   _dump_onode<30>(cct, *newo);
   return 0;
@@ -18227,27 +17435,6 @@ int BlueStore::_rename(TransContext *txc,
   // and read newo's metadata via the old name).
   txc->note_modified_object(oldo);
 
-#ifdef HAVE_LIBZBD
-  if (bdev->is_smr()) {
-    // adjust zone refs
-    for (auto& [zone, offset] : newo->onode.zone_offset_refs) {
-      dout(20) << __func__ << " rm ref zone 0x" << std::hex << zone
-	       << " offset 0x" << offset << std::dec
-	       << " -> " << oldo->oid << dendl;
-      string key;
-      get_zone_offset_object_key(zone, offset, oldo->oid, &key);
-      txc->t->rmkey(PREFIX_ZONED_CL_INFO, key);
-
-      dout(20) << __func__ << " add ref zone 0x" << std::hex << zone
-	       << " offset 0x" << offset << std::dec
-	       << " -> " << newo->oid << dendl;
-      get_zone_offset_object_key(zone, offset, newo->oid, &key);
-      bufferlist v;
-      txc->t->set(PREFIX_ZONED_CL_INFO, key, v);
-    }
-  }
-#endif
-
  out:
   dout(10) << __func__ << " " << c->cid << " " << old_oid << " -> "
 	   << new_oid << " = " << r << dendl;
@@ -19861,7 +19048,6 @@ Allocator* BlueStore::create_bitmap_allocator(uint64_t bdev_size) {
   // create allocator
   uint64_t alloc_size = min_alloc_size;
   Allocator* alloc = Allocator::create(cct, "bitmap", bdev_size, alloc_size,
-				       zone_size, first_sequential_zone,
 				       "recovery");
   if (alloc) {
     return alloc;
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 03c1ce124de1..4b832f7cce1a 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1376,17 +1376,6 @@ class BlueStore : public ObjectStore,
     void rewrite_omap_key(const std::string& old, std::string *out);
     void decode_omap_key(const std::string& key, std::string *user_key);
 
-#ifdef HAVE_LIBZBD
-    // Return the offset of an object on disk.  This function is intended *only*
-    // for use with zoned storage devices because in these devices, the objects
-    // are laid out contiguously on disk, which is not the case in general.
-    // Also, it should always be called after calling extent_map.fault_range(),
-    // so that the extent map is loaded.
-    int64_t zoned_get_ondisk_starting_offset() const {
-      return extent_map.extent_map.begin()->blob->
-	  get_blob().calc_offset(0, nullptr);
-    }
-#endif
 private:
     void _decode(const ceph::buffer::list& v);
   };
@@ -1847,16 +1836,6 @@ class BlueStore : public ObjectStore,
     std::set<OnodeRef> onodes;     ///< these need to be updated/written
     std::set<OnodeRef> modified_objects;  ///< objects we modified (and need a ref)
 
-#ifdef HAVE_LIBZBD
-    // zone refs to add/remove.  each zone ref is a (zone, offset) tuple.  The offset
-    // is the first offset in the zone that the onode touched; subsequent writes
-    // to that zone do not generate additional refs.  This is a bit imprecise but
-    // is sufficient to generate reasonably sequential reads when doing zone
-    // cleaning with less metadata than a ref for every extent.
-    std::map<std::pair<OnodeRef, uint32_t>, uint64_t> new_zone_offset_refs;
-    std::map<std::pair<OnodeRef, uint32_t>, uint64_t> old_zone_offset_refs;
-#endif
-    
     std::set<SharedBlobRef> shared_blobs;  ///< these need to be updated/written
     std::set<BlobRef> blobs_written; ///< update these on io completion
     KeyValueDB::Transaction t; ///< then we will commit this
@@ -1928,17 +1907,6 @@ class BlueStore : public ObjectStore,
       onodes.erase(o);
     }
 
-#ifdef HAVE_LIBZBD
-    void note_write_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) {
-      o->onode.zone_offset_refs[zone] = offset;
-      new_zone_offset_refs[std::make_pair(o, zone)] = offset;
-    }
-    void note_release_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) {
-      old_zone_offset_refs[std::make_pair(o, zone)] = offset;
-      o->onode.zone_offset_refs.erase(zone);
-    }
-#endif
-
     void aio_finish(BlueStore *store) override {
       store->txc_aio_finish(this);
     }
@@ -2242,17 +2210,6 @@ class BlueStore : public ObjectStore,
     }
   };
 
-#ifdef HAVE_LIBZBD
-  struct ZonedCleanerThread : public Thread {
-    BlueStore *store;
-    explicit ZonedCleanerThread(BlueStore *s) : store(s) {}
-    void *entry() override {
-      store->_zoned_cleaner_thread();
-      return nullptr;
-    }
-  };
-#endif
-  
   struct BigDeferredWriteContext {
     uint64_t off = 0;     // original logical offset
     uint32_t b_off = 0;   // blob relative offset
@@ -2349,15 +2306,6 @@ class BlueStore : public ObjectStore,
   std::deque<DeferredBatch*> deferred_stable_to_finalize; ///< pending finalization
   bool kv_finalize_in_progress = false;
 
-#ifdef HAVE_LIBZBD
-  ZonedCleanerThread zoned_cleaner_thread;
-  ceph::mutex zoned_cleaner_lock = ceph::make_mutex("BlueStore::zoned_cleaner_lock");
-  ceph::condition_variable zoned_cleaner_cond;
-  bool zoned_cleaner_started = false;
-  bool zoned_cleaner_stop = false;
-  std::deque<uint64_t> zoned_cleaner_queue;
-#endif
-
   PerfCounters *logger = nullptr;
 
   std::list<CollectionRef> removed_collections;
@@ -2382,10 +2330,6 @@ class BlueStore : public ObjectStore,
 		"not enough bits for min_alloc_size");
   bool elastic_shared_blobs = false; ///< use smart ExtentMap::dup to reduce shared blob count
 
-  // smr-only
-  uint64_t zone_size = 0;              ///< number of SMR zones 
-  uint64_t first_sequential_zone = 0;  ///< first SMR zone that is sequential-only
-
   enum {
     // Please preserve the order since it's DB persistent
     OMAP_BULK = 0,
@@ -2820,16 +2764,6 @@ class BlueStore : public ObjectStore,
   void _kv_sync_thread();
   void _kv_finalize_thread();
 
-#ifdef HAVE_LIBZBD
-  void _zoned_cleaner_start();
-  void _zoned_cleaner_stop();
-  void _zoned_cleaner_thread();
-  void _zoned_clean_zone(uint64_t zone_num,
-			 class ZonedAllocator *a,
-			 class ZonedFreelistManager *f);
-  void _clean_some(ghobject_t oid, uint32_t zone_num);
-#endif
-
   bluestore_deferred_op_t *_get_deferred_op(TransContext *txc, uint64_t len);
   void _deferred_queue(TransContext *txc);
 public:
diff --git a/src/os/bluestore/FreelistManager.cc b/src/os/bluestore/FreelistManager.cc
index 69866fa40cb5..37347ced66bd 100644
--- a/src/os/bluestore/FreelistManager.cc
+++ b/src/os/bluestore/FreelistManager.cc
@@ -27,17 +27,6 @@ FreelistManager *FreelistManager::create(
     return fm;
   }
 
-#ifdef HAVE_LIBZBD
-  // With zoned drives there is only one FreelistManager implementation that we
-  // can use, and we also know if a drive is zoned right after opening it
-  // (BlueStore::_open_bdev).  Hence, we set freelist_type to "zoned" whenever
-  // we open the device and it turns out to be is zoned.  We ignore |prefix|
-  // passed to create and use the prefixes defined for zoned devices at the top
-  // of BlueStore.cc.
-  if (type == "zoned")
-    return new ZonedFreelistManager(cct, "Z", "z");
-#endif
-
   return NULL;
 }
 
diff --git a/src/os/bluestore/FreelistManager.h b/src/os/bluestore/FreelistManager.h
index 7f44fe957316..b647e55c2c16 100644
--- a/src/os/bluestore/FreelistManager.h
+++ b/src/os/bluestore/FreelistManager.h
@@ -26,7 +26,6 @@ class FreelistManager {
   static void setup_merge_operators(KeyValueDB *db, const std::string &type);
 
   virtual int create(uint64_t size, uint64_t granularity,
-		     uint64_t zone_size, uint64_t first_sequential_zone,
 		     KeyValueDB::Transaction txn) = 0;
 
   virtual int init(KeyValueDB *kvdb, bool db_in_read_only,
diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc
deleted file mode 100644
index 4139b4755697..000000000000
--- a/src/os/bluestore/ZonedAllocator.cc
+++ /dev/null
@@ -1,240 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-// 
-// A simple allocator that just hands out space from the next empty zone.  This
-// is temporary, just to get the simplest append-only write workload to work.
-//
-// Copyright (C) 2020 Abutalib Aghayev
-//
-
-#include "ZonedAllocator.h"
-#include "bluestore_types.h"
-#include "zoned_types.h"
-#include "common/debug.h"
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_bluestore
-#undef dout_prefix
-#define dout_prefix *_dout << "ZonedAllocator(" << this << ") " << __func__ << " "
-
-ZonedAllocator::ZonedAllocator(CephContext* cct,
-			       int64_t size,
-			       int64_t blk_size,
-			       int64_t _zone_size,
-			       int64_t _first_sequential_zone,
-			       std::string_view name)
-    : Allocator(name, size, blk_size),
-      cct(cct),
-      size(size),
-      conventional_size(_first_sequential_zone * _zone_size),
-      sequential_size(size - conventional_size),
-      num_sequential_free(0),
-      block_size(blk_size),
-      zone_size(_zone_size),
-      first_seq_zone_num(_first_sequential_zone),
-      starting_zone_num(first_seq_zone_num),
-      num_zones(size / zone_size)
-{
-  ldout(cct, 10) << " size 0x" << std::hex << size
-		 << ", zone size 0x" << zone_size << std::dec
-		 << ", number of zones 0x" << num_zones
-		 << ", first sequential zone 0x" << starting_zone_num
-		 << ", sequential size 0x" << sequential_size
-		 << std::dec
-		 << dendl;
-  ceph_assert(size % zone_size == 0);
-
-  zone_states.resize(num_zones);
-}
-
-ZonedAllocator::~ZonedAllocator()
-{
-}
-
-int64_t ZonedAllocator::allocate(
-  uint64_t want_size,
-  uint64_t alloc_unit,
-  uint64_t max_alloc_size,
-  int64_t hint,
-  PExtentVector *extents)
-{
-  std::lock_guard l(lock);
-
-  ceph_assert(want_size % 4096 == 0);
-
-  ldout(cct, 10) << " trying to allocate 0x"
-		 << std::hex << want_size << std::dec << dendl;
-
-  uint64_t left = num_zones - first_seq_zone_num;
-  uint64_t zone_num = starting_zone_num;
-  for ( ; left > 0; ++zone_num, --left) {
-    if (zone_num == num_zones) {
-      zone_num = first_seq_zone_num;
-    }
-    if (zone_num == cleaning_zone) {
-      ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num
-		     << " because we are cleaning it" << std::dec << dendl;
-      continue;
-    }
-    if (!fits(want_size, zone_num)) {
-      ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num
-		     << " because there is not enough space: "
-		     << " want_size = 0x" << want_size
-		     << " available = 0x" << get_remaining_space(zone_num)
-		     << std::dec
-		     << dendl;
-      continue;
-    }
-    break;
-  }
-
-  if (left == 0) {
-    ldout(cct, 10) << " failed to allocate" << dendl;
-    return -ENOSPC;
-  }
-
-  uint64_t offset = get_offset(zone_num);
-
-  ldout(cct, 10) << " moving zone 0x" << std::hex
-		 << zone_num << " write pointer from 0x" << offset
-		 << " -> 0x" << offset + want_size
-		 << std::dec << dendl;
-
-  increment_write_pointer(zone_num, want_size);
-  num_sequential_free -= want_size;
-  if (get_remaining_space(zone_num) == 0) {
-    starting_zone_num = zone_num + 1;
-  }
-
-  ldout(cct, 10) << " allocated 0x" << std::hex << offset << "~" << want_size
-		 << " from zone 0x" << zone_num
-		 << " and zone offset 0x" << (offset % zone_size)
-		 << std::dec << dendl;
-
-  extents->emplace_back(bluestore_pextent_t(offset, want_size));
-  return want_size;
-}
-
-void ZonedAllocator::release(const interval_set<uint64_t>& release_set)
-{
-  std::lock_guard l(lock);
-  for (auto p = cbegin(release_set); p != cend(release_set); ++p) {
-    auto offset = p.get_start();
-    auto length = p.get_len();
-    uint64_t zone_num = offset / zone_size;
-    ldout(cct, 10) << " 0x" << std::hex << offset << "~" << length
-		   << " from zone 0x" << zone_num << std::dec << dendl;
-    uint64_t num_dead = std::min(zone_size - offset % zone_size, length);
-    for ( ; length; ++zone_num) {
-      increment_num_dead_bytes(zone_num, num_dead);
-      length -= num_dead;
-      num_dead = std::min(zone_size, length);
-    }
-  }
-}
-
-uint64_t ZonedAllocator::get_free()
-{
-  return num_sequential_free;
-}
-
-void ZonedAllocator::dump()
-{
-  std::lock_guard l(lock);
-}
-
-void ZonedAllocator::foreach(
-  std::function<void(uint64_t offset, uint64_t length)> notify)
-{
-  std::lock_guard l(lock);
-}
-
-void ZonedAllocator::init_from_zone_pointers(
-  std::vector<zone_state_t> &&_zone_states)
-{
-  // this is called once, based on the device's zone pointers
-  std::lock_guard l(lock);
-  ldout(cct, 10) << dendl;
-  zone_states = std::move(_zone_states);
-  num_sequential_free = 0;
-  for (size_t i = first_seq_zone_num; i < num_zones; ++i) {
-    num_sequential_free += zone_size - (zone_states[i].write_pointer % zone_size);
-  }
-  ldout(cct, 10) << "free 0x" << std::hex << num_sequential_free
-		 << " / 0x" << sequential_size << std::dec
-		 << dendl;
-}
-
-int64_t ZonedAllocator::pick_zone_to_clean(float min_score, uint64_t min_saved)
-{
-  std::lock_guard l(lock);
-  int32_t best = -1;
-  float best_score = 0.0;
-  for (size_t i = first_seq_zone_num; i < num_zones; ++i) {
-    // value (score) = benefit / cost
-    //    benefit = how much net free space we'll get (dead bytes)
-    //    cost = how many bytes we'll have to rewrite (live bytes)
-    // avoid divide by zero on a zone with no live bytes
-    float score =
-      (float)zone_states[i].num_dead_bytes /
-      (float)(zone_states[i].get_num_live_bytes() + 1);
-    if (score > 0) {
-      ldout(cct, 20) << " zone 0x" << std::hex << i
-		     << " dead 0x" << zone_states[i].num_dead_bytes
-		     << " score " << score
-		     << dendl;
-    }
-    if (zone_states[i].num_dead_bytes < min_saved) {
-      continue;
-    }
-    if (best < 0 || score > best_score) {
-      best = i;
-      best_score = score;
-    }
-  }
-  if (best_score >= min_score) {
-    ldout(cct, 10) << " zone 0x" << std::hex << best << " with score " << best_score
-		   << ": 0x" << zone_states[best].num_dead_bytes
-		   << " dead and 0x"
-		   << zone_states[best].write_pointer - zone_states[best].num_dead_bytes
-		   << " live bytes" << std::dec << dendl;
-  } else if (best > 0) {
-    ldout(cct, 10) << " zone 0x" << std::hex << best << " with score " << best_score
-		   << ": 0x" << zone_states[best].num_dead_bytes
-		   << " dead and 0x"
-		   << zone_states[best].write_pointer - zone_states[best].num_dead_bytes
-		   << " live bytes" << std::dec
-		   << " but below min_score " << min_score
-		   << dendl;
-    best = -1;
-  } else {
-    ldout(cct, 10) << " no zones found that are good cleaning candidates" << dendl;
-  }
-  return best;
-}
-
-void ZonedAllocator::reset_zone(uint32_t zone)
-{
-  num_sequential_free += zone_states[zone].write_pointer;
-  zone_states[zone].reset();
-}
-
-bool ZonedAllocator::low_on_space(void)
-{
-  std::lock_guard l(lock);
-  double free_ratio = static_cast<double>(num_sequential_free) / sequential_size;
-
-  ldout(cct, 10) << " free 0x" << std::hex << num_sequential_free
-		 << "/ 0x" << sequential_size << std::dec
-		 << ", free ratio is " << free_ratio << dendl;
-  ceph_assert(num_sequential_free <= (int64_t)sequential_size);
-
-  // TODO: make 0.25 tunable
-  return free_ratio <= 0.25;
-}
-
-void ZonedAllocator::shutdown()
-{
-  ldout(cct, 1) << dendl;
-}
diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h
deleted file mode 100644
index 0778bd0da9e6..000000000000
--- a/src/os/bluestore/ZonedAllocator.h
+++ /dev/null
@@ -1,120 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-// 
-// A simple allocator that just hands out space from the next empty zone.  This
-// is temporary, just to get the simplest append-only write workload to work.
-//
-// Copyright (C) 2020 Abutalib Aghayev
-//
-
-#ifndef CEPH_OS_BLUESTORE_ZONEDALLOCATOR_H
-#define CEPH_OS_BLUESTORE_ZONEDALLOCATOR_H
-
-#include <mutex>
-
-#include "Allocator.h"
-#include "common/ceph_mutex.h"
-#include "include/btree_map.h"
-#include "include/interval_set.h"
-#include "include/mempool.h"
-#include "bluestore_types.h"
-#include "zoned_types.h"
-
-class ZonedAllocator : public Allocator {
-  CephContext* cct;
-
-  // Currently only one thread at a time calls into ZonedAllocator due to
-  // atomic_alloc_and_submit_lock in BlueStore.cc, but we do locking anyway
-  // because eventually ZONE_APPEND support will land and
-  // atomic_alloc_and_submit_lock will be removed.
-  ceph::mutex lock = ceph::make_mutex("ZonedAllocator::lock");
-
-  uint64_t size;
-  uint64_t conventional_size, sequential_size;
-  std::atomic<int64_t> num_sequential_free;  ///< total bytes in freelist
-  uint64_t block_size;
-  uint64_t zone_size;
-  uint64_t first_seq_zone_num;
-  uint64_t starting_zone_num;
-  uint64_t num_zones;
-  std::atomic<uint32_t> cleaning_zone = -1;
-  std::vector<zone_state_t> zone_states;
-
-  inline uint64_t get_offset(uint64_t zone_num) const {
-    return zone_num * zone_size + get_write_pointer(zone_num);
-  }
-
-public:
-  inline uint64_t get_write_pointer(uint64_t zone_num) const {
-    return zone_states[zone_num].get_write_pointer();
-  }
-private:
-  inline uint64_t get_remaining_space(uint64_t zone_num) const {
-    return zone_size - get_write_pointer(zone_num);
-  }
-
-  inline void increment_write_pointer(uint64_t zone_num, uint64_t want_size) {
-    zone_states[zone_num].increment_write_pointer(want_size);
-  }
-
-  inline void increment_num_dead_bytes(uint64_t zone_num, uint64_t length) {
-    zone_states[zone_num].increment_num_dead_bytes(length);
-  }
-
-  inline bool fits(uint64_t want_size, uint64_t zone_num) const {
-    return want_size <= get_remaining_space(zone_num);
-  }
-
-public:
-  ZonedAllocator(CephContext* cct, int64_t size, int64_t block_size,
-		 int64_t _zone_size,
-		 int64_t _first_sequential_zone,
-		 std::string_view name);
-  ~ZonedAllocator() override;
-
-  const char *get_type() const override {
-    return "zoned";
-  }
-
-  uint64_t get_dead_bytes(uint32_t zone) {
-    return zone_states[zone].num_dead_bytes;
-  }
-  uint64_t get_live_bytes(uint32_t zone) {
-    std::scoped_lock l(lock);
-    return zone_states[zone].write_pointer - zone_states[zone].num_dead_bytes;
-  }
-
-  int64_t allocate(
-    uint64_t want_size, uint64_t alloc_unit, uint64_t max_alloc_size,
-    int64_t hint, PExtentVector *extents) override;
-
-  void release(const interval_set<uint64_t>& release_set) override;
-
-  uint64_t get_free() override;
-
-  void dump() override;
-  void foreach(
-    std::function<void(uint64_t offset, uint64_t length)> notify) override;
-
-  int64_t pick_zone_to_clean(float min_score, uint64_t min_saved);
-  void set_cleaning_zone(uint32_t zone) {
-    cleaning_zone = zone;
-  }
-  void clear_cleaning_zone(uint32_t zone) {
-    cleaning_zone = -1;
-  }
-  void reset_zone(uint32_t zone);
-
-  void init_from_zone_pointers(
-    std::vector<zone_state_t> &&_zone_states);
-  void init_add_free(uint64_t offset, uint64_t length) override {}
-  void init_rm_free(uint64_t offset, uint64_t length) override {}
-
-  void shutdown() override;
-
-private:
-  bool low_on_space(void);
-};
-
-#endif
diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc
deleted file mode 100644
index 3a5bce66fe5d..000000000000
--- a/src/os/bluestore/ZonedFreelistManager.cc
+++ /dev/null
@@ -1,372 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-//
-// A freelist manager for zoned devices.  This iteration just keeps the write
-// pointer per zone.  Following iterations will add enough information to enable
-// cleaning of zones.
-//
-// Copyright (C) 2020 Abutalib Aghayev
-//
-
-#include "ZonedFreelistManager.h"
-#include "bluestore_common.h"
-#include "include/stringify.h"
-#include "kv/KeyValueDB.h"
-#include "os/kv.h"
-#include "zoned_types.h"
-
-#include "common/debug.h"
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_bluestore
-#undef dout_prefix
-#define dout_prefix *_dout << "zoned freelist "
-
-using std::string;
-
-using ceph::bufferlist;
-using ceph::bufferptr;
-using ceph::decode;
-using ceph::encode;
-
-void ZonedFreelistManager::write_zone_state_delta_to_db(
-  uint64_t zone_num,
-  const zone_state_t &zone_state,
-  KeyValueDB::Transaction txn)
-{
-  string key;
-  _key_encode_u64(zone_num, &key);
-  bufferlist bl;
-  zone_state.encode(bl);
-  txn->merge(info_prefix, key, bl);
-}
-
-void ZonedFreelistManager::write_zone_state_reset_to_db(
-  uint64_t zone_num,
-  const zone_state_t &zone_state,
-  KeyValueDB::Transaction txn)
-{
-  string key;
-  _key_encode_u64(zone_num, &key);
-  bufferlist bl;
-  zone_state.encode(bl);
-  txn->set(info_prefix, key, bl);
-}
-
-void ZonedFreelistManager::load_zone_state_from_db(
-  uint64_t zone_num,
-  zone_state_t &zone_state,
-  KeyValueDB::Iterator& it) const
-{
-  string k = it->key();
-  uint64_t zone_num_from_db;
-  _key_decode_u64(k.c_str(), &zone_num_from_db);
-  ceph_assert(zone_num_from_db == zone_num);
-
-  bufferlist bl = it->value();
-  auto p = bl.cbegin();
-  zone_state.decode(p);
-}
-
-void ZonedFreelistManager::init_zone_states(KeyValueDB::Transaction txn)
-{
-  dout(10) << __func__ << dendl;
-  for (uint64_t zone_num = 0; zone_num < num_zones; ++zone_num) {
-    zone_state_t zone_state;
-    write_zone_state_reset_to_db(zone_num, zone_state, txn);
-  }
-}
-
-void ZonedFreelistManager::setup_merge_operator(KeyValueDB *db, string prefix)
-{
-  std::shared_ptr<Int64ArrayMergeOperator> merge_op(
-    new Int64ArrayMergeOperator);
-  db->set_merge_operator(prefix, merge_op);
-}
-
-ZonedFreelistManager::ZonedFreelistManager(
-  CephContext* cct,
-  string meta_prefix,
-  string info_prefix)
-  : FreelistManager(cct),
-    meta_prefix(meta_prefix),
-    info_prefix(info_prefix),
-    enumerate_zone_num(~0UL)
-{
-}
-
-int ZonedFreelistManager::create(
-  uint64_t new_size,
-  uint64_t granularity,
-  uint64_t new_zone_size,
-  uint64_t first_sequential_zone,
-  KeyValueDB::Transaction txn)
-{
-  size = new_size;
-  bytes_per_block = granularity;
-  zone_size = new_zone_size;
-  num_zones = size / zone_size;
-  starting_zone_num = first_sequential_zone;
-  enumerate_zone_num = ~0UL;
-
-  ceph_assert(size % zone_size == 0);
-
-  dout(1) << __func__ << std::hex
-	  << " size 0x" << size
-	  << " bytes_per_block 0x" << bytes_per_block
-	  << " zone size 0x " << zone_size
-	  << " num_zones 0x" << num_zones
-	  << " starting_zone 0x" << starting_zone_num << dendl;
-  {
-    bufferlist bl;
-    encode(size, bl);
-    txn->set(meta_prefix, "size", bl);
-  }
-  {
-    bufferlist bl;
-    encode(bytes_per_block, bl);
-    txn->set(meta_prefix, "bytes_per_block", bl);
-  }
-  {
-    bufferlist bl;
-    encode(zone_size, bl);
-    txn->set(meta_prefix, "zone_size", bl);
-  }
-  {
-    bufferlist bl;
-    encode(num_zones, bl);
-    txn->set(meta_prefix, "num_zones", bl);
-  }
-  {
-    bufferlist bl;
-    encode(starting_zone_num, bl);
-    txn->set(meta_prefix, "starting_zone_num", bl);
-  }
-
-  init_zone_states(txn);
-
-  return 0;
-}
-
-int ZonedFreelistManager::init(
-  KeyValueDB *kvdb,
-  bool db_in_read_only,
-  cfg_reader_t cfg_reader)
-{
-  dout(1) << __func__ << dendl;
-  int r = _read_cfg(cfg_reader);
-  if (r != 0) {
-    return r;
-  }
-
-  ceph_assert(num_zones == size / zone_size);
-
-  dout(10) << __func__ << std::hex
-	   << " size 0x" << size
-	   << " bytes_per_block 0x" << bytes_per_block
-	   << " zone size 0x" << zone_size
-	   << " num_zones 0x" << num_zones
-	   << " starting_zone 0x" << starting_zone_num
-	   << std::dec << dendl;
-  return 0;
-}
-
-void ZonedFreelistManager::sync(KeyValueDB* kvdb)
-{
-}
-
-void ZonedFreelistManager::shutdown()
-{
-  dout(1) << __func__ << dendl;
-}
-
-void ZonedFreelistManager::enumerate_reset()
-{
-  std::lock_guard l(lock);
-
-  dout(1) << __func__ << dendl;
-
-  enumerate_p.reset();
-  enumerate_zone_num = ~0UL;
-}
-
-// Currently, this just iterates over the list of zones and sets |offset| and
-// |length| to the write pointer and the number of remaining free bytes in a
-// given zone.  Hence, it can set |length| to 0 if a zone is full, and it can
-// also return two contiguous empty zones in two calls.  This does not violate
-// current semantics of the call and appears to work fine with the clients of
-// this call.
-bool ZonedFreelistManager::enumerate_next(
-  KeyValueDB *kvdb,
-  uint64_t *offset,
-  uint64_t *length)
-{
-  std::lock_guard l(lock);
-
-  // starting case
-  if (enumerate_zone_num == ~0UL) {
-    dout(30) << __func__ << " start" << dendl;
-    enumerate_p = kvdb->get_iterator(info_prefix);
-    enumerate_p->lower_bound(string());
-    ceph_assert(enumerate_p->valid());
-    enumerate_zone_num = 0;
-  } else {
-    enumerate_p->next();
-    if (!enumerate_p->valid()) {
-      dout(30) << __func__ << " end" << dendl;
-      return false;
-    }
-    ++enumerate_zone_num;
-  }
-
-  zone_state_t zone_state;
-  load_zone_state_from_db(enumerate_zone_num, zone_state, enumerate_p);
-
-  *offset = enumerate_zone_num * zone_size + zone_state.get_write_pointer();
-  *length = zone_size - zone_state.get_write_pointer();
-
-  dout(30) << __func__ << std::hex << " 0x" << *offset << "~" << *length
-	   << std::dec << dendl;
-
-  return true;
-}
-
-void ZonedFreelistManager::dump(KeyValueDB *kvdb)
-{
-  enumerate_reset();
-  uint64_t offset, length;
-  while (enumerate_next(kvdb, &offset, &length)) {
-    dout(20) << __func__ << " 0x" << std::hex << offset << "~" << length
-	     << std::dec << dendl;
-  }
-}
-
-// Advances the write pointer and writes the updated write pointer to database.
-void ZonedFreelistManager::allocate(
-  uint64_t offset,
-  uint64_t length,
-  KeyValueDB::Transaction txn)
-{
-  while (length > 0) {
-    uint64_t zone_num = offset / zone_size;
-    uint64_t this_len = std::min(length, zone_size - offset % zone_size);
-    dout(10) << __func__ << " 0x" << std::hex << offset << "~" << this_len
-	     << " zone 0x" << zone_num << std::dec << dendl;
-    zone_state_t zone_state;
-    zone_state.increment_write_pointer(this_len);
-    write_zone_state_delta_to_db(zone_num, zone_state, txn);
-    offset += this_len;
-    length -= this_len;
-  }
-}
-
-// Increments the number of dead bytes in a zone and writes the updated value to
-// database.  The dead bytes in the zone are not usable.  The cleaner will later
-// copy live objects from the zone to another zone an make the zone writable
-// again.  The number of dead bytes in a zone is used by the cleaner to select
-// which zones to clean -- the ones with most dead bytes are good candidates
-// since they require less I/O.
-void ZonedFreelistManager::release(
-  uint64_t offset,
-  uint64_t length,
-  KeyValueDB::Transaction txn)
-{
-  while (length > 0) {
-    uint64_t zone_num = offset / zone_size;
-    uint64_t this_len = std::min(length, zone_size - offset % zone_size);
-    dout(10) << __func__ << " 0x" << std::hex << offset << "~" << this_len
-	     << " zone 0x" << zone_num << std::dec << dendl;
-    zone_state_t zone_state;
-    zone_state.increment_num_dead_bytes(this_len);
-    write_zone_state_delta_to_db(zone_num, zone_state, txn);
-    length -= this_len;
-    offset += this_len;
-  }
-}
-
-void ZonedFreelistManager::get_meta(
-  uint64_t target_size,
-  std::vector<std::pair<string, string>>* res) const
-{
-  // We do not support expanding devices for now.
-  ceph_assert(target_size == 0);
-  res->emplace_back("zfm_size", stringify(size));
-  res->emplace_back("zfm_bytes_per_block", stringify(bytes_per_block));
-  res->emplace_back("zfm_zone_size", stringify(zone_size));
-  res->emplace_back("zfm_num_zones", stringify(num_zones));
-  res->emplace_back("zfm_starting_zone_num", stringify(starting_zone_num));
-}
-
-std::vector<zone_state_t> ZonedFreelistManager::get_zone_states(
-  KeyValueDB *kvdb) const
-{
-  std::vector<zone_state_t> zone_states;
-  auto p = kvdb->get_iterator(info_prefix);
-  uint64_t zone_num = 0;
-  for (p->lower_bound(string()); p->valid(); p->next(), ++zone_num) {
-    zone_state_t zone_state;
-    load_zone_state_from_db(zone_num, zone_state, p);
-    zone_states.emplace_back(zone_state);
-  }
-  return zone_states;
-}
-
-// TODO: The following function is copied almost verbatim from
-// BitmapFreelistManager.  Eliminate duplication.
-int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader)
-{
-  dout(1) << __func__ << dendl;
-
-  string err;
-
-  const size_t key_count = 5;
-  string keys[key_count] = {
-    "zfm_size",
-    "zfm_bytes_per_block",
-    "zfm_zone_size",
-    "zfm_num_zones",
-    "zfm_starting_zone_num"
-  };
-  uint64_t* vals[key_count] = {
-    &size,
-    &bytes_per_block,
-    &zone_size,
-    &num_zones,
-    &starting_zone_num};
-
-  for (size_t i = 0; i < key_count; i++) {
-    string val;
-    int r = cfg_reader(keys[i], &val);
-    if (r == 0) {
-      *(vals[i]) = strict_iecstrtoll(val.c_str(), &err);
-      if (!err.empty()) {
-        derr << __func__ << " Failed to parse - "
-          << keys[i] << ":" << val
-          << ", error: " << err << dendl;
-        return -EINVAL;
-      }
-    } else {
-      // this is expected for legacy deployed OSDs
-      dout(0) << __func__ << " " << keys[i] << " not found in bdev meta" << dendl;
-      return r;
-    }
-  }
-  return 0;
-}
-
-void ZonedFreelistManager::mark_zone_to_clean_free(
-  uint64_t zone,
-  KeyValueDB *kvdb)
-{
-  dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl;
-
-  KeyValueDB::Transaction txn = kvdb->get_transaction();
-
-  zone_state_t empty_zone_state;
-  write_zone_state_reset_to_db(zone, empty_zone_state, txn);
-
-  // block here until this commits so that we don't end up starting to allocate and
-  // write to the new zone before this fully commits.
-  kvdb->submit_transaction_sync(txn);
-}
diff --git a/src/os/bluestore/ZonedFreelistManager.h b/src/os/bluestore/ZonedFreelistManager.h
deleted file mode 100644
index 378a20f0a796..000000000000
--- a/src/os/bluestore/ZonedFreelistManager.h
+++ /dev/null
@@ -1,113 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-//
-// A freelist manager for zoned devices.
-//
-// Copyright (C) 2020 Abutalib Aghayev
-//
-
-#ifndef CEPH_OS_BLUESTORE_ZONEDFREELISTMANAGER_H
-#define CEPH_OS_BLUESTORE_ZONEDFREELISTMANAGER_H
-
-#include "FreelistManager.h"
-
-#include <string>
-#include <mutex>
-
-#include "common/ceph_mutex.h"
-#include "include/buffer.h"
-#include "kv/KeyValueDB.h"
-#include "zoned_types.h"
-
-using cfg_reader_t = std::function<int(const std::string&, std::string*)>;
-
-class ZonedFreelistManager : public FreelistManager {
-  std::string meta_prefix;    ///< device size, zone size, etc.
-  std::string info_prefix;    ///< per zone write pointer, dead bytes
-  mutable ceph::mutex lock = ceph::make_mutex("ZonedFreelistManager::lock");
-
-  uint64_t size;	      ///< size of sequential region (bytes)
-  uint64_t bytes_per_block;   ///< bytes per allocation unit (bytes)
-  uint64_t zone_size;	      ///< size of a single zone (bytes)
-  uint64_t num_zones;	      ///< number of sequential zones
-  uint64_t starting_zone_num; ///< the first sequential zone number
-
-  KeyValueDB::Iterator enumerate_p;
-  uint64_t enumerate_zone_num;
-
-  void write_zone_state_delta_to_db(uint64_t zone_num,
-				    const zone_state_t &zone_state,
-				    KeyValueDB::Transaction txn);
-  void write_zone_state_reset_to_db(uint64_t zone_num,
-				    const zone_state_t &zone_state,
-				    KeyValueDB::Transaction txn);
-  void load_zone_state_from_db(uint64_t zone_num,
-			       zone_state_t &zone_state,
-			       KeyValueDB::Iterator &it) const;
-
-  void init_zone_states(KeyValueDB::Transaction txn);
-
-  void increment_write_pointer(
-      uint64_t zone, uint64_t length, KeyValueDB::Transaction txn);
-  void increment_num_dead_bytes(
-      uint64_t zone, uint64_t num_bytes, KeyValueDB::Transaction txn);
-
-  int _read_cfg(cfg_reader_t cfg_reader);
-
-public:
-  ZonedFreelistManager(CephContext* cct,
-		       std::string meta_prefix,
-		       std::string info_prefix);
-
-  static void setup_merge_operator(KeyValueDB *db, std::string prefix);
-
-  int create(uint64_t size,
-	     uint64_t granularity,
-	     uint64_t zone_size,
-	     uint64_t first_sequential_zone,
-	     KeyValueDB::Transaction txn) override;
-
-  int init(KeyValueDB *kvdb,
-	   bool db_in_read_only,
-	   cfg_reader_t cfg_reader) override;
-
-  void shutdown() override;
-  void sync(KeyValueDB* kvdb) override;
-  void dump(KeyValueDB *kvdb) override;
-
-  void enumerate_reset() override;
-  bool enumerate_next(KeyValueDB *kvdb,
-		      uint64_t *offset,
-		      uint64_t *length) override;
-
-  void allocate(uint64_t offset,
-		uint64_t length,
-		KeyValueDB::Transaction txn) override;
-
-  void release(uint64_t offset,
-	       uint64_t length,
-	       KeyValueDB::Transaction txn) override;
-
-  inline uint64_t get_size() const override {
-    return size;
-  }
-
-  inline uint64_t get_alloc_units() const override {
-    return size / bytes_per_block;
-  }
-
-  inline uint64_t get_alloc_size() const override {
-    return bytes_per_block;
-  }
-
-  void get_meta(uint64_t target_size,
-		std::vector<std::pair<std::string, std::string>>*) const override;
-
-  std::vector<zone_state_t> get_zone_states(KeyValueDB *kvdb) const;
-
-  void mark_zone_to_clean_free(uint64_t zone,
-			       KeyValueDB *kvdb);
-};
-
-#endif
diff --git a/src/os/bluestore/zoned_types.h b/src/os/bluestore/zoned_types.h
deleted file mode 100644
index d8ca3a0c7c63..000000000000
--- a/src/os/bluestore/zoned_types.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-#ifndef CEPH_OS_BLUESTORE_ZONED_TYPES_H
-#define CEPH_OS_BLUESTORE_ZONED_TYPES_H
-
-#include "include/types.h"
-#include "kv/KeyValueDB.h"
-#include "os/kv.h"
-
-// Tracks two bits of information about the state of a zone: (1) number of dead
-// bytes in a zone and (2) the write pointer.  We use the existing
-// Int64ArrayMergeOperator for merge and avoid the cost of point queries.
-//
-// We use the same struct for an on-disk and in-memory representation of the
-// state.
-struct zone_state_t {
-  uint64_t num_dead_bytes = 0;  ///< dead bytes deallocated (behind the write pointer)
-  uint64_t write_pointer = 0;   ///< relative offset within the zone
-
-  void encode(ceph::buffer::list &bl) const {
-    using ceph::encode;
-    encode(write_pointer, bl);
-    encode(num_dead_bytes, bl);
-  }
-  void decode(ceph::buffer::list::const_iterator &p) {
-    using ceph::decode;
-    decode(write_pointer, p);
-    decode(num_dead_bytes, p);
-  }
-
-  void reset() {
-    write_pointer = 0;
-    num_dead_bytes = 0;
-  }
-
-  uint64_t get_num_dead_bytes() const {
-    return num_dead_bytes;
-  }
-
-  uint64_t get_num_live_bytes() const {
-    return write_pointer - num_dead_bytes;
-  }
-
-  uint64_t get_write_pointer() const {
-    return write_pointer;
-  }
-
-  void increment_num_dead_bytes(uint64_t num_bytes) {
-    num_dead_bytes += num_bytes;
-  }
-
-  void increment_write_pointer(uint64_t num_bytes) {
-    write_pointer += num_bytes;
-  }
-
-  friend std::ostream& operator<<(
-    std::ostream& out,
-    const zone_state_t& zone_state) {
-    return out << std::hex
-	       << " dead bytes: 0x" << zone_state.get_num_dead_bytes()
-	       << " write pointer: 0x"  << zone_state.get_write_pointer()
-	       << " " << std::dec;
-  }
-};
-
-#endif
diff --git a/src/test/objectstore/run_smr_bluestore_test.sh b/src/test/objectstore/run_smr_bluestore_test.sh
deleted file mode 100644
index d689cf2c5011..000000000000
--- a/src/test/objectstore/run_smr_bluestore_test.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash -ex
-
-# 1) run_smr_bluestore_test.sh
-# Setup smr device, run all tests
-
-# 2) run_smr_bluestore_test.sh --smr
-# Setup smr device but skip tests failing on smr
-
-
-before_creation=$(mktemp)
-lsscsi > $before_creation
-
-echo "cd /backstores/user:zbc
-create name=zbc0 size=20G cfgstring=model-HM/zsize-256/conv-10@zbc0.raw
-/loopback create
-cd /loopback
-create naa.50014055e5f25aa0
-cd naa.50014055e5f25aa0/luns
-create /backstores/user:zbc/zbc0 0
-" | sudo targetcli
-
-sleep 1 #if too fast device does not show up
-after_creation=$(mktemp)
-lsscsi > $after_creation
-if [[ $(diff $before_creation $after_creation | wc -l ) != 2 ]]
-then
-    echo New zbc device not created
-    false
-fi
-
-function cleanup() {
-    echo "cd /loopback
-delete naa.50014055e5f25aa0
-cd /backstores/user:zbc
-delete zbc0" | sudo targetcli
-    sudo rm -f zbc0.raw
-    rm -f $before_creation $after_creation
-}
-trap cleanup EXIT
-
-DEV=$(diff $before_creation $after_creation |grep zbc |sed "s@.* /@/@")
-sudo chmod 666 $DEV
-# Need sudo
-# https://patchwork.kernel.org/project/linux-block/patch/20210811110505.29649-3-Niklas.Cassel@wdc.com/
-sudo ceph_test_objectstore \
-    --bluestore-block-path $DEV \
-    --gtest_filter=*/2 \
-    $*
diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 99da6002ef00..de779eb039dc 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -55,8 +55,6 @@ typedef boost::mt11213b gen_type;
 const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000;
 #define dout_context g_ceph_context
 
-bool smr = false;
-
 static bool bl_eq(bufferlist& expected, bufferlist& actual)
 {
   if (expected.contents_equal(actual))
@@ -1278,16 +1276,11 @@ void StoreTest::doCompressionTest()
 TEST_P(StoreTest, CompressionTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "TODO: need to adjust statfs check for smr" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_compression_algorithm", "snappy");
   SetVal(g_conf(), "bluestore_compression_mode", "force");
   g_ceph_context->_conf.apply_changes(nullptr);
   doCompressionTest();
-
   SetVal(g_conf(), "bluestore_compression_algorithm", "zlib");
   SetVal(g_conf(), "bluestore_compression_mode", "aggressive");
   g_ceph_context->_conf.apply_changes(nullptr);
@@ -1514,10 +1507,6 @@ TEST_P(StoreTest, SimpleObjectTest) {
 TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) {
   if(string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP (smr)" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_max_blob_size", "524288");
   SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd");
@@ -1613,10 +1602,6 @@ TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) {
 TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) {
   if(string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "TODO: fix this for smr" << std::endl;
-    return;
-  }
   SetVal(g_conf(), "bluestore_block_db_path", "");
   StartDeferred(65536);
   SetVal(g_conf(), "bluestore_compression_mode", "force");
@@ -2147,10 +2132,6 @@ TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) {
 TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) {
   if(string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "TODO: fix this for smr" << std::endl;
-    return;
-  }
   SetVal(g_conf(), "bluestore_block_db_path", "");
   StartDeferred(0x10000);
 
@@ -3782,10 +3763,6 @@ TEST_P(StoreTest, SimpleCloneRangeTest) {
 TEST_P(StoreTest, BlueStoreUnshareBlobTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: non-deterministic behavior with smr" << std::endl;
-    return;
-  }
   int r;
   coll_t cid;
   auto ch = store->create_new_collection(cid);
@@ -7164,11 +7141,6 @@ void doMany4KWritesTest(ObjectStore* store,
 TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply"
-	 << std::endl;
-    return;
-  }
 
   StartDeferred(0x10000);
 
@@ -7179,11 +7151,6 @@ TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) {
 TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply"
-	 << std::endl;
-    return;
-  }
   StartDeferred(0x10000);
   SetVal(g_conf(), "bluestore_csum_type", "none");
   g_ceph_context->_conf.apply_changes(nullptr);
@@ -7195,11 +7162,6 @@ TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) {
 TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply"
-	 << std::endl;
-    return;
-  }
   StartDeferred(0x10000);
   const unsigned max_object = 4*1024*1024;
   doMany4KWritesTest(store.get(), 1, 1000, max_object, 4*1024, 0);
@@ -7575,9 +7537,6 @@ TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionSmallOverwrite) {
   if (string(GetParam()) != "bluestore" || !cct->_conf->bluestore_zero_block_detection) {
     GTEST_SKIP() << "not bluestore or bluestore_zero_block_detection=false, skipping";
   }
-  if (smr) {
-    GTEST_SKIP() << "smr, skipping";
-  }
 
   size_t block_size = 65536;
   StartDeferred(block_size);
@@ -7755,9 +7714,6 @@ TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionBigOverwrite) {
   if (string(GetParam()) != "bluestore" || !cct->_conf->bluestore_zero_block_detection) {
     GTEST_SKIP() << "not bluestore or bluestore_zero_block_detection=false, skipping";
   }
-  if (smr) {
-    GTEST_SKIP() << "smr, skipping";
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -7858,10 +7814,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite1) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -8046,10 +7998,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -8196,10 +8144,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -8362,10 +8306,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite4) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   SetVal(g_conf(), "bluestore_block_db_create", "true");
@@ -8440,10 +8380,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite5) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   SetVal(g_conf(), "bluestore_block_db_create", "true");
@@ -8518,10 +8454,6 @@ TEST_P(StoreTestSpecificAUSize, DeferredDifferentChunks) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t alloc_size = 4096;
   size_t large_object_size = 1 * 1024 * 1024;
@@ -8625,10 +8557,6 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no overwrite" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -8805,10 +8733,6 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no overwrite" << std::endl;
-    return;
-  }
 
   size_t block_size = 4096;
   StartDeferred(block_size);
@@ -8971,10 +8895,6 @@ TEST_P(StoreTestSpecificAUSize, ReproBug56488Test) {
 
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: no deferred" << std::endl;
-    return;
-  }
 
   size_t alloc_size = 65536;
   size_t write_size = 4096;
@@ -9147,10 +9067,6 @@ TEST_P(StoreTestSpecificAUSize, garbageCollection) {
   int write_offset = buf_len;
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: assertions about allocations need to be adjusted" << std::endl;
-    return;
-  }
 
 #define WRITE_AT(offset, _length) {\
       ObjectStore::Transaction t;\
@@ -9346,10 +9262,6 @@ namespace {
 TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "TODO: repair mismatched write pointer (+ dead bytes mismatch)" << std::endl;
-    return;
-  }
   const size_t offs_base = 65536 / 2;
 
 
@@ -9554,10 +9466,6 @@ TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) {
 TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: smr repair is different" << std::endl;
-    return;
-  }
   SetVal(g_conf(), "bluestore_fsck_on_mount", "false");
   SetVal(g_conf(), "bluestore_fsck_on_umount", "false");
 
@@ -9627,10 +9535,6 @@ TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) {
 TEST_P(StoreTestSpecificAUSize, BluestoreRepairSharedBlobTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "TODO: repair mismatched write pointer (+ dead bytes mismatch)" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_fsck_on_mount", "false");
   SetVal(g_conf(), "bluestore_fsck_on_umount", "false");
@@ -9703,10 +9607,6 @@ TEST_P(StoreTestSpecificAUSize, BluestoreRepairSharedBlobTest) {
 TEST_P(StoreTestSpecificAUSize, BluestoreBrokenNoSharedBlobRepairTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: smr repair is different" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_fsck_on_mount", "false");
   SetVal(g_conf(), "bluestore_fsck_on_umount", "false");
@@ -10320,41 +10220,6 @@ TEST_P(StoreTest, mergeRegionTest) {
   }
 }
 
-TEST_P(StoreTest, FixSMRWritePointer) {
-  if(string(GetParam()) != "bluestore")
-    return;
-  if (!smr)
-    return;
-  int r = store->umount();
-  ASSERT_EQ(0, r);
-
-  // copied from StoreTestFixture
-  std::string path = GetParam() + ".test_temp_dir"s;
-
-  std::string p = path + "/block";
-  BlockDevice* bdev = BlockDevice::create(g_ceph_context, p, nullptr, nullptr, nullptr, nullptr);
-  r = bdev->open(p);
-  ASSERT_EQ(0, r);
-  ASSERT_EQ(true, bdev->is_smr());
-
-  std::vector<uint64_t> wp = bdev->get_zones();
-  uint64_t first_seq_zone = bdev->get_conventional_region_size() / bdev->get_zone_size();
-
-  IOContext ioc(g_ceph_context, NULL, true);
-  bufferlist bl;
-  bl.append(std::string(1024 * 1024, 'x'));
-  r = bdev->aio_write(wp[first_seq_zone], bl, &ioc, false);
-  ASSERT_EQ(0, r);
-  bdev->aio_submit(&ioc);
-  ioc.aio_wait();
-  bdev->close();
-  delete bdev;
-
-  r = store->mount();
-  ASSERT_EQ(0, r);
-}
-
-
 TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) {
   if (string(GetParam()) != "bluestore")
     return;
@@ -10425,10 +10290,6 @@ TEST_P(StoreTestSpecificAUSize, ReproNoBlobMultiTest) {
 
   if(string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP (FIXME): bluestore gc does not seem to do the trick here" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "4294967296");
@@ -10546,10 +10407,6 @@ void doManySetAttr(ObjectStore* store,
 TEST_P(StoreTestSpecificAUSize, SpilloverLegacyTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
@@ -10581,10 +10438,6 @@ TEST_P(StoreTestSpecificAUSize, SpilloverLegacyTest) {
 TEST_P(StoreTestSpecificAUSize, SpilloverLegacyFixedByFitToFastTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
@@ -10632,10 +10485,6 @@ void do_bluefs_write(BlueFS* _fs,
 TEST_P(StoreTestSpecificAUSize, SpilloverTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
@@ -10666,10 +10515,6 @@ TEST_P(StoreTestSpecificAUSize, SpilloverTest) {
 TEST_P(StoreTestSpecificAUSize, SpilloverFixedCompletelyTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", "3221225472");
@@ -10693,10 +10538,6 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixedCompletelyTest) {
 TEST_P(StoreTestSpecificAUSize, SpilloverFixedPartialTest) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl;
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_block_db_create", "true");
   SetVal(g_conf(), "bluestore_block_db_size", stringify(3ull << 30).c_str());
@@ -10732,9 +10573,6 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixedPartialTest) {
 TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) {
   if (string(GetParam()) != "bluestore")
     return;
-  if (smr) {
-    return;
-  }
 
   SetVal(g_conf(), "bluestore_default_buffered_write", "true");
   SetVal(g_conf(), "bluestore_max_blob_size", "65536");
@@ -11059,18 +10897,6 @@ int main(int argc, char **argv) {
 			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
   common_init_finish(g_ceph_context);
 
-  for (auto& i : args) {
-    if (i == "--smr"s) {
-#if defined(HAVE_LIBZBD)
-      derr << "Adjusting tests for smr mode." << dendl;
-      smr = true;
-#else
-      derr << "smr mode selected, but support not compiled in" << dendl;
-      return 1;
-#endif
-    }
-  }
-
   // make sure we can adjust any config settings
   g_ceph_context->_conf._clear_safe_to_start_threads();
 
diff --git a/src/vstart.sh b/src/vstart.sh
index 1187e86b9ed8..a76eadfb54ec 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -252,7 +252,6 @@ options:
 	--bluestore-devs: comma-separated list of blockdevs to use for bluestore
 	--bluestore-db-devs: comma-separated list of db-devs to use for bluestore
 	--bluestore-wal-devs: comma-separated list of wal-devs to use for bluestore
-	--bluestore-zoned: blockdevs listed by --bluestore-devs are zoned devices (HM-SMR HDD or ZNS SSD)
 	--bluestore-io-uring: enable io_uring backend
 	--inc-osd: append some more osds into existing vcluster
 	--cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub]
@@ -586,9 +585,6 @@ case $1 in
         parse_bluestore_wal_devs --bluestore-wal-devs "$2"
         shift
         ;;
-    --bluestore-zoned)
-        zoned_enabled=1
-        ;;
     --bluestore-io-uring)
         io_uring_enabled=1
         shift
@@ -852,7 +848,6 @@ EOF
         bluestore prefer deferred size = 0
         bluestore prefer deferred size hdd = 0
         bluestore prefer deferred size ssd = 0
-        bluestore allocator = zoned"
         fi
         if [ "$io_uring_enabled" -eq 1 ]; then
             BLUESTORE_OPTS+="

From 159751b68085fbe0fe10a881ff8bedecda11142f Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Fri, 22 Dec 2023 22:55:29 +0000
Subject: [PATCH 1206/2492] doc/rados/operations: document `ceph balancer
 status detail`

Document change in https://github.com/ceph/ceph/pull/54801

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 PendingReleaseNotes               | 4 ++++
 doc/rados/operations/balancer.rst | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 8041ab81ab75..4038993d8e47 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -99,6 +99,10 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   by disabling async recovery by setting osd_async_recovery_min_cost to a very
   large value on all OSDs until the upgrade is complete:
   ``ceph config set osd osd_async_recovery_min_cost 1099511627776``
+* RADOS: A detailed version of the `balancer status` CLI command in the balancer
+  module is now available. Users may run `ceph balancer status detail` to see more
+  details about which PGs were updated in the balancer's last optimization.
+  See https://docs.ceph.com/en/latest/rados/operations/balancer/ for more information.
 
 >=18.0.0
 
diff --git a/doc/rados/operations/balancer.rst b/doc/rados/operations/balancer.rst
index aa4eab93cf96..5f87865bcf32 100644
--- a/doc/rados/operations/balancer.rst
+++ b/doc/rados/operations/balancer.rst
@@ -205,6 +205,12 @@ command:
 
       ceph balancer status
 
+To see the status in greater detail, run the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer status detail
+
 To evaluate the distribution that would result from executing a specific plan,
 run the following command:
 

From 1e905d2c414d750e67516b299a42f23809c87d25 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 26 Sep 2023 12:03:58 -0400
Subject: [PATCH 1207/2492] qa/rgw/upgrade: add reef as a starting release

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/upgrade/1-install/reef/%                         | 0
 qa/suites/rgw/upgrade/1-install/reef/.qa                       | 1 +
 qa/suites/rgw/upgrade/1-install/reef/distro$/.qa               | 1 +
 .../rgw/upgrade/1-install/reef/distro$/centos_latest.yaml      | 1 +
 .../rgw/upgrade/1-install/reef/distro$/ubuntu_latest.yaml      | 1 +
 qa/suites/rgw/upgrade/1-install/reef/install.yaml              | 3 +++
 qa/suites/rgw/upgrade/1-install/reef/overrides.yaml            | 3 +++
 7 files changed, 10 insertions(+)
 create mode 100644 qa/suites/rgw/upgrade/1-install/reef/%
 create mode 120000 qa/suites/rgw/upgrade/1-install/reef/.qa
 create mode 120000 qa/suites/rgw/upgrade/1-install/reef/distro$/.qa
 create mode 120000 qa/suites/rgw/upgrade/1-install/reef/distro$/centos_latest.yaml
 create mode 120000 qa/suites/rgw/upgrade/1-install/reef/distro$/ubuntu_latest.yaml
 create mode 100644 qa/suites/rgw/upgrade/1-install/reef/install.yaml
 create mode 100644 qa/suites/rgw/upgrade/1-install/reef/overrides.yaml

diff --git a/qa/suites/rgw/upgrade/1-install/reef/% b/qa/suites/rgw/upgrade/1-install/reef/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/upgrade/1-install/reef/.qa b/qa/suites/rgw/upgrade/1-install/reef/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/reef/distro$/.qa b/qa/suites/rgw/upgrade/1-install/reef/distro$/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/distro$/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/reef/distro$/centos_latest.yaml b/qa/suites/rgw/upgrade/1-install/reef/distro$/centos_latest.yaml
new file mode 120000
index 000000000000..bd9854e70298
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/distro$/centos_latest.yaml
@@ -0,0 +1 @@
+.qa/distros/supported/centos_latest.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/reef/distro$/ubuntu_latest.yaml b/qa/suites/rgw/upgrade/1-install/reef/distro$/ubuntu_latest.yaml
new file mode 120000
index 000000000000..3a09f9abb05c
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/distro$/ubuntu_latest.yaml
@@ -0,0 +1 @@
+.qa/distros/supported/ubuntu_latest.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/upgrade/1-install/reef/install.yaml b/qa/suites/rgw/upgrade/1-install/reef/install.yaml
new file mode 100644
index 000000000000..b3b7bbfab5a0
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/install.yaml
@@ -0,0 +1,3 @@
+tasks:
+- install:
+    branch: reef
diff --git a/qa/suites/rgw/upgrade/1-install/reef/overrides.yaml b/qa/suites/rgw/upgrade/1-install/reef/overrides.yaml
new file mode 100644
index 000000000000..8380c4c1b9a0
--- /dev/null
+++ b/qa/suites/rgw/upgrade/1-install/reef/overrides.yaml
@@ -0,0 +1,3 @@
+overrides:
+  ragweed:
+    default-branch: ceph-reef

From 883f99dd3e17f88fd90cdb6b7c221d3a63bce6b9 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 9 Jan 2023 16:07:20 +0200
Subject: [PATCH 1208/2492] crimson/osd/object_metadata_helper: Introduce
 calc_*_subsets

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/CMakeLists.txt                |   1 +
 src/crimson/osd/object_metadata_helper.cc     | 222 +++++++++++++++
 src/crimson/osd/object_metadata_helper.h      |  24 ++
 src/crimson/osd/replicated_recovery_backend.h |   1 +
 src/test/crimson/CMakeLists.txt               |   7 +
 src/test/crimson/test_calc_subsets.cc         | 255 ++++++++++++++++++
 6 files changed, 510 insertions(+)
 create mode 100644 src/crimson/osd/object_metadata_helper.cc
 create mode 100644 src/crimson/osd/object_metadata_helper.h
 create mode 100644 src/test/crimson/test_calc_subsets.cc

diff --git a/src/crimson/osd/CMakeLists.txt b/src/crimson/osd/CMakeLists.txt
index 65fb7201f766..c77da7575a50 100644
--- a/src/crimson/osd/CMakeLists.txt
+++ b/src/crimson/osd/CMakeLists.txt
@@ -15,6 +15,7 @@ add_executable(crimson-osd
   pg_shard_manager.cc
   object_context.cc
   object_context_loader.cc
+  object_metadata_helper.cc
   ops_executer.cc
   osd_operation.cc
   osd_operations/client_request.cc
diff --git a/src/crimson/osd/object_metadata_helper.cc b/src/crimson/osd/object_metadata_helper.cc
new file mode 100644
index 000000000000..3016b0ae406b
--- /dev/null
+++ b/src/crimson/osd/object_metadata_helper.cc
@@ -0,0 +1,222 @@
+#include "crimson/osd/object_metadata_helper.h"
+
+namespace {
+  seastar::logger& logger() {
+    return crimson::get_logger(ceph_subsys_osd);
+  }
+}
+
+namespace crimson::osd {
+
+/*
+ *   The clone object content may already overlap with the
+ *   next older and the next newest clone obejct.
+ *   Use the existing (next) clones object overlaps instead
+ *   of pushing the whole clone object to the replica.
+ */
+
+subsets_t calc_clone_subsets(
+  SnapSet& snapset, const hobject_t& soid,
+  const pg_missing_t& missing,
+  const hobject_t &last_backfill)
+{
+  subsets_t subsets;
+  logger().debug("{}: {} clone_overlap {} ",
+                 __func__, soid, snapset.clone_overlap);
+
+  uint64_t size = snapset.clone_size[soid.snap];
+  if (size) {
+    subsets.data_subset.insert(0, size);
+  }
+
+  // TODO: make sure CEPH_FEATURE_OSD_CACHEPOOL is not supported in Crimson
+  // Skips clone subsets if caching was enabled (allow_incomplete_clones).
+
+#ifndef UNIT_TESTS_BUILT
+  if (!crimson::common::local_conf()->osd_recover_clone_overlap) {
+    logger().debug("{} {} -- osd_recover_clone_overlap is disabled",
+                   __func__, soid); ;
+    return subsets;
+  }
+#endif
+
+  if (snapset.clones.empty()) {
+    logger().debug("{} {} -- no clones", __func__, soid);
+    return subsets;
+  }
+
+  auto soid_snap_iter = find(snapset.clones.begin(),
+                             snapset.clones.end(),
+                             soid.snap);
+  assert(soid_snap_iter != snapset.clones.end());
+  auto soid_snap_index = soid_snap_iter - snapset.clones.begin();
+
+  // any overlap with next older clone?
+  interval_set<uint64_t> cloning;
+  interval_set<uint64_t> prev;
+  if (size) {
+    prev.insert(0, size);
+  }
+  for (int i = soid_snap_index - 1; i >= 0; i--) {
+    hobject_t clone = soid;
+    clone.snap = snapset.clones[i];
+    // clone_overlap of i holds the overlap between i to i+1
+    prev.intersection_of(snapset.clone_overlap[snapset.clones[i]]);
+    if (!missing.is_missing(clone) && clone < last_backfill) {
+      logger().debug("{} {} has prev {} overlap {}",
+                     __func__, soid, clone, prev);
+      subsets.clone_subsets[clone] = prev;
+      cloning.union_of(prev);
+      break;
+    }
+    logger().debug("{} {} does not have prev {} overlap {}",
+                   __func__, soid, clone, prev);
+  }
+
+  // overlap with next newest?
+  interval_set<uint64_t> next;
+  if (size) {
+    next.insert(0, size);
+  }
+  for (unsigned i = soid_snap_index+1;
+       i < snapset.clones.size(); i++) {
+    hobject_t clone = soid;
+    clone.snap = snapset.clones[i];
+    // clone_overlap of i-1 holds the overlap between i-1 to i
+    next.intersection_of(snapset.clone_overlap[snapset.clones[i - 1]]);
+    if (!missing.is_missing(clone) && clone < last_backfill) {
+      logger().debug("{} {} has next {} overlap {}",
+                     __func__, soid, clone, next);
+      subsets.clone_subsets[clone] = next;
+      cloning.union_of(next);
+      break;
+    }
+    logger().debug("{} {} does not have next {} overlap {}",
+                   __func__, soid, clone, next);
+  }
+
+#ifndef UNIT_TESTS_BUILT
+  if (cloning.num_intervals() >
+      crimson::common::local_conf().get_val<uint64_t>
+      ("osd_recover_clone_overlap_limit")) {
+    logger().debug("skipping clone, too many holes");
+    subsets.clone_subsets.clear();
+    cloning.clear();
+  }
+#endif
+
+  // what's left for us to push?
+  subsets.data_subset.subtract(cloning);
+  logger().debug("{} {} data_subsets {}"
+                 "clone_subsets {}",
+                 __func__, soid, subsets.data_subset, subsets.clone_subsets);
+  return subsets;
+}
+
+/*
+ * Instead of pushing the whole object to the replica,
+ * make use of:
+ * 1) ObjectCleanRegion - push modified content only.
+ *    - See: dev/osd_internals/partial_object_recovery
+ * 2) The modified content may already overlap with the
+ *    next older clone obejct. Use the existing clone
+ *    object overlap as well.
+ */
+
+subsets_t calc_head_subsets(
+  uint64_t obj_size,
+  SnapSet& snapset,
+  const hobject_t& head,
+  const pg_missing_t& missing,
+  const hobject_t &last_backfill)
+{
+  logger().debug("{}: {} clone_overlap {} ",
+                 __func__, head, snapset.clone_overlap);
+
+  subsets_t subsets;
+
+// 1) Calculate modified content only
+  if (obj_size) {
+    subsets.data_subset.insert(0, obj_size);
+  }
+  assert(missing.get_items().contains(head));
+  const pg_missing_item missing_item = missing.get_items().at(head);
+  // let data_subset store only the modified content of the object.
+  subsets.data_subset.intersection_of(missing_item.clean_regions.get_dirty_regions());
+  logger().debug("{} {} data_subset {}",
+                 __func__, head, subsets.data_subset);
+
+  // TODO: make sure CEPH_FEATURE_OSD_CACHEPOOL is not supported in Crimson
+  // Skips clone subsets if caching was enabled (allow_incomplete_clones).
+
+#ifndef UNIT_TESTS_BUILT
+  if (!crimson::common::local_conf()->osd_recover_clone_overlap) {
+    logger().debug("{} {} -- osd_recover_clone_overlap is disabled",
+                   __func__, head);
+    return subsets;
+  }
+#endif
+
+  if (snapset.clones.empty()) {
+    logger().debug("{} {} -- no clones", __func__, head);
+    return subsets;
+  }
+
+  // 2) Find any overlap with next older clone
+  interval_set<uint64_t> cloning;
+  interval_set<uint64_t> prev;
+  hobject_t clone = head;
+  if (obj_size) {
+    prev.insert(0, obj_size);
+  }
+  for (int i = snapset.clones.size()-1; i >= 0; i--) {
+    clone.snap = snapset.clones[i];
+    // let prev store only the overlap with clone i
+    prev.intersection_of(snapset.clone_overlap[snapset.clones[i]]);
+    if (!missing.is_missing(clone) && clone < last_backfill) {
+      logger().debug("{} {} has prev {} overlap {}",
+                     __func__, head, clone, prev);
+      cloning = prev;
+      break;
+    }
+    logger().debug("{} {} does not have prev {} overlap {}",
+                   __func__, head, clone, prev);
+  }
+
+  // let cloning store only the overlap with data_subset
+  cloning.intersection_of(subsets.data_subset);
+  if (cloning.empty()) {
+    logger().debug("skipping clone, nothing needs to clone");
+    return subsets;
+  }
+
+#ifndef UNIT_TESTS_BUILT
+  if (cloning.num_intervals() >
+      crimson::common::local_conf().get_val<uint64_t>
+      ("osd_recover_clone_overlap_limit")) {
+    logger().debug("skipping clone, too many holes");
+    subsets.clone_subsets.clear();
+    cloning.clear();
+  }
+#endif
+
+  // what's left for us to push?
+  subsets.clone_subsets[clone] = cloning;
+  subsets.data_subset.subtract(cloning);
+  logger().debug("{} {} data_subsets {}"
+                 "clone_subsets {}",
+                 __func__, head, subsets.data_subset, subsets.clone_subsets);
+
+  return subsets;
+}
+
+void set_subsets(
+  const subsets_t& subsets,
+  ObjectRecoveryInfo& recovery_info)
+{
+  recovery_info.copy_subset = subsets.data_subset;
+  recovery_info.clone_subset = subsets.clone_subsets;
+}
+
+
+}
diff --git a/src/crimson/osd/object_metadata_helper.h b/src/crimson/osd/object_metadata_helper.h
new file mode 100644
index 000000000000..927fc4888111
--- /dev/null
+++ b/src/crimson/osd/object_metadata_helper.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "osd/osd_types_fmt.h"
+
+namespace crimson::osd {
+  struct subsets_t {
+    interval_set<uint64_t> data_subset;
+    std::map<hobject_t, interval_set<uint64_t>> clone_subsets;
+  };
+
+  subsets_t calc_clone_subsets(
+    SnapSet& snapset, const hobject_t& soid,
+    const pg_missing_t& missing,
+    const hobject_t &last_backfill);
+  subsets_t calc_head_subsets(
+    uint64_t obj_size,
+    SnapSet& snapset,
+    const hobject_t& head,
+    const pg_missing_t& missing,
+    const hobject_t &last_backfill);
+  void set_subsets(
+    const subsets_t& subsets,
+    ObjectRecoveryInfo& recovery_info);
+}
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index 16d6369a91f3..8fdaf9a10f8c 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -6,6 +6,7 @@
 #include "crimson/common/interruptible_future.h"
 #include "crimson/osd/pg_interval_interrupt_condition.h"
 #include "crimson/osd/recovery_backend.h"
+#include "crimson/osd/object_metadata_helper.h"
 
 #include "messages/MOSDPGPull.h"
 #include "messages/MOSDPGPush.h"
diff --git a/src/test/crimson/CMakeLists.txt b/src/test/crimson/CMakeLists.txt
index c943ff885464..e1a5dfe73dfd 100644
--- a/src/test/crimson/CMakeLists.txt
+++ b/src/test/crimson/CMakeLists.txt
@@ -71,6 +71,13 @@ add_ceph_unittest(unittest-seastar-lru
   --memory 256M --smp 1)
 target_link_libraries(unittest-seastar-lru crimson GTest::Main)
 
+add_executable(unittest-seastar-calc-subsets
+    ${PROJECT_SOURCE_DIR}/src/crimson/osd/object_metadata_helper.cc
+  test_calc_subsets.cc)
+add_ceph_unittest(unittest-seastar-calc-subsets
+  --memory 256M --smp 1)
+target_link_libraries(unittest-seastar-calc-subsets crimson GTest::Main)
+
 add_executable(unittest-fixed-kv-node-layout
   test_fixed_kv_node_layout.cc)
 add_ceph_unittest(unittest-fixed-kv-node-layout)
diff --git a/src/test/crimson/test_calc_subsets.cc b/src/test/crimson/test_calc_subsets.cc
new file mode 100644
index 000000000000..7d23810c2e60
--- /dev/null
+++ b/src/test/crimson/test_calc_subsets.cc
@@ -0,0 +1,255 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "gtest/gtest.h"
+#include "crimson/osd/object_metadata_helper.h"
+
+
+TEST(head_subsets, dirty_region)
+{
+  uint64_t obj_size = 10;
+  SnapSet empty_ss;
+  hobject_t head{object_t{"foo"}, "foo", CEPH_NOSNAP, 42, 0, "nspace"};
+  pg_missing_t missing;
+  pg_missing_item item;
+  uint64_t offset_1, len_1;
+  offset_1 = 3;
+  len_1 = 2;
+  item.clean_regions.mark_data_region_dirty(offset_1, len_1);
+  missing.add(head, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+  interval_set<uint64_t> expect_data_region;
+  expect_data_region.insert(offset_1, len_1);
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_head_subsets(obj_size,
+                                    empty_ss,
+                                    head,
+                                    missing,
+                                    last_backfill);
+
+  EXPECT_TRUE(result.clone_subsets.empty());
+  EXPECT_TRUE(result.data_subset == expect_data_region);
+}
+
+TEST(head_subsets, head_all_clean)
+{
+  uint64_t obj_size = 10;
+  SnapSet empty_ss;
+  hobject_t head{object_t{"foo"}, "foo", CEPH_NOSNAP, 42, 0, "nspace"};
+  pg_missing_t missing;
+  pg_missing_item item;
+  missing.add(head, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_head_subsets(obj_size,
+                                    empty_ss,
+                                    head,
+                                    missing,
+                                    last_backfill);
+
+  EXPECT_TRUE(result.clone_subsets.empty());
+  EXPECT_TRUE(result.data_subset.empty());
+}
+
+TEST(head_subsets, all_dirty)
+{
+  uint64_t obj_size = 10;
+  SnapSet empty_ss;
+  hobject_t head{object_t{"foo"}, "foo", CEPH_NOSNAP, 42, 0, "nspace"};
+  pg_missing_t missing;
+  pg_missing_item item;
+  item.clean_regions.mark_fully_dirty();
+  missing.add(head, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_head_subsets(obj_size,
+                                    empty_ss,
+                                    head,
+                                    missing,
+                                    last_backfill);
+
+  EXPECT_TRUE(result.clone_subsets.empty());
+  EXPECT_TRUE(result.data_subset.size() == obj_size);
+}
+
+TEST(head_subsets, clone_overlap)
+{
+  uint64_t obj_size = 10;
+  SnapSet ss;
+  hobject_t head{object_t{"foo"}, "foo", CEPH_NOSNAP, 42, 0, "nspace"};
+  pg_missing_t missing;
+  pg_missing_item item;
+  item.clean_regions.mark_fully_dirty();
+  missing.add(head, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+
+  // Clone object:
+  hobject_t clone = head;
+  clone.snap = 0;
+  std::map<snapid_t, interval_set<uint64_t>> clone_overlap;  // overlap w/ next
+  interval_set<uint64_t> overlap;
+  uint64_t offset_2, len_2;
+  offset_2 = 2;
+  len_2 = 2;
+  overlap.insert(offset_2, len_2);
+  clone_overlap[clone.snap] = overlap;
+
+  // Snapset:
+  // ss.seq = 0;
+  // ss.snaps = snaps; (legacy)
+  ss.clones.push_back(clone.snap);
+  ss.clone_overlap = clone_overlap;
+  // ss.clone_size = clone_size;
+  // ss.clone_snaps = clone_snaps;
+
+  // Expected intervals:
+  interval_set<uint64_t> expect_clone_subset;
+  expect_clone_subset.insert(offset_2, len_2);
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_head_subsets(obj_size,
+                                    ss,
+                                    head,
+                                    missing,
+                                    last_backfill);
+  EXPECT_TRUE(result.clone_subsets[clone] == expect_clone_subset);
+}
+
+TEST(head_subsets, dirty_region_and_clone_overlap)
+{
+  uint64_t obj_size = 100;
+  SnapSet ss;
+  hobject_t head{object_t{"foo"}, "foo", CEPH_NOSNAP, 42, 0, "nspace"};
+  pg_missing_t missing;
+  pg_missing_item item;
+  uint64_t offset_1, len_1;
+  offset_1 = 3;
+  len_1 = 2;
+  item.clean_regions.mark_data_region_dirty(offset_1, len_1);
+  missing.add(head, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+  interval_set<uint64_t> expect_data_region;
+  expect_data_region.insert(offset_1, len_1);
+
+  // Clone object:
+  hobject_t clone = head;
+  clone.snap = 0;
+  std::map<snapid_t, interval_set<uint64_t>> clone_overlap;  // overlap w/ next
+  interval_set<uint64_t> overlap;
+  uint64_t offset_2, len_2;
+  offset_2 = 2;
+  len_2 = 2;
+  overlap.insert(offset_2, len_2);
+  clone_overlap[clone.snap] = overlap;
+
+  // Snapset:
+  // ss.seq = 0;
+  // ss.snaps = snaps; (legacy)
+  ss.clones.push_back(clone.snap);
+  ss.clone_overlap = clone_overlap;
+  // ss.clone_size = clone_size;
+  // ss.clone_snaps = clone_snaps;
+
+  // Expected intervals:
+  interval_set<uint64_t> expect_clone_subset;
+  expect_clone_subset.insert(offset_2, len_2);
+  expect_clone_subset.intersection_of(expect_data_region);
+  expect_data_region.subtract(expect_clone_subset);
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_head_subsets(obj_size,
+                                    ss,
+                                    head,
+                                    missing,
+                                    last_backfill);
+  EXPECT_TRUE(result.clone_subsets[clone] == expect_clone_subset);
+  EXPECT_TRUE(result.data_subset == expect_data_region);
+}
+
+TEST(clone_subsets, overlap)
+{
+  uint64_t clone_size = 10;
+  SnapSet ss;
+  hobject_t clone{object_t{"foo"}, "foo", 1, 42, 0, "nspace"};
+  ss.clone_size[1] = clone_size;
+  ss.clones.push_back(snapid_t(0));
+  ss.clones.push_back(snapid_t(1));
+  ss.clones.push_back(snapid_t(2));
+  pg_missing_t missing;
+  pg_missing_item item;
+  missing.add(clone, std::move(item));
+  hobject_t last_backfill{object_t{"foo1"}, "foo1", CEPH_NOSNAP, 42, 0, "nspace"};
+
+  interval_set<uint64_t> expect_clone_subset1, expect_clone_subset2;
+
+  // Next older clone:
+  hobject_t older_clone = clone;
+  older_clone.snap = 0;
+  {
+    std::map<snapid_t, interval_set<uint64_t>> clone_overlap;  // overlap w/ next
+    interval_set<uint64_t> overlap;
+    uint64_t offset_2, len_2;
+    offset_2 = 4;
+    len_2 = 2;
+    overlap.insert(offset_2, len_2);
+    ss.clone_overlap[older_clone.snap] = overlap;
+
+    // Snapset:
+    // ss.seq = 0;
+    // ss.snaps = snaps; (legacy)
+    // ss.clones.push_back(snapid_t());
+    // ss.clone_overlap = clone_overlap;
+    // ss.clone_size = clone_size;
+    // ss.clone_snaps = clone_snaps;
+
+    // Expected intervals:
+    expect_clone_subset1.insert(offset_2, len_2);
+  }
+
+  // Next newest clone:
+  hobject_t newest_clone = clone;
+  newest_clone.snap = 2;
+  {
+    std::map<snapid_t, interval_set<uint64_t>> clone_overlap;  // overlap w/ next
+    interval_set<uint64_t> overlap;
+    uint64_t offset_2, len_2;
+    offset_2 = 2;
+    len_2 = 2;
+    overlap.insert(offset_2, len_2);
+    ss.clone_overlap[newest_clone.snap - 1] = overlap;
+
+    // Snapset:
+    // ss.seq = 0;
+    // ss.snaps = snaps; (legacy)
+    // ss.clones.push_back(snapid_t());
+    // ss.clone_overlap = clone_overlap;
+    // ss.clone_size = clone_size;
+    // ss.clone_snaps = clone_snaps;
+
+    // Expected intervals:
+    expect_clone_subset2.insert(offset_2, len_2);
+  }
+
+// ****
+
+  crimson::osd::subsets_t result =
+    crimson::osd::calc_clone_subsets(ss,
+                                     clone,
+                                     missing,
+                                     last_backfill);
+  EXPECT_TRUE(result.clone_subsets[older_clone] == expect_clone_subset1);
+  EXPECT_TRUE(result.clone_subsets[newest_clone] == expect_clone_subset2);
+}

From 35e14f8ff8ad50fb9b2ac212b325c0d1345db167 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 31 Dec 2023 12:09:38 +0000
Subject: [PATCH 1209/2492] crimson/osd/replicated_recovery_backend:
 prepare_pull to support calc_clone_subsets

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../osd/replicated_recovery_backend.cc        | 51 ++++++++++++++-----
 src/crimson/osd/replicated_recovery_backend.h |  4 ++
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 803d067fd653..266e3ebcf34a 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -111,8 +111,10 @@ ReplicatedRecoveryBackend::maybe_pull_missing_obj(
   const hobject_t& soid,
   eversion_t need)
 {
+  logger().debug("{}: {}, {}", __func__, soid, need);
   pg_missing_tracker_t local_missing = pg.get_local_missing();
   if (!local_missing.is_missing(soid)) {
+    // object is not missing, don't pull
     return seastar::make_ready_future<>();
   }
   return pg.obc_loader.with_obc<RWState::RWREAD>(soid.get_head(),
@@ -371,19 +373,11 @@ void ReplicatedRecoveryBackend::prepare_pull(
   const auto missing_iter = local_missing.get_items().find(soid);
   auto m = pg.get_missing_loc_shards();
   pg_shard_t fromshard = *(m[soid].begin());
+  const auto& last_backfill =
+    pg.get_peering_state().get_peer_info(fromshard).last_backfill;
 
-  //TODO: skipped snap objects case for now
-  pull_op.recovery_info.copy_subset.insert(0, (uint64_t) -1);
-  pull_op.recovery_info.copy_subset.intersection_of(
-    missing_iter->second.clean_regions.get_dirty_regions());
-  if (soid.is_snap()) {
-    assert(head_obc->ssc);
-    pull_op.recovery_info.ss = head_obc->ssc->snapset;
-  }
-  pull_op.recovery_info.size = ((uint64_t) -1);
-  pull_op.recovery_info.object_exist =
-    missing_iter->second.clean_regions.object_is_exist();
-  pull_op.recovery_info.soid = soid;
+  pull_op.recovery_info =
+    set_recovery_info(soid, head_obc->ssc, last_backfill);
   pull_op.soid = soid;
   pull_op.recovery_progress.data_complete = false;
   pull_op.recovery_progress.omap_complete =
@@ -393,10 +387,43 @@ void ReplicatedRecoveryBackend::prepare_pull(
 
   pull_info.from = fromshard;
   pull_info.soid = soid;
+  pull_info.head_ctx = head_obc;
   pull_info.recovery_info = pull_op.recovery_info;
   pull_info.recovery_progress = pull_op.recovery_progress;
 }
 
+ObjectRecoveryInfo ReplicatedRecoveryBackend::set_recovery_info(
+  const hobject_t& soid,
+  const crimson::osd::SnapSetContextRef ssc,
+  const hobject_t& last_backfill)
+{
+  pg_missing_tracker_t local_missing = pg.get_local_missing();
+  const auto missing_iter = local_missing.get_items().find(soid);
+  ObjectRecoveryInfo recovery_info;
+  if (soid.is_snap()) {
+    assert(!local_missing.is_missing(soid.get_head()));
+    assert(ssc);
+    recovery_info.ss = ssc->snapset;
+    auto subsets = crimson::osd::calc_clone_subsets(
+      ssc->snapset, soid, local_missing, last_backfill);
+    crimson::osd::set_subsets(subsets, recovery_info);
+    logger().debug("{}: pulling {}", __func__, recovery_info);
+    ceph_assert(ssc->snapset.clone_size.count(soid.snap));
+    recovery_info.size = ssc->snapset.clone_size[soid.snap];
+  } else {
+    // pulling head or unversioned object.
+    // always pull the whole thing.
+    recovery_info.copy_subset.insert(0, (uint64_t) -1);
+    recovery_info.copy_subset.intersection_of(
+      missing_iter->second.clean_regions.get_dirty_regions());
+    recovery_info.size = ((uint64_t) -1);
+  }
+  recovery_info.object_exist =
+    missing_iter->second.clean_regions.object_is_exist();
+  recovery_info.soid = soid;
+  return recovery_info;
+}
+
 RecoveryBackend::interruptible_future<PushOp>
 ReplicatedRecoveryBackend::build_push_op(
     const ObjectRecoveryInfo& recovery_info,
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index 8fdaf9a10f8c..cfce12c691a3 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -60,6 +60,10 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
     pull_info_t& pull_info,
     const hobject_t& soid,
     eversion_t need);
+  ObjectRecoveryInfo set_recovery_info(
+    const hobject_t& soid,
+    const crimson::osd::SnapSetContextRef ssc,
+    const hobject_t& last_backfill);
   std::vector<pg_shard_t> get_shards_to_push(
     const hobject_t& soid) const;
   interruptible_future<PushOp> build_push_op(

From 09b638c8dbd4745b1753b8ed26f27b0ca7237e68 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 31 Dec 2023 14:38:30 +0000
Subject: [PATCH 1210/2492] rename prep_push to prep_push_to_replica

dismiss head_ctx

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/replicated_recovery_backend.cc | 7 ++-----
 src/crimson/osd/replicated_recovery_backend.h  | 3 +--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 266e3ebcf34a..92435a9c925c 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -61,7 +61,7 @@ ReplicatedRecoveryBackend::maybe_push_shards(
     return interruptor::parallel_for_each(
       shards,
       [this, need, soid, head_obc](auto shard) {
-      return prep_push(head_obc, soid, need, shard
+      return prep_push_to_replica(soid, need, shard
       ).then_interruptible([this, soid, shard](auto push) {
         auto msg = crimson::make_message<MOSDPGPush>();
         msg->from = pg.get_pg_whoami();
@@ -310,8 +310,7 @@ ReplicatedRecoveryBackend::recover_delete(
 }
 
 RecoveryBackend::interruptible_future<PushOp>
-ReplicatedRecoveryBackend::prep_push(
-  const crimson::osd::ObjectContextRef &head_obc,
+ReplicatedRecoveryBackend::prep_push_to_replica(
   const hobject_t& soid,
   eversion_t need,
   pg_shard_t pg_shard)
@@ -342,8 +341,6 @@ ReplicatedRecoveryBackend::prep_push(
   push_info.recovery_info.copy_subset = data_subset;
   push_info.recovery_info.soid = soid;
   push_info.recovery_info.oi = obc->obs.oi;
-  assert(head_obc->ssc);
-  push_info.recovery_info.ss = head_obc->ssc->snapset;
   push_info.recovery_info.version = obc->obs.oi.version;
   push_info.recovery_info.object_exist =
     missing_iter->second.clean_regions.object_is_exist();
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index cfce12c691a3..8d0e5ecabb78 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -49,8 +49,7 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
     Ref<MOSDPGRecoveryDelete> m);
   interruptible_future<> handle_recovery_delete_reply(
     Ref<MOSDPGRecoveryDeleteReply> m);
-  interruptible_future<PushOp> prep_push(
-    const crimson::osd::ObjectContextRef &head_obc,
+  interruptible_future<PushOp> prep_push_to_replica(
     const hobject_t& soid,
     eversion_t need,
     pg_shard_t pg_shard);

From 869fc82a74b73c8e5661db21d48d7b8953f71c85 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 31 Dec 2023 15:00:35 +0000
Subject: [PATCH 1211/2492] crimson/osd/replicated_recovery_backend: seperate
 prep_push_to_replica and prep_push

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../osd/replicated_recovery_backend.cc        | 34 +++++++++++++------
 src/crimson/osd/replicated_recovery_backend.h |  5 +++
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 92435a9c925c..da416427d627 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -319,16 +319,26 @@ ReplicatedRecoveryBackend::prep_push_to_replica(
 
   auto& recovery_waiter = get_recovering(soid);
   auto& obc = recovery_waiter.obc;
-  interval_set<uint64_t> data_subset;
-  if (obc->obs.oi.size) {
-    data_subset.insert(0, obc->obs.oi.size);
-  }
-  const auto& missing = pg.get_shard_missing().find(pg_shard)->second;
-  const auto it = missing.get_items().find(soid);
-  assert(it != missing.get_items().end());
-  data_subset.intersection_of(it->second.clean_regions.get_dirty_regions());
-  logger().debug("prep_push: {} data_subset {} to {}",
-                 soid, data_subset, pg_shard);
+  // TODO: use calc_clone_subsets
+  crimson::osd::subsets_t subsets;
+
+  return prep_push(soid,
+                   need,
+                   pg_shard,
+                   subsets);
+
+}
+
+RecoveryBackend::interruptible_future<PushOp>
+ReplicatedRecoveryBackend::prep_push(
+  const hobject_t& soid,
+  eversion_t need,
+  pg_shard_t pg_shard,
+  const crimson::osd::subsets_t& subsets)
+{
+  logger().debug("{}: {}, {}", __func__, soid, need);
+  auto& recovery_waiter = get_recovering(soid);
+  auto& obc = recovery_waiter.obc;
 
   auto& push_info = recovery_waiter.pushing[pg_shard];
   pg.begin_peer_recover(pg_shard, soid);
@@ -338,7 +348,8 @@ ReplicatedRecoveryBackend::prep_push_to_replica(
 
   push_info.obc = obc;
   push_info.recovery_info.size = obc->obs.oi.size;
-  push_info.recovery_info.copy_subset = data_subset;
+  push_info.recovery_info.copy_subset = subsets.data_subset;
+  push_info.recovery_info.clone_subset = subsets.clone_subsets;
   push_info.recovery_info.soid = soid;
   push_info.recovery_info.oi = obc->obs.oi;
   push_info.recovery_info.version = obc->obs.oi.version;
@@ -356,6 +367,7 @@ ReplicatedRecoveryBackend::prep_push_to_replica(
     push_info.recovery_progress = push_op.after_progress;
     return push_op;
   });
+
 }
 
 void ReplicatedRecoveryBackend::prepare_pull(
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index 8d0e5ecabb78..00e1021b9eb1 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -53,6 +53,11 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
     const hobject_t& soid,
     eversion_t need,
     pg_shard_t pg_shard);
+  interruptible_future<PushOp> prep_push(
+    const hobject_t& soid,
+    eversion_t need,
+    pg_shard_t pg_shard,
+    const crimson::osd::subsets_t& subsets);
   void prepare_pull(
     const crimson::osd::ObjectContextRef &head_obc,
     PullOp& pull_op,

From 0cbe98226fe9279a872598aecbeee8b84827a1ab Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 31 Dec 2023 15:28:12 +0000
Subject: [PATCH 1212/2492] =?UTF-8?q?mson/osd/replicated=5Frecovery=5Fback?=
 =?UTF-8?q?end:=20Introduce=20prep=5Fpush=5Fto=5Frepli=E2=80=A6=20?=
 =?UTF-8?q?=E2=80=A6ca()?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Intelligently push an object to a replica.  make use of existing
clones/heads and dup data ranges where possible.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../osd/replicated_recovery_backend.cc        | 59 +++++++++++++++++--
 src/crimson/osd/replicated_recovery_backend.h |  3 +-
 2 files changed, 56 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index da416427d627..f78994e8a5af 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -319,14 +319,61 @@ ReplicatedRecoveryBackend::prep_push_to_replica(
 
   auto& recovery_waiter = get_recovering(soid);
   auto& obc = recovery_waiter.obc;
-  // TODO: use calc_clone_subsets
+  SnapSet push_info_ss; // only populated if soid is_snap()
   crimson::osd::subsets_t subsets;
-
+  const auto& missing =
+    pg.get_shard_missing().find(pg_shard)->second;
+
+  // are we doing a clone on the replica?
+  if (soid.snap && soid.snap < CEPH_NOSNAP) {
+    hobject_t head = soid;
+    head.snap = CEPH_NOSNAP;
+
+    // try to base push off of clones that succeed/preceed poid
+    // we need the head (and current SnapSet) locally to do that.
+    if (pg.get_local_missing().is_missing(head)) {
+      logger().debug("{} missing head {}, pushing raw clone",
+                     __func__, head);
+      if (obc->obs.oi.size) {
+        subsets.data_subset.insert(0, obc->obs.oi.size);
+      }
+      return prep_push(soid,
+                       need,
+                       pg_shard,
+                       subsets,
+                       push_info_ss);
+    }
+    auto ssc = obc->ssc;
+    ceph_assert(ssc);
+    push_info_ss = ssc->snapset;
+    logger().debug("push_to_replica snapset is {}",
+                   ssc->snapset);
+
+    subsets = crimson::osd::calc_clone_subsets(
+      ssc->snapset, soid,
+      missing,
+      // get_peer_info() asserts `peer_info` existence.
+      pg.get_peering_state().get_peer_info(
+        pg_shard).last_backfill);
+  } else if (soid.snap == CEPH_NOSNAP) {
+    // pushing head or unversioned object.
+    // base this on partially on replica's clones?
+    auto ssc = obc->ssc;
+    ceph_assert(ssc);
+    logger().debug("push_to_replica snapset is {}",
+                   ssc->snapset);
+    subsets = crimson::osd::calc_head_subsets(
+      obc->obs.oi.size,
+      ssc->snapset, soid,
+      missing,
+      pg.get_peering_state().get_peer_info(
+        pg_shard).last_backfill);
+  }
   return prep_push(soid,
                    need,
                    pg_shard,
-                   subsets);
-
+                   subsets,
+                   push_info_ss);
 }
 
 RecoveryBackend::interruptible_future<PushOp>
@@ -334,7 +381,8 @@ ReplicatedRecoveryBackend::prep_push(
   const hobject_t& soid,
   eversion_t need,
   pg_shard_t pg_shard,
-  const crimson::osd::subsets_t& subsets)
+  const crimson::osd::subsets_t& subsets,
+  const SnapSet push_info_ss)
 {
   logger().debug("{}: {}, {}", __func__, soid, need);
   auto& recovery_waiter = get_recovering(soid);
@@ -350,6 +398,7 @@ ReplicatedRecoveryBackend::prep_push(
   push_info.recovery_info.size = obc->obs.oi.size;
   push_info.recovery_info.copy_subset = subsets.data_subset;
   push_info.recovery_info.clone_subset = subsets.clone_subsets;
+  push_info.recovery_info.ss = push_info_ss;
   push_info.recovery_info.soid = soid;
   push_info.recovery_info.oi = obc->obs.oi;
   push_info.recovery_info.version = obc->obs.oi.version;
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index 00e1021b9eb1..8faed2fbf3f2 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -57,7 +57,8 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
     const hobject_t& soid,
     eversion_t need,
     pg_shard_t pg_shard,
-    const crimson::osd::subsets_t& subsets);
+    const crimson::osd::subsets_t& subsets,
+    const SnapSet push_info_ss);
   void prepare_pull(
     const crimson::osd::ObjectContextRef &head_obc,
     PullOp& pull_op,

From 13674901150b6554ec33fa057e07553cc2f4e586 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 4 Jan 2023 12:45:51 +0000
Subject: [PATCH 1213/2492] crimson/osd/replicated_recovery_backend:
 handle_pull_response() - recalc_subsets()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../osd/replicated_recovery_backend.cc        | 27 +++++++++++++++++--
 src/crimson/osd/replicated_recovery_backend.h |  3 +++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index f78994e8a5af..1ff0623abdc0 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -799,11 +799,23 @@ ReplicatedRecoveryBackend::_handle_pull_response(
   if (pull_info.recovery_progress.first) {
     prepare_waiter = pg.obc_loader.with_obc<RWState::RWNONE>(
       pull_info.recovery_info.soid,
-      [&pull_info, &recovery_waiter, &push_op](auto, auto obc) {
+      [this, &pull_info, &recovery_waiter, &push_op](auto, auto obc) {
         pull_info.obc = obc;
         recovery_waiter.obc = obc;
-        obc->obs.oi.decode_no_oid(push_op.attrset.at(OI_ATTR), push_op.soid);
+        obc->obs.oi.decode_no_oid(push_op.attrset.at(OI_ATTR),
+                                  push_op.soid);
+        if (push_op.attrset.contains(SS_ATTR)) {
+          bufferlist ssbl = push_op.attrset.at(SS_ATTR);
+          SnapSet ss(ssbl);
+          assert(!pull_info.obc->ssc->exists ||
+                 ss.seq == pull_info.obc->ssc->snapset.seq);
+        }
         pull_info.recovery_info.oi = obc->obs.oi;
+        if (pull_info.recovery_info.soid.snap &&
+            pull_info.recovery_info.soid.snap < CEPH_NOSNAP) {
+            recalc_subsets(pull_info.recovery_info,
+                           pull_info.obc->ssc);
+        }
         return crimson::osd::PG::load_obc_ertr::now();
       }).handle_error_interruptible(crimson::ct_error::assert_all{});
   };
@@ -857,6 +869,17 @@ ReplicatedRecoveryBackend::_handle_pull_response(
   });
 }
 
+void ReplicatedRecoveryBackend::recalc_subsets(
+    ObjectRecoveryInfo& recovery_info,
+    crimson::osd::SnapSetContextRef ssc)
+{
+  assert(ssc);
+  auto subsets = crimson::osd::calc_clone_subsets(
+    ssc->snapset, recovery_info.soid, pg.get_local_missing(),
+    pg.get_info().last_backfill);
+  crimson::osd::set_subsets(subsets, recovery_info);
+}
+
 RecoveryBackend::interruptible_future<>
 ReplicatedRecoveryBackend::handle_pull_response(
   Ref<MOSDPGPush> m)
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index 8faed2fbf3f2..cd1d9e067289 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -82,6 +82,9 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
     PushOp& push_op,
     PullOp* response,
     ceph::os::Transaction* t);
+  void recalc_subsets(
+    ObjectRecoveryInfo& recovery_info,
+    crimson::osd::SnapSetContextRef ssc);
   std::pair<interval_set<uint64_t>, ceph::bufferlist> trim_pushed_data(
     const interval_set<uint64_t> &copy_subset,
     const interval_set<uint64_t> &intervals_received,

From 964e5a7121be13d207f061a282c903295c482e09 Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Wed, 6 Dec 2023 22:28:29 +0530
Subject: [PATCH 1214/2492] src/crimson: Add support for the OSD to receive
 config changes

1. Adds MMgrReport message in manager client
2. MonClient subscribes to config

Fixes: https://tracker.ceph.com/issues/59241
Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 src/crimson/mgr/client.cc    | 32 +++++++++++++++++++++++++++++++-
 src/crimson/mgr/client.h     |  3 +++
 src/crimson/mon/MonClient.cc |  1 +
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/crimson/mgr/client.cc b/src/crimson/mgr/client.cc
index 169915c9eb3b..d2a1ea5136a0 100644
--- a/src/crimson/mgr/client.cc
+++ b/src/crimson/mgr/client.cc
@@ -11,6 +11,7 @@
 #include "messages/MMgrConfigure.h"
 #include "messages/MMgrMap.h"
 #include "messages/MMgrOpen.h"
+#include "messages/MMgrReport.h"
 
 namespace {
   seastar::logger& logger()
@@ -156,9 +157,10 @@ seastar::future<> Client::handle_mgr_conf(crimson::net::ConnectionRef,
 
 void Client::report()
 {
+  _send_report();
   gate.dispatch_in_background(__func__, *this, [this] {
     if (!conn) {
-      logger().warn("report: no conn available; raport skipped");
+      logger().warn("report: no conn available; report skipped");
       return seastar::now();
     }
     return with_stats.get_stats(
@@ -168,6 +170,34 @@ void Client::report()
   });
 }
 
+void Client::_send_report()
+{
+  // TODO: implement daemon_health_metrics support
+  // https://tracker.ceph.com/issues/63766
+  gate.dispatch_in_background(__func__, *this, [this] {
+    if (!conn) {
+      logger().warn("cannot send report; no conn available");
+      return seastar::now();
+    }
+    auto report = make_message<MMgrReport>();
+    // Adding empty information since we don't support perfcounters yet
+    report->undeclare_types.emplace_back();
+    ENCODE_START(1, 1, report->packed);
+    report->declare_types.emplace_back();
+    ENCODE_FINISH(report->packed);
+
+    if (daemon_name.size()) {
+      report->daemon_name = daemon_name;
+    } else {
+      report->daemon_name = local_conf()->name.get_id();
+    }
+    report->service_name = service_name;
+    local_conf().get_config_bl(last_config_bl_version, &report->config_bl,
+	                      &last_config_bl_version);
+    return conn->send(std::move(report));
+  });
+}
+
 void Client::print(std::ostream& out) const
 {
   out << "mgrc ";
diff --git a/src/crimson/mgr/client.h b/src/crimson/mgr/client.h
index b88c60c5e4a4..7f4e62fd7d2d 100644
--- a/src/crimson/mgr/client.h
+++ b/src/crimson/mgr/client.h
@@ -57,6 +57,9 @@ class Client : public crimson::net::Dispatcher {
   seastar::timer<seastar::lowres_clock> report_timer;
   crimson::common::Gated gate;
   uint64_t last_config_bl_version = 0;
+  std::string service_name, daemon_name;
+
+  void _send_report();
 };
 
 inline std::ostream& operator<<(std::ostream& out, const Client& client) {
diff --git a/src/crimson/mon/MonClient.cc b/src/crimson/mon/MonClient.cc
index 7be09915a946..761c6b1a5c85 100644
--- a/src/crimson/mon/MonClient.cc
+++ b/src/crimson/mon/MonClient.cc
@@ -432,6 +432,7 @@ Client::~Client() = default;
 seastar::future<> Client::start() {
   entity_name = crimson::common::local_conf()->name;
   auth_registry.refresh_config();
+  sub.want("config", 0, 0);
   return load_keyring().then([this] {
     return monmap.build_initial(crimson::common::local_conf(), false);
   }).then([this] {

From b5df21f55606d58077333df42bc3af98fa1a3d3c Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 1 Jan 2023 11:41:34 +0000
Subject: [PATCH 1215/2492] crimson/osd: Keep track of modified_ranges

* `modifies_ranges` interval_set is added to osd_op_params_t
* keep track of modified_ranges while executing relevant ops
* Add `osd_op_params` parameter to `PGBackend::remove()`.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/ops_executer.cc               |  23 +++-
 src/crimson/osd/osd_operations/osdop_params.h |   3 +-
 src/crimson/osd/pg_backend.cc                 | 105 ++++++++++++++----
 src/crimson/osd/pg_backend.h                  |   5 +-
 4 files changed, 108 insertions(+), 28 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index d57b668c5c70..034fdde71693 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -674,7 +674,16 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
       whiteout = true;
     }
     return do_write_op([this, whiteout](auto& backend, auto& os, auto& txn) {
-      return backend.remove(os, txn, delta_stats, whiteout);
+      int num_bytes = 0;
+      // Calculate num_bytes to be removed
+      if (obc->obs.oi.soid.is_snap()) {
+        ceph_assert(obc->ssc->snapset.clone_overlap.count(obc->obs.oi.soid.snap));
+        num_bytes = obc->ssc->snapset.get_clone_bytes(obc->obs.oi.soid.snap);
+      } else {
+        num_bytes = obc->obs.oi.size;
+      }
+      return backend.remove(os, txn, *osd_op_params,
+                            delta_stats, whiteout, num_bytes);
     });
   }
   case CEPH_OSD_OP_CALL:
@@ -961,7 +970,17 @@ std::unique_ptr<OpsExecuter::CloningContext> OpsExecuter::execute_clone(
   osd_op_params->at_version.version++;
   encode(cloned_snaps, cloning_ctx->log_entry.snaps);
 
-  // TODO: update most recent clone_overlap and usage stats
+  // update most recent clone_overlap and usage stats
+  assert(cloning_ctx->new_snapset.clones.size() > 0);
+  // In classic, we check for evicted clones before
+  // adjusting the clone_overlap.
+  // This check is redundant here since `clone_obc`
+  // was just created (See prepare_clone()).
+  interval_set<uint64_t> &newest_overlap =
+    cloning_ctx->new_snapset.clone_overlap.rbegin()->second;
+  osd_op_params->modified_ranges.intersection_of(newest_overlap);
+  delta_stats.num_bytes += osd_op_params->modified_ranges.size();
+  newest_overlap.subtract(osd_op_params->modified_ranges);
   return cloning_ctx;
 }
 
diff --git a/src/crimson/osd/osd_operations/osdop_params.h b/src/crimson/osd/osd_operations/osdop_params.h
index c7b81e765d9d..0f842f185f41 100644
--- a/src/crimson/osd/osd_operations/osdop_params.h
+++ b/src/crimson/osd/osd_operations/osdop_params.h
@@ -17,6 +17,7 @@ struct osd_op_params_t {
   version_t user_at_version = 0;
   bool user_modify = false;
   ObjectCleanRegions clean_regions;
-
+  interval_set<uint64_t> modified_ranges;
+  //TODO: Move delta_stats to osd_op_params_t
   osd_op_params_t() = default;
 };
diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index a15b6f4bef88..8c68301530b4 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -506,7 +506,9 @@ PGBackend::write_iertr::future<> PGBackend::_writefull(
       coll->get_cid(), ghobject_t{os.oi.soid}, 0, bl.length(),
       bl, flags);
     update_size_and_usage(
-      delta_stats, os.oi, 0,
+      delta_stats,
+      osd_op_params.modified_ranges,
+      os.oi, 0,
       bl.length(), true);
     osd_op_params.clean_regions.mark_data_region_dirty(
       0,
@@ -543,7 +545,9 @@ PGBackend::write_iertr::future<> PGBackend::_truncate(
       coll->get_cid(),
       ghobject_t{os.oi.soid}, offset);
     if (os.oi.size > offset) {
-      // TODO: modified_ranges.union_of(trim);
+      interval_set<uint64_t> trim;
+      trim.insert(offset, os.oi.size - offset);
+      osd_op_params.modified_ranges.union_of(trim);
       osd_op_params.clean_regions.mark_data_region_dirty(
         offset,
 	os.oi.size - offset);
@@ -581,9 +585,19 @@ bool PGBackend::maybe_create_new_object(
 }
 
 void PGBackend::update_size_and_usage(object_stat_sum_t& delta_stats,
+  interval_set<uint64_t>& modified,
   object_info_t& oi, uint64_t offset,
   uint64_t length, bool write_full)
 {
+  interval_set<uint64_t> ch;
+  if (write_full) {
+    if (oi.size) {
+      ch.insert(0, oi.size);
+    } else if (length) {
+      ch.insert(offset, length);
+    }
+    modified.union_of(ch);
+  }
   if (write_full ||
       (offset + length > oi.size && length)) {
     uint64_t new_size = offset + length;
@@ -681,12 +695,14 @@ PGBackend::write_iertr::future<> PGBackend::write(
                    ghobject_t{os.oi.soid}, op.extent.truncate_size);
       if (op.extent.truncate_size != os.oi.size) {
         os.oi.size = length;
-        if (op.extent.truncate_size > os.oi.size) {
-          osd_op_params.clean_regions.mark_data_region_dirty(os.oi.size,
-              op.extent.truncate_size - os.oi.size);
-        } else {
-          osd_op_params.clean_regions.mark_data_region_dirty(op.extent.truncate_size,
-              os.oi.size - op.extent.truncate_size);
+        if (op.extent.truncate_size < os.oi.size) {
+          interval_set<uint64_t> trim;
+          trim.insert(op.extent.truncate_size,
+            os.oi.size - op.extent.truncate_size);
+          osd_op_params.modified_ranges.union_of(trim);
+          osd_op_params.clean_regions.mark_data_region_dirty(
+            op.extent.truncate_size, os.oi.size - op.extent.truncate_size);
+          os.oi.clear_data_digest();
         }
       }
       truncate_update_size_and_usage(delta_stats, os.oi, op.extent.truncate_size);
@@ -705,10 +721,12 @@ PGBackend::write_iertr::future<> PGBackend::write(
   } else {
     txn.write(coll->get_cid(), ghobject_t{os.oi.soid},
 	      offset, length, std::move(buf), op.flags);
-    update_size_and_usage(delta_stats, os.oi, offset, length);
+    update_size_and_usage(delta_stats, osd_op_params.modified_ranges,
+                          os.oi, offset, length);
   }
   osd_op_params.clean_regions.mark_data_region_dirty(op.extent.offset,
 						     op.extent.length);
+  logger().debug("{} clean_regions modified", __func__);
 
   return seastar::now();
 }
@@ -738,7 +756,8 @@ PGBackend::interruptible_future<> PGBackend::write_same(
   txn.write(coll->get_cid(), ghobject_t{os.oi.soid},
             op.writesame.offset, len,
             std::move(repeated_indata), op.flags);
-  update_size_and_usage(delta_stats, os.oi, op.writesame.offset, len);
+  update_size_and_usage(delta_stats, osd_op_params.modified_ranges,
+                        os.oi, op.writesame.offset, len);
   osd_op_params.clean_regions.mark_data_region_dirty(op.writesame.offset, len);
   return seastar::now();
 }
@@ -788,7 +807,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
   target_coid.snap = snapid;
   return obc_loader.with_clone_obc_only<RWState::RWWRITE>(
     head, target_coid,
-    [this, &os, &txn, &delta_stats, &osd_op_params]
+    [this, &os, &txn, &delta_stats, &osd_op_params, &snapid]
     (auto, auto resolved_obc) {
     if (resolved_obc->obs.oi.soid.is_head()) {
       // no-op: The resolved oid returned the head object
@@ -824,9 +843,24 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
     osd_op_params.clean_regions.mark_data_region_dirty(0,
       std::max(os.oi.size, resolved_obc->obs.oi.size));
     osd_op_params.clean_regions.mark_omap_dirty();
-    // TODO: 3) Calculate clone_overlaps by following overlaps
-    //          forward from rollback snapshot
-    //          https://tracker.ceph.com/issues/58263
+
+    // 3) Calculate clone_overlaps by following overlaps
+    const auto& clone_overlap =
+      resolved_obc->ssc->snapset.clone_overlap;
+    auto iter = clone_overlap.lower_bound(snapid);
+    ceph_assert(iter != clone_overlap.end());
+    interval_set<uint64_t> overlaps = iter->second;
+    for (const auto&i: clone_overlap) {
+      overlaps.intersection_of(i.second);
+    }
+
+    if (os.oi.size > 0) {
+      interval_set<uint64_t> modified;
+      modified.insert(0, os.oi.size);
+      overlaps.intersection_of(modified);
+      modified.subtract(overlaps);
+      osd_op_params.modified_ranges.union_of(modified);
+    }
     return rollback_iertr::now();
   }).safe_then_interruptible([] {
     logger().debug("PGBackend::rollback succefully");
@@ -835,12 +869,13 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
     // if there's no snapshot, we delete the object;
     // otherwise, do nothing.
     crimson::ct_error::enoent::handle(
-    [this, &os, &snapid, &txn, &delta_stats, &snapc, &ss] {
+    [this, &os, &snapid, &txn, &delta_stats, &snapc, &ss, &osd_op_params] {
       logger().debug("PGBackend::rollback: deleting head on {}"
                      " with snap_id of {}"
                      " because got ENOENT|whiteout on obc lookup",
                      os.oi.soid, snapid);
-      return remove(os, txn, delta_stats, should_whiteout(ss, snapc));
+      return remove(os, txn, osd_op_params, delta_stats,
+                    should_whiteout(ss, snapc), os.oi.size);
     }),
     rollback_ertr::pass_further{},
     crimson::ct_error::assert_all{"unexpected error in rollback"}
@@ -863,8 +898,9 @@ PGBackend::append_ierrorator::future<> PGBackend::append(
     txn.write(coll->get_cid(), ghobject_t{os.oi.soid},
               os.oi.size /* offset */, op.extent.length,
               std::move(osd_op.indata), op.flags);
-    update_size_and_usage(delta_stats, os.oi, os.oi.size,
-      op.extent.length);
+    update_size_and_usage(delta_stats,
+                          osd_op_params.modified_ranges,
+                          os.oi, os.oi.size, op.extent.length);
     osd_op_params.clean_regions.mark_data_region_dirty(os.oi.size,
                                                        op.extent.length);
   }
@@ -921,7 +957,9 @@ PGBackend::write_iertr::future<> PGBackend::zero(
            ghobject_t{os.oi.soid},
            op.extent.offset,
            op.extent.length);
-  // TODO: modified_ranges.union_of(zeroed);
+  interval_set<uint64_t> ch;
+  ch.insert(op.extent.offset, op.extent.length);
+  osd_op_params.modified_ranges.union_of(ch);
   osd_op_params.clean_regions.mark_data_region_dirty(op.extent.offset,
 						     op.extent.length);
   delta_stats.num_wr++;
@@ -975,7 +1013,10 @@ PGBackend::remove(ObjectState& os, ceph::os::Transaction& txn)
 
 PGBackend::remove_iertr::future<>
 PGBackend::remove(ObjectState& os, ceph::os::Transaction& txn,
-  object_stat_sum_t& delta_stats, bool whiteout)
+  osd_op_params_t& osd_op_params,
+  object_stat_sum_t& delta_stats,
+  bool whiteout,
+  int num_bytes)
 {
   if (!os.exists) {
     return crimson::ct_error::enoent::make();
@@ -991,17 +1032,28 @@ PGBackend::remove(ObjectState& os, ceph::os::Transaction& txn,
   }
   txn.remove(coll->get_cid(),
 	     ghobject_t{os.oi.soid, ghobject_t::NO_GEN, shard});
-  delta_stats.num_bytes -= os.oi.size;
 
   if (os.oi.is_omap()) {
     os.oi.clear_flag(object_info_t::FLAG_OMAP);
     delta_stats.num_objects_omap--;
   }
 
+  if (os.oi.size > 0) {
+    interval_set<uint64_t> ch;
+    ch.insert(0, os.oi.size);
+    osd_op_params.modified_ranges.union_of(ch);
+    osd_op_params.clean_regions.mark_data_region_dirty(0, os.oi.size);
+  }
+
+  osd_op_params.clean_regions.mark_omap_dirty();
+  delta_stats.num_wr++;
+  // num_bytes of the removed clone or head object
+  delta_stats.num_bytes -= num_bytes;
   os.oi.size = 0;
   os.oi.new_object();
 
-  // todo: clone_overlap
+  // todo: update watchers
+
   if (whiteout) {
     logger().debug("{} setting whiteout on {} ",__func__, os.oi.soid);
     os.oi.set_flag(object_info_t::FLAG_WHITEOUT);
@@ -1010,12 +1062,17 @@ PGBackend::remove(ObjectState& os, ceph::os::Transaction& txn,
                ghobject_t{os.oi.soid, ghobject_t::NO_GEN, shard});
     return seastar::now();
   }
-  // todo: update watchers
+
+  // delete the head
+  delta_stats.num_objects--;
+  if (os.oi.soid.is_snap()) {
+    delta_stats.num_object_clones--;
+  }
   if (os.oi.is_whiteout()) {
+    logger().debug("{} deleting whiteout on {}", __func__, os.oi.soid);
     os.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
     delta_stats.num_whiteouts--;
   }
-  delta_stats.num_objects--;
   os.exists = false;
   return seastar::now();
 }
diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h
index aa26b2cd2c1e..981a983075bc 100644
--- a/src/crimson/osd/pg_backend.h
+++ b/src/crimson/osd/pg_backend.h
@@ -149,8 +149,10 @@ class PGBackend
   remove_iertr::future<> remove(
     ObjectState& os,
     ceph::os::Transaction& txn,
+    osd_op_params_t& osd_op_params,
     object_stat_sum_t& delta_stats,
-    bool whiteout);
+    bool whiteout,
+    int num_bytes);
   interruptible_future<> remove(
     ObjectState& os,
     ceph::os::Transaction& txn);
@@ -432,6 +434,7 @@ class PGBackend
     ceph::os::Transaction& txn,
     object_stat_sum_t& delta_stats);
   void update_size_and_usage(object_stat_sum_t& delta_stats,
+    interval_set<uint64_t>& modified,
     object_info_t& oi, uint64_t offset,
     uint64_t length, bool write_full = false);
   void truncate_update_size_and_usage(

From b37839502ea1bd9054d0eda67dc893f593d17372 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 19 Dec 2023 16:31:20 +0000
Subject: [PATCH 1216/2492] rgw/notifications: dont send notifications if op
 has failed

Fixes: https://tracker.ceph.com/issues/63855

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/rgw_op.cc | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 0a1c3b1cf459..5c504769fcd7 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4532,6 +4532,9 @@ void RGWPutObj::execute(optional_yield y)
                                (user_data.empty() ? nullptr : &user_data), nullptr, nullptr,
                                rctx);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
+  if (op_ret < 0) {
+    return;
+  }
 
   // send request to notification manager
   int ret = res->publish_commit(this, s->obj_size, mtime, etag, s->object->get_instance());
@@ -5383,6 +5386,10 @@ void RGWDeleteObj::execute(optional_yield y)
     rgw::op_counters::inc(counters, l_rgw_op_del_obj_b, obj_size);
     rgw::op_counters::tinc(counters, l_rgw_op_del_obj_lat, s->time_elapsed());
 
+    if (op_ret < 0) {
+      return;
+    }
+
     // send request to notification manager
     int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
     if (ret < 0) {
@@ -5835,6 +5842,10 @@ void RGWCopyObj::execute(optional_yield y)
 	   this,
 	   s->yield);
 
+  if (op_ret < 0) {
+    return;
+  }
+
   // send request to notification manager
   int ret = res->publish_commit(this, obj_size, mtime, etag, s->object->get_instance());
   if (ret < 0) {
@@ -7266,13 +7277,13 @@ void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key& o, optional_
   if (op_ret == -ENOENT) {
     op_ret = 0;
   }
-
-
-  // send request to notification manager
-  int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
-  if (ret < 0) {
-    ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
-    // too late to rollback operation, hence op_ret is not set here
+  if (op_ret == 0) {
+    // send request to notification manager
+    int ret = res->publish_commit(this, obj_size, ceph::real_clock::now(), etag, version_id);
+    if (ret < 0) {
+      ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
+      // too late to rollback operation, hence op_ret is not set here
+    }
   }
   
   send_partial_response(o, del_op->result.delete_marker, del_op->result.version_id, op_ret, formatter_flush_cond);

From 9183de730967ab837a19130756e777a31e88b5ca Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 8 Nov 2023 14:02:33 -0500
Subject: [PATCH 1217/2492] rgwlc:  additional lc notify cleanups

Follow the same formula to build up obj_state and version_id
in all call sites.

Resolves: rhbz#2163667

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 42 ++++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index f117aaa81f84..fec80339d0ff 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -525,9 +525,6 @@ static int remove_expired_obj(
   auto& version_id = obj_key.instance;
   std::unique_ptr<rgw::sal::Notification> notify;
 
-  std::unique_ptr<rgw::sal::User> user;
-  user = driver->get_user(bucket_info.owner);
-
   /* per discussion w/Daniel, Casey,and Eric, we *do need*
    * a new sal object handle, based on the following decision
    * to clear obj_key.instance--which happens in the case
@@ -555,7 +552,7 @@ static int remove_expired_obj(
   del_op->params.unmod_since = meta.mtime;
 
   // notification supported only for RADOS driver for now
-  notify = driver->get_notification(dpp, oc.obj.get(), nullptr, event_type,
+  notify = driver->get_notification(dpp, obj.get(), nullptr, event_type,
 				   oc.bucket, lc_id,
 				   const_cast<std::string&>(oc.bucket->get_tenant()),
 				   lc_req_id, null_yield);
@@ -834,18 +831,25 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
     int ret{0};
     auto wt = boost::get<std::tuple<lc_op, rgw_bucket_dir_entry>>(wi);
     auto& [rule, obj] = wt;
+
     if (obj_has_expired(this, cct, obj.meta.mtime, rule.mp_expiration)) {
       rgw_obj_key key(obj.key);
-      std::unique_ptr<rgw::sal::MultipartUpload> mpu = target->get_multipart_upload(key.name);
-      std::unique_ptr<rgw::sal::Object> sal_obj
-	= target->get_object(key);
+      auto mpu = target->get_multipart_upload(key.name);
+      auto sal_obj = target->get_object(key);
+
+      RGWObjState* obj_state{nullptr};
+      ret = sal_obj->get_obj_state(this, &obj_state, null_yield, true);
+      if (ret < 0) {
+	return ret;
+      }
+
       std::unique_ptr<rgw::sal::Notification> notify
 	= driver->get_notification(
 	  this, sal_obj.get(), nullptr, event_type,
 	  target, lc_id,
 	  const_cast<std::string&>(target->get_tenant()),
 	  lc_req_id, null_yield);
-			auto& version_id = obj.key.instance;
+      auto& version_id = obj.key.instance;
 
       ret = notify->publish_reserve(this, nullptr);
       if (ret < 0) {
@@ -860,12 +864,13 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
       ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
         int publish_ret = notify->publish_commit(
-            this, sal_obj->get_obj_size(), ceph::real_clock::now(),
-            sal_obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
-						version_id);
+            this, obj_state->size,
+	    ceph::real_clock::now(),
+            obj_state->attrset[RGW_ATTR_ETAG].to_str(),
+	    version_id);
         if (publish_ret < 0) {
           ldpp_dout(wk->get_lc(), 5)
-              << "WARNING: notify publish_commit failed, with error: " << publish_ret
+              << "WARNING: notify publish_commit failed, with error: " << ret
               << dendl;
         }
         if (perfcounter) {
@@ -1332,10 +1337,14 @@ class LCOpAction_Transition : public LCOpAction {
 
     /* notifications */
     auto& bucket = oc.bucket;
-    std::string version_id;
-
     auto& obj = oc.obj;
 
+    RGWObjState* obj_state{nullptr};
+    ret = obj->get_obj_state(oc.dpp, &obj_state, null_yield, true);
+    if (ret < 0) {
+      return ret;
+    }
+
     const auto event_type = (bucket->versioned() &&
 			     oc.o.is_current() && !oc.o.is_delete_marker()) ?
       rgw::notify::ObjectTransitionCurrent :
@@ -1347,6 +1356,7 @@ class LCOpAction_Transition : public LCOpAction {
 	bucket, lc_id,
 	const_cast<std::string&>(oc.bucket->get_tenant()),
 	lc_req_id, null_yield);
+    auto& version_id = oc.o.key.instance;
 
     ret = notify->publish_reserve(oc.dpp, nullptr);
     if (ret < 0) {
@@ -1365,9 +1375,9 @@ class LCOpAction_Transition : public LCOpAction {
       return ret;
     } else {
       // send request to notification manager
-      int publish_ret = notify->publish_commit(oc.dpp, obj->get_obj_size(),
+      int publish_ret =  notify->publish_commit(oc.dpp, obj_state->size,
 				    ceph::real_clock::now(),
-				    obj->get_attrs()[RGW_ATTR_ETAG].to_str(),
+				    obj_state->attrset[RGW_ATTR_ETAG].to_str(),
 				    version_id);
       if (publish_ret < 0) {
 	ldpp_dout(oc.dpp, 5) <<

From 8a935bb115d53f94d2156bd0631a8ddc2e205d9e Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 15 Nov 2023 13:39:18 -0500
Subject: [PATCH 1218/2492] rgwlc:  even current object versions have a unique
 instance

Fixes: https://tracker.ceph.com/issues/63546

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index fec80339d0ff..4364c2e683c8 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -522,7 +522,7 @@ static int remove_expired_obj(
   auto obj_key = o.key;
   auto& meta = o.meta;
   int ret;
-  auto& version_id = obj_key.instance;
+  auto version_id = obj_key.instance; // deep copy, so not cleared below
   std::unique_ptr<rgw::sal::Notification> notify;
 
   /* per discussion w/Daniel, Casey,and Eric, we *do need*
@@ -849,7 +849,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 	  target, lc_id,
 	  const_cast<std::string&>(target->get_tenant()),
 	  lc_req_id, null_yield);
-      auto& version_id = obj.key.instance;
+      auto version_id = obj.key.instance;
 
       ret = notify->publish_reserve(this, nullptr);
       if (ret < 0) {
@@ -1356,7 +1356,7 @@ class LCOpAction_Transition : public LCOpAction {
 	bucket, lc_id,
 	const_cast<std::string&>(oc.bucket->get_tenant()),
 	lc_req_id, null_yield);
-    auto& version_id = oc.o.key.instance;
+    auto version_id = oc.o.key.instance;
 
     ret = notify->publish_reserve(oc.dpp, nullptr);
     if (ret < 0) {

From 1b29691318b93dd3e2a20cae0bc7b878f3a837c3 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Tue, 12 Dec 2023 18:55:47 +0900
Subject: [PATCH 1219/2492] crimson/os/seastore/cache: reset version to 1 if
 inplace write occurs before in prepare_record

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 8645a3071062..fe42efa74ad7 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1096,6 +1096,20 @@ record_t Cache::prepare_record(
     if (!i->is_exist_mutation_pending()) {
       DEBUGT("commit replace extent ... -- {}, prior={}",
 	     t, *i, *i->prior_instance);
+      // If inplace rewrite occurs during mutation, prev->version will
+      // be zero. Although this results in the version mismatch here, we can
+      // correct this by changing version to 1. This is because the inplace rewrite
+      // does not introduce any actual modification that could negatively
+      // impact system reliability
+      if (i->prior_instance->version == 0 && i->version > 1) {
+	assert(can_inplace_rewrite(i->get_type()));
+	assert(can_inplace_rewrite(i->prior_instance->get_type()));
+	assert(i->prior_instance->dirty_from_or_retired_at == JOURNAL_SEQ_MIN);
+	assert(i->prior_instance->state == CachedExtent::extent_state_t::CLEAN);
+	assert(i->prior_instance->get_paddr().get_addr_type() ==
+	  paddr_types_t::RANDOM_BLOCK);
+	i->version = 1;
+      }
       // extent with EXIST_MUTATION_PENDING doesn't have
       // prior_instance field so skip these extents.
       // the existing extents should be added into Cache

From 78dc32bada360f2db91b30cb50624f3cde84b192 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Thu, 16 Nov 2023 13:52:47 +0000
Subject: [PATCH 1220/2492] test/crimson/seastore: add a random overwrite test
 to test_objec_date_handler

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/async_cleaner.cc      |  2 +-
 .../seastore/test_object_data_handler.cc      | 47 ++++++++++++++++++-
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc
index d7e398f5f732..fd043f4ea35e 100644
--- a/src/crimson/os/seastore/async_cleaner.cc
+++ b/src/crimson/os/seastore/async_cleaner.cc
@@ -388,7 +388,7 @@ JournalTrimmerImpl::config_t::get_test(
     max_journal_bytes = 4 * roll_size;
   } else {
     assert(type == journal_type_t::RANDOM_BLOCK);
-    target_dirty_bytes = roll_size / 4;
+    target_dirty_bytes = roll_size / 36;
     target_alloc_bytes = roll_size / 4;
     max_journal_bytes = roll_size / 2;
   }
diff --git a/src/test/crimson/seastore/test_object_data_handler.cc b/src/test/crimson/seastore/test_object_data_handler.cc
index ae44cb94a62d..d054fc18f480 100644
--- a/src/test/crimson/seastore/test_object_data_handler.cc
+++ b/src/test/crimson/seastore/test_object_data_handler.cc
@@ -110,12 +110,17 @@ struct object_data_handler_test_t:
 
   bufferptr known_contents;
   extent_len_t size = 0;
+  std::random_device rd;
+  std::mt19937 gen;
 
-  object_data_handler_test_t() {}
+  object_data_handler_test_t() : gen(rd()) {}
 
   void write(Transaction &t, objaddr_t offset, extent_len_t len, char fill) {
     ceph_assert(offset + len <= known_contents.length());
     size = std::max<extent_len_t>(size, offset + len);
+    Option::size_t olen = crimson::common::local_conf().get_val<Option::size_t>(
+      "seastore_data_delta_based_overwrite");
+    ceph_assert(olen == 0 || len <= olen);
     memset(
       known_contents.c_str() + offset,
       fill,
@@ -233,12 +238,18 @@ struct object_data_handler_test_t:
   }
 
   void set_overwrite_threshold() {
-    crimson::common::local_conf().set_val("seastore_data_delta_based_overwrite", "131072").get();
+    crimson::common::local_conf().set_val("seastore_data_delta_based_overwrite",
+      "16777216").get();
   }
   void unset_overwrite_threshold() {
     crimson::common::local_conf().set_val("seastore_data_delta_based_overwrite", "0").get();
   }
 
+  laddr_t get_random_laddr(size_t block_size, laddr_t limit) {
+    return block_size *
+      std::uniform_int_distribution<>(0, (limit / block_size) - 1)(gen);
+  }
+
   void test_multi_write() {
     write((1<<20) - (4<<10), 4<<10, 'a');
     write(1<<20, 4<<10, 'b');
@@ -671,6 +682,38 @@ TEST_P(object_data_handler_test_t, multiple_overwrite) {
   });
 }
 
+TEST_P(object_data_handler_test_t, random_overwrite) {
+  constexpr size_t TOTAL = 4<<20;
+  constexpr size_t BSIZE = 4<<10;
+  constexpr size_t BLOCKS = TOTAL / BSIZE;
+  run_async([this] {
+    set_overwrite_threshold();
+    size_t wsize = std::uniform_int_distribution<>(10, BSIZE - 1)(gen);
+    uint8_t div[3] = {1, 2, 4};
+    uint8_t block_num = div[std::uniform_int_distribution<>(0, 2)(gen)];
+    for (unsigned i = 0; i < BLOCKS / block_num; ++i) {
+      auto t = create_mutate_transaction();
+      write(i * (BSIZE * block_num), BSIZE * block_num, 'a');
+    }
+
+    for (unsigned i = 0; i < 4; ++i) {
+      for (unsigned j = 0; j < 100; ++j) {
+	auto t = create_mutate_transaction();
+	for (unsigned k = 0; k < 2; ++k) {
+	  write(*t, get_random_laddr(BSIZE, TOTAL), wsize,
+	    (char)((j*k) % std::numeric_limits<char>::max()));
+	}
+	submit_transaction(std::move(t));
+      }
+      restart();
+      epm->check_usage();
+      logger().info("random_writes: {} done replaying/checking", i);
+    }
+    read(0, 4<<20);
+    unset_overwrite_threshold();
+  });
+}
+
 INSTANTIATE_TEST_SUITE_P(
   object_data_handler_test,
   object_data_handler_test_t,

From 5307f252d6d31d49aa2a62ca0b61cb92ea46e5a2 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Sun, 10 Dec 2023 12:34:03 +0900
Subject: [PATCH 1221/2492] crimson/os/seastore: write only modified region
 when rewriting the extent in an inplace manner

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cached_extent.h       |  8 +++++++
 .../os/seastore/extent_placement_manager.cc   | 24 +++++++++++++++----
 src/crimson/os/seastore/object_data_handler.h | 14 +++++++++++
 src/test/crimson/seastore/test_block.h        | 14 +++++++++++
 4 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 18c102ed3e17..a72011db2752 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1244,6 +1244,14 @@ class LogicalCachedExtent : public ChildableCachedExtent {
 
   virtual void clear_delta() {}
 
+  struct modified_region_t {
+    extent_len_t offset;
+    extent_len_t len;
+  };
+  virtual std::optional<modified_region_t> get_modified_region() {
+    return std::nullopt;
+  }
+
   virtual ~LogicalCachedExtent();
 protected:
 
diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc
index f53b9f5be916..45f51a401e08 100644
--- a/src/crimson/os/seastore/extent_placement_manager.cc
+++ b/src/crimson/os/seastore/extent_placement_manager.cc
@@ -790,19 +790,33 @@ RandomBlockOolWriter::do_write(
     stats.num_records += 1;
 
     ex->prepare_write();
-    return rbm->write(paddr,
-      ex->get_bptr()
+    extent_len_t offset = 0;
+    bufferptr bp;
+    if (can_inplace_rewrite(t, ex)) {
+      auto r = ex->get_modified_region();
+      if (r.has_value() && r->len > rbm->get_block_size()) {
+	offset = p2align(r->offset, rbm->get_block_size());
+	extent_len_t len =
+	  p2roundup(r->offset + r->len, rbm->get_block_size()) - offset;
+	bp = ceph::bufferptr(ex->get_bptr(), offset, len);
+      } else {
+	bp = ex->get_bptr();
+      }
+    } else {
+      bp = ex->get_bptr();
+    }
+    return rbm->write(paddr + offset,
+      bp
     ).handle_error(
       alloc_write_iertr::pass_further{},
       crimson::ct_error::assert_all{
 	"Invalid error when writing record"}
-    ).safe_then([&t, &ex, paddr, FNAME]() {
+    ).safe_then([&t, &ex, paddr, this, FNAME]() {
       TRACET("ool extent written at {} -- {}",
 	     t, paddr, *ex);
       if (ex->is_initial_pending()) {
 	t.mark_allocated_extent_ool(ex);
-      } else if (ex->is_dirty()) {
-	assert(t.get_src() == transaction_type_t::TRIM_DIRTY);
+      } else if (can_inplace_rewrite(t, ex)) {
 	t.mark_inplace_rewrite_extent_ool(ex);
       } else {
 	ceph_assert("impossible");
diff --git a/src/crimson/os/seastore/object_data_handler.h b/src/crimson/os/seastore/object_data_handler.h
index 76d69eb73f9b..2b01089214c2 100644
--- a/src/crimson/os/seastore/object_data_handler.h
+++ b/src/crimson/os/seastore/object_data_handler.h
@@ -64,6 +64,20 @@ struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
   void clear_delta() final {
     delta.clear();
   }
+
+  std::optional<modified_region_t> get_modified_region() final {
+    interval_set<extent_len_t> range;
+    for (auto &p : delta) {
+      if (p.len > 0) {
+	range.union_insert(p.offset, p.len);
+      }
+    }
+    if (range.empty()) {
+      return std::nullopt;
+    }
+    return modified_region_t{range.range_start(),
+      range.range_end() - range.range_start()};
+  }
 };
 using ObjectDataBlockRef = TCachedExtentRef<ObjectDataBlock>;
 
diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h
index bfb50670420c..8ddb3880b4f5 100644
--- a/src/test/crimson/seastore/test_block.h
+++ b/src/test/crimson/seastore/test_block.h
@@ -83,6 +83,20 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
   void clear_delta() final {
     delta.clear();
   }
+
+  std::optional<modified_region_t> get_modified_region() final {
+    interval_set<extent_len_t> range;
+    for (auto &p : delta) {
+      if (p.len > 0) {
+	range.union_insert(p.offset, p.len);
+      }
+    }
+    if (range.empty()) {
+      return std::nullopt;
+    }
+    return modified_region_t{range.range_start(),
+      range.range_end() - range.range_start()};
+  }
 };
 using TestBlockRef = TCachedExtentRef<TestBlock>;
 

From 3ca63d79b86cec82cb33ad4fc05866e61949967d Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Sun, 17 Dec 2023 17:51:22 +0900
Subject: [PATCH 1222/2492] crimson/os/seastore: introduce modified_region in
 DATA_BLOCK to keep track of modified region

It has a limitation to keep track of the modified region using the existing
deltas because we can not get the correct region in two cases: 1) a case where replay
is done and 2) duplicate_for_write. This commit introduces modified region
to solve the problem.

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc              |  2 +-
 src/crimson/os/seastore/cached_extent.h       |  4 +--
 .../os/seastore/extent_placement_manager.cc   | 13 ++++-----
 .../os/seastore/object_data_handler.cc        |  1 +
 src/crimson/os/seastore/object_data_handler.h | 29 ++++++++++---------
 src/test/crimson/seastore/test_block.cc       |  1 +
 src/test/crimson/seastore/test_block.h        | 29 ++++++++++---------
 7 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index fe42efa74ad7..24fa9788fe89 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1288,7 +1288,7 @@ record_t Cache::prepare_record(
     i->dirty_from_or_retired_at = JOURNAL_SEQ_MIN;
     i->state = CachedExtent::extent_state_t::CLEAN;
     assert(i->is_logical());
-    i->cast<LogicalCachedExtent>()->clear_delta();
+    i->clear_modified_region();
     touch_extent(*i);
     DEBUGT("inplace rewrite ool block is commmitted -- {}", t, *i);
   }
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index a72011db2752..487b0f3555e1 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1242,8 +1242,6 @@ class LogicalCachedExtent : public ChildableCachedExtent {
 
   void on_replace_prior(Transaction &t) final;
 
-  virtual void clear_delta() {}
-
   struct modified_region_t {
     extent_len_t offset;
     extent_len_t len;
@@ -1252,6 +1250,8 @@ class LogicalCachedExtent : public ChildableCachedExtent {
     return std::nullopt;
   }
 
+  virtual void clear_modified_region() {}
+
   virtual ~LogicalCachedExtent();
 protected:
 
diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc
index 45f51a401e08..0562d975337c 100644
--- a/src/crimson/os/seastore/extent_placement_manager.cc
+++ b/src/crimson/os/seastore/extent_placement_manager.cc
@@ -794,14 +794,11 @@ RandomBlockOolWriter::do_write(
     bufferptr bp;
     if (can_inplace_rewrite(t, ex)) {
       auto r = ex->get_modified_region();
-      if (r.has_value() && r->len > rbm->get_block_size()) {
-	offset = p2align(r->offset, rbm->get_block_size());
-	extent_len_t len =
-	  p2roundup(r->offset + r->len, rbm->get_block_size()) - offset;
-	bp = ceph::bufferptr(ex->get_bptr(), offset, len);
-      } else {
-	bp = ex->get_bptr();
-      }
+      ceph_assert(r.has_value());
+      offset = p2align(r->offset, rbm->get_block_size());
+      extent_len_t len =
+	p2roundup(r->offset + r->len, rbm->get_block_size()) - offset;
+      bp = ceph::bufferptr(ex->get_bptr(), offset, len);
     } else {
       bp = ex->get_bptr();
     }
diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 3333fcfad9d4..5e49a48a1880 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -423,6 +423,7 @@ void ObjectDataBlock::apply_delta(const ceph::bufferlist &bl) {
   for (auto &&d : deltas) {
     auto iter = d.bl.cbegin();
     iter.copy(d.len, get_bptr().c_str() + d.offset);
+    modified_region.union_insert(d.offset, d.len);
   }
 }
 
diff --git a/src/crimson/os/seastore/object_data_handler.h b/src/crimson/os/seastore/object_data_handler.h
index 2b01089214c2..783d0919ce5d 100644
--- a/src/crimson/os/seastore/object_data_handler.h
+++ b/src/crimson/os/seastore/object_data_handler.h
@@ -35,10 +35,12 @@ struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
 
   std::vector<block_delta_t> delta = {};
 
+  interval_set<extent_len_t> modified_region;
+
   explicit ObjectDataBlock(ceph::bufferptr &&ptr)
     : LogicalCachedExtent(std::move(ptr)) {}
   explicit ObjectDataBlock(const ObjectDataBlock &other)
-    : LogicalCachedExtent(other) {}
+    : LogicalCachedExtent(other), modified_region(other.modified_region) {}
   explicit ObjectDataBlock(extent_len_t length)
     : LogicalCachedExtent(length) {}
 
@@ -55,28 +57,27 @@ struct ObjectDataBlock : crimson::os::seastore::LogicalCachedExtent {
     auto iter = bl.cbegin();
     iter.copy(bl.length(), get_bptr().c_str() + offset);
     delta.push_back({offset, bl.length(), bl});
+    modified_region.union_insert(offset, bl.length());
   }
 
   ceph::bufferlist get_delta() final;
 
   void apply_delta(const ceph::bufferlist &bl) final;
 
-  void clear_delta() final {
-    delta.clear();
-  }
-
   std::optional<modified_region_t> get_modified_region() final {
-    interval_set<extent_len_t> range;
-    for (auto &p : delta) {
-      if (p.len > 0) {
-	range.union_insert(p.offset, p.len);
-      }
-    }
-    if (range.empty()) {
+    if (modified_region.empty()) {
       return std::nullopt;
     }
-    return modified_region_t{range.range_start(),
-      range.range_end() - range.range_start()};
+    return modified_region_t{modified_region.range_start(),
+      modified_region.range_end() - modified_region.range_start()};
+  }
+
+  void clear_modified_region() final {
+    modified_region.clear();
+  }
+
+  void logical_on_delta_write() final {
+    delta.clear();
   }
 };
 using ObjectDataBlockRef = TCachedExtentRef<ObjectDataBlock>;
diff --git a/src/test/crimson/seastore/test_block.cc b/src/test/crimson/seastore/test_block.cc
index f7a39b0ef59c..7d673d8c2362 100644
--- a/src/test/crimson/seastore/test_block.cc
+++ b/src/test/crimson/seastore/test_block.cc
@@ -19,6 +19,7 @@ void TestBlock::apply_delta(const ceph::bufferlist &bl) {
   decode(deltas, biter);
   for (auto &&d : deltas) {
     set_contents(d.val, d.offset, d.len);
+    modified_region.union_insert(d.offset, d.len);
   }
 }
 
diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h
index 8ddb3880b4f5..2c70b7dbb876 100644
--- a/src/test/crimson/seastore/test_block.h
+++ b/src/test/crimson/seastore/test_block.h
@@ -49,10 +49,12 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
 
   std::vector<test_block_delta_t> delta = {};
 
+  interval_set<extent_len_t> modified_region;
+
   TestBlock(ceph::bufferptr &&ptr)
     : LogicalCachedExtent(std::move(ptr)) {}
   TestBlock(const TestBlock &other)
-    : LogicalCachedExtent(other) {}
+    : LogicalCachedExtent(other), modified_region(other.modified_region) {}
 
   CachedExtentRef duplicate_for_write(Transaction&) final {
     return CachedExtentRef(new TestBlock(*this));
@@ -68,6 +70,7 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
   void set_contents(char c, uint16_t offset, uint16_t len) {
     ::memset(get_bptr().c_str() + offset, c, len);
     delta.push_back({c, offset, len});
+    modified_region.union_insert(offset, len);
   }
 
   void set_contents(char c) {
@@ -80,22 +83,20 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
 
   void apply_delta(const ceph::bufferlist &bl) final;
 
-  void clear_delta() final {
-    delta.clear();
-  }
-
   std::optional<modified_region_t> get_modified_region() final {
-    interval_set<extent_len_t> range;
-    for (auto &p : delta) {
-      if (p.len > 0) {
-	range.union_insert(p.offset, p.len);
-      }
-    }
-    if (range.empty()) {
+    if (modified_region.empty()) {
       return std::nullopt;
     }
-    return modified_region_t{range.range_start(),
-      range.range_end() - range.range_start()};
+    return modified_region_t{modified_region.range_start(),
+      modified_region.range_end() - modified_region.range_start()};
+  }
+
+  void clear_modified_region() final {
+    modified_region.clear();
+  }
+
+  void logical_on_delta_write() final {
+    delta.clear();
   }
 };
 using TestBlockRef = TCachedExtentRef<TestBlock>;

From 6d42ff4c7e6d3f2e5b97d3d4e16a4e1b1552af29 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Tue, 12 Dec 2023 18:22:13 +0900
Subject: [PATCH 1223/2492] crimson/os/seastore/epm: detect transaction confict
 during rbm write using OOL

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 .../os/seastore/extent_placement_manager.cc      | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc
index 0562d975337c..3acd3dbd63de 100644
--- a/src/crimson/os/seastore/extent_placement_manager.cc
+++ b/src/crimson/os/seastore/extent_placement_manager.cc
@@ -802,13 +802,15 @@ RandomBlockOolWriter::do_write(
     } else {
       bp = ex->get_bptr();
     }
-    return rbm->write(paddr + offset,
-      bp
-    ).handle_error(
-      alloc_write_iertr::pass_further{},
-      crimson::ct_error::assert_all{
-	"Invalid error when writing record"}
-    ).safe_then([&t, &ex, paddr, this, FNAME]() {
+    return trans_intr::make_interruptible(
+      rbm->write(paddr + offset,
+	bp
+      ).handle_error(
+	alloc_write_iertr::pass_further{},
+	crimson::ct_error::assert_all{
+	  "Invalid error when writing record"}
+      )
+    ).si_then([this, &t, &ex, paddr, FNAME] {
       TRACET("ool extent written at {} -- {}",
 	     t, paddr, *ex);
       if (ex->is_initial_pending()) {

From cba051d6e0d964ae3bc7337aee942ee91b9c663e Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Sun, 17 Dec 2023 18:04:10 +0900
Subject: [PATCH 1224/2492] test/crimson/seastore: fix overflow issue and make
 len > 0 when set_contents is called

set_contents causes the overflow at times because
alloc_extent is allowed to use uint32_t.

Specifically, in random_writes case, PADDING_SIZE is 256<<10,
whereas set_contents's len is uint16_t.

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/test/crimson/seastore/test_block.h | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h
index 2c70b7dbb876..3bf119f77260 100644
--- a/src/test/crimson/seastore/test_block.h
+++ b/src/test/crimson/seastore/test_block.h
@@ -24,8 +24,8 @@ struct test_extent_desc_t {
 
 struct test_block_delta_t {
   int8_t val = 0;
-  uint16_t offset = 0;
-  uint16_t len = 0;
+  extent_len_t offset = 0;
+  extent_len_t len = 0;
 
 
   DENC(test_block_delta_t, v, p) {
@@ -67,7 +67,9 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
 
   ceph::bufferlist get_delta() final;
 
-  void set_contents(char c, uint16_t offset, uint16_t len) {
+  void set_contents(char c, extent_len_t offset, extent_len_t len) {
+    assert(offset + len <= get_length());
+    assert(len > 0);
     ::memset(get_bptr().c_str() + offset, c, len);
     delta.push_back({c, offset, len});
     modified_region.union_insert(offset, len);
@@ -121,7 +123,7 @@ struct TestBlockPhysical : crimson::os::seastore::CachedExtent{
     return TYPE;
   }
 
-  void set_contents(char c, uint16_t offset, uint16_t len) {
+  void set_contents(char c, extent_len_t offset, extent_len_t len) {
     ::memset(get_bptr().c_str() + offset, c, len);
     delta.push_back({c, offset, len});
   }
@@ -142,13 +144,13 @@ struct test_block_mutator_t {
     std::numeric_limits<int8_t>::min(),
     std::numeric_limits<int8_t>::max());
 
-  std::uniform_int_distribution<uint16_t>
-  offset_distribution = std::uniform_int_distribution<uint16_t>(
+  std::uniform_int_distribution<extent_len_t>
+  offset_distribution = std::uniform_int_distribution<extent_len_t>(
     0, TestBlock::SIZE - 1);
 
-  std::uniform_int_distribution<uint16_t> length_distribution(uint16_t offset) {
-    return std::uniform_int_distribution<uint16_t>(
-      0, TestBlock::SIZE - offset - 1);
+  std::uniform_int_distribution<extent_len_t> length_distribution(extent_len_t offset) {
+    return std::uniform_int_distribution<extent_len_t>(
+      1, TestBlock::SIZE - offset);
   }
 
 

From ad8b3f29ada57c06af900c9a3b357806e097d4ab Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Wed, 20 Dec 2023 20:35:10 +0900
Subject: [PATCH 1225/2492] crimson/os/seastore: check correct crc for
 inplace-rewritten extents  after replay is done

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc              | 34 +++++++++++--------
 src/crimson/os/seastore/cache.h               |  3 +-
 src/crimson/os/seastore/cached_extent.h       |  4 +++
 src/crimson/os/seastore/journal.h             |  3 +-
 .../journal/circular_bounded_journal.cc       | 26 +++++++++++---
 .../os/seastore/journal/segmented_journal.cc  |  3 +-
 src/test/crimson/seastore/test_cbjournal.cc   |  6 ++--
 .../crimson/seastore/test_seastore_journal.cc |  3 +-
 8 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 24fa9788fe89..7513cd601871 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1732,14 +1732,16 @@ Cache::replay_delta(
               segment_seq_printer_t{delta_paddr_segment_seq},
               delta_paddr_segment_type,
               delta);
-        return replay_delta_ertr::make_ready_future<bool>(false);
+        return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+	  std::make_pair(false, nullptr));
       }
     }
   }
 
   if (delta.type == extent_types_t::JOURNAL_TAIL) {
     // this delta should have been dealt with during segment cleaner mounting
-    return replay_delta_ertr::make_ready_future<bool>(false);
+    return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+      std::make_pair(false, nullptr));
   }
 
   // replay alloc
@@ -1747,7 +1749,8 @@ Cache::replay_delta(
     if (journal_seq < alloc_tail) {
       DEBUG("journal_seq {} < alloc_tail {}, don't replay {}",
 	journal_seq, alloc_tail, delta);
-      return replay_delta_ertr::make_ready_future<bool>(false);
+      return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+	std::make_pair(false, nullptr));
     }
 
     alloc_delta_t alloc_delta;
@@ -1771,14 +1774,16 @@ Cache::replay_delta(
     if (!backref_list.empty()) {
       backref_batch_update(std::move(backref_list), journal_seq);
     }
-    return replay_delta_ertr::make_ready_future<bool>(true);
+    return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+      std::make_pair(true, nullptr));
   }
 
   // replay dirty
   if (journal_seq < dirty_tail) {
     DEBUG("journal_seq {} < dirty_tail {}, don't replay {}",
       journal_seq, dirty_tail, delta);
-    return replay_delta_ertr::make_ready_future<bool>(false);
+    return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+      std::make_pair(false, nullptr));
   }
 
   if (delta.type == extent_types_t::ROOT) {
@@ -1792,7 +1797,8 @@ Cache::replay_delta(
           journal_seq, record_base, delta, *root);
     root->set_modify_time(modify_time);
     add_extent(root);
-    return replay_delta_ertr::make_ready_future<bool>(true);
+    return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+      std::make_pair(true, root));
   } else {
     auto _get_extent_if_cached = [this](paddr_t addr)
       -> get_extent_ertr::future<CachedExtentRef> {
@@ -1832,7 +1838,8 @@ Cache::replay_delta(
 	DEBUG("replay extent is not present, so delta is obsolete at {} {} -- {}",
 	      journal_seq, record_base, delta);
 	assert(delta.pversion > 0);
-	return replay_delta_ertr::make_ready_future<bool>(true);
+	return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+	  std::make_pair(true, nullptr));
       }
 
       DEBUG("replay extent delta at {} {} ... -- {}, prv_extent={}",
@@ -1840,20 +1847,16 @@ Cache::replay_delta(
 
       if (delta.paddr.get_addr_type() == paddr_types_t::SEGMENT ||
 	  !can_inplace_rewrite(delta.type)) {
-	assert(extent->last_committed_crc == delta.prev_crc);
+	ceph_assert_always(extent->last_committed_crc == delta.prev_crc);
 	assert(extent->version == delta.pversion);
 	extent->apply_delta_and_adjust_crc(record_base, delta.bl);
 	extent->set_modify_time(modify_time);
-	assert(extent->last_committed_crc == delta.final_crc);
+	ceph_assert_always(extent->last_committed_crc == delta.final_crc);
       } else {
 	assert(delta.paddr.get_addr_type() == paddr_types_t::RANDOM_BLOCK);
 	extent->apply_delta_and_adjust_crc(record_base, delta.bl);
 	extent->set_modify_time(modify_time);
-	// Since rewrite_dirty can conflict with other transaction after 
-	// inplace rewrite is complete, crc may not be matched
-	if (delta.final_crc == extent->last_committed_crc) {
-	  assert(extent->version == delta.pversion);
-	}
+	// crc will be checked after journal replay is done
       }
 
       extent->version++;
@@ -1866,7 +1869,8 @@ Cache::replay_delta(
               journal_seq, record_base, delta, *extent);
       }
       mark_dirty(extent);
-      return replay_delta_ertr::make_ready_future<bool>(true);
+      return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
+	std::make_pair(true, extent));
     });
   }
 }
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index c79473f98ba2..0f32bfe87e66 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -1039,7 +1039,8 @@ class Cache {
    */
   using replay_delta_ertr = crimson::errorator<
     crimson::ct_error::input_output_error>;
-  using replay_delta_ret = replay_delta_ertr::future<bool>;
+  using replay_delta_ret = replay_delta_ertr::future<
+    std::pair<bool, CachedExtentRef>>;
   replay_delta_ret replay_delta(
     journal_seq_t seq,
     paddr_t record_block_base,
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 487b0f3555e1..4cb7e2b490cb 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -611,6 +611,10 @@ class CachedExtent
     return prior_instance;
   }
 
+  uint32_t get_last_committed_crc() const {
+    return last_committed_crc;
+  }
+
 private:
   template <typename T>
   friend class read_set_item_t;
diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h
index 18c0797a8b8b..633aa84d7dbc 100644
--- a/src/crimson/os/seastore/journal.h
+++ b/src/crimson/os/seastore/journal.h
@@ -8,6 +8,7 @@
 #include "crimson/os/seastore/ordering_handle.h"
 #include "crimson/os/seastore/seastore_types.h"
 #include "crimson/os/seastore/segment_seq_allocator.h"
+#include "crimson/os/seastore/cached_extent.h"
 
 namespace crimson::os::seastore {
 
@@ -88,7 +89,7 @@ class Journal {
     crimson::ct_error::erange>;
   using replay_ret = replay_ertr::future<>;
   using delta_handler_t = std::function<
-    replay_ertr::future<bool>(
+    replay_ertr::future<std::pair<bool, CachedExtentRef>>(
       const record_locator_t&,
       const delta_info_t&,
       const journal_seq_t&, // dirty_tail
diff --git a/src/crimson/os/seastore/journal/circular_bounded_journal.cc b/src/crimson/os/seastore/journal/circular_bounded_journal.cc
index ec41bfab1426..a02c74508417 100644
--- a/src/crimson/os/seastore/journal/circular_bounded_journal.cc
+++ b/src/crimson/os/seastore/journal/circular_bounded_journal.cc
@@ -316,7 +316,8 @@ Journal::replay_ret CircularBoundedJournal::replay(
     return seastar::do_with(
       std::move(delta_handler),
       std::map<paddr_t, journal_seq_t>(),
-      [this](auto &d_handler, auto &map) {
+      std::map<paddr_t, std::pair<CachedExtentRef, uint32_t>>(),
+      [this](auto &d_handler, auto &map, auto &crc_info) {
       auto build_paddr_seq_map = [&map](
         const auto &offsets,
         const auto &e,
@@ -339,8 +340,8 @@ Journal::replay_ret CircularBoundedJournal::replay(
       // The first pass to build the paddr->journal_seq_t map 
       // from extent allocations
       return scan_valid_record_delta(std::move(build_paddr_seq_map), tail
-      ).safe_then([this, &map, &d_handler, tail]() {
-	auto call_d_handler_if_valid = [this, &map, &d_handler](
+      ).safe_then([this, &map, &d_handler, tail, &crc_info]() {
+	auto call_d_handler_if_valid = [this, &map, &d_handler, &crc_info](
 	  const auto &offsets,
 	  const auto &e,
 	  sea_time_point modify_time)
@@ -353,12 +354,27 @@ Journal::replay_ret CircularBoundedJournal::replay(
 	      get_dirty_tail(),
 	      get_alloc_tail(),
 	      modify_time
-	    );
+	    ).safe_then([&e, &crc_info](auto ret) {
+	      auto [applied, ext] = ret;
+	      if (applied && ext && can_inplace_rewrite(
+		  ext->get_type())) {
+		crc_info[ext->get_paddr()] =
+		  std::make_pair(ext, e.final_crc);
+	      }
+	      return replay_ertr::make_ready_future<bool>(applied);
+	    });
 	  }
 	  return replay_ertr::make_ready_future<bool>(true);
 	};
 	// The second pass to replay deltas
-	return scan_valid_record_delta(std::move(call_d_handler_if_valid), tail);
+	return scan_valid_record_delta(std::move(call_d_handler_if_valid), tail
+	).safe_then([&crc_info]() {
+	  for (auto p : crc_info) {
+	    ceph_assert_always(p.second.first->get_last_committed_crc() == p.second.second);	
+	  }
+	  crc_info.clear();
+	  return replay_ertr::now();
+	});
       });
     }).safe_then([this]() {
       // make sure that committed_to is JOURNAL_SEQ_NULL if jounal is the initial state
diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc
index 58df91374932..c40295dbacef 100644
--- a/src/crimson/os/seastore/journal/segmented_journal.cc
+++ b/src/crimson/os/seastore/journal/segmented_journal.cc
@@ -291,7 +291,8 @@ SegmentedJournal::replay_segment(
 	      trimmer.get_dirty_tail(),
 	      trimmer.get_alloc_tail(),
               modify_time
-            ).safe_then([&stats, delta_type=delta.type](bool is_applied) {
+            ).safe_then([&stats, delta_type=delta.type](auto ret) {
+	      auto [is_applied, ext] = ret;
               if (is_applied) {
                 // see Cache::replay_delta()
                 assert(delta_type != extent_types_t::JOURNAL_TAIL);
diff --git a/src/test/crimson/seastore/test_cbjournal.cc b/src/test/crimson/seastore/test_cbjournal.cc
index 0bf2d41358bf..bacb3cd2f78e 100644
--- a/src/test/crimson/seastore/test_cbjournal.cc
+++ b/src/test/crimson/seastore/test_cbjournal.cc
@@ -246,7 +246,8 @@ struct cbjournal_test_t : public seastar_test_suite_t, JournalTrimmer
 	}
       }
       assert(found == true);
-      return Journal::replay_ertr::make_ready_future<bool>(true);
+      return Journal::replay_ertr::make_ready_future<
+	std::pair<bool, CachedExtentRef>>(true, nullptr);
     });
   }
 
@@ -576,7 +577,8 @@ TEST_F(cbjournal_test_t, multiple_submit_at_end)
 	     auto &dirty_seq,
 	     auto &alloc_seq,
 	     auto last_modified) {
-      return Journal::replay_ertr::make_ready_future<bool>(true);
+      return Journal::replay_ertr::make_ready_future<
+	std::pair<bool, CachedExtentRef>>(true, nullptr);
     }).unsafe_get0();
     assert(get_written_to() == old_written_to);
   });
diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc
index 46ec723a3524..ddd894349d21 100644
--- a/src/test/crimson/seastore/test_seastore_journal.cc
+++ b/src/test/crimson/seastore/test_seastore_journal.cc
@@ -218,7 +218,8 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider, JournalTrimmer {
 	  delta_checker = std::nullopt;
 	  advance();
 	}
-	return Journal::replay_ertr::make_ready_future<bool>(true);
+	return Journal::replay_ertr::make_ready_future<
+	  std::pair<bool, CachedExtentRef>>(true, nullptr);
       }).unsafe_get0();
     ASSERT_EQ(record_iter, records.end());
     for (auto &i : records) {

From 72c9d6d17f6d5b8527123c769e956c859c00acf7 Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Fri, 5 Jan 2024 15:46:43 +0900
Subject: [PATCH 1226/2492] crimson/os/seastore/cache: return false explicitly
 if delta is not replayed

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 7513cd601871..b81fe62906c7 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1839,7 +1839,7 @@ Cache::replay_delta(
 	      journal_seq, record_base, delta);
 	assert(delta.pversion > 0);
 	return replay_delta_ertr::make_ready_future<std::pair<bool, CachedExtentRef>>(
-	  std::make_pair(true, nullptr));
+	  std::make_pair(false, nullptr));
       }
 
       DEBUG("replay extent delta at {} {} ... -- {}, prv_extent={}",

From 1f0ff27898dfd06a17b18fadd52f68dfb73ccb94 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 5 Jan 2024 21:11:59 +1000
Subject: [PATCH 1227/2492] doc/radosrgw: edit admin.rst

Edit "Add / Remove Admin Capabilities" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 39 +++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index f922ad2b8df6..6eb36452edea 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -353,32 +353,43 @@ To remove a Swift secret key, run a command of the following form:
    radosgw-admin key rm --subuser=foo:bar --key-type=swift
 
 
-Add / Remove Admin Capabilities
--------------------------------
+Add or Remove Admin Capabilities
+--------------------------------
 
-The Ceph Storage Cluster provides an administrative API that enables  users to
-execute administrative functions via the REST API. By default, users do NOT have
-access to this API. To enable a user to exercise  administrative functionality,
-provide the user with administrative capabilities.
+The Ceph Storage Cluster provides an administrative API that enables users to
+execute administrative functions via the REST API. By default, users do NOT
+have access to this API. To enable a user to exercise administrative
+functionality, provide the user with administrative capabilities.
 
-To add administrative capabilities to a user, execute the following:: 
+To add administrative capabilities to a user, run a command of the following
+form: 
 
-	radosgw-admin caps add --uid={uid} --caps={caps}
+.. prompt:: bash
 
+   radosgw-admin caps add --uid={uid} --caps={caps}
 
-You can add read, write or all capabilities to users, buckets, metadata and 
-usage (utilization). For example::
 
-	--caps="[users|buckets|metadata|usage|zone|amz-cache|info|bilog|mdlog|datalog|user-policy|oidc-provider|roles|ratelimit]=[*|read|write|read, write]"
+You can add read, write or all capabilities to users, buckets, metadata and
+usage (utilization). To do this, use a command-line option of the following
+form:
 
-For example::
+.. prompt:: bash
+
+   --caps="[users|buckets|metadata|usage|zone|amz-cache|info|bilog|mdlog|datalog|user-policy|oidc-provider|roles|ratelimit]=[\*|read|write|read, write]"
+
+For example:
+
+.. prompt:: bash
 
 	radosgw-admin caps add --uid=johndoe --caps="users=*;buckets=*"
 
+To remove administrative capabilities from a user, run a command of the
+following form: 
 
-To remove administrative capabilities from a user, execute the following:: 
+.. prompt:: bash
 
-	radosgw-admin caps rm --uid=johndoe --caps={caps}
+   radosgw-admin caps rm --uid=johndoe --caps={caps}
+  
 
 
 Quota Management

From 569c07f979d9f0846f803565dfc35ed0f9acaba8 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 23 Dec 2023 10:18:41 -0600
Subject: [PATCH 1228/2492] osd/scrub: manage queue registration lifetime in
 the FSM

As the state of 'being registered in the OSDs scrub queue'
corresponds to the PrimaryActive FSM state.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                          |  1 -
 src/osd/PrimaryLogPG.cc                |  3 +--
 src/osd/scrubber/pg_scrubber.cc        | 30 +++++++++-----------------
 src/osd/scrubber/pg_scrubber.h         |  2 +-
 src/osd/scrubber/scrub_machine.cc      |  2 ++
 src/osd/scrubber/scrub_machine_lstnr.h |  9 ++++++++
 src/osd/scrubber_common.h              |  8 -------
 7 files changed, 23 insertions(+), 32 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 3138b8c32f9c..eb7a332f6c2f 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1847,7 +1847,6 @@ void PG::on_activate(interval_set<snapid_t> snaps)
   snap_trimq = snaps;
   release_pg_backoffs();
   projected_last_update = info.last_update;
-  m_scrubber->on_pg_activate(m_planned_scrub);
 }
 
 void PG::on_replica_activate()
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 8120d76e7ea8..d57ab432d4ac 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -12879,8 +12879,7 @@ void PrimaryLogPG::on_shutdown()
     osd->clear_queued_recovery(this);
   }
 
-  m_scrubber->scrub_clear_state();
-  m_scrubber->rm_from_osd_scrubbing();
+  m_scrubber->on_new_interval();
 
   vector<ceph_tid_t> tids;
   cancel_copy_ops(false, &tids);
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index dd279f720add..ab9b86e5b35c 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -460,14 +460,12 @@ void PgScrubber::on_new_interval()
 		  (is_primary() ? "Primary" : "Replica/other"),
 		  is_scrub_active(), is_queued_or_active())
 	   << dendl;
-
   m_fsm->process_event(IntervalChanged{});
   // the following asserts were added due to a past bug, where PG flags were
   // left set in some scenarios.
   ceph_assert(!is_queued_or_active());
   ceph_assert(!state_test(PG_STATE_SCRUBBING));
   ceph_assert(!state_test(PG_STATE_DEEP_SCRUB));
-  rm_from_osd_scrubbing();
 }
 
 bool PgScrubber::is_scrub_registered() const
@@ -493,35 +491,31 @@ void PgScrubber::rm_from_osd_scrubbing()
   }
 }
 
-void PgScrubber::on_pg_activate(const requested_scrub_t& request_flags)
+/*
+ * Note: referring to m_planned_scrub here is temporary, as this set of
+ * scheduling flags will be removed in a followup PR.
+ */
+void PgScrubber::schedule_scrub_with_osd()
 {
   ceph_assert(is_primary());
-  if (!m_scrub_job) {
-    // we won't have a chance to see more logs from this function, thus:
-    dout(2) << fmt::format(
-		   "{}: flags:<{}> {}.Reg-state:{:.7}. No scrub-job", __func__,
-		   request_flags, (is_primary() ? "Primary" : "Replica/other"),
-		   registration_state())
-	    << dendl;
-    return;
-  }
+  ceph_assert(m_scrub_job);
 
-  ceph_assert(!is_queued_or_active());
   auto pre_state = m_scrub_job->state_desc();
   auto pre_reg = registration_state();
 
   auto suggested = m_osds->get_scrub_services().determine_scrub_time(
-      request_flags, m_pg->info, m_pg->get_pgpool().info.opts);
+      m_planned_scrub, m_pg->info, m_pg->get_pgpool().info.opts);
   m_osds->get_scrub_services().register_with_osd(m_scrub_job, suggested);
 
   dout(10) << fmt::format(
 		  "{}: <flags:{}> {} <{:.5}>&<{:.10}> --> <{:.5}>&<{:.14}>",
-		  __func__, request_flags,
+		  __func__, m_planned_scrub,
 		  (is_primary() ? "Primary" : "Replica/other"), pre_reg,
 		  pre_state, registration_state(), m_scrub_job->state_desc())
 	   << dendl;
 }
 
+
 void PgScrubber::on_primary_active_clean()
 {
   dout(10) << fmt::format(
@@ -2177,11 +2171,7 @@ void PgScrubber::handle_query_state(ceph::Formatter* f)
 PgScrubber::~PgScrubber()
 {
   m_fsm->process_event(IntervalChanged{});
-  if (m_scrub_job) {
-    // make sure the OSD won't try to scrub this one just now
-    rm_from_osd_scrubbing();
-    m_scrub_job.reset();
-  }
+  m_scrub_job.reset();
 }
 
 PgScrubber::PgScrubber(PG* pg)
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index af667f732143..8360b4c038f7 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -252,7 +252,7 @@ class PgScrubber : public ScrubPgIF,
 
   void rm_from_osd_scrubbing() final;
 
-  void on_pg_activate(const requested_scrub_t& request_flags) final;
+  void schedule_scrub_with_osd() final;
 
   scrub_level_t scrub_requested(
       scrub_level_t scrub_level,
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 1928eed7d0cd..26054bf3f764 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -123,6 +123,8 @@ PrimaryActive::PrimaryActive(my_context ctx)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "-- state -->> PrimaryActive" << dendl;
+  // insert this PG into the OSD scrub queue. Calculate initial schedule
+  scrbr->schedule_scrub_with_osd();
 }
 
 PrimaryActive::~PrimaryActive()
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 8b110fe62719..086802ee813e 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -107,6 +107,15 @@ struct ScrubMachineListener {
 
   [[nodiscard]] virtual bool is_primary() const = 0;
 
+  /// dequeue this PG from the OSD's scrub-queue
+  virtual void rm_from_osd_scrubbing() = 0;
+
+  /**
+   * the FSM has entered the PrimaryActive state. That happens when
+   * peered as a Primary, and achieving the 'active' state.
+   */
+  virtual void schedule_scrub_with_osd() = 0;
+
   virtual void select_range_n_notify() = 0;
 
   /// walk the log to find the latest update that affects our chunk
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index fbbef578ae69..b41a9b4bba84 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -409,12 +409,6 @@ struct ScrubPgIF {
    */
   virtual bool reserve_local() = 0;
 
-  /**
-   * if activated as a Primary - register the scrub job with the OSD
-   * scrub queue
-   */
-  virtual void on_pg_activate(const requested_scrub_t& request_flags) = 0;
-
   /**
    * Recalculate the required scrub time.
    *
@@ -431,8 +425,6 @@ struct ScrubPgIF {
    */
   virtual void handle_scrub_reserve_msgs(OpRequestRef op) = 0;
 
-  virtual void rm_from_osd_scrubbing() = 0;
-
   virtual scrub_level_t scrub_requested(
       scrub_level_t scrub_level,
       scrub_type_t scrub_type,

From 6d6cc7145636c4482b81af02993b9d83043d3ec0 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 28 Dec 2023 13:41:19 -0600
Subject: [PATCH 1229/2492] osd/scrub: avoid "over clearing" queued_or_active
 flag

If two StartScrub messages are received in quick succession, the earlier
one might clear the queued_or_active flag as it fails for being from an old
interval.
When that happens - a 3'rd scrub request will actually be allowed to go
through, while the scrubber is still handling the second one.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc        | 21 +++++++++++----------
 src/osd/scrubber/pg_scrubber.h         |  2 +-
 src/osd/scrubber/scrub_machine.cc      |  7 +++++++
 src/osd/scrubber/scrub_machine.h       |  1 +
 src/osd/scrubber/scrub_machine_lstnr.h |  3 +++
 5 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index ab9b86e5b35c..f7e027a84f07 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -208,17 +208,18 @@ bool PgScrubber::should_abort() const
 
 void PgScrubber::initiate_regular_scrub(epoch_t epoch_queued)
 {
-  dout(15) << __func__ << " epoch: " << epoch_queued << dendl;
+  dout(10) << fmt::format(
+		  "{}: epoch:{} is PrimaryIdle:{}", __func__, epoch_queued,
+		  m_fsm->is_primary_idle())
+	   << dendl;
+
   // we may have lost our Primary status while the message languished in the
   // queue
   if (check_interval(epoch_queued)) {
     dout(10) << "scrubber event -->> StartScrub epoch: " << epoch_queued
 	     << dendl;
-    reset_epoch(epoch_queued);
     m_fsm->process_event(StartScrub{});
     dout(10) << "scrubber event --<< StartScrub" << dendl;
-  } else {
-    clear_queued_or_active();  // also restarts snap trimming
   }
 }
 
@@ -229,17 +230,17 @@ void PgScrubber::advance_token()
 
 void PgScrubber::initiate_scrub_after_repair(epoch_t epoch_queued)
 {
-  dout(15) << __func__ << " epoch: " << epoch_queued << dendl;
+  dout(10) << fmt::format(
+		  "{}: epoch:{} is PrimaryIdle:{}", __func__, epoch_queued,
+		  m_fsm->is_primary_idle())
+	   << dendl;
   // we may have lost our Primary status while the message languished in the
   // queue
   if (check_interval(epoch_queued)) {
     dout(10) << "scrubber event -->> AfterRepairScrub epoch: " << epoch_queued
 	     << dendl;
-    reset_epoch(epoch_queued);
     m_fsm->process_event(AfterRepairScrub{});
     dout(10) << "scrubber event --<< AfterRepairScrub" << dendl;
-  } else {
-    clear_queued_or_active();  // also restarts snap trimming
   }
 }
 
@@ -403,13 +404,13 @@ bool PgScrubber::is_reserving() const
   return m_fsm->is_reserving();
 }
 
-void PgScrubber::reset_epoch(epoch_t epoch_queued)
+void PgScrubber::reset_epoch()
 {
   dout(10) << __func__ << " state deep? " << state_test(PG_STATE_DEEP_SCRUB)
 	   << dendl;
   m_fsm->assert_not_in_session();
 
-  m_epoch_start = epoch_queued;
+  m_epoch_start = m_pg->get_same_interval_since();
   ceph_assert(m_is_deep == state_test(PG_STATE_DEEP_SCRUB));
   update_op_mode_text();
 }
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 8360b4c038f7..adf50f97d92a 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -556,7 +556,7 @@ class PgScrubber : public ScrubPgIF,
    *  - the epoch when started;
    *  - the depth of the scrub requested (from the PG_STATE variable)
    */
-  void reset_epoch(epoch_t epoch_queued);
+  void reset_epoch() final;
 
   void run_callbacks();
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 26054bf3f764..9aa5842c4908 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -57,6 +57,11 @@ bool ScrubMachine::is_reserving() const
   return state_cast<const ReservingReplicas*>();
 }
 
+bool ScrubMachine::is_primary_idle() const
+{
+  return state_cast<const PrimaryIdle*>();
+}
+
 bool ScrubMachine::is_accepting_updates() const
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
@@ -150,6 +155,7 @@ sc::result PrimaryIdle::react(const StartScrub&)
 {
   dout(10) << "PrimaryIdle::react(const StartScrub&)" << dendl;
   DECLARE_LOCALS;
+  scrbr->reset_epoch();
   return transit<ReservingReplicas>();
 }
 
@@ -157,6 +163,7 @@ sc::result PrimaryIdle::react(const AfterRepairScrub&)
 {
   dout(10) << "PrimaryIdle::react(const AfterRepairScrub&)" << dendl;
   DECLARE_LOCALS;
+  scrbr->reset_epoch();
   return transit<ReservingReplicas>();
 }
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index fbd8f8b3a359..8fa96da14052 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -244,6 +244,7 @@ class ScrubMachine : public sc::state_machine<ScrubMachine, NotActive> {
   void assert_not_in_session() const;
   [[nodiscard]] bool is_reserving() const;
   [[nodiscard]] bool is_accepting_updates() const;
+  [[nodiscard]] bool is_primary_idle() const;
 
   // elapsed time for the currently active scrub.session
   ceph::timespan get_time_scrubbing() const;
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 086802ee813e..f2ddd9c87168 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -212,6 +212,9 @@ struct ScrubMachineListener {
   virtual void set_queued_or_active() = 0;
   virtual void clear_queued_or_active() = 0;
 
+  /// note the epoch when the scrub session started
+  virtual void reset_epoch() = 0;
+
   /**
    * Our scrubbing is blocked, waiting for an excessive length of time for
    * our target chunk to be unlocked. We will set the corresponding flags,

From 937aa7ec21be2c5ef4ab5fa4b7a69669b8354617 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarit@redhat.com>
Date: Fri, 5 Jan 2024 17:44:30 +0000
Subject: [PATCH 1230/2492] mgr/telemetry: add pool flags

Pool flags can help us learn how certain features are being used, for
example, the "crimson" flag implies a crimson cluster.

We add a basic_pool_flags collection to the "basic" channel for this
purpose.

Signed-off-by: Yaarit Hatuka <yaarit@redhat.com>
---
 doc/dev/release-checklists.rst     |  1 +
 doc/mgr/telemetry.rst              |  1 +
 src/pybind/mgr/telemetry/module.py | 37 ++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
index 3a3b2a98cd76..f6a4dca39077 100644
--- a/doc/dev/release-checklists.rst
+++ b/doc/dev/release-checklists.rst
@@ -146,6 +146,7 @@ First release candidate
 
 - [ ] src/ceph_release: change type to `rc`
 - [ ] opt-in to all telemetry channels, generate telemetry reports, and verify no sensitive details (like pools names) are collected
+- [ ] check if new pool flags exist in pg_pool_t (osd/osd_types.h), and add them to telemetry's basic_pool_flags collection, in case they are not sensitive
 
 
 First stable release
diff --git a/doc/mgr/telemetry.rst b/doc/mgr/telemetry.rst
index 90d45766c15b..2d09bb9f3577 100644
--- a/doc/mgr/telemetry.rst
+++ b/doc/mgr/telemetry.rst
@@ -186,6 +186,7 @@ List all collections with::
   NAME                            STATUS                                               DESC
   basic_base                      NOT REPORTING: NOT OPTED-IN                          Basic information about the cluster (capacity, number and type of daemons, version, etc.)
   basic_mds_metadata              NOT REPORTING: NOT OPTED-IN                          MDS metadata
+  basic_pool_flags                NOT REPORTING: NOT OPTED-IN                          Per-pool flags
   basic_pool_options_bluestore    NOT REPORTING: NOT OPTED-IN                          Per-pool bluestore config options
   basic_pool_usage                NOT REPORTING: NOT OPTED-IN                          Default pool application and usage statistics
   basic_rook_v01                  NOT REPORTING: NOT OPTED-IN                          Basic Rook deployment data
diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py
index f729b9180cfb..e4e238a88f28 100644
--- a/src/pybind/mgr/telemetry/module.py
+++ b/src/pybind/mgr/telemetry/module.py
@@ -71,6 +71,7 @@ class Collection(str, enum.Enum):
     basic_rook_v01 = 'basic_rook_v01'
     perf_memory_metrics = 'perf_memory_metrics'
     basic_pool_options_bluestore = 'basic_pool_options_bluestore'
+    basic_pool_flags = 'basic_pool_flags'
 
 MODULE_COLLECTION : List[Dict] = [
     {
@@ -139,6 +140,12 @@ class Collection(str, enum.Enum):
         "channel": "basic",
         "nag": False
     },
+    {
+        "name": Collection.basic_pool_flags,
+        "description": "Per-pool flags",
+        "channel": "basic",
+        "nag": False
+    },
 ]
 
 ROOK_KEYS_BY_COLLECTION : List[Tuple[str, Collection]] = [
@@ -1109,7 +1116,37 @@ def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any]
                         for option in bluestore_options:
                             if option in pool['options']:
                                 pool_data['options'][option] = pool['options'][option]
+
+                # basic_pool_flags collection
+                if self.is_enabled_collection(Collection.basic_pool_flags):
+                    if 'flags_names' in pool and pool['flags_names'] is not None:
+                        # flags are defined in pg_pool_t (src/osd/osd_types.h)
+                        flags_to_report = [
+                            'hashpspool',
+                            'full',
+                            'ec_overwrites',
+                            'incomplete_clones',
+                            'nodelete',
+                            'nopgchange',
+                            'nosizechange',
+                            'write_fadvise_dontneed',
+                            'noscrub',
+                            'nodeep-scrub',
+                            'full_quota',
+                            'nearfull',
+                            'backfillfull',
+                            'selfmanaged_snaps',
+                            'pool_snaps',
+                            'creating',
+                            'eio',
+                            'bulk',
+                            'crimson',
+                            ]
+
+                        pool_data['flags_names'] = [flag for flag in pool['flags_names'].split(',') if flag in flags_to_report]
+
                 cast(List[Dict[str, Any]], report['pools']).append(pool_data)
+
                 if 'rbd' in pool['application_metadata']:
                     rbd_num_pools += 1
                     ioctx = self.rados.open_ioctx(pool['pool_name'])

From e6c58dcd614f71efa4de2f781b8425c480af285e Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Fri, 5 Jan 2024 20:13:41 +0000
Subject: [PATCH 1231/2492] doc: update rgw admin api req params for get user
 info

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 doc/radosgw/adminops.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst
index 0974b95c5862..f1faac6b1c2f 100644
--- a/doc/radosgw/adminops.rst
+++ b/doc/radosgw/adminops.rst
@@ -275,6 +275,9 @@ Get User Info
 
 Get user information.
 
+Either a ``uid`` or ``access-key`` must be supplied as a request parameter. We recommend supplying uid.
+If both are provided but correspond to different users, the info for the user specified with ``uid`` will be returned.
+
 :caps: users=read
 
 
@@ -297,6 +300,13 @@ Request Parameters
 :Example: ``foo_user``
 :Required: Yes
 
+``access-key``
+
+:Description: The S3 access key of the user for which the information is requested.
+:Type: String
+:Example: ``ABCD0EF12GHIJ2K34LMN``
+:Required: No
+
 
 Response Entities
 ~~~~~~~~~~~~~~~~~

From 950aa4963d3671ea84b47f239e8743775632acba Mon Sep 17 00:00:00 2001
From: Prashant D <pdhange@redhat.com>
Date: Fri, 5 Jan 2024 17:26:46 -0500
Subject: [PATCH 1232/2492] mon: initialize ProgressEvent::add_to_ceph_s

The add_to_ceph_s field was not initialized causing unit
test failures from check-generated.sh:

**** ProgressEvent test 1 dump_json check failed ****
   ceph-dencoder type ProgressEvent select_test 1 dump_json > /tmp/typ-akwRrjNtt
   ceph-dencoder type ProgressEvent select_test 1 encode decode dump_json > /tmp/typ-iDmGaq17A
4c4
<     "add_to_ceph_s": false
---
>     "add_to_ceph_s": true

Fixes: https://tracker.ceph.com/issues/63950

Signed-off-by: Prashant D <pdhange@redhat.com>
---
 src/mon/mon_types.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index 91502443d426..3429a8e99916 100644
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -684,7 +684,7 @@ inline std::ostream& operator<<(std::ostream& out, const mon_feature_t& f) {
 struct ProgressEvent {
   std::string message;                  ///< event description
   float progress = 0.0f;                  ///< [0..1]
-  bool add_to_ceph_s;
+  bool add_to_ceph_s = false;
   void encode(ceph::buffer::list& bl) const {
     ENCODE_START(2, 1, bl);
     encode(message, bl);

From d2205bf8247edec08a456377c1e8988428e9ba29 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 7 Dec 2023 14:03:39 +0530
Subject: [PATCH 1233/2492] mgr/volumes: add remote helper methods to fetch
 subvolume info

  * add remote method to fetch subvolume path
  * add remote method to list subvolumes
  * add remote method to fetch subvolume attributes

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/pybind/mgr/volumes/fs/volume.py |  1 +
 src/pybind/mgr/volumes/module.py    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index e19d1eafb2a6..c896fd73d0b0 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -422,6 +422,7 @@ def subvolume_info(self, **kwargs):
 
                         subvol_info_dict = subvolume.info()
                         subvol_info_dict["mon_addrs"] = mon_addr_lst
+                        subvol_info_dict["flavor"] = subvolume.VERSION
                         ret = 0, json.dumps(subvol_info_dict, indent=4, sort_keys=True), ""
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index 6227276fcaf5..68031ed55a3b 100644
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -855,3 +855,19 @@ def _cmd_fs_clone_status(self, inbuf, cmd):
     def _cmd_fs_clone_cancel(self, inbuf, cmd):
         return self.vc.clone_cancel(
             vol_name=cmd['vol_name'], clone_name=cmd['clone_name'], group_name=cmd.get('group_name', None))
+
+    # remote method
+    def subvolume_getpath(self, vol_name, subvol, group_name):
+        return self.vc.subvolume_getpath(vol_name=vol_name,
+                                         sub_name=subvol,
+                                         group_name=group_name)
+
+    # remote method
+    def subvolume_ls(self, vol_name, group_name):
+        return self.vc.list_subvolumes(vol_name=vol_name, group_name=group_name)
+
+    # remote method
+    def subvolume_info(self, vol_name, subvol, group_name):
+        return self.vc.subvolume_info(vol_name=vol_name,
+                                      sub_name=subvol,
+                                      group_name=group_name)

From ffa197c03159e2acb7c3892f09a1ed616b9c1cd8 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 9 Nov 2023 12:38:48 +0530
Subject: [PATCH 1234/2492] mds/snap_schedule: add subvolume group column
 management

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/pybind/mgr/snap_schedule/fs/schedule.py   | 10 +++--
 .../mgr/snap_schedule/fs/schedule_client.py   | 40 ++++++++++++++++++-
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/snap_schedule/fs/schedule.py b/src/pybind/mgr/snap_schedule/fs/schedule.py
index 3538fe501ddb..ba4390939bf7 100644
--- a/src/pybind/mgr/snap_schedule/fs/schedule.py
+++ b/src/pybind/mgr/snap_schedule/fs/schedule.py
@@ -89,6 +89,7 @@ def __init__(self,
                  rel_path: str,
                  start: Optional[str] = None,
                  subvol: Optional[str] = None,
+                 group: Optional[str] = None,
                  retention_policy: str = '{}',
                  created: Optional[str] = None,
                  first: Optional[str] = None,
@@ -100,6 +101,7 @@ def __init__(self,
                  ) -> None:
         self.fs = fs_name
         self.subvol = subvol
+        self.group = group
         self.path = path
         self.rel_path = rel_path
         self.schedule = schedule
@@ -145,6 +147,7 @@ def _from_db_row(cls, table_row: TableRowT, fs: str) -> 'Schedule':
                    cast(str, table_row['rel_path']),
                    cast(str, table_row['start']),
                    cast(str, table_row['subvol']),
+                   cast(str, table_row['group_name']),
                    cast(str, table_row['retention']),
                    cast(str, table_row['created']),
                    cast(str, table_row['first']),
@@ -200,7 +203,7 @@ def json_list(self) -> str:
         ORDER BY until;'''
 
     PROTO_GET_SCHEDULES = '''SELECT
-          s.path, s.subvol, s.rel_path, sm.active,
+          s.path, s.subvol, s.group_name, s.rel_path, sm.active,
           sm.schedule, s.retention, sm.start, sm.first, sm.last,
           sm.last_pruned, sm.created, sm.created_count, sm.pruned_count
           FROM schedules s
@@ -255,8 +258,8 @@ def list_all_schedules(cls,
             return [cls._from_db_row(row, fs) for row in c.fetchall()]
 
     INSERT_SCHEDULE = '''INSERT INTO
-        schedules(path, subvol, retention, rel_path)
-        Values(?, ?, ?, ?);'''
+        schedules(path, subvol, group_name, retention, rel_path)
+        Values(?, ?, ?, ?, ?);'''
     INSERT_SCHEDULE_META = '''INSERT INTO
         schedules_meta(schedule_id, start, created, repeat, schedule,
         active)
@@ -270,6 +273,7 @@ def store_schedule(self, db: sqlite3.Connection) -> None:
                 c = db.execute(self.INSERT_SCHEDULE,
                                (self.path,
                                 self.subvol,
+                                self.group,
                                 json.dumps(self.retention),
                                 self.rel_path,))
                 sched_id = c.lastrowid
diff --git a/src/pybind/mgr/snap_schedule/fs/schedule_client.py b/src/pybind/mgr/snap_schedule/fs/schedule_client.py
index ec3d32fb4438..8b199b632c32 100644
--- a/src/pybind/mgr/snap_schedule/fs/schedule_client.py
+++ b/src/pybind/mgr/snap_schedule/fs/schedule_client.py
@@ -180,6 +180,41 @@ def allow_minute_snaps(self) -> None:
     def dump_on_update(self) -> None:
         return self.mgr.get_module_option('dump_on_update')
 
+    def _create_snap_schedule_kv_db(self, db: sqlite3.Connection) -> None:
+        SQL = """
+        CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV (
+          key TEXT PRIMARY KEY,
+          value NOT NULL
+        ) WITHOUT ROWID;
+        INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1);
+        """
+        db.executescript(SQL)
+
+    def _get_snap_schedule_db_version(self, db: sqlite3.Connection) -> int:
+        SQL = """
+        SELECT value
+        FROM SnapScheduleModuleKV
+        WHERE key = '__snap_schedule_db_version';
+        """
+        cur = db.execute(SQL)
+        row = cur.fetchone()
+        assert row is not None
+        return int(row[0])
+
+    # add all upgrades here
+    def _upgrade_snap_schedule_db_schema(self, db: sqlite3.Connection) -> None:
+        # add a column to hold the subvolume group name
+        if self._get_snap_schedule_db_version(db) < 2:
+            SQL = """
+            ALTER TABLE schedules
+            ADD COLUMN group_name TEXT;
+            """
+            db.executescript(SQL)
+
+            # bump up the snap-schedule db version to 2
+            SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';"
+            db.execute(SQL, (2,))
+
     def get_schedule_db(self, fs: str) -> DBConnectionManager:
         dbinfo = None
         self.conn_lock.acquire()
@@ -206,6 +241,8 @@ def get_schedule_db(self, fs: str) -> DBConnectionManager:
                 except rados.ObjectNotFound:
                     log.debug(f'No legacy schedule DB found in {fs}')
             db.executescript(Schedule.CREATE_TABLES)
+            self._create_snap_schedule_kv_db(db)
+            self._upgrade_snap_schedule_db_schema(db)
             self.sqlite_connections[fs] = DBInfo(fs, db)
         dbinfo = self.sqlite_connections[fs]
         self.conn_lock.release()
@@ -370,7 +407,8 @@ def list_snap_schedules(self,
     def store_snap_schedule(self,
                             fs: str, path_: str,
                             args: Tuple[str, str, str, str,
-                                        Optional[str], Optional[str]]) -> None:
+                                        Optional[str], Optional[str],
+                                        Optional[str]]) -> None:
         sched = Schedule(*args)
         log.debug(f'repeat is {sched.repeat}')
         if sched.parse_schedule(sched.schedule)[1] == 'm' and not self.allow_minute_snaps:

From 5b250907d03196bf99fe4a56ea00245a4c830264 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 9 Nov 2023 13:25:02 +0530
Subject: [PATCH 1235/2492] mgr/snap_schedule: add subvol and subvol group
 arguments to cli

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/pybind/mgr/snap_schedule/module.py | 134 ++++++++++++++++++++-----
 1 file changed, 107 insertions(+), 27 deletions(-)

diff --git a/src/pybind/mgr/snap_schedule/module.py b/src/pybind/mgr/snap_schedule/module.py
index 8ff7ff372889..d8f04a62b94e 100644
--- a/src/pybind/mgr/snap_schedule/module.py
+++ b/src/pybind/mgr/snap_schedule/module.py
@@ -6,7 +6,7 @@
 import errno
 import json
 import sqlite3
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple, Union
 from .fs.schedule_client import SnapSchedClient
 from mgr_module import MgrModule, CLIReadCommand, CLIWriteCommand, Option
 from mgr_util import CephfsConnectionException
@@ -37,6 +37,44 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self._initialized = Event()
         self.client = SnapSchedClient(self)
 
+    def _subvolume_exist(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> bool:
+        rc, subvolumes, err = self.remote('volumes', 'subvolume_ls', fs, group)
+        if rc == 0:
+            for svj in json.loads(subvolumes):
+                if subvol == svj['name']:
+                    return True
+
+        return False
+
+    def _subvolume_flavor(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> str:
+        rc, subvol_info, err = self.remote('volumes', 'subvolume_info', fs, subvol, group)
+        svi_json = json.loads(subvol_info)
+        return svi_json.get('flavor', 'bad_flavor')  # 1 or 2 etc.
+
+    def _resolve_subvolume_path(self, fs: str, path: str, subvol: Union[str, None], group: Union[str, None]) -> Tuple[int, str, str]:
+        if subvol is None and group is None:
+            return 0, path, ''
+
+        rc = -1
+        subvol_path = ''
+        if self._subvolume_exist(fs, subvol, group):
+            rc, subvol_path, err = self.remote('volumes', 'subvolume_getpath', fs, subvol, group)
+            if rc != 0:
+                return rc, '', f'Could not resolve subvol:{subvol} path in fs:{fs}'
+            else:
+                subvol_flavor = self._subvolume_flavor(fs, subvol, group)
+                if subvol_flavor == 1:  # v1
+                    return 0, subvol_path, f'Ignoring user specified path:{path} for subvol'
+                if subvol_flavor == 2:  # v2
+                    err = '';
+                    if path != "/..":
+                        err = f'Ignoring user specified path:{path} for subvol'
+                    return 0, subvol_path + "/..", err
+
+                return -errno.EINVAL, '', f'Unhandled subvol flavor:{subvol_flavor}'
+        else:
+            return -errno.EINVAL, '', f'No such subvol: {group}::{subvol}'
+
     @property
     def _default_fs(self) -> Tuple[int, str, str]:
         fs_map = self.get('fs_map')
@@ -53,11 +91,11 @@ def _validate_fs(self, fs: Optional[str]) -> Tuple[int, str, str]:
             rc, fs, err = self._default_fs
             if rc < 0:
                 return rc, fs, err
-        if not self.has_fs(fs):
+        if not self._has_fs(fs):
             return -errno.EINVAL, '', f"no such file system: {fs}"
         return 0, fs, 'Success'
 
-    def has_fs(self, fs_name: str) -> bool:
+    def _has_fs(self, fs_name: str) -> bool:
         return fs_name in self.client.get_all_filesystems()
 
     def serve(self) -> None:
@@ -71,6 +109,8 @@ def handle_command(self, inbuf: str, cmd: Dict[str, str]) -> Tuple[int, str, str
     def snap_schedule_get(self,
                           path: str = '/',
                           fs: Optional[str] = None,
+                          subvol: Optional[str] = None,
+                          group: Optional[str] = None,
                           format: Optional[str] = 'plain') -> Tuple[int, str, str]:
         '''
         List current snapshot schedules
@@ -78,8 +118,12 @@ def snap_schedule_get(self,
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        errstr = 'Success'
         try:
-            ret_scheds = self.client.get_snap_schedules(fs, path)
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
+            ret_scheds = self.client.get_snap_schedules(fs, abs_path)
         except CephfsConnectionException as e:
             return e.to_tuple()
         except Exception as e:
@@ -87,12 +131,15 @@ def snap_schedule_get(self,
         if format == 'json':
             json_report = ','.join([ret_sched.report_json() for ret_sched in ret_scheds])
             return 0, f'[{json_report}]', ''
+        self.log.info(errstr)
         return 0, '\n===\n'.join([ret_sched.report() for ret_sched in ret_scheds]), ''
 
     @CLIReadCommand('fs snap-schedule list')
     def snap_schedule_list(self, path: str,
                            recursive: bool = False,
                            fs: Optional[str] = None,
+                           subvol: Optional[str] = None,
+                           group: Optional[str] = None,
                            format: Optional[str] = 'plain') -> Tuple[int, str, str]:
         '''
         Get current snapshot schedule for <path>
@@ -100,8 +147,12 @@ def snap_schedule_list(self, path: str,
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            scheds = self.client.list_snap_schedules(fs, path, recursive)
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
+            scheds = self.client.list_snap_schedules(fs, abs_path, recursive)
             self.log.debug(f'recursive is {recursive}')
         except CephfsConnectionException as e:
             return e.to_tuple()
@@ -116,7 +167,7 @@ def snap_schedule_list(self, path: str,
             # json_list = ','.join([sched.json_list() for sched in scheds])
             schedule_list = [sched.schedule for sched in scheds]
             retention_list = [sched.retention for sched in scheds]
-            out = {'path': path, 'schedule': schedule_list, 'retention': retention_list}
+            out = {'path': abs_path, 'schedule': schedule_list, 'retention': retention_list}
             return 0, json.dumps(out), ''
         return 0, '\n'.join([str(sched) for sched in scheds]), ''
 
@@ -125,23 +176,27 @@ def snap_schedule_add(self,
                           path: str,
                           snap_schedule: str,
                           start: Optional[str] = None,
-                          fs: Optional[str] = None) -> Tuple[int, str, str]:
+                          fs: Optional[str] = None,
+                          subvol: Optional[str] = None,
+                          group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Set a snapshot schedule for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
-            subvol = None
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.store_snap_schedule(fs,
                                             abs_path,
                                             (abs_path, snap_schedule,
-                                             fs, path, start, subvol))
-            suc_msg = f'Schedule set for path {path}'
+                                             fs, abs_path, start, subvol, group))
+            suc_msg = f'Schedule set for path {abs_path}'
         except sqlite3.IntegrityError:
-            existing_scheds = self.client.get_snap_schedules(fs, path)
+            existing_scheds = self.client.get_snap_schedules(fs, abs_path)
             report = [s.report() for s in existing_scheds]
             error_msg = f'Found existing schedule {report}'
             self.log.error(error_msg)
@@ -159,15 +214,20 @@ def snap_schedule_rm(self,
                          path: str,
                          repeat: Optional[str] = None,
                          start: Optional[str] = None,
-                         fs: Optional[str] = None) -> Tuple[int, str, str]:
+                         fs: Optional[str] = None,
+                         subvol: Optional[str] = None,
+                         group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Remove a snapshot schedule for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.rm_snap_schedule(fs, abs_path, repeat, start)
         except ValueError as e:
             return -errno.ENOENT, '', str(e)
@@ -175,22 +235,27 @@ def snap_schedule_rm(self,
             return e.to_tuple()
         except Exception as e:
             return -errno.EIO, '', str(e)
-        return 0, 'Schedule removed for path {}'.format(path), ''
+        return 0, 'Schedule removed for path {}'.format(abs_path), ''
 
     @CLIWriteCommand('fs snap-schedule retention add')
     def snap_schedule_retention_add(self,
                                     path: str,
                                     retention_spec_or_period: str,
                                     retention_count: Optional[str] = None,
-                                    fs: Optional[str] = None) -> Tuple[int, str, str]:
+                                    fs: Optional[str] = None,
+                                    subvol: Optional[str] = None,
+                                    group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Set a retention specification for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.add_retention_spec(fs, abs_path,
                                            retention_spec_or_period,
                                            retention_count)
@@ -200,22 +265,27 @@ def snap_schedule_retention_add(self,
             return e.to_tuple()
         except Exception as e:
             return -errno.EIO, '', str(e)
-        return 0, 'Retention added to path {}'.format(path), ''
+        return 0, 'Retention added to path {}'.format(abs_path), ''
 
     @CLIWriteCommand('fs snap-schedule retention remove')
     def snap_schedule_retention_rm(self,
                                    path: str,
                                    retention_spec_or_period: str,
                                    retention_count: Optional[str] = None,
-                                   fs: Optional[str] = None) -> Tuple[int, str, str]:
+                                   fs: Optional[str] = None,
+                                   subvol: Optional[str] = None,
+                                   group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Remove a retention specification for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.rm_retention_spec(fs, abs_path,
                                           retention_spec_or_period,
                                           retention_count)
@@ -225,22 +295,27 @@ def snap_schedule_retention_rm(self,
             return e.to_tuple()
         except Exception as e:
             return -errno.EIO, '', str(e)
-        return 0, 'Retention removed from path {}'.format(path), ''
+        return 0, 'Retention removed from path {}'.format(abs_path), ''
 
     @CLIWriteCommand('fs snap-schedule activate')
     def snap_schedule_activate(self,
                                path: str,
                                repeat: Optional[str] = None,
                                start: Optional[str] = None,
-                               fs: Optional[str] = None) -> Tuple[int, str, str]:
+                               fs: Optional[str] = None,
+                               subvol: Optional[str] = None,
+                               group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Activate a snapshot schedule for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.activate_snap_schedule(fs, abs_path, repeat, start)
         except ValueError as e:
             return -errno.ENOENT, '', str(e)
@@ -248,22 +323,27 @@ def snap_schedule_activate(self,
             return e.to_tuple()
         except Exception as e:
             return -errno.EIO, '', str(e)
-        return 0, 'Schedule activated for path {}'.format(path), ''
+        return 0, 'Schedule activated for path {}'.format(abs_path), ''
 
     @CLIWriteCommand('fs snap-schedule deactivate')
     def snap_schedule_deactivate(self,
                                  path: str,
                                  repeat: Optional[str] = None,
                                  start: Optional[str] = None,
-                                 fs: Optional[str] = None) -> Tuple[int, str, str]:
+                                 fs: Optional[str] = None,
+                                 subvol: Optional[str] = None,
+                                 group: Optional[str] = None) -> Tuple[int, str, str]:
         '''
         Deactivate a snapshot schedule for <path>
         '''
         rc, fs, err = self._validate_fs(fs)
         if rc < 0:
             return rc, fs, err
+        abs_path = ""
         try:
-            abs_path = path
+            rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+            if rc != 0:
+                return rc, '', errstr
             self.client.deactivate_snap_schedule(fs, abs_path, repeat, start)
         except ValueError as e:
             return -errno.ENOENT, '', str(e)
@@ -271,4 +351,4 @@ def snap_schedule_deactivate(self,
             return e.to_tuple()
         except Exception as e:
             return -errno.EIO, '', str(e)
-        return 0, 'Schedule deactivated for path {}'.format(path), ''
+        return 0, 'Schedule deactivated for path {}'.format(abs_path), ''

From 614d112cff522439c08e554a14472f1f391dd125 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Wed, 22 Nov 2023 15:20:17 +0530
Subject: [PATCH 1236/2492] mgr/volumes: ensure correct init of v1 subvol

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
index 88d186f3f150..90f35a4c90b3 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
@@ -98,7 +98,6 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
             create_base_dir(self.fs, self.group.path, self.vol_spec.DEFAULT_MODE)
             # create directory and set attributes
             self.fs.mkdirs(subvol_path, mode)
-            self.mark_subvolume()
             attrs = {
                 'uid': uid,
                 'gid': gid,
@@ -111,6 +110,7 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
             # persist subvolume metadata
             qpath = subvol_path.decode('utf-8')
             self.init_config(SubvolumeV1.VERSION, subvolume_type, qpath, initial_state)
+            self.mark_subvolume()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 log.info("cleaning up subvolume with path: {0}".format(self.subvolname))

From 0c5804ab4e5eb284d127044c5e51fbc870f5727a Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 7 Dec 2023 14:01:42 +0530
Subject: [PATCH 1237/2492] mgr/volumes: conditionalize subvolume upgrade

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 .../volumes/fs/operations/versions/__init__.py    | 15 +++++++++++++++
 .../fs/operations/versions/metadata_manager.py    |  1 +
 2 files changed, 16 insertions(+)

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/__init__.py b/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
index 6c96c4570199..097620d73780 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/__init__.py
@@ -49,6 +49,18 @@ def _get_subvolume_version(self, version):
     def get_subvolume_object_max(self, mgr, fs, vol_spec, group, subvolname):
         return self._get_subvolume_version(self.max_version)(mgr, fs, vol_spec, group, subvolname)
 
+    def allow_subvolume_upgrade(self, subvolume):
+        asu = True
+        try:
+            opt = subvolume.metadata_mgr.get_global_option(MetadataManager.GLOBAL_META_KEY_ALLOW_SUBVOLUME_UPGRADE)
+            asu = False if opt == "0" else True
+        except MetadataMgrException:
+            # this key is injected for QA testing and will not be available in
+            # production
+            pass
+
+        return asu
+
     def upgrade_to_v2_subvolume(self, subvolume):
         # legacy mode subvolumes cannot be upgraded to v2
         if subvolume.legacy_mode:
@@ -58,6 +70,9 @@ def upgrade_to_v2_subvolume(self, subvolume):
         if version >= SubvolumeV2.version():
             return
 
+        if not self.allow_subvolume_upgrade(subvolume):
+            return
+
         v1_subvolume = self._get_subvolume_version(version)(subvolume.mgr, subvolume.fs, subvolume.vol_spec, subvolume.group, subvolume.subvolname)
         try:
             v1_subvolume.open(SubvolumeOpType.SNAP_LIST)
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py b/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
index b33a2b48b775..610a61e6a4c1 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/metadata_manager.py
@@ -58,6 +58,7 @@ class MetadataManager(object):
     GLOBAL_META_KEY_TYPE    = "type"
     GLOBAL_META_KEY_PATH    = "path"
     GLOBAL_META_KEY_STATE   = "state"
+    GLOBAL_META_KEY_ALLOW_SUBVOLUME_UPGRADE   = "allow_subvolume_upgrade"
 
     CLONE_FAILURE_SECTION = "CLONE_FAILURE"
     CLONE_FAILURE_META_KEY_ERRNO = "errno"

From 006c68f492044327594fbf7204774d7d7d4ee518 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Mon, 20 Nov 2023 16:09:57 +0530
Subject: [PATCH 1238/2492] qa: add test cases for testing --subvol and --group
 arguments

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 qa/tasks/cephfs/test_snap_schedules.py | 481 +++++++++++++++++++++++--
 1 file changed, 459 insertions(+), 22 deletions(-)

diff --git a/qa/tasks/cephfs/test_snap_schedules.py b/qa/tasks/cephfs/test_snap_schedules.py
index 8f9132566da5..8bbd679efa30 100644
--- a/qa/tasks/cephfs/test_snap_schedules.py
+++ b/qa/tasks/cephfs/test_snap_schedules.py
@@ -3,6 +3,7 @@
 import time
 import errno
 import logging
+import uuid
 
 from tasks.cephfs.cephfs_test_case import CephFSTestCase
 from teuthology.exceptions import CommandFailedError
@@ -28,6 +29,29 @@ class TestSnapSchedulesHelper(CephFSTestCase):
     # this should be in sync with snap_schedule format
     SNAPSHOT_TS_FORMAT = '%Y-%m-%d-%H_%M_%S'
 
+    def remove_snapshots(self, dir_path, sdn):
+        snap_path = f'{dir_path}/{sdn}'
+
+        snapshots = self.mount_a.ls(path=snap_path)
+        for snapshot in snapshots:
+            if snapshot.startswith("_scheduled"):
+                continue
+            snapshot_path = os.path.join(snap_path, snapshot)
+            log.debug(f'removing snapshot: {snapshot_path}')
+            self.mount_a.run_shell(['sudo', 'rmdir', snapshot_path])
+
+    def get_snap_dir_name(self):
+        from .fuse_mount import FuseMount
+        from .kernel_mount import KernelMount
+
+        if isinstance(self.mount_a, KernelMount):
+            sdn = self.mount_a.client_config.get('snapdirname', '.snap')
+        elif isinstance(self.mount_a, FuseMount):
+            sdn = self.mount_a.client_config.get('client_snapdir', '.snap')
+            self.fs.set_ceph_conf('client', 'client snapdir', sdn)
+            self.mount_a.remount()
+        return sdn
+
     def check_scheduled_snapshot(self, exec_time, timo):
         now = time.time()
         delta = now - exec_time
@@ -170,7 +194,7 @@ def verify_schedule(self, dir_path, schedules, retentions=[]):
             self.assertTrue(schedule in json_res['schedule'])
         for retention in retentions:
             self.assertTrue(retention in json_res['retention'])
-    
+
 class TestSnapSchedules(TestSnapSchedulesHelper):
     def remove_snapshots(self, dir_path):
         snap_path = f'{dir_path}/.snap'
@@ -462,7 +486,7 @@ def test_snap_schedule_with_mgr_restart(self):
         # cleanup
         self.fs_snap_schedule_cmd('remove', path=testdir, snap_schedule='1m')
         self.remove_snapshots(testdir[1:])
-        self.mount_a.run_shell(['rmdir', testdir[1:]])    
+        self.mount_a.run_shell(['rmdir', testdir[1:]])
 
     def test_schedule_auto_deactivation_for_non_existent_path(self):
         """
@@ -577,28 +601,441 @@ def test_snap_schedule_all_periods(self):
         self.mount_a.run_shell(['rmdir', test_dir])
 
 
-class TestSnapSchedulesSnapdir(TestSnapSchedulesHelper):
-    def remove_snapshots(self, dir_path, sdn):
-        snap_path = f'{dir_path}/{sdn}'
+class TestSnapSchedulesSubvolAndGroupArguments(TestSnapSchedulesHelper):
+    def setUp(self):
+        super(TestSnapSchedulesSubvolAndGroupArguments, self).setUp()
+        self.CREATE_VERSION = int(self.mount_a.ctx['config']['overrides']['subvolume_version'])
+
+    def _create_v1_subvolume(self, subvol_name, subvol_group=None, has_snapshot=False, subvol_type='subvolume', state='complete'):
+        group = subvol_group if subvol_group is not None else '_nogroup'
+        basepath = os.path.join("volumes", group, subvol_name)
+        uuid_str = str(uuid.uuid4())
+        createpath = os.path.join(basepath, uuid_str)
+        self.mount_a.run_shell(['sudo', 'mkdir', '-p', createpath], omit_sudo=False)
+        self.mount_a.setfattr(createpath, 'ceph.dir.subvolume', '1', sudo=True)
+
+        # create a v1 snapshot, to prevent auto upgrades
+        if has_snapshot:
+            snappath = os.path.join(createpath, self.get_snap_dir_name(), "fake")
+            self.mount_a.run_shell(['sudo', 'mkdir', '-p', snappath], omit_sudo=False)
+
+        # add required xattrs to subvolume
+        default_pool = self.mount_a.getfattr(".", "ceph.dir.layout.pool")
+        self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool, sudo=True)
+
+        # create a v1 .meta file
+        cp = "/" + createpath
+        meta_contents = f"[GLOBAL]\nversion = 1\ntype = {subvol_type}\npath = {cp}\nstate = {state}\n"
+        meta_contents += "allow_subvolume_upgrade = 0\n"  # boolean
+        if state == 'pending':
+            # add a fake clone source
+            meta_contents = meta_contents + '[source]\nvolume = fake\nsubvolume = fake\nsnapshot = fake\n'
+        meta_filepath1 = os.path.join(self.mount_a.mountpoint, basepath, ".meta")
+        self.mount_a.client_remote.write_file(meta_filepath1, meta_contents, sudo=True)
+        return createpath
+
+    def _create_subvolume(self, version, subvol_name, subvol_group=None):
+        if version == 1:
+            self._create_v1_subvolume(subvol_name, subvol_group)
+        elif version >= 2:
+            if subvol_group:
+                self._fs_cmd('subvolume', 'create', 'cephfs', subvol_name, '--group_name', subvol_group)
+            else:
+                self._fs_cmd('subvolume', 'create', 'cephfs', subvol_name)
+        else:
+            self.assertTrue('NoSuchSubvolumeVersion' == None)
+
+    def _get_subvol_snapdir_path(self, version, subvol, group):
+        args = ['subvolume', 'getpath', 'cephfs', subvol]
+        if group:
+            args += ['--group_name', group]
+
+        path = self.get_ceph_cmd_stdout("fs", *args).rstrip()
+        if version >= 2:
+            path += "/.."
+        return path[1:]
+
+    def _verify_snap_schedule(self, version, subvol, group):
+        time.sleep(75)
+        path = self._get_subvol_snapdir_path(version, subvol, group)
+        path += "/" + self.get_snap_dir_name()
+        snaps = self.mount_a.ls(path=path)
+        log.debug(f"snaps:{snaps}")
+        count = 0
+        for snapname in snaps:
+            if snapname.startswith("scheduled-"):
+                count += 1
+        # confirm presence of snapshot dir under .snap dir
+        self.assertGreater(count, 0)
+
+    def test_snap_schedule_subvol_and_group_arguments_01(self):
+        """
+        Test subvol schedule creation succeeds for default subvolgroup.
+        """
+        self._create_subvolume(self.CREATE_VERSION, 'sv01')
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv01', path='.', snap_schedule='1m')
 
-        snapshots = self.mount_a.ls(path=snap_path)
-        for snapshot in snapshots:
-            snapshot_path = os.path.join(snap_path, snapshot)
-            log.debug(f'removing snapshot: {snapshot_path}')
-            self.mount_a.run_shell(['rmdir', snapshot_path])
+        self._verify_snap_schedule(self.CREATE_VERSION, 'sv01', None)
+        path = self._get_subvol_snapdir_path(self.CREATE_VERSION, 'sv01', None)
+        self.remove_snapshots(path, self.get_snap_dir_name())
 
-    def get_snap_dir_name(self):
-        from .fuse_mount import FuseMount
-        from .kernel_mount import KernelMount
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv01', path='.', snap_schedule='1m')
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv01')
 
-        if isinstance(self.mount_a, KernelMount):
-            sdn = self.mount_a.client_config.get('snapdirname', '.snap')
-        elif isinstance(self.mount_a, FuseMount):
-            sdn = self.mount_a.client_config.get('client_snapdir', '.snap')
-            self.fs.set_ceph_conf('client', 'client snapdir', sdn)
-            self.mount_a.remount()
-        return sdn
+    def test_snap_schedule_subvol_and_group_arguments_02(self):
+        """
+        Test subvol schedule creation fails for non-default subvolgroup.
+        """
+        self._create_subvolume(self.CREATE_VERSION, 'sv02')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('add', '--subvol', 'sv02', '--group', 'mygrp02', path='.', snap_schedule='1m')
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv02')
+
+    def test_snap_schedule_subvol_and_group_arguments_03(self):
+        """
+        Test subvol schedule creation fails when subvol exists only under default group.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp03')
+        self._create_subvolume(self.CREATE_VERSION, 'sv03', 'mygrp03')
+
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('add', '--subvol', 'sv03', path='.', snap_schedule='1m')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv03', '--group_name', 'mygrp03')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp03')
+
+    def test_snap_schedule_subvol_and_group_arguments_04(self):
+        """
+        Test subvol schedule creation fails without subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp04')
+        self._create_subvolume(self.CREATE_VERSION, 'sv04', 'mygrp04')
+
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('add', '--group', 'mygrp04', path='.', snap_schedule='1m')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv04', '--group_name', 'mygrp04')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp04')
+
+    def test_snap_schedule_subvol_and_group_arguments_05(self):
+        """
+        Test subvol schedule creation succeeds for a subvol under a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp05')
+        self._create_subvolume(self.CREATE_VERSION, 'sv05', 'mygrp05')
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv05', '--group', 'mygrp05', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._verify_snap_schedule(self.CREATE_VERSION, 'sv05', 'mygrp05')
+        path = self._get_subvol_snapdir_path(self.CREATE_VERSION, 'sv05', 'mygrp05')
+        self.remove_snapshots(path, self.get_snap_dir_name())
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv05', '--group_name', 'mygrp05')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp05')
+
+    def test_snap_schedule_subvol_and_group_arguments_06(self):
+        """
+        Test subvol schedule listing fails without a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp06')
+        self._create_subvolume(self.CREATE_VERSION, 'sv06', 'mygrp06')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv06', '--group', 'mygrp06', path='.', snap_schedule='1m', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('list', '--subvol', 'sv06', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv06', '--group', 'mygrp06', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv06', '--group_name', 'mygrp06')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp06')
+
+    def test_snap_schedule_subvol_and_group_arguments_07(self):
+        """
+        Test subvol schedule listing fails without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp07')
+        self._create_subvolume(self.CREATE_VERSION, 'sv07', 'mygrp07')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv07', '--group', 'mygrp07', path='.', snap_schedule='1m', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('list', '--group', 'mygrp07', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv07', '--group', 'mygrp07', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv07', '--group_name', 'mygrp07')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp07')
+
+    def test_snap_schedule_subvol_and_group_arguments_08(self):
+        """
+        Test subvol schedule listing succeeds with a subvol and a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp08')
+        self._create_subvolume(self.CREATE_VERSION, 'sv08', 'mygrp08')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv08', '--group', 'mygrp08', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('list', '--subvol', 'sv08', '--group', 'mygrp08', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv08', '--group', 'mygrp08', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv08', '--group_name', 'mygrp08')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp08')
+
+    def test_snap_schedule_subvol_and_group_arguments_09(self):
+        """
+        Test subvol schedule retention add fails for a subvol without a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp09')
+        self._create_subvolume(self.CREATE_VERSION, 'sv09', 'mygrp09')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv09', '--group', 'mygrp09', path='.', snap_schedule='1m', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv09', path='.', retention_spec_or_period='h', retention_count='5')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv09', '--group', 'mygrp09', path='.', snap_schedule='1m', fs='cephfs')
 
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv09', '--group_name', 'mygrp09')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp09')
+
+    def test_snap_schedule_subvol_and_group_arguments_10(self):
+        """
+        Test subvol schedule retention add fails for a subvol without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp10')
+        self._create_subvolume(self.CREATE_VERSION, 'sv10', 'mygrp10')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv10', '--group', 'mygrp10', path='.', snap_schedule='1m', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('retention', 'add', '--group', 'mygrp10', path='.', retention_spec_or_period='h', retention_count='5')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv10', '--group', 'mygrp10', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv10', '--group_name', 'mygrp10')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp10')
+
+    def test_snap_schedule_subvol_and_group_arguments_11(self):
+        """
+        Test subvol schedule retention add succeeds for a subvol within a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp11')
+        self._create_subvolume(self.CREATE_VERSION, 'sv11', 'mygrp11')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv11', '--group', 'mygrp11', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv11', '--group', 'mygrp11', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv11', '--group', 'mygrp11', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv11', '--group_name', 'mygrp11')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp11')
+
+    def test_snap_schedule_subvol_and_group_arguments_12(self):
+        """
+        Test subvol schedule activation fails for a subvol without a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp12')
+        self._create_subvolume(self.CREATE_VERSION, 'sv12', 'mygrp12')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv12', '--group', 'mygrp12', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv12', '--group', 'mygrp12', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('activate', '--subvol', 'sv12', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv12', '--group', 'mygrp12', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv12', '--group_name', 'mygrp12')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp12')
+
+    def test_snap_schedule_subvol_and_group_arguments_13(self):
+        """
+        Test subvol schedule activation fails for a subvol without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp13')
+        self._create_subvolume(self.CREATE_VERSION, 'sv13', 'mygrp13')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv13', '--group', 'mygrp13', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv13', '--group', 'mygrp13', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('activate', '--group', 'mygrp13', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv13', '--group', 'mygrp13', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv13', '--group_name', 'mygrp13')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp13')
+
+    def test_snap_schedule_subvol_and_group_arguments_14(self):
+        """
+        Test subvol schedule activation succeeds for a subvol within a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp14')
+        self._create_subvolume(self.CREATE_VERSION, 'sv14', 'mygrp14')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv14', '--group', 'mygrp14', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv14', '--group', 'mygrp14', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv14', '--group', 'mygrp14', path='.', fs='cephfs')
+
+        self._verify_snap_schedule(self.CREATE_VERSION, 'sv14', 'mygrp14')
+        path = self._get_subvol_snapdir_path(self.CREATE_VERSION, 'sv14', 'mygrp14')
+        self.remove_snapshots(path, self.get_snap_dir_name())
+
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv14', '--group', 'mygrp14', path='.', snap_schedule='1m', fs='cephfs')
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv14', '--group_name', 'mygrp14')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp14')
+
+    def test_snap_schedule_subvol_and_group_arguments_15(self):
+        """
+        Test subvol schedule deactivation fails for a subvol without a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp15')
+        self._create_subvolume(self.CREATE_VERSION, 'sv15', 'mygrp15')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv15', '--group', 'mygrp15', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv15', '--group', 'mygrp15', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv15', '--group', 'mygrp15', path='.', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv15', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv15', '--group', 'mygrp15', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv15', '--group_name', 'mygrp15')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp15')
+
+    def test_snap_schedule_subvol_and_group_arguments_16(self):
+        """
+        Test subvol schedule deactivation fails for a subvol without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp16')
+        self._create_subvolume(self.CREATE_VERSION, 'sv16', 'mygrp16')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv16', '--group', 'mygrp16', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv16', '--group', 'mygrp16', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv16', '--group', 'mygrp16', path='.', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('deactivate', '--group', 'mygrp16', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv16', '--group', 'mygrp16', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv16', '--group_name', 'mygrp16')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp16')
+
+    def test_snap_schedule_subvol_and_group_arguments_17(self):
+        """
+        Test subvol schedule deactivation succeeds for a subvol within a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp17')
+        self._create_subvolume(self.CREATE_VERSION, 'sv17', 'mygrp17')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv17', '--group', 'mygrp17', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv17', '--group', 'mygrp17', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv17', '--group', 'mygrp17', path='.', fs='cephfs')
+
+        self._verify_snap_schedule(self.CREATE_VERSION, 'sv17', 'mygrp17')
+        path = self._get_subvol_snapdir_path(self.CREATE_VERSION, 'sv17', 'mygrp17')
+        self.remove_snapshots(path, self.get_snap_dir_name())
+
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv17', '--group', 'mygrp17', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv17', '--group', 'mygrp17', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv17', '--group_name', 'mygrp17')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp17')
+
+    def test_snap_schedule_subvol_and_group_arguments_18(self):
+        """
+        Test subvol schedule retention remove fails for a subvol without a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp18')
+        self._create_subvolume(self.CREATE_VERSION, 'sv18', 'mygrp18')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv18', '--group', 'mygrp18', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv18', '--group', 'mygrp18', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv18', '--group', 'mygrp18', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv18', '--group', 'mygrp18', path='.', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('retention', 'remove', '--subvol', 'sv18', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv18', '--group', 'mygrp18', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv18', '--group_name', 'mygrp18')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp18')
+
+    def test_snap_schedule_subvol_and_group_arguments_19(self):
+        """
+        Test subvol schedule retention remove fails for a subvol without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp19')
+        self._create_subvolume(self.CREATE_VERSION, 'sv19', 'mygrp19')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv19', '--group', 'mygrp19', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv19', '--group', 'mygrp19', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv19', '--group', 'mygrp19', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv19', '--group', 'mygrp19', path='.', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('retention', 'remove', '--group', 'mygrp19', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv19', '--group', 'mygrp19', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv19', '--group_name', 'mygrp19')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp19')
+
+    def test_snap_schedule_subvol_and_group_arguments_20(self):
+        """
+        Test subvol schedule retention remove succeeds for a subvol within a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp20')
+        self._create_subvolume(self.CREATE_VERSION, 'sv20', 'mygrp20')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv20', '--group', 'mygrp20', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv20', '--group', 'mygrp20', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv20', '--group', 'mygrp20', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv20', '--group', 'mygrp20', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'remove', '--subvol', 'sv20', '--group', 'mygrp20', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv20', '--group', 'mygrp20', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv20', '--group_name', 'mygrp20')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp20')
+
+    def test_snap_schedule_subvol_and_group_arguments_21(self):
+        """
+        Test subvol schedule remove fails for a subvol without a subvolgroup argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp21')
+        self._create_subvolume(self.CREATE_VERSION, 'sv21', 'mygrp21')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv21', '--group', 'mygrp21', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv21', '--group', 'mygrp21', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv21', '--group', 'mygrp21', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv21', '--group', 'mygrp21', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'remove', '--subvol', 'sv21', '--group', 'mygrp21', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('remove', '--subvol', 'sv21', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv21', '--group', 'mygrp21', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv21', '--group_name', 'mygrp21')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp21')
+
+    def test_snap_schedule_subvol_and_group_arguments_22(self):
+        """
+        Test subvol schedule remove fails for a subvol without a subvol argument.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp22')
+        self._create_subvolume(self.CREATE_VERSION, 'sv22', 'mygrp22')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv22', '--group', 'mygrp22', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv22', '--group', 'mygrp22', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv22', '--group', 'mygrp22', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv22', '--group', 'mygrp22', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'remove', '--subvol', 'sv22', '--group', 'mygrp22', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        with self.assertRaises(CommandFailedError):
+            self.fs_snap_schedule_cmd('remove', '--group', 'mygrp22', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv22', '--group', 'mygrp22', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv22', '--group_name', 'mygrp22')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp22')
+
+    def test_snap_schedule_subvol_and_group_arguments_23(self):
+        """
+        Test subvol schedule remove succeeds for a subvol within a subvolgroup.
+        """
+        self._fs_cmd('subvolumegroup', 'create', 'cephfs', 'mygrp23')
+        self._create_subvolume(self.CREATE_VERSION, 'sv23', 'mygrp23')
+
+        self.fs_snap_schedule_cmd('add', '--subvol', 'sv23', '--group', 'mygrp23', path='.', snap_schedule='1m', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'add', '--subvol', 'sv23', '--group', 'mygrp23', path='.', retention_spec_or_period='h', retention_count=5, fs='cephfs')
+        self.fs_snap_schedule_cmd('activate', '--subvol', 'sv23', '--group', 'mygrp23', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('deactivate', '--subvol', 'sv23', '--group', 'mygrp23', path='.', fs='cephfs')
+        self.fs_snap_schedule_cmd('retention', 'remove', '--subvol', 'sv23', '--group', 'mygrp23', path='.', retention_spec_or_period='h', retention_count='5', fs='cephfs')
+        self.fs_snap_schedule_cmd('remove', '--subvol', 'sv23', '--group', 'mygrp23', path='.', snap_schedule='1m', fs='cephfs')
+
+        self._fs_cmd('subvolume', 'rm', 'cephfs', 'sv23', '--group_name', 'mygrp23')
+        self._fs_cmd('subvolumegroup', 'rm', 'cephfs', 'mygrp23')
+
+
+class TestSnapSchedulesSnapdir(TestSnapSchedulesHelper):
     def test_snap_dir_name(self):
         """Test the correctness of snap directory name"""
         self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedulesSnapdir.TEST_DIRECTORY])
@@ -611,10 +1048,10 @@ def test_snap_dir_name(self):
         timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1m')
         sdn = self.get_snap_dir_name()
         log.info(f'expecting snap {TestSnapSchedulesSnapdir.TEST_DIRECTORY}/{sdn}/scheduled-{snap_sfx} in ~{timo}s...')
-        
+
         # verify snapshot schedule
         self.verify_schedule(TestSnapSchedulesSnapdir.TEST_DIRECTORY, ['1m'], retentions=[{'m':1}])
-        
+
         # remove snapshot schedule
         self.fs_snap_schedule_cmd('remove', path=TestSnapSchedulesSnapdir.TEST_DIRECTORY)
 

From 2533ac5b90f367dd55f61c1e95dee0e154940930 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 14 Dec 2023 16:31:05 +0530
Subject: [PATCH 1239/2492] qa: add yaml for on demand subvol version testing

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 .../overrides/subvol_versions/create_subvol_version_v1.yaml     | 2 ++
 .../overrides/subvol_versions/create_subvol_version_v2.yaml     | 2 ++
 qa/suites/fs/functional/subvol_versions/.qa                     | 1 +
 .../fs/functional/subvol_versions/create_subvol_version_v1.yaml | 1 +
 .../fs/functional/subvol_versions/create_subvol_version_v2.yaml | 1 +
 5 files changed, 7 insertions(+)
 create mode 100644 qa/cephfs/overrides/subvol_versions/create_subvol_version_v1.yaml
 create mode 100644 qa/cephfs/overrides/subvol_versions/create_subvol_version_v2.yaml
 create mode 120000 qa/suites/fs/functional/subvol_versions/.qa
 create mode 120000 qa/suites/fs/functional/subvol_versions/create_subvol_version_v1.yaml
 create mode 120000 qa/suites/fs/functional/subvol_versions/create_subvol_version_v2.yaml

diff --git a/qa/cephfs/overrides/subvol_versions/create_subvol_version_v1.yaml b/qa/cephfs/overrides/subvol_versions/create_subvol_version_v1.yaml
new file mode 100644
index 000000000000..120b2bf04bee
--- /dev/null
+++ b/qa/cephfs/overrides/subvol_versions/create_subvol_version_v1.yaml
@@ -0,0 +1,2 @@
+overrides:
+  subvolume_version: 1
diff --git a/qa/cephfs/overrides/subvol_versions/create_subvol_version_v2.yaml b/qa/cephfs/overrides/subvol_versions/create_subvol_version_v2.yaml
new file mode 100644
index 000000000000..c8bcf95c0567
--- /dev/null
+++ b/qa/cephfs/overrides/subvol_versions/create_subvol_version_v2.yaml
@@ -0,0 +1,2 @@
+overrides:
+  subvolume_version: 2
diff --git a/qa/suites/fs/functional/subvol_versions/.qa b/qa/suites/fs/functional/subvol_versions/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/fs/functional/subvol_versions/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/fs/functional/subvol_versions/create_subvol_version_v1.yaml b/qa/suites/fs/functional/subvol_versions/create_subvol_version_v1.yaml
new file mode 120000
index 000000000000..09cfdb59edae
--- /dev/null
+++ b/qa/suites/fs/functional/subvol_versions/create_subvol_version_v1.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/subvol_versions/create_subvol_version_v1.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/functional/subvol_versions/create_subvol_version_v2.yaml b/qa/suites/fs/functional/subvol_versions/create_subvol_version_v2.yaml
new file mode 120000
index 000000000000..5a4de14e7e00
--- /dev/null
+++ b/qa/suites/fs/functional/subvol_versions/create_subvol_version_v2.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/subvol_versions/create_subvol_version_v2.yaml
\ No newline at end of file

From dfc35a4679495a3d7136927374e838409c29ab4a Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Tue, 17 Oct 2023 10:06:48 +0530
Subject: [PATCH 1240/2492] snap_schedule/tests: fix db upgrade issue

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 .../mgr/snap_schedule/tests/conftest.py       | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/src/pybind/mgr/snap_schedule/tests/conftest.py b/src/pybind/mgr/snap_schedule/tests/conftest.py
index 35255b8d486b..92d4c9bc132b 100644
--- a/src/pybind/mgr/snap_schedule/tests/conftest.py
+++ b/src/pybind/mgr/snap_schedule/tests/conftest.py
@@ -31,4 +31,41 @@ def db():
         db.row_factory = sqlite3.Row
         db.execute("PRAGMA FOREIGN_KEYS = 1")
         db.executescript(Schedule.CREATE_TABLES)
+        _create_snap_schedule_kv_db(db)
+        _upgrade_snap_schedule_db_schema(db)
     return db
+
+def _create_snap_schedule_kv_db(db):
+    SQL = """
+    CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV (
+      key TEXT PRIMARY KEY,
+      value NOT NULL
+    ) WITHOUT ROWID;
+    INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1);
+    """
+    db.executescript(SQL)
+
+def _get_snap_schedule_db_version(db):
+    SQL = """
+    SELECT value
+    FROM SnapScheduleModuleKV
+    WHERE key = '__snap_schedule_db_version';
+    """
+    cur = db.execute(SQL)
+    row = cur.fetchone()
+    assert row is not None
+    return int(row[0])
+
+# add all upgrades here
+def _upgrade_snap_schedule_db_schema(db):
+    # add a column to hold the subvolume group name
+    if _get_snap_schedule_db_version(db) < 2:
+        SQL = """
+        ALTER TABLE schedules
+        ADD COLUMN group_name TEXT;
+        """
+        db.executescript(SQL)
+
+        # bump up the snap-schedule db version to 2
+        SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';"
+        db.execute(SQL, (2,))

From b3ff5f794985c173ba87926f34e6d9793c5d5c67 Mon Sep 17 00:00:00 2001
From: Milind Changire <mchangir@redhat.com>
Date: Thu, 14 Dec 2023 12:55:08 +0530
Subject: [PATCH 1241/2492] PendingReleaseNotes: support for subvolumes and
 subvolume groups in snap_schedule

Signed-off-by: Milind Changire <mchangir@redhat.com>
---
 PendingReleaseNotes | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 4038993d8e47..b3fa9b9990e2 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -103,6 +103,8 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   module is now available. Users may run `ceph balancer status detail` to see more
   details about which PGs were updated in the balancer's last optimization.
   See https://docs.ceph.com/en/latest/rados/operations/balancer/ for more information.
+* CephFS: Full support for subvolumes and subvolume groups is now available
+  for snap_schedule Manager module.
 
 >=18.0.0
 

From f9d92f1817ed438209dc3915d6ed09389e51b03f Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 25 Dec 2023 12:57:26 -0600
Subject: [PATCH 1242/2492] osd/scrub: remove scrub_clear_state()

Its functionality is now handled by the FSM when existing
'Session' or 'ActiveReplicaOp' states.
For a PG that is not active - no need to do anything.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PrimaryLogPG.cc                |  3 +--
 src/osd/scrubber/pg_scrubber.cc        | 22 +---------------------
 src/osd/scrubber/pg_scrubber.h         | 13 ++++++-------
 src/osd/scrubber/scrub_machine_lstnr.h | 11 +++++++++--
 src/osd/scrubber_common.h              | 11 +++++++----
 5 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index d57ab432d4ac..4ecbe58a1f38 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -13027,8 +13027,7 @@ void PrimaryLogPG::on_change(ObjectStore::Transaction &t)
     finish_degraded_object(p->first);
   }
 
-  // requeues waiting_for_scrub
-  m_scrubber->scrub_clear_state();
+  ceph_assert(waiting_for_scrub.empty());
 
   for (auto p = waiting_for_blocked_object.begin();
        p != waiting_for_blocked_object.end();
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index f7e027a84f07..2dae53273a70 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -2212,32 +2212,12 @@ PerfCounters& PgScrubber::get_counters_set() const
 void PgScrubber::cleanup_on_finish()
 {
   dout(10) << __func__ << dendl;
-  ceph_assert(m_pg->is_locked());
-
-  state_clear(PG_STATE_SCRUBBING);
-  state_clear(PG_STATE_DEEP_SCRUB);
-
-  m_local_osd_resource.reset();
-  requeue_waiting();
-
-  reset_internal_state();
-  m_flags = scrub_flags_t{};
+  clear_pgscrub_state();
 
-  // type-specific state clear
-  _scrub_clear_state();
   // PG state flags changed:
   m_pg->publish_stats_to_osd();
 }
 
-// uses process_event(), so must be invoked externally
-void PgScrubber::scrub_clear_state()
-{
-  dout(10) << __func__ << dendl;
-
-  clear_pgscrub_state();
-  m_fsm->process_event(FullReset{});
-}
-
 /*
  * note: does not access the state-machine
  */
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index adf50f97d92a..5911b67263f4 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -318,8 +318,6 @@ class PgScrubber : public ScrubPgIF,
 
   void on_replica_activate() final;
 
-  void scrub_clear_state() final;
-
   bool is_queued_or_active() const final;
 
   /**
@@ -421,11 +419,8 @@ class PgScrubber : public ScrubPgIF,
   void on_replica_init() final;
   void replica_handling_done() final;
 
-  /// the version of 'scrub_clear_state()' that does not try to invoke FSM
-  /// services (thus can be called from FSM reactions)
   void clear_pgscrub_state() final;
 
-
   std::chrono::milliseconds get_scrub_sleep_time() const final;
   void queue_for_scrub_resched(Scrub::scrub_prio_t prio) final;
 
@@ -622,8 +617,12 @@ class PgScrubber : public ScrubPgIF,
    */
   std::unique_ptr<Scrub::LocalResourceWrapper> m_local_osd_resource;
 
-  void cleanup_on_finish();  // scrub_clear_state() as called for a Primary when
-			     // Active->NotActive
+  /**
+   * clearing the scrubber state & the PG's scrub-related flags
+   * (calls clear_pgscrub_state()).
+   * Also - publishes the PG stats.
+   */
+  void cleanup_on_finish();
 
  protected:
   PG* const m_pg;
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index f2ddd9c87168..7849d14e4e41 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -135,8 +135,15 @@ struct ScrubMachineListener {
 
   virtual void replica_handling_done() = 0;
 
-  /// the version of 'scrub_clear_state()' that does not try to invoke FSM
-  /// services (thus can be called from FSM reactions)
+  /**
+   * clears both internal scrub state, and some PG-visible flags:
+   * - the two scrubbing PG state flags;
+   * - primary/replica scrub position (chunk boundaries);
+   * - primary/replica interaction state;
+   * - the backend state;
+   * Also runs pending callbacks, and clears the active flags.
+   * Does not try to invoke FSM events.
+   */
   virtual void clear_pgscrub_state() = 0;
 
   /// Get time to sleep before next scrub
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index b41a9b4bba84..5323b14677b0 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -326,8 +326,6 @@ struct ScrubPgIF {
   /// we are peered as a replica
   virtual void on_replica_activate() = 0;
 
-  virtual void scrub_clear_state() = 0;
-
   virtual void handle_query_state(ceph::Formatter* f) = 0;
 
   virtual pg_scrubbing_status_t get_schedule() const = 0;
@@ -382,8 +380,13 @@ struct ScrubPgIF {
 					const hobject_t& soid) = 0;
 
   /**
-   * the version of 'scrub_clear_state()' that does not try to invoke FSM
-   * services (thus can be called from FSM reactions)
+   * clears both internal scrub state, and some PG-visible flags:
+   * - the two scrubbing PG state flags;
+   * - primary/replica scrub position (chunk boundaries);
+   * - primary/replica interaction state;
+   * - the backend state
+   * Also runs pending callbacks, and clears the active flags.
+   * Does not try to invoke FSM events.
    */
   virtual void clear_pgscrub_state() = 0;
 

From 50a4cd3a18ce510f25908531d6228e7447f5e72c Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Wed, 20 Dec 2023 20:12:45 -0500
Subject: [PATCH 1243/2492] orchestrator: Add summary line to orch device ls

This patch just adds a summary line to the plain
text output of orch device ls when the --summary
switch is given. This helps to quickly understand your
device countswhen managing hosts with many devices.

Fixes: https://tracker.ceph.com/issues/63864

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 src/pybind/mgr/orchestrator/module.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index d6f36e81b718..309035c3e937 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -635,7 +635,8 @@ def _list_devices(self,
                       hostname: Optional[List[str]] = None,
                       format: Format = Format.plain,
                       refresh: bool = False,
-                      wide: bool = False) -> HandleCommandResult:
+                      wide: bool = False,
+                      summary: bool = False) -> HandleCommandResult:
         """
         List devices on a host
         """
@@ -682,9 +683,23 @@ def _list_devices(self,
             table.left_padding_width = 0
             table.right_padding_width = 2
             now = datetime_now()
+            host_count = 0
+            available_count = 0
+            device_count = {
+                "hdd": 0,
+                "ssd": 0}
+
             for host_ in natsorted(inv_hosts, key=lambda h: h.name):  # type: InventoryHost
+                host_count += 1
                 for d in sorted(host_.devices.devices, key=lambda d: d.path):  # type: Device
 
+                    if d.available:
+                        available_count += 1
+                    try:
+                        device_count[d.human_readable_type] += 1
+                    except KeyError:
+                        device_count[d.human_readable_type] = 1
+
                     led_ident = 'N/A'
                     led_fail = 'N/A'
                     if d.lsm_data.get('ledSupport', None):
@@ -723,6 +738,11 @@ def _list_devices(self,
                             )
                         )
             out.append(table.get_string())
+
+            if summary:
+                device_summary = [f"{device_count[devtype]} {devtype.upper()}" for devtype in sorted(device_count.keys())]
+                out.append(f"{host_count} host(s), {', '.join(device_summary)}, {available_count} available")
+
             return HandleCommandResult(stdout='\n'.join(out))
 
     @_cli_write_command('orch device zap')

From 594d1e5e83e3d4c318ab1dd79660670a0852f7e6 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Mon, 8 Jan 2024 08:44:51 +0000
Subject: [PATCH 1244/2492] win32_deps_build.sh: change Boost URL

The Boost download URL is broken again [1], we'll switch to a
working mirror.

[1] https://github.com/boostorg/boost/issues/842

Fixes: https://tracker.ceph.com/issues/63952

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 win32_deps_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index c61cd7cb78a8..1434d8b7ca66 100755
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -16,7 +16,7 @@ sslDir="${depsToolsetDir}/openssl"
 sslSrcDir="${depsSrcDir}/openssl"
 
 # For now, we'll keep the version number within the file path when not using git.
-boostUrl="https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+boostUrl="https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz"
 boostSha256Sum="66a469b6e608a51f8347236f4912e27dc5c60c60d7d53ae9bfe4683316c6f04c"
 boostSrcDir="${depsSrcDir}/boost_1_82_0"
 boostDir="${depsToolsetDir}/boost"

From 22fce3c7a55591f6755355751157bc7231776137 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 7 Jan 2024 22:10:59 +1000
Subject: [PATCH 1245/2492] doc/radosgw: edit admin.rst - quota

Edit the "Quota Management" section (all of it) in
doc/radosgw/admin.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 169 ++++++++++++++++++++++++++----------------
 1 file changed, 104 insertions(+), 65 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 6eb36452edea..a06b7a1eb3c8 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -395,9 +395,9 @@ following form:
 Quota Management
 ================
 
-The Ceph Object Gateway enables you to set quotas on users and buckets owned by
-users. Quotas include the maximum number of objects in a bucket and the maximum
-storage size a bucket can hold.
+The Ceph Object Gateway makes it possible for you to set quotas on users and
+buckets owned by users. Quotas include the maximum number of objects in a
+bucket and the maximum storage size a bucket can hold.
 
 - **Bucket:** The ``--bucket`` option allows you to specify a quota for
   buckets the user owns.
@@ -406,149 +406,188 @@ storage size a bucket can hold.
   the maximum number of objects. A negative value disables this setting.
   
 - **Maximum Size:** The ``--max-size`` option allows you to specify a quota
-  size in B/K/M/G/T, where B is the default. A negative value disables this setting.
+  size in B/K/M/G/T, where B is the default. A negative value disables this
+  setting.
   
 - **Quota Scope:** The ``--quota-scope`` option sets the scope for the quota.
-  The options are ``bucket`` and ``user``. Bucket quotas apply to buckets a 
-  user owns. User quotas apply to a user.
+  The options are ``bucket`` and ``user``. Bucket quotas apply to each bucket
+  owned by the user. User Quotas are summed across all buckets owned by the
+  user. 
 
 
 Set User Quota
 --------------
 
 Before you enable a quota, you must first set the quota parameters.
-For example:: 
+To set quota parameters, run a command of the following form: 
 
-	radosgw-admin quota set --quota-scope=user --uid=<uid> [--max-objects=<num objects>] [--max-size=<max size>]
+.. prompt:: bash
 
-For example:: 
+   radosgw-admin quota set --quota-scope=user --uid=<uid> [--max-objects=<num objects>] [--max-size=<max size>]
 
-	radosgw-admin quota set --quota-scope=user --uid=johndoe --max-objects=1024 --max-size=1024B
+For example:
 
+.. prompt:: bash
 
-A negative value for num objects and / or max size means that the
-specific quota attribute check is disabled.
+   radosgw-admin quota set --quota-scope=user --uid=johndoe --max-objects=1024 --max-size=1024B
 
+Passing a negative value as an argument of ``--max-objects`` or ``--max-size``
+disables the given quota attribute.  
 
-Enable/Disable User Quota
--------------------------
 
-Once you set a user quota, you may enable it. For example:: 
+Enabling and Disabling User Quota
+---------------------------------
 
-	radosgw-admin quota enable --quota-scope=user --uid=<uid>
+After a user quota is set, it must be enabled in order to take effect. To enable a user quota, run a command of the following form: 
 
-You may disable an enabled user quota. For example:: 
+.. prompt:: bash
+
+   radosgw-admin quota enable --quota-scope=user --uid=<uid>
 
-	radosgw-admin quota disable --quota-scope=user --uid=<uid>
+To disable an enabled user quota, run a command of the following form: 
+
+.. prompt:: bash
+
+   radosgw-admin quota disable --quota-scope=user --uid=<uid>
 
 
 Set Bucket Quota
 ----------------
 
 Bucket quotas apply to the buckets owned by the specified ``uid``. They are
-independent of the user. ::
+independent of the user. To set a bucket quota, run a command of the following
+form:
+
+.. prompt:: bash
 
-	radosgw-admin quota set --uid=<uid> --quota-scope=bucket [--max-objects=<num objects>] [--max-size=<max size]
+   radosgw-admin quota set --uid=<uid> --quota-scope=bucket [--max-objects=<num objects>] [--max-size=<max size]
 
-A negative value for num objects and / or max size means that the
-specific quota attribute check is disabled.
+A negative value for ``--max-objects`` or ``--max-size`` means that the
+specific quota attribute is disabled.
 
 
-Enable/Disable Bucket Quota
----------------------------
+Enable and Disabling Bucket Quota
+---------------------------------
 
-Once you set a bucket quota, you may enable it. For example:: 
+After a bucket quota has been set, it must be enabled in order to take effect.
+To enable a bucket quota, run a command of the following form:
 
-	radosgw-admin quota enable --quota-scope=bucket --uid=<uid>
+.. prompt:: bash
+
+   radosgw-admin quota enable --quota-scope=bucket --uid=<uid>
 
-You may disable an enabled bucket quota. For example:: 
+To disable an enabled bucket quota, run a command of the following form: 
+
+.. prompt:: bash
 
-	radosgw-admin quota disable --quota-scope=bucket --uid=<uid>
+   radosgw-admin quota disable --quota-scope=bucket --uid=<uid>
 
 
 Get Quota Settings
 ------------------
 
-You may access each user's quota settings via the user information
+You can access each user's quota settings via the user information
 API. To read user quota setting information with the CLI interface, 
-execute the following::
+run a command of the following form:
 
-	radosgw-admin user info --uid=<uid>
+.. prompt:: bash
+
+   radosgw-admin user info --uid=<uid>
 
 
 Update Quota Stats
 ------------------
 
-Quota stats get updated asynchronously. You can update quota
-statistics for all users and all buckets manually to retrieve
-the latest quota stats. ::
+Quota stats are updated asynchronously. You can update quota statistics for all
+users and all buckets manually to force an update of the latest quota stats. To
+update quota statistics for all users and all buckets in order to retrieve the
+latest quota statistics, run a command of the following form:
 
-	radosgw-admin user stats --uid=<uid> --sync-stats
+.. prompt:: bash
+
+   radosgw-admin user stats --uid=<uid> --sync-stats
 
 .. _rgw_user_usage_stats:
 
 Get User Usage Stats
 --------------------
 
-To see how much of the quota a user has consumed, execute the following::
+To see how much of a quota a user has consumed, run a command of the following
+form: 
 
-	radosgw-admin user stats --uid=<uid>
+.. prompt:: bash
+
+   radosgw-admin user stats --uid=<uid>
 
-.. note:: You should execute ``radosgw-admin user stats`` with the 
-   ``--sync-stats`` option to receive the latest data.
+.. note:: Run ``radosgw-admin user stats`` with the ``--sync-stats`` option to
+   receive the latest data.
 
 Default Quotas
 --------------
 
-You can set default quotas in the config.  These defaults are used when
-creating a new user and have no effect on existing users. If the
-relevant default quota is set in config, then that quota is set on the
-new user, and that quota is enabled.  See ``rgw_bucket_default_quota_max_objects``,
-``rgw_bucket_default_quota_max_size``, ``rgw_user_default_quota_max_objects``, and
-``rgw_user_default_quota_max_size`` in `Ceph Object Gateway Config Reference`_
+You can set default quotas in the Ceph Object Gateway config. **These defaults
+will be used only when creating new users and will have no effect on existing
+users.** If a default quota is set in the Ceph Object Gateway Config, then that
+quota is set for all subsequently-created users, and that quota is enabled. See
+``rgw_bucket_default_quota_max_objects``,
+``rgw_bucket_default_quota_max_size``, ``rgw_user_default_quota_max_objects``,
+and ``rgw_user_default_quota_max_size`` in `Ceph Object Gateway Config
+Reference`_
 
 Quota Cache
 -----------
 
-Quota statistics are cached on each RGW instance.  If there are multiple
-instances, then the cache can keep quotas from being perfectly enforced, as
-each instance will have a different view of quotas.  The options that control
-this are ``rgw_bucket_quota_ttl``, ``rgw_user_quota_bucket_sync_interval`` and
-``rgw_user_quota_sync_interval``.  The higher these values are, the more
-efficient quota operations are, but the more out-of-sync multiple instances
-will be.  The lower these values are, the closer to perfect enforcement
-multiple instances will achieve.  If all three are 0, then quota caching is
-effectively disabled, and multiple instances will have perfect quota
-enforcement.  See `Ceph Object Gateway Config Reference`_
+Quota statistics are cached by each RGW instance. If multiple RGW instances are
+deployed, then this cache may prevent quotas from being perfectly enforced,
+because each instance may have a different set of quota settings.  
+
+The options that control this behavior are 
+
+#. ``rgw_bucket_quota_ttl``
+#. ``rgw_user_quota_bucket_sync_interval`` and 
+#. ``rgw_user_quota_sync_interval``.
+
+Increasing these values will make quota operations more efficient at the cost
+of increasing the likelihood that the multiple RGW instances may not
+consistently have the latest quota settings. Decreasing these values brings
+the multiple RGW instances closer to perfect quota synchronization. 
+
+If all three values are set to ``0`` , then quota caching is effectively
+disabled, and multiple instances will have perfect quota enforcement.  See
+`Ceph Object Gateway Config Reference`_.
 
 Reading / Writing Global Quotas
 -------------------------------
 
 You can read and write global quota settings in the period configuration. To
-view the global quota settings::
+view the global quota settings, run the following command:
+
+.. prompt:: bash
 
-	radosgw-admin global quota get
+   radosgw-admin global quota get
 
-The global quota settings can be manipulated with the ``global quota``
+Global quota settings can be manipulated with the ``global quota``
 counterparts of the ``quota set``, ``quota enable``, and ``quota disable``
-commands. ::
+commands, as in the following examples:  
+
+.. prompt:: bash
 
 	radosgw-admin global quota set --quota-scope bucket --max-objects 1024
 	radosgw-admin global quota enable --quota-scope bucket
 
-.. note:: In a multisite configuration, where there is a realm and period
+.. note:: In a multisite configuration where there is a realm and period
    present, changes to the global quotas must be committed using ``period
-   update --commit``. If there is no period present, the rados gateway(s) must
+   update --commit``. If no period is present, the RGW instances must
    be restarted for the changes to take effect.
 
 
 Rate Limit Management
 =====================
 
-The Ceph Object Gateway makes it possible to set rate limits on users and
-buckets.  "Rate limit" includes the maximum number of read operations (read
-ops) and write operations (write ops) per minute and the number of bytes per
-minute that can be written or read per user or per bucket.
+Quotas may be set for The Ceph Object Gateway on users and buckets.  "Rate
+limit" includes the maximum number of read operations (read ops) and write
+operations (write ops) per minute and the number of bytes per minute that can
+be written or read per user or per bucket.
 
 Operations that use the ``GET`` method or the ``HEAD`` method in their REST
 requests are "read requests". All other requests are "write requests".  

From f9aea9105b6c1a8d7bff0ec0675f84f2ffb1db6f Mon Sep 17 00:00:00 2001
From: Florian Weimer <fweimer@redhat.com>
Date: Wed, 20 Dec 2023 14:16:19 +0100
Subject: [PATCH 1246/2492] tracing: Fix C type errors in librados tracing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes type errors like this:

In file included from /usr/include/lttng/tracepoint-event.h:69,
                 from …-build/include/tracing/librados.h:4143,
                 from …/src/tracing/librados.c:6
:
…-build/include/tracing/librados.h:
 In function ‘lttng_ust__event_probe__librados___rados_mon_command_exit’:
…-build/include/tracing/librados.h:477:9: error: initialization of ‘size_t’ {aka ‘long unsigned int’} from ‘size_t *’ {aka ‘long unsigned int *’} makes integer from pointer without a cast
  477 |         ceph_ctf_integerp(size_t, outslen, outslen)
      |         ^~~~~~~~~~~~~~~~~

GCC 14 will likely treat these type mismatches as an error
and fail the build.

Signed-off-by: Florian Weimer <fweimer@redhat.com>
---
 src/tracing/librados.tp      | 4 ++--
 src/tracing/tracing-common.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tracing/librados.tp b/src/tracing/librados.tp
index 8b5e78ef15d6..8e116124b83d 100644
--- a/src/tracing/librados.tp
+++ b/src/tracing/librados.tp
@@ -2628,7 +2628,7 @@ TRACEPOINT_EVENT(librados, rados_watch3_enter,
     TP_FIELDS(
         ctf_integer_hex(rados_ioctx_t, ioctx, ioctx)
         ctf_string(oid, oid)
-        ctf_integer_hex(uint64_t, phandle, phandle)
+        ctf_integer_hex(uint64_t*, phandle, phandle)
         ctf_integer_hex(rados_watchcb2_t, callback, callback)
         ctf_integer(uint32_t, timeout, timeout)
         ctf_integer_hex(void*, arg, arg)
@@ -2658,7 +2658,7 @@ TRACEPOINT_EVENT(librados, rados_aio_watch2_enter,
         ctf_integer_hex(rados_ioctx_t, ioctx, ioctx)
         ctf_string(oid, oid)
         ctf_integer_hex(rados_completion_t, completion, completion)
-        ctf_integer_hex(uint64_t, phandle, phandle)
+        ctf_integer_hex(uint64_t*, phandle, phandle)
         ctf_integer_hex(rados_watchcb2_t, callback, callback)
         ctf_integer(uint32_t, timeout, timeout)
         ctf_integer_hex(void*, arg, arg)
diff --git a/src/tracing/tracing-common.h b/src/tracing/tracing-common.h
index 3e07f9de8e85..03449ab58861 100644
--- a/src/tracing/tracing-common.h
+++ b/src/tracing/tracing-common.h
@@ -21,7 +21,7 @@
 // type should be an integer type
 // val should have type type*
 #define ceph_ctf_integerp(type, field, val) \
-    ctf_integer(type, field, (val) == NULL ? 0 : (val)) \
+    ctf_integer(type, field, (val) == NULL ? 0 : *(val)) \
     ctf_integer(uint8_t, field##_isnull, (val) == NULL)
 
 // val should have type char*

From a49d154f4a8e493baf2296a15c7b5c56cd25e993 Mon Sep 17 00:00:00 2001
From: Florian Weimer <fweimer@redhat.com>
Date: Wed, 20 Dec 2023 14:59:19 +0100
Subject: [PATCH 1247/2492] pybind: Fix C type errors in Cython-generated
 Python bindings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Several Ceph APIs use bool * types, which correspond to
libcpp.bool * types in Cython.  The bint type has an incorrect
size 4 and cannot be used as a replacement.

This prevents a compilation failure with future compilers:

…-build/src/pybind/rbd/rbd.c: In function ‘__pyx_pf_3rbd_3RBD_104namespace_exists’:
…-build/src/pybind/rbd/rbd.c:42165:76: error: passing argument 3 of ‘rbd_namespace_exists’ from incompatible pointer type
42165 |         __pyx_v_ret = rbd_namespace_exists(__pyx_v__ioctx, __pyx_v__name, (&__pyx_v__exists));
      |                                                                           ~^~~~~~~~~~~~~~~~~
      |                                                                            |
      |                                                                            int *
In file included from …-build/src/pybind/rbd/rbd.c:1268:
…/src/include/rbd/librbd.h:1496:45: note: expected ‘_Bool *’ but argument is of type ‘int *’
 1496 |                                       bool *exists);
      |                                             ^

Signed-off-by: Florian Weimer <fweimer@redhat.com>
---
 src/pybind/rbd/c_rbd.pxd    | 5 +++--
 src/pybind/rbd/mock_rbd.pxi | 9 +++++++--
 src/pybind/rbd/rbd.pyx      | 9 +++++----
 src/pybind/rgw/mock_rgw.pxi | 9 +++++++--
 src/pybind/rgw/rgw.pyx      | 3 ++-
 5 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/pybind/rbd/c_rbd.pxd b/src/pybind/rbd/c_rbd.pxd
index 885f7bd46abd..bda23bbc4735 100644
--- a/src/pybind/rbd/c_rbd.pxd
+++ b/src/pybind/rbd/c_rbd.pxd
@@ -2,6 +2,7 @@
 
 from libc.stdint cimport *
 from ctime cimport time_t, timespec
+cimport libcpp
 
 cdef extern from "rados/librados.h":
     enum:
@@ -525,7 +526,7 @@ cdef extern from "rbd/librbd.h" nogil:
     int rbd_snap_unprotect(rbd_image_t image, const char *snap_name)
     int rbd_snap_is_protected(rbd_image_t image, const char *snap_name,
                               int *is_protected)
-    int rbd_snap_exists(rbd_image_t image, const char *snapname, bint *exists)
+    int rbd_snap_exists(rbd_image_t image, const char *snapname, libcpp.bool *exists)
     int rbd_snap_get_limit(rbd_image_t image, uint64_t *limit)
     int rbd_snap_set_limit(rbd_image_t image, uint64_t limit)
     int rbd_snap_get_timestamp(rbd_image_t image, uint64_t snap_id, timespec *timestamp)
@@ -711,7 +712,7 @@ cdef extern from "rbd/librbd.h" nogil:
     int rbd_namespace_list(rados_ioctx_t io, char *namespace_names,
                            size_t *size)
     int rbd_namespace_exists(rados_ioctx_t io, const char *namespace_name,
-                             bint *exists)
+                             libcpp.bool *exists)
 
     int rbd_pool_init(rados_ioctx_t, bint force)
 
diff --git a/src/pybind/rbd/mock_rbd.pxi b/src/pybind/rbd/mock_rbd.pxi
index 11872bd8146d..364f965fbad4 100644
--- a/src/pybind/rbd/mock_rbd.pxi
+++ b/src/pybind/rbd/mock_rbd.pxi
@@ -3,6 +3,11 @@
 from libc.stdint cimport *
 from ctime cimport time_t, timespec
 
+# Make the bool type available as libcpp.bool, for both C and C++.
+cimport libcpp
+cdef extern from "<stdbool.h>":
+    pass
+
 cdef nogil:
     enum:
         _LIBRADOS_SNAP_HEAD "LIBRADOS_SNAP_HEAD"
@@ -637,7 +642,7 @@ cdef nogil:
     int rbd_snap_is_protected(rbd_image_t image, const char *snap_name,
                               int *is_protected):
         pass
-    int rbd_snap_exists(rbd_image_t image, const char *snapname, bint *exists):
+    int rbd_snap_exists(rbd_image_t image, const char *snapname, libcpp.bool *exists):
         pass
     int rbd_snap_get_limit(rbd_image_t image, uint64_t *limit):
         pass
@@ -896,7 +901,7 @@ cdef nogil:
                            size_t *size):
         pass
     int rbd_namespace_exists(rados_ioctx_t io, const char *namespace_name,
-                             bint *exists):
+                             libcpp.bool *exists):
         pass
     int rbd_pool_init(rados_ioctx_t io, bint force):
         pass
diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx
index fcb2fb347060..f59ba23f0fe2 100644
--- a/src/pybind/rbd/rbd.pyx
+++ b/src/pybind/rbd/rbd.pyx
@@ -23,6 +23,7 @@ from libc cimport errno
 from libc.stdint cimport *
 from libc.stdlib cimport malloc, realloc, free
 from libc.string cimport strdup, memset
+cimport libcpp
 
 try:
     from collections.abc import Iterable
@@ -1935,12 +1936,12 @@ class RBD(object):
         cdef:
             rados_ioctx_t _ioctx = convert_ioctx(ioctx)
             const char *_name = name
-            bint _exists = False
+            libcpp.bool _exists = False
         with nogil:
             ret = rbd_namespace_exists(_ioctx, _name, &_exists)
         if ret != 0:
             raise make_ex(ret, 'error verifying namespace')
-        return bool(_exists != 0)
+        return _exists
 
     def namespace_list(self, ioctx):
         """
@@ -3679,12 +3680,12 @@ cdef class Image(object):
         name = cstr(name, 'name')
         cdef:
             char *_name = name
-            bint _exists = False
+            libcpp.bool _exists = False
         with nogil:
             ret = rbd_snap_exists(self.image, _name, &_exists)
         if ret != 0:
             raise make_ex(ret, 'error getting snapshot exists for %s' % self.name)
-        return bool(_exists != 0)
+        return _exists
 
     @requires_not_closed
     def get_snap_limit(self):
diff --git a/src/pybind/rgw/mock_rgw.pxi b/src/pybind/rgw/mock_rgw.pxi
index ca893a5bb8a1..806d4df75de0 100644
--- a/src/pybind/rgw/mock_rgw.pxi
+++ b/src/pybind/rgw/mock_rgw.pxi
@@ -1,5 +1,10 @@
 # cython: embedsignature=True
 
+# Make the bool type available as libcpp.bool, for both C and C++.
+cimport libcpp
+cdef extern from "<stdbool.h>":
+    pass
+
 cdef nogil:
     ctypedef void* librgw_t
 
@@ -111,8 +116,8 @@ cdef nogil:
 
     int rgw_readdir(rgw_fs *fs,
                     rgw_file_handle *parent_fh, uint64_t *offset,
-                    bint (*cb)(const char *name, void *arg, uint64_t offset, stat *st, uint32_t st_mask, uint32_t flags) nogil except? -9000,
-                    void *cb_arg, bint *eof, uint32_t flags) except? -9000:
+                    libcpp.bool (*cb)(const char *name, void *arg, uint64_t offset, stat *st, uint32_t st_mask, uint32_t flags) nogil except? -9000,
+                    void *cb_arg, libcpp.bool *eof, uint32_t flags) except? -9000:
         pass
 
     int rgw_getattr(rgw_fs *fs,
diff --git a/src/pybind/rgw/rgw.pyx b/src/pybind/rgw/rgw.pyx
index 9bbcdfff586a..d210a70bbb8e 100644
--- a/src/pybind/rgw/rgw.pyx
+++ b/src/pybind/rgw/rgw.pyx
@@ -7,6 +7,7 @@ from cpython cimport PyObject, ref, exc, array
 from libc.stdint cimport *
 from libc.stdlib cimport malloc, realloc, free
 from cstat cimport stat
+cimport libcpp
 
 IF BUILD_DOC:
     include "mock_rgw.pxi"
@@ -373,7 +374,7 @@ cdef class LibRGWFS(object):
         cdef:
             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
             uint64_t _offset = offset
-            bint _eof
+            libcpp.bool _eof
             uint32_t _flags = flags
         with nogil:
             ret = rgw_readdir(self.fs, _dir_handler, &_offset, &readdir_cb,

From 655aedf605e8b5bc99976499ae5bdb8d11aa4a93 Mon Sep 17 00:00:00 2001
From: Anoop C S <anoopcs@cryptolab.net>
Date: Mon, 8 Jan 2024 17:15:26 +0530
Subject: [PATCH 1248/2492] client: Fix parsing of xattr namespace in
 removexattr

To match the kernel ceph client, as mentioned by the inline comments,
remove the comparison with "system." xattr namespace in removexattr()
code path. This is in preparation to handle the case where POSIX ACL
xattrs are treated differently by kernel during removexattr operation.

ref: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/ceph/xattr.c#n21

Signed-off-by: Anoop C S <anoopcs@cryptolab.net>
---
 src/client/Client.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 195738efc87d..9479ebfa2170 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -14069,7 +14069,6 @@ int Client::_removexattr(Inode *in, const char *name, const UserPerm& perms)
 
   // same xattrs supported by kernel client
   if (strncmp(name, "user.", 5) &&
-      strncmp(name, "system.", 7) &&
       strncmp(name, "security.", 9) &&
       strncmp(name, "trusted.", 8) &&
       strncmp(name, "ceph.", 5))

From 67c5669157e1771ab075bef095fb6bcd29975a38 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 16 Aug 2023 19:56:38 -0400
Subject: [PATCH 1249/2492] qa/cephadm: support to pull stable branch compiled
 cephadm

This is to allow us to pull the latest build of
cephadm off of a stable branch (currently the only
valid option for that is reef, although this hopefully
will work with squid, T release, etc. in the future).

This should allow us to bootstrap cliusters based on
those stable branches for use in upgrade testing

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/tasks/cephadm.py     | 51 +++++++++++++++++++++++++++++++++++++++++
 qa/tasks/util/chacra.py | 10 +++++++-
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 1fdf6ee012e3..9e386bbd5c29 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -169,6 +169,8 @@ def download_cephadm(ctx, config, ref):
         # cephadm
         elif 'cephadm_git_url' in config and 'cephadm_branch' in config:
             _fetch_cephadm_from_github(ctx, config, ref)
+        elif 'compiled_cephadm_branch' in config:
+            _fetch_stable_branch_cephadm_from_chacra(ctx, config, cluster_name)
         else:
             _fetch_cephadm_from_chachra(ctx, config, cluster_name)
 
@@ -291,6 +293,55 @@ def _fetch_cephadm_from_chachra(ctx, config, cluster_name):
         ],
     )
 
+def _fetch_stable_branch_cephadm_from_chacra(ctx, config, cluster_name):
+    branch = config.get('compiled_cephadm_branch', 'reef')
+    flavor = config.get('flavor', 'default')
+
+    log.info(f'Downloading "compiled" cephadm from cachra for {branch}')
+
+    bootstrap_remote = ctx.ceph[cluster_name].bootstrap_remote
+    bp = packaging.get_builder_project()(
+        config.get('project', 'ceph'),
+        config,
+        ctx=ctx,
+        remote=bootstrap_remote,
+    )
+    log.info('builder_project result: %s' % (bp._result.json()))
+
+    # pull the cephadm binary from chacra
+    url = chacra.get_binary_url(
+            'cephadm',
+            project=bp.project,
+            distro=bp.distro.split('/')[0],
+            release=bp.distro.split('/')[1],
+            arch=bp.arch,
+            flavor=flavor,
+            branch=branch,
+    )
+    log.info("Discovered cachra url: %s", url)
+    ctx.cluster.run(
+        args=[
+            'curl', '--silent', '-L', url,
+            run.Raw('>'),
+            ctx.cephadm,
+            run.Raw('&&'),
+            'ls', '-l',
+            ctx.cephadm,
+        ],
+    )
+
+    # sanity-check the resulting file and set executable bit
+    cephadm_file_size = '$(stat -c%s {})'.format(ctx.cephadm)
+    ctx.cluster.run(
+        args=[
+            'test', '-s', ctx.cephadm,
+            run.Raw('&&'),
+            'test', run.Raw(cephadm_file_size), "-gt", run.Raw('1000'),
+            run.Raw('&&'),
+            'chmod', '+x', ctx.cephadm,
+        ],
+    )
+
 
 def _rm_cluster(ctx, cluster_name):
     log.info('Removing cluster...')
diff --git a/qa/tasks/util/chacra.py b/qa/tasks/util/chacra.py
index ed9358a59749..52414c890f9c 100644
--- a/qa/tasks/util/chacra.py
+++ b/qa/tasks/util/chacra.py
@@ -71,7 +71,15 @@ def get_binary_url(
     if len(result) == 0:
         raise RuntimeError(f'no results found at {resp.url}')
 
-    # TODO: filter the result down to the correct arch etc.?
+    # if arch was supplied, filter down to only results
+    # that include the desired arch
+    if arch:
+        result = [r for r in result if ('archs' in r and arch in r['archs'])]
+
+    # TODO: Is there any further filtering we should do beyond arch?
+    # We already use flavor, ref, etc. in our search.
+
+    # TODO: After filtering, does it matter which result we take?
     result = result[0]
 
     status = result['status']

From 967716f2e6c8cd3f7d810a5f8e5e49448717e513 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 29 Nov 2023 11:49:38 -0500
Subject: [PATCH 1250/2492] qa/upgrade/reef-x: pull compiled cephadm to start
 upgrades from reef

The compiled zipapp cephadm that began in reef needs
to be pulled differently than the old single python script
cephadm from earlier releases. This commit updates the reef-x
upgrade suite to pull cephadm in this new way.

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/suites/upgrade/reef-x/parallel/1-tasks.yaml     | 3 +--
 qa/suites/upgrade/reef-x/stress-split/1-start.yaml | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/qa/suites/upgrade/reef-x/parallel/1-tasks.yaml b/qa/suites/upgrade/reef-x/parallel/1-tasks.yaml
index 6e3ede31c04c..7f7ad8c57d3d 100644
--- a/qa/suites/upgrade/reef-x/parallel/1-tasks.yaml
+++ b/qa/suites/upgrade/reef-x/parallel/1-tasks.yaml
@@ -7,8 +7,7 @@ tasks:
 - print: "**** done start installing reef cephadm ..."
 - cephadm:
     image: quay.ceph.io/ceph-ci/ceph:reef
-    cephadm_branch: reef
-    cephadm_git_url: https://github.com/ceph/ceph
+    compiled_cephadm_branch: reef
     conf:
       osd:
         #set config option for which cls modules are allowed to be loaded / used
diff --git a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
index b4a67744f355..1add459f439e 100644
--- a/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
+++ b/qa/suites/upgrade/reef-x/stress-split/1-start.yaml
@@ -6,8 +6,7 @@ tasks:
 
 - cephadm:
     image: quay.ceph.io/ceph-ci/ceph:reef
-    cephadm_branch: reef
-    cephadm_git_url: https://github.com/ceph/ceph
+    compiled_cephadm_branch: reef
     conf:
       osd:
         #set config option for which cls modules are allowed to be loaded / used

From df2e42c02f2df1e126aa233f599a4286b3efa9ac Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 4 Aug 2023 13:30:55 -0400
Subject: [PATCH 1251/2492] qa/cephadm: mgr-nfs-upgrade, match any migration >
 2

I believe this check was originally added because
the 2->3 migration migrated some nfs related bits. Since
then we've had to update the migration this checks
for every time we bump the max migration. This change
is intended to instead just have it check for a
miration > 2 so we don't have to keep updating it.

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
index 3a9169659090..7559885b9715 100644
--- a/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
@@ -7,4 +7,8 @@ tasks:
       - ceph nfs cluster ls | grep foo
       - ceph nfs export ls foo --detailed
       - rados -p .nfs --all ls -
-      - ceph config get mgr mgr/cephadm/migration_current | grep 6
+- cephadm.shell:
+    host.a:
+      - |
+        set -ex
+        [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]]

From 2ba7d6e59e8750867059c29a33c22bcb3e358b65 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 8 Jan 2024 08:24:18 -0800
Subject: [PATCH 1252/2492] make-dist: don't use --continue option for wget

the boost jfrog mirror is broken and returns an HTML error page instead
of the archive. the file size of this page is 11534 bytes

when download_from() retries the download from download.ceph.com, the -c
option tells it to resume the download of the existing file. the
resulting boost_1_82_0.tar.bz2 ends up with the correct total file size
of 121325129 bytes, but the first 11534 bytes still correspond to the
HTML from jfrog. that causes the sha256sum mismatch

remove the -c option so that wget fetches the archive in its entirety

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 make-dist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/make-dist b/make-dist
index 4add880180eb..45cbf1381ccd 100755
--- a/make-dist
+++ b/make-dist
@@ -55,7 +55,7 @@ download_from() {
             exit
         fi
         url=$url_base/$fname
-        wget -c --no-verbose -O $fname $url
+        wget --no-verbose -O $fname $url
         if [ $? != 0 -o ! -e $fname ]; then
             echo "Download of $url failed"
         elif [ $(sha256sum $fname | awk '{print $1}') != $sha256 ]; then

From a17dccebba0fbc46ac0d3d57fca3f671b6a99bec Mon Sep 17 00:00:00 2001
From: Anoop C S <anoopcs@cryptolab.net>
Date: Mon, 8 Jan 2024 17:17:59 +0530
Subject: [PATCH 1253/2492] client: Add exception to POSIX ACL xattrs in
 removexattr

With an exception to POSIX ACL xattrs all others from "system."
namespace are found to return ENOTSUP during a removal attempt.
Hence "system.posix_acl_access" and "system.posix_acl_default"
are exempted thereby returning success despite their presence
on files.

Signed-off-by: Anoop C S <anoopcs@cryptolab.net>
---
 src/client/Client.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 9479ebfa2170..bfed869a0ec3 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -14071,7 +14071,9 @@ int Client::_removexattr(Inode *in, const char *name, const UserPerm& perms)
   if (strncmp(name, "user.", 5) &&
       strncmp(name, "security.", 9) &&
       strncmp(name, "trusted.", 8) &&
-      strncmp(name, "ceph.", 5))
+      strncmp(name, "ceph.", 5) &&
+      strcmp(name, ACL_EA_ACCESS) &&
+      strcmp(name, ACL_EA_DEFAULT))
     return -CEPHFS_EOPNOTSUPP;
 
   const VXattr *vxattr = _match_vxattr(in, name);
@@ -14087,6 +14089,11 @@ int Client::_removexattr(Inode *in, const char *name, const UserPerm& perms)
  
   int res = make_request(req, perms);
 
+  if ((!strcmp(name, ACL_EA_ACCESS) ||
+      !strcmp(name, ACL_EA_DEFAULT)) &&
+      res == -CEPHFS_ENODATA)
+    res = 0;
+
   trim_cache();
   ldout(cct, 8) << "_removexattr(" << in->ino << ", \"" << name << "\") = " << res << dendl;
   return res;

From eaf9c170ca88c7279546f1861682915e2d745869 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 8 Jan 2024 13:08:03 -0500
Subject: [PATCH 1254/2492] make-dist: prefer download.ceph.com mirror for
 boost sources

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 make-dist | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/make-dist b/make-dist
index 45cbf1381ccd..d5bcf44ff898 100755
--- a/make-dist
+++ b/make-dist
@@ -193,8 +193,8 @@ tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec
 # at the three URLs referenced below (may involve uploading to download.ceph.com)
 boost_version=1.82.0
 download_boost $boost_version a6e1ab9b0860e6a2881dd7b21fe9f737a095e5f33a3a874afc6a345228597ee6 \
-               https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source \
-               https://download.ceph.com/qa
+               https://download.ceph.com/qa \
+               https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source
 download_liburing 0.7 8e2842cfe947f3a443af301bdd6d034455536c38a455c7a700d0c1ad165a7543 \
                   https://github.com/axboe/liburing/archive \
                   https://git.kernel.dk/cgit/liburing/snapshot

From 5697d81188cafe19dde96e9c080b858f3a0f7861 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 8 Jan 2024 13:08:41 -0500
Subject: [PATCH 1255/2492] make-dist: replace boostorg.jfrog.io mirror with
 archives.boost.io

change the jfrog mirror to the same one used by win32_deps_build.sh
after 594d1e5e83e3d4c318ab1dd79660670a0852f7e6

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 make-dist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/make-dist b/make-dist
index d5bcf44ff898..41cee50a6d39 100755
--- a/make-dist
+++ b/make-dist
@@ -194,7 +194,7 @@ tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec
 boost_version=1.82.0
 download_boost $boost_version a6e1ab9b0860e6a2881dd7b21fe9f737a095e5f33a3a874afc6a345228597ee6 \
                https://download.ceph.com/qa \
-               https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source
+               https://archives.boost.io/release/$boost_version/source
 download_liburing 0.7 8e2842cfe947f3a443af301bdd6d034455536c38a455c7a700d0c1ad165a7543 \
                   https://github.com/axboe/liburing/archive \
                   https://git.kernel.dk/cgit/liburing/snapshot

From ec9fdd4a9bee1fb6b1abb2455e8b7cea7b8223aa Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 30 Nov 2023 21:56:17 -0800
Subject: [PATCH 1256/2492] OSDMonitor: update require_osd_release for crush
 features

The existing implementation checks that the currently live
OSDs support the pending crush features, but we don't
actually check that require_osd_release is new enough.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/mon/OSDMonitor.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f8e379326f25..05fabca0d09a 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -7562,6 +7562,12 @@ bool OSDMonitor::validate_crush_against_features(const CrushWrapper *newcrush,
 	 << newmap.require_min_compat_client;
       return false;
     }
+    if (mv > newmap.require_osd_release) {
+      ss << "new crush map requires client version " << mv
+	 << " but require_osd_release is "
+	 << newmap.require_osd_release;
+      return false;
+    }
   }
 
   // osd compat

From 22ab2747c56e6fe69b13968941cd33c9b0eeb79b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 19:59:10 -0800
Subject: [PATCH 1257/2492] src/crush: add a rule type enum

We're going to add more rule types that don't map directly onto
pool types.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc | 8 ++++----
 src/crush/crush.h          | 7 ++++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 5e51aad8dba4..f7df0c6670ce 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -363,10 +363,10 @@ int CrushCompiler::decompile(ostream &out)
     out << "\tid " << i << "\n";
 
     switch (crush.get_rule_type(i)) {
-    case CEPH_PG_TYPE_REPLICATED:
+    case CRUSH_RULE_TYPE_REPLICATED:
       out << "\ttype replicated\n";
       break;
-    case CEPH_PG_TYPE_ERASURE:
+    case CRUSH_RULE_TYPE_ERASURE:
       out << "\ttype erasure\n";
       break;
     default:
@@ -781,9 +781,9 @@ int CrushCompiler::parse_rule(iter_t const& i)
   string tname = string_node(i->children[start+2]);
   int type;
   if (tname == "replicated")
-    type = CEPH_PG_TYPE_REPLICATED;
+    type = CRUSH_RULE_TYPE_REPLICATED;
   else if (tname == "erasure")
-    type = CEPH_PG_TYPE_ERASURE;
+    type = CRUSH_RULE_TYPE_ERASURE;
   else 
     ceph_abort();
 
diff --git a/src/crush/crush.h b/src/crush/crush.h
index fde2df6a8a3e..263922615b63 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -87,7 +87,12 @@ struct crush_rule {
 #define crush_rule_size(len) (sizeof(struct crush_rule) + \
 			      (len)*sizeof(struct crush_rule_step))
 
-
+enum crush_rule_type {
+	CRUSH_RULE_TYPE_REPLICATED = 1,
+	CRUSH_RULE_TYPE_ERASURE = 3,
+	CRUSH_RULE_TYPE_MSR_FIRSTN = 4,
+	CRUSH_RULE_TYPE_MSR_INDEP = 5
+};
 
 /*
  * A bucket is a named container of other items (either devices or

From 196205dcf52f5fad3d7f6fd051958e9497c9b94d Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:09:45 -0800
Subject: [PATCH 1258/2492] crush: add msr rule types and CrushWrapper
 infrastructure

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc  | 10 ++++++++++
 src/crush/CrushWrapper.cc   | 24 ++++++++++++++++++++++++
 src/crush/CrushWrapper.h    |  6 +++++-
 src/crush/grammar.h         |  2 +-
 src/include/ceph_features.h |  6 ++++--
 src/osd/OSDMap.cc           |  5 +++++
 6 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index f7df0c6670ce..740f003005b9 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -369,6 +369,12 @@ int CrushCompiler::decompile(ostream &out)
     case CRUSH_RULE_TYPE_ERASURE:
       out << "\ttype erasure\n";
       break;
+    case CRUSH_RULE_TYPE_MSR_FIRSTN:
+      out << "\ttype msr_firstn\n";
+      break;
+    case CRUSH_RULE_TYPE_MSR_INDEP:
+      out << "\ttype msr_indep\n";
+      break;
     default:
       out << "\ttype " << crush.get_rule_type(i) << "\n";
     }
@@ -784,6 +790,10 @@ int CrushCompiler::parse_rule(iter_t const& i)
     type = CRUSH_RULE_TYPE_REPLICATED;
   else if (tname == "erasure")
     type = CRUSH_RULE_TYPE_ERASURE;
+  else if (tname == "msr_firstn")
+    type = CRUSH_RULE_TYPE_MSR_FIRSTN;
+  else if (tname == "msr_indep")
+    type = CRUSH_RULE_TYPE_MSR_INDEP;
   else 
     ceph_abort();
 
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 0f40e6875e1b..5e3694c5193b 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -135,6 +135,29 @@ bool CrushWrapper::is_v5_rule(unsigned ruleid) const
   return false;
 }
 
+bool CrushWrapper::has_msr_rules() const
+{
+  for (unsigned i=0; i<crush->max_rules; i++) {
+    if (is_msr_rule(i)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool CrushWrapper::is_msr_rule(unsigned ruleid) const
+{
+  if (ruleid >= crush->max_rules)
+    return false;
+  
+  crush_rule *r = crush->rules[ruleid];
+  if (!r)
+    return false;
+
+  return r->type == CRUSH_RULE_TYPE_MSR_INDEP ||
+    r->type == CRUSH_RULE_TYPE_MSR_FIRSTN;
+}
+
 bool CrushWrapper::has_choose_args() const
 {
   return !choose_args.empty();
@@ -3515,6 +3538,7 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("has_v4_buckets", (int)has_v4_buckets());
   f->dump_int("require_feature_tunables5", (int)has_nondefault_tunables5());
   f->dump_int("has_v5_rules", (int)has_v5_rules());
+  f->dump_int("has_msr_rules", (int)has_msr_rules());
 }
 
 void CrushWrapper::dump_choose_args(Formatter *f) const
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index b8caa24ce621..e84a829a2d3d 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -329,13 +329,17 @@ class CrushWrapper {
   bool has_v5_rules() const;
   bool has_choose_args() const;          // any choose_args
   bool has_incompat_choose_args() const; // choose_args that can't be made compat
+  bool has_msr_rules() const;
 
   bool is_v2_rule(unsigned ruleid) const;
   bool is_v3_rule(unsigned ruleid) const;
   bool is_v5_rule(unsigned ruleid) const;
+  bool is_msr_rule(unsigned ruleid) const;
 
   std::string get_min_required_version() const {
-    if (has_v5_rules() || has_nondefault_tunables5())
+    if (has_msr_rules())
+      return "squid";
+    else if (has_v5_rules() || has_nondefault_tunables5())
       return "jewel";
     else if (has_v4_buckets())
       return "hammer";
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index b5c356a49e17..072b36b073aa 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -170,7 +170,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_emit );
       crushrule = str_p("rule") >> !name >> '{'
 				>> (str_p("id") | str_p("ruleset")) >> posint
-				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") )
+				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") | str_p("msr_firstn") | str_p("msr_indep") )
 				>> !(str_p("min_size") >> posint)
 				>> !(str_p("max_size") >> posint)
 			   >> +step
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 1937eeb4c698..23175adfa2c8 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -137,7 +137,7 @@ DEFINE_CEPH_FEATURE(34, 3, RANGE_BLOCKLIST)
 DEFINE_CEPH_FEATURE(35, 1, OSD_CACHEPOOL)    // 3.14
 DEFINE_CEPH_FEATURE(36, 1, CRUSH_V2)         // 3.14
 DEFINE_CEPH_FEATURE(37, 1, EXPORT_PEER)      // 3.14
-DEFINE_CEPH_FEATURE_RETIRED(38, 1, OSD_ERASURE_CODES, MIMIC, OCTOPUS)
+DEFINE_CEPH_FEATURE(38, 2, CRUSH_MSR)        // X.XX TODOSAM kernel version?
 // available
 DEFINE_CEPH_FEATURE(39, 1, OSDMAP_ENC)       // 3.15
 DEFINE_CEPH_FEATURE(40, 1, MDS_INLINE_DATA)  // 3.19
@@ -218,6 +218,7 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_OSD_CACHEPOOL |	    \
 	 CEPH_FEATURE_CRUSH_V2 |	    \
 	 CEPH_FEATURE_EXPORT_PEER |	    \
+	 CEPH_FEATURE_CRUSH_MSR |	    \
 	 CEPH_FEATURE_OSDMAP_ENC |          \
 	 CEPH_FEATURE_MDS_INLINE_DATA |	    \
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |	    \
@@ -265,9 +266,10 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_CRUSH_TUNABLES2 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES5 |		\
+	 CEPH_FEATURE_CRUSH_MSR |		\
 	 CEPH_FEATURE_CRUSH_V2 |		\
 	 CEPH_FEATURE_CRUSH_V4 |		\
-	 CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS)
+	 CEPH_FEATUREMASK_CRUSH_MSR)
 
 /*
  * make sure we don't try to use the reserved features
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 5773695b77ad..209ec6db203d 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1789,6 +1789,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
 	features |= CEPH_FEATURE_CRUSH_TUNABLES3;
       if (crush->is_v5_rule(ruleid))
 	features |= CEPH_FEATURE_CRUSH_TUNABLES5;
+      if (crush->is_msr_rule(ruleid))
+	features |= CEPH_FEATURE_CRUSH_MSR;
     }
   }
   mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;
@@ -1843,6 +1845,9 @@ ceph_release_t OSDMap::get_min_compat_client() const
 {
   uint64_t f = get_features(CEPH_ENTITY_TYPE_CLIENT, nullptr);
 
+  if (HAVE_FEATURE(f, CRUSH_MSR)) {      // TODOSAM -- add version right before merge
+    return ceph_release_t::squid;        // v19.2.0
+  }
   if (HAVE_FEATURE(f, OSDMAP_PG_UPMAP) ||      // v12.0.0-1733-g27d6f43
       HAVE_FEATURE(f, CRUSH_CHOOSE_ARGS)) {    // v12.0.1-2172-gef1ef28
     return ceph_release_t::luminous;  // v12.2.0

From 52654551cc6c87ba00cd20587ff3fa3dc8142459 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:33:54 -0800
Subject: [PATCH 1259/2492] osd/OSDMap: minor formatting nit

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/osd/OSDMap.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 209ec6db203d..305431071029 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1764,9 +1764,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_V4;
   if (crush->has_nondefault_tunables5())
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
-  if (crush->has_incompat_choose_args()) {
+  if (crush->has_incompat_choose_args())
     features |= CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS;
-  }
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty() || !pg_upmap_primaries.empty())

From 88a7fc2713f372ff95cff02f43f3dababfff756e Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:20:07 -0800
Subject: [PATCH 1260/2492] crush: add msr tunables

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc | 50 +++++++++++++++++++++++++++++++++++
 src/crush/CrushWrapper.cc  | 25 ++++++++++++++++++
 src/crush/CrushWrapper.h   | 53 +++++++++++++++++++++++++++++++++-----
 src/crush/crush.h          | 13 +++++++++-
 src/crush/grammar.h        |  8 ++++++
 src/osd/OSDMap.cc          |  2 ++
 6 files changed, 144 insertions(+), 7 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 740f003005b9..c884caed00e6 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -321,6 +321,13 @@ int CrushCompiler::decompile(ostream &out)
   if (crush.get_allowed_bucket_algs() != CRUSH_LEGACY_ALLOWED_BUCKET_ALGS)
     out << "tunable allowed_bucket_algs " << crush.get_allowed_bucket_algs()
 	<< "\n";
+  if (crush.has_nondefault_tunables_msr()) {
+    out << "tunable msr_descents " << crush.get_msr_descents()
+	<< "\n";
+    out << "tunable msr_collision_tries "
+	<< crush.get_msr_collision_tries()
+	<< "\n";
+  }
 
   out << "\n# devices\n";
   for (int i=0; i<crush.get_max_devices(); i++) {
@@ -428,6 +435,15 @@ int CrushCompiler::decompile(ostream &out)
 	out << "\tstep set_chooseleaf_stable " << crush.get_rule_arg1(i, j)
 	    << "\n";
 	break;
+      case CRUSH_RULE_SET_MSR_DESCENTS:
+	out << "\tstep set_msr_descents " << crush.get_rule_arg1(i, j)
+	    << "\n";
+	break;
+      case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+	out << "\tstep set_msr_collision_tries "
+	    << crush.get_rule_arg1(i, j)
+	    << "\n";
+	break;
       case CRUSH_RULE_CHOOSE_FIRSTN:
 	out << "\tstep choose firstn "
 	    << crush.get_rule_arg1(i, j) 
@@ -456,6 +472,13 @@ int CrushCompiler::decompile(ostream &out)
 	print_type_name(out, crush.get_rule_arg2(i, j), crush);
 	out << "\n";
 	break;
+      case CRUSH_RULE_CHOOSE_MSR:
+	out << "\tstep choosemsr "
+	    << crush.get_rule_arg1(i, j) 
+	    << " type ";
+	print_type_name(out, crush.get_rule_arg2(i, j), crush);
+	out << "\n";
+	break;
       }
     }
     out << "}\n";
@@ -538,6 +561,10 @@ int CrushCompiler::parse_tunable(iter_t const& i)
     crush.set_straw_calc_version(val);
   else if (name == "allowed_bucket_algs")
     crush.set_allowed_bucket_algs(val);
+  else if (name == "msr_descents")
+    crush.set_msr_descents(val);
+  else if (name == "msr_collision_tries")
+    crush.set_msr_collision_tries(val);
   else {
     err << "tunable " << name << " not recognized" << std::endl;
     return -1;
@@ -915,6 +942,18 @@ int CrushCompiler::parse_rule(iter_t const& i)
 	crush.set_rule_step_set_chooseleaf_stable(ruleno, step++, val);
       }
       break;
+    case crush_grammar::_step_set_msr_descents:
+      {
+	int val = int_node(s->children[1]);
+	crush.set_rule_step_set_msr_descents(ruleno, step++, val);
+      }
+      break;
+    case crush_grammar::_step_set_msr_collision_tries:
+      {
+	int val = int_node(s->children[1]);
+	crush.set_rule_step_set_msr_collision_tries(ruleno, step++, val);
+      }
+      break;
 
     case crush_grammar::_step_choose:
     case crush_grammar::_step_chooseleaf:
@@ -942,6 +981,17 @@ int CrushCompiler::parse_rule(iter_t const& i)
       }
       break;
 
+    case crush_grammar::_step_choose_msr:
+      {
+	string type = string_node(s->children[3]);
+	if (!type_id.count(type)) {
+	  err << "in rule '" << rname << "' type '" << type << "' not defined" << std::endl;
+	  return -1;
+	}
+	crush.set_rule_step_choose_msr(ruleno, step++, int_node(s->children[1]), type_id[type]);
+      }
+      break;
+
     case crush_grammar::_step_emit:
       crush.set_rule_step_emit(ruleno, step++);
       break;
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 5e3694c5193b..69936a53d651 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -3103,6 +3103,10 @@ void CrushWrapper::encode(bufferlist& bl, uint64_t features) const
       }
     }
   }
+  if (HAVE_FEATURE(features, CRUSH_MSR)) {
+    encode(crush->msr_descents, bl);
+    encode(crush->msr_collision_tries, bl);
+  }
 }
 
 static void decode_32_or_64_string_map(map<int32_t,string>& m, bufferlist::const_iterator& blp)
@@ -3253,6 +3257,12 @@ void CrushWrapper::decode(bufferlist::const_iterator& blp)
 	choose_args[choose_args_index] = arg_map;
       }
     }
+    if (!blp.end()) {
+      decode(crush->msr_descents, blp);
+      decode(crush->msr_collision_tries, blp);
+    } else {
+      set_default_msr_tunables();
+    }
     update_choose_args(nullptr); // in case we decode a legacy "corrupted" map
     finalize();
   }
@@ -3508,6 +3518,8 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("chooseleaf_descend_once", get_chooseleaf_descend_once());
   f->dump_int("chooseleaf_vary_r", get_chooseleaf_vary_r());
   f->dump_int("chooseleaf_stable", get_chooseleaf_stable());
+  f->dump_int("msr_descents", get_msr_descents());
+  f->dump_int("msr_collision_tries", get_msr_collision_tries());
   f->dump_int("straw_calc_version", get_straw_calc_version());
   f->dump_int("allowed_bucket_algs", get_allowed_bucket_algs());
 
@@ -3637,6 +3649,11 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_int("num", get_rule_arg1(rule_id, j));
       f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
       break;
+    case CRUSH_RULE_CHOOSE_MSR:
+      f->dump_string("op", "choosemsr");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
+      break;
     case CRUSH_RULE_SET_CHOOSE_TRIES:
       f->dump_string("op", "set_choose_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
@@ -3645,6 +3662,14 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_string("op", "set_chooseleaf_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
       break;
+    case CRUSH_RULE_SET_MSR_DESCENTS:
+      f->dump_string("op", "set_msr_descents");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      break;
+    case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+      f->dump_string("op", "set_msr_collision_tries");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      break;
     default:
       f->dump_int("opcode", get_rule_op(rule_id, j));
       f->dump_int("arg1", get_rule_arg1(rule_id, j));
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index e84a829a2d3d..8419a92a410d 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -125,6 +125,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_bobtail() {
     crush->choose_local_tries = 0;
@@ -134,6 +135,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_firefly() {
     crush->choose_local_tries = 0;
@@ -143,6 +145,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 1;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_hammer() {
     crush->choose_local_tries = 0;
@@ -156,6 +159,7 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
+    set_default_msr_tunables();
   }
   void set_tunables_jewel() {
     crush->choose_local_tries = 0;
@@ -169,6 +173,7 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
+    set_default_msr_tunables();
   }
 
   void set_tunables_legacy() {
@@ -233,6 +238,24 @@ class CrushWrapper {
     crush->straw_calc_version = n;
   }
 
+  int get_msr_descents() const {
+    return crush->msr_descents;
+  }
+  void set_msr_descents(int n) {
+    crush->msr_descents = n;
+  }
+
+  int get_msr_collision_tries() const {
+    return crush->msr_collision_tries;
+  }
+  void set_msr_collision_tries(int n) {
+    crush->msr_collision_tries = n;
+  }
+  void set_default_msr_tunables() {
+    set_msr_descents(100);
+    set_msr_collision_tries(100);
+  }
+
   unsigned get_allowed_bucket_algs() const {
     return crush->allowed_bucket_algs;
   }
@@ -248,7 +271,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 0 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_bobtail_tunables() const {
     return
@@ -258,7 +282,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_firefly_tunables() const {
     return
@@ -268,7 +293,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 1 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_hammer_tunables() const {
     return
@@ -281,7 +307,8 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2));
+				      (1 << CRUSH_BUCKET_STRAW2)) &&
+      !has_nondefault_tunables_msr();
   }
   bool has_jewel_tunables() const {
     return
@@ -294,7 +321,8 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2));
+				      (1 << CRUSH_BUCKET_STRAW2)) &&
+      !has_nondefault_tunables_msr();
   }
 
   bool has_optimal_tunables() const {
@@ -322,6 +350,11 @@ class CrushWrapper {
     return
         crush->chooseleaf_stable != 0;
   }
+  bool has_nondefault_tunables_msr() const {
+    return
+      crush->msr_descents != 100 ||
+      crush->msr_collision_tries != 100;
+  }
 
   bool has_v2_rules() const;
   bool has_v3_rules() const;
@@ -337,7 +370,7 @@ class CrushWrapper {
   bool is_msr_rule(unsigned ruleid) const;
 
   std::string get_min_required_version() const {
-    if (has_msr_rules())
+    if (has_msr_rules() || has_nondefault_tunables_msr())
       return "squid";
     else if (has_v5_rules() || has_nondefault_tunables5())
       return "jewel";
@@ -1155,6 +1188,14 @@ class CrushWrapper {
   int set_rule_step_set_chooseleaf_stable(unsigned ruleno, unsigned step, int val) {
     return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_STABLE, val, 0);
   }
+
+  int set_rule_step_set_msr_descents(unsigned ruleno, unsigned step, int val) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_DESCENTS, val, 0);
+  }
+  int set_rule_step_set_msr_collision_tries(unsigned ruleno, unsigned step, int val) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_COLLISION_TRIES, val, 0);
+  }
+
   int set_rule_step_choose_firstn(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_FIRSTN, val, type);
   }
diff --git a/src/crush/crush.h b/src/crush/crush.h
index 263922615b63..236ffea419c5 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -65,7 +65,12 @@ enum crush_opcodes {
 	CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES = 10,
 	CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES = 11,
 	CRUSH_RULE_SET_CHOOSELEAF_VARY_R = 12,
-	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13
+	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13,
+
+	/* set choose_msr_total_tries */
+	CRUSH_RULE_SET_MSR_DESCENTS = 14,
+	/* set choose_msr_local_collision_tries */
+	CRUSH_RULE_SET_MSR_COLLISION_TRIES = 15,
 };
 
 /*
@@ -415,6 +420,12 @@ struct crush_map {
          */
 	__u8 chooseleaf_stable;
 
+	/*! Sets total descents for MSR rules */
+	__u8 msr_descents;
+
+	/*! Sets local collision retries for MSR rules */
+	__u8 msr_collision_tries;
+
         /*! @cond INTERNAL */
 	/* This value is calculated after decode or construction by
 	   the builder. It is exposed here (rather than having a
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 072b36b073aa..9e955091c6a8 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -50,6 +50,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     _step_set_choose_tries,
     _step_set_choose_local_tries,
     _step_set_choose_local_fallback_tries,
+    _step_set_msr_descents,
+    _step_set_msr_collision_tries,
     _step_choose,
     _step_chooseleaf,
     _step_emit,
@@ -91,6 +93,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_tries> >    step_set_chooseleaf_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_vary_r> >    step_set_chooseleaf_vary_r;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_stable> >    step_set_chooseleaf_stable;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_descents> >    step_set_msr_descents;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_collision_tries> >    step_set_msr_collision_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose> >    step_choose;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_chooseleaf> >      step_chooseleaf;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_emit> >      step_emit;
@@ -149,6 +153,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
       step_set_chooseleaf_tries = str_p("set_chooseleaf_tries") >> posint;
       step_set_chooseleaf_vary_r = str_p("set_chooseleaf_vary_r") >> posint;
       step_set_chooseleaf_stable = str_p("set_chooseleaf_stable") >> posint;
+      step_set_msr_descents = str_p("set_msr_descents") >> posint;
+      step_set_msr_collision_tries = str_p("set_msr_collision_tries") >> posint;
       step_choose = str_p("choose")
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
@@ -165,6 +171,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_set_chooseleaf_tries |
 				step_set_chooseleaf_vary_r |
 				step_set_chooseleaf_stable |
+				step_set_msr_descents |
+				step_set_msr_collision_tries |
 				step_choose |
 				step_chooseleaf |
 				step_emit );
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 305431071029..8b3d2ad3711d 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1766,6 +1766,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
   if (crush->has_incompat_choose_args())
     features |= CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS;
+  if (crush->has_nondefault_tunables_msr())
+    features |= CEPH_FEATURE_CRUSH_MSR;
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty() || !pg_upmap_primaries.empty())

From d82257223ba9bac1bf5769b8e9c8bfbd3f98ee65 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 18 Dec 2023 12:58:15 -0800
Subject: [PATCH 1261/2492] crush: add choosemsr rule

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.h | 3 +++
 src/crush/crush.h        | 3 +++
 src/crush/grammar.h      | 6 ++++++
 3 files changed, 12 insertions(+)

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 8419a92a410d..78e38fde4c5d 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -1208,6 +1208,9 @@ class CrushWrapper {
   int set_rule_step_choose_leaf_indep(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSELEAF_INDEP, val, type);
   }
+  int set_rule_step_choose_msr(unsigned ruleno, unsigned step, int val, int type) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_MSR, val, type);
+  }
   int set_rule_step_emit(unsigned ruleno, unsigned step) {
     return set_rule_step(ruleno, step, CRUSH_RULE_EMIT, 0, 0);
   }
diff --git a/src/crush/crush.h b/src/crush/crush.h
index 236ffea419c5..bdcdc97eef2b 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -71,6 +71,9 @@ enum crush_opcodes {
 	CRUSH_RULE_SET_MSR_DESCENTS = 14,
 	/* set choose_msr_local_collision_tries */
 	CRUSH_RULE_SET_MSR_COLLISION_TRIES = 15,
+
+	/* choose variant without FIRSTN|INDEP */
+	CRUSH_RULE_CHOOSE_MSR = 16
 };
 
 /*
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 9e955091c6a8..0c9a2da7d770 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -54,6 +54,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     _step_set_msr_collision_tries,
     _step_choose,
     _step_chooseleaf,
+    _step_choose_msr,
     _step_emit,
     _step,
     _crushrule,
@@ -97,6 +98,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_collision_tries> >    step_set_msr_collision_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose> >    step_choose;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_chooseleaf> >      step_chooseleaf;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose_msr> >      step_choose_msr;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_emit> >      step_emit;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step> >      step;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_crushrule> >      crushrule;
@@ -163,6 +165,9 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
 	>> str_p("type") >> name;
+      step_choose_msr = str_p("choosemsr")
+	>> integer
+	>> str_p("type") >> name;
       step_emit = str_p("emit");
       step = str_p("step") >> ( step_take |
 				step_set_choose_tries |
@@ -175,6 +180,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_set_msr_collision_tries |
 				step_choose |
 				step_chooseleaf |
+				step_choose_msr |
 				step_emit );
       crushrule = str_p("rule") >> !name >> '{'
 				>> (str_p("id") | str_p("ruleset")) >> posint

From 2c7b6bb16dfe269cbbb1fc4809512833283a01ba Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:22:19 -0800
Subject: [PATCH 1262/2492] crush/mapper: add support for MSR types

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/mapper.c | 1070 +++++++++++++++++++++++++++++++++++++++++---
 src/crush/mapper.h |   14 +-
 2 files changed, 1016 insertions(+), 68 deletions(-)

diff --git a/src/crush/mapper.c b/src/crush/mapper.c
index 736cc6162c97..afeaffc5a8d8 100644
--- a/src/crush/mapper.c
+++ b/src/crush/mapper.c
@@ -27,6 +27,9 @@
 
 #define dprintk(args...) /* printf(args) */
 
+#define MIN(x, y) ((x) > (y) ? (y) : (x))
+#define MAX(y, x) ((x) < (y) ? (y) : (x))
+
 /*
  * Implement the core CRUSH mapping algorithm.
  */
@@ -820,65 +823,11 @@ static void crush_choose_indep(const struct crush_map *map,
 #endif
 }
 
-
-/* This takes a chunk of memory and sets it up to be a shiny new
-   working area for a CRUSH placement computation. It must be called
-   on any newly allocated memory before passing it in to
-   crush_do_rule. It may be used repeatedly after that, so long as the
-   map has not changed. If the map /has/ changed, you must make sure
-   the working size is no smaller than what was allocated and re-run
-   crush_init_workspace.
-
-   If you do retain the working space between calls to crush, make it
-   thread-local. If you reinstitute the locking I've spent so much
-   time getting rid of, I will be very unhappy with you. */
-
-void crush_init_workspace(const struct crush_map *m, void *v) {
-	/* We work by moving through the available space and setting
-	   values and pointers as we go.
-
-	   It's a bit like Forth's use of the 'allot' word since we
-	   set the pointer first and then reserve the space for it to
-	   point to by incrementing the point. */
-	struct crush_work *w = (struct crush_work *)v;
-	char *point = (char *)v;
-	__s32 b;
-	point += sizeof(struct crush_work);
-	w->work = (struct crush_work_bucket **)point;
-	point += m->max_buckets * sizeof(struct crush_work_bucket *);
-	for (b = 0; b < m->max_buckets; ++b) {
-		if (m->buckets[b] == 0)
-			continue;
-
-		w->work[b] = (struct crush_work_bucket *) point;
-		switch (m->buckets[b]->alg) {
-		default:
-			point += sizeof(struct crush_work_bucket);
-			break;
-		}
-		w->work[b]->perm_x = 0;
-		w->work[b]->perm_n = 0;
-		w->work[b]->perm = (__u32 *)point;
-		point += m->buckets[b]->size * sizeof(__u32);
-	}
-	BUG_ON((char *)point - (char *)w != m->working_size);
-}
-
-/**
- * crush_do_rule - calculate a mapping with the given input and rule
- * @map: the crush_map
- * @ruleno: the rule id
- * @x: hash input
- * @result: pointer to result vector
- * @result_max: maximum result size
- * @weight: weight vector (for map leaves)
- * @weight_max: size of weight vector
- * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
- */
-int crush_do_rule(const struct crush_map *map,
-		  int ruleno, int x, int *result, int result_max,
-		  const __u32 *weight, int weight_max,
-		  void *cwin, const struct crush_choose_arg *choose_args)
+static int crush_do_rule_no_retry(
+	const struct crush_map *map,
+	int ruleno, int x, int *result, int result_max,
+	const __u32 *weight, int weight_max,
+	void *cwin, const struct crush_choose_arg *choose_args)
 {
 	int result_len;
 	struct crush_work *cw = cwin;
@@ -1081,3 +1030,1006 @@ int crush_do_rule(const struct crush_map *map,
 
 	return result_len;
 }
+
+/// invariant through crush_msr_do_rule invocation
+struct crush_msr_input {
+	const struct crush_map *map;
+	const struct crush_rule *rule;
+	
+	const unsigned result_max;
+	
+	const unsigned weight_len;
+	const __u32 *weights;
+	
+	const int map_input;
+	const struct crush_choose_arg *choose_args;
+	
+	const unsigned msr_descents;
+	const unsigned msr_collision_tries;
+};
+
+/// encapsulates work space, invariant within an EMIT block
+struct crush_msr_workspace {
+	const unsigned start_stepno;
+	const unsigned end_stepno;
+
+	const unsigned result_len;
+
+	const struct crush_work *crush_work;
+
+	// int[end_stepno - start_stepno][result_len]
+	int **step_vecs;
+};
+
+/// encapsulates output space, invariant through crush_msr_do_rule invocation
+struct crush_msr_output {
+	const unsigned result_len;
+	unsigned returned_so_far;
+	int *out;
+};
+
+/**
+ * crush_msr_scan_config_steps
+ *
+ * Scans possibly empty sequence of CRUSH_RULE_SET_CHOOSE_MSR_*_TRIES
+ * steps at the start of the rule.  Returns index of next step.
+ * Populates *msr_descents and *msr_collision_tries (if non-null) with
+ * last matching rule.
+ * @steps: steps to scan
+ * @step_len: length of steps
+ * @msr_descents: out param for CRUSH_RULE_SET_MSR_DESCENTS
+ * @msr_collision_tries: out param for CRUSH_RULE_SET_MSR_COLLISION_TRIES
+ */
+static unsigned crush_msr_scan_config_steps(
+	const struct crush_rule_step *steps,
+	unsigned step_len,
+	unsigned *msr_descents,
+	unsigned *msr_collision_tries) {
+	unsigned stepno = 0;
+	for (; stepno < step_len; ++stepno) {
+		const struct crush_rule_step *step = &steps[stepno];
+		switch (step->op) {
+		case CRUSH_RULE_SET_MSR_DESCENTS:
+			if (msr_descents) *msr_descents = step->arg1;
+			break;
+		case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+			if (msr_collision_tries) *msr_collision_tries = step->arg1;
+			break;
+		default:
+			return stepno;
+		}
+	}
+	return stepno;
+}
+
+/// clear workspace represented by *ws
+static void crush_msr_clear_workspace(
+	struct crush_msr_workspace *ws)
+{
+	for (unsigned stepno = ws->start_stepno; stepno < ws->end_stepno;
+	     ++stepno) {
+		for (unsigned i = 0; i < ws->result_len; ++i) {
+			ws->step_vecs[stepno - ws->start_stepno][i] =
+				CRUSH_ITEM_UNDEF;
+		}
+	}
+}
+
+/**
+ * crush_msr_scan_next
+ *
+ * Validates an EMIT block of the form (TAKE CHOOSE_MSR* EMIT)
+ * If sequence is valid, populates total_children with the width
+ * of the mapping from the choose steps and next_emit with the
+ * index of the next EMIT step.
+ *
+ * @rule: rule to scan
+ * @result_max: max number of results to return
+ * @max_steps: length of longest string of choosemsr steps
+ * @return 0 if valid, -1 if there were validation errors
+ */
+static int crush_msr_scan_next(
+	const struct crush_rule *rule,
+	unsigned result_max,
+	unsigned stepno,
+	unsigned *total_children,
+	unsigned *next_emit)
+{
+	if (stepno + 1 >= rule->len) {
+		dprintk("stepno too large\n");
+		return -1;
+	}
+	if (rule->steps[stepno].op != CRUSH_RULE_TAKE) {
+		dprintk("first rule not CRUSH_RULE_TAKE\n");
+		return -1;
+	}
+	++stepno;
+
+	if (total_children) *total_children = 1;
+	for (; stepno < rule->len; ++stepno) {
+		const struct crush_rule_step *curstep =
+			&(rule->steps[stepno]);
+		if (curstep->op == CRUSH_RULE_EMIT) {
+			break;
+		}
+		if (rule->steps[stepno].op != CRUSH_RULE_CHOOSE_MSR) {
+			dprintk("found non-choose non-emit step %d\n", stepno);
+			return -1;
+		}
+		if (total_children) {
+			*total_children *= curstep->arg1 ? curstep->arg1
+				: result_max;
+		}
+	}
+	if (stepno >= rule->len) {
+		dprintk("did not find emit\n");
+		return -1;
+	}
+	if (next_emit) {
+		*next_emit = stepno;
+	}
+	return 0;
+}
+
+/**
+ * crush_msr_scan_rule
+ *
+ * MSR rules must have the form:
+ * 1) Possibly empty sequence of CRUSH_RULE_SET_CHOOSE_MSR_.*_TRIES steps
+ * 2) A sequence of EMIT blocks of the form
+ *   (TAKE CHOOSE_MSR* EMIT)*
+ *
+ * crush_msr_scan_rule validates that the form obeys the above form and
+ * popualtes max_steps with the length of the longest sequence of CHOOSE_MSR
+ * steps.
+ *
+ * crush_msr_scan_rule replicates the scan behavior of crush_msr_do_rule.
+ *
+ * @rule: rule to scan
+ * @result_max: max number of results to return
+ * @max_steps: length of longest string of choosemsr steps
+ * @return 0 if valid, -1 otherwise
+ */
+static int crush_msr_scan_rule(
+	const struct crush_rule *rule,
+	unsigned result_max,
+	unsigned *max_steps)
+{
+	if (max_steps) *max_steps = 0;
+	unsigned next_stepno = crush_msr_scan_config_steps(
+		rule->steps,
+		rule->len,
+		NULL, NULL);
+	while (next_stepno < rule->len) {
+		unsigned next_emit_stepno;
+		int r = crush_msr_scan_next(
+			rule, result_max, next_stepno,
+			NULL, &next_emit_stepno);
+		if (r < 0) return r;
+
+		if (max_steps) {
+			*max_steps = MAX(
+				*max_steps,
+				next_emit_stepno - (next_stepno + 1));
+		}
+		next_stepno = next_emit_stepno + 1;
+	}
+	return 0;
+}
+
+/// Returns true if all leaf slots in [start, end) are mapped
+static int crush_msr_leaf_vec_populated(
+	const struct crush_msr_workspace *workspace,
+	const unsigned start, const unsigned end)
+{
+	BUG_ON(start >= end);
+	BUG_ON(end > workspace->result_len);
+	BUG_ON(workspace->end_stepno <= workspace->start_stepno);
+	// we check the last step vector here because output
+	// won't be ordered by index for FIRSTN rules
+	int *leaf_vec = workspace->step_vecs[
+	  workspace->end_stepno - workspace->start_stepno - 1];
+	for (unsigned i = start; i < end; ++i) {
+		if (leaf_vec[i] == CRUSH_ITEM_UNDEF) {
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/// Returns try value to pass to crush based on index, tries, and local_tries
+static unsigned crush_msr_get_retry_value(
+	const unsigned result_max,
+	const unsigned index,
+	const unsigned msr_descents,
+	const unsigned msr_collision_tries)
+{
+	const unsigned total_index = (msr_descents * result_max) + index;
+	return (total_index << 16) + msr_collision_tries;
+}
+
+/**
+ * crush_msr_descend
+ *
+ * Descend recursively from bucket until we either hit a leaf or an
+ * interior node of type type.
+ * @input: crush input information
+ * @workspace: struct with working space
+ * @bucket: bucket from which to descend
+ * @type: target node type
+ * @tryno: top level try number, incremented with each call into crush_msr_choose
+ *         from crush_msr_do_rule
+ * @local_tryno: local collision try number, incremented with each call into
+ *               crush_msr_descend from crush_msr_choose after collision
+ * @index: mapping index
+ */
+static int crush_msr_descend(
+	const struct crush_msr_input *input,
+	const struct crush_msr_workspace *workspace,
+	const struct crush_bucket *bucket,
+	const int type,
+	const unsigned tryno,
+	const unsigned local_tryno,
+	const unsigned index)
+{
+	dprintk(" crush_msr_descend type %d tryno %d local_tryno %d index %d\n",
+		type, tryno, local_tryno, index);
+	while (1) {
+		const int child_bucket_candidate = crush_bucket_choose(
+			bucket,
+			workspace->crush_work->work[-1 - bucket->id],
+			input->map_input,
+			crush_msr_get_retry_value(
+				input->result_max,
+				index, tryno, local_tryno),
+			(input->choose_args ?
+			 &(input->choose_args[-1 - bucket->id]) : 0),
+			index);
+
+		if (child_bucket_candidate >= 0) {
+			return child_bucket_candidate;
+		}
+
+		bucket = input->map->buckets[-1 - child_bucket_candidate];
+		if (bucket->type == type) {
+			return child_bucket_candidate;
+		}
+	}
+}
+
+/**
+ * crush_msr_valid_candidate  
+ *
+ * Checks whether candidate is a valid choice given buckets already
+ * mapped for step stepno.
+ * 
+ * If candidate has already been mapped for a position in
+ * [include_start, include_end), candidate is valid.
+ *
+ * Else, if candidate has already been mapped for a position in
+ * [exclude_start, exclude_end), candidate is invalid.
+ *
+ * Otherwise, candidate is valid.
+ *
+ * @stepno: step to check
+ * @exclude_start: start of exclusion range
+ * @exclude_end: end of exlusion range
+ * @include_start: start of inclusion range
+ * @include_end: end of inclusion range
+ * @candidate: bucket to check
+ *
+ * Note, [exclude_start, exclude_end) must contain [include_start, include_end).
+ */
+static int crush_msr_valid_candidate(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned exclude_start,
+	unsigned exclude_end,
+	unsigned include_start,
+	unsigned include_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(exclude_end <= exclude_start);
+	BUG_ON(include_end <= include_start);
+
+	BUG_ON(exclude_start > include_start);
+	BUG_ON(exclude_end < include_end);
+
+	BUG_ON(exclude_end > workspace->result_len);
+
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = exclude_start; i < exclude_end; ++i) {
+		if (vec[i] == candidate) {
+			if (i >= include_start && i < include_end) {
+				dprintk(" crush_msr_valid_candidate: "
+					"candidate %d already chosen for "
+					"stride\n",
+					candidate);
+				return 1;
+			} else {
+				dprintk(" crush_msr_valid_candidate: "
+					"candidate %d collision\n",
+					candidate);
+				return 0;
+			}
+		}
+	}
+	dprintk(" crush_msr_valid_candidate: candidate %d no collision\n",
+		candidate);
+	return 1;
+}
+
+/**
+ * crush_msr_push_used
+ *
+ * See crush_msr_choose for details, used to push bucket indicies onto collision
+ * set for specified stride.  User is responsible for ensuring that
+ * [stride_start, stride_end) never holds more than stride_end - stride_start
+ * entries.
+ * @workspace: holds working space information
+ * @stepno: index of step
+ * @stride_start: start of stride
+ * @stride_end: one past end of stride
+ * @candidate: element to add to set
+ * @return 1 if added (not already present), 0 if not added due to already
+ *           being present
+ */
+static int crush_msr_push_used(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned stride_start,
+	unsigned stride_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(stride_end <= stride_start);
+	BUG_ON(stride_end > workspace->result_len);
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = stride_start; i < stride_end; ++i) {
+		if (vec[i] == candidate) {
+			return 0;
+		} else if (vec[i] == CRUSH_ITEM_UNDEF) {
+			vec[i] = candidate;
+			return 1;
+		}
+	}
+	BUG_ON("impossible");
+	return 0;
+}
+
+/**
+ * crush_msr_push_used
+ *
+ * See crush_msr_choose for details, used to pop bucket indicies from collision
+ * set for specified stride.  If an element is to be popped, crush_msr_pop_used
+ * must be called prior to pushing another element.
+ * @workspace: holds working space information
+ * @stepno: index of step
+ * @stride_start: start of stride
+ * @stride_end: one past end of stride
+ * @candidate: element to pop from set
+ */
+static void crush_msr_pop_used(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned stride_start,
+	unsigned stride_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(stride_end <= stride_start);
+	BUG_ON(stride_end > workspace->result_len);
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = stride_end; i > stride_start;) {
+		--i;
+		if (vec[i] != CRUSH_ITEM_UNDEF) {
+			BUG_ON(vec[i] != candidate);
+			vec[i] = CRUSH_ITEM_UNDEF;
+			return;
+		}
+	}
+	BUG_ON(0 == "impossible");
+}
+
+/**
+ * crush_msr_emit_result
+ *
+ * Outputs mapping result from specified position.  Position in output
+ * buffer depends on rule type -- FIRSTN outputs in output order, INDEP
+ * outputs into specified position.
+ * @output: output buffer
+ * @rule_type: CRUSH_RULE_TYPE_MSR_FIRSTN or CRUSH_RULE_TYPE_MSR_INDEP
+ * @position: mapping position
+ * @result: mapping value to output
+ */
+static void crush_msr_emit_result(
+	struct crush_msr_output *output,
+	int rule_type,
+	unsigned position,
+	int result)
+{
+	BUG_ON(position >= output->result_len);
+	BUG_ON(output->returned_so_far >= output->result_len);
+	if (rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
+		BUG_ON(output->out[output->returned_so_far] != CRUSH_ITEM_NONE);
+		output->out[(output->returned_so_far)++] = result;
+	} else {
+		BUG_ON(output->out[position] != CRUSH_ITEM_NONE);
+		output->out[position] = result;
+		++output->returned_so_far;
+	}
+	dprintk(" emit: %d, returned_so_far: %d\n",
+		result, output->returned_so_far);
+}
+
+/**
+ * crush_msr_choose
+ *
+ * Performs mapping for a single EMIT block containing CHOOSE steps
+ * [current_stepno, end_stepno) into mapping indices [start_index, end_index).
+ *
+ * Like chooseleaf, crush_msr_choose is essentially depth-first -- it chooses
+ * an item and all of the descendents under that item before moving to the
+ * next item.  Each choose step in the block gets its own workspace for
+ * collision detection.
+ *
+ * crush_msr_choose (and its recursive calls) will locally retry any bucket
+ * selections that produce a collision (up to msr_collision_tries times), but
+ * won't retry if it hits an out osd -- that's handled by calling back into
+ * crush_msr_choose up to msr_descents times.
+ *
+ * @input: crush input information
+ * @workspace: working space for this EMIT block
+ * @output: crush mapping output buffer specification
+ * @total_children: total number of children implied by the step sequence, may
+ *                  be larger than end_index - start_index.
+ * @start_index: start mapping index
+ * @end_index: end mapping index
+ * @current_stepno: first choose step
+ * @end_stepno: one past last choose step, must be an EMIT
+ * @tryno: try number, see crush_msr_do_rule
+ */
+static unsigned crush_msr_choose(
+	const struct crush_msr_input *input,
+	const struct crush_msr_workspace *workspace,
+	struct crush_msr_output *output,
+	const struct crush_bucket *bucket,
+	const unsigned total_descendants,
+	const unsigned start_index, const unsigned end_index,
+	const unsigned current_stepno, const unsigned end_stepno,
+	const unsigned tryno)
+{
+	dprintk("crush_msr_choose: bucket %d, start_index %d, end_index %d\n",
+		bucket->id, start_index, end_index);
+
+	BUG_ON(current_stepno >= input->rule->len);
+	const struct crush_rule_step *curstep =
+		&(input->rule->steps[current_stepno]);
+	BUG_ON(curstep->op != CRUSH_RULE_CHOOSE_MSR);
+
+	/* This call into crush_msr_choose is responsible, ultimately, for
+	 * populating indices [start_index, end_index).  We do this by
+	 * dividing that range into a set of strides specified in the
+	 * step -- choosemsr 4 host would dictate that the range be divided
+	 * into 4 strides.
+	 *
+	 * If the full rule is
+	 *
+	 * ...
+	 * step take root
+	 * step choosemsr 4 host
+	 * step choosemsr 2 osd
+	 * step emit
+	 *
+	 * total_descendants for the initial call would be 8 (4*2) with
+	 * num_stride=4 (4 hosts) and stride_length = 2 (2 osds per host).
+	 * For the recursive calls, total_descendants would be 2 (8 / 4),
+	 * stride_length would be 1 and num_strides would be 2.
+	 */
+
+	// choosemsr 0 host should select result_max hosts
+	const unsigned num_strides = curstep->arg1 ? curstep->arg1
+		: input->result_max;
+
+	// total_descendants is the product of the steps in the block
+	BUG_ON(total_descendants % num_strides != 0);
+	const unsigned stride_length = total_descendants / num_strides;
+
+	/* MSR steps like
+	 *
+	 * step choosemsr 4 host
+	 *
+	 * guarantee that the output mapping will be divided into at least
+	 * 4 hosts, not exactly 4 hosts.  We achieve this by ensuring that
+	 * the sets of hosts for each stride are disjoint -- a host selected
+	 * for stride 0 will not be used for any other stride.
+	 *
+	 * However, a single stride might end up using more than one host.
+	 * If an OSD on a host is marked out, crush_msr_choose will simply
+	 * skip that index when it hits it.  crush_msr_do_rule will then
+	 * call back into crush_msr_choose and eventually find another OSD
+	 * either on the same host or on another one not already used in
+	 * another stride. For this reason, a single stride may need to
+	 * remember up to stride_length entries for collision detection
+	 * purposes.
+	 *
+	 * Unfortunately, we only have stride_length entries to work with
+	 * in workspace.  Thus, prior to returning from crush_msr_choose,
+	 * we remove entries that didn't actually result in a mapping.  We
+	 * use the following undo vector to achieve this -- any strides that
+	 * didn't result in a successful mapping are set in undo to be undone
+	 * immediately prior to returning.
+	 *
+	 * Why prior to returning and not immediately?  Selecting a bucket in
+	 * a stride impacts subsequent choices as they may have collided.  In
+	 * order to limit the impact of marking an OSD out, we treat it as
+	 * collidable until the next pass.
+	 */
+	int undo[num_strides];
+	for (unsigned stride = 0; stride < num_strides; ++stride) {
+		undo[stride] = CRUSH_ITEM_UNDEF;
+	}
+
+	dprintk("crush_msr_choose: bucket %d, start_index %d, "
+		"end_index %d, stride_length %d\n",
+		bucket->id, start_index, end_index, stride_length);
+
+	unsigned mapped = 0;
+	unsigned stride_index = 0;
+	for (unsigned stride_start = start_index;
+	     stride_start < end_index;
+	     stride_start += stride_length, ++stride_index) {
+		const unsigned stride_end =
+		  MIN(stride_start + stride_length, end_index);
+    
+		// all descendants for this stride have been mapped already
+		if (crush_msr_leaf_vec_populated(
+		      workspace, stride_start, stride_end)) {
+		  continue;
+		}
+
+		int found = 0;
+		int child_bucket_candidate;
+		for (unsigned local_tryno = 0;
+		     local_tryno <= input->msr_collision_tries;
+		     ++local_tryno) {
+			child_bucket_candidate = crush_msr_descend(
+				input, workspace, bucket,
+				curstep->arg2, tryno, local_tryno,
+				stride_index);
+
+			/* candidate is valid if:
+			 * - we already chose it for this stride
+			 * - it hasn't been chosen for any stride */
+			if (crush_msr_valid_candidate(
+				    workspace,
+				    current_stepno,
+				    // Collision on elements in [start_index, end_index)
+				    start_index, end_index,
+				    // ...unless in [stride_start, stride_end)
+				    stride_start, stride_end,
+				    child_bucket_candidate)) {
+				found = 1;
+				break;
+			}
+		}
+
+		/* failed to find non-colliding choice after msr_collision_tries
+		 * attempts */
+		if (!found) continue;
+
+		if (curstep->arg2 == 0 /* leaf */) {
+			if (stride_length != 1 ||
+			    (current_stepno + 1 != end_stepno)) {
+				/* Either condition above implies that there's
+				 * another step after a choosemsr step for the
+				 * leaf type, rule is malformed, bail */
+				continue;
+			}
+			if (is_out(input->map, input->weights,
+				   input->weight_len,
+				   child_bucket_candidate, input->map_input)) {
+				dprintk(" crush_msr_choose: item %d out\n",
+					child_bucket_candidate);
+				/* crush_msr_do_rule will try again,
+				 * msr_descents permitting */
+				continue;
+			}
+			// for collision detection
+			int pushed = crush_msr_push_used(
+				workspace, current_stepno, stride_start, stride_end,
+				child_bucket_candidate);
+			/* stride_length == 1, can't already be there */
+			BUG_ON(!pushed);
+			// final output, ordering depending on input->rule->type
+			crush_msr_emit_result(
+				output, input->rule->type,
+				stride_start, child_bucket_candidate);
+			mapped++;
+		} else /* not leaf */ {
+			if (current_stepno + 1 >= end_stepno) {
+				/* Type isn't leaf, rule is malformed since there
+				 * isn't another step */
+				continue;
+			}
+			struct crush_bucket *child_bucket = input->map->buckets[
+				-1 - child_bucket_candidate];
+			unsigned child_mapped = crush_msr_choose(
+				input, workspace, output,
+				child_bucket,
+				// total_descendants for recursive call
+				stride_length,
+				// recursive call populates
+				// [stride_start, stride_end)
+				stride_start, stride_end,
+				// next step
+				current_stepno + 1, end_stepno,
+				tryno);
+			int pushed = crush_msr_push_used(
+				workspace,
+				current_stepno,
+				stride_start,
+				stride_end,
+				child_bucket_candidate);
+			/* pushed may be false if we already chose this bucket
+			 * for this stride.  If so, child_mapped must have been
+			 * != 0 at the time, so we still retain it */
+			if (pushed && (child_mapped == 0)) {
+				// no child mapped, and we didn't choose it
+				// before
+				undo[stride_index] = child_bucket_candidate;
+			} else {
+				mapped += child_mapped;
+			}
+		}
+	}
+
+	// pop unused buckets
+	stride_index = 0;
+	for (unsigned stride_start = start_index;
+	     stride_start < end_index;
+	     stride_start += stride_length, ++stride_index) {
+		if (undo[stride_index] != CRUSH_ITEM_UNDEF) {
+			unsigned stride_end =
+			  MIN(stride_start + stride_length, end_index);
+			crush_msr_pop_used(
+				workspace,
+				current_stepno,
+				stride_start,
+				stride_end,
+				undo[stride_index]);
+		}
+	}
+  
+	return mapped;
+}
+
+/**
+ * crush_msr_do_rule - calculate a mapping with the given input and msr rule
+ *
+ * msr_firstn and msr_indep rules are intended to address a limitation of
+ * conventional crush rules in that they do not retry steps outside of
+ * a CHOOSELEAF step.  In the case of a crush rule like
+ *
+ * rule replicated_rule_1 {
+ *   ...
+ *   step take default class hdd
+ *   step chooseleaf firstn 3 type host
+ *   step emit
+ * }
+ *
+ * the chooseleaf step will ensure that if all of the osds on a
+ * particular host are marked out, mappings including those OSDs would
+ * end up on another host (provided that there are enough hosts).
+ *
+ * However, if the rule used two choose steps instead
+ *
+ * rule replicated_rule_1 {
+ *   ...
+ *   step take default class hdd
+ *   step choose firstn 3 type host
+ *   step choose firstn 1 type osd
+ *   step emit
+ * }
+ *
+ * marking an OSD down could cause it to be remapped to another on the same
+ * host, but not to another host.  If all of the OSDs on a host are marked
+ * down, the PGs will simply be degraded and unable to remap until the host
+ * is removed from the CRUSH heirarchy or reweighted to 0.
+ *
+ * Normally, we can comfortably work around this by using a chooseleaf
+ * step as in the first example, but there are cases where we want to map
+ * multiple OSDs to each host (wide EC codes on small clusters, for
+ * example) which can't be handled with chooseleaf as it currently
+ * exists.
+ *
+ * rule ecpool-86 {
+ *   type msr_indep
+ *   ...
+ *   step choosemsr 4 type host
+ *   step choosemsr 4 type osd
+ *   step emit
+ * }
+ *
+ * With an 8+6 code, this rule can tolerate a host and a single OSD down without
+ * becoming unavailable on 4 hosts.  It relies on ensuring that no more than 4
+ * OSDs are mapped to any single host, however, which can't be done with a
+ * conventional CRUSH rule without the drawback described above.  By using
+ * msr_indep, this rule can deal with an OSD failure by remapping to another
+ * host.
+ *
+ * MSR rules have some structural differences from conventional rules:
+ * - The rule type determines whether the mapping is FIRSTN or INDEP.  Because
+ *   the descent can retry steps, it doesn't really make sense for steps to
+ *   individually specify output order and I'm not really aware of any use cases
+ *   that would benefit from it.
+ * - MSR rules *must* be structured as a (possibly empty) prefix of config
+ *   steps (CRUSH_RULE_SET_CHOOSE_MSR*) followed by a sequence of EMIT blocks
+ *   each comprised of a TAKE step, a sequence of CHOOSE_MSR steps, and
+ *   ended by an EMIT step.
+ * - MSR choose steps must be choosemsr.  choose and chooseleaf are not permitted.
+ *
+ * MSR rules also have different requirements for working space.  Conventional CRUSH
+ * requires 3 vectors of size result_max to use for working space -- two to alternate
+ * as it processes each rule and one, additionally, for chooseleaf.  MSR rules
+ * need N vectors where N is the number of choosemsr in the longest EMIT block since
+ * it needs to retain all of the choices made as part of each descent.
+ *
+ * See crush_msr_choose for details.
+ *
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
+ */
+static int crush_msr_do_rule(
+	const struct crush_map *map,
+	int ruleno, int map_input, int *result, int result_max,
+	const __u32 *weight, int weight_max,
+	void *cwin, const struct crush_choose_arg *choose_args)
+{
+	unsigned msr_descents = map->msr_descents;
+	unsigned msr_collision_tries = map->msr_collision_tries;
+	struct crush_rule *rule = map->rules[ruleno];
+	unsigned start_stepno = crush_msr_scan_config_steps(
+		rule->steps, rule->len,
+		&msr_descents, &msr_collision_tries);
+
+	struct crush_msr_input input = {
+		.map = map,
+		.rule = map->rules[ruleno],
+		.result_max = result_max,
+		.weight_len = weight_max,
+		.weights = weight,
+		.map_input = map_input,
+		.choose_args = choose_args,
+		.msr_descents = msr_descents,
+		.msr_collision_tries = msr_collision_tries 
+	};
+
+	struct crush_msr_output output = {
+		.result_len = result_max,
+		.returned_so_far = 0,
+		.out = result
+	};
+	for (unsigned i = 0; i < output.result_len; ++i) {
+		output.out[i] = CRUSH_ITEM_NONE;
+	}
+
+	unsigned start_index = 0;
+	while (start_stepno < input.rule->len) {
+		unsigned emit_stepno, total_children;
+		if (crush_msr_scan_next(
+			    input.rule, input.result_max,
+			    start_stepno, &total_children,
+			    &emit_stepno) != 0) {
+			// invalid rule, return whatever we have
+			dprintk("crush_msr_scan_returned -1\n");
+			return 0;
+		}
+
+		const struct crush_rule_step *take_step =
+			&(input.rule->steps[start_stepno]);
+		BUG_ON(take_step->op != CRUSH_RULE_TAKE);
+
+		if (take_step->arg1 >= 0) {
+			if (start_stepno + 1 != emit_stepno) {
+				// invalid rule
+				dprintk("take step specifies osd, but "
+					"there are subsequent choose steps\n");
+				return 0;
+			} else {
+				crush_msr_emit_result(
+					&output, input.rule->type,
+					start_index, take_step->arg1);
+			}
+		} else {
+			dprintk("start_stepno %d\n", start_stepno);
+			dprintk("root bucket: %d\n",
+				input.rule->steps[start_stepno].arg1);
+			struct crush_bucket *root_bucket = input.map->buckets[
+				-1 - input.rule->steps[start_stepno].arg1];
+			dprintk(
+				"root bucket: %d %p\n",
+				input.rule->steps[start_stepno].arg1,
+				root_bucket);
+
+			++start_stepno;
+			BUG_ON(emit_stepno >= input.rule->len);
+			BUG_ON(emit_stepno < start_stepno);
+			BUG_ON(start_stepno >= input.rule->len);
+
+			struct crush_work *cw = cwin;
+			int *out_vecs[input.rule->len];
+			for (unsigned stepno = 0; stepno < input.rule->len; ++stepno) {
+				out_vecs[stepno] = (int*)((char*)cw + map->working_size) +
+					(stepno * result_max);
+			}
+			struct crush_msr_workspace workspace = {
+				.start_stepno = start_stepno,
+				.end_stepno = emit_stepno,
+				.result_len = result_max,
+				.crush_work = cw,
+				.step_vecs = out_vecs
+			};
+			crush_msr_clear_workspace(&workspace);
+
+
+			unsigned tries_so_far = 0;
+			unsigned end_index = MIN(start_index + total_children,
+						 input.result_max);
+			while (tries_so_far <= input.msr_descents &&
+			       output.returned_so_far < input.result_max) {
+				crush_msr_choose(
+					&input, &workspace, &output,
+					root_bucket,
+					total_children,
+					start_index,
+					end_index,
+					start_stepno, emit_stepno,
+					tries_so_far);
+				dprintk("returned_so_far: %d\n",
+					output.returned_so_far);
+				++tries_so_far;
+			}
+			start_index = end_index;
+			start_stepno = emit_stepno + 1;
+		}
+	}
+
+	if (rule->type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
+	  return output.returned_so_far;
+	} else {
+	  return input.result_max;
+	}
+}
+
+/// Return 1 if msr, 0 otherwise
+static int rule_type_is_msr(int type)
+{
+	return type == CRUSH_RULE_TYPE_MSR_FIRSTN ||
+		type == CRUSH_RULE_TYPE_MSR_INDEP;
+}
+
+size_t crush_work_size(const struct crush_map *map,
+		       int result_max)
+{
+	unsigned ruleno;
+	unsigned out_vecs = 3; /* normal do_rule needs 3 outvecs */
+	for (ruleno = 0; ruleno < map->max_rules; ++ruleno) {
+		const struct crush_rule *rule = map->rules[ruleno];
+		if (!rule) continue;
+		if (!rule_type_is_msr(rule->type))
+			continue;
+		unsigned rule_max_msr_steps;
+		// we ignore the return value because rule_max_msr_steps will be
+		// populated with the longest step sequence before hitting
+		// the error
+		crush_msr_scan_rule(rule, result_max, &rule_max_msr_steps);
+		out_vecs = MAX(rule_max_msr_steps, out_vecs);
+	}
+	return map->working_size + result_max * out_vecs * sizeof(__u32);
+}
+
+/* This takes a chunk of memory and sets it up to be a shiny new
+   working area for a CRUSH placement computation. It must be called
+   on any newly allocated memory before passing it in to
+   crush_do_rule. It may be used repeatedly after that, so long as the
+   map has not changed. If the map /has/ changed, you must make sure
+   the working size is no smaller than what was allocated and re-run
+   crush_init_workspace.
+
+   If you do retain the working space between calls to crush, make it
+   thread-local. If you reinstitute the locking I've spent so much
+   time getting rid of, I will be very unhappy with you. */
+
+void crush_init_workspace(const struct crush_map *m, void *v) {
+	/* We work by moving through the available space and setting
+	   values and pointers as we go.
+
+	   It's a bit like Forth's use of the 'allot' word since we
+	   set the pointer first and then reserve the space for it to
+	   point to by incrementing the point. */
+	struct crush_work *w = (struct crush_work *)v;
+	char *point = (char *)v;
+	__s32 b;
+	point += sizeof(struct crush_work);
+	w->work = (struct crush_work_bucket **)point;
+	point += m->max_buckets * sizeof(struct crush_work_bucket *);
+	for (b = 0; b < m->max_buckets; ++b) {
+		if (m->buckets[b] == 0)
+			continue;
+
+		w->work[b] = (struct crush_work_bucket *) point;
+		switch (m->buckets[b]->alg) {
+		default:
+			point += sizeof(struct crush_work_bucket);
+			break;
+		}
+		w->work[b]->perm_x = 0;
+		w->work[b]->perm_n = 0;
+		w->work[b]->perm = (__u32 *)point;
+		point += m->buckets[b]->size * sizeof(__u32);
+	}
+	BUG_ON((char *)point - (char *)w != m->working_size);
+}
+
+/**
+ * crush_do_rule - calculate a mapping with the given input and rule
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
+ */
+int crush_do_rule(const struct crush_map *map,
+		  int ruleno, int x, int *result, int result_max,
+		  const __u32 *weight, int weight_max,
+		  void *cwin, const struct crush_choose_arg *choose_args)
+{
+	const struct crush_rule *rule;
+
+	if ((__u32)ruleno >= map->max_rules) {
+		dprintk(" bad ruleno %d\n", ruleno);
+		return 0;
+	}
+
+	rule = map->rules[ruleno];
+	if (rule_type_is_msr(rule->type)) {
+		return crush_msr_do_rule(
+			map,
+			ruleno,
+			x,
+			result,
+			result_max,
+			weight,
+			weight_max,
+			cwin,
+			choose_args);
+	} else {
+		return crush_do_rule_no_retry(
+			map,
+			ruleno,
+			x,
+			result,
+			result_max,
+			weight,
+			weight_max,
+			cwin,
+			choose_args);
+	}
+}
diff --git a/src/crush/mapper.h b/src/crush/mapper.h
index 0ec927d9e616..98c7bf11c0d9 100644
--- a/src/crush/mapper.h
+++ b/src/crush/mapper.h
@@ -77,15 +77,11 @@ extern int crush_do_rule(const struct crush_map *map,
 			 const __u32 *weights, int weight_max,
 			 void *cwin, const struct crush_choose_arg *choose_args);
 
-/* Returns the exact amount of workspace that will need to be used
-   for a given combination of crush_map and result_max. The caller can
-   then allocate this much on its own, either on the stack, in a
-   per-thread long-lived buffer, or however it likes. */
-
-static inline size_t crush_work_size(const struct crush_map *map,
-				     int result_max) {
-	return map->working_size + result_max * 3 * sizeof(__u32);
-}
+/* Returns enough workspace for any crush rule within map to generate
+   result_max outputs. The caller can then allocate this much on its own,
+   either on the stack, in a per-thread long-lived buffer, or however it likes.*/
+extern size_t crush_work_size(const struct crush_map *map,
+			      int result_max);
 
 extern void crush_init_workspace(const struct crush_map *m, void *v);
 

From 1199166c4517fbacaf5aa458252f481ae641db9d Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 21 Nov 2023 15:39:23 -0800
Subject: [PATCH 1263/2492] mon/OSDMonitor: generalize rule type check for
 pools

Add rule_valid_for_pool_type to CrushWrapper to generalize
rule type <-> pool type mapping to include the new MSR
types.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.h | 15 +++++++++++++++
 src/mon/OSDMonitor.cc    |  6 +++---
 src/osd/OSDMap.cc        |  2 +-
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 78e38fde4c5d..9cb1c487de95 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -602,6 +602,21 @@ class CrushWrapper {
     if (have_rmaps)
       rule_name_rmap[name] = i;
   }
+  bool rule_valid_for_pool_type(int rule_id, int ptype) const {
+    auto rule_type = get_rule_type(rule_id);
+    switch (ptype) {
+    case CEPH_PG_TYPE_REPLICATED:
+      return rule_type == CRUSH_RULE_TYPE_REPLICATED ||
+	rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN;
+    case CEPH_PG_TYPE_ERASURE:
+      return rule_type == CRUSH_RULE_TYPE_ERASURE ||
+	rule_type == CRUSH_RULE_TYPE_MSR_INDEP;
+    default:
+      ceph_assert(0 == "impossible");
+      return false;
+    }
+  }
+
   bool is_shadow_item(int id) const {
     const char *name = get_item_name(id);
     return name && !is_valid_crush_name(name);
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 05fabca0d09a..696d7f3185b3 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -8078,7 +8078,7 @@ int OSDMonitor::prepare_new_pool(string& name,
     return r;
   }
 
-  if (osdmap.crush->get_rule_type(crush_rule) != (int)pool_type) {
+  if (!osdmap.crush->rule_valid_for_pool_type(crush_rule, pool_type)) {
     *ss << "crush rule " << crush_rule << " type does not match pool";
     return -EINVAL;
   }
@@ -8350,7 +8350,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
 	return -EPERM;
       }
     }
-    if (osdmap.crush->get_rule_type(p.get_crush_rule()) != (int)p.type) {
+    if (!osdmap.crush->rule_valid_for_pool_type(p.get_crush_rule(), p.type)) {
       ss << "crush rule " << p.get_crush_rule() << " type does not match pool";
       return -EINVAL;
     }
@@ -8583,7 +8583,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
       ss << cpp_strerror(id);
       return -ENOENT;
     }
-    if (osdmap.crush->get_rule_type(id) != (int)p.get_type()) {
+    if (!osdmap.crush->rule_valid_for_pool_type(id, p.get_type())) {
       ss << "crush rule " << id << " type does not match pool";
       return -EINVAL;
     }
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 8b3d2ad3711d..76552333dfff 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -4530,7 +4530,7 @@ int OSDMap::validate_crush_rules(CrushWrapper *newcrush,
 	  << " but it is not present";
       return -EINVAL;
     }
-    if (newcrush->get_rule_type(ruleno) != (int)pool.get_type()) {
+    if (!newcrush->rule_valid_for_pool_type(ruleno, pool.get_type())) {
       *ss << "pool " << i.first << " type does not match rule " << ruleno;
       return -EINVAL;
     }

From 9adedac20d951a260deafe339aa7efaabe896b7a Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:24:30 -0800
Subject: [PATCH 1264/2492] test/crush: convert indep test cases to test MSR as
 well

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 285 ++++++++++++++++++++++++++++++----------
 1 file changed, 216 insertions(+), 69 deletions(-)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 1f53084a70af..11335643eecf 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -18,69 +18,11 @@
 #include "include/stringify.h"
 
 #include "crush/CrushWrapper.h"
+#include "crush/CrushCompiler.h"
 #include "osd/osd_types.h"
 
 using namespace std;
 
-std::unique_ptr<CrushWrapper> build_indep_map(CephContext *cct, int num_rack,
-                              int num_host, int num_osd)
-{
-  std::unique_ptr<CrushWrapper> c(new CrushWrapper);
-  c->create();
-
-  c->set_type_name(5, "root");
-  c->set_type_name(4, "row");
-  c->set_type_name(3, "rack");
-  c->set_type_name(2, "chasis");
-  c->set_type_name(1, "host");
-  c->set_type_name(0, "osd");
-
-  int rootno;
-  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
-		5, 0, NULL, NULL, &rootno);
-  c->set_item_name(rootno, "default");
-
-  map<string,string> loc;
-  loc["root"] = "default";
-
-  int osd = 0;
-  for (int r=0; r<num_rack; ++r) {
-    loc["rack"] = string("rack-") + stringify(r);
-    for (int h=0; h<num_host; ++h) {
-      loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
-      for (int o=0; o<num_osd; ++o, ++osd) {
-	c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
-      }
-    }
-  }
-  int ret;
-  int ruleno = 0;
-  ret = c->add_rule(ruleno, 4, 123);
-  ceph_assert(ret == ruleno);
-  ret = c->set_rule_step(ruleno, 0, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 1, CRUSH_RULE_TAKE, rootno, 0);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 2, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 3, CRUSH_RULE_EMIT, 0, 0);
-  ceph_assert(ret == 0);
-  c->set_rule_name(ruleno, "data");
-
-  c->finalize();
-
-  if (false) {
-    Formatter *f = Formatter::create("json-pretty");
-    f->open_object_section("crush_map");
-    c->dump(f);
-    f->close_section();
-    f->flush(cout);
-    delete f;
-  }
-
-  return c;
-}
-
 int get_num_dups(const vector<int>& v)
 {
   std::set<int> s;
@@ -94,7 +36,21 @@ int get_num_dups(const vector<int>& v)
   return dups;
 }
 
-class CRUSHTest : public ::testing::Test
+class RuleType {
+  bool msr;
+
+public:
+  RuleType(bool msr) : msr(msr) {}
+
+  bool is_msr() const { return msr; }
+  
+  friend std::ostream &operator<<(std::ostream &, RuleType);
+};
+std::ostream &operator<<(std::ostream &lhs, RuleType rhs) {
+  return lhs << (rhs.msr ? "MSR" : "NORMAL");
+}
+
+class IndepTest : public ::testing::TestWithParam<RuleType>
 {
 public:
   void SetUp() final
@@ -108,11 +64,91 @@ class CRUSHTest : public ::testing::Test
     cct->put();
     cct = nullptr;
   }
+
+  std::unique_ptr<CrushWrapper> build_indep_map(
+    CephContext *cct, int num_rack, int num_host, int num_osd)
+  {
+    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
+    c->create();
+    c->set_tunables_optimal();
+
+    c->set_type_name(5, "root");
+    c->set_type_name(4, "row");
+    c->set_type_name(3, "rack");
+    c->set_type_name(2, "chasis");
+    c->set_type_name(1, "host");
+    c->set_type_name(0, "osd");
+
+    int rootno;
+    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		  5, 0, NULL, NULL, &rootno);
+    c->set_item_name(rootno, "default");
+
+    map<string,string> loc;
+    loc["root"] = "default";
+
+    int osd = 0;
+    for (int r=0; r<num_rack; ++r) {
+      loc["rack"] = string("rack-") + stringify(r);
+      for (int h=0; h<num_host; ++h) {
+	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+	for (int o=0; o<num_osd; ++o, ++osd) {
+	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+	}
+      }
+    }
+    int ret;
+    int ruleno = 0;
+
+    if (GetParam().is_msr()) {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    } else {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    }
+
+    c->set_rule_name(ruleno, "data");
+    c->finalize();
+
+    if (false) {
+      Formatter *f = Formatter::create("json-pretty");
+      f->open_object_section("crush_map");
+      c->dump(f);
+      f->close_section();
+      f->flush(cout);
+      delete f;
+    }
+
+    return c;
+  }
+
 protected:
   CephContext *cct = nullptr;
 };
 
-TEST_F(CRUSHTest, indep_toosmall) {
+TEST_P(IndepTest, toosmall) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 1, 3, 1));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, NULL);
@@ -131,7 +167,7 @@ TEST_F(CRUSHTest, indep_toosmall) {
   }
 }
 
-TEST_F(CRUSHTest, indep_basic) {
+TEST_P(IndepTest, basic) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, NULL);
@@ -150,7 +186,88 @@ TEST_F(CRUSHTest, indep_basic) {
   }
 }
 
-TEST_F(CRUSHTest, indep_out_alt) {
+TEST_P(IndepTest, single_out_first) {
+  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+
+    int num_none = 0;
+    for (unsigned i=0; i<out.size(); ++i) {
+      if (out[i] == CRUSH_ITEM_NONE)
+	num_none++;
+    }
+    ASSERT_EQ(0, num_none);
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[0]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 5, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    // First item should have been remapped
+    ASSERT_NE(CRUSH_ITEM_NONE, out2[0]);
+    ASSERT_NE(out[0], out2[0]);
+    for (unsigned i=1; i<out.size(); ++i) {
+      // but none of the others
+      ASSERT_EQ(out[i], out2[i]);
+    }
+    ASSERT_EQ(0, get_num_dups(out2));
+  }
+}
+
+TEST_P(IndepTest, single_out_last) {
+  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+
+    int num_none = 0;
+    for (unsigned i=0; i<out.size(); ++i) {
+      if (out[i] == CRUSH_ITEM_NONE)
+	num_none++;
+    }
+    ASSERT_EQ(0, num_none);
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    unsigned last = out.size() - 1;
+    weight[out[last]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 5, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    // Last
+    ASSERT_NE(CRUSH_ITEM_NONE, out2[last]);
+    ASSERT_NE(out[last], out2[last]);
+    for (unsigned i=0; i<last; ++i) {
+      // but none of the others
+      ASSERT_EQ(out[i], out2[i]);
+    }
+    ASSERT_EQ(0, get_num_dups(out2));
+  }
+}
+
+TEST_P(IndepTest, out_alt) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -176,7 +293,7 @@ TEST_F(CRUSHTest, indep_out_alt) {
   }
 }
 
-TEST_F(CRUSHTest, indep_out_contig) {
+TEST_P(IndepTest, out_contig) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -201,8 +318,7 @@ TEST_F(CRUSHTest, indep_out_contig) {
   }
 }
 
-
-TEST_F(CRUSHTest, indep_out_progressive) {
+TEST_P(IndepTest, out_progressive) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   c->set_choose_total_tries(100);
   vector<__u32> tweight(c->get_max_devices(), 0x10000);
@@ -217,8 +333,15 @@ TEST_F(CRUSHTest, indep_out_progressive) {
     for (unsigned i=0; i<weight.size(); ++i) {
       vector<int> out;
       c->do_rule(0, x, out, 7, weight, 0);
-      cout << "(" << i << "/" << weight.size() << " out) "
-	   << x << " -> " << out << std::endl;
+      cout << "(" << i << "/" << weight.size() << " out) ";
+      if (i > 0) cout << "marked out " << i - 1 << " ";
+      cout << x << " -> " << out << std::endl;
+
+      int num_none = 0;
+      for (unsigned k=0; k<out.size(); ++k) {
+	if (out[k] == CRUSH_ITEM_NONE)
+	  num_none++;
+      }
       ASSERT_EQ(0, get_num_dups(out));
 
       // make sure nothing moved
@@ -238,7 +361,6 @@ TEST_F(CRUSHTest, indep_out_progressive) {
 	    cout << " " << out[j] << " moved from " << pos[out[j]] << " to " << j << std::endl;
 	    ++moved;
 	  }
-	  //ASSERT_EQ(j, pos[out[j]]);
 	}
       }
       if (moved || changed)
@@ -260,6 +382,31 @@ TEST_F(CRUSHTest, indep_out_progressive) {
 
 }
 
+INSTANTIATE_TEST_SUITE_P(
+  IndepTest,
+  IndepTest,
+  ::testing::Values(RuleType(true), RuleType(false)),
+  testing::PrintToStringParamName());
+
+
+class CRUSHTest : public ::testing::Test
+{
+public:
+  void SetUp() final
+  {
+    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
+    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  }
+  void TearDown() final
+  {
+    cct->put();
+    cct = nullptr;
+  }
+protected:
+  CephContext *cct = nullptr;
+};
+
 TEST_F(CRUSHTest, straw_zero) {
   // zero weight items should have no effect on placement.
 

From 6276009459f7397950dfec0d8cbb1767c9c31c7f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Dec 2023 17:00:44 -0800
Subject: [PATCH 1265/2492] test/crush: add test variants for firstn rules

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 303 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 303 insertions(+)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 11335643eecf..8349d6ca8ce5 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -388,6 +388,309 @@ INSTANTIATE_TEST_SUITE_P(
   ::testing::Values(RuleType(true), RuleType(false)),
   testing::PrintToStringParamName());
 
+class FirstnTest : public ::testing::TestWithParam<RuleType>
+{
+public:
+  void SetUp() final
+  {
+    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
+    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  }
+  void TearDown() final
+  {
+    cct->put();
+    cct = nullptr;
+  }
+
+  std::unique_ptr<CrushWrapper> build_firstn_map(
+    CephContext *cct, int num_rack, int num_host, int num_osd)
+  {
+    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
+    c->create();
+    c->set_tunables_optimal();
+
+    c->set_type_name(5, "root");
+    c->set_type_name(4, "row");
+    c->set_type_name(3, "rack");
+    c->set_type_name(2, "chasis");
+    c->set_type_name(1, "host");
+    c->set_type_name(0, "osd");
+
+    int rootno;
+    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		  5, 0, NULL, NULL, &rootno);
+    c->set_item_name(rootno, "default");
+
+    map<string,string> loc;
+    loc["root"] = "default";
+
+    int osd = 0;
+    for (int r=0; r<num_rack; ++r) {
+      loc["rack"] = string("rack-") + stringify(r);
+      for (int h=0; h<num_host; ++h) {
+	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+	for (int o=0; o<num_osd; ++o, ++osd) {
+	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+	}
+      }
+    }
+    int ret;
+    int ruleno = 0;
+
+    if (GetParam().is_msr()) {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_FIRSTN);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    } else {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 0, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_FIRSTN, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    }
+
+    c->set_rule_name(ruleno, "data");
+    c->finalize();
+
+    if (false) {
+      Formatter *f = Formatter::create("json-pretty");
+      f->open_object_section("crush_map");
+      c->dump(f);
+      f->close_section();
+      f->flush(cout);
+      delete f;
+    }
+
+    return c;
+  }
+
+protected:
+  CephContext *cct = nullptr;
+};
+
+TEST_P(FirstnTest, basic) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, toosmall) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 1, 3, 1));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, single_out_first) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[0]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 3, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    ASSERT_EQ(3, out2.size());
+    ASSERT_EQ(0, get_num_dups(out2));
+    for (unsigned i=0; i<out2.size(); ++i) {
+      EXPECT_NE(out2[i], out[0]);
+    }
+    if (GetParam().is_msr()) {
+      // normal crush doesn't guarantee this reliably
+      ASSERT_EQ(out2[0], out[1]);
+      ASSERT_EQ(out2[1], out[2]);
+      ASSERT_NE(out2[2], out[0]);
+    }
+  }
+}
+
+TEST_P(FirstnTest, single_out_last) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, NULL);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[2]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 3, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    ASSERT_EQ(3, out2.size());
+    ASSERT_EQ(0, get_num_dups(out2));
+    for (unsigned i=0; i<out2.size(); ++i) {
+      EXPECT_NE(out2[i], out[2]);
+    }
+    ASSERT_EQ(out2[0], out[0]);
+    ASSERT_EQ(out2[1], out[1]);
+    ASSERT_NE(out2[2], out[2]);
+  }
+}
+
+TEST_P(FirstnTest, out_alt) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; i<num / 2; ++i)
+    weight[i*2] = 0;
+  c->dump_tree(&cout, NULL);
+
+  // need more retries to get 9/9 hosts for x in 0..99
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 9, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    ASSERT_EQ(9, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, out_contig) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; i<num / 3; ++i)
+    weight[i] = 0;
+  c->dump_tree(&cout, NULL);
+
+  // need more retries to get 7/7 hosts for x in 0..99
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 7, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    ASSERT_EQ(6, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, out_progressive) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  vector<__u32> tweight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, NULL);
+
+  int tchanged = 0;
+  for (int x = 1; x < 5; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+    std::set<int> prev;
+    for (unsigned i=0; i<weight.size(); ++i) {
+      vector<int> out;
+      c->do_rule(0, x, out, 7, weight, 0);
+      cout << "(" << i << "/" << weight.size() << " out) ";
+      if (i > 0) cout << "marked out " << i - 1 << " ";
+      cout << x << " -> " << out << std::endl;
+
+      ASSERT_EQ(0, get_num_dups(out));
+
+      int changed = 0;
+      for (unsigned j=0; j<out.size(); ++j) {
+	if (i && prev.count(out[j]) == 0) {
+	  ++changed;
+	  ++tchanged;
+	}
+      }
+      if (changed)
+	cout << " " << changed << " changed" << std::endl;
+      ASSERT_LE(changed, 3);
+
+      // mark another osd out
+      weight[i] = 0;
+      prev = std::set<int>{out.begin(), out.end()};
+    }
+  }
+  cout << tchanged << " total changed" << std::endl;
+}
+
+INSTANTIATE_TEST_SUITE_P(
+  FirstnTest,
+  FirstnTest,
+  ::testing::Values(RuleType(true), RuleType(false)),
+  testing::PrintToStringParamName());
 
 class CRUSHTest : public ::testing::Test
 {

From 845d91d6d5276496c997c541a904f899760d0c3d Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:26:07 -0800
Subject: [PATCH 1266/2492] test/crush: add tests specifically for MSR

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 456 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 456 insertions(+)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 8349d6ca8ce5..9e2a2c99fd22 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -1103,3 +1103,459 @@ TEST_F(CRUSHTest, straw2_reweight) {
     cout << "     vs " << estddev << std::endl;
   }
 }
+
+struct cluster_test_spec_t {
+  const int num_osds_per_host;
+  const int num_hosts;
+
+  const int num_hosts_mapped;
+  const int num_mapped_per_host;
+  const int num_mapped_size;
+
+  const int num_osds;
+
+  cluster_test_spec_t(
+    int num_osds_per_host, int num_hosts,
+    int num_hosts_mapped, int num_mapped_per_host, int num_mapped_size)
+    : num_osds_per_host(num_osds_per_host), num_hosts(num_hosts),
+      num_hosts_mapped(num_hosts_mapped),
+      num_mapped_per_host(num_mapped_per_host),
+      num_mapped_size(num_mapped_size),
+      num_osds(num_osds_per_host * num_hosts) {}
+
+  void validate_osd(int osd) const {
+    EXPECT_GE(osd, 0);
+    EXPECT_LT(osd, num_osds);
+  }
+
+  bool check_osd(int osd) const {
+    return osd >= 0 && osd < num_osds;
+  }
+
+  void validate_host(int host) const {
+    assert(host >= 0);
+    assert(host < num_hosts);
+  }
+
+  std::pair<int, int> host_to_osd_range(int host) const {
+    validate_host(host);
+    auto first = host * num_osds_per_host;
+    return std::make_pair(first, first + num_osds_per_host);
+  }
+
+  int osd_to_host(int osd) const {
+    validate_osd(osd);
+    return osd / num_osds_per_host;
+  }
+};
+
+static constexpr int ROOT_TYPE = 2;
+static constexpr int HOST_TYPE = 1;
+static constexpr int OSD_TYPE = 0;
+std::pair<int, std::unique_ptr<CrushWrapper>> create_crush_heirarchy(
+  CephContext *cct,
+  const cluster_test_spec_t &spec)
+{
+  auto c = std::make_unique<CrushWrapper>();
+  c->create();
+  c->set_tunables_optimal();
+
+  
+  c->set_type_name(ROOT_TYPE, "root");
+  c->set_type_name(HOST_TYPE, "host");
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
+	       ROOT_TYPE, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  for (auto host_id = 0; host_id < spec.num_hosts; ++host_id) {
+    const std::string host_name = fmt::format("host{}", host_id);
+    const auto first_host_osd = host_id * spec.num_osds_per_host;
+    const auto next_first_host_osd = first_host_osd + spec.num_osds_per_host;
+    for (auto osd_id = first_host_osd; osd_id < next_first_host_osd; ++osd_id) {
+      const std::string osd_name = fmt::format("osd{}", osd_id);
+      auto ret = c->insert_item(
+	cct, osd_id, 1.0, osd_name,
+	{{ "root", "default"}, {"host", host_name}});
+      EXPECT_EQ(ret, 0);
+    }
+  }
+
+  c->finalize();
+  return std::make_pair(rootno, std::move(c));
+}
+
+std::vector<uint32_t> create_weight_vector(
+  const cluster_test_spec_t &spec)
+{
+  return std::vector<uint32_t>(spec.num_osds, CEPH_OSD_IN);
+}
+
+std::vector<uint32_t> create_weight_vector_first_osd_out(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &mapping)
+{
+  auto weights = create_weight_vector(spec);
+  spec.validate_osd(mapping[0]);
+  weights[mapping[0]] = CEPH_OSD_OUT;
+  return weights;
+}
+
+std::vector<uint32_t> create_weight_vector_first_host_out(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &mapping)
+{
+  auto weights = create_weight_vector(spec);
+  const auto [first, end] = spec.host_to_osd_range(spec.osd_to_host(mapping[0]));
+  for (auto i = first; i < end; ++i) {
+    weights[i] = CEPH_OSD_OUT;
+  }
+  return weights;
+}
+
+enum class mapping_change_t {
+  SAME,
+  FAILURE,
+  SAME_HOST,
+  NEW_HOST
+};
+void compare_mappings(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &before,
+  const std::vector<int> &after,
+  mapping_change_t expectation,
+  const std::pair<int, int> &range)
+{
+  const auto &[begin, end] = range;
+  for (auto i = begin; i < end; ++i) {
+    switch (expectation) {
+    case mapping_change_t::SAME:
+      EXPECT_EQ(before[i], after[i]);
+      break;
+    case mapping_change_t::FAILURE:
+      EXPECT_EQ(CRUSH_ITEM_NONE, after[i]);
+      break;
+    case mapping_change_t::SAME_HOST:
+      EXPECT_NE(before[i], after[i]);
+      if (!spec.check_osd(after[i])) {
+	spec.validate_osd(after[i]);
+      } else {
+	EXPECT_EQ(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
+      }
+      break;
+    case mapping_change_t::NEW_HOST:
+      EXPECT_NE(before[i], after[i]);
+      if (!spec.check_osd(after[i])) {
+	spec.validate_osd(after[i]);
+      } else {
+	EXPECT_NE(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
+      }
+      break;
+    }
+  }
+}
+
+std::vector<int> get_mapping(
+  const cluster_test_spec_t &spec,
+  CrushWrapper &c,
+  const std::vector<uint32_t> &weights,
+  int ruleno)
+{
+  std::vector<int> out;
+  c.do_rule(
+    ruleno, 0 /* seed */, out, spec.num_mapped_size,
+    weights,
+    0);
+  EXPECT_EQ(std::size(out), spec.num_mapped_size);
+  return out;
+}
+
+unsigned count_mapped(const auto &v) {
+  unsigned ret = 0;
+  for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
+  return ret;
+}
+
+TEST_F(CRUSHTest, msr_4_host_2_choose_rule) {
+  cluster_test_spec_t spec{3, 4, 3, 1, 3};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
+  EXPECT_EQ(
+    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, 2, 1, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  auto count_mapped = [](const auto &v) {
+    unsigned ret = 0;
+    for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
+    return ret;
+  };
+
+  EXPECT_EQ(count_mapped(before), count_mapped(after_host_out));
+
+  auto weights_osd_out = create_weight_vector_first_osd_out(spec, before);
+  auto after_osd_out = get_mapping(spec, *c, weights_osd_out, ruleno);
+  EXPECT_EQ(count_mapped(before), count_mapped(after_osd_out));
+}
+
+TEST_F(CRUSHTest, msr_2_host_2_osd) {
+  cluster_test_spec_t spec{2, 3, 2, 2, 3};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
+  EXPECT_EQ(
+    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, 2, spec.num_mapped_per_host, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  ASSERT_EQ(count_mapped(before), 3);
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::NEW_HOST,
+    {0, spec.num_mapped_per_host});
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::SAME,
+    {spec.num_mapped_per_host, spec.num_mapped_size});
+}
+
+TEST_F(CRUSHTest, msr_5_host_8_6_ec_choose) {
+  cluster_test_spec_t spec{4, 5, 4, 4, 14};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  unsigned step_id = 0;
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, step_id++, rootno));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, step_id++, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, step_id++, spec.num_mapped_per_host, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, step_id++));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::NEW_HOST,
+    {0, spec.num_mapped_per_host});
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::SAME,
+    {spec.num_mapped_per_host, spec.num_mapped_size});
+}
+
+TEST_F(CRUSHTest, msr_multi_root) {
+  constexpr unsigned NUM_HOSTS = 4;
+  constexpr unsigned NUM_OSDS_PER_HOST = 3;
+
+  auto c = CrushWrapper();
+  c.create();
+  c.set_tunables_optimal();
+
+  c.set_type_name(ROOT_TYPE, "root");
+  c.set_type_name(HOST_TYPE, "host");
+  c.set_type_name(OSD_TYPE, "osd");
+
+  std::map<int, std::pair<std::string, std::string>> osd_id_to_host_root;
+  std::map<std::string, int> root_name_to_id;
+  std::map<std::string, std::vector<int>> host_name_to_osds;
+  unsigned next_osd_id = 0;
+
+  auto populate_root = [&](const auto &root_name) {
+    int rootno;
+    c.add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
+		 ROOT_TYPE, 0, NULL, NULL, &rootno);
+    c.set_item_name(rootno, root_name);
+    root_name_to_id[root_name] = rootno;
+
+    for (unsigned host_id = 0; host_id < NUM_HOSTS; ++host_id) {
+      const std::string host_name =
+	fmt::format("{}-host{}", root_name, host_id);
+      for (unsigned osd = 0; osd < NUM_OSDS_PER_HOST; ++osd) {
+	const int osd_id = next_osd_id++;
+	const std::string osd_name = fmt::format("{}-osd{}", root_name, osd_id);
+	auto ret = c.insert_item(
+	  cct, osd_id, 1.0, osd_name,
+	  {{ "root", root_name }, { "host", host_name }});
+	osd_id_to_host_root[osd_id] = std::make_pair(host_name, root_name);
+	host_name_to_osds[host_name].push_back(osd_id);
+	EXPECT_EQ(ret, 0);
+      }
+    }
+  };
+
+  int ruleno = 0;
+  int ret = c.add_rule(ruleno, 8, CRUSH_RULE_TYPE_MSR_INDEP);
+  ceph_assert(ret == ruleno);
+
+  unsigned step_id = 0;
+  auto populate_rule = [&](const auto &rule_name) {
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_TAKE, root_name_to_id[rule_name], 0);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, HOST_TYPE);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, OSD_TYPE);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+    ceph_assert(ret == 0);
+  };
+
+  for (const auto &root_name : { "ssd", "hdd" }) {
+    populate_root(root_name);
+    populate_rule(root_name);
+  }
+  c.set_rule_name(ruleno, "rule_name");
+  c.finalize();
+
+  constexpr unsigned ACTING_SIZE = 8;
+  constexpr unsigned OSDS_PER_ROOT = 4;
+  constexpr unsigned OSDS_PER_HOST = 2;
+  auto validate_output = [&](const auto &out) {
+    std::set<std::string> hosts;
+    for (unsigned host = 0; host < (ACTING_SIZE / OSDS_PER_HOST); ++host) {
+      std::set<std::string> hosts_this_failure_domain;
+      unsigned start = host * OSDS_PER_HOST;
+      unsigned end = (host + 1) * OSDS_PER_HOST;
+      for (unsigned i = start; i < end; ++i) {
+	EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+	EXPECT_EQ(osd_id_to_host_root.count(out[i]), 1);
+	const auto &[host_name, root_name] = osd_id_to_host_root[out[start]];
+	EXPECT_EQ(i < OSDS_PER_ROOT ? "ssd" : "hdd", root_name);
+	hosts_this_failure_domain.insert(host_name);
+      }
+      for (const auto &i: hosts_this_failure_domain) {
+	EXPECT_EQ(hosts.count(i), 0);
+	hosts.insert(i);
+      }
+    }
+  };
+
+  const std::vector<uint32_t> all_in(next_osd_id, CEPH_OSD_IN);
+  for (int x = 0; x < 1000; ++x) {
+    std::vector<int> out;
+    c.do_rule(ruleno, x, out, 8, all_in, 0);
+    EXPECT_EQ(count_mapped(out), 8);
+    validate_output(out);
+
+    {
+      std::vector<uint32_t> osds_out_weight = all_in;
+      std::set<unsigned> osd_idx_out{{1, 5}};
+      for (const auto &i: osd_idx_out) {
+	osds_out_weight[out[i]] = CEPH_OSD_OUT;
+      }
+      std::vector<int> osds_out;
+      c.do_rule(ruleno, x, osds_out, 8, osds_out_weight, 0);
+      EXPECT_EQ(count_mapped(osds_out), 8);
+      validate_output(osds_out);
+      for (unsigned i = 0; i < osds_out.size(); ++i) {
+	if (osd_idx_out.count(i)) {
+	  EXPECT_NE(osds_out[i], out[i]);
+	} else {
+	  EXPECT_EQ(osds_out[i], out[i]);
+	}
+      }
+    }
+
+    {
+      std::vector<uint32_t> hosts_out_weight = all_in;
+      std::set<unsigned> osd_ids_out;
+
+      for (const auto &i : {2, 6}) {
+	const auto &[host_name, _] = osd_id_to_host_root[out[i]];
+	for (const auto &osd_id: host_name_to_osds[host_name]) {
+	  osd_ids_out.insert(osd_id);
+	  hosts_out_weight[osd_id] = CEPH_OSD_OUT;
+	}
+      }
+
+      std::vector<int> hosts_out;
+      c.do_rule(ruleno, x, hosts_out, 8, hosts_out_weight, 0);
+      EXPECT_EQ(count_mapped(hosts_out), 8);
+      validate_output(hosts_out);
+      for (unsigned i = 0; i < hosts_out.size(); ++i) {
+	if (osd_ids_out.count(out[i])) {
+	  EXPECT_NE(hosts_out[i], out[i]);
+	} else {
+	  EXPECT_EQ(hosts_out[i], out[i]);
+	}
+      }
+    }
+  }
+}

From 791a7b1c6cb9b6b494205d359bbbd6b4e98a91c6 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 20 Nov 2023 20:43:18 -0800
Subject: [PATCH 1267/2492] vstart.sh: add --osds-per-host

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/vstart.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index 3765928aa0c3..01fc493978ae 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -176,6 +176,7 @@ zoned_enabled=0
 io_uring_enabled=0
 with_jaeger=0
 force_addr=0
+osds_per_host=0
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -599,6 +600,11 @@ case $1 in
         with_jaeger=1
         echo "with_jaeger $with_jaeger"
         ;;
+    --osds-per-host)
+        osds_per_host="$2"
+        shift
+        echo "osds_per_host $osds_per_host"
+        ;;
     *)
         usage_exit
 esac
@@ -1128,6 +1134,13 @@ start_osd() {
 [osd.$osd]
         host = $HOSTNAME
 EOF
+
+            if [ "$osds_per_host" -gt 0 ]; then
+                wconf <<EOF
+        crush location = root=default host=$HOSTNAME-$(echo "$osd / $osds_per_host" | bc)
+EOF
+            fi
+
             if [ "$spdk_enabled" -eq 1 ]; then
                 wconf <<EOF
         bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}

From 144163495b73eb5bde44819362c47dd3238f06bf Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 30 Nov 2023 22:12:19 -0800
Subject: [PATCH 1268/2492] vstart.sh: add --require-osd-and-client-version and
 --use-crush-tunables flags

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/vstart.sh | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index 01fc493978ae..23ae7ef6cc8d 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -177,6 +177,8 @@ io_uring_enabled=0
 with_jaeger=0
 force_addr=0
 osds_per_host=0
+require_osd_and_client_version=""
+use_crush_tunables=""
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -605,6 +607,16 @@ case $1 in
         shift
         echo "osds_per_host $osds_per_host"
         ;;
+    --require-osd-and-client-version)
+        require_osd_and_client_version="$2"
+        shift
+        echo "require_osd_and_client_version $require_osd_and_client_version"
+        ;;
+    --use-crush-tunables)
+        use_crush_tunables="$2"
+        shift
+        echo "use_crush_tunables $use_crush_tunables"
+        ;;
     *)
         usage_exit
 esac
@@ -1101,6 +1113,15 @@ EOF
     if [ "$crimson" -eq 1 ]; then
         $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it
     fi
+
+    if [ -n "$require_osd_and_client_version" ]; then
+        $CEPH_BIN/ceph osd set-require-min-compat-client $require_osd_and_client_version
+        $CEPH_BIN/ceph osd require-osd-release $require_osd_and_client_version --yes-i-really-mean-it
+    fi
+
+    if [ -n "$use_crush_tunables" ]; then
+        $CEPH_BIN/ceph osd crush tunables $use_crush_tunables
+    fi
 }
 
 start_osd() {

From 404e84f9aa8d20ec7232cb5b72f6a1619151d3df Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 12 Dec 2023 18:31:46 -0800
Subject: [PATCH 1269/2492] erasure-code: add support for multiple osds in a
 single failure domain

Adds support for crush-osds-per-failure-domain and
crush-num-failure-domains via MSR rules.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.cc       | 141 +++++++++++++++++++++++++++++---
 src/crush/CrushWrapper.h        |  38 ++++++++-
 src/erasure-code/ErasureCode.cc |  46 ++++++++---
 src/erasure-code/ErasureCode.h  |   2 +
 4 files changed, 200 insertions(+), 27 deletions(-)

diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 69936a53d651..4850e36f9b5c 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -2261,6 +2261,7 @@ void CrushWrapper::reweight_bucket(
 int CrushWrapper::add_simple_rule_at(
   string name, string root_name,
   string failure_domain_name,
+  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   int rno,
@@ -2332,17 +2333,19 @@ int CrushWrapper::add_simple_rule_at(
   }
   crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
   if (type)
-    crush_rule_set_step(rule, step++,
-			mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
-			CRUSH_RULE_CHOOSELEAF_INDEP,
-			CRUSH_CHOOSE_N,
-			type);
+    crush_rule_set_step(
+      rule, step++,
+      mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
+      CRUSH_RULE_CHOOSELEAF_INDEP,
+      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
+      type);
   else
-    crush_rule_set_step(rule, step++,
-			mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
-			CRUSH_RULE_CHOOSE_INDEP,
-			CRUSH_CHOOSE_N,
-			0);
+    crush_rule_set_step(
+      rule, step++,
+      mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
+      CRUSH_RULE_CHOOSE_INDEP,
+      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
+      0);
   crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
 
   int ret = crush_add_rule(crush, rule, rno);
@@ -2358,13 +2361,125 @@ int CrushWrapper::add_simple_rule_at(
 int CrushWrapper::add_simple_rule(
   string name, string root_name,
   string failure_domain_name,
+  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   ostream *err)
 {
-  return add_simple_rule_at(name, root_name, failure_domain_name, device_class,
-			    mode,
-			    rule_type, -1, err);
+  return add_simple_rule_at(
+    name, root_name, failure_domain_name, num_failure_domains,
+    device_class,
+    mode,
+    rule_type, -1, err);
+}
+
+int CrushWrapper::add_multi_osd_per_failure_domain_rule_at(
+  string name, string root_name, string failure_domain_name,
+  int num_failure_domains,
+  int osds_per_failure_domain,
+  string device_class,
+  crush_rule_type rule_type,
+  int rno,
+  ostream *err)
+{
+  if (rule_exists(name)) {
+    if (err)
+      *err << "rule " << name << " exists";
+    return -EEXIST;
+  }
+  if (rno >= 0) {
+    if (rule_exists(rno)) {
+      if (err)
+        *err << "rule with ruleno " << rno << " exists";
+      return -EEXIST;
+    }
+  } else {
+    for (rno = 0; rno < get_max_rules(); rno++) {
+      if (!rule_exists(rno))
+        break;
+    }
+  }
+  if (!name_exists(root_name)) {
+    if (err)
+      *err << "root item " << root_name << " does not exist";
+    return -ENOENT;
+  }
+  int root = get_item_id(root_name);
+  int type = 0;
+  if (failure_domain_name.length()) {
+    type = get_type_id(failure_domain_name);
+    if (type < 0) {
+      if (err)
+	*err << "unknown type " << failure_domain_name;
+      return -EINVAL;
+    }
+  }
+  if (device_class.size()) {
+    if (!class_exists(device_class)) {
+      if (err)
+	*err << "device class " << device_class << " does not exist";
+      return -EINVAL;
+    }
+    int c = get_class_id(device_class);
+    if (class_bucket.count(root) == 0 ||
+	class_bucket[root].count(c) == 0) {
+      if (err)
+	*err << "root " << root_name << " has no devices with class "
+	     << device_class;
+      return -EINVAL;
+    }
+    root = class_bucket[root][c];
+  }
+  if (rule_type != CRUSH_RULE_TYPE_MSR_INDEP &&
+      rule_type != CRUSH_RULE_TYPE_MSR_FIRSTN) {
+    if (err)
+      *err << "unknown rule_type " << rule_type;
+    return -EINVAL;
+  }
+
+  int steps = 4;
+  crush_rule *rule = crush_make_rule(steps, rule_type);
+  ceph_assert(rule);
+  int step = 0;
+  crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
+  crush_rule_set_step(rule, step++,
+		      CRUSH_RULE_CHOOSE_MSR,
+		      num_failure_domains,
+		      type);
+  crush_rule_set_step(rule, step++,
+		      CRUSH_RULE_CHOOSE_MSR,
+		      osds_per_failure_domain,
+		      0);
+  crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+
+  int ret = crush_add_rule(crush, rule, rno);
+  if(ret < 0) {
+    *err << "failed to add rule " << rno << " because " << cpp_strerror(ret);
+    return ret;
+  }
+  set_rule_name(rno, name);
+  have_rmaps = false;
+  return rno;
+}
+
+
+int CrushWrapper::add_indep_multi_osd_per_failure_domain_rule(
+  string name, string root_name,
+  string failure_domain_name,
+  int num_failure_domains,
+  int osds_per_failure_domain,
+  string device_class,
+  ostream *err)
+{
+  return add_multi_osd_per_failure_domain_rule_at(
+    name, root_name,
+    failure_domain_name,
+    num_failure_domains,
+    osds_per_failure_domain,
+    device_class,
+    CRUSH_RULE_TYPE_MSR_INDEP,
+    -1,
+    err);
 }
 
 float CrushWrapper::_get_take_weight_osd_map(int root,
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 9cb1c487de95..317f4c28bdd6 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -1232,16 +1232,52 @@ class CrushWrapper {
 
   int add_simple_rule(
     std::string name, std::string root_name, std::string failure_domain_type,
+    int num_failure_domains,
     std::string device_class, std::string mode, int rule_type,
     std::ostream *err = 0);
+  int add_simple_rule(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    std::string device_class, std::string mode, int rule_type,
+    std::ostream *err = 0) {
+    return add_simple_rule(
+      name, root_name, failure_domain_type, -1,
+      device_class, mode, rule_type, err);
+  }
+
+  int add_indep_multi_osd_per_failure_domain_rule(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    int osds_per_failure_domain,
+    int num_failure_domains,
+    std::string device_class,
+    std::ostream *err = 0);
 
   /**
    * @param rno rule[set] id to use, -1 to pick the lowest available
    */
   int add_simple_rule_at(
     std::string name, std::string root_name,
-    std::string failure_domain_type, std::string device_class, std::string mode,
+    std::string failure_domain_type,
+    int num_failure_domains,
+    std::string device_class, std::string mode,
     int rule_type, int rno, std::ostream *err = 0);
+  int add_simple_rule_at(
+    std::string name, std::string root_name,
+    std::string failure_domain_type,
+    std::string device_class, std::string mode,
+    int rule_type, int rno, std::ostream *err = 0) {
+    return add_simple_rule_at(
+      name, root_name, failure_domain_type, -1,
+      device_class, mode, rule_type, rno, err);
+  }
+
+  int add_multi_osd_per_failure_domain_rule_at(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    int osds_per_failure_domain,
+    int num_failure_domains,
+    std::string device_class,
+    crush_rule_type rule_type,
+    int rno,
+    std::ostream *err = 0);
 
   int remove_rule(int ruleno);
 
diff --git a/src/erasure-code/ErasureCode.cc b/src/erasure-code/ErasureCode.cc
index 5212baee2518..928d05f2adb0 100644
--- a/src/erasure-code/ErasureCode.cc
+++ b/src/erasure-code/ErasureCode.cc
@@ -52,6 +52,12 @@ int ErasureCode::init(
   err |= to_string("crush-failure-domain", profile,
 		   &rule_failure_domain,
 		   DEFAULT_RULE_FAILURE_DOMAIN, ss);
+  err |= to_int("crush-osds-per-failure-domain", profile,
+		&rule_osds_per_failure_domain,
+		"0", ss);
+  err |= to_int("crush-num-failure-domains", profile,
+		&rule_num_failure_domains,
+		"0", ss);
   err |= to_string("crush-device-class", profile,
 		   &rule_device_class,
 		   "", ss);
@@ -66,19 +72,33 @@ int ErasureCode::create_rule(
   CrushWrapper &crush,
   std::ostream *ss) const
 {
-  int ruleid = crush.add_simple_rule(
-    name,
-    rule_root,
-    rule_failure_domain,
-    rule_device_class,
-    "indep",
-    pg_pool_t::TYPE_ERASURE,
-    ss);
-
-  if (ruleid < 0)
-    return ruleid;
-
-  return ruleid;
+  if (rule_osds_per_failure_domain <= 1) {
+    return crush.add_simple_rule(
+      name,
+      rule_root,
+      rule_failure_domain,
+      rule_num_failure_domains,
+      rule_device_class,
+      "indep",
+      pg_pool_t::TYPE_ERASURE,
+      ss);
+  } else {
+    if (rule_num_failure_domains < 1)  {
+      if (ss) {
+	*ss << "crush-num-failure-domains " << rule_num_failure_domains 
+	    << " must be >= 1 if crush-osds-per-failure-domain specified";
+	return -EINVAL;
+      }
+    }
+    return crush.add_indep_multi_osd_per_failure_domain_rule(
+      name,
+      rule_root,
+      rule_failure_domain,
+      rule_num_failure_domains,
+      rule_osds_per_failure_domain,
+      rule_device_class,
+      ss);
+  }
 }
 
 int ErasureCode::sanity_check_k_m(int k, int m, ostream *ss)
diff --git a/src/erasure-code/ErasureCode.h b/src/erasure-code/ErasureCode.h
index c246d5dc6b67..fd6d1a41f714 100644
--- a/src/erasure-code/ErasureCode.h
+++ b/src/erasure-code/ErasureCode.h
@@ -37,6 +37,8 @@ namespace ceph {
     std::string rule_root;
     std::string rule_failure_domain;
     std::string rule_device_class;
+    int rule_osds_per_failure_domain = -1;
+    int rule_num_failure_domains = -1;
 
     ~ErasureCode() override {}
 

From e6a3b55e5c9b94104d325d7b9e9a8f66ebdd0bdc Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 11 Dec 2023 13:06:42 -0800
Subject: [PATCH 1270/2492] doc/rados/operations: add CRUSH MSR documentation

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 doc/rados/operations/crush-map-edits.rst | 30 +++++++++++++++++++++---
 doc/rados/operations/crush-map.rst       | 22 +++++++++++++++++
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/doc/rados/operations/crush-map-edits.rst b/doc/rados/operations/crush-map-edits.rst
index 46a4a4f74e87..22e7e2f3772f 100644
--- a/doc/rados/operations/crush-map-edits.rst
+++ b/doc/rados/operations/crush-map-edits.rst
@@ -419,7 +419,7 @@ centers for three-way replication, and yet another rule for erasure coding acros
 six storage devices. For a detailed discussion of CRUSH rules, see **Section 3.2**
 of `CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_.
 
-A rule takes the following form::
+A normal CRUSH rule takes the following form::
 
     rule <rulename> {
 
@@ -430,6 +430,18 @@ A rule takes the following form::
         step emit
     }
 
+CRUSH MSR rules are a distinct type of CRUSH rule which supports retrying steps
+and provides better support for configurations that require multiple OSDs within
+each failure domain.  MSR rules take the following form::
+
+    rule <rulename> {
+
+        id [a unique integer ID]
+        type [msr_indep|msr_firsn]
+        step take <bucket-name> [class <device-class>]
+        step choosemsr <N> type <bucket-type>
+        step emit
+    }
 
 ``id``
    :Description: A unique integer that identifies the rule.
@@ -441,12 +453,14 @@ A rule takes the following form::
 
 ``type``
    :Description: Denotes the type of replication strategy to be enforced by the
-                 rule.
+                 rule.  msr_firstn and msr_indep are a distinct descent algorithm
+		 which supports retrying steps within the rule and therefore
+		 multiple OSDs per failure domain.
    :Purpose: A component of the rule mask.
    :Type: String
    :Required: Yes
    :Default: ``replicated``
-   :Valid Values: ``replicated`` or ``erasure``
+   :Valid Values: ``replicated``, ``erasure``, ``msr_firstn``, ``msr_indep``
 
 
 ``step take <bucket-name> [class <device-class>]``
@@ -525,6 +539,16 @@ A rule takes the following form::
                  final CRUSH mapping transformation is therefore 1, 2, 3, 4, 5
                  → 1, 2, 6, 4, 5.
 
+``step choosemsr {num} type {bucket-type}``
+   :Description: Selects a num buckets of type bucket-type.  msr_firstn and msr_indep
+		 must use choosemsr rather than choose or chooseleaf.
+
+                 - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (as many buckets as are available).
+                 - If ``pool-num-replicas > {num} > 0``, choose that many buckets.
+   :Purpose: Choose step required for msr_firstn and msr_indep rules.
+   :Prerequisite: Follows ``step take`` and precedes ``step emit``
+   :Example: ``step choosemsr 3 type host``
+
 .. _crush-reclassify:
 
 Migrating from a legacy SSD rule to device classes
diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst
index 39151e6d4a76..e18d593253d9 100644
--- a/doc/rados/operations/crush-map.rst
+++ b/doc/rados/operations/crush-map.rst
@@ -709,6 +709,13 @@ The relevant erasure-code profile properties are as follows:
    [default: ``default``].
  * **crush-failure-domain**: the CRUSH bucket type used in the distribution of
    erasure-coded shards [default: ``host``].
+ * **crush-osds-per-failure-domain**: Maximum number of OSDs to place in each
+   failure domain -- defaults to 1.  Using a value greater than one will
+   cause a CRUSH MSR rule to be created, see below.  Must be specified if
+   crush-num-failure-domains is specified.
+ * **crush-num-failure-domains**: Number of failure domains to map.  Must be
+   specified if crush-osds-per-failure-domain is specified.  Results in
+   a CRUSH MSR rule being created.
  * **crush-device-class**: the device class on which to place data [default:
    none, which means that all devices are used].
  * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the
@@ -726,6 +733,21 @@ The relevant erasure-code profile properties are as follows:
    argument is omitted, then Ceph will create the CRUSH rule automatically.
 
 
+CRUSH MSR Rules
+---------------
+
+Creating an erasure-code profile with a crush-osds-per-failure-domain
+value greater than one will cause a CRUSH MSR rule type to be created
+instead of a normal CRUSH rule.  Normal crush rules cannot retry prior
+steps when an out OSD is encountered and rely on CHOOSELEAF steps to
+permit moving OSDs to new hosts.  However, CHOOSELEAF rules don't
+support more than a single OSD per failure domain.  MSR rules, new in
+squid, support multiple OSDs per failure domain by retrying all prior
+steps when an out OSD is encountered.  Using MSR rules requires that
+OSDs and clients be required to support the CRUSH_MSR feature bit
+(squid or newer).
+
+
 Deleting rules
 --------------
 

From 67e71ae0bf0562003c9576a89de646bd50150162 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 20 Dec 2023 19:20:32 -0800
Subject: [PATCH 1271/2492] qa/erasure-code: modify jerasure 4/2 ec test case
 to use msr

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
index dfcc61607a7d..a0cd68a55f53 100644
--- a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
+++ b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
@@ -11,7 +11,9 @@ tasks:
       k: 4
       m: 2
       technique: reed_sol_van
-      crush-failure-domain: osd
+      crush-failure-domain: host
+      crush-osds-per-failure-domain: 2
+      crush-num-failure-domains: 3
     op_weights:
       read: 100
       write: 0

From 72996a96c167ea6a85561796d0053838612611bf Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 8 Jan 2024 19:24:41 -0800
Subject: [PATCH 1272/2492] test/cli/crushtool/choose-args.t: add msr related
 json output

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/cli/crushtool/choose-args.t | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/cli/crushtool/choose-args.t b/src/test/cli/crushtool/choose-args.t
index e0956ec0a754..99120f0f211f 100644
--- a/src/test/cli/crushtool/choose-args.t
+++ b/src/test/cli/crushtool/choose-args.t
@@ -159,6 +159,8 @@
           "chooseleaf_descend_once": 0,
           "chooseleaf_vary_r": 0,
           "chooseleaf_stable": 0,
+          "msr_descents": 100,
+          "msr_collision_tries": 100,
           "straw_calc_version": 0,
           "allowed_bucket_algs": 22,
           "profile": "argonaut",
@@ -172,7 +174,8 @@
           "has_v3_rules": 0,
           "has_v4_buckets": 1,
           "require_feature_tunables5": 0,
-          "has_v5_rules": 0
+          "has_v5_rules": 0,
+          "has_msr_rules": 0
       },
       "choose_args": {
           "1": [],

From 38bd19a433a29d9a74610b76c0e90ca848f41e18 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 8 Jan 2024 19:25:05 -0800
Subject: [PATCH 1273/2492] test/cli/osdmaptool/crush.t: adjust --import-crush
 size output

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/cli/osdmaptool/crush.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/cli/osdmaptool/crush.t b/src/test/cli/osdmaptool/crush.t
index 520f11e50d5c..695c4e0c22d9 100644
--- a/src/test/cli/osdmaptool/crush.t
+++ b/src/test/cli/osdmaptool/crush.t
@@ -6,7 +6,7 @@
   osdmaptool: exported crush map to oc
   $ osdmaptool --import-crush oc myosdmap
   osdmaptool: osdmap file 'myosdmap'
-  osdmaptool: imported 497 byte crush map from oc
+  osdmaptool: imported 499 byte crush map from oc
   osdmaptool: writing epoch 3 to myosdmap
   $ osdmaptool --adjust-crush-weight 0:5 myosdmap
   osdmaptool: osdmap file 'myosdmap'

From 8431207a71aed498389e92a0d5975f2b0935f88e Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 5 Sep 2023 09:26:17 +0800
Subject: [PATCH 1274/2492] crimson/osd/pg: set obc's ssc when handling pull
 responses

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/replicated_recovery_backend.cc | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 1ff0623abdc0..9a1060c440e2 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -804,11 +804,21 @@ ReplicatedRecoveryBackend::_handle_pull_response(
         recovery_waiter.obc = obc;
         obc->obs.oi.decode_no_oid(push_op.attrset.at(OI_ATTR),
                                   push_op.soid);
-        if (push_op.attrset.contains(SS_ATTR)) {
-          bufferlist ssbl = push_op.attrset.at(SS_ATTR);
-          SnapSet ss(ssbl);
+        auto ss_attr_iter = push_op.attrset.find(SS_ATTR);
+        if (ss_attr_iter != push_op.attrset.end()) {
+          if (!obc->ssc) {
+            obc->ssc = new crimson::osd::SnapSetContext(
+              push_op.soid.get_snapdir());
+          }
+          try {
+            obc->ssc->snapset = SnapSet(ss_attr_iter->second);
+            obc->ssc->exists = true;
+          } catch (const buffer::error&) {
+            logger().warn("unable to decode SnapSet");
+            throw crimson::osd::invalid_argument();
+          }
           assert(!pull_info.obc->ssc->exists ||
-                 ss.seq == pull_info.obc->ssc->snapset.seq);
+                 obc->ssc->snapset.seq == pull_info.obc->ssc->snapset.seq);
         }
         pull_info.recovery_info.oi = obc->obs.oi;
         if (pull_info.recovery_info.soid.snap &&

From 542e0f419e42350a1678d9bb1b9aecaac923077b Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Sun, 20 Aug 2023 14:50:48 +0800
Subject: [PATCH 1275/2492] crimson/osd/object_context: assert that all
 in-cache obcs are fully loaded

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/object_context.h         | 7 +++++++
 src/crimson/osd/object_context_loader.cc | 1 +
 2 files changed, 8 insertions(+)

diff --git a/src/crimson/osd/object_context.h b/src/crimson/osd/object_context.h
index 8abf6d3f71e9..106968e60c3d 100644
--- a/src/crimson/osd/object_context.h
+++ b/src/crimson/osd/object_context.h
@@ -94,11 +94,13 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
     ceph_assert(is_head());
     obs = std::move(_obs);
     ssc = std::move(_ssc);
+    fully_loaded = true;
   }
 
   void set_clone_state(ObjectState &&_obs) {
     ceph_assert(!is_head());
     obs = std::move(_obs);
+    fully_loaded = true;
   }
 
   /// pass the provided exception to any waiting consumers of this ObjectContext
@@ -110,6 +112,10 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
     }
   }
 
+  bool is_fully_loaded() const {
+    return fully_loaded;
+  }
+
 private:
   tri_mutex lock;
   bool recovery_read_marker = false;
@@ -126,6 +132,7 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
 
   boost::intrusive::list_member_hook<> list_hook;
   uint64_t list_link_cnt = 0;
+  bool fully_loaded = false;
 
 public:
 
diff --git a/src/crimson/osd/object_context_loader.cc b/src/crimson/osd/object_context_loader.cc
index d7d2b6d98c67..f49de73f5d40 100644
--- a/src/crimson/osd/object_context_loader.cc
+++ b/src/crimson/osd/object_context_loader.cc
@@ -168,6 +168,7 @@ using crimson::common::local_conf;
     auto loaded =
       load_obc_iertr::make_ready_future<ObjectContextRef>(obc);
     if (existed) {
+      ceph_assert(obc->is_fully_loaded());
       DEBUGDPP("cache hit on {}", dpp, obc->get_oid());
     } else {
       DEBUGDPP("cache miss on {}", dpp, obc->get_oid());

From be2125127830668353615734d32aa669ed7c584b Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 24 Nov 2023 10:55:07 +0530
Subject: [PATCH 1276/2492] Revert "mgr/dashboard: allow tls 1.2 with a config
 option"

This reverts commit 219c62bea60083c0e59a86454b0cecf8afaf9780.

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/module.py   | 12 +++---------
 src/pybind/mgr/dashboard/settings.py |  2 --
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index 68725be6e355..efef273bda06 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -33,7 +33,7 @@
 from .services.exception import dashboard_exception_handler
 from .services.rgw_client import configure_rgw_credentials
 from .services.sso import SSO_COMMANDS, handle_sso_command
-from .settings import Settings, handle_option_command, options_command_list, options_schema_list
+from .settings import handle_option_command, options_command_list, options_schema_list
 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
     prepare_url_prefix, str_to_bool
 
@@ -178,15 +178,9 @@ def _configure(self):
             context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
             context.load_cert_chain(cert_fname, pkey_fname)
             if sys.version_info >= (3, 7):
-                if Settings.UNSAFE_TLS_v1_2:
-                    context.minimum_version = ssl.TLSVersion.TLSv1_2
-                else:
-                    context.minimum_version = ssl.TLSVersion.TLSv1_3
+                context.minimum_version = ssl.TLSVersion.TLSv1_3
             else:
-                if Settings.UNSAFE_TLS_v1_2:
-                    context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
-                else:
-                    context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2
+                context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2
 
             config['server.ssl_module'] = 'builtin'
             config['server.ssl_certificate'] = cert_fname
diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py
index d4e06a9cc8dc..6018f0d7f9c7 100644
--- a/src/pybind/mgr/dashboard/settings.py
+++ b/src/pybind/mgr/dashboard/settings.py
@@ -119,8 +119,6 @@ class Options(object):
                                                   'gateway', 'logs', 'crush', 'maps']),
                                         [str])
 
-    UNSAFE_TLS_v1_2 = Setting(False, [bool])
-
     @staticmethod
     def has_default_value(name):
         return getattr(Settings, name, None) is None or \

From a4b357c10d581d3c97d4a47f6210c6c00ba409fb Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 09:48:46 +0000
Subject: [PATCH 1277/2492] Revert "crimson/osd/osd_operations/snaptrim_event:
 lifetime fixes"

This reverts commit abceb1652239629ed11187a5fc670a3b1a3a5bb1.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index e90c7a213faf..64b318580a47 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -90,14 +90,7 @@ SnapTrimEvent::start()
 {
   ShardServices &shard_services = pg->get_shard_services();
   IRef ref = this;
-  return interruptor::with_interruption(
-    // SnapTrimEvent is a background operation,
-    // it's lifetime is not guarnteed since the caller
-    // returned future is being ignored. We should capture
-    // a self reference thourhgout the entire execution
-    // progress (not only on finally() continuations).
-    // See: PG::on_active_actmap()
-    [&shard_services, this, ref] {
+  return interruptor::with_interruption([&shard_services, this] {
     return enter_stage<interruptor>(
       client_pp().wait_for_active
     ).then_interruptible([this] {
@@ -200,8 +193,7 @@ SnapTrimEvent::start()
         });
       });
     });
-  }, [this, ref]
-     (std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
+  }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
   }, pg).finally([this, ref=std::move(ref)] {

From 1b85b8c1f639303ebd88f69aae3e1e8578bd2521 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 09:49:38 +0000
Subject: [PATCH 1278/2492] Revert "crimson/osd/osd_operations/snaptrim_event:
 fix lifetime on finally()"

This reverts commit 84c5b6c578c1e64807eca5098d51193ee71e6106.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index 64b318580a47..da2c6298177b 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -89,7 +89,6 @@ SnapTrimEvent::snap_trim_ertr::future<seastar::stop_iteration>
 SnapTrimEvent::start()
 {
   ShardServices &shard_services = pg->get_shard_services();
-  IRef ref = this;
   return interruptor::with_interruption([&shard_services, this] {
     return enter_stage<interruptor>(
       client_pp().wait_for_active
@@ -196,7 +195,7 @@ SnapTrimEvent::start()
   }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
-  }, pg).finally([this, ref=std::move(ref)] {
+  }, pg).finally([this] {
     logger().debug("{}: exit", *this);
     handle.exit();
   });

From 1f478ca39b3e2a97df6cd9aca02756dd5185426d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 11:33:28 +0000
Subject: [PATCH 1279/2492] crimson/osd/osd_operations/snaptrim_event: add op
 lifetime comment

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index da2c6298177b..b2e294316dbc 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -196,6 +196,8 @@ SnapTrimEvent::start()
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
   }, pg).finally([this] {
+    // This SnapTrimEvent op lifetime is maintained within
+    // PerShardState::start_operation() implementation.
     logger().debug("{}: exit", *this);
     handle.exit();
   });

From 6ec48eeb261bc8aceb1bfa3ee3732acfb0974478 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Tue, 19 Dec 2023 16:48:32 +0100
Subject: [PATCH 1280/2492] mgr/dashboard: fix e2e failure related to landing
 page

Fixes: https://tracker.ceph.com/issues/59142
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../cypress/e2e/orchestrator/04-osds.e2e-spec.ts       | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/04-osds.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/04-osds.e2e-spec.ts
index e80398d5a472..74b46054afd2 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/04-osds.e2e-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/04-osds.e2e-spec.ts
@@ -1,9 +1,19 @@
 import { OSDsPageHelper } from '../cluster/osds.po';
 import { DashboardPageHelper } from '../ui/dashboard.po';
+import { ManagerModulesPageHelper } from '../cluster/mgr-modules.po';
 
 describe('OSDs page', () => {
   const osds = new OSDsPageHelper();
   const dashboard = new DashboardPageHelper();
+  const mgrmodules = new ManagerModulesPageHelper();
+
+  before(() => {
+    cy.login();
+    mgrmodules.navigateTo();
+    mgrmodules.navigateEdit('dashboard');
+    cy.get('#FEATURE_TOGGLE_DASHBOARD').uncheck();
+    cy.contains('button', 'Update').click();
+  });
 
   beforeEach(() => {
     cy.login();

From 82561ba77c4b896d4b56aed453f0425172bd76f0 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Tue, 9 Jan 2024 08:15:34 +0000
Subject: [PATCH 1281/2492] win32_deps_build.sh: download Boost from Ceph
 mirror

We've recently changed the Boost mirror since the previous one
became unavailable.

As pointed out in PR comments, it's even better to use the Ceph
hosted source in order to save Ceph lab bandwidth.

We're adding bzip2 as a dependency. xz-utils was missing, so we're
adding it to our list.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 win32_deps_build.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/win32_deps_build.sh b/win32_deps_build.sh
index 1434d8b7ca66..491eb28093fa 100755
--- a/win32_deps_build.sh
+++ b/win32_deps_build.sh
@@ -16,8 +16,8 @@ sslDir="${depsToolsetDir}/openssl"
 sslSrcDir="${depsSrcDir}/openssl"
 
 # For now, we'll keep the version number within the file path when not using git.
-boostUrl="https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz"
-boostSha256Sum="66a469b6e608a51f8347236f4912e27dc5c60c60d7d53ae9bfe4683316c6f04c"
+boostUrl="https://download.ceph.com/qa/boost_1_82_0.tar.bz2"
+boostSha256Sum="a6e1ab9b0860e6a2881dd7b21fe9f737a095e5f33a3a874afc6a345228597ee6"
 boostSrcDir="${depsSrcDir}/boost_1_82_0"
 boostDir="${depsToolsetDir}/boost"
 zlibDir="${depsToolsetDir}/zlib"
@@ -70,6 +70,8 @@ case "$OS" in
             libtool \
             ninja-build \
             zip \
+            bzip2 \
+            xz \
             python3-PyYAML \
             gcc \
             diffutils \
@@ -83,12 +85,12 @@ case "$OS" in
         sudo env DEBIAN_FRONTEND=noninteractive apt-get -y install \
             mingw-w64 g++ cmake pkg-config \
             python3-dev python3-yaml \
-                autoconf libtool ninja-build wget zip \
+                autoconf libtool ninja-build wget xz-utils zip bzip2 \
                 git
         ;;
     suse)
         for PKG in mingw64-cross-gcc-c++ mingw64-libgcc_s_seh1 mingw64-libstdc++6 \
-                cmake pkgconf python3-devel autoconf libtool ninja zip \
+                cmake pkgconf python3-devel autoconf libtool ninja xz zip bzip2 \
                 python3-PyYAML \
                 gcc patch wget git; do
             rpm -q $PKG >/dev/null || zypper -n install $PKG
@@ -160,14 +162,14 @@ echo "Building boost."
 cd $depsSrcDir
 if [[ ! -d $boostSrcDir ]]; then
     echo "Downloading boost."
-    wget -q -O boost.tar.gz $boostUrl
-    checksum=`sha256sum boost.tar.gz | cut -d ' ' -f 1`
+    wget -q -O boost.tar.bz2 $boostUrl
+    checksum=`sha256sum boost.tar.bz2 | cut -d ' ' -f 1`
     if [[ "$boostSha256Sum" != "$checksum" ]]; then
         echo "Invalid boost checksum: $checksum" >&2
         exit 1
     fi
-    tar xzf boost.tar.gz
-    rm boost.tar.gz
+    tar -xf boost.tar.bz2
+    rm boost.tar.bz2
 fi
 
 cd $boostSrcDir

From f5d1b7df2ed090b45dd74634e3c92d4df2c4017c Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 27 Dec 2023 14:44:53 +0530
Subject: [PATCH 1282/2492] mgr/dashboard: subvolume snapshot creation form

Fixes: https://tracker.ceph.com/issues/63934
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  27 ++++
 ...fs-subvolume-snapshots-form.component.html |  98 ++++++++++++++
 ...fs-subvolume-snapshots-form.component.scss |   0
 ...subvolume-snapshots-form.component.spec.ts |  41 ++++++
 ...phfs-subvolume-snapshots-form.component.ts | 127 ++++++++++++++++++
 ...fs-subvolume-snapshots-list.component.html |  13 +-
 ...phfs-subvolume-snapshots-list.component.ts |  51 ++++++-
 .../src/app/ceph/cephfs/cephfs.module.ts      |   4 +-
 .../api/cephfs-subvolume.service.spec.ts      |   6 +
 .../shared/api/cephfs-subvolume.service.ts    |  44 ++++++
 .../src/app/shared/forms/cd-validators.ts     |   6 +-
 .../src/app/shared/pipes/cd-date.pipe.spec.ts |  14 +-
 .../src/app/shared/pipes/cd-date.pipe.ts      |  24 +++-
 .../shared/services/task-message.service.ts   |   8 ++
 src/pybind/mgr/dashboard/openapi.yaml         |  88 ++++++++++++
 15 files changed, 530 insertions(+), 21 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.ts

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 61b31eb809fd..712efe11b0dd 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -864,6 +864,33 @@ def get(self, vol_name: str, subvol_name, group_name: str = '', info=True):
                 snapshot['info'] = json.loads(out)
         return snapshots
 
+    @RESTController.Resource('GET')
+    def info(self, vol_name: str, subvol_name: str, snap_name: str, group_name: str = ''):
+        params = {'vol_name': vol_name, 'sub_name': subvol_name, 'snap_name': snap_name}
+        if group_name:
+            params['group_name'] = group_name
+        error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_info', None,
+                                          params)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to get info for subvolume snapshot {snap_name}: {err}'
+            )
+        return json.loads(out)
+
+    def create(self, vol_name: str, subvol_name: str, snap_name: str, group_name=''):
+        params = {'vol_name': vol_name, 'sub_name': subvol_name, 'snap_name': snap_name}
+        if group_name:
+            params['group_name'] = group_name
+
+        error_code, _, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_create', None,
+                                        params)
+
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to create subvolume snapshot {snap_name}: {err}'
+            )
+        return f'Subvolume snapshot {snap_name} created successfully'
+
 
 @APIRouter('/cephfs/snaphost/schedule', Scope.CEPHFS)
 @APIDoc("Cephfs Snapshot Scheduling API", "CephFSSnapshotSchedule")
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.html
new file mode 100644
index 000000000000..867ed1bbfc16
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.html
@@ -0,0 +1,98 @@
+<cd-modal [modalRef]="activeModal">
+  <ng-container i18n="form title"
+                class="modal-title">{{ action | titlecase }} {{ resource | upperFirst }}</ng-container>
+
+  <ng-container class="modal-content"
+                *cdFormLoading="loading">
+    <form name="snapshotForm"
+          #formDir="ngForm"
+          [formGroup]="snapshotForm"
+          novalidate>
+      <div class="modal-body">
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="snapshotName"
+                 i18n>Name</label>
+          <div class="cd-col-form-input">
+            <input class="form-control"
+                   type="text"
+                   placeholder="Snapshot name..."
+                   id="snapshotName"
+                   name="snapshotName"
+                   formControlName="snapshotName"
+                   autofocus>
+            <span class="invalid-feedback"
+                  *ngIf="snapshotForm.showError('snapshotName', formDir, 'required')"
+                  i18n>This field is required.</span>
+            <span class="invalid-feedback"
+                  *ngIf="snapshotForm.showError('snapshotName', formDir, 'notUnique')"
+                  i18n>The snapshot already exists.</span>
+          </div>
+        </div>
+
+        <!-- Volume name -->
+        <div class="form-group row">
+          <label class="cd-col-form-label"
+                 for="volumeName"
+                 i18n>Volume name</label>
+          <div class="cd-col-form-input">
+            <input class="form-control"
+                   id="volumeName"
+                   name="volumeName"
+                   formControlName="volumeName">
+          </div>
+        </div>
+
+        <!--Subvolume Group name -->
+        <div class="form-group row">
+          <label class="cd-col-form-label"
+                 for="subvolumeGroupName"
+                 i18n>Subvolume group
+          </label>
+          <div class="cd-col-form-input">
+            <select class="form-select"
+                    id="subvolumeGroupName"
+                    name="subvolumeGroupName"
+                    formControlName="subvolumeGroupName"
+                    #selection
+                    (change)="onSelectionChange(selection.value)"
+                    *ngIf="subVolumeGroups">
+              <ng-container *ngFor="let subvolumegroup of subVolumeGroups">
+                <option *ngIf="subvolumegroup == ''"
+                        value="">_nogroup</option>
+                <option [value]="subvolumegroup"
+                        *ngIf="subvolumegroup !== ''">{{ subvolumegroup }}</option>
+              </ng-container>
+            </select>
+          </div>
+        </div>
+
+        <!--Subvolume name -->
+        <div class="form-group row">
+          <label class="cd-col-form-label"
+                 for="subVolumeName"
+                 i18n>Subvolume
+          </label>
+          <div class="cd-col-form-input">
+            <select class="form-select"
+                    id="subVolumeName"
+                    name="subVolumeName"
+                    formControlName="subVolumeName"
+                    #selection
+                    (change)="resetValidators(selection.value)"
+                    *ngIf="subVolumes$ | async as subVolumes">
+              <option *ngFor="let subVolume of subVolumes"
+                      [value]="subVolume.name">{{ subVolume.name }}</option>
+            </select>
+          </div>
+        </div>
+      </div>
+
+      <div class="modal-footer">
+        <cd-form-button-panel (submitActionEvent)="submit()"
+                              [form]="snapshotForm"
+                              [submitText]="(action | titlecase) + ' ' + (resource | upperFirst)"></cd-form-button-panel>
+      </div>
+    </form>
+  </ng-container>
+</cd-modal>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.spec.ts
new file mode 100644
index 000000000000..a6eb923cdb2d
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.spec.ts
@@ -0,0 +1,41 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-form.component';
+import { configureTestBed } from '~/testing/unit-test-helper';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { SharedModule } from '~/app/shared/shared.module';
+import { ToastrModule } from 'ngx-toastr';
+import { ReactiveFormsModule } from '@angular/forms';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { RouterTestingModule } from '@angular/router/testing';
+
+describe('CephfsSubvolumeSnapshotsFormComponent', () => {
+  let component: CephfsSubvolumeSnapshotsFormComponent;
+  let fixture: ComponentFixture<CephfsSubvolumeSnapshotsFormComponent>;
+
+  configureTestBed({
+    declarations: [CephfsSubvolumeSnapshotsFormComponent],
+    providers: [NgbActiveModal],
+    imports: [
+      SharedModule,
+      ToastrModule.forRoot(),
+      ReactiveFormsModule,
+      HttpClientTestingModule,
+      RouterTestingModule
+    ]
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(CephfsSubvolumeSnapshotsFormComponent);
+    component = fixture.componentInstance;
+    component.fsName = 'test_volume';
+    component.subVolumeName = 'test_subvolume';
+    component.subVolumeGroupName = 'test_subvolume_group';
+    component.ngOnInit();
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.ts
new file mode 100644
index 000000000000..92757d334acd
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component.ts
@@ -0,0 +1,127 @@
+import { Component, OnInit } from '@angular/core';
+import { FormControl, Validators } from '@angular/forms';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import moment from 'moment';
+import { Observable } from 'rxjs';
+import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
+import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants';
+import { CdForm } from '~/app/shared/forms/cd-form';
+import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
+import { CdValidators } from '~/app/shared/forms/cd-validators';
+import { CephfsSubvolume } from '~/app/shared/models/cephfs-subvolume.model';
+import { FinishedTask } from '~/app/shared/models/finished-task';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+
+@Component({
+  selector: 'cd-cephfs-subvolume-snapshots-form',
+  templateUrl: './cephfs-subvolume-snapshots-form.component.html',
+  styleUrls: ['./cephfs-subvolume-snapshots-form.component.scss']
+})
+export class CephfsSubvolumeSnapshotsFormComponent extends CdForm implements OnInit {
+  fsName: string;
+  subVolumeName: string;
+  subVolumeGroupName: string;
+  subVolumeGroups: string[];
+
+  isEdit = false;
+
+  snapshotForm: CdFormGroup;
+
+  action: string;
+  resource: string;
+
+  subVolumes$: Observable<CephfsSubvolume[]>;
+
+  constructor(
+    public activeModal: NgbActiveModal,
+    private actionLabels: ActionLabelsI18n,
+    private taskWrapper: TaskWrapperService,
+    private cephFsSubvolumeService: CephfsSubvolumeService
+  ) {
+    super();
+    this.resource = $localize`snapshot`;
+    this.action = this.actionLabels.CREATE;
+  }
+
+  ngOnInit(): void {
+    this.createForm();
+
+    this.subVolumes$ = this.cephFsSubvolumeService.get(this.fsName, this.subVolumeGroupName, false);
+    this.loadingReady();
+  }
+
+  createForm() {
+    this.snapshotForm = new CdFormGroup({
+      snapshotName: new FormControl(moment().toISOString(true), {
+        validators: [Validators.required],
+        asyncValidators: [
+          CdValidators.unique(
+            this.cephFsSubvolumeService.snapshotExists,
+            this.cephFsSubvolumeService,
+            null,
+            null,
+            this.fsName,
+            this.subVolumeName,
+            this.subVolumeGroupName
+          )
+        ]
+      }),
+      volumeName: new FormControl({ value: this.fsName, disabled: true }),
+      subVolumeName: new FormControl(this.subVolumeName),
+      subvolumeGroupName: new FormControl(this.subVolumeGroupName)
+    });
+  }
+
+  onSelectionChange(groupName: string) {
+    this.subVolumeGroupName = groupName;
+    this.subVolumes$ = this.cephFsSubvolumeService.get(this.fsName, this.subVolumeGroupName, false);
+    this.subVolumes$.subscribe((subVolumes) => {
+      this.subVolumeName = subVolumes[0].name;
+      this.snapshotForm.get('subVolumeName').setValue(this.subVolumeName);
+
+      this.resetValidators();
+    });
+  }
+
+  resetValidators(subVolumeName?: string) {
+    this.subVolumeName = subVolumeName;
+    this.snapshotForm
+      .get('snapshotName')
+      .setAsyncValidators(
+        CdValidators.unique(
+          this.cephFsSubvolumeService.snapshotExists,
+          this.cephFsSubvolumeService,
+          null,
+          null,
+          this.fsName,
+          this.subVolumeName,
+          this.subVolumeGroupName
+        )
+      );
+    this.snapshotForm.get('snapshotName').updateValueAndValidity();
+  }
+
+  submit() {
+    const snapshotName = this.snapshotForm.getValue('snapshotName');
+    const subVolumeName = this.snapshotForm.getValue('subVolumeName');
+    const subVolumeGroupName = this.snapshotForm.getValue('subvolumeGroupName');
+    const volumeName = this.snapshotForm.getValue('volumeName');
+
+    this.taskWrapper
+      .wrapTaskAroundCall({
+        task: new FinishedTask('cephfs/subvolume/snapshot/' + URLVerbs.CREATE, {
+          snapshotName: snapshotName
+        }),
+        call: this.cephFsSubvolumeService.createSnapshot(
+          volumeName,
+          snapshotName,
+          subVolumeName,
+          subVolumeGroupName
+        )
+      })
+      .subscribe({
+        error: () => this.snapshotForm.setErrors({ cdSubmitButton: true }),
+        complete: () => this.activeModal.close()
+      });
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
index de3117236524..190072027bcf 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.html
@@ -26,7 +26,18 @@
               [columns]="columns"
               selectionType="single"
               [hasDetails]="false"
-              (fetchData)="fetchData()"></cd-table>
+              (fetchData)="fetchData()"
+              (updateSelection)="updateSelection($event)">
+
+      <div class="table-actions btn-toolbar">
+        <cd-table-actions [permission]="permissions.cephfs"
+                          [selection]="selection"
+                          class="btn-group"
+                          id="cephfs-snapshot-actions"
+                          [tableActions]="tableActions">
+        </cd-table-actions>
+      </div>
+    </cd-table>
   </div>
 </div>
 <ng-template #noGroupsTpl>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
index 251314c3e86a..9970d5988879 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
@@ -3,10 +3,19 @@ import { BehaviorSubject, Observable, forkJoin, of } from 'rxjs';
 import { catchError, shareReplay, switchMap, tap } from 'rxjs/operators';
 import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
 import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { CdTableAction } from '~/app/shared/models/cd-table-action';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
 import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
 import { CephfsSubvolume, SubvolumeSnapshot } from '~/app/shared/models/cephfs-subvolume.model';
+import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component';
+import { ModalService } from '~/app/shared/services/modal.service';
+import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
+import { Permissions } from '~/app/shared/models/permissions';
+import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
+import { CdDatePipe } from '~/app/shared/pipes/cd-date.pipe';
 
 @Component({
   selector: 'cd-cephfs-subvolume-snapshots-list',
@@ -18,6 +27,9 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
 
   context: CdTableFetchDataContext;
   columns: CdTableColumn[] = [];
+  tableActions: CdTableAction[];
+  selection = new CdTableSelection();
+  permissions: Permissions;
 
   subVolumes$: Observable<CephfsSubvolume[]>;
   snapshots$: Observable<any[]>;
@@ -37,8 +49,14 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
 
   constructor(
     private cephfsSubvolumeGroupService: CephfsSubvolumeGroupService,
-    private cephfsSubvolumeService: CephfsSubvolumeService
-  ) {}
+    private cephfsSubvolumeService: CephfsSubvolumeService,
+    private actionLabels: ActionLabelsI18n,
+    private modalService: ModalService,
+    private authStorageService: AuthStorageService,
+    private cdDatePipe: CdDatePipe
+  ) {
+    this.permissions = this.authStorageService.getPermissions();
+  }
 
   ngOnInit(): void {
     this.columns = [
@@ -51,7 +69,7 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
         name: $localize`Created`,
         prop: 'info.created_at',
         flexGrow: 1,
-        cellTransformation: CellTemplate.timeAgo
+        pipe: this.cdDatePipe
       },
       {
         name: $localize`Pending Clones`,
@@ -67,6 +85,15 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
       }
     ];
 
+    this.tableActions = [
+      {
+        name: this.actionLabels.CREATE,
+        permission: 'create',
+        icon: Icons.add,
+        click: () => this.openModal()
+      }
+    ];
+
     this.cephfsSubvolumeGroupService
       .get(this.fsName)
       .pipe(
@@ -145,4 +172,22 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
   fetchData() {
     this.snapshotSubject.next([]);
   }
+
+  openModal(edit = false) {
+    this.modalService.show(
+      CephfsSubvolumeSnapshotsFormComponent,
+      {
+        fsName: this.fsName,
+        subVolumeName: this.activeSubVolumeName,
+        subVolumeGroupName: this.activeGroupName,
+        subVolumeGroups: this.subvolumeGroupList,
+        isEdit: edit
+      },
+      { size: 'lg' }
+    );
+  }
+
+  updateSelection(selection: CdTableSelection) {
+    this.selection = selection;
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index 53544ccd9ed4..41451d9e3c0c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -22,6 +22,7 @@ import { CephfsSubvolumegroupFormComponent } from './cephfs-subvolumegroup-form/
 import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component';
 import { CephfsSnapshotscheduleListComponent } from './cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component';
 import { DataTableModule } from '../../shared/datatable/datatable.module';
+import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component';
 
 @NgModule({
   imports: [
@@ -51,7 +52,8 @@ import { DataTableModule } from '../../shared/datatable/datatable.module';
     CephfsSubvolumeGroupComponent,
     CephfsSubvolumegroupFormComponent,
     CephfsSubvolumeSnapshotsListComponent,
-    CephfsSnapshotscheduleListComponent
+    CephfsSnapshotscheduleListComponent,
+    CephfsSubvolumeSnapshotsFormComponent
   ]
 })
 export class CephfsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
index 2e8448ff1a22..10ef5ea00f73 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.spec.ts
@@ -48,4 +48,10 @@ describe('CephfsSubvolumeService', () => {
     );
     expect(req.request.method).toBe('GET');
   });
+
+  it('should call createSnapshot', () => {
+    service.createSnapshot('testFS', 'testSnap', 'testSubvol').subscribe();
+    const req = httpTesting.expectOne('api/cephfs/subvolume/snapshot/');
+    expect(req.request.method).toBe('POST');
+  });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
index d76523aafd2a..1995fd293bae 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
@@ -117,4 +117,48 @@ export class CephfsSubvolumeService {
       }
     );
   }
+
+  getSnapshotInfo(snapshotName: string, fsName: string, subVolumeName: string, groupName = '') {
+    return this.http.get(`${this.baseURL}/snapshot/${fsName}/${subVolumeName}/info`, {
+      params: {
+        snap_name: snapshotName,
+        group_name: groupName
+      }
+    });
+  }
+
+  snapshotExists(
+    fsName: string,
+    snapshotName: string,
+    subVolumeName: string,
+    groupName: string = ''
+  ): Observable<boolean> {
+    return this.getSnapshotInfo(fsName, snapshotName, subVolumeName, groupName).pipe(
+      mapTo(true),
+      catchError((error: Event) => {
+        if (_.isFunction(error.preventDefault)) {
+          error.preventDefault();
+        }
+        return of(false);
+      })
+    );
+  }
+
+  createSnapshot(
+    fsName: string,
+    snapshotName: string,
+    subVolumeName: string,
+    groupName: string = ''
+  ) {
+    return this.http.post(
+      `${this.baseURL}/snapshot/`,
+      {
+        vol_name: fsName,
+        subvol_name: subVolumeName,
+        snap_name: snapshotName,
+        group_name: groupName
+      },
+      { observe: 'response' }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
index bea426724e07..602a11e7343c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
@@ -18,7 +18,7 @@ export function isEmptyInputValue(value: any): boolean {
   return value == null || value.length === 0;
 }
 
-export type existsServiceFn = (value: any, args?: any) => Observable<boolean>;
+export type existsServiceFn = (value: any, ...args: any[]) => Observable<boolean>;
 
 export class CdValidators {
   /**
@@ -359,7 +359,7 @@ export class CdValidators {
     serviceFnThis: any = null,
     usernameFn?: Function,
     uidField = false,
-    extraArgs = ''
+    ...extraArgs: any[]
   ): AsyncValidatorFn {
     let uName: string;
     return (control: AbstractControl): Observable<ValidationErrors | null> => {
@@ -378,7 +378,7 @@ export class CdValidators {
       }
 
       return observableTimer().pipe(
-        switchMapTo(serviceFn.call(serviceFnThis, uName, extraArgs)),
+        switchMapTo(serviceFn.call(serviceFnThis, uName, ...extraArgs)),
         map((resp: boolean) => {
           if (!resp) {
             return null;
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.spec.ts
index b67ed62c8a6c..b711bdbb1cef 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.spec.ts
@@ -1,15 +1,12 @@
-import { DatePipe } from '@angular/common';
-
 import moment from 'moment';
 
 import { CdDatePipe } from './cd-date.pipe';
 
 describe('CdDatePipe', () => {
-  const datePipe = new DatePipe('en-US');
-  let pipe = new CdDatePipe(datePipe);
+  let pipe = new CdDatePipe();
 
   it('create an instance', () => {
-    pipe = new CdDatePipe(datePipe);
+    pipe = new CdDatePipe();
     expect(pipe).toBeTruthy();
   });
 
@@ -18,7 +15,12 @@ describe('CdDatePipe', () => {
   });
 
   it('transforms with some date', () => {
-    const result = moment(1527085564486).format('M/D/YY LTS');
+    const result = moment
+      .parseZone(moment.unix(1527085564486))
+      .utc()
+      .utcOffset(moment().utcOffset())
+      .local()
+      .format('D/M/YY hh:mm A');
     expect(pipe.transform(1527085564486)).toBe(result);
   });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.ts
index 911f320410f4..887d8d6bfb98 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/cd-date.pipe.ts
@@ -1,20 +1,30 @@
-import { DatePipe } from '@angular/common';
 import { Pipe, PipeTransform } from '@angular/core';
+import _ from 'lodash';
+import moment from 'moment';
 
 @Pipe({
   name: 'cdDate'
 })
 export class CdDatePipe implements PipeTransform {
-  constructor(private datePipe: DatePipe) {}
+  constructor() {}
 
   transform(value: any): any {
     if (value === null || value === '') {
       return '';
     }
-    return (
-      this.datePipe.transform(value, 'shortDate') +
-      ' ' +
-      this.datePipe.transform(value, 'mediumTime')
-    );
+    let date: string;
+    const offset = moment().utcOffset();
+    if (_.isNumber(value)) {
+      date = moment
+        .parseZone(moment.unix(value))
+        .utc()
+        .utcOffset(offset)
+        .local()
+        .format('D/M/YY hh:mm A');
+    } else {
+      value = value?.replace('Z', '');
+      date = moment.parseZone(value).utc().utcOffset(offset).local().format('D/M/YY hh:mm A');
+    }
+    return date;
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index f6969c2e8e1b..c1165d318a36 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -379,6 +379,10 @@ export class TaskMessageService {
     ),
     'cephfs/subvolume/group/remove': this.newTaskMessage(this.commonOperations.remove, (metadata) =>
       this.subvolumegroup(metadata)
+    ),
+    'cephfs/subvolume/snapshot/create': this.newTaskMessage(
+      this.commonOperations.create,
+      (metadata) => this.snapshot(metadata)
     )
   };
 
@@ -447,6 +451,10 @@ export class TaskMessageService {
     return $localize`subvolume group '${metadata.subvolumegroupName}'`;
   }
 
+  snapshot(metadata: any) {
+    return $localize`snapshot '${metadata.snapshotName}'`;
+  }
+
   crudMessageId(id: string) {
     return $localize`${id}`;
   }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 0ed83aab6c8e..6129321c7df2 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -2019,6 +2019,52 @@ paths:
       - jwt: []
       tags:
       - CephfsSubvolumeGroup
+  /api/cephfs/subvolume/snapshot:
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                group_name:
+                  default: ''
+                  type: string
+                snap_name:
+                  type: string
+                subvol_name:
+                  type: string
+                vol_name:
+                  type: string
+              required:
+              - vol_name
+              - subvol_name
+              - snap_name
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephfsSubvolumeSnapshot
   /api/cephfs/subvolume/snapshot/{vol_name}/{subvol_name}:
     get:
       parameters:
@@ -2061,6 +2107,48 @@ paths:
       - jwt: []
       tags:
       - CephfsSubvolumeSnapshot
+  /api/cephfs/subvolume/snapshot/{vol_name}/{subvol_name}/info:
+    get:
+      parameters:
+      - in: path
+        name: vol_name
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: subvol_name
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: snap_name
+        required: true
+        schema:
+          type: string
+      - default: ''
+        in: query
+        name: group_name
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephfsSubvolumeSnapshot
   /api/cephfs/subvolume/{vol_name}:
     delete:
       parameters:

From bb02c5e3c5f104f057bf63c14b48a858f3109d24 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Thu, 29 Jun 2023 14:28:38 +0000
Subject: [PATCH 1283/2492] osd: s/read/recovery_read/ in
 ECBackend::RecoveryMessages

`read()` is simply too common for grep. Also, this commit
will ensure there is no unnoticed new call site which
could be problematic in the spot of the upcoming recovery
read rework.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 3083f6e698bc..baef8b16f126 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -275,19 +275,20 @@ struct OnRecoveryReadComplete :
 
 struct RecoveryMessages {
   map<hobject_t,
-      ECBackend::read_request_t> reads;
+      ECBackend::read_request_t> recovery_reads;
   map<hobject_t, set<int>> want_to_read;
-  void read(
+  void recovery_read(
     ECBackend *ec,
     const hobject_t &hoid, uint64_t off, uint64_t len,
     set<int> &&_want_to_read,
     const map<pg_shard_t, vector<pair<int, int>>> &need,
-    bool attrs) {
+    bool attrs)
+  {
     list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     to_read.push_back(boost::make_tuple(off, len, 0));
-    ceph_assert(!reads.count(hoid));
+    ceph_assert(!recovery_reads.count(hoid));
     want_to_read.insert(make_pair(hoid, std::move(_want_to_read)));
-    reads.insert(
+    recovery_reads.insert(
       make_pair(
 	hoid,
 	ECBackend::read_request_t(
@@ -582,12 +583,12 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
     get_parent()->queue_transaction(std::move(m.t));
   } 
 
-  if (m.reads.empty())
+  if (m.recovery_reads.empty())
     return;
   start_read_op(
     priority,
     m.want_to_read,
-    m.reads,
+    m.recovery_reads,
     OpRequestRef(),
     false, true);
 }
@@ -636,7 +637,7 @@ void ECBackend::continue_recovery_op(
 	recovery_ops.erase(op.hoid);
 	return;
       }
-      m->read(
+      m->recovery_read(
 	this,
 	op.hoid,
 	op.recovery_progress.data_recovered_to,

From 31efa5bf550f366f71801ad39650feb3d7db0eec Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Thu, 29 Jun 2023 14:34:45 +0000
Subject: [PATCH 1284/2492] osd: s/pg/backend/ in
 ECBackend::OnRecoveryReadComplete

`pg` for `ECBackend*` is simply misleading.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index baef8b16f126..d99c989f6783 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -254,18 +254,18 @@ void ECBackend::_failed_push(const hobject_t &hoid,
 
 struct OnRecoveryReadComplete :
   public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
-  ECBackend *pg;
+  ECBackend *backend;
   hobject_t hoid;
-  OnRecoveryReadComplete(ECBackend *pg, const hobject_t &hoid)
-    : pg(pg), hoid(hoid) {}
+  OnRecoveryReadComplete(ECBackend *backend, const hobject_t &hoid)
+    : backend(backend), hoid(hoid) {}
   void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) override {
     ECBackend::read_result_t &res = in.second;
     if (!(res.r == 0 && res.errors.empty())) {
-        pg->_failed_push(hoid, in);
+        backend->_failed_push(hoid, in);
         return;
     }
     ceph_assert(res.returned.size() == 1);
-    pg->handle_recovery_read_complete(
+    backend->handle_recovery_read_complete(
       hoid,
       res.returned.back(),
       res.attrs,

From f5fd8bd9e3b0188d6a23339366da1804957c14ce Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Thu, 29 Jun 2023 19:01:19 +0000
Subject: [PATCH 1285/2492] osd: decrease the coupling between client and
 recovery reads in ECBackend

`complete_read_op()` being aware about `RecoveryMessages` was too much.

TODO:
  * rename or rework `RecoveryMessages`. Since now it's also
    a callack for `ReadOp::on_complete`. I don't like that.
  * drop the `pair<RecoveryMessages*, read_request_t&>`.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 53 +++++++++++++++++++++++++++-----------------
 src/osd/ECBackend.h  | 20 ++++++++++++-----
 2 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index d99c989f6783..5f91d85ce904 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -254,10 +254,12 @@ void ECBackend::_failed_push(const hobject_t &hoid,
 
 struct OnRecoveryReadComplete :
   public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
+  struct RecoveryMessages* rm;
   ECBackend *backend;
   hobject_t hoid;
-  OnRecoveryReadComplete(ECBackend *backend, const hobject_t &hoid)
-    : backend(backend), hoid(hoid) {}
+
+  OnRecoveryReadComplete(RecoveryMessages* rm, ECBackend *backend, const hobject_t &hoid)
+    : rm(rm), backend(backend), hoid(hoid) {}
   void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) override {
     ECBackend::read_result_t &res = in.second;
     if (!(res.r == 0 && res.errors.empty())) {
@@ -269,14 +271,18 @@ struct OnRecoveryReadComplete :
       hoid,
       res.returned.back(),
       res.attrs,
-      in.first);
+      rm);
   }
 };
 
-struct RecoveryMessages {
+struct RecoveryMessages : GenContext<int> {
+  ECBackend *ec;
   map<hobject_t,
       ECBackend::read_request_t> recovery_reads;
   map<hobject_t, set<int>> want_to_read;
+
+  RecoveryMessages(ECBackend* ec) : ec(ec) {}
+
   void recovery_read(
     ECBackend *ec,
     const hobject_t &hoid, uint64_t off, uint64_t len,
@@ -296,6 +302,7 @@ struct RecoveryMessages {
 	  need,
 	  attrs,
 	  new OnRecoveryReadComplete(
+	    this,
 	    ec,
 	    hoid))));
   }
@@ -305,6 +312,10 @@ struct RecoveryMessages {
   ObjectStore::Transaction t;
   RecoveryMessages() {}
   ~RecoveryMessages() {}
+
+  void finish(int priority) override {
+    ec->dispatch_recovery_messages(*this, priority);
+  }
 };
 
 void ECBackend::handle_recovery_push(
@@ -590,7 +601,7 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
     m.want_to_read,
     m.recovery_reads,
     OpRequestRef(),
-    false, true);
+    false, true, new RecoveryMessages{});
 }
 
 void ECBackend::continue_recovery_op(
@@ -755,7 +766,7 @@ void ECBackend::run_recovery_op(
   int priority)
 {
   ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
-  RecoveryMessages m;
+  RecoveryMessages m{this};
   for (list<RecoveryOp>::iterator i = h->ops.begin();
        i != h->ops.end();
        ++i) {
@@ -859,14 +870,14 @@ bool ECBackend::_handle_message(
     // buffers.  It does not conflict with ECSubReadReply operator<<.
     MOSDECSubOpReadReply *op = static_cast<MOSDECSubOpReadReply*>(
       _op->get_nonconst_req());
-    RecoveryMessages rm;
-    handle_sub_read_reply(op->op.from, op->op, &rm, _op->pg_trace);
-    dispatch_recovery_messages(rm, priority);
+    handle_sub_read_reply(op->op.from, op->op, _op->pg_trace);
+    // dispatch_recovery_messages() in the case of recovery_reads
+    // is called via the `on_complete` callback
     return true;
   }
   case MSG_OSD_PG_PUSH: {
     auto op = _op->get_req<MOSDPGPush>();
-    RecoveryMessages rm;
+    RecoveryMessages rm{this};
     for (vector<PushOp>::const_iterator i = op->pushes.begin();
 	 i != op->pushes.end();
 	 ++i) {
@@ -878,7 +889,7 @@ bool ECBackend::_handle_message(
   case MSG_OSD_PG_PUSH_REPLY: {
     const MOSDPGPushReply *op = static_cast<const MOSDPGPushReply *>(
       _op->get_req());
-    RecoveryMessages rm;
+    RecoveryMessages rm{this};
     for (vector<PushReplyOp>::const_iterator i = op->replies.begin();
 	 i != op->replies.end();
 	 ++i) {
@@ -1196,7 +1207,6 @@ void ECBackend::handle_sub_write_reply(
 void ECBackend::handle_sub_read_reply(
   pg_shard_t from,
   ECSubReadReply &op,
-  RecoveryMessages *m,
   const ZTracer::Trace &trace)
 {
   trace.event("ec sub read reply");
@@ -1335,13 +1345,13 @@ void ECBackend::handle_sub_read_reply(
              is_complete == rop.complete.size()) {
     dout(20) << __func__ << " Complete: " << rop << dendl;
     rop.trace.event("ec read complete");
-    complete_read_op(rop, m);
+    complete_read_op(rop);
   } else {
     dout(10) << __func__ << " readop not complete: " << rop << dendl;
   }
 }
 
-void ECBackend::complete_read_op(ReadOp &rop, RecoveryMessages *m)
+void ECBackend::complete_read_op(ReadOp &rop)
 {
   map<hobject_t, read_request_t>::iterator reqiter =
     rop.to_read.begin();
@@ -1364,6 +1374,10 @@ void ECBackend::complete_read_op(ReadOp &rop, RecoveryMessages *m)
   }
   rop.in_progress.clear();
   tid_to_read_map.erase(rop.tid);
+  if (rop.on_complete) {
+    rop.on_complete->complete(rop.priority);
+    rop.on_complete = nullptr;
+  }
 }
 
 struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
@@ -1373,10 +1387,7 @@ struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
   void finish(ThreadPool::TPHandle &handle) override {
     auto ropiter = ec->tid_to_read_map.find(tid);
     ceph_assert(ropiter != ec->tid_to_read_map.end());
-    int priority = ropiter->second.priority;
-    RecoveryMessages rm;
-    ec->complete_read_op(ropiter->second, &rm);
-    ec->dispatch_recovery_messages(rm, priority);
+    ec->complete_read_op(ropiter->second);
   }
 };
 
@@ -1808,7 +1819,8 @@ void ECBackend::start_read_op(
   map<hobject_t, read_request_t> &to_read,
   OpRequestRef _op,
   bool do_redundant_reads,
-  bool for_recovery)
+  bool for_recovery,
+  GenContext<int> *on_complete)
 {
   ceph_tid_t tid = get_parent()->get_tid();
   ceph_assert(!tid_to_read_map.count(tid));
@@ -1819,6 +1831,7 @@ void ECBackend::start_read_op(
       tid,
       do_redundant_reads,
       for_recovery,
+      on_complete,
       _op,
       std::move(want_to_read),
       std::move(to_read))).first->second;
@@ -2501,7 +2514,7 @@ void ECBackend::objects_read_and_reconstruct(
     obj_want_to_read,
     for_read_op,
     OpRequestRef(),
-    fast_read, false);
+    fast_read, false, nullptr);
   return;
 }
 
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index d879f4294435..0483fa191294 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -82,7 +82,6 @@ class ECBackend : public PGBackend {
   void handle_sub_read_reply(
     pg_shard_t from,
     ECSubReadReply &op,
-    RecoveryMessages *m,
     const ZTracer::Trace &trace
     );
 
@@ -277,6 +276,7 @@ class ECBackend : public PGBackend {
   void continue_recovery_op(
     RecoveryOp &op,
     RecoveryMessages *m);
+  friend struct RecoveryMessages;
   void dispatch_recovery_messages(RecoveryMessages &m, int priority);
   friend struct OnRecoveryReadComplete;
   void handle_recovery_read_complete(
@@ -356,6 +356,7 @@ class ECBackend : public PGBackend {
     // True if reading for recovery which could possibly reading only a subset
     // of the available shards.
     bool for_recovery;
+    GenContext<int> *on_complete;
 
     ZTracer::Trace trace;
 
@@ -375,11 +376,17 @@ class ECBackend : public PGBackend {
       ceph_tid_t tid,
       bool do_redundant_reads,
       bool for_recovery,
+      GenContext<int> *on_complete,
       OpRequestRef op,
       std::map<hobject_t, std::set<int>> &&_want_to_read,
       std::map<hobject_t, read_request_t> &&_to_read)
-      : priority(priority), tid(tid), op(op), do_redundant_reads(do_redundant_reads),
-	for_recovery(for_recovery), want_to_read(std::move(_want_to_read)),
+      : priority(priority),
+        tid(tid),
+        op(op),
+        do_redundant_reads(do_redundant_reads),
+        for_recovery(for_recovery),
+        on_complete(on_complete),
+        want_to_read(std::move(_want_to_read)),
 	to_read(std::move(_to_read)) {
       for (auto &&hpair: to_read) {
 	auto &returned = complete[hpair.first].returned;
@@ -400,8 +407,9 @@ class ECBackend : public PGBackend {
   void filter_read_op(
     const OSDMapRef& osdmap,
     ReadOp &op);
-  void complete_read_op(ReadOp &rop, RecoveryMessages *m);
+  void complete_read_op(ReadOp &rop);
   friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+
   std::map<ceph_tid_t, ReadOp> tid_to_read_map;
   std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
   void start_read_op(
@@ -409,7 +417,9 @@ class ECBackend : public PGBackend {
     std::map<hobject_t, std::set<int>> &want_to_read,
     std::map<hobject_t, read_request_t> &to_read,
     OpRequestRef op,
-    bool do_redundant_reads, bool for_recovery);
+    bool do_redundant_reads,
+    bool for_recovery,
+    GenContext<int> *on_complete);
 
   void do_read_op(ReadOp &rop);
   int send_all_remaining_reads(

From 5a3010db3b988e98a5fa416af3df2de595ab54d9 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Thu, 29 Jun 2023 19:20:30 +0000
Subject: [PATCH 1286/2492] osd: decouple client and recovery reads in
 ECBackend, part 2

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 23 ++++++++---------------
 src/osd/ECBackend.h  |  7 +++----
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 5f91d85ce904..378f77e4fd40 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -233,10 +233,8 @@ PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
   return new ECRecoveryHandle;
 }
 
-void ECBackend::_failed_push(const hobject_t &hoid,
-  pair<RecoveryMessages *, ECBackend::read_result_t &> &in)
+void ECBackend::_failed_push(const hobject_t &hoid, ECBackend::read_result_t &res)
 {
-  ECBackend::read_result_t &res = in.second;
   dout(10) << __func__ << ": Read error " << hoid << " r="
 	   << res.r << " errors=" << res.errors << dendl;
   dout(10) << __func__ << ": canceling recovery op for obj " << hoid
@@ -253,17 +251,16 @@ void ECBackend::_failed_push(const hobject_t &hoid,
 }
 
 struct OnRecoveryReadComplete :
-  public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
+  public GenContext<ECBackend::read_result_t&> {
   struct RecoveryMessages* rm;
   ECBackend *backend;
   hobject_t hoid;
 
   OnRecoveryReadComplete(RecoveryMessages* rm, ECBackend *backend, const hobject_t &hoid)
     : rm(rm), backend(backend), hoid(hoid) {}
-  void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) override {
-    ECBackend::read_result_t &res = in.second;
+  void finish(ECBackend::read_result_t &res) override {
     if (!(res.r == 0 && res.errors.empty())) {
-        backend->_failed_push(hoid, in);
+        backend->_failed_push(hoid, res);
         return;
     }
     ceph_assert(res.returned.size() == 1);
@@ -1360,9 +1357,7 @@ void ECBackend::complete_read_op(ReadOp &rop)
   ceph_assert(rop.to_read.size() == rop.complete.size());
   for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
     if (reqiter->second.cb) {
-      pair<RecoveryMessages *, read_result_t &> arg(
-	m, resiter->second);
-      reqiter->second.cb->complete(arg);
+      reqiter->second.cb->complete(resiter->second);
       reqiter->second.cb = nullptr;
     }
   }
@@ -2407,7 +2402,7 @@ void ECBackend::objects_read_async(
 }
 
 struct CallClientContexts :
-  public GenContext<pair<RecoveryMessages*, ECBackend::read_result_t& > &> {
+  public GenContext<ECBackend::read_result_t&> {
   hobject_t hoid;
   ECBackend *ec;
   ECBackend::ClientAsyncReadStatus *status;
@@ -2418,8 +2413,7 @@ struct CallClientContexts :
     ECBackend::ClientAsyncReadStatus *status,
     const list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read)
     : hoid(hoid), ec(ec), status(status), to_read(to_read) {}
-  void finish(pair<RecoveryMessages *, ECBackend::read_result_t &> &in) override {
-    ECBackend::read_result_t &res = in.second;
+  void finish(ECBackend::read_result_t &res) override {
     extent_map result;
     if (res.r != 0)
       goto out;
@@ -2536,8 +2530,7 @@ int ECBackend::send_all_remaining_reads(
 
   list<boost::tuple<uint64_t, uint64_t, uint32_t> > offsets =
     rop.to_read.find(hoid)->second.to_read;
-  GenContext<pair<RecoveryMessages *, read_result_t& > &> *c =
-    rop.to_read.find(hoid)->second.cb;
+  GenContext<read_result_t&> *c = rop.to_read.find(hoid)->second.cb;
 
   // (Note cuixf) If we need to read attrs and we read failed, try to read again.
   bool want_attrs =
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 0483fa191294..d604a176ac7d 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -334,12 +334,12 @@ class ECBackend : public PGBackend {
     const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
     bool want_attrs;
-    GenContext<std::pair<RecoveryMessages *, read_result_t& > &> *cb;
+    GenContext<read_result_t&> *cb;
     read_request_t(
       const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
       const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
       bool want_attrs,
-      GenContext<std::pair<RecoveryMessages *, read_result_t& > &> *cb)
+      GenContext<read_result_t&> *cb)
       : to_read(to_read), need(need), want_attrs(want_attrs),
 	cb(cb) {}
   };
@@ -749,8 +749,7 @@ class ECBackend : public PGBackend {
   uint64_t be_get_ondisk_size(uint64_t logical_size) const final {
     return sinfo.logical_to_next_chunk_offset(logical_size);
   }
-  void _failed_push(const hobject_t &hoid,
-    std::pair<RecoveryMessages *, ECBackend::read_result_t &> &in);
+  void _failed_push(const hobject_t &hoid, ECBackend::read_result_t &res);
 };
 ostream &operator<<(ostream &lhs, const ECBackend::RMWPipeline::pipeline_state_t &rhs);
 

From faefa5bbb798f65d5faf921b83bf142c94c03c60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 6 Jul 2023 12:17:45 +0200
Subject: [PATCH 1287/2492] osd: drop unused param name in
 ECBackend::FinishReadOp::finish()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 378f77e4fd40..906b1d664504 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1379,7 +1379,7 @@ struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
   ECBackend *ec;
   ceph_tid_t tid;
   FinishReadOp(ECBackend *ec, ceph_tid_t tid) : ec(ec), tid(tid) {}
-  void finish(ThreadPool::TPHandle &handle) override {
+  void finish(ThreadPool::TPHandle&) override {
     auto ropiter = ec->tid_to_read_map.find(tid);
     ceph_assert(ropiter != ec->tid_to_read_map.end());
     ec->complete_read_op(ropiter->second);

From 416d3281f1009ad539ff04c03c2801affa7634d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 6 Jul 2023 13:30:53 +0200
Subject: [PATCH 1288/2492] osd: dissect the EC read pipeline from ECBackend
 into dedicated class
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 161 ++++++++++++++++++++++++++++---------------
 src/osd/ECBackend.h  | 125 +++++++++++++++++++++------------
 2 files changed, 188 insertions(+), 98 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 906b1d664504..4bc7695ebb16 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -58,6 +58,9 @@ static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
 static ostream& _prefix(std::ostream *_dout, ECBackend::RMWPipeline *rmw_pipeline) {
   return rmw_pipeline->get_parent()->gen_dbg_prefix(*_dout);
 }
+static ostream& _prefix(std::ostream *_dout, ECBackend::ReadPipeline *read_pipeline) {
+  return read_pipeline->get_parent()->gen_dbg_prefix(*_dout);
+}
 
 struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
   list<ECBackend::RecoveryOp> ops;
@@ -221,6 +224,7 @@ ECBackend::ECBackend(
   ErasureCodeInterfaceRef ec_impl,
   uint64_t stripe_width)
   : PGBackend(cct, pg, store, coll, ch),
+    read_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
@@ -593,7 +597,7 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
 
   if (m.recovery_reads.empty())
     return;
-  start_read_op(
+  read_pipeline.start_read_op(
     priority,
     m.want_to_read,
     m.recovery_reads,
@@ -1208,8 +1212,8 @@ void ECBackend::handle_sub_read_reply(
 {
   trace.event("ec sub read reply");
   dout(10) << __func__ << ": reply " << op << dendl;
-  map<ceph_tid_t, ReadOp>::iterator iter = tid_to_read_map.find(op.tid);
-  if (iter == tid_to_read_map.end()) {
+  map<ceph_tid_t, ReadOp>::iterator iter = read_pipeline.tid_to_read_map.find(op.tid);
+  if (iter == read_pipeline.tid_to_read_map.end()) {
     //canceled
     dout(20) << __func__ << ": dropped " << op << dendl;
     return;
@@ -1265,8 +1269,8 @@ void ECBackend::handle_sub_read_reply(
   }
 
   map<pg_shard_t, set<ceph_tid_t> >::iterator siter =
-					shard_to_read_map.find(from);
-  ceph_assert(siter != shard_to_read_map.end());
+					read_pipeline.shard_to_read_map.find(from);
+  ceph_assert(siter != read_pipeline.shard_to_read_map.end());
   ceph_assert(siter->second.count(op.tid));
   siter->second.erase(op.tid);
 
@@ -1298,7 +1302,7 @@ void ECBackend::handle_sub_read_reply(
 	  // During recovery there may be multiple osds with copies of the same shard,
 	  // so getting EIO from one may result in multiple passes through this code path.
 	  if (!rop.do_redundant_reads) {
-	    int r = send_all_remaining_reads(iter->first, rop);
+	    int r = read_pipeline.send_all_remaining_reads(iter->first, rop);
 	    if (r == 0) {
 	      // We changed the rop's to_read and not incrementing is_complete
 	      need_resend = true;
@@ -1337,18 +1341,18 @@ void ECBackend::handle_sub_read_reply(
     }
   }
   if (need_resend) {
-    do_read_op(rop);
+    read_pipeline.do_read_op(rop);
   } else if (rop.in_progress.empty() || 
              is_complete == rop.complete.size()) {
     dout(20) << __func__ << " Complete: " << rop << dendl;
     rop.trace.event("ec read complete");
-    complete_read_op(rop);
+    read_pipeline.complete_read_op(rop);
   } else {
     dout(10) << __func__ << " readop not complete: " << rop << dendl;
   }
 }
 
-void ECBackend::complete_read_op(ReadOp &rop)
+void ECBackend::ReadPipeline::complete_read_op(ReadOp &rop)
 {
   map<hobject_t, read_request_t>::iterator reqiter =
     rop.to_read.begin();
@@ -1376,17 +1380,18 @@ void ECBackend::complete_read_op(ReadOp &rop)
 }
 
 struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
-  ECBackend *ec;
+  ECBackend::ReadPipeline& read_pipeline;
   ceph_tid_t tid;
-  FinishReadOp(ECBackend *ec, ceph_tid_t tid) : ec(ec), tid(tid) {}
+  FinishReadOp(ECBackend::ReadPipeline& read_pipeline, ceph_tid_t tid)
+    : read_pipeline(read_pipeline), tid(tid) {}
   void finish(ThreadPool::TPHandle&) override {
-    auto ropiter = ec->tid_to_read_map.find(tid);
-    ceph_assert(ropiter != ec->tid_to_read_map.end());
-    ec->complete_read_op(ropiter->second);
+    auto ropiter = read_pipeline.tid_to_read_map.find(tid);
+    ceph_assert(ropiter != read_pipeline.tid_to_read_map.end());
+    read_pipeline.complete_read_op(ropiter->second);
   }
 };
 
-void ECBackend::filter_read_op(
+void ECBackend::ReadPipeline::filter_read_op(
   const OSDMapRef& osdmap,
   ReadOp &op)
 {
@@ -1438,7 +1443,8 @@ void ECBackend::filter_read_op(
 
     op.to_read.erase(*i);
     op.complete.erase(*i);
-    recovery_ops.erase(*i);
+    // TODO: meh, this doesn't look like a part of the read pipeline
+    //recovery_ops.erase(*i);
   }
 
   if (op.in_progress.empty()) {
@@ -1459,21 +1465,22 @@ void ECBackend::filter_read_op(
      */
     get_parent()->schedule_recovery_work(
       get_parent()->bless_unlocked_gencontext(
-        new FinishReadOp(this, op.tid)),
+        new FinishReadOp(*this, op.tid)),
       1);
   }
 }
 
 void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
 {
+  // TODO: dissect into ReadPipeline
   set<ceph_tid_t> tids_to_filter;
   for (map<pg_shard_t, set<ceph_tid_t> >::iterator 
-       i = shard_to_read_map.begin();
-       i != shard_to_read_map.end();
+       i = read_pipeline.shard_to_read_map.begin();
+       i != read_pipeline.shard_to_read_map.end();
        ) {
     if (osdmap->is_down(i->first.osd)) {
       tids_to_filter.insert(i->second.begin(), i->second.end());
-      shard_to_read_map.erase(i++);
+      read_pipeline.shard_to_read_map.erase(i++);
     } else {
       ++i;
     }
@@ -1481,12 +1488,31 @@ void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
   for (set<ceph_tid_t>::iterator i = tids_to_filter.begin();
        i != tids_to_filter.end();
        ++i) {
-    map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
-    ceph_assert(j != tid_to_read_map.end());
-    filter_read_op(osdmap, j->second);
+    map<ceph_tid_t, ReadOp>::iterator j = read_pipeline.tid_to_read_map.find(*i);
+    ceph_assert(j != read_pipeline.tid_to_read_map.end());
+    read_pipeline.filter_read_op(osdmap, j->second);
   }
 }
 
+void ECBackend::ReadPipeline::on_change()
+{
+  for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
+       i != tid_to_read_map.end();
+       ++i) {
+    dout(10) << __func__ << ": cancelling " << i->second << dendl;
+    for (map<hobject_t, read_request_t>::iterator j =
+	   i->second.to_read.begin();
+	 j != i->second.to_read.end();
+	 ++j) {
+      delete j->second.cb;
+      j->second.cb = nullptr;
+    }
+  }
+  tid_to_read_map.clear();
+  shard_to_read_map.clear();
+  in_progress_client_reads.clear();
+}
+
 void ECBackend::RMWPipeline::on_change()
 {
   dout(10) << __func__ << dendl;
@@ -1506,21 +1532,7 @@ void ECBackend::RMWPipeline::on_change()
 void ECBackend::on_change()
 {
   rmw_pipeline.on_change();
-  for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
-       i != tid_to_read_map.end();
-       ++i) {
-    dout(10) << __func__ << ": cancelling " << i->second << dendl;
-    for (map<hobject_t, read_request_t>::iterator j =
-	   i->second.to_read.begin();
-	 j != i->second.to_read.end();
-	 ++j) {
-      delete j->second.cb;
-      j->second.cb = nullptr;
-    }
-  }
-  tid_to_read_map.clear();
-  shard_to_read_map.clear();
-  in_progress_client_reads.clear();
+  read_pipeline.on_change();
   clear_recovery_state();
 }
 
@@ -1541,8 +1553,8 @@ void ECBackend::dump_recovery_info(Formatter *f) const
   }
   f->close_section();
   f->open_array_section("read_ops");
-  for (map<ceph_tid_t, ReadOp>::const_iterator i = tid_to_read_map.begin();
-       i != tid_to_read_map.end();
+  for (map<ceph_tid_t, ReadOp>::const_iterator i = read_pipeline.tid_to_read_map.begin();
+       i != read_pipeline.tid_to_read_map.end();
        ++i) {
     f->open_object_section("read_op");
     i->second.dump(f);
@@ -1658,7 +1670,7 @@ void ECBackend::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb)
   }
 }
 
-void ECBackend::get_all_avail_shards(
+void ECBackend::ReadPipeline::get_all_avail_shards(
   const hobject_t &hoid,
   const set<pg_shard_t> &error_shards,
   set<int> &have,
@@ -1737,7 +1749,7 @@ int ECBackend::get_min_avail_to_read_shards(
   map<shard_id_t, pg_shard_t> shards;
   set<pg_shard_t> error_shards;
 
-  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
+  read_pipeline.get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
 
   map<int, vector<pair<int, int>>> need;
   int r = ec_impl->minimum_to_decode(want, have, &need);
@@ -1762,7 +1774,7 @@ int ECBackend::get_min_avail_to_read_shards(
   return 0;
 }
 
-int ECBackend::get_remaining_shards(
+int ECBackend::ReadPipeline::get_remaining_shards(
   const hobject_t &hoid,
   const set<int> &avail,
   const set<int> &want,
@@ -1808,7 +1820,7 @@ int ECBackend::get_remaining_shards(
   return 0;
 }
 
-void ECBackend::start_read_op(
+void ECBackend::ReadPipeline::start_read_op(
   int priority,
   map<hobject_t, set<int>> &want_to_read,
   map<hobject_t, read_request_t> &to_read,
@@ -1838,7 +1850,7 @@ void ECBackend::start_read_op(
   do_read_op(op);
 }
 
-void ECBackend::do_read_op(ReadOp &op)
+void ECBackend::ReadPipeline::do_read_op(ReadOp &op)
 {
   int priority = op.priority;
   ceph_tid_t tid = op.tid;
@@ -2404,15 +2416,15 @@ void ECBackend::objects_read_async(
 struct CallClientContexts :
   public GenContext<ECBackend::read_result_t&> {
   hobject_t hoid;
-  ECBackend *ec;
+  ECBackend::ReadPipeline &read_pipeline;
   ECBackend::ClientAsyncReadStatus *status;
   list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
   CallClientContexts(
     hobject_t hoid,
-    ECBackend *ec,
+    ECBackend::ReadPipeline &read_pipeline,
     ECBackend::ClientAsyncReadStatus *status,
     const list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read)
-    : hoid(hoid), ec(ec), status(status), to_read(to_read) {}
+      : hoid(hoid), read_pipeline(read_pipeline), status(status), to_read(to_read) {}
   void finish(ECBackend::read_result_t &res) override {
     extent_map result;
     if (res.r != 0)
@@ -2421,7 +2433,7 @@ struct CallClientContexts :
     ceph_assert(res.errors.empty());
     for (auto &&read: to_read) {
       pair<uint64_t, uint64_t> adjusted =
-	ec->sinfo.offset_len_to_stripe_bounds(
+	read_pipeline.sinfo.offset_len_to_stripe_bounds(
 	  make_pair(read.get<0>(), read.get<1>()));
       ceph_assert(res.returned.front().get<0>() == adjusted.first);
       ceph_assert(res.returned.front().get<1>() == adjusted.second);
@@ -2434,8 +2446,8 @@ struct CallClientContexts :
 	to_decode[j->first.shard] = std::move(j->second);
       }
       int r = ECUtil::decode(
-	ec->sinfo,
-	ec->ec_impl,
+	read_pipeline.sinfo,
+	read_pipeline.ec_impl,
 	to_decode,
 	&bl);
       if (r < 0) {
@@ -2454,7 +2466,7 @@ struct CallClientContexts :
     }
 out:
     status->complete_object(hoid, res.r, std::move(result));
-    ec->kick_reads();
+    read_pipeline.kick_reads();
   }
 };
 
@@ -2465,6 +2477,32 @@ void ECBackend::objects_read_and_reconstruct(
   bool fast_read,
   GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
 {
+  return read_pipeline.objects_read_and_reconstruct(
+    *this, reads, fast_read, std::move(func));
+}
+
+void ECBackend::ReadPipeline::objects_read_and_reconstruct(
+  ECBackend& ecbackend,
+  const map<hobject_t,
+    std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+  > &reads,
+  bool fast_read,
+  GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
+{
+  return [this,
+    kick_reads=[this] (auto...) { return this->kick_reads();},
+    get_want_to_read_shards=[&ecbackend] (auto&&... args) {
+      return ecbackend.get_want_to_read_shards(std::forward<decltype(args)>(args)...);
+    },
+    get_min_avail_to_read_shards=[&ecbackend] (auto&&... args) {
+      return ecbackend.get_min_avail_to_read_shards(std::forward<decltype(args)>(args)...);
+    },
+    cct=(CephContext*)nullptr,
+    // params
+    &reads,
+    fast_read,
+    func=std::move(func)
+  ]() mutable {
   in_progress_client_reads.emplace_back(
     reads.size(), std::move(func));
   if (!reads.size()) {
@@ -2489,7 +2527,7 @@ void ECBackend::objects_read_and_reconstruct(
 
     CallClientContexts *c = new CallClientContexts(
       to_read.first,
-      this,
+      *this,
       &(in_progress_client_reads.back()),
       to_read.second);
     for_read_op.insert(
@@ -2509,11 +2547,12 @@ void ECBackend::objects_read_and_reconstruct(
     for_read_op,
     OpRequestRef(),
     fast_read, false, nullptr);
+  }();
   return;
 }
 
 
-int ECBackend::send_all_remaining_reads(
+int ECBackend::ReadPipeline::send_all_remaining_reads(
   const hobject_t &hoid,
   ReadOp &rop)
 {
@@ -2551,6 +2590,20 @@ int ECBackend::send_all_remaining_reads(
   return 0;
 }
 
+void ECBackend::ReadPipeline::kick_reads()
+{
+  while (in_progress_client_reads.size() &&
+         in_progress_client_reads.front().is_complete()) {
+    in_progress_client_reads.front().run();
+    in_progress_client_reads.pop_front();
+  }
+}
+
+void ECBackend::kick_reads() {
+  read_pipeline.kick_reads();
+}
+
+
 int ECBackend::objects_get_attrs(
   const hobject_t &hoid,
   map<string, bufferlist, less<>> *out)
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index d604a176ac7d..b3951f664a8f 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -168,7 +168,6 @@ class ECBackend : public PGBackend {
       func.release()->complete(std::move(results));
     }
   };
-  std::list<ClientAsyncReadStatus> in_progress_client_reads;
   void objects_read_async(
     const hobject_t &hoid,
     const std::list<std::pair<boost::tuple<uint64_t, uint64_t, uint32_t>,
@@ -176,13 +175,7 @@ class ECBackend : public PGBackend {
     Context *on_complete,
     bool fast_read = false) override;
 
-  void kick_reads() {
-    while (in_progress_client_reads.size() &&
-	   in_progress_client_reads.front().is_complete()) {
-      in_progress_client_reads.front().run();
-      in_progress_client_reads.pop_front();
-    }
-  }
+  void kick_reads();
 
 private:
   friend struct ECRecoveryHandle;
@@ -292,12 +285,6 @@ class ECBackend : public PGBackend {
     const PushReplyOp &op,
     pg_shard_t from,
     RecoveryMessages *m);
-  void get_all_avail_shards(
-    const hobject_t &hoid,
-    const std::set<pg_shard_t> &error_shards,
-    std::set<int> &have,
-    std::map<shard_id_t, pg_shard_t> &shards,
-    bool for_recovery);
 
 public:
   /**
@@ -403,28 +390,86 @@ class ECBackend : public PGBackend {
     ReadOp(const ReadOp &) = default;
     ReadOp(ReadOp &&) = default;
   };
-  friend struct FinishReadOp;
-  void filter_read_op(
-    const OSDMapRef& osdmap,
-    ReadOp &op);
-  void complete_read_op(ReadOp &rop);
-  friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
-
-  std::map<ceph_tid_t, ReadOp> tid_to_read_map;
-  std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
-  void start_read_op(
-    int priority,
-    std::map<hobject_t, std::set<int>> &want_to_read,
-    std::map<hobject_t, read_request_t> &to_read,
-    OpRequestRef op,
-    bool do_redundant_reads,
-    bool for_recovery,
-    GenContext<int> *on_complete);
-
-  void do_read_op(ReadOp &rop);
-  int send_all_remaining_reads(
-    const hobject_t &hoid,
-    ReadOp &rop);
+
+  struct ReadPipeline {
+    void objects_read_and_reconstruct(
+      ECBackend& ecbackend,
+      const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+      > &reads,
+      bool fast_read,
+      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
+
+    void filter_read_op(
+      const OSDMapRef& osdmap,
+      ReadOp &op);
+
+    void complete_read_op(ReadOp &rop);
+
+    void start_read_op(
+      int priority,
+      std::map<hobject_t, std::set<int>> &want_to_read,
+      std::map<hobject_t, read_request_t> &to_read,
+      OpRequestRef op,
+      bool do_redundant_reads,
+      bool for_recovery,
+      GenContext<int> *on_complete);
+
+    void do_read_op(ReadOp &rop);
+
+    int send_all_remaining_reads(
+      const hobject_t &hoid,
+      ReadOp &rop);
+
+    void on_change();
+
+    void kick_reads();
+
+    std::map<ceph_tid_t, ReadOp> tid_to_read_map;
+    std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
+    std::list<ClientAsyncReadStatus> in_progress_client_reads;
+
+    CephContext* cct;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    PGBackend::Listener* parent;
+    // TODO: lay an interface down here
+    ECBackend& ec_backend;
+
+    PGBackend::Listener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+    ReadPipeline(CephContext* cct,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+                PGBackend::Listener* parent,
+                ECBackend& ec_backend)
+      : cct(cct),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+        parent(parent),
+        ec_backend(ec_backend) {
+    }
+
+    int get_remaining_shards(
+      const hobject_t &hoid,
+      const std::set<int> &avail,
+      const std::set<int> &want,
+      const read_result_t &result,
+      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
+      bool for_recovery);
+
+    void get_all_avail_shards(
+      const hobject_t &hoid,
+      const std::set<pg_shard_t> &error_shards,
+      std::set<int> &have,
+      std::map<shard_id_t, pg_shard_t> &shards,
+      bool for_recovery);
+
+    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+    friend struct FinishReadOp;
+  } read_pipeline;
 
 
   /**
@@ -721,14 +766,6 @@ class ECBackend : public PGBackend {
     std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
     ); ///< @return error code, 0 on success
 
-  int get_remaining_shards(
-    const hobject_t &hoid,
-    const std::set<int> &avail,
-    const std::set<int> &want,
-    const read_result_t &result,
-    std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
-    bool for_recovery);
-
   int objects_get_attrs(
     const hobject_t &hoid,
     std::map<std::string, ceph::buffer::list, std::less<>> *out) override;

From d609bbe960ce1662d7a2e30e6cefaea1cb5c66e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 6 Jul 2023 15:39:19 +0200
Subject: [PATCH 1289/2492] osd: decouple reads and recovery in
 ECBackend::check_recovery_sources()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 32 +++++++++++++++++++++-----------
 src/osd/ECBackend.h  |  7 ++++++-
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 4bc7695ebb16..3982c0441435 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1391,9 +1391,11 @@ struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
   }
 };
 
+template <class F>
 void ECBackend::ReadPipeline::filter_read_op(
   const OSDMapRef& osdmap,
-  ReadOp &op)
+  ReadOp &op,
+  F&& on_erase)
 {
   set<hobject_t> to_cancel;
   for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
@@ -1443,8 +1445,7 @@ void ECBackend::ReadPipeline::filter_read_op(
 
     op.to_read.erase(*i);
     op.complete.erase(*i);
-    // TODO: meh, this doesn't look like a part of the read pipeline
-    //recovery_ops.erase(*i);
+    on_erase(*i);
   }
 
   if (op.in_progress.empty()) {
@@ -1470,17 +1471,19 @@ void ECBackend::ReadPipeline::filter_read_op(
   }
 }
 
-void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
+template <class F>
+void ECBackend::ReadPipeline::check_recovery_sources(
+  const OSDMapRef& osdmap,
+  F&& on_erase)
 {
-  // TODO: dissect into ReadPipeline
   set<ceph_tid_t> tids_to_filter;
   for (map<pg_shard_t, set<ceph_tid_t> >::iterator 
-       i = read_pipeline.shard_to_read_map.begin();
-       i != read_pipeline.shard_to_read_map.end();
+       i = shard_to_read_map.begin();
+       i != shard_to_read_map.end();
        ) {
     if (osdmap->is_down(i->first.osd)) {
       tids_to_filter.insert(i->second.begin(), i->second.end());
-      read_pipeline.shard_to_read_map.erase(i++);
+      shard_to_read_map.erase(i++);
     } else {
       ++i;
     }
@@ -1488,12 +1491,19 @@ void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
   for (set<ceph_tid_t>::iterator i = tids_to_filter.begin();
        i != tids_to_filter.end();
        ++i) {
-    map<ceph_tid_t, ReadOp>::iterator j = read_pipeline.tid_to_read_map.find(*i);
-    ceph_assert(j != read_pipeline.tid_to_read_map.end());
-    read_pipeline.filter_read_op(osdmap, j->second);
+    map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
+    ceph_assert(j != tid_to_read_map.end());
+    filter_read_op(osdmap, j->second, on_erase);
   }
 }
 
+void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
+{
+  read_pipeline.check_recovery_sources(osdmap, [this] (const hobject_t& obj) {
+    recovery_ops.erase(obj);
+  });
+}
+
 void ECBackend::ReadPipeline::on_change()
 {
   for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index b3951f664a8f..94267144ba4c 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -399,9 +399,14 @@ class ECBackend : public PGBackend {
       bool fast_read,
       GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
 
+    template <class F>
     void filter_read_op(
       const OSDMapRef& osdmap,
-      ReadOp &op);
+      ReadOp &op,
+      F&& on_erase);
+
+    template <class F>
+    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
 
     void complete_read_op(ReadOp &rop);
 

From c00606e71ffe08fb8317bd5699e440b440113677 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 1 Sep 2023 16:42:54 +0200
Subject: [PATCH 1290/2492] fix a crash due to RecoveryMessages life-time mgmt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 3982c0441435..6c7364f5f835 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -260,6 +260,7 @@ struct OnRecoveryReadComplete :
   ECBackend *backend;
   hobject_t hoid;
 
+  OnRecoveryReadComplete() = delete;
   OnRecoveryReadComplete(RecoveryMessages* rm, ECBackend *backend, const hobject_t &hoid)
     : rm(rm), backend(backend), hoid(hoid) {}
   void finish(ECBackend::read_result_t &res) override {
@@ -280,6 +281,7 @@ struct RecoveryMessages : GenContext<int> {
   ECBackend *ec;
   map<hobject_t,
       ECBackend::read_request_t> recovery_reads;
+  RecoveryMessages* next_recovery_messages = nullptr;
   map<hobject_t, set<int>> want_to_read;
 
   RecoveryMessages(ECBackend* ec) : ec(ec) {}
@@ -291,6 +293,9 @@ struct RecoveryMessages : GenContext<int> {
     const map<pg_shard_t, vector<pair<int, int>>> &need,
     bool attrs)
   {
+    if (!next_recovery_messages) {
+      next_recovery_messages = new RecoveryMessages{ec};
+    }
     list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     to_read.push_back(boost::make_tuple(off, len, 0));
     ceph_assert(!recovery_reads.count(hoid));
@@ -303,7 +308,7 @@ struct RecoveryMessages : GenContext<int> {
 	  need,
 	  attrs,
 	  new OnRecoveryReadComplete(
-	    this,
+	    next_recovery_messages,
 	    ec,
 	    hoid))));
   }
@@ -311,7 +316,7 @@ struct RecoveryMessages : GenContext<int> {
   map<pg_shard_t, vector<PushOp> > pushes;
   map<pg_shard_t, vector<PushReplyOp> > push_replies;
   ObjectStore::Transaction t;
-  RecoveryMessages() {}
+  RecoveryMessages() = delete;
   ~RecoveryMessages() {}
 
   void finish(int priority) override {
@@ -602,7 +607,8 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
     m.want_to_read,
     m.recovery_reads,
     OpRequestRef(),
-    false, true, new RecoveryMessages{});
+    false, true, m.next_recovery_messages);
+  m.next_recovery_messages = nullptr;
 }
 
 void ECBackend::continue_recovery_op(

From e6d3659021dc17cdb5712d58787f3aaf54ebe8a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 7 Sep 2023 12:07:13 +0200
Subject: [PATCH 1291/2492] osd: refactor accessors of
 RecoveryBackend::temp_contents
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1. Move some of them to .cc and
2. switch their implementations to use lower-layer methods
   instead of touching `temp_contents` directly.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/crimson/osd/pg_backend.h        | 18 ++++++++++++++++++
 src/crimson/osd/recovery_backend.cc | 10 ++++++++++
 src/crimson/osd/recovery_backend.h  |  9 +++------
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h
index 981a983075bc..de7223173f1b 100644
--- a/src/crimson/osd/pg_backend.h
+++ b/src/crimson/osd/pg_backend.h
@@ -450,4 +450,22 @@ class PGBackend
 		      std::vector<pg_log_entry_t>&& log_entries) = 0;
   friend class ReplicatedRecoveryBackend;
   friend class ::crimson::osd::PG;
+
+protected:
+  boost::container::flat_set<hobject_t> temp_contents;
+
+  template <class... Args>
+  void add_temp_obj(Args&&... args) {
+    temp_contents.insert(std::forward<Args>(args)...);
+  }
+  void clear_temp_obj(const hobject_t &oid) {
+    temp_contents.erase(oid);
+  }
+  template <class T>
+  void clear_temp_objs(const T &cont) {
+    for (const auto& oid : cont) {
+      clear_temp_obj(oid);
+    }
+  }
+  friend class RecoveryBackend;
 };
diff --git a/src/crimson/osd/recovery_backend.cc b/src/crimson/osd/recovery_backend.cc
index b5394bfdc485..c9d7246f2f55 100644
--- a/src/crimson/osd/recovery_backend.cc
+++ b/src/crimson/osd/recovery_backend.cc
@@ -32,6 +32,16 @@ hobject_t RecoveryBackend::get_temp_recovery_object(
   return hoid;
 }
 
+void RecoveryBackend::add_temp_obj(const hobject_t &oid)
+{
+  backend->add_temp_obj(oid);
+}
+
+void RecoveryBackend::clear_temp_obj(const hobject_t &oid)
+{
+  backend->clear_temp_obj(oid);
+}
+
 void RecoveryBackend::clean_up(ceph::os::Transaction& t,
 			       std::string_view why)
 {
diff --git a/src/crimson/osd/recovery_backend.h b/src/crimson/osd/recovery_backend.h
index abf695891596..7d219a8d9eca 100644
--- a/src/crimson/osd/recovery_backend.h
+++ b/src/crimson/osd/recovery_backend.h
@@ -201,12 +201,9 @@ class RecoveryBackend {
 
   boost::container::flat_set<hobject_t> temp_contents;
 
-  void add_temp_obj(const hobject_t &oid) {
-    temp_contents.insert(oid);
-  }
-  void clear_temp_obj(const hobject_t &oid) {
-    temp_contents.erase(oid);
-  }
+  void add_temp_obj(const hobject_t &oid);
+  void clear_temp_obj(const hobject_t &oid);
+
   void clean_up(ceph::os::Transaction& t, std::string_view why);
   virtual seastar::future<> on_stop() = 0;
 private:

From 25c16d78838b14d319da72ba36a1898373ecdb8d Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 9 Jan 2024 10:54:01 -0600
Subject: [PATCH 1292/2492] qa/tasks: fix syntax for deep-scrub command

Fixes: https://tracker.ceph.com/issues/63967
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/tasks/ceph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index e478fd40af2c..39f4033ac4f0 100644
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -1303,7 +1303,7 @@ def osd_scrub_pgs(ctx, config):
                     # request was missed.  do not do it every time because
                     # the scrub may be in progress or not reported yet and
                     # we will starve progress.
-                    manager.raw_cluster_cmd('tell', pgid, 'deep_scrub')
+                    manager.raw_cluster_cmd('tell', pgid, 'deep-scrub')
             if gap_cnt > retries:
                 raise RuntimeError('Exiting scrub checking -- not all pgs scrubbed.')
         if loop:

From 18043c5e88d241f43b8fd23a8cbc1b15a3854de9 Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Thu, 13 Jul 2023 23:35:42 +0530
Subject: [PATCH 1293/2492] osd: Remove usage of unsupported objstores for QoS

mClock is supported on Bluestore and a check is currently done to eliminate other unsupported object stores.
With Filestore no longer in the code base, this check can be removed.
In addition, make sure that osd bench will no longer run on setups with memstore.

Fixes: https://tracker.ceph.com/issues/59531
Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 src/osd/OSD.cc | 19 ++++---------------
 src/osd/OSD.h  |  1 -
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index ab83ef7ba4f6..9e6b3fd9d929 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -9991,7 +9991,7 @@ void OSD::maybe_override_max_osd_capacity_for_qos()
   // osd bench test. This is later used to setup mclock.
   if ((op_queue_type_t::mClockScheduler == osd_op_queue_type()) &&
       (cct->_conf.get_val<bool>("osd_mclock_skip_benchmark") == false) &&
-      (!unsupported_objstore_for_qos())) {
+      (store->get_type() != "memstore")) {
     std::string max_capacity_iops_config;
     bool force_run_benchmark =
       cct->_conf.get_val<bool>("osd_mclock_force_run_benchmark_on_init");
@@ -10089,8 +10089,7 @@ bool OSD::maybe_override_options_for_qos(const std::set<std::string> *changed)
 {
   // Override options only if the scheduler enabled is mclock and the
   // underlying objectstore is supported by mclock
-  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
-      !unsupported_objstore_for_qos()) {
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
     static const std::map<std::string, uint64_t> recovery_qos_defaults {
       {"osd_recovery_max_active", 0},
       {"osd_recovery_max_active_hdd", 3},
@@ -10191,8 +10190,7 @@ void OSD::maybe_override_sleep_options_for_qos()
 {
   // Override options only if the scheduler enabled is mclock and the
   // underlying objectstore is supported by mclock
-  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
-      !unsupported_objstore_for_qos()) {
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
     // Override the various sleep settings
     // Disable recovery sleep
     cct->_conf.set_val("osd_recovery_sleep", std::to_string(0));
@@ -10221,8 +10219,7 @@ void OSD::maybe_override_cost_for_qos()
 {
   // If the scheduler enabled is mclock, override the default PG deletion cost
   // so that mclock can meet the QoS goals.
-  if (op_queue_type_t::mClockScheduler == osd_op_queue_type() &&
-      !unsupported_objstore_for_qos()) {
+  if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
     uint64_t pg_delete_cost = 15728640;
     cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost));
   }
@@ -10287,14 +10284,6 @@ void OSD::mon_cmd_set_config(const std::string &key, const std::string &val)
   monc->start_mon_command(vcmd, {}, nullptr, nullptr, on_finish);
 }
 
-bool OSD::unsupported_objstore_for_qos()
-{
-  static const std::vector<std::string> unsupported_objstores = { "filestore" };
-  return std::find(unsupported_objstores.begin(),
-                   unsupported_objstores.end(),
-                   store->get_type()) != unsupported_objstores.end();
-}
-
 op_queue_type_t OSD::osd_op_queue_type() const
 {
   /**
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 7cd7f8423f09..5f561e643182 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1992,7 +1992,6 @@ class OSD : public Dispatcher,
                          double *elapsed,
                          std::ostream& ss);
   void mon_cmd_set_config(const std::string &key, const std::string &val);
-  bool unsupported_objstore_for_qos();
 
   void scrub_purged_snaps();
   void probe_smart(const std::string& devid, std::ostream& ss);

From ef461914eec14bafb68c9d3c04d3596e547ed559 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 9 Jan 2024 11:35:45 -0800
Subject: [PATCH 1294/2492] tasks/.../test_erasure_code_profile: assertSubset
 in test_create_plugin

Newly added profile options may break this test otherwise.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/tasks/mgr/dashboard/test_erasure_code_profile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
index 7fb7c1c8270f..a50914008934 100644
--- a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
+++ b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
@@ -79,7 +79,7 @@ def test_create_plugin(self):
         self.assertStatus(201)
 
         self._get('/api/erasure_code_profile/lrc')
-        self.assertJsonBody({
+        self.assertJsonSubset({
             'crush-device-class': '',
             'crush-failure-domain': 'host',
             'crush-root': 'default',

From 03e11acca048a43ce183be68a5db1c039b7ca485 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Thu, 4 Jan 2024 14:30:31 +0100
Subject: [PATCH 1295/2492] os/bluestore: remove zoned from crimson

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 CMakeLists.txt                                |   7 -
 ceph.spec.in                                  |   7 -
 install-deps.sh                               |  23 +--
 src/blk/BlockDevice.cc                        |  18 ---
 src/blk/BlockDevice.h                         |   3 -
 src/blk/CMakeLists.txt                        |   9 --
 src/blk/zoned/HMSMRDevice.cc                  | 131 ------------------
 src/blk/zoned/HMSMRDevice.h                   |  52 -------
 src/crimson/os/alienstore/CMakeLists.txt      |   5 -
 src/include/config-h.in.cmake                 |   3 -
 src/os/bluestore/Allocator.cc                 |   3 -
 src/os/bluestore/FreelistManager.cc           |  10 +-
 src/test/objectstore/Allocator_test.cc        |   3 +-
 src/test/objectstore/allocator_replay_test.cc |   2 +-
 src/test/objectstore/test_bluefs.cc           |   6 +-
 src/vstart.sh                                 |   8 --
 16 files changed, 7 insertions(+), 283 deletions(-)
 delete mode 100644 src/blk/zoned/HMSMRDevice.cc
 delete mode 100644 src/blk/zoned/HMSMRDevice.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f2df55a75e12..582eda5c74dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -233,13 +233,6 @@ endif()
 
 include(CMakeDependentOption)
 
-CMAKE_DEPENDENT_OPTION(WITH_ZBD "Enable libzbd bluestore backend" OFF
-  "WITH_BLUESTORE" OFF)
-if(WITH_ZBD)
-  find_package(zbd REQUIRED)
-  set(HAVE_LIBZBD ${ZBD_FOUND})
-endif()
-
 CMAKE_DEPENDENT_OPTION(WITH_LIBURING "Enable io_uring bluestore backend" ON
   "WITH_BLUESTORE;HAVE_LIBAIO" OFF)
 set(HAVE_LIBURING ${WITH_LIBURING})
diff --git a/ceph.spec.in b/ceph.spec.in
index 058c79c5e7f8..f813407b76af 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -21,7 +21,6 @@
 # https://rpm-software-management.github.io/rpm/manual/conditionalbuilds.html
 #################################################################################
 %bcond_with make_check
-%bcond_with zbd
 %bcond_with cmake_verbose_logging
 %bcond_without ceph_test_package
 %ifarch s390
@@ -300,9 +299,6 @@ BuildRequires:	socat
 BuildRequires:	python%{python3_pkgversion}-asyncssh
 BuildRequires:	python%{python3_pkgversion}-natsort
 %endif
-%if 0%{with zbd}
-BuildRequires:  libzbd-devel
-%endif
 %if 0%{?suse_version}
 BuildRequires:  libthrift-devel >= 0.13.0
 %else
@@ -1403,9 +1399,6 @@ cmake .. \
 %if 0%{without lua_packages}
     -DWITH_RADOSGW_LUA_PACKAGES:BOOL=OFF \
 %endif
-%if 0%{with zbd}
-    -DWITH_ZBD:BOOL=ON \
-%endif
 %if 0%{with cmake_verbose_logging}
     -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 %endif
diff --git a/install-deps.sh b/install-deps.sh
index d01bc6464e6f..5956b586e526 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -36,8 +36,6 @@ ARCH=$(uname -m)
 function munge_ceph_spec_in {
     local with_seastar=$1
     shift
-    local with_zbd=$1
-    shift
     local for_make_check=$1
     shift
     local OUTFILE=$1
@@ -46,9 +44,6 @@ function munge_ceph_spec_in {
     if $with_seastar; then
         sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE
     fi
-    if $with_zbd; then
-        sed -i -e 's/%bcond_with zbd/%bcond_without zbd/g' $OUTFILE
-    fi
     if $for_make_check; then
         sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE
     fi
@@ -221,19 +216,6 @@ function install_boost_on_ubuntu {
 
 }
 
-function install_libzbd_on_ubuntu {
-    ci_debug "Running install_libzbd_on_ubuntu() in install-deps.sh"
-    local codename=$1
-    local project=libzbd
-    local sha1=1fadde94b08fab574b17637c2bebd2b1e7f9127b
-    install_pkg_on_ubuntu \
-        $project \
-        $sha1 \
-        $codename \
-        check \
-        libzbd-dev
-}
-
 motr_pkgs_url='https://github.com/Seagate/cortx-motr/releases/download/2.0.0-rgw'
 
 function install_cortx_motr_on_ubuntu {
@@ -423,7 +405,6 @@ if [ x$(uname)x = xFreeBSDx ]; then
     exit
 else
     [ $WITH_SEASTAR ] && with_seastar=true || with_seastar=false
-    [ $WITH_ZBD ] && with_zbd=true || with_zbd=false
     [ $WITH_PMEM ] && with_pmem=true || with_pmem=false
     [ $WITH_RADOSGW_MOTR ] && with_rgw_motr=true || with_rgw_motr=false
     source /etc/os-release
@@ -452,12 +433,10 @@ else
             *Bionic*)
                 ensure_decent_gcc_on_ubuntu 9 bionic
                 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu bionic
-                $with_zbd && install_libzbd_on_ubuntu bionic
                 ;;
             *Focal*)
                 ensure_decent_gcc_on_ubuntu 11 focal
                 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu focal
-                $with_zbd && install_libzbd_on_ubuntu focal
                 ;;
             *Jammy*)
                 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu jammy
@@ -549,7 +528,7 @@ else
         if [ "$INSTALL_EXTRA_PACKAGES" ]; then
             $SUDO dnf install -y $INSTALL_EXTRA_PACKAGES
         fi
-        munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec
+        munge_ceph_spec_in $with_seastar $for_make_check $DIR/ceph.spec
         # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
         $SUDO dnf install -y python3-devel
         $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out
diff --git a/src/blk/BlockDevice.cc b/src/blk/BlockDevice.cc
index fd07e443c136..33ceacc8909e 100644
--- a/src/blk/BlockDevice.cc
+++ b/src/blk/BlockDevice.cc
@@ -31,10 +31,6 @@
 #include "pmem/PMEMDevice.h"
 #endif
 
-#if defined(HAVE_LIBZBD)
-#include "zoned/HMSMRDevice.h"
-#endif
-
 #include "common/debug.h"
 #include "common/EventTrace.h"
 #include "common/errno.h"
@@ -113,11 +109,6 @@ BlockDevice::detect_device_type(const std::string& path)
     return block_device_t::pmem;
   }
 #endif
-#if (defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)) && defined(HAVE_LIBZBD)
-  if (HMSMRDevice::support(path)) {
-    return block_device_t::hm_smr;
-  }
-#endif
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
   return block_device_t::aio;
 #else
@@ -142,11 +133,6 @@ BlockDevice::device_type_from_name(const std::string& blk_dev_name)
   if (blk_dev_name == "pmem") {
     return block_device_t::pmem;
   }
-#endif
-#if (defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)) && defined(HAVE_LIBZBD)
-  if (blk_dev_name == "hm_smr") {
-    return block_device_t::hm_smr;
-  }
 #endif
   return block_device_t::unknown;
 }
@@ -168,10 +154,6 @@ BlockDevice* BlockDevice::create_with_type(block_device_t device_type,
 #if defined(HAVE_BLUESTORE_PMEM)
   case block_device_t::pmem:
     return new PMEMDevice(cct, cb, cbpriv);
-#endif
-#if (defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)) && defined(HAVE_LIBZBD)
-  case block_device_t::hm_smr:
-    return new HMSMRDevice(cct, cb, cbpriv, d_cb, d_cbpriv);
 #endif
   default:
     ceph_abort_msg("unsupported device");
diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h
index 440faf3d4b4c..6c55646fc76d 100644
--- a/src/blk/BlockDevice.h
+++ b/src/blk/BlockDevice.h
@@ -156,9 +156,6 @@ class BlockDevice {
     unknown,
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
     aio,
-#if defined(HAVE_LIBZBD)
-    hm_smr,
-#endif
 #endif
 #if defined(HAVE_SPDK)
     spdk,
diff --git a/src/blk/CMakeLists.txt b/src/blk/CMakeLists.txt
index 5d9fd6a52953..62c2a5c29f4b 100644
--- a/src/blk/CMakeLists.txt
+++ b/src/blk/CMakeLists.txt
@@ -20,11 +20,6 @@ if(WITH_SPDK)
     spdk/NVMEDevice.cc)
 endif()
 
-if(WITH_ZBD)
-  list(APPEND libblk_srcs
-    zoned/HMSMRDevice.cc)
-endif()
-
 if(libblk_srcs)
   add_library(blk STATIC ${libblk_srcs})
   target_include_directories(blk PRIVATE "./")
@@ -39,10 +34,6 @@ if(WITH_SPDK)
     PRIVATE spdk::spdk)
 endif()
 
-if(WITH_ZBD)
-  target_link_libraries(blk PRIVATE ${ZBD_LIBRARIES})
-endif()
-
 if(WITH_BLUESTORE_PMEM)
   if(HAVE_LIBDML)
     target_link_libraries(blk PRIVATE dml::dml dml::dmlhl)
diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc
deleted file mode 100644
index 416eae4e49fc..000000000000
--- a/src/blk/zoned/HMSMRDevice.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2014 Red Hat
- * Copyright (C) 2020 Abutalib Aghayev
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include "HMSMRDevice.h"
-extern "C" {
-#include <libzbd/zbd.h>
-}
-#include "common/debug.h"
-#include "common/errno.h"
-
-#define dout_context cct
-#define dout_subsys ceph_subsys_bdev
-#undef dout_prefix
-#define dout_prefix *_dout << "smrbdev(" << this << " " << path << ") "
-
-using namespace std;
-
-HMSMRDevice::HMSMRDevice(CephContext* cct,
-			 aio_callback_t cb,
-			 void *cbpriv,
-			 aio_callback_t d_cb,
-			 void *d_cbpriv)
-  : KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv)
-{
-}
-
-bool HMSMRDevice::support(const std::string& path)
-{
-  return zbd_device_is_zoned(path.c_str()) == 1;
-}
-
-int HMSMRDevice::_post_open()
-{
-  dout(10) << __func__ << dendl;
-
-  zbd_fd = zbd_open(path.c_str(), O_RDWR | O_DIRECT | O_LARGEFILE, nullptr);
-  int r;
-  if (zbd_fd < 0) {
-    r = errno;
-    derr << __func__ << " zbd_open failed on " << path << ": "
-	 << cpp_strerror(r) << dendl;
-    return -r;
-  }
-
-  unsigned int nr_zones = 0;
-  std::vector<zbd_zone> zones;
-  if (zbd_report_nr_zones(zbd_fd, 0, 0, ZBD_RO_NOT_WP, &nr_zones) != 0) {
-    r = -errno;
-    derr << __func__ << " zbd_report_nr_zones failed on " << path << ": "
-	 << cpp_strerror(r) << dendl;
-    goto fail;
-  }
-
-  zones.resize(nr_zones);
-  if (zbd_report_zones(zbd_fd, 0, 0, ZBD_RO_NOT_WP, zones.data(), &nr_zones) != 0) {
-    r = -errno;
-    derr << __func__ << " zbd_report_zones failed on " << path << dendl;
-    goto fail;
-  }
-
-  zone_size = zbd_zone_len(&zones[0]);
-  conventional_region_size = nr_zones * zone_size;
-
-  dout(10) << __func__ << " setting zone size to " << zone_size
-	   << " and conventional region size to " << conventional_region_size
-           << dendl;
-
-  return 0;
-
-fail:
-  zbd_close(zbd_fd);
-  zbd_fd = -1;
-  return r;
-}
-
-
-void HMSMRDevice::_pre_close()
-{
-  if (zbd_fd >= 0) {
-    zbd_close(zbd_fd);
-    zbd_fd = -1;
-  }
-}
-
-void HMSMRDevice::reset_all_zones()
-{
-  dout(10) << __func__ << dendl;
-  zbd_reset_zones(zbd_fd, conventional_region_size, 0);
-}
-
-void HMSMRDevice::reset_zone(uint64_t zone)
-{
-  dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl;
-  if (zbd_reset_zones(zbd_fd, zone * zone_size, zone_size) != 0) {
-    derr << __func__ << " resetting zone failed for zone 0x" << std::hex
-	 << zone << std::dec << dendl;
-    ceph_abort("zbd_reset_zones failed");
-  }
-}
-
-std::vector<uint64_t> HMSMRDevice::get_zones()
-{
-  std::vector<zbd_zone> zones;
-  unsigned int num_zones = size / zone_size;
-  zones.resize(num_zones);
-
-  int r = zbd_report_zones(zbd_fd, 0, 0, ZBD_RO_ALL, zones.data(), &num_zones);
-  if (r != 0) {
-    derr << __func__ << " zbd_report_zones failed on " << path << ": "
-	 << cpp_strerror(errno) << dendl;
-    ceph_abort("zbd_report_zones failed");
-  }
-
-  std::vector<uint64_t> wp(num_zones);
-  for (unsigned i = 0; i < num_zones; ++i) {
-    wp[i] = zones[i].wp;
-  }
-  return wp;
-}
diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h
deleted file mode 100644
index edf18b5f0ba3..000000000000
--- a/src/blk/zoned/HMSMRDevice.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2014 Red Hat
- * Copyright (C) 2020 Abutalib Aghayev
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef CEPH_BLK_HMSMRDEVICE_H
-#define CEPH_BLK_HMSMRDEVICE_H
-
-#include <atomic>
-
-#include "include/types.h"
-#include "include/interval_set.h"
-#include "common/Thread.h"
-#include "include/utime.h"
-
-#include "aio/aio.h"
-#include "BlockDevice.h"
-#include "../kernel/KernelDevice.h"
-
-
-class HMSMRDevice final : public KernelDevice {
-  int zbd_fd = -1;	///< fd for the zoned block device
-
-public:
-  HMSMRDevice(CephContext* cct, aio_callback_t cb, void *cbpriv,
-              aio_callback_t d_cb, void *d_cbpriv);
-
-  static bool support(const std::string& path);
-
-  // open/close hooks for libzbd
-  int _post_open() override;
-  void _pre_close() override;
-
-  // smr-specific methods
-  bool is_smr() const final { return true; }
-  void reset_all_zones() override;
-  void reset_zone(uint64_t zone) override;
-  std::vector<uint64_t> get_zones() override;
-
-};
-
-#endif //CEPH_BLK_HMSMRDEVICE_H
diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt
index 20e03d3305ff..e446d4099f7b 100644
--- a/src/crimson/os/alienstore/CMakeLists.txt
+++ b/src/crimson/os/alienstore/CMakeLists.txt
@@ -63,11 +63,6 @@ set(alien_store_srcs
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/StupidAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/memstore/MemStore.cc)
-if(WITH_ZBD)
-  list(APPEND alien_store_srcs
-    ${PROJECT_SOURCE_DIR}/src/os/bluestore/ZonedFreelistManager.cc
-    ${PROJECT_SOURCE_DIR}/src/os/bluestore/ZonedAllocator.cc)
-endif()
 add_library(crimson-alienstore STATIC
   ${alien_store_srcs})
 if(WITH_LTTNG)
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index f14a1f43a600..c983eff39637 100644
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -87,9 +87,6 @@
 /* Defined if you have libdml */
 #cmakedefine HAVE_LIBDML
 
-/* Defined if you have libzbd */
-#cmakedefine HAVE_LIBZBD
-
 /* Defined if you have liburing */
 #cmakedefine HAVE_LIBURING
 
diff --git a/src/os/bluestore/Allocator.cc b/src/os/bluestore/Allocator.cc
index 7029420b53d0..60265939988e 100644
--- a/src/os/bluestore/Allocator.cc
+++ b/src/os/bluestore/Allocator.cc
@@ -8,9 +8,6 @@
 #include "AvlAllocator.h"
 #include "BtreeAllocator.h"
 #include "HybridAllocator.h"
-#ifdef HAVE_LIBZBD
-#include "ZonedAllocator.h"
-#endif
 #include "common/debug.h"
 #include "common/admin_socket.h"
 #define dout_subsys ceph_subsys_bluestore
diff --git a/src/os/bluestore/FreelistManager.cc b/src/os/bluestore/FreelistManager.cc
index 37347ced66bd..1ea07bf6e4c9 100644
--- a/src/os/bluestore/FreelistManager.cc
+++ b/src/os/bluestore/FreelistManager.cc
@@ -3,9 +3,6 @@
 
 #include "FreelistManager.h"
 #include "BitmapFreelistManager.h"
-#ifdef HAVE_LIBZBD
-#include "ZonedFreelistManager.h"
-#endif
 
 FreelistManager *FreelistManager::create(
   CephContext* cct,
@@ -33,10 +30,5 @@ FreelistManager *FreelistManager::create(
 void FreelistManager::setup_merge_operators(KeyValueDB *db,
 					    const std::string& type)
 {
-#ifdef HAVE_LIBZBD
-  if (type == "zoned")
-    ZonedFreelistManager::setup_merge_operator(db, "z");
-  else
-#endif
-    BitmapFreelistManager::setup_merge_operator(db, "b");
+  BitmapFreelistManager::setup_merge_operator(db, "b");
 }
diff --git a/src/test/objectstore/Allocator_test.cc b/src/test/objectstore/Allocator_test.cc
index 0e76c479002a..1325e89b0da8 100644
--- a/src/test/objectstore/Allocator_test.cc
+++ b/src/test/objectstore/Allocator_test.cc
@@ -26,8 +26,7 @@ class AllocTest : public ::testing::TestWithParam<const char*> {
   void init_alloc(int64_t size, uint64_t min_alloc_size) {
     std::cout << "Creating alloc type " << string(GetParam()) << " \n";
     alloc.reset(Allocator::create(g_ceph_context, GetParam(), size,
-				  min_alloc_size,
-				  256*1048576, 100*256*1048576ull));
+				  min_alloc_size));
   }
 
   void init_close() {
diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc
index 7d0d9420f6be..4405c30611ef 100644
--- a/src/test/objectstore/allocator_replay_test.cc
+++ b/src/test/objectstore/allocator_replay_test.cc
@@ -391,7 +391,7 @@ int replay_free_dump_and_apply(char* fname,
                        std::string_view alloc_name) {
     alloc.reset(
       Allocator::create(
-        g_ceph_context, alloc_type, capacity, alloc_unit, 0, 0, alloc_name));
+        g_ceph_context, alloc_type, capacity, alloc_unit, alloc_name));
   };
   auto add_fn = [&](uint64_t offset,
                    uint64_t len) {
diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc
index 007f47837cef..795b8b054c90 100644
--- a/src/test/objectstore/test_bluefs.cc
+++ b/src/test/objectstore/test_bluefs.cc
@@ -1125,7 +1125,7 @@ TEST(BlueFS, test_shared_alloc) {
   uint64_t shared_alloc_unit = 4096;
   shared_alloc.set(
     Allocator::create(g_ceph_context, g_ceph_context->_conf->bluefs_allocator,
-                      size, shared_alloc_unit, 0, 0, "test shared allocator"),
+                      size, shared_alloc_unit, "test shared allocator"),
     shared_alloc_unit);
   shared_alloc.a->init_add_free(0, size);
 
@@ -1196,7 +1196,7 @@ TEST(BlueFS, test_shared_alloc_sparse) {
   bluefs_shared_alloc_context_t shared_alloc;
   shared_alloc.set(
     Allocator::create(g_ceph_context, g_ceph_context->_conf->bluefs_allocator,
-                      size, main_unit, 0, 0, "test shared allocator"),
+                      size, main_unit, "test shared allocator"),
     main_unit);
   // prepare sparse free space but let's have a continuous chunk at
   // the beginning to fit initial log's fnode into superblock,
@@ -1273,7 +1273,7 @@ TEST(BlueFS, test_4k_shared_alloc) {
   bluefs_shared_alloc_context_t shared_alloc;
   shared_alloc.set(
     Allocator::create(g_ceph_context, g_ceph_context->_conf->bluefs_allocator,
-                      size, main_unit, 0, 0, "test shared allocator"),
+                      size, main_unit, "test shared allocator"),
     main_unit);
   shared_alloc.a->init_add_free(bluefs_alloc_unit, size - bluefs_alloc_unit);
 
diff --git a/src/vstart.sh b/src/vstart.sh
index a76eadfb54ec..d860a6e104e0 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -172,7 +172,6 @@ rgw_store="rados"
 lockdep=${LOCKDEP:-1}
 spdk_enabled=0 # disable SPDK by default
 pmem_enabled=0
-zoned_enabled=0
 io_uring_enabled=0
 with_jaeger=0
 
@@ -842,13 +841,6 @@ EOF
                 BLUESTORE_OPTS=""
             fi
         fi
-        if [ "$zoned_enabled" -eq 1 ]; then
-            BLUESTORE_OPTS+="
-        bluestore min alloc size = 65536
-        bluestore prefer deferred size = 0
-        bluestore prefer deferred size hdd = 0
-        bluestore prefer deferred size ssd = 0
-        fi
         if [ "$io_uring_enabled" -eq 1 ]; then
             BLUESTORE_OPTS+="
         bdev ioring = true"

From aeeee3f2becae65d24274c46638a38bdafe7120e Mon Sep 17 00:00:00 2001
From: Myoungwon Oh <myoungwon.oh@samsung.com>
Date: Wed, 30 Aug 2023 16:41:43 +0900
Subject: [PATCH 1296/2492] osd: check if adjacent clones are unreadable when
 rollback is called

In rollback, if the head object is not manfest and the rollback_to is manifest,
the head object becomes the manifest object. At this point,
we need to check adjacent clonces to the head object to calculate
correct reference count for deduped chunks because the head object is now manifest.
Plus, in the meantime of the waiting for recovery, subsequent requests should be blocked
to ensure transaction order.

To fix the issue, this commit is to wait until the adjacent
clones are readable and delay future incoming ops.

fixes: https://tracker.ceph.com/issues/62167

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/osd/PG.h            |   1 +
 src/osd/PrimaryLogPG.cc | 122 ++++++++++++++++++++++++++++++++--------
 src/osd/PrimaryLogPG.h  |   3 +
 3 files changed, 103 insertions(+), 23 deletions(-)

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2a823f675969..966a83545037 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1115,6 +1115,7 @@ class PG : public DoutPrefixProvider,
 
   std::set<hobject_t> objects_blocked_on_cache_full;
   std::map<hobject_t,snapid_t> objects_blocked_on_degraded_snap;
+  std::map<hobject_t,snapid_t> objects_blocked_on_unreadable_snap;
   std::map<hobject_t,ObjectContextRef> objects_blocked_on_snap_promotion;
 
   // Callbacks should assume pg (and nothing else) is locked
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index d57ab432d4ac..a4e45e318f81 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -467,6 +467,7 @@ void PrimaryLogPG::on_local_recover(
 	dout(20) << " kicking unreadable waiters on " << hoid << dendl;
 	requeue_ops(unreadable_object_entry->second);
 	waiting_for_unreadable_object.erase(unreadable_object_entry);
+	finish_unreadable_object(unreadable_object_entry->first);
       }
     }
   } else {
@@ -520,6 +521,7 @@ void PrimaryLogPG::on_global_recover(
     waiting_for_unreadable_object.erase(unreadable_object_entry);
   }
   finish_degraded_object(soid);
+  finish_unreadable_object(soid);
 }
 
 void PrimaryLogPG::schedule_recovery_work(
@@ -747,6 +749,18 @@ void PrimaryLogPG::block_write_on_degraded_snap(
   wait_for_degraded_object(snap, op);
 }
 
+void PrimaryLogPG::block_write_on_unreadable_snap(
+  const hobject_t& snap, OpRequestRef op)
+{
+  dout(20) << __func__ << ": blocking object " << snap.get_head()
+	   << " on unreadable snap " << snap << dendl;
+  // otherwise, we'd have blocked in do_op
+  ceph_assert(objects_blocked_on_unreadable_snap.count(snap.get_head()) == 0);
+  objects_blocked_on_unreadable_snap[snap.get_head()] = snap.snap;
+  // the op must be queued before calling block_write_on_unreadable_snap
+  ceph_assert(waiting_for_unreadable_object.count(snap) == 1);
+}
+
 bool PrimaryLogPG::maybe_await_blocked_head(
   const hobject_t &hoid,
   OpRequestRef op)
@@ -2196,6 +2210,14 @@ void PrimaryLogPG::do_op(OpRequestRef& op)
       return;
     }
 
+    if (auto blocked_iter = objects_blocked_on_unreadable_snap.find(head);
+	blocked_iter != std::end(objects_blocked_on_unreadable_snap)) {
+      hobject_t to_wait_on(head);
+      to_wait_on.snap = blocked_iter->second;
+      wait_for_unreadable_object(to_wait_on, op);
+      return;
+    }
+
     // blocked on snap?
     if (auto blocked_iter = objects_blocked_on_degraded_snap.find(head);
 	blocked_iter != std::end(objects_blocked_on_degraded_snap)) {
@@ -3468,47 +3490,56 @@ int PrimaryLogPG::get_manifest_ref_count(ObjectContextRef obc, std::string& fp_o
   return cnt;
 }
 
-bool PrimaryLogPG::recover_adjacent_clones(ObjectContextRef obc, OpRequestRef op)
+snapid_t PrimaryLogPG::do_recover_adjacent_clones(ObjectContextRef obc, OpRequestRef op) 
 {
-  if (!obc->ssc || !obc->ssc->snapset.clones.size()) {
-    return false;
-  }
-  MOSDOp *m = static_cast<MOSDOp*>(op->get_nonconst_req());
-  bool has_manifest_op = std::any_of(
-    begin(m->ops),
-    end(m->ops),
-    [](const auto& osd_op) {
-       return osd_op.op.op == CEPH_OSD_OP_SET_CHUNK;
-    });
-  if (!obc->obs.oi.manifest.is_chunked() && !has_manifest_op) {
-    return false;
-  }
   ceph_assert(op);
-
   const SnapSet& snapset = obc->ssc->snapset;
   auto s = std::find(snapset.clones.begin(), snapset.clones.end(), obc->obs.oi.soid.snap);
-  auto is_unreadable_snap = [this, obc, &snapset, op](auto iter) -> bool {
+  auto is_unreadable_snap = [this, obc, &snapset, op](auto iter) -> snapid_t {
     hobject_t cid = obc->obs.oi.soid;
     cid.snap = (iter == snapset.clones.end()) ? snapid_t(CEPH_NOSNAP) : *iter;
     if (is_unreadable_object(cid)) {
       dout(10) << __func__ << ": clone " << cid
 	       << " is unreadable, waiting" << dendl;
       wait_for_unreadable_object(cid, op);
-      return true;
+      return cid.snap;
     }
-    return false;
+    return snapid_t();
   };
   if (s != snapset.clones.begin()) {
-    if (is_unreadable_snap(s - 1)) {
-      return true;
+    snapid_t snap = is_unreadable_snap(s - 1);
+    if (snap != snapid_t()) {
+      return snap;
     }
   }
   if (s != snapset.clones.end()) {
-    if (is_unreadable_snap(s + 1)) {
-      return true;
+    snapid_t snap = is_unreadable_snap(s + 1);
+    if (snap != snapid_t()) {
+      return snap;
     }
   }
-  return false;
+  return snapid_t();
+}
+
+bool PrimaryLogPG::recover_adjacent_clones(ObjectContextRef obc, OpRequestRef op)
+{
+  if (!obc->ssc || !obc->ssc->snapset.clones.size()) {
+    return false;
+  }
+  MOSDOp *m = static_cast<MOSDOp*>(op->get_nonconst_req());
+  bool has_manifest_op = false;
+  for (auto& osd_op : m->ops) {
+    if (osd_op.op.op == CEPH_OSD_OP_ROLLBACK) {
+      return false;
+    } else if (osd_op.op.op == CEPH_OSD_OP_SET_CHUNK) {
+      has_manifest_op = true;	
+      break;
+    }
+  }
+  if (!obc->obs.oi.manifest.is_chunked() && !has_manifest_op) {
+    return false;
+  }
+  return do_recover_adjacent_clones(obc, op) != snapid_t();
 }
 
 ObjectContextRef PrimaryLogPG::get_prev_clone_obc(ObjectContextRef obc)
@@ -8289,6 +8320,40 @@ int PrimaryLogPG::_rollback_to(OpContext *ctx, OSDOp& op)
     block_write_on_degraded_snap(missing_oid, ctx->op);
     return ret;
   }
+  /*
+   * In rollback, if the head object is not manfest and the rollback_to is manifest,
+   * the head object will become the manifest object. At this point,
+   * we need to check adjacent clones beside the head object to calculate 
+   * correct reference count for deduped chunks because the head object is now 
+   * manifest. The reverse is also true---the head object is manifest, but the rollback_to
+   * is not manifest.
+   * Therefore, the following lines inserts the op to the waiting queue to wait until
+   * unreadable object is recovered if either adjacent clones is 
+   * unreadable to calculate chunk references.
+   */
+  auto block_write_if_unreadable = [this](ObjectContextRef obc, OpRequestRef op) {
+    snapid_t sid = do_recover_adjacent_clones(obc, op);
+    if (sid != snapid_t()) {
+      hobject_t oid = obc->obs.oi.soid; 
+      oid.snap = sid;
+      block_write_on_unreadable_snap(oid, op);
+      return -EAGAIN;
+    } 
+    return 0;
+  };
+  if (oi.has_manifest() && oi.manifest.is_chunked()) {
+    int r = block_write_if_unreadable(ctx->obc, ctx->op);
+    if (r < 0) {
+      return r;
+    }
+  }
+  if (rollback_to && rollback_to->obs.oi.has_manifest() &&
+      rollback_to->obs.oi.manifest.is_chunked()) {
+    int r = block_write_if_unreadable(rollback_to, ctx->op);
+    if (r < 0) {
+      return r;
+    }
+  }
   {
     ObjectContextRef promote_obc;
     cache_result_t tier_mode_result;
@@ -12431,6 +12496,16 @@ void PrimaryLogPG::finish_degraded_object(const hobject_t oid)
     objects_blocked_on_degraded_snap.erase(i);
 }
 
+void PrimaryLogPG::finish_unreadable_object(const hobject_t oid)
+{
+  dout(10) << __func__ << " " << oid << dendl;
+  map<hobject_t, snapid_t>::iterator i = objects_blocked_on_unreadable_snap.find(
+    oid.get_head());
+  if (i != objects_blocked_on_unreadable_snap.end() &&
+      i->second == oid.snap)
+    objects_blocked_on_unreadable_snap.erase(i);
+}
+
 void PrimaryLogPG::_committed_pushed_object(
   epoch_t epoch, eversion_t last_complete)
 {
@@ -13172,6 +13247,7 @@ void PrimaryLogPG::cancel_pull(const hobject_t &soid)
   if (is_missing_object(soid))
     recovery_state.set_last_requested(0);
   finish_degraded_object(soid);
+  finish_unreadable_object(soid);
 }
 
 void PrimaryLogPG::check_recovery_sources(const OSDMapRef& osdmap)
diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 6ed29927463d..a8a0c5a7fc0e 100644
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -1451,6 +1451,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   void dec_refcount_by_dirty(OpContext* ctx);
   ObjectContextRef get_prev_clone_obc(ObjectContextRef obc);
   bool recover_adjacent_clones(ObjectContextRef obc, OpRequestRef op);
+  snapid_t do_recover_adjacent_clones(ObjectContextRef obc, OpRequestRef op);
   void get_adjacent_clones(ObjectContextRef src_obc, 
 			   ObjectContextRef& _l, ObjectContextRef& _g);
   bool inc_refcount_by_set(OpContext* ctx, object_manifest_t& tgt,
@@ -1833,6 +1834,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   }
   void maybe_kick_recovery(const hobject_t &soid);
   void wait_for_unreadable_object(const hobject_t& oid, OpRequestRef op);
+  void finish_unreadable_object(const hobject_t oid);
 
   int get_manifest_ref_count(ObjectContextRef obc, std::string& fp_oid, OpRequestRef op);
 
@@ -1863,6 +1865,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   void block_write_on_snap_rollback(
     const hobject_t& oid, ObjectContextRef obc, OpRequestRef op);
   void block_write_on_degraded_snap(const hobject_t& oid, OpRequestRef op);
+  void block_write_on_unreadable_snap(const hobject_t& snap, OpRequestRef op);
 
   bool maybe_await_blocked_head(const hobject_t &soid, OpRequestRef op);
   void wait_for_blocked_object(const hobject_t& soid, OpRequestRef op);

From 38e74cfe96bec00253b8818301425ddf7c274cad Mon Sep 17 00:00:00 2001
From: avanthakkar <avanjohn@gmail.com>
Date: Tue, 19 Sep 2023 15:50:10 +0530
Subject: [PATCH 1297/2492] cephadm/rgw: make rgw perf counters cache and it's
 size configurable

Fixes: https://tracker.ceph.com/issues/62877
Signed-off-by: avanthakkar <avanjohn@gmail.com>
---
 .../mgr/cephadm/services/cephadmservice.py    | 34 ++++++++++++++++++-
 .../ceph/deployment/service_spec.py           | 15 +++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py
index f1d405edda0b..c2cf64475e0b 100644
--- a/src/pybind/mgr/cephadm/services/cephadmservice.py
+++ b/src/pybind/mgr/cephadm/services/cephadmservice.py
@@ -1022,14 +1022,46 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
             args.extend(spec.rgw_frontend_extra_args)
 
         frontend = f'{ftype} {" ".join(args)}'
+        daemon_name = utils.name_to_config_section(daemon_spec.name())
 
         ret, out, err = self.mgr.check_mon_command({
             'prefix': 'config set',
-            'who': utils.name_to_config_section(daemon_spec.name()),
+            'who': daemon_name,
             'name': 'rgw_frontends',
             'value': frontend
         })
 
+        if spec.rgw_user_counters_cache:
+            ret, out, err = self.mgr.check_mon_command({
+                'prefix': 'config set',
+                'who': daemon_name,
+                'name': 'rgw_user_counters_cache',
+                'value': 'true',
+            })
+        if spec.rgw_bucket_counters_cache:
+            ret, out, err = self.mgr.check_mon_command({
+                'prefix': 'config set',
+                'who': daemon_name,
+                'name': 'rgw_bucket_counters_cache',
+                'value': 'true',
+            })
+
+        if spec.rgw_user_counters_cache_size:
+            ret, out, err = self.mgr.check_mon_command({
+                'prefix': 'config set',
+                'who': daemon_name,
+                'name': 'rgw_user_counters_cache_size',
+                'value': str(spec.rgw_user_counters_cache_size),
+            })
+
+        if spec.rgw_bucket_counters_cache_size:
+            ret, out, err = self.mgr.check_mon_command({
+                'prefix': 'config set',
+                'who': daemon_name,
+                'name': 'rgw_bucket_counters_cache_size',
+                'value': str(spec.rgw_bucket_counters_cache_size),
+            })
+
         daemon_spec.keyring = keyring
         daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
 
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index cb81cf7b5c8b..4432d1c9f5f0 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1167,7 +1167,11 @@ def __init__(self,
                  custom_configs: Optional[List[CustomConfig]] = None,
                  rgw_realm_token: Optional[str] = None,
                  update_endpoints: Optional[bool] = False,
-                 zone_endpoints: Optional[str] = None  # commad separated endpoints list
+                 zone_endpoints: Optional[str] = None,  # comma separated endpoints list
+                 rgw_user_counters_cache: Optional[bool] = False,
+                 rgw_user_counters_cache_size: Optional[int] = None,
+                 rgw_bucket_counters_cache: Optional[bool] = False,
+                 rgw_bucket_counters_cache_size: Optional[int] = None
                  ):
         assert service_type == 'rgw', service_type
 
@@ -1208,6 +1212,15 @@ def __init__(self,
         self.update_endpoints = update_endpoints
         self.zone_endpoints = zone_endpoints
 
+        #: To track op metrics by user config value rgw_user_counters_cache must be set to true
+        self.rgw_user_counters_cache = rgw_user_counters_cache
+        #: Used to set number of entries in each cache of user counters
+        self.rgw_user_counters_cache_size = rgw_user_counters_cache_size
+        #: To track op metrics by bucket config value rgw_bucket_counters_cache must be set to true
+        self.rgw_bucket_counters_cache = rgw_bucket_counters_cache
+        #: Used to set number of entries in each cache of bucket counters
+        self.rgw_bucket_counters_cache_size = rgw_bucket_counters_cache_size
+
     def get_port_start(self) -> List[int]:
         return [self.get_port()]
 

From 2a6ce1341a58bba227efb6a916cdb98e85a0af1a Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 9 Jan 2024 06:51:46 -0600
Subject: [PATCH 1298/2492] osd/scrub: allow replicas to scrub even if OSD has
 recovery in progress

Limit the check ("not scrubbing if the OSD is performing recovery on
some PG") to the primary OSD. Otherwise - a small number of backfills may
prevent a large number of PGs from scrubbing.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_machine.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 9aa5842c4908..eac26a85f851 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -819,11 +819,6 @@ ReplicaActive::ReservationAttemptRes ReplicaActive::get_remote_reservation()
 {
   using ReservationAttemptRes = ReplicaActive::ReservationAttemptRes;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  if (!scrbr->get_pg_cct()->_conf.get_val<bool>("osd_scrub_during_recovery") &&
-      m_osds->is_recovery_active()) {
-    return ReservationAttemptRes{
-	MOSDScrubReserve::REJECT, "recovery is active", false};
-  }
 
   if (m_osds->get_scrub_services().inc_scrubs_remote(scrbr->get_spgid().pgid)) {
     return ReservationAttemptRes{MOSDScrubReserve::GRANT, "", true};

From dcac099a46596091292296fc5e20f14f704a30a6 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 11:33:48 +0000
Subject: [PATCH 1299/2492] crimson/osd/osd_operations/snaptrim_event: move
 SubOpBlocker to seperate file

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/common/subop_blocker.h            | 53 +++++++++++++++++++
 .../osd/osd_operations/snaptrim_event.cc      | 27 +---------
 .../osd/osd_operations/snaptrim_event.h       | 18 +------
 3 files changed, 56 insertions(+), 42 deletions(-)
 create mode 100644 src/crimson/common/subop_blocker.h

diff --git a/src/crimson/common/subop_blocker.h b/src/crimson/common/subop_blocker.h
new file mode 100644
index 000000000000..a4bb3d8f7108
--- /dev/null
+++ b/src/crimson/common/subop_blocker.h
@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include "osd/osd_op_util.h"
+#include "crimson/osd/osd_operation.h"
+
+namespace crimson::osd {
+
+using interruptor =
+  ::crimson::interruptible::interruptor<
+    ::crimson::osd::IOInterruptCondition>;
+
+// bases on 998cb8c141bb89aafae298a9d5e130fbd78fe5f2
+template <typename T>
+struct SubOpBlocker : crimson::BlockerT<SubOpBlocker<T>> {
+  static constexpr const char* type_name = "CompoundOpBlocker";
+
+  using id_done_t = std::pair<crimson::OperationRef, T>;
+
+  void dump_detail(Formatter *f) const final {
+    f->open_array_section("dependent_operations");
+    {
+      for (const auto &kv : subops) {
+        f->dump_unsigned("op_id", kv.first->get_id());
+      }
+    }
+    f->close_section();
+  }
+
+  template <class... Args>
+  void emplace_back(Args&&... args) {
+      subops.emplace_back(std::forward<Args>(args)...);
+  };
+
+  T interruptible_wait_completion() {
+    return interruptor::do_for_each(subops, [](auto&& kv) {
+      return std::move(kv.second);
+    });
+  }
+
+  T wait_completion() {
+    return seastar::do_for_each(subops, [](auto&& kv) {
+      return std::move(kv.second);
+    });
+  }
+
+private:
+  std::vector<id_done_t> subops;
+};
+
+} // namespace crimson::osd
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index b2e294316dbc..01ea2a710c33 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -39,31 +39,6 @@ PG::BackgroundProcessLock::lock_with_op(SnapTrimEvent &st_event) noexcept
   });
 }
 
-void SnapTrimEvent::SubOpBlocker::dump_detail(Formatter *f) const
-{
-  f->open_array_section("dependent_operations");
-  {
-    for (const auto &kv : subops) {
-      f->dump_unsigned("op_id", kv.first->get_id());
-    }
-  }
-  f->close_section();
-}
-
-template <class... Args>
-void SnapTrimEvent::SubOpBlocker::emplace_back(Args&&... args)
-{
-  subops.emplace_back(std::forward<Args>(args)...);
-};
-
-SnapTrimEvent::remove_or_update_iertr::future<>
-SnapTrimEvent::SubOpBlocker::wait_completion()
-{
-  return interruptor::do_for_each(subops, [](auto&& kv) {
-    return std::move(kv.second);
-  });
-}
-
 void SnapTrimEvent::print(std::ostream &lhs) const
 {
   lhs << "SnapTrimEvent("
@@ -161,7 +136,7 @@ SnapTrimEvent::start()
 	  return enter_stage<interruptor>(wait_subop);
 	}).then_interruptible([this] {
           logger().debug("{}: awaiting completion", *this);
-          return subop_blocker.wait_completion();
+          return subop_blocker.interruptible_wait_completion();
         }).finally([this] {
 	  pg->background_process_lock.unlock();
 	}).si_then([this] {
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 39038c71ef91..0548b396a075 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -8,6 +8,7 @@
 
 #include "crimson/osd/osdmap_gate.h"
 #include "crimson/osd/osd_operation.h"
+#include "crimson/common/subop_blocker.h"
 #include "crimson/osd/osd_operations/common/pg_pipeline.h"
 #include "crimson/osd/pg.h"
 #include "crimson/osd/pg_activation_blocker.h"
@@ -57,22 +58,7 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
 private:
   CommonPGPipeline& client_pp();
 
-  // bases on 998cb8c141bb89aafae298a9d5e130fbd78fe5f2
-  struct SubOpBlocker : crimson::BlockerT<SubOpBlocker> {
-    static constexpr const char* type_name = "CompoundOpBlocker";
-
-    using id_done_t = std::pair<crimson::OperationRef,
-                                remove_or_update_iertr::future<>>;
-
-    void dump_detail(Formatter *f) const final;
-
-    template <class... Args>
-    void emplace_back(Args&&... args);
-
-    remove_or_update_iertr::future<> wait_completion();
-  private:
-    std::vector<id_done_t> subops;
-  } subop_blocker;
+  SubOpBlocker<remove_or_update_iertr::future<>> subop_blocker;
 
   // we don't need to synchronize with other instances of SnapTrimEvent;
   // it's here for the sake of op tracking.

From f776610eeacbc7de97bc691face2043e6013baae Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 12:11:38 +0000
Subject: [PATCH 1300/2492] crimson/osd/osd_operations/snaptrim_event: using
 snap_trim_*_ret_t

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc |  8 ++++----
 src/crimson/osd/osd_operations/snaptrim_event.h  | 14 ++++++++++----
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index 01ea2a710c33..a29c57d102c1 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -60,7 +60,7 @@ CommonPGPipeline& SnapTrimEvent::client_pp()
   return pg->request_pg_pipeline;
 }
 
-SnapTrimEvent::snap_trim_ertr::future<seastar::stop_iteration>
+SnapTrimEvent::snap_trim_event_ret_t
 SnapTrimEvent::start()
 {
   ShardServices &shard_services = pg->get_shard_services();
@@ -167,7 +167,7 @@ SnapTrimEvent::start()
         });
       });
     });
-  }, [this](std::exception_ptr eptr) -> snap_trim_ertr::future<seastar::stop_iteration> {
+  }, [this](std::exception_ptr eptr) -> snap_trim_event_ret_t {
     logger().debug("{}: interrupted {}", *this, eptr);
     return crimson::ct_error::eagain::make();
   }, pg).finally([this] {
@@ -184,7 +184,7 @@ CommonPGPipeline& SnapTrimObjSubEvent::client_pp()
   return pg->request_pg_pipeline;
 }
 
-SnapTrimObjSubEvent::remove_or_update_iertr::future<>
+SnapTrimObjSubEvent::snap_trim_obj_subevent_ret_t
 SnapTrimObjSubEvent::remove_clone(
   ObjectContextRef obc,
   ObjectContextRef head_obc,
@@ -431,7 +431,7 @@ SnapTrimObjSubEvent::remove_or_update(
   });
 }
 
-SnapTrimObjSubEvent::remove_or_update_iertr::future<>
+SnapTrimObjSubEvent::snap_trim_obj_subevent_ret_t
 SnapTrimObjSubEvent::start()
 {
   return enter_stage<interruptor>(
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 0548b396a075..01cd82a245a7 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -39,6 +39,10 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
     crimson::ct_error::eagain>;
   using snap_trim_iertr = remove_or_update_iertr::extend<
     crimson::ct_error::eagain>;
+  using snap_trim_event_ret_t =
+    snap_trim_ertr::future<seastar::stop_iteration>;
+  using snap_trim_obj_subevent_ret_t =
+      remove_or_update_iertr::future<>;
 
   static constexpr OperationTypeCode type = OperationTypeCode::snaptrim_event;
 
@@ -53,12 +57,12 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
 
   void print(std::ostream &) const final;
   void dump_detail(ceph::Formatter* f) const final;
-  snap_trim_ertr::future<seastar::stop_iteration> start();
+  snap_trim_event_ret_t start();
 
 private:
   CommonPGPipeline& client_pp();
 
-  SubOpBlocker<remove_or_update_iertr::future<>> subop_blocker;
+  SubOpBlocker<snap_trim_obj_subevent_ret_t> subop_blocker;
 
   // we don't need to synchronize with other instances of SnapTrimEvent;
   // it's here for the sake of op tracking.
@@ -108,6 +112,8 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
   using remove_or_update_iertr =
     crimson::interruptible::interruptible_errorator<
       IOInterruptCondition, remove_or_update_ertr>;
+  using snap_trim_obj_subevent_ret_t =
+      remove_or_update_iertr::future<>;
 
   static constexpr OperationTypeCode type =
     OperationTypeCode::snaptrimobj_subevent;
@@ -123,7 +129,7 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
 
   void print(std::ostream &) const final;
   void dump_detail(ceph::Formatter* f) const final;
-  remove_or_update_iertr::future<> start();
+  snap_trim_obj_subevent_ret_t start();
 
   CommonPGPipeline& client_pp();
 
@@ -132,7 +138,7 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
    * https://tracker.ceph.com/issues/63307 */
   object_stat_sum_t delta_stats;
 
-  remove_or_update_iertr::future<> remove_clone(
+  snap_trim_obj_subevent_ret_t remove_clone(
     ObjectContextRef obc,
     ObjectContextRef head_obc,
     ceph::os::Transaction& txn);

From 7a56ae1edb950568eee60fafd88b9201a8f6f977 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 27 Nov 2023 16:08:01 +0000
Subject: [PATCH 1301/2492] crimson/osd: cleanup SubOpBlocker::emplace_back
 usage

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index a29c57d102c1..1a2292ae90fd 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -121,15 +121,12 @@ SnapTrimEvent::start()
         return [&shard_services, this](const auto &to_trim) {
 	  for (const auto& object : to_trim) {
 	    logger().debug("{}: trimming {}", *this, object);
-	    auto [op, fut] = shard_services.start_operation_may_interrupt<
-	      interruptor, SnapTrimObjSubEvent>(
-	      pg,
-	      object,
-	      snapid);
 	    subop_blocker.emplace_back(
-	      std::move(op),
-	      std::move(fut)
-	    );
+	      shard_services.start_operation_may_interrupt<
+	      interruptor, SnapTrimObjSubEvent>(
+	        pg,
+	        object,
+	        snapid));
 	  }
 	  return interruptor::now();
 	}(to_trim).then_interruptible([this] {

From 3a520eb217e0024db6e2dcee5e4fc792f915d070 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 28 Nov 2023 15:58:18 +0000
Subject: [PATCH 1302/2492] crimson/osd/osd_operations/background_recovery:
 cleanup unused logger

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/background_recovery.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc
index 1b9c5ba9b49d..f74933ec2660 100644
--- a/src/crimson/osd/osd_operations/background_recovery.cc
+++ b/src/crimson/osd/osd_operations/background_recovery.cc
@@ -12,12 +12,6 @@
 #include "crimson/osd/osd_operation_external_tracking.h"
 #include "crimson/osd/osd_operations/background_recovery.h"
 
-namespace {
-  seastar::logger& logger() {
-    return crimson::get_logger(ceph_subsys_osd);
-  }
-}
-
 namespace crimson {
   template <>
   struct EventBackendRegistry<osd::UrgentRecovery> {

From a59e708b788d915dec90c1679f42a3a0c309ac59 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 28 Nov 2023 15:41:35 +0000
Subject: [PATCH 1303/2492] crimson/osd/osd_operations: let pg reference
 destruct last

Prevent from SnapTrimEvent::handle destruct before SnapTrimEvent::pg.
The pg reference should be destructed last.

May releate to: https://tracker.ceph.com/issues/63647

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/pg_advance_map.cc | 4 ++--
 src/crimson/osd/osd_operations/pg_advance_map.h  | 4 ++--
 src/crimson/osd/osd_operations/snaptrim_event.h  | 2 +-
 src/crimson/osd/shard_services.cc                | 5 +++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/crimson/osd/osd_operations/pg_advance_map.cc b/src/crimson/osd/osd_operations/pg_advance_map.cc
index ba63212fc7fc..832794a56003 100644
--- a/src/crimson/osd/osd_operations/pg_advance_map.cc
+++ b/src/crimson/osd/osd_operations/pg_advance_map.cc
@@ -21,9 +21,9 @@ namespace {
 namespace crimson::osd {
 
 PGAdvanceMap::PGAdvanceMap(
-  ShardServices &shard_services, Ref<PG> pg, epoch_t to,
+  Ref<PG> pg, ShardServices &shard_services, epoch_t to,
   PeeringCtx &&rctx, bool do_init)
-  : shard_services(shard_services), pg(pg), to(to),
+  : pg(pg), shard_services(shard_services), to(to),
     rctx(std::move(rctx)), do_init(do_init)
 {
   logger().debug("{}: created", *this);
diff --git a/src/crimson/osd/osd_operations/pg_advance_map.h b/src/crimson/osd/osd_operations/pg_advance_map.h
index b712cc12e477..43be7319545b 100644
--- a/src/crimson/osd/osd_operations/pg_advance_map.h
+++ b/src/crimson/osd/osd_operations/pg_advance_map.h
@@ -25,8 +25,8 @@ class PGAdvanceMap : public PhasedOperationT<PGAdvanceMap> {
   static constexpr OperationTypeCode type = OperationTypeCode::pg_advance_map;
 
 protected:
-  ShardServices &shard_services;
   Ref<PG> pg;
+  ShardServices &shard_services;
   PipelineHandle handle;
 
   std::optional<epoch_t> from;
@@ -37,7 +37,7 @@ class PGAdvanceMap : public PhasedOperationT<PGAdvanceMap> {
 
 public:
   PGAdvanceMap(
-    ShardServices &shard_services, Ref<PG> pg, epoch_t to,
+    Ref<PG> pg, ShardServices &shard_services, epoch_t to,
     PeeringCtx &&rctx, bool do_init);
   ~PGAdvanceMap();
 
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 01cd82a245a7..23f8be26c841 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -77,8 +77,8 @@ class SnapTrimEvent final : public PhasedOperationT<SnapTrimEvent> {
     static constexpr auto type_name = "SnapTrimEvent::wait_trim_timer";
   } wait_trim_timer;
 
-  PipelineHandle handle;
   Ref<PG> pg;
+  PipelineHandle handle;
   SnapMapper& snap_mapper;
   const snapid_t snapid;
   const bool needs_pause;
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 8da4ccd8262c..d21c951b88cd 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -97,8 +97,9 @@ seastar::future<> PerShardState::broadcast_map_to_pgs(
     pgs.begin(), pgs.end(),
     [=, &shard_services](auto& pg) {
       return shard_services.start_operation<PGAdvanceMap>(
+	pg.second,
 	shard_services,
-	pg.second, epoch,
+	epoch,
 	PeeringCtx{}, false).second;
     });
 }
@@ -689,7 +690,7 @@ seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
 	    rctx.transaction);
 
 	  return start_operation<PGAdvanceMap>(
-	    *this, pg, get_map()->get_epoch(), std::move(rctx), true
+	    pg, *this, get_map()->get_epoch(), std::move(rctx), true
 	  ).second.then([pg=pg] {
 	    return seastar::make_ready_future<Ref<PG>>(pg);
 	  });

From 47fabe757cff630aaf23a85d268b7406057244e2 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 9 Jan 2024 23:42:34 +1000
Subject: [PATCH 1304/2492] doc/rgw: edit admin.rst - rate limit management

Edit the "Rate Limit Management" section in doc/radosgw/admin.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index a06b7a1eb3c8..88a7215fb156 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -584,20 +584,24 @@ commands, as in the following examples:
 Rate Limit Management
 =====================
 
-Quotas may be set for The Ceph Object Gateway on users and buckets.  "Rate
+Quotas can be set for The Ceph Object Gateway on users and buckets. The "rate
 limit" includes the maximum number of read operations (read ops) and write
-operations (write ops) per minute and the number of bytes per minute that can
-be written or read per user or per bucket.
+operations (write ops) per minute as well as the number of bytes per minute
+that can be written or read per user or per bucket.
 
+Read Requests and Write Requests
+--------------------------------
 Operations that use the ``GET`` method or the ``HEAD`` method in their REST
 requests are "read requests". All other requests are "write requests".  
 
+How Metrics Work
+----------------
 Each object gateway tracks per-user metrics separately from bucket metrics.
 These metrics are not shared with other gateways. The configured limits should
 be divided by the number of active object gateways. For example, if "user A" is
 to be be limited to 10 ops per minute and there are two object gateways in the
 cluster, then the limit on "user A" should be ``5`` (10 ops per minute / 2
-RGWs).  If the requests are **not** balanced between RGWs, the rate limit might
+RGWs). If the requests are **not** balanced between RGWs, the rate limit might
 be underutilized. For example: if the ops limit is ``5`` and there are two
 RGWs, **but** the Load Balancer sends load to only one of those RGWs, the
 effective limit is 5 ops, because this limit is enforced per RGW. If the rate
@@ -626,22 +630,25 @@ time has elapsed, "user A" will be able to send ``GET`` requests again.
 - **User:** The ``--uid`` option allows you to specify a rate limit for a
   user.
 
-- **Maximum Read Ops:** The ``--max-read-ops`` setting allows you to specify
-  the maximum number of read ops per minute per RGW. A 0 value disables this setting (which means unlimited access).
+- **Maximum Read Ops:** The ``--max-read-ops`` setting allows you to limit read
+  bytes per minute per RGW instance. A ``0`` value disables throttling. 
   
-- **Maximum Read Bytes:** The ``--max-read-bytes`` setting allows you to specify
-  the maximum number of read bytes per minute per RGW. A 0 value disables this setting (which means unlimited access).
+- **Maximum Read Bytes:** The ``--max-read-bytes`` setting allows you to limit
+  read bytes per minute per RGW instance. A ``0`` value disables throttling. 
 
 - **Maximum Write Ops:** The ``--max-write-ops`` setting allows you to specify
-  the maximum number of write ops per minute per RGW. A 0 value disables this setting (which means unlimited access).
+  the maximum number of write ops per minute per RGW instance. A ``0`` value
+  disables throttling.
   
-- **Maximum Write Bytes:** The ``--max-write-bytes`` setting allows you to specify
-  the maximum number of write bytes per minute per RGW. A 0 value disables this setting (which means unlimited access).
+- **Maximum Write Bytes:** The ``--max-write-bytes`` setting allows you to
+  specify the maximum number of write bytes per minute per RGW instance. A
+  ``0`` value disables throttling.
  
-- **Rate Limit Scope:** The ``--ratelimit-scope`` option sets the scope for the rate limit.
-  The options are ``bucket`` , ``user`` and ``anonymous``. Bucket rate limit apply to buckets. 
-  The user rate limit applies to a user. Anonymous applies to an unauthenticated user.
-  Anonymous scope is only available for global rate limit.
+- **Rate Limit Scope:** The ``--ratelimit-scope`` option sets the scope for the
+  rate limit.  The options are ``bucket`` , ``user`` and ``anonymous``. Bucket
+  rate limit apply to buckets.  The user rate limit applies to a user.  The
+  ``anonymous`` option applies to an unauthenticated user. Anonymous scope is
+  available only for global rate limit.
 
 
 Set User Rate Limit

From b37416b2f15e2622ed5c9d9176770b5d7a308120 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Mon, 11 Sep 2023 14:00:16 +0200
Subject: [PATCH 1305/2492] osd: rework the callback infrastructure around
 read_pipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 168 +++++++++++++++++++------------------------
 src/osd/ECBackend.h  |  29 +++++---
 2 files changed, 92 insertions(+), 105 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 6c7364f5f835..386c1eeabe32 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -254,48 +254,17 @@ void ECBackend::_failed_push(const hobject_t &hoid, ECBackend::read_result_t &re
   get_parent()->on_failed_pull(fl, hoid, v);
 }
 
-struct OnRecoveryReadComplete :
-  public GenContext<ECBackend::read_result_t&> {
-  struct RecoveryMessages* rm;
-  ECBackend *backend;
-  hobject_t hoid;
-
-  OnRecoveryReadComplete() = delete;
-  OnRecoveryReadComplete(RecoveryMessages* rm, ECBackend *backend, const hobject_t &hoid)
-    : rm(rm), backend(backend), hoid(hoid) {}
-  void finish(ECBackend::read_result_t &res) override {
-    if (!(res.r == 0 && res.errors.empty())) {
-        backend->_failed_push(hoid, res);
-        return;
-    }
-    ceph_assert(res.returned.size() == 1);
-    backend->handle_recovery_read_complete(
-      hoid,
-      res.returned.back(),
-      res.attrs,
-      rm);
-  }
-};
-
-struct RecoveryMessages : GenContext<int> {
-  ECBackend *ec;
+struct RecoveryMessages {
   map<hobject_t,
       ECBackend::read_request_t> recovery_reads;
-  RecoveryMessages* next_recovery_messages = nullptr;
   map<hobject_t, set<int>> want_to_read;
 
-  RecoveryMessages(ECBackend* ec) : ec(ec) {}
-
   void recovery_read(
-    ECBackend *ec,
     const hobject_t &hoid, uint64_t off, uint64_t len,
     set<int> &&_want_to_read,
     const map<pg_shard_t, vector<pair<int, int>>> &need,
     bool attrs)
   {
-    if (!next_recovery_messages) {
-      next_recovery_messages = new RecoveryMessages{ec};
-    }
     list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     to_read.push_back(boost::make_tuple(off, len, 0));
     ceph_assert(!recovery_reads.count(hoid));
@@ -306,22 +275,12 @@ struct RecoveryMessages : GenContext<int> {
 	ECBackend::read_request_t(
 	  to_read,
 	  need,
-	  attrs,
-	  new OnRecoveryReadComplete(
-	    next_recovery_messages,
-	    ec,
-	    hoid))));
+	  attrs)));
   }
 
   map<pg_shard_t, vector<PushOp> > pushes;
   map<pg_shard_t, vector<PushReplyOp> > push_replies;
   ObjectStore::Transaction t;
-  RecoveryMessages() = delete;
-  ~RecoveryMessages() {}
-
-  void finish(int priority) override {
-    ec->dispatch_recovery_messages(*this, priority);
-  }
 };
 
 void ECBackend::handle_recovery_push(
@@ -556,6 +515,36 @@ struct SendPushReplies : public Context {
   }
 };
 
+struct RecoveryReadCompleter : ECBackend::ReadCompleter {
+  RecoveryReadCompleter(ECBackend& backend)
+    : backend(backend) {}
+
+  void finish_single_request(
+    const hobject_t &hoid,
+    ECBackend::read_result_t &res,
+    list<boost::tuple<uint64_t, uint64_t, uint32_t> >) override
+  {
+    if (!(res.r == 0 && res.errors.empty())) {
+      backend._failed_push(hoid, res);
+      return;
+    }
+    ceph_assert(res.returned.size() == 1);
+    backend.handle_recovery_read_complete(
+      hoid,
+      res.returned.back(),
+      res.attrs,
+      &rm);
+  }
+
+  void finish(int priority) && override
+  {
+    backend.dispatch_recovery_messages(rm, priority);
+  }
+
+  ECBackend& backend;
+  RecoveryMessages rm;
+};
+
 void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
 {
   for (map<pg_shard_t, vector<PushOp> >::iterator i = m.pushes.begin();
@@ -607,8 +596,9 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
     m.want_to_read,
     m.recovery_reads,
     OpRequestRef(),
-    false, true, m.next_recovery_messages);
-  m.next_recovery_messages = nullptr;
+    false,
+    true,
+    std::make_unique<RecoveryReadCompleter>(*this));
 }
 
 void ECBackend::continue_recovery_op(
@@ -656,7 +646,6 @@ void ECBackend::continue_recovery_op(
 	return;
       }
       m->recovery_read(
-	this,
 	op.hoid,
 	op.recovery_progress.data_recovered_to,
 	amount,
@@ -773,7 +762,7 @@ void ECBackend::run_recovery_op(
   int priority)
 {
   ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
-  RecoveryMessages m{this};
+  RecoveryMessages m;
   for (list<RecoveryOp>::iterator i = h->ops.begin();
        i != h->ops.end();
        ++i) {
@@ -884,7 +873,7 @@ bool ECBackend::_handle_message(
   }
   case MSG_OSD_PG_PUSH: {
     auto op = _op->get_req<MOSDPGPush>();
-    RecoveryMessages rm{this};
+    RecoveryMessages rm;
     for (vector<PushOp>::const_iterator i = op->pushes.begin();
 	 i != op->pushes.end();
 	 ++i) {
@@ -896,7 +885,7 @@ bool ECBackend::_handle_message(
   case MSG_OSD_PG_PUSH_REPLY: {
     const MOSDPGPushReply *op = static_cast<const MOSDPGPushReply *>(
       _op->get_req());
-    RecoveryMessages rm{this};
+    RecoveryMessages rm;
     for (vector<PushReplyOp>::const_iterator i = op->replies.begin();
 	 i != op->replies.end();
 	 ++i) {
@@ -1366,11 +1355,14 @@ void ECBackend::ReadPipeline::complete_read_op(ReadOp &rop)
     rop.complete.begin();
   ceph_assert(rop.to_read.size() == rop.complete.size());
   for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
-    if (reqiter->second.cb) {
-      reqiter->second.cb->complete(resiter->second);
-      reqiter->second.cb = nullptr;
-    }
-  }
+    rop.on_complete->finish_single_request(
+      reqiter->first,
+      resiter->second,
+      reqiter->second.to_read);
+  }
+  ceph_assert(rop.on_complete);
+  std::move(*rop.on_complete).finish(rop.priority);
+  rop.on_complete = nullptr;
   // if the read op is over. clean all the data of this tid.
   for (set<pg_shard_t>::iterator iter = rop.in_progress.begin();
     iter != rop.in_progress.end();
@@ -1379,10 +1371,6 @@ void ECBackend::ReadPipeline::complete_read_op(ReadOp &rop)
   }
   rop.in_progress.clear();
   tid_to_read_map.erase(rop.tid);
-  if (rop.on_complete) {
-    rop.on_complete->complete(rop.priority);
-    rop.on_complete = nullptr;
-  }
 }
 
 struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
@@ -1445,10 +1433,6 @@ void ECBackend::ReadPipeline::filter_read_op(
     read_request_t &req = op.to_read.find(*i)->second;
     dout(10) << __func__ << ": canceling " << req
 	     << "  for obj " << *i << dendl;
-    ceph_assert(req.cb);
-    delete req.cb;
-    req.cb = nullptr;
-
     op.to_read.erase(*i);
     op.complete.erase(*i);
     on_erase(*i);
@@ -1516,13 +1500,6 @@ void ECBackend::ReadPipeline::on_change()
        i != tid_to_read_map.end();
        ++i) {
     dout(10) << __func__ << ": cancelling " << i->second << dendl;
-    for (map<hobject_t, read_request_t>::iterator j =
-	   i->second.to_read.begin();
-	 j != i->second.to_read.end();
-	 ++j) {
-      delete j->second.cb;
-      j->second.cb = nullptr;
-    }
   }
   tid_to_read_map.clear();
   shard_to_read_map.clear();
@@ -1843,7 +1820,7 @@ void ECBackend::ReadPipeline::start_read_op(
   OpRequestRef _op,
   bool do_redundant_reads,
   bool for_recovery,
-  GenContext<int> *on_complete)
+  std::unique_ptr<ECBackend::ReadCompleter> on_complete)
 {
   ceph_tid_t tid = get_parent()->get_tid();
   ceph_assert(!tid_to_read_map.count(tid));
@@ -1854,7 +1831,7 @@ void ECBackend::ReadPipeline::start_read_op(
       tid,
       do_redundant_reads,
       for_recovery,
-      on_complete,
+      std::move(on_complete),
       _op,
       std::move(want_to_read),
       std::move(to_read))).first->second;
@@ -2429,19 +2406,17 @@ void ECBackend::objects_read_async(
 	   on_complete)));
 }
 
-struct CallClientContexts :
-  public GenContext<ECBackend::read_result_t&> {
-  hobject_t hoid;
-  ECBackend::ReadPipeline &read_pipeline;
-  ECBackend::ClientAsyncReadStatus *status;
-  list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
-  CallClientContexts(
-    hobject_t hoid,
-    ECBackend::ReadPipeline &read_pipeline,
-    ECBackend::ClientAsyncReadStatus *status,
-    const list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read)
-      : hoid(hoid), read_pipeline(read_pipeline), status(status), to_read(to_read) {}
-  void finish(ECBackend::read_result_t &res) override {
+struct ClientReadCompleter : ECBackend::ReadCompleter {
+  ClientReadCompleter(ECBackend::ReadPipeline &read_pipeline,
+                      ECBackend::ClientAsyncReadStatus *status)
+    : read_pipeline(read_pipeline),
+      status(status) {}
+
+  void finish_single_request(
+    const hobject_t &hoid,
+    ECBackend::read_result_t &res,
+    list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) override
+  {
     extent_map result;
     if (res.r != 0)
       goto out;
@@ -2484,8 +2459,17 @@ struct CallClientContexts :
     status->complete_object(hoid, res.r, std::move(result));
     read_pipeline.kick_reads();
   }
+
+  void finish(int priority) && override
+  {
+    // NOP
+  }
+
+  ECBackend::ReadPipeline &read_pipeline;
+  ECBackend::ClientAsyncReadStatus *status;
 };
 
+
 void ECBackend::objects_read_and_reconstruct(
   const map<hobject_t,
     std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
@@ -2541,19 +2525,13 @@ void ECBackend::ReadPipeline::objects_read_and_reconstruct(
       &shards);
     ceph_assert(r == 0);
 
-    CallClientContexts *c = new CallClientContexts(
-      to_read.first,
-      *this,
-      &(in_progress_client_reads.back()),
-      to_read.second);
     for_read_op.insert(
       make_pair(
 	to_read.first,
 	read_request_t(
 	  to_read.second,
 	  shards,
-	  false,
-	  c)));
+	  false)));
     obj_want_to_read.insert(make_pair(to_read.first, want_to_read));
   }
 
@@ -2562,7 +2540,9 @@ void ECBackend::ReadPipeline::objects_read_and_reconstruct(
     obj_want_to_read,
     for_read_op,
     OpRequestRef(),
-    fast_read, false, nullptr);
+    fast_read,
+    false,
+    std::make_unique<ClientReadCompleter>(*this, &(in_progress_client_reads.back())));
   }();
   return;
 }
@@ -2585,7 +2565,6 @@ int ECBackend::ReadPipeline::send_all_remaining_reads(
 
   list<boost::tuple<uint64_t, uint64_t, uint32_t> > offsets =
     rop.to_read.find(hoid)->second.to_read;
-  GenContext<read_result_t&> *c = rop.to_read.find(hoid)->second.cb;
 
   // (Note cuixf) If we need to read attrs and we read failed, try to read again.
   bool want_attrs =
@@ -2601,8 +2580,7 @@ int ECBackend::ReadPipeline::send_all_remaining_reads(
       read_request_t(
 	offsets,
 	shards,
-	want_attrs,
-	c)));
+	want_attrs)));
   return 0;
 }
 
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 94267144ba4c..b859414160bd 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -272,6 +272,7 @@ class ECBackend : public PGBackend {
   friend struct RecoveryMessages;
   void dispatch_recovery_messages(RecoveryMessages &m, int priority);
   friend struct OnRecoveryReadComplete;
+  friend struct RecoveryReadCompleter;
   void handle_recovery_read_complete(
     const hobject_t &hoid,
     boost::tuple<uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > &to_read,
@@ -321,17 +322,25 @@ class ECBackend : public PGBackend {
     const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
     bool want_attrs;
-    GenContext<read_result_t&> *cb;
     read_request_t(
       const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
       const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
-      bool want_attrs,
-      GenContext<read_result_t&> *cb)
-      : to_read(to_read), need(need), want_attrs(want_attrs),
-	cb(cb) {}
+      bool want_attrs)
+      : to_read(to_read), need(need), want_attrs(want_attrs) {}
   };
   friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
 
+  struct ReadCompleter {
+    virtual void finish_single_request(
+      const hobject_t &hoid,
+      ECBackend::read_result_t &res,
+      std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) = 0;
+
+    virtual void finish(int priority) && = 0;
+
+    virtual ~ReadCompleter() = default;
+  };
+
   struct ReadOp {
     int priority;
     ceph_tid_t tid;
@@ -343,7 +352,7 @@ class ECBackend : public PGBackend {
     // True if reading for recovery which could possibly reading only a subset
     // of the available shards.
     bool for_recovery;
-    GenContext<int> *on_complete;
+    std::unique_ptr<ReadCompleter> on_complete;
 
     ZTracer::Trace trace;
 
@@ -363,7 +372,7 @@ class ECBackend : public PGBackend {
       ceph_tid_t tid,
       bool do_redundant_reads,
       bool for_recovery,
-      GenContext<int> *on_complete,
+      std::unique_ptr<ReadCompleter> _on_complete,
       OpRequestRef op,
       std::map<hobject_t, std::set<int>> &&_want_to_read,
       std::map<hobject_t, read_request_t> &&_to_read)
@@ -372,7 +381,7 @@ class ECBackend : public PGBackend {
         op(op),
         do_redundant_reads(do_redundant_reads),
         for_recovery(for_recovery),
-        on_complete(on_complete),
+        on_complete(std::move(_on_complete)),
         want_to_read(std::move(_want_to_read)),
 	to_read(std::move(_to_read)) {
       for (auto &&hpair: to_read) {
@@ -387,7 +396,7 @@ class ECBackend : public PGBackend {
       }
     }
     ReadOp() = delete;
-    ReadOp(const ReadOp &) = default;
+    ReadOp(const ReadOp &) = delete; // due to on_complete being unique_ptr
     ReadOp(ReadOp &&) = default;
   };
 
@@ -417,7 +426,7 @@ class ECBackend : public PGBackend {
       OpRequestRef op,
       bool do_redundant_reads,
       bool for_recovery,
-      GenContext<int> *on_complete);
+      std::unique_ptr<ReadCompleter> on_complete);
 
     void do_read_op(ReadOp &rop);
 

From 6ed1a6306fd3fa41d11a86641be39c97fc5aa473 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 13 Sep 2023 18:20:31 +0200
Subject: [PATCH 1306/2492] osd: ReadPipeline doesn't depend on ECBackend
 anymore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 2 +-
 src/osd/ECBackend.h  | 9 +++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 386c1eeabe32..d72751c4c36c 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -224,7 +224,7 @@ ECBackend::ECBackend(
   ErasureCodeInterfaceRef ec_impl,
   uint64_t stripe_width)
   : PGBackend(cct, pg, store, coll, ch),
-    read_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
+    read_pipeline(cct, ec_impl, this->sinfo, get_parent()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index b859414160bd..85e7a5b91595 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -445,9 +445,8 @@ class ECBackend : public PGBackend {
     CephContext* cct;
     ceph::ErasureCodeInterfaceRef ec_impl;
     const ECUtil::stripe_info_t& sinfo;
-    PGBackend::Listener* parent;
     // TODO: lay an interface down here
-    ECBackend& ec_backend;
+    PGBackend::Listener* parent;
 
     PGBackend::Listener *get_parent() const { return parent; }
     const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
@@ -457,13 +456,11 @@ class ECBackend : public PGBackend {
     ReadPipeline(CephContext* cct,
                 ceph::ErasureCodeInterfaceRef ec_impl,
                 const ECUtil::stripe_info_t& sinfo,
-                PGBackend::Listener* parent,
-                ECBackend& ec_backend)
+                PGBackend::Listener* parent)
       : cct(cct),
         ec_impl(std::move(ec_impl)),
         sinfo(sinfo),
-        parent(parent),
-        ec_backend(ec_backend) {
+        parent(parent) {
     }
 
     int get_remaining_shards(

From 9b0234501b89e652e9f4ef035d6e4203d8c50b04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 14 Sep 2023 15:02:05 +0200
Subject: [PATCH 1307/2492] osd: drop the lambda wrapping from
 objects_read_and_reconstruct()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index d72751c4c36c..a1d777cba248 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -2482,27 +2482,13 @@ void ECBackend::objects_read_and_reconstruct(
 }
 
 void ECBackend::ReadPipeline::objects_read_and_reconstruct(
-  ECBackend& ecbackend,
+  ECBackend& ec_backend,
   const map<hobject_t,
     std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
   > &reads,
   bool fast_read,
   GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
 {
-  return [this,
-    kick_reads=[this] (auto...) { return this->kick_reads();},
-    get_want_to_read_shards=[&ecbackend] (auto&&... args) {
-      return ecbackend.get_want_to_read_shards(std::forward<decltype(args)>(args)...);
-    },
-    get_min_avail_to_read_shards=[&ecbackend] (auto&&... args) {
-      return ecbackend.get_min_avail_to_read_shards(std::forward<decltype(args)>(args)...);
-    },
-    cct=(CephContext*)nullptr,
-    // params
-    &reads,
-    fast_read,
-    func=std::move(func)
-  ]() mutable {
   in_progress_client_reads.emplace_back(
     reads.size(), std::move(func));
   if (!reads.size()) {
@@ -2512,12 +2498,12 @@ void ECBackend::ReadPipeline::objects_read_and_reconstruct(
 
   map<hobject_t, set<int>> obj_want_to_read;
   set<int> want_to_read;
-  get_want_to_read_shards(&want_to_read);
+  ec_backend.get_want_to_read_shards(&want_to_read);
     
   map<hobject_t, read_request_t> for_read_op;
   for (auto &&to_read: reads) {
     map<pg_shard_t, vector<pair<int, int>>> shards;
-    int r = get_min_avail_to_read_shards(
+    int r = ec_backend.get_min_avail_to_read_shards(
       to_read.first,
       want_to_read,
       false,
@@ -2543,8 +2529,6 @@ void ECBackend::ReadPipeline::objects_read_and_reconstruct(
     fast_read,
     false,
     std::make_unique<ClientReadCompleter>(*this, &(in_progress_client_reads.back())));
-  }();
-  return;
 }
 
 

From 498d2d1e6309de7de4d2f651ade1cbc1ccebcfe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 14 Sep 2023 15:03:00 +0200
Subject: [PATCH 1308/2492] osd: drop unneeded get_hash_info() from EC's
 RMWPipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.h | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 85e7a5b91595..0bc4f9f423be 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -663,13 +663,6 @@ class ECBackend : public PGBackend {
     ) {
       ec_backend.handle_sub_write(from, std::move(msg), op, trace);
     }
-
-    ECUtil::HashInfoRef get_hash_info(
-      const hobject_t &hoid,
-      bool create
-    ) {
-      return ec_backend.get_hash_info(hoid, create);
-    }
     // end of iface
 
     ceph::ErasureCodeInterfaceRef ec_impl;

From 1b3040a24814a28b283e3eca57b257820b040b27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 15 Sep 2023 14:07:46 +0200
Subject: [PATCH 1309/2492] osd: lay down the ECListener for
 ECBackend::ReadPipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc    |  4 +--
 src/osd/ECBackend.h     | 54 ++++++++++++++++++++++++++++++++++++++---
 src/osd/PGBackend.h     | 21 ++--------------
 src/osd/PrimaryLogPG.cc |  5 ++++
 src/osd/PrimaryLogPG.h  | 37 +++++++++++++++++++++++++---
 5 files changed, 94 insertions(+), 27 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index a1d777cba248..e597129626f5 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -224,7 +224,7 @@ ECBackend::ECBackend(
   ErasureCodeInterfaceRef ec_impl,
   uint64_t stripe_width)
   : PGBackend(cct, pg, store, coll, ch),
-    read_pipeline(cct, ec_impl, this->sinfo, get_parent()),
+    read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
@@ -1897,7 +1897,7 @@ void ECBackend::ReadPipeline::do_read_op(ReadOp &op)
     MOSDECSubOpRead *msg = new MOSDECSubOpRead;
     msg->set_priority(priority);
     msg->pgid = spg_t(
-      get_parent()->whoami_spg_t().pgid,
+      get_info().pgid.pgid,
       i->first.shard);
     msg->map_epoch = get_osdmap_epoch();
     msg->min_epoch = get_parent()->get_interval_start_epoch();
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 0bc4f9f423be..238895d3e5b3 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -32,6 +32,54 @@ struct ECSubRead;
 struct ECSubReadReply;
 
 struct RecoveryMessages;
+
+  // ECListener -- an interface decoupling the pipelines from
+  // particular implementation of ECBackend (crimson vs cassical).
+  // https://stackoverflow.com/q/7872958
+  struct ECListener {
+    virtual ~ECListener() = default;
+    virtual const OSDMapRef& pgb_get_osdmap() const = 0;
+    virtual epoch_t pgb_get_osdmap_epoch() const = 0;
+    virtual const pg_info_t &get_info() const = 0;
+    /**
+     * Called when a pull on soid cannot be completed due to
+     * down peers
+     */
+    virtual void cancel_pull(
+      const hobject_t &soid) = 0;
+    virtual void schedule_recovery_work(
+      GenContext<ThreadPool::TPHandle&> *c,
+      uint64_t cost) = 0;
+
+    virtual epoch_t get_interval_start_epoch() const = 0;
+    virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
+    virtual const std::set<pg_shard_t> &get_backfill_shards() const = 0;
+    virtual const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_loc_shards()
+      const = 0;
+
+    virtual const std::map<pg_shard_t,
+			   pg_missing_t> &get_shard_missing() const = 0;
+    virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
+#if 1
+    virtual const pg_missing_const_i * maybe_get_shard_missing(
+      pg_shard_t peer) const = 0;
+    virtual const pg_info_t &get_shard_info(pg_shard_t peer) const = 0;
+#endif
+    virtual ceph_tid_t get_tid() = 0;
+    virtual pg_shard_t whoami_shard() const = 0;
+#if 0
+    int whoami() const {
+      return whoami_shard().osd;
+    }
+    spg_t whoami_spg_t() const {
+      return get_info().pgid;
+    }
+#endif
+    virtual void send_message_osd_cluster(
+      std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
+
+    virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
+  };
 class ECBackend : public PGBackend {
 public:
   RecoveryHandle *open_recovery_op() override;
@@ -446,9 +494,9 @@ class ECBackend : public PGBackend {
     ceph::ErasureCodeInterfaceRef ec_impl;
     const ECUtil::stripe_info_t& sinfo;
     // TODO: lay an interface down here
-    PGBackend::Listener* parent;
+    ECListener* parent;
 
-    PGBackend::Listener *get_parent() const { return parent; }
+    ECListener *get_parent() const { return parent; }
     const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
     epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
     const pg_info_t &get_info() { return get_parent()->get_info(); }
@@ -456,7 +504,7 @@ class ECBackend : public PGBackend {
     ReadPipeline(CephContext* cct,
                 ceph::ErasureCodeInterfaceRef ec_impl,
                 const ECUtil::stripe_info_t& sinfo,
-                PGBackend::Listener* parent)
+                ECListener* parent)
       : cct(cct),
         ec_impl(std::move(ec_impl)),
         sinfo(sinfo),
diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h
index ac17f05035db..274d1fb874bc 100644
--- a/src/osd/PGBackend.h
+++ b/src/osd/PGBackend.h
@@ -171,25 +171,7 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
      virtual void add_local_next_event(const pg_log_entry_t& e) = 0;
      virtual const std::map<pg_shard_t, pg_missing_t> &get_shard_missing()
        const = 0;
-     virtual const pg_missing_const_i * maybe_get_shard_missing(
-       pg_shard_t peer) const {
-       if (peer == primary_shard()) {
-	 return &get_local_missing();
-       } else {
-	 std::map<pg_shard_t, pg_missing_t>::const_iterator i =
-	   get_shard_missing().find(peer);
-	 if (i == get_shard_missing().end()) {
-	   return nullptr;
-	 } else {
-	   return &(i->second);
-	 }
-       }
-     }
-     virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const {
-       auto m = maybe_get_shard_missing(peer);
-       ceph_assert(m);
-       return *m;
-     }
+     virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
 
      virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
      virtual const pg_info_t &get_shard_info(pg_shard_t peer) const {
@@ -305,6 +287,7 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
      virtual void pg_add_num_bytes(int64_t num_bytes) = 0;
      virtual void pg_sub_num_bytes(int64_t num_bytes) = 0;
      virtual bool maybe_preempt_replica_scrub(const hobject_t& oid) = 0;
+     virtual struct ECListener *get_eclistener() = 0;
      virtual ~Listener() {}
    };
    Listener *parent;
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 4ecbe58a1f38..f3d03df1d368 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -15799,6 +15799,11 @@ bool PrimaryLogPG::maybe_preempt_replica_scrub(const hobject_t& oid)
   return m_scrubber->write_blocked_by_scrub(oid);
 }
 
+struct ECListener *PrimaryLogPG::get_eclistener()
+{
+  return this;
+}
+
 void intrusive_ptr_add_ref(PrimaryLogPG *pg) { pg->get("intptr"); }
 void intrusive_ptr_release(PrimaryLogPG *pg) { pg->put("intptr"); }
 
diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 6ed29927463d..420871b1990e 100644
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -29,6 +29,7 @@
 #include "common/sharedptr_registry.hpp"
 #include "common/shared_cache.hpp"
 #include "ReplicatedBackend.h"
+#include "ECBackend.h"
 #include "PGTransaction.h"
 #include "cls/cas/cls_cas_ops.h"
 
@@ -55,7 +56,9 @@ void put_with_id(PrimaryLogPG *pg, uint64_t id);
 
 struct inconsistent_snapset_wrapper;
 
-class PrimaryLogPG : public PG, public PGBackend::Listener {
+class PrimaryLogPG : public PG,
+		     public PGBackend::Listener,
+		     public ECListener {
   friend class OSD;
   friend class Watch;
   friend class PrimaryLogScrub;
@@ -391,11 +394,24 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   const std::map<pg_shard_t, pg_missing_t> &get_shard_missing() const override {
     return recovery_state.get_peer_missing();
   }
-  using PGBackend::Listener::get_shard_missing;
+  const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const override {
+    auto m = maybe_get_shard_missing(peer);
+    ceph_assert(m);
+    return *m;
+  }
   const std::map<pg_shard_t, pg_info_t> &get_shard_info() const override {
     return recovery_state.get_peer_info();
   }
-  using PGBackend::Listener::get_shard_info;
+  const pg_info_t &get_shard_info(pg_shard_t peer) const override {
+    if (peer == primary_shard()) {
+      return get_info();
+    } else {
+      std::map<pg_shard_t, pg_info_t>::const_iterator i =
+        get_shard_info().find(peer);
+      ceph_assert(i != get_shard_info().end());
+      return i->second;
+    }
+  }
   const pg_missing_tracker_t &get_local_missing() const override {
     return recovery_state.get_pg_log().get_missing();
   }
@@ -1892,6 +1908,21 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   void on_shutdown() override;
   bool check_failsafe_full() override;
   bool maybe_preempt_replica_scrub(const hobject_t& oid) override;
+  struct ECListener *get_eclistener() override;
+  const pg_missing_const_i * maybe_get_shard_missing(
+    pg_shard_t peer) const {
+    if (peer == primary_shard()) {
+      return &get_local_missing();
+    } else {
+      std::map<pg_shard_t, pg_missing_t>::const_iterator i =
+        get_shard_missing().find(peer);
+      if (i == get_shard_missing().end()) {
+        return nullptr;
+      } else {
+        return &(i->second);
+      }
+    }
+  }
   int rep_repair_primary_object(const hobject_t& soid, OpContext *ctx);
 
   // attr cache handling

From 99ca0448cbb984772e34c137ae44915ce4bce2ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 15 Sep 2023 15:15:01 +0200
Subject: [PATCH 1310/2492] osd: lay down the ECListener for
 ECBackend::RMWPipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc |  2 +-
 src/osd/ECBackend.h  | 21 +++++++++++++++++----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index e597129626f5..ec8fdc0637a2 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -225,7 +225,7 @@ ECBackend::ECBackend(
   uint64_t stripe_width)
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
-    rmw_pipeline(cct, ec_impl, this->sinfo, get_parent(), *this),
+    rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
   ceph_assert((ec_impl->get_data_chunk_count() *
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 238895d3e5b3..2a351113f3ff 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -79,6 +79,20 @@ struct RecoveryMessages;
       std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
 
     virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
+
+    // RMWPipeline
+    virtual const pg_pool_t &get_pool() const = 0;
+    virtual const std::set<pg_shard_t> &get_acting_recovery_backfill_shards() const = 0;
+    virtual bool should_send_op(
+      pg_shard_t peer,
+      const hobject_t &hoid) = 0;
+    virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
+    virtual spg_t primary_spg_t() const = 0;
+    virtual const PGLog &get_log() const = 0;
+    virtual DoutPrefixProvider *get_dpp() = 0;
+    virtual void apply_stats(
+       const hobject_t &soid,
+       const object_stat_sum_t &delta_stats) = 0;
   };
 class ECBackend : public PGBackend {
 public:
@@ -678,12 +692,11 @@ class ECBackend : public PGBackend {
     void call_write_ordered(std::function<void(void)> &&cb);
 
     CephContext* cct;
-    PGBackend::Listener *get_parent() const { return parent; }
+    ECListener *get_parent() const { return parent; }
     const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
     epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
     const pg_info_t &get_info() { return get_parent()->get_info(); }
 
-    // TODO: this will is going to be the RMWPipeline::Listener
     template <typename Func>
     void objects_read_async_no_cache(
       const std::map<hobject_t,extent_set> &to_read,
@@ -715,7 +728,7 @@ class ECBackend : public PGBackend {
 
     ceph::ErasureCodeInterfaceRef ec_impl;
     const ECUtil::stripe_info_t& sinfo;
-    PGBackend::Listener* parent;
+    ECListener* parent;
 
     // TODO: lay an interface down here
     ECBackend& ec_backend;
@@ -723,7 +736,7 @@ class ECBackend : public PGBackend {
     RMWPipeline(CephContext* cct,
                 ceph::ErasureCodeInterfaceRef ec_impl,
                 const ECUtil::stripe_info_t& sinfo,
-                PGBackend::Listener* parent,
+                ECListener* parent,
                 ECBackend& ec_backend)
       : cct(cct),
         ec_impl(std::move(ec_impl)),

From cb7437a700bd550ef6ea0ce0c17bf804333b819f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 14:48:08 +0200
Subject: [PATCH 1311/2492] osd: dissect crimson-shareable parts of ECBackend
 into ECCommon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc |  60 +++---
 src/osd/ECBackend.h  | 447 ++++++++++++++++++++++---------------------
 2 files changed, 258 insertions(+), 249 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index ec8fdc0637a2..a19e95cba9d1 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -58,7 +58,7 @@ static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
 static ostream& _prefix(std::ostream *_dout, ECBackend::RMWPipeline *rmw_pipeline) {
   return rmw_pipeline->get_parent()->gen_dbg_prefix(*_dout);
 }
-static ostream& _prefix(std::ostream *_dout, ECBackend::ReadPipeline *read_pipeline) {
+static ostream& _prefix(std::ostream *_dout, ECCommon::ReadPipeline *read_pipeline) {
   return read_pipeline->get_parent()->gen_dbg_prefix(*_dout);
 }
 
@@ -112,7 +112,7 @@ static ostream &operator<<(
 	     << rhs.get<1>() << ", " << rhs.get<2>() << ")";
 }
 
-ostream &operator<<(ostream &lhs, const ECBackend::read_request_t &rhs)
+ostream &operator<<(ostream &lhs, const ECCommon::read_request_t &rhs)
 {
   return lhs << "read_request_t(to_read=[" << rhs.to_read << "]"
 	     << ", need=" << rhs.need
@@ -120,7 +120,7 @@ ostream &operator<<(ostream &lhs, const ECBackend::read_request_t &rhs)
 	     << ")";
 }
 
-ostream &operator<<(ostream &lhs, const ECBackend::read_result_t &rhs)
+ostream &operator<<(ostream &lhs, const ECCommon::read_result_t &rhs)
 {
   lhs << "read_result_t(r=" << rhs.r
       << ", errors=" << rhs.errors;
@@ -132,7 +132,7 @@ ostream &operator<<(ostream &lhs, const ECBackend::read_result_t &rhs)
   return lhs << ", returned=" << rhs.returned << ")";
 }
 
-ostream &operator<<(ostream &lhs, const ECBackend::ReadOp &rhs)
+ostream &operator<<(ostream &lhs, const ECCommon::ReadOp &rhs)
 {
   lhs << "ReadOp(tid=" << rhs.tid;
   if (rhs.op && rhs.op->get_req()) {
@@ -147,7 +147,7 @@ ostream &operator<<(ostream &lhs, const ECBackend::ReadOp &rhs)
 	     << ", in_progress=" << rhs.in_progress << ")";
 }
 
-void ECBackend::ReadOp::dump(Formatter *f) const
+void ECCommon::ReadOp::dump(Formatter *f) const
 {
   f->dump_unsigned("tid", tid);
   if (op && op->get_req()) {
@@ -237,7 +237,7 @@ PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
   return new ECRecoveryHandle;
 }
 
-void ECBackend::_failed_push(const hobject_t &hoid, ECBackend::read_result_t &res)
+void ECBackend::_failed_push(const hobject_t &hoid, ECCommon::read_result_t &res)
 {
   dout(10) << __func__ << ": Read error " << hoid << " r="
 	   << res.r << " errors=" << res.errors << dendl;
@@ -256,7 +256,7 @@ void ECBackend::_failed_push(const hobject_t &hoid, ECBackend::read_result_t &re
 
 struct RecoveryMessages {
   map<hobject_t,
-      ECBackend::read_request_t> recovery_reads;
+      ECCommon::read_request_t> recovery_reads;
   map<hobject_t, set<int>> want_to_read;
 
   void recovery_read(
@@ -272,7 +272,7 @@ struct RecoveryMessages {
     recovery_reads.insert(
       make_pair(
 	hoid,
-	ECBackend::read_request_t(
+	ECCommon::read_request_t(
 	  to_read,
 	  need,
 	  attrs)));
@@ -515,13 +515,13 @@ struct SendPushReplies : public Context {
   }
 };
 
-struct RecoveryReadCompleter : ECBackend::ReadCompleter {
+struct RecoveryReadCompleter : ECCommon::ReadCompleter {
   RecoveryReadCompleter(ECBackend& backend)
     : backend(backend) {}
 
   void finish_single_request(
     const hobject_t &hoid,
-    ECBackend::read_result_t &res,
+    ECCommon::read_result_t &res,
     list<boost::tuple<uint64_t, uint64_t, uint32_t> >) override
   {
     if (!(res.r == 0 && res.errors.empty())) {
@@ -1347,7 +1347,7 @@ void ECBackend::handle_sub_read_reply(
   }
 }
 
-void ECBackend::ReadPipeline::complete_read_op(ReadOp &rop)
+void ECCommon::ReadPipeline::complete_read_op(ReadOp &rop)
 {
   map<hobject_t, read_request_t>::iterator reqiter =
     rop.to_read.begin();
@@ -1374,9 +1374,9 @@ void ECBackend::ReadPipeline::complete_read_op(ReadOp &rop)
 }
 
 struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
-  ECBackend::ReadPipeline& read_pipeline;
+  ECCommon::ReadPipeline& read_pipeline;
   ceph_tid_t tid;
-  FinishReadOp(ECBackend::ReadPipeline& read_pipeline, ceph_tid_t tid)
+  FinishReadOp(ECCommon::ReadPipeline& read_pipeline, ceph_tid_t tid)
     : read_pipeline(read_pipeline), tid(tid) {}
   void finish(ThreadPool::TPHandle&) override {
     auto ropiter = read_pipeline.tid_to_read_map.find(tid);
@@ -1386,7 +1386,7 @@ struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
 };
 
 template <class F>
-void ECBackend::ReadPipeline::filter_read_op(
+void ECCommon::ReadPipeline::filter_read_op(
   const OSDMapRef& osdmap,
   ReadOp &op,
   F&& on_erase)
@@ -1462,7 +1462,7 @@ void ECBackend::ReadPipeline::filter_read_op(
 }
 
 template <class F>
-void ECBackend::ReadPipeline::check_recovery_sources(
+void ECCommon::ReadPipeline::check_recovery_sources(
   const OSDMapRef& osdmap,
   F&& on_erase)
 {
@@ -1494,7 +1494,7 @@ void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
   });
 }
 
-void ECBackend::ReadPipeline::on_change()
+void ECCommon::ReadPipeline::on_change()
 {
   for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
        i != tid_to_read_map.end();
@@ -1663,7 +1663,7 @@ void ECBackend::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb)
   }
 }
 
-void ECBackend::ReadPipeline::get_all_avail_shards(
+void ECCommon::ReadPipeline::get_all_avail_shards(
   const hobject_t &hoid,
   const set<pg_shard_t> &error_shards,
   set<int> &have,
@@ -1767,7 +1767,7 @@ int ECBackend::get_min_avail_to_read_shards(
   return 0;
 }
 
-int ECBackend::ReadPipeline::get_remaining_shards(
+int ECCommon::ReadPipeline::get_remaining_shards(
   const hobject_t &hoid,
   const set<int> &avail,
   const set<int> &want,
@@ -1813,14 +1813,14 @@ int ECBackend::ReadPipeline::get_remaining_shards(
   return 0;
 }
 
-void ECBackend::ReadPipeline::start_read_op(
+void ECCommon::ReadPipeline::start_read_op(
   int priority,
   map<hobject_t, set<int>> &want_to_read,
   map<hobject_t, read_request_t> &to_read,
   OpRequestRef _op,
   bool do_redundant_reads,
   bool for_recovery,
-  std::unique_ptr<ECBackend::ReadCompleter> on_complete)
+  std::unique_ptr<ECCommon::ReadCompleter> on_complete)
 {
   ceph_tid_t tid = get_parent()->get_tid();
   ceph_assert(!tid_to_read_map.count(tid));
@@ -1843,7 +1843,7 @@ void ECBackend::ReadPipeline::start_read_op(
   do_read_op(op);
 }
 
-void ECBackend::ReadPipeline::do_read_op(ReadOp &op)
+void ECCommon::ReadPipeline::do_read_op(ReadOp &op)
 {
   int priority = op.priority;
   ceph_tid_t tid = op.tid;
@@ -2406,15 +2406,15 @@ void ECBackend::objects_read_async(
 	   on_complete)));
 }
 
-struct ClientReadCompleter : ECBackend::ReadCompleter {
-  ClientReadCompleter(ECBackend::ReadPipeline &read_pipeline,
-                      ECBackend::ClientAsyncReadStatus *status)
+struct ClientReadCompleter : ECCommon::ReadCompleter {
+  ClientReadCompleter(ECCommon::ReadPipeline &read_pipeline,
+                      ECCommon::ClientAsyncReadStatus *status)
     : read_pipeline(read_pipeline),
       status(status) {}
 
   void finish_single_request(
     const hobject_t &hoid,
-    ECBackend::read_result_t &res,
+    ECCommon::read_result_t &res,
     list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) override
   {
     extent_map result;
@@ -2465,8 +2465,8 @@ struct ClientReadCompleter : ECBackend::ReadCompleter {
     // NOP
   }
 
-  ECBackend::ReadPipeline &read_pipeline;
-  ECBackend::ClientAsyncReadStatus *status;
+  ECCommon::ReadPipeline &read_pipeline;
+  ECCommon::ClientAsyncReadStatus *status;
 };
 
 
@@ -2481,7 +2481,7 @@ void ECBackend::objects_read_and_reconstruct(
     *this, reads, fast_read, std::move(func));
 }
 
-void ECBackend::ReadPipeline::objects_read_and_reconstruct(
+void ECCommon::ReadPipeline::objects_read_and_reconstruct(
   ECBackend& ec_backend,
   const map<hobject_t,
     std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
@@ -2532,7 +2532,7 @@ void ECBackend::ReadPipeline::objects_read_and_reconstruct(
 }
 
 
-int ECBackend::ReadPipeline::send_all_remaining_reads(
+int ECCommon::ReadPipeline::send_all_remaining_reads(
   const hobject_t &hoid,
   ReadOp &rop)
 {
@@ -2568,7 +2568,7 @@ int ECBackend::ReadPipeline::send_all_remaining_reads(
   return 0;
 }
 
-void ECBackend::ReadPipeline::kick_reads()
+void ECCommon::ReadPipeline::kick_reads()
 {
   while (in_progress_client_reads.size() &&
          in_progress_client_reads.front().is_complete()) {
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 2a351113f3ff..30b6f017f968 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -94,7 +94,231 @@ struct RecoveryMessages;
        const hobject_t &soid,
        const object_stat_sum_t &delta_stats) = 0;
   };
-class ECBackend : public PGBackend {
+
+struct ECBackend;
+struct ECCommon {
+  struct read_request_t {
+    const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
+    std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
+    bool want_attrs;
+    read_request_t(
+      const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
+      const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
+      bool want_attrs)
+      : to_read(to_read), need(need), want_attrs(want_attrs) {}
+  };
+  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
+  struct ReadOp;
+  /**
+   * Low level async read mechanism
+   *
+   * To avoid duplicating the logic for requesting and waiting for
+   * multiple object shards, there is a common async read mechanism
+   * taking a std::map of hobject_t->read_request_t which defines callbacks
+   * taking read_result_ts as arguments.
+   *
+   * tid_to_read_map gives open read ops.  check_recovery_sources uses
+   * shard_to_read_map and ReadOp::source_to_obj to restart reads
+   * involving down osds.
+   *
+   * The user is responsible for specifying replicas on which to read
+   * and for reassembling the buffer on the other side since client
+   * reads require the original object buffer while recovery only needs
+   * the missing pieces.
+   *
+   * Rather than handling reads on the primary directly, we simply send
+   * ourselves a message.  This avoids a dedicated primary path for that
+   * part.
+   */
+  struct read_result_t {
+    int r;
+    std::map<pg_shard_t, int> errors;
+    std::optional<std::map<std::string, ceph::buffer::list, std::less<>> > attrs;
+    std::list<
+      boost::tuple<
+	uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > > returned;
+    read_result_t() : r(0) {}
+  };
+
+  struct ReadCompleter {
+    virtual void finish_single_request(
+      const hobject_t &hoid,
+      read_result_t &res,
+      std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) = 0;
+
+    virtual void finish(int priority) && = 0;
+
+    virtual ~ReadCompleter() = default;
+  };
+
+  friend struct CallClientContexts;
+  struct ClientAsyncReadStatus {
+    unsigned objects_to_read;
+    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> func;
+    std::map<hobject_t,std::pair<int, extent_map> > results;
+    explicit ClientAsyncReadStatus(
+      unsigned objects_to_read,
+      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func)
+      : objects_to_read(objects_to_read), func(std::move(func)) {}
+    void complete_object(
+      const hobject_t &hoid,
+      int err,
+      extent_map &&buffers) {
+      ceph_assert(objects_to_read);
+      --objects_to_read;
+      ceph_assert(!results.count(hoid));
+      results.emplace(hoid, std::make_pair(err, std::move(buffers)));
+    }
+    bool is_complete() const {
+      return objects_to_read == 0;
+    }
+    void run() {
+      func.release()->complete(std::move(results));
+    }
+  };
+
+  struct ReadOp {
+    int priority;
+    ceph_tid_t tid;
+    OpRequestRef op; // may be null if not on behalf of a client
+    // True if redundant reads are issued, false otherwise,
+    // this is useful to tradeoff some resources (redundant ops) for
+    // low latency read, especially on relatively idle cluster
+    bool do_redundant_reads;
+    // True if reading for recovery which could possibly reading only a subset
+    // of the available shards.
+    bool for_recovery;
+    std::unique_ptr<ReadCompleter> on_complete;
+
+    ZTracer::Trace trace;
+
+    std::map<hobject_t, std::set<int>> want_to_read;
+    std::map<hobject_t, read_request_t> to_read;
+    std::map<hobject_t, read_result_t> complete;
+
+    std::map<hobject_t, std::set<pg_shard_t>> obj_to_source;
+    std::map<pg_shard_t, std::set<hobject_t> > source_to_obj;
+
+    void dump(ceph::Formatter *f) const;
+
+    std::set<pg_shard_t> in_progress;
+
+    ReadOp(
+      int priority,
+      ceph_tid_t tid,
+      bool do_redundant_reads,
+      bool for_recovery,
+      std::unique_ptr<ReadCompleter> _on_complete,
+      OpRequestRef op,
+      std::map<hobject_t, std::set<int>> &&_want_to_read,
+      std::map<hobject_t, read_request_t> &&_to_read)
+      : priority(priority),
+        tid(tid),
+        op(op),
+        do_redundant_reads(do_redundant_reads),
+        for_recovery(for_recovery),
+        on_complete(std::move(_on_complete)),
+        want_to_read(std::move(_want_to_read)),
+	to_read(std::move(_to_read)) {
+      for (auto &&hpair: to_read) {
+	auto &returned = complete[hpair.first].returned;
+	for (auto &&extent: hpair.second.to_read) {
+	  returned.push_back(
+	    boost::make_tuple(
+	      extent.get<0>(),
+	      extent.get<1>(),
+	      std::map<pg_shard_t, ceph::buffer::list>()));
+	}
+      }
+    }
+    ReadOp() = delete;
+    ReadOp(const ReadOp &) = delete; // due to on_complete being unique_ptr
+    ReadOp(ReadOp &&) = default;
+  };
+  struct ReadPipeline {
+    void objects_read_and_reconstruct(
+      ECBackend& ecbackend,
+      const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+      > &reads,
+      bool fast_read,
+      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
+
+    template <class F>
+    void filter_read_op(
+      const OSDMapRef& osdmap,
+      ReadOp &op,
+      F&& on_erase);
+
+    template <class F>
+    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
+
+    void complete_read_op(ReadOp &rop);
+
+    void start_read_op(
+      int priority,
+      std::map<hobject_t, std::set<int>> &want_to_read,
+      std::map<hobject_t, read_request_t> &to_read,
+      OpRequestRef op,
+      bool do_redundant_reads,
+      bool for_recovery,
+      std::unique_ptr<ReadCompleter> on_complete);
+
+    void do_read_op(ReadOp &rop);
+
+    int send_all_remaining_reads(
+      const hobject_t &hoid,
+      ReadOp &rop);
+
+    void on_change();
+
+    void kick_reads();
+
+    std::map<ceph_tid_t, ReadOp> tid_to_read_map;
+    std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
+    std::list<ClientAsyncReadStatus> in_progress_client_reads;
+
+    CephContext* cct;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    // TODO: lay an interface down here
+    ECListener* parent;
+
+    ECListener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+    ReadPipeline(CephContext* cct,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+                ECListener* parent)
+      : cct(cct),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+        parent(parent) {
+    }
+
+    int get_remaining_shards(
+      const hobject_t &hoid,
+      const std::set<int> &avail,
+      const std::set<int> &want,
+      const read_result_t &result,
+      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
+      bool for_recovery);
+
+    void get_all_avail_shards(
+      const hobject_t &hoid,
+      const std::set<pg_shard_t> &error_shards,
+      std::set<int> &have,
+      std::map<shard_id_t, pg_shard_t> &shards,
+      bool for_recovery);
+
+    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+    friend struct FinishReadOp;
+  };
+};
+
+class ECBackend : public PGBackend, public ECCommon {
 public:
   RecoveryHandle *open_recovery_op() override;
 
@@ -205,31 +429,6 @@ class ECBackend : public PGBackend {
     bool fast_read,
     GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
 
-  friend struct CallClientContexts;
-  struct ClientAsyncReadStatus {
-    unsigned objects_to_read;
-    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> func;
-    std::map<hobject_t,std::pair<int, extent_map> > results;
-    explicit ClientAsyncReadStatus(
-      unsigned objects_to_read,
-      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func)
-      : objects_to_read(objects_to_read), func(std::move(func)) {}
-    void complete_object(
-      const hobject_t &hoid,
-      int err,
-      extent_map &&buffers) {
-      ceph_assert(objects_to_read);
-      --objects_to_read;
-      ceph_assert(!results.count(hoid));
-      results.emplace(hoid, std::make_pair(err, std::move(buffers)));
-    }
-    bool is_complete() const {
-      return objects_to_read == 0;
-    }
-    void run() {
-      func.release()->complete(std::move(results));
-    }
-  };
   void objects_read_async(
     const hobject_t &hoid,
     const std::list<std::pair<boost::tuple<uint64_t, uint64_t, uint32_t>,
@@ -246,6 +445,7 @@ class ECBackend : public PGBackend {
 			sinfo.get_stripe_width());
   }
 
+public:
   void get_want_to_read_shards(std::set<int> *want_to_read) const {
     const std::vector<int> &chunk_mapping = ec_impl->get_chunk_mapping();
     for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
@@ -253,6 +453,7 @@ class ECBackend : public PGBackend {
       want_to_read->insert(chunk);
     }
   }
+private:
 
   /**
    * Recovery
@@ -350,199 +551,7 @@ class ECBackend : public PGBackend {
     RecoveryMessages *m);
 
 public:
-  /**
-   * Low level async read mechanism
-   *
-   * To avoid duplicating the logic for requesting and waiting for
-   * multiple object shards, there is a common async read mechanism
-   * taking a std::map of hobject_t->read_request_t which defines callbacks
-   * taking read_result_ts as arguments.
-   *
-   * tid_to_read_map gives open read ops.  check_recovery_sources uses
-   * shard_to_read_map and ReadOp::source_to_obj to restart reads
-   * involving down osds.
-   *
-   * The user is responsible for specifying replicas on which to read
-   * and for reassembling the buffer on the other side since client
-   * reads require the original object buffer while recovery only needs
-   * the missing pieces.
-   *
-   * Rather than handling reads on the primary directly, we simply send
-   * ourselves a message.  This avoids a dedicated primary path for that
-   * part.
-   */
-  struct read_result_t {
-    int r;
-    std::map<pg_shard_t, int> errors;
-    std::optional<std::map<std::string, ceph::buffer::list, std::less<>> > attrs;
-    std::list<
-      boost::tuple<
-	uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > > returned;
-    read_result_t() : r(0) {}
-  };
-  struct read_request_t {
-    const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
-    std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
-    bool want_attrs;
-    read_request_t(
-      const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
-      const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
-      bool want_attrs)
-      : to_read(to_read), need(need), want_attrs(want_attrs) {}
-  };
-  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
-
-  struct ReadCompleter {
-    virtual void finish_single_request(
-      const hobject_t &hoid,
-      ECBackend::read_result_t &res,
-      std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) = 0;
-
-    virtual void finish(int priority) && = 0;
-
-    virtual ~ReadCompleter() = default;
-  };
-
-  struct ReadOp {
-    int priority;
-    ceph_tid_t tid;
-    OpRequestRef op; // may be null if not on behalf of a client
-    // True if redundant reads are issued, false otherwise,
-    // this is useful to tradeoff some resources (redundant ops) for
-    // low latency read, especially on relatively idle cluster
-    bool do_redundant_reads;
-    // True if reading for recovery which could possibly reading only a subset
-    // of the available shards.
-    bool for_recovery;
-    std::unique_ptr<ReadCompleter> on_complete;
-
-    ZTracer::Trace trace;
-
-    std::map<hobject_t, std::set<int>> want_to_read;
-    std::map<hobject_t, read_request_t> to_read;
-    std::map<hobject_t, read_result_t> complete;
-
-    std::map<hobject_t, std::set<pg_shard_t>> obj_to_source;
-    std::map<pg_shard_t, std::set<hobject_t> > source_to_obj;
-
-    void dump(ceph::Formatter *f) const;
-
-    std::set<pg_shard_t> in_progress;
-
-    ReadOp(
-      int priority,
-      ceph_tid_t tid,
-      bool do_redundant_reads,
-      bool for_recovery,
-      std::unique_ptr<ReadCompleter> _on_complete,
-      OpRequestRef op,
-      std::map<hobject_t, std::set<int>> &&_want_to_read,
-      std::map<hobject_t, read_request_t> &&_to_read)
-      : priority(priority),
-        tid(tid),
-        op(op),
-        do_redundant_reads(do_redundant_reads),
-        for_recovery(for_recovery),
-        on_complete(std::move(_on_complete)),
-        want_to_read(std::move(_want_to_read)),
-	to_read(std::move(_to_read)) {
-      for (auto &&hpair: to_read) {
-	auto &returned = complete[hpair.first].returned;
-	for (auto &&extent: hpair.second.to_read) {
-	  returned.push_back(
-	    boost::make_tuple(
-	      extent.get<0>(),
-	      extent.get<1>(),
-	      std::map<pg_shard_t, ceph::buffer::list>()));
-	}
-      }
-    }
-    ReadOp() = delete;
-    ReadOp(const ReadOp &) = delete; // due to on_complete being unique_ptr
-    ReadOp(ReadOp &&) = default;
-  };
-
-  struct ReadPipeline {
-    void objects_read_and_reconstruct(
-      ECBackend& ecbackend,
-      const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
-      > &reads,
-      bool fast_read,
-      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
-
-    template <class F>
-    void filter_read_op(
-      const OSDMapRef& osdmap,
-      ReadOp &op,
-      F&& on_erase);
-
-    template <class F>
-    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
-
-    void complete_read_op(ReadOp &rop);
-
-    void start_read_op(
-      int priority,
-      std::map<hobject_t, std::set<int>> &want_to_read,
-      std::map<hobject_t, read_request_t> &to_read,
-      OpRequestRef op,
-      bool do_redundant_reads,
-      bool for_recovery,
-      std::unique_ptr<ReadCompleter> on_complete);
-
-    void do_read_op(ReadOp &rop);
-
-    int send_all_remaining_reads(
-      const hobject_t &hoid,
-      ReadOp &rop);
-
-    void on_change();
-
-    void kick_reads();
-
-    std::map<ceph_tid_t, ReadOp> tid_to_read_map;
-    std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
-    std::list<ClientAsyncReadStatus> in_progress_client_reads;
-
-    CephContext* cct;
-    ceph::ErasureCodeInterfaceRef ec_impl;
-    const ECUtil::stripe_info_t& sinfo;
-    // TODO: lay an interface down here
-    ECListener* parent;
-
-    ECListener *get_parent() const { return parent; }
-    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
-    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
-    const pg_info_t &get_info() { return get_parent()->get_info(); }
-
-    ReadPipeline(CephContext* cct,
-                ceph::ErasureCodeInterfaceRef ec_impl,
-                const ECUtil::stripe_info_t& sinfo,
-                ECListener* parent)
-      : cct(cct),
-        ec_impl(std::move(ec_impl)),
-        sinfo(sinfo),
-        parent(parent) {
-    }
-
-    int get_remaining_shards(
-      const hobject_t &hoid,
-      const std::set<int> &avail,
-      const std::set<int> &want,
-      const read_result_t &result,
-      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
-      bool for_recovery);
-
-    void get_all_avail_shards(
-      const hobject_t &hoid,
-      const std::set<pg_shard_t> &error_shards,
-      std::set<int> &have,
-      std::map<shard_id_t, pg_shard_t> &shards,
-      bool for_recovery);
-
-    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
-    friend struct FinishReadOp;
-  } read_pipeline;
+  struct ReadPipeline read_pipeline;
 
 
   /**

From 98f9a254ec5133d60f7da2d6144de667931cbe7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 14:51:53 +0200
Subject: [PATCH 1312/2492] osd: ReadPipeline doesn't depend on classical
 ECBackend at all
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 23 ++++++++++++++++-------
 src/osd/ECBackend.h  | 33 ++++++++++++---------------------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index a19e95cba9d1..6750a8a538d8 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -634,7 +634,7 @@ void ECBackend::continue_recovery_op(
       }
 
       map<pg_shard_t, vector<pair<int, int>>> to_read;
-      int r = get_min_avail_to_read_shards(
+      int r = read_pipeline.get_min_avail_to_read_shards(
 	op.hoid, want, true, false, &to_read);
       if (r != 0) {
 	// we must have lost a recovery source
@@ -1728,7 +1728,7 @@ void ECCommon::ReadPipeline::get_all_avail_shards(
   }
 }
 
-int ECBackend::get_min_avail_to_read_shards(
+int ECCommon::ReadPipeline::get_min_avail_to_read_shards(
   const hobject_t &hoid,
   const set<int> &want,
   bool for_recovery,
@@ -1742,7 +1742,7 @@ int ECBackend::get_min_avail_to_read_shards(
   map<shard_id_t, pg_shard_t> shards;
   set<pg_shard_t> error_shards;
 
-  read_pipeline.get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
+  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
 
   map<int, vector<pair<int, int>>> need;
   int r = ec_impl->minimum_to_decode(want, have, &need);
@@ -2478,11 +2478,20 @@ void ECBackend::objects_read_and_reconstruct(
   GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
 {
   return read_pipeline.objects_read_and_reconstruct(
-    *this, reads, fast_read, std::move(func));
+    reads, fast_read, std::move(func));
+}
+
+void ECCommon::ReadPipeline::get_want_to_read_shards(
+  std::set<int> *want_to_read) const
+{
+  const std::vector<int> &chunk_mapping = ec_impl->get_chunk_mapping();
+  for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
+    int chunk = (int)chunk_mapping.size() > i ? chunk_mapping[i] : i;
+    want_to_read->insert(chunk);
+  }
 }
 
 void ECCommon::ReadPipeline::objects_read_and_reconstruct(
-  ECBackend& ec_backend,
   const map<hobject_t,
     std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
   > &reads,
@@ -2498,12 +2507,12 @@ void ECCommon::ReadPipeline::objects_read_and_reconstruct(
 
   map<hobject_t, set<int>> obj_want_to_read;
   set<int> want_to_read;
-  ec_backend.get_want_to_read_shards(&want_to_read);
+  get_want_to_read_shards(&want_to_read);
     
   map<hobject_t, read_request_t> for_read_op;
   for (auto &&to_read: reads) {
     map<pg_shard_t, vector<pair<int, int>>> shards;
-    int r = ec_backend.get_min_avail_to_read_shards(
+    int r = get_min_avail_to_read_shards(
       to_read.first,
       want_to_read,
       false,
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 30b6f017f968..ae169fd8ff34 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -90,12 +90,12 @@ struct RecoveryMessages;
     virtual spg_t primary_spg_t() const = 0;
     virtual const PGLog &get_log() const = 0;
     virtual DoutPrefixProvider *get_dpp() = 0;
+    // XXX
     virtual void apply_stats(
        const hobject_t &soid,
        const object_stat_sum_t &delta_stats) = 0;
   };
 
-struct ECBackend;
 struct ECCommon {
   struct read_request_t {
     const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
@@ -237,7 +237,6 @@ struct ECCommon {
   };
   struct ReadPipeline {
     void objects_read_and_reconstruct(
-      ECBackend& ecbackend,
       const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
       > &reads,
       bool fast_read,
@@ -315,6 +314,17 @@ struct ECCommon {
 
     friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
     friend struct FinishReadOp;
+
+    void get_want_to_read_shards(std::set<int> *want_to_read) const;
+
+    /// Returns to_read replicas sufficient to reconstruct want
+    int get_min_avail_to_read_shards(
+      const hobject_t &hoid,     ///< [in] object
+      const std::set<int> &want,      ///< [in] desired shards
+      bool for_recovery,         ///< [in] true if we may use non-acting replicas
+      bool do_redundant_reads,   ///< [in] true if we want to issue redundant reads to reduce latency
+      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
+      ); ///< @return error code, 0 on success
   };
 };
 
@@ -445,16 +455,6 @@ class ECBackend : public PGBackend, public ECCommon {
 			sinfo.get_stripe_width());
   }
 
-public:
-  void get_want_to_read_shards(std::set<int> *want_to_read) const {
-    const std::vector<int> &chunk_mapping = ec_impl->get_chunk_mapping();
-    for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
-      int chunk = (int)chunk_mapping.size() > i ? chunk_mapping[i] : i;
-      want_to_read->insert(chunk);
-    }
-  }
-private:
-
   /**
    * Recovery
    *
@@ -831,15 +831,6 @@ class ECBackend : public PGBackend, public ECCommon {
     ceph::ErasureCodeInterfaceRef ec_impl,
     uint64_t stripe_width);
 
-  /// Returns to_read replicas sufficient to reconstruct want
-  int get_min_avail_to_read_shards(
-    const hobject_t &hoid,     ///< [in] object
-    const std::set<int> &want,      ///< [in] desired shards
-    bool for_recovery,         ///< [in] true if we may use non-acting replicas
-    bool do_redundant_reads,   ///< [in] true if we want to issue redundant reads to reduce latency
-    std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
-    ); ///< @return error code, 0 on success
-
   int objects_get_attrs(
     const hobject_t &hoid,
     std::map<std::string, ceph::buffer::list, std::less<>> *out) override;

From 4d6970e7006ed21143d9e0e0e8d02174b46baeda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 15:11:12 +0200
Subject: [PATCH 1313/2492] osd: decouple RMWPipeline from ECBackend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.h | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index ae169fd8ff34..a2769e6cb6dc 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -97,6 +97,21 @@ struct RecoveryMessages;
   };
 
 struct ECCommon {
+  virtual ~ECCommon() = default;
+
+  virtual void handle_sub_write(
+    pg_shard_t from,
+    OpRequestRef msg,
+    ECSubWrite &op,
+    const ZTracer::Trace &trace
+    ) = 0;
+
+  virtual void objects_read_and_reconstruct(
+    const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+    > &reads,
+    bool fast_read,
+    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func) = 0;
+
   struct read_request_t {
     const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
     std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
@@ -363,7 +378,7 @@ class ECBackend : public PGBackend, public ECCommon {
     OpRequestRef msg,
     ECSubWrite &op,
     const ZTracer::Trace &trace
-    );
+    ) override;
   void handle_sub_read(
     pg_shard_t from,
     const ECSubRead &op,
@@ -437,7 +452,7 @@ class ECBackend : public PGBackend, public ECCommon {
     const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
     > &reads,
     bool fast_read,
-    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
+    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func) override;
 
   void objects_read_async(
     const hobject_t &hoid,
@@ -738,15 +753,13 @@ class ECBackend : public PGBackend, public ECCommon {
     ceph::ErasureCodeInterfaceRef ec_impl;
     const ECUtil::stripe_info_t& sinfo;
     ECListener* parent;
-
-    // TODO: lay an interface down here
-    ECBackend& ec_backend;
+    ECCommon& ec_backend;
 
     RMWPipeline(CephContext* cct,
                 ceph::ErasureCodeInterfaceRef ec_impl,
                 const ECUtil::stripe_info_t& sinfo,
                 ECListener* parent,
-                ECBackend& ec_backend)
+                ECCommon& ec_backend)
       : cct(cct),
         ec_impl(std::move(ec_impl)),
         sinfo(sinfo),

From 882bbb930432cb61e33464a378ef0b00b1078aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 15:23:31 +0200
Subject: [PATCH 1314/2492] osd: move ReadPipeline from ECBackend to shareable
 ECCommon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc |  26 +--
 src/osd/ECBackend.h  | 400 +++++++++++++++++++++----------------------
 2 files changed, 213 insertions(+), 213 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 6750a8a538d8..4e8f0b4e019a 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -55,7 +55,7 @@ using ceph::Formatter;
 static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
   return pgb->get_parent()->gen_dbg_prefix(*_dout);
 }
-static ostream& _prefix(std::ostream *_dout, ECBackend::RMWPipeline *rmw_pipeline) {
+static ostream& _prefix(std::ostream *_dout, ECCommon::RMWPipeline *rmw_pipeline) {
   return rmw_pipeline->get_parent()->gen_dbg_prefix(*_dout);
 }
 static ostream& _prefix(std::ostream *_dout, ECCommon::ReadPipeline *read_pipeline) {
@@ -66,11 +66,11 @@ struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
   list<ECBackend::RecoveryOp> ops;
 };
 
-ostream &operator<<(ostream &lhs, const ECBackend::RMWPipeline::pipeline_state_t &rhs) {
+ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::pipeline_state_t &rhs) {
   switch (rhs.pipeline_state) {
-  case ECBackend::RMWPipeline::pipeline_state_t::CACHE_VALID:
+  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_VALID:
     return lhs << "CACHE_VALID";
-  case ECBackend::RMWPipeline::pipeline_state_t::CACHE_INVALID:
+  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_INVALID:
     return lhs << "CACHE_INVALID";
   default:
     ceph_abort_msg("invalid pipeline state");
@@ -161,7 +161,7 @@ void ECCommon::ReadOp::dump(Formatter *f) const
   f->dump_stream("in_progress") << in_progress;
 }
 
-ostream &operator<<(ostream &lhs, const ECBackend::RMWPipeline::Op &rhs)
+ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::Op &rhs)
 {
   lhs << "Op(" << rhs.hoid
       << " v=" << rhs.version
@@ -1506,7 +1506,7 @@ void ECCommon::ReadPipeline::on_change()
   in_progress_client_reads.clear();
 }
 
-void ECBackend::RMWPipeline::on_change()
+void ECCommon::RMWPipeline::on_change()
 {
   dout(10) << __func__ << dendl;
 
@@ -1556,7 +1556,7 @@ void ECBackend::dump_recovery_info(Formatter *f) const
   f->close_section();
 }
 
-struct ECClassicalOp : ECBackend::RMWPipeline::Op {
+struct ECClassicalOp : ECCommon::RMWPipeline::Op {
   PGTransactionUPtr t;
 
   void generate_transactions(
@@ -1652,7 +1652,7 @@ void ECBackend::submit_transaction(
   rmw_pipeline.start_rmw(std::move(op));
 }
 
-void ECBackend::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
+void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
   if (!waiting_state.empty()) {
     waiting_state.back().on_write.emplace_back(std::move(cb));
   } else if (!waiting_reads.empty()) {
@@ -1978,7 +1978,7 @@ ECUtil::HashInfoRef ECBackend::get_hash_info(
   return ref;
 }
 
-void ECBackend::RMWPipeline::start_rmw(OpRef op)
+void ECCommon::RMWPipeline::start_rmw(OpRef op)
 {
   ceph_assert(op);
   dout(10) << __func__ << ": " << *op << dendl;
@@ -1989,7 +1989,7 @@ void ECBackend::RMWPipeline::start_rmw(OpRef op)
   check_ops();
 }
 
-bool ECBackend::RMWPipeline::try_state_to_reads()
+bool ECCommon::RMWPipeline::try_state_to_reads()
 {
   if (waiting_state.empty())
     return false;
@@ -2063,7 +2063,7 @@ bool ECBackend::RMWPipeline::try_state_to_reads()
   return true;
 }
 
-bool ECBackend::RMWPipeline::try_reads_to_commit()
+bool ECCommon::RMWPipeline::try_reads_to_commit()
 {
   if (waiting_reads.empty())
     return false;
@@ -2221,7 +2221,7 @@ bool ECBackend::RMWPipeline::try_reads_to_commit()
   return true;
 }
 
-bool ECBackend::RMWPipeline::try_finish_rmw()
+bool ECCommon::RMWPipeline::try_finish_rmw()
 {
   if (waiting_commit.empty())
     return false;
@@ -2270,7 +2270,7 @@ bool ECBackend::RMWPipeline::try_finish_rmw()
   return true;
 }
 
-void ECBackend::RMWPipeline::check_ops()
+void ECCommon::RMWPipeline::check_ops()
 {
   while (try_state_to_reads() ||
 	 try_reads_to_commit() ||
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index a2769e6cb6dc..5fbe8ccbfc67 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -341,6 +341,205 @@ struct ECCommon {
       std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
       ); ///< @return error code, 0 on success
   };
+
+  /**
+   * Client writes
+   *
+   * ECTransaction is responsible for generating a transaction for
+   * each shard to which we need to send the write.  As required
+   * by the PGBackend interface, the ECBackend write mechanism
+   * passes trim information with the write and last_complete back
+   * with the reply.
+   *
+   * As with client reads, there is a possibility of out-of-order
+   * completions. Thus, callbacks and completion are called in order
+   * on the writing std::list.
+   */
+
+  struct RMWPipeline {
+    struct Op : boost::intrusive::list_base_hook<> {
+      /// From submit_transaction caller, describes operation
+      hobject_t hoid;
+      object_stat_sum_t delta_stats;
+      eversion_t version;
+      eversion_t trim_to;
+      std::optional<pg_hit_set_history_t> updated_hit_set_history;
+      std::vector<pg_log_entry_t> log_entries;
+      ceph_tid_t tid;
+      osd_reqid_t reqid;
+      ZTracer::Trace trace;
+
+      eversion_t roll_forward_to; /// Soon to be generated internally
+
+      /// Ancillary also provided from submit_transaction caller
+      std::map<hobject_t, ObjectContextRef> obc_map;
+
+      /// see call_write_ordered
+      std::list<std::function<void(void)> > on_write;
+
+      /// Generated internally
+      std::set<hobject_t> temp_added;
+      std::set<hobject_t> temp_cleared;
+
+      ECTransaction::WritePlan plan;
+      bool requires_rmw() const { return !plan.to_read.empty(); }
+      bool invalidates_cache() const { return plan.invalidates_cache; }
+
+      // must be true if requires_rmw(), must be false if invalidates_cache()
+      bool using_cache = true;
+
+      /// In progress read state;
+      std::map<hobject_t,extent_set> pending_read; // subset already being read
+      std::map<hobject_t,extent_set> remote_read;  // subset we must read
+      std::map<hobject_t,extent_map> remote_read_result;
+      bool read_in_progress() const {
+        return !remote_read.empty() && remote_read_result.empty();
+      }
+
+      /// In progress write state.
+      std::set<pg_shard_t> pending_commit;
+      // we need pending_apply for pre-mimic peers so that we don't issue a
+      // read on a remote shard before it has applied a previous write.  We can
+      // remove this after nautilus.
+      std::set<pg_shard_t> pending_apply;
+      bool write_in_progress() const {
+        return !pending_commit.empty() || !pending_apply.empty();
+      }
+
+      /// optional, may be null, for tracking purposes
+      OpRequestRef client_op;
+
+      /// pin for cache
+      ExtentCache::write_pin pin;
+
+      /// Callbacks
+      Context *on_all_commit = nullptr;
+      virtual ~Op() {
+        delete on_all_commit;
+      }
+
+      virtual void generate_transactions(
+        ceph::ErasureCodeInterfaceRef &ecimpl,
+        pg_t pgid,
+        const ECUtil::stripe_info_t &sinfo,
+        std::map<hobject_t,extent_map> *written,
+        std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+        DoutPrefixProvider *dpp,
+        const ceph_release_t require_osd_release = ceph_release_t::unknown) = 0;
+    };
+    using OpRef = std::unique_ptr<Op>;
+    using op_list = boost::intrusive::list<Op>;
+    friend ostream &operator<<(ostream &lhs, const Op &rhs);
+
+    ExtentCache cache;
+    std::map<ceph_tid_t, OpRef> tid_to_op_map; /// Owns Op structure
+    /**
+     * We model the possible rmw states as a std::set of waitlists.
+     * All writes at this time complete in order, so a write blocked
+     * at waiting_state blocks all writes behind it as well (same for
+     * other states).
+     *
+     * Future work: We can break this up into a per-object pipeline
+     * (almost).  First, provide an ordering token to submit_transaction
+     * and require that all operations within a single transaction take
+     * place on a subset of hobject_t space partitioned by that token
+     * (the hashid seem about right to me -- even works for temp objects
+     * if you recall that a temp object created for object head foo will
+     * only ever be referenced by other transactions on foo and aren't
+     * reused).  Next, factor this part into a class and maintain one per
+     * ordering token.  Next, fixup PrimaryLogPG's repop queue to be
+     * partitioned by ordering token.  Finally, refactor the op pipeline
+     * so that the log entries passed into submit_transaction aren't
+     * versioned.  We can't assign versions to them until we actually
+     * submit the operation.  That's probably going to be the hard part.
+     */
+    class pipeline_state_t {
+      enum {
+        CACHE_VALID = 0,
+        CACHE_INVALID = 1
+      } pipeline_state = CACHE_VALID;
+    public:
+      bool caching_enabled() const {
+        return pipeline_state == CACHE_VALID;
+      }
+      bool cache_invalid() const {
+        return !caching_enabled();
+      }
+      void invalidate() {
+        pipeline_state = CACHE_INVALID;
+      }
+      void clear() {
+        pipeline_state = CACHE_VALID;
+      }
+      friend ostream &operator<<(ostream &lhs, const pipeline_state_t &rhs);
+    } pipeline_state;
+
+    op_list waiting_state;        /// writes waiting on pipe_state
+    op_list waiting_reads;        /// writes waiting on partial stripe reads
+    op_list waiting_commit;       /// writes waiting on initial commit
+    eversion_t completed_to;
+    eversion_t committed_to;
+    void start_rmw(OpRef op);
+    bool try_state_to_reads();
+    bool try_reads_to_commit();
+    bool try_finish_rmw();
+    void check_ops();
+
+    void on_change();
+    void call_write_ordered(std::function<void(void)> &&cb);
+
+    CephContext* cct;
+    ECListener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+    template <typename Func>
+    void objects_read_async_no_cache(
+      const std::map<hobject_t,extent_set> &to_read,
+      Func &&on_complete
+    ) {
+      std::map<hobject_t,std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > > _to_read;
+      for (auto &&hpair: to_read) {
+        auto &l = _to_read[hpair.first];
+        for (auto extent: hpair.second) {
+          l.emplace_back(extent.first, extent.second, 0);
+        }
+      }
+      ec_backend.objects_read_and_reconstruct(
+        _to_read,
+        false,
+        make_gen_lambda_context<
+        std::map<hobject_t,std::pair<int, extent_map> > &&, Func>(
+            std::forward<Func>(on_complete)));
+    }
+    void handle_sub_write(
+      pg_shard_t from,
+      OpRequestRef msg,
+      ECSubWrite &op,
+      const ZTracer::Trace &trace
+    ) {
+      ec_backend.handle_sub_write(from, std::move(msg), op, trace);
+    }
+    // end of iface
+
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    ECListener* parent;
+    ECCommon& ec_backend;
+
+    RMWPipeline(CephContext* cct,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+                ECListener* parent,
+                ECCommon& ec_backend)
+      : cct(cct),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+        parent(parent),
+        ec_backend(ec_backend) {
+    }
+  };
 };
 
 class ECBackend : public PGBackend, public ECCommon {
@@ -567,206 +766,7 @@ class ECBackend : public PGBackend, public ECCommon {
 
 public:
   struct ReadPipeline read_pipeline;
-
-
-  /**
-   * Client writes
-   *
-   * ECTransaction is responsible for generating a transaction for
-   * each shard to which we need to send the write.  As required
-   * by the PGBackend interface, the ECBackend write mechanism
-   * passes trim information with the write and last_complete back
-   * with the reply.
-   *
-   * As with client reads, there is a possibility of out-of-order
-   * completions. Thus, callbacks and completion are called in order
-   * on the writing std::list.
-   */
-
-  struct RMWPipeline {
-    struct Op : boost::intrusive::list_base_hook<> {
-      /// From submit_transaction caller, describes operation
-      hobject_t hoid;
-      object_stat_sum_t delta_stats;
-      eversion_t version;
-      eversion_t trim_to;
-      std::optional<pg_hit_set_history_t> updated_hit_set_history;
-      std::vector<pg_log_entry_t> log_entries;
-      ceph_tid_t tid;
-      osd_reqid_t reqid;
-      ZTracer::Trace trace;
-
-      eversion_t roll_forward_to; /// Soon to be generated internally
-
-      /// Ancillary also provided from submit_transaction caller
-      std::map<hobject_t, ObjectContextRef> obc_map;
-
-      /// see call_write_ordered
-      std::list<std::function<void(void)> > on_write;
-
-      /// Generated internally
-      std::set<hobject_t> temp_added;
-      std::set<hobject_t> temp_cleared;
-
-      ECTransaction::WritePlan plan;
-      bool requires_rmw() const { return !plan.to_read.empty(); }
-      bool invalidates_cache() const { return plan.invalidates_cache; }
-
-      // must be true if requires_rmw(), must be false if invalidates_cache()
-      bool using_cache = true;
-
-      /// In progress read state;
-      std::map<hobject_t,extent_set> pending_read; // subset already being read
-      std::map<hobject_t,extent_set> remote_read;  // subset we must read
-      std::map<hobject_t,extent_map> remote_read_result;
-      bool read_in_progress() const {
-        return !remote_read.empty() && remote_read_result.empty();
-      }
-
-      /// In progress write state.
-      std::set<pg_shard_t> pending_commit;
-      // we need pending_apply for pre-mimic peers so that we don't issue a
-      // read on a remote shard before it has applied a previous write.  We can
-      // remove this after nautilus.
-      std::set<pg_shard_t> pending_apply;
-      bool write_in_progress() const {
-        return !pending_commit.empty() || !pending_apply.empty();
-      }
-
-      /// optional, may be null, for tracking purposes
-      OpRequestRef client_op;
-
-      /// pin for cache
-      ExtentCache::write_pin pin;
-
-      /// Callbacks
-      Context *on_all_commit = nullptr;
-      virtual ~Op() {
-        delete on_all_commit;
-      }
-
-      virtual void generate_transactions(
-        ceph::ErasureCodeInterfaceRef &ecimpl,
-        pg_t pgid,
-        const ECUtil::stripe_info_t &sinfo,
-        std::map<hobject_t,extent_map> *written,
-        std::map<shard_id_t, ObjectStore::Transaction> *transactions,
-        DoutPrefixProvider *dpp,
-        const ceph_release_t require_osd_release = ceph_release_t::unknown) = 0;
-    };
-    using OpRef = std::unique_ptr<Op>;
-    using op_list = boost::intrusive::list<Op>;
-    friend ostream &operator<<(ostream &lhs, const Op &rhs);
-
-    ExtentCache cache;
-    std::map<ceph_tid_t, OpRef> tid_to_op_map; /// Owns Op structure
-    /**
-     * We model the possible rmw states as a std::set of waitlists.
-     * All writes at this time complete in order, so a write blocked
-     * at waiting_state blocks all writes behind it as well (same for
-     * other states).
-     *
-     * Future work: We can break this up into a per-object pipeline
-     * (almost).  First, provide an ordering token to submit_transaction
-     * and require that all operations within a single transaction take
-     * place on a subset of hobject_t space partitioned by that token
-     * (the hashid seem about right to me -- even works for temp objects
-     * if you recall that a temp object created for object head foo will
-     * only ever be referenced by other transactions on foo and aren't
-     * reused).  Next, factor this part into a class and maintain one per
-     * ordering token.  Next, fixup PrimaryLogPG's repop queue to be
-     * partitioned by ordering token.  Finally, refactor the op pipeline
-     * so that the log entries passed into submit_transaction aren't
-     * versioned.  We can't assign versions to them until we actually
-     * submit the operation.  That's probably going to be the hard part.
-     */
-    class pipeline_state_t {
-      enum {
-        CACHE_VALID = 0,
-        CACHE_INVALID = 1
-      } pipeline_state = CACHE_VALID;
-    public:
-      bool caching_enabled() const {
-        return pipeline_state == CACHE_VALID;
-      }
-      bool cache_invalid() const {
-        return !caching_enabled();
-      }
-      void invalidate() {
-        pipeline_state = CACHE_INVALID;
-      }
-      void clear() {
-        pipeline_state = CACHE_VALID;
-      }
-      friend ostream &operator<<(ostream &lhs, const pipeline_state_t &rhs);
-    } pipeline_state;
-
-    op_list waiting_state;        /// writes waiting on pipe_state
-    op_list waiting_reads;        /// writes waiting on partial stripe reads
-    op_list waiting_commit;       /// writes waiting on initial commit
-    eversion_t completed_to;
-    eversion_t committed_to;
-    void start_rmw(OpRef op);
-    bool try_state_to_reads();
-    bool try_reads_to_commit();
-    bool try_finish_rmw();
-    void check_ops();
-
-    void on_change();
-    void call_write_ordered(std::function<void(void)> &&cb);
-
-    CephContext* cct;
-    ECListener *get_parent() const { return parent; }
-    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
-    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
-    const pg_info_t &get_info() { return get_parent()->get_info(); }
-
-    template <typename Func>
-    void objects_read_async_no_cache(
-      const std::map<hobject_t,extent_set> &to_read,
-      Func &&on_complete
-    ) {
-      std::map<hobject_t,std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > > _to_read;
-      for (auto &&hpair: to_read) {
-        auto &l = _to_read[hpair.first];
-        for (auto extent: hpair.second) {
-          l.emplace_back(extent.first, extent.second, 0);
-        }
-      }
-      ec_backend.objects_read_and_reconstruct(
-        _to_read,
-        false,
-        make_gen_lambda_context<
-        std::map<hobject_t,std::pair<int, extent_map> > &&, Func>(
-            std::forward<Func>(on_complete)));
-    }
-    void handle_sub_write(
-      pg_shard_t from,
-      OpRequestRef msg,
-      ECSubWrite &op,
-      const ZTracer::Trace &trace
-    ) {
-      ec_backend.handle_sub_write(from, std::move(msg), op, trace);
-    }
-    // end of iface
-
-    ceph::ErasureCodeInterfaceRef ec_impl;
-    const ECUtil::stripe_info_t& sinfo;
-    ECListener* parent;
-    ECCommon& ec_backend;
-
-    RMWPipeline(CephContext* cct,
-                ceph::ErasureCodeInterfaceRef ec_impl,
-                const ECUtil::stripe_info_t& sinfo,
-                ECListener* parent,
-                ECCommon& ec_backend)
-      : cct(cct),
-        ec_impl(std::move(ec_impl)),
-        sinfo(sinfo),
-        parent(parent),
-        ec_backend(ec_backend) {
-    }
-  } rmw_pipeline;
+  struct RMWPipeline rmw_pipeline;
 
   ceph::ErasureCodeInterfaceRef ec_impl;
 

From 15c5cbbbbafe07a730b91aac15093837027da4e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 16:26:54 +0200
Subject: [PATCH 1315/2492] osd, fixup: reindent ECListener
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.h | 120 +++++++++++++++++++++++---------------------
 1 file changed, 62 insertions(+), 58 deletions(-)

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 5fbe8ccbfc67..92a8c0b2091f 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -33,68 +33,72 @@ struct ECSubReadReply;
 
 struct RecoveryMessages;
 
-  // ECListener -- an interface decoupling the pipelines from
-  // particular implementation of ECBackend (crimson vs cassical).
-  // https://stackoverflow.com/q/7872958
-  struct ECListener {
-    virtual ~ECListener() = default;
-    virtual const OSDMapRef& pgb_get_osdmap() const = 0;
-    virtual epoch_t pgb_get_osdmap_epoch() const = 0;
-    virtual const pg_info_t &get_info() const = 0;
-    /**
-     * Called when a pull on soid cannot be completed due to
-     * down peers
-     */
-    virtual void cancel_pull(
-      const hobject_t &soid) = 0;
-    virtual void schedule_recovery_work(
-      GenContext<ThreadPool::TPHandle&> *c,
-      uint64_t cost) = 0;
-
-    virtual epoch_t get_interval_start_epoch() const = 0;
-    virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
-    virtual const std::set<pg_shard_t> &get_backfill_shards() const = 0;
-    virtual const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_loc_shards()
-      const = 0;
-
-    virtual const std::map<pg_shard_t,
-			   pg_missing_t> &get_shard_missing() const = 0;
-    virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
+// ECListener -- an interface decoupling the pipelines from
+// particular implementation of ECBackend (crimson vs cassical).
+// https://stackoverflow.com/q/7872958
+struct ECListener {
+  virtual ~ECListener() = default;
+  virtual const OSDMapRef& pgb_get_osdmap() const = 0;
+  virtual epoch_t pgb_get_osdmap_epoch() const = 0;
+  virtual const pg_info_t &get_info() const = 0;
+  /**
+   * Called when a pull on soid cannot be completed due to
+   * down peers
+   */
+  // XXX
+  virtual void cancel_pull(
+    const hobject_t &soid) = 0;
+  // XXX
+  virtual void schedule_recovery_work(
+    GenContext<ThreadPool::TPHandle&> *c,
+    uint64_t cost) = 0;
+
+  virtual epoch_t get_interval_start_epoch() const = 0;
+  virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
+  virtual const std::set<pg_shard_t> &get_backfill_shards() const = 0;
+  virtual const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_loc_shards()
+    const = 0;
+
+  virtual const std::map<pg_shard_t,
+			 pg_missing_t> &get_shard_missing() const = 0;
+  virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
 #if 1
-    virtual const pg_missing_const_i * maybe_get_shard_missing(
-      pg_shard_t peer) const = 0;
-    virtual const pg_info_t &get_shard_info(pg_shard_t peer) const = 0;
+  virtual const pg_missing_const_i * maybe_get_shard_missing(
+    pg_shard_t peer) const = 0;
+  virtual const pg_info_t &get_shard_info(pg_shard_t peer) const = 0;
 #endif
-    virtual ceph_tid_t get_tid() = 0;
-    virtual pg_shard_t whoami_shard() const = 0;
+  virtual ceph_tid_t get_tid() = 0;
+  virtual pg_shard_t whoami_shard() const = 0;
 #if 0
-    int whoami() const {
-      return whoami_shard().osd;
-    }
-    spg_t whoami_spg_t() const {
-      return get_info().pgid;
-    }
+  int whoami() const {
+    return whoami_shard().osd;
+  }
+  spg_t whoami_spg_t() const {
+    return get_info().pgid;
+  }
 #endif
-    virtual void send_message_osd_cluster(
-      std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
-
-    virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
-
-    // RMWPipeline
-    virtual const pg_pool_t &get_pool() const = 0;
-    virtual const std::set<pg_shard_t> &get_acting_recovery_backfill_shards() const = 0;
-    virtual bool should_send_op(
-      pg_shard_t peer,
-      const hobject_t &hoid) = 0;
-    virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
-    virtual spg_t primary_spg_t() const = 0;
-    virtual const PGLog &get_log() const = 0;
-    virtual DoutPrefixProvider *get_dpp() = 0;
-    // XXX
-    virtual void apply_stats(
-       const hobject_t &soid,
-       const object_stat_sum_t &delta_stats) = 0;
-  };
+  // XXX
+  virtual void send_message_osd_cluster(
+    std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
+
+  virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
+
+  // RMWPipeline
+  virtual const pg_pool_t &get_pool() const = 0;
+  virtual const std::set<pg_shard_t> &get_acting_recovery_backfill_shards() const = 0;
+  // XXX
+  virtual bool should_send_op(
+    pg_shard_t peer,
+    const hobject_t &hoid) = 0;
+  virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
+  virtual spg_t primary_spg_t() const = 0;
+  virtual const PGLog &get_log() const = 0;
+  virtual DoutPrefixProvider *get_dpp() = 0;
+  // XXX
+  virtual void apply_stats(
+     const hobject_t &soid,
+     const object_stat_sum_t &delta_stats) = 0;
+};
 
 struct ECCommon {
   virtual ~ECCommon() = default;

From bf29a11f0ad2e64bcf493758b851ce5e2a8d18b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Sep 2023 16:36:50 +0200
Subject: [PATCH 1316/2492] osd: dissect ECCommon and ECListener into separated
 header
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.h | 514 +----------------------------------------
 src/osd/ECCommon.h  | 545 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 546 insertions(+), 513 deletions(-)
 create mode 100644 src/osd/ECCommon.h

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 92a8c0b2091f..df8cc6dc54aa 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -18,6 +18,7 @@
 #include <boost/intrusive/set.hpp>
 #include <boost/intrusive/list.hpp>
 
+#include "ECCommon.h"
 #include "OSD.h"
 #include "PGBackend.h"
 #include "erasure-code/ErasureCodeInterface.h"
@@ -33,519 +34,6 @@ struct ECSubReadReply;
 
 struct RecoveryMessages;
 
-// ECListener -- an interface decoupling the pipelines from
-// particular implementation of ECBackend (crimson vs cassical).
-// https://stackoverflow.com/q/7872958
-struct ECListener {
-  virtual ~ECListener() = default;
-  virtual const OSDMapRef& pgb_get_osdmap() const = 0;
-  virtual epoch_t pgb_get_osdmap_epoch() const = 0;
-  virtual const pg_info_t &get_info() const = 0;
-  /**
-   * Called when a pull on soid cannot be completed due to
-   * down peers
-   */
-  // XXX
-  virtual void cancel_pull(
-    const hobject_t &soid) = 0;
-  // XXX
-  virtual void schedule_recovery_work(
-    GenContext<ThreadPool::TPHandle&> *c,
-    uint64_t cost) = 0;
-
-  virtual epoch_t get_interval_start_epoch() const = 0;
-  virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
-  virtual const std::set<pg_shard_t> &get_backfill_shards() const = 0;
-  virtual const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_loc_shards()
-    const = 0;
-
-  virtual const std::map<pg_shard_t,
-			 pg_missing_t> &get_shard_missing() const = 0;
-  virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
-#if 1
-  virtual const pg_missing_const_i * maybe_get_shard_missing(
-    pg_shard_t peer) const = 0;
-  virtual const pg_info_t &get_shard_info(pg_shard_t peer) const = 0;
-#endif
-  virtual ceph_tid_t get_tid() = 0;
-  virtual pg_shard_t whoami_shard() const = 0;
-#if 0
-  int whoami() const {
-    return whoami_shard().osd;
-  }
-  spg_t whoami_spg_t() const {
-    return get_info().pgid;
-  }
-#endif
-  // XXX
-  virtual void send_message_osd_cluster(
-    std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
-
-  virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
-
-  // RMWPipeline
-  virtual const pg_pool_t &get_pool() const = 0;
-  virtual const std::set<pg_shard_t> &get_acting_recovery_backfill_shards() const = 0;
-  // XXX
-  virtual bool should_send_op(
-    pg_shard_t peer,
-    const hobject_t &hoid) = 0;
-  virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
-  virtual spg_t primary_spg_t() const = 0;
-  virtual const PGLog &get_log() const = 0;
-  virtual DoutPrefixProvider *get_dpp() = 0;
-  // XXX
-  virtual void apply_stats(
-     const hobject_t &soid,
-     const object_stat_sum_t &delta_stats) = 0;
-};
-
-struct ECCommon {
-  virtual ~ECCommon() = default;
-
-  virtual void handle_sub_write(
-    pg_shard_t from,
-    OpRequestRef msg,
-    ECSubWrite &op,
-    const ZTracer::Trace &trace
-    ) = 0;
-
-  virtual void objects_read_and_reconstruct(
-    const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
-    > &reads,
-    bool fast_read,
-    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func) = 0;
-
-  struct read_request_t {
-    const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
-    std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
-    bool want_attrs;
-    read_request_t(
-      const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
-      const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
-      bool want_attrs)
-      : to_read(to_read), need(need), want_attrs(want_attrs) {}
-  };
-  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
-  struct ReadOp;
-  /**
-   * Low level async read mechanism
-   *
-   * To avoid duplicating the logic for requesting and waiting for
-   * multiple object shards, there is a common async read mechanism
-   * taking a std::map of hobject_t->read_request_t which defines callbacks
-   * taking read_result_ts as arguments.
-   *
-   * tid_to_read_map gives open read ops.  check_recovery_sources uses
-   * shard_to_read_map and ReadOp::source_to_obj to restart reads
-   * involving down osds.
-   *
-   * The user is responsible for specifying replicas on which to read
-   * and for reassembling the buffer on the other side since client
-   * reads require the original object buffer while recovery only needs
-   * the missing pieces.
-   *
-   * Rather than handling reads on the primary directly, we simply send
-   * ourselves a message.  This avoids a dedicated primary path for that
-   * part.
-   */
-  struct read_result_t {
-    int r;
-    std::map<pg_shard_t, int> errors;
-    std::optional<std::map<std::string, ceph::buffer::list, std::less<>> > attrs;
-    std::list<
-      boost::tuple<
-	uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > > returned;
-    read_result_t() : r(0) {}
-  };
-
-  struct ReadCompleter {
-    virtual void finish_single_request(
-      const hobject_t &hoid,
-      read_result_t &res,
-      std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) = 0;
-
-    virtual void finish(int priority) && = 0;
-
-    virtual ~ReadCompleter() = default;
-  };
-
-  friend struct CallClientContexts;
-  struct ClientAsyncReadStatus {
-    unsigned objects_to_read;
-    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> func;
-    std::map<hobject_t,std::pair<int, extent_map> > results;
-    explicit ClientAsyncReadStatus(
-      unsigned objects_to_read,
-      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func)
-      : objects_to_read(objects_to_read), func(std::move(func)) {}
-    void complete_object(
-      const hobject_t &hoid,
-      int err,
-      extent_map &&buffers) {
-      ceph_assert(objects_to_read);
-      --objects_to_read;
-      ceph_assert(!results.count(hoid));
-      results.emplace(hoid, std::make_pair(err, std::move(buffers)));
-    }
-    bool is_complete() const {
-      return objects_to_read == 0;
-    }
-    void run() {
-      func.release()->complete(std::move(results));
-    }
-  };
-
-  struct ReadOp {
-    int priority;
-    ceph_tid_t tid;
-    OpRequestRef op; // may be null if not on behalf of a client
-    // True if redundant reads are issued, false otherwise,
-    // this is useful to tradeoff some resources (redundant ops) for
-    // low latency read, especially on relatively idle cluster
-    bool do_redundant_reads;
-    // True if reading for recovery which could possibly reading only a subset
-    // of the available shards.
-    bool for_recovery;
-    std::unique_ptr<ReadCompleter> on_complete;
-
-    ZTracer::Trace trace;
-
-    std::map<hobject_t, std::set<int>> want_to_read;
-    std::map<hobject_t, read_request_t> to_read;
-    std::map<hobject_t, read_result_t> complete;
-
-    std::map<hobject_t, std::set<pg_shard_t>> obj_to_source;
-    std::map<pg_shard_t, std::set<hobject_t> > source_to_obj;
-
-    void dump(ceph::Formatter *f) const;
-
-    std::set<pg_shard_t> in_progress;
-
-    ReadOp(
-      int priority,
-      ceph_tid_t tid,
-      bool do_redundant_reads,
-      bool for_recovery,
-      std::unique_ptr<ReadCompleter> _on_complete,
-      OpRequestRef op,
-      std::map<hobject_t, std::set<int>> &&_want_to_read,
-      std::map<hobject_t, read_request_t> &&_to_read)
-      : priority(priority),
-        tid(tid),
-        op(op),
-        do_redundant_reads(do_redundant_reads),
-        for_recovery(for_recovery),
-        on_complete(std::move(_on_complete)),
-        want_to_read(std::move(_want_to_read)),
-	to_read(std::move(_to_read)) {
-      for (auto &&hpair: to_read) {
-	auto &returned = complete[hpair.first].returned;
-	for (auto &&extent: hpair.second.to_read) {
-	  returned.push_back(
-	    boost::make_tuple(
-	      extent.get<0>(),
-	      extent.get<1>(),
-	      std::map<pg_shard_t, ceph::buffer::list>()));
-	}
-      }
-    }
-    ReadOp() = delete;
-    ReadOp(const ReadOp &) = delete; // due to on_complete being unique_ptr
-    ReadOp(ReadOp &&) = default;
-  };
-  struct ReadPipeline {
-    void objects_read_and_reconstruct(
-      const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
-      > &reads,
-      bool fast_read,
-      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
-
-    template <class F>
-    void filter_read_op(
-      const OSDMapRef& osdmap,
-      ReadOp &op,
-      F&& on_erase);
-
-    template <class F>
-    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
-
-    void complete_read_op(ReadOp &rop);
-
-    void start_read_op(
-      int priority,
-      std::map<hobject_t, std::set<int>> &want_to_read,
-      std::map<hobject_t, read_request_t> &to_read,
-      OpRequestRef op,
-      bool do_redundant_reads,
-      bool for_recovery,
-      std::unique_ptr<ReadCompleter> on_complete);
-
-    void do_read_op(ReadOp &rop);
-
-    int send_all_remaining_reads(
-      const hobject_t &hoid,
-      ReadOp &rop);
-
-    void on_change();
-
-    void kick_reads();
-
-    std::map<ceph_tid_t, ReadOp> tid_to_read_map;
-    std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
-    std::list<ClientAsyncReadStatus> in_progress_client_reads;
-
-    CephContext* cct;
-    ceph::ErasureCodeInterfaceRef ec_impl;
-    const ECUtil::stripe_info_t& sinfo;
-    // TODO: lay an interface down here
-    ECListener* parent;
-
-    ECListener *get_parent() const { return parent; }
-    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
-    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
-    const pg_info_t &get_info() { return get_parent()->get_info(); }
-
-    ReadPipeline(CephContext* cct,
-                ceph::ErasureCodeInterfaceRef ec_impl,
-                const ECUtil::stripe_info_t& sinfo,
-                ECListener* parent)
-      : cct(cct),
-        ec_impl(std::move(ec_impl)),
-        sinfo(sinfo),
-        parent(parent) {
-    }
-
-    int get_remaining_shards(
-      const hobject_t &hoid,
-      const std::set<int> &avail,
-      const std::set<int> &want,
-      const read_result_t &result,
-      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
-      bool for_recovery);
-
-    void get_all_avail_shards(
-      const hobject_t &hoid,
-      const std::set<pg_shard_t> &error_shards,
-      std::set<int> &have,
-      std::map<shard_id_t, pg_shard_t> &shards,
-      bool for_recovery);
-
-    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
-    friend struct FinishReadOp;
-
-    void get_want_to_read_shards(std::set<int> *want_to_read) const;
-
-    /// Returns to_read replicas sufficient to reconstruct want
-    int get_min_avail_to_read_shards(
-      const hobject_t &hoid,     ///< [in] object
-      const std::set<int> &want,      ///< [in] desired shards
-      bool for_recovery,         ///< [in] true if we may use non-acting replicas
-      bool do_redundant_reads,   ///< [in] true if we want to issue redundant reads to reduce latency
-      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
-      ); ///< @return error code, 0 on success
-  };
-
-  /**
-   * Client writes
-   *
-   * ECTransaction is responsible for generating a transaction for
-   * each shard to which we need to send the write.  As required
-   * by the PGBackend interface, the ECBackend write mechanism
-   * passes trim information with the write and last_complete back
-   * with the reply.
-   *
-   * As with client reads, there is a possibility of out-of-order
-   * completions. Thus, callbacks and completion are called in order
-   * on the writing std::list.
-   */
-
-  struct RMWPipeline {
-    struct Op : boost::intrusive::list_base_hook<> {
-      /// From submit_transaction caller, describes operation
-      hobject_t hoid;
-      object_stat_sum_t delta_stats;
-      eversion_t version;
-      eversion_t trim_to;
-      std::optional<pg_hit_set_history_t> updated_hit_set_history;
-      std::vector<pg_log_entry_t> log_entries;
-      ceph_tid_t tid;
-      osd_reqid_t reqid;
-      ZTracer::Trace trace;
-
-      eversion_t roll_forward_to; /// Soon to be generated internally
-
-      /// Ancillary also provided from submit_transaction caller
-      std::map<hobject_t, ObjectContextRef> obc_map;
-
-      /// see call_write_ordered
-      std::list<std::function<void(void)> > on_write;
-
-      /// Generated internally
-      std::set<hobject_t> temp_added;
-      std::set<hobject_t> temp_cleared;
-
-      ECTransaction::WritePlan plan;
-      bool requires_rmw() const { return !plan.to_read.empty(); }
-      bool invalidates_cache() const { return plan.invalidates_cache; }
-
-      // must be true if requires_rmw(), must be false if invalidates_cache()
-      bool using_cache = true;
-
-      /// In progress read state;
-      std::map<hobject_t,extent_set> pending_read; // subset already being read
-      std::map<hobject_t,extent_set> remote_read;  // subset we must read
-      std::map<hobject_t,extent_map> remote_read_result;
-      bool read_in_progress() const {
-        return !remote_read.empty() && remote_read_result.empty();
-      }
-
-      /// In progress write state.
-      std::set<pg_shard_t> pending_commit;
-      // we need pending_apply for pre-mimic peers so that we don't issue a
-      // read on a remote shard before it has applied a previous write.  We can
-      // remove this after nautilus.
-      std::set<pg_shard_t> pending_apply;
-      bool write_in_progress() const {
-        return !pending_commit.empty() || !pending_apply.empty();
-      }
-
-      /// optional, may be null, for tracking purposes
-      OpRequestRef client_op;
-
-      /// pin for cache
-      ExtentCache::write_pin pin;
-
-      /// Callbacks
-      Context *on_all_commit = nullptr;
-      virtual ~Op() {
-        delete on_all_commit;
-      }
-
-      virtual void generate_transactions(
-        ceph::ErasureCodeInterfaceRef &ecimpl,
-        pg_t pgid,
-        const ECUtil::stripe_info_t &sinfo,
-        std::map<hobject_t,extent_map> *written,
-        std::map<shard_id_t, ObjectStore::Transaction> *transactions,
-        DoutPrefixProvider *dpp,
-        const ceph_release_t require_osd_release = ceph_release_t::unknown) = 0;
-    };
-    using OpRef = std::unique_ptr<Op>;
-    using op_list = boost::intrusive::list<Op>;
-    friend ostream &operator<<(ostream &lhs, const Op &rhs);
-
-    ExtentCache cache;
-    std::map<ceph_tid_t, OpRef> tid_to_op_map; /// Owns Op structure
-    /**
-     * We model the possible rmw states as a std::set of waitlists.
-     * All writes at this time complete in order, so a write blocked
-     * at waiting_state blocks all writes behind it as well (same for
-     * other states).
-     *
-     * Future work: We can break this up into a per-object pipeline
-     * (almost).  First, provide an ordering token to submit_transaction
-     * and require that all operations within a single transaction take
-     * place on a subset of hobject_t space partitioned by that token
-     * (the hashid seem about right to me -- even works for temp objects
-     * if you recall that a temp object created for object head foo will
-     * only ever be referenced by other transactions on foo and aren't
-     * reused).  Next, factor this part into a class and maintain one per
-     * ordering token.  Next, fixup PrimaryLogPG's repop queue to be
-     * partitioned by ordering token.  Finally, refactor the op pipeline
-     * so that the log entries passed into submit_transaction aren't
-     * versioned.  We can't assign versions to them until we actually
-     * submit the operation.  That's probably going to be the hard part.
-     */
-    class pipeline_state_t {
-      enum {
-        CACHE_VALID = 0,
-        CACHE_INVALID = 1
-      } pipeline_state = CACHE_VALID;
-    public:
-      bool caching_enabled() const {
-        return pipeline_state == CACHE_VALID;
-      }
-      bool cache_invalid() const {
-        return !caching_enabled();
-      }
-      void invalidate() {
-        pipeline_state = CACHE_INVALID;
-      }
-      void clear() {
-        pipeline_state = CACHE_VALID;
-      }
-      friend ostream &operator<<(ostream &lhs, const pipeline_state_t &rhs);
-    } pipeline_state;
-
-    op_list waiting_state;        /// writes waiting on pipe_state
-    op_list waiting_reads;        /// writes waiting on partial stripe reads
-    op_list waiting_commit;       /// writes waiting on initial commit
-    eversion_t completed_to;
-    eversion_t committed_to;
-    void start_rmw(OpRef op);
-    bool try_state_to_reads();
-    bool try_reads_to_commit();
-    bool try_finish_rmw();
-    void check_ops();
-
-    void on_change();
-    void call_write_ordered(std::function<void(void)> &&cb);
-
-    CephContext* cct;
-    ECListener *get_parent() const { return parent; }
-    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
-    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
-    const pg_info_t &get_info() { return get_parent()->get_info(); }
-
-    template <typename Func>
-    void objects_read_async_no_cache(
-      const std::map<hobject_t,extent_set> &to_read,
-      Func &&on_complete
-    ) {
-      std::map<hobject_t,std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > > _to_read;
-      for (auto &&hpair: to_read) {
-        auto &l = _to_read[hpair.first];
-        for (auto extent: hpair.second) {
-          l.emplace_back(extent.first, extent.second, 0);
-        }
-      }
-      ec_backend.objects_read_and_reconstruct(
-        _to_read,
-        false,
-        make_gen_lambda_context<
-        std::map<hobject_t,std::pair<int, extent_map> > &&, Func>(
-            std::forward<Func>(on_complete)));
-    }
-    void handle_sub_write(
-      pg_shard_t from,
-      OpRequestRef msg,
-      ECSubWrite &op,
-      const ZTracer::Trace &trace
-    ) {
-      ec_backend.handle_sub_write(from, std::move(msg), op, trace);
-    }
-    // end of iface
-
-    ceph::ErasureCodeInterfaceRef ec_impl;
-    const ECUtil::stripe_info_t& sinfo;
-    ECListener* parent;
-    ECCommon& ec_backend;
-
-    RMWPipeline(CephContext* cct,
-                ceph::ErasureCodeInterfaceRef ec_impl,
-                const ECUtil::stripe_info_t& sinfo,
-                ECListener* parent,
-                ECCommon& ec_backend)
-      : cct(cct),
-        ec_impl(std::move(ec_impl)),
-        sinfo(sinfo),
-        parent(parent),
-        ec_backend(ec_backend) {
-    }
-  };
-};
-
 class ECBackend : public PGBackend, public ECCommon {
 public:
   RecoveryHandle *open_recovery_op() override;
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
new file mode 100644
index 000000000000..bdbc059d42bb
--- /dev/null
+++ b/src/osd/ECCommon.h
@@ -0,0 +1,545 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <boost/intrusive/set.hpp>
+#include <boost/intrusive/list.hpp>
+
+#include "OSD.h"
+#include "PGBackend.h"
+#include "erasure-code/ErasureCodeInterface.h"
+#include "ECUtil.h"
+#include "ECTransaction.h"
+#include "ExtentCache.h"
+
+//forward declaration
+struct ECSubWrite;
+struct ECSubWriteReply;
+struct ECSubRead;
+struct ECSubReadReply;
+
+// ECListener -- an interface decoupling the pipelines from
+// particular implementation of ECBackend (crimson vs cassical).
+// https://stackoverflow.com/q/7872958
+struct ECListener {
+  virtual ~ECListener() = default;
+  virtual const OSDMapRef& pgb_get_osdmap() const = 0;
+  virtual epoch_t pgb_get_osdmap_epoch() const = 0;
+  virtual const pg_info_t &get_info() const = 0;
+  /**
+   * Called when a pull on soid cannot be completed due to
+   * down peers
+   */
+  // XXX
+  virtual void cancel_pull(
+    const hobject_t &soid) = 0;
+  // XXX
+  virtual void schedule_recovery_work(
+    GenContext<ThreadPool::TPHandle&> *c,
+    uint64_t cost) = 0;
+
+  virtual epoch_t get_interval_start_epoch() const = 0;
+  virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
+  virtual const std::set<pg_shard_t> &get_backfill_shards() const = 0;
+  virtual const std::map<hobject_t, std::set<pg_shard_t>> &get_missing_loc_shards()
+    const = 0;
+
+  virtual const std::map<pg_shard_t,
+			 pg_missing_t> &get_shard_missing() const = 0;
+  virtual const pg_missing_const_i &get_shard_missing(pg_shard_t peer) const = 0;
+#if 1
+  virtual const pg_missing_const_i * maybe_get_shard_missing(
+    pg_shard_t peer) const = 0;
+  virtual const pg_info_t &get_shard_info(pg_shard_t peer) const = 0;
+#endif
+  virtual ceph_tid_t get_tid() = 0;
+  virtual pg_shard_t whoami_shard() const = 0;
+#if 0
+  int whoami() const {
+    return whoami_shard().osd;
+  }
+  spg_t whoami_spg_t() const {
+    return get_info().pgid;
+  }
+#endif
+  // XXX
+  virtual void send_message_osd_cluster(
+    std::vector<std::pair<int, Message*>>& messages, epoch_t from_epoch) = 0;
+
+  virtual std::ostream& gen_dbg_prefix(std::ostream& out) const = 0;
+
+  // RMWPipeline
+  virtual const pg_pool_t &get_pool() const = 0;
+  virtual const std::set<pg_shard_t> &get_acting_recovery_backfill_shards() const = 0;
+  // XXX
+  virtual bool should_send_op(
+    pg_shard_t peer,
+    const hobject_t &hoid) = 0;
+  virtual const std::map<pg_shard_t, pg_info_t> &get_shard_info() const = 0;
+  virtual spg_t primary_spg_t() const = 0;
+  virtual const PGLog &get_log() const = 0;
+  virtual DoutPrefixProvider *get_dpp() = 0;
+  // XXX
+  virtual void apply_stats(
+     const hobject_t &soid,
+     const object_stat_sum_t &delta_stats) = 0;
+};
+
+struct ECCommon {
+  virtual ~ECCommon() = default;
+
+  virtual void handle_sub_write(
+    pg_shard_t from,
+    OpRequestRef msg,
+    ECSubWrite &op,
+    const ZTracer::Trace &trace
+    ) = 0;
+
+  virtual void objects_read_and_reconstruct(
+    const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+    > &reads,
+    bool fast_read,
+    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func) = 0;
+
+  struct read_request_t {
+    const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read;
+    std::map<pg_shard_t, std::vector<std::pair<int, int>>> need;
+    bool want_attrs;
+    read_request_t(
+      const std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > &to_read,
+      const std::map<pg_shard_t, std::vector<std::pair<int, int>>> &need,
+      bool want_attrs)
+      : to_read(to_read), need(need), want_attrs(want_attrs) {}
+  };
+  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
+  struct ReadOp;
+  /**
+   * Low level async read mechanism
+   *
+   * To avoid duplicating the logic for requesting and waiting for
+   * multiple object shards, there is a common async read mechanism
+   * taking a std::map of hobject_t->read_request_t which defines callbacks
+   * taking read_result_ts as arguments.
+   *
+   * tid_to_read_map gives open read ops.  check_recovery_sources uses
+   * shard_to_read_map and ReadOp::source_to_obj to restart reads
+   * involving down osds.
+   *
+   * The user is responsible for specifying replicas on which to read
+   * and for reassembling the buffer on the other side since client
+   * reads require the original object buffer while recovery only needs
+   * the missing pieces.
+   *
+   * Rather than handling reads on the primary directly, we simply send
+   * ourselves a message.  This avoids a dedicated primary path for that
+   * part.
+   */
+  struct read_result_t {
+    int r;
+    std::map<pg_shard_t, int> errors;
+    std::optional<std::map<std::string, ceph::buffer::list, std::less<>> > attrs;
+    std::list<
+      boost::tuple<
+	uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > > returned;
+    read_result_t() : r(0) {}
+  };
+
+  struct ReadCompleter {
+    virtual void finish_single_request(
+      const hobject_t &hoid,
+      read_result_t &res,
+      std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) = 0;
+
+    virtual void finish(int priority) && = 0;
+
+    virtual ~ReadCompleter() = default;
+  };
+
+  friend struct CallClientContexts;
+  struct ClientAsyncReadStatus {
+    unsigned objects_to_read;
+    GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> func;
+    std::map<hobject_t,std::pair<int, extent_map> > results;
+    explicit ClientAsyncReadStatus(
+      unsigned objects_to_read,
+      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func)
+      : objects_to_read(objects_to_read), func(std::move(func)) {}
+    void complete_object(
+      const hobject_t &hoid,
+      int err,
+      extent_map &&buffers) {
+      ceph_assert(objects_to_read);
+      --objects_to_read;
+      ceph_assert(!results.count(hoid));
+      results.emplace(hoid, std::make_pair(err, std::move(buffers)));
+    }
+    bool is_complete() const {
+      return objects_to_read == 0;
+    }
+    void run() {
+      func.release()->complete(std::move(results));
+    }
+  };
+
+  struct ReadOp {
+    int priority;
+    ceph_tid_t tid;
+    OpRequestRef op; // may be null if not on behalf of a client
+    // True if redundant reads are issued, false otherwise,
+    // this is useful to tradeoff some resources (redundant ops) for
+    // low latency read, especially on relatively idle cluster
+    bool do_redundant_reads;
+    // True if reading for recovery which could possibly reading only a subset
+    // of the available shards.
+    bool for_recovery;
+    std::unique_ptr<ReadCompleter> on_complete;
+
+    ZTracer::Trace trace;
+
+    std::map<hobject_t, std::set<int>> want_to_read;
+    std::map<hobject_t, read_request_t> to_read;
+    std::map<hobject_t, read_result_t> complete;
+
+    std::map<hobject_t, std::set<pg_shard_t>> obj_to_source;
+    std::map<pg_shard_t, std::set<hobject_t> > source_to_obj;
+
+    void dump(ceph::Formatter *f) const;
+
+    std::set<pg_shard_t> in_progress;
+
+    ReadOp(
+      int priority,
+      ceph_tid_t tid,
+      bool do_redundant_reads,
+      bool for_recovery,
+      std::unique_ptr<ReadCompleter> _on_complete,
+      OpRequestRef op,
+      std::map<hobject_t, std::set<int>> &&_want_to_read,
+      std::map<hobject_t, read_request_t> &&_to_read)
+      : priority(priority),
+        tid(tid),
+        op(op),
+        do_redundant_reads(do_redundant_reads),
+        for_recovery(for_recovery),
+        on_complete(std::move(_on_complete)),
+        want_to_read(std::move(_want_to_read)),
+	to_read(std::move(_to_read)) {
+      for (auto &&hpair: to_read) {
+	auto &returned = complete[hpair.first].returned;
+	for (auto &&extent: hpair.second.to_read) {
+	  returned.push_back(
+	    boost::make_tuple(
+	      extent.get<0>(),
+	      extent.get<1>(),
+	      std::map<pg_shard_t, ceph::buffer::list>()));
+	}
+      }
+    }
+    ReadOp() = delete;
+    ReadOp(const ReadOp &) = delete; // due to on_complete being unique_ptr
+    ReadOp(ReadOp &&) = default;
+  };
+  struct ReadPipeline {
+    void objects_read_and_reconstruct(
+      const std::map<hobject_t, std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+      > &reads,
+      bool fast_read,
+      GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
+
+    template <class F>
+    void filter_read_op(
+      const OSDMapRef& osdmap,
+      ReadOp &op,
+      F&& on_erase);
+
+    template <class F>
+    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
+
+    void complete_read_op(ReadOp &rop);
+
+    void start_read_op(
+      int priority,
+      std::map<hobject_t, std::set<int>> &want_to_read,
+      std::map<hobject_t, read_request_t> &to_read,
+      OpRequestRef op,
+      bool do_redundant_reads,
+      bool for_recovery,
+      std::unique_ptr<ReadCompleter> on_complete);
+
+    void do_read_op(ReadOp &rop);
+
+    int send_all_remaining_reads(
+      const hobject_t &hoid,
+      ReadOp &rop);
+
+    void on_change();
+
+    void kick_reads();
+
+    std::map<ceph_tid_t, ReadOp> tid_to_read_map;
+    std::map<pg_shard_t, std::set<ceph_tid_t> > shard_to_read_map;
+    std::list<ClientAsyncReadStatus> in_progress_client_reads;
+
+    CephContext* cct;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    // TODO: lay an interface down here
+    ECListener* parent;
+
+    ECListener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+    ReadPipeline(CephContext* cct,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+                ECListener* parent)
+      : cct(cct),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+        parent(parent) {
+    }
+
+    int get_remaining_shards(
+      const hobject_t &hoid,
+      const std::set<int> &avail,
+      const std::set<int> &want,
+      const read_result_t &result,
+      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read,
+      bool for_recovery);
+
+    void get_all_avail_shards(
+      const hobject_t &hoid,
+      const std::set<pg_shard_t> &error_shards,
+      std::set<int> &have,
+      std::map<shard_id_t, pg_shard_t> &shards,
+      bool for_recovery);
+
+    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+    friend struct FinishReadOp;
+
+    void get_want_to_read_shards(std::set<int> *want_to_read) const;
+
+    /// Returns to_read replicas sufficient to reconstruct want
+    int get_min_avail_to_read_shards(
+      const hobject_t &hoid,     ///< [in] object
+      const std::set<int> &want,      ///< [in] desired shards
+      bool for_recovery,         ///< [in] true if we may use non-acting replicas
+      bool do_redundant_reads,   ///< [in] true if we want to issue redundant reads to reduce latency
+      std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
+      ); ///< @return error code, 0 on success
+  };
+
+  /**
+   * Client writes
+   *
+   * ECTransaction is responsible for generating a transaction for
+   * each shard to which we need to send the write.  As required
+   * by the PGBackend interface, the ECBackend write mechanism
+   * passes trim information with the write and last_complete back
+   * with the reply.
+   *
+   * As with client reads, there is a possibility of out-of-order
+   * completions. Thus, callbacks and completion are called in order
+   * on the writing std::list.
+   */
+
+  struct RMWPipeline {
+    struct Op : boost::intrusive::list_base_hook<> {
+      /// From submit_transaction caller, describes operation
+      hobject_t hoid;
+      object_stat_sum_t delta_stats;
+      eversion_t version;
+      eversion_t trim_to;
+      std::optional<pg_hit_set_history_t> updated_hit_set_history;
+      std::vector<pg_log_entry_t> log_entries;
+      ceph_tid_t tid;
+      osd_reqid_t reqid;
+      ZTracer::Trace trace;
+
+      eversion_t roll_forward_to; /// Soon to be generated internally
+
+      /// Ancillary also provided from submit_transaction caller
+      std::map<hobject_t, ObjectContextRef> obc_map;
+
+      /// see call_write_ordered
+      std::list<std::function<void(void)> > on_write;
+
+      /// Generated internally
+      std::set<hobject_t> temp_added;
+      std::set<hobject_t> temp_cleared;
+
+      ECTransaction::WritePlan plan;
+      bool requires_rmw() const { return !plan.to_read.empty(); }
+      bool invalidates_cache() const { return plan.invalidates_cache; }
+
+      // must be true if requires_rmw(), must be false if invalidates_cache()
+      bool using_cache = true;
+
+      /// In progress read state;
+      std::map<hobject_t,extent_set> pending_read; // subset already being read
+      std::map<hobject_t,extent_set> remote_read;  // subset we must read
+      std::map<hobject_t,extent_map> remote_read_result;
+      bool read_in_progress() const {
+        return !remote_read.empty() && remote_read_result.empty();
+      }
+
+      /// In progress write state.
+      std::set<pg_shard_t> pending_commit;
+      // we need pending_apply for pre-mimic peers so that we don't issue a
+      // read on a remote shard before it has applied a previous write.  We can
+      // remove this after nautilus.
+      std::set<pg_shard_t> pending_apply;
+      bool write_in_progress() const {
+        return !pending_commit.empty() || !pending_apply.empty();
+      }
+
+      /// optional, may be null, for tracking purposes
+      OpRequestRef client_op;
+
+      /// pin for cache
+      ExtentCache::write_pin pin;
+
+      /// Callbacks
+      Context *on_all_commit = nullptr;
+      virtual ~Op() {
+        delete on_all_commit;
+      }
+
+      virtual void generate_transactions(
+        ceph::ErasureCodeInterfaceRef &ecimpl,
+        pg_t pgid,
+        const ECUtil::stripe_info_t &sinfo,
+        std::map<hobject_t,extent_map> *written,
+        std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+        DoutPrefixProvider *dpp,
+        const ceph_release_t require_osd_release = ceph_release_t::unknown) = 0;
+    };
+    using OpRef = std::unique_ptr<Op>;
+    using op_list = boost::intrusive::list<Op>;
+    friend ostream &operator<<(ostream &lhs, const Op &rhs);
+
+    ExtentCache cache;
+    std::map<ceph_tid_t, OpRef> tid_to_op_map; /// Owns Op structure
+    /**
+     * We model the possible rmw states as a std::set of waitlists.
+     * All writes at this time complete in order, so a write blocked
+     * at waiting_state blocks all writes behind it as well (same for
+     * other states).
+     *
+     * Future work: We can break this up into a per-object pipeline
+     * (almost).  First, provide an ordering token to submit_transaction
+     * and require that all operations within a single transaction take
+     * place on a subset of hobject_t space partitioned by that token
+     * (the hashid seem about right to me -- even works for temp objects
+     * if you recall that a temp object created for object head foo will
+     * only ever be referenced by other transactions on foo and aren't
+     * reused).  Next, factor this part into a class and maintain one per
+     * ordering token.  Next, fixup PrimaryLogPG's repop queue to be
+     * partitioned by ordering token.  Finally, refactor the op pipeline
+     * so that the log entries passed into submit_transaction aren't
+     * versioned.  We can't assign versions to them until we actually
+     * submit the operation.  That's probably going to be the hard part.
+     */
+    class pipeline_state_t {
+      enum {
+        CACHE_VALID = 0,
+        CACHE_INVALID = 1
+      } pipeline_state = CACHE_VALID;
+    public:
+      bool caching_enabled() const {
+        return pipeline_state == CACHE_VALID;
+      }
+      bool cache_invalid() const {
+        return !caching_enabled();
+      }
+      void invalidate() {
+        pipeline_state = CACHE_INVALID;
+      }
+      void clear() {
+        pipeline_state = CACHE_VALID;
+      }
+      friend ostream &operator<<(ostream &lhs, const pipeline_state_t &rhs);
+    } pipeline_state;
+
+    op_list waiting_state;        /// writes waiting on pipe_state
+    op_list waiting_reads;        /// writes waiting on partial stripe reads
+    op_list waiting_commit;       /// writes waiting on initial commit
+    eversion_t completed_to;
+    eversion_t committed_to;
+    void start_rmw(OpRef op);
+    bool try_state_to_reads();
+    bool try_reads_to_commit();
+    bool try_finish_rmw();
+    void check_ops();
+
+    void on_change();
+    void call_write_ordered(std::function<void(void)> &&cb);
+
+    CephContext* cct;
+    ECListener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+
+    template <typename Func>
+    void objects_read_async_no_cache(
+      const std::map<hobject_t,extent_set> &to_read,
+      Func &&on_complete
+    ) {
+      std::map<hobject_t,std::list<boost::tuple<uint64_t, uint64_t, uint32_t> > > _to_read;
+      for (auto &&hpair: to_read) {
+        auto &l = _to_read[hpair.first];
+        for (auto extent: hpair.second) {
+          l.emplace_back(extent.first, extent.second, 0);
+        }
+      }
+      ec_backend.objects_read_and_reconstruct(
+        _to_read,
+        false,
+        make_gen_lambda_context<
+        std::map<hobject_t,std::pair<int, extent_map> > &&, Func>(
+            std::forward<Func>(on_complete)));
+    }
+    void handle_sub_write(
+      pg_shard_t from,
+      OpRequestRef msg,
+      ECSubWrite &op,
+      const ZTracer::Trace &trace
+    ) {
+      ec_backend.handle_sub_write(from, std::move(msg), op, trace);
+    }
+    // end of iface
+
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    ECListener* parent;
+    ECCommon& ec_backend;
+
+    RMWPipeline(CephContext* cct,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+                ECListener* parent,
+                ECCommon& ec_backend)
+      : cct(cct),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+        parent(parent),
+        ec_backend(ec_backend) {
+    }
+  };
+};
+

From 7dfa6d412cc8db47c0fc98474de42caf1d4381c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 19 Dec 2023 18:13:56 +0100
Subject: [PATCH 1317/2492] osd: settle RMWPipeline and RMWPipeline within
 ECBackend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 10 +++++--
 src/osd/ECBackend.h  |  3 ++-
 src/osd/ECCommon.h   | 64 +++++++++++++++++++++++++++++++++++---------
 src/osd/OSDCap.h     |  7 ++---
 src/osd/Session.h    |  1 +
 5 files changed, 67 insertions(+), 18 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 4e8f0b4e019a..b63ab0cca1cf 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -840,7 +840,7 @@ bool ECBackend::_handle_message(
     MOSDECSubOpWrite *op = static_cast<MOSDECSubOpWrite*>(
       _op->get_nonconst_req());
     parent->maybe_preempt_replica_scrub(op->op.soid);
-    handle_sub_write(op->op.from, _op, op->op, _op->pg_trace);
+    handle_sub_write(op->op.from, _op, op->op, _op->pg_trace, *get_parent()->get_eclistener());
     return true;
   }
   case MSG_OSD_EC_WRITE_REPLY: {
@@ -958,7 +958,8 @@ void ECBackend::handle_sub_write(
   pg_shard_t from,
   OpRequestRef msg,
   ECSubWrite &op,
-  const ZTracer::Trace &trace)
+  const ZTracer::Trace &trace,
+  ECListener&)
 {
   if (msg) {
     msg->mark_event("sub_op_started");
@@ -1454,10 +1455,15 @@ void ECCommon::ReadPipeline::filter_read_op(
      *    the pull on the affected objects and pushes from in-memory buffers
      *    on any now complete unaffected objects.
      */
+#ifndef WITH_SEASTAR
     get_parent()->schedule_recovery_work(
       get_parent()->bless_unlocked_gencontext(
         new FinishReadOp(*this, op.tid)),
       1);
+#else
+    // TODO
+    ceph_abort_msg("not yet implemented");
+#endif
   }
 }
 
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index df8cc6dc54aa..739e35563bfd 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -68,7 +68,8 @@ class ECBackend : public PGBackend, public ECCommon {
     pg_shard_t from,
     OpRequestRef msg,
     ECSubWrite &op,
-    const ZTracer::Trace &trace
+    const ZTracer::Trace &trace,
+    ECListener& eclistener
     ) override;
   void handle_sub_read(
     pg_shard_t from,
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index bdbc059d42bb..a7585ea13ddd 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -17,18 +17,37 @@
 #include <boost/intrusive/set.hpp>
 #include <boost/intrusive/list.hpp>
 
-#include "OSD.h"
-#include "PGBackend.h"
 #include "erasure-code/ErasureCodeInterface.h"
 #include "ECUtil.h"
+#if WITH_SEASTAR
+#include "ExtentCache.h"
+#include "crimson/osd/object_context.h"
+#include "os/Transaction.h"
+#include "osd/OSDMap.h"
+#include "osd/osd_op_util.h"
+
+struct ECTransaction {
+  struct WritePlan {
+    bool invalidates_cache = false; // Yes, both are possible
+    std::map<hobject_t,extent_set> to_read;
+    std::map<hobject_t,extent_set> will_write; // superset of to_read
+
+    std::map<hobject_t,ECUtil::HashInfoRef> hash_infos;
+  };
+};
+
+typedef void* OpRequestRef;
+typedef crimson::osd::ObjectContextRef ObjectContextRef;
+#else
+#include "common/WorkQueue.h"
+#endif
+
 #include "ECTransaction.h"
 #include "ExtentCache.h"
 
 //forward declaration
 struct ECSubWrite;
-struct ECSubWriteReply;
-struct ECSubRead;
-struct ECSubReadReply;
+struct PGLog;
 
 // ECListener -- an interface decoupling the pipelines from
 // particular implementation of ECBackend (crimson vs cassical).
@@ -46,9 +65,14 @@ struct ECListener {
   virtual void cancel_pull(
     const hobject_t &soid) = 0;
   // XXX
+#ifndef WITH_SEASTAR
+  virtual GenContext<ThreadPool::TPHandle&> *bless_unlocked_gencontext(
+    GenContext<ThreadPool::TPHandle&> *c) = 0;
+
   virtual void schedule_recovery_work(
     GenContext<ThreadPool::TPHandle&> *c,
     uint64_t cost) = 0;
+#endif
 
   virtual epoch_t get_interval_start_epoch() const = 0;
   virtual const std::set<pg_shard_t> &get_acting_shards() const = 0;
@@ -95,6 +119,21 @@ struct ECListener {
   virtual void apply_stats(
      const hobject_t &soid,
      const object_stat_sum_t &delta_stats) = 0;
+
+  // new batch
+  virtual bool is_missing_object(const hobject_t& oid) const = 0;
+  virtual void add_local_next_event(const pg_log_entry_t& e) = 0;
+  virtual void log_operation(
+    std::vector<pg_log_entry_t>&& logv,
+    const std::optional<pg_hit_set_history_t> &hset_history,
+    const eversion_t &trim_to,
+    const eversion_t &roll_forward_to,
+    const eversion_t &min_last_complete_ondisk,
+    bool transaction_applied,
+    ObjectStore::Transaction &t,
+    bool async = false) = 0;
+  virtual void op_applied(
+    const eversion_t &applied_version) = 0;
 };
 
 struct ECCommon {
@@ -104,7 +143,8 @@ struct ECCommon {
     pg_shard_t from,
     OpRequestRef msg,
     ECSubWrite &op,
-    const ZTracer::Trace &trace
+    const ZTracer::Trace &trace,
+    ECListener& eclistener
     ) = 0;
 
   virtual void objects_read_and_reconstruct(
@@ -123,7 +163,7 @@ struct ECCommon {
       bool want_attrs)
       : to_read(to_read), need(need), want_attrs(want_attrs) {}
   };
-  friend ostream &operator<<(ostream &lhs, const read_request_t &rhs);
+  friend std::ostream &operator<<(std::ostream &lhs, const read_request_t &rhs);
   struct ReadOp;
   /**
    * Low level async read mechanism
@@ -328,7 +368,7 @@ struct ECCommon {
       std::map<shard_id_t, pg_shard_t> &shards,
       bool for_recovery);
 
-    friend ostream &operator<<(ostream &lhs, const ReadOp &rhs);
+    friend std::ostream &operator<<(std::ostream &lhs, const ReadOp &rhs);
     friend struct FinishReadOp;
 
     void get_want_to_read_shards(std::set<int> *want_to_read) const;
@@ -424,13 +464,13 @@ struct ECCommon {
         pg_t pgid,
         const ECUtil::stripe_info_t &sinfo,
         std::map<hobject_t,extent_map> *written,
-        std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+        std::map<shard_id_t, ceph::os::Transaction> *transactions,
         DoutPrefixProvider *dpp,
         const ceph_release_t require_osd_release = ceph_release_t::unknown) = 0;
     };
     using OpRef = std::unique_ptr<Op>;
     using op_list = boost::intrusive::list<Op>;
-    friend ostream &operator<<(ostream &lhs, const Op &rhs);
+    friend std::ostream &operator<<(std::ostream &lhs, const Op &rhs);
 
     ExtentCache cache;
     std::map<ceph_tid_t, OpRef> tid_to_op_map; /// Owns Op structure
@@ -472,7 +512,7 @@ struct ECCommon {
       void clear() {
         pipeline_state = CACHE_VALID;
       }
-      friend ostream &operator<<(ostream &lhs, const pipeline_state_t &rhs);
+      friend std::ostream &operator<<(std::ostream &lhs, const pipeline_state_t &rhs);
     } pipeline_state;
 
     op_list waiting_state;        /// writes waiting on pipe_state
@@ -520,7 +560,7 @@ struct ECCommon {
       ECSubWrite &op,
       const ZTracer::Trace &trace
     ) {
-      ec_backend.handle_sub_write(from, std::move(msg), op, trace);
+      ec_backend.handle_sub_write(from, std::move(msg), op, trace, *get_parent());
     }
     // end of iface
 
diff --git a/src/osd/OSDCap.h b/src/osd/OSDCap.h
index caf6cd788d73..8aed09adf9bb 100644
--- a/src/osd/OSDCap.h
+++ b/src/osd/OSDCap.h
@@ -29,14 +29,15 @@
 #include <ostream>
 using std::ostream;
 
-#include "include/types.h"
-#include "OpRequest.h"
-
 #include <list>
 #include <vector>
 #include <boost/optional.hpp>
 #include <boost/fusion/include/adapt_struct.hpp>
 
+#include "include/types.h"
+#include "osd/osd_op_util.h"
+
+
 static const __u8 OSD_CAP_R     = (1 << 1);      // read
 static const __u8 OSD_CAP_W     = (1 << 2);      // write
 static const __u8 OSD_CAP_CLS_R = (1 << 3);      // class read
diff --git a/src/osd/Session.h b/src/osd/Session.h
index 3c3eae211cf5..9fa9c6554563 100644
--- a/src/osd/Session.h
+++ b/src/osd/Session.h
@@ -20,6 +20,7 @@
 #include "global/global_context.h"
 #include "include/spinlock.h"
 #include "OSDCap.h"
+#include "OpRequest.h"
 #include "Watch.h"
 #include "OSDMap.h"
 #include "PeeringState.h"

From fae1cde05126595e5eb2656dff54dc31bf1105d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 26 Sep 2023 17:46:58 +0200
Subject: [PATCH 1318/2492] osd: make ECBackend::kick_reads() private
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 739e35563bfd..54b8e16119eb 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -153,10 +153,11 @@ class ECBackend : public PGBackend, public ECCommon {
     Context *on_complete,
     bool fast_read = false) override;
 
-  void kick_reads();
-
 private:
   friend struct ECRecoveryHandle;
+
+  void kick_reads();
+
   uint64_t get_recovery_chunk_size() const {
     return round_up_to(cct->_conf->osd_recovery_max_chunk,
 			sinfo.get_stripe_width());

From cb8078a9545b46d3833faa20a9fe53cedae4758f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 27 Sep 2023 12:46:12 +0200
Subject: [PATCH 1319/2492] osd: decouple classical and dummy ops for
 RMWPipeline::try_finish_rmw()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 49 ++++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index b63ab0cca1cf..467315a8b7f3 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1574,23 +1574,21 @@ struct ECClassicalOp : ECCommon::RMWPipeline::Op {
       DoutPrefixProvider *dpp,
       const ceph_release_t require_osd_release) final
   {
-    // there is the special case of t-empty op generated in try_finish_rmw()
-    if (t) {
-      ECTransaction::generate_transactions(
-        t.get(),
-        plan,
-        ecimpl,
-        pgid,
-        sinfo,
-        remote_read_result,
-        log_entries,
-        written,
-        transactions,
-        &temp_added,
-        &temp_cleared,
-        dpp,
-        require_osd_release);
-    }
+    assert(t);
+    ECTransaction::generate_transactions(
+      t.get(),
+      plan,
+      ecimpl,
+      pgid,
+      sinfo,
+      remote_read_result,
+      log_entries,
+      written,
+      transactions,
+      &temp_added,
+      &temp_cleared,
+      dpp,
+      require_osd_release);
   }
 
   template <typename F>
@@ -2227,6 +2225,21 @@ bool ECCommon::RMWPipeline::try_reads_to_commit()
   return true;
 }
 
+struct ECDummyOp : ECCommon::RMWPipeline::Op {
+  void generate_transactions(
+      ceph::ErasureCodeInterfaceRef &ecimpl,
+      pg_t pgid,
+      const ECUtil::stripe_info_t &sinfo,
+      std::map<hobject_t,extent_map> *written,
+      std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+      DoutPrefixProvider *dpp,
+      const ceph_release_t require_osd_release) final
+  {
+    // NOP, as -- in constrast to ECClassicalOp -- there is no
+    // transaction involved
+  }
+};
+
 bool ECCommon::RMWPipeline::try_finish_rmw()
 {
   if (waiting_commit.empty())
@@ -2250,7 +2263,7 @@ bool ECCommon::RMWPipeline::try_finish_rmw()
 	waiting_commit.empty()) {
       // submit a dummy, transaction-empty op to kick the rollforward
       auto tid = get_parent()->get_tid();
-      auto nop = std::make_unique<ECClassicalOp>();
+      auto nop = std::make_unique<ECDummyOp>();
       nop->hoid = op->hoid;
       nop->trim_to = op->trim_to;
       nop->roll_forward_to = op->version;

From 6509f78a5e08c07143ab48fadf493edfad4c543e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 27 Sep 2023 14:17:06 +0200
Subject: [PATCH 1320/2492] osd: shuffle the ECCommon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/CMakeLists.txt |   1 +
 src/osd/ECBackend.cc   | 840 +-------------------------------------
 src/osd/ECCommon.cc    | 907 +++++++++++++++++++++++++++++++++++++++++
 src/osd/ECCommon.h     |  13 +-
 4 files changed, 922 insertions(+), 839 deletions(-)
 create mode 100644 src/osd/ECCommon.cc

diff --git a/src/osd/CMakeLists.txt b/src/osd/CMakeLists.txt
index 7043f27e2932..5f1e96573c3b 100644
--- a/src/osd/CMakeLists.txt
+++ b/src/osd/CMakeLists.txt
@@ -17,6 +17,7 @@ set(osd_srcs
   PGLog.cc
   PrimaryLogPG.cc
   ReplicatedBackend.cc
+  ECCommon.cc
   ECBackend.cc
   ECTransaction.cc
   PGBackend.cc
diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 467315a8b7f3..afa5d5bba44e 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -66,18 +66,6 @@ struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
   list<ECBackend::RecoveryOp> ops;
 };
 
-ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::pipeline_state_t &rhs) {
-  switch (rhs.pipeline_state) {
-  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_VALID:
-    return lhs << "CACHE_VALID";
-  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_INVALID:
-    return lhs << "CACHE_INVALID";
-  default:
-    ceph_abort_msg("invalid pipeline state");
-  }
-  return lhs; // unreachable
-}
-
 static ostream &operator<<(ostream &lhs, const map<pg_shard_t, bufferlist> &rhs)
 {
   lhs << "[";
@@ -112,80 +100,6 @@ static ostream &operator<<(
 	     << rhs.get<1>() << ", " << rhs.get<2>() << ")";
 }
 
-ostream &operator<<(ostream &lhs, const ECCommon::read_request_t &rhs)
-{
-  return lhs << "read_request_t(to_read=[" << rhs.to_read << "]"
-	     << ", need=" << rhs.need
-	     << ", want_attrs=" << rhs.want_attrs
-	     << ")";
-}
-
-ostream &operator<<(ostream &lhs, const ECCommon::read_result_t &rhs)
-{
-  lhs << "read_result_t(r=" << rhs.r
-      << ", errors=" << rhs.errors;
-  if (rhs.attrs) {
-    lhs << ", attrs=" << *(rhs.attrs);
-  } else {
-    lhs << ", noattrs";
-  }
-  return lhs << ", returned=" << rhs.returned << ")";
-}
-
-ostream &operator<<(ostream &lhs, const ECCommon::ReadOp &rhs)
-{
-  lhs << "ReadOp(tid=" << rhs.tid;
-  if (rhs.op && rhs.op->get_req()) {
-    lhs << ", op=";
-    rhs.op->get_req()->print(lhs);
-  }
-  return lhs << ", to_read=" << rhs.to_read
-	     << ", complete=" << rhs.complete
-	     << ", priority=" << rhs.priority
-	     << ", obj_to_source=" << rhs.obj_to_source
-	     << ", source_to_obj=" << rhs.source_to_obj
-	     << ", in_progress=" << rhs.in_progress << ")";
-}
-
-void ECCommon::ReadOp::dump(Formatter *f) const
-{
-  f->dump_unsigned("tid", tid);
-  if (op && op->get_req()) {
-    f->dump_stream("op") << *(op->get_req());
-  }
-  f->dump_stream("to_read") << to_read;
-  f->dump_stream("complete") << complete;
-  f->dump_int("priority", priority);
-  f->dump_stream("obj_to_source") << obj_to_source;
-  f->dump_stream("source_to_obj") << source_to_obj;
-  f->dump_stream("in_progress") << in_progress;
-}
-
-ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::Op &rhs)
-{
-  lhs << "Op(" << rhs.hoid
-      << " v=" << rhs.version
-      << " tt=" << rhs.trim_to
-      << " tid=" << rhs.tid
-      << " reqid=" << rhs.reqid;
-  if (rhs.client_op && rhs.client_op->get_req()) {
-    lhs << " client_op=";
-    rhs.client_op->get_req()->print(lhs);
-  }
-  lhs << " roll_forward_to=" << rhs.roll_forward_to
-      << " temp_added=" << rhs.temp_added
-      << " temp_cleared=" << rhs.temp_cleared
-      << " pending_read=" << rhs.pending_read
-      << " remote_read=" << rhs.remote_read
-      << " remote_read_result=" << rhs.remote_read_result
-      << " pending_apply=" << rhs.pending_apply
-      << " pending_commit=" << rhs.pending_commit
-      << " plan.to_read=" << rhs.plan.to_read
-      << " plan.will_write=" << rhs.plan.will_write
-      << ")";
-  return lhs;
-}
-
 ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
 {
   return lhs << "RecoveryOp("
@@ -1348,184 +1262,13 @@ void ECBackend::handle_sub_read_reply(
   }
 }
 
-void ECCommon::ReadPipeline::complete_read_op(ReadOp &rop)
-{
-  map<hobject_t, read_request_t>::iterator reqiter =
-    rop.to_read.begin();
-  map<hobject_t, read_result_t>::iterator resiter =
-    rop.complete.begin();
-  ceph_assert(rop.to_read.size() == rop.complete.size());
-  for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
-    rop.on_complete->finish_single_request(
-      reqiter->first,
-      resiter->second,
-      reqiter->second.to_read);
-  }
-  ceph_assert(rop.on_complete);
-  std::move(*rop.on_complete).finish(rop.priority);
-  rop.on_complete = nullptr;
-  // if the read op is over. clean all the data of this tid.
-  for (set<pg_shard_t>::iterator iter = rop.in_progress.begin();
-    iter != rop.in_progress.end();
-    iter++) {
-    shard_to_read_map[*iter].erase(rop.tid);
-  }
-  rop.in_progress.clear();
-  tid_to_read_map.erase(rop.tid);
-}
-
-struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
-  ECCommon::ReadPipeline& read_pipeline;
-  ceph_tid_t tid;
-  FinishReadOp(ECCommon::ReadPipeline& read_pipeline, ceph_tid_t tid)
-    : read_pipeline(read_pipeline), tid(tid) {}
-  void finish(ThreadPool::TPHandle&) override {
-    auto ropiter = read_pipeline.tid_to_read_map.find(tid);
-    ceph_assert(ropiter != read_pipeline.tid_to_read_map.end());
-    read_pipeline.complete_read_op(ropiter->second);
-  }
-};
-
-template <class F>
-void ECCommon::ReadPipeline::filter_read_op(
-  const OSDMapRef& osdmap,
-  ReadOp &op,
-  F&& on_erase)
-{
-  set<hobject_t> to_cancel;
-  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
-       i != op.source_to_obj.end();
-       ++i) {
-    if (osdmap->is_down(i->first.osd)) {
-      to_cancel.insert(i->second.begin(), i->second.end());
-      op.in_progress.erase(i->first);
-      continue;
-    }
-  }
-
-  if (to_cancel.empty())
-    return;
-
-  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
-       i != op.source_to_obj.end();
-       ) {
-    for (set<hobject_t>::iterator j = i->second.begin();
-	 j != i->second.end();
-	 ) {
-      if (to_cancel.count(*j))
-	i->second.erase(j++);
-      else
-	++j;
-    }
-    if (i->second.empty()) {
-      op.source_to_obj.erase(i++);
-    } else {
-      ceph_assert(!osdmap->is_down(i->first.osd));
-      ++i;
-    }
-  }
-
-  for (set<hobject_t>::iterator i = to_cancel.begin();
-       i != to_cancel.end();
-       ++i) {
-    get_parent()->cancel_pull(*i);
-
-    ceph_assert(op.to_read.count(*i));
-    read_request_t &req = op.to_read.find(*i)->second;
-    dout(10) << __func__ << ": canceling " << req
-	     << "  for obj " << *i << dendl;
-    op.to_read.erase(*i);
-    op.complete.erase(*i);
-    on_erase(*i);
-  }
-
-  if (op.in_progress.empty()) {
-    /* This case is odd.  filter_read_op gets called while processing
-     * an OSDMap.  Normal, non-recovery reads only happen from acting
-     * set osds.  For this op to have had a read source go down and
-     * there not be an interval change, it must be part of a pull during
-     * log-based recovery.
-     *
-     * This callback delays calling complete_read_op until later to avoid
-     * dealing with recovery while handling an OSDMap.  We assign a
-     * cost here of 1 because:
-     * 1) This should be very rare, and the operation itself was already
-     *    throttled.
-     * 2) It shouldn't result in IO, rather it should result in restarting
-     *    the pull on the affected objects and pushes from in-memory buffers
-     *    on any now complete unaffected objects.
-     */
-#ifndef WITH_SEASTAR
-    get_parent()->schedule_recovery_work(
-      get_parent()->bless_unlocked_gencontext(
-        new FinishReadOp(*this, op.tid)),
-      1);
-#else
-    // TODO
-    ceph_abort_msg("not yet implemented");
-#endif
-  }
-}
-
-template <class F>
-void ECCommon::ReadPipeline::check_recovery_sources(
-  const OSDMapRef& osdmap,
-  F&& on_erase)
-{
-  set<ceph_tid_t> tids_to_filter;
-  for (map<pg_shard_t, set<ceph_tid_t> >::iterator 
-       i = shard_to_read_map.begin();
-       i != shard_to_read_map.end();
-       ) {
-    if (osdmap->is_down(i->first.osd)) {
-      tids_to_filter.insert(i->second.begin(), i->second.end());
-      shard_to_read_map.erase(i++);
-    } else {
-      ++i;
-    }
-  }
-  for (set<ceph_tid_t>::iterator i = tids_to_filter.begin();
-       i != tids_to_filter.end();
-       ++i) {
-    map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
-    ceph_assert(j != tid_to_read_map.end());
-    filter_read_op(osdmap, j->second, on_erase);
-  }
-}
-
 void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
 {
+#if 0
   read_pipeline.check_recovery_sources(osdmap, [this] (const hobject_t& obj) {
     recovery_ops.erase(obj);
   });
-}
-
-void ECCommon::ReadPipeline::on_change()
-{
-  for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
-       i != tid_to_read_map.end();
-       ++i) {
-    dout(10) << __func__ << ": cancelling " << i->second << dendl;
-  }
-  tid_to_read_map.clear();
-  shard_to_read_map.clear();
-  in_progress_client_reads.clear();
-}
-
-void ECCommon::RMWPipeline::on_change()
-{
-  dout(10) << __func__ << dendl;
-
-  completed_to = eversion_t();
-  committed_to = eversion_t();
-  pipeline_state.clear();
-  waiting_reads.clear();
-  waiting_state.clear();
-  waiting_commit.clear();
-  for (auto &&op: tid_to_op_map) {
-    cache.release_write_pin(op.second->pin);
-  }
-  tid_to_op_map.clear();
+#endif
 }
 
 void ECBackend::on_change()
@@ -1656,271 +1399,6 @@ void ECBackend::submit_transaction(
   rmw_pipeline.start_rmw(std::move(op));
 }
 
-void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
-  if (!waiting_state.empty()) {
-    waiting_state.back().on_write.emplace_back(std::move(cb));
-  } else if (!waiting_reads.empty()) {
-    waiting_reads.back().on_write.emplace_back(std::move(cb));
-  } else {
-    // Nothing earlier in the pipeline, just call it
-    cb();
-  }
-}
-
-void ECCommon::ReadPipeline::get_all_avail_shards(
-  const hobject_t &hoid,
-  const set<pg_shard_t> &error_shards,
-  set<int> &have,
-  map<shard_id_t, pg_shard_t> &shards,
-  bool for_recovery)
-{
-  for (set<pg_shard_t>::const_iterator i =
-	 get_parent()->get_acting_shards().begin();
-       i != get_parent()->get_acting_shards().end();
-       ++i) {
-    dout(10) << __func__ << ": checking acting " << *i << dendl;
-    const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
-    if (error_shards.find(*i) != error_shards.end())
-      continue;
-    if (!missing.is_missing(hoid)) {
-      ceph_assert(!have.count(i->shard));
-      have.insert(i->shard);
-      ceph_assert(!shards.count(i->shard));
-      shards.insert(make_pair(i->shard, *i));
-    }
-  }
-
-  if (for_recovery) {
-    for (set<pg_shard_t>::const_iterator i =
-	   get_parent()->get_backfill_shards().begin();
-	 i != get_parent()->get_backfill_shards().end();
-	 ++i) {
-      if (error_shards.find(*i) != error_shards.end())
-	continue;
-      if (have.count(i->shard)) {
-	ceph_assert(shards.count(i->shard));
-	continue;
-      }
-      dout(10) << __func__ << ": checking backfill " << *i << dendl;
-      ceph_assert(!shards.count(i->shard));
-      const pg_info_t &info = get_parent()->get_shard_info(*i);
-      const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
-      if (hoid < info.last_backfill &&
-	  !missing.is_missing(hoid)) {
-	have.insert(i->shard);
-	shards.insert(make_pair(i->shard, *i));
-      }
-    }
-
-    map<hobject_t, set<pg_shard_t>>::const_iterator miter =
-      get_parent()->get_missing_loc_shards().find(hoid);
-    if (miter != get_parent()->get_missing_loc_shards().end()) {
-      for (set<pg_shard_t>::iterator i = miter->second.begin();
-	   i != miter->second.end();
-	   ++i) {
-	dout(10) << __func__ << ": checking missing_loc " << *i << dendl;
-	auto m = get_parent()->maybe_get_shard_missing(*i);
-	if (m) {
-	  ceph_assert(!(*m).is_missing(hoid));
-	}
-	if (error_shards.find(*i) != error_shards.end())
-	  continue;
-	have.insert(i->shard);
-	shards.insert(make_pair(i->shard, *i));
-      }
-    }
-  }
-}
-
-int ECCommon::ReadPipeline::get_min_avail_to_read_shards(
-  const hobject_t &hoid,
-  const set<int> &want,
-  bool for_recovery,
-  bool do_redundant_reads,
-  map<pg_shard_t, vector<pair<int, int>>> *to_read)
-{
-  // Make sure we don't do redundant reads for recovery
-  ceph_assert(!for_recovery || !do_redundant_reads);
-
-  set<int> have;
-  map<shard_id_t, pg_shard_t> shards;
-  set<pg_shard_t> error_shards;
-
-  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
-
-  map<int, vector<pair<int, int>>> need;
-  int r = ec_impl->minimum_to_decode(want, have, &need);
-  if (r < 0)
-    return r;
-
-  if (do_redundant_reads) {
-      vector<pair<int, int>> subchunks_list;
-      subchunks_list.push_back(make_pair(0, ec_impl->get_sub_chunk_count()));
-      for (auto &&i: have) {
-        need[i] = subchunks_list;
-      }
-  } 
-
-  if (!to_read)
-    return 0;
-
-  for (auto &&i:need) {
-    ceph_assert(shards.count(shard_id_t(i.first)));
-    to_read->insert(make_pair(shards[shard_id_t(i.first)], i.second));
-  }
-  return 0;
-}
-
-int ECCommon::ReadPipeline::get_remaining_shards(
-  const hobject_t &hoid,
-  const set<int> &avail,
-  const set<int> &want,
-  const read_result_t &result,
-  map<pg_shard_t, vector<pair<int, int>>> *to_read,
-  bool for_recovery)
-{
-  ceph_assert(to_read);
-
-  set<int> have;
-  map<shard_id_t, pg_shard_t> shards;
-  set<pg_shard_t> error_shards;
-  for (auto &p : result.errors) {
-    error_shards.insert(p.first);
-  }
-
-  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
-
-  map<int, vector<pair<int, int>>> need;
-  int r = ec_impl->minimum_to_decode(want, have, &need);
-  if (r < 0) {
-    dout(0) << __func__ << " not enough shards left to try for " << hoid
-	    << " read result was " << result << dendl;
-    return -EIO;
-  }
-
-  set<int> shards_left;
-  for (auto p : need) {
-    if (avail.find(p.first) == avail.end()) {
-      shards_left.insert(p.first);
-    }
-  }
-
-  vector<pair<int, int>> subchunks;
-  subchunks.push_back(make_pair(0, ec_impl->get_sub_chunk_count()));
-  for (set<int>::iterator i = shards_left.begin();
-       i != shards_left.end();
-       ++i) {
-    ceph_assert(shards.count(shard_id_t(*i)));
-    ceph_assert(avail.find(*i) == avail.end());
-    to_read->insert(make_pair(shards[shard_id_t(*i)], subchunks));
-  }
-  return 0;
-}
-
-void ECCommon::ReadPipeline::start_read_op(
-  int priority,
-  map<hobject_t, set<int>> &want_to_read,
-  map<hobject_t, read_request_t> &to_read,
-  OpRequestRef _op,
-  bool do_redundant_reads,
-  bool for_recovery,
-  std::unique_ptr<ECCommon::ReadCompleter> on_complete)
-{
-  ceph_tid_t tid = get_parent()->get_tid();
-  ceph_assert(!tid_to_read_map.count(tid));
-  auto &op = tid_to_read_map.emplace(
-    tid,
-    ReadOp(
-      priority,
-      tid,
-      do_redundant_reads,
-      for_recovery,
-      std::move(on_complete),
-      _op,
-      std::move(want_to_read),
-      std::move(to_read))).first->second;
-  dout(10) << __func__ << ": starting " << op << dendl;
-  if (_op) {
-    op.trace = _op->pg_trace;
-    op.trace.event("start ec read");
-  }
-  do_read_op(op);
-}
-
-void ECCommon::ReadPipeline::do_read_op(ReadOp &op)
-{
-  int priority = op.priority;
-  ceph_tid_t tid = op.tid;
-
-  dout(10) << __func__ << ": starting read " << op << dendl;
-
-  map<pg_shard_t, ECSubRead> messages;
-  for (map<hobject_t, read_request_t>::iterator i = op.to_read.begin();
-       i != op.to_read.end();
-       ++i) {
-    bool need_attrs = i->second.want_attrs;
-
-    for (auto j = i->second.need.begin();
-	 j != i->second.need.end();
-	 ++j) {
-      if (need_attrs) {
-	messages[j->first].attrs_to_read.insert(i->first);
-	need_attrs = false;
-      }
-      messages[j->first].subchunks[i->first] = j->second;
-      op.obj_to_source[i->first].insert(j->first);
-      op.source_to_obj[j->first].insert(i->first);
-    }
-    for (list<boost::tuple<uint64_t, uint64_t, uint32_t> >::const_iterator j =
-	   i->second.to_read.begin();
-	 j != i->second.to_read.end();
-	 ++j) {
-      pair<uint64_t, uint64_t> chunk_off_len =
-	sinfo.aligned_offset_len_to_chunk(make_pair(j->get<0>(), j->get<1>()));
-      for (auto k = i->second.need.begin();
-	   k != i->second.need.end();
-	   ++k) {
-	messages[k->first].to_read[i->first].push_back(
-	  boost::make_tuple(
-	    chunk_off_len.first,
-	    chunk_off_len.second,
-	    j->get<2>()));
-      }
-      ceph_assert(!need_attrs);
-    }
-  }
-
-  std::vector<std::pair<int, Message*>> m;
-  m.reserve(messages.size());
-  for (map<pg_shard_t, ECSubRead>::iterator i = messages.begin();
-       i != messages.end();
-       ++i) {
-    op.in_progress.insert(i->first);
-    shard_to_read_map[i->first].insert(op.tid);
-    i->second.tid = tid;
-    MOSDECSubOpRead *msg = new MOSDECSubOpRead;
-    msg->set_priority(priority);
-    msg->pgid = spg_t(
-      get_info().pgid.pgid,
-      i->first.shard);
-    msg->map_epoch = get_osdmap_epoch();
-    msg->min_epoch = get_parent()->get_interval_start_epoch();
-    msg->op = i->second;
-    msg->op.from = get_parent()->whoami_shard();
-    msg->op.tid = tid;
-    if (op.trace) {
-      // initialize a child span for this shard
-      msg->trace.init("ec sub read", nullptr, &op.trace);
-      msg->trace.keyval("shard", i->first.shard.id);
-    }
-    m.push_back(std::make_pair(i->first.osd, msg));
-  }
-  if (!m.empty()) {
-    get_parent()->send_message_osd_cluster(m, get_osdmap_epoch());
-  }
-
-  dout(10) << __func__ << ": started " << op << dendl;
-}
 
 ECUtil::HashInfoRef ECBackend::get_hash_info(
   const hobject_t &hoid, bool create, const map<string,bufferptr,less<>> *attrs)
@@ -1982,320 +1460,6 @@ ECUtil::HashInfoRef ECBackend::get_hash_info(
   return ref;
 }
 
-void ECCommon::RMWPipeline::start_rmw(OpRef op)
-{
-  ceph_assert(op);
-  dout(10) << __func__ << ": " << *op << dendl;
-
-  ceph_assert(!tid_to_op_map.count(op->tid));
-  waiting_state.push_back(*op);
-  tid_to_op_map[op->tid] = std::move(op);
-  check_ops();
-}
-
-bool ECCommon::RMWPipeline::try_state_to_reads()
-{
-  if (waiting_state.empty())
-    return false;
-
-  Op *op = &(waiting_state.front());
-  if (op->requires_rmw() && pipeline_state.cache_invalid()) {
-    ceph_assert(get_parent()->get_pool().allows_ecoverwrites());
-    dout(20) << __func__ << ": blocking " << *op
-	     << " because it requires an rmw and the cache is invalid "
-	     << pipeline_state
-	     << dendl;
-    return false;
-  }
-
-  if (!pipeline_state.caching_enabled()) {
-    op->using_cache = false;
-  } else if (op->invalidates_cache()) {
-    dout(20) << __func__ << ": invalidating cache after this op"
-	     << dendl;
-    pipeline_state.invalidate();
-  }
-
-  waiting_state.pop_front();
-  waiting_reads.push_back(*op);
-
-  if (op->using_cache) {
-    cache.open_write_pin(op->pin);
-
-    extent_set empty;
-    for (auto &&hpair: op->plan.will_write) {
-      auto to_read_plan_iter = op->plan.to_read.find(hpair.first);
-      const extent_set &to_read_plan =
-	to_read_plan_iter == op->plan.to_read.end() ?
-	empty :
-	to_read_plan_iter->second;
-
-      extent_set remote_read = cache.reserve_extents_for_rmw(
-	hpair.first,
-	op->pin,
-	hpair.second,
-	to_read_plan);
-
-      extent_set pending_read = to_read_plan;
-      pending_read.subtract(remote_read);
-
-      if (!remote_read.empty()) {
-	op->remote_read[hpair.first] = std::move(remote_read);
-      }
-      if (!pending_read.empty()) {
-	op->pending_read[hpair.first] = std::move(pending_read);
-      }
-    }
-  } else {
-    op->remote_read = op->plan.to_read;
-  }
-
-  dout(10) << __func__ << ": " << *op << dendl;
-
-  if (!op->remote_read.empty()) {
-    ceph_assert(get_parent()->get_pool().allows_ecoverwrites());
-    objects_read_async_no_cache(
-      op->remote_read,
-      [op, this](map<hobject_t,pair<int, extent_map> > &&results) {
-	for (auto &&i: results) {
-	  op->remote_read_result.emplace(i.first, i.second.second);
-	}
-	check_ops();
-      });
-  }
-
-  return true;
-}
-
-bool ECCommon::RMWPipeline::try_reads_to_commit()
-{
-  if (waiting_reads.empty())
-    return false;
-  Op *op = &(waiting_reads.front());
-  if (op->read_in_progress())
-    return false;
-  waiting_reads.pop_front();
-  waiting_commit.push_back(*op);
-
-  dout(10) << __func__ << ": starting commit on " << *op << dendl;
-  dout(20) << __func__ << ": " << cache << dendl;
-
-  get_parent()->apply_stats(
-    op->hoid,
-    op->delta_stats);
-
-  if (op->using_cache) {
-    for (auto &&hpair: op->pending_read) {
-      op->remote_read_result[hpair.first].insert(
-	cache.get_remaining_extents_for_rmw(
-	  hpair.first,
-	  op->pin,
-	  hpair.second));
-    }
-    op->pending_read.clear();
-  } else {
-    ceph_assert(op->pending_read.empty());
-  }
-
-  map<shard_id_t, ObjectStore::Transaction> trans;
-  for (set<pg_shard_t>::const_iterator i =
-	 get_parent()->get_acting_recovery_backfill_shards().begin();
-       i != get_parent()->get_acting_recovery_backfill_shards().end();
-       ++i) {
-    trans[i->shard];
-  }
-
-  op->trace.event("start ec write");
-
-  map<hobject_t,extent_map> written;
-  op->generate_transactions(
-    ec_impl,
-    get_parent()->get_info().pgid.pgid,
-    sinfo,
-    &written,
-    &trans,
-    get_parent()->get_dpp(),
-    get_osdmap()->require_osd_release);
-
-  dout(20) << __func__ << ": " << cache << dendl;
-  dout(20) << __func__ << ": written: " << written << dendl;
-  dout(20) << __func__ << ": op: " << *op << dendl;
-
-  if (!get_parent()->get_pool().allows_ecoverwrites()) {
-    for (auto &&i: op->log_entries) {
-      if (i.requires_kraken()) {
-	derr << __func__ << ": log entry " << i << " requires kraken"
-	     << " but overwrites are not enabled!" << dendl;
-	ceph_abort();
-      }
-    }
-  }
-
-  map<hobject_t,extent_set> written_set;
-  for (auto &&i: written) {
-    written_set[i.first] = i.second.get_interval_set();
-  }
-  dout(20) << __func__ << ": written_set: " << written_set << dendl;
-  ceph_assert(written_set == op->plan.will_write);
-
-  if (op->using_cache) {
-    for (auto &&hpair: written) {
-      dout(20) << __func__ << ": " << hpair << dendl;
-      cache.present_rmw_update(hpair.first, op->pin, hpair.second);
-    }
-  }
-  op->remote_read.clear();
-  op->remote_read_result.clear();
-
-  ObjectStore::Transaction empty;
-  bool should_write_local = false;
-  ECSubWrite local_write_op;
-  std::vector<std::pair<int, Message*>> messages;
-  messages.reserve(get_parent()->get_acting_recovery_backfill_shards().size());
-  set<pg_shard_t> backfill_shards = get_parent()->get_backfill_shards();
-  for (set<pg_shard_t>::const_iterator i =
-	 get_parent()->get_acting_recovery_backfill_shards().begin();
-       i != get_parent()->get_acting_recovery_backfill_shards().end();
-       ++i) {
-    op->pending_apply.insert(*i);
-    op->pending_commit.insert(*i);
-    map<shard_id_t, ObjectStore::Transaction>::iterator iter =
-      trans.find(i->shard);
-    ceph_assert(iter != trans.end());
-    bool should_send = get_parent()->should_send_op(*i, op->hoid);
-    const pg_stat_t &stats =
-      (should_send || !backfill_shards.count(*i)) ?
-      get_info().stats :
-      get_parent()->get_shard_info().find(*i)->second.stats;
-
-    ECSubWrite sop(
-      get_parent()->whoami_shard(),
-      op->tid,
-      op->reqid,
-      op->hoid,
-      stats,
-      should_send ? iter->second : empty,
-      op->version,
-      op->trim_to,
-      op->roll_forward_to,
-      op->log_entries,
-      op->updated_hit_set_history,
-      op->temp_added,
-      op->temp_cleared,
-      !should_send);
-
-    ZTracer::Trace trace;
-    if (op->trace) {
-      // initialize a child span for this shard
-      trace.init("ec sub write", nullptr, &op->trace);
-      trace.keyval("shard", i->shard.id);
-    }
-
-    if (*i == get_parent()->whoami_shard()) {
-      should_write_local = true;
-      local_write_op.claim(sop);
-    } else {
-      MOSDECSubOpWrite *r = new MOSDECSubOpWrite(sop);
-      r->pgid = spg_t(get_parent()->primary_spg_t().pgid, i->shard);
-      r->map_epoch = get_osdmap_epoch();
-      r->min_epoch = get_parent()->get_interval_start_epoch();
-      r->trace = trace;
-      messages.push_back(std::make_pair(i->osd, r));
-    }
-  }
-
-  if (!messages.empty()) {
-    get_parent()->send_message_osd_cluster(messages, get_osdmap_epoch());
-  }
-
-  if (should_write_local) {
-    handle_sub_write(
-      get_parent()->whoami_shard(),
-      op->client_op,
-      local_write_op,
-      op->trace);
-  }
-
-  for (auto i = op->on_write.begin();
-       i != op->on_write.end();
-       op->on_write.erase(i++)) {
-    (*i)();
-  }
-
-  return true;
-}
-
-struct ECDummyOp : ECCommon::RMWPipeline::Op {
-  void generate_transactions(
-      ceph::ErasureCodeInterfaceRef &ecimpl,
-      pg_t pgid,
-      const ECUtil::stripe_info_t &sinfo,
-      std::map<hobject_t,extent_map> *written,
-      std::map<shard_id_t, ObjectStore::Transaction> *transactions,
-      DoutPrefixProvider *dpp,
-      const ceph_release_t require_osd_release) final
-  {
-    // NOP, as -- in constrast to ECClassicalOp -- there is no
-    // transaction involved
-  }
-};
-
-bool ECCommon::RMWPipeline::try_finish_rmw()
-{
-  if (waiting_commit.empty())
-    return false;
-  Op *op = &(waiting_commit.front());
-  if (op->write_in_progress())
-    return false;
-  waiting_commit.pop_front();
-
-  dout(10) << __func__ << ": " << *op << dendl;
-  dout(20) << __func__ << ": " << cache << dendl;
-
-  if (op->roll_forward_to > completed_to)
-    completed_to = op->roll_forward_to;
-  if (op->version > committed_to)
-    committed_to = op->version;
-
-  if (get_osdmap()->require_osd_release >= ceph_release_t::kraken) {
-    if (op->version > get_parent()->get_log().get_can_rollback_to() &&
-	waiting_reads.empty() &&
-	waiting_commit.empty()) {
-      // submit a dummy, transaction-empty op to kick the rollforward
-      auto tid = get_parent()->get_tid();
-      auto nop = std::make_unique<ECDummyOp>();
-      nop->hoid = op->hoid;
-      nop->trim_to = op->trim_to;
-      nop->roll_forward_to = op->version;
-      nop->tid = tid;
-      nop->reqid = op->reqid;
-      waiting_reads.push_back(*nop);
-      tid_to_op_map[tid] = std::move(nop);
-    }
-  }
-
-  if (op->using_cache) {
-    cache.release_write_pin(op->pin);
-  }
-  tid_to_op_map.erase(op->tid);
-
-  if (waiting_reads.empty() &&
-      waiting_commit.empty()) {
-    pipeline_state.clear();
-    dout(20) << __func__ << ": clearing pipeline_state "
-	     << pipeline_state
-	     << dendl;
-  }
-  return true;
-}
-
-void ECCommon::RMWPipeline::check_ops()
-{
-  while (try_state_to_reads() ||
-	 try_reads_to_commit() ||
-	 try_finish_rmw());
-}
-
 int ECBackend::objects_read_sync(
   const hobject_t &hoid,
   uint64_t off,
diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
new file mode 100644
index 000000000000..373cc4079d3e
--- /dev/null
+++ b/src/osd/ECCommon.cc
@@ -0,0 +1,907 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank Storage, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <iostream>
+#include <sstream>
+
+#include "ECCommon.h"
+#include "messages/MOSDPGPush.h"
+#include "messages/MOSDPGPushReply.h"
+#include "messages/MOSDECSubOpWrite.h"
+#include "messages/MOSDECSubOpWriteReply.h"
+#include "messages/MOSDECSubOpRead.h"
+#include "messages/MOSDECSubOpReadReply.h"
+#include "ECMsgTypes.h"
+#include "PGLog.h"
+
+#include "osd_tracer.h"
+
+#define dout_context cct
+#define dout_subsys ceph_subsys_osd
+#define DOUT_PREFIX_ARGS this
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, this)
+
+using std::dec;
+using std::hex;
+using std::less;
+using std::list;
+using std::make_pair;
+using std::map;
+using std::pair;
+using std::ostream;
+using std::set;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using ceph::bufferhash;
+using ceph::bufferlist;
+using ceph::bufferptr;
+using ceph::ErasureCodeInterfaceRef;
+using ceph::Formatter;
+
+static ostream& _prefix(std::ostream *_dout, ECCommon::RMWPipeline *rmw_pipeline) {
+  return rmw_pipeline->get_parent()->gen_dbg_prefix(*_dout);
+}
+static ostream& _prefix(std::ostream *_dout, ECCommon::ReadPipeline *read_pipeline) {
+  return read_pipeline->get_parent()->gen_dbg_prefix(*_dout);
+}
+
+ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::pipeline_state_t &rhs) {
+  switch (rhs.pipeline_state) {
+  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_VALID:
+    return lhs << "CACHE_VALID";
+  case ECCommon::RMWPipeline::pipeline_state_t::CACHE_INVALID:
+    return lhs << "CACHE_INVALID";
+  default:
+    ceph_abort_msg("invalid pipeline state");
+  }
+  return lhs; // unreachable
+}
+
+ostream &operator<<(ostream &lhs, const ECCommon::read_request_t &rhs)
+{
+  return lhs << "read_request_t(to_read=[" << rhs.to_read << "]"
+	     << ", need=" << rhs.need
+	     << ", want_attrs=" << rhs.want_attrs
+	     << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECCommon::read_result_t &rhs)
+{
+  lhs << "read_result_t(r=" << rhs.r
+      << ", errors=" << rhs.errors;
+  if (rhs.attrs) {
+    lhs << ", attrs=" << *(rhs.attrs);
+  } else {
+    lhs << ", noattrs";
+  }
+  return lhs << ", returned=" << rhs.returned << ")";
+}
+
+ostream &operator<<(ostream &lhs, const ECCommon::ReadOp &rhs)
+{
+  lhs << "ReadOp(tid=" << rhs.tid;
+#ifndef WITH_SEASTAR
+  if (rhs.op && rhs.op->get_req()) {
+    lhs << ", op=";
+    rhs.op->get_req()->print(lhs);
+  }
+#endif
+  return lhs << ", to_read=" << rhs.to_read
+	     << ", complete=" << rhs.complete
+	     << ", priority=" << rhs.priority
+	     << ", obj_to_source=" << rhs.obj_to_source
+	     << ", source_to_obj=" << rhs.source_to_obj
+	     << ", in_progress=" << rhs.in_progress << ")";
+}
+
+void ECCommon::ReadOp::dump(Formatter *f) const
+{
+  f->dump_unsigned("tid", tid);
+#ifndef WITH_SEASTAR
+  if (op && op->get_req()) {
+    f->dump_stream("op") << *(op->get_req());
+  }
+#endif
+  f->dump_stream("to_read") << to_read;
+  f->dump_stream("complete") << complete;
+  f->dump_int("priority", priority);
+  f->dump_stream("obj_to_source") << obj_to_source;
+  f->dump_stream("source_to_obj") << source_to_obj;
+  f->dump_stream("in_progress") << in_progress;
+}
+
+ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::Op &rhs)
+{
+  lhs << "Op(" << rhs.hoid
+      << " v=" << rhs.version
+      << " tt=" << rhs.trim_to
+      << " tid=" << rhs.tid
+      << " reqid=" << rhs.reqid;
+#ifndef WITH_SEASTAR
+  if (rhs.client_op && rhs.client_op->get_req()) {
+    lhs << " client_op=";
+    rhs.client_op->get_req()->print(lhs);
+  }
+#endif
+  lhs << " roll_forward_to=" << rhs.roll_forward_to
+      << " temp_added=" << rhs.temp_added
+      << " temp_cleared=" << rhs.temp_cleared
+      << " pending_read=" << rhs.pending_read
+      << " remote_read=" << rhs.remote_read
+      << " remote_read_result=" << rhs.remote_read_result
+      << " pending_apply=" << rhs.pending_apply
+      << " pending_commit=" << rhs.pending_commit
+      << " plan.to_read=" << rhs.plan.to_read
+      << " plan.will_write=" << rhs.plan.will_write
+      << ")";
+  return lhs;
+}
+
+void ECCommon::ReadPipeline::complete_read_op(ReadOp &rop)
+{
+  map<hobject_t, read_request_t>::iterator reqiter =
+    rop.to_read.begin();
+  map<hobject_t, read_result_t>::iterator resiter =
+    rop.complete.begin();
+  ceph_assert(rop.to_read.size() == rop.complete.size());
+  for (; reqiter != rop.to_read.end(); ++reqiter, ++resiter) {
+    rop.on_complete->finish_single_request(
+      reqiter->first,
+      resiter->second,
+      reqiter->second.to_read);
+  }
+  ceph_assert(rop.on_complete);
+  std::move(*rop.on_complete).finish(rop.priority);
+  rop.on_complete = nullptr;
+  // if the read op is over. clean all the data of this tid.
+  for (set<pg_shard_t>::iterator iter = rop.in_progress.begin();
+    iter != rop.in_progress.end();
+    iter++) {
+    shard_to_read_map[*iter].erase(rop.tid);
+  }
+  rop.in_progress.clear();
+  tid_to_read_map.erase(rop.tid);
+}
+
+struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
+  ECCommon::ReadPipeline& read_pipeline;
+  ceph_tid_t tid;
+  FinishReadOp(ECCommon::ReadPipeline& read_pipeline, ceph_tid_t tid)
+    : read_pipeline(read_pipeline), tid(tid) {}
+  void finish(ThreadPool::TPHandle&) override {
+    auto ropiter = read_pipeline.tid_to_read_map.find(tid);
+    ceph_assert(ropiter != read_pipeline.tid_to_read_map.end());
+    read_pipeline.complete_read_op(ropiter->second);
+  }
+};
+
+template <class F>
+void ECCommon::ReadPipeline::filter_read_op(
+  const OSDMapRef& osdmap,
+  ReadOp &op,
+  F&& on_erase)
+{
+  set<hobject_t> to_cancel;
+  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ++i) {
+    if (osdmap->is_down(i->first.osd)) {
+      to_cancel.insert(i->second.begin(), i->second.end());
+      op.in_progress.erase(i->first);
+      continue;
+    }
+  }
+
+  if (to_cancel.empty())
+    return;
+
+  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ) {
+    for (set<hobject_t>::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ) {
+      if (to_cancel.count(*j))
+	i->second.erase(j++);
+      else
+	++j;
+    }
+    if (i->second.empty()) {
+      op.source_to_obj.erase(i++);
+    } else {
+      ceph_assert(!osdmap->is_down(i->first.osd));
+      ++i;
+    }
+  }
+
+  for (set<hobject_t>::iterator i = to_cancel.begin();
+       i != to_cancel.end();
+       ++i) {
+    get_parent()->cancel_pull(*i);
+
+    ceph_assert(op.to_read.count(*i));
+    read_request_t &req = op.to_read.find(*i)->second;
+    dout(10) << __func__ << ": canceling " << req
+	     << "  for obj " << *i << dendl;
+    op.to_read.erase(*i);
+    op.complete.erase(*i);
+    on_erase(*i);
+  }
+
+  if (op.in_progress.empty()) {
+    /* This case is odd.  filter_read_op gets called while processing
+     * an OSDMap.  Normal, non-recovery reads only happen from acting
+     * set osds.  For this op to have had a read source go down and
+     * there not be an interval change, it must be part of a pull during
+     * log-based recovery.
+     *
+     * This callback delays calling complete_read_op until later to avoid
+     * dealing with recovery while handling an OSDMap.  We assign a
+     * cost here of 1 because:
+     * 1) This should be very rare, and the operation itself was already
+     *    throttled.
+     * 2) It shouldn't result in IO, rather it should result in restarting
+     *    the pull on the affected objects and pushes from in-memory buffers
+     *    on any now complete unaffected objects.
+     */
+#ifndef WITH_SEASTAR
+    get_parent()->schedule_recovery_work(
+      get_parent()->bless_unlocked_gencontext(
+        new FinishReadOp(*this, op.tid)),
+      1);
+#else
+    // TODO
+    ceph_abort_msg("not yet implemented");
+#endif
+  }
+}
+
+template <class F>
+void ECCommon::ReadPipeline::check_recovery_sources(
+  const OSDMapRef& osdmap,
+  F&& on_erase)
+{
+  set<ceph_tid_t> tids_to_filter;
+  for (map<pg_shard_t, set<ceph_tid_t> >::iterator 
+       i = shard_to_read_map.begin();
+       i != shard_to_read_map.end();
+       ) {
+    if (osdmap->is_down(i->first.osd)) {
+      tids_to_filter.insert(i->second.begin(), i->second.end());
+      shard_to_read_map.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+  for (set<ceph_tid_t>::iterator i = tids_to_filter.begin();
+       i != tids_to_filter.end();
+       ++i) {
+    map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
+    ceph_assert(j != tid_to_read_map.end());
+    filter_read_op(osdmap, j->second, on_erase);
+  }
+}
+
+void ECCommon::ReadPipeline::on_change()
+{
+  for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
+       i != tid_to_read_map.end();
+       ++i) {
+    dout(10) << __func__ << ": cancelling " << i->second << dendl;
+  }
+  tid_to_read_map.clear();
+  shard_to_read_map.clear();
+  in_progress_client_reads.clear();
+}
+
+void ECCommon::RMWPipeline::on_change()
+{
+  dout(10) << __func__ << dendl;
+
+  completed_to = eversion_t();
+  committed_to = eversion_t();
+  pipeline_state.clear();
+  waiting_reads.clear();
+  waiting_state.clear();
+  waiting_commit.clear();
+  for (auto &&op: tid_to_op_map) {
+    cache.release_write_pin(op.second->pin);
+  }
+  tid_to_op_map.clear();
+}
+
+void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
+  if (!waiting_state.empty()) {
+    waiting_state.back().on_write.emplace_back(std::move(cb));
+  } else if (!waiting_reads.empty()) {
+    waiting_reads.back().on_write.emplace_back(std::move(cb));
+  } else {
+    // Nothing earlier in the pipeline, just call it
+    cb();
+  }
+}
+
+void ECCommon::ReadPipeline::get_all_avail_shards(
+  const hobject_t &hoid,
+  const set<pg_shard_t> &error_shards,
+  set<int> &have,
+  map<shard_id_t, pg_shard_t> &shards,
+  bool for_recovery)
+{
+  for (set<pg_shard_t>::const_iterator i =
+	 get_parent()->get_acting_shards().begin();
+       i != get_parent()->get_acting_shards().end();
+       ++i) {
+    dout(10) << __func__ << ": checking acting " << *i << dendl;
+    const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+    if (error_shards.find(*i) != error_shards.end())
+      continue;
+    if (!missing.is_missing(hoid)) {
+      ceph_assert(!have.count(i->shard));
+      have.insert(i->shard);
+      ceph_assert(!shards.count(i->shard));
+      shards.insert(make_pair(i->shard, *i));
+    }
+  }
+
+  if (for_recovery) {
+    for (set<pg_shard_t>::const_iterator i =
+	   get_parent()->get_backfill_shards().begin();
+	 i != get_parent()->get_backfill_shards().end();
+	 ++i) {
+      if (error_shards.find(*i) != error_shards.end())
+	continue;
+      if (have.count(i->shard)) {
+	ceph_assert(shards.count(i->shard));
+	continue;
+      }
+      dout(10) << __func__ << ": checking backfill " << *i << dendl;
+      ceph_assert(!shards.count(i->shard));
+      const pg_info_t &info = get_parent()->get_shard_info(*i);
+      const pg_missing_t &missing = get_parent()->get_shard_missing(*i);
+      if (hoid < info.last_backfill &&
+	  !missing.is_missing(hoid)) {
+	have.insert(i->shard);
+	shards.insert(make_pair(i->shard, *i));
+      }
+    }
+
+    map<hobject_t, set<pg_shard_t>>::const_iterator miter =
+      get_parent()->get_missing_loc_shards().find(hoid);
+    if (miter != get_parent()->get_missing_loc_shards().end()) {
+      for (set<pg_shard_t>::iterator i = miter->second.begin();
+	   i != miter->second.end();
+	   ++i) {
+	dout(10) << __func__ << ": checking missing_loc " << *i << dendl;
+	auto m = get_parent()->maybe_get_shard_missing(*i);
+	if (m) {
+	  ceph_assert(!(*m).is_missing(hoid));
+	}
+	if (error_shards.find(*i) != error_shards.end())
+	  continue;
+	have.insert(i->shard);
+	shards.insert(make_pair(i->shard, *i));
+      }
+    }
+  }
+}
+
+int ECCommon::ReadPipeline::get_min_avail_to_read_shards(
+  const hobject_t &hoid,
+  const set<int> &want,
+  bool for_recovery,
+  bool do_redundant_reads,
+  map<pg_shard_t, vector<pair<int, int>>> *to_read)
+{
+  // Make sure we don't do redundant reads for recovery
+  ceph_assert(!for_recovery || !do_redundant_reads);
+
+  set<int> have;
+  map<shard_id_t, pg_shard_t> shards;
+  set<pg_shard_t> error_shards;
+
+  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
+
+  map<int, vector<pair<int, int>>> need;
+  int r = ec_impl->minimum_to_decode(want, have, &need);
+  if (r < 0)
+    return r;
+
+  if (do_redundant_reads) {
+      vector<pair<int, int>> subchunks_list;
+      subchunks_list.push_back(make_pair(0, ec_impl->get_sub_chunk_count()));
+      for (auto &&i: have) {
+        need[i] = subchunks_list;
+      }
+  } 
+
+  if (!to_read)
+    return 0;
+
+  for (auto &&i:need) {
+    ceph_assert(shards.count(shard_id_t(i.first)));
+    to_read->insert(make_pair(shards[shard_id_t(i.first)], i.second));
+  }
+  return 0;
+}
+
+int ECCommon::ReadPipeline::get_remaining_shards(
+  const hobject_t &hoid,
+  const set<int> &avail,
+  const set<int> &want,
+  const read_result_t &result,
+  map<pg_shard_t, vector<pair<int, int>>> *to_read,
+  bool for_recovery)
+{
+  ceph_assert(to_read);
+
+  set<int> have;
+  map<shard_id_t, pg_shard_t> shards;
+  set<pg_shard_t> error_shards;
+  for (auto &p : result.errors) {
+    error_shards.insert(p.first);
+  }
+
+  get_all_avail_shards(hoid, error_shards, have, shards, for_recovery);
+
+  map<int, vector<pair<int, int>>> need;
+  int r = ec_impl->minimum_to_decode(want, have, &need);
+  if (r < 0) {
+    dout(0) << __func__ << " not enough shards left to try for " << hoid
+	    << " read result was " << result << dendl;
+    return -EIO;
+  }
+
+  set<int> shards_left;
+  for (auto p : need) {
+    if (avail.find(p.first) == avail.end()) {
+      shards_left.insert(p.first);
+    }
+  }
+
+  vector<pair<int, int>> subchunks;
+  subchunks.push_back(make_pair(0, ec_impl->get_sub_chunk_count()));
+  for (set<int>::iterator i = shards_left.begin();
+       i != shards_left.end();
+       ++i) {
+    ceph_assert(shards.count(shard_id_t(*i)));
+    ceph_assert(avail.find(*i) == avail.end());
+    to_read->insert(make_pair(shards[shard_id_t(*i)], subchunks));
+  }
+  return 0;
+}
+
+void ECCommon::ReadPipeline::start_read_op(
+  int priority,
+  map<hobject_t, set<int>> &want_to_read,
+  map<hobject_t, read_request_t> &to_read,
+  OpRequestRef _op,
+  bool do_redundant_reads,
+  bool for_recovery,
+  std::unique_ptr<ECCommon::ReadCompleter> on_complete)
+{
+  ceph_tid_t tid = get_parent()->get_tid();
+  ceph_assert(!tid_to_read_map.count(tid));
+  auto &op = tid_to_read_map.emplace(
+    tid,
+    ReadOp(
+      priority,
+      tid,
+      do_redundant_reads,
+      for_recovery,
+      std::move(on_complete),
+      _op,
+      std::move(want_to_read),
+      std::move(to_read))).first->second;
+  dout(10) << __func__ << ": starting " << op << dendl;
+  if (_op) {
+#ifndef WITH_SEASTAR
+    op.trace = _op->pg_trace;
+#endif
+    op.trace.event("start ec read");
+  }
+  do_read_op(op);
+}
+
+void ECCommon::ReadPipeline::do_read_op(ReadOp &op)
+{
+  int priority = op.priority;
+  ceph_tid_t tid = op.tid;
+
+  dout(10) << __func__ << ": starting read " << op << dendl;
+
+  map<pg_shard_t, ECSubRead> messages;
+  for (map<hobject_t, read_request_t>::iterator i = op.to_read.begin();
+       i != op.to_read.end();
+       ++i) {
+    bool need_attrs = i->second.want_attrs;
+
+    for (auto j = i->second.need.begin();
+	 j != i->second.need.end();
+	 ++j) {
+      if (need_attrs) {
+	messages[j->first].attrs_to_read.insert(i->first);
+	need_attrs = false;
+      }
+      messages[j->first].subchunks[i->first] = j->second;
+      op.obj_to_source[i->first].insert(j->first);
+      op.source_to_obj[j->first].insert(i->first);
+    }
+    for (list<boost::tuple<uint64_t, uint64_t, uint32_t> >::const_iterator j =
+	   i->second.to_read.begin();
+	 j != i->second.to_read.end();
+	 ++j) {
+      pair<uint64_t, uint64_t> chunk_off_len =
+	sinfo.aligned_offset_len_to_chunk(make_pair(j->get<0>(), j->get<1>()));
+      for (auto k = i->second.need.begin();
+	   k != i->second.need.end();
+	   ++k) {
+	messages[k->first].to_read[i->first].push_back(
+	  boost::make_tuple(
+	    chunk_off_len.first,
+	    chunk_off_len.second,
+	    j->get<2>()));
+      }
+      ceph_assert(!need_attrs);
+    }
+  }
+
+  std::vector<std::pair<int, Message*>> m;
+  m.reserve(messages.size());
+  for (map<pg_shard_t, ECSubRead>::iterator i = messages.begin();
+       i != messages.end();
+       ++i) {
+    op.in_progress.insert(i->first);
+    shard_to_read_map[i->first].insert(op.tid);
+    i->second.tid = tid;
+    MOSDECSubOpRead *msg = new MOSDECSubOpRead;
+    msg->set_priority(priority);
+    msg->pgid = spg_t(
+      get_info().pgid.pgid,
+      i->first.shard);
+    msg->map_epoch = get_osdmap_epoch();
+    msg->min_epoch = get_parent()->get_interval_start_epoch();
+    msg->op = i->second;
+    msg->op.from = get_parent()->whoami_shard();
+    msg->op.tid = tid;
+    if (op.trace) {
+      // initialize a child span for this shard
+      msg->trace.init("ec sub read", nullptr, &op.trace);
+      msg->trace.keyval("shard", i->first.shard.id);
+    }
+    m.push_back(std::make_pair(i->first.osd, msg));
+  }
+  if (!m.empty()) {
+    get_parent()->send_message_osd_cluster(m, get_osdmap_epoch());
+  }
+
+  dout(10) << __func__ << ": started " << op << dendl;
+}
+
+void ECCommon::RMWPipeline::start_rmw(OpRef op)
+{
+  ceph_assert(op);
+  dout(10) << __func__ << ": " << *op << dendl;
+
+  ceph_assert(!tid_to_op_map.count(op->tid));
+  waiting_state.push_back(*op);
+  tid_to_op_map[op->tid] = std::move(op);
+  check_ops();
+}
+
+bool ECCommon::RMWPipeline::try_state_to_reads()
+{
+  if (waiting_state.empty())
+    return false;
+
+  Op *op = &(waiting_state.front());
+  if (op->requires_rmw() && pipeline_state.cache_invalid()) {
+    ceph_assert(get_parent()->get_pool().allows_ecoverwrites());
+    dout(20) << __func__ << ": blocking " << *op
+	     << " because it requires an rmw and the cache is invalid "
+	     << pipeline_state
+	     << dendl;
+    return false;
+  }
+
+  if (!pipeline_state.caching_enabled()) {
+    op->using_cache = false;
+  } else if (op->invalidates_cache()) {
+    dout(20) << __func__ << ": invalidating cache after this op"
+	     << dendl;
+    pipeline_state.invalidate();
+  }
+
+  waiting_state.pop_front();
+  waiting_reads.push_back(*op);
+
+  if (op->using_cache) {
+    cache.open_write_pin(op->pin);
+
+    extent_set empty;
+    for (auto &&hpair: op->plan.will_write) {
+      auto to_read_plan_iter = op->plan.to_read.find(hpair.first);
+      const extent_set &to_read_plan =
+	to_read_plan_iter == op->plan.to_read.end() ?
+	empty :
+	to_read_plan_iter->second;
+
+      extent_set remote_read = cache.reserve_extents_for_rmw(
+	hpair.first,
+	op->pin,
+	hpair.second,
+	to_read_plan);
+
+      extent_set pending_read = to_read_plan;
+      pending_read.subtract(remote_read);
+
+      if (!remote_read.empty()) {
+	op->remote_read[hpair.first] = std::move(remote_read);
+      }
+      if (!pending_read.empty()) {
+	op->pending_read[hpair.first] = std::move(pending_read);
+      }
+    }
+  } else {
+    op->remote_read = op->plan.to_read;
+  }
+
+  dout(10) << __func__ << ": " << *op << dendl;
+
+  if (!op->remote_read.empty()) {
+    ceph_assert(get_parent()->get_pool().allows_ecoverwrites());
+    objects_read_async_no_cache(
+      op->remote_read,
+      [op, this](map<hobject_t,pair<int, extent_map> > &&results) {
+	for (auto &&i: results) {
+	  op->remote_read_result.emplace(i.first, i.second.second);
+	}
+	check_ops();
+      });
+  }
+
+  return true;
+}
+
+bool ECCommon::RMWPipeline::try_reads_to_commit()
+{
+  if (waiting_reads.empty())
+    return false;
+  Op *op = &(waiting_reads.front());
+  if (op->read_in_progress())
+    return false;
+  waiting_reads.pop_front();
+  waiting_commit.push_back(*op);
+
+  dout(10) << __func__ << ": starting commit on " << *op << dendl;
+  dout(20) << __func__ << ": " << cache << dendl;
+
+  get_parent()->apply_stats(
+    op->hoid,
+    op->delta_stats);
+
+  if (op->using_cache) {
+    for (auto &&hpair: op->pending_read) {
+      op->remote_read_result[hpair.first].insert(
+	cache.get_remaining_extents_for_rmw(
+	  hpair.first,
+	  op->pin,
+	  hpair.second));
+    }
+    op->pending_read.clear();
+  } else {
+    ceph_assert(op->pending_read.empty());
+  }
+
+  map<shard_id_t, ObjectStore::Transaction> trans;
+  for (set<pg_shard_t>::const_iterator i =
+	 get_parent()->get_acting_recovery_backfill_shards().begin();
+       i != get_parent()->get_acting_recovery_backfill_shards().end();
+       ++i) {
+    trans[i->shard];
+  }
+
+  op->trace.event("start ec write");
+
+  map<hobject_t,extent_map> written;
+  op->generate_transactions(
+    ec_impl,
+    get_parent()->get_info().pgid.pgid,
+    sinfo,
+    &written,
+    &trans,
+    get_parent()->get_dpp(),
+    get_osdmap()->require_osd_release);
+
+  dout(20) << __func__ << ": " << cache << dendl;
+  dout(20) << __func__ << ": written: " << written << dendl;
+  dout(20) << __func__ << ": op: " << *op << dendl;
+
+  if (!get_parent()->get_pool().allows_ecoverwrites()) {
+    for (auto &&i: op->log_entries) {
+      if (i.requires_kraken()) {
+	derr << __func__ << ": log entry " << i << " requires kraken"
+	     << " but overwrites are not enabled!" << dendl;
+	ceph_abort();
+      }
+    }
+  }
+
+  map<hobject_t,extent_set> written_set;
+  for (auto &&i: written) {
+    written_set[i.first] = i.second.get_interval_set();
+  }
+  dout(20) << __func__ << ": written_set: " << written_set << dendl;
+  ceph_assert(written_set == op->plan.will_write);
+
+  if (op->using_cache) {
+    for (auto &&hpair: written) {
+      dout(20) << __func__ << ": " << hpair << dendl;
+      cache.present_rmw_update(hpair.first, op->pin, hpair.second);
+    }
+  }
+  op->remote_read.clear();
+  op->remote_read_result.clear();
+
+  ObjectStore::Transaction empty;
+  bool should_write_local = false;
+  ECSubWrite local_write_op;
+  std::vector<std::pair<int, Message*>> messages;
+  messages.reserve(get_parent()->get_acting_recovery_backfill_shards().size());
+  set<pg_shard_t> backfill_shards = get_parent()->get_backfill_shards();
+  for (set<pg_shard_t>::const_iterator i =
+	 get_parent()->get_acting_recovery_backfill_shards().begin();
+       i != get_parent()->get_acting_recovery_backfill_shards().end();
+       ++i) {
+    op->pending_apply.insert(*i);
+    op->pending_commit.insert(*i);
+    map<shard_id_t, ObjectStore::Transaction>::iterator iter =
+      trans.find(i->shard);
+    ceph_assert(iter != trans.end());
+    bool should_send = get_parent()->should_send_op(*i, op->hoid);
+    const pg_stat_t &stats =
+      (should_send || !backfill_shards.count(*i)) ?
+      get_info().stats :
+      get_parent()->get_shard_info().find(*i)->second.stats;
+
+    ECSubWrite sop(
+      get_parent()->whoami_shard(),
+      op->tid,
+      op->reqid,
+      op->hoid,
+      stats,
+      should_send ? iter->second : empty,
+      op->version,
+      op->trim_to,
+      op->roll_forward_to,
+      op->log_entries,
+      op->updated_hit_set_history,
+      op->temp_added,
+      op->temp_cleared,
+      !should_send);
+
+    ZTracer::Trace trace;
+    if (op->trace) {
+      // initialize a child span for this shard
+      trace.init("ec sub write", nullptr, &op->trace);
+      trace.keyval("shard", i->shard.id);
+    }
+
+    if (*i == get_parent()->whoami_shard()) {
+      should_write_local = true;
+      local_write_op.claim(sop);
+    } else {
+      MOSDECSubOpWrite *r = new MOSDECSubOpWrite(sop);
+      r->pgid = spg_t(get_parent()->primary_spg_t().pgid, i->shard);
+      r->map_epoch = get_osdmap_epoch();
+      r->min_epoch = get_parent()->get_interval_start_epoch();
+      r->trace = trace;
+      messages.push_back(std::make_pair(i->osd, r));
+    }
+  }
+
+  if (!messages.empty()) {
+    get_parent()->send_message_osd_cluster(messages, get_osdmap_epoch());
+  }
+
+  if (should_write_local) {
+    handle_sub_write(
+      get_parent()->whoami_shard(),
+      op->client_op,
+      local_write_op,
+      op->trace);
+  }
+
+  for (auto i = op->on_write.begin();
+       i != op->on_write.end();
+       op->on_write.erase(i++)) {
+    (*i)();
+  }
+
+  return true;
+}
+
+struct ECDummyOp : ECCommon::RMWPipeline::Op {
+  void generate_transactions(
+      ceph::ErasureCodeInterfaceRef &ecimpl,
+      pg_t pgid,
+      const ECUtil::stripe_info_t &sinfo,
+      std::map<hobject_t,extent_map> *written,
+      std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+      DoutPrefixProvider *dpp,
+      const ceph_release_t require_osd_release) final
+  {
+    // NOP, as -- in constrast to ECClassicalOp -- there is no
+    // transaction involved
+  }
+};
+
+bool ECCommon::RMWPipeline::try_finish_rmw()
+{
+  if (waiting_commit.empty())
+    return false;
+  Op *op = &(waiting_commit.front());
+  if (op->write_in_progress())
+    return false;
+  waiting_commit.pop_front();
+
+  dout(10) << __func__ << ": " << *op << dendl;
+  dout(20) << __func__ << ": " << cache << dendl;
+
+  if (op->roll_forward_to > completed_to)
+    completed_to = op->roll_forward_to;
+  if (op->version > committed_to)
+    committed_to = op->version;
+
+  if (get_osdmap()->require_osd_release >= ceph_release_t::kraken) {
+    if (op->version > get_parent()->get_log().get_can_rollback_to() &&
+	waiting_reads.empty() &&
+	waiting_commit.empty()) {
+      // submit a dummy, transaction-empty op to kick the rollforward
+      auto tid = get_parent()->get_tid();
+      auto nop = std::make_unique<ECDummyOp>();
+      nop->hoid = op->hoid;
+      nop->trim_to = op->trim_to;
+      nop->roll_forward_to = op->version;
+      nop->tid = tid;
+      nop->reqid = op->reqid;
+      waiting_reads.push_back(*nop);
+      tid_to_op_map[tid] = std::move(nop);
+    }
+  }
+
+  if (op->using_cache) {
+    cache.release_write_pin(op->pin);
+  }
+  tid_to_op_map.erase(op->tid);
+
+  if (waiting_reads.empty() &&
+      waiting_commit.empty()) {
+    pipeline_state.clear();
+    dout(20) << __func__ << ": clearing pipeline_state "
+	     << pipeline_state
+	     << dendl;
+  }
+  return true;
+}
+
+void ECCommon::RMWPipeline::check_ops()
+{
+  while (try_state_to_reads() ||
+	 try_reads_to_commit() ||
+	 try_finish_rmw());
+}
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index a7585ea13ddd..46f8bb14772b 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -130,7 +130,7 @@ struct ECListener {
     const eversion_t &roll_forward_to,
     const eversion_t &min_last_complete_ondisk,
     bool transaction_applied,
-    ObjectStore::Transaction &t,
+    ceph::os::Transaction &t,
     bool async = false) = 0;
   virtual void op_applied(
     const eversion_t &applied_version) = 0;
@@ -583,3 +583,14 @@ struct ECCommon {
   };
 };
 
+std::ostream &operator<<(std::ostream &lhs,
+			 const ECCommon::RMWPipeline::pipeline_state_t &rhs);
+std::ostream &operator<<(std::ostream &lhs,
+			 const ECCommon::read_request_t &rhs);
+std::ostream &operator<<(std::ostream &lhs,
+			 const ECCommon::read_result_t &rhs);
+std::ostream &operator<<(std::ostream &lhs,
+			 const ECCommon::ReadOp &rhs);
+std::ostream &operator<<(std::ostream &lhs,
+			 const ECCommon::RMWPipeline::Op &rhs);
+

From 80b72bc2efb758a4d5da6baad31e04a302182e8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 28 Sep 2023 14:51:56 +0200
Subject: [PATCH 1321/2492] osd: consolidate RMWPipeline code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit just moved the code to have all methods of
`RMWPipeline` groupped in the file.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECCommon.cc | 54 ++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index 373cc4079d3e..7ed15a061f81 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -308,33 +308,6 @@ void ECCommon::ReadPipeline::on_change()
   in_progress_client_reads.clear();
 }
 
-void ECCommon::RMWPipeline::on_change()
-{
-  dout(10) << __func__ << dendl;
-
-  completed_to = eversion_t();
-  committed_to = eversion_t();
-  pipeline_state.clear();
-  waiting_reads.clear();
-  waiting_state.clear();
-  waiting_commit.clear();
-  for (auto &&op: tid_to_op_map) {
-    cache.release_write_pin(op.second->pin);
-  }
-  tid_to_op_map.clear();
-}
-
-void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
-  if (!waiting_state.empty()) {
-    waiting_state.back().on_write.emplace_back(std::move(cb));
-  } else if (!waiting_reads.empty()) {
-    waiting_reads.back().on_write.emplace_back(std::move(cb));
-  } else {
-    // Nothing earlier in the pipeline, just call it
-    cb();
-  }
-}
-
 void ECCommon::ReadPipeline::get_all_avail_shards(
   const hobject_t &hoid,
   const set<pg_shard_t> &error_shards,
@@ -905,3 +878,30 @@ void ECCommon::RMWPipeline::check_ops()
 	 try_reads_to_commit() ||
 	 try_finish_rmw());
 }
+
+void ECCommon::RMWPipeline::on_change()
+{
+  dout(10) << __func__ << dendl;
+
+  completed_to = eversion_t();
+  committed_to = eversion_t();
+  pipeline_state.clear();
+  waiting_reads.clear();
+  waiting_state.clear();
+  waiting_commit.clear();
+  for (auto &&op: tid_to_op_map) {
+    cache.release_write_pin(op.second->pin);
+  }
+  tid_to_op_map.clear();
+}
+
+void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
+  if (!waiting_state.empty()) {
+    waiting_state.back().on_write.emplace_back(std::move(cb));
+  } else if (!waiting_reads.empty()) {
+    waiting_reads.back().on_write.emplace_back(std::move(cb));
+  } else {
+    // Nothing earlier in the pipeline, just call it
+    cb();
+  }
+}

From f80e367c7e483ad2543f228c5ceee71760a1b961 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 28 Sep 2023 18:17:59 +0200
Subject: [PATCH 1322/2492] osd: make {PG,EC}Transaction.h consumable for
 crimson
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECTransaction.h | 7 +++----
 src/osd/PGTransaction.h | 5 ++++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/osd/ECTransaction.h b/src/osd/ECTransaction.h
index 56f6e341b525..2aa2941f8db5 100644
--- a/src/osd/ECTransaction.h
+++ b/src/osd/ECTransaction.h
@@ -15,12 +15,11 @@
 #ifndef ECTRANSACTION_H
 #define ECTRANSACTION_H
 
-#include "OSD.h"
-#include "PGBackend.h"
 #include "ECUtil.h"
+#include "ExtentCache.h"
 #include "erasure-code/ErasureCodeInterface.h"
+#include "os/Transaction.h"
 #include "PGTransaction.h"
-#include "ExtentCache.h"
 
 namespace ECTransaction {
   struct WritePlan {
@@ -186,7 +185,7 @@ namespace ECTransaction {
     const std::map<hobject_t,extent_map> &partial_extents,
     std::vector<pg_log_entry_t> &entries,
     std::map<hobject_t,extent_map> *written,
-    std::map<shard_id_t, ObjectStore::Transaction> *transactions,
+    std::map<shard_id_t, ceph::os::Transaction> *transactions,
     std::set<hobject_t> *temp_added,
     std::set<hobject_t> *temp_removed,
     DoutPrefixProvider *dpp,
diff --git a/src/osd/PGTransaction.h b/src/osd/PGTransaction.h
index e6f57c90fa1e..bb6e4efc3f47 100644
--- a/src/osd/PGTransaction.h
+++ b/src/osd/PGTransaction.h
@@ -19,8 +19,11 @@
 #include <optional>
 
 #include "common/hobject.h"
-#include "osd/osd_types.h"
+#ifndef WITH_SEASTAR
 #include "osd/osd_internal_types.h"
+#else
+#include "crimson/osd/object_context.h"
+#endif
 #include "common/interval_map.h"
 #include "common/inline_variant.h"
 

From 76d780f743cbdd648bd9eca73e0c8d2fc4c10532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 28 Sep 2023 18:22:41 +0200
Subject: [PATCH 1323/2492] osd: finish shuffling ECCommon::ReadPipeline to
 ECCommon.cc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 183 -------------------------------------------
 src/osd/ECCommon.cc  | 169 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+), 183 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index afa5d5bba44e..e29d181ec0b6 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -55,12 +55,6 @@ using ceph::Formatter;
 static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
   return pgb->get_parent()->gen_dbg_prefix(*_dout);
 }
-static ostream& _prefix(std::ostream *_dout, ECCommon::RMWPipeline *rmw_pipeline) {
-  return rmw_pipeline->get_parent()->gen_dbg_prefix(*_dout);
-}
-static ostream& _prefix(std::ostream *_dout, ECCommon::ReadPipeline *read_pipeline) {
-  return read_pipeline->get_parent()->gen_dbg_prefix(*_dout);
-}
 
 struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
   list<ECBackend::RecoveryOp> ops;
@@ -92,14 +86,6 @@ static ostream &operator<<(ostream &lhs, const map<int, bufferlist> &rhs)
   return lhs << "]";
 }
 
-static ostream &operator<<(
-  ostream &lhs,
-  const boost::tuple<uint64_t, uint64_t, map<pg_shard_t, bufferlist> > &rhs)
-{
-  return lhs << "(" << rhs.get<0>() << ", "
-	     << rhs.get<1>() << ", " << rhs.get<2>() << ")";
-}
-
 ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
 {
   return lhs << "RecoveryOp("
@@ -1589,70 +1575,6 @@ void ECBackend::objects_read_async(
 	   on_complete)));
 }
 
-struct ClientReadCompleter : ECCommon::ReadCompleter {
-  ClientReadCompleter(ECCommon::ReadPipeline &read_pipeline,
-                      ECCommon::ClientAsyncReadStatus *status)
-    : read_pipeline(read_pipeline),
-      status(status) {}
-
-  void finish_single_request(
-    const hobject_t &hoid,
-    ECCommon::read_result_t &res,
-    list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) override
-  {
-    extent_map result;
-    if (res.r != 0)
-      goto out;
-    ceph_assert(res.returned.size() == to_read.size());
-    ceph_assert(res.errors.empty());
-    for (auto &&read: to_read) {
-      pair<uint64_t, uint64_t> adjusted =
-	read_pipeline.sinfo.offset_len_to_stripe_bounds(
-	  make_pair(read.get<0>(), read.get<1>()));
-      ceph_assert(res.returned.front().get<0>() == adjusted.first);
-      ceph_assert(res.returned.front().get<1>() == adjusted.second);
-      map<int, bufferlist> to_decode;
-      bufferlist bl;
-      for (map<pg_shard_t, bufferlist>::iterator j =
-	     res.returned.front().get<2>().begin();
-	   j != res.returned.front().get<2>().end();
-	   ++j) {
-	to_decode[j->first.shard] = std::move(j->second);
-      }
-      int r = ECUtil::decode(
-	read_pipeline.sinfo,
-	read_pipeline.ec_impl,
-	to_decode,
-	&bl);
-      if (r < 0) {
-        res.r = r;
-        goto out;
-      }
-      bufferlist trimmed;
-      trimmed.substr_of(
-	bl,
-	read.get<0>() - adjusted.first,
-	std::min(read.get<1>(),
-	    bl.length() - (read.get<0>() - adjusted.first)));
-      result.insert(
-	read.get<0>(), trimmed.length(), std::move(trimmed));
-      res.returned.pop_front();
-    }
-out:
-    status->complete_object(hoid, res.r, std::move(result));
-    read_pipeline.kick_reads();
-  }
-
-  void finish(int priority) && override
-  {
-    // NOP
-  }
-
-  ECCommon::ReadPipeline &read_pipeline;
-  ECCommon::ClientAsyncReadStatus *status;
-};
-
-
 void ECBackend::objects_read_and_reconstruct(
   const map<hobject_t,
     std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
@@ -1664,111 +1586,6 @@ void ECBackend::objects_read_and_reconstruct(
     reads, fast_read, std::move(func));
 }
 
-void ECCommon::ReadPipeline::get_want_to_read_shards(
-  std::set<int> *want_to_read) const
-{
-  const std::vector<int> &chunk_mapping = ec_impl->get_chunk_mapping();
-  for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
-    int chunk = (int)chunk_mapping.size() > i ? chunk_mapping[i] : i;
-    want_to_read->insert(chunk);
-  }
-}
-
-void ECCommon::ReadPipeline::objects_read_and_reconstruct(
-  const map<hobject_t,
-    std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
-  > &reads,
-  bool fast_read,
-  GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
-{
-  in_progress_client_reads.emplace_back(
-    reads.size(), std::move(func));
-  if (!reads.size()) {
-    kick_reads();
-    return;
-  }
-
-  map<hobject_t, set<int>> obj_want_to_read;
-  set<int> want_to_read;
-  get_want_to_read_shards(&want_to_read);
-    
-  map<hobject_t, read_request_t> for_read_op;
-  for (auto &&to_read: reads) {
-    map<pg_shard_t, vector<pair<int, int>>> shards;
-    int r = get_min_avail_to_read_shards(
-      to_read.first,
-      want_to_read,
-      false,
-      fast_read,
-      &shards);
-    ceph_assert(r == 0);
-
-    for_read_op.insert(
-      make_pair(
-	to_read.first,
-	read_request_t(
-	  to_read.second,
-	  shards,
-	  false)));
-    obj_want_to_read.insert(make_pair(to_read.first, want_to_read));
-  }
-
-  start_read_op(
-    CEPH_MSG_PRIO_DEFAULT,
-    obj_want_to_read,
-    for_read_op,
-    OpRequestRef(),
-    fast_read,
-    false,
-    std::make_unique<ClientReadCompleter>(*this, &(in_progress_client_reads.back())));
-}
-
-
-int ECCommon::ReadPipeline::send_all_remaining_reads(
-  const hobject_t &hoid,
-  ReadOp &rop)
-{
-  set<int> already_read;
-  const set<pg_shard_t>& ots = rop.obj_to_source[hoid];
-  for (set<pg_shard_t>::iterator i = ots.begin(); i != ots.end(); ++i)
-    already_read.insert(i->shard);
-  dout(10) << __func__ << " have/error shards=" << already_read << dendl;
-  map<pg_shard_t, vector<pair<int, int>>> shards;
-  int r = get_remaining_shards(hoid, already_read, rop.want_to_read[hoid],
-			       rop.complete[hoid], &shards, rop.for_recovery);
-  if (r)
-    return r;
-
-  list<boost::tuple<uint64_t, uint64_t, uint32_t> > offsets =
-    rop.to_read.find(hoid)->second.to_read;
-
-  // (Note cuixf) If we need to read attrs and we read failed, try to read again.
-  bool want_attrs =
-    rop.to_read.find(hoid)->second.want_attrs &&
-    (!rop.complete[hoid].attrs || rop.complete[hoid].attrs->empty());
-  if (want_attrs) {
-    dout(10) << __func__ << " want attrs again" << dendl;
-  }
-
-  rop.to_read.erase(hoid);
-  rop.to_read.insert(make_pair(
-      hoid,
-      read_request_t(
-	offsets,
-	shards,
-	want_attrs)));
-  return 0;
-}
-
-void ECCommon::ReadPipeline::kick_reads()
-{
-  while (in_progress_client_reads.size() &&
-         in_progress_client_reads.front().is_complete()) {
-    in_progress_client_reads.front().run();
-    in_progress_client_reads.pop_front();
-  }
-}
-
 void ECBackend::kick_reads() {
   read_pipeline.kick_reads();
 }
diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index 7ed15a061f81..e1bbfce7796c 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -565,6 +565,175 @@ void ECCommon::ReadPipeline::do_read_op(ReadOp &op)
   dout(10) << __func__ << ": started " << op << dendl;
 }
 
+void ECCommon::ReadPipeline::get_want_to_read_shards(
+  std::set<int> *want_to_read) const
+{
+  const std::vector<int> &chunk_mapping = ec_impl->get_chunk_mapping();
+  for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) {
+    int chunk = (int)chunk_mapping.size() > i ? chunk_mapping[i] : i;
+    want_to_read->insert(chunk);
+  }
+}
+
+struct ClientReadCompleter : ECCommon::ReadCompleter {
+  ClientReadCompleter(ECCommon::ReadPipeline &read_pipeline,
+                      ECCommon::ClientAsyncReadStatus *status)
+    : read_pipeline(read_pipeline),
+      status(status) {}
+
+  void finish_single_request(
+    const hobject_t &hoid,
+    ECCommon::read_result_t &res,
+    list<boost::tuple<uint64_t, uint64_t, uint32_t> > to_read) override
+  {
+    extent_map result;
+    if (res.r != 0)
+      goto out;
+    ceph_assert(res.returned.size() == to_read.size());
+    ceph_assert(res.errors.empty());
+    for (auto &&read: to_read) {
+      pair<uint64_t, uint64_t> adjusted =
+	read_pipeline.sinfo.offset_len_to_stripe_bounds(
+	  make_pair(read.get<0>(), read.get<1>()));
+      ceph_assert(res.returned.front().get<0>() == adjusted.first);
+      ceph_assert(res.returned.front().get<1>() == adjusted.second);
+      map<int, bufferlist> to_decode;
+      bufferlist bl;
+      for (map<pg_shard_t, bufferlist>::iterator j =
+	     res.returned.front().get<2>().begin();
+	   j != res.returned.front().get<2>().end();
+	   ++j) {
+	to_decode[j->first.shard] = std::move(j->second);
+      }
+      int r = ECUtil::decode(
+	read_pipeline.sinfo,
+	read_pipeline.ec_impl,
+	to_decode,
+	&bl);
+      if (r < 0) {
+        res.r = r;
+        goto out;
+      }
+      bufferlist trimmed;
+      trimmed.substr_of(
+	bl,
+	read.get<0>() - adjusted.first,
+	std::min(read.get<1>(),
+	    bl.length() - (read.get<0>() - adjusted.first)));
+      result.insert(
+	read.get<0>(), trimmed.length(), std::move(trimmed));
+      res.returned.pop_front();
+    }
+out:
+    status->complete_object(hoid, res.r, std::move(result));
+    read_pipeline.kick_reads();
+  }
+
+  void finish(int priority) && override
+  {
+    // NOP
+  }
+
+  ECCommon::ReadPipeline &read_pipeline;
+  ECCommon::ClientAsyncReadStatus *status;
+};
+
+void ECCommon::ReadPipeline::objects_read_and_reconstruct(
+  const map<hobject_t,
+    std::list<boost::tuple<uint64_t, uint64_t, uint32_t> >
+  > &reads,
+  bool fast_read,
+  GenContextURef<map<hobject_t,pair<int, extent_map> > &&> &&func)
+{
+  in_progress_client_reads.emplace_back(
+    reads.size(), std::move(func));
+  if (!reads.size()) {
+    kick_reads();
+    return;
+  }
+
+  map<hobject_t, set<int>> obj_want_to_read;
+  set<int> want_to_read;
+  get_want_to_read_shards(&want_to_read);
+    
+  map<hobject_t, read_request_t> for_read_op;
+  for (auto &&to_read: reads) {
+    map<pg_shard_t, vector<pair<int, int>>> shards;
+    int r = get_min_avail_to_read_shards(
+      to_read.first,
+      want_to_read,
+      false,
+      fast_read,
+      &shards);
+    ceph_assert(r == 0);
+
+    for_read_op.insert(
+      make_pair(
+	to_read.first,
+	read_request_t(
+	  to_read.second,
+	  shards,
+	  false)));
+    obj_want_to_read.insert(make_pair(to_read.first, want_to_read));
+  }
+
+  start_read_op(
+    CEPH_MSG_PRIO_DEFAULT,
+    obj_want_to_read,
+    for_read_op,
+    OpRequestRef(),
+    fast_read,
+    false,
+    std::make_unique<ClientReadCompleter>(*this, &(in_progress_client_reads.back())));
+}
+
+
+int ECCommon::ReadPipeline::send_all_remaining_reads(
+  const hobject_t &hoid,
+  ReadOp &rop)
+{
+  set<int> already_read;
+  const set<pg_shard_t>& ots = rop.obj_to_source[hoid];
+  for (set<pg_shard_t>::iterator i = ots.begin(); i != ots.end(); ++i)
+    already_read.insert(i->shard);
+  dout(10) << __func__ << " have/error shards=" << already_read << dendl;
+  map<pg_shard_t, vector<pair<int, int>>> shards;
+  int r = get_remaining_shards(hoid, already_read, rop.want_to_read[hoid],
+			       rop.complete[hoid], &shards, rop.for_recovery);
+  if (r)
+    return r;
+
+  list<boost::tuple<uint64_t, uint64_t, uint32_t> > offsets =
+    rop.to_read.find(hoid)->second.to_read;
+
+  // (Note cuixf) If we need to read attrs and we read failed, try to read again.
+  bool want_attrs =
+    rop.to_read.find(hoid)->second.want_attrs &&
+    (!rop.complete[hoid].attrs || rop.complete[hoid].attrs->empty());
+  if (want_attrs) {
+    dout(10) << __func__ << " want attrs again" << dendl;
+  }
+
+  rop.to_read.erase(hoid);
+  rop.to_read.insert(make_pair(
+      hoid,
+      read_request_t(
+	offsets,
+	shards,
+	want_attrs)));
+  return 0;
+}
+
+void ECCommon::ReadPipeline::kick_reads()
+{
+  while (in_progress_client_reads.size() &&
+         in_progress_client_reads.front().is_complete()) {
+    in_progress_client_reads.front().run();
+    in_progress_client_reads.pop_front();
+  }
+}
+
+
 void ECCommon::RMWPipeline::start_rmw(OpRef op)
 {
   ceph_assert(op);

From 93e41ec624393bef5503e58a1e7df9f34bf16a6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Mon, 2 Oct 2023 21:52:44 +0200
Subject: [PATCH 1324/2492] osd: refactor ECTransaction with C++17 structured
 bindings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For better understandability.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECTransaction.h | 46 ++++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/osd/ECTransaction.h b/src/osd/ECTransaction.h
index 2aa2941f8db5..8ca7b4da53ed 100644
--- a/src/osd/ECTransaction.h
+++ b/src/osd/ECTransaction.h
@@ -39,20 +39,21 @@ namespace ECTransaction {
     WritePlan plan;
     t.safe_create_traverse(
       [&](std::pair<const hobject_t, PGTransaction::ObjectOperation> &i) {
-	ECUtil::HashInfoRef hinfo = get_hinfo(i.first);
-	plan.hash_infos[i.first] = hinfo;
+        const auto& [obj, op] = i;
+	ECUtil::HashInfoRef hinfo = get_hinfo(obj);
+	plan.hash_infos[obj] = hinfo;
 
 	uint64_t projected_size =
 	  hinfo->get_projected_total_logical_size(sinfo);
 
-	if (i.second.deletes_first()) {
+	if (op.deletes_first()) {
 	  ldpp_dout(dpp, 20) << __func__ << ": delete, setting projected size"
 			     << " to 0" << dendl;
 	  projected_size = 0;
 	}
 
 	hobject_t source;
-	if (i.second.has_source(&source)) {
+	if (op.has_source(&source)) {
 	  // typically clone or mv
 	  plan.invalidates_cache = true;
 
@@ -61,27 +62,27 @@ namespace ECTransaction {
 	  plan.hash_infos[source] = shinfo;
 	}
 
-	auto &will_write = plan.will_write[i.first];
-	if (i.second.truncate &&
-	    i.second.truncate->first < projected_size) {
+	auto &will_write = plan.will_write[obj];
+	if (op.truncate &&
+	    op.truncate->first < projected_size) {
 	  if (!(sinfo.logical_offset_is_stripe_aligned(
-		  i.second.truncate->first))) {
-	    plan.to_read[i.first].union_insert(
-	      sinfo.logical_to_prev_stripe_offset(i.second.truncate->first),
+		  op.truncate->first))) {
+	    plan.to_read[obj].union_insert(
+	      sinfo.logical_to_prev_stripe_offset(op.truncate->first),
 	      sinfo.get_stripe_width());
 
 	    ldpp_dout(dpp, 20) << __func__ << ": unaligned truncate" << dendl;
 
 	    will_write.union_insert(
-	      sinfo.logical_to_prev_stripe_offset(i.second.truncate->first),
+	      sinfo.logical_to_prev_stripe_offset(op.truncate->first),
 	      sinfo.get_stripe_width());
 	  }
 	  projected_size = sinfo.logical_to_next_stripe_offset(
-	    i.second.truncate->first);
+	    op.truncate->first);
 	}
 
 	extent_set raw_write_set;
-	for (auto &&extent: i.second.buffer_updates) {
+	for (auto &&extent: op.buffer_updates) {
 	  using BufferUpdate = PGTransaction::ObjectOperation::BufferUpdate;
 	  if (boost::get<BufferUpdate::CloneRange>(&(extent.get_val()))) {
 	    ceph_assert(
@@ -109,7 +110,7 @@ namespace ECTransaction {
 	    ldpp_dout(dpp, 20) << __func__ << ": reading partial head stripe "
 			       << head_start << "~" << sinfo.get_stripe_width()
 			       << dendl;
-	    plan.to_read[i.first].union_insert(
+	    plan.to_read[obj].union_insert(
 	      head_start, sinfo.get_stripe_width());
 	  }
 
@@ -127,7 +128,7 @@ namespace ECTransaction {
 	    ldpp_dout(dpp, 20) << __func__ << ": reading partial tail stripe "
 			       << tail_start << "~" << sinfo.get_stripe_width()
 			       << dendl;
-	    plan.to_read[i.first].union_insert(
+	    plan.to_read[obj].union_insert(
 	      tail_start, sinfo.get_stripe_width());
 	  }
 
@@ -145,10 +146,9 @@ namespace ECTransaction {
 	  }
 	}
 
-	if (i.second.truncate &&
-	    i.second.truncate->second > projected_size) {
+	if (op.truncate && op.truncate->second > projected_size) {
 	  uint64_t truncating_to =
-	    sinfo.logical_to_next_stripe_offset(i.second.truncate->second);
+	    sinfo.logical_to_next_stripe_offset(op.truncate->second);
 	  ldpp_dout(dpp, 20) << __func__ << ": truncating out to "
 			     <<  truncating_to
 			     << dendl;
@@ -157,7 +157,7 @@ namespace ECTransaction {
 	  projected_size = truncating_to;
 	}
 
-	ldpp_dout(dpp, 20) << __func__ << ": " << i.first
+	ldpp_dout(dpp, 20) << __func__ << ": " << obj
 			   << " projected size "
 			   << projected_size
 			   << dendl;
@@ -166,11 +166,11 @@ namespace ECTransaction {
 	  projected_size);
 
 	/* validate post conditions:
-	 * to_read should have an entry for i.first iff it isn't empty
-	 * and if we are reading from i.first, we can't be renaming or
+	 * to_read should have an entry for `obj` if it isn't empty
+	 * and if we are reading from `obj`, we can't be renaming or
 	 * cloning it */
-	ceph_assert(plan.to_read.count(i.first) == 0 ||
-	       (!plan.to_read.at(i.first).empty() &&
+	ceph_assert(plan.to_read.count(obj) == 0 ||
+	       (!plan.to_read.at(obj).empty() &&
 		!i.second.has_source()));
       });
     return plan;

From 6f261a63322595d8313e8104f579698674e288b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Mon, 9 Oct 2023 21:40:47 +0200
Subject: [PATCH 1325/2492] osd: improve comments in
 ECBackend::objects_read_async()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index e29d181ec0b6..1322f7f47f31 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1523,6 +1523,7 @@ void ECBackend::objects_read_async(
       int r = 0;
       for (auto &&read: to_read) {
 	if (got.first < 0) {
+	  // error handling
 	  if (read.second.second) {
 	    read.second.second->complete(got.first);
 	  }

From 4bb022ba4ba32a22cf573d587098dfcea5030ed6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 24 Oct 2023 12:40:15 +0200
Subject: [PATCH 1326/2492] mon/OSDMonitor: let crimson handle ECPools
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/mon/OSDMonitor.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f8e379326f25..09ff97e6748e 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -8013,10 +8013,7 @@ int OSDMonitor::prepare_new_pool(string& name,
     /* crimson-osd requires that the pool be replicated and that pg_num/pgp_num
      * be static.  User must also have specified set-allow-crimson */
     const auto *suffix = " (--crimson specified or osd_pool_default_crimson set)";
-    if (pool_type != pg_pool_t::TYPE_REPLICATED) {
-      *ss << "crimson-osd only supports replicated pools" << suffix;
-      return -EINVAL;
-    } else if (pg_autoscale_mode != "off") {
+    if (pg_autoscale_mode != "off") {
       *ss << "crimson-osd does not support changing pg_num or pgp_num, "
 	  << "pg_autoscale_mode must be set to 'off'" << suffix;
       return -EINVAL;

From 1a4d3f01816cedb15106fe2cdb52322029482827 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 7 Nov 2023 16:22:46 +0100
Subject: [PATCH 1327/2492] osd: ECBackend::get_hash_info() makes use of the
 obc::attr_cache
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
(cherry picked from commit cf07db1a09044c689f5b595d5088a61e3d098e15)
---
 src/common/scrub_types.cc                     | 10 ++--
 .../osd/osd_operations/scrub_events.cc        |  2 +-
 src/crimson/osd/scrub/scrub_validator.cc      | 12 ++---
 src/osd/ECBackend.cc                          | 42 ++++++++---------
 src/osd/ECBackend.h                           |  4 +-
 src/osd/osd_types.cc                          | 12 ++++-
 src/osd/osd_types.h                           |  2 +-
 src/osd/osd_types_fmt.h                       |  6 +--
 src/osd/scrubber/pg_scrubber.cc               |  9 ++--
 src/osd/scrubber/scrub_backend.cc             | 46 ++++++-------------
 src/test/crimson/test_crimson_scrub.cc        |  9 ++--
 src/test/osd/scrubber_generators.cc           | 24 +++++-----
 12 files changed, 76 insertions(+), 102 deletions(-)

diff --git a/src/common/scrub_types.cc b/src/common/scrub_types.cc
index 9168ee0a2793..b03a3cab70c8 100644
--- a/src/common/scrub_types.cc
+++ b/src/common/scrub_types.cc
@@ -55,10 +55,12 @@ static void encode(const osd_shard_t& shard, bufferlist& bl) {
 
 void shard_info_wrapper::set_object(const ScrubMap::object& object)
 {
-  for (auto attr : object.attrs) {
-    bufferlist bl;
-    bl.push_back(attr.second);
-    attrs.insert(std::make_pair(attr.first, std::move(bl)));
+  // logically no-op, changes the comparator from std::less<void>
+  // while avoiding `reinterpret_cast<const std::map<std::string,
+  // ceph::bufferlist>&>(object.attrs)`
+  attrs.clear();
+  for (const auto& kv : object.attrs) {
+    attrs.insert(kv);
   }
   size = object.size;
   if (object.omap_digest_present) {
diff --git a/src/crimson/osd/osd_operations/scrub_events.cc b/src/crimson/osd/osd_operations/scrub_events.cc
index 4f54cf0b274f..a488077d6a77 100644
--- a/src/crimson/osd/osd_operations/scrub_events.cc
+++ b/src/crimson/osd/osd_operations/scrub_events.cc
@@ -220,7 +220,7 @@ ScrubScan::ifut<> ScrubScan::scan_object(
       if (i.second.length() == 0) {
 	entry.attrs[i.first];
       } else {
-	entry.attrs.emplace(i.first, i.second.front());
+	entry.attrs.emplace(i.first, i.second);
       }
     }
   }).handle_error_interruptible(
diff --git a/src/crimson/osd/scrub/scrub_validator.cc b/src/crimson/osd/scrub/scrub_validator.cc
index b7dcc46a35e7..a3979f790bca 100644
--- a/src/crimson/osd/scrub/scrub_validator.cc
+++ b/src/crimson/osd/scrub/scrub_validator.cc
@@ -96,11 +96,9 @@ shard_evaluation_t evaluate_object_shard(
     if (xiter == obj.attrs.end()) {
       ret.shard_info.set_info_missing();
     } else {
-      bufferlist bl;
-      bl.push_back(xiter->second);
       ret.object_info = object_info_t{};
       try {
-	auto bliter = bl.cbegin();
+	auto bliter = xiter->second.cbegin();
 	::decode(*(ret.object_info), bliter);
       } catch (...) {
 	ret.shard_info.set_info_corrupted();
@@ -120,11 +118,9 @@ shard_evaluation_t evaluate_object_shard(
     if (xiter == obj.attrs.end()) {
       ret.shard_info.set_snapset_missing();
     } else {
-      bufferlist bl;
-      bl.push_back(xiter->second);
       ret.snapset = SnapSet{};
       try {
-	auto bliter = bl.cbegin();
+	auto bliter = xiter->second.cbegin();
 	::decode(*(ret.snapset), bliter);
       } catch (...) {
 	ret.shard_info.set_snapset_corrupted();
@@ -138,11 +134,9 @@ shard_evaluation_t evaluate_object_shard(
     if (xiter == obj.attrs.end()) {
       ret.shard_info.set_hinfo_missing();
     } else {
-      bufferlist bl;
-      bl.push_back(xiter->second);
       ret.hinfo = ECUtil::HashInfo{};
       try {
-	auto bliter = bl.cbegin();
+	auto bliter = xiter->second.cbegin();
 	decode(*(ret.hinfo), bliter);
       } catch (...) {
 	ret.shard_info.set_hinfo_corrupted();
diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 1322f7f47f31..836bf96dde44 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -518,7 +518,7 @@ void ECBackend::continue_recovery_op(
 
       if (op.recovery_progress.first && op.obc) {
 	/* We've got the attrs and the hinfo, might as well use them */
-	op.hinfo = get_hash_info(op.hoid);
+	op.hinfo = get_hash_info(op.hoid, false, &op.obc->attr_cache);
 	if (!op.hinfo) {
           derr << __func__ << ": " << op.hoid << " has inconsistent hinfo"
                << dendl;
@@ -1012,7 +1012,14 @@ void ECBackend::handle_sub_read(
 	// Do NOT check osd_read_eio_on_bad_digest here.  We need to report
 	// the state of our chunk in case other chunks could substitute.
         ECUtil::HashInfoRef hinfo;
-        hinfo = get_hash_info(i->first);
+        map<string, bufferlist, less<>> attrs;
+        int r = PGBackend::objects_get_attrs(i->first, &attrs);
+        if (r >= 0) {
+          hinfo = get_hash_info(i->first, false, &attrs);
+	} else {
+          derr << "get_hash_info" << ": stat " << i->first << " failed: "
+               << cpp_strerror(r) << dendl;
+	}
         if (!hinfo) {
           r = -EIO;
           get_parent()->clog_error() << "Corruption detected: object "
@@ -1370,7 +1377,7 @@ void ECBackend::submit_transaction(
     sinfo,
     *(op->t),
     [&](const hobject_t &i) {
-      ECUtil::HashInfoRef ref = get_hash_info(i, true);
+      ECUtil::HashInfoRef ref = get_hash_info(i, true, &op->t->obc_map[hoid]->attr_cache);
       if (!ref) {
 	derr << __func__ << ": get_hash_info(" << i << ")"
 	     << " returned a null pointer and there is no "
@@ -1387,7 +1394,7 @@ void ECBackend::submit_transaction(
 
 
 ECUtil::HashInfoRef ECBackend::get_hash_info(
-  const hobject_t &hoid, bool create, const map<string,bufferptr,less<>> *attrs)
+  const hobject_t &hoid, bool create, const map<string,bufferlist,less<>> *attrs)
 {
   dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
   ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
@@ -1402,23 +1409,12 @@ ECUtil::HashInfoRef ECBackend::get_hash_info(
     if (r >= 0) {
       dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
       bufferlist bl;
-      if (attrs) {
-	map<string, bufferptr>::const_iterator k = attrs->find(ECUtil::get_hinfo_key());
-	if (k == attrs->end()) {
-	  dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
-	} else {
-	  bl.push_back(k->second);
-	}
+      ceph_assert(attrs);
+      map<string, bufferlist>::const_iterator k = attrs->find(ECUtil::get_hinfo_key());
+      if (k == attrs->end()) {
+        dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
       } else {
-	r = store->getattr(
-	  ch,
-	  ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
-	  ECUtil::get_hinfo_key(),
-	  bl);
-	if (r < 0) {
-	  dout(5) << __func__ << ": getattr failed: " << cpp_strerror(r) << dendl;
-	  bl.clear(); // just in case
-	}
+        bl = k->second;
       }
       if (bl.length() > 0) {
 	auto bp = bl.cbegin();
@@ -1596,10 +1592,8 @@ int ECBackend::objects_get_attrs(
   const hobject_t &hoid,
   map<string, bufferlist, less<>> *out)
 {
-  int r = store->getattrs(
-    ch,
-    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
-    *out);
+  // call from parents -- get raw attrs, without any filtering for hinfo
+  int r = PGBackend::objects_get_attrs(hoid, out);
   if (r < 0)
     return r;
 
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 54b8e16119eb..27176bf18f92 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -325,8 +325,8 @@ class ECBackend : public PGBackend, public ECCommon {
   const ECUtil::stripe_info_t sinfo;
   /// If modified, ensure that the ref is held until the update is applied
   SharedPtrRegistry<hobject_t, ECUtil::HashInfo> unstable_hashinfo_registry;
-  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool create = false,
-				    const std::map<std::string, ceph::buffer::ptr, std::less<>> *attr = NULL);
+  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool create,
+				    const std::map<std::string, ceph::buffer::list, std::less<>> *attr);
 
 public:
   ECBackend(
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 7596723a0e30..1a6b6041d66f 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -7148,8 +7148,16 @@ void ScrubMap::object::generate_test_instances(list<object*>& o)
   o.back()->negative = true;
   o.push_back(new object);
   o.back()->size = 123;
-  o.back()->attrs["foo"] = ceph::buffer::copy("foo", 3);
-  o.back()->attrs["bar"] = ceph::buffer::copy("barval", 6);
+  {
+    bufferlist foobl;
+    foobl.push_back(ceph::buffer::copy("foo", 3));
+    o.back()->attrs["foo"] = std::move(foobl);
+  }
+  {
+    bufferlist barbl;
+    barbl.push_back(ceph::buffer::copy("barval", 6));
+    o.back()->attrs["bar"] = std::move(barbl);
+  }
 }
 
 // -- OSDOp --
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index ff02ca5730f7..a8e6b5994a93 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -6233,7 +6233,7 @@ std::ostream& operator<<(std::ostream& out, const PushOp &op);
  */
 struct ScrubMap {
   struct object {
-    std::map<std::string, ceph::buffer::ptr, std::less<>> attrs;
+    std::map<std::string, ceph::buffer::list, std::less<>> attrs;
     uint64_t size;
     __u32 omap_digest;         ///< omap crc32c
     __u32 digest;              ///< data crc32c
diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h
index d6d746d295f7..6ffe95a3e95f 100644
--- a/src/osd/osd_types_fmt.h
+++ b/src/osd/osd_types_fmt.h
@@ -275,15 +275,13 @@ struct fmt::formatter<ScrubMap::object> {
 
     // note the special handling of (1) OI_ATTR and (2) non-printables
     for (auto [k, v] : so.attrs) {
-      std::string bkstr{v.raw_c_str(), v.raw_length()};
+      std::string bkstr = v.to_str();
       if (k == std::string{OI_ATTR}) {
 	/// \todo consider parsing the OI args here. Maybe add a specific format
 	/// specifier
 	fmt::format_to(ctx.out(), "{{{}:<<OI_ATTR>>({})}} ", k, bkstr.length());
       } else if (k == std::string{SS_ATTR}) {
-	bufferlist bl;
-	bl.push_back(v);
-	SnapSet sns{bl};
+	SnapSet sns{v};
 	fmt::format_to(ctx.out(), "{{{}:{:D}}} ", k, sns);
       } else {
 	fmt::format_to(ctx.out(), "{{{}:{}({})}} ", k, bkstr, bkstr.length());
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 2dae53273a70..3bfc3a10f10a 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -1321,11 +1321,9 @@ void PgScrubber::repair_oinfo_oid(ScrubMap& smap)
     if (o.attrs.find(OI_ATTR) == o.attrs.end()) {
       continue;
     }
-    bufferlist bl;
-    bl.push_back(o.attrs[OI_ATTR]);
     object_info_t oi;
     try {
-      oi.decode(bl);
+      oi.decode(o.attrs[OI_ATTR]);
     } catch (...) {
       continue;
     }
@@ -1340,13 +1338,12 @@ void PgScrubber::repair_oinfo_oid(ScrubMap& smap)
         << "...repaired";
       // Fix object info
       oi.soid = hoid;
-      bl.clear();
+      bufferlist bl;
       encode(oi,
              bl,
              m_pg->get_osdmap()->get_features(CEPH_ENTITY_TYPE_OSD, nullptr));
 
-      bufferptr bp(bl.c_str(), bl.length());
-      o.attrs[OI_ATTR] = bp;
+      o.attrs[OI_ATTR] = std::move(bl);
 
       t.setattr(m_pg->coll, ghobject_t(hoid), OI_ATTR, bl);
       int r = m_pg->osd->store->queue_transaction(m_pg->ch, std::move(t));
diff --git a/src/osd/scrubber/scrub_backend.cc b/src/osd/scrubber/scrub_backend.cc
index e25c5b99da09..2d7d1a4ecf17 100644
--- a/src/osd/scrubber/scrub_backend.cc
+++ b/src/osd/scrubber/scrub_backend.cc
@@ -376,7 +376,7 @@ void ScrubBackend::repair_object(const hobject_t& soid,
   try {
     bufferlist bv;
     if (po.attrs.count(OI_ATTR)) {
-      bv.push_back(po.attrs.find(OI_ATTR)->second);
+      bv = po.attrs.find(OI_ATTR)->second;
     }
     auto bliter = bv.cbegin();
     decode(oi, bliter);
@@ -634,9 +634,8 @@ shard_as_auth_t ScrubBackend::possible_auth_shard(const hobject_t& obj,
                        &shard_info_wrapper::set_snapset_missing,
                        "candidate had a missing snapset key"sv,
                        errstream)) {
-      bufferlist ss_bl;
+      const bufferlist& ss_bl = k->second;
       SnapSet snapset;
-      ss_bl.push_back(k->second);
       try {
         auto bliter = ss_bl.cbegin();
         decode(snapset, bliter);
@@ -671,9 +670,8 @@ shard_as_auth_t ScrubBackend::possible_auth_shard(const hobject_t& obj,
                        &shard_info_wrapper::set_hinfo_missing,
                        "candidate had a missing hinfo key"sv,
                        errstream)) {
-      bufferlist hk_bl;
+      const bufferlist& hk_bl = k->second;
       ECUtil::HashInfo hi;
-      hk_bl.push_back(k->second);
       try {
         auto bliter = hk_bl.cbegin();
         decode(hi, bliter);
@@ -704,10 +702,8 @@ shard_as_auth_t ScrubBackend::possible_auth_shard(const hobject_t& obj,
       return shard_as_auth_t{errstream.str()};
     }
 
-    bufferlist bl;
-    bl.push_back(k->second);
     try {
-      auto bliter = bl.cbegin();
+      auto bliter = k->second.cbegin();
       decode(oi, bliter);
     } catch (...) {
       // invalid object info, probably corrupt
@@ -1232,13 +1228,11 @@ bool ScrubBackend::compare_obj_details(pg_shard_t auth_shard,
 
     auto can_attr = candidate.attrs.find(OI_ATTR);
     ceph_assert(can_attr != candidate.attrs.end());
-    bufferlist can_bl;
-    can_bl.push_back(can_attr->second);
+    const bufferlist& can_bl = can_attr->second;
 
     auto auth_attr = auth.attrs.find(OI_ATTR);
     ceph_assert(auth_attr != auth.attrs.end());
-    bufferlist auth_bl;
-    auth_bl.push_back(auth_attr->second);
+    const bufferlist& auth_bl = auth_attr->second;
 
     if (!can_bl.contents_equal(auth_bl)) {
       fmt::format_to(std::back_inserter(out),
@@ -1254,13 +1248,11 @@ bool ScrubBackend::compare_obj_details(pg_shard_t auth_shard,
 
       auto can_attr = candidate.attrs.find(SS_ATTR);
       ceph_assert(can_attr != candidate.attrs.end());
-      bufferlist can_bl;
-      can_bl.push_back(can_attr->second);
+      const bufferlist& can_bl = can_attr->second;
 
       auto auth_attr = auth.attrs.find(SS_ATTR);
       ceph_assert(auth_attr != auth.attrs.end());
-      bufferlist auth_bl;
-      auth_bl.push_back(auth_attr->second);
+      const bufferlist& auth_bl = auth_attr->second;
 
       if (!can_bl.contents_equal(auth_bl)) {
         fmt::format_to(std::back_inserter(out),
@@ -1279,13 +1271,11 @@ bool ScrubBackend::compare_obj_details(pg_shard_t auth_shard,
 
       auto can_hi = candidate.attrs.find(ECUtil::get_hinfo_key());
       ceph_assert(can_hi != candidate.attrs.end());
-      bufferlist can_bl;
-      can_bl.push_back(can_hi->second);
+      const bufferlist& can_bl = can_hi->second;
 
       auto auth_hi = auth.attrs.find(ECUtil::get_hinfo_key());
       ceph_assert(auth_hi != auth.attrs.end());
-      bufferlist auth_bl;
-      auth_bl.push_back(auth_hi->second);
+      const bufferlist& auth_bl = auth_hi->second;
 
       if (!can_bl.contents_equal(auth_bl)) {
         fmt::format_to(std::back_inserter(out),
@@ -1351,7 +1341,7 @@ bool ScrubBackend::compare_obj_details(pg_shard_t auth_shard,
 		     sep(error),
 		     k);
       obj_result.set_attr_name_mismatch();
-    } else if (cand->second.cmp(v)) {
+    } else if (!cand->second.contents_equal(v)) {
       fmt::format_to(std::back_inserter(out),
 		     "{}attr value mismatch '{}'",
 		     sep(error),
@@ -1463,10 +1453,8 @@ void ScrubBackend::scrub_snapshot_metadata(ScrubMap& map)
       this_chunk->m_error_counts.shallow_errors++;
       soid_error.set_info_missing();
     } else {
-      bufferlist bv;
-      bv.push_back(p->second.attrs[OI_ATTR]);
       try {
-        oi = object_info_t(bv);
+        oi = object_info_t(std::as_const(p->second.attrs[OI_ATTR]));
       } catch (ceph::buffer::error& e) {
         oi = std::nullopt;
         clog.error() << m_mode_desc << " " << m_pg_id << " " << soid
@@ -1592,13 +1580,11 @@ void ScrubBackend::scrub_snapshot_metadata(ScrubMap& map)
         snapset = std::nullopt;
         head_error.set_snapset_missing();
       } else {
-        bufferlist bl;
-        bl.push_back(p->second.attrs[SS_ATTR]);
-        auto blp = bl.cbegin();
+        auto blp = p->second.attrs[SS_ATTR].cbegin();
         try {
           snapset = SnapSet();  // Initialize optional<> before decoding into it
           decode(*snapset, blp);
-          head_error.ss_bl.push_back(p->second.attrs[SS_ATTR]);
+          head_error.ss_bl.append(p->second.attrs[SS_ATTR]);
         } catch (ceph::buffer::error& e) {
           snapset = std::nullopt;
           clog.error() << m_mode_desc << " " << m_pg_id << " " << soid
@@ -1789,13 +1775,11 @@ std::vector<snap_mapper_fix_t> ScrubBackend::scan_snaps(
 
     if (hoid.is_head()) {
       // parse the SnapSet
-      bufferlist bl;
       if (o.attrs.find(SS_ATTR) == o.attrs.end()) {
 	// no snaps for this head
 	continue;
       }
-      bl.push_back(o.attrs[SS_ATTR]);
-      auto p = bl.cbegin();
+      auto p = o.attrs[SS_ATTR].cbegin();
       try {
 	decode(snapset, p);
       } catch (...) {
diff --git a/src/test/crimson/test_crimson_scrub.cc b/src/test/crimson/test_crimson_scrub.cc
index 65b1f3152539..488d2246436d 100644
--- a/src/test/crimson/test_crimson_scrub.cc
+++ b/src/test/crimson/test_crimson_scrub.cc
@@ -24,22 +24,21 @@ constexpr static size_t TEST_OMAP_BYTES_LIMIT = 1<<30;
 
 void so_set_attr_len(ScrubMap::object &obj, const std::string &name, size_t len)
 {
-  obj.attrs[name] = buffer::ptr(len);
+  obj.attrs[name] = bufferlist();
+  obj.attrs[name].push_back(buffer::ptr(len));
 }
 
 void so_set_attr(ScrubMap::object &obj, const std::string &name, bufferlist bl)
 {
   bl.rebuild();
-  obj.attrs[name] = bl.front();
+  obj.attrs[name] = bl;
 }
 
 std::optional<bufferlist> so_get_attr(
   ScrubMap::object &obj, const std::string &name)
 {
   if (obj.attrs.count(name)) {
-    bufferlist bl;
-    bl.push_back(obj.attrs[name]);
-    return bl;
+    return obj.attrs[name];
   } else {
     return std::nullopt;
   }
diff --git a/src/test/osd/scrubber_generators.cc b/src/test/osd/scrubber_generators.cc
index 0f2f371e714b..19f64bb05ad8 100644
--- a/src/test/osd/scrubber_generators.cc
+++ b/src/test/osd/scrubber_generators.cc
@@ -8,7 +8,7 @@
 using namespace ScrubGenerator;
 
 // ref: PGLogTestRebuildMissing()
-bufferptr create_object_info(const ScrubGenerator::RealObj& objver)
+bufferlist create_object_info(const ScrubGenerator::RealObj& objver)
 {
   object_info_t oi{};
   oi.soid = objver.ghobj.hobj;
@@ -18,25 +18,23 @@ bufferptr create_object_info(const ScrubGenerator::RealObj& objver)
   bufferlist bl;
   oi.encode(bl,
 	    0 /*get_osdmap()->get_features(CEPH_ENTITY_TYPE_OSD, nullptr)*/);
-  bufferptr bp(bl.c_str(), bl.length());
-  return bp;
+  return bl;
 }
 
-std::pair<bufferptr, std::vector<snapid_t>> create_object_snapset(
+std::pair<bufferlist, std::vector<snapid_t>> create_object_snapset(
   const ScrubGenerator::RealObj& robj,
   const SnapsetMockData* snapset_mock_data)
 {
   if (!snapset_mock_data) {
-    return {bufferptr(), {}};
+    return {bufferlist(), {}};
   }
   /// \todo fill in missing version/osd details from the robj
   auto sns = snapset_mock_data->make_snapset();
   bufferlist bl;
   encode(sns, bl);
-  bufferptr bp = bufferptr(bl.c_str(), bl.length());
 
   // extract the set of object snaps
-  return {bp, sns.snaps};
+  return {bl, sns.snaps};
 }
 
 RealObjsConfList ScrubGenerator::make_real_objs_conf(
@@ -70,9 +68,9 @@ ScrubGenerator::SmapEntry ScrubGenerator::make_smobject(
   ret.ghobj = blueprint.ghobj;
   ret.smobj.attrs[OI_ATTR] = create_object_info(blueprint);
   if (blueprint.snapset_mock_data) {
-    auto [bp, snaps] =
+    auto [bl, snaps] =
       create_object_snapset(blueprint, blueprint.snapset_mock_data);
-    ret.smobj.attrs[SS_ATTR] = bp;
+    ret.smobj.attrs[SS_ATTR] = bl;
     std::cout << fmt::format("{}: ({}) osd:{} snaps:{}",
 			     __func__,
 			     ret.ghobj.hobj,
@@ -82,12 +80,12 @@ ScrubGenerator::SmapEntry ScrubGenerator::make_smobject(
   }
 
   for (const auto& [at_k, at_v] : blueprint.data.attrs) {
-    ret.smobj.attrs[at_k] = ceph::buffer::copy(at_v.c_str(), at_v.size());
+    // deep copy assignment
+    ret.smobj.attrs[at_k].clear();
+    ret.smobj.attrs[at_k].append(at_v.c_str(), at_v.size());
     {
       // verifying (to be removed after dev phase)
-      auto bk = ret.smobj.attrs[at_k].begin_deep().get_ptr(
-	ret.smobj.attrs[at_k].length());
-      std::string bkstr{bk.raw_c_str(), bk.raw_length()};
+      std::string bkstr = ret.smobj.attrs[at_k].to_str();
       std::cout << fmt::format("{}: verification: {}", __func__, bkstr)
 		<< std::endl;
     }

From 4917790bea02f00762c32f9865103e918be01971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 7 Nov 2023 17:30:51 +0100
Subject: [PATCH 1328/2492] osd: attrs are obligatory param for
 ECBackend::get_hash_info()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
(cherry picked from commit 149c61c180fadeae7fe5a1574aa8eba01bc0550b)
---
 src/osd/ECBackend.cc | 15 +++++++--------
 src/osd/ECBackend.h  |  2 +-
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 836bf96dde44..686e179f987a 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -518,7 +518,7 @@ void ECBackend::continue_recovery_op(
 
       if (op.recovery_progress.first && op.obc) {
 	/* We've got the attrs and the hinfo, might as well use them */
-	op.hinfo = get_hash_info(op.hoid, false, &op.obc->attr_cache);
+	op.hinfo = get_hash_info(op.hoid, false, op.obc->attr_cache);
 	if (!op.hinfo) {
           derr << __func__ << ": " << op.hoid << " has inconsistent hinfo"
                << dendl;
@@ -1015,7 +1015,7 @@ void ECBackend::handle_sub_read(
         map<string, bufferlist, less<>> attrs;
         int r = PGBackend::objects_get_attrs(i->first, &attrs);
         if (r >= 0) {
-          hinfo = get_hash_info(i->first, false, &attrs);
+          hinfo = get_hash_info(i->first, false, attrs);
 	} else {
           derr << "get_hash_info" << ": stat " << i->first << " failed: "
                << cpp_strerror(r) << dendl;
@@ -1377,7 +1377,7 @@ void ECBackend::submit_transaction(
     sinfo,
     *(op->t),
     [&](const hobject_t &i) {
-      ECUtil::HashInfoRef ref = get_hash_info(i, true, &op->t->obc_map[hoid]->attr_cache);
+      ECUtil::HashInfoRef ref = get_hash_info(i, true, op->t->obc_map[hoid]->attr_cache);
       if (!ref) {
 	derr << __func__ << ": get_hash_info(" << i << ")"
 	     << " returned a null pointer and there is no "
@@ -1394,7 +1394,7 @@ void ECBackend::submit_transaction(
 
 
 ECUtil::HashInfoRef ECBackend::get_hash_info(
-  const hobject_t &hoid, bool create, const map<string,bufferlist,less<>> *attrs)
+  const hobject_t &hoid, bool create, const map<string,bufferlist,less<>>& attrs)
 {
   dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
   ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
@@ -1409,9 +1409,8 @@ ECUtil::HashInfoRef ECBackend::get_hash_info(
     if (r >= 0) {
       dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
       bufferlist bl;
-      ceph_assert(attrs);
-      map<string, bufferlist>::const_iterator k = attrs->find(ECUtil::get_hinfo_key());
-      if (k == attrs->end()) {
+      map<string, bufferlist>::const_iterator k = attrs.find(ECUtil::get_hinfo_key());
+      if (k == attrs.end()) {
         dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
       } else {
         bl = k->second;
@@ -1677,7 +1676,7 @@ int ECBackend::be_deep_scrub(
     return -EINPROGRESS;
   }
 
-  ECUtil::HashInfoRef hinfo = get_hash_info(poid, false, &o.attrs);
+  ECUtil::HashInfoRef hinfo = get_hash_info(poid, false, o.attrs);
   if (!hinfo) {
     dout(0) << "_scan_list  " << poid << " could not retrieve hash info" << dendl;
     o.read_error = true;
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 27176bf18f92..a134ea58e695 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -326,7 +326,7 @@ class ECBackend : public PGBackend, public ECCommon {
   /// If modified, ensure that the ref is held until the update is applied
   SharedPtrRegistry<hobject_t, ECUtil::HashInfo> unstable_hashinfo_registry;
   ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool create,
-				    const std::map<std::string, ceph::buffer::list, std::less<>> *attr);
+				    const std::map<std::string, ceph::buffer::list, std::less<>>& attr);
 
 public:
   ECBackend(

From 3f10311b7a87118dc63c8092b9ecbf4df8c71c96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 8 Nov 2023 18:41:19 +0100
Subject: [PATCH 1329/2492] osd: ECBackend::get_hash_info() takes external
 object size
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This allows to reduce the interactions with `ObjectStore`
by letting callers `get_hash_info()` to provide the object
size from external source, from a cache in partiular.

The same idea will be reused in crimson but with an extra
benefit: no need for green threads within `submit_transaction()`
of `ECBackend`.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
(cherry picked from commit 98347a6013c9244c8768b1e570d36d0e17cd45d3)
---
 src/osd/ECBackend.cc | 94 +++++++++++++++++++++++++-------------------
 src/osd/ECBackend.h  |  8 +++-
 2 files changed, 59 insertions(+), 43 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 686e179f987a..8847d5b09073 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -518,7 +518,7 @@ void ECBackend::continue_recovery_op(
 
       if (op.recovery_progress.first && op.obc) {
 	/* We've got the attrs and the hinfo, might as well use them */
-	op.hinfo = get_hash_info(op.hoid, false, op.obc->attr_cache);
+	op.hinfo = get_hash_info(op.hoid, false, op.obc->attr_cache, op.obc->obs.oi.size);
 	if (!op.hinfo) {
           derr << __func__ << ": " << op.hoid << " has inconsistent hinfo"
                << dendl;
@@ -1013,12 +1013,17 @@ void ECBackend::handle_sub_read(
 	// the state of our chunk in case other chunks could substitute.
         ECUtil::HashInfoRef hinfo;
         map<string, bufferlist, less<>> attrs;
-        int r = PGBackend::objects_get_attrs(i->first, &attrs);
+	struct stat st;
+	int r = object_stat(i->first, &st);
         if (r >= 0) {
-          hinfo = get_hash_info(i->first, false, attrs);
+	  dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
+	  r = PGBackend::objects_get_attrs(i->first, &attrs);
+	}
+	if (r >= 0) {
+	  hinfo = get_hash_info(i->first, false, attrs, st.st_size);
 	} else {
-          derr << "get_hash_info" << ": stat " << i->first << " failed: "
-               << cpp_strerror(r) << dendl;
+	  derr << __func__ << ": access (attrs) on " << i->first << " failed: "
+	       << cpp_strerror(r) << dendl;
 	}
         if (!hinfo) {
           r = -EIO;
@@ -1377,7 +1382,11 @@ void ECBackend::submit_transaction(
     sinfo,
     *(op->t),
     [&](const hobject_t &i) {
-      ECUtil::HashInfoRef ref = get_hash_info(i, true, op->t->obc_map[hoid]->attr_cache);
+      ECUtil::HashInfoRef ref = get_hash_info(
+	i,
+	true,
+	op->t->obc_map[hoid]->attr_cache,
+	op->t->obc_map[hoid]->obs.oi.size);
       if (!ref) {
 	derr << __func__ << ": get_hash_info(" << i << ")"
 	     << " returned a null pointer and there is no "
@@ -1394,49 +1403,42 @@ void ECBackend::submit_transaction(
 
 
 ECUtil::HashInfoRef ECBackend::get_hash_info(
-  const hobject_t &hoid, bool create, const map<string,bufferlist,less<>>& attrs)
+  const hobject_t &hoid,
+  bool create,
+  const map<string, bufferlist, less<>>& attrs,
+  uint64_t size)
 {
   dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
   ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
   if (!ref) {
     dout(10) << __func__ << ": not in cache " << hoid << dendl;
-    struct stat st;
-    int r = store->stat(
-      ch,
-      ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
-      &st);
     ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
-    if (r >= 0) {
-      dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
-      bufferlist bl;
-      map<string, bufferlist>::const_iterator k = attrs.find(ECUtil::get_hinfo_key());
-      if (k == attrs.end()) {
-        dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
-      } else {
-        bl = k->second;
+    bufferlist bl;
+    map<string, bufferlist>::const_iterator k = attrs.find(ECUtil::get_hinfo_key());
+    if (k == attrs.end()) {
+      dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
+    } else {
+      bl = k->second;
+    }
+    if (bl.length() > 0) {
+      auto bp = bl.cbegin();
+      try {
+        decode(hinfo, bp);
+      } catch(...) {
+        dout(0) << __func__ << ": Can't decode hinfo for " << hoid << dendl;
+        return ECUtil::HashInfoRef();
       }
-      if (bl.length() > 0) {
-	auto bp = bl.cbegin();
-        try {
-	  decode(hinfo, bp);
-        } catch(...) {
-	  dout(0) << __func__ << ": Can't decode hinfo for " << hoid << dendl;
-	  return ECUtil::HashInfoRef();
-        }
-	if (hinfo.get_total_chunk_size() != (uint64_t)st.st_size) {
-	  dout(0) << __func__ << ": Mismatch of total_chunk_size "
-			       << hinfo.get_total_chunk_size() << dendl;
-	  return ECUtil::HashInfoRef();
-	}
-      } else if (st.st_size > 0) { // If empty object and no hinfo, create it
-	return ECUtil::HashInfoRef();
+      if (hinfo.get_total_chunk_size() != size) {
+        dout(0) << __func__ << ": Mismatch of total_chunk_size "
+      		       << hinfo.get_total_chunk_size() << dendl;
+        return ECUtil::HashInfoRef();
       }
-    } else if (r != -ENOENT || !create) {
-      derr << __func__ << ": stat " << hoid << " failed: " << cpp_strerror(r)
-           << dendl;
-      return ECUtil::HashInfoRef();
+    } else if (size == 0) { // If empty object and no hinfo, create it
+      create = true;
+    }
+    if (create) {
+      ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
     }
-    ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
   }
   return ref;
 }
@@ -1586,6 +1588,16 @@ void ECBackend::kick_reads() {
   read_pipeline.kick_reads();
 }
 
+int ECBackend::object_stat(
+  const hobject_t &hoid,
+  struct stat* st)
+{
+  int r = store->stat(
+    ch,
+    ghobject_t{hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard},
+    st);
+  return r;
+}
 
 int ECBackend::objects_get_attrs(
   const hobject_t &hoid,
@@ -1676,7 +1688,7 @@ int ECBackend::be_deep_scrub(
     return -EINPROGRESS;
   }
 
-  ECUtil::HashInfoRef hinfo = get_hash_info(poid, false, o.attrs);
+  ECUtil::HashInfoRef hinfo = get_hash_info(poid, false, o.attrs, o.size);
   if (!hinfo) {
     dout(0) << "_scan_list  " << poid << " could not retrieve hash info" << dendl;
     o.read_error = true;
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index a134ea58e695..0b3d79d8bf0a 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -325,8 +325,12 @@ class ECBackend : public PGBackend, public ECCommon {
   const ECUtil::stripe_info_t sinfo;
   /// If modified, ensure that the ref is held until the update is applied
   SharedPtrRegistry<hobject_t, ECUtil::HashInfo> unstable_hashinfo_registry;
-  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool create,
-				    const std::map<std::string, ceph::buffer::list, std::less<>>& attr);
+  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid,
+				    bool create,
+				    const std::map<std::string, ceph::buffer::list, std::less<>>& attr,
+				    uint64_t size);
+
+  int object_stat(const hobject_t &hoid, struct stat* st);
 
 public:
   ECBackend(

From d9d28a4eb45da81829ccaf69b9290b41a94167c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 28 Nov 2023 14:14:51 +0100
Subject: [PATCH 1330/2492] osd: dissect get_hash_info() with the registry into
 a dedicated class
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 29 +++++++++++++++++++++--------
 src/osd/ECBackend.h  | 30 ++++++++++++++++++++++++------
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 8847d5b09073..4a537766d0b5 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -126,6 +126,7 @@ ECBackend::ECBackend(
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
+    unstable_hashinfo_registry(cct, ec_impl),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
   ceph_assert((ec_impl->get_data_chunk_count() *
@@ -375,7 +376,7 @@ void ECBackend::handle_recovery_read_complete(
       auto bp = op.xattrs[ECUtil::get_hinfo_key()].cbegin();
       decode(hinfo, bp);
     }
-    op.hinfo = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+    op.hinfo = unstable_hashinfo_registry.maybe_put_hash_info(hoid, std::move(hinfo));
   }
   ceph_assert(op.xattrs.size());
   ceph_assert(op.obc);
@@ -501,6 +502,12 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
     std::make_unique<RecoveryReadCompleter>(*this));
 }
 
+void ECBackend::RecoveryBackend::continue_recovery_op(
+  RecoveryBackend::RecoveryOp &op,
+  RecoveryMessages *m)
+{
+}
+
 void ECBackend::continue_recovery_op(
   RecoveryOp &op,
   RecoveryMessages *m)
@@ -518,7 +525,7 @@ void ECBackend::continue_recovery_op(
 
       if (op.recovery_progress.first && op.obc) {
 	/* We've got the attrs and the hinfo, might as well use them */
-	op.hinfo = get_hash_info(op.hoid, false, op.obc->attr_cache, op.obc->obs.oi.size);
+	op.hinfo = unstable_hashinfo_registry.get_hash_info(op.hoid, false, op.obc->attr_cache, op.obc->obs.oi.size);
 	if (!op.hinfo) {
           derr << __func__ << ": " << op.hoid << " has inconsistent hinfo"
                << dendl;
@@ -1020,7 +1027,7 @@ void ECBackend::handle_sub_read(
 	  r = PGBackend::objects_get_attrs(i->first, &attrs);
 	}
 	if (r >= 0) {
-	  hinfo = get_hash_info(i->first, false, attrs, st.st_size);
+	  hinfo = unstable_hashinfo_registry.get_hash_info(i->first, false, attrs, st.st_size);
 	} else {
 	  derr << __func__ << ": access (attrs) on " << i->first << " failed: "
 	       << cpp_strerror(r) << dendl;
@@ -1382,7 +1389,7 @@ void ECBackend::submit_transaction(
     sinfo,
     *(op->t),
     [&](const hobject_t &i) {
-      ECUtil::HashInfoRef ref = get_hash_info(
+      ECUtil::HashInfoRef ref = unstable_hashinfo_registry.get_hash_info(
 	i,
 	true,
 	op->t->obc_map[hoid]->attr_cache,
@@ -1401,15 +1408,21 @@ void ECBackend::submit_transaction(
   rmw_pipeline.start_rmw(std::move(op));
 }
 
+ECUtil::HashInfoRef ECBackend::UnstableHashInfoRegistry::maybe_put_hash_info(
+  const hobject_t &hoid,
+  ECUtil::HashInfo &&hinfo)
+{
+  return registry.lookup_or_create(hoid, hinfo);
+}
 
-ECUtil::HashInfoRef ECBackend::get_hash_info(
+ECUtil::HashInfoRef ECBackend::UnstableHashInfoRegistry::get_hash_info(
   const hobject_t &hoid,
   bool create,
   const map<string, bufferlist, less<>>& attrs,
   uint64_t size)
 {
   dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
-  ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid);
+  ECUtil::HashInfoRef ref = registry.lookup(hoid);
   if (!ref) {
     dout(10) << __func__ << ": not in cache " << hoid << dendl;
     ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
@@ -1437,7 +1450,7 @@ ECUtil::HashInfoRef ECBackend::get_hash_info(
       create = true;
     }
     if (create) {
-      ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo);
+      ref = registry.lookup_or_create(hoid, hinfo);
     }
   }
   return ref;
@@ -1688,7 +1701,7 @@ int ECBackend::be_deep_scrub(
     return -EINPROGRESS;
   }
 
-  ECUtil::HashInfoRef hinfo = get_hash_info(poid, false, o.attrs, o.size);
+  ECUtil::HashInfoRef hinfo = unstable_hashinfo_registry.get_hash_info(poid, false, o.attrs, o.size);
   if (!hinfo) {
     dout(0) << "_scan_list  " << poid << " could not retrieve hash info" << dendl;
     o.read_error = true;
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 0b3d79d8bf0a..745ad4546b68 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -323,12 +323,30 @@ class ECBackend : public PGBackend, public ECCommon {
 
 
   const ECUtil::stripe_info_t sinfo;
-  /// If modified, ensure that the ref is held until the update is applied
-  SharedPtrRegistry<hobject_t, ECUtil::HashInfo> unstable_hashinfo_registry;
-  ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid,
-				    bool create,
-				    const std::map<std::string, ceph::buffer::list, std::less<>>& attr,
-				    uint64_t size);
+
+  class UnstableHashInfoRegistry {
+    CephContext *cct;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    /// If modified, ensure that the ref is held until the update is applied
+    SharedPtrRegistry<hobject_t, ECUtil::HashInfo> registry;
+
+  public:
+    UnstableHashInfoRegistry(
+      CephContext *cct,
+      ceph::ErasureCodeInterfaceRef ec_impl)
+      : cct(cct),
+	ec_impl(std::move(ec_impl)) {}
+
+    ECUtil::HashInfoRef maybe_put_hash_info(
+      const hobject_t &hoid,
+      ECUtil::HashInfo &&hinfo);
+
+    ECUtil::HashInfoRef get_hash_info(
+      const hobject_t &hoid,
+      bool create,
+      const std::map<std::string, ceph::buffer::list, std::less<>>& attr,
+      uint64_t size);
+  } unstable_hashinfo_registry;
 
   int object_stat(const hobject_t &hoid, struct stat* st);
 

From 09db3b9410bc41070a0ff9d849c0504029ca0993 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 28 Nov 2023 14:55:58 +0100
Subject: [PATCH 1331/2492] osd: shuffle UnstableHashInfoRegistry to ECCommon
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 48 ---------------------------------------
 src/osd/ECBackend.h  | 24 +-------------------
 src/osd/ECCommon.cc  | 53 ++++++++++++++++++++++++++++++++++++++++++++
 src/osd/ECCommon.h   | 25 +++++++++++++++++++++
 4 files changed, 79 insertions(+), 71 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 4a537766d0b5..472acdf88487 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1408,54 +1408,6 @@ void ECBackend::submit_transaction(
   rmw_pipeline.start_rmw(std::move(op));
 }
 
-ECUtil::HashInfoRef ECBackend::UnstableHashInfoRegistry::maybe_put_hash_info(
-  const hobject_t &hoid,
-  ECUtil::HashInfo &&hinfo)
-{
-  return registry.lookup_or_create(hoid, hinfo);
-}
-
-ECUtil::HashInfoRef ECBackend::UnstableHashInfoRegistry::get_hash_info(
-  const hobject_t &hoid,
-  bool create,
-  const map<string, bufferlist, less<>>& attrs,
-  uint64_t size)
-{
-  dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
-  ECUtil::HashInfoRef ref = registry.lookup(hoid);
-  if (!ref) {
-    dout(10) << __func__ << ": not in cache " << hoid << dendl;
-    ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
-    bufferlist bl;
-    map<string, bufferlist>::const_iterator k = attrs.find(ECUtil::get_hinfo_key());
-    if (k == attrs.end()) {
-      dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
-    } else {
-      bl = k->second;
-    }
-    if (bl.length() > 0) {
-      auto bp = bl.cbegin();
-      try {
-        decode(hinfo, bp);
-      } catch(...) {
-        dout(0) << __func__ << ": Can't decode hinfo for " << hoid << dendl;
-        return ECUtil::HashInfoRef();
-      }
-      if (hinfo.get_total_chunk_size() != size) {
-        dout(0) << __func__ << ": Mismatch of total_chunk_size "
-      		       << hinfo.get_total_chunk_size() << dendl;
-        return ECUtil::HashInfoRef();
-      }
-    } else if (size == 0) { // If empty object and no hinfo, create it
-      create = true;
-    }
-    if (create) {
-      ref = registry.lookup_or_create(hoid, hinfo);
-    }
-  }
-  return ref;
-}
-
 int ECBackend::objects_read_sync(
   const hobject_t &hoid,
   uint64_t off,
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 745ad4546b68..1f10269947a4 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -324,29 +324,7 @@ class ECBackend : public PGBackend, public ECCommon {
 
   const ECUtil::stripe_info_t sinfo;
 
-  class UnstableHashInfoRegistry {
-    CephContext *cct;
-    ceph::ErasureCodeInterfaceRef ec_impl;
-    /// If modified, ensure that the ref is held until the update is applied
-    SharedPtrRegistry<hobject_t, ECUtil::HashInfo> registry;
-
-  public:
-    UnstableHashInfoRegistry(
-      CephContext *cct,
-      ceph::ErasureCodeInterfaceRef ec_impl)
-      : cct(cct),
-	ec_impl(std::move(ec_impl)) {}
-
-    ECUtil::HashInfoRef maybe_put_hash_info(
-      const hobject_t &hoid,
-      ECUtil::HashInfo &&hinfo);
-
-    ECUtil::HashInfoRef get_hash_info(
-      const hobject_t &hoid,
-      bool create,
-      const std::map<std::string, ceph::buffer::list, std::less<>>& attr,
-      uint64_t size);
-  } unstable_hashinfo_registry;
+  ECCommon::UnstableHashInfoRegistry unstable_hashinfo_registry;
 
   int object_stat(const hobject_t &hoid, struct stat* st);
 
diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index e1bbfce7796c..2c6534d081f1 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -58,6 +58,11 @@ static ostream& _prefix(std::ostream *_dout, ECCommon::RMWPipeline *rmw_pipeline
 static ostream& _prefix(std::ostream *_dout, ECCommon::ReadPipeline *read_pipeline) {
   return read_pipeline->get_parent()->gen_dbg_prefix(*_dout);
 }
+static ostream& _prefix(std::ostream *_dout,
+			ECCommon::UnstableHashInfoRegistry *unstable_hash_info_registry) {
+  // TODO: backref to ECListener?
+  return *_dout;
+}
 
 ostream &operator<<(ostream &lhs, const ECCommon::RMWPipeline::pipeline_state_t &rhs) {
   switch (rhs.pipeline_state) {
@@ -1074,3 +1079,51 @@ void ECCommon::RMWPipeline::call_write_ordered(std::function<void(void)> &&cb) {
     cb();
   }
 }
+
+ECUtil::HashInfoRef ECCommon::UnstableHashInfoRegistry::maybe_put_hash_info(
+  const hobject_t &hoid,
+  ECUtil::HashInfo &&hinfo)
+{
+  return registry.lookup_or_create(hoid, hinfo);
+}
+
+ECUtil::HashInfoRef ECCommon::UnstableHashInfoRegistry::get_hash_info(
+  const hobject_t &hoid,
+  bool create,
+  const map<string, bufferlist, less<>>& attrs,
+  uint64_t size)
+{
+  dout(10) << __func__ << ": Getting attr on " << hoid << dendl;
+  ECUtil::HashInfoRef ref = registry.lookup(hoid);
+  if (!ref) {
+    dout(10) << __func__ << ": not in cache " << hoid << dendl;
+    ECUtil::HashInfo hinfo(ec_impl->get_chunk_count());
+    bufferlist bl;
+    map<string, bufferlist>::const_iterator k = attrs.find(ECUtil::get_hinfo_key());
+    if (k == attrs.end()) {
+      dout(5) << __func__ << " " << hoid << " missing hinfo attr" << dendl;
+    } else {
+      bl = k->second;
+    }
+    if (bl.length() > 0) {
+      auto bp = bl.cbegin();
+      try {
+        decode(hinfo, bp);
+      } catch(...) {
+        dout(0) << __func__ << ": Can't decode hinfo for " << hoid << dendl;
+        return ECUtil::HashInfoRef();
+      }
+      if (hinfo.get_total_chunk_size() != size) {
+        dout(0) << __func__ << ": Mismatch of total_chunk_size "
+      		       << hinfo.get_total_chunk_size() << dendl;
+        return ECUtil::HashInfoRef();
+      }
+    } else if (size == 0) { // If empty object and no hinfo, create it
+      create = true;
+    }
+    if (create) {
+      ref = registry.lookup_or_create(hoid, hinfo);
+    }
+  }
+  return ref;
+}
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index 46f8bb14772b..83e1712b0438 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -17,6 +17,7 @@
 #include <boost/intrusive/set.hpp>
 #include <boost/intrusive/list.hpp>
 
+#include "common/sharedptr_registry.hpp"
 #include "erasure-code/ErasureCodeInterface.h"
 #include "ECUtil.h"
 #if WITH_SEASTAR
@@ -581,6 +582,30 @@ struct ECCommon {
         ec_backend(ec_backend) {
     }
   };
+
+  class UnstableHashInfoRegistry {
+    CephContext *cct;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    /// If modified, ensure that the ref is held until the update is applied
+    SharedPtrRegistry<hobject_t, ECUtil::HashInfo> registry;
+
+  public:
+    UnstableHashInfoRegistry(
+      CephContext *cct,
+      ceph::ErasureCodeInterfaceRef ec_impl)
+      : cct(cct),
+	ec_impl(std::move(ec_impl)) {}
+
+    ECUtil::HashInfoRef maybe_put_hash_info(
+      const hobject_t &hoid,
+      ECUtil::HashInfo &&hinfo);
+
+    ECUtil::HashInfoRef get_hash_info(
+      const hobject_t &hoid,
+      bool create,
+      const std::map<std::string, ceph::buffer::list, std::less<>>& attr,
+      uint64_t size);
+  };
 };
 
 std::ostream &operator<<(std::ostream &lhs,

From deffa8209f9c0bd300cfdb54d358402bfc6e41c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Tue, 28 Nov 2023 12:58:01 +0100
Subject: [PATCH 1332/2492] osd: dissect the recovery-related parts out from
 ECBackend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For the sake of sharing with crimson.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc   | 165 ++++++++++++++++++++++++++---------------
 src/osd/ECBackend.h    |  87 ++++++++++++++++++----
 src/osd/ECCommon.h     |  70 +++++++++++++++++
 src/osd/PrimaryLogPG.h |   2 +
 4 files changed, 250 insertions(+), 74 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 472acdf88487..ecc5d5c82374 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -56,8 +56,12 @@ static ostream& _prefix(std::ostream *_dout, ECBackend *pgb) {
   return pgb->get_parent()->gen_dbg_prefix(*_dout);
 }
 
+static ostream& _prefix(std::ostream *_dout, ECBackend::RecoveryBackend *pgb) {
+  return pgb->get_parent()->gen_dbg_prefix(*_dout);
+}
+
 struct ECRecoveryHandle : public PGBackend::RecoveryHandle {
-  list<ECBackend::RecoveryOp> ops;
+  list<ECBackend::RecoveryBackend::RecoveryOp> ops;
 };
 
 static ostream &operator<<(ostream &lhs, const map<pg_shard_t, bufferlist> &rhs)
@@ -86,7 +90,7 @@ static ostream &operator<<(ostream &lhs, const map<int, bufferlist> &rhs)
   return lhs << "]";
 }
 
-ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
+ostream &operator<<(ostream &lhs, const ECBackend::RecoveryBackend::RecoveryOp &rhs)
 {
   return lhs << "RecoveryOp("
 	     << "hoid=" << rhs.hoid
@@ -96,13 +100,13 @@ ostream &operator<<(ostream &lhs, const ECBackend::RecoveryOp &rhs)
 	     << " recovery_info=" << rhs.recovery_info
 	     << " recovery_progress=" << rhs.recovery_progress
 	     << " obc refcount=" << rhs.obc.use_count()
-	     << " state=" << ECBackend::RecoveryOp::tostr(rhs.state)
+	     << " state=" << ECBackend::RecoveryBackend::RecoveryOp::tostr(rhs.state)
 	     << " waiting_on_pushes=" << rhs.waiting_on_pushes
 	     << " extent_requested=" << rhs.extent_requested
 	     << ")";
 }
 
-void ECBackend::RecoveryOp::dump(Formatter *f) const
+void ECBackend::RecoveryBackend::RecoveryOp::dump(Formatter *f) const
 {
   f->dump_stream("hoid") << hoid;
   f->dump_stream("v") << v;
@@ -126,19 +130,25 @@ ECBackend::ECBackend(
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
-    unstable_hashinfo_registry(cct, ec_impl),
+    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent()->get_eclistener()),
     ec_impl(ec_impl),
-    sinfo(ec_impl->get_data_chunk_count(), stripe_width) {
+    sinfo(ec_impl->get_data_chunk_count(), stripe_width),
+    unstable_hashinfo_registry(cct, ec_impl) {
   ceph_assert((ec_impl->get_data_chunk_count() *
 	  ec_impl->get_chunk_size(stripe_width)) == stripe_width);
 }
 
 PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
+{
+  return recovery_backend.open_recovery_op();
+}
+
+PGBackend::RecoveryHandle *ECBackend::RecoveryBackend::open_recovery_op()
 {
   return new ECRecoveryHandle;
 }
 
-void ECBackend::_failed_push(const hobject_t &hoid, ECCommon::read_result_t &res)
+void ECBackend::RecoveryBackend::_failed_push(const hobject_t &hoid, ECCommon::read_result_t &res)
 {
   dout(10) << __func__ << ": Read error " << hoid << " r="
 	   << res.r << " errors=" << res.errors << dendl;
@@ -188,6 +198,48 @@ void ECBackend::handle_recovery_push(
   const PushOp &op,
   RecoveryMessages *m,
   bool is_repair)
+{
+  if (get_parent()->pg_is_remote_backfilling()) {
+    get_parent()->pg_add_local_num_bytes(op.data.length());
+    get_parent()->pg_add_num_bytes(op.data.length() * get_ec_data_chunk_count());
+    dout(10) << __func__ << " " << op.soid
+             << " add new actual data by " << op.data.length()
+             << " add new num_bytes by " << op.data.length() * get_ec_data_chunk_count()
+             << dendl;
+  }
+
+  recovery_backend.handle_recovery_push(op, m, is_repair);
+
+  if (op.after_progress.data_complete) {
+    if ((get_parent()->pgb_is_primary())) {
+      if (get_parent()->pg_is_repair() || is_repair)
+        get_parent()->inc_osd_stat_repaired();
+    } else {
+      // If primary told us this is a repair, bump osd_stat_t::num_objects_repaired
+      if (is_repair)
+        get_parent()->inc_osd_stat_repaired();
+      if (get_parent()->pg_is_remote_backfilling()) {
+        struct stat st;
+        int r = store->stat(ch, ghobject_t(op.soid, ghobject_t::NO_GEN,
+                            get_parent()->whoami_shard().shard), &st);
+        if (r == 0) {
+          get_parent()->pg_sub_local_num_bytes(st.st_size);
+         // XXX: This can be way overestimated for small objects
+         get_parent()->pg_sub_num_bytes(st.st_size * get_ec_data_chunk_count());
+         dout(10) << __func__ << " " << op.soid
+                  << " sub actual data by " << st.st_size
+                  << " sub num_bytes by " << st.st_size * get_ec_data_chunk_count()
+                  << dendl;
+        }
+      }
+    }
+  }
+}
+
+void ECBackend::RecoveryBackend::handle_recovery_push(
+  const PushOp &op,
+  RecoveryMessages *m,
+  bool is_repair)
 {
   if (get_parent()->check_failsafe_full()) {
     dout(10) << __func__ << " Out of space (failsafe) processing push request." << dendl;
@@ -231,15 +283,6 @@ void ECBackend::handle_recovery_push(
     ceph_assert(op.data.length() == 0);
   }
 
-  if (get_parent()->pg_is_remote_backfilling()) {
-    get_parent()->pg_add_local_num_bytes(op.data.length());
-    get_parent()->pg_add_num_bytes(op.data.length() * get_ec_data_chunk_count());
-    dout(10) << __func__ << " " << op.soid
-             << " add new actual data by " << op.data.length()
-             << " add new num_bytes by " << op.data.length() * get_ec_data_chunk_count()
-             << dendl;
-  }
-
   if (op.before_progress.first) {
     ceph_assert(op.attrset.count(string("_")));
     m->t.setattrs(
@@ -281,27 +324,13 @@ void ECBackend::handle_recovery_push(
 	ObjectContextRef(),
 	false,
 	&m->t);
-      if (get_parent()->pg_is_remote_backfilling()) {
-        struct stat st;
-        int r = store->stat(ch, ghobject_t(op.soid, ghobject_t::NO_GEN,
-                            get_parent()->whoami_shard().shard), &st);
-        if (r == 0) {
-          get_parent()->pg_sub_local_num_bytes(st.st_size);
-         // XXX: This can be way overestimated for small objects
-         get_parent()->pg_sub_num_bytes(st.st_size * get_ec_data_chunk_count());
-         dout(10) << __func__ << " " << op.soid
-                  << " sub actual data by " << st.st_size
-                  << " sub num_bytes by " << st.st_size * get_ec_data_chunk_count()
-                  << dendl;
-        }
-      }
     }
   }
   m->push_replies[get_parent()->primary_shard()].push_back(PushReplyOp());
   m->push_replies[get_parent()->primary_shard()].back().soid = op.soid;
 }
 
-void ECBackend::handle_recovery_push_reply(
+void ECBackend::RecoveryBackend::handle_recovery_push_reply(
   const PushReplyOp &op,
   pg_shard_t from,
   RecoveryMessages *m)
@@ -314,7 +343,7 @@ void ECBackend::handle_recovery_push_reply(
   continue_recovery_op(rop, m);
 }
 
-void ECBackend::handle_recovery_read_complete(
+void ECBackend::RecoveryBackend::handle_recovery_read_complete(
   const hobject_t &hoid,
   boost::tuple<uint64_t, uint64_t, map<pg_shard_t, bufferlist> > &to_read,
   std::optional<map<string, bufferlist, less<>> > attrs,
@@ -327,7 +356,7 @@ void ECBackend::handle_recovery_read_complete(
 	   << ")"
 	   << dendl;
   ceph_assert(recovery_ops.count(hoid));
-  RecoveryOp &op = recovery_ops[hoid];
+  RecoveryBackend::RecoveryOp &op = recovery_ops[hoid];
   ceph_assert(op.returned_data.empty());
   map<int, bufferlist*> target;
   for (set<shard_id_t>::iterator i = op.missing_on_shards.begin();
@@ -417,7 +446,7 @@ struct SendPushReplies : public Context {
 };
 
 struct RecoveryReadCompleter : ECCommon::ReadCompleter {
-  RecoveryReadCompleter(ECBackend& backend)
+  RecoveryReadCompleter(ECBackend::RecoveryBackend& backend)
     : backend(backend) {}
 
   void finish_single_request(
@@ -442,27 +471,28 @@ struct RecoveryReadCompleter : ECCommon::ReadCompleter {
     backend.dispatch_recovery_messages(rm, priority);
   }
 
-  ECBackend& backend;
+  ECBackend::RecoveryBackend& backend;
   RecoveryMessages rm;
 };
 
-void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
+void ECBackend::RecoveryBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
 {
   for (map<pg_shard_t, vector<PushOp> >::iterator i = m.pushes.begin();
        i != m.pushes.end();
        m.pushes.erase(i++)) {
     MOSDPGPush *msg = new MOSDPGPush();
     msg->set_priority(priority);
-    msg->map_epoch = get_osdmap_epoch();
+    msg->map_epoch = get_parent()->pgb_get_osdmap_epoch();
     msg->min_epoch = get_parent()->get_last_peering_reset_epoch();
     msg->from = get_parent()->whoami_shard();
     msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
     msg->pushes.swap(i->second);
     msg->compute_cost(cct);
     msg->is_repair = get_parent()->pg_is_repair();
-    get_parent()->send_message(
-      i->first.osd,
-      msg);
+    std::vector wrapped_msg {
+      std::make_pair(i->first.osd, static_cast<Message*>(msg))
+    };
+    get_parent()->send_message_osd_cluster(wrapped_msg, msg->map_epoch);
   }
   map<int, MOSDPGPushReply*> replies;
   for (map<pg_shard_t, vector<PushReplyOp> >::iterator i =
@@ -471,7 +501,7 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
        m.push_replies.erase(i++)) {
     MOSDPGPushReply *msg = new MOSDPGPushReply();
     msg->set_priority(priority);
-    msg->map_epoch = get_osdmap_epoch();
+    msg->map_epoch = get_parent()->pgb_get_osdmap_epoch();
     msg->min_epoch = get_parent()->get_last_peering_reset_epoch();
     msg->from = get_parent()->whoami_shard();
     msg->pgid = spg_t(get_parent()->get_info().pgid.pgid, i->first.shard);
@@ -505,14 +535,9 @@ void ECBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
 void ECBackend::RecoveryBackend::continue_recovery_op(
   RecoveryBackend::RecoveryOp &op,
   RecoveryMessages *m)
-{
-}
-
-void ECBackend::continue_recovery_op(
-  RecoveryOp &op,
-  RecoveryMessages *m)
 {
   dout(10) << __func__ << ": continuing " << op << dendl;
+  using RecoveryOp = RecoveryBackend::RecoveryOp;
   while (1) {
     switch (op.state) {
     case RecoveryOp::IDLE: {
@@ -532,6 +557,7 @@ void ECBackend::continue_recovery_op(
           ceph_assert(recovery_ops.count(op.hoid));
           eversion_t v = recovery_ops[op.hoid].v;
           recovery_ops.erase(op.hoid);
+	  // TODO: not in crimson yet
           get_parent()->on_failed_pull({get_parent()->whoami_shard()},
                                        op.hoid, v);
           return;
@@ -548,6 +574,7 @@ void ECBackend::continue_recovery_op(
 	ceph_assert(!op.recovery_progress.first);
 	dout(10) << __func__ << ": canceling recovery op for obj " << op.hoid
 		 << dendl;
+	// in crimson
 	get_parent()->cancel_pull(op.hoid);
 	recovery_ops.erase(op.hoid);
 	return;
@@ -611,6 +638,7 @@ void ECBackend::continue_recovery_op(
 	pop.before_progress = op.recovery_progress;
 	pop.after_progress = after_progress;
 	if (*mi != get_parent()->primary_shard())
+	  // already in crimson -- junction point with PeeringState
 	  get_parent()->begin_peer_recover(
 	    *mi,
 	    op.hoid);
@@ -641,8 +669,10 @@ void ECBackend::continue_recovery_op(
 	  stat.num_bytes_recovered = op.recovery_info.size;
 	  stat.num_keys_recovered = 0; // ??? op ... omap_entries.size(); ?
 	  stat.num_objects_recovered = 1;
+	  // TODO: not in crimson yet
 	  if (get_parent()->pg_is_repair())
 	    stat.num_objects_repaired = 1;
+	  // pg_recovery.cc in crimson has it
 	  get_parent()->on_global_recover(op.hoid, stat, false);
 	  dout(10) << __func__ << ": WRITING return " << op << dendl;
 	  recovery_ops.erase(op.hoid);
@@ -668,20 +698,27 @@ void ECBackend::run_recovery_op(
   RecoveryHandle *_h,
   int priority)
 {
-  ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
+  ceph_assert(_h);
+  ECRecoveryHandle &h = static_cast<ECRecoveryHandle&>(*_h);
+  recovery_backend.run_recovery_op(h, priority);
+  send_recovery_deletes(priority, h.deletes);
+  delete _h;
+}
+
+void ECBackend::RecoveryBackend::run_recovery_op(
+  ECRecoveryHandle &h,
+  int priority)
+{
   RecoveryMessages m;
-  for (list<RecoveryOp>::iterator i = h->ops.begin();
-       i != h->ops.end();
+  for (list<RecoveryOp>::iterator i = h.ops.begin();
+       i != h.ops.end();
        ++i) {
     dout(10) << __func__ << ": starting " << *i << dendl;
     ceph_assert(!recovery_ops.count(i->hoid));
     RecoveryOp &op = recovery_ops.insert(make_pair(i->hoid, *i)).first->second;
     continue_recovery_op(op, &m);
   }
-
   dispatch_recovery_messages(m, priority);
-  send_recovery_deletes(priority, h->deletes);
-  delete _h;
 }
 
 int ECBackend::recover_object(
@@ -690,6 +727,16 @@ int ECBackend::recover_object(
   ObjectContextRef head,
   ObjectContextRef obc,
   RecoveryHandle *_h)
+{
+  return recovery_backend.recover_object(hoid, v, head, obc, _h);
+}
+
+int ECBackend::RecoveryBackend::recover_object(
+  const hobject_t &hoid,
+  eversion_t v,
+  ObjectContextRef head,
+  ObjectContextRef obc,
+  RecoveryHandle *_h)
 {
   ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
   h->ops.push_back(RecoveryOp());
@@ -786,7 +833,7 @@ bool ECBackend::_handle_message(
 	 ++i) {
       handle_recovery_push(*i, &rm, op->is_repair);
     }
-    dispatch_recovery_messages(rm, priority);
+    recovery_backend.dispatch_recovery_messages(rm, priority);
     return true;
   }
   case MSG_OSD_PG_PUSH_REPLY: {
@@ -796,9 +843,9 @@ bool ECBackend::_handle_message(
     for (vector<PushReplyOp>::const_iterator i = op->replies.begin();
 	 i != op->replies.end();
 	 ++i) {
-      handle_recovery_push_reply(*i, op->from, &rm);
+      recovery_backend.handle_recovery_push_reply(*i, op->from, &rm);
     }
-    dispatch_recovery_messages(rm, priority);
+    recovery_backend.dispatch_recovery_messages(rm, priority);
     return true;
   }
   default:
@@ -1285,14 +1332,14 @@ void ECBackend::on_change()
 
 void ECBackend::clear_recovery_state()
 {
-  recovery_ops.clear();
+  recovery_backend.recovery_ops.clear();
 }
 
 void ECBackend::dump_recovery_info(Formatter *f) const
 {
   f->open_array_section("recovery_ops");
-  for (map<hobject_t, RecoveryOp>::const_iterator i = recovery_ops.begin();
-       i != recovery_ops.end();
+  for (map<hobject_t, RecoveryBackend::RecoveryOp>::const_iterator i = recovery_backend.recovery_ops.begin();
+       i != recovery_backend.recovery_ops.end();
        ++i) {
     f->open_object_section("op");
     i->second.dump(f);
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 1f10269947a4..21f8ed29244c 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -158,11 +158,6 @@ class ECBackend : public PGBackend, public ECCommon {
 
   void kick_reads();
 
-  uint64_t get_recovery_chunk_size() const {
-    return round_up_to(cct->_conf->osd_recovery_max_chunk,
-			sinfo.get_stripe_width());
-  }
-
   /**
    * Recovery
    *
@@ -193,6 +188,40 @@ class ECBackend : public PGBackend, public ECCommon {
    * Transaction, and reads in a RecoveryMessages object which is passed
    * among the recovery methods.
    */
+public:
+  struct RecoveryBackend {
+    CephContext* cct;
+    const coll_t &coll;
+    ceph::ErasureCodeInterfaceRef ec_impl;
+    const ECUtil::stripe_info_t& sinfo;
+    ReadPipeline& read_pipeline;
+    UnstableHashInfoRegistry& unstable_hashinfo_registry;
+    // TODO: lay an interface down here
+    ECListener* parent;
+
+    ECListener *get_parent() const { return parent; }
+    const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
+    epoch_t get_osdmap_epoch() const { return get_parent()->pgb_get_osdmap_epoch(); }
+    const pg_info_t &get_info() { return get_parent()->get_info(); }
+    void add_temp_obj(const hobject_t &oid) { get_parent()->add_temp_obj(oid); }
+    void clear_temp_obj(const hobject_t &oid) { get_parent()->clear_temp_obj(oid); }
+
+    RecoveryBackend(CephContext* cct,
+		    const coll_t &coll,
+                ceph::ErasureCodeInterfaceRef ec_impl,
+                const ECUtil::stripe_info_t& sinfo,
+		ReadPipeline& read_pipeline,
+		UnstableHashInfoRegistry& unstable_hashinfo_registry,
+                ECListener* parent)
+      : cct(cct),
+        coll(coll),
+        ec_impl(std::move(ec_impl)),
+        sinfo(sinfo),
+	read_pipeline(read_pipeline),
+	unstable_hashinfo_registry(unstable_hashinfo_registry),
+        parent(parent) {
+    }
+  // <<<----
   struct RecoveryOp {
     hobject_t hoid;
     eversion_t v;
@@ -206,13 +235,13 @@ class ECBackend : public PGBackend, public ECCommon {
 
     static const char* tostr(state_t state) {
       switch (state) {
-      case ECBackend::RecoveryOp::IDLE:
+      case RecoveryOp::IDLE:
 	return "IDLE";
-      case ECBackend::RecoveryOp::READING:
+      case RecoveryOp::READING:
 	return "READING";
-      case ECBackend::RecoveryOp::WRITING:
+      case RecoveryOp::WRITING:
 	return "WRITING";
-      case ECBackend::RecoveryOp::COMPLETE:
+      case RecoveryOp::COMPLETE:
 	return "COMPLETE";
       default:
 	ceph_abort();
@@ -237,13 +266,26 @@ class ECBackend : public PGBackend, public ECCommon {
   friend ostream &operator<<(ostream &lhs, const RecoveryOp &rhs);
   std::map<hobject_t, RecoveryOp> recovery_ops;
 
+  uint64_t get_recovery_chunk_size() const {
+    return round_up_to(cct->_conf->osd_recovery_max_chunk,
+			sinfo.get_stripe_width());
+  }
+
+  void dispatch_recovery_messages(RecoveryMessages &m, int priority);
+
+  RecoveryHandle *open_recovery_op();
+  void run_recovery_op(
+    struct ECRecoveryHandle &h,
+    int priority);
+  int recover_object(
+    const hobject_t &hoid,
+    eversion_t v,
+    ObjectContextRef head,
+    ObjectContextRef obc,
+    RecoveryHandle *h);
   void continue_recovery_op(
-    RecoveryOp &op,
+    RecoveryBackend::RecoveryOp &op,
     RecoveryMessages *m);
-  friend struct RecoveryMessages;
-  void dispatch_recovery_messages(RecoveryMessages &m, int priority);
-  friend struct OnRecoveryReadComplete;
-  friend struct RecoveryReadCompleter;
   void handle_recovery_read_complete(
     const hobject_t &hoid,
     boost::tuple<uint64_t, uint64_t, std::map<pg_shard_t, ceph::buffer::list> > &to_read,
@@ -257,10 +299,26 @@ class ECBackend : public PGBackend, public ECCommon {
     const PushReplyOp &op,
     pg_shard_t from,
     RecoveryMessages *m);
+  friend struct RecoveryMessages;
+  int get_ec_data_chunk_count() const {
+    return ec_impl->get_data_chunk_count();
+  }
+  void _failed_push(const hobject_t &hoid, ECCommon::read_result_t &res);
+  };
+  friend ostream &operator<<(ostream &lhs, const RecoveryBackend::RecoveryOp &rhs);
+  friend struct RecoveryMessages;
+  friend struct OnRecoveryReadComplete;
+  friend struct RecoveryReadCompleter;
+
+  void handle_recovery_push(
+    const PushOp &op,
+    RecoveryMessages *m,
+    bool is_repair);
 
 public:
   struct ReadPipeline read_pipeline;
   struct RMWPipeline rmw_pipeline;
+  struct RecoveryBackend recovery_backend;
 
   ceph::ErasureCodeInterfaceRef ec_impl;
 
@@ -358,7 +416,6 @@ class ECBackend : public PGBackend, public ECCommon {
   uint64_t be_get_ondisk_size(uint64_t logical_size) const final {
     return sinfo.logical_to_next_chunk_offset(logical_size);
   }
-  void _failed_push(const hobject_t &hoid, ECBackend::read_result_t &res);
 };
 ostream &operator<<(ostream &lhs, const ECBackend::RMWPipeline::pipeline_state_t &rhs);
 
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index 83e1712b0438..2ef088f5bf1c 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -65,6 +65,76 @@ struct ECListener {
   // XXX
   virtual void cancel_pull(
     const hobject_t &soid) = 0;
+
+#ifndef WITH_SEASTAR
+  // XXX
+  virtual pg_shard_t primary_shard() const = 0;
+  virtual bool pgb_is_primary() const = 0;
+
+  /**
+   * Called when a read from a std::set of replicas/primary fails
+   */
+  virtual void on_failed_pull(
+    const std::set<pg_shard_t> &from,
+    const hobject_t &soid,
+    const eversion_t &v
+    ) = 0;
+
+     /**
+      * Called with the transaction recovering oid
+      */
+     virtual void on_local_recover(
+       const hobject_t &oid,
+       const ObjectRecoveryInfo &recovery_info,
+       ObjectContextRef obc,
+       bool is_delete,
+       ceph::os::Transaction *t
+       ) = 0;
+
+  /**
+   * Called when transaction recovering oid is durable and
+   * applied on all replicas
+   */
+  virtual void on_global_recover(
+    const hobject_t &oid,
+    const object_stat_sum_t &stat_diff,
+    bool is_delete
+    ) = 0;
+
+  /**
+   * Called when peer is recovered
+   */
+  virtual void on_peer_recover(
+    pg_shard_t peer,
+    const hobject_t &oid,
+    const ObjectRecoveryInfo &recovery_info
+    ) = 0;
+
+  virtual void begin_peer_recover(
+    pg_shard_t peer,
+    const hobject_t oid) = 0;
+
+  virtual bool pg_is_repair() const = 0;
+
+     virtual ObjectContextRef get_obc(
+       const hobject_t &hoid,
+       const std::map<std::string, ceph::buffer::list, std::less<>> &attrs) = 0;
+
+     virtual bool check_failsafe_full() = 0;
+     virtual hobject_t get_temp_recovery_object(const hobject_t& target,
+						eversion_t version) = 0;
+     virtual bool pg_is_remote_backfilling() = 0;
+     virtual void pg_add_local_num_bytes(int64_t num_bytes) = 0;
+     //virtual void pg_sub_local_num_bytes(int64_t num_bytes) = 0;
+     virtual void pg_add_num_bytes(int64_t num_bytes) = 0;
+     //virtual void pg_sub_num_bytes(int64_t num_bytes) = 0;
+     virtual void inc_osd_stat_repaired() = 0;
+
+   virtual void add_temp_obj(const hobject_t &oid) = 0;
+   virtual void clear_temp_obj(const hobject_t &oid) = 0;
+     virtual epoch_t get_last_peering_reset_epoch() const = 0;
+#endif
+
   // XXX
 #ifndef WITH_SEASTAR
   virtual GenContext<ThreadPool::TPHandle&> *bless_unlocked_gencontext(
diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 420871b1990e..85849bb85b93 100644
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -304,6 +304,8 @@ class PrimaryLogPG : public PG,
   }
 
   /// Listener methods
+  void add_temp_obj(const hobject_t &oid) override { get_pgbackend()->add_temp_obj(oid); }
+  void clear_temp_obj(const hobject_t &oid) override { get_pgbackend()->clear_temp_obj(oid); }
   DoutPrefixProvider *get_dpp() override {
     return this;
   }

From a7222fd65cd9ee518bc8a90cc13db4d5184f231e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 28 Sep 2023 18:23:47 +0200
Subject: [PATCH 1333/2492] osd: make ECCommon compatible with libfmt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For the crimson's sake.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECCommon.h   | 6 ++++++
 src/osd/ECMsgTypes.h | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index 2ef088f5bf1c..f074a79abaf8 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -16,6 +16,7 @@
 
 #include <boost/intrusive/set.hpp>
 #include <boost/intrusive/list.hpp>
+#include <fmt/format.h>
 
 #include "common/sharedptr_registry.hpp"
 #include "erasure-code/ErasureCodeInterface.h"
@@ -689,3 +690,8 @@ std::ostream &operator<<(std::ostream &lhs,
 std::ostream &operator<<(std::ostream &lhs,
 			 const ECCommon::RMWPipeline::Op &rhs);
 
+template <> struct fmt::formatter<ECCommon::RMWPipeline::pipeline_state_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECCommon::read_request_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECCommon::read_result_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECCommon::ReadOp> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECCommon::RMWPipeline::Op> : fmt::ostream_formatter {};
diff --git a/src/osd/ECMsgTypes.h b/src/osd/ECMsgTypes.h
index bf10704125c0..2d0bc5c12216 100644
--- a/src/osd/ECMsgTypes.h
+++ b/src/osd/ECMsgTypes.h
@@ -15,6 +15,8 @@
 #ifndef ECBMSGTYPES_H
 #define ECBMSGTYPES_H
 
+#include <fmt/format.h>
+
 #include "osd_types.h"
 #include "include/buffer.h"
 #include "os/ObjectStore.h"
@@ -137,4 +139,9 @@ std::ostream &operator<<(
 std::ostream &operator<<(
   std::ostream &lhs, const ECSubReadReply &rhs);
 
+template <> struct fmt::formatter<ECSubWrite> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECSubWriteReply> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECSubRead> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<ECSubReadReply> : fmt::ostream_formatter {};
+
 #endif

From e46590f76236cd1428e947fcbd14aacf58f238b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Wed, 13 Dec 2023 15:08:08 +0100
Subject: [PATCH 1334/2492] osd: move templated
 ReadPipeline::check_recovery_sources() to .h
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Behind shluffing, this commit includes also adding `std` namespaces
to dissecting the `schedule_recovery_work()` to keep it in .cc.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECCommon.cc | 98 +--------------------------------------------
 src/osd/ECCommon.h  | 98 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+), 96 deletions(-)

diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index 2c6534d081f1..5d7c7a48cb20 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -194,111 +194,17 @@ struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
   }
 };
 
-template <class F>
-void ECCommon::ReadPipeline::filter_read_op(
-  const OSDMapRef& osdmap,
-  ReadOp &op,
-  F&& on_erase)
+void ECCommon::ReadPipeline::schedule_recovery_work()
 {
-  set<hobject_t> to_cancel;
-  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
-       i != op.source_to_obj.end();
-       ++i) {
-    if (osdmap->is_down(i->first.osd)) {
-      to_cancel.insert(i->second.begin(), i->second.end());
-      op.in_progress.erase(i->first);
-      continue;
-    }
-  }
-
-  if (to_cancel.empty())
-    return;
-
-  for (map<pg_shard_t, set<hobject_t> >::iterator i = op.source_to_obj.begin();
-       i != op.source_to_obj.end();
-       ) {
-    for (set<hobject_t>::iterator j = i->second.begin();
-	 j != i->second.end();
-	 ) {
-      if (to_cancel.count(*j))
-	i->second.erase(j++);
-      else
-	++j;
-    }
-    if (i->second.empty()) {
-      op.source_to_obj.erase(i++);
-    } else {
-      ceph_assert(!osdmap->is_down(i->first.osd));
-      ++i;
-    }
-  }
-
-  for (set<hobject_t>::iterator i = to_cancel.begin();
-       i != to_cancel.end();
-       ++i) {
-    get_parent()->cancel_pull(*i);
-
-    ceph_assert(op.to_read.count(*i));
-    read_request_t &req = op.to_read.find(*i)->second;
-    dout(10) << __func__ << ": canceling " << req
-	     << "  for obj " << *i << dendl;
-    op.to_read.erase(*i);
-    op.complete.erase(*i);
-    on_erase(*i);
-  }
-
-  if (op.in_progress.empty()) {
-    /* This case is odd.  filter_read_op gets called while processing
-     * an OSDMap.  Normal, non-recovery reads only happen from acting
-     * set osds.  For this op to have had a read source go down and
-     * there not be an interval change, it must be part of a pull during
-     * log-based recovery.
-     *
-     * This callback delays calling complete_read_op until later to avoid
-     * dealing with recovery while handling an OSDMap.  We assign a
-     * cost here of 1 because:
-     * 1) This should be very rare, and the operation itself was already
-     *    throttled.
-     * 2) It shouldn't result in IO, rather it should result in restarting
-     *    the pull on the affected objects and pushes from in-memory buffers
-     *    on any now complete unaffected objects.
-     */
 #ifndef WITH_SEASTAR
     get_parent()->schedule_recovery_work(
       get_parent()->bless_unlocked_gencontext(
-        new FinishReadOp(*this, op.tid)),
+        nullptr), //new struct FinishReadOp(*this, op.tid)),
       1);
 #else
     // TODO
     ceph_abort_msg("not yet implemented");
 #endif
-  }
-}
-
-template <class F>
-void ECCommon::ReadPipeline::check_recovery_sources(
-  const OSDMapRef& osdmap,
-  F&& on_erase)
-{
-  set<ceph_tid_t> tids_to_filter;
-  for (map<pg_shard_t, set<ceph_tid_t> >::iterator 
-       i = shard_to_read_map.begin();
-       i != shard_to_read_map.end();
-       ) {
-    if (osdmap->is_down(i->first.osd)) {
-      tids_to_filter.insert(i->second.begin(), i->second.end());
-      shard_to_read_map.erase(i++);
-    } else {
-      ++i;
-    }
-  }
-  for (set<ceph_tid_t>::iterator i = tids_to_filter.begin();
-       i != tids_to_filter.end();
-       ++i) {
-    map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
-    ceph_assert(j != tid_to_read_map.end());
-    filter_read_op(osdmap, j->second, on_erase);
-  }
 }
 
 void ECCommon::ReadPipeline::on_change()
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index f074a79abaf8..1e30b6d80ff9 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -453,6 +453,8 @@ struct ECCommon {
       bool do_redundant_reads,   ///< [in] true if we want to issue redundant reads to reduce latency
       std::map<pg_shard_t, std::vector<std::pair<int, int>>> *to_read   ///< [out] shards, corresponding subchunks to read
       ); ///< @return error code, 0 on success
+
+    void schedule_recovery_work();
   };
 
   /**
@@ -695,3 +697,99 @@ template <> struct fmt::formatter<ECCommon::read_request_t> : fmt::ostream_forma
 template <> struct fmt::formatter<ECCommon::read_result_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<ECCommon::ReadOp> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<ECCommon::RMWPipeline::Op> : fmt::ostream_formatter {};
+
+template <class F>
+void ECCommon::ReadPipeline::check_recovery_sources(
+  const OSDMapRef& osdmap,
+  F&& on_erase)
+{
+  std::set<ceph_tid_t> tids_to_filter;
+  for (std::map<pg_shard_t, std::set<ceph_tid_t> >::iterator 
+       i = shard_to_read_map.begin();
+       i != shard_to_read_map.end();
+       ) {
+    if (osdmap->is_down(i->first.osd)) {
+      tids_to_filter.insert(i->second.begin(), i->second.end());
+      shard_to_read_map.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+  for (std::set<ceph_tid_t>::iterator i = tids_to_filter.begin();
+       i != tids_to_filter.end();
+       ++i) {
+    std::map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
+    ceph_assert(j != tid_to_read_map.end());
+    filter_read_op(osdmap, j->second, on_erase);
+  }
+}
+
+template <class F>
+void ECCommon::ReadPipeline::filter_read_op(
+  const OSDMapRef& osdmap,
+  ReadOp &op,
+  F&& on_erase)
+{
+  std::set<hobject_t> to_cancel;
+  for (std::map<pg_shard_t, std::set<hobject_t> >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ++i) {
+    if (osdmap->is_down(i->first.osd)) {
+      to_cancel.insert(i->second.begin(), i->second.end());
+      op.in_progress.erase(i->first);
+      continue;
+    }
+  }
+
+  if (to_cancel.empty())
+    return;
+
+  for (std::map<pg_shard_t, std::set<hobject_t> >::iterator i = op.source_to_obj.begin();
+       i != op.source_to_obj.end();
+       ) {
+    for (std::set<hobject_t>::iterator j = i->second.begin();
+	 j != i->second.end();
+	 ) {
+      if (to_cancel.count(*j))
+	i->second.erase(j++);
+      else
+	++j;
+    }
+    if (i->second.empty()) {
+      op.source_to_obj.erase(i++);
+    } else {
+      ceph_assert(!osdmap->is_down(i->first.osd));
+      ++i;
+    }
+  }
+
+  for (std::set<hobject_t>::iterator i = to_cancel.begin();
+       i != to_cancel.end();
+       ++i) {
+    get_parent()->cancel_pull(*i);
+
+    ceph_assert(op.to_read.count(*i));
+    op.to_read.erase(*i);
+    op.complete.erase(*i);
+    on_erase(*i);
+  }
+
+  if (op.in_progress.empty()) {
+    /* This case is odd.  filter_read_op gets called while processing
+     * an OSDMap.  Normal, non-recovery reads only happen from acting
+     * set osds.  For this op to have had a read source go down and
+     * there not be an interval change, it must be part of a pull during
+     * log-based recovery.
+     *
+     * This callback delays calling complete_read_op until later to avoid
+     * dealing with recovery while handling an OSDMap.  We assign a
+     * cost here of 1 because:
+     * 1) This should be very rare, and the operation itself was already
+     *    throttled.
+     * 2) It shouldn't result in IO, rather it should result in restarting
+     *    the pull on the affected objects and pushes from in-memory buffers
+     *    on any now complete unaffected objects.
+     */
+    schedule_recovery_work();
+  }
+}

From da502e7127654ccb647b560e22ae69247f059f5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 15 Dec 2023 22:59:03 +0100
Subject: [PATCH 1335/2492] osd: abstract EC read pipeline's
 check_recovery_sources() from IO
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For the sake of crimson.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 27 ++++++++++++++++++++++-----
 src/osd/ECCommon.cc  | 25 -------------------------
 src/osd/ECCommon.h   | 26 ++++++++++++++++----------
 3 files changed, 38 insertions(+), 40 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index ecc5d5c82374..8ae0fe8c5c5c 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1316,11 +1316,28 @@ void ECBackend::handle_sub_read_reply(
 
 void ECBackend::check_recovery_sources(const OSDMapRef& osdmap)
 {
-#if 0
-  read_pipeline.check_recovery_sources(osdmap, [this] (const hobject_t& obj) {
-    recovery_ops.erase(obj);
-  });
-#endif
+  struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
+    ECCommon::ReadPipeline& read_pipeline;
+    ceph_tid_t tid;
+    FinishReadOp(ECCommon::ReadPipeline& read_pipeline, ceph_tid_t tid)
+      : read_pipeline(read_pipeline), tid(tid) {}
+    void finish(ThreadPool::TPHandle&) override {
+      auto ropiter = read_pipeline.tid_to_read_map.find(tid);
+      ceph_assert(ropiter != read_pipeline.tid_to_read_map.end());
+      read_pipeline.complete_read_op(ropiter->second);
+    }
+  };
+  read_pipeline.check_recovery_sources(
+    osdmap,
+    [this] (const hobject_t& obj) {
+      recovery_backend.recovery_ops.erase(obj);
+    },
+    [this] (const ReadOp& op) {
+      get_parent()->schedule_recovery_work(
+        get_parent()->bless_unlocked_gencontext(
+          new FinishReadOp(read_pipeline, op.tid)),
+        1);
+    });
 }
 
 void ECBackend::on_change()
diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index 5d7c7a48cb20..2d7acf15d18e 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -182,31 +182,6 @@ void ECCommon::ReadPipeline::complete_read_op(ReadOp &rop)
   tid_to_read_map.erase(rop.tid);
 }
 
-struct FinishReadOp : public GenContext<ThreadPool::TPHandle&>  {
-  ECCommon::ReadPipeline& read_pipeline;
-  ceph_tid_t tid;
-  FinishReadOp(ECCommon::ReadPipeline& read_pipeline, ceph_tid_t tid)
-    : read_pipeline(read_pipeline), tid(tid) {}
-  void finish(ThreadPool::TPHandle&) override {
-    auto ropiter = read_pipeline.tid_to_read_map.find(tid);
-    ceph_assert(ropiter != read_pipeline.tid_to_read_map.end());
-    read_pipeline.complete_read_op(ropiter->second);
-  }
-};
-
-void ECCommon::ReadPipeline::schedule_recovery_work()
-{
-#ifndef WITH_SEASTAR
-    get_parent()->schedule_recovery_work(
-      get_parent()->bless_unlocked_gencontext(
-        nullptr), //new struct FinishReadOp(*this, op.tid)),
-      1);
-#else
-    // TODO
-    ceph_abort_msg("not yet implemented");
-#endif
-}
-
 void ECCommon::ReadPipeline::on_change()
 {
   for (map<ceph_tid_t, ReadOp>::iterator i = tid_to_read_map.begin();
diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h
index 1e30b6d80ff9..3ceb3d295b2a 100644
--- a/src/osd/ECCommon.h
+++ b/src/osd/ECCommon.h
@@ -370,14 +370,18 @@ struct ECCommon {
       bool fast_read,
       GenContextURef<std::map<hobject_t,std::pair<int, extent_map> > &&> &&func);
 
-    template <class F>
+    template <class F, class G>
     void filter_read_op(
       const OSDMapRef& osdmap,
       ReadOp &op,
-      F&& on_erase);
+      F&& on_erase,
+      G&& on_schedule_recovery);
 
-    template <class F>
-    void check_recovery_sources(const OSDMapRef& osdmap, F&& on_erase);
+    template <class F, class G>
+    void check_recovery_sources(
+      const OSDMapRef& osdmap,
+      F&& on_erase,
+      G&& on_schedule_recovery);
 
     void complete_read_op(ReadOp &rop);
 
@@ -698,10 +702,11 @@ template <> struct fmt::formatter<ECCommon::read_result_t> : fmt::ostream_format
 template <> struct fmt::formatter<ECCommon::ReadOp> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<ECCommon::RMWPipeline::Op> : fmt::ostream_formatter {};
 
-template <class F>
+template <class F, class G>
 void ECCommon::ReadPipeline::check_recovery_sources(
   const OSDMapRef& osdmap,
-  F&& on_erase)
+  F&& on_erase,
+  G&& on_schedule_recovery)
 {
   std::set<ceph_tid_t> tids_to_filter;
   for (std::map<pg_shard_t, std::set<ceph_tid_t> >::iterator 
@@ -720,15 +725,16 @@ void ECCommon::ReadPipeline::check_recovery_sources(
        ++i) {
     std::map<ceph_tid_t, ReadOp>::iterator j = tid_to_read_map.find(*i);
     ceph_assert(j != tid_to_read_map.end());
-    filter_read_op(osdmap, j->second, on_erase);
+    filter_read_op(osdmap, j->second, on_erase, on_schedule_recovery);
   }
 }
 
-template <class F>
+template <class F, class G>
 void ECCommon::ReadPipeline::filter_read_op(
   const OSDMapRef& osdmap,
   ReadOp &op,
-  F&& on_erase)
+  F&& on_erase,
+  G&& on_schedule_recovery)
 {
   std::set<hobject_t> to_cancel;
   for (std::map<pg_shard_t, std::set<hobject_t> >::iterator i = op.source_to_obj.begin();
@@ -790,6 +796,6 @@ void ECCommon::ReadPipeline::filter_read_op(
      *    the pull on the affected objects and pushes from in-memory buffers
      *    on any now complete unaffected objects.
      */
-    schedule_recovery_work();
+    on_schedule_recovery(op);
   }
 }

From ea06991506345e453027e417c3e675bc21cfa0fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Fri, 15 Dec 2023 23:07:37 +0100
Subject: [PATCH 1336/2492] osd: abstract EC recovery's
 dispatch_recovery_messages() from IO
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 44 +++++++++++++++++++++++++++++++++---------
 src/osd/ECBackend.h  | 46 +++++++++++++++++++++++++++++---------------
 2 files changed, 66 insertions(+), 24 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 8ae0fe8c5c5c..318b1aca1f2f 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -130,7 +130,7 @@ ECBackend::ECBackend(
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
-    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent()->get_eclistener()),
+    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent()),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width),
     unstable_hashinfo_registry(cct, ec_impl) {
@@ -143,6 +143,23 @@ PGBackend::RecoveryHandle *ECBackend::open_recovery_op()
   return recovery_backend.open_recovery_op();
 }
 
+ECBackend::RecoveryBackend::RecoveryBackend(
+  CephContext* cct,
+  const coll_t &coll,
+  ceph::ErasureCodeInterfaceRef ec_impl,
+  const ECUtil::stripe_info_t& sinfo,
+  ReadPipeline& read_pipeline,
+  UnstableHashInfoRegistry& unstable_hashinfo_registry,
+  ECListener* parent)
+  : cct(cct),
+    coll(coll),
+    ec_impl(std::move(ec_impl)),
+    sinfo(sinfo),
+    read_pipeline(read_pipeline),
+    unstable_hashinfo_registry(unstable_hashinfo_registry),
+    parent(parent) {
+}
+
 PGBackend::RecoveryHandle *ECBackend::RecoveryBackend::open_recovery_op()
 {
   return new ECRecoveryHandle;
@@ -475,6 +492,19 @@ struct RecoveryReadCompleter : ECCommon::ReadCompleter {
   RecoveryMessages rm;
 };
 
+void ECBackend::ECRecoveryBackend::commit_txn_send_replies(
+  ceph::os::Transaction&& txn,
+  std::map<int, MOSDPGPushReply*> replies)
+{
+  txn.register_on_complete(
+      get_parent()->bless_context(
+        new SendPushReplies(
+          get_parent(),
+          get_osdmap_epoch(),
+          replies)));
+  get_parent()->queue_transaction(std::move(txn));
+}
+
 void ECBackend::RecoveryBackend::dispatch_recovery_messages(RecoveryMessages &m, int priority)
 {
   for (map<pg_shard_t, vector<PushOp> >::iterator i = m.pushes.begin();
@@ -510,15 +540,11 @@ void ECBackend::RecoveryBackend::dispatch_recovery_messages(RecoveryMessages &m,
     replies.insert(make_pair(i->first.osd, msg));
   }
 
+#if 1
   if (!replies.empty()) {
-    (m.t).register_on_complete(
-	get_parent()->bless_context(
-	  new SendPushReplies(
-	    get_parent(),
-	    get_osdmap_epoch(),
-	    replies)));
-    get_parent()->queue_transaction(std::move(m.t));
-  } 
+    commit_txn_send_replies(std::move(m.t), std::move(replies));
+  }
+#endif
 
   if (m.recovery_reads.empty())
     return;
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 21f8ed29244c..56e1c972fe24 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -208,20 +208,11 @@ class ECBackend : public PGBackend, public ECCommon {
 
     RecoveryBackend(CephContext* cct,
 		    const coll_t &coll,
-                ceph::ErasureCodeInterfaceRef ec_impl,
-                const ECUtil::stripe_info_t& sinfo,
-		ReadPipeline& read_pipeline,
-		UnstableHashInfoRegistry& unstable_hashinfo_registry,
-                ECListener* parent)
-      : cct(cct),
-        coll(coll),
-        ec_impl(std::move(ec_impl)),
-        sinfo(sinfo),
-	read_pipeline(read_pipeline),
-	unstable_hashinfo_registry(unstable_hashinfo_registry),
-        parent(parent) {
-    }
-  // <<<----
+		    ceph::ErasureCodeInterfaceRef ec_impl,
+		    const ECUtil::stripe_info_t& sinfo,
+		    ReadPipeline& read_pipeline,
+		    UnstableHashInfoRegistry& unstable_hashinfo_registry,
+		    ECListener* parent);
   struct RecoveryOp {
     hobject_t hoid;
     eversion_t v;
@@ -271,6 +262,10 @@ class ECBackend : public PGBackend, public ECCommon {
 			sinfo.get_stripe_width());
   }
 
+  virtual ~RecoveryBackend() = default;
+  virtual void commit_txn_send_replies(
+    ceph::os::Transaction&& txn,
+    std::map<int, MOSDPGPushReply*> replies) = 0;
   void dispatch_recovery_messages(RecoveryMessages &m, int priority);
 
   RecoveryHandle *open_recovery_op();
@@ -305,6 +300,27 @@ class ECBackend : public PGBackend, public ECCommon {
   }
   void _failed_push(const hobject_t &hoid, ECCommon::read_result_t &res);
   };
+  struct ECRecoveryBackend : RecoveryBackend {
+    ECRecoveryBackend(CephContext* cct,
+		      const coll_t &coll,
+		      ceph::ErasureCodeInterfaceRef ec_impl,
+		      const ECUtil::stripe_info_t& sinfo,
+		      ReadPipeline& read_pipeline,
+		      UnstableHashInfoRegistry& unstable_hashinfo_registry,
+		      Listener* parent)
+      : RecoveryBackend(cct, coll, std::move(ec_impl), sinfo, read_pipeline, unstable_hashinfo_registry, parent->get_eclistener()),
+	parent(parent) {
+    }
+
+    void commit_txn_send_replies(
+      ceph::os::Transaction&& txn,
+      std::map<int, MOSDPGPushReply*> replies) override;
+
+    Listener *get_parent() const { return parent; }
+
+  private:
+    Listener *parent;
+  };
   friend ostream &operator<<(ostream &lhs, const RecoveryBackend::RecoveryOp &rhs);
   friend struct RecoveryMessages;
   friend struct OnRecoveryReadComplete;
@@ -318,7 +334,7 @@ class ECBackend : public PGBackend, public ECCommon {
 public:
   struct ReadPipeline read_pipeline;
   struct RMWPipeline rmw_pipeline;
-  struct RecoveryBackend recovery_backend;
+  struct ECRecoveryBackend recovery_backend;
 
   ceph::ErasureCodeInterfaceRef ec_impl;
 

From 2e2a3f15cdd9125029fca78b834b8b5bb53edbb1 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 10 Jan 2024 12:50:51 -0500
Subject: [PATCH 1337/2492] mon/MgrMonitor: add "down" setting to simplify
 testing

This flag prevents promotion of a standby manager to the active. It also drops
the current active.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mon/MgrMap.h      | 13 ++++++--
 src/mon/MgrMonitor.cc | 69 +++++++++++++++++++++++++++++++++----------
 src/mon/MgrMonitor.h  |  3 +-
 src/mon/MonCommands.h |  4 +++
 4 files changed, 70 insertions(+), 19 deletions(-)

diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h
index 0a906cd039fc..82f6ea880462 100644
--- a/src/mon/MgrMap.h
+++ b/src/mon/MgrMap.h
@@ -268,6 +268,10 @@ class MgrMap
   epoch_t epoch = 0;
   epoch_t last_failure_osd_epoch = 0;
 
+
+  static const uint64_t FLAG_DOWN = (1<<0);
+  uint64_t flags = 0;
+
   /// global_id of the ceph-mgr instance selected as a leader
   uint64_t active_gid = 0;
   /// server address reported by the leader once it is active
@@ -444,7 +448,7 @@ class MgrMap
       ENCODE_FINISH(bl);
       return;
     }
-    ENCODE_START(12, 6, bl);
+    ENCODE_START(13, 6, bl);
     encode(epoch, bl);
     encode(active_addrs, bl, features);
     encode(active_gid, bl);
@@ -468,13 +472,14 @@ class MgrMap
     // backwards compatible messsage for older monitors.
     encode(clients_addrs, bl, features);
     encode(clients_names, bl, features);
+    encode(flags, bl);
     ENCODE_FINISH(bl);
     return;
   }
 
   void decode(ceph::buffer::list::const_iterator& p)
   {
-    DECODE_START(12, p);
+    DECODE_START(13, p);
     decode(epoch, p);
     decode(active_addrs, p);
     decode(active_gid, p);
@@ -541,12 +546,16 @@ class MgrMap
 	}
       }
     }
+    if (struct_v >= 13) {
+      decode(flags, p);
+    }
     DECODE_FINISH(p);
   }
 
   void dump(ceph::Formatter *f) const
   {
     f->dump_int("epoch", epoch);
+    f->dump_int("flags", flags);
     f->dump_int("active_gid", get_active_gid());
     f->dump_string("active_name", get_active_name());
     f->dump_object("active_addrs", active_addrs);
diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc
index a1a2c256df83..3c37865721d0 100644
--- a/src/mon/MgrMonitor.cc
+++ b/src/mon/MgrMonitor.cc
@@ -615,22 +615,23 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
     if (pending_map.standbys.count(m->get_gid())) {
       drop_standby(m->get_gid(), false);
     }
-    dout(4) << "selecting new active " << m->get_gid()
-	    << " " << m->get_name()
-	    << " (was " << pending_map.active_gid << " "
-	    << pending_map.active_name << ")" << dendl;
-    pending_map.active_gid = m->get_gid();
-    pending_map.active_name = m->get_name();
-    pending_map.active_change = ceph_clock_now();
-    pending_map.active_mgr_features = m->get_mgr_features();
-    pending_map.available_modules = m->get_available_modules();
-    encode(m->get_metadata(), pending_metadata[m->get_name()]);
-    pending_metadata_rm.erase(m->get_name());
-
-    mon.clog->info() << "Activating manager daemon "
-                      << pending_map.active_name;
+    if (!(pending_map.flags & MgrMap::FLAG_DOWN)) {
+      dout(4) << "selecting new active " << m->get_gid()
+	      << " " << m->get_name()
+	      << " (was " << pending_map.active_gid << " "
+	      << pending_map.active_name << ")" << dendl;
+      pending_map.active_gid = m->get_gid();
+      pending_map.active_name = m->get_name();
+      pending_map.active_change = ceph_clock_now();
+      pending_map.active_mgr_features = m->get_mgr_features();
+      pending_map.available_modules = m->get_available_modules();
+      encode(m->get_metadata(), pending_metadata[m->get_name()]);
+      pending_metadata_rm.erase(m->get_name());
 
-    updated = true;
+      mon.clog->info() << "Activating manager daemon "
+                       << pending_map.active_name;
+      updated = true;
+    }
   } else {
     if (pending_map.standbys.count(m->get_gid()) > 0) {
       dout(10) << "from existing standby " << m->get_gid() << dendl;
@@ -905,6 +906,9 @@ void MgrMonitor::on_restart()
 bool MgrMonitor::promote_standby()
 {
   ceph_assert(pending_map.active_gid == 0);
+  if (pending_map.flags & MgrMap::FLAG_DOWN) {
+    return false;
+  }
   if (pending_map.standbys.size()) {
     // Promote a replacement (arbitrary choice of standby)
     auto replacement_gid = pending_map.standbys.begin()->first;
@@ -918,6 +922,9 @@ bool MgrMonitor::promote_standby()
     pending_map.active_addrs = entity_addrvec_t();
     pending_map.active_change = ceph_clock_now();
 
+    mon.clog->info() << "Activating manager daemon "
+                     << pending_map.active_name;
+
     drop_standby(replacement_gid, false);
 
     return true;
@@ -1209,7 +1216,37 @@ bool MgrMonitor::prepare_command(MonOpRequestRef op)
   int r = 0;
   bool plugged = false;
 
-  if (prefix == "mgr fail") {
+  if (prefix == "mgr set") {
+    std::string var;
+    if (!cmd_getval(cmdmap, "var", var) || var.empty()) {
+      ss << "Invalid variable";
+      return -EINVAL;
+    }
+    string val;
+    if (!cmd_getval(cmdmap, "val", val)) {
+      return -EINVAL;
+    }
+
+    if (var == "down") {
+      bool enable_down = false;
+      int r = parse_bool(val, &enable_down, ss);
+      if (r != 0) {
+        return r;
+      }
+      if (enable_down) {
+        if (!mon.osdmon()->is_writeable()) {
+          mon.osdmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+          return false;
+        }
+        pending_map.flags |= MgrMap::FLAG_DOWN;
+        plugged |= drop_active();
+      } else {
+        pending_map.flags &= ~(MgrMap::FLAG_DOWN);
+      }
+    } else {
+      return -EINVAL;
+    }
+  } else if (prefix == "mgr fail") {
     string who;
     if (!cmd_getval(cmdmap, "who", who)) {
       if (!map.active_gid) {
diff --git a/src/mon/MgrMonitor.h b/src/mon/MgrMonitor.h
index 79d4e50051d8..a2a84c141f71 100644
--- a/src/mon/MgrMonitor.h
+++ b/src/mon/MgrMonitor.h
@@ -21,8 +21,9 @@
 #include "MgrMap.h"
 #include "PaxosService.h"
 #include "MonCommand.h"
+#include "CommandHandler.h"
 
-class MgrMonitor: public PaxosService
+class MgrMonitor: public PaxosService, public CommandHandler
 {
   MgrMap map;
   MgrMap pending_map;
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index d8fb3763d406..954dcb077fbc 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1296,6 +1296,10 @@ COMMAND("mgr dump "
 	"name=epoch,type=CephInt,range=0,req=false",
 	"dump the latest MgrMap",
 	"mgr", "r")
+COMMAND("mgr set "
+	"name=var,type=CephChoices,strings=down "
+	"name=val,type=CephString ",
+	"set mgr parameter <var> to <val>", "mgr", "rw")
 COMMAND("mgr fail name=who,type=CephString,req=false",
 	"treat the named manager daemon as failed", "mgr", "rw")
 COMMAND("mgr module ls",

From 9ebcbdbed0fbaf37d60fbd8db321742ee5721d8c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 10 Jan 2024 12:52:04 -0500
Subject: [PATCH 1338/2492] qa: add unit tests for MgrMap down flag

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/mgr/mgr_test_case.py |  2 ++
 qa/tasks/mgr/test_cli.py      | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 qa/tasks/mgr/test_cli.py

diff --git a/qa/tasks/mgr/mgr_test_case.py b/qa/tasks/mgr/mgr_test_case.py
index 275a567da29c..b96f17d81f38 100644
--- a/qa/tasks/mgr/mgr_test_case.py
+++ b/qa/tasks/mgr/mgr_test_case.py
@@ -77,6 +77,8 @@ def setup_mgrs(cls):
         for daemon in cls.mgr_cluster.mgr_daemons.values():
             daemon.stop()
 
+        cls.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "set", "down", "false")
+
         for mgr_id in cls.mgr_cluster.mgr_ids:
             cls.mgr_cluster.mgr_fail(mgr_id)
 
diff --git a/qa/tasks/mgr/test_cli.py b/qa/tasks/mgr/test_cli.py
new file mode 100644
index 000000000000..a43be90ea035
--- /dev/null
+++ b/qa/tasks/mgr/test_cli.py
@@ -0,0 +1,32 @@
+import logging
+
+from .mgr_test_case import MgrTestCase
+
+log = logging.getLogger(__name__)
+
+
+class TestCLI(MgrTestCase):
+    MGRS_REQUIRED = 2
+
+    def setUp(self):
+        super(TestCLI, self).setUp()
+        self.setup_mgrs()
+
+    def test_set_down(self):
+        """
+        That setting the down flag prevents a standby from promoting.
+        """
+
+        with self.assert_cluster_log("Activating manager daemon", present=False):
+            self.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'set', 'down', 'true')
+            self.wait_until_true(lambda: self.mgr_cluster.get_active_id() == "", timeout=60)
+
+    def test_set_down_off(self):
+        """
+        That removing the down flag allows a standby to promote.
+        """
+
+        with self.assert_cluster_log("Activating manager daemon"):
+            self.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'set', 'down', 'true')
+            self.wait_until_true(lambda: self.mgr_cluster.get_active_id() == "", timeout=60)
+            self.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'set', 'down', 'false')

From a066467c9dc6f34a58e4085d05238a5db3f45cba Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Dec 2023 11:11:03 -0500
Subject: [PATCH 1339/2492] qa: allow failing whatever the active mgr is

Vanilla `mgr fail` will just fail the active.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/mgr/mgr_test_case.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/mgr/mgr_test_case.py b/qa/tasks/mgr/mgr_test_case.py
index b96f17d81f38..aa5bc6e56a9f 100644
--- a/qa/tasks/mgr/mgr_test_case.py
+++ b/qa/tasks/mgr/mgr_test_case.py
@@ -29,8 +29,11 @@ def __init__(self, ctx):
     def mgr_stop(self, mgr_id):
         self.mgr_daemons[mgr_id].stop()
 
-    def mgr_fail(self, mgr_id):
-        self.mon_manager.raw_cluster_cmd("mgr", "fail", mgr_id)
+    def mgr_fail(self, mgr_id=None):
+        if mgr_id is None:
+            self.mon_manager.raw_cluster_cmd("mgr", "fail")
+        else:
+            self.mon_manager.raw_cluster_cmd("mgr", "fail", mgr_id)
 
     def mgr_restart(self, mgr_id):
         self.mgr_daemons[mgr_id].restart()

From aa30adbaa7616c8e24b3ccaad9dbcda7c0c663fc Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Dec 2023 10:39:03 -0500
Subject: [PATCH 1340/2492] qa: test devicehealth legacy load of deleted snap
 obj

Failure without fix looks like:

    2023-12-21T16:05:55.737+0000 7fbe585b0700  0 [devicehealth DEBUG root] loading object ABC_DEADB33F_FA
    2023-12-21T16:05:55.737+0000 7fbe585b0700 -1 log_channel(cluster) log [ERR] : Unhandled exception from module 'devicehealth' while running on mgr.x: [errno 2] RADOS object not found (Failed to operate read op for oid ABC_DEADB33F_FA)
    2023-12-21T16:05:55.737+0000 7fbe585b0700 -1 devicehealth.serve:
    2023-12-21T16:05:55.737+0000 7fbe585b0700 -1 Traceback (most recent call last):
    File "/home/pdonnell/ceph/src/pybind/mgr/devicehealth/module.py", line 394, in serve
        self._do_serve()
    File "/home/pdonnell/ceph/src/pybind/mgr/mgr_module.py", line 524, in check
        return func(self, *args, **kwargs)
    File "/home/pdonnell/ceph/src/pybind/mgr/devicehealth/module.py", line 354, in _do_serve
        finished_loading_legacy = self.check_legacy_pool()
    File "/home/pdonnell/ceph/src/pybind/mgr/devicehealth/module.py", line 326, in check_legacy_pool
        if self._load_legacy_object(ioctx, obj.key):
    File "/home/pdonnell/ceph/src/pybind/mgr/devicehealth/module.py", line 300, in _load_legacy_object
        ioctx.operate_read_op(op, oid)
    File "rados.pyx", line 3723, in rados.Ioctx.operate_read_op
    rados.ObjectNotFound: [errno 2] RADOS object not found (Failed to operate read op for oid ABC_DEADB33F_FA)

Credit to Greg Farnum for postulating the cause.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/mgr/test_devicehealth.py | 33 +++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 qa/tasks/mgr/test_devicehealth.py

diff --git a/qa/tasks/mgr/test_devicehealth.py b/qa/tasks/mgr/test_devicehealth.py
new file mode 100644
index 000000000000..d3aa33fc0951
--- /dev/null
+++ b/qa/tasks/mgr/test_devicehealth.py
@@ -0,0 +1,33 @@
+from io import StringIO
+import logging
+
+from .mgr_test_case import MgrTestCase
+
+log = logging.getLogger(__name__)
+
+
+class TestDeviceHealth(MgrTestCase):
+    MGRS_REQUIRED = 1
+
+    def setUp(self):
+        super(TestDeviceHealth, self).setUp()
+        self.setup_mgrs()
+
+    def tearDown(self):
+        self.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'set', 'down', 'true')
+        self.mgr_cluster.mon_manager.raw_cluster_cmd('config', 'set', 'mon', 'mon_allow_pool_delete', 'true')
+        self.mgr_cluster.mon_manager.raw_cluster_cmd('osd', 'pool', 'rm', '.mgr', '.mgr', '--yes-i-really-really-mean-it-not-faking')
+        self.mgr_cluster.mon_manager.raw_cluster_cmd('mgr', 'set', 'down', 'false')
+
+    def test_legacy_upgrade_snap(self):
+        """
+        """
+
+        o = "ABC_DEADB33F_FA"
+        self.mon_manager.do_rados(["put", o, "-"], pool=".mgr", stdin=StringIO("junk"))
+        self.mon_manager.do_rados(["mksnap", "foo"], pool=".mgr")
+        self.mon_manager.do_rados(["rm", o], pool=".mgr")
+        self.mgr_cluster.mgr_fail()
+
+        with self.assert_cluster_log("Unhandled exception from module 'devicehealth' while running", present=False):
+            self.wait_until_true(lambda: self.mgr_cluster.get_active_id() is not None, timeout=60)

From 5e6fc0bf5f52732966d5cf2987e679abee8a384d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 21 Dec 2023 08:48:33 -0500
Subject: [PATCH 1341/2492] pybind/mgr/devicehealth: skip legacy objects that
 cannot be loaded

Log looks like after test:

    2023-12-21T16:09:28.804+0000 7fbe7fd86700  0 [devicehealth DEBUG root] loading object ABC_DEADB33F_FA
    2023-12-21T16:09:28.805+0000 7fbe7fd86700  0 [devicehealth DEBUG root] object rados.Object(ioctx=<rados.Ioctx object at 0x7fbeee0c4668>,key=ABC_DEADB33F_FA,nspace=--default--,locator=None) does not exist because it is deleted in HEAD
    2023-12-21T16:09:28.805+0000 7fbe7fd86700  0 [devicehealth DEBUG root] finished reading legacy pool, complete = True

Credit to Greg Farnum for postulating the cause.

Fixes: https://tracker.ceph.com/issues/63882
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/pybind/mgr/devicehealth/module.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index 86ea1d280720..e4356175c6a4 100644
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -327,6 +327,11 @@ def check_legacy_pool(self) -> bool:
                         count += 1
                 except json.decoder.JSONDecodeError:
                     pass
+                except rados.ObjectNotFound:
+                    # https://tracker.ceph.com/issues/63882
+                    # Sometimes an object appears in the pool listing but cannot be interacted with?
+                    self.log.debug(f"object {obj} does not exist because it is deleted in HEAD")
+                    pass
                 if count >= 10:
                     break
             done = count < 10

From 4341fc51d04cd62843ed6e98a4cad2745192ca95 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Sun, 3 Dec 2023 19:58:33 -0500
Subject: [PATCH 1342/2492] rgw/lc: do not add datalog/bilog for some lc
 actions

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc            |   9 +-
 src/rgw/driver/d4n/rgw_sal_d4n.h             |   5 +-
 src/rgw/driver/daos/rgw_sal_daos.cc          |  11 +-
 src/rgw/driver/daos/rgw_sal_daos.h           |  13 +-
 src/rgw/driver/motr/rgw_sal_motr.cc          |  13 +-
 src/rgw/driver/motr/rgw_sal_motr.h           |  13 +-
 src/rgw/driver/posix/rgw_sal_posix.cc        |  13 +-
 src/rgw/driver/posix/rgw_sal_posix.h         |  13 +-
 src/rgw/driver/rados/rgw_putobj_processor.cc |  14 +-
 src/rgw/driver/rados/rgw_putobj_processor.h  |   9 +-
 src/rgw/driver/rados/rgw_rados.cc            | 179 ++++++++++++-------
 src/rgw/driver/rados/rgw_rados.h             |  52 ++++--
 src/rgw/driver/rados/rgw_sal_rados.cc        |  32 ++--
 src/rgw/driver/rados/rgw_sal_rados.h         |  16 +-
 src/rgw/rgw_lc.cc                            |  32 +++-
 src/rgw/rgw_sal.h                            |  13 +-
 src/rgw/rgw_sal_dbstore.cc                   |  15 +-
 src/rgw/rgw_sal_dbstore.h                    |  13 +-
 src/rgw/rgw_sal_filter.cc                    |  16 +-
 src/rgw/rgw_sal_filter.h                     |  10 +-
 20 files changed, 308 insertions(+), 183 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index a25d5bbdfd95..b10dd760d78a 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -245,7 +245,7 @@ int D4NFilterObject::modify_obj_attrs(const char* attr_name, bufferlist& attr_va
 }
 
 int D4NFilterObject::delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name,
-                               optional_yield y) 
+                               optional_yield y)
 {
   std::vector<std::string> delFields;
   delFields.push_back((std::string)attr_name);
@@ -361,7 +361,7 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix
 }
 
 int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-					   optional_yield y)
+                                                   optional_yield y, bool log_op)
 {
   int delDirReturn = source->filter->get_block_dir()->delValue(source->filter->get_cache_block());
 
@@ -415,7 +415,8 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   cache_block* temp_cache_block = filter->get_cache_block();
   RGWBlockDirectory* temp_block_dir = filter->get_block_dir();
@@ -437,7 +438,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
   RGWObjState* astate;
   int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, rctx);
+			canceled, rctx, log_op);
   obj->get_obj_attrs(rctx.y, save_dpp, NULL);
   obj->get_obj_state(save_dpp, &astate, rctx.y);
 
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index cebae1eaf6ba..95696efc543e 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -111,7 +111,7 @@ class D4NFilterObject : public FilterObject {
 										     source(_source) {}
       virtual ~D4NFilterDeleteOp() = default;
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
     };
 
     D4NFilterObject(std::unique_ptr<Object> _next, D4NFilterDriver* _filter) : FilterObject(std::move(_next)),
@@ -177,7 +177,8 @@ class D4NFilterWriter : public FilterWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; }
 };
diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 21252c906daa..6e6581be9018 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -1017,7 +1017,8 @@ std::unique_ptr<MPSerializer> DaosObject::get_serializer(
 int DaosObject::transition(Bucket* bucket,
                            const rgw_placement_rule& placement_rule,
                            const real_time& mtime, uint64_t olh_epoch,
-                           const DoutPrefixProvider* dpp, optional_yield y) {
+                           const DoutPrefixProvider* dpp, optional_yield y,
+                           bool log_op) {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
 
@@ -1167,7 +1168,7 @@ DaosObject::DaosDeleteOp::DaosDeleteOp(DaosObject* _source) : source(_source) {}
 // 3. Handle empty directories
 // 4. Fail when file doesn't exist
 int DaosObject::DaosDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-                                         optional_yield y) {
+                                         optional_yield y, bool log_op) {
   ldpp_dout(dpp, 20) << "DaosDeleteOp::delete_obj "
                      << source->get_key().get_oid() << " from "
                      << source->get_bucket()->get_name() << dendl;
@@ -1506,7 +1507,7 @@ int DaosAtomicWriter::complete(
     ceph::real_time set_mtime, std::map<std::string, bufferlist>& attrs,
     ceph::real_time delete_at, const char* if_match, const char* if_nomatch,
     const std::string* user_data, rgw_zone_set* zones_trace, bool* canceled,
-    optional_yield y) {
+    optional_yield y, bool log_op) {
   ldpp_dout(dpp, 20) << "DEBUG: complete" << dendl;
   bufferlist bl;
   rgw_bucket_dir_entry ent;
@@ -1563,7 +1564,7 @@ int DaosAtomicWriter::complete(
 }
 
 int DaosMultipartUpload::abort(const DoutPrefixProvider* dpp,
-                               CephContext* cct) {
+                               CephContext* cct, optional_yield y, bool log_op) {
   // Remove upload from bucket multipart index
   ldpp_dout(dpp, 20) << "DEBUG: abort" << dendl;
   return ds3_upload_remove(bucket->get_name().c_str(), get_upload_id().c_str(),
@@ -2030,7 +2031,7 @@ int DaosMultipartWriter::complete(
     ceph::real_time set_mtime, std::map<std::string, bufferlist>& attrs,
     ceph::real_time delete_at, const char* if_match, const char* if_nomatch,
     const std::string* user_data, rgw_zone_set* zones_trace, bool* canceled,
-    const req_context& rctx) {
+    const req_context& rctx, bool log_op) {
   ldpp_dout(dpp, 20) << "DaosMultipartWriter::complete(): enter part="
                      << part_num_str << dendl;
 
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 146ba7077a40..3b067bcdffbd 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -581,7 +581,7 @@ class DaosObject : public StoreObject {
     DaosDeleteOp(DaosObject* _source);
 
     virtual int delete_obj(const DoutPrefixProvider* dpp,
-                           optional_yield y) override;
+                           optional_yield y, bool log_op = true) override;
   };
 
   ds3_obj_t* ds3o = nullptr;
@@ -641,7 +641,8 @@ class DaosObject : public StoreObject {
                          const rgw_placement_rule& placement_rule,
                          const real_time& mtime, uint64_t olh_epoch,
                          const DoutPrefixProvider* dpp,
-                         optional_yield y) override;
+                         optional_yield y,
+                         bool log_op = true) override;
   virtual int transition_to_cloud(Bucket* bucket, rgw::sal::PlacementTier* tier,
                                   rgw_bucket_dir_entry& o,
                                   std::set<std::string>& cloud_targets,
@@ -745,7 +746,8 @@ class DaosAtomicWriter : public StoreWriter {
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class DaosMultipartWriter : public StoreWriter {
@@ -790,7 +792,8 @@ class DaosMultipartWriter : public StoreWriter {
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
-                       optional_yield y) override;
+                       optional_yield y,
+                       bool log_op = true) override;
 
   const std::string& get_bucket_name();
 };
@@ -846,7 +849,7 @@ class DaosMultipartUpload : public StoreMultipartUpload {
                          int num_parts, int marker, int* next_marker,
                          bool* truncated,
                          bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
   virtual int complete(const DoutPrefixProvider* dpp, optional_yield y,
                        CephContext* cct, std::map<int, std::string>& part_etags,
                        std::list<rgw_obj_index_key>& remove_objs,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 6a97ef2f01e8..6bb7f19f5cce 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -1294,7 +1294,8 @@ int MotrObject::transition(Bucket* bucket,
     const real_time& mtime,
     uint64_t olh_epoch,
     const DoutPrefixProvider* dpp,
-    optional_yield y)
+    optional_yield y,
+    bool log_op)
 {
   return 0;
 }
@@ -1456,7 +1457,7 @@ MotrObject::MotrDeleteOp::MotrDeleteOp(MotrObject *_source) :
 // Delete::delete_obj() in rgw_rados.cc shows how rados backend process the
 // params.
 // 2. Delete an object when its versioning is turned on.
-int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y)
+int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
 {
   ldpp_dout(dpp, 20) << "delete " << source->get_key().get_oid() << " from " << source->get_bucket()->get_name() << dendl;
 
@@ -2324,7 +2325,8 @@ int MotrAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   int rc = 0;
 
@@ -2463,7 +2465,7 @@ int MotrMultipartUpload::delete_parts(const DoutPrefixProvider *dpp)
   return store->delete_motr_idx_by_name(obj_part_iname);
 }
 
-int MotrMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct)
+int MotrMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
 {
   int rc;
   // Check if multipart upload exists
@@ -2981,7 +2983,8 @@ int MotrMultipartWriter::complete(size_t accounted_size, const std::string& etag
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y)
+                       optional_yield y,
+                       bool log_op)
 {
   // Should the dir entry(object metadata) be updated? For example
   // mtime.
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index ab0fc1d5283b..a6f096424221 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -660,7 +660,7 @@ class MotrObject : public StoreObject {
       public:
         MotrDeleteOp(MotrObject* _source);
 
-        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
     };
 
     MotrObject() = default;
@@ -710,7 +710,8 @@ class MotrObject : public StoreObject {
         const real_time& mtime,
         uint64_t olh_epoch,
         const DoutPrefixProvider* dpp,
-        optional_yield y) override;
+        optional_yield y,
+        bool log_op = true) override;
     virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
     virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -812,7 +813,8 @@ class MotrAtomicWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 
   unsigned populate_bvec(unsigned len, bufferlist::iterator &bi);
   void cleanup();
@@ -859,7 +861,8 @@ class MotrMultipartWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       optional_yield y) override;
+                       optional_yield y,
+                       bool log_op = true) override;
 };
 
 // The implementation of multipart upload in POC roughly follows the
@@ -934,7 +937,7 @@ class MotrMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index f5b79f9b0933..2670f7872cd2 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -1658,7 +1658,8 @@ int POSIXObject::transition(Bucket* bucket,
 			    const real_time& mtime,
 			    uint64_t olh_epoch,
 			    const DoutPrefixProvider* dpp,
-			    optional_yield y)
+			    optional_yield y,
+                            bool log_op)
 {
   return -ERR_NOT_IMPLEMENTED;
 }
@@ -2366,7 +2367,7 @@ int POSIXObject::POSIXReadOp::get_attr(const DoutPrefixProvider* dpp, const char
 }
 
 int POSIXObject::POSIXDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-					   optional_yield y)
+					   optional_yield y, bool log_op)
 {
   return source->delete_object(dpp, y, false);
 }
@@ -2572,7 +2573,7 @@ int POSIXMultipartUpload::list_parts(const DoutPrefixProvider *dpp, CephContext
   return 0;
 }
 
-int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
+int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
 {
   int ret;
 
@@ -2809,7 +2810,8 @@ int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   int ret;
   POSIXUploadPartInfo info;
@@ -2876,7 +2878,8 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   int ret;
 
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 3b76295b0c47..67931a4677a4 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -298,7 +298,7 @@ class POSIXObject : public StoreObject {
       source(_source) {}
     virtual ~POSIXDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
   };
 
   POSIXObject(POSIXDriver *_dr, const rgw_obj_key& _k)
@@ -355,7 +355,8 @@ class POSIXObject : public StoreObject {
 			 const real_time& mtime,
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
-			 optional_yield y) override;
+			 optional_yield y,
+                         bool log_op = true) override;
   virtual int transition_to_cloud(Bucket* bucket,
 			 rgw::sal::PlacementTier* tier,
 			 rgw_bucket_dir_entry& o,
@@ -544,7 +545,7 @@ class POSIXMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -606,7 +607,8 @@ class POSIXAtomicWriter : public StoreWriter {
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
-		       const req_context& rctx) override;
+		       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class POSIXMultipartWriter : public StoreWriter {
@@ -645,7 +647,8 @@ class POSIXMultipartWriter : public StoreWriter {
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
-		       const req_context& rctx) override;
+		       const req_context& rctx,
+                       bool log_op = true) override;
 
 };
 
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 67e8a3a998ef..8b86b570480b 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -344,7 +344,8 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
                                     const std::string *user_data,
                                     rgw_zone_set *zones_trace,
                                     bool *pcanceled, 
-                                    const req_context& rctx)
+                                    const req_context& rctx,
+                                    bool log_op)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -381,7 +382,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
 
   read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
 
-  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx, log_op);
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -486,7 +487,8 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
                                        const std::string *user_data,
                                        rgw_zone_set *zones_trace,
                                        bool *pcanceled, 
-                                       const req_context& rctx)
+                                       const req_context& rctx,
+                                       bool log_op)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -510,7 +512,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx, log_op);
   if (r < 0)
     return r;
 
@@ -691,7 +693,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
                                     ceph::real_time set_mtime, rgw::sal::Attrs& attrs,
                                     ceph::real_time delete_at, const char *if_match, const char *if_nomatch,
                                     const string *user_data, rgw_zone_set *zones_trace, bool *pcanceled,
-                                    const req_context& rctx)
+                                    const req_context& rctx, bool log_op)
 {
   int r = writer.drain();
   if (r < 0)
@@ -749,7 +751,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
   }
   r = obj_op.write_meta(actual_size + cur_size,
 			accounted_size + *cur_accounted_size,
-			attrs, rctx);
+			attrs, rctx, log_op);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index ce163e36f777..7b92402d2d7d 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -190,7 +190,8 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                const char *if_match, const char *if_nomatch,
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
-               const req_context& rctx) override;
+               const req_context& rctx,
+               bool log_op = true) override;
 
 };
 
@@ -237,7 +238,8 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                const char *if_match, const char *if_nomatch,
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
-               const req_context& rctx) override;
+               const req_context& rctx,
+               bool log_op = true) override;
 
 };
 
@@ -273,7 +275,8 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                  std::map<std::string, bufferlist>& attrs, ceph::real_time delete_at,
                  const char *if_match, const char *if_nomatch, const std::string *user_data,
                  rgw_zone_set *zones_trace, bool *canceled,
-                 const req_context& rctx) override;
+                 const req_context& rctx,
+                 bool log_op = true) override;
   };
 
 } // namespace putobj
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 93e8840eafdb..b2ffef0589fa 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -926,9 +926,11 @@ void RGWIndexCompletionManager::process()
         continue;
       }
 
-      // This null_yield can stay, for now, since we're in our own thread
-      add_datalog_entry(&dpp, store->svc.datalog_rados, bucket_info,
-			bs.shard_id, null_yield);
+      if (c->log_op) {
+        // This null_yield can stay, for now, since we're in our own thread
+        add_datalog_entry(&dpp, store->svc.datalog_rados, bucket_info,
+                          bs.shard_id, null_yield);
+      }
     }
   }
 }
@@ -3040,7 +3042,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
 int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_size,
                                            map<string, bufferlist>& attrs,
                                            bool assume_noent, bool modify_tail,
-                                           void *_index_op, const req_context& rctx)
+                                           void *_index_op, const req_context& rctx,
+                                           bool log_op)
 {
   RGWRados::Bucket::UpdateIndex *index_op = static_cast<RGWRados::Bucket::UpdateIndex *>(_index_op);
   RGWRados *store = target->get_store();
@@ -3206,7 +3209,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
 
   if (!index_op->is_prepared()) {
     tracepoint(rgw_rados, prepare_enter, req_id.c_str());
-    r = index_op->prepare(rctx.dpp, CLS_RGW_OP_ADD, &state->write_tag, rctx.y);
+    r = index_op->prepare(rctx.dpp, CLS_RGW_OP_ADD, &state->write_tag, rctx.y, log_op);
     tracepoint(rgw_rados, prepare_exit, req_id.c_str());
     if (r < 0)
       return r;
@@ -3240,7 +3243,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
                         meta.set_mtime, etag, content_type,
                         storage_class, &acl_bl,
 			 meta.category, meta.remove_objs, rctx.y,
-			 meta.user_data, meta.appendable);
+			 meta.user_data, meta.appendable, log_op);
   tracepoint(rgw_rados, complete_exit, req_id.c_str());
   if (r < 0)
     goto done_cancel;
@@ -3254,7 +3257,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   state = NULL;
 
   if (versioned_op && meta.olh_epoch) {
-    r = store->set_olh(rctx.dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, rctx.y, meta.zones_trace);
+    r = store->set_olh(rctx.dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, rctx.y, meta.zones_trace, log_op);
     if (r < 0) {
       return r;
     }
@@ -3285,7 +3288,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   return 0;
 
 done_cancel:
-  int ret = index_op->cancel(rctx.dpp, meta.remove_objs, rctx.y);
+  int ret = index_op->cancel(rctx.dpp, meta.remove_objs, rctx.y, log_op);
   if (ret < 0) {
     ldpp_dout(rctx.dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
   }
@@ -3330,7 +3333,8 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
 }
 
 int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
-                                           map<string, bufferlist>& attrs, const req_context& rctx)
+                                        map<string, bufferlist>& attrs, const req_context& rctx,
+                                        bool log_op)
 {
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -3341,13 +3345,13 @@ int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
   bool assume_noent = (meta.if_match == NULL && meta.if_nomatch == NULL);
   int r;
   if (assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
     if (r == -EEXIST) {
       assume_noent = false;
     }
   }
   if (!assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
   }
   return r;
 }
@@ -4901,7 +4905,8 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 	       real_time delete_at,
                string *petag,
                const DoutPrefixProvider *dpp,
-               optional_yield y)
+               optional_yield y,
+               bool log_op)
 {
   string tag;
   append_rand_alpha(cct, tag, tag, 32);
@@ -4965,7 +4970,7 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 
   const req_context rctx{dpp, y, nullptr};
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-                            nullptr, nullptr, nullptr, nullptr, nullptr, rctx);
+                            nullptr, nullptr, nullptr, nullptr, nullptr, rctx, log_op);
 }
 
 int RGWRados::transition_obj(RGWObjectCtx& obj_ctx,
@@ -4975,7 +4980,8 @@ int RGWRados::transition_obj(RGWObjectCtx& obj_ctx,
                              const real_time& mtime,
                              uint64_t olh_epoch,
                              const DoutPrefixProvider *dpp,
-                             optional_yield y)
+                             optional_yield y,
+                             bool log_op)
 {
   rgw::sal::Attrs attrs;
   real_time read_mtime;
@@ -5016,7 +5022,8 @@ int RGWRados::transition_obj(RGWObjectCtx& obj_ctx,
                       real_time(),
                       nullptr /* petag */,
                       dpp,
-                      y);
+                      y,
+                      log_op);
   if (ret < 0) {
     return ret;
   }
@@ -5587,7 +5594,7 @@ struct tombstone_entry {
  * obj: name of the object to delete
  * Returns: 0 on success, -ERR# otherwise.
  */
-int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvider *dpp)
+int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvider *dpp, bool log_op)
 {
   RGWRados *store = target->get_store();
   const rgw_obj& src_obj = target->get_obj();
@@ -5601,6 +5608,8 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
   bool explicit_marker_version = (!params.marker_version_id.empty());
 
   if (params.versioning_status & BUCKET_VERSIONED || explicit_marker_version) {
+    bool add_log = log_op && store->svc.zone->need_to_log_data();
+
     if (instance.empty() || explicit_marker_version) {
       rgw_obj marker = obj;
       marker.key.instance.clear();
@@ -5629,7 +5638,9 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
         meta.mtime = params.mtime;
       }
 
-      int r = store->set_olh(dpp, target->get_ctx(), target->get_bucket_info(), marker, true, &meta, params.olh_epoch, params.unmod_since, params.high_precision_time, y, params.zones_trace);
+      int r = store->set_olh(dpp, target->get_ctx(), target->get_bucket_info(), marker, true,
+                             &meta, params.olh_epoch, params.unmod_since, params.high_precision_time,
+                             y, params.zones_trace, add_log);
       if (r < 0) {
         return r;
       }
@@ -5641,7 +5652,8 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
         return r;
       }
       result.delete_marker = dirent.is_delete_marker();
-      r = store->unlink_obj_instance(dpp, target->get_ctx(), target->get_bucket_info(), obj, params.olh_epoch, y, params.zones_trace);
+      r = store->unlink_obj_instance(dpp, target->get_ctx(), target->get_bucket_info(), obj, params.olh_epoch,
+                                     y, params.zones_trace, log_op);
       if (r < 0) {
         return r;
       }
@@ -5655,8 +5667,10 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
       return r;
     }
 
-    add_datalog_entry(dpp, store->svc.datalog_rados,
-                      target->get_bucket_info(), bs->shard_id, y);
+    if (add_log) {
+      add_datalog_entry(dpp, store->svc.datalog_rados,
+                        target->get_bucket_info(), bs->shard_id, y);
+    }
 
     return 0;
   }
@@ -5735,7 +5749,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
   index_op.set_zones_trace(params.zones_trace);
   index_op.set_bilog_flags(params.bilog_flags);
 
-  r = index_op.prepare(dpp, CLS_RGW_OP_DEL, &state->write_tag, y);
+  r = index_op.prepare(dpp, CLS_RGW_OP_DEL, &state->write_tag, y, log_op);
   if (r < 0)
     return r;
 
@@ -5754,7 +5768,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
       tombstone_entry entry{*state};
       obj_tombstone_cache->add(obj, entry);
     }
-    r = index_op.complete_del(dpp, poolid, ioctx.get_last_version(), state->mtime, params.remove_objs, y);
+    r = index_op.complete_del(dpp, poolid, ioctx.get_last_version(), state->mtime, params.remove_objs, y, log_op);
 
     int ret = target->complete_atomic_modification(dpp, y);
     if (ret < 0) {
@@ -5762,7 +5776,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
     }
     /* other than that, no need to propagate error */
   } else {
-    int ret = index_op.cancel(dpp, params.remove_objs, y);
+    int ret = index_op.cancel(dpp, params.remove_objs, y, log_op);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "ERROR: index_op.cancel() returned ret=" << ret << dendl;
     }
@@ -5788,7 +5802,8 @@ int RGWRados::delete_obj(const DoutPrefixProvider *dpp,
                          int versioning_status, optional_yield y,// versioning flags defined in enum RGWBucketFlags
                          uint16_t bilog_flags,
                          const real_time& expiration_time,
-                         rgw_zone_set *zones_trace)
+                         rgw_zone_set *zones_trace,
+                         bool log_op)
 {
   RGWRados::Object del_target(this, bucket_info, obj_ctx, obj);
   RGWRados::Object::Delete del_op(&del_target);
@@ -5799,7 +5814,7 @@ int RGWRados::delete_obj(const DoutPrefixProvider *dpp,
   del_op.params.expiration_time = expiration_time;
   del_op.params.zones_trace = zones_trace;
 
-  return del_op.delete_obj(y, dpp);
+  return del_op.delete_obj(y, dpp, log_op);
 }
 
 int RGWRados::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y)
@@ -6827,7 +6842,8 @@ int RGWRados::Bucket::UpdateIndex::guard_reshard(const DoutPrefixProvider *dpp,
   return 0;
 }
 
-int RGWRados::Bucket::UpdateIndex::prepare(const DoutPrefixProvider *dpp, RGWModifyOp op, const string *write_tag, optional_yield y)
+int RGWRados::Bucket::UpdateIndex::prepare(const DoutPrefixProvider *dpp, RGWModifyOp op, const string *write_tag,
+                                           optional_yield y, bool log_op)
 {
   if (blind) {
     return 0;
@@ -6842,8 +6858,10 @@ int RGWRados::Bucket::UpdateIndex::prepare(const DoutPrefixProvider *dpp, RGWMod
     }
   }
 
+  bool add_log = log_op && store->svc.zone->need_to_log_data();
+
   int r = guard_reshard(dpp, obj, nullptr, [&](BucketShard *bs) -> int {
-				   return store->cls_obj_prepare_op(dpp, *bs, op, optag, obj, bilog_flags, y, zones_trace);
+				   return store->cls_obj_prepare_op(dpp, *bs, op, optag, obj, bilog_flags, y, zones_trace, add_log);
 				 }, y);
   if (r < 0) {
     return r;
@@ -6862,7 +6880,8 @@ int RGWRados::Bucket::UpdateIndex::complete(const DoutPrefixProvider *dpp, int64
                                             list<rgw_obj_index_key> *remove_objs,
 					    optional_yield y,
 					    const string *user_data,
-                                            bool appendable)
+                                            bool appendable,
+                                            bool log_op)
 {
   if (blind) {
     return 0;
@@ -6898,10 +6917,13 @@ int RGWRados::Bucket::UpdateIndex::complete(const DoutPrefixProvider *dpp, int64
   ent.meta.content_type = content_type;
   ent.meta.appendable = appendable;
 
-  ret = store->cls_obj_complete_add(*bs, obj, optag, poolid, epoch, ent, category, remove_objs, bilog_flags, zones_trace);
+  bool add_log = log_op && store->svc.zone->need_to_log_data();
 
-  add_datalog_entry(dpp, store->svc.datalog_rados,
-                    target->bucket_info, bs->shard_id, y);
+  ret = store->cls_obj_complete_add(*bs, obj, optag, poolid, epoch, ent, category, remove_objs, bilog_flags, zones_trace, add_log);
+  if (add_log) {
+    add_datalog_entry(dpp, store->svc.datalog_rados,
+                      target->bucket_info, bs->shard_id, y);
+  }
 
   return ret;
 }
@@ -6910,7 +6932,8 @@ int RGWRados::Bucket::UpdateIndex::complete_del(const DoutPrefixProvider *dpp,
                                                 int64_t poolid, uint64_t epoch,
                                                 real_time& removed_mtime,
                                                 list<rgw_obj_index_key> *remove_objs,
-						optional_yield y)
+                                                optional_yield y,
+                                                bool log_op)
 {
   if (blind) {
     return 0;
@@ -6924,10 +6947,14 @@ int RGWRados::Bucket::UpdateIndex::complete_del(const DoutPrefixProvider *dpp,
     return ret;
   }
 
-  ret = store->cls_obj_complete_del(*bs, optag, poolid, epoch, obj, removed_mtime, remove_objs, bilog_flags, zones_trace);
+  bool add_log = log_op && store->svc.zone->need_to_log_data();
 
-  add_datalog_entry(dpp, store->svc.datalog_rados,
-                    target->bucket_info, bs->shard_id, y);
+  ret = store->cls_obj_complete_del(*bs, optag, poolid, epoch, obj, removed_mtime, remove_objs, bilog_flags, zones_trace, add_log);
+
+  if (add_log) {
+    add_datalog_entry(dpp, store->svc.datalog_rados,
+                      target->bucket_info, bs->shard_id, y);
+  }
 
   return ret;
 }
@@ -6935,7 +6962,8 @@ int RGWRados::Bucket::UpdateIndex::complete_del(const DoutPrefixProvider *dpp,
 
 int RGWRados::Bucket::UpdateIndex::cancel(const DoutPrefixProvider *dpp,
                                           list<rgw_obj_index_key> *remove_objs,
-					  optional_yield y)
+                                          optional_yield y,
+                                          bool log_op)
 {
     if (blind) {
     return 0;
@@ -6943,17 +6971,21 @@ int RGWRados::Bucket::UpdateIndex::cancel(const DoutPrefixProvider *dpp,
   RGWRados *store = target->get_store();
   BucketShard *bs;
 
+  bool add_log = log_op && store->svc.zone->need_to_log_data();
+
   int ret = guard_reshard(dpp, obj, &bs, [&](BucketShard *bs) -> int {
-				 return store->cls_obj_complete_cancel(*bs, optag, obj, remove_objs, bilog_flags, zones_trace);
+				 return store->cls_obj_complete_cancel(*bs, optag, obj, remove_objs, bilog_flags, zones_trace, add_log);
 			       }, y);
 
-  /*
-   * need to update data log anyhow, so that whoever follows needs to update its internal markers
-   * for following the specific bucket shard log. Otherwise they end up staying behind, and users
-   * have no way to tell that they're all caught up
-   */
-  add_datalog_entry(dpp, store->svc.datalog_rados,
-                    target->bucket_info, bs->shard_id, y);
+  if (add_log) {
+    /*
+     * need to update data log anyhow, so that whoever follows needs to update its internal markers
+     * for following the specific bucket shard log. Otherwise they end up staying behind, and users
+     * have no way to tell that they're all caught up
+     */
+    add_datalog_entry(dpp, store->svc.datalog_rados,
+                      target->bucket_info, bs->shard_id, y);
+  }
 
   return ret;
 }
@@ -7705,7 +7737,7 @@ int RGWRados::bucket_index_link_olh(const DoutPrefixProvider *dpp, RGWBucketInfo
 		      cls_rgw_bucket_link_olh(op, key, olh_state.olh_tag,
                                               delete_marker, op_tag, meta, olh_epoch,
 					      unmod_since, high_precision_time,
-					      svc.zone->need_to_log_data(), zones_trace);
+					      log_data_change, zones_trace);
                       return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
                     }, y);
   if (r < 0) {
@@ -7730,7 +7762,8 @@ int RGWRados::bucket_index_unlink_instance(const DoutPrefixProvider *dpp,
                                            RGWBucketInfo& bucket_info,
                                            const rgw_obj& obj_instance,
                                            const string& op_tag, const string& olh_tag,
-                                           uint64_t olh_epoch, optional_yield y, rgw_zone_set *_zones_trace)
+                                           uint64_t olh_epoch, optional_yield y,
+                                           rgw_zone_set *_zones_trace, bool log_op)
 {
   rgw_rados_ref ref;
   int r = get_obj_head_ref(dpp, bucket_info, obj_instance, &ref);
@@ -7754,7 +7787,7 @@ int RGWRados::bucket_index_unlink_instance(const DoutPrefixProvider *dpp,
 		      op.assert_exists(); // bucket index shard must exist
 		      cls_rgw_guard_bucket_resharding(op, -ERR_BUSY_RESHARDING);
 		      cls_rgw_bucket_unlink_instance(op, key, op_tag,
-						     olh_tag, olh_epoch, svc.zone->need_to_log_data(), zones_trace);
+						     olh_tag, olh_epoch, log_op, zones_trace);
                       return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
                     }, y);
   if (r < 0) {
@@ -7957,7 +7990,8 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
 			    bufferlist& olh_tag,
 			    std::map<uint64_t, std::vector<rgw_bucket_olh_log_entry> >& log,
 			    uint64_t *plast_ver,
-			    optional_yield y, rgw_zone_set* zones_trace)
+			    optional_yield y, rgw_zone_set* zones_trace,
+                            bool log_op)
 {
   if (log.empty()) {
     return 0;
@@ -8070,7 +8104,7 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp,
        liter != remove_instances.end(); ++liter) {
     cls_rgw_obj_key& key = *liter;
     rgw_obj obj_instance(bucket, key);
-    int ret = delete_obj(dpp, obj_ctx, bucket_info, obj_instance, 0, y, RGW_BILOG_FLAG_VERSIONED_OP, ceph::real_time(), zones_trace);
+    int ret = delete_obj(dpp, obj_ctx, bucket_info, obj_instance, 0, y, RGW_BILOG_FLAG_VERSIONED_OP, ceph::real_time(), zones_trace, log_op);
     if (ret < 0 && ret != -ENOENT) {
       ldpp_dout(dpp, 0) << "ERROR: delete_obj() returned " << ret << " obj_instance=" << obj_instance << dendl;
       return ret;
@@ -8174,7 +8208,7 @@ int RGWRados::clear_olh(const DoutPrefixProvider *dpp,
 /*
  * read olh log and apply it
  */
-int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y, rgw_zone_set *zones_trace)
+int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y, rgw_zone_set *zones_trace, bool log_op)
 {
   map<uint64_t, vector<rgw_bucket_olh_log_entry> > log;
   bool is_truncated;
@@ -8185,7 +8219,7 @@ int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, R
     if (ret < 0) {
       return ret;
     }
-    ret = apply_olh_log(dpp, obj_ctx, *state, bucket_info, obj, state->olh_tag, log, &ver_marker, y, zones_trace);
+    ret = apply_olh_log(dpp, obj_ctx, *state, bucket_info, obj, state->olh_tag, log, &ver_marker, y, zones_trace, log_op);
     if (ret < 0) {
       return ret;
     }
@@ -8255,7 +8289,7 @@ int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx,
       // it's possible that the pending xattr from this op prevented the olh
       // object from being cleaned by another thread that was deleting the last
       // existing version. We invoke a best-effort update_olh here to handle this case.
-      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y);
+      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, log_data_change);
       if (r < 0 && r != -ECANCELED) {
         ldpp_dout(dpp, 20) << "update_olh() target_obj=" << olh_obj << " returned " << r << dendl;
       }
@@ -8274,7 +8308,7 @@ int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx,
     return 0;
   }
 
-  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y);
+  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, log_data_change);
   if (ret == -ECANCELED) { /* already did what we needed, no need to retry, raced with another user */
     ret = 0;
   }
@@ -8287,7 +8321,7 @@ int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx,
 }
 
 int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj,
-                                  uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace)
+                                  uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace, bool log_op)
 {
   string op_tag;
 
@@ -8320,7 +8354,7 @@ int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& o
 
     string olh_tag(state->olh_tag.c_str(), state->olh_tag.length());
 
-    ret = bucket_index_unlink_instance(dpp, bucket_info, target_obj, op_tag, olh_tag, olh_epoch, y, zones_trace);
+    ret = bucket_index_unlink_instance(dpp, bucket_info, target_obj, op_tag, olh_tag, olh_epoch, y, zones_trace, log_op);
     if (ret < 0) {
       olh_cancel_modification(dpp, bucket_info, *state, olh_obj, op_tag, y);
       ldpp_dout(dpp, 20) << "bucket_index_unlink_instance() target_obj=" << target_obj << " returned " << ret << dendl;
@@ -8330,7 +8364,7 @@ int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& o
       // it's possible that the pending xattr from this op prevented the olh
       // object from being cleaned by another thread that was deleting the last
       // existing version. We invoke a best-effort update_olh here to handle this case.
-      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace);
+      int r = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, log_op);
       if (r < 0 && r != -ECANCELED) {
         ldpp_dout(dpp, 20) << "update_olh() target_obj=" << olh_obj << " returned " << r << dendl;
       }
@@ -8344,7 +8378,7 @@ int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& o
     return -EIO;
   }
 
-  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace);
+  ret = update_olh(dpp, obj_ctx, state, bucket_info, olh_obj, y, zones_trace, log_op);
   if (ret == -ECANCELED) { /* already did what we needed, no need to retry, raced with another user */
     return 0;
   }
@@ -9145,7 +9179,8 @@ bool RGWRados::process_expire_objects(const DoutPrefixProvider *dpp, optional_yi
 }
 
 int RGWRados::cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, string& tag,
-                                 rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *_zones_trace)
+                                 rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *_zones_trace,
+                                 bool log_op)
 {
   const bool bitx = cct->_conf->rgw_bucket_index_transaction_instrumentation;
   ldout_bitx(bitx, dpp, 10) << "ENTERING " << __func__ << ": bucket-shard=" << bs << " obj=" << obj << " tag=" << tag << " op=" << op << dendl_bitx;
@@ -9162,7 +9197,7 @@ int RGWRados::cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs,
 
   cls_rgw_obj_key key(obj.key.get_index_key_name(), obj.key.instance);
   cls_rgw_guard_bucket_resharding(o, -ERR_BUSY_RESHARDING);
-  cls_rgw_bucket_prepare_op(o, op, tag, key, obj.key.get_loc(), svc.zone->need_to_log_data(), bilog_flags, zones_trace);
+  cls_rgw_bucket_prepare_op(o, op, tag, key, obj.key.get_loc(), log_op, bilog_flags, zones_trace);
   int ret = bs.bucket_obj.operate(dpp, &o, y);
   ldout_bitx(bitx, dpp, 10) << "EXITING " << __func__ << ": ret=" << ret << dendl_bitx;
   return ret;
@@ -9171,12 +9206,14 @@ int RGWRados::cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs,
 int RGWRados::cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModifyOp op, string& tag,
                                   int64_t pool, uint64_t epoch,
                                   rgw_bucket_dir_entry& ent, RGWObjCategory category,
-				  list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *_zones_trace)
+                                  list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags,
+                                  rgw_zone_set *_zones_trace, bool log_op)
 {
   const bool bitx = cct->_conf->rgw_bucket_index_transaction_instrumentation;
   ldout_bitx_c(bitx, cct, 10) << "ENTERING " << __func__ << ": bucket-shard=" << bs <<
     " obj=" << obj << " tag=" << tag << " op=" << op <<
-    ", remove_objs=" << (remove_objs ? *remove_objs : std::list<rgw_obj_index_key>()) << dendl_bitx;
+    ", remove_objs=" << (remove_objs ? *remove_objs : std::list<rgw_obj_index_key>()) <<
+    ", log_op=" << log_op << dendl_bitx;
   ldout_bitx_c(bitx, cct, 25) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl_bitx;
 
   ObjectWriteOperation o;
@@ -9198,10 +9235,10 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModify
   cls_rgw_obj_key key(ent.key.name, ent.key.instance);
   cls_rgw_guard_bucket_resharding(o, -ERR_BUSY_RESHARDING);
   cls_rgw_bucket_complete_op(o, op, tag, ver, key, dir_meta, remove_objs,
-                             svc.zone->need_to_log_data(), bilog_flags, &zones_trace, obj.key.get_loc());
+                             log_op, bilog_flags, &zones_trace, obj.key.get_loc());
   complete_op_data *arg;
   index_completion_manager->create_completion(obj, op, tag, ver, key, dir_meta, remove_objs,
-                                              svc.zone->need_to_log_data(), bilog_flags, &zones_trace, &arg);
+                                              log_op, bilog_flags, &zones_trace, &arg);
   librados::AioCompletion *completion = arg->rados_completion;
   int ret = bs.bucket_obj.aio_operate(arg->rados_completion, &o);
   completion->release(); /* can't reference arg here, as it might have already been released */
@@ -9213,9 +9250,12 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModify
 int RGWRados::cls_obj_complete_add(BucketShard& bs, const rgw_obj& obj, string& tag,
                                    int64_t pool, uint64_t epoch,
                                    rgw_bucket_dir_entry& ent, RGWObjCategory category,
-                                   list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace)
+                                   list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags,
+                                   rgw_zone_set *zones_trace, bool log_op)
 {
-  return cls_obj_complete_op(bs, obj, CLS_RGW_OP_ADD, tag, pool, epoch, ent, category, remove_objs, bilog_flags, zones_trace);
+  return cls_obj_complete_op(bs, obj, CLS_RGW_OP_ADD, tag, pool, epoch,
+                             ent, category, remove_objs, bilog_flags,
+                             zones_trace, log_op);
 }
 
 int RGWRados::cls_obj_complete_del(BucketShard& bs, string& tag,
@@ -9224,26 +9264,27 @@ int RGWRados::cls_obj_complete_del(BucketShard& bs, string& tag,
                                    real_time& removed_mtime,
                                    list<rgw_obj_index_key> *remove_objs,
                                    uint16_t bilog_flags,
-                                   rgw_zone_set *zones_trace)
+                                   rgw_zone_set *zones_trace,
+                                   bool log_op)
 {
   rgw_bucket_dir_entry ent;
   ent.meta.mtime = removed_mtime;
   obj.key.get_index_key(&ent.key);
   return cls_obj_complete_op(bs, obj, CLS_RGW_OP_DEL, tag, pool, epoch,
 			     ent, RGWObjCategory::None, remove_objs,
-			     bilog_flags, zones_trace);
+			     bilog_flags, zones_trace, log_op);
 }
 
 int RGWRados::cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj,
                                       list<rgw_obj_index_key> *remove_objs,
-                                      uint16_t bilog_flags, rgw_zone_set *zones_trace)
+                                      uint16_t bilog_flags, rgw_zone_set *zones_trace, bool log_op)
 {
   rgw_bucket_dir_entry ent;
   obj.key.get_index_key(&ent.key);
   return cls_obj_complete_op(bs, obj, CLS_RGW_OP_CANCEL, tag,
 			     -1 /* pool id */, 0, ent,
 			     RGWObjCategory::None, remove_objs, bilog_flags,
-			     zones_trace);
+			     zones_trace, log_op);
 }
 
 int RGWRados::cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout)
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 986dc091163a..9245dc1512bd 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -822,9 +822,11 @@ class RGWRados
       int _do_write_meta(uint64_t size, uint64_t accounted_size,
                      std::map<std::string, bufferlist>& attrs,
                      bool modify_tail, bool assume_noent,
-                     void *index_op, const req_context& rctx);
+                     void *index_op, const req_context& rctx,
+                     bool log_op = true);
       int write_meta(uint64_t size, uint64_t accounted_size,
-                     std::map<std::string, bufferlist>& attrs, const req_context& rctx);
+                     std::map<std::string, bufferlist>& attrs,
+                     const req_context& rctx, bool log_op = true);
       int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive);
       const req_state* get_req_state() {
         return nullptr;  /* XXX dang Only used by LTTng, and it handles null anyway */
@@ -862,7 +864,7 @@ class RGWRados
 
       explicit Delete(RGWRados::Object *_target) : target(_target) {}
 
-      int delete_obj(optional_yield y, const DoutPrefixProvider *dpp);
+      int delete_obj(optional_yield y, const DoutPrefixProvider *dpp, bool log_op = true);
     }; // struct RGWRados::Object::Delete
 
     struct Stat {
@@ -968,7 +970,7 @@ class RGWRados
         zones_trace = _zones_trace;
       }
 
-      int prepare(const DoutPrefixProvider *dpp, RGWModifyOp, const std::string *write_tag, optional_yield y);
+      int prepare(const DoutPrefixProvider *dpp, RGWModifyOp, const std::string *write_tag, optional_yield y, bool log_op = true);
       int complete(const DoutPrefixProvider *dpp, int64_t poolid, uint64_t epoch, uint64_t size,
                    uint64_t accounted_size, const ceph::real_time& ut,
                    const std::string& etag, const std::string& content_type,
@@ -977,15 +979,18 @@ class RGWRados
 		   std::list<rgw_obj_index_key> *remove_objs,
 		   optional_yield y,
 		   const std::string *user_data = nullptr,
-		   bool appendable = false);
+		   bool appendable = false,
+                   bool log_op = true);
       int complete_del(const DoutPrefixProvider *dpp,
                        int64_t poolid, uint64_t epoch,
                        ceph::real_time& removed_mtime, /* mtime of removed object */
                        std::list<rgw_obj_index_key> *remove_objs,
-		       optional_yield y);
+                       optional_yield y,
+                       bool log_op = true);
       int cancel(const DoutPrefixProvider *dpp,
                  std::list<rgw_obj_index_key> *remove_objs,
-		 optional_yield y);
+                 optional_yield y,
+                 bool log_op = true);
 
       const std::string *get_optag() { return &optag; }
 
@@ -1208,7 +1213,8 @@ class RGWRados
 	       ceph::real_time delete_at,
                std::string *petag,
                const DoutPrefixProvider *dpp,
-               optional_yield y);
+               optional_yield y,
+               bool log_op = true);
 
   int transition_obj(RGWObjectCtx& obj_ctx,
                      RGWBucketInfo& bucket_info,
@@ -1217,7 +1223,8 @@ class RGWRados
                      const real_time& mtime,
                      uint64_t olh_epoch,
                      const DoutPrefixProvider *dpp,
-                     optional_yield y);
+                     optional_yield y,
+                     bool log_op = true);
 
   int check_bucket_empty(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, optional_yield y);
 
@@ -1247,7 +1254,8 @@ class RGWRados
 		 int versioning_status, optional_yield y,  // versioning flags defined in enum RGWBucketFlags
 		 uint16_t bilog_flags = 0,
 		 const ceph::real_time& expiration_time = ceph::real_time(),
-		 rgw_zone_set *zones_trace = nullptr);
+		 rgw_zone_set *zones_trace = nullptr,
+                 bool log_op = true);
 
   int delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y);
 
@@ -1337,7 +1345,9 @@ class RGWRados
                                    RGWBucketInfo& bucket_info,
                                    const rgw_obj& obj_instance,
                                    const std::string& op_tag, const std::string& olh_tag,
-                                   uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr);
+                                   uint64_t olh_epoch, optional_yield y,
+                                   rgw_zone_set *zones_trace = nullptr,
+                                   bool log_op = true);
   int bucket_index_read_olh_log(const DoutPrefixProvider *dpp,
                                 RGWBucketInfo& bucket_info, RGWObjState& state,
                                 const rgw_obj& obj_instance, uint64_t ver_marker,
@@ -1346,9 +1356,9 @@ class RGWRados
   int bucket_index_clear_olh(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const std::string& olh_tag, const rgw_obj& obj_instance, optional_yield y);
   int apply_olh_log(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState& obj_state, RGWBucketInfo& bucket_info, const rgw_obj& obj,
                     bufferlist& obj_tag, std::map<uint64_t, std::vector<rgw_bucket_olh_log_entry> >& log,
-                    uint64_t *plast_ver, optional_yield y, rgw_zone_set *zones_trace = nullptr);
+                    uint64_t *plast_ver, optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y,
-		 rgw_zone_set *zones_trace = nullptr);
+		 rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int clear_olh(const DoutPrefixProvider *dpp,
                 RGWObjectCtx& obj_ctx,
                 const rgw_obj& obj,
@@ -1372,7 +1382,7 @@ class RGWRados
   int repair_olh(const DoutPrefixProvider *dpp, RGWObjState* state, const RGWBucketInfo& bucket_info,
                  const rgw_obj& obj, optional_yield y);
   int unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWBucketInfo& bucket_info, const rgw_obj& target_obj,
-                          uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr);
+                          uint64_t olh_epoch, optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
 
   void check_pending_olh_entries(const DoutPrefixProvider *dpp, std::map<std::string, bufferlist>& pending_entries, std::map<std::string, bufferlist> *rm_pending_entries);
   int remove_olh_pending_entries(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, std::map<std::string, bufferlist>& pending_attrs, optional_yield y);
@@ -1430,16 +1440,20 @@ class RGWRados
 			     const std::map<std::string, bufferlist> *pattrs, bool create_entry_point,
                              const DoutPrefixProvider *dpp, optional_yield y);
 
-  int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, std::string& tag, rgw_obj& obj, uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr);
+  int cls_obj_prepare_op(const DoutPrefixProvider *dpp, BucketShard& bs, RGWModifyOp op, std::string& tag, rgw_obj& obj,
+                         uint16_t bilog_flags, optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int cls_obj_complete_op(BucketShard& bs, const rgw_obj& obj, RGWModifyOp op, std::string& tag, int64_t pool, uint64_t epoch,
-                          rgw_bucket_dir_entry& ent, RGWObjCategory category, std::list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr);
+                          rgw_bucket_dir_entry& ent, RGWObjCategory category, std::list<rgw_obj_index_key> *remove_objs,
+                          uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int cls_obj_complete_add(BucketShard& bs, const rgw_obj& obj, std::string& tag, int64_t pool, uint64_t epoch, rgw_bucket_dir_entry& ent,
-                           RGWObjCategory category, std::list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr);
+                           RGWObjCategory category, std::list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags,
+                           rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int cls_obj_complete_del(BucketShard& bs, std::string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj,
-                           ceph::real_time& removed_mtime, std::list<rgw_obj_index_key> *remove_objs, uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr);
+                           ceph::real_time& removed_mtime, std::list<rgw_obj_index_key> *remove_objs,
+                           uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int cls_obj_complete_cancel(BucketShard& bs, std::string& tag, rgw_obj& obj,
                               std::list<rgw_obj_index_key> *remove_objs,
-                              uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr);
+                              uint16_t bilog_flags, rgw_zone_set *zones_trace = nullptr, bool log_op = true);
   int cls_obj_set_bucket_tag_timeout(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, uint64_t timeout);
 
   using ent_map_t =
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index f005d3960646..e6dc69e4c088 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1702,9 +1702,10 @@ int RadosObject::transition(Bucket* bucket,
 			    const real_time& mtime,
 			    uint64_t olh_epoch,
 			    const DoutPrefixProvider* dpp,
-			    optional_yield y)
+			    optional_yield y,
+                            bool log_op)
 {
-  return store->getRados()->transition_obj(*rados_ctx, bucket->get_info(), get_obj(), placement_rule, mtime, olh_epoch, dpp, y);
+  return store->getRados()->transition_obj(*rados_ctx, bucket->get_info(), get_obj(), placement_rule, mtime, olh_epoch, dpp, y, log_op);
 }
 
 int RadosObject::transition_to_cloud(Bucket* bucket,
@@ -1986,7 +1987,7 @@ RadosObject::RadosDeleteOp::RadosDeleteOp(RadosObject *_source) :
 	parent_op(&op_target)
 { }
 
-int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y)
+int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
 {
   parent_op.params.bucket_owner = params.bucket_owner.id;
   parent_op.params.versioning_status = params.versioning_status;
@@ -2003,7 +2004,7 @@ int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, option
   parent_op.params.abortmp = params.abortmp;
   parent_op.params.parts_accounted_size = params.parts_accounted_size;
 
-  int ret = parent_op.delete_obj(y, dpp);
+  int ret = parent_op.delete_obj(y, dpp, log_op);
   if (ret < 0)
     return ret;
 
@@ -2154,7 +2155,7 @@ int RadosMultipartUpload::cleanup_part_history(const DoutPrefixProvider* dpp,
 }
 
 
-int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
+int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
 {
   std::unique_ptr<rgw::sal::Object> meta_obj = get_meta_obj();
   meta_obj->set_in_extra_data(true);
@@ -2182,7 +2183,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
 	std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(
 				    rgw_obj_key(obj_part->oid, std::string(), RGW_OBJ_NS_MULTIPART));
 	obj->set_hash_source(mp_obj.get_key());
-	ret = obj->delete_object(dpp, y);
+	ret = obj->delete_object(dpp, y, log_op);
         if (ret < 0 && ret != -ENOENT)
           return ret;
       } else {
@@ -2232,7 +2233,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
   del_op->params.parts_accounted_size = parts_accounted_size;
 
   // and also remove the metadata obj
-  ret = del_op->delete_obj(dpp, y);
+  ret = del_op->delete_obj(dpp, y, log_op);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " <<
       ret << dendl;
@@ -2849,10 +2850,11 @@ int RadosAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
 }
 
 int RadosAppendWriter::prepare(optional_yield y)
@@ -2872,10 +2874,11 @@ int RadosAppendWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
 }
 
 int RadosMultipartWriter::prepare(optional_yield y)
@@ -2895,10 +2898,11 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
 }
 
 bool RadosZoneGroup::placement_target_exists(std::string& target) const
@@ -3046,7 +3050,7 @@ const std::string_view RadosZone::get_tier_type()
   if (local_zone)
     return store->svc()->zone->get_zone().tier_type;
 
-  return rgw_zone.id;
+  return rgw_zone.tier_type;
 }
 
 RGWBucketSyncPolicyHandlerRef RadosZone::get_sync_policy_handler()
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index fbf14410c173..efd36be19e97 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -332,7 +332,7 @@ class RadosObject : public StoreObject {
     public:
       RadosDeleteOp(RadosObject* _source);
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
     };
 
     RadosObject(RadosStore *_st, const rgw_obj_key& _k)
@@ -413,7 +413,8 @@ class RadosObject : public StoreObject {
 			   const real_time& mtime,
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
-			   optional_yield y) override;
+			   optional_yield y,
+                           bool log_op = true) override;
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
 			   rgw_bucket_dir_entry& o,
@@ -609,7 +610,7 @@ class RadosMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -752,7 +753,8 @@ class RadosAtomicWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class RadosAppendWriter : public StoreWriter {
@@ -799,7 +801,8 @@ class RadosAppendWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class RadosMultipartWriter : public StoreWriter {
@@ -844,7 +847,8 @@ class RadosMultipartWriter : public StoreWriter {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class RadosLuaManager : public StoreLuaManager {
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 4364c2e683c8..43fe22eab1f0 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -512,6 +512,28 @@ struct lc_op_ctx {
 static std::string lc_id = "rgw lifecycle";
 static std::string lc_req_id = "0";
 
+/* do all zones in the zone group process LC? */
+static bool zonegroup_lc_check(const DoutPrefixProvider *dpp, rgw::sal::Zone* zone)
+{
+  auto& zonegroup = zone->get_zonegroup();
+  std::list<std::string> ids;
+  int ret = zonegroup.list_zones(ids);
+  if (ret < 0) {
+    return false;
+  }
+
+  return std::all_of(ids.begin(), ids.end(), [&](const auto& id) {
+    std::unique_ptr<rgw::sal::Zone> zone;
+    ret = zonegroup.get_zone_by_id(id, &zone);
+    if (ret < 0) {
+      return false;
+    }
+    const auto& tier_type = zone->get_tier_type();
+    ldpp_dout(dpp, 20) << "checking zone tier_type=" << tier_type << dendl;
+    return (tier_type == "rgw" || tier_type == "archive" || tier_type == "");
+  });
+}
+
 static int remove_expired_obj(
   const DoutPrefixProvider *dpp, lc_op_ctx& oc, bool remove_indeed,
   rgw::notify::EventType event_type)
@@ -565,7 +587,9 @@ static int remove_expired_obj(
       << dendl;
     return ret;
   }
-  ret =  del_op->delete_obj(dpp, null_yield);
+
+  bool log_op = !remove_indeed || !zonegroup_lc_check(dpp, oc.driver->get_zone());
+  ret =  del_op->delete_obj(dpp, null_yield, log_op);
   if (ret < 0) {
     ldpp_dout(dpp, 1) <<
       fmt::format("ERROR: {} failed, with error: {}", __func__, ret) << dendl;
@@ -861,7 +885,8 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
         return ret;
       }
 
-      ret = mpu->abort(this, cct, null_yield);
+      bool log_op = !zonegroup_lc_check(wk->get_lc(), driver->get_zone());
+      ret = mpu->abort(this, cct, null_yield, log_op);
       if (ret == 0) {
         int publish_ret = notify->publish_commit(
             this, obj_state->size,
@@ -1440,8 +1465,9 @@ class LCOpAction_Transition : public LCOpAction {
         return -EINVAL;
       }
 
+      bool log_op = !zonegroup_lc_check(oc.dpp, oc.driver->get_zone());
       int r = oc.obj->transition(oc.bucket, target_placement, o.meta.mtime,
-	  		         o.versioned_epoch, oc.dpp, null_yield);
+                                 o.versioned_epoch, oc.dpp, null_yield, log_op);
       if (r < 0) {
         ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj " 
 			     << oc.bucket << ":" << o.key 
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 8771b74c87aa..218aad8884b1 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -224,7 +224,8 @@ class ObjectProcessor : public DataProcessor {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) = 0;
+                       const req_context& rctx,
+                       bool log_op = true) = 0;
 };
 
 /** A list of key-value attributes */
@@ -887,7 +888,7 @@ class Object {
       virtual ~DeleteOp() = default;
 
       /** Delete the object */
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) = 0;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) = 0;
     };
 
     Object() {}
@@ -964,7 +965,8 @@ class Object {
 			   const real_time& mtime,
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
-			   optional_yield y) = 0;
+			   optional_yield y,
+                           bool log_op = true) = 0;
     /** Move an object to the cloud */
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
@@ -1143,7 +1145,7 @@ class MultipartUpload {
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) = 0;
   /** Abort this upload */
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) = 0;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) = 0;
   /** Complete this upload, making it available as a normal object */
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
@@ -1372,7 +1374,8 @@ class Writer : public ObjectProcessor {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) = 0;
+                       const req_context& rctx,
+                       bool log_op = true) = 0;
 };
 
 
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 4edb30166dbd..68baf3d8bd46 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -618,7 +618,8 @@ namespace rgw::sal {
       const real_time& mtime,
       uint64_t olh_epoch,
       const DoutPrefixProvider* dpp,
-      optional_yield y)
+      optional_yield y,
+      bool log_op)
   {
     DB::Object op_target(store->getDB(),
         get_bucket()->get_info(), get_obj());
@@ -699,7 +700,7 @@ namespace rgw::sal {
     parent_op(&op_target)
   { }
 
-  int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y)
+  int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
   {
     parent_op.params.bucket_owner = params.bucket_owner.id;
     parent_op.params.versioning_status = params.versioning_status;
@@ -785,7 +786,7 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
+  int DBMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
   {
     std::unique_ptr<rgw::sal::Object> meta_obj = get_meta_obj();
     meta_obj->set_in_extra_data(true);
@@ -799,7 +800,7 @@ namespace rgw::sal {
     // Since the data objects are associated with meta obj till
     // MultipartUpload::Complete() is done, removing the metadata obj
     // should remove all the uploads so far.
-    ret = del_op->delete_obj(dpp, null_yield);
+    ret = del_op->delete_obj(dpp, null_yield, log_op);
     if (ret < 0) {
       ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " <<
         ret << dendl;
@@ -1226,7 +1227,8 @@ namespace rgw::sal {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
   {
     /* XXX: same as AtomicWriter..consolidate code */
     parent_op.meta.mtime = mtime;
@@ -1380,7 +1382,8 @@ namespace rgw::sal {
                          const char *if_match, const char *if_nomatch,
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
-                         const req_context& rctx)
+                         const req_context& rctx,
+                         bool log_op)
   {
     parent_op.meta.mtime = mtime;
     parent_op.meta.delete_at = delete_at;
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index bfa424e269fe..0da065e4a8f5 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -462,7 +462,7 @@ class DBNotification : public StoreNotification {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
+    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
     virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -524,7 +524,7 @@ class DBNotification : public StoreNotification {
         public:
           DBDeleteOp(DBObject* _source);
 
-          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
       };
 
       DBObject() = default;
@@ -579,7 +579,8 @@ class DBNotification : public StoreNotification {
           const real_time& mtime,
           uint64_t olh_epoch,
           const DoutPrefixProvider* dpp,
-          optional_yield y) override;
+          optional_yield y,
+          bool log_op = true) override;
       virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
       virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -655,7 +656,8 @@ class DBNotification : public StoreNotification {
                          const char *if_match, const char *if_nomatch,
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
-                         const req_context& rctx) override;
+                         const req_context& rctx,
+                         bool log_op = true) override;
   };
 
   class DBMultipartWriter : public StoreWriter {
@@ -703,7 +705,8 @@ class DBNotification : public StoreNotification {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
   };
 
   class DBStore : public StoreDriver {
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index decd648adb0f..ebb72e212366 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -856,10 +856,11 @@ int FilterObject::transition(Bucket* bucket,
 			     const real_time& mtime,
 			     uint64_t olh_epoch,
 			     const DoutPrefixProvider* dpp,
-			     optional_yield y)
+			     optional_yield y,
+                             bool log_op)
 {
   return next->transition(nextBucket(bucket), placement_rule, mtime, olh_epoch,
-			  dpp, y);
+			  dpp, y, log_op);
 }
 
 int FilterObject::transition_to_cloud(Bucket* bucket,
@@ -979,11 +980,11 @@ int FilterObject::FilterReadOp::iterate(const DoutPrefixProvider* dpp, int64_t o
 }
 
 int FilterObject::FilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-					   optional_yield y)
+					   optional_yield y, bool log_op)
 {
   /* Copy params into next */
   next->params = params;
-  int ret = next->delete_obj(dpp, y);
+  int ret = next->delete_obj(dpp, y, log_op);
   /* Copy result back */
   result = next->result;
   return ret;
@@ -1024,7 +1025,7 @@ int FilterMultipartUpload::list_parts(const DoutPrefixProvider *dpp, CephContext
   return 0;
 }
 
-int FilterMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
+int FilterMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
 {
   return next->abort(dpp, cct, y);
 }
@@ -1201,11 +1202,12 @@ int FilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx)
+                       const req_context& rctx,
+                       bool log_op)
 {
   return next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, rctx);
+			canceled, rctx, log_op);
 }
 
 int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index aebbe383714d..ff1c346950ae 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -521,7 +521,7 @@ class FilterObject : public Object {
     FilterDeleteOp(std::unique_ptr<DeleteOp> _next) : next(std::move(_next)) {}
     virtual ~FilterDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
   };
 
   FilterObject(std::unique_ptr<Object> _next) : next(std::move(_next)) {}
@@ -583,7 +583,8 @@ class FilterObject : public Object {
 			 const real_time& mtime,
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
-			 optional_yield y) override;
+			 optional_yield y,
+                         bool log_op = true) override;
   virtual int transition_to_cloud(Bucket* bucket,
 				  rgw::sal::PlacementTier* tier,
 				  rgw_bucket_dir_entry& o,
@@ -696,7 +697,7 @@ class FilterMultipartUpload : public MultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -840,7 +841,8 @@ class FilterWriter : public Writer {
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx) override;
+                       const req_context& rctx,
+                       bool log_op = true) override;
 };
 
 class FilterLuaManager : public LuaManager {

From a15b7d1cc5197f778d2fe0f4a8c2fb2be6cd9435 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Thu, 7 Dec 2023 13:02:28 -0500
Subject: [PATCH 1343/2492] rgw/lc: remove default arguments in virtual
 functions

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc     |  2 +-
 src/rgw/driver/d4n/rgw_sal_d4n.h      |  4 ++--
 src/rgw/driver/daos/rgw_sal_daos.h    | 10 +++++-----
 src/rgw/driver/motr/rgw_sal_motr.h    | 10 +++++-----
 src/rgw/driver/posix/rgw_sal_posix.h  | 10 +++++-----
 src/rgw/driver/rados/rgw_cr_rados.cc  |  2 +-
 src/rgw/driver/rados/rgw_rados.cc     |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.h  | 12 ++++++------
 src/rgw/rgw_file.cc                   |  2 +-
 src/rgw/rgw_op.cc                     | 14 +++++++-------
 src/rgw/rgw_sal.h                     | 10 +++++-----
 src/rgw/rgw_sal_dbstore.h             | 10 +++++-----
 src/rgw/rgw_sal_filter.cc             |  2 +-
 src/rgw/rgw_sal_filter.h              |  8 ++++----
 src/test/rgw/test_d4n_filter.cc       | 12 ++++++------
 16 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index b10dd760d78a..8647519855d3 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -379,7 +379,7 @@ int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp
     ldpp_dout(dpp, 20) << "D4N Filter: Cache delete operation succeeded." << dendl;
   }
 
-  return next->delete_obj(dpp, y);
+  return next->delete_obj(dpp, y, log_op);
 }
 
 int D4NFilterWriter::prepare(optional_yield y) 
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index 95696efc543e..d6ab2f7351bf 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -111,7 +111,7 @@ class D4NFilterObject : public FilterObject {
 										     source(_source) {}
       virtual ~D4NFilterDeleteOp() = default;
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
     };
 
     D4NFilterObject(std::unique_ptr<Object> _next, D4NFilterDriver* _filter) : FilterObject(std::move(_next)),
@@ -178,7 +178,7 @@ class D4NFilterWriter : public FilterWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; }
 };
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 3b067bcdffbd..db024169a8c9 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -581,7 +581,7 @@ class DaosObject : public StoreObject {
     DaosDeleteOp(DaosObject* _source);
 
     virtual int delete_obj(const DoutPrefixProvider* dpp,
-                           optional_yield y, bool log_op = true) override;
+                           optional_yield y, bool log_op) override;
   };
 
   ds3_obj_t* ds3o = nullptr;
@@ -642,7 +642,7 @@ class DaosObject : public StoreObject {
                          const real_time& mtime, uint64_t olh_epoch,
                          const DoutPrefixProvider* dpp,
                          optional_yield y,
-                         bool log_op = true) override;
+                         bool log_op) override;
   virtual int transition_to_cloud(Bucket* bucket, rgw::sal::PlacementTier* tier,
                                   rgw_bucket_dir_entry& o,
                                   std::set<std::string>& cloud_targets,
@@ -747,7 +747,7 @@ class DaosAtomicWriter : public StoreWriter {
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class DaosMultipartWriter : public StoreWriter {
@@ -793,7 +793,7 @@ class DaosMultipartWriter : public StoreWriter {
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
                        optional_yield y,
-                       bool log_op = true) override;
+                       bool log_op) override;
 
   const std::string& get_bucket_name();
 };
@@ -849,7 +849,7 @@ class DaosMultipartUpload : public StoreMultipartUpload {
                          int num_parts, int marker, int* next_marker,
                          bool* truncated,
                          bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
   virtual int complete(const DoutPrefixProvider* dpp, optional_yield y,
                        CephContext* cct, std::map<int, std::string>& part_etags,
                        std::list<rgw_obj_index_key>& remove_objs,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index a6f096424221..c15212415d80 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -660,7 +660,7 @@ class MotrObject : public StoreObject {
       public:
         MotrDeleteOp(MotrObject* _source);
 
-        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
     };
 
     MotrObject() = default;
@@ -711,7 +711,7 @@ class MotrObject : public StoreObject {
         uint64_t olh_epoch,
         const DoutPrefixProvider* dpp,
         optional_yield y,
-        bool log_op = true) override;
+        bool log_op) override;
     virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
     virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -814,7 +814,7 @@ class MotrAtomicWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 
   unsigned populate_bvec(unsigned len, bufferlist::iterator &bi);
   void cleanup();
@@ -862,7 +862,7 @@ class MotrMultipartWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        optional_yield y,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 // The implementation of multipart upload in POC roughly follows the
@@ -937,7 +937,7 @@ class MotrMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 67931a4677a4..42c46242640c 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -298,7 +298,7 @@ class POSIXObject : public StoreObject {
       source(_source) {}
     virtual ~POSIXDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
   };
 
   POSIXObject(POSIXDriver *_dr, const rgw_obj_key& _k)
@@ -356,7 +356,7 @@ class POSIXObject : public StoreObject {
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
 			 optional_yield y,
-                         bool log_op = true) override;
+                         bool log_op) override;
   virtual int transition_to_cloud(Bucket* bucket,
 			 rgw::sal::PlacementTier* tier,
 			 rgw_bucket_dir_entry& o,
@@ -545,7 +545,7 @@ class POSIXMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -608,7 +608,7 @@ class POSIXAtomicWriter : public StoreWriter {
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
 		       const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class POSIXMultipartWriter : public StoreWriter {
@@ -648,7 +648,7 @@ class POSIXMultipartWriter : public StoreWriter {
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
 		       const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 
 };
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index 2a4a02f29adc..b7d362423e53 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -922,7 +922,7 @@ int RGWAsyncRemoveObj::_send_request(const DoutPrefixProvider *dpp)
   del_op->params.high_precision_time = true;
   del_op->params.zones_trace = &zones_trace;
 
-  ret = del_op->delete_obj(dpp, null_yield);
+  ret = del_op->delete_obj(dpp, null_yield, true);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << "(): delete_obj() obj=" << obj << " returned ret=" << ret << dendl;
   }
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b2ffef0589fa..a8ecb4bfa099 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -5653,7 +5653,7 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi
       }
       result.delete_marker = dirent.is_delete_marker();
       r = store->unlink_obj_instance(dpp, target->get_ctx(), target->get_bucket_info(), obj, params.olh_epoch,
-                                     y, params.zones_trace, log_op);
+                                     y, params.zones_trace, add_log);
       if (r < 0) {
         return r;
       }
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index e6dc69e4c088..e5ac7229bf6c 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -811,7 +811,7 @@ int RadosBucket::abort_multiparts(const DoutPrefixProvider* dpp,
 
     if (!uploads.empty()) {
       for (const auto& upload : uploads) {
-	ret = upload->abort(dpp, cct, y);
+	ret = upload->abort(dpp, cct, y, true);
         if (ret < 0) {
 	  // we're doing a best-effort; if something cannot be found,
 	  // log it and keep moving forward
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index efd36be19e97..a05403856111 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -332,7 +332,7 @@ class RadosObject : public StoreObject {
     public:
       RadosDeleteOp(RadosObject* _source);
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
     };
 
     RadosObject(RadosStore *_st, const rgw_obj_key& _k)
@@ -414,7 +414,7 @@ class RadosObject : public StoreObject {
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
 			   optional_yield y,
-                           bool log_op = true) override;
+                           bool log_op) override;
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
 			   rgw_bucket_dir_entry& o,
@@ -610,7 +610,7 @@ class RadosMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -754,7 +754,7 @@ class RadosAtomicWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class RadosAppendWriter : public StoreWriter {
@@ -802,7 +802,7 @@ class RadosAppendWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class RadosMultipartWriter : public StoreWriter {
@@ -848,7 +848,7 @@ class RadosMultipartWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class RadosLuaManager : public StoreLuaManager {
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index f0100e37c2bb..f69ba543d898 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -2020,7 +2020,7 @@ namespace rgw {
     op_ret = processor->complete(state->obj_size, etag, &mtime, real_time(), attrs,
                                  (delete_at ? *delete_at : real_time()),
                                 if_match, if_nomatch, nullptr, nullptr, nullptr,
-                                rctx);
+                                rctx, true);
     if (op_ret != 0) {
       /* revert attr updates */
       rgw_fh->set_mtime(omtime);
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 521630e05b0f..12fc4a706d9c 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4530,7 +4530,7 @@ void RGWPutObj::execute(optional_yield y)
   op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs,
                                (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
                                (user_data.empty() ? nullptr : &user_data), nullptr, nullptr,
-                               rctx);
+                               rctx, true);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
 
   // send request to notification manager
@@ -4799,7 +4799,7 @@ void RGWPostObj::execute(optional_yield y)
     op_ret = processor->complete(s->obj_size, etag, nullptr, real_time(), attrs,
                                 (delete_at ? *delete_at : real_time()),
                                 nullptr, nullptr, nullptr, nullptr, nullptr,
-                                rctx);
+                                rctx, true);
     if (op_ret < 0) {
       return;
     }
@@ -5357,7 +5357,7 @@ void RGWDeleteObj::execute(optional_yield y)
       del_op->params.olh_epoch = epoch;
       del_op->params.marker_version_id = version_id;
 
-      op_ret = del_op->delete_obj(this, y);
+      op_ret = del_op->delete_obj(this, y, true);
       if (op_ret >= 0) {
 	delete_marker = del_op->result.delete_marker;
 	version_id = del_op->result.version_id;
@@ -6896,7 +6896,7 @@ void RGWAbortMultipart::execute(optional_yield y)
   }
   multipart_trace = tracing::rgw::tracer.add_span(name(), trace_ctx);
 
-  op_ret = upload->abort(this, s->cct, y);
+  op_ret = upload->abort(this, s->cct, y, true);
 }
 
 int RGWListMultipart::verify_permission(optional_yield y)
@@ -7262,7 +7262,7 @@ void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key& o, optional_
   del_op->params.bucket_owner = s->bucket_owner;
   del_op->params.marker_version_id = version_id;
 
-  op_ret = del_op->delete_obj(this, y);
+  op_ret = del_op->delete_obj(this, y, true);
   if (op_ret == -ENOENT) {
     op_ret = 0;
   }
@@ -7436,7 +7436,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
     del_op->params.obj_owner = bowner;
     del_op->params.bucket_owner = bucket_owner;
 
-    ret = del_op->delete_obj(dpp, y);
+    ret = del_op->delete_obj(dpp, y, true);
     if (ret < 0) {
       goto delop_fail;
     }
@@ -7949,7 +7949,7 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   op_ret = processor->complete(size, etag, nullptr, ceph::real_time(),
                               attrs, ceph::real_time() /* delete_at */,
                               nullptr, nullptr, nullptr, nullptr, nullptr,
-                              rctx);
+                              rctx, true);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "processor::complete returned op_ret=" << op_ret << dendl;
   }
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 218aad8884b1..33891d08a4c1 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -225,7 +225,7 @@ class ObjectProcessor : public DataProcessor {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) = 0;
+                       bool log_op) = 0;
 };
 
 /** A list of key-value attributes */
@@ -888,7 +888,7 @@ class Object {
       virtual ~DeleteOp() = default;
 
       /** Delete the object */
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) = 0;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) = 0;
     };
 
     Object() {}
@@ -966,7 +966,7 @@ class Object {
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
 			   optional_yield y,
-                           bool log_op = true) = 0;
+                           bool log_op) = 0;
     /** Move an object to the cloud */
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
@@ -1145,7 +1145,7 @@ class MultipartUpload {
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) = 0;
   /** Abort this upload */
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) = 0;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) = 0;
   /** Complete this upload, making it available as a normal object */
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
@@ -1375,7 +1375,7 @@ class Writer : public ObjectProcessor {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) = 0;
+                       bool log_op) = 0;
 };
 
 
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 0da065e4a8f5..d39fb3f62550 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -462,7 +462,7 @@ class DBNotification : public StoreNotification {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
     virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -524,7 +524,7 @@ class DBNotification : public StoreNotification {
         public:
           DBDeleteOp(DBObject* _source);
 
-          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
       };
 
       DBObject() = default;
@@ -580,7 +580,7 @@ class DBNotification : public StoreNotification {
           uint64_t olh_epoch,
           const DoutPrefixProvider* dpp,
           optional_yield y,
-          bool log_op = true) override;
+          bool log_op) override;
       virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
       virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -657,7 +657,7 @@ class DBNotification : public StoreNotification {
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
                          const req_context& rctx,
-                         bool log_op = true) override;
+                         bool log_op) override;
   };
 
   class DBMultipartWriter : public StoreWriter {
@@ -706,7 +706,7 @@ class DBNotification : public StoreNotification {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
   };
 
   class DBStore : public StoreDriver {
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index ebb72e212366..953cf54e0408 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -1027,7 +1027,7 @@ int FilterMultipartUpload::list_parts(const DoutPrefixProvider *dpp, CephContext
 
 int FilterMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
 {
-  return next->abort(dpp, cct, y);
+  return next->abort(dpp, cct, y, log_op);
 }
 
 int FilterMultipartUpload::complete(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index ff1c346950ae..d258cda4a654 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -521,7 +521,7 @@ class FilterObject : public Object {
     FilterDeleteOp(std::unique_ptr<DeleteOp> _next) : next(std::move(_next)) {}
     virtual ~FilterDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op = true) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
   };
 
   FilterObject(std::unique_ptr<Object> _next) : next(std::move(_next)) {}
@@ -584,7 +584,7 @@ class FilterObject : public Object {
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
 			 optional_yield y,
-                         bool log_op = true) override;
+                         bool log_op) override;
   virtual int transition_to_cloud(Bucket* bucket,
 				  rgw::sal::PlacementTier* tier,
 				  rgw_bucket_dir_entry& o,
@@ -697,7 +697,7 @@ class FilterMultipartUpload : public MultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op = true) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -842,7 +842,7 @@ class FilterWriter : public Writer {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op = true) override;
+                       bool log_op) override;
 };
 
 class FilterLuaManager : public LuaManager {
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index 4f54d9b86bb0..b1483f26e0db 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -169,7 +169,7 @@ class D4NFilterFixture : public ::testing::Test {
                        &if_match, &if_nomatch,
                        &user_data,
                        &zones_trace, &canceled,
-                       rctx);
+                       rctx, true);
 
       return ret;
     }
@@ -429,7 +429,7 @@ TEST_F(D4NFilterFixture, CopyObjectReplace) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   rctx), 0);
+		   rctx, true), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -554,7 +554,7 @@ TEST_F(D4NFilterFixture, CopyObjectMerge) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   rctx), 0);
+		   rctx, true), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -639,7 +639,7 @@ TEST_F(D4NFilterFixture, DelObject) {
   unique_ptr<rgw::sal::Object::DeleteOp> testDOp = testObject_DelObject->get_delete_op();
 
   EXPECT_NE(testDOp, nullptr);
-  EXPECT_EQ(testDOp->delete_obj(dpp, null_yield), 0);
+  EXPECT_EQ(testDOp->delete_obj(dpp, null_yield, true), 0);
 
   /* Check the object does not exist after delete op */
   client.exists(keys, [](cpp_redis::reply& reply) {
@@ -1881,7 +1881,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 rctx), 0);
+		 rctx, true), 0);
  
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&data](cpp_redis::reply& reply) {
     if (reply.is_string()) {
@@ -1906,7 +1906,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 rctx), 0);
+		 rctx, true), 0);
 
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&dataNew](cpp_redis::reply& reply) {
     if (reply.is_string()) {

From 17ca50e70421cfe263784e8d65008145221e41c3 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Tue, 12 Dec 2023 13:57:00 -0500
Subject: [PATCH 1344/2492] rgw/lc: use flag and named constants instead of
 booleans as argument in interface

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/driver/d4n/rgw_sal_d4n.cc             |  8 ++---
 src/rgw/driver/d4n/rgw_sal_d4n.h              |  4 +--
 src/rgw/driver/daos/rgw_sal_daos.cc           | 14 ++++----
 src/rgw/driver/daos/rgw_sal_daos.h            | 12 +++----
 src/rgw/driver/motr/rgw_sal_motr.cc           | 16 ++++-----
 src/rgw/driver/motr/rgw_sal_motr.h            | 12 +++----
 src/rgw/driver/posix/rgw_sal_posix.cc         | 24 ++++++-------
 src/rgw/driver/posix/rgw_sal_posix.h          | 14 ++++----
 src/rgw/driver/rados/rgw_bucket.cc            |  2 +-
 .../driver/rados/rgw_object_expirer_core.cc   |  2 +-
 src/rgw/driver/rados/rgw_putobj_processor.cc  | 14 ++++----
 src/rgw/driver/rados/rgw_putobj_processor.h   |  6 ++--
 src/rgw/driver/rados/rgw_rados.cc             |  7 ++--
 src/rgw/driver/rados/rgw_sal_rados.cc         | 36 ++++++++++---------
 src/rgw/driver/rados/rgw_sal_rados.h          | 14 ++++----
 src/rgw/rgw_data_access.cc                    |  3 +-
 src/rgw/rgw_lc.cc                             | 13 +++----
 src/rgw/rgw_op.cc                             | 16 ++++-----
 src/rgw/rgw_sal.h                             | 15 ++++----
 src/rgw/rgw_sal_dbstore.cc                    | 14 ++++----
 src/rgw/rgw_sal_dbstore.h                     | 12 +++----
 src/rgw/rgw_sal_filter.cc                     | 20 +++++------
 src/rgw/rgw_sal_filter.h                      | 10 +++---
 23 files changed, 149 insertions(+), 139 deletions(-)

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index 8647519855d3..e2624690ebdd 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -361,7 +361,7 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix
 }
 
 int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-                                                   optional_yield y, bool log_op)
+                                                   optional_yield y, uint32_t flags)
 {
   int delDirReturn = source->filter->get_block_dir()->delValue(source->filter->get_cache_block());
 
@@ -379,7 +379,7 @@ int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp
     ldpp_dout(dpp, 20) << "D4N Filter: Cache delete operation succeeded." << dendl;
   }
 
-  return next->delete_obj(dpp, y, log_op);
+  return next->delete_obj(dpp, y, flags);
 }
 
 int D4NFilterWriter::prepare(optional_yield y) 
@@ -416,7 +416,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   cache_block* temp_cache_block = filter->get_cache_block();
   RGWBlockDirectory* temp_block_dir = filter->get_block_dir();
@@ -438,7 +438,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
   RGWObjState* astate;
   int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, rctx, log_op);
+			canceled, rctx, flags);
   obj->get_obj_attrs(rctx.y, save_dpp, NULL);
   obj->get_obj_state(save_dpp, &astate, rctx.y);
 
diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index d6ab2f7351bf..840eb99f6048 100644
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -111,7 +111,7 @@ class D4NFilterObject : public FilterObject {
 										     source(_source) {}
       virtual ~D4NFilterDeleteOp() = default;
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
     };
 
     D4NFilterObject(std::unique_ptr<Object> _next, D4NFilterDriver* _filter) : FilterObject(std::move(_next)),
@@ -178,7 +178,7 @@ class D4NFilterWriter : public FilterWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; }
 };
diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 6e6581be9018..43ae35be4265 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -1018,7 +1018,7 @@ int DaosObject::transition(Bucket* bucket,
                            const rgw_placement_rule& placement_rule,
                            const real_time& mtime, uint64_t olh_epoch,
                            const DoutPrefixProvider* dpp, optional_yield y,
-                           bool log_op) {
+                           uint32_t flags) {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
 
@@ -1168,7 +1168,7 @@ DaosObject::DaosDeleteOp::DaosDeleteOp(DaosObject* _source) : source(_source) {}
 // 3. Handle empty directories
 // 4. Fail when file doesn't exist
 int DaosObject::DaosDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-                                         optional_yield y, bool log_op) {
+                                         optional_yield y, uint32_t flags) {
   ldpp_dout(dpp, 20) << "DaosDeleteOp::delete_obj "
                      << source->get_key().get_oid() << " from "
                      << source->get_bucket()->get_name() << dendl;
@@ -1197,13 +1197,13 @@ int DaosObject::DaosDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
 }
 
 int DaosObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y,
-                              bool prevent_versioning) {
+                              uint32_t flags) {
   ldpp_dout(dpp, 20) << "DEBUG: delete_object" << dendl;
   DaosObject::DaosDeleteOp del_op(this);
   del_op.params.bucket_owner = bucket->get_info().owner;
   del_op.params.versioning_status = bucket->get_info().versioning_status();
 
-  return del_op.delete_obj(dpp, y);
+  return del_op.delete_obj(dpp, y, flags);
 }
 
 int DaosObject::copy_object(
@@ -1507,7 +1507,7 @@ int DaosAtomicWriter::complete(
     ceph::real_time set_mtime, std::map<std::string, bufferlist>& attrs,
     ceph::real_time delete_at, const char* if_match, const char* if_nomatch,
     const std::string* user_data, rgw_zone_set* zones_trace, bool* canceled,
-    optional_yield y, bool log_op) {
+    optional_yield y, uint32_t flags) {
   ldpp_dout(dpp, 20) << "DEBUG: complete" << dendl;
   bufferlist bl;
   rgw_bucket_dir_entry ent;
@@ -1564,7 +1564,7 @@ int DaosAtomicWriter::complete(
 }
 
 int DaosMultipartUpload::abort(const DoutPrefixProvider* dpp,
-                               CephContext* cct, optional_yield y, bool log_op) {
+                               CephContext* cct, optional_yield y) {
   // Remove upload from bucket multipart index
   ldpp_dout(dpp, 20) << "DEBUG: abort" << dendl;
   return ds3_upload_remove(bucket->get_name().c_str(), get_upload_id().c_str(),
@@ -2031,7 +2031,7 @@ int DaosMultipartWriter::complete(
     ceph::real_time set_mtime, std::map<std::string, bufferlist>& attrs,
     ceph::real_time delete_at, const char* if_match, const char* if_nomatch,
     const std::string* user_data, rgw_zone_set* zones_trace, bool* canceled,
-    const req_context& rctx, bool log_op) {
+    const req_context& rctx, uint32_t flags) {
   ldpp_dout(dpp, 20) << "DaosMultipartWriter::complete(): enter part="
                      << part_num_str << dendl;
 
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index db024169a8c9..c5cfefc222d1 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -581,7 +581,7 @@ class DaosObject : public StoreObject {
     DaosDeleteOp(DaosObject* _source);
 
     virtual int delete_obj(const DoutPrefixProvider* dpp,
-                           optional_yield y, bool log_op) override;
+                           optional_yield y, uint32_t flags) override;
   };
 
   ds3_obj_t* ds3o = nullptr;
@@ -598,7 +598,7 @@ class DaosObject : public StoreObject {
   virtual ~DaosObject();
 
   virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y,
-                            bool prevent_versioning = false) override;
+                            uint32_t flags) override;
   virtual int copy_object(
       User* user, req_info* info, const rgw_zone_id& source_zone,
       rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -642,7 +642,7 @@ class DaosObject : public StoreObject {
                          const real_time& mtime, uint64_t olh_epoch,
                          const DoutPrefixProvider* dpp,
                          optional_yield y,
-                         bool log_op) override;
+                         uint32_t flags) override;
   virtual int transition_to_cloud(Bucket* bucket, rgw::sal::PlacementTier* tier,
                                   rgw_bucket_dir_entry& o,
                                   std::set<std::string>& cloud_targets,
@@ -747,7 +747,7 @@ class DaosAtomicWriter : public StoreWriter {
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class DaosMultipartWriter : public StoreWriter {
@@ -793,7 +793,7 @@ class DaosMultipartWriter : public StoreWriter {
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
                        optional_yield y,
-                       bool log_op) override;
+                       uint32_t flags) override;
 
   const std::string& get_bucket_name();
 };
@@ -849,7 +849,7 @@ class DaosMultipartUpload : public StoreMultipartUpload {
                          int num_parts, int marker, int* next_marker,
                          bool* truncated,
                          bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
   virtual int complete(const DoutPrefixProvider* dpp, optional_yield y,
                        CephContext* cct, std::map<int, std::string>& part_etags,
                        std::list<rgw_obj_index_key>& remove_objs,
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 6bb7f19f5cce..55a396436167 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -584,7 +584,7 @@ int MotrBucket::remove(const DoutPrefixProvider *dpp, bool delete_children, opti
 
       std::unique_ptr<rgw::sal::Object> object = get_object(key);
 
-      ret = object->delete_object(dpp, null_yield);
+      ret = object->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP);
       if (ret < 0 && ret != -ENOENT) {
         ldpp_dout(dpp, 0) << "ERROR: remove_bucket rgw_remove_object failed rc=" << ret << dendl;
 	      return ret;
@@ -1295,7 +1295,7 @@ int MotrObject::transition(Bucket* bucket,
     uint64_t olh_epoch,
     const DoutPrefixProvider* dpp,
     optional_yield y,
-    bool log_op)
+    uint32_t flags)
 {
   return 0;
 }
@@ -1457,7 +1457,7 @@ MotrObject::MotrDeleteOp::MotrDeleteOp(MotrObject *_source) :
 // Delete::delete_obj() in rgw_rados.cc shows how rados backend process the
 // params.
 // 2. Delete an object when its versioning is turned on.
-int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
+int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
 {
   ldpp_dout(dpp, 20) << "delete " << source->get_key().get_oid() << " from " << source->get_bucket()->get_name() << dendl;
 
@@ -1501,13 +1501,13 @@ int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional
   return 0;
 }
 
-int MotrObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, bool prevent_versioning)
+int MotrObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
 {
   MotrObject::MotrDeleteOp del_op(this);
   del_op.params.bucket_owner = bucket->get_info().owner;
   del_op.params.versioning_status = bucket->get_info().versioning_status();
 
-  return del_op.delete_obj(dpp, y);
+  return del_op.delete_obj(dpp, y, flags);
 }
 
 int MotrObject::copy_object(User* user,
@@ -2326,7 +2326,7 @@ int MotrAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   int rc = 0;
 
@@ -2465,7 +2465,7 @@ int MotrMultipartUpload::delete_parts(const DoutPrefixProvider *dpp)
   return store->delete_motr_idx_by_name(obj_part_iname);
 }
 
-int MotrMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
+int MotrMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
 {
   int rc;
   // Check if multipart upload exists
@@ -2984,7 +2984,7 @@ int MotrMultipartWriter::complete(size_t accounted_size, const std::string& etag
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        optional_yield y,
-                       bool log_op)
+                       uint32_t flags)
 {
   // Should the dir entry(object metadata) be updated? For example
   // mtime.
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index c15212415d80..3cc3b37fa9ad 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -660,7 +660,7 @@ class MotrObject : public StoreObject {
       public:
         MotrDeleteOp(MotrObject* _source);
 
-        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+        virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
     };
 
     MotrObject() = default;
@@ -676,7 +676,7 @@ class MotrObject : public StoreObject {
 
     virtual int delete_object(const DoutPrefixProvider* dpp,
         optional_yield y,
-        bool prevent_versioning = false) override;
+        uint32_t flags) override;
     virtual int copy_object(User* user,
         req_info* info, const rgw_zone_id& source_zone,
         rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -711,7 +711,7 @@ class MotrObject : public StoreObject {
         uint64_t olh_epoch,
         const DoutPrefixProvider* dpp,
         optional_yield y,
-        bool log_op) override;
+        uint32_t flags) override;
     virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
     virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -814,7 +814,7 @@ class MotrAtomicWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 
   unsigned populate_bvec(unsigned len, bufferlist::iterator &bi);
   void cleanup();
@@ -862,7 +862,7 @@ class MotrMultipartWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        optional_yield y,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 // The implementation of multipart upload in POC roughly follows the
@@ -937,7 +937,7 @@ class MotrMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 2670f7872cd2..8ca6a09275bd 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -1371,7 +1371,7 @@ int POSIXBucket::copy(const DoutPrefixProvider *dpp, optional_yield y,
   std::unique_ptr<POSIXBucket> dsb;
 
   // Delete the target, in case it's not a multipart
-  int ret = dest->delete_object(dpp, y);
+  int ret = dest->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
                       << dest->get_name() << dendl;
@@ -1423,7 +1423,7 @@ int POSIXBucket::copy(const DoutPrefixProvider *dpp, optional_yield y,
 
 int POSIXObject::delete_object(const DoutPrefixProvider* dpp,
 				optional_yield y,
-				bool prevent_versioning)
+				uint32_t flags)
 {
   POSIXBucket *b = static_cast<POSIXBucket*>(get_bucket());
   if (!b) {
@@ -1659,7 +1659,7 @@ int POSIXObject::transition(Bucket* bucket,
 			    uint64_t olh_epoch,
 			    const DoutPrefixProvider* dpp,
 			    optional_yield y,
-                            bool log_op)
+                            uint32_t flags)
 {
   return -ERR_NOT_IMPLEMENTED;
 }
@@ -1890,7 +1890,7 @@ int POSIXObject::open(const DoutPrefixProvider* dpp, bool create, bool temp_file
   return 0;
 }
 
-int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y)
+int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y, uint32_t flags)
 {
   if (obj_fd < 0) {
     return 0;
@@ -1916,7 +1916,7 @@ int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y)
   }
 
   // Delete the target, in case it's a multipart
-  ret = delete_object(dpp, y);
+  ret = delete_object(dpp, y, flags);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
                       << get_name() << dendl;
@@ -2367,9 +2367,9 @@ int POSIXObject::POSIXReadOp::get_attr(const DoutPrefixProvider* dpp, const char
 }
 
 int POSIXObject::POSIXDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-					   optional_yield y, bool log_op)
+					   optional_yield y, uint32_t flags)
 {
-  return source->delete_object(dpp, y, false);
+  return source->delete_object(dpp, y, flags);
 }
 
 int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y,
@@ -2385,7 +2385,7 @@ int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y,
   }
 
   // Delete the target, in case it's a multipart
-  ret = dobj->delete_object(dpp, y);
+  ret = dobj->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: could not remove dest object "
                       << dobj->get_name() << dendl;
@@ -2573,7 +2573,7 @@ int POSIXMultipartUpload::list_parts(const DoutPrefixProvider *dpp, CephContext
   return 0;
 }
 
-int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
+int POSIXMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
 {
   int ret;
 
@@ -2811,7 +2811,7 @@ int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   int ret;
   POSIXUploadPartInfo info;
@@ -2879,7 +2879,7 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   int ret;
 
@@ -2928,7 +2928,7 @@ int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
     }
   }
 
-  ret = obj.link_temp_file(rctx.dpp, rctx.y);
+  ret = obj.link_temp_file(rctx.dpp, rctx.y, flags);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << "ERROR: POSIXAtomicWriter failed writing temp file" << dendl;
     return ret;
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 42c46242640c..61ca13dc76f5 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -298,7 +298,7 @@ class POSIXObject : public StoreObject {
       source(_source) {}
     virtual ~POSIXDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
   };
 
   POSIXObject(POSIXDriver *_dr, const rgw_obj_key& _k)
@@ -319,7 +319,7 @@ class POSIXObject : public StoreObject {
 
   virtual int delete_object(const DoutPrefixProvider* dpp,
 			    optional_yield y,
-			    bool prevent_versioning = false) override;
+			    uint32_t flags) override;
   virtual int copy_object(User* user,
                req_info* info, const rgw_zone_id& source_zone,
                rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -356,7 +356,7 @@ class POSIXObject : public StoreObject {
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
 			 optional_yield y,
-                         bool log_op) override;
+                         uint32_t flags) override;
   virtual int transition_to_cloud(Bucket* bucket,
 			 rgw::sal::PlacementTier* tier,
 			 rgw_bucket_dir_entry& o,
@@ -387,7 +387,7 @@ class POSIXObject : public StoreObject {
   int close();
   int write(int64_t ofs, bufferlist& bl, const DoutPrefixProvider* dpp, optional_yield y);
   int write_attr(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& value);
-  int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y);
+  int link_temp_file(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags);
   void gen_temp_fname();
   /* TODO dang Escape the object name for file use */
   const std::string get_fname();
@@ -545,7 +545,7 @@ class POSIXMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -608,7 +608,7 @@ class POSIXAtomicWriter : public StoreWriter {
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
 		       const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class POSIXMultipartWriter : public StoreWriter {
@@ -648,7 +648,7 @@ class POSIXMultipartWriter : public StoreWriter {
 		       const std::string *user_data,
 		       rgw_zone_set *zones_trace, bool *canceled,
 		       const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 
 };
 
diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index aac80909634f..d0eacf81e8c1 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -148,7 +148,7 @@ int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, r
 
   std::unique_ptr<rgw::sal::Object> object = bucket->get_object(key);
 
-  return object->delete_object(dpp, y);
+  return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP);
 }
 
 static void set_err_msg(std::string *sink, std::string msg)
diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc
index 269043f94df1..c285443d0b0a 100644
--- a/src/rgw/driver/rados/rgw_object_expirer_core.cc
+++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc
@@ -225,7 +225,7 @@ int RGWObjectExpirer::garbage_single_object(const DoutPrefixProvider *dpp, objex
 
   std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(key);
   obj->set_atomic();
-  ret = obj->delete_object(dpp, null_yield);
+  ret = obj->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP);
 
   return ret;
 }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index 8b86b570480b..d7462587c87c 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -345,7 +345,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
                                     rgw_zone_set *zones_trace,
                                     bool *pcanceled, 
                                     const req_context& rctx,
-                                    bool log_op)
+                                    uint32_t flags)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -382,7 +382,8 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
 
   read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
 
-  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx, log_op);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
+                        flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -488,7 +489,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
                                        rgw_zone_set *zones_trace,
                                        bool *pcanceled, 
                                        const req_context& rctx,
-                                       bool log_op)
+                                       uint32_t flags)
 {
   int r = writer.drain();
   if (r < 0) {
@@ -512,7 +513,8 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx, log_op);
+  r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
+                        flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0)
     return r;
 
@@ -693,7 +695,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
                                     ceph::real_time set_mtime, rgw::sal::Attrs& attrs,
                                     ceph::real_time delete_at, const char *if_match, const char *if_nomatch,
                                     const string *user_data, rgw_zone_set *zones_trace, bool *pcanceled,
-                                    const req_context& rctx, bool log_op)
+                                    const req_context& rctx, uint32_t flags)
 {
   int r = writer.drain();
   if (r < 0)
@@ -751,7 +753,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
   }
   r = obj_op.write_meta(actual_size + cur_size,
 			accounted_size + *cur_accounted_size,
-			attrs, rctx, log_op);
+			attrs, rctx, flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index 7b92402d2d7d..53b14c41ee08 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -191,7 +191,7 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
                const req_context& rctx,
-               bool log_op = true) override;
+               uint32_t flags) override;
 
 };
 
@@ -239,7 +239,7 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                const std::string *user_data,
                rgw_zone_set *zones_trace, bool *canceled,
                const req_context& rctx,
-               bool log_op = true) override;
+               uint32_t flags) override;
 
 };
 
@@ -276,7 +276,7 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                  const char *if_match, const char *if_nomatch, const std::string *user_data,
                  rgw_zone_set *zones_trace, bool *canceled,
                  const req_context& rctx,
-                 bool log_op = true) override;
+                 uint32_t flags) override;
   };
 
 } // namespace putobj
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index a8ecb4bfa099..f167dce6e992 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4432,7 +4432,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     bool canceled = false;
     ret = processor.complete(accounted_size, etag, mtime, set_mtime,
                              attrs, delete_at, nullptr, nullptr, nullptr,
-                             zones_trace, &canceled, rctx);
+                             zones_trace, &canceled, rctx, rgw::sal::FLAG_LOG_OP);
     if (ret < 0) {
       goto set_err_state;
     }
@@ -4970,7 +4970,8 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 
   const req_context rctx{dpp, y, nullptr};
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-                            nullptr, nullptr, nullptr, nullptr, nullptr, rctx, log_op);
+                            nullptr, nullptr, nullptr, nullptr, nullptr, rctx,
+                            log_op ? rgw::sal::FLAG_LOG_OP : 0);
 }
 
 int RGWRados::transition_obj(RGWObjectCtx& obj_ctx,
@@ -5814,7 +5815,7 @@ int RGWRados::delete_obj(const DoutPrefixProvider *dpp,
   del_op.params.expiration_time = expiration_time;
   del_op.params.zones_trace = zones_trace;
 
-  return del_op.delete_obj(y, dpp, log_op);
+  return del_op.delete_obj(y, dpp, log_op ? rgw::sal::FLAG_LOG_OP : 0);
 }
 
 int RGWRados::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y)
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index e5ac7229bf6c..709e748cb499 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -811,7 +811,7 @@ int RadosBucket::abort_multiparts(const DoutPrefixProvider* dpp,
 
     if (!uploads.empty()) {
       for (const auto& upload : uploads) {
-	ret = upload->abort(dpp, cct, y, true);
+	ret = upload->abort(dpp, cct, y);
         if (ret < 0) {
 	  // we're doing a best-effort; if something cannot be found,
 	  // log it and keep moving forward
@@ -1703,9 +1703,10 @@ int RadosObject::transition(Bucket* bucket,
 			    uint64_t olh_epoch,
 			    const DoutPrefixProvider* dpp,
 			    optional_yield y,
-                            bool log_op)
+                            uint32_t flags)
 {
-  return store->getRados()->transition_obj(*rados_ctx, bucket->get_info(), get_obj(), placement_rule, mtime, olh_epoch, dpp, y, log_op);
+  return store->getRados()->transition_obj(*rados_ctx, bucket->get_info(), get_obj(), placement_rule,
+                                           mtime, olh_epoch, dpp, y, flags & FLAG_LOG_OP);
 }
 
 int RadosObject::transition_to_cloud(Bucket* bucket,
@@ -1987,7 +1988,7 @@ RadosObject::RadosDeleteOp::RadosDeleteOp(RadosObject *_source) :
 	parent_op(&op_target)
 { }
 
-int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
+int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
 {
   parent_op.params.bucket_owner = params.bucket_owner.id;
   parent_op.params.versioning_status = params.versioning_status;
@@ -2004,7 +2005,7 @@ int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, option
   parent_op.params.abortmp = params.abortmp;
   parent_op.params.parts_accounted_size = params.parts_accounted_size;
 
-  int ret = parent_op.delete_obj(y, dpp, log_op);
+  int ret = parent_op.delete_obj(y, dpp, flags & FLAG_LOG_OP);
   if (ret < 0)
     return ret;
 
@@ -2016,15 +2017,16 @@ int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, option
 
 int RadosObject::delete_object(const DoutPrefixProvider* dpp,
 			       optional_yield y,
-			       bool prevent_versioning)
+			       uint32_t flags)
 {
   RGWRados::Object del_target(store->getRados(), bucket->get_info(), *rados_ctx, get_obj());
   RGWRados::Object::Delete del_op(&del_target);
 
   del_op.params.bucket_owner = bucket->get_info().owner;
-  del_op.params.versioning_status = prevent_versioning ? 0 : bucket->get_info().versioning_status();
+  del_op.params.versioning_status = (flags & FLAG_PREVENT_VERSIONING)
+                                    ? 0 : bucket->get_info().versioning_status();
 
-  return del_op.delete_obj(y, dpp);
+  return del_op.delete_obj(y, dpp, flags & FLAG_LOG_OP);
 }
 
 int RadosObject::copy_object(User* user,
@@ -2155,7 +2157,7 @@ int RadosMultipartUpload::cleanup_part_history(const DoutPrefixProvider* dpp,
 }
 
 
-int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
+int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
 {
   std::unique_ptr<rgw::sal::Object> meta_obj = get_meta_obj();
   meta_obj->set_in_extra_data(true);
@@ -2183,7 +2185,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
 	std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(
 				    rgw_obj_key(obj_part->oid, std::string(), RGW_OBJ_NS_MULTIPART));
 	obj->set_hash_source(mp_obj.get_key());
-	ret = obj->delete_object(dpp, y, log_op);
+	ret = obj->delete_object(dpp, y, 0);
         if (ret < 0 && ret != -ENOENT)
           return ret;
       } else {
@@ -2233,7 +2235,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
   del_op->params.parts_accounted_size = parts_accounted_size;
 
   // and also remove the metadata obj
-  ret = del_op->delete_obj(dpp, y, log_op);
+  ret = del_op->delete_obj(dpp, y, 0);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " <<
       ret << dendl;
@@ -2851,10 +2853,10 @@ int RadosAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, flags);
 }
 
 int RadosAppendWriter::prepare(optional_yield y)
@@ -2875,10 +2877,10 @@ int RadosAppendWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, flags);
 }
 
 int RadosMultipartWriter::prepare(optional_yield y)
@@ -2899,10 +2901,10 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, log_op);
+			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, flags);
 }
 
 bool RadosZoneGroup::placement_target_exists(std::string& target) const
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index a05403856111..7b810504457b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -332,7 +332,7 @@ class RadosObject : public StoreObject {
     public:
       RadosDeleteOp(RadosObject* _source);
 
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
     };
 
     RadosObject(RadosStore *_st, const rgw_obj_key& _k)
@@ -364,7 +364,7 @@ class RadosObject : public StoreObject {
       rados_ctx->invalidate(get_obj());
     }
     virtual int delete_object(const DoutPrefixProvider* dpp,
-			      optional_yield y, bool prevent_versioning) override;
+			      optional_yield y, uint32_t flags) override;
     virtual int copy_object(User* user,
                req_info* info, const rgw_zone_id& source_zone,
                rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -414,7 +414,7 @@ class RadosObject : public StoreObject {
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
 			   optional_yield y,
-                           bool log_op) override;
+                           uint32_t flags) override;
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
 			   rgw_bucket_dir_entry& o,
@@ -610,7 +610,7 @@ class RadosMultipartUpload : public StoreMultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -754,7 +754,7 @@ class RadosAtomicWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class RadosAppendWriter : public StoreWriter {
@@ -802,7 +802,7 @@ class RadosAppendWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class RadosMultipartWriter : public StoreWriter {
@@ -848,7 +848,7 @@ class RadosMultipartWriter : public StoreWriter {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class RadosLuaManager : public StoreLuaManager {
diff --git a/src/rgw/rgw_data_access.cc b/src/rgw/rgw_data_access.cc
index 07bf12e37460..76cadc9c8bfc 100644
--- a/src/rgw/rgw_data_access.cc
+++ b/src/rgw/rgw_data_access.cc
@@ -212,7 +212,8 @@ int RGWDataAccess::Object::put(bufferlist& data,
 			    attrs, delete_at,
                             nullptr, nullptr,
                             puser_data,
-                            nullptr, nullptr, rctx);
+                            nullptr, nullptr,
+                            rctx, rgw::sal::FLAG_LOG_OP);
 }
 
 void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy)
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 43fe22eab1f0..c25e4ffc648b 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -588,8 +588,9 @@ static int remove_expired_obj(
     return ret;
   }
 
-  bool log_op = !remove_indeed || !zonegroup_lc_check(dpp, oc.driver->get_zone());
-  ret =  del_op->delete_obj(dpp, null_yield, log_op);
+  uint32_t flags = (!remove_indeed || !zonegroup_lc_check(dpp, oc.driver->get_zone()))
+                   ? rgw::sal::FLAG_LOG_OP : 0;
+  ret =  del_op->delete_obj(dpp, null_yield, flags);
   if (ret < 0) {
     ldpp_dout(dpp, 1) <<
       fmt::format("ERROR: {} failed, with error: {}", __func__, ret) << dendl;
@@ -885,8 +886,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
         return ret;
       }
 
-      bool log_op = !zonegroup_lc_check(wk->get_lc(), driver->get_zone());
-      ret = mpu->abort(this, cct, null_yield, log_op);
+      ret = mpu->abort(this, cct, null_yield);
       if (ret == 0) {
         int publish_ret = notify->publish_commit(
             this, obj_state->size,
@@ -1465,9 +1465,10 @@ class LCOpAction_Transition : public LCOpAction {
         return -EINVAL;
       }
 
-      bool log_op = !zonegroup_lc_check(oc.dpp, oc.driver->get_zone());
+      uint32_t flags = !zonegroup_lc_check(oc.dpp, oc.driver->get_zone())
+                       ? rgw::sal::FLAG_LOG_OP : 0;
       int r = oc.obj->transition(oc.bucket, target_placement, o.meta.mtime,
-                                 o.versioned_epoch, oc.dpp, null_yield, log_op);
+                                 o.versioned_epoch, oc.dpp, null_yield, flags);
       if (r < 0) {
         ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj " 
 			     << oc.bucket << ":" << o.key 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 12fc4a706d9c..9103db49cdd9 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4530,7 +4530,7 @@ void RGWPutObj::execute(optional_yield y)
   op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs,
                                (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
                                (user_data.empty() ? nullptr : &user_data), nullptr, nullptr,
-                               rctx, true);
+                               rctx, rgw::sal::FLAG_LOG_OP);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
 
   // send request to notification manager
@@ -4799,7 +4799,7 @@ void RGWPostObj::execute(optional_yield y)
     op_ret = processor->complete(s->obj_size, etag, nullptr, real_time(), attrs,
                                 (delete_at ? *delete_at : real_time()),
                                 nullptr, nullptr, nullptr, nullptr, nullptr,
-                                rctx, true);
+                                rctx, rgw::sal::FLAG_LOG_OP);
     if (op_ret < 0) {
       return;
     }
@@ -5357,7 +5357,7 @@ void RGWDeleteObj::execute(optional_yield y)
       del_op->params.olh_epoch = epoch;
       del_op->params.marker_version_id = version_id;
 
-      op_ret = del_op->delete_obj(this, y, true);
+      op_ret = del_op->delete_obj(this, y, rgw::sal::FLAG_LOG_OP);
       if (op_ret >= 0) {
 	delete_marker = del_op->result.delete_marker;
 	version_id = del_op->result.version_id;
@@ -6787,7 +6787,7 @@ void RGWCompleteMultipart::complete()
   // when the bucket is, as that would add an unneeded delete marker
   // moved to complete to prevent segmentation fault in publish commit
   if (meta_obj.get() != nullptr) {
-    int ret = meta_obj->delete_object(this, null_yield, true /* prevent versioning */);
+    int ret = meta_obj->delete_object(this, null_yield, rgw::sal::FLAG_PREVENT_VERSIONING | rgw::sal::FLAG_LOG_OP);
     if (ret >= 0) {
       /* serializer's exclusive lock is released */
       serializer->clear_locked();
@@ -6896,7 +6896,7 @@ void RGWAbortMultipart::execute(optional_yield y)
   }
   multipart_trace = tracing::rgw::tracer.add_span(name(), trace_ctx);
 
-  op_ret = upload->abort(this, s->cct, y, true);
+  op_ret = upload->abort(this, s->cct, y);
 }
 
 int RGWListMultipart::verify_permission(optional_yield y)
@@ -7262,7 +7262,7 @@ void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key& o, optional_
   del_op->params.bucket_owner = s->bucket_owner;
   del_op->params.marker_version_id = version_id;
 
-  op_ret = del_op->delete_obj(this, y, true);
+  op_ret = del_op->delete_obj(this, y, rgw::sal::FLAG_LOG_OP);
   if (op_ret == -ENOENT) {
     op_ret = 0;
   }
@@ -7436,7 +7436,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
     del_op->params.obj_owner = bowner;
     del_op->params.bucket_owner = bucket_owner;
 
-    ret = del_op->delete_obj(dpp, y, true);
+    ret = del_op->delete_obj(dpp, y, rgw::sal::FLAG_LOG_OP);
     if (ret < 0) {
       goto delop_fail;
     }
@@ -7949,7 +7949,7 @@ int RGWBulkUploadOp::handle_file(const std::string_view path,
   op_ret = processor->complete(size, etag, nullptr, ceph::real_time(),
                               attrs, ceph::real_time() /* delete_at */,
                               nullptr, nullptr, nullptr, nullptr, nullptr,
-                              rctx, true);
+                              rctx, rgw::sal::FLAG_LOG_OP);
   if (op_ret < 0) {
     ldpp_dout(this, 20) << "processor::complete returned op_ret=" << op_ret << dendl;
   }
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 33891d08a4c1..943905af17a4 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -191,6 +191,9 @@ enum AttrsMod {
   ATTRSMOD_MERGE   = 2
 };
 
+static constexpr uint32_t FLAG_LOG_OP = 0x0001;
+static constexpr uint32_t FLAG_PREVENT_VERSIONING = 0x0002;
+
 // a simple streaming data processing abstraction
 /**
  * @brief A simple streaming data processing abstraction
@@ -225,7 +228,7 @@ class ObjectProcessor : public DataProcessor {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) = 0;
+                       uint32_t flags) = 0;
 };
 
 /** A list of key-value attributes */
@@ -888,7 +891,7 @@ class Object {
       virtual ~DeleteOp() = default;
 
       /** Delete the object */
-      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) = 0;
+      virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) = 0;
     };
 
     Object() {}
@@ -897,7 +900,7 @@ class Object {
     /** Shortcut synchronous delete call for common deletes */
     virtual int delete_object(const DoutPrefixProvider* dpp,
 			      optional_yield y,
-			      bool prevent_versioning = false) = 0;
+			      uint32_t flags) = 0;
     /** Copy an this object to another object. */
     virtual int copy_object(User* user,
                req_info* info, const rgw_zone_id& source_zone,
@@ -966,7 +969,7 @@ class Object {
 			   uint64_t olh_epoch,
 			   const DoutPrefixProvider* dpp,
 			   optional_yield y,
-                           bool log_op) = 0;
+                           uint32_t flags) = 0;
     /** Move an object to the cloud */
     virtual int transition_to_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
@@ -1145,7 +1148,7 @@ class MultipartUpload {
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) = 0;
   /** Abort this upload */
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) = 0;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) = 0;
   /** Complete this upload, making it available as a normal object */
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
@@ -1375,7 +1378,7 @@ class Writer : public ObjectProcessor {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) = 0;
+                       uint32_t flags) = 0;
 };
 
 
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 68baf3d8bd46..2dc9fc95d105 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -619,7 +619,7 @@ namespace rgw::sal {
       uint64_t olh_epoch,
       const DoutPrefixProvider* dpp,
       optional_yield y,
-      bool log_op)
+      uint32_t flags)
   {
     DB::Object op_target(store->getDB(),
         get_bucket()->get_info(), get_obj());
@@ -700,7 +700,7 @@ namespace rgw::sal {
     parent_op(&op_target)
   { }
 
-  int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op)
+  int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
   {
     parent_op.params.bucket_owner = params.bucket_owner.id;
     parent_op.params.versioning_status = params.versioning_status;
@@ -727,7 +727,7 @@ namespace rgw::sal {
     return ret;
   }
 
-  int DBObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, bool prevent_versioning)
+  int DBObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags)
   {
     DB::Object del_target(store->getDB(), bucket->get_info(), get_obj());
     DB::Object::Delete del_op(&del_target);
@@ -786,7 +786,7 @@ namespace rgw::sal {
     return 0;
   }
 
-  int DBMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
+  int DBMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
   {
     std::unique_ptr<rgw::sal::Object> meta_obj = get_meta_obj();
     meta_obj->set_in_extra_data(true);
@@ -800,7 +800,7 @@ namespace rgw::sal {
     // Since the data objects are associated with meta obj till
     // MultipartUpload::Complete() is done, removing the metadata obj
     // should remove all the uploads so far.
-    ret = del_op->delete_obj(dpp, null_yield, log_op);
+    ret = del_op->delete_obj(dpp, null_yield, 0);
     if (ret < 0) {
       ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " <<
         ret << dendl;
@@ -1228,7 +1228,7 @@ namespace rgw::sal {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
   {
     /* XXX: same as AtomicWriter..consolidate code */
     parent_op.meta.mtime = mtime;
@@ -1383,7 +1383,7 @@ namespace rgw::sal {
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
                          const req_context& rctx,
-                         bool log_op)
+                         uint32_t flags)
   {
     parent_op.meta.mtime = mtime;
     parent_op.meta.delete_at = delete_at;
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index d39fb3f62550..3c0c7c765198 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -462,7 +462,7 @@ class DBNotification : public StoreNotification {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+    virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
     virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -524,7 +524,7 @@ class DBNotification : public StoreNotification {
         public:
           DBDeleteOp(DBObject* _source);
 
-          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+          virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
       };
 
       DBObject() = default;
@@ -543,7 +543,7 @@ class DBNotification : public StoreNotification {
 
       virtual int delete_object(const DoutPrefixProvider* dpp,
           optional_yield y,
-          bool prevent_versioning = false) override;
+          uint32_t flags) override;
       virtual int copy_object(User* user,
           req_info* info, const rgw_zone_id& source_zone,
           rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -580,7 +580,7 @@ class DBNotification : public StoreNotification {
           uint64_t olh_epoch,
           const DoutPrefixProvider* dpp,
           optional_yield y,
-          bool log_op) override;
+          uint32_t flags) override;
       virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
       virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
 
@@ -657,7 +657,7 @@ class DBNotification : public StoreNotification {
                          const std::string *user_data,
                          rgw_zone_set *zones_trace, bool *canceled,
                          const req_context& rctx,
-                         bool log_op) override;
+                         uint32_t flags) override;
   };
 
   class DBMultipartWriter : public StoreWriter {
@@ -706,7 +706,7 @@ class DBNotification : public StoreNotification {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
   };
 
   class DBStore : public StoreDriver {
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 953cf54e0408..6e0ca8ccd821 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -755,9 +755,9 @@ int FilterBucket::abort_multiparts(const DoutPrefixProvider* dpp, CephContext* c
 
 int FilterObject::delete_object(const DoutPrefixProvider* dpp,
 				optional_yield y,
-				bool prevent_versioning)
+				uint32_t flags)
 {
-  return next->delete_object(dpp, y, prevent_versioning);
+  return next->delete_object(dpp, y, flags);
 }
 
 int FilterObject::copy_object(User* user,
@@ -857,10 +857,10 @@ int FilterObject::transition(Bucket* bucket,
 			     uint64_t olh_epoch,
 			     const DoutPrefixProvider* dpp,
 			     optional_yield y,
-                             bool log_op)
+                             uint32_t flags)
 {
   return next->transition(nextBucket(bucket), placement_rule, mtime, olh_epoch,
-			  dpp, y, log_op);
+			  dpp, y, flags);
 }
 
 int FilterObject::transition_to_cloud(Bucket* bucket,
@@ -980,11 +980,11 @@ int FilterObject::FilterReadOp::iterate(const DoutPrefixProvider* dpp, int64_t o
 }
 
 int FilterObject::FilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
-					   optional_yield y, bool log_op)
+					   optional_yield y, uint32_t flags)
 {
   /* Copy params into next */
   next->params = params;
-  int ret = next->delete_obj(dpp, y, log_op);
+  int ret = next->delete_obj(dpp, y, flags);
   /* Copy result back */
   result = next->result;
   return ret;
@@ -1025,9 +1025,9 @@ int FilterMultipartUpload::list_parts(const DoutPrefixProvider *dpp, CephContext
   return 0;
 }
 
-int FilterMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y, bool log_op)
+int FilterMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, optional_yield y)
 {
-  return next->abort(dpp, cct, y, log_op);
+  return next->abort(dpp, cct, y);
 }
 
 int FilterMultipartUpload::complete(const DoutPrefixProvider *dpp,
@@ -1203,11 +1203,11 @@ int FilterWriter::complete(size_t accounted_size, const std::string& etag,
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op)
+                       uint32_t flags)
 {
   return next->complete(accounted_size, etag, mtime, set_mtime, attrs,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
-			canceled, rctx, log_op);
+			canceled, rctx, flags);
 }
 
 int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y,
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index d258cda4a654..80e236a25d94 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -521,7 +521,7 @@ class FilterObject : public Object {
     FilterDeleteOp(std::unique_ptr<DeleteOp> _next) : next(std::move(_next)) {}
     virtual ~FilterDeleteOp() = default;
 
-    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, bool log_op) override;
+    virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) override;
   };
 
   FilterObject(std::unique_ptr<Object> _next) : next(std::move(_next)) {}
@@ -535,7 +535,7 @@ class FilterObject : public Object {
 
   virtual int delete_object(const DoutPrefixProvider* dpp,
 			    optional_yield y,
-			    bool prevent_versioning = false) override;
+			    uint32_t flags) override;
   virtual int copy_object(User* user,
                req_info* info, const rgw_zone_id& source_zone,
 	       rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket,
@@ -584,7 +584,7 @@ class FilterObject : public Object {
 			 uint64_t olh_epoch,
 			 const DoutPrefixProvider* dpp,
 			 optional_yield y,
-                         bool log_op) override;
+                         uint32_t flags) override;
   virtual int transition_to_cloud(Bucket* bucket,
 				  rgw::sal::PlacementTier* tier,
 				  rgw_bucket_dir_entry& o,
@@ -697,7 +697,7 @@ class FilterMultipartUpload : public MultipartUpload {
 			 int num_parts, int marker,
 			 int* next_marker, bool* truncated, optional_yield y,
 			 bool assume_unsorted = false) override;
-  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y, bool log_op) override;
+  virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, optional_yield y) override;
   virtual int complete(const DoutPrefixProvider* dpp,
 		       optional_yield y, CephContext* cct,
 		       std::map<int, std::string>& part_etags,
@@ -842,7 +842,7 @@ class FilterWriter : public Writer {
                        const std::string *user_data,
                        rgw_zone_set *zones_trace, bool *canceled,
                        const req_context& rctx,
-                       bool log_op) override;
+                       uint32_t flags) override;
 };
 
 class FilterLuaManager : public LuaManager {

From 8660b4edc71113d5d2d51438c9fd6f388b35a214 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 11 Jan 2024 18:32:09 +1000
Subject: [PATCH 1345/2492] doc/radosgw - edit admin.rst "set user rate limit"

Edit "Set User Rate Limit" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 88a7215fb156..a178d69421fb 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -654,19 +654,27 @@ time has elapsed, "user A" will be able to send ``GET`` requests again.
 Set User Rate Limit
 -------------------
 
-Before you enable a rate limit, you must first set the rate limit parameters.
-For example:: 
+Before you can enable a rate limit, you must first set the rate limit
+parameters. The following is the general form of commands that set rate limit
+parameters: 
 
-	radosgw-admin ratelimit set --ratelimit-scope=user --uid=<uid> <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
-  [--max-write-ops=<num ops>] [--max-write-bytes=<num bytes>]>
+.. prompt:: bash
 
-For example:: 
+   radosgw-admin ratelimit set --ratelimit-scope=user --uid=<uid>
+   <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
+   [--max-write-ops=<num ops>] [--max-write-bytes=<num bytes>]>
 
-	radosgw-admin ratelimit set --ratelimit-scope=user --uid=johndoe --max-read-ops=1024 --max-write-bytes=10240
+An example of using ``radosgw-admin ratelimit set`` to set a rate limit might
+look like this: 
 
+.. prompt:: bash
 
-A 0 value for num ops and / or num bytes means that the
-specific rate limit attribute check is disabled.
+   radosgw-admin ratelimit set --ratelimit-scope=user --uid=johndoe --max-read-ops=1024 --max-write-bytes=10240
+
+
+A value of ``0`` assigned to ``--max-read-ops``, ``--max-read-bytes``,
+``--max-write-ops``, or ``--max-write-bytes`` disables the specified rate
+limit.  
 
 Get User Rate Limit
 -------------------

From f75a9da2871113a156092ff28f29b9d12f465975 Mon Sep 17 00:00:00 2001
From: Afreen Misbah <afreen23.git@gmail.com>
Date: Thu, 11 Jan 2024 13:19:39 +0530
Subject: [PATCH 1346/2492] mgr/dashboard: Fixes multisite topology page
 breadcrumb

The multi-site topology page just says object in breadcrumb.
The fix adds the missing "multi-site" breadcrumb.

Fixes https://tracker.ceph.com/issues/63635

Signed-off-by: Afreen Misbah <afreen23.git@gmail.com>
---
 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
index 8fcf3c6c6645..04755928b0a9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw.module.ts
@@ -192,6 +192,7 @@ const routes: Routes = [
   },
   {
     path: 'multisite',
+    data: { breadcrumbs: 'Multi-site' },
     children: [{ path: '', component: RgwMultisiteDetailsComponent }]
   }
 ];

From 1c561e2beac67c85c40f2adf387c42aba8175ecb Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 11 Jan 2024 23:40:25 +1000
Subject: [PATCH 1347/2492] doc/radosgw: edit admin.rst "get user rate limit"

Edit "Get User Rate Limit" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 88a7215fb156..8a4d1d1af842 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -671,18 +671,26 @@ specific rate limit attribute check is disabled.
 Get User Rate Limit
 -------------------
 
-Get the current configured rate limit parameters
-For example:: 
+The ``radosgw-admin ratelimit get`` command returns the current configured rate
+limit parameters.
 
-	radosgw-admin ratelimit get --ratelimit-scope=user --uid=<uid>
+The following is the general form of the command that returns the current
+configured limit parameters:  
 
-For example:: 
+.. prompt:: bash
 
-	radosgw-admin ratelimit get --ratelimit-scope=user --uid=johndoe
+   radosgw-admin ratelimit get --ratelimit-scope=user --uid=<uid>
 
+An example of using ``radosgw-admin ratelimit get`` to return the rate limit
+parameters might look like this: 
 
-A 0 value for num ops and / or num bytes means that the
-specific rate limit attribute check is disabled.
+.. prompt:: bash
+
+   radosgw-admin ratelimit get --ratelimit-scope=user --uid=johndoe
+
+A value of ``0`` assigned to ``--max-read-ops``, ``--max-read-bytes``,
+``--max-write-ops``, or ``--max-write-bytes`` disables the specified rate
+limit.  
 
 
 Enable/Disable User Rate Limit

From 87a596e4e26064b85e7da4d58484d52231e8c877 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 11 Jan 2024 13:59:43 +0000
Subject: [PATCH 1348/2492] osd/PeeringState: introduce
 osd_skip_check_past_interval_bounds

Fixes: https://tracker.ceph.com/issues/64002

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/common/options/global.yaml.in | 6 ++++++
 src/osd/PeeringState.cc           | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 286a1126a467..e1c06e2402f6 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -3228,6 +3228,12 @@ options:
   level: dev
   default: false
   with_legacy: true
+- name: osd_skip_check_past_interval_bounds
+  type: bool
+  level: dev
+  desc: See https://tracker.ceph.com/issues/64002
+  default: false
+  with_legacy: true
 - name: osd_debug_pretend_recovery_active
   type: bool
   level: dev
diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index c81cb958a7a9..68b427bff2e9 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -922,6 +922,10 @@ static pair<epoch_t, epoch_t> get_required_past_interval_bounds(
 
 void PeeringState::check_past_interval_bounds() const
 {
+  // See: https://tracker.ceph.com/issues/64002
+  if (cct->_conf.get_val<bool>("osd_skip_check_past_interval_bounds")) {
+    return;
+  }
   // cluster_osdmap_trim_lower_bound gives us a bound on needed
   // intervals, see doc/dev/osd_internals/past_intervals.rst
   auto oldest_epoch = pl->cluster_osdmap_trim_lower_bound();

From 209e3ed6f11af0cc1dde070e21e0867412e7d5e9 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 11 Jan 2024 10:28:01 -0500
Subject: [PATCH 1349/2492] rgw: fix use of creds in forward_iam_request()

variable `creds` was moved into the `RGWRESTConn` constructor before being
passed into `forward_iam_request()`. change `forward_iam_request()` so
it uses the member variable from the constructor instead of taking it as
an argument

Fixes: https://tracker.ceph.com/issues/63994

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_rest_conn.cc | 2 +-
 src/rgw/rgw_rest_conn.h  | 2 +-
 src/rgw/rgw_rest_role.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index a9a6bc7c56cb..235de2b7363f 100644
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -109,7 +109,7 @@ int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, con
   return req.forward_request(dpp, key, info, max_response, inbl, outbl, y);
 }
 
-int RGWRESTConn::forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
+int RGWRESTConn::forward_iam_request(const DoutPrefixProvider *dpp, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 {
   string url;
   int ret = get_url(url);
diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 4e3c88af219b..26e71bef38a9 100644
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -128,7 +128,7 @@ class RGWRESTConn
   int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 
   /* sync request */
-  int forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
+  int forward_iam_request(const DoutPrefixProvider *dpp, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 
 
   /* async requests */
diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index 2b8d3920acea..83c8fb902c16 100644
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -59,7 +59,7 @@ int forward_iam_request_to_master(const DoutPrefixProvider* dpp,
                           std::move(creds), zg->second.id, zg->second.api_name};
   bufferlist outdata;
   constexpr size_t max_response_size = 128 * 1024; // we expect a very small response
-  int ret = conn.forward_iam_request(dpp, creds, req, nullptr, max_response_size,
+  int ret = conn.forward_iam_request(dpp, req, nullptr, max_response_size,
                                      &indata, &outdata, y);
   if (ret < 0) {
     return ret;

From c32a042433887aadab49443809de66a2a33750cb Mon Sep 17 00:00:00 2001
From: Marcus Watts <mwatts@redhat.com>
Date: Mon, 22 Aug 2022 03:37:27 -0400
Subject: [PATCH 1350/2492] common: hmacsha512 for radosgw

swift tempurl (and formpost) now suport sha256 and sha512.
hmacsha256 was there, adding hmacsha512 to round out the collection.

Fixes: https://tracker.ceph.com/issues/56564
Signed-off-by: Marcus Watts <mwatts@redhat.com>
---
 src/common/ceph_crypto.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h
index 5beda7a12522..ed93d09e6e27 100644
--- a/src/common/ceph_crypto.h
+++ b/src/common/ceph_crypto.h
@@ -14,6 +14,7 @@
 #define CEPH_CRYPTO_SHA1_DIGESTSIZE 20
 #define CEPH_CRYPTO_HMACSHA256_DIGESTSIZE 32
 #define CEPH_CRYPTO_SHA256_DIGESTSIZE 32
+#define CEPH_CRYPTO_HMACSHA512_DIGESTSIZE 64
 #define CEPH_CRYPTO_SHA512_DIGESTSIZE 64
 
 #include <openssl/evp.h>
@@ -187,6 +188,12 @@ namespace TOPNSPC::crypto {
       : HMAC(EVP_sha256(), key, length) {
     }
   };
+
+  struct HMACSHA512 : public HMAC {
+    HMACSHA512 (const unsigned char *key, size_t length)
+      : HMAC(EVP_sha512(), key, length) {
+    }
+  };
 }
 
 
@@ -197,6 +204,7 @@ namespace TOPNSPC::crypto {
 
   using ssl::HMACSHA256;
   using ssl::HMACSHA1;
+  using ssl::HMACSHA512;
 
 template<class Digest>
 auto digest(const ceph::buffer::list& bl)

From 86c2c9a0aa6fc709022b4beb4fcd456c1320e03a Mon Sep 17 00:00:00 2001
From: Marcus Watts <mwatts@redhat.com>
Date: Mon, 22 Aug 2022 03:37:48 -0400
Subject: [PATCH 1351/2492] rgw: tempurl signature fixes: also support sha256
 and sha512 in formpost

In openstack swift, formpost and tempurl share the same signature
logic, so both now support sha1 / sha256 / sha512.  Here are changes
to ceph's formpost logic to match what swift does.  Most of the
guts of the signature logic are moved from .h to .cc because
this logic is only useful to the actual formpost code.

Fixes: https://tracker.ceph.com/issues/56564
Signed-off-by: Marcus Watts <mwatts@redhat.com>
---
 src/rgw/rgw_rest_swift.cc | 121 +++++++++++++++++++++++++++++++++++---
 src/rgw/rgw_rest_swift.h  |  60 +------------------
 2 files changed, 116 insertions(+), 65 deletions(-)

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 46f02805cd1b..cc75b8146524 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -42,6 +42,109 @@
 
 using namespace std;
 
+template <class HASHFLAVOR, rgw::auth::swift::SignatureFlavor SIGNATUREFLAVOR>
+class FormPostSignatureT: public rgw::auth::swift::FormatSignature<HASHFLAVOR,SIGNATUREFLAVOR>
+{
+  using UCHARPTR = const unsigned char*;
+  using base_t = rgw::auth::swift::SignatureHelperT<HASHFLAVOR>;
+  using format_signature_t = rgw::auth::swift::FormatSignature<HASHFLAVOR,SIGNATUREFLAVOR>;
+public:
+  const char* calc(const std::string& key,
+      const std::string_view& path_info,
+      const std::string_view& redirect,
+      const std::string_view& max_file_size,
+      const std::string_view& max_file_count,
+      const std::string_view& expires) {
+    HASHFLAVOR hmac((UCHARPTR) key.data(), key.size());
+
+    hmac.Update((UCHARPTR) path_info.data(), path_info.size());
+    hmac.Update((UCHARPTR) "\n", 1);
+
+    hmac.Update((UCHARPTR) redirect.data(), redirect.size());
+    hmac.Update((UCHARPTR) "\n", 1);
+
+    hmac.Update((UCHARPTR) max_file_size.data(), max_file_size.size());
+    hmac.Update((UCHARPTR) "\n", 1);
+
+    hmac.Update((UCHARPTR) max_file_count.data(), max_file_count.size());
+    hmac.Update((UCHARPTR) "\n", 1);
+
+    hmac.Update((UCHARPTR) expires.data(), expires.size());
+
+    hmac.Final(base_t::dest);
+
+    return format_signature_t::result();
+  }
+};
+class RGWFormPost::SignatureHelper {
+public:
+  virtual ~SignatureHelper() {};
+  virtual const char* calc(const std::string& key,
+    const std::string_view& path_info,
+    const std::string_view& redirect,
+    const std::string_view& max_file_size,
+    const std::string_view& max_file_count,
+    const std::string_view& expires) {
+    return nullptr;
+  };
+  virtual const char* get_signature() const {
+    return nullptr;
+  };
+  virtual bool is_equal_to(const std::string& rhs) {
+    return false;
+  };
+  static std::unique_ptr<SignatureHelper> get_sig_helper(std::string_view x);
+};
+template<typename HASHFLAVOR, rgw::auth::swift::SignatureFlavor SIGNATUREFLAVOR>
+class RGWFormPost::SignatureHelper_x : public RGWFormPost::SignatureHelper
+{
+  friend RGWFormPost;
+private:
+  FormPostSignatureT<HASHFLAVOR,SIGNATUREFLAVOR> d;
+public:
+  ~SignatureHelper_x() { };
+  SignatureHelper_x() {};
+  virtual const char* calc(const std::string& key,
+    const std::string_view& path_info,
+    const std::string_view& redirect,
+    const std::string_view& max_file_size,
+    const std::string_view& max_file_count,
+    const std::string_view& expires) {
+    return d.calc(key,path_info,redirect,
+      max_file_size,max_file_count,expires) ;
+  };
+  virtual const char* get_signature() const {
+    return d.get_signature();
+  };
+  virtual bool is_equal_to(const std::string& rhs) {
+    return d.is_equal_to(rhs);
+  };
+};
+
+std::unique_ptr<RGWFormPost::SignatureHelper> RGWFormPost::SignatureHelper::get_sig_helper(std::string_view x) {
+  size_t pos = x.find(':');
+  if (pos == x.npos || pos <= 0) {
+    switch(x.length()) {
+    case CEPH_CRYPTO_HMACSHA1_DIGESTSIZE*2:
+      return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA1,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    case CEPH_CRYPTO_HMACSHA256_DIGESTSIZE*2:
+      return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA256,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    case CEPH_CRYPTO_HMACSHA512_DIGESTSIZE*2:
+      return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA512,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    }
+    return std::make_unique<BadSignatureHelper>();
+  }
+  std::string_view type { x.substr(0,pos) };
+  if (type == "sha1") {
+    return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA1,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  } else if (type == "sha256") {
+    return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA256,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  } else if (type == "sha512") {
+    return std::make_unique<SignatureHelper_x<ceph::crypto::HMACSHA512,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  }
+  return std::make_unique<BadSignatureHelper>();
+};
+
 int RGWListBuckets_ObjStore_SWIFT::get_params(optional_yield y)
 {
   prefix = s->info.args.get("prefix");
@@ -2034,6 +2137,7 @@ bool RGWFormPost::is_non_expired()
 bool RGWFormPost::is_integral()
 {
   const std::string form_signature = get_part_str(ctrl_parts, "signature");
+  bool r = false;
 
   try {
     get_owner_info(s, s->user->get_info());
@@ -2051,28 +2155,31 @@ bool RGWFormPost::is_integral()
       continue;
     }
 
-    SignatureHelper sig_helper;
-    sig_helper.calc(temp_url_key,
+    auto sig_helper{ RGWFormPost::SignatureHelper::get_sig_helper(form_signature) };
+    sig_helper->calc(temp_url_key,
                     s->info.request_uri,
                     get_part_str(ctrl_parts, "redirect"),
                     get_part_str(ctrl_parts, "max_file_size", "0"),
                     get_part_str(ctrl_parts, "max_file_count", "0"),
                     get_part_str(ctrl_parts, "expires", "0"));
 
-    const auto local_sig = sig_helper.get_signature();
+    const char* local_sig = sig_helper->get_signature();
+    if (!local_sig) local_sig = "???";
 
     ldpp_dout(this, 20) << "FormPost signature [" << temp_url_key_num << "]"
                       << " (calculated): " << local_sig << dendl;
 
-    if (sig_helper.is_equal_to(form_signature)) {
-      return true;
-    } else {
+    r = sig_helper->is_equal_to(form_signature);
+    if (!r) {
       ldpp_dout(this, 5) << "FormPost's signature mismatch: "
                        << local_sig << " != " << form_signature << dendl;
     }
+    if (r) {
+      break;
+    }
   }
 
-  return false;
+  return r;
 }
 
 void RGWFormPost::get_owner_info(const req_state* const s,
diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h
index 08b5a77dabbe..154a069d73ff 100644
--- a/src/rgw/rgw_rest_swift.h
+++ b/src/rgw/rgw_rest_swift.h
@@ -263,6 +263,8 @@ class RGWFormPost : public RGWPostObj_ObjStore {
   bool stream_done = false;
 
   class SignatureHelper;
+  using BadSignatureHelper = SignatureHelper;
+  template<typename HASHFLAVOR, rgw::auth::swift::SignatureFlavor SIGNATUREFLAVOR> class SignatureHelper_x;
 public:
   RGWFormPost() = default;
   ~RGWFormPost() = default;
@@ -278,64 +280,6 @@ class RGWFormPost : public RGWPostObj_ObjStore {
   static bool is_formpost_req(req_state* const s);
 };
 
-class RGWFormPost::SignatureHelper
-{
-private:
-  static constexpr uint32_t output_size =
-    CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2 + 1;
-
-  unsigned char dest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE]; // 20
-  char dest_str[output_size];
-
-public:
-  SignatureHelper() = default;
-
-  const char* calc(const std::string& key,
-                   const std::string_view& path_info,
-                   const std::string_view& redirect,
-                   const std::string_view& max_file_size,
-                   const std::string_view& max_file_count,
-                   const std::string_view& expires) {
-    using ceph::crypto::HMACSHA1;
-    using UCHARPTR = const unsigned char*;
-
-    HMACSHA1 hmac((UCHARPTR) key.data(), key.size());
-
-    hmac.Update((UCHARPTR) path_info.data(), path_info.size());
-    hmac.Update((UCHARPTR) "\n", 1);
-
-    hmac.Update((UCHARPTR) redirect.data(), redirect.size());
-    hmac.Update((UCHARPTR) "\n", 1);
-
-    hmac.Update((UCHARPTR) max_file_size.data(), max_file_size.size());
-    hmac.Update((UCHARPTR) "\n", 1);
-
-    hmac.Update((UCHARPTR) max_file_count.data(), max_file_count.size());
-    hmac.Update((UCHARPTR) "\n", 1);
-
-    hmac.Update((UCHARPTR) expires.data(), expires.size());
-
-    hmac.Final(dest);
-
-    buf_to_hex((UCHARPTR) dest, sizeof(dest), dest_str);
-
-    return dest_str;
-  }
-
-  const char* get_signature() const {
-    return dest_str;
-  }
-
-  bool is_equal_to(const std::string& rhs) const {
-    /* never allow out-of-range exception */
-    if (rhs.size() < (output_size - 1)) {
-      return false;
-    }
-    return rhs.compare(0 /* pos */,  output_size, dest_str) == 0;
-  }
-
-}; /* RGWFormPost::SignatureHelper */
-
 
 class RGWSwiftWebsiteHandler {
   rgw::sal::Driver* const driver;

From c15e56e386251403a876454f6a4aa186284565e1 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Wed, 27 Dec 2023 13:16:11 +0800
Subject: [PATCH 1352/2492] crimson/os/alienstore: submit from the alien world
 concurrently

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/alienstore/alien_log.cc   | 1 +
 src/crimson/os/alienstore/alien_store.cc | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/alienstore/alien_log.cc b/src/crimson/os/alienstore/alien_log.cc
index b371af897a2f..a0f5b03a5d39 100644
--- a/src/crimson/os/alienstore/alien_log.cc
+++ b/src/crimson/os/alienstore/alien_log.cc
@@ -17,6 +17,7 @@ CnLog::~CnLog() {
 }
 
 void CnLog::_flush(EntryVector& q, bool crash) {
+  // XXX: the waiting here will block the thread for an indeterministic peroid
   seastar::alien::submit_to(inst, shard, [&q] {
     for (auto& it : q) {
       crimson::get_logger(it.m_subsys).log(
diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 593716df3519..d53dacb18246 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -57,10 +57,11 @@ class OnCommit final: public Context
   }
 
   void finish(int) final {
-    return seastar::alien::submit_to(alien, cpuid, [this] {
-      alien_done.set_value();
+    std::ignore = seastar::alien::submit_to(alien, cpuid,
+        [&_alien_done=this->alien_done] {
+      _alien_done.set_value();
       return seastar::make_ready_future<>();
-    }).wait();
+    });
   }
 };
 }

From c3d7f70b7a7d594050e3e231cd7d6544179fdfeb Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 10 Jan 2024 12:12:23 +0530
Subject: [PATCH 1353/2492] mgr/dashboard: delete cephfs snapshot

Fixes: https://tracker.ceph.com/issues/63990
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       | 13 +++++
 ...subvolume-snapshots-list.component.spec.ts |  3 +-
 ...phfs-subvolume-snapshots-list.component.ts | 46 ++++++++++++++++-
 .../shared/api/cephfs-subvolume.service.ts    | 10 ++++
 .../shared/services/task-message.service.ts   |  4 ++
 src/pybind/mgr/dashboard/openapi.yaml         | 51 +++++++++++++++++++
 6 files changed, 125 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 712efe11b0dd..d747ba1e0f02 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -891,6 +891,19 @@ def create(self, vol_name: str, subvol_name: str, snap_name: str, group_name='')
             )
         return f'Subvolume snapshot {snap_name} created successfully'
 
+    def delete(self, vol_name: str, subvol_name: str, snap_name: str, group_name='', force=True):
+        params = {'vol_name': vol_name, 'sub_name': subvol_name, 'snap_name': snap_name}
+        if group_name:
+            params['group_name'] = group_name
+        params['force'] = str_to_bool(force)
+        error_code, _, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_rm', None,
+                                        params)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to delete subvolume snapshot {snap_name}: {err}'
+            )
+        return f'Subvolume snapshot {snap_name} removed successfully'
+
 
 @APIRouter('/cephfs/snaphost/schedule', Scope.CEPHFS)
 @APIDoc("Cephfs Snapshot Scheduling API", "CephFSSnapshotSchedule")
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
index 1d03cf2a8bca..c69f916c2c16 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.spec.ts
@@ -3,6 +3,7 @@ import { ComponentFixture, TestBed } from '@angular/core/testing';
 import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapshots-list.component';
 import { HttpClientTestingModule } from '@angular/common/http/testing';
 import { SharedModule } from '~/app/shared/shared.module';
+import { ToastrModule } from 'ngx-toastr';
 
 describe('CephfsSubvolumeSnapshotsListComponent', () => {
   let component: CephfsSubvolumeSnapshotsListComponent;
@@ -11,7 +12,7 @@ describe('CephfsSubvolumeSnapshotsListComponent', () => {
   beforeEach(async () => {
     await TestBed.configureTestingModule({
       declarations: [CephfsSubvolumeSnapshotsListComponent],
-      imports: [HttpClientTestingModule, SharedModule]
+      imports: [HttpClientTestingModule, SharedModule, ToastrModule.forRoot()]
     }).compileComponents();
 
     fixture = TestBed.createComponent(CephfsSubvolumeSnapshotsListComponent);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
index 9970d5988879..798307a0cf9f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
@@ -16,6 +16,10 @@ import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
 import { Permissions } from '~/app/shared/models/permissions';
 import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
 import { CdDatePipe } from '~/app/shared/pipes/cd-date.pipe';
+import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
+import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
+import { FinishedTask } from '~/app/shared/models/finished-task';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 
 @Component({
   selector: 'cd-cephfs-subvolume-snapshots-list',
@@ -30,6 +34,7 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
   tableActions: CdTableAction[];
   selection = new CdTableSelection();
   permissions: Permissions;
+  modalRef: NgbModalRef;
 
   subVolumes$: Observable<CephfsSubvolume[]>;
   snapshots$: Observable<any[]>;
@@ -53,7 +58,8 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
     private actionLabels: ActionLabelsI18n,
     private modalService: ModalService,
     private authStorageService: AuthStorageService,
-    private cdDatePipe: CdDatePipe
+    private cdDatePipe: CdDatePipe,
+    private taskWrapper: TaskWrapperService
   ) {
     this.permissions = this.authStorageService.getPermissions();
   }
@@ -91,6 +97,12 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
         permission: 'create',
         icon: Icons.add,
         click: () => this.openModal()
+      },
+      {
+        name: this.actionLabels.REMOVE,
+        permission: 'delete',
+        icon: Icons.destroy,
+        click: () => this.deleteSnapshot()
       }
     ];
 
@@ -190,4 +202,36 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
   updateSelection(selection: CdTableSelection) {
     this.selection = selection;
   }
+
+  deleteSnapshot() {
+    const snapshotName = this.selection.first().name;
+    const subVolumeName = this.activeSubVolumeName;
+    const subVolumeGroupName = this.activeGroupName;
+    const fsName = this.fsName;
+    this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      actionDescription: 'Remove',
+      itemNames: [snapshotName],
+      itemDescription: 'Snapshot',
+      submitAction: () =>
+        this.taskWrapper
+          .wrapTaskAroundCall({
+            task: new FinishedTask('cephfs/subvolume/snapshot/delete', {
+              fsName: fsName,
+              subVolumeName: subVolumeName,
+              subVolumeGroupName: subVolumeGroupName,
+              snapshotName: snapshotName
+            }),
+            call: this.cephfsSubvolumeService.deleteSnapshot(
+              fsName,
+              subVolumeName,
+              snapshotName,
+              subVolumeGroupName
+            )
+          })
+          .subscribe({
+            complete: () => this.modalRef.close(),
+            error: () => this.modalRef.componentInstance.stopLoadingSpinner()
+          })
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
index 1995fd293bae..ad0ce248064d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
@@ -161,4 +161,14 @@ export class CephfsSubvolumeService {
       { observe: 'response' }
     );
   }
+
+  deleteSnapshot(fsName: string, subVolumeName: string, snapshotName: string, groupName = '') {
+    return this.http.delete(`${this.baseURL}/snapshot/${fsName}/${subVolumeName}`, {
+      params: {
+        snap_name: snapshotName,
+        group_name: groupName
+      },
+      observe: 'response'
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index c1165d318a36..dba742fbf783 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -383,6 +383,10 @@ export class TaskMessageService {
     'cephfs/subvolume/snapshot/create': this.newTaskMessage(
       this.commonOperations.create,
       (metadata) => this.snapshot(metadata)
+    ),
+    'cephfs/subvolume/snapshot/delete': this.newTaskMessage(
+      this.commonOperations.delete,
+      (metadata) => this.snapshot(metadata)
     )
   };
 
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 6129321c7df2..fb2b7059a53a 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -2066,6 +2066,57 @@ paths:
       tags:
       - CephfsSubvolumeSnapshot
   /api/cephfs/subvolume/snapshot/{vol_name}/{subvol_name}:
+    delete:
+      parameters:
+      - in: path
+        name: vol_name
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: subvol_name
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: snap_name
+        required: true
+        schema:
+          type: string
+      - default: ''
+        in: query
+        name: group_name
+        schema:
+          type: string
+      - default: true
+        in: query
+        name: force
+        schema:
+          type: boolean
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephfsSubvolumeSnapshot
     get:
       parameters:
       - in: path

From aedde7fa2ca20e5ee6f749eaff1fbb00f1875728 Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Wed, 10 Jan 2024 16:18:57 +0800
Subject: [PATCH 1354/2492] common: fix ambiguous error when using gcc 13

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/common/scrub_types.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/scrub_types.h b/src/common/scrub_types.h
index 972d0993193f..dd206f56f603 100644
--- a/src/common/scrub_types.h
+++ b/src/common/scrub_types.h
@@ -220,7 +220,7 @@ struct fmt::formatter<librados::object_id_t> {
   template <typename FormatContext>
   auto format(const auto &oid, FormatContext& ctx) const
   {
-    return format_to(ctx.out(), "{}/{}/{}", oid.locator, oid.nspace, oid.name);
+    return fmt::format_to(ctx.out(), "{}/{}/{}", oid.locator, oid.nspace, oid.name);
   }
 };
 

From b65046e37f62f2c65b17ba6f3434a19d3d68c983 Mon Sep 17 00:00:00 2001
From: Cory Snyder <csnyder@1111systems.com>
Date: Fri, 12 Jan 2024 14:41:31 +0000
Subject: [PATCH 1355/2492] rgw: add reproducer for bug with concurrent
 versioned object deletes

Adds a test case to reproduce a scenario where concurrent versioned
object deletes can cause leftover OLH entries to be left behind.

Signed-off-by: Cory Snyder <csnyder@1111systems.com>
---
 qa/workunits/rgw/test_rgw_versioning.py | 29 ++++++++++++++++++++++++-
 src/common/options/rgw.yaml.in          | 10 +++++++++
 src/rgw/driver/rados/rgw_rados.cc       |  6 +++++
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/qa/workunits/rgw/test_rgw_versioning.py b/qa/workunits/rgw/test_rgw_versioning.py
index fc69e138d41f..f175203ea0bf 100755
--- a/qa/workunits/rgw/test_rgw_versioning.py
+++ b/qa/workunits/rgw/test_rgw_versioning.py
@@ -5,6 +5,7 @@
 import uuid
 import botocore
 import time
+import threading
 from common import exec_cmd, create_user, boto_connect
 from botocore.config import Config
 
@@ -100,7 +101,33 @@ def main():
         exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err')
     get_resp = bucket.Object(key).get()
     assert put_resp.e_tag == get_resp['ETag'], 'get did not return null version with correct etag'
-        
+
+    # TESTCASE 'verify that concurrent delete requests do not leave behind olh entries'
+    log.debug('TEST: verify that concurrent delete requests do not leave behind olh entries\n')
+    bucket.object_versions.all().delete()
+    
+    key = 'concurrent-delete'
+    # create a delete marker
+    resp = bucket.Object(key).delete()
+    version_id = resp['ResponseMetadata']['HTTPHeaders']['x-amz-version-id']
+    try:
+        exec_cmd('ceph config set client rgw_debug_inject_latency_bi_unlink 2')
+        time.sleep(1)
+
+        def do_delete():
+            connection.ObjectVersion(bucket.name, key, version_id).delete()
+            
+        t2 = threading.Thread(target=do_delete)
+        t2.start()
+        do_delete()
+        t2.join()
+    finally:
+        exec_cmd('ceph config rm client rgw_debug_inject_latency_bi_unlink')
+    out = exec_cmd(f'radosgw-admin bucket check olh --bucket {bucket.name} --dump-keys')
+    num_leftover_olh_entries = len(json.loads(out))
+    assert num_leftover_olh_entries == 0, \
+      'Found leftover olh entries after concurrent deletes'
+
     # Clean up
     log.debug("Deleting bucket {}".format(BUCKET_NAME))
     bucket.object_versions.all().delete()
diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index a37079775225..b0f442366f4e 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -2642,6 +2642,16 @@ options:
   - rgw
   - rgw
   min: 30
+- name: rgw_debug_inject_latency_bi_unlink
+  type: uint
+  level: dev
+  desc: Latency (in seconds) injected before rgw bucket index unlink op calls to simulate
+    queueing latency and validate behavior of simultaneuous delete requests which
+    target the same object.
+  default: 0
+  with_legacy: true
+  services:
+  - rgw
 - name: rgw_debug_inject_set_olh_err
   type: uint
   level: dev
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 93e8840eafdb..7d4a765cbba6 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -8319,6 +8319,12 @@ int RGWRados::unlink_obj_instance(const DoutPrefixProvider *dpp, RGWObjectCtx& o
     }
 
     string olh_tag(state->olh_tag.c_str(), state->olh_tag.length());
+    
+    if (cct->_conf->rgw_debug_inject_latency_bi_unlink) {
+      // simulates queue latency for unlink ops to validate behavior with
+      // concurrent delete requests for the same object version instance
+      std::this_thread::sleep_for(cct->_conf->rgw_debug_inject_latency_bi_unlink * std::chrono::seconds{1});
+    }
 
     ret = bucket_index_unlink_instance(dpp, bucket_info, target_obj, op_tag, olh_tag, olh_epoch, y, zones_trace);
     if (ret < 0) {

From 66ac828f97c1b368e41c51be4e5699f3e92d47f9 Mon Sep 17 00:00:00 2001
From: Cory Snyder <csnyder@1111systems.com>
Date: Fri, 12 Jan 2024 14:51:14 +0000
Subject: [PATCH 1356/2492] rgw: fix issue with concurrent versioned deletes
 leaving behind olh entries

Fixes a scenario where BI OLH entries were left behind after racing
delete requests for the same versioned object instance.

Fixes: https://tracker.ceph.com/issues/64014
Signed-off-by: Cory Snyder <csnyder@1111systems.com>
---
 src/cls/rgw/cls_rgw.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 034660a11ed1..b957f0aaaa69 100644
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -1906,11 +1906,10 @@ static int rgw_bucket_unlink_instance(cls_method_context_t hctx, bufferlist *in,
   BIOLHEntry olh(hctx, dest_key);
 
   int ret = obj.init();
-  if (ret == -ENOENT) {
-    return 0; /* already removed */
-  }
   if (ret < 0) {
-    CLS_LOG(0, "ERROR: obj.init() returned ret=%d", ret);
+    if (ret != -ENOENT) {
+      CLS_LOG(0, "ERROR: obj.init() returned ret=%d", ret);
+    }
     return ret;
   }
 

From d5a08ba98756d2f4eeef438d89fa1ea959e62fc3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 12 Jan 2024 11:05:48 -0500
Subject: [PATCH 1357/2492] test/rgw: metadata sync checkpoint waits on global
 'sync' status

in testing, i was seeing meta sync checkpoints finish even though sync
hadn't started yet:
```
rgw_multi.tests: DEBUG: current meta sync status={
    "sync_status": {
        "info": {
            "status": "building-full-sync-maps",
```

wait for the global status to reach "sync" before starting to compare
period epochs or sync markers

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/rgw/rgw_multi/tests.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index c720423e9239..3f12d25f75a2 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -131,7 +131,7 @@ def parse_meta_sync_status(meta_sync_status_json):
         else:
             markers[i] = sync_markers[i]['val']['marker']
 
-    return period, realm_epoch, num_shards, markers
+    return global_sync_status, period, realm_epoch, num_shards, markers
 
 def meta_sync_status(zone):
     for _ in range(config.checkpoint_retries):
@@ -182,8 +182,10 @@ def zone_meta_checkpoint(zone, meta_master_zone = None, master_status = None):
     log.info('starting meta checkpoint for zone=%s', zone.name)
 
     for _ in range(config.checkpoint_retries):
-        period, realm_epoch, num_shards, sync_status = meta_sync_status(zone)
-        if realm_epoch < current_realm_epoch:
+        global_status, period, realm_epoch, num_shards, sync_status = meta_sync_status(zone)
+        if global_status != 'sync':
+            log.warning('zone %s has not started sync yet, state=%s', zone.name, global_status)
+        elif realm_epoch < current_realm_epoch:
             log.warning('zone %s is syncing realm epoch=%d, behind current realm epoch=%d',
                         zone.name, realm_epoch, current_realm_epoch)
         else:

From 9a93e9375f236379faaf0a7103ad4b74c9a72de4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 12 Jan 2024 11:08:38 -0500
Subject: [PATCH 1358/2492] rgw: forward_iam_request_to_master() trims trailing
 null from response

fix a regression from commit d3ad0efaee6a19c84b2af9b3dfa1b6c902674774
which changed how we parse the response bufferlist:

-  std::string r = response.c_str();
+  std::string r = response.to_str();

when the response contains a trailing null character, this now ends up in
`r` and breaks json parsing in `parser.parse(r.c_str(), r.length(), 1)`

replace `response.to_str()` with `rgw_bl_str(response)` which trims
trailing nulls

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_rest_role.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index 83c8fb902c16..fd537c0c9946 100644
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -65,7 +65,7 @@ int forward_iam_request_to_master(const DoutPrefixProvider* dpp,
     return ret;
   }
 
-  std::string r = outdata.to_str();
+  std::string r = rgw_bl_str(outdata);
   boost::replace_all(r, "&quot;", "\"");
 
   if (!parser.parse(r.c_str(), r.length(), 1)) {

From 26b361d1edbdf070724f0e0c5ba30accaf57d475 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 12 Jan 2024 11:19:11 -0500
Subject: [PATCH 1359/2492] rgw/common: rgw_bl_str(bl) avoids bl.c_str()

where bl contains multiple buffer segments, c_str() has to
rellocate and copy those segments into a single buffer. use c_str()
instead, which just copies each segment into the resulting string

this allows the function to take the bufferlist argument by const ref

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_common.h | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 2cf3d77f9e48..ef9aa5420558 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1824,13 +1824,14 @@ static inline ssize_t rgw_unescape_str(const std::string& s, ssize_t ofs,
   return std::string::npos;
 }
 
-static inline std::string rgw_bl_str(ceph::buffer::list& raw)
+/// Return a string copy of the given bufferlist with trailing nulls removed
+static inline std::string rgw_bl_str(const ceph::buffer::list& bl)
 {
-  size_t len = raw.length();
-  std::string s(raw.c_str(), len);
-  while (len && !s[len - 1]) {
-    --len;
-    s.resize(len);
+  // use to_str() instead of c_str() so we don't reallocate a flat bufferlist
+  std::string s = bl.to_str();
+  // with to_str(), the result may include null characters. trim trailing nulls
+  while (!s.empty() && s.back() == '\0') {
+    s.pop_back();
   }
   return s;
 }

From c636e3ed2d459a063ae6ab0e2dc876db5c58c7e6 Mon Sep 17 00:00:00 2001
From: tobydarling <anothercoffee@gmail.com>
Date: Fri, 12 Jan 2024 14:10:13 +0000
Subject: [PATCH 1360/2492] doc/rados/operations: Fix off-by-one errors in
 control.rst

Description of examples was off by one

Signed-off-by: tobydarling <anothercoffee@gmail.com>
---
 doc/rados/operations/control.rst | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/doc/rados/operations/control.rst b/doc/rados/operations/control.rst
index 033f831cd8fc..32d043f1f31a 100644
--- a/doc/rados/operations/control.rst
+++ b/doc/rados/operations/control.rst
@@ -474,27 +474,25 @@ following command:
 
    ceph tell mds.{mds-id} config set {setting} {value}
 
-Example:
+Example: to enable debug messages, run the following command:
 
 .. prompt:: bash $
 
    ceph tell mds.0 config set debug_ms 1
 
-To enable debug messages, run the following command:
+To display the status of all metadata servers, run the following command:
 
 .. prompt:: bash $
 
    ceph mds stat
 
-To display the status of all metadata servers, run the following command:
+To mark the active metadata server as failed (and to trigger failover to a
+standby if a standby is present), run the following command:
 
 .. prompt:: bash $
 
    ceph mds fail 0
 
-To mark the active metadata server as failed (and to trigger failover to a
-standby if a standby is present), run the following command:
-
 .. todo:: ``ceph mds`` subcommands missing docs: set, dump, getmap, stop, setmap
 
 

From 1cd996b6fe2e3c923001a69772c7b7635d058f2b Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 14 Jan 2024 04:06:12 +1000
Subject: [PATCH 1361/2492] docs/radosgw: edit admin.rst "enable/disable user
 rate limit"

Edit "Enable/Disable User Rate Limit" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 57d38c97ab2f..4fb8ffe749ab 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -701,16 +701,21 @@ A value of ``0`` assigned to ``--max-read-ops``, ``--max-read-bytes``,
 limit.  
 
 
-Enable/Disable User Rate Limit
-------------------------------
+Enable and Disable User Rate Limit
+----------------------------------
 
-Once you set a user rate limit, you may enable it. For example:: 
+After you have set a user rate limit, you must enable it in order for it to
+take effect. Run a command of the following form to enable a user rate limit: 
 
-	radosgw-admin ratelimit enable --ratelimit-scope=user --uid=<uid>
+.. prompt:: bash
+
+   radosgw-admin ratelimit enable --ratelimit-scope=user --uid=<uid>
 
-You may disable an enabled user rate limit. For example:: 
+To disable an enabled user rate limit, run a command of the following form: 
+
+.. prompt:: bash
 
-	radosgw-admin ratelimit disable --ratelimit-scope=user --uid=johndoe
+   radosgw-admin ratelimit disable --ratelimit-scope=user --uid=johndoe
 
 
 Set Bucket Rate Limit

From e9b3ef4cb11760392ff7f766d652289636fcf5eb Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 14 Jan 2024 20:23:25 +1000
Subject: [PATCH 1362/2492] doc/radosgw: edit admin.rst "set bucket rate limit"

Edit "Set Bucket Rate Limit" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 57d38c97ab2f..a8dc8a3a59b4 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -717,18 +717,24 @@ Set Bucket Rate Limit
 ---------------------
 
 Before you enable a rate limit, you must first set the rate limit parameters.
-For example:: 
+The following is the general form of commands that set rate limit parameters:
+
+.. prompt:: bash
 
-	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=<bucket> <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
+   radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=<bucket> <[--max-read-ops=<num ops>] [--max-read-bytes=<num bytes>]
   [--max-write-ops=<num ops>] [--max-write-bytes=<num bytes>]>
 
-For example:: 
+An example of using ``radosgw-admin ratelimit set`` to set a rate limit for a
+bucket might look like this: 
 
-	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=mybucket --max-read-ops=1024 --max-write-bytes=10240
+.. prompt:: bash
 
+   radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=mybucket --max-read-ops=1024 --max-write-bytes=10240
 
-A 0 value for num ops and / or num bytes means that the
-specific rate limit attribute check is disabled.
+
+A value of ``0`` assigned to ``--max-read-ops``, ``--max-read-bytes``,
+``--max-write-ops``, or ``-max-write-bytes`` disables the specified bucket rate
+limit. 
 
 Get Bucket Rate Limit
 ---------------------

From f5e6b607b97b4fb4b3d19429d49a75b9ef24d303 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 14 Jan 2024 20:53:38 +0200
Subject: [PATCH 1363/2492] test/scrub: modify selection of max-scrubs
 configuration values

As the osd-max-scrubs default was increased from 1 to (currently) 3, the
original set of optional values under rados/thrash/3-scrub-overrides are
no longer useful.  This commits changes the set of optional values to
reflect the current default.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 qa/suites/rados/thrash/3-scrub-overrides/default.yaml        | 0
 ...ltaneous-scrubs-2.yaml => max-simultaneous-scrubs-1.yaml} | 2 +-
 .../thrash/3-scrub-overrides/max-simultaneous-scrubs-5.yaml  | 5 +++++
 3 files changed, 6 insertions(+), 1 deletion(-)
 delete mode 100644 qa/suites/rados/thrash/3-scrub-overrides/default.yaml
 rename qa/suites/rados/thrash/3-scrub-overrides/{max-simultaneous-scrubs-2.yaml => max-simultaneous-scrubs-1.yaml} (60%)
 create mode 100644 qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-5.yaml

diff --git a/qa/suites/rados/thrash/3-scrub-overrides/default.yaml b/qa/suites/rados/thrash/3-scrub-overrides/default.yaml
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml b/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-1.yaml
similarity index 60%
rename from qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml
rename to qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-1.yaml
index abf852e98510..9a63786f80d7 100644
--- a/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml
+++ b/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-1.yaml
@@ -2,4 +2,4 @@ overrides:
   ceph:
     conf:
       osd:
-        osd max scrubs: 2
+        osd max scrubs: 1
diff --git a/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-5.yaml b/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-5.yaml
new file mode 100644
index 000000000000..bb36e9970ca7
--- /dev/null
+++ b/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-5.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd max scrubs: 5

From 65ec74434d6297c50311bdda313732f0460da403 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Tue, 5 Dec 2023 11:28:25 +0530
Subject: [PATCH 1364/2492] mgr/dashboard: increase the number of plottable
 graphs in charts

Fixes: https://tracker.ceph.com/issues/64024

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../dashboard-area-chart.component.html       |  27 ++--
 .../dashboard-area-chart.component.scss       |   9 +-
 .../dashboard-area-chart.component.spec.ts    |  46 +++---
 .../dashboard-area-chart.component.ts         | 148 ++++++++++--------
 .../dashboard/dashboard-v3.component.html     |  90 +++++------
 .../dashboard/dashboard-v3.component.ts       |  20 +--
 .../rgw-overview-dashboard.component.html     |  16 +-
 .../rgw-overview-dashboard.component.ts       |  10 +-
 .../src/app/shared/api/prometheus.service.ts  |   2 +
 .../styles/defaults/_bootstrap-defaults.scss  |  18 ++-
 10 files changed, 196 insertions(+), 190 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
index cebfcf903781..6151843e4e01 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
@@ -3,29 +3,22 @@
     <br>
     <b class="chartTitle pb-2"
        i18n>{{ chartTitle }}</b>
-    <div
+    <div *ngFor="let data of currentChartData.dataset"
          i18n>
-      <div class="d-inline-flex align-items-center gap-1">
-        <div *ngIf="!maxValue"
-             class="blue-box">
-      </div>
-        <div *ngIf="label2">{{ label }}:
+      <div *ngIf="data.data.length !== 0"
+           class="d-inline-flex align-items-center gap-1">
+        <div class="box"
+             [style.background-color]="data.pointBackgroundColor">
         </div>
-        {{ currentData || 'N/A' }} {{ currentDataUnits }}
-        <div *ngIf="maxValue && currentData"> used of
-          {{ maxConvertedValue }} {{ maxConvertedValueUnits }}
+        <div *ngIf="!chartTitle.includes(data.label)">{{ data.label }}:</div>
+          {{ data?.currentData || 'N/A' }} {{ data?.currentDataUnits }}
+        <div *ngIf="maxValue && data.currentData">
+          used of {{ maxConvertedValue }} {{ maxConvertedValueUnits }}
         </div>
       </div>
     </div>
-    <div *ngIf="label2"
-         i18n>
-      <div class="d-inline-flex align-items-center gap-1">
-        <div class="yellow-box"></div>
-        <div *ngIf="label2 !== chartTitle" >{{ label2 }}: </div>
-        <div>{{ currentData2 || 'N/A' }} {{ currentDataUnits2 }}</div>
-      </div>
-    </div>
   </div>
+
   <div class="col-9 d-flex flex-column">
     <div class="chart mt-3">
       <canvas baseChart
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.scss
index 02310e37e1b3..6b802653230f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.scss
@@ -4,16 +4,9 @@
   height: 9vh;
 }
 
-.blue-box {
+.box {
   background-color: vv.$chart-color-strong-blue;
   border: 2px double vv.$chart-color-light-gray;
   height: 13px;
   width: 13px;
 }
-
-.yellow-box {
-  background-color: vv.$chart-color-orange;
-  border: 2px double vv.$chart-color-light-gray;
-  height: 13px;
-  width: 13px;
-}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
index ec71070544fa..80cdb1ea2369 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
@@ -28,10 +28,21 @@ describe('DashboardAreaChartComponent', () => {
   beforeEach(() => {
     fixture = TestBed.createComponent(DashboardAreaChartComponent);
     component = fixture.componentInstance;
-    component.data = [
-      [1, '110'],
-      [3, '130']
+    component.dataArray = [
+      [
+        [1, '110'],
+        [3, '130']
+      ],
+      [
+        [2, '120'],
+        [4, '140']
+      ],
+      [
+        [5, '150'],
+        [6, '160']
+      ]
     ];
+    component.labelsArray = ['Read', 'Write', 'Total'];
   });
 
   it('should create', () => {
@@ -43,25 +54,22 @@ describe('DashboardAreaChartComponent', () => {
     expect(chartElement).toBeTruthy();
   });
 
-  it('should have two datasets', () => {
-    component.data2 = [
-      [2, '120'],
-      [4, '140']
-    ];
+  it('should have three datasets', () => {
+    component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
     expect(component.chartData.dataset[0].data).toBeDefined();
     expect(component.chartData.dataset[1].data).toBeDefined();
+    expect(component.chartData.dataset[2].data).toBeDefined();
   });
 
   it('should set label', () => {
-    component.label = 'Write';
-    expect(component.label).toBe('Write');
+    component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
+    expect(component.chartData.dataset[0].label).toEqual('Read');
+    expect(component.chartData.dataset[1].label).toEqual('Write');
+    expect(component.chartData.dataset[2].label).toEqual('Total');
   });
 
   it('should transform and update data', () => {
-    expect(component.chartData.dataset[0].data).toEqual([{ x: 0, y: 0 }]);
-
-    component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
-
+    component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
     expect(component.chartData.dataset[0].data).toEqual([
       { x: 1000, y: 110 },
       { x: 3000, y: 130 }
@@ -69,8 +77,8 @@ describe('DashboardAreaChartComponent', () => {
   });
 
   it('should set currentData to last value', () => {
-    component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
-    expect(component.currentData).toBe('130');
+    component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
+    expect(component.currentChartData.dataset[0].currentData).toBe('130');
   });
 
   it('should keep data units consistency', () => {
@@ -78,12 +86,8 @@ describe('DashboardAreaChartComponent', () => {
     setTimeout(() => {
       fixture.detectChanges();
 
-      component.data = [
-        [1, '1100'],
-        [3, '1300']
-      ];
       component.dataUnits = 'B';
-      component.ngOnChanges({ data: new SimpleChange(null, component.data, false) });
+      component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
 
       expect(component.currentDataUnits).toBe('KiB');
       expect(component.chartDataUnits).toBe('KiB');
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index cbf97691d27d..4a0d95f5421d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -1,4 +1,4 @@
-import { Component, Input, OnChanges, SimpleChanges, ViewChild } from '@angular/core';
+import { Component, Input, ViewChild, OnChanges, SimpleChanges } from '@angular/core';
 
 import { CssHelper } from '~/app/shared/classes/css-helper';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
@@ -24,26 +24,48 @@ export class DashboardAreaChartComponent implements OnChanges {
   @Input()
   dataUnits: string;
   @Input()
-  data: Array<[number, string]>;
+  dataArray?: Array<Array<[number, string]>>; // Array of query results
   @Input()
-  data2?: Array<[number, string]>;
-  @Input()
-  label: string;
-  @Input()
-  label2?: string;
+  labelsArray?: string[] = []; // Array of chart labels
   @Input()
   decimals?: number = 1;
 
   currentDataUnits: string;
   currentData: number;
-  currentDataUnits2?: string;
-  currentData2?: number;
   maxConvertedValue?: number;
   maxConvertedValueUnits?: string;
 
   chartDataUnits: string;
-  chartData: any;
-  options: any;
+  chartData: any = { dataset: [] };
+  options: any = {};
+  currentChartData: any = {};
+
+  chartColors: any[] = [
+    [
+      this.cssHelper.propertyValue('chart-color-strong-blue'),
+      this.cssHelper.propertyValue('chart-color-translucent-blue')
+    ],
+    [
+      this.cssHelper.propertyValue('chart-color-orange'),
+      this.cssHelper.propertyValue('chart-color-translucent-orange')
+    ],
+    [
+      this.cssHelper.propertyValue('chart-color-green'),
+      this.cssHelper.propertyValue('chart-color-translucent-green')
+    ],
+    [
+      this.cssHelper.propertyValue('chart-color-cyan'),
+      this.cssHelper.propertyValue('chart-color-translucent-cyan')
+    ],
+    [
+      this.cssHelper.propertyValue('chart-color-purple'),
+      this.cssHelper.propertyValue('chart-color-translucent-purple')
+    ],
+    [
+      this.cssHelper.propertyValue('chart-color-red'),
+      this.cssHelper.propertyValue('chart-color-translucent-red')
+    ]
+  ];
 
   public chartAreaBorderPlugin: any[] = [
     {
@@ -74,35 +96,6 @@ export class DashboardAreaChartComponent implements OnChanges {
     private formatter: FormatterService,
     private numberFormatter: NumberFormatterService
   ) {
-    this.chartData = {
-      dataset: [
-        {
-          label: '',
-          data: [{ x: 0, y: 0 }],
-          tension: 0.2,
-          pointBackgroundColor: this.cssHelper.propertyValue('chart-color-strong-blue'),
-          backgroundColor: this.cssHelper.propertyValue('chart-color-translucent-blue'),
-          borderColor: this.cssHelper.propertyValue('chart-color-strong-blue'),
-          borderWidth: 1,
-          fill: {
-            target: 'origin'
-          }
-        },
-        {
-          label: '',
-          data: [],
-          tension: 0.2,
-          pointBackgroundColor: this.cssHelper.propertyValue('chart-color-orange'),
-          backgroundColor: this.cssHelper.propertyValue('chart-color-translucent-yellow'),
-          borderColor: this.cssHelper.propertyValue('chart-color-orange'),
-          borderWidth: 1,
-          fill: {
-            target: 'origin'
-          }
-        }
-      ]
-    };
-
     this.options = {
       plugins: {
         legend: {
@@ -179,27 +172,50 @@ export class DashboardAreaChartComponent implements OnChanges {
     this.updateChartData(changes);
   }
 
+  ngAfterViewInit() {
+    this.updateChartData(null);
+  }
+
   private updateChartData(changes: SimpleChanges): void {
-    this.chartData.dataset[0].label = this.label;
-    this.chartData.dataset[1].label = this.label2;
-    this.setChartTicks();
-    if (changes.data && changes.data.currentValue) {
-      this.data = changes.data.currentValue;
-      this.chartData.dataset[0].data = this.formatData(this.data);
-      [this.currentData, this.currentDataUnits] = this.convertUnits(
-        this.data[this.data.length - 1][1]
-      ).split(' ');
-      [this.maxConvertedValue, this.maxConvertedValueUnits] = this.convertUnits(
-        this.maxValue
-      ).split(' ');
+    for (let index = 0; index < this.labelsArray.length; index++) {
+      const colorIndex = index % this.chartColors.length;
+      this.chartData.dataset[index] = {
+        label: '',
+        data: [],
+        tension: 0.2,
+        pointBackgroundColor: this.chartColors[colorIndex][0],
+        backgroundColor: this.chartColors[colorIndex][1],
+        borderColor: this.chartColors[colorIndex][0],
+        borderWidth: 1,
+        fill: {
+          target: 'origin'
+        }
+      };
+      this.chartData.dataset[index].label = this.labelsArray[index];
     }
-    if (changes.data2 && changes.data2.currentValue) {
-      this.data2 = changes.data2.currentValue;
-      this.chartData.dataset[1].data = this.formatData(this.data2);
-      [this.currentData2, this.currentDataUnits2] = this.convertUnits(
-        this.data2[this.data2.length - 1][1]
-      ).split(' ');
+
+    this.setChartTicks();
+
+    if (this.dataArray) {
+      this.dataArray = changes?.dataArray?.currentValue || this.dataArray;
+      this.currentChartData = this.chartData;
+      for (let index = 0; index < this.dataArray.length; index++) {
+        this.chartData.dataset[index].data = this.formatData(this.dataArray[index]);
+        let currentDataValue = this.dataArray[index][this.dataArray[index].length - 1]
+          ? this.dataArray[index][this.dataArray[index].length - 1][1]
+          : 0;
+        if (currentDataValue) {
+          [
+            this.currentChartData.dataset[index]['currentData'],
+            this.currentChartData.dataset[index]['currentDataUnits']
+          ] = this.convertUnits(currentDataValue).split(' ');
+          [this.maxConvertedValue, this.maxConvertedValueUnits] = this.convertUnits(
+            this.maxValue
+          ).split(' ');
+        }
+      }
     }
+
     if (this.chart) {
       this.chart.chart.update();
     }
@@ -273,16 +289,12 @@ export class DashboardAreaChartComponent implements OnChanges {
     let maxValue = 0;
     let maxValueDataUnits = '';
 
-    if (this.data) {
-      let maxValueData = Math.max(...this.data.map((values: any) => values[1]));
-      if (this.data2) {
-        let maxValueData2 = Math.max(...this.data2.map((values: any) => values[1]));
-        maxValue = Math.max(maxValueData, maxValueData2);
-      } else {
-        maxValue = maxValueData;
-      }
-      [maxValue, maxValueDataUnits] = this.convertUnits(maxValue).split(' ');
-    }
+    const allDataValues = this.dataArray.reduce((array: string[], data) => {
+      return array.concat(data.map((values: [number, string]) => values[1]));
+    }, []);
+
+    maxValue = Math.max(...allDataValues.map(Number));
+    [maxValue, maxValueDataUnits] = this.convertUnits(maxValue).split(' ');
 
     const yAxesTicks = this.chart.chart.options.scales.y;
     yAxesTicks.ticks.callback = (value: any) => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
index 85dc5c96970b..4c290746b45b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
@@ -4,7 +4,7 @@
   <div class="row d-flex flex-row ps-3">
 
     <!-- First Grid to hold Details and Inventory Card-->
-    <div class="col-sm-3 d-flex flex-column ps-2 pe-4">
+    <div class="col-sm-3 d-flex flex-column ps-2">
 
       <!-- Details Card-->
       <cd-card cardTitle="Details"
@@ -216,54 +216,48 @@
         </div>
 
       <!-- This column will hold Cluster Utlization card -->
-        <div class="col-sm-12 d-flex flex-column pt-4">
-          <cd-card cardTitle="Cluster Utilization"
-                   i18n-title
-                   aria-label="Cluster utilization card">
-            <div class="ms-4 me-4 mt-0">
-              <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
-              </cd-dashboard-time-selector>
-              <ng-container *ngIf="capacity">
-                <cd-dashboard-area-chart chartTitle="Used Capacity (RAW)"
-                                         [maxValue]="capacity.total_bytes"
-                                         dataUnits="B"
-                                         label="Used Capacity"
-                                         [data]="queriesResults.USEDCAPACITY">
-                </cd-dashboard-area-chart>
-              </ng-container>
-              <cd-dashboard-area-chart chartTitle="IOPS"
-                                       dataUnits=""
-                                       decimals="0"
-                                       label="Reads"
-                                       label2="Writes"
-                                       [data]="queriesResults.READIOPS"
-                                       [data2]="queriesResults.WRITEIOPS">
-              </cd-dashboard-area-chart>
-              <cd-dashboard-area-chart chartTitle="OSD Latencies"
-                                       dataUnits="ms"
-                                       decimals="2"
-                                       label="Apply"
-                                       label2="Commit"
-                                       [data]="queriesResults.READLATENCY"
-                                       [data2]="queriesResults.WRITELATENCY">
-              </cd-dashboard-area-chart>
-              <cd-dashboard-area-chart chartTitle="Client Throughput"
-                                       dataUnits="B/s"
-                                       decimals="2"
-                                       label="Reads"
-                                       label2="Writes"
-                                       [data]="queriesResults.READCLIENTTHROUGHPUT"
-                                       [data2]="queriesResults.WRITECLIENTTHROUGHPUT">
+      <div class="col-sm-12 d-flex flex-column pt-4">
+        <cd-card cardTitle="Cluster Utilization"
+                 i18n-title
+                 aria-label="Cluster utilization card">
+          <div class="ms-4 me-4 mt-0">
+            <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
+            </cd-dashboard-time-selector>
+            <ng-container *ngIf="capacity">
+              <cd-dashboard-area-chart chartTitle="Used Capacity (RAW)"
+                                       [maxValue]="capacity.total_bytes"
+                                       dataUnits="B"
+                                       [labelsArray]="['Used Capacity']"
+                                       [dataArray]="[queriesResults.USEDCAPACITY]">
               </cd-dashboard-area-chart>
-              <cd-dashboard-area-chart chartTitle="Recovery Throughput"
-                                       dataUnits="B/s"
-                                       decimals="2"
-                                       label="Recovery Throughput"
-                                       [data]="queriesResults.RECOVERYBYTES">
-              </cd-dashboard-area-chart>
-            </div>
-          </cd-card>
-        </div>
+            </ng-container>
+            <cd-dashboard-area-chart chartTitle="IOPS"
+                                     dataUnits=""
+                                     decimals="0"
+                                     [labelsArray]="['Reads', 'Writes']"
+                                     [dataArray]="[queriesResults.READIOPS, queriesResults.WRITEIOPS]">
+            </cd-dashboard-area-chart>
+            <cd-dashboard-area-chart chartTitle="OSD Latencies"
+                                     dataUnits="ms"
+                                     decimals="2"
+                                     [labelsArray]="['Apply', 'Commit']"
+                                     [dataArray]="[queriesResults.READLATENCY, queriesResults.WRITELATENCY]">
+            </cd-dashboard-area-chart>
+            <cd-dashboard-area-chart chartTitle="Client Throughput"
+                                     dataUnits="B/s"
+                                     decimals="2"
+                                     [labelsArray]="['Reads', 'Writes']"
+                                     [dataArray]="[queriesResults.READCLIENTTHROUGHPUT, queriesResults.WRITECLIENTTHROUGHPUT]">
+            </cd-dashboard-area-chart>
+            <cd-dashboard-area-chart chartTitle="Recovery Throughput"
+                                     dataUnits="B/s"
+                                     decimals="2"
+                                     [labelsArray]="['Recovery Throughput']"
+                                     [dataArray]="[queriesResults.RECOVERYBYTES]">
+            </cd-dashboard-area-chart>
+          </div>
+        </cd-card>
+      </div>
       </div>
     </div>
   </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
index 7ec0cd4495be..3c44bd36a890 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
@@ -54,15 +54,17 @@ export class DashboardV3Component extends PrometheusListHelper implements OnInit
   healthData: any;
   categoryPgAmount: Record<string, number> = {};
   totalPgs = 0;
-  queriesResults: any = {
-    USEDCAPACITY: '',
-    IPS: '',
-    OPS: '',
-    READLATENCY: '',
-    WRITELATENCY: '',
-    READCLIENTTHROUGHPUT: '',
-    WRITECLIENTTHROUGHPUT: '',
-    RECOVERYBYTES: ''
+  queriesResults: { [key: string]: [] } = {
+    USEDCAPACITY: [],
+    IPS: [],
+    OPS: [],
+    READLATENCY: [],
+    WRITELATENCY: [],
+    READCLIENTTHROUGHPUT: [],
+    WRITECLIENTTHROUGHPUT: [],
+    RECOVERYBYTES: [],
+    READIOPS: [],
+    WRITEIOPS: []
   };
   telemetryEnabled: boolean;
   telemetryURL = 'https://telemetry-public.ceph.com/';
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.html
index 0bcc48b4be2d..66f3ec5a5a7a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.html
@@ -55,22 +55,18 @@
         </cd-dashboard-time-selector>
         <cd-dashboard-area-chart chartTitle="Requests/sec"
                                  dataUnits=""
-                                 label="Requests/sec"
-                                 [data]="queriesResults.RGW_REQUEST_PER_SECOND">
+                                 [labelsArray]="['Requests/sec']"
+                                 [dataArray]="[queriesResults.RGW_REQUEST_PER_SECOND]">
         </cd-dashboard-area-chart>
         <cd-dashboard-area-chart chartTitle="Latency"
                                  dataUnits="ms"
-                                 label="GET"
-                                 label2="PUT"
-                                 [data]="queriesResults.AVG_GET_LATENCY"
-                                 [data2]="queriesResults.AVG_PUT_LATENCY">
+                                 [labelsArray]="['GET', 'PUT']"
+                                 [dataArray]="[queriesResults.AVG_GET_LATENCY, queriesResults.AVG_PUT_LATENCY]">
         </cd-dashboard-area-chart>
         <cd-dashboard-area-chart chartTitle="Bandwidth"
                                  dataUnits="B"
-                                 label="GET"
-                                 label2="PUT"
-                                 [data]="queriesResults.GET_BANDWIDTH"
-                                 [data2]="queriesResults.PUT_BANDWIDTH">
+                                 [labelsArray]="['GET', 'PUT']"
+                                 [dataArray]="[queriesResults.GET_BANDWIDTH, queriesResults.PUT_BANDWIDTH]">
         </cd-dashboard-area-chart>
       </div>
     </cd-card>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.ts
index 00537b32af00..8b5901769c35 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-overview-dashboard/rgw-overview-dashboard.component.ts
@@ -46,11 +46,11 @@ export class RgwOverviewDashboardComponent implements OnInit, OnDestroy {
   ZoneSUb: Subscription;
   HealthSub: Subscription;
   BucketSub: Subscription;
-  queriesResults: any = {
-    RGW_REQUEST_PER_SECOND: '',
-    BANDWIDTH: '',
-    AVG_GET_LATENCY: '',
-    AVG_PUT_LATENCY: ''
+  queriesResults: { [key: string]: [] } = {
+    RGW_REQUEST_PER_SECOND: [],
+    BANDWIDTH: [],
+    AVG_GET_LATENCY: [],
+    AVG_PUT_LATENCY: []
   };
   timerGetPrometheusDataSub: Subscription;
   chartTitles = ['Metadata Sync', 'Data Sync'];
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
index 6917b37662a6..e1aa7a07cafc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
@@ -154,6 +154,8 @@ export class PrometheusService {
             }).subscribe((data: any) => {
               if (data.result.length) {
                 queriesResults[queryName] = data.result[0].values;
+              } else {
+                queriesResults[queryName] = [];
               }
               if (
                 queriesResults[queryName] !== undefined &&
diff --git a/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss b/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
index d69abf12bc8e..8147d9381ce6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/styles/defaults/_bootstrap-defaults.scss
@@ -15,13 +15,18 @@ $black: #000 !default;
 $blue: #007bff !default;
 $indigo: #6610f2 !default;
 $purple: #6f42c1 !default;
+$purple-dim: #6f42c180 !default;
 $pink: #a94442 !default;
 $red: #dc3545 !default;
+$red-dim: #dc354580 !default;
 $orange: #fd7e14 !default;
+$orange-dim: #fd7e1480 !default;
 $yellow: #d48200 !default;
 $green: #008a00 !default;
+$green-dim: #008a0080 !default;
 $teal: #20c997 !default;
 $cyan: #17a2b8 !default;
+$cyan-dim: #17a2b880 !default;
 $barley-white: #fcecba !default;
 
 $primary: #25828e !default;
@@ -75,17 +80,22 @@ $health-color-warning-800: #9d6d10 !default;
 
 // Chart colors.
 $chart-color-red: $red !default;
-$chart-color-blue: #06c !default;
-$chart-color-orange: #ef9234 !default;
 $chart-color-yellow: #f6d173 !default;
+$chart-color-translucent-red: $red-dim !default;
+$chart-color-blue: $blue !default;
+$chart-color-orange: $orange !default;
+$chart-color-translucent-orange: $orange-dim !default;
+$chart-color-translucent-green: $green-dim !default;
+$chart-color-translucent-cyan: $cyan-dim !default;
+$chart-color-yellow: $yellow !default;
 $chart-color-green: $green !default;
 $chart-color-gray: #ededed !default;
 $chart-color-cyan: $primary-500 !default;
 $chart-color-light-gray: #f0f0f0 !default;
 $chart-color-slight-dark-gray: #d7d7d7 !default;
 $chart-color-dark-gray: #afafaf !default;
-$chart-color-cyan: #73c5c5 !default;
-$chart-color-purple: #3c3d99 !default;
+$chart-color-purple: $purple !default;
+$chart-color-translucent-purple: $purple-dim !default;
 $chart-color-white: #fff !default;
 $chart-color-center-text: #151515 !default;
 $chart-color-center-text-description: #72767b !default;

From b0e6ce3967b3e59c4ae77f1a1eff50b22c6f12de Mon Sep 17 00:00:00 2001
From: zhangjianwei2 <zhangjianwei2_yewu@cmss.chinamobile.com>
Date: Thu, 4 Jan 2024 17:28:29 +0800
Subject: [PATCH 1365/2492] osd/OSD: reply pg_created when pg is peered

Problem:
  when PG is active+clean
  ceph osd force-create-pg 1.0 --yes-i-really-mean-it
  because osd is not reply pg_created to mon.
Affects:
  pool remains creating flags,
  mon creating_pgs.pg is not empty,
  lead to mon cannot trim OSDMaps
Solution:
  add PgCreateEvt for pg create
  if pg Active and is_peered,
  reply pg_created to mon.

Fixes: https://tracker.ceph.com/issues/63912

Signed-off-by: zhangjianwei2 <zhangjianwei2@cmss.chinamobile.com>
---
 src/osd/OSD.cc           |  4 +++-
 src/osd/PGPeeringEvent.h |  1 +
 src/osd/PeeringState.cc  | 16 ++++++++++++++++
 src/osd/PeeringState.h   |  7 ++++++-
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index ab83ef7ba4f6..ec90af93b201 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1265,6 +1265,7 @@ void OSDService::send_pg_created(pg_t pgid)
   auto o = get_osdmap();
   if (o->require_osd_release >= ceph_release_t::luminous) {
     pg_created.insert(pgid);
+    dout(20) << __func__ << " reply to mon " << pgid << " created." << dendl;
     monc->send_mon_message(new MOSDPGCreated(pgid));
   }
 }
@@ -1276,6 +1277,7 @@ void OSDService::send_pg_created()
   auto o = get_osdmap();
   if (o->require_osd_release >= ceph_release_t::luminous) {
     for (auto pgid : pg_created) {
+      dout(20) << __func__ << " reply to mon " << pgid << " created!" << dendl;
       monc->send_mon_message(new MOSDPGCreated(pgid));
     }
   }
@@ -9222,7 +9224,7 @@ void OSD::handle_fast_pg_create(MOSDPGCreate2 *m)
 	    std::make_shared<PGPeeringEvent>(
 	      m->epoch,
 	      m->epoch,
-	      NullEvt(),
+	      PgCreateEvt(),
 	      true,
 	      new PGCreateInfo(
 		pgid,
diff --git a/src/osd/PGPeeringEvent.h b/src/osd/PGPeeringEvent.h
index 2828880f6565..ceabcb799ec6 100644
--- a/src/osd/PGPeeringEvent.h
+++ b/src/osd/PGPeeringEvent.h
@@ -193,6 +193,7 @@ struct RequestRecoveryPrio : boost::statechart::event< RequestRecoveryPrio > {
   };
 
 TrivialEvent(NullEvt)
+TrivialEvent(PgCreateEvt)
 TrivialEvent(RemoteBackfillReserved)
 TrivialEvent(RemoteReservationRejectedTooFull)
 TrivialEvent(RemoteReservationRevokedTooFull)
diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index c81cb958a7a9..dbe94d9d5f4d 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -6332,6 +6332,22 @@ boost::statechart::result PeeringState::Active::react(const CheckReadable &evt)
   return discard_event();
 }
 
+boost::statechart::result PeeringState::Active::react(const PgCreateEvt &evt)
+{
+  DECLARE_LOCALS;
+  pg_t pgid = context< PeeringMachine >().spgid.pgid;
+
+  psdout(10) << __func__ << " receive PgCreateEvt"
+             << " is_peered=" << ps->is_peered() << dendl;
+
+  if (ps->is_peered()) {
+    psdout(10) << __func__ << " pg is peered, reply pg_created" << dendl;
+    pl->send_pg_created(pgid);
+  }
+
+  return discard_event();
+}
+
 /*
  * update info.history.last_epoch_started ONLY after we and all
  * replicas have activated AND committed the activate transaction
diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h
index 89ef702e0fea..f036bb44b115 100644
--- a/src/osd/PeeringState.h
+++ b/src/osd/PeeringState.h
@@ -690,6 +690,7 @@ class PeeringState : public MissingLoc::MappingInfo {
     typedef boost::mpl::list <
       boost::statechart::transition< Initialize, Reset >,
       boost::statechart::custom_reaction< NullEvt >,
+      boost::statechart::custom_reaction< PgCreateEvt >,
       boost::statechart::transition< boost::statechart::event_base, Crashed >
       > reactions;
 
@@ -711,6 +712,7 @@ class PeeringState : public MissingLoc::MappingInfo {
       boost::statechart::custom_reaction< AdvMap >,
       boost::statechart::custom_reaction< ActMap >,
       boost::statechart::custom_reaction< NullEvt >,
+      boost::statechart::custom_reaction< PgCreateEvt >,
       boost::statechart::custom_reaction< IntervalFlush >,
       boost::statechart::transition< boost::statechart::event_base, Crashed >
       > reactions;
@@ -737,6 +739,7 @@ class PeeringState : public MissingLoc::MappingInfo {
       boost::statechart::custom_reaction< IntervalFlush >,
       // ignored
       boost::statechart::custom_reaction< NullEvt >,
+      boost::statechart::custom_reaction< PgCreateEvt >,
       boost::statechart::custom_reaction<SetForceRecovery>,
       boost::statechart::custom_reaction<UnsetForceRecovery>,
       boost::statechart::custom_reaction<SetForceBackfill>,
@@ -867,7 +870,8 @@ class PeeringState : public MissingLoc::MappingInfo {
       boost::statechart::custom_reaction< DoRecovery>,
       boost::statechart::custom_reaction< RenewLease>,
       boost::statechart::custom_reaction< MLeaseAck>,
-      boost::statechart::custom_reaction< CheckReadable>
+      boost::statechart::custom_reaction< CheckReadable>,
+      boost::statechart::custom_reaction< PgCreateEvt >
       > reactions;
     boost::statechart::result react(const QueryState& q);
     boost::statechart::result react(const QueryUnfound& q);
@@ -906,6 +910,7 @@ class PeeringState : public MissingLoc::MappingInfo {
       return discard_event();
     }
     boost::statechart::result react(const CheckReadable&);
+    boost::statechart::result react(const PgCreateEvt&);
     void all_activated_and_committed();
   };
 

From ecf546f93b9676fab8df3dc606839a09600ccecd Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 16 Jan 2024 09:49:39 +0530
Subject: [PATCH 1366/2492] qa: remove redundant test
 test_volumes.TestRenameCmd.test_rename_when_fs_is_online

See - https://github.com/ceph/ceph/pull/53899/files#r1382869363

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index f5894dbf7907..d86af3412dcf 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -745,21 +745,6 @@ def test_volume_rename_for_more_than_one_data_pool(self):
         # data pool names unchanged
         self.assertCountEqual(orig_data_pool_names, list(self.fs.data_pools.values()))
 
-    def test_rename_when_fs_is_online(self):
-        for m in self.mounts:
-            m.umount_wait()
-        newvolname = self._gen_vol_name()
-
-        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session true')
-        self.negtest_ceph_cmd(
-            args=(f'fs volume rename {self.volname} {newvolname} '
-                   '--yes-i-really-mean-it'),
-            errmsgs=(f"CephFS '{self.volname}' is not offline. Before "
-                      "renaming a CephFS, it must be marked as down. See "
-                      "`ceph fs fail`."),
-            retval=errno.EPERM)
-        self.run_ceph_cmd(f'fs set {self.volname} refuse_client_session false')
-
     def test_rename_when_clients_arent_refused(self):
         newvolname = self._gen_vol_name()
         for m in self.mounts:

From dad7e5f67c095d04cca4156c062d6629776678c4 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 16 Jan 2024 14:32:29 +0800
Subject: [PATCH 1367/2492] common/options/crimson: increase
 crimson_osd_obc_lru_size to 512

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/common/options/crimson.yaml.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index 8f0af93f3ea8..d3c2c57a674c 100644
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -6,7 +6,7 @@ options:
   type: uint
   level: advanced
   desc: Number of obcs to cache
-  default: 10
+  default: 512
 - name: crimson_osd_scheduler_concurrency
   type: uint
   level: advanced

From 6bd05fcf62d4182e8494ad4a5dadda3f304922b0 Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Tue, 16 Jan 2024 16:13:03 +0800
Subject: [PATCH 1368/2492] crimson/os/seastore: correct the parameter passed
 to LBAManager::clone_mapping

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/crimson/os/seastore/transaction_manager.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 911ebbf6008f..2bfed6ac9f66 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -517,6 +517,10 @@ class TransactionManager : public ExtentCallbackInterface {
       mapping.is_indirect()
 	? mapping.get_intermediate_key()
 	: mapping.get_key();
+    auto intermediate_base =
+      mapping.is_indirect()
+        ? mapping.get_intermediate_base()
+        : mapping.get_key();
 
     LOG_PREFIX(TransactionManager::clone_pin);
     SUBDEBUGT(seastore_tm, "len={}, laddr_hint={}, clone_offset {}",
@@ -528,7 +532,7 @@ class TransactionManager : public ExtentCallbackInterface {
       mapping.get_length(),
       intermediate_key,
       mapping.get_val(),
-      intermediate_key
+      intermediate_base
     );
   }
 

From a916feeee757e4e196967944feeb1e9b1f92c398 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 16 Jan 2024 10:51:56 +0530
Subject: [PATCH 1369/2492] admin/doc-requirements: bump Sphinx to 5.0.2

```
Running Sphinx v4.5.0

Sphinx version error:
The sphinxcontrib.applehelp extension used by this project needs at least Sphinx v5.0; it therefore cannot be built with this version.
```

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 admin/doc-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/admin/doc-requirements.txt b/admin/doc-requirements.txt
index 2f4970e3fbb3..636f7e138511 100644
--- a/admin/doc-requirements.txt
+++ b/admin/doc-requirements.txt
@@ -1,4 +1,4 @@
-Sphinx == 4.5.0
+Sphinx == 5.0.2
 git+https://github.com/ceph/sphinx-ditaa.git@py3#egg=sphinx-ditaa
 git+https://github.com/vlasovskikh/funcparserlib.git
 breathe >= 4.20.0,!=4.33

From d169a206d11283428f7eb74a928525bc93f58915 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Wed, 18 Oct 2023 13:53:51 +0530
Subject: [PATCH 1370/2492] mgr/dashboard: add frontend unit tests for rgw
 multisite sync status card

Fixes: https://tracker.ceph.com/issues/64039

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../rgw-sync-data-info.component.spec.ts      | 53 +++++++++++++++++-
 .../rgw-sync-metadata-info.component.spec.ts  | 54 ++++++++++++++++++-
 .../rgw-sync-primary-zone.component.spec.ts   | 17 ++++++
 3 files changed, 121 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-data-info/rgw-sync-data-info.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-data-info/rgw-sync-data-info.component.spec.ts
index 1c7ce8a78657..944435cba16a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-data-info/rgw-sync-data-info.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-data-info/rgw-sync-data-info.component.spec.ts
@@ -1,15 +1,17 @@
-import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { ComponentFixture, TestBed, fakeAsync } from '@angular/core/testing';
 
 import { RgwSyncDataInfoComponent } from './rgw-sync-data-info.component';
 import { NgbPopoverModule } from '@ng-bootstrap/ng-bootstrap';
 import { configureTestBed } from '~/testing/unit-test-helper';
+import { RelativeDatePipe } from '~/app/shared/pipes/relative-date.pipe';
+import { By } from '@angular/platform-browser';
 
 describe('RgwSyncDataInfoComponent', () => {
   let component: RgwSyncDataInfoComponent;
   let fixture: ComponentFixture<RgwSyncDataInfoComponent>;
 
   configureTestBed({
-    declarations: [RgwSyncDataInfoComponent],
+    declarations: [RgwSyncDataInfoComponent, RelativeDatePipe],
     imports: [NgbPopoverModule]
   });
 
@@ -22,4 +24,51 @@ describe('RgwSyncDataInfoComponent', () => {
   it('should create', () => {
     expect(component).toBeTruthy();
   });
+
+  it('should display "Up to Date" badge when zone is up to date', () => {
+    component.zone = {
+      timestamp: null
+    };
+    fixture.detectChanges();
+    const upToDateBadge = fixture.debugElement.query(By.css('.badge-success'));
+    expect(upToDateBadge).toBeTruthy();
+    expect(upToDateBadge.nativeElement.textContent).toEqual('Up to Date');
+  });
+
+  it('should display correct sync status and last synced time', fakeAsync(() => {
+    component.zone = { syncstatus: 'Syncing', timestamp: new Date(Date.now() - 10 * 60 * 1000) };
+    fixture.detectChanges();
+
+    const statusElement = fixture.debugElement.query(By.css('li b'));
+    expect(statusElement.nativeElement.textContent).toContain('Status:');
+
+    const lastSyncedElement = fixture.debugElement.query(By.css('li.mt-4.fw-bold'));
+    expect(lastSyncedElement.nativeElement.textContent).toContain('Last Synced:');
+    const lastSyncedTimestamp = fixture.debugElement.query(By.css('.badge-info'));
+    expect(lastSyncedTimestamp.nativeElement.textContent).toEqual('10 minutes ago');
+  }));
+
+  it('should display sync status in the popover', () => {
+    component.zone = {
+      syncstatus: 'Syncing',
+      timestamp: new Date(Date.now() - 10 * 60 * 1000),
+      fullSyncStatus: [
+        'full sync: 0/128 shards',
+        'incremental sync:128/128 shards',
+        'Data is behind on 31 shards'
+      ]
+    };
+    fixture.detectChanges();
+    const syncStatus = fixture.debugElement.query(By.css('.text-primary'));
+    expect(syncStatus).toBeTruthy();
+    expect(syncStatus.nativeElement.textContent).toEqual('Syncing');
+    const syncPopover = fixture.debugElement.query(By.css('a'));
+    syncPopover.triggerEventHandler('click', null);
+    fixture.detectChanges();
+    expect(syncPopover).toBeTruthy();
+    const syncPopoverText = fixture.debugElement.query(By.css('.text-center'));
+    expect(syncPopoverText.nativeElement.textContent).toEqual(
+      'Sync Status:Full Sync:0/128 Shards Incremental Sync:128/128 Shards: Data Is Behind On 31 Shards'
+    );
+  });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-metadata-info/rgw-sync-metadata-info.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-metadata-info/rgw-sync-metadata-info.component.spec.ts
index df3748b175b3..0b7e4ede377d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-metadata-info/rgw-sync-metadata-info.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-metadata-info/rgw-sync-metadata-info.component.spec.ts
@@ -3,13 +3,15 @@ import { ComponentFixture, TestBed } from '@angular/core/testing';
 import { RgwSyncMetadataInfoComponent } from './rgw-sync-metadata-info.component';
 import { NgbPopoverModule } from '@ng-bootstrap/ng-bootstrap';
 import { configureTestBed } from '~/testing/unit-test-helper';
+import { By } from '@angular/platform-browser';
+import { RelativeDatePipe } from '~/app/shared/pipes/relative-date.pipe';
 
 describe('RgwSyncMetadataInfoComponent', () => {
   let component: RgwSyncMetadataInfoComponent;
   let fixture: ComponentFixture<RgwSyncMetadataInfoComponent>;
 
   configureTestBed({
-    declarations: [RgwSyncMetadataInfoComponent],
+    declarations: [RgwSyncMetadataInfoComponent, RelativeDatePipe],
     imports: [NgbPopoverModule]
   });
 
@@ -22,4 +24,54 @@ describe('RgwSyncMetadataInfoComponent', () => {
   it('should create', () => {
     expect(component).toBeTruthy();
   });
+
+  it('should display "Up to Date" badge when zone is up to date', () => {
+    component.metadataSyncInfo = {
+      timestamp: null
+    };
+    fixture.detectChanges();
+    const upToDateBadge = fixture.debugElement.query(By.css('.badge-success'));
+    expect(upToDateBadge).toBeTruthy();
+    expect(upToDateBadge.nativeElement.textContent).toEqual('Up to Date');
+  });
+
+  it('should display correct sync status and last synced time', () => {
+    component.metadataSyncInfo = {
+      syncstatus: 'Syncing',
+      timestamp: new Date(Date.now() - 10 * 60 * 1000)
+    };
+    fixture.detectChanges();
+
+    const statusElement = fixture.debugElement.query(By.css('li b'));
+    expect(statusElement.nativeElement.textContent).toContain('Status:');
+
+    const lastSyncedElement = fixture.debugElement.query(By.css('li.mt-4.fw-bold'));
+    expect(lastSyncedElement.nativeElement.textContent).toContain('Last Synced:');
+    const lastSyncedTimestamp = fixture.debugElement.query(By.css('.badge-info'));
+    expect(lastSyncedTimestamp.nativeElement.textContent).toEqual('10 minutes ago');
+  });
+
+  it('should display sync status in the popover', () => {
+    component.metadataSyncInfo = {
+      syncstatus: 'Syncing',
+      timestamp: new Date(Date.now() - 10 * 60 * 1000),
+      fullSyncStatus: [
+        'full sync:0/128 shards',
+        'incremental sync:128/128 shards',
+        'Data is behind on 31 shards'
+      ]
+    };
+    fixture.detectChanges();
+    const syncStatus = fixture.debugElement.query(By.css('.text-primary'));
+    expect(syncStatus).toBeTruthy();
+    expect(syncStatus.nativeElement.textContent).toEqual('Syncing');
+    const syncPopover = fixture.debugElement.query(By.css('a'));
+    syncPopover.triggerEventHandler('click', null);
+    fixture.detectChanges();
+    expect(syncPopover).toBeTruthy();
+    const syncPopoverText = fixture.debugElement.query(By.css('.text-center'));
+    expect(syncPopoverText.nativeElement.textContent).toEqual(
+      'Metadata Sync Status:Full Sync:0/128 Shards Incremental Sync:128/128 Shards Data Is Behind On 31 Shards'
+    );
+  });
 });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-primary-zone/rgw-sync-primary-zone.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-primary-zone/rgw-sync-primary-zone.component.spec.ts
index aefb32794a71..1bce83fcc5a6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-primary-zone/rgw-sync-primary-zone.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-sync-primary-zone/rgw-sync-primary-zone.component.spec.ts
@@ -2,6 +2,7 @@ import { ComponentFixture, TestBed } from '@angular/core/testing';
 
 import { RgwSyncPrimaryZoneComponent } from './rgw-sync-primary-zone.component';
 import { configureTestBed } from '~/testing/unit-test-helper';
+import { By } from '@angular/platform-browser';
 
 describe('RgwSyncPrimaryZoneComponent', () => {
   let component: RgwSyncPrimaryZoneComponent;
@@ -20,4 +21,20 @@ describe('RgwSyncPrimaryZoneComponent', () => {
   it('should create', () => {
     expect(component).toBeTruthy();
   });
+
+  it('should display realm, zonegroup, and zone in badges', () => {
+    component.realm = 'Realm';
+    component.zonegroup = 'Zonegroup';
+    component.zone = 'Zone';
+    fixture.detectChanges();
+
+    const realmBadge = fixture.debugElement.query(By.css('li:nth-child(2)'));
+    expect(realmBadge.nativeElement.textContent).toContain('Realm');
+
+    const zonegroupBadge = fixture.debugElement.query(By.css('p'));
+    expect(zonegroupBadge.nativeElement.textContent).toContain('Zonegroup');
+
+    const zoneBadge = fixture.debugElement.query(By.css('li:nth-child(8)'));
+    expect(zoneBadge.nativeElement.textContent).toContain('Zone');
+  });
 });

From a1fca50dfdcb30e58ebcc4ef60ef921b977efc81 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 16 Jan 2024 20:37:17 +1000
Subject: [PATCH 1371/2492] doc/radosgw: edit "Enable/Disable Bucket Rate
 Limit"

Edit "Enable/Disable Bucket Rate Limit" in doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 57d38c97ab2f..03dad19c2a92 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -747,17 +747,22 @@ A 0 value for num ops and / or num bytes means that the
 specific rate limit attribute check is disabled.
 
 
-Enable/Disable Bucket Rate Limit
---------------------------------
+Enable and Disable Bucket Rate Limit
+------------------------------------
+
+After you set a bucket rate limit, you can enable it. The following is the
+general form of the ``radosgw-admin ratelimit enable`` command that enables
+bucket rate limits: 
 
-Once you set a bucket rate limit, you may enable it. For example:: 
+.. prompt:: bash
 
-	radosgw-admin ratelimit enable --ratelimit-scope=bucket --bucket=<bucket>
+   radosgw-admin ratelimit enable --ratelimit-scope=bucket --bucket=<bucket>
 
-You may disable an enabled bucket rate limit. For example:: 
+An enabled bucket rate limit can be disabled by running a command of the following form:
 
-	radosgw-admin ratelimit disable --ratelimit-scope=bucket --uid=mybucket
+.. prompt:: bash
 
+   radosgw-admin ratelimit disable --ratelimit-scope=bucket --uid=mybucket
 
 Reading / Writing Global Rate Limit Configuration
 -------------------------------------------------

From d8d18011a82f3e69f5726827347f94b5953a84ce Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 16 Jan 2024 18:09:54 +0530
Subject: [PATCH 1372/2492] qa: typo fixes in ceph_test_case docstrings

Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/ceph_test_case.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py
index 877f79d06a3b..bbb9c4fb73e4 100644
--- a/qa/tasks/ceph_test_case.py
+++ b/qa/tasks/ceph_test_case.py
@@ -20,7 +20,7 @@ class RunCephCmd:
     def run_ceph_cmd(self, *args, **kwargs):
         """
         *args and **kwargs must contain arguments that are accepted by
-        vstart_runner.LocalRemote._do_run() or teuhology.orchestra.run.run()
+        vstart_runner.LocalRemote._do_run() or teuthology.orchestra.run.run()
         methods.
         """
         if kwargs.get('args') is None and args:
@@ -32,7 +32,7 @@ def run_ceph_cmd(self, *args, **kwargs):
     def get_ceph_cmd_result(self, *args, **kwargs):
         """
         *args and **kwargs must contain arguments that are accepted by
-        vstart_runner.LocalRemote._do_run() or teuhology.orchestra.run.run()
+        vstart_runner.LocalRemote._do_run() or teuthology.orchestra.run.run()
         methods.
         """
         if kwargs.get('args') is None and args:
@@ -44,7 +44,7 @@ def get_ceph_cmd_result(self, *args, **kwargs):
     def get_ceph_cmd_stdout(self, *args, **kwargs):
         """
         *args and **kwargs must contain arguments that are accepted by
-        vstart_runner.LocalRemote._do_run() or teuhology.orchestra.run.run()
+        vstart_runner.LocalRemote._do_run() or teuthology.orchestra.run.run()
         methods.
         """
         if kwargs.get('args') is None and args:
@@ -94,7 +94,7 @@ def negtest_ceph_cmd(self, args, retval=None, errmsgs=None, **kwargs):
         failure.
 
         *args and **kwargs must contain arguments that are accepted by
-        vstart_runner.LocalRemote._do_run() or teuhology.orchestra.run.run()
+        vstart_runner.LocalRemote._do_run() or teuthology.orchestra.run.run()
         methods.
 
         NOTE: errmsgs is expected to be a tuple, but in case there's only one

From 841bc653efe7df12f12185e728d93935e66c8d6b Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 3 Dec 2023 10:27:23 +0000
Subject: [PATCH 1373/2492] crimson/osd/pg: do_osd_ops_execute drop
 error_func_ptr

std::move to the only user instead

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index f20ea15d66f5..a605745a3bef 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -832,7 +832,6 @@ PG::do_osd_ops_execute(
     }
     return seastar::now();
   };
-  auto error_func_ptr = seastar::make_lw_shared(std::move(maybe_submit_error_log));
   auto failure_func_ptr = seastar::make_lw_shared(std::move(failure_func));
   return interruptor::do_for_each(ops, [ox](OSDOp& osd_op) {
     logger().debug(
@@ -892,7 +891,7 @@ PG::do_osd_ops_execute(
           std::move(log_entries));
     });
   }).safe_then_unpack_interruptible(
-    [success_func=std::move(success_func), error_func_ptr, rollbacker, this, failure_func_ptr]
+    [success_func=std::move(success_func), rollbacker, this, failure_func_ptr]
     (auto submitted_fut, auto _all_completed_fut) mutable {
 
     auto all_completed_fut = _all_completed_fut.safe_then_interruptible_tuple(
@@ -929,14 +928,16 @@ PG::do_osd_ops_execute(
       std::move(all_completed_fut)
     );
   }, OpsExecuter::osd_op_errorator::all_same_way(
-    [this, error_func_ptr, rollbacker, failure_func_ptr]
+    [this, maybe_submit_error_log=std::move(maybe_submit_error_log),
+     rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
 
     ceph_tid_t rep_tid = shard_services.get_tid();
     return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-    [error_func_ptr, e, rep_tid, failure_func_ptr] {
+    [maybe_submit_error_log=std::move(maybe_submit_error_log),
+     e, rep_tid, failure_func_ptr] {
       // record error log
-      return (*error_func_ptr)(e, rep_tid).then(
+      return maybe_submit_error_log(e, rep_tid).then(
       [failure_func_ptr, e, rep_tid] {
         return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
           std::move(seastar::now()),

From b44c41f08c2c2196197a5ce535466c94bb09398a Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 7 Dec 2023 13:48:02 +0000
Subject: [PATCH 1374/2492] crimson/osd/pg: submit_error_log cleanup

* error log completion logic is moved into maybe_submit_error_log
* renamed it and it2
* maybe_submit_error_log is moved outside of failure_func
* failure_func no longer gets rep_tid and record_error params
* log_entry_version is removed, submit_error_log returns the version instead

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 104 ++++++++++++++++++++++--------------------
 src/crimson/osd/pg.h  |   5 +-
 2 files changed, 58 insertions(+), 51 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index a605745a3bef..cd2478d58eac 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -323,7 +323,6 @@ void PG::on_removal(ceph::os::Transaction &t) {
 void PG::clear_log_entry_maps()
 {
   log_entry_update_waiting_on.clear();
-  log_entry_version.clear();
 }
 
 void PG::on_activate(interval_set<snapid_t> snaps)
@@ -830,7 +829,7 @@ PG::do_osd_ops_execute(
         return submit_error_log(m, op_info, obc, e, rep_tid);
       }
     }
-    return seastar::now();
+    return seastar::make_ready_future<std::optional<eversion_t>>(std::nullopt);
   };
   auto failure_func_ptr = seastar::make_lw_shared(std::move(failure_func));
   return interruptor::do_for_each(ops, [ox](OSDOp& osd_op) {
@@ -910,16 +909,16 @@ PG::do_osd_ops_execute(
         });
       });
     }), OpsExecuter::osd_op_errorator::all_same_way(
-        [this, rollbacker, failure_func_ptr]
+        [rollbacker, failure_func_ptr]
         (const std::error_code& e) mutable {
           // handle non-fatal errors only
           ceph_assert(e.value() == EDQUOT ||
                       e.value() == ENOSPC ||
                       e.value() == EAGAIN);
           return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-          [this, e, failure_func_ptr] {
+          [e, failure_func_ptr] {
             // no need to record error log
-            return (*failure_func_ptr)(e , shard_services.get_tid(), false);
+            return (*failure_func_ptr)(e);
           });
     }));
 
@@ -935,20 +934,59 @@ PG::do_osd_ops_execute(
     ceph_tid_t rep_tid = shard_services.get_tid();
     return rollbacker.rollback_obc_if_modified(e).then_interruptible(
     [maybe_submit_error_log=std::move(maybe_submit_error_log),
-     e, rep_tid, failure_func_ptr] {
+     this, e, rep_tid, failure_func_ptr] {
       // record error log
       return maybe_submit_error_log(e, rep_tid).then(
-      [failure_func_ptr, e, rep_tid] {
+      [this, failure_func_ptr, e, rep_tid] (auto version) {
+        auto all_completed =
+        [this, failure_func_ptr, e, rep_tid,  version] {
+          if (version.has_value()) {
+            return complete_error_log(rep_tid, version.value()).then(
+            [failure_func_ptr, e] {
+              return (*failure_func_ptr)(e);
+            });
+          } else {
+            return (*failure_func_ptr)(e);
+          }
+        };
         return PG::do_osd_ops_iertr::make_ready_future<pg_rep_op_fut_t<Ret>>(
           std::move(seastar::now()),
-          std::move((*failure_func_ptr)(e, rep_tid, true))
+          std::move(all_completed())
         );
       });
     });
   }));
 }
 
-seastar::future<> PG::submit_error_log(
+seastar::future<> PG::complete_error_log(const ceph_tid_t& rep_tid,
+                                         const eversion_t& version)
+{
+  auto result = seastar::now();
+  auto last_complete = peering_state.get_info().last_complete;
+  ceph_assert(log_entry_update_waiting_on.contains(rep_tid));
+  auto& log_update = log_entry_update_waiting_on[rep_tid];
+  ceph_assert(log_update.waiting_on.contains(pg_whoami));
+  log_update.waiting_on.erase(pg_whoami);
+  if (log_update.waiting_on.empty()) {
+    log_entry_update_waiting_on.erase(rep_tid);
+    peering_state.complete_write(version, last_complete);
+    logger().debug("complete_error_log: write complete,"
+                   " erasing rep_tid {}", rep_tid);
+  } else {
+    logger().debug("complete_error_log: rep_tid {} awaiting update from {}",
+                   rep_tid, log_update.waiting_on);
+    result = log_update.all_committed.get_shared_future().then(
+    [this, last_complete, rep_tid, version] {
+      logger().debug("complete_error_log: rep_tid {} awaited ", rep_tid);
+      peering_state.complete_write(version, last_complete);
+      ceph_assert(!log_entry_update_waiting_on.contains(rep_tid));
+      return seastar::now();
+    });
+  }
+  return result;
+}
+
+seastar::future<std::optional<eversion_t>> PG::submit_error_log(
   Ref<MOSDOp> m,
   const OpInfo &op_info,
   ObjectContextRef obc,
@@ -971,7 +1009,7 @@ seastar::future<> PG::submit_error_log(
   ceph_assert(is_primary());
   ceph_assert(!log_entries.empty());
   ceph_assert(log_entries.rbegin()->version >= projected_last_update);
-  log_entry_version[rep_tid] = projected_last_update = log_entries.rbegin()->version;
+  projected_last_update = log_entries.rbegin()->version;
   ceph::os::Transaction t;
   peering_state.merge_new_log_entries(
     log_entries, t, peering_state.get_pg_trim_to(),
@@ -1014,7 +1052,7 @@ seastar::future<> PG::submit_error_log(
         get_collection_ref(), std::move(t)
       ).then([this] {
         peering_state.update_trim_to();
-        return seastar::now();
+        return seastar::make_ready_future<std::optional<eversion_t>>(projected_last_update);
       });
     });
   });
@@ -1078,42 +1116,11 @@ PG::do_osd_ops(
         std::move(reply));
     },
     // failure_func
-    [m, &op_info, obc, this]
-    (const std::error_code& e, const ceph_tid_t& rep_tid, bool record_error) {
-    logger().error("do_osd_ops_execute::failure_func {} got error: {} record_error: {}",
-                    *m, e, record_error);
-    epoch_t epoch = get_osdmap_epoch();
-    auto last_complete = peering_state.get_info().last_complete;
-    auto fut = seastar::now();
-    if (record_error && !peering_state.pg_has_reset_since(epoch) && op_info.may_write()) {
-      logger().debug("do_osd_ops_execute::failure_func finding rep_tid {}",
-                      rep_tid);
-      ceph_assert(log_entry_version.contains(rep_tid));
-      auto it = log_entry_update_waiting_on.find(rep_tid);
-      ceph_assert(it != log_entry_update_waiting_on.end());
-      auto it2 = it->second.waiting_on.find(pg_whoami);
-      ceph_assert(it2 != it->second.waiting_on.end());
-      it->second.waiting_on.erase(it2);
-      if (it->second.waiting_on.empty()) {
-        log_entry_update_waiting_on.erase(it);
-        peering_state.complete_write(log_entry_version[rep_tid], last_complete);
-        log_entry_version.erase(rep_tid);
-        logger().debug("do_osd_ops_execute::failure_func write complete,"
-                        " erasing rep_tid {}", rep_tid);
-
-      } else {
-        fut = it->second.all_committed.get_shared_future().then(
-          [this, last_complete, rep_tid] {
-          logger().debug("do_osd_ops_execute::failure_func awaited {}", rep_tid);
-          peering_state.complete_write(log_entry_version[rep_tid], last_complete);
-          ceph_assert(!log_entry_update_waiting_on.contains(rep_tid));
-          return seastar::now();
-        });
-      }
-    }
-    return fut.then([this, m, e] {
-      return log_reply(m, e);
-    });
+    [m, this]
+    (const std::error_code& e) {
+    logger().error("do_osd_ops_execute::failure_func {} got error: {}",
+                    *m, e);
+    return log_reply(m, e);
   });
 }
 
@@ -1177,7 +1184,7 @@ PG::do_osd_ops(
         return do_osd_ops_iertr::now();
       },
       // failure_func
-      [] (const std::error_code& e, const ceph_tid_t& rep_tid, bool record_error) {
+      [] (const std::error_code& e) {
         return do_osd_ops_iertr::now();
       });
   });
@@ -1442,7 +1449,6 @@ PG::interruptible_future<> PG::do_update_log_missing_reply(
       logger().debug("{}: erasing rep_tid {}",
                      __func__, m->get_tid());
       log_entry_update_waiting_on.erase(it);
-      log_entry_version.erase(m->get_tid());
     }
   } else {
     logger().error("{} : {} got reply {} on unknown tid {}",
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 80a181f24a23..b829ea177dbb 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -533,7 +533,9 @@ class PG : public boost::intrusive_ref_counter<
 
   void print(std::ostream& os) const;
   void dump_primary(Formatter*);
-  seastar::future<> submit_error_log(
+  seastar::future<> complete_error_log(const ceph_tid_t& rep_tid,
+                                       const eversion_t& version);
+  seastar::future<std::optional<eversion_t>> submit_error_log(
     Ref<MOSDOp> m,
     const OpInfo &op_info,
     ObjectContextRef obc,
@@ -791,7 +793,6 @@ class PG : public boost::intrusive_ref_counter<
   };
 
   std::map<ceph_tid_t, log_update_t> log_entry_update_waiting_on;
-  std::map<ceph_tid_t, eversion_t> log_entry_version;
   // snap trimming
   interval_set<snapid_t> snap_trimq;
 };

From 0f99cdc81ee141869a92c5a8107bae82d215c292 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 30 Nov 2023 14:57:06 +0000
Subject: [PATCH 1375/2492] crimson/osd/pg: on_removal to assert empty
 log_entry map

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index cd2478d58eac..8304697d5683 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -309,7 +309,7 @@ unsigned PG::get_target_pg_log_entries() const
 }
 
 void PG::on_removal(ceph::os::Transaction &t) {
-  clear_log_entry_maps();
+  ceph_assert(log_entry_update_waiting_on.empty());
   t.register_on_commit(
     new LambdaContext(
       [this](int r) {

From 4997eadda7443d01655cdf89c0e3c6e831e0a9c7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 20 Dec 2023 08:29:45 +0000
Subject: [PATCH 1376/2492] crimson/osd/pg: move maybe_submit_error_log

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 8304697d5683..3813b882a3d5 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -821,16 +821,6 @@ PG::do_osd_ops_execute(
     return obc_loader.reload_obc(obc).handle_error_interruptible(
       load_obc_ertr::assert_all{"can't live with object state messed up"});
   });
-  auto maybe_submit_error_log = [&, op_info, m, obc]
-    (const std::error_code& e, const ceph_tid_t& rep_tid) {
-    // call submit_error_log only for non-internal clients
-    if constexpr (!std::is_same_v<Ret, void>) {
-      if(op_info.may_write()) {
-        return submit_error_log(m, op_info, obc, e, rep_tid);
-      }
-    }
-    return seastar::make_ready_future<std::optional<eversion_t>>(std::nullopt);
-  };
   auto failure_func_ptr = seastar::make_lw_shared(std::move(failure_func));
   return interruptor::do_for_each(ops, [ox](OSDOp& osd_op) {
     logger().debug(
@@ -927,16 +917,24 @@ PG::do_osd_ops_execute(
       std::move(all_completed_fut)
     );
   }, OpsExecuter::osd_op_errorator::all_same_way(
-    [this, maybe_submit_error_log=std::move(maybe_submit_error_log),
+    [this, op_info, m, obc,
      rollbacker, failure_func_ptr]
     (const std::error_code& e) mutable {
-
     ceph_tid_t rep_tid = shard_services.get_tid();
     return rollbacker.rollback_obc_if_modified(e).then_interruptible(
-    [maybe_submit_error_log=std::move(maybe_submit_error_log),
+    [&, op_info, m, obc,
      this, e, rep_tid, failure_func_ptr] {
       // record error log
-      return maybe_submit_error_log(e, rep_tid).then(
+      auto maybe_submit_error_log =
+        seastar::make_ready_future<std::optional<eversion_t>>(std::nullopt);
+      // call submit_error_log only for non-internal clients
+      if constexpr (!std::is_same_v<Ret, void>) {
+        if(op_info.may_write()) {
+          maybe_submit_error_log =
+            submit_error_log(m, op_info, obc, e, rep_tid);
+        }
+      }
+      return maybe_submit_error_log.then(
       [this, failure_func_ptr, e, rep_tid] (auto version) {
         auto all_completed =
         [this, failure_func_ptr, e, rep_tid,  version] {

From 98f3c19104720ff2ab929c0891bd33e2634c87bc Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 14 Jan 2024 12:33:52 +0000
Subject: [PATCH 1377/2492] crimson/osd/osd_operations/client_request: return
 OpInfo errors

See NeoRadosCls.DNE or LibRadosCls.DNE

Fixes: https://tracker.ceph.com/issues/64000

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd_operations/client_request.cc | 5 ++++-
 src/crimson/osd/pg.cc                            | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 120b92ae0949..0936889e37e3 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -249,7 +249,10 @@ ClientRequest::process_op(
 	  -> PG::load_obc_iertr::future<> {
 	  DEBUGDPP("{}.{}: entered get_obc stage, about to wait_scrub",
 		   *pg, *this, this_instance_id);
-          op_info.set_from_op(&*m, *pg->get_osdmap());
+          if (int res = op_info.set_from_op(&*m, *pg->get_osdmap());
+              res != 0) {
+	    return reply_op_error(pg, res);
+          }
 	  return ihref.enter_blocker(
 	    *this,
 	    pg->scrubber,
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index f20ea15d66f5..48544ea2566a 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1217,7 +1217,7 @@ hobject_t PG::get_oid(const hobject_t& hobj)
 
 RWState::State PG::get_lock_type(const OpInfo &op_info)
 {
-
+  ceph_assert(op_info.get_flags());
   if (op_info.rwordered() && op_info.may_read()) {
     return RWState::RWEXCL;
   } else if (op_info.rwordered()) {

From da362521bfbe455a6240506000b05b9800adb0b2 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Tue, 7 Nov 2023 13:37:26 +0200
Subject: [PATCH 1378/2492] common/admin_socket: improvements to the RaiseHook

When porting the original feature to older releases
an issue was spotted that resulted in an effectively
infinite loop inside the killer fork

The problem was that the timespan deduced by the `auto`
variable definition ended up to be unsigned (the default)
As a result, when the difference between now and ref
was negative, it began awaiting some insane 64bit-second period

This improvement both addresses some inefficiency with the
spin loop and avoids using the `auto` to explicitly request
the expected span types.

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/common/admin_socket.cc | 42 ++++++++++++++++++++++----------------
 src/test/admin_socket.cc   |  1 +
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc
index c4a37b29ed98..343cb4414172 100644
--- a/src/common/admin_socket.cc
+++ b/src/common/admin_socket.cc
@@ -749,21 +749,22 @@ static std::string strsignal_compat(int signal) {
 }
 
 class RaiseHook: public AdminSocketHook {
+  using clock = ceph::coarse_mono_clock;
   struct Killer {
     CephContext* m_cct;
     pid_t pid;
     int signal;
-    ceph::coarse_mono_clock::time_point due;
+    clock::time_point due;
 
     std::string describe()
     {
       using std::chrono::duration_cast;
       using std::chrono::seconds;
-      auto remaining = (due - coarse_mono_clock::now());
+      auto remaining = (due - clock::now());
       return fmt::format(
         "pending signal ({}) due in {}", 
         strsignal_compat(signal),
-        duration_cast<seconds>(remaining));
+        duration_cast<seconds>(remaining).count());
     }
 
     bool cancel()
@@ -801,7 +802,7 @@ class RaiseHook: public AdminSocketHook {
     static std::optional<Killer> fork(CephContext *m_cct, int signal_to_send, double delay) {
 #   ifndef WIN32
       pid_t victim = getpid();
-      auto until = ceph::coarse_mono_clock::now() + ceph::make_timespan(delay);
+      clock::time_point until = clock::now() + ceph::make_timespan(delay);
 
       int fresult = ::fork();
       if (fresult < 0) {
@@ -814,19 +815,24 @@ class RaiseHook: public AdminSocketHook {
         return {{m_cct, fresult, signal_to_send, until}};
       }
 
-      const auto poll_interval = ceph::make_timespan(0.1);
-      auto remaining = (until - ceph::coarse_mono_clock::now());
-      do {
-        using std::chrono::duration_cast;
-        using std::chrono::nanoseconds;
-        std::this_thread::sleep_for(duration_cast<nanoseconds>(std::min(remaining, poll_interval)));
-        if (getppid() != victim) {
-          // suicide if my parent has changed
-          // this means that the original parent process has terminated
-          _exit(1);
+      const ceph::signedspan poll_interval = ceph::make_timespan(0.1);
+      while (getppid() == victim) {
+        ceph::signedspan remaining = (until - clock::now());
+        if (remaining.count() > 0) {
+          using std::chrono::duration_cast;
+          using std::chrono::nanoseconds;
+          std::this_thread::sleep_for(duration_cast<nanoseconds>(std::min(remaining, poll_interval)));
+        } else {
+          break;
         }
-        remaining = (until - ceph::coarse_mono_clock::now());
-      } while (remaining > ceph::signedspan::zero());
+      }
+
+      if (getppid() != victim) {
+        // suicide if my parent has changed
+        // this means that the original parent process has terminated
+        ldout(m_cct, 5) << __func__ << "my parent isn't what it used to be, i'm out" << strerror(errno) << dendl;
+        _exit(1);
+      }
 
       int status = kill(victim, signal_to_send);
       if (0 != status) {
@@ -846,10 +852,10 @@ class RaiseHook: public AdminSocketHook {
     int result = 0;
     std::transform(sigdesc.begin(), sigdesc.end(), sigdesc.begin(),
         [](unsigned char c) { return std::toupper(c); });
-    if (sigdesc.starts_with("-")) {
+    if (0 == sigdesc.find("-")) {
       sigdesc.erase(0, 1);
     }
-    if (sigdesc.starts_with("SIG")) {
+    if (0 == sigdesc.find("SIG")) {
       sigdesc.erase(0, 3);
     }
 
diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc
index 03bbb72b2a9a..a16a0cbb1f15 100644
--- a/src/test/admin_socket.cc
+++ b/src/test/admin_socket.cc
@@ -19,6 +19,7 @@
 #include "common/ceph_argparse.h"
 #include "json_spirit/json_spirit.h"
 #include "gtest/gtest.h"
+#include "fmt/format.h"
 
 #include <stdint.h>
 #include <string.h>

From e200499bb3c5703862b92a4d7fb534d98601f1bf Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Tue, 5 Sep 2023 14:56:15 -0400
Subject: [PATCH 1379/2492] rgw/multisite: maintain endpoints connectable
 status and retry the requests to them when appropriate

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/driver/rados/rgw_data_sync.cc | 150 +++++++------
 src/rgw/driver/rados/rgw_rados.cc     | 113 ++++++----
 src/rgw/driver/rados/rgw_sync.cc      | 170 +++++++++------
 src/rgw/rgw_coroutine.cc              |  23 +-
 src/rgw/rgw_coroutine.h               |   7 +-
 src/rgw/rgw_cr_rest.h                 |  20 +-
 src/rgw/rgw_http_client.cc            |   1 +
 src/rgw/rgw_http_client.h             |   5 +
 src/rgw/rgw_rest_conn.cc              | 296 +++++++++++++++++++-------
 src/rgw/rgw_rest_conn.h               |  19 ++
 10 files changed, 548 insertions(+), 256 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index 7b273e681293..0a7d21a72778 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -233,6 +233,9 @@ class RGWReadRemoteDataLogShardInfoCR : public RGWCoroutine {
   int shard_id;
   RGWDataChangesLogInfo *shard_info;
 
+  int tries{0};
+  int op_ret{0};
+
 public:
   RGWReadRemoteDataLogShardInfoCR(RGWDataSyncCtx *_sc,
                                   int _shard_id, RGWDataChangesLogInfo *_shard_info) : RGWCoroutine(_sc->cct),
@@ -243,41 +246,48 @@ class RGWReadRemoteDataLogShardInfoCR : public RGWCoroutine {
                                                       shard_info(_shard_info) {
   }
 
-  ~RGWReadRemoteDataLogShardInfoCR() override {
-    if (http_op) {
-      http_op->put();
-    }
-  }
-
   int operate(const DoutPrefixProvider *dpp) override {
     reenter(this) {
-      yield {
-	char buf[16];
-	snprintf(buf, sizeof(buf), "%d", shard_id);
-        rgw_http_param_pair pairs[] = { { "type" , "data" },
-	                                { "id", buf },
-					{ "info" , NULL },
-	                                { NULL, NULL } };
+      static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+      for (tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+        ldpp_dout(dpp, 20) << "read remote datalog shard info. shard_id=" << shard_id << " retries=" << tries << dendl;
 
-        string p = "/admin/log/";
+        yield {
+          char buf[16];
+          snprintf(buf, sizeof(buf), "%d", shard_id);
+          rgw_http_param_pair pairs[] = { { "type" , "data" },
+                                          { "id", buf },
+                                          { "info" , NULL },
+                                          { NULL, NULL } };
 
-        http_op = new RGWRESTReadResource(sc->conn, p, pairs, NULL, sync_env->http_manager);
+          string p = "/admin/log/";
 
-        init_new_io(http_op);
+          http_op = new RGWRESTReadResource(sc->conn, p, pairs, NULL, sync_env->http_manager);
 
-        int ret = http_op->aio_read(dpp);
-        if (ret < 0) {
-          ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl;
-          log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
-          return set_cr_error(ret);
+          init_new_io(http_op);
+
+          int ret = http_op->aio_read(dpp);
+          if (ret < 0) {
+            ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl;
+            log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+            http_op->put();
+            return set_cr_error(ret);
+          }
+
+          return io_block(0);
+        }
+        yield {
+          op_ret = http_op->wait(shard_info, null_yield);
+          http_op->put();
         }
 
-        return io_block(0);
-      }
-      yield {
-        int ret = http_op->wait(shard_info, null_yield);
-        if (ret < 0) {
-          return set_cr_error(ret);
+        if (op_ret < 0) {
+          if (op_ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+            ldpp_dout(dpp, 20) << "failed to fetch remote datalog shard info. retry. shard_id=" << shard_id << dendl;
+            continue;
+          } else {
+            return set_cr_error(op_ret);
+          }
         }
         return set_cr_done();
       }
@@ -315,6 +325,9 @@ class RGWReadRemoteDataLogShardCR : public RGWCoroutine {
   read_remote_data_log_response response;
   std::optional<TOPNSPC::common::PerfGuard> timer;
 
+  int tries{0};
+  int op_ret{0};
+
 public:
   RGWReadRemoteDataLogShardCR(RGWDataSyncCtx *_sc, int _shard_id,
                               const std::string& marker, string *pnext_marker,
@@ -324,53 +337,62 @@ class RGWReadRemoteDataLogShardCR : public RGWCoroutine {
       shard_id(_shard_id), marker(marker), pnext_marker(pnext_marker),
       entries(_entries), truncated(_truncated) {
   }
-  ~RGWReadRemoteDataLogShardCR() override {
-    if (http_op) {
-      http_op->put();
-    }
-  }
 
   int operate(const DoutPrefixProvider *dpp) override {
     reenter(this) {
-      yield {
-	char buf[16];
-	snprintf(buf, sizeof(buf), "%d", shard_id);
-        rgw_http_param_pair pairs[] = { { "type" , "data" },
-	                                { "id", buf },
-	                                { "marker", marker.c_str() },
-	                                { "extra-info", "true" },
-	                                { NULL, NULL } };
+      static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+      for (tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+        ldpp_dout(dpp, 20) << "read remote datalog shard. shard_id=" << shard_id << " retries=" << tries << dendl;
 
-        string p = "/admin/log/";
+        yield {
+          char buf[16];
+          snprintf(buf, sizeof(buf), "%d", shard_id);
+          rgw_http_param_pair pairs[] = { { "type" , "data" },
+                                          { "id", buf },
+                                          { "marker", marker.c_str() },
+                                          { "extra-info", "true" },
+                                          { NULL, NULL } };
 
-        http_op = new RGWRESTReadResource(sc->conn, p, pairs, NULL, sync_env->http_manager);
+          string p = "/admin/log/";
 
-        init_new_io(http_op);
+          http_op = new RGWRESTReadResource(sc->conn, p, pairs, NULL, sync_env->http_manager);
+
+          init_new_io(http_op);
 
-        if (sync_env->counters) {
-          timer.emplace(sync_env->counters, sync_counters::l_poll);
-        }
-        int ret = http_op->aio_read(dpp);
-        if (ret < 0) {
-          ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl;
-          log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
           if (sync_env->counters) {
-            sync_env->counters->inc(sync_counters::l_poll_err);
+            timer.emplace(sync_env->counters, sync_counters::l_poll);
+          }
+          int ret = http_op->aio_read(dpp);
+          if (ret < 0) {
+            ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl;
+            log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+            if (sync_env->counters) {
+              sync_env->counters->inc(sync_counters::l_poll_err);
+            }
+            http_op->put();
+            return set_cr_error(ret);
           }
-          return set_cr_error(ret);
+
+          return io_block(0);
+        }
+        yield {
+          timer.reset();
+          op_ret = http_op->wait(&response, null_yield);
+          http_op->put();
         }
 
-        return io_block(0);
-      }
-      yield {
-        timer.reset();
-        int ret = http_op->wait(&response, null_yield);
-        if (ret < 0) {
-          if (sync_env->counters && ret != -ENOENT) {
-            sync_env->counters->inc(sync_counters::l_poll_err);
+        if (op_ret < 0) {
+          if (op_ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+            ldpp_dout(dpp, 20) << "failed to read remote datalog shard. retry. shard_id=" << shard_id << dendl;
+            continue;
+          } else {
+            if (sync_env->counters && op_ret != -ENOENT) {
+              sync_env->counters->inc(sync_counters::l_poll_err);
+            }
+            return set_cr_error(op_ret);
           }
-          return set_cr_error(ret);
         }
+
         entries->clear();
         entries->swap(response.entries);
         *pnext_marker = response.marker;
@@ -421,6 +443,8 @@ bool RGWReadRemoteDataLogInfoCR::spawn_next() {
 }
 
 class RGWListRemoteDataLogShardCR : public RGWSimpleCoroutine {
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+
   RGWDataSyncCtx *sc;
   RGWDataSyncEnv *sync_env;
   RGWRESTReadResource *http_op;
@@ -434,7 +458,7 @@ class RGWListRemoteDataLogShardCR : public RGWSimpleCoroutine {
   RGWListRemoteDataLogShardCR(RGWDataSyncCtx *sc, int _shard_id,
                               const string& _marker, uint32_t _max_entries,
                               rgw_datalog_shard_data *_result)
-    : RGWSimpleCoroutine(sc->cct), sc(sc), sync_env(sc->env), http_op(NULL),
+    : RGWSimpleCoroutine(sc->cct, NUM_ENPOINT_IOERROR_RETRIES), sc(sc), sync_env(sc->env), http_op(NULL),
       shard_id(_shard_id), marker(_marker), max_entries(_max_entries), result(_result) {}
 
   int send_request(const DoutPrefixProvider *dpp) override {
@@ -474,7 +498,7 @@ class RGWListRemoteDataLogShardCR : public RGWSimpleCoroutine {
     int ret = http_op->wait(result, null_yield);
     http_op->put();
     if (ret < 0 && ret != -ENOENT) {
-      ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to list remote datalog shard, ret=" << ret << dendl;
+      ldpp_dout(sync_env->dpp, 5) << "ERROR: failed to list remote datalog shard, ret=" << ret << dendl;
       return ret;
     }
     return 0;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 93e8840eafdb..1f42c1b8f039 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4028,19 +4028,28 @@ int RGWRados::stat_remote_obj(const DoutPrefixProvider *dpp,
   constexpr bool sync_manifest = true;
   constexpr bool skip_decrypt = true;
   constexpr bool sync_cloudtiered = true;
-  int ret = conn->get_obj(dpp, user_id, info, src_obj, pmod, unmod_ptr,
-                      dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
-                      prepend_meta, get_op, rgwx_stat,
-                      sync_manifest, skip_decrypt, nullptr, sync_cloudtiered,
-                      true, &cb, &in_stream_req);
-  if (ret < 0) {
-    return ret;
-  }
 
-  ret = conn->complete_request(in_stream_req, nullptr, &set_mtime, psize,
-                               nullptr, pheaders, y);
-  if (ret < 0) {
-    return ret;
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    int ret = conn->get_obj(dpp, user_id, info, src_obj, pmod, unmod_ptr,
+                        dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
+                        prepend_meta, get_op, rgwx_stat,
+                        sync_manifest, skip_decrypt, nullptr, sync_cloudtiered,
+                        true, &cb, &in_stream_req);
+    if (ret < 0) {
+      return ret;
+    }
+
+    ret = conn->complete_request(in_stream_req, nullptr, &set_mtime, psize,
+                                 nullptr, pheaders, y);
+    if (ret < 0) {
+      if (ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to fetch object from remote. retries=" << tries << dendl;
+        continue;
+      }
+      return ret;
+    }
+    break;
   }
 
   bufferlist& extra_data_bl = cb.get_extra_data();
@@ -4248,20 +4257,29 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   static constexpr bool sync_manifest = true;
   static constexpr bool skip_decrypt = true;
   static constexpr bool sync_cloudtiered = true;
-  ret = conn->get_obj(rctx.dpp, user_id, info, src_obj, pmod, unmod_ptr,
-                      dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
-                      prepend_meta, get_op, rgwx_stat,
-                      sync_manifest, skip_decrypt, &dst_zone_trace,
-                      sync_cloudtiered, true,
-                      &cb, &in_stream_req);
-  if (ret < 0) {
-    goto set_err_state;
-  }
 
-  ret = conn->complete_request(in_stream_req, &etag, &set_mtime,
-                               &accounted_size, nullptr, nullptr, rctx.y);
-  if (ret < 0) {
-    goto set_err_state;
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    ret = conn->get_obj(rctx.dpp, user_id, info, src_obj, pmod, unmod_ptr,
+                        dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
+                        prepend_meta, get_op, rgwx_stat,
+                        sync_manifest, skip_decrypt, &dst_zone_trace,
+                        sync_cloudtiered, true,
+                        &cb, &in_stream_req);
+    if (ret < 0) {
+      goto set_err_state;
+    }
+
+    ret = conn->complete_request(in_stream_req, &etag, &set_mtime,
+                                 &accounted_size, nullptr, nullptr, rctx.y);
+    if (ret < 0) {
+      if (ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(rctx.dpp, 20) << __func__  << "(): failed to fetch object from remote. retries=" << tries << dendl;
+        continue;
+      }
+      goto set_err_state;
+    }
+    break;
   }
   ret = cb.flush();
   if (ret < 0) {
@@ -4495,28 +4513,37 @@ int RGWRados::copy_obj_to_remote_dest(const DoutPrefixProvider *dpp,
 
   auto rest_master_conn = svc.zone->get_master_conn();
 
-  int ret = rest_master_conn->put_obj_async_init(dpp, user_id, dest_obj, src_attrs, &out_stream_req);
-  if (ret < 0) {
-    return ret;
-  }
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    int ret = rest_master_conn->put_obj_async_init(dpp, user_id, dest_obj, src_attrs, &out_stream_req);
+    if (ret < 0) {
+      return ret;
+    }
 
-  out_stream_req->set_send_length(astate->size);
+    out_stream_req->set_send_length(astate->size);
 
-  ret = RGWHTTP::send(out_stream_req);
-  if (ret < 0) {
-    delete out_stream_req;
-    return ret;
-  }
+    ret = RGWHTTP::send(out_stream_req);
+    if (ret < 0) {
+      delete out_stream_req;
+      return ret;
+    }
 
-  ret = read_op.iterate(dpp, 0, astate->size - 1, out_stream_req->get_out_cb(), y);
-  if (ret < 0) {
-    delete out_stream_req;
-    return ret;
-  }
+    ret = read_op.iterate(dpp, 0, astate->size - 1, out_stream_req->get_out_cb(), y);
+    if (ret < 0) {
+      delete out_stream_req;
+      return ret;
+    }
 
-  ret = rest_master_conn->complete_request(out_stream_req, etag, mtime, y);
-  if (ret < 0)
-    return ret;
+    ret = rest_master_conn->complete_request(out_stream_req, etag, mtime, y);
+    if (ret < 0) {
+      if (ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to put_obj_async_init. retries=" << tries << dendl;
+        continue;
+      }
+      return ret;
+    }
+    break;
+  }
 
   return 0;
 }
diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc
index 251e9b395e0c..c5ea9f99ec5f 100644
--- a/src/rgw/driver/rados/rgw_sync.cc
+++ b/src/rgw/driver/rados/rgw_sync.cc
@@ -461,6 +461,9 @@ class RGWReadRemoteMDLogShardInfoCR : public RGWCoroutine {
   int shard_id;
   RGWMetadataLogInfo *shard_info;
 
+  int tries{0};
+  int op_ret{0};
+
 public:
   RGWReadRemoteMDLogShardInfoCR(RGWMetaSyncEnv *env, const std::string& period,
                                 int _shard_id, RGWMetadataLogInfo *_shard_info)
@@ -471,37 +474,48 @@ class RGWReadRemoteMDLogShardInfoCR : public RGWCoroutine {
     auto store = env->store;
     RGWRESTConn *conn = store->svc()->zone->get_master_conn();
     reenter(this) {
-      yield {
-	char buf[16];
-	snprintf(buf, sizeof(buf), "%d", shard_id);
-        rgw_http_param_pair pairs[] = { { "type" , "metadata" },
-	                                { "id", buf },
-	                                { "period", period.c_str() },
-					{ "info" , NULL },
-	                                { NULL, NULL } };
-
-        string p = "/admin/log/";
-
-        http_op = new RGWRESTReadResource(conn, p, pairs, NULL,
-                                          env->http_manager);
+      static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+      for (tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+        ldpp_dout(dpp, 20) << "read remote metadata log shard info. shard_is=" << shard_id << " retries=" << tries << dendl;
 
-        init_new_io(http_op);
+        yield {
+          char buf[16];
+          snprintf(buf, sizeof(buf), "%d", shard_id);
+          rgw_http_param_pair pairs[] = { { "type" , "metadata" },
+                                          { "id", buf },
+                                          { "period", period.c_str() },
+                                          { "info" , NULL },
+                                          { NULL, NULL } };
+
+          string p = "/admin/log/";
+
+          http_op = new RGWRESTReadResource(conn, p, pairs, NULL,
+                                            env->http_manager);
+
+          init_new_io(http_op);
+
+          int ret = http_op->aio_read(dpp);
+          if (ret < 0) {
+            ldpp_dout(env->dpp, 0) << "ERROR: failed to read from " << p << dendl;
+            log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+            http_op->put();
+            return set_cr_error(ret);
+          }
 
-        int ret = http_op->aio_read(dpp);
-        if (ret < 0) {
-          ldpp_dout(env->dpp, 0) << "ERROR: failed to read from " << p << dendl;
-          log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+          return io_block(0);
+        }
+        yield {
+          op_ret = http_op->wait(shard_info, null_yield);
           http_op->put();
-          return set_cr_error(ret);
         }
 
-        return io_block(0);
-      }
-      yield {
-        int ret = http_op->wait(shard_info, null_yield);
-        http_op->put();
-        if (ret < 0) {
-          return set_cr_error(ret);
+        if (op_ret < 0) {
+          if (op_ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+            ldpp_dout(dpp, 20) << "failed to read remote metadata log shard info. retry. shard_id=" << shard_id << dendl;
+            continue;
+          } else {
+            return set_cr_error(op_ret);
+          }
         }
         return set_cr_done();
       }
@@ -519,6 +533,8 @@ RGWCoroutine* create_read_remote_mdlog_shard_info_cr(RGWMetaSyncEnv *env,
 }
 
 class RGWListRemoteMDLogShardCR : public RGWSimpleCoroutine {
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+
   RGWMetaSyncEnv *sync_env;
   RGWRESTReadResource *http_op;
 
@@ -532,7 +548,7 @@ class RGWListRemoteMDLogShardCR : public RGWSimpleCoroutine {
   RGWListRemoteMDLogShardCR(RGWMetaSyncEnv *env, const std::string& period,
                             int _shard_id, const string& _marker, uint32_t _max_entries,
                             rgw_mdlog_shard_data *_result)
-    : RGWSimpleCoroutine(env->store->ctx()), sync_env(env), http_op(NULL),
+    : RGWSimpleCoroutine(env->store->ctx(), NUM_ENPOINT_IOERROR_RETRIES), sync_env(env), http_op(NULL),
       period(period), shard_id(_shard_id), marker(_marker), max_entries(_max_entries), result(_result) {}
 
   int send_request(const DoutPrefixProvider *dpp) override {
@@ -573,7 +589,7 @@ class RGWListRemoteMDLogShardCR : public RGWSimpleCoroutine {
     int ret = http_op->wait(result, null_yield);
     http_op->put();
     if (ret < 0 && ret != -ENOENT) {
-      ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to list remote mdlog shard, ret=" << ret << dendl;
+      ldpp_dout(sync_env->dpp, 5) << "ERROR: failed to list remote mdlog shard, ret=" << ret << dendl;
       return ret;
     }
     return 0;
@@ -1027,6 +1043,9 @@ class RGWReadRemoteMetadataCR : public RGWCoroutine {
 
   RGWSyncTraceNodeRef tn;
 
+  int tries{0};
+  int op_ret{0};
+
 public:
   RGWReadRemoteMetadataCR(RGWMetaSyncEnv *_sync_env,
                                                       const string& _section, const string& _key, bufferlist *_pbl,
@@ -1034,7 +1053,7 @@ class RGWReadRemoteMetadataCR : public RGWCoroutine {
                                                       http_op(NULL),
                                                       section(_section),
                                                       key(_key),
-						      pbl(_pbl) {
+                                                      pbl(_pbl) {
     tn = sync_env->sync_tracer->add_node(_tn_parent, "read_remote_meta",
                                          section + ":" + key);
   }
@@ -1042,33 +1061,44 @@ class RGWReadRemoteMetadataCR : public RGWCoroutine {
   int operate(const DoutPrefixProvider *dpp) override {
     RGWRESTConn *conn = sync_env->conn;
     reenter(this) {
-      yield {
-        string key_encode;
-        url_encode(key, key_encode);
-        rgw_http_param_pair pairs[] = { { "key" , key.c_str()},
-	                                { NULL, NULL } };
+      static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+      for (tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+        ldpp_dout(dpp, 20) << "read remote metadata.  retries=" << tries << dendl;
 
-        string p = string("/admin/metadata/") + section + "/" + key_encode;
+        yield {
+          string key_encode;
+          url_encode(key, key_encode);
+          rgw_http_param_pair pairs[] = { { "key" , key.c_str()},
+                                          { NULL, NULL } };
 
-        http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager);
+          string p = string("/admin/metadata/") + section + "/" + key_encode;
 
-        init_new_io(http_op);
+          http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager);
 
-        int ret = http_op->aio_read(dpp);
-        if (ret < 0) {
-          ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl;
-          log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+          init_new_io(http_op);
+
+          int ret = http_op->aio_read(dpp);
+          if (ret < 0) {
+            ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl;
+            log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl;
+            http_op->put();
+            return set_cr_error(ret);
+          }
+
+          return io_block(0);
+        }
+        yield {
+          op_ret = http_op->wait(pbl, null_yield);
           http_op->put();
-          return set_cr_error(ret);
         }
 
-        return io_block(0);
-      }
-      yield {
-        int ret = http_op->wait(pbl, null_yield);
-        http_op->put();
-        if (ret < 0) {
-          return set_cr_error(ret);
+        if (op_ret < 0) {
+          if (op_ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+            ldpp_dout(dpp, 20) << "failed to read remote metadata. retry. section=" << section << " key=" << key << dendl;
+            continue;
+          } else {
+            return set_cr_error(op_ret);
+          }
         }
         return set_cr_done();
       }
@@ -1365,6 +1395,9 @@ class RGWCloneMetaLogCoroutine : public RGWCoroutine {
   RGWMetadataLogInfo shard_info;
   rgw_mdlog_shard_data data;
 
+  int tries{0};
+  int op_ret{0};
+
 public:
   RGWCloneMetaLogCoroutine(RGWMetaSyncEnv *_sync_env, RGWMetadataLog* mdlog,
                            const std::string& period, int _id,
@@ -2392,14 +2425,27 @@ int RGWCloneMetaLogCoroutine::operate(const DoutPrefixProvider *dpp)
         ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": reading shard status complete" << dendl;
         return state_read_shard_status_complete();
       }
-      yield {
-        ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": sending rest request" << dendl;
-        return state_send_rest_request(dpp);
-      }
-      yield {
-        ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": receiving rest response" << dendl;
-        return state_receive_rest_response();
+
+      static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+      for (tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+        yield {
+          ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": sending rest request" << dendl;
+          return state_send_rest_request(dpp);
+        }
+        yield {
+          ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": receiving rest response" << dendl;
+          return state_receive_rest_response();
+        }
+
+        if (op_ret == -EIO && tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+          ldout(cct, 20) << __func__ << ": request IO error. retries=" << tries << dendl;
+          continue;
+        } else if (op_ret < 0) {
+          return set_cr_error(op_ret);
+        }
+        break;
       }
+
       yield {
         ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": storing mdlog entries" << dendl;
         return state_store_mdlog_entries();
@@ -2497,17 +2543,21 @@ int RGWCloneMetaLogCoroutine::state_send_rest_request(const DoutPrefixProvider *
 
 int RGWCloneMetaLogCoroutine::state_receive_rest_response()
 {
-  int ret = http_op->wait(&data, null_yield);
-  if (ret < 0) {
+  op_ret = http_op->wait(&data, null_yield);
+  if (op_ret < 0 && op_ret != -EIO) {
     error_stream << "http operation failed: " << http_op->to_str() << " status=" << http_op->get_http_status() << std::endl;
-    ldpp_dout(sync_env->dpp, 5) << "failed to wait for op, ret=" << ret << dendl;
+    ldpp_dout(sync_env->dpp, 5) << "failed to wait for op, ret=" << op_ret << dendl;
     http_op->put();
     http_op = NULL;
-    return set_cr_error(ret);
+    return set_cr_error(op_ret);
   }
   http_op->put();
   http_op = NULL;
 
+  if (op_ret == -EIO) {
+    return 0;
+  }
+
   ldpp_dout(sync_env->dpp, 20) << "remote mdlog, shard_id=" << shard_id << " num of shard entries: " << data.entries.size() << dendl;
 
   truncated = ((int)data.entries.size() == max_entries);
diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc
index a9c9c38e3bc3..9e69c0876ef9 100644
--- a/src/rgw/rgw_coroutine.cc
+++ b/src/rgw/rgw_coroutine.cc
@@ -1077,8 +1077,21 @@ int RGWSimpleCoroutine::operate(const DoutPrefixProvider *dpp)
   int ret = 0;
   reenter(this) {
     yield return state_init();
-    yield return state_send_request(dpp);
-    yield return state_request_complete();
+
+    for (tries = 0; tries < max_eio_retries; tries++) {
+      yield return state_send_request(dpp);
+      yield return state_request_complete();
+
+      if (op_ret == -EIO && tries < max_eio_retries - 1) {
+        ldout(cct, 20) << "request IO error. retries=" << tries << dendl;
+        continue;
+      } else if (op_ret < 0) {
+        call_cleanup();
+        return set_state(RGWCoroutine_Error, op_ret);
+      }
+      break;
+    }
+
     yield return state_all_complete();
     drain_all();
     call_cleanup();
@@ -1109,10 +1122,10 @@ int RGWSimpleCoroutine::state_send_request(const DoutPrefixProvider *dpp)
 
 int RGWSimpleCoroutine::state_request_complete()
 {
-  int ret = request_complete();
-  if (ret < 0) {
+  op_ret = request_complete();
+  if (op_ret < 0 && op_ret != -EIO) {
     call_cleanup();
-    return set_state(RGWCoroutine_Error, ret);
+    return set_state(RGWCoroutine_Error, op_ret);
   }
   return 0;
 }
diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h
index 9c0915a3b1e4..44ac256f0c47 100644
--- a/src/rgw/rgw_coroutine.h
+++ b/src/rgw/rgw_coroutine.h
@@ -700,6 +700,10 @@ RGWAioCompletionNotifier *RGWCoroutinesStack::create_completion_notifier(T value
 
 class RGWSimpleCoroutine : public RGWCoroutine {
   bool called_cleanup;
+  const int max_eio_retries;
+
+  int tries{0};
+  int op_ret{0};
 
   int operate(const DoutPrefixProvider *dpp) override;
 
@@ -711,7 +715,8 @@ class RGWSimpleCoroutine : public RGWCoroutine {
   void call_cleanup();
 
 public:
-  RGWSimpleCoroutine(CephContext *_cct) : RGWCoroutine(_cct), called_cleanup(false) {}
+  RGWSimpleCoroutine(CephContext *_cct) : RGWCoroutine(_cct), called_cleanup(false), max_eio_retries(1) {}
+  RGWSimpleCoroutine(CephContext *_cct, const int _max_eio_retries) : RGWCoroutine(_cct), called_cleanup(false), max_eio_retries(_max_eio_retries) {}
   virtual ~RGWSimpleCoroutine() override;
 
   virtual int init() { return 0; }
diff --git a/src/rgw/rgw_cr_rest.h b/src/rgw/rgw_cr_rest.h
index ba47c3dd622c..531d7ee68ef8 100644
--- a/src/rgw/rgw_cr_rest.h
+++ b/src/rgw/rgw_cr_rest.h
@@ -24,6 +24,8 @@ struct rgw_rest_obj {
 };
 
 class RGWReadRawRESTResourceCR : public RGWSimpleCoroutine {
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+
   bufferlist *result;
  protected:
   RGWRESTConn *conn;
@@ -36,21 +38,21 @@ class RGWReadRawRESTResourceCR : public RGWSimpleCoroutine {
   RGWReadRawRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
                            RGWHTTPManager *_http_manager, const std::string& _path,
                            rgw_http_param_pair *params, bufferlist *_result)
-    : RGWSimpleCoroutine(_cct), result(_result), conn(_conn), http_manager(_http_manager),
+    : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), result(_result), conn(_conn), http_manager(_http_manager),
     path(_path), params(make_param_list(params))
   {}
 
  RGWReadRawRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
                           RGWHTTPManager *_http_manager, const std::string& _path,
                           rgw_http_param_pair *params)
-   : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+   : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
     path(_path), params(make_param_list(params))
   {}
 
   RGWReadRawRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
                            RGWHTTPManager *_http_manager, const std::string& _path,
                            rgw_http_param_pair *params, param_vec_t &hdrs)
-    : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+    : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
       path(_path), params(make_param_list(params)),
       extra_headers(hdrs)
   {}
@@ -59,7 +61,7 @@ class RGWReadRawRESTResourceCR : public RGWSimpleCoroutine {
                           RGWHTTPManager *_http_manager, const std::string& _path,
                           rgw_http_param_pair *params,
                           std::map <std::string, std::string> *hdrs)
-   : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+   : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
     path(_path), params(make_param_list(params)),
     extra_headers(make_param_list(hdrs))
     {}
@@ -144,6 +146,8 @@ class RGWReadRESTResourceCR : public RGWReadRawRESTResourceCR {
 
 template <class T, class E = int>
 class RGWSendRawRESTResourceCR: public RGWSimpleCoroutine {
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+
  protected:
   RGWRESTConn *conn;
   RGWHTTPManager *http_manager;
@@ -167,7 +171,7 @@ class RGWSendRawRESTResourceCR: public RGWSimpleCoroutine {
                           bufferlist& _input, T *_result,
                           bool _send_content_length,
                           E *_err_result = nullptr)
-   : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+   : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
      method(_method), path(_path), params(make_param_list(_params)),
      headers(make_param_list(_attrs)), attrs(_attrs),
      result(_result), err_result(_err_result),
@@ -178,7 +182,7 @@ class RGWSendRawRESTResourceCR: public RGWSimpleCoroutine {
                           const std::string& _method, const std::string& _path,
                           rgw_http_param_pair *_params, std::map<std::string, std::string> *_attrs,
                           T *_result, E *_err_result = nullptr)
-   : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+   : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
     method(_method), path(_path), params(make_param_list(_params)), headers(make_param_list(_attrs)), attrs(_attrs), result(_result),
     err_result(_err_result) {}
 
@@ -321,6 +325,8 @@ class RGWPutRESTResourceCR : public RGWSendRESTResourceCR<S, T, E> {
 };
 
 class RGWDeleteRESTResourceCR : public RGWSimpleCoroutine {
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+
   RGWRESTConn *conn;
   RGWHTTPManager *http_manager;
   std::string path;
@@ -333,7 +339,7 @@ class RGWDeleteRESTResourceCR : public RGWSimpleCoroutine {
                         RGWHTTPManager *_http_manager,
                         const std::string& _path,
                         rgw_http_param_pair *_params)
-    : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
+    : RGWSimpleCoroutine(_cct, NUM_ENPOINT_IOERROR_RETRIES), conn(_conn), http_manager(_http_manager),
       path(_path), params(make_param_list(_params))
   {}
 
diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index aacc3f40b82c..587b33d04780 100644
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -306,6 +306,7 @@ RGWHTTPClient::RGWHTTPClient(CephContext *cct,
       verify_ssl(cct->_conf->rgw_verify_ssl),
       cct(cct),
       method(_method),
+      url_orig(_url),
       url(_url) {
   init();
 }
diff --git a/src/rgw/rgw_http_client.h b/src/rgw/rgw_http_client.h
index dbd705a1880d..ef1883903266 100644
--- a/src/rgw/rgw_http_client.h
+++ b/src/rgw/rgw_http_client.h
@@ -52,6 +52,7 @@ class RGWHTTPClient : public RGWIOProvider,
   CephContext *cct;
 
   std::string method;
+  std::string url_orig;
   std::string url;
 
   std::string protocol;
@@ -166,6 +167,10 @@ class RGWHTTPClient : public RGWIOProvider,
     url = _url;
   }
 
+  const std::string& get_url_orig() const {
+    return url_orig;
+  }
+
   void set_method(const std::string& _method) {
     method = _method;
   }
diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 235de2b7363f..2c1ff1a0a3f0 100644
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -21,6 +21,12 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Driver* driver,
     api_name(_api_name),
     host_style(_host_style)
 {
+  endpoints_status.reserve(remote_endpoints.size());
+  std::for_each(remote_endpoints.begin(), remote_endpoints.end(),
+                [this](const auto& url) {
+                  this->endpoints_status.emplace(url, ceph::real_clock::zero());
+                });
+
   if (driver) {
     key = driver->get_zone()->get_system_key();
     self_zone_group = driver->get_zone()->get_zonegroup().get_id();
@@ -42,11 +48,17 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct,
     api_name(_api_name),
     host_style(_host_style)
 {
+  endpoints_status.reserve(remote_endpoints.size());
+  std::for_each(remote_endpoints.begin(), remote_endpoints.end(),
+                [this](const auto& url) {
+                  this->endpoints_status.emplace(url, ceph::real_clock::zero());
+                });
 }
 
 RGWRESTConn::RGWRESTConn(RGWRESTConn&& other)
   : cct(other.cct),
     endpoints(std::move(other.endpoints)),
+    endpoints_status(std::move(other.endpoints_status)),
     key(std::move(other.key)),
     self_zone_group(std::move(other.self_zone_group)),
     remote_id(std::move(other.remote_id)),
@@ -58,6 +70,7 @@ RGWRESTConn& RGWRESTConn::operator=(RGWRESTConn&& other)
 {
   cct = other.cct;
   endpoints = std::move(other.endpoints);
+  endpoints_status = std::move(other.endpoints_status);
   key = std::move(other.key);
   self_zone_group = std::move(other.self_zone_group);
   remote_id = std::move(other.remote_id);
@@ -69,11 +82,47 @@ int RGWRESTConn::get_url(string& endpoint)
 {
   if (endpoints.empty()) {
     ldout(cct, 0) << "ERROR: endpoints not configured for upstream zone" << dendl;
-    return -EIO;
+    return -EINVAL;
   }
 
-  int i = ++counter;
-  endpoint = endpoints[i % endpoints.size()];
+  size_t num = 0;
+  while (num < endpoints.size()) {
+    int i = ++counter;
+    endpoint = endpoints[i % endpoints.size()];
+
+    if (endpoints_status.find(endpoint) == endpoints_status.end()) {
+      ldout(cct, 1) << "ERROR: missing status for endpoint " << endpoint << dendl;
+      num++;
+      continue;
+    }
+
+    const auto& upd_time = endpoints_status[endpoint].load();
+
+    if (ceph::real_clock::is_zero(upd_time)) {
+      break;
+    }
+
+    auto diff = ceph::to_seconds<double>(ceph::real_clock::now() - upd_time);
+
+    ldout(cct, 20) << "endpoint url=" << endpoint
+                   << " last endpoint status update time="
+                   << ceph::real_clock::to_double(upd_time)
+                   << " diff=" << diff << dendl;
+
+    static constexpr uint32_t CONN_STATUS_EXPIRE_SECS = 2;
+    if (diff >= CONN_STATUS_EXPIRE_SECS) {
+      endpoints_status[endpoint].store(ceph::real_clock::zero());
+      ldout(cct, 10) << "endpoint " << endpoint << " unconnectable status expired. mark it connectable" << dendl;
+      break;
+    }
+    num++;
+  };
+
+  if (num == endpoints.size()) {
+    ldout(cct, 5) << "ERROR: no valid endpoint" << dendl;
+    return -EINVAL;
+  }
+  ldout(cct, 20) << "get_url picked endpoint=" << endpoint << dendl;
 
   return 0;
 }
@@ -85,6 +134,19 @@ string RGWRESTConn::get_url()
   return endpoint;
 }
 
+void RGWRESTConn::set_url_unconnectable(const std::string& endpoint)
+{
+  if (endpoint.empty() || endpoints_status.find(endpoint) == endpoints_status.end()) {
+    ldout(cct, 0) << "ERROR: endpoint is not a valid or doesn't have status. endpoint="
+                  << endpoint << dendl;
+    return;
+  }
+
+  endpoints_status[endpoint].store(ceph::real_clock::now());
+
+  ldout(cct, 10) << "set endpoint unconnectable. url=" << endpoint << dendl;
+}
+
 void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
 {
   populate_uid(params, uid);
@@ -93,39 +155,67 @@ void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, cons
 
 int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 {
-  string url;
-  int ret = get_url(url);
-  if (ret < 0)
-    return ret;
-  param_vec_t params;
-  populate_params(params, &uid, self_zone_group);
-  if (objv) {
-    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
-    char buf[16];
-    snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
-    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "ver", buf));
-  }
-  RGWRESTSimpleRequest req(cct, info.method, url, NULL, &params, api_name);
-  return req.forward_request(dpp, key, info, max_response, inbl, outbl, y);
+  int ret = 0;
+
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    string url;
+    ret = get_url(url);
+    if (ret < 0)
+      return ret;
+    param_vec_t params;
+    populate_params(params, &uid, self_zone_group);
+    if (objv) {
+      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
+      char buf[16];
+      snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
+      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "ver", buf));
+    }
+    RGWRESTSimpleRequest req(cct, info.method, url, NULL, &params, api_name);
+    ret = req.forward_request(dpp, key, info, max_response, inbl, outbl, y);
+    if (ret == -EIO) {
+      set_url_unconnectable(url);
+      if (tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to forward request. retries=" << tries << dendl;
+        continue;
+      }
+    }
+    break;
+  }
+  return ret;
 }
 
 int RGWRESTConn::forward_iam_request(const DoutPrefixProvider *dpp, const req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 {
-  string url;
-  int ret = get_url(url);
-  if (ret < 0)
-    return ret;
-  param_vec_t params;
-  if (objv) {
-    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
-    char buf[16];
-    snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
-    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "ver", buf));
-  }
-  std::string service = "iam";
-  RGWRESTSimpleRequest req(cct, info.method, url, NULL, &params, api_name);
-  // coverity[uninit_use_in_call:SUPPRESS]
-  return req.forward_request(dpp, key, info, max_response, inbl, outbl, y, service);
+  int ret = 0;
+
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    string url;
+    ret = get_url(url);
+    if (ret < 0)
+      return ret;
+    param_vec_t params;
+    if (objv) {
+      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
+      char buf[16];
+      snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
+      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "ver", buf));
+    }
+    std::string service = "iam";
+    RGWRESTSimpleRequest req(cct, info.method, url, NULL, &params, api_name);
+    // coverity[uninit_use_in_call:SUPPRESS]
+    ret = req.forward_request(dpp, key, info, max_response, inbl, outbl, y, service);
+    if (ret == -EIO) {
+      set_url_unconnectable(url);
+      if (tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to forward request. retries=" << tries << dendl;
+        continue;
+      }
+    }
+    break;
+  }
+  return ret;
 }
 
 int RGWRESTConn::put_obj_send_init(const rgw_obj& obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req)
@@ -172,6 +262,11 @@ int RGWRESTConn::complete_request(RGWRESTStreamS3PutObj *req, string& etag,
                                   real_time *mtime, optional_yield y)
 {
   int ret = req->complete_request(y, &etag, mtime);
+  if (ret == -EIO) {
+    ldout(cct, 5) << __func__ << ": complete_request() returned ret=" << ret << dendl;
+    set_url_unconnectable(req->get_url_orig());
+  }
+
   delete req;
 
   return ret;
@@ -323,6 +418,10 @@ int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req,
                                   optional_yield y)
 {
   int ret = req->complete_request(y, etag, mtime, psize, pattrs, pheaders);
+  if (ret == -EIO) {
+    ldout(cct, 5) << __func__ << ": complete_request() returned ret=" << ret << dendl;
+    set_url_unconnectable(req->get_url_orig());
+  }
   delete req;
 
   return ret;
@@ -337,35 +436,53 @@ int RGWRESTConn::get_resource(const DoutPrefixProvider *dpp,
 		     RGWHTTPManager *mgr,
 		     optional_yield y)
 {
-  string url;
-  int ret = get_url(url);
-  if (ret < 0)
-    return ret;
+  int ret = 0;
 
-  param_vec_t params;
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    string url;
+    ret = get_url(url);
+    if (ret < 0)
+      return ret;
 
-  if (extra_params) {
-    params.insert(params.end(), extra_params->begin(), extra_params->end());
-  }
+    param_vec_t params;
 
-  populate_params(params, nullptr, self_zone_group);
+    if (extra_params) {
+      params.insert(params.end(), extra_params->begin(), extra_params->end());
+    }
 
-  RGWStreamIntoBufferlist cb(bl);
+    populate_params(params, nullptr, self_zone_group);
 
-  RGWRESTStreamReadRequest req(cct, url, &cb, NULL, &params, api_name, host_style);
+    RGWStreamIntoBufferlist cb(bl);
 
-  map<string, string> headers;
-  if (extra_headers) {
-    headers.insert(extra_headers->begin(), extra_headers->end());
-  }
+    RGWRESTStreamReadRequest req(cct, url, &cb, NULL, &params, api_name, host_style);
 
-  ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
-  if (ret < 0) {
-    ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
-    return ret;
+    map<string, string> headers;
+    if (extra_headers) {
+      headers.insert(extra_headers->begin(), extra_headers->end());
+    }
+
+    ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
+    if (ret < 0) {
+      ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
+      return ret;
+    }
+
+    ret = req.complete_request(y);
+    if (ret == -EIO) {
+      set_url_unconnectable(url);
+      if (tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to get resource. retries=" << tries << dendl;
+        continue;
+      }
+    }
+    if (ret < 0) {
+      ldpp_dout(dpp, 5) << __func__ << ": complete_request() returned ret=" << ret << dendl;
+    }
+    break;
   }
 
-  return req.complete_request(y);
+  return ret;
 }
 
 int RGWRESTConn::send_resource(const DoutPrefixProvider *dpp, const std::string& method,
@@ -373,37 +490,50 @@ int RGWRESTConn::send_resource(const DoutPrefixProvider *dpp, const std::string&
 		                std::map<std::string, std::string> *extra_headers, bufferlist& bl,
                         bufferlist *send_data, RGWHTTPManager *mgr, optional_yield y)
 {
-  std::string url;
-  int ret = get_url(url);
-  if (ret < 0)
-    return ret;
+  int ret = 0;
 
-  param_vec_t params;
+  static constexpr int NUM_ENPOINT_IOERROR_RETRIES = 20;
+  for (int tries = 0; tries < NUM_ENPOINT_IOERROR_RETRIES; tries++) {
+    std::string url;
+    ret = get_url(url);
+    if (ret < 0)
+      return ret;
 
-  if (extra_params) {
-    params = make_param_list(extra_params);
-  }
+    param_vec_t params;
 
-  populate_params(params, nullptr, self_zone_group);
+    if (extra_params) {
+      params = make_param_list(extra_params);
+    }
 
-  RGWStreamIntoBufferlist cb(bl);
+    populate_params(params, nullptr, self_zone_group);
 
-  RGWRESTStreamSendRequest req(cct, method, url, &cb, NULL, &params, api_name, host_style);
+    RGWStreamIntoBufferlist cb(bl);
 
-  std::map<std::string, std::string> headers;
-  if (extra_headers) {
-    headers.insert(extra_headers->begin(), extra_headers->end());
-  }
+    RGWRESTStreamSendRequest req(cct, method, url, &cb, NULL, &params, api_name, host_style);
 
-  ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
-  if (ret < 0) {
-    ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
-    return ret;
-  }
+    std::map<std::string, std::string> headers;
+    if (extra_headers) {
+      headers.insert(extra_headers->begin(), extra_headers->end());
+    }
 
-  ret = req.complete_request(y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 5) << __func__ << ": complete_request() resource=" << resource << " returned ret=" << ret << dendl;
+    ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
+    if (ret < 0) {
+      ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
+      return ret;
+    }
+
+    ret = req.complete_request(y);
+    if (ret == -EIO) {
+      set_url_unconnectable(url);
+      if (tries < NUM_ENPOINT_IOERROR_RETRIES - 1) {
+        ldpp_dout(dpp, 20) << __func__  << "(): failed to send resource. retries=" << tries << dendl;
+        continue;
+      }
+    }
+    if (ret < 0) {
+      ldpp_dout(dpp, 5) << __func__ << ": complete_request() resource=" << resource << " returned ret=" << ret << dendl;
+    }
+    break;
   }
 
   return ret;
@@ -451,7 +581,13 @@ int RGWRESTReadResource::read(const DoutPrefixProvider *dpp, optional_yield y)
     return ret;
   }
 
-  return req.complete_request(y);
+  ret = req.complete_request(y);
+  if (ret == -EIO) {
+    conn->set_url_unconnectable(req.get_url_orig());
+    ldpp_dout(dpp, 20) << __func__ << ": complete_request() returned ret=" << ret << dendl;
+  }
+
+  return ret;
 }
 
 int RGWRESTReadResource::aio_read(const DoutPrefixProvider *dpp)
@@ -512,7 +648,13 @@ int RGWRESTSendResource::send(const DoutPrefixProvider *dpp, bufferlist& outbl,
     return ret;
   }
 
-  return req.complete_request(y);
+  ret = req.complete_request(y);
+  if (ret == -EIO) {
+    conn->set_url_unconnectable(req.get_url_orig());
+    ldpp_dout(dpp, 20) << __func__ << ": complete_request() returned ret=" << ret << dendl;
+  }
+
+  return ret;
 }
 
 int RGWRESTSendResource::aio_send(const DoutPrefixProvider *dpp, bufferlist& outbl)
diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 26e71bef38a9..07ca83212cac 100644
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -67,8 +67,12 @@ inline param_vec_t make_param_list(const std::map<std::string, std::string> *pp)
 
 class RGWRESTConn
 {
+  /* the endpoint is not able to connect if the timestamp is not real_clock::zero */
+  using endpoint_status_map = std::unordered_map<std::string, std::atomic<ceph::real_time>>;
+
   CephContext *cct;
   std::vector<std::string> endpoints;
+  endpoint_status_map endpoints_status;
   RGWAccessKey key;
   std::string self_zone_group;
   std::string remote_id;
@@ -99,6 +103,7 @@ class RGWRESTConn
 
   int get_url(std::string& endpoint);
   std::string get_url();
+  void set_url_unconnectable(const std::string& endpoint);
   const std::string& get_self_zonegroup() {
     return self_zone_group;
   }
@@ -344,6 +349,9 @@ class RGWRESTReadResource : public RefCountedObject, public RGWIOProvider {
   int wait(bufferlist *pbl, optional_yield y) {
     int ret = req.wait(y);
     if (ret < 0) {
+      if (ret == -EIO) {
+        conn->set_url_unconnectable(req.get_url_orig());
+      }
       return ret;
     }
 
@@ -396,6 +404,9 @@ int RGWRESTReadResource::wait(T *dest, optional_yield y)
 {
   int ret = req.wait(y);
   if (ret < 0) {
+    if (ret == -EIO) {
+      conn->set_url_unconnectable(req.get_url_orig());
+    }
     return ret;
   }
 
@@ -467,6 +478,10 @@ class RGWRESTSendResource : public RefCountedObject, public RGWIOProvider {
     int ret = req.wait(y);
     *pbl = bl;
 
+    if (ret == -EIO) {
+      conn->set_url_unconnectable(req.get_url_orig());
+    }
+
     if (ret < 0 && err_result ) {
       ret = parse_decode_json(*err_result, bl);
     }
@@ -482,6 +497,10 @@ template <class T, class E>
 int RGWRESTSendResource::wait(T *dest, optional_yield y, E *err_result)
 {
   int ret = req.wait(y);
+  if (ret == -EIO) {
+    conn->set_url_unconnectable(req.get_url_orig());
+  }
+
   if (ret >= 0) {
     ret = req.get_status();
   }

From 06532d8545b59503e799904ab371dc7c7cec5f89 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 23 Aug 2023 09:58:47 +0800
Subject: [PATCH 1380/2492] crimson/osd/object_context: rename
 ObjectContext::list_hook to obc_accessing_hook

There's already a "list_hook" in the base class "intrusive_lru_base"

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/object_context.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/object_context.h b/src/crimson/osd/object_context.h
index 106968e60c3d..75c99bb54680 100644
--- a/src/crimson/osd/object_context.h
+++ b/src/crimson/osd/object_context.h
@@ -130,7 +130,7 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
     });
   }
 
-  boost::intrusive::list_member_hook<> list_hook;
+  boost::intrusive::list_member_hook<> obc_accessing_hook;
   uint64_t list_link_cnt = 0;
   bool fully_loaded = false;
 
@@ -154,7 +154,7 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
   using obc_accessing_option_t = boost::intrusive::member_hook<
     ObjectContext,
     boost::intrusive::list_member_hook<>,
-    &ObjectContext::list_hook>;
+    &ObjectContext::obc_accessing_hook>;
 
   template<RWState::State Type, typename InterruptCond = void, typename Func>
   auto with_lock(Func&& func) {

From 4eeffee677bc7010c47ab7c7e516083f47cdced9 Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Wed, 17 Jan 2024 11:39:53 +0800
Subject: [PATCH 1381/2492] crimson/osd: get SnapSetContext from head_obc

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/crimson/osd/pg_backend.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index 8c68301530b4..a3a777f848ad 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -808,7 +808,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
   return obc_loader.with_clone_obc_only<RWState::RWWRITE>(
     head, target_coid,
     [this, &os, &txn, &delta_stats, &osd_op_params, &snapid]
-    (auto, auto resolved_obc) {
+    (auto head_obc, auto resolved_obc) {
     if (resolved_obc->obs.oi.soid.is_head()) {
       // no-op: The resolved oid returned the head object
       logger().debug("PGBackend::rollback: loaded head_obc: {}"
@@ -846,7 +846,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
 
     // 3) Calculate clone_overlaps by following overlaps
     const auto& clone_overlap =
-      resolved_obc->ssc->snapset.clone_overlap;
+      head_obc->ssc->snapset.clone_overlap;
     auto iter = clone_overlap.lower_bound(snapid);
     ceph_assert(iter != clone_overlap.end());
     interval_set<uint64_t> overlaps = iter->second;

From 315616783a652f2e3880d1a32088b1bc4ad2d509 Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Fri, 12 Jan 2024 09:10:30 +0000
Subject: [PATCH 1382/2492] mon: add created_at and ceph_version_when_created
 meta

Fixes: https://tracker.ceph.com/issues/57515
Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 src/mon/Monitor.cc       | 13 +++++++++++++
 src/mon/MonitorDBStore.h | 14 +++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 5543ff316fff..a0fe3fa038e6 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2377,6 +2377,19 @@ void Monitor::collect_metadata(Metadata *m)
   for (auto& i : errs) {
     dout(1) << __func__ << " " << i.first << ": " << i.second << dendl;
   }
+
+  string ceph_version_when_created;
+  int r = store->read_meta("ceph_version_when_created", &ceph_version_when_created);
+  if (r < 0 || ceph_version_when_created.empty()) {
+    ceph_version_when_created = "";
+  }
+  (*m)["ceph_version_when_created"] = ceph_version_when_created;
+  string created_at;
+  r = store->read_meta("created_at", &created_at);
+  if (r < 0 || created_at.empty()) {
+    created_at = "";
+  }
+  (*m)["created_at"] = created_at;
 }
 
 void Monitor::finish_election()
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index b7dfc50b0a17..a0718f439b64 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -31,6 +31,7 @@
 #include "common/safe_io.h"
 #include "common/blkdev.h"
 #include "common/PriorityCache.h"
+#include "common/version.h"
 
 #define dout_context g_ceph_context
 
@@ -698,9 +699,20 @@ class MonitorDBStore
   }
 
   int create_and_open(std::ostream &out) {
+    int r = write_meta("ceph_version_when_created", pretty_version_to_str());
+    if (r < 0)
+      return r;
+
+    std::ostringstream created_at;
+    utime_t now = ceph_clock_now();
+    now.gmtime(created_at);
+    r = write_meta("created_at", created_at.str());
+    if (r < 0)
+      return r;
+
     // record the type before open
     std::string kv_type;
-    int r = read_meta("kv_backend", &kv_type);
+    r = read_meta("kv_backend", &kv_type);
     if (r < 0) {
       kv_type = g_conf()->mon_keyvaluedb;
       r = write_meta("kv_backend", kv_type);

From 3ae10a22cab6e79edf020d102c9d95812b422cf0 Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Wed, 17 Jan 2024 10:07:46 +0800
Subject: [PATCH 1383/2492] crimson/osd: fix dangling reference bug

Cherry-picked from: 3fad54e1547c2e5d8b5c54cfcdc48f2a520c3292

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/crimson/osd/pg_backend.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/pg_backend.cc b/src/crimson/osd/pg_backend.cc
index a3a777f848ad..9914a8845e4e 100644
--- a/src/crimson/osd/pg_backend.cc
+++ b/src/crimson/osd/pg_backend.cc
@@ -807,7 +807,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
   target_coid.snap = snapid;
   return obc_loader.with_clone_obc_only<RWState::RWWRITE>(
     head, target_coid,
-    [this, &os, &txn, &delta_stats, &osd_op_params, &snapid]
+    [this, &os, &txn, &delta_stats, &osd_op_params, snapid]
     (auto head_obc, auto resolved_obc) {
     if (resolved_obc->obs.oi.soid.is_head()) {
       // no-op: The resolved oid returned the head object
@@ -869,7 +869,7 @@ PGBackend::rollback_iertr::future<> PGBackend::rollback(
     // if there's no snapshot, we delete the object;
     // otherwise, do nothing.
     crimson::ct_error::enoent::handle(
-    [this, &os, &snapid, &txn, &delta_stats, &snapc, &ss, &osd_op_params] {
+    [this, &os, snapid, &txn, &delta_stats, &snapc, &ss, &osd_op_params] {
       logger().debug("PGBackend::rollback: deleting head on {}"
                      " with snap_id of {}"
                      " because got ENOENT|whiteout on obc lookup",

From c843ea616739e0c57681001526eea18fd3b596ce Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 15 Jan 2024 14:53:17 +0000
Subject: [PATCH 1384/2492] test/neorados/cls: Crimson - Skip
 NeoRadosCls.RemoteReads

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/test/neorados/cls.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/neorados/cls.cc b/src/test/neorados/cls.cc
index c3578da78064..aadc8172098a 100644
--- a/src/test/neorados/cls.cc
+++ b/src/test/neorados/cls.cc
@@ -53,6 +53,7 @@ CORO_TEST_F(NeoRadosCls, DNE, NeoRadosTest)
 
 CORO_TEST_F(NeoRadosCls, RemoteReads, NeoRadosTest)
 {
+  SKIP_IF_CRIMSON();
   static constexpr std::size_t object_size = 4096;
   static constexpr std::array oids{"src_object.1"sv, "src_object.2"sv,
 				   "src_object.3"sv};

From 3a75a60aed83bd163237449b763ad7724cae4b18 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 17 Jan 2024 11:04:43 +0000
Subject: [PATCH 1385/2492] test/neorados/io: Crimson - Skip NeoRadosIo.Limits

See: https://tracker.ceph.com/issues/64040

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/test/neorados/io.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/neorados/io.cc b/src/test/neorados/io.cc
index 3cc2d4d4c993..a73d6d36e076 100644
--- a/src/test/neorados/io.cc
+++ b/src/test/neorados/io.cc
@@ -45,6 +45,7 @@ using neorados::WriteOp;
 static constexpr auto oid = "oid"sv;
 
 CORO_TEST_F(NeoRadosIo, Limits, NeoRadosTest) {
+  SKIP_IF_CRIMSON(); // See: https://tracker.ceph.com/issues/64040
   co_await expect_error_code(
     execute(oid, WriteOp{}
 	    .write(std::numeric_limits<std::uint64_t>::max(), {})),

From b99ffc4f3e980d20374df4afeac2375eb36e37bb Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 14 Jan 2024 14:05:39 +0000
Subject: [PATCH 1386/2492] test/neorados/aio_cxx: Seperate NeoRadosECTest from
 NeoRadosTest

SKIP_IF_CRIMSON won't work here since we try to create EC pools
prior to the test being run.
Skip if the entire test instead by seperating EC tests.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 .../basic/tasks/rados_api_tests.yaml          |   2 +-
 qa/workunits/rados/test.sh                    |  10 +-
 src/test/neorados/CMakeLists.txt              |  16 +++
 src/test/neorados/ec_io.cc                    | 136 ++++++++++++++++++
 src/test/neorados/io.cc                       |  91 ------------
 5 files changed, 162 insertions(+), 93 deletions(-)
 create mode 100644 src/test/neorados/ec_io.cc

diff --git a/qa/suites/crimson-rados/basic/tasks/rados_api_tests.yaml b/qa/suites/crimson-rados/basic/tasks/rados_api_tests.yaml
index ad8c921425b1..7a209a461d6f 100644
--- a/qa/suites/crimson-rados/basic/tasks/rados_api_tests.yaml
+++ b/qa/suites/crimson-rados/basic/tasks/rados_api_tests.yaml
@@ -24,5 +24,5 @@ tasks:
 - workunit:
     clients:
       client.0:
-        - rados/test.sh
+        - rados/test.sh --crimson
         - rados/test_pool_quota.sh
diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
index 8d5f40088950..e64fc488a2dd 100755
--- a/qa/workunits/rados/test.sh
+++ b/qa/workunits/rados/test.sh
@@ -4,6 +4,10 @@ set -ex
 parallel=1
 [ "$1" = "--serial" ] && parallel=0
 
+# let crimson run in serial mode
+crimson=0
+[ "$1" = "--crimson" ] && parallel=0 && crimson=1
+
 color=""
 [ -t 1 ] && color="--gtest_color=yes"
 
@@ -49,7 +53,7 @@ do
 done
 
 for f in \
-    cls cmd handler_error io list misc pool read_operations snapshots \
+    cls cmd handler_error io ec_io list misc pool read_operations snapshots \
     watch_notify write_operations
 do
     if [ $parallel -eq 1 ]; then
@@ -60,6 +64,10 @@ do
 	echo "test $f on pid $pid"
 	pids[$f]=$pid
     else
+	if [ $crimson -eq 1 ] && [ $f = "ec_io" ]; then
+		echo "Skipping EC with Crimson"
+		continue
+	fi
 	ceph_test_neorados_$f
     fi
 done
diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index 97b99a04e5d7..62937f5c0778 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -122,6 +122,22 @@ install(TARGETS
   ceph_test_neorados_io
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
+add_executable(ceph_test_neorados_ec_io
+  ec_io.cc
+  )
+target_link_libraries(ceph_test_neorados_ec_io
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_ec_io
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 add_executable(ceph_test_neorados_list
   list.cc
   )
diff --git a/src/test/neorados/ec_io.cc b/src/test/neorados/ec_io.cc
new file mode 100644
index 000000000000..ae3c3ba8be5b
--- /dev/null
+++ b/src/test/neorados/ec_io.cc
@@ -0,0 +1,136 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <array>
+#include <coroutine>
+#include <cstdint>
+#include <limits>
+#include <utility>
+
+#include <fmt/format.h>
+
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_map.hpp>
+
+#include <boost/system/errc.hpp>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "include/buffer.h"
+#include "include/stringify.h"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace buffer = ceph::buffer;
+namespace container = boost::container;
+namespace sys = boost::system;
+
+using namespace std::literals;
+
+using neorados::ReadOp;
+using neorados::WriteOp;
+
+static constexpr auto oid = "oid"sv;
+
+CORO_TEST_F(NeoRadosECIo, SimpleWrite, NeoRadosECTest) {
+  co_return;
+  static constexpr auto nspace = "nspace";
+  auto pool2 = pool();
+  const auto bl = filled_buffer_list(0xcc, 128);
+
+  pool2.set_ns(nspace);
+  EXPECT_EQ(nspace, pool2.get_ns());
+  sleep(10);
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl));
+    auto resbl = co_await read(oid);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  {
+    co_await execute(oid, WriteOp().write(0, bl), pool2);
+    auto resbl = co_await read(oid, pool2);
+    EXPECT_EQ(bl, resbl);
+  }
+
+  co_return;
+}
+
+CORO_TEST_F(NeoRadosECIo, ReadOp, NeoRadosECTest) {
+  const auto refbl = filled_buffer_list(0xcc, 128);
+
+  co_await execute(oid, WriteOp{}.write_full(refbl));
+  {
+    buffer::list op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+  {
+    buffer::list read_bl, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, 0, &read_bl),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, op_bl);
+  }
+
+  {
+    buffer::list read_bl, read_bl2, op_bl;
+    // 0 means read the whole object data.
+    co_await rados().execute(oid, pool(), ReadOp{}
+			     .read(0, 0, &read_bl)
+			     .read(0, 0, &read_bl2),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, read_bl);
+    EXPECT_EQ(refbl, read_bl2);
+    buffer::list bl2;
+    bl2.append(refbl);
+    bl2.append(refbl);
+    EXPECT_EQ(bl2, op_bl);
+  }
+  {
+    // Read into buffer with a cached crc
+    auto op_bl = filled_buffer_list('z', refbl.length());
+    EXPECT_NE(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+
+    co_await rados().execute(oid, pool(),
+			     ReadOp().read(0, refbl.length(), nullptr),
+			     &op_bl, asio::use_awaitable);
+    EXPECT_EQ(refbl, op_bl);
+    EXPECT_EQ(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
+  }
+
+  co_return;
+}
diff --git a/src/test/neorados/io.cc b/src/test/neorados/io.cc
index a73d6d36e076..82fabf185b9b 100644
--- a/src/test/neorados/io.cc
+++ b/src/test/neorados/io.cc
@@ -379,94 +379,3 @@ CORO_TEST_F(NeoRadosIo, GetXattrs, NeoRadosTest) {
 
   co_return;
 }
-
-CORO_TEST_F(NeoRadosECIo, SimpleWrite, NeoRadosECTest) {
-  SKIP_IF_CRIMSON();
-  static constexpr auto nspace = "nspace";
-  auto pool2 = pool();
-  const auto bl = filled_buffer_list(0xcc, 128);
-
-  pool2.set_ns(nspace);
-  EXPECT_EQ(nspace, pool2.get_ns());
-
-  {
-    co_await execute(oid, WriteOp().write(0, bl));
-    auto resbl = co_await read(oid);
-    EXPECT_EQ(bl, resbl);
-  }
-
-  {
-    co_await execute(oid, WriteOp().write(0, bl), pool2);
-    auto resbl = co_await read(oid, pool2);
-    EXPECT_EQ(bl, resbl);
-  }
-
-  co_return;
-}
-
-CORO_TEST_F(NeoRadosECIo, ReadOp, NeoRadosECTest) {
-  SKIP_IF_CRIMSON();
-  const auto refbl = filled_buffer_list(0xcc, 128);
-
-  co_await execute(oid, WriteOp{}.write_full(refbl));
-  {
-    buffer::list op_bl;
-    co_await rados().execute(oid, pool(),
-			     ReadOp().read(0, refbl.length(), nullptr),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, op_bl);
-  }
-  {
-    buffer::list op_bl;
-    // 0 means read the whole object data.
-    co_await rados().execute(oid, pool(),
-			     ReadOp().read(0, 0, nullptr),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, op_bl);
-  }
-  {
-    buffer::list read_bl, op_bl;
-    co_await rados().execute(oid, pool(),
-			     ReadOp().read(0, refbl.length(), &read_bl),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, read_bl);
-    EXPECT_EQ(refbl, op_bl);
-  }
-  {
-    buffer::list read_bl, op_bl;
-    // 0 means read the whole object data.
-    co_await rados().execute(oid, pool(),
-			     ReadOp().read(0, 0, &read_bl),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, read_bl);
-    EXPECT_EQ(refbl, op_bl);
-  }
-
-  {
-    buffer::list read_bl, read_bl2, op_bl;
-    // 0 means read the whole object data.
-    co_await rados().execute(oid, pool(), ReadOp{}
-			     .read(0, 0, &read_bl)
-			     .read(0, 0, &read_bl2),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, read_bl);
-    EXPECT_EQ(refbl, read_bl2);
-    buffer::list bl2;
-    bl2.append(refbl);
-    bl2.append(refbl);
-    EXPECT_EQ(bl2, op_bl);
-  }
-  {
-    // Read into buffer with a cached crc
-    auto op_bl = filled_buffer_list('z', refbl.length());
-    EXPECT_NE(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
-
-    co_await rados().execute(oid, pool(),
-			     ReadOp().read(0, refbl.length(), nullptr),
-			     &op_bl, asio::use_awaitable);
-    EXPECT_EQ(refbl, op_bl);
-    EXPECT_EQ(refbl.crc32c(0), op_bl.crc32c(0));  // cache 'x' crc
-  }
-
-  co_return;
-}

From 73b5e8c15dacc763489fd307c93ae45cd2c67aa6 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 15 Jan 2024 14:47:00 +0000
Subject: [PATCH 1387/2492] test/neorados/list: Seperate NeoRadosECTest from
 NeoRadosTest

otherwise SKIP_IF_CRIMSON won't work in this case

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/workunits/rados/test.sh       |  10 ++-
 src/test/neorados/CMakeLists.txt |  16 ++++
 src/test/neorados/ec_list.cc     | 148 +++++++++++++++++++++++++++++++
 src/test/neorados/list.cc        |  94 --------------------
 4 files changed, 170 insertions(+), 98 deletions(-)
 create mode 100644 src/test/neorados/ec_list.cc

diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
index e64fc488a2dd..c6bcca1f0af8 100755
--- a/qa/workunits/rados/test.sh
+++ b/qa/workunits/rados/test.sh
@@ -53,7 +53,7 @@ do
 done
 
 for f in \
-    cls cmd handler_error io ec_io list misc pool read_operations snapshots \
+    cls cmd handler_error io ec_io list ec_list misc pool read_operations snapshots \
     watch_notify write_operations
 do
     if [ $parallel -eq 1 ]; then
@@ -64,9 +64,11 @@ do
 	echo "test $f on pid $pid"
 	pids[$f]=$pid
     else
-	if [ $crimson -eq 1 ] && [ $f = "ec_io" ]; then
-		echo "Skipping EC with Crimson"
-		continue
+	if [ $crimson -eq 1 ]; then
+		if [ $f = "ec_io" ] || [ $f = "ec_list" ]; then
+			echo "Skipping EC with Crimson"
+			continue
+		fi
 	fi
 	ceph_test_neorados_$f
     fi
diff --git a/src/test/neorados/CMakeLists.txt b/src/test/neorados/CMakeLists.txt
index 62937f5c0778..968ef609cdca 100644
--- a/src/test/neorados/CMakeLists.txt
+++ b/src/test/neorados/CMakeLists.txt
@@ -154,6 +154,22 @@ install(TARGETS
   ceph_test_neorados_list
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
+add_executable(ceph_test_neorados_ec_list
+  ec_list.cc
+  )
+target_link_libraries(ceph_test_neorados_ec_list
+  libneorados
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  ${CRYPTO_LIBS}
+  ${EXTRALIBS}
+  neoradostest-support
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_neorados_ec_list
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 add_executable(ceph_test_neorados_misc
   misc.cc
   )
diff --git a/src/test/neorados/ec_list.cc b/src/test/neorados/ec_list.cc
new file mode 100644
index 000000000000..5a4b76179db7
--- /dev/null
+++ b/src/test/neorados/ec_list.cc
@@ -0,0 +1,148 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM
+ *
+ * See file COPYING for license information.
+ *
+ */
+
+#include <coroutine>
+#include <cstdint>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/use_awaitable.hpp>
+
+#include <boost/container/flat_set.hpp>
+
+#include <fmt/format.h>
+
+#include "include/neorados/RADOS.hpp"
+
+#include "test/neorados/common_tests.h"
+
+#include "gtest/gtest.h"
+
+namespace asio = boost::asio;
+namespace container = boost::container;
+
+using namespace std::literals;
+
+using neorados::Cursor;
+using neorados::IOContext;
+using neorados::WriteOp;
+
+using Entries = std::vector<neorados::Entry>;
+using REntries = container::flat_set<neorados::Entry>;
+
+asio::awaitable<void> populate(neorados::RADOS& rados, const IOContext& pool,
+			       const REntries& entries) {
+  for (const auto& entry : entries) {
+    co_await ::create_obj(rados, entry.oid, pool, asio::use_awaitable);
+  }
+  co_return;
+};
+
+void compare(const REntries& ref, const Entries& res) {
+  EXPECT_EQ(ref.size(), res.size());
+  for (const auto& e : res) {
+    EXPECT_TRUE(ref.contains(e));
+  }
+  return;
+};
+
+CORO_TEST_F(NeoradosECList, ListObjects, NeoRadosECTest) {
+  static constexpr auto oid = "foo";
+  co_await execute(oid, WriteOp{}.create(true));
+  auto [entries, cursor] = co_await
+    rados().enumerate_objects(pool(), Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  EXPECT_EQ(1, entries.size());
+  EXPECT_EQ(oid, entries.front().oid);
+  co_return;
+}
+
+CORO_TEST_F(NeoradosECList, ListObjectsNS, NeoRadosECTest) {
+  auto pdef = pool();
+  IOContext p1{pool().get_pool(), "ns1"};
+  IOContext p2{pool().get_pool(), "ns2"};
+  IOContext pall{pool().get_pool(), neorados::all_nspaces};
+
+  neorados::Entry meow{.oid="foo1"s};
+  REntries def{
+    {.oid = "foo1"s},
+    {.oid = "foo2"s},
+    {.oid = "foo3"s}
+  };
+  REntries ns1{
+    {.nspace = "ns1"s, .oid = "foo1"s},
+    {.nspace = "ns1"s, .oid = "foo4"s},
+    {.nspace = "ns1"s, .oid = "foo5"s},
+    {.nspace = "ns1"s, .oid = "foo6"s},
+    {.nspace = "ns1"s, .oid = "foo7"s}
+  };
+  REntries ns2{
+    {.nspace = "ns2"s, .oid = "foo6"s},
+    {.nspace = "ns2"s, .oid = "foo7"s}
+  };
+  REntries all{def};
+  all.insert(ns1.begin(), ns1.end());
+  all.insert(ns2.begin(), ns2.end());
+
+  co_await populate(rados(), pdef, def);
+  co_await populate(rados(), p1, ns1);
+  co_await populate(rados(), p2, ns2);
+
+  auto [resdef, cdef] = co_await
+    rados().enumerate_objects(pdef, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res1, c1] = co_await
+    rados().enumerate_objects(p1, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [res2, c2] = co_await
+    rados().enumerate_objects(p2, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+  auto [resall, call] = co_await
+    rados().enumerate_objects(pall, Cursor::begin(), Cursor::end(), 1'000, {},
+			      asio::use_awaitable);
+
+  compare(def, resdef);
+  compare(ns1, res1);
+  compare(ns2, res2);
+  compare(all, resall);
+
+  co_return;
+}
+
+CORO_TEST_F(NeoradosECList, ListObjectsMany, NeoRadosECTest) {
+  REntries ref;
+  for (auto i = 0u; i < 512; ++i) {
+    ref.insert({.oid = fmt::format("{:0>3}", i)});
+  }
+  co_await populate(rados(), pool(), ref);
+  REntries res;
+  {
+    Cursor c;
+    Entries e;
+    static constexpr auto per = 10;
+    e.reserve(per);
+    while (c != Cursor::end()) {
+      std::tie(e, c) = co_await
+	rados().enumerate_objects(pool(), c, Cursor::end(), per, {},
+				  asio::use_awaitable);
+      for (auto&& n : e) {
+	res.insert(std::move(n));
+      }
+      e.clear();
+    }
+  }
+  EXPECT_EQ(ref, res);
+
+  co_return;
+}
diff --git a/src/test/neorados/list.cc b/src/test/neorados/list.cc
index 56cc042a2b88..bcfafc2c4e9e 100644
--- a/src/test/neorados/list.cc
+++ b/src/test/neorados/list.cc
@@ -151,97 +151,3 @@ CORO_TEST_F(NeoradosList, ListObjectsMany, NeoRadosTest) {
 // Sadly I don't think there's a good way to templatize testcases over
 // fixture.
 
-
-CORO_TEST_F(NeoradosECList, ListObjects, NeoRadosECTest) {
-  SKIP_IF_CRIMSON();
-  static constexpr auto oid = "foo";
-  co_await execute(oid, WriteOp{}.create(true));
-  auto [entries, cursor] = co_await
-    rados().enumerate_objects(pool(), Cursor::begin(), Cursor::end(), 1'000, {},
-			      asio::use_awaitable);
-
-  EXPECT_EQ(1, entries.size());
-  EXPECT_EQ(oid, entries.front().oid);
-  co_return;
-}
-
-CORO_TEST_F(NeoradosECList, ListObjectsNS, NeoRadosECTest) {
-  SKIP_IF_CRIMSON();
-  auto pdef = pool();
-  IOContext p1{pool().get_pool(), "ns1"};
-  IOContext p2{pool().get_pool(), "ns2"};
-  IOContext pall{pool().get_pool(), neorados::all_nspaces};
-
-  neorados::Entry meow{.oid="foo1"s};
-  REntries def{
-    {.oid = "foo1"s},
-    {.oid = "foo2"s},
-    {.oid = "foo3"s}
-  };
-  REntries ns1{
-    {.nspace = "ns1"s, .oid = "foo1"s},
-    {.nspace = "ns1"s, .oid = "foo4"s},
-    {.nspace = "ns1"s, .oid = "foo5"s},
-    {.nspace = "ns1"s, .oid = "foo6"s},
-    {.nspace = "ns1"s, .oid = "foo7"s}
-  };
-  REntries ns2{
-    {.nspace = "ns2"s, .oid = "foo6"s},
-    {.nspace = "ns2"s, .oid = "foo7"s}
-  };
-  REntries all{def};
-  all.insert(ns1.begin(), ns1.end());
-  all.insert(ns2.begin(), ns2.end());
-
-  co_await populate(rados(), pdef, def);
-  co_await populate(rados(), p1, ns1);
-  co_await populate(rados(), p2, ns2);
-
-  auto [resdef, cdef] = co_await
-    rados().enumerate_objects(pdef, Cursor::begin(), Cursor::end(), 1'000, {},
-			      asio::use_awaitable);
-  auto [res1, c1] = co_await
-    rados().enumerate_objects(p1, Cursor::begin(), Cursor::end(), 1'000, {},
-			      asio::use_awaitable);
-  auto [res2, c2] = co_await
-    rados().enumerate_objects(p2, Cursor::begin(), Cursor::end(), 1'000, {},
-			      asio::use_awaitable);
-  auto [resall, call] = co_await
-    rados().enumerate_objects(pall, Cursor::begin(), Cursor::end(), 1'000, {},
-			      asio::use_awaitable);
-
-  compare(def, resdef);
-  compare(ns1, res1);
-  compare(ns2, res2);
-  compare(all, resall);
-
-  co_return;
-}
-
-CORO_TEST_F(NeoradosECList, ListObjectsMany, NeoRadosECTest) {
-  SKIP_IF_CRIMSON();
-  REntries ref;
-  for (auto i = 0u; i < 512; ++i) {
-    ref.insert({.oid = fmt::format("{:0>3}", i)});
-  }
-  co_await populate(rados(), pool(), ref);
-  REntries res;
-  {
-    Cursor c;
-    Entries e;
-    static constexpr auto per = 10;
-    e.reserve(per);
-    while (c != Cursor::end()) {
-      std::tie(e, c) = co_await
-	rados().enumerate_objects(pool(), c, Cursor::end(), per, {},
-				  asio::use_awaitable);
-      for (auto&& n : e) {
-	res.insert(std::move(n));
-      }
-      e.clear();
-    }
-  }
-  EXPECT_EQ(ref, res);
-
-  co_return;
-}

From 678e8eaf60210a4e6906704b57fcc1bfb99dec1d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 15 Jan 2024 14:50:15 +0000
Subject: [PATCH 1388/2492] test/librados/aio_cxx: Crimson - Skip
 LibRadosAio.MultiReads

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/test/librados/aio_cxx.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/librados/aio_cxx.cc b/src/test/librados/aio_cxx.cc
index 5647bd9c0f1d..92326e4dbc07 100644
--- a/src/test/librados/aio_cxx.cc
+++ b/src/test/librados/aio_cxx.cc
@@ -2424,6 +2424,7 @@ TEST(LibRadosAio, PoolEIOFlag) {
 
 // This test case reproduces https://tracker.ceph.com/issues/57152
 TEST(LibRadosAio, MultiReads) {
+  SKIP_IF_CRIMSON();
 
   // here we test multithreaded aio reads
 

From 76b64e1db13d18795173b138e88d7411f5d39bfb Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 16 Jan 2024 11:00:37 +0000
Subject: [PATCH 1389/2492] test/neorados/misc: Crimson - Skip
 NeoRadosMisc.WriteSame

See: https://tracker.ceph.com/issues/64040

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/test/neorados/misc.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/neorados/misc.cc b/src/test/neorados/misc.cc
index 2c761bd7c7dc..201a1eee390f 100644
--- a/src/test/neorados/misc.cc
+++ b/src/test/neorados/misc.cc
@@ -261,6 +261,7 @@ CORO_TEST_F(NeoRadosMisc, BigAttr, NeoRadosTest) {
 }
 
 CORO_TEST_F(NeoRadosMisc, WriteSame, NeoRadosTest) {
+  SKIP_IF_CRIMSON(); // See: https://tracker.ceph.com/issues/64040
   static constexpr auto patlen = 128u;
   static constexpr auto samelen = patlen * 4;
   static constexpr char fill = 0xcc;

From 19e943627d97d1146802ad5a24496b46db1c11b7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 17 Jan 2024 11:51:50 +0000
Subject: [PATCH 1390/2492] qa/config/crimson_qa_overrides.yaml:  Decrease
 lru_size for testing

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/config/crimson_qa_overrides.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/config/crimson_qa_overrides.yaml b/qa/config/crimson_qa_overrides.yaml
index c2bcd0539025..8720103dab62 100644
--- a/qa/config/crimson_qa_overrides.yaml
+++ b/qa/config/crimson_qa_overrides.yaml
@@ -9,6 +9,7 @@ overrides:
         osd pool default crimson: true
       osd:
         crimson seastar smp: 3
+        crimson osd obc lru size: 10
     flavor: crimson
   workunit:
     env:

From ee50056b4421500d187e28edc49461ee9569002e Mon Sep 17 00:00:00 2001
From: Samarah <samarah.uriarte@ibm.com>
Date: Wed, 17 Jan 2024 16:05:02 +0000
Subject: [PATCH 1391/2492] rgw/asio: Add asserts to ensure asio isn't running
 during `run` and `stop` calls

Signed-off-by: Samarah <samarah.uriarte@ibm.com>
---
 src/rgw/rgw_asio_frontend.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index 1e443a84f90d..cdc7f840da7c 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -1069,6 +1069,8 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
 
 int AsioFrontend::run()
 {
+  ceph_assert(!is_asio_thread);
+
   auto cct = ctx();
   const int thread_count = cct->_conf->rgw_thread_pool_size;
   threads.reserve(thread_count);
@@ -1093,6 +1095,8 @@ int AsioFrontend::run()
 
 void AsioFrontend::stop()
 {
+  ceph_assert(!is_asio_thread);
+
   ldout(ctx(), 4) << "frontend initiating shutdown..." << dendl;
 
   going_down = true;

From 9e1999c13d5cab65e88200aa2bfc5ce385d98679 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sat, 13 Jan 2024 12:48:58 -0500
Subject: [PATCH 1392/2492] doc/rados/operations: add EC overhead table to
 erasure-code.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/rados/operations/erasure-code.rst | 178 ++++++++++++++++++++++++++
 1 file changed, 178 insertions(+)

diff --git a/doc/rados/operations/erasure-code.rst b/doc/rados/operations/erasure-code.rst
index e2bd3c296625..e53f348cdf4c 100644
--- a/doc/rados/operations/erasure-code.rst
+++ b/doc/rados/operations/erasure-code.rst
@@ -179,6 +179,8 @@ This can be enabled only on a pool residing on BlueStore OSDs, since
 BlueStore's checksumming is used during deep scrubs to detect bitrot
 or other corruption. Using Filestore with EC overwrites is not only
 unsafe, but it also results in lower performance compared to BlueStore.
+Moreover, Filestore is deprecated and any Filestore OSDs in your cluster
+should be migrated to BlueStore.
 
 Erasure-coded pools do not support omap, so to use them with RBD and
 CephFS you must instruct them to store their data in an EC pool and
@@ -192,6 +194,182 @@ erasure-coded pool as the ``--data-pool`` during image creation:
 For CephFS, an erasure-coded pool can be set as the default data pool during
 file system creation or via `file layouts <../../../cephfs/file-layouts>`_.
 
+Erasure-coded pool overhead
+---------------------------
+
+The overhead factor (space amplification) of an erasure-coded pool
+is `(k+m) / k`.  For a 4,2 profile, the overhead is
+thus 1.5, which means that 1.5 GiB of underlying storage are used to store
+1 GiB of user data.  Contrast with default three-way replication, with
+which the overhead factor is 3.0.  Do not mistake erasure coding for a free
+lunch: there is a significant performance tradeoff, especially when using HDDs
+and when performing cluster recovery or backfill.
+
+Below is a table showing the overhead factors for various values of `k` and `m`.
+As `m` increases above 2, the incremental capacity overhead gain quickly
+experiences diminishing returns but the performance impact grows proportionally.
+We recommend that you do not choose a profile with `k` > 4 or `m` > 2 until
+and unless you fully understand the ramifications, including the number of
+failure domains your cluster topology must contain.  If  you choose `m=1`,
+expect data unavailability during maintenance and data loss if component
+failures overlap.
+
+.. list-table:: Erasure coding overhead
+   :widths: 4 4 4 4 4 4 4 4 4 4 4 4
+   :header-rows: 1
+   :stub-columns: 1
+
+   * -
+     - m=1
+     - m=2
+     - m=3
+     - m=4
+     - m=4
+     - m=6
+     - m=7
+     - m=8
+     - m=9
+     - m=10
+     - m=11
+   * - k=1
+     - 2.00
+     - 3.00
+     - 4.00
+     - 5.00
+     - 6.00
+     - 7.00
+     - 8.00
+     - 9.00
+     - 10.00
+     - 11.00
+     - 12.00
+   * - k=2
+     - 1.50
+     - 2.00
+     - 2.50
+     - 3.00
+     - 3.50
+     - 4.00
+     - 4.50
+     - 5.00
+     - 5.50
+     - 6.00
+     - 6.50
+   * - k=3
+     - 1.33
+     - 1.67
+     - 2.00
+     - 2.33
+     - 2.67
+     - 3.00
+     - 3.33
+     - 3.67
+     - 4.00
+     - 4.33
+     - 4.67
+   * - k=4
+     - 1.25
+     - 1.50
+     - 1.75
+     - 2.00
+     - 2.25
+     - 2.50
+     - 2.75
+     - 3.00
+     - 3.25
+     - 3.50
+     - 3.75
+   * - k=5
+     - 1.20
+     - 1.40
+     - 1.60
+     - 1.80
+     - 2.00
+     - 2.20
+     - 2.40
+     - 2.60
+     - 2.80
+     - 3.00
+     - 3.20
+   * - k=6
+     - 1.16
+     - 1.33
+     - 1.50
+     - 1.66
+     - 1.83
+     - 2.00
+     - 2.17
+     - 2.33
+     - 2.50
+     - 2.66
+     - 2.83
+   * - k=7
+     - 1.14
+     - 1.29
+     - 1.43
+     - 1.58
+     - 1.71
+     - 1.86
+     - 2.00
+     - 2.14
+     - 2.29
+     - 2.43
+     - 2.58
+   * - k=8
+     - 1.13
+     - 1.25
+     - 1.38
+     - 1.50
+     - 1.63
+     - 1.75
+     - 1.88
+     - 2.00
+     - 2.13
+     - 2.25
+     - 2.38
+   * - k=9
+     - 1.11
+     - 1.22
+     - 1.33
+     - 1.44
+     - 1.56
+     - 1.67
+     - 1.78
+     - 1.88
+     - 2.00
+     - 2.11
+     - 2.22
+   * - k=10
+     - 1.10
+     - 1.20
+     - 1.30
+     - 1.40
+     - 1.50
+     - 1.60
+     - 1.70
+     - 1.80
+     - 1.90
+     - 2.00
+     - 2.10
+   * - k=11
+     - 1.09
+     - 1.18
+     - 1.27
+     - 1.36
+     - 1.45
+     - 1.54
+     - 1.63
+     - 1.72
+     - 1.82
+     - 1.91
+     - 2.00
+
+
+
+
+
+
+
 
 Erasure-coded pools and cache tiering
 -------------------------------------

From 11a9fe8947b94cda49a2248c8708552da412485d Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Mon, 15 Jan 2024 13:41:06 +0100
Subject: [PATCH 1393/2492] mgr/rook: always recreate kvm default network + fix
 groups refresh Fixes: https://tracker.ceph.com/issues/64027

This change also includes:
- adding ~/.local/bin to path so behave binary can be found
- adding requirements.txt file for testing dependencies
- increasing timeout used to wait for tools deployment to 90s
- increasing timeout used to wait for kvm network to 20s

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/ci/requirements.txt       |  1 +
 src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh  |  3 ++
 .../rook/ci/scripts/bootstrap-rook-cluster.sh | 49 +++++++++++++++----
 3 files changed, 44 insertions(+), 9 deletions(-)
 create mode 100644 src/pybind/mgr/rook/ci/requirements.txt

diff --git a/src/pybind/mgr/rook/ci/requirements.txt b/src/pybind/mgr/rook/ci/requirements.txt
new file mode 100644
index 000000000000..9684f7742bdd
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/requirements.txt
@@ -0,0 +1 @@
+behave
diff --git a/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
index a43e01a89861..58d55475774f 100755
--- a/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
+++ b/src/pybind/mgr/rook/ci/run-rook-e2e-tests.sh
@@ -2,8 +2,11 @@
 
 set -ex
 
+export PATH=$PATH:~/.local/bin # behave is installed on this directory
+
 # Execute tests
 : ${CEPH_DEV_FOLDER:=${PWD}}
 ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
 cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/tests
+pip install --upgrade --force-reinstall -r ../requirements.txt
 behave
diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index 4b97df6bac14..eb4f9fb66ce7 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -15,14 +15,6 @@ on_error() {
     minikube delete
 }
 
-configure_libvirt(){
-    sudo usermod -aG libvirt $(id -un)
-    sudo su -l $USER  # Avoid having to log out and log in for group addition to take effect.
-    sudo systemctl enable --now libvirtd
-    sudo systemctl restart libvirtd
-    sleep 10 # wait some time for libvirtd service to restart
-}
-
 setup_minikube_env() {
 
     # Check if Minikube is running
@@ -93,7 +85,7 @@ wait_for_ceph_cluster() {
     local max_attempts=10
     local sleep_interval=20
     local attempts=0
-    $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=30s
+    $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=90s
     while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do
 	echo "Waiting for Ceph cluster installed"
 	sleep $sleep_interval
@@ -118,12 +110,51 @@ show_info() {
     echo "==========================="
 }
 
+configure_libvirt(){
+    if sudo usermod -aG libvirt $(id -un); then
+	echo "User added to libvirt group successfully."
+	sudo systemctl enable --now libvirtd
+	sudo systemctl restart libvirtd
+	sleep 10 # wait some time for libvirtd service to restart
+	newgrp libvirt
+    else
+	echo "Error adding user to libvirt group."
+	return 1
+    fi
+}
+
+recreate_default_network(){
+
+    # destroy any existing kvm default network
+    if sudo virsh net-destroy default; then
+	sudo virsh net-undefine default
+    fi
+
+    # let's create a new kvm default network
+    sudo virsh net-define /usr/share/libvirt/networks/default.xml
+    if sudo virsh net-start default; then
+        echo "Network 'default' started successfully."
+    else
+        # Optionally, handle the error
+        echo "Failed to start network 'default', but continuing..."
+    fi
+
+    # restart libvirtd service and wait a little bit for the service
+    sudo systemctl restart libvirtd
+    sleep 20
+
+    # Just some debugging information
+    all_networks=$(virsh net-list --all)
+    groups=$(groups)
+}
+
 ####################################################################
 ####################################################################
 
 trap 'on_error $? $LINENO' ERR
 
 configure_libvirt
+recreate_default_network
 setup_minikube_env
 build_ceph_image
 create_rook_cluster

From 5c056d6ca0483d65574f870137ffc9a06ee8d499 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 16 Jan 2024 23:38:51 +0000
Subject: [PATCH 1394/2492] common/intrusive_lru: clarify invariants

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/intrusive_lru.h | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/common/intrusive_lru.h b/src/common/intrusive_lru.h
index fc63bea2636a..6ae13c8f99fc 100644
--- a/src/common/intrusive_lru.h
+++ b/src/common/intrusive_lru.h
@@ -12,13 +12,12 @@ namespace ceph::common {
 /**
  * intrusive_lru: lru implementation with embedded map and list hook
  *
- * Elements will be stored in an intrusive set. Once an element is no longer
- * referenced it will remain in the set. The unreferenced elements will be
- * evicted from the set once the set size exceeds the `lru_target_size`.
- * Referenced elements will not be evicted as this is a registery with
- * extra caching capabilities.
+ * Elements with live references are guarranteed to remain accessible.
+ * Elements without live references may remain accessible -- implementation
+ * will release unreferenced elements based on lru_target_size.
  *
- * Note, this implementation currently is entirely thread-unsafe.
+ * Accesses, mutations, and references must be confined to a single thread or
+ * serialized via some other mechanism.
  */
 
 template <typename K, typename V, typename VToK>
@@ -43,11 +42,26 @@ void intrusive_ptr_release(intrusive_lru_base<Config> *p);
 
 template <typename Config>
 class intrusive_lru_base {
+  /* object invariants
+   *
+   * intrusive_lru objects may be in one of two states:
+   * 1. referenced
+   *    - intrusive_lru_base::lru is points to parent intrusive_lru
+   *    - present in lru_set
+   *    - use_count > 0
+   *    - not eligible for eviction
+   *    - intrusive_lru_release may be invoked externally
+   * 2. unreferenced
+   *    - intrusive_lru_base::lru is null
+   *    - present in lru_set
+   *    - present in intrusive_lru::unreferenced_list
+   *    - use_count == 0
+   *    - eligible for eviction
+   *    - intrusive_lru_release cannot be invoked
+   */
   unsigned use_count = 0;
 
-  // lru points to the corresponding intrusive_lru
-  // which will be set to null if its use_count
-  // is zero (aka unreferenced).
+  // See above, points at intrusive_lru iff referenced
   intrusive_lru<Config> *lru = nullptr;
 
 public:
@@ -232,8 +246,11 @@ void intrusive_ptr_add_ref(intrusive_lru_base<Config> *p) {
 
 template <typename Config>
 void intrusive_ptr_release(intrusive_lru_base<Config> *p) {
+  /* See object invariants above -- intrusive_ptr_release can only be invoked on
+   * referenced objects */
   assert(p);
   assert(p->use_count > 0);
+  assert(is_referenced());
   --p->use_count;
   if (p->use_count == 0) {
     p->lru->mark_as_unreferenced(*p);

From 10c9fb3240f987b605de6fe7c3c1e66bb32d89aa Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 17 Jan 2024 20:22:10 -0800
Subject: [PATCH 1395/2492] common/intrusive_lru: generalize evict() slightly
 to allow other target values

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/intrusive_lru.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/common/intrusive_lru.h b/src/common/intrusive_lru.h
index 6ae13c8f99fc..fb94b5fd9ead 100644
--- a/src/common/intrusive_lru.h
+++ b/src/common/intrusive_lru.h
@@ -122,9 +122,9 @@ class intrusive_lru {
 
   // when the lru_set exceeds its target size, evict
   // only unreferenced elements from it (if any).
-  void evict() {
+  void evict(unsigned target_size) {
     while (!unreferenced_list.empty() &&
-	   lru_set.size() > lru_target_size) {
+	   lru_set.size() > target_size) {
       auto &evict_target = unreferenced_list.front();
       assert(evict_target.is_unreferenced());
       unreferenced_list.pop_front();
@@ -150,7 +150,7 @@ class intrusive_lru {
     assert(b.is_unreferenced());
     lru_set.insert(b);
     b.lru = this;
-    evict();
+    evict(lru_target_size);
   }
 
   // an element in the lru_set has no users,
@@ -159,7 +159,7 @@ class intrusive_lru {
     assert(b.is_referenced());
     unreferenced_list.push_back(b);
     b.lru = nullptr;
-    evict();
+    evict(lru_target_size);
   }
 
 public:
@@ -226,7 +226,7 @@ class intrusive_lru {
 
   void set_target_size(size_t target_size) {
     lru_target_size = target_size;
-    evict();
+    evict(lru_target_size);
   }
 
   ~intrusive_lru() {

From e2023d28dc6e6e835303716e7235df720d33a01c Mon Sep 17 00:00:00 2001
From: Marcus Watts <mwatts@redhat.com>
Date: Mon, 22 Aug 2022 03:38:18 -0400
Subject: [PATCH 1396/2492] rgw: tempurl signature fixes: support sha256 and
 sha512

The latest openstack swift logic supports sha256 and sha512 checksums
for tempurl and formpost, which can be either hexadecimal or
hashname:base64.  This adds logic to inspect the incoming signature
for type and to internally construct the same hash to compare.

Also:: fixes an incidental crash if a malformed swift path does not
contain an object name.

Fixes: https://tracker.ceph.com/issues/56564
Signed-off-by: Marcus Watts <mwatts@redhat.com>
---
 src/rgw/rgw_swift_auth.cc | 118 ++++++++++++++++++++++++++------------
 src/rgw/rgw_swift_auth.h  | 116 +++++++++++++++++++++++++++++++++++++
 2 files changed, 196 insertions(+), 38 deletions(-)

diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc
index 765721df190d..aa31adc1d0a4 100644
--- a/src/rgw/rgw_swift_auth.cc
+++ b/src/rgw/rgw_swift_auth.cc
@@ -82,7 +82,7 @@ void TempURLEngine::get_owner_info(const DoutPrefixProvider* dpp, const req_stat
   const string& bucket_name = s->init_state.url_bucket;
 
   /* TempURL requires that bucket and object names are specified. */
-  if (bucket_name.empty() || s->object->empty()) {
+  if (bucket_name.empty() || rgw::sal::Object::empty(s->object)) {
     throw -EPERM;
   }
 
@@ -190,66 +190,108 @@ std::string extract_swift_subuser(const std::string& swift_user_name)
   }
 }
 
-class TempURLEngine::SignatureHelper
-{
-private:
-  static constexpr uint32_t output_size =
-    CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2 + 1;
-
-  unsigned char dest[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE]; // 20
-  char dest_str[output_size];
-
+template <class HASHFLAVOR, SignatureFlavor SIGNATUREFLAVOR>
+class TempURLSignatureT : public rgw::auth::swift::FormatSignature<HASHFLAVOR,SIGNATUREFLAVOR> {
+  using UCHARPTR = const unsigned char*;
+  using base_t = SignatureHelperT<HASHFLAVOR>;
+  using format_signature_t = rgw::auth::swift::FormatSignature<HASHFLAVOR,SIGNATUREFLAVOR>;
 public:
-  SignatureHelper() = default;
-
   const char* calc(const std::string& key,
                    const std::string_view& method,
                    const std::string_view& path,
                    const std::string& expires) {
+    HASHFLAVOR hmac((UCHARPTR) key.data(), key.size());
 
-    using ceph::crypto::HMACSHA1;
-    using UCHARPTR = const unsigned char*;
-
-    HMACSHA1 hmac((UCHARPTR) key.c_str(), key.size());
     hmac.Update((UCHARPTR) method.data(), method.size());
     hmac.Update((UCHARPTR) "\n", 1);
     hmac.Update((UCHARPTR) expires.c_str(), expires.size());
     hmac.Update((UCHARPTR) "\n", 1);
     hmac.Update((UCHARPTR) path.data(), path.size());
-    hmac.Final(dest);
+    hmac.Final(base_t::dest);
 
-    buf_to_hex((UCHARPTR) dest, sizeof(dest), dest_str);
-
-    return dest_str;
+    return  format_signature_t::result();
   }
-
-  bool is_equal_to(const std::string& rhs) const {
-    /* never allow out-of-range exception */
-    if (rhs.size() < (output_size - 1)) {
-      return false;
+}; /* TempURLSignatureT */
+class TempURLEngine::SignatureHelper {
+public:
+  SignatureHelper() {};
+  virtual ~SignatureHelper() {};
+  virtual const char* calc(const std::string& key,
+    const std::string_view& method,
+    const std::string_view& path,
+    const std::string& expires) {
+    return nullptr;
+  }
+  virtual bool is_equal_to(const std::string& rhs) {
+    return false;
+  };
+  static std::unique_ptr<SignatureHelper> get_sig_helper(std::string_view x);
+};
+class TempURLSignature {
+  friend TempURLEngine;
+  using BadSignatureHelper = TempURLEngine::SignatureHelper;
+  template<typename HASHFLAVOR, SignatureFlavor SIGNATUREFLAVOR>
+  class SignatureHelper_x : public TempURLEngine::SignatureHelper
+  {
+    friend TempURLEngine;
+    TempURLSignatureT<HASHFLAVOR,SIGNATUREFLAVOR> d;
+  public:
+    SignatureHelper_x() {};
+    ~SignatureHelper_x() { };
+    virtual const char* calc(const std::string& key,
+      const std::string_view& method,
+      const std::string_view& path,
+      const std::string& expires) {
+      return d.calc(key,method,path,expires);
     }
-    return rhs.compare(0 /* pos */,  output_size, dest_str) == 0;
+    virtual bool is_equal_to(const std::string& rhs) {
+      return d.is_equal_to(rhs);
+    };
+  };
+};
+
+std::unique_ptr<TempURLEngine::SignatureHelper> TempURLEngine::SignatureHelper::get_sig_helper(std::string_view x) {
+  size_t pos = x.find(':');
+  if (pos == x.npos || pos <= 0) {
+    switch(x.length()) {
+    case CEPH_CRYPTO_HMACSHA1_DIGESTSIZE*2:
+      return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA1,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    case CEPH_CRYPTO_HMACSHA256_DIGESTSIZE*2:
+      return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA256,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    case CEPH_CRYPTO_HMACSHA512_DIGESTSIZE*2:
+      return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA512,rgw::auth::swift::SignatureFlavor::BARE_HEX>>();
+    }
+    return std::make_unique<TempURLSignature::BadSignatureHelper>();
   }
+  std::string_view type { x.substr(0,pos) };
+  if (type == "sha1") {
+    return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA1,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  } else if (type == "sha256") {
+    return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA256,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  } else if (type == "sha512") {
+    return std::make_unique<TempURLSignature::SignatureHelper_x<ceph::crypto::HMACSHA512,rgw::auth::swift::SignatureFlavor::NAMED_BASE64>>();
+  }
+  return std::make_unique<TempURLSignature::BadSignatureHelper>();
+};
 
-}; /* TempURLEngine::SignatureHelper */
-
-class TempURLEngine::PrefixableSignatureHelper
-    : private TempURLEngine::SignatureHelper {
-  using base_t = SignatureHelper;
+class TempURLEngine::PrefixableSignatureHelper {
 
   const std::string_view decoded_uri;
   const std::string_view object_name;
   std::string_view no_obj_uri;
 
   const boost::optional<const std::string&> prefix;
+  std::unique_ptr<SignatureHelper> base_sig_helper;
 
 public:
-  PrefixableSignatureHelper(const std::string& _decoded_uri,
+  PrefixableSignatureHelper(const std::string_view sig,
+	                    const std::string& _decoded_uri,
 	                    const std::string& object_name,
                             const boost::optional<const std::string&> prefix)
     : decoded_uri(_decoded_uri),
       object_name(object_name),
-      prefix(prefix) {
+      prefix(prefix),
+      base_sig_helper(TempURLEngine::SignatureHelper::get_sig_helper(sig)) {
     /* Transform: v1/acct/cont/obj - > v1/acct/cont/
      *
      * NOTE(rzarzynski): we really want to substr() on std::string_view,
@@ -257,23 +299,23 @@ class TempURLEngine::PrefixableSignatureHelper
      * a temporary. */
     no_obj_uri = \
       decoded_uri.substr(0, decoded_uri.length() - object_name.length());
-  }
+  };
 
   const char* calc(const std::string& key,
                    const std::string_view& method,
                    const std::string_view& path,
                    const std::string& expires) {
     if (!prefix) {
-      return base_t::calc(key, method, path, expires);
+      return base_sig_helper->calc(key, method, path, expires);
     } else {
       const auto prefixed_path = \
         string_cat_reserve("prefix:", no_obj_uri, *prefix);
-      return base_t::calc(key, method, prefixed_path, expires);
+      return base_sig_helper->calc(key, method, prefixed_path, expires);
     }
   }
 
   bool is_equal_to(const std::string& rhs) const {
-    bool is_auth_ok = base_t::is_equal_to(rhs);
+    bool is_auth_ok = base_sig_helper->is_equal_to(rhs);
 
     if (prefix && is_auth_ok) {
       const auto prefix_uri = string_cat_reserve(no_obj_uri, *prefix);
@@ -360,6 +402,7 @@ TempURLEngine::authenticate(const DoutPrefixProvider* dpp, const req_state* cons
 
   /* Need to try each combination of keys, allowed path and methods. */
   PrefixableSignatureHelper sig_helper {
+    temp_url_sig,
     s->decoded_uri,
     s->object->get_name(),
     temp_url_prefix
@@ -772,4 +815,3 @@ RGWOp *RGWHandler_SWIFT_Auth::op_get()
 {
   return new RGW_SWIFT_Auth_Get;
 }
-
diff --git a/src/rgw/rgw_swift_auth.h b/src/rgw/rgw_swift_auth.h
index 85a103dbfd5a..3564a6b39b5c 100644
--- a/src/rgw/rgw_swift_auth.h
+++ b/src/rgw/rgw_swift_auth.h
@@ -11,6 +11,7 @@
 #include "rgw_auth_keystone.h"
 #include "rgw_auth_filters.h"
 #include "rgw_sal.h"
+#include "rgw_b64.h"
 
 #define RGW_SWIFT_TOKEN_EXPIRATION (15 * 60)
 
@@ -39,6 +40,7 @@ class TempURLApplier : public rgw::auth::LocalApplier {
 
 /* TempURL: engine */
 class TempURLEngine : public rgw::auth::Engine {
+  friend class TempURLSignature;
   using result_t = rgw::auth::Engine::result_t;
 
   CephContext* const cct;
@@ -303,6 +305,120 @@ class DefaultStrategy : public rgw::auth::Strategy,
   }
 };
 
+// shared logic for swift tempurl and formpost signatures
+template <class HASHFLAVOR>
+inline constexpr uint32_t signature_hash_size = -1;
+template <>
+inline constexpr uint32_t signature_hash_size<ceph::crypto::HMACSHA1> = CEPH_CRYPTO_HMACSHA1_DIGESTSIZE;
+template<>
+inline constexpr uint32_t signature_hash_size<ceph::crypto::HMACSHA256> = CEPH_CRYPTO_HMACSHA256_DIGESTSIZE;
+template<>
+inline constexpr uint32_t signature_hash_size<ceph::crypto::HMACSHA512> = CEPH_CRYPTO_HMACSHA512_DIGESTSIZE;
+
+const char sha1_name[] = "sha1";
+const char sha256_name[] = "sha256";
+const char sha512_name[] = "sha512";
+
+template <class HASHFLAVOR>
+const char * signature_hash_name;
+template<>
+inline constexpr const char * signature_hash_name<ceph::crypto::HMACSHA1> = sha1_name;;
+template<>
+inline constexpr const char * signature_hash_name<ceph::crypto::HMACSHA256> = sha256_name;
+template<>
+inline constexpr const char * signature_hash_name<ceph::crypto::HMACSHA512> = sha512_name;
+
+template <class HASHFLAVOR>
+inline const uint32_t signature_hash_name_size = -1;
+template<>
+inline constexpr uint32_t signature_hash_name_size<ceph::crypto::HMACSHA1> = sizeof sha1_name;;
+template<>
+inline constexpr uint32_t signature_hash_name_size<ceph::crypto::HMACSHA256> = sizeof sha256_name;
+template<>
+inline constexpr uint32_t signature_hash_name_size<ceph::crypto::HMACSHA512> = sizeof sha512_name;
+
+template <class HASHFLAVOR>
+class SignatureHelperT {
+protected:
+  static constexpr uint32_t hash_size = signature_hash_size<HASHFLAVOR>;
+  static constexpr uint32_t output_size = hash_size * 2 + 1;
+  const char * signature_name = signature_hash_name<HASHFLAVOR>;
+  uint32_t signature_name_size = signature_hash_name_size<HASHFLAVOR>;
+  char dest_str[output_size];
+  uint32_t dest_size = 0;
+  unsigned char dest[hash_size];
+
+public:
+  ~SignatureHelperT() { };
+
+  void Update(const unsigned char *input, size_t length);
+
+  const char* get_signature() const {
+    return dest_str;
+  }
+
+  bool is_equal_to(const std::string& rhs) const {
+    /* never allow out-of-range exception */
+    if (!dest_size || rhs.size() < dest_size) {
+      return false;
+    }
+    return rhs.compare(0 /* pos */,  dest_size + 1, dest_str) == 0;
+  }
+};
+
+enum class SignatureFlavor {
+  BARE_HEX,
+  NAMED_BASE64
+};
+
+template <typename HASHFLAVOR, SignatureFlavor SIGNATUREFLAVOR>
+class FormatSignature {
+};
+
+// hexadecimal
+template <typename HASHFLAVOR>
+class FormatSignature<HASHFLAVOR, SignatureFlavor::BARE_HEX> : public SignatureHelperT<HASHFLAVOR> {
+  using UCHARPTR = const unsigned char*;
+  using base_t = SignatureHelperT<HASHFLAVOR>;
+public:
+  const char *result() {
+    buf_to_hex((UCHARPTR) base_t::dest,
+      signature_hash_size<HASHFLAVOR>,
+      base_t::dest_str);
+    base_t::dest_size = strlen(base_t::dest_str);
+    return base_t::dest_str;
+  };
+};
+
+// prefix:base64
+template <typename HASHFLAVOR>
+class FormatSignature<HASHFLAVOR, SignatureFlavor::NAMED_BASE64> : public SignatureHelperT<HASHFLAVOR> {
+  using UCHARPTR = const unsigned char*;
+  using base_t = SignatureHelperT<HASHFLAVOR>;
+public:
+  char * const result() {
+    const char *prefix = base_t::signature_name;
+    const int prefix_size = base_t::signature_name_size;
+    std::string_view dest_view((char*)base_t::dest, sizeof base_t::dest);
+    auto b { rgw::to_base64(dest_view) };
+    for (auto &v: b ) {	// translate to "url safe" (rfc 4648 section 5)
+      switch(v) {
+      case '+': v = '-'; break;
+      case '/': v = '_'; break;
+      }
+    }
+    base_t::dest_size = prefix_size + b.length();
+    if (base_t::dest_size < base_t::output_size) {
+      ::memcpy(base_t::dest_str, prefix, prefix_size - 1);
+      base_t::dest_str[prefix_size-1] = ':';
+      ::strcpy(base_t::dest_str + prefix_size, b.c_str());
+    } else {
+      base_t::dest_size = 0;
+    }
+    return base_t::dest_str;
+  };
+};
+
 } /* namespace swift */
 } /* namespace auth */
 } /* namespace rgw */

From 7edfa3f2fb145253a49c155847212097b3bfb9da Mon Sep 17 00:00:00 2001
From: Tobias Urdin <tobias.urdin@binero.se>
Date: Thu, 18 Jan 2024 08:15:51 +0000
Subject: [PATCH 1397/2492] rgw: fix http error check in keystone
 issue_admin_token_request

This changes so that we don't return before even checking
if the request failed by checking the response code like
was done for other methods in [1].

[1] https://github.com/ceph/ceph/pull/53846

Signed-off-by: Tobias Urdin <tobias.urdin@binero.se>
---
 src/rgw/rgw_keystone.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc
index 7d5264f980ee..e6fbeea71b46 100644
--- a/src/rgw/rgw_keystone.cc
+++ b/src/rgw/rgw_keystone.cc
@@ -211,9 +211,6 @@ int Service::issue_admin_token_request(const DoutPrefixProvider *dpp,
   token_req.set_url(token_url);
 
   const int ret = token_req.process(y);
-  if (ret < 0) {
-    return ret;
-  }
 
   /* Detect rejection earlier than during the token parsing step. */
   if (token_req.get_http_status() ==
@@ -221,6 +218,11 @@ int Service::issue_admin_token_request(const DoutPrefixProvider *dpp,
     return -EACCES;
   }
 
+  // throw any other http or connection errors
+  if (ret < 0) {
+    return ret;
+  }
+
   if (t.parse(dpp, token_req.get_subject_token(), token_bl,
               keystone_version) != 0) {
     return -EINVAL;

From ab49906b7a81f9110145b820db3fb7b69e28032b Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 21 Sep 2023 02:14:13 -0400
Subject: [PATCH 1398/2492] mds: adhere to mds_health_summarize_threshold
 config when reporting laggy clients

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/Beacon.cc         | 29 +++++++++++++++++++----------
 src/messages/MMDSBeacon.h |  2 ++
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/mds/Beacon.cc b/src/mds/Beacon.cc
index 4990f0b249f9..c71030d7d211 100644
--- a/src/mds/Beacon.cc
+++ b/src/mds/Beacon.cc
@@ -506,17 +506,26 @@ void Beacon::notify_health(MDSRank const *mds)
       return map.any_osd_laggy(); });
     auto&& laggy_clients = mds->server->get_laggy_clients();
     if (defer_client_eviction && !laggy_clients.empty()) {
-      std::vector<MDSHealthMetric> laggy_clients_metrics;
-      for (const auto& laggy_client: laggy_clients) {
-        CachedStackStringStream css;
-        *css << "Client " << laggy_client << " is laggy; not evicted"
-             << " because some OSD(s) is/are laggy";
-        MDSHealthMetric m(MDS_HEALTH_CLIENTS_LAGGY, HEALTH_WARN, css->strv());
-        laggy_clients_metrics.emplace_back(std::move(m));
+      if (laggy_clients.size() <= (size_t)g_conf()->mds_health_summarize_threshold) {
+	std::vector<MDSHealthMetric> laggy_clients_metrics;
+	for (const auto& laggy_client: laggy_clients) {
+	  CachedStackStringStream css;
+	  *css << "Client " << laggy_client << " is laggy; not evicted"
+	       << " because some OSD(s) is/are laggy";
+	  MDSHealthMetric m(MDS_HEALTH_CLIENTS_LAGGY, HEALTH_WARN, css->strv());
+	  laggy_clients_metrics.emplace_back(std::move(m));
+	}
+	auto&& m = laggy_clients_metrics;
+	health.metrics.insert(std::end(health.metrics), std::cbegin(m),
+			      std::cend(m));
+      } else {
+	CachedStackStringStream css;
+	*css << "Many client (" << laggy_clients.size()
+	     << ") are laggy; not evicting since some OSD(s) are laggy";
+	MDSHealthMetric m(MDS_HEALTH_CLIENTS_LAGGY_MANY, HEALTH_WARN, css->strv());
+	m.metadata["client_count"] = stringify(laggy_clients.size());
+	health.metrics.push_back(std::move(m));
       }
-      auto&& m = laggy_clients_metrics;
-      health.metrics.insert(std::end(health.metrics), std::cbegin(m),
-                            std::cend(m));
     }
   }
 }
diff --git a/src/messages/MMDSBeacon.h b/src/messages/MMDSBeacon.h
index d843b0e76610..ba2d52d11ee3 100644
--- a/src/messages/MMDSBeacon.h
+++ b/src/messages/MMDSBeacon.h
@@ -46,6 +46,7 @@ enum mds_metric_t {
   MDS_HEALTH_CACHE_OVERSIZED,
   MDS_HEALTH_SLOW_METADATA_IO,
   MDS_HEALTH_CLIENTS_LAGGY,
+  MDS_HEALTH_CLIENTS_LAGGY_MANY,
   MDS_HEALTH_DUMMY, // not a real health warning, for testing
 };
 
@@ -65,6 +66,7 @@ inline const char *mds_metric_name(mds_metric_t m)
   case MDS_HEALTH_CACHE_OVERSIZED: return "MDS_CACHE_OVERSIZED";
   case MDS_HEALTH_SLOW_METADATA_IO: return "MDS_SLOW_METADATA_IO";
   case MDS_HEALTH_CLIENTS_LAGGY: return "MDS_CLIENTS_LAGGY";
+  case MDS_HEALTH_CLIENTS_LAGGY_MANY: return "MDS_CLIENTS_LAGGY_MANY";
   case MDS_HEALTH_DUMMY: return "MDS_DUMMY";
   default:
     return "???";

From f14899ce0859af5225074f9b19d018ba1cc8fcf6 Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Wed, 20 Dec 2023 14:15:35 +0530
Subject: [PATCH 1399/2492] src: kernel mount command returning misleading
 error message

Fixes: https://tracker.ceph.com/issues/63866
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 src/mount/mount.ceph.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mount/mount.ceph.c b/src/mount/mount.ceph.c
index 631b0b3fcf18..6efb154d31bb 100644
--- a/src/mount/mount.ceph.c
+++ b/src/mount/mount.ceph.c
@@ -912,7 +912,8 @@ static int do_mount(const char *dev, const char *node,
 			fprintf(stderr, "mount error: ceph filesystem not supported by the system\n");
 			break;
 		case EHOSTUNREACH:
-			fprintf(stderr, "mount error: no mds server is up or the cluster is laggy\n");
+			fprintf(stderr, "mount error: no mds (Metadata Server) is up. "
+			"The cluster might be laggy, or you may not be authorized\n");
 			break;
 		default:
 			fprintf(stderr, "mount error %d = %s\n", errno, strerror(errno));

From bac9e74abf0f5cb60c74640d6c4689169acf7abe Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 9 Oct 2023 17:22:10 -0400
Subject: [PATCH 1400/2492] qa/distros: bump container-hosts from centos 8 to 9

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 .../centos_9.stream_container_tools.yaml         | 14 ++++++++++++++
 .../centos_9.stream_container_tools_crun.yaml    | 16 ++++++++++++++++
 qa/distros/single-container-host.yaml            |  2 +-
 qa/suites/orch/cephadm/smoke-small/0-distro/.qa  |  1 +
 .../centos_8.stream_container_tools_crun.yaml    |  1 -
 .../centos_9.stream_container_tools_crun.yaml    |  1 +
 ...1-start-centos_9.stream_container-tools.yaml} |  2 +-
 7 files changed, 34 insertions(+), 3 deletions(-)
 create mode 100644 qa/distros/container-hosts/centos_9.stream_container_tools.yaml
 create mode 100644 qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
 create mode 120000 qa/suites/orch/cephadm/smoke-small/0-distro/.qa
 delete mode 120000 qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml
 create mode 120000 qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
 rename qa/suites/orch/cephadm/upgrade/1-start-distro/{1-start-centos_8.stream_container-tools.yaml => 1-start-centos_9.stream_container-tools.yaml} (96%)

diff --git a/qa/distros/container-hosts/centos_9.stream_container_tools.yaml b/qa/distros/container-hosts/centos_9.stream_container_tools.yaml
new file mode 100644
index 000000000000..eb542a92cc87
--- /dev/null
+++ b/qa/distros/container-hosts/centos_9.stream_container_tools.yaml
@@ -0,0 +1,14 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
+tasks:
+- pexec:
+    all:
+    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
+    - sudo dnf -y  module reset container-tools
+    - sudo dnf -y  module install container-tools --allowerasing --nobest
+    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml b/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
new file mode 100644
index 000000000000..1dafe3216908
--- /dev/null
+++ b/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
@@ -0,0 +1,16 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
+tasks:
+- pexec:
+    all:
+    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
+    - sudo dnf -y  module reset container-tools
+    - sudo dnf -y  module install container-tools --allowerasing --nobest
+    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
+    - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf
+    - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf
diff --git a/qa/distros/single-container-host.yaml b/qa/distros/single-container-host.yaml
index f71756d42e87..dbda08f27f74 120000
--- a/qa/distros/single-container-host.yaml
+++ b/qa/distros/single-container-host.yaml
@@ -1 +1 @@
-container-hosts/centos_8.stream_container_tools_crun.yaml
\ No newline at end of file
+container-hosts/centos_9.stream_container_tools_crun.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/.qa b/qa/suites/orch/cephadm/smoke-small/0-distro/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/0-distro/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml
deleted file mode 120000
index 83fe02026e1b..000000000000
--- a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml
+++ /dev/null
@@ -1 +0,0 @@
-../.qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
new file mode 120000
index 000000000000..52bac4e665e9
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
@@ -0,0 +1 @@
+.qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml
similarity index 96%
rename from qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
rename to qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml
index db57ea8874f3..c752194a6759 100644
--- a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml
@@ -1,5 +1,5 @@
 os_type: centos
-os_version: "8.stream"
+os_version: "9.stream"
 
 tasks:
 - pexec:

From 00b3eef5b846dee6e9aae1f1295271c89f2801e5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 9 Oct 2023 17:25:13 -0400
Subject: [PATCH 1401/2492] qa/distros: bump podman from centos 8 to 9

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 .../podman/centos_9.stream_container_tools.yaml    | 14 ++++++++++++++
 .../centos_8.stream_container_tools.yaml           |  1 -
 .../centos_9.stream_container_tools.yaml           |  1 +
 .../0-centos_8.stream_container_tools.yaml         |  1 -
 .../0-centos_9.stream_container_tools.yaml         |  1 +
 .../centos_8.stream_container_tools.yaml           |  1 -
 .../centos_9.stream_container_tools.yaml           |  1 +
 7 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 qa/distros/podman/centos_9.stream_container_tools.yaml
 delete mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.stream_container_tools.yaml
 create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
 delete mode 120000 qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml
 create mode 120000 qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
 delete mode 120000 qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml
 create mode 120000 qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml

diff --git a/qa/distros/podman/centos_9.stream_container_tools.yaml b/qa/distros/podman/centos_9.stream_container_tools.yaml
new file mode 100644
index 000000000000..eb542a92cc87
--- /dev/null
+++ b/qa/distros/podman/centos_9.stream_container_tools.yaml
@@ -0,0 +1,14 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
+tasks:
+- pexec:
+    all:
+    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
+    - sudo dnf -y  module reset container-tools
+    - sudo dnf -y  module install container-tools --allowerasing --nobest
+    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.stream_container_tools.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.stream_container_tools.yaml
deleted file mode 120000
index 7a86f967f020..000000000000
--- a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_8.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
new file mode 120000
index 000000000000..0ad8250bb855
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml
deleted file mode 120000
index 7a86f967f020..000000000000
--- a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_8.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
new file mode 120000
index 000000000000..0ad8250bb855
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml
deleted file mode 120000
index 7a86f967f020..000000000000
--- a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_8.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml
new file mode 120000
index 000000000000..0ad8250bb855
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file

From b9ba9642fea18055e768c3ddee5ba5e372264ede Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 9 Oct 2023 17:28:48 -0400
Subject: [PATCH 1402/2492] qa/suites/fs: workload suite uses centos 9 instead
 of rhel

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml | 1 +
 qa/suites/fs/workload/0-rhel_8.yaml                          | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 120000 qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
 delete mode 120000 qa/suites/fs/workload/0-rhel_8.yaml

diff --git a/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml b/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
new file mode 120000
index 000000000000..0ad8250bb855
--- /dev/null
+++ b/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/workload/0-rhel_8.yaml b/qa/suites/fs/workload/0-rhel_8.yaml
deleted file mode 120000
index c9abcd7b83f3..000000000000
--- a/qa/suites/fs/workload/0-rhel_8.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/rhel_8.6_container_tools_rhel8.yaml
\ No newline at end of file

From 7193f76a1502de392af0b407db0073266f9bc9c8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 9 Oct 2023 17:29:21 -0400
Subject: [PATCH 1403/2492] qa/distos: remove rhel from podman and
 container-hosts

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 .../rhel_8.6_container_tools_3.0.yaml               | 13 -------------
 .../rhel_8.6_container_tools_rhel8.yaml             | 13 -------------
 qa/distros/podman/rhel_8.6_container_tools_3.0.yaml | 13 -------------
 .../podman/rhel_8.6_container_tools_rhel8.yaml      | 13 -------------
 4 files changed, 52 deletions(-)
 delete mode 100644 qa/distros/container-hosts/rhel_8.6_container_tools_3.0.yaml
 delete mode 100644 qa/distros/container-hosts/rhel_8.6_container_tools_rhel8.yaml
 delete mode 100644 qa/distros/podman/rhel_8.6_container_tools_3.0.yaml
 delete mode 100644 qa/distros/podman/rhel_8.6_container_tools_rhel8.yaml

diff --git a/qa/distros/container-hosts/rhel_8.6_container_tools_3.0.yaml b/qa/distros/container-hosts/rhel_8.6_container_tools_3.0.yaml
deleted file mode 100644
index 361d8546e2b7..000000000000
--- a/qa/distros/container-hosts/rhel_8.6_container_tools_3.0.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-os_type: rhel
-os_version: "8.6"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools:3.0 --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/container-hosts/rhel_8.6_container_tools_rhel8.yaml b/qa/distros/container-hosts/rhel_8.6_container_tools_rhel8.yaml
deleted file mode 100644
index be94ed69e7d6..000000000000
--- a/qa/distros/container-hosts/rhel_8.6_container_tools_rhel8.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-os_type: rhel
-os_version: "8.6"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools:rhel8 --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/podman/rhel_8.6_container_tools_3.0.yaml b/qa/distros/podman/rhel_8.6_container_tools_3.0.yaml
deleted file mode 100644
index 361d8546e2b7..000000000000
--- a/qa/distros/podman/rhel_8.6_container_tools_3.0.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-os_type: rhel
-os_version: "8.6"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools:3.0 --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/podman/rhel_8.6_container_tools_rhel8.yaml b/qa/distros/podman/rhel_8.6_container_tools_rhel8.yaml
deleted file mode 100644
index be94ed69e7d6..000000000000
--- a/qa/distros/podman/rhel_8.6_container_tools_rhel8.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-os_type: rhel
-os_version: "8.6"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools:rhel8 --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf

From 12254d8f5418b31437ba7077fbfc2d015bd88fe1 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 9 Oct 2023 19:02:31 -0400
Subject: [PATCH 1404/2492] qa/distros: bump container-hosts from ubuntu 20 to
 22

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/distros/container-hosts/ubuntu_20.04.yaml         | 9 ---------
 qa/distros/container-hosts/ubuntu_22.04.yaml         | 1 +
 qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml | 1 -
 qa/suites/orch/rook/smoke/0-distro/ubuntu_22.04.yaml | 1 +
 4 files changed, 2 insertions(+), 10 deletions(-)
 delete mode 100644 qa/distros/container-hosts/ubuntu_20.04.yaml
 create mode 120000 qa/distros/container-hosts/ubuntu_22.04.yaml
 delete mode 120000 qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml
 create mode 120000 qa/suites/orch/rook/smoke/0-distro/ubuntu_22.04.yaml

diff --git a/qa/distros/container-hosts/ubuntu_20.04.yaml b/qa/distros/container-hosts/ubuntu_20.04.yaml
deleted file mode 100644
index bb9f5c00a098..000000000000
--- a/qa/distros/container-hosts/ubuntu_20.04.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-os_type: ubuntu
-os_version: "20.04"
-# the normal ubuntu 20.04 kernel (5.4.0-88-generic currently) have a bug that prevents the nvme_loop
-# from behaving.  I think it is this:
-#   https://lkml.org/lkml/2020/9/21/1456
-# (at least, that is the symptom: nvme nvme1: Connect command failed, error wo/DNR bit: 880)
-overrides:
-  kernel:
-    hwe: true
diff --git a/qa/distros/container-hosts/ubuntu_22.04.yaml b/qa/distros/container-hosts/ubuntu_22.04.yaml
new file mode 120000
index 000000000000..0a708b4dbd27
--- /dev/null
+++ b/qa/distros/container-hosts/ubuntu_22.04.yaml
@@ -0,0 +1 @@
+.qa/distros/all/ubuntu_22.04.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml b/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml
deleted file mode 120000
index f62164f9166f..000000000000
--- a/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/container-hosts/ubuntu_20.04.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/rook/smoke/0-distro/ubuntu_22.04.yaml b/qa/suites/orch/rook/smoke/0-distro/ubuntu_22.04.yaml
new file mode 120000
index 000000000000..e4835e4f7682
--- /dev/null
+++ b/qa/suites/orch/rook/smoke/0-distro/ubuntu_22.04.yaml
@@ -0,0 +1 @@
+.qa/distros/container-hosts/ubuntu_22.04.yaml
\ No newline at end of file

From c8873c6591d368e12907669c274fd3d6391e3f68 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Thu, 30 Nov 2023 17:14:59 -0500
Subject: [PATCH 1405/2492] qa: remove container hosts install from centos 9
 distros

We needed the container-hosts version of podman on centos 8,
but there's no need to keep using it in centos 9

Additionally, changes from crun variants to runc variants since
crun is now the default container runtime on centos 9

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/distros/container-hosts/centos_9.stream.yaml  |  7 +++++++
 .../centos_9.stream_container_tools.yaml         | 14 --------------
 .../centos_9.stream_container_tools_crun.yaml    | 16 ----------------
 .../container-hosts/centos_9.stream_runc.yaml    | 13 +++++++++++++
 qa/distros/podman/centos_9.stream.yaml           |  7 +++++++
 .../podman/centos_9.stream_container_tools.yaml  | 14 --------------
 qa/distros/single-container-host.yaml            |  2 +-
 .../mds_upgrade_sequence/centos_9.stream.yaml    |  1 +
 .../centos_9.stream_container_tools.yaml         |  1 -
 qa/suites/fs/workload/0-centos_9.stream.yaml     |  1 +
 .../0-centos_9.stream_container_tools.yaml       |  1 -
 .../mgr-nfs-upgrade/0-centos_9.stream.yaml       |  1 +
 .../0-centos_9.stream_container_tools.yaml       |  1 -
 .../centos_9.stream_container_tools_crun.yaml    |  1 -
 .../0-distro/centos_9.stream_runc.yaml           |  1 +
 ...r-tools.yaml => 1-start-centos_9.stream.yaml} |  6 ------
 .../test_iscsi_container/centos_9.stream.yaml    |  1 +
 .../centos_9.stream_container_tools.yaml         |  1 -
 18 files changed, 33 insertions(+), 56 deletions(-)
 create mode 100644 qa/distros/container-hosts/centos_9.stream.yaml
 delete mode 100644 qa/distros/container-hosts/centos_9.stream_container_tools.yaml
 delete mode 100644 qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
 create mode 100644 qa/distros/container-hosts/centos_9.stream_runc.yaml
 create mode 100644 qa/distros/podman/centos_9.stream.yaml
 delete mode 100644 qa/distros/podman/centos_9.stream_container_tools.yaml
 create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream.yaml
 delete mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
 create mode 120000 qa/suites/fs/workload/0-centos_9.stream.yaml
 delete mode 120000 qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
 create mode 120000 qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream.yaml
 delete mode 120000 qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
 delete mode 120000 qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
 create mode 120000 qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_runc.yaml
 rename qa/suites/orch/cephadm/upgrade/1-start-distro/{1-start-centos_9.stream_container-tools.yaml => 1-start-centos_9.stream.yaml} (59%)
 create mode 120000 qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream.yaml
 delete mode 120000 qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml

diff --git a/qa/distros/container-hosts/centos_9.stream.yaml b/qa/distros/container-hosts/centos_9.stream.yaml
new file mode 100644
index 000000000000..d3bc430e63b9
--- /dev/null
+++ b/qa/distros/container-hosts/centos_9.stream.yaml
@@ -0,0 +1,7 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
diff --git a/qa/distros/container-hosts/centos_9.stream_container_tools.yaml b/qa/distros/container-hosts/centos_9.stream_container_tools.yaml
deleted file mode 100644
index eb542a92cc87..000000000000
--- a/qa/distros/container-hosts/centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-os_type: centos
-os_version: "9.stream"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml b/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
deleted file mode 100644
index 1dafe3216908..000000000000
--- a/qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-os_type: centos
-os_version: "9.stream"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
-    - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf
-    - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf
diff --git a/qa/distros/container-hosts/centos_9.stream_runc.yaml b/qa/distros/container-hosts/centos_9.stream_runc.yaml
new file mode 100644
index 000000000000..2e4f9fe3a61e
--- /dev/null
+++ b/qa/distros/container-hosts/centos_9.stream_runc.yaml
@@ -0,0 +1,13 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
+tasks:
+- pexec:
+    all:
+    - sudo dnf install runc -y
+    - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf
+    - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf
diff --git a/qa/distros/podman/centos_9.stream.yaml b/qa/distros/podman/centos_9.stream.yaml
new file mode 100644
index 000000000000..d3bc430e63b9
--- /dev/null
+++ b/qa/distros/podman/centos_9.stream.yaml
@@ -0,0 +1,7 @@
+os_type: centos
+os_version: "9.stream"
+overrides:
+  selinux:
+    whitelist:
+      - scontext=system_u:system_r:logrotate_t:s0
+
diff --git a/qa/distros/podman/centos_9.stream_container_tools.yaml b/qa/distros/podman/centos_9.stream_container_tools.yaml
deleted file mode 100644
index eb542a92cc87..000000000000
--- a/qa/distros/podman/centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-os_type: centos
-os_version: "9.stream"
-overrides:
-  selinux:
-    whitelist:
-      - scontext=system_u:system_r:logrotate_t:s0
-
-tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools --allowerasing --nobest
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
diff --git a/qa/distros/single-container-host.yaml b/qa/distros/single-container-host.yaml
index dbda08f27f74..08e1be9c25d5 120000
--- a/qa/distros/single-container-host.yaml
+++ b/qa/distros/single-container-host.yaml
@@ -1 +1 @@
-container-hosts/centos_9.stream_container_tools_crun.yaml
\ No newline at end of file
+container-hosts/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream.yaml
new file mode 120000
index 000000000000..dca92ddbf45e
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
deleted file mode 120000
index 0ad8250bb855..000000000000
--- a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/workload/0-centos_9.stream.yaml b/qa/suites/fs/workload/0-centos_9.stream.yaml
new file mode 120000
index 000000000000..dca92ddbf45e
--- /dev/null
+++ b/qa/suites/fs/workload/0-centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml b/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
deleted file mode 120000
index 0ad8250bb855..000000000000
--- a/qa/suites/fs/workload/0-centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream.yaml
new file mode 120000
index 000000000000..dca92ddbf45e
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
deleted file mode 120000
index 0ad8250bb855..000000000000
--- a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
deleted file mode 120000
index 52bac4e665e9..000000000000
--- a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_container_tools_crun.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/container-hosts/centos_9.stream_container_tools_crun.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_runc.yaml b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_runc.yaml
new file mode 120000
index 000000000000..5ef2595cf740
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_9.stream_runc.yaml
@@ -0,0 +1 @@
+.qa/distros/container-hosts/centos_9.stream_runc.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream.yaml
similarity index 59%
rename from qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml
rename to qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream.yaml
index c752194a6759..885efe6bf6f6 100644
--- a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream_container-tools.yaml
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_9.stream.yaml
@@ -2,12 +2,6 @@ os_type: centos
 os_version: "9.stream"
 
 tasks:
-- pexec:
-    all:
-    - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
-    - sudo dnf -y  module reset container-tools
-    - sudo dnf -y  module install container-tools
-    - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
 - cephadm:
     image: quay.io/ceph/ceph:v17.2.0
     cephadm_branch: v17.2.0
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream.yaml
new file mode 120000
index 000000000000..dca92ddbf45e
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml
deleted file mode 120000
index 0ad8250bb855..000000000000
--- a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_9.stream_container_tools.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/podman/centos_9.stream_container_tools.yaml
\ No newline at end of file

From 80dc6ce72a2c209424698ce31549362e9148022f Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 11 Dec 2023 13:59:42 -0500
Subject: [PATCH 1406/2492] qa/cephadm: use quincy for add-repo test

There are no centos 9 build for octopus, so if we
want to start testing on cnetos 9 as a distro we need
the add-repo test to be done on a newer release
for which there are actual builds

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/workunits/cephadm/test_repos.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/workunits/cephadm/test_repos.sh b/qa/workunits/cephadm/test_repos.sh
index 221585fd0fef..5c17e51060e9 100755
--- a/qa/workunits/cephadm/test_repos.sh
+++ b/qa/workunits/cephadm/test_repos.sh
@@ -30,7 +30,7 @@ function test_install_uninstall() {
 	      sudo zypper -n remove cephadm )
 }
 
-sudo $CEPHADM -v add-repo --release octopus
+sudo $CEPHADM -v add-repo --release quincy
 test_install_uninstall
 sudo $CEPHADM -v rm-repo
 
@@ -38,7 +38,7 @@ sudo $CEPHADM -v add-repo --dev main
 test_install_uninstall
 sudo $CEPHADM -v rm-repo
 
-sudo $CEPHADM -v add-repo --release 15.2.7
+sudo $CEPHADM -v add-repo --release 17.2.6
 test_install_uninstall
 sudo $CEPHADM -v rm-repo
 

From 97efa97c2c72dba32774d0f105ae90462b93ac4e Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 11 Dec 2023 15:44:30 -0500
Subject: [PATCH 1407/2492] qa/cephadm: fix iscsi pids limit check for centos 9

Centos 9 uses cgroups v2 which has a slightly
different file location for the pids.max. This commit
updates the test to also check the new location
so the test can pass on centos 9

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/workunits/cephadm/test_iscsi_pids_limit.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/qa/workunits/cephadm/test_iscsi_pids_limit.sh b/qa/workunits/cephadm/test_iscsi_pids_limit.sh
index bed4cc9e274f..40bc6085667a 100755
--- a/qa/workunits/cephadm/test_iscsi_pids_limit.sh
+++ b/qa/workunits/cephadm/test_iscsi_pids_limit.sh
@@ -12,7 +12,17 @@ test ${CONT_COUNT} -eq 2
 
 for i in ${ISCSI_CONT_IDS}
 do
-  test $(sudo podman exec ${i} cat /sys/fs/cgroup/pids/pids.max) == max
+  # cgroups v1 and v2 have slightly different file locations for the pids.max
+  # so check both spots
+  if [ $(sudo podman exec ${i} cat /sys/fs/cgroup/pids/pids.max) ]; then
+    pid_limit=$(sudo podman exec ${i} cat /sys/fs/cgroup/pids/pids.max)
+  elif [ $(sudo podman exec ${i} cat /sys/fs/cgroup/pids.max) ]; then
+    pid_limit=$(sudo podman exec ${i} cat /sys/fs/cgroup/pids.max)
+  else
+    echo "could not find pids.max inside container"
+    exit 1
+  fi
+  test $pid_limit == max
 done
 
 for i in ${ISCSI_CONT_IDS}

From 9fd22e142ff1482079561c6a9f55adaddcdf413f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 17 Jan 2024 20:26:07 -0800
Subject: [PATCH 1408/2492] common/intrusive_lru: add clear() mechanism and
 state for removed items with live references

We need to be able to clear the lru without waiting for all outstanding
references to be released.  Items with such references enter an
invalidated state where references can still be added and removed with
deletion at use_count == 0, but can't be accessed any longer via the lru.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/common/intrusive_lru.h            | 48 ++++++++++++++---
 src/test/common/test_intrusive_lru.cc | 78 ++++++++++++++++++++++++++-
 2 files changed, 117 insertions(+), 9 deletions(-)

diff --git a/src/common/intrusive_lru.h b/src/common/intrusive_lru.h
index fb94b5fd9ead..564cceef1cc2 100644
--- a/src/common/intrusive_lru.h
+++ b/src/common/intrusive_lru.h
@@ -44,20 +44,30 @@ template <typename Config>
 class intrusive_lru_base {
   /* object invariants
    *
-   * intrusive_lru objects may be in one of two states:
+   * intrusive_lru objects may be in one of three states:
    * 1. referenced
+   *    - accessible via intrusive_lru
    *    - intrusive_lru_base::lru is points to parent intrusive_lru
-   *    - present in lru_set
+   *    - present in intrusive_lru::lru_set
+   *    - absent from intrusive_lru::unreferenced_list
    *    - use_count > 0
    *    - not eligible for eviction
    *    - intrusive_lru_release may be invoked externally
    * 2. unreferenced
+   *    - accessible via intrusive_lru
    *    - intrusive_lru_base::lru is null
-   *    - present in lru_set
+   *    - present in intrusive_lru::lru_set
    *    - present in intrusive_lru::unreferenced_list
    *    - use_count == 0
    *    - eligible for eviction
    *    - intrusive_lru_release cannot be invoked
+   * 3. invalidated
+   *    - inaccessible via intrusive_lru
+   *    - intrusive_lru_base::lru is null
+   *    - absent from intrusive_lru::lru_set
+   *    - absent from intrusive_lru::unreferenced_list
+   *    - use_count > 0
+   *    - intrusive_lru_release may be invoked externally
    */
   unsigned use_count = 0;
 
@@ -69,7 +79,10 @@ class intrusive_lru_base {
     return static_cast<bool>(lru);
   }
   bool is_unreferenced() const {
-    return !is_referenced();
+    return !is_referenced() && use_count == 0;
+  }
+  bool is_invalidated() const {
+    return !is_referenced() && use_count > 0;
   }
   boost::intrusive::set_member_hook<> set_hook;
   boost::intrusive::list_member_hook<> list_hook;
@@ -203,6 +216,21 @@ class intrusive_lru {
       }
   }
 
+  /// drop all elements from lru, invoke f on any with outstanding references
+  template <typename F>
+  void clear(F &&f) {
+    evict(0);
+    assert(unreferenced_list.empty());
+    for (auto &i: lru_set) {
+      std::invoke(f, static_cast<T&>(i));
+      i.lru = nullptr;
+      assert(i.is_invalidated());
+    }
+    lru_set.clear_and_dispose([](auto *i){
+      assert(i->use_count > 0); /* don't delete, still has a ref count */
+    });
+  }
+
   template <class F>
   void for_each(F&& f) {
     for (auto& v : lru_set) {
@@ -240,20 +268,24 @@ class intrusive_lru {
 template <typename Config>
 void intrusive_ptr_add_ref(intrusive_lru_base<Config> *p) {
   assert(p);
-  assert(p->lru);
   p->use_count++;
+  assert(p->is_referenced() || p->is_invalidated());
 }
 
 template <typename Config>
 void intrusive_ptr_release(intrusive_lru_base<Config> *p) {
   /* See object invariants above -- intrusive_ptr_release can only be invoked on
-   * referenced objects */
+   * is_referenced() or is_invalidated() objects with live external references */
   assert(p);
   assert(p->use_count > 0);
-  assert(is_referenced());
+  assert(p->is_referenced() || p->is_invalidated());
   --p->use_count;
   if (p->use_count == 0) {
-    p->lru->mark_as_unreferenced(*p);
+    if (p->lru) {
+      p->lru->mark_as_unreferenced(*p);
+    } else {
+      delete p;
+    }
   }
 }
 
diff --git a/src/test/common/test_intrusive_lru.cc b/src/test/common/test_intrusive_lru.cc
index 0654bd97d81e..af8edb8e2bf3 100644
--- a/src/test/common/test_intrusive_lru.cc
+++ b/src/test/common/test_intrusive_lru.cc
@@ -13,14 +13,21 @@ struct item_to_unsigned {
   }
 };
 
+
+static int LIVE_TEST_LRU_ITEMS = 0;
 struct TestLRUItem : public ceph::common::intrusive_lru_base<
   ceph::common::intrusive_lru_config<
     unsigned, TestLRUItem, item_to_unsigned<TestLRUItem>>> {
   unsigned key = 0;
   int value = 0;
+  bool invalidated = false;
 
-  TestLRUItem(unsigned key) : key(key) {}
+  TestLRUItem(unsigned key) : key(key) {
+    ++LIVE_TEST_LRU_ITEMS;
+  }
+  ~TestLRUItem() { --LIVE_TEST_LRU_ITEMS; }
 };
+using TestLRUItemRef = boost::intrusive_ptr<TestLRUItem>;
 
 class LRUTest : public TestLRUItem::lru_t {
 public:
@@ -206,3 +213,72 @@ TEST(LRU, clear_range) {
     ASSERT_FALSE(existed);
   }
 }
+
+TEST(LRU, clear) {
+  LRUTest cache;
+  const unsigned SIZE = 10;
+  cache.set_target_size(SIZE);
+  
+  std::vector<TestLRUItemRef> refs;
+  for (unsigned i = 0; i < 100; ++i) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_FALSE(existed);
+    if ((i % 2) == 0) {
+      refs.push_back(ref);
+    }
+  }
+
+  for (unsigned i = 0; i < 100; i += 2) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_TRUE(existed);
+  }
+
+  cache.clear([](auto &i) { i.invalidated = true; });
+  ASSERT_EQ(refs.size(), LIVE_TEST_LRU_ITEMS);
+
+  for (auto &i: refs) {
+    ASSERT_TRUE(i->invalidated);
+  }
+
+  std::vector<TestLRUItemRef> refs_new;
+  for (unsigned i = 0; i < 100; ++i) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_FALSE(existed);
+    ASSERT_FALSE(ref->invalidated);
+    if ((i % 2) == 0) {
+      refs_new.push_back(ref);
+    }
+  }
+
+  for (unsigned i = 0; i < 100; i += 2) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_TRUE(existed);
+    ASSERT_FALSE(ref->invalidated);
+  }
+
+  refs.clear();
+  cache.set_target_size(0);
+  ASSERT_EQ(refs_new.size(), LIVE_TEST_LRU_ITEMS);
+  cache.set_target_size(SIZE);
+
+  for (unsigned i = 100; i < 200; ++i) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_FALSE(existed);
+    ASSERT_FALSE(ref->invalidated);
+    if ((i % 2) == 0) {
+      refs_new.push_back(ref);
+    }
+  }
+
+  for (unsigned i = 0; i < 200; i += 2) {
+    auto [ref, existed] = cache.add(i, i);
+    ASSERT_TRUE(existed);
+    ASSERT_FALSE(ref->invalidated);
+  }
+
+  ASSERT_EQ(refs_new.size(), LIVE_TEST_LRU_ITEMS);
+  refs_new.clear();
+  ASSERT_EQ(SIZE, LIVE_TEST_LRU_ITEMS);
+  cache.set_target_size(0);
+  ASSERT_EQ(0, LIVE_TEST_LRU_ITEMS);
+}

From 04691123ed5f0c34afca25cba63b9609f2490d9b Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Fri, 19 Jan 2024 10:53:04 +0800
Subject: [PATCH 1409/2492] install-deps: Force remove ceph-libboost* packages

As `apt-get remove ceph-libboost*` show the error and cause the make check failed.
`E: The package ceph-libboost1.79-dev needs to be reinstalled, but I can't find an archive for it.`
The ceph-libboost* packages may be not removed, so use `dpkg` to force remove ceph-libboost*.

Fixes: https://tracker.ceph.com/issues/64032

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 install-deps.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/install-deps.sh b/install-deps.sh
index d01bc6464e6f..a334a3b29a30 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -177,6 +177,14 @@ function clean_boost_on_ubuntu {
     # so no need to spare it.
     if test -n "$installed_ver"; then
 	$SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y --fix-missing remove "ceph-libboost*"
+	# When an error occurs during `apt-get remove ceph-libboost*`, ceph-libboost* packages
+	# may be not removed, so use `dpkg` to force remove ceph-libboost*.
+	local ceph_libboost_pkgs=$(dpkg -l | grep ceph-libboost* | awk '{print $2}' |
+		                        awk -F: '{print $1}')
+	if test -n "$ceph_libboost_pkgs"; then
+	    ci_debug "Force remove ceph-libboost* packages $ceph_libboost_pkgs"
+	    $SUDO dpkg --purge --force-all $ceph_libboost_pkgs
+	fi
     fi
 }
 

From 4bb49478fae09ead4646c1baada3bbc9a2555130 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 19 Jan 2024 13:56:21 -0500
Subject: [PATCH 1410/2492] rgw/auth/s3: validate x-amz-content-sha256 for
 empty payloads

when is_v4_payload_empty(), we return a null completer so never try to
validate the x-amz-content-sha256 for signed payloads. add this
checksum comparison to get_auth_data_v4() before we create the completer

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_rest_s3.cc | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 9791cab8a71e..a91814c737dc 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5805,6 +5805,19 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
                                      std::placeholders::_3,
                                      s);
 
+  // some ops don't expect a request body at all, so never call complete() to
+  // validate the payload hash. check empty signed payloads now and return a
+  // null completer below
+  constexpr std::string_view empty_sha256sum = // echo -n | sha256sum
+      "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+  if (is_v4_payload_empty(s) &&
+      !is_v4_payload_unsigned(exp_payload_hash) &&
+      exp_payload_hash != empty_sha256sum) {
+    ldpp_dout(s, 4) << "ERROR: empty payload checksum mismatch, expected "
+        << empty_sha256sum << " got " << exp_payload_hash << dendl;
+    throw -ERR_AMZ_CONTENT_SHA256_MISMATCH;
+  }
+
   /* Requests authenticated with the Query Parameters are treated as unsigned.
    * From "Authenticating Requests: Using Query Parameters (AWS Signature
    * Version 4)":

From 90b4376bcfd6e9f51f5cb982490f524eb7703454 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Fri, 19 Jan 2024 19:48:23 +0000
Subject: [PATCH 1411/2492] tools/cot: respect generic, Ceph-wide argvs

```
$ bin/ceph-objectstore-tool --version
Must provide --data-path

Allowed options:
  --help                      produce help message
  --type arg                  Arg is one of [bluestore (default), memstore]
```

```
$ bin/ceph-bluestore-tool --version
ceph version 19.0.0-642-g6d93c157068 (6d93c1570681fbf25fff5c48a9ff5bcb7d6d2210) squid (dev)
```

Fixes: https://tracker.ceph.com/issues/64104
Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/tools/ceph_objectstore_tool.cc | 35 ++++++++++++++++--------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index bb051df64cd6..eeb15a39248a 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -3490,12 +3490,25 @@ int main(int argc, char **argv)
     tmp.swap(ceph_option_strings);
   }
 
-  vector<const char *> ceph_options;
-  ceph_options.reserve(ceph_options.size() + ceph_option_strings.size());
-  for (vector<string>::iterator i = ceph_option_strings.begin();
-       i != ceph_option_strings.end();
-       ++i) {
-    ceph_options.push_back(i->c_str());
+  boost::intrusive_ptr<CephContext> cct;
+  {
+    vector<const char *> ceph_options;
+    ceph_options.reserve(ceph_options.size() + ceph_option_strings.size());
+    for (vector<string>::iterator i = ceph_option_strings.begin();
+         i != ceph_option_strings.end();
+         ++i) {
+      ceph_options.push_back(i->c_str());
+    }
+    int init_flags = 0;
+    if (vm.count("no-mon-config") > 0) {
+      init_flags |= CINIT_FLAG_NO_MON_CONFIG;
+    }
+    cct = global_init(
+      nullptr,
+      ceph_options,
+      CEPH_ENTITY_TYPE_OSD,
+      CODE_ENVIRONMENT_UTILITY_NODOUT,
+      init_flags);
   }
 
   snprintf(fn, sizeof(fn), "%s/type", dpath.c_str());
@@ -3570,16 +3583,6 @@ int main(int argc, char **argv)
     perror(err.c_str());
     return 1;
   }
-  int init_flags = 0;
-  if (vm.count("no-mon-config") > 0) {
-    init_flags |= CINIT_FLAG_NO_MON_CONFIG;
-  }
-
-  auto cct = global_init(
-    NULL, ceph_options,
-    CEPH_ENTITY_TYPE_OSD,
-    CODE_ENVIRONMENT_UTILITY_NODOUT,
-    init_flags);
   common_init_finish(g_ceph_context);
   if (debug) {
     g_conf().set_val_or_die("log_to_stderr", "true");

From 83d535cd27e8acd373fa4fbc9405d26272446028 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 17 Jan 2024 21:07:15 -0800
Subject: [PATCH 1412/2492] crimson/osd: invalidate contents of obc_registry on
 interval_change

We can't guarantee that all continuations will complete before on_flush,
so we unhook and invalidate all live obcs from the registry upon
interval_change.  As long as any lingering continuations simply note the
interval change and terminate, this should be safe.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/object_context.h         | 13 +++++++++++--
 src/crimson/osd/object_context_loader.cc | 10 +++++++++-
 src/crimson/osd/pg.cc                    |  1 +
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/object_context.h b/src/crimson/osd/object_context.h
index 75c99bb54680..f38e52f009ce 100644
--- a/src/crimson/osd/object_context.h
+++ b/src/crimson/osd/object_context.h
@@ -112,8 +112,8 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
     }
   }
 
-  bool is_fully_loaded() const {
-    return fully_loaded;
+  bool is_loaded_and_valid() const {
+    return fully_loaded && !invalidated_by_interval_change;
   }
 
 private:
@@ -133,7 +133,10 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
   boost::intrusive::list_member_hook<> obc_accessing_hook;
   uint64_t list_link_cnt = 0;
   bool fully_loaded = false;
+  bool invalidated_by_interval_change = false;
 
+  friend class ObjectContextRegistry;
+  friend class ObjectContextLoader;
 public:
 
   template <typename ListType>
@@ -267,6 +270,12 @@ class ObjectContextRegistry : public md_config_obs_t  {
     obc_lru.clear_range(from, to);
   }
 
+  void invalidate_on_interval_change() {
+    obc_lru.clear([](auto &obc) {
+      obc.invalidated_by_interval_change = true;
+    });
+  }
+
   template <class F>
   void for_each(F&& f) {
     obc_lru.for_each(std::forward<F>(f));
diff --git a/src/crimson/osd/object_context_loader.cc b/src/crimson/osd/object_context_loader.cc
index f49de73f5d40..9772ce59776a 100644
--- a/src/crimson/osd/object_context_loader.cc
+++ b/src/crimson/osd/object_context_loader.cc
@@ -168,7 +168,15 @@ using crimson::common::local_conf;
     auto loaded =
       load_obc_iertr::make_ready_future<ObjectContextRef>(obc);
     if (existed) {
-      ceph_assert(obc->is_fully_loaded());
+      if (!obc->is_loaded_and_valid()) {
+	ERRORDPP(
+	  "obc for {} invalid -- fully_loaded={}, "
+	  "invalidated_by_interval_change={}",
+	  dpp, obc->get_oid(),
+	  obc->fully_loaded, obc->invalidated_by_interval_change
+	);
+      }
+      ceph_assert(obc->is_loaded_and_valid());
       DEBUGDPP("cache hit on {}", dpp, obc->get_oid());
     } else {
       DEBUGDPP("cache miss on {}", dpp, obc->get_oid());
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index f20ea15d66f5..b9cfdb8c19fc 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1524,6 +1524,7 @@ void PG::on_change(ceph::os::Transaction &t) {
     client_request_orderer.clear_and_cancel(*this);
   }
   scrubber.on_interval_change();
+  obc_registry.invalidate_on_interval_change();
 }
 
 void PG::context_registry_on_change() {

From b6436f47e871d787535ea80a301aff543fd77739 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 19 Jan 2024 17:41:01 -0800
Subject: [PATCH 1413/2492] crimson/osd/ops_executor: simplify prepare_clone,
 use set_clone_state

- Remove static_snap_oi name -- left over from old classic implementation,
  misleading.
- OpsExecutor::prepare_clone can only be invoked on a primary, remove
  the branch.
- Create the obs directly and pass to obc via set_clone_state, which
  sets fully_loaded.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/ops_executer.cc | 47 +++++++++++++--------------------
 src/crimson/osd/ops_executer.h  |  2 +-
 2 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 034fdde71693..5fe7ab4a7178 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -931,12 +931,12 @@ std::unique_ptr<OpsExecuter::CloningContext> OpsExecuter::execute_clone(
     return std::vector<snapid_t>{std::begin(snapc.snaps), last};
   }();
 
-  auto [snap_oi, clone_obc] = prepare_clone(coid);
+  auto clone_obc = prepare_clone(coid);
   // make clone
-  backend.clone(snap_oi, initial_obs, clone_obc->obs, txn);
+  backend.clone(clone_obc->obs.oi, initial_obs, clone_obc->obs, txn);
 
   delta_stats.num_objects++;
-  if (snap_oi.is_omap()) {
+  if (clone_obc->obs.oi.is_omap()) {
     delta_stats.num_objects_omap++;
   }
   delta_stats.num_object_clones++;
@@ -960,7 +960,7 @@ std::unique_ptr<OpsExecuter::CloningContext> OpsExecuter::execute_clone(
   cloning_ctx->log_entry = {
     pg_log_entry_t::CLONE,
     coid,
-    snap_oi.version,
+    clone_obc->obs.oi.version,
     initial_obs.oi.version,
     initial_obs.oi.user_version,
     osd_reqid_t(),
@@ -1028,32 +1028,23 @@ OpsExecuter::flush_clone_metadata(
   });
 }
 
-// TODO: make this static
-std::pair<object_info_t, ObjectContextRef> OpsExecuter::prepare_clone(
+ObjectContextRef OpsExecuter::prepare_clone(
   const hobject_t& coid)
 {
-  object_info_t static_snap_oi(coid);
-  static_snap_oi.version = osd_op_params->at_version;
-  static_snap_oi.prior_version = obc->obs.oi.version;
-  static_snap_oi.copy_user_bits(obc->obs.oi);
-  if (static_snap_oi.is_whiteout()) {
-    // clone shouldn't be marked as whiteout
-    static_snap_oi.clear_flag(object_info_t::FLAG_WHITEOUT);
-  }
-
-  ObjectContextRef clone_obc;
-  if (pg->is_primary()) {
-    // lookup_or_create
-    auto [c_obc, existed] =
-      pg->obc_registry.get_cached_obc(std::move(coid));
-    assert(!existed);
-    c_obc->obs.oi = static_snap_oi;
-    c_obc->obs.exists = true;
-    c_obc->ssc = obc->ssc;
-    logger().debug("clone_obc: {}", c_obc->obs.oi);
-    clone_obc = std::move(c_obc);
-  }
-  return std::make_pair(std::move(static_snap_oi), std::move(clone_obc));
+  ceph_assert(pg->is_primary());
+  ObjectState clone_obs{coid};
+  clone_obs.exists = true;
+  clone_obs.oi.version = osd_op_params->at_version;
+  clone_obs.oi.prior_version = obc->obs.oi.version;
+  clone_obs.oi.copy_user_bits(obc->obs.oi);
+  clone_obs.oi.clear_flag(object_info_t::FLAG_WHITEOUT);
+
+  auto [clone_obc, existed] = pg->obc_registry.get_cached_obc(std::move(coid));
+  ceph_assert(!existed);
+
+  clone_obc->set_clone_state(std::move(clone_obs));
+  clone_obc->ssc = obc->ssc;
+  return clone_obc;
 }
 
 void OpsExecuter::apply_stats()
diff --git a/src/crimson/osd/ops_executer.h b/src/crimson/osd/ops_executer.h
index 1230b1c5a2e5..1d861d49bf1f 100644
--- a/src/crimson/osd/ops_executer.h
+++ b/src/crimson/osd/ops_executer.h
@@ -449,7 +449,7 @@ class OpsExecuter : public seastar::enable_lw_shared_from_this<OpsExecuter> {
 
   version_t get_last_user_version() const;
 
-  std::pair<object_info_t, ObjectContextRef> prepare_clone(
+  ObjectContextRef prepare_clone(
     const hobject_t& coid);
 
   void apply_stats();

From 7c4fdd3762831cb86993b1b4a814ea68a9ea2401 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 15 Jan 2024 09:29:23 +1000
Subject: [PATCH 1414/2492] doc/radosgw: edit admin.rst "Get Bucket Rate Limit"

Edit "Get Bucket Rate Limit" in doc/radosgw/admin.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index fe8e84b05062..c09f6d591d8c 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -679,8 +679,8 @@ limit.
 Get User Rate Limit
 -------------------
 
-The ``radosgw-admin ratelimit get`` command returns the current configured rate
-limit parameters.
+The ``radosgw-admin ratelimit get`` command returns the currently configured
+rate limit parameters.
 
 The following is the general form of the command that returns the current
 configured limit parameters:  
@@ -744,18 +744,26 @@ limit.
 Get Bucket Rate Limit
 ---------------------
 
-Get the current configured rate limit parameters
-For example:: 
+The ``radosgw-admin ratelimit get`` command returns the current configured rate
+limit parameters.
+
+The following is the general form of the command that returns the current
+configured limit parameters:
+
+.. prompt:: bash
 
-	radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket=<bucket>
+   radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket=<bucket>
 
-For example:: 
+An example of using ``radosgw-admin ratelimit get`` to return the rate limit
+parameters for a bucket might look like this:
 
-	radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket=mybucket
+.. prompt:: bash
 
+   radosgw-admin ratelimit get --ratelimit-scope=bucket --bucket=mybucket
 
-A 0 value for num ops and / or num bytes means that the
-specific rate limit attribute check is disabled.
+A value of ``0`` assigned to ``--max-read-ops``, ``--max-read-bytes``,
+``--max-write-ops``, or ``--max-write-bytes`` disables the specified rate
+limit.
 
 
 Enable/Disable Bucket Rate Limit

From 64a5afcaad7b61a20f630aca064c2953380c70c1 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 21 Dec 2023 18:08:21 +0100
Subject: [PATCH 1415/2492] test/librbd: improve
 TestMockObjectMapDiffRequest.InvalidStartSnap

Use a range where only snap_id_start is invalid.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/object_map/test_mock_DiffRequest.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index aefebf4a05bb..2d136e8ed75d 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -207,7 +207,7 @@ TEST_P(TestMockObjectMapDiffRequest, InvalidStartSnap) {
   InSequence seq;
 
   C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, CEPH_NOSNAP, 0,
+  auto req = new MockDiffRequest(&mock_image_ctx, CEPH_NOSNAP, CEPH_NOSNAP,
                                  is_diff_iterate(), &m_object_diff_state,
                                  &ctx);
   req->send();

From 718f6b5546890179f66d5ffadbae9e9cb0e6c97b Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 21 Dec 2023 16:50:31 +0100
Subject: [PATCH 1416/2492] test/librbd: refactor TestMockObjectMapDiffRequest
 tests

In preparation for multiple similarly configured MockTestImageCtx
objects being used in a single test, centralize their creation and add
a couple of helpers for setting expectations from a callback.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../object_map/test_mock_DiffRequest.cc       | 575 ++++++++----------
 1 file changed, 254 insertions(+), 321 deletions(-)

diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 2d136e8ed75d..fb52327e88cb 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -18,6 +18,8 @@ struct MockTestImageCtx : public MockImageCtx {
   }
 };
 
+void noop(MockTestImageCtx&) {}
+
 } // anonymous namespace
 } // namespace librbd
 
@@ -160,7 +162,7 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
   }
 
   bool is_diff_iterate() const {
-    return GetParam();
+    return !GetParam();
   }
 
   void expect_get_flags(MockTestImageCtx& mock_image_ctx, uint64_t snap_id,
@@ -197,69 +199,88 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
     expect_load_map(mock_image_ctx, snap_id, object_map, r, [](){});
   }
 
+  template <typename F>
+  int do_diff(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+              bool diff_iterate_range) {
+    InSequence seq;
+
+    MockTestImageCtx mock_image_ctx(*m_image_ctx);
+    std::forward<F>(f)(mock_image_ctx);
+
+    C_SaferCond ctx;
+    auto req = new MockDiffRequest(&mock_image_ctx, start_snap_id,
+                                   end_snap_id, diff_iterate_range,
+                                   &m_diff_state, &ctx);
+    req->send();
+    return ctx.wait();
+  }
+
+  template <typename F>
+  void test_diff_iterate(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+                         const BitVector<2>& expected_diff_state) {
+    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id, true));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+  }
+
+  template <typename F>
+  void test_deep_copy(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+                      const BitVector<2>& expected_diff_state) {
+    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id, false));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+  }
+
   librbd::ImageCtx* m_image_ctx = nullptr;
-  BitVector<2> m_object_diff_state;
+  BitVector<2> m_diff_state;
 };
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidStartSnap) {
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-
-  InSequence seq;
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, CEPH_NOSNAP, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-EINVAL, ctx.wait());
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-
-  InSequence seq;
+  BitVector<2> expected_diff_state;
 
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 1, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-  ASSERT_EQ(0U, m_object_diff_state.size());
+  if (is_diff_iterate()) {
+    ASSERT_EQ(0, do_diff(noop, 1, 1, true));
+  } else {
+    ASSERT_EQ(0, do_diff(noop, 1, 1, false));
+  }
+  ASSERT_EQ(expected_diff_state, m_diff_state);
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   // negative test -- object-map implicitly enables fast-diff
   REQUIRE(!is_feature_enabled(RBD_FEATURE_OBJECT_MAP));
 
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-
-  InSequence seq;
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-EINVAL, ctx.wait());
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FastDiffInvalid) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
   };
 
-  InSequence seq;
-  expect_get_flags(mock_image_ctx, 1U, RBD_FLAG_FAST_DIFF_INVALID, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-EINVAL, ctx.wait());
+  auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
@@ -267,10 +288,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
 
   uint32_t object_count = std::size(from_beginning_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -283,18 +302,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
     expected_diff_state[i] = from_beginning_table[i][1];
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, 1, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 1, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 1, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
@@ -302,12 +318,10 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
 
   uint32_t object_count = std::size(from_beginning_intermediate_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -327,21 +341,17 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
     }
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, 2U, 0, 0);
-  expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, 2, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 2, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
@@ -350,8 +360,6 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
   uint32_t object_count = std::size(from_beginning_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
 
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-
   BitVector<2> object_map_head;
   object_map_head.resize(object_count);
   BitVector<2> expected_diff_state;
@@ -361,19 +369,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
     expected_diff_state[i] = from_beginning_table[i][1];
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
@@ -381,10 +385,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
 
   uint32_t object_count = std::size(from_beginning_intermediate_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -404,22 +406,17 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
     }
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
@@ -427,12 +424,10 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
 
   uint32_t object_count = std::size(from_snap_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -448,21 +443,17 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
     expected_diff_state[i] = from_snap_table[i][2];
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, 2U, 0, 0);
-  expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 2, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
@@ -470,14 +461,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
 
   uint32_t object_count = std::size(from_snap_intermediate_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {3U, {"snap3", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {3U, {"snap3", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -500,24 +489,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
     }
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, 2U, 0, 0);
-  expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
-
-  expect_get_flags(mock_image_ctx, 3U, 0, 0);
-  expect_load_map(mock_image_ctx, 3U, object_map_3, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 3, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    expect_get_flags(mock_image_ctx, 3, 0, 0);
+    expect_load_map(mock_image_ctx, 3, object_map_3, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 3, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 3, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
@@ -525,10 +509,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
 
   uint32_t object_count = std::size(from_snap_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -544,22 +526,17 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
     expected_diff_state[i] = from_snap_table[i][2];
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
@@ -567,12 +544,10 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
 
   uint32_t object_count = std::size(from_snap_intermediate_table);
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -595,25 +570,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
     }
   }
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  expect_get_flags(mock_image_ctx, 2U, 0, 0);
-  expect_load_map(mock_image_ctx, 2U, object_map_2, 0);
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
@@ -621,21 +590,16 @@ TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-ENOENT, ctx.wait());
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
@@ -643,26 +607,23 @@ TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
   BitVector<2> object_map_1;
   object_map_1.resize(object_count);
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
 
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 1, 2, is_diff_iterate(),
-                                 &m_object_diff_state, &ctx);
-  req->send();
-  ASSERT_EQ(-ENOENT, ctx.wait());
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, true));
+  } else {
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
@@ -670,43 +631,34 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
   BitVector<2> object_map_1;
   object_map_1.resize(object_count);
-  object_map_1[1] = OBJECT_EXISTS_CLEAN;
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0,
-                  [&mock_image_ctx]() { mock_image_ctx.snap_info.erase(2); });
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-
   BitVector<2> object_map_head;
   object_map_head.resize(object_count);
   object_map_head[1] = OBJECT_EXISTS_CLEAN;
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
   BitVector<2> expected_diff_state;
   expected_diff_state.resize(object_count);
   expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0,
+                    [&mock_image_ctx]() { mock_image_ctx.snap_info.erase(2); });
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
@@ -715,21 +667,17 @@ TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
 
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-
   BitVector<2> object_map_head;
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, -ENOENT);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-ENOENT, ctx.wait());
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, -ENOENT);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
@@ -737,38 +685,30 @@ TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
   BitVector<2> object_map_1;
-  expect_load_map(mock_image_ctx, 1U, object_map_1, -ENOENT);
-
-  expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-
   BitVector<2> object_map_head;
   object_map_head.resize(object_count);
   object_map_head[1] = OBJECT_EXISTS_CLEAN;
-  expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(0, ctx.wait());
-
   BitVector<2> expected_diff_state;
   expected_diff_state.resize(object_count);
   expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
-  ASSERT_EQ(expected_diff_state, m_object_diff_state);
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapError) {
@@ -776,26 +716,22 @@ TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapError) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
   BitVector<2> object_map_1;
-  expect_load_map(mock_image_ctx, 1U, object_map_1, -EPERM);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-EPERM, ctx.wait());
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, false));
+  }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
@@ -803,26 +739,23 @@ TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-
-  MockTestImageCtx mock_image_ctx(*m_image_ctx);
-  mock_image_ctx.snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, mock_image_ctx.size, {},
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
-  InSequence seq;
-
-  expect_get_flags(mock_image_ctx, 1U, 0, 0);
-
   BitVector<2> object_map_1;
-  expect_load_map(mock_image_ctx, 1U, object_map_1, 0);
-
-  C_SaferCond ctx;
-  auto req = new MockDiffRequest(&mock_image_ctx, 0, CEPH_NOSNAP,
-                                 is_diff_iterate(), &m_object_diff_state,
-                                 &ctx);
-  req->send();
-  ASSERT_EQ(-EINVAL, ctx.wait());
+  object_map_1.resize(object_count - 1);
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, false));
+  }
 }
 
 INSTANTIATE_TEST_SUITE_P(MockObjectMapDiffRequestTests,

From 4e036d65b9146f28a2f6c0dfb353120baea8d62d Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 20 Dec 2023 12:22:17 +0100
Subject: [PATCH 1417/2492] librbd/object_map: drop bogus if in
 handle_load_object_map()

It became redundant with commit b81cd2460de7 ("librbd/object_map: diff
state machine should track object existence") -- it != end_it condition
in the loop is sufficient.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc | 72 ++++++++++++++--------------
 1 file changed, 35 insertions(+), 37 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 5b785aeaf3c7..7fc3c900c14d 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -235,48 +235,46 @@ void DiffRequest<I>::handle_load_object_map(int r) {
 
   bool diff_from_start = (m_snap_id_start == 0);
   auto end_it = m_object_map.end();
-  if (m_object_map.size() > prev_object_diff_state_size) {
-    for (; it != end_it; ++it,++diff_it, ++i) {
-      uint8_t object_map_state = *it;
-      if (object_map_state == OBJECT_NONEXISTENT) {
-        *diff_it = DIFF_STATE_HOLE;
-      } else if (diff_from_start ||
-                 (m_object_diff_state_valid &&
-                  object_map_state != OBJECT_EXISTS_CLEAN)) {
-        // diffing against the beginning of time or image was grown
-        // (implicit) starting state is HOLE, this is the first object
-        // map after
-        if (m_diff_iterate_range) {
-          // for diff-iterate, if the object is discarded prior to or
-          // in the end version, result should be HOLE
-          // since DATA_UPDATED can transition only to HOLE_UPDATED,
-          // stay in HOLE on intermediate snapshots -- another way to
-          // put this is that when starting with a hole, intermediate
-          // snapshots can be ignored as the result depends only on the
-          // end version
-          if (m_current_snap_id == m_snap_id_end) {
-            *diff_it = DIFF_STATE_DATA_UPDATED;
-          } else {
-            *diff_it = DIFF_STATE_HOLE;
-          }
-        } else {
-          // for deep-copy, if the object is discarded prior to or
-          // in the end version, result should be HOLE_UPDATED
+  for (; it != end_it; ++it, ++diff_it, ++i) {
+    uint8_t object_map_state = *it;
+    if (object_map_state == OBJECT_NONEXISTENT) {
+      *diff_it = DIFF_STATE_HOLE;
+    } else if (diff_from_start ||
+               (m_object_diff_state_valid &&
+                object_map_state != OBJECT_EXISTS_CLEAN)) {
+      // diffing against the beginning of time or image was grown
+      // (implicit) starting state is HOLE, this is the first object
+      // map after
+      if (m_diff_iterate_range) {
+        // for diff-iterate, if the object is discarded prior to or
+        // in the end version, result should be HOLE
+        // since DATA_UPDATED can transition only to HOLE_UPDATED,
+        // stay in HOLE on intermediate snapshots -- another way to
+        // put this is that when starting with a hole, intermediate
+        // snapshots can be ignored as the result depends only on the
+        // end version
+        if (m_current_snap_id == m_snap_id_end) {
           *diff_it = DIFF_STATE_DATA_UPDATED;
-        }
-      } else {
-        // diffing against a snapshot, this is its object map
-        if (object_map_state != OBJECT_PENDING) {
-          *diff_it = DIFF_STATE_DATA;
         } else {
-          *diff_it = DIFF_STATE_DATA_UPDATED;
+          *diff_it = DIFF_STATE_HOLE;
         }
+      } else {
+        // for deep-copy, if the object is discarded prior to or
+        // in the end version, result should be HOLE_UPDATED
+        *diff_it = DIFF_STATE_DATA_UPDATED;
+      }
+    } else {
+      // diffing against a snapshot, this is its object map
+      if (object_map_state != OBJECT_PENDING) {
+        *diff_it = DIFF_STATE_DATA;
+      } else {
+        *diff_it = DIFF_STATE_DATA_UPDATED;
       }
-
-      ldout(cct, 20) << "object state: " << i << " "
-                     << "->" << static_cast<uint32_t>(*diff_it) << " ("
-                     << static_cast<uint32_t>(*it) << ")" << dendl;
     }
+
+    ldout(cct, 20) << "object state: " << i << " "
+                   << "->" << static_cast<uint32_t>(*diff_it) << " ("
+                   << static_cast<uint32_t>(*it) << ")" << dendl;
   }
   ldout(cct, 20) << "computed resize diffs" << dendl;
 

From 34386d29a8b96de79c38465062b8e93e7fc6e184 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 22 Dec 2023 16:10:12 +0100
Subject: [PATCH 1418/2492] librbd/object_map: fix diff from snapshot when
 image is grown

Commit 399a45e11332 ("librbd/object_map: rbd diff between two
snapshots lists entire image content") fixed most of the damage caused
by commit b81cd2460de7 ("librbd/object_map: diff state machine should
track object existence"), but the case of a "resize diff" when diffing
from snapshot was missed.  An area that was freshly allocated in image
resize is the same in principle as a freshly created image and objects
marked OBJECT_EXISTS_CLEAN are no exception.  Diff for such objects in
such an area should be set to DIFF_STATE_DATA_UPDATED, however
currently when diffing from snapshot, it's set to DIFF_STATE_DATA.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc          |   7 +-
 src/librbd/object_map/DiffRequest.h           |   1 -
 .../object_map/test_mock_DiffRequest.cc       | 304 ++++++++++++++++++
 3 files changed, 305 insertions(+), 7 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 7fc3c900c14d..08a25b026527 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -233,15 +233,12 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   }
   ldout(cct, 20) << "computed overlap diffs" << dendl;
 
-  bool diff_from_start = (m_snap_id_start == 0);
   auto end_it = m_object_map.end();
   for (; it != end_it; ++it, ++diff_it, ++i) {
     uint8_t object_map_state = *it;
     if (object_map_state == OBJECT_NONEXISTENT) {
       *diff_it = DIFF_STATE_HOLE;
-    } else if (diff_from_start ||
-               (m_object_diff_state_valid &&
-                object_map_state != OBJECT_EXISTS_CLEAN)) {
+    } else if (m_current_snap_id != m_snap_id_start) {
       // diffing against the beginning of time or image was grown
       // (implicit) starting state is HOLE, this is the first object
       // map after
@@ -278,8 +275,6 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   }
   ldout(cct, 20) << "computed resize diffs" << dendl;
 
-  m_object_diff_state_valid = true;
-
   std::shared_lock image_locker{m_image_ctx->image_lock};
   load_object_map(&image_locker);
 }
diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
index b02ef34ba347..f2fb9975f751 100644
--- a/src/librbd/object_map/DiffRequest.h
+++ b/src/librbd/object_map/DiffRequest.h
@@ -69,7 +69,6 @@ class DiffRequest {
   uint64_t m_current_size = 0;
 
   BitVector<2> m_object_map;
-  bool m_object_diff_state_valid = false;
 
   bufferlist m_out_bl;
 
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index fb52327e88cb..74dd2215ff38 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -354,6 +354,86 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrow) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(from_beginning_intermediate_table);
+  uint32_t object_count_2 = object_count_1 + std::size(from_beginning_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_2);
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_2[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+  for (uint32_t i = object_count_1; i < object_count_2; i++) {
+    object_map_2[i] = from_beginning_table[i - object_count_1][0];
+    expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 2, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrowFromZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_2 = std::size(from_beginning_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_2);
+  for (uint32_t i = 0; i < object_count_2; i++) {
+    object_map_2[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 2, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -419,6 +499,82 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrow) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(from_beginning_intermediate_table);
+  uint32_t object_count_head = object_count_1 + std::size(from_beginning_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_head);
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_head[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+  for (uint32_t i = object_count_1; i < object_count_head; i++) {
+    object_map_head[i] = from_beginning_table[i - object_count_1][0];
+    expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrowFromZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_head = std::size(from_beginning_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_head);
+  for (uint32_t i = 0; i < object_count_head; i++) {
+    object_map_head[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -456,6 +612,82 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapGrow) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(from_snap_table);
+  uint32_t object_count_2 = object_count_1 + std::size(from_beginning_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_2);
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_2[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+  for (uint32_t i = object_count_1; i < object_count_2; i++) {
+    object_map_2[i] = from_beginning_table[i - object_count_1][0];
+    expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 2, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapGrowFromZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_2 = std::size(from_beginning_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_2);
+  for (uint32_t i = 0; i < object_count_2; i++) {
+    object_map_2[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 2, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -539,6 +771,78 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadGrow) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(from_snap_table);
+  uint32_t object_count_head = object_count_1 + std::size(from_beginning_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_head);
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_head[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+  for (uint32_t i = object_count_1; i < object_count_head; i++) {
+    object_map_head[i] = from_beginning_table[i - object_count_1][0];
+    expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadGrowFromZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_head = std::size(from_beginning_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count_head);
+  for (uint32_t i = 0; i < object_count_head; i++) {
+    object_map_head[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 

From 19c7c4a5359fa9c1d06cc11187e300251249ad9e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 22 Dec 2023 18:50:20 +0100
Subject: [PATCH 1419/2492] librbd/object_map: resurrect diff-iterate behavior
 when image is shrunk

The new "track the largest of all versions in the set, diff state is
only ever grown" semantics introduced in commit 330f2a7bb94f ("librbd:
helper state machine for computing diffs between object-maps") don't
make sense for diff-iterate.  It's a waste because DiffIterate won't
query beyond the end version size -- this is baked into the API.

Limit this behavior to deep-copy and resurrect the original behavior
from 2015 for diff-iterate.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc          |  24 +-
 .../object_map/test_mock_DiffRequest.cc       | 354 ++++++++++++++++++
 2 files changed, 369 insertions(+), 9 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 08a25b026527..d14367010ed7 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -176,15 +176,21 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   }
 
   uint64_t prev_object_diff_state_size = m_object_diff_state->size();
-  if (prev_object_diff_state_size < num_objs) {
-    // the diff state should be the largest of all snapshots in the set
-    m_object_diff_state->resize(num_objs);
-  }
-  if (m_object_map.size() < m_object_diff_state->size()) {
-    // the image was shrunk so expanding the object map will flag end objects
-    // as non-existent and they will be compared against the previous object
-    // diff state
-    m_object_map.resize(m_object_diff_state->size());
+  if (m_diff_iterate_range) {
+    if (m_object_diff_state->size() != m_object_map.size()) {
+      m_object_diff_state->resize(m_object_map.size());
+    }
+  } else {
+    // for deep-copy, the object diff state should be the largest of
+    // all versions in the set, so it's only ever grown
+    if (m_object_diff_state->size() < m_object_map.size()) {
+      m_object_diff_state->resize(m_object_map.size());
+    } else if (m_object_diff_state->size() > m_object_map.size()) {
+      // the image was shrunk so expanding the object map will flag end objects
+      // as non-existent and they will be compared against the previous object
+      // diff state
+      m_object_map.resize(m_object_diff_state->size());
+    }
   }
 
   uint64_t overlap = std::min(m_object_map.size(), prev_object_diff_state_size);
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 74dd2215ff38..8d9ded985a26 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -150,6 +150,14 @@ static constexpr uint8_t from_snap_intermediate_table[][5] = {
   { OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  OBJECT_EXISTS_CLEAN,  DIFF_STATE_DATA,          DIFF_STATE_DATA }
 };
 
+static constexpr uint8_t shrink_table[][2] = {
+  //      shrunk             deep-copy expected
+  { OBJECT_NONEXISTENT,   DIFF_STATE_HOLE },
+  { OBJECT_EXISTS,        DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_PENDING,       DIFF_STATE_HOLE_UPDATED },
+  { OBJECT_EXISTS_CLEAN,  DIFF_STATE_HOLE_UPDATED }
+};
+
 class TestMockObjectMapDiffRequest : public TestMockFixture,
                                      public ::testing::WithParamInterface<bool> {
 public:
@@ -434,6 +442,96 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrowFrom
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrink) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_2 = std::size(from_beginning_intermediate_table);
+  uint32_t object_count_1 = object_count_2 + std::size(shrink_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  if (is_diff_iterate()) {
+    expected_diff_state.resize(object_count_2);
+  } else {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_2; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_2[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+  for (uint32_t i = object_count_2; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i - object_count_2][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i - object_count_2][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 2, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrinkToZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(shrink_table);
+  m_image_ctx->size = 0;
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  BitVector<2> expected_diff_state;
+  if (!is_diff_iterate()) {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 2, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -575,6 +673,93 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrowFrom
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrink) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_head = std::size(from_beginning_intermediate_table);
+  uint32_t object_count_1 = object_count_head + std::size(shrink_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  if (is_diff_iterate()) {
+    expected_diff_state.resize(object_count_head);
+  } else {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_head; i++) {
+    object_map_1[i] = from_beginning_intermediate_table[i][0];
+    object_map_head[i] = from_beginning_intermediate_table[i][1];
+    if (is_diff_iterate()) {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][2];
+    } else {
+      expected_diff_state[i] = from_beginning_intermediate_table[i][3];
+    }
+  }
+  for (uint32_t i = object_count_head; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i - object_count_head][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i - object_count_head][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrinkToZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(shrink_table);
+  m_image_ctx->size = 0;
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  BitVector<2> expected_diff_state;
+  if (!is_diff_iterate()) {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -688,6 +873,92 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapGrowFromZero) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapShrink) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_2 = std::size(from_snap_table);
+  uint32_t object_count_1 = object_count_2 + std::size(shrink_table);
+  m_image_ctx->size = object_count_2 * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_2 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count_2);
+  BitVector<2> expected_diff_state;
+  if (is_diff_iterate()) {
+    expected_diff_state.resize(object_count_2);
+  } else {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_2; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_2[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+  for (uint32_t i = object_count_2; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i - object_count_2][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i - object_count_2][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 2, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapShrinkToZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(shrink_table);
+  m_image_ctx->size = 0;
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_2;
+  BitVector<2> expected_diff_state;
+  if (!is_diff_iterate()) {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, 2, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, 2, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -843,6 +1114,89 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadGrowFromZero) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadShrink) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_head = std::size(from_snap_table);
+  uint32_t object_count_1 = object_count_head + std::size(shrink_table);
+  m_image_ctx->size = object_count_head * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count_head);
+  BitVector<2> expected_diff_state;
+  if (is_diff_iterate()) {
+    expected_diff_state.resize(object_count_head);
+  } else {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_head; i++) {
+    object_map_1[i] = from_snap_table[i][0];
+    object_map_head[i] = from_snap_table[i][1];
+    expected_diff_state[i] = from_snap_table[i][2];
+  }
+  for (uint32_t i = object_count_head; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i - object_count_head][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i - object_count_head][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadShrinkToZero) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count_1 = std::size(shrink_table);
+  m_image_ctx->size = 0;
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}},
+          object_count_1 * (1 << m_image_ctx->order), {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count_1);
+  BitVector<2> object_map_head;
+  BitVector<2> expected_diff_state;
+  if (!is_diff_iterate()) {
+    expected_diff_state.resize(object_count_1);
+  }
+  for (uint32_t i = 0; i < object_count_1; i++) {
+    object_map_1[i] = shrink_table[i][0];
+    if (!is_diff_iterate()) {
+      expected_diff_state[i] = shrink_table[i][1];
+    }
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 

From 23c675f04a4b4bb00794bbc59493d9591625a0a7 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 23 Dec 2023 14:47:54 +0100
Subject: [PATCH 1420/2492] librbd/object_map: allow intermediate snaps to be
 skipped on diff-iterate

In case of diff-iterate against the beginning of time, the result
depends only on the end version.  Loading and processing object maps
or intermediate snapshots is redundant and can be skipped.

This optimization is made possible by commit be507aaed15f ("librbd:
diff-iterate shouldn't ever report "new hole" against a hole") and, to
a lesser extent, the previous commit.

Getting FastDiffInvalid, LoadObjectMapError and ObjectMapTooSmall to
pass required tweaking not just expectations, but also start/end snap
ids and thus also the meaning of these tests.  This is addressed in the
next commit.

Fixes: https://tracker.ceph.com/issues/63341
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc          |  21 +-
 .../object_map/test_mock_DiffRequest.cc       | 218 +++++++++++-------
 2 files changed, 148 insertions(+), 91 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index d14367010ed7..4d23bcbad4da 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -38,16 +38,19 @@ void DiffRequest<I>::send() {
 
   m_object_diff_state->clear();
 
-  // collect all the snap ids in the provided range (inclusive)
-  if (m_snap_id_start != 0) {
-    m_snap_ids.insert(m_snap_id_start);
-  }
-
+  // collect all the snap ids in the provided range (inclusive) unless
+  // this is diff-iterate against the beginning of time, in which case
+  // only the end version matters
   std::shared_lock image_locker{m_image_ctx->image_lock};
-  auto snap_info_it = m_image_ctx->snap_info.upper_bound(m_snap_id_start);
-  auto snap_info_it_end = m_image_ctx->snap_info.lower_bound(m_snap_id_end);
-  for (; snap_info_it != snap_info_it_end; ++snap_info_it) {
-    m_snap_ids.insert(snap_info_it->first);
+  if (!m_diff_iterate_range || m_snap_id_start != 0) {
+    if (m_snap_id_start != 0) {
+      m_snap_ids.insert(m_snap_id_start);
+    }
+    auto snap_info_it = m_image_ctx->snap_info.upper_bound(m_snap_id_start);
+    auto snap_info_it_end = m_image_ctx->snap_info.lower_bound(m_snap_id_end);
+    for (; snap_info_it != snap_info_it_end; ++snap_info_it) {
+      m_snap_ids.insert(snap_info_it->first);
+    }
   }
   m_snap_ids.insert(m_snap_id_end);
 
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 8d9ded985a26..8646f09354be 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -271,13 +271,15 @@ TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, FastDiffInvalid) {
+TEST_P(TestMockObjectMapDiffRequest, StartFastDiffInvalid) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
   m_image_ctx->snap_info = {
     {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -285,9 +287,9 @@ TEST_P(TestMockObjectMapDiffRequest, FastDiffInvalid) {
     expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, true));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, false));
   }
 }
 
@@ -349,15 +351,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, 2, 0, 0);
-    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_diff_iterate(load, 0, 2, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_deep_copy(load, 0, 2, expected_diff_state);
   }
 }
@@ -395,15 +401,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrow) {
     expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, 2, 0, 0);
-    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_diff_iterate(load, 0, 2, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_deep_copy(load, 0, 2, expected_diff_state);
   }
 }
@@ -429,15 +439,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrowFrom
     expected_diff_state[i] = from_beginning_table[i][1];
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, 2, 0, 0);
-    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_diff_iterate(load, 0, 2, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_deep_copy(load, 0, 2, expected_diff_state);
   }
 }
@@ -481,15 +495,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrink)
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, 2, 0, 0);
-    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_diff_iterate(load, 0, 2, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_deep_copy(load, 0, 2, expected_diff_state);
   }
 }
@@ -519,15 +537,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrinkTo
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, 2, 0, 0);
-    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_diff_iterate(load, 0, 2, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
     test_deep_copy(load, 0, 2, expected_diff_state);
   }
 }
@@ -584,15 +606,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -628,15 +654,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrow) {
     expected_diff_state[i] = from_beginning_table[i - object_count_1][1];
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -660,15 +690,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrowFrom
     expected_diff_state[i] = from_beginning_table[i][1];
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -710,15 +744,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrink)
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -747,15 +785,19 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrinkTo
     }
   }
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -1305,16 +1347,20 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
   expected_diff_state.resize(object_count);
   expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0,
-                    [&mock_image_ctx]() { mock_image_ctx.snap_info.erase(2); });
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0,
+                      [&mock_image_ctx]() { mock_image_ctx.snap_info.erase(2); });
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
@@ -1356,26 +1402,32 @@ TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
   expected_diff_state.resize(object_count);
   expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
-    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
-  };
   if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
     test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapError) {
+TEST_P(TestMockObjectMapDiffRequest, StartObjectMapLoadError) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
   m_image_ctx->snap_info = {
     {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -1386,19 +1438,21 @@ TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapError) {
     expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, true));
   } else {
-    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, false));
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
+TEST_P(TestMockObjectMapDiffRequest, StartObjectMapTooSmall) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
   m_image_ctx->snap_info = {
     {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
           {}, {}, {}}}
   };
 
@@ -1410,9 +1464,9 @@ TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooSmall) {
     expect_load_map(mock_image_ctx, 1, object_map_1, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, true));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, false));
   }
 }
 

From 9931282bfd2260d654325970555cab8c617e8f14 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 23 Dec 2023 15:19:09 +0100
Subject: [PATCH 1421/2492] test/librbd: expand TestMockObjectMapDiffRequest
 edge case coverage

For each covered edge case or error, run through the following
scenarios:

- where the edge case concerns snap_id_start
- where the edge case concerns snap_id_end
- where the edge case concerns intermediate snapshot and
  snap_id_start == 0 (diff against the beginning of time)
- where the edge case concerns intermediate snapshot and
  snap_id_start != 0 (diff from snapshot)

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 .../object_map/test_mock_DiffRequest.cc       | 474 ++++++++++++++++--
 1 file changed, 438 insertions(+), 36 deletions(-)

diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 8646f09354be..9be4995e00a7 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -271,28 +271,6 @@ TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, StartFastDiffInvalid) {
-  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
-
-  uint32_t object_count = 5;
-  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
-  m_image_ctx->snap_info = {
-    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
-          {}, {}, {}}},
-    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
-          {}, {}, {}}}
-  };
-
-  auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
-  };
-  if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, true));
-  } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, false));
-  }
-}
-
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -1296,9 +1274,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
   };
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, CEPH_NOSNAP, true));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, true));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, CEPH_NOSNAP, false));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, false));
   }
 }
 
@@ -1315,18 +1293,18 @@ TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
   BitVector<2> object_map_1;
   object_map_1.resize(object_count);
 
-  auto load = [&](MockTestImageCtx& mock_image_ctx) {
-    expect_get_flags(mock_image_ctx, 1, 0, 0);
-    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
-  };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, true));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 0, 2, true));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, false));
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    };
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, false));
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNEFromBeginning) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -1365,26 +1343,105 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNE) {
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, LoadObjectMapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNEFromSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
   m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
 
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
   BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
 
   auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0,
+                    [&mock_image_ctx]() { mock_image_ctx.snap_info.erase(2); });
     expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
-    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, -ENOENT);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, CEPH_NOSNAP, true));
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, CEPH_NOSNAP, false));
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
-TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
+TEST_P(TestMockObjectMapDiffRequest, StartObjectMapDNE) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, true));
+  } else {
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, EndObjectMapDNE) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  object_map_2[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
+    };
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, true));
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
+    };
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapDNEFromBeginning) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
   uint32_t object_count = 5;
@@ -1419,6 +1476,153 @@ TEST_P(TestMockObjectMapDiffRequest, LoadIntermediateObjectMapDNE) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapDNEFromSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, StartFastDiffInvalid) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, EndFastDiffInvalid) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+
+  if (is_diff_iterate()) {
+    auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, true));
+  } else {
+    auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromBeginning) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+
+  auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, false));
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, StartObjectMapLoadError) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -1444,6 +1648,106 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapLoadError) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, EndObjectMapLoadError) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count);
+  object_map_2[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
+    };
+    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, true));
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
+    };
+    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromBeginning) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
+    };
+    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, false));
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, StartObjectMapTooSmall) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -1470,6 +1774,104 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapTooSmall) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, EndObjectMapTooSmall) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count - 1);
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, true));
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromBeginning) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count - 1);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, false));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}},
+    {2U, {"snap2", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> object_map_2;
+  object_map_2.resize(object_count - 1);
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+    expect_get_flags(mock_image_ctx, 2, 0, 0);
+    expect_load_map(mock_image_ctx, 2, object_map_2, 0);
+  };
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, true));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, false));
+  }
+}
+
 INSTANTIATE_TEST_SUITE_P(MockObjectMapDiffRequestTests,
                          TestMockObjectMapDiffRequest, ::testing::Bool());
 

From 94bf3a5d74cfd61e0bd881ca2863fe2205f27767 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 27 Dec 2023 16:18:50 +0100
Subject: [PATCH 1422/2492] include/intarith: introduce round_down_to()

Same as with round_up_to(), d isn't required to be a power of two.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/include/intarith.h        | 4 ++++
 src/librbd/api/DiffIterate.cc | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/include/intarith.h b/src/include/intarith.h
index 68b0345a457e..92f827d88e58 100644
--- a/src/include/intarith.h
+++ b/src/include/intarith.h
@@ -25,6 +25,10 @@ constexpr inline std::make_unsigned_t<std::common_type_t<T, U>> div_round_up(T n
   return (n + d - 1) / d;
 }
 
+template<typename T, typename U>
+constexpr inline std::make_unsigned_t<std::common_type_t<T, U>> round_down_to(T n, U d) {
+  return n - n % d;
+}
 
 template<typename T, typename U>
 constexpr inline std::make_unsigned_t<std::common_type_t<T, U>> round_up_to(T n, U d) {
diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index 7679bf77759d..4bbc8ebdc98a 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -293,7 +293,7 @@ int DiffIterate<I>::execute() {
   uint64_t left = m_length;
 
   while (left > 0) {
-    uint64_t period_off = off - (off % period);
+    uint64_t period_off = round_down_to(off, period);
     uint64_t read_len = std::min(period_off + period - off, left);
 
     if (fast_diff_enabled) {

From 0b5ba5fedf704ada74a65108af129eae6baea5c5 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 27 Dec 2023 18:07:05 +0100
Subject: [PATCH 1423/2492] librbd/object_map: add support for ranged
 diff-iterate

Currently diff-iterate in fast-diff mode is performed on the entire
image no matter what image extent is passed to the API.  Then, unused
diff just gets discarded as DiffIterate ends up querying only objects
that the passed image extent maps to.  This hasn't been an issue for
internal consumers ("rbd du", "rbd diff", etc) because they work on the
entire image, but turns out to lead to quadratic slowdown in some QEMU
use cases.

0..UINT64_MAX range is carved out for deep-copy which is unranged by
definition.  To get effectively unranged diff-iterate, 0..UINT64_MAX-1
range can be used.

Fixes: https://tracker.ceph.com/issues/63341
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/api/DiffIterate.cc                 |  34 +++-
 src/librbd/api/DiffIterate.h                  |   3 +
 src/librbd/deep_copy/ImageCopyRequest.cc      |   2 +-
 src/librbd/object_map/DiffRequest.cc          |  82 +++++++--
 src/librbd/object_map/DiffRequest.h           |  24 +--
 .../deep_copy/test_mock_ImageCopyRequest.cc   |   2 +-
 .../object_map/test_mock_DiffRequest.cc       | 169 ++++++++++++++----
 7 files changed, 243 insertions(+), 73 deletions(-)

diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index 4bbc8ebdc98a..a464b942ae09 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -211,6 +211,29 @@ int DiffIterate<I>::diff_iterate(I *ictx,
   return r;
 }
 
+template <typename I>
+std::pair<uint64_t, uint64_t> DiffIterate<I>::calc_object_diff_range() {
+  uint64_t period = m_image_ctx.get_stripe_period();
+  uint64_t first_period_off = round_down_to(m_offset, period);
+  uint64_t last_period_off = round_down_to(m_offset + m_length - 1, period);
+
+  striper::LightweightObjectExtents object_extents;
+  if (first_period_off != last_period_off) {
+    // map only the tail of the first period and the front of the last
+    // period instead of the entire range for efficiency
+    Striper::file_to_extents(m_image_ctx.cct, &m_image_ctx.layout,
+                             m_offset, first_period_off + period - m_offset,
+                             0, 0, &object_extents);
+    Striper::file_to_extents(m_image_ctx.cct, &m_image_ctx.layout,
+                        last_period_off, m_offset + m_length - last_period_off,
+                        0, 0, &object_extents);
+  } else {
+    Striper::file_to_extents(m_image_ctx.cct, &m_image_ctx.layout, m_offset,
+                             m_length, 0, 0, &object_extents);
+  }
+  return {object_extents.front().object_no, object_extents.back().object_no + 1};
+}
+
 template <typename I>
 int DiffIterate<I>::execute() {
   CephContext* cct = m_image_ctx.cct;
@@ -245,20 +268,24 @@ int DiffIterate<I>::execute() {
 
   int r;
   bool fast_diff_enabled = false;
+  uint64_t start_object_no, end_object_no;
   BitVector<2> object_diff_state;
   interval_set<uint64_t> parent_diff;
   if (m_whole_object) {
+    std::tie(start_object_no, end_object_no) = calc_object_diff_range();
+
     C_SaferCond ctx;
     auto req = object_map::DiffRequest<I>::create(&m_image_ctx, from_snap_id,
-                                                  end_snap_id, true,
+                                                  end_snap_id, start_object_no,
+                                                  end_object_no,
                                                   &object_diff_state, &ctx);
     req->send();
-
     r = ctx.wait();
     if (r < 0) {
       ldout(cct, 5) << "fast diff disabled" << dendl;
     } else {
       ldout(cct, 5) << "fast diff enabled" << dendl;
+      ceph_assert(object_diff_state.size() == end_object_no - start_object_no);
       fast_diff_enabled = true;
 
       // check parent overlap only if we are comparing to the beginning of time
@@ -308,7 +335,8 @@ int DiffIterate<I>::execute() {
       io::SparseExtents aggregate_sparse_extents;
       for (auto& [object, extents] : object_extents) {
         const uint64_t object_no = extents.front().objectno;
-        uint8_t diff_state = object_diff_state[object_no];
+        ceph_assert(object_no >= start_object_no && object_no < end_object_no);
+        uint8_t diff_state = object_diff_state[object_no - start_object_no];
         ldout(cct, 20) << "object " << object << ": diff_state="
                        << (int)diff_state << dendl;
 
diff --git a/src/librbd/api/DiffIterate.h b/src/librbd/api/DiffIterate.h
index e6074d9cb3d6..8ce7f6afd7c3 100644
--- a/src/librbd/api/DiffIterate.h
+++ b/src/librbd/api/DiffIterate.h
@@ -7,6 +7,7 @@
 #include "include/int_types.h"
 #include "common/bit_vector.hpp"
 #include "cls/rbd/cls_rbd_types.h"
+#include <utility>
 
 namespace librbd {
 
@@ -51,6 +52,8 @@ class DiffIterate {
   {
   }
 
+  std::pair<uint64_t, uint64_t> calc_object_diff_range();
+
   int execute();
 
   int diff_object_map(uint64_t from_snap_id, uint64_t to_snap_id,
diff --git a/src/librbd/deep_copy/ImageCopyRequest.cc b/src/librbd/deep_copy/ImageCopyRequest.cc
index 0ba9bc82f18d..7f684c5b0a53 100644
--- a/src/librbd/deep_copy/ImageCopyRequest.cc
+++ b/src/librbd/deep_copy/ImageCopyRequest.cc
@@ -110,7 +110,7 @@ void ImageCopyRequest<I>::compute_diff() {
     ImageCopyRequest<I>, &ImageCopyRequest<I>::handle_compute_diff>(this);
   auto req = object_map::DiffRequest<I>::create(m_src_image_ctx,
                                                 m_src_snap_id_start,
-                                                m_src_snap_id_end, false,
+                                                m_src_snap_id_end, 0, UINT64_MAX,
                                                 &m_object_diff_state, ctx);
   req->send();
 }
diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 4d23bcbad4da..b8666fb9809a 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -20,6 +20,29 @@ namespace object_map {
 
 using util::create_rados_callback;
 
+template <typename I>
+DiffRequest<I>::DiffRequest(I* image_ctx,
+                            uint64_t snap_id_start, uint64_t snap_id_end,
+                            uint64_t start_object_no, uint64_t end_object_no,
+                            BitVector<2>* object_diff_state,
+                            Context* on_finish)
+    : m_image_ctx(image_ctx), m_snap_id_start(snap_id_start),
+      m_snap_id_end(snap_id_end), m_start_object_no(start_object_no),
+      m_end_object_no(end_object_no), m_object_diff_state(object_diff_state),
+      m_on_finish(on_finish) {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << "snap_id_start=" << m_snap_id_start
+                 << ", snap_id_end=" << m_snap_id_end
+                 << ", start_object_no=" << m_start_object_no
+                 << ", end_object_no=" << m_end_object_no
+                 << dendl;
+}
+
+template <typename I>
+bool DiffRequest<I>::is_diff_iterate() const {
+  return m_start_object_no != 0 || m_end_object_no != UINT64_MAX;
+}
+
 template <typename I>
 void DiffRequest<I>::send() {
   auto cct = m_image_ctx->cct;
@@ -30,19 +53,34 @@ void DiffRequest<I>::send() {
                << "snap_id_end=" << m_snap_id_end << dendl;
     finish(-EINVAL);
     return;
-  } else if (m_snap_id_start == m_snap_id_end) {
-    // no delta between the same snapshot
-    finish(0);
+  }
+  if (m_start_object_no == UINT64_MAX || m_start_object_no > m_end_object_no ||
+      (m_start_object_no != 0 && m_end_object_no == UINT64_MAX)) {
+    lderr(cct) << "invalid start/end object numbers: "
+               << "start_object_no=" << m_start_object_no << ", "
+               << "end_object_no=" << m_end_object_no << dendl;
+    finish(-EINVAL);
     return;
   }
 
   m_object_diff_state->clear();
 
+  if (m_snap_id_start == m_snap_id_end) {
+    // no delta between the same snapshot
+    finish(0);
+    return;
+  }
+  if (m_start_object_no == m_end_object_no) {
+    // no objects in the provided range (half-open)
+    finish(0);
+    return;
+  }
+
   // collect all the snap ids in the provided range (inclusive) unless
   // this is diff-iterate against the beginning of time, in which case
   // only the end version matters
   std::shared_lock image_locker{m_image_ctx->image_lock};
-  if (!m_diff_iterate_range || m_snap_id_start != 0) {
+  if (!is_diff_iterate() || m_snap_id_start != 0) {
     if (m_snap_id_start != 0) {
       m_snap_ids.insert(m_snap_id_start);
     }
@@ -178,10 +216,14 @@ void DiffRequest<I>::handle_load_object_map(int r) {
     m_object_map.resize(num_objs);
   }
 
+  uint64_t start_object_no, end_object_no;
   uint64_t prev_object_diff_state_size = m_object_diff_state->size();
-  if (m_diff_iterate_range) {
-    if (m_object_diff_state->size() != m_object_map.size()) {
-      m_object_diff_state->resize(m_object_map.size());
+  if (is_diff_iterate()) {
+    start_object_no = std::min(m_start_object_no, m_object_map.size());
+    end_object_no = std::min(m_end_object_no, m_object_map.size());
+    uint64_t num_objs_in_range = end_object_no - start_object_no;
+    if (m_object_diff_state->size() != num_objs_in_range) {
+      m_object_diff_state->resize(num_objs_in_range);
     }
   } else {
     // for deep-copy, the object diff state should be the largest of
@@ -194,21 +236,23 @@ void DiffRequest<I>::handle_load_object_map(int r) {
       // diff state
       m_object_map.resize(m_object_diff_state->size());
     }
+    start_object_no = 0;
+    end_object_no = m_object_diff_state->size();
   }
 
-  uint64_t overlap = std::min(m_object_map.size(), prev_object_diff_state_size);
-  auto it = m_object_map.begin();
-  auto overlap_end_it = it + overlap;
+  uint64_t overlap = std::min(m_object_diff_state->size(),
+                              prev_object_diff_state_size);
+  auto it = m_object_map.begin() + start_object_no;
   auto diff_it = m_object_diff_state->begin();
-  uint64_t i = 0;
-  for (; it != overlap_end_it; ++it, ++diff_it, ++i) {
+  uint64_t ono = start_object_no;
+  for (; ono < start_object_no + overlap; ++it, ++diff_it, ++ono) {
     uint8_t object_map_state = *it;
     uint8_t prev_object_diff_state = *diff_it;
     switch (prev_object_diff_state) {
     case DIFF_STATE_HOLE:
       if (object_map_state != OBJECT_NONEXISTENT) {
         // stay in HOLE on intermediate snapshots for diff-iterate
-        if (!m_diff_iterate_range || m_current_snap_id == m_snap_id_end) {
+        if (!is_diff_iterate() || m_current_snap_id == m_snap_id_end) {
           *diff_it = DIFF_STATE_DATA_UPDATED;
         }
       }
@@ -234,7 +278,7 @@ void DiffRequest<I>::handle_load_object_map(int r) {
       ceph_abort();
     }
 
-    ldout(cct, 20) << "object state: " << i << " "
+    ldout(cct, 20) << "object state: " << ono << " "
                    << static_cast<uint32_t>(prev_object_diff_state)
                    << "->" << static_cast<uint32_t>(*diff_it) << " ("
                    << static_cast<uint32_t>(object_map_state) << ")"
@@ -242,8 +286,9 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   }
   ldout(cct, 20) << "computed overlap diffs" << dendl;
 
-  auto end_it = m_object_map.end();
-  for (; it != end_it; ++it, ++diff_it, ++i) {
+  ceph_assert(diff_it == m_object_diff_state->end() ||
+              end_object_no <= m_object_map.size());
+  for (; ono < end_object_no; ++it, ++diff_it, ++ono) {
     uint8_t object_map_state = *it;
     if (object_map_state == OBJECT_NONEXISTENT) {
       *diff_it = DIFF_STATE_HOLE;
@@ -251,7 +296,7 @@ void DiffRequest<I>::handle_load_object_map(int r) {
       // diffing against the beginning of time or image was grown
       // (implicit) starting state is HOLE, this is the first object
       // map after
-      if (m_diff_iterate_range) {
+      if (is_diff_iterate()) {
         // for diff-iterate, if the object is discarded prior to or
         // in the end version, result should be HOLE
         // since DATA_UPDATED can transition only to HOLE_UPDATED,
@@ -278,12 +323,13 @@ void DiffRequest<I>::handle_load_object_map(int r) {
       }
     }
 
-    ldout(cct, 20) << "object state: " << i << " "
+    ldout(cct, 20) << "object state: " << ono << " "
                    << "->" << static_cast<uint32_t>(*diff_it) << " ("
                    << static_cast<uint32_t>(*it) << ")" << dendl;
   }
   ldout(cct, 20) << "computed resize diffs" << dendl;
 
+  ceph_assert(diff_it == m_object_diff_state->end());
   std::shared_lock image_locker{m_image_ctx->image_lock};
   load_object_map(&image_locker);
 }
diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
index f2fb9975f751..6a8ecb9dc930 100644
--- a/src/librbd/object_map/DiffRequest.h
+++ b/src/librbd/object_map/DiffRequest.h
@@ -21,21 +21,20 @@ namespace object_map {
 template <typename ImageCtxT>
 class DiffRequest {
 public:
-  static DiffRequest* create(ImageCtxT* image_ctx, uint64_t snap_id_start,
-                             uint64_t snap_id_end, bool diff_iterate_range,
+  static DiffRequest* create(ImageCtxT* image_ctx,
+                             uint64_t snap_id_start, uint64_t snap_id_end,
+                             uint64_t start_object_no, uint64_t end_object_no,
                              BitVector<2>* object_diff_state,
                              Context* on_finish) {
     return new DiffRequest(image_ctx, snap_id_start, snap_id_end,
-                           diff_iterate_range, object_diff_state, on_finish);
+                           start_object_no, end_object_no, object_diff_state,
+                           on_finish);
   }
 
-  DiffRequest(ImageCtxT* image_ctx, uint64_t snap_id_start,
-              uint64_t snap_id_end, bool diff_iterate_range,
-              BitVector<2>* object_diff_state, Context* on_finish)
-    : m_image_ctx(image_ctx), m_snap_id_start(snap_id_start),
-      m_snap_id_end(snap_id_end), m_diff_iterate_range(diff_iterate_range),
-      m_object_diff_state(object_diff_state), m_on_finish(on_finish) {
-  }
+  DiffRequest(ImageCtxT* image_ctx,
+              uint64_t snap_id_start, uint64_t snap_id_end,
+              uint64_t start_object_no, uint64_t end_object_no,
+              BitVector<2>* object_diff_state, Context* on_finish);
 
   void send();
 
@@ -58,7 +57,8 @@ class DiffRequest {
   ImageCtxT* m_image_ctx;
   uint64_t m_snap_id_start;
   uint64_t m_snap_id_end;
-  bool m_diff_iterate_range;
+  uint64_t m_start_object_no;
+  uint64_t m_end_object_no;
   BitVector<2>* m_object_diff_state;
   Context* m_on_finish;
 
@@ -72,6 +72,8 @@ class DiffRequest {
 
   bufferlist m_out_bl;
 
+  bool is_diff_iterate() const;
+
   void load_object_map(std::shared_lock<ceph::shared_mutex>* image_locker);
   void handle_load_object_map(int r);
 
diff --git a/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc b/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
index 634cabfa79e3..2c42d5075ccf 100644
--- a/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
+++ b/src/test/librbd/deep_copy/test_mock_ImageCopyRequest.cc
@@ -92,7 +92,7 @@ struct DiffRequest<MockTestImageCtx> {
   static DiffRequest* s_instance;
   static DiffRequest* create(MockTestImageCtx *image_ctx,
                              uint64_t snap_id_start, uint64_t snap_id_end,
-                             bool diff_iterate_range,
+                             uint64_t start_object_no, uint64_t end_object_no,
                              BitVector<2>* object_diff_state,
                              Context* on_finish) {
     ceph_assert(s_instance != nullptr);
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 9be4995e00a7..c7afbe939c29 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -209,7 +209,7 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
 
   template <typename F>
   int do_diff(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
-              bool diff_iterate_range) {
+              uint64_t start_object_no, uint64_t end_object_no) {
     InSequence seq;
 
     MockTestImageCtx mock_image_ctx(*m_image_ctx);
@@ -217,7 +217,7 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
 
     C_SaferCond ctx;
     auto req = new MockDiffRequest(&mock_image_ctx, start_snap_id,
-                                   end_snap_id, diff_iterate_range,
+                                   end_snap_id, start_object_no, end_object_no,
                                    &m_diff_state, &ctx);
     req->send();
     return ctx.wait();
@@ -226,14 +226,29 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
   template <typename F>
   void test_diff_iterate(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
                          const BitVector<2>& expected_diff_state) {
-    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id, true));
+    // ranged -- run through all ranges (substrings) in expected_diff_state
+    for (uint64_t i = 0; i < expected_diff_state.size(); i++) {
+      for (uint64_t j = i + 1; j <= expected_diff_state.size(); j++) {
+        ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
+                             i, j));
+        ASSERT_EQ(j - i, m_diff_state.size());
+        for (uint64_t k = 0; k < m_diff_state.size(); k++) {
+          ASSERT_EQ(expected_diff_state[i + k], m_diff_state[k]);
+        }
+      }
+    }
+
+    // unranged -- equivalent to i=0, j=expected_diff_state.size() range
+    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
+                         0, UINT64_MAX - 1));
     ASSERT_EQ(expected_diff_state, m_diff_state);
   }
 
   template <typename F>
   void test_deep_copy(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
                       const BitVector<2>& expected_diff_state) {
-    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id, false));
+    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
+                         0, UINT64_MAX));
     ASSERT_EQ(expected_diff_state, m_diff_state);
   }
 
@@ -243,9 +258,17 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidStartSnap) {
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, 123, 456));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, 0, UINT64_MAX));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, InvalidEndSnap) {
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 2, 1, 123, 456));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(noop, 2, 1, 0, UINT64_MAX));
   }
 }
 
@@ -253,21 +276,48 @@ TEST_P(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
   BitVector<2> expected_diff_state;
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(0, do_diff(noop, 1, 1, true));
+    ASSERT_EQ(0, do_diff(noop, 1, 1, 123, 456));
   } else {
-    ASSERT_EQ(0, do_diff(noop, 1, 1, false));
+    ASSERT_EQ(0, do_diff(noop, 1, 1, 0, UINT64_MAX));
   }
   ASSERT_EQ(expected_diff_state, m_diff_state);
 }
 
+TEST_P(TestMockObjectMapDiffRequest, InvalidStartObject) {
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, UINT64_MAX, UINT64_MAX));
+  } else {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, 123, UINT64_MAX));
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, InvalidEndObject) {
+  if (is_diff_iterate()) {
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, 456, 123));
+  } else {
+    SUCCEED();
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, StartEndObjectEqual) {
+  BitVector<2> expected_diff_state;
+
+  if (is_diff_iterate()) {
+    ASSERT_EQ(0, do_diff(noop, 0, 1, 123, 123));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+  } else {
+    SUCCEED();
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   // negative test -- object-map implicitly enables fast-diff
   REQUIRE(!is_feature_enabled(RBD_FEATURE_OBJECT_MAP));
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, 123, 456));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -301,6 +351,28 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapEmpty) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  m_image_ctx->size = 0;
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, {}, {}, {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  BitVector<2> expected_diff_state;
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, 1, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, 1, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -558,6 +630,25 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadEmpty) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  m_image_ctx->size = 0;
+
+  BitVector<2> object_map_head;
+  BitVector<2> expected_diff_state;
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
   REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
 
@@ -1274,9 +1365,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
   };
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, true));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, false));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1294,13 +1385,13 @@ TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
   object_map_1.resize(object_count);
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 0, 2, true));
+    ASSERT_EQ(-ENOENT, do_diff(noop, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, false));
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1397,9 +1488,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapDNE) {
     expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, true));
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, false));
+    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1429,7 +1520,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapDNE) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, true));
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1437,7 +1528,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapDNE) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, false));
+    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1529,9 +1620,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartFastDiffInvalid) {
     expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, true));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, false));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1554,14 +1645,14 @@ TEST_P(TestMockObjectMapDiffRequest, EndFastDiffInvalid) {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, true));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, 0, object_count));
   } else {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
       expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, false));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1592,7 +1683,7 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromBeginning) {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1617,9 +1708,9 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromSnap) {
     expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1642,9 +1733,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapLoadError) {
     expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, true));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, false));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1674,7 +1765,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapLoadError) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, true));
+    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1682,7 +1773,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapLoadError) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, false));
+    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1715,7 +1806,7 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromBeginning
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1742,9 +1833,9 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromSnap) {
     expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1768,9 +1859,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapTooSmall) {
     expect_load_map(mock_image_ctx, 1, object_map_1, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, true));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, false));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1796,7 +1887,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapTooSmall) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, true));
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1804,7 +1895,7 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapTooSmall) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, false));
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1838,7 +1929,7 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromBeginning)
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1866,9 +1957,9 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromSnap) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, true));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, false));
+    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 

From 7677d4b1b52ab68484545d0fcd7348f2f8e9f263 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 28 Dec 2023 10:14:18 +0100
Subject: [PATCH 1424/2492] librbd: propagate diff-iterate range to parent in
 fast-diff mode

When getting parent diff, pass the overlap-reduced image extent instead
of the entire 0..overlap range to avoid a similar quadratic slowdown on
cloned images.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/api/DiffIterate.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index a464b942ae09..b114c32fabad 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -293,12 +293,14 @@ int DiffIterate<I>::execute() {
         std::shared_lock image_locker{m_image_ctx.image_lock};
         uint64_t raw_overlap = 0;
         m_image_ctx.get_parent_overlap(m_image_ctx.snap_id, &raw_overlap);
-        auto overlap = m_image_ctx.reduce_parent_overlap(raw_overlap, false);
-        if (overlap.first > 0 && overlap.second == io::ImageArea::DATA) {
+        io::Extents parent_extents = {{m_offset, m_length}};
+        if (m_image_ctx.prune_parent_extents(parent_extents, io::ImageArea::DATA,
+                                             raw_overlap, false) > 0) {
           ldout(cct, 10) << " first getting parent diff" << dendl;
-          DiffIterate diff_parent(*m_image_ctx.parent, {}, nullptr, 0,
-                                  overlap.first, true, true, &simple_diff_cb,
-                                  &parent_diff);
+          DiffIterate diff_parent(*m_image_ctx.parent, {}, nullptr,
+                                  parent_extents[0].first,
+                                  parent_extents[0].second, true, true,
+                                  &simple_diff_cb, &parent_diff);
           r = diff_parent.execute();
           if (r < 0) {
             return r;

From 1503b96bf91d358540fc56a69b3cf18aa7eab68e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 28 Dec 2023 10:52:11 +0100
Subject: [PATCH 1425/2492] librbd: drop DiffIterate::diff_object_map()
 declaration

This is a leftover from commit 2b3a46801d39 ("librbd: switch
diff-iterate API to use new object-map diff helper").

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/api/DiffIterate.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/librbd/api/DiffIterate.h b/src/librbd/api/DiffIterate.h
index 8ce7f6afd7c3..c53b0e995b6d 100644
--- a/src/librbd/api/DiffIterate.h
+++ b/src/librbd/api/DiffIterate.h
@@ -55,10 +55,6 @@ class DiffIterate {
   std::pair<uint64_t, uint64_t> calc_object_diff_range();
 
   int execute();
-
-  int diff_object_map(uint64_t from_snap_id, uint64_t to_snap_id,
-                      BitVector<2>* object_diff_state);
-
 };
 
 } // namespace api

From 2ab5b52f71c88cb55f8ed82f1dfd0115fdd6e022 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 6 Jan 2024 12:22:35 +0100
Subject: [PATCH 1426/2492] common/bit_vector: fix IteratorImpl post-increment
 operator

It's totally broken: instead of returning the current position and
moving to the next position, it returns the next position and doesn't
move anywhere.  Luckily it hasn't been used until now.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/common/bit_vector.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/bit_vector.hpp b/src/common/bit_vector.hpp
index 9ce3e8b1ebb2..09ac98b32c3b 100644
--- a/src/common/bit_vector.hpp
+++ b/src/common/bit_vector.hpp
@@ -129,7 +129,7 @@ class BitVector
 
     inline IteratorImpl operator++(int) {
       IteratorImpl iterator_impl(*this);
-      ++iterator_impl;
+      ++*this;
       return iterator_impl;
     }
     inline IteratorImpl operator+(uint64_t offset) {

From 275a299cd48d2ddac36608d6633a6b79c8927351 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 4 Jan 2024 11:39:20 +0100
Subject: [PATCH 1427/2492] librbd/object_map: don't resize object map in
 handle_load_object_map()

Currently it's done in two cases:

- if the loaded object map is larger than expected based on byte size,
  it's truncated to expected number of objects
- in case of deep-copy, if the loaded object map is smaller than diff
  state, it's expanded to get "track the largest of all versions in the
  set" semantics

Both of these cases can be easily dealt with without modifying the
object map.  Being able to process a const object map is needed for
working on in-memory object map which is external to DiffRequest.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc          | 23 +++++-------
 .../object_map/test_mock_DiffRequest.cc       | 36 +++++++++++++++++++
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index b8666fb9809a..0d33078c173d 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -212,15 +212,13 @@ void DiffRequest<I>::handle_load_object_map(int r) {
                   << m_object_map.size() << " < " << num_objs << dendl;
     finish(-EINVAL);
     return;
-  } else {
-    m_object_map.resize(num_objs);
   }
 
   uint64_t start_object_no, end_object_no;
   uint64_t prev_object_diff_state_size = m_object_diff_state->size();
   if (is_diff_iterate()) {
-    start_object_no = std::min(m_start_object_no, m_object_map.size());
-    end_object_no = std::min(m_end_object_no, m_object_map.size());
+    start_object_no = std::min(m_start_object_no, num_objs);
+    end_object_no = std::min(m_end_object_no, num_objs);
     uint64_t num_objs_in_range = end_object_no - start_object_no;
     if (m_object_diff_state->size() != num_objs_in_range) {
       m_object_diff_state->resize(num_objs_in_range);
@@ -228,13 +226,10 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   } else {
     // for deep-copy, the object diff state should be the largest of
     // all versions in the set, so it's only ever grown
-    if (m_object_diff_state->size() < m_object_map.size()) {
-      m_object_diff_state->resize(m_object_map.size());
-    } else if (m_object_diff_state->size() > m_object_map.size()) {
-      // the image was shrunk so expanding the object map will flag end objects
-      // as non-existent and they will be compared against the previous object
-      // diff state
-      m_object_map.resize(m_object_diff_state->size());
+    // shrink is handled by flagging trimmed objects as non-existent
+    // and comparing against the previous object diff state as usual
+    if (m_object_diff_state->size() < num_objs) {
+      m_object_diff_state->resize(num_objs);
     }
     start_object_no = 0;
     end_object_no = m_object_diff_state->size();
@@ -245,8 +240,8 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   auto it = m_object_map.begin() + start_object_no;
   auto diff_it = m_object_diff_state->begin();
   uint64_t ono = start_object_no;
-  for (; ono < start_object_no + overlap; ++it, ++diff_it, ++ono) {
-    uint8_t object_map_state = *it;
+  for (; ono < start_object_no + overlap; ++diff_it, ++ono) {
+    uint8_t object_map_state = (ono < num_objs ? *it++ : OBJECT_NONEXISTENT);
     uint8_t prev_object_diff_state = *diff_it;
     switch (prev_object_diff_state) {
     case DIFF_STATE_HOLE:
@@ -287,7 +282,7 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   ldout(cct, 20) << "computed overlap diffs" << dendl;
 
   ceph_assert(diff_it == m_object_diff_state->end() ||
-              end_object_no <= m_object_map.size());
+              end_object_no <= num_objs);
   for (; ono < end_object_no; ++it, ++diff_it, ++ono) {
     uint8_t object_map_state = *it;
     if (object_map_state == OBJECT_NONEXISTENT) {
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index c7afbe939c29..8d6ee2d27b1c 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -1963,6 +1963,42 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromSnap) {
   }
 }
 
+TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooLarge) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = 5;
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count + 12);
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count + 34);
+  object_map_head[1] = OBJECT_EXISTS_CLEAN;
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  expected_diff_state[1] = DIFF_STATE_DATA_UPDATED;
+
+  if (is_diff_iterate()) {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
+    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    auto load = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+    };
+    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
 INSTANTIATE_TEST_SUITE_P(MockObjectMapDiffRequestTests,
                          TestMockObjectMapDiffRequest, ::testing::Bool());
 

From 232ad1a5fb6248d7b3fbfaec5944a90a71a95806 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Thu, 4 Jan 2024 11:44:46 +0100
Subject: [PATCH 1428/2492] librbd/object_map: make object map in
 handle_load_object_map() local

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc | 15 ++++++++-------
 src/librbd/object_map/DiffRequest.h  |  2 --
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 0d33078c173d..abb5353b0c18 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -185,13 +185,14 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   auto cct = m_image_ctx->cct;
   ldout(cct, 10) << "r=" << r << dendl;
 
+  BitVector<2> object_map;
+  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
+                                               m_current_snap_id));
+
   if (r == 0) {
     auto bl_it = m_out_bl.cbegin();
-    r = cls_client::object_map_load_finish(&bl_it, &m_object_map);
+    r = cls_client::object_map_load_finish(&bl_it, &object_map);
   }
-
-  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
-                                               m_current_snap_id));
   if (r == -ENOENT && m_ignore_enoent) {
     ldout(cct, 10) << "object map " << oid << " does not exist" << dendl;
 
@@ -207,9 +208,9 @@ void DiffRequest<I>::handle_load_object_map(int r) {
 
   uint64_t num_objs = Striper::get_num_objects(m_image_ctx->layout,
                                                m_current_size);
-  if (m_object_map.size() < num_objs) {
+  if (object_map.size() < num_objs) {
     ldout(cct, 1) << "object map too small: "
-                  << m_object_map.size() << " < " << num_objs << dendl;
+                  << object_map.size() << " < " << num_objs << dendl;
     finish(-EINVAL);
     return;
   }
@@ -237,7 +238,7 @@ void DiffRequest<I>::handle_load_object_map(int r) {
 
   uint64_t overlap = std::min(m_object_diff_state->size(),
                               prev_object_diff_state_size);
-  auto it = m_object_map.begin() + start_object_no;
+  auto it = object_map.begin() + start_object_no;
   auto diff_it = m_object_diff_state->begin();
   uint64_t ono = start_object_no;
   for (; ono < start_object_no + overlap; ++diff_it, ++ono) {
diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
index 6a8ecb9dc930..84211c6fc4a4 100644
--- a/src/librbd/object_map/DiffRequest.h
+++ b/src/librbd/object_map/DiffRequest.h
@@ -68,8 +68,6 @@ class DiffRequest {
 
   uint64_t m_current_size = 0;
 
-  BitVector<2> m_object_map;
-
   bufferlist m_out_bl;
 
   bool is_diff_iterate() const;

From 45d534553abaca81e26574fd5a7b17b9219c0dd0 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 5 Jan 2024 12:23:24 +0100
Subject: [PATCH 1429/2492] common/bit_vector: fix iterator vs reference
 constness confusion

T (ConstIterator or Iterator) is confused with const T here:
IteratorImpl dereference operator is wrongly overloaded on const
and returns Reference instead of ConstReference for ConstIterator.
This then fails inside bufferlist bowels because Reference is
incompatible with bufferlist::const_iterator.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/common/bit_vector.hpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/common/bit_vector.hpp b/src/common/bit_vector.hpp
index 09ac98b32c3b..961d9a0192ee 100644
--- a/src/common/bit_vector.hpp
+++ b/src/common/bit_vector.hpp
@@ -83,7 +83,7 @@ class BitVector
   };
 
 public:
-  template <typename BitVectorT, typename DataIterator>
+  template <typename BitVectorT, typename DataIteratorT, typename ReferenceT>
   class IteratorImpl {
   private:
     friend class BitVector;
@@ -94,7 +94,7 @@ class BitVector
     // cached derived values
     uint64_t m_index = 0;
     uint64_t m_shift = 0;
-    DataIterator m_data_iterator;
+    DataIteratorT m_data_iterator;
 
     IteratorImpl(BitVectorT *bit_vector, uint64_t offset)
       : m_bit_vector(bit_vector),
@@ -145,17 +145,15 @@ class BitVector
       return (m_offset != rhs.m_offset || m_bit_vector != rhs.m_bit_vector);
     }
 
-    inline ConstReference operator*() const {
-      return ConstReference(m_data_iterator, m_shift);
-    }
-    inline Reference operator*() {
-      return Reference(m_data_iterator, m_shift);
+    inline ReferenceT operator*() const {
+      return ReferenceT(m_data_iterator, m_shift);
     }
   };
 
   typedef IteratorImpl<const BitVector,
-                       bufferlist::const_iterator> ConstIterator;
-  typedef IteratorImpl<BitVector, bufferlist::iterator> Iterator;
+                       bufferlist::const_iterator,
+                       ConstReference> ConstIterator;
+  typedef IteratorImpl<BitVector, bufferlist::iterator, Reference> Iterator;
 
   static const uint32_t BLOCK_SIZE;
   static const uint8_t BIT_COUNT = _bit_count;

From dabb677ba5923f347c5b4b81b6a86214699a52bf Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Fri, 5 Jan 2024 13:15:54 +0100
Subject: [PATCH 1430/2492] librbd/object_map: decouple object map processing
 in DiffRequest

In preparation for potentially using in-memory object map, decouple
object map processing from loading object maps and place the logic in
prepare_for_object_map() and process_object_map().

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/object_map/DiffRequest.cc | 275 ++++++++++++++-------------
 src/librbd/object_map/DiffRequest.h  |   3 +
 2 files changed, 145 insertions(+), 133 deletions(-)

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index abb5353b0c18..7da6152058b0 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -44,88 +44,17 @@ bool DiffRequest<I>::is_diff_iterate() const {
 }
 
 template <typename I>
-void DiffRequest<I>::send() {
-  auto cct = m_image_ctx->cct;
-
-  if (m_snap_id_start == CEPH_NOSNAP || m_snap_id_start > m_snap_id_end) {
-    lderr(cct) << "invalid start/end snap ids: "
-               << "snap_id_start=" << m_snap_id_start << ", "
-               << "snap_id_end=" << m_snap_id_end << dendl;
-    finish(-EINVAL);
-    return;
-  }
-  if (m_start_object_no == UINT64_MAX || m_start_object_no > m_end_object_no ||
-      (m_start_object_no != 0 && m_end_object_no == UINT64_MAX)) {
-    lderr(cct) << "invalid start/end object numbers: "
-               << "start_object_no=" << m_start_object_no << ", "
-               << "end_object_no=" << m_end_object_no << dendl;
-    finish(-EINVAL);
-    return;
-  }
-
-  m_object_diff_state->clear();
-
-  if (m_snap_id_start == m_snap_id_end) {
-    // no delta between the same snapshot
-    finish(0);
-    return;
-  }
-  if (m_start_object_no == m_end_object_no) {
-    // no objects in the provided range (half-open)
-    finish(0);
-    return;
-  }
-
-  // collect all the snap ids in the provided range (inclusive) unless
-  // this is diff-iterate against the beginning of time, in which case
-  // only the end version matters
-  std::shared_lock image_locker{m_image_ctx->image_lock};
-  if (!is_diff_iterate() || m_snap_id_start != 0) {
-    if (m_snap_id_start != 0) {
-      m_snap_ids.insert(m_snap_id_start);
-    }
-    auto snap_info_it = m_image_ctx->snap_info.upper_bound(m_snap_id_start);
-    auto snap_info_it_end = m_image_ctx->snap_info.lower_bound(m_snap_id_end);
-    for (; snap_info_it != snap_info_it_end; ++snap_info_it) {
-      m_snap_ids.insert(snap_info_it->first);
-    }
-  }
-  m_snap_ids.insert(m_snap_id_end);
-
-  load_object_map(&image_locker);
-}
-
-template <typename I>
-void DiffRequest<I>::load_object_map(
-    std::shared_lock<ceph::shared_mutex>* image_locker) {
+int DiffRequest<I>::prepare_for_object_map() {
   ceph_assert(ceph_mutex_is_locked(m_image_ctx->image_lock));
 
-  if (m_snap_ids.empty()) {
-    image_locker->unlock();
-
-    finish(0);
-    return;
-  }
-
-  m_current_snap_id = *m_snap_ids.begin();
-  m_snap_ids.erase(m_current_snap_id);
-
   auto cct = m_image_ctx->cct;
-  ldout(cct, 10) << "snap_id=" << m_current_snap_id << dendl;
+  ldout(cct, 20) << "snap_id=" << m_current_snap_id << dendl;
 
   if ((m_image_ctx->features & RBD_FEATURE_FAST_DIFF) == 0) {
-    image_locker->unlock();
-
     ldout(cct, 10) << "fast-diff feature not enabled" << dendl;
-    finish(-EINVAL);
-    return;
+    return -EINVAL;
   }
 
-  // ignore ENOENT with intermediate snapshots since deleted
-  // snaps will get merged with later snapshots
-  m_ignore_enoent = (m_current_snap_id != m_snap_id_start &&
-                     m_current_snap_id != m_snap_id_end);
-
   if (m_current_snap_id == CEPH_NOSNAP) {
     m_current_size = m_image_ctx->size;
   } else {
@@ -133,86 +62,35 @@ void DiffRequest<I>::load_object_map(
     if (snap_it == m_image_ctx->snap_info.end()) {
       ldout(cct, 10) << "snapshot " << m_current_snap_id << " does not exist"
                      << dendl;
-      if (!m_ignore_enoent) {
-        image_locker->unlock();
-
-        finish(-ENOENT);
-        return;
-      }
-
-      load_object_map(image_locker);
-      return;
+      return -ENOENT;
     }
-
     m_current_size = snap_it->second.size;
   }
 
-  uint64_t flags = 0;
+  uint64_t flags;
   int r = m_image_ctx->get_flags(m_current_snap_id, &flags);
-  if (r < 0) {
-    image_locker->unlock();
-
-    lderr(cct) << "failed to retrieve image flags: " << cpp_strerror(r)
-               << dendl;
-    finish(r);
-    return;
-  }
-  image_locker->unlock();
+  ceph_assert(r == 0);
 
   if ((flags & RBD_FLAG_FAST_DIFF_INVALID) != 0) {
     ldout(cct, 1) << "cannot perform fast diff on invalid object map"
                   << dendl;
-    finish(-EINVAL);
-    return;
+    return -EINVAL;
   }
 
-  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
-                                               m_current_snap_id));
-
-  librados::ObjectReadOperation op;
-  cls_client::object_map_load_start(&op);
-
-  m_out_bl.clear();
-  auto aio_comp = create_rados_callback<
-    DiffRequest<I>, &DiffRequest<I>::handle_load_object_map>(this);
-  r = m_image_ctx->md_ctx.aio_operate(oid, aio_comp, &op, &m_out_bl);
-  ceph_assert(r == 0);
-  aio_comp->release();
+  return 0;
 }
 
 template <typename I>
-void DiffRequest<I>::handle_load_object_map(int r) {
+int DiffRequest<I>::process_object_map(const BitVector<2>& object_map) {
   auto cct = m_image_ctx->cct;
-  ldout(cct, 10) << "r=" << r << dendl;
-
-  BitVector<2> object_map;
-  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
-                                               m_current_snap_id));
-
-  if (r == 0) {
-    auto bl_it = m_out_bl.cbegin();
-    r = cls_client::object_map_load_finish(&bl_it, &object_map);
-  }
-  if (r == -ENOENT && m_ignore_enoent) {
-    ldout(cct, 10) << "object map " << oid << " does not exist" << dendl;
-
-    std::shared_lock image_locker{m_image_ctx->image_lock};
-    load_object_map(&image_locker);
-    return;
-  } else if (r < 0) {
-    lderr(cct) << "failed to load object map: " << oid << dendl;
-    finish(r);
-    return;
-  }
-  ldout(cct, 20) << "loaded object map " << oid << dendl;
+  ldout(cct, 20) << "snap_id=" << m_current_snap_id << dendl;
 
   uint64_t num_objs = Striper::get_num_objects(m_image_ctx->layout,
                                                m_current_size);
   if (object_map.size() < num_objs) {
     ldout(cct, 1) << "object map too small: "
                   << object_map.size() << " < " << num_objs << dendl;
-    finish(-EINVAL);
-    return;
+    return -EINVAL;
   }
 
   uint64_t start_object_no, end_object_no;
@@ -326,6 +204,137 @@ void DiffRequest<I>::handle_load_object_map(int r) {
   ldout(cct, 20) << "computed resize diffs" << dendl;
 
   ceph_assert(diff_it == m_object_diff_state->end());
+  return 0;
+}
+
+template <typename I>
+void DiffRequest<I>::send() {
+  auto cct = m_image_ctx->cct;
+
+  if (m_snap_id_start == CEPH_NOSNAP || m_snap_id_start > m_snap_id_end) {
+    lderr(cct) << "invalid start/end snap ids: "
+               << "snap_id_start=" << m_snap_id_start << ", "
+               << "snap_id_end=" << m_snap_id_end << dendl;
+    finish(-EINVAL);
+    return;
+  }
+  if (m_start_object_no == UINT64_MAX || m_start_object_no > m_end_object_no ||
+      (m_start_object_no != 0 && m_end_object_no == UINT64_MAX)) {
+    lderr(cct) << "invalid start/end object numbers: "
+               << "start_object_no=" << m_start_object_no << ", "
+               << "end_object_no=" << m_end_object_no << dendl;
+    finish(-EINVAL);
+    return;
+  }
+
+  m_object_diff_state->clear();
+
+  if (m_snap_id_start == m_snap_id_end) {
+    // no delta between the same snapshot
+    finish(0);
+    return;
+  }
+  if (m_start_object_no == m_end_object_no) {
+    // no objects in the provided range (half-open)
+    finish(0);
+    return;
+  }
+
+  // collect all the snap ids in the provided range (inclusive) unless
+  // this is diff-iterate against the beginning of time, in which case
+  // only the end version matters
+  std::shared_lock image_locker{m_image_ctx->image_lock};
+  if (!is_diff_iterate() || m_snap_id_start != 0) {
+    if (m_snap_id_start != 0) {
+      m_snap_ids.insert(m_snap_id_start);
+    }
+    auto snap_info_it = m_image_ctx->snap_info.upper_bound(m_snap_id_start);
+    auto snap_info_it_end = m_image_ctx->snap_info.lower_bound(m_snap_id_end);
+    for (; snap_info_it != snap_info_it_end; ++snap_info_it) {
+      m_snap_ids.insert(snap_info_it->first);
+    }
+  }
+  m_snap_ids.insert(m_snap_id_end);
+
+  load_object_map(&image_locker);
+}
+
+template <typename I>
+void DiffRequest<I>::load_object_map(
+    std::shared_lock<ceph::shared_mutex>* image_locker) {
+  ceph_assert(ceph_mutex_is_locked(m_image_ctx->image_lock));
+
+  if (m_snap_ids.empty()) {
+    image_locker->unlock();
+
+    finish(0);
+    return;
+  }
+
+  m_current_snap_id = *m_snap_ids.begin();
+  m_snap_ids.erase(m_current_snap_id);
+
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << "snap_id=" << m_current_snap_id << dendl;
+
+  // ignore ENOENT with intermediate snapshots since deleted
+  // snaps will get merged with later snapshots
+  m_ignore_enoent = (m_current_snap_id != m_snap_id_start &&
+                     m_current_snap_id != m_snap_id_end);
+
+  int r = prepare_for_object_map();
+  if (r == -ENOENT && m_ignore_enoent) {
+    load_object_map(image_locker);
+    return;
+  } else if (r < 0) {
+    image_locker->unlock();
+
+    finish(r);
+    return;
+  }
+  image_locker->unlock();
+
+  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
+                                               m_current_snap_id));
+
+  librados::ObjectReadOperation op;
+  cls_client::object_map_load_start(&op);
+
+  m_out_bl.clear();
+  auto aio_comp = create_rados_callback<
+    DiffRequest<I>, &DiffRequest<I>::handle_load_object_map>(this);
+  r = m_image_ctx->md_ctx.aio_operate(oid, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void DiffRequest<I>::handle_load_object_map(int r) {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  BitVector<2> object_map;
+  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
+                                               m_current_snap_id));
+
+  if (r == 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = cls_client::object_map_load_finish(&bl_it, &object_map);
+  }
+  if (r == -ENOENT && m_ignore_enoent) {
+    ldout(cct, 10) << "object map " << oid << " does not exist" << dendl;
+  } else if (r < 0) {
+    lderr(cct) << "failed to load object map: " << oid << dendl;
+    finish(r);
+    return;
+  } else {
+    r = process_object_map(object_map);
+    if (r < 0) {
+      finish(r);
+      return;
+    }
+  }
+
   std::shared_lock image_locker{m_image_ctx->image_lock};
   load_object_map(&image_locker);
 }
diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
index 84211c6fc4a4..740f4e02a117 100644
--- a/src/librbd/object_map/DiffRequest.h
+++ b/src/librbd/object_map/DiffRequest.h
@@ -72,6 +72,9 @@ class DiffRequest {
 
   bool is_diff_iterate() const;
 
+  int prepare_for_object_map();
+  int process_object_map(const BitVector<2>& object_map);
+
   void load_object_map(std::shared_lock<ceph::shared_mutex>* image_locker);
   void handle_load_object_map(int r);
 

From 0c4bb58c900efa2356ea8526d3432b2922787afa Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 6 Jan 2024 17:05:39 +0100
Subject: [PATCH 1431/2492] librbd/object_map: potentially use in-memory object
 map in DiffRequest

If the object map for the end version is around (already loaded in
memory, either due to the end version being a snapshot or due to
exclusive lock being held), use it to run diff-iterate against the
beginning of time.  Since it's the only object map needed in that
case, such calls would be satisfied locally.

Fixes: https://tracker.ceph.com/issues/63341
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/ObjectMap.h                        |   6 +
 src/librbd/object_map/DiffRequest.cc          |  22 +-
 src/test/librbd/mock/MockObjectMap.h          |  12 +-
 .../object_map/test_mock_DiffRequest.cc       | 510 ++++++++++++++----
 4 files changed, 431 insertions(+), 119 deletions(-)

diff --git a/src/librbd/ObjectMap.h b/src/librbd/ObjectMap.h
index 8b5b352ef8aa..35ea4cb88f93 100644
--- a/src/librbd/ObjectMap.h
+++ b/src/librbd/ObjectMap.h
@@ -45,6 +45,12 @@ class ObjectMap : public RefCountedObject {
     return m_object_map.size();
   }
 
+  template <typename F, typename... Args>
+  auto with_object_map(F&& f, Args&&... args) const {
+    std::shared_lock locker(m_lock);
+    return std::forward<F>(f)(m_object_map, std::forward<Args>(args)...);
+  }
+
   inline void set_state(uint64_t object_no, uint8_t new_state,
                         const boost::optional<uint8_t> &current_state) {
     std::unique_lock locker{m_lock};
diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
index 7da6152058b0..acaf31a39bfb 100644
--- a/src/librbd/object_map/DiffRequest.cc
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -240,10 +240,30 @@ void DiffRequest<I>::send() {
     return;
   }
 
+  std::shared_lock image_locker{m_image_ctx->image_lock};
+  if (is_diff_iterate() &&
+      m_snap_id_start == 0 &&
+      m_snap_id_end == m_image_ctx->snap_id &&
+      m_image_ctx->object_map != nullptr) {
+    ldout(cct, 10) << "using in-memory object map" << dendl;
+    m_current_snap_id = m_snap_id_end;
+
+    int r = prepare_for_object_map();
+    if (r == 0) {
+      r = m_image_ctx->object_map->with_object_map(
+        [this](const BitVector<2>& object_map) {
+          return process_object_map(object_map);
+        });
+    }
+    image_locker.unlock();
+
+    finish(r);
+    return;
+  }
+
   // collect all the snap ids in the provided range (inclusive) unless
   // this is diff-iterate against the beginning of time, in which case
   // only the end version matters
-  std::shared_lock image_locker{m_image_ctx->image_lock};
   if (!is_diff_iterate() || m_snap_id_start != 0) {
     if (m_snap_id_start != 0) {
       m_snap_ids.insert(m_snap_id_start);
diff --git a/src/test/librbd/mock/MockObjectMap.h b/src/test/librbd/mock/MockObjectMap.h
index 2a1adbcaebab..5e3235cf023d 100644
--- a/src/test/librbd/mock/MockObjectMap.h
+++ b/src/test/librbd/mock/MockObjectMap.h
@@ -4,19 +4,27 @@
 #ifndef CEPH_TEST_LIBRBD_MOCK_OBJECT_MAP_H
 #define CEPH_TEST_LIBRBD_MOCK_OBJECT_MAP_H
 
+#include "common/bit_vector.hpp"
 #include "librbd/Utils.h"
 #include "gmock/gmock.h"
 
 namespace librbd {
 
 struct MockObjectMap {
-  MOCK_METHOD1(at, uint8_t(uint64_t));
-  uint8_t operator[](uint64_t object_no) {
+  MOCK_CONST_METHOD1(at, uint8_t(uint64_t));
+  uint8_t operator[](uint64_t object_no) const {
     return at(object_no);
   }
 
   MOCK_CONST_METHOD0(size, uint64_t());
 
+  MOCK_CONST_METHOD0(with, ceph::BitVector<2>());
+  template <typename F, typename... Args>
+  auto with_object_map(F&& f, Args&&... args) const {
+    const ceph::BitVector<2> object_map = with();
+    return std::forward<F>(f)(object_map, std::forward<Args>(args)...);
+  }
+
   MOCK_METHOD1(open, void(Context *on_finish));
   MOCK_METHOD1(close, void(Context *on_finish));
 
diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc
index 8d6ee2d27b1c..9b88f0d75ca9 100644
--- a/src/test/librbd/object_map/test_mock_DiffRequest.cc
+++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc
@@ -28,6 +28,7 @@ void noop(MockTestImageCtx&) {}
 using ::testing::_;
 using ::testing::Invoke;
 using ::testing::InSequence;
+using ::testing::Return;
 using ::testing::StrEq;
 using ::testing::WithArg;
 
@@ -207,12 +208,23 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
     expect_load_map(mock_image_ctx, snap_id, object_map, r, [](){});
   }
 
+  void expect_with_map(MockTestImageCtx& mock_image_ctx,
+                       const BitVector<2>& object_map) {
+    EXPECT_CALL(*mock_image_ctx.object_map, with()).WillOnce(Return(object_map));
+  }
+
   template <typename F>
-  int do_diff(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+  int do_diff(bool want_object_map, F&& f,
+              uint64_t start_snap_id, uint64_t end_snap_id,
               uint64_t start_object_no, uint64_t end_object_no) {
     InSequence seq;
 
     MockTestImageCtx mock_image_ctx(*m_image_ctx);
+    MockObjectMap mock_object_map;
+    if (want_object_map) {
+      mock_image_ctx.object_map = &mock_object_map;
+    }
+    mock_image_ctx.snap_id = end_snap_id;
     std::forward<F>(f)(mock_image_ctx);
 
     C_SaferCond ctx;
@@ -224,13 +236,14 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
   }
 
   template <typename F>
-  void test_diff_iterate(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+  void test_diff_iterate(bool want_object_map, F&& f,
+                         uint64_t start_snap_id, uint64_t end_snap_id,
                          const BitVector<2>& expected_diff_state) {
     // ranged -- run through all ranges (substrings) in expected_diff_state
     for (uint64_t i = 0; i < expected_diff_state.size(); i++) {
       for (uint64_t j = i + 1; j <= expected_diff_state.size(); j++) {
-        ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
-                             i, j));
+        ASSERT_EQ(0, do_diff(want_object_map, std::forward<F>(f),
+                             start_snap_id, end_snap_id, i, j));
         ASSERT_EQ(j - i, m_diff_state.size());
         for (uint64_t k = 0; k < m_diff_state.size(); k++) {
           ASSERT_EQ(expected_diff_state[i + k], m_diff_state[k]);
@@ -239,16 +252,17 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
     }
 
     // unranged -- equivalent to i=0, j=expected_diff_state.size() range
-    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
-                         0, UINT64_MAX - 1));
+    ASSERT_EQ(0, do_diff(want_object_map, std::forward<F>(f),
+                         start_snap_id, end_snap_id, 0, UINT64_MAX - 1));
     ASSERT_EQ(expected_diff_state, m_diff_state);
   }
 
   template <typename F>
-  void test_deep_copy(F&& f, uint64_t start_snap_id, uint64_t end_snap_id,
+  void test_deep_copy(bool want_object_map, F&& f,
+                      uint64_t start_snap_id, uint64_t end_snap_id,
                       const BitVector<2>& expected_diff_state) {
-    ASSERT_EQ(0, do_diff(std::forward<F>(f), start_snap_id, end_snap_id,
-                         0, UINT64_MAX));
+    ASSERT_EQ(0, do_diff(want_object_map, std::forward<F>(f),
+                         start_snap_id, end_snap_id, 0, UINT64_MAX));
     ASSERT_EQ(expected_diff_state, m_diff_state);
   }
 
@@ -258,17 +272,21 @@ class TestMockObjectMapDiffRequest : public TestMockFixture,
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidStartSnap) {
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, CEPH_NOSNAP, CEPH_NOSNAP, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, CEPH_NOSNAP, CEPH_NOSNAP, 123, 456));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, CEPH_NOSNAP, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, CEPH_NOSNAP, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, CEPH_NOSNAP, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidEndSnap) {
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 2, 1, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 2, 1, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 2, 1, 123, 456));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 2, 1, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 2, 1, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 2, 1, 0, UINT64_MAX));
   }
 }
 
@@ -276,24 +294,32 @@ TEST_P(TestMockObjectMapDiffRequest, StartEndSnapEqual) {
   BitVector<2> expected_diff_state;
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(0, do_diff(noop, 1, 1, 123, 456));
+    ASSERT_EQ(0, do_diff(false, noop, 1, 1, 123, 456));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+    ASSERT_EQ(0, do_diff(true, noop, 1, 1, 123, 456));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
   } else {
-    ASSERT_EQ(0, do_diff(noop, 1, 1, 0, UINT64_MAX));
+    ASSERT_EQ(0, do_diff(false, noop, 1, 1, 0, UINT64_MAX));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+    ASSERT_EQ(0, do_diff(true, noop, 1, 1, 0, UINT64_MAX));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
   }
-  ASSERT_EQ(expected_diff_state, m_diff_state);
 }
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidStartObject) {
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, UINT64_MAX, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 0, 1, UINT64_MAX, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 0, 1, UINT64_MAX, UINT64_MAX));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, 123, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 0, 1, 123, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 0, 1, 123, UINT64_MAX));
   }
 }
 
 TEST_P(TestMockObjectMapDiffRequest, InvalidEndObject) {
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, 1, 456, 123));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 0, 1, 456, 123));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 0, 1, 456, 123));
   } else {
     SUCCEED();
   }
@@ -303,7 +329,9 @@ TEST_P(TestMockObjectMapDiffRequest, StartEndObjectEqual) {
   BitVector<2> expected_diff_state;
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(0, do_diff(noop, 0, 1, 123, 123));
+    ASSERT_EQ(0, do_diff(false, noop, 0, 1, 123, 123));
+    ASSERT_EQ(expected_diff_state, m_diff_state);
+    ASSERT_EQ(0, do_diff(true, noop, 0, 1, 123, 123));
     ASSERT_EQ(expected_diff_state, m_diff_state);
   } else {
     SUCCEED();
@@ -315,9 +343,11 @@ TEST_P(TestMockObjectMapDiffRequest, FastDiffDisabled) {
   REQUIRE(!is_feature_enabled(RBD_FEATURE_OBJECT_MAP));
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 0, CEPH_NOSNAP, 123, 456));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 0, CEPH_NOSNAP, 123, 456));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(noop, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, noop, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, noop, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -345,9 +375,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnap) {
     expect_load_map(mock_image_ctx, 1, object_map_1, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 0, 1, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_1);
+    };
+    test_diff_iterate(false, load, 0, 1, expected_diff_state);
+    test_diff_iterate(true, with, 0, 1, expected_diff_state);
   } else {
-    test_deep_copy(load, 0, 1, expected_diff_state);
+    test_deep_copy(false, load, 0, 1, expected_diff_state);
+    test_deep_copy(true, load, 0, 1, expected_diff_state);
   }
 }
 
@@ -367,9 +403,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapEmpty) {
     expect_load_map(mock_image_ctx, 1, object_map_1, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 0, 1, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 1, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_1);
+    };
+    test_diff_iterate(false, load, 0, 1, expected_diff_state);
+    test_diff_iterate(true, with, 0, 1, expected_diff_state);
   } else {
-    test_deep_copy(load, 0, 1, expected_diff_state);
+    test_deep_copy(false, load, 0, 1, expected_diff_state);
+    test_deep_copy(true, load, 0, 1, expected_diff_state);
   }
 }
 
@@ -406,7 +448,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_diff_iterate(load, 0, 2, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    test_diff_iterate(false, load, 0, 2, expected_diff_state);
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -414,7 +461,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnap) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_deep_copy(load, 0, 2, expected_diff_state);
+    test_deep_copy(false, load, 0, 2, expected_diff_state);
+    test_deep_copy(true, load, 0, 2, expected_diff_state);
   }
 }
 
@@ -456,7 +504,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrow) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_diff_iterate(load, 0, 2, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    test_diff_iterate(false, load, 0, 2, expected_diff_state);
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -464,7 +517,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrow) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_deep_copy(load, 0, 2, expected_diff_state);
+    test_deep_copy(false, load, 0, 2, expected_diff_state);
+    test_deep_copy(true, load, 0, 2, expected_diff_state);
   }
 }
 
@@ -494,7 +548,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrowFrom
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_diff_iterate(load, 0, 2, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    test_diff_iterate(false, load, 0, 2, expected_diff_state);
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -502,7 +561,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapGrowFrom
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_deep_copy(load, 0, 2, expected_diff_state);
+    test_deep_copy(false, load, 0, 2, expected_diff_state);
+    test_deep_copy(true, load, 0, 2, expected_diff_state);
   }
 }
 
@@ -550,7 +610,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrink)
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_diff_iterate(load, 0, 2, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    test_diff_iterate(false, load, 0, 2, expected_diff_state);
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -558,7 +623,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrink)
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_deep_copy(load, 0, 2, expected_diff_state);
+    test_deep_copy(false, load, 0, 2, expected_diff_state);
+    test_deep_copy(true, load, 0, 2, expected_diff_state);
   }
 }
 
@@ -592,7 +658,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrinkTo
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_diff_iterate(load, 0, 2, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    test_diff_iterate(false, load, 0, 2, expected_diff_state);
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -600,7 +671,41 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToSnapIntermediateSnapShrinkTo
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    test_deep_copy(load, 0, 2, expected_diff_state);
+    test_deep_copy(false, load, 0, 2, expected_diff_state);
+    test_deep_copy(true, load, 0, 2, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToUnsetSnap) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_beginning_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+  m_image_ctx->snap_info = {
+    {1U, {"snap1", {cls::rbd::UserSnapshotNamespace{}}, m_image_ctx->size, {},
+          {}, {}, {}}}
+  };
+
+  BitVector<2> object_map_1;
+  object_map_1.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_1[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    mock_image_ctx.snap_id = 123;
+    expect_get_flags(mock_image_ctx, 1, 0, 0);
+    expect_load_map(mock_image_ctx, 1, object_map_1, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(false, load, 0, 1, expected_diff_state);
+    test_diff_iterate(true, load, 0, 1, expected_diff_state);
+  } else {
+    test_deep_copy(false, load, 0, 1, expected_diff_state);
+    test_deep_copy(true, load, 0, 1, expected_diff_state);
   }
 }
 
@@ -624,9 +729,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHead) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -643,9 +754,15 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadEmpty) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -680,7 +797,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -688,7 +810,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnap) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -728,7 +851,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrow) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -736,7 +864,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrow) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -764,7 +893,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrowFrom
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -772,7 +906,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapGrowFrom
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -818,7 +953,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrink)
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -826,7 +966,8 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrink)
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -859,7 +1000,12 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrinkTo
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -867,7 +1013,37 @@ TEST_P(TestMockObjectMapDiffRequest, FromBeginningToHeadIntermediateSnapShrinkTo
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
+  }
+}
+
+TEST_P(TestMockObjectMapDiffRequest, FromBeginningToUnsetHead) {
+  REQUIRE_FEATURE(RBD_FEATURE_FAST_DIFF);
+
+  uint32_t object_count = std::size(from_beginning_table);
+  m_image_ctx->size = object_count * (1 << m_image_ctx->order);
+
+  BitVector<2> object_map_head;
+  object_map_head.resize(object_count);
+  BitVector<2> expected_diff_state;
+  expected_diff_state.resize(object_count);
+  for (uint32_t i = 0; i < object_count; i++) {
+    object_map_head[i] = from_beginning_table[i][0];
+    expected_diff_state[i] = from_beginning_table[i][1];
+  }
+
+  auto load = [&](MockTestImageCtx& mock_image_ctx) {
+    mock_image_ctx.snap_id = 123;
+    expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+    expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
+  };
+  if (is_diff_iterate()) {
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 0, CEPH_NOSNAP, expected_diff_state);
+  } else {
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -902,9 +1078,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnap) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 2, expected_diff_state);
+    test_diff_iterate(false, load, 1, 2, expected_diff_state);
+    test_diff_iterate(true, load, 1, 2, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 2, expected_diff_state);
+    test_deep_copy(false, load, 1, 2, expected_diff_state);
+    test_deep_copy(true, load, 1, 2, expected_diff_state);
   }
 }
 
@@ -944,9 +1122,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapGrow) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 2, expected_diff_state);
+    test_diff_iterate(false, load, 1, 2, expected_diff_state);
+    test_diff_iterate(true, load, 1, 2, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 2, expected_diff_state);
+    test_deep_copy(false, load, 1, 2, expected_diff_state);
+    test_deep_copy(true, load, 1, 2, expected_diff_state);
   }
 }
 
@@ -978,9 +1158,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapGrowFromZero) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 2, expected_diff_state);
+    test_diff_iterate(false, load, 1, 2, expected_diff_state);
+    test_diff_iterate(true, load, 1, 2, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 2, expected_diff_state);
+    test_deep_copy(false, load, 1, 2, expected_diff_state);
+    test_deep_copy(true, load, 1, 2, expected_diff_state);
   }
 }
 
@@ -1026,9 +1208,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapShrink) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 2, expected_diff_state);
+    test_diff_iterate(false, load, 1, 2, expected_diff_state);
+    test_diff_iterate(true, load, 1, 2, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 2, expected_diff_state);
+    test_deep_copy(false, load, 1, 2, expected_diff_state);
+    test_deep_copy(true, load, 1, 2, expected_diff_state);
   }
 }
 
@@ -1064,9 +1248,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapShrinkToZero) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 2, expected_diff_state);
+    test_diff_iterate(false, load, 1, 2, expected_diff_state);
+    test_diff_iterate(true, load, 1, 2, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 2, expected_diff_state);
+    test_deep_copy(false, load, 1, 2, expected_diff_state);
+    test_deep_copy(true, load, 1, 2, expected_diff_state);
   }
 }
 
@@ -1112,9 +1298,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToSnapIntermediateSnap) {
     expect_load_map(mock_image_ctx, 3, object_map_3, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, 3, expected_diff_state);
+    test_diff_iterate(false, load, 1, 3, expected_diff_state);
+    test_diff_iterate(true, load, 1, 3, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, 3, expected_diff_state);
+    test_deep_copy(false, load, 1, 3, expected_diff_state);
+    test_deep_copy(true, load, 1, 3, expected_diff_state);
   }
 }
 
@@ -1147,9 +1335,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHead) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1187,9 +1377,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadGrow) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1219,9 +1411,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadGrowFromZero) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1265,9 +1459,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadShrink) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1302,9 +1498,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadShrinkToZero) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1348,9 +1546,11 @@ TEST_P(TestMockObjectMapDiffRequest, FromSnapToHeadIntermediateSnap) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1365,9 +1565,11 @@ TEST_P(TestMockObjectMapDiffRequest, StartSnapDNE) {
   };
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(false, noop, 1, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(true, noop, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(false, noop, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(true, noop, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1385,13 +1587,15 @@ TEST_P(TestMockObjectMapDiffRequest, EndSnapDNE) {
   object_map_1.resize(object_count);
 
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(noop, 0, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(false, noop, 0, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(true, noop, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(false, load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(true, load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1421,7 +1625,12 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNEFromBeginning) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1430,7 +1639,8 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNEFromBeginning) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1463,9 +1673,11 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateSnapDNEFromSnap) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1488,9 +1700,11 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapDNE) {
     expect_load_map(mock_image_ctx, 1, object_map_1, -ENOENT);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(false, load, 1, 2, 0, object_count));
+    ASSERT_EQ(-ENOENT, do_diff(true, load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-ENOENT, do_diff(load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(false, load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(true, load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1520,7 +1734,12 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapDNE) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, object_count));
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    ASSERT_EQ(-ENOENT, do_diff(false, load, 0, 2, 0, object_count));
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1528,7 +1747,8 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapDNE) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -ENOENT);
     };
-    ASSERT_EQ(-ENOENT, do_diff(load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(false, load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-ENOENT, do_diff(true, load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1555,7 +1775,12 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapDNEFromBeginning) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1563,7 +1788,8 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapDNEFromBeginning) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1598,9 +1824,11 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapDNEFromSnap) {
     expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
   };
   if (is_diff_iterate()) {
-    test_diff_iterate(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   } else {
-    test_deep_copy(load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 1, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 1, CEPH_NOSNAP, expected_diff_state);
   }
 }
 
@@ -1620,9 +1848,11 @@ TEST_P(TestMockObjectMapDiffRequest, StartFastDiffInvalid) {
     expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 1, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1645,14 +1875,16 @@ TEST_P(TestMockObjectMapDiffRequest, EndFastDiffInvalid) {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 0, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 0, 2, 0, object_count));
   } else {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
       expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1678,12 +1910,18 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromBeginning) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto get_flags = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, RBD_FLAG_FAST_DIFF_INVALID, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1708,9 +1946,11 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateFastDiffInvalidFromSnap) {
     expect_get_flags(mock_image_ctx, 2, RBD_FLAG_FAST_DIFF_INVALID, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(get_flags, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, get_flags, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, get_flags, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1733,9 +1973,11 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapLoadError) {
     expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, 0, object_count));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 1, 2, 0, object_count));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1765,7 +2007,12 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapLoadError) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, 0, object_count));
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    ASSERT_EQ(-EPERM, do_diff(false, load, 0, 2, 0, object_count));
+    test_diff_iterate(true, with, 0, 2, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1773,7 +2020,8 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapLoadError) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1800,13 +2048,19 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromBeginning
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, -EPERM);
     };
-    ASSERT_EQ(-EPERM, do_diff(load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1833,9 +2087,11 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapLoadErrorFromSnap) {
     expect_load_map(mock_image_ctx, 2, object_map_2, -EPERM);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EPERM, do_diff(load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(false, load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EPERM, do_diff(true, load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1859,9 +2115,11 @@ TEST_P(TestMockObjectMapDiffRequest, StartObjectMapTooSmall) {
     expect_load_map(mock_image_ctx, 1, object_map_1, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 1, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 1, 2, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 1, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 1, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1887,7 +2145,12 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapTooSmall) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, 0, object_count));
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, 2, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_2);
+    };
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 0, 2, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, with, 0, 2, 0, object_count));
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1895,7 +2158,8 @@ TEST_P(TestMockObjectMapDiffRequest, EndObjectMapTooSmall) {
       expect_get_flags(mock_image_ctx, 2, 0, 0);
       expect_load_map(mock_image_ctx, 2, object_map_2, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 0, 2, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 0, 2, 0, UINT64_MAX));
   }
 }
 
@@ -1923,13 +2187,19 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromBeginning)
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
       expect_load_map(mock_image_ctx, 1, object_map_1, 0);
     };
-    ASSERT_EQ(-EINVAL, do_diff(load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 0, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1957,9 +2227,11 @@ TEST_P(TestMockObjectMapDiffRequest, IntermediateObjectMapTooSmallFromSnap) {
     expect_load_map(mock_image_ctx, 2, object_map_2, 0);
   };
   if (is_diff_iterate()) {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 1, CEPH_NOSNAP, 0, object_count));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 1, CEPH_NOSNAP, 0, object_count));
   } else {
-    ASSERT_EQ(-EINVAL, do_diff(load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(false, load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
+    ASSERT_EQ(-EINVAL, do_diff(true, load, 1, CEPH_NOSNAP, 0, UINT64_MAX));
   }
 }
 
@@ -1987,7 +2259,12 @@ TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooLarge) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_diff_iterate(load, 0, CEPH_NOSNAP, expected_diff_state);
+    auto with = [&](MockTestImageCtx& mock_image_ctx) {
+      expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
+      expect_with_map(mock_image_ctx, object_map_head);
+    };
+    test_diff_iterate(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_diff_iterate(true, with, 0, CEPH_NOSNAP, expected_diff_state);
   } else {
     auto load = [&](MockTestImageCtx& mock_image_ctx) {
       expect_get_flags(mock_image_ctx, 1, 0, 0);
@@ -1995,7 +2272,8 @@ TEST_P(TestMockObjectMapDiffRequest, ObjectMapTooLarge) {
       expect_get_flags(mock_image_ctx, CEPH_NOSNAP, 0, 0);
       expect_load_map(mock_image_ctx, CEPH_NOSNAP, object_map_head, 0);
     };
-    test_deep_copy(load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(false, load, 0, CEPH_NOSNAP, expected_diff_state);
+    test_deep_copy(true, load, 0, CEPH_NOSNAP, expected_diff_state);
   }
 }
 

From 89b0d9e7b40a5f962094428e613315d3697d261f Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 6 Jan 2024 17:08:04 +0100
Subject: [PATCH 1432/2492] librbd: try to preserve object map for diff-iterate
 in fast-diff mode

As an optimization, try to ensure that the object map for the end
version is preloaded through the acquisition of exclusive lock and
as a consequence remains around until exclusive lock is released.
If it's not around, DiffRequest would (re)load it on each call.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/ImageCtx.h          |  2 +
 src/librbd/api/DiffIterate.cc  | 71 +++++++++++++++++++++++++++----
 src/test/librbd/test_librbd.cc | 77 ++++++++++++++++++++++++++++++++++
 3 files changed, 141 insertions(+), 9 deletions(-)

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 0f6f742fcbd9..197fea305b02 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -148,6 +148,7 @@ namespace librbd {
                        // encryption_format
 
     ceph::shared_mutex timestamp_lock; // protects (create/access/modify)_timestamp
+                                       // and internal diff_iterate_lock_timestamp
     ceph::mutex async_ops_lock; // protects async_ops and async_requests
     ceph::mutex copyup_list_lock; // protects copyup_waiting_list
 
@@ -173,6 +174,7 @@ namespace librbd {
     utime_t create_timestamp;
     utime_t access_timestamp;
     utime_t modify_timestamp;
+    utime_t diff_iterate_lock_timestamp;
 
     file_layout_t layout;
 
diff --git a/src/librbd/api/DiffIterate.cc b/src/librbd/api/DiffIterate.cc
index b114c32fabad..717110bd38a1 100644
--- a/src/librbd/api/DiffIterate.cc
+++ b/src/librbd/api/DiffIterate.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "librbd/api/DiffIterate.h"
+#include "librbd/ExclusiveLock.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
 #include "librbd/ObjectMap.h"
@@ -30,6 +31,8 @@ namespace api {
 
 namespace {
 
+constexpr uint32_t LOCK_INTERVAL_SECONDS = 5;
+
 struct DiffContext {
   DiffIterate<>::Callback callback;
   void *callback_arg;
@@ -149,6 +152,35 @@ class C_DiffObject : public Context {
   }
 };
 
+template <typename I>
+bool should_try_acquire_lock(I* image_ctx) {
+  if (image_ctx->exclusive_lock == nullptr ||
+      image_ctx->exclusive_lock->is_lock_owner()) {
+    return false;
+  }
+  if ((image_ctx->features & RBD_FEATURE_FAST_DIFF) == 0) {
+    return false;
+  }
+
+  utime_t now = ceph_clock_now();
+  utime_t cutoff = now - utime_t(LOCK_INTERVAL_SECONDS, 0);
+
+  {
+    std::shared_lock timestamp_locker{image_ctx->timestamp_lock};
+    if (image_ctx->diff_iterate_lock_timestamp > cutoff) {
+      return false;
+    }
+  }
+
+  std::unique_lock timestamp_locker{image_ctx->timestamp_lock};
+  if (image_ctx->diff_iterate_lock_timestamp > cutoff) {
+    return false;
+  }
+
+  image_ctx->diff_iterate_lock_timestamp = now;
+  return true;
+}
+
 int simple_diff_cb(uint64_t off, size_t len, int exists, void *arg) {
   // This reads the existing extents in a parent from the beginning
   // of time.  Since images are thin-provisioned, the extents will
@@ -168,10 +200,14 @@ int DiffIterate<I>::diff_iterate(I *ictx,
                                  uint64_t off, uint64_t len,
                                  bool include_parent, bool whole_object,
                                  int (*cb)(uint64_t, size_t, int, void *),
-                                 void *arg)
-{
-  ldout(ictx->cct, 20) << "diff_iterate " << ictx << " off = " << off
-      		 << " len = " << len << dendl;
+                                 void *arg) {
+  ldout(ictx->cct, 10) << "from_snap_namespace=" << from_snap_namespace
+                       << ", fromsnapname=" << (fromsnapname ?: "")
+                       << ", off=" << off
+                       << ", len=" << len
+                       << ", include_parent=" << include_parent
+                       << ", whole_object=" << whole_object
+                       << dendl;
 
   if (!ictx->data_ctx.is_valid()) {
     return -ENODEV;
@@ -198,11 +234,28 @@ int DiffIterate<I>::diff_iterate(I *ictx,
     return r;
   }
 
-  ictx->image_lock.lock_shared();
-  r = clip_io(ictx, off, &len, io::ImageArea::DATA);
-  ictx->image_lock.unlock_shared();
-  if (r < 0) {
-    return r;
+  {
+    std::shared_lock owner_locker{ictx->owner_lock};
+    std::shared_lock image_locker{ictx->image_lock};
+
+    r = clip_io(ictx, off, &len, io::ImageArea::DATA);
+    if (r < 0) {
+      return r;
+    }
+
+    // optimization: hang onto the only object map needed to run fast
+    // diff against the beginning of time -- it's loaded when exclusive
+    // lock is acquired
+    // acquire exclusive lock only if not busy (i.e. don't request),
+    // throttle acquisition attempts and ignore errors
+    if (fromsnapname == nullptr && whole_object &&
+        should_try_acquire_lock(ictx)) {
+      C_SaferCond lock_ctx;
+      ictx->exclusive_lock->try_acquire_lock(&lock_ctx);
+      image_locker.unlock();
+      owner_locker.unlock();
+      lock_ctx.wait();
+    }
   }
 
   DiffIterate command(*ictx, from_snap_namespace, fromsnapname, off, len,
diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index bd73ecb93133..2b9557795348 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -8205,6 +8205,83 @@ TYPED_TEST(DiffIterateTest, DiffIterateUnaligned)
   ioctx.close();
 }
 
+TYPED_TEST(DiffIterateTest, DiffIterateTryAcquireLock)
+{
+  REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
+
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
+
+  {
+    librbd::RBD rbd;
+    int order = 22;
+    std::string name = this->get_temp_image_name();
+    ssize_t size = 20 << 20;
+
+    uint64_t object_size = 0;
+    if (this->whole_object) {
+      object_size = 1 << order;
+    }
+
+    ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+
+    librbd::Image image1;
+    ASSERT_EQ(0, rbd.open(ioctx, image1, name.c_str(), NULL));
+
+    ceph::bufferlist bl;
+    bl.append(std::string(256, '1'));
+    ASSERT_EQ(256, image1.write(0, 256, bl));
+    ASSERT_EQ(0, image1.flush());
+
+    bool lock_owner;
+    ASSERT_EQ(0, image1.is_exclusive_lock_owner(&lock_owner));
+    ASSERT_TRUE(lock_owner);
+
+    librbd::Image image2;
+    ASSERT_EQ(0, rbd.open(ioctx, image2, name.c_str(), NULL));
+
+    std::vector<diff_extent> extents;
+    ASSERT_EQ(0, image2.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                      vector_iterate_cb, &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+    extents.clear();
+
+    ASSERT_EQ(0, image2.is_exclusive_lock_owner(&lock_owner));
+    ASSERT_FALSE(lock_owner);
+
+    ASSERT_EQ(0, image1.close());
+    ASSERT_EQ(0, image2.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                      vector_iterate_cb, &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+    extents.clear();
+
+    ASSERT_EQ(0, image2.is_exclusive_lock_owner(&lock_owner));
+    ASSERT_FALSE(lock_owner);
+
+    sleep(5);
+    ASSERT_EQ(0, image2.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                      vector_iterate_cb, &extents));
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 256, true, object_size), extents[0]);
+    extents.clear();
+
+    ASSERT_EQ(0, image2.is_exclusive_lock_owner(&lock_owner));
+    if (this->whole_object &&
+        (is_feature_enabled(RBD_FEATURE_OBJECT_MAP) ||
+         is_feature_enabled(RBD_FEATURE_FAST_DIFF))) {
+      ASSERT_TRUE(lock_owner);
+    } else {
+      ASSERT_FALSE(lock_owner);
+    }
+
+    ASSERT_PASSED(this->validate_object_map, image2);
+  }
+
+  ioctx.close();
+}
+
 TYPED_TEST(DiffIterateTest, DiffIterateStriping)
 {
   REQUIRE_FEATURE(RBD_FEATURE_STRIPINGV2);

From 40e8813e9c705838eee42c98de717b20851aed72 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 20 Jan 2024 16:00:46 +0100
Subject: [PATCH 1433/2492] PendingReleaseNotes: add rbd_diff_iterate2 note

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 PendingReleaseNotes | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index b3fa9b9990e2..0de7e784507b 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -105,6 +105,11 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   See https://docs.ceph.com/en/latest/rados/operations/balancer/ for more information.
 * CephFS: Full support for subvolumes and subvolume groups is now available
   for snap_schedule Manager module.
+* RBD: When diffing against the beginning of time (`fromsnapname == NULL`) in
+  fast-diff mode (`whole_object == true` with `fast-diff` image feature enabled
+  and valid), diff-iterate is now guaranteed to execute locally if exclusive
+  lock is available.  This brings a dramatic performance improvement for QEMU
+  live disk synchronization and backup use cases.
 
 >=18.0.0
 

From 511af83e2747361350b60ce0ce88e67a726d9343 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 21 Jan 2024 09:33:59 +0000
Subject: [PATCH 1434/2492] crimson/os/alienstore/alien_log: _flush
 concurrently

In continuation to c15e56e386251403a876454f6a4aa186284565e1

Authored-by: Yingxin Cheng <yingxin.cheng@intel.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/os/alienstore/alien_log.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/alienstore/alien_log.cc b/src/crimson/os/alienstore/alien_log.cc
index a0f5b03a5d39..822b2f3ab519 100644
--- a/src/crimson/os/alienstore/alien_log.cc
+++ b/src/crimson/os/alienstore/alien_log.cc
@@ -17,8 +17,7 @@ CnLog::~CnLog() {
 }
 
 void CnLog::_flush(EntryVector& q, bool crash) {
-  // XXX: the waiting here will block the thread for an indeterministic peroid
-  seastar::alien::submit_to(inst, shard, [&q] {
+  std::ignore = seastar::alien::submit_to(inst, shard, [&q] {
     for (auto& it : q) {
       crimson::get_logger(it.m_subsys).log(
         crimson::to_log_level(it.m_prio),
@@ -26,7 +25,7 @@ void CnLog::_flush(EntryVector& q, bool crash) {
         it.strv());
     }
     return seastar::make_ready_future<>();
-  }).wait();
+  });
   q.clear();
   return;
 }

From c5157bb670c4c55d089dd1fb953a5faabd3480d2 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 8 Jan 2024 14:25:20 +0800
Subject: [PATCH 1435/2492] crimson/osd/osd_operations/client_request: cleanup
 shard_services

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 .../osd/osd_operations/client_request.cc      | 30 +++++++++----------
 .../osd/osd_operations/client_request.h       | 10 +++----
 src/crimson/osd/pg.cc                         |  2 +-
 3 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 0936889e37e3..1611531fcd0b 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -19,14 +19,13 @@ SET_SUBSYS(osd);
 namespace crimson::osd {
 
 
-void ClientRequest::Orderer::requeue(
-  ShardServices &shard_services, Ref<PG> pg)
+void ClientRequest::Orderer::requeue(Ref<PG> pg)
 {
   LOG_PREFIX(ClientRequest::Orderer::requeue);
   for (auto &req: list) {
     DEBUGDPP("requeueing {}", *pg, req);
     req.reset_instance_handle();
-    std::ignore = req.with_pg_int(shard_services, pg);
+    std::ignore = req.with_pg_int(pg);
   }
 }
 
@@ -47,9 +46,9 @@ void ClientRequest::complete_request()
 }
 
 ClientRequest::ClientRequest(
-  ShardServices &shard_services, crimson::net::ConnectionRef conn,
+  ShardServices &_shard_services, crimson::net::ConnectionRef conn,
   Ref<MOSDOp> &&m)
-  : put_historic_shard_services(&shard_services),
+  : shard_services(&_shard_services),
     conn(std::move(conn)),
     m(std::move(m)),
     instance_handle(new instance_handle_t)
@@ -98,9 +97,10 @@ bool ClientRequest::is_pg_op() const
     [](auto& op) { return ceph_osd_op_type_pg(op.op.op); });
 }
 
-seastar::future<> ClientRequest::with_pg_int(
-  ShardServices &shard_services, Ref<PG> pgref)
+seastar::future<> ClientRequest::with_pg_int(Ref<PG> pgref)
 {
+  ceph_assert_always(shard_services);
+
   LOG_PREFIX(ClientRequest::with_pg_int);
   epoch_t same_interval_since = pgref->get_interval_start_epoch();
   DEBUGDPP("{}: same_interval_since: {}", *pgref, *this, same_interval_since);
@@ -112,11 +112,11 @@ seastar::future<> ClientRequest::with_pg_int(
   auto instance_handle = get_instance_handle();
   auto &ihref = *instance_handle;
   return interruptor::with_interruption(
-    [FNAME, this, pgref, this_instance_id, &ihref, &shard_services]() mutable {
+    [FNAME, this, pgref, this_instance_id, &ihref]() mutable {
       DEBUGDPP("{} start", *pgref, *this);
       PG &pg = *pgref;
       if (pg.can_discard_op(*m)) {
-	return shard_services.send_incremental_map(
+	return shard_services->send_incremental_map(
 	  std::ref(*conn), m->get_map_epoch()
 	).then([FNAME, this, this_instance_id, pgref] {
 	  DEBUGDPP("{}: discarding {}", *pgref, *this, this_instance_id);
@@ -179,14 +179,12 @@ seastar::future<> ClientRequest::with_pg_int(
 }
 
 seastar::future<> ClientRequest::with_pg(
-  ShardServices &shard_services, Ref<PG> pgref)
+  ShardServices &_shard_services, Ref<PG> pgref)
 {
-  put_historic_shard_services = &shard_services;
+  shard_services = &_shard_services;
   pgref->client_request_orderer.add_request(*this);
   auto ret = on_complete.get_future();
-  std::ignore = with_pg_int(
-    shard_services, std::move(pgref)
-  );
+  std::ignore = with_pg_int(std::move(pgref));
   return ret;
 }
 
@@ -417,8 +415,8 @@ bool ClientRequest::is_misdirected(const PG& pg) const
 
 void ClientRequest::put_historic() const
 {
-  ceph_assert_always(put_historic_shard_services);
-  put_historic_shard_services->get_registry().put_historic(*this);
+  ceph_assert_always(shard_services);
+  shard_services->get_registry().put_historic(*this);
 }
 
 const SnapContext ClientRequest::get_snapc(
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 80645885a141..b374aacbbe94 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -29,9 +29,8 @@ class ShardServices;
 
 class ClientRequest final : public PhasedOperationT<ClientRequest>,
                             private CommonClientRequest {
-  // Initially set to primary core, updated to pg core after move,
-  // used by put_historic
-  ShardServices *put_historic_shard_services = nullptr;
+  // Initially set to primary core, updated to pg core after with_pg()
+  ShardServices *shard_services = nullptr;
 
   crimson::net::ConnectionRef conn;
   // must be after conn due to ConnectionPipeline's life-time
@@ -199,7 +198,7 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
       list.erase(list_t::s_iterator_to(request));
       intrusive_ptr_release(&request);
     }
-    void requeue(ShardServices &shard_services, Ref<PG> pg);
+    void requeue(Ref<PG> pg);
     void clear_and_cancel(PG &pg);
   };
   void complete_request();
@@ -243,8 +242,7 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
     conn = make_local_shared_foreign(std::move(_conn));
   }
 
-  seastar::future<> with_pg_int(
-    ShardServices &shard_services, Ref<PG> pg);
+  seastar::future<> with_pg_int(Ref<PG> pg);
 
 public:
   seastar::future<> with_pg(
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 0e16f6f118ba..7638b2fcdbeb 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1523,7 +1523,7 @@ void PG::on_change(ceph::os::Transaction &t) {
   wait_for_active_blocker.unblock();
   if (is_primary()) {
     logger().debug("{} {}: requeueing", *this, __func__);
-    client_request_orderer.requeue(shard_services, this);
+    client_request_orderer.requeue(this);
   } else {
     logger().debug("{} {}: dropping requests", *this, __func__);
     client_request_orderer.clear_and_cancel(*this);

From 829a9b4d5f61d97ef1d45a6ca972b0cd65d1306c Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 9 Jan 2024 14:39:41 +0800
Subject: [PATCH 1436/2492] crimson/osd/osd_operations/client_request: don't
 use and move pgref at the same time

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_operations/client_request.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 1611531fcd0b..d1eb9fbb350d 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -171,7 +171,7 @@ seastar::future<> ClientRequest::with_pg_int(Ref<PG> pgref)
       DEBUGDPP("{}.{}: interrupted due to {}",
 	       *pgref, *this, this_instance_id, eptr);
     }, pgref).finally(
-      [this, FNAME, opref=std::move(opref), pgref=std::move(pgref),
+      [this, FNAME, opref=std::move(opref), pgref,
        this_instance_id, instance_handle=std::move(instance_handle), &ihref] {
 	DEBUGDPP("{}.{}: exit", *pgref, *this, this_instance_id);
 	ihref.handle.exit();

From 3eb3327b848e511ea52e58669bbfa668093cdfcb Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 11 Dec 2023 14:38:51 +0800
Subject: [PATCH 1437/2492] crimson/osd: drop a foreign-copy to shard-0 for
 every pg operation

By using ConnectionRef before pg submission, and after that, change to
use ConnectionXcoreRef.

The intent is to drop the foreign copy of the connection to shard 0 at
pg submission time. This should remove two pairs of crosscore
communications in shard 0 for each I/O, one for connection-ref foreign
copy, another for connection-ref destruction.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/common/local_shared_foreign_ptr.h |  6 +--
 src/crimson/net/Fwd.h                         |  2 +
 src/crimson/osd/ops_executer.cc               | 10 ++--
 src/crimson/osd/ops_executer.h                | 10 ++--
 .../osd/osd_operations/client_request.cc      | 26 ++++++----
 .../osd/osd_operations/client_request.h       | 39 +++++++++------
 .../osd/osd_operations/logmissing_request.cc  |  7 +--
 .../osd/osd_operations/logmissing_request.h   | 39 +++++++++------
 .../logmissing_request_reply.cc               |  5 +-
 .../osd_operations/logmissing_request_reply.h | 39 +++++++++------
 .../osd/osd_operations/peering_event.cc       |  3 +-
 .../osd/osd_operations/peering_event.h        | 41 ++++++++++------
 .../osd/osd_operations/recovery_subrequest.cc |  5 +-
 .../osd/osd_operations/recovery_subrequest.h  | 41 ++++++++++------
 .../osd/osd_operations/replicated_request.cc  |  5 +-
 .../osd/osd_operations/replicated_request.h   | 39 +++++++++------
 .../osd/osd_operations/scrub_events.cc        |  3 +-
 src/crimson/osd/osd_operations/scrub_events.h | 49 ++++++++++++-------
 src/crimson/osd/pg.cc                         |  4 +-
 src/crimson/osd/pg.h                          |  8 +--
 src/crimson/osd/pg_shard_manager.h            | 17 ++++---
 src/crimson/osd/recovery_backend.cc           | 10 ++--
 src/crimson/osd/recovery_backend.h            | 10 ++--
 .../osd/replicated_recovery_backend.cc        |  2 +-
 src/crimson/osd/replicated_recovery_backend.h |  2 +-
 src/crimson/osd/watch.cc                      |  4 +-
 src/crimson/osd/watch.h                       | 12 ++---
 27 files changed, 267 insertions(+), 171 deletions(-)

diff --git a/src/crimson/common/local_shared_foreign_ptr.h b/src/crimson/common/local_shared_foreign_ptr.h
index c4bd1099a029..675442273e5b 100644
--- a/src/crimson/common/local_shared_foreign_ptr.h
+++ b/src/crimson/common/local_shared_foreign_ptr.h
@@ -27,9 +27,6 @@ namespace crimson {
  */
 template <typename PtrType>
 class local_shared_foreign_ptr {
-  using element_type = typename std::pointer_traits<PtrType>::element_type;
-  using pointer = element_type*;
-
   seastar::lw_shared_ptr<seastar::foreign_ptr<PtrType>> ptr;
 
   /// Wraps a pointer object and remembers the current core.
@@ -43,6 +40,9 @@ class local_shared_foreign_ptr {
     seastar::foreign_ptr<T> &&);
 
 public:
+  using element_type = typename std::pointer_traits<PtrType>::element_type;
+  using pointer = element_type*;
+
   /// Constructs a null local_shared_foreign_ptr<>.
   local_shared_foreign_ptr() = default;
 
diff --git a/src/crimson/net/Fwd.h b/src/crimson/net/Fwd.h
index 3a56cf5bb0a6..ad8eedd47773 100644
--- a/src/crimson/net/Fwd.h
+++ b/src/crimson/net/Fwd.h
@@ -38,6 +38,8 @@ class Connection;
 using ConnectionLRef = seastar::shared_ptr<Connection>;
 using ConnectionFRef = seastar::foreign_ptr<ConnectionLRef>;
 using ConnectionRef = ::crimson::local_shared_foreign_ptr<ConnectionLRef>;
+using ConnectionFFRef = seastar::foreign_ptr<ConnectionRef>;
+using ConnectionXcoreRef = ::crimson::local_shared_foreign_ptr<ConnectionRef>;
 
 class Dispatcher;
 class ChainedDispatchers;
diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 034fdde71693..4e2e1861f999 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -160,13 +160,13 @@ OpsExecuter::watch_ierrorator::future<> OpsExecuter::do_op_watch_subop_watch(
   logger().debug("{}", __func__);
   struct connect_ctx_t {
     ObjectContext::watch_key_t key;
-    crimson::net::ConnectionRef conn;
+    crimson::net::ConnectionXcoreRef conn;
     watch_info_t info;
 
     connect_ctx_t(
       const OSDOp& osd_op,
       const ExecutableMessage& msg,
-      crimson::net::ConnectionRef conn)
+      crimson::net::ConnectionXcoreRef conn)
       : key(osd_op.op.watch.cookie, msg.get_reqid().name),
         conn(conn),
         info(create_watch_info(osd_op, msg, conn->get_peer_addr())) {
@@ -323,13 +323,13 @@ OpsExecuter::watch_ierrorator::future<> OpsExecuter::do_op_notify(
     return crimson::ct_error::enoent::make();
   }
   struct notify_ctx_t {
-    crimson::net::ConnectionRef conn;
+    crimson::net::ConnectionXcoreRef conn;
     notify_info_t ninfo;
     const uint64_t client_gid;
     const epoch_t epoch;
 
     notify_ctx_t(const ExecutableMessage& msg,
-                 crimson::net::ConnectionRef conn)
+                 crimson::net::ConnectionXcoreRef conn)
       : conn(conn),
         client_gid(msg.get_reqid().name.num()),
         epoch(msg.get_map_epoch()) {
@@ -1067,7 +1067,7 @@ OpsExecuter::OpsExecuter(Ref<PG> pg,
                          ObjectContextRef _obc,
                          const OpInfo& op_info,
                          abstracted_msg_t&& msg,
-                         crimson::net::ConnectionRef conn,
+                         crimson::net::ConnectionXcoreRef conn,
                          const SnapContext& _snapc)
   : pg(std::move(pg)),
     obc(std::move(_obc)),
diff --git a/src/crimson/osd/ops_executer.h b/src/crimson/osd/ops_executer.h
index 1230b1c5a2e5..556e2f3b94a3 100644
--- a/src/crimson/osd/ops_executer.h
+++ b/src/crimson/osd/ops_executer.h
@@ -113,10 +113,10 @@ class OpsExecuter : public seastar::enable_lw_shared_from_this<OpsExecuter> {
   class ExecutableMessagePimpl final : ExecutableMessage {
     const ImplT* pimpl;
     // In crimson, conn is independently maintained outside Message.
-    const crimson::net::ConnectionRef conn;
+    const crimson::net::ConnectionXcoreRef conn;
   public:
     ExecutableMessagePimpl(const ImplT* pimpl,
-                           const crimson::net::ConnectionRef conn)
+                           const crimson::net::ConnectionXcoreRef conn)
       : pimpl(pimpl), conn(conn) {
     }
 
@@ -185,7 +185,7 @@ class OpsExecuter : public seastar::enable_lw_shared_from_this<OpsExecuter> {
     ceph::static_ptr<ExecutableMessage,
                      sizeof(ExecutableMessagePimpl<void>)>;
   abstracted_msg_t msg;
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionXcoreRef conn;
   std::optional<osd_op_params_t> osd_op_params;
   bool user_modify = false;
   ceph::os::Transaction txn;
@@ -372,7 +372,7 @@ class OpsExecuter : public seastar::enable_lw_shared_from_this<OpsExecuter> {
               ObjectContextRef obc,
               const OpInfo& op_info,
               abstracted_msg_t&& msg,
-              crimson::net::ConnectionRef conn,
+              crimson::net::ConnectionXcoreRef conn,
               const SnapContext& snapc);
 
 public:
@@ -381,7 +381,7 @@ class OpsExecuter : public seastar::enable_lw_shared_from_this<OpsExecuter> {
               ObjectContextRef obc,
               const OpInfo& op_info,
               const MsgT& msg,
-              crimson::net::ConnectionRef conn,
+              crimson::net::ConnectionXcoreRef conn,
               const SnapContext& snapc)
     : OpsExecuter(
         std::move(pg),
diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index d1eb9fbb350d..460f413efe38 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -49,7 +49,7 @@ ClientRequest::ClientRequest(
   ShardServices &_shard_services, crimson::net::ConnectionRef conn,
   Ref<MOSDOp> &&m)
   : shard_services(&_shard_services),
-    conn(std::move(conn)),
+    l_conn(std::move(conn)),
     m(std::move(m)),
     instance_handle(new instance_handle_t)
 {}
@@ -76,7 +76,8 @@ void ClientRequest::dump_detail(Formatter *f) const
 
 ConnectionPipeline &ClientRequest::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).client_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).client_request_conn_pipeline;
 }
 
 PerShardPipeline &ClientRequest::get_pershard_pipeline(
@@ -117,7 +118,7 @@ seastar::future<> ClientRequest::with_pg_int(Ref<PG> pgref)
       PG &pg = *pgref;
       if (pg.can_discard_op(*m)) {
 	return shard_services->send_incremental_map(
-	  std::ref(*conn), m->get_map_epoch()
+	  std::ref(get_foreign_connection()), m->get_map_epoch()
 	).then([FNAME, this, this_instance_id, pgref] {
 	  DEBUGDPP("{}: discarding {}", *pgref, *this, this_instance_id);
 	  pgref->client_request_orderer.remove_request(*this);
@@ -196,7 +197,7 @@ ClientRequest::process_pg_op(
     m
   ).then_interruptible([this, pg=std::move(pg)](MURef<MOSDOpReply> reply) {
     // TODO: gate the crosscore sending
-    return conn->send_with_throttling(std::move(reply));
+    return get_foreign_connection().send_with_throttling(std::move(reply));
   });
 }
 
@@ -211,7 +212,7 @@ auto ClientRequest::reply_op_error(const Ref<PG>& pg, int err)
   reply->set_reply_versions(eversion_t(), 0);
   reply->set_op_returns(std::vector<pg_log_op_return_item_t>{});
   // TODO: gate the crosscore sending
-  return conn->send_with_throttling(std::move(reply));
+  return get_foreign_connection().send_with_throttling(std::move(reply));
 }
 
 ClientRequest::interruptible_future<>
@@ -237,7 +238,7 @@ ClientRequest::process_op(
           CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK, false);
 	reply->set_reply_versions(completed->version, completed->user_version);
         // TODO: gate the crosscore sending
-        return conn->send_with_throttling(std::move(reply));
+        return get_foreign_connection().send_with_throttling(std::move(reply));
       } else {
 	DEBUGDPP("{}.{}: not completed, entering get_obc stage",
 		 *pg, *this, this_instance_id);
@@ -323,9 +324,10 @@ ClientRequest::do_process(
     return reply_op_error(pg, -ENAMETOOLONG);
   } else if (m->get_hobj().oid.name.empty()) {
     return reply_op_error(pg, -EINVAL);
-  } else if (pg->get_osdmap()->is_blocklisted(conn->get_peer_addr())) {
+  } else if (pg->get_osdmap()->is_blocklisted(
+        get_foreign_connection().get_peer_addr())) {
     DEBUGDPP("{}.{}: {} is blocklisted",
-	     *pg, *this, this_instance_id, conn->get_peer_addr());
+	     *pg, *this, this_instance_id, get_foreign_connection().get_peer_addr());
     return reply_op_error(pg, -EBLOCKLISTED);
   }
 
@@ -361,7 +363,9 @@ ClientRequest::do_process(
 	       *pg, *this, this_instance_id, m->get_hobj());
     }
   }
-  return pg->do_osd_ops(m, conn, obc, op_info, snapc).safe_then_unpack_interruptible(
+  return pg->do_osd_ops(
+    m, r_conn, obc, op_info, snapc
+  ).safe_then_unpack_interruptible(
     [FNAME, this, pg, this_instance_id, &ihref](
       auto submitted, auto all_completed) mutable {
       return submitted.then_interruptible(
@@ -379,7 +383,9 @@ ClientRequest::do_process(
 		 reply=std::move(reply)]() mutable {
 		  DEBUGDPP("{}.{}: sending response",
 			   *pg, *this, this_instance_id);
-		  return conn->send(std::move(reply));
+		  // TODO: gate the crosscore sending
+		  return get_foreign_connection(
+                      ).send_with_throttling(std::move(reply));
 		});
 	    }, crimson::ct_error::eagain::handle(
 	      [this, pg, this_instance_id, &ihref]() mutable {
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index b374aacbbe94..8bf396232a66 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -32,7 +32,9 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
   // Initially set to primary core, updated to pg core after with_pg()
   ShardServices *shard_services = nullptr;
 
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   // must be after conn due to ConnectionPipeline's life-time
   Ref<MOSDOp> m;
   OpInfo op_info;
@@ -224,22 +226,31 @@ class ClientRequest final : public PhasedOperationT<ClientRequest>,
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 
   seastar::future<> with_pg_int(Ref<PG> pg);
diff --git a/src/crimson/osd/osd_operations/logmissing_request.cc b/src/crimson/osd/osd_operations/logmissing_request.cc
index 5f92ccddfcdc..7e979131f060 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request.cc
@@ -22,7 +22,7 @@ namespace crimson::osd {
 
 LogMissingRequest::LogMissingRequest(crimson::net::ConnectionRef&& conn,
 		       Ref<MOSDPGUpdateLogMissing> &&req)
-  : conn{std::move(conn)},
+  : l_conn{std::move(conn)},
     req{std::move(req)}
 {}
 
@@ -48,7 +48,8 @@ void LogMissingRequest::dump_detail(Formatter *f) const
 
 ConnectionPipeline &LogMissingRequest::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).client_request_conn_pipeline;
 }
 
 PerShardPipeline &LogMissingRequest::get_pershard_pipeline(
@@ -81,7 +82,7 @@ seastar::future<> LogMissingRequest::with_pg(
           std::move(trigger), req->min_epoch);
       });
     }).then_interruptible([this, pg](auto) {
-      return pg->do_update_log_missing(req, conn);
+      return pg->do_update_log_missing(req, r_conn);
     }).then_interruptible([this] {
       logger().debug("{}: complete", *this);
       return handle.complete();
diff --git a/src/crimson/osd/osd_operations/logmissing_request.h b/src/crimson/osd/osd_operations/logmissing_request.h
index 5b01fee17b86..51c9d540cb5a 100644
--- a/src/crimson/osd/osd_operations/logmissing_request.h
+++ b/src/crimson/osd/osd_operations/logmissing_request.h
@@ -41,22 +41,31 @@ class LogMissingRequest final : public PhasedOperationT<LogMissingRequest> {
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 
   seastar::future<> with_pg(
@@ -77,7 +86,9 @@ class LogMissingRequest final : public PhasedOperationT<LogMissingRequest> {
 private:
   ClientRequest::PGPipeline &client_pp(PG &pg);
 
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   // must be after `conn` to ensure the ConnectionPipeline's is alive
   PipelineHandle handle;
   Ref<MOSDPGUpdateLogMissing> req;
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.cc b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
index 20a9bf7dd00e..39b3debf10f5 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.cc
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.cc
@@ -21,7 +21,7 @@ namespace crimson::osd {
 LogMissingRequestReply::LogMissingRequestReply(
   crimson::net::ConnectionRef&& conn,
   Ref<MOSDPGUpdateLogMissingReply> &&req)
-  : conn{std::move(conn)},
+  : l_conn{std::move(conn)},
     req{std::move(req)}
 {}
 
@@ -46,7 +46,8 @@ void LogMissingRequestReply::dump_detail(Formatter *f) const
 
 ConnectionPipeline &LogMissingRequestReply::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).replicated_request_conn_pipeline;
 }
 
 PerShardPipeline &LogMissingRequestReply::get_pershard_pipeline(
diff --git a/src/crimson/osd/osd_operations/logmissing_request_reply.h b/src/crimson/osd/osd_operations/logmissing_request_reply.h
index b01cae15421d..c741b41bd0f5 100644
--- a/src/crimson/osd/osd_operations/logmissing_request_reply.h
+++ b/src/crimson/osd/osd_operations/logmissing_request_reply.h
@@ -41,22 +41,31 @@ class LogMissingRequestReply final : public PhasedOperationT<LogMissingRequestRe
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 
   seastar::future<> with_pg(
@@ -75,7 +84,9 @@ class LogMissingRequestReply final : public PhasedOperationT<LogMissingRequestRe
 private:
   ClientRequest::PGPipeline &client_pp(PG &pg);
 
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   // must be after `conn` to ensure the ConnectionPipeline's is alive
   PipelineHandle handle;
   Ref<MOSDPGUpdateLogMissingReply> req;
diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc
index f12699e5e6a2..5c5c73e00867 100644
--- a/src/crimson/osd/osd_operations/peering_event.cc
+++ b/src/crimson/osd/osd_operations/peering_event.cc
@@ -135,7 +135,8 @@ PeeringEvent<T>::complete_rctx(ShardServices &shard_services, Ref<PG> pg)
 
 ConnectionPipeline &RemotePeeringEvent::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).peering_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).client_request_conn_pipeline;
 }
 
 PerShardPipeline &RemotePeeringEvent::get_pershard_pipeline(
diff --git a/src/crimson/osd/osd_operations/peering_event.h b/src/crimson/osd/osd_operations/peering_event.h
index dad1076a93bb..1e6bd957289f 100644
--- a/src/crimson/osd/osd_operations/peering_event.h
+++ b/src/crimson/osd/osd_operations/peering_event.h
@@ -101,7 +101,9 @@ class PeeringEvent : public PhasedOperationT<T> {
 
 class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
 protected:
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   // must be after conn due to ConnectionPipeline's life-time
   PipelineHandle handle;
 
@@ -117,7 +119,7 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
   template <typename... Args>
   RemotePeeringEvent(crimson::net::ConnectionRef conn, Args&&... args) :
     PeeringEvent(std::forward<Args>(args)...),
-    conn(conn)
+    l_conn(conn)
   {}
 
   std::tuple<
@@ -146,22 +148,31 @@ class RemotePeeringEvent : public PeeringEvent<RemotePeeringEvent> {
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 };
 
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.cc b/src/crimson/osd/osd_operations/recovery_subrequest.cc
index 1ea7ae12869e..06a3be1662c2 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.cc
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.cc
@@ -35,7 +35,7 @@ seastar::future<> RecoverySubRequest::with_pg(
   return interruptor::with_interruption([this, pgref] {
     LOG_PREFIX(RecoverySubRequest::with_pg);
     DEBUGI("{}: {}", "RecoverySubRequest::with_pg", *this);
-    return pgref->get_recovery_backend()->handle_recovery_op(m, conn
+    return pgref->get_recovery_backend()->handle_recovery_op(m, r_conn
     ).then_interruptible([this] {
       LOG_PREFIX(RecoverySubRequest::with_pg);
       DEBUGI("{}: complete", *this);
@@ -52,7 +52,8 @@ seastar::future<> RecoverySubRequest::with_pg(
 
 ConnectionPipeline &RecoverySubRequest::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).peering_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).client_request_conn_pipeline;
 }
 
 PerShardPipeline &RecoverySubRequest::get_pershard_pipeline(
diff --git a/src/crimson/osd/osd_operations/recovery_subrequest.h b/src/crimson/osd/osd_operations/recovery_subrequest.h
index 31e6045cb0ee..17c2faf97ea9 100644
--- a/src/crimson/osd/osd_operations/recovery_subrequest.h
+++ b/src/crimson/osd/osd_operations/recovery_subrequest.h
@@ -22,7 +22,7 @@ class RecoverySubRequest final : public PhasedOperationT<RecoverySubRequest> {
   RecoverySubRequest(
     crimson::net::ConnectionRef conn,
     Ref<MOSDFastDispatchOp>&& m)
-    : conn(conn), m(m) {}
+    : l_conn(conn), m(m) {}
 
   void print(std::ostream& out) const final
   {
@@ -44,22 +44,31 @@ class RecoverySubRequest final : public PhasedOperationT<RecoverySubRequest> {
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 
   seastar::future<> with_pg(
@@ -77,7 +86,9 @@ class RecoverySubRequest final : public PhasedOperationT<RecoverySubRequest> {
   > tracking_events;
 
 private:
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   // must be after `conn` to ensure the ConnectionPipeline's is alive
   PipelineHandle handle;
   Ref<MOSDFastDispatchOp> m;
diff --git a/src/crimson/osd/osd_operations/replicated_request.cc b/src/crimson/osd/osd_operations/replicated_request.cc
index 4e97db5be982..dc2adc37efef 100644
--- a/src/crimson/osd/osd_operations/replicated_request.cc
+++ b/src/crimson/osd/osd_operations/replicated_request.cc
@@ -22,7 +22,7 @@ namespace crimson::osd {
 
 RepRequest::RepRequest(crimson::net::ConnectionRef&& conn,
 		       Ref<MOSDRepOp> &&req)
-  : conn{std::move(conn)},
+  : l_conn{std::move(conn)},
     req{std::move(req)}
 {}
 
@@ -48,7 +48,8 @@ void RepRequest::dump_detail(Formatter *f) const
 
 ConnectionPipeline &RepRequest::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).replicated_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).client_request_conn_pipeline;
 }
 
 PerShardPipeline &RepRequest::get_pershard_pipeline(
diff --git a/src/crimson/osd/osd_operations/replicated_request.h b/src/crimson/osd/osd_operations/replicated_request.h
index 32cf271788ba..ff5dea6d6db3 100644
--- a/src/crimson/osd/osd_operations/replicated_request.h
+++ b/src/crimson/osd/osd_operations/replicated_request.h
@@ -41,22 +41,31 @@ class RepRequest final : public PhasedOperationT<RepRequest> {
 
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
   }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
   }
 
   seastar::future<> with_pg(
@@ -77,7 +86,9 @@ class RepRequest final : public PhasedOperationT<RepRequest> {
 private:
   ClientRequest::PGPipeline &client_pp(PG &pg);
 
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   PipelineHandle handle;
   Ref<MOSDRepOp> req;
 };
diff --git a/src/crimson/osd/osd_operations/scrub_events.cc b/src/crimson/osd/osd_operations/scrub_events.cc
index 4f54cf0b274f..9246233448b5 100644
--- a/src/crimson/osd/osd_operations/scrub_events.cc
+++ b/src/crimson/osd/osd_operations/scrub_events.cc
@@ -20,7 +20,8 @@ PGPeeringPipeline &RemoteScrubEventBaseT<T>::get_peering_pipeline(PG &pg)
 template <class T>
 ConnectionPipeline &RemoteScrubEventBaseT<T>::get_connection_pipeline()
 {
-  return get_osd_priv(conn.get()).peering_request_conn_pipeline;
+  return get_osd_priv(&get_local_connection()
+         ).peering_request_conn_pipeline;
 }
 
 template <class T>
diff --git a/src/crimson/osd/osd_operations/scrub_events.h b/src/crimson/osd/osd_operations/scrub_events.h
index 0793983d8c66..6aa7035d1856 100644
--- a/src/crimson/osd/osd_operations/scrub_events.h
+++ b/src/crimson/osd/osd_operations/scrub_events.h
@@ -24,7 +24,9 @@ class RemoteScrubEventBaseT : public PhasedOperationT<T> {
 
   PipelineHandle handle;
 
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionRef l_conn;
+  crimson::net::ConnectionXcoreRef r_conn;
+
   epoch_t epoch;
   spg_t pgid;
 
@@ -38,17 +40,41 @@ class RemoteScrubEventBaseT : public PhasedOperationT<T> {
 public:
   RemoteScrubEventBaseT(
     crimson::net::ConnectionRef conn, epoch_t epoch, spg_t pgid)
-    : conn(conn), epoch(epoch), pgid(pgid) {}
+    : l_conn(std::move(conn)), epoch(epoch), pgid(pgid) {}
 
   PGPeeringPipeline &get_peering_pipeline(PG &pg);
+
   ConnectionPipeline &get_connection_pipeline();
+
   PerShardPipeline &get_pershard_pipeline(ShardServices &);
 
-  crimson::net::Connection &get_connection() {
-    assert(conn);
-    return *conn;
+  crimson::net::Connection &get_local_connection() {
+    assert(l_conn);
+    assert(!r_conn);
+    return *l_conn;
   };
 
+  crimson::net::Connection &get_foreign_connection() {
+    assert(r_conn);
+    assert(!l_conn);
+    return *r_conn;
+  };
+
+  crimson::net::ConnectionFFRef prepare_remote_submission() {
+    assert(l_conn);
+    assert(!r_conn);
+    auto ret = seastar::make_foreign(std::move(l_conn));
+    l_conn.reset();
+    return ret;
+  }
+
+  void finish_remote_submission(crimson::net::ConnectionFFRef conn) {
+    assert(conn);
+    assert(!l_conn);
+    assert(!r_conn);
+    r_conn = make_local_shared_foreign(std::move(conn));
+  }
+
   static constexpr bool can_create() { return false; }
 
   spg_t get_pgid() const {
@@ -58,19 +84,6 @@ class RemoteScrubEventBaseT : public PhasedOperationT<T> {
   PipelineHandle &get_handle() { return handle; }
   epoch_t get_epoch() const { return epoch; }
 
-  seastar::future<crimson::net::ConnectionFRef> prepare_remote_submission() {
-    assert(conn);
-    return conn.get_foreign(
-    ).then([this](auto f_conn) {
-      conn.reset();
-      return f_conn;
-    });
-  }
-  void finish_remote_submission(crimson::net::ConnectionFRef _conn) {
-    assert(!conn);
-    conn = make_local_shared_foreign(std::move(_conn));
-  }
-
   seastar::future<> with_pg(
     ShardServices &shard_services, Ref<PG> pg);
 
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 7638b2fcdbeb..8d12c4287384 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -1059,7 +1059,7 @@ seastar::future<std::optional<eversion_t>> PG::submit_error_log(
 PG::do_osd_ops_iertr::future<PG::pg_rep_op_fut_t<MURef<MOSDOpReply>>>
 PG::do_osd_ops(
   Ref<MOSDOp> m,
-  crimson::net::ConnectionRef conn,
+  crimson::net::ConnectionXcoreRef conn,
   ObjectContextRef obc,
   const OpInfo &op_info,
   const SnapContext& snapc)
@@ -1382,7 +1382,7 @@ void PG::handle_rep_op_reply(const MOSDRepOpReply& m)
 
 PG::interruptible_future<> PG::do_update_log_missing(
   Ref<MOSDPGUpdateLogMissing> m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   if (__builtin_expect(stopping, false)) {
     return seastar::make_exception_future<>(
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index b829ea177dbb..e103d3bb5eec 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -526,7 +526,7 @@ class PG : public boost::intrusive_ref_counter<
   void handle_rep_op_reply(const MOSDRepOpReply& m);
   interruptible_future<> do_update_log_missing(
     Ref<MOSDPGUpdateLogMissing> m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
   interruptible_future<> do_update_log_missing_reply(
                          Ref<MOSDPGUpdateLogMissingReply> m);
 
@@ -570,7 +570,7 @@ class PG : public boost::intrusive_ref_counter<
                do_osd_ops_iertr::future<Ret>>;
   do_osd_ops_iertr::future<pg_rep_op_fut_t<MURef<MOSDOpReply>>> do_osd_ops(
     Ref<MOSDOp> m,
-    crimson::net::ConnectionRef conn,
+    crimson::net::ConnectionXcoreRef conn,
     ObjectContextRef obc,
     const OpInfo &op_info,
     const SnapContext& snapc);
@@ -798,7 +798,7 @@ class PG : public boost::intrusive_ref_counter<
 };
 
 struct PG::do_osd_ops_params_t {
-  crimson::net::ConnectionRef &get_connection() const {
+  crimson::net::ConnectionXcoreRef &get_connection() const {
     return conn;
   }
   osd_reqid_t get_reqid() const {
@@ -826,7 +826,7 @@ struct PG::do_osd_ops_params_t {
     return orig_source_inst.name;
   }
 
-  crimson::net::ConnectionRef &conn;
+  crimson::net::ConnectionXcoreRef &conn;
   osd_reqid_t reqid;
   utime_t mtime;
   epoch_t map_epoch;
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index 1559dde0cb1b..011a11605fd9 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -156,7 +156,8 @@ class PGShardManager {
       ShardServices &target_shard_services,
       typename T::IRef &&op,
       F &&f) {
-    auto &crosscore_ordering = get_osd_priv(&op->get_connection()).crosscore_ordering;
+    auto &crosscore_ordering = get_osd_priv(
+        &op->get_foreign_connection()).crosscore_ordering;
     if (crosscore_ordering.proceed_or_wait(cc_seq)) {
       return std::invoke(
         std::move(f),
@@ -182,6 +183,8 @@ class PGShardManager {
       F &&f) {
     ceph_assert(op->use_count() == 1);
     if (seastar::this_shard_id() == core) {
+      auto f_conn = op->prepare_remote_submission();
+      op->finish_remote_submission(std::move(f_conn));
       auto &target_shard_services = shard_services.local();
       return std::invoke(
         std::move(f),
@@ -190,17 +193,17 @@ class PGShardManager {
     }
     // Note: the ordering in only preserved until f is invoked.
     auto &opref = *op;
-    auto &crosscore_ordering = get_osd_priv(&opref.get_connection()).crosscore_ordering;
+    auto &crosscore_ordering = get_osd_priv(
+        &opref.get_local_connection()).crosscore_ordering;
     auto cc_seq = crosscore_ordering.prepare_submit(core);
     auto &logger = crimson::get_logger(ceph_subsys_osd);
     logger.debug("{}: send {} to the remote pg core {}",
                  opref, cc_seq, core);
     return opref.get_handle().complete(
-    ).then([&opref, this] {
-      get_local_state().registry.remove_from_registry(opref);
-      return opref.prepare_remote_submission();
-    }).then([op=std::move(op), f=std::move(f), this, core, cc_seq
-            ](auto f_conn) mutable {
+    ).then([this, core, cc_seq,
+            op=std::move(op), f=std::move(f)]() mutable {
+      get_local_state().registry.remove_from_registry(*op);
+      auto f_conn = op->prepare_remote_submission();
       return shard_services.invoke_on(
         core,
         [this, cc_seq,
diff --git a/src/crimson/osd/recovery_backend.cc b/src/crimson/osd/recovery_backend.cc
index b5394bfdc485..e5ba9f8d562b 100644
--- a/src/crimson/osd/recovery_backend.cc
+++ b/src/crimson/osd/recovery_backend.cc
@@ -70,7 +70,7 @@ void RecoveryBackend::WaitForObjectRecovery::stop() {
 
 void RecoveryBackend::handle_backfill_finish(
   MOSDPGBackfill& m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   logger().debug("{}", __func__);
   ceph_assert(!pg.is_primary());
@@ -125,7 +125,7 @@ RecoveryBackend::handle_backfill_finish_ack(
 RecoveryBackend::interruptible_future<>
 RecoveryBackend::handle_backfill(
   MOSDPGBackfill& m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   logger().debug("{}", __func__);
   if (pg.old_peering_msg(m.map_epoch, m.query_epoch)) {
@@ -227,7 +227,7 @@ RecoveryBackend::scan_for_backfill(
 RecoveryBackend::interruptible_future<>
 RecoveryBackend::handle_scan_get_digest(
   MOSDPGScan& m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   logger().debug("{}", __func__);
   if (false /* FIXME: check for backfill too full */) {
@@ -289,7 +289,7 @@ RecoveryBackend::handle_scan_digest(
 RecoveryBackend::interruptible_future<>
 RecoveryBackend::handle_scan(
   MOSDPGScan& m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   logger().debug("{}", __func__);
   if (pg.old_peering_msg(m.map_epoch, m.query_epoch)) {
@@ -311,7 +311,7 @@ RecoveryBackend::handle_scan(
 RecoveryBackend::interruptible_future<>
 RecoveryBackend::handle_recovery_op(
   Ref<MOSDFastDispatchOp> m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   switch (m->get_header().type) {
   case MSG_OSD_PG_BACKFILL:
diff --git a/src/crimson/osd/recovery_backend.h b/src/crimson/osd/recovery_backend.h
index abf695891596..c65bf5653924 100644
--- a/src/crimson/osd/recovery_backend.h
+++ b/src/crimson/osd/recovery_backend.h
@@ -66,7 +66,7 @@ class RecoveryBackend {
 
   virtual interruptible_future<> handle_recovery_op(
     Ref<MOSDFastDispatchOp> m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
 
   virtual interruptible_future<> recover_object(
     const hobject_t& soid,
@@ -212,22 +212,22 @@ class RecoveryBackend {
 private:
   void handle_backfill_finish(
     MOSDPGBackfill& m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
   interruptible_future<> handle_backfill_progress(
     MOSDPGBackfill& m);
   interruptible_future<> handle_backfill_finish_ack(
     MOSDPGBackfill& m);
   interruptible_future<> handle_backfill(
     MOSDPGBackfill& m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
 
   interruptible_future<> handle_scan_get_digest(
     MOSDPGScan& m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
   interruptible_future<> handle_scan_digest(
     MOSDPGScan& m);
   interruptible_future<> handle_scan(
     MOSDPGScan& m,
-    crimson::net::ConnectionRef conn);
+    crimson::net::ConnectionXcoreRef conn);
   interruptible_future<> handle_backfill_remove(MOSDPGBackfillRemove& m);
 };
diff --git a/src/crimson/osd/replicated_recovery_backend.cc b/src/crimson/osd/replicated_recovery_backend.cc
index 9a1060c440e2..a62cb793272d 100644
--- a/src/crimson/osd/replicated_recovery_backend.cc
+++ b/src/crimson/osd/replicated_recovery_backend.cc
@@ -1305,7 +1305,7 @@ ReplicatedRecoveryBackend::handle_recovery_delete_reply(
 RecoveryBackend::interruptible_future<>
 ReplicatedRecoveryBackend::handle_recovery_op(
   Ref<MOSDFastDispatchOp> m,
-  crimson::net::ConnectionRef conn)
+  crimson::net::ConnectionXcoreRef conn)
 {
   switch (m->get_header().type) {
   case MSG_OSD_PG_PULL:
diff --git a/src/crimson/osd/replicated_recovery_backend.h b/src/crimson/osd/replicated_recovery_backend.h
index cd1d9e067289..aac4531ee95a 100644
--- a/src/crimson/osd/replicated_recovery_backend.h
+++ b/src/crimson/osd/replicated_recovery_backend.h
@@ -25,7 +25,7 @@ class ReplicatedRecoveryBackend : public RecoveryBackend {
   {}
   interruptible_future<> handle_recovery_op(
     Ref<MOSDFastDispatchOp> m,
-    crimson::net::ConnectionRef conn) final;
+    crimson::net::ConnectionXcoreRef conn) final;
 
   interruptible_future<> recover_object(
     const hobject_t& soid,
diff --git a/src/crimson/osd/watch.cc b/src/crimson/osd/watch.cc
index 4573333c3dab..088054c5a6ed 100644
--- a/src/crimson/osd/watch.cc
+++ b/src/crimson/osd/watch.cc
@@ -78,7 +78,7 @@ Watch::~Watch()
   logger().debug("{} gid={} cookie={}", __func__, get_watcher_gid(), get_cookie());
 }
 
-seastar::future<> Watch::connect(crimson::net::ConnectionRef conn, bool)
+seastar::future<> Watch::connect(crimson::net::ConnectionXcoreRef conn, bool)
 {
   if (this->conn == conn) {
     logger().debug("conn={} already connected", conn);
@@ -235,7 +235,7 @@ std::ostream &operator<<(std::ostream &out, const notify_reply_t &rhs)
   return out;
 }
 
-Notify::Notify(crimson::net::ConnectionRef conn,
+Notify::Notify(crimson::net::ConnectionXcoreRef conn,
                const notify_info_t& ninfo,
                const uint64_t client_gid,
                const uint64_t user_version)
diff --git a/src/crimson/osd/watch.h b/src/crimson/osd/watch.h
index b3982141d86e..64708febd09c 100644
--- a/src/crimson/osd/watch.h
+++ b/src/crimson/osd/watch.h
@@ -34,7 +34,7 @@ class Watch : public seastar::enable_shared_from_this<Watch> {
   struct private_ctag_t{};
 
   std::set<NotifyRef, std::less<>> in_progress_notifies;
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionXcoreRef conn;
   crimson::osd::ObjectContextRef obc;
 
   watch_info_t winfo;
@@ -67,7 +67,7 @@ class Watch : public seastar::enable_shared_from_this<Watch> {
   }
   ~Watch();
 
-  seastar::future<> connect(crimson::net::ConnectionRef, bool);
+  seastar::future<> connect(crimson::net::ConnectionXcoreRef, bool);
   void disconnect();
   bool is_alive() const {
     return true;
@@ -131,7 +131,7 @@ std::ostream &operator<<(std::ostream &out, const notify_reply_t &rhs);
 class Notify : public seastar::enable_shared_from_this<Notify> {
   std::set<WatchRef> watchers;
   const notify_info_t ninfo;
-  crimson::net::ConnectionRef conn;
+  crimson::net::ConnectionXcoreRef conn;
   const uint64_t client_gid;
   const uint64_t user_version;
   bool complete{false};
@@ -154,14 +154,14 @@ class Notify : public seastar::enable_shared_from_this<Notify> {
   /// Called on Notify timeout
   void do_notify_timeout();
 
-  Notify(crimson::net::ConnectionRef conn,
+  Notify(crimson::net::ConnectionXcoreRef conn,
          const notify_info_t& ninfo,
          const uint64_t client_gid,
          const uint64_t user_version);
   template <class WatchIteratorT>
   Notify(WatchIteratorT begin,
          WatchIteratorT end,
-         crimson::net::ConnectionRef conn,
+         crimson::net::ConnectionXcoreRef conn,
          const notify_info_t& ninfo,
          const uint64_t client_gid,
          const uint64_t user_version);
@@ -207,7 +207,7 @@ class Notify : public seastar::enable_shared_from_this<Notify> {
 template <class WatchIteratorT>
 Notify::Notify(WatchIteratorT begin,
                WatchIteratorT end,
-               crimson::net::ConnectionRef conn,
+               crimson::net::ConnectionXcoreRef conn,
                const notify_info_t& ninfo,
                const uint64_t client_gid,
                const uint64_t user_version)

From 12e4686aee4cae72453c09f13ec877e9b23a5a65 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 22 Jan 2024 16:53:16 +0800
Subject: [PATCH 1438/2492] crimson/osd/osd_operations/scrub_events: fix to use
 get_peering_request_pipeline()

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/osd_operations/scrub_events.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd_operations/scrub_events.cc b/src/crimson/osd/osd_operations/scrub_events.cc
index 9246233448b5..9777ce20ef1b 100644
--- a/src/crimson/osd/osd_operations/scrub_events.cc
+++ b/src/crimson/osd/osd_operations/scrub_events.cc
@@ -28,7 +28,7 @@ template <class T>
 PerShardPipeline &RemoteScrubEventBaseT<T>::get_pershard_pipeline(
   ShardServices &shard_services)
 {
-  return shard_services.get_client_request_pipeline();
+  return shard_services.get_peering_request_pipeline();
 }
 
 template <class T>

From df23e4b2ea4f8647271a9ce541a1fdbc4d9fe4a6 Mon Sep 17 00:00:00 2001
From: Tobias Urdin <tobias.urdin@binero.se>
Date: Thu, 18 Jan 2024 09:29:05 +0000
Subject: [PATCH 1439/2492] rgw: invalidate and retry keystone admin token

We validate client tokens against the Keystone API by
sending our own "admin token" that is allowed to lookup
client tokens.

This "admin token" is cached and upon checking the cache
we verify the expiration on the token before using it but
we have no logic to invalidate the cache if the response
from the Keystone API says that the "admin token" is invalid.

Since we don't invalidate it and it still has not expired
it will stay in our cache and continue to cause Swift API
requests for clients to be dropped because of the invalid
admin token, until service is restarted, admin token is
expired (which it can already be) or until
the whole cache is dropped or TokenCache::invalidate()
called on the admin token.

There is probably multiple places in Keystone where it
invalidates tokens, but one example where the "admin token"
would be invalidated and return  HTTP 401 status code is if
the user that is configured in rgw_keystone_admin_user has
it's password changed (even if it's the same password as the
current one) then Keystone will invalidate it's cache and
invalidated existing tokens even if they have not expired yet.

Fixes: https://tracker.ceph.com/issues/64094

Signed-off-by: Tobias Urdin <tobias.urdin@binero.se>
---
 src/rgw/rgw_auth_keystone.cc | 44 +++++++++++++++++++++++++++---------
 src/rgw/rgw_keystone.cc      |  9 +++++++-
 src/rgw/rgw_keystone.h       |  4 +++-
 3 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc
index 552159823bb4..7b24306a15b0 100644
--- a/src/rgw/rgw_auth_keystone.cc
+++ b/src/rgw/rgw_auth_keystone.cc
@@ -48,6 +48,10 @@ TokenEngine::get_from_keystone(const DoutPrefixProvider* dpp,
   using RGWValidateKeystoneToken = \
     rgw::keystone::Service::RGWValidateKeystoneToken;
 
+  bool admin_token_retried = false;
+
+admin_token_retry:
+
   /* The container for plain response obtained from Keystone. It will be
    * parsed token_envelope_t::parse method. */
   ceph::bufferlist token_body_bl;
@@ -72,8 +76,10 @@ TokenEngine::get_from_keystone(const DoutPrefixProvider* dpp,
   }
 
   std::string admin_token;
-  if (rgw::keystone::Service::get_admin_token(dpp, token_cache, config,
-                                              y, admin_token) < 0) {
+  bool admin_token_cached = false;
+  int ret = rgw::keystone::Service::get_admin_token(dpp, token_cache, config,
+                                                    y, admin_token, admin_token_cached);
+  if (ret < 0) {
     throw -EINVAL;
   }
 
@@ -82,7 +88,7 @@ TokenEngine::get_from_keystone(const DoutPrefixProvider* dpp,
 
   validate.set_url(url);
 
-  int ret = validate.process(y);
+  ret = validate.process(y);
 
   /* NULL terminate for debug output. */
   token_body_bl.append(static_cast<char>(0));
@@ -91,12 +97,26 @@ TokenEngine::get_from_keystone(const DoutPrefixProvider* dpp,
    * Although failure at the parsing phase doesn't impose a threat,
    * this allows to return proper error code (EACCESS instead of EINVAL
    * or similar) and thus improves logging. */
-  if (validate.get_http_status() ==
-          /* Most likely: wrong admin credentials or admin token. */
-          RGWValidateKeystoneToken::HTTP_STATUS_UNAUTHORIZED ||
-      validate.get_http_status() ==
-          /* Most likely: non-existent token supplied by the client. */
-          RGWValidateKeystoneToken::HTTP_STATUS_NOTFOUND) {
+
+  /* If admin token is invalid we should expire it from the cache and
+     try one last time without the cache. */
+  bool admin_token_unauthorized = (validate.get_http_status() ==
+    RGWValidateKeystoneToken::HTTP_STATUS_UNAUTHORIZED);
+
+  if (admin_token_unauthorized && admin_token_cached) {
+    ldpp_dout(dpp, 20) << "invalidating admin_token cache due to 401" << dendl;
+    token_cache.invalidate_admin(dpp);
+
+    if (!admin_token_retried) {
+      ldpp_dout(dpp, 20) << "retrying with uncached admin_token" << dendl;
+      admin_token_retried = true;
+      goto admin_token_retry;
+    }
+  }
+
+  /* If admin token is invalid or token supplied by client is non-existent. */
+  if (admin_token_unauthorized || validate.get_http_status() ==
+        RGWValidateKeystoneToken::HTTP_STATUS_NOTFOUND) {
     ldpp_dout(dpp, 5) << "Failed keystone auth from " << url << " with "
                   << validate.get_http_status() << dendl;
     return boost::none;
@@ -404,8 +424,9 @@ EC2Engine::get_from_keystone(const DoutPrefixProvider* dpp, const std::string_vi
 
   /* get authentication token for Keystone. */
   std::string admin_token;
+  bool admin_token_cached = false;
   int ret = rgw::keystone::Service::get_admin_token(dpp, token_cache, config,
-                                                    y, admin_token);
+                                                    y, admin_token, admin_token_cached);
   if (ret < 0) {
     ldpp_dout(dpp, 2) << "s3 keystone: cannot get token for keystone access"
                   << dendl;
@@ -500,8 +521,9 @@ auto EC2Engine::get_secret_from_keystone(const DoutPrefixProvider* dpp,
 
   /* get authentication token for Keystone. */
   std::string admin_token;
+  bool admin_token_cached = false;
   int ret = rgw::keystone::Service::get_admin_token(dpp, token_cache, config,
-                                                    y, admin_token);
+                                                    y, admin_token, admin_token_cached);
   if (ret < 0) {
     ldpp_dout(dpp, 2) << "s3 keystone: cannot get token for keystone access"
                   << dendl;
diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc
index 7d5264f980ee..7713d3f2a50a 100644
--- a/src/rgw/rgw_keystone.cc
+++ b/src/rgw/rgw_keystone.cc
@@ -140,7 +140,8 @@ int Service::get_admin_token(const DoutPrefixProvider *dpp,
                              TokenCache& token_cache,
                              const Config& config,
                              optional_yield y,
-                             std::string& token)
+                             std::string& token,
+                             bool& token_cached)
 {
   /* Let's check whether someone uses the deprecated "admin token" feature
    * based on a shared secret from keystone.conf file. */
@@ -156,6 +157,7 @@ int Service::get_admin_token(const DoutPrefixProvider *dpp,
   if (token_cache.find_admin(t)) {
     ldpp_dout(dpp, 20) << "found cached admin token" << dendl;
     token = t.token.id;
+    token_cached = true;
     return 0;
   }
 
@@ -521,6 +523,11 @@ void TokenCache::invalidate(const DoutPrefixProvider *dpp, const std::string& to
   tokens.erase(iter);
 }
 
+void TokenCache::invalidate_admin(const DoutPrefixProvider *dpp)
+{
+  invalidate(dpp, admin_token_id);
+}
+
 bool TokenCache::going_down() const
 {
   return down_flag;
diff --git a/src/rgw/rgw_keystone.h b/src/rgw/rgw_keystone.h
index f800830767d3..7b18b1c546a3 100644
--- a/src/rgw/rgw_keystone.h
+++ b/src/rgw/rgw_keystone.h
@@ -123,7 +123,8 @@ class Service {
                              TokenCache& token_cache,
                              const Config& config,
                              optional_yield y,
-                             std::string& token);
+                             std::string& token,
+                             bool& token_cached);
   static int issue_admin_token_request(const DoutPrefixProvider *dpp,
                                        const Config& config,
                                        optional_yield y,
@@ -284,6 +285,7 @@ class TokenCache {
   void add_admin(const TokenEnvelope& token);
   void add_barbican(const TokenEnvelope& token);
   void invalidate(const DoutPrefixProvider *dpp, const std::string& token_id);
+  void invalidate_admin(const DoutPrefixProvider *dpp);
   bool going_down() const;
 private:
   void add_locked(const std::string& token_id, const TokenEnvelope& token,

From e60c698338510e4939f201de6fed4f569ea9ad01 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 30 Dec 2023 06:36:26 -0600
Subject: [PATCH 1440/2492] osd/scrub: remove the 'penalized jobs' queue

The 'penalized jobs' queue was used to track scrub jobs that had failed
to acquire their replicas, and to prevent those jobs from being retried
too quickly.  This functionality will be replaced by a
simple 'not before' delay (see the next commits).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_types.h                 |   4 +-
 src/osd/scrubber/osd_scrub_sched.cc | 108 ++--------------------------
 src/osd/scrubber/osd_scrub_sched.h  |  23 ------
 src/osd/scrubber/scrub_job.h        |  15 ++--
 src/test/osd/test_scrub_sched.cc    |   1 -
 5 files changed, 12 insertions(+), 139 deletions(-)

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index ff02ca5730f7..66f39a91ac4d 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -403,7 +403,7 @@ struct pg_t {
   uint32_t m_seed;
 
   pg_t() : m_pool(0), m_seed(0) {}
-  pg_t(ps_t seed, uint64_t pool) :
+  constexpr pg_t(ps_t seed, uint64_t pool) :
     m_pool(pool), m_seed(seed) {}
   // cppcheck-suppress noExplicitConstructor
   pg_t(const ceph_pg& cpg) :
@@ -521,7 +521,7 @@ struct spg_t {
   pg_t pgid;
   shard_id_t shard;
   spg_t() : shard(shard_id_t::NO_SHARD) {}
-  spg_t(pg_t pgid, shard_id_t shard) : pgid(pgid), shard(shard) {}
+  constexpr spg_t(pg_t pgid, shard_id_t shard) : pgid(pgid), shard(shard) {}
   explicit spg_t(pg_t pgid) : pgid(pgid), shard(shard_id_t::NO_SHARD) {}
   auto operator<=>(const spg_t&) const = default;
   unsigned get_split_bits(unsigned pg_num) const {
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index 1b3506a35e50..bb72ae4cb3fb 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -188,75 +188,27 @@ sched_params_t ScrubQueue::determine_scrub_time(
 }
 
 
-// used under jobs_lock
-void ScrubQueue::move_failed_pgs(utime_t now_is)
-{
-  int punished_cnt{0};	// for log/debug only
-
-  for (auto job = to_scrub.begin(); job != to_scrub.end();) {
-    if ((*job)->resources_failure) {
-      auto sjob = *job;
-
-      // last time it was scheduled for a scrub, this PG failed in securing
-      // remote resources. Move it to the secondary scrub queue.
-
-      dout(15) << "moving " << sjob->pgid
-	       << " state: " << ScrubJob::qu_state_text(sjob->state) << dendl;
-
-      // determine the penalty time, after which the job should be reinstated
-      utime_t after = now_is;
-      after += conf()->osd_scrub_sleep * 2 + utime_t{300'000ms};
-
-      // note: currently - not taking 'deadline' into account when determining
-      // 'penalty_timeout'.
-      sjob->penalty_timeout = after;
-      sjob->resources_failure = false;
-      sjob->updated = false;  // as otherwise will be pardoned immediately
-
-      // place in the penalty list, and remove from the to-scrub group
-      penalized.push_back(sjob);
-      job = to_scrub.erase(job);
-      punished_cnt++;
-    } else {
-      job++;
-    }
-  }
-
-  if (punished_cnt) {
-    dout(15) << "# of jobs penalized: " << punished_cnt << dendl;
-  }
-}
-
 std::vector<ScrubTargetId> ScrubQueue::ready_to_scrub(
     OSDRestrictions restrictions,  // note: 4B in size! (copy)
     utime_t scrub_tick)
 {
   dout(10) << fmt::format(
-		  " @{:s}: reg./pen. sizes: {} / {} ({})", scrub_tick,
-		  to_scrub.size(), penalized.size(), restrictions)
+		  " @{:s}: registered: {} ({})", scrub_tick,
+		  to_scrub.size(), restrictions)
 	   << dendl;
+
   //  create a list of candidates (copying, as otherwise creating a deadlock):
-  //  - possibly restore penalized
   //  - (if we didn't handle directly) remove invalid jobs
   //  - create a copy of the to_scrub (possibly up to first not-ripe)
-  //  - same for the penalized (although that usually be a waste)
   //  unlock, then try the lists
-
   std::unique_lock lck{jobs_lock};
 
-  // pardon all penalized jobs that have deadlined (or were updated)
-  scan_penalized(restore_penalized, scrub_tick);
-  restore_penalized = false;
-
   // remove the 'updated' flag from all entries
   std::for_each(
       to_scrub.begin(), to_scrub.end(),
       [](const auto& jobref) -> void { jobref->updated = false; });
 
-  // add failed scrub attempts to the penalized list
-  move_failed_pgs(scrub_tick);
-
-  // collect all valid & ripe jobs from the two lists. Note that we must copy,
+  // collect all valid & ripe jobs. Note that we must copy,
   // as when we use the lists we will not be holding jobs_lock (see
   // explanation above)
 
@@ -264,7 +216,6 @@ std::vector<ScrubTargetId> ScrubQueue::ready_to_scrub(
   // transformed into a vector of targets (which, in this phase, are
   // the PG id-s).
   auto to_scrub_copy = collect_ripe_jobs(to_scrub, restrictions, scrub_tick);
-  auto penalized_copy = collect_ripe_jobs(penalized, restrictions, scrub_tick);
   lck.unlock();
 
   std::vector<ScrubTargetId> all_ready;
@@ -272,13 +223,6 @@ std::vector<ScrubTargetId> ScrubQueue::ready_to_scrub(
       to_scrub_copy.cbegin(), to_scrub_copy.cend(),
       std::back_inserter(all_ready),
       [](const auto& jobref) -> ScrubTargetId { return jobref->pgid; });
-  // not bothering to handle the "reached the penalized - so all should be
-  // forgiven" case, as the penalty queue is destined to be removed in a
-  // followup PR.
-  std::transform(
-      penalized_copy.cbegin(), penalized_copy.cend(),
-      std::back_inserter(all_ready),
-      [](const auto& jobref) -> ScrubTargetId { return jobref->pgid; });
   return all_ready;
 }
 
@@ -389,71 +333,29 @@ Scrub::scrub_schedule_t ScrubQueue::adjust_target_time(
 }
 
 
-// note: called with jobs_lock held
-void ScrubQueue::scan_penalized(bool forgive_all, utime_t time_now)
-{
-  dout(20) << time_now << (forgive_all ? " all " : " - ") << penalized.size()
-	   << dendl;
-
-  // clear dead entries (deleted PGs, or those PGs we are no longer their
-  // primary)
-  rm_unregistered_jobs(penalized);
-
-  if (forgive_all) {
-
-    std::copy(penalized.begin(), penalized.end(), std::back_inserter(to_scrub));
-    penalized.clear();
-
-  } else {
-
-    auto forgiven_last = std::partition(
-      penalized.begin(),
-      penalized.end(),
-      [time_now](const auto& e) {
-	return (*e).updated || ((*e).penalty_timeout <= time_now);
-      });
-
-    std::copy(penalized.begin(), forgiven_last, std::back_inserter(to_scrub));
-    penalized.erase(penalized.begin(), forgiven_last);
-    dout(20) << "penalized after screening: " << penalized.size() << dendl;
-  }
-}
-
 void ScrubQueue::dump_scrubs(ceph::Formatter* f) const
 {
   ceph_assert(f != nullptr);
   std::lock_guard lck(jobs_lock);
 
   f->open_array_section("scrubs");
-
   std::for_each(
       to_scrub.cbegin(), to_scrub.cend(),
       [&f](const Scrub::ScrubJobRef& j) { j->dump(f); });
-
-  std::for_each(
-      penalized.cbegin(), penalized.cend(),
-      [&f](const Scrub::ScrubJobRef& j) { j->dump(f); });
-
   f->close_section();
 }
 
 ScrubQContainer ScrubQueue::list_registered_jobs() const
 {
   ScrubQContainer all_jobs;
-  all_jobs.reserve(to_scrub.size() + penalized.size());
+  all_jobs.reserve(to_scrub.size());
   dout(20) << " size: " << all_jobs.capacity() << dendl;
 
   std::lock_guard lck{jobs_lock};
-
   std::copy_if(to_scrub.begin(),
 	       to_scrub.end(),
 	       std::back_inserter(all_jobs),
 	       registered_job);
-  std::copy_if(penalized.begin(),
-	       penalized.end(),
-	       std::back_inserter(all_jobs),
-	       registered_job);
-
   return all_jobs;
 }
 
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index bd6de1c93478..83578dff7d93 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -30,7 +30,6 @@
 │                                        │
 │                                        │
 │  ScrubQContainer    to_scrub <>────────┼────────┐
-│  ScrubQContainer    penalized          │        │
 │                                        │        │
 │                                        │        │
 │  OSD_wide resource counters            │        │
@@ -146,11 +145,6 @@ class ScrubSchedListener {
  * the queue of PGs waiting to be scrubbed.
  * Main operations are scheduling/unscheduling a PG to be scrubbed at a certain
  * time.
- *
- * A "penalty" queue maintains those PGs that have failed to reserve the
- * resources of their replicas. The PGs in this list will be reinstated into the
- * scrub queue when all eligible PGs were already handled, or after a timeout
- * (or if their deadline has passed [[disabled at this time]]).
  */
 class ScrubQueue {
  public:
@@ -282,8 +276,6 @@ class ScrubQueue {
   mutable ceph::mutex jobs_lock = ceph::make_mutex("ScrubQueue::jobs_lock");
 
   Scrub::ScrubQContainer to_scrub;   ///< scrub jobs (i.e. PGs) to scrub
-  Scrub::ScrubQContainer penalized;  ///< those that failed to reserve remote resources
-  bool restore_penalized{false};
 
   static inline constexpr auto registered_job = [](const auto& jobref) -> bool {
     return jobref->state == Scrub::qu_state_t::registered;
@@ -293,11 +285,6 @@ class ScrubQueue {
     return jobref->state == Scrub::qu_state_t::not_registered;
   };
 
-  /**
-   * Are there scrub jobs that should be reinstated?
-   */
-  void scan_penalized(bool forgive_all, utime_t time_now);
-
   /**
    * clear dead entries (unregistered, or belonging to removed PGs) from a
    * queue. Job state is changed to match new status.
@@ -353,16 +340,6 @@ class ScrubQueue {
   Scrub::scrub_schedule_t adjust_target_time(
     const Scrub::sched_params_t& recomputed_params) const;
 
-  /**
-   * Look for scrub jobs that have their 'resources_failure' set. These jobs
-   * have failed to acquire remote resources last time we've initiated a scrub
-   * session on them. They are now moved from the 'to_scrub' queue to the
-   * 'penalized' set.
-   *
-   * locking: called with job_lock held
-   */
-  void move_failed_pgs(utime_t now_is);
-
 protected: // used by the unit-tests
   /**
    * unit-tests will override this function to return a mock time
diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index 11e7388f6362..b75141ea179c 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -77,8 +77,6 @@ class ScrubJob final : public RefCountedObject {
    * 'updated' is a temporary flag, used to create a barrier after
    * 'sched_time' and 'deadline' (or any other job entry) were modified by
    * different task.
-   * 'updated' also signals the need to move a job back from the penalized
-   * queue to the regular one.
    */
   std::atomic_bool updated{false};
 
@@ -89,8 +87,6 @@ class ScrubJob final : public RefCountedObject {
   bool blocked{false};
   utime_t blocked_since{};
 
-  utime_t penalty_timeout{0, 0};
-
   CephContext* cct;
 
   bool high_priority{false};
@@ -231,12 +227,11 @@ struct formatter<Scrub::ScrubJob> {
   {
     return fmt::format_to(
 	ctx.out(),
-	"pg[{}] @ {:s} (dl:{:s}) - <{}> / failure: {} / pen. t.o.: {:s} / "
-	"queue "
-	"state: {:.7}",
-	sjob.pgid, sjob.schedule.scheduled_at, sjob.schedule.deadline,
-	sjob.registration_state(), sjob.resources_failure, sjob.penalty_timeout,
-	sjob.state.load(std::memory_order_relaxed));
+	"pg[{}] @ {:s} (dl:{:s}) - <{}> / failure: {} / queue state: "
+	"{:.7}",
+	sjob.pgid, sjob.schedule.scheduled_at,
+	sjob.schedule.deadline, sjob.registration_state(),
+	sjob.resources_failure, sjob.state.load(std::memory_order_relaxed));
   }
 };
 
diff --git a/src/test/osd/test_scrub_sched.cc b/src/test/osd/test_scrub_sched.cc
index da8fb3bb5e3e..afeb32b21d9d 100644
--- a/src/test/osd/test_scrub_sched.cc
+++ b/src/test/osd/test_scrub_sched.cc
@@ -57,7 +57,6 @@ class ScrubSchedTestWrapper : public ScrubQueue {
   void rm_unregistered_jobs()
   {
     ScrubQueue::rm_unregistered_jobs(to_scrub);
-    ScrubQueue::rm_unregistered_jobs(penalized);
   }
 
   ScrubQContainer collect_ripe_jobs()

From d294ea80cc018dc3f893fcb2c4df9a1ad8bb6a04 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 31 Dec 2023 10:18:09 -0600
Subject: [PATCH 1441/2492] osd/scrub: introduce a 'not before' attribute for
 scrub jobs

The NB enables the OSD to delay the next attempt to schedule a specific
scrub job.  This is useful for jobs that have failed for whatever
reason, especially if the primary has failed to acquire the replicas.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc       |  5 +++--
 src/osd/scrubber/osd_scrub.h        | 10 +++++----
 src/osd/scrubber/osd_scrub_sched.cc | 22 +++++++++++---------
 src/osd/scrubber/osd_scrub_sched.h  | 12 +++++++----
 src/osd/scrubber/pg_scrubber.cc     |  6 +++---
 src/osd/scrubber/scrub_job.cc       | 32 ++++++++++++++++++++---------
 src/osd/scrubber/scrub_job.h        | 19 ++++++++++++-----
 7 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index e5abe03feaad..a721dc1cb9c0 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -429,9 +429,10 @@ Scrub::sched_params_t OsdScrub::determine_scrub_time(
 
 void OsdScrub::update_job(
     Scrub::ScrubJobRef sjob,
-    const Scrub::sched_params_t& suggested)
+    const Scrub::sched_params_t& suggested,
+    bool reset_notbefore)
 {
-  m_queue.update_job(sjob, suggested);
+  m_queue.update_job(sjob, suggested, reset_notbefore);
 }
 
 void OsdScrub::register_with_osd(
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index ce7b8524d69c..407eae059079 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -90,21 +90,23 @@ class OsdScrub {
    *   the registration will be with "beginning of time" target, making the
    *   scrub-job eligible to immediate scrub (given that external conditions
    *   do not prevent scrubbing)
-   *
    * - 'must' is asserted, and the suggested time is 'now':
    *   This happens if our stats are unknown. The results are similar to the
    *   previous scenario.
-   *
    * - not a 'must': we take the suggested time as a basis, and add to it some
    *   configuration / random delays.
-   *
    *  ('must' is Scrub::sched_params_t.is_must)
    *
+   *  'reset_notbefore' is used to reset the 'not_before' time to the updated
+   *  'scheduled_at' time. This is used whenever the scrub-job schedule is
+   *  updated not as a result of a scrub attempt failure.
+   *
    *  locking: not using the jobs_lock
    */
   void update_job(
       Scrub::ScrubJobRef sjob,
-      const Scrub::sched_params_t& suggested);
+      const Scrub::sched_params_t& suggested,
+      bool reset_notbefore);
 
   /**
    * Add the scrub job to the list of jobs (i.e. list of PGs) to be periodically
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index bb72ae4cb3fb..836bf740ec08 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -94,7 +94,7 @@ void ScrubQueue::register_with_osd(
   switch (state_at_entry) {
     case qu_state_t::registered:
       // just updating the schedule?
-      update_job(scrub_job, suggested);
+      update_job(scrub_job, suggested, false /* keep n.b. delay */);
       break;
 
     case qu_state_t::not_registered:
@@ -110,7 +110,7 @@ void ScrubQueue::register_with_osd(
 	  break;
 	}
 
-	update_job(scrub_job, suggested);
+	update_job(scrub_job, suggested, true /* resets not_before */);
 	to_scrub.push_back(scrub_job);
 	scrub_job->in_queues = true;
 	scrub_job->state = qu_state_t::registered;
@@ -124,7 +124,7 @@ void ScrubQueue::register_with_osd(
 	// at any minute
 	std::lock_guard lck{jobs_lock};
 
-	update_job(scrub_job, suggested);
+	update_job(scrub_job, suggested, true /* resets not_before */);
 	if (scrub_job->state == qu_state_t::not_registered) {
 	  dout(5) << " scrub job state changed to 'not registered'" << dendl;
 	  to_scrub.push_back(scrub_job);
@@ -138,18 +138,19 @@ void ScrubQueue::register_with_osd(
   dout(10) << fmt::format(
 		"pg[{}] sched-state changed from <{:.14}> to <{:.14}> (@{:s})",
 		scrub_job->pgid, state_at_entry, scrub_job->state.load(),
-		scrub_job->schedule.scheduled_at)
+		scrub_job->schedule.not_before)
 	   << dendl;
 }
 
-// look mommy - no locks!
+
 void ScrubQueue::update_job(Scrub::ScrubJobRef scrub_job,
-			    const sched_params_t& suggested)
+			    const sched_params_t& suggested,
+                            bool reset_nb)
 {
   // adjust the suggested scrub time according to OSD-wide status
   auto adjusted = adjust_target_time(suggested);
-  scrub_job->update_schedule(adjusted);
   scrub_job->high_priority = suggested.is_must == must_scrub_t::mandatory;
+  scrub_job->update_schedule(adjusted, reset_nb);
 }
 
 sched_params_t ScrubQueue::determine_scrub_time(
@@ -262,7 +263,7 @@ ScrubQContainer ScrubQueue::collect_ripe_jobs(
     utime_t time_now)
 {
   auto filtr = [time_now, rst = restrictions](const auto& jobref) -> bool {
-    return jobref->schedule.scheduled_at <= time_now &&
+    return jobref->schedule.not_before <= time_now &&
 	   (!rst.high_priority_only || jobref->high_priority) &&
 	   (!rst.only_deadlined || (!jobref->schedule.deadline.is_zero() &&
 				    jobref->schedule.deadline <= time_now));
@@ -280,7 +281,8 @@ ScrubQContainer ScrubQueue::collect_ripe_jobs(
     for (const auto& jobref : group) {
       if (!filtr(jobref)) {
 	dout(20) << fmt::format(
-			" not ripe: {} @ {:s}", jobref->pgid,
+			" not ripe: {} @ {:s} ({:s})", jobref->pgid,
+                        jobref->schedule.not_before,
 			jobref->schedule.scheduled_at)
 		 << dendl;
       }
@@ -295,7 +297,7 @@ Scrub::scrub_schedule_t ScrubQueue::adjust_target_time(
   const sched_params_t& times) const
 {
   Scrub::scrub_schedule_t sched_n_dead{
-    times.proposed_time, times.proposed_time};
+    times.proposed_time, times.proposed_time, times.proposed_time};
 
   if (times.is_must == Scrub::must_scrub_t::not_mandatory) {
     // unless explicitly requested, postpone the scrub with a random delay
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index 83578dff7d93..d94ca8cf7579 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -202,19 +202,23 @@ class ScrubQueue {
    *   the registration will be with "beginning of time" target, making the
    *   scrub-job eligible to immediate scrub (given that external conditions
    *   do not prevent scrubbing)
-   *
    * - 'must' is asserted, and the suggested time is 'now':
    *   This happens if our stats are unknown. The results are similar to the
    *   previous scenario.
-   *
    * - not a 'must': we take the suggested time as a basis, and add to it some
    *   configuration / random delays.
-   *
    *  ('must' is sched_params_t.is_must)
    *
+   *  'reset_notbefore' is used to reset the 'not_before' time to the updated
+   *  'scheduled_at' time. This is used whenever the scrub-job schedule is
+   *  updated not as a result of a scrub attempt failure.
+   *
    *  locking: not using the jobs_lock
    */
-  void update_job(Scrub::ScrubJobRef sjob, const sched_params_t& suggested);
+  void update_job(
+      Scrub::ScrubJobRef sjob,
+      const sched_params_t& suggested,
+      bool reset_notbefore);
 
   sched_params_t determine_scrub_time(const requested_scrub_t& request_flags,
 				      const pg_info_t& pg_info,
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 2dae53273a70..9cd082a87947 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -547,7 +547,7 @@ void PgScrubber::update_scrub_job(const requested_scrub_t& request_flags)
     ceph_assert(m_pg->is_locked());
     auto suggested = m_osds->get_scrub_services().determine_scrub_time(
 	request_flags, m_pg->info, m_pg->get_pgpool().info.opts);
-    m_osds->get_scrub_services().update_job(m_scrub_job, suggested);
+    m_osds->get_scrub_services().update_job(m_scrub_job, suggested, true);
     m_pg->publish_stats_to_osd();
   }
 
@@ -2126,7 +2126,7 @@ pg_scrubbing_status_t PgScrubber::get_schedule() const
 		  !m_planned_scrub.must_deep_scrub;
 
   // are we ripe for scrubbing?
-  if (now_is > m_scrub_job->schedule.scheduled_at) {
+  if (now_is > m_scrub_job->schedule.not_before) {
     // we are waiting for our turn at the OSD.
     return pg_scrubbing_status_t{m_scrub_job->schedule.scheduled_at,
 				 0,
@@ -2136,7 +2136,7 @@ pg_scrubbing_status_t PgScrubber::get_schedule() const
 				 periodic};
   }
 
-  return pg_scrubbing_status_t{m_scrub_job->schedule.scheduled_at,
+  return pg_scrubbing_status_t{m_scrub_job->schedule.not_before,
 			       0,
 			       pg_scrub_sched_status_t::scheduled,
 			       false,
diff --git a/src/osd/scrubber/scrub_job.cc b/src/osd/scrubber/scrub_job.cc
index 35071af5fd51..1875641e754a 100644
--- a/src/osd/scrubber/scrub_job.cc
+++ b/src/osd/scrubber/scrub_job.cc
@@ -43,18 +43,30 @@ ostream& operator<<(ostream& out, const ScrubJob& sjob)
 }
 }  // namespace std
 
-void ScrubJob::update_schedule(const Scrub::scrub_schedule_t& adjusted)
+void ScrubJob::update_schedule(
+    const Scrub::scrub_schedule_t& adjusted,
+    bool reset_nb)
 {
-  schedule = adjusted;
-  penalty_timeout = utime_t(0, 0);  // helps with debugging
+  dout(15)
+      << fmt::format(
+	     "was: nb:{:s}({:s}). Called with: rest?{} nb:{:s} ({:s}) ({})",
+	     schedule.not_before, schedule.scheduled_at, reset_nb,
+	     adjusted.not_before, adjusted.scheduled_at, registration_state())
+      << dendl;
+  schedule.scheduled_at = adjusted.scheduled_at;
+  schedule.deadline = adjusted.deadline;
+
+  if (reset_nb || schedule.not_before < schedule.scheduled_at) {
+    schedule.not_before = schedule.scheduled_at;
+  }
 
   // 'updated' is changed here while not holding jobs_lock. That's OK, as
   // the (atomic) flag will only be cleared by select_pg_and_scrub() after
   // scan_penalized() is called and the job was moved to the to_scrub queue.
   updated = true;
   dout(10) << fmt::format(
-		  "adjusted: {:s} ({})", schedule.scheduled_at,
-		  registration_state())
+		  "adjusted: nb:{:s} ({:s}) ({})", schedule.not_before,
+		  schedule.scheduled_at, registration_state())
 	   << dendl;
 }
 
@@ -67,15 +79,14 @@ std::string ScrubJob::scheduling_state(utime_t now_is, bool is_deep_expected)
   }
 
   // if the time has passed, we are surely in the queue
-  // (note that for now we do not tell client if 'penalized')
-  if (now_is > schedule.scheduled_at) {
+  if (now_is > schedule.not_before) {
     // we are never sure that the next scrub will indeed be shallow:
     return fmt::format("queued for {}scrub", (is_deep_expected ? "deep " : ""));
   }
 
   return fmt::format(
-      "{}scrub scheduled @ {:s}", (is_deep_expected ? "deep " : ""),
-      schedule.scheduled_at);
+      "{}scrub scheduled @ {:s} ({:s})", (is_deep_expected ? "deep " : ""),
+      schedule.not_before, schedule.scheduled_at);
 }
 
 std::ostream& ScrubJob::gen_prefix(std::ostream& out, std::string_view fn) const
@@ -100,7 +111,8 @@ void ScrubJob::dump(ceph::Formatter* f) const
 {
   f->open_object_section("scrub");
   f->dump_stream("pgid") << pgid;
-  f->dump_stream("sched_time") << schedule.scheduled_at;
+  f->dump_stream("sched_time") << schedule.not_before;
+  f->dump_stream("orig_sched_time") << schedule.scheduled_at;
   f->dump_stream("deadline") << schedule.deadline;
   f->dump_bool("forced",
 	       schedule.scheduled_at == PgScrubber::scrub_must_stamp());
diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index b75141ea179c..e6765c767b96 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -38,6 +38,7 @@ enum class qu_state_t {
 struct scrub_schedule_t {
   utime_t scheduled_at{};
   utime_t deadline{0, 0};
+  utime_t not_before{utime_t::max()};
 };
 
 struct sched_params_t {
@@ -66,7 +67,7 @@ class ScrubJob final : public RefCountedObject {
 
   /**
    * the old 'is_registered'. Set whenever the job is registered with the OSD,
-   * i.e. is in either the 'to_scrub' or the 'penalized' vectors.
+   * i.e. is in 'to_scrub'.
    */
   std::atomic_bool in_queues{false};
 
@@ -93,7 +94,7 @@ class ScrubJob final : public RefCountedObject {
 
   ScrubJob(CephContext* cct, const spg_t& pg, int node_id);
 
-  utime_t get_sched_time() const { return schedule.scheduled_at; }
+  utime_t get_sched_time() const { return schedule.not_before; }
 
   static std::string_view qu_state_text(qu_state_t st);
 
@@ -107,7 +108,15 @@ class ScrubJob final : public RefCountedObject {
     return qu_state_text(state.load(std::memory_order_relaxed));
   }
 
-  void update_schedule(const scrub_schedule_t& adjusted);
+  /**
+   * 'reset_failure_penalty' is used to reset the 'not_before' jo attribute to
+   * the updated 'scheduled_at' time. This is used whenever the scrub-job
+   * schedule is updated, and the update is not a result of a scrub attempt
+   * failure.
+   */
+  void update_schedule(
+      const scrub_schedule_t& adjusted,
+      bool reset_failure_penalty);
 
   void dump(ceph::Formatter* f) const;
 
@@ -227,9 +236,9 @@ struct formatter<Scrub::ScrubJob> {
   {
     return fmt::format_to(
 	ctx.out(),
-	"pg[{}] @ {:s} (dl:{:s}) - <{}> / failure: {} / queue state: "
+	"pg[{}] @ {:s} ({:s}) (dl:{:s}) - <{}> / failure: {} / queue state: "
 	"{:.7}",
-	sjob.pgid, sjob.schedule.scheduled_at,
+	sjob.pgid, sjob.schedule.not_before, sjob.schedule.scheduled_at,
 	sjob.schedule.deadline, sjob.registration_state(),
 	sjob.resources_failure, sjob.state.load(std::memory_order_relaxed));
   }

From af8939dd881aecc146dba80e1c129a2c8cc43f94 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 9 Jan 2024 08:15:33 -0600
Subject: [PATCH 1442/2492] osd/scrub: fix set_last_deep_scrub_stamp()

The call should update last_scrub_stamp, too, without
requiring an extra call to on_scrub_schedule_input_change()

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.h                    | 1 +
 src/osd/scrubber/pg_scrubber.cc | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2a823f675969..677f4e93a079 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -281,6 +281,7 @@ class PG : public DoutPrefixProvider,
     recovery_state.update_stats(
       [t](auto &history, auto &stats) {
 	set_last_deep_scrub_stamp(t, history, stats);
+	set_last_scrub_stamp(t, history, stats);
 	return true;
       });
     on_scrub_schedule_input_change();
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 9cd082a87947..55fd7b18dcde 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -715,10 +715,11 @@ void PgScrubber::on_operator_periodic_cmd(
   asok_response_section(f, true, scrub_level, stamp);
 
   if (scrub_level == scrub_level_t::deep) {
+    // this call sets both stamps
     m_pg->set_last_deep_scrub_stamp(stamp);
+  } else {
+    m_pg->set_last_scrub_stamp(stamp);
   }
-  // and in both cases:
-  m_pg->set_last_scrub_stamp(stamp);
 }
 
 // when asked to force a high-priority scrub

From 375d01a6aa54ef5ef4d4f571c85b794edbea403e Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 2 Jan 2024 10:09:06 -0600
Subject: [PATCH 1443/2492] osd/scrub: update job's NB on failure

When a scrub job fails, update its NB to the current time plus a
fixed delay.  This prevents the job from being scheduled again
immediately.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PG.cc                          |  7 ++++
 src/osd/scrubber/osd_scrub.cc          | 10 ++++++
 src/osd/scrubber/osd_scrub.h           | 11 ++++++
 src/osd/scrubber/osd_scrub_sched.cc    | 21 +++++++++--
 src/osd/scrubber/osd_scrub_sched.h     |  6 ++++
 src/osd/scrubber/pg_scrubber.cc        | 14 +++++++-
 src/osd/scrubber/pg_scrubber.h         |  5 +--
 src/osd/scrubber/scrub_job.cc          | 29 +++++++++------
 src/osd/scrubber/scrub_job.h           | 33 +++++++++++++----
 src/osd/scrubber/scrub_machine.cc      |  3 ++
 src/osd/scrubber/scrub_machine_lstnr.h |  7 ++--
 src/osd/scrubber_common.h              | 49 ++++++++++++++++++++++++++
 12 files changed, 170 insertions(+), 25 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index eb7a332f6c2f..cb12dae7c5b1 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1342,9 +1342,12 @@ Scrub::schedule_result_t PG::start_scrubbing(
 	   << dendl;
   ceph_assert(ceph_mutex_is_locked(_lock));
 
+  // recheck PG status (as the PG was unlocked for a time after being selected
+  // for scrubbing)
   if (!is_primary() || !is_active() || !is_clean()) {
     dout(10) << __func__ << ": cannot scrub (not a clean and active primary)"
 	     << dendl;
+    m_scrubber->penalize_next_scrub(Scrub::delay_cause_t::pg_state);
     return schedule_result_t::target_specific_failure;
   }
 
@@ -1361,6 +1364,7 @@ Scrub::schedule_result_t PG::start_scrubbing(
 	     << ": skipping this PG as repairing was not explicitly "
 		"requested for it"
 	     << dendl;
+    m_scrubber->penalize_next_scrub(Scrub::delay_cause_t::scrub_params);
     return schedule_result_t::target_specific_failure;
   }
 
@@ -1369,6 +1373,7 @@ Scrub::schedule_result_t PG::start_scrubbing(
     // (on the transition from NotTrimming to Trimming/WaitReservation),
     // i.e. some time before setting 'snaptrim'.
     dout(10) << __func__ << ": cannot scrub while snap-trimming" << dendl;
+    m_scrubber->penalize_next_scrub(Scrub::delay_cause_t::pg_state);
     return schedule_result_t::target_specific_failure;
   }
 
@@ -1381,6 +1386,7 @@ Scrub::schedule_result_t PG::start_scrubbing(
     // (due to configuration or priority issues)
     // The reason was already reported by the callee.
     dout(10) << __func__ << ": failed to initiate a scrub" << dendl;
+    m_scrubber->penalize_next_scrub(Scrub::delay_cause_t::scrub_params);
     return schedule_result_t::target_specific_failure;
   }
 
@@ -1388,6 +1394,7 @@ Scrub::schedule_result_t PG::start_scrubbing(
   // be retried by the OSD later on.
   if (!m_scrubber->reserve_local()) {
     dout(10) << __func__ << ": failed to reserve locally" << dendl;
+    m_scrubber->penalize_next_scrub(Scrub::delay_cause_t::local_resources);
     return schedule_result_t::osd_wide_failure;
   }
 
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index a721dc1cb9c0..bc65d64bb422 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -435,6 +435,16 @@ void OsdScrub::update_job(
   m_queue.update_job(sjob, suggested, reset_notbefore);
 }
 
+void OsdScrub::delay_on_failure(
+      Scrub::ScrubJobRef sjob,
+      std::chrono::seconds delay,
+      Scrub::delay_cause_t delay_cause,
+      utime_t now_is)
+{
+  m_queue.delay_on_failure(sjob, delay, delay_cause, now_is);
+}
+
+
 void OsdScrub::register_with_osd(
     Scrub::ScrubJobRef sjob,
     const Scrub::sched_params_t& suggested)
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 407eae059079..12e9db312312 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -149,6 +149,17 @@ class OsdScrub {
 
   void clear_reserving_now(spg_t reserving_id);
 
+  /**
+   * push the 'not_before' time out by 'delay' seconds, so that this scrub target
+   * would not be retried before 'delay' seconds have passed.
+   */
+  void delay_on_failure(
+      Scrub::ScrubJobRef sjob,
+      std::chrono::seconds delay,
+      Scrub::delay_cause_t delay_cause,
+      utime_t now_is);
+
+
   /**
    * \returns true if the current time is within the scrub time window
    */
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index 836bf740ec08..36644211ec39 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -153,6 +153,21 @@ void ScrubQueue::update_job(Scrub::ScrubJobRef scrub_job,
   scrub_job->update_schedule(adjusted, reset_nb);
 }
 
+
+void ScrubQueue::delay_on_failure(
+    Scrub::ScrubJobRef sjob,
+    std::chrono::seconds delay,
+    Scrub::delay_cause_t delay_cause,
+    utime_t now_is)
+{
+  dout(10) << fmt::format(
+		  "pg[{}] delay_on_failure: delay:{} now:{:s}",
+		  sjob->pgid, delay, now_is)
+	   << dendl;
+  sjob->delay_on_failure(delay, delay_cause, now_is);
+}
+
+
 sched_params_t ScrubQueue::determine_scrub_time(
   const requested_scrub_t& request_flags,
   const pg_info_t& pg_info,
@@ -281,9 +296,9 @@ ScrubQContainer ScrubQueue::collect_ripe_jobs(
     for (const auto& jobref : group) {
       if (!filtr(jobref)) {
 	dout(20) << fmt::format(
-			" not ripe: {} @ {:s} ({:s})", jobref->pgid,
-                        jobref->schedule.not_before,
-			jobref->schedule.scheduled_at)
+			" not eligible: {} @ {:s} ({:s},{:s})", jobref->pgid,
+			jobref->schedule.not_before,
+			jobref->schedule.scheduled_at, jobref->last_issue)
 		 << dendl;
       }
     }
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index d94ca8cf7579..0eddb22ea7dd 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -220,6 +220,12 @@ class ScrubQueue {
       const sched_params_t& suggested,
       bool reset_notbefore);
 
+  void delay_on_failure(
+      Scrub::ScrubJobRef sjob,
+      std::chrono::seconds delay,
+      Scrub::delay_cause_t delay_cause,
+      utime_t now_is);
+
   sched_params_t determine_scrub_time(const requested_scrub_t& request_flags,
 				      const pg_info_t& pg_info,
 				      const pool_opts_t& pool_conf) const;
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 55fd7b18dcde..3ef5a2ef567d 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -1756,7 +1756,9 @@ void PgScrubber::clear_scrub_blocked()
 
 void PgScrubber::flag_reservations_failure()
 {
-  m_scrub_job->resources_failure = true;
+  dout(10) << __func__ << dendl;
+  // delay the next invocation of the scrubber on this target
+  penalize_next_scrub(Scrub::delay_cause_t::replicas);
 }
 
 /*
@@ -1968,6 +1970,16 @@ void PgScrubber::scrub_finish()
   }
 }
 
+/*
+ * note: arbitrary delay used in this early version of the
+ * scheduler refactoring.
+ */
+void PgScrubber::penalize_next_scrub(Scrub::delay_cause_t cause)
+{
+  m_osds->get_scrub_services().delay_on_failure(
+      m_scrub_job, 5s, cause, ceph_clock_now());
+}
+
 void PgScrubber::on_digest_updates()
 {
   dout(10) << __func__ << " #pending: " << num_digest_updates_pending << " "
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 5911b67263f4..9c29d5fdedb9 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -365,8 +365,7 @@ class PgScrubber : public ScrubPgIF,
   PG* get_pg() const final { return m_pg; }
   PerfCounters& get_counters_set() const final;
 
-  // temporary interface (to be discarded in a follow-up PR)
-  /// set the 'resources_failure' flag in the scrub-job object
+  /// delay next retry of this PG after a replica reservation failure
   void flag_reservations_failure();
 
   scrubber_callback_cancel_token_t schedule_callback_after(
@@ -430,6 +429,8 @@ class PgScrubber : public ScrubPgIF,
 
   void scrub_finish() final;
 
+  void penalize_next_scrub(Scrub::delay_cause_t cause) final;
+
   ScrubMachineListener::MsgAndEpoch prep_replica_map_msg(
     Scrub::PreemptionNoted was_preempted) final;
 
diff --git a/src/osd/scrubber/scrub_job.cc b/src/osd/scrubber/scrub_job.cc
index 1875641e754a..6564b129b0ef 100644
--- a/src/osd/scrubber/scrub_job.cc
+++ b/src/osd/scrubber/scrub_job.cc
@@ -45,24 +45,21 @@ ostream& operator<<(ostream& out, const ScrubJob& sjob)
 
 void ScrubJob::update_schedule(
     const Scrub::scrub_schedule_t& adjusted,
-    bool reset_nb)
+    bool reset_failure_penalty)
 {
-  dout(15)
-      << fmt::format(
-	     "was: nb:{:s}({:s}). Called with: rest?{} nb:{:s} ({:s}) ({})",
-	     schedule.not_before, schedule.scheduled_at, reset_nb,
-	     adjusted.not_before, adjusted.scheduled_at, registration_state())
-      << dendl;
+  dout(15) << fmt::format(
+		  "was: nb:{:s}({:s}). Called with: rest?{} {:s} ({})",
+		  schedule.not_before, schedule.scheduled_at,
+		  reset_failure_penalty, adjusted.scheduled_at,
+		  registration_state())
+	   << dendl;
   schedule.scheduled_at = adjusted.scheduled_at;
   schedule.deadline = adjusted.deadline;
 
-  if (reset_nb || schedule.not_before < schedule.scheduled_at) {
+  if (reset_failure_penalty || (schedule.not_before < schedule.scheduled_at)) {
     schedule.not_before = schedule.scheduled_at;
   }
 
-  // 'updated' is changed here while not holding jobs_lock. That's OK, as
-  // the (atomic) flag will only be cleared by select_pg_and_scrub() after
-  // scan_penalized() is called and the job was moved to the to_scrub queue.
   updated = true;
   dout(10) << fmt::format(
 		  "adjusted: nb:{:s} ({:s}) ({})", schedule.not_before,
@@ -70,6 +67,16 @@ void ScrubJob::update_schedule(
 	   << dendl;
 }
 
+void ScrubJob::delay_on_failure(
+    std::chrono::seconds delay,
+    Scrub::delay_cause_t delay_cause,
+    utime_t scrub_clock_now)
+{
+  schedule.not_before =
+      std::max(scrub_clock_now, schedule.not_before) + utime_t{delay};
+  last_issue = delay_cause;
+}
+
 std::string ScrubJob::scheduling_state(utime_t now_is, bool is_deep_expected)
     const
 {
diff --git a/src/osd/scrubber/scrub_job.h b/src/osd/scrubber/scrub_job.h
index e6765c767b96..98a73477b092 100644
--- a/src/osd/scrubber/scrub_job.h
+++ b/src/osd/scrubber/scrub_job.h
@@ -71,8 +71,8 @@ class ScrubJob final : public RefCountedObject {
    */
   std::atomic_bool in_queues{false};
 
-  /// last scrub attempt failed to secure replica resources
-  bool resources_failure{false};
+  /// how the last attempt to scrub this PG ended
+  delay_cause_t last_issue{delay_cause_t::none};
 
   /**
    * 'updated' is a temporary flag, used to create a barrier after
@@ -118,6 +118,15 @@ class ScrubJob final : public RefCountedObject {
       const scrub_schedule_t& adjusted,
       bool reset_failure_penalty);
 
+  /**
+   * push the 'not_before' time out by 'delay' seconds, so that this scrub target
+   * would not be retried before 'delay' seconds have passed.
+   */
+  void delay_on_failure(
+      std::chrono::seconds delay,
+      delay_cause_t delay_cause,
+      utime_t scrub_clock_now);
+
   void dump(ceph::Formatter* f) const;
 
   /*
@@ -227,6 +236,20 @@ struct formatter<Scrub::qu_state_t> : formatter<std::string_view> {
   }
 };
 
+template <>
+struct formatter<Scrub::sched_params_t> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+  template <typename FormatContext>
+  auto format(const Scrub::sched_params_t& pm, FormatContext& ctx)
+  {
+    return fmt::format_to(
+	ctx.out(), "(proposed:{:s} min/max:{:.3f}/{:.3f} must:{:2s})",
+        utime_t{pm.proposed_time}, pm.min_interval, pm.max_interval,
+        pm.is_must == Scrub::must_scrub_t::mandatory ? "true" : "false");
+  }
+};
+
+
 template <>
 struct formatter<Scrub::ScrubJob> {
   constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
@@ -235,12 +258,10 @@ struct formatter<Scrub::ScrubJob> {
   auto format(const Scrub::ScrubJob& sjob, FormatContext& ctx)
   {
     return fmt::format_to(
-	ctx.out(),
-	"pg[{}] @ {:s} ({:s}) (dl:{:s}) - <{}> / failure: {} / queue state: "
-	"{:.7}",
+	ctx.out(), "pg[{}] @ nb:{:s} ({:s}) (dl:{:s}) - <{}> queue state:{:.7}",
 	sjob.pgid, sjob.schedule.not_before, sjob.schedule.scheduled_at,
 	sjob.schedule.deadline, sjob.registration_state(),
-	sjob.resources_failure, sjob.state.load(std::memory_order_relaxed));
+	sjob.state.load(std::memory_order_relaxed));
   }
 };
 
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index eac26a85f851..d0ab07fe1dfa 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -339,6 +339,9 @@ ActiveScrubbing::~ActiveScrubbing()
   // if the begin-time stamp was not set 'off' (as done if the scrubbing
   // completed successfully), we use it now to set the 'failed scrub' duration.
   if (session.m_session_started_at != ScrubTimePoint{}) {
+    // delay the next invocation of the scrubber on this target
+    scrbr->penalize_next_scrub(Scrub::delay_cause_t::aborted);
+
     auto logged_duration = ScrubClock::now() - session.m_session_started_at;
     session.m_perf_set->tinc(scrbcnt_failed_elapsed, logged_duration);
     session.m_perf_set->inc(scrbcnt_failed);
diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h
index 7849d14e4e41..ea893ba81f01 100644
--- a/src/osd/scrubber/scrub_machine_lstnr.h
+++ b/src/osd/scrubber/scrub_machine_lstnr.h
@@ -162,6 +162,10 @@ struct ScrubMachineListener {
   /// the part that actually finalizes a scrub
   virtual void scrub_finish() = 0;
 
+  /// notify the scrubber about a scrub failure
+  /// (note: temporary implementation)
+  virtual void penalize_next_scrub(Scrub::delay_cause_t cause) = 0;
+
   /**
    * Prepare a MOSDRepScrubMap message carrying the requested scrub map
    * @param was_preempted - were we preempted?
@@ -252,8 +256,7 @@ struct ScrubMachineListener {
   /// sending cluster-log warnings
   virtual void log_cluster_warning(const std::string& msg) const = 0;
 
-  // temporary interface (to be discarded in a follow-up PR)
-  /// set the 'resources_failure' flag in the scrub-job object
+  /// delay next retry of this PG after a replica reservation failure
   virtual void flag_reservations_failure() = 0;
 
   /// is this scrub more than just regular periodic scrub?
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 5323b14677b0..067b9754c110 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -85,6 +85,53 @@ struct formatter<Scrub::OSDRestrictions> {
 };
 }  // namespace fmt
 
+namespace Scrub {
+
+/**
+ * the result of the last attempt to schedule a scrub for a specific PG.
+ * The enum value itself is mostly used for logging purposes.
+ */
+enum class delay_cause_t {
+  none,		    ///< scrub attempt was successful
+  replicas,	    ///< failed to reserve replicas
+  flags,	    ///< noscrub or nodeep-scrub
+  pg_state,	    ///< e.g. snap-trimming
+  restricted_time,  ///< time restrictions or busy CPU
+  local_resources,  ///< too many scrubbing PGs
+  aborted,	    ///< scrub was aborted w/ unspecified reason
+  interval,	    ///< the interval had ended mid-scrub
+  scrub_params,     ///< the specific scrub type is not allowed
+};
+}  // namespace Scrub
+
+namespace fmt {
+// clang-format off
+template <>
+struct formatter<Scrub::delay_cause_t> : ::fmt::formatter<std::string_view> {
+  template <typename FormatContext>
+  auto format(Scrub::delay_cause_t cause, FormatContext& ctx)
+  {
+    using enum Scrub::delay_cause_t;
+    std::string_view desc;
+    switch (cause) {
+      case none:                desc = "ok"; break;
+      case replicas:            desc = "replicas"; break;
+      case flags:               desc = "noscrub"; break;
+      case pg_state:            desc = "pg-state"; break;
+      case restricted_time:     desc = "time/load"; break;
+      case local_resources:     desc = "local-cnt"; break;
+      case aborted:             desc = "aborted"; break;
+      case interval:            desc = "interval"; break;
+      case scrub_params:        desc = "scrub-mode"; break;
+      // better to not have a default case, so that the compiler will warn
+    }
+    return ::fmt::formatter<string_view>::format(desc, ctx);
+  }
+};
+// clang-format on
+}  // namespace fmt
+
+
 namespace Scrub {
 
 /// PG services used by the scrubber backend
@@ -330,6 +377,8 @@ struct ScrubPgIF {
 
   virtual pg_scrubbing_status_t get_schedule() const = 0;
 
+  /// notify the scrubber about a scrub failure
+  virtual void penalize_next_scrub(Scrub::delay_cause_t cause) = 0;
 
   // // perform 'scrub'/'deep_scrub' asok commands
 

From 657f59ccedac21b3a08e8cd4380f6e8ef42ef818 Mon Sep 17 00:00:00 2001
From: Athos Ribeiro <athos.ribeiro@canonical.com>
Date: Mon, 22 Jan 2024 15:06:01 -0300
Subject: [PATCH 1444/2492] cmake: quote variable to allow empty values

Use quotes to allow empty submodules variable during the cmake build
configuration. This is specially useful for lean build environments
where git is not available (which would result in an empty submodules
variable).

Further context is available at
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1056793#10.

Signed-off-by: Athos Ribeiro <athos.ribeiro@canonical.com>
---
 cmake/modules/CTags.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/CTags.cmake b/cmake/modules/CTags.cmake
index c3e1b3799b02..ee1912f3c5a7 100644
--- a/cmake/modules/CTags.cmake
+++ b/cmake/modules/CTags.cmake
@@ -16,10 +16,10 @@ function(add_tags name)
       OUTPUT_VARIABLE submodules
       OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(${result_code} EQUAL 0)
-      string(REPLACE "${TAGS_SRC_DIR}/" "" submodules ${submodules})
+      string(REPLACE "${TAGS_SRC_DIR}/" "" submodules "${submodules}")
       # cmake list uses ";" as the delimiter, so split the string manually
       # before iterating in it.
-      string(REPLACE "\n" ";" submodules ${submodules})
+      string(REPLACE "\n" ";" submodules "${submodules}")
       list(APPEND excludes ${submodules})
     endif()
   endif()

From 6383093e2731c2aec87b5ab286f48dc161edea37 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Fri, 19 Jan 2024 16:31:50 -0500
Subject: [PATCH 1445/2492] rgw/multisite: avoid writing multipart parts to the
 bucket index log

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/cls/rgw/cls_rgw.cc                | 15 ++-------------
 src/rgw/driver/rados/rgw_sal_rados.cc |  4 ++--
 src/rgw/rgw_op.cc                     |  8 ++++++--
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 034660a11ed1..d4350e667518 100644
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -963,7 +963,7 @@ static int read_key_entry(cls_method_context_t hctx, const cls_rgw_obj_key& key,
 // called by rgw_bucket_complete_op() for each item in op.remove_objs
 static int complete_remove_obj(cls_method_context_t hctx,
                                rgw_bucket_dir_header& header,
-                               const cls_rgw_obj_key& key, bool log_op)
+                               const cls_rgw_obj_key& key)
 {
   rgw_bucket_dir_entry entry;
   string idx;
@@ -978,17 +978,6 @@ static int complete_remove_obj(cls_method_context_t hctx,
           int(entry.meta.category));
   unaccount_entry(header, entry);
 
-  if (log_op) {
-    ++header.ver; // increment index version, or we'll overwrite keys previously written
-    const std::string tag;
-    ret = log_index_operation(hctx, key, CLS_RGW_OP_DEL, tag, entry.meta.mtime,
-                              entry.ver, CLS_RGW_STATE_COMPLETE, header.ver,
-                              header.max_marker, 0, nullptr, nullptr, nullptr);
-    if (ret < 0) {
-      return ret;
-    }
-  }
-
   ret = cls_cxx_map_remove_key(hctx, idx);
   if (ret < 0) {
     CLS_LOG(1, "%s: cls_cxx_map_remove_key failed with %d", __func__, ret);
@@ -1210,7 +1199,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist
     CLS_LOG_BITX(bitx_inst, 20,
 		 "INFO: %s: completing object remove key=%s",
 		 __func__, escape_str(remove_key.to_string()).c_str());
-    rc = complete_remove_obj(hctx, header, remove_key, default_log_op);
+    rc = complete_remove_obj(hctx, header, remove_key);
     if (rc < 0) {
       CLS_LOG_BITX(bitx_inst, 1,
 		   "WARNING: %s: complete_remove_obj, failed to remove entry, "
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 709e748cb499..64c3c1a9e5e3 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2289,7 +2289,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     encode(upload_info, bl);
     obj_op.meta.data = &bl;
 
-    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx);
+    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx, false);
   } while (ret == -EEXIST);
 
   return ret;
@@ -2904,7 +2904,7 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta
                        uint32_t flags)
 {
   return processor.complete(accounted_size, etag, mtime, set_mtime, attrs, delete_at,
-			    if_match, if_nomatch, user_data, zones_trace, canceled, rctx, flags);
+                            if_match, if_nomatch, user_data, zones_trace, canceled, rctx, flags);
 }
 
 bool RadosZoneGroup::placement_target_exists(std::string& target) const
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 8f1ae6f4068e..f6011dbe5c9d 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4525,12 +4525,16 @@ void RGWPutObj::execute(optional_yield y)
     emplace_attr(RGW_ATTR_OBJECT_RETENTION, std::move(obj_retention_bl));
   }
 
+  // don't track the individual parts of multipart uploads. they replicate in
+  // full after CompleteMultipart
+  const uint32_t complete_flags = multipart ? 0 : rgw::sal::FLAG_LOG_OP;
+
   tracepoint(rgw_op, processor_complete_enter, s->req_id.c_str());
   const req_context rctx{this, s->yield, s->trace.get()};
   op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs,
                                (delete_at ? *delete_at : real_time()), if_match, if_nomatch,
                                (user_data.empty() ? nullptr : &user_data), nullptr, nullptr,
-                               rctx, rgw::sal::FLAG_LOG_OP);
+                               rctx, complete_flags);
   tracepoint(rgw_op, processor_complete_exit, s->req_id.c_str());
   if (op_ret < 0) {
     return;
@@ -6798,7 +6802,7 @@ void RGWCompleteMultipart::complete()
   // when the bucket is, as that would add an unneeded delete marker
   // moved to complete to prevent segmentation fault in publish commit
   if (meta_obj.get() != nullptr) {
-    int ret = meta_obj->delete_object(this, null_yield, rgw::sal::FLAG_PREVENT_VERSIONING | rgw::sal::FLAG_LOG_OP);
+    int ret = meta_obj->delete_object(this, null_yield, rgw::sal::FLAG_PREVENT_VERSIONING);
     if (ret >= 0) {
       /* serializer's exclusive lock is released */
       serializer->clear_locked();

From 9b36e312ddac8f5dc6d881a10841571894c62f84 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Wed, 10 Jan 2024 00:40:35 -0500
Subject: [PATCH 1446/2492] rgw/multisite: add multisite test cases with some
 rgw instances down

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/test/rgw/rgw_multi/tests.py | 191 ++++++++++++++++++++++++++++++++
 1 file changed, 191 insertions(+)

diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 3f12d25f75a2..3f5ae0fea5d3 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -2885,3 +2885,194 @@ def test_sync_single_bucket_to_multiple():
     remove_sync_policy_group(c1, "sync-bucket", bucketA.name)
     remove_sync_policy_group(c1, "sync-group")
     return
+
+def stop_2nd_rgw(zonegroup):
+    rgw_down = False
+    for z in zonegroup.zones:
+        if len(z.gateways) <= 1:
+            continue
+        z.gateways[1].stop()
+        log.info('gateway stopped zone=%s gateway=%s', z.name, z.gateways[1].endpoint())
+        rgw_down = True
+    return rgw_down
+
+def start_2nd_rgw(zonegroup):
+    for z in zonegroup.zones:
+        if len(z.gateways) <= 1:
+            continue
+        z.gateways[1].start()
+        log.info('gateway started zone=%s gateway=%s', z.name, z.gateways[1].endpoint())
+
+@attr('rgw_down')
+def test_bucket_create_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_create_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        zonegroup_conns = ZonegroupConns(zonegroup)
+        buckets, _ = create_bucket_per_zone(zonegroup_conns, 2)
+        zonegroup_meta_checkpoint(zonegroup)
+
+        for zone in zonegroup_conns.zones:
+            assert check_all_buckets_exist(zone, buckets)
+
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_bucket_remove_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_remove_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        zonegroup_conns = ZonegroupConns(zonegroup)
+        buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, 2)
+        zonegroup_meta_checkpoint(zonegroup)
+
+        for zone in zonegroup_conns.zones:
+            assert check_all_buckets_exist(zone, buckets)
+
+        for zone, bucket_name in zone_bucket:
+            zone.conn.delete_bucket(bucket_name)
+
+        zonegroup_meta_checkpoint(zonegroup)
+
+        for zone in zonegroup_conns.zones:
+            assert check_all_buckets_dont_exist(zone, buckets)
+
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_object_sync_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_object_sync_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_object_sync()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_object_delete_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_object_delete_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_object_delete()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_concurrent_versioned_object_incremental_sync_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_concurrent_versioned_object_incremental_sync_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_concurrent_versioned_object_incremental_sync()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_suspended_delete_marker_full_sync_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_suspended_delete_marker_full_sync_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_suspended_delete_marker_full_sync()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_bucket_acl_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_acl_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_bucket_acl()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_bucket_sync_enable_right_after_disable_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_sync_enable_right_after_disable_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_bucket_sync_enable_right_after_disable()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_multipart_object_sync_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_multipart_object_sync_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_multipart_object_sync()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_bucket_sync_run_basic_incremental_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_sync_run_basic_incremental_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_bucket_sync_run_basic_incremental()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_role_sync_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_role_sync_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_role_sync()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_bucket_full_sync_after_data_sync_init_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_bucket_full_sync_after_data_sync_init_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_bucket_full_sync_after_data_sync_init()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_sync_policy_config_zonegroup_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_sync_policy_config_zonegroup_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_sync_policy_config_zonegroup()
+    finally:
+        start_2nd_rgw(zonegroup)
+
+@attr('rgw_down')
+def test_sync_flow_symmetrical_zonegroup_all_rgw_down():
+    zonegroup = realm.master_zonegroup()
+    try:
+        if not stop_2nd_rgw(zonegroup):
+            raise SkipTest("test_sync_flow_symmetrical_zonegroup_all_rgw_down skipped. More than one rgw needed in any one or multiple zone(s).")
+
+        test_sync_flow_symmetrical_zonegroup_all()
+    finally:
+        start_2nd_rgw(zonegroup)

From c67a5e5d4bad17e7ae799dd62a66d1e23ec18942 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 23 Jan 2024 12:13:10 +1000
Subject: [PATCH 1447/2492] doc/radosgw: edit "read/write global rate limit"
 admin.rst

Edit "Reading/Writing Global Rate Limit Configuration" in
doc/radosgw/admin.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 49 ++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 19 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index b011fa265109..8e846874a5f8 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -783,35 +783,46 @@ An enabled bucket rate limit can be disabled by running a command of the followi
 
    radosgw-admin ratelimit disable --ratelimit-scope=bucket --uid=mybucket
 
-Reading / Writing Global Rate Limit Configuration
--------------------------------------------------
+Reading and Writing Global Rate Limit Configuration
+---------------------------------------------------
 
-You can read and write global rate limit settings in the period configuration. To
-view the global rate limit settings::
+You can read and write global rate limit settings in the period's configuration.
+To view the global rate limit settings, run the following command:
 
-	radosgw-admin global ratelimit get
+.. prompt:: bash
+
+   radosgw-admin global ratelimit get
 
 The global rate limit settings can be manipulated with the ``global ratelimit``
-counterparts of the ``ratelimit set``, ``ratelimit enable``, and ``ratelimit disable``
-commands. Per user and per bucket ratelimit configuration is overriding the global configuration::
+counterparts of the ``ratelimit set``, ``ratelimit enable``, and ``ratelimit
+disable`` commands. Per-user and per-bucket ratelimit configurations override
+the global configuration:
 
-	radosgw-admin global ratelimit set --ratelimit-scope bucket --max-read-ops=1024
-	radosgw-admin global ratelimit enable --ratelimit-scope bucket
+.. prompt:: bash
 
-The global rate limit can configure rate limit scope for all authenticated users::
+   radosgw-admin global ratelimit set --ratelimit-scope bucket --max-read-ops=1024
+   radosgw-admin global ratelimit enable --ratelimit-scope bucket
 
-  radosgw-admin global ratelimit set --ratelimit-scope user --max-read-ops=1024
-  radosgw-admin global ratelimit enable --ratelimit-scope user
+The global rate limit can be used to configure the scope of the rate limit for
+all authenticated users:
+
+.. prompt:: bash
 
-The global rate limit can configure rate limit scope for all unauthenticated users::
+   radosgw-admin global ratelimit set --ratelimit-scope user --max-read-ops=1024
+   radosgw-admin global ratelimit enable --ratelimit-scope user
+
+The global rate limit can be used to configure the scope of the rate limit for
+all unauthenticated users:
+
+.. prompt:: bash
   
-  radosgw-admin global ratelimit set --ratelimit-scope=anonymous --max-read-ops=1024
-  radosgw-admin global ratelimit enable --ratelimit-scope=anonymous
+   radosgw-admin global ratelimit set --ratelimit-scope=anonymous --max-read-ops=1024
+   radosgw-admin global ratelimit enable --ratelimit-scope=anonymous
 
-.. note:: In a multisite configuration, where there is a realm and period
-   present, changes to the global rate limit must be committed using ``period
-   update --commit``. If there is no period present, the rados gateway(s) must
-   be restarted for the changes to take effect.
+.. note:: In a multisite configuration where a realm and a period are present,
+   any changes to the global rate limit must be committed using ``period update
+   --commit``. If no period is present, the rados gateway(s) must be restarted
+   for the changes to take effect.
 
 Usage
 =====

From 7559a6c0d11b5c8f2443e208c8188a9201b82a4d Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 17 Jan 2024 09:36:16 -0600
Subject: [PATCH 1448/2492] osd/scrub: check reservation replies for relevance

Compare a token (nonce) carried in the reservation reply with the remembered
token of the reservation request.  If they don't match, the reply is
stale and should be ignored (and logged).

Fixes: https://tracker.ceph.com/issues/64052

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/messages/MOSDScrubReserve.h        |  21 ++++-
 src/osd/scrubber/scrub_machine.cc      |  38 +++++---
 src/osd/scrubber/scrub_machine.h       |  16 ++++
 src/osd/scrubber/scrub_reservations.cc | 117 ++++++++++++++++++-------
 src/osd/scrubber/scrub_reservations.h  |  60 +++++++++++--
 5 files changed, 200 insertions(+), 52 deletions(-)

diff --git a/src/messages/MOSDScrubReserve.h b/src/messages/MOSDScrubReserve.h
index c7ab98541175..97b6dff4dd00 100644
--- a/src/messages/MOSDScrubReserve.h
+++ b/src/messages/MOSDScrubReserve.h
@@ -19,9 +19,11 @@
 
 class MOSDScrubReserve : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 1;
+  static constexpr int HEAD_VERSION = 2;
   static constexpr int COMPAT_VERSION = 1;
 public:
+  using reservation_nonce_t = uint32_t;
+
   spg_t pgid;
   epoch_t map_epoch;
   enum ReserveMsgOp {
@@ -32,6 +34,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   };
   int32_t type;
   pg_shard_t from;
+  reservation_nonce_t reservation_nonce{0};
 
   epoch_t get_map_epoch() const override {
     return map_epoch;
@@ -46,10 +49,11 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   MOSDScrubReserve(spg_t pgid,
 		   epoch_t map_epoch,
 		   int type,
-		   pg_shard_t from)
+		   pg_shard_t from,
+		   reservation_nonce_t nonce)
     : MOSDFastDispatchOp{MSG_OSD_SCRUB_RESERVE, HEAD_VERSION, COMPAT_VERSION},
       pgid(pgid), map_epoch(map_epoch),
-      type(type), from(from) {}
+      type(type), from(from), reservation_nonce{nonce} {}
 
   std::string_view get_type_name() const {
     return "MOSDScrubReserve";
@@ -71,7 +75,8 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
       out << "RELEASE ";
       break;
     }
-    out << "e" << map_epoch << ")";
+    out << "e" << map_epoch << " from: " << from
+	<< " reservation_nonce: " << reservation_nonce << ")";
     return;
   }
 
@@ -82,6 +87,13 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     decode(map_epoch, p);
     decode(type, p);
     decode(from, p);
+    if (header.version >= 2) {
+      decode(reservation_nonce, p);
+    } else {
+      // a zero nonce (identifying legacy senders) is ignored when
+      // checking the request for obsolescence
+      reservation_nonce = 0;
+    }
   }
 
   void encode_payload(uint64_t features) {
@@ -90,6 +102,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     encode(map_epoch, payload);
     encode(type, payload);
     encode(from, payload);
+    encode(reservation_nonce, payload);
   }
 private:
   template<class T, typename... Args>
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index d0ab07fe1dfa..d9d03fe68896 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -230,7 +230,9 @@ ReservingReplicas::ReservingReplicas(my_context ctx)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
 
   // initiate the reservation process
-  session.m_reservations.emplace(*scrbr, *session.m_perf_set);
+  session.m_reservations.emplace(
+      *scrbr, context<PrimaryActive>().last_request_sent_nonce,
+      *session.m_perf_set);
 
   if (session.m_reservations->get_last_sent()) {
     // the 1'st reservation request was sent
@@ -263,9 +265,9 @@ sc::result ReservingReplicas::react(const ReplicaGrant& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReservingReplicas::react(const ReplicaGrant&)" << dendl;
+  const auto& m = ev.m_op->get_req<MOSDScrubReserve>();
 
-  if (context<Session>().m_reservations->handle_reserve_grant(
-	  ev.m_op, ev.m_from)) {
+  if (context<Session>().m_reservations->handle_reserve_grant(*m, ev.m_from)) {
     // we are done with the reservation process
     return transit<ActiveScrubbing>();
   }
@@ -277,13 +279,21 @@ sc::result ReservingReplicas::react(const ReplicaReject& ev)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   auto& session = context<Session>();
   dout(10) << "ReservingReplicas::react(const ReplicaReject&)" << dendl;
-  session.m_reservations->log_failure_and_duration(scrbcnt_resrv_rejected);
-
-  // manipulate the 'next to reserve' iterator to exclude
-  // the rejecting replica from the set of replicas requiring release
-  session.m_reservations->verify_rejections_source(ev.m_op, ev.m_from);
+  const auto m = ev.m_op->get_req<MOSDScrubReserve>();
+
+  // Verify that the message is from the replica we were expecting a reply from,
+  // and that the message is not stale. If all is well - this is a real rejection:
+  // - log required details;
+  // - manipulate the 'next to reserve' iterator to exclude
+  //   the rejecting replica from the set of replicas requiring release
+  if (!session.m_reservations->handle_reserve_rejection(*m, ev.m_from)) {
+    // stale or unexpected
+    return discard_event();
+  }
 
-  // set 'reservation failure' as the scrub termination cause (affecting
+  // The rejection was carrying the correct reservation_nonce. It was
+  // logged by handle_reserve_rejection().
+  // Set 'reservation failure' as the scrub termination cause (affecting
   // the rescheduling of this PG)
   scrbr->flag_reservations_failure();
 
@@ -777,7 +787,13 @@ ReplicaActive::~ReplicaActive()
 void ReplicaActive::on_reserve_req(const ReplicaReserveReq& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "ReplicaActive::on_reserve_req()" << dendl;
+  const auto m = ev.m_op->get_req<MOSDScrubReserve>();
+  const auto msg_nonce = m->reservation_nonce;
+  dout(10)
+      << fmt::format(
+	     "ReplicaActive::on_reserve_req() from {} (reservation_nonce:{})",
+	     ev.m_from, msg_nonce)
+      << dendl;
 
   if (reserved_by_my_primary) {
     dout(10) << "ReplicaActive::on_reserve_req(): already reserved" << dendl;
@@ -797,7 +813,7 @@ void ReplicaActive::on_reserve_req(const ReplicaReserveReq& ev)
 
   Message* reply = new MOSDScrubReserve(
       spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch, ret.op,
-      m_pg->pg_whoami);
+      m_pg->pg_whoami, msg_nonce);
   m_osds->send_message_osd_cluster(reply, ev.m_op->get_req()->get_connection());
 }
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 8fa96da14052..beb4d7a4c0fa 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -440,6 +440,22 @@ struct PrimaryActive : sc::state<PrimaryActive, ScrubMachine, PrimaryIdle>,
   using reactions = mpl::list<
       // when the interval ends - we may not be a primary anymore
       sc::transition<IntervalChanged, NotActive>>;
+
+ /**
+  * Identifies a specific reservation request.
+  * The primary is permitted to cancel outstanding reservation requests without
+  * waiting for the pending response from the replica.  Thus, we may, in general,
+  * see responses from prior reservation attempts that we need to ignore.  Each
+  * reservation request is therefore associated with a nonce incremented within
+  * an interval with each reservation request.  Any response with a non-matching
+  * nonce must be from a reservation request we canceled.  Note that this check
+  * occurs after validating that the message is from the current interval, so
+  * reusing nonces between intervals is safe.
+  *
+  * 0 is a special value used to indicate that the sender did not include a nonce due
+  * to not being a sufficiently recent version.
+  */
+  reservation_nonce_t last_request_sent_nonce{1};
 };
 
 /**
diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 3faafe9cb0a1..284d90290578 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -6,7 +6,6 @@
 #include <span>
 
 #include "common/ceph_time.h"
-#include "messages/MOSDScrubReserve.h"
 #include "osd/OSD.h"
 #include "osd/PG.h"
 #include "osd/osd_types_fmt.h"
@@ -31,11 +30,13 @@ namespace Scrub {
 
 ReplicaReservations::ReplicaReservations(
     ScrubMachineListener& scrbr,
+    reservation_nonce_t& nonce,
     PerfCounters& pc)
     : m_scrubber{scrbr}
     , m_pg{m_scrubber.get_pg()}
     , m_pgid{m_scrubber.get_spgid().pgid}
     , m_osds{m_pg->get_pg_osd(ScrubberPasskey())}
+    , m_last_request_sent_nonce{nonce}
     , m_perf_set{pc}
 {
   // the acting set is sorted by pg_shard_t. The reservations are to be issued
@@ -80,7 +81,7 @@ void ReplicaReservations::release_all()
   for (const auto& peer : replicas) {
     auto m = make_message<MOSDScrubReserve>(
 	spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::RELEASE,
-	m_pg->pg_whoami);
+	m_pg->pg_whoami, 0);
     m_pg->send_cluster_message(peer.osd, m, epoch, false);
   }
 
@@ -125,16 +126,50 @@ ReplicaReservations::~ReplicaReservations()
   log_failure_and_duration(scrbcnt_resrv_aborted);
 }
 
-bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
+bool ReplicaReservations::is_reservation_response_relevant(
+    reservation_nonce_t msg_nonce) const
 {
-  // verify that the grant is from the peer we expected. If not?
-  // for now - abort the OSD. \todo reconsider the reaction.
-  if (!get_last_sent().has_value() || from != *get_last_sent()) {
+  return (msg_nonce == 0) || (msg_nonce == m_last_request_sent_nonce);
+}
+
+bool ReplicaReservations::is_msg_source_correct(pg_shard_t from) const
+{
+  const auto exp_source = get_last_sent();
+  return exp_source && from == *exp_source;
+}
+
+bool ReplicaReservations::handle_reserve_grant(
+    const MOSDScrubReserve& msg,
+    pg_shard_t from)
+{
+  if (!is_reservation_response_relevant(msg.reservation_nonce)) {
+    // this is a stale response to a previous request (e.g. one that
+    // timed-out). See m_last_request_sent_nonce for details.
     dout(1) << fmt::format(
-		   "unexpected grant from {} (expected {})", from,
-		   get_last_sent().value_or(pg_shard_t{}))
+		   "stale reservation response from {} with nonce {} vs. "
+		   "expected {} (e:{})",
+		   from, msg.reservation_nonce, m_last_request_sent_nonce,
+		   msg.map_epoch)
 	    << dendl;
-    ceph_assert(from == get_last_sent());
+    return false;
+  }
+
+  // verify that the grant is from the peer we expected. If not?
+  // for now - abort the OSD. There is no known scenario in which a
+  // grant message with a correct nonce can arrive from the wrong peer.
+  // (we would not abort for arriving messages with nonce 0, as those
+  // are legacy messages, for which the nonce was not verified).
+  if (!is_msg_source_correct(from)) {
+    const auto error_text = fmt::format(
+	"unexpected reservation grant from {} vs. the expected {} (e:{} "
+	"message nonce:{})",
+	from, get_last_sent().value_or(pg_shard_t{}), msg.map_epoch,
+	msg.reservation_nonce);
+    dout(1) << error_text << dendl;
+    if (msg.reservation_nonce != 0) {
+      m_osds->clog->error() << error_text;
+      ceph_abort_msg(error_text);
+    }
     return false;
   }
 
@@ -143,15 +178,15 @@ bool ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from)
   // log a warning if the response was slow to arrive
   if ((m_slow_response_warn_timeout > 0ms) &&
       (elapsed > m_slow_response_warn_timeout)) {
-    dout(1) << fmt::format(
-		   "slow reservation response from {} ({}ms)", from,
-		   duration_cast<milliseconds>(elapsed).count())
-	    << dendl;
+    m_osds->clog->warn() << fmt::format(
+	"slow reservation response from {} ({}ms)", from,
+	duration_cast<milliseconds>(elapsed).count());
     // prevent additional warnings
     m_slow_response_warn_timeout = 0ms;
   }
   dout(10) << fmt::format(
-		  "granted by {} ({} of {}) in {}ms", from,
+		  "(e:{} nonce:{}) granted by {} ({} of {}) in {}ms",
+		  msg.map_epoch, msg.reservation_nonce, from,
 		  active_requests_cnt(), m_sorted_secondaries.size(),
 		  duration_cast<milliseconds>(elapsed).count())
 	   << dendl;
@@ -170,44 +205,64 @@ bool ReplicaReservations::send_next_reservation_or_complete()
   // send the next reservation request
   const auto peer = *m_next_to_request;
   const auto epoch = m_pg->get_osdmap_epoch();
+  m_last_request_sent_nonce++;
+
   auto m = make_message<MOSDScrubReserve>(
-      spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST,
-      m_pg->pg_whoami);
+      spg_t{m_pgid, peer.shard}, epoch, MOSDScrubReserve::REQUEST, m_pg->pg_whoami,
+      m_last_request_sent_nonce);
   m_pg->send_cluster_message(peer.osd, m, epoch, false);
   m_last_request_sent_at = ScrubClock::now();
   dout(10) << fmt::format(
-		  "reserving {} (the {} of {} replicas)", *m_next_to_request,
-		  active_requests_cnt() + 1, m_sorted_secondaries.size())
+		  "reserving {} (the {} of {} replicas) e:{} nonce:{}",
+		  *m_next_to_request, active_requests_cnt() + 1,
+		  m_sorted_secondaries.size(), epoch, m_last_request_sent_nonce)
 	   << dendl;
   m_next_to_request++;
   return false;
 }
 
-void ReplicaReservations::verify_rejections_source(
-    OpRequestRef op,
+bool ReplicaReservations::handle_reserve_rejection(
+    const MOSDScrubReserve& msg,
     pg_shard_t from)
 {
   // a convenient log message for the reservation process conclusion
   // (matches the one in send_next_reservation_or_complete())
   dout(10) << fmt::format(
-		  "remote reservation failure. Rejected by {} ({})", from,
-		  *op->get_req())
+		  "remote reservation failure. Rejected by {} ({})", from, msg)
 	   << dendl;
 
+  if (!is_reservation_response_relevant(msg.reservation_nonce)) {
+    // this is a stale response to a previous request (e.g. one that
+    // timed-out). See m_last_request_sent_nonce for details.
+    dout(10) << fmt::format(
+		    "stale reservation response from {} with reservation_nonce "
+		    "{} vs. expected {} (e:{})",
+		    from, msg.reservation_nonce, m_last_request_sent_nonce,
+		    msg.map_epoch)
+	     << dendl;
+    return false;
+  }
+
+  log_failure_and_duration(scrbcnt_resrv_rejected);
+
+  // we should never see a rejection carrying a valid
+  // reservation nonce - arriving while we have no pending requests
+  ceph_assert(get_last_sent().has_value() || msg.reservation_nonce == 0);
+
   // verify that the denial is from the peer we expected. If not?
+  // There is no known scenario in which this can happen, but if it does -
   // we should treat it as though the *correct* peer has rejected the request,
   // but remember to release that peer, too.
-
-  ceph_assert(get_last_sent().has_value());
-  const auto expected = *get_last_sent();
-  if (from != expected) {
-    dout(1) << fmt::format(
-		   "unexpected rejection from {} (expected {})", from, expected)
-	    << dendl;
-  } else {
-    // correct peer, wrong answer...
+  if (is_msg_source_correct(from)) {
     m_next_to_request--;  // no need to release this one
+  } else {
+    m_osds->clog->warn() << fmt::format(
+	"unexpected reservation denial from {} vs the expected {} (e:{} "
+	"message reservation_nonce:{})",
+	from, get_last_sent().value_or(pg_shard_t{}), msg.map_epoch,
+	msg.reservation_nonce);
   }
+  return true;
 }
 
 std::optional<pg_shard_t> ReplicaReservations::get_last_sent() const
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index 9d59033bac87..2e13760ffa1f 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -8,6 +8,7 @@
 #include <string_view>
 #include <vector>
 
+#include "messages/MOSDScrubReserve.h"
 #include "osd/scrubber_common.h"
 
 #include "osd_scrub_sched.h"
@@ -15,6 +16,8 @@
 
 namespace Scrub {
 
+using reservation_nonce_t = MOSDScrubReserve::reservation_nonce_t;
+
 /**
  * Reserving/freeing scrub resources at the replicas.
  *
@@ -44,6 +47,21 @@ namespace Scrub {
  *  that have been acquired until that moment.
  *  (Why? because we have encountered instances where a reservation request was
  *  lost - either due to a bug or due to a network issue.)
+ *
+ * Keeping primary & replica in sync:
+ *
+ * Reservation requests may be canceled by the primary independently of the
+ * replica's response. Depending on timing, a cancellation by the primary might
+ * or might not be processed by a replica prior to sending a response (either
+ * rejection or success).  Thus, we associate each reservation request with a
+ * nonce incremented with each reservation during an interval and drop any
+ * responses that do not match our current nonce.
+ * This check occurs after rejecting any messages from prior intervals, so
+ * reusing nonces between intervals is not a problem.  Note that epoch would
+ * not suffice as it is possible for this sequence to occur several times
+ * without a new map epoch.
+ * Note - 'release' messages, which are not replied to by the replica,
+ * do not need or use that field.
  */
 class ReplicaReservations {
   ScrubMachineListener& m_scrubber;
@@ -64,6 +82,14 @@ class ReplicaReservations {
   /// for logs, and for detecting slow peers
   ScrubTimePoint m_last_request_sent_at;
 
+  /**
+   * A ref to PrimaryActive::last_request_sent_nonce.
+   * Identifies a specific request sent, to verify against grant/deny
+   * responses.
+   * See PrimaryActive::last_request_sent_nonce for details.
+   */
+  reservation_nonce_t& m_last_request_sent_nonce;
+
   /// the 'slow response' timeout (in milliseconds) - as configured.
   /// Doubles as a 'do once' flag for the warning.
   std::chrono::milliseconds m_slow_response_warn_timeout;
@@ -77,7 +103,10 @@ class ReplicaReservations {
   std::optional<ScrubTimePoint> m_process_started_at;
 
  public:
-  ReplicaReservations(ScrubMachineListener& scrubber, PerfCounters& pc);
+  ReplicaReservations(
+      ScrubMachineListener& scrubber,
+      reservation_nonce_t& nonce,
+      PerfCounters& pc);
 
   ~ReplicaReservations();
 
@@ -90,18 +119,23 @@ class ReplicaReservations {
    * \returns true if there are no more replicas to send reservation requests
    * (i.e., the scrubber should proceed to the next phase), false otherwise.
    */
-  bool handle_reserve_grant(OpRequestRef op, pg_shard_t from);
+  bool handle_reserve_grant(const MOSDScrubReserve& msg, pg_shard_t from);
 
   /**
+   * React to an incoming reservation rejection.
+   *
    * Verify that the sender of the received rejection is the replica we
-   * were expecting a reply from.
-   * If this is so - just mark the fact that the specific peer need not
-   * be released.
+   * were expecting a reply from, and that the message isn't stale (see
+   * m_last_request_sent_nonce for details).
+   * If a valid rejection: log it, and mark the fact that the specific peer
+   * need not be released.
    *
    * Note - the actual handling of scrub session termination and of
    * releasing the reserved replicas is done by the caller (the FSM).
+   *
+   * Returns true if the rejection is valid, false otherwise.
    */
-  void verify_rejections_source(OpRequestRef op, pg_shard_t from);
+  bool handle_reserve_rejection(const MOSDScrubReserve& msg, pg_shard_t from);
 
   /**
    * Notifies implementation that it is no longer responsible for releasing
@@ -140,6 +174,20 @@ class ReplicaReservations {
    */
   bool send_next_reservation_or_complete();
 
+  /**
+   * is this is a reply to our last request?
+   * Checks response once against m_last_request_sent_nonce. See
+   * m_last_request_sent_nonce for details.
+   */
+  bool is_reservation_response_relevant(reservation_nonce_t msg_nonce) const;
+
+  /**
+   * is this reply coming from the expected replica?
+   * Now that we check the nonce before checking the sender - this
+   * check should never fail.
+   */
+  bool is_msg_source_correct(pg_shard_t from) const;
+
   // ---   perf counters helpers
 
   /**

From aeb8afe1d5a958cfd9343bbe941df2e1451019f4 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Tue, 23 Jan 2024 08:04:32 +0530
Subject: [PATCH 1449/2492] mgr/dashboard: Fix inconsistency in capitalisation
 of "Multi-site"

fixes https://tracker.ceph.com/issues/64125

Across the dashboard, two instances are present: Multi-site and
Multi-Site.
Making it consistent all over by using Multi-site.

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../rgw-multisite-export/rgw-multisite-export.component.html    | 2 +-
 .../rgw-multisite-import/rgw-multisite-import.component.html    | 2 +-
 .../rgw-multisite-migrate/rgw-multisite-migrate.component.html  | 2 +-
 .../frontend/src/app/shared/constants/app.constants.ts          | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-export/rgw-multisite-export.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-export/rgw-multisite-export.component.html
index b399f934aa64..779884936938 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-export/rgw-multisite-export.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-export/rgw-multisite-export.component.html
@@ -1,6 +1,6 @@
 <cd-modal [modalRef]="activeModal">
   <ng-container i18n="form title"
-                class="modal-title">Export Multi-Site Realm Token</ng-container>
+                class="modal-title">Export Multi-site Realm Token</ng-container>
 
   <ng-container class="modal-content">
     <form name="exportTokenForm"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
index 70c07e8acadc..ee5633ec5fe1 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-import/rgw-multisite-import.component.html
@@ -1,6 +1,6 @@
 <cd-modal [modalRef]="activeModal">
   <ng-container i18n="form title"
-                class="modal-title">Import Multi-Site Token</ng-container>
+                class="modal-title">Import Multi-site Token</ng-container>
 
   <ng-container class="modal-content">
     <form name="importTokenForm"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
index f3f23feec7a5..b18c5a0b9d76 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
@@ -1,6 +1,6 @@
 <cd-modal [modalRef]="activeModal">
   <ng-container i18n="form title"
-                class="modal-title">Migrate Single Site to Multi-Site
+                class="modal-title">Migrate Single Site to Multi-site
     <cd-helper>
       <span>Migrate from a single-site deployment with a default zone group and zone to a multi-site system</span>
     </cd-helper>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index d299f59fefd0..0f41e2832204 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -150,7 +150,7 @@ export class ActionLabelsI18n {
 
     this.IMPORT = $localize`Import`;
 
-    this.MIGRATE = $localize`Migrate to Multi-Site`;
+    this.MIGRATE = $localize`Migrate to Multi-site`;
 
     /* Destroy an existing item */
     this.DELETE = $localize`Delete`;

From e539ca98d62ca63983d4066feecd04d64819d7a4 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 23 Jan 2024 11:09:26 +0000
Subject: [PATCH 1450/2492] rgw/lua: fix compilation issue when lua packages
 are disabled

Fixes: https://tracker.ceph.com/issues/63578#change-253102

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 709e748cb499..32d11a151b5e 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -3270,6 +3270,7 @@ void RadosLuaManager::handle_reload_notify(const DoutPrefixProvider* dpp, option
     return;
   }
 
+#ifdef WITH_RADOSGW_LUA_PACKAGES
   rgw::lua::packages_t failed_packages;
   std::string install_dir;
   auto r = rgw::lua::install_packages(dpp, store, 
@@ -3284,7 +3285,9 @@ void RadosLuaManager::handle_reload_notify(const DoutPrefixProvider* dpp, option
     ldpp_dout(dpp, 5) << "WARNING: failed to install Lua package: " << p
             << " from allowlist" << dendl;
   }
-  
+#else 
+  const int r = 0;
+#endif  
   ack_reload(dpp, notify_id, cookie, r);
 }
 

From 886af37744847246b3e70f54b8577ed4f9815c20 Mon Sep 17 00:00:00 2001
From: Himura Kazuto <vladislav.glagolev@devexpress.com>
Date: Tue, 23 Jan 2024 12:59:10 +0000
Subject: [PATCH 1451/2492] doc: specify correct fs type for mkfs

The default value is ext2, which is not supported (anymore?).

Signed-off-by: Vladislav Glagolev <vladislav.glagolev@devexpress.com>
---
 doc/rbd/rbd-nomad.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/rbd/rbd-nomad.rst b/doc/rbd/rbd-nomad.rst
index 66d87d6cee16..747bc3acaf79 100644
--- a/doc/rbd/rbd-nomad.rst
+++ b/doc/rbd/rbd-nomad.rst
@@ -372,6 +372,7 @@ using the newly created nomad user id and cephx key::
       clusterID = "b9127830-b0cc-4e34-aa47-9d1a2e9949a8"
       pool = "nomad"
       imageFeatures = "layering"
+      mkfsOptions = "-t ext4"
     }
 
 After the ``ceph-volume.hcl`` file has been generated, create the volume:

From a602e5f0e0e48c40fd728ae382502826ca354828 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Mon, 22 Jan 2024 20:52:58 +0000
Subject: [PATCH 1452/2492] qa/workunits/cephadm: enable nodejs:18 on centos 9
 stream

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/workunits/cephadm/test_dashboard_e2e.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qa/workunits/cephadm/test_dashboard_e2e.sh b/qa/workunits/cephadm/test_dashboard_e2e.sh
index 32e0bcc771d3..13746ec6de1a 100755
--- a/qa/workunits/cephadm/test_dashboard_e2e.sh
+++ b/qa/workunits/cephadm/test_dashboard_e2e.sh
@@ -20,6 +20,9 @@ install_common () {
         $SUDO apt-get update
         $SUDO apt-get install nodejs
     elif grep -q rhel /etc/*-release; then
+        if grep -q "CentOS Stream 9" /etc/*-release; then
+            NODEJS_VERSION="18"
+        fi
         $SUDO yum module -y enable nodejs:$NODEJS_VERSION
         $SUDO yum install -y jq npm
     else

From 852c1d97356caa2feb7c5c5062e8f78ffc26c408 Mon Sep 17 00:00:00 2001
From: galsalomon66 <gal.salomon@gmail.com>
Date: Mon, 15 Jan 2024 16:38:19 +0200
Subject: [PATCH 1453/2492] bug fixes per QE recent defects update for the
 engine_version message s3select submodule

Signed-off-by: galsalomon66 <gal.salomon@gmail.com>
---
 src/s3select | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/s3select b/src/s3select
index 9ade26c63ef1..91d843873ff0 160000
--- a/src/s3select
+++ b/src/s3select
@@ -1 +1 @@
-Subproject commit 9ade26c63ef1016dac868f53270a61e6232be9ba
+Subproject commit 91d843873ff0411ec08445ab71ec85e62f5e2ea7

From b121118e5a06f2acd5d9e5d5630af93c3847bee7 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 23 Jan 2024 16:38:40 +0200
Subject: [PATCH 1454/2492] osd: remove extra '>' in spg_t Formatter

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_types_fmt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h
index d6d746d295f7..e467d5d23044 100644
--- a/src/osd/osd_types_fmt.h
+++ b/src/osd/osd_types_fmt.h
@@ -131,7 +131,7 @@ struct fmt::formatter<spg_t> {
     if (shard_id_t::NO_SHARD == spg.shard.id) {
       return fmt::format_to(ctx.out(), "{}", spg.pgid);
     } else {
-      return fmt::format_to(ctx.out(), "{}s{}>", spg.pgid, spg.shard.id);
+      return fmt::format_to(ctx.out(), "{}s{}", spg.pgid, spg.shard.id);
     }
   }
 };

From f557827011e93d836f4334ca9c4efd5b9ef70c9b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:45:31 -0500
Subject: [PATCH 1455/2492] src/mypy.ini: disable namespace packages

Configure mypy not to use namespace packages as the cython source dirs
in src/pybind/{cephfs,rados,rbd}, etc. confuse mypy into thinking
it's an empty namespace package rather than a missing module.
Found using mypy 0.990.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/mypy.ini | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/mypy.ini b/src/mypy.ini
index bd4b436b4926..ce3e4a2ff4bb 100755
--- a/src/mypy.ini
+++ b/src/mypy.ini
@@ -6,6 +6,10 @@ check_untyped_defs = True
 show_error_context = True
 allow_redefinition = True
 disallow_untyped_defs = True
+# Disable namespace packages as the cephfs, rados, rbd, etc. dirs lack an
+# __init__.py and thus confuse mypy 0.990 (and up) into thinking these are
+# empty namespaces causing the ignore_missing_imports rules to stop working.
+namespace_packages = False
 
 [mypy-rados]
 # This would require a rados.pyi file

From c14287f749e18467b3e2d0a1da0d5c21257d68b4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 14:33:57 -0500
Subject: [PATCH 1456/2492] pybind/mgr/orchestrator: workaround spurious mypy
 issue

When run locally mypy complains about the `name` field, possibly due to
us using an old mypy version on newer versions of python (we're stuck on
a Sept. 2022 version if you weren't aware). However, this could still be
an issue in mypy as there are open issues related to the `.name` field
of an enum. Regardless, we wouldn't know - so just work around it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/orchestrator/module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 0000d25dd49c..8c496b777c5e 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -139,7 +139,7 @@ def yaml_representer(dumper: 'yaml.SafeDumper', data: 'HostDetails') -> Any:
 class DaemonFields(enum.Enum):
     service_name = 'service_name'
     daemon_type = 'daemon_type'
-    name = 'name'
+    name = 'name'  # type: ignore
     host = 'host'
     status = 'status'
     refreshed = 'refreshed'

From e71e7c14453bc69f43a7c225ffe691c4ae5e8794 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:45:46 -0500
Subject: [PATCH 1457/2492] pybind/mgr/cephadm: fix incorrect return type

Found using mypy 0.990. Newer versions of mypy see that the callback
function passed to `addFilter` are expected to return a bool.  Update
`cherrypy_filter` to do have the correct type.

I did not try to fix the issue that this same function seems copy-pasted
into three different files. >:-}

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/agent.py             | 2 +-
 src/pybind/mgr/cephadm/http_server.py       | 2 +-
 src/pybind/mgr/cephadm/service_discovery.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 93a08cb3439b..ba9c21de07b6 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -29,7 +29,7 @@ class Server:  # type: ignore
     from cephadm.module import CephadmOrchestrator
 
 
-def cherrypy_filter(record: logging.LogRecord) -> int:
+def cherrypy_filter(record: logging.LogRecord) -> bool:
     blocked = [
         'TLSV1_ALERT_DECRYPT_ERROR'
     ]
diff --git a/src/pybind/mgr/cephadm/http_server.py b/src/pybind/mgr/cephadm/http_server.py
index ef29d3b4e753..56a87bdcf642 100644
--- a/src/pybind/mgr/cephadm/http_server.py
+++ b/src/pybind/mgr/cephadm/http_server.py
@@ -12,7 +12,7 @@
     from cephadm.module import CephadmOrchestrator
 
 
-def cherrypy_filter(record: logging.LogRecord) -> int:
+def cherrypy_filter(record: logging.LogRecord) -> bool:
     blocked = [
         'TLSV1_ALERT_DECRYPT_ERROR'
     ]
diff --git a/src/pybind/mgr/cephadm/service_discovery.py b/src/pybind/mgr/cephadm/service_discovery.py
index ddc0574e2b12..b681cc8e7ff7 100644
--- a/src/pybind/mgr/cephadm/service_discovery.py
+++ b/src/pybind/mgr/cephadm/service_discovery.py
@@ -24,7 +24,7 @@ class Server:  # type: ignore
     from cephadm.module import CephadmOrchestrator
 
 
-def cherrypy_filter(record: logging.LogRecord) -> int:
+def cherrypy_filter(record: logging.LogRecord) -> bool:
     blocked = [
         'TLSV1_ALERT_DECRYPT_ERROR'
     ]

From f581ecf4b8866f2ba13f966af76baf0cef8ba500 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:45:46 -0500
Subject: [PATCH 1458/2492] pybind/mgr/cephadm: fix typing issue

Found using mypy 0.990.
Newer mypy versions see the check as tautological because `self.func` is
typed as always being `Callable` and thus always true. I don't know if
the typing is wrong or if the if-block is tautological (and redundant).
For now, silence the error by using `getattr` instead of direct
attribute access.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/configchecks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/configchecks.py b/src/pybind/mgr/cephadm/configchecks.py
index b9dcb18f478a..38cde7806afb 100644
--- a/src/pybind/mgr/cephadm/configchecks.py
+++ b/src/pybind/mgr/cephadm/configchecks.py
@@ -150,7 +150,7 @@ def to_json(self) -> Dict[str, Any]:
             "description": self.description,
             "name": self.name,
             "status": self.status,
-            "valid": True if self.func else False
+            "valid": True if getattr(self, 'func', None) else False
         }
 
 

From a354f626fe8a1a4468ed7ff4f253091bfb9f743d Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:45:46 -0500
Subject: [PATCH 1459/2492] pybind/mgr/dashboard: fix typing issues

Found using mypy 0.990.
Fix mypy error by excluding both imports in the try-except block.
Type-narrow the `ex.request` variable with an assert.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/dashboard/rest_client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/rest_client.py b/src/pybind/mgr/dashboard/rest_client.py
index 69240bace866..9e27cb57802d 100644
--- a/src/pybind/mgr/dashboard/rest_client.py
+++ b/src/pybind/mgr/dashboard/rest_client.py
@@ -23,7 +23,7 @@
 from .settings import Settings
 
 try:
-    from requests.packages.urllib3.exceptions import SSLError
+    from requests.packages.urllib3.exceptions import SSLError  # type: ignore
 except ImportError:
     from urllib3.exceptions import SSLError  # type: ignore
 
@@ -433,6 +433,7 @@ def do_request(self,
                              method.upper(), str(ex))
             raise RequestException(str(ex))
         except Timeout as ex:
+            assert ex.request
             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
                 ex.request.url)

From fe649aa78cd8a4fd1775f6cb9c74c7002779f24b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:45:46 -0500
Subject: [PATCH 1460/2492] pybind/mgr/orchestrator: fix yaml representer
 return type

Found using mypy 0.990. The `yaml_representer` function returns a Node
type, not Any. Newer versions of mypy understand this and complain.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/orchestrator/_interface.py | 4 ++--
 src/pybind/mgr/orchestrator/module.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index 5bde317d19e6..81aa66964eea 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -1277,7 +1277,7 @@ def __copy__(self) -> 'DaemonDescription':
         return DaemonDescription.from_json(self.to_json())
 
     @staticmethod
-    def yaml_representer(dumper: 'yaml.SafeDumper', data: 'DaemonDescription') -> Any:
+    def yaml_representer(dumper: 'yaml.Dumper', data: 'DaemonDescription') -> yaml.Node:
         return dumper.represent_dict(cast(Mapping, data.to_json().items()))
 
 
@@ -1410,7 +1410,7 @@ def from_json(cls, data: dict) -> 'ServiceDescription':
         return cls(spec=spec, events=events, **c_status)
 
     @staticmethod
-    def yaml_representer(dumper: 'yaml.SafeDumper', data: 'ServiceDescription') -> Any:
+    def yaml_representer(dumper: 'yaml.Dumper', data: 'ServiceDescription') -> yaml.Node:
         return dumper.represent_dict(cast(Mapping, data.to_json().items()))
 
 
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 8c496b777c5e..a7d86cbe2e5e 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -129,7 +129,7 @@ def from_json(cls, host_details: dict) -> 'HostDetails':
         return _cls
 
     @staticmethod
-    def yaml_representer(dumper: 'yaml.SafeDumper', data: 'HostDetails') -> Any:
+    def yaml_representer(dumper: 'yaml.Dumper', data: 'HostDetails') -> yaml.Node:
         return dumper.represent_dict(cast(Mapping, data.to_json().items()))
 
 

From d6af577221c96c49704b10373dc5beb423416597 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 11:30:40 -0500
Subject: [PATCH 1461/2492] qa: disable namespace packages on mypy runs

Namespace package support confuses mypy in regards to the
pybind/{cephfs,rados,rbd}, etc when using mypy >= 0.990.
Turning them off seems to not be a problem for the ceph code.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tox.ini b/qa/tox.ini
index 234873c43386..af4c69991990 100644
--- a/qa/tox.ini
+++ b/qa/tox.ini
@@ -26,7 +26,7 @@ deps =
   types-cryptography
   types-python-dateutil
   -c{toxinidir}/../src/mypy-constrains.txt
-commands = mypy {posargs:.}
+commands = mypy --no-namespace-packages {posargs:.}
 
 [testenv:py3]
 basepython = python3

From afc470ae636256dd97263ef80ff863f5bc032b9f Mon Sep 17 00:00:00 2001
From: Marcus Watts <mwatts@redhat.com>
Date: Tue, 26 Sep 2023 03:04:35 -0400
Subject: [PATCH 1462/2492] Update libkmip submodule to pull in some
 portability changes.

Signed-off-by: Marcus Watts <mwatts@redhat.com>
---
 src/libkmip | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libkmip b/src/libkmip
index b25cde94c9b8..c05329f82a1a 160000
--- a/src/libkmip
+++ b/src/libkmip
@@ -1 +1 @@
-Subproject commit b25cde94c9b8686988ed1236bd807afe74991333
+Subproject commit c05329f82a1a0e6d9bc4bae6fb25ce3d8e733f6c

From 4d52d1d22b74c3fcbc43db0973ef48d11e91748b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 23 Jan 2024 21:47:27 +0000
Subject: [PATCH 1463/2492] Revert "crimson/os/alienstore/alien_log: _flush
 concurrently"

While submitting the log line asyncronously is reasonable,
with this implementation the EntryVector &q parameter does
not necessarily outlive the submission continuation.

This reverts commit 511af83e2747361350b60ce0ce88e67a726d9343.

Fixes: https://tracker.ceph.com/issues/64140
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/os/alienstore/alien_log.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/alienstore/alien_log.cc b/src/crimson/os/alienstore/alien_log.cc
index 822b2f3ab519..a0f5b03a5d39 100644
--- a/src/crimson/os/alienstore/alien_log.cc
+++ b/src/crimson/os/alienstore/alien_log.cc
@@ -17,7 +17,8 @@ CnLog::~CnLog() {
 }
 
 void CnLog::_flush(EntryVector& q, bool crash) {
-  std::ignore = seastar::alien::submit_to(inst, shard, [&q] {
+  // XXX: the waiting here will block the thread for an indeterministic peroid
+  seastar::alien::submit_to(inst, shard, [&q] {
     for (auto& it : q) {
       crimson::get_logger(it.m_subsys).log(
         crimson::to_log_level(it.m_prio),
@@ -25,7 +26,7 @@ void CnLog::_flush(EntryVector& q, bool crash) {
         it.strv());
     }
     return seastar::make_ready_future<>();
-  });
+  }).wait();
   q.clear();
   return;
 }

From c90e8148e46ad6b85850ecfa8abdc38449986727 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Wed, 24 Jan 2024 01:42:30 -0500
Subject: [PATCH 1464/2492] rgw/lc: pass in flag as an argument when calling
 function complete(..)

fixup for 17ca50e70421cfe263784e8d65008145221e41c3

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/rgw_file.cc             |  2 +-
 src/test/rgw/test_d4n_filter.cc | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index f69ba543d898..3424d4b04d7a 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -2020,7 +2020,7 @@ namespace rgw {
     op_ret = processor->complete(state->obj_size, etag, &mtime, real_time(), attrs,
                                  (delete_at ? *delete_at : real_time()),
                                 if_match, if_nomatch, nullptr, nullptr, nullptr,
-                                rctx, true);
+                                rctx, rgw::sal::FLAG_LOG_OP);
     if (op_ret != 0) {
       /* revert attr updates */
       rgw_fh->set_mtime(omtime);
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index b1483f26e0db..fc9255e14383 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -169,7 +169,7 @@ class D4NFilterFixture : public ::testing::Test {
                        &if_match, &if_nomatch,
                        &user_data,
                        &zones_trace, &canceled,
-                       rctx, true);
+                       rctx, rgw::sal::FLAG_LOG_OP);
 
       return ret;
     }
@@ -429,7 +429,7 @@ TEST_F(D4NFilterFixture, CopyObjectReplace) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   rctx, true), 0);
+		   rctx, rgw::sal::FLAG_LOG_OP), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -554,7 +554,7 @@ TEST_F(D4NFilterFixture, CopyObjectMerge) {
 		   &if_match, &if_nomatch,
 		   &user_data,
 		   &zones_trace, &canceled,
-		   rctx, true), 0);
+		   rctx, rgw::sal::FLAG_LOG_OP), 0);
 
   unique_ptr<rgw::sal::Object> testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy"));
 
@@ -1881,7 +1881,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 rctx, true), 0);
+		 rctx, rgw::sal::FLAG_LOG_OP), 0);
  
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&data](cpp_redis::reply& reply) {
     if (reply.is_string()) {
@@ -1906,7 +1906,7 @@ TEST_F(D4NFilterFixture, DataCheck) {
 		 &if_match, &if_nomatch,
 		 &user_data,
 		 &zones_trace, &canceled,
-		 rctx, true), 0);
+		 rctx, rgw::sal::FLAG_LOG_OP), 0);
 
   client.hget("rgw-object:test_object_DataCheck:cache", "data", [&dataNew](cpp_redis::reply& reply) {
     if (reply.is_string()) {

From 144618bea5a5f730baa8d8ea5ac158137ba05b50 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Thu, 18 Jan 2024 15:00:49 +0800
Subject: [PATCH 1465/2492] src/script/lib-build.sh: clang should > 12.

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/script/lib-build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 src/script/lib-build.sh

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
old mode 100644
new mode 100755
index 950c1ab7387c..a96a20ea8265
--- a/src/script/lib-build.sh
+++ b/src/script/lib-build.sh
@@ -64,7 +64,7 @@ function discover_compiler() {
     local cxx_compiler=g++
     local c_compiler=gcc
     # ubuntu/debian ci builds prefer clang
-    for i in {17..10}; do
+    for i in {17..12}; do
         if type -t "clang-$i" > /dev/null; then
             cxx_compiler="clang++-$i"
             c_compiler="clang-$i"

From 260a76fd9a31b268044cbd84aa09a87c5e776f38 Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Wed, 24 Jan 2024 14:33:45 +0530
Subject: [PATCH 1466/2492] qa/workunits/rados/test.sh: fix GTEST_OUTPUT path

Currently, GTEST_OUTPUT is hardcoded to
'/home/ubuntu/cephtest/archive/gtest_xml_report'.
It causes errors on non-ubuntu OS.

This PR changes that to instead save all xml outputs to
TESTDIR or new tmp directory. This still saves xml files
in teuthology archive and fixes potential errors for local
test runs on non-ubuntu based OS.

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 qa/tasks/workunit.py       | 2 +-
 qa/workunits/rados/test.sh | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/workunit.py b/qa/tasks/workunit.py
index 1876b7842192..f6e55c48cd67 100644
--- a/qa/tasks/workunit.py
+++ b/qa/tasks/workunit.py
@@ -427,7 +427,7 @@ def _run_tests(ctx, refspec, role, tests, env, basedir,
                         logger=log.getChild(role),
                         args=args + optional_args,
                         label="workunit test {workunit}".format(workunit=workunit),
-                        xml_path_regex=f'{testdir}/archive/gtest_xml_report-*.xml',
+                        xml_path_regex=f'{testdir}/archive/unit_test_xml_report/*.xml',
                         output_yaml=os.path.join(ctx.archive, 'unit_test_summary.yaml'),
                     )
                 else:
diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
index 8d5f40088950..2fbb2124d658 100755
--- a/qa/workunits/rados/test.sh
+++ b/qa/workunits/rados/test.sh
@@ -12,7 +12,8 @@ function cleanup() {
 }
 trap cleanup EXIT ERR HUP INT QUIT
 
-GTEST_OUTPUT="xml:/home/ubuntu/cephtest/archive/gtest_xml_report"
+GTEST_OUTPUT_DIR=${TESTDIR:-$(mktemp -d)}/archive/unit_test_xml_report
+mkdir -p $GTEST_OUTPUT_DIR
 
 declare -A pids
 
@@ -39,7 +40,7 @@ do
     if [ $parallel -eq 1 ]; then
 	r=`printf '%25s' $f`
 	ff=`echo $f | awk '{print $1}'`
-	bash -o pipefail -exc "ceph_test_rados_$f --gtest_output=$GTEST_OUTPUT-$f.xml $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" &
+	bash -o pipefail -exc "ceph_test_rados_$f --gtest_output=xml:$GTEST_OUTPUT_DIR/$f.xml $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" &
 	pid=$!
 	echo "test $f on pid $pid"
 	pids[$f]=$pid

From 1b3454914ee6be7973f8391e1fb7d0d67d9092ce Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Mon, 15 Jan 2024 15:25:02 +0100
Subject: [PATCH 1467/2492] mgr/rook: adding some basic rook e2e testing Fixes:
 https://tracker.ceph.com/issues/64029

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 .../cluster-on-pvc-minikube.yaml              | 198 ++++++++++++++++++
 .../rook/ci/scripts/bootstrap-rook-cluster.sh |  65 +++---
 .../mgr/rook/ci/tests/features/rook.feature   |  59 +++++-
 .../ci/tests/features/steps/implementation.py |  16 +-
 .../mgr/rook/ci/tests/features/steps/utils.py |  23 ++
 5 files changed, 328 insertions(+), 33 deletions(-)
 create mode 100644 src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml

diff --git a/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml b/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
new file mode 100644
index 000000000000..2732286aba0c
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
@@ -0,0 +1,198 @@
+#################################################################################################################
+# Define the settings for the rook-ceph cluster with settings for a minikube cluster with a single node
+
+# This example expects a single node minikube cluster with three extra disks: vdb, vdc and vdd. Please modify
+# it according to your environment. See the documentation for more details on storage settings available.
+
+# For example, to create the cluster:
+#   kubectl create -f crds.yaml -f common.yaml -f operator.yaml
+#   kubectl create -f cluster-on-pvc-minikube.yaml
+#################################################################################################################
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: local-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-0
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Filesystem
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdb
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-1
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdc
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-2
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vdd
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: local0-3
+spec:
+  storageClassName: local-storage
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  # PV for mon must be a filesystem volume.
+  volumeMode: Block
+  local:
+    # To use dm devices like logical volume, please replace `/dev/sdb` with their device names like `/dev/vg-name/lv-name`.
+    path: /dev/vde
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - minikube
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: my-cluster
+  namespace: rook-ceph # namespace:cluster
+spec:
+  dataDirHostPath: /var/lib/rook
+  mon:
+    count: 1
+    allowMultiplePerNode: true
+    volumeClaimTemplate:
+      spec:
+        storageClassName: local-storage
+        resources:
+          requests:
+            storage: 10Gi
+  mgr:
+    count: 1
+    modules:
+      - name: pg_autoscaler
+        enabled: true
+  dashboard:
+    enabled: true
+    ssl: false
+  crashCollector:
+    disable: false
+  cephVersion:
+    image: quay.io/ceph/daemon-base:latest-main
+    allowUnsupported: true
+  skipUpgradeChecks: false
+  continueUpgradeAfterChecksEvenIfNotHealthy: false
+  storage:
+    storageClassDeviceSets:
+      - name: set1
+        count: 3
+        portable: false
+        tuneDeviceClass: true
+        tuneFastDeviceClass: false
+        encrypted: false
+        placement:
+        preparePlacement:
+        volumeClaimTemplates:
+          - metadata:
+              name: data
+              # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph
+              # annotations:
+              #   crushDeviceClass: hybrid
+            spec:
+              resources:
+                requests:
+                  storage: 20Gi
+              # IMPORTANT: Change the storage class depending on your environment
+              storageClassName: local-storage
+              volumeMode: Block
+              accessModes:
+                - ReadWriteOnce
+    # when onlyApplyOSDPlacement is false, will merge both placement.All() and storageClassDeviceSets.Placement
+    onlyApplyOSDPlacement: false
+  priorityClassNames:
+    mon: system-node-critical
+    osd: system-node-critical
+    mgr: system-cluster-critical
+  disruptionManagement:
+    managePodBudgets: true
+    osdMaintenanceTimeout: 30
+    pgHealthCheckTimeout: 0
+  cephConfig:
+    global:
+      mon_warn_on_pool_no_redundancy: "false"
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: builtin-mgr
+  namespace: rook-ceph # namespace:cluster
+spec:
+  name: .mgr
+  failureDomain: osd
+  replicated:
+    size: 1
+    requireSafeReplicaSize: false
diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index eb4f9fb66ce7..48181708201c 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -3,7 +3,10 @@
 set -eEx
 
 : ${CEPH_DEV_FOLDER:=${PWD}}
+CLUSTER_SPEC=${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci/cluster-specs/cluster-on-pvc-minikube.yaml
+DEFAULT_NS="rook-ceph"
 KUBECTL="minikube kubectl --"
+export ROOK_CLUSTER_NS="${ROOK_CLUSTER_NS:=$DEFAULT_NS}" ## CephCluster namespace
 
 # We build a local ceph image that contains the latest code
 # plus changes from the PR. This image will be used by the docker
@@ -27,14 +30,14 @@ setup_minikube_env() {
     fi
 
     rm -rf ~/.minikube
-    minikube start --memory="4096" --cpus="2" --disk-size=10g --extra-disks=1 --driver kvm2
+    minikube start --disk-size=20g --extra-disks=4 --driver kvm2
     # point Docker env to use docker daemon running on minikube
     eval $(minikube docker-env -p minikube)
 }
 
 build_ceph_image() {
-    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
-    CURR_CEPH_IMG=$(grep -E '^\s*image:\s+' cluster-test.yaml | sed 's/.*image: *\([^ ]*\)/\1/')
+
+    CURR_CEPH_IMG=$(grep -E '^\s*image:\s+' $CLUSTER_SPEC | sed 's/.*image: *\([^ ]*\)/\1/')
 
     cd ${CEPH_DEV_FOLDER}/src/pybind/mgr/rook/ci
     mkdir -p tmp_build/rook
@@ -54,28 +57,39 @@ build_ceph_image() {
 }
 
 create_rook_cluster() {
-    wget -q -O cluster-test.yaml https://raw.githubusercontent.com/rook/rook/master/deploy/examples/cluster-test.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/crds.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/common.yaml
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/operator.yaml
-    $KUBECTL create -f cluster-test.yaml
-    $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/dashboard-external-http.yaml
+    $KUBECTL create -f $CLUSTER_SPEC
     $KUBECTL create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/toolbox.yaml
 }
 
+is_operator_ready() {
+    local phase
+    phase=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
+    echo "PHASE: $phase"
+    [[ "$phase" == "Ready" ]]
+}
+
 wait_for_rook_operator() {
     local max_attempts=10
     local sleep_interval=20
     local attempts=0
+
     $KUBECTL rollout status deployment rook-ceph-operator -n rook-ceph --timeout=180s
-    PHASE=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
-    echo "PHASE: $PHASE"
-    while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}' | grep -q "Ready"; do
-	echo "Waiting for cluster to be ready..."
-	sleep $sleep_interval
-	attempts=$((attempts+1))
+
+    while ! is_operator_ready; do
+        echo "Waiting for rook operator to be ready..."
+        sleep $sleep_interval
+
+	# log current cluster state and pods info for debugging
+        PHASE=$($KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.phase}')
+        $KUBECTL -n rook-ceph get pods
+
+        attempts=$((attempts + 1))
         if [ $attempts -ge $max_attempts ]; then
             echo "Maximum number of attempts ($max_attempts) reached. Exiting..."
+            $KUBECTL -n rook-ceph get pods | grep operator | awk '{print $1}' | xargs $KUBECTL -n rook-ceph logs
             return 1
         fi
     done
@@ -87,7 +101,7 @@ wait_for_ceph_cluster() {
     local attempts=0
     $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=90s
     while ! $KUBECTL get cephclusters.ceph.rook.io -n rook-ceph -o jsonpath='{.items[?(@.kind == "CephCluster")].status.ceph.health}' | grep -q "HEALTH_OK"; do
-	echo "Waiting for Ceph cluster installed"
+	echo "Waiting for Ceph cluster to enter HEALTH_OK" state
 	sleep $sleep_interval
 	attempts=$((attempts+1))
         if [ $attempts -ge $max_attempts ]; then
@@ -96,18 +110,9 @@ wait_for_ceph_cluster() {
         fi
     done
     echo "Ceph cluster installed and running"
-}
 
-show_info() {
-    DASHBOARD_PASSWORD=$($KUBECTL -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo)
-    IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n rook-ceph --output jsonpath='{.items[*].status.hostIP}')
-    PORT="$($KUBECTL -n rook-ceph -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)"
-    BASE_URL="http://$IP_ADDR:$PORT"
-    echo "==========================="
-    echo "Ceph Dashboard:  "
-    echo "   IP_ADDRESS: $BASE_URL"
-    echo "   PASSWORD: $DASHBOARD_PASSWORD"
-    echo "==========================="
+    # add an additional wait to cover with any subttle change in the state
+    sleep 20
 }
 
 configure_libvirt(){
@@ -141,13 +146,21 @@ recreate_default_network(){
 
     # restart libvirtd service and wait a little bit for the service
     sudo systemctl restart libvirtd
-    sleep 20
+    sleep 10
 
     # Just some debugging information
     all_networks=$(virsh net-list --all)
     groups=$(groups)
 }
 
+enable_rook_orchestrator() {
+    echo "Enabling rook orchestrator"
+    $KUBECTL rollout status deployment rook-ceph-tools -n "$ROOK_CLUSTER_NS" --timeout=90s
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch set backend rook
+    $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status
+}
+
 ####################################################################
 ####################################################################
 
@@ -160,7 +173,7 @@ build_ceph_image
 create_rook_cluster
 wait_for_rook_operator
 wait_for_ceph_cluster
-show_info
+enable_rook_orchestrator
 
 ####################################################################
 ####################################################################
diff --git a/src/pybind/mgr/rook/ci/tests/features/rook.feature b/src/pybind/mgr/rook/ci/tests/features/rook.feature
index ae0478f8b9cd..acf733f55b49 100644
--- a/src/pybind/mgr/rook/ci/tests/features/rook.feature
+++ b/src/pybind/mgr/rook/ci/tests/features/rook.feature
@@ -1,8 +1,8 @@
 Feature: Testing Rook orchestrator commands
-    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml and
+    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml
 
     Scenario: Verify ceph cluster health
-      When I run
+      When I run ceph command
           """
           ceph health | grep HEALTH
           """
@@ -10,3 +10,58 @@ Feature: Testing Rook orchestrator commands
           """
           HEALTH_OK
           """
+
+    Scenario: Verify rook orchestrator has been enabled correctly
+      When I run ceph command
+          """
+          ceph mgr module ls | grep rook
+          """
+      Then I get something like
+          """
+          rook +on
+          """
+
+    Scenario: Verify rook orchestrator lists services correctly
+        When I run ceph command
+            """
+            ceph orch ls
+            """
+        Then I get something like
+            """
+            NAME +PORTS +RUNNING +REFRESHED +AGE +PLACEMENT
+            crash +1/1 .+
+            mgr +1/1 .+
+            mon +1/1 .+
+            osd +3 .+
+            """
+
+    Scenario: Verify rook orchestrator lists daemons correctly
+        When I run ceph command
+            """
+            ceph orch ps
+            """
+        Then I get something like
+            """
+            NAME +HOST +PORTS +STATUS +REFRESHED +AGE +MEM +USE +MEM +LIM +VERSION +IMAGE +ID
+            ceph-exporter.exporter +minikube +running .+
+            crashcollector.crash +minikube +running .+
+            mgr.a +minikube +running .+
+            mon.a +minikube +running .+
+            osd.0 +minikube +running .+
+            osd.1 +minikube +running .+
+            osd.2 +minikube +running .+
+            """
+
+    Scenario: Verify rook orchestrator lists devices correctly
+        When I run ceph command
+            """
+            ceph orch device ls
+            """
+        Then I get something like
+            """
+            HOST +PATH +TYPE +DEVICE +ID +SIZE +AVAILABLE +REFRESHED +REJECT +REASONS
+            minikube +/dev/vdb  +unknown +None +10.0G .+
+            minikube +/dev/vdc  +unknown +None +20.0G .+
+            minikube +/dev/vdd  +unknown +None +20.0G .+
+            minikube +/dev/vde  +unknown +None +20.0G .+
+            """
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
index adde61afd384..69dcde458855 100644
--- a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
@@ -2,14 +2,19 @@
 from utils import *
 import re
 
-@when("I run")
+@when("I run ceph command")
 def run_step(context):
-    context.output = run_commands(context.text)
+    context.output = run_ceph_commands(context.text)
+
+@when("I run k8s command")
+def run_step(context):
+    context.output = run_k8s_commands(context.text)
 
 @then("I get")
 def verify_result_step(context):
-    print(f"Output is:\n{context.output}\n--------------\n")
-    assert context.text == context.output
+    if (context.text != context.output):
+        display_side_by_side(context.text, context.output)
+    assert context.text == context.output, ""
 
 @then("I get something like")
 def verify_fuzzy_result_step(context):
@@ -18,4 +23,5 @@ def verify_fuzzy_result_step(context):
     num_lines = min(len(output_lines), len(expected_lines))
     for n in range(num_lines):
         if not re.match(expected_lines[n], output_lines[n]):
-            raise
+            display_side_by_side(expected_lines[n], output_lines[n])
+            assert False, ""
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/utils.py b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
index 41a71d0fb1fc..f711ec3fe6ca 100644
--- a/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/utils.py
@@ -1,4 +1,5 @@
 import subprocess
+from difflib import unified_diff
 
 ROOK_CEPH_COMMAND = "minikube kubectl -- -n rook-ceph exec -it deploy/rook-ceph-tools -- "
 CLUSTER_COMMAND = "minikube kubectl -- "
@@ -27,3 +28,25 @@ def run_commands(commands: str) -> str:
         output = execute_command(command)
 
     return output.strip("\n")
+
+def run_k8s_commands(commands: str) -> str:
+    commands_list = commands.split("\n")
+    output = ""
+    for cmd in commands_list:
+        command = CLUSTER_COMMAND + cmd
+        output = execute_command(command)
+
+    return output.strip("\n")
+
+def run_ceph_commands(commands: str) -> str:
+    commands_list = commands.split("\n")
+    output = ""
+    for cmd in commands_list:
+        command = ROOK_CEPH_COMMAND + cmd
+        output = execute_command(command)
+
+    return output.strip("\n")
+
+def display_side_by_side(expected, got):
+    diff = unified_diff(expected.splitlines(), got.splitlines(), lineterm='')
+    print('\n'.join(diff))

From 195409de94f124c63e69a75e1bba5ce568b81e05 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 24 Jan 2024 09:00:02 +0000
Subject: [PATCH 1468/2492] crimson/osd/pg_map: move maybe_create_pg and remove
 pg form header

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc | 65 +++++++++++++++++++++++++++++++++++++++
 src/crimson/osd/pg_map.h  | 62 ++-----------------------------------
 2 files changed, 67 insertions(+), 60 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 193781250f7e..5c99b82dc9dc 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -16,6 +16,71 @@ using std::make_pair;
 
 namespace crimson::osd {
 
+seastar::future<core_id_t> PGShardMapping::maybe_create_pg(
+  spg_t pgid,
+  core_id_t core)
+{
+  auto find_iter = pg_to_core.find(pgid);
+  if (find_iter != pg_to_core.end()) {
+    ceph_assert_always(find_iter->second != NULL_CORE);
+    if (core != NULL_CORE) {
+      ceph_assert_always(find_iter->second == core);
+    }
+    return seastar::make_ready_future<core_id_t>(find_iter->second);
+  } else {
+    return container().invoke_on(0,[pgid, core]
+      (auto &primary_mapping) {
+      auto [insert_iter, inserted] = primary_mapping.pg_to_core.emplace(pgid, core);
+      ceph_assert_always(inserted);
+      ceph_assert_always(primary_mapping.core_to_num_pgs.size() > 0);
+      std::map<core_id_t, unsigned>::iterator core_iter;
+      if (core == NULL_CORE) {
+        core_iter = std::min_element(
+          primary_mapping.core_to_num_pgs.begin(),
+          primary_mapping.core_to_num_pgs.end(),
+            [](const auto &left, const auto &right) {
+            return left.second < right.second;
+        });
+      } else {
+        core_iter = primary_mapping.core_to_num_pgs.find(core);
+      }
+      ceph_assert_always(primary_mapping.core_to_num_pgs.end() != core_iter);
+      insert_iter->second = core_iter->first;
+      core_iter->second++;
+      return primary_mapping.container().invoke_on_others(
+        [pgid = insert_iter->first, core = insert_iter->second]
+        (auto &other_mapping) {
+        ceph_assert_always(core != NULL_CORE);
+        auto [insert_iter, inserted] = other_mapping.pg_to_core.emplace(pgid, core);
+        ceph_assert_always(inserted);
+      });
+    }).then([this, pgid] {
+      auto find_iter = pg_to_core.find(pgid);
+      return seastar::make_ready_future<core_id_t>(find_iter->second);
+    });
+  }
+}
+
+seastar::future<> PGShardMapping::remove_pg(spg_t pgid) {
+  return container().invoke_on(0, [pgid](auto &primary_mapping) {
+    auto iter = primary_mapping.pg_to_core.find(pgid);
+    ceph_assert_always(iter != primary_mapping.pg_to_core.end());
+    ceph_assert_always(iter->second != NULL_CORE);
+    auto count_iter = primary_mapping.core_to_num_pgs.find(iter->second);
+    ceph_assert_always(count_iter != primary_mapping.core_to_num_pgs.end());
+    ceph_assert_always(count_iter->second > 0);
+    --(count_iter->second);
+    primary_mapping.pg_to_core.erase(iter);
+    return primary_mapping.container().invoke_on_others(
+      [pgid](auto &other_mapping) {
+      auto iter = other_mapping.pg_to_core.find(pgid);
+      ceph_assert_always(iter != other_mapping.pg_to_core.end());
+      ceph_assert_always(iter->second != NULL_CORE);
+      other_mapping.pg_to_core.erase(iter);
+    });
+  });
+}
+
 PGMap::PGCreationState::PGCreationState(spg_t pgid) : pgid(pgid) {}
 PGMap::PGCreationState::~PGCreationState() {}
 
diff --git a/src/crimson/osd/pg_map.h b/src/crimson/osd/pg_map.h
index 3269de43497f..a4a0a6dfb726 100644
--- a/src/crimson/osd/pg_map.h
+++ b/src/crimson/osd/pg_map.h
@@ -37,68 +37,10 @@ class PGShardMapping : public seastar::peering_sharded_service<PGShardMapping> {
   /// Returns mapping for pgid, creates new one if it doesn't already exist
   seastar::future<core_id_t> maybe_create_pg(
     spg_t pgid,
-    core_id_t core = NULL_CORE) {
-    auto find_iter = pg_to_core.find(pgid);
-    if (find_iter != pg_to_core.end()) {
-      ceph_assert_always(find_iter->second != NULL_CORE);
-      if (core != NULL_CORE) {
-        ceph_assert_always(find_iter->second == core);
-      }
-      return seastar::make_ready_future<core_id_t>(find_iter->second);
-    } else {
-      return container().invoke_on(0,[pgid, core]
-        (auto &primary_mapping) {
-        auto [insert_iter, inserted] = primary_mapping.pg_to_core.emplace(pgid, core);
-        ceph_assert_always(inserted);
-        ceph_assert_always(primary_mapping.core_to_num_pgs.size() > 0);
-        std::map<core_id_t, unsigned>::iterator core_iter;
-        if (core == NULL_CORE) {
-          core_iter = std::min_element(
-            primary_mapping.core_to_num_pgs.begin(),
-            primary_mapping.core_to_num_pgs.end(),
-              [](const auto &left, const auto &right) {
-              return left.second < right.second;
-          });
-        } else {
-          core_iter = primary_mapping.core_to_num_pgs.find(core);
-        }
-        ceph_assert_always(primary_mapping.core_to_num_pgs.end() != core_iter);
-        insert_iter->second = core_iter->first;
-        core_iter->second++;
-        return primary_mapping.container().invoke_on_others(
-          [pgid = insert_iter->first, core = insert_iter->second]
-          (auto &other_mapping) {
-          ceph_assert_always(core != NULL_CORE);
-          auto [insert_iter, inserted] = other_mapping.pg_to_core.emplace(pgid, core);
-          ceph_assert_always(inserted);
-        });
-      }).then([this, pgid] {
-        auto find_iter = pg_to_core.find(pgid);
-        return seastar::make_ready_future<core_id_t>(find_iter->second);
-      });
-    }
-  }
+    core_id_t core = NULL_CORE);
 
   /// Remove pgid
-  seastar::future<> remove_pg(spg_t pgid) {
-    return container().invoke_on(0, [pgid](auto &primary_mapping) {
-      auto iter = primary_mapping.pg_to_core.find(pgid);
-      ceph_assert_always(iter != primary_mapping.pg_to_core.end());
-      ceph_assert_always(iter->second != NULL_CORE);
-      auto count_iter = primary_mapping.core_to_num_pgs.find(iter->second);
-      ceph_assert_always(count_iter != primary_mapping.core_to_num_pgs.end());
-      ceph_assert_always(count_iter->second > 0);
-      --(count_iter->second);
-      primary_mapping.pg_to_core.erase(iter);
-      return primary_mapping.container().invoke_on_others(
-        [pgid](auto &other_mapping) {
-        auto iter = other_mapping.pg_to_core.find(pgid);
-        ceph_assert_always(iter != other_mapping.pg_to_core.end());
-        ceph_assert_always(iter->second != NULL_CORE);
-        other_mapping.pg_to_core.erase(iter);
-      });
-    });
-  }
+  seastar::future<> remove_pg(spg_t pgid);
 
   size_t get_num_pgs() const { return pg_to_core.size(); }
 

From 384f6532d3a9052b26b79763cd7a0f14ea9030c1 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 24 Jan 2024 10:22:00 +0000
Subject: [PATCH 1469/2492] crimson/osd:pg_map: rename create or remove pg

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc           | 4 ++--
 src/crimson/osd/pg_map.h            | 8 ++++----
 src/crimson/osd/pg_shard_manager.cc | 2 +-
 src/crimson/osd/pg_shard_manager.h  | 2 +-
 src/crimson/osd/shard_services.h    | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 5c99b82dc9dc..a9f1ef12230d 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -16,7 +16,7 @@ using std::make_pair;
 
 namespace crimson::osd {
 
-seastar::future<core_id_t> PGShardMapping::maybe_create_pg(
+seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
   spg_t pgid,
   core_id_t core)
 {
@@ -61,7 +61,7 @@ seastar::future<core_id_t> PGShardMapping::maybe_create_pg(
   }
 }
 
-seastar::future<> PGShardMapping::remove_pg(spg_t pgid) {
+seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
   return container().invoke_on(0, [pgid](auto &primary_mapping) {
     auto iter = primary_mapping.pg_to_core.find(pgid);
     ceph_assert_always(iter != primary_mapping.pg_to_core.end());
diff --git a/src/crimson/osd/pg_map.h b/src/crimson/osd/pg_map.h
index a4a0a6dfb726..2ea3f239b9b6 100644
--- a/src/crimson/osd/pg_map.h
+++ b/src/crimson/osd/pg_map.h
@@ -23,7 +23,7 @@ class PG;
  *
  * Maintains a mapping from spg_t to the core containing that PG.  Internally, each
  * core has a local copy of the mapping to enable core-local lookups.  Updates
- * are proxied to core 0, and the back out to all other cores -- see maybe_create_pg.
+ * are proxied to core 0, and the back out to all other cores -- see get_or_create_pg_mapping.
  */
 class PGShardMapping : public seastar::peering_sharded_service<PGShardMapping> {
 public:
@@ -35,12 +35,12 @@ class PGShardMapping : public seastar::peering_sharded_service<PGShardMapping> {
   }
 
   /// Returns mapping for pgid, creates new one if it doesn't already exist
-  seastar::future<core_id_t> maybe_create_pg(
+  seastar::future<core_id_t> get_or_create_pg_mapping(
     spg_t pgid,
     core_id_t core = NULL_CORE);
 
-  /// Remove pgid
-  seastar::future<> remove_pg(spg_t pgid);
+  /// Remove pgid mapping
+  seastar::future<> remove_pg_mapping(spg_t pgid);
 
   size_t get_num_pgs() const { return pg_to_core.size(); }
 
diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc
index 1980b36271c2..a32a1581adcd 100644
--- a/src/crimson/osd/pg_shard_manager.cc
+++ b/src/crimson/osd/pg_shard_manager.cc
@@ -23,7 +23,7 @@ seastar::future<> PGShardManager::load_pgs(crimson::os::FuturizedStore& store)
         auto[coll, shard_core] = coll_core;
 	spg_t pgid;
 	if (coll.is_pg(&pgid)) {
-          return get_pg_to_shard_mapping().maybe_create_pg(
+          return get_pg_to_shard_mapping().get_or_create_pg_mapping(
             pgid, shard_core
           ).then([this, pgid] (auto core) {
             return this->template with_remote_shard_state(
diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index 1559dde0cb1b..a3f0c8b7cc57 100644
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -373,7 +373,7 @@ class PGShardManager {
       return opref.template enter_stage<>(
 	opref.get_connection_pipeline().get_pg_mapping);
     }).then([this, &opref] {
-      return get_pg_to_shard_mapping().maybe_create_pg(opref.get_pgid());
+      return get_pg_to_shard_mapping().get_or_create_pg_mapping(opref.get_pgid());
     }).then_wrapped([this, &logger, op=std::move(op)](auto fut) mutable {
       if (unlikely(fut.failed())) {
         logger.error("{}: failed before with_pg", *op);
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index fbfe0e55c001..57dff9d2ee3e 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -404,7 +404,7 @@ class ShardServices : public OSDMapService {
 
   auto remove_pg(spg_t pgid) {
     local_state.pg_map.remove_pg(pgid);
-    return pg_to_shard_mapping.remove_pg(pgid);
+    return pg_to_shard_mapping.remove_pg_mapping(pgid);
   }
 
   crimson::common::CephContext *get_cct() {

From f3b39ab923e6aeab9d21911631da368222b56008 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 24 Jan 2024 10:44:44 +0000
Subject: [PATCH 1470/2492] crimson/osd_pg_map: migrate logging to new style

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index a9f1ef12230d..234db5863480 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -2,15 +2,11 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "crimson/osd/pg_map.h"
-
+#include "crimson/common/log.h"
 #include "crimson/osd/pg.h"
 #include "common/Formatter.h"
 
-namespace {
-  seastar::logger& logger() {
-    return crimson::get_logger(ceph_subsys_osd);
-  }
-}
+SET_SUBSYS(osd);
 
 using std::make_pair;
 
@@ -121,7 +117,8 @@ Ref<PG> PGMap::get_pg(spg_t pgid)
 
 void PGMap::set_creating(spg_t pgid)
 {
-  logger().debug("Creating {}", pgid);
+  LOG_PREFIX(PGMap::set_creating);
+  DEBUG("Creating {}", pgid);
   ceph_assert(pgs.count(pgid) == 0);
   auto pg = pgs_creating.find(pgid);
   ceph_assert(pg != pgs_creating.end());
@@ -131,7 +128,8 @@ void PGMap::set_creating(spg_t pgid)
 
 void PGMap::pg_created(spg_t pgid, Ref<PG> pg)
 {
-  logger().debug("Created {}", pgid);
+  LOG_PREFIX(PGMap::pg_created);
+  DEBUG("Created {}", pgid);
   ceph_assert(!pgs.count(pgid));
   pgs.emplace(pgid, pg);
 
@@ -150,7 +148,8 @@ void PGMap::pg_loaded(spg_t pgid, Ref<PG> pg)
 
 void PGMap::pg_creation_canceled(spg_t pgid)
 {
-  logger().debug("PGMap::pg_creation_canceled: {}", pgid);
+  LOG_PREFIX(PGMap::pg_creation_canceled);
+  DEBUG("{}", pgid);
   ceph_assert(!pgs.count(pgid));
 
   auto creating_iter = pgs_creating.find(pgid);

From e128396eca58579cade1c79bdfd45a3c2747bc52 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 24 Jan 2024 10:56:01 +0000
Subject: [PATCH 1471/2492] crimson/osd/pg_map: add logs

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 234db5863480..14f2b9f03e84 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -16,6 +16,7 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
   spg_t pgid,
   core_id_t core)
 {
+  LOG_PREFIX(PGShardMapping::get_or_create_pg_mapping);
   auto find_iter = pg_to_core.find(pgid);
   if (find_iter != pg_to_core.end()) {
     ceph_assert_always(find_iter->second != NULL_CORE);
@@ -24,7 +25,7 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
     }
     return seastar::make_ready_future<core_id_t>(find_iter->second);
   } else {
-    return container().invoke_on(0,[pgid, core]
+    return container().invoke_on(0,[pgid, core, FNAME]
       (auto &primary_mapping) {
       auto [insert_iter, inserted] = primary_mapping.pg_to_core.emplace(pgid, core);
       ceph_assert_always(inserted);
@@ -43,6 +44,8 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
       ceph_assert_always(primary_mapping.core_to_num_pgs.end() != core_iter);
       insert_iter->second = core_iter->first;
       core_iter->second++;
+      DEBUG("mapping pg {} to core: {} with num_pgs of: {}",
+            pgid, insert_iter->second, core_iter->second);
       return primary_mapping.container().invoke_on_others(
         [pgid = insert_iter->first, core = insert_iter->second]
         (auto &other_mapping) {
@@ -58,6 +61,8 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
 }
 
 seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
+  LOG_PREFIX(PGShardMapping::remove_pg_mapping);
+  DEBUG("{}", pgid);
   return container().invoke_on(0, [pgid](auto &primary_mapping) {
     auto iter = primary_mapping.pg_to_core.find(pgid);
     ceph_assert_always(iter != primary_mapping.pg_to_core.end());

From 5c1a57a1afed1e4a632fb6df6de34ff9689460db Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 24 Jan 2024 09:44:25 -0500
Subject: [PATCH 1472/2492] cmake/arrow: don't treat warnings as errors

with arrow's default BUILD_WARNING_LEVEL, -Werror is added to cflags and
debug builds fail on a warning about _FORTIFY_SOURCE

Fixes: https://tracker.ceph.com/issues/63130

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 cmake/modules/BuildArrow.cmake | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmake/modules/BuildArrow.cmake b/cmake/modules/BuildArrow.cmake
index 691108a40c55..8a16b9b8899d 100644
--- a/cmake/modules/BuildArrow.cmake
+++ b/cmake/modules/BuildArrow.cmake
@@ -86,6 +86,9 @@ function(build_arrow)
   else()
     list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
   endif()
+  # don't add -Werror or debug package builds fail with:
+  #warning _FORTIFY_SOURCE requires compiling with optimization (-O)
+  list(APPEND arrow_CMAKE_ARGS -DBUILD_WARNING_LEVEL=PRODUCTION)
 
   # we use an external project and copy the sources to bin directory to ensure
   # that object files are built outside of the source tree.

From b62c6521fda75e5b39be42e18a95dcad587fc8b8 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Wed, 24 Jan 2024 20:39:54 +0530
Subject: [PATCH 1473/2492] mgr/dashboard: Fix issue with Object's Overview
 page

Fixes https://tracker.ceph.com/issues/64150

Issue: Due to absence of null checks, the overview page was failing to
load.

Fix: Added, null check at relevant places to ensure data availability
before accessing the data.

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../dashboard-area-chart/dashboard-area-chart.component.ts  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index 4a0d95f5421d..ac0b9ac2ff43 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -196,7 +196,7 @@ export class DashboardAreaChartComponent implements OnChanges {
 
     this.setChartTicks();
 
-    if (this.dataArray) {
+    if (this.dataArray && this.dataArray.length && this.dataArray[0] && this.dataArray[0].length) {
       this.dataArray = changes?.dataArray?.currentValue || this.dataArray;
       this.currentChartData = this.chartData;
       for (let index = 0; index < this.dataArray.length; index++) {
@@ -223,7 +223,7 @@ export class DashboardAreaChartComponent implements OnChanges {
 
   private formatData(array: Array<any>): any {
     let formattedData = {};
-    formattedData = array.map((data: any) => ({
+    formattedData = array?.map((data: any) => ({
       x: data[0] * 1000,
       y: Number(this.convertToChartDataUnits(data[1]).replace(/[^\d,.]+/g, ''))
     }));
@@ -290,7 +290,7 @@ export class DashboardAreaChartComponent implements OnChanges {
     let maxValueDataUnits = '';
 
     const allDataValues = this.dataArray.reduce((array: string[], data) => {
-      return array.concat(data.map((values: [number, string]) => values[1]));
+      return array.concat(data?.map((values: [number, string]) => values[1]));
     }, []);
 
     maxValue = Math.max(...allDataValues.map(Number));

From 39ca72469b0763b4228ccc8b715a78a5cf1e2fa1 Mon Sep 17 00:00:00 2001
From: igomon <igomon@bloomberg.net>
Date: Tue, 16 Jan 2024 14:28:01 -0500
Subject: [PATCH 1474/2492] rgw/s3-notifications: reservations are now
 committed asynchronously

Signed-off-by: Igor Gomon <igomon@bloomberg.net>
---
 src/rgw/driver/rados/rgw_notify.cc | 32 +++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 6711f310c268..9baed182ed94 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -84,6 +84,25 @@ auto make_stack_allocator() {
 
 const std::string Q_LIST_OBJECT_NAME = "queues_list_object";
 
+struct PublishCommitCompleteArg {
+
+    PublishCommitCompleteArg(std::string _queue_name, const DoutPrefixProvider *_dpp)
+            : queue_name{std::move(_queue_name)}, dpp{_dpp} {}
+
+    std::string queue_name;
+    const DoutPrefixProvider *dpp;
+};
+
+void publish_commit_completion(rados_completion_t completion, void *arg) {
+    auto *comp_obj = reinterpret_cast<librados::AioCompletionImpl *>(completion);
+    std::unique_ptr<PublishCommitCompleteArg> pcc_arg(reinterpret_cast<PublishCommitCompleteArg *>(arg));
+    if (comp_obj->get_return_value() < 0) {
+        ldpp_dout(pcc_arg->dpp, 1) << "ERROR: failed to commit reservation to queue: "
+                                   << pcc_arg->queue_name << ". error: " << comp_obj->get_return_value()
+                                   << dendl;
+    }
+};
+
 class Manager : public DoutPrefixProvider {
   const size_t max_queue_size;
   const uint32_t queues_update_period_ms;
@@ -1087,16 +1106,19 @@ int publish_commit(rgw::sal::Object* obj,
       std::vector<buffer::list> bl_data_vec{std::move(bl)};
       librados::ObjectWriteOperation op;
       cls_2pc_queue_commit(op, bl_data_vec, topic.res_id);
-      const auto ret = rgw_rados_operate(
-	dpp, res.store->getRados()->get_notif_pool_ctx(),
-	queue_name, &op, res.yield);
+      aio_completion_ptr completion {librados::Rados::aio_create_completion()};
+      auto pcc_arg = make_unique<PublishCommitCompleteArg>(queue_name, dpp);
+      completion->set_complete_callback(pcc_arg.get(), publish_commit_completion);
+      auto &io_ctx = res.store->getRados()->get_notif_pool_ctx();
+      int ret = io_ctx.aio_operate(queue_name, completion.get(), &op);
       topic.res_id = cls_2pc_reservation::NO_ID;
       if (ret < 0) {
         ldpp_dout(dpp, 1) << "ERROR: failed to commit reservation to queue: "
-			  << queue_name << ". error: " << ret
-			  << dendl;
+                          << queue_name << ". error: " << ret << dendl;
         return ret;
       }
+      pcc_arg.release();
+      completion.release();
     } else {
       try {
         // TODO add endpoint LRU cache

From 310eb4f9b7da7eeb1bc48f89e62ce78f8c63cfae Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 24 Jan 2024 19:03:56 +0100
Subject: [PATCH 1475/2492] mgr/rook: increase minikube mem to 6GB to avoid
 stability issues

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index 48181708201c..d3c35544131e 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -30,7 +30,7 @@ setup_minikube_env() {
     fi
 
     rm -rf ~/.minikube
-    minikube start --disk-size=20g --extra-disks=4 --driver kvm2
+    minikube start --memory="6144" --disk-size=20g --extra-disks=4 --driver kvm2
     # point Docker env to use docker daemon running on minikube
     eval $(minikube docker-env -p minikube)
 }

From 1b5a12b5699f5bbaa0a03f706c6b4045b79a4f38 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 10:57:42 -0500
Subject: [PATCH 1476/2492] qa/workunits/windows: alter python default argument
 types

Found using mypy 1.1.1.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/workunits/windows/test_rbd_wnbd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/qa/workunits/windows/test_rbd_wnbd.py b/qa/workunits/windows/test_rbd_wnbd.py
index db14234a2b5f..f233876cb35a 100644
--- a/qa/workunits/windows/test_rbd_wnbd.py
+++ b/qa/workunits/windows/test_rbd_wnbd.py
@@ -75,7 +75,7 @@
 class CephTestException(Exception):
     msg_fmt = "An exception has been encountered."
 
-    def __init__(self, message: str = None, **kwargs):
+    def __init__(self, message: str = '', **kwargs):
         self.kwargs = kwargs
         if not message:
             message = self.msg_fmt % kwargs
@@ -520,7 +520,7 @@ def cleanup(self):
     @classmethod
     def print_results(cls,
                       title: str = "Test results",
-                      description: str = None):
+                      description: str = ''):
         pass
 
 
@@ -551,7 +551,7 @@ class RbdFioTest(RbdTest):
 
     def __init__(self,
                  *args,
-                 fio_size_mb: int = None,
+                 fio_size_mb: int = 0,
                  iterations: int = 1,
                  workers: int = 1,
                  bs: str = "2M",
@@ -603,7 +603,7 @@ def _get_fio_path(self):
         return self.image.path
 
     @Tracer.trace
-    def _run_fio(self, fio_size_mb=None):
+    def _run_fio(self, fio_size_mb: int = 0) -> None:
         LOG.info("Starting FIO test.")
         cmd = [
             "fio", "--thread", "--output-format=json",
@@ -628,7 +628,7 @@ def run(self):
     @classmethod
     def print_results(cls,
                       title: str = "Benchmark results",
-                      description: str = None):
+                      description: str = ''):
         if description:
             title = "%s (%s)" % (title, description)
 

From 86f23452752032f6768d312e2d772dca8b3c6beb Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 11:07:54 -0500
Subject: [PATCH 1477/2492] pybind/mgr/dashboard: correct type signature of
 CRUDEndpoint arg

Typing issue was found by mypy 1.5.1. The type only accepted values
of str but based on the data and code using set_column data the
boolean assignments to the 'isHidden' key were correct and so
I updated the type to also accept bool.

A more correct fix might be to use typing.TypedDict - this would
require python 3.8 (or typing extensions) and someone more familiar
with the dashboard code :-)

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/_crud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/controllers/_crud.py b/src/pybind/mgr/dashboard/controllers/_crud.py
index d65649cadb4b..855832ff5508 100644
--- a/src/pybind/mgr/dashboard/controllers/_crud.py
+++ b/src/pybind/mgr/dashboard/controllers/_crud.py
@@ -332,7 +332,7 @@ class CRUDEndpoint:
     CRUDClassMetadata: Optional[RESTController] = None
 
     def __init__(self, router: APIRouter, doc: APIDoc,
-                 set_column: Optional[Dict[str, Dict[str, str]]] = None,
+                 set_column: Optional[Dict[str, Dict[str, Union[str, bool]]]] = None,
                  actions: Optional[List[TableAction]] = None,
                  permissions: Optional[List[str]] = None, forms: Optional[List[Form]] = None,
                  column_key: Optional[str] = None,

From 90f9a98c8cdbff957414ee7952b4632a2d387bb8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 20 Jan 2024 11:22:51 -0500
Subject: [PATCH 1478/2492] pybind/mgr/rook: use AllMatcher when size is not
 provided

Issue found by mypy 1.6.1. The previous code was:
`and ((sizematcher != None) or sizematcher.compare(device)`
meaning that if sizematcher is not none "return" true, but if
sizematcher is not none execute the compare method. This is of course
impossible as None will never have a compare method. I assume that the

At Kefu Chai's suggestion we can replace the None condition with an
AllMatcher object when no size is provided to create a size matcher.
This both corrects the error and makes the code much simpler and more
readable.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/rook/rook_cluster.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index 21a8e77ba1ff..c7d2a34a7b75 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -33,7 +33,11 @@
     HostPattern,
 )
 from ceph.utils import datetime_now
-from ceph.deployment.drive_selection.matchers import SizeMatcher
+from ceph.deployment.drive_selection.matchers import (
+    AllMatcher,
+    Matcher,
+    SizeMatcher,
+)
 from nfs.cluster import create_ganesha_pool
 from nfs.module import Module
 from nfs.export import NFSRados
@@ -416,7 +420,7 @@ def device_to_device_set(self, drive_group: DriveGroupSpec, d: Device) -> ccl.St
     def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupSpec, matching_hosts: List[str]) -> List[Device]:
         device_list = []
         assert drive_group.data_devices is not None
-        sizematcher: Optional[SizeMatcher] = None
+        sizematcher: Matcher = AllMatcher('', None)
         if drive_group.data_devices.size:
             sizematcher = SizeMatcher('size', drive_group.data_devices.size)
         limit = getattr(drive_group.data_devices, 'limit', None)
@@ -444,7 +448,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
                             all 
                             or (
                                 device.sys_api['node'] in matching_hosts
-                                and ((sizematcher != None) or sizematcher.compare(device))
+                                and sizematcher.compare(device)
                                 and (
                                     not drive_group.data_devices.paths
                                     or (device.path in paths)
@@ -481,7 +485,7 @@ class LSOCreator(DefaultCreator):
     def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupSpec, matching_hosts: List[str]) -> List[Device]:
         device_list = []
         assert drive_group.data_devices is not None
-        sizematcher = None
+        sizematcher: Matcher = AllMatcher('', None)
         if drive_group.data_devices.size:
             sizematcher = SizeMatcher('size', drive_group.data_devices.size)
         limit = getattr(drive_group.data_devices, 'limit', None)
@@ -511,7 +515,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
                             all 
                             or (
                                 device.sys_api['node'] in matching_hosts
-                                and ((sizematcher != None) or sizematcher.compare(device))
+                                and sizematcher.compare(device)
                                 and (
                                     not drive_group.data_devices.paths
                                     or device.path in paths

From a450820b474338e5f5ac19c593c73f5d2aa08557 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 22 Jan 2024 13:37:24 -0500
Subject: [PATCH 1479/2492] pybind/mgr/rook: rename variable from all to _all

Every time I looked at these functions my brain would be thrown off by
`all or ...` as I see all as the builtin function `all`:
    https://docs.python.org/3/library/functions.html#all

Rename `all` so that it avoids shadowing the function `all` and avoid
some confusion for people like me.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/rook/rook_cluster.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index c7d2a34a7b75..ff8af5f38df4 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -425,7 +425,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
             sizematcher = SizeMatcher('size', drive_group.data_devices.size)
         limit = getattr(drive_group.data_devices, 'limit', None)
         count = 0
-        all = getattr(drive_group.data_devices, 'all', None)
+        _all = getattr(drive_group.data_devices, 'all', None)
         paths = [device.path for device in drive_group.data_devices.paths]
         osd_list = []
         for pod in rook_pods.items:
@@ -445,7 +445,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
                 if not limit or (count < limit):
                     if device.available:
                         if (
-                            all 
+                            _all
                             or (
                                 device.sys_api['node'] in matching_hosts
                                 and sizematcher.compare(device)
@@ -489,7 +489,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
         if drive_group.data_devices.size:
             sizematcher = SizeMatcher('size', drive_group.data_devices.size)
         limit = getattr(drive_group.data_devices, 'limit', None)
-        all = getattr(drive_group.data_devices, 'all', None)
+        _all = getattr(drive_group.data_devices, 'all', None)
         paths = [device.path for device in drive_group.data_devices.paths]
         vendor = getattr(drive_group.data_devices, 'vendor', None)
         model = getattr(drive_group.data_devices, 'model', None)
@@ -512,7 +512,7 @@ def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupS
                 if not limit or (count < limit):
                     if device.available:
                         if (
-                            all 
+                            _all
                             or (
                                 device.sys_api['node'] in matching_hosts
                                 and sizematcher.compare(device)

From 1eebb7ba7903c6db0ab37a0457b263a1b2b00ff5 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Mon, 22 Jan 2024 17:06:58 -0500
Subject: [PATCH 1480/2492] rbd_nbd: fix resize of images mapped using netlink

Include device identifier or cookie in the message sent to the kernel
to resize images mapped to NBD devices using netlink. Otherwise,
netlink_resize() fails and the size of the device isn't updated.

Fixes: https://tracker.ceph.com/issues/64139
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 qa/workunits/rbd/rbd-nbd.sh  |  5 ++++-
 src/tools/rbd_nbd/rbd-nbd.cc | 19 +++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh
index bc89e9be5a18..8e1b05b3f782 100755
--- a/qa/workunits/rbd/rbd-nbd.sh
+++ b/qa/workunits/rbd/rbd-nbd.sh
@@ -202,8 +202,11 @@ provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
 used=`rbd -p ${POOL} --format xml du ${IMAGE} |
   $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
 [ "${used}" -lt "${provisioned}" ]
+unmap_device ${DEV} ${PID}
 
 # resize test
+DEV=`_sudo rbd device -t nbd -o try-netlink map ${POOL}/${IMAGE}`
+get_pid ${POOL}
 devname=$(basename ${DEV})
 blocks=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
 test -n "${blocks}"
@@ -216,9 +219,9 @@ rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M
 blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
 test -n "${blocks2}"
 test ${blocks2} -eq ${blocks}
+unmap_device ${DEV} ${PID}
 
 # read-only option test
-unmap_device ${DEV} ${PID}
 DEV=`_sudo rbd --device-type nbd map --read-only ${POOL}/${IMAGE}`
 PID=$(rbd device --device-type nbd list | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \
     '$2 == pool && $3 == img && $5 == dev {print $1}')
diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc
index e348bd8fe431..3626002685bb 100644
--- a/src/tools/rbd_nbd/rbd-nbd.cc
+++ b/src/tools/rbd_nbd/rbd-nbd.cc
@@ -194,7 +194,8 @@ static EventSocket terminate_event_sock;
 static int parse_args(vector<const char*>& args, std::ostream *err_msg,
                       Config *cfg);
 static int netlink_disconnect(int index);
-static int netlink_resize(int nbd_index, uint64_t size);
+static int netlink_resize(int nbd_index, const std::string& cookie,
+                          uint64_t size);
 
 static int run_quiesce_hook(const std::string &quiesce_hook,
                             const std::string &devpath,
@@ -744,6 +745,7 @@ class NBDWatchCtx : public librbd::UpdateWatchCtx
   ceph::mutex lock = ceph::make_mutex("NBDWatchCtx::Locker");
   bool notify = false;
   bool terminated = false;
+  std::string cookie;
 
   bool wait_notify() {
     dout(10) << __func__ << dendl;
@@ -779,11 +781,11 @@ class NBDWatchCtx : public librbd::UpdateWatchCtx
              << dendl;
       }
       if (use_netlink) {
-        ret = netlink_resize(nbd_index, new_size);
+        ret = netlink_resize(nbd_index, cookie, new_size);
       } else {
         ret = ioctl(fd, NBD_SET_SIZE, new_size);
         if (ret < 0) {
-          derr << "resize failed: " << cpp_strerror(errno) << dendl;
+          derr << "ioctl resize failed: " << cpp_strerror(errno) << dendl;
         }
       }
       if (!ret) {
@@ -805,13 +807,15 @@ class NBDWatchCtx : public librbd::UpdateWatchCtx
               bool _use_netlink,
               librados::IoCtx &_io_ctx,
               librbd::Image &_image,
-              unsigned long _size)
+              unsigned long _size,
+              std::string _cookie)
     : fd(_fd)
     , nbd_index(_nbd_index)
     , use_netlink(_use_netlink)
     , io_ctx(_io_ctx)
     , image(_image)
     , size(_size)
+    , cookie(std::move(_cookie))
   {
     handle_notify_thread = make_named_thread("rbd_handle_notify",
                                              &NBDWatchCtx::handle_notify_entry,
@@ -1319,7 +1323,8 @@ static int netlink_disconnect_by_path(const std::string& devpath)
   return netlink_disconnect(index);
 }
 
-static int netlink_resize(int nbd_index, uint64_t size)
+static int netlink_resize(int nbd_index, const std::string& cookie,
+                          uint64_t size)
 {
   struct nl_sock *sock;
   struct nl_msg *msg;
@@ -1347,6 +1352,8 @@ static int netlink_resize(int nbd_index, uint64_t size)
 
   NLA_PUT_U32(msg, NBD_ATTR_INDEX, nbd_index);
   NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
+  if (!cookie.empty())
+    NLA_PUT_STRING(msg, NBD_ATTR_BACKEND_IDENTIFIER, cookie.c_str());
 
   ret = nl_send_sync(sock, msg);
   if (ret < 0) {
@@ -1896,7 +1903,7 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect)
     uint64_t handle;
 
     NBDWatchCtx watch_ctx(nbd, nbd_index, use_netlink, io_ctx, image,
-                          info.size);
+                          info.size, cfg->cookie);
     r = image.update_watch(&watch_ctx, &handle);
     if (r < 0)
       goto close_nbd;

From 1712b95c784c5ce381fbf4b09e8219ea40bd99a8 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Tue, 23 Jan 2024 16:07:04 -0500
Subject: [PATCH 1481/2492] rbd-nbd: log errors during netlink_resize() using
 derr

When using rbd CLI to map the images to NBD devices via netlink,
any errors that arose during image resizing in netlink_resize()
were not logged. Switching the error logging from using cerr to
derr helps log the errors from netlink_resize().

Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 src/tools/rbd_nbd/rbd-nbd.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc
index 3626002685bb..f2dfa1f660e8 100644
--- a/src/tools/rbd_nbd/rbd-nbd.cc
+++ b/src/tools/rbd_nbd/rbd-nbd.cc
@@ -1332,21 +1332,21 @@ static int netlink_resize(int nbd_index, const std::string& cookie,
 
   sock = netlink_init(&nl_id);
   if (!sock) {
-    cerr << "rbd-nbd: Netlink interface not supported." << std::endl;
-    return 1;
+    derr << __func__ << ": netlink interface not supported" << dendl;
+    return -EINVAL;
   }
 
   nl_socket_modify_cb(sock, NL_CB_VALID, NL_CB_CUSTOM, genl_handle_msg, NULL);
 
   msg = nlmsg_alloc();
   if (!msg) {
-    cerr << "rbd-nbd: Could not allocate netlink message." << std::endl;
+    derr << __func__ << ": could not allocate netlink message" << dendl;
     goto free_sock;
   }
 
   if (!genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, nl_id, 0, 0,
                    NBD_CMD_RECONFIGURE, 0)) {
-    cerr << "rbd-nbd: Could not setup message." << std::endl;
+    derr << __func__ << ": could not setup netlink message" << dendl;
     goto free_msg;
   }
 
@@ -1357,7 +1357,8 @@ static int netlink_resize(int nbd_index, const std::string& cookie,
 
   ret = nl_send_sync(sock, msg);
   if (ret < 0) {
-    cerr << "rbd-nbd: netlink resize failed: " << nl_geterror(ret) << std::endl;
+    derr << __func__ << ": netlink resize failed: " << nl_geterror(ret)
+         << dendl;
     goto free_sock;
   }
 

From 2b694195bff06d0f92c69d9d9c4144ec13ee649d Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Wed, 24 Jan 2024 20:37:04 +0000
Subject: [PATCH 1482/2492] .github: fix length of dependabot commit message
 prefix

Followup to commit 265f727481c729fbe78dce7410def7de6fe869d1.

Fixes this error mesage under GitHub Insights (https://github.com/ceph/ceph/network/updates):

```
The property '#/updates/1/commit-message/prefix' was not of a maximum string length of 15
```

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 .github/dependabot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 98b7d53d8119..d23134597030 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -17,6 +17,6 @@ updates:
     schedule:
       interval: "daily"
     commit-message:
-      prefix: ".github/workflows:"
+      prefix: ".github:"
     pull-request-branch-name:
       separator: "-"

From 809e1304b09b9aeb1605ee010a0f14414723cf8d Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 10 Jan 2024 12:37:15 -0500
Subject: [PATCH 1483/2492] include/inline_memory.h: Add missing includes to
 fix build

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/include/inline_memory.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/include/inline_memory.h b/src/include/inline_memory.h
index 6a1fe34800ab..e2283830103c 100644
--- a/src/include/inline_memory.h
+++ b/src/include/inline_memory.h
@@ -14,6 +14,10 @@
 #ifndef CEPH_INLINE_MEMORY_H
 #define CEPH_INLINE_MEMORY_H
 
+#include <cstdint>
+#include <cstddef> // For size_t
+#include <cstring> // for memcpy
+
 #if defined(__GNUC__)
 
 // optimize for the common case, which is very small copies
@@ -25,7 +29,7 @@ void *maybe_inline_memcpy(void *dest, const void *src, size_t l,
 			 size_t inline_len)
 {
   if (l > inline_len) {
-    return memcpy(dest, src, l);
+    return std::memcpy(dest, src, l);
   }
   switch (l) {
   case 8:
@@ -63,7 +67,7 @@ void *maybe_inline_memcpy(void *dest, const void *src, size_t l,
 
 #else
 
-#define maybe_inline_memcpy(d, s, l, x) memcpy(d, s, l)
+#define maybe_inline_memcpy(d, s, l, x) std::memcpy(d, s, l)
 
 #endif
 

From c7789f10fee63dac2103908941a375faa32acf5a Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 14 Apr 2023 14:58:16 -0400
Subject: [PATCH 1484/2492] common/async: `io_context_pool` constructor takes
 int64_t

Just because this is what Ceph's config uses and it saves a narrowing
conversion. If we want to set a max value on the thread count, we
should do it in config.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/common/async/context_pool.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/async/context_pool.h b/src/common/async/context_pool.h
index 9c6cab7677db..0fee5dd7fcac 100644
--- a/src/common/async/context_pool.h
+++ b/src/common/async/context_pool.h
@@ -46,7 +46,7 @@ class io_context_pool {
   }
 public:
   io_context_pool() noexcept {}
-  io_context_pool(std::int16_t threadcnt) noexcept {
+  io_context_pool(std::int64_t threadcnt) noexcept {
     start(threadcnt);
   }
   ~io_context_pool() {

From 82a40dc151cc4ec1576241b88f099abefc5b4b9a Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 14 Apr 2023 15:43:36 -0400
Subject: [PATCH 1485/2492] ceph/async: `io_context_pool` constructor/start
 takes init function

Add overloads of the constructor and `start` that take an init
function, for setting up the thread before dropping into
`io_context::run`.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/common/async/context_pool.h | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/common/async/context_pool.h b/src/common/async/context_pool.h
index 0fee5dd7fcac..91a7090ade76 100644
--- a/src/common/async/context_pool.h
+++ b/src/common/async/context_pool.h
@@ -16,6 +16,7 @@
 #ifndef CEPH_COMMON_ASYNC_CONTEXT_POOL_H
 #define CEPH_COMMON_ASYNC_CONTEXT_POOL_H
 
+#include <concepts>
 #include <cstddef>
 #include <cstdint>
 #include <mutex>
@@ -46,9 +47,14 @@ class io_context_pool {
   }
 public:
   io_context_pool() noexcept {}
+
   io_context_pool(std::int64_t threadcnt) noexcept {
     start(threadcnt);
   }
+  template<std::invocable<> Init>
+  io_context_pool(std::int64_t threadcnt, Init&& init) noexcept {
+    start(threadcnt, std::move(init));
+  }
   ~io_context_pool() {
     stop();
   }
@@ -59,7 +65,22 @@ class io_context_pool {
       ioctx.restart();
       for (std::int16_t i = 0; i < threadcnt; ++i) {
 	threadvec.emplace_back(make_named_thread("io_context_pool",
-						 [this]() {
+						 [this] {
+						   ioctx.run();
+						 }));
+      }
+    }
+  }
+  template<std::invocable<> Init>
+  void start(std::int16_t threadcnt, Init&& init) noexcept {
+    auto l = std::scoped_lock(m);
+    if (threadvec.empty()) {
+      guard.emplace(boost::asio::make_work_guard(ioctx));
+      ioctx.restart();
+      for (std::int16_t i = 0; i < threadcnt; ++i) {
+	threadvec.emplace_back(make_named_thread("io_context_pool",
+						 [this, init=std::move(init)] {
+						   std::move(init)();
 						   ioctx.run();
 						 }));
       }

From 7c4eee61c4ab6e54f9298053eaecde682b19cc2b Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 14 Apr 2023 14:59:19 -0400
Subject: [PATCH 1486/2492] rgw: SAL drivers take `boost::asio::io_context`

`RadosDriver` needs it, and since SAL generally uses `optional_yield`,
other stores are going to need it to implement that sensibly.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/CMakeLists.txt                |  1 +
 src/rgw/driver/rados/rgw_rados.cc     | 16 +++++++----
 src/rgw/driver/rados/rgw_sal_rados.cc | 22 +++++++++++++--
 src/rgw/driver/rados/rgw_sal_rados.h  | 13 +++++++--
 src/rgw/driver/rados/rgw_service.cc   | 40 ++++++++++++++-------------
 src/rgw/driver/rados/rgw_service.h    | 26 +++++++++--------
 src/rgw/rgw_admin.cc                  |  9 ++++--
 src/rgw/rgw_appmain.cc                |  3 +-
 src/rgw/rgw_main.h                    |  6 +++-
 src/rgw/rgw_object_expirer.cc         |  5 +++-
 src/rgw/rgw_realm_reloader.cc         |  6 ++--
 src/rgw/rgw_realm_reloader.h          |  5 +++-
 src/rgw/rgw_sal.cc                    | 11 ++++----
 src/rgw/rgw_sal.h                     | 12 +++++---
 src/test/rgw/rgw_cr_test.cc           |  4 ++-
 src/test/rgw/test_d4n_filter.cc       |  4 +++
 src/test/rgw/test_rgw_iam_policy.cc   |  4 ++-
 src/test/rgw/test_rgw_lua.cc          |  7 +++--
 18 files changed, 132 insertions(+), 62 deletions(-)

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index a308c833ddce..3c2f1423808b 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -270,6 +270,7 @@ target_link_libraries(rgw_common
     cls_user_client
     cls_version_client
     librados
+    libneorados
     rt
     ICU::uc
     OATH::OATH
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 207ed5c919c0..fe0d14d1029a 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1365,11 +1365,10 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
 int RGWRados::init_svc(bool raw, const DoutPrefixProvider *dpp)
 {
   if (raw) {
-    return svc.init_raw(cct, use_cache, get_rados_handle(), null_yield, dpp);
+    return svc.init_raw(cct, driver, use_cache, null_yield, dpp);
   }
 
-  return svc.init(cct, use_cache, run_sync_thread, get_rados_handle(),
-		  null_yield, dpp);
+  return svc.init(cct, driver, use_cache, run_sync_thread, null_yield, dpp);
 }
 
 int RGWRados::init_ctl(const DoutPrefixProvider *dpp)
@@ -1383,9 +1382,16 @@ int RGWRados::init_ctl(const DoutPrefixProvider *dpp)
  */
 int RGWRados::init_begin(const DoutPrefixProvider *dpp)
 {
-  int ret = init_rados();
+  int ret;
+
+  ret = driver->init_neorados(dpp);
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: failed to initialize neorados (ret=" << cpp_strerror(-ret) << ")" << dendl;
+    return ret;
+  }
+  ret = init_rados();
   if (ret < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to init rados (ret=" << cpp_strerror(-ret) << ")" << dendl;
+    ldpp_dout(dpp, 0) << "ERROR: failed to initialize librados (ret=" << cpp_strerror(-ret) << ")" << dendl;
     return ret;
   }
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 32d11a151b5e..600e54d97d50 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -22,6 +22,8 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/process.hpp>
 
+#include "common/async/blocked_completion.h"
+
 #include "common/Clock.h"
 #include "common/errno.h"
 
@@ -1124,12 +1126,25 @@ int RadosStore::get_raw_chunk_size(const DoutPrefixProvider* dpp, const rgw_raw_
   return rados->get_max_chunk_size(obj.pool, chunk_size, dpp);
 }
 
+int RadosStore::init_neorados(const DoutPrefixProvider* dpp) {
+  if (!neorados) try {
+      neorados = neorados::RADOS::make_with_cct(dpp->get_cct(), io_context,
+						ceph::async::use_blocked);
+    } catch (const boost::system::system_error& e) {
+      ldpp_dout(dpp, 0) << "ERROR: creating neorados handle failed: "
+			<< e.what() << dendl;
+      return ceph::from_error_code(e.code());
+    }
+  return 0;
+}
+
 int RadosStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp)
 {
   std::unique_ptr<ZoneGroup> zg =
     std::make_unique<RadosZoneGroup>(this, svc()->zone->get_zonegroup());
   zone = make_unique<RadosZone>(this, std::move(zg));
-  return 0;
+
+  return init_neorados(dpp);
 }
 
 int RadosStore::log_usage(const DoutPrefixProvider *dpp, map<rgw_user_bucket, RGWUsageBatch>& usage_info, optional_yield y)
@@ -3718,9 +3733,10 @@ int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y)
 
 extern "C" {
 
-void* newRadosStore(void)
+void* newRadosStore(void* io_context)
 {
-  rgw::sal::RadosStore* store = new rgw::sal::RadosStore();
+  rgw::sal::RadosStore* store = new rgw::sal::RadosStore(
+    *static_cast<boost::asio::io_context*>(io_context));
   if (store) {
     RGWRados* rados = new RGWRados();
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 7b810504457b..d5174f39bc8f 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -15,6 +15,10 @@
 
 #pragma once
 
+#include "include/neorados/RADOS.hpp"
+
+#include <boost/asio/io_context.hpp>
+
 #include "rgw_sal_store.h"
 #include "rgw_rados.h"
 #include "rgw_notify.h"
@@ -112,19 +116,22 @@ class RadosZone : public StoreZone {
 
 class RadosStore : public StoreDriver {
   private:
+    boost::asio::io_context& io_context;
     RGWRados* rados;
     RGWUserCtl* user_ctl;
     std::unique_ptr<RadosZone> zone;
+    std::optional<neorados::RADOS> neorados;
     std::string topics_oid(const std::string& tenant) const;
 
   public:
-    RadosStore()
-      : rados(nullptr) {
+    RadosStore(boost::asio::io_context& io_context)
+      : io_context(io_context), rados(nullptr) {
       }
     ~RadosStore() {
       delete rados;
     }
 
+    int init_neorados(const DoutPrefixProvider* dpp);
     virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override;
     virtual const std::string get_name() const override {
       return "rados";
@@ -240,6 +247,8 @@ class RadosStore : public StoreDriver {
 
     void setRados(RGWRados * st) { rados = st; }
     RGWRados* getRados(void) { return rados; }
+    boost::asio::io_context& get_io_context() { return io_context; }
+    neorados::RADOS& get_neorados() { return *neorados; }
 
     RGWServices* svc() { return &rados->svc; }
     const RGWServices* svc() const { return &rados->svc; }
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 5b78472dfe3b..4be0738bae2b 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -34,6 +34,7 @@
 #include "rgw_datalog.h"
 #include "rgw_metadata.h"
 #include "rgw_otp.h"
+#include "rgw_sal_rados.h"
 #include "rgw_user.h"
 #include "rgw_role.h"
 
@@ -48,10 +49,10 @@ RGWServices_Def::~RGWServices_Def()
 }
 
 int RGWServices_Def::init(CephContext *cct,
+			  rgw::sal::RadosStore* driver,
 			  bool have_cache,
                           bool raw,
 			  bool run_sync,
-			  librados::Rados* rados,
 			  optional_yield y,
                           const DoutPrefixProvider *dpp)
 {
@@ -88,7 +89,8 @@ int RGWServices_Def::init(CephContext *cct,
 
   async_processor->start();
   finisher->init();
-  bi_rados->init(zone.get(), rados, bilog_rados.get(), datalog_rados.get());
+  bi_rados->init(zone.get(), driver->getRados()->get_rados_handle(),
+		 bilog_rados.get(), datalog_rados.get());
   bilog_rados->init(bi_rados.get());
   bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(),
                     bi_rados.get(), meta.get(), meta_be_sobj.get(),
@@ -97,27 +99,29 @@ int RGWServices_Def::init(CephContext *cct,
                          sysobj.get(),
                          sysobj_cache.get(),
                          bucket_sobj.get());
-  cls->init(zone.get(), rados);
-  config_key_rados->init(rados);
-  mdlog->init(rados, zone.get(), sysobj.get(), cls.get(),
-	      async_processor.get());
+  cls->init(zone.get(), driver->getRados()->get_rados_handle());
+  config_key_rados->init(driver->getRados()->get_rados_handle());
+  mdlog->init(driver->getRados()->get_rados_handle(), zone.get(), sysobj.get(),
+	      cls.get(), async_processor.get());
   meta->init(sysobj.get(), mdlog.get(), meta_bes);
   meta_be_sobj->init(sysobj.get(), mdlog.get());
   meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get());
-  notify->init(zone.get(), rados, finisher.get());
+  notify->init(zone.get(), driver->getRados()->get_rados_handle(),
+	       finisher.get());
   otp->init(zone.get(), meta.get(), meta_be_otp.get());
-  zone->init(sysobj.get(), rados, sync_modules.get(), bucket_sync_sobj.get());
-  zone_utils->init(rados, zone.get());
+  zone->init(sysobj.get(), driver->getRados()->get_rados_handle(),
+	     sync_modules.get(), bucket_sync_sobj.get());
+  zone_utils->init(driver->getRados()->get_rados_handle(), zone.get());
   quota->init(zone.get());
   sync_modules->init(zone.get());
-  sysobj_core->core_init(rados, zone.get());
+  sysobj_core->core_init(driver->getRados()->get_rados_handle(), zone.get());
   if (have_cache) {
-    sysobj_cache->init(rados, zone.get(), notify.get());
-    sysobj->init(rados, sysobj_cache.get());
+    sysobj_cache->init(driver->getRados()->get_rados_handle(), zone.get(), notify.get());
+    sysobj->init(driver->getRados()->get_rados_handle(), sysobj_cache.get());
   } else {
-    sysobj->init(rados, sysobj_core.get());
+    sysobj->init(driver->getRados()->get_rados_handle(), sysobj_core.get());
   }
-  user_rados->init(rados, zone.get(), sysobj.get(), sysobj_cache.get(),
+  user_rados->init(driver->getRados()->get_rados_handle(), zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
   role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
 
@@ -146,7 +150,7 @@ int RGWServices_Def::init(CephContext *cct,
 
     r = datalog_rados->start(dpp, &zone->get_zone(),
 			     zone->get_zone_params(),
-			     rados);
+			     driver->getRados()->get_rados_handle());
     if (r < 0) {
       ldpp_dout(dpp, 0) << "ERROR: failed to start datalog_rados service (" << cpp_strerror(-r) << dendl;
       return r;
@@ -301,13 +305,11 @@ void RGWServices_Def::shutdown()
   has_shutdown = true;
 }
 
-int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw,
-			 bool run_sync, librados::Rados* rados,
-			 optional_yield y, const DoutPrefixProvider *dpp)
+int RGWServices::do_init(CephContext *_cct, rgw::sal::RadosStore* driver, bool have_cache, bool raw, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp)
 {
   cct = _cct;
 
-  int r = _svc.init(cct, have_cache, raw, run_sync, rados, y, dpp);
+  int r = _svc.init(cct, driver, have_cache, raw, run_sync, y, dpp);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 7c05f043a47c..9996b42e2514 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -11,6 +11,10 @@
 
 #include "rgw_common.h"
 
+namespace rgw::sal {
+class RadosStore;
+}
+
 struct RGWServices_Def;
 
 class RGWServiceInstance
@@ -108,8 +112,8 @@ struct RGWServices_Def
   RGWServices_Def();
   ~RGWServices_Def();
 
-  int init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync,
-	   librados::Rados* rados, optional_yield y,
+  int init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
+	   bool raw_storage, bool run_sync, optional_yield y,
 	   const DoutPrefixProvider *dpp);
   void shutdown();
 };
@@ -150,19 +154,19 @@ struct RGWServices
   RGWSI_Role_RADOS *role{nullptr};
   RGWAsyncRadosProcessor* async_processor;
 
-  int do_init(CephContext *cct, bool have_cache, bool raw_storage,
-	      bool run_sync, librados::Rados* rados, optional_yield y,
+  int do_init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
+	      bool raw_storage, bool run_sync, optional_yield y,
 	      const DoutPrefixProvider *dpp);
 
-  int init(CephContext *cct, bool have_cache, bool run_sync,
-	   librados::Rados* rados, optional_yield y,
-	   const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, false, run_sync, rados, y, dpp);
+  int init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
+	   bool run_sync, optional_yield y, const DoutPrefixProvider *dpp) {
+    return do_init(cct, store, have_cache, false, run_sync, y, dpp);
   }
 
-  int init_raw(CephContext *cct, bool have_cache, librados::Rados* rados,
-	       optional_yield y, const DoutPrefixProvider *dpp) {
-    return do_init(cct, have_cache, true, false, rados, y, dpp);
+  int init_raw(CephContext *cct, rgw::sal::RadosStore* store,
+	       bool have_cache, optional_yield y,
+	       const DoutPrefixProvider *dpp) {
+    return do_init(cct, store, have_cache, true, false, y, dpp);
   }
   void shutdown() {
     _svc.shutdown();
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 3d817c6039bf..5a3e5102bf7f 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -17,6 +17,8 @@ extern "C" {
 #include "auth/Crypto.h"
 #include "compressor/Compressor.h"
 
+#include "common/async/context_pool.h"
+
 #include "common/armor.h"
 #include "common/ceph_json.h"
 #include "common/config.h"
@@ -3309,6 +3311,7 @@ int main(int argc, const char **argv)
 
   auto cct = rgw_global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
 			     CODE_ENVIRONMENT_UTILITY, 0);
+  ceph::async::io_context_pool context_pool(cct->_conf->rgw_thread_pool_size);
 
   // for region -> zonegroup conversion (must happen before common_init_finish())
   if (!g_conf()->rgw_region.empty() && g_conf()->rgw_zonegroup.empty()) {
@@ -4256,13 +4259,13 @@ int main(int argc, const char **argv)
     }
 
     if (raw_storage_op) {
-      driver = DriverManager::get_raw_storage(dpp(),
-					    g_ceph_context,
-					    cfg);
+      driver = DriverManager::get_raw_storage(dpp(), g_ceph_context,
+					      cfg, context_pool);
     } else {
       driver = DriverManager::get_storage(dpp(),
 					g_ceph_context,
 					cfg,
+					context_pool,
 					false,
 					false,
 					false,
diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
index 57a1a16783a9..d37721ef4b9d 100644
--- a/src/rgw/rgw_appmain.cc
+++ b/src/rgw/rgw_appmain.cc
@@ -237,6 +237,7 @@ int rgw::AppMain::init_storage()
   DriverManager::Config cfg = DriverManager::get_config(false, g_ceph_context);
   env.driver = DriverManager::get_storage(dpp, dpp->get_cct(),
           cfg,
+	  context_pool,
           run_gc,
           run_lc,
           run_quota,
@@ -515,7 +516,7 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
       rgw_pauser->add_pauser(env.lua.background);
     }
     reloader = std::make_unique<RGWRealmReloader>(
-        env, *implicit_tenant_context, service_map_meta, rgw_pauser.get());
+      env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), context_pool);
     realm_watcher = std::make_unique<RGWRealmWatcher>(dpp, g_ceph_context,
 				  static_cast<rgw::sal::RadosStore*>(env.driver)->svc()->zone->get_realm());
     realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader);
diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h
index 49846c5d595a..9bdea60e2860 100644
--- a/src/rgw/rgw_main.h
+++ b/src/rgw/rgw_main.h
@@ -18,6 +18,9 @@
 #include <vector>
 #include <map>
 #include <string>
+
+#include "common/async/context_pool.h"
+
 #include "rgw_common.h"
 #include "rgw_rest.h"
 #include "rgw_frontend.h"
@@ -81,7 +84,8 @@ class AppMain {
   SiteConfig site;
   const DoutPrefixProvider* dpp;
   RGWProcessEnv env;
-
+  ceph::async::io_context_pool context_pool{
+    dpp->get_cct()->_conf->rgw_thread_pool_size};
 public:
   AppMain(const DoutPrefixProvider* dpp);
   ~AppMain();
diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc
index 0470b1a6e6fc..e3a816865f80 100644
--- a/src/rgw/rgw_object_expirer.cc
+++ b/src/rgw/rgw_object_expirer.cc
@@ -9,6 +9,8 @@
 
 #include "auth/Crypto.h"
 
+#include "common/async/context_pool.h"
+
 #include "common/armor.h"
 #include "common/ceph_json.h"
 #include "common/config.h"
@@ -81,12 +83,13 @@ int main(const int argc, const char **argv)
   }
 
   common_init_finish(g_ceph_context);
+  ceph::async::io_context_pool context_pool{cct->_conf->rgw_thread_pool_size};
 
   const DoutPrefix dp(cct.get(), dout_subsys, "rgw object expirer: ");
   DriverManager::Config cfg;
   cfg.store_name = "rados";
   cfg.filter_name = "none";
-  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, false, false, false, false, false, false, null_yield);
+  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, context_pool, false, false, false, false, false, false, null_yield);
   if (!driver) {
     std::cerr << "couldn't init storage provider" << std::endl;
     return EIO;
diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc
index b3a511becb68..26dffa99cf4f 100644
--- a/src/rgw/rgw_realm_reloader.cc
+++ b/src/rgw/rgw_realm_reloader.cc
@@ -31,11 +31,13 @@ static constexpr bool USE_SAFE_TIMER_CALLBACKS = false;
 RGWRealmReloader::RGWRealmReloader(RGWProcessEnv& env,
                                    const rgw::auth::ImplicitTenants& implicit_tenants,
                                    std::map<std::string, std::string>& service_map_meta,
-                                   Pauser* frontends)
+                                   Pauser* frontends,
+				   boost::asio::io_context& io_context)
   : env(env),
     implicit_tenants(implicit_tenants),
     service_map_meta(service_map_meta),
     frontends(frontends),
+    io_context(io_context),
     timer(env.driver->ctx(), mutex, USE_SAFE_TIMER_CALLBACKS),
     mutex(ceph::make_mutex("RGWRealmReloader")),
     reload_scheduled(nullptr)
@@ -118,7 +120,7 @@ void RGWRealmReloader::reload()
       DriverManager::Config cfg;
       cfg.store_name = "rados";
       cfg.filter_name = "none";
-      env.driver = DriverManager::get_storage(&dp, cct, cfg,
+      env.driver = DriverManager::get_storage(&dp, cct, cfg, io_context,
           cct->_conf->rgw_enable_gc_threads,
           cct->_conf->rgw_enable_lc_threads,
           cct->_conf->rgw_enable_quota_threads,
diff --git a/src/rgw/rgw_realm_reloader.h b/src/rgw/rgw_realm_reloader.h
index 25082a2e490d..6cf969da55ad 100644
--- a/src/rgw/rgw_realm_reloader.h
+++ b/src/rgw/rgw_realm_reloader.h
@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include <boost/asio/io_context.hpp>
+
 #include "rgw_realm_watcher.h"
 #include "common/Cond.h"
 #include "rgw_sal_fwd.h"
@@ -36,7 +38,7 @@ class RGWRealmReloader : public RGWRealmWatcher::Watcher {
   RGWRealmReloader(RGWProcessEnv& env,
                    const rgw::auth::ImplicitTenants& implicit_tenants,
                    std::map<std::string, std::string>& service_map_meta,
-                   Pauser* frontends);
+                   Pauser* frontends, boost::asio::io_context& io_context);
   ~RGWRealmReloader() override;
 
   /// respond to realm notifications by scheduling a reload()
@@ -52,6 +54,7 @@ class RGWRealmReloader : public RGWRealmWatcher::Watcher {
   const rgw::auth::ImplicitTenants& implicit_tenants;
   std::map<std::string, std::string>& service_map_meta;
   Pauser *const frontends;
+  boost::asio::io_context& io_context;
 
   /// reload() takes a significant amount of time, so we don't want to run
   /// it in the handle_notify() thread. we choose a timer thread instead of a
diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc
index 6c0212dae817..a70813435bc0 100644
--- a/src/rgw/rgw_sal.cc
+++ b/src/rgw/rgw_sal.cc
@@ -46,7 +46,7 @@
 #define dout_subsys ceph_subsys_rgw
 
 extern "C" {
-extern rgw::sal::Driver* newRadosStore(void);
+extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context);
 #ifdef WITH_RADOSGW_DBSTORE
 extern rgw::sal::Driver* newDBStore(CephContext *cct);
 #endif
@@ -103,6 +103,7 @@ RGWObjState::RGWObjState(const RGWObjState& rhs) : obj (rhs.obj) {
 rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* dpp,
 						     CephContext* cct,
 						     const Config& cfg,
+						     boost::asio::io_context& io_context,
 						     bool use_gc_thread,
 						     bool use_lc_thread,
 						     bool quota_threads,
@@ -115,7 +116,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
   rgw::sal::Driver* driver{nullptr};
 
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore();
+    driver = newRadosStore(&io_context);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     if ((*rados).set_use_cache(use_cache)
@@ -141,7 +142,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
     }
   }
   else if (cfg.store_name.compare("d3n") == 0) {
-    driver = new rgw::sal::RadosStore();
+    driver = new rgw::sal::RadosStore(io_context);
     RGWRados* rados = new D3nRGWDataCache<RGWRados>;
     dynamic_cast<rgw::sal::RadosStore*>(driver)->setRados(rados);
     rados->set_store(static_cast<rgw::sal::RadosStore* >(driver));
@@ -261,11 +262,11 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
   return driver;
 }
 
-rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg)
+rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg, boost::asio::io_context& io_context)
 {
   rgw::sal::Driver* driver = nullptr;
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore();
+    driver = newRadosStore(&io_context);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     rados->set_context(cct);
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 943905af17a4..fbff4f60645c 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1536,6 +1536,7 @@ class DriverManager {
   static rgw::sal::Driver* get_storage(const DoutPrefixProvider* dpp,
 				      CephContext* cct,
 				      const Config& cfg,
+				      boost::asio::io_context& io_context,
 				      bool use_gc_thread,
 				      bool use_lc_thread,
 				      bool quota_threads,
@@ -1544,7 +1545,7 @@ class DriverManager {
 				      bool run_notification_thread, optional_yield y,
 				      bool use_cache = true,
 				      bool use_gc = true) {
-    rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, use_gc_thread,
+    rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context, use_gc_thread,
 						   use_lc_thread,
 						   quota_threads,
 						   run_sync_thread,
@@ -1555,14 +1556,16 @@ class DriverManager {
   }
   /** Get a stripped down driver by service name */
   static rgw::sal::Driver* get_raw_storage(const DoutPrefixProvider* dpp,
-					  CephContext* cct, const Config& cfg) {
-    rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg);
+					  CephContext* cct, const Config& cfg,
+					  boost::asio::io_context& io_context) {
+    rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg, io_context);
     return driver;
   }
   /** Initialize a new full Driver */
   static rgw::sal::Driver* init_storage_provider(const DoutPrefixProvider* dpp,
 						CephContext* cct,
 						const Config& cfg,
+						boost::asio::io_context& io_context,
 						bool use_gc_thread,
 						bool use_lc_thread,
 						bool quota_threads,
@@ -1574,7 +1577,8 @@ class DriverManager {
   /** Initialize a new raw Driver */
   static rgw::sal::Driver* init_raw_storage_provider(const DoutPrefixProvider* dpp,
 						    CephContext* cct,
-						    const Config& cfg);
+						    const Config& cfg,
+						    boost::asio::io_context& io_context);
   /** Close a Driver when it's no longer needed */
   static void close_storage(rgw::sal::Driver* driver);
 
diff --git a/src/test/rgw/rgw_cr_test.cc b/src/test/rgw/rgw_cr_test.cc
index 2c250b390a98..db59dd1841b3 100644
--- a/src/test/rgw/rgw_cr_test.cc
+++ b/src/test/rgw/rgw_cr_test.cc
@@ -10,7 +10,7 @@
 
 #include "include/rados/librados.hpp"
 
-#include "rgw_tools.h"
+#include "common/async/context_pool.h"
 
 #include "common/common_init.h"
 #include "common/config.h"
@@ -321,12 +321,14 @@ int main(int argc, const char **argv)
   common_init_finish(g_ceph_context);
 
 
+  ceph::async::io_context_pool context_pool{cct->_conf->rgw_thread_pool_size};
   DriverManager::Config cfg = DriverManager::get_config(true, g_ceph_context);
 
   store = static_cast<rgw::sal::RadosStore*>(
     DriverManager::get_storage(dpp(),
 			      g_ceph_context,
 			      cfg,
+			      context_pool,
 			      false,
 			      false,
 			      false,
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index b1483f26e0db..00f7f24e00f8 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -12,6 +12,8 @@
 #include "rgw_auth.h"
 #include "rgw_auth_registry.h"
 
+#include <boost/asio/io_context.hpp>
+
 #define dout_subsys ceph_subsys_rgw
 
 #define METADATA_LENGTH 22
@@ -33,6 +35,7 @@ class StoreObject : public rgw::sal::StoreObject {
 };
 
 class Environment : public ::testing::Environment {
+  boost::asio::io_context ioc;
   public:
     Environment() {}
     
@@ -61,6 +64,7 @@ class Environment : public ::testing::Environment {
       
       driver = DriverManager::get_storage(dpp, dpp->get_cct(),
               cfg,
+              ioc,
               false,
               false,
               false,
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index 9c5f8d9caa3f..b7cd66f6ea5a 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -20,6 +20,7 @@
 #include <gtest/gtest.h>
 
 #include "include/stringify.h"
+#include "common/async/context_pool.h"
 #include "common/code_environment.h"
 #include "common/ceph_context.h"
 #include "global/global_init.h"
@@ -912,7 +913,8 @@ TEST_F(IPPolicyTest, IPEnvironment) {
   RGWProcessEnv penv;
   // Unfortunately RGWCivetWeb is too tightly tied to civetweb to test RGWCivetWeb::init_env.
   RGWEnv rgw_env;
-  rgw::sal::RadosStore store;
+  ceph::async::io_context_pool context_pool(cct->_conf->rgw_thread_pool_size); \
+  rgw::sal::RadosStore store(context_pool);
   std::unique_ptr<rgw::sal::User> user = store.get_user(rgw_user());
   rgw_env.set("REMOTE_ADDR", "192.168.1.1");
   rgw_env.set("HTTP_HOST", "1.2.3.4");
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 0485e71ede37..1ae7976c7944 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -1,4 +1,5 @@
 #include <gtest/gtest.h>
+#include "common/async/context_pool.h"
 #include "common/ceph_context.h"
 #include "rgw_common.h"
 #include "rgw_auth_registry.h"
@@ -159,8 +160,10 @@ CctCleaner cleaner(g_cct);
 
 tracing::Tracer tracer;
 
-#define MAKE_STORE auto store = std::unique_ptr<sal::RadosStore>(new sal::RadosStore); \
-                        store->setRados(new RGWRados);
+#define MAKE_STORE \
+  ceph::async::io_context_pool context_pool(g_cct->_conf->rgw_thread_pool_size); \
+  auto store = std::unique_ptr<sal::RadosStore>(new sal::RadosStore(context_pool)); \
+  store->setRados(new RGWRados);
 
 #define DEFINE_REQ_STATE RGWProcessEnv pe; \
   MAKE_STORE; \

From 31e4b8de9631eef2b4b5d9865725b0520637d603 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Thu, 25 Jan 2024 10:22:13 +1300
Subject: [PATCH 1487/2492] cephadm: Add nvmeof to autotuner calculation

Add nvmeof to the list of daemons when calculating the
memory to use for OSDs.

Fixes: https://tracker.ceph.com/issues/64020

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 src/pybind/mgr/cephadm/autotune.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pybind/mgr/cephadm/autotune.py b/src/pybind/mgr/cephadm/autotune.py
index 51c931cbad63..72ebcd66064c 100644
--- a/src/pybind/mgr/cephadm/autotune.py
+++ b/src/pybind/mgr/cephadm/autotune.py
@@ -15,6 +15,7 @@ class MemoryAutotuner(object):
         'crash': 128 * 1048576,
         'keepalived': 128 * 1048576,
         'haproxy': 128 * 1048576,
+        'nvmeof': 4096 * 1048576,
     }
     default_size = 1024 * 1048576
 

From 2d4bd1604246436136e11d14bc447c31a1e26a97 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Thu, 25 Jan 2024 10:22:37 +1300
Subject: [PATCH 1488/2492] cephadm: add testcase to autotuner

Adds a testcase for the presence of the nvmeof daemon

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 src/pybind/mgr/cephadm/tests/test_autotune.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/pybind/mgr/cephadm/tests/test_autotune.py b/src/pybind/mgr/cephadm/tests/test_autotune.py
index 524da9c0008c..7994c390a7e8 100644
--- a/src/pybind/mgr/cephadm/tests/test_autotune.py
+++ b/src/pybind/mgr/cephadm/tests/test_autotune.py
@@ -46,6 +46,17 @@
             ],
             {},
             62 * 1024 * 1024 * 1024,
+        ),
+        (
+            128 * 1024 * 1024 * 1024,
+            [
+                DaemonDescription('mgr', 'a', 'host1'),
+                DaemonDescription('osd', '1', 'host1'),
+                DaemonDescription('osd', '2', 'host1'),
+                DaemonDescription('nvmeof', 'a', 'host1'),
+            ],
+            {},
+            60 * 1024 * 1024 * 1024,
         )
     ])
 def test_autotune(total, daemons, config, result):

From dc262061fed890f1c69a5569b040cddbbeb8c741 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Fri, 14 Apr 2023 15:45:04 -0400
Subject: [PATCH 1489/2492] rgw: Asio frontend shares `io_context` with the
 rest of RGW

Pull the `io_context` and threads out of `AsioFrontend`, pass in a
reference to `io_context_pool` so it can be shut down at `AsioFrontend::join`

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
 src/rgw/rgw_appmain.cc       | 48 ++++++++++++++++++--------
 src/rgw/rgw_asio_frontend.cc | 66 ++++++++++--------------------------
 src/rgw/rgw_asio_frontend.h  |  6 +++-
 src/rgw/rgw_main.h           |  4 +--
 4 files changed, 58 insertions(+), 66 deletions(-)

diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
index d37721ef4b9d..096b7d006e1e 100644
--- a/src/rgw/rgw_appmain.cc
+++ b/src/rgw/rgw_appmain.cc
@@ -203,6 +203,17 @@ void rgw::AppMain::init_numa()
   }
 } /* init_numa */
 
+void rgw::AppMain::need_context_pool() {
+  if (!context_pool) {
+    context_pool.emplace(
+      dpp->get_cct()->_conf->rgw_thread_pool_size,
+      [] {
+	// request warnings on synchronous librados calls in this thread
+	is_asio_thread = true;
+      });
+  }
+}
+
 int rgw::AppMain::init_storage()
 {
   auto config_store_type = g_conf().get_val<std::string>("rgw_config_store");
@@ -234,10 +245,11 @@ int rgw::AppMain::init_storage()
     (g_conf()->rgw_run_sync_thread &&
       ((!nfs) || (nfs && g_conf()->rgw_nfs_run_sync_thread)));
 
+  need_context_pool();
   DriverManager::Config cfg = DriverManager::get_config(false, g_ceph_context);
   env.driver = DriverManager::get_storage(dpp, dpp->get_cct(),
           cfg,
-	  context_pool,
+	  *context_pool,
           run_gc,
           run_lc,
           run_quota,
@@ -457,7 +469,8 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
       fe = new RGWLoadGenFrontend(env, config);
     }
     else if (framework == "beast") {
-      fe = new RGWAsioFrontend(env, config, *sched_ctx);
+      need_context_pool();
+      fe = new RGWAsioFrontend(env, config, *sched_ctx, *context_pool);
     }
     else if (framework == "rgw-nfs") {
       fe = new RGWLibFrontend(env, config);
@@ -515,8 +528,9 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
     if (env.lua.background) {
       rgw_pauser->add_pauser(env.lua.background);
     }
+    need_context_pool();
     reloader = std::make_unique<RGWRealmReloader>(
-      env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), context_pool);
+      env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), *context_pool);
     realm_watcher = std::make_unique<RGWRealmWatcher>(dpp, g_ceph_context,
 				  static_cast<rgw::sal::RadosStore*>(env.driver)->svc()->zone->get_realm());
     realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader);
@@ -588,28 +602,34 @@ void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
     fe->stop();
   }
 
-  for (auto& fe : fes) {
-    fe->join();
-    delete fe;
-  }
-
-  for (auto& fec : fe_configs) {
-    delete fec;
-  }
-
   ldh.reset(nullptr); // deletes
-  finalize_async_signals(); // callback
   rgw_log_usage_finalize();
-  
+
   delete olog;
 
   if (lua_background) {
     lua_background->shutdown();
   }
 
+  // Do this before closing storage so requests don't try to call into
+  // closed storage.
+  context_pool->finish();
+
   cfgstore.reset(); // deletes
   DriverManager::close_storage(env.driver);
 
+  // Fe can't be deleted until nobody's exeucting `io_context::run`
+  for (auto& fe : fes) {
+    fe->join();
+    delete fe;
+  }
+
+  for (auto& fec : fe_configs) {
+    delete fec;
+  }
+
+  finalize_async_signals(); // callback
+
   rgw_tools_cleanup();
   rgw_shutdown_resolver();
   rgw_http_client_cleanup();
diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index cdc7f840da7c..42ad12061488 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -3,7 +3,6 @@
 
 #include <atomic>
 #include <ctime>
-#include <thread>
 #include <vector>
 
 #include <boost/asio/error.hpp>
@@ -401,8 +400,9 @@ class ConnectionList {
 namespace dmc = rgw::dmclock;
 class AsioFrontend {
   RGWProcessEnv& env;
+  boost::intrusive_ptr<CephContext> cct{env.driver->ctx()};
   RGWFrontendConfig* conf;
-  boost::asio::io_context context;
+  boost::asio::io_context& context;
   std::string uri_prefix;
   ceph::timespan request_timeout = std::chrono::milliseconds(REQUEST_TIMEOUT);
   size_t header_limit = 16384;
@@ -432,22 +432,19 @@ class AsioFrontend {
 
   ConnectionList connections;
 
-  // work guard to keep run() threads busy while listeners are paused
-  using Executor = boost::asio::io_context::executor_type;
-  std::optional<boost::asio::executor_work_guard<Executor>> work;
-
-  std::vector<std::thread> threads;
   std::atomic<bool> going_down{false};
 
-  CephContext* ctx() const { return env.driver->ctx(); }
+  CephContext* ctx() const { return cct.get(); }
   std::optional<dmc::ClientCounters> client_counters;
   std::unique_ptr<dmc::ClientConfig> client_config;
   void accept(Listener& listener, boost::system::error_code ec);
 
  public:
   AsioFrontend(RGWProcessEnv& env, RGWFrontendConfig* conf,
-	       dmc::SchedulerCtx& sched_ctx)
-    : env(env), conf(conf), pause_mutex(context.get_executor())
+	       dmc::SchedulerCtx& sched_ctx,
+	       boost::asio::io_context& context)
+    : env(env), conf(conf), context(context),
+      pause_mutex(context.get_executor())
   {
     auto sched_t = dmc::get_scheduler_t(ctx());
     switch(sched_t){
@@ -469,7 +466,9 @@ class AsioFrontend {
   }
 
   int init();
-  int run();
+  int run() {
+    return 0;
+  }
   void stop();
   void join();
   void pause();
@@ -487,7 +486,7 @@ unsigned short parse_port(const char *input, boost::system::error_code& ec)
   }
   return port;
 }
-	
+
 tcp::endpoint parse_endpoint(boost::asio::string_view input,
                              unsigned short default_port,
                              boost::system::error_code& ec)
@@ -1067,32 +1066,6 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
   }
 }
 
-int AsioFrontend::run()
-{
-  ceph_assert(!is_asio_thread);
-
-  auto cct = ctx();
-  const int thread_count = cct->_conf->rgw_thread_pool_size;
-  threads.reserve(thread_count);
-
-  ldout(cct, 4) << "frontend spawning " << thread_count << " threads" << dendl;
-
-  // the worker threads call io_context::run(), which will return when there's
-  // no work left. hold a work guard to keep these threads going until join()
-  work.emplace(boost::asio::make_work_guard(context));
-
-  for (int i = 0; i < thread_count; i++) {
-    threads.emplace_back([this]() noexcept {
-      // request warnings on synchronous librados calls in this thread
-      is_asio_thread = true;
-      // Have uncaught exceptions kill the process and give a
-      // stacktrace, not be swallowed.
-      context.run();
-    });
-  }
-  return 0;
-}
-
 void AsioFrontend::stop()
 {
   ceph_assert(!is_asio_thread);
@@ -1116,13 +1089,6 @@ void AsioFrontend::join()
   if (!going_down) {
     stop();
   }
-  work.reset();
-
-  ldout(ctx(), 4) << "frontend joining threads..." << dendl;
-  for (auto& thread : threads) {
-    thread.join();
-  }
-  ldout(ctx(), 4) << "frontend done" << dendl;
 }
 
 void AsioFrontend::pause()
@@ -1166,14 +1132,16 @@ void AsioFrontend::unpause()
 class RGWAsioFrontend::Impl : public AsioFrontend {
  public:
   Impl(RGWProcessEnv& env, RGWFrontendConfig* conf,
-       rgw::dmclock::SchedulerCtx& sched_ctx)
-    : AsioFrontend(env, conf, sched_ctx) {}
+       rgw::dmclock::SchedulerCtx& sched_ctx,
+       boost::asio::io_context& context)
+    : AsioFrontend(env, conf, sched_ctx, context) {}
 };
 
 RGWAsioFrontend::RGWAsioFrontend(RGWProcessEnv& env,
                                  RGWFrontendConfig* conf,
-				 rgw::dmclock::SchedulerCtx& sched_ctx)
-  : impl(new Impl(env, conf, sched_ctx))
+				 rgw::dmclock::SchedulerCtx& sched_ctx,
+				 boost::asio::io_context& context)
+  : impl(new Impl(env, conf, sched_ctx, context))
 {
 }
 
diff --git a/src/rgw/rgw_asio_frontend.h b/src/rgw/rgw_asio_frontend.h
index 2de6f337a9fb..8f642bb526f5 100644
--- a/src/rgw/rgw_asio_frontend.h
+++ b/src/rgw/rgw_asio_frontend.h
@@ -4,6 +4,9 @@
 #pragma once
 
 #include <memory>
+
+#include <boost/asio/io_context.hpp>
+
 #include "rgw_frontend.h"
 #define REQUEST_TIMEOUT 65000
 
@@ -12,7 +15,8 @@ class RGWAsioFrontend : public RGWFrontend {
   std::unique_ptr<Impl> impl;
 public:
   RGWAsioFrontend(RGWProcessEnv& env, RGWFrontendConfig* conf,
-		  rgw::dmclock::SchedulerCtx& sched_ctx);
+		  rgw::dmclock::SchedulerCtx& sched_ctx,
+		  boost::asio::io_context& io_context);
   ~RGWAsioFrontend() override;
 
   int init() override;
diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h
index 9bdea60e2860..caa6a0822828 100644
--- a/src/rgw/rgw_main.h
+++ b/src/rgw/rgw_main.h
@@ -84,8 +84,8 @@ class AppMain {
   SiteConfig site;
   const DoutPrefixProvider* dpp;
   RGWProcessEnv env;
-  ceph::async::io_context_pool context_pool{
-    dpp->get_cct()->_conf->rgw_thread_pool_size};
+  void need_context_pool();
+  std::optional<ceph::async::io_context_pool> context_pool;
 public:
   AppMain(const DoutPrefixProvider* dpp);
   ~AppMain();

From 0e584be9029dd4520e5c05a4d198bf1f9eeea40e Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Thu, 9 Nov 2023 15:23:53 -0500
Subject: [PATCH 1490/2492] rgw: Thread site config into RADOS driver

Make this available so multisite sync doesn't have to rely on the zone
service.

Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc |  5 ++--
 src/rgw/driver/rados/rgw_sal_rados.h  |  7 +++--
 src/rgw/driver/rados/rgw_zone.cc      | 13 ++++++--
 src/rgw/driver/rados/rgw_zone.h       | 13 +++++++-
 src/rgw/rgw_admin.cc                  | 19 ++++++++++--
 src/rgw/rgw_appmain.cc                |  1 +
 src/rgw/rgw_object_expirer.cc         | 18 ++++++++++-
 src/rgw/rgw_realm_reloader.cc         |  1 +
 src/rgw/rgw_sal.cc                    | 14 +++++----
 src/rgw/rgw_sal.h                     | 16 +++++++---
 src/test/rgw/rgw_cr_test.cc           | 16 ++++++++++
 src/test/rgw/test_d4n_filter.cc       | 14 +++++++--
 src/test/rgw/test_rgw_common.h        |  2 ++
 src/test/rgw/test_rgw_iam_policy.cc   |  8 ++---
 src/test/rgw/test_rgw_lua.cc          | 43 +++++++++++++++++++--------
 15 files changed, 153 insertions(+), 37 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 600e54d97d50..50c49d24fc34 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -3733,10 +3733,11 @@ int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y)
 
 extern "C" {
 
-void* newRadosStore(void* io_context)
+void* newRadosStore(void* io_context, const void* site_config)
 {
   rgw::sal::RadosStore* store = new rgw::sal::RadosStore(
-    *static_cast<boost::asio::io_context*>(io_context));
+    *static_cast<boost::asio::io_context*>(io_context),
+    *static_cast<const rgw::SiteConfig*>(site_config));
   if (store) {
     RGWRados* rados = new RGWRados();
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index d5174f39bc8f..d4bd19a55fe1 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -117,6 +117,7 @@ class RadosZone : public StoreZone {
 class RadosStore : public StoreDriver {
   private:
     boost::asio::io_context& io_context;
+    const rgw::SiteConfig& site_config;
     RGWRados* rados;
     RGWUserCtl* user_ctl;
     std::unique_ptr<RadosZone> zone;
@@ -124,8 +125,9 @@ class RadosStore : public StoreDriver {
     std::string topics_oid(const std::string& tenant) const;
 
   public:
-    RadosStore(boost::asio::io_context& io_context)
-      : io_context(io_context), rados(nullptr) {
+    RadosStore(boost::asio::io_context& io_context,
+	       const rgw::SiteConfig& site_config)
+      : io_context(io_context), site_config(site_config), rados(nullptr) {
       }
     ~RadosStore() {
       delete rados;
@@ -248,6 +250,7 @@ class RadosStore : public StoreDriver {
     void setRados(RGWRados * st) { rados = st; }
     RGWRados* getRados(void) { return rados; }
     boost::asio::io_context& get_io_context() { return io_context; }
+    const rgw::SiteConfig& get_siteconfig() { return site_config; }
     neorados::RADOS& get_neorados() { return *neorados; }
 
     RGWServices* svc() { return &rados->svc; }
diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index e6cc54d69d89..c8ebb79f542f 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -1165,7 +1165,7 @@ static int read_or_create_default_zonegroup(const DoutPrefixProvider* dpp,
 }
 
 int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y,
-                     sal::ConfigStore* cfgstore)
+                     sal::ConfigStore* cfgstore, bool force_local_zonegroup)
 {
   // clear existing configuration
   zone = nullptr;
@@ -1219,7 +1219,7 @@ int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y,
     }
   }
 
-  if (realm) {
+  if (realm && !force_local_zonegroup) {
     // try to load the realm's period
     r = load_period_zonegroup(dpp, y, cfgstore, *realm, zone_params.id);
   } else {
@@ -1230,6 +1230,15 @@ int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y,
   return r;
 }
 
+std::unique_ptr<SiteConfig> SiteConfig::make_fake() {
+  auto fake = std::make_unique<SiteConfig>();
+  fake->local_zonegroup.emplace();
+  fake->local_zonegroup->zones.emplace(""s, RGWZone{});
+  fake->zonegroup = &*fake->local_zonegroup;
+  fake->zone = &fake->zonegroup->zones.begin()->second;
+  return fake;
+}
+
 int SiteConfig::load_period_zonegroup(const DoutPrefixProvider* dpp,
                                       optional_yield y,
                                       sal::ConfigStore* cfgstore,
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 446f1efda91a..f0dccdbc4e94 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -973,7 +973,17 @@ class SiteConfig {
   /// Load or reload the multisite configuration from storage. This is not
   /// thread-safe, so requires careful coordination with the RGWRealmReloader.
   int load(const DoutPrefixProvider* dpp, optional_yield y,
-           sal::ConfigStore* cfgstore);
+           sal::ConfigStore* cfgstore, bool force_local_zonegroup = false);
+
+  /// Create a fake site config to be used by tests and similar, just
+  /// to have a site config.
+  ///
+  /// \warning Do not use this anywhere but unittests where we need to
+  /// bring up parts of RGW that require a SiteConfig exist, but need
+  /// to run without a cluster.
+  static std::unique_ptr<SiteConfig> make_fake();
+
+  virtual ~SiteConfig() = default;
 
  private:
   int load_period_zonegroup(const DoutPrefixProvider* dpp, optional_yield y,
@@ -991,4 +1001,5 @@ class SiteConfig {
   const RGWZone* zone = nullptr;
 };
 
+
 } // namespace rgw
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 5a3e5102bf7f..8265852973f9 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -87,7 +87,6 @@ using namespace std;
 static rgw::sal::Driver* driver = NULL;
 static constexpr auto dout_subsys = ceph_subsys_rgw;
 
-
 static const DoutPrefixProvider* dpp() {
   struct GlobalPrefix : public DoutPrefixProvider {
     CephContext *get_cct() const override { return dout_context; }
@@ -4130,6 +4129,11 @@ int main(int argc, const char **argv)
     // not a raw op if 'period pull' needs to read zone/period configuration
     bool raw_period_pull = opt_cmd == OPT::PERIOD_PULL && !url.empty();
 
+    // Before a period commit or pull, our zonegroup may not be in the
+    // period, causing `load_period_zonegroup` to fail.
+    bool localzonegroup_op = ((opt_cmd == OPT::PERIOD_UPDATE && commit) ||
+			      (opt_cmd == OPT::PERIOD_PULL && url.empty()));
+
     std::set<OPT> raw_storage_ops_list = {OPT::ZONEGROUP_ADD, OPT::ZONEGROUP_CREATE,
 			 OPT::ZONEGROUP_DELETE,
 			 OPT::ZONEGROUP_GET, OPT::ZONEGROUP_LIST,
@@ -4258,14 +4262,25 @@ int main(int argc, const char **argv)
       return EIO;
     }
 
+    std::unique_ptr<rgw::SiteConfig> site;
+
     if (raw_storage_op) {
+      site = rgw::SiteConfig::make_fake();
       driver = DriverManager::get_raw_storage(dpp(), g_ceph_context,
-					      cfg, context_pool);
+					      cfg, context_pool, *site);
     } else {
+      site = std::make_unique<rgw::SiteConfig>();
+      auto r = site->load(dpp(), null_yield, cfgstore.get(), localzonegroup_op);
+      if (r < 0) {
+	std::cerr << "Unable to initialize site config." << std::endl;
+	exit(1);
+      }
+
       driver = DriverManager::get_storage(dpp(),
 					g_ceph_context,
 					cfg,
 					context_pool,
+					*site,
 					false,
 					false,
 					false,
diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
index 096b7d006e1e..b4aa1019a739 100644
--- a/src/rgw/rgw_appmain.cc
+++ b/src/rgw/rgw_appmain.cc
@@ -250,6 +250,7 @@ int rgw::AppMain::init_storage()
   env.driver = DriverManager::get_storage(dpp, dpp->get_cct(),
           cfg,
 	  *context_pool,
+	  site,
           run_gc,
           run_lc,
           run_quota,
diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc
index e3a816865f80..58723503c5e9 100644
--- a/src/rgw/rgw_object_expirer.cc
+++ b/src/rgw/rgw_object_expirer.cc
@@ -31,6 +31,8 @@
 #include "rgw_formats.h"
 #include "rgw_usage.h"
 #include "rgw_object_expirer_core.h"
+#include "driver/rados/rgw_zone.h"
+#include "rgw_sal_config.h"
 
 #define dout_subsys ceph_subsys_rgw
 
@@ -89,7 +91,21 @@ int main(const int argc, const char **argv)
   DriverManager::Config cfg;
   cfg.store_name = "rados";
   cfg.filter_name = "none";
-  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, context_pool, false, false, false, false, false, false, null_yield);
+  std::unique_ptr<rgw::sal::ConfigStore> cfgstore;
+  auto config_store_type = g_conf().get_val<std::string>("rgw_config_store");
+  cfgstore = DriverManager::create_config_store(&dp, config_store_type);
+  if (!cfgstore) {
+    std::cerr << "Unable to initialize config store." << std::endl;
+    exit(1);
+  }
+  rgw::SiteConfig site;
+  auto r = site.load(&dp, null_yield, cfgstore.get());
+  if (r < 0) {
+    std::cerr << "Unable to initialize config store." << std::endl;
+    exit(1);
+  }
+
+  driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, context_pool, site, false, false, false, false, false, false, null_yield);
   if (!driver) {
     std::cerr << "couldn't init storage provider" << std::endl;
     return EIO;
diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc
index 26dffa99cf4f..d425793a3fa4 100644
--- a/src/rgw/rgw_realm_reloader.cc
+++ b/src/rgw/rgw_realm_reloader.cc
@@ -121,6 +121,7 @@ void RGWRealmReloader::reload()
       cfg.store_name = "rados";
       cfg.filter_name = "none";
       env.driver = DriverManager::get_storage(&dp, cct, cfg, io_context,
+	  *env.site,
           cct->_conf->rgw_enable_gc_threads,
           cct->_conf->rgw_enable_lc_threads,
           cct->_conf->rgw_enable_quota_threads,
diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc
index a70813435bc0..5e4603b81162 100644
--- a/src/rgw/rgw_sal.cc
+++ b/src/rgw/rgw_sal.cc
@@ -46,7 +46,8 @@
 #define dout_subsys ceph_subsys_rgw
 
 extern "C" {
-extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context);
+extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context,
+				       const rgw::SiteConfig* site_config);
 #ifdef WITH_RADOSGW_DBSTORE
 extern rgw::sal::Driver* newDBStore(CephContext *cct);
 #endif
@@ -104,6 +105,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
 						     CephContext* cct,
 						     const Config& cfg,
 						     boost::asio::io_context& io_context,
+						     const rgw::SiteConfig& site_config,
 						     bool use_gc_thread,
 						     bool use_lc_thread,
 						     bool quota_threads,
@@ -116,7 +118,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
   rgw::sal::Driver* driver{nullptr};
 
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore(&io_context);
+    driver = newRadosStore(&io_context, &site_config);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     if ((*rados).set_use_cache(use_cache)
@@ -142,7 +144,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
     }
   }
   else if (cfg.store_name.compare("d3n") == 0) {
-    driver = new rgw::sal::RadosStore(io_context);
+    driver = new rgw::sal::RadosStore(io_context, site_config);
     RGWRados* rados = new D3nRGWDataCache<RGWRados>;
     dynamic_cast<rgw::sal::RadosStore*>(driver)->setRados(rados);
     rados->set_store(static_cast<rgw::sal::RadosStore* >(driver));
@@ -262,11 +264,13 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
   return driver;
 }
 
-rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg, boost::asio::io_context& io_context)
+rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct,
+							   const Config& cfg, boost::asio::io_context& io_context,
+							   const rgw::SiteConfig& site_config)
 {
   rgw::sal::Driver* driver = nullptr;
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore(&io_context);
+    driver = newRadosStore(&io_context, &site_config);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     rados->set_context(cct);
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index fbff4f60645c..6ee02be0b67f 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1537,6 +1537,7 @@ class DriverManager {
 				      CephContext* cct,
 				      const Config& cfg,
 				      boost::asio::io_context& io_context,
+				      const rgw::SiteConfig& site_config,
 				      bool use_gc_thread,
 				      bool use_lc_thread,
 				      bool quota_threads,
@@ -1545,7 +1546,9 @@ class DriverManager {
 				      bool run_notification_thread, optional_yield y,
 				      bool use_cache = true,
 				      bool use_gc = true) {
-    rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context, use_gc_thread,
+    rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context,
+						   site_config,
+						   use_gc_thread,
 						   use_lc_thread,
 						   quota_threads,
 						   run_sync_thread,
@@ -1557,8 +1560,11 @@ class DriverManager {
   /** Get a stripped down driver by service name */
   static rgw::sal::Driver* get_raw_storage(const DoutPrefixProvider* dpp,
 					  CephContext* cct, const Config& cfg,
-					  boost::asio::io_context& io_context) {
-    rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg, io_context);
+					  boost::asio::io_context& io_context,
+					  const rgw::SiteConfig& site_config) {
+    rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg,
+							 io_context,
+							 site_config);
     return driver;
   }
   /** Initialize a new full Driver */
@@ -1566,6 +1572,7 @@ class DriverManager {
 						CephContext* cct,
 						const Config& cfg,
 						boost::asio::io_context& io_context,
+						const rgw::SiteConfig& site_config,
 						bool use_gc_thread,
 						bool use_lc_thread,
 						bool quota_threads,
@@ -1578,7 +1585,8 @@ class DriverManager {
   static rgw::sal::Driver* init_raw_storage_provider(const DoutPrefixProvider* dpp,
 						    CephContext* cct,
 						    const Config& cfg,
-						    boost::asio::io_context& io_context);
+						    boost::asio::io_context& io_context,
+						    const rgw::SiteConfig& site_config);
   /** Close a Driver when it's no longer needed */
   static void close_storage(rgw::sal::Driver* driver);
 
diff --git a/src/test/rgw/rgw_cr_test.cc b/src/test/rgw/rgw_cr_test.cc
index db59dd1841b3..d75f776eb749 100644
--- a/src/test/rgw/rgw_cr_test.cc
+++ b/src/test/rgw/rgw_cr_test.cc
@@ -21,6 +21,8 @@
 #include "rgw_cr_rados.h"
 #include "rgw_sal.h"
 #include "rgw_sal_rados.h"
+#include "driver/rados/rgw_zone.h"
+#include "rgw_sal_config.h"
 
 #include "gtest/gtest.h"
 
@@ -323,12 +325,26 @@ int main(int argc, const char **argv)
 
   ceph::async::io_context_pool context_pool{cct->_conf->rgw_thread_pool_size};
   DriverManager::Config cfg = DriverManager::get_config(true, g_ceph_context);
+  auto config_store_type = g_conf().get_val<std::string>("rgw_config_store");
+  std::unique_ptr<rgw::sal::ConfigStore> cfgstore
+    = DriverManager::create_config_store(dpp(), config_store_type);
+  if (!cfgstore) {
+    std::cerr << "Unable to initialize config store." << std::endl;
+    exit(1);
+  }
+  rgw::SiteConfig site;
+  auto r = site.load(dpp(), null_yield, cfgstore.get());
+  if (r < 0) {
+    std::cerr << "Unable to initialize config store." << std::endl;
+    exit(1);
+  }
 
   store = static_cast<rgw::sal::RadosStore*>(
     DriverManager::get_storage(dpp(),
 			      g_ceph_context,
 			      cfg,
 			      context_pool,
+			      site,
 			      false,
 			      false,
 			      false,
diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc
index 00f7f24e00f8..9b2491600918 100644
--- a/src/test/rgw/test_d4n_filter.cc
+++ b/src/test/rgw/test_d4n_filter.cc
@@ -11,6 +11,8 @@
 #include "rgw_sal.h"
 #include "rgw_auth.h"
 #include "rgw_auth_registry.h"
+#include "driver/rados/rgw_zone.h"
+#include "rgw_sal_config.h"
 
 #include <boost/asio/io_context.hpp>
 
@@ -55,16 +57,24 @@ class Environment : public ::testing::Environment {
       cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, 
 		        CODE_ENVIRONMENT_UTILITY, 
 			CINIT_FLAG_NO_MON_CONFIG);
-      
+
       dpp = new DoutPrefix(cct->get(), dout_subsys, "d4n test: ");
       DriverManager::Config cfg;
 
       cfg.store_name = "dbstore";
       cfg.filter_name = "d4n";
-      
+      auto config_store_type = g_conf().get_val<std::string>("rgw_config_store");
+      std::unique_ptr<rgw::sal::ConfigStore> cfgstore
+        = DriverManager::create_config_store(dpp, config_store_type);
+      ASSERT_TRUE(cfgstore);
+      rgw::SiteConfig site;
+      auto r = site.load(dpp, null_yield, cfgstore.get());
+      ASSERT_GT(r, 0);
+
       driver = DriverManager::get_storage(dpp, dpp->get_cct(),
               cfg,
               ioc,
+              site,
               false,
               false,
               false,
diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h
index 664e0b22e804..ece16e844037 100644
--- a/src/test/rgw/test_rgw_common.h
+++ b/src/test/rgw/test_rgw_common.h
@@ -502,5 +502,7 @@ void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const
 void test_rgw_init_bucket(rgw_bucket *bucket, const char *name);
 rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns);
 
+
+
 #endif
 
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index b7cd66f6ea5a..e69910395a8f 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -30,14 +30,13 @@
 #include "rgw_op.h"
 #include "rgw_process_env.h"
 #include "rgw_sal_rados.h"
-
+#include "driver/rados/rgw_zone.h"
+#include "rgw_sal_config.h"
 
 using std::string;
-using std::vector;
 
 using boost::container::flat_set;
 using boost::intrusive_ptr;
-using boost::make_optional;
 using boost::none;
 
 using rgw::auth::Identity;
@@ -914,7 +913,8 @@ TEST_F(IPPolicyTest, IPEnvironment) {
   // Unfortunately RGWCivetWeb is too tightly tied to civetweb to test RGWCivetWeb::init_env.
   RGWEnv rgw_env;
   ceph::async::io_context_pool context_pool(cct->_conf->rgw_thread_pool_size); \
-  rgw::sal::RadosStore store(context_pool);
+  auto site = rgw::SiteConfig::make_fake();
+  rgw::sal::RadosStore store(context_pool, *site);
   std::unique_ptr<rgw::sal::User> user = store.get_user(rgw_user());
   rgw_env.set("REMOTE_ADDR", "192.168.1.1");
   rgw_env.set("HTTP_HOST", "1.2.3.4");
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 1ae7976c7944..8aed91bb6f5a 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -8,6 +8,8 @@
 #include "rgw_lua_request.h"
 #include "rgw_lua_background.h"
 #include "rgw_lua_data_filter.h"
+#include "driver/rados/rgw_zone.h"
+#include "rgw_sal_config.h"
 
 using namespace std;
 using namespace rgw;
@@ -160,13 +162,30 @@ CctCleaner cleaner(g_cct);
 
 tracing::Tracer tracer;
 
-#define MAKE_STORE \
-  ceph::async::io_context_pool context_pool(g_cct->_conf->rgw_thread_pool_size); \
-  auto store = std::unique_ptr<sal::RadosStore>(new sal::RadosStore(context_pool)); \
-  store->setRados(new RGWRados);
+inline std::unique_ptr<sal::RadosStore> make_store() {
+  auto context_pool = std::make_unique<ceph::async::io_context_pool>(
+    g_cct->_conf->rgw_thread_pool_size);
+  std::unique_ptr<rgw::SiteConfig> site = rgw::SiteConfig::make_fake();
+
+
+  struct StoreBundle : public sal::RadosStore {
+    std::unique_ptr<ceph::async::io_context_pool> context_pool;
+    std::unique_ptr<rgw::SiteConfig> site;
+    StoreBundle(std::unique_ptr<ceph::async::io_context_pool> context_pool_,
+                std::unique_ptr<rgw::SiteConfig> site_)
+      : sal::RadosStore(*context_pool_.get(), *site_),
+        context_pool(std::move(context_pool_)),
+        site(std::move(site_)) {
+      setRados(new RGWRados);
+    }
+    virtual ~StoreBundle() = default;
+  };
+  return std::make_unique<StoreBundle>(std::move(context_pool),
+                                       std::move(site));
+};
 
 #define DEFINE_REQ_STATE RGWProcessEnv pe; \
-  MAKE_STORE; \
+  auto store = make_store();                   \
   pe.lua.manager = store->get_lua_manager(""); \
   RGWEnv e; \
   req_state s(g_cct, pe, &e, 0);
@@ -861,7 +880,7 @@ class TestBackground : public rgw::lua::Background {
 
 TEST(TestRGWLuaBackground, Start)
 {
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   {
     // ctr and dtor without running
@@ -895,7 +914,7 @@ TEST(TestRGWLuaBackground, Script)
     RGW[key] = value
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get());
   lua_background.start();
@@ -948,7 +967,7 @@ TEST(TestRGWLuaBackground, Pause)
     end
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get());
   lua_background.start();
@@ -974,7 +993,7 @@ TEST(TestRGWLuaBackground, PauseWhileReading)
     end
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get(), 2);
   lua_background.start();
@@ -996,7 +1015,7 @@ TEST(TestRGWLuaBackground, ReadWhilePaused)
     RGW[key] = value
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get());
   lua_background.pause();
@@ -1020,7 +1039,7 @@ TEST(TestRGWLuaBackground, PauseResume)
     end
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get());
   lua_background.start();
@@ -1049,7 +1068,7 @@ TEST(TestRGWLuaBackground, MultipleStarts)
     end
   )";
 
-  MAKE_STORE;
+  auto store = make_store();
   auto manager = store->get_lua_manager("");
   TestBackground lua_background(store.get(), script, manager.get());
   lua_background.start();

From 6ad175f0b9b561b05a99246553b5d1d2542922fd Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 6 Jan 2024 15:32:03 -0800
Subject: [PATCH 1491/2492] crimson/osd/osd.cc: migrate logging to new style

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd.cc | 267 ++++++++++++++++++++++-------------------
 1 file changed, 144 insertions(+), 123 deletions(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index eb9ba8ef1c86..063f9d86ea77 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -39,6 +39,7 @@
 #include "crimson/admin/pg_commands.h"
 #include "crimson/common/buffer_io.h"
 #include "crimson/common/exception.h"
+#include "crimson/common/log.h"
 #include "crimson/mon/MonClient.h"
 #include "crimson/net/Connection.h"
 #include "crimson/net/Messenger.h"
@@ -58,10 +59,9 @@
 #include "crimson/osd/osd_operation_external_tracking.h"
 #include "crimson/crush/CrushLocation.h"
 
+SET_SUBSYS(osd);
+
 namespace {
-  seastar::logger& logger() {
-    return crimson::get_logger(ceph_subsys_osd);
-  }
   static constexpr int TICK_INTERVAL = 1;
 }
 
@@ -112,6 +112,7 @@ OSD::OSD(int id, uint32_t nonce,
     log_client(cluster_msgr.get(), LogClient::NO_FLAGS),
     clog(log_client.create_channel())
 {
+  LOG_PREFIX(OSD::OSD);
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
   for (auto msgr : {std::ref(cluster_msgr), std::ref(public_msgr),
                     std::ref(hb_front_msgr), std::ref(hb_back_msgr)}) {
@@ -122,11 +123,11 @@ OSD::OSD(int id, uint32_t nonce,
   if (local_conf()->osd_open_classes_on_start) {
     const int r = ClassHandler::get_instance().open_all_classes();
     if (r) {
-      logger().warn("{} warning: got an error loading one or more classes: {}",
-                    __func__, cpp_strerror(r));
+      WARN("warning: got an error loading one or more classes: {}",
+	   cpp_strerror(r));
     }
   }
-  logger().info("{}: nonce is {}", __func__, nonce);
+  INFO("nonce is {}", nonce);
   monc->set_log_client(&log_client);
   clog->set_log_to_monitors(true);
 }
@@ -195,20 +196,21 @@ seastar::future<> OSD::mkfs(
   uuid_d cluster_fsid,
   std::string osdspec_affinity)
 {
-  return store.start().then([&store, osd_uuid] {
+  LOG_PREFIX(OSD::mkfs);
+  return store.start().then([&store, FNAME, osd_uuid] {
     return store.mkfs(osd_uuid).handle_error(
-      crimson::stateful_ec::handle([] (const auto& ec) {
-        logger().error("error creating empty object store in {}: ({}) {}",
-                       local_conf().get_val<std::string>("osd_data"),
-                       ec.value(), ec.message());
+      crimson::stateful_ec::handle([FNAME] (const auto& ec) {
+        ERROR("error creating empty object store in {}: ({}) {}",
+	      local_conf().get_val<std::string>("osd_data"),
+	      ec.value(), ec.message());
         std::exit(EXIT_FAILURE);
       }));
-  }).then([&store] {
+  }).then([&store, FNAME] {
     return store.mount().handle_error(
-      crimson::stateful_ec::handle([](const auto& ec) {
-        logger().error("error mounting object store in {}: ({}) {}",
-                       local_conf().get_val<std::string>("osd_data"),
-                       ec.value(), ec.message());
+      crimson::stateful_ec::handle([FNAME](const auto& ec) {
+        ERROR("error mounting object store in {}: ({}) {}",
+	      local_conf().get_val<std::string>("osd_data"),
+	      ec.value(), ec.message());
         std::exit(EXIT_FAILURE);
       }));
   }).then([&store] {
@@ -248,34 +250,35 @@ seastar::future<> OSD::_write_superblock(
   OSDMeta meta_coll,
   OSDSuperblock superblock)
 {
+  LOG_PREFIX(OSD::_write_superblock);
   return seastar::do_with(
     std::move(meta_coll),
     std::move(superblock),
-    [&store](auto &meta_coll, auto &superblock) {
+    [&store, FNAME](auto &meta_coll, auto &superblock) {
       return meta_coll.load_superblock(
-      ).safe_then([&superblock](OSDSuperblock&& sb) {
+      ).safe_then([&superblock, FNAME](OSDSuperblock&& sb) {
 	if (sb.cluster_fsid != superblock.cluster_fsid) {
-	  logger().error("provided cluster fsid {} != superblock's {}",
-			 sb.cluster_fsid, superblock.cluster_fsid);
+	  ERROR("provided cluster fsid {} != superblock's {}",
+		sb.cluster_fsid, superblock.cluster_fsid);
 	  throw std::invalid_argument("mismatched fsid");
 	}
 	if (sb.whoami != superblock.whoami) {
-	  logger().error("provided osd id {} != superblock's {}",
-			 sb.whoami, superblock.whoami);
+	  ERROR("provided osd id {} != superblock's {}",
+		sb.whoami, superblock.whoami);
 	  throw std::invalid_argument("mismatched osd id");
 	}
       }).handle_error(
-	crimson::ct_error::enoent::handle([&store, &meta_coll, &superblock] {
+	crimson::ct_error::enoent::handle([&store, &meta_coll, &superblock,
+					   FNAME] {
 	  // meta collection does not yet, create superblock
-	  logger().info(
-	    "{} writing superblock cluster_fsid {} osd_fsid {}",
-	    "_write_superblock",
-	    superblock.cluster_fsid,
-	    superblock.osd_fsid);
+	  INFO("{} writing superblock cluster_fsid {} osd_fsid {}",
+	       "_write_superblock",
+	       superblock.cluster_fsid,
+	       superblock.osd_fsid);
 	  ceph::os::Transaction t;
 	  meta_coll.create(t);
 	  meta_coll.store_superblock(t, superblock);
-	  logger().debug("OSD::_write_superblock: do_transaction...");
+	  DEBUG("OSD::_write_superblock: do_transaction...");
 	  return store.get_sharded_store().do_transaction(
 	    meta_coll.collection(),
 	    std::move(t));
@@ -294,7 +297,7 @@ static std::string to_string(const seastar::temporary_buffer<char>& temp_buf)
 
 seastar::future<> OSD::_write_key_meta(FuturizedStore &store)
 {
-
+  LOG_PREFIX(OSD::_write_key_meta);
   if (auto key = local_conf().get_val<std::string>("key"); !std::empty(key)) {
     return store.write_meta("osd_key", key);
   } else if (auto keyfile = local_conf().get_val<std::string>("keyfile");
@@ -302,9 +305,9 @@ seastar::future<> OSD::_write_key_meta(FuturizedStore &store)
     return read_file(keyfile).then([&store](const auto& temp_buf) {
       // it's on a truly cold path, so don't worry about memcpy.
       return store.write_meta("osd_key", to_string(temp_buf));
-    }).handle_exception([keyfile] (auto ep) {
-      logger().error("_write_key_meta: failed to handle keyfile {}: {}",
-                     keyfile, ep);
+    }).handle_exception([FNAME, keyfile] (auto ep) {
+      ERROR("_write_key_meta: failed to handle keyfile {}: {}",
+	    keyfile, ep);
       ceph_abort();
     });
   } else {
@@ -314,6 +317,7 @@ seastar::future<> OSD::_write_key_meta(FuturizedStore &store)
 
 namespace {
   entity_addrvec_t pick_addresses(int what) {
+    LOG_PREFIX(osd.cc:pick_addresses);
     entity_addrvec_t addrs;
     crimson::common::CephContext cct;
     // we're interested solely in v2; crimson doesn't do v1
@@ -322,7 +326,7 @@ namespace {
       throw std::runtime_error("failed to pick address");
     }
     for (auto addr : addrs.v) {
-      logger().info("picked address {}", addr);
+      INFO("picked address {}", addr);
     }
     return addrs;
   }
@@ -357,7 +361,8 @@ namespace {
 
 seastar::future<> OSD::start()
 {
-  logger().info("start");
+  LOG_PREFIX(OSD::start);
+  INFO();
 
   startup_time = ceph::mono_clock::now();
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
@@ -381,15 +386,15 @@ seastar::future<> OSD::start()
         std::ref(store),
         std::ref(osd_states));
     });
-  }).then([this] {
+  }).then([this, FNAME] {
     heartbeat.reset(new Heartbeat{
 	whoami, get_shard_services(),
 	*monc, *hb_front_msgr, *hb_back_msgr});
     return store.mount().handle_error(
-      crimson::stateful_ec::handle([] (const auto& ec) {
-        logger().error("error mounting object store in {}: ({}) {}",
-                       local_conf().get_val<std::string>("osd_data"),
-                       ec.value(), ec.message());
+      crimson::stateful_ec::handle([FNAME] (const auto& ec) {
+        ERROR("error mounting object store in {}: ({}) {}",
+	      local_conf().get_val<std::string>("osd_data"),
+	      ec.value(), ec.message());
         std::exit(EXIT_FAILURE);
       }));
   }).then([this] {
@@ -417,7 +422,7 @@ seastar::future<> OSD::start()
   }).then([this] {
     bind_epoch = osdmap->get_epoch();
     return pg_shard_manager.load_pgs(store);
-  }).then([this] {
+  }).then([this, FNAME] {
     uint64_t osd_required =
       CEPH_FEATURE_UID |
       CEPH_FEATURE_PGID64 |
@@ -446,16 +451,16 @@ seastar::future<> OSD::start()
         .safe_then([this, dispatchers]() mutable {
 	  return cluster_msgr->start(dispatchers);
         }, crimson::net::Messenger::bind_ertr::all_same_way(
-            [] (const std::error_code& e) {
-          logger().error("cluster messenger bind(): {}", e);
+            [FNAME] (const std::error_code& e) {
+          ERROR("cluster messenger bind(): {}", e);
           ceph_abort();
         })),
       public_msgr->bind(pick_addresses(CEPH_PICK_ADDRESS_PUBLIC))
         .safe_then([this, dispatchers]() mutable {
 	  return public_msgr->start(dispatchers);
         }, crimson::net::Messenger::bind_ertr::all_same_way(
-            [] (const std::error_code& e) {
-          logger().error("public messenger bind(): {}", e);
+            [FNAME] (const std::error_code& e) {
+          ERROR("public messenger bind(): {}", e);
           ceph_abort();
         })));
   }).then_unpack([this] {
@@ -468,11 +473,11 @@ seastar::future<> OSD::start()
     monc->sub_want("mgrmap", 0, 0);
     monc->sub_want("osdmap", 0, 0);
     return monc->renew_subs();
-  }).then([this] {
+  }).then([FNAME, this] {
     if (auto [addrs, changed] =
         replace_unknown_addrs(cluster_msgr->get_myaddrs(),
                               public_msgr->get_myaddrs()); changed) {
-      logger().debug("replacing unkwnown addrs of cluster messenger");
+      DEBUG("replacing unkwnown addrs of cluster messenger");
       cluster_msgr->set_myaddrs(addrs);
     }
     return heartbeat->start(pick_addresses(CEPH_PICK_ADDRESS_PUBLIC),
@@ -499,22 +504,23 @@ seastar::future<> OSD::start_boot()
 
 seastar::future<> OSD::_preboot(version_t oldest, version_t newest)
 {
-  logger().info("osd.{}: _preboot", whoami);
+  LOG_PREFIX(OSD::_preboot);
+  INFO("osd.{}", whoami);
   if (osdmap->get_epoch() == 0) {
-    logger().info("waiting for initial osdmap");
+    INFO("waiting for initial osdmap");
   } else if (osdmap->is_destroyed(whoami)) {
-    logger().warn("osdmap says I am destroyed");
+    INFO("osdmap says I am destroyed");
     // provide a small margin so we don't livelock seeing if we
     // un-destroyed ourselves.
     if (osdmap->get_epoch() > newest - 1) {
       throw std::runtime_error("i am destroyed");
     }
   } else if (osdmap->is_noup(whoami)) {
-    logger().warn("osdmap NOUP flag is set, waiting for it to clear");
+    WARN("osdmap NOUP flag is set, waiting for it to clear");
   } else if (!osdmap->test_flag(CEPH_OSDMAP_SORTBITWISE)) {
-    logger().error("osdmap SORTBITWISE OSDMap flag is NOT set; please set it");
+    ERROR("osdmap SORTBITWISE OSDMap flag is NOT set; please set it");
   } else if (osdmap->require_osd_release < ceph_release_t::octopus) {
-    logger().error("osdmap require_osd_release < octopus; please upgrade to octopus");
+    ERROR("osdmap require_osd_release < octopus; please upgrade to octopus");
   } else if (false) {
     // TODO: update mon if current fullness state is different from osdmap
   } else if (version_t n = local_conf()->osd_map_message_max;
@@ -532,6 +538,7 @@ seastar::future<> OSD::_preboot(version_t oldest, version_t newest)
 
 seastar::future<> OSD::_send_boot()
 {
+  LOG_PREFIX(OSD::_send_boot);
   pg_shard_manager.set_booting();
 
   entity_addrvec_t public_addrs = public_msgr->get_myaddrs();
@@ -547,9 +554,9 @@ seastar::future<> OSD::_send_boot()
   if (heartbeat->get_front_msgr().set_addr_unknowns(public_addrs)) {
     hb_front_addrs = heartbeat->get_front_addrs();
   }
-  logger().info("hb_back_msgr: {}", hb_back_addrs);
-  logger().info("hb_front_msgr: {}", hb_front_addrs);
-  logger().info("cluster_msgr: {}", cluster_addrs);
+  INFO("hb_back_msgr: {}", hb_back_addrs);
+  INFO("hb_front_msgr: {}", hb_front_addrs);
+  INFO("cluster_msgr: {}", cluster_addrs);
 
   auto m = crimson::make_message<MOSDBoot>(superblock,
                                   osdmap->get_epoch(),
@@ -568,6 +575,7 @@ seastar::future<> OSD::_send_boot()
 
 seastar::future<> OSD::_add_me_to_crush()
 {
+  LOG_PREFIX(OSD::_add_me_to_crush);
   if (!local_conf().get_val<bool>("osd_crush_update_on_start")) {
     return seastar::now();
   }
@@ -584,13 +592,13 @@ seastar::future<> OSD::_add_me_to_crush()
        });
     }
   };
-  return get_weight().then([this](auto weight) {
+  return get_weight().then([FNAME, this](auto weight) {
     const crimson::crush::CrushLocation loc;
     return seastar::do_with(
       std::move(loc),
-      [this, weight] (crimson::crush::CrushLocation& loc) {
-      return loc.init_on_startup().then([this, weight, &loc]() {
-        logger().info("crush location is {}", loc);
+      [FNAME, this, weight] (crimson::crush::CrushLocation& loc) {
+      return loc.init_on_startup().then([FNAME, this, weight, &loc]() {
+        INFO("crush location is {}", loc);
         string cmd = fmt::format(R"({{
           "prefix": "osd crush create-or-move",
           "id": {},
@@ -600,13 +608,13 @@ seastar::future<> OSD::_add_me_to_crush()
         return monc->run_command(std::move(cmd), {});
       });
     });
-  }).then([](auto&& command_result) {
+  }).then([FNAME](auto&& command_result) {
     [[maybe_unused]] auto [code, message, out] = std::move(command_result);
     if (code) {
-      logger().warn("fail to add to crush: {} ({})", message, code);
+      WARN("fail to add to crush: {} ({})", message, code);
       throw std::runtime_error("fail to add to crush");
     } else {
-      logger().info("added to crush: {}", message);
+      INFO("added to crush: {}", message);
     }
     return seastar::now();
   });
@@ -664,14 +672,15 @@ seastar::future<> OSD::start_asok_admin()
 
 seastar::future<> OSD::stop()
 {
-  logger().info("stop");
+  LOG_PREFIX(OSD::stop);
+  INFO();
   beacon_timer.cancel();
   tick_timer.cancel();
   // see also OSD::shutdown()
   return prepare_to_stop().then([this] {
     return pg_shard_manager.set_stopping();
-  }).then([this] {
-    logger().debug("prepared to stop");
+  }).then([FNAME, this] {
+    DEBUG("prepared to stop");
     public_msgr->stop();
     cluster_msgr->stop();
     auto gate_close_fut = gate.close();
@@ -703,8 +712,8 @@ seastar::future<> OSD::stop()
       return when_all_succeed(
 	  public_msgr->shutdown(),
 	  cluster_msgr->shutdown()).discard_result();
-    }).handle_exception([](auto ep) {
-      logger().error("error while stopping osd: {}", ep);
+    }).handle_exception([FNAME](auto ep) {
+      ERROR("error while stopping osd: {}", ep);
     });
   });
 }
@@ -845,20 +854,23 @@ OSD::do_ms_dispatch(
 void OSD::ms_handle_reset(crimson::net::ConnectionRef conn, bool is_replace)
 {
   // TODO: cleanup the session attached to this connection
-  logger().warn("ms_handle_reset");
+  LOG_PREFIX(OSD::ms_handle_reset);
+  WARN("{}", *conn);
 }
 
 void OSD::ms_handle_remote_reset(crimson::net::ConnectionRef conn)
 {
-  logger().warn("ms_handle_remote_reset");
+  LOG_PREFIX(OSD::ms_handle_remote_reset);
+  WARN("{}", *conn);
 }
 
 void OSD::handle_authentication(const EntityName& name,
 				const AuthCapsInfo& caps_info)
 {
+  LOG_PREFIX(OSD::handle_authentication);
   // TODO: store the parsed cap and associate it with the connection
   if (caps_info.allow_all) {
-    logger().debug("{} {} has all caps", __func__, name);
+    DEBUG("{} has all caps", name);
     return;
   }
   if (caps_info.caps.length() > 0) {
@@ -867,14 +879,14 @@ void OSD::handle_authentication(const EntityName& name,
     try {
       decode(str, p);
     } catch (ceph::buffer::error& e) {
-      logger().warn("{} {} failed to decode caps string", __func__, name);
+      WARN("{} failed to decode caps string", name);
       return;
     }
     OSDCap caps;
     if (caps.parse(str)) {
-      logger().debug("{} {} has caps {}", __func__, name, str);
+      DEBUG("{} has caps {}", name, str);
     } else {
-      logger().warn("{} {} failed to parse caps {}", __func__, name, str);
+      WARN("{} failed to parse caps {}", name, str);
     }
   }
 }
@@ -960,28 +972,29 @@ seastar::future<> OSD::handle_osd_map(Ref<MOSDMap> m)
 
 seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
 {
-  logger().info("handle_osd_map {}", *m);
+  LOG_PREFIX(OSD::_handle_osd_map);
+  INFO("{}", *m);
   if (m->fsid != superblock.cluster_fsid) {
-    logger().warn("fsid mismatched");
+    WARN("fsid mismatched");
     return seastar::now();
   }
   if (pg_shard_manager.is_initializing()) {
-    logger().warn("i am still initializing");
+    WARN("i am still initializing");
     return seastar::now();
   }
 
   const auto first = m->get_first();
   const auto last = m->get_last();
-  logger().info("handle_osd_map epochs [{}..{}], i have {}, src has [{}..{}]",
-                first, last, superblock.get_newest_map(),
-                m->cluster_osdmap_trim_lower_bound, m->newest_map);
+  INFO(" epochs [{}..{}], i have {}, src has [{}..{}]",
+       first, last, superblock.get_newest_map(),
+       m->cluster_osdmap_trim_lower_bound, m->newest_map);
 
   if (superblock.cluster_osdmap_trim_lower_bound <
       m->cluster_osdmap_trim_lower_bound) {
     superblock.cluster_osdmap_trim_lower_bound =
       m->cluster_osdmap_trim_lower_bound;
-    logger().debug("{} superblock cluster_osdmap_trim_lower_bound new epoch is: {}",
-                   __func__, superblock.cluster_osdmap_trim_lower_bound);
+    DEBUG("superblock cluster_osdmap_trim_lower_bound new epoch is: {}",
+	  superblock.cluster_osdmap_trim_lower_bound);
     ceph_assert(
       superblock.cluster_osdmap_trim_lower_bound >= superblock.get_oldest_map());
   }
@@ -993,8 +1006,8 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
   // missing some?
   epoch_t start = superblock.get_newest_map() + 1;
   if (first > start) {
-    logger().info("handle_osd_map message skips epochs {}..{}",
-                  start, first - 1);
+    INFO("message skips epochs {}..{}",
+	 start, first - 1);
     if (m->cluster_osdmap_trim_lower_bound <= start) {
       return get_shard_services().osdmap_subscribe(start, false);
     }
@@ -1030,8 +1043,8 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
       }
       pg_shard_manager.get_meta_coll().store_superblock(t, superblock);
       return pg_shard_manager.set_superblock(superblock).then(
-      [this, &t] {
-        logger().debug("OSD::handle_osd_map: do_transaction...");
+      [FNAME, this, &t] {
+        DEBUG("submitting transaction");
         return store.get_sharded_store().do_transaction(
           pg_shard_manager.get_meta_coll().collection(),
           std::move(t));
@@ -1048,8 +1061,9 @@ seastar::future<> OSD::committed_osd_maps(
   version_t last,
   Ref<MOSDMap> m)
 {
+  LOG_PREFIX(OSD::committed_osd_maps);
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
-  logger().info("osd.{}: committed_osd_maps({}, {})", whoami, first, last);
+  INFO("osd.{} ({}, {})", whoami, first, last);
   // advance through the new maps
   return seastar::do_for_each(boost::make_counting_iterator(first),
                               boost::make_counting_iterator(last + 1),
@@ -1074,17 +1088,17 @@ seastar::future<> OSD::committed_osd_maps(
 	return seastar::now();
       }
     });
-  }).then([m, this] {
+  }).then([FNAME, m, this] {
     auto fut = seastar::now();
     if (osdmap->is_up(whoami)) {
       const auto up_from = osdmap->get_up_from(whoami);
-      logger().info("osd.{}: map e {} marked me up: up_from {}, bind_epoch {}, state {}",
-                    whoami, osdmap->get_epoch(), up_from, bind_epoch,
-		    pg_shard_manager.get_osd_state_string());
+      INFO("osd.{}: map e {} marked me up: up_from {}, bind_epoch {}, state {}",
+	   whoami, osdmap->get_epoch(), up_from, bind_epoch,
+	   pg_shard_manager.get_osd_state_string());
       if (bind_epoch < up_from &&
           osdmap->get_addrs(whoami) == public_msgr->get_myaddrs() &&
           pg_shard_manager.is_booting()) {
-        logger().info("osd.{}: activating...", whoami);
+        INFO("osd.{}: activating...", whoami);
         fut = pg_shard_manager.set_active().then([this] {
           beacon_timer.arm_periodic(
             std::chrono::seconds(local_conf()->osd_beacon_report_interval));
@@ -1099,17 +1113,17 @@ seastar::future<> OSD::committed_osd_maps(
 	return seastar::now();
       }
     }
-    return fut.then([this] {
-      return check_osdmap_features().then([this] {
+    return fut.then([FNAME, this] {
+      return check_osdmap_features().then([FNAME, this] {
         // yay!
-        logger().info("osd.{}: committed_osd_maps: broadcasting osdmaps up"
-                      " to {} epoch to pgs", whoami, osdmap->get_epoch());
+        INFO("osd.{}: committed_osd_maps: broadcasting osdmaps up"
+	     " to {} epoch to pgs", whoami, osdmap->get_epoch());
         return pg_shard_manager.broadcast_map_to_pgs(osdmap->get_epoch());
       });
     });
-  }).then([m, this] {
+  }).then([FNAME, m, this] {
     if (pg_shard_manager.is_active()) {
-      logger().info("osd.{}: now active", whoami);
+      INFO("osd.{}: now active", whoami);
       if (!osdmap->exists(whoami) ||
 	  osdmap->is_stop(whoami)) {
         return shutdown();
@@ -1120,18 +1134,18 @@ seastar::future<> OSD::committed_osd_maps(
         return seastar::now();
       }
     } else if (pg_shard_manager.is_preboot()) {
-      logger().info("osd.{}: now preboot", whoami);
+      INFO("osd.{}: now preboot", whoami);
 
       if (m->get_source().is_mon()) {
         return _preboot(
           m->cluster_osdmap_trim_lower_bound, m->newest_map);
       } else {
-        logger().info("osd.{}: start_boot", whoami);
+        INFO("osd.{}: start_boot", whoami);
         return start_boot();
       }
     } else {
-      logger().info("osd.{}: now {}", whoami,
-		    pg_shard_manager.get_osd_state_string());
+      INFO("osd.{}: now {}", whoami,
+	   pg_shard_manager.get_osd_state_string());
       // XXX
       return seastar::now();
     }
@@ -1152,20 +1166,21 @@ seastar::future<> OSD::handle_pg_create(
   crimson::net::ConnectionRef conn,
   Ref<MOSDPGCreate2> m)
 {
-  return seastar::do_for_each(m->pgs, [this, conn, m](auto& pg) {
+  LOG_PREFIX(OSD::handle_pg_create);
+  return seastar::do_for_each(m->pgs, [FNAME, this, conn, m](auto& pg) {
     auto& [pgid, when] = pg;
     const auto &[created, created_stamp] = when;
     auto q = m->pg_extra.find(pgid);
     ceph_assert(q != m->pg_extra.end());
     auto& [history, pi] = q->second;
-    logger().debug(
-      "{}: {} e{} @{} "
+    DEBUG(
+      "e{} @{} "
       "history {} pi {}",
-      __func__, pgid, created, created_stamp,
+      pgid, created, created_stamp,
       history, pi);
     if (!pi.empty() &&
 	m->epoch < pi.get_bounds().second) {
-      logger().error(
+      ERROR(
         "got pg_create on {} epoch {}  "
         "unmatched past_intervals {} (history {})",
         pgid, m->epoch,
@@ -1217,15 +1232,16 @@ seastar::future<> OSD::handle_rep_op_reply(
   crimson::net::ConnectionRef conn,
   Ref<MOSDRepOpReply> m)
 {
+  LOG_PREFIX(OSD::handle_rep_op_reply);
   spg_t pgid = m->get_spg();
   return pg_shard_manager.with_pg(
     pgid,
-    [m=std::move(m)](auto &&pg) {
+    [FNAME, m=std::move(m)](auto &&pg) {
       if (pg) {
 	m->finish_decode();
 	pg->handle_rep_op_reply(*m);
       } else {
-	logger().warn("stale reply: {}", *m);
+	WARN("stale reply: {}", *m);
       }
       return seastar::now();
     });
@@ -1235,8 +1251,9 @@ seastar::future<> OSD::handle_scrub_command(
   crimson::net::ConnectionRef conn,
   Ref<MOSDScrub2> m)
 {
+  LOG_PREFIX(OSD::handle_scrub_command);
   if (m->fsid != superblock.cluster_fsid) {
-    logger().warn("fsid mismatched");
+    WARN("fsid mismatched");
     return seastar::now();
   }
   return seastar::parallel_for_each(std::move(m->scrub_pgs),
@@ -1278,21 +1295,22 @@ seastar::future<> OSD::handle_recovery_subreq(
 
 bool OSD::should_restart() const
 {
+  LOG_PREFIX(OSD::should_restart);
   if (!osdmap->is_up(whoami)) {
-    logger().info("map e {} marked osd.{} down",
-                  osdmap->get_epoch(), whoami);
+    INFO("map e {} marked osd.{} down",
+	 osdmap->get_epoch(), whoami);
     return true;
   } else if (osdmap->get_addrs(whoami) != public_msgr->get_myaddrs()) {
-    logger().error("map e {} had wrong client addr ({} != my {})",
-                   osdmap->get_epoch(),
-                   osdmap->get_addrs(whoami),
-                   public_msgr->get_myaddrs());
+    ERROR("map e {} had wrong client addr ({} != my {})",
+	  osdmap->get_epoch(),
+	  osdmap->get_addrs(whoami),
+	  public_msgr->get_myaddrs());
     return true;
   } else if (osdmap->get_cluster_addrs(whoami) != cluster_msgr->get_myaddrs()) {
-    logger().error("map e {} had wrong cluster addr ({} != my {})",
-                   osdmap->get_epoch(),
-                   osdmap->get_cluster_addrs(whoami),
-                   cluster_msgr->get_myaddrs());
+    ERROR("map e {} had wrong cluster addr ({} != my {})",
+	  osdmap->get_epoch(),
+	  osdmap->get_cluster_addrs(whoami),
+	  cluster_msgr->get_myaddrs());
     return true;
   } else {
     return false;
@@ -1315,13 +1333,15 @@ seastar::future<> OSD::restart()
 
 seastar::future<> OSD::shutdown()
 {
-  logger().info("shutting down per osdmap");
+  LOG_PREFIX(OSD::shutdown);
+  INFO("shutting down per osdmap");
   abort_source.request_abort();
   return seastar::now();
 }
 
 seastar::future<> OSD::send_beacon()
 {
+  LOG_PREFIX(OSD::send_beacon);
   if (!pg_shard_manager.is_active()) {
     return seastar::now();
   }
@@ -1330,7 +1350,7 @@ seastar::future<> OSD::send_beacon()
                                     superblock.last_purged_snaps_scrub,
                                     local_conf()->osd_beacon_report_interval);
   beacon->pgs = min_last_epoch_clean_pgs;
-  logger().debug("{} {}", __func__, *beacon);
+  DEBUG("{}", *beacon);
   return monc->send_message(std::move(beacon));
 }
 
@@ -1361,8 +1381,9 @@ seastar::future<> OSD::handle_peering_op(
   crimson::net::ConnectionRef conn,
   Ref<MOSDPeeringOp> m)
 {
+  LOG_PREFIX(OSD::handle_peering_op);
   const int from = m->get_source().num();
-  logger().debug("handle_peering_op on {} from {}", m->get_spg(), from);
+  DEBUG("{} from {}", m->get_spg(), from);
   m->set_features(conn->get_features());
   std::unique_ptr<PGPeeringEvent> evt(m->get_event());
   return pg_shard_manager.start_pg_operation<RemotePeeringEvent>(

From cb02911686f3c413e687c8af067dd8cd3df40d54 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 10 Jan 2024 17:16:49 +0000
Subject: [PATCH 1492/2492] crimson/osd/shard_service.cc: convert to newer
 logging machinery

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/shard_services.cc | 167 +++++++++++++++---------------
 1 file changed, 82 insertions(+), 85 deletions(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index d21c951b88cd..26993daa659e 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -13,6 +13,7 @@
 #include "osd/osd_perf_counters.h"
 #include "osd/PeeringState.h"
 #include "crimson/common/config_proxy.h"
+#include "crimson/common/log.h"
 #include "crimson/mgr/client.h"
 #include "crimson/mon/MonClient.h"
 #include "crimson/net/Messenger.h"
@@ -23,11 +24,7 @@
 #include "crimson/osd/pg.h"
 #include "crimson/osd/pg_meta.h"
 
-namespace {
-  seastar::logger& logger() {
-    return crimson::get_logger(ceph_subsys_osd);
-  }
-}
+SET_SUBSYS(osd);
 
 using std::vector;
 
@@ -166,12 +163,13 @@ OSDSingletonState::OSDSingletonState(
 seastar::future<> OSDSingletonState::send_to_osd(
   int peer, MessageURef m, epoch_t from_epoch)
 {
+  LOG_PREFIX(OSDSingletonState::send_to_osd);
   if (osdmap->is_down(peer)) {
-    logger().info("{}: osd.{} is_down", __func__, peer);
+    INFO("osd.{} is_down", peer);
     return seastar::now();
   } else if (osdmap->get_info(peer).up_from > from_epoch) {
-    logger().info("{}: osd.{} {} > {}", __func__, peer,
-		    osdmap->get_info(peer).up_from, from_epoch);
+    INFO("osd.{} {} > {}", peer,
+	 osdmap->get_info(peer).up_from, from_epoch);
     return seastar::now();
   } else {
     auto conn = cluster_msgr.connect(
@@ -184,7 +182,8 @@ seastar::future<> OSDSingletonState::send_to_osd(
 seastar::future<> OSDSingletonState::osdmap_subscribe(
   version_t epoch, bool force_request)
 {
-  logger().info("{}({})", __func__, epoch);
+  LOG_PREFIX(OSDSingletonState::osdmap_subscribe);
+  INFO("epoch {}");
   if (monc.sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
       force_request) {
     return monc.renew_subs();
@@ -214,13 +213,13 @@ void OSDSingletonState::remove_want_pg_temp(pg_t pgid)
 
 void OSDSingletonState::requeue_pg_temp()
 {
+  LOG_PREFIX(OSDSingletonState::requeue_pg_temp);
   unsigned old_wanted = pg_temp_wanted.size();
   unsigned old_pending = pg_temp_pending.size();
   pg_temp_wanted.merge(pg_temp_pending);
   pg_temp_pending.clear();
-  logger().debug(
-    "{}: {} + {} -> {}",
-    __func__ ,
+  DEBUG(
+    "{} + {} -> {}",
     old_wanted,
     old_pending,
     pg_temp_wanted.size());
@@ -228,9 +227,10 @@ void OSDSingletonState::requeue_pg_temp()
 
 seastar::future<> OSDSingletonState::send_pg_temp()
 {
+  LOG_PREFIX(OSDSingletonState::send_pg_temp);
   if (pg_temp_wanted.empty())
     return seastar::now();
-  logger().debug("{}: {}", __func__, pg_temp_wanted);
+  DEBUG("{}", pg_temp_wanted);
   MURef<MOSDPGTemp> ms[2] = {nullptr, nullptr};
   for (auto& [pgid, pg_temp] : pg_temp_wanted) {
     auto& m = ms[pg_temp.forced];
@@ -265,7 +265,8 @@ std::ostream& operator<<(
 
 seastar::future<> OSDSingletonState::send_pg_created(pg_t pgid)
 {
-  logger().debug(__func__);
+  LOG_PREFIX(OSDSingletonState::send_pg_created);
+  DEBUG();
   auto o = get_osdmap();
   ceph_assert(o->require_osd_release >= ceph_release_t::luminous);
   pg_created.insert(pgid);
@@ -274,7 +275,8 @@ seastar::future<> OSDSingletonState::send_pg_created(pg_t pgid)
 
 seastar::future<> OSDSingletonState::send_pg_created()
 {
-  logger().debug(__func__);
+  LOG_PREFIX(OSDSingletonState::send_pg_created);
+  DEBUG();
   auto o = get_osdmap();
   ceph_assert(o->require_osd_release >= ceph_release_t::luminous);
   return seastar::parallel_for_each(pg_created,
@@ -285,16 +287,17 @@ seastar::future<> OSDSingletonState::send_pg_created()
 
 void OSDSingletonState::prune_pg_created()
 {
-  logger().debug(__func__);
+  LOG_PREFIX(OSDSingletonState::prune_pg_created);
+  DEBUG();
   auto o = get_osdmap();
   auto i = pg_created.begin();
   while (i != pg_created.end()) {
     auto p = o->get_pg_pool(i->pool());
     if (!p || !p->has_flag(pg_pool_t::FLAG_CREATING)) {
-      logger().debug("{} pruning {}", __func__, *i);
+      DEBUG("pruning {}", *i);
       i = pg_created.erase(i);
     } else {
-      logger().debug(" keeping {}", __func__, *i);
+      DEBUG("keeping {}", *i);
       ++i;
     }
   }
@@ -302,29 +305,24 @@ void OSDSingletonState::prune_pg_created()
 
 seastar::future<> OSDSingletonState::send_alive(const epoch_t want)
 {
-  logger().info(
-    "{} want={} up_thru_wanted={}",
-    __func__,
-    want,
-    up_thru_wanted);
-
+  LOG_PREFIX(OSDSingletonState::send_alive);
+  INFO("want={} up_thru_wanted={}", want, up_thru_wanted);
   if (want > up_thru_wanted) {
     up_thru_wanted = want;
   } else {
-    logger().debug("{} want={} <= up_thru_wanted={}; skipping",
-                   __func__, want, up_thru_wanted);
+    DEBUG("want={} <= up_thru_wanted={}; skipping", want, up_thru_wanted);
     return seastar::now();
   }
   if (!osdmap->exists(whoami)) {
-    logger().warn("{} DNE", __func__);
+    WARN("DNE");
     return seastar::now();
   } if (const epoch_t up_thru = osdmap->get_up_thru(whoami);
         up_thru_wanted > up_thru) {
-    logger().debug("{} up_thru_wanted={} up_thru={}", __func__, want, up_thru);
+    DEBUG("up_thru_wanted={} up_thru={}", want, up_thru);
     return monc.send_message(
       crimson::make_message<MOSDAlive>(osdmap->get_epoch(), want));
   } else {
-    logger().debug("{} {} <= {}", __func__, want, osdmap->get_up_thru(whoami));
+    DEBUG("{} <= {}", want, osdmap->get_up_thru(whoami));
     return seastar::now();
   }
 }
@@ -360,11 +358,12 @@ void OSDSingletonState::handle_conf_change(
 seastar::future<OSDSingletonState::local_cached_map_t>
 OSDSingletonState::get_local_map(epoch_t e)
 {
+  LOG_PREFIX(OSDSingletonState::get_local_map);
   if (auto found = osdmaps.find(e); found) {
-    logger().debug("{} osdmap.{} found in cache", __func__, e);
+    DEBUG("osdmap.{} found in cache", e);
     return seastar::make_ready_future<local_cached_map_t>(std::move(found));
   } else {
-    logger().debug("{} loading osdmap.{} from disk", __func__, e);
+    DEBUG("loading osdmap.{} from disk", e);
     return load_map(e).then([e, this](std::unique_ptr<OSDMap> osdmap) {
       return seastar::make_ready_future<local_cached_map_t>(
 	osdmaps.insert(e, std::move(osdmap)));
@@ -391,11 +390,12 @@ void OSDSingletonState::store_inc_map_bl(
 seastar::future<bufferlist> OSDSingletonState::load_map_bl(
   epoch_t e)
 {
+  LOG_PREFIX(OSDSingletonState::load_map_bl);
   if (std::optional<bufferlist> found = map_bl_cache.find(e); found) {
-    logger().debug("{} osdmap.{} found in cache", __func__, e);
+    DEBUG("osdmap.{} found in cache", e);
     return seastar::make_ready_future<bufferlist>(*found);
   } else {
-    logger().debug("{} loading osdmap.{} from disk", __func__, e);
+    DEBUG("loading osdmap.{} from disk", e);
     return meta_coll->load_map(e).then([this, e](auto&& bl) {
       map_bl_cache.insert(e, bl);
       return seastar::make_ready_future<bufferlist>(std::move(bl));
@@ -406,11 +406,12 @@ seastar::future<bufferlist> OSDSingletonState::load_map_bl(
 read_errorator::future<ceph::bufferlist> OSDSingletonState::load_inc_map_bl(
   epoch_t e)
 {
+  LOG_PREFIX(OSDSingletonState::load_inc_map_bl);
   if (std::optional<bufferlist> found = inc_map_bl_cache.find(e); found) {
-    logger().debug("{} inc map.{} found in cache", __func__, e);
+    DEBUG("inc map.{} found in cache", e);
     return read_errorator::make_ready_future<bufferlist>(*found);
   } else {
-    logger().debug("{} loading inc map.{} from disk", __func__, e);
+    DEBUG("loading inc map.{} from disk", e);
     return meta_coll->load_inc_map(e).safe_then([this, e](auto&& bl) {
       inc_map_bl_cache.insert(e, bl);
       return seastar::make_ready_future<bufferlist>(std::move(bl));
@@ -422,19 +423,18 @@ seastar::future<OSDMapService::bls_map_t> OSDSingletonState::load_map_bls(
   epoch_t first,
   epoch_t last)
 {
-  logger().debug("{} loading maps [{},{}]",
-                 __func__, first, last);
+  LOG_PREFIX(OSDSingletonState::load_map_bl);
+  DEBUG("loading maps [{},{}]", first, last);
   ceph_assert(first <= last);
   return seastar::map_reduce(boost::make_counting_iterator<epoch_t>(first),
 			     boost::make_counting_iterator<epoch_t>(last + 1),
-			     [this](epoch_t e) {
+			     [this, FNAME](epoch_t e) {
     return load_inc_map_bl(e).safe_then([](auto&& bl) {
       return seastar::make_ready_future<OSDMapService::bls_pair>(
         std::make_pair(OSDMapService::encoded_osdmap_type_t::INCMAP,
                        std::move(bl)));
-    }, read_errorator::all_same_way([this, e] {
-      logger().debug("load_map_bls: can't load inc map {}, attempting full map instread",
-                      e);
+    }, read_errorator::all_same_way([this, FNAME, e] {
+      DEBUG("can't load inc map {}, attempting full map instread", e);
       return load_map_bl(e).then([](auto&& bl) {
         return seastar::make_ready_future<OSDMapService::bls_pair>(
           std::make_pair(OSDMapService::encoded_osdmap_type_t::FULLMAP,
@@ -454,8 +454,9 @@ seastar::future<OSDMapService::bls_map_t> OSDSingletonState::load_map_bls(
 
 seastar::future<std::unique_ptr<OSDMap>> OSDSingletonState::load_map(epoch_t e)
 {
+  LOG_PREFIX(OSDSingletonState::load_map_bl);
   auto o = std::make_unique<OSDMap>();
-  logger().info("{} osdmap.{}", __func__, e);
+  INFO("osdmap.{}", e);
   if (e == 0) {
     return seastar::make_ready_future<std::unique_ptr<OSDMap>>(std::move(o));
   }
@@ -468,28 +469,28 @@ seastar::future<std::unique_ptr<OSDMap>> OSDSingletonState::load_map(epoch_t e)
 seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
                                   epoch_t start, Ref<MOSDMap> m)
 {
+  LOG_PREFIX(OSDSingletonState::store_maps);
   return seastar::do_with(
     std::map<epoch_t, OSDMap*>(),
-    [&t, m, start, this](auto &added_maps) {
+    [&t, FNAME, m, start, this](auto &added_maps) {
     return seastar::do_for_each(
       boost::make_counting_iterator(start),
       boost::make_counting_iterator(m->get_last() + 1),
-      [&t, m, this, &added_maps](epoch_t e) {
+      [&t, FNAME, m, this, &added_maps](epoch_t e) {
       if (auto p = m->maps.find(e); p != m->maps.end()) {
 	auto o = std::make_unique<OSDMap>();
 	o->decode(p->second);
-	logger().info("store_maps storing osdmap.{}", e);
+	INFO("storing osdmap.{}", e);
 	store_map_bl(t, e, std::move(std::move(p->second)));
 	added_maps.emplace(e, o.get());
 	osdmaps.insert(e, std::move(o));
 	return seastar::now();
       } else if (auto p = m->incremental_maps.find(e);
 		 p != m->incremental_maps.end()) {
-	logger().info("store_maps found osdmap.{} incremental map, "
-	              "loading osdmap.{}", e, e - 1);
+	INFO("found osdmap.{} incremental map, loading osdmap.{}", e, e - 1);
 	ceph_assert(std::cmp_greater(e, 0u));
 	return load_map(e - 1).then(
-	  [&added_maps, e, bl=p->second, &t, this](auto o) mutable {
+          [&added_maps, FNAME, e, bl=p->second, &t, this](auto o) mutable {
 	  OSDMap::Incremental inc;
 	  auto i = bl.cbegin();
 	  inc.decode(i);
@@ -497,17 +498,17 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	  store_inc_map_bl(t, e, std::move(bl));
 	  bufferlist fbl;
 	  o->encode(fbl, inc.encode_features | CEPH_FEATURE_RESERVED);
-	  logger().info("store_maps storing osdmap.{}", o->get_epoch());
+	  INFO("storing osdmap.{}", o->get_epoch());
 	  store_map_bl(t, e, std::move(fbl));
 	  added_maps.emplace(e, o.get());
 	  osdmaps.insert(e, std::move(o));
 	  return seastar::now();
 	});
       } else {
-	logger().error("MOSDMap lied about what maps it had?");
+	ERROR("MOSDMap lied about what maps it had?");
 	return seastar::now();
       }
-    }).then([&t, this, &added_maps] {
+    }).then([&t, FNAME, this, &added_maps] {
       auto [e, map] = *added_maps.begin();
       auto lastmap = osdmaps.find(e - 1).get();
       meta_coll->store_final_pool_info(t, lastmap, added_maps);
@@ -521,6 +522,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 void OSDSingletonState::trim_maps(ceph::os::Transaction& t,
                                   OSDSuperblock& superblock)
 {
+  LOG_PREFIX(OSDSingletonState::trim_maps);
   epoch_t min =
     std::min(superblock.cluster_osdmap_trim_lower_bound,
              osdmaps.cached_key_lower_bound());
@@ -528,13 +530,13 @@ void OSDSingletonState::trim_maps(ceph::os::Transaction& t,
   if (min <= superblock.get_oldest_map()) {
     return;
   }
-  logger().debug("{}: min={} oldest_map={}", __func__, min,  superblock.get_oldest_map());
+  DEBUG("min={} oldest_map={}", min,  superblock.get_oldest_map());
 
   // Trim from the superblock's oldest_map up to `min`.
   // Break if we have exceeded the txn target size.
   while (superblock.get_oldest_map() < min &&
          t.get_num_ops() < crimson::common::local_conf()->osd_target_transaction_size) {
-    logger().debug("{}: removing old osdmap epoch {}", __func__, superblock.get_oldest_map());
+    DEBUG("removing old osdmap epoch {}", superblock.get_oldest_map());
     meta_coll->remove_map(t, superblock.get_oldest_map());
     meta_coll->remove_inc_map(t, superblock.get_oldest_map());
     superblock.maps.erase(superblock.get_oldest_map());
@@ -600,31 +602,26 @@ seastar::future<Ref<PG>> ShardServices::make_pg(
 
 seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
   std::unique_ptr<PGCreateInfo> info) {
+  LOG_PREFIX(OSDSingletonState::trim_maps);
   return seastar::do_with(
     std::move(info),
-    [this](auto &info)
+    [FNAME, this](auto &info)
     -> seastar::future<Ref<PG>> {
       return get_map(info->epoch).then(
-	[&info, this](cached_map_t startmap)
+	[&info, FNAME, this](cached_map_t startmap)
 	-> seastar::future<std::tuple<Ref<PG>, cached_map_t>> {
 	  const spg_t &pgid = info->pgid;
 	  if (info->by_mon) {
 	    int64_t pool_id = pgid.pgid.pool();
 	    const pg_pool_t *pool = get_map()->get_pg_pool(pool_id);
 	    if (!pool) {
-	      logger().debug(
-		"{} ignoring pgid {}, pool dne",
-		__func__,
-		pgid);
+	      DEBUG("ignoring pgid {}, pool dne", pgid);
 	      local_state.pg_map.pg_creation_canceled(pgid);
 	      return seastar::make_ready_future<
 		std::tuple<Ref<PG>, OSDMapService::cached_map_t>
 		>(std::make_tuple(Ref<PG>(), startmap));
 	    } else if (!pool->is_crimson()) {
-	      logger().debug(
-		"{} ignoring pgid {}, pool lacks crimson flag",
-		__func__,
-		pgid);
+	      DEBUG("ignoring pgid {}, pool lacks crimson flag", pgid);
 	      local_state.pg_map.pg_creation_canceled(pgid);
 	      return seastar::make_ready_future<
 		std::tuple<Ref<PG>, OSDMapService::cached_map_t>
@@ -636,10 +633,7 @@ seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
 	      // this ensures we do not process old creating messages after the
 	      // pool's initial pgs have been created (and pg are subsequently
 	      // allowed to split or merge).
-	      logger().debug(
-		"{} dropping {} create, pool does not have CREATING flag set",
-		__func__,
-		pgid);
+	      DEBUG("dropping {} create, pool does not have CREATING flag set", pgid);
 	      local_state.pg_map.pg_creation_canceled(pgid);
 	      return seastar::make_ready_future<
 		std::tuple<Ref<PG>, OSDMapService::cached_map_t>
@@ -731,7 +725,8 @@ ShardServices::wait_for_pg(
 seastar::future<Ref<PG>> ShardServices::load_pg(spg_t pgid)
 
 {
-  logger().debug("{}: {}", __func__, pgid);
+  LOG_PREFIX(OSDSingletonState::load_pg);
+  DEBUG("{}", pgid);
 
   return seastar::do_with(PGMeta(get_store(), pgid), [](auto& pg_meta) {
     return pg_meta.get_epoch();
@@ -743,8 +738,8 @@ seastar::future<Ref<PG>> ShardServices::load_pg(spg_t pgid)
     return pg->read_state(&get_store()).then([pg] {
 	return seastar::make_ready_future<Ref<PG>>(std::move(pg));
     });
-  }).handle_exception([pgid](auto ep) {
-    logger().info("pg {} saw exception on load {}", pgid, ep);
+  }).handle_exception([FNAME, pgid](auto ep) {
+    INFO("pg {} saw exception on load {}", pgid, ep);
     ceph_abort("Could not load pg" == 0);
     return seastar::make_exception_future<Ref<PG>>(ep);
   });
@@ -752,12 +747,13 @@ seastar::future<Ref<PG>> ShardServices::load_pg(spg_t pgid)
 
 seastar::future<> ShardServices::dispatch_context_transaction(
   crimson::os::CollectionRef col, PeeringCtx &ctx) {
+  LOG_PREFIX(OSDSingletonState::dispatch_context_transaction);
   if (ctx.transaction.empty()) {
-    logger().debug("ShardServices::dispatch_context_transaction: empty transaction");
+    DEBUG("empty transaction");
     return seastar::now();
   }
 
-  logger().debug("ShardServices::dispatch_context_transaction: do_transaction ...");
+  DEBUG("do_transaction ...");
   auto ret = get_store().do_transaction(
     col,
     std::move(ctx.transaction));
@@ -768,10 +764,11 @@ seastar::future<> ShardServices::dispatch_context_transaction(
 seastar::future<> ShardServices::dispatch_context_messages(
   BufferedRecoveryMessages &&ctx)
 {
+  LOG_PREFIX(OSDSingletonState::dispatch_context_transaction);
   auto ret = seastar::parallel_for_each(std::move(ctx.message_map),
-    [this](auto& osd_messages) {
+    [FNAME, this](auto& osd_messages) {
       auto& [peer, messages] = osd_messages;
-      logger().debug("dispatch_context_messages sending messages to {}", peer);
+      DEBUG("sending messages to {}", peer);
       return seastar::parallel_for_each(
         std::move(messages), [=, peer=peer, this](auto& m) {
         return send_to_osd(peer, std::move(m), local_state.osdmap->get_epoch());
@@ -799,19 +796,19 @@ seastar::future<MURef<MOSDMap>> OSDSingletonState::build_incremental_map_msg(
   epoch_t first,
   epoch_t last)
 {
+  LOG_PREFIX(OSDSingletonState::build_incremental_map_msg);
   return seastar::do_with(crimson::common::local_conf()->osd_map_message_max,
                           crimson::make_message<MOSDMap>(
                             monc.get_fsid(),
                             osdmap->get_encoding_features()),
-                          [this, &first, last](unsigned int map_message_max,
+                          [this, &first, FNAME, last](unsigned int map_message_max,
                                                       auto& m) {
     m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
     m->newest_map = superblock.get_newest_map();
     auto maybe_handle_mapgap = seastar::now();
     if (first < superblock.cluster_osdmap_trim_lower_bound) {
-      logger().info("{}: cluster osdmap lower bound: {} "
-                    " > first {}, starting with full map",
-                    __func__, superblock.cluster_osdmap_trim_lower_bound, first);
+      INFO("cluster osdmap lower bound: {}  > first {}, starting with full map",
+	    superblock.cluster_osdmap_trim_lower_bound, first);
       // we don't have the next map the target wants,
       // so start with a full map.
       first = superblock.cluster_osdmap_trim_lower_bound;
@@ -856,14 +853,13 @@ seastar::future<> OSDSingletonState::send_incremental_map(
   crimson::net::Connection &conn,
   epoch_t first)
 {
+  LOG_PREFIX(OSDSingletonState::send_incremental_map);
   epoch_t to = osdmap->get_epoch();
-  logger().info("{}: first osdmap: {} "
-                "superblock's oldest map: {}, "
-                "to {}",
-                __func__, first, superblock.get_oldest_map(), to);
+  INFO("first osdmap: {} superblock's oldest map: {}, to {}",
+       first, superblock.get_oldest_map(), to);
   if (to > first && (int64_t)(to - first) > crimson::common::local_conf()->osd_map_share_max_epochs) {
-    logger().debug("{} {} > max epochs to send of {}, only sending most recent,",
-                   __func__, (to - first), crimson::common::local_conf()->osd_map_share_max_epochs);
+    DEBUG("{} > max epochs to send of {}, only sending most recent,",
+	  (to - first), crimson::common::local_conf()->osd_map_share_max_epochs);
     first = to - crimson::common::local_conf()->osd_map_share_max_epochs;
   }
   return build_incremental_map_msg(first, to).then([&conn](auto&& m) {
@@ -875,8 +871,9 @@ seastar::future<> OSDSingletonState::send_incremental_map_to_osd(
   int osd,
   epoch_t first)
 {
+  LOG_PREFIX(OSDSingletonState::send_incremental_map);
   if (osdmap->is_down(osd)) {
-    logger().info("{}: osd.{} is_down", __func__, osd);
+    INFO("osd.{} is_down", osd);
     return seastar::now();
   } else {
     auto conn = cluster_msgr.connect(

From db3addf01ca7a99b7bd383c3003c09ba38c2e3f0 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 10 Jan 2024 09:43:45 -0800
Subject: [PATCH 1493/2492] crimson/osd/shard_services: retain map references
 in OSDSingletonState::store_maps

Introduced: 3f11cd94
Fixes: https://tracker.ceph.com/issues/63996
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd_meta.cc       | 16 ++++++++--------
 src/crimson/osd/osd_meta.h        |  5 +++--
 src/crimson/osd/shard_services.cc | 30 +++++++++++++++++++-----------
 3 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/src/crimson/osd/osd_meta.cc b/src/crimson/osd/osd_meta.cc
index 551229a4c6eb..0317da84020f 100644
--- a/src/crimson/osd/osd_meta.cc
+++ b/src/crimson/osd/osd_meta.cc
@@ -105,31 +105,31 @@ OSDMeta::load_final_pool_info(int64_t pool) {
 
 void OSDMeta::store_final_pool_info(
   ceph::os::Transaction &t,
-  OSDMap* lastmap,
-  std::map<epoch_t, OSDMap*> &added_map)
+  LocalOSDMapRef previous,
+  std::map<epoch_t, LocalOSDMapRef> &added_map)
 {
   for (auto [e, map] : added_map) {
-    if (!lastmap) {
-      lastmap = map;
+    if (!previous) {
+      previous = map;
       continue;
     }
-    for (auto &[pool_id, pool] : lastmap->get_pools()) {
+    for (auto &[pool_id, pool] : previous->get_pools()) {
       if (!map->have_pg_pool(pool_id)) {
 	ghobject_t obj = final_pool_info_oid(pool_id);
 	bufferlist bl;
 	encode(pool, bl, CEPH_FEATURES_ALL);
-	string name = lastmap->get_pool_name(pool_id);
+	string name = previous->get_pool_name(pool_id);
 	encode(name, bl);
 	std::map<string, string> profile;
 	if (pool.is_erasure()) {
-	  profile = lastmap->get_erasure_code_profile(
+	  profile = previous->get_erasure_code_profile(
 	    pool.erasure_code_profile);
 	}
 	encode(profile, bl);
 	t.write(coll->get_cid(), obj, 0, bl.length(), bl);
       }
     }
-    lastmap = map;
+    previous = map;
   }
 }
 
diff --git a/src/crimson/osd/osd_meta.h b/src/crimson/osd/osd_meta.h
index 506007e397d6..0c6738aed8fe 100644
--- a/src/crimson/osd/osd_meta.h
+++ b/src/crimson/osd/osd_meta.h
@@ -6,6 +6,7 @@
 #include <map>
 #include <string>
 #include <seastar/core/future.hh>
+#include "osd/OSDMap.h"
 #include "osd/osd_types.h"
 #include "crimson/os/futurized_collection.h"
 #include "crimson/os/futurized_store.h"
@@ -63,8 +64,8 @@ class OSDMeta {
 			     ec_profile_t>> load_final_pool_info(int64_t pool);
   void store_final_pool_info(
     ceph::os::Transaction&,
-    OSDMap* lastmap,
-    std::map<epoch_t, OSDMap*>&);
+    LocalOSDMapRef lastmap,
+    std::map<epoch_t, LocalOSDMapRef>&);
 private:
   static ghobject_t osdmap_oid(epoch_t epoch);
   static ghobject_t inc_osdmap_oid(epoch_t epoch);
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 26993daa659e..0a6a739af274 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -466,12 +466,13 @@ seastar::future<std::unique_ptr<OSDMap>> OSDSingletonState::load_map(epoch_t e)
   });
 }
 
-seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
-                                  epoch_t start, Ref<MOSDMap> m)
+seastar::future<> OSDSingletonState::store_maps(
+  ceph::os::Transaction& t,
+  epoch_t start, Ref<MOSDMap> m)
 {
   LOG_PREFIX(OSDSingletonState::store_maps);
   return seastar::do_with(
-    std::map<epoch_t, OSDMap*>(),
+    std::map<epoch_t, local_cached_map_t>(),
     [&t, FNAME, m, start, this](auto &added_maps) {
     return seastar::do_for_each(
       boost::make_counting_iterator(start),
@@ -482,8 +483,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	o->decode(p->second);
 	INFO("storing osdmap.{}", e);
 	store_map_bl(t, e, std::move(std::move(p->second)));
-	added_maps.emplace(e, o.get());
-	osdmaps.insert(e, std::move(o));
+	added_maps.emplace(e, osdmaps.insert(e, std::move(o)));
 	return seastar::now();
       } else if (auto p = m->incremental_maps.find(e);
 		 p != m->incremental_maps.end()) {
@@ -500,8 +500,7 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	  o->encode(fbl, inc.encode_features | CEPH_FEATURE_RESERVED);
 	  INFO("storing osdmap.{}", o->get_epoch());
 	  store_map_bl(t, e, std::move(fbl));
-	  added_maps.emplace(e, o.get());
-	  osdmaps.insert(e, std::move(o));
+	  added_maps.emplace(e, osdmaps.insert(e, std::move(o)));
 	  return seastar::now();
 	});
       } else {
@@ -509,10 +508,19 @@ seastar::future<> OSDSingletonState::store_maps(ceph::os::Transaction& t,
 	return seastar::now();
       }
     }).then([&t, FNAME, this, &added_maps] {
-      auto [e, map] = *added_maps.begin();
-      auto lastmap = osdmaps.find(e - 1).get();
-      meta_coll->store_final_pool_info(t, lastmap, added_maps);
-      return seastar::now();
+      epoch_t last_map_epoch = superblock.get_newest_map();
+      auto last_map_fut = last_map_epoch > 0
+	? get_local_map(last_map_epoch)
+	: seastar::make_ready_future<local_cached_map_t>();
+      return last_map_fut.then(
+	[&t, FNAME, last_map_epoch, this, &added_maps](auto lastmap) {
+	INFO("storing final pool info lastmap epoch {}, added maps {}->{}",
+	     last_map_epoch,
+	     added_maps.begin()->first,
+	     added_maps.rbegin()->first);
+	meta_coll->store_final_pool_info(t, lastmap, added_maps);
+	return seastar::now();
+      });
     });
   });
 }

From 2a6c1dc36d97b71986885708b9c78b7ba69ad75f Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 25 Jan 2024 12:08:50 +0530
Subject: [PATCH 1494/2492] qa: `fs volume rename` requires `fs fail` and
 `refuse_client_session` set

Introduced-by: fe3a4b9683d7b6f72f57f5ed8bc324bdbb24351f
Fixes: http://tracker.ceph.com/issues/64174
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/suites/fs/cephadm/renamevolume/1-rename.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qa/suites/fs/cephadm/renamevolume/1-rename.yaml b/qa/suites/fs/cephadm/renamevolume/1-rename.yaml
index 7f9bc890617a..e1d5b9b33cd2 100644
--- a/qa/suites/fs/cephadm/renamevolume/1-rename.yaml
+++ b/qa/suites/fs/cephadm/renamevolume/1-rename.yaml
@@ -1,7 +1,11 @@
 tasks:
 - cephadm.shell:
     host.a:
+      - ceph fs fail foo
+      - ceph fs set foo refuse_client_session true
       - ceph fs volume rename foo bar --yes-i-really-mean-it
+      - ceph fs set bar joinable true
+      - ceph fs set bar refuse_client_session false
 - fs.ready:
     timeout: 300
 - cephadm.shell:

From 129f1db734777f3df551965c35f40ee0ab7d467a Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 23 Jan 2024 22:17:46 +0530
Subject: [PATCH 1495/2492] mgr/dashboard: create cephfs snapshot clone

Fixes: https://tracker.ceph.com/issues/64175
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       | 44 +++++++++-
 .../cephfs-subvolume-list.component.ts        |  2 +-
 ...phfs-subvolume-snapshots-list.component.ts | 80 ++++++++++++++++++-
 .../shared/api/cephfs-subvolume.service.ts    | 22 +++++
 .../form-modal/form-modal.component.ts        |  8 +-
 .../models/cd-form-modal-field-config.ts      |  3 +-
 src/pybind/mgr/dashboard/openapi.yaml         | 55 +++++++++++++
 7 files changed, 203 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index d747ba1e0f02..01827e3ef0ec 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import errno
 import json
 import logging
 import os
@@ -693,11 +694,17 @@ def get(self, vol_name: str, group_name: str = "", info=True):
                 params['sub_name'] = subvolume['name']
                 error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_info', None,
                                                   params)
-                if error_code != 0:
+                # just ignore this error for now so the subvolumes page will load.
+                # the ideal solution is to implement a status page where clone status
+                # can be displayed
+                if error_code == -errno.EAGAIN:
+                    pass
+                elif error_code != 0:
                     raise DashboardException(
                         f'Failed to get info for subvolume {subvolume["name"]}: {err}'
                     )
-                subvolume['info'] = json.loads(out)
+                if out:
+                    subvolume['info'] = json.loads(out)
         return subvolumes
 
     @RESTController.Resource('GET')
@@ -857,11 +864,17 @@ def get(self, vol_name: str, subvol_name, group_name: str = '', info=True):
                 params['snap_name'] = snapshot['name']
                 error_code, out, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_info',
                                                   None, params)
-                if error_code != 0:
+                # just ignore this error for now so the subvolumes page will load.
+                # the ideal solution is to implement a status page where clone status
+                # can be displayed
+                if error_code == -errno.EAGAIN:
+                    pass
+                elif error_code != 0:
                     raise DashboardException(
                         f'Failed to get info for subvolume snapshot {snapshot["name"]}: {err}'
                     )
-                snapshot['info'] = json.loads(out)
+                if out:
+                    snapshot['info'] = json.loads(out)
         return snapshots
 
     @RESTController.Resource('GET')
@@ -905,6 +918,29 @@ def delete(self, vol_name: str, subvol_name: str, snap_name: str, group_name='',
         return f'Subvolume snapshot {snap_name} removed successfully'
 
 
+@APIRouter('/cephfs/subvolume/snapshot/clone', Scope.CEPHFS)
+@APIDoc("Cephfs Snapshot Clone Management API", "CephfsSnapshotClone")
+class CephFsSnapshotClone(RESTController):
+    @EndpointDoc("Create a clone of a subvolume snapshot")
+    def create(self, vol_name: str, subvol_name: str, snap_name: str, clone_name: str,
+               group_name='', target_group_name=''):
+        params = {'vol_name': vol_name, 'sub_name': subvol_name, 'snap_name': snap_name,
+                  'target_sub_name': clone_name}
+        if group_name:
+            params['group_name'] = group_name
+
+        if target_group_name:
+            params['target_group_name'] = target_group_name
+
+        error_code, _, err = mgr.remote('volumes', '_cmd_fs_subvolume_snapshot_clone', None,
+                                        params)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to create clone {clone_name}: {err}'
+            )
+        return f'Clone {clone_name} created successfully'
+
+
 @APIRouter('/cephfs/snaphost/schedule', Scope.CEPHFS)
 @APIDoc("Cephfs Snapshot Scheduling API", "CephFSSnapshotSchedule")
 class CephFSSnapshotSchedule(RESTController):
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
index 4f9cf27db0ff..92c139f8e5dd 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
@@ -245,7 +245,7 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
       switchMap(() =>
         this.cephfsSubVolumeService.get(this.fsName, this.activeGroupName).pipe(
           catchError(() => {
-            this.context.error();
+            this.context?.error();
             return of(null);
           })
         )
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
index 798307a0cf9f..3cb5b0bd47b5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-list.component.ts
@@ -20,6 +20,12 @@ import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
 import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
 import { FinishedTask } from '~/app/shared/models/finished-task';
 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+import { FormModalComponent } from '~/app/shared/components/form-modal/form-modal.component';
+import { NotificationService } from '~/app/shared/services/notification.service';
+import { NotificationType } from '~/app/shared/enum/notification-type.enum';
+import moment from 'moment';
+import { Validators } from '@angular/forms';
+import { CdValidators } from '~/app/shared/forms/cd-validators';
 
 @Component({
   selector: 'cd-cephfs-subvolume-snapshots-list',
@@ -59,7 +65,8 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
     private modalService: ModalService,
     private authStorageService: AuthStorageService,
     private cdDatePipe: CdDatePipe,
-    private taskWrapper: TaskWrapperService
+    private taskWrapper: TaskWrapperService,
+    private notificationService: NotificationService
   ) {
     this.permissions = this.authStorageService.getPermissions();
   }
@@ -99,9 +106,17 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
         click: () => this.openModal()
       },
       {
-        name: this.actionLabels.REMOVE,
+        name: this.actionLabels.CLONE,
+        permission: 'create',
+        icon: Icons.clone,
+        disable: () => !this.selection.hasSingleSelection,
+        click: () => this.cloneModal()
+      },
+      {
+        name: this.actionLabels.DELETE,
         permission: 'delete',
         icon: Icons.destroy,
+        disable: () => !this.selection.hasSingleSelection,
         click: () => this.deleteSnapshot()
       }
     ];
@@ -209,7 +224,7 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
     const subVolumeGroupName = this.activeGroupName;
     const fsName = this.fsName;
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
-      actionDescription: 'Remove',
+      actionDescription: 'Delete',
       itemNames: [snapshotName],
       itemDescription: 'Snapshot',
       submitAction: () =>
@@ -234,4 +249,63 @@ export class CephfsSubvolumeSnapshotsListComponent implements OnInit, OnChanges
           })
     });
   }
+
+  cloneModal() {
+    const cloneName = `clone_${moment().toISOString(true)}`;
+    const allGroups = Array.from(this.subvolumeGroupList).map((group) => {
+      return { value: group, text: group === '' ? '_nogroup' : group };
+    });
+    this.modalService.show(FormModalComponent, {
+      titleText: $localize`Create clone`,
+      fields: [
+        {
+          type: 'text',
+          name: 'cloneName',
+          value: cloneName,
+          label: $localize`Name`,
+          validators: [Validators.required, Validators.pattern(/^[.A-Za-z0-9_+:-]+$/)],
+          asyncValidators: [
+            CdValidators.unique(
+              this.cephfsSubvolumeService.exists,
+              this.cephfsSubvolumeService,
+              null,
+              null,
+              this.fsName
+            )
+          ],
+          required: true,
+          errors: {
+            pattern: $localize`Allowed characters are letters, numbers, '.', '-', '+', ':' or '_'`
+          }
+        },
+        {
+          type: 'select',
+          name: 'groupName',
+          value: this.activeGroupName,
+          label: $localize`Group Name`,
+          typeConfig: {
+            options: allGroups
+          }
+        }
+      ],
+      submitButtonText: $localize`Create Clone`,
+      onSubmit: (value: any) => {
+        this.cephfsSubvolumeService
+          .createSnapshotClone(
+            this.fsName,
+            this.activeSubVolumeName,
+            this.selection.first().name,
+            value.cloneName,
+            this.activeGroupName,
+            value.groupName
+          )
+          .subscribe(() =>
+            this.notificationService.show(
+              NotificationType.success,
+              $localize`Created Clone "${value.cloneName}" successfully.`
+            )
+          );
+      }
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
index ad0ce248064d..6a88fa1d5297 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
@@ -171,4 +171,26 @@ export class CephfsSubvolumeService {
       observe: 'response'
     });
   }
+
+  createSnapshotClone(
+    fsName: string,
+    subVolumeName: string,
+    snapshotName: string,
+    cloneName: string,
+    groupName = '',
+    targetGroupName = ''
+  ) {
+    return this.http.post(
+      `${this.baseURL}/snapshot/clone`,
+      {
+        vol_name: fsName,
+        subvol_name: subVolumeName,
+        snap_name: snapshotName,
+        clone_name: cloneName,
+        group_name: groupName,
+        target_group_name: targetGroupName
+      },
+      { observe: 'response' }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-modal/form-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-modal/form-modal.component.ts
index 59b0d2a8560a..1b4af6cd69fc 100755
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-modal/form-modal.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-modal/form-modal.component.ts
@@ -1,5 +1,5 @@
 import { Component, OnInit } from '@angular/core';
-import { UntypedFormControl, ValidatorFn, Validators } from '@angular/forms';
+import { AsyncValidatorFn, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms';
 
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
 import _ from 'lodash';
@@ -47,18 +47,22 @@ export class FormModalComponent implements OnInit {
 
   private createFormControl(field: CdFormModalFieldConfig): UntypedFormControl {
     let validators: ValidatorFn[] = [];
+    let asyncValidators: AsyncValidatorFn[] = [];
     if (_.isBoolean(field.required) && field.required) {
       validators.push(Validators.required);
     }
     if (field.validators) {
       validators = validators.concat(field.validators);
     }
+    if (field.asyncValidators) {
+      asyncValidators = asyncValidators.concat(field.asyncValidators);
+    }
     return new UntypedFormControl(
       _.defaultTo(
         field.type === 'binary' ? this.dimlessBinaryPipe.transform(field.value) : field.value,
         null
       ),
-      { validators }
+      { validators, asyncValidators }
     );
   }
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-form-modal-field-config.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-form-modal-field-config.ts
index e327be59a27a..a899e6daa690 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-form-modal-field-config.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-form-modal-field-config.ts
@@ -1,4 +1,4 @@
-import { ValidatorFn } from '@angular/forms';
+import { AsyncValidatorFn, ValidatorFn } from '@angular/forms';
 
 export class CdFormModalFieldConfig {
   // --- Generic field properties ---
@@ -11,6 +11,7 @@ export class CdFormModalFieldConfig {
   value?: any;
   errors?: { [errorName: string]: string };
   validators: ValidatorFn[];
+  asyncValidators?: AsyncValidatorFn[];
 
   // --- Specific field properties ---
   typeConfig?: {
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index fb2b7059a53a..1ea06dd43362 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -2065,6 +2065,59 @@ paths:
       - jwt: []
       tags:
       - CephfsSubvolumeSnapshot
+  /api/cephfs/subvolume/snapshot/clone:
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                clone_name:
+                  type: string
+                group_name:
+                  default: ''
+                  type: string
+                snap_name:
+                  type: string
+                subvol_name:
+                  type: string
+                target_group_name:
+                  default: ''
+                  type: string
+                vol_name:
+                  type: string
+              required:
+              - vol_name
+              - subvol_name
+              - snap_name
+              - clone_name
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Create a clone of a subvolume snapshot
+      tags:
+      - CephfsSnapshotClone
   /api/cephfs/subvolume/snapshot/{vol_name}/{subvol_name}:
     delete:
       parameters:
@@ -13114,6 +13167,8 @@ tags:
   name: CephFSSubvolume
 - description: Cephfs Management API
   name: Cephfs
+- description: Cephfs Snapshot Clone Management API
+  name: CephfsSnapshotClone
 - description: Cephfs Subvolume Group Management API
   name: CephfsSubvolumeGroup
 - description: Cephfs Subvolume Snapshot Management API

From 946a26c41e9f3a4f6d62fa3d1286c72985774c04 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Thu, 25 Jan 2024 16:51:06 +0530
Subject: [PATCH 1496/2492] mgr/dashboard:  Code refactor rgw migrate component
 for using correctly the MIGRATE action verb

fixes https://tracker.ceph.com/issues/64152

this.MIGRATE = $localize`Migrate to Multi-Site`;

Just like other action verbs we should set this.Migrate = "MIGRATE" only.
This will require rephrasing in the following places as well:
1. https://github.com/ceph/ceph/blob/d3256c484136a1b32b79a904861f681a9248ba3c/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-details/rgw-multisite-details.component.ts#L223-L228

2. https://github.com/ceph/ceph/blob/d3256c484136a1b32b79a904861f681a9248ba3c/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.ts#L180-L18

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../rgw-multisite-migrate/rgw-multisite-migrate.component.html  | 2 +-
 .../rgw-multisite-migrate/rgw-multisite-migrate.component.ts    | 2 +-
 .../frontend/src/app/shared/constants/app.constants.ts          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
index b18c5a0b9d76..51f72dd7f897 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.html
@@ -146,7 +146,7 @@
     </div>
     <div class="modal-footer">
       <cd-form-button-panel (submitActionEvent)="submit()"
-                            [submitText]="actionLabels.MIGRATE"
+                            [submitText]="actionLabels.MIGRATE + ' ' + 'to Multi-site'"
                             [form]="multisiteMigrateForm"></cd-form-button-panel>
     </div>
     </form>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.ts
index 4c2f53b6af1f..1073dee429a5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-migrate/rgw-multisite-migrate.component.ts
@@ -181,7 +181,7 @@ export class RgwMultisiteMigrateComponent implements OnInit {
       () => {
         this.notificationService.show(
           NotificationType.success,
-          $localize`${this.actionLabels.MIGRATE} done successfully`
+          $localize`Migration done successfully`
         );
         this.submitAction.emit();
         this.activeModal.close();
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index 0f41e2832204..7edce5ff6671 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -150,7 +150,7 @@ export class ActionLabelsI18n {
 
     this.IMPORT = $localize`Import`;
 
-    this.MIGRATE = $localize`Migrate to Multi-site`;
+    this.MIGRATE = $localize`Migrate`;
 
     /* Destroy an existing item */
     this.DELETE = $localize`Delete`;

From 191fda84b38b364fe686a273c1b6e8f8a6156d22 Mon Sep 17 00:00:00 2001
From: cloudbehl <cloudbehl@gmail.com>
Date: Thu, 25 Jan 2024 18:28:37 +0530
Subject: [PATCH 1497/2492] mgr/dashboard: Fixing RGW graph panels - Fixing
 grafana panels for rgw dashboards - Fixing RGW overview dashboard queries

fixes https://tracker.ceph.com/issues/64177

Signed-off-by: cloudbehl <cloudbehl@gmail.com>
---
 .../ceph-mixin/dashboards/rgw.libsonnet       | 36 +++++++++----------
 .../app/shared/enum/dashboard-promqls.enum.ts |  8 ++---
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards/rgw.libsonnet b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
index 932eb2122ca7..a660f870a0f3 100644
--- a/monitoring/ceph-mixin/dashboards/rgw.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/rgw.libsonnet
@@ -263,8 +263,8 @@ local u = import 'utils.libsonnet';
         'short',
         |||
           label_replace(
-            rate(ceph_rgw_get_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-              rate(ceph_rgw_get_initial_lat_count{%(matchers)s}[$__rate_interval]) *
+            rate(ceph_rgw_op_get_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+              rate(ceph_rgw_op_get_obj_lat_count{%(matchers)s}[$__rate_interval]) *
               on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s},
             "rgw_host", "$1", "ceph_daemon", "rgw.(.*)"
           )
@@ -279,8 +279,8 @@ local u = import 'utils.libsonnet';
           $.addTargetSchema(
             |||
               label_replace(
-                rate(ceph_rgw_put_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-                  rate(ceph_rgw_put_initial_lat_count{%(matchers)s}[$__rate_interval]) *
+                rate(ceph_rgw_op_put_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+                  rate(ceph_rgw_op_put_obj_lat_count{%(matchers)s}[$__rate_interval]) *
                   on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s},
                 "rgw_host", "$1", "ceph_daemon", "rgw.(.*)"
               )
@@ -316,8 +316,8 @@ local u = import 'utils.libsonnet';
         'short',
         |||
           label_replace(
-            rate(ceph_rgw_get_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-              rate(ceph_rgw_get_initial_lat_count{%(matchers)s}[$__rate_interval]) *
+            rate(ceph_rgw_op_get_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+              rate(ceph_rgw_op_get_obj_lat_count{%(matchers)s}[$__rate_interval]) *
               on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s},
             "rgw_host", "$1", "ceph_daemon", "rgw.(.*)"
           )
@@ -333,14 +333,14 @@ local u = import 'utils.libsonnet';
         'Total bytes transferred in/out of all radosgw instances within the cluster',
         'bytes',
         'short',
-        'sum(rate(ceph_rgw_get_b{%(matchers)s}[$__rate_interval]))' % $.matchers(),
+        'sum(rate(ceph_rgw_op_get_obj_bytes{%(matchers)s}[$__rate_interval]))' % $.matchers(),
         'GETs',
         0,
         8,
         8,
         6
       ).addTargets(
-        [$.addTargetSchema('sum(rate(ceph_rgw_put_b{%(matchers)s}[$__rate_interval]))' % $.matchers(),
+        [$.addTargetSchema('sum(rate(ceph_rgw_op_put_obj_bytes{%(matchers)s}[$__rate_interval]))' % $.matchers(),
                            'PUTs')]
       ),
       RgwOverviewPanel(
@@ -350,8 +350,8 @@ local u = import 'utils.libsonnet';
         'short',
         |||
           label_replace(sum by (instance_id) (
-            rate(ceph_rgw_get_b{%(matchers)s}[$__rate_interval]) +
-              rate(ceph_rgw_put_b{%(matchers)s}[$__rate_interval])) *
+            rate(ceph_rgw_op_get_obj_bytes{%(matchers)s}[$__rate_interval]) +
+              rate(ceph_rgw_op_put_obj_bytes{%(matchers)s}[$__rate_interval])) *
               on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s},
             "rgw_host", "$1", "ceph_daemon", "rgw.(.*)"
           )
@@ -369,8 +369,8 @@ local u = import 'utils.libsonnet';
         'short',
         |||
           label_replace(
-            rate(ceph_rgw_put_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-              rate(ceph_rgw_put_initial_lat_count{%(matchers)s}[$__rate_interval]) *
+            rate(ceph_rgw_op_put_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+              rate(ceph_rgw_op_put_obj_lat_count{%(matchers)s}[$__rate_interval]) *
               on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s},
             "rgw_host", "$1", "ceph_daemon", "rgw.(.*)"
           )
@@ -736,14 +736,14 @@ local u = import 'utils.libsonnet';
         'short',
         |||
           sum by (instance_id) (
-            rate(ceph_rgw_get_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-              rate(ceph_rgw_get_initial_lat_count{%(matchers)s}[$__rate_interval])
+            rate(ceph_rgw_op_get_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+              rate(ceph_rgw_op_get_obj_lat_count{%(matchers)s}[$__rate_interval])
           ) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s, ceph_daemon=~"$rgw_servers"}
         ||| % $.matchers(),
         |||
           sum by (instance_id) (
-            rate(ceph_rgw_put_initial_lat_sum{%(matchers)s}[$__rate_interval]) /
-              rate(ceph_rgw_put_initial_lat_count{%(matchers)s}[$__rate_interval])
+            rate(ceph_rgw_op_put_obj_lat_sum{%(matchers)s}[$__rate_interval]) /
+              rate(ceph_rgw_op_put_obj_lat_count{%(matchers)s}[$__rate_interval])
           ) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s, ceph_daemon=~"$rgw_servers"}
         ||| % $.matchers(),
         'GET {{ceph_daemon}}',
@@ -760,11 +760,11 @@ local u = import 'utils.libsonnet';
         'bytes',
         'short',
         |||
-          rate(ceph_rgw_get_b{%(matchers)s}[$__rate_interval]) *
+          rate(ceph_rgw_op_get_obj_bytes{%(matchers)s}[$__rate_interval]) *
             on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{%(matchers)s, ceph_daemon=~"$rgw_servers"}
         ||| % $.matchers(),
         |||
-          rate(ceph_rgw_put_b{%(matchers)s}[$__rate_interval]) *
+          rate(ceph_rgw_op_put_obj_bytes{%(matchers)s}[$__rate_interval]) *
             on (instance_id) group_left (ceph_daemon)
             ceph_rgw_metadata{%(matchers)s, ceph_daemon=~"$rgw_servers"}
         ||| % $.matchers(),
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
index 515fefcdb610..2d8aa22819dc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
@@ -11,8 +11,8 @@ export enum Promqls {
 
 export enum RgwPromqls {
   RGW_REQUEST_PER_SECOND = 'sum(rate(ceph_rgw_req[1m]))',
-  AVG_GET_LATENCY = 'sum(rate(ceph_rgw_get_initial_lat_sum[1m])) / sum(rate(ceph_rgw_get_initial_lat_count[1m]))',
-  AVG_PUT_LATENCY = 'sum(rate(ceph_rgw_put_initial_lat_sum[1m])) / sum(rate(ceph_rgw_put_initial_lat_count[1m]))',
-  GET_BANDWIDTH = 'sum(rate(ceph_rgw_get_b[1m]))',
-  PUT_BANDWIDTH = 'sum(rate(ceph_rgw_put_b[1m]))'
+  AVG_GET_LATENCY = 'sum(rate(ceph_rgw_op_get_obj_lat_sum[1m])) / sum(rate(ceph_rgw_op_get_obj_lat_count[1m]))',
+  AVG_PUT_LATENCY = 'sum(rate(ceph_rgw_op_put_obj_lat_sum[1m])) / sum(rate(ceph_rgw_op_put_obj_lat_count[1m]))',
+  GET_BANDWIDTH = 'sum(rate(ceph_rgw_op_get_obj_bytes[1m]))',
+  PUT_BANDWIDTH = 'sum(rate(ceph_rgw_op_put_obj_bytes[1m]))'
 }

From 1c402576529edafdb8aa0aef241965e06fa4c151 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 8 Mar 2023 15:27:57 +0100
Subject: [PATCH 1498/2492] First hardware-monitoring draft version

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/cephadm/node-proxy/data.py                |  93 +++++++++++
 .../fake_cephadm/cephadm_mgr_module.py        |  51 +++++++
 src/cephadm/node-proxy/main.py                |  12 ++
 src/cephadm/node-proxy/redfish-test.py        |  27 ++++
 src/cephadm/node-proxy/redfish_client.py      |  37 +++++
 .../redfish_json_samples/interface_sample     |  19 +++
 .../redfish_json_samples/interfaces_sample    |  21 +++
 .../node-proxy/redfish_json_samples/memory    |  65 ++++++++
 .../redfish_json_samples/memory_socket        |  21 +++
 .../node-proxy/redfish_json_samples/processor | 117 ++++++++++++++
 .../redfish_json_samples/processors           |  13 ++
 .../redfish_json_samples/storage_sample       |  19 +++
 .../node-proxy/redfish_json_samples/system    | 144 ++++++++++++++++++
 src/cephadm/node-proxy/redfish_system.py      |  91 +++++++++++
 src/cephadm/node-proxy/reporter.py            |  28 ++++
 src/cephadm/node-proxy/server.py              |  49 ++++++
 src/cephadm/node-proxy/system.py              |  29 ++++
 17 files changed, 836 insertions(+)
 create mode 100644 src/cephadm/node-proxy/data.py
 create mode 100644 src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py
 create mode 100644 src/cephadm/node-proxy/main.py
 create mode 100644 src/cephadm/node-proxy/redfish-test.py
 create mode 100644 src/cephadm/node-proxy/redfish_client.py
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/interface_sample
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/interfaces_sample
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/memory
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/memory_socket
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/processor
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/processors
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/storage_sample
 create mode 100644 src/cephadm/node-proxy/redfish_json_samples/system
 create mode 100644 src/cephadm/node-proxy/redfish_system.py
 create mode 100644 src/cephadm/node-proxy/reporter.py
 create mode 100644 src/cephadm/node-proxy/server.py
 create mode 100644 src/cephadm/node-proxy/system.py

diff --git a/src/cephadm/node-proxy/data.py b/src/cephadm/node-proxy/data.py
new file mode 100644
index 000000000000..b8e69f39078b
--- /dev/null
+++ b/src/cephadm/node-proxy/data.py
@@ -0,0 +1,93 @@
+
+system_1 = {
+
+    'metadata': {
+        'name': 'xx',
+        'manufacturer': 'Dell',
+        'model': 'HP PowerEdge',
+        'chassis': 'xxx',
+        'xxx': '',
+    },
+
+    'status': {
+        'State': 'Enabled',
+        'Health': 'OK'
+    },
+
+    'processor': {
+        'description': '',
+        'count': '',
+        'type': '',
+        'model': '',
+        'temperature': '',
+        'status': {
+            'State': 'Enabled',
+            'Health': 'OK'
+        }
+    },
+
+    'memory': {
+        'description': '',
+        'total': 'xx',
+        'status': {
+            'State': 'Enabled',
+            'Health': 'OK'
+        },
+    },
+
+    'network': {
+        'interfaces': [
+            {
+                'type': 'ethernet',
+                'description': 'my ethertnet interface',
+                'name': 'name of the interface',
+                'description': 'description of the interface',
+                'speed_mbps': 'xxx',
+                'status': {
+                    'State': 'Enabled',
+                    'Health': 'OK'
+                },
+            }
+        ]
+    },
+
+    'storage': {
+        'drives': [
+            {
+                'device': 'devc',
+                'description': 'Milk, Cheese, Bread, Fruit, Vegetables',
+                'model': 'Buy groceries',
+                'type': 'ssd|rotate|nvme',
+                'capacity_bytes': '',
+                'usage_bytes': '',
+                'status': {
+                    'State': 'Enabled',
+                    'Health': 'OK'
+                },
+            }
+        ]
+    },
+
+    'power': {
+        'power_supplies': [
+            'type': 'xx',
+            'manufacturer': 'xxx',
+            'model': 'xx',
+            'properties': {},
+            'power_control': 'xx',
+            'status': {
+                'State': 'Enabled',
+                'Health': 'OK'
+            }
+        ]
+    },
+
+    'cooling': {
+        'fans': [
+            {
+                'id': 1,
+                'status': '',
+            }
+        ]
+    },
+}
diff --git a/src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py b/src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py
new file mode 100644
index 000000000000..6d46de40dd22
--- /dev/null
+++ b/src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+License: MIT License
+Copyright (c) 2023 Miel Donkers
+
+Very simple HTTP server in python for logging requests
+Usage::
+    ./server.py [<port>]
+"""
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import logging
+
+class S(BaseHTTPRequestHandler):
+    def _set_response(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_GET(self):
+        logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
+        self._set_response()
+        self.wfile.write("GET request for {}".format(self.path).encode('utf-8'))
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+        post_data = self.rfile.read(content_length) # <--- Gets the data itself
+        logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",
+                str(self.path), str(self.headers), post_data.decode('utf-8'))
+
+        self._set_response()
+        self.wfile.write("POST request for {}".format(self.path).encode('utf-8'))
+
+def run(server_class=HTTPServer, handler_class=S, port=8000):
+    logging.basicConfig(level=logging.INFO)
+    server_address = ('', port)
+    httpd = server_class(server_address, handler_class)
+    logging.info(f'Starting httpd on port {port}...\n')
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+    logging.info('Stopping httpd...\n')
+
+if __name__ == '__main__':
+    from sys import argv
+
+    if len(argv) == 2:
+        run(port=int(argv[1]))
+    else:
+        run()
diff --git a/src/cephadm/node-proxy/main.py b/src/cephadm/node-proxy/main.py
new file mode 100644
index 000000000000..45cd573192a9
--- /dev/null
+++ b/src/cephadm/node-proxy/main.py
@@ -0,0 +1,12 @@
+from redfish_system import RedfishSystem
+import time
+
+host = "https://x.x.x.x:8443"
+username = "myuser"
+password = "mypassword"
+
+system = RedfishSystem(host, username, password)
+system.start_update_loop()
+time.sleep(20)
+print(system.get_status())
+system.stop_update_loop()
diff --git a/src/cephadm/node-proxy/redfish-test.py b/src/cephadm/node-proxy/redfish-test.py
new file mode 100644
index 000000000000..3aaab2d2f097
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish-test.py
@@ -0,0 +1,27 @@
+from redfish.rest.v1 import ServerDownOrUnreachableError
+import redfish
+import sys
+
+
+login_host = "https://x.x.x.x:8443"
+login_account = "myuser"
+login_password = "mypassword"
+
+REDFISH_OBJ = redfish.redfish_client(base_url=login_host, username=login_account, password=login_password, default_prefix='/redfish/v1/')
+
+# Login
+try:
+    REDFISH_OBJ.login(auth="session")
+except ServerDownOrUnreachableError as excp:
+    sys.stderr.write("Error: server not reachable or does not support RedFish.\n")
+    sys.exit()
+
+# Get the system information /redfish/v1/Systems/1/SmartStorage/
+# /redfish/v1/Systems/1/Processors/
+# /redfish/v1/Systems/1/Memory/proc1dimm1/
+response = REDFISH_OBJ.get(sys.argv[1])
+# Print the system information
+print(response.dict)
+
+# Logout
+REDFISH_OBJ.logout()
diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
new file mode 100644
index 000000000000..f4a06ffc309b
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -0,0 +1,37 @@
+from redfish.rest.v1 import ServerDownOrUnreachableError
+import redfish
+import logging
+
+class RedFishClient:
+
+    PREFIX = '/redfish/v1'
+
+    def __init__(self, host, username, password):
+        self.host = host
+        self.username = username
+        self.password = password
+        self.redfish_obj = None
+
+    def login(self):
+        self.redfish_obj = redfish.redfish_client(base_url=self.host,
+                                                  username=self.username,
+                                                  password=self.password,
+                                                  default_prefix=self.PREFIX)
+        try:
+            self.redfish_obj.login(auth="session")
+        except ServerDownOrUnreachableError as e:
+            logging.error(f"Server not reachable or does not support RedFish {e}", e)
+
+    def get_path(self, path):
+        try:
+            if self.PREFIX not in path:
+                path = f"{self.PREFIX}{path}"
+            print(f"getting: {path}")
+            response = self.redfish_obj.get(path)
+            return response.dict
+        except Exception as e:
+            #TODO
+            pass
+
+    def logout(self):
+        self.redfish_obj.logout()
diff --git a/src/cephadm/node-proxy/redfish_json_samples/interface_sample b/src/cephadm/node-proxy/redfish_json_samples/interface_sample
new file mode 100644
index 000000000000..6d351cfbc61e
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/interface_sample
@@ -0,0 +1,19 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces/Members/$entity',
+	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/',
+	'@odata.type': '#EthernetInterface.1.0.0.EthernetInterface',
+	'Id': '1',
+	'Name': 'System Ethernet Interface',
+	'Oem': {
+		'Hp': {
+			'@odata.type': '#HpiLOEthernetNetworkInterface.1.0.0.HpiLOEthernetNetworkInterface',
+			'DHCPv4': None,
+			'DHCPv6': None,
+			'IPv4': None,
+			'IPv6': None,
+			'SharedNetworkPortOptions': None
+		}
+	},
+	'SettingsResult': None,
+	'Status': None
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/interfaces_sample b/src/cephadm/node-proxy/redfish_json_samples/interfaces_sample
new file mode 100644
index 000000000000..811a7720d791
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/interfaces_sample
@@ -0,0 +1,21 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces',
+	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/',
+	'@odata.type': '#EthernetInterfaceCollection.EthernetInterfaceCollection',
+	'Description': 'Collection of System Network Interfaces',
+	'Members': [{
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/2/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/3/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/4/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/5/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/6/'
+	}],
+	'Members@odata.count': 6,
+	'Name': 'System Network Interfaces'
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/memory b/src/cephadm/node-proxy/redfish_json_samples/memory
new file mode 100644
index 000000000000..fba0606f750a
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/memory
@@ -0,0 +1,65 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Memory',
+	'@odata.id': '/redfish/v1/Systems/1/Memory/',
+	'@odata.type': '#HpMemoryCollection.HpMemoryCollection',
+	'Description': 'Memory DIMM Collection',
+	'Members': [{
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm1/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm2/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm3/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm4/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm5/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm6/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm7/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm8/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm9/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm10/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm11/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm12/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm1/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm2/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm3/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm4/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm5/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm6/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm7/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm8/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm9/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm10/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm11/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm12/'
+	}],
+	'Members@odata.count': 24,
+	'Name': 'Memory DIMM Collection',
+	'Oem': {
+		'Hp': {
+			'@odata.type': '#HpAdvancedMemoryProtection.1.0.0.HpAdvancedMemoryProtection',
+			'AmpModeActive': 'AdvancedECC',
+			'AmpModeStatus': 'AdvancedECC',
+			'AmpModeSupported': ['AdvancedECC', 'OnlineSpareRank']
+		}
+	}
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/memory_socket b/src/cephadm/node-proxy/redfish_json_samples/memory_socket
new file mode 100644
index 000000000000..283c7d41e143
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/memory_socket
@@ -0,0 +1,21 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Memory/Members/$entity',
+	'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm1/',
+	'@odata.type': '#HpMemory.1.0.0.HpMemory',
+	'DIMMStatus': 'GoodInUse',
+	'DIMMTechnology': 'RDIMM',
+	'DIMMType': 'DDR3',
+	'DataWidth': 64,
+	'ErrorCorrection': 'SingleBitECC',
+	'HPMemoryType': 'HPSmartMemory',
+	'Id': 'proc1dimm1',
+	'Manufacturer': 'HP     ',
+	'MaximumFrequencyMHz': 1600,
+	'MinimumVoltageVoltsX10': 13,
+	'Name': 'proc1dimm1',
+	'PartNumber': '713756-081          ',
+	'Rank': 2,
+	'SizeMB': 16384,
+	'SocketLocator': 'PROC  1 DIMM  1 ',
+	'TotalWidth': 72
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/processor b/src/cephadm/node-proxy/redfish_json_samples/processor
new file mode 100644
index 000000000000..bc381fb5185c
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/processor
@@ -0,0 +1,117 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Processors/Members/$entity',
+	'@odata.id': '/redfish/v1/Systems/1/Processors/1/',
+	'@odata.type': '#Processor.1.0.0.Processor',
+	'Id': '1',
+	'InstructionSet': 'x86-64',
+	'Manufacturer': 'Intel',
+	'MaxSpeedMHz': 4800,
+	'Model': ' Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz      ',
+	'Name': 'Processors',
+	'Oem': {
+		'Hp': {
+			'@odata.type': '#HpProcessorExt.1.0.0.HpProcessorExt',
+			'AssetTag': '',
+			'Cache': [{
+				'Associativity': '8waySetAssociative',
+				'CacheSpeedns': 0,
+				'CurrentSRAMType': ['Burst'],
+				'EccType': 'SingleBitECC',
+				'InstalledSizeKB': 256,
+				'Location': 'Internal',
+				'MaximumSizeKB': 384,
+				'Name': 'Processor 1 Internal L1 Cache',
+				'Policy': 'WriteBack',
+				'Socketed': False,
+				'SupportedSRAMType': ['Burst'],
+				'SystemCacheType': 'Data'
+			}, {
+				'Associativity': '8waySetAssociative',
+				'CacheSpeedns': 0,
+				'CurrentSRAMType': ['Burst'],
+				'EccType': 'SingleBitECC',
+				'InstalledSizeKB': 2048,
+				'Location': 'Internal',
+				'MaximumSizeKB': 3072,
+				'Name': 'Processor 1 Internal L2 Cache',
+				'Policy': 'WriteBack',
+				'Socketed': False,
+				'SupportedSRAMType': ['Burst'],
+				'SystemCacheType': None
+			}, {
+				'Associativity': '20waySetAssociative',
+				'CacheSpeedns': 0,
+				'CurrentSRAMType': ['Burst'],
+				'EccType': 'SingleBitECC',
+				'InstalledSizeKB': 20480,
+				'Location': 'Internal',
+				'MaximumSizeKB': 30720,
+				'Name': 'Processor 1 Internal L3 Cache',
+				'Policy': 'WriteBack',
+				'Socketed': False,
+				'SupportedSRAMType': ['Burst'],
+				'SystemCacheType': None
+			}],
+			'Characteristics': ['64Bit'],
+			'ConfigStatus': {
+				'Populated': True,
+				'State': 'Enabled'
+			},
+			'CoresEnabled': 8,
+			'ExternalClockMHz': 100,
+			'MicrocodePatches': [{
+				'CpuId': '0x000206D2',
+				'Date': '2011-05-03T00:00:00Z',
+				'PatchId': '0x8000020C'
+			}, {
+				'CpuId': '0x000206D3',
+				'Date': '2011-04-20T00:00:00Z',
+				'PatchId': '0x80000304'
+			}, {
+				'CpuId': '0x000206D5',
+				'Date': '2011-10-13T00:00:00Z',
+				'PatchId': '0x00000513'
+			}, {
+				'CpuId': '0x000206D6',
+				'Date': '2018-01-30T00:00:00Z',
+				'PatchId': '0x0000061C'
+			}, {
+				'CpuId': '0x000206D7',
+				'Date': '2018-01-26T00:00:00Z',
+				'PatchId': '0x00000713'
+			}, {
+				'CpuId': '0x000306E2',
+				'Date': '2013-03-21T00:00:00Z',
+				'PatchId': '0x0000020D'
+			}, {
+				'CpuId': '0x000306E3',
+				'Date': '2013-03-21T00:00:00Z',
+				'PatchId': '0x00000308'
+			}, {
+				'CpuId': '0x000306E4',
+				'Date': '2018-01-25T00:00:00Z',
+				'PatchId': '0x0000042C'
+			}],
+			'PartNumber': '',
+			'RatedSpeedMHz': 2000,
+			'SerialNumber': '',
+			'VoltageVoltsX10': 14
+		}
+	},
+	'ProcessorArchitecture': 'x86',
+	'ProcessorId': {
+		'EffectiveFamily': '179',
+		'EffectiveModel': '14',
+		'IdentificationRegisters': '0x06e40003fbffbfeb',
+		'MicrocodeInfo': None,
+		'Step': '4',
+		'VendorId': 'Intel'
+	},
+	'ProcessorType': 'CPU',
+	'Socket': 'Proc 1',
+	'Status': {
+		'Health': 'OK'
+	},
+	'TotalCores': 8,
+	'TotalThreads': 16
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/processors b/src/cephadm/node-proxy/redfish_json_samples/processors
new file mode 100644
index 000000000000..c2fb740a4cce
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/processors
@@ -0,0 +1,13 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Processors',
+	'@odata.id': '/redfish/v1/Systems/1/Processors/',
+	'@odata.type': '#ProcessorCollection.ProcessorCollection',
+	'Description': 'Processors view',
+	'Members': [{
+		'@odata.id': '/redfish/v1/Systems/1/Processors/1/'
+	}, {
+		'@odata.id': '/redfish/v1/Systems/1/Processors/2/'
+	}],
+	'Members@odata.count': 2,
+	'Name': 'Processors Collection'
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/storage_sample b/src/cephadm/node-proxy/redfish_json_samples/storage_sample
new file mode 100644
index 000000000000..6d351cfbc61e
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/storage_sample
@@ -0,0 +1,19 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces/Members/$entity',
+	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/',
+	'@odata.type': '#EthernetInterface.1.0.0.EthernetInterface',
+	'Id': '1',
+	'Name': 'System Ethernet Interface',
+	'Oem': {
+		'Hp': {
+			'@odata.type': '#HpiLOEthernetNetworkInterface.1.0.0.HpiLOEthernetNetworkInterface',
+			'DHCPv4': None,
+			'DHCPv6': None,
+			'IPv4': None,
+			'IPv6': None,
+			'SharedNetworkPortOptions': None
+		}
+	},
+	'SettingsResult': None,
+	'Status': None
+}
diff --git a/src/cephadm/node-proxy/redfish_json_samples/system b/src/cephadm/node-proxy/redfish_json_samples/system
new file mode 100644
index 000000000000..5bd20170b235
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_json_samples/system
@@ -0,0 +1,144 @@
+{
+	'@odata.context': '/redfish/v1/$metadata#Systems/Members/$entity',
+	'@odata.id': '/redfish/v1/Systems/1/',
+	'@odata.type': '#ComputerSystem.1.0.1.ComputerSystem',
+	'Actions': {
+		'#ComputerSystem.Reset': {
+			'ResetType@Redfish.AllowableValues': ['On', 'ForceOff', 'ForceRestart', 'Nmi', 'PushPowerButton'],
+			'target': '/redfish/v1/Systems/1/Actions/ComputerSystem.Reset/'
+		}
+	},
+	'AssetTag': '                                ',
+	'BiosVersion': 'P71 01/22/2018',
+	'Boot': {
+		'BootSourceOverrideEnabled': 'Disabled',
+		'BootSourceOverrideSupported': ['None', 'Floppy', 'Cd', 'Hdd', 'Usb', 'Utilities', 'BiosSetup', 'Pxe'],
+		'BootSourceOverrideTarget': 'None'
+	},
+	'Description': 'Computer System View',
+	'EthernetInterfaces': {
+		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/'
+	},
+	'HostName': 'hive1',
+	'Id': '1',
+	'IndicatorLED': 'Off',
+	'Links': {
+		'Chassis': [{
+			'@odata.id': '/redfish/v1/Chassis/1/'
+		}],
+		'ManagedBy': [{
+			'@odata.id': '/redfish/v1/Managers/1/'
+		}]
+	},
+	'LogServices': {
+		'@odata.id': '/redfish/v1/Systems/1/LogServices/'
+	},
+	'Manufacturer': 'HPE',
+	'MemorySummary': {
+		'Status': {
+			'HealthRollup': 'OK'
+		},
+		'TotalSystemMemoryGiB': 384
+	},
+	'Model': 'ProLiant DL360p Gen8',
+	'Name': 'Computer System',
+	'Oem': {
+		'Hp': {
+			'@odata.type': '#HpComputerSystemExt.1.2.2.HpComputerSystemExt',
+			'Actions': {
+				'#HpComputerSystemExt.PowerButton': {
+					'PushType@Redfish.AllowableValues': ['Press', 'PressAndHold'],
+					'target': '/redfish/v1/Systems/1/Actions/Oem/Hp/ComputerSystemExt.PowerButton/'
+				},
+				'#HpComputerSystemExt.SystemReset': {
+					'ResetType@Redfish.AllowableValues': ['ColdBoot', 'AuxCycle'],
+					'target': '/redfish/v1/Systems/1/Actions/Oem/Hp/ComputerSystemExt.SystemReset/'
+				}
+			},
+			'Bios': {
+				'Backup': {
+					'Date': '07/01/2015',
+					'Family': 'P71',
+					'VersionString': 'P71 07/01/2015'
+				},
+				'Bootblock': {
+					'Date': '03/05/2013',
+					'Family': 'P71',
+					'VersionString': 'P71 03/05/2013'
+				},
+				'Current': {
+					'Date': '01/22/2018',
+					'Family': 'P71',
+					'VersionString': 'P71 01/22/2018'
+				},
+				'UefiClass': 0
+			},
+			'DeviceDiscoveryComplete': {
+				'AMSDeviceDiscovery': 'NoAMS',
+				'DeviceDiscovery': 'vMainDeviceDiscoveryComplete',
+				'SmartArrayDiscovery': 'Complete'
+			},
+			'IntelligentProvisioningIndex': 3,
+			'IntelligentProvisioningLocation': 'System Board',
+			'IntelligentProvisioningVersion': 'N/A',
+			'Links': {
+				'BIOS': {
+					'@odata.id': '/redfish/v1/Systems/1/Bios/'
+				},
+				'EthernetInterfaces': {
+					'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/'
+				},
+				'FirmwareInventory': {
+					'@odata.id': '/redfish/v1/Systems/1/FirmwareInventory/'
+				},
+				'Memory': {
+					'@odata.id': '/redfish/v1/Systems/1/Memory/'
+				},
+				'NetworkAdapters': {
+				'@odata.id': '/redfish/v1/Systems/1/NetworkAdapters/'
+				},
+				'PCIDevices': {
+					'@odata.id': '/redfish/v1/Systems/1/PCIDevices/'
+				},
+				'PCISlots': {
+					'@odata.id': '/redfish/v1/Systems/1/PCISlots/'
+				},
+				'SmartStorage': {
+					'@odata.id': '/redfish/v1/Systems/1/SmartStorage/'
+				},
+				'SoftwareInventory': {
+					'@odata.id': '/redfish/v1/Systems/1/SoftwareInventory/'
+				}
+			},
+			'PostState': 'FinishedPost',
+			'PowerAllocationLimit': 1500,
+			'PowerAutoOn': 'Restore',
+			'PowerOnDelay': 'Minimum',
+			'PowerRegulatorMode': 'Dynamic',
+			'PowerRegulatorModesSupported': ['OSControl', 'Dynamic', 'Max', 'Min'],
+			'TrustedModules': [{
+				'Status': 'NotPresent'
+			}],
+			'VirtualProfile': 'Inactive'
+		}
+	},
+	'PowerState': 'On',
+	'ProcessorSummary': {
+		'Count': 2,
+		'Model': ' Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz      ',
+		'Status': {
+			'HealthRollup': 'OK'
+		}
+	},
+	'Processors': {
+		'@odata.id': '/redfish/v1/Systems/1/Processors/'
+	},
+	'SKU': '654081-B21      ',
+	'SerialNumber': 'CZJ4320228      ',
+	'Status': {
+		'Health': 'Warning',
+		'State': 'Enabled'
+	},
+	'SystemType': 'Physical',
+	'UUID': '30343536-3138-5A43-4A34-333230323238'
+}
\ No newline at end of file
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
new file mode 100644
index 000000000000..921b7520640a
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -0,0 +1,91 @@
+from system import System
+from redfish_client import RedFishClient
+from threading import Thread
+from time import sleep
+
+class RedfishSystem(System):
+    def __init__(self, host, username, password):
+        self.client = RedFishClient(host, username, password)
+        self.client.login()
+        self._system = {}
+        self.run = False
+        self.thread = None
+
+    def get_system(self):
+        return self._system
+
+    def get_status(self):
+        return self._system['Status']
+
+    def get_metadata(self):
+        return self._system['metadata']
+
+    def get_memory(self):
+        return self._system['memory']
+
+    def get_power(self):
+        return self._system['power']
+
+    def get_processor(self):
+        return self._system['processor']
+
+    def get_network(self):
+        return self._system['network']
+
+    def get_storage(self):
+        return self._system['storage']
+
+    def _process_redfish_system(self, redfish_system):
+        return redfish_system
+
+    def _update_system(self):
+        redfish_system = self.client.get_path('/Systems/1')
+        self._system = self._process_redfish_system(redfish_system)
+
+    def _update_metadata(self):
+        print("Updating metadata")
+        pass
+
+    def _update_memory(self):
+        print("Updating memory")
+        pass
+
+    def _update_power(self):
+        print("Updating power")
+        pass
+
+    def _update_network(self):
+        print("Updating network")
+        net_path = self._system['EthernetInterfaces']['@odata.id']
+        network_info = self.client.get_path(net_path)
+        self._system['network'] = {}
+        for interface in network_info['Members']:
+            interface_path = interface['@odata.id']
+            interface_info = self.client.get_path(interface_path)
+            self._system['network'][interface_info['Id']] = interface_info
+
+    def _update_storage(self):
+        print("Updating storage")
+        pass
+
+    def start_update_loop(self):
+        self.run = True
+        self.thread = Thread(target=self.update)
+        self.thread.start()
+
+    def stop_update_loop(self):
+        self.run = False
+        self.thread.join()
+
+    def update(self):
+        #  this loop can have:
+        #  - caching logic
+        while self.run:
+            self._update_system()
+            # following calls in theory can be done in parallel
+            self._update_metadata()
+            self._update_memory()
+            self._update_power()
+            self._update_network()
+            self._update_storage()
+            sleep(3)
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
new file mode 100644
index 000000000000..3942f38cb5df
--- /dev/null
+++ b/src/cephadm/node-proxy/reporter.py
@@ -0,0 +1,28 @@
+from threading import Thread
+import requests
+import time
+
+class Reporter:
+    def __init__(self, system, observer_url):
+        self.system = system
+        self.observer_url = observer_url
+        self.finish = False
+
+    def stop(self):
+        self.finish = True
+        self.thread.join()
+
+    def run(self):
+        self.thread = Thread(target=self.loop)
+        self.thread.start()
+
+    def loop(self):
+        while not self.finish:
+            # Any logic to avoid sending the all the system
+            # information every loop can go here. In a real
+            # scenario probably we should just send the sub-parts
+            # that have changed to minimize the traffic in
+            # dense clusters
+            d = self.system.get_system()
+            requests.post(self.observer_url, json=d)
+            time.sleep(10)
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
new file mode 100644
index 000000000000..fc09a4fc8e5a
--- /dev/null
+++ b/src/cephadm/node-proxy/server.py
@@ -0,0 +1,49 @@
+from flask import Flask, request, jsonify
+from system import System
+from redfish_system import RedfishSystem
+from reporter import Reporter
+import time
+
+# must be passed as arguments
+host = "https://x.x.x.x:8443"
+username = "myuser"
+password = "mypassword"
+
+# create the redfish system and the obsever
+system = RedfishSystem(host, username, password)
+reporter_agent = Reporter(system, "http://127.0.0.1:8000")
+
+app = Flask(__name__)
+
+@app.route('/system', methods=['GET'])
+def get_system():
+    return jsonify({'system': system.get_system()})
+
+@app.route('/system/memory', methods=['GET'])
+def get_system_memory():
+    return jsonify({'memory': system.get_memory()})
+
+@app.route('/system/network', methods=['GET'])
+def get_system_network():
+    return jsonify({'network': system.get_network()})
+
+@app.route('/system/status', methods=['GET'])
+def get_system_status():
+    return jsonify({'status': system.get_status()})
+
+@app.route('/system/actions/', methods=['POST'])
+def post_system():
+    pass
+
+@app.route('/system/actions/', methods=['PUT'])
+def put_system():
+    pass
+
+@app.route('/system/control/', methods=['DELETE'])
+def delete():
+    pass
+
+if __name__ == '__main__':
+    system.start_update_loop()
+    reporter_agent.run()
+    app.run(debug=True)
diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/system.py
new file mode 100644
index 000000000000..c43bd35ebf0c
--- /dev/null
+++ b/src/cephadm/node-proxy/system.py
@@ -0,0 +1,29 @@
+
+
+class System:
+    def __init__(self):
+        self._system = {}
+
+    def get_system(self):
+        return self._system
+
+    def get_status(self):
+        return self._system['status']
+
+    def get_metadata(self):
+        return self._system['metadata']
+
+    def get_processor(self):
+        return self._system['processor']
+
+    def get_memory(self):
+        return self._system['memory']
+
+    def get_power(self):
+        return self._system['power']
+
+    def get_network(self):
+        return self._system['network']
+
+    def get_storage(self):
+        return self._system['storage']

From ecc84f5b5aa8f8e45d5068956117ca793f805e18 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 21 Mar 2023 07:07:54 +0100
Subject: [PATCH 1499/2492] node-proxy: various unified interface changes

this slightly modifies the data structure of the unified interface.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
(cherry picked from commit b853761836febe92f6460a13d554cd966ff2e529)
---
 src/cephadm/node-proxy/data.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/node-proxy/data.py b/src/cephadm/node-proxy/data.py
index b8e69f39078b..70339011e4ac 100644
--- a/src/cephadm/node-proxy/data.py
+++ b/src/cephadm/node-proxy/data.py
@@ -14,9 +14,10 @@
         'Health': 'OK'
     },
 
-    'processor': {
+    'processors': [{
         'description': '',
-        'count': '',
+        'cores': '',
+        'threads': '',
         'type': '',
         'model': '',
         'temperature': '',
@@ -24,7 +25,7 @@
             'State': 'Enabled',
             'Health': 'OK'
         }
-    },
+    }],
 
     'memory': {
         'description': '',
@@ -56,6 +57,9 @@
             {
                 'device': 'devc',
                 'description': 'Milk, Cheese, Bread, Fruit, Vegetables',
+                'serial_number': 'xxxxx',
+                'location': '1I:x:y',
+                'interface_type': 'SATA',
                 'model': 'Buy groceries',
                 'type': 'ssd|rotate|nvme',
                 'capacity_bytes': '',
@@ -68,8 +72,8 @@
         ]
     },
 
-    'power': {
-        'power_supplies': [
+    'power':
+        [{
             'type': 'xx',
             'manufacturer': 'xxx',
             'model': 'xx',
@@ -79,14 +83,16 @@
                 'State': 'Enabled',
                 'Health': 'OK'
             }
-        ]
-    },
+        }],
 
-    'cooling': {
+    'thermal': {
         'fans': [
             {
                 'id': 1,
-                'status': '',
+                'status': {
+                    'State': 'Enabled',
+                    'Health': 'OK'
+                }
             }
         ]
     },

From 906286426f02e068f5f8379e9330b2dcbaace050 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 5 Apr 2023 14:08:38 +0200
Subject: [PATCH 1500/2492] node-proxy: improve logging

this adds a new file `util.py` with a logger function in order
to improve a bit the logging.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 11 ++++++++---
 src/cephadm/node-proxy/redfish_system.py | 16 +++++++++++-----
 src/cephadm/node-proxy/server.py         |  4 ++++
 src/cephadm/node-proxy/util.py           | 12 ++++++++++++
 4 files changed, 35 insertions(+), 8 deletions(-)
 create mode 100644 src/cephadm/node-proxy/util.py

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index f4a06ffc309b..8b85daeace38 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -1,12 +1,15 @@
 from redfish.rest.v1 import ServerDownOrUnreachableError
 import redfish
-import logging
+from util import logger
+
+log = logger(__name__, level=10)
 
 class RedFishClient:
 
     PREFIX = '/redfish/v1'
 
     def __init__(self, host, username, password):
+        log.info(f"redfish client initialization...")
         self.host = host
         self.username = username
         self.password = password
@@ -19,14 +22,15 @@ def login(self):
                                                   default_prefix=self.PREFIX)
         try:
             self.redfish_obj.login(auth="session")
+            log.info(f"Logging to redfish api at {self.host} with user: {self.username}")
         except ServerDownOrUnreachableError as e:
-            logging.error(f"Server not reachable or does not support RedFish {e}", e)
+            log.error(f"Server not reachable or does not support RedFish {e}", e)
 
     def get_path(self, path):
         try:
             if self.PREFIX not in path:
                 path = f"{self.PREFIX}{path}"
-            print(f"getting: {path}")
+            log.debug(f"getting: {path}")
             response = self.redfish_obj.get(path)
             return response.dict
         except Exception as e:
@@ -34,4 +38,5 @@ def get_path(self, path):
             pass
 
     def logout(self):
+        log.info('logging out...')
         self.redfish_obj.logout()
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 921b7520640a..07f9cd69973a 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,9 +2,15 @@
 from redfish_client import RedFishClient
 from threading import Thread
 from time import sleep
+from flask import request
+from util import logger
+
+log = logger(__name__)
+
 
 class RedfishSystem(System):
     def __init__(self, host, username, password):
+        log.info(f"redfish system initialization, host: {host}, user: {username}")
         self.client = RedFishClient(host, username, password)
         self.client.login()
         self._system = {}
@@ -43,20 +49,20 @@ def _update_system(self):
         self._system = self._process_redfish_system(redfish_system)
 
     def _update_metadata(self):
-        print("Updating metadata")
+        log.info("Updating metadata")
         pass
 
     def _update_memory(self):
-        print("Updating memory")
+        log.info("Updating memory")
         pass
 
     def _update_power(self):
-        print("Updating power")
+        log.info("Updating power")
         pass
 
     def _update_network(self):
-        print("Updating network")
         net_path = self._system['EthernetInterfaces']['@odata.id']
+        log.info("Updating network")
         network_info = self.client.get_path(net_path)
         self._system['network'] = {}
         for interface in network_info['Members']:
@@ -65,7 +71,7 @@ def _update_network(self):
             self._system['network'][interface_info['Id']] = interface_info
 
     def _update_storage(self):
-        print("Updating storage")
+        log.info("Updating storage")
         pass
 
     def start_update_loop(self):
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index fc09a4fc8e5a..14c8db941ee7 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -2,8 +2,11 @@
 from system import System
 from redfish_system import RedfishSystem
 from reporter import Reporter
+from util import logger
 import time
 
+log = logger(__name__)
+
 # must be passed as arguments
 host = "https://x.x.x.x:8443"
 username = "myuser"
@@ -11,6 +14,7 @@
 
 # create the redfish system and the obsever
 system = RedfishSystem(host, username, password)
+log.info(f"Server initialization...")
 reporter_agent = Reporter(system, "http://127.0.0.1:8000")
 
 app = Flask(__name__)
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
new file mode 100644
index 000000000000..d70e99fc4f38
--- /dev/null
+++ b/src/cephadm/node-proxy/util.py
@@ -0,0 +1,12 @@
+import logging
+
+def logger(name, level=logging.INFO):
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    handler = logging.StreamHandler()
+    handler.setLevel(level)
+    fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler.setFormatter(fmt)
+    logger.addHandler(handler)
+
+    return logger

From e80cd0286a34a352fc098d72a9740e25156de9a8 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 5 Apr 2023 14:10:41 +0200
Subject: [PATCH 1501/2492] node-proxy: variabilize the system_endpoint

This makes it possible to define the value of the 'System endpoint'.
This can be different according to the hardware.

This probably means that the class `RedfishSystem` should be split itself.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 9 +++++++--
 src/cephadm/node-proxy/server.py         | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 07f9cd69973a..e4a59320768c 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -9,13 +9,18 @@
 
 
 class RedfishSystem(System):
-    def __init__(self, host, username, password):
+    def __init__(self,
+                 host,
+                 username,
+                 password,
+                 system_endpoint='/Systems/1'):
         log.info(f"redfish system initialization, host: {host}, user: {username}")
         self.client = RedFishClient(host, username, password)
         self.client.login()
         self._system = {}
         self.run = False
         self.thread = None
+        self.system_endpoint = system_endpoint
 
     def get_system(self):
         return self._system
@@ -45,7 +50,7 @@ def _process_redfish_system(self, redfish_system):
         return redfish_system
 
     def _update_system(self):
-        redfish_system = self.client.get_path('/Systems/1')
+        redfish_system = self.client.get_path(self.system_endpoint)
         self._system = self._process_redfish_system(redfish_system)
 
     def _update_metadata(self):
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 14c8db941ee7..582a5e615758 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -13,8 +13,8 @@
 password = "mypassword"
 
 # create the redfish system and the obsever
-system = RedfishSystem(host, username, password)
 log.info(f"Server initialization...")
+system = RedfishSystem(host, username, password, system_endpoint='/Systems/System.Embedded.1')
 reporter_agent = Reporter(system, "http://127.0.0.1:8000")
 
 app = Flask(__name__)

From 7c602947e45ceff719c105b0277a10a6e72831e5 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 5 Apr 2023 14:14:40 +0200
Subject: [PATCH 1502/2492] node-proxy: logout from redfish api on Exception

Otherwise it ends up recreating new session each time whereas the previous session
is left. After multiple failures, it reaches the limit and left sessions need to be
cleaned up manually.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index e4a59320768c..de80d83d87d0 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -91,12 +91,18 @@ def stop_update_loop(self):
     def update(self):
         #  this loop can have:
         #  - caching logic
-        while self.run:
-            self._update_system()
-            # following calls in theory can be done in parallel
-            self._update_metadata()
-            self._update_memory()
-            self._update_power()
-            self._update_network()
-            self._update_storage()
-            sleep(3)
+        try:
+            while self.run:
+                self._update_system()
+                # following calls in theory can be done in parallel
+                self._update_metadata()
+                self._update_memory()
+                self._update_power()
+                self._update_network()
+                self._update_storage()
+                sleep(3)
+        # Catching 'Exception' is probably not a good idea (devel only)
+        except Exception:
+            log.error(f"Error detected, logging out from redfish api")
+            self.client.logout()
+            raise

From e06e65b78bdf44d38a3d47ab2040dc88e5cd130f Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 5 Apr 2023 14:16:29 +0200
Subject: [PATCH 1503/2492] node-proxy: add a /shutdown endpoint

Add a '/shutdown' endpoint to force the client to logout and delete its current
session.
This is for devel puroposes and probably not intended to be kept.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 582a5e615758..b2fef2845f23 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -19,6 +19,12 @@
 
 app = Flask(__name__)
 
+@app.route('/shutdown', methods=['POST'])
+def shutdown():
+    system.stop_update_loop()
+    system.client.logout()
+    return 'Server shutting down...\n'
+
 @app.route('/system', methods=['GET'])
 def get_system():
     return jsonify({'system': system.get_system()})

From dcbdfd33feda40df82dc898011bfbd690c7aca31 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 5 Apr 2023 14:18:19 +0200
Subject: [PATCH 1504/2492] node-proxy: use `use_reloader=False`

In order to prevent the server from restarting in a loop
when an error shows up. Otherwise, it creates a bunch of new
redfish client session and make it quickly unavailable due to the
session limit.
Probably not intended to be kept.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index b2fef2845f23..8928cf8de0eb 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -56,4 +56,4 @@ def delete():
 if __name__ == '__main__':
     system.start_update_loop()
     reporter_agent.run()
-    app.run(debug=True)
+    app.run(debug=True, use_reloader=False)

From 693a05a0cb38f3ca91d8aa3a67c0da23a491aa23 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 6 Apr 2023 14:53:41 +0200
Subject: [PATCH 1505/2492] node-proxy: implement processors endpoint

This adds the required logic for the endpoint '/system/processors'
to gather and return data about CPUs.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 26 ++++++++++++++++++++++--
 src/cephadm/node-proxy/server.py         |  4 ++++
 src/cephadm/node-proxy/system.py         |  4 ++--
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index de80d83d87d0..3d05718be171 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -37,8 +37,8 @@ def get_memory(self):
     def get_power(self):
         return self._system['power']
 
-    def get_processor(self):
-        return self._system['processor']
+    def get_processors(self):
+        return self._system['processors']
 
     def get_network(self):
         return self._system['network']
@@ -75,6 +75,27 @@ def _update_network(self):
             interface_info = self.client.get_path(interface_path)
             self._system['network'][interface_info['Id']] = interface_info
 
+    def _update_processors(self):
+        cpus_path = self._system['Processors']['@odata.id']
+        log.info("Updating processors")
+        cpus_info = self.client.get_path(cpus_path)
+        self._system['processors'] = {}
+        result = dict()
+        for cpu in cpus_info['Members']:
+            cpu_path = cpu['@odata.id']
+            cpu_info = self.client.get_path(cpu_path)
+            cpu_id = cpu_info['Id']
+            result[cpu_id] = dict()
+            result[cpu_id]['description'] = cpu_info['Description']
+            result[cpu_id]['cores'] = cpu_info['TotalCores']
+            result[cpu_id]['threads'] = cpu_info['TotalThreads']
+            result[cpu_id]['type'] = cpu_info['ProcessorType']
+            result[cpu_id]['model'] = cpu_info['Model']
+            result[cpu_id]['status'] = cpu_info['Status']
+            result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
+        self._system['processors'] = result
+
+
     def _update_storage(self):
         log.info("Updating storage")
         pass
@@ -99,6 +120,7 @@ def update(self):
                 self._update_memory()
                 self._update_power()
                 self._update_network()
+                self._update_processors()
                 self._update_storage()
                 sleep(3)
         # Catching 'Exception' is probably not a good idea (devel only)
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 8928cf8de0eb..6e4da362329b 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -37,6 +37,10 @@ def get_system_memory():
 def get_system_network():
     return jsonify({'network': system.get_network()})
 
+@app.route('/system/processors', methods=['GET'])
+def get_system_processors():
+    return jsonify({'processors': system.get_processors()})
+
 @app.route('/system/status', methods=['GET'])
 def get_system_status():
     return jsonify({'status': system.get_status()})
diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/system.py
index c43bd35ebf0c..cab408d1a7fd 100644
--- a/src/cephadm/node-proxy/system.py
+++ b/src/cephadm/node-proxy/system.py
@@ -13,8 +13,8 @@ def get_status(self):
     def get_metadata(self):
         return self._system['metadata']
 
-    def get_processor(self):
-        return self._system['processor']
+    def get_processors(self):
+        return self._system['processors']
 
     def get_memory(self):
         return self._system['memory']

From 4b9bc24f0dd1ab261c4b87667c6cdb77d3785185 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 6 Apr 2023 14:55:41 +0200
Subject: [PATCH 1506/2492] node-proxy: implement network endpoint

This adds the required logic for the endpoint '/system/network'
to gather and return data about network interfaces.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 3d05718be171..76d518354159 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -70,10 +70,17 @@ def _update_network(self):
         log.info("Updating network")
         network_info = self.client.get_path(net_path)
         self._system['network'] = {}
+        result = dict()
         for interface in network_info['Members']:
             interface_path = interface['@odata.id']
             interface_info = self.client.get_path(interface_path)
-            self._system['network'][interface_info['Id']] = interface_info
+            interface_id = interface_info['Id']
+            result[interface_id] = dict()
+            result[interface_id]['description'] = interface_info['Description']
+            result[interface_id]['name'] = interface_info['Name']
+            result[interface_id]['speed_mbps'] = interface_info['SpeedMbps']
+            result[interface_id]['status'] = interface_info['Status']
+        self._system['network'] = result
 
     def _update_processors(self):
         cpus_path = self._system['Processors']['@odata.id']

From d919132be3e69e5494e44519e46a864833802b96 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 6 Apr 2023 14:56:48 +0200
Subject: [PATCH 1507/2492] node-proxy: implement storage endpoint

This adds the required logic for the endpoint '/system/storage'
to gather and return data about physical drives.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 20 +++++++++++++++++++-
 src/cephadm/node-proxy/server.py         |  4 ++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 76d518354159..483bdb90a7ae 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -104,8 +104,26 @@ def _update_processors(self):
 
 
     def _update_storage(self):
+        storage_path = self._system['Storage']['@odata.id']
         log.info("Updating storage")
-        pass
+        storage_info = self.client.get_path(storage_path)
+        result = dict()
+        for storage in storage_info['Members']:
+           entity_path = storage['@odata.id']
+           entity_info = self.client.get_path(entity_path)
+           for drive in entity_info['Drives']:
+               drive_path = drive['@odata.id']
+               drive_info = self.client.get_path(drive_path)
+               drive_id = drive_info['Id']
+               result[drive_id] = dict()
+               result[drive_id]['description'] = drive_info['Description']
+               result[drive_id]['capacity_bytes'] = drive_info['CapacityBytes']
+               result[drive_id]['model'] = drive_info['Model']
+               result[drive_id]['protocol'] = drive_info['Protocol']
+               result[drive_id]['serial_number'] = drive_info['SerialNumber']
+               result[drive_id]['status'] = drive_info['Status']
+               result[drive_id]['location'] = drive_info['PhysicalLocation']
+        self._system['storage'] = result
 
     def start_update_loop(self):
         self.run = True
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 6e4da362329b..11ce48890d94 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -41,6 +41,10 @@ def get_system_network():
 def get_system_processors():
     return jsonify({'processors': system.get_processors()})
 
+@app.route('/system/storage', methods=['GET'])
+def get_system_storage():
+    return jsonify({'storage': system.get_storage()})
+
 @app.route('/system/status', methods=['GET'])
 def get_system_status():
     return jsonify({'status': system.get_status()})

From 4454c64f94bb2fd93ed796a832dedb42faecf3f9 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 6 Apr 2023 17:29:28 +0200
Subject: [PATCH 1508/2492] node-proxy: split RedfishSystem class

This class should be split because the logic will be different depending on the
hardware.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_dell.py   | 82 ++++++++++++++++++++++++
 src/cephadm/node-proxy/redfish_system.py | 81 ++++-------------------
 src/cephadm/node-proxy/server.py         |  4 +-
 src/cephadm/node-proxy/system.py         | 21 +++---
 4 files changed, 108 insertions(+), 80 deletions(-)
 create mode 100644 src/cephadm/node-proxy/redfish_dell.py

diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
new file mode 100644
index 000000000000..9a611fb7bbe1
--- /dev/null
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -0,0 +1,82 @@
+from redfish_system import RedfishSystem
+from util import logger
+
+log = logger(__name__)
+
+
+class RedfishDell(RedfishSystem):
+    def __init__(self, **kw):
+        if kw.get('system_endpoint') is None:
+            kw['system_endpoint'] = '/Systems/System.Embedded.1'
+        super().__init__(**kw)
+
+    def _update_network(self):
+        net_path = self._system['EthernetInterfaces']['@odata.id']
+        log.info("Updating network")
+        network_info = self.client.get_path(net_path)
+        self._system['network'] = {}
+        result = dict()
+        for interface in network_info['Members']:
+            interface_path = interface['@odata.id']
+            interface_info = self.client.get_path(interface_path)
+            interface_id = interface_info['Id']
+            result[interface_id] = dict()
+            result[interface_id]['description'] = interface_info['Description']
+            result[interface_id]['name'] = interface_info['Name']
+            result[interface_id]['speed_mbps'] = interface_info['SpeedMbps']
+            result[interface_id]['status'] = interface_info['Status']
+        self._system['network'] = result
+
+    def _update_processors(self):
+        cpus_path = self._system['Processors']['@odata.id']
+        log.info("Updating processors")
+        cpus_info = self.client.get_path(cpus_path)
+        self._system['processors'] = {}
+        result = dict()
+        for cpu in cpus_info['Members']:
+            cpu_path = cpu['@odata.id']
+            cpu_info = self.client.get_path(cpu_path)
+            cpu_id = cpu_info['Id']
+            result[cpu_id] = dict()
+            result[cpu_id]['description'] = cpu_info['Description']
+            result[cpu_id]['cores'] = cpu_info['TotalCores']
+            result[cpu_id]['threads'] = cpu_info['TotalThreads']
+            result[cpu_id]['type'] = cpu_info['ProcessorType']
+            result[cpu_id]['model'] = cpu_info['Model']
+            result[cpu_id]['status'] = cpu_info['Status']
+            result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
+        self._system['processors'] = result
+
+    def _update_storage(self):
+        storage_path = self._system['Storage']['@odata.id']
+        log.info("Updating storage")
+        storage_info = self.client.get_path(storage_path)
+        result = dict()
+        for storage in storage_info['Members']:
+            entity_path = storage['@odata.id']
+            entity_info = self.client.get_path(entity_path)
+            for drive in entity_info['Drives']:
+                drive_path = drive['@odata.id']
+                drive_info = self.client.get_path(drive_path)
+                drive_id = drive_info['Id']
+                result[drive_id] = dict()
+                result[drive_id]['description'] = drive_info['Description']
+                result[drive_id]['capacity_bytes'] = drive_info['CapacityBytes']
+                result[drive_id]['model'] = drive_info['Model']
+                result[drive_id]['protocol'] = drive_info['Protocol']
+                result[drive_id]['serial_number'] = drive_info['SerialNumber']
+                result[drive_id]['status'] = drive_info['Status']
+                result[drive_id]['location'] = drive_info['PhysicalLocation']
+        self._system['storage'] = result
+
+    def _update_metadata(self):
+        log.info("Updating metadata")
+        pass
+
+    def _update_memory(self):
+        log.info("Updating memory")
+        pass
+
+    def _update_power(self):
+        log.info("Updating power")
+        pass
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 483bdb90a7ae..df012aa964f8 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -9,18 +9,17 @@
 
 
 class RedfishSystem(System):
-    def __init__(self,
-                 host,
-                 username,
-                 password,
-                 system_endpoint='/Systems/1'):
-        log.info(f"redfish system initialization, host: {host}, user: {username}")
-        self.client = RedFishClient(host, username, password)
+    def __init__(self, **kw):
+        self.host = kw.get('host')
+        self.username = kw.get('username')
+        self.password = kw.get('password')
+        self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
+        log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
+        self.client = RedFishClient(self.host, self.username, self.password)
         self.client.login()
         self._system = {}
         self.run = False
         self.thread = None
-        self.system_endpoint = system_endpoint
 
     def get_system(self):
         return self._system
@@ -54,76 +53,22 @@ def _update_system(self):
         self._system = self._process_redfish_system(redfish_system)
 
     def _update_metadata(self):
-        log.info("Updating metadata")
-        pass
+        raise NotImplementedError()
 
     def _update_memory(self):
-        log.info("Updating memory")
-        pass
+        raise NotImplementedError()
 
     def _update_power(self):
-        log.info("Updating power")
-        pass
+        raise NotImplementedError()
 
     def _update_network(self):
-        net_path = self._system['EthernetInterfaces']['@odata.id']
-        log.info("Updating network")
-        network_info = self.client.get_path(net_path)
-        self._system['network'] = {}
-        result = dict()
-        for interface in network_info['Members']:
-            interface_path = interface['@odata.id']
-            interface_info = self.client.get_path(interface_path)
-            interface_id = interface_info['Id']
-            result[interface_id] = dict()
-            result[interface_id]['description'] = interface_info['Description']
-            result[interface_id]['name'] = interface_info['Name']
-            result[interface_id]['speed_mbps'] = interface_info['SpeedMbps']
-            result[interface_id]['status'] = interface_info['Status']
-        self._system['network'] = result
+        raise NotImplementedError()
 
     def _update_processors(self):
-        cpus_path = self._system['Processors']['@odata.id']
-        log.info("Updating processors")
-        cpus_info = self.client.get_path(cpus_path)
-        self._system['processors'] = {}
-        result = dict()
-        for cpu in cpus_info['Members']:
-            cpu_path = cpu['@odata.id']
-            cpu_info = self.client.get_path(cpu_path)
-            cpu_id = cpu_info['Id']
-            result[cpu_id] = dict()
-            result[cpu_id]['description'] = cpu_info['Description']
-            result[cpu_id]['cores'] = cpu_info['TotalCores']
-            result[cpu_id]['threads'] = cpu_info['TotalThreads']
-            result[cpu_id]['type'] = cpu_info['ProcessorType']
-            result[cpu_id]['model'] = cpu_info['Model']
-            result[cpu_id]['status'] = cpu_info['Status']
-            result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
-        self._system['processors'] = result
-
+        raise NotImplementedError()
 
     def _update_storage(self):
-        storage_path = self._system['Storage']['@odata.id']
-        log.info("Updating storage")
-        storage_info = self.client.get_path(storage_path)
-        result = dict()
-        for storage in storage_info['Members']:
-           entity_path = storage['@odata.id']
-           entity_info = self.client.get_path(entity_path)
-           for drive in entity_info['Drives']:
-               drive_path = drive['@odata.id']
-               drive_info = self.client.get_path(drive_path)
-               drive_id = drive_info['Id']
-               result[drive_id] = dict()
-               result[drive_id]['description'] = drive_info['Description']
-               result[drive_id]['capacity_bytes'] = drive_info['CapacityBytes']
-               result[drive_id]['model'] = drive_info['Model']
-               result[drive_id]['protocol'] = drive_info['Protocol']
-               result[drive_id]['serial_number'] = drive_info['SerialNumber']
-               result[drive_id]['status'] = drive_info['Status']
-               result[drive_id]['location'] = drive_info['PhysicalLocation']
-        self._system['storage'] = result
+        raise NotImplementedError()
 
     def start_update_loop(self):
         self.run = True
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 11ce48890d94..f450f0523957 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -1,6 +1,6 @@
 from flask import Flask, request, jsonify
 from system import System
-from redfish_system import RedfishSystem
+from redfish_dell import RedfishDell
 from reporter import Reporter
 from util import logger
 import time
@@ -14,7 +14,7 @@
 
 # create the redfish system and the obsever
 log.info(f"Server initialization...")
-system = RedfishSystem(host, username, password, system_endpoint='/Systems/System.Embedded.1')
+system = RedfishDell(host=host, username=username, password=password, system_endpoint='/Systems/System.Embedded.1')
 reporter_agent = Reporter(system, "http://127.0.0.1:8000")
 
 app = Flask(__name__)
diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/system.py
index cab408d1a7fd..1b70bcd2492d 100644
--- a/src/cephadm/node-proxy/system.py
+++ b/src/cephadm/node-proxy/system.py
@@ -1,29 +1,30 @@
-
+from util import Config
 
 class System:
-    def __init__(self):
+    def __init__(self, **kw):
         self._system = {}
+        self.config: Config = kw['config']
 
     def get_system(self):
-        return self._system
+        raise NotImplementedError()
 
     def get_status(self):
-        return self._system['status']
+        raise NotImplementedError()
 
     def get_metadata(self):
-        return self._system['metadata']
+        raise NotImplementedError()
 
     def get_processors(self):
-        return self._system['processors']
+        raise NotImplementedError()
 
     def get_memory(self):
-        return self._system['memory']
+        raise NotImplementedError()
 
     def get_power(self):
-        return self._system['power']
+        raise NotImplementedError()
 
     def get_network(self):
-        return self._system['network']
+        raise NotImplementedError()
 
     def get_storage(self):
-        return self._system['storage']
+        raise NotImplementedError()

From 15b1122f7af6df3faac21ed4f1834ae826f5abb3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 11 May 2023 13:23:22 +0200
Subject: [PATCH 1509/2492] node-proxy: add normalize_dict() function

this is to make sure all keys are converted into
lowercase.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_dell.py | 8 ++++----
 src/cephadm/node-proxy/util.py         | 9 +++++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index 9a611fb7bbe1..c4ccd9add4a6 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -1,5 +1,5 @@
 from redfish_system import RedfishSystem
-from util import logger
+from util import logger, normalize_dict
 
 log = logger(__name__)
 
@@ -25,7 +25,7 @@ def _update_network(self):
             result[interface_id]['name'] = interface_info['Name']
             result[interface_id]['speed_mbps'] = interface_info['SpeedMbps']
             result[interface_id]['status'] = interface_info['Status']
-        self._system['network'] = result
+        self._system['network'] = normalize_dict(result)
 
     def _update_processors(self):
         cpus_path = self._system['Processors']['@odata.id']
@@ -45,7 +45,7 @@ def _update_processors(self):
             result[cpu_id]['model'] = cpu_info['Model']
             result[cpu_id]['status'] = cpu_info['Status']
             result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
-        self._system['processors'] = result
+        self._system['processors'] = normalize_dict(result)
 
     def _update_storage(self):
         storage_path = self._system['Storage']['@odata.id']
@@ -67,7 +67,7 @@ def _update_storage(self):
                 result[drive_id]['serial_number'] = drive_info['SerialNumber']
                 result[drive_id]['status'] = drive_info['Status']
                 result[drive_id]['location'] = drive_info['PhysicalLocation']
-        self._system['storage'] = result
+        self._system['storage'] = normalize_dict(result)
 
     def _update_metadata(self):
         log.info("Updating metadata")
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index d70e99fc4f38..f976814c0ce1 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -10,3 +10,12 @@ def logger(name, level=logging.INFO):
     logger.addHandler(handler)
 
     return logger
+
+def normalize_dict(test_dict):
+    res = dict()
+    for key in test_dict.keys():
+        if isinstance(test_dict[key], dict):
+            res[key.lower()] = normalize_dict(test_dict[key])
+        else:
+            res[key.lower()] = test_dict[key]
+    return res

From 6ae1687f5f27b6d03dd2c46735de837c7429ae5b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 11 May 2023 13:25:36 +0200
Subject: [PATCH 1510/2492] node-proxy: merge self._system with current values

Otherwise `self._system` gets reset in each iteration.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index df012aa964f8..ae02062a469f 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -50,7 +50,8 @@ def _process_redfish_system(self, redfish_system):
 
     def _update_system(self):
         redfish_system = self.client.get_path(self.system_endpoint)
-        self._system = self._process_redfish_system(redfish_system)
+        _system = self._process_redfish_system(redfish_system)
+        self._system = {**_system, **self._system}
 
     def _update_metadata(self):
         raise NotImplementedError()

From 4b9a4ec55fbcbeed48b1dc01594cf8ed65a23ef5 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 11 May 2023 13:29:05 +0200
Subject: [PATCH 1511/2492] node-proxy: display error messages when Exception
 is caught

This is mostly for development purposes.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 1 +
 src/cephadm/node-proxy/redfish_system.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index 8b85daeace38..52ae3c089def 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -35,6 +35,7 @@ def get_path(self, path):
             return response.dict
         except Exception as e:
             #TODO
+            log.error(f"Error detected.\n{e}")
             pass
 
     def logout(self):
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index ae02062a469f..6dd457774d9f 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -95,7 +95,7 @@ def update(self):
                 self._update_storage()
                 sleep(3)
         # Catching 'Exception' is probably not a good idea (devel only)
-        except Exception:
-            log.error(f"Error detected, logging out from redfish api")
+        except Exception as e:
+            log.error(f"Error detected, logging out from redfish api.\n{e}")
             self.client.logout()
             raise

From 9b2a3345b6e52b152ebf680abf319300dad513d2 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:09:03 +0200
Subject: [PATCH 1512/2492] node-proxy: drop
 redfish_system._process_redfish_system method

This method isn't needed, let's drop it.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 6dd457774d9f..f10a30ece4c8 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -45,13 +45,9 @@ def get_network(self):
     def get_storage(self):
         return self._system['storage']
 
-    def _process_redfish_system(self, redfish_system):
-        return redfish_system
-
     def _update_system(self):
         redfish_system = self.client.get_path(self.system_endpoint)
-        _system = self._process_redfish_system(redfish_system)
-        self._system = {**_system, **self._system}
+        self._system = {**redfish_system, **self._system}
 
     def _update_metadata(self):
         raise NotImplementedError()

From 6f9d3d9e15305e80ea5797ff2f0dd0b929e70822 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:15:05 +0200
Subject: [PATCH 1513/2492] node-proxy: add method start_client()
 redfish_system class

This is going to be useful for a new endpoint '/start'

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index f10a30ece4c8..8b8e96f7a34a 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -16,10 +16,15 @@ def __init__(self, **kw):
         self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
         log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
-        self.client.login()
         self._system = {}
         self.run = False
         self.thread = None
+        self.start_client()
+
+    def start_client(self):
+        log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
+        self.client = RedFishClient(self.host, self.username, self.password)
+        self.client.login()
 
     def get_system(self):
         return self._system

From 213320d33863b2e76bb81e8de33bb78d0970dd28 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:19:09 +0200
Subject: [PATCH 1514/2492] node-proxy: migrate to cherrypy

cherrypy is already widely used in Ceph.
Let's not add new dependencies and use cherrypy instead of
python-flask

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py |   1 -
 src/cephadm/node-proxy/server-v2.py      | 136 +++++++++++++++++++++++
 2 files changed, 136 insertions(+), 1 deletion(-)
 create mode 100644 src/cephadm/node-proxy/server-v2.py

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 8b8e96f7a34a..94c668277439 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,7 +2,6 @@
 from redfish_client import RedFishClient
 from threading import Thread
 from time import sleep
-from flask import request
 from util import logger
 
 log = logger(__name__)
diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server-v2.py
new file mode 100644
index 000000000000..3f2e3df471e0
--- /dev/null
+++ b/src/cephadm/node-proxy/server-v2.py
@@ -0,0 +1,136 @@
+import cherrypy
+from redfish_dell import RedfishDell
+from reporter import Reporter
+from util import logger
+import sys
+
+# for devel purposes
+import os
+DEVEL_ENV_VARS = ['REDFISH_HOST',
+                  'REDFISH_USERNAME',
+                  'REDFISH_PASSWORD']
+for env_var in DEVEL_ENV_VARS:
+    if os.environ.get(env_var) is None:
+        print(f"{env_var} environment variable must be set.")
+        sys.exit(1)
+
+log = logger(__name__)
+
+# must be passed as arguments
+host = os.environ.get('REDFISH_HOST')
+username = os.environ.get('REDFISH_USERNAME')
+password = os.environ.get('REDFISH_PASSWORD')
+
+# create the redfish system and the obsever
+log.info("Server initialization...")
+system = RedfishDell(host=host, username=username, password=password, system_endpoint='/Systems/System.Embedded.1')
+reporter_agent = Reporter(system, "http://127.0.0.1:8000")
+
+
+class Memory:
+    exposed = True
+
+    @cherrypy.tools.json_out()
+    def GET(self):
+        return {'memory': system.get_memory()}
+
+
+class Network:
+    exposed = True
+
+    @cherrypy.tools.json_out()
+    def GET(self):
+        return {'network': system.get_network()}
+
+
+class Processors:
+    exposed = True
+
+    @cherrypy.tools.json_out()
+    def GET(self):
+        return {'processors': system.get_processors()}
+
+
+class Storage:
+    exposed = True
+
+    @cherrypy.tools.json_out()
+    def GET(self):
+        return {'storage': system.get_storage()}
+
+
+class Status:
+    exposed = True
+
+    @cherrypy.tools.json_out()
+    def GET(self):
+        return {'status': system.get_status()}
+
+
+class System:
+    exposed = True
+    memory = Memory()
+    network = Network()
+    processors = Processors()
+    storage = Storage()
+    status = Status()
+    # actions = Actions()
+    # control = Control()
+
+
+class Shutdown:
+    exposed = True
+
+    def POST(self):
+        _stop()
+        cherrypy.engine.exit()
+        return 'Server shutdown...'
+
+
+def _stop():
+    system.stop_update_loop()
+    system.client.logout()
+    reporter_agent.stop()
+
+
+class Start:
+    exposed = True
+
+    def POST(self):
+        system.start_client()
+        system.start_update_loop()
+        reporter_agent.run()
+        return 'node-proxy daemon started'
+
+
+class Stop:
+    exposed = True
+
+    def POST(self):
+        _stop()
+        return 'node-proxy daemon stopped'
+
+
+class API:
+    exposed = True
+
+    system = System()
+    shutdown = Shutdown()
+    start = Start()
+    stop = Stop()
+
+    def GET(self):
+        return 'use /system'
+
+
+if __name__ == '__main__':
+    cherrypy.config.update({
+        'server.socket_port': 8080
+    })
+    config = {'/': {
+        'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
+        'request.dispatch': cherrypy.dispatch.MethodDispatcher()
+    }}
+    system.start_update_loop()
+    reporter_agent.run()
+    cherrypy.quickstart(API(), config=config)

From fe03bf3676ee2b351a0155491bc5eb4bb7b3d1a3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:20:54 +0200
Subject: [PATCH 1515/2492] node-proxy: add a lock mechanism

The loop in the reporter agent has to wait that the data are all
collected before checking and pushing them to the ceph-mgr (if needed).
The idea is to use the lock mechanism offered by the threading module
from python.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 31 ++++++++++++++++--------
 src/cephadm/node-proxy/reporter.py       | 19 ++++++++++++---
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 94c668277439..7c6ebe6de734 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -1,6 +1,6 @@
 from system import System
 from redfish_client import RedFishClient
-from threading import Thread
+from threading import Thread, Lock
 from time import sleep
 from util import logger
 
@@ -19,6 +19,9 @@ def __init__(self, **kw):
         self.run = False
         self.thread = None
         self.start_client()
+        self.data_ready = False
+        self.previous_data = {}
+        self.lock = Lock()
 
     def start_client(self):
         log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
@@ -85,15 +88,23 @@ def update(self):
         #  - caching logic
         try:
             while self.run:
-                self._update_system()
-                # following calls in theory can be done in parallel
-                self._update_metadata()
-                self._update_memory()
-                self._update_power()
-                self._update_network()
-                self._update_processors()
-                self._update_storage()
-                sleep(3)
+                log.debug("waiting for a lock.")
+                self.lock.acquire()
+                log.debug("lock acquired.")
+                try:
+                    self._update_system()
+                    # following calls in theory can be done in parallel
+                    self._update_metadata()
+                    self._update_memory()
+                    self._update_power()
+                    self._update_network()
+                    self._update_processors()
+                    self._update_storage()
+                    self.data_ready = True
+                    sleep(5)
+                finally:
+                    self.lock.release()
+                    log.debug("lock released.")
         # Catching 'Exception' is probably not a good idea (devel only)
         except Exception as e:
             log.error(f"Error detected, logging out from redfish api.\n{e}")
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index 3942f38cb5df..ec27e36e3db3 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -23,6 +23,19 @@ def loop(self):
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
-            d = self.system.get_system()
-            requests.post(self.observer_url, json=d)
-            time.sleep(10)
+            if self.system.data_ready:
+                log.debug("waiting for a lock.")
+                try:
+                    self.system.lock.acquire()
+                    log.debug("lock acquired.")
+                    if not self.system.get_system() == self.system.previous_data:
+                        self.system.previous_data = self.system.get_system()
+                        log.info('data has changed since last iteration.')
+                        d = self.system.get_system()
+                        requests.post(f"{self.observer_url}/fake_endpoint", json=d)
+                    else:
+                        log.info('no diff, not sending data to the mgr.')
+                finally:
+                    self.system.lock.release()
+                    log.debug("lock released.")
+            time.sleep(20)

From 9f72e688c79ebf7883801f108cec3772b16e8d3c Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:25:35 +0200
Subject: [PATCH 1516/2492] node-proxy: redfish_system.get_system refactor

This method should return the 'unified structure' version of the
collected data instead of the huge json returned by redfish.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 7c6ebe6de734..fbb26a33e4bd 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -29,7 +29,12 @@ def start_client(self):
         self.client.login()
 
     def get_system(self):
-        return self._system
+        result = {
+            'storage': self.get_storage(),
+            'processors': self.get_processors(),
+            'network': self.get_network(),
+        }
+        return result
 
     def get_status(self):
         return self._system['Status']

From 7d30c787779078b653d29d31be812580a86602d6 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:26:54 +0200
Subject: [PATCH 1517/2492] node-proxy: fix a typo in
 redfish_system.get_status()

s/Status/status

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index fbb26a33e4bd..fffb45c64dec 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -37,7 +37,7 @@ def get_system(self):
         return result
 
     def get_status(self):
-        return self._system['Status']
+        return self._system['status']
 
     def get_metadata(self):
         return self._system['metadata']

From 102a80fc298a4292e14554e7d57db6c541889468 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 22 May 2023 14:27:48 +0200
Subject: [PATCH 1518/2492] node-proxy: add some logging in the reporter agent

This adds some calls to the logging module, mostly for
devel/debug purposes at the moment.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/reporter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index ec27e36e3db3..a454b8a5bb74 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -1,6 +1,9 @@
 from threading import Thread
 import requests
 import time
+from util import logger
+
+log = logger(__name__, level=10)
 
 class Reporter:
     def __init__(self, system, observer_url):

From c8653e4cf64af5156d571d5e2ffe7e912ac0a78e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 7 Jun 2023 14:20:07 +0200
Subject: [PATCH 1519/2492] node-proxy: catch more error in redfish_client

This catches more potential exceptions in the redfish_client
class.
So if an error is caught we can log a more accurate and nicer message.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index 52ae3c089def..3948b7444f37 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -1,5 +1,8 @@
-from redfish.rest.v1 import ServerDownOrUnreachableError
+from redfish.rest.v1 import ServerDownOrUnreachableError, \
+    SessionCreationError, \
+    InvalidCredentialsError
 import redfish
+import sys
 from util import logger
 
 log = logger(__name__, level=10)
@@ -23,8 +26,12 @@ def login(self):
         try:
             self.redfish_obj.login(auth="session")
             log.info(f"Logging to redfish api at {self.host} with user: {self.username}")
-        except ServerDownOrUnreachableError as e:
-            log.error(f"Server not reachable or does not support RedFish {e}", e)
+            return self.redfish_obj
+        except InvalidCredentialsError as e:
+            log.error(f"Invalid credentials for {self.username} at {self.host}:\n{e}")
+        except (SessionCreationError, ServerDownOrUnreachableError) as e:
+            log.error(f"Server not reachable or does not support RedFish:\n{e}")
+        sys.exit(1)
 
     def get_path(self, path):
         try:

From 6d9198519d7b0d51e00d785d7be1f06e2e7509e3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 7 Jun 2023 14:23:57 +0200
Subject: [PATCH 1520/2492] node-proxy: catch RequestException in reporter

This catches the requests.exceptions.RequestException
exception in the reporter agent so we can better handle the
case where it can't reach the endpoint when trying to send the
collected data.
Before this change, if for some reason the refreshed data couldn't be
sent to the endpoint, it wouldn't have retried because
`self.system.previous_data` was overwritten anyway.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/reporter.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index a454b8a5bb74..c20195b535d4 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -28,17 +28,21 @@ def loop(self):
             # dense clusters
             if self.system.data_ready:
                 log.debug("waiting for a lock.")
-                try:
-                    self.system.lock.acquire()
-                    log.debug("lock acquired.")
-                    if not self.system.get_system() == self.system.previous_data:
-                        self.system.previous_data = self.system.get_system()
-                        log.info('data has changed since last iteration.')
-                        d = self.system.get_system()
+                self.system.lock.acquire()
+                log.debug("lock acquired.")
+                if not self.system.get_system() == self.system.previous_data:
+                    log.info('data has changed since last iteration.')
+                    d = self.system.get_system()
+                    try:
                         requests.post(f"{self.observer_url}/fake_endpoint", json=d)
+                    except requests.exceptions.RequestException as e:
+                        log.error(f"The reporter couldn't send data to the mgr: {e}")
+                        # Need to add a new parameter 'max_retries' to the reporter if it can't
+                        # send the data for more than x times, maybe the daemon should stop altogether
                     else:
-                        log.info('no diff, not sending data to the mgr.')
-                finally:
-                    self.system.lock.release()
-                    log.debug("lock released.")
-            time.sleep(20)
+                        self.system.previous_data = self.system.get_system()
+                else:
+                    log.info('no diff, not sending data to the mgr.')
+                self.system.lock.release()
+                log.debug("lock released.")
+            time.sleep(5)

From c5acf8183c7d6d02fb8fa301b2acdec096e37059 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 8 Jun 2023 15:12:16 +0200
Subject: [PATCH 1521/2492] node-proxy: implement config & logging management

This adds the classes 'Config' and 'Logger' in order to manage
the logging and the configuration within the node-proxy daemon.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 19 +++----
 src/cephadm/node-proxy/redfish_dell.py   | 16 +++---
 src/cephadm/node-proxy/redfish_system.py | 18 ++++---
 src/cephadm/node-proxy/reporter.py       | 16 +++---
 src/cephadm/node-proxy/server-v2.py      | 49 ++++++++++++++---
 src/cephadm/node-proxy/system.py         |  4 +-
 src/cephadm/node-proxy/util.py           | 69 ++++++++++++++++++++----
 7 files changed, 138 insertions(+), 53 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index 3948b7444f37..709130bd7232 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -3,16 +3,16 @@
     InvalidCredentialsError
 import redfish
 import sys
-from util import logger
+from util import Logger
 
-log = logger(__name__, level=10)
+log = Logger(__name__)
 
 class RedFishClient:
 
     PREFIX = '/redfish/v1'
 
     def __init__(self, host, username, password):
-        log.info(f"redfish client initialization...")
+        log.logger.info("redfish client initialization...")
         self.host = host
         self.username = username
         self.password = password
@@ -24,27 +24,28 @@ def login(self):
                                                   password=self.password,
                                                   default_prefix=self.PREFIX)
         try:
+            # TODO: add a retry? check for a timeout setting
             self.redfish_obj.login(auth="session")
-            log.info(f"Logging to redfish api at {self.host} with user: {self.username}")
+            log.logger.info(f"Logging to redfish api at {self.host} with user: {self.username}")
             return self.redfish_obj
         except InvalidCredentialsError as e:
-            log.error(f"Invalid credentials for {self.username} at {self.host}:\n{e}")
+            log.logger.error(f"Invalid credentials for {self.username} at {self.host}:\n{e}")
         except (SessionCreationError, ServerDownOrUnreachableError) as e:
-            log.error(f"Server not reachable or does not support RedFish:\n{e}")
+            log.logger.error(f"Server not reachable or does not support RedFish:\n{e}")
         sys.exit(1)
 
     def get_path(self, path):
         try:
             if self.PREFIX not in path:
                 path = f"{self.PREFIX}{path}"
-            log.debug(f"getting: {path}")
+            log.logger.debug(f"getting: {path}")
             response = self.redfish_obj.get(path)
             return response.dict
         except Exception as e:
             #TODO
-            log.error(f"Error detected.\n{e}")
+            log.logger.error(f"Error detected.\n{e}")
             pass
 
     def logout(self):
-        log.info('logging out...')
+        log.logger.info('logging out...')
         self.redfish_obj.logout()
diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index c4ccd9add4a6..bebce7e2a19b 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -1,7 +1,7 @@
 from redfish_system import RedfishSystem
-from util import logger, normalize_dict
+from util import Logger, normalize_dict
 
-log = logger(__name__)
+log = Logger(__name__)
 
 
 class RedfishDell(RedfishSystem):
@@ -12,7 +12,7 @@ def __init__(self, **kw):
 
     def _update_network(self):
         net_path = self._system['EthernetInterfaces']['@odata.id']
-        log.info("Updating network")
+        log.logger.info("Updating network")
         network_info = self.client.get_path(net_path)
         self._system['network'] = {}
         result = dict()
@@ -29,7 +29,7 @@ def _update_network(self):
 
     def _update_processors(self):
         cpus_path = self._system['Processors']['@odata.id']
-        log.info("Updating processors")
+        log.logger.info("Updating processors")
         cpus_info = self.client.get_path(cpus_path)
         self._system['processors'] = {}
         result = dict()
@@ -49,7 +49,7 @@ def _update_processors(self):
 
     def _update_storage(self):
         storage_path = self._system['Storage']['@odata.id']
-        log.info("Updating storage")
+        log.logger.info("Updating storage")
         storage_info = self.client.get_path(storage_path)
         result = dict()
         for storage in storage_info['Members']:
@@ -70,13 +70,13 @@ def _update_storage(self):
         self._system['storage'] = normalize_dict(result)
 
     def _update_metadata(self):
-        log.info("Updating metadata")
+        log.logger.info("Updating metadata")
         pass
 
     def _update_memory(self):
-        log.info("Updating memory")
+        log.logger.info("Updating memory")
         pass
 
     def _update_power(self):
-        log.info("Updating power")
+        log.logger.info("Updating power")
         pass
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index fffb45c64dec..358d03d5bdf8 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,19 +2,21 @@
 from redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from util import logger
+from util import Logger
 
-log = logger(__name__)
+log = Logger(__name__)
 
 
 class RedfishSystem(System):
     def __init__(self, **kw):
+        super().__init__(**kw)
         self.host = kw.get('host')
         self.username = kw.get('username')
         self.password = kw.get('password')
         self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
-        log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
+        log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
+
         self._system = {}
         self.run = False
         self.thread = None
@@ -24,7 +26,7 @@ def __init__(self, **kw):
         self.lock = Lock()
 
     def start_client(self):
-        log.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
+        log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
         self.client.login()
 
@@ -93,9 +95,9 @@ def update(self):
         #  - caching logic
         try:
             while self.run:
-                log.debug("waiting for a lock.")
+                log.logger.debug("waiting for a lock.")
                 self.lock.acquire()
-                log.debug("lock acquired.")
+                log.logger.debug("lock acquired.")
                 try:
                     self._update_system()
                     # following calls in theory can be done in parallel
@@ -109,9 +111,9 @@ def update(self):
                     sleep(5)
                 finally:
                     self.lock.release()
-                    log.debug("lock released.")
+                    log.logger.debug("lock released.")
         # Catching 'Exception' is probably not a good idea (devel only)
         except Exception as e:
-            log.error(f"Error detected, logging out from redfish api.\n{e}")
+            log.logger.error(f"Error detected, logging out from redfish api.\n{e}")
             self.client.logout()
             raise
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index c20195b535d4..d3be2f3393bd 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -1,9 +1,9 @@
 from threading import Thread
 import requests
 import time
-from util import logger
+from util import Logger
 
-log = logger(__name__, level=10)
+log = Logger(__name__)
 
 class Reporter:
     def __init__(self, system, observer_url):
@@ -27,22 +27,22 @@ def loop(self):
             # that have changed to minimize the traffic in
             # dense clusters
             if self.system.data_ready:
-                log.debug("waiting for a lock.")
+                log.logger.debug("waiting for a lock.")
                 self.system.lock.acquire()
-                log.debug("lock acquired.")
+                log.logger.debug("lock acquired.")
                 if not self.system.get_system() == self.system.previous_data:
-                    log.info('data has changed since last iteration.')
+                    log.logger.info('data has changed since last iteration.')
                     d = self.system.get_system()
                     try:
                         requests.post(f"{self.observer_url}/fake_endpoint", json=d)
                     except requests.exceptions.RequestException as e:
-                        log.error(f"The reporter couldn't send data to the mgr: {e}")
+                        log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't
                         # send the data for more than x times, maybe the daemon should stop altogether
                     else:
                         self.system.previous_data = self.system.get_system()
                 else:
-                    log.info('no diff, not sending data to the mgr.')
+                    log.logger.info('no diff, not sending data to the mgr.')
                 self.system.lock.release()
-                log.debug("lock released.")
+                log.logger.debug("lock released.")
             time.sleep(5)
diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server-v2.py
index 3f2e3df471e0..aaf7a75e1c89 100644
--- a/src/cephadm/node-proxy/server-v2.py
+++ b/src/cephadm/node-proxy/server-v2.py
@@ -1,7 +1,7 @@
 import cherrypy
 from redfish_dell import RedfishDell
 from reporter import Reporter
-from util import logger
+from util import Config, Logger
 import sys
 
 # for devel purposes
@@ -9,21 +9,43 @@
 DEVEL_ENV_VARS = ['REDFISH_HOST',
                   'REDFISH_USERNAME',
                   'REDFISH_PASSWORD']
+
+DEFAULT_CONFIG = {
+    'reporter': {
+        'check_interval': 5,
+        'push_data_max_retries': 30,
+    },
+    'system': {
+        'refresh_interval': 5
+    },
+    'server': {
+        'port': 8080,
+    },
+    'logging': {
+        'level': 20,
+    }
+}
+
 for env_var in DEVEL_ENV_VARS:
     if os.environ.get(env_var) is None:
         print(f"{env_var} environment variable must be set.")
         sys.exit(1)
 
-log = logger(__name__)
+config = Config(default_config=DEFAULT_CONFIG)
 
+log = Logger(__name__, level=config.logging['level'])
 # must be passed as arguments
 host = os.environ.get('REDFISH_HOST')
 username = os.environ.get('REDFISH_USERNAME')
 password = os.environ.get('REDFISH_PASSWORD')
 
 # create the redfish system and the obsever
-log.info("Server initialization...")
-system = RedfishDell(host=host, username=username, password=password, system_endpoint='/Systems/System.Embedded.1')
+log.logger.info("Server initialization...")
+system = RedfishDell(host=host,
+                     username=username,
+                     password=password,
+                     system_endpoint='/Systems/System.Embedded.1',
+                     config=config)
 reporter_agent = Reporter(system, "http://127.0.0.1:8000")
 
 
@@ -111,6 +133,17 @@ def POST(self):
         return 'node-proxy daemon stopped'
 
 
+class ConfigReload:
+    exposed = True
+
+    def __init__(self, config):
+        self.config = config
+
+    def POST(self):
+        self.config['node_proxy'].reload()
+        return 'node-proxy config reloaded'
+
+
 class API:
     exposed = True
 
@@ -118,6 +151,7 @@ class API:
     shutdown = Shutdown()
     start = Start()
     stop = Stop()
+    config_reload = ConfigReload(cherrypy.config)
 
     def GET(self):
         return 'use /system'
@@ -125,12 +159,13 @@ def GET(self):
 
 if __name__ == '__main__':
     cherrypy.config.update({
-        'server.socket_port': 8080
+        'node_proxy': config,
+        'server.socket_port': config.server['port']
     })
-    config = {'/': {
+    c = {'/': {
         'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
         'request.dispatch': cherrypy.dispatch.MethodDispatcher()
     }}
     system.start_update_loop()
     reporter_agent.run()
-    cherrypy.quickstart(API(), config=config)
+    cherrypy.quickstart(API(), config=c)
diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/system.py
index 1b70bcd2492d..4b34001dd817 100644
--- a/src/cephadm/node-proxy/system.py
+++ b/src/cephadm/node-proxy/system.py
@@ -1,9 +1,7 @@
-from util import Config
-
 class System:
     def __init__(self, **kw):
         self._system = {}
-        self.config: Config = kw['config']
+        self.config = kw['config']
 
     def get_system(self):
         raise NotImplementedError()
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index f976814c0ce1..27cfffdf3c29 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -1,15 +1,7 @@
 import logging
+import yaml
+import os
 
-def logger(name, level=logging.INFO):
-    logger = logging.getLogger(name)
-    logger.setLevel(level)
-    handler = logging.StreamHandler()
-    handler.setLevel(level)
-    fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    handler.setFormatter(fmt)
-    logger.addHandler(handler)
-
-    return logger
 
 def normalize_dict(test_dict):
     res = dict()
@@ -19,3 +11,60 @@ def normalize_dict(test_dict):
         else:
             res[key.lower()] = test_dict[key]
     return res
+
+
+class Config:
+
+    def __init__(self,
+                 config_file='/etc/ceph/node-proxy.yaml',
+                 default_config={}):
+        self.config_file = config_file
+        self.default_config = default_config
+
+        self.load_config()
+
+    def load_config(self):
+        if os.path.exists(self.config_file):
+            with open(self.config_file, 'r') as f:
+                self.config = yaml.safe_load(f)
+        else:
+            self.config = self.default_config
+
+        for k, v in self.default_config.items():
+            if k not in self.config.keys():
+                self.config[k] = v
+
+        for k, v in self.config.items():
+            setattr(self, k, v)
+
+        # TODO: need to be improved
+        for _l in Logger._Logger:
+            _l.logger.setLevel(self.logging['level'])
+            _l.logger.handlers[0].setLevel(self.logging['level'])
+
+    def reload(self, config_file=None):
+        if config_file != '':
+            self.config_file = config_file
+        self.load_config()
+
+
+class Logger:
+    _Logger = []
+
+    def __init__(self, name, level=logging.INFO):
+        self.name = name
+        self.level = level
+
+        Logger._Logger.append(self)
+        self.logger = self.get_logger()
+
+    def get_logger(self):
+        logger = logging.getLogger(self.name)
+        logger.setLevel(self.level)
+        handler = logging.StreamHandler()
+        handler.setLevel(self.level)
+        fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
+        return logger

From 4d63a0a18dbbb5259dad098ea0184edd5c3655bb Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 8 Jun 2023 18:22:26 +0200
Subject: [PATCH 1522/2492] node-proxy: address some flake8 linting errors

This addresses some flake8 errors.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 3 ++-
 src/cephadm/node-proxy/reporter.py       | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index 709130bd7232..fdf0106c173b 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -7,6 +7,7 @@
 
 log = Logger(__name__)
 
+
 class RedFishClient:
 
     PREFIX = '/redfish/v1'
@@ -42,7 +43,7 @@ def get_path(self, path):
             response = self.redfish_obj.get(path)
             return response.dict
         except Exception as e:
-            #TODO
+            # TODO
             log.logger.error(f"Error detected.\n{e}")
             pass
 
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index d3be2f3393bd..25dde2d85da2 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -5,6 +5,7 @@
 
 log = Logger(__name__)
 
+
 class Reporter:
     def __init__(self, system, observer_url):
         self.system = system

From ee8e28baafbe6861a21514c2af05b77a42d6f963 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 8 Jun 2023 18:31:38 +0200
Subject: [PATCH 1523/2492] node-proxy: add type annotation

This commit adds the type annotation in all files.
This was missing since the initial implementation, let's add
it before the project gets bigger.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py | 14 ++++--
 src/cephadm/node-proxy/redfish_dell.py   | 21 +++++----
 src/cephadm/node-proxy/redfish_system.py | 59 ++++++++++++------------
 src/cephadm/node-proxy/reporter.py       |  9 ++--
 src/cephadm/node-proxy/server-v2.py      | 27 +++++------
 src/cephadm/node-proxy/system.py         | 26 ++++++-----
 src/cephadm/node-proxy/util.py           | 21 +++++----
 7 files changed, 95 insertions(+), 82 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index fdf0106c173b..bfec7d9ad4e9 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -4,6 +4,7 @@
 import redfish
 import sys
 from util import Logger
+from typing import Dict
 
 log = Logger(__name__)
 
@@ -12,14 +13,17 @@ class RedFishClient:
 
     PREFIX = '/redfish/v1'
 
-    def __init__(self, host, username, password):
+    def __init__(self,
+                 host: str,
+                 username: str,
+                 password: str) -> None:
         log.logger.info("redfish client initialization...")
         self.host = host
         self.username = username
         self.password = password
-        self.redfish_obj = None
+        self.redfish_obj: 'redfish.redfish_client' = None
 
-    def login(self):
+    def login(self) -> 'redfish.redfish_client':
         self.redfish_obj = redfish.redfish_client(base_url=self.host,
                                                   username=self.username,
                                                   password=self.password,
@@ -35,7 +39,7 @@ def login(self):
             log.logger.error(f"Server not reachable or does not support RedFish:\n{e}")
         sys.exit(1)
 
-    def get_path(self, path):
+    def get_path(self, path: str) -> Dict:
         try:
             if self.PREFIX not in path:
                 path = f"{self.PREFIX}{path}"
@@ -47,6 +51,6 @@ def get_path(self, path):
             log.logger.error(f"Error detected.\n{e}")
             pass
 
-    def logout(self):
+    def logout(self) -> None:
         log.logger.info('logging out...')
         self.redfish_obj.logout()
diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index bebce7e2a19b..80e54f1cff31 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -1,21 +1,22 @@
 from redfish_system import RedfishSystem
 from util import Logger, normalize_dict
+from typing import Dict, Any
 
 log = Logger(__name__)
 
 
 class RedfishDell(RedfishSystem):
-    def __init__(self, **kw):
+    def __init__(self, **kw: Any) -> None:
         if kw.get('system_endpoint') is None:
             kw['system_endpoint'] = '/Systems/System.Embedded.1'
         super().__init__(**kw)
 
-    def _update_network(self):
+    def _update_network(self) -> None:
         net_path = self._system['EthernetInterfaces']['@odata.id']
         log.logger.info("Updating network")
         network_info = self.client.get_path(net_path)
         self._system['network'] = {}
-        result = dict()
+        result: Dict[str, Dict[str, Dict]] = dict()
         for interface in network_info['Members']:
             interface_path = interface['@odata.id']
             interface_info = self.client.get_path(interface_path)
@@ -27,12 +28,12 @@ def _update_network(self):
             result[interface_id]['status'] = interface_info['Status']
         self._system['network'] = normalize_dict(result)
 
-    def _update_processors(self):
+    def _update_processors(self) -> None:
         cpus_path = self._system['Processors']['@odata.id']
         log.logger.info("Updating processors")
         cpus_info = self.client.get_path(cpus_path)
         self._system['processors'] = {}
-        result = dict()
+        result: Dict[str, Dict[str, Dict]] = dict()
         for cpu in cpus_info['Members']:
             cpu_path = cpu['@odata.id']
             cpu_info = self.client.get_path(cpu_path)
@@ -47,11 +48,11 @@ def _update_processors(self):
             result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
         self._system['processors'] = normalize_dict(result)
 
-    def _update_storage(self):
+    def _update_storage(self) -> None:
         storage_path = self._system['Storage']['@odata.id']
         log.logger.info("Updating storage")
         storage_info = self.client.get_path(storage_path)
-        result = dict()
+        result: Dict[str, Dict[str, Dict]] = dict()
         for storage in storage_info['Members']:
             entity_path = storage['@odata.id']
             entity_info = self.client.get_path(entity_path)
@@ -69,14 +70,14 @@ def _update_storage(self):
                 result[drive_id]['location'] = drive_info['PhysicalLocation']
         self._system['storage'] = normalize_dict(result)
 
-    def _update_metadata(self):
+    def _update_metadata(self) -> None:
         log.logger.info("Updating metadata")
         pass
 
-    def _update_memory(self):
+    def _update_memory(self) -> None:
         log.logger.info("Updating memory")
         pass
 
-    def _update_power(self):
+    def _update_power(self) -> None:
         log.logger.info("Updating power")
         pass
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 358d03d5bdf8..7b95005b9546 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -3,34 +3,35 @@
 from threading import Thread, Lock
 from time import sleep
 from util import Logger
+from typing import Dict, Any
 
 log = Logger(__name__)
 
 
 class RedfishSystem(System):
-    def __init__(self, **kw):
+    def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
-        self.host = kw.get('host')
-        self.username = kw.get('username')
-        self.password = kw.get('password')
+        self.host: str = kw['host']
+        self.username: str = kw['username']
+        self.password: str = kw['password']
         self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
         log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
 
-        self._system = {}
-        self.run = False
-        self.thread = None
+        self._system: Dict[str, Dict[str, Any]] = {}
+        self.run: bool = False
+        self.thread: Thread
         self.start_client()
-        self.data_ready = False
-        self.previous_data = {}
-        self.lock = Lock()
+        self.data_ready: bool = False
+        self.previous_data: Dict = {}
+        self.lock: Lock = Lock()
 
-    def start_client(self):
+    def start_client(self) -> None:
         log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
         self.client.login()
 
-    def get_system(self):
+    def get_system(self) -> Dict[str, Dict[str, Dict]]:
         result = {
             'storage': self.get_storage(),
             'processors': self.get_processors(),
@@ -38,59 +39,59 @@ def get_system(self):
         }
         return result
 
-    def get_status(self):
+    def get_status(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['status']
 
-    def get_metadata(self):
+    def get_metadata(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['metadata']
 
-    def get_memory(self):
+    def get_memory(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['memory']
 
-    def get_power(self):
+    def get_power(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['power']
 
-    def get_processors(self):
+    def get_processors(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['processors']
 
-    def get_network(self):
+    def get_network(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['network']
 
-    def get_storage(self):
+    def get_storage(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['storage']
 
-    def _update_system(self):
+    def _update_system(self) -> None:
         redfish_system = self.client.get_path(self.system_endpoint)
         self._system = {**redfish_system, **self._system}
 
-    def _update_metadata(self):
+    def _update_metadata(self) -> None:
         raise NotImplementedError()
 
-    def _update_memory(self):
+    def _update_memory(self) -> None:
         raise NotImplementedError()
 
-    def _update_power(self):
+    def _update_power(self) -> None:
         raise NotImplementedError()
 
-    def _update_network(self):
+    def _update_network(self) -> None:
         raise NotImplementedError()
 
-    def _update_processors(self):
+    def _update_processors(self) -> None:
         raise NotImplementedError()
 
-    def _update_storage(self):
+    def _update_storage(self) -> None:
         raise NotImplementedError()
 
-    def start_update_loop(self):
+    def start_update_loop(self) -> None:
         self.run = True
         self.thread = Thread(target=self.update)
         self.thread.start()
 
-    def stop_update_loop(self):
+    def stop_update_loop(self) -> None:
         self.run = False
         self.thread.join()
 
-    def update(self):
+    def update(self) -> None:
         #  this loop can have:
         #  - caching logic
         try:
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index 25dde2d85da2..6be8d24c6b0a 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -2,25 +2,26 @@
 import requests
 import time
 from util import Logger
+from typing import Any
 
 log = Logger(__name__)
 
 
 class Reporter:
-    def __init__(self, system, observer_url):
+    def __init__(self, system: Any, observer_url: str) -> None:
         self.system = system
         self.observer_url = observer_url
         self.finish = False
 
-    def stop(self):
+    def stop(self) -> None:
         self.finish = True
         self.thread.join()
 
-    def run(self):
+    def run(self) -> None:
         self.thread = Thread(target=self.loop)
         self.thread.start()
 
-    def loop(self):
+    def loop(self) -> None:
         while not self.finish:
             # Any logic to avoid sending the all the system
             # information every loop can go here. In a real
diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server-v2.py
index aaf7a75e1c89..d1924629e6bd 100644
--- a/src/cephadm/node-proxy/server-v2.py
+++ b/src/cephadm/node-proxy/server-v2.py
@@ -2,6 +2,7 @@
 from redfish_dell import RedfishDell
 from reporter import Reporter
 from util import Config, Logger
+from typing import Dict
 import sys
 
 # for devel purposes
@@ -53,7 +54,7 @@ class Memory:
     exposed = True
 
     @cherrypy.tools.json_out()
-    def GET(self):
+    def GET(self) -> Dict[str, Dict[str, Dict]]:
         return {'memory': system.get_memory()}
 
 
@@ -61,7 +62,7 @@ class Network:
     exposed = True
 
     @cherrypy.tools.json_out()
-    def GET(self):
+    def GET(self) -> Dict[str, Dict[str, Dict]]:
         return {'network': system.get_network()}
 
 
@@ -69,7 +70,7 @@ class Processors:
     exposed = True
 
     @cherrypy.tools.json_out()
-    def GET(self):
+    def GET(self) -> Dict[str, Dict[str, Dict]]:
         return {'processors': system.get_processors()}
 
 
@@ -77,7 +78,7 @@ class Storage:
     exposed = True
 
     @cherrypy.tools.json_out()
-    def GET(self):
+    def GET(self) -> Dict[str, Dict[str, Dict]]:
         return {'storage': system.get_storage()}
 
 
@@ -85,7 +86,7 @@ class Status:
     exposed = True
 
     @cherrypy.tools.json_out()
-    def GET(self):
+    def GET(self) -> Dict[str, Dict[str, Dict]]:
         return {'status': system.get_status()}
 
 
@@ -103,13 +104,13 @@ class System:
 class Shutdown:
     exposed = True
 
-    def POST(self):
+    def POST(self) -> str:
         _stop()
         cherrypy.engine.exit()
         return 'Server shutdown...'
 
 
-def _stop():
+def _stop() -> None:
     system.stop_update_loop()
     system.client.logout()
     reporter_agent.stop()
@@ -118,7 +119,7 @@ def _stop():
 class Start:
     exposed = True
 
-    def POST(self):
+    def POST(self) -> str:
         system.start_client()
         system.start_update_loop()
         reporter_agent.run()
@@ -128,7 +129,7 @@ def POST(self):
 class Stop:
     exposed = True
 
-    def POST(self):
+    def POST(self) -> str:
         _stop()
         return 'node-proxy daemon stopped'
 
@@ -136,10 +137,10 @@ def POST(self):
 class ConfigReload:
     exposed = True
 
-    def __init__(self, config):
+    def __init__(self, config: cherrypy.config) -> None:
         self.config = config
 
-    def POST(self):
+    def POST(self) -> str:
         self.config['node_proxy'].reload()
         return 'node-proxy config reloaded'
 
@@ -153,14 +154,14 @@ class API:
     stop = Stop()
     config_reload = ConfigReload(cherrypy.config)
 
-    def GET(self):
+    def GET(self) -> str:
         return 'use /system'
 
 
 if __name__ == '__main__':
     cherrypy.config.update({
         'node_proxy': config,
-        'server.socket_port': config.server['port']
+        'server.socket_port': config.__dict__['server']['port']
     })
     c = {'/': {
         'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/system.py
index 4b34001dd817..9e8f97ef59d6 100644
--- a/src/cephadm/node-proxy/system.py
+++ b/src/cephadm/node-proxy/system.py
@@ -1,28 +1,32 @@
+from util import Config
+from typing import Dict, Any
+
+
 class System:
-    def __init__(self, **kw):
-        self._system = {}
-        self.config = kw['config']
+    def __init__(self, **kw: Any) -> None:
+        self._system: Dict = {}
+        self.config: Config = kw['config']
 
-    def get_system(self):
+    def get_system(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_status(self):
+    def get_status(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_metadata(self):
+    def get_metadata(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_processors(self):
+    def get_processors(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_memory(self):
+    def get_memory(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_power(self):
+    def get_power(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_network(self):
+    def get_network(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
-    def get_storage(self):
+    def get_storage(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index 27cfffdf3c29..817583e6ff7f 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -1,9 +1,10 @@
 import logging
 import yaml
 import os
+from typing import Dict, List
 
 
-def normalize_dict(test_dict):
+def normalize_dict(test_dict: Dict) -> Dict:
     res = dict()
     for key in test_dict.keys():
         if isinstance(test_dict[key], dict):
@@ -16,14 +17,14 @@ def normalize_dict(test_dict):
 class Config:
 
     def __init__(self,
-                 config_file='/etc/ceph/node-proxy.yaml',
-                 default_config={}):
+                 config_file: str = '/etc/ceph/node-proxy.yaml',
+                 default_config: Dict[str, Any] = {}) -> None:
         self.config_file = config_file
         self.default_config = default_config
 
         self.load_config()
 
-    def load_config(self):
+    def load_config(self) -> None:
         if os.path.exists(self.config_file):
             with open(self.config_file, 'r') as f:
                 self.config = yaml.safe_load(f)
@@ -39,26 +40,26 @@ def load_config(self):
 
         # TODO: need to be improved
         for _l in Logger._Logger:
-            _l.logger.setLevel(self.logging['level'])
-            _l.logger.handlers[0].setLevel(self.logging['level'])
+            _l.logger.setLevel(self.logging['level'])  # type: ignore
+            _l.logger.handlers[0].setLevel(self.logging['level'])  # type: ignore
 
-    def reload(self, config_file=None):
+    def reload(self, config_file: str = '') -> None:
         if config_file != '':
             self.config_file = config_file
         self.load_config()
 
 
 class Logger:
-    _Logger = []
+    _Logger: List['Logger'] = []
 
-    def __init__(self, name, level=logging.INFO):
+    def __init__(self, name: str, level: int = logging.INFO):
         self.name = name
         self.level = level
 
         Logger._Logger.append(self)
         self.logger = self.get_logger()
 
-    def get_logger(self):
+    def get_logger(self) -> logging.Logger:
         logger = logging.getLogger(self.name)
         logger.setLevel(self.level)
         handler = logging.StreamHandler()

From 9b88e5a782b2c10ce782ca09ff2bb56bb0a82200 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 9 Jun 2023 14:58:02 +0200
Subject: [PATCH 1524/2492] node-proxy: add a decorator 'retry'

This decorator will be useful for calls that should do multiple
attempts before actually failing.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/util.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index 817583e6ff7f..6420e904876f 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -1,7 +1,8 @@
 import logging
 import yaml
 import os
-from typing import Dict, List
+import time
+from typing import Dict, List, Callable, Any
 
 
 def normalize_dict(test_dict: Dict) -> Dict:
@@ -69,3 +70,20 @@ def get_logger(self) -> logging.Logger:
         logger.addHandler(handler)
 
         return logger
+
+
+def retry(exceptions: Any = Exception, retries: int = 20, delay: int = 1) -> Callable:
+    def decorator(f: Callable) -> Callable:
+        def _retry(*args: str, **kwargs: Any) -> Callable:
+            _tries = retries
+            while _tries > 1:
+                try:
+                    print("{}".format(_tries))
+                    return f(*args, **kwargs)
+                except exceptions:
+                    time.sleep(delay)
+                    _tries -= 1
+            print("{} has failed after {} tries".format(f, retries))
+            return f(*args, **kwargs)
+        return _retry
+    return decorator

From c8f31a1ef01d777e9ef8aae1a895dfcf0a6dea8b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 9 Jun 2023 15:03:24 +0200
Subject: [PATCH 1525/2492] node-proxy: add a retry on
 redfish_client.get_path() calls

The idea is to retry multiple times before stating the endpoint is
definitely unreachable.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_client.py |  5 ++---
 src/cephadm/node-proxy/redfish_dell.py   | 14 +++++++-------
 src/cephadm/node-proxy/redfish_system.py | 10 +++++++++-
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index bfec7d9ad4e9..6a3e384893d2 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -47,9 +47,8 @@ def get_path(self, path: str) -> Dict:
             response = self.redfish_obj.get(path)
             return response.dict
         except Exception as e:
-            # TODO
-            log.logger.error(f"Error detected.\n{e}")
-            pass
+            log.logger.error(f"Error getting path:\n{e}")
+            return {}
 
     def logout(self) -> None:
         log.logger.info('logging out...')
diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index 80e54f1cff31..f940a9ec3c30 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -14,12 +14,12 @@ def __init__(self, **kw: Any) -> None:
     def _update_network(self) -> None:
         net_path = self._system['EthernetInterfaces']['@odata.id']
         log.logger.info("Updating network")
-        network_info = self.client.get_path(net_path)
+        network_info = self._get_path(net_path)
         self._system['network'] = {}
         result: Dict[str, Dict[str, Dict]] = dict()
         for interface in network_info['Members']:
             interface_path = interface['@odata.id']
-            interface_info = self.client.get_path(interface_path)
+            interface_info = self._get_path(interface_path)
             interface_id = interface_info['Id']
             result[interface_id] = dict()
             result[interface_id]['description'] = interface_info['Description']
@@ -31,12 +31,12 @@ def _update_network(self) -> None:
     def _update_processors(self) -> None:
         cpus_path = self._system['Processors']['@odata.id']
         log.logger.info("Updating processors")
-        cpus_info = self.client.get_path(cpus_path)
+        cpus_info = self._get_path(cpus_path)
         self._system['processors'] = {}
         result: Dict[str, Dict[str, Dict]] = dict()
         for cpu in cpus_info['Members']:
             cpu_path = cpu['@odata.id']
-            cpu_info = self.client.get_path(cpu_path)
+            cpu_info = self._get_path(cpu_path)
             cpu_id = cpu_info['Id']
             result[cpu_id] = dict()
             result[cpu_id]['description'] = cpu_info['Description']
@@ -51,14 +51,14 @@ def _update_processors(self) -> None:
     def _update_storage(self) -> None:
         storage_path = self._system['Storage']['@odata.id']
         log.logger.info("Updating storage")
-        storage_info = self.client.get_path(storage_path)
+        storage_info = self._get_path(storage_path)
         result: Dict[str, Dict[str, Dict]] = dict()
         for storage in storage_info['Members']:
             entity_path = storage['@odata.id']
-            entity_info = self.client.get_path(entity_path)
+            entity_info = self._get_path(entity_path)
             for drive in entity_info['Drives']:
                 drive_path = drive['@odata.id']
-                drive_info = self.client.get_path(drive_path)
+                drive_info = self._get_path(drive_path)
                 drive_id = drive_info['Id']
                 result[drive_id] = dict()
                 result[drive_id]['description'] = drive_info['Description']
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 7b95005b9546..0dc269deb508 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,7 +2,7 @@
 from redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from util import Logger
+from util import Logger, retry
 from typing import Dict, Any
 
 log = Logger(__name__)
@@ -26,6 +26,14 @@ def __init__(self, **kw: Any) -> None:
         self.previous_data: Dict = {}
         self.lock: Lock = Lock()
 
+    @retry(retries=10, delay=2)
+    def _get_path(self, path: str) -> Dict:
+        result = self.client.get_path(path)
+        if result is None:
+            log.logger.error(f"The client reported an error when getting path: {path}")
+            raise RuntimeError(f"Could not get path: {path}")
+        return result
+
     def start_client(self) -> None:
         log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)

From 31b46ff9b8901d0a54cfedaf219a280c4802676a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 12 Jun 2023 14:36:54 +0200
Subject: [PATCH 1526/2492] node-proxy: add requirements.txt

This adds the requirements.txt file in order to manage the required
libraries.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/requirements.txt | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 src/cephadm/node-proxy/requirements.txt

diff --git a/src/cephadm/node-proxy/requirements.txt b/src/cephadm/node-proxy/requirements.txt
new file mode 100644
index 000000000000..ab20bcc64b46
--- /dev/null
+++ b/src/cephadm/node-proxy/requirements.txt
@@ -0,0 +1,8 @@
+pyyaml
+types-PyYAML
+types-requests
+mypy
+flake8
+redfish
+CherryPy
+requests

From 415dc693ffaab4e6bbcfd5e2891625c4707bd7e3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 15 Jun 2023 16:19:27 +0200
Subject: [PATCH 1527/2492] node-proxy: remove a redundant message

This message is not needed given that there's the same in
the RedFishClient class.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 0dc269deb508..7254236938a1 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -35,7 +35,6 @@ def _get_path(self, path: str) -> Dict:
         return result
 
     def start_client(self) -> None:
-        log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
         self.client.login()
 

From ee1d4e49d1431365ceed4043a59d9f91123c4506 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 15 Jun 2023 16:20:31 +0200
Subject: [PATCH 1528/2492] node-proxy: (Redfish_System) reuse the existing
 client when possible

Otherwise, the method start_client() recreates a new client.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_system.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 7254236938a1..ffa530cefbf0 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -35,7 +35,8 @@ def _get_path(self, path: str) -> Dict:
         return result
 
     def start_client(self) -> None:
-        self.client = RedFishClient(self.host, self.username, self.password)
+        if not self.client:
+            self.client = RedFishClient(self.host, self.username, self.password)
         self.client.login()
 
     def get_system(self) -> Dict[str, Dict[str, Dict]]:

From a3aff1b848a3785dd2e3752a79c8c819e6445239 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 15 Jun 2023 16:23:13 +0200
Subject: [PATCH 1529/2492] node-proxy: add a timeout when posting data

if this call is stuck for any reason, the report will block
the whole daemon given that at this point it has acquired a lock.
We need to make sure this call won't block the daemon for a long time,
let's add a timeout.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/reporter.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index 6be8d24c6b0a..3636e1ab3fa6 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -36,7 +36,8 @@ def loop(self) -> None:
                     log.logger.info('data has changed since last iteration.')
                     d = self.system.get_system()
                     try:
-                        requests.post(f"{self.observer_url}/fake_endpoint", json=d)
+                        # TODO: add a timeout parameter to the reporter in the config file
+                        requests.post(f"{self.observer_url}/fake_endpoint", json=d, timeout=5)
                     except requests.exceptions.RequestException as e:
                         log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't

From 3bb2863d5ac14fbadd609cfb3c494acc3ba8c9f0 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 08:08:38 +0200
Subject: [PATCH 1530/2492] node-proxy: rename System to BaseSystem

In order to avoid confusion or redefinition issue with class System()
defined in server.py.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/{system.py => basesystem.py} | 2 +-
 src/cephadm/node-proxy/redfish_system.py            | 4 ++--
 src/cephadm/node-proxy/server-v2.py                 | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)
 rename src/cephadm/node-proxy/{system.py => basesystem.py} (98%)

diff --git a/src/cephadm/node-proxy/system.py b/src/cephadm/node-proxy/basesystem.py
similarity index 98%
rename from src/cephadm/node-proxy/system.py
rename to src/cephadm/node-proxy/basesystem.py
index 9e8f97ef59d6..8dcdecbc3a74 100644
--- a/src/cephadm/node-proxy/system.py
+++ b/src/cephadm/node-proxy/basesystem.py
@@ -2,7 +2,7 @@
 from typing import Dict, Any
 
 
-class System:
+class BaseSystem:
     def __init__(self, **kw: Any) -> None:
         self._system: Dict = {}
         self.config: Config = kw['config']
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index ffa530cefbf0..eb1f323acb2c 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -1,4 +1,4 @@
-from system import System
+from basesystem import BaseSystem
 from redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
@@ -8,7 +8,7 @@
 log = Logger(__name__)
 
 
-class RedfishSystem(System):
+class RedfishSystem(BaseSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
         self.host: str = kw['host']
diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server-v2.py
index d1924629e6bd..f4afba4daa11 100644
--- a/src/cephadm/node-proxy/server-v2.py
+++ b/src/cephadm/node-proxy/server-v2.py
@@ -3,6 +3,7 @@
 from reporter import Reporter
 from util import Config, Logger
 from typing import Dict
+from basesystem import BaseSystem
 import sys
 
 # for devel purposes

From f2f87f4259bbfe1014f5a2309a82f5b08a8d78d3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 11:13:56 +0200
Subject: [PATCH 1531/2492] node-proxy: create entrypoint main()

This creates a `main()` function in server.py that will be the
entrypoint of node-proxy.

This also implement arg parsing and add a `--config` parameter
to specify the configuration file.

Finally, this introduce a small refactor of class `Config` and class
`Logger` in util.py because there was a circular dependency between them.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server-v2.py | 145 +++++++++++++++++++---------
 src/cephadm/node-proxy/util.py      |  59 +++++------
 2 files changed, 129 insertions(+), 75 deletions(-)

diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server-v2.py
index f4afba4daa11..4eef854bea55 100644
--- a/src/cephadm/node-proxy/server-v2.py
+++ b/src/cephadm/node-proxy/server-v2.py
@@ -5,6 +5,7 @@
 from typing import Dict
 from basesystem import BaseSystem
 import sys
+import argparse
 
 # for devel purposes
 import os
@@ -33,133 +34,179 @@
         print(f"{env_var} environment variable must be set.")
         sys.exit(1)
 
-config = Config(default_config=DEFAULT_CONFIG)
-
-log = Logger(__name__, level=config.logging['level'])
-# must be passed as arguments
-host = os.environ.get('REDFISH_HOST')
-username = os.environ.get('REDFISH_USERNAME')
-password = os.environ.get('REDFISH_PASSWORD')
-
-# create the redfish system and the obsever
-log.logger.info("Server initialization...")
-system = RedfishDell(host=host,
-                     username=username,
-                     password=password,
-                     system_endpoint='/Systems/System.Embedded.1',
-                     config=config)
-reporter_agent = Reporter(system, "http://127.0.0.1:8000")
-
 
 class Memory:
     exposed = True
 
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
     @cherrypy.tools.json_out()
     def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'memory': system.get_memory()}
+        return {'memory': self.backend.get_memory()}
 
 
 class Network:
     exposed = True
 
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
     @cherrypy.tools.json_out()
     def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'network': system.get_network()}
+        return {'network': self.backend.get_network()}
 
 
 class Processors:
     exposed = True
 
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
     @cherrypy.tools.json_out()
     def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'processors': system.get_processors()}
+        return {'processors': self.backend.get_processors()}
 
 
 class Storage:
     exposed = True
 
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
     @cherrypy.tools.json_out()
     def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'storage': system.get_storage()}
+        return {'storage': self.backend.get_storage()}
 
 
 class Status:
     exposed = True
 
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
     @cherrypy.tools.json_out()
     def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'status': system.get_status()}
+        return {'status': self.backend.get_status()}
 
 
 class System:
     exposed = True
-    memory = Memory()
-    network = Network()
-    processors = Processors()
-    storage = Storage()
-    status = Status()
-    # actions = Actions()
-    # control = Control()
+
+    def __init__(self, backend: BaseSystem) -> None:
+        self.memory = Memory(backend)
+        self.network = Network(backend)
+        self.processors = Processors(backend)
+        self.storage = Storage(backend)
+        self.status = Status(backend)
+        # actions = Actions()
+        # control = Control()
 
 
 class Shutdown:
     exposed = True
 
+    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
+        self.backend = backend
+        self.reporter = reporter
+
     def POST(self) -> str:
-        _stop()
+        _stop(self.backend, self.reporter)
         cherrypy.engine.exit()
         return 'Server shutdown...'
 
 
-def _stop() -> None:
-    system.stop_update_loop()
-    system.client.logout()
-    reporter_agent.stop()
+def _stop(backend: BaseSystem, reporter: Reporter) -> None:
+    backend.stop_update_loop()
+    backend.client.logout()
+    reporter.stop()
 
 
 class Start:
     exposed = True
 
+    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
+        self.backend = backend
+        self.reporter = reporter
+
     def POST(self) -> str:
-        system.start_client()
-        system.start_update_loop()
-        reporter_agent.run()
+        self.backend.start_client()
+        self.backend.start_update_loop()
+        self.reporter.run()
         return 'node-proxy daemon started'
 
 
 class Stop:
     exposed = True
 
+    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
+        self.backend = backend
+        self.reporter = reporter
+
     def POST(self) -> str:
-        _stop()
+        _stop(self.backend, self.reporter)
         return 'node-proxy daemon stopped'
 
 
 class ConfigReload:
     exposed = True
 
-    def __init__(self, config: cherrypy.config) -> None:
+    def __init__(self, config: Config) -> None:
         self.config = config
 
     def POST(self) -> str:
-        self.config['node_proxy'].reload()
+        self.config.reload()
         return 'node-proxy config reloaded'
 
 
 class API:
     exposed = True
 
-    system = System()
-    shutdown = Shutdown()
-    start = Start()
-    stop = Stop()
-    config_reload = ConfigReload(cherrypy.config)
+    def __init__(self,
+                 backend: BaseSystem,
+                 reporter: Reporter,
+                 config: Config) -> None:
+
+        self.system = System(backend)
+        self.shutdown = Shutdown(backend, reporter)
+        self.start = Start(backend, reporter)
+        self.stop = Stop(backend, reporter)
+        self.config_reload = ConfigReload(config)
 
     def GET(self) -> str:
         return 'use /system'
 
 
-if __name__ == '__main__':
+def main() -> None:
+
+    parser = argparse.ArgumentParser(
+        prog='node-proxy',
+    )
+    parser.add_argument(
+        '--config',
+        dest='config',
+        type=str,
+        required=False,
+        default='/etc/ceph/node-proxy.yml'
+    )
+
+    args = parser.parse_args()
+    config = Config(args.config, default_config=DEFAULT_CONFIG)
+
+    log = Logger(__name__, level=config.logging['level'])
+    # must be passed as arguments
+    host = os.environ.get('REDFISH_HOST')
+    username = os.environ.get('REDFISH_USERNAME')
+    password = os.environ.get('REDFISH_PASSWORD')
+
+    # create the redfish system and the obsever
+    log.logger.info("Server initialization...")
+    system = RedfishDell(host=host,
+                         username=username,
+                         password=password,
+                         system_endpoint='/Systems/System.Embedded.1',
+                         config=config)
+    reporter_agent = Reporter(system, "http://127.0.0.1:8000")
     cherrypy.config.update({
         'node_proxy': config,
         'server.socket_port': config.__dict__['server']['port']
@@ -170,4 +217,8 @@ def GET(self) -> str:
     }}
     system.start_update_loop()
     reporter_agent.run()
-    cherrypy.quickstart(API(), config=c)
+    cherrypy.quickstart(API(system, reporter_agent, config), config=c)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index 6420e904876f..112652f7bd76 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -5,14 +5,26 @@
 from typing import Dict, List, Callable, Any
 
 
-def normalize_dict(test_dict: Dict) -> Dict:
-    res = dict()
-    for key in test_dict.keys():
-        if isinstance(test_dict[key], dict):
-            res[key.lower()] = normalize_dict(test_dict[key])
-        else:
-            res[key.lower()] = test_dict[key]
-    return res
+class Logger:
+    _Logger: List['Logger'] = []
+
+    def __init__(self, name: str, level: int = logging.INFO):
+        self.name = name
+        self.level = level
+
+        Logger._Logger.append(self)
+        self.logger = self.get_logger()
+
+    def get_logger(self) -> logging.Logger:
+        logger = logging.getLogger(self.name)
+        logger.setLevel(self.level)
+        handler = logging.StreamHandler()
+        handler.setLevel(self.level)
+        fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(fmt)
+        logger.addHandler(handler)
+
+        return logger
 
 
 class Config:
@@ -50,26 +62,17 @@ def reload(self, config_file: str = '') -> None:
         self.load_config()
 
 
-class Logger:
-    _Logger: List['Logger'] = []
-
-    def __init__(self, name: str, level: int = logging.INFO):
-        self.name = name
-        self.level = level
+log = Logger(__name__)
 
-        Logger._Logger.append(self)
-        self.logger = self.get_logger()
 
-    def get_logger(self) -> logging.Logger:
-        logger = logging.getLogger(self.name)
-        logger.setLevel(self.level)
-        handler = logging.StreamHandler()
-        handler.setLevel(self.level)
-        fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        handler.setFormatter(fmt)
-        logger.addHandler(handler)
-
-        return logger
+def normalize_dict(test_dict: Dict) -> Dict:
+    res = dict()
+    for key in test_dict.keys():
+        if isinstance(test_dict[key], dict):
+            res[key.lower()] = normalize_dict(test_dict[key])
+        else:
+            res[key.lower()] = test_dict[key]
+    return res
 
 
 def retry(exceptions: Any = Exception, retries: int = 20, delay: int = 1) -> Callable:
@@ -78,12 +81,12 @@ def _retry(*args: str, **kwargs: Any) -> Callable:
             _tries = retries
             while _tries > 1:
                 try:
-                    print("{}".format(_tries))
+                    log.logger.debug("{} {} attempt(s) left.".format(f, _tries - 1))
                     return f(*args, **kwargs)
                 except exceptions:
                     time.sleep(delay)
                     _tries -= 1
-            print("{} has failed after {} tries".format(f, retries))
+            log.logger.warn("{} has failed after {} tries".format(f, retries))
             return f(*args, **kwargs)
         return _retry
     return decorator

From 8c1036374008d1422e2f3485012231a3d1da77b8 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 13:04:56 +0200
Subject: [PATCH 1532/2492] node-proxy: drop old server.py

This version relies on flask.
At the end, we decided to migrate to cherrypy given that
we already use it quite a lot in ceph/ceph

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 67 --------------------------------
 1 file changed, 67 deletions(-)
 delete mode 100644 src/cephadm/node-proxy/server.py

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
deleted file mode 100644
index f450f0523957..000000000000
--- a/src/cephadm/node-proxy/server.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from flask import Flask, request, jsonify
-from system import System
-from redfish_dell import RedfishDell
-from reporter import Reporter
-from util import logger
-import time
-
-log = logger(__name__)
-
-# must be passed as arguments
-host = "https://x.x.x.x:8443"
-username = "myuser"
-password = "mypassword"
-
-# create the redfish system and the obsever
-log.info(f"Server initialization...")
-system = RedfishDell(host=host, username=username, password=password, system_endpoint='/Systems/System.Embedded.1')
-reporter_agent = Reporter(system, "http://127.0.0.1:8000")
-
-app = Flask(__name__)
-
-@app.route('/shutdown', methods=['POST'])
-def shutdown():
-    system.stop_update_loop()
-    system.client.logout()
-    return 'Server shutting down...\n'
-
-@app.route('/system', methods=['GET'])
-def get_system():
-    return jsonify({'system': system.get_system()})
-
-@app.route('/system/memory', methods=['GET'])
-def get_system_memory():
-    return jsonify({'memory': system.get_memory()})
-
-@app.route('/system/network', methods=['GET'])
-def get_system_network():
-    return jsonify({'network': system.get_network()})
-
-@app.route('/system/processors', methods=['GET'])
-def get_system_processors():
-    return jsonify({'processors': system.get_processors()})
-
-@app.route('/system/storage', methods=['GET'])
-def get_system_storage():
-    return jsonify({'storage': system.get_storage()})
-
-@app.route('/system/status', methods=['GET'])
-def get_system_status():
-    return jsonify({'status': system.get_status()})
-
-@app.route('/system/actions/', methods=['POST'])
-def post_system():
-    pass
-
-@app.route('/system/actions/', methods=['PUT'])
-def put_system():
-    pass
-
-@app.route('/system/control/', methods=['DELETE'])
-def delete():
-    pass
-
-if __name__ == '__main__':
-    system.start_update_loop()
-    reporter_agent.run()
-    app.run(debug=True, use_reloader=False)

From 37f33ec87e830989dadd17dbfc0dfde1f58877c1 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 13:06:03 +0200
Subject: [PATCH 1533/2492] node-proxy: rename server-v2.py

As the previous version has been removed, let's rename this file.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/{server-v2.py => server.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/cephadm/node-proxy/{server-v2.py => server.py} (100%)

diff --git a/src/cephadm/node-proxy/server-v2.py b/src/cephadm/node-proxy/server.py
similarity index 100%
rename from src/cephadm/node-proxy/server-v2.py
rename to src/cephadm/node-proxy/server.py

From 5b6e762383efa7d1e846ac6c3ec1f912f6d60248 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 13:07:34 +0200
Subject: [PATCH 1534/2492] node-proxy: fix mypy warning regarding
 Config.logging

Config's attributes are dynamically created so mypy complains.
using `__dict__['logging']` addresses that.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 4eef854bea55..8d408aa7762b 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -193,7 +193,7 @@ def main() -> None:
     args = parser.parse_args()
     config = Config(args.config, default_config=DEFAULT_CONFIG)
 
-    log = Logger(__name__, level=config.logging['level'])
+    log = Logger(__name__, level=config.__dict__['logging']['level'])
     # must be passed as arguments
     host = os.environ.get('REDFISH_HOST')
     username = os.environ.get('REDFISH_USERNAME')

From 5cd39211401fcbbcb8a8e3441fd42043b45238dd Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 16 Jun 2023 13:09:48 +0200
Subject: [PATCH 1535/2492] node-proxy: RedfishClient class refactor

This implements BaseClient class and make RedfishClient inherit from it.
Same logic as BaseSystem / RedfishSystem given that any other backend could
need to implement a new client for collecting the data.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/baseclient.py     | 20 ++++++++++++++++++++
 src/cephadm/node-proxy/basesystem.py     | 11 +++++++++++
 src/cephadm/node-proxy/redfish_client.py |  7 +++----
 3 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100644 src/cephadm/node-proxy/baseclient.py

diff --git a/src/cephadm/node-proxy/baseclient.py b/src/cephadm/node-proxy/baseclient.py
new file mode 100644
index 000000000000..735dd11e96d9
--- /dev/null
+++ b/src/cephadm/node-proxy/baseclient.py
@@ -0,0 +1,20 @@
+from typing import Dict
+
+
+class BaseClient:
+    def __init__(self,
+                 host: str,
+                 username: str,
+                 password: str) -> None:
+        self.host = host
+        self.username = username
+        self.password = password
+
+    def login(self) -> None:
+        raise NotImplementedError()
+
+    def logout(self) -> None:
+        raise NotImplementedError()
+
+    def get_path(self, path: str) -> Dict:
+        raise NotImplementedError()
diff --git a/src/cephadm/node-proxy/basesystem.py b/src/cephadm/node-proxy/basesystem.py
index 8dcdecbc3a74..1ec6998add22 100644
--- a/src/cephadm/node-proxy/basesystem.py
+++ b/src/cephadm/node-proxy/basesystem.py
@@ -1,11 +1,13 @@
 from util import Config
 from typing import Dict, Any
+from baseclient import BaseClient
 
 
 class BaseSystem:
     def __init__(self, **kw: Any) -> None:
         self._system: Dict = {}
         self.config: Config = kw['config']
+        self.client: BaseClient
 
     def get_system(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
@@ -30,3 +32,12 @@ def get_network(self) -> Dict[str, Dict[str, Dict]]:
 
     def get_storage(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
+
+    def start_update_loop(self) -> None:
+        raise NotImplementedError()
+
+    def stop_update_loop(self) -> None:
+        raise NotImplementedError()
+
+    def start_client(self) -> None:
+        raise NotImplementedError()
diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node-proxy/redfish_client.py
index 6a3e384893d2..77353cd47819 100644
--- a/src/cephadm/node-proxy/redfish_client.py
+++ b/src/cephadm/node-proxy/redfish_client.py
@@ -4,12 +4,13 @@
 import redfish
 import sys
 from util import Logger
+from baseclient import BaseClient
 from typing import Dict
 
 log = Logger(__name__)
 
 
-class RedFishClient:
+class RedFishClient(BaseClient):
 
     PREFIX = '/redfish/v1'
 
@@ -18,9 +19,7 @@ def __init__(self,
                  username: str,
                  password: str) -> None:
         log.logger.info("redfish client initialization...")
-        self.host = host
-        self.username = username
-        self.password = password
+        super().__init__(host, username, password)
         self.redfish_obj: 'redfish.redfish_client' = None
 
     def login(self) -> 'redfish.redfish_client':

From c538030f9e70afc687ed1e5734d0d603fc4b0a31 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 13:28:55 +0200
Subject: [PATCH 1536/2492] node-proxy: redfish_dell.py refactor

This commit introduces a small refactor of `redfish_dell.py` in order
to avoid code redundancy.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_dell.py   | 67 ++++++++----------------
 src/cephadm/node-proxy/redfish_system.py | 24 ++++++++-
 src/cephadm/node-proxy/util.py           |  6 +++
 3 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index f940a9ec3c30..9bf69b324707 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -1,5 +1,5 @@
 from redfish_system import RedfishSystem
-from util import Logger, normalize_dict
+from util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any
 
 log = Logger(__name__)
@@ -12,62 +12,39 @@ def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
 
     def _update_network(self) -> None:
-        net_path = self._system['EthernetInterfaces']['@odata.id']
+        fields = ['Description', 'Name', 'SpeedMbps', 'Status']
         log.logger.info("Updating network")
-        network_info = self._get_path(net_path)
-        self._system['network'] = {}
-        result: Dict[str, Dict[str, Dict]] = dict()
-        for interface in network_info['Members']:
-            interface_path = interface['@odata.id']
-            interface_info = self._get_path(interface_path)
-            interface_id = interface_info['Id']
-            result[interface_id] = dict()
-            result[interface_id]['description'] = interface_info['Description']
-            result[interface_id]['name'] = interface_info['Name']
-            result[interface_id]['speed_mbps'] = interface_info['SpeedMbps']
-            result[interface_id]['status'] = interface_info['Status']
-        self._system['network'] = normalize_dict(result)
+        self._system['network'] = self.build_data(fields, 'EthernetInterfaces')
 
     def _update_processors(self) -> None:
-        cpus_path = self._system['Processors']['@odata.id']
+        fields = ['Description',
+                  'TotalCores',
+                  'TotalThreads',
+                  'ProcessorType',
+                  'Model',
+                  'Status',
+                  'Manufacturer']
         log.logger.info("Updating processors")
-        cpus_info = self._get_path(cpus_path)
-        self._system['processors'] = {}
-        result: Dict[str, Dict[str, Dict]] = dict()
-        for cpu in cpus_info['Members']:
-            cpu_path = cpu['@odata.id']
-            cpu_info = self._get_path(cpu_path)
-            cpu_id = cpu_info['Id']
-            result[cpu_id] = dict()
-            result[cpu_id]['description'] = cpu_info['Description']
-            result[cpu_id]['cores'] = cpu_info['TotalCores']
-            result[cpu_id]['threads'] = cpu_info['TotalThreads']
-            result[cpu_id]['type'] = cpu_info['ProcessorType']
-            result[cpu_id]['model'] = cpu_info['Model']
-            result[cpu_id]['status'] = cpu_info['Status']
-            result[cpu_id]['manufacturer'] = cpu_info['Manufacturer']
-        self._system['processors'] = normalize_dict(result)
+        self._system['processors'] = self.build_data(fields, 'Processors')
 
     def _update_storage(self) -> None:
-        storage_path = self._system['Storage']['@odata.id']
+        fields = ['Description',
+                  'CapacityBytes',
+                  'Model', 'Protocol',
+                  'SerialNumber', 'Status',
+                  'PhysicalLocation']
+        entities = self.get_members('Storage')
         log.logger.info("Updating storage")
-        storage_info = self._get_path(storage_path)
         result: Dict[str, Dict[str, Dict]] = dict()
-        for storage in storage_info['Members']:
-            entity_path = storage['@odata.id']
-            entity_info = self._get_path(entity_path)
-            for drive in entity_info['Drives']:
+        for entity in entities:
+            for drive in entity['Drives']:
                 drive_path = drive['@odata.id']
                 drive_info = self._get_path(drive_path)
                 drive_id = drive_info['Id']
                 result[drive_id] = dict()
-                result[drive_id]['description'] = drive_info['Description']
-                result[drive_id]['capacity_bytes'] = drive_info['CapacityBytes']
-                result[drive_id]['model'] = drive_info['Model']
-                result[drive_id]['protocol'] = drive_info['Protocol']
-                result[drive_id]['serial_number'] = drive_info['SerialNumber']
-                result[drive_id]['status'] = drive_info['Status']
-                result[drive_id]['location'] = drive_info['PhysicalLocation']
+                for field in fields:
+                    result[drive_id][to_snake_case(field)] = drive_info[field]
+                    result[drive_id]['entity'] = entity['Id']
         self._system['storage'] = normalize_dict(result)
 
     def _update_metadata(self) -> None:
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index eb1f323acb2c..a41a861df46c 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -2,8 +2,8 @@
 from redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from util import Logger, retry
-from typing import Dict, Any
+from util import Logger, retry, normalize_dict, to_snake_case
+from typing import Dict, Any, List
 
 log = Logger(__name__)
 
@@ -34,6 +34,26 @@ def _get_path(self, path: str) -> Dict:
             raise RuntimeError(f"Could not get path: {path}")
         return result
 
+    def get_members(self, path: str) -> List:
+        _path = self._system[path]['@odata.id']
+        data = self._get_path(_path)
+        return [self._get_path(member['@odata.id']) for member in data['Members']]
+
+    def build_data(self,
+                   fields: List,
+                   path: str) -> Dict[str, Dict[str, Dict]]:
+        result: Dict[str, Dict[str, Dict]] = dict()
+        for member_info in self.get_members(path):
+            member_id = member_info['Id']
+            result[member_id] = dict()
+            for field in fields:
+                try:
+                    result[member_id][to_snake_case(field)] = member_info[field]
+                except KeyError:
+                    log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
+
+        return normalize_dict(result)
+
     def start_client(self) -> None:
         if not self.client:
             self.client = RedFishClient(self.host, self.username, self.password)
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node-proxy/util.py
index 112652f7bd76..98c1a7d36712 100644
--- a/src/cephadm/node-proxy/util.py
+++ b/src/cephadm/node-proxy/util.py
@@ -2,6 +2,7 @@
 import yaml
 import os
 import time
+import re
 from typing import Dict, List, Callable, Any
 
 
@@ -65,6 +66,11 @@ def reload(self, config_file: str = '') -> None:
 log = Logger(__name__)
 
 
+def to_snake_case(name: str) -> str:
+    name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
+
+
 def normalize_dict(test_dict: Dict) -> Dict:
     res = dict()
     for key in test_dict.keys():

From b1d00d9a5a63fed9d866bc7c44c89b0b1580301d Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 13:30:36 +0200
Subject: [PATCH 1537/2492] node-proxy: implement _update_memory() in
 redfish_dell.py

This implements the `_update_memory()` method in redfish_dell.py

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/redfish_dell.py   | 6 +++++-
 src/cephadm/node-proxy/redfish_system.py | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node-proxy/redfish_dell.py
index 9bf69b324707..0f4467bad5ff 100644
--- a/src/cephadm/node-proxy/redfish_dell.py
+++ b/src/cephadm/node-proxy/redfish_dell.py
@@ -52,8 +52,12 @@ def _update_metadata(self) -> None:
         pass
 
     def _update_memory(self) -> None:
+        fields = ['Description',
+                  'MemoryDeviceType',
+                  'CapacityMiB',
+                  'Status']
         log.logger.info("Updating memory")
-        pass
+        self._system['memory'] = self.build_data(fields, 'Memory')
 
     def _update_power(self) -> None:
         log.logger.info("Updating power")
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index a41a861df46c..3da45ff11a54 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -64,6 +64,7 @@ def get_system(self) -> Dict[str, Dict[str, Dict]]:
             'storage': self.get_storage(),
             'processors': self.get_processors(),
             'network': self.get_network(),
+            'memory': self.get_memory(),
         }
         return result
 

From f71dad1a11abf73ab17028e8a983439401c3893f Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 13:31:40 +0200
Subject: [PATCH 1538/2492] node-proxy: update endpoint url in Reporter.loop()

change the path of the endpoint to something more generic

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index 3636e1ab3fa6..a5ba531d1a8d 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -37,7 +37,7 @@ def loop(self) -> None:
                     d = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
-                        requests.post(f"{self.observer_url}/fake_endpoint", json=d, timeout=5)
+                        requests.post(f"{self.observer_url}/", json=d, timeout=5)
                     except requests.exceptions.RequestException as e:
                         log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't

From ecbbcb432f1b4d08f4e2d011d821a30e102dd89a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 13:33:14 +0200
Subject: [PATCH 1539/2492] node-proxy: variabilize the observer_url

create a new parameter in DEFAULT_CONFIG for the reporter agent.
The default value, (especially the tcp port) still has to be defined though.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 8d408aa7762b..5f50ef3a03ac 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -17,6 +17,7 @@
     'reporter': {
         'check_interval': 5,
         'push_data_max_retries': 30,
+        'endpoint': 'http://127.0.0.1:8150',
     },
     'system': {
         'refresh_interval': 5
@@ -206,7 +207,7 @@ def main() -> None:
                          password=password,
                          system_endpoint='/Systems/System.Embedded.1',
                          config=config)
-    reporter_agent = Reporter(system, "http://127.0.0.1:8000")
+    reporter_agent = Reporter(system, config.__dict__['reporter']['endpoint'])
     cherrypy.config.update({
         'node_proxy': config,
         'server.socket_port': config.__dict__['server']['port']

From 3f7384c7e1a9656dcc91fcd9e34c9095371a2a1e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 14:24:42 +0200
Subject: [PATCH 1540/2492] node-proxy: try to acquire lock early in reporter's
 loop

The lock should be acquired early in this loop.

If the lock gets acquired by another call after we enter that condition *and*
before Reporter.loop() actually acquires it, it can lead to issue if during
this short amount of time the value of `data_ready` gets modified

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/reporter.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node-proxy/reporter.py
index a5ba531d1a8d..07ac637bf1c8 100644
--- a/src/cephadm/node-proxy/reporter.py
+++ b/src/cephadm/node-proxy/reporter.py
@@ -28,10 +28,11 @@ def loop(self) -> None:
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
+            log.logger.debug("waiting for a lock.")
+            self.system.lock.acquire()
+            log.logger.debug("lock acquired.")
             if self.system.data_ready:
-                log.logger.debug("waiting for a lock.")
-                self.system.lock.acquire()
-                log.logger.debug("lock acquired.")
+                log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
                     log.logger.info('data has changed since last iteration.')
                     d = self.system.get_system()
@@ -46,6 +47,6 @@ def loop(self) -> None:
                         self.system.previous_data = self.system.get_system()
                 else:
                     log.logger.info('no diff, not sending data to the mgr.')
-                self.system.lock.release()
-                log.logger.debug("lock released.")
+            self.system.lock.release()
+            log.logger.debug("lock released.")
             time.sleep(5)

From 6677a6838493d5c6c6600edcf02d17a95f36b965 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 14:33:42 +0200
Subject: [PATCH 1541/2492] node-proxy: add new endpoint for flushing the data

Although this is mostly for devel and debug purposes at the moment,
it might be useful to be able to flush the data whenever the user needs it.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/basesystem.py     |  3 +++
 src/cephadm/node-proxy/redfish_system.py | 12 ++++++++++++
 src/cephadm/node-proxy/server.py         | 10 ++++++++++
 3 files changed, 25 insertions(+)

diff --git a/src/cephadm/node-proxy/basesystem.py b/src/cephadm/node-proxy/basesystem.py
index 1ec6998add22..a56ad7e8f9d8 100644
--- a/src/cephadm/node-proxy/basesystem.py
+++ b/src/cephadm/node-proxy/basesystem.py
@@ -41,3 +41,6 @@ def stop_update_loop(self) -> None:
 
     def start_client(self) -> None:
         raise NotImplementedError()
+
+    def flush(self) -> None:
+        raise NotImplementedError()
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node-proxy/redfish_system.py
index 3da45ff11a54..f23e41af79ec 100644
--- a/src/cephadm/node-proxy/redfish_system.py
+++ b/src/cephadm/node-proxy/redfish_system.py
@@ -147,3 +147,15 @@ def update(self) -> None:
             log.logger.error(f"Error detected, logging out from redfish api.\n{e}")
             self.client.logout()
             raise
+
+    def flush(self) -> None:
+        log.logger.info("Acquiring lock to flush data.")
+        self.lock.acquire()
+        log.logger.info("Lock acquired, flushing data.")
+        self._system = {}
+        self.previous_data = {}
+        log.logger.info("Data flushed.")
+        self.data_ready = False
+        log.logger.info("Data marked as not ready.")
+        self.lock.release()
+        log.logger.info("Lock released.")
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 5f50ef3a03ac..21a3c29b165b 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -160,6 +160,16 @@ def POST(self) -> str:
         return 'node-proxy config reloaded'
 
 
+class Flush:
+    exposed = True
+
+    def __init__(self, backend: BaseSystem) -> None:
+        self.backend = backend
+
+    def POST(self) -> str:
+        self.backend.flush()
+        return 'node-proxy data flushed'
+
 class API:
     exposed = True
 

From 2995c6a277159735002686d48484df7d6ae25ac0 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 20 Jun 2023 14:35:02 +0200
Subject: [PATCH 1542/2492] node-proxy: move administration operations to
 /admin path

This adds a new path /admin where all administrator operation are grouped.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/node-proxy/server.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node-proxy/server.py
index 21a3c29b165b..d05dbebd9c68 100644
--- a/src/cephadm/node-proxy/server.py
+++ b/src/cephadm/node-proxy/server.py
@@ -170,6 +170,18 @@ def POST(self) -> str:
         self.backend.flush()
         return 'node-proxy data flushed'
 
+
+class Admin:
+    exposed = False
+
+    def __init__(self, backend: BaseSystem, config: Config, reporter: Reporter) -> None:
+        self.reload = ConfigReload(config)
+        self.flush = Flush(backend)
+        self.shutdown = Shutdown(backend, reporter)
+        self.start = Start(backend, reporter)
+        self.stop = Stop(backend, reporter)
+
+
 class API:
     exposed = True
 
@@ -179,13 +191,10 @@ def __init__(self,
                  config: Config) -> None:
 
         self.system = System(backend)
-        self.shutdown = Shutdown(backend, reporter)
-        self.start = Start(backend, reporter)
-        self.stop = Stop(backend, reporter)
-        self.config_reload = ConfigReload(config)
+        self.admin = Admin(backend, config, reporter)
 
     def GET(self) -> str:
-        return 'use /system'
+        return 'use /system or /admin endpoints'
 
 
 def main() -> None:

From 300c99a2f0afd5999938e7e614188b80ee61853b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 22 Jun 2023 15:54:55 +0200
Subject: [PATCH 1543/2492] node-proxy: add unit tests for node-proxy endpoint

This adds some unit tests for the node-proxy endpoint recently added to
the mgr.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../mgr/cephadm/tests/node_proxy_data.py      | 340 ++++++++++++++++++
 .../mgr/cephadm/tests/test_node_proxy.py      | 297 +++++++++++++++
 2 files changed, 637 insertions(+)
 create mode 100644 src/pybind/mgr/cephadm/tests/node_proxy_data.py
 create mode 100644 src/pybind/mgr/cephadm/tests/test_node_proxy.py

diff --git a/src/pybind/mgr/cephadm/tests/node_proxy_data.py b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
new file mode 100644
index 000000000000..1073bed274cd
--- /dev/null
+++ b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
@@ -0,0 +1,340 @@
+full_set_with_critical = {
+  "host": "host01",
+  "sn": "12345",
+  "status": {
+    "storage": {
+      "disk.bay.0:enclosure.internal.0-1:raid.integrated.1-1": {
+        "description": "Solid State Disk 0:1:0",
+        "entity": "RAID.Integrated.1-1",
+        "capacity_bytes": 959656755200,
+        "model": "KPM5XVUG960G",
+        "protocol": "SAS",
+        "serial_number": "8080A1CRTP5F",
+        "status": {
+          "health": "Critical",
+          "healthrollup": "OK",
+          "state": "Enabled"
+        },
+        "physical_location": {
+          "partlocation": {
+            "locationordinalvalue": 0,
+            "locationtype": "Slot"
+          }
+        }
+      },
+      "disk.bay.9:enclosure.internal.0-1": {
+        "description": "PCIe SSD in Slot 9 in Bay 1",
+        "entity": "CPU.1",
+        "capacity_bytes": 1600321314816,
+        "model": "Dell Express Flash NVMe P4610 1.6TB SFF",
+        "protocol": "PCIe",
+        "serial_number": "PHLN035305MN1P6AGN",
+        "status": {
+          "health": "Critical",
+          "healthrollup": "OK",
+          "state": "Enabled"
+        },
+        "physical_location": {
+          "partlocation": {
+            "locationordinalvalue": 9,
+            "locationtype": "Slot"
+          }
+        }
+      }
+    },
+    "processors": {
+      "cpu.socket.2": {
+        "description": "Represents the properties of a Processor attached to this System",
+        "total_cores": 20,
+        "total_threads": 40,
+        "processor_type": "CPU",
+        "model": "Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz",
+        "status": {
+          "health": "OK",
+          "state": "Enabled"
+        },
+        "manufacturer": "Intel"
+      },
+      
+    },
+    "network": {
+      "nic.slot.1-1-1": {
+        "description": "NIC in Slot 1 Port 1 Partition 1",
+        "name": "System Ethernet Interface",
+        "speed_mbps": 0,
+        "status": {
+          "health": "OK",
+          "state": "StandbyOffline"
+        }
+      }
+    },
+    "memory": {
+      "dimm.socket.a1": {
+        "description": "DIMM A1",
+        "memory_device_type": "DDR4",
+        "capacity_mi_b": 31237,
+        "status": {
+          "health": "Critical",
+          "state": "Enabled"
+        }
+      }
+    }
+  },
+  "firmwares": {
+    
+  }
+}
+
+mgr_inventory_cache = {"host01": {"hostname": "host01",
+                                  "addr": "10.10.10.11",
+                                  "labels": ["_admin"],
+                                  "status": "",
+                                  "idrac": {"hostname": "10.10.10.11",
+                                            "username": "root",
+                                            "password": "ceph123"}},
+                       "host02": {"hostname": "host02",
+                                  "addr": "10.10.10.12",
+                                  "labels": [],
+                                  "status": "",
+                                  "idrac": {"hostname": "10.10.10.12",
+                                            "username": "root",
+                                            "password": "ceph123"}}}
+
+full_set = {
+  "host01": {
+    "host": "host01",
+    "sn": "FR8Y5X3",
+    "status": {
+      "storage": {
+        "disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1": {
+          "description": "Disk 8 in Backplane 1 of Storage Controller in Slot 2",
+          "entity": "NonRAID.Slot.2-1",
+          "capacity_bytes": 20000588955136,
+          "model": "ST20000NM008D-3D",
+          "protocol": "SATA",
+          "serial_number": "ZVT99QLL",
+          "status": {
+            "health": "OK",
+            "healthrollup": "OK",
+            "state": "Enabled"
+          },
+          "physical_location": {
+            "partlocation": {
+              "locationordinalvalue": 8,
+              "locationtype": "Slot"
+            }
+          }
+        }
+      },
+      "processors": {
+        "cpu.socket.2": {
+          "description": "Represents the properties of a Processor attached to this System",
+          "total_cores": 16,
+          "total_threads": 32,
+          "processor_type": "CPU",
+          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          },
+          "manufacturer": "Intel"
+        },
+        "cpu.socket.1": {
+          "description": "Represents the properties of a Processor attached to this System",
+          "total_cores": 16,
+          "total_threads": 32,
+          "processor_type": "CPU",
+          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          },
+          "manufacturer": "Intel"
+        }
+      },
+      "network": {
+        "oslogicalnetwork.2": {
+          "description": "eno8303",
+          "name": "eno8303",
+          "speed_mbps": 0,
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "memory": {
+        "dimm.socket.a1": {
+          "description": "DIMM A1",
+          "memory_device_type": "DDR4",
+          "capacity_mi_b": 16384,
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "power": {
+        "0": {
+          "name": "PS1 Status",
+          "model": "PWR SPLY,800W,RDNT,LTON",
+          "manufacturer": "DELL",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        },
+        "1": {
+          "name": "PS2 Status",
+          "model": "PWR SPLY,800W,RDNT,LTON",
+          "manufacturer": "DELL",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "fans": {
+        "0": {
+          "name": "System Board Fan1A",
+          "physical_context": "SystemBoard",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      }
+    },
+    "firmwares": {
+      "installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1": {
+        "name": "Lifecycle Controller",
+        "description": "Represents Firmware Inventory",
+        "release_date": "00:00:00Z",
+        "version": "6.10.30.20",
+        "updateable": True,
+        "status": {
+          "health": "OK",
+          "state": "Enabled"
+        }
+      }
+    }
+  },
+"host02": {
+    "host": "host02",
+    "sn": "FR8Y5X4",
+    "status": {
+      "storage": {
+        "disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1": {
+          "description": "Disk 8 in Backplane 1 of Storage Controller in Slot 2",
+          "entity": "NonRAID.Slot.2-1",
+          "capacity_bytes": 20000588955136,
+          "model": "ST20000NM008D-3D",
+          "protocol": "SATA",
+          "serial_number": "ZVT99QLL",
+          "status": {
+            "health": "OK",
+            "healthrollup": "OK",
+            "state": "Enabled"
+          },
+          "physical_location": {
+            "partlocation": {
+              "locationordinalvalue": 8,
+              "locationtype": "Slot"
+            }
+          }
+        }
+      },
+      "processors": {
+        "cpu.socket.2": {
+          "description": "Represents the properties of a Processor attached to this System",
+          "total_cores": 16,
+          "total_threads": 32,
+          "processor_type": "CPU",
+          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          },
+          "manufacturer": "Intel"
+        },
+        "cpu.socket.1": {
+          "description": "Represents the properties of a Processor attached to this System",
+          "total_cores": 16,
+          "total_threads": 32,
+          "processor_type": "CPU",
+          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          },
+          "manufacturer": "Intel"
+        }
+      },
+      "network": {
+        "oslogicalnetwork.2": {
+          "description": "eno8303",
+          "name": "eno8303",
+          "speed_mbps": 0,
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "memory": {
+        "dimm.socket.a1": {
+          "description": "DIMM A1",
+          "memory_device_type": "DDR4",
+          "capacity_mi_b": 16384,
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "power": {
+        "0": {
+          "name": "PS1 Status",
+          "model": "PWR SPLY,800W,RDNT,LTON",
+          "manufacturer": "DELL",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        },
+        "1": {
+          "name": "PS2 Status",
+          "model": "PWR SPLY,800W,RDNT,LTON",
+          "manufacturer": "DELL",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      },
+      "fans": {
+        "0": {
+          "name": "System Board Fan1A",
+          "physical_context": "SystemBoard",
+          "status": {
+            "health": "OK",
+            "state": "Enabled"
+          }
+        }
+      }
+    },
+    "firmwares": {
+      "installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1": {
+        "name": "Lifecycle Controller",
+        "description": "Represents Firmware Inventory",
+        "release_date": "00:00:00Z",
+        "version": "6.10.30.20",
+        "updateable": True,
+        "status": {
+          "health": "OK",
+          "state": "Enabled"
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
new file mode 100644
index 000000000000..42ab39139dda
--- /dev/null
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -0,0 +1,297 @@
+import pytest
+import cherrypy
+import json
+from _pytest.monkeypatch import MonkeyPatch
+from cherrypy.test import helper
+from cephadm.agent import NodeProxy
+from unittest.mock import MagicMock, call
+from cephadm.http_server import CephadmHttpServer
+from cephadm.inventory import AgentCache, NodeProxyCache, Inventory
+from cephadm.ssl_cert_utils import SSLCerts
+from urllib.error import URLError
+from . import node_proxy_data
+
+PORT = 58585
+
+
+class FakeMgr:
+    def __init__(self) -> None:
+        self.log = MagicMock()
+        self.get_store = MagicMock(return_value=json.dumps(node_proxy_data.mgr_inventory_cache))
+        self.set_store = MagicMock()
+        self.set_health_warning = MagicMock()
+        self.remove_health_warning = MagicMock()
+        self.inventory = Inventory(self)
+        self.agent_cache = AgentCache(self)
+        self.node_proxy = NodeProxyCache(self)
+        self.node_proxy.save = MagicMock()
+        self.http_server = MagicMock()
+        self.http_server.agent = MagicMock()
+        self.http_server.agent.ssl_certs = SSLCerts()
+        self.http_server.agent.ssl_certs.generate_root_cert(self.get_mgr_ip())
+
+    def get_mgr_ip(self) -> str:
+        return '0.0.0.0'
+
+class TestNodeProxy(helper.CPWebCase):
+    mgr = FakeMgr()
+    app = NodeProxy(mgr)
+    mgr.agent_cache.agent_keys = {"host01": "fake-secret01",
+                                  "host02": "fake-secret02"}
+    mgr.node_proxy.idrac = {"host01": {"username": "idrac-user01",
+                                       "password": "idrac-pass01"},
+                            "host02": {"username": "idrac-user02",
+                                       "password": "idrac-pass02"}}
+    mgr.node_proxy.data = node_proxy_data.full_set
+
+    @classmethod
+    def setup_server(cls):
+        # cherrypy.tree.mount(NodeProxy(TestNodeProxy.mgr))
+        cherrypy.tree.mount(TestNodeProxy.app)
+        cherrypy.config.update({'global': {
+            'server.socket_host': '127.0.0.1',
+            'server.socket_port': PORT}})
+
+    def setUp(self):
+        self.PORT = PORT
+        self.monkeypatch = MonkeyPatch()
+
+    def test_idrac_data_misses_cephx_field(self):
+        data = '{}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('400 Bad Request')
+
+    def test_idrac_data_misses_name_field(self):
+        data = '{"cephx": {"secret": "fake-secret"}}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('400 Bad Request')
+
+    def test_idrac_data_misses_secret_field(self):
+        data = '{"cephx": {"name": "host01"}}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('400 Bad Request')
+
+    def test_idrac_agent_not_running(self):
+        data = '{"cephx": {"name": "host03", "secret": "fake-secret03"}}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('502 Bad Gateway')
+
+    def test_idrac_wrong_keyring(self):
+        data = '{"cephx": {"name": "host01", "secret": "wrong-keyring"}}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('403 Forbidden')
+
+    def test_idrac_ok(self):
+        data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
+        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                  ('Content-Length', str(len(data)))])
+        self.assertStatus('200 OK')
+
+    def test_data_missing_patch(self):
+        data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
+        self.getPage("/data", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+                                                                 ('Content-Length', str(len(data)))])
+        self.assertStatus('400 Bad Request')
+
+    def test_data_raises_alert(self):
+        patch = node_proxy_data.full_set_with_critical
+        data = {"cephx": {"name": "host01", "secret": "fake-secret01"}, "patch": patch}
+        data_str = json.dumps(data)
+        self.getPage("/data", method="POST", body=data_str, headers=[('Content-Type', 'application/json'),
+                                                                     ('Content-Length', str(len(data_str)))])
+        self.assertStatus('200 OK')
+
+        calls = [call('HARDWARE_STORAGE',
+                      count=2,
+                      detail=['disk.bay.0:enclosure.internal.0-1:raid.integrated.1-1 is critical: Enabled',
+                              'disk.bay.9:enclosure.internal.0-1 is critical: Enabled'],
+                      summary='2 storage members are not ok'),
+                 call('HARDWARE_MEMORY',
+                      count=1,
+                      detail=['dimm.socket.a1 is critical: Enabled'],
+                      summary='1 memory member is not ok')]
+
+        assert TestNodeProxy.mgr.set_health_warning.mock_calls == calls
+
+    # @pytest.mark.parametrize("method", ["GET", "PATCH"])
+    # def test_led_no_hostname(self, method):
+    #     self.getPage("/led", method=method)
+    #     self.assertStatus('501 Not Implemented')
+
+    def test_led_GET_no_hostname(self):
+        self.getPage("/led", method="GET")
+        self.assertStatus('501 Not Implemented')
+
+    def test_led_PATCH_no_hostname(self):
+        data = "{}"
+        self.getPage("/led", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
+                                                                 ('Content-Length', str(len(data)))])
+        self.assertStatus('501 Not Implemented')
+
+    def test_set_led(self):
+        data = '{"state": "on"}'
+        TestNodeProxy.app.query_endpoint = MagicMock(return_value=(200, "OK"))
+        # self.monkeypatch.setattr(NodeProxy, "query_endpoint", lambda *a, **kw: (200, "OK"))
+        self.getPage("/host01/led", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
+                                                                        ('Content-Length', str(len(data)))])
+
+        calls = [call(addr='10.10.10.11',
+                      data='{"state": "on"}',
+                      endpoint='/led',
+                      headers={'Authorization': 'Basic aWRyYWMtdXNlcjAxOmlkcmFjLXBhc3MwMQ=='},
+                      method='PATCH',
+                      port=8080,
+                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+        self.assertStatus('200 OK')
+        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
+
+    def test_get_led(self):
+        TestNodeProxy.app.query_endpoint = MagicMock(return_value=(200, "OK"))
+        self.getPage("/host01/led", method="GET")
+        calls = [call(addr='10.10.10.11',
+                      data=None,
+                      endpoint='/led',
+                      headers={},
+                      method='GET',
+                      port=8080,
+                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+        self.assertStatus('200 OK')
+        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
+
+    def test_led_endpoint_unreachable(self):
+        TestNodeProxy.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
+        self.getPage("/host02/led", method="GET")
+        calls = [call(addr='10.10.10.12',
+                      data=None,
+                      endpoint='/led',
+                      headers={},
+                      method='GET',
+                      port=8080,
+                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+        self.assertStatus('502 Bad Gateway')
+        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
+
+    def test_fullreport_with_valid_hostname(self):
+        self.getPage("/host02/fullreport", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_fullreport_no_hostname(self):
+        self.getPage("/fullreport", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_fullreport_with_invalid_hostname(self):
+        self.getPage("/host03/fullreport", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_summary_with_valid_hostname(self):
+        self.getPage("/host02/summary", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_summary_no_hostname(self):
+        self.getPage("/summary", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_summary_with_invalid_hostname(self):
+        self.getPage("/host03/summary", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_criticals_with_valid_hostname(self):
+        self.getPage("/host02/criticals", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_criticals_no_hostname(self):
+        self.getPage("/criticals", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_criticals_with_invalid_hostname(self):
+        self.getPage("/host03/criticals", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_memory_with_valid_hostname(self):
+        self.getPage("/host02/memory", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_memory_no_hostname(self):
+        self.getPage("/memory", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_memory_with_invalid_hostname(self):
+        self.getPage("/host03/memory", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_network_with_valid_hostname(self):
+        self.getPage("/host02/network", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_network_no_hostname(self):
+        self.getPage("/network", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_network_with_invalid_hostname(self):
+        self.getPage("/host03/network", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_processors_with_valid_hostname(self):
+        self.getPage("/host02/processors", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_processors_no_hostname(self):
+        self.getPage("/processors", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_processors_with_invalid_hostname(self):
+        self.getPage("/host03/processors", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_storage_with_valid_hostname(self):
+        self.getPage("/host02/storage", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_storage_no_hostname(self):
+        self.getPage("/storage", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_storage_with_invalid_hostname(self):
+        self.getPage("/host03/storage", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_power_with_valid_hostname(self):
+        self.getPage("/host02/power", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_power_no_hostname(self):
+        self.getPage("/power", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_power_with_invalid_hostname(self):
+        self.getPage("/host03/power", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_fans_with_valid_hostname(self):
+        self.getPage("/host02/fans", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_fans_no_hostname(self):
+        self.getPage("/fans", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_fans_with_invalid_hostname(self):
+        self.getPage("/host03/fans", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_firmwares_with_valid_hostname(self):
+        self.getPage("/host02/firmwares", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_firmwares_no_hostname(self):
+        self.getPage("/firmwares", method="GET")
+        self.assertStatus('200 OK')
+
+    def test_firmwares_with_invalid_hostname(self):
+        self.getPage("/host03/firmwares", method="GET")
+        self.assertStatus('404 Not Found')
\ No newline at end of file

From 400edcbd05305baed8d790aeefe48958a28d2b18 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 17 Aug 2023 11:18:10 +0200
Subject: [PATCH 1544/2492] node-proxy: rename directory

this renames the node-proxy directory node-proxy > node_proxy

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/{node-proxy => node_proxy}/baseclient.py              | 0
 src/cephadm/{node-proxy => node_proxy}/basesystem.py              | 0
 src/cephadm/{node-proxy => node_proxy}/data.py                    | 0
 .../{node-proxy => node_proxy}/fake_cephadm/cephadm_mgr_module.py | 0
 src/cephadm/{node-proxy => node_proxy}/main.py                    | 0
 src/cephadm/{node-proxy => node_proxy}/redfish-test.py            | 0
 src/cephadm/{node-proxy => node_proxy}/redfish_client.py          | 0
 src/cephadm/{node-proxy => node_proxy}/redfish_dell.py            | 0
 .../redfish_json_samples/interface_sample                         | 0
 .../redfish_json_samples/interfaces_sample                        | 0
 .../{node-proxy => node_proxy}/redfish_json_samples/memory        | 0
 .../{node-proxy => node_proxy}/redfish_json_samples/memory_socket | 0
 .../{node-proxy => node_proxy}/redfish_json_samples/processor     | 0
 .../{node-proxy => node_proxy}/redfish_json_samples/processors    | 0
 .../redfish_json_samples/storage_sample                           | 0
 .../{node-proxy => node_proxy}/redfish_json_samples/system        | 0
 src/cephadm/{node-proxy => node_proxy}/redfish_system.py          | 0
 src/cephadm/{node-proxy => node_proxy}/reporter.py                | 0
 src/cephadm/{node-proxy => node_proxy}/requirements.txt           | 0
 src/cephadm/{node-proxy => node_proxy}/server.py                  | 0
 src/cephadm/{node-proxy => node_proxy}/util.py                    | 0
 21 files changed, 0 insertions(+), 0 deletions(-)
 rename src/cephadm/{node-proxy => node_proxy}/baseclient.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/basesystem.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/data.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/fake_cephadm/cephadm_mgr_module.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/main.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish-test.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_client.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_dell.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/interface_sample (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/interfaces_sample (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/memory (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/memory_socket (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/processor (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/processors (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/storage_sample (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_json_samples/system (100%)
 rename src/cephadm/{node-proxy => node_proxy}/redfish_system.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/reporter.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/requirements.txt (100%)
 rename src/cephadm/{node-proxy => node_proxy}/server.py (100%)
 rename src/cephadm/{node-proxy => node_proxy}/util.py (100%)

diff --git a/src/cephadm/node-proxy/baseclient.py b/src/cephadm/node_proxy/baseclient.py
similarity index 100%
rename from src/cephadm/node-proxy/baseclient.py
rename to src/cephadm/node_proxy/baseclient.py
diff --git a/src/cephadm/node-proxy/basesystem.py b/src/cephadm/node_proxy/basesystem.py
similarity index 100%
rename from src/cephadm/node-proxy/basesystem.py
rename to src/cephadm/node_proxy/basesystem.py
diff --git a/src/cephadm/node-proxy/data.py b/src/cephadm/node_proxy/data.py
similarity index 100%
rename from src/cephadm/node-proxy/data.py
rename to src/cephadm/node_proxy/data.py
diff --git a/src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py b/src/cephadm/node_proxy/fake_cephadm/cephadm_mgr_module.py
similarity index 100%
rename from src/cephadm/node-proxy/fake_cephadm/cephadm_mgr_module.py
rename to src/cephadm/node_proxy/fake_cephadm/cephadm_mgr_module.py
diff --git a/src/cephadm/node-proxy/main.py b/src/cephadm/node_proxy/main.py
similarity index 100%
rename from src/cephadm/node-proxy/main.py
rename to src/cephadm/node_proxy/main.py
diff --git a/src/cephadm/node-proxy/redfish-test.py b/src/cephadm/node_proxy/redfish-test.py
similarity index 100%
rename from src/cephadm/node-proxy/redfish-test.py
rename to src/cephadm/node_proxy/redfish-test.py
diff --git a/src/cephadm/node-proxy/redfish_client.py b/src/cephadm/node_proxy/redfish_client.py
similarity index 100%
rename from src/cephadm/node-proxy/redfish_client.py
rename to src/cephadm/node_proxy/redfish_client.py
diff --git a/src/cephadm/node-proxy/redfish_dell.py b/src/cephadm/node_proxy/redfish_dell.py
similarity index 100%
rename from src/cephadm/node-proxy/redfish_dell.py
rename to src/cephadm/node_proxy/redfish_dell.py
diff --git a/src/cephadm/node-proxy/redfish_json_samples/interface_sample b/src/cephadm/node_proxy/redfish_json_samples/interface_sample
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/interface_sample
rename to src/cephadm/node_proxy/redfish_json_samples/interface_sample
diff --git a/src/cephadm/node-proxy/redfish_json_samples/interfaces_sample b/src/cephadm/node_proxy/redfish_json_samples/interfaces_sample
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/interfaces_sample
rename to src/cephadm/node_proxy/redfish_json_samples/interfaces_sample
diff --git a/src/cephadm/node-proxy/redfish_json_samples/memory b/src/cephadm/node_proxy/redfish_json_samples/memory
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/memory
rename to src/cephadm/node_proxy/redfish_json_samples/memory
diff --git a/src/cephadm/node-proxy/redfish_json_samples/memory_socket b/src/cephadm/node_proxy/redfish_json_samples/memory_socket
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/memory_socket
rename to src/cephadm/node_proxy/redfish_json_samples/memory_socket
diff --git a/src/cephadm/node-proxy/redfish_json_samples/processor b/src/cephadm/node_proxy/redfish_json_samples/processor
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/processor
rename to src/cephadm/node_proxy/redfish_json_samples/processor
diff --git a/src/cephadm/node-proxy/redfish_json_samples/processors b/src/cephadm/node_proxy/redfish_json_samples/processors
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/processors
rename to src/cephadm/node_proxy/redfish_json_samples/processors
diff --git a/src/cephadm/node-proxy/redfish_json_samples/storage_sample b/src/cephadm/node_proxy/redfish_json_samples/storage_sample
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/storage_sample
rename to src/cephadm/node_proxy/redfish_json_samples/storage_sample
diff --git a/src/cephadm/node-proxy/redfish_json_samples/system b/src/cephadm/node_proxy/redfish_json_samples/system
similarity index 100%
rename from src/cephadm/node-proxy/redfish_json_samples/system
rename to src/cephadm/node_proxy/redfish_json_samples/system
diff --git a/src/cephadm/node-proxy/redfish_system.py b/src/cephadm/node_proxy/redfish_system.py
similarity index 100%
rename from src/cephadm/node-proxy/redfish_system.py
rename to src/cephadm/node_proxy/redfish_system.py
diff --git a/src/cephadm/node-proxy/reporter.py b/src/cephadm/node_proxy/reporter.py
similarity index 100%
rename from src/cephadm/node-proxy/reporter.py
rename to src/cephadm/node_proxy/reporter.py
diff --git a/src/cephadm/node-proxy/requirements.txt b/src/cephadm/node_proxy/requirements.txt
similarity index 100%
rename from src/cephadm/node-proxy/requirements.txt
rename to src/cephadm/node_proxy/requirements.txt
diff --git a/src/cephadm/node-proxy/server.py b/src/cephadm/node_proxy/server.py
similarity index 100%
rename from src/cephadm/node-proxy/server.py
rename to src/cephadm/node_proxy/server.py
diff --git a/src/cephadm/node-proxy/util.py b/src/cephadm/node_proxy/util.py
similarity index 100%
rename from src/cephadm/node-proxy/util.py
rename to src/cephadm/node_proxy/util.py

From 83661b6c1a25b2d40f3cefa9f5de094c644a1e4e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 17 Aug 2023 11:21:00 +0200
Subject: [PATCH 1545/2492] node-proxy: migrate to cephadm-agent

This moves the existing files to the new directory 'cephadmlib' so
we can make the existing code for node-proxy run within the cephadm
agent. Indeed, we can leverage the existing code for the cephadm agent
given that both daemons would achieve the same thing.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        |  4 ++
 .../{ => cephadmlib}/node_proxy/baseclient.py |  0
 .../{ => cephadmlib}/node_proxy/basesystem.py |  4 +-
 .../{ => cephadmlib}/node_proxy/data.py       |  0
 .../fake_cephadm/cephadm_mgr_module.py        |  0
 .../{ => cephadmlib}/node_proxy/main.py       |  0
 .../node_proxy/redfish-test.py                |  0
 .../node_proxy/redfish_client.py              |  4 +-
 .../node_proxy/redfish_dell.py                |  4 +-
 .../redfish_json_samples/interface_sample     |  0
 .../redfish_json_samples/interfaces_sample    |  0
 .../node_proxy/redfish_json_samples/memory    |  0
 .../redfish_json_samples/memory_socket        |  0
 .../node_proxy/redfish_json_samples/processor |  0
 .../redfish_json_samples/processors           |  0
 .../redfish_json_samples/storage_sample       |  0
 .../node_proxy/redfish_json_samples/system    |  0
 .../node_proxy/redfish_system.py              |  6 +--
 .../{ => cephadmlib}/node_proxy/reporter.py   |  2 +-
 .../{ => cephadmlib}/node_proxy/server.py     | 42 +++++++++----------
 .../{ => cephadmlib}/node_proxy/util.py       |  0
 src/cephadm/node_proxy/requirements.txt       |  8 ----
 22 files changed, 35 insertions(+), 39 deletions(-)
 rename src/cephadm/{ => cephadmlib}/node_proxy/baseclient.py (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/basesystem.py (95%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/data.py (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/fake_cephadm/cephadm_mgr_module.py (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/main.py (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish-test.py (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_client.py (97%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_dell.py (95%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/interface_sample (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/interfaces_sample (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/memory (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/memory_socket (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/processor (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/processors (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/storage_sample (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_json_samples/system (100%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/redfish_system.py (97%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/reporter.py (98%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/server.py (89%)
 rename src/cephadm/{ => cephadmlib}/node_proxy/util.py (100%)
 delete mode 100644 src/cephadm/node_proxy/requirements.txt

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 636c21615474..ad21fc3a89c3 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -30,6 +30,7 @@
 from threading import Thread, Event
 from urllib.request import urlopen, Request
 from pathlib import Path
+import cephadmlib.node_proxy.server
 
 from cephadmlib.constants import (
     # default images
@@ -1453,6 +1454,9 @@ def pull_conf_settings(self) -> None:
     def run(self) -> None:
         self.pull_conf_settings()
 
+        t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main)
+        t_node_proxy.start()
+
         try:
             for _ in range(1001):
                 if not port_in_use(self.ctx, EndPoint('0.0.0.0', self.starting_port)):
diff --git a/src/cephadm/node_proxy/baseclient.py b/src/cephadm/cephadmlib/node_proxy/baseclient.py
similarity index 100%
rename from src/cephadm/node_proxy/baseclient.py
rename to src/cephadm/cephadmlib/node_proxy/baseclient.py
diff --git a/src/cephadm/node_proxy/basesystem.py b/src/cephadm/cephadmlib/node_proxy/basesystem.py
similarity index 95%
rename from src/cephadm/node_proxy/basesystem.py
rename to src/cephadm/cephadmlib/node_proxy/basesystem.py
index a56ad7e8f9d8..d3571e921f92 100644
--- a/src/cephadm/node_proxy/basesystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/basesystem.py
@@ -1,6 +1,6 @@
-from util import Config
+from .util import Config
 from typing import Dict, Any
-from baseclient import BaseClient
+from .baseclient import BaseClient
 
 
 class BaseSystem:
diff --git a/src/cephadm/node_proxy/data.py b/src/cephadm/cephadmlib/node_proxy/data.py
similarity index 100%
rename from src/cephadm/node_proxy/data.py
rename to src/cephadm/cephadmlib/node_proxy/data.py
diff --git a/src/cephadm/node_proxy/fake_cephadm/cephadm_mgr_module.py b/src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py
similarity index 100%
rename from src/cephadm/node_proxy/fake_cephadm/cephadm_mgr_module.py
rename to src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py
diff --git a/src/cephadm/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
similarity index 100%
rename from src/cephadm/node_proxy/main.py
rename to src/cephadm/cephadmlib/node_proxy/main.py
diff --git a/src/cephadm/node_proxy/redfish-test.py b/src/cephadm/cephadmlib/node_proxy/redfish-test.py
similarity index 100%
rename from src/cephadm/node_proxy/redfish-test.py
rename to src/cephadm/cephadmlib/node_proxy/redfish-test.py
diff --git a/src/cephadm/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
similarity index 97%
rename from src/cephadm/node_proxy/redfish_client.py
rename to src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 77353cd47819..26f71358593b 100644
--- a/src/cephadm/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -3,8 +3,8 @@
     InvalidCredentialsError
 import redfish
 import sys
-from util import Logger
-from baseclient import BaseClient
+from .util import Logger
+from .baseclient import BaseClient
 from typing import Dict
 
 log = Logger(__name__)
diff --git a/src/cephadm/node_proxy/redfish_dell.py b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
similarity index 95%
rename from src/cephadm/node_proxy/redfish_dell.py
rename to src/cephadm/cephadmlib/node_proxy/redfish_dell.py
index 0f4467bad5ff..f1cd6f6509be 100644
--- a/src/cephadm/node_proxy/redfish_dell.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
@@ -1,5 +1,5 @@
-from redfish_system import RedfishSystem
-from util import Logger, normalize_dict, to_snake_case
+from .redfish_system import RedfishSystem
+from .util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any
 
 log = Logger(__name__)
diff --git a/src/cephadm/node_proxy/redfish_json_samples/interface_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/interface_sample
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample
diff --git a/src/cephadm/node_proxy/redfish_json_samples/interfaces_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/interfaces_sample
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample
diff --git a/src/cephadm/node_proxy/redfish_json_samples/memory b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/memory
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory
diff --git a/src/cephadm/node_proxy/redfish_json_samples/memory_socket b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/memory_socket
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket
diff --git a/src/cephadm/node_proxy/redfish_json_samples/processor b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/processor
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor
diff --git a/src/cephadm/node_proxy/redfish_json_samples/processors b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/processors
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors
diff --git a/src/cephadm/node_proxy/redfish_json_samples/storage_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/storage_sample
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample
diff --git a/src/cephadm/node_proxy/redfish_json_samples/system b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system
similarity index 100%
rename from src/cephadm/node_proxy/redfish_json_samples/system
rename to src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system
diff --git a/src/cephadm/node_proxy/redfish_system.py b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
similarity index 97%
rename from src/cephadm/node_proxy/redfish_system.py
rename to src/cephadm/cephadmlib/node_proxy/redfish_system.py
index f23e41af79ec..bb8286b127e1 100644
--- a/src/cephadm/node_proxy/redfish_system.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
@@ -1,8 +1,8 @@
-from basesystem import BaseSystem
-from redfish_client import RedFishClient
+from .basesystem import BaseSystem
+from .redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from util import Logger, retry, normalize_dict, to_snake_case
+from .util import Logger, retry, normalize_dict, to_snake_case
 from typing import Dict, Any, List
 
 log = Logger(__name__)
diff --git a/src/cephadm/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
similarity index 98%
rename from src/cephadm/node_proxy/reporter.py
rename to src/cephadm/cephadmlib/node_proxy/reporter.py
index 07ac637bf1c8..43b57388e3b0 100644
--- a/src/cephadm/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -1,7 +1,7 @@
 from threading import Thread
 import requests
 import time
-from util import Logger
+from .util import Logger
 from typing import Any
 
 log = Logger(__name__)
diff --git a/src/cephadm/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
similarity index 89%
rename from src/cephadm/node_proxy/server.py
rename to src/cephadm/cephadmlib/node_proxy/server.py
index d05dbebd9c68..714dafc31f5b 100644
--- a/src/cephadm/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -1,9 +1,9 @@
 import cherrypy
-from redfish_dell import RedfishDell
-from reporter import Reporter
-from util import Config, Logger
+from .redfish_dell import RedfishDell
+from .reporter import Reporter
+from .util import Config, Logger
 from typing import Dict
-from basesystem import BaseSystem
+from .basesystem import BaseSystem
 import sys
 import argparse
 
@@ -30,10 +30,10 @@
     }
 }
 
-for env_var in DEVEL_ENV_VARS:
-    if os.environ.get(env_var) is None:
-        print(f"{env_var} environment variable must be set.")
-        sys.exit(1)
+#for env_var in DEVEL_ENV_VARS:
+#    if os.environ.get(env_var) is None:
+#        print(f"{env_var} environment variable must be set.")
+#        sys.exit(1)
 
 
 class Memory:
@@ -199,19 +199,19 @@ def GET(self) -> str:
 
 def main() -> None:
 
-    parser = argparse.ArgumentParser(
-        prog='node-proxy',
-    )
-    parser.add_argument(
-        '--config',
-        dest='config',
-        type=str,
-        required=False,
-        default='/etc/ceph/node-proxy.yml'
-    )
-
-    args = parser.parse_args()
-    config = Config(args.config, default_config=DEFAULT_CONFIG)
+    # parser = argparse.ArgumentParser(
+    #     prog='node-proxy',
+    # )
+    # parser.add_argument(
+    #     '--config',
+    #     dest='config',
+    #     type=str,
+    #     required=False,
+    #     default='/etc/ceph/node-proxy.yml'
+    # )
+
+    # args = parser.parse_args()
+    config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
 
     log = Logger(__name__, level=config.__dict__['logging']['level'])
     # must be passed as arguments
diff --git a/src/cephadm/node_proxy/util.py b/src/cephadm/cephadmlib/node_proxy/util.py
similarity index 100%
rename from src/cephadm/node_proxy/util.py
rename to src/cephadm/cephadmlib/node_proxy/util.py
diff --git a/src/cephadm/node_proxy/requirements.txt b/src/cephadm/node_proxy/requirements.txt
deleted file mode 100644
index ab20bcc64b46..000000000000
--- a/src/cephadm/node_proxy/requirements.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-pyyaml
-types-PyYAML
-types-requests
-mypy
-flake8
-redfish
-CherryPy
-requests

From 22247f0b1e39bc485fa66fbd7a802203eb5279a9 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 15:16:57 +0000
Subject: [PATCH 1546/2492] python-common: update HostSpec

This adds new parameters to the current spec 'HostSpec'.

The idea is to make it possible to pass idrac credentials so
it will be possible for the node-proxy agent to consume them in order
to communicate with the redfish API.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/python-common/ceph/deployment/hostspec.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/python-common/ceph/deployment/hostspec.py b/src/python-common/ceph/deployment/hostspec.py
index 0c448bf1313b..8fc9fdc508d8 100644
--- a/src/python-common/ceph/deployment/hostspec.py
+++ b/src/python-common/ceph/deployment/hostspec.py
@@ -16,6 +16,15 @@ def assert_valid_host(name: str) -> None:
         raise SpecValidationError(str(e) + f'. Got "{name}"')
 
 
+def assert_valid_idrac(idrac: Dict[str, str]) -> None:
+    fields = ['username', 'password']
+    try:
+        for field in fields:
+            assert field in idrac.keys()
+    except AssertionError as e:
+        raise SpecValidationError(str(e))
+
+
 class SpecValidationError(Exception):
     """
     Defining an exception here is a bit problematic, cause you cannot properly catch it,
@@ -38,6 +47,7 @@ def __init__(self,
                  labels: Optional[List[str]] = None,
                  status: Optional[str] = None,
                  location: Optional[Dict[str, str]] = None,
+                 idrac: Optional[Dict[str, str]] = None,
                  ):
         self.service_type = 'host'
 
@@ -55,8 +65,13 @@ def __init__(self,
 
         self.location = location
 
+        #: idrac details, if provided
+        self.idrac = idrac
+
     def validate(self) -> None:
         assert_valid_host(self.hostname)
+        if self.idrac:
+            assert_valid_idrac(self.idrac)
 
     def to_json(self) -> Dict[str, Any]:
         r: Dict[str, Any] = {
@@ -67,6 +82,8 @@ def to_json(self) -> Dict[str, Any]:
         }
         if self.location:
             r['location'] = self.location
+        if self.idrac:
+            r['idrac'] = self.idrac
         return r
 
     @classmethod
@@ -79,6 +96,7 @@ def from_json(cls, host_spec: dict) -> 'HostSpec':
                 host_spec['labels'])) if 'labels' in host_spec else None,
             host_spec['status'] if 'status' in host_spec else None,
             host_spec.get('location'),
+            host_spec['idrac'] if 'idrac' in host_spec else None,
         )
         return _cls
 

From 08e3d7ff5f70a1c6faafb64d982be6c684cfef06 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 15:27:45 +0000
Subject: [PATCH 1547/2492] mgr/cephadm: store oob mgmt credentials in mon kv
 store

The idea is to store the oob mgmt credentials into the monitor kv store
when they are passed via a host spec.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index b8f9d648ed4f..8c1e3a482847 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1613,6 +1613,19 @@ def _add_host(self, spec):
         if spec.hostname in self.inventory and self.inventory.get_addr(spec.hostname) != spec.addr:
             self.cache.refresh_all_host_info(spec.hostname)
 
+        if spec.idrac:
+            if not spec.idrac.get('addr'):
+                spec.idrac['addr'] = spec.hostname
+            if not spec.idrac.get('port'):
+                spec.idrac['port'] = '443'
+            data = json.loads(self.get_store('node_proxy/idrac', '{}'))
+            data[spec.hostname] = dict()
+            data[spec.hostname]['addr'] = spec.idrac['addr']
+            data[spec.hostname]['port'] = spec.idrac['port']
+            data[spec.hostname]['username'] = spec.idrac['username']
+            data[spec.hostname]['password'] = spec.idrac['password']
+            self.set_store('node_proxy/idrac', json.dumps(data))
+
         # prime crush map?
         if spec.location:
             self.check_mon_command({

From 7544406be33a579b3d0c63ee4c78ae91b02dfb0e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 15:32:38 +0000
Subject: [PATCH 1548/2492] cephadm/binary: add `query_endpoint()` method

This encapsulates the existing code in a new method
`query_endpoint()`.
The idea is to avoid duplicating code if we need to make multiple
calls to the agent endpoint from the `run()` method.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ad21fc3a89c3..efdd85c9fdff 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1451,6 +1451,28 @@ def pull_conf_settings(self) -> None:
             self.device_enhanced_scan = True
         self.volume_gatherer.update_func(lambda: self._ceph_volume(enhanced=self.device_enhanced_scan))
 
+    def query_endpoint(self,
+                       addr: str = '',
+                       port: str = '',
+                       data: Optional[Union[Dict[str, str], str]] = None,
+                       endpoint: str = '',
+                       ssl_ctx: Optional[Any] = None) -> str:
+        _addr = addr if addr else self.target_ip
+        _port = port if port else self.target_port
+        url = f'https://{_addr}:{_port}{endpoint}'
+
+        try:
+            req = Request(url, data, {'Content-Type': 'application/json'})
+            send_time = time.monotonic()
+            with urlopen(req, context=ssl_ctx) as response:
+                response_str = response.read()
+                response_json = json.loads(response_str)
+                total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
+                logger.info(f'Received mgr response: "{response_json["result"]}" {total_request_time} seconds after sending request.')
+        except Exception:
+            raise
+        return response_str
+
     def run(self) -> None:
         self.pull_conf_settings()
 
@@ -1507,15 +1529,10 @@ def run(self) -> None:
                                'port': self.listener_port})
             data = data.encode('ascii')
 
-            url = f'https://{self.target_ip}:{self.target_port}/data/'
             try:
-                req = Request(url, data, {'Content-Type': 'application/json'})
-                send_time = time.monotonic()
-                with urlopen(req, context=ssl_ctx) as response:
-                    response_str = response.read()
-                    response_json = json.loads(response_str)
-                    total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
-                    logger.info(f'Received mgr response: "{response_json["result"]}" {total_request_time} seconds after sending request.')
+                self.query_endpoint(data=data,
+                                    endpoint='/data/',
+                                    ssl_ctx=ssl_ctx)
             except Exception as e:
                 logger.error(f'Failed to send metadata to mgr: {e}')
 

From c1324cd821ef005474eddd5d009e499de1a51ee3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 15:41:32 +0000
Subject: [PATCH 1549/2492] mgr/cephadm: add node-proxy endpoints to the mgr

This adds 2 endpoints to the existing http agent endpoint:

- '/node_proxy/idrac': support POST requests only although this endpoint
  is intended for fetching the idrac credentials of a given node. As we pass
  sensitive details (ceph secret) I didn't want to pass it as a query parameter
  in the url. Passing it in a HTTP header is perhaps a better approach but we already
  do similar thing for endpoint '/data' (agent) so for consistency reason I stick to
  that.

- '/node_proxy/data': support GET and POST requests. A GET will return the
  aggregated data for all nodes within the cluster. node-proxy will use a POST
  request to that endpoint to push its collected data.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py          |  9 +++---
 src/pybind/mgr/cephadm/agent.py | 56 +++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index efdd85c9fdff..c0625a0f39bb 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1477,6 +1477,10 @@ def run(self) -> None:
         self.pull_conf_settings()
 
         t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main)
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = True
+        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx.load_verify_locations(self.ca_path)
         t_node_proxy.start()
 
         try:
@@ -1499,11 +1503,6 @@ def run(self) -> None:
         if not self.volume_gatherer.is_alive():
             self.volume_gatherer.start()
 
-        ssl_ctx = ssl.create_default_context()
-        ssl_ctx.check_hostname = True
-        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        ssl_ctx.load_verify_locations(self.ca_path)
-
         while not self.stop:
             start_time = time.monotonic()
             ack = self.ack
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 93a08cb3439b..40853f7806ef 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -57,6 +57,12 @@ def configure_routes(self) -> None:
         d.connect(name='host-data', route='/data/',
                   controller=self.host_data.POST,
                   conditions=dict(method=['POST']))
+        d.connect(name='node-proxy-idrac', route='/node-proxy/idrac',
+                  controller=self.host_data.node_proxy_idrac,
+                  conditions=dict(method=['POST']))
+        d.connect(name='node-proxy-data', route='/node-proxy/data',
+                  controller=self.host_data.node_proxy_data,
+                  conditions=dict(method=['GET','POST']))
         cherrypy.tree.mount(None, '/', config={'/': {'request.dispatch': d}})
 
     def configure_tls(self, server: Server) -> None:
@@ -126,6 +132,56 @@ def POST(self) -> Dict[str, Any]:
             results['result'] = self.handle_metadata(data)
         return results
 
+    def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
+        if 'host' not in data:
+            cherrypy.response.status = 400
+            self.mgr.log.warning('The field \'host\' must be provided.')
+        elif 'keyring' not in data:
+            cherrypy.response.status = 400
+            self.mgr.log.warning(f'The agent keyring must be provided.')
+        elif not self.mgr.agent_cache.agent_keys.get(data['host']):
+            cherrypy.response.status = 400
+            self.mgr.log.warning(f'Make sure the agent is running on {data["host"]}')
+        elif data['keyring'] != self.mgr.agent_cache.agent_keys[data['host']]:
+            cherrypy.response.status = 403
+            self.mgr.log.warning(f'Got wrong keyring from agent on host {data["host"]}.')
+        else:
+            cherrypy.response.status = 200
+
+        return cherrypy.response.status == 200
+
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def node_proxy_idrac(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = cherrypy.request.json
+        results: Dict[str, Any] = {}
+
+        if self.validate_node_proxy_data(data):
+            idrac_details = self.mgr.get_store('node_proxy/idrac')
+            idrac_details_json = json.loads(idrac_details)
+            self.mgr.log.warning(f"{idrac_details_json}")
+            results['result'] = idrac_details_json[data["host"]]
+
+        return results
+
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def node_proxy_data(self) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+
+        if cherrypy.request.method == 'POST':
+            data: Dict[str, Any] = cherrypy.request.json
+            if self.validate_node_proxy_data(data):
+                self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
+                self.mgr.log.warning(f"{data}")
+                results['result'] = data
+
+        if cherrypy.request.method == 'GET':
+            for k, v in self.mgr.get_store_prefix('node_proxy/data').items():
+                host = k.split('/')[-1:][0]
+                results[host] = json.loads(v)
+        return results
+
     def check_request_fields(self, data: Dict[str, Any]) -> None:
         fields = '{' + ', '.join([key for key in data.keys()]) + '}'
         if 'host' not in data:

From c5e705abaa9df28862f88ba319e8dd9c6d710fac Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 15:53:34 +0000
Subject: [PATCH 1550/2492] node-proxy: fetch idrac details from ceph

The idrac details are now fetched from ceph (monitor kv store) and
passed by the cephadm binary at the agent startup.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                      | 13 +++++++++--
 src/cephadm/cephadmlib/node_proxy/server.py | 24 +++++++--------------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index c0625a0f39bb..ee2bca70ddd6 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1475,12 +1475,21 @@ def query_endpoint(self,
 
     def run(self) -> None:
         self.pull_conf_settings()
-
-        t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main)
         ssl_ctx = ssl.create_default_context()
         ssl_ctx.check_hostname = True
         ssl_ctx.verify_mode = ssl.CERT_REQUIRED
         ssl_ctx.load_verify_locations(self.ca_path)
+        node_proxy_data = json.dumps({'keyring': self.keyring,
+                                      'host': self.host})
+        node_proxy_data = node_proxy_data.encode('ascii')
+        result = self.query_endpoint(data=node_proxy_data,
+                                     endpoint='/node-proxy/idrac',
+                                     ssl_ctx=ssl_ctx)
+        result_json = json.loads(result)
+        t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main,
+                              kwargs={'host': result_json['result']['addr'],
+                                      'username': result_json['result']['username'],
+                                      'password': result_json['result']['password'])
         t_node_proxy.start()
 
         try:
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 714dafc31f5b..b3c2306d88fc 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -7,12 +7,6 @@
 import sys
 import argparse
 
-# for devel purposes
-import os
-DEVEL_ENV_VARS = ['REDFISH_HOST',
-                  'REDFISH_USERNAME',
-                  'REDFISH_PASSWORD']
-
 DEFAULT_CONFIG = {
     'reporter': {
         'check_interval': 5,
@@ -30,11 +24,6 @@
     }
 }
 
-#for env_var in DEVEL_ENV_VARS:
-#    if os.environ.get(env_var) is None:
-#        print(f"{env_var} environment variable must be set.")
-#        sys.exit(1)
-
 
 class Memory:
     exposed = True
@@ -197,7 +186,10 @@ def GET(self) -> str:
         return 'use /system or /admin endpoints'
 
 
-def main() -> None:
+def main(host: str = '',
+         username: str = '',
+         password: str = '') -> None:
+    # TODO: add a check and fail if host/username/password/data aren't passed
 
     # parser = argparse.ArgumentParser(
     #     prog='node-proxy',
@@ -214,10 +206,10 @@ def main() -> None:
     config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
 
     log = Logger(__name__, level=config.__dict__['logging']['level'])
-    # must be passed as arguments
-    host = os.environ.get('REDFISH_HOST')
-    username = os.environ.get('REDFISH_USERNAME')
-    password = os.environ.get('REDFISH_PASSWORD')
+
+    host = host
+    username = username
+    password = password
 
     # create the redfish system and the obsever
     log.logger.info("Server initialization...")

From 9a305c5c8e94e12b6c103b3b3f4201f4fc3616c9 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 16:08:26 +0000
Subject: [PATCH 1551/2492] node-proxy: update reporter agent

This commit introduces the required changes in order to make
the reporter agent query the new mgr endpoint '/node-proxy/data'

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        |  3 ++-
 src/cephadm/cephadmlib/node_proxy/reporter.py | 13 ++++++++-----
 src/cephadm/cephadmlib/node_proxy/server.py   |  7 +++++--
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ee2bca70ddd6..2fdaaa2854ad 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1489,7 +1489,8 @@ def run(self) -> None:
         t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main,
                               kwargs={'host': result_json['result']['addr'],
                                       'username': result_json['result']['username'],
-                                      'password': result_json['result']['password'])
+                                      'password': result_json['result']['password'],
+                                      'data': node_proxy_data})
         t_node_proxy.start()
 
         try:
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 43b57388e3b0..380884dc5b40 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -2,16 +2,17 @@
 import requests
 import time
 from .util import Logger
-from typing import Any
+from typing import Dict, Any
 
 log = Logger(__name__)
 
 
 class Reporter:
-    def __init__(self, system: Any, observer_url: str) -> None:
+    def __init__(self, system: Any, data: Dict[str, Any], observer_url: str) -> None:
         self.system = system
         self.observer_url = observer_url
         self.finish = False
+        self.data = data
 
     def stop(self) -> None:
         self.finish = True
@@ -35,11 +36,13 @@ def loop(self) -> None:
                 log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
                     log.logger.info('data has changed since last iteration.')
-                    d = self.system.get_system()
+                    self.data['data'] = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
-                        requests.post(f"{self.observer_url}/", json=d, timeout=5)
-                    except requests.exceptions.RequestException as e:
+                        log.logger.info(f"sending data to {self.observer_url}")
+                        r = requests.post(f"{self.observer_url}", json=self.data, timeout=5, verify=False)
+                    except (requests.exceptions.RequestException,
+                            requests.exceptions.ConnectionError) as e:
                         log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't
                         # send the data for more than x times, maybe the daemon should stop altogether
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index b3c2306d88fc..18cfedb05012 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -6,6 +6,7 @@
 from .basesystem import BaseSystem
 import sys
 import argparse
+import json
 
 DEFAULT_CONFIG = {
     'reporter': {
@@ -188,7 +189,8 @@ def GET(self) -> str:
 
 def main(host: str = '',
          username: str = '',
-         password: str = '') -> None:
+         password: str = '',
+         data: str = '') -> None:
     # TODO: add a check and fail if host/username/password/data aren't passed
 
     # parser = argparse.ArgumentParser(
@@ -210,6 +212,7 @@ def main(host: str = '',
     host = host
     username = username
     password = password
+    data = json.loads(data)
 
     # create the redfish system and the obsever
     log.logger.info("Server initialization...")
@@ -218,7 +221,7 @@ def main(host: str = '',
                          password=password,
                          system_endpoint='/Systems/System.Embedded.1',
                          config=config)
-    reporter_agent = Reporter(system, config.__dict__['reporter']['endpoint'])
+    reporter_agent = Reporter(system, data, config.__dict__['reporter']['endpoint'])
     cherrypy.config.update({
         'node_proxy': config,
         'server.socket_port': config.__dict__['server']['port']

From 74df8b711f81138b38a960ac9cf39291f7d7d906 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 14 Sep 2023 16:10:01 +0000
Subject: [PATCH 1552/2492] node-proxy: modify the endpoint url from default
 config

This updates the endpoint url from DEFAULT_CONFIG in order
to match the new endpoint recently added.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 18cfedb05012..54396f445dda 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -12,7 +12,7 @@
     'reporter': {
         'check_interval': 5,
         'push_data_max_retries': 30,
-        'endpoint': 'http://127.0.0.1:8150',
+        'endpoint': 'https://127.0.0.1:7150/node-proxy/data',
     },
     'system': {
         'refresh_interval': 5

From 043e827c2d7c4ac78808efff5627d75a3ed5a3bb Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 18 Sep 2023 06:50:24 +0000
Subject: [PATCH 1553/2492] node-proxy: parametrize reporter url

node-proxy entrypoint (`server.main()`) now takes two parameters
(addr / port) in order to make the reporter agent know how to reach
the http agent endpoint hosted in the mgr daemon.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                      | 4 +++-
 src/cephadm/cephadmlib/node_proxy/server.py | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 2fdaaa2854ad..450c91c24ceb 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1490,7 +1490,9 @@ def run(self) -> None:
                               kwargs={'host': result_json['result']['addr'],
                                       'username': result_json['result']['username'],
                                       'password': result_json['result']['password'],
-                                      'data': node_proxy_data})
+                                      'data': node_proxy_data,
+                                      'mgr_target_ip': self.target_ip,
+                                      'mgr_target_port': self.target_port})
         t_node_proxy.start()
 
         try:
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 54396f445dda..78d1e7065a96 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -190,7 +190,9 @@ def GET(self) -> str:
 def main(host: str = '',
          username: str = '',
          password: str = '',
-         data: str = '') -> None:
+         data: str = '',
+         mgr_target_ip: str = '',
+         mgr_target_port: str = '') -> None:
     # TODO: add a check and fail if host/username/password/data aren't passed
 
     # parser = argparse.ArgumentParser(
@@ -221,7 +223,7 @@ def main(host: str = '',
                          password=password,
                          system_endpoint='/Systems/System.Embedded.1',
                          config=config)
-    reporter_agent = Reporter(system, data, config.__dict__['reporter']['endpoint'])
+    reporter_agent = Reporter(system, data, f"https://{mgr_target_ip}:{mgr_target_port}/node-proxy/data")
     cherrypy.config.update({
         'node_proxy': config,
         'server.socket_port': config.__dict__['server']['port']

From f4d3c59feb6fe5969bbd578850079226d1af6ad2 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Sep 2023 07:41:57 +0000
Subject: [PATCH 1554/2492] node-proxy: add __init__.py file

In order to make node-proxy a package.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/node_proxy/__init__.py

diff --git a/src/cephadm/cephadmlib/node_proxy/__init__.py b/src/cephadm/cephadmlib/node_proxy/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1

From 88ad166b21815c775c50d902c573a65206e40f3e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Sep 2023 07:46:42 +0000
Subject: [PATCH 1555/2492] node-proxy: logging refactor

This makes `logger` a class attribute so we don't have
the `Logger` instantiation outside of the different classes.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/redfish_dell.py     | 15 +++++------
 .../cephadmlib/node_proxy/redfish_system.py   | 27 +++++++++----------
 src/cephadm/cephadmlib/node_proxy/reporter.py | 20 +++++++-------
 3 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
index f1cd6f6509be..796c989f8834 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
@@ -2,18 +2,17 @@
 from .util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any
 
-log = Logger(__name__)
-
 
 class RedfishDell(RedfishSystem):
     def __init__(self, **kw: Any) -> None:
+        self.log = Logger(__name__)
         if kw.get('system_endpoint') is None:
             kw['system_endpoint'] = '/Systems/System.Embedded.1'
         super().__init__(**kw)
 
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
-        log.logger.info("Updating network")
+        self.log.logger.info("Updating network")
         self._system['network'] = self.build_data(fields, 'EthernetInterfaces')
 
     def _update_processors(self) -> None:
@@ -24,7 +23,7 @@ def _update_processors(self) -> None:
                   'Model',
                   'Status',
                   'Manufacturer']
-        log.logger.info("Updating processors")
+        self.log.logger.info("Updating processors")
         self._system['processors'] = self.build_data(fields, 'Processors')
 
     def _update_storage(self) -> None:
@@ -34,7 +33,7 @@ def _update_storage(self) -> None:
                   'SerialNumber', 'Status',
                   'PhysicalLocation']
         entities = self.get_members('Storage')
-        log.logger.info("Updating storage")
+        self.log.logger.info("Updating storage")
         result: Dict[str, Dict[str, Dict]] = dict()
         for entity in entities:
             for drive in entity['Drives']:
@@ -48,7 +47,7 @@ def _update_storage(self) -> None:
         self._system['storage'] = normalize_dict(result)
 
     def _update_metadata(self) -> None:
-        log.logger.info("Updating metadata")
+        self.log.logger.info("Updating metadata")
         pass
 
     def _update_memory(self) -> None:
@@ -56,9 +55,9 @@ def _update_memory(self) -> None:
                   'MemoryDeviceType',
                   'CapacityMiB',
                   'Status']
-        log.logger.info("Updating memory")
+        self.log.logger.info("Updating memory")
         self._system['memory'] = self.build_data(fields, 'Memory')
 
     def _update_power(self) -> None:
-        log.logger.info("Updating power")
+        self.log.logger.info("Updating power")
         pass
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_system.py b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
index bb8286b127e1..ad556ab8dea9 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_system.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
@@ -5,18 +5,17 @@
 from .util import Logger, retry, normalize_dict, to_snake_case
 from typing import Dict, Any, List
 
-log = Logger(__name__)
-
 
 class RedfishSystem(BaseSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
+        self.log = Logger(__name__)
         self.host: str = kw['host']
         self.username: str = kw['username']
         self.password: str = kw['password']
         self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
-        log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
         self.client = RedFishClient(self.host, self.username, self.password)
+        self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
 
         self._system: Dict[str, Dict[str, Any]] = {}
         self.run: bool = False
@@ -30,7 +29,7 @@ def __init__(self, **kw: Any) -> None:
     def _get_path(self, path: str) -> Dict:
         result = self.client.get_path(path)
         if result is None:
-            log.logger.error(f"The client reported an error when getting path: {path}")
+            self.log.logger.error(f"The client reported an error when getting path: {path}")
             raise RuntimeError(f"Could not get path: {path}")
         return result
 
@@ -50,7 +49,7 @@ def build_data(self,
                 try:
                     result[member_id][to_snake_case(field)] = member_info[field]
                 except KeyError:
-                    log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
+                    self.log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
 
         return normalize_dict(result)
 
@@ -125,9 +124,9 @@ def update(self) -> None:
         #  - caching logic
         try:
             while self.run:
-                log.logger.debug("waiting for a lock.")
+                self.log.logger.debug("waiting for a lock.")
                 self.lock.acquire()
-                log.logger.debug("lock acquired.")
+                self.log.logger.debug("lock acquired.")
                 try:
                     self._update_system()
                     # following calls in theory can be done in parallel
@@ -141,21 +140,21 @@ def update(self) -> None:
                     sleep(5)
                 finally:
                     self.lock.release()
-                    log.logger.debug("lock released.")
+                    self.log.logger.debug("lock released.")
         # Catching 'Exception' is probably not a good idea (devel only)
         except Exception as e:
-            log.logger.error(f"Error detected, logging out from redfish api.\n{e}")
+            self.log.logger.error(f"Error detected, logging out from redfish api.\n{e}")
             self.client.logout()
             raise
 
     def flush(self) -> None:
-        log.logger.info("Acquiring lock to flush data.")
+        self.log.logger.info("Acquiring lock to flush data.")
         self.lock.acquire()
-        log.logger.info("Lock acquired, flushing data.")
+        self.log.logger.info("Lock acquired, flushing data.")
         self._system = {}
         self.previous_data = {}
-        log.logger.info("Data flushed.")
+        self.log.logger.info("Data flushed.")
         self.data_ready = False
-        log.logger.info("Data marked as not ready.")
+        self.log.logger.info("Data marked as not ready.")
         self.lock.release()
-        log.logger.info("Lock released.")
+        self.log.logger.info("Lock released.")
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 380884dc5b40..79de9509ac86 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -4,8 +4,6 @@
 from .util import Logger
 from typing import Dict, Any
 
-log = Logger(__name__)
-
 
 class Reporter:
     def __init__(self, system: Any, data: Dict[str, Any], observer_url: str) -> None:
@@ -13,6 +11,8 @@ def __init__(self, system: Any, data: Dict[str, Any], observer_url: str) -> None
         self.observer_url = observer_url
         self.finish = False
         self.data = data
+        self.log = Logger(__name__)
+        self.log.logger.info(f'Observer url set to {self.observer_url}')
 
     def stop(self) -> None:
         self.finish = True
@@ -29,27 +29,27 @@ def loop(self) -> None:
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
-            log.logger.debug("waiting for a lock.")
+            self.log.logger.debug("waiting for a lock.")
             self.system.lock.acquire()
-            log.logger.debug("lock acquired.")
+            self.log.logger.debug("lock acquired.")
             if self.system.data_ready:
-                log.logger.info('data ready to be sent to the mgr.')
+                self.log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
-                    log.logger.info('data has changed since last iteration.')
+                    self.log.logger.info('data has changed since last iteration.')
                     self.data['data'] = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
-                        log.logger.info(f"sending data to {self.observer_url}")
+                        self.log.logger.info(f"sending data to {self.observer_url}")
                         r = requests.post(f"{self.observer_url}", json=self.data, timeout=5, verify=False)
                     except (requests.exceptions.RequestException,
                             requests.exceptions.ConnectionError) as e:
-                        log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
+                        self.log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't
                         # send the data for more than x times, maybe the daemon should stop altogether
                     else:
                         self.system.previous_data = self.system.get_system()
                 else:
-                    log.logger.info('no diff, not sending data to the mgr.')
+                    self.log.logger.info('no diff, not sending data to the mgr.')
             self.system.lock.release()
-            log.logger.debug("lock released.")
+            self.log.logger.debug("lock released.")
             time.sleep(5)

From 9da37815ad48f00088ae028041b4621e91725985 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Sep 2023 07:55:54 +0000
Subject: [PATCH 1556/2492] node-proxy: drop redfish library dependency

Given that this library isn't packaged for both
upstream and downstream and we can achieve what it was used for
directly with a lib such `urllib` (basically just auth), let's
drop this dependency.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/baseclient.py       |   6 +-
 .../cephadmlib/node_proxy/redfish_client.py   | 121 +++++++++++-------
 .../cephadmlib/node_proxy/redfish_system.py   |   4 +-
 3 files changed, 83 insertions(+), 48 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseclient.py b/src/cephadm/cephadmlib/node_proxy/baseclient.py
index 735dd11e96d9..1415bc3fe194 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseclient.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseclient.py
@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Any
 
 
 class BaseClient:
@@ -10,10 +10,10 @@ def __init__(self,
         self.username = username
         self.password = password
 
-    def login(self) -> None:
+    def login(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
-    def logout(self) -> None:
+    def logout(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
     def get_path(self, path: str) -> Dict:
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 26f71358593b..437e4be131a4 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -1,54 +1,89 @@
-from redfish.rest.v1 import ServerDownOrUnreachableError, \
-    SessionCreationError, \
-    InvalidCredentialsError
-import redfish
-import sys
-from .util import Logger
+import time
+import datetime
+import ssl
+import json
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen, Request
 from .baseclient import BaseClient
-from typing import Dict
-
-log = Logger(__name__)
+from .util import Logger
+from typing import Dict, Any, Tuple, Optional
 
 
 class RedFishClient(BaseClient):
-
-    PREFIX = '/redfish/v1'
-
+    PREFIX = '/redfish/v1/'
     def __init__(self,
-                 host: str,
-                 username: str,
-                 password: str) -> None:
-        log.logger.info("redfish client initialization...")
+                 host: str = "",
+                 port: str = "443",
+                 username: str = "",
+                 password: str = ""):
         super().__init__(host, username, password)
-        self.redfish_obj: 'redfish.redfish_client' = None
+        self.log: Logger = Logger(__name__)
+        self.log.logger.info(f"Initializing redfish client {__name__}")
+        self.host: str = f"https://{host}:{port}"
+        self.token: Dict[str, str] = {}
+        self.location: str = ''
+
+    def login(self) -> Dict[str, Any]:
+        self.log.logger.info(f"Logging in to {self.host} as '{self.username}'")
+        idrac_credentials = json.dumps({"UserName": self.username, "Password": self.password})
+        headers = {"Content-Type": "application/json"}
 
-    def login(self) -> 'redfish.redfish_client':
-        self.redfish_obj = redfish.redfish_client(base_url=self.host,
-                                                  username=self.username,
-                                                  password=self.password,
-                                                  default_prefix=self.PREFIX)
         try:
-            # TODO: add a retry? check for a timeout setting
-            self.redfish_obj.login(auth="session")
-            log.logger.info(f"Logging to redfish api at {self.host} with user: {self.username}")
-            return self.redfish_obj
-        except InvalidCredentialsError as e:
-            log.logger.error(f"Invalid credentials for {self.username} at {self.host}:\n{e}")
-        except (SessionCreationError, ServerDownOrUnreachableError) as e:
-            log.logger.error(f"Server not reachable or does not support RedFish:\n{e}")
-        sys.exit(1)
-
-    def get_path(self, path: str) -> Dict:
+            _headers, _data = self.query(data=idrac_credentials,
+                                         headers=headers,
+                                         endpoint='/redfish/v1/SessionService/Sessions/')
+        except URLError as e:
+            self.log.logger.error(f"Can't log in to {self.host} as '{self.username}'.\n{e}")
+            return {}
+        self.token = {"X-Auth-Token": _headers['X-Auth-Token']}
+        self.location = _headers['Location']
+
+        return json.loads(_data)
+
+    def logout(self) -> Dict[str, Any]:
         try:
-            if self.PREFIX not in path:
-                path = f"{self.PREFIX}{path}"
-            log.logger.debug(f"getting: {path}")
-            response = self.redfish_obj.get(path)
-            return response.dict
-        except Exception as e:
-            log.logger.error(f"Error getting path:\n{e}")
+            _, _data = self.query(method='DELETE', headers=self.token, endpoint=self.location)
+        except URLError as e:
+            self.log.logger.error(f"Can't log out from {self.host}")
             return {}
 
-    def logout(self) -> None:
-        log.logger.info('logging out...')
-        self.redfish_obj.logout()
+        response_str = _data
+
+        return json.loads(response_str)
+
+    def get_path(self, path: str) -> Dict[str, Any]:
+        if self.PREFIX not in path:
+            path = f"{self.PREFIX}{path}"
+        try:
+            _, result = self.query(headers=self.token, endpoint=path)
+            result_json = json.loads(result)
+            return result_json
+        except URLError as e:
+            self.log.logger.error(f"Can't get path {path}:\n{e}")
+            return {}
+
+    def query(self,
+              data: Optional[str] = None,
+              headers: Dict[str, str] = {},
+              method: Optional[str] = None,
+              endpoint: str = '') -> Tuple[Dict[str, str], str]:
+        url = f'{self.host}{endpoint}'
+
+        # ssl_ctx = ssl.create_default_context()
+        # ssl_ctx.check_hostname = True
+        # ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx = ssl._create_unverified_context()
+        _data = bytes(data, 'ascii') if data else None
+        try:
+            req = Request(url, _data, headers=headers, method=method)
+            send_time = time.monotonic()
+            with urlopen(req, context=ssl_ctx) as response:
+                response_str = response.read()
+                response_headers = response.headers
+                response_json = json.loads(response_str)
+                total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
+        except Exception as e:
+            self.log.logger.error(f"{e}")
+            raise
+
+        return response_headers, response_str
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_system.py b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
index ad556ab8dea9..95c82960eae1 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_system.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_system.py
@@ -14,7 +14,7 @@ def __init__(self, **kw: Any) -> None:
         self.username: str = kw['username']
         self.password: str = kw['password']
         self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
-        self.client = RedFishClient(self.host, self.username, self.password)
+        self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
         self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
 
         self._system: Dict[str, Dict[str, Any]] = {}
@@ -55,7 +55,7 @@ def build_data(self,
 
     def start_client(self) -> None:
         if not self.client:
-            self.client = RedFishClient(self.host, self.username, self.password)
+            self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
         self.client.login()
 
     def get_system(self) -> Dict[str, Dict[str, Dict]]:

From b45ba22920afbd1471ad3163157f7dc612e6a1f1 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Sep 2023 11:49:44 +0000
Subject: [PATCH 1557/2492] node-proxy: raise ceph warning(s) if needed

This makes the agent endpoint raise alert(s) when one or multiple
members of a component are critical.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 46 +++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 40853f7806ef..144780b65b4f 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -174,6 +174,8 @@ def node_proxy_data(self) -> Dict[str, Any]:
             if self.validate_node_proxy_data(data):
                 self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
                 self.mgr.log.warning(f"{data}")
+                self.raise_alert(data)
+
                 results['result'] = data
 
         if cherrypy.request.method == 'GET':
@@ -182,6 +184,50 @@ def node_proxy_data(self) -> Dict[str, Any]:
                 results[host] = json.loads(v)
         return results
 
+    def get_nok_members(self,
+                        component: str,
+                        data: Dict[str, Any]) -> List[Dict[str, str]]:
+        nok_members: List[Dict[str, str]] = []
+
+        for member in data[component].keys():
+            # Force a fake error for testing purpose
+            if component == 'storage':
+                _status = 'critical'
+                state = "Fake error"
+            else:
+                _status = data[component][member]['status']['health'].lower()
+            if _status.lower() != 'ok':
+                # state = data[component][member]['status']['state']
+                _member = dict(
+                    member=member,
+                    status=_status,
+                    state=state
+                )
+                nok_members.append(_member)
+
+        return nok_members
+
+    def raise_alert(self, data: Dict[str, Any]) -> None:
+        mapping: Dict[str, str] = {
+            'storage': 'NODE_PROXY_STORAGE',
+            'memory': 'NODE_PROXY_MEMORY',
+            'processors': 'NODE_PROXY_PROCESSORS',
+            'network': 'NODE_PROXY_NETWORK',
+        }
+
+        for component in data['data'].keys():
+            nok_members = self.get_nok_members(component,
+                                                data['data'])
+
+            if nok_members:
+                count = len(nok_members)
+                self.mgr.set_health_warning(
+                    mapping[component],
+                    summary=f'{count} {component} member{"s" if count > 1 else ""} {"are" if count > 1 else "is"} not ok',
+                    count=count,
+                    detail=[f"{member['member']} is {member['status']}: {member['state']}" for member in nok_members],
+                )
+
     def check_request_fields(self, data: Dict[str, Any]) -> None:
         fields = '{' + ', '.join([key for key in data.keys()]) + '}'
         if 'host' not in data:

From 27b7f98e5c0816d07327bae22d39453608860390 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 21 Sep 2023 14:52:01 +0000
Subject: [PATCH 1558/2492] cephadm/agent: endpoint refactor

These changes are required in order to be able to re-use
the existing agent endpoint. The current code doesn't ease/allow
adding a new application. The idea here is to add a new class for
handling the '/node-proxy' endpoint.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py          |   2 +-
 src/pybind/mgr/cephadm/agent.py | 241 ++++++++++++++++++++++----------
 2 files changed, 172 insertions(+), 71 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 450c91c24ceb..98196c355ece 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1542,7 +1542,7 @@ def run(self) -> None:
 
             try:
                 self.query_endpoint(data=data,
-                                    endpoint='/data/',
+                                    endpoint='/data',
                                     ssl_ctx=ssl_ctx)
             except Exception as e:
                 logger.error(f'Failed to send metadata to mgr: {e}')
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 144780b65b4f..d3376d235889 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -53,17 +53,10 @@ def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.server_addr = self.mgr.get_mgr_ip()
 
     def configure_routes(self) -> None:
-        d = cherrypy.dispatch.RoutesDispatcher()
-        d.connect(name='host-data', route='/data/',
-                  controller=self.host_data.POST,
-                  conditions=dict(method=['POST']))
-        d.connect(name='node-proxy-idrac', route='/node-proxy/idrac',
-                  controller=self.host_data.node_proxy_idrac,
-                  conditions=dict(method=['POST']))
-        d.connect(name='node-proxy-data', route='/node-proxy/data',
-                  controller=self.host_data.node_proxy_data,
-                  conditions=dict(method=['GET','POST']))
-        cherrypy.tree.mount(None, '/', config={'/': {'request.dispatch': d}})
+        conf = {'/': {'tools.trailing_slash.on': False}}
+
+        cherrypy.tree.mount(self.host_data, '/data', config=conf)
+        cherrypy.tree.mount(self.node_proxy, '/node-proxy', config=conf)
 
     def configure_tls(self, server: Server) -> None:
         old_cert = self.mgr.get_store(self.KV_STORE_AGENT_ROOT_CERT)
@@ -93,43 +86,38 @@ def find_free_port(self) -> None:
 
     def configure(self) -> None:
         self.host_data = HostData(self.mgr, self.server_port, self.server_addr)
+        self.node_proxy = NodeProxy(self.mgr)
         self.configure_tls(self.host_data)
         self.configure_routes()
         self.find_free_port()
 
 
-class HostData(Server):
-    exposed = True
-
-    def __init__(self, mgr: "CephadmOrchestrator", port: int, host: str):
+class NodeProxy:
+    def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
-        super().__init__()
-        self.socket_port = port
-        self.socket_host = host
-        self.subscribe()
 
-    def stop(self) -> None:
-        # we must call unsubscribe before stopping the server,
-        # otherwise the port is not released and we will get
-        # an exception when trying to restart it
-        self.unsubscribe()
-        super().stop()
+    def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
+        if len(vpath) == 2:
+            hostname = vpath.pop(0)
+            cherrypy.request.params['hostname'] = hostname
+        cmd = vpath.pop(0)
+        cherrypy.request.params['cmd'] = cmd
+
+        return self
 
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['POST'])
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    def POST(self) -> Dict[str, Any]:
+    def idrac(self) -> Dict[str, Any]:
         data: Dict[str, Any] = cherrypy.request.json
         results: Dict[str, Any] = {}
-        try:
-            self.check_request_fields(data)
-        except Exception as e:
-            results['result'] = f'Bad metadata: {e}'
-            self.mgr.log.warning(f'Received bad metadata from an agent: {e}')
-        else:
-            # if we got here, we've already verified the keyring of the agent. If
-            # host agent is reporting on is marked offline, it shouldn't be any more
-            self.mgr.offline_hosts_remove(data['host'])
-            results['result'] = self.handle_metadata(data)
+
+        if self.validate_node_proxy_data(data):
+            idrac_details = self.mgr.get_store('node_proxy/idrac')
+            idrac_details_json = json.loads(idrac_details)
+            results['result'] = idrac_details_json[data["host"]]
+
         return results
 
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
@@ -150,40 +138,6 @@ def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
 
         return cherrypy.response.status == 200
 
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    def node_proxy_idrac(self) -> Dict[str, Any]:
-        data: Dict[str, Any] = cherrypy.request.json
-        results: Dict[str, Any] = {}
-
-        if self.validate_node_proxy_data(data):
-            idrac_details = self.mgr.get_store('node_proxy/idrac')
-            idrac_details_json = json.loads(idrac_details)
-            self.mgr.log.warning(f"{idrac_details_json}")
-            results['result'] = idrac_details_json[data["host"]]
-
-        return results
-
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    def node_proxy_data(self) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-
-        if cherrypy.request.method == 'POST':
-            data: Dict[str, Any] = cherrypy.request.json
-            if self.validate_node_proxy_data(data):
-                self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
-                self.mgr.log.warning(f"{data}")
-                self.raise_alert(data)
-
-                results['result'] = data
-
-        if cherrypy.request.method == 'GET':
-            for k, v in self.mgr.get_store_prefix('node_proxy/data').items():
-                host = k.split('/')[-1:][0]
-                results[host] = json.loads(v)
-        return results
-
     def get_nok_members(self,
                         component: str,
                         data: Dict[str, Any]) -> List[Dict[str, str]]:
@@ -228,6 +182,153 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
                     detail=[f"{member['member']} is {member['status']}: {member['state']}" for member in nok_members],
                 )
 
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['POST'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def data(self) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+
+        data: Dict[str, Any] = cherrypy.request.json
+        if self.validate_node_proxy_data(data):
+            self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
+            self.raise_alert(data)
+
+            results['result'] = data
+
+        return results
+
+    def get_full_report(self) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+
+        for k, v in self.mgr.get_store_prefix('node_proxy/data').items():
+            host = k.split('/')[-1:][0]
+            results[host] = json.loads(v)
+        return results
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def fullreport(self, **kw: Any) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+        results = self.get_full_report()
+        hostname = kw.get('hostname',)
+
+        if hostname not in results.keys():
+            return results
+        else:
+            return results[hostname]
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def summary(self, **kw: Any) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+        status: List[str] = []
+        hostname = kw.get('hostname',)
+
+        results = self.get_full_report()
+
+        mapper: Dict[bool, str] = {
+            True: 'error',
+            False: 'ok'
+        }
+
+        _result = {}
+
+        for host, data in results.items():
+            _result[host] = {}
+            for component, details in data.items():
+                res = any([member['status']['health'].lower() != 'ok' for member in data[component].values()])
+                _result[host][component] = mapper[res]
+
+        if hostname and hostname in results.keys():
+            return _result[hostname]
+        else:
+            return _result
+
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def common(self, **kw) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+        status: List[str] = []
+        hostname = kw.get('hostname',)
+        cmd = kw.get('cmd',)
+        results = self.get_full_report()
+
+        _result = {}
+
+        for host, data in results.items():
+            try:
+                _result[host] = data[cmd]
+            except KeyError:
+                raise RuntimeError(f'invalid endpoint {cmd}')
+
+        if hostname and hostname in results.keys():
+            return _result[hostname]
+        else:
+            return _result
+
+    def dispatch(self, hostname='', cmd=''):
+        kw = dict(hostname=hostname, cmd=cmd)
+        try:
+            func = getattr(self, cmd)
+            result = func(**kw)
+        except AttributeError:
+            try:
+                result = self.common(**kw)
+            except RuntimeError:
+                cherrypy.response.status = 404
+                result = {}
+            return {"error": "Not a valid endpoint."}
+        finally:
+            return result
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def index(self, hostname=None, cmd=''):
+        result = self.dispatch(hostname, cmd)
+        return result
+
+
+class HostData(Server):
+    exposed = True
+
+    def __init__(self, mgr: "CephadmOrchestrator", port: int, host: str):
+        self.mgr = mgr
+        super().__init__()
+        self.socket_port = port
+        self.socket_host = host
+        self.subscribe()
+
+    def stop(self) -> None:
+        # we must call unsubscribe before stopping the server,
+        # otherwise the port is not released and we will get
+        # an exception when trying to restart it
+        self.unsubscribe()
+        super().stop()
+
+    @cherrypy.tools.allow(methods=['POST'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.expose
+    def index(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = cherrypy.request.json
+        results: Dict[str, Any] = {}
+        try:
+            self.check_request_fields(data)
+        except Exception as e:
+            results['result'] = f'Bad metadata: {e}'
+            self.mgr.log.warning(f'Received bad metadata from an agent: {e}')
+        else:
+            # if we got here, we've already verified the keyring of the agent. If
+            # host agent is reporting on is marked offline, it shouldn't be any more
+            self.mgr.offline_hosts_remove(data['host'])
+            results['result'] = self.handle_metadata(data)
+        return results
+
     def check_request_fields(self, data: Dict[str, Any]) -> None:
         fields = '{' + ', '.join([key for key in data.keys()]) + '}'
         if 'host' not in data:

From db0172186a753d57c357a5396378d1158e3167e3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 27 Sep 2023 08:27:28 +0000
Subject: [PATCH 1559/2492] node-proxy: split redfishdell class

This refactors split the redfishdell class in order
to collect power and thermal details from the redfish API.

'power' and 'thermal' details are very different in many points:

- not available at the same endpoint,
- data structure is different.

For these two reasons, let's split that class.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 ...redfish_system.py => baseredfishsystem.py} | 149 ++++++++----------
 .../cephadmlib/node_proxy/redfish_dell.py     |  65 +-------
 .../node_proxy/redfishdellchassis.py          |  67 ++++++++
 .../node_proxy/redfishdellsystem.py           |  95 +++++++++++
 src/pybind/mgr/cephadm/agent.py               |   7 +-
 5 files changed, 240 insertions(+), 143 deletions(-)
 rename src/cephadm/cephadmlib/node_proxy/{redfish_system.py => baseredfishsystem.py} (76%)
 create mode 100644 src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
 create mode 100644 src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py

diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_system.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
similarity index 76%
rename from src/cephadm/cephadmlib/node_proxy/redfish_system.py
rename to src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 95c82960eae1..2e1d9087d797 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_system.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -2,114 +2,35 @@
 from .redfish_client import RedFishClient
 from threading import Thread, Lock
 from time import sleep
-from .util import Logger, retry, normalize_dict, to_snake_case
+from .util import Logger, retry
 from typing import Dict, Any, List
 
 
-class RedfishSystem(BaseSystem):
+class BaseRedfishSystem(BaseSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
         self.log = Logger(__name__)
         self.host: str = kw['host']
         self.username: str = kw['username']
         self.password: str = kw['password']
-        self.system_endpoint = kw.get('system_endpoint', '/Systems/1')
+        # move the following line (class attribute?)
         self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
         self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
 
-        self._system: Dict[str, Dict[str, Any]] = {}
         self.run: bool = False
         self.thread: Thread
-        self.start_client()
         self.data_ready: bool = False
         self.previous_data: Dict = {}
         self.lock: Lock = Lock()
-
-    @retry(retries=10, delay=2)
-    def _get_path(self, path: str) -> Dict:
-        result = self.client.get_path(path)
-        if result is None:
-            self.log.logger.error(f"The client reported an error when getting path: {path}")
-            raise RuntimeError(f"Could not get path: {path}")
-        return result
-
-    def get_members(self, path: str) -> List:
-        _path = self._system[path]['@odata.id']
-        data = self._get_path(_path)
-        return [self._get_path(member['@odata.id']) for member in data['Members']]
-
-    def build_data(self,
-                   fields: List,
-                   path: str) -> Dict[str, Dict[str, Dict]]:
-        result: Dict[str, Dict[str, Dict]] = dict()
-        for member_info in self.get_members(path):
-            member_id = member_info['Id']
-            result[member_id] = dict()
-            for field in fields:
-                try:
-                    result[member_id][to_snake_case(field)] = member_info[field]
-                except KeyError:
-                    self.log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
-
-        return normalize_dict(result)
+        self.data: Dict[str, Dict[str, Any]] = {}
+        self._system: Dict[str, Dict[str, Any]] = {}
+        self.start_client()
 
     def start_client(self) -> None:
         if not self.client:
             self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
         self.client.login()
 
-    def get_system(self) -> Dict[str, Dict[str, Dict]]:
-        result = {
-            'storage': self.get_storage(),
-            'processors': self.get_processors(),
-            'network': self.get_network(),
-            'memory': self.get_memory(),
-        }
-        return result
-
-    def get_status(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['status']
-
-    def get_metadata(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['metadata']
-
-    def get_memory(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['memory']
-
-    def get_power(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['power']
-
-    def get_processors(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['processors']
-
-    def get_network(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['network']
-
-    def get_storage(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['storage']
-
-    def _update_system(self) -> None:
-        redfish_system = self.client.get_path(self.system_endpoint)
-        self._system = {**redfish_system, **self._system}
-
-    def _update_metadata(self) -> None:
-        raise NotImplementedError()
-
-    def _update_memory(self) -> None:
-        raise NotImplementedError()
-
-    def _update_power(self) -> None:
-        raise NotImplementedError()
-
-    def _update_network(self) -> None:
-        raise NotImplementedError()
-
-    def _update_processors(self) -> None:
-        raise NotImplementedError()
-
-    def _update_storage(self) -> None:
-        raise NotImplementedError()
-
     def start_update_loop(self) -> None:
         self.run = True
         self.thread = Thread(target=self.update)
@@ -133,6 +54,7 @@ def update(self) -> None:
                     self._update_metadata()
                     self._update_memory()
                     self._update_power()
+                    self._update_fans()
                     self._update_network()
                     self._update_processors()
                     self._update_storage()
@@ -158,3 +80,60 @@ def flush(self) -> None:
         self.log.logger.info("Data marked as not ready.")
         self.lock.release()
         self.log.logger.info("Lock released.")
+
+    @retry(retries=10, delay=2)
+    def _get_path(self, path: str) -> Dict:
+        result = self.client.get_path(path)
+        if result is None:
+            self.log.logger.error(f"The client reported an error when getting path: {path}")
+            raise RuntimeError(f"Could not get path: {path}")
+        return result
+
+    def get_members(self, path: str) -> List:
+        _path = self._system[path]['@odata.id']
+        data = self._get_path(_path)
+        return [self._get_path(member['@odata.id']) for member in data['Members']]
+
+    def build_data(self,
+                   fields: List,
+                   path: str) -> Dict[str, Dict[str, Dict]]:
+        raise NotImplementedError()
+
+    # def _update_system(self) -> None:
+    #     raise NotImplementedError()
+
+    def get_system(self) -> Dict[str, Dict[str, Dict]]:
+        result = {
+            'storage': self.get_storage(),
+            'processors': self.get_processors(),
+            'network': self.get_network(),
+            'memory': self.get_memory(),
+            'power': self.get_power(),
+            'fans': self.get_fans()
+        }
+        return result
+
+    def _update_system(self) -> None:
+        redfish_system = self.client.get_path(self.system_endpoint)
+        self._system = {**redfish_system, **self._system}
+
+    def _update_metadata(self) -> None:
+        raise NotImplementedError()
+
+    def _update_memory(self) -> None:
+        raise NotImplementedError()
+
+    def _update_power(self) -> None:
+        raise NotImplementedError()
+
+    def _update_fans(self) -> None:
+        raise NotImplementedError()
+
+    def _update_network(self) -> None:
+        raise NotImplementedError()
+
+    def _update_processors(self) -> None:
+        raise NotImplementedError()
+
+    def _update_storage(self) -> None:
+        raise NotImplementedError()
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
index 796c989f8834..f6a01664629c 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
@@ -1,63 +1,14 @@
-from .redfish_system import RedfishSystem
-from .util import Logger, normalize_dict, to_snake_case
-from typing import Dict, Any
+from .redfishdellchassis import RedfishDellChassis
+from .redfishdellsystem import RedfishDellSystem
+from .util import Logger
+from typing import Any
 
 
-class RedfishDell(RedfishSystem):
+class RedfishDell(RedfishDellSystem, RedfishDellChassis):
     def __init__(self, **kw: Any) -> None:
-        self.log = Logger(__name__)
         if kw.get('system_endpoint') is None:
             kw['system_endpoint'] = '/Systems/System.Embedded.1'
+        if kw.get('chassis_endpoint') is None:
+            kw['chassis_endpoint'] = '/Chassis/System.Embedded.1'
         super().__init__(**kw)
-
-    def _update_network(self) -> None:
-        fields = ['Description', 'Name', 'SpeedMbps', 'Status']
-        self.log.logger.info("Updating network")
-        self._system['network'] = self.build_data(fields, 'EthernetInterfaces')
-
-    def _update_processors(self) -> None:
-        fields = ['Description',
-                  'TotalCores',
-                  'TotalThreads',
-                  'ProcessorType',
-                  'Model',
-                  'Status',
-                  'Manufacturer']
-        self.log.logger.info("Updating processors")
-        self._system['processors'] = self.build_data(fields, 'Processors')
-
-    def _update_storage(self) -> None:
-        fields = ['Description',
-                  'CapacityBytes',
-                  'Model', 'Protocol',
-                  'SerialNumber', 'Status',
-                  'PhysicalLocation']
-        entities = self.get_members('Storage')
-        self.log.logger.info("Updating storage")
-        result: Dict[str, Dict[str, Dict]] = dict()
-        for entity in entities:
-            for drive in entity['Drives']:
-                drive_path = drive['@odata.id']
-                drive_info = self._get_path(drive_path)
-                drive_id = drive_info['Id']
-                result[drive_id] = dict()
-                for field in fields:
-                    result[drive_id][to_snake_case(field)] = drive_info[field]
-                    result[drive_id]['entity'] = entity['Id']
-        self._system['storage'] = normalize_dict(result)
-
-    def _update_metadata(self) -> None:
-        self.log.logger.info("Updating metadata")
-        pass
-
-    def _update_memory(self) -> None:
-        fields = ['Description',
-                  'MemoryDeviceType',
-                  'CapacityMiB',
-                  'Status']
-        self.log.logger.info("Updating memory")
-        self._system['memory'] = self.build_data(fields, 'Memory')
-
-    def _update_power(self) -> None:
-        self.log.logger.info("Updating power")
-        pass
+        self.log = Logger(__name__)
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py b/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
new file mode 100644
index 000000000000..39610dc74456
--- /dev/null
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
@@ -0,0 +1,67 @@
+from .baseredfishsystem import BaseRedfishSystem
+from .redfish_client import RedFishClient
+from threading import Thread, Lock
+from time import sleep
+from .util import Logger, retry, normalize_dict, to_snake_case
+from typing import Dict, Any, List, Union
+
+
+class RedfishDellChassis(BaseRedfishSystem):
+    def __init__(self, **kw: Any) -> None:
+        self.chassis_endpoint = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
+        super().__init__(**kw)
+        self.log = Logger(__name__)
+        self.log.logger.info(f"{__name__} initialization.")
+
+    def get_power(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['power']
+
+    def get_fans(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['fans']
+
+    def get_chassis(self) -> Dict[str, Dict[str, Dict]]:
+        result = {
+            'power': self.get_power(),
+            'fans': self.get_fans()
+        }
+        return result
+
+    def _update_power(self) -> None:
+        fields = {
+            "PowerSupplies": [
+                "Name",
+                "Model",
+                "Manufacturer",
+                "Status"
+            ]
+        }
+        self.log.logger.info("Updating powersupplies")
+        self._system['power'] = self.build_chassis_data(fields, 'Power')
+
+    def _update_fans(self) -> None:
+        fields = {
+            "Fans": [
+                "Name",
+                "PhysicalContext",
+                "Status"
+            ],
+        }
+        self.log.logger.info("Updating fans")
+        self._system['fans'] = self.build_chassis_data(fields, 'Thermal')
+
+    def build_chassis_data(self,
+                   fields: Dict[str, List[str]],
+                   path: str) -> Dict[str, Dict[str, Dict]]:
+        result: Dict[str, Dict[str, Dict]] = dict()
+        data = self._get_path(f"{self.chassis_endpoint}/{path}")
+
+        for elt, _fields in fields.items():
+            for member_elt in data[elt]:
+                _id = member_elt['MemberId']
+                result[_id] = dict()
+                for field in _fields:
+                    try:
+                        result[_id][to_snake_case(field)] = member_elt[field]
+                    except KeyError:
+                        self.log.logger.warning(f"Could not find field: {field} in data: {data[elt]}")
+        return normalize_dict(result)
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
new file mode 100644
index 000000000000..de9756fe79fe
--- /dev/null
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -0,0 +1,95 @@
+from .baseredfishsystem import BaseRedfishSystem
+from .util import Logger, normalize_dict, to_snake_case
+from typing import Dict, Any, List
+
+
+class RedfishDellSystem(BaseRedfishSystem):
+    def __init__(self, **kw: Any) -> None:
+        self.system_endpoint = kw.get('systemd_endpoint', '/Systems/System.Embedded.1')
+        super().__init__(**kw)
+        self.log = Logger(__name__)
+
+    def build_system_data(self,
+                   fields: List,
+                   path: str) -> Dict[str, Dict[str, Dict]]:
+        result: Dict[str, Dict[str, Dict]] = dict()
+        for member_info in self.get_members(path):
+            member_id = member_info['Id']
+            result[member_id] = dict()
+            for field in fields:
+                try:
+                    result[member_id][to_snake_case(field)] = member_info[field]
+                except KeyError:
+                    self.log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
+
+        return normalize_dict(result)
+
+    def get_status(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['status']
+
+    def get_metadata(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['metadata']
+
+    def get_memory(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['memory']
+
+    def get_processors(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['processors']
+
+    def get_network(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['network']
+
+    def get_storage(self) -> Dict[str, Dict[str, Dict]]:
+        return self._system['storage']
+
+    # def _update_system(self) -> None:
+    #     redfish_system = self.client.get_path(self.system_endpoint)
+    #     self._system = {**redfish_system, **self._system}
+
+    def _update_network(self) -> None:
+        fields = ['Description', 'Name', 'SpeedMbps', 'Status']
+        self.log.logger.info("Updating network")
+        self._system['network'] = self.build_system_data(fields, 'EthernetInterfaces')
+
+    def _update_processors(self) -> None:
+        fields = ['Description',
+                  'TotalCores',
+                  'TotalThreads',
+                  'ProcessorType',
+                  'Model',
+                  'Status',
+                  'Manufacturer']
+        self.log.logger.info("Updating processors")
+        self._system['processors'] = self.build_system_data(fields, 'Processors')
+
+    def _update_storage(self) -> None:
+        fields = ['Description',
+                  'CapacityBytes',
+                  'Model', 'Protocol',
+                  'SerialNumber', 'Status',
+                  'PhysicalLocation']
+        entities = self.get_members('Storage')
+        self.log.logger.info("Updating storage")
+        result: Dict[str, Dict[str, Dict]] = dict()
+        for entity in entities:
+            for drive in entity['Drives']:
+                drive_path = drive['@odata.id']
+                drive_info = self._get_path(drive_path)
+                drive_id = drive_info['Id']
+                result[drive_id] = dict()
+                for field in fields:
+                    result[drive_id][to_snake_case(field)] = drive_info[field]
+                    result[drive_id]['entity'] = entity['Id']
+        self._system['storage'] = normalize_dict(result)
+
+    def _update_metadata(self) -> None:
+        self.log.logger.info("Updating metadata")
+        pass
+
+    def _update_memory(self) -> None:
+        fields = ['Description',
+                  'MemoryDeviceType',
+                  'CapacityMiB',
+                  'Status']
+        self.log.logger.info("Updating memory")
+        self._system['memory'] = self.build_system_data(fields, 'Memory')
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index d3376d235889..697f097e1434 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -147,7 +147,10 @@ def get_nok_members(self,
             # Force a fake error for testing purpose
             if component == 'storage':
                 _status = 'critical'
-                state = "Fake error"
+                state = "[Fake error] device is faulty."
+            elif component == 'power':
+                _status = 'critical'
+                state = "[Fake error] power supply unplugged."
             else:
                 _status = data[component][member]['status']['health'].lower()
             if _status.lower() != 'ok':
@@ -167,6 +170,8 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
             'memory': 'NODE_PROXY_MEMORY',
             'processors': 'NODE_PROXY_PROCESSORS',
             'network': 'NODE_PROXY_NETWORK',
+            'power': 'NODE_PROXY_POWER',
+            'fans': 'NODE_PROXY_FANS'
         }
 
         for component in data['data'].keys():

From 14bfc07a1a3ad483f2f91c5d1fde9f073c12867e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 27 Sep 2023 09:41:49 +0000
Subject: [PATCH 1560/2492] node-proxy: update alert names

Given that the 'node-proxy' terminology is internal, let's change
the few node-proxy related alert names to something
more user friendly as they are intended to be seen by the user

(NODE_PROXY_xxx > HARDWARE_xxx).

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 697f097e1434..2c99149998c2 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -166,12 +166,12 @@ def get_nok_members(self,
 
     def raise_alert(self, data: Dict[str, Any]) -> None:
         mapping: Dict[str, str] = {
-            'storage': 'NODE_PROXY_STORAGE',
-            'memory': 'NODE_PROXY_MEMORY',
-            'processors': 'NODE_PROXY_PROCESSORS',
-            'network': 'NODE_PROXY_NETWORK',
-            'power': 'NODE_PROXY_POWER',
-            'fans': 'NODE_PROXY_FANS'
+            'storage': 'HARDWARE_STORAGE',
+            'memory': 'HARDWARE_MEMORY',
+            'processors': 'HARDWARE_PROCESSORS',
+            'network': 'HARDWARE_NETWORK',
+            'power': 'HARDWARE_POWER',
+            'fans': 'HARDWARE_FANS'
         }
 
         for component in data['data'].keys():

From 9fee8362a91fc4c263a69003733d6c1fde37db5e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 27 Sep 2023 13:00:17 +0000
Subject: [PATCH 1561/2492] node-proxy: lower verbosity level

This reduces the verbosity level for some messages.
These are generating a lot of messages while they can be needed
only for debugging purposes.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/redfishdellchassis.py        |  4 ++--
 src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py b/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
index 39610dc74456..3238f9e8a373 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
@@ -35,7 +35,7 @@ def _update_power(self) -> None:
                 "Status"
             ]
         }
-        self.log.logger.info("Updating powersupplies")
+        self.log.logger.debug("Updating powersupplies")
         self._system['power'] = self.build_chassis_data(fields, 'Power')
 
     def _update_fans(self) -> None:
@@ -46,7 +46,7 @@ def _update_fans(self) -> None:
                 "Status"
             ],
         }
-        self.log.logger.info("Updating fans")
+        self.log.logger.debug("Updating fans")
         self._system['fans'] = self.build_chassis_data(fields, 'Thermal')
 
     def build_chassis_data(self,
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index de9756fe79fe..a79a3fcf52be 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -48,7 +48,7 @@ def get_storage(self) -> Dict[str, Dict[str, Dict]]:
 
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
-        self.log.logger.info("Updating network")
+        self.log.logger.debug('Updating network')
         self._system['network'] = self.build_system_data(fields, 'EthernetInterfaces')
 
     def _update_processors(self) -> None:
@@ -59,7 +59,7 @@ def _update_processors(self) -> None:
                   'Model',
                   'Status',
                   'Manufacturer']
-        self.log.logger.info("Updating processors")
+        self.log.logger.debug('Updating processors')
         self._system['processors'] = self.build_system_data(fields, 'Processors')
 
     def _update_storage(self) -> None:
@@ -69,7 +69,7 @@ def _update_storage(self) -> None:
                   'SerialNumber', 'Status',
                   'PhysicalLocation']
         entities = self.get_members('Storage')
-        self.log.logger.info("Updating storage")
+        self.log.logger.debug('Updating storage')
         result: Dict[str, Dict[str, Dict]] = dict()
         for entity in entities:
             for drive in entity['Drives']:
@@ -83,7 +83,7 @@ def _update_storage(self) -> None:
         self._system['storage'] = normalize_dict(result)
 
     def _update_metadata(self) -> None:
-        self.log.logger.info("Updating metadata")
+        self.log.logger.debug('Updating metadata')
         pass
 
     def _update_memory(self) -> None:
@@ -91,5 +91,5 @@ def _update_memory(self) -> None:
                   'MemoryDeviceType',
                   'CapacityMiB',
                   'Status']
-        self.log.logger.info("Updating memory")
+        self.log.logger.debug('Updating memory')
         self._system['memory'] = self.build_system_data(fields, 'Memory')

From 05cc6afe4b76d35f549bb8928e459bc8fb93697c Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 29 Sep 2023 13:05:31 +0000
Subject: [PATCH 1562/2492] node-proxy: validate_node_proxy_data() refactor

This introduces minor changes in order to improve error
handling in validate_node_proxy_data()

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 2c99149998c2..0228d73f8285 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -117,24 +117,26 @@ def idrac(self) -> Dict[str, Any]:
             idrac_details = self.mgr.get_store('node_proxy/idrac')
             idrac_details_json = json.loads(idrac_details)
             results['result'] = idrac_details_json[data["host"]]
+        else:
+            results['result'] = self.validate_msg
 
         return results
 
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
+        self.validate_msg = 'valid node-proxy data received.'
+        cherrypy.response.status = 200
         if 'host' not in data:
             cherrypy.response.status = 400
-            self.mgr.log.warning('The field \'host\' must be provided.')
+            self.validate_msg = 'The field \'host\' must be provided.'
         elif 'keyring' not in data:
             cherrypy.response.status = 400
-            self.mgr.log.warning(f'The agent keyring must be provided.')
+            self.validate_msg = 'The agent keyring must be provided.'
         elif not self.mgr.agent_cache.agent_keys.get(data['host']):
             cherrypy.response.status = 400
-            self.mgr.log.warning(f'Make sure the agent is running on {data["host"]}')
+            self.validate_msg = f'Make sure the agent is running on {data["host"]}'
         elif data['keyring'] != self.mgr.agent_cache.agent_keys[data['host']]:
             cherrypy.response.status = 403
-            self.mgr.log.warning(f'Got wrong keyring from agent on host {data["host"]}.')
-        else:
-            cherrypy.response.status = 200
+            self.validate_msg = f'Got wrong keyring from agent on host {data["host"]}.'
 
         return cherrypy.response.status == 200
 
@@ -199,7 +201,7 @@ def data(self) -> Dict[str, Any]:
             self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
             self.raise_alert(data)
 
-            results['result'] = data
+        results["result"] = self.validate_msg
 
         return results
 

From 76d8e0bbbf2c5130a325943ffe09791cbd4f2feb Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 4 Oct 2023 10:00:26 +0000
Subject: [PATCH 1563/2492] monitoring: add new alerts

This adds new hardware monitoring alerts.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../ceph-mixin/prometheus_alerts.libsonnet    |  65 ++++++++
 monitoring/ceph-mixin/prometheus_alerts.yml   |  62 ++++++++
 .../ceph-mixin/tests_alerts/test_alerts.yml   | 144 ++++++++++++++++++
 monitoring/snmp/README.md                     |   1 +
 4 files changed, 272 insertions(+)

diff --git a/monitoring/ceph-mixin/prometheus_alerts.libsonnet b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
index b90573be6c01..b5fed4fe7cf7 100644
--- a/monitoring/ceph-mixin/prometheus_alerts.libsonnet
+++ b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
@@ -689,6 +689,71 @@
         },
       ],
     },
+    {
+      name: 'hardware',
+      rules: [
+        {
+          alert: 'HardwareStorageError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_STORAGE"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.1' },
+          annotations: {
+            summary: 'Storage devices error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'Some storage devices are in error. Check `ceph health detail`.',
+          },
+        },
+        {
+          alert: 'HardwareMemoryError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_MEMORY"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.2' },
+          annotations: {
+            summary: 'DIMM error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'DIMM error(s) detected. Check `ceph health detail`.',
+          },
+        },
+        {
+          alert: 'HardwareProcessorError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.3' },
+          annotations: {
+            summary: 'Processor error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'Processor error(s) detected. Check `ceph health detail`.',
+          },
+        },
+        {
+          alert: 'HardwareNetworkError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_NETWORK"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.4' },
+          annotations: {
+            summary: 'Network error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'Network error(s) detected. Check `ceph health detail`.',
+          },
+        },
+        {
+          alert: 'HardwarePowerError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_POWER"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.5' },
+          annotations: {
+            summary: 'Power supply error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'Power supply error(s) detected. Check `ceph health detail`.',
+          },
+        },
+        {
+          alert: 'HardwareFanError',
+          'for': '30s',
+          expr: 'ceph_health_detail{name="HARDWARE_FANS"} > 0',
+          labels: { severity: 'critical', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.13.6' },
+          annotations: {
+            summary: 'Fan error(s) detected%(cluster)s' % $.MultiClusterSummary(),
+            description: 'Fan error(s) detected. Check `ceph health detail`.',
+          },
+        },
+      ],
+    },
     {
       name: 'PrometheusServer',
       rules: [
diff --git a/monitoring/ceph-mixin/prometheus_alerts.yml b/monitoring/ceph-mixin/prometheus_alerts.yml
index b549d6bf49a1..5b011aa3568a 100644
--- a/monitoring/ceph-mixin/prometheus_alerts.yml
+++ b/monitoring/ceph-mixin/prometheus_alerts.yml
@@ -614,6 +614,68 @@ groups:
         labels:
           severity: "warning"
           type: "ceph_default"
+  - name: "hardware"
+    rules:
+      - alert: "HardwareStorageError"
+        annotations:
+          description: "Some storage devices are in error. Check `ceph health detail`."
+          summary: "Storage devices error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_STORAGE\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.1"
+          severity: "critical"
+          type: "ceph_default"
+      - alert: "HardwareMemoryError"
+        annotations:
+          description: "DIMM error(s) detected. Check `ceph health detail`."
+          summary: "DIMM error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_MEMORY\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.2"
+          severity: "critical"
+          type: "ceph_default"
+      - alert: "HardwareProcessorError"
+        annotations:
+          description: "Processor error(s) detected. Check `ceph health detail`."
+          summary: "Processor error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_PROCESSOR\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.3"
+          severity: "critical"
+          type: "ceph_default"
+      - alert: "HardwareNetworkError"
+        annotations:
+          description: "Network error(s) detected. Check `ceph health detail`."
+          summary: "Network error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_NETWORK\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.4"
+          severity: "critical"
+          type: "ceph_default"
+      - alert: "HardwarePowerError"
+        annotations:
+          description: "Power supply error(s) detected. Check `ceph health detail`."
+          summary: "Power supply error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_POWER\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.5"
+          severity: "critical"
+          type: "ceph_default"
+      - alert: "HardwareFanError"
+        annotations:
+          description: "Fan error(s) detected. Check `ceph health detail`."
+          summary: "Fan error(s) detected"
+        expr: "ceph_health_detail{name=\"HARDWARE_FANS\"} > 0"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.13.6"
+          severity: "critical"
+          type: "ceph_default"
   - name: "PrometheusServer"
     rules:
       - alert: "PrometheusJobMissing"
diff --git a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
index 9eaa096d5db0..9cb688e9ca0e 100644
--- a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
+++ b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
@@ -2075,3 +2075,147 @@ tests:
            description: "Detected a heavy increase in bandwidth for rbd replications (over 80%) in the last 30 min. This might not be a problem, but it is good to review the number of images being replicated simultaneously"
            summary: "The replication network usage has been increased over 80% in the last 30 minutes. Review the number of images being replicated. This alert will be cleaned automatically after 30 minutes"
 
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_STORAGE"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_STORAGE"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_STORAGE"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwareStorageError
+    - eval_time: 5m
+      alertname: HardwareStorageError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_STORAGE
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.1
+        exp_annotations:
+          summary: Storage devices error(s) detected
+          description: "Some storage devices are in error. Check `ceph health detail`."
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_MEMORY"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_MEMORY"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_MEMORY"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwareMemoryError
+    - eval_time: 5m
+      alertname: HardwareMemoryError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_MEMORY
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.2
+        exp_annotations:
+          summary: DIMM error(s) detected
+          description: "DIMM error(s) detected. Check `ceph health detail`."
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_PROCESSOR"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_PROCESSOR"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_PROCESSOR"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwareProcessorError
+    - eval_time: 5m
+      alertname: HardwareProcessorError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_PROCESSOR
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.3
+        exp_annotations:
+          summary: Processor error(s) detected
+          description: "Processor error(s) detected. Check `ceph health detail`."
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_NETWORK"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_NETWORK"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_NETWORK"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwareNetworkError
+    - eval_time: 5m
+      alertname: HardwareNetworkError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_NETWORK
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.4
+        exp_annotations:
+          summary: Network error(s) detected
+          description: "Network error(s) detected. Check `ceph health detail`."
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_POWER"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_POWER"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_POWER"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwarePowerError
+    - eval_time: 5m
+      alertname: HardwarePowerError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_POWER
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.5
+        exp_annotations:
+          summary: Power supply error(s) detected
+          description: "Power supply error(s) detected. Check `ceph health detail`."
+ - interval: 30s
+   input_series:
+    - series: 'ceph_health_detail{name="HARDWARE_FANS"}'
+      values: '1+0x40'
+   promql_expr_test:
+     - expr: ceph_health_detail{name="HARDWARE_FANS"} > 0
+       eval_time: 2m
+       exp_samples:
+         - labels: '{__name__="ceph_health_detail", name="HARDWARE_FANS"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 1m
+      alertname: HardwareFanError
+    - eval_time: 5m
+      alertname: HardwareFanError
+      exp_alerts:
+      - exp_labels:
+          name: HARDWARE_FANS
+          severity: critical
+          type: ceph_default
+          oid: 1.3.6.1.4.1.50495.1.2.1.13.6
+        exp_annotations:
+          summary: Fan error(s) detected
+          description: "Fan error(s) detected. Check `ceph health detail`."
diff --git a/monitoring/snmp/README.md b/monitoring/snmp/README.md
index 1a5b609556df..c96dffa3d68d 100644
--- a/monitoring/snmp/README.md
+++ b/monitoring/snmp/README.md
@@ -40,6 +40,7 @@ internet private enterprise   ceph   ceph    Notifications   Prometheus  Notific
                                                                             .10  (Rados)
                                                                             .11 (cephadm)
                                                                             .12 (prometheus)
+                                                                            .13 (hardware)
 
 ```
 Individual alerts are placed within the appropriate alert category. For example, to add

From a48c34ef0034de335c1ec5d599272fc9d958a506 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 6 Oct 2023 11:10:39 +0000
Subject: [PATCH 1564/2492] mgr/cephadm: add NodeProxyCache class

This is for tracking and caching any node-proxy data.
The node-proxy API now uses this class to serve its data.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py     | 71 ++++---------------------
 src/pybind/mgr/cephadm/inventory.py | 80 +++++++++++++++++++++++++++++
 src/pybind/mgr/cephadm/module.py    |  5 +-
 src/pybind/mgr/cephadm/serve.py     |  3 ++
 4 files changed, 98 insertions(+), 61 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 0228d73f8285..648b3b58d49b 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -198,85 +198,36 @@ def data(self) -> Dict[str, Any]:
 
         data: Dict[str, Any] = cherrypy.request.json
         if self.validate_node_proxy_data(data):
-            self.mgr.set_store(f'node_proxy/data/{data["host"]}', json.dumps(data['data']))
+            host = data['host']
+            self.mgr.node_proxy.save(host, data['data'])
             self.raise_alert(data)
 
         results["result"] = self.validate_msg
 
         return results
 
-    def get_full_report(self) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-
-        for k, v in self.mgr.get_store_prefix('node_proxy/data').items():
-            host = k.split('/')[-1:][0]
-            results[host] = json.loads(v)
-        return results
-
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     def fullreport(self, **kw: Any) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-        results = self.get_full_report()
-        hostname = kw.get('hostname',)
+        return self.mgr.node_proxy.fullreport(**kw)
 
-        if hostname not in results.keys():
-            return results
-        else:
-            return results[hostname]
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def criticals(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.criticals()
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     def summary(self, **kw: Any) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-        status: List[str] = []
-        hostname = kw.get('hostname',)
-
-        results = self.get_full_report()
+        return self.mgr.node_proxy.summary(**kw)
 
-        mapper: Dict[bool, str] = {
-            True: 'error',
-            False: 'ok'
-        }
-
-        _result = {}
-
-        for host, data in results.items():
-            _result[host] = {}
-            for component, details in data.items():
-                res = any([member['status']['health'].lower() != 'ok' for member in data[component].values()])
-                _result[host][component] = mapper[res]
-
-        if hostname and hostname in results.keys():
-            return _result[hostname]
-        else:
-            return _result
-
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def common(self, **kw) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-        status: List[str] = []
-        hostname = kw.get('hostname',)
-        cmd = kw.get('cmd',)
-        results = self.get_full_report()
-
-        _result = {}
-
-        for host, data in results.items():
-            try:
-                _result[host] = data[cmd]
-            except KeyError:
-                raise RuntimeError(f'invalid endpoint {cmd}')
-
-        if hostname and hostname in results.keys():
-            return _result[hostname]
-        else:
-            return _result
+        return self.mgr.node_proxy.common(**kw)
 
     def dispatch(self, hostname='', cmd=''):
         kw = dict(hostname=hostname, cmd=cmd)
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 329501fc7dcb..8f83dcb13089 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -29,6 +29,7 @@
 HOST_CACHE_PREFIX = "host."
 SPEC_STORE_PREFIX = "spec."
 AGENT_CACHE_PREFIX = 'agent.'
+NODE_PROXY_CACHE_PREFIX = 'node_proxy/data'
 
 
 class HostCacheStatus(enum.Enum):
@@ -1405,6 +1406,85 @@ def get_scheduled_daemon_action(self, host: str, daemon: str) -> Optional[str]:
         return self.scheduled_daemon_actions.get(host, {}).get(daemon)
 
 
+class NodeProxyCache:
+    def __init__(self, mgr: "CephadmOrchestrator") -> None:
+        self.mgr = mgr
+        self.data: Dict[str, Any] = {}
+        self.idrac = {}
+
+    def load(self) -> None:
+        _idrac = self.mgr.get_store('node_proxy/idrac', "{}")
+        self.idrac = json.loads(_idrac)
+
+        for k, v in self.mgr.get_store_prefix(NODE_PROXY_CACHE_PREFIX).items():
+            host = k.split('/')[-1:][0]
+
+            if host not in self.mgr.inventory.keys():
+                # remove entry for host that no longer exists
+                self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", None)
+                try:
+                    self.idrac.pop(host)
+                    self.data.pop(host)
+                except KeyError:
+                    pass
+                continue
+
+            self.data[host] = json.loads(v)
+
+    def save(self,
+             host: str = '',
+             data: Dict[str, Any] = {}) -> None:
+        self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", json.dumps(data))
+
+    def fullreport(self, **kw: Any) -> Dict[str, Any]:
+        hostname = kw.get('hostname')
+        if hostname not in self.data.keys():
+            return self.data
+        else:
+            return self.data[hostname]
+
+    def summary(self, **kw: Any) -> Dict[str, Any]:
+        hostname = kw.get('hostname')
+        results = self.data
+
+        mapper: Dict[bool, str] = {
+            True: 'error',
+            False: 'ok'
+        }
+
+        _result: Dict[str, Any] = {}
+
+        for host, data in results.items():
+            _result[host] = {}
+            for component, details in data.items():
+                res = any([member['status']['health'].lower() != 'ok' for member in data[component].values()])
+                _result[host][component] = mapper[res]
+
+        if hostname and hostname in results.keys():
+            return _result[hostname]
+        else:
+            return _result
+
+    def common(self, **kw):
+        hostname = kw.get('hostname',)
+        cmd = kw.get('cmd',)
+        _result = {}
+
+        for host, data in self.data.items():
+            try:
+                _result[host] = data[cmd]
+            except KeyError:
+                raise RuntimeError(f'Invalid node-proxy category {cmd}')
+
+        if hostname and hostname in self.data.keys():
+            return _result[hostname]
+        else:
+            return _result
+
+    def criticals(self, **kw):
+        return {}
+
+
 class AgentCache():
     """
     AgentCache is used for storing metadata about agent daemons that must be kept
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 8c1e3a482847..8e54381b18cf 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -74,7 +74,7 @@
 from .services.jaeger import ElasticSearchService, JaegerAgentService, JaegerCollectorService, JaegerQueryService
 from .schedule import HostAssignment
 from .inventory import Inventory, SpecStore, HostCache, AgentCache, EventStore, \
-    ClientKeyringStore, ClientKeyringSpec, TunedProfileStore
+    ClientKeyringStore, ClientKeyringSpec, TunedProfileStore, NodeProxyCache
 from .upgrade import CephadmUpgrade
 from .template import TemplateMgr
 from .utils import CEPH_IMAGE_TYPES, RESCHEDULE_FROM_OFFLINE_HOSTS_TYPES, forall_hosts, \
@@ -600,6 +600,9 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.cache = HostCache(self)
         self.cache.load()
 
+        self.node_proxy = NodeProxyCache(self)
+        self.node_proxy.load()
+
         self.agent_cache = AgentCache(self)
         self.agent_cache.load()
 
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 116e97238691..ecd94ac084e8 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -116,6 +116,9 @@ def serve(self) -> None:
                     if self.mgr.upgrade.continue_upgrade():
                         continue
 
+                    # refresh node-proxy cache
+                    self.mgr.node_proxy.load()
+
             except OrchestratorError as e:
                 if e.event_subject:
                     self.mgr.events.from_orch_error(e)

From 12b735204734a7a1c6196d330b9babc3a3860f33 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 25 Jan 2024 09:45:08 -0500
Subject: [PATCH 1565/2492] mon: zero-initialize MonCommand::flags

causing failures in check-generated.sh:

2 MonCommand
/tmp/typ-qmnDZ1ahR /tmp/typ-lIgJTZiUe differ: byte 100, line 6
**** MonCommand test 1 dump_json check failed ****
   ceph-dencoder type MonCommand select_test 1 dump_json > /tmp/typ-qmnDZ1ahR
   ceph-dencoder type MonCommand select_test 1 encode decode dump_json > /tmp/typ-lIgJTZiUe
6c6
<     "flags": 94661933599904
---
>     "flags": 94342467308704

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/mon/MonCommand.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mon/MonCommand.h b/src/mon/MonCommand.h
index 5caebfc6f134..2421229c84d6 100644
--- a/src/mon/MonCommand.h
+++ b/src/mon/MonCommand.h
@@ -22,7 +22,7 @@ struct MonCommand {
   std::string helpstring;
   std::string module;
   std::string req_perms;
-  uint64_t flags;
+  uint64_t flags = 0;
 
   // MonCommand flags
   static const uint64_t FLAG_NONE       = 0;

From d43452d57f4342a8d0bf0b38e138e92945ba4eb6 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 6 Oct 2023 13:55:21 +0000
Subject: [PATCH 1566/2492] cephadm/node-proxy: logging issues / error handling
 refactor

- fix multiple logging issue because of new handler
  added each time `Logger` is called
- do not propagate to parent (root) logger: as it makes it log the messages too
- implement a new method `is_logged()` in `RedFishClient`
- refactor the logic regarding caught errors in `RedFishClient`

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/baseclient.py       |  2 +-
 .../node_proxy/baseredfishsystem.py           | 56 +++++++--------
 .../cephadmlib/node_proxy/redfish_client.py   | 72 ++++++++++++-------
 src/cephadm/cephadmlib/node_proxy/server.py   | 16 +++--
 src/cephadm/cephadmlib/node_proxy/util.py     |  2 +
 5 files changed, 88 insertions(+), 60 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseclient.py b/src/cephadm/cephadmlib/node_proxy/baseclient.py
index 1415bc3fe194..6b46561486d5 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseclient.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseclient.py
@@ -10,7 +10,7 @@ def __init__(self,
         self.username = username
         self.password = password
 
-    def login(self) -> Dict[str, Any]:
+    def login(self) -> None:
         raise NotImplementedError()
 
     def logout(self) -> Dict[str, Any]:
diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 2e1d9087d797..b61d63a5bcc3 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -27,9 +27,8 @@ def __init__(self, **kw: Any) -> None:
         self.start_client()
 
     def start_client(self) -> None:
-        if not self.client:
-            self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
         self.client.login()
+        self.start_update_loop()
 
     def start_update_loop(self) -> None:
         self.run = True
@@ -43,31 +42,29 @@ def stop_update_loop(self) -> None:
     def update(self) -> None:
         #  this loop can have:
         #  - caching logic
-        try:
-            while self.run:
-                self.log.logger.debug("waiting for a lock.")
-                self.lock.acquire()
-                self.log.logger.debug("lock acquired.")
-                try:
-                    self._update_system()
-                    # following calls in theory can be done in parallel
-                    self._update_metadata()
-                    self._update_memory()
-                    self._update_power()
-                    self._update_fans()
-                    self._update_network()
-                    self._update_processors()
-                    self._update_storage()
-                    self.data_ready = True
-                    sleep(5)
-                finally:
-                    self.lock.release()
-                    self.log.logger.debug("lock released.")
-        # Catching 'Exception' is probably not a good idea (devel only)
-        except Exception as e:
-            self.log.logger.error(f"Error detected, logging out from redfish api.\n{e}")
-            self.client.logout()
-            raise
+        while self.run:
+            self.log.logger.debug("waiting for a lock.")
+            self.lock.acquire()
+            self.log.logger.debug("lock acquired.")
+            try:
+                self._update_system()
+                # following calls in theory can be done in parallel
+                self._update_metadata()
+                self._update_memory()
+                self._update_power()
+                self._update_fans()
+                self._update_network()
+                self._update_processors()
+                self._update_storage()
+                self.data_ready = True
+                sleep(5)
+            except RuntimeError as e:
+                self.run = False
+                self.log.logger.error(f"Error detected, trying to gracefully log out from redfish api.\n{e}")
+                self.client.logout()
+            finally:
+                self.lock.release()
+                self.log.logger.debug("lock released.")
 
     def flush(self) -> None:
         self.log.logger.info("Acquiring lock to flush data.")
@@ -83,7 +80,10 @@ def flush(self) -> None:
 
     @retry(retries=10, delay=2)
     def _get_path(self, path: str) -> Dict:
-        result = self.client.get_path(path)
+        try:
+            result = self.client.get_path(path)
+        except RuntimeError:
+            raise
         if result is None:
             self.log.logger.error(f"The client reported an error when getting path: {path}")
             raise RuntimeError(f"Could not get path: {path}")
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 437e4be131a4..439ad750a465 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -11,6 +11,7 @@
 
 class RedFishClient(BaseClient):
     PREFIX = '/redfish/v1/'
+
     def __init__(self,
                  host: str = "",
                  port: str = "443",
@@ -23,27 +24,49 @@ def __init__(self,
         self.token: Dict[str, str] = {}
         self.location: str = ''
 
-    def login(self) -> Dict[str, Any]:
-        self.log.logger.info(f"Logging in to {self.host} as '{self.username}'")
-        idrac_credentials = json.dumps({"UserName": self.username, "Password": self.password})
-        headers = {"Content-Type": "application/json"}
+    def login(self) -> None:
+        if not self.is_logged_in():
+            self.log.logger.info("Logging in to "
+                                 f"{self.host} as '{self.username}'")
+            idrac_credentials = json.dumps({"UserName": self.username,
+                                            "Password": self.password})
+            headers = {"Content-Type": "application/json"}
+
+            try:
+                _headers, _data, _status_code = self.query(data=idrac_credentials,
+                                                           headers=headers,
+                                                           endpoint='/redfish/v1/SessionService/Sessions/')
+                if _status_code != 201:
+                    self.log.logger.error(f"Can't log in to {self.host} as '{self.username}': {_status_code}")
+                    raise RuntimeError
+            except URLError as e:
+                msg = f"Can't log in to {self.host} as '{self.username}': {e}"
+                self.log.logger.error(msg)
+                raise RuntimeError
+            self.token = {"X-Auth-Token": _headers['X-Auth-Token']}
+            self.location = _headers['Location']
 
+    def is_logged_in(self) -> bool:
+        self.log.logger.debug(f"Checking token validity for {self.host}")
+        if not self.location or not self.token.get('X-Auth-Token'):
+            self.log.logger.debug(f"No token found for {self.host}.")
+            return False
+        headers = {"X-Auth-Token": self.token['X-Auth-Token']}
         try:
-            _headers, _data = self.query(data=idrac_credentials,
-                                         headers=headers,
-                                         endpoint='/redfish/v1/SessionService/Sessions/')
+            _headers, _data, _status_code = self.query(headers=headers,
+                                                       endpoint=self.location)
         except URLError as e:
-            self.log.logger.error(f"Can't log in to {self.host} as '{self.username}'.\n{e}")
-            return {}
-        self.token = {"X-Auth-Token": _headers['X-Auth-Token']}
-        self.location = _headers['Location']
-
-        return json.loads(_data)
+            self.log.logger.error("Can't check token "
+                                  f"validity for {self.host}: {e}")
+            raise RuntimeError
+        return _status_code == 200
 
     def logout(self) -> Dict[str, Any]:
         try:
-            _, _data = self.query(method='DELETE', headers=self.token, endpoint=self.location)
-        except URLError as e:
+            _, _data, _status_code = self.query(method='DELETE',
+                                                headers=self.token,
+                                                endpoint=self.location)
+        except URLError:
             self.log.logger.error(f"Can't log out from {self.host}")
             return {}
 
@@ -55,18 +78,20 @@ def get_path(self, path: str) -> Dict[str, Any]:
         if self.PREFIX not in path:
             path = f"{self.PREFIX}{path}"
         try:
-            _, result = self.query(headers=self.token, endpoint=path)
+            _, result, _status_code = self.query(headers=self.token,
+                                                 endpoint=path)
             result_json = json.loads(result)
             return result_json
         except URLError as e:
             self.log.logger.error(f"Can't get path {path}:\n{e}")
-            return {}
+            raise RuntimeError
 
     def query(self,
               data: Optional[str] = None,
               headers: Dict[str, str] = {},
               method: Optional[str] = None,
-              endpoint: str = '') -> Tuple[Dict[str, str], str]:
+              endpoint: str = '',
+              timeout: int = 10) -> Tuple[Dict[str, str], str, int]:
         url = f'{self.host}{endpoint}'
 
         # ssl_ctx = ssl.create_default_context()
@@ -76,14 +101,11 @@ def query(self,
         _data = bytes(data, 'ascii') if data else None
         try:
             req = Request(url, _data, headers=headers, method=method)
-            send_time = time.monotonic()
-            with urlopen(req, context=ssl_ctx) as response:
+            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
                 response_headers = response.headers
-                response_json = json.loads(response_str)
-                total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
-        except Exception as e:
-            self.log.logger.error(f"{e}")
+        except URLError as e:
+            self.log.logger.debug(f"{e}")
             raise
 
-        return response_headers, response_str
+        return response_headers, response_str, response.status
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 78d1e7065a96..9f7ed6c1cb6a 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -122,7 +122,7 @@ def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
 
     def POST(self) -> str:
         self.backend.start_client()
-        self.backend.start_update_loop()
+        # self.backend.start_update_loop()
         self.reporter.run()
         return 'node-proxy daemon started'
 
@@ -218,11 +218,15 @@ def main(host: str = '',
 
     # create the redfish system and the obsever
     log.logger.info("Server initialization...")
-    system = RedfishDell(host=host,
-                         username=username,
-                         password=password,
-                         system_endpoint='/Systems/System.Embedded.1',
-                         config=config)
+    try:
+        system = RedfishDell(host=host,
+                            username=username,
+                            password=password,
+                            system_endpoint='/Systems/System.Embedded.1',
+                            config=config)
+    except RuntimeError:
+        log.logger.error(f"Can't initialize the redfish system.")
+
     reporter_agent = Reporter(system, data, f"https://{mgr_target_ip}:{mgr_target_port}/node-proxy/data")
     cherrypy.config.update({
         'node_proxy': config,
diff --git a/src/cephadm/cephadmlib/node_proxy/util.py b/src/cephadm/cephadmlib/node_proxy/util.py
index 98c1a7d36712..40b01f0c7640 100644
--- a/src/cephadm/cephadmlib/node_proxy/util.py
+++ b/src/cephadm/cephadmlib/node_proxy/util.py
@@ -23,7 +23,9 @@ def get_logger(self) -> logging.Logger:
         handler.setLevel(self.level)
         fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
         handler.setFormatter(fmt)
+        logger.handlers.clear()
         logger.addHandler(handler)
+        logger.propagate = False
 
         return logger
 

From ccf9af06f372bfd0b6d5883bd28e8bd01a5088b6 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 10 Oct 2023 12:38:12 +0000
Subject: [PATCH 1567/2492] node-proxy: drop current main.py

This file was there for devel purposes.
Let's drop it as it is not used any longer.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/main.py | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/main.py

diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
deleted file mode 100644
index 45cd573192a9..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from redfish_system import RedfishSystem
-import time
-
-host = "https://x.x.x.x:8443"
-username = "myuser"
-password = "mypassword"
-
-system = RedfishSystem(host, username, password)
-system.start_update_loop()
-time.sleep(20)
-print(system.get_status())
-system.stop_update_loop()

From 40930412e95993eb6b63a39a8962b35a6e3ddcc1 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 10 Oct 2023 12:41:09 +0000
Subject: [PATCH 1568/2492] node-proxy: subclass Thread class

The idea is to subclass Thread so I can catch
exceptions in threads from the main process.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                      |  46 +++++---
 src/cephadm/cephadmlib/node_proxy/server.py | 119 ++++++++++----------
 2 files changed, 92 insertions(+), 73 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 98196c355ece..3009e4ff845b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -30,7 +30,7 @@
 from threading import Thread, Event
 from urllib.request import urlopen, Request
 from pathlib import Path
-import cephadmlib.node_proxy.server
+from cephadmlib.node_proxy.server import NodeProxy
 
 from cephadmlib.constants import (
     # default images
@@ -1473,27 +1473,32 @@ def query_endpoint(self,
             raise
         return response_str
 
-    def run(self) -> None:
-        self.pull_conf_settings()
-        ssl_ctx = ssl.create_default_context()
-        ssl_ctx.check_hostname = True
-        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        ssl_ctx.load_verify_locations(self.ca_path)
+    def init_node_proxy(self, ssl_ctx: Any) -> None:
         node_proxy_data = json.dumps({'keyring': self.keyring,
-                                      'host': self.host})
+                                          'host': self.host})
         node_proxy_data = node_proxy_data.encode('ascii')
         result = self.query_endpoint(data=node_proxy_data,
                                      endpoint='/node-proxy/idrac',
                                      ssl_ctx=ssl_ctx)
         result_json = json.loads(result)
-        t_node_proxy = Thread(target=cephadmlib.node_proxy.server.main,
-                              kwargs={'host': result_json['result']['addr'],
-                                      'username': result_json['result']['username'],
-                                      'password': result_json['result']['password'],
-                                      'data': node_proxy_data,
-                                      'mgr_target_ip': self.target_ip,
-                                      'mgr_target_port': self.target_port})
-        t_node_proxy.start()
+        kwargs = {
+            'host': result_json['result']['addr'],
+            'username': result_json['result']['username'],
+            'password': result_json['result']['password'],
+            'data': node_proxy_data,
+            'mgr_target_ip': self.target_ip,
+            'mgr_target_port': self.target_port,
+        }
+
+        self.t_node_proxy = NodeProxy(**kwargs)
+        self.t_node_proxy.start()
+
+    def run(self) -> None:
+        self.pull_conf_settings()
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = True
+        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx.load_verify_locations(self.ca_path)
 
         try:
             for _ in range(1001):
@@ -1515,7 +1520,16 @@ def run(self) -> None:
         if not self.volume_gatherer.is_alive():
             self.volume_gatherer.start()
 
+        # initiate node-proxy thread
+        self.init_node_proxy(ssl_ctx)
+
         while not self.stop:
+            try:
+                _mapper = {True: 'Ok', False: 'Critical'}
+                logger.debug(f'node-proxy status: {_mapper[self.t_node_proxy.check_status()]}')
+            except Exception as e:
+                logger.error(f'node-proxy failure: {e.__class__.__name__}: {e}')
+                self.init_node_proxy(ssl_ctx)
             start_time = time.monotonic()
             ack = self.ack
 
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/server.py
index 9f7ed6c1cb6a..25a42e19f2f3 100644
--- a/src/cephadm/cephadmlib/node_proxy/server.py
+++ b/src/cephadm/cephadmlib/node_proxy/server.py
@@ -1,8 +1,9 @@
 import cherrypy
+from threading import Thread
 from .redfish_dell import RedfishDell
 from .reporter import Reporter
 from .util import Config, Logger
-from typing import Dict
+from typing import Dict, Any, Optional
 from .basesystem import BaseSystem
 import sys
 import argparse
@@ -187,59 +188,63 @@ def GET(self) -> str:
         return 'use /system or /admin endpoints'
 
 
-def main(host: str = '',
-         username: str = '',
-         password: str = '',
-         data: str = '',
-         mgr_target_ip: str = '',
-         mgr_target_port: str = '') -> None:
-    # TODO: add a check and fail if host/username/password/data aren't passed
-
-    # parser = argparse.ArgumentParser(
-    #     prog='node-proxy',
-    # )
-    # parser.add_argument(
-    #     '--config',
-    #     dest='config',
-    #     type=str,
-    #     required=False,
-    #     default='/etc/ceph/node-proxy.yml'
-    # )
-
-    # args = parser.parse_args()
-    config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-
-    log = Logger(__name__, level=config.__dict__['logging']['level'])
-
-    host = host
-    username = username
-    password = password
-    data = json.loads(data)
-
-    # create the redfish system and the obsever
-    log.logger.info("Server initialization...")
-    try:
-        system = RedfishDell(host=host,
-                            username=username,
-                            password=password,
-                            system_endpoint='/Systems/System.Embedded.1',
-                            config=config)
-    except RuntimeError:
-        log.logger.error(f"Can't initialize the redfish system.")
-
-    reporter_agent = Reporter(system, data, f"https://{mgr_target_ip}:{mgr_target_port}/node-proxy/data")
-    cherrypy.config.update({
-        'node_proxy': config,
-        'server.socket_port': config.__dict__['server']['port']
-    })
-    c = {'/': {
-        'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
-        'request.dispatch': cherrypy.dispatch.MethodDispatcher()
-    }}
-    system.start_update_loop()
-    reporter_agent.run()
-    cherrypy.quickstart(API(system, reporter_agent, config), config=c)
-
-
-if __name__ == '__main__':
-    main()
+class NodeProxy(Thread):
+    def __init__(self, **kw: Dict[str, Any]) -> None:
+        super().__init__()
+        for k, v in kw.items():
+            setattr(self, k, v)
+        self.exc: Optional[Exception] = None
+
+    def run(self) -> None:
+        try:
+            self.main()
+        except Exception as e:
+            self.exc = e
+            return
+
+    def check_status(self) -> bool:
+        if self.__dict__.get('system') and not self.system.run:
+            raise RuntimeError("node-proxy encountered an error.")
+        if self.exc:
+            raise self.exc
+        return True
+
+    def main(self) -> None:
+        # TODO: add a check and fail if host/username/password/data aren't passed
+
+        config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
+
+        log = Logger(__name__, level=config.__dict__['logging']['level'])
+
+        self.data = json.loads(self.__dict__['data'])
+
+        # create the redfish system and the obsever
+        log.logger.info(f"Server initialization...")
+        try:
+            self.system = RedfishDell(host=self.__dict__['host'],
+                                      username=self.__dict__['username'],
+                                      password=self.__dict__['password'],
+                                      system_endpoint='/Systems/System.Embedded.1',
+                                      config=config)
+        except RuntimeError:
+            log.logger.error("Can't initialize the redfish system.")
+            raise
+
+        try:
+            reporter_agent = Reporter(self.system,
+                                      self.data,
+                                      f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
+        except RuntimeError:
+            log.logger.error("Can't initialize the reporter.")
+            raise
+
+        cherrypy.config.update({
+            'node_proxy': config,
+            'server.socket_port': config.__dict__['server']['port']
+        })
+        c = {'/': {
+            'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
+            'request.dispatch': cherrypy.dispatch.MethodDispatcher()
+        }}
+        reporter_agent.run()
+        cherrypy.quickstart(API(self.system, reporter_agent, config), config=c)

From 77beb5f333549e9e8e6157f474f6f7197daf173e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 10 Oct 2023 12:42:42 +0000
Subject: [PATCH 1569/2492] node-proxy: rename server.py -> main.py

This is going to be the entrypoint of node-proxy, let's rename
this file to main.py

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                                   | 2 +-
 src/cephadm/cephadmlib/node_proxy/{server.py => main.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename src/cephadm/cephadmlib/node_proxy/{server.py => main.py} (100%)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 3009e4ff845b..0a4d1eca2c02 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -30,7 +30,7 @@
 from threading import Thread, Event
 from urllib.request import urlopen, Request
 from pathlib import Path
-from cephadmlib.node_proxy.server import NodeProxy
+from cephadmlib.node_proxy.main import NodeProxy
 
 from cephadmlib.constants import (
     # default images
diff --git a/src/cephadm/cephadmlib/node_proxy/server.py b/src/cephadm/cephadmlib/node_proxy/main.py
similarity index 100%
rename from src/cephadm/cephadmlib/node_proxy/server.py
rename to src/cephadm/cephadmlib/node_proxy/main.py

From d46be38feb61d4d3dcf4b101db84e9a3eb410f17 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 11 Oct 2023 08:34:38 +0000
Subject: [PATCH 1570/2492] cephadm/node-proxy: reset ceph warning when needed

This makes the mgr reset the warning when the alert is fixed.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 648b3b58d49b..adcff4e67c33 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -177,8 +177,9 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
         }
 
         for component in data['data'].keys():
+            self.mgr.remove_health_warning(mapping[component])
             nok_members = self.get_nok_members(component,
-                                                data['data'])
+                                               data['data'])
 
             if nok_members:
                 count = len(nok_members)

From b6cca33c7e637abc667d091244823c23db7a1973 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 11 Oct 2023 14:50:40 +0000
Subject: [PATCH 1571/2492] node-proxy: run all update functions in parallel

This makes the update logic run faster.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../node_proxy/baseredfishsystem.py           | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index b61d63a5bcc3..850a86e933d6 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -1,3 +1,4 @@
+import concurrent.futures
 from .basesystem import BaseSystem
 from .redfish_client import RedFishClient
 from threading import Thread, Lock
@@ -48,14 +49,17 @@ def update(self) -> None:
             self.log.logger.debug("lock acquired.")
             try:
                 self._update_system()
-                # following calls in theory can be done in parallel
-                self._update_metadata()
-                self._update_memory()
-                self._update_power()
-                self._update_fans()
-                self._update_network()
-                self._update_processors()
-                self._update_storage()
+                update_funcs = [self._update_metadata,
+                                self._update_memory,
+                                self._update_power,
+                                self._update_fans,
+                                self._update_network,
+                                self._update_processors,
+                                self._update_storage]
+
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    executor.map(lambda f: f(), update_funcs)
+
                 self.data_ready = True
                 sleep(5)
             except RuntimeError as e:

From 236fa29d75fdc28df813b93f6eb2f4dfcc720517 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 11 Oct 2023 15:15:50 +0000
Subject: [PATCH 1572/2492] node-proxy: quick clean up

This removes some files which are not needed any longer.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/data.py     |  99 ------------
 .../fake_cephadm/cephadm_mgr_module.py        |  51 -------
 .../redfish_json_samples/interface_sample     |  19 ---
 .../redfish_json_samples/interfaces_sample    |  21 ---
 .../node_proxy/redfish_json_samples/memory    |  65 --------
 .../redfish_json_samples/memory_socket        |  21 ---
 .../node_proxy/redfish_json_samples/processor | 117 --------------
 .../redfish_json_samples/processors           |  13 --
 .../redfish_json_samples/storage_sample       |  19 ---
 .../node_proxy/redfish_json_samples/system    | 144 ------------------
 10 files changed, 569 deletions(-)
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/data.py
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system

diff --git a/src/cephadm/cephadmlib/node_proxy/data.py b/src/cephadm/cephadmlib/node_proxy/data.py
deleted file mode 100644
index 70339011e4ac..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/data.py
+++ /dev/null
@@ -1,99 +0,0 @@
-
-system_1 = {
-
-    'metadata': {
-        'name': 'xx',
-        'manufacturer': 'Dell',
-        'model': 'HP PowerEdge',
-        'chassis': 'xxx',
-        'xxx': '',
-    },
-
-    'status': {
-        'State': 'Enabled',
-        'Health': 'OK'
-    },
-
-    'processors': [{
-        'description': '',
-        'cores': '',
-        'threads': '',
-        'type': '',
-        'model': '',
-        'temperature': '',
-        'status': {
-            'State': 'Enabled',
-            'Health': 'OK'
-        }
-    }],
-
-    'memory': {
-        'description': '',
-        'total': 'xx',
-        'status': {
-            'State': 'Enabled',
-            'Health': 'OK'
-        },
-    },
-
-    'network': {
-        'interfaces': [
-            {
-                'type': 'ethernet',
-                'description': 'my ethertnet interface',
-                'name': 'name of the interface',
-                'description': 'description of the interface',
-                'speed_mbps': 'xxx',
-                'status': {
-                    'State': 'Enabled',
-                    'Health': 'OK'
-                },
-            }
-        ]
-    },
-
-    'storage': {
-        'drives': [
-            {
-                'device': 'devc',
-                'description': 'Milk, Cheese, Bread, Fruit, Vegetables',
-                'serial_number': 'xxxxx',
-                'location': '1I:x:y',
-                'interface_type': 'SATA',
-                'model': 'Buy groceries',
-                'type': 'ssd|rotate|nvme',
-                'capacity_bytes': '',
-                'usage_bytes': '',
-                'status': {
-                    'State': 'Enabled',
-                    'Health': 'OK'
-                },
-            }
-        ]
-    },
-
-    'power':
-        [{
-            'type': 'xx',
-            'manufacturer': 'xxx',
-            'model': 'xx',
-            'properties': {},
-            'power_control': 'xx',
-            'status': {
-                'State': 'Enabled',
-                'Health': 'OK'
-            }
-        }],
-
-    'thermal': {
-        'fans': [
-            {
-                'id': 1,
-                'status': {
-                    'State': 'Enabled',
-                    'Health': 'OK'
-                }
-            }
-        ]
-    },
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py b/src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py
deleted file mode 100644
index 6d46de40dd22..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/fake_cephadm/cephadm_mgr_module.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-"""
-License: MIT License
-Copyright (c) 2023 Miel Donkers
-
-Very simple HTTP server in python for logging requests
-Usage::
-    ./server.py [<port>]
-"""
-from http.server import BaseHTTPRequestHandler, HTTPServer
-import logging
-
-class S(BaseHTTPRequestHandler):
-    def _set_response(self):
-        self.send_response(200)
-        self.send_header('Content-type', 'text/html')
-        self.end_headers()
-
-    def do_GET(self):
-        logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
-        self._set_response()
-        self.wfile.write("GET request for {}".format(self.path).encode('utf-8'))
-
-    def do_POST(self):
-        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
-        post_data = self.rfile.read(content_length) # <--- Gets the data itself
-        logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",
-                str(self.path), str(self.headers), post_data.decode('utf-8'))
-
-        self._set_response()
-        self.wfile.write("POST request for {}".format(self.path).encode('utf-8'))
-
-def run(server_class=HTTPServer, handler_class=S, port=8000):
-    logging.basicConfig(level=logging.INFO)
-    server_address = ('', port)
-    httpd = server_class(server_address, handler_class)
-    logging.info(f'Starting httpd on port {port}...\n')
-    try:
-        httpd.serve_forever()
-    except KeyboardInterrupt:
-        pass
-    httpd.server_close()
-    logging.info('Stopping httpd...\n')
-
-if __name__ == '__main__':
-    from sys import argv
-
-    if len(argv) == 2:
-        run(port=int(argv[1]))
-    else:
-        run()
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample
deleted file mode 100644
index 6d351cfbc61e..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interface_sample
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces/Members/$entity',
-	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/',
-	'@odata.type': '#EthernetInterface.1.0.0.EthernetInterface',
-	'Id': '1',
-	'Name': 'System Ethernet Interface',
-	'Oem': {
-		'Hp': {
-			'@odata.type': '#HpiLOEthernetNetworkInterface.1.0.0.HpiLOEthernetNetworkInterface',
-			'DHCPv4': None,
-			'DHCPv6': None,
-			'IPv4': None,
-			'IPv6': None,
-			'SharedNetworkPortOptions': None
-		}
-	},
-	'SettingsResult': None,
-	'Status': None
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample
deleted file mode 100644
index 811a7720d791..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/interfaces_sample
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces',
-	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/',
-	'@odata.type': '#EthernetInterfaceCollection.EthernetInterfaceCollection',
-	'Description': 'Collection of System Network Interfaces',
-	'Members': [{
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/2/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/3/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/4/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/5/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/6/'
-	}],
-	'Members@odata.count': 6,
-	'Name': 'System Network Interfaces'
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory
deleted file mode 100644
index fba0606f750a..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory
+++ /dev/null
@@ -1,65 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Memory',
-	'@odata.id': '/redfish/v1/Systems/1/Memory/',
-	'@odata.type': '#HpMemoryCollection.HpMemoryCollection',
-	'Description': 'Memory DIMM Collection',
-	'Members': [{
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm1/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm2/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm3/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm4/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm5/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm6/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm7/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm8/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm9/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm10/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm11/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm12/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm1/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm2/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm3/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm4/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm5/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm6/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm7/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm8/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm9/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm10/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm11/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Memory/proc2dimm12/'
-	}],
-	'Members@odata.count': 24,
-	'Name': 'Memory DIMM Collection',
-	'Oem': {
-		'Hp': {
-			'@odata.type': '#HpAdvancedMemoryProtection.1.0.0.HpAdvancedMemoryProtection',
-			'AmpModeActive': 'AdvancedECC',
-			'AmpModeStatus': 'AdvancedECC',
-			'AmpModeSupported': ['AdvancedECC', 'OnlineSpareRank']
-		}
-	}
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket
deleted file mode 100644
index 283c7d41e143..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/memory_socket
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Memory/Members/$entity',
-	'@odata.id': '/redfish/v1/Systems/1/Memory/proc1dimm1/',
-	'@odata.type': '#HpMemory.1.0.0.HpMemory',
-	'DIMMStatus': 'GoodInUse',
-	'DIMMTechnology': 'RDIMM',
-	'DIMMType': 'DDR3',
-	'DataWidth': 64,
-	'ErrorCorrection': 'SingleBitECC',
-	'HPMemoryType': 'HPSmartMemory',
-	'Id': 'proc1dimm1',
-	'Manufacturer': 'HP     ',
-	'MaximumFrequencyMHz': 1600,
-	'MinimumVoltageVoltsX10': 13,
-	'Name': 'proc1dimm1',
-	'PartNumber': '713756-081          ',
-	'Rank': 2,
-	'SizeMB': 16384,
-	'SocketLocator': 'PROC  1 DIMM  1 ',
-	'TotalWidth': 72
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor
deleted file mode 100644
index bc381fb5185c..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processor
+++ /dev/null
@@ -1,117 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Processors/Members/$entity',
-	'@odata.id': '/redfish/v1/Systems/1/Processors/1/',
-	'@odata.type': '#Processor.1.0.0.Processor',
-	'Id': '1',
-	'InstructionSet': 'x86-64',
-	'Manufacturer': 'Intel',
-	'MaxSpeedMHz': 4800,
-	'Model': ' Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz      ',
-	'Name': 'Processors',
-	'Oem': {
-		'Hp': {
-			'@odata.type': '#HpProcessorExt.1.0.0.HpProcessorExt',
-			'AssetTag': '',
-			'Cache': [{
-				'Associativity': '8waySetAssociative',
-				'CacheSpeedns': 0,
-				'CurrentSRAMType': ['Burst'],
-				'EccType': 'SingleBitECC',
-				'InstalledSizeKB': 256,
-				'Location': 'Internal',
-				'MaximumSizeKB': 384,
-				'Name': 'Processor 1 Internal L1 Cache',
-				'Policy': 'WriteBack',
-				'Socketed': False,
-				'SupportedSRAMType': ['Burst'],
-				'SystemCacheType': 'Data'
-			}, {
-				'Associativity': '8waySetAssociative',
-				'CacheSpeedns': 0,
-				'CurrentSRAMType': ['Burst'],
-				'EccType': 'SingleBitECC',
-				'InstalledSizeKB': 2048,
-				'Location': 'Internal',
-				'MaximumSizeKB': 3072,
-				'Name': 'Processor 1 Internal L2 Cache',
-				'Policy': 'WriteBack',
-				'Socketed': False,
-				'SupportedSRAMType': ['Burst'],
-				'SystemCacheType': None
-			}, {
-				'Associativity': '20waySetAssociative',
-				'CacheSpeedns': 0,
-				'CurrentSRAMType': ['Burst'],
-				'EccType': 'SingleBitECC',
-				'InstalledSizeKB': 20480,
-				'Location': 'Internal',
-				'MaximumSizeKB': 30720,
-				'Name': 'Processor 1 Internal L3 Cache',
-				'Policy': 'WriteBack',
-				'Socketed': False,
-				'SupportedSRAMType': ['Burst'],
-				'SystemCacheType': None
-			}],
-			'Characteristics': ['64Bit'],
-			'ConfigStatus': {
-				'Populated': True,
-				'State': 'Enabled'
-			},
-			'CoresEnabled': 8,
-			'ExternalClockMHz': 100,
-			'MicrocodePatches': [{
-				'CpuId': '0x000206D2',
-				'Date': '2011-05-03T00:00:00Z',
-				'PatchId': '0x8000020C'
-			}, {
-				'CpuId': '0x000206D3',
-				'Date': '2011-04-20T00:00:00Z',
-				'PatchId': '0x80000304'
-			}, {
-				'CpuId': '0x000206D5',
-				'Date': '2011-10-13T00:00:00Z',
-				'PatchId': '0x00000513'
-			}, {
-				'CpuId': '0x000206D6',
-				'Date': '2018-01-30T00:00:00Z',
-				'PatchId': '0x0000061C'
-			}, {
-				'CpuId': '0x000206D7',
-				'Date': '2018-01-26T00:00:00Z',
-				'PatchId': '0x00000713'
-			}, {
-				'CpuId': '0x000306E2',
-				'Date': '2013-03-21T00:00:00Z',
-				'PatchId': '0x0000020D'
-			}, {
-				'CpuId': '0x000306E3',
-				'Date': '2013-03-21T00:00:00Z',
-				'PatchId': '0x00000308'
-			}, {
-				'CpuId': '0x000306E4',
-				'Date': '2018-01-25T00:00:00Z',
-				'PatchId': '0x0000042C'
-			}],
-			'PartNumber': '',
-			'RatedSpeedMHz': 2000,
-			'SerialNumber': '',
-			'VoltageVoltsX10': 14
-		}
-	},
-	'ProcessorArchitecture': 'x86',
-	'ProcessorId': {
-		'EffectiveFamily': '179',
-		'EffectiveModel': '14',
-		'IdentificationRegisters': '0x06e40003fbffbfeb',
-		'MicrocodeInfo': None,
-		'Step': '4',
-		'VendorId': 'Intel'
-	},
-	'ProcessorType': 'CPU',
-	'Socket': 'Proc 1',
-	'Status': {
-		'Health': 'OK'
-	},
-	'TotalCores': 8,
-	'TotalThreads': 16
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors
deleted file mode 100644
index c2fb740a4cce..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/processors
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/Processors',
-	'@odata.id': '/redfish/v1/Systems/1/Processors/',
-	'@odata.type': '#ProcessorCollection.ProcessorCollection',
-	'Description': 'Processors view',
-	'Members': [{
-		'@odata.id': '/redfish/v1/Systems/1/Processors/1/'
-	}, {
-		'@odata.id': '/redfish/v1/Systems/1/Processors/2/'
-	}],
-	'Members@odata.count': 2,
-	'Name': 'Processors Collection'
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample
deleted file mode 100644
index 6d351cfbc61e..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/storage_sample
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/1/EthernetInterfaces/Members/$entity',
-	'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/1/',
-	'@odata.type': '#EthernetInterface.1.0.0.EthernetInterface',
-	'Id': '1',
-	'Name': 'System Ethernet Interface',
-	'Oem': {
-		'Hp': {
-			'@odata.type': '#HpiLOEthernetNetworkInterface.1.0.0.HpiLOEthernetNetworkInterface',
-			'DHCPv4': None,
-			'DHCPv6': None,
-			'IPv4': None,
-			'IPv6': None,
-			'SharedNetworkPortOptions': None
-		}
-	},
-	'SettingsResult': None,
-	'Status': None
-}
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system b/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system
deleted file mode 100644
index 5bd20170b235..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_json_samples/system
+++ /dev/null
@@ -1,144 +0,0 @@
-{
-	'@odata.context': '/redfish/v1/$metadata#Systems/Members/$entity',
-	'@odata.id': '/redfish/v1/Systems/1/',
-	'@odata.type': '#ComputerSystem.1.0.1.ComputerSystem',
-	'Actions': {
-		'#ComputerSystem.Reset': {
-			'ResetType@Redfish.AllowableValues': ['On', 'ForceOff', 'ForceRestart', 'Nmi', 'PushPowerButton'],
-			'target': '/redfish/v1/Systems/1/Actions/ComputerSystem.Reset/'
-		}
-	},
-	'AssetTag': '                                ',
-	'BiosVersion': 'P71 01/22/2018',
-	'Boot': {
-		'BootSourceOverrideEnabled': 'Disabled',
-		'BootSourceOverrideSupported': ['None', 'Floppy', 'Cd', 'Hdd', 'Usb', 'Utilities', 'BiosSetup', 'Pxe'],
-		'BootSourceOverrideTarget': 'None'
-	},
-	'Description': 'Computer System View',
-	'EthernetInterfaces': {
-		'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/'
-	},
-	'HostName': 'hive1',
-	'Id': '1',
-	'IndicatorLED': 'Off',
-	'Links': {
-		'Chassis': [{
-			'@odata.id': '/redfish/v1/Chassis/1/'
-		}],
-		'ManagedBy': [{
-			'@odata.id': '/redfish/v1/Managers/1/'
-		}]
-	},
-	'LogServices': {
-		'@odata.id': '/redfish/v1/Systems/1/LogServices/'
-	},
-	'Manufacturer': 'HPE',
-	'MemorySummary': {
-		'Status': {
-			'HealthRollup': 'OK'
-		},
-		'TotalSystemMemoryGiB': 384
-	},
-	'Model': 'ProLiant DL360p Gen8',
-	'Name': 'Computer System',
-	'Oem': {
-		'Hp': {
-			'@odata.type': '#HpComputerSystemExt.1.2.2.HpComputerSystemExt',
-			'Actions': {
-				'#HpComputerSystemExt.PowerButton': {
-					'PushType@Redfish.AllowableValues': ['Press', 'PressAndHold'],
-					'target': '/redfish/v1/Systems/1/Actions/Oem/Hp/ComputerSystemExt.PowerButton/'
-				},
-				'#HpComputerSystemExt.SystemReset': {
-					'ResetType@Redfish.AllowableValues': ['ColdBoot', 'AuxCycle'],
-					'target': '/redfish/v1/Systems/1/Actions/Oem/Hp/ComputerSystemExt.SystemReset/'
-				}
-			},
-			'Bios': {
-				'Backup': {
-					'Date': '07/01/2015',
-					'Family': 'P71',
-					'VersionString': 'P71 07/01/2015'
-				},
-				'Bootblock': {
-					'Date': '03/05/2013',
-					'Family': 'P71',
-					'VersionString': 'P71 03/05/2013'
-				},
-				'Current': {
-					'Date': '01/22/2018',
-					'Family': 'P71',
-					'VersionString': 'P71 01/22/2018'
-				},
-				'UefiClass': 0
-			},
-			'DeviceDiscoveryComplete': {
-				'AMSDeviceDiscovery': 'NoAMS',
-				'DeviceDiscovery': 'vMainDeviceDiscoveryComplete',
-				'SmartArrayDiscovery': 'Complete'
-			},
-			'IntelligentProvisioningIndex': 3,
-			'IntelligentProvisioningLocation': 'System Board',
-			'IntelligentProvisioningVersion': 'N/A',
-			'Links': {
-				'BIOS': {
-					'@odata.id': '/redfish/v1/Systems/1/Bios/'
-				},
-				'EthernetInterfaces': {
-					'@odata.id': '/redfish/v1/Systems/1/EthernetInterfaces/'
-				},
-				'FirmwareInventory': {
-					'@odata.id': '/redfish/v1/Systems/1/FirmwareInventory/'
-				},
-				'Memory': {
-					'@odata.id': '/redfish/v1/Systems/1/Memory/'
-				},
-				'NetworkAdapters': {
-				'@odata.id': '/redfish/v1/Systems/1/NetworkAdapters/'
-				},
-				'PCIDevices': {
-					'@odata.id': '/redfish/v1/Systems/1/PCIDevices/'
-				},
-				'PCISlots': {
-					'@odata.id': '/redfish/v1/Systems/1/PCISlots/'
-				},
-				'SmartStorage': {
-					'@odata.id': '/redfish/v1/Systems/1/SmartStorage/'
-				},
-				'SoftwareInventory': {
-					'@odata.id': '/redfish/v1/Systems/1/SoftwareInventory/'
-				}
-			},
-			'PostState': 'FinishedPost',
-			'PowerAllocationLimit': 1500,
-			'PowerAutoOn': 'Restore',
-			'PowerOnDelay': 'Minimum',
-			'PowerRegulatorMode': 'Dynamic',
-			'PowerRegulatorModesSupported': ['OSControl', 'Dynamic', 'Max', 'Min'],
-			'TrustedModules': [{
-				'Status': 'NotPresent'
-			}],
-			'VirtualProfile': 'Inactive'
-		}
-	},
-	'PowerState': 'On',
-	'ProcessorSummary': {
-		'Count': 2,
-		'Model': ' Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz      ',
-		'Status': {
-			'HealthRollup': 'OK'
-		}
-	},
-	'Processors': {
-		'@odata.id': '/redfish/v1/Systems/1/Processors/'
-	},
-	'SKU': '654081-B21      ',
-	'SerialNumber': 'CZJ4320228      ',
-	'Status': {
-		'Health': 'Warning',
-		'State': 'Enabled'
-	},
-	'SystemType': 'Physical',
-	'UUID': '30343536-3138-5A43-4A34-333230323238'
-}
\ No newline at end of file

From b8f9a232339305c3c884b03d0dcdfdc83036ae25 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 12 Oct 2023 13:29:19 +0000
Subject: [PATCH 1573/2492] node-proxy: update the JSON data structure

Change the data structure from:
```
{
  "storage": "ok",
  "processors": "ok",
  "network": "ok",
  "memory": "ok",
  "power": "ok",
  "fans": "ok"
}
```
to:

```
{
    "host": "node1",
    "sn": "xxxx",
    "status": {
        "storage": {
        }
    }
}
```

In order to provide a unique key (sn) which is more reliable at the top
level of the dict.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        | 13 +++++---
 .../node_proxy/baseredfishsystem.py           | 16 +++++----
 .../cephadmlib/node_proxy/basesystem.py       |  7 ++++
 src/cephadm/cephadmlib/node_proxy/main.py     |  4 +--
 .../node_proxy/redfishdellsystem.py           |  3 ++
 src/cephadm/cephadmlib/node_proxy/reporter.py |  8 +++--
 src/pybind/mgr/cephadm/agent.py               | 33 +++++++++----------
 src/pybind/mgr/cephadm/inventory.py           |  8 ++---
 8 files changed, 54 insertions(+), 38 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 0a4d1eca2c02..6fdbc398b54a 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1474,10 +1474,13 @@ def query_endpoint(self,
         return response_str
 
     def init_node_proxy(self, ssl_ctx: Any) -> None:
-        node_proxy_data = json.dumps({'keyring': self.keyring,
-                                          'host': self.host})
-        node_proxy_data = node_proxy_data.encode('ascii')
-        result = self.query_endpoint(data=node_proxy_data,
+        node_proxy_meta = {
+            'cephx': {
+                'name': self.host,
+                'secret': self.keyring
+            }
+        }
+        result = self.query_endpoint(data=json.dumps(node_proxy_meta).encode('ascii'),
                                      endpoint='/node-proxy/idrac',
                                      ssl_ctx=ssl_ctx)
         result_json = json.loads(result)
@@ -1485,7 +1488,7 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
             'host': result_json['result']['addr'],
             'username': result_json['result']['username'],
             'password': result_json['result']['password'],
-            'data': node_proxy_data,
+            'cephx': node_proxy_meta['cephx'],
             'mgr_target_ip': self.target_ip,
             'mgr_target_port': self.target_port,
         }
diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 850a86e933d6..b41983000cf1 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -108,12 +108,16 @@ def build_data(self,
 
     def get_system(self) -> Dict[str, Dict[str, Dict]]:
         result = {
-            'storage': self.get_storage(),
-            'processors': self.get_processors(),
-            'network': self.get_network(),
-            'memory': self.get_memory(),
-            'power': self.get_power(),
-            'fans': self.get_fans()
+            'host': self.get_host(),
+            'sn': self.get_sn(),
+            'status': {
+                'storage': self.get_storage(),
+                'processors': self.get_processors(),
+                'network': self.get_network(),
+                'memory': self.get_memory(),
+                'power': self.get_power(),
+                'fans': self.get_fans()
+            }
         }
         return result
 
diff --git a/src/cephadm/cephadmlib/node_proxy/basesystem.py b/src/cephadm/cephadmlib/node_proxy/basesystem.py
index d3571e921f92..d853e967cd34 100644
--- a/src/cephadm/cephadmlib/node_proxy/basesystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/basesystem.py
@@ -1,3 +1,4 @@
+import socket
 from .util import Config
 from typing import Dict, Any
 from .baseclient import BaseClient
@@ -33,6 +34,12 @@ def get_network(self) -> Dict[str, Dict[str, Dict]]:
     def get_storage(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
+    def get_sn(self) -> str:
+        raise NotImplementedError()
+
+    def get_host(self) -> str:
+        return socket.gethostname()
+
     def start_update_loop(self) -> None:
         raise NotImplementedError()
 
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 25a42e19f2f3..a906a3098c16 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -216,8 +216,6 @@ def main(self) -> None:
 
         log = Logger(__name__, level=config.__dict__['logging']['level'])
 
-        self.data = json.loads(self.__dict__['data'])
-
         # create the redfish system and the obsever
         log.logger.info(f"Server initialization...")
         try:
@@ -232,7 +230,7 @@ def main(self) -> None:
 
         try:
             reporter_agent = Reporter(self.system,
-                                      self.data,
+                                      self.__dict__['cephx'],
                                       f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
         except RuntimeError:
             log.logger.error("Can't initialize the reporter.")
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index a79a3fcf52be..7d4d6f1a4a01 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -24,6 +24,9 @@ def build_system_data(self,
 
         return normalize_dict(result)
 
+    def get_sn(self) -> str:
+        return self._system['SKU']
+
     def get_status(self) -> Dict[str, Dict[str, Dict]]:
         return self._system['status']
 
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 79de9509ac86..5089dd131979 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -6,11 +6,11 @@
 
 
 class Reporter:
-    def __init__(self, system: Any, data: Dict[str, Any], observer_url: str) -> None:
+    def __init__(self, system: Any, cephx: Dict[str, Any], observer_url: str) -> None:
         self.system = system
         self.observer_url = observer_url
         self.finish = False
-        self.data = data
+        self.cephx = cephx
         self.log = Logger(__name__)
         self.log.logger.info(f'Observer url set to {self.observer_url}')
 
@@ -36,7 +36,9 @@ def loop(self) -> None:
                 self.log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
                     self.log.logger.info('data has changed since last iteration.')
-                    self.data['data'] = self.system.get_system()
+                    self.data = {}
+                    self.data['cephx'] = self.cephx
+                    self.data['patch'] = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
                         self.log.logger.info(f"sending data to {self.observer_url}")
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index adcff4e67c33..fab4f2ca8a0c 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -116,7 +116,7 @@ def idrac(self) -> Dict[str, Any]:
         if self.validate_node_proxy_data(data):
             idrac_details = self.mgr.get_store('node_proxy/idrac')
             idrac_details_json = json.loads(idrac_details)
-            results['result'] = idrac_details_json[data["host"]]
+            results['result'] = idrac_details_json[data["cephx"]["name"]]
         else:
             results['result'] = self.validate_msg
 
@@ -125,18 +125,18 @@ def idrac(self) -> Dict[str, Any]:
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
         self.validate_msg = 'valid node-proxy data received.'
         cherrypy.response.status = 200
-        if 'host' not in data:
+        if 'cephx' not in data:
             cherrypy.response.status = 400
             self.validate_msg = 'The field \'host\' must be provided.'
-        elif 'keyring' not in data:
+        elif 'secret' not in data['cephx']:
             cherrypy.response.status = 400
             self.validate_msg = 'The agent keyring must be provided.'
-        elif not self.mgr.agent_cache.agent_keys.get(data['host']):
+        elif not self.mgr.agent_cache.agent_keys.get(data['cephx']['name']):
             cherrypy.response.status = 400
-            self.validate_msg = f'Make sure the agent is running on {data["host"]}'
-        elif data['keyring'] != self.mgr.agent_cache.agent_keys[data['host']]:
+            self.validate_msg = f'Make sure the agent is running on {data["cephx"]["name"]}'
+        elif data['cephx']['secret'] != self.mgr.agent_cache.agent_keys[data['cephx']['name']]:
             cherrypy.response.status = 403
-            self.validate_msg = f'Got wrong keyring from agent on host {data["host"]}.'
+            self.validate_msg = f'Got wrong keyring from agent on host {data["cephx"]["name"]}.'
 
         return cherrypy.response.status == 200
 
@@ -145,7 +145,7 @@ def get_nok_members(self,
                         data: Dict[str, Any]) -> List[Dict[str, str]]:
         nok_members: List[Dict[str, str]] = []
 
-        for member in data[component].keys():
+        for member in data.keys():
             # Force a fake error for testing purpose
             if component == 'storage':
                 _status = 'critical'
@@ -154,9 +154,9 @@ def get_nok_members(self,
                 _status = 'critical'
                 state = "[Fake error] power supply unplugged."
             else:
-                _status = data[component][member]['status']['health'].lower()
+                _status = data[member]['status']['health'].lower()
             if _status.lower() != 'ok':
-                # state = data[component][member]['status']['state']
+                # state = data[member]['status']['state']
                 _member = dict(
                     member=member,
                     status=_status,
@@ -176,10 +176,9 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
             'fans': 'HARDWARE_FANS'
         }
 
-        for component in data['data'].keys():
+        for component in data['patch']['status'].keys():
             self.mgr.remove_health_warning(mapping[component])
-            nok_members = self.get_nok_members(component,
-                                               data['data'])
+            nok_members = self.get_nok_members(component, data['patch']['status'][component])
 
             if nok_members:
                 count = len(nok_members)
@@ -199,8 +198,8 @@ def data(self) -> Dict[str, Any]:
 
         data: Dict[str, Any] = cherrypy.request.json
         if self.validate_node_proxy_data(data):
-            host = data['host']
-            self.mgr.node_proxy.save(host, data['data'])
+            host = data['cephx']['name']
+            self.mgr.node_proxy.save(host, data['patch'])
             self.raise_alert(data)
 
         results["result"] = self.validate_msg
@@ -238,9 +237,9 @@ def dispatch(self, hostname='', cmd=''):
         except AttributeError:
             try:
                 result = self.common(**kw)
-            except RuntimeError:
+            except RuntimeError as e:
                 cherrypy.response.status = 404
-                result = {}
+                result = {"error": f"{e}"}
             return {"error": "Not a valid endpoint."}
         finally:
             return result
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 8f83dcb13089..d112af710e31 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1439,7 +1439,7 @@ def save(self,
     def fullreport(self, **kw: Any) -> Dict[str, Any]:
         hostname = kw.get('hostname')
         if hostname not in self.data.keys():
-            return self.data
+            return [self.data[h] for h in self.data.keys()]
         else:
             return self.data[hostname]
 
@@ -1456,8 +1456,8 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
 
         for host, data in results.items():
             _result[host] = {}
-            for component, details in data.items():
-                res = any([member['status']['health'].lower() != 'ok' for member in data[component].values()])
+            for component, details in data['status'].items():
+                res = any([member['status']['health'].lower() != 'ok' for member in data['status'][component].values()])
                 _result[host][component] = mapper[res]
 
         if hostname and hostname in results.keys():
@@ -1472,7 +1472,7 @@ def common(self, **kw):
 
         for host, data in self.data.items():
             try:
-                _result[host] = data[cmd]
+                _result[host] = data['status'][cmd]
             except KeyError:
                 raise RuntimeError(f'Invalid node-proxy category {cmd}')
 

From 800bd02dd698486c6f13ad9533bfd0dc7d636b7c Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 13 Oct 2023 12:09:56 +0000
Subject: [PATCH 1574/2492] node-proxy: collect firmwares details

This makes all the required changes in order to support
collecting, pushing and exposing data regarding firmwares
status and versions for all the underlying hardware.
This also refactors the redfish dell corresponding logic:
Having so many nested/inheritance classes seems unnecessary.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../node_proxy/baseredfishsystem.py           |  41 ++++---
 .../cephadmlib/node_proxy/basesystem.py       |   3 +
 src/cephadm/cephadmlib/node_proxy/main.py     |  13 +-
 .../cephadmlib/node_proxy/redfish_dell.py     |  14 ---
 .../node_proxy/redfishdellchassis.py          |  67 ----------
 .../node_proxy/redfishdellsystem.py           | 115 ++++++++++++++----
 src/pybind/mgr/cephadm/agent.py               |   6 +
 src/pybind/mgr/cephadm/inventory.py           |  19 +++
 8 files changed, 145 insertions(+), 133 deletions(-)
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish_dell.py
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index b41983000cf1..76edee0b2514 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -10,6 +10,9 @@
 class BaseRedfishSystem(BaseSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
+        self.common_endpoints: List[str] = kw.get('common_endpoints', ['/Systems/System.Embedded.1',
+                                                                       '/UpdateService'])
+        self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
         self.log = Logger(__name__)
         self.host: str = kw['host']
         self.username: str = kw['username']
@@ -25,6 +28,7 @@ def __init__(self, **kw: Any) -> None:
         self.lock: Lock = Lock()
         self.data: Dict[str, Dict[str, Any]] = {}
         self._system: Dict[str, Dict[str, Any]] = {}
+        self._sys: Dict[str, Any] = {}
         self.start_client()
 
     def start_client(self) -> None:
@@ -49,13 +53,14 @@ def update(self) -> None:
             self.log.logger.debug("lock acquired.")
             try:
                 self._update_system()
-                update_funcs = [self._update_metadata,
-                                self._update_memory,
+                self._update_sn()
+                update_funcs = [self._update_memory,
                                 self._update_power,
                                 self._update_fans,
                                 self._update_network,
                                 self._update_processors,
-                                self._update_storage]
+                                self._update_storage,
+                                self._update_firmwares]
 
                 with concurrent.futures.ThreadPoolExecutor() as executor:
                     executor.map(lambda f: f(), update_funcs)
@@ -93,18 +98,10 @@ def _get_path(self, path: str) -> Dict:
             raise RuntimeError(f"Could not get path: {path}")
         return result
 
-    def get_members(self, path: str) -> List:
-        _path = self._system[path]['@odata.id']
-        data = self._get_path(_path)
-        return [self._get_path(member['@odata.id']) for member in data['Members']]
-
-    def build_data(self,
-                   fields: List,
-                   path: str) -> Dict[str, Dict[str, Dict]]:
-        raise NotImplementedError()
-
-    # def _update_system(self) -> None:
-    #     raise NotImplementedError()
+    def get_members(self, data: Dict[str, Any], path: str) -> List:
+        _path = data[path]['@odata.id']
+        _data = self._get_path(_path)
+        return [self._get_path(member['@odata.id']) for member in _data['Members']]
 
     def get_system(self) -> Dict[str, Dict[str, Dict]]:
         result = {
@@ -117,15 +114,18 @@ def get_system(self) -> Dict[str, Dict[str, Dict]]:
                 'memory': self.get_memory(),
                 'power': self.get_power(),
                 'fans': self.get_fans()
-            }
+            },
+            'firmwares': self.get_firmwares()
         }
         return result
 
     def _update_system(self) -> None:
-        redfish_system = self.client.get_path(self.system_endpoint)
-        self._system = {**redfish_system, **self._system}
+        for endpoint in self.common_endpoints:
+            result = self.client.get_path(endpoint)
+            _endpoint = endpoint.strip('/').split('/')[0]
+            self._system[_endpoint] = result
 
-    def _update_metadata(self) -> None:
+    def _update_sn(self) -> None:
         raise NotImplementedError()
 
     def _update_memory(self) -> None:
@@ -145,3 +145,6 @@ def _update_processors(self) -> None:
 
     def _update_storage(self) -> None:
         raise NotImplementedError()
+
+    def _update_firmwares(self) -> None:
+        raise NotImplementedError()
diff --git a/src/cephadm/cephadmlib/node_proxy/basesystem.py b/src/cephadm/cephadmlib/node_proxy/basesystem.py
index d853e967cd34..6c5804063346 100644
--- a/src/cephadm/cephadmlib/node_proxy/basesystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/basesystem.py
@@ -34,6 +34,9 @@ def get_network(self) -> Dict[str, Dict[str, Dict]]:
     def get_storage(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
+    def get_firmwares(self) -> Dict[str, Dict[str, Dict]]:
+        raise NotImplementedError()
+
     def get_sn(self) -> str:
         raise NotImplementedError()
 
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index a906a3098c16..394c111bc7fb 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -1,6 +1,6 @@
 import cherrypy
 from threading import Thread
-from .redfish_dell import RedfishDell
+from .redfishdellsystem import RedfishDellSystem
 from .reporter import Reporter
 from .util import Config, Logger
 from typing import Dict, Any, Optional
@@ -211,19 +211,16 @@ def check_status(self) -> bool:
 
     def main(self) -> None:
         # TODO: add a check and fail if host/username/password/data aren't passed
-
         config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-
         log = Logger(__name__, level=config.__dict__['logging']['level'])
 
         # create the redfish system and the obsever
         log.logger.info(f"Server initialization...")
         try:
-            self.system = RedfishDell(host=self.__dict__['host'],
-                                      username=self.__dict__['username'],
-                                      password=self.__dict__['password'],
-                                      system_endpoint='/Systems/System.Embedded.1',
-                                      config=config)
+            self.system = RedfishDellSystem(host=self.__dict__['host'],
+                                            username=self.__dict__['username'],
+                                            password=self.__dict__['password'],
+                                            config=config)
         except RuntimeError:
             log.logger.error("Can't initialize the redfish system.")
             raise
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py b/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
deleted file mode 100644
index f6a01664629c..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish_dell.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from .redfishdellchassis import RedfishDellChassis
-from .redfishdellsystem import RedfishDellSystem
-from .util import Logger
-from typing import Any
-
-
-class RedfishDell(RedfishDellSystem, RedfishDellChassis):
-    def __init__(self, **kw: Any) -> None:
-        if kw.get('system_endpoint') is None:
-            kw['system_endpoint'] = '/Systems/System.Embedded.1'
-        if kw.get('chassis_endpoint') is None:
-            kw['chassis_endpoint'] = '/Chassis/System.Embedded.1'
-        super().__init__(**kw)
-        self.log = Logger(__name__)
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py b/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
deleted file mode 100644
index 3238f9e8a373..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellchassis.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from .baseredfishsystem import BaseRedfishSystem
-from .redfish_client import RedFishClient
-from threading import Thread, Lock
-from time import sleep
-from .util import Logger, retry, normalize_dict, to_snake_case
-from typing import Dict, Any, List, Union
-
-
-class RedfishDellChassis(BaseRedfishSystem):
-    def __init__(self, **kw: Any) -> None:
-        self.chassis_endpoint = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
-        super().__init__(**kw)
-        self.log = Logger(__name__)
-        self.log.logger.info(f"{__name__} initialization.")
-
-    def get_power(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['power']
-
-    def get_fans(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['fans']
-
-    def get_chassis(self) -> Dict[str, Dict[str, Dict]]:
-        result = {
-            'power': self.get_power(),
-            'fans': self.get_fans()
-        }
-        return result
-
-    def _update_power(self) -> None:
-        fields = {
-            "PowerSupplies": [
-                "Name",
-                "Model",
-                "Manufacturer",
-                "Status"
-            ]
-        }
-        self.log.logger.debug("Updating powersupplies")
-        self._system['power'] = self.build_chassis_data(fields, 'Power')
-
-    def _update_fans(self) -> None:
-        fields = {
-            "Fans": [
-                "Name",
-                "PhysicalContext",
-                "Status"
-            ],
-        }
-        self.log.logger.debug("Updating fans")
-        self._system['fans'] = self.build_chassis_data(fields, 'Thermal')
-
-    def build_chassis_data(self,
-                   fields: Dict[str, List[str]],
-                   path: str) -> Dict[str, Dict[str, Dict]]:
-        result: Dict[str, Dict[str, Dict]] = dict()
-        data = self._get_path(f"{self.chassis_endpoint}/{path}")
-
-        for elt, _fields in fields.items():
-            for member_elt in data[elt]:
-                _id = member_elt['MemberId']
-                result[_id] = dict()
-                for field in _fields:
-                    try:
-                        result[_id][to_snake_case(field)] = member_elt[field]
-                    except KeyError:
-                        self.log.logger.warning(f"Could not find field: {field} in data: {data[elt]}")
-        return normalize_dict(result)
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index 7d4d6f1a4a01..b1edba425e49 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -5,15 +5,15 @@
 
 class RedfishDellSystem(BaseRedfishSystem):
     def __init__(self, **kw: Any) -> None:
-        self.system_endpoint = kw.get('systemd_endpoint', '/Systems/System.Embedded.1')
         super().__init__(**kw)
         self.log = Logger(__name__)
 
-    def build_system_data(self,
-                   fields: List,
-                   path: str) -> Dict[str, Dict[str, Dict]]:
+    def build_common_data(self,
+                          data: Dict[str, Any],
+                          fields: List,
+                          path: str) -> Dict[str, Dict[str, Dict]]:
         result: Dict[str, Dict[str, Dict]] = dict()
-        for member_info in self.get_members(path):
+        for member_info in self.get_members(data, path):
             member_id = member_info['Id']
             result[member_id] = dict()
             for field in fields:
@@ -24,35 +24,57 @@ def build_system_data(self,
 
         return normalize_dict(result)
 
+    def build_chassis_data(self,
+                           fields: Dict[str, List[str]],
+                           path: str) -> Dict[str, Dict[str, Dict]]:
+        result: Dict[str, Dict[str, Dict]] = dict()
+        data = self._get_path(f"{self.chassis_endpoint}/{path}")
+
+        for elt, _fields in fields.items():
+            for member_elt in data[elt]:
+                _id = member_elt['MemberId']
+                result[_id] = dict()
+                for field in _fields:
+                    try:
+                        result[_id][to_snake_case(field)] = member_elt[field]
+                    except KeyError:
+                        self.log.logger.warning(f"Could not find field: {field} in data: {data[elt]}")
+        return normalize_dict(result)
+
+
     def get_sn(self) -> str:
-        return self._system['SKU']
+        return self._sys['SKU']
 
     def get_status(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['status']
-
-    def get_metadata(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['metadata']
+        return self._sys['status']
 
     def get_memory(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['memory']
+        return self._sys['memory']
 
     def get_processors(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['processors']
+        return self._sys['processors']
 
     def get_network(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['network']
+        return self._sys['network']
 
     def get_storage(self) -> Dict[str, Dict[str, Dict]]:
-        return self._system['storage']
+        return self._sys['storage']
+
+    def get_firmwares(self) -> Dict[str, Dict[str, Dict]]:
+        return self._sys['firmwares']
+
+    def get_power(self) -> Dict[str, Dict[str, Dict]]:
+        return self._sys['power']
 
-    # def _update_system(self) -> None:
-    #     redfish_system = self.client.get_path(self.system_endpoint)
-    #     self._system = {**redfish_system, **self._system}
+    def get_fans(self) -> Dict[str, Dict[str, Dict]]:
+        return self._sys['fans']
 
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
         self.log.logger.debug('Updating network')
-        self._system['network'] = self.build_system_data(fields, 'EthernetInterfaces')
+        self._sys['network'] = self.build_common_data(data=self._system['Systems'],
+                                                      fields=fields,
+                                                      path='EthernetInterfaces')
 
     def _update_processors(self) -> None:
         fields = ['Description',
@@ -63,7 +85,9 @@ def _update_processors(self) -> None:
                   'Status',
                   'Manufacturer']
         self.log.logger.debug('Updating processors')
-        self._system['processors'] = self.build_system_data(fields, 'Processors')
+        self._sys['processors'] = self.build_common_data(data=self._system['Systems'],
+                                                         fields=fields,
+                                                         path='Processors')
 
     def _update_storage(self) -> None:
         fields = ['Description',
@@ -71,7 +95,8 @@ def _update_storage(self) -> None:
                   'Model', 'Protocol',
                   'SerialNumber', 'Status',
                   'PhysicalLocation']
-        entities = self.get_members('Storage')
+        entities = self.get_members(data=self._system['Systems'],
+                                    path='Storage')
         self.log.logger.debug('Updating storage')
         result: Dict[str, Dict[str, Dict]] = dict()
         for entity in entities:
@@ -83,11 +108,11 @@ def _update_storage(self) -> None:
                 for field in fields:
                     result[drive_id][to_snake_case(field)] = drive_info[field]
                     result[drive_id]['entity'] = entity['Id']
-        self._system['storage'] = normalize_dict(result)
+        self._sys['storage'] = normalize_dict(result)
 
-    def _update_metadata(self) -> None:
-        self.log.logger.debug('Updating metadata')
-        pass
+    def _update_sn(self) -> None:
+        self.log.logger.debug('Updating serial number')
+        self._sys['SKU'] = self._system['Systems']['SKU']
 
     def _update_memory(self) -> None:
         fields = ['Description',
@@ -95,4 +120,44 @@ def _update_memory(self) -> None:
                   'CapacityMiB',
                   'Status']
         self.log.logger.debug('Updating memory')
-        self._system['memory'] = self.build_system_data(fields, 'Memory')
+        self._sys['memory'] = self.build_common_data(data=self._system['Systems'],
+                                                     fields=fields,
+                                                     path='Memory')
+
+    def _update_power(self) -> None:
+        fields = {
+            'PowerSupplies': [
+                'Name',
+                'Model',
+                'Manufacturer',
+                'Status'
+            ]
+        }
+        self.log.logger.debug('Updating powersupplies')
+        self._sys['power'] = self.build_chassis_data(fields, 'Power')
+
+    def _update_fans(self) -> None:
+        fields = {
+            'Fans': [
+                'Name',
+                'PhysicalContext',
+                'Status'
+            ],
+        }
+        self.log.logger.debug('Updating fans')
+        self._sys['fans'] = self.build_chassis_data(fields, 'Thermal')
+
+    def _update_firmwares(self) -> None:
+        fields = [
+            'Name',
+            'Description',
+            'ReleaseDate',
+            'Version',
+            'Updateable',
+            'Status',
+        ]
+        self.log.logger.debug('Updating firmwares')
+        self._sys['firmwares'] = self.build_common_data(data=self._system['UpdateService'],
+                                                        fields=fields,
+                                                        path='FirmwareInventory')
+        self.log.logger.warning(f"guits-debug1:{self._sys['firmwares']}")
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index fab4f2ca8a0c..28e1516f84f9 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -229,6 +229,12 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
     def common(self, **kw) -> Dict[str, Any]:
         return self.mgr.node_proxy.common(**kw)
 
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def firmwares(self, **kw) -> Dict[str, Any]:
+        return self.mgr.node_proxy.firmwares(**kw)
+
     def dispatch(self, hostname='', cmd=''):
         kw = dict(hostname=hostname, cmd=cmd)
         try:
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index d112af710e31..625753a8825f 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1481,6 +1481,25 @@ def common(self, **kw):
         else:
             return _result
 
+    def firmwares(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Retrieves firmware information for a specific hostname or all hosts.
+
+        If a 'hostname' is provided in the keyword arguments, retrieves firmware
+        information for that specific host. Otherwise, retrieves firmware
+        information for all available hosts.
+
+        :param kw: Keyword arguments, including 'hostname' if specified.
+        :type kw: dict
+
+        :return: A dictionary containing firmware information for each host.
+        :rtype: Dict[str, Any]
+        """
+        hostname = kw.get('hostname')
+        hosts = [hostname] if hostname else self.data.keys()
+
+        return {host: self.data[host]['firmwares'] for host in hosts}
+
     def criticals(self, **kw):
         return {}
 

From 00299b28b03dbee7fc4d9256373ce9a9d2fb50f1 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 13 Oct 2023 12:15:21 +0000
Subject: [PATCH 1575/2492] node-proxy: clean up node_proxy dir

This removes a legacy file that is not needed any longer.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/redfish-test.py     | 27 -------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/redfish-test.py

diff --git a/src/cephadm/cephadmlib/node_proxy/redfish-test.py b/src/cephadm/cephadmlib/node_proxy/redfish-test.py
deleted file mode 100644
index 3aaab2d2f097..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/redfish-test.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from redfish.rest.v1 import ServerDownOrUnreachableError
-import redfish
-import sys
-
-
-login_host = "https://x.x.x.x:8443"
-login_account = "myuser"
-login_password = "mypassword"
-
-REDFISH_OBJ = redfish.redfish_client(base_url=login_host, username=login_account, password=login_password, default_prefix='/redfish/v1/')
-
-# Login
-try:
-    REDFISH_OBJ.login(auth="session")
-except ServerDownOrUnreachableError as excp:
-    sys.stderr.write("Error: server not reachable or does not support RedFish.\n")
-    sys.exit()
-
-# Get the system information /redfish/v1/Systems/1/SmartStorage/
-# /redfish/v1/Systems/1/Processors/
-# /redfish/v1/Systems/1/Memory/proc1dimm1/
-response = REDFISH_OBJ.get(sys.argv[1])
-# Print the system information
-print(response.dict)
-
-# Logout
-REDFISH_OBJ.logout()

From 659584168ca47790151542ac1c504c87558e6138 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 19 Oct 2023 07:42:24 +0000
Subject: [PATCH 1576/2492] node-proxy: local API (NodeProxy) refactor

- subclass cherrypy._cpserver.Server,
  - drop cherrypy.quickstart() call,
  - drop nested classes approach,
- make it run over https
- print tracebacks when an exception is raised

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                    |   3 +
 src/cephadm/cephadmlib/node_proxy/main.py | 298 +++++++++++-----------
 2 files changed, 145 insertions(+), 156 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 6fdbc398b54a..ec0dca656115 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1491,6 +1491,9 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
             'cephx': node_proxy_meta['cephx'],
             'mgr_target_ip': self.target_ip,
             'mgr_target_port': self.target_port,
+            # re-use listener ssl certificate instead of generating new ones...
+            'ssl_crt_path': self.listener_cert_path,
+            'ssl_key_path': self.listener_key_path
         }
 
         self.t_node_proxy = NodeProxy(**kwargs)
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 394c111bc7fb..34f7d979fd3c 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -1,5 +1,6 @@
 import cherrypy
-from threading import Thread
+from cherrypy._cpserver import Server
+from threading import Thread, Event
 from .redfishdellsystem import RedfishDellSystem
 from .reporter import Reporter
 from .util import Config, Logger
@@ -8,6 +9,7 @@
 import sys
 import argparse
 import json
+import traceback
 
 DEFAULT_CONFIG = {
     'reporter': {
@@ -27,165 +29,116 @@
 }
 
 
-class Memory:
-    exposed = True
+@cherrypy.tools.auth_basic(on=True)
+@cherrypy.tools.allow(methods=['PUT'])
+@cherrypy.tools.json_out()
+class Admin():
+    def __init__(self, api: 'API') -> None:
+        self.api = api
 
-    def __init__(self, backend: BaseSystem) -> None:
-        self.backend = backend
-
-    @cherrypy.tools.json_out()
-    def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'memory': self.backend.get_memory()}
+    @cherrypy.expose
+    def start(self) -> Dict[str, str]:
+        self.api.backend.start_client()
+        # self.backend.start_update_loop()
+        self.api.reporter.run()
+        return {"ok": "node-proxy daemon started"}
+
+    @cherrypy.expose
+    def reload(self) -> Dict[str, str]:
+        self.api.config.reload()
+        return {"ok": "node-proxy config reloaded"}
+
+    def _stop(self) -> None:
+        self.api.backend.stop_update_loop()
+        self.api.backend.client.logout()
+        self.api.reporter.stop()
+
+    @cherrypy.expose
+    def stop(self) -> Dict[str, str]:
+        self._stop()
+        return {"ok": "node-proxy daemon stopped"}
+
+    @cherrypy.expose
+    def shutdown(self) -> Dict[str, str]:
+        self._stop()
+        cherrypy.engine.exit()
+        return {"ok": "Server shutdown."}
 
+    @cherrypy.expose
+    def flush(self) -> Dict[str, str]:
+        self.api.backend.flush()
+        return {"ok": "node-proxy data flushed"}
 
-class Network:
-    exposed = True
 
-    def __init__(self, backend: BaseSystem) -> None:
+class API(Server):
+    def __init__(self,
+                 backend: BaseSystem,
+                 reporter: Reporter,
+                 config: Config,
+                 addr: str = '0.0.0.0',
+                 port: int = 0) -> None:
+        super().__init__()
+        self.log = Logger(__name__)
         self.backend = backend
+        self.reporter = reporter
+        self.config = config
+        self.socket_port = self.config.__dict__['server']['port'] if not port else port
+        self.socket_host = addr
+        self.subscribe()
 
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'network': self.backend.get_network()}
-
-
-class Processors:
-    exposed = True
+    def memory(self) -> Dict[str, Any]:
+        return {'memory': self.backend.get_memory()}
 
-    def __init__(self, backend: BaseSystem) -> None:
-        self.backend = backend
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def network(self) -> Dict[str, Any]:
+        return {'network': self.backend.get_network()}
 
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def GET(self) -> Dict[str, Dict[str, Dict]]:
+    def processors(self) -> Dict[str, Any]:
         return {'processors': self.backend.get_processors()}
 
-
-class Storage:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem) -> None:
-        self.backend = backend
-
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def GET(self) -> Dict[str, Dict[str, Dict]]:
+    def storage(self) -> Dict[str, Any]:
         return {'storage': self.backend.get_storage()}
 
-
-class Status:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem) -> None:
-        self.backend = backend
-
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def GET(self) -> Dict[str, Dict[str, Dict]]:
-        return {'status': self.backend.get_status()}
-
-
-class System:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem) -> None:
-        self.memory = Memory(backend)
-        self.network = Network(backend)
-        self.processors = Processors(backend)
-        self.storage = Storage(backend)
-        self.status = Status(backend)
-        # actions = Actions()
-        # control = Control()
-
-
-class Shutdown:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
-        self.backend = backend
-        self.reporter = reporter
-
-    def POST(self) -> str:
-        _stop(self.backend, self.reporter)
-        cherrypy.engine.exit()
-        return 'Server shutdown...'
-
+    def power(self) -> Dict[str, Any]:
+        return {'power': self.backend.get_power()}
 
-def _stop(backend: BaseSystem, reporter: Reporter) -> None:
-    backend.stop_update_loop()
-    backend.client.logout()
-    reporter.stop()
-
-
-class Start:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
-        self.backend = backend
-        self.reporter = reporter
-
-    def POST(self) -> str:
-        self.backend.start_client()
-        # self.backend.start_update_loop()
-        self.reporter.run()
-        return 'node-proxy daemon started'
-
-
-class Stop:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem, reporter: Reporter) -> None:
-        self.backend = backend
-        self.reporter = reporter
-
-    def POST(self) -> str:
-        _stop(self.backend, self.reporter)
-        return 'node-proxy daemon stopped'
-
-
-class ConfigReload:
-    exposed = True
-
-    def __init__(self, config: Config) -> None:
-        self.config = config
-
-    def POST(self) -> str:
-        self.config.reload()
-        return 'node-proxy config reloaded'
-
-
-class Flush:
-    exposed = True
-
-    def __init__(self, backend: BaseSystem) -> None:
-        self.backend = backend
-
-    def POST(self) -> str:
-        self.backend.flush()
-        return 'node-proxy data flushed'
-
-
-class Admin:
-    exposed = False
-
-    def __init__(self, backend: BaseSystem, config: Config, reporter: Reporter) -> None:
-        self.reload = ConfigReload(config)
-        self.flush = Flush(backend)
-        self.shutdown = Shutdown(backend, reporter)
-        self.start = Start(backend, reporter)
-        self.stop = Stop(backend, reporter)
-
-
-class API:
-    exposed = True
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def fans(self) -> Dict[str, Any]:
+        return {'fans': self.backend.get_fans()}
 
-    def __init__(self,
-                 backend: BaseSystem,
-                 reporter: Reporter,
-                 config: Config) -> None:
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def firmwares(self) -> Dict[str, Any]:
+        return {'firmwares': self.backend.get_firmwares()}
 
-        self.system = System(backend)
-        self.admin = Admin(backend, config, reporter)
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_in()
+    def index(self, endpoint: str) -> Dict[str, Any]:
+        kw = dict(endpoint=endpoint)
+        result = self.common(**kw)
+        return result
 
-    def GET(self) -> str:
-        return 'use /system or /admin endpoints'
+    def stop(self) -> None:
+        self.unsubscribe()
+        super().stop()
 
 
 class NodeProxy(Thread):
@@ -194,6 +147,8 @@ def __init__(self, **kw: Dict[str, Any]) -> None:
         for k, v in kw.items():
             setattr(self, k, v)
         self.exc: Optional[Exception] = None
+        self.cp_shutdown_event = Event()
+        self.log = Logger(__name__)
 
     def run(self) -> None:
         try:
@@ -202,44 +157,75 @@ def run(self) -> None:
             self.exc = e
             return
 
+    def check_auth(self, realm: str, username: str, password: str) -> bool:
+        return self.__dict__['username'] == username and \
+            self.__dict__['password'] == password
+
     def check_status(self) -> bool:
         if self.__dict__.get('system') and not self.system.run:
             raise RuntimeError("node-proxy encountered an error.")
         if self.exc:
+            traceback.print_tb(self.exc.__traceback__)
+            self.log.logger.error(f"{self.exc.__class__.__name__}: {self.exc}")
             raise self.exc
         return True
 
+    def start_api(self) -> None:
+        cherrypy.server.unsubscribe()
+        cherrypy.engine.start()
+        self.reporter_agent.run()
+        self.cp_shutdown_event.wait()
+        self.cp_shutdown_event.clear()
+        cherrypy.engine.stop()
+        cherrypy.server.httpserver = None
+        self.log.logger.info("node-proxy shutdown.")
+
     def main(self) -> None:
         # TODO: add a check and fail if host/username/password/data aren't passed
-        config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-        log = Logger(__name__, level=config.__dict__['logging']['level'])
+        self.config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
+        self.log = Logger(__name__, level=self.config.__dict__['logging']['level'])
 
         # create the redfish system and the obsever
-        log.logger.info(f"Server initialization...")
+        self.log.logger.info(f"Server initialization...")
         try:
             self.system = RedfishDellSystem(host=self.__dict__['host'],
                                             username=self.__dict__['username'],
                                             password=self.__dict__['password'],
-                                            config=config)
+                                            config=self.config)
         except RuntimeError:
-            log.logger.error("Can't initialize the redfish system.")
+            self.log.logger.error("Can't initialize the redfish system.")
             raise
 
         try:
-            reporter_agent = Reporter(self.system,
-                                      self.__dict__['cephx'],
-                                      f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
+            self.reporter_agent = Reporter(self.system,
+                                           self.__dict__['cephx'],
+                                           f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
         except RuntimeError:
-            log.logger.error("Can't initialize the reporter.")
+            self.log.logger.error("Can't initialize the reporter.")
             raise
-
+        self.api = API(self.system,
+                       self.reporter_agent,
+                       self.config)
+        self.admin = Admin(self.api)
+        self.configure()
+        self.start_api()
+
+    def configure(self) -> None:
         cherrypy.config.update({
-            'node_proxy': config,
-            'server.socket_port': config.__dict__['server']['port']
+            'environment': 'production',
+            'engine.autoreload.on': False,
         })
-        c = {'/': {
+        config = {'/': {
             'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
-            'request.dispatch': cherrypy.dispatch.MethodDispatcher()
+            'tools.trailing_slash.on': False,
+            'tools.auth_basic.realm': 'localhost',
+            'tools.auth_basic.checkpassword': self.check_auth
         }}
-        reporter_agent.run()
-        cherrypy.quickstart(API(self.system, reporter_agent, config), config=c)
+        cherrypy.tree.mount(self.api, '/', config=config)
+        cherrypy.tree.mount(self.admin, '/admin', config=config)
+        self.api.ssl_certificate = self.__dict__['ssl_crt_path']
+        self.api.ssl_private_key = self.__dict__['ssl_key_path']
+
+    def shutdown(self) -> None:
+        self.log.logger.info("Shutting node-proxy down...")
+        self.cp_shutdown_event.set()

From b65c872b1e4de48b35c2f82cdd02dbd23dced68e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 20 Oct 2023 09:21:16 +0000
Subject: [PATCH 1577/2492] node-proxy: drop dispatch() in NodeProxy()

The current logic prevents from using any cherrypy decorators
on actual endpoints as we use a set of 'proxy functions'
(index and dispatch) instead.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py     | 55 ++++++++++++++++-------------
 src/pybind/mgr/cephadm/inventory.py |  9 +++--
 2 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 28e1516f84f9..82f289e50778 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -100,9 +100,6 @@ def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
         if len(vpath) == 2:
             hostname = vpath.pop(0)
             cherrypy.request.params['hostname'] = hostname
-        cmd = vpath.pop(0)
-        cherrypy.request.params['cmd'] = cmd
-
         return self
 
     @cherrypy.expose
@@ -224,37 +221,47 @@ def criticals(self, **kw: Any) -> Dict[str, Any]:
     def summary(self, **kw: Any) -> Dict[str, Any]:
         return self.mgr.node_proxy.summary(**kw)
 
+    @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def common(self, **kw) -> Dict[str, Any]:
-        return self.mgr.node_proxy.common(**kw)
+    def memory(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('memory', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def firmwares(self, **kw) -> Dict[str, Any]:
-        return self.mgr.node_proxy.firmwares(**kw)
+    def network(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('network', **kw)
 
-    def dispatch(self, hostname='', cmd=''):
-        kw = dict(hostname=hostname, cmd=cmd)
-        try:
-            func = getattr(self, cmd)
-            result = func(**kw)
-        except AttributeError:
-            try:
-                result = self.common(**kw)
-            except RuntimeError as e:
-                cherrypy.response.status = 404
-                result = {"error": f"{e}"}
-            return {"error": "Not a valid endpoint."}
-        finally:
-            return result
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def processors(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('processors', **kw)
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def storage(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('storage', **kw)
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def power(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('power', **kw)
 
     @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
-    def index(self, hostname=None, cmd=''):
-        result = self.dispatch(hostname, cmd)
-        return result
+    def fans(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.common('fans', **kw)
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def firmwares(self, **kw: Any) -> Dict[str, Any]:
+        return self.mgr.node_proxy.firmwares(**kw)
 
 
 class HostData(Server):
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 625753a8825f..c4e0b34f2cbf 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1465,16 +1465,15 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
         else:
             return _result
 
-    def common(self, **kw):
-        hostname = kw.get('hostname',)
-        cmd = kw.get('cmd',)
+    def common(self, endpoint: str, **kw: Any) -> Dict[str, Any]:
+        hostname = kw.get('hostname')
         _result = {}
 
         for host, data in self.data.items():
             try:
-                _result[host] = data['status'][cmd]
+                _result[host] = data['status'][endpoint]
             except KeyError:
-                raise RuntimeError(f'Invalid node-proxy category {cmd}')
+                raise RuntimeError(f'Invalid node-proxy endpoint {endpoint}')
 
         if hostname and hostname in self.data.keys():
             return _result[hostname]

From cd5141da0d5530c724614cfb148d38eb545c9003 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 20 Oct 2023 16:12:55 +0000
Subject: [PATCH 1578/2492] node-proxy: implement /led endpoint

This is the first 'act on node' feature implementation.

This adds the endpoint /led

a GET request to this endpoint returns the current status
of the enclosure LED.
a PATCH request to this endpoint allows to set the
enclosure LED status.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/basesystem.py       |   6 +
 src/cephadm/cephadmlib/node_proxy/main.py     |  30 ++++-
 .../cephadmlib/node_proxy/redfish_client.py   |  21 ++--
 .../node_proxy/redfishdellsystem.py           |  38 +++++-
 src/pybind/mgr/cephadm/agent.py               | 112 ++++++++++++++++--
 5 files changed, 180 insertions(+), 27 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/basesystem.py b/src/cephadm/cephadmlib/node_proxy/basesystem.py
index 6c5804063346..d4cda344ddbf 100644
--- a/src/cephadm/cephadmlib/node_proxy/basesystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/basesystem.py
@@ -40,6 +40,12 @@ def get_firmwares(self) -> Dict[str, Dict[str, Dict]]:
     def get_sn(self) -> str:
         raise NotImplementedError()
 
+    def get_led(self) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    def set_led(self, data: Dict[str, str]) -> int:
+        raise NotImplementedError()
+
     def get_host(self) -> str:
         return socket.gethostname()
 
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 34f7d979fd3c..8d3da58369ce 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -4,7 +4,7 @@
 from .redfishdellsystem import RedfishDellSystem
 from .reporter import Reporter
 from .util import Config, Logger
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 from .basesystem import BaseSystem
 import sys
 import argparse
@@ -128,12 +128,34 @@ def fans(self) -> Dict[str, Any]:
     def firmwares(self) -> Dict[str, Any]:
         return {'firmwares': self.backend.get_firmwares()}
 
+    def _cp_dispatch(self, vpath: List[str]) -> "API":
+        if vpath[0] == 'led':
+            if cherrypy.request.method == 'GET':
+                return self.get_led
+            if cherrypy.request.method == 'PATCH':
+                return self.set_led
+        return self
+
     @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
+    def get_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
+        return self.backend.get_led()
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['PATCH'])
     @cherrypy.tools.json_in()
-    def index(self, endpoint: str) -> Dict[str, Any]:
-        kw = dict(endpoint=endpoint)
-        result = self.common(**kw)
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.auth_basic(on=True)
+    def set_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
+        data = cherrypy.request.json
+        rc = self.backend.set_led(data)
+
+        if rc != 200:
+            cherrypy.response.status = rc
+            result = {"state": f"error: please, verify the data you sent."}
+        else:
+            result = {"state": data["state"].lower()}
         return result
 
     def stop(self) -> None:
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 439ad750a465..f7ec01ae5b42 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -21,7 +21,7 @@ def __init__(self,
         self.log: Logger = Logger(__name__)
         self.log.logger.info(f"Initializing redfish client {__name__}")
         self.host: str = f"https://{host}:{port}"
-        self.token: Dict[str, str] = {}
+        self.token: str = ''
         self.location: str = ''
 
     def login(self) -> None:
@@ -43,15 +43,15 @@ def login(self) -> None:
                 msg = f"Can't log in to {self.host} as '{self.username}': {e}"
                 self.log.logger.error(msg)
                 raise RuntimeError
-            self.token = {"X-Auth-Token": _headers['X-Auth-Token']}
+            self.token = _headers['X-Auth-Token']
             self.location = _headers['Location']
 
     def is_logged_in(self) -> bool:
         self.log.logger.debug(f"Checking token validity for {self.host}")
-        if not self.location or not self.token.get('X-Auth-Token'):
+        if not self.location or not self.token:
             self.log.logger.debug(f"No token found for {self.host}.")
             return False
-        headers = {"X-Auth-Token": self.token['X-Auth-Token']}
+        headers = {"X-Auth-Token": self.token}
         try:
             _headers, _data, _status_code = self.query(headers=headers,
                                                        endpoint=self.location)
@@ -64,7 +64,7 @@ def is_logged_in(self) -> bool:
     def logout(self) -> Dict[str, Any]:
         try:
             _, _data, _status_code = self.query(method='DELETE',
-                                                headers=self.token,
+                                                headers={"X-Auth-Token": self.token},
                                                 endpoint=self.location)
         except URLError:
             self.log.logger.error(f"Can't log out from {self.host}")
@@ -78,8 +78,7 @@ def get_path(self, path: str) -> Dict[str, Any]:
         if self.PREFIX not in path:
             path = f"{self.PREFIX}{path}"
         try:
-            _, result, _status_code = self.query(headers=self.token,
-                                                 endpoint=path)
+            _, result, _status_code = self.query(endpoint=path)
             result_json = json.loads(result)
             return result_json
         except URLError as e:
@@ -93,14 +92,18 @@ def query(self,
               endpoint: str = '',
               timeout: int = 10) -> Tuple[Dict[str, str], str, int]:
         url = f'{self.host}{endpoint}'
-
+        _headers = headers.copy() if headers else {}
+        if self.token:
+            _headers['X-Auth-Token'] = self.token
+        if not _headers.get('Content-Type') and method in ['POST', 'PUT', 'PATCH']:
+            _headers['Content-Type'] = 'application/json'
         # ssl_ctx = ssl.create_default_context()
         # ssl_ctx.check_hostname = True
         # ssl_ctx.verify_mode = ssl.CERT_REQUIRED
         ssl_ctx = ssl._create_unverified_context()
         _data = bytes(data, 'ascii') if data else None
         try:
-            req = Request(url, _data, headers=headers, method=method)
+            req = Request(url, _data, headers=_headers, method=method)
             with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
                 response_headers = response.headers
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index b1edba425e49..9ae370ea4a6d 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -1,3 +1,4 @@
+import json
 from .baseredfishsystem import BaseRedfishSystem
 from .util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any, List
@@ -69,6 +70,42 @@ def get_power(self) -> Dict[str, Dict[str, Dict]]:
     def get_fans(self) -> Dict[str, Dict[str, Dict]]:
         return self._sys['fans']
 
+    def get_led(self) -> Dict[str, Dict[str, Dict]]:
+        endpoint = f"/redfish/v1/{self.chassis_endpoint}"
+        result = self.client.query(method='GET',
+                                   endpoint=endpoint,
+                                   timeout=10)
+        response_json = json.loads(result[1])
+        mapper = {
+                'true': 'on',
+                'false': 'off'
+        }
+        if result[2] == 200:
+            return {"state": mapper[str(response_json['LocationIndicatorActive']).lower()]}
+        else:
+            return {"error": "Couldn't retrieve enclosure LED status."}
+
+    def set_led(self, data: Dict[str, str]) -> int:
+        # '{"IndicatorLED": "Lit"}'      -> LocationIndicatorActive = false
+        # '{"IndicatorLED": "Blinking"}' -> LocationIndicatorActive = true
+        mapper = {
+            "on": 'Blinking',
+            "off": 'Lit'
+        }
+        try:
+            _data = {
+                "IndicatorLED": mapper[data["state"].lower()]
+            }
+            _, response, status = self.client.query(
+                data=json.dumps(_data),
+                method='PATCH',
+                endpoint=f"/redfish/v1{self.chassis_endpoint}"
+            )
+        except KeyError:
+            status = 400
+        result = status
+        return result
+
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
         self.log.logger.debug('Updating network')
@@ -160,4 +197,3 @@ def _update_firmwares(self) -> None:
         self._sys['firmwares'] = self.build_common_data(data=self._system['UpdateService'],
                                                         fields=fields,
                                                         path='FirmwareInventory')
-        self.log.logger.warning(f"guits-debug1:{self._sys['firmwares']}")
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 82f289e50778..f2ae92a603fc 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -13,6 +13,7 @@ class Server:  # type: ignore
 import tempfile
 import threading
 import time
+import base64
 
 from orchestrator import DaemonDescriptionStatus
 from orchestrator._interface import daemon_type_to_service
@@ -22,8 +23,10 @@ class Server:  # type: ignore
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
 from cephadm.ssl_cert_utils import SSLCerts
 from mgr_util import test_port_allocation, PortAlreadyInUse
+from urllib.request import urlopen, Request
+from urllib.error import HTTPError, URLError
 
-from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional
+from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -86,8 +89,8 @@ def find_free_port(self) -> None:
 
     def configure(self) -> None:
         self.host_data = HostData(self.mgr, self.server_port, self.server_addr)
-        self.node_proxy = NodeProxy(self.mgr)
         self.configure_tls(self.host_data)
+        self.node_proxy = NodeProxy(self.mgr)
         self.configure_routes()
         self.find_free_port()
 
@@ -95,6 +98,11 @@ def configure(self) -> None:
 class NodeProxy:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
+        self.ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        self.ssl_ctx = ssl.create_default_context()
+        self.ssl_ctx.check_hostname = True
+        self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        self.ssl_ctx.load_verify_locations(cadata=self.ssl_root_crt)
 
     def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
         if len(vpath) == 2:
@@ -122,18 +130,25 @@ def idrac(self) -> Dict[str, Any]:
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
         self.validate_msg = 'valid node-proxy data received.'
         cherrypy.response.status = 200
-        if 'cephx' not in data:
-            cherrypy.response.status = 400
-            self.validate_msg = 'The field \'host\' must be provided.'
-        elif 'secret' not in data['cephx']:
-            cherrypy.response.status = 400
-            self.validate_msg = 'The agent keyring must be provided.'
-        elif not self.mgr.agent_cache.agent_keys.get(data['cephx']['name']):
+        try:
+            if 'cephx' not in data.keys():
+                cherrypy.response.status = 400
+                self.validate_msg = 'The field \'cephx\' must be provided.'
+            elif 'name' not in data['cephx'].keys():
+                cherrypy.response.status = 400
+                self.validate_msg = 'The field \'host\' must be provided.'
+            elif 'secret' not in data['cephx'].keys():
+                cherrypy.response.status = 400
+                self.validate_msg = 'The agent keyring must be provided.'
+            elif not self.mgr.agent_cache.agent_keys.get(data['cephx']['name']):
+                cherrypy.response.status = 400
+                self.validate_msg = f'Make sure the agent is running on {data["cephx"]["name"]}'
+            elif data['cephx']['secret'] != self.mgr.agent_cache.agent_keys[data['cephx']['name']]:
+                cherrypy.response.status = 403
+                self.validate_msg = f'Got wrong keyring from agent on host {data["cephx"]["name"]}.'
+        except AttributeError:
             cherrypy.response.status = 400
-            self.validate_msg = f'Make sure the agent is running on {data["cephx"]["name"]}'
-        elif data['cephx']['secret'] != self.mgr.agent_cache.agent_keys[data['cephx']['name']]:
-            cherrypy.response.status = 403
-            self.validate_msg = f'Got wrong keyring from agent on host {data["cephx"]["name"]}.'
+            self.validate_msg = 'Malformed data received.'
 
         return cherrypy.response.status == 200
 
@@ -203,6 +218,77 @@ def data(self) -> Dict[str, Any]:
 
         return results
 
+    def query_endpoint(self,
+                       addr: str = '',
+                       port: str = '',
+                       method: Optional[str] = None,
+                       headers: Optional[Dict[str, str]] = {},
+                       data: Optional[bytes] = None,
+                       endpoint: str = '',
+                       ssl_ctx: Optional[Any] = None) -> Tuple[int, Dict[str, Any]]:
+        url = f'https://{addr}:{port}{endpoint}'
+        _headers = headers
+        response_json = {}
+        if not _headers.get('Content-Type'):
+            # default to application/json if nothing provided
+            _headers['Content-Type'] = 'application/json'
+        _data = bytes(data, 'ascii') if data else None
+        try:
+            req = Request(url, _data, _headers, method=method)
+            with urlopen(req, context=ssl_ctx) as response:
+                response_str = response.read()
+                response_json = json.loads(response_str)
+                response_status = response.status
+        except HTTPError as e:
+            self.mgr.log.debug(f"{e.code} {e.reason}")
+            response_status = e.code
+        except URLError as e:
+            self.mgr.log.debug(f"{e.reason}")
+            raise
+        except Exception as e:
+            self.mgr.log.error(f"{e}")
+            raise
+        return (response_status, response_json)
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET', 'PATCH'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    def led(self, **kw: Any) -> Dict[str, Any]:
+        method: str = cherrypy.request.method
+        hostname: Optional[str] = kw.get('hostname')
+        headers: Dict[str, str] = {}
+
+        if not hostname:
+            msg: str = "listing enclosure LED status for all nodes is not implemented."
+            self.mgr.log.debug(msg)
+            raise cherrypy.HTTPError(501, msg)
+
+        addr = self.mgr.inventory.get_addr(hostname)
+
+        if method == 'PATCH':
+            # TODO(guits): need to check the request is authorized
+            # allowing a specific keyring only ? (client.admin or client.agent.. ?)
+            data: str = json.dumps(cherrypy.request.json)
+            username = self.mgr.node_proxy.idrac[hostname]['username']
+            password = self.mgr.node_proxy.idrac[hostname]['password']
+            auth = f"{username}:{password}".encode("utf-8")
+            auth = base64.b64encode(auth).decode("utf-8")
+            headers = {"Authorization": f"Basic {auth}"}
+
+        try:
+            status, result = self.query_endpoint(data=data,
+                                                headers=headers,
+                                                addr=addr,
+                                                method=method,
+                                                port=8080,
+                                                endpoint='/led',
+                                                ssl_ctx=self.ssl_ctx)
+        except URLError as e:
+            raise cherrypy.HTTPError(502, f"{e}")
+        cherrypy.response.status = status
+        return result
+
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()

From f5fc77ec7d3500346518f1d011a899e10ca83167 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 23 Oct 2023 13:42:09 +0000
Subject: [PATCH 1579/2492] cephadm: add new option to CLI

this adds the `--deploy-cephadm-agent` option to the cephadm
CLI's bootstrap subcommand.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index ec0dca656115..e65e837d9f3d 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -2969,6 +2969,10 @@ def mgr_has_latest_epoch():
                 'For more information see:\n\n'
                 '\thttps://docs.ceph.com/en/latest/mgr/telemetry/\n')
     logger.info('Bootstrap complete.')
+
+    if getattr(ctx, 'deploy_cephadm_agent', None):
+        cli(['config', 'set', 'mgr', 'mgr/cephadm/use_agent', 'true'])
+
     return ctx.error_code
 
 ##################################
@@ -5328,6 +5332,10 @@ def _get_parser():
         '--log-to-file',
         action='store_true',
         help='configure cluster to log to traditional log files in /var/log/ceph/$fsid')
+    parser_bootstrap.add_argument(
+        '--deploy-cephadm-agent',
+        action='store_true',
+        help='deploy the cephadm-agent')
 
     parser_deploy = subparsers.add_parser(
         'deploy', help='deploy a daemon')

From 6276c68877eac8547f6cba23bb20bb725d17529d Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 23 Oct 2023 15:28:35 +0000
Subject: [PATCH 1580/2492] node-proxy: parametrize idrac port

This adds the missing piece to make the idrac port
a parameter that one can customize.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                                 | 2 ++
 src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py | 3 ++-
 src/cephadm/cephadmlib/node_proxy/main.py              | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index e65e837d9f3d..8ee75cb65203 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1495,6 +1495,8 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
             'ssl_crt_path': self.listener_cert_path,
             'ssl_key_path': self.listener_key_path
         }
+        if result_json['result'].get('port'):
+            kwargs['port'] = result_json['result']['port']
 
         self.t_node_proxy = NodeProxy(**kwargs)
         self.t_node_proxy.start()
diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 76edee0b2514..ccf61e9cf879 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -15,10 +15,11 @@ def __init__(self, **kw: Any) -> None:
         self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
         self.log = Logger(__name__)
         self.host: str = kw['host']
+        self.port: int = kw['port']
         self.username: str = kw['username']
         self.password: str = kw['password']
         # move the following line (class attribute?)
-        self.client = RedFishClient(host=self.host, username=self.username, password=self.password)
+        self.client = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
         self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
 
         self.run: bool = False
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 8d3da58369ce..cd36639bd1fe 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -211,6 +211,7 @@ def main(self) -> None:
         self.log.logger.info(f"Server initialization...")
         try:
             self.system = RedfishDellSystem(host=self.__dict__['host'],
+                                            port=self.__dict__.get('port', 443),
                                             username=self.__dict__['username'],
                                             password=self.__dict__['password'],
                                             config=self.config)

From ac654eb43c7a183d6df085f9f316ac61819139c3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 24 Oct 2023 08:43:53 +0000
Subject: [PATCH 1581/2492] node-proxy: fetch idrac details from
 NodeProxyCache()

The class ` NodeProxyCache()` is intended for that, it already
has this information so there's no need to make a call to `get_store()`
each time we want to access idrac details.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index f2ae92a603fc..86726728bccb 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -119,9 +119,8 @@ def idrac(self) -> Dict[str, Any]:
         results: Dict[str, Any] = {}
 
         if self.validate_node_proxy_data(data):
-            idrac_details = self.mgr.get_store('node_proxy/idrac')
-            idrac_details_json = json.loads(idrac_details)
-            results['result'] = idrac_details_json[data["cephx"]["name"]]
+            host = data["cephx"]["name"]
+            results['result'] = self.mgr.node_proxy.idrac.get(host)
         else:
             results['result'] = self.validate_msg
 

From 8343968af89c30bb6005fdbe640520d78258356b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 24 Oct 2023 11:28:11 +0000
Subject: [PATCH 1582/2492] node-proxy: address mypy and flake8 errors

This addresses some flake8 and python typing errors.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py | 4 ++--
 src/cephadm/cephadmlib/node_proxy/basesystem.py        | 5 ++++-
 src/cephadm/cephadmlib/node_proxy/main.py              | 3 ---
 src/cephadm/cephadmlib/node_proxy/redfish_client.py    | 8 +++-----
 src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py | 7 +++----
 5 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index ccf61e9cf879..c4675e5b8f03 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -19,7 +19,7 @@ def __init__(self, **kw: Any) -> None:
         self.username: str = kw['username']
         self.password: str = kw['password']
         # move the following line (class attribute?)
-        self.client = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
+        self.client: RedFishClient = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
         self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
 
         self.run: bool = False
@@ -104,7 +104,7 @@ def get_members(self, data: Dict[str, Any], path: str) -> List:
         _data = self._get_path(_path)
         return [self._get_path(member['@odata.id']) for member in _data['Members']]
 
-    def get_system(self) -> Dict[str, Dict[str, Dict]]:
+    def get_system(self) -> Dict[str, Any]:
         result = {
             'host': self.get_host(),
             'sn': self.get_sn(),
diff --git a/src/cephadm/cephadmlib/node_proxy/basesystem.py b/src/cephadm/cephadmlib/node_proxy/basesystem.py
index d4cda344ddbf..4fb1b7b85534 100644
--- a/src/cephadm/cephadmlib/node_proxy/basesystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/basesystem.py
@@ -10,7 +10,7 @@ def __init__(self, **kw: Any) -> None:
         self.config: Config = kw['config']
         self.client: BaseClient
 
-    def get_system(self) -> Dict[str, Dict[str, Dict]]:
+    def get_system(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
     def get_status(self) -> Dict[str, Dict[str, Dict]]:
@@ -25,6 +25,9 @@ def get_processors(self) -> Dict[str, Dict[str, Dict]]:
     def get_memory(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
+    def get_fans(self) -> Dict[str, Dict[str, Dict]]:
+        raise NotImplementedError()
+
     def get_power(self) -> Dict[str, Dict[str, Dict]]:
         raise NotImplementedError()
 
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index cd36639bd1fe..82f941a08512 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -6,9 +6,6 @@
 from .util import Config, Logger
 from typing import Dict, Any, Optional, List
 from .basesystem import BaseSystem
-import sys
-import argparse
-import json
 import traceback
 
 DEFAULT_CONFIG = {
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index f7ec01ae5b42..3046a634c58e 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -1,8 +1,6 @@
-import time
-import datetime
 import ssl
 import json
-from urllib.error import HTTPError, URLError
+from urllib.error import URLError
 from urllib.request import urlopen, Request
 from .baseclient import BaseClient
 from .util import Logger
@@ -14,13 +12,13 @@ class RedFishClient(BaseClient):
 
     def __init__(self,
                  host: str = "",
-                 port: str = "443",
+                 port: int = 443,
                  username: str = "",
                  password: str = ""):
         super().__init__(host, username, password)
         self.log: Logger = Logger(__name__)
         self.log.logger.info(f"Initializing redfish client {__name__}")
-        self.host: str = f"https://{host}:{port}"
+        self.host: str = f"https://{host}:{str(port)}"
         self.token: str = ''
         self.location: str = ''
 
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index 9ae370ea4a6d..516272c7223e 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -42,7 +42,6 @@ def build_chassis_data(self,
                         self.log.logger.warning(f"Could not find field: {field} in data: {data[elt]}")
         return normalize_dict(result)
 
-
     def get_sn(self) -> str:
         return self._sys['SKU']
 
@@ -70,15 +69,15 @@ def get_power(self) -> Dict[str, Dict[str, Dict]]:
     def get_fans(self) -> Dict[str, Dict[str, Dict]]:
         return self._sys['fans']
 
-    def get_led(self) -> Dict[str, Dict[str, Dict]]:
+    def get_led(self) -> Dict[str, Any]:
         endpoint = f"/redfish/v1/{self.chassis_endpoint}"
         result = self.client.query(method='GET',
                                    endpoint=endpoint,
                                    timeout=10)
         response_json = json.loads(result[1])
         mapper = {
-                'true': 'on',
-                'false': 'off'
+            'true': 'on',
+            'false': 'off'
         }
         if result[2] == 200:
             return {"state": mapper[str(response_json['LocationIndicatorActive']).lower()]}

From c4875f4cf4dab2841ac1a138ed3b564cd827a77c Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 25 Oct 2023 15:07:09 +0000
Subject: [PATCH 1583/2492] node-proxy: implement http_query() helper function

so we can drop the dependency to `requests` and
use same helper function from both reporter.py and redfish_client.py

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadmlib/node_proxy/main.py     |  5 +-
 .../cephadmlib/node_proxy/redfish_client.py   | 52 +++++++++----------
 src/cephadm/cephadmlib/node_proxy/reporter.py | 39 ++++++++++----
 src/cephadm/cephadmlib/node_proxy/util.py     | 40 +++++++++++++-
 4 files changed, 97 insertions(+), 39 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 82f941a08512..e9c99f0983dc 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -219,7 +219,10 @@ def main(self) -> None:
         try:
             self.reporter_agent = Reporter(self.system,
                                            self.__dict__['cephx'],
-                                           f"https://{self.__dict__['mgr_target_ip']}:{self.__dict__['mgr_target_port']}/node-proxy/data")
+                                           reporter_scheme=self.__dict__.get('reporter_scheme', 'https'),
+                                           reporter_hostname=self.__dict__['mgr_target_ip'],
+                                           reporter_port=self.__dict__['mgr_target_port'],
+                                           reporter_endpoint=self.__dict__.get('reporter_endpoint', '/node-proxy/data'))
         except RuntimeError:
             self.log.logger.error("Can't initialize the reporter.")
             raise
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 3046a634c58e..4107fb57c8f0 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -1,10 +1,9 @@
-import ssl
 import json
-from urllib.error import URLError
-from urllib.request import urlopen, Request
+from urllib.error import HTTPError, URLError
 from .baseclient import BaseClient
-from .util import Logger
+from .util import Logger, http_req
 from typing import Dict, Any, Tuple, Optional
+from http.client import HTTPMessage
 
 
 class RedFishClient(BaseClient):
@@ -18,14 +17,16 @@ def __init__(self,
         super().__init__(host, username, password)
         self.log: Logger = Logger(__name__)
         self.log.logger.info(f"Initializing redfish client {__name__}")
-        self.host: str = f"https://{host}:{str(port)}"
+        self.host: str = host
+        self.port: int = port
+        self.url: str = f"https://{self.host}:{self.port}"
         self.token: str = ''
         self.location: str = ''
 
     def login(self) -> None:
         if not self.is_logged_in():
             self.log.logger.info("Logging in to "
-                                 f"{self.host} as '{self.username}'")
+                                 f"{self.url} as '{self.username}'")
             idrac_credentials = json.dumps({"UserName": self.username,
                                             "Password": self.password})
             headers = {"Content-Type": "application/json"}
@@ -35,19 +36,19 @@ def login(self) -> None:
                                                            headers=headers,
                                                            endpoint='/redfish/v1/SessionService/Sessions/')
                 if _status_code != 201:
-                    self.log.logger.error(f"Can't log in to {self.host} as '{self.username}': {_status_code}")
+                    self.log.logger.error(f"Can't log in to {self.url} as '{self.username}': {_status_code}")
                     raise RuntimeError
             except URLError as e:
-                msg = f"Can't log in to {self.host} as '{self.username}': {e}"
+                msg = f"Can't log in to {self.url} as '{self.username}': {e}"
                 self.log.logger.error(msg)
                 raise RuntimeError
             self.token = _headers['X-Auth-Token']
             self.location = _headers['Location']
 
     def is_logged_in(self) -> bool:
-        self.log.logger.debug(f"Checking token validity for {self.host}")
+        self.log.logger.debug(f"Checking token validity for {self.url}")
         if not self.location or not self.token:
-            self.log.logger.debug(f"No token found for {self.host}.")
+            self.log.logger.debug(f"No token found for {self.url}.")
             return False
         headers = {"X-Auth-Token": self.token}
         try:
@@ -55,7 +56,7 @@ def is_logged_in(self) -> bool:
                                                        endpoint=self.location)
         except URLError as e:
             self.log.logger.error("Can't check token "
-                                  f"validity for {self.host}: {e}")
+                                  f"validity for {self.url}: {e}")
             raise RuntimeError
         return _status_code == 200
 
@@ -65,7 +66,7 @@ def logout(self) -> Dict[str, Any]:
                                                 headers={"X-Auth-Token": self.token},
                                                 endpoint=self.location)
         except URLError:
-            self.log.logger.error(f"Can't log out from {self.host}")
+            self.log.logger.error(f"Can't log out from {self.url}")
             return {}
 
         response_str = _data
@@ -88,25 +89,24 @@ def query(self,
               headers: Dict[str, str] = {},
               method: Optional[str] = None,
               endpoint: str = '',
-              timeout: int = 10) -> Tuple[Dict[str, str], str, int]:
-        url = f'{self.host}{endpoint}'
+              timeout: int = 10) -> Tuple[HTTPMessage, str, int]:
         _headers = headers.copy() if headers else {}
         if self.token:
             _headers['X-Auth-Token'] = self.token
         if not _headers.get('Content-Type') and method in ['POST', 'PUT', 'PATCH']:
             _headers['Content-Type'] = 'application/json'
-        # ssl_ctx = ssl.create_default_context()
-        # ssl_ctx.check_hostname = True
-        # ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        ssl_ctx = ssl._create_unverified_context()
-        _data = bytes(data, 'ascii') if data else None
         try:
-            req = Request(url, _data, headers=_headers, method=method)
-            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
-                response_str = response.read()
-                response_headers = response.headers
-        except URLError as e:
+            (response_headers,
+             response_str,
+             response_status) = http_req(hostname=self.host,
+                                         port=self.port,
+                                         endpoint=endpoint,
+                                         headers=_headers,
+                                         method=method,
+                                         data=data,
+                                         timeout=timeout)
+
+            return response_headers, response_str, response_status
+        except (HTTPError, URLError) as e:
             self.log.logger.debug(f"{e}")
             raise
-
-        return response_headers, response_str, response.status
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 5089dd131979..aa0ecdf93f98 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -1,18 +1,32 @@
 from threading import Thread
-import requests
 import time
-from .util import Logger
+import json
+from .util import Logger, http_req
+from urllib.error import HTTPError, URLError
 from typing import Dict, Any
 
 
 class Reporter:
-    def __init__(self, system: Any, cephx: Dict[str, Any], observer_url: str) -> None:
+    def __init__(self,
+                 system: Any,
+                 cephx: Dict[str, Any],
+                 reporter_scheme: str = 'https',
+                 reporter_hostname: str = '',
+                 reporter_port: int = 443,
+                 reporter_endpoint: str = '/node-proxy/data') -> None:
         self.system = system
-        self.observer_url = observer_url
+        self.data: Dict[str, Any] = {}
         self.finish = False
         self.cephx = cephx
+        self.data['cephx'] = self.cephx
+        self.reporter_scheme: str = reporter_scheme
+        self.reporter_hostname: str = reporter_hostname
+        self.reporter_port: int = reporter_port
+        self.reporter_endpoint: str = reporter_endpoint
         self.log = Logger(__name__)
-        self.log.logger.info(f'Observer url set to {self.observer_url}')
+        self.reporter_url: str = (f"{reporter_scheme}:{reporter_hostname}:"
+                                  f"{reporter_port}{reporter_endpoint}")
+        self.log.logger.info(f'Reporter url set to {self.reporter_url}')
 
     def stop(self) -> None:
         self.finish = True
@@ -36,15 +50,18 @@ def loop(self) -> None:
                 self.log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
                     self.log.logger.info('data has changed since last iteration.')
-                    self.data = {}
-                    self.data['cephx'] = self.cephx
                     self.data['patch'] = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
-                        self.log.logger.info(f"sending data to {self.observer_url}")
-                        r = requests.post(f"{self.observer_url}", json=self.data, timeout=5, verify=False)
-                    except (requests.exceptions.RequestException,
-                            requests.exceptions.ConnectionError) as e:
+                        self.log.logger.info(f"sending data to {self.reporter_url}")
+                        http_req(hostname=self.reporter_hostname,
+                                 port=self.reporter_port,
+                                 method='POST',
+                                 headers={'Content-Type': 'application/json'},
+                                 endpoint=self.reporter_endpoint,
+                                 scheme=self.reporter_scheme,
+                                 data=json.dumps(self.data))
+                    except (HTTPError, URLError) as e:
                         self.log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
                         # Need to add a new parameter 'max_retries' to the reporter if it can't
                         # send the data for more than x times, maybe the daemon should stop altogether
diff --git a/src/cephadm/cephadmlib/node_proxy/util.py b/src/cephadm/cephadmlib/node_proxy/util.py
index 40b01f0c7640..da46ebabda0d 100644
--- a/src/cephadm/cephadmlib/node_proxy/util.py
+++ b/src/cephadm/cephadmlib/node_proxy/util.py
@@ -3,7 +3,10 @@
 import os
 import time
 import re
-from typing import Dict, List, Callable, Any
+import ssl
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen, Request
+from typing import Dict, List, Callable, Any, Optional, MutableMapping, Tuple
 
 
 class Logger:
@@ -98,3 +101,38 @@ def _retry(*args: str, **kwargs: Any) -> Callable:
             return f(*args, **kwargs)
         return _retry
     return decorator
+
+
+def http_req(hostname: str = '',
+             port: int = 443,
+             method: Optional[str] = None,
+             headers: MutableMapping[str, str] = {},
+             data: Optional[str] = None,
+             endpoint: str = '/',
+             scheme: str = 'https',
+             ssl_verify: bool = False,
+             timeout: Optional[int] = None,
+             ssl_ctx: Optional[Any] = None) -> Tuple[Any, Any, Any]:
+
+    if not ssl_ctx:
+        ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        if not ssl_verify:
+            ssl_ctx.check_hostname = False
+            ssl_ctx.verify_mode = ssl.CERT_NONE
+        else:
+            ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+
+    url: str = f'{scheme}://{hostname}:{str(port)}{endpoint}'
+    _data = bytes(data, 'ascii') if data else None
+
+    try:
+        req = Request(url, _data, headers, method=method)
+        with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
+            response_str = response.read()
+            response_headers = response.headers
+            response_code = response.code
+        return response_headers, response_str.decode(), response_code
+    except (HTTPError, URLError) as e:
+        print(f'{e}')
+        # handle error here if needed
+        raise

From 6117f79ebaad427c8d9ec869c9b258ea139a527e Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 16 Nov 2023 09:48:02 +0000
Subject: [PATCH 1584/2492] node-proxy: validate_node_proxy_data() refactor

raise cherrypy.HTTPError() when the received data is
not valid instead of returning `self.validate_msg`

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 47 ++++++++++++---------------------
 1 file changed, 17 insertions(+), 30 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 86726728bccb..fb45d01f46bc 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -118,38 +118,29 @@ def idrac(self) -> Dict[str, Any]:
         data: Dict[str, Any] = cherrypy.request.json
         results: Dict[str, Any] = {}
 
-        if self.validate_node_proxy_data(data):
-            host = data["cephx"]["name"]
-            results['result'] = self.mgr.node_proxy.idrac.get(host)
-        else:
-            results['result'] = self.validate_msg
+        self.validate_node_proxy_data(data)
+
+        host = data["cephx"]["name"]
+        results['result'] = self.mgr.node_proxy.idrac.get(host)
 
         return results
 
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
-        self.validate_msg = 'valid node-proxy data received.'
         cherrypy.response.status = 200
         try:
             if 'cephx' not in data.keys():
-                cherrypy.response.status = 400
-                self.validate_msg = 'The field \'cephx\' must be provided.'
+                raise cherrypy.HTTPError(400, 'The field \'cephx\' must be provided.')
             elif 'name' not in data['cephx'].keys():
                 cherrypy.response.status = 400
-                self.validate_msg = 'The field \'host\' must be provided.'
+                raise cherrypy.HTTPError(400, 'The field \'host\' must be provided.')
             elif 'secret' not in data['cephx'].keys():
-                cherrypy.response.status = 400
-                self.validate_msg = 'The agent keyring must be provided.'
+                raise cherrypy.HTTPError(400, 'The agent keyring must be provided.')
             elif not self.mgr.agent_cache.agent_keys.get(data['cephx']['name']):
-                cherrypy.response.status = 400
-                self.validate_msg = f'Make sure the agent is running on {data["cephx"]["name"]}'
+                raise cherrypy.HTTPError(502, f'Make sure the agent is running on {data["cephx"]["name"]}')
             elif data['cephx']['secret'] != self.mgr.agent_cache.agent_keys[data['cephx']['name']]:
-                cherrypy.response.status = 403
-                self.validate_msg = f'Got wrong keyring from agent on host {data["cephx"]["name"]}.'
+                raise cherrypy.HTTPError(403, f'Got wrong keyring from agent on host {data["cephx"]["name"]}.')
         except AttributeError:
-            cherrypy.response.status = 400
-            self.validate_msg = 'Malformed data received.'
-
-        return cherrypy.response.status == 200
+            raise cherrypy.HTTPError(400, 'Malformed data received.')
 
     def get_nok_members(self,
                         component: str,
@@ -204,18 +195,14 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
     @cherrypy.tools.allow(methods=['POST'])
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    def data(self) -> Dict[str, Any]:
-        results: Dict[str, Any] = {}
-
+    def data(self) -> None:
         data: Dict[str, Any] = cherrypy.request.json
-        if self.validate_node_proxy_data(data):
-            host = data['cephx']['name']
-            self.mgr.node_proxy.save(host, data['patch'])
-            self.raise_alert(data)
-
-        results["result"] = self.validate_msg
-
-        return results
+        self.validate_node_proxy_data(data)
+        if 'patch' not in data.keys():
+            raise cherrypy.HTTPError(400, 'Malformed data received.')
+        host = data['cephx']['name']
+        self.mgr.node_proxy.save(host, data['patch'])
+        self.raise_alert(data)
 
     def query_endpoint(self,
                        addr: str = '',

From 70e55db636c48140efc05c4dc9ae6da0ace49e43 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 26 Oct 2023 14:34:10 +0000
Subject: [PATCH 1585/2492] orch/cephadm: implement `ceph orch hardware`
 command

This adds a first implementation of the `ceph orch hardware` CLI.

Usage:

```
ceph orch hardware status [<hostname>] [--category <value>]
```

Omitting the `[<hostname>]` argument will generate a report for all hosts.
The default for argument `[--category]` is `summary`.

Example with `--category` :

```
+------------+-------------+-------+--------+---------+
|    HOST    |     NAME    | SPEED | STATUS |  STATE  |
+------------+-------------+-------+--------+---------+
| ceph-00001 |   eno8303   |   0   |   OK   | Enabled |
| ceph-00001 |   eno8403   |   0   |   OK   | Enabled |
| ceph-00001 | eno12399np0 | 10000 |   OK   | Enabled |
| ceph-00001 | eno12409np1 | 10000 |   OK   | Enabled |
| ceph-00001 |    bond0    | 10000 |   OK   | Enabled |
+------------+-------------+-------+--------+---------+
```

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py          | 90 +++++++++++++++++++++++
 src/pybind/mgr/orchestrator/_interface.py |  9 +++
 src/pybind/mgr/orchestrator/module.py     |  7 ++
 3 files changed, 106 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 8e54381b18cf..ed142adfc2a4 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1652,6 +1652,96 @@ def _add_host(self, spec):
     def add_host(self, spec: HostSpec) -> str:
         return self._add_host(spec)
 
+    @handle_orch_error
+    def hardware_status(self,
+                        hostname: Optional[str] = None,
+                        category: str = 'summary',
+                        format: Format = Format.plain) -> str:
+        """
+        Display hardware status summary
+
+        :param hostname: hostname
+        """
+
+        table_heading_mapping = {
+            'summary': ['HOST', 'STORAGE', 'CPU', 'NET', 'MEMORY', 'POWER', 'FANS'],
+            'firmwares': ['HOST', 'COMPONENT', 'NAME', 'DATE', 'VERSION', 'STATUS'],
+            'criticals': ['HOST', 'COMPONENT', 'NAME', 'STATUS', 'STATE'],
+            'memory': ['HOST', 'NAME', 'STATUS', 'STATE'],
+            'storage': ['HOST', 'NAME', 'MODEL', 'SIZE', 'PROTOCOL', 'SN', 'STATUS', 'STATE'],
+            'processors': ['HOST', 'NAME', 'MODEL', 'CORES', 'THREADS', 'STATUS', 'STATE'],
+            'network': ['HOST', 'NAME', 'SPEED', 'STATUS', 'STATE'],
+            'power': ['HOST', 'ID', 'NAME', 'MODEL', 'MANUFACTURER', 'STATUS', 'STATE'],
+            'fans': ['HOST', 'ID', 'NAME', 'STATUS', 'STATE']
+        }
+        table_headings = table_heading_mapping.get(category, [])
+        table = PrettyTable(
+            table_headings,
+            border=True)
+        if category not in table_heading_mapping.keys():
+            return f"'{category}' is not a valid category."
+        if category == 'summary':
+            data = self.node_proxy.summary(hostname=hostname)
+            for k, v in data.items():
+                row = [k]
+                row.extend([v[key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
+                table.add_row(row)
+            output = table.get_string()
+        elif category == 'firmwares':
+            output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table)
+        elif category == 'criticals':
+            output = self._criticals_table(hostname, table)
+        else:
+            output = self._common_table(category, hostname, table)
+
+        return output if 'output' in locals() else table.get_string()
+
+    def _firmwares_table(self, hostname, table):
+        data = self.node_proxy.firmwares(hostname=hostname)
+        for host, details in data.items():
+            for k, v in details.items():
+                table.add_row((host, k, v['name'], v['release_date'], v['version'], v['status']['health']))
+        return table.get_string()
+
+    def _criticals_table(self, hostname, table):
+        data = self.node_proxy.criticals(hostname=hostname)
+        for host, host_details in data.items():
+            for component, component_details in host_details.items():
+                for member, member_details in component_details.items():
+                    description = member_details.get('description') or member_details.get('name')
+                    table.add_row((host, component, description, member_details['status']['health'], member_details['status']['state']))
+        return table.get_string()
+
+    def _common_table(self, category, hostname, table):
+        data = self.node_proxy.common(endpoint=category, hostname=hostname)
+        mapping = {
+            'memory': ('description', 'health', 'state'),
+            'storage': ('description', 'model', 'capacity_bytes', 'protocol', 'serial_number', 'health', 'state'),
+            'processors': ('model', 'total_cores', 'total_threads', 'health', 'state'),
+            'network': ('name', 'speed_mbps', 'health', 'state'),
+            'power': ('name', 'model', 'manufacturer', 'health', 'state'),
+            'fans': ('name', 'health', 'state')
+        }
+
+        fields = mapping.get(category, ())
+        for host, details in data.items():
+            for k, v in details.items():
+                row = []
+                for field in fields:
+                    if field in v:
+                        row.append(v[field])
+                    elif field in v.get('status', {}):
+                        row.append(v['status'][field])
+                    else:
+                        row.append('')
+                if category in ('power', 'fans', 'processors'):
+                    table.add_row((host,) + (k,) + tuple(row))
+                else:
+                    table.add_row((host,) + tuple(row))
+
+
+        return table.get_string()
+
     @handle_orch_error
     def remove_host(self, host: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> str:
         """
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index 5bde317d19e6..b36ffd4f89a3 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -359,6 +359,15 @@ def add_host(self, host_spec: HostSpec) -> OrchResult[str]:
         """
         raise NotImplementedError()
 
+    def hardware_status(self, hostname: Optional[str] = None, category: Optional[str] = 'summary') -> OrchResult[str]:
+        """
+        Display hardware status.
+
+        :param category: category
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
     def remove_host(self, host: str, force: bool, offline: bool, rm_crush_entry: bool) -> OrchResult[str]:
         """
         Remove a host from the orchestrator inventory.
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 0000d25dd49c..9c5e2aec7719 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -488,6 +488,13 @@ def _add_host(self,
 
         return self._apply_misc([s], False, Format.plain)
 
+    @_cli_write_command('orch hardware status')
+    def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int = 0, category: str = 'summary') -> HandleCommandResult:
+        """Display hardware status"""
+        completion = self.hardware_status(hostname, category)
+        raise_if_exception(completion)
+        return HandleCommandResult(stdout=completion.result_str())
+
     @_cli_write_command('orch host rm')
     def _remove_host(self, hostname: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> HandleCommandResult:
         """Remove a host"""

From 81150e213ba157e4d943ed6c7f99305b0082b824 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 30 Oct 2023 15:51:56 +0000
Subject: [PATCH 1586/2492] node-proxy: implement criticals endpoint

This adds the required changes in order to implement the endpoint
'/criticals'.

The goal of this endpoint is to provide a report of all critical statuses
for either a given host or all hosts across the cluster.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py     | 24 +++++++-----------
 src/pybind/mgr/cephadm/inventory.py | 39 +++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index fb45d01f46bc..39b0cb3d0ef8 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -142,23 +142,16 @@ def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
         except AttributeError:
             raise cherrypy.HTTPError(400, 'Malformed data received.')
 
+    # TODO(guits): refactor this
+    # TODO(guits): use self.node_proxy.get_critical_from_host() ?
     def get_nok_members(self,
-                        component: str,
                         data: Dict[str, Any]) -> List[Dict[str, str]]:
         nok_members: List[Dict[str, str]] = []
 
         for member in data.keys():
-            # Force a fake error for testing purpose
-            if component == 'storage':
-                _status = 'critical'
-                state = "[Fake error] device is faulty."
-            elif component == 'power':
-                _status = 'critical'
-                state = "[Fake error] power supply unplugged."
-            else:
-                _status = data[member]['status']['health'].lower()
+            _status = data[member]['status']['health'].lower()
             if _status.lower() != 'ok':
-                # state = data[member]['status']['state']
+                state = data[member]['status']['state']
                 _member = dict(
                     member=member,
                     status=_status,
@@ -179,13 +172,14 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
         }
 
         for component in data['patch']['status'].keys():
-            self.mgr.remove_health_warning(mapping[component])
-            nok_members = self.get_nok_members(component, data['patch']['status'][component])
+            alert_name = f"HARDWARE_{component.upper()}"
+            self.mgr.remove_health_warning(alert_name)
+            nok_members = self.get_nok_members(data['patch']['status'][component])
 
             if nok_members:
                 count = len(nok_members)
                 self.mgr.set_health_warning(
-                    mapping[component],
+                    alert_name,
                     summary=f'{count} {component} member{"s" if count > 1 else ""} {"are" if count > 1 else "is"} not ok',
                     count=count,
                     detail=[f"{member['member']} is {member['status']}: {member['state']}" for member in nok_members],
@@ -285,7 +279,7 @@ def fullreport(self, **kw: Any) -> Dict[str, Any]:
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def criticals(self, **kw: Any) -> Dict[str, Any]:
-        return self.mgr.node_proxy.criticals()
+        return self.mgr.node_proxy.criticals(**kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index c4e0b34f2cbf..9d3eb4904e55 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1499,8 +1499,43 @@ def firmwares(self, **kw: Any) -> Dict[str, Any]:
 
         return {host: self.data[host]['firmwares'] for host in hosts}
 
-    def criticals(self, **kw):
-        return {}
+    def get_critical_from_host(self, hostname: str) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+        for component, data_component in self.data[hostname]['status'].items():
+            if component not in results.keys():
+                results[component] = {}
+            for member, data_member in data_component.items():
+                if component == 'power':
+                    data_member['status']['health'] = 'critical'
+                    data_member['status']['state'] = 'unplugged'
+                if component == 'memory':
+                    data_member['status']['health'] = 'critical'
+                    data_member['status']['state'] = 'errors detected'
+                if data_member['status']['health'].lower() != 'ok':
+                    results[component][member] = data_member
+        return results
+
+    def criticals(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Retrieves critical information for a specific hostname or all hosts.
+
+        If a 'hostname' is provided in the keyword arguments, retrieves critical
+        information for that specific host. Otherwise, retrieves critical
+        information for all available hosts.
+
+        :param kw: Keyword arguments, including 'hostname' if specified.
+        :type kw: dict
+
+        :return: A dictionary containing critical information for each host.
+        :rtype: List[Dict[str, Any]]
+        """
+        hostname = kw.get('hostname')
+        results: Dict[str, Any] = {}
+
+        hosts = [hostname] if hostname else self.data.keys()
+        for host in hosts:
+            results[host] = self.get_critical_from_host(host)
+        return results
 
 
 class AgentCache():

From 846cd8752adba4c2d696a5671d07d1bccc4fbda7 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 16 Nov 2023 13:35:51 +0000
Subject: [PATCH 1587/2492] cephadm/agent: add docstring to NodeProxy class

In order to document that part of the code and it might
help to generate API spec and documentation.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 216 ++++++++++++++++++++++++++++++--
 1 file changed, 207 insertions(+), 9 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 39b0cb3d0ef8..50652867e74b 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -115,6 +115,12 @@ def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     def idrac(self) -> Dict[str, Any]:
+        """
+        Get the iDrac details for a given host.
+
+        :return: idrac details.
+        :rtype: dict
+        """
         data: Dict[str, Any] = cherrypy.request.json
         results: Dict[str, Any] = {}
 
@@ -125,7 +131,16 @@ def idrac(self) -> Dict[str, Any]:
 
         return results
 
-    def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
+    def validate_node_proxy_data(self, data: Dict[str, Any]) -> None:
+        """
+        Validate received data.
+
+        :param data: data to validate.
+        :type data: dict
+
+        :raises cherrypy.HTTPError 400: If the data is not valid (missing fields)
+        :raises cherrypy.HTTPError 403: If the secret provided is wrong.
+        """
         cherrypy.response.status = 200
         try:
             if 'cephx' not in data.keys():
@@ -146,6 +161,18 @@ def validate_node_proxy_data(self, data: Dict[str, Any]) -> bool:
     # TODO(guits): use self.node_proxy.get_critical_from_host() ?
     def get_nok_members(self,
                         data: Dict[str, Any]) -> List[Dict[str, str]]:
+        """
+        Retrieves members whose status is not 'ok'.
+
+        :param data: Data containing information about members.
+        :type data: dict
+
+        :return: A list containing dictionaries of members whose status is not 'ok'.
+        :rtype: List[Dict[str, str]]
+
+        :return: None
+        :rtype: None
+        """
         nok_members: List[Dict[str, str]] = []
 
         for member in data.keys():
@@ -162,14 +189,27 @@ def get_nok_members(self,
         return nok_members
 
     def raise_alert(self, data: Dict[str, Any]) -> None:
-        mapping: Dict[str, str] = {
-            'storage': 'HARDWARE_STORAGE',
-            'memory': 'HARDWARE_MEMORY',
-            'processors': 'HARDWARE_PROCESSORS',
-            'network': 'HARDWARE_NETWORK',
-            'power': 'HARDWARE_POWER',
-            'fans': 'HARDWARE_FANS'
-        }
+        """
+        Raises hardware alerts based on the provided patch status.
+
+        :param data: Data containing patch status information.
+        :type data: dict
+
+        This function iterates through the provided status
+        information to raise hardware alerts.
+        For each component in the provided data, it removes any
+        existing health warnings associated with it and checks
+        for non-okay members using the `get_nok_members` method.
+        If non-okay members are found, it sets a new health
+        warning for that component and generates a report detailing
+        the non-okay members' statuses.
+
+        Note: This function relies on the `get_nok_members` method to
+        identify non-okay members.
+
+        :return: None
+        :rtype: None
+        """
 
         for component in data['patch']['status'].keys():
             alert_name = f"HARDWARE_{component.upper()}"
@@ -190,6 +230,19 @@ def raise_alert(self, data: Dict[str, Any]) -> None:
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     def data(self) -> None:
+        """
+        Handles incoming data via a POST request.
+
+        This function is exposed to handle POST requests and expects incoming
+        JSON data. It processes the incoming data by first validating it
+        through the `validate_node_proxy_data` method. Subsequently, it
+        extracts the hostname from the data and saves the information
+        using `mgr.node_proxy.save`. Finally, it raises alerts based on the
+        provided status through the `raise_alert` method.
+
+        :return: None
+        :rtype: None
+        """
         data: Dict[str, Any] = cherrypy.request.json
         self.validate_node_proxy_data(data)
         if 'patch' not in data.keys():
@@ -235,6 +288,25 @@ def query_endpoint(self,
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     def led(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles enclosure LED operations for the specified hostname.
+
+        This function handles GET and PATCH requests related to LED status for a
+        specific hostname. It identifies the request method and provided hostname.
+        If the hostname is missing, it logs an error and returns an error message.
+
+        For PATCH requests, it prepares authorization headers based on the
+        provided ID and password, encodes them, and constructs the authorization
+        header.
+
+        After processing, it queries the endpoint and returns the result.
+
+        :param kw: Keyword arguments including 'hostname'.
+        :type kw: dict
+
+        :return: Result of the LED-related operation.
+        :rtype: dict[str, Any]
+        """
         method: str = cherrypy.request.method
         hostname: Optional[str] = kw.get('hostname')
         headers: Dict[str, str] = {}
@@ -273,60 +345,186 @@ def led(self, **kw: Any) -> Dict[str, Any]:
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def fullreport(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve a full report.
+
+        This function is exposed to handle GET requests and retrieves a comprehensive
+        report using the 'fullreport' method from the NodeProxyCache class.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: The full report data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.fullreport(**kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def criticals(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve critical information.
+
+        This function is exposed to handle GET requests and fetches critical data
+        using the 'criticals' method from the NodeProxyCache class.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Critical information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.criticals(**kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def summary(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve summary information.
+
+        This function is exposed to handle GET requests and fetches summary
+        data using the 'summary' method from the NodeProxyCache class.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Summary information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.summary(**kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def memory(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('memory', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def network(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('network', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def processors(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('processors', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def storage(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('storage', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def power(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('power', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def fans(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve specific information.
+
+        This function is exposed to handle GET requests
+        and fetch specific data using the 'common' method
+        from the NodeProxyCache class with.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Specific information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.common('fans', **kw)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
     @cherrypy.tools.json_out()
     def firmwares(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Handles GET request to retrieve firmware information.
+
+        This function is exposed to handle GET requests and fetches firmware data using
+        the 'firmwares' method from the NodeProxyCache class.
+
+        :param kw: Keyword arguments for the request.
+        :type kw: dict
+
+        :return: Firmware information data.
+        :rtype: dict[str, Any]
+        """
         return self.mgr.node_proxy.firmwares(**kw)
 
 

From f703e9118fcf431448a407ea7fce30c9dc7b3356 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 20 Nov 2023 14:55:26 +0000
Subject: [PATCH 1588/2492] cephadm: inventory.NodeProxyCache() refactor

This modifies fullreport(), summary() and common() methods
so they use the same logic as firmwares() and criticals()

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/inventory.py | 72 +++++++++++++++++++++--------
 1 file changed, 53 insertions(+), 19 deletions(-)

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 9d3eb4904e55..a320fb1f2395 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1437,16 +1437,41 @@ def save(self,
         self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", json.dumps(data))
 
     def fullreport(self, **kw: Any) -> Dict[str, Any]:
+        """
+        Retrieves the full report for the specified hostname.
+
+        If a hostname is provided in the keyword arguments, it retrieves the full report
+        data for that specific host. If no hostname is provided, it fetches the full
+        report data for all hosts available.
+
+        :param kw: Keyword arguments including 'hostname'.
+        :type kw: dict
+
+        :return: The full report data for the specified hostname(s).
+        :rtype: dict
+        """
         hostname = kw.get('hostname')
-        if hostname not in self.data.keys():
-            return [self.data[h] for h in self.data.keys()]
-        else:
-            return self.data[hostname]
+        hosts = [hostname] if hostname else self.data.keys()
+        return {host: self.data[host] for host in hosts}
 
     def summary(self, **kw: Any) -> Dict[str, Any]:
-        hostname = kw.get('hostname')
-        results = self.data
+        """
+        Summarizes the health status of components for specified hosts or all hosts.
 
+        Generates a summary of the health status of components for given hosts. If
+        no hostname is provided, it generates the health status summary for all hosts.
+        It inspects the status of each component and categorizes it as 'ok' or 'error'
+        based on the health status of its members.
+
+        :param kw: Keyword arguments including 'hostname'.
+        :type kw: dict
+
+        :return: A dictionary containing the health status summary for each specified
+                host or all hosts and their components.
+        :rtype: Dict[str, Dict[str, str]]
+        """
+        hostname = kw.get('hostname')
+        hosts = [hostname] if hostname else self.data.keys()
         mapper: Dict[bool, str] = {
             True: 'error',
             False: 'ok'
@@ -1454,31 +1479,40 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
 
         _result: Dict[str, Any] = {}
 
-        for host, data in results.items():
+        for host in hosts:
             _result[host] = {}
+            data = self.data[host]
             for component, details in data['status'].items():
                 res = any([member['status']['health'].lower() != 'ok' for member in data['status'][component].values()])
                 _result[host][component] = mapper[res]
 
-        if hostname and hostname in results.keys():
-            return _result[hostname]
-        else:
-            return _result
+        return _result
 
     def common(self, endpoint: str, **kw: Any) -> Dict[str, Any]:
+        """
+        Retrieves specific endpoint information for a specific hostname or all hosts.
+
+        Retrieves information from the specified 'endpoint' for all available hosts.
+        If 'hostname' is provided, retrieves the specified 'endpoint' information for that host.
+
+        :param endpoint: The endpoint for which information is retrieved.
+        :type endpoint: str
+        :param kw: Keyword arguments, including 'hostname' if specified.
+        :type kw: dict
+
+        :return: Endpoint information for the specified host(s).
+        :rtype: Union[Dict[str, Any], Any]
+        """
         hostname = kw.get('hostname')
         _result = {}
+        hosts = [hostname] if hostname else self.data.keys()
 
-        for host, data in self.data.items():
+        for host in hosts:
             try:
-                _result[host] = data['status'][endpoint]
+                _result[host] = self.data[host]['status'][endpoint]
             except KeyError:
-                raise RuntimeError(f'Invalid node-proxy endpoint {endpoint}')
-
-        if hostname and hostname in self.data.keys():
-            return _result[hostname]
-        else:
-            return _result
+                raise KeyError(f'Invalid host {host} or component {endpoint}.')
+        return _result
 
     def firmwares(self, **kw: Any) -> Dict[str, Any]:
         """

From ff72f18b5b199408b3789a5da500d5e6d4fbc74a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 22 Nov 2023 14:27:09 +0000
Subject: [PATCH 1589/2492] node-proxy: run only when idrac details provided

This agent shouldn't run when no idrac details are
available.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                    | 59 +++++++++++++++++------
 src/cephadm/cephadmlib/node_proxy/main.py |  8 +++
 src/pybind/mgr/cephadm/agent.py           |  8 +--
 3 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 8ee75cb65203..95735512963f 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -30,7 +30,7 @@
 from threading import Thread, Event
 from urllib.request import urlopen, Request
 from pathlib import Path
-from cephadmlib.node_proxy.main import NodeProxy
+from cephadmlib.node_proxy.main import NodeProxy, NodeProxyInitialization, NodeProxyFetchIdracError
 
 from cephadmlib.constants import (
     # default images
@@ -1358,6 +1358,7 @@ def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str] =
         self.recent_iteration_run_times: List[float] = [0.0, 0.0, 0.0]
         self.recent_iteration_index: int = 0
         self.cached_ls_values: Dict[str, Dict[str, str]] = {}
+        self.t_node_proxy: Optional["NodeProxy"] = None
 
     def validate(self, config: Dict[str, str] = {}) -> None:
         # check for the required files
@@ -1456,22 +1457,51 @@ def query_endpoint(self,
                        port: str = '',
                        data: Optional[Union[Dict[str, str], str]] = None,
                        endpoint: str = '',
-                       ssl_ctx: Optional[Any] = None) -> str:
+                       ssl_ctx: Optional[Any] = None,
+                       timeout: Optional[int] = 10) -> Tuple[int, str]:
         _addr = addr if addr else self.target_ip
         _port = port if port else self.target_port
         url = f'https://{_addr}:{_port}{endpoint}'
-
+        logger.info(f"sending query to {url}")
         try:
             req = Request(url, data, {'Content-Type': 'application/json'})
             send_time = time.monotonic()
-            with urlopen(req, context=ssl_ctx) as response:
+            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
                 response_json = json.loads(response_str)
                 total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
                 logger.info(f'Received mgr response: "{response_json["result"]}" {total_request_time} seconds after sending request.')
+                response_status = response.status
+        except HTTPError as e:
+            logger.debug(f"{e.code} {e.reason}")
+            response_status = e.code
+            response_str = e.reason
+        except URLError as e:
+            logger.debug(f"{e.reason}")
+            response_status = -1
+            response_str = e.reason
         except Exception:
             raise
-        return response_str
+        return (response_status, response_str)
+
+    def node_proxy_loop_check(self, ssl_ctx: Any) -> None:
+        while True:
+            try:
+                if isinstance(self.t_node_proxy, NodeProxy):
+                    status = self.t_node_proxy.check_status()
+                    label = 'Ok' if status else 'Critical'
+                    logger.debug(f'node-proxy status: {label}')
+                else:
+                    raise NodeProxyInitialization("starting node-proxy...")
+            except Exception as e:
+                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
+                try:
+                    self.init_node_proxy(ssl_ctx)
+                except NodeProxyFetchIdracError:
+                    logger.info("No iDrac details could be loaded. "
+                                "Aborting node-proxy initialization. "
+                                "Will retry in 120s.")
+                    time.sleep(120)
 
     def init_node_proxy(self, ssl_ctx: Any) -> None:
         node_proxy_meta = {
@@ -1480,9 +1510,13 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
                 'secret': self.keyring
             }
         }
-        result = self.query_endpoint(data=json.dumps(node_proxy_meta).encode('ascii'),
-                                     endpoint='/node-proxy/idrac',
-                                     ssl_ctx=ssl_ctx)
+        status, result = self.query_endpoint(data=json.dumps(node_proxy_meta).encode('ascii'),
+                                             endpoint='/node-proxy/idrac',
+                                             ssl_ctx=ssl_ctx)
+        if status != 200:
+            msg = f"Couldn't load iDrac details: {status}, {result}"
+            logger.debug(msg)
+            raise NodeProxyFetchIdracError(msg)
         result_json = json.loads(result)
         kwargs = {
             'host': result_json['result']['addr'],
@@ -1529,15 +1563,10 @@ def run(self) -> None:
             self.volume_gatherer.start()
 
         # initiate node-proxy thread
-        self.init_node_proxy(ssl_ctx)
+        node_proxy_loop_thread = Thread(target=self.node_proxy_loop_check, args=(ssl_ctx,))
+        node_proxy_loop_thread.start()
 
         while not self.stop:
-            try:
-                _mapper = {True: 'Ok', False: 'Critical'}
-                logger.debug(f'node-proxy status: {_mapper[self.t_node_proxy.check_status()]}')
-            except Exception as e:
-                logger.error(f'node-proxy failure: {e.__class__.__name__}: {e}')
-                self.init_node_proxy(ssl_ctx)
             start_time = time.monotonic()
             ack = self.ack
 
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index e9c99f0983dc..106d3b447bcc 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -26,6 +26,14 @@
 }
 
 
+class NodeProxyInitialization(Exception):
+    pass
+
+
+class NodeProxyFetchIdracError(Exception):
+    pass
+
+
 @cherrypy.tools.auth_basic(on=True)
 @cherrypy.tools.allow(methods=['PUT'])
 @cherrypy.tools.json_out()
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 50652867e74b..b8453f719af8 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -128,7 +128,8 @@ def idrac(self) -> Dict[str, Any]:
 
         host = data["cephx"]["name"]
         results['result'] = self.mgr.node_proxy.idrac.get(host)
-
+        if not results['result']:
+            raise cherrypy.HTTPError(400, 'The provided host has no iDrac details.')
         return results
 
     def validate_node_proxy_data(self, data: Dict[str, Any]) -> None:
@@ -258,7 +259,8 @@ def query_endpoint(self,
                        headers: Optional[Dict[str, str]] = {},
                        data: Optional[bytes] = None,
                        endpoint: str = '',
-                       ssl_ctx: Optional[Any] = None) -> Tuple[int, Dict[str, Any]]:
+                       ssl_ctx: Optional[Any] = None,
+                       timeout: Optional[int] = 10) -> Tuple[int, Dict[str, Any]]:
         url = f'https://{addr}:{port}{endpoint}'
         _headers = headers
         response_json = {}
@@ -268,7 +270,7 @@ def query_endpoint(self,
         _data = bytes(data, 'ascii') if data else None
         try:
             req = Request(url, _data, _headers, method=method)
-            with urlopen(req, context=ssl_ctx) as response:
+            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
                 response_json = json.loads(response_str)
                 response_status = response.status

From 5cb8f2af44ea587f3c85d72bee385f571e6cc2cb Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 23 Nov 2023 16:08:18 +0000
Subject: [PATCH 1590/2492] node-proxy: raise HTTPError 404 error when no host
 is found

Raise a 404 HTTPError when these differents endpoints
are passed an inexisting hostname.
Otherwise the code will fail with a `KeyError` exception.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 80 ++++++++++++++++++++++++++++-----
 1 file changed, 70 insertions(+), 10 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index b8453f719af8..316cc7c0da30 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -358,8 +358,14 @@ def fullreport(self, **kw: Any) -> Dict[str, Any]:
 
         :return: The full report data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.fullreport(**kw)
+        try:
+            results = self.mgr.node_proxy.fullreport(**kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -376,8 +382,14 @@ def criticals(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Critical information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.criticals(**kw)
+        try:
+            results = self.mgr.node_proxy.criticals(**kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -394,8 +406,14 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Summary information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.summary(**kw)
+        try:
+            results = self.mgr.node_proxy.summary(**kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -413,8 +431,14 @@ def memory(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('memory', **kw)
+        try:
+            results = self.mgr.node_proxy.common('memory', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -432,8 +456,14 @@ def network(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('network', **kw)
+        try:
+            results = self.mgr.node_proxy.common('network', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -451,8 +481,14 @@ def processors(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('processors', **kw)
+        try:
+            results = self.mgr.node_proxy.common('processors', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -470,8 +506,14 @@ def storage(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('storage', **kw)
+        try:
+            results = self.mgr.node_proxy.common('storage', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -489,8 +531,14 @@ def power(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('power', **kw)
+        try:
+            results = self.mgr.node_proxy.common('power', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -508,8 +556,14 @@ def fans(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Specific information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.common('fans', **kw)
+        try:
+            results = self.mgr.node_proxy.common('fans', **kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -526,8 +580,14 @@ def firmwares(self, **kw: Any) -> Dict[str, Any]:
 
         :return: Firmware information data.
         :rtype: dict[str, Any]
+
+        :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
-        return self.mgr.node_proxy.firmwares(**kw)
+        try:
+            results = self.mgr.node_proxy.firmwares(**kw)
+        except KeyError:
+            raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
+        return results
 
 
 class HostData(Server):

From 17959de64017466305d1a86dd5ed2492cf423468 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 28 Nov 2023 08:05:47 +0000
Subject: [PATCH 1591/2492] node-proxy: change 'idrac' terminology

The 'idrac' terminology is too specific, let's change this
to something more generic.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        | 12 +++----
 src/cephadm/cephadmlib/node_proxy/main.py     |  2 +-
 .../cephadmlib/node_proxy/redfish_client.py   |  6 ++--
 src/pybind/mgr/cephadm/agent.py               | 12 +++----
 src/pybind/mgr/cephadm/inventory.py           |  8 ++---
 src/pybind/mgr/cephadm/module.py              | 22 ++++++-------
 .../mgr/cephadm/tests/node_proxy_data.py      | 12 +++----
 .../mgr/cephadm/tests/test_node_proxy.py      | 32 +++++++++----------
 src/python-common/ceph/deployment/hostspec.py | 20 ++++++------
 9 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 95735512963f..5db12f9dfc30 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -30,7 +30,7 @@
 from threading import Thread, Event
 from urllib.request import urlopen, Request
 from pathlib import Path
-from cephadmlib.node_proxy.main import NodeProxy, NodeProxyInitialization, NodeProxyFetchIdracError
+from cephadmlib.node_proxy.main import NodeProxy, NodeProxyInitialization, NodeProxyFetchOobError
 
 from cephadmlib.constants import (
     # default images
@@ -1497,8 +1497,8 @@ def node_proxy_loop_check(self, ssl_ctx: Any) -> None:
                 logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
                 try:
                     self.init_node_proxy(ssl_ctx)
-                except NodeProxyFetchIdracError:
-                    logger.info("No iDrac details could be loaded. "
+                except NodeProxyFetchOobError:
+                    logger.info("No oob details could be loaded. "
                                 "Aborting node-proxy initialization. "
                                 "Will retry in 120s.")
                     time.sleep(120)
@@ -1511,12 +1511,12 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
             }
         }
         status, result = self.query_endpoint(data=json.dumps(node_proxy_meta).encode('ascii'),
-                                             endpoint='/node-proxy/idrac',
+                                             endpoint='/node-proxy/oob',
                                              ssl_ctx=ssl_ctx)
         if status != 200:
-            msg = f"Couldn't load iDrac details: {status}, {result}"
+            msg = f"Couldn't load oob details: {status}, {result}"
             logger.debug(msg)
-            raise NodeProxyFetchIdracError(msg)
+            raise NodeProxyFetchOobError(msg)
         result_json = json.loads(result)
         kwargs = {
             'host': result_json['result']['addr'],
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 106d3b447bcc..66da0038ecc9 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -30,7 +30,7 @@ class NodeProxyInitialization(Exception):
     pass
 
 
-class NodeProxyFetchIdracError(Exception):
+class NodeProxyFetchOobError(Exception):
     pass
 
 
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index 4107fb57c8f0..dec35a2c5728 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -27,12 +27,12 @@ def login(self) -> None:
         if not self.is_logged_in():
             self.log.logger.info("Logging in to "
                                  f"{self.url} as '{self.username}'")
-            idrac_credentials = json.dumps({"UserName": self.username,
-                                            "Password": self.password})
+            oob_credentials = json.dumps({"UserName": self.username,
+                                          "Password": self.password})
             headers = {"Content-Type": "application/json"}
 
             try:
-                _headers, _data, _status_code = self.query(data=idrac_credentials,
+                _headers, _data, _status_code = self.query(data=oob_credentials,
                                                            headers=headers,
                                                            endpoint='/redfish/v1/SessionService/Sessions/')
                 if _status_code != 201:
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 316cc7c0da30..d5563eee6c74 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -114,11 +114,11 @@ def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
     @cherrypy.tools.allow(methods=['POST'])
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    def idrac(self) -> Dict[str, Any]:
+    def oob(self) -> Dict[str, Any]:
         """
-        Get the iDrac details for a given host.
+        Get the out-of-band management tool details for a given host.
 
-        :return: idrac details.
+        :return: oob details.
         :rtype: dict
         """
         data: Dict[str, Any] = cherrypy.request.json
@@ -127,7 +127,7 @@ def idrac(self) -> Dict[str, Any]:
         self.validate_node_proxy_data(data)
 
         host = data["cephx"]["name"]
-        results['result'] = self.mgr.node_proxy.idrac.get(host)
+        results['result'] = self.mgr.node_proxy.oob.get(host)
         if not results['result']:
             raise cherrypy.HTTPError(400, 'The provided host has no iDrac details.')
         return results
@@ -324,8 +324,8 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             # TODO(guits): need to check the request is authorized
             # allowing a specific keyring only ? (client.admin or client.agent.. ?)
             data: str = json.dumps(cherrypy.request.json)
-            username = self.mgr.node_proxy.idrac[hostname]['username']
-            password = self.mgr.node_proxy.idrac[hostname]['password']
+            username = self.mgr.node_proxy.oob[hostname]['username']
+            password = self.mgr.node_proxy.oob[hostname]['password']
             auth = f"{username}:{password}".encode("utf-8")
             auth = base64.b64encode(auth).decode("utf-8")
             headers = {"Authorization": f"Basic {auth}"}
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index a320fb1f2395..15e81fa57f40 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1410,11 +1410,11 @@ class NodeProxyCache:
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
         self.data: Dict[str, Any] = {}
-        self.idrac = {}
+        self.oob: Dict[str, Any] = {}
 
     def load(self) -> None:
-        _idrac = self.mgr.get_store('node_proxy/idrac', "{}")
-        self.idrac = json.loads(_idrac)
+        _oob = self.mgr.get_store('node_proxy/oob', "{}")
+        self.oob = json.loads(_oob)
 
         for k, v in self.mgr.get_store_prefix(NODE_PROXY_CACHE_PREFIX).items():
             host = k.split('/')[-1:][0]
@@ -1423,7 +1423,7 @@ def load(self) -> None:
                 # remove entry for host that no longer exists
                 self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", None)
                 try:
-                    self.idrac.pop(host)
+                    self.oob.pop(host)
                     self.data.pop(host)
                 except KeyError:
                     pass
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index ed142adfc2a4..1bf86a65dd89 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1616,18 +1616,18 @@ def _add_host(self, spec):
         if spec.hostname in self.inventory and self.inventory.get_addr(spec.hostname) != spec.addr:
             self.cache.refresh_all_host_info(spec.hostname)
 
-        if spec.idrac:
-            if not spec.idrac.get('addr'):
-                spec.idrac['addr'] = spec.hostname
-            if not spec.idrac.get('port'):
-                spec.idrac['port'] = '443'
-            data = json.loads(self.get_store('node_proxy/idrac', '{}'))
+        if spec.oob:
+            if not spec.oob.get('addr'):
+                spec.oob['addr'] = spec.hostname
+            if not spec.oob.get('port'):
+                spec.oob['port'] = '443'
+            data = json.loads(self.get_store('node_proxy/oob', '{}'))
             data[spec.hostname] = dict()
-            data[spec.hostname]['addr'] = spec.idrac['addr']
-            data[spec.hostname]['port'] = spec.idrac['port']
-            data[spec.hostname]['username'] = spec.idrac['username']
-            data[spec.hostname]['password'] = spec.idrac['password']
-            self.set_store('node_proxy/idrac', json.dumps(data))
+            data[spec.hostname]['addr'] = spec.oob['addr']
+            data[spec.hostname]['port'] = spec.oob['port']
+            data[spec.hostname]['username'] = spec.oob['username']
+            data[spec.hostname]['password'] = spec.oob['password']
+            self.set_store('node_proxy/oob', json.dumps(data))
 
         # prime crush map?
         if spec.location:
diff --git a/src/pybind/mgr/cephadm/tests/node_proxy_data.py b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
index 1073bed274cd..70afc8a76483 100644
--- a/src/pybind/mgr/cephadm/tests/node_proxy_data.py
+++ b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
@@ -89,16 +89,16 @@
                                   "addr": "10.10.10.11",
                                   "labels": ["_admin"],
                                   "status": "",
-                                  "idrac": {"hostname": "10.10.10.11",
-                                            "username": "root",
-                                            "password": "ceph123"}},
+                                  "oob": {"hostname": "10.10.10.11",
+                                          "username": "root",
+                                          "password": "ceph123"}},
                        "host02": {"hostname": "host02",
                                   "addr": "10.10.10.12",
                                   "labels": [],
                                   "status": "",
-                                  "idrac": {"hostname": "10.10.10.12",
-                                            "username": "root",
-                                            "password": "ceph123"}}}
+                                  "oob": {"hostname": "10.10.10.12",
+                                          "username": "root",
+                                          "password": "ceph123"}}}
 
 full_set = {
   "host01": {
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index 42ab39139dda..fce7152bd86b 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -38,10 +38,10 @@ class TestNodeProxy(helper.CPWebCase):
     app = NodeProxy(mgr)
     mgr.agent_cache.agent_keys = {"host01": "fake-secret01",
                                   "host02": "fake-secret02"}
-    mgr.node_proxy.idrac = {"host01": {"username": "idrac-user01",
-                                       "password": "idrac-pass01"},
-                            "host02": {"username": "idrac-user02",
-                                       "password": "idrac-pass02"}}
+    mgr.node_proxy.oob = {"host01": {"username": "oob-user01",
+                                     "password": "oob-pass01"},
+                          "host02": {"username": "oob-user02",
+                                     "password": "oob-pass02"}}
     mgr.node_proxy.data = node_proxy_data.full_set
 
     @classmethod
@@ -56,39 +56,39 @@ def setUp(self):
         self.PORT = PORT
         self.monkeypatch = MonkeyPatch()
 
-    def test_idrac_data_misses_cephx_field(self):
+    def test_oob_data_misses_cephx_field(self):
         data = '{}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
-    def test_idrac_data_misses_name_field(self):
+    def test_oob_data_misses_name_field(self):
         data = '{"cephx": {"secret": "fake-secret"}}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
-    def test_idrac_data_misses_secret_field(self):
+    def test_oob_data_misses_secret_field(self):
         data = '{"cephx": {"name": "host01"}}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
-    def test_idrac_agent_not_running(self):
+    def test_oob_agent_not_running(self):
         data = '{"cephx": {"name": "host03", "secret": "fake-secret03"}}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('502 Bad Gateway')
 
-    def test_idrac_wrong_keyring(self):
+    def test_oob_wrong_keyring(self):
         data = '{"cephx": {"name": "host01", "secret": "wrong-keyring"}}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('403 Forbidden')
 
-    def test_idrac_ok(self):
+    def test_oob_ok(self):
         data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
-        self.getPage("/idrac", method="POST", body=data, headers=[('Content-Type', 'application/json'),
+        self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                   ('Content-Length', str(len(data)))])
         self.assertStatus('200 OK')
 
diff --git a/src/python-common/ceph/deployment/hostspec.py b/src/python-common/ceph/deployment/hostspec.py
index 8fc9fdc508d8..f17ba81cf09b 100644
--- a/src/python-common/ceph/deployment/hostspec.py
+++ b/src/python-common/ceph/deployment/hostspec.py
@@ -16,11 +16,11 @@ def assert_valid_host(name: str) -> None:
         raise SpecValidationError(str(e) + f'. Got "{name}"')
 
 
-def assert_valid_idrac(idrac: Dict[str, str]) -> None:
+def assert_valid_oob(oob: Dict[str, str]) -> None:
     fields = ['username', 'password']
     try:
         for field in fields:
-            assert field in idrac.keys()
+            assert field in oob.keys()
     except AssertionError as e:
         raise SpecValidationError(str(e))
 
@@ -47,7 +47,7 @@ def __init__(self,
                  labels: Optional[List[str]] = None,
                  status: Optional[str] = None,
                  location: Optional[Dict[str, str]] = None,
-                 idrac: Optional[Dict[str, str]] = None,
+                 oob: Optional[Dict[str, str]] = None,
                  ):
         self.service_type = 'host'
 
@@ -65,13 +65,13 @@ def __init__(self,
 
         self.location = location
 
-        #: idrac details, if provided
-        self.idrac = idrac
+        #: oob details, if provided
+        self.oob = oob
 
     def validate(self) -> None:
         assert_valid_host(self.hostname)
-        if self.idrac:
-            assert_valid_idrac(self.idrac)
+        if self.oob:
+            assert_valid_oob(self.oob)
 
     def to_json(self) -> Dict[str, Any]:
         r: Dict[str, Any] = {
@@ -82,8 +82,8 @@ def to_json(self) -> Dict[str, Any]:
         }
         if self.location:
             r['location'] = self.location
-        if self.idrac:
-            r['idrac'] = self.idrac
+        if self.oob:
+            r['oob'] = self.oob
         return r
 
     @classmethod
@@ -96,7 +96,7 @@ def from_json(cls, host_spec: dict) -> 'HostSpec':
                 host_spec['labels'])) if 'labels' in host_spec else None,
             host_spec['status'] if 'status' in host_spec else None,
             host_spec.get('location'),
-            host_spec['idrac'] if 'idrac' in host_spec else None,
+            host_spec['oob'] if 'oob' in host_spec else None,
         )
         return _cls
 

From 16b840f46dadc98d7e79e3b65d452aec389c3847 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 28 Nov 2023 13:17:47 +0000
Subject: [PATCH 1592/2492] node-proxy: drop local API

This was intented to address the case where the Ceph
manager can't talk directly to the oob management tool because
of network restrictions (subnets not inter-connecter, etc.).

If for any reason the host is stuck or unreachable, that local API won't
be helpful anyway, as a result any actions the Ceph mgr would be asked
to perform on the node would fail.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        |   5 +-
 src/cephadm/cephadmlib/node_proxy/main.py     | 179 +-----------------
 .../node_proxy/redfishdellsystem.py           |  37 ----
 src/pybind/mgr/cephadm/agent.py               | 142 ++++++++++----
 4 files changed, 109 insertions(+), 254 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5db12f9dfc30..aa5ae7570531 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1524,10 +1524,7 @@ def init_node_proxy(self, ssl_ctx: Any) -> None:
             'password': result_json['result']['password'],
             'cephx': node_proxy_meta['cephx'],
             'mgr_target_ip': self.target_ip,
-            'mgr_target_port': self.target_port,
-            # re-use listener ssl certificate instead of generating new ones...
-            'ssl_crt_path': self.listener_cert_path,
-            'ssl_key_path': self.listener_key_path
+            'mgr_target_port': self.target_port
         }
         if result_json['result'].get('port'):
             kwargs['port'] = result_json['result']['port']
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 66da0038ecc9..0575340d0ecd 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -1,11 +1,8 @@
-import cherrypy
-from cherrypy._cpserver import Server
-from threading import Thread, Event
+from threading import Thread
 from .redfishdellsystem import RedfishDellSystem
 from .reporter import Reporter
 from .util import Config, Logger
-from typing import Dict, Any, Optional, List
-from .basesystem import BaseSystem
+from typing import Dict, Any, Optional
 import traceback
 
 DEFAULT_CONFIG = {
@@ -34,147 +31,12 @@ class NodeProxyFetchOobError(Exception):
     pass
 
 
-@cherrypy.tools.auth_basic(on=True)
-@cherrypy.tools.allow(methods=['PUT'])
-@cherrypy.tools.json_out()
-class Admin():
-    def __init__(self, api: 'API') -> None:
-        self.api = api
-
-    @cherrypy.expose
-    def start(self) -> Dict[str, str]:
-        self.api.backend.start_client()
-        # self.backend.start_update_loop()
-        self.api.reporter.run()
-        return {"ok": "node-proxy daemon started"}
-
-    @cherrypy.expose
-    def reload(self) -> Dict[str, str]:
-        self.api.config.reload()
-        return {"ok": "node-proxy config reloaded"}
-
-    def _stop(self) -> None:
-        self.api.backend.stop_update_loop()
-        self.api.backend.client.logout()
-        self.api.reporter.stop()
-
-    @cherrypy.expose
-    def stop(self) -> Dict[str, str]:
-        self._stop()
-        return {"ok": "node-proxy daemon stopped"}
-
-    @cherrypy.expose
-    def shutdown(self) -> Dict[str, str]:
-        self._stop()
-        cherrypy.engine.exit()
-        return {"ok": "Server shutdown."}
-
-    @cherrypy.expose
-    def flush(self) -> Dict[str, str]:
-        self.api.backend.flush()
-        return {"ok": "node-proxy data flushed"}
-
-
-class API(Server):
-    def __init__(self,
-                 backend: BaseSystem,
-                 reporter: Reporter,
-                 config: Config,
-                 addr: str = '0.0.0.0',
-                 port: int = 0) -> None:
-        super().__init__()
-        self.log = Logger(__name__)
-        self.backend = backend
-        self.reporter = reporter
-        self.config = config
-        self.socket_port = self.config.__dict__['server']['port'] if not port else port
-        self.socket_host = addr
-        self.subscribe()
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def memory(self) -> Dict[str, Any]:
-        return {'memory': self.backend.get_memory()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def network(self) -> Dict[str, Any]:
-        return {'network': self.backend.get_network()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def processors(self) -> Dict[str, Any]:
-        return {'processors': self.backend.get_processors()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def storage(self) -> Dict[str, Any]:
-        return {'storage': self.backend.get_storage()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def power(self) -> Dict[str, Any]:
-        return {'power': self.backend.get_power()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def fans(self) -> Dict[str, Any]:
-        return {'fans': self.backend.get_fans()}
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def firmwares(self) -> Dict[str, Any]:
-        return {'firmwares': self.backend.get_firmwares()}
-
-    def _cp_dispatch(self, vpath: List[str]) -> "API":
-        if vpath[0] == 'led':
-            if cherrypy.request.method == 'GET':
-                return self.get_led
-            if cherrypy.request.method == 'PATCH':
-                return self.set_led
-        return self
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['GET'])
-    @cherrypy.tools.json_out()
-    def get_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
-        return self.backend.get_led()
-
-    @cherrypy.expose
-    @cherrypy.tools.allow(methods=['PATCH'])
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @cherrypy.tools.auth_basic(on=True)
-    def set_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
-        data = cherrypy.request.json
-        rc = self.backend.set_led(data)
-
-        if rc != 200:
-            cherrypy.response.status = rc
-            result = {"state": f"error: please, verify the data you sent."}
-        else:
-            result = {"state": data["state"].lower()}
-        return result
-
-    def stop(self) -> None:
-        self.unsubscribe()
-        super().stop()
-
-
 class NodeProxy(Thread):
     def __init__(self, **kw: Dict[str, Any]) -> None:
         super().__init__()
         for k, v in kw.items():
             setattr(self, k, v)
         self.exc: Optional[Exception] = None
-        self.cp_shutdown_event = Event()
         self.log = Logger(__name__)
 
     def run(self) -> None:
@@ -197,16 +59,6 @@ def check_status(self) -> bool:
             raise self.exc
         return True
 
-    def start_api(self) -> None:
-        cherrypy.server.unsubscribe()
-        cherrypy.engine.start()
-        self.reporter_agent.run()
-        self.cp_shutdown_event.wait()
-        self.cp_shutdown_event.clear()
-        cherrypy.engine.stop()
-        cherrypy.server.httpserver = None
-        self.log.logger.info("node-proxy shutdown.")
-
     def main(self) -> None:
         # TODO: add a check and fail if host/username/password/data aren't passed
         self.config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
@@ -231,32 +83,7 @@ def main(self) -> None:
                                            reporter_hostname=self.__dict__['mgr_target_ip'],
                                            reporter_port=self.__dict__['mgr_target_port'],
                                            reporter_endpoint=self.__dict__.get('reporter_endpoint', '/node-proxy/data'))
+            self.reporter_agent.run()
         except RuntimeError:
             self.log.logger.error("Can't initialize the reporter.")
             raise
-        self.api = API(self.system,
-                       self.reporter_agent,
-                       self.config)
-        self.admin = Admin(self.api)
-        self.configure()
-        self.start_api()
-
-    def configure(self) -> None:
-        cherrypy.config.update({
-            'environment': 'production',
-            'engine.autoreload.on': False,
-        })
-        config = {'/': {
-            'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
-            'tools.trailing_slash.on': False,
-            'tools.auth_basic.realm': 'localhost',
-            'tools.auth_basic.checkpassword': self.check_auth
-        }}
-        cherrypy.tree.mount(self.api, '/', config=config)
-        cherrypy.tree.mount(self.admin, '/admin', config=config)
-        self.api.ssl_certificate = self.__dict__['ssl_crt_path']
-        self.api.ssl_private_key = self.__dict__['ssl_key_path']
-
-    def shutdown(self) -> None:
-        self.log.logger.info("Shutting node-proxy down...")
-        self.cp_shutdown_event.set()
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index 516272c7223e..8bf4bc6befdf 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -1,4 +1,3 @@
-import json
 from .baseredfishsystem import BaseRedfishSystem
 from .util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any, List
@@ -69,42 +68,6 @@ def get_power(self) -> Dict[str, Dict[str, Dict]]:
     def get_fans(self) -> Dict[str, Dict[str, Dict]]:
         return self._sys['fans']
 
-    def get_led(self) -> Dict[str, Any]:
-        endpoint = f"/redfish/v1/{self.chassis_endpoint}"
-        result = self.client.query(method='GET',
-                                   endpoint=endpoint,
-                                   timeout=10)
-        response_json = json.loads(result[1])
-        mapper = {
-            'true': 'on',
-            'false': 'off'
-        }
-        if result[2] == 200:
-            return {"state": mapper[str(response_json['LocationIndicatorActive']).lower()]}
-        else:
-            return {"error": "Couldn't retrieve enclosure LED status."}
-
-    def set_led(self, data: Dict[str, str]) -> int:
-        # '{"IndicatorLED": "Lit"}'      -> LocationIndicatorActive = false
-        # '{"IndicatorLED": "Blinking"}' -> LocationIndicatorActive = true
-        mapper = {
-            "on": 'Blinking',
-            "off": 'Lit'
-        }
-        try:
-            _data = {
-                "IndicatorLED": mapper[data["state"].lower()]
-            }
-            _, response, status = self.client.query(
-                data=json.dumps(_data),
-                method='PATCH',
-                endpoint=f"/redfish/v1{self.chassis_endpoint}"
-            )
-        except KeyError:
-            status = 400
-        result = status
-        return result
-
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
         self.log.logger.debug('Updating network')
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index d5563eee6c74..d5f7d3161cf9 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -13,7 +13,6 @@ class Server:  # type: ignore
 import tempfile
 import threading
 import time
-import base64
 
 from orchestrator import DaemonDescriptionStatus
 from orchestrator._interface import daemon_type_to_service
@@ -25,8 +24,9 @@ class Server:  # type: ignore
 from mgr_util import test_port_allocation, PortAlreadyInUse
 from urllib.request import urlopen, Request
 from urllib.error import HTTPError, URLError
+from contextlib import contextmanager
 
-from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING, Optional
+from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING, Optional, Generator
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -99,10 +99,15 @@ class NodeProxy:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
         self.ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
-        self.ssl_ctx = ssl.create_default_context()
-        self.ssl_ctx.check_hostname = True
-        self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        self.ssl_ctx.load_verify_locations(cadata=self.ssl_root_crt)
+        self.ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        self.ssl_ctx.check_hostname = False
+        self.ssl_ctx.verify_mode = ssl.CERT_NONE
+        # self.ssl_ctx = ssl.create_default_context()
+        # self.ssl_ctx.check_hostname = True
+        # self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        # self.ssl_ctx.load_verify_locations(cadata=self.ssl_root_crt)
+        self.redfish_token: str = ''
+        self.redfish_session_location: str = ''
 
     def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
         if len(vpath) == 2:
@@ -252,15 +257,77 @@ def data(self) -> None:
         self.mgr.node_proxy.save(host, data['patch'])
         self.raise_alert(data)
 
-    def query_endpoint(self,
-                       addr: str = '',
-                       port: str = '',
-                       method: Optional[str] = None,
-                       headers: Optional[Dict[str, str]] = {},
-                       data: Optional[bytes] = None,
-                       endpoint: str = '',
-                       ssl_ctx: Optional[Any] = None,
-                       timeout: Optional[int] = 10) -> Tuple[int, Dict[str, Any]]:
+    def login(self,
+              addr: str,
+              username: str,
+              password: str,
+              port: str = '443') -> None:
+        self.mgr.log.info("Logging in to "
+                          f"{addr}:{port} as '{username}'")
+        oob_credentials = json.dumps({"UserName": username,
+                                      "Password": password})
+        headers = {"Content-Type": "application/json"}
+
+        try:
+            _status_code, _data, _headers = self.query(addr=addr,
+                                                       port=port,
+                                                       data=oob_credentials,
+                                                       headers=headers,
+                                                       endpoint="/redfish/v1/SessionService/Sessions/",
+                                                       method="POST")
+            if _status_code != 201:
+                self.mgr.log.error(f"Can't log in to {addr}:{port} as '{username}': {_status_code}")
+                raise RuntimeError
+            self.redfish_token = _headers['X-Auth-Token']
+            self.redfish_session_location = _headers['Location']
+        except URLError as e:
+            msg = f"Can't log in to {addr}:{port} as '{username}': {e}"
+            self.mgr.log.error(msg)
+            raise RuntimeError
+
+    def logout(self,
+               addr: str,
+               port: str = '443') -> None:
+        try:
+            _status_code, _data, _ = self.query(addr=addr,
+                                                port=port,
+                                                method='DELETE',
+                                                headers={"X-Auth-Token": self.redfish_token},
+                                                endpoint=self.redfish_session_location)
+        except URLError:
+            msg = f"Can't log out from {addr}:{port}"
+            self.mgr.log.error(msg)
+            raise cherrypy.HTTPError(502, msg)
+
+    @contextmanager
+    def redfish_session(self,
+                        addr: str,
+                        username: str,
+                        password: str,
+                        port: str = '443') -> Generator:
+        try:
+            self.login(addr=addr,
+                       port=port,
+                       username=username,
+                       password=password)
+            yield
+        except Exception:
+            raise
+        else:
+            self.logout(addr=addr,
+                        port=port)
+
+    def query(self,
+              addr: str = '',
+              port: str = '',
+              method: Optional[str] = None,
+              headers: Dict[str, str] = {},
+              data: Optional[bytes] = None,
+              endpoint: str = '',
+              ssl_ctx: Optional[Any] = None,
+              timeout: Optional[int] = 10) -> Tuple[int, Dict[str, Any], Dict[str, Any]]:
+        if not ssl_ctx:
+            ssl_ctx = self.ssl_ctx
         url = f'https://{addr}:{port}{endpoint}'
         _headers = headers
         response_json = {}
@@ -272,18 +339,19 @@ def query_endpoint(self,
             req = Request(url, _data, _headers, method=method)
             with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
+                response_headers = response.headers
                 response_json = json.loads(response_str)
                 response_status = response.status
         except HTTPError as e:
-            self.mgr.log.debug(f"{e.code} {e.reason}")
+            self.mgr.log.error(f"{e.code} {e.reason}")
             response_status = e.code
         except URLError as e:
-            self.mgr.log.debug(f"{e.reason}")
+            self.mgr.log.error(f"{e.reason}")
             raise
         except Exception as e:
             self.mgr.log.error(f"{e}")
             raise
-        return (response_status, response_json)
+        return (response_status, response_json, response_headers)
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET', 'PATCH'])
@@ -311,37 +379,37 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         """
         method: str = cherrypy.request.method
         hostname: Optional[str] = kw.get('hostname')
-        headers: Dict[str, str] = {}
 
         if not hostname:
             msg: str = "listing enclosure LED status for all nodes is not implemented."
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(501, msg)
 
-        addr = self.mgr.inventory.get_addr(hostname)
+        addr = self.mgr.node_proxy.oob[hostname]['addr']
+        port = self.mgr.node_proxy.oob[hostname]['port']
+        username = self.mgr.node_proxy.oob[hostname]['username']
+        password = self.mgr.node_proxy.oob[hostname]['password']
 
         if method == 'PATCH':
             # TODO(guits): need to check the request is authorized
             # allowing a specific keyring only ? (client.admin or client.agent.. ?)
             data: str = json.dumps(cherrypy.request.json)
-            username = self.mgr.node_proxy.oob[hostname]['username']
-            password = self.mgr.node_proxy.oob[hostname]['password']
-            auth = f"{username}:{password}".encode("utf-8")
-            auth = base64.b64encode(auth).decode("utf-8")
-            headers = {"Authorization": f"Basic {auth}"}
 
-        try:
-            status, result = self.query_endpoint(data=data,
-                                                headers=headers,
-                                                addr=addr,
-                                                method=method,
-                                                port=8080,
-                                                endpoint='/led',
-                                                ssl_ctx=self.ssl_ctx)
-        except URLError as e:
-            raise cherrypy.HTTPError(502, f"{e}")
-        cherrypy.response.status = status
-        return result
+        with self.redfish_session(addr, username, password, port=port):
+            try:
+                status, result, _ = self.query(data=bytes(data, 'ascii'),
+                                               addr=addr,
+                                               method=method,
+                                               headers={"X-Auth-Token": self.redfish_token},
+                                               port=port,
+                                               endpoint='/redfish/v1/Chassis/System.Embedded.1',
+                                               ssl_ctx=self.ssl_ctx)
+            except (URLError, HTTPError) as e:
+                raise cherrypy.HTTPError(502, f"{e}")
+            if method == 'GET':
+                result = {"LocationIndicatorActive": result['LocationIndicatorActive']}
+            cherrypy.response.status = status
+            return result
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])

From a943217ad3e03635f4e82be6e2e620d87a28562a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 28 Nov 2023 16:28:46 +0000
Subject: [PATCH 1593/2492] node-proxy: update the data structure for summary
 report

This extends the current data structure for the 'summary' report.
It adds `sn` (serial number information) and the `firmwares` dict
to the current data structure.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/inventory.py | 7 +++++--
 src/pybind/mgr/cephadm/module.py    | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 15e81fa57f40..27bf55c921c3 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -1481,11 +1481,14 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
 
         for host in hosts:
             _result[host] = {}
+            _result[host]['status'] = {}
             data = self.data[host]
             for component, details in data['status'].items():
                 res = any([member['status']['health'].lower() != 'ok' for member in data['status'][component].values()])
-                _result[host][component] = mapper[res]
-
+                _result[host]['status'][component] = mapper[res]
+            _result[host]['sn'] = data['sn']
+            _result[host]['host'] = data['host']
+            _result[host]['firmwares'] = data['firmwares']
         return _result
 
     def common(self, endpoint: str, **kw: Any) -> Dict[str, Any]:
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 1bf86a65dd89..648cdc179c0f 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1684,7 +1684,7 @@ def hardware_status(self,
             data = self.node_proxy.summary(hostname=hostname)
             for k, v in data.items():
                 row = [k]
-                row.extend([v[key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
+                row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
                 table.add_row(row)
             output = table.get_string()
         elif category == 'firmwares':

From 9e591b0ed54c8cee387a8280a8e9321ab1b1bee0 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 29 Nov 2023 13:44:55 +0000
Subject: [PATCH 1594/2492] orch/cephadm: add json format support to `ceph orch
 hardware`

This adds `--format json` option support to the `ceph orch hardware` CLI
command.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py      | 43 +++++++++++++++++----------
 src/pybind/mgr/orchestrator/module.py |  4 +--
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 648cdc179c0f..85d5e5b0fbfc 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1674,37 +1674,46 @@ def hardware_status(self,
             'power': ['HOST', 'ID', 'NAME', 'MODEL', 'MANUFACTURER', 'STATUS', 'STATE'],
             'fans': ['HOST', 'ID', 'NAME', 'STATUS', 'STATE']
         }
-        table_headings = table_heading_mapping.get(category, [])
-        table = PrettyTable(
-            table_headings,
-            border=True)
+
         if category not in table_heading_mapping.keys():
             return f"'{category}' is not a valid category."
+
+        table_headings = table_heading_mapping.get(category, [])
+        table = PrettyTable(table_headings,border=True)
+        output = None
+
         if category == 'summary':
             data = self.node_proxy.summary(hostname=hostname)
-            for k, v in data.items():
-                row = [k]
-                row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
-                table.add_row(row)
-            output = table.get_string()
+            if format == Format.json:
+                output = json.dumps(data)
+            else:
+                for k, v in data.items():
+                    row = [k]
+                    row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
+                    table.add_row(row)
+                output = table.get_string()
         elif category == 'firmwares':
-            output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table)
+            output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table, format)
         elif category == 'criticals':
-            output = self._criticals_table(hostname, table)
+            output = self._criticals_table(hostname, table, format)
         else:
-            output = self._common_table(category, hostname, table)
+            output = self._common_table(category, hostname, table, format)
 
-        return output if 'output' in locals() else table.get_string()
+        return output if output else table.get_string()
 
-    def _firmwares_table(self, hostname, table):
+    def _firmwares_table(self, hostname, table, format):
         data = self.node_proxy.firmwares(hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
         for host, details in data.items():
             for k, v in details.items():
                 table.add_row((host, k, v['name'], v['release_date'], v['version'], v['status']['health']))
         return table.get_string()
 
-    def _criticals_table(self, hostname, table):
+    def _criticals_table(self, hostname, table, format):
         data = self.node_proxy.criticals(hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
         for host, host_details in data.items():
             for component, component_details in host_details.items():
                 for member, member_details in component_details.items():
@@ -1712,8 +1721,10 @@ def _criticals_table(self, hostname, table):
                     table.add_row((host, component, description, member_details['status']['health'], member_details['status']['state']))
         return table.get_string()
 
-    def _common_table(self, category, hostname, table):
+    def _common_table(self, category, hostname, table, format):
         data = self.node_proxy.common(endpoint=category, hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
         mapping = {
             'memory': ('description', 'health', 'state'),
             'storage': ('description', 'model', 'capacity_bytes', 'protocol', 'serial_number', 'health', 'state'),
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 9c5e2aec7719..31a219f06b06 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -489,9 +489,9 @@ def _add_host(self,
         return self._apply_misc([s], False, Format.plain)
 
     @_cli_write_command('orch hardware status')
-    def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int = 0, category: str = 'summary') -> HandleCommandResult:
+    def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int = 0, category: str = 'summary', format: Format = Format.plain) -> HandleCommandResult:
         """Display hardware status"""
-        completion = self.hardware_status(hostname, category)
+        completion = self.hardware_status(hostname, category, format)
         raise_if_exception(completion)
         return HandleCommandResult(stdout=completion.result_str())
 

From e9a2450f897aca61181522bf64b215d5295154b0 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 1 Dec 2023 08:03:58 +0000
Subject: [PATCH 1595/2492] cephadm: gracefully shutdown the agent prior to
 removing

When the agent is removed, the daemon is abruptly stopped.
Since the node-proxy logic runs from within the cephadm agent,
it leaves an active RedFish session. The idea is to gracefully
shutdown the agent so node-proxy can catch that event and make sure
it closes the current active RedFish session prior to shutting down.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        | 41 +++++++++++--------
 src/pybind/mgr/cephadm/agent.py               | 19 +++++++--
 .../mgr/cephadm/services/cephadmservice.py    | 10 +++++
 3 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index aa5ae7570531..e7193ccc29b6 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1280,12 +1280,13 @@ def run(self) -> None:
                         conn.send(err_str.encode())
                         logger.error(err_str)
                     else:
-                        conn.send(b'ACK')
-                        if 'config' in data:
-                            self.agent.wakeup()
-                        self.agent.ls_gatherer.wakeup()
-                        self.agent.volume_gatherer.wakeup()
-                        logger.debug(f'Got mgr message {data}')
+                        if 'counter' in data:
+                            conn.send(b'ACK')
+                            if 'config' in data:
+                                self.agent.wakeup()
+                            self.agent.ls_gatherer.wakeup()
+                            self.agent.volume_gatherer.wakeup()
+                            logger.debug(f'Got mgr message {data}')
             except Exception as e:
                 logger.error(f'Mgr Listener encountered exception: {e}')
 
@@ -1293,17 +1294,23 @@ def shutdown(self) -> None:
         self.stop = True
 
     def handle_json_payload(self, data: Dict[Any, Any]) -> None:
-        self.agent.ack = int(data['counter'])
-        if 'config' in data:
-            logger.info('Received new config from mgr')
-            config = data['config']
-            for filename in config:
-                if filename in self.agent.required_files:
-                    file_path = os.path.join(self.agent.daemon_dir, filename)
-                    with write_new(file_path) as f:
-                        f.write(config[filename])
-            self.agent.pull_conf_settings()
-            self.agent.wakeup()
+        if 'counter' in data:
+            self.agent.ack = int(data['counter'])
+            if 'config' in data:
+                logger.info('Received new config from mgr')
+                config = data['config']
+                for filename in config:
+                    if filename in self.agent.required_files:
+                        file_path = os.path.join(self.agent.daemon_dir, filename)
+                        with write_new(file_path) as f:
+                            f.write(config[filename])
+                self.agent.pull_conf_settings()
+                self.agent.wakeup()
+        elif 'node_proxy_shutdown' in data:
+            self.agent.shutdown()
+        else:
+            raise RuntimeError('No valid data received.')
+
 
 
 @register_daemon_form
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index d5f7d3161cf9..03266a6c7a75 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -896,6 +896,16 @@ def _request_agent_acks(self, hosts: Set[str], increment: bool = False, daemon_s
                 host, self.mgr.agent_cache.agent_ports[host], payload, self.mgr, daemon_spec)
             message_thread.start()
 
+    def _shutdown_node_proxy(self) -> None:
+        hosts = set([h for h in self.mgr.cache.get_hosts() if
+                     (h in self.mgr.agent_cache.agent_ports and not self.mgr.agent_cache.messaging_agent(h))])
+
+        for host in hosts:
+            payload: Dict[str, Any] = {'node_proxy_shutdown': host}
+            message_thread = AgentMessageThread(
+                host, self.mgr.agent_cache.agent_ports[host], payload, self.mgr)
+            message_thread.start()
+
     def _request_ack_all_not_up_to_date(self) -> None:
         self.mgr.agent_helpers._request_agent_acks(
             set([h for h in self.mgr.cache.get_hosts() if
@@ -971,10 +981,11 @@ def _handle_use_agent_setting(self) -> bool:
             if 'agent' in self.mgr.spec_store:
                 self.mgr.spec_store.rm('agent')
                 need_apply = True
-            self.mgr.agent_cache.agent_counter = {}
-            self.mgr.agent_cache.agent_timestamp = {}
-            self.mgr.agent_cache.agent_keys = {}
-            self.mgr.agent_cache.agent_ports = {}
+            if not self.mgr.cache.get_daemons_by_service('agent'):
+                self.mgr.agent_cache.agent_counter = {}
+                self.mgr.agent_cache.agent_timestamp = {}
+                self.mgr.agent_cache.agent_keys = {}
+                self.mgr.agent_cache.agent_ports = {}
         return need_apply
 
     def _check_agent(self, host: str) -> bool:
diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py
index f1d405edda0b..1681be003487 100644
--- a/src/pybind/mgr/cephadm/services/cephadmservice.py
+++ b/src/pybind/mgr/cephadm/services/cephadmservice.py
@@ -1236,6 +1236,16 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
 
         return daemon_spec
 
+    def pre_remove(self, daemon: DaemonDescription) -> None:
+        super().pre_remove(daemon)
+
+        assert daemon.daemon_id is not None
+        daemon_id: str = daemon.daemon_id
+
+        logger.info('Removing agent %s...' % daemon_id)
+
+        self.mgr.agent_helpers._shutdown_node_proxy()
+
     def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
         agent = self.mgr.http_server.agent
         try:

From 8d129efd850dff1306c4084f52da84594d8146ef Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 1 Dec 2023 08:11:31 +0000
Subject: [PATCH 1596/2492] node-proxy: Add a `NodeProxyManager` class

The current approach with `init_node_proxy()` and `node_proxy_loop_check()`
is 'cumbersome' and gives the heebie-jeebies.

Sub-classing `Thread()` makes the code a bit more clearer and readable.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                    | 185 +++++++++++-----------
 src/cephadm/cephadmlib/agent.py           |  33 ++++
 src/cephadm/cephadmlib/node_proxy/main.py |  14 +-
 src/cephadm/tests/test_agent.py           |   4 +-
 4 files changed, 133 insertions(+), 103 deletions(-)
 create mode 100644 src/cephadm/cephadmlib/agent.py

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index e7193ccc29b6..879aec568b5b 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -28,9 +28,8 @@
 from glob import glob
 from io import StringIO
 from threading import Thread, Event
-from urllib.request import urlopen, Request
 from pathlib import Path
-from cephadmlib.node_proxy.main import NodeProxy, NodeProxyInitialization, NodeProxyFetchOobError
+from cephadmlib.node_proxy.main import NodeProxy
 
 from cephadmlib.constants import (
     # default images
@@ -178,6 +177,7 @@
     SNMPGateway,
     Tracing,
 )
+from cephadmlib.agent import http_query
 
 
 FuncT = TypeVar('FuncT', bound=Callable)
@@ -1312,6 +1312,71 @@ def handle_json_payload(self, data: Dict[Any, Any]) -> None:
             raise RuntimeError('No valid data received.')
 
 
+class NodeProxyManager(Thread):
+    def __init__(self, agent: 'CephadmAgent', event: Event):
+        super().__init__()
+        self.agent = agent
+        self.event = event
+        self.stop = False
+
+    def run(self) -> None:
+        self.event.wait()
+        self.ssl_ctx = self.agent.ssl_ctx
+        self.init()
+        self.loop()
+
+    def init(self) -> None:
+        node_proxy_meta = {
+            'cephx': {
+                'name': self.agent.host,
+                'secret': self.agent.keyring
+            }
+        }
+        status, result = http_query(addr=self.agent.target_ip,
+                                    port=self.agent.target_port,
+                                    data=json.dumps(node_proxy_meta).encode('ascii'),
+                                    endpoint='/node-proxy/oob',
+                                    ssl_ctx=self.ssl_ctx)
+        if status != 200:
+            msg = f'No out of band tool details could be loaded: {status}, {result}'
+            logger.debug(msg)
+            raise RuntimeError(msg)
+
+        result_json = json.loads(result)
+        kwargs = {
+            'host': result_json['result']['addr'],
+            'username': result_json['result']['username'],
+            'password': result_json['result']['password'],
+            'cephx': node_proxy_meta['cephx'],
+            'mgr_target_ip': self.agent.target_ip,
+            'mgr_target_port': self.agent.target_port
+        }
+        if result_json['result'].get('port'):
+            kwargs['port'] = result_json['result']['port']
+
+        self.node_proxy = NodeProxy(**kwargs)
+        self.node_proxy.start()
+
+    def loop(self) -> None:
+        while not self.stop:
+            try:
+                status = self.node_proxy.check_status()
+                label = 'Ok' if status else 'Critical'
+                logger.debug(f'node-proxy status: {label}')
+            except Exception as e:
+                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
+                time.sleep(120)
+                self.init()
+            else:
+                logger.debug('node-proxy alive, next check in 60sec.')
+                time.sleep(60)
+
+    def shutdown(self) -> None:
+        self.stop = True
+        # if `self.node_proxy.shutdown()` is called before self.start(), it will fail.
+        if self.__dict__.get('node_proxy'):
+            self.node_proxy.shutdown()
+
 
 @register_daemon_form
 class CephadmAgent(DaemonForm):
@@ -1365,7 +1430,11 @@ def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str] =
         self.recent_iteration_run_times: List[float] = [0.0, 0.0, 0.0]
         self.recent_iteration_index: int = 0
         self.cached_ls_values: Dict[str, Dict[str, str]] = {}
-        self.t_node_proxy: Optional["NodeProxy"] = None
+        self.ssl_ctx = ssl.create_default_context()
+        self.ssl_ctx.check_hostname = True
+        self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        self.node_proxy_mgr_event = Event()
+        self.node_proxy_mgr = NodeProxyManager(self, self.node_proxy_mgr_event)
 
     def validate(self, config: Dict[str, str] = {}) -> None:
         # check for the required files
@@ -1421,6 +1490,8 @@ def unit_file(self) -> str:
 
     def shutdown(self) -> None:
         self.stop = True
+        if self.node_proxy_mgr.is_alive():
+            self.node_proxy_mgr.shutdown()
         if self.mgr_listener.is_alive():
             self.mgr_listener.shutdown()
         if self.ls_gatherer.is_alive():
@@ -1459,92 +1530,12 @@ def pull_conf_settings(self) -> None:
             self.device_enhanced_scan = True
         self.volume_gatherer.update_func(lambda: self._ceph_volume(enhanced=self.device_enhanced_scan))
 
-    def query_endpoint(self,
-                       addr: str = '',
-                       port: str = '',
-                       data: Optional[Union[Dict[str, str], str]] = None,
-                       endpoint: str = '',
-                       ssl_ctx: Optional[Any] = None,
-                       timeout: Optional[int] = 10) -> Tuple[int, str]:
-        _addr = addr if addr else self.target_ip
-        _port = port if port else self.target_port
-        url = f'https://{_addr}:{_port}{endpoint}'
-        logger.info(f"sending query to {url}")
-        try:
-            req = Request(url, data, {'Content-Type': 'application/json'})
-            send_time = time.monotonic()
-            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
-                response_str = response.read()
-                response_json = json.loads(response_str)
-                total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
-                logger.info(f'Received mgr response: "{response_json["result"]}" {total_request_time} seconds after sending request.')
-                response_status = response.status
-        except HTTPError as e:
-            logger.debug(f"{e.code} {e.reason}")
-            response_status = e.code
-            response_str = e.reason
-        except URLError as e:
-            logger.debug(f"{e.reason}")
-            response_status = -1
-            response_str = e.reason
-        except Exception:
-            raise
-        return (response_status, response_str)
-
-    def node_proxy_loop_check(self, ssl_ctx: Any) -> None:
-        while True:
-            try:
-                if isinstance(self.t_node_proxy, NodeProxy):
-                    status = self.t_node_proxy.check_status()
-                    label = 'Ok' if status else 'Critical'
-                    logger.debug(f'node-proxy status: {label}')
-                else:
-                    raise NodeProxyInitialization("starting node-proxy...")
-            except Exception as e:
-                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
-                try:
-                    self.init_node_proxy(ssl_ctx)
-                except NodeProxyFetchOobError:
-                    logger.info("No oob details could be loaded. "
-                                "Aborting node-proxy initialization. "
-                                "Will retry in 120s.")
-                    time.sleep(120)
-
-    def init_node_proxy(self, ssl_ctx: Any) -> None:
-        node_proxy_meta = {
-            'cephx': {
-                'name': self.host,
-                'secret': self.keyring
-            }
-        }
-        status, result = self.query_endpoint(data=json.dumps(node_proxy_meta).encode('ascii'),
-                                             endpoint='/node-proxy/oob',
-                                             ssl_ctx=ssl_ctx)
-        if status != 200:
-            msg = f"Couldn't load oob details: {status}, {result}"
-            logger.debug(msg)
-            raise NodeProxyFetchOobError(msg)
-        result_json = json.loads(result)
-        kwargs = {
-            'host': result_json['result']['addr'],
-            'username': result_json['result']['username'],
-            'password': result_json['result']['password'],
-            'cephx': node_proxy_meta['cephx'],
-            'mgr_target_ip': self.target_ip,
-            'mgr_target_port': self.target_port
-        }
-        if result_json['result'].get('port'):
-            kwargs['port'] = result_json['result']['port']
-
-        self.t_node_proxy = NodeProxy(**kwargs)
-        self.t_node_proxy.start()
-
     def run(self) -> None:
         self.pull_conf_settings()
-        ssl_ctx = ssl.create_default_context()
-        ssl_ctx.check_hostname = True
-        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        ssl_ctx.load_verify_locations(self.ca_path)
+        self.ssl_ctx.load_verify_locations(self.ca_path)
+        # only after self.pull_conf_settings() was called we can actually start
+        # node-proxy
+        self.node_proxy_mgr_event.set()
 
         try:
             for _ in range(1001):
@@ -1566,9 +1557,8 @@ def run(self) -> None:
         if not self.volume_gatherer.is_alive():
             self.volume_gatherer.start()
 
-        # initiate node-proxy thread
-        node_proxy_loop_thread = Thread(target=self.node_proxy_loop_check, args=(ssl_ctx,))
-        node_proxy_loop_thread.start()
+        if not self.node_proxy_mgr.is_alive():
+            self.node_proxy_mgr.start()
 
         while not self.stop:
             start_time = time.monotonic()
@@ -1596,9 +1586,18 @@ def run(self) -> None:
             data = data.encode('ascii')
 
             try:
-                self.query_endpoint(data=data,
-                                    endpoint='/data',
-                                    ssl_ctx=ssl_ctx)
+                send_time = time.monotonic()
+                status, response = http_query(addr=self.target_ip,
+                                              port=self.target_port,
+                                              data=data,
+                                              endpoint='/data',
+                                              ssl_ctx=self.ssl_ctx)
+                response_json = json.loads(response)
+                if status != 200:
+                    logger.error(f'HTTP error {status} while querying agent endpoint: {response}')
+                    raise RuntimeError
+                total_request_time = datetime.timedelta(seconds=(time.monotonic() - send_time)).total_seconds()
+                logger.info(f'Received mgr response: "{response_json["result"]}" {total_request_time} seconds after sending request.')
             except Exception as e:
                 logger.error(f'Failed to send metadata to mgr: {e}')
 
diff --git a/src/cephadm/cephadmlib/agent.py b/src/cephadm/cephadmlib/agent.py
new file mode 100644
index 000000000000..71924c39ccfe
--- /dev/null
+++ b/src/cephadm/cephadmlib/agent.py
@@ -0,0 +1,33 @@
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen, Request
+from typing import Optional, Any, Tuple
+import logging
+
+logger = logging.getLogger()
+
+
+def http_query(addr: str = '',
+               port: str = '',
+               data: Optional[bytes] = None,
+               endpoint: str = '',
+               ssl_ctx: Optional[Any] = None,
+               timeout: Optional[int] = 10) -> Tuple[int, str]:
+
+    url = f'https://{addr}:{port}{endpoint}'
+    logger.debug(f'sending query to {url}')
+    try:
+        req = Request(url, data, {'Content-Type': 'application/json'})
+        with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
+            response_str = response.read()
+            response_status = response.status
+    except HTTPError as e:
+        logger.debug(f'{e.code} {e.reason}')
+        response_status = e.code
+        response_str = e.reason
+    except URLError as e:
+        logger.debug(f'{e.reason}')
+        response_status = -1
+        response_str = e.reason
+    except Exception:
+        raise
+    return (response_status, response_str)
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 0575340d0ecd..339d0d2c8533 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -23,14 +23,6 @@
 }
 
 
-class NodeProxyInitialization(Exception):
-    pass
-
-
-class NodeProxyFetchOobError(Exception):
-    pass
-
-
 class NodeProxy(Thread):
     def __init__(self, **kw: Dict[str, Any]) -> None:
         super().__init__()
@@ -46,6 +38,12 @@ def run(self) -> None:
             self.exc = e
             return
 
+    def shutdown(self) -> None:
+        self.log.logger.info('Shutting down node-proxy...')
+        self.system.client.logout()
+        self.system.stop_update_loop()
+        self.reporter_agent.stop()
+
     def check_auth(self, realm: str, username: str, password: str) -> bool:
         return self.__dict__['username'] == username and \
             self.__dict__['password'] == password
diff --git a/src/cephadm/tests/test_agent.py b/src/cephadm/tests/test_agent.py
index 4904cb4f61fc..52cce74e1fb8 100644
--- a/src/cephadm/tests/test_agent.py
+++ b/src/cephadm/tests/test_agent.py
@@ -416,7 +416,7 @@ def test_agent_get_ls(_ls_subset, _ls, cephadm_fs):
 @mock.patch("threading.Event.clear")
 @mock.patch("threading.Event.wait")
 @mock.patch("urllib.request.Request.__init__")
-@mock.patch("cephadm.urlopen")
+@mock.patch("cephadmlib.agent.urlopen")
 @mock.patch("cephadm.list_networks")
 @mock.patch("cephadm.HostFacts.dump")
 @mock.patch("cephadm.HostFacts.__init__", lambda _, __: None)
@@ -531,7 +531,7 @@ class EventCleared(Exception):
            'port': str(open_listener_port)
         }
         _RQ_init.assert_called_with(
-            f'https://{target_ip}:{target_port}/data/',
+            f'https://{target_ip}:{target_port}/data',
             json.dumps(expected_data).encode('ascii'),
             {'Content-Type': 'application/json'}
         )

From d9ce32796db4ba987b9ea00883f8f379d94d9608 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 1 Dec 2023 08:18:25 +0000
Subject: [PATCH 1597/2492] node-proxy: code change for hdd blinkenlight
 pre-requisites

This is mainly for anticipating the case where hdd blinkenlight via RedFish
works (testing has to be done). This introduces the required changes so the
endpoint `/led` can support blinkenlight for both chassis and disks.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../node_proxy/baseredfishsystem.py           |   3 +-
 .../node_proxy/redfishdellsystem.py           |   1 +
 src/pybind/mgr/cephadm/agent.py               |  51 ++++++-
 .../mgr/cephadm/tests/test_node_proxy.py      | 129 ++++++++++++------
 4 files changed, 139 insertions(+), 45 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index c4675e5b8f03..44bb3427b713 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -116,7 +116,8 @@ def get_system(self) -> Dict[str, Any]:
                 'power': self.get_power(),
                 'fans': self.get_fans()
             },
-            'firmwares': self.get_firmwares()
+            'firmwares': self.get_firmwares(),
+            'chassis': {'redfish_endpoint': f'/redfish/v1{self.chassis_endpoint}'}  # TODO(guits): not ideal
         }
         return result
 
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index 8bf4bc6befdf..b41ade2e68fd 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -104,6 +104,7 @@ def _update_storage(self) -> None:
                 drive_info = self._get_path(drive_path)
                 drive_id = drive_info['Id']
                 result[drive_id] = dict()
+                result[drive_id]['redfish_endpoint'] = drive['@odata.id']
                 for field in fields:
                     result[drive_id][to_snake_case(field)] = drive_info[field]
                     result[drive_id]['entity'] = entity['Id']
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 03266a6c7a75..b760fcfb93d1 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -110,9 +110,19 @@ def __init__(self, mgr: "CephadmOrchestrator"):
         self.redfish_session_location: str = ''
 
     def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
-        if len(vpath) == 2:
-            hostname = vpath.pop(0)
+        if len(vpath) > 1:  # /{hostname}/<endpoint>
+            hostname = vpath.pop(0)  # /<endpoint>
             cherrypy.request.params['hostname'] = hostname
+            # /{hostname}/led/{type}/{drive} eg: /{hostname}/led/chassis or /{hostname}/led/drive/{id}
+            if vpath[0] == 'led' and len(vpath) > 1:  # /led/{type}/{id}
+                _type = vpath[1]
+                cherrypy.request.params['type'] = _type
+                vpath.pop(1)  # /led/{id} or # /led
+                if _type == 'drive' and len(vpath) > 1:  # /led/{id}
+                    _id = vpath[1]
+                    vpath.pop(1)  # /led
+                    cherrypy.request.params['id'] = _id
+        # /<endpoint>
         return self
 
     @cherrypy.expose
@@ -331,6 +341,7 @@ def query(self,
         url = f'https://{addr}:{port}{endpoint}'
         _headers = headers
         response_json = {}
+        response_headers = {}
         if not _headers.get('Content-Type'):
             # default to application/json if nothing provided
             _headers['Content-Type'] = 'application/json'
@@ -379,12 +390,34 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         """
         method: str = cherrypy.request.method
         hostname: Optional[str] = kw.get('hostname')
+        led_type: Optional[str] = kw.get('type')
+        id_drive: Optional[str] = kw.get('id')
 
         if not hostname:
             msg: str = "listing enclosure LED status for all nodes is not implemented."
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(501, msg)
 
+        if not led_type:
+            msg = "the led type must be provided (either 'chassis' or 'drive')."
+            self.mgr.log.debug(msg)
+            raise cherrypy.HTTPError(400, msg)
+
+        if led_type == 'drive' and not id_drive:
+            msg = "the id of the drive must be provided when type is 'drive'."
+            self.mgr.log.debug(msg)
+            raise cherrypy.HTTPError(400, msg)
+
+        if hostname not in self.mgr.node_proxy.data.keys():
+            # TODO(guits): update unit test for this
+            msg = f"'{hostname}' not found."
+            self.mgr.log.debug(msg)
+            raise cherrypy.HTTPError(400, msg)
+
+        # if led_type not in ['chassis', 'drive']:
+        #     # TODO(guits): update unit test for this
+        #     raise cherrypy.HTTPError(404, 'LED type must be either "chassis" or "drive"')
+
         addr = self.mgr.node_proxy.oob[hostname]['addr']
         port = self.mgr.node_proxy.oob[hostname]['port']
         username = self.mgr.node_proxy.oob[hostname]['username']
@@ -395,6 +428,16 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             # allowing a specific keyring only ? (client.admin or client.agent.. ?)
             data: str = json.dumps(cherrypy.request.json)
 
+            if led_type == 'drive':
+                if id_drive not in self.mgr.node_proxy.data[hostname]['status']['storage'].keys():
+                    # TODO(guits): update unit test for this
+                    msg = f"'{id_drive}' not found."
+                    self.mgr.log.debug(msg)
+                    raise cherrypy.HTTPError(400, msg)
+                endpoint = self.mgr.node_proxy.data[hostname]['status']['storage'][id_drive].get('redfish_endpoint')
+            else:
+                endpoint = self.mgr.node_proxy.data[hostname]['chassis']['redfish_endpoint']
+
         with self.redfish_session(addr, username, password, port=port):
             try:
                 status, result, _ = self.query(data=bytes(data, 'ascii'),
@@ -402,9 +445,9 @@ def led(self, **kw: Any) -> Dict[str, Any]:
                                                method=method,
                                                headers={"X-Auth-Token": self.redfish_token},
                                                port=port,
-                                               endpoint='/redfish/v1/Chassis/System.Embedded.1',
+                                               endpoint=endpoint,
                                                ssl_ctx=self.ssl_ctx)
-            except (URLError, HTTPError) as e:
+            except (URLError, HTTPError, RuntimeError) as e:
                 raise cherrypy.HTTPError(502, f"{e}")
             if method == 'GET':
                 result = {"LocationIndicatorActive": result['LocationIndicatorActive']}
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index fce7152bd86b..1ca8b762cd39 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -4,11 +4,9 @@
 from _pytest.monkeypatch import MonkeyPatch
 from cherrypy.test import helper
 from cephadm.agent import NodeProxy
-from unittest.mock import MagicMock, call
-from cephadm.http_server import CephadmHttpServer
+from unittest.mock import MagicMock, call, patch
 from cephadm.inventory import AgentCache, NodeProxyCache, Inventory
 from cephadm.ssl_cert_utils import SSLCerts
-from urllib.error import URLError
 from . import node_proxy_data
 
 PORT = 58585
@@ -23,6 +21,7 @@ def __init__(self) -> None:
         self.remove_health_warning = MagicMock()
         self.inventory = Inventory(self)
         self.agent_cache = AgentCache(self)
+        self.agent_cache.agent_ports = {"host01": 1234}
         self.node_proxy = NodeProxyCache(self)
         self.node_proxy.save = MagicMock()
         self.http_server = MagicMock()
@@ -133,48 +132,98 @@ def test_led_PATCH_no_hostname(self):
                                                                  ('Content-Length', str(len(data)))])
         self.assertStatus('501 Not Implemented')
 
-    def test_set_led(self):
-        data = '{"state": "on"}'
-        TestNodeProxy.app.query_endpoint = MagicMock(return_value=(200, "OK"))
-        # self.monkeypatch.setattr(NodeProxy, "query_endpoint", lambda *a, **kw: (200, "OK"))
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_set_led_no_type(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        data = '{"IndicatorLED": "Blinking"}'
         self.getPage("/host01/led", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
                                                                         ('Content-Length', str(len(data)))])
+        self.assertStatus('400 Bad Request')
 
-        calls = [call(addr='10.10.10.11',
-                      data='{"state": "on"}',
-                      endpoint='/led',
-                      headers={'Authorization': 'Basic aWRyYWMtdXNlcjAxOmlkcmFjLXBhc3MwMQ=='},
-                      method='PATCH',
-                      port=8080,
-                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
-        self.assertStatus('200 OK')
-        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
-
-    def test_get_led(self):
-        TestNodeProxy.app.query_endpoint = MagicMock(return_value=(200, "OK"))
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_set_chassis_led(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        data = '{"IndicatorLED": "Blinking"}'
+        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
+            a.return_value = '{"http_code": 200}'
+            self.getPage("/host01/led/chassis", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
+                                                                                    ('Content-Length', str(len(data)))])
+            self.assertStatus('200 OK')
+
+    def test_get_led_missing_type(self):
         self.getPage("/host01/led", method="GET")
-        calls = [call(addr='10.10.10.11',
-                      data=None,
-                      endpoint='/led',
-                      headers={},
-                      method='GET',
-                      port=8080,
-                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
-        self.assertStatus('200 OK')
-        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
-
-    def test_led_endpoint_unreachable(self):
-        TestNodeProxy.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
-        self.getPage("/host02/led", method="GET")
-        calls = [call(addr='10.10.10.12',
-                      data=None,
-                      endpoint='/led',
-                      headers={},
-                      method='GET',
-                      port=8080,
-                      ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+        self.assertStatus('400 Bad Request')
+
+    def test_get_led_no_hostname(self):
+        self.getPage("/led", method="GET")
+        self.assertStatus('501 Not Implemented')
+
+    def test_get_led_type_chassis_no_hostname(self):
+        self.getPage("/led/chassis", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_get_led_type_drive_no_hostname(self):
+        self.getPage("/led/chassis", method="GET")
+        self.assertStatus('404 Not Found')
+
+    def test_get_led_type_drive_missing_id(self):
+        self.getPage("/host01/led/drive", method="GET")
+        self.assertStatus('400 Bad Request')
+
+    def test_get_led_type_chassis_answer_invalid_json(self):
+        self.getPage("/host01/led/chassis", method="GET")
+        self.assertStatus('503 Service Unavailable')
+
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_get_led_type_chassis_answer_no_http_code(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
+            a.return_value = '{"foo": "bar"}'
+            self.getPage("/host01/led/chassis", method="GET")
+            self.assertStatus('503 Service Unavailable')
+
+    def test_get_led_status_not_200(self):
+        self.getPage("/host01/led/chassis", method="GET")
+        self.assertStatus('503 Service Unavailable')
+
+    def test_get_led_key_error(self):
+        self.getPage("/host02/led/chassis", method="GET")
         self.assertStatus('502 Bad Gateway')
-        assert TestNodeProxy.app.query_endpoint.mock_calls == calls
+
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_get_chassis_led_ok(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
+            a.return_value = '{"http_code": 200}'
+            self.getPage("/host01/led/chassis", method="GET")
+            self.assertStatus('200 OK')
+
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_get_drive_led_without_id(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        self.getPage("/host01/led/drive", method="GET")
+        self.assertStatus('400 Bad Request')
+
+    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
+    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
+    def test_get_drive_led_with_id(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
+            a.return_value = '{"http_code": 200}'
+            self.getPage("/host01/led/drive/123", method="GET")
+            self.assertStatus('200 OK')
+
+    # def test_led_endpoint_unreachable(self):
+    #     TestNodeProxy.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
+    #     self.getPage("/host02/led", method="GET")
+    #     calls = [call(addr='10.10.10.12',
+    #                   data=None,
+    #                   endpoint='/led',
+    #                   headers={},
+    #                   method='GET',
+    #                   port=8080,
+    #                   ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+    #     self.assertStatus('502 Bad Gateway')
+    #     assert TestNodeProxy.app.query_endpoint.mock_calls == calls
 
     def test_fullreport_with_valid_hostname(self):
         self.getPage("/host02/fullreport", method="GET")

From 45ce4989166dc73467a6bf8a4b59cb94a66f9314 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 1 Dec 2023 08:56:23 +0000
Subject: [PATCH 1598/2492] node-proxy: address flake8 'Q000' warnings

This addresses the flake8 warning 'Q000':

`Q000 Double quotes found but single quotes preferred`

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../node_proxy/baseredfishsystem.py           | 24 +++++++-------
 src/cephadm/cephadmlib/node_proxy/main.py     |  6 ++--
 .../cephadmlib/node_proxy/redfish_client.py   | 32 +++++++++----------
 .../node_proxy/redfishdellsystem.py           |  6 ++--
 src/cephadm/cephadmlib/node_proxy/reporter.py | 12 +++----
 src/cephadm/cephadmlib/node_proxy/util.py     |  4 +--
 6 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 44bb3427b713..587974f8db20 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -20,7 +20,7 @@ def __init__(self, **kw: Any) -> None:
         self.password: str = kw['password']
         # move the following line (class attribute?)
         self.client: RedFishClient = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
-        self.log.logger.info(f"redfish system initialization, host: {self.host}, user: {self.username}")
+        self.log.logger.info(f'redfish system initialization, host: {self.host}, user: {self.username}')
 
         self.run: bool = False
         self.thread: Thread
@@ -49,9 +49,9 @@ def update(self) -> None:
         #  this loop can have:
         #  - caching logic
         while self.run:
-            self.log.logger.debug("waiting for a lock.")
+            self.log.logger.debug('waiting for a lock.')
             self.lock.acquire()
-            self.log.logger.debug("lock acquired.")
+            self.log.logger.debug('lock acquired.')
             try:
                 self._update_system()
                 self._update_sn()
@@ -70,23 +70,23 @@ def update(self) -> None:
                 sleep(5)
             except RuntimeError as e:
                 self.run = False
-                self.log.logger.error(f"Error detected, trying to gracefully log out from redfish api.\n{e}")
+                self.log.logger.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
                 self.client.logout()
             finally:
                 self.lock.release()
-                self.log.logger.debug("lock released.")
+                self.log.logger.debug('lock released.')
 
     def flush(self) -> None:
-        self.log.logger.info("Acquiring lock to flush data.")
+        self.log.logger.info('Acquiring lock to flush data.')
         self.lock.acquire()
-        self.log.logger.info("Lock acquired, flushing data.")
+        self.log.logger.info('Lock acquired, flushing data.')
         self._system = {}
         self.previous_data = {}
-        self.log.logger.info("Data flushed.")
+        self.log.logger.info('Data flushed.')
         self.data_ready = False
-        self.log.logger.info("Data marked as not ready.")
+        self.log.logger.info('Data marked as not ready.')
         self.lock.release()
-        self.log.logger.info("Lock released.")
+        self.log.logger.info('Lock released.')
 
     @retry(retries=10, delay=2)
     def _get_path(self, path: str) -> Dict:
@@ -95,8 +95,8 @@ def _get_path(self, path: str) -> Dict:
         except RuntimeError:
             raise
         if result is None:
-            self.log.logger.error(f"The client reported an error when getting path: {path}")
-            raise RuntimeError(f"Could not get path: {path}")
+            self.log.logger.error(f'The client reported an error when getting path: {path}')
+            raise RuntimeError(f'Could not get path: {path}')
         return result
 
     def get_members(self, data: Dict[str, Any], path: str) -> List:
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 339d0d2c8533..813b3e3edf84 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -50,10 +50,10 @@ def check_auth(self, realm: str, username: str, password: str) -> bool:
 
     def check_status(self) -> bool:
         if self.__dict__.get('system') and not self.system.run:
-            raise RuntimeError("node-proxy encountered an error.")
+            raise RuntimeError('node-proxy encountered an error.')
         if self.exc:
             traceback.print_tb(self.exc.__traceback__)
-            self.log.logger.error(f"{self.exc.__class__.__name__}: {self.exc}")
+            self.log.logger.error(f'{self.exc.__class__.__name__}: {self.exc}')
             raise self.exc
         return True
 
@@ -63,7 +63,7 @@ def main(self) -> None:
         self.log = Logger(__name__, level=self.config.__dict__['logging']['level'])
 
         # create the redfish system and the obsever
-        self.log.logger.info(f"Server initialization...")
+        self.log.logger.info('Server initialization...')
         try:
             self.system = RedfishDellSystem(host=self.__dict__['host'],
                                             port=self.__dict__.get('port', 443),
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index dec35a2c5728..c07e1a6b6f39 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -10,26 +10,26 @@ class RedFishClient(BaseClient):
     PREFIX = '/redfish/v1/'
 
     def __init__(self,
-                 host: str = "",
+                 host: str = '',
                  port: int = 443,
-                 username: str = "",
-                 password: str = ""):
+                 username: str = '',
+                 password: str = ''):
         super().__init__(host, username, password)
         self.log: Logger = Logger(__name__)
-        self.log.logger.info(f"Initializing redfish client {__name__}")
+        self.log.logger.info(f'Initializing redfish client {__name__}')
         self.host: str = host
         self.port: int = port
-        self.url: str = f"https://{self.host}:{self.port}"
+        self.url: str = f'https://{self.host}:{self.port}'
         self.token: str = ''
         self.location: str = ''
 
     def login(self) -> None:
         if not self.is_logged_in():
-            self.log.logger.info("Logging in to "
+            self.log.logger.info('Logging in to '
                                  f"{self.url} as '{self.username}'")
-            oob_credentials = json.dumps({"UserName": self.username,
-                                          "Password": self.password})
-            headers = {"Content-Type": "application/json"}
+            oob_credentials = json.dumps({'UserName': self.username,
+                                          'Password': self.password})
+            headers = {'Content-Type': 'application/json'}
 
             try:
                 _headers, _data, _status_code = self.query(data=oob_credentials,
@@ -46,24 +46,24 @@ def login(self) -> None:
             self.location = _headers['Location']
 
     def is_logged_in(self) -> bool:
-        self.log.logger.debug(f"Checking token validity for {self.url}")
+        self.log.logger.debug(f'Checking token validity for {self.url}')
         if not self.location or not self.token:
-            self.log.logger.debug(f"No token found for {self.url}.")
+            self.log.logger.debug(f'No token found for {self.url}.')
             return False
-        headers = {"X-Auth-Token": self.token}
+        headers = {'X-Auth-Token': self.token}
         try:
             _headers, _data, _status_code = self.query(headers=headers,
                                                        endpoint=self.location)
         except URLError as e:
             self.log.logger.error("Can't check token "
-                                  f"validity for {self.url}: {e}")
+                                  f'validity for {self.url}: {e}')
             raise RuntimeError
         return _status_code == 200
 
     def logout(self) -> Dict[str, Any]:
         try:
             _, _data, _status_code = self.query(method='DELETE',
-                                                headers={"X-Auth-Token": self.token},
+                                                headers={'X-Auth-Token': self.token},
                                                 endpoint=self.location)
         except URLError:
             self.log.logger.error(f"Can't log out from {self.url}")
@@ -75,7 +75,7 @@ def logout(self) -> Dict[str, Any]:
 
     def get_path(self, path: str) -> Dict[str, Any]:
         if self.PREFIX not in path:
-            path = f"{self.PREFIX}{path}"
+            path = f'{self.PREFIX}{path}'
         try:
             _, result, _status_code = self.query(endpoint=path)
             result_json = json.loads(result)
@@ -108,5 +108,5 @@ def query(self,
 
             return response_headers, response_str, response_status
         except (HTTPError, URLError) as e:
-            self.log.logger.debug(f"{e}")
+            self.log.logger.debug(f'{e}')
             raise
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index b41ade2e68fd..12d2466a88ff 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -20,7 +20,7 @@ def build_common_data(self,
                 try:
                     result[member_id][to_snake_case(field)] = member_info[field]
                 except KeyError:
-                    self.log.logger.warning(f"Could not find field: {field} in member_info: {member_info}")
+                    self.log.logger.warning(f'Could not find field: {field} in member_info: {member_info}')
 
         return normalize_dict(result)
 
@@ -28,7 +28,7 @@ def build_chassis_data(self,
                            fields: Dict[str, List[str]],
                            path: str) -> Dict[str, Dict[str, Dict]]:
         result: Dict[str, Dict[str, Dict]] = dict()
-        data = self._get_path(f"{self.chassis_endpoint}/{path}")
+        data = self._get_path(f'{self.chassis_endpoint}/{path}')
 
         for elt, _fields in fields.items():
             for member_elt in data[elt]:
@@ -38,7 +38,7 @@ def build_chassis_data(self,
                     try:
                         result[_id][to_snake_case(field)] = member_elt[field]
                     except KeyError:
-                        self.log.logger.warning(f"Could not find field: {field} in data: {data[elt]}")
+                        self.log.logger.warning(f'Could not find field: {field} in data: {data[elt]}')
         return normalize_dict(result)
 
     def get_sn(self) -> str:
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index aa0ecdf93f98..21183c980357 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -24,8 +24,8 @@ def __init__(self,
         self.reporter_port: int = reporter_port
         self.reporter_endpoint: str = reporter_endpoint
         self.log = Logger(__name__)
-        self.reporter_url: str = (f"{reporter_scheme}:{reporter_hostname}:"
-                                  f"{reporter_port}{reporter_endpoint}")
+        self.reporter_url: str = (f'{reporter_scheme}:{reporter_hostname}:'
+                                  f'{reporter_port}{reporter_endpoint}')
         self.log.logger.info(f'Reporter url set to {self.reporter_url}')
 
     def stop(self) -> None:
@@ -43,9 +43,9 @@ def loop(self) -> None:
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
-            self.log.logger.debug("waiting for a lock.")
+            self.log.logger.debug('waiting for a lock.')
             self.system.lock.acquire()
-            self.log.logger.debug("lock acquired.")
+            self.log.logger.debug('lock acquired.')
             if self.system.data_ready:
                 self.log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
@@ -53,7 +53,7 @@ def loop(self) -> None:
                     self.data['patch'] = self.system.get_system()
                     try:
                         # TODO: add a timeout parameter to the reporter in the config file
-                        self.log.logger.info(f"sending data to {self.reporter_url}")
+                        self.log.logger.info(f'sending data to {self.reporter_url}')
                         http_req(hostname=self.reporter_hostname,
                                  port=self.reporter_port,
                                  method='POST',
@@ -70,5 +70,5 @@ def loop(self) -> None:
                 else:
                     self.log.logger.info('no diff, not sending data to the mgr.')
             self.system.lock.release()
-            self.log.logger.debug("lock released.")
+            self.log.logger.debug('lock released.')
             time.sleep(5)
diff --git a/src/cephadm/cephadmlib/node_proxy/util.py b/src/cephadm/cephadmlib/node_proxy/util.py
index da46ebabda0d..f154d83daafb 100644
--- a/src/cephadm/cephadmlib/node_proxy/util.py
+++ b/src/cephadm/cephadmlib/node_proxy/util.py
@@ -92,12 +92,12 @@ def _retry(*args: str, **kwargs: Any) -> Callable:
             _tries = retries
             while _tries > 1:
                 try:
-                    log.logger.debug("{} {} attempt(s) left.".format(f, _tries - 1))
+                    log.logger.debug('{} {} attempt(s) left.'.format(f, _tries - 1))
                     return f(*args, **kwargs)
                 except exceptions:
                     time.sleep(delay)
                     _tries -= 1
-            log.logger.warn("{} has failed after {} tries".format(f, retries))
+            log.logger.warn('{} has failed after {} tries'.format(f, retries))
             return f(*args, **kwargs)
         return _retry
     return decorator

From e45cf32511750911bc485f901b0d4894fdb3f95b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 6 Dec 2023 12:27:46 +0000
Subject: [PATCH 1599/2492] node-proxy: address a typing issue in
 agent.NodeProxy.query()

The current logic supports str and bytes types for parameter
`data`. This doesn't make sense, let's drop this logic.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index b760fcfb93d1..49f45bffef42 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -281,7 +281,7 @@ def login(self,
         try:
             _status_code, _data, _headers = self.query(addr=addr,
                                                        port=port,
-                                                       data=oob_credentials,
+                                                       data=bytes(oob_credentials, 'ascii'),
                                                        headers=headers,
                                                        endpoint="/redfish/v1/SessionService/Sessions/",
                                                        method="POST")
@@ -345,9 +345,9 @@ def query(self,
         if not _headers.get('Content-Type'):
             # default to application/json if nothing provided
             _headers['Content-Type'] = 'application/json'
-        _data = bytes(data, 'ascii') if data else None
+
         try:
-            req = Request(url, _data, _headers, method=method)
+            req = Request(url, data, _headers, method=method)
             with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
                 response_str = response.read()
                 response_headers = response.headers

From f262579ef0cdebb4bed1b149f11a2db163488707 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 6 Dec 2023 14:25:28 +0000
Subject: [PATCH 1600/2492] node-proxy: move the output formatting logic to
 orchestrator

Implementing this in the cephadm module doesn't follow the general idea
of the orchestrator interface. This is where the output formatting should
be done so let's move the logic to the orchestrator module.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py          | 107 +++-------------------
 src/pybind/mgr/orchestrator/_interface.py |  32 +++++++
 src/pybind/mgr/orchestrator/module.py     | 104 ++++++++++++++++++++-
 3 files changed, 143 insertions(+), 100 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 85d5e5b0fbfc..3029b7b6b5b4 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1653,105 +1653,20 @@ def add_host(self, spec: HostSpec) -> str:
         return self._add_host(spec)
 
     @handle_orch_error
-    def hardware_status(self,
-                        hostname: Optional[str] = None,
-                        category: str = 'summary',
-                        format: Format = Format.plain) -> str:
-        """
-        Display hardware status summary
-
-        :param hostname: hostname
-        """
-
-        table_heading_mapping = {
-            'summary': ['HOST', 'STORAGE', 'CPU', 'NET', 'MEMORY', 'POWER', 'FANS'],
-            'firmwares': ['HOST', 'COMPONENT', 'NAME', 'DATE', 'VERSION', 'STATUS'],
-            'criticals': ['HOST', 'COMPONENT', 'NAME', 'STATUS', 'STATE'],
-            'memory': ['HOST', 'NAME', 'STATUS', 'STATE'],
-            'storage': ['HOST', 'NAME', 'MODEL', 'SIZE', 'PROTOCOL', 'SN', 'STATUS', 'STATE'],
-            'processors': ['HOST', 'NAME', 'MODEL', 'CORES', 'THREADS', 'STATUS', 'STATE'],
-            'network': ['HOST', 'NAME', 'SPEED', 'STATUS', 'STATE'],
-            'power': ['HOST', 'ID', 'NAME', 'MODEL', 'MANUFACTURER', 'STATUS', 'STATE'],
-            'fans': ['HOST', 'ID', 'NAME', 'STATUS', 'STATE']
-        }
-
-        if category not in table_heading_mapping.keys():
-            return f"'{category}' is not a valid category."
+    def node_proxy_summary(self, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy.summary(hostname=hostname)
 
-        table_headings = table_heading_mapping.get(category, [])
-        table = PrettyTable(table_headings,border=True)
-        output = None
-
-        if category == 'summary':
-            data = self.node_proxy.summary(hostname=hostname)
-            if format == Format.json:
-                output = json.dumps(data)
-            else:
-                for k, v in data.items():
-                    row = [k]
-                    row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
-                    table.add_row(row)
-                output = table.get_string()
-        elif category == 'firmwares':
-            output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table, format)
-        elif category == 'criticals':
-            output = self._criticals_table(hostname, table, format)
-        else:
-            output = self._common_table(category, hostname, table, format)
-
-        return output if output else table.get_string()
-
-    def _firmwares_table(self, hostname, table, format):
-        data = self.node_proxy.firmwares(hostname=hostname)
-        if format == Format.json:
-            return json.dumps(data)
-        for host, details in data.items():
-            for k, v in details.items():
-                table.add_row((host, k, v['name'], v['release_date'], v['version'], v['status']['health']))
-        return table.get_string()
-
-    def _criticals_table(self, hostname, table, format):
-        data = self.node_proxy.criticals(hostname=hostname)
-        if format == Format.json:
-            return json.dumps(data)
-        for host, host_details in data.items():
-            for component, component_details in host_details.items():
-                for member, member_details in component_details.items():
-                    description = member_details.get('description') or member_details.get('name')
-                    table.add_row((host, component, description, member_details['status']['health'], member_details['status']['state']))
-        return table.get_string()
-
-    def _common_table(self, category, hostname, table, format):
-        data = self.node_proxy.common(endpoint=category, hostname=hostname)
-        if format == Format.json:
-            return json.dumps(data)
-        mapping = {
-            'memory': ('description', 'health', 'state'),
-            'storage': ('description', 'model', 'capacity_bytes', 'protocol', 'serial_number', 'health', 'state'),
-            'processors': ('model', 'total_cores', 'total_threads', 'health', 'state'),
-            'network': ('name', 'speed_mbps', 'health', 'state'),
-            'power': ('name', 'model', 'manufacturer', 'health', 'state'),
-            'fans': ('name', 'health', 'state')
-        }
-
-        fields = mapping.get(category, ())
-        for host, details in data.items():
-            for k, v in details.items():
-                row = []
-                for field in fields:
-                    if field in v:
-                        row.append(v[field])
-                    elif field in v.get('status', {}):
-                        row.append(v['status'][field])
-                    else:
-                        row.append('')
-                if category in ('power', 'fans', 'processors'):
-                    table.add_row((host,) + (k,) + tuple(row))
-                else:
-                    table.add_row((host,) + tuple(row))
+    @handle_orch_error
+    def node_proxy_firmwares(self, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy.firmwares(hostname=hostname)
 
+    @handle_orch_error
+    def node_proxy_criticals(self, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy.criticals(hostname=hostname)
 
-        return table.get_string()
+    @handle_orch_error
+    def node_proxy_common(self, category: str, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy.common(category, hostname=hostname)
 
     @handle_orch_error
     def remove_host(self, host: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> str:
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index b36ffd4f89a3..f97b61e8f9b5 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -368,6 +368,38 @@ def hardware_status(self, hostname: Optional[str] = None, category: Optional[str
         """
         raise NotImplementedError()
 
+    def node_proxy_summary(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy summary
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
+    def node_proxy_firmwares(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy firmwares report
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
+    def node_proxy_criticals(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy criticals report
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
+    def node_proxy_common(self, category: str, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy generic report
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
     def remove_host(self, host: str, force: bool, offline: bool, rm_crush_entry: bool) -> OrchResult[str]:
         """
         Remove a host from the orchestrator inventory.
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 31a219f06b06..27d91d1a4d35 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -490,10 +490,106 @@ def _add_host(self,
 
     @_cli_write_command('orch hardware status')
     def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int = 0, category: str = 'summary', format: Format = Format.plain) -> HandleCommandResult:
-        """Display hardware status"""
-        completion = self.hardware_status(hostname, category, format)
-        raise_if_exception(completion)
-        return HandleCommandResult(stdout=completion.result_str())
+        """
+        Display hardware status summary
+
+        :param hostname: hostname
+        """
+        table_heading_mapping = {
+            'summary': ['HOST', 'STORAGE', 'CPU', 'NET', 'MEMORY', 'POWER', 'FANS'],
+            'firmwares': ['HOST', 'COMPONENT', 'NAME', 'DATE', 'VERSION', 'STATUS'],
+            'criticals': ['HOST', 'COMPONENT', 'NAME', 'STATUS', 'STATE'],
+            'memory': ['HOST', 'NAME', 'STATUS', 'STATE'],
+            'storage': ['HOST', 'NAME', 'MODEL', 'SIZE', 'PROTOCOL', 'SN', 'STATUS', 'STATE'],
+            'processors': ['HOST', 'NAME', 'MODEL', 'CORES', 'THREADS', 'STATUS', 'STATE'],
+            'network': ['HOST', 'NAME', 'SPEED', 'STATUS', 'STATE'],
+            'power': ['HOST', 'ID', 'NAME', 'MODEL', 'MANUFACTURER', 'STATUS', 'STATE'],
+            'fans': ['HOST', 'ID', 'NAME', 'STATUS', 'STATE']
+        }
+
+        if category not in table_heading_mapping.keys():
+            return HandleCommandResult(stdout=f"'{category}' is not a valid category.")
+
+        table_headings = table_heading_mapping.get(category, [])
+        table = PrettyTable(table_headings, border=True)
+        output = ''
+
+        if category == 'summary':
+            completion = self.node_proxy_summary(hostname=hostname)
+            summary: Dict[str, Any] = raise_if_exception(completion)
+            if format == Format.json:
+                output = json.dumps(summary)
+            else:
+                for k, v in summary.items():
+                    row = [k]
+                    row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
+                    table.add_row(row)
+                output = table.get_string()
+        elif category == 'firmwares':
+            output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table, format)
+        elif category == 'criticals':
+            output = self._criticals_table(hostname, table, format)
+        else:
+            output = self._common_table(category, hostname, table, format)
+
+        return HandleCommandResult(stdout=output)
+
+    def _firmwares_table(self, hostname: Optional[str], table: PrettyTable, format: Format) -> str:
+        completion = self.node_proxy_firmwares(hostname=hostname)
+        data = raise_if_exception(completion)
+        # data = self.node_proxy_firmware(hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
+        for host, details in data.items():
+            for k, v in details.items():
+                table.add_row((host, k, v['name'], v['release_date'], v['version'], v['status']['health']))
+        return table.get_string()
+
+    def _criticals_table(self, hostname: Optional[str], table: PrettyTable, format: Format) -> str:
+        completion = self.node_proxy_criticals(hostname=hostname)
+        data = raise_if_exception(completion)
+        # data = self.node_proxy_criticals(hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
+        for host, host_details in data.items():
+            for component, component_details in host_details.items():
+                for member, member_details in component_details.items():
+                    description = member_details.get('description') or member_details.get('name')
+                    table.add_row((host, component, description, member_details['status']['health'], member_details['status']['state']))
+        return table.get_string()
+
+    def _common_table(self, category: str, hostname: Optional[str], table: PrettyTable, format: Format) -> str:
+        completion = self.node_proxy_common(category=category, hostname=hostname)
+        data = raise_if_exception(completion)
+        # data = self.node_proxy_common(category=category, hostname=hostname)
+        if format == Format.json:
+            return json.dumps(data)
+        mapping = {
+            'memory': ('description', 'health', 'state'),
+            'storage': ('description', 'model', 'capacity_bytes', 'protocol', 'serial_number', 'health', 'state'),
+            'processors': ('model', 'total_cores', 'total_threads', 'health', 'state'),
+            'network': ('name', 'speed_mbps', 'health', 'state'),
+            'power': ('name', 'model', 'manufacturer', 'health', 'state'),
+            'fans': ('name', 'health', 'state')
+        }
+
+        fields = mapping.get(category, ())
+        for host, details in data.items():
+            for k, v in details.items():
+                row = []
+                for field in fields:
+                    if field in v:
+                        row.append(v[field])
+                    elif field in v.get('status', {}):
+                        row.append(v['status'][field])
+                    else:
+                        row.append('')
+                if category in ('power', 'fans', 'processors'):
+                    table.add_row((host,) + (k,) + tuple(row))
+                else:
+                    table.add_row((host,) + tuple(row))
+
+        return table.get_string()
 
     @_cli_write_command('orch host rm')
     def _remove_host(self, hostname: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> HandleCommandResult:

From f9a6467fdceb0ffe0b5bd2c61a55f3cbe9cf1518 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 6 Dec 2023 15:01:29 +0000
Subject: [PATCH 1601/2492] node-proxy: address flake8 errors in tests

This addresses a lot of flake8 errors in node-proxy tests:

E121 continuation line under-indented for hanging indent

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../mgr/cephadm/tests/node_proxy_data.py      | 343 +-----------------
 .../mgr/cephadm/tests/test_node_proxy.py      |  16 +-
 2 files changed, 11 insertions(+), 348 deletions(-)

diff --git a/src/pybind/mgr/cephadm/tests/node_proxy_data.py b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
index 70afc8a76483..37e6aaa46c83 100644
--- a/src/pybind/mgr/cephadm/tests/node_proxy_data.py
+++ b/src/pybind/mgr/cephadm/tests/node_proxy_data.py
@@ -1,340 +1,3 @@
-full_set_with_critical = {
-  "host": "host01",
-  "sn": "12345",
-  "status": {
-    "storage": {
-      "disk.bay.0:enclosure.internal.0-1:raid.integrated.1-1": {
-        "description": "Solid State Disk 0:1:0",
-        "entity": "RAID.Integrated.1-1",
-        "capacity_bytes": 959656755200,
-        "model": "KPM5XVUG960G",
-        "protocol": "SAS",
-        "serial_number": "8080A1CRTP5F",
-        "status": {
-          "health": "Critical",
-          "healthrollup": "OK",
-          "state": "Enabled"
-        },
-        "physical_location": {
-          "partlocation": {
-            "locationordinalvalue": 0,
-            "locationtype": "Slot"
-          }
-        }
-      },
-      "disk.bay.9:enclosure.internal.0-1": {
-        "description": "PCIe SSD in Slot 9 in Bay 1",
-        "entity": "CPU.1",
-        "capacity_bytes": 1600321314816,
-        "model": "Dell Express Flash NVMe P4610 1.6TB SFF",
-        "protocol": "PCIe",
-        "serial_number": "PHLN035305MN1P6AGN",
-        "status": {
-          "health": "Critical",
-          "healthrollup": "OK",
-          "state": "Enabled"
-        },
-        "physical_location": {
-          "partlocation": {
-            "locationordinalvalue": 9,
-            "locationtype": "Slot"
-          }
-        }
-      }
-    },
-    "processors": {
-      "cpu.socket.2": {
-        "description": "Represents the properties of a Processor attached to this System",
-        "total_cores": 20,
-        "total_threads": 40,
-        "processor_type": "CPU",
-        "model": "Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz",
-        "status": {
-          "health": "OK",
-          "state": "Enabled"
-        },
-        "manufacturer": "Intel"
-      },
-      
-    },
-    "network": {
-      "nic.slot.1-1-1": {
-        "description": "NIC in Slot 1 Port 1 Partition 1",
-        "name": "System Ethernet Interface",
-        "speed_mbps": 0,
-        "status": {
-          "health": "OK",
-          "state": "StandbyOffline"
-        }
-      }
-    },
-    "memory": {
-      "dimm.socket.a1": {
-        "description": "DIMM A1",
-        "memory_device_type": "DDR4",
-        "capacity_mi_b": 31237,
-        "status": {
-          "health": "Critical",
-          "state": "Enabled"
-        }
-      }
-    }
-  },
-  "firmwares": {
-    
-  }
-}
-
-mgr_inventory_cache = {"host01": {"hostname": "host01",
-                                  "addr": "10.10.10.11",
-                                  "labels": ["_admin"],
-                                  "status": "",
-                                  "oob": {"hostname": "10.10.10.11",
-                                          "username": "root",
-                                          "password": "ceph123"}},
-                       "host02": {"hostname": "host02",
-                                  "addr": "10.10.10.12",
-                                  "labels": [],
-                                  "status": "",
-                                  "oob": {"hostname": "10.10.10.12",
-                                          "username": "root",
-                                          "password": "ceph123"}}}
-
-full_set = {
-  "host01": {
-    "host": "host01",
-    "sn": "FR8Y5X3",
-    "status": {
-      "storage": {
-        "disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1": {
-          "description": "Disk 8 in Backplane 1 of Storage Controller in Slot 2",
-          "entity": "NonRAID.Slot.2-1",
-          "capacity_bytes": 20000588955136,
-          "model": "ST20000NM008D-3D",
-          "protocol": "SATA",
-          "serial_number": "ZVT99QLL",
-          "status": {
-            "health": "OK",
-            "healthrollup": "OK",
-            "state": "Enabled"
-          },
-          "physical_location": {
-            "partlocation": {
-              "locationordinalvalue": 8,
-              "locationtype": "Slot"
-            }
-          }
-        }
-      },
-      "processors": {
-        "cpu.socket.2": {
-          "description": "Represents the properties of a Processor attached to this System",
-          "total_cores": 16,
-          "total_threads": 32,
-          "processor_type": "CPU",
-          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          },
-          "manufacturer": "Intel"
-        },
-        "cpu.socket.1": {
-          "description": "Represents the properties of a Processor attached to this System",
-          "total_cores": 16,
-          "total_threads": 32,
-          "processor_type": "CPU",
-          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          },
-          "manufacturer": "Intel"
-        }
-      },
-      "network": {
-        "oslogicalnetwork.2": {
-          "description": "eno8303",
-          "name": "eno8303",
-          "speed_mbps": 0,
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "memory": {
-        "dimm.socket.a1": {
-          "description": "DIMM A1",
-          "memory_device_type": "DDR4",
-          "capacity_mi_b": 16384,
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "power": {
-        "0": {
-          "name": "PS1 Status",
-          "model": "PWR SPLY,800W,RDNT,LTON",
-          "manufacturer": "DELL",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        },
-        "1": {
-          "name": "PS2 Status",
-          "model": "PWR SPLY,800W,RDNT,LTON",
-          "manufacturer": "DELL",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "fans": {
-        "0": {
-          "name": "System Board Fan1A",
-          "physical_context": "SystemBoard",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      }
-    },
-    "firmwares": {
-      "installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1": {
-        "name": "Lifecycle Controller",
-        "description": "Represents Firmware Inventory",
-        "release_date": "00:00:00Z",
-        "version": "6.10.30.20",
-        "updateable": True,
-        "status": {
-          "health": "OK",
-          "state": "Enabled"
-        }
-      }
-    }
-  },
-"host02": {
-    "host": "host02",
-    "sn": "FR8Y5X4",
-    "status": {
-      "storage": {
-        "disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1": {
-          "description": "Disk 8 in Backplane 1 of Storage Controller in Slot 2",
-          "entity": "NonRAID.Slot.2-1",
-          "capacity_bytes": 20000588955136,
-          "model": "ST20000NM008D-3D",
-          "protocol": "SATA",
-          "serial_number": "ZVT99QLL",
-          "status": {
-            "health": "OK",
-            "healthrollup": "OK",
-            "state": "Enabled"
-          },
-          "physical_location": {
-            "partlocation": {
-              "locationordinalvalue": 8,
-              "locationtype": "Slot"
-            }
-          }
-        }
-      },
-      "processors": {
-        "cpu.socket.2": {
-          "description": "Represents the properties of a Processor attached to this System",
-          "total_cores": 16,
-          "total_threads": 32,
-          "processor_type": "CPU",
-          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          },
-          "manufacturer": "Intel"
-        },
-        "cpu.socket.1": {
-          "description": "Represents the properties of a Processor attached to this System",
-          "total_cores": 16,
-          "total_threads": 32,
-          "processor_type": "CPU",
-          "model": "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          },
-          "manufacturer": "Intel"
-        }
-      },
-      "network": {
-        "oslogicalnetwork.2": {
-          "description": "eno8303",
-          "name": "eno8303",
-          "speed_mbps": 0,
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "memory": {
-        "dimm.socket.a1": {
-          "description": "DIMM A1",
-          "memory_device_type": "DDR4",
-          "capacity_mi_b": 16384,
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "power": {
-        "0": {
-          "name": "PS1 Status",
-          "model": "PWR SPLY,800W,RDNT,LTON",
-          "manufacturer": "DELL",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        },
-        "1": {
-          "name": "PS2 Status",
-          "model": "PWR SPLY,800W,RDNT,LTON",
-          "manufacturer": "DELL",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      },
-      "fans": {
-        "0": {
-          "name": "System Board Fan1A",
-          "physical_context": "SystemBoard",
-          "status": {
-            "health": "OK",
-            "state": "Enabled"
-          }
-        }
-      }
-    },
-    "firmwares": {
-      "installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1": {
-        "name": "Lifecycle Controller",
-        "description": "Represents Firmware Inventory",
-        "release_date": "00:00:00Z",
-        "version": "6.10.30.20",
-        "updateable": True,
-        "status": {
-          "health": "OK",
-          "state": "Enabled"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
+full_set_with_critical = {'host': 'host01', 'sn': '12345', 'status': {'storage': {'disk.bay.0:enclosure.internal.0-1:raid.integrated.1-1': {'description': 'Solid State Disk 0:1:0', 'entity': 'RAID.Integrated.1-1', 'capacity_bytes': 959656755200, 'model': 'KPM5XVUG960G', 'protocol': 'SAS', 'serial_number': '8080A1CRTP5F', 'status': {'health': 'Critical', 'healthrollup': 'OK', 'state': 'Enabled'}, 'physical_location': {'partlocation': {'locationordinalvalue': 0, 'locationtype': 'Slot'}}}, 'disk.bay.9:enclosure.internal.0-1': {'description': 'PCIe SSD in Slot 9 in Bay 1', 'entity': 'CPU.1', 'capacity_bytes': 1600321314816, 'model': 'Dell Express Flash NVMe P4610 1.6TB SFF', 'protocol': 'PCIe', 'serial_number': 'PHLN035305MN1P6AGN', 'status': {'health': 'Critical', 'healthrollup': 'OK', 'state': 'Enabled'}, 'physical_location': {'partlocation': {'locationordinalvalue': 9, 'locationtype': 'Slot'}}}}, 'processors': {'cpu.socket.2': {'description': 'Represents the properties of a Processor attached to this System', 'total_cores': 20, 'total_threads': 40, 'processor_type': 'CPU', 'model': 'Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz', 'status': {'health': 'OK', 'state': 'Enabled'}, 'manufacturer': 'Intel'}}, 'network': {'nic.slot.1-1-1': {'description': 'NIC in Slot 1 Port 1 Partition 1', 'name': 'System Ethernet Interface', 'speed_mbps': 0, 'status': {'health': 'OK', 'state': 'StandbyOffline'}}}, 'memory': {'dimm.socket.a1': {'description': 'DIMM A1', 'memory_device_type': 'DDR4', 'capacity_mi_b': 31237, 'status': {'health': 'Critical', 'state': 'Enabled'}}}}, 'firmwares': {}}
+mgr_inventory_cache = {'host01': {'hostname': 'host01', 'addr': '10.10.10.11', 'labels': ['_admin'], 'status': '', 'oob': {'hostname': '10.10.10.11', 'username': 'root', 'password': 'ceph123'}}, 'host02': {'hostname': 'host02', 'addr': '10.10.10.12', 'labels': [], 'status': '', 'oob': {'hostname': '10.10.10.12', 'username': 'root', 'password': 'ceph123'}}}
+full_set = {'host01': {'host': 'host01', 'sn': 'FR8Y5X3', 'status': {'storage': {'disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1': {'description': 'Disk 8 in Backplane 1 of Storage Controller in Slot 2', 'entity': 'NonRAID.Slot.2-1', 'capacity_bytes': 20000588955136, 'model': 'ST20000NM008D-3D', 'protocol': 'SATA', 'serial_number': 'ZVT99QLL', 'status': {'health': 'OK', 'healthrollup': 'OK', 'state': 'Enabled'}, 'physical_location': {'partlocation': {'locationordinalvalue': 8, 'locationtype': 'Slot'}}}}, 'processors': {'cpu.socket.2': {'description': 'Represents the properties of a Processor attached to this System', 'total_cores': 16, 'total_threads': 32, 'processor_type': 'CPU', 'model': 'Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz', 'status': {'health': 'OK', 'state': 'Enabled'}, 'manufacturer': 'Intel'}, 'cpu.socket.1': {'description': 'Represents the properties of a Processor attached to this System', 'total_cores': 16, 'total_threads': 32, 'processor_type': 'CPU', 'model': 'Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz', 'status': {'health': 'OK', 'state': 'Enabled'}, 'manufacturer': 'Intel'}}, 'network': {'oslogicalnetwork.2': {'description': 'eno8303', 'name': 'eno8303', 'speed_mbps': 0, 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'memory': {'dimm.socket.a1': {'description': 'DIMM A1', 'memory_device_type': 'DDR4', 'capacity_mi_b': 16384, 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'power': {'0': {'name': 'PS1 Status', 'model': 'PWR SPLY,800W,RDNT,LTON', 'manufacturer': 'DELL', 'status': {'health': 'OK', 'state': 'Enabled'}}, '1': {'name': 'PS2 Status', 'model': 'PWR SPLY,800W,RDNT,LTON', 'manufacturer': 'DELL', 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'fans': {'0': {'name': 'System Board Fan1A', 'physical_context': 'SystemBoard', 'status': {'health': 'OK', 'state': 'Enabled'}}}}, 'firmwares': {'installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1': {'name': 'Lifecycle Controller', 'description': 'Represents Firmware Inventory', 'release_date': '00:00:00Z', 'version': '6.10.30.20', 'updateable': True, 'status': {'health': 'OK', 'state': 'Enabled'}}}}, 'host02': {'host': 'host02', 'sn': 'FR8Y5X4', 'status': {'storage': {'disk.bay.8:enclosure.internal.0-1:nonraid.slot.2-1': {'description': 'Disk 8 in Backplane 1 of Storage Controller in Slot 2', 'entity': 'NonRAID.Slot.2-1', 'capacity_bytes': 20000588955136, 'model': 'ST20000NM008D-3D', 'protocol': 'SATA', 'serial_number': 'ZVT99QLL', 'status': {'health': 'OK', 'healthrollup': 'OK', 'state': 'Enabled'}, 'physical_location': {'partlocation': {'locationordinalvalue': 8, 'locationtype': 'Slot'}}}}, 'processors': {'cpu.socket.2': {'description': 'Represents the properties of a Processor attached to this System', 'total_cores': 16, 'total_threads': 32, 'processor_type': 'CPU', 'model': 'Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz', 'status': {'health': 'OK', 'state': 'Enabled'}, 'manufacturer': 'Intel'}, 'cpu.socket.1': {'description': 'Represents the properties of a Processor attached to this System', 'total_cores': 16, 'total_threads': 32, 'processor_type': 'CPU', 'model': 'Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz', 'status': {'health': 'OK', 'state': 'Enabled'}, 'manufacturer': 'Intel'}}, 'network': {'oslogicalnetwork.2': {'description': 'eno8303', 'name': 'eno8303', 'speed_mbps': 0, 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'memory': {'dimm.socket.a1': {'description': 'DIMM A1', 'memory_device_type': 'DDR4', 'capacity_mi_b': 16384, 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'power': {'0': {'name': 'PS1 Status', 'model': 'PWR SPLY,800W,RDNT,LTON', 'manufacturer': 'DELL', 'status': {'health': 'OK', 'state': 'Enabled'}}, '1': {'name': 'PS2 Status', 'model': 'PWR SPLY,800W,RDNT,LTON', 'manufacturer': 'DELL', 'status': {'health': 'OK', 'state': 'Enabled'}}}, 'fans': {'0': {'name': 'System Board Fan1A', 'physical_context': 'SystemBoard', 'status': {'health': 'OK', 'state': 'Enabled'}}}}, 'firmwares': {'installed-28897-6.10.30.20__usc.embedded.1:lc.embedded.1': {'name': 'Lifecycle Controller', 'description': 'Represents Firmware Inventory', 'release_date': '00:00:00Z', 'version': '6.10.30.20', 'updateable': True, 'status': {'health': 'OK', 'state': 'Enabled'}}}}}
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index 1ca8b762cd39..3b05a4f39a54 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -1,4 +1,3 @@
-import pytest
 import cherrypy
 import json
 from _pytest.monkeypatch import MonkeyPatch
@@ -32,6 +31,7 @@ def __init__(self) -> None:
     def get_mgr_ip(self) -> str:
         return '0.0.0.0'
 
+
 class TestNodeProxy(helper.CPWebCase):
     mgr = FakeMgr()
     app = NodeProxy(mgr)
@@ -58,37 +58,37 @@ def setUp(self):
     def test_oob_data_misses_cephx_field(self):
         data = '{}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
     def test_oob_data_misses_name_field(self):
         data = '{"cephx": {"secret": "fake-secret"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
     def test_oob_data_misses_secret_field(self):
         data = '{"cephx": {"name": "host01"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
     def test_oob_agent_not_running(self):
         data = '{"cephx": {"name": "host03", "secret": "fake-secret03"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('502 Bad Gateway')
 
     def test_oob_wrong_keyring(self):
         data = '{"cephx": {"name": "host01", "secret": "wrong-keyring"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('403 Forbidden')
 
     def test_oob_ok(self):
         data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
-                                                                  ('Content-Length', str(len(data)))])
+                                                                ('Content-Length', str(len(data)))])
         self.assertStatus('200 OK')
 
     def test_data_missing_patch(self):
@@ -343,4 +343,4 @@ def test_firmwares_no_hostname(self):
 
     def test_firmwares_with_invalid_hostname(self):
         self.getPage("/host03/firmwares", method="GET")
-        self.assertStatus('404 Not Found')
\ No newline at end of file
+        self.assertStatus('404 Not Found')

From 04f8b5b85c628f8efc422b0cdf52d6522f6a3898 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 6 Dec 2023 15:09:44 +0000
Subject: [PATCH 1602/2492] cephadm: add `types-PyYAML` dependency in mypy
 testing

In order to address the following error:

```
cephadmlib/node_proxy/util.py:2: error: Library stubs not installed for "yaml" (or incompatible with Python 3.9)
cephadmlib/node_proxy/util.py:2: note: Hint: "python3 -m pip install types-PyYAML"
cephadmlib/node_proxy/util.py:2: note: (or run "mypy --install-types" to install all missing stub packages)
cephadmlib/node_proxy/util.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
```

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 67ed18c97249..38241a1c763b 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -44,6 +44,7 @@ commands=pytest {posargs}
 basepython = python3
 deps =
     mypy
+    types-PyYAML
     -rzipapp-reqs.txt
     -c{toxinidir}/../mypy-constrains.txt
 commands = mypy --config-file ../mypy.ini {posargs:cephadm.py cephadmlib}

From 17c9a5250744cef089be2f5781c1031679d4b00d Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 7 Dec 2023 14:20:43 +0000
Subject: [PATCH 1603/2492] node-proxy: send oob management requests to the
 MgrListener()

Note that this won't be a true out of band management.
In the case where the host hangs, this won't work. The oob
management should be reached directly but most of the time
the oob network is isolated. The idea is to send queries to the
the tcp server exposed by the cephadm agent (MgrListener) so it
can send itself queries to the redfish API using the IP address
exposed on the OS.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/cephadm.py                        |  34 +++-
 .../node_proxy/redfishdellsystem.py           |  24 +++
 src/pybind/mgr/cephadm/agent.py               | 165 ++++--------------
 3 files changed, 91 insertions(+), 132 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 879aec568b5b..1fc5768ca968 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -1287,13 +1287,28 @@ def run(self) -> None:
                             self.agent.ls_gatherer.wakeup()
                             self.agent.volume_gatherer.wakeup()
                             logger.debug(f'Got mgr message {data}')
+                        if 'node_proxy_oob_cmd' in data:
+                            if data['node_proxy_oob_cmd']['action'] in ['get_led', 'set_led']:
+                                conn.send(bytes(json.dumps(self.node_proxy_oob_cmd_result), 'utf-8'))
             except Exception as e:
                 logger.error(f'Mgr Listener encountered exception: {e}')
 
     def shutdown(self) -> None:
         self.stop = True
 
+    def validate_node_proxy_payload(self, data: Dict[str, Any]) -> None:
+        if 'action' not in data.keys():
+            raise RuntimeError('node-proxy oob command needs an action.')
+        if data['action'] in ['get_led', 'set_led']:
+            fields = ['type', 'id']
+            if data['type'] not in ['chassis', 'drive']:
+                raise RuntimeError('the LED type must be either "chassis" or "drive".')
+            for field in fields:
+                if field not in data.keys():
+                    raise RuntimeError('Received invalid node-proxy cmd.')
+
     def handle_json_payload(self, data: Dict[Any, Any]) -> None:
+        self.node_proxy_oob_cmd_result: Dict[str, Any] = {}
         if 'counter' in data:
             self.agent.ack = int(data['counter'])
             if 'config' in data:
@@ -1307,7 +1322,24 @@ def handle_json_payload(self, data: Dict[Any, Any]) -> None:
                 self.agent.pull_conf_settings()
                 self.agent.wakeup()
         elif 'node_proxy_shutdown' in data:
+            logger.info('Received node_proxy_shutdown command.')
             self.agent.shutdown()
+        elif 'node_proxy_oob_cmd' in data:
+            node_proxy_cmd: Dict[str, Any] = data['node_proxy_oob_cmd']
+            try:
+                self.validate_node_proxy_payload(node_proxy_cmd)
+            except RuntimeError as e:
+                logger.error(f"Couldn't validate node-proxy payload:\n{node_proxy_cmd}\n{e}")
+                raise
+            logger.info(f'Received node_proxy_oob_cmd command: {node_proxy_cmd}')
+            if node_proxy_cmd['action'] == 'get_led':
+                if node_proxy_cmd['type'] == 'chassis':
+                    self.node_proxy_oob_cmd_result = self.agent.node_proxy_mgr.node_proxy.system.get_chassis_led()
+            if node_proxy_cmd['action'] == 'set_led':
+                if node_proxy_cmd['type'] == 'chassis':
+                    _data: Dict[str, Any] = json.loads(node_proxy_cmd['data'])
+                    _result: int = self.agent.node_proxy_mgr.node_proxy.system.set_chassis_led(_data)
+                    self.node_proxy_oob_cmd_result = {'http_code': _result}
         else:
             raise RuntimeError('No valid data received.')
 
@@ -1354,7 +1386,7 @@ def init(self) -> None:
         if result_json['result'].get('port'):
             kwargs['port'] = result_json['result']['port']
 
-        self.node_proxy = NodeProxy(**kwargs)
+        self.node_proxy: NodeProxy = NodeProxy(**kwargs)
         self.node_proxy.start()
 
     def loop(self) -> None:
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
index 12d2466a88ff..71006c48bf86 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
@@ -1,3 +1,4 @@
+import json
 from .baseredfishsystem import BaseRedfishSystem
 from .util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any, List
@@ -160,3 +161,26 @@ def _update_firmwares(self) -> None:
         self._sys['firmwares'] = self.build_common_data(data=self._system['UpdateService'],
                                                         fields=fields,
                                                         path='FirmwareInventory')
+
+    def get_chassis_led(self) -> Dict[str, Any]:
+        endpoint = f'/redfish/v1/{self.chassis_endpoint}'
+        result = self.client.query(method='GET',
+                                   endpoint=endpoint,
+                                   timeout=10)
+        response_json = json.loads(result[1])
+        _result: Dict[str, Any] = {'http_code': result[2]}
+        if result[2] == 200:
+            _result['LocationIndicatorActive'] = response_json['LocationIndicatorActive']
+        else:
+            _result['LocationIndicatorActive'] = None
+        return _result
+
+    def set_chassis_led(self, data: Dict[str, str]) -> int:
+        # '{"IndicatorLED": "Lit"}'      -> LocationIndicatorActive = false
+        # '{"IndicatorLED": "Blinking"}' -> LocationIndicatorActive = true
+        _, response, status = self.client.query(
+            data=json.dumps(data),
+            method='PATCH',
+            endpoint=f'/redfish/v1{self.chassis_endpoint}'
+        )
+        return status
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 49f45bffef42..d2a155d5de0c 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -22,11 +22,8 @@ class Server:  # type: ignore
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
 from cephadm.ssl_cert_utils import SSLCerts
 from mgr_util import test_port_allocation, PortAlreadyInUse
-from urllib.request import urlopen, Request
-from urllib.error import HTTPError, URLError
-from contextlib import contextmanager
 
-from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING, Optional, Generator
+from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -267,103 +264,6 @@ def data(self) -> None:
         self.mgr.node_proxy.save(host, data['patch'])
         self.raise_alert(data)
 
-    def login(self,
-              addr: str,
-              username: str,
-              password: str,
-              port: str = '443') -> None:
-        self.mgr.log.info("Logging in to "
-                          f"{addr}:{port} as '{username}'")
-        oob_credentials = json.dumps({"UserName": username,
-                                      "Password": password})
-        headers = {"Content-Type": "application/json"}
-
-        try:
-            _status_code, _data, _headers = self.query(addr=addr,
-                                                       port=port,
-                                                       data=bytes(oob_credentials, 'ascii'),
-                                                       headers=headers,
-                                                       endpoint="/redfish/v1/SessionService/Sessions/",
-                                                       method="POST")
-            if _status_code != 201:
-                self.mgr.log.error(f"Can't log in to {addr}:{port} as '{username}': {_status_code}")
-                raise RuntimeError
-            self.redfish_token = _headers['X-Auth-Token']
-            self.redfish_session_location = _headers['Location']
-        except URLError as e:
-            msg = f"Can't log in to {addr}:{port} as '{username}': {e}"
-            self.mgr.log.error(msg)
-            raise RuntimeError
-
-    def logout(self,
-               addr: str,
-               port: str = '443') -> None:
-        try:
-            _status_code, _data, _ = self.query(addr=addr,
-                                                port=port,
-                                                method='DELETE',
-                                                headers={"X-Auth-Token": self.redfish_token},
-                                                endpoint=self.redfish_session_location)
-        except URLError:
-            msg = f"Can't log out from {addr}:{port}"
-            self.mgr.log.error(msg)
-            raise cherrypy.HTTPError(502, msg)
-
-    @contextmanager
-    def redfish_session(self,
-                        addr: str,
-                        username: str,
-                        password: str,
-                        port: str = '443') -> Generator:
-        try:
-            self.login(addr=addr,
-                       port=port,
-                       username=username,
-                       password=password)
-            yield
-        except Exception:
-            raise
-        else:
-            self.logout(addr=addr,
-                        port=port)
-
-    def query(self,
-              addr: str = '',
-              port: str = '',
-              method: Optional[str] = None,
-              headers: Dict[str, str] = {},
-              data: Optional[bytes] = None,
-              endpoint: str = '',
-              ssl_ctx: Optional[Any] = None,
-              timeout: Optional[int] = 10) -> Tuple[int, Dict[str, Any], Dict[str, Any]]:
-        if not ssl_ctx:
-            ssl_ctx = self.ssl_ctx
-        url = f'https://{addr}:{port}{endpoint}'
-        _headers = headers
-        response_json = {}
-        response_headers = {}
-        if not _headers.get('Content-Type'):
-            # default to application/json if nothing provided
-            _headers['Content-Type'] = 'application/json'
-
-        try:
-            req = Request(url, data, _headers, method=method)
-            with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
-                response_str = response.read()
-                response_headers = response.headers
-                response_json = json.loads(response_str)
-                response_status = response.status
-        except HTTPError as e:
-            self.mgr.log.error(f"{e.code} {e.reason}")
-            response_status = e.code
-        except URLError as e:
-            self.mgr.log.error(f"{e.reason}")
-            raise
-        except Exception as e:
-            self.mgr.log.error(f"{e}")
-            raise
-        return (response_status, response_json, response_headers)
-
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET', 'PATCH'])
     @cherrypy.tools.json_in()
@@ -392,6 +292,9 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         hostname: Optional[str] = kw.get('hostname')
         led_type: Optional[str] = kw.get('type')
         id_drive: Optional[str] = kw.get('id')
+        data: Optional[str] = None
+        # this method is restricted to 'GET' or 'PATCH'
+        action: str = 'get_led' if method == 'GET' else 'set_led'
 
         if not hostname:
             msg: str = "listing enclosure LED status for all nodes is not implemented."
@@ -414,19 +317,10 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
 
-        # if led_type not in ['chassis', 'drive']:
-        #     # TODO(guits): update unit test for this
-        #     raise cherrypy.HTTPError(404, 'LED type must be either "chassis" or "drive"')
-
-        addr = self.mgr.node_proxy.oob[hostname]['addr']
-        port = self.mgr.node_proxy.oob[hostname]['port']
-        username = self.mgr.node_proxy.oob[hostname]['username']
-        password = self.mgr.node_proxy.oob[hostname]['password']
-
         if method == 'PATCH':
             # TODO(guits): need to check the request is authorized
             # allowing a specific keyring only ? (client.admin or client.agent.. ?)
-            data: str = json.dumps(cherrypy.request.json)
+            data = json.dumps(cherrypy.request.json)
 
             if led_type == 'drive':
                 if id_drive not in self.mgr.node_proxy.data[hostname]['status']['storage'].keys():
@@ -434,25 +328,30 @@ def led(self, **kw: Any) -> Dict[str, Any]:
                     msg = f"'{id_drive}' not found."
                     self.mgr.log.debug(msg)
                     raise cherrypy.HTTPError(400, msg)
-                endpoint = self.mgr.node_proxy.data[hostname]['status']['storage'][id_drive].get('redfish_endpoint')
-            else:
-                endpoint = self.mgr.node_proxy.data[hostname]['chassis']['redfish_endpoint']
 
-        with self.redfish_session(addr, username, password, port=port):
-            try:
-                status, result, _ = self.query(data=bytes(data, 'ascii'),
-                                               addr=addr,
-                                               method=method,
-                                               headers={"X-Auth-Token": self.redfish_token},
-                                               port=port,
-                                               endpoint=endpoint,
-                                               ssl_ctx=self.ssl_ctx)
-            except (URLError, HTTPError, RuntimeError) as e:
-                raise cherrypy.HTTPError(502, f"{e}")
-            if method == 'GET':
-                result = {"LocationIndicatorActive": result['LocationIndicatorActive']}
-            cherrypy.response.status = status
-            return result
+        payload: Dict[str, Any] = {"node_proxy_oob_cmd":
+                                   {"action": action,
+                                    "type": led_type,
+                                    "id": id_drive,
+                                    "host": hostname,
+                                    "data": data}}
+        try:
+            message_thread = AgentMessageThread(
+                hostname, self.mgr.agent_cache.agent_ports[hostname], payload, self.mgr)
+            message_thread.start()
+            message_thread.join()  # TODO(guits): Add a timeout?
+        except KeyError:
+            raise cherrypy.HTTPError(502, f"{hostname}'s agent not running, please check.")
+        agent_response = message_thread.get_agent_response()
+        try:
+            response_json: Dict[str, Any] = json.loads(agent_response)
+        except json.decoder.JSONDecodeError:
+            cherrypy.response.status = 503
+        else:
+            cherrypy.response.status = response_json.get('http_code', 503)
+        if cherrypy.response.status != 200:
+            raise cherrypy.HTTPError(cherrypy.response.status, "Couldn't change the LED status.")
+        return response_json
 
     @cherrypy.expose
     @cherrypy.tools.allow(methods=['GET'])
@@ -845,6 +744,7 @@ def __init__(self, host: str, port: int, data: Dict[Any, Any], mgr: "CephadmOrch
         self.port = port
         self.data: str = json.dumps(data)
         self.daemon_spec: Optional[CephadmDaemonDeploySpec] = daemon_spec
+        self.agent_response: str = ''
         super().__init__(target=self.run)
 
     def run(self) -> None:
@@ -897,8 +797,8 @@ def run(self) -> None:
                 secure_agent_socket.connect((self.addr, self.port))
                 msg = (bytes_len + self.data)
                 secure_agent_socket.sendall(msg.encode('utf-8'))
-                agent_response = secure_agent_socket.recv(1024).decode()
-                self.mgr.log.debug(f'Received "{agent_response}" from agent on host {self.host}')
+                self.agent_response = secure_agent_socket.recv(1024).decode()
+                self.mgr.log.debug(f'Received "{self.agent_response}" from agent on host {self.host}')
                 if self.daemon_spec:
                     self.mgr.agent_cache.agent_config_successfully_delivered(self.daemon_spec)
                 self.mgr.agent_cache.sending_agent_message[self.host] = False
@@ -918,6 +818,9 @@ def run(self) -> None:
         self.mgr.agent_cache.sending_agent_message[self.host] = False
         return
 
+    def get_agent_response(self) -> str:
+        return self.agent_response
+
 
 class CephadmAgentHelpers:
     def __init__(self, mgr: "CephadmOrchestrator"):

From 56a939af49af975346636b43e486227cbcdd2572 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 18 Dec 2023 14:26:04 +0000
Subject: [PATCH 1604/2492] cephadm/tests: add pyyaml dependency

node-proxy requires this dependency so it needs to be added as
dependency for tox testing.

Typical failure:

```
ImportError while importing test module '/root/ceph/src/cephadm/tests/test_agent.py'.
Hint: make sure your test modules/packages have valid Python names.
Traceback:
/usr/lib64/python3.9/importlib/__init__.py:127: in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
tests/test_agent.py:10: in <module>
    _cephadm = import_cephadm()
tests/fixtures.py:14: in import_cephadm
    import cephadm as _cephadm
cephadm.py:32: in <module>
    from cephadmlib.node_proxy.main import NodeProxy
cephadmlib/node_proxy/main.py:2: in <module>
    from .redfishdellsystem import RedfishDellSystem
cephadmlib/node_proxy/redfishdellsystem.py:2: in <module>
    from .baseredfishsystem import BaseRedfishSystem
cephadmlib/node_proxy/baseredfishsystem.py:2: in <module>
    from .basesystem import BaseSystem
cephadmlib/node_proxy/basesystem.py:2: in <module>
    from .util import Config
cephadmlib/node_proxy/util.py:2: in <module>
    import yaml
E   ModuleNotFoundError: No module named 'yaml'
```

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/cephadm/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 38241a1c763b..4871c186b3e3 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -38,6 +38,7 @@ deps =
   pyfakefs >= 5, < 6 ; python_version >= "3.7"
   mock
   pytest
+  pyyaml
 commands=pytest {posargs}
 
 [testenv:mypy]

From a77a13f6af0d3214124aa49e5e1e1d063df8226b Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Dec 2023 09:14:31 +0000
Subject: [PATCH 1605/2492] node-proxy: explicitly set NodeProxy's attributes

The current logic using `setattr()` makes mypy complain:

"NodeProxy" has no attribute "xxx"

Using `self.__dict['xxx']` addresses this mypy error but the
downside of this is that the code isn't clear and less readable.

Explicitly setting the different attributes makes the code clearer
and more readable.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../node_proxy/baseredfishsystem.py           |  2 +-
 src/cephadm/cephadmlib/node_proxy/main.py     | 35 +++++++++++--------
 .../cephadmlib/node_proxy/redfish_client.py   |  4 +--
 src/cephadm/cephadmlib/node_proxy/reporter.py |  4 +--
 src/cephadm/cephadmlib/node_proxy/util.py     |  4 +--
 5 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 587974f8db20..6102c1a8044d 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -15,7 +15,7 @@ def __init__(self, **kw: Any) -> None:
         self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
         self.log = Logger(__name__)
         self.host: str = kw['host']
-        self.port: int = kw['port']
+        self.port: str = kw['port']
         self.username: str = kw['username']
         self.password: str = kw['password']
         # move the following line (class attribute?)
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
index 813b3e3edf84..968dfd3c1ca6 100644
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ b/src/cephadm/cephadmlib/node_proxy/main.py
@@ -24,10 +24,17 @@
 
 
 class NodeProxy(Thread):
-    def __init__(self, **kw: Dict[str, Any]) -> None:
+    def __init__(self, **kw: Any) -> None:
         super().__init__()
-        for k, v in kw.items():
-            setattr(self, k, v)
+        self.username: str = kw.get('username', '')
+        self.password: str = kw.get('password', '')
+        self.host: str = kw.get('host', '')
+        self.port: int = kw.get('port', 443)
+        self.cephx: Dict[str, Any] = kw.get('cephx', {})
+        self.reporter_scheme: str = kw.get('reporter_scheme', 'https')
+        self.mgr_target_ip: str = kw.get('mgr_target_ip', '')
+        self.mgr_target_port: str = kw.get('mgr_target_port', '')
+        self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
         self.exc: Optional[Exception] = None
         self.log = Logger(__name__)
 
@@ -45,8 +52,8 @@ def shutdown(self) -> None:
         self.reporter_agent.stop()
 
     def check_auth(self, realm: str, username: str, password: str) -> bool:
-        return self.__dict__['username'] == username and \
-            self.__dict__['password'] == password
+        return self.username == username and \
+            self.password == password
 
     def check_status(self) -> bool:
         if self.__dict__.get('system') and not self.system.run:
@@ -65,10 +72,10 @@ def main(self) -> None:
         # create the redfish system and the obsever
         self.log.logger.info('Server initialization...')
         try:
-            self.system = RedfishDellSystem(host=self.__dict__['host'],
-                                            port=self.__dict__.get('port', 443),
-                                            username=self.__dict__['username'],
-                                            password=self.__dict__['password'],
+            self.system = RedfishDellSystem(host=self.host,
+                                            port=self.port,
+                                            username=self.username,
+                                            password=self.password,
                                             config=self.config)
         except RuntimeError:
             self.log.logger.error("Can't initialize the redfish system.")
@@ -76,11 +83,11 @@ def main(self) -> None:
 
         try:
             self.reporter_agent = Reporter(self.system,
-                                           self.__dict__['cephx'],
-                                           reporter_scheme=self.__dict__.get('reporter_scheme', 'https'),
-                                           reporter_hostname=self.__dict__['mgr_target_ip'],
-                                           reporter_port=self.__dict__['mgr_target_port'],
-                                           reporter_endpoint=self.__dict__.get('reporter_endpoint', '/node-proxy/data'))
+                                           self.cephx,
+                                           reporter_scheme=self.reporter_scheme,
+                                           reporter_hostname=self.mgr_target_ip,
+                                           reporter_port=self.mgr_target_port,
+                                           reporter_endpoint=self.reporter_endpoint)
             self.reporter_agent.run()
         except RuntimeError:
             self.log.logger.error("Can't initialize the reporter.")
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
index c07e1a6b6f39..040db8ce2521 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/cephadm/cephadmlib/node_proxy/redfish_client.py
@@ -11,14 +11,14 @@ class RedFishClient(BaseClient):
 
     def __init__(self,
                  host: str = '',
-                 port: int = 443,
+                 port: str = '443',
                  username: str = '',
                  password: str = ''):
         super().__init__(host, username, password)
         self.log: Logger = Logger(__name__)
         self.log.logger.info(f'Initializing redfish client {__name__}')
         self.host: str = host
-        self.port: int = port
+        self.port: str = port
         self.url: str = f'https://{self.host}:{self.port}'
         self.token: str = ''
         self.location: str = ''
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 21183c980357..765374483b1f 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -12,7 +12,7 @@ def __init__(self,
                  cephx: Dict[str, Any],
                  reporter_scheme: str = 'https',
                  reporter_hostname: str = '',
-                 reporter_port: int = 443,
+                 reporter_port: str = '443',
                  reporter_endpoint: str = '/node-proxy/data') -> None:
         self.system = system
         self.data: Dict[str, Any] = {}
@@ -21,7 +21,7 @@ def __init__(self,
         self.data['cephx'] = self.cephx
         self.reporter_scheme: str = reporter_scheme
         self.reporter_hostname: str = reporter_hostname
-        self.reporter_port: int = reporter_port
+        self.reporter_port: str = reporter_port
         self.reporter_endpoint: str = reporter_endpoint
         self.log = Logger(__name__)
         self.reporter_url: str = (f'{reporter_scheme}:{reporter_hostname}:'
diff --git a/src/cephadm/cephadmlib/node_proxy/util.py b/src/cephadm/cephadmlib/node_proxy/util.py
index f154d83daafb..31c1c00a0a15 100644
--- a/src/cephadm/cephadmlib/node_proxy/util.py
+++ b/src/cephadm/cephadmlib/node_proxy/util.py
@@ -104,7 +104,7 @@ def _retry(*args: str, **kwargs: Any) -> Callable:
 
 
 def http_req(hostname: str = '',
-             port: int = 443,
+             port: str = '443',
              method: Optional[str] = None,
              headers: MutableMapping[str, str] = {},
              data: Optional[str] = None,
@@ -122,7 +122,7 @@ def http_req(hostname: str = '',
         else:
             ssl_ctx.verify_mode = ssl.CERT_REQUIRED
 
-    url: str = f'{scheme}://{hostname}:{str(port)}{endpoint}'
+    url: str = f'{scheme}://{hostname}:{port}{endpoint}'
     _data = bytes(data, 'ascii') if data else None
 
     try:

From da347d235d3fc5d6ecc98ea7ffd93f4c3c966d43 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 19 Dec 2023 09:23:42 +0000
Subject: [PATCH 1606/2492] node-proxy: enhance debug log messages for locking
 operations

This commit updates the debug log messages in the BaseRedfishSystem
and Reporter classes. The adjustments made enhance the clarity and
precision of the messages by specifically identifying acquired
and released locks, detailing their context, thereby improving the
understanding of the control flow during locking operations
in these components.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../cephadmlib/node_proxy/baseredfishsystem.py     | 14 +++++++-------
 src/cephadm/cephadmlib/node_proxy/reporter.py      |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
index 6102c1a8044d..45c80e0209d4 100644
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
@@ -49,9 +49,9 @@ def update(self) -> None:
         #  this loop can have:
         #  - caching logic
         while self.run:
-            self.log.logger.debug('waiting for a lock.')
+            self.log.logger.debug('waiting for a lock in the update loop.')
             self.lock.acquire()
-            self.log.logger.debug('lock acquired.')
+            self.log.logger.debug('lock acquired in the update loop.')
             try:
                 self._update_system()
                 self._update_sn()
@@ -74,19 +74,19 @@ def update(self) -> None:
                 self.client.logout()
             finally:
                 self.lock.release()
-                self.log.logger.debug('lock released.')
+                self.log.logger.debug('lock released in the update loop.')
 
     def flush(self) -> None:
-        self.log.logger.info('Acquiring lock to flush data.')
+        self.log.logger.debug('Acquiring lock to flush data.')
         self.lock.acquire()
-        self.log.logger.info('Lock acquired, flushing data.')
+        self.log.logger.debug('Lock acquired, flushing data.')
         self._system = {}
         self.previous_data = {}
         self.log.logger.info('Data flushed.')
         self.data_ready = False
-        self.log.logger.info('Data marked as not ready.')
+        self.log.logger.debug('Data marked as not ready.')
         self.lock.release()
-        self.log.logger.info('Lock released.')
+        self.log.logger.debug('Released the lock after flushing data.')
 
     @retry(retries=10, delay=2)
     def _get_path(self, path: str) -> Dict:
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/cephadm/cephadmlib/node_proxy/reporter.py
index 765374483b1f..fb92a4523467 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/cephadm/cephadmlib/node_proxy/reporter.py
@@ -43,9 +43,9 @@ def loop(self) -> None:
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
-            self.log.logger.debug('waiting for a lock.')
+            self.log.logger.debug('waiting for a lock in reporter loop.')
             self.system.lock.acquire()
-            self.log.logger.debug('lock acquired.')
+            self.log.logger.debug('lock acquired in reporter loop.')
             if self.system.data_ready:
                 self.log.logger.info('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
@@ -70,5 +70,5 @@ def loop(self) -> None:
                 else:
                     self.log.logger.info('no diff, not sending data to the mgr.')
             self.system.lock.release()
-            self.log.logger.debug('lock released.')
+            self.log.logger.debug('lock released in reporter loop.')
             time.sleep(5)

From f99decff89e594ecc7bd3b19d71cbc77559eaf07 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 17 Jan 2024 08:47:36 +0000
Subject: [PATCH 1607/2492] node-proxy: rename attribute and class

This renames the mgr's NodeProxyCache attribute from
`self.node_proxy` to `self.node_proxy_cache` and the
class `NodeProxy` in agent.py from `NodeProxy` to
`NodeProxyEndpoint` to make it clearer and avoid confusion.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py               | 36 +++++++++----------
 src/pybind/mgr/cephadm/module.py              | 12 +++----
 src/pybind/mgr/cephadm/serve.py               |  2 +-
 .../mgr/cephadm/tests/test_node_proxy.py      | 22 ++++++------
 4 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index d2a155d5de0c..b589886e566c 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -56,7 +56,7 @@ def configure_routes(self) -> None:
         conf = {'/': {'tools.trailing_slash.on': False}}
 
         cherrypy.tree.mount(self.host_data, '/data', config=conf)
-        cherrypy.tree.mount(self.node_proxy, '/node-proxy', config=conf)
+        cherrypy.tree.mount(self.node_proxy_endpoint, '/node-proxy', config=conf)
 
     def configure_tls(self, server: Server) -> None:
         old_cert = self.mgr.get_store(self.KV_STORE_AGENT_ROOT_CERT)
@@ -87,12 +87,12 @@ def find_free_port(self) -> None:
     def configure(self) -> None:
         self.host_data = HostData(self.mgr, self.server_port, self.server_addr)
         self.configure_tls(self.host_data)
-        self.node_proxy = NodeProxy(self.mgr)
+        self.node_proxy_endpoint = NodeProxyEndpoint(self.mgr)
         self.configure_routes()
         self.find_free_port()
 
 
-class NodeProxy:
+class NodeProxyEndpoint:
     def __init__(self, mgr: "CephadmOrchestrator"):
         self.mgr = mgr
         self.ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
@@ -106,7 +106,7 @@ def __init__(self, mgr: "CephadmOrchestrator"):
         self.redfish_token: str = ''
         self.redfish_session_location: str = ''
 
-    def _cp_dispatch(self, vpath: List[str]) -> "NodeProxy":
+    def _cp_dispatch(self, vpath: List[str]) -> "NodeProxyEndpoint":
         if len(vpath) > 1:  # /{hostname}/<endpoint>
             hostname = vpath.pop(0)  # /<endpoint>
             cherrypy.request.params['hostname'] = hostname
@@ -139,7 +139,7 @@ def oob(self) -> Dict[str, Any]:
         self.validate_node_proxy_data(data)
 
         host = data["cephx"]["name"]
-        results['result'] = self.mgr.node_proxy.oob.get(host)
+        results['result'] = self.mgr.node_proxy_cache.oob.get(host)
         if not results['result']:
             raise cherrypy.HTTPError(400, 'The provided host has no iDrac details.')
         return results
@@ -261,7 +261,7 @@ def data(self) -> None:
         if 'patch' not in data.keys():
             raise cherrypy.HTTPError(400, 'Malformed data received.')
         host = data['cephx']['name']
-        self.mgr.node_proxy.save(host, data['patch'])
+        self.mgr.node_proxy_cache.save(host, data['patch'])
         self.raise_alert(data)
 
     @cherrypy.expose
@@ -311,7 +311,7 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
 
-        if hostname not in self.mgr.node_proxy.data.keys():
+        if hostname not in self.mgr.node_proxy_cache.data.keys():
             # TODO(guits): update unit test for this
             msg = f"'{hostname}' not found."
             self.mgr.log.debug(msg)
@@ -323,7 +323,7 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             data = json.dumps(cherrypy.request.json)
 
             if led_type == 'drive':
-                if id_drive not in self.mgr.node_proxy.data[hostname]['status']['storage'].keys():
+                if id_drive not in self.mgr.node_proxy_cache.data[hostname]['status']['storage'].keys():
                     # TODO(guits): update unit test for this
                     msg = f"'{id_drive}' not found."
                     self.mgr.log.debug(msg)
@@ -372,7 +372,7 @@ def fullreport(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.fullreport(**kw)
+            results = self.mgr.node_proxy_cache.fullreport(**kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -396,7 +396,7 @@ def criticals(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.criticals(**kw)
+            results = self.mgr.node_proxy_cache.criticals(**kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -420,7 +420,7 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.summary(**kw)
+            results = self.mgr.node_proxy_cache.summary(**kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -445,7 +445,7 @@ def memory(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('memory', **kw)
+            results = self.mgr.node_proxy_cache.common('memory', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -470,7 +470,7 @@ def network(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('network', **kw)
+            results = self.mgr.node_proxy_cache.common('network', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -495,7 +495,7 @@ def processors(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('processors', **kw)
+            results = self.mgr.node_proxy_cache.common('processors', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -520,7 +520,7 @@ def storage(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('storage', **kw)
+            results = self.mgr.node_proxy_cache.common('storage', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -545,7 +545,7 @@ def power(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('power', **kw)
+            results = self.mgr.node_proxy_cache.common('power', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -570,7 +570,7 @@ def fans(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.common('fans', **kw)
+            results = self.mgr.node_proxy_cache.common('fans', **kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
@@ -594,7 +594,7 @@ def firmwares(self, **kw: Any) -> Dict[str, Any]:
         :raises cherrypy.HTTPError 404: If the passed hostname is not found.
         """
         try:
-            results = self.mgr.node_proxy.firmwares(**kw)
+            results = self.mgr.node_proxy_cache.firmwares(**kw)
         except KeyError:
             raise cherrypy.HTTPError(404, f"{kw.get('hostname')} not found.")
         return results
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 3029b7b6b5b4..b82f861ce6ad 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -600,8 +600,8 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.cache = HostCache(self)
         self.cache.load()
 
-        self.node_proxy = NodeProxyCache(self)
-        self.node_proxy.load()
+        self.node_proxy_cache = NodeProxyCache(self)
+        self.node_proxy_cache.load()
 
         self.agent_cache = AgentCache(self)
         self.agent_cache.load()
@@ -1654,19 +1654,19 @@ def add_host(self, spec: HostSpec) -> str:
 
     @handle_orch_error
     def node_proxy_summary(self, hostname: Optional[str] = None) -> Dict[str, Any]:
-        return self.node_proxy.summary(hostname=hostname)
+        return self.node_proxy_cache.summary(hostname=hostname)
 
     @handle_orch_error
     def node_proxy_firmwares(self, hostname: Optional[str] = None) -> Dict[str, Any]:
-        return self.node_proxy.firmwares(hostname=hostname)
+        return self.node_proxy_cache.firmwares(hostname=hostname)
 
     @handle_orch_error
     def node_proxy_criticals(self, hostname: Optional[str] = None) -> Dict[str, Any]:
-        return self.node_proxy.criticals(hostname=hostname)
+        return self.node_proxy_cache.criticals(hostname=hostname)
 
     @handle_orch_error
     def node_proxy_common(self, category: str, hostname: Optional[str] = None) -> Dict[str, Any]:
-        return self.node_proxy.common(category, hostname=hostname)
+        return self.node_proxy_cache.common(category, hostname=hostname)
 
     @handle_orch_error
     def remove_host(self, host: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> str:
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index ecd94ac084e8..262ecbd27f6a 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -117,7 +117,7 @@ def serve(self) -> None:
                         continue
 
                     # refresh node-proxy cache
-                    self.mgr.node_proxy.load()
+                    self.mgr.node_proxy_cache.load()
 
             except OrchestratorError as e:
                 if e.event_subject:
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index 3b05a4f39a54..b713d04cd597 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -2,7 +2,7 @@
 import json
 from _pytest.monkeypatch import MonkeyPatch
 from cherrypy.test import helper
-from cephadm.agent import NodeProxy
+from cephadm.agent import NodeProxyEndpoint
 from unittest.mock import MagicMock, call, patch
 from cephadm.inventory import AgentCache, NodeProxyCache, Inventory
 from cephadm.ssl_cert_utils import SSLCerts
@@ -21,8 +21,8 @@ def __init__(self) -> None:
         self.inventory = Inventory(self)
         self.agent_cache = AgentCache(self)
         self.agent_cache.agent_ports = {"host01": 1234}
-        self.node_proxy = NodeProxyCache(self)
-        self.node_proxy.save = MagicMock()
+        self.node_proxy_cache = NodeProxyCache(self)
+        self.node_proxy_cache.save = MagicMock()
         self.http_server = MagicMock()
         self.http_server.agent = MagicMock()
         self.http_server.agent.ssl_certs = SSLCerts()
@@ -32,9 +32,9 @@ def get_mgr_ip(self) -> str:
         return '0.0.0.0'
 
 
-class TestNodeProxy(helper.CPWebCase):
+class TestNodeProxyEndpoint(helper.CPWebCase):
     mgr = FakeMgr()
-    app = NodeProxy(mgr)
+    app = NodeProxyEndpoint(mgr)
     mgr.agent_cache.agent_keys = {"host01": "fake-secret01",
                                   "host02": "fake-secret02"}
     mgr.node_proxy.oob = {"host01": {"username": "oob-user01",
@@ -45,8 +45,8 @@ class TestNodeProxy(helper.CPWebCase):
 
     @classmethod
     def setup_server(cls):
-        # cherrypy.tree.mount(NodeProxy(TestNodeProxy.mgr))
-        cherrypy.tree.mount(TestNodeProxy.app)
+        # cherrypy.tree.mount(NodeProxyEndpoint(TestNodeProxyEndpoint.mgr))
+        cherrypy.tree.mount(TestNodeProxyEndpoint.app)
         cherrypy.config.update({'global': {
             'server.socket_host': '127.0.0.1',
             'server.socket_port': PORT}})
@@ -115,7 +115,7 @@ def test_data_raises_alert(self):
                       detail=['dimm.socket.a1 is critical: Enabled'],
                       summary='1 memory member is not ok')]
 
-        assert TestNodeProxy.mgr.set_health_warning.mock_calls == calls
+        assert TestNodeProxyEndpoint.mgr.set_health_warning.mock_calls == calls
 
     # @pytest.mark.parametrize("method", ["GET", "PATCH"])
     # def test_led_no_hostname(self, method):
@@ -213,7 +213,7 @@ def test_get_drive_led_with_id(self, m_agent_msg_thread_start, m_agent_msg_threa
             self.assertStatus('200 OK')
 
     # def test_led_endpoint_unreachable(self):
-    #     TestNodeProxy.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
+    #     TestNodeProxyEndpoint.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
     #     self.getPage("/host02/led", method="GET")
     #     calls = [call(addr='10.10.10.12',
     #                   data=None,
@@ -221,9 +221,9 @@ def test_get_drive_led_with_id(self, m_agent_msg_thread_start, m_agent_msg_threa
     #                   headers={},
     #                   method='GET',
     #                   port=8080,
-    #                   ssl_ctx=TestNodeProxy.app.ssl_ctx)]
+    #                   ssl_ctx=TestNodeProxyEndpoint.app.ssl_ctx)]
     #     self.assertStatus('502 Bad Gateway')
-    #     assert TestNodeProxy.app.query_endpoint.mock_calls == calls
+    #     assert TestNodeProxyEndpoint.app.query_endpoint.mock_calls == calls
 
     def test_fullreport_with_valid_hostname(self):
         self.getPage("/host02/fullreport", method="GET")

From 3b8c945a6afeb7c8ab9f862468929f579c78adc3 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 15 Jan 2024 12:38:39 +0000
Subject: [PATCH 1608/2492] node-proxy: make it a separate daemon

The current implementation requires the inclusion of all the recent
modifications in the cephadm binary, which won't be backported.

Since we need the node-proxy code backported to reef, let's move the
code make it a separate daemon.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
Co-authored-by: Adam King <adking@redhat.com>
---
 .../ceph_node_proxy}/__init__.py              |   0
 src/ceph-node-proxy/ceph_node_proxy/api.py    | 291 ++++++++++++++++++
 .../ceph_node_proxy}/baseclient.py            |   0
 .../ceph_node_proxy/baseredfishsystem.py      | 289 +++++++++++++++++
 .../ceph_node_proxy}/basesystem.py            |  34 +-
 src/ceph-node-proxy/ceph_node_proxy/main.py   | 244 +++++++++++++++
 .../ceph_node_proxy}/redfish_client.py        |  21 +-
 .../ceph_node_proxy}/redfishdellsystem.py     |  31 +-
 .../ceph_node_proxy}/reporter.py              |   2 +-
 .../ceph_node_proxy}/util.py                  |  17 +-
 src/cephadm/cephadm.py                        | 117 +------
 src/cephadm/cephadmlib/daemons/__init__.py    |   2 +
 src/cephadm/cephadmlib/daemons/node_proxy.py  | 145 +++++++++
 .../node_proxy/baseredfishsystem.py           | 152 ---------
 src/cephadm/cephadmlib/node_proxy/main.py     |  94 ------
 src/mypy.ini                                  |   2 +
 src/pybind/mgr/cephadm/agent.py               | 104 ++++---
 src/pybind/mgr/cephadm/inventory.py           |  23 +-
 src/pybind/mgr/cephadm/module.py              |  84 ++++-
 src/pybind/mgr/cephadm/serve.py               |   3 +
 .../mgr/cephadm/services/cephadmservice.py    |  12 +-
 src/pybind/mgr/cephadm/services/node_proxy.py | 180 +++++++++++
 .../mgr/cephadm/tests/test_node_proxy.py      |  16 +-
 src/pybind/mgr/cephadm/utils.py               |   2 +-
 src/pybind/mgr/orchestrator/_interface.py     |  29 ++
 src/pybind/mgr/orchestrator/module.py         |  45 +++
 .../ceph/deployment/service_spec.py           |   3 +-
 src/python-common/ceph/utils.py               |  48 ++-
 28 files changed, 1511 insertions(+), 479 deletions(-)
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/__init__.py (100%)
 create mode 100644 src/ceph-node-proxy/ceph_node_proxy/api.py
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/baseclient.py (100%)
 create mode 100644 src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/basesystem.py (64%)
 create mode 100644 src/ceph-node-proxy/ceph_node_proxy/main.py
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/redfish_client.py (88%)
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/redfishdellsystem.py (84%)
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/reporter.py (98%)
 rename src/{cephadm/cephadmlib/node_proxy => ceph-node-proxy/ceph_node_proxy}/util.py (88%)
 create mode 100644 src/cephadm/cephadmlib/daemons/node_proxy.py
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
 delete mode 100644 src/cephadm/cephadmlib/node_proxy/main.py
 create mode 100644 src/pybind/mgr/cephadm/services/node_proxy.py

diff --git a/src/cephadm/cephadmlib/node_proxy/__init__.py b/src/ceph-node-proxy/ceph_node_proxy/__init__.py
similarity index 100%
rename from src/cephadm/cephadmlib/node_proxy/__init__.py
rename to src/ceph-node-proxy/ceph_node_proxy/__init__.py
diff --git a/src/ceph-node-proxy/ceph_node_proxy/api.py b/src/ceph-node-proxy/ceph_node_proxy/api.py
new file mode 100644
index 000000000000..93e41def3bfe
--- /dev/null
+++ b/src/ceph-node-proxy/ceph_node_proxy/api.py
@@ -0,0 +1,291 @@
+import cherrypy
+from urllib.error import HTTPError
+from cherrypy._cpserver import Server
+from threading import Thread, Event
+from typing import Dict, Any, List
+from ceph_node_proxy.util import Config, Logger, write_tmp_file
+from ceph_node_proxy.basesystem import BaseSystem
+from ceph_node_proxy.reporter import Reporter
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from ceph_node_proxy.main import NodeProxy
+
+
+@cherrypy.tools.auth_basic(on=True)
+@cherrypy.tools.allow(methods=['PUT'])
+@cherrypy.tools.json_out()
+class Admin():
+    def __init__(self, api: 'API') -> None:
+        self.api = api
+
+    @cherrypy.expose
+    def start(self) -> Dict[str, str]:
+        self.api.backend.start_client()
+        # self.backend.start_update_loop()
+        self.api.reporter.run()
+        return {'ok': 'node-proxy daemon started'}
+
+    @cherrypy.expose
+    def reload(self) -> Dict[str, str]:
+        self.api.config.reload()
+        return {'ok': 'node-proxy config reloaded'}
+
+    def _stop(self) -> None:
+        self.api.backend.stop_update_loop()
+        self.api.backend.client.logout()
+        self.api.reporter.stop()
+
+    @cherrypy.expose
+    def stop(self) -> Dict[str, str]:
+        self._stop()
+        return {'ok': 'node-proxy daemon stopped'}
+
+    @cherrypy.expose
+    def shutdown(self) -> Dict[str, str]:
+        self._stop()
+        cherrypy.engine.exit()
+        return {'ok': 'Server shutdown.'}
+
+    @cherrypy.expose
+    def flush(self) -> Dict[str, str]:
+        self.api.backend.flush()
+        return {'ok': 'node-proxy data flushed'}
+
+
+class API(Server):
+    def __init__(self,
+                 backend: 'BaseSystem',
+                 reporter: 'Reporter',
+                 config: 'Config',
+                 addr: str = '0.0.0.0',
+                 port: int = 0) -> None:
+        super().__init__()
+        self.log = Logger(__name__)
+        self.backend = backend
+        self.reporter = reporter
+        self.config = config
+        self.socket_port = self.config.__dict__['server']['port'] if not port else port
+        self.socket_host = addr
+        self.subscribe()
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def memory(self) -> Dict[str, Any]:
+        return {'memory': self.backend.get_memory()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def network(self) -> Dict[str, Any]:
+        return {'network': self.backend.get_network()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def processors(self) -> Dict[str, Any]:
+        return {'processors': self.backend.get_processors()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def storage(self) -> Dict[str, Any]:
+        return {'storage': self.backend.get_storage()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def power(self) -> Dict[str, Any]:
+        return {'power': self.backend.get_power()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def fans(self) -> Dict[str, Any]:
+        return {'fans': self.backend.get_fans()}
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def firmwares(self) -> Dict[str, Any]:
+        return {'firmwares': self.backend.get_firmwares()}
+
+    def _cp_dispatch(self, vpath: List[str]) -> 'API':
+        if vpath[0] == 'led' and len(vpath) > 1:  # /led/{type}/{id}
+            _type = vpath[1]
+            cherrypy.request.params['type'] = _type
+            vpath.pop(1)  # /led/{id} or # /led
+            if _type == 'drive' and len(vpath) > 1:  # /led/{id}
+                _id = vpath[1]
+                vpath.pop(1)  # /led
+                cherrypy.request.params['id'] = _id
+            vpath[0] = '_led'
+        # /<endpoint>
+        return self
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['POST'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.auth_basic(on=True)
+    def shutdown(self, **kw: Any) -> int:
+        data: Dict[str, bool] = cherrypy.request.json
+
+        if 'force' not in data.keys():
+            msg = "The key 'force' wasn't passed."
+            self.log.logger.debug(msg)
+            raise cherrypy.HTTPError(400, msg)
+        try:
+            result: int = self.backend.shutdown(force=data['force'])
+        except HTTPError as e:
+            raise cherrypy.HTTPError(e.code, e.reason)
+        return result
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['POST'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.auth_basic(on=True)
+    def powercycle(self, **kw: Any) -> int:
+        try:
+            result: int = self.backend.powercycle()
+        except HTTPError as e:
+            raise cherrypy.HTTPError(e.code, e.reason)
+        return result
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET', 'PATCH'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.auth_basic(on=True)
+    def _led(self, **kw: Any) -> Dict[str, Any]:
+        method: str = cherrypy.request.method
+        led_type: Optional[str] = kw.get('type')
+        id_drive: Optional[str] = kw.get('id')
+        result: Dict[str, Any] = dict()
+
+        if not led_type:
+            msg = "the led type must be provided (either 'chassis' or 'drive')."
+            self.log.logger.debug(msg)
+            raise cherrypy.HTTPError(400, msg)
+
+        if led_type == 'drive':
+            id_drive_required = not id_drive
+            if id_drive_required or id_drive not in self.backend.get_storage():
+                msg = 'A valid device ID must be provided.'
+                self.log.logger.debug(msg)
+                raise cherrypy.HTTPError(400, msg)
+
+        try:
+            if method == 'PATCH':
+                data: Dict[str, Any] = cherrypy.request.json
+
+                if 'state' not in data or data['state'] not in ['on', 'off']:
+                    msg = "Invalid data. 'state' must be provided and have a valid value (on|off)."
+                    self.log.logger.error(msg)
+                    raise cherrypy.HTTPError(400, msg)
+
+                func: Any = (self.backend.device_led_on if led_type == 'drive' and data['state'] == 'on' else
+                             self.backend.device_led_off if led_type == 'drive' and data['state'] == 'off' else
+                             self.backend.chassis_led_on if led_type != 'drive' and data['state'] == 'on' else
+                             self.backend.chassis_led_off if led_type != 'drive' and data['state'] == 'off' else None)
+
+            else:
+                func = self.backend.get_device_led if led_type == 'drive' else self.backend.get_chassis_led
+
+            result = func(id_drive) if led_type == 'drive' else func()
+
+        except HTTPError as e:
+            raise cherrypy.HTTPError(e.code, e.reason)
+        return result
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['GET'])
+    @cherrypy.tools.json_out()
+    def get_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
+        return self.backend.get_led()
+
+    @cherrypy.expose
+    @cherrypy.tools.allow(methods=['PATCH'])
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.auth_basic(on=True)
+    def set_led(self, **kw: Dict[str, Any]) -> Dict[str, Any]:
+        data = cherrypy.request.json
+        rc = self.backend.set_led(data)
+
+        if rc != 200:
+            cherrypy.response.status = rc
+            result = {'state': 'error: please, verify the data you sent.'}
+        else:
+            result = {'state': data['state'].lower()}
+        return result
+
+    def stop(self) -> None:
+        self.unsubscribe()
+        super().stop()
+
+
+class NodeProxyApi(Thread):
+    def __init__(self,
+                 node_proxy: 'NodeProxy',
+                 username: str,
+                 password: str,
+                 ssl_crt: str,
+                 ssl_key: str) -> None:
+        super().__init__()
+        self.log = Logger(__name__)
+        self.cp_shutdown_event = Event()
+        self.node_proxy = node_proxy
+        self.username = username
+        self.password = password
+        self.ssl_crt = ssl_crt
+        self.ssl_key = ssl_key
+        self.api = API(self.node_proxy.system,
+                       self.node_proxy.reporter_agent,
+                       self.node_proxy.config)
+
+    def check_auth(self, realm: str, username: str, password: str) -> bool:
+        return self.username == username and \
+            self.password == password
+
+    def shutdown(self) -> None:
+        self.log.logger.info('Stopping node-proxy API...')
+        self.cp_shutdown_event.set()
+
+    def run(self) -> None:
+        self.log.logger.info('node-proxy API configuration...')
+        cherrypy.config.update({
+            'environment': 'production',
+            'engine.autoreload.on': False,
+            'log.screen': True,
+        })
+        config = {'/': {
+            'request.methods_with_bodies': ('POST', 'PUT', 'PATCH'),
+            'tools.trailing_slash.on': False,
+            'tools.auth_basic.realm': 'localhost',
+            'tools.auth_basic.checkpassword': self.check_auth
+        }}
+        cherrypy.tree.mount(self.api, '/', config=config)
+        # cherrypy.tree.mount(admin, '/admin', config=config)
+
+        ssl_crt = write_tmp_file(self.ssl_crt,
+                                 prefix_name='listener-crt-')
+        ssl_key = write_tmp_file(self.ssl_key,
+                                 prefix_name='listener-key-')
+
+        self.api.ssl_certificate = ssl_crt.name
+        self.api.ssl_private_key = ssl_key.name
+
+        cherrypy.server.unsubscribe()
+        try:
+            cherrypy.engine.start()
+            self.log.logger.info('node-proxy API started.')
+            self.cp_shutdown_event.wait()
+            self.cp_shutdown_event.clear()
+            cherrypy.engine.stop()
+            cherrypy.server.httpserver = None
+            self.log.logger.info('node-proxy API shutdown.')
+        except Exception as e:
+            self.log.logger.error(f'node-proxy API error: {e}')
diff --git a/src/cephadm/cephadmlib/node_proxy/baseclient.py b/src/ceph-node-proxy/ceph_node_proxy/baseclient.py
similarity index 100%
rename from src/cephadm/cephadmlib/node_proxy/baseclient.py
rename to src/ceph-node-proxy/ceph_node_proxy/baseclient.py
diff --git a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
new file mode 100644
index 000000000000..98f117196158
--- /dev/null
+++ b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
@@ -0,0 +1,289 @@
+import concurrent.futures
+import json
+from ceph_node_proxy.basesystem import BaseSystem
+from ceph_node_proxy.redfish_client import RedFishClient
+from threading import Thread, Lock
+from time import sleep
+from ceph_node_proxy.util import Logger, retry
+from typing import Dict, Any, List, Callable, Union
+from urllib.error import HTTPError, URLError
+
+
+class BaseRedfishSystem(BaseSystem):
+    def __init__(self, **kw: Any) -> None:
+        super().__init__(**kw)
+        self.common_endpoints: List[str] = kw.get('common_endpoints', ['/Systems/System.Embedded.1',
+                                                                       '/UpdateService'])
+        self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
+        self.log = Logger(__name__)
+        self.host: str = kw['host']
+        self.port: str = kw['port']
+        self.username: str = kw['username']
+        self.password: str = kw['password']
+        # move the following line (class attribute?)
+        self.client: RedFishClient = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
+        self.log.logger.info(f'redfish system initialization, host: {self.host}, user: {self.username}')
+
+        self.run: bool = False
+        self.thread: Thread
+        self.data_ready: bool = False
+        self.previous_data: Dict = {}
+        self.lock: Lock = Lock()
+        self.data: Dict[str, Dict[str, Any]] = {}
+        self._system: Dict[str, Dict[str, Any]] = {}
+        self._sys: Dict[str, Any] = {}
+        self.job_service_endpoint: str = ''
+        self.create_reboot_job_endpoint: str = ''
+        self.setup_job_queue_endpoint: str = ''
+
+        self.start_client()
+
+    def start_client(self) -> None:
+        self.client.login()
+        self.start_update_loop()
+
+    def start_update_loop(self) -> None:
+        self.run = True
+        self.thread = Thread(target=self.update)
+        self.thread.start()
+
+    def stop_update_loop(self) -> None:
+        self.run = False
+        self.thread.join()
+
+    def update(self) -> None:
+        #  this loop can have:
+        #  - caching logic
+        while self.run:
+            self.log.logger.debug('waiting for a lock in the update loop.')
+            self.lock.acquire()
+            self.log.logger.debug('lock acquired in the update loop.')
+            try:
+                self._update_system()
+                self._update_sn()
+                update_funcs = [self._update_memory,
+                                self._update_power,
+                                self._update_fans,
+                                self._update_network,
+                                self._update_processors,
+                                self._update_storage,
+                                self._update_firmwares]
+
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    executor.map(lambda f: f(), update_funcs)
+
+                self.data_ready = True
+                sleep(5)
+            except RuntimeError as e:
+                self.run = False
+                self.log.logger.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
+                self.client.logout()
+            finally:
+                self.lock.release()
+                self.log.logger.debug('lock released in the update loop.')
+
+    def flush(self) -> None:
+        self.log.logger.debug('Acquiring lock to flush data.')
+        self.lock.acquire()
+        self.log.logger.debug('Lock acquired, flushing data.')
+        self._system = {}
+        self.previous_data = {}
+        self.log.logger.info('Data flushed.')
+        self.data_ready = False
+        self.log.logger.debug('Data marked as not ready.')
+        self.lock.release()
+        self.log.logger.debug('Released the lock after flushing data.')
+
+    @retry(retries=10, delay=2)
+    def _get_path(self, path: str) -> Dict:
+        try:
+            result = self.client.get_path(path)
+        except RuntimeError:
+            raise
+        if result is None:
+            self.log.logger.error(f'The client reported an error when getting path: {path}')
+            raise RuntimeError(f'Could not get path: {path}')
+        return result
+
+    def get_members(self, data: Dict[str, Any], path: str) -> List:
+        _path = data[path]['@odata.id']
+        _data = self._get_path(_path)
+        return [self._get_path(member['@odata.id']) for member in _data['Members']]
+
+    def get_system(self) -> Dict[str, Any]:
+        result = {
+            'host': self.get_host(),
+            'sn': self.get_sn(),
+            'status': {
+                'storage': self.get_storage(),
+                'processors': self.get_processors(),
+                'network': self.get_network(),
+                'memory': self.get_memory(),
+                'power': self.get_power(),
+                'fans': self.get_fans()
+            },
+            'firmwares': self.get_firmwares(),
+            'chassis': {'redfish_endpoint': f'/redfish/v1{self.chassis_endpoint}'}  # TODO(guits): not ideal
+        }
+        return result
+
+    def _update_system(self) -> None:
+        for endpoint in self.common_endpoints:
+            result = self.client.get_path(endpoint)
+            _endpoint = endpoint.strip('/').split('/')[0]
+            self._system[_endpoint] = result
+
+    def _update_sn(self) -> None:
+        raise NotImplementedError()
+
+    def _update_memory(self) -> None:
+        raise NotImplementedError()
+
+    def _update_power(self) -> None:
+        raise NotImplementedError()
+
+    def _update_fans(self) -> None:
+        raise NotImplementedError()
+
+    def _update_network(self) -> None:
+        raise NotImplementedError()
+
+    def _update_processors(self) -> None:
+        raise NotImplementedError()
+
+    def _update_storage(self) -> None:
+        raise NotImplementedError()
+
+    def _update_firmwares(self) -> None:
+        raise NotImplementedError()
+
+    def device_led_on(self, device: str) -> int:
+        data: Dict[str, bool] = {'LocationIndicatorActive': True}
+        try:
+            result = self.set_device_led(device, data)
+        except (HTTPError, KeyError):
+            return 0
+        return result
+
+    def device_led_off(self, device: str) -> int:
+        data: Dict[str, bool] = {'LocationIndicatorActive': False}
+        try:
+            result = self.set_device_led(device, data)
+        except (HTTPError, KeyError):
+            return 0
+        return result
+
+    def chassis_led_on(self) -> int:
+        data: Dict[str, str] = {'IndicatorLED': 'Blinking'}
+        result = self.set_chassis_led(data)
+        return result
+
+    def chassis_led_off(self) -> int:
+        data: Dict[str, str] = {'IndicatorLED': 'Lit'}
+        result = self.set_chassis_led(data)
+        return result
+
+    def get_device_led(self, device: str) -> Dict[str, Any]:
+        endpoint = self._sys['storage'][device]['redfish_endpoint']
+        try:
+            result = self.client.query(method='GET',
+                                       endpoint=endpoint,
+                                       timeout=10)
+        except HTTPError as e:
+            self.log.logger.error(f"Couldn't get the ident device LED status for device '{device}': {e}")
+            raise
+        response_json = json.loads(result[1])
+        _result: Dict[str, Any] = {'http_code': result[2]}
+        if result[2] == 200:
+            _result['LocationIndicatorActive'] = response_json['LocationIndicatorActive']
+        else:
+            _result['LocationIndicatorActive'] = None
+        return _result
+
+    def set_device_led(self, device: str, data: Dict[str, bool]) -> int:
+        try:
+            _, response, status = self.client.query(
+                data=json.dumps(data),
+                method='PATCH',
+                endpoint=self._sys['storage'][device]['redfish_endpoint']
+            )
+        except (HTTPError, KeyError) as e:
+            self.log.logger.error(f"Couldn't set the ident device LED for device '{device}': {e}")
+            raise
+        return status
+
+    def get_chassis_led(self) -> Dict[str, Any]:
+        endpoint = f'/redfish/v1/{self.chassis_endpoint}'
+        try:
+            result = self.client.query(method='GET',
+                                       endpoint=endpoint,
+                                       timeout=10)
+        except HTTPError as e:
+            self.log.logger.error(f"Couldn't get the ident chassis LED status: {e}")
+            raise
+        response_json = json.loads(result[1])
+        _result: Dict[str, Any] = {'http_code': result[2]}
+        if result[2] == 200:
+            _result['LocationIndicatorActive'] = response_json['LocationIndicatorActive']
+        else:
+            _result['LocationIndicatorActive'] = None
+        return _result
+
+    def set_chassis_led(self, data: Dict[str, str]) -> int:
+        # '{"IndicatorLED": "Lit"}'      -> LocationIndicatorActive = false
+        # '{"IndicatorLED": "Blinking"}' -> LocationIndicatorActive = true
+        try:
+            _, response, status = self.client.query(
+                data=json.dumps(data),
+                method='PATCH',
+                endpoint=f'/redfish/v1{self.chassis_endpoint}'
+            )
+        except HTTPError as e:
+            self.log.logger.error(f"Couldn't set the ident chassis LED: {e}")
+            raise
+        return status
+
+    def shutdown(self, force: bool = False) -> int:
+        reboot_type: str = 'GracefulRebootWithForcedShutdown' if force else 'GracefulRebootWithoutForcedShutdown'
+
+        try:
+            job_id: str = self.create_reboot_job(reboot_type)
+            status = self.schedule_reboot_job(job_id)
+        except (HTTPError, KeyError) as e:
+            self.log.logger.error(f"Couldn't create the reboot job: {e}")
+            raise
+        return status
+
+    def powercycle(self) -> int:
+        try:
+            job_id: str = self.create_reboot_job('PowerCycle')
+            status = self.schedule_reboot_job(job_id)
+        except (HTTPError, URLError) as e:
+            self.log.logger.error(f"Couldn't perform power cycle: {e}")
+            raise
+        return status
+
+    def create_reboot_job(self, reboot_type: str) -> str:
+        data: Dict[str, str] = dict(RebootJobType=reboot_type)
+        try:
+            headers, response, status = self.client.query(
+                data=json.dumps(data),
+                endpoint=self.create_reboot_job_endpoint
+            )
+            job_id: str = headers['Location'].split('/')[-1]
+        except (HTTPError, URLError) as e:
+            self.log.logger.error(f"Couldn't create the reboot job: {e}")
+            raise
+        return job_id
+
+    def schedule_reboot_job(self, job_id: str) -> int:
+        data: Dict[str, Union[List[str], str]] = dict(JobArray=[job_id], StartTimeInterval='TIME_NOW')
+        try:
+            headers, response, status = self.client.query(
+                data=json.dumps(data),
+                endpoint=self.setup_job_queue_endpoint
+            )
+        except (HTTPError, KeyError) as e:
+            self.log.logger.error(f"Couldn't schedule the reboot job: {e}")
+            raise
+        return status
diff --git a/src/cephadm/cephadmlib/node_proxy/basesystem.py b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
similarity index 64%
rename from src/cephadm/cephadmlib/node_proxy/basesystem.py
rename to src/ceph-node-proxy/ceph_node_proxy/basesystem.py
index 4fb1b7b85534..c2389d8dc85e 100644
--- a/src/cephadm/cephadmlib/node_proxy/basesystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
@@ -1,7 +1,7 @@
 import socket
-from .util import Config
+from ceph_node_proxy.util import Config
 from typing import Dict, Any
-from .baseclient import BaseClient
+from ceph_node_proxy.baseclient import BaseClient
 
 
 class BaseSystem:
@@ -49,6 +49,30 @@ def get_led(self) -> Dict[str, Any]:
     def set_led(self, data: Dict[str, str]) -> int:
         raise NotImplementedError()
 
+    def get_chassis_led(self) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    def set_chassis_led(self, data: Dict[str, str]) -> int:
+        raise NotImplementedError()
+
+    def device_led_on(self, device: str) -> int:
+        raise NotImplementedError()
+
+    def device_led_off(self, device: str) -> int:
+        raise NotImplementedError()
+
+    def get_device_led(self, device: str) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    def set_device_led(self, device: str, data: Dict[str, bool]) -> int:
+        raise NotImplementedError()
+
+    def chassis_led_on(self) -> int:
+        raise NotImplementedError()
+
+    def chassis_led_off(self) -> int:
+        raise NotImplementedError()
+
     def get_host(self) -> str:
         return socket.gethostname()
 
@@ -63,3 +87,9 @@ def start_client(self) -> None:
 
     def flush(self) -> None:
         raise NotImplementedError()
+
+    def shutdown(self, force: bool = False) -> int:
+        raise NotImplementedError()
+
+    def powercycle(self) -> int:
+        raise NotImplementedError()
diff --git a/src/ceph-node-proxy/ceph_node_proxy/main.py b/src/ceph-node-proxy/ceph_node_proxy/main.py
new file mode 100644
index 000000000000..689089aa609f
--- /dev/null
+++ b/src/ceph-node-proxy/ceph_node_proxy/main.py
@@ -0,0 +1,244 @@
+from threading import Thread
+from ceph_node_proxy.redfishdellsystem import RedfishDellSystem
+from ceph_node_proxy.api import NodeProxyApi
+from ceph_node_proxy.reporter import Reporter
+from ceph_node_proxy.util import Config, Logger, http_req, write_tmp_file
+from typing import Dict, Any, Optional
+
+import argparse
+import traceback
+import logging
+import os
+import ssl
+import json
+import time
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CONFIG = {
+    'reporter': {
+        'check_interval': 5,
+        'push_data_max_retries': 30,
+        'endpoint': 'https://127.0.0.1:7150/node-proxy/data',
+    },
+    'system': {
+        'refresh_interval': 5
+    },
+    'server': {
+        'port': 8080,
+    },
+    'logging': {
+        'level': 20,
+    }
+}
+
+
+class NodeProxyManager(Thread):
+    def __init__(self,
+                 mgr_host: str,
+                 cephx_name: str,
+                 cephx_secret: str,
+                 ca_path: str,
+                 api_ssl_crt: str,
+                 api_ssl_key: str,
+                 mgr_agent_port: int = 7150):
+        super().__init__()
+        self.mgr_host = mgr_host
+        self.cephx_name = cephx_name
+        self.cephx_secret = cephx_secret
+        self.ca_path = ca_path
+        self.api_ssl_crt = api_ssl_crt
+        self.api_ssl_key = api_ssl_key
+        self.mgr_agent_port = str(mgr_agent_port)
+        self.stop = False
+        self.ssl_ctx = ssl.create_default_context()
+        self.ssl_ctx.check_hostname = True
+        self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        self.ssl_ctx.load_verify_locations(self.ca_path)
+
+    def run(self) -> None:
+        self.init()
+        self.loop()
+
+    def init(self) -> None:
+        node_proxy_meta = {
+            'cephx': {
+                'name': self.cephx_name,
+                'secret': self.cephx_secret
+            }
+        }
+        headers, result, status = http_req(hostname=self.mgr_host,
+                                           port=self.mgr_agent_port,
+                                           data=json.dumps(node_proxy_meta),
+                                           endpoint='/node-proxy/oob',
+                                           ssl_ctx=self.ssl_ctx)
+        if status != 200:
+            msg = f'No out of band tool details could be loaded: {status}, {result}'
+            logger.debug(msg)
+            raise RuntimeError(msg)
+
+        result_json = json.loads(result)
+        kwargs = {
+            'host': result_json['result']['addr'],
+            'username': result_json['result']['username'],
+            'password': result_json['result']['password'],
+            'cephx': node_proxy_meta['cephx'],
+            'mgr_host': self.mgr_host,
+            'mgr_agent_port': self.mgr_agent_port,
+            'api_ssl_crt': self.api_ssl_crt,
+            'api_ssl_key': self.api_ssl_key
+        }
+        if result_json['result'].get('port'):
+            kwargs['port'] = result_json['result']['port']
+
+        self.node_proxy: NodeProxy = NodeProxy(**kwargs)
+        self.node_proxy.start()
+
+    def loop(self) -> None:
+        while not self.stop:
+            try:
+                status = self.node_proxy.check_status()
+                label = 'Ok' if status else 'Critical'
+                logger.debug(f'node-proxy status: {label}')
+            except Exception as e:
+                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
+                time.sleep(120)
+                self.init()
+            else:
+                logger.debug('node-proxy alive, next check in 60sec.')
+                time.sleep(60)
+
+    def shutdown(self) -> None:
+        self.stop = True
+        # if `self.node_proxy.shutdown()` is called before self.start(), it will fail.
+        if self.__dict__.get('node_proxy'):
+            self.node_proxy.shutdown()
+
+
+class NodeProxy(Thread):
+    def __init__(self, **kw: Any) -> None:
+        super().__init__()
+        self.username: str = kw.get('username', '')
+        self.password: str = kw.get('password', '')
+        self.host: str = kw.get('host', '')
+        self.port: int = kw.get('port', 443)
+        self.cephx: Dict[str, Any] = kw.get('cephx', {})
+        self.reporter_scheme: str = kw.get('reporter_scheme', 'https')
+        self.mgr_host: str = kw.get('mgr_host', '')
+        self.mgr_agent_port: str = kw.get('mgr_agent_port', '')
+        self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
+        self.api_ssl_crt: str = kw.get('api_ssl_crt', '')
+        self.api_ssl_key: str = kw.get('api_ssl_key', '')
+        self.exc: Optional[Exception] = None
+        self.log = Logger(__name__)
+
+    def run(self) -> None:
+        try:
+            self.main()
+        except Exception as e:
+            self.exc = e
+            return
+
+    def shutdown(self) -> None:
+        self.log.logger.info('Shutting down node-proxy...')
+        self.system.client.logout()
+        self.system.stop_update_loop()
+        self.reporter_agent.stop()
+
+    def check_status(self) -> bool:
+        if self.__dict__.get('system') and not self.system.run:
+            raise RuntimeError('node-proxy encountered an error.')
+        if self.exc:
+            traceback.print_tb(self.exc.__traceback__)
+            self.log.logger.error(f'{self.exc.__class__.__name__}: {self.exc}')
+            raise self.exc
+        return True
+
+    def main(self) -> None:
+        # TODO: add a check and fail if host/username/password/data aren't passed
+        self.config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
+        self.log = Logger(__name__, level=self.config.__dict__['logging']['level'])
+
+        # create the redfish system and the obsever
+        self.log.logger.info('Server initialization...')
+        try:
+            self.system = RedfishDellSystem(host=self.host,
+                                            port=self.port,
+                                            username=self.username,
+                                            password=self.password,
+                                            config=self.config)
+        except RuntimeError:
+            self.log.logger.error("Can't initialize the redfish system.")
+            raise
+
+        try:
+            self.reporter_agent = Reporter(self.system,
+                                           self.cephx,
+                                           reporter_scheme=self.reporter_scheme,
+                                           reporter_hostname=self.mgr_host,
+                                           reporter_port=self.mgr_agent_port,
+                                           reporter_endpoint=self.reporter_endpoint)
+            self.reporter_agent.run()
+        except RuntimeError:
+            self.log.logger.error("Can't initialize the reporter.")
+            raise
+
+        try:
+            self.log.logger.info('Starting node-proxy API...')
+            self.api = NodeProxyApi(self,
+                                    username=self.username,
+                                    password=self.password,
+                                    ssl_crt=self.api_ssl_crt,
+                                    ssl_key=self.api_ssl_key)
+            self.api.start()
+        except Exception as e:
+            self.log.logger.error(f"Can't start node-proxy API: {e}")
+            raise
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description='Ceph Node-Proxy for HW Monitoring',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        '--config',
+        help='path of config file in json format',
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.config):
+        raise Exception(f'No config file found at provided config path: {args.config}')
+
+    with open(args.config, 'r') as f:
+        try:
+            config_json = f.read()
+            config = json.loads(config_json)
+        except Exception as e:
+            raise Exception(f'Failed to load json config: {str(e)}')
+
+    target_ip = config['target_ip']
+    target_port = config['target_port']
+    keyring = config['keyring']
+    root_cert = config['root_cert.pem']
+    listener_cert = config['listener.crt']
+    listener_key = config['listener.key']
+    name = config['name']
+
+    f = write_tmp_file(root_cert,
+                       prefix_name='cephadm-endpoint-root-cert')
+
+    node_proxy_mgr = NodeProxyManager(mgr_host=target_ip,
+                                      cephx_name=name,
+                                      cephx_secret=keyring,
+                                      mgr_agent_port=target_port,
+                                      ca_path=f.name,
+                                      api_ssl_crt=listener_cert,
+                                      api_ssl_key=listener_key)
+    if not node_proxy_mgr.is_alive():
+        node_proxy_mgr.start()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/cephadm/cephadmlib/node_proxy/redfish_client.py b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
similarity index 88%
rename from src/cephadm/cephadmlib/node_proxy/redfish_client.py
rename to src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
index 040db8ce2521..eeca2e5ba217 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfish_client.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
@@ -1,7 +1,7 @@
 import json
 from urllib.error import HTTPError, URLError
-from .baseclient import BaseClient
-from .util import Logger, http_req
+from ceph_node_proxy.baseclient import BaseClient
+from ceph_node_proxy.util import Logger, http_req
 from typing import Dict, Any, Tuple, Optional
 from http.client import HTTPMessage
 
@@ -57,21 +57,24 @@ def is_logged_in(self) -> bool:
         except URLError as e:
             self.log.logger.error("Can't check token "
                                   f'validity for {self.url}: {e}')
-            raise RuntimeError
+            raise
         return _status_code == 200
 
     def logout(self) -> Dict[str, Any]:
+        result: Dict[str, Any] = {}
         try:
-            _, _data, _status_code = self.query(method='DELETE',
-                                                headers={'X-Auth-Token': self.token},
-                                                endpoint=self.location)
+            if self.is_logged_in():
+                _, _data, _status_code = self.query(method='DELETE',
+                                                    headers={'X-Auth-Token': self.token},
+                                                    endpoint=self.location)
+                result = json.loads(_data)
         except URLError:
             self.log.logger.error(f"Can't log out from {self.url}")
-            return {}
 
-        response_str = _data
+        self.location = ''
+        self.token = ''
 
-        return json.loads(response_str)
+        return result
 
     def get_path(self, path: str) -> Dict[str, Any]:
         if self.PREFIX not in path:
diff --git a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
similarity index 84%
rename from src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
rename to src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
index 71006c48bf86..0424bb38bd1e 100644
--- a/src/cephadm/cephadmlib/node_proxy/redfishdellsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
@@ -1,6 +1,5 @@
-import json
-from .baseredfishsystem import BaseRedfishSystem
-from .util import Logger, normalize_dict, to_snake_case
+from ceph_node_proxy.baseredfishsystem import BaseRedfishSystem
+from ceph_node_proxy.util import Logger, normalize_dict, to_snake_case
 from typing import Dict, Any, List
 
 
@@ -8,6 +7,9 @@ class RedfishDellSystem(BaseRedfishSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
         self.log = Logger(__name__)
+        self.job_service_endpoint: str = '/redfish/v1/Managers/iDRAC.Embedded.1/Oem/Dell/DellJobService'
+        self.create_reboot_job_endpoint: str = f'{self.job_service_endpoint}/Actions/DellJobService.CreateRebootJob'
+        self.setup_job_queue_endpoint: str = f'{self.job_service_endpoint}/Actions/DellJobService.SetupJobQueue'
 
     def build_common_data(self,
                           data: Dict[str, Any],
@@ -161,26 +163,3 @@ def _update_firmwares(self) -> None:
         self._sys['firmwares'] = self.build_common_data(data=self._system['UpdateService'],
                                                         fields=fields,
                                                         path='FirmwareInventory')
-
-    def get_chassis_led(self) -> Dict[str, Any]:
-        endpoint = f'/redfish/v1/{self.chassis_endpoint}'
-        result = self.client.query(method='GET',
-                                   endpoint=endpoint,
-                                   timeout=10)
-        response_json = json.loads(result[1])
-        _result: Dict[str, Any] = {'http_code': result[2]}
-        if result[2] == 200:
-            _result['LocationIndicatorActive'] = response_json['LocationIndicatorActive']
-        else:
-            _result['LocationIndicatorActive'] = None
-        return _result
-
-    def set_chassis_led(self, data: Dict[str, str]) -> int:
-        # '{"IndicatorLED": "Lit"}'      -> LocationIndicatorActive = false
-        # '{"IndicatorLED": "Blinking"}' -> LocationIndicatorActive = true
-        _, response, status = self.client.query(
-            data=json.dumps(data),
-            method='PATCH',
-            endpoint=f'/redfish/v1{self.chassis_endpoint}'
-        )
-        return status
diff --git a/src/cephadm/cephadmlib/node_proxy/reporter.py b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
similarity index 98%
rename from src/cephadm/cephadmlib/node_proxy/reporter.py
rename to src/ceph-node-proxy/ceph_node_proxy/reporter.py
index fb92a4523467..9e5521aa280b 100644
--- a/src/cephadm/cephadmlib/node_proxy/reporter.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
@@ -1,7 +1,7 @@
 from threading import Thread
 import time
 import json
-from .util import Logger, http_req
+from ceph_node_proxy.util import Logger, http_req
 from urllib.error import HTTPError, URLError
 from typing import Dict, Any
 
diff --git a/src/cephadm/cephadmlib/node_proxy/util.py b/src/ceph-node-proxy/ceph_node_proxy/util.py
similarity index 88%
rename from src/cephadm/cephadmlib/node_proxy/util.py
rename to src/ceph-node-proxy/ceph_node_proxy/util.py
index 31c1c00a0a15..a94acc9ed556 100644
--- a/src/cephadm/cephadmlib/node_proxy/util.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/util.py
@@ -4,6 +4,7 @@
 import time
 import re
 import ssl
+from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen, Request
 from typing import Dict, List, Callable, Any, Optional, MutableMapping, Tuple
@@ -124,9 +125,13 @@ def http_req(hostname: str = '',
 
     url: str = f'{scheme}://{hostname}:{port}{endpoint}'
     _data = bytes(data, 'ascii') if data else None
-
+    _headers = headers
+    if data and not method:
+        method = 'POST'
+    if not _headers.get('Content-Type') and method in ['POST', 'PATCH']:
+        _headers['Content-Type'] = 'application/json'
     try:
-        req = Request(url, _data, headers, method=method)
+        req = Request(url, _data, _headers, method=method)
         with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
             response_str = response.read()
             response_headers = response.headers
@@ -136,3 +141,11 @@ def http_req(hostname: str = '',
         print(f'{e}')
         # handle error here if needed
         raise
+
+
+def write_tmp_file(data: str, prefix_name: str = 'node-proxy-') -> _TemporaryFileWrapper:
+    f = NamedTemporaryFile(prefix=prefix_name)
+    os.fchmod(f.fileno(), 0o600)
+    f.write(data.encode('utf-8'))
+    f.flush()
+    return f
diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 1fc5768ca968..5a7ed9e76656 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -29,7 +29,6 @@
 from io import StringIO
 from threading import Thread, Event
 from pathlib import Path
-from cephadmlib.node_proxy.main import NodeProxy
 
 from cephadmlib.constants import (
     # default images
@@ -176,6 +175,7 @@
     NFSGanesha,
     SNMPGateway,
     Tracing,
+    NodeProxy,
 )
 from cephadmlib.agent import http_query
 
@@ -226,6 +226,7 @@ def get_supported_daemons():
     supported_daemons.append(CephadmAgent.daemon_type)
     supported_daemons.append(SNMPGateway.daemon_type)
     supported_daemons.extend(Tracing.components)
+    supported_daemons.append(NodeProxy.daemon_type)
     assert len(supported_daemons) == len(set(supported_daemons))
     return supported_daemons
 
@@ -800,6 +801,10 @@ def create_daemon_dirs(
         sg = SNMPGateway.init(ctx, fsid, ident.daemon_id)
         sg.create_daemon_conf()
 
+    elif daemon_type == NodeProxy.daemon_type:
+        node_proxy = NodeProxy.init(ctx, fsid, ident.daemon_id)
+        node_proxy.create_daemon_dirs(data_dir, uid, gid)
+
     _write_custom_conf_files(ctx, ident, uid, gid)
 
 
@@ -1287,28 +1292,13 @@ def run(self) -> None:
                             self.agent.ls_gatherer.wakeup()
                             self.agent.volume_gatherer.wakeup()
                             logger.debug(f'Got mgr message {data}')
-                        if 'node_proxy_oob_cmd' in data:
-                            if data['node_proxy_oob_cmd']['action'] in ['get_led', 'set_led']:
-                                conn.send(bytes(json.dumps(self.node_proxy_oob_cmd_result), 'utf-8'))
             except Exception as e:
                 logger.error(f'Mgr Listener encountered exception: {e}')
 
     def shutdown(self) -> None:
         self.stop = True
 
-    def validate_node_proxy_payload(self, data: Dict[str, Any]) -> None:
-        if 'action' not in data.keys():
-            raise RuntimeError('node-proxy oob command needs an action.')
-        if data['action'] in ['get_led', 'set_led']:
-            fields = ['type', 'id']
-            if data['type'] not in ['chassis', 'drive']:
-                raise RuntimeError('the LED type must be either "chassis" or "drive".')
-            for field in fields:
-                if field not in data.keys():
-                    raise RuntimeError('Received invalid node-proxy cmd.')
-
     def handle_json_payload(self, data: Dict[Any, Any]) -> None:
-        self.node_proxy_oob_cmd_result: Dict[str, Any] = {}
         if 'counter' in data:
             self.agent.ack = int(data['counter'])
             if 'config' in data:
@@ -1321,95 +1311,10 @@ def handle_json_payload(self, data: Dict[Any, Any]) -> None:
                             f.write(config[filename])
                 self.agent.pull_conf_settings()
                 self.agent.wakeup()
-        elif 'node_proxy_shutdown' in data:
-            logger.info('Received node_proxy_shutdown command.')
-            self.agent.shutdown()
-        elif 'node_proxy_oob_cmd' in data:
-            node_proxy_cmd: Dict[str, Any] = data['node_proxy_oob_cmd']
-            try:
-                self.validate_node_proxy_payload(node_proxy_cmd)
-            except RuntimeError as e:
-                logger.error(f"Couldn't validate node-proxy payload:\n{node_proxy_cmd}\n{e}")
-                raise
-            logger.info(f'Received node_proxy_oob_cmd command: {node_proxy_cmd}')
-            if node_proxy_cmd['action'] == 'get_led':
-                if node_proxy_cmd['type'] == 'chassis':
-                    self.node_proxy_oob_cmd_result = self.agent.node_proxy_mgr.node_proxy.system.get_chassis_led()
-            if node_proxy_cmd['action'] == 'set_led':
-                if node_proxy_cmd['type'] == 'chassis':
-                    _data: Dict[str, Any] = json.loads(node_proxy_cmd['data'])
-                    _result: int = self.agent.node_proxy_mgr.node_proxy.system.set_chassis_led(_data)
-                    self.node_proxy_oob_cmd_result = {'http_code': _result}
         else:
             raise RuntimeError('No valid data received.')
 
 
-class NodeProxyManager(Thread):
-    def __init__(self, agent: 'CephadmAgent', event: Event):
-        super().__init__()
-        self.agent = agent
-        self.event = event
-        self.stop = False
-
-    def run(self) -> None:
-        self.event.wait()
-        self.ssl_ctx = self.agent.ssl_ctx
-        self.init()
-        self.loop()
-
-    def init(self) -> None:
-        node_proxy_meta = {
-            'cephx': {
-                'name': self.agent.host,
-                'secret': self.agent.keyring
-            }
-        }
-        status, result = http_query(addr=self.agent.target_ip,
-                                    port=self.agent.target_port,
-                                    data=json.dumps(node_proxy_meta).encode('ascii'),
-                                    endpoint='/node-proxy/oob',
-                                    ssl_ctx=self.ssl_ctx)
-        if status != 200:
-            msg = f'No out of band tool details could be loaded: {status}, {result}'
-            logger.debug(msg)
-            raise RuntimeError(msg)
-
-        result_json = json.loads(result)
-        kwargs = {
-            'host': result_json['result']['addr'],
-            'username': result_json['result']['username'],
-            'password': result_json['result']['password'],
-            'cephx': node_proxy_meta['cephx'],
-            'mgr_target_ip': self.agent.target_ip,
-            'mgr_target_port': self.agent.target_port
-        }
-        if result_json['result'].get('port'):
-            kwargs['port'] = result_json['result']['port']
-
-        self.node_proxy: NodeProxy = NodeProxy(**kwargs)
-        self.node_proxy.start()
-
-    def loop(self) -> None:
-        while not self.stop:
-            try:
-                status = self.node_proxy.check_status()
-                label = 'Ok' if status else 'Critical'
-                logger.debug(f'node-proxy status: {label}')
-            except Exception as e:
-                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
-                time.sleep(120)
-                self.init()
-            else:
-                logger.debug('node-proxy alive, next check in 60sec.')
-                time.sleep(60)
-
-    def shutdown(self) -> None:
-        self.stop = True
-        # if `self.node_proxy.shutdown()` is called before self.start(), it will fail.
-        if self.__dict__.get('node_proxy'):
-            self.node_proxy.shutdown()
-
-
 @register_daemon_form
 class CephadmAgent(DaemonForm):
 
@@ -1465,8 +1370,6 @@ def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str] =
         self.ssl_ctx = ssl.create_default_context()
         self.ssl_ctx.check_hostname = True
         self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
-        self.node_proxy_mgr_event = Event()
-        self.node_proxy_mgr = NodeProxyManager(self, self.node_proxy_mgr_event)
 
     def validate(self, config: Dict[str, str] = {}) -> None:
         # check for the required files
@@ -1522,8 +1425,6 @@ def unit_file(self) -> str:
 
     def shutdown(self) -> None:
         self.stop = True
-        if self.node_proxy_mgr.is_alive():
-            self.node_proxy_mgr.shutdown()
         if self.mgr_listener.is_alive():
             self.mgr_listener.shutdown()
         if self.ls_gatherer.is_alive():
@@ -1565,9 +1466,6 @@ def pull_conf_settings(self) -> None:
     def run(self) -> None:
         self.pull_conf_settings()
         self.ssl_ctx.load_verify_locations(self.ca_path)
-        # only after self.pull_conf_settings() was called we can actually start
-        # node-proxy
-        self.node_proxy_mgr_event.set()
 
         try:
             for _ in range(1001):
@@ -1589,9 +1487,6 @@ def run(self) -> None:
         if not self.volume_gatherer.is_alive():
             self.volume_gatherer.start()
 
-        if not self.node_proxy_mgr.is_alive():
-            self.node_proxy_mgr.start()
-
         while not self.stop:
             start_time = time.monotonic()
             ack = self.ack
diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index cf572d487c9b..29f150694832 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -7,6 +7,7 @@
 from .nvmeof import CephNvmeof
 from .snmp import SNMPGateway
 from .tracing import Tracing
+from .node_proxy import NodeProxy
 
 __all__ = [
     'Ceph',
@@ -21,4 +22,5 @@
     'OSD',
     'SNMPGateway',
     'Tracing',
+    'NodeProxy',
 ]
diff --git a/src/cephadm/cephadmlib/daemons/node_proxy.py b/src/cephadm/cephadmlib/daemons/node_proxy.py
new file mode 100644
index 000000000000..a4cce11a53ca
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/node_proxy.py
@@ -0,0 +1,145 @@
+import logging
+import os
+
+from typing import Dict, List, Optional, Tuple
+
+from ..constants import DEFAULT_IMAGE
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_types import CephContainer, extract_uid_gid
+from ..context import CephadmContext
+from ..context_getters import fetch_configs, get_config_and_keyring
+from ..daemon_form import register as register_daemon_form
+from ..daemon_identity import DaemonIdentity
+from ..data_utils import dict_get, is_fsid
+from ..deployment_utils import to_deployment_container
+from ..exceptions import Error
+from ..file_utils import populate_files
+
+logger = logging.getLogger()
+
+
+@register_daemon_form
+class NodeProxy(ContainerDaemonForm):
+    """Defines a node-proxy container"""
+
+    daemon_type = 'node-proxy'
+    # TODO: update this if we make node-proxy an executable
+    entrypoint = 'python3'
+    required_files = ['node-proxy.json']
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(
+        self,
+        ctx: CephadmContext,
+        ident: DaemonIdentity,
+        config_json: Dict,
+        image: str = DEFAULT_IMAGE,
+    ):
+        self.ctx = ctx
+        self._identity = ident
+        self.image = image
+
+        # config-json options
+        config = dict_get(config_json, 'node-proxy.json', {})
+        self.files = {'node-proxy.json': config}
+
+        # validate the supplied args
+        self.validate()
+
+    @classmethod
+    def init(
+        cls, ctx: CephadmContext, fsid: str, daemon_id: str
+    ) -> 'NodeProxy':
+        return cls.create(
+            ctx, DaemonIdentity(fsid, cls.daemon_type, daemon_id)
+        )
+
+    @classmethod
+    def create(
+        cls, ctx: CephadmContext, ident: DaemonIdentity
+    ) -> 'NodeProxy':
+        return cls(ctx, ident, fetch_configs(ctx), ctx.image)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    @property
+    def fsid(self) -> str:
+        return self._identity.fsid
+
+    @property
+    def daemon_id(self) -> str:
+        return self._identity.daemon_id
+
+    def customize_container_mounts(
+        self, ctx: CephadmContext, mounts: Dict[str, str]
+    ) -> None:
+        data_dir = self.identity.data_dir(ctx.data_dir)
+        # TODO: update this when we have the actual location
+        # in the ceph container we are going to keep node-proxy
+        mounts.update({os.path.join(data_dir, 'node-proxy.json'): '/usr/share/ceph/node-proxy.json:z'})
+
+    def customize_process_args(self, ctx: CephadmContext, args: List[str]) -> None:
+        # TODO: this corresponds with the mount location of
+        # the config in _get_container_mounts above. They
+        # will both need to be updated when we have a proper
+        # location in the container for node-proxy
+        args.extend(['/usr/share/ceph/ceph_node_proxy/main.py', '--config', '/usr/share/ceph/node-proxy.json'])
+
+    def validate(self):
+        # type: () -> None
+        if not is_fsid(self.fsid):
+            raise Error('not an fsid: %s' % self.fsid)
+        if not self.daemon_id:
+            raise Error('invalid daemon_id: %s' % self.daemon_id)
+        if not self.image:
+            raise Error('invalid image: %s' % self.image)
+        # check for the required files
+        if self.required_files:
+            for fname in self.required_files:
+                if fname not in self.files:
+                    raise Error(
+                        'required file missing from config-json: %s' % fname
+                    )
+
+    def get_daemon_name(self):
+        # type: () -> str
+        return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+    def get_container_name(self, desc=None):
+        # type: (Optional[str]) -> str
+        cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+        if desc:
+            cname = '%s-%s' % (cname, desc)
+        return cname
+
+    def create_daemon_dirs(self, data_dir, uid, gid):
+        # type: (str, int, int) -> None
+        """Create files under the container data dir"""
+        if not os.path.isdir(data_dir):
+            raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+        logger.info('Writing node-proxy config...')
+        # populate files from the config-json
+        populate_files(data_dir, self.files, uid, gid)
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        # So the container can modprobe iscsi_target_mod and have write perms
+        # to configfs we need to make this a privileged container.
+        ctr = daemon_to_container(ctx, self, privileged=True, envs=['PYTHONPATH=$PYTHONPATH:/usr/share/ceph'])
+        return to_deployment_container(ctx, ctr)
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return get_config_and_keyring(ctx)
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return extract_uid_gid(ctx)
+
+    def default_entrypoint(self) -> str:
+        return self.entrypoint
diff --git a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py b/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
deleted file mode 100644
index 45c80e0209d4..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/baseredfishsystem.py
+++ /dev/null
@@ -1,152 +0,0 @@
-import concurrent.futures
-from .basesystem import BaseSystem
-from .redfish_client import RedFishClient
-from threading import Thread, Lock
-from time import sleep
-from .util import Logger, retry
-from typing import Dict, Any, List
-
-
-class BaseRedfishSystem(BaseSystem):
-    def __init__(self, **kw: Any) -> None:
-        super().__init__(**kw)
-        self.common_endpoints: List[str] = kw.get('common_endpoints', ['/Systems/System.Embedded.1',
-                                                                       '/UpdateService'])
-        self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
-        self.log = Logger(__name__)
-        self.host: str = kw['host']
-        self.port: str = kw['port']
-        self.username: str = kw['username']
-        self.password: str = kw['password']
-        # move the following line (class attribute?)
-        self.client: RedFishClient = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
-        self.log.logger.info(f'redfish system initialization, host: {self.host}, user: {self.username}')
-
-        self.run: bool = False
-        self.thread: Thread
-        self.data_ready: bool = False
-        self.previous_data: Dict = {}
-        self.lock: Lock = Lock()
-        self.data: Dict[str, Dict[str, Any]] = {}
-        self._system: Dict[str, Dict[str, Any]] = {}
-        self._sys: Dict[str, Any] = {}
-        self.start_client()
-
-    def start_client(self) -> None:
-        self.client.login()
-        self.start_update_loop()
-
-    def start_update_loop(self) -> None:
-        self.run = True
-        self.thread = Thread(target=self.update)
-        self.thread.start()
-
-    def stop_update_loop(self) -> None:
-        self.run = False
-        self.thread.join()
-
-    def update(self) -> None:
-        #  this loop can have:
-        #  - caching logic
-        while self.run:
-            self.log.logger.debug('waiting for a lock in the update loop.')
-            self.lock.acquire()
-            self.log.logger.debug('lock acquired in the update loop.')
-            try:
-                self._update_system()
-                self._update_sn()
-                update_funcs = [self._update_memory,
-                                self._update_power,
-                                self._update_fans,
-                                self._update_network,
-                                self._update_processors,
-                                self._update_storage,
-                                self._update_firmwares]
-
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                    executor.map(lambda f: f(), update_funcs)
-
-                self.data_ready = True
-                sleep(5)
-            except RuntimeError as e:
-                self.run = False
-                self.log.logger.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
-                self.client.logout()
-            finally:
-                self.lock.release()
-                self.log.logger.debug('lock released in the update loop.')
-
-    def flush(self) -> None:
-        self.log.logger.debug('Acquiring lock to flush data.')
-        self.lock.acquire()
-        self.log.logger.debug('Lock acquired, flushing data.')
-        self._system = {}
-        self.previous_data = {}
-        self.log.logger.info('Data flushed.')
-        self.data_ready = False
-        self.log.logger.debug('Data marked as not ready.')
-        self.lock.release()
-        self.log.logger.debug('Released the lock after flushing data.')
-
-    @retry(retries=10, delay=2)
-    def _get_path(self, path: str) -> Dict:
-        try:
-            result = self.client.get_path(path)
-        except RuntimeError:
-            raise
-        if result is None:
-            self.log.logger.error(f'The client reported an error when getting path: {path}')
-            raise RuntimeError(f'Could not get path: {path}')
-        return result
-
-    def get_members(self, data: Dict[str, Any], path: str) -> List:
-        _path = data[path]['@odata.id']
-        _data = self._get_path(_path)
-        return [self._get_path(member['@odata.id']) for member in _data['Members']]
-
-    def get_system(self) -> Dict[str, Any]:
-        result = {
-            'host': self.get_host(),
-            'sn': self.get_sn(),
-            'status': {
-                'storage': self.get_storage(),
-                'processors': self.get_processors(),
-                'network': self.get_network(),
-                'memory': self.get_memory(),
-                'power': self.get_power(),
-                'fans': self.get_fans()
-            },
-            'firmwares': self.get_firmwares(),
-            'chassis': {'redfish_endpoint': f'/redfish/v1{self.chassis_endpoint}'}  # TODO(guits): not ideal
-        }
-        return result
-
-    def _update_system(self) -> None:
-        for endpoint in self.common_endpoints:
-            result = self.client.get_path(endpoint)
-            _endpoint = endpoint.strip('/').split('/')[0]
-            self._system[_endpoint] = result
-
-    def _update_sn(self) -> None:
-        raise NotImplementedError()
-
-    def _update_memory(self) -> None:
-        raise NotImplementedError()
-
-    def _update_power(self) -> None:
-        raise NotImplementedError()
-
-    def _update_fans(self) -> None:
-        raise NotImplementedError()
-
-    def _update_network(self) -> None:
-        raise NotImplementedError()
-
-    def _update_processors(self) -> None:
-        raise NotImplementedError()
-
-    def _update_storage(self) -> None:
-        raise NotImplementedError()
-
-    def _update_firmwares(self) -> None:
-        raise NotImplementedError()
diff --git a/src/cephadm/cephadmlib/node_proxy/main.py b/src/cephadm/cephadmlib/node_proxy/main.py
deleted file mode 100644
index 968dfd3c1ca6..000000000000
--- a/src/cephadm/cephadmlib/node_proxy/main.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from threading import Thread
-from .redfishdellsystem import RedfishDellSystem
-from .reporter import Reporter
-from .util import Config, Logger
-from typing import Dict, Any, Optional
-import traceback
-
-DEFAULT_CONFIG = {
-    'reporter': {
-        'check_interval': 5,
-        'push_data_max_retries': 30,
-        'endpoint': 'https://127.0.0.1:7150/node-proxy/data',
-    },
-    'system': {
-        'refresh_interval': 5
-    },
-    'server': {
-        'port': 8080,
-    },
-    'logging': {
-        'level': 20,
-    }
-}
-
-
-class NodeProxy(Thread):
-    def __init__(self, **kw: Any) -> None:
-        super().__init__()
-        self.username: str = kw.get('username', '')
-        self.password: str = kw.get('password', '')
-        self.host: str = kw.get('host', '')
-        self.port: int = kw.get('port', 443)
-        self.cephx: Dict[str, Any] = kw.get('cephx', {})
-        self.reporter_scheme: str = kw.get('reporter_scheme', 'https')
-        self.mgr_target_ip: str = kw.get('mgr_target_ip', '')
-        self.mgr_target_port: str = kw.get('mgr_target_port', '')
-        self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
-        self.exc: Optional[Exception] = None
-        self.log = Logger(__name__)
-
-    def run(self) -> None:
-        try:
-            self.main()
-        except Exception as e:
-            self.exc = e
-            return
-
-    def shutdown(self) -> None:
-        self.log.logger.info('Shutting down node-proxy...')
-        self.system.client.logout()
-        self.system.stop_update_loop()
-        self.reporter_agent.stop()
-
-    def check_auth(self, realm: str, username: str, password: str) -> bool:
-        return self.username == username and \
-            self.password == password
-
-    def check_status(self) -> bool:
-        if self.__dict__.get('system') and not self.system.run:
-            raise RuntimeError('node-proxy encountered an error.')
-        if self.exc:
-            traceback.print_tb(self.exc.__traceback__)
-            self.log.logger.error(f'{self.exc.__class__.__name__}: {self.exc}')
-            raise self.exc
-        return True
-
-    def main(self) -> None:
-        # TODO: add a check and fail if host/username/password/data aren't passed
-        self.config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-        self.log = Logger(__name__, level=self.config.__dict__['logging']['level'])
-
-        # create the redfish system and the obsever
-        self.log.logger.info('Server initialization...')
-        try:
-            self.system = RedfishDellSystem(host=self.host,
-                                            port=self.port,
-                                            username=self.username,
-                                            password=self.password,
-                                            config=self.config)
-        except RuntimeError:
-            self.log.logger.error("Can't initialize the redfish system.")
-            raise
-
-        try:
-            self.reporter_agent = Reporter(self.system,
-                                           self.cephx,
-                                           reporter_scheme=self.reporter_scheme,
-                                           reporter_hostname=self.mgr_target_ip,
-                                           reporter_port=self.mgr_target_port,
-                                           reporter_endpoint=self.reporter_endpoint)
-            self.reporter_agent.run()
-        except RuntimeError:
-            self.log.logger.error("Can't initialize the reporter.")
-            raise
diff --git a/src/mypy.ini b/src/mypy.ini
index bd4b436b4926..1b158822b526 100755
--- a/src/mypy.ini
+++ b/src/mypy.ini
@@ -45,6 +45,8 @@ ignore_missing_imports = True
 [mypy-kubernetes.*]
 ignore_missing_imports = True
 
+[mypy-setuptools]
+ignore_missing_imports = True
 
 # Make dashboard happy:
 [mypy-coverage]
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index b589886e566c..68495d3bc061 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -16,14 +16,15 @@ class Server:  # type: ignore
 
 from orchestrator import DaemonDescriptionStatus
 from orchestrator._interface import daemon_type_to_service
-from ceph.utils import datetime_now
+from ceph.utils import datetime_now, http_req
 from ceph.deployment.inventory import Devices
 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
 from cephadm.services.cephadmservice import CephadmDaemonDeploySpec
 from cephadm.ssl_cert_utils import SSLCerts
 from mgr_util import test_port_allocation, PortAlreadyInUse
 
-from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional
+from urllib.error import HTTPError, URLError
+from typing import Any, Dict, List, Set, TYPE_CHECKING, Optional, MutableMapping
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -138,8 +139,9 @@ def oob(self) -> Dict[str, Any]:
 
         self.validate_node_proxy_data(data)
 
-        host = data["cephx"]["name"]
-        results['result'] = self.mgr.node_proxy_cache.oob.get(host)
+        # expecting name to be "node-proxy.<hostname>"
+        hostname = data['cephx']['name'][11:]
+        results['result'] = self.mgr.node_proxy_cache.oob.get(hostname, '')
         if not results['result']:
             raise cherrypy.HTTPError(400, 'The provided host has no iDrac details.')
         return results
@@ -160,13 +162,15 @@ def validate_node_proxy_data(self, data: Dict[str, Any]) -> None:
                 raise cherrypy.HTTPError(400, 'The field \'cephx\' must be provided.')
             elif 'name' not in data['cephx'].keys():
                 cherrypy.response.status = 400
-                raise cherrypy.HTTPError(400, 'The field \'host\' must be provided.')
-            elif 'secret' not in data['cephx'].keys():
-                raise cherrypy.HTTPError(400, 'The agent keyring must be provided.')
-            elif not self.mgr.agent_cache.agent_keys.get(data['cephx']['name']):
-                raise cherrypy.HTTPError(502, f'Make sure the agent is running on {data["cephx"]["name"]}')
-            elif data['cephx']['secret'] != self.mgr.agent_cache.agent_keys[data['cephx']['name']]:
-                raise cherrypy.HTTPError(403, f'Got wrong keyring from agent on host {data["cephx"]["name"]}.')
+                raise cherrypy.HTTPError(400, 'The field \'name\' must be provided.')
+            # expecting name to be "node-proxy.<hostname>"
+            hostname = data['cephx']['name'][11:]
+            if 'secret' not in data['cephx'].keys():
+                raise cherrypy.HTTPError(400, 'The node-proxy keyring must be provided.')
+            elif not self.mgr.node_proxy_cache.keyrings.get(hostname, ''):
+                raise cherrypy.HTTPError(502, f'Make sure the node-proxy is running on {hostname}')
+            elif data['cephx']['secret'] != self.mgr.node_proxy_cache.keyrings[hostname]:
+                raise cherrypy.HTTPError(403, f'Got wrong keyring from agent on host {hostname}.')
         except AttributeError:
             raise cherrypy.HTTPError(400, 'Malformed data received.')
 
@@ -289,12 +293,19 @@ def led(self, **kw: Any) -> Dict[str, Any]:
         :rtype: dict[str, Any]
         """
         method: str = cherrypy.request.method
+        header: MutableMapping[str, str] = {}
         hostname: Optional[str] = kw.get('hostname')
         led_type: Optional[str] = kw.get('type')
         id_drive: Optional[str] = kw.get('id')
-        data: Optional[str] = None
-        # this method is restricted to 'GET' or 'PATCH'
-        action: str = 'get_led' if method == 'GET' else 'set_led'
+        payload: Optional[Dict[str, str]] = None
+        endpoint: List[Any] = ['led', led_type]
+        device: str = id_drive if id_drive else ''
+
+        ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = True
+        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx.load_verify_locations(cadata=ssl_root_crt)
 
         if not hostname:
             msg: str = "listing enclosure LED status for all nodes is not implemented."
@@ -311,16 +322,32 @@ def led(self, **kw: Any) -> Dict[str, Any]:
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
 
+        if led_type == 'drive':
+            endpoint.append(device)
+
         if hostname not in self.mgr.node_proxy_cache.data.keys():
             # TODO(guits): update unit test for this
             msg = f"'{hostname}' not found."
             self.mgr.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
 
+        addr: str = self.mgr.inventory.get_addr(hostname)
+
         if method == 'PATCH':
             # TODO(guits): need to check the request is authorized
             # allowing a specific keyring only ? (client.admin or client.agent.. ?)
-            data = json.dumps(cherrypy.request.json)
+            data: Dict[str, Any] = cherrypy.request.json
+            if 'state' not in data.keys():
+                msg = "'state' key not provided."
+                raise cherrypy.HTTPError(400, msg)
+            if 'keyring' not in data.keys():
+                msg = "'keyring' key must be provided."
+                raise cherrypy.HTTPError(400, msg)
+            if data['keyring'] != self.mgr.node_proxy_cache.keyrings.get(hostname):
+                msg = 'wrong keyring provided.'
+                raise cherrypy.HTTPError(401, msg)
+            payload = {}
+            payload['state'] = data['state']
 
             if led_type == 'drive':
                 if id_drive not in self.mgr.node_proxy_cache.data[hostname]['status']['storage'].keys():
@@ -329,28 +356,23 @@ def led(self, **kw: Any) -> Dict[str, Any]:
                     self.mgr.log.debug(msg)
                     raise cherrypy.HTTPError(400, msg)
 
-        payload: Dict[str, Any] = {"node_proxy_oob_cmd":
-                                   {"action": action,
-                                    "type": led_type,
-                                    "id": id_drive,
-                                    "host": hostname,
-                                    "data": data}}
-        try:
-            message_thread = AgentMessageThread(
-                hostname, self.mgr.agent_cache.agent_ports[hostname], payload, self.mgr)
-            message_thread.start()
-            message_thread.join()  # TODO(guits): Add a timeout?
-        except KeyError:
-            raise cherrypy.HTTPError(502, f"{hostname}'s agent not running, please check.")
-        agent_response = message_thread.get_agent_response()
+        endpoint = f'/{"/".join(endpoint)}'
+        header = self.mgr.node_proxy.generate_auth_header(hostname)
+
         try:
-            response_json: Dict[str, Any] = json.loads(agent_response)
-        except json.decoder.JSONDecodeError:
-            cherrypy.response.status = 503
-        else:
-            cherrypy.response.status = response_json.get('http_code', 503)
-        if cherrypy.response.status != 200:
-            raise cherrypy.HTTPError(cherrypy.response.status, "Couldn't change the LED status.")
+            headers, result, status = http_req(hostname=addr,
+                                               port='8080',
+                                               headers=header,
+                                               method=method,
+                                               data=json.dumps(payload),
+                                               endpoint=endpoint,
+                                               ssl_ctx=ssl_ctx)
+            response_json = json.loads(result)
+        except HTTPError as e:
+            self.mgr.log.debug(e)
+        except URLError:
+            raise cherrypy.HTTPError(502, f'Make sure the node-proxy agent is deployed and running on {hostname}')
+
         return response_json
 
     @cherrypy.expose
@@ -842,16 +864,6 @@ def _request_agent_acks(self, hosts: Set[str], increment: bool = False, daemon_s
                 host, self.mgr.agent_cache.agent_ports[host], payload, self.mgr, daemon_spec)
             message_thread.start()
 
-    def _shutdown_node_proxy(self) -> None:
-        hosts = set([h for h in self.mgr.cache.get_hosts() if
-                     (h in self.mgr.agent_cache.agent_ports and not self.mgr.agent_cache.messaging_agent(h))])
-
-        for host in hosts:
-            payload: Dict[str, Any] = {'node_proxy_shutdown': host}
-            message_thread = AgentMessageThread(
-                host, self.mgr.agent_cache.agent_ports[host], payload, self.mgr)
-            message_thread.start()
-
     def _request_ack_all_not_up_to_date(self) -> None:
         self.mgr.agent_helpers._request_agent_acks(
             set([h for h in self.mgr.cache.get_hosts() if
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 27bf55c921c3..235737ef10e7 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -29,7 +29,7 @@
 HOST_CACHE_PREFIX = "host."
 SPEC_STORE_PREFIX = "spec."
 AGENT_CACHE_PREFIX = 'agent.'
-NODE_PROXY_CACHE_PREFIX = 'node_proxy/data'
+NODE_PROXY_CACHE_PREFIX = 'node_proxy'
 
 
 class HostCacheStatus(enum.Enum):
@@ -1411,20 +1411,25 @@ def __init__(self, mgr: "CephadmOrchestrator") -> None:
         self.mgr = mgr
         self.data: Dict[str, Any] = {}
         self.oob: Dict[str, Any] = {}
+        self.keyrings: Dict[str, str] = {}
 
     def load(self) -> None:
-        _oob = self.mgr.get_store('node_proxy/oob', "{}")
+        _oob = self.mgr.get_store(f'{NODE_PROXY_CACHE_PREFIX}/oob', '{}')
         self.oob = json.loads(_oob)
 
-        for k, v in self.mgr.get_store_prefix(NODE_PROXY_CACHE_PREFIX).items():
+        _keyrings = self.mgr.get_store(f'{NODE_PROXY_CACHE_PREFIX}/keyrings', '{}')
+        self.keyrings = json.loads(_keyrings)
+
+        for k, v in self.mgr.get_store_prefix(f'{NODE_PROXY_CACHE_PREFIX}/data').items():
             host = k.split('/')[-1:][0]
 
             if host not in self.mgr.inventory.keys():
                 # remove entry for host that no longer exists
-                self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", None)
+                self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/data/{host}", None)
                 try:
                     self.oob.pop(host)
                     self.data.pop(host)
+                    self.keyrings.pop(host)
                 except KeyError:
                     pass
                 continue
@@ -1434,7 +1439,15 @@ def load(self) -> None:
     def save(self,
              host: str = '',
              data: Dict[str, Any] = {}) -> None:
-        self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/{host}", json.dumps(data))
+        self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/data/{host}", json.dumps(data))
+
+    def update_oob(self, host: str, host_oob_info: Dict[str, str]) -> None:
+        self.oob[host] = host_oob_info
+        self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/oob", json.dumps(self.oob))
+
+    def update_keyring(self, host: str, key: str) -> None:
+        self.keyrings[host] = key
+        self.mgr.set_store(f"{NODE_PROXY_CACHE_PREFIX}/keyrings", json.dumps(self.keyrings))
 
     def fullreport(self, **kw: Any) -> Dict[str, Any]:
         """
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index b82f861ce6ad..5c5e602e944f 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -11,6 +11,7 @@
 from contextlib import contextmanager
 from functools import wraps
 from tempfile import TemporaryDirectory, NamedTemporaryFile
+from urllib.error import HTTPError
 from threading import Event
 
 from cephadm.service_discovery import ServiceDiscovery
@@ -72,6 +73,7 @@
 from .services.monitoring import GrafanaService, AlertmanagerService, PrometheusService, \
     NodeExporterService, SNMPGatewayService, LokiService, PromtailService
 from .services.jaeger import ElasticSearchService, JaegerAgentService, JaegerCollectorService, JaegerQueryService
+from .services.node_proxy import NodeProxy
 from .schedule import HostAssignment
 from .inventory import Inventory, SpecStore, HostCache, AgentCache, EventStore, \
     ClientKeyringStore, ClientKeyringSpec, TunedProfileStore, NodeProxyCache
@@ -443,6 +445,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             default=3.0,
             desc='Multiplied by agent refresh rate to calculate how long agent must not report before being marked down'
         ),
+        Option(
+            'hw_monitoring',
+            type='bool',
+            default=False,
+            desc='Deploy hw monitoring daemon on every host.'
+        ),
         Option(
             'max_osd_draining_count',
             type='int',
@@ -560,6 +568,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.agent_refresh_rate = 0
             self.agent_down_multiplier = 0.0
             self.agent_starting_port = 0
+            self.hw_monitoring = False
             self.service_discovery_port = 0
             self.secure_monitoring_stack = False
             self.apply_spec_fails: List[Tuple[str, str]] = []
@@ -640,7 +649,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             PrometheusService, NodeExporterService, LokiService, PromtailService, CrashService, IscsiService,
             IngressService, CustomContainerService, CephfsMirrorService, NvmeofService,
             CephadmAgent, CephExporterService, SNMPGatewayService, ElasticSearchService,
-            JaegerQueryService, JaegerAgentService, JaegerCollectorService
+            JaegerQueryService, JaegerAgentService, JaegerCollectorService, NodeProxy
         ]
 
         # https://github.com/python/mypy/issues/8993
@@ -651,6 +660,7 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.osd_service: OSDService = cast(OSDService, self.cephadm_services['osd'])
         self.iscsi_service: IscsiService = cast(IscsiService, self.cephadm_services['iscsi'])
         self.nvmeof_service: NvmeofService = cast(NvmeofService, self.cephadm_services['nvmeof'])
+        self.node_proxy_service: NodeProxy = cast(NodeProxy, self.cephadm_services['node-proxy'])
 
         self.scheduled_async_actions: List[Callable] = []
 
@@ -663,6 +673,9 @@ def __init__(self, *args: Any, **kwargs: Any):
 
         self.http_server = CephadmHttpServer(self)
         self.http_server.start()
+
+        self.node_proxy = NodeProxy(self)
+
         self.agent_helpers = CephadmAgentHelpers(self)
         if self.use_agent:
             self.agent_helpers._apply_agent()
@@ -829,7 +842,7 @@ def get_unique_name(
         Generate a unique random service name
         """
         suffix = daemon_type not in [
-            'mon', 'crash', 'ceph-exporter',
+            'mon', 'crash', 'ceph-exporter', 'node-proxy',
             'prometheus', 'node-exporter', 'grafana', 'alertmanager',
             'container', 'agent', 'snmp-gateway', 'loki', 'promtail',
             'elasticsearch', 'jaeger-collector', 'jaeger-agent', 'jaeger-query'
@@ -1621,13 +1634,12 @@ def _add_host(self, spec):
                 spec.oob['addr'] = spec.hostname
             if not spec.oob.get('port'):
                 spec.oob['port'] = '443'
-            data = json.loads(self.get_store('node_proxy/oob', '{}'))
-            data[spec.hostname] = dict()
-            data[spec.hostname]['addr'] = spec.oob['addr']
-            data[spec.hostname]['port'] = spec.oob['port']
-            data[spec.hostname]['username'] = spec.oob['username']
-            data[spec.hostname]['password'] = spec.oob['password']
-            self.set_store('node_proxy/oob', json.dumps(data))
+            host_oob_info = dict()
+            host_oob_info['addr'] = spec.oob['addr']
+            host_oob_info['port'] = spec.oob['port']
+            host_oob_info['username'] = spec.oob['username']
+            host_oob_info['password'] = spec.oob['password']
+            self.node_proxy_cache.update_oob(spec.hostname, host_oob_info)
 
         # prime crush map?
         if spec.location:
@@ -1652,6 +1664,51 @@ def _add_host(self, spec):
     def add_host(self, spec: HostSpec) -> str:
         return self._add_host(spec)
 
+    @handle_orch_error
+    def hardware_light(self, light_type: str, action: str, hostname: str, device: Optional[str] = None) -> Dict[str, Any]:
+        try:
+            result = self.node_proxy.led(light_type=light_type,
+                                         action=action,
+                                         hostname=hostname,
+                                         device=device)
+        except RuntimeError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f'Make sure the node-proxy agent is deployed and running on {hostname}')
+        except HTTPError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f"http error while querying node-proxy API: {e}")
+        return result
+
+    @handle_orch_error
+    def hardware_shutdown(self, hostname: str, force: Optional[bool] = False, yes_i_really_mean_it: bool = False) -> str:
+        if not yes_i_really_mean_it:
+            raise OrchestratorError("you must pass --yes-i-really-mean-it")
+
+        try:
+            self.node_proxy.shutdown(hostname, force)
+        except RuntimeError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f'Make sure the node-proxy agent is deployed and running on {hostname}')
+        except HTTPError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f"Can't shutdown node {hostname}: {e}")
+        return f'Shutdown scheduled on {hostname}'
+
+    @handle_orch_error
+    def hardware_powercycle(self, hostname: str, yes_i_really_mean_it: bool = False) -> str:
+        if not yes_i_really_mean_it:
+            raise OrchestratorError("you must pass --yes-i-really-mean-it")
+
+        try:
+            self.node_proxy.powercycle(hostname)
+        except RuntimeError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f'Make sure the node-proxy agent is deployed and running on {hostname}')
+        except HTTPError as e:
+            self.log.error(e)
+            raise OrchestratorValidationError(f"Can't perform powercycle on node {hostname}: {e}")
+        return f'Powercycle scheduled on {hostname}'
+
     @handle_orch_error
     def node_proxy_summary(self, hostname: Optional[str] = None) -> Dict[str, Any]:
         return self.node_proxy_cache.summary(hostname=hostname)
@@ -2724,6 +2781,15 @@ def get_daemon_names(daemons: List[str]) -> List[str]:
                 pass
             deps = sorted([self.get_mgr_ip(), server_port, root_cert,
                            str(self.device_enhanced_scan)])
+        elif daemon_type == 'node-proxy':
+            root_cert = ''
+            server_port = ''
+            try:
+                server_port = str(self.http_server.agent.server_port)
+                root_cert = self.http_server.agent.ssl_certs.get_root_cert()
+            except Exception:
+                pass
+            deps = sorted([self.get_mgr_ip(), server_port, root_cert])
         elif daemon_type == 'iscsi':
             if spec:
                 iscsi_spec = cast(IscsiServiceSpec, spec)
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 262ecbd27f6a..600329509a0e 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -113,6 +113,9 @@ def serve(self) -> None:
                     if self.mgr.agent_helpers._handle_use_agent_setting():
                         continue
 
+                    if self.mgr.node_proxy_service.handle_hw_monitoring_setting():
+                        continue
+
                     if self.mgr.upgrade.continue_upgrade():
                         continue
 
diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py
index 1681be003487..115ee8d01024 100644
--- a/src/pybind/mgr/cephadm/services/cephadmservice.py
+++ b/src/pybind/mgr/cephadm/services/cephadmservice.py
@@ -42,7 +42,7 @@ def get_auth_entity(daemon_type: str, daemon_id: str, host: str = "") -> AuthEnt
     # the CephService class refers to service types, not daemon types
     if daemon_type in ['rgw', 'rbd-mirror', 'cephfs-mirror', 'nfs', "iscsi", 'nvmeof', 'ingress', 'ceph-exporter']:
         return AuthEntity(f'client.{daemon_type}.{daemon_id}')
-    elif daemon_type in ['crash', 'agent']:
+    elif daemon_type in ['crash', 'agent', 'node-proxy']:
         if host == "":
             raise OrchestratorError(
                 f'Host not provided to generate <{daemon_type}> auth entity name')
@@ -1236,16 +1236,6 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
 
         return daemon_spec
 
-    def pre_remove(self, daemon: DaemonDescription) -> None:
-        super().pre_remove(daemon)
-
-        assert daemon.daemon_id is not None
-        daemon_id: str = daemon.daemon_id
-
-        logger.info('Removing agent %s...' % daemon_id)
-
-        self.mgr.agent_helpers._shutdown_node_proxy()
-
     def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
         agent = self.mgr.http_server.agent
         try:
diff --git a/src/pybind/mgr/cephadm/services/node_proxy.py b/src/pybind/mgr/cephadm/services/node_proxy.py
new file mode 100644
index 000000000000..ebbbaf212c74
--- /dev/null
+++ b/src/pybind/mgr/cephadm/services/node_proxy.py
@@ -0,0 +1,180 @@
+import json
+import ssl
+import base64
+
+from urllib.error import HTTPError, URLError
+from typing import List, Any, Dict, Tuple, Optional, MutableMapping
+
+from .cephadmservice import CephadmDaemonDeploySpec, CephService
+from ceph.deployment.service_spec import ServiceSpec, PlacementSpec
+from ceph.utils import http_req
+from orchestrator import OrchestratorError
+
+
+class NodeProxy(CephService):
+    TYPE = 'node-proxy'
+
+    def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
+        assert self.TYPE == daemon_spec.daemon_type
+        daemon_id, host = daemon_spec.daemon_id, daemon_spec.host
+
+        if not self.mgr.http_server.agent:
+            raise OrchestratorError('Cannot deploy node-proxy before creating cephadm endpoint')
+
+        keyring = self.get_keyring_with_caps(self.get_auth_entity(daemon_id, host=host), [])
+        daemon_spec.keyring = keyring
+        self.mgr.node_proxy_cache.update_keyring(host, keyring)
+
+        daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
+
+        return daemon_spec
+
+    def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
+        # node-proxy is re-using the agent endpoint and therefore
+        # needs similar checks to see if the endpoint is ready.
+        self.agent_endpoint = self.mgr.http_server.agent
+        try:
+            assert self.agent_endpoint
+            assert self.agent_endpoint.ssl_certs.get_root_cert()
+            assert self.agent_endpoint.server_port
+        except Exception:
+            raise OrchestratorError(
+                'Cannot deploy node-proxy daemons until cephadm endpoint has finished generating certs')
+
+        listener_cert, listener_key = self.agent_endpoint.ssl_certs.generate_cert(daemon_spec.host, self.mgr.inventory.get_addr(daemon_spec.host))
+        cfg = {
+            'target_ip': self.mgr.get_mgr_ip(),
+            'target_port': self.agent_endpoint.server_port,
+            'name': f'node-proxy.{daemon_spec.host}',
+            'keyring': daemon_spec.keyring,
+            'root_cert.pem': self.agent_endpoint.ssl_certs.get_root_cert(),
+            'listener.crt': listener_cert,
+            'listener.key': listener_key,
+        }
+        config = {'node-proxy.json': json.dumps(cfg)}
+
+        return config, sorted([str(self.mgr.get_mgr_ip()), str(self.agent_endpoint.server_port),
+                               self.agent_endpoint.ssl_certs.get_root_cert()])
+
+    def handle_hw_monitoring_setting(self) -> bool:
+        # function to apply or remove node-proxy service spec depending
+        # on whether the hw_mointoring config option is set or not.
+        # It should return True when it either creates or deletes a spec
+        # and False otherwise.
+        if self.mgr.hw_monitoring:
+            if 'node-proxy' not in self.mgr.spec_store:
+                spec = ServiceSpec(
+                    service_type='node-proxy',
+                    placement=PlacementSpec(host_pattern='*')
+                )
+                self.mgr.spec_store.save(spec)
+                return True
+            return False
+        else:
+            if 'node-proxy' in self.mgr.spec_store:
+                self.mgr.spec_store.rm('node-proxy')
+                return True
+            return False
+
+    def get_ssl_ctx(self) -> ssl.SSLContext:
+        ssl_root_crt = self.mgr.http_server.agent.ssl_certs.get_root_cert()
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = True
+        ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+        ssl_ctx.load_verify_locations(cadata=ssl_root_crt)
+        return ssl_ctx
+
+    def led(self, light_type: str, action: str, hostname: str, device: Optional[str] = None) -> Dict[str, Any]:
+        ssl_ctx: ssl.SSLContext = self.get_ssl_ctx()
+        header: MutableMapping[str, str] = {}
+        method: str = 'PATCH' if action in ['on', 'off'] else 'GET'
+        payload: Optional[Dict[str, str]] = None
+        addr: str = self.mgr.inventory.get_addr(hostname)
+        endpoint: List[str] = ['led', light_type]
+        _device: str = device if device else ''
+
+        if light_type == 'drive':
+            endpoint.append(_device)
+
+        if method == 'PATCH':
+            payload = dict(state=action)
+
+        header = self.generate_auth_header(hostname)
+
+        endpoint = f'/{"/".join(endpoint)}'
+
+        try:
+            headers, result, status = http_req(hostname=addr,
+                                               port='8080',
+                                               headers=header,
+                                               method=method,
+                                               data=json.dumps(payload),
+                                               endpoint=endpoint,
+                                               ssl_ctx=ssl_ctx)
+            result_json = json.loads(result)
+        except HTTPError as e:
+            self.mgr.log.error(e)
+            raise
+        except URLError as e:
+            raise RuntimeError(e)
+
+        return result_json
+
+    def generate_auth_header(self, hostname: str) -> Dict[str, str]:
+        try:
+            username = self.mgr.node_proxy_cache.oob[hostname]['username']
+            password = self.mgr.node_proxy_cache.oob[hostname]['password']
+            auth: bytes = f'{username}:{password}'.encode('utf-8')
+            auth_str: str = base64.b64encode(auth).decode('utf-8')
+            header = {'Authorization': f'Basic {auth_str}'}
+        except KeyError as e:
+            self.mgr.log.error(f'Check oob information is provided for {hostname}.')
+            raise RuntimeError(e)
+        return header
+
+    def shutdown(self, hostname: str, force: Optional[bool] = False) -> Dict[str, Any]:
+        ssl_ctx: ssl.SSLContext = self.get_ssl_ctx()
+        header: Dict[str, str] = self.generate_auth_header(hostname)
+        addr: str = self.mgr.inventory.get_addr(hostname)
+
+        endpoint = '/shutdown'
+        payload: Dict[str, Optional[bool]] = dict(force=force)
+
+        try:
+            headers, result, status = http_req(hostname=addr,
+                                               port='8080',
+                                               headers=header,
+                                               data=json.dumps(payload),
+                                               endpoint=endpoint,
+                                               ssl_ctx=ssl_ctx)
+            result_json = json.loads(result)
+        except HTTPError as e:
+            self.mgr.log.error(e)
+            raise
+        except URLError as e:
+            raise RuntimeError(e)
+
+        return result_json
+
+    def powercycle(self, hostname: str) -> Dict[str, Any]:
+        ssl_ctx: ssl.SSLContext = self.get_ssl_ctx()
+        header: Dict[str, str] = self.generate_auth_header(hostname)
+        addr: str = self.mgr.inventory.get_addr(hostname)
+
+        endpoint = '/powercycle'
+
+        try:
+            headers, result, status = http_req(hostname=addr,
+                                               port='8080',
+                                               headers=header,
+                                               data="{}",
+                                               endpoint=endpoint,
+                                               ssl_ctx=ssl_ctx)
+            result_json = json.loads(result)
+        except HTTPError as e:
+            self.mgr.log.error(e)
+            raise
+        except URLError as e:
+            raise RuntimeError(e)
+
+        return result_json
diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index b713d04cd597..0c9ee127547c 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -35,8 +35,8 @@ def get_mgr_ip(self) -> str:
 class TestNodeProxyEndpoint(helper.CPWebCase):
     mgr = FakeMgr()
     app = NodeProxyEndpoint(mgr)
-    mgr.agent_cache.agent_keys = {"host01": "fake-secret01",
-                                  "host02": "fake-secret02"}
+    mgr.node_proxy.keyrings = {"host01": "fake-secret01",
+                               "host02": "fake-secret02"}
     mgr.node_proxy.oob = {"host01": {"username": "oob-user01",
                                      "password": "oob-pass01"},
                           "host02": {"username": "oob-user02",
@@ -68,38 +68,38 @@ def test_oob_data_misses_name_field(self):
         self.assertStatus('400 Bad Request')
 
     def test_oob_data_misses_secret_field(self):
-        data = '{"cephx": {"name": "host01"}}'
+        data = '{"cephx": {"name": "node-proxy.host01"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                 ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
     def test_oob_agent_not_running(self):
-        data = '{"cephx": {"name": "host03", "secret": "fake-secret03"}}'
+        data = '{"cephx": {"name": "node-proxy.host03", "secret": "fake-secret03"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                 ('Content-Length', str(len(data)))])
         self.assertStatus('502 Bad Gateway')
 
     def test_oob_wrong_keyring(self):
-        data = '{"cephx": {"name": "host01", "secret": "wrong-keyring"}}'
+        data = '{"cephx": {"name": "node-proxy.host01", "secret": "wrong-keyring"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                 ('Content-Length', str(len(data)))])
         self.assertStatus('403 Forbidden')
 
     def test_oob_ok(self):
-        data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
+        data = '{"cephx": {"name": "node-proxy.host01", "secret": "fake-secret01"}}'
         self.getPage("/oob", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                 ('Content-Length', str(len(data)))])
         self.assertStatus('200 OK')
 
     def test_data_missing_patch(self):
-        data = '{"cephx": {"name": "host01", "secret": "fake-secret01"}}'
+        data = '{"cephx": {"name": "node-proxy.host01", "secret": "fake-secret01"}}'
         self.getPage("/data", method="POST", body=data, headers=[('Content-Type', 'application/json'),
                                                                  ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
     def test_data_raises_alert(self):
         patch = node_proxy_data.full_set_with_critical
-        data = {"cephx": {"name": "host01", "secret": "fake-secret01"}, "patch": patch}
+        data = {"cephx": {"name": "node-proxy.host01", "secret": "fake-secret01"}, "patch": patch}
         data_str = json.dumps(data)
         self.getPage("/data", method="POST", body=data_str, headers=[('Content-Type', 'application/json'),
                                                                      ('Content-Length', str(len(data_str)))])
diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py
index 63672936c7cb..3aedfbd86f00 100644
--- a/src/pybind/mgr/cephadm/utils.py
+++ b/src/pybind/mgr/cephadm/utils.py
@@ -31,7 +31,7 @@ class CephadmNoImage(Enum):
 CEPH_UPGRADE_ORDER = CEPH_TYPES + GATEWAY_TYPES + MONITORING_STACK_TYPES
 
 # these daemon types use the ceph container image
-CEPH_IMAGE_TYPES = CEPH_TYPES + ['iscsi', 'nfs']
+CEPH_IMAGE_TYPES = CEPH_TYPES + ['iscsi', 'nfs', 'node-proxy']
 
 # these daemons do not use the ceph image. There are other daemons
 # that also don't use the ceph image, but we only care about those
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index f97b61e8f9b5..362badcc35ee 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -359,6 +359,33 @@ def add_host(self, host_spec: HostSpec) -> OrchResult[str]:
         """
         raise NotImplementedError()
 
+    def hardware_light(self, light_type: str, action: str, hostname: str, device: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Light a chassis or device ident LED.
+
+        :param light_type: led type (chassis or device).
+        :param action: set or get status led.
+        :param hostname: the name of the host.
+        :param device: the device id (when light_type = 'device')
+        """
+        raise NotImplementedError()
+
+    def hardware_powercycle(self, hostname: str, yes_i_really_mean_it: bool = False) -> OrchResult[str]:
+        """
+        Reboot a host.
+
+        :param hostname: the name of the host being rebooted.
+        """
+        raise NotImplementedError()
+
+    def hardware_shutdown(self, hostname: str, force: Optional[bool] = False, yes_i_really_mean_it: bool = False) -> OrchResult[str]:
+        """
+        Shutdown a host.
+
+        :param hostname: the name of the host to shutdown.
+        """
+        raise NotImplementedError()
+
     def hardware_status(self, hostname: Optional[str] = None, category: Optional[str] = 'summary') -> OrchResult[str]:
         """
         Display hardware status.
@@ -869,6 +896,7 @@ def daemon_type_to_service(dtype: str) -> str:
         'crashcollector': 'crash',  # Specific Rook Daemon
         'container': 'container',
         'agent': 'agent',
+        'node-proxy': 'node-proxy',
         'snmp-gateway': 'snmp-gateway',
         'elasticsearch': 'elasticsearch',
         'jaeger-agent': 'jaeger-agent',
@@ -901,6 +929,7 @@ def service_to_daemon_types(stype: str) -> List[str]:
         'crash': ['crash'],
         'container': ['container'],
         'agent': ['agent'],
+        'node-proxy': ['node-proxy'],
         'snmp-gateway': ['snmp-gateway'],
         'elasticsearch': ['elasticsearch'],
         'jaeger-agent': ['jaeger-agent'],
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 27d91d1a4d35..22e1dd2f6527 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -591,6 +591,51 @@ def _common_table(self, category: str, hostname: Optional[str], table: PrettyTab
 
         return table.get_string()
 
+    class HardwareLightType(enum.Enum):
+        chassis = 'chassis'
+        device = 'drive'
+
+    class HardwareLightAction(enum.Enum):
+        on = 'on'
+        off = 'off'
+        get = 'get'
+
+    @_cli_write_command('orch hardware light')
+    def _hardware_light(self,
+                        light_type: HardwareLightType, action: HardwareLightAction,
+                        hostname: str, device: Optional[str] = None) -> HandleCommandResult:
+        """Enable or Disable a device or chassis LED"""
+        if light_type == self.HardwareLightType.device and not device:
+            return HandleCommandResult(stderr='you must pass a device ID.',
+                                       retval=-errno.ENOENT)
+
+        completion = self.hardware_light(light_type.value, action.value, hostname, device)
+        data = raise_if_exception(completion)
+        output: str = ''
+        if action == self.HardwareLightAction.get:
+            status = 'on' if data["LocationIndicatorActive"] else 'off'
+            if light_type == self.HardwareLightType.device:
+                output = f'ident LED for {device} on {hostname} is: {status}'
+            else:
+                output = f'ident chassis LED for {hostname} is: {status}'
+        else:
+            pass
+        return HandleCommandResult(stdout=output)
+
+    @_cli_write_command('orch hardware powercycle')
+    def _hardware_powercycle(self, hostname: str, yes_i_really_mean_it: bool = False) -> HandleCommandResult:
+        """Reboot a host"""
+        completion = self.hardware_powercycle(hostname, yes_i_really_mean_it=yes_i_really_mean_it)
+        raise_if_exception(completion)
+        return HandleCommandResult(stdout=completion.result_str())
+
+    @_cli_write_command('orch hardware shutdown')
+    def _hardware_shutdown(self, hostname: str, force: Optional[bool] = False, yes_i_really_mean_it: bool = False) -> HandleCommandResult:
+        """Shutdown a host"""
+        completion = self.hardware_shutdown(hostname, force, yes_i_really_mean_it=yes_i_really_mean_it)
+        raise_if_exception(completion)
+        return HandleCommandResult(stdout=completion.result_str())
+
     @_cli_write_command('orch host rm')
     def _remove_host(self, hostname: str, force: bool = False, offline: bool = False, rm_crush_entry: bool = False) -> HandleCommandResult:
         """Remove a host"""
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index cb81cf7b5c8b..a8b679ee3969 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -755,7 +755,8 @@ class ServiceSpec(object):
     KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi nvmeof loki promtail mds mgr mon nfs ' \
                           'node-exporter osd prometheus rbd-mirror rgw agent ceph-exporter ' \
                           'container ingress cephfs-mirror snmp-gateway jaeger-tracing ' \
-                          'elasticsearch jaeger-agent jaeger-collector jaeger-query'.split()
+                          'elasticsearch jaeger-agent jaeger-collector jaeger-query ' \
+                          'node-proxy'.split()
     REQUIRES_SERVICE_ID = 'iscsi nvmeof mds nfs rgw container ingress '.split()
     MANAGED_CONFIG_OPTIONS = [
         'mds_join_fs',
diff --git a/src/python-common/ceph/utils.py b/src/python-common/ceph/utils.py
index 643be06580b6..e92a2d1de7db 100644
--- a/src/python-common/ceph/utils.py
+++ b/src/python-common/ceph/utils.py
@@ -1,8 +1,15 @@
 import datetime
 import re
 import string
+import ssl
 
-from typing import Optional
+from typing import Optional, MutableMapping, Tuple, Any
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen, Request
+
+import logging
+
+log = logging.getLogger(__name__)
 
 
 def datetime_now() -> datetime.datetime:
@@ -121,3 +128,42 @@ def is_hex(s: str, strict: bool = True) -> bool:
             return False
 
     return True
+
+
+def http_req(hostname: str = '',
+             port: str = '443',
+             method: Optional[str] = None,
+             headers: MutableMapping[str, str] = {},
+             data: Optional[str] = None,
+             endpoint: str = '/',
+             scheme: str = 'https',
+             ssl_verify: bool = False,
+             timeout: Optional[int] = None,
+             ssl_ctx: Optional[Any] = None) -> Tuple[Any, Any, Any]:
+
+    if not ssl_ctx:
+        ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        if not ssl_verify:
+            ssl_ctx.check_hostname = False
+            ssl_ctx.verify_mode = ssl.CERT_NONE
+        else:
+            ssl_ctx.verify_mode = ssl.CERT_REQUIRED
+
+    url: str = f'{scheme}://{hostname}:{port}{endpoint}'
+    _data = bytes(data, 'ascii') if data else None
+    _headers = headers
+    if data and not method:
+        method = 'POST'
+    if not _headers.get('Content-Type') and method in ['POST', 'PATCH']:
+        _headers['Content-Type'] = 'application/json'
+    try:
+        req = Request(url, _data, _headers, method=method)
+        with urlopen(req, context=ssl_ctx, timeout=timeout) as response:
+            response_str = response.read()
+            response_headers = response.headers
+            response_code = response.code
+        return response_headers, response_str.decode(), response_code
+    except (HTTPError, URLError) as e:
+        log.error(e)
+        # handle error here if needed
+        raise

From c4675a6c97a8fe331f836113e5ffde4a09717dc4 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 12 Jan 2024 09:09:15 +0000
Subject: [PATCH 1609/2492] node-proxy: address a typo

while checking logs, I noticed the following message:

```
2024-01-12 09:08:03,751 - reporter - INFO - Reporter url set to https:10.10.10.11:7150/node-proxy/data
```

Although this is only a cosmetic issue as this variable
is only used for logging messages, let's fix it.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/reporter.py b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
index 9e5521aa280b..ab0fbc1af515 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/reporter.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
@@ -24,7 +24,7 @@ def __init__(self,
         self.reporter_port: str = reporter_port
         self.reporter_endpoint: str = reporter_endpoint
         self.log = Logger(__name__)
-        self.reporter_url: str = (f'{reporter_scheme}:{reporter_hostname}:'
+        self.reporter_url: str = (f'{reporter_scheme}://{reporter_hostname}:'
                                   f'{reporter_port}{reporter_endpoint}')
         self.log.logger.info(f'Reporter url set to {self.reporter_url}')
 

From c3682f0e663a6fc4c471030d7f5ff15ef0239a49 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 12 Jan 2024 09:11:21 +0000
Subject: [PATCH 1610/2492] node-proxy: fix a thread/locking issue

This `sleep(5)` should be initiated *after* the lock is released.
Otherwise, it can cause troubles with the reporter loop which can
never acquire the lock.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
index 98f117196158..74d9219d1ce5 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
@@ -73,13 +73,13 @@ def update(self) -> None:
                     executor.map(lambda f: f(), update_funcs)
 
                 self.data_ready = True
-                sleep(5)
             except RuntimeError as e:
                 self.run = False
                 self.log.logger.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
                 self.client.logout()
             finally:
                 self.lock.release()
+                sleep(5)
                 self.log.logger.debug('lock released in the update loop.')
 
     def flush(self) -> None:

From a724e4cfcf51a5b609d171f300d5bf7bc09cdc5a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 12 Jan 2024 09:15:02 +0000
Subject: [PATCH 1611/2492] node-proxy: reduce log level in reporter agent

the following messages get logged quite a lot while
this is not a very useful information in a normal situation:

```
2024-01-12 09:09:40,604 - reporter - INFO - data ready to be sent to the mgr.
2024-01-12 09:09:40,604 - reporter - INFO - no diff, not sending data to the mgr.
2024-01-12 09:10:15,022 - reporter - INFO - data ready to be sent to the mgr.
2024-01-12 09:10:15,022 - reporter - INFO - no diff, not sending data to the mgr.
...
```

This commit changes the log level to DEBUG.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/reporter.py b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
index ab0fbc1af515..aa16d83421f3 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/reporter.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
@@ -47,7 +47,7 @@ def loop(self) -> None:
             self.system.lock.acquire()
             self.log.logger.debug('lock acquired in reporter loop.')
             if self.system.data_ready:
-                self.log.logger.info('data ready to be sent to the mgr.')
+                self.log.logger.debug('data ready to be sent to the mgr.')
                 if not self.system.get_system() == self.system.previous_data:
                     self.log.logger.info('data has changed since last iteration.')
                     self.data['patch'] = self.system.get_system()
@@ -68,7 +68,7 @@ def loop(self) -> None:
                     else:
                         self.system.previous_data = self.system.get_system()
                 else:
-                    self.log.logger.info('no diff, not sending data to the mgr.')
+                    self.log.logger.debug('no diff, not sending data to the mgr.')
             self.system.lock.release()
             self.log.logger.debug('lock released in reporter loop.')
             time.sleep(5)

From 0dd73643649ddc2366e60de4fe6c078b6e112091 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 15 Jan 2024 14:09:23 +0000
Subject: [PATCH 1612/2492] node-proxy: add packaging related changes

This adds the required changes to build an RPM of node-proxy.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 ceph.spec.in                                  | 16 ++++++++
 src/CMakeLists.txt                            |  1 +
 src/ceph-node-proxy/CMakeLists.txt            | 25 ++++++++++++
 src/ceph-node-proxy/MANIFEST.in               |  2 +
 .../ceph_node_proxy/__init__.py               |  2 +
 src/ceph-node-proxy/setup.py                  | 39 +++++++++++++++++++
 6 files changed, 85 insertions(+)
 create mode 100644 src/ceph-node-proxy/CMakeLists.txt
 create mode 100644 src/ceph-node-proxy/MANIFEST.in
 create mode 100644 src/ceph-node-proxy/setup.py

diff --git a/ceph.spec.in b/ceph.spec.in
index 058c79c5e7f8..f6b9f0459446 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -1296,6 +1296,15 @@ Group:          System/Monitoring
 %description mib
 This package provides a Ceph MIB for SNMP traps.
 
+%package node-proxy
+Summary:        hw monitoring agent for Ceph
+BuildArch:      noarch
+%if 0%{?suse_version}
+Group:          System/Monitoring
+%endif
+%description node-proxy
+This package provides a Ceph hardware monitoring agent.
+
 #################################################################################
 # common
 #################################################################################
@@ -2638,4 +2647,11 @@ exit 0
 %attr(0755,root,root) %dir %{_datadir}/snmp
 %{_datadir}/snmp/mibs
 
+%files node-proxy
+%{_sbindir}/ceph-node-proxy
+%dir %{python3_sitelib}/ceph_node_proxy
+%{python3_sitelib}/ceph_node_proxy/*
+%{python3_sitelib}/ceph_node_proxy-*
+#%{_mandir}/man8/ceph-node-proxy.8*
+
 %changelog
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ba6549673e6c..afa2442d5709 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -662,6 +662,7 @@ endif()
 if(NOT WIN32)
 add_subdirectory(pybind)
 add_subdirectory(ceph-volume)
+add_subdirectory(ceph-node-proxy)
 add_subdirectory(python-common)
 add_subdirectory(cephadm)
 endif(NOT WIN32)
diff --git a/src/ceph-node-proxy/CMakeLists.txt b/src/ceph-node-proxy/CMakeLists.txt
new file mode 100644
index 000000000000..0f83b0b6caa2
--- /dev/null
+++ b/src/ceph-node-proxy/CMakeLists.txt
@@ -0,0 +1,25 @@
+
+include(Distutils)
+
+distutils_install_module(ceph_node_proxy
+  INSTALL_SCRIPT ${CMAKE_INSTALL_FULL_SBINDIR})
+
+# Required for running ceph-node-proxy in a vstart environment
+set(CEPH_NODE_PROXY_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/ceph-node-proxy-virtualenv)
+
+add_custom_command(
+  OUTPUT ${CEPH_NODE_PROXY_VIRTUALENV}/bin/python
+  COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh --python=${Python3_EXECUTABLE} ${CEPH_NODE_PROXY_VIRTUALENV}
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/ceph-node-proxy
+  COMMENT "ceph-node-proxy venv is being created")
+
+add_custom_command(
+  OUTPUT ${CEPH_NODE_PROXY_VIRTUALENV}/bin/ceph-node-proxy
+  DEPENDS ${CEPH_NODE_PROXY_VIRTUALENV}/bin/python
+  COMMAND . ${CEPH_NODE_PROXY_VIRTUALENV}/bin/activate && ${CEPH_NODE_PROXY_VIRTUALENV}/bin/python setup.py develop && deactivate
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/ceph-node-proxy
+  COMMENT "${CMAKE_SOURCE_DIR}/src/ceph-node-proxy")
+
+add_custom_target(ceph-node-proxy-venv-setup
+  DEPENDS ${CEPH_NODE_PROXY_VIRTUALENV}/bin/ceph-node-proxy)
+
diff --git a/src/ceph-node-proxy/MANIFEST.in b/src/ceph-node-proxy/MANIFEST.in
new file mode 100644
index 000000000000..3e6850fe101a
--- /dev/null
+++ b/src/ceph-node-proxy/MANIFEST.in
@@ -0,0 +1,2 @@
+include bin/ceph-node-proxy
+
diff --git a/src/ceph-node-proxy/ceph_node_proxy/__init__.py b/src/ceph-node-proxy/ceph_node_proxy/__init__.py
index e69de29bb2d1..20403aa92bbf 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/__init__.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/__init__.py
@@ -0,0 +1,2 @@
+__version__ = '1.0.0'
+__release__ = 'squid'
diff --git a/src/ceph-node-proxy/setup.py b/src/ceph-node-proxy/setup.py
new file mode 100644
index 000000000000..7dcc7cdf5bf8
--- /dev/null
+++ b/src/ceph-node-proxy/setup.py
@@ -0,0 +1,39 @@
+from setuptools import setup, find_packages
+import os
+
+
+setup(
+    name='ceph-node-proxy',
+    version='1.0.0',
+    packages=find_packages(),
+
+    author='',
+    author_email='gabrioux@ibm.com',
+    description='node-proxy agent to inventory and report hardware statuses.',
+    license='LGPLv2+',
+    keywords='ceph hardware inventory monitoring',
+    url='https://github.com/ceph/ceph',
+    zip_safe=False,
+    install_requires='ceph',
+    dependency_links=[''.join(['file://', os.path.join(os.getcwd(), '../',
+                                                       'python-common#egg=ceph-1.0.0')])],
+    tests_require=[
+        'pytest >=2.1.3',
+        'tox',
+        'ceph',
+    ],
+    entry_points=dict(
+        console_scripts=[
+            'ceph-node-proxy = ceph_node_proxy.main:main',
+        ],
+    ),
+    classifiers=[
+        'Environment :: Console',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'Operating System :: POSIX :: Linux',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3.9',
+    ]
+)

From 316d032148d8ac8792eb08a80388630866e25f04 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 23 Jan 2024 09:36:00 +0000
Subject: [PATCH 1613/2492] node-proxy: add new attribute to
 BaseRedfishSystem()

This adds `self.component_list()` in order to parametrize
which categories the agent will collect.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../ceph_node_proxy/baseredfishsystem.py      | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
index 74d9219d1ce5..e80523fed185 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
@@ -35,6 +35,20 @@ def __init__(self, **kw: Any) -> None:
         self.job_service_endpoint: str = ''
         self.create_reboot_job_endpoint: str = ''
         self.setup_job_queue_endpoint: str = ''
+        self.component_list: List[str] = kw.get('component_list', ['memory',
+                                                                   'power',
+                                                                   'fans',
+                                                                   'network',
+                                                                   'processors',
+                                                                   'storage',
+                                                                   'firmwares'])
+        self.update_funcs: List[Callable] = []
+        for component in self.component_list:
+            self.log.logger.debug(f'adding: {component} to hw component gathered list.')
+            func = f'_update_{component}'
+            if hasattr(self, func):
+                f = getattr(self, func)
+                self.update_funcs.append(f)
 
         self.start_client()
 
@@ -61,16 +75,9 @@ def update(self) -> None:
             try:
                 self._update_system()
                 self._update_sn()
-                update_funcs = [self._update_memory,
-                                self._update_power,
-                                self._update_fans,
-                                self._update_network,
-                                self._update_processors,
-                                self._update_storage,
-                                self._update_firmwares]
 
                 with concurrent.futures.ThreadPoolExecutor() as executor:
-                    executor.map(lambda f: f(), update_funcs)
+                    executor.map(lambda f: f(), self.update_funcs)
 
                 self.data_ready = True
             except RuntimeError as e:

From c07482a86a4d1a683d19e25a604c6f5fab370957 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 23 Jan 2024 09:41:39 +0000
Subject: [PATCH 1614/2492] node-proxy: collect `LocationIndicatorActive`
 property (storage)

This makes node-proxy collect the `LocationIndicatorActive`
property for storage component.
This can be needed for the Blinkenlight feature.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
index 0424bb38bd1e..83b73657b1aa 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
@@ -95,6 +95,7 @@ def _update_storage(self) -> None:
         fields = ['Description',
                   'CapacityBytes',
                   'Model', 'Protocol',
+                  'LocationIndicatorActive',
                   'SerialNumber', 'Status',
                   'PhysicalLocation']
         entities = self.get_members(data=self._system['Systems'],

From 6a130a70077dced7104678c8f8933e2d56f64492 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 23 Jan 2024 12:18:05 +0000
Subject: [PATCH 1615/2492] crimson/os/cyanstore: support OP_RMCOLL

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/os/cyanstore/cyan_store.cc | 17 +++++++++++++++++
 src/crimson/os/cyanstore/cyan_store.h  |  1 +
 2 files changed, 18 insertions(+)

diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc
index f2a6018e36ab..36230893d36b 100644
--- a/src/crimson/os/cyanstore/cyan_store.cc
+++ b/src/crimson/os/cyanstore/cyan_store.cc
@@ -494,6 +494,12 @@ seastar::future<> CyanStore::Shard::do_transaction_no_callbacks(
         r = _create_collection(cid, op->split_bits);
       }
       break;
+      case Transaction::OP_RMCOLL:
+      {
+        coll_t cid = i.get_cid(op->cid);
+        r = _remove_collection(cid);
+      }
+      break;
       case Transaction::OP_SETALLOCHINT:
       {
         r = 0;
@@ -863,6 +869,17 @@ int CyanStore::Shard::_create_collection(const coll_t& cid, int bits)
   return 0;
 }
 
+int CyanStore::Shard::_remove_collection(const coll_t& cid)
+{
+  logger().debug("{} cid={}", __func__, cid);
+  auto c = _get_collection(cid);
+  if (!c) {
+    return -ENOENT;
+  }
+  coll_map.erase(cid);
+  return 0;
+}
+
 boost::intrusive_ptr<Collection>
 CyanStore::Shard::_get_collection(const coll_t& cid)
 {
diff --git a/src/crimson/os/cyanstore/cyan_store.h b/src/crimson/os/cyanstore/cyan_store.h
index 307f9ec32eda..518222d82f65 100644
--- a/src/crimson/os/cyanstore/cyan_store.h
+++ b/src/crimson/os/cyanstore/cyan_store.h
@@ -148,6 +148,7 @@ class CyanStore final : public FuturizedStore {
 		 std::string_view name);
     int _rm_attrs(const coll_t& cid, const ghobject_t& oid);
     int _create_collection(const coll_t& cid, int bits);
+    int _remove_collection(const coll_t& cid);
     boost::intrusive_ptr<Collection> _get_collection(const coll_t& cid);
 
   private:

From fcbf7367d285629b382e3d9d32ac354319d1cc66 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Wed, 17 Jan 2024 13:24:36 -0500
Subject: [PATCH 1616/2492] rbd-nbd: map using netlink interface by default

Mapping rbd images to nbd devices using ioctl interface is not
robust. It was discovered that the device size or the md5 checksum
of the nbd device was incorrect immediately after mapping using
ioctl method. When using the nbd netlink interface to map RBD images
the issue was not encountered. Switch to using nbd netlink interface
for mapping.

Fixes: https://tracker.ceph.com/issues/64063
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 PendingReleaseNotes          |  3 +++
 qa/workunits/rbd/rbd-nbd.sh  |  7 ++++---
 src/tools/rbd_nbd/rbd-nbd.cc | 32 ++++++++++++++------------------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 0de7e784507b..f11772762728 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -110,6 +110,9 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   and valid), diff-iterate is now guaranteed to execute locally if exclusive
   lock is available.  This brings a dramatic performance improvement for QEMU
   live disk synchronization and backup use cases.
+* RBD: The ``try-netlink`` mapping option for rbd-nbd has become the default
+  and is now deprecated. If the NBD netlink interface is not supported by the
+  kernel, then the mapping is retried using the legacy ioctl interface.
 
 >=18.0.0
 
diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh
index 8e1b05b3f782..98b3aff1370d 100755
--- a/qa/workunits/rbd/rbd-nbd.sh
+++ b/qa/workunits/rbd/rbd-nbd.sh
@@ -205,6 +205,7 @@ used=`rbd -p ${POOL} --format xml du ${IMAGE} |
 unmap_device ${DEV} ${PID}
 
 # resize test
+# also test that try-netlink option is accepted for compatibility
 DEV=`_sudo rbd device -t nbd -o try-netlink map ${POOL}/${IMAGE}`
 get_pid ${POOL}
 devname=$(basename ${DEV})
@@ -391,7 +392,7 @@ cat ${LOG_FILE}
 expect_false grep 'quiesce failed' ${LOG_FILE}
 
 # test detach/attach
-OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map ${POOL}/${IMAGE}`
+OUT=`_sudo rbd device --device-type nbd --show-cookie map ${POOL}/${IMAGE}`
 read DEV COOKIE <<< "${OUT}"
 get_pid ${POOL}
 _sudo mount ${DEV} ${TEMPDIR}/mnt
@@ -419,7 +420,7 @@ _sudo umount ${TEMPDIR}/mnt
 unmap_device ${DEV} ${PID}
 # if kernel supports cookies
 if [ -n "${COOKIE}" ]; then
-    OUT=`_sudo rbd device --device-type nbd --show-cookie --cookie "abc de" --options try-netlink map ${POOL}/${IMAGE}`
+    OUT=`_sudo rbd device --device-type nbd --show-cookie --cookie "abc de" map ${POOL}/${IMAGE}`
     read DEV ANOTHER_COOKIE <<< "${OUT}"
     get_pid ${POOL}
     test "${ANOTHER_COOKIE}" = "abc de"
@@ -429,7 +430,7 @@ DEV=
 
 # test detach/attach with --snap-id
 SNAPID=`rbd snap ls ${POOL}/${IMAGE} | awk '$2 == "snap" {print $1}'`
-OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map --snap-id ${SNAPID} ${POOL}/${IMAGE}`
+OUT=`_sudo rbd device --device-type nbd --show-cookie map --snap-id ${SNAPID} ${POOL}/${IMAGE}`
 read DEV COOKIE <<< "${OUT}"
 get_pid ${POOL}
 _sudo rbd device detach ${POOL}/${IMAGE} --snap-id ${SNAPID} --device-type nbd
diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc
index f2dfa1f660e8..325dbdb5246b 100644
--- a/src/tools/rbd_nbd/rbd-nbd.cc
+++ b/src/tools/rbd_nbd/rbd-nbd.cc
@@ -106,7 +106,6 @@ struct Config {
   bool quiesce = false;
   bool readonly = false;
   bool set_max_part = false;
-  bool try_netlink = false;
   bool show_cookie = false;
 
   std::string poolname;
@@ -166,7 +165,6 @@ static void usage()
             << "  --read-only                   Map read-only\n"
             << "  --reattach-timeout <sec>      Set nbd re-attach timeout\n"
             << "                                (default: " << Config().reattach_timeout << ")\n"
-            << "  --try-netlink                 Use the nbd netlink interface\n"
             << "  --show-cookie                 Show device cookie\n"
             << "  --cookie                      Specify device cookie\n"
             << "  --snap-id <snap-id>           Specify snapshot by ID instead of by name\n"
@@ -1682,7 +1680,7 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect)
   unsigned long flags;
   unsigned long size;
   unsigned long blksize = RBD_NBD_BLKSIZE;
-  bool use_netlink;
+  bool use_netlink = true;
 
   int fd[2];
 
@@ -1859,20 +1857,17 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect)
 
   server = start_server(fd[1], image, cfg);
 
-  use_netlink = cfg->try_netlink || reconnect;
-  if (use_netlink) {
-    // generate when the cookie is not supplied at CLI
-    if (!reconnect && cfg->cookie.empty()) {
-      uuid_d uuid_gen;
-      uuid_gen.generate_random();
-      cfg->cookie = uuid_gen.to_string();
-    }
-    r = try_netlink_setup(cfg, fd[0], size, flags, reconnect);
-    if (r < 0) {
-      goto free_server;
-    } else if (r == 1) {
-      use_netlink = false;
-    }
+  // generate when the cookie is not supplied at CLI
+  if (!reconnect && cfg->cookie.empty()) {
+    uuid_d uuid_gen;
+    uuid_gen.generate_random();
+    cfg->cookie = uuid_gen.to_string();
+  }
+  r = try_netlink_setup(cfg, fd[0], size, flags, reconnect);
+  if (r < 0) {
+    goto free_server;
+  } else if (r == 1) {
+    use_netlink = false;
   }
 
   if (!use_netlink) {
@@ -2216,7 +2211,8 @@ static int parse_args(vector<const char*>& args, std::ostream *err_msg,
     } else if (ceph_argparse_flag(args, i, "--pretty-format", (char *)NULL)) {
       cfg->pretty_format = true;
     } else if (ceph_argparse_flag(args, i, "--try-netlink", (char *)NULL)) {
-      cfg->try_netlink = true;
+      // netlink used by default. option not required anymore.
+      // accept for compatibility.
     } else if (ceph_argparse_flag(args, i, "--show-cookie", (char *)NULL)) {
       cfg->show_cookie = true;
     } else if (ceph_argparse_witharg(args, i, &cfg->cookie, "--cookie", (char *)NULL)) {

From 5acd763010344725617b8237e2844e18651d46d2 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 18 Jan 2024 09:32:17 -0500
Subject: [PATCH 1617/2492] qa: use centos 9.stream for cephfs stock kernel
 testing

RHEL8 is no longer supported in Squid. RHEL9 is not yet available in FOG.

Fixes: https://tracker.ceph.com/issues/64085
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../mount/kclient/overrides/distro/stock/centos_9.stream.yaml    | 1 +
 qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml       | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 120000 qa/cephfs/mount/kclient/overrides/distro/stock/centos_9.stream.yaml
 delete mode 120000 qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml

diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/centos_9.stream.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/centos_9.stream.yaml
new file mode 120000
index 000000000000..dc442c241498
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/stock/centos_9.stream.yaml
@@ -0,0 +1 @@
+.qa/distros/all/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml
deleted file mode 120000
index 133acf27bff7..000000000000
--- a/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/all/rhel_8.yaml
\ No newline at end of file

From 5d0477eb1bda0f895ed2ee19db6f9cd3fc47f900 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 25 Jan 2024 11:26:07 -0500
Subject: [PATCH 1618/2492] qa/tempest: use default
 tempurl_digest_hashlib=sha256

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/tempest/tasks/tempest.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/qa/suites/rgw/tempest/tasks/tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml
index 44afe7efede3..cae0b3362ca1 100644
--- a/qa/suites/rgw/tempest/tasks/tempest.yaml
+++ b/qa/suites/rgw/tempest/tasks/tempest.yaml
@@ -27,8 +27,6 @@ tasks:
       object-storage-feature-enabled:
         container_sync: false
         discoverability: true
-        # TODO(tobias-urdin): Use sha256 when supported in RadosGW
-        tempurl_digest_hashlib: sha1
       blocklist:
         - .*test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota
         - .*test_container_acl_negative.ObjectACLsNegativeTest.*

From 0eace4ea9ea42412d4d6a16d24a8660642e41173 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 24 Jan 2024 17:22:44 +0000
Subject: [PATCH 1619/2492] common/dout: fix FTBFS on GCC 14
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The following problem has been reported by Kaleb Keithley:

```
/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h: In lambda function:
/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:184:73: error: call to non-‘constexpr’ function ‘virtual unsigned int DoutPrefixProvider::get_subsys() const’
  184 |     dout_impl(pdpp->get_cct(), ceph::dout::need_dynamic(pdpp->get_subsys()), v) \
      |                                                         ~~~~~~~~~~~~~~~~^~
/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:155:58: note: in definition of macro ‘dout_impl’
  155 |       return (cctX->_conf->subsys.template should_gather<sub, v>());    \
      |                                                          ^~~
/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h:3617:3: note: in expansion of macro ‘ldpp_dout’
 3617 |   ldpp_dout(dpp, 10) << "build_prior all_probe " << all_probe << dendl;
      |   ^~~~~~~~~
```

For details of the problem and the idea behind the fix,
please refer to the comment this commit brings to `dout.h`.

The minimized replicator that the facilitated Goldbot-based
investigation:

```cpp
namespace ceph::dout {

template<typename T>
struct dynamic_marker_t {
  T value;
  // constexpr ctor isn't needed as it's an aggregate type
  constexpr operator T() const { return value; }
};

template<typename T>
constexpr dynamic_marker_t<T> need_dynamic(T&& t) {
  return dynamic_marker_t<T>{ std::forward<T>(t) };
}

template<typename T>
struct is_dynamic : public std::false_type {};

template<typename T>
struct is_dynamic<dynamic_marker_t<T>> : public std::true_type {};

} // ceph::dout

struct subsys_t {
  template <unsigned SubV, int LvlV>
  bool should_gather() const {
    return true;
  }
  bool should_gather(const unsigned sub, int level) const {
    return false;
  }
};

static subsys_t subsys;

  do {                                                                  \
  const bool should_gather = [&](const auto cctX) {                     \
    if constexpr (ceph::dout::is_dynamic<decltype(sub)>::value ||       \
                  ceph::dout::is_dynamic<decltype(v)>::value) {         \
      std::cout << "the dynamic path" << std::endl;                     \
      return subsys.should_gather(sub, v);                              \
    } else {                                                            \
      /* The parentheses are **essential** because commas in angle      \
       * brackets are NOT ignored on macro expansion! A language's      \
       * limitation, sorry. */                                          \
      std::cout << "the static path" << std::endl;                      \
      /*return subsys.should_gather(sub, v);*/                              \
      return (subsys.template should_gather<sub, v>());             \
    }                                                                   \
  }(cct);                                                               \
  } while (0)

  if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \
    dout_impl(42, sub, v)

  if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \
    dout_impl(42, ceph::dout::need_dynamic(42), v)

int main() {
    std::random_device dev;
    std::mt19937 rng(dev());
    std::uniform_int_distribution<std::mt19937::result_type> dist6(1,6); // distribution in range [1, 6]

    int sub = dist6(rng);
    ldpp_dout("mocked out", sub);
    //ldpp_subdout("mocked out", 4, 3);
}
```

Fixes: https://tracker.ceph.com/issues/64050
Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/common/dout.h | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/common/dout.h b/src/common/dout.h
index 4cd60efff8fe..6516060c5438 100644
--- a/src/common/dout.h
+++ b/src/common/dout.h
@@ -144,17 +144,27 @@ struct is_dynamic<dynamic_marker_t<T>> : public std::true_type {};
 #else
 #define dout_impl(cct, sub, v)						\
   do {									\
-  const bool should_gather = [&](const auto cctX) {			\
-    if constexpr (ceph::dout::is_dynamic<decltype(sub)>::value ||	\
-		  ceph::dout::is_dynamic<decltype(v)>::value) {		\
+  const bool should_gather = [&](const auto cctX, auto sub_, auto v_) {	\
+    /* The check is performed on `sub_` and `v_` to leverage the C++'s 	\
+     * guarantee on _discarding_ one of blocks of `if constexpr`, which	\
+     * includes also the checks for ill-formed code (`should_gather<>`	\
+     * must not be feed with non-const expresions), BUT ONLY within	\
+     * a template (thus the generic lambda) and under the restriction	\
+     * it's dependant on a parameter of this template).			\
+     * GCC prior to v14 was not enforcing these restrictions. */	\
+    if constexpr (ceph::dout::is_dynamic<decltype(sub_)>::value ||	\
+		  ceph::dout::is_dynamic<decltype(v_)>::value) {	\
       return cctX->_conf->subsys.should_gather(sub, v);			\
     } else {								\
+      constexpr auto sub_helper = static_cast<decltype(sub_)>(sub);	\
+      constexpr auto v_helper = static_cast<decltype(v_)>(v);		\
       /* The parentheses are **essential** because commas in angle	\
        * brackets are NOT ignored on macro expansion! A language's	\
        * limitation, sorry. */						\
-      return (cctX->_conf->subsys.template should_gather<sub, v>());	\
+      return (cctX->_conf->subsys.template should_gather<sub_helper,	\
+							 v_helper>());	\
     }									\
-  }(cct);								\
+  }(cct, sub, v);							\
 									\
   if (should_gather) {							\
     ceph::logging::MutableEntry _dout_e(v, sub);                        \

From 34bed3e36a05b8281b6248fc9a1340bd0f1828a8 Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Tue, 5 Dec 2023 15:28:50 +0000
Subject: [PATCH 1620/2492] osd: Change PG Deletion cost for mClock

With the osd_delete_sleep_ssd and osd_delete_sleep_hdd options disabled with mClock, it was noticed that PG deletion was completing much faster with mClock scheduler.
In order to give mClock a more accurate cost of the PG Deletion operation, we calculate it by taking into consideration how many objects are being deleted.

Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 src/osd/OSD.cc          | 13 ++++++++++---
 src/osd/OSD.h           |  2 +-
 src/osd/PG.cc           |  9 ++++++---
 src/osd/PG.h            |  4 +++-
 src/osd/PrimaryLogPG.cc |  4 +++-
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 9e6b3fd9d929..46e8eb5e3513 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1875,14 +1875,21 @@ void OSDService::queue_scrub_next_chunk(PG *pg, Scrub::scrub_prio_t with_priorit
   queue_scrub_event_msg<PGScrubGetNextChunk>(pg, with_priority);
 }
 
-void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e)
+void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects)
 {
   dout(10) << __func__ << " on " << pgid << " e " << e  << dendl;
+  uint64_t cost_for_queue = [this, num_objects] {
+    if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) {
+      return num_objects * cct->_conf->osd_pg_delete_cost;
+    } else {
+      return cct->_conf->osd_pg_delete_cost;
+    }
+  }();
   enqueue_back(
     OpSchedulerItem(
       unique_ptr<OpSchedulerItem::OpQueueable>(
 	new PGDelete(pgid, e)),
-      cct->_conf->osd_pg_delete_cost,
+      cost_for_queue,
       cct->_conf->osd_pg_delete_priority,
       ceph_clock_now(),
       0,
@@ -10220,7 +10227,7 @@ void OSD::maybe_override_cost_for_qos()
   // If the scheduler enabled is mclock, override the default PG deletion cost
   // so that mclock can meet the QoS goals.
   if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
-    uint64_t pg_delete_cost = 15728640;
+    uint64_t pg_delete_cost = 1048576;
     cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost));
   }
 }
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 5f561e643182..6aface4b3ee0 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -553,7 +553,7 @@ class OSDService : public Scrub::ScrubSchedListener {
 				   unsigned int qu_priority,
 				   Scrub::act_token_t act_token);
 
-  void queue_for_pg_delete(spg_t pgid, epoch_t e);
+  void queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects);
   bool try_finish_pg_delete(PG *pg, unsigned old_pg_num);
 
 private:
diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index cb12dae7c5b1..7a6b7b011803 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2665,7 +2665,8 @@ void PG::C_DeleteMore::complete(int r) {
   ceph_assert(r == 0);
   pg->lock();
   if (!pg->pg_has_reset_since(epoch)) {
-    pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch);
+    pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch,
+	                         num_objects);
   }
   pg->unlock();
   delete this;
@@ -2689,7 +2690,9 @@ std::pair<ghobject_t, bool> PG::do_delete_work(
         std::scoped_lock locker{*this};
         delete_needs_sleep = false;
         if (!pg_has_reset_since(e)) {
-          osd->queue_for_pg_delete(get_pgid(), e);
+	  // We pass 1 for num_objects here as only wpq uses this code path
+	  // and it will be ignored
+          osd->queue_for_pg_delete(get_pgid(), e, 1);
         }
       });
 
@@ -2762,7 +2765,7 @@ std::pair<ghobject_t, bool> PG::do_delete_work(
   bool running = true;
   if (num) {
     dout(20) << __func__ << " deleting " << num << " objects" << dendl;
-    Context *fin = new C_DeleteMore(this, get_osdmap_epoch());
+    Context *fin = new C_DeleteMore(this, get_osdmap_epoch(), num);
     t.register_on_commit(fin);
   } else {
     if (cct->_conf->osd_inject_failure_on_pg_removal) {
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 677f4e93a079..9abf92e90855 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -776,7 +776,9 @@ class PG : public DoutPrefixProvider,
   struct C_DeleteMore : public Context {
     PGRef pg;
     epoch_t epoch;
-    C_DeleteMore(PG *p, epoch_t e) : pg(p), epoch(e) {}
+    int64_t num_objects;
+    C_DeleteMore(PG *p, epoch_t e, int64_t num) : pg(p), epoch(e),
+	                                          num_objects(num){}
     void finish(int r) override {
       ceph_abort();
     }
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index ca83a563e4f8..dea154144a5d 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -12849,7 +12849,9 @@ void PrimaryLogPG::on_removal(ObjectStore::Transaction &t)
 
   on_shutdown();
 
-  t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch()));
+  // starting PG deletion, num_objects can be 1
+  // do_delete_work will update num_objects
+  t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch(), 1));
 }
 
 void PrimaryLogPG::clear_async_reads()

From 9d6b463a6442ed5ea3ada5f0926214173f4b7576 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Tue, 23 Jan 2024 10:25:44 -0500
Subject: [PATCH 1621/2492] rgw/lc: decorating log events with more details

  * some minor typos in the log event strings
  * correcting the names of the owning functions in some of the log events
  * adding worker index to the events in LCWorker::entry()
  * adding worker index to the cycle-finished events
  * adding bucket name to the interval budget expired events
  * adding bucket name to the events found in RGWLC::bucket_lc_process()
  * adding event to capture the end and the return code for the call to
    bucket_lc_process()

Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_lc.cc | 55 ++++++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index c25e4ffc648b..0bf95bdff9f8 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -219,13 +219,13 @@ void *RGWLC::LCWorker::entry() {
     std::unique_ptr<rgw::sal::Bucket> all_buckets; // empty restriction
     utime_t start = ceph_clock_now();
     if (should_work(start)) {
-      ldpp_dout(dpp, 2) << "life cycle: start" << dendl;
+      ldpp_dout(dpp, 2) << "life cycle: start worker=" << ix << dendl;
       int r = lc->process(this, all_buckets, false /* once */);
       if (r < 0) {
         ldpp_dout(dpp, 0) << "ERROR: do life cycle process() returned error r="
-			  << r << dendl;
+			  << r << " worker=" << ix << dendl;
       }
-      ldpp_dout(dpp, 2) << "life cycle: stop" << dendl;
+      ldpp_dout(dpp, 2) << "life cycle: stop worker=" << ix << dendl;
       cloud_targets.clear(); // clear cloud targets
     }
     if (lc->going_down())
@@ -236,8 +236,8 @@ void *RGWLC::LCWorker::entry() {
     utime_t next;
     next.set_from_double(end + secs);
 
-    ldpp_dout(dpp, 5) << "schedule life cycle next start time: "
-		      << rgw_to_asctime(next) << dendl;
+    ldpp_dout(dpp, 5) << "schedule life cycle next start time="
+		      << rgw_to_asctime(next) << " worker=" << ix << dendl;
 
     std::unique_lock l{lock};
     cond.wait_for(l, std::chrono::seconds(secs));
@@ -924,8 +924,8 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
        ++prefix_iter) {
 
     if (worker_should_stop(stop_at, once)) {
-      ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker "
-		     << worker->ix
+      ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker="
+		     << worker->ix << " bucket=" << target->get_name()
 		     << dendl;
       return 0;
     }
@@ -956,8 +956,8 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 
       if ((offset % 100) == 0) {
 	if (worker_should_stop(stop_at, once)) {
-	  ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker "
-			     << worker->ix
+	  ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker="
+			     << worker->ix << " bucket=" << target->get_name()
 			     << dendl;
 	  return 0;
 	}
@@ -1643,7 +1643,7 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
   string bucket_name = result[1];
   string bucket_marker = result[2];
 
-  ldpp_dout(this, 5) << "RGWLC::bucket_lc_process ENTER " << bucket_name << dendl;
+  ldpp_dout(this, 5) << "RGWLC::bucket_lc_process ENTER bucket=" << bucket_name << dendl;
   if (unlikely(cct->_conf->rgwlc_skip_bucket_step)) {
     return 0;
   }
@@ -1684,7 +1684,7 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
   try {
       config.decode(iter);
     } catch (const buffer::error& e) {
-      ldpp_dout(this, 0) << __func__ <<  "() decode life cycle config failed"
+      ldpp_dout(this, 0) << __func__ <<  "() decode life cycle config failed bucket=" << bucket_name
 			 << dendl;
       return -1;
     }
@@ -1692,7 +1692,7 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
   /* fetch information for zone checks */
   rgw::sal::Zone* zone = driver->get_zone();
 
-  auto pf = [](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) {
+  auto pf = [&bucket_name](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) {
     auto wt =
       boost::get<std::tuple<LCOpRule, rgw_bucket_dir_entry>>(wi);
     auto& [op_rule, o] = wt;
@@ -1704,7 +1704,8 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
     if (ret < 0) {
       ldpp_dout(wk->get_lc(), 20)
 	<< "ERROR: orule.process() returned ret=" << ret
-	<< "thread:" << wq->thr_name()
+	<< " thread=" << wq->thr_name()
+	<< " bucket=" << bucket_name
 	<< dendl;
     }
   };
@@ -1721,8 +1722,8 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
       ++prefix_iter) {
 
     if (worker_should_stop(stop_at, once)) {
-      ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker "
-		     << worker->ix
+      ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker="
+		     << worker->ix << " bucket=" << bucket_name
 		     << dendl;
       return 0;
     }
@@ -1768,8 +1769,8 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
       worker->workpool->enqueue(WorkItem{t1});
       if ((offset % 100) == 0) {
 	if (worker_should_stop(stop_at, once)) {
-	  ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker "
-			     << worker->ix
+	  ldpp_dout(this, 5) << __func__ << " interval budget EXPIRED worker="
+			     << worker->ix << " bucket=" << bucket_name
 			     << dendl;
 	  return 0;
 	}
@@ -1861,7 +1862,7 @@ int RGWLC::bucket_lc_post(int index, int max_lock_sec,
 
     ret = sal_lc->set_entry(obj_names[index],  entry);
     if (ret < 0) {
-      ldpp_dout(this, 0) << "RGWLC::process() failed to set entry on "
+      ldpp_dout(this, 0) << "RGWLC::bucket_lc_post() failed to set entry on "
           << obj_names[index] << dendl;
     }
 clean:
@@ -2024,7 +2025,7 @@ int RGWLC::process_bucket(int index, int max_lock_secs, LCWorker* worker,
   ret = serializer->try_lock(this, time, null_yield);
   if (ret == -EBUSY || ret == -EEXIST) {
     /* already locked by another lc processor */
-    ldpp_dout(this, 0) << "RGWLC::process() failed to acquire lock on "
+    ldpp_dout(this, 0) << "RGWLC::process_bucket() failed to acquire lock on "
 		       << obj_names[index] << dendl;
     return -EBUSY;
   }
@@ -2077,6 +2078,8 @@ int RGWLC::process_bucket(int index, int max_lock_secs, LCWorker* worker,
 
   lock.unlock();
   ret = bucket_lc_process(entry->get_bucket(), worker, thread_stop_at(), once);
+  ldpp_dout(this, 5) << "RGWLC::process_bucket(): END entry 2: " << entry
+    << " index: " << index << " worker ix: " << worker->ix << " ret: " << ret << dendl;
   bucket_lc_post(index, max_lock_secs, *entry, ret, worker);
 
   return ret;
@@ -2266,7 +2269,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
 	  if (head->get_marker().empty()) {
 	    ldpp_dout(this, 5) <<
 	      "RGWLC::process() cycle finished lc_shard="
-			       << lc_shard
+			       << lc_shard << " worker=" << worker->ix
 			       << dendl;
 	    head->set_shard_rollover_date(ceph_clock_now());
 	    ret = sal_lc->put_head(lc_shard, *head.get());
@@ -2286,14 +2289,14 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
 	  if (advance_head(lc_shard, *head.get(), *entry.get(), now) < 0) {
 	    goto exit;
 	  }
-	  ldpp_dout(this, 5) << "RGWLC::process() worker ix; " << worker->ix
+	  ldpp_dout(this, 5) << "RGWLC::process() worker ix: " << worker->ix
 			     << " SKIP processing for already-processed bucket " << entry->get_bucket()
 			     << dendl;
 	  /* done with this shard */
 	  if (head->get_marker().empty()) {
 	    ldpp_dout(this, 5) <<
 	      "RGWLC::process() cycle finished lc_shard="
-			       << lc_shard
+			       << lc_shard << " worker=" << worker->ix
 			       << dendl;
 	    head->set_shard_rollover_date(ceph_clock_now());
 	    ret = sal_lc->put_head(lc_shard, *head.get());
@@ -2346,6 +2349,8 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
      * bucket is being processed */
     lock->unlock();
     ret = bucket_lc_process(entry->get_bucket(), worker, thread_stop_at(), once);
+    ldpp_dout(this, 5) << "RGWLC::process(): END entry 2: " << entry
+      << " index: " << index << " worker ix: " << worker->ix << " ret: " << ret << dendl;
 
     /* postamble */
     //bucket_lc_post(index, max_lock_secs, entry, ret, worker);
@@ -2375,8 +2380,8 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
       }
       ret = sal_lc->set_entry(lc_shard, *entry);
       if (ret < 0) {
-        ldpp_dout(this, 0) << "RGWLC::process() failed to set entry on "
-                           << lc_shard
+        ldpp_dout(this, 0) << "RGWLC::process() failed to set entry on lc_shard="
+                           << lc_shard << " entry=" << entry
                            << dendl;
         /* fatal, locked */
         goto exit;
@@ -2387,7 +2392,7 @@ int RGWLC::process(int index, int max_lock_secs, LCWorker* worker,
     if (head->get_marker().empty()) {
       ldpp_dout(this, 5) <<
 	"RGWLC::process() cycle finished lc_shard="
-			 << lc_shard
+			 << lc_shard << " worker=" << worker->ix
 			 << dendl;
       head->set_shard_rollover_date(ceph_clock_now());
       ret = sal_lc->put_head(lc_shard,  *head.get());

From d8df6f61e817a34c2c3282224cff117ae43e3f98 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 25 Jan 2024 05:13:00 +1000
Subject: [PATCH 1622/2492] doc/radosgw: edit "Usage" admin.rst

Edit "Usage" in doc/radosgw/admin.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 53 +++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 8e846874a5f8..5717e367018a 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -827,53 +827,66 @@ all unauthenticated users:
 Usage
 =====
 
-The Ceph Object Gateway logs usage for each user. You can track
-user usage within date ranges too.
+The Ceph Object Gateway logs the usage of each user. You can track the usage of
+each user within a specified date range.
+
+- Add ``rgw_enable_usage_log = true`` in the ``[client.rgw]`` section of
+  ``ceph.conf`` and restart the ``radosgw`` service. 
+
+  .. note:: Until Ceph has a linkable macro that handles all the many ways that options can be set, we advise that you set ``rgw_enable_usage_log = true`` in central config or in ``ceph.conf`` and restart all RGWs.
 
-- Add ``rgw_enable_usage_log = true`` in [client.rgw] section of ceph.conf and restart the radosgw service. 
 
 Options include: 
 
 - **Start Date:** The ``--start-date`` option allows you to filter usage
-  stats from a particular start date and an optional start time
+  stats from a specified start date and an optional start time
   (**format:** ``yyyy-mm-dd [HH:MM:SS]``).
 
 - **End Date:** The ``--end-date`` option allows you to filter usage up
-  to a particular date and an optional end time
+  to a particular end date and an optional end time
   (**format:** ``yyyy-mm-dd [HH:MM:SS]``). 
   
 - **Log Entries:** The ``--show-log-entries`` option allows you to specify
-  whether or not to include log entries with the usage stats 
+  whether to include log entries with the usage stats 
   (options: ``true`` | ``false``).
 
-.. note:: You may specify time with minutes and seconds, but it is stored 
-   with 1 hour resolution.
+.. note:: You can specify time to a precision of minutes and seconds, but the
+   specified time is stored only with a one-hour resolution.
 
 
 Show Usage
 ----------
 
-To show usage statistics, specify the ``usage show``. To show usage for a
-particular user, you must specify a user ID. You may also specify a start date,
-end date, and whether or not to show log entries.::
+To show usage statistics, use the ``radosgw-admin usage show`` command. To show
+usage for a particular user, you must specify a user ID. You can also specify a
+start date, end date, and whether to show log entries. The following is an example
+of such a command:
+
+.. prompt:: bash $
 
-	radosgw-admin usage show --uid=johndoe --start-date=2012-03-01 --end-date=2012-04-01
+   radosgw-admin usage show --uid=johndoe --start-date=2012-03-01 --end-date=2012-04-01
 
-You may also show a summary of usage information for all users by omitting a user ID. ::
+You can show a summary of usage information for all users by omitting the user
+ID, as in the following example command:
 
-	radosgw-admin usage show --show-log-entries=false
+.. prompt:: bash $
+
+   radosgw-admin usage show --show-log-entries=false
 
 
 Trim Usage
 ----------
 
-With heavy use, usage logs can begin to take up storage space. You can trim
-usage logs for all users and for specific users. You may also specify date
-ranges for trim operations. ::
+Usage logs can consume significant storage space, especially over time and with
+heavy use. You can trim the usage logs for all users and for specific users.
+You can also specify date ranges for trim operations, as in the following
+example commands:
+
+.. prompt:: bash $
 
-	radosgw-admin usage trim --start-date=2010-01-01 --end-date=2010-12-31
-	radosgw-admin usage trim --uid=johndoe	
-	radosgw-admin usage trim --uid=johndoe --end-date=2013-12-31
+   radosgw-admin usage trim --start-date=2010-01-01 --end-date=2010-12-31
+   radosgw-admin usage trim --uid=johndoe	
+   radosgw-admin usage trim --uid=johndoe --end-date=2013-12-31
 
 
 .. _radosgw-admin: ../../man/8/radosgw-admin/

From 5ae5925a929cf831c7c02372c128fda920e3c788 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 23:57:13 +0000
Subject: [PATCH 1623/2492] .github: Bump gregsdennis/dependencies-action from
 1.2.3 to 1.3.2

Bumps [gregsdennis/dependencies-action](https://github.com/gregsdennis/dependencies-action) from 1.2.3 to 1.3.2.
- [Release notes](https://github.com/gregsdennis/dependencies-action/releases)
- [Commits](https://github.com/gregsdennis/dependencies-action/compare/80b5ffec566913b1494d5a8577ab0d60e476271d...f98d55eee1f66e7aaea4a60e71892736ae2548c7)

---
updated-dependencies:
- dependency-name: gregsdennis/dependencies-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/pr-check-deps.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-check-deps.yml b/.github/workflows/pr-check-deps.yml
index 7815b8fe486f..a7258d187f13 100644
--- a/.github/workflows/pr-check-deps.yml
+++ b/.github/workflows/pr-check-deps.yml
@@ -5,6 +5,6 @@ jobs:
     runs-on: ubuntu-latest
     name: Check PR Dependencies
     steps:
-    - uses: gregsdennis/dependencies-action@80b5ffec566913b1494d5a8577ab0d60e476271d
+    - uses: gregsdennis/dependencies-action@f98d55eee1f66e7aaea4a60e71892736ae2548c7
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From e03f8a8c163725ef6b3df68aa146e326f68e2476 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 25 Jan 2024 23:57:22 +0000
Subject: [PATCH 1624/2492] .github: Bump actions/labeler from 4.0.2 to 5.0.0

Bumps [actions/labeler](https://github.com/actions/labeler) from 4.0.2 to 5.0.0.
- [Release notes](https://github.com/actions/labeler/releases)
- [Commits](https://github.com/actions/labeler/compare/5c7539237e04b714afd8ad9b4aed733815b9fab4...8558fd74291d67161a8a78ce36a881fa63b766a9)

---
updated-dependencies:
- dependency-name: actions/labeler
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/pr-triage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml
index 24b1c97db708..e94e90146213 100644
--- a/.github/workflows/pr-triage.yml
+++ b/.github/workflows/pr-triage.yml
@@ -9,7 +9,7 @@ jobs:
     steps:
       - name: Assign labels based on modified files
         # https://github.com/marketplace/actions/labeler?version=v4.0.2
-        uses: actions/labeler@5c7539237e04b714afd8ad9b4aed733815b9fab4
+        uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9
         with:
           sync-labels: ''
           repo-token: "${{ secrets.GITHUB_TOKEN }}"

From 4bdc5d18dd68b95c6ccd4c0e77a1bd04ad86dbb8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 26 Jan 2024 09:53:30 -0500
Subject: [PATCH 1625/2492] rgw/rest: fix url decode of post params for
 iam/sts/sns

add the `in_query=true` argument to `url_decode()` to replace '+' with ' '

Fixes: https://tracker.ceph.com/issues/64189

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_rest_pubsub.cc      | 4 ++--
 src/rgw/rgw_rest_s3.cc          | 3 ++-
 src/rgw/rgw_rest_user_policy.cc | 6 +++---
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index f43a9ef27228..611589d721f3 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -152,7 +152,7 @@ class RGWPSCreateTopicOp : public RGWOp {
       return -EINVAL;
     }
     // Store topic Policy.
-    policy_text = url_decode(s->info.args.get("Policy"), true);
+    policy_text = s->info.args.get("Policy");
     if (!policy_text.empty() && !get_policy_from_text(s, policy_text)) {
       return -ERR_MALFORMED_DOC;
     }
@@ -562,7 +562,7 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
         return -EINVAL;
       }
     } else if (attribute_name == "Policy") {
-      policy_text = url_decode(s->info.args.get("AttributeValue"), true);
+      policy_text = s->info.args.get("AttributeValue");
       if (!policy_text.empty() && !get_policy_from_text(s, policy_text)) {
         return -ERR_MALFORMED_DOC;
       }
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 9791cab8a71e..b3d3891b0ea9 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5198,8 +5198,9 @@ void parse_post_action(const std::string& post_body, req_state* s)
           if (boost::starts_with(key, "Attributes.")) {
             update_attribute_map(t, map);
           } else {
+            constexpr bool in_query = true; // replace '+' with ' '
             s->info.args.append(t.substr(0, pos),
-                              url_decode(t.substr(pos+1, t.size() -1)));
+                              url_decode(t.substr(pos+1, t.size() -1), in_query));
           }
         }
       }
diff --git a/src/rgw/rgw_rest_user_policy.cc b/src/rgw/rgw_rest_user_policy.cc
index ddca86a95d86..4103e4aff772 100644
--- a/src/rgw/rgw_rest_user_policy.cc
+++ b/src/rgw/rgw_rest_user_policy.cc
@@ -92,9 +92,9 @@ uint64_t RGWPutUserPolicy::get_op()
 
 int RGWPutUserPolicy::get_params()
 {
-  policy_name = url_decode(s->info.args.get("PolicyName"), true);
-  user_name = url_decode(s->info.args.get("UserName"), true);
-  policy = url_decode(s->info.args.get("PolicyDocument"), true);
+  policy_name = s->info.args.get("PolicyName");
+  user_name = s->info.args.get("UserName");
+  policy = s->info.args.get("PolicyDocument");
 
   if (policy_name.empty() || user_name.empty() || policy.empty()) {
     ldpp_dout(this, 20) << "ERROR: one of policy name, user name or policy document is empty"

From b5ad8cb3250093639f1ac29ec69697961853570f Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Fri, 26 Jan 2024 09:59:18 -0600
Subject: [PATCH 1626/2492] .github/workflows: update comment to reflect
 version change

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 .github/workflows/pr-triage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml
index e94e90146213..81c23128ae28 100644
--- a/.github/workflows/pr-triage.yml
+++ b/.github/workflows/pr-triage.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Assign labels based on modified files
-        # https://github.com/marketplace/actions/labeler?version=v4.0.2
+        # https://github.com/marketplace/actions/labeler?version=v5.0.0
         uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9
         with:
           sync-labels: ''

From 11a37da05305420ca37f5a3af5b8991e1e2967cb Mon Sep 17 00:00:00 2001
From: Alexander Indenbaum <aindenba@redhat.com>
Date: Fri, 26 Jan 2024 19:50:19 +0000
Subject: [PATCH 1627/2492] build dependencies: centos9

- ceph.spec.in: declare git as build dependency
- install-deps.sh: enable CRB repo

Test procedure:
    docker run --rm -ti  -v /home/baum/ceph-ci:/home/ceph quay.io/centos/centos:stream9 bash
    [root@a3c4b1545e93 /]# cd /home/ceph/
    [root@a3c4b1545e93 ceph]# ./install-deps.sh 2>&1 tee install-deps.log

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 ceph.spec.in    | 1 +
 install-deps.sh | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 058c79c5e7f8..c0dbe41b7263 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -213,6 +213,7 @@ BuildRequires:	selinux-policy-devel
 BuildRequires:	gperf
 BuildRequires:  cmake > 3.5
 BuildRequires:	fuse-devel
+BuildRequires:	git
 %if 0%{?fedora} || 0%{?suse_version} > 1500 || 0%{?rhel} == 9 || 0%{?openEuler}
 BuildRequires:	gcc-c++ >= 11
 %endif
diff --git a/install-deps.sh b/install-deps.sh
index a334a3b29a30..257c43111e6c 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -545,6 +545,8 @@ else
                     $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
                     $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
                     $SUDO dnf -y module enable javapackages-tools
+                elif test $ID = centos -a $MAJOR_VERSION = 9 ; then
+                    $SUDO dnf config-manager --set-enabled crb
                 elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then
                     dts_ver=11
                     $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-8-${ARCH}-rpms"

From a5ce9c3863f98992193a9913344a046112c28dea Mon Sep 17 00:00:00 2001
From: Samuel Just <rexludorum@gmail.com>
Date: Fri, 26 Jan 2024 12:23:03 -0800
Subject: [PATCH 1628/2492] Revert "crush: add multistep retry rules"

This PR was merged by accident before it was ready.
Let's revert for now and open a new PR.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 doc/rados/operations/crush-map-edits.rst      |   30 +-
 doc/rados/operations/crush-map.rst            |   22 -
 .../ec-rados-plugin=jerasure-k=4-m=2.yaml     |    4 +-
 .../dashboard/test_erasure_code_profile.py    |    2 +-
 src/crush/CrushCompiler.cc                    |   68 +-
 src/crush/CrushWrapper.cc                     |  190 +--
 src/crush/CrushWrapper.h                      |  113 +-
 src/crush/crush.h                             |   23 +-
 src/crush/grammar.h                           |   16 +-
 src/crush/mapper.c                            | 1070 +----------------
 src/crush/mapper.h                            |   14 +-
 src/erasure-code/ErasureCode.cc               |   46 +-
 src/erasure-code/ErasureCode.h                |    2 -
 src/include/ceph_features.h                   |    6 +-
 src/mon/OSDMonitor.cc                         |   12 +-
 src/osd/OSDMap.cc                             |   12 +-
 src/test/cli/crushtool/choose-args.t          |    5 +-
 src/test/cli/osdmaptool/crush.t               |    2 +-
 src/test/crush/crush.cc                       | 1044 ++--------------
 src/vstart.sh                                 |   34 -
 20 files changed, 192 insertions(+), 2523 deletions(-)

diff --git a/doc/rados/operations/crush-map-edits.rst b/doc/rados/operations/crush-map-edits.rst
index 22e7e2f3772f..46a4a4f74e87 100644
--- a/doc/rados/operations/crush-map-edits.rst
+++ b/doc/rados/operations/crush-map-edits.rst
@@ -419,7 +419,7 @@ centers for three-way replication, and yet another rule for erasure coding acros
 six storage devices. For a detailed discussion of CRUSH rules, see **Section 3.2**
 of `CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_.
 
-A normal CRUSH rule takes the following form::
+A rule takes the following form::
 
     rule <rulename> {
 
@@ -430,18 +430,6 @@ A normal CRUSH rule takes the following form::
         step emit
     }
 
-CRUSH MSR rules are a distinct type of CRUSH rule which supports retrying steps
-and provides better support for configurations that require multiple OSDs within
-each failure domain.  MSR rules take the following form::
-
-    rule <rulename> {
-
-        id [a unique integer ID]
-        type [msr_indep|msr_firsn]
-        step take <bucket-name> [class <device-class>]
-        step choosemsr <N> type <bucket-type>
-        step emit
-    }
 
 ``id``
    :Description: A unique integer that identifies the rule.
@@ -453,14 +441,12 @@ each failure domain.  MSR rules take the following form::
 
 ``type``
    :Description: Denotes the type of replication strategy to be enforced by the
-                 rule.  msr_firstn and msr_indep are a distinct descent algorithm
-		 which supports retrying steps within the rule and therefore
-		 multiple OSDs per failure domain.
+                 rule.
    :Purpose: A component of the rule mask.
    :Type: String
    :Required: Yes
    :Default: ``replicated``
-   :Valid Values: ``replicated``, ``erasure``, ``msr_firstn``, ``msr_indep``
+   :Valid Values: ``replicated`` or ``erasure``
 
 
 ``step take <bucket-name> [class <device-class>]``
@@ -539,16 +525,6 @@ each failure domain.  MSR rules take the following form::
                  final CRUSH mapping transformation is therefore 1, 2, 3, 4, 5
                  → 1, 2, 6, 4, 5.
 
-``step choosemsr {num} type {bucket-type}``
-   :Description: Selects a num buckets of type bucket-type.  msr_firstn and msr_indep
-		 must use choosemsr rather than choose or chooseleaf.
-
-                 - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (as many buckets as are available).
-                 - If ``pool-num-replicas > {num} > 0``, choose that many buckets.
-   :Purpose: Choose step required for msr_firstn and msr_indep rules.
-   :Prerequisite: Follows ``step take`` and precedes ``step emit``
-   :Example: ``step choosemsr 3 type host``
-
 .. _crush-reclassify:
 
 Migrating from a legacy SSD rule to device classes
diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst
index e18d593253d9..39151e6d4a76 100644
--- a/doc/rados/operations/crush-map.rst
+++ b/doc/rados/operations/crush-map.rst
@@ -709,13 +709,6 @@ The relevant erasure-code profile properties are as follows:
    [default: ``default``].
  * **crush-failure-domain**: the CRUSH bucket type used in the distribution of
    erasure-coded shards [default: ``host``].
- * **crush-osds-per-failure-domain**: Maximum number of OSDs to place in each
-   failure domain -- defaults to 1.  Using a value greater than one will
-   cause a CRUSH MSR rule to be created, see below.  Must be specified if
-   crush-num-failure-domains is specified.
- * **crush-num-failure-domains**: Number of failure domains to map.  Must be
-   specified if crush-osds-per-failure-domain is specified.  Results in
-   a CRUSH MSR rule being created.
  * **crush-device-class**: the device class on which to place data [default:
    none, which means that all devices are used].
  * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the
@@ -733,21 +726,6 @@ The relevant erasure-code profile properties are as follows:
    argument is omitted, then Ceph will create the CRUSH rule automatically.
 
 
-CRUSH MSR Rules
----------------
-
-Creating an erasure-code profile with a crush-osds-per-failure-domain
-value greater than one will cause a CRUSH MSR rule type to be created
-instead of a normal CRUSH rule.  Normal crush rules cannot retry prior
-steps when an out OSD is encountered and rely on CHOOSELEAF steps to
-permit moving OSDs to new hosts.  However, CHOOSELEAF rules don't
-support more than a single OSD per failure domain.  MSR rules, new in
-squid, support multiple OSDs per failure domain by retrying all prior
-steps when an out OSD is encountered.  Using MSR rules requires that
-OSDs and clients be required to support the CRUSH_MSR feature bit
-(squid or newer).
-
-
 Deleting rules
 --------------
 
diff --git a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
index a0cd68a55f53..dfcc61607a7d 100644
--- a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
+++ b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
@@ -11,9 +11,7 @@ tasks:
       k: 4
       m: 2
       technique: reed_sol_van
-      crush-failure-domain: host
-      crush-osds-per-failure-domain: 2
-      crush-num-failure-domains: 3
+      crush-failure-domain: osd
     op_weights:
       read: 100
       write: 0
diff --git a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
index a50914008934..7fb7c1c8270f 100644
--- a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
+++ b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
@@ -79,7 +79,7 @@ def test_create_plugin(self):
         self.assertStatus(201)
 
         self._get('/api/erasure_code_profile/lrc')
-        self.assertJsonSubset({
+        self.assertJsonBody({
             'crush-device-class': '',
             'crush-failure-domain': 'host',
             'crush-root': 'default',
diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index c884caed00e6..5e51aad8dba4 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -321,13 +321,6 @@ int CrushCompiler::decompile(ostream &out)
   if (crush.get_allowed_bucket_algs() != CRUSH_LEGACY_ALLOWED_BUCKET_ALGS)
     out << "tunable allowed_bucket_algs " << crush.get_allowed_bucket_algs()
 	<< "\n";
-  if (crush.has_nondefault_tunables_msr()) {
-    out << "tunable msr_descents " << crush.get_msr_descents()
-	<< "\n";
-    out << "tunable msr_collision_tries "
-	<< crush.get_msr_collision_tries()
-	<< "\n";
-  }
 
   out << "\n# devices\n";
   for (int i=0; i<crush.get_max_devices(); i++) {
@@ -370,18 +363,12 @@ int CrushCompiler::decompile(ostream &out)
     out << "\tid " << i << "\n";
 
     switch (crush.get_rule_type(i)) {
-    case CRUSH_RULE_TYPE_REPLICATED:
+    case CEPH_PG_TYPE_REPLICATED:
       out << "\ttype replicated\n";
       break;
-    case CRUSH_RULE_TYPE_ERASURE:
+    case CEPH_PG_TYPE_ERASURE:
       out << "\ttype erasure\n";
       break;
-    case CRUSH_RULE_TYPE_MSR_FIRSTN:
-      out << "\ttype msr_firstn\n";
-      break;
-    case CRUSH_RULE_TYPE_MSR_INDEP:
-      out << "\ttype msr_indep\n";
-      break;
     default:
       out << "\ttype " << crush.get_rule_type(i) << "\n";
     }
@@ -435,15 +422,6 @@ int CrushCompiler::decompile(ostream &out)
 	out << "\tstep set_chooseleaf_stable " << crush.get_rule_arg1(i, j)
 	    << "\n";
 	break;
-      case CRUSH_RULE_SET_MSR_DESCENTS:
-	out << "\tstep set_msr_descents " << crush.get_rule_arg1(i, j)
-	    << "\n";
-	break;
-      case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
-	out << "\tstep set_msr_collision_tries "
-	    << crush.get_rule_arg1(i, j)
-	    << "\n";
-	break;
       case CRUSH_RULE_CHOOSE_FIRSTN:
 	out << "\tstep choose firstn "
 	    << crush.get_rule_arg1(i, j) 
@@ -472,13 +450,6 @@ int CrushCompiler::decompile(ostream &out)
 	print_type_name(out, crush.get_rule_arg2(i, j), crush);
 	out << "\n";
 	break;
-      case CRUSH_RULE_CHOOSE_MSR:
-	out << "\tstep choosemsr "
-	    << crush.get_rule_arg1(i, j) 
-	    << " type ";
-	print_type_name(out, crush.get_rule_arg2(i, j), crush);
-	out << "\n";
-	break;
       }
     }
     out << "}\n";
@@ -561,10 +532,6 @@ int CrushCompiler::parse_tunable(iter_t const& i)
     crush.set_straw_calc_version(val);
   else if (name == "allowed_bucket_algs")
     crush.set_allowed_bucket_algs(val);
-  else if (name == "msr_descents")
-    crush.set_msr_descents(val);
-  else if (name == "msr_collision_tries")
-    crush.set_msr_collision_tries(val);
   else {
     err << "tunable " << name << " not recognized" << std::endl;
     return -1;
@@ -814,13 +781,9 @@ int CrushCompiler::parse_rule(iter_t const& i)
   string tname = string_node(i->children[start+2]);
   int type;
   if (tname == "replicated")
-    type = CRUSH_RULE_TYPE_REPLICATED;
+    type = CEPH_PG_TYPE_REPLICATED;
   else if (tname == "erasure")
-    type = CRUSH_RULE_TYPE_ERASURE;
-  else if (tname == "msr_firstn")
-    type = CRUSH_RULE_TYPE_MSR_FIRSTN;
-  else if (tname == "msr_indep")
-    type = CRUSH_RULE_TYPE_MSR_INDEP;
+    type = CEPH_PG_TYPE_ERASURE;
   else 
     ceph_abort();
 
@@ -942,18 +905,6 @@ int CrushCompiler::parse_rule(iter_t const& i)
 	crush.set_rule_step_set_chooseleaf_stable(ruleno, step++, val);
       }
       break;
-    case crush_grammar::_step_set_msr_descents:
-      {
-	int val = int_node(s->children[1]);
-	crush.set_rule_step_set_msr_descents(ruleno, step++, val);
-      }
-      break;
-    case crush_grammar::_step_set_msr_collision_tries:
-      {
-	int val = int_node(s->children[1]);
-	crush.set_rule_step_set_msr_collision_tries(ruleno, step++, val);
-      }
-      break;
 
     case crush_grammar::_step_choose:
     case crush_grammar::_step_chooseleaf:
@@ -981,17 +932,6 @@ int CrushCompiler::parse_rule(iter_t const& i)
       }
       break;
 
-    case crush_grammar::_step_choose_msr:
-      {
-	string type = string_node(s->children[3]);
-	if (!type_id.count(type)) {
-	  err << "in rule '" << rname << "' type '" << type << "' not defined" << std::endl;
-	  return -1;
-	}
-	crush.set_rule_step_choose_msr(ruleno, step++, int_node(s->children[1]), type_id[type]);
-      }
-      break;
-
     case crush_grammar::_step_emit:
       crush.set_rule_step_emit(ruleno, step++);
       break;
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 4850e36f9b5c..0f40e6875e1b 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -135,29 +135,6 @@ bool CrushWrapper::is_v5_rule(unsigned ruleid) const
   return false;
 }
 
-bool CrushWrapper::has_msr_rules() const
-{
-  for (unsigned i=0; i<crush->max_rules; i++) {
-    if (is_msr_rule(i)) {
-      return true;
-    }
-  }
-  return false;
-}
-
-bool CrushWrapper::is_msr_rule(unsigned ruleid) const
-{
-  if (ruleid >= crush->max_rules)
-    return false;
-  
-  crush_rule *r = crush->rules[ruleid];
-  if (!r)
-    return false;
-
-  return r->type == CRUSH_RULE_TYPE_MSR_INDEP ||
-    r->type == CRUSH_RULE_TYPE_MSR_FIRSTN;
-}
-
 bool CrushWrapper::has_choose_args() const
 {
   return !choose_args.empty();
@@ -2261,7 +2238,6 @@ void CrushWrapper::reweight_bucket(
 int CrushWrapper::add_simple_rule_at(
   string name, string root_name,
   string failure_domain_name,
-  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   int rno,
@@ -2333,19 +2309,17 @@ int CrushWrapper::add_simple_rule_at(
   }
   crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
   if (type)
-    crush_rule_set_step(
-      rule, step++,
-      mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
-      CRUSH_RULE_CHOOSELEAF_INDEP,
-      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
-      type);
+    crush_rule_set_step(rule, step++,
+			mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
+			CRUSH_RULE_CHOOSELEAF_INDEP,
+			CRUSH_CHOOSE_N,
+			type);
   else
-    crush_rule_set_step(
-      rule, step++,
-      mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
-      CRUSH_RULE_CHOOSE_INDEP,
-      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
-      0);
+    crush_rule_set_step(rule, step++,
+			mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
+			CRUSH_RULE_CHOOSE_INDEP,
+			CRUSH_CHOOSE_N,
+			0);
   crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
 
   int ret = crush_add_rule(crush, rule, rno);
@@ -2361,125 +2335,13 @@ int CrushWrapper::add_simple_rule_at(
 int CrushWrapper::add_simple_rule(
   string name, string root_name,
   string failure_domain_name,
-  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   ostream *err)
 {
-  return add_simple_rule_at(
-    name, root_name, failure_domain_name, num_failure_domains,
-    device_class,
-    mode,
-    rule_type, -1, err);
-}
-
-int CrushWrapper::add_multi_osd_per_failure_domain_rule_at(
-  string name, string root_name, string failure_domain_name,
-  int num_failure_domains,
-  int osds_per_failure_domain,
-  string device_class,
-  crush_rule_type rule_type,
-  int rno,
-  ostream *err)
-{
-  if (rule_exists(name)) {
-    if (err)
-      *err << "rule " << name << " exists";
-    return -EEXIST;
-  }
-  if (rno >= 0) {
-    if (rule_exists(rno)) {
-      if (err)
-        *err << "rule with ruleno " << rno << " exists";
-      return -EEXIST;
-    }
-  } else {
-    for (rno = 0; rno < get_max_rules(); rno++) {
-      if (!rule_exists(rno))
-        break;
-    }
-  }
-  if (!name_exists(root_name)) {
-    if (err)
-      *err << "root item " << root_name << " does not exist";
-    return -ENOENT;
-  }
-  int root = get_item_id(root_name);
-  int type = 0;
-  if (failure_domain_name.length()) {
-    type = get_type_id(failure_domain_name);
-    if (type < 0) {
-      if (err)
-	*err << "unknown type " << failure_domain_name;
-      return -EINVAL;
-    }
-  }
-  if (device_class.size()) {
-    if (!class_exists(device_class)) {
-      if (err)
-	*err << "device class " << device_class << " does not exist";
-      return -EINVAL;
-    }
-    int c = get_class_id(device_class);
-    if (class_bucket.count(root) == 0 ||
-	class_bucket[root].count(c) == 0) {
-      if (err)
-	*err << "root " << root_name << " has no devices with class "
-	     << device_class;
-      return -EINVAL;
-    }
-    root = class_bucket[root][c];
-  }
-  if (rule_type != CRUSH_RULE_TYPE_MSR_INDEP &&
-      rule_type != CRUSH_RULE_TYPE_MSR_FIRSTN) {
-    if (err)
-      *err << "unknown rule_type " << rule_type;
-    return -EINVAL;
-  }
-
-  int steps = 4;
-  crush_rule *rule = crush_make_rule(steps, rule_type);
-  ceph_assert(rule);
-  int step = 0;
-  crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
-  crush_rule_set_step(rule, step++,
-		      CRUSH_RULE_CHOOSE_MSR,
-		      num_failure_domains,
-		      type);
-  crush_rule_set_step(rule, step++,
-		      CRUSH_RULE_CHOOSE_MSR,
-		      osds_per_failure_domain,
-		      0);
-  crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
-
-  int ret = crush_add_rule(crush, rule, rno);
-  if(ret < 0) {
-    *err << "failed to add rule " << rno << " because " << cpp_strerror(ret);
-    return ret;
-  }
-  set_rule_name(rno, name);
-  have_rmaps = false;
-  return rno;
-}
-
-
-int CrushWrapper::add_indep_multi_osd_per_failure_domain_rule(
-  string name, string root_name,
-  string failure_domain_name,
-  int num_failure_domains,
-  int osds_per_failure_domain,
-  string device_class,
-  ostream *err)
-{
-  return add_multi_osd_per_failure_domain_rule_at(
-    name, root_name,
-    failure_domain_name,
-    num_failure_domains,
-    osds_per_failure_domain,
-    device_class,
-    CRUSH_RULE_TYPE_MSR_INDEP,
-    -1,
-    err);
+  return add_simple_rule_at(name, root_name, failure_domain_name, device_class,
+			    mode,
+			    rule_type, -1, err);
 }
 
 float CrushWrapper::_get_take_weight_osd_map(int root,
@@ -3218,10 +3080,6 @@ void CrushWrapper::encode(bufferlist& bl, uint64_t features) const
       }
     }
   }
-  if (HAVE_FEATURE(features, CRUSH_MSR)) {
-    encode(crush->msr_descents, bl);
-    encode(crush->msr_collision_tries, bl);
-  }
 }
 
 static void decode_32_or_64_string_map(map<int32_t,string>& m, bufferlist::const_iterator& blp)
@@ -3372,12 +3230,6 @@ void CrushWrapper::decode(bufferlist::const_iterator& blp)
 	choose_args[choose_args_index] = arg_map;
       }
     }
-    if (!blp.end()) {
-      decode(crush->msr_descents, blp);
-      decode(crush->msr_collision_tries, blp);
-    } else {
-      set_default_msr_tunables();
-    }
     update_choose_args(nullptr); // in case we decode a legacy "corrupted" map
     finalize();
   }
@@ -3633,8 +3485,6 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("chooseleaf_descend_once", get_chooseleaf_descend_once());
   f->dump_int("chooseleaf_vary_r", get_chooseleaf_vary_r());
   f->dump_int("chooseleaf_stable", get_chooseleaf_stable());
-  f->dump_int("msr_descents", get_msr_descents());
-  f->dump_int("msr_collision_tries", get_msr_collision_tries());
   f->dump_int("straw_calc_version", get_straw_calc_version());
   f->dump_int("allowed_bucket_algs", get_allowed_bucket_algs());
 
@@ -3665,7 +3515,6 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("has_v4_buckets", (int)has_v4_buckets());
   f->dump_int("require_feature_tunables5", (int)has_nondefault_tunables5());
   f->dump_int("has_v5_rules", (int)has_v5_rules());
-  f->dump_int("has_msr_rules", (int)has_msr_rules());
 }
 
 void CrushWrapper::dump_choose_args(Formatter *f) const
@@ -3764,11 +3613,6 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_int("num", get_rule_arg1(rule_id, j));
       f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
       break;
-    case CRUSH_RULE_CHOOSE_MSR:
-      f->dump_string("op", "choosemsr");
-      f->dump_int("num", get_rule_arg1(rule_id, j));
-      f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
-      break;
     case CRUSH_RULE_SET_CHOOSE_TRIES:
       f->dump_string("op", "set_choose_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
@@ -3777,14 +3621,6 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_string("op", "set_chooseleaf_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
       break;
-    case CRUSH_RULE_SET_MSR_DESCENTS:
-      f->dump_string("op", "set_msr_descents");
-      f->dump_int("num", get_rule_arg1(rule_id, j));
-      break;
-    case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
-      f->dump_string("op", "set_msr_collision_tries");
-      f->dump_int("num", get_rule_arg1(rule_id, j));
-      break;
     default:
       f->dump_int("opcode", get_rule_op(rule_id, j));
       f->dump_int("arg1", get_rule_arg1(rule_id, j));
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 317f4c28bdd6..b8caa24ce621 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -125,7 +125,6 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
-    set_default_msr_tunables();
   }
   void set_tunables_bobtail() {
     crush->choose_local_tries = 0;
@@ -135,7 +134,6 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
-    set_default_msr_tunables();
   }
   void set_tunables_firefly() {
     crush->choose_local_tries = 0;
@@ -145,7 +143,6 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 1;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
-    set_default_msr_tunables();
   }
   void set_tunables_hammer() {
     crush->choose_local_tries = 0;
@@ -159,7 +156,6 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
-    set_default_msr_tunables();
   }
   void set_tunables_jewel() {
     crush->choose_local_tries = 0;
@@ -173,7 +169,6 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
-    set_default_msr_tunables();
   }
 
   void set_tunables_legacy() {
@@ -238,24 +233,6 @@ class CrushWrapper {
     crush->straw_calc_version = n;
   }
 
-  int get_msr_descents() const {
-    return crush->msr_descents;
-  }
-  void set_msr_descents(int n) {
-    crush->msr_descents = n;
-  }
-
-  int get_msr_collision_tries() const {
-    return crush->msr_collision_tries;
-  }
-  void set_msr_collision_tries(int n) {
-    crush->msr_collision_tries = n;
-  }
-  void set_default_msr_tunables() {
-    set_msr_descents(100);
-    set_msr_collision_tries(100);
-  }
-
   unsigned get_allowed_bucket_algs() const {
     return crush->allowed_bucket_algs;
   }
@@ -271,8 +248,7 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 0 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
-      !has_nondefault_tunables_msr();
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
   }
   bool has_bobtail_tunables() const {
     return
@@ -282,8 +258,7 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
-      !has_nondefault_tunables_msr();
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
   }
   bool has_firefly_tunables() const {
     return
@@ -293,8 +268,7 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 1 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
-      !has_nondefault_tunables_msr();
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
   }
   bool has_hammer_tunables() const {
     return
@@ -307,8 +281,7 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2)) &&
-      !has_nondefault_tunables_msr();
+				      (1 << CRUSH_BUCKET_STRAW2));
   }
   bool has_jewel_tunables() const {
     return
@@ -321,8 +294,7 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2)) &&
-      !has_nondefault_tunables_msr();
+				      (1 << CRUSH_BUCKET_STRAW2));
   }
 
   bool has_optimal_tunables() const {
@@ -350,11 +322,6 @@ class CrushWrapper {
     return
         crush->chooseleaf_stable != 0;
   }
-  bool has_nondefault_tunables_msr() const {
-    return
-      crush->msr_descents != 100 ||
-      crush->msr_collision_tries != 100;
-  }
 
   bool has_v2_rules() const;
   bool has_v3_rules() const;
@@ -362,17 +329,13 @@ class CrushWrapper {
   bool has_v5_rules() const;
   bool has_choose_args() const;          // any choose_args
   bool has_incompat_choose_args() const; // choose_args that can't be made compat
-  bool has_msr_rules() const;
 
   bool is_v2_rule(unsigned ruleid) const;
   bool is_v3_rule(unsigned ruleid) const;
   bool is_v5_rule(unsigned ruleid) const;
-  bool is_msr_rule(unsigned ruleid) const;
 
   std::string get_min_required_version() const {
-    if (has_msr_rules() || has_nondefault_tunables_msr())
-      return "squid";
-    else if (has_v5_rules() || has_nondefault_tunables5())
+    if (has_v5_rules() || has_nondefault_tunables5())
       return "jewel";
     else if (has_v4_buckets())
       return "hammer";
@@ -602,21 +565,6 @@ class CrushWrapper {
     if (have_rmaps)
       rule_name_rmap[name] = i;
   }
-  bool rule_valid_for_pool_type(int rule_id, int ptype) const {
-    auto rule_type = get_rule_type(rule_id);
-    switch (ptype) {
-    case CEPH_PG_TYPE_REPLICATED:
-      return rule_type == CRUSH_RULE_TYPE_REPLICATED ||
-	rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN;
-    case CEPH_PG_TYPE_ERASURE:
-      return rule_type == CRUSH_RULE_TYPE_ERASURE ||
-	rule_type == CRUSH_RULE_TYPE_MSR_INDEP;
-    default:
-      ceph_assert(0 == "impossible");
-      return false;
-    }
-  }
-
   bool is_shadow_item(int id) const {
     const char *name = get_item_name(id);
     return name && !is_valid_crush_name(name);
@@ -1203,14 +1151,6 @@ class CrushWrapper {
   int set_rule_step_set_chooseleaf_stable(unsigned ruleno, unsigned step, int val) {
     return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_STABLE, val, 0);
   }
-
-  int set_rule_step_set_msr_descents(unsigned ruleno, unsigned step, int val) {
-    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_DESCENTS, val, 0);
-  }
-  int set_rule_step_set_msr_collision_tries(unsigned ruleno, unsigned step, int val) {
-    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_COLLISION_TRIES, val, 0);
-  }
-
   int set_rule_step_choose_firstn(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_FIRSTN, val, type);
   }
@@ -1223,61 +1163,22 @@ class CrushWrapper {
   int set_rule_step_choose_leaf_indep(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSELEAF_INDEP, val, type);
   }
-  int set_rule_step_choose_msr(unsigned ruleno, unsigned step, int val, int type) {
-    return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_MSR, val, type);
-  }
   int set_rule_step_emit(unsigned ruleno, unsigned step) {
     return set_rule_step(ruleno, step, CRUSH_RULE_EMIT, 0, 0);
   }
 
   int add_simple_rule(
     std::string name, std::string root_name, std::string failure_domain_type,
-    int num_failure_domains,
     std::string device_class, std::string mode, int rule_type,
     std::ostream *err = 0);
-  int add_simple_rule(
-    std::string name, std::string root_name, std::string failure_domain_type,
-    std::string device_class, std::string mode, int rule_type,
-    std::ostream *err = 0) {
-    return add_simple_rule(
-      name, root_name, failure_domain_type, -1,
-      device_class, mode, rule_type, err);
-  }
-
-  int add_indep_multi_osd_per_failure_domain_rule(
-    std::string name, std::string root_name, std::string failure_domain_type,
-    int osds_per_failure_domain,
-    int num_failure_domains,
-    std::string device_class,
-    std::ostream *err = 0);
 
   /**
    * @param rno rule[set] id to use, -1 to pick the lowest available
    */
   int add_simple_rule_at(
     std::string name, std::string root_name,
-    std::string failure_domain_type,
-    int num_failure_domains,
-    std::string device_class, std::string mode,
+    std::string failure_domain_type, std::string device_class, std::string mode,
     int rule_type, int rno, std::ostream *err = 0);
-  int add_simple_rule_at(
-    std::string name, std::string root_name,
-    std::string failure_domain_type,
-    std::string device_class, std::string mode,
-    int rule_type, int rno, std::ostream *err = 0) {
-    return add_simple_rule_at(
-      name, root_name, failure_domain_type, -1,
-      device_class, mode, rule_type, rno, err);
-  }
-
-  int add_multi_osd_per_failure_domain_rule_at(
-    std::string name, std::string root_name, std::string failure_domain_type,
-    int osds_per_failure_domain,
-    int num_failure_domains,
-    std::string device_class,
-    crush_rule_type rule_type,
-    int rno,
-    std::ostream *err = 0);
 
   int remove_rule(int ruleno);
 
diff --git a/src/crush/crush.h b/src/crush/crush.h
index bdcdc97eef2b..fde2df6a8a3e 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -65,15 +65,7 @@ enum crush_opcodes {
 	CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES = 10,
 	CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES = 11,
 	CRUSH_RULE_SET_CHOOSELEAF_VARY_R = 12,
-	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13,
-
-	/* set choose_msr_total_tries */
-	CRUSH_RULE_SET_MSR_DESCENTS = 14,
-	/* set choose_msr_local_collision_tries */
-	CRUSH_RULE_SET_MSR_COLLISION_TRIES = 15,
-
-	/* choose variant without FIRSTN|INDEP */
-	CRUSH_RULE_CHOOSE_MSR = 16
+	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13
 };
 
 /*
@@ -95,12 +87,7 @@ struct crush_rule {
 #define crush_rule_size(len) (sizeof(struct crush_rule) + \
 			      (len)*sizeof(struct crush_rule_step))
 
-enum crush_rule_type {
-	CRUSH_RULE_TYPE_REPLICATED = 1,
-	CRUSH_RULE_TYPE_ERASURE = 3,
-	CRUSH_RULE_TYPE_MSR_FIRSTN = 4,
-	CRUSH_RULE_TYPE_MSR_INDEP = 5
-};
+
 
 /*
  * A bucket is a named container of other items (either devices or
@@ -423,12 +410,6 @@ struct crush_map {
          */
 	__u8 chooseleaf_stable;
 
-	/*! Sets total descents for MSR rules */
-	__u8 msr_descents;
-
-	/*! Sets local collision retries for MSR rules */
-	__u8 msr_collision_tries;
-
         /*! @cond INTERNAL */
 	/* This value is calculated after decode or construction by
 	   the builder. It is exposed here (rather than having a
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 0c9a2da7d770..b5c356a49e17 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -50,11 +50,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     _step_set_choose_tries,
     _step_set_choose_local_tries,
     _step_set_choose_local_fallback_tries,
-    _step_set_msr_descents,
-    _step_set_msr_collision_tries,
     _step_choose,
     _step_chooseleaf,
-    _step_choose_msr,
     _step_emit,
     _step,
     _crushrule,
@@ -94,11 +91,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_tries> >    step_set_chooseleaf_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_vary_r> >    step_set_chooseleaf_vary_r;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_stable> >    step_set_chooseleaf_stable;
-    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_descents> >    step_set_msr_descents;
-    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_collision_tries> >    step_set_msr_collision_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose> >    step_choose;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_chooseleaf> >      step_chooseleaf;
-    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose_msr> >      step_choose_msr;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_emit> >      step_emit;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step> >      step;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_crushrule> >      crushrule;
@@ -155,8 +149,6 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
       step_set_chooseleaf_tries = str_p("set_chooseleaf_tries") >> posint;
       step_set_chooseleaf_vary_r = str_p("set_chooseleaf_vary_r") >> posint;
       step_set_chooseleaf_stable = str_p("set_chooseleaf_stable") >> posint;
-      step_set_msr_descents = str_p("set_msr_descents") >> posint;
-      step_set_msr_collision_tries = str_p("set_msr_collision_tries") >> posint;
       step_choose = str_p("choose")
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
@@ -165,9 +157,6 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
 	>> str_p("type") >> name;
-      step_choose_msr = str_p("choosemsr")
-	>> integer
-	>> str_p("type") >> name;
       step_emit = str_p("emit");
       step = str_p("step") >> ( step_take |
 				step_set_choose_tries |
@@ -176,15 +165,12 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_set_chooseleaf_tries |
 				step_set_chooseleaf_vary_r |
 				step_set_chooseleaf_stable |
-				step_set_msr_descents |
-				step_set_msr_collision_tries |
 				step_choose |
 				step_chooseleaf |
-				step_choose_msr |
 				step_emit );
       crushrule = str_p("rule") >> !name >> '{'
 				>> (str_p("id") | str_p("ruleset")) >> posint
-				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") | str_p("msr_firstn") | str_p("msr_indep") )
+				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") )
 				>> !(str_p("min_size") >> posint)
 				>> !(str_p("max_size") >> posint)
 			   >> +step
diff --git a/src/crush/mapper.c b/src/crush/mapper.c
index afeaffc5a8d8..736cc6162c97 100644
--- a/src/crush/mapper.c
+++ b/src/crush/mapper.c
@@ -27,9 +27,6 @@
 
 #define dprintk(args...) /* printf(args) */
 
-#define MIN(x, y) ((x) > (y) ? (y) : (x))
-#define MAX(y, x) ((x) < (y) ? (y) : (x))
-
 /*
  * Implement the core CRUSH mapping algorithm.
  */
@@ -823,11 +820,65 @@ static void crush_choose_indep(const struct crush_map *map,
 #endif
 }
 
-static int crush_do_rule_no_retry(
-	const struct crush_map *map,
-	int ruleno, int x, int *result, int result_max,
-	const __u32 *weight, int weight_max,
-	void *cwin, const struct crush_choose_arg *choose_args)
+
+/* This takes a chunk of memory and sets it up to be a shiny new
+   working area for a CRUSH placement computation. It must be called
+   on any newly allocated memory before passing it in to
+   crush_do_rule. It may be used repeatedly after that, so long as the
+   map has not changed. If the map /has/ changed, you must make sure
+   the working size is no smaller than what was allocated and re-run
+   crush_init_workspace.
+
+   If you do retain the working space between calls to crush, make it
+   thread-local. If you reinstitute the locking I've spent so much
+   time getting rid of, I will be very unhappy with you. */
+
+void crush_init_workspace(const struct crush_map *m, void *v) {
+	/* We work by moving through the available space and setting
+	   values and pointers as we go.
+
+	   It's a bit like Forth's use of the 'allot' word since we
+	   set the pointer first and then reserve the space for it to
+	   point to by incrementing the point. */
+	struct crush_work *w = (struct crush_work *)v;
+	char *point = (char *)v;
+	__s32 b;
+	point += sizeof(struct crush_work);
+	w->work = (struct crush_work_bucket **)point;
+	point += m->max_buckets * sizeof(struct crush_work_bucket *);
+	for (b = 0; b < m->max_buckets; ++b) {
+		if (m->buckets[b] == 0)
+			continue;
+
+		w->work[b] = (struct crush_work_bucket *) point;
+		switch (m->buckets[b]->alg) {
+		default:
+			point += sizeof(struct crush_work_bucket);
+			break;
+		}
+		w->work[b]->perm_x = 0;
+		w->work[b]->perm_n = 0;
+		w->work[b]->perm = (__u32 *)point;
+		point += m->buckets[b]->size * sizeof(__u32);
+	}
+	BUG_ON((char *)point - (char *)w != m->working_size);
+}
+
+/**
+ * crush_do_rule - calculate a mapping with the given input and rule
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
+ */
+int crush_do_rule(const struct crush_map *map,
+		  int ruleno, int x, int *result, int result_max,
+		  const __u32 *weight, int weight_max,
+		  void *cwin, const struct crush_choose_arg *choose_args)
 {
 	int result_len;
 	struct crush_work *cw = cwin;
@@ -1030,1006 +1081,3 @@ static int crush_do_rule_no_retry(
 
 	return result_len;
 }
-
-/// invariant through crush_msr_do_rule invocation
-struct crush_msr_input {
-	const struct crush_map *map;
-	const struct crush_rule *rule;
-	
-	const unsigned result_max;
-	
-	const unsigned weight_len;
-	const __u32 *weights;
-	
-	const int map_input;
-	const struct crush_choose_arg *choose_args;
-	
-	const unsigned msr_descents;
-	const unsigned msr_collision_tries;
-};
-
-/// encapsulates work space, invariant within an EMIT block
-struct crush_msr_workspace {
-	const unsigned start_stepno;
-	const unsigned end_stepno;
-
-	const unsigned result_len;
-
-	const struct crush_work *crush_work;
-
-	// int[end_stepno - start_stepno][result_len]
-	int **step_vecs;
-};
-
-/// encapsulates output space, invariant through crush_msr_do_rule invocation
-struct crush_msr_output {
-	const unsigned result_len;
-	unsigned returned_so_far;
-	int *out;
-};
-
-/**
- * crush_msr_scan_config_steps
- *
- * Scans possibly empty sequence of CRUSH_RULE_SET_CHOOSE_MSR_*_TRIES
- * steps at the start of the rule.  Returns index of next step.
- * Populates *msr_descents and *msr_collision_tries (if non-null) with
- * last matching rule.
- * @steps: steps to scan
- * @step_len: length of steps
- * @msr_descents: out param for CRUSH_RULE_SET_MSR_DESCENTS
- * @msr_collision_tries: out param for CRUSH_RULE_SET_MSR_COLLISION_TRIES
- */
-static unsigned crush_msr_scan_config_steps(
-	const struct crush_rule_step *steps,
-	unsigned step_len,
-	unsigned *msr_descents,
-	unsigned *msr_collision_tries) {
-	unsigned stepno = 0;
-	for (; stepno < step_len; ++stepno) {
-		const struct crush_rule_step *step = &steps[stepno];
-		switch (step->op) {
-		case CRUSH_RULE_SET_MSR_DESCENTS:
-			if (msr_descents) *msr_descents = step->arg1;
-			break;
-		case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
-			if (msr_collision_tries) *msr_collision_tries = step->arg1;
-			break;
-		default:
-			return stepno;
-		}
-	}
-	return stepno;
-}
-
-/// clear workspace represented by *ws
-static void crush_msr_clear_workspace(
-	struct crush_msr_workspace *ws)
-{
-	for (unsigned stepno = ws->start_stepno; stepno < ws->end_stepno;
-	     ++stepno) {
-		for (unsigned i = 0; i < ws->result_len; ++i) {
-			ws->step_vecs[stepno - ws->start_stepno][i] =
-				CRUSH_ITEM_UNDEF;
-		}
-	}
-}
-
-/**
- * crush_msr_scan_next
- *
- * Validates an EMIT block of the form (TAKE CHOOSE_MSR* EMIT)
- * If sequence is valid, populates total_children with the width
- * of the mapping from the choose steps and next_emit with the
- * index of the next EMIT step.
- *
- * @rule: rule to scan
- * @result_max: max number of results to return
- * @max_steps: length of longest string of choosemsr steps
- * @return 0 if valid, -1 if there were validation errors
- */
-static int crush_msr_scan_next(
-	const struct crush_rule *rule,
-	unsigned result_max,
-	unsigned stepno,
-	unsigned *total_children,
-	unsigned *next_emit)
-{
-	if (stepno + 1 >= rule->len) {
-		dprintk("stepno too large\n");
-		return -1;
-	}
-	if (rule->steps[stepno].op != CRUSH_RULE_TAKE) {
-		dprintk("first rule not CRUSH_RULE_TAKE\n");
-		return -1;
-	}
-	++stepno;
-
-	if (total_children) *total_children = 1;
-	for (; stepno < rule->len; ++stepno) {
-		const struct crush_rule_step *curstep =
-			&(rule->steps[stepno]);
-		if (curstep->op == CRUSH_RULE_EMIT) {
-			break;
-		}
-		if (rule->steps[stepno].op != CRUSH_RULE_CHOOSE_MSR) {
-			dprintk("found non-choose non-emit step %d\n", stepno);
-			return -1;
-		}
-		if (total_children) {
-			*total_children *= curstep->arg1 ? curstep->arg1
-				: result_max;
-		}
-	}
-	if (stepno >= rule->len) {
-		dprintk("did not find emit\n");
-		return -1;
-	}
-	if (next_emit) {
-		*next_emit = stepno;
-	}
-	return 0;
-}
-
-/**
- * crush_msr_scan_rule
- *
- * MSR rules must have the form:
- * 1) Possibly empty sequence of CRUSH_RULE_SET_CHOOSE_MSR_.*_TRIES steps
- * 2) A sequence of EMIT blocks of the form
- *   (TAKE CHOOSE_MSR* EMIT)*
- *
- * crush_msr_scan_rule validates that the form obeys the above form and
- * popualtes max_steps with the length of the longest sequence of CHOOSE_MSR
- * steps.
- *
- * crush_msr_scan_rule replicates the scan behavior of crush_msr_do_rule.
- *
- * @rule: rule to scan
- * @result_max: max number of results to return
- * @max_steps: length of longest string of choosemsr steps
- * @return 0 if valid, -1 otherwise
- */
-static int crush_msr_scan_rule(
-	const struct crush_rule *rule,
-	unsigned result_max,
-	unsigned *max_steps)
-{
-	if (max_steps) *max_steps = 0;
-	unsigned next_stepno = crush_msr_scan_config_steps(
-		rule->steps,
-		rule->len,
-		NULL, NULL);
-	while (next_stepno < rule->len) {
-		unsigned next_emit_stepno;
-		int r = crush_msr_scan_next(
-			rule, result_max, next_stepno,
-			NULL, &next_emit_stepno);
-		if (r < 0) return r;
-
-		if (max_steps) {
-			*max_steps = MAX(
-				*max_steps,
-				next_emit_stepno - (next_stepno + 1));
-		}
-		next_stepno = next_emit_stepno + 1;
-	}
-	return 0;
-}
-
-/// Returns true if all leaf slots in [start, end) are mapped
-static int crush_msr_leaf_vec_populated(
-	const struct crush_msr_workspace *workspace,
-	const unsigned start, const unsigned end)
-{
-	BUG_ON(start >= end);
-	BUG_ON(end > workspace->result_len);
-	BUG_ON(workspace->end_stepno <= workspace->start_stepno);
-	// we check the last step vector here because output
-	// won't be ordered by index for FIRSTN rules
-	int *leaf_vec = workspace->step_vecs[
-	  workspace->end_stepno - workspace->start_stepno - 1];
-	for (unsigned i = start; i < end; ++i) {
-		if (leaf_vec[i] == CRUSH_ITEM_UNDEF) {
-			return 0;
-		}
-	}
-	return 1;
-}
-
-/// Returns try value to pass to crush based on index, tries, and local_tries
-static unsigned crush_msr_get_retry_value(
-	const unsigned result_max,
-	const unsigned index,
-	const unsigned msr_descents,
-	const unsigned msr_collision_tries)
-{
-	const unsigned total_index = (msr_descents * result_max) + index;
-	return (total_index << 16) + msr_collision_tries;
-}
-
-/**
- * crush_msr_descend
- *
- * Descend recursively from bucket until we either hit a leaf or an
- * interior node of type type.
- * @input: crush input information
- * @workspace: struct with working space
- * @bucket: bucket from which to descend
- * @type: target node type
- * @tryno: top level try number, incremented with each call into crush_msr_choose
- *         from crush_msr_do_rule
- * @local_tryno: local collision try number, incremented with each call into
- *               crush_msr_descend from crush_msr_choose after collision
- * @index: mapping index
- */
-static int crush_msr_descend(
-	const struct crush_msr_input *input,
-	const struct crush_msr_workspace *workspace,
-	const struct crush_bucket *bucket,
-	const int type,
-	const unsigned tryno,
-	const unsigned local_tryno,
-	const unsigned index)
-{
-	dprintk(" crush_msr_descend type %d tryno %d local_tryno %d index %d\n",
-		type, tryno, local_tryno, index);
-	while (1) {
-		const int child_bucket_candidate = crush_bucket_choose(
-			bucket,
-			workspace->crush_work->work[-1 - bucket->id],
-			input->map_input,
-			crush_msr_get_retry_value(
-				input->result_max,
-				index, tryno, local_tryno),
-			(input->choose_args ?
-			 &(input->choose_args[-1 - bucket->id]) : 0),
-			index);
-
-		if (child_bucket_candidate >= 0) {
-			return child_bucket_candidate;
-		}
-
-		bucket = input->map->buckets[-1 - child_bucket_candidate];
-		if (bucket->type == type) {
-			return child_bucket_candidate;
-		}
-	}
-}
-
-/**
- * crush_msr_valid_candidate  
- *
- * Checks whether candidate is a valid choice given buckets already
- * mapped for step stepno.
- * 
- * If candidate has already been mapped for a position in
- * [include_start, include_end), candidate is valid.
- *
- * Else, if candidate has already been mapped for a position in
- * [exclude_start, exclude_end), candidate is invalid.
- *
- * Otherwise, candidate is valid.
- *
- * @stepno: step to check
- * @exclude_start: start of exclusion range
- * @exclude_end: end of exlusion range
- * @include_start: start of inclusion range
- * @include_end: end of inclusion range
- * @candidate: bucket to check
- *
- * Note, [exclude_start, exclude_end) must contain [include_start, include_end).
- */
-static int crush_msr_valid_candidate(
-	const struct crush_msr_workspace *workspace,
-	unsigned stepno,
-	unsigned exclude_start,
-	unsigned exclude_end,
-	unsigned include_start,
-	unsigned include_end,
-	int candidate)
-{
-	BUG_ON(stepno >= workspace->end_stepno);
-	BUG_ON(stepno < workspace->start_stepno);
-
-	BUG_ON(exclude_end <= exclude_start);
-	BUG_ON(include_end <= include_start);
-
-	BUG_ON(exclude_start > include_start);
-	BUG_ON(exclude_end < include_end);
-
-	BUG_ON(exclude_end > workspace->result_len);
-
-	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
-	for (unsigned i = exclude_start; i < exclude_end; ++i) {
-		if (vec[i] == candidate) {
-			if (i >= include_start && i < include_end) {
-				dprintk(" crush_msr_valid_candidate: "
-					"candidate %d already chosen for "
-					"stride\n",
-					candidate);
-				return 1;
-			} else {
-				dprintk(" crush_msr_valid_candidate: "
-					"candidate %d collision\n",
-					candidate);
-				return 0;
-			}
-		}
-	}
-	dprintk(" crush_msr_valid_candidate: candidate %d no collision\n",
-		candidate);
-	return 1;
-}
-
-/**
- * crush_msr_push_used
- *
- * See crush_msr_choose for details, used to push bucket indicies onto collision
- * set for specified stride.  User is responsible for ensuring that
- * [stride_start, stride_end) never holds more than stride_end - stride_start
- * entries.
- * @workspace: holds working space information
- * @stepno: index of step
- * @stride_start: start of stride
- * @stride_end: one past end of stride
- * @candidate: element to add to set
- * @return 1 if added (not already present), 0 if not added due to already
- *           being present
- */
-static int crush_msr_push_used(
-	const struct crush_msr_workspace *workspace,
-	unsigned stepno,
-	unsigned stride_start,
-	unsigned stride_end,
-	int candidate)
-{
-	BUG_ON(stepno >= workspace->end_stepno);
-	BUG_ON(stepno < workspace->start_stepno);
-
-	BUG_ON(stride_end <= stride_start);
-	BUG_ON(stride_end > workspace->result_len);
-	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
-	for (unsigned i = stride_start; i < stride_end; ++i) {
-		if (vec[i] == candidate) {
-			return 0;
-		} else if (vec[i] == CRUSH_ITEM_UNDEF) {
-			vec[i] = candidate;
-			return 1;
-		}
-	}
-	BUG_ON("impossible");
-	return 0;
-}
-
-/**
- * crush_msr_push_used
- *
- * See crush_msr_choose for details, used to pop bucket indicies from collision
- * set for specified stride.  If an element is to be popped, crush_msr_pop_used
- * must be called prior to pushing another element.
- * @workspace: holds working space information
- * @stepno: index of step
- * @stride_start: start of stride
- * @stride_end: one past end of stride
- * @candidate: element to pop from set
- */
-static void crush_msr_pop_used(
-	const struct crush_msr_workspace *workspace,
-	unsigned stepno,
-	unsigned stride_start,
-	unsigned stride_end,
-	int candidate)
-{
-	BUG_ON(stepno >= workspace->end_stepno);
-	BUG_ON(stepno < workspace->start_stepno);
-
-	BUG_ON(stride_end <= stride_start);
-	BUG_ON(stride_end > workspace->result_len);
-	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
-	for (unsigned i = stride_end; i > stride_start;) {
-		--i;
-		if (vec[i] != CRUSH_ITEM_UNDEF) {
-			BUG_ON(vec[i] != candidate);
-			vec[i] = CRUSH_ITEM_UNDEF;
-			return;
-		}
-	}
-	BUG_ON(0 == "impossible");
-}
-
-/**
- * crush_msr_emit_result
- *
- * Outputs mapping result from specified position.  Position in output
- * buffer depends on rule type -- FIRSTN outputs in output order, INDEP
- * outputs into specified position.
- * @output: output buffer
- * @rule_type: CRUSH_RULE_TYPE_MSR_FIRSTN or CRUSH_RULE_TYPE_MSR_INDEP
- * @position: mapping position
- * @result: mapping value to output
- */
-static void crush_msr_emit_result(
-	struct crush_msr_output *output,
-	int rule_type,
-	unsigned position,
-	int result)
-{
-	BUG_ON(position >= output->result_len);
-	BUG_ON(output->returned_so_far >= output->result_len);
-	if (rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
-		BUG_ON(output->out[output->returned_so_far] != CRUSH_ITEM_NONE);
-		output->out[(output->returned_so_far)++] = result;
-	} else {
-		BUG_ON(output->out[position] != CRUSH_ITEM_NONE);
-		output->out[position] = result;
-		++output->returned_so_far;
-	}
-	dprintk(" emit: %d, returned_so_far: %d\n",
-		result, output->returned_so_far);
-}
-
-/**
- * crush_msr_choose
- *
- * Performs mapping for a single EMIT block containing CHOOSE steps
- * [current_stepno, end_stepno) into mapping indices [start_index, end_index).
- *
- * Like chooseleaf, crush_msr_choose is essentially depth-first -- it chooses
- * an item and all of the descendents under that item before moving to the
- * next item.  Each choose step in the block gets its own workspace for
- * collision detection.
- *
- * crush_msr_choose (and its recursive calls) will locally retry any bucket
- * selections that produce a collision (up to msr_collision_tries times), but
- * won't retry if it hits an out osd -- that's handled by calling back into
- * crush_msr_choose up to msr_descents times.
- *
- * @input: crush input information
- * @workspace: working space for this EMIT block
- * @output: crush mapping output buffer specification
- * @total_children: total number of children implied by the step sequence, may
- *                  be larger than end_index - start_index.
- * @start_index: start mapping index
- * @end_index: end mapping index
- * @current_stepno: first choose step
- * @end_stepno: one past last choose step, must be an EMIT
- * @tryno: try number, see crush_msr_do_rule
- */
-static unsigned crush_msr_choose(
-	const struct crush_msr_input *input,
-	const struct crush_msr_workspace *workspace,
-	struct crush_msr_output *output,
-	const struct crush_bucket *bucket,
-	const unsigned total_descendants,
-	const unsigned start_index, const unsigned end_index,
-	const unsigned current_stepno, const unsigned end_stepno,
-	const unsigned tryno)
-{
-	dprintk("crush_msr_choose: bucket %d, start_index %d, end_index %d\n",
-		bucket->id, start_index, end_index);
-
-	BUG_ON(current_stepno >= input->rule->len);
-	const struct crush_rule_step *curstep =
-		&(input->rule->steps[current_stepno]);
-	BUG_ON(curstep->op != CRUSH_RULE_CHOOSE_MSR);
-
-	/* This call into crush_msr_choose is responsible, ultimately, for
-	 * populating indices [start_index, end_index).  We do this by
-	 * dividing that range into a set of strides specified in the
-	 * step -- choosemsr 4 host would dictate that the range be divided
-	 * into 4 strides.
-	 *
-	 * If the full rule is
-	 *
-	 * ...
-	 * step take root
-	 * step choosemsr 4 host
-	 * step choosemsr 2 osd
-	 * step emit
-	 *
-	 * total_descendants for the initial call would be 8 (4*2) with
-	 * num_stride=4 (4 hosts) and stride_length = 2 (2 osds per host).
-	 * For the recursive calls, total_descendants would be 2 (8 / 4),
-	 * stride_length would be 1 and num_strides would be 2.
-	 */
-
-	// choosemsr 0 host should select result_max hosts
-	const unsigned num_strides = curstep->arg1 ? curstep->arg1
-		: input->result_max;
-
-	// total_descendants is the product of the steps in the block
-	BUG_ON(total_descendants % num_strides != 0);
-	const unsigned stride_length = total_descendants / num_strides;
-
-	/* MSR steps like
-	 *
-	 * step choosemsr 4 host
-	 *
-	 * guarantee that the output mapping will be divided into at least
-	 * 4 hosts, not exactly 4 hosts.  We achieve this by ensuring that
-	 * the sets of hosts for each stride are disjoint -- a host selected
-	 * for stride 0 will not be used for any other stride.
-	 *
-	 * However, a single stride might end up using more than one host.
-	 * If an OSD on a host is marked out, crush_msr_choose will simply
-	 * skip that index when it hits it.  crush_msr_do_rule will then
-	 * call back into crush_msr_choose and eventually find another OSD
-	 * either on the same host or on another one not already used in
-	 * another stride. For this reason, a single stride may need to
-	 * remember up to stride_length entries for collision detection
-	 * purposes.
-	 *
-	 * Unfortunately, we only have stride_length entries to work with
-	 * in workspace.  Thus, prior to returning from crush_msr_choose,
-	 * we remove entries that didn't actually result in a mapping.  We
-	 * use the following undo vector to achieve this -- any strides that
-	 * didn't result in a successful mapping are set in undo to be undone
-	 * immediately prior to returning.
-	 *
-	 * Why prior to returning and not immediately?  Selecting a bucket in
-	 * a stride impacts subsequent choices as they may have collided.  In
-	 * order to limit the impact of marking an OSD out, we treat it as
-	 * collidable until the next pass.
-	 */
-	int undo[num_strides];
-	for (unsigned stride = 0; stride < num_strides; ++stride) {
-		undo[stride] = CRUSH_ITEM_UNDEF;
-	}
-
-	dprintk("crush_msr_choose: bucket %d, start_index %d, "
-		"end_index %d, stride_length %d\n",
-		bucket->id, start_index, end_index, stride_length);
-
-	unsigned mapped = 0;
-	unsigned stride_index = 0;
-	for (unsigned stride_start = start_index;
-	     stride_start < end_index;
-	     stride_start += stride_length, ++stride_index) {
-		const unsigned stride_end =
-		  MIN(stride_start + stride_length, end_index);
-    
-		// all descendants for this stride have been mapped already
-		if (crush_msr_leaf_vec_populated(
-		      workspace, stride_start, stride_end)) {
-		  continue;
-		}
-
-		int found = 0;
-		int child_bucket_candidate;
-		for (unsigned local_tryno = 0;
-		     local_tryno <= input->msr_collision_tries;
-		     ++local_tryno) {
-			child_bucket_candidate = crush_msr_descend(
-				input, workspace, bucket,
-				curstep->arg2, tryno, local_tryno,
-				stride_index);
-
-			/* candidate is valid if:
-			 * - we already chose it for this stride
-			 * - it hasn't been chosen for any stride */
-			if (crush_msr_valid_candidate(
-				    workspace,
-				    current_stepno,
-				    // Collision on elements in [start_index, end_index)
-				    start_index, end_index,
-				    // ...unless in [stride_start, stride_end)
-				    stride_start, stride_end,
-				    child_bucket_candidate)) {
-				found = 1;
-				break;
-			}
-		}
-
-		/* failed to find non-colliding choice after msr_collision_tries
-		 * attempts */
-		if (!found) continue;
-
-		if (curstep->arg2 == 0 /* leaf */) {
-			if (stride_length != 1 ||
-			    (current_stepno + 1 != end_stepno)) {
-				/* Either condition above implies that there's
-				 * another step after a choosemsr step for the
-				 * leaf type, rule is malformed, bail */
-				continue;
-			}
-			if (is_out(input->map, input->weights,
-				   input->weight_len,
-				   child_bucket_candidate, input->map_input)) {
-				dprintk(" crush_msr_choose: item %d out\n",
-					child_bucket_candidate);
-				/* crush_msr_do_rule will try again,
-				 * msr_descents permitting */
-				continue;
-			}
-			// for collision detection
-			int pushed = crush_msr_push_used(
-				workspace, current_stepno, stride_start, stride_end,
-				child_bucket_candidate);
-			/* stride_length == 1, can't already be there */
-			BUG_ON(!pushed);
-			// final output, ordering depending on input->rule->type
-			crush_msr_emit_result(
-				output, input->rule->type,
-				stride_start, child_bucket_candidate);
-			mapped++;
-		} else /* not leaf */ {
-			if (current_stepno + 1 >= end_stepno) {
-				/* Type isn't leaf, rule is malformed since there
-				 * isn't another step */
-				continue;
-			}
-			struct crush_bucket *child_bucket = input->map->buckets[
-				-1 - child_bucket_candidate];
-			unsigned child_mapped = crush_msr_choose(
-				input, workspace, output,
-				child_bucket,
-				// total_descendants for recursive call
-				stride_length,
-				// recursive call populates
-				// [stride_start, stride_end)
-				stride_start, stride_end,
-				// next step
-				current_stepno + 1, end_stepno,
-				tryno);
-			int pushed = crush_msr_push_used(
-				workspace,
-				current_stepno,
-				stride_start,
-				stride_end,
-				child_bucket_candidate);
-			/* pushed may be false if we already chose this bucket
-			 * for this stride.  If so, child_mapped must have been
-			 * != 0 at the time, so we still retain it */
-			if (pushed && (child_mapped == 0)) {
-				// no child mapped, and we didn't choose it
-				// before
-				undo[stride_index] = child_bucket_candidate;
-			} else {
-				mapped += child_mapped;
-			}
-		}
-	}
-
-	// pop unused buckets
-	stride_index = 0;
-	for (unsigned stride_start = start_index;
-	     stride_start < end_index;
-	     stride_start += stride_length, ++stride_index) {
-		if (undo[stride_index] != CRUSH_ITEM_UNDEF) {
-			unsigned stride_end =
-			  MIN(stride_start + stride_length, end_index);
-			crush_msr_pop_used(
-				workspace,
-				current_stepno,
-				stride_start,
-				stride_end,
-				undo[stride_index]);
-		}
-	}
-  
-	return mapped;
-}
-
-/**
- * crush_msr_do_rule - calculate a mapping with the given input and msr rule
- *
- * msr_firstn and msr_indep rules are intended to address a limitation of
- * conventional crush rules in that they do not retry steps outside of
- * a CHOOSELEAF step.  In the case of a crush rule like
- *
- * rule replicated_rule_1 {
- *   ...
- *   step take default class hdd
- *   step chooseleaf firstn 3 type host
- *   step emit
- * }
- *
- * the chooseleaf step will ensure that if all of the osds on a
- * particular host are marked out, mappings including those OSDs would
- * end up on another host (provided that there are enough hosts).
- *
- * However, if the rule used two choose steps instead
- *
- * rule replicated_rule_1 {
- *   ...
- *   step take default class hdd
- *   step choose firstn 3 type host
- *   step choose firstn 1 type osd
- *   step emit
- * }
- *
- * marking an OSD down could cause it to be remapped to another on the same
- * host, but not to another host.  If all of the OSDs on a host are marked
- * down, the PGs will simply be degraded and unable to remap until the host
- * is removed from the CRUSH heirarchy or reweighted to 0.
- *
- * Normally, we can comfortably work around this by using a chooseleaf
- * step as in the first example, but there are cases where we want to map
- * multiple OSDs to each host (wide EC codes on small clusters, for
- * example) which can't be handled with chooseleaf as it currently
- * exists.
- *
- * rule ecpool-86 {
- *   type msr_indep
- *   ...
- *   step choosemsr 4 type host
- *   step choosemsr 4 type osd
- *   step emit
- * }
- *
- * With an 8+6 code, this rule can tolerate a host and a single OSD down without
- * becoming unavailable on 4 hosts.  It relies on ensuring that no more than 4
- * OSDs are mapped to any single host, however, which can't be done with a
- * conventional CRUSH rule without the drawback described above.  By using
- * msr_indep, this rule can deal with an OSD failure by remapping to another
- * host.
- *
- * MSR rules have some structural differences from conventional rules:
- * - The rule type determines whether the mapping is FIRSTN or INDEP.  Because
- *   the descent can retry steps, it doesn't really make sense for steps to
- *   individually specify output order and I'm not really aware of any use cases
- *   that would benefit from it.
- * - MSR rules *must* be structured as a (possibly empty) prefix of config
- *   steps (CRUSH_RULE_SET_CHOOSE_MSR*) followed by a sequence of EMIT blocks
- *   each comprised of a TAKE step, a sequence of CHOOSE_MSR steps, and
- *   ended by an EMIT step.
- * - MSR choose steps must be choosemsr.  choose and chooseleaf are not permitted.
- *
- * MSR rules also have different requirements for working space.  Conventional CRUSH
- * requires 3 vectors of size result_max to use for working space -- two to alternate
- * as it processes each rule and one, additionally, for chooseleaf.  MSR rules
- * need N vectors where N is the number of choosemsr in the longest EMIT block since
- * it needs to retain all of the choices made as part of each descent.
- *
- * See crush_msr_choose for details.
- *
- * @map: the crush_map
- * @ruleno: the rule id
- * @x: hash input
- * @result: pointer to result vector
- * @result_max: maximum result size
- * @weight: weight vector (for map leaves)
- * @weight_max: size of weight vector
- * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
- */
-static int crush_msr_do_rule(
-	const struct crush_map *map,
-	int ruleno, int map_input, int *result, int result_max,
-	const __u32 *weight, int weight_max,
-	void *cwin, const struct crush_choose_arg *choose_args)
-{
-	unsigned msr_descents = map->msr_descents;
-	unsigned msr_collision_tries = map->msr_collision_tries;
-	struct crush_rule *rule = map->rules[ruleno];
-	unsigned start_stepno = crush_msr_scan_config_steps(
-		rule->steps, rule->len,
-		&msr_descents, &msr_collision_tries);
-
-	struct crush_msr_input input = {
-		.map = map,
-		.rule = map->rules[ruleno],
-		.result_max = result_max,
-		.weight_len = weight_max,
-		.weights = weight,
-		.map_input = map_input,
-		.choose_args = choose_args,
-		.msr_descents = msr_descents,
-		.msr_collision_tries = msr_collision_tries 
-	};
-
-	struct crush_msr_output output = {
-		.result_len = result_max,
-		.returned_so_far = 0,
-		.out = result
-	};
-	for (unsigned i = 0; i < output.result_len; ++i) {
-		output.out[i] = CRUSH_ITEM_NONE;
-	}
-
-	unsigned start_index = 0;
-	while (start_stepno < input.rule->len) {
-		unsigned emit_stepno, total_children;
-		if (crush_msr_scan_next(
-			    input.rule, input.result_max,
-			    start_stepno, &total_children,
-			    &emit_stepno) != 0) {
-			// invalid rule, return whatever we have
-			dprintk("crush_msr_scan_returned -1\n");
-			return 0;
-		}
-
-		const struct crush_rule_step *take_step =
-			&(input.rule->steps[start_stepno]);
-		BUG_ON(take_step->op != CRUSH_RULE_TAKE);
-
-		if (take_step->arg1 >= 0) {
-			if (start_stepno + 1 != emit_stepno) {
-				// invalid rule
-				dprintk("take step specifies osd, but "
-					"there are subsequent choose steps\n");
-				return 0;
-			} else {
-				crush_msr_emit_result(
-					&output, input.rule->type,
-					start_index, take_step->arg1);
-			}
-		} else {
-			dprintk("start_stepno %d\n", start_stepno);
-			dprintk("root bucket: %d\n",
-				input.rule->steps[start_stepno].arg1);
-			struct crush_bucket *root_bucket = input.map->buckets[
-				-1 - input.rule->steps[start_stepno].arg1];
-			dprintk(
-				"root bucket: %d %p\n",
-				input.rule->steps[start_stepno].arg1,
-				root_bucket);
-
-			++start_stepno;
-			BUG_ON(emit_stepno >= input.rule->len);
-			BUG_ON(emit_stepno < start_stepno);
-			BUG_ON(start_stepno >= input.rule->len);
-
-			struct crush_work *cw = cwin;
-			int *out_vecs[input.rule->len];
-			for (unsigned stepno = 0; stepno < input.rule->len; ++stepno) {
-				out_vecs[stepno] = (int*)((char*)cw + map->working_size) +
-					(stepno * result_max);
-			}
-			struct crush_msr_workspace workspace = {
-				.start_stepno = start_stepno,
-				.end_stepno = emit_stepno,
-				.result_len = result_max,
-				.crush_work = cw,
-				.step_vecs = out_vecs
-			};
-			crush_msr_clear_workspace(&workspace);
-
-
-			unsigned tries_so_far = 0;
-			unsigned end_index = MIN(start_index + total_children,
-						 input.result_max);
-			while (tries_so_far <= input.msr_descents &&
-			       output.returned_so_far < input.result_max) {
-				crush_msr_choose(
-					&input, &workspace, &output,
-					root_bucket,
-					total_children,
-					start_index,
-					end_index,
-					start_stepno, emit_stepno,
-					tries_so_far);
-				dprintk("returned_so_far: %d\n",
-					output.returned_so_far);
-				++tries_so_far;
-			}
-			start_index = end_index;
-			start_stepno = emit_stepno + 1;
-		}
-	}
-
-	if (rule->type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
-	  return output.returned_so_far;
-	} else {
-	  return input.result_max;
-	}
-}
-
-/// Return 1 if msr, 0 otherwise
-static int rule_type_is_msr(int type)
-{
-	return type == CRUSH_RULE_TYPE_MSR_FIRSTN ||
-		type == CRUSH_RULE_TYPE_MSR_INDEP;
-}
-
-size_t crush_work_size(const struct crush_map *map,
-		       int result_max)
-{
-	unsigned ruleno;
-	unsigned out_vecs = 3; /* normal do_rule needs 3 outvecs */
-	for (ruleno = 0; ruleno < map->max_rules; ++ruleno) {
-		const struct crush_rule *rule = map->rules[ruleno];
-		if (!rule) continue;
-		if (!rule_type_is_msr(rule->type))
-			continue;
-		unsigned rule_max_msr_steps;
-		// we ignore the return value because rule_max_msr_steps will be
-		// populated with the longest step sequence before hitting
-		// the error
-		crush_msr_scan_rule(rule, result_max, &rule_max_msr_steps);
-		out_vecs = MAX(rule_max_msr_steps, out_vecs);
-	}
-	return map->working_size + result_max * out_vecs * sizeof(__u32);
-}
-
-/* This takes a chunk of memory and sets it up to be a shiny new
-   working area for a CRUSH placement computation. It must be called
-   on any newly allocated memory before passing it in to
-   crush_do_rule. It may be used repeatedly after that, so long as the
-   map has not changed. If the map /has/ changed, you must make sure
-   the working size is no smaller than what was allocated and re-run
-   crush_init_workspace.
-
-   If you do retain the working space between calls to crush, make it
-   thread-local. If you reinstitute the locking I've spent so much
-   time getting rid of, I will be very unhappy with you. */
-
-void crush_init_workspace(const struct crush_map *m, void *v) {
-	/* We work by moving through the available space and setting
-	   values and pointers as we go.
-
-	   It's a bit like Forth's use of the 'allot' word since we
-	   set the pointer first and then reserve the space for it to
-	   point to by incrementing the point. */
-	struct crush_work *w = (struct crush_work *)v;
-	char *point = (char *)v;
-	__s32 b;
-	point += sizeof(struct crush_work);
-	w->work = (struct crush_work_bucket **)point;
-	point += m->max_buckets * sizeof(struct crush_work_bucket *);
-	for (b = 0; b < m->max_buckets; ++b) {
-		if (m->buckets[b] == 0)
-			continue;
-
-		w->work[b] = (struct crush_work_bucket *) point;
-		switch (m->buckets[b]->alg) {
-		default:
-			point += sizeof(struct crush_work_bucket);
-			break;
-		}
-		w->work[b]->perm_x = 0;
-		w->work[b]->perm_n = 0;
-		w->work[b]->perm = (__u32 *)point;
-		point += m->buckets[b]->size * sizeof(__u32);
-	}
-	BUG_ON((char *)point - (char *)w != m->working_size);
-}
-
-/**
- * crush_do_rule - calculate a mapping with the given input and rule
- * @map: the crush_map
- * @ruleno: the rule id
- * @x: hash input
- * @result: pointer to result vector
- * @result_max: maximum result size
- * @weight: weight vector (for map leaves)
- * @weight_max: size of weight vector
- * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
- */
-int crush_do_rule(const struct crush_map *map,
-		  int ruleno, int x, int *result, int result_max,
-		  const __u32 *weight, int weight_max,
-		  void *cwin, const struct crush_choose_arg *choose_args)
-{
-	const struct crush_rule *rule;
-
-	if ((__u32)ruleno >= map->max_rules) {
-		dprintk(" bad ruleno %d\n", ruleno);
-		return 0;
-	}
-
-	rule = map->rules[ruleno];
-	if (rule_type_is_msr(rule->type)) {
-		return crush_msr_do_rule(
-			map,
-			ruleno,
-			x,
-			result,
-			result_max,
-			weight,
-			weight_max,
-			cwin,
-			choose_args);
-	} else {
-		return crush_do_rule_no_retry(
-			map,
-			ruleno,
-			x,
-			result,
-			result_max,
-			weight,
-			weight_max,
-			cwin,
-			choose_args);
-	}
-}
diff --git a/src/crush/mapper.h b/src/crush/mapper.h
index 98c7bf11c0d9..0ec927d9e616 100644
--- a/src/crush/mapper.h
+++ b/src/crush/mapper.h
@@ -77,11 +77,15 @@ extern int crush_do_rule(const struct crush_map *map,
 			 const __u32 *weights, int weight_max,
 			 void *cwin, const struct crush_choose_arg *choose_args);
 
-/* Returns enough workspace for any crush rule within map to generate
-   result_max outputs. The caller can then allocate this much on its own,
-   either on the stack, in a per-thread long-lived buffer, or however it likes.*/
-extern size_t crush_work_size(const struct crush_map *map,
-			      int result_max);
+/* Returns the exact amount of workspace that will need to be used
+   for a given combination of crush_map and result_max. The caller can
+   then allocate this much on its own, either on the stack, in a
+   per-thread long-lived buffer, or however it likes. */
+
+static inline size_t crush_work_size(const struct crush_map *map,
+				     int result_max) {
+	return map->working_size + result_max * 3 * sizeof(__u32);
+}
 
 extern void crush_init_workspace(const struct crush_map *m, void *v);
 
diff --git a/src/erasure-code/ErasureCode.cc b/src/erasure-code/ErasureCode.cc
index 928d05f2adb0..5212baee2518 100644
--- a/src/erasure-code/ErasureCode.cc
+++ b/src/erasure-code/ErasureCode.cc
@@ -52,12 +52,6 @@ int ErasureCode::init(
   err |= to_string("crush-failure-domain", profile,
 		   &rule_failure_domain,
 		   DEFAULT_RULE_FAILURE_DOMAIN, ss);
-  err |= to_int("crush-osds-per-failure-domain", profile,
-		&rule_osds_per_failure_domain,
-		"0", ss);
-  err |= to_int("crush-num-failure-domains", profile,
-		&rule_num_failure_domains,
-		"0", ss);
   err |= to_string("crush-device-class", profile,
 		   &rule_device_class,
 		   "", ss);
@@ -72,33 +66,19 @@ int ErasureCode::create_rule(
   CrushWrapper &crush,
   std::ostream *ss) const
 {
-  if (rule_osds_per_failure_domain <= 1) {
-    return crush.add_simple_rule(
-      name,
-      rule_root,
-      rule_failure_domain,
-      rule_num_failure_domains,
-      rule_device_class,
-      "indep",
-      pg_pool_t::TYPE_ERASURE,
-      ss);
-  } else {
-    if (rule_num_failure_domains < 1)  {
-      if (ss) {
-	*ss << "crush-num-failure-domains " << rule_num_failure_domains 
-	    << " must be >= 1 if crush-osds-per-failure-domain specified";
-	return -EINVAL;
-      }
-    }
-    return crush.add_indep_multi_osd_per_failure_domain_rule(
-      name,
-      rule_root,
-      rule_failure_domain,
-      rule_num_failure_domains,
-      rule_osds_per_failure_domain,
-      rule_device_class,
-      ss);
-  }
+  int ruleid = crush.add_simple_rule(
+    name,
+    rule_root,
+    rule_failure_domain,
+    rule_device_class,
+    "indep",
+    pg_pool_t::TYPE_ERASURE,
+    ss);
+
+  if (ruleid < 0)
+    return ruleid;
+
+  return ruleid;
 }
 
 int ErasureCode::sanity_check_k_m(int k, int m, ostream *ss)
diff --git a/src/erasure-code/ErasureCode.h b/src/erasure-code/ErasureCode.h
index fd6d1a41f714..c246d5dc6b67 100644
--- a/src/erasure-code/ErasureCode.h
+++ b/src/erasure-code/ErasureCode.h
@@ -37,8 +37,6 @@ namespace ceph {
     std::string rule_root;
     std::string rule_failure_domain;
     std::string rule_device_class;
-    int rule_osds_per_failure_domain = -1;
-    int rule_num_failure_domains = -1;
 
     ~ErasureCode() override {}
 
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 23175adfa2c8..1937eeb4c698 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -137,7 +137,7 @@ DEFINE_CEPH_FEATURE(34, 3, RANGE_BLOCKLIST)
 DEFINE_CEPH_FEATURE(35, 1, OSD_CACHEPOOL)    // 3.14
 DEFINE_CEPH_FEATURE(36, 1, CRUSH_V2)         // 3.14
 DEFINE_CEPH_FEATURE(37, 1, EXPORT_PEER)      // 3.14
-DEFINE_CEPH_FEATURE(38, 2, CRUSH_MSR)        // X.XX TODOSAM kernel version?
+DEFINE_CEPH_FEATURE_RETIRED(38, 1, OSD_ERASURE_CODES, MIMIC, OCTOPUS)
 // available
 DEFINE_CEPH_FEATURE(39, 1, OSDMAP_ENC)       // 3.15
 DEFINE_CEPH_FEATURE(40, 1, MDS_INLINE_DATA)  // 3.19
@@ -218,7 +218,6 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_OSD_CACHEPOOL |	    \
 	 CEPH_FEATURE_CRUSH_V2 |	    \
 	 CEPH_FEATURE_EXPORT_PEER |	    \
-	 CEPH_FEATURE_CRUSH_MSR |	    \
 	 CEPH_FEATURE_OSDMAP_ENC |          \
 	 CEPH_FEATURE_MDS_INLINE_DATA |	    \
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |	    \
@@ -266,10 +265,9 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_CRUSH_TUNABLES2 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES5 |		\
-	 CEPH_FEATURE_CRUSH_MSR |		\
 	 CEPH_FEATURE_CRUSH_V2 |		\
 	 CEPH_FEATURE_CRUSH_V4 |		\
-	 CEPH_FEATUREMASK_CRUSH_MSR)
+	 CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS)
 
 /*
  * make sure we don't try to use the reserved features
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 696d7f3185b3..f8e379326f25 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -7562,12 +7562,6 @@ bool OSDMonitor::validate_crush_against_features(const CrushWrapper *newcrush,
 	 << newmap.require_min_compat_client;
       return false;
     }
-    if (mv > newmap.require_osd_release) {
-      ss << "new crush map requires client version " << mv
-	 << " but require_osd_release is "
-	 << newmap.require_osd_release;
-      return false;
-    }
   }
 
   // osd compat
@@ -8078,7 +8072,7 @@ int OSDMonitor::prepare_new_pool(string& name,
     return r;
   }
 
-  if (!osdmap.crush->rule_valid_for_pool_type(crush_rule, pool_type)) {
+  if (osdmap.crush->get_rule_type(crush_rule) != (int)pool_type) {
     *ss << "crush rule " << crush_rule << " type does not match pool";
     return -EINVAL;
   }
@@ -8350,7 +8344,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
 	return -EPERM;
       }
     }
-    if (!osdmap.crush->rule_valid_for_pool_type(p.get_crush_rule(), p.type)) {
+    if (osdmap.crush->get_rule_type(p.get_crush_rule()) != (int)p.type) {
       ss << "crush rule " << p.get_crush_rule() << " type does not match pool";
       return -EINVAL;
     }
@@ -8583,7 +8577,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
       ss << cpp_strerror(id);
       return -ENOENT;
     }
-    if (!osdmap.crush->rule_valid_for_pool_type(id, p.get_type())) {
+    if (osdmap.crush->get_rule_type(id) != (int)p.get_type()) {
       ss << "crush rule " << id << " type does not match pool";
       return -EINVAL;
     }
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 76552333dfff..5773695b77ad 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1764,10 +1764,9 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_V4;
   if (crush->has_nondefault_tunables5())
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
-  if (crush->has_incompat_choose_args())
+  if (crush->has_incompat_choose_args()) {
     features |= CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS;
-  if (crush->has_nondefault_tunables_msr())
-    features |= CEPH_FEATURE_CRUSH_MSR;
+  }
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty() || !pg_upmap_primaries.empty())
@@ -1790,8 +1789,6 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
 	features |= CEPH_FEATURE_CRUSH_TUNABLES3;
       if (crush->is_v5_rule(ruleid))
 	features |= CEPH_FEATURE_CRUSH_TUNABLES5;
-      if (crush->is_msr_rule(ruleid))
-	features |= CEPH_FEATURE_CRUSH_MSR;
     }
   }
   mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;
@@ -1846,9 +1843,6 @@ ceph_release_t OSDMap::get_min_compat_client() const
 {
   uint64_t f = get_features(CEPH_ENTITY_TYPE_CLIENT, nullptr);
 
-  if (HAVE_FEATURE(f, CRUSH_MSR)) {      // TODOSAM -- add version right before merge
-    return ceph_release_t::squid;        // v19.2.0
-  }
   if (HAVE_FEATURE(f, OSDMAP_PG_UPMAP) ||      // v12.0.0-1733-g27d6f43
       HAVE_FEATURE(f, CRUSH_CHOOSE_ARGS)) {    // v12.0.1-2172-gef1ef28
     return ceph_release_t::luminous;  // v12.2.0
@@ -4530,7 +4524,7 @@ int OSDMap::validate_crush_rules(CrushWrapper *newcrush,
 	  << " but it is not present";
       return -EINVAL;
     }
-    if (!newcrush->rule_valid_for_pool_type(ruleno, pool.get_type())) {
+    if (newcrush->get_rule_type(ruleno) != (int)pool.get_type()) {
       *ss << "pool " << i.first << " type does not match rule " << ruleno;
       return -EINVAL;
     }
diff --git a/src/test/cli/crushtool/choose-args.t b/src/test/cli/crushtool/choose-args.t
index 99120f0f211f..e0956ec0a754 100644
--- a/src/test/cli/crushtool/choose-args.t
+++ b/src/test/cli/crushtool/choose-args.t
@@ -159,8 +159,6 @@
           "chooseleaf_descend_once": 0,
           "chooseleaf_vary_r": 0,
           "chooseleaf_stable": 0,
-          "msr_descents": 100,
-          "msr_collision_tries": 100,
           "straw_calc_version": 0,
           "allowed_bucket_algs": 22,
           "profile": "argonaut",
@@ -174,8 +172,7 @@
           "has_v3_rules": 0,
           "has_v4_buckets": 1,
           "require_feature_tunables5": 0,
-          "has_v5_rules": 0,
-          "has_msr_rules": 0
+          "has_v5_rules": 0
       },
       "choose_args": {
           "1": [],
diff --git a/src/test/cli/osdmaptool/crush.t b/src/test/cli/osdmaptool/crush.t
index 695c4e0c22d9..520f11e50d5c 100644
--- a/src/test/cli/osdmaptool/crush.t
+++ b/src/test/cli/osdmaptool/crush.t
@@ -6,7 +6,7 @@
   osdmaptool: exported crush map to oc
   $ osdmaptool --import-crush oc myosdmap
   osdmaptool: osdmap file 'myosdmap'
-  osdmaptool: imported 499 byte crush map from oc
+  osdmaptool: imported 497 byte crush map from oc
   osdmaptool: writing epoch 3 to myosdmap
   $ osdmaptool --adjust-crush-weight 0:5 myosdmap
   osdmaptool: osdmap file 'myosdmap'
diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 9e2a2c99fd22..1f53084a70af 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -18,11 +18,69 @@
 #include "include/stringify.h"
 
 #include "crush/CrushWrapper.h"
-#include "crush/CrushCompiler.h"
 #include "osd/osd_types.h"
 
 using namespace std;
 
+std::unique_ptr<CrushWrapper> build_indep_map(CephContext *cct, int num_rack,
+                              int num_host, int num_osd)
+{
+  std::unique_ptr<CrushWrapper> c(new CrushWrapper);
+  c->create();
+
+  c->set_type_name(5, "root");
+  c->set_type_name(4, "row");
+  c->set_type_name(3, "rack");
+  c->set_type_name(2, "chasis");
+  c->set_type_name(1, "host");
+  c->set_type_name(0, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		5, 0, NULL, NULL, &rootno);
+  c->set_item_name(rootno, "default");
+
+  map<string,string> loc;
+  loc["root"] = "default";
+
+  int osd = 0;
+  for (int r=0; r<num_rack; ++r) {
+    loc["rack"] = string("rack-") + stringify(r);
+    for (int h=0; h<num_host; ++h) {
+      loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+      for (int o=0; o<num_osd; ++o, ++osd) {
+	c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+      }
+    }
+  }
+  int ret;
+  int ruleno = 0;
+  ret = c->add_rule(ruleno, 4, 123);
+  ceph_assert(ret == ruleno);
+  ret = c->set_rule_step(ruleno, 0, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
+  ceph_assert(ret == 0);
+  ret = c->set_rule_step(ruleno, 1, CRUSH_RULE_TAKE, rootno, 0);
+  ceph_assert(ret == 0);
+  ret = c->set_rule_step(ruleno, 2, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
+  ceph_assert(ret == 0);
+  ret = c->set_rule_step(ruleno, 3, CRUSH_RULE_EMIT, 0, 0);
+  ceph_assert(ret == 0);
+  c->set_rule_name(ruleno, "data");
+
+  c->finalize();
+
+  if (false) {
+    Formatter *f = Formatter::create("json-pretty");
+    f->open_object_section("crush_map");
+    c->dump(f);
+    f->close_section();
+    f->flush(cout);
+    delete f;
+  }
+
+  return c;
+}
+
 int get_num_dups(const vector<int>& v)
 {
   std::set<int> s;
@@ -36,21 +94,7 @@ int get_num_dups(const vector<int>& v)
   return dups;
 }
 
-class RuleType {
-  bool msr;
-
-public:
-  RuleType(bool msr) : msr(msr) {}
-
-  bool is_msr() const { return msr; }
-  
-  friend std::ostream &operator<<(std::ostream &, RuleType);
-};
-std::ostream &operator<<(std::ostream &lhs, RuleType rhs) {
-  return lhs << (rhs.msr ? "MSR" : "NORMAL");
-}
-
-class IndepTest : public ::testing::TestWithParam<RuleType>
+class CRUSHTest : public ::testing::Test
 {
 public:
   void SetUp() final
@@ -64,91 +108,11 @@ class IndepTest : public ::testing::TestWithParam<RuleType>
     cct->put();
     cct = nullptr;
   }
-
-  std::unique_ptr<CrushWrapper> build_indep_map(
-    CephContext *cct, int num_rack, int num_host, int num_osd)
-  {
-    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
-    c->create();
-    c->set_tunables_optimal();
-
-    c->set_type_name(5, "root");
-    c->set_type_name(4, "row");
-    c->set_type_name(3, "rack");
-    c->set_type_name(2, "chasis");
-    c->set_type_name(1, "host");
-    c->set_type_name(0, "osd");
-
-    int rootno;
-    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
-		  5, 0, NULL, NULL, &rootno);
-    c->set_item_name(rootno, "default");
-
-    map<string,string> loc;
-    loc["root"] = "default";
-
-    int osd = 0;
-    for (int r=0; r<num_rack; ++r) {
-      loc["rack"] = string("rack-") + stringify(r);
-      for (int h=0; h<num_host; ++h) {
-	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
-	for (int o=0; o<num_osd; ++o, ++osd) {
-	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
-	}
-      }
-    }
-    int ret;
-    int ruleno = 0;
-
-    if (GetParam().is_msr()) {
-      unsigned step_id = 0;
-      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_INDEP);
-      ceph_assert(ret == ruleno);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
-      ceph_assert(ret == 0);
-    } else {
-      unsigned step_id = 0;
-      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
-      ceph_assert(ret == ruleno);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
-      ceph_assert(ret == 0);
-    }
-
-    c->set_rule_name(ruleno, "data");
-    c->finalize();
-
-    if (false) {
-      Formatter *f = Formatter::create("json-pretty");
-      f->open_object_section("crush_map");
-      c->dump(f);
-      f->close_section();
-      f->flush(cout);
-      delete f;
-    }
-
-    return c;
-  }
-
 protected:
   CephContext *cct = nullptr;
 };
 
-TEST_P(IndepTest, toosmall) {
+TEST_F(CRUSHTest, indep_toosmall) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 1, 3, 1));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, NULL);
@@ -167,7 +131,7 @@ TEST_P(IndepTest, toosmall) {
   }
 }
 
-TEST_P(IndepTest, basic) {
+TEST_F(CRUSHTest, indep_basic) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, NULL);
@@ -186,88 +150,7 @@ TEST_P(IndepTest, basic) {
   }
 }
 
-TEST_P(IndepTest, single_out_first) {
-  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 1000; ++x) {
-    vector<__u32> weight(c->get_max_devices(), 0x10000);
-    vector<int> out;
-    c->do_rule(0, x, out, 5, weight, 0);
-
-    int num_none = 0;
-    for (unsigned i=0; i<out.size(); ++i) {
-      if (out[i] == CRUSH_ITEM_NONE)
-	num_none++;
-    }
-    ASSERT_EQ(0, num_none);
-    ASSERT_EQ(0, get_num_dups(out));
-
-    // mark first osd out
-    weight[out[0]] = 0;
-
-    vector<int> out2;
-    c->do_rule(0, x, out2, 5, weight, 0);
-
-    cout << "input " << x
-	 << " marked out " << out[0]
-	 << " out " << out
-	 << " -> out2 " << out2
-	 << std::endl;
-
-    // First item should have been remapped
-    ASSERT_NE(CRUSH_ITEM_NONE, out2[0]);
-    ASSERT_NE(out[0], out2[0]);
-    for (unsigned i=1; i<out.size(); ++i) {
-      // but none of the others
-      ASSERT_EQ(out[i], out2[i]);
-    }
-    ASSERT_EQ(0, get_num_dups(out2));
-  }
-}
-
-TEST_P(IndepTest, single_out_last) {
-  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 1000; ++x) {
-    vector<__u32> weight(c->get_max_devices(), 0x10000);
-    vector<int> out;
-    c->do_rule(0, x, out, 5, weight, 0);
-
-    int num_none = 0;
-    for (unsigned i=0; i<out.size(); ++i) {
-      if (out[i] == CRUSH_ITEM_NONE)
-	num_none++;
-    }
-    ASSERT_EQ(0, num_none);
-    ASSERT_EQ(0, get_num_dups(out));
-
-    // mark first osd out
-    unsigned last = out.size() - 1;
-    weight[out[last]] = 0;
-
-    vector<int> out2;
-    c->do_rule(0, x, out2, 5, weight, 0);
-
-    cout << "input " << x
-	 << " marked out " << out[0]
-	 << " out " << out
-	 << " -> out2 " << out2
-	 << std::endl;
-
-    // Last
-    ASSERT_NE(CRUSH_ITEM_NONE, out2[last]);
-    ASSERT_NE(out[last], out2[last]);
-    for (unsigned i=0; i<last; ++i) {
-      // but none of the others
-      ASSERT_EQ(out[i], out2[i]);
-    }
-    ASSERT_EQ(0, get_num_dups(out2));
-  }
-}
-
-TEST_P(IndepTest, out_alt) {
+TEST_F(CRUSHTest, indep_out_alt) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -293,7 +176,7 @@ TEST_P(IndepTest, out_alt) {
   }
 }
 
-TEST_P(IndepTest, out_contig) {
+TEST_F(CRUSHTest, indep_out_contig) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -318,7 +201,8 @@ TEST_P(IndepTest, out_contig) {
   }
 }
 
-TEST_P(IndepTest, out_progressive) {
+
+TEST_F(CRUSHTest, indep_out_progressive) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   c->set_choose_total_tries(100);
   vector<__u32> tweight(c->get_max_devices(), 0x10000);
@@ -333,15 +217,8 @@ TEST_P(IndepTest, out_progressive) {
     for (unsigned i=0; i<weight.size(); ++i) {
       vector<int> out;
       c->do_rule(0, x, out, 7, weight, 0);
-      cout << "(" << i << "/" << weight.size() << " out) ";
-      if (i > 0) cout << "marked out " << i - 1 << " ";
-      cout << x << " -> " << out << std::endl;
-
-      int num_none = 0;
-      for (unsigned k=0; k<out.size(); ++k) {
-	if (out[k] == CRUSH_ITEM_NONE)
-	  num_none++;
-      }
+      cout << "(" << i << "/" << weight.size() << " out) "
+	   << x << " -> " << out << std::endl;
       ASSERT_EQ(0, get_num_dups(out));
 
       // make sure nothing moved
@@ -361,6 +238,7 @@ TEST_P(IndepTest, out_progressive) {
 	    cout << " " << out[j] << " moved from " << pos[out[j]] << " to " << j << std::endl;
 	    ++moved;
 	  }
+	  //ASSERT_EQ(j, pos[out[j]]);
 	}
       }
       if (moved || changed)
@@ -382,334 +260,6 @@ TEST_P(IndepTest, out_progressive) {
 
 }
 
-INSTANTIATE_TEST_SUITE_P(
-  IndepTest,
-  IndepTest,
-  ::testing::Values(RuleType(true), RuleType(false)),
-  testing::PrintToStringParamName());
-
-class FirstnTest : public ::testing::TestWithParam<RuleType>
-{
-public:
-  void SetUp() final
-  {
-    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
-    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
-			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
-  }
-  void TearDown() final
-  {
-    cct->put();
-    cct = nullptr;
-  }
-
-  std::unique_ptr<CrushWrapper> build_firstn_map(
-    CephContext *cct, int num_rack, int num_host, int num_osd)
-  {
-    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
-    c->create();
-    c->set_tunables_optimal();
-
-    c->set_type_name(5, "root");
-    c->set_type_name(4, "row");
-    c->set_type_name(3, "rack");
-    c->set_type_name(2, "chasis");
-    c->set_type_name(1, "host");
-    c->set_type_name(0, "osd");
-
-    int rootno;
-    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
-		  5, 0, NULL, NULL, &rootno);
-    c->set_item_name(rootno, "default");
-
-    map<string,string> loc;
-    loc["root"] = "default";
-
-    int osd = 0;
-    for (int r=0; r<num_rack; ++r) {
-      loc["rack"] = string("rack-") + stringify(r);
-      for (int h=0; h<num_host; ++h) {
-	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
-	for (int o=0; o<num_osd; ++o, ++osd) {
-	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
-	}
-      }
-    }
-    int ret;
-    int ruleno = 0;
-
-    if (GetParam().is_msr()) {
-      unsigned step_id = 0;
-      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_FIRSTN);
-      ceph_assert(ret == ruleno);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
-      ceph_assert(ret == 0);
-    } else {
-      unsigned step_id = 0;
-      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
-      ceph_assert(ret == ruleno);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 0, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(
-	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_FIRSTN, CRUSH_CHOOSE_N, 1);
-      ceph_assert(ret == 0);
-      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
-      ceph_assert(ret == 0);
-    }
-
-    c->set_rule_name(ruleno, "data");
-    c->finalize();
-
-    if (false) {
-      Formatter *f = Formatter::create("json-pretty");
-      f->open_object_section("crush_map");
-      c->dump(f);
-      f->close_section();
-      f->flush(cout);
-      delete f;
-    }
-
-    return c;
-  }
-
-protected:
-  CephContext *cct = nullptr;
-};
-
-TEST_P(FirstnTest, basic) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  vector<__u32> weight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 100; ++x) {
-    vector<int> out;
-    c->do_rule(0, x, out, 3, weight, 0);
-    cout << x << " -> " << out << std::endl;
-    for (unsigned i=0; i<out.size(); ++i) {
-      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
-    }
-    ASSERT_EQ(3, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-  }
-}
-
-TEST_P(FirstnTest, toosmall) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 1, 3, 1));
-  vector<__u32> weight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 100; ++x) {
-    vector<int> out;
-    c->do_rule(0, x, out, 5, weight, 0);
-    cout << x << " -> " << out << std::endl;
-    for (unsigned i=0; i<out.size(); ++i) {
-      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
-    }
-    ASSERT_EQ(3, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-  }
-}
-
-TEST_P(FirstnTest, single_out_first) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 1000; ++x) {
-    vector<__u32> weight(c->get_max_devices(), 0x10000);
-    vector<int> out;
-    c->do_rule(0, x, out, 3, weight, 0);
-
-    for (unsigned i=0; i<out.size(); ++i) {
-      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
-    }
-    ASSERT_EQ(3, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-
-    // mark first osd out
-    weight[out[0]] = 0;
-
-    vector<int> out2;
-    c->do_rule(0, x, out2, 3, weight, 0);
-
-    cout << "input " << x
-	 << " marked out " << out[0]
-	 << " out " << out
-	 << " -> out2 " << out2
-	 << std::endl;
-
-    ASSERT_EQ(3, out2.size());
-    ASSERT_EQ(0, get_num_dups(out2));
-    for (unsigned i=0; i<out2.size(); ++i) {
-      EXPECT_NE(out2[i], out[0]);
-    }
-    if (GetParam().is_msr()) {
-      // normal crush doesn't guarantee this reliably
-      ASSERT_EQ(out2[0], out[1]);
-      ASSERT_EQ(out2[1], out[2]);
-      ASSERT_NE(out2[2], out[0]);
-    }
-  }
-}
-
-TEST_P(FirstnTest, single_out_last) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  c->dump_tree(&cout, NULL);
-
-  for (int x = 0; x < 1000; ++x) {
-    vector<__u32> weight(c->get_max_devices(), 0x10000);
-    vector<int> out;
-    c->do_rule(0, x, out, 3, weight, 0);
-
-    for (unsigned i=0; i<out.size(); ++i) {
-      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
-    }
-    ASSERT_EQ(3, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-
-    // mark first osd out
-    weight[out[2]] = 0;
-
-    vector<int> out2;
-    c->do_rule(0, x, out2, 3, weight, 0);
-
-    cout << "input " << x
-	 << " marked out " << out[0]
-	 << " out " << out
-	 << " -> out2 " << out2
-	 << std::endl;
-
-    ASSERT_EQ(3, out2.size());
-    ASSERT_EQ(0, get_num_dups(out2));
-    for (unsigned i=0; i<out2.size(); ++i) {
-      EXPECT_NE(out2[i], out[2]);
-    }
-    ASSERT_EQ(out2[0], out[0]);
-    ASSERT_EQ(out2[1], out[1]);
-    ASSERT_NE(out2[2], out[2]);
-  }
-}
-
-TEST_P(FirstnTest, out_alt) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  vector<__u32> weight(c->get_max_devices(), 0x10000);
-
-  // mark a bunch of osds out
-  int num = 3*3*3;
-  for (int i=0; i<num / 2; ++i)
-    weight[i*2] = 0;
-  c->dump_tree(&cout, NULL);
-
-  // need more retries to get 9/9 hosts for x in 0..99
-  if (!GetParam().is_msr()) {
-    c->set_choose_total_tries(500);
-  }
-  for (int x = 0; x < 100; ++x) {
-    vector<int> out;
-    c->do_rule(0, x, out, 9, weight, 0);
-    cout << x << " -> " << out << std::endl;
-    ASSERT_EQ(9, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-  }
-}
-
-TEST_P(FirstnTest, out_contig) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  vector<__u32> weight(c->get_max_devices(), 0x10000);
-
-  // mark a bunch of osds out
-  int num = 3*3*3;
-  for (int i=0; i<num / 3; ++i)
-    weight[i] = 0;
-  c->dump_tree(&cout, NULL);
-
-  // need more retries to get 7/7 hosts for x in 0..99
-  if (!GetParam().is_msr()) {
-    c->set_choose_total_tries(500);
-  }
-  for (int x = 0; x < 100; ++x) {
-    vector<int> out;
-    c->do_rule(0, x, out, 7, weight, 0);
-    cout << x << " -> " << out << std::endl;
-    ASSERT_EQ(6, out.size());
-    ASSERT_EQ(0, get_num_dups(out));
-  }
-}
-
-TEST_P(FirstnTest, out_progressive) {
-  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
-  if (!GetParam().is_msr()) {
-    c->set_choose_total_tries(500);
-  }
-  vector<__u32> tweight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
-
-  int tchanged = 0;
-  for (int x = 1; x < 5; ++x) {
-    vector<__u32> weight(c->get_max_devices(), 0x10000);
-
-    std::set<int> prev;
-    for (unsigned i=0; i<weight.size(); ++i) {
-      vector<int> out;
-      c->do_rule(0, x, out, 7, weight, 0);
-      cout << "(" << i << "/" << weight.size() << " out) ";
-      if (i > 0) cout << "marked out " << i - 1 << " ";
-      cout << x << " -> " << out << std::endl;
-
-      ASSERT_EQ(0, get_num_dups(out));
-
-      int changed = 0;
-      for (unsigned j=0; j<out.size(); ++j) {
-	if (i && prev.count(out[j]) == 0) {
-	  ++changed;
-	  ++tchanged;
-	}
-      }
-      if (changed)
-	cout << " " << changed << " changed" << std::endl;
-      ASSERT_LE(changed, 3);
-
-      // mark another osd out
-      weight[i] = 0;
-      prev = std::set<int>{out.begin(), out.end()};
-    }
-  }
-  cout << tchanged << " total changed" << std::endl;
-}
-
-INSTANTIATE_TEST_SUITE_P(
-  FirstnTest,
-  FirstnTest,
-  ::testing::Values(RuleType(true), RuleType(false)),
-  testing::PrintToStringParamName());
-
-class CRUSHTest : public ::testing::Test
-{
-public:
-  void SetUp() final
-  {
-    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
-    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
-			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
-  }
-  void TearDown() final
-  {
-    cct->put();
-    cct = nullptr;
-  }
-protected:
-  CephContext *cct = nullptr;
-};
-
 TEST_F(CRUSHTest, straw_zero) {
   // zero weight items should have no effect on placement.
 
@@ -1103,459 +653,3 @@ TEST_F(CRUSHTest, straw2_reweight) {
     cout << "     vs " << estddev << std::endl;
   }
 }
-
-struct cluster_test_spec_t {
-  const int num_osds_per_host;
-  const int num_hosts;
-
-  const int num_hosts_mapped;
-  const int num_mapped_per_host;
-  const int num_mapped_size;
-
-  const int num_osds;
-
-  cluster_test_spec_t(
-    int num_osds_per_host, int num_hosts,
-    int num_hosts_mapped, int num_mapped_per_host, int num_mapped_size)
-    : num_osds_per_host(num_osds_per_host), num_hosts(num_hosts),
-      num_hosts_mapped(num_hosts_mapped),
-      num_mapped_per_host(num_mapped_per_host),
-      num_mapped_size(num_mapped_size),
-      num_osds(num_osds_per_host * num_hosts) {}
-
-  void validate_osd(int osd) const {
-    EXPECT_GE(osd, 0);
-    EXPECT_LT(osd, num_osds);
-  }
-
-  bool check_osd(int osd) const {
-    return osd >= 0 && osd < num_osds;
-  }
-
-  void validate_host(int host) const {
-    assert(host >= 0);
-    assert(host < num_hosts);
-  }
-
-  std::pair<int, int> host_to_osd_range(int host) const {
-    validate_host(host);
-    auto first = host * num_osds_per_host;
-    return std::make_pair(first, first + num_osds_per_host);
-  }
-
-  int osd_to_host(int osd) const {
-    validate_osd(osd);
-    return osd / num_osds_per_host;
-  }
-};
-
-static constexpr int ROOT_TYPE = 2;
-static constexpr int HOST_TYPE = 1;
-static constexpr int OSD_TYPE = 0;
-std::pair<int, std::unique_ptr<CrushWrapper>> create_crush_heirarchy(
-  CephContext *cct,
-  const cluster_test_spec_t &spec)
-{
-  auto c = std::make_unique<CrushWrapper>();
-  c->create();
-  c->set_tunables_optimal();
-
-  
-  c->set_type_name(ROOT_TYPE, "root");
-  c->set_type_name(HOST_TYPE, "host");
-  c->set_type_name(OSD_TYPE, "osd");
-
-  int rootno;
-  c->add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
-	       ROOT_TYPE, 0, NULL, NULL, &rootno);
-  c->set_item_name(rootno, "default");
-
-  for (auto host_id = 0; host_id < spec.num_hosts; ++host_id) {
-    const std::string host_name = fmt::format("host{}", host_id);
-    const auto first_host_osd = host_id * spec.num_osds_per_host;
-    const auto next_first_host_osd = first_host_osd + spec.num_osds_per_host;
-    for (auto osd_id = first_host_osd; osd_id < next_first_host_osd; ++osd_id) {
-      const std::string osd_name = fmt::format("osd{}", osd_id);
-      auto ret = c->insert_item(
-	cct, osd_id, 1.0, osd_name,
-	{{ "root", "default"}, {"host", host_name}});
-      EXPECT_EQ(ret, 0);
-    }
-  }
-
-  c->finalize();
-  return std::make_pair(rootno, std::move(c));
-}
-
-std::vector<uint32_t> create_weight_vector(
-  const cluster_test_spec_t &spec)
-{
-  return std::vector<uint32_t>(spec.num_osds, CEPH_OSD_IN);
-}
-
-std::vector<uint32_t> create_weight_vector_first_osd_out(
-  const cluster_test_spec_t &spec,
-  const std::vector<int> &mapping)
-{
-  auto weights = create_weight_vector(spec);
-  spec.validate_osd(mapping[0]);
-  weights[mapping[0]] = CEPH_OSD_OUT;
-  return weights;
-}
-
-std::vector<uint32_t> create_weight_vector_first_host_out(
-  const cluster_test_spec_t &spec,
-  const std::vector<int> &mapping)
-{
-  auto weights = create_weight_vector(spec);
-  const auto [first, end] = spec.host_to_osd_range(spec.osd_to_host(mapping[0]));
-  for (auto i = first; i < end; ++i) {
-    weights[i] = CEPH_OSD_OUT;
-  }
-  return weights;
-}
-
-enum class mapping_change_t {
-  SAME,
-  FAILURE,
-  SAME_HOST,
-  NEW_HOST
-};
-void compare_mappings(
-  const cluster_test_spec_t &spec,
-  const std::vector<int> &before,
-  const std::vector<int> &after,
-  mapping_change_t expectation,
-  const std::pair<int, int> &range)
-{
-  const auto &[begin, end] = range;
-  for (auto i = begin; i < end; ++i) {
-    switch (expectation) {
-    case mapping_change_t::SAME:
-      EXPECT_EQ(before[i], after[i]);
-      break;
-    case mapping_change_t::FAILURE:
-      EXPECT_EQ(CRUSH_ITEM_NONE, after[i]);
-      break;
-    case mapping_change_t::SAME_HOST:
-      EXPECT_NE(before[i], after[i]);
-      if (!spec.check_osd(after[i])) {
-	spec.validate_osd(after[i]);
-      } else {
-	EXPECT_EQ(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
-      }
-      break;
-    case mapping_change_t::NEW_HOST:
-      EXPECT_NE(before[i], after[i]);
-      if (!spec.check_osd(after[i])) {
-	spec.validate_osd(after[i]);
-      } else {
-	EXPECT_NE(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
-      }
-      break;
-    }
-  }
-}
-
-std::vector<int> get_mapping(
-  const cluster_test_spec_t &spec,
-  CrushWrapper &c,
-  const std::vector<uint32_t> &weights,
-  int ruleno)
-{
-  std::vector<int> out;
-  c.do_rule(
-    ruleno, 0 /* seed */, out, spec.num_mapped_size,
-    weights,
-    0);
-  EXPECT_EQ(std::size(out), spec.num_mapped_size);
-  return out;
-}
-
-unsigned count_mapped(const auto &v) {
-  unsigned ret = 0;
-  for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
-  return ret;
-}
-
-TEST_F(CRUSHTest, msr_4_host_2_choose_rule) {
-  cluster_test_spec_t spec{3, 4, 3, 1, 3};
-  auto [rootno, c] = create_crush_heirarchy(cct, spec);
-
-  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
-  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
-  EXPECT_EQ(
-    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
-  EXPECT_EQ(
-    0,
-    c->set_rule_step_choose_msr(
-      ruleno, 2, 1, OSD_TYPE));
-  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
-
-  auto weights_all_in = create_weight_vector(spec);
-  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
-  for (auto i : before) { spec.validate_osd(i); }
-
-  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
-   * a retry of the previous step, so marking all of the osds on a host
-   * out will not cause positions mapped to that pg to remap.
-   * However, because the above is an MSR rule type, hitting an out osd
-   * will cause a retry of the previous steps as well.
-   * See https://tracker.ceph.com/issues/62214 for the original motivation */
-  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
-  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
-
-  CrushCompiler cc{*c, std::cout};
-  cc.decompile(std::cout);
-
-  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
-  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
-  fmt::print("before        : {}\n", fmt::join(before, ", "));
-  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
-
-  auto count_mapped = [](const auto &v) {
-    unsigned ret = 0;
-    for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
-    return ret;
-  };
-
-  EXPECT_EQ(count_mapped(before), count_mapped(after_host_out));
-
-  auto weights_osd_out = create_weight_vector_first_osd_out(spec, before);
-  auto after_osd_out = get_mapping(spec, *c, weights_osd_out, ruleno);
-  EXPECT_EQ(count_mapped(before), count_mapped(after_osd_out));
-}
-
-TEST_F(CRUSHTest, msr_2_host_2_osd) {
-  cluster_test_spec_t spec{2, 3, 2, 2, 3};
-  auto [rootno, c] = create_crush_heirarchy(cct, spec);
-
-  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
-  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
-  EXPECT_EQ(
-    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
-  EXPECT_EQ(
-    0,
-    c->set_rule_step_choose_msr(
-      ruleno, 2, spec.num_mapped_per_host, OSD_TYPE));
-  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
-
-  auto weights_all_in = create_weight_vector(spec);
-  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
-  for (auto i : before) { spec.validate_osd(i); }
-
-  fmt::print("before        : {}\n", fmt::join(before, ", "));
-  ASSERT_EQ(count_mapped(before), 3);
-
-  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
-   * a retry of the previous step, so marking all of the osds on a host
-   * out will not cause positions mapped to that pg to remap.
-   * However, because the above is an MSR rule type, hitting an out osd
-   * will cause a retry of the previous steps as well.
-   * See https://tracker.ceph.com/issues/62214 for the original motivation */
-  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
-  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
-
-  CrushCompiler cc{*c, std::cout};
-  cc.decompile(std::cout);
-
-  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
-  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
-  fmt::print("before        : {}\n", fmt::join(before, ", "));
-  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
-
-  compare_mappings(
-    spec, before, after_host_out, mapping_change_t::NEW_HOST,
-    {0, spec.num_mapped_per_host});
-  compare_mappings(
-    spec, before, after_host_out, mapping_change_t::SAME,
-    {spec.num_mapped_per_host, spec.num_mapped_size});
-}
-
-TEST_F(CRUSHTest, msr_5_host_8_6_ec_choose) {
-  cluster_test_spec_t spec{4, 5, 4, 4, 14};
-  auto [rootno, c] = create_crush_heirarchy(cct, spec);
-
-  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
-  unsigned step_id = 0;
-  EXPECT_EQ(0, c->set_rule_step_take(ruleno, step_id++, rootno));
-  EXPECT_EQ(
-    0,
-    c->set_rule_step_choose_msr(
-      ruleno, step_id++, spec.num_hosts_mapped, HOST_TYPE));
-  EXPECT_EQ(
-    0,
-    c->set_rule_step_choose_msr(
-      ruleno, step_id++, spec.num_mapped_per_host, OSD_TYPE));
-  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, step_id++));
-
-  auto weights_all_in = create_weight_vector(spec);
-  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
-  for (auto i : before) { spec.validate_osd(i); }
-
-  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
-   * a retry of the previous step, so marking all of the osds on a host
-   * out will not cause positions mapped to that pg to remap.
-   * However, because the above is an MSR rule type, hitting an out osd
-   * will cause a retry of the previous steps as well.
-   * See https://tracker.ceph.com/issues/62214 for the original motivation */
-  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
-  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
-
-  CrushCompiler cc{*c, std::cout};
-  cc.decompile(std::cout);
-
-  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
-  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
-  fmt::print("before        : {}\n", fmt::join(before, ", "));
-  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
-
-  compare_mappings(
-    spec, before, after_host_out, mapping_change_t::NEW_HOST,
-    {0, spec.num_mapped_per_host});
-  compare_mappings(
-    spec, before, after_host_out, mapping_change_t::SAME,
-    {spec.num_mapped_per_host, spec.num_mapped_size});
-}
-
-TEST_F(CRUSHTest, msr_multi_root) {
-  constexpr unsigned NUM_HOSTS = 4;
-  constexpr unsigned NUM_OSDS_PER_HOST = 3;
-
-  auto c = CrushWrapper();
-  c.create();
-  c.set_tunables_optimal();
-
-  c.set_type_name(ROOT_TYPE, "root");
-  c.set_type_name(HOST_TYPE, "host");
-  c.set_type_name(OSD_TYPE, "osd");
-
-  std::map<int, std::pair<std::string, std::string>> osd_id_to_host_root;
-  std::map<std::string, int> root_name_to_id;
-  std::map<std::string, std::vector<int>> host_name_to_osds;
-  unsigned next_osd_id = 0;
-
-  auto populate_root = [&](const auto &root_name) {
-    int rootno;
-    c.add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
-		 ROOT_TYPE, 0, NULL, NULL, &rootno);
-    c.set_item_name(rootno, root_name);
-    root_name_to_id[root_name] = rootno;
-
-    for (unsigned host_id = 0; host_id < NUM_HOSTS; ++host_id) {
-      const std::string host_name =
-	fmt::format("{}-host{}", root_name, host_id);
-      for (unsigned osd = 0; osd < NUM_OSDS_PER_HOST; ++osd) {
-	const int osd_id = next_osd_id++;
-	const std::string osd_name = fmt::format("{}-osd{}", root_name, osd_id);
-	auto ret = c.insert_item(
-	  cct, osd_id, 1.0, osd_name,
-	  {{ "root", root_name }, { "host", host_name }});
-	osd_id_to_host_root[osd_id] = std::make_pair(host_name, root_name);
-	host_name_to_osds[host_name].push_back(osd_id);
-	EXPECT_EQ(ret, 0);
-      }
-    }
-  };
-
-  int ruleno = 0;
-  int ret = c.add_rule(ruleno, 8, CRUSH_RULE_TYPE_MSR_INDEP);
-  ceph_assert(ret == ruleno);
-
-  unsigned step_id = 0;
-  auto populate_rule = [&](const auto &rule_name) {
-    ret = c.set_rule_step(
-      ruleno, step_id++, CRUSH_RULE_TAKE, root_name_to_id[rule_name], 0);
-    ceph_assert(ret == 0);
-    ret = c.set_rule_step(
-      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, HOST_TYPE);
-    ceph_assert(ret == 0);
-    ret = c.set_rule_step(
-      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, OSD_TYPE);
-    ceph_assert(ret == 0);
-    ret = c.set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
-    ceph_assert(ret == 0);
-  };
-
-  for (const auto &root_name : { "ssd", "hdd" }) {
-    populate_root(root_name);
-    populate_rule(root_name);
-  }
-  c.set_rule_name(ruleno, "rule_name");
-  c.finalize();
-
-  constexpr unsigned ACTING_SIZE = 8;
-  constexpr unsigned OSDS_PER_ROOT = 4;
-  constexpr unsigned OSDS_PER_HOST = 2;
-  auto validate_output = [&](const auto &out) {
-    std::set<std::string> hosts;
-    for (unsigned host = 0; host < (ACTING_SIZE / OSDS_PER_HOST); ++host) {
-      std::set<std::string> hosts_this_failure_domain;
-      unsigned start = host * OSDS_PER_HOST;
-      unsigned end = (host + 1) * OSDS_PER_HOST;
-      for (unsigned i = start; i < end; ++i) {
-	EXPECT_NE(out[i], CRUSH_ITEM_NONE);
-	EXPECT_EQ(osd_id_to_host_root.count(out[i]), 1);
-	const auto &[host_name, root_name] = osd_id_to_host_root[out[start]];
-	EXPECT_EQ(i < OSDS_PER_ROOT ? "ssd" : "hdd", root_name);
-	hosts_this_failure_domain.insert(host_name);
-      }
-      for (const auto &i: hosts_this_failure_domain) {
-	EXPECT_EQ(hosts.count(i), 0);
-	hosts.insert(i);
-      }
-    }
-  };
-
-  const std::vector<uint32_t> all_in(next_osd_id, CEPH_OSD_IN);
-  for (int x = 0; x < 1000; ++x) {
-    std::vector<int> out;
-    c.do_rule(ruleno, x, out, 8, all_in, 0);
-    EXPECT_EQ(count_mapped(out), 8);
-    validate_output(out);
-
-    {
-      std::vector<uint32_t> osds_out_weight = all_in;
-      std::set<unsigned> osd_idx_out{{1, 5}};
-      for (const auto &i: osd_idx_out) {
-	osds_out_weight[out[i]] = CEPH_OSD_OUT;
-      }
-      std::vector<int> osds_out;
-      c.do_rule(ruleno, x, osds_out, 8, osds_out_weight, 0);
-      EXPECT_EQ(count_mapped(osds_out), 8);
-      validate_output(osds_out);
-      for (unsigned i = 0; i < osds_out.size(); ++i) {
-	if (osd_idx_out.count(i)) {
-	  EXPECT_NE(osds_out[i], out[i]);
-	} else {
-	  EXPECT_EQ(osds_out[i], out[i]);
-	}
-      }
-    }
-
-    {
-      std::vector<uint32_t> hosts_out_weight = all_in;
-      std::set<unsigned> osd_ids_out;
-
-      for (const auto &i : {2, 6}) {
-	const auto &[host_name, _] = osd_id_to_host_root[out[i]];
-	for (const auto &osd_id: host_name_to_osds[host_name]) {
-	  osd_ids_out.insert(osd_id);
-	  hosts_out_weight[osd_id] = CEPH_OSD_OUT;
-	}
-      }
-
-      std::vector<int> hosts_out;
-      c.do_rule(ruleno, x, hosts_out, 8, hosts_out_weight, 0);
-      EXPECT_EQ(count_mapped(hosts_out), 8);
-      validate_output(hosts_out);
-      for (unsigned i = 0; i < hosts_out.size(); ++i) {
-	if (osd_ids_out.count(out[i])) {
-	  EXPECT_NE(hosts_out[i], out[i]);
-	} else {
-	  EXPECT_EQ(hosts_out[i], out[i]);
-	}
-      }
-    }
-  }
-}
diff --git a/src/vstart.sh b/src/vstart.sh
index 13155003ab47..0c9ef3237791 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -176,9 +176,6 @@ zoned_enabled=0
 io_uring_enabled=0
 with_jaeger=0
 force_addr=0
-osds_per_host=0
-require_osd_and_client_version=""
-use_crush_tunables=""
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -602,21 +599,6 @@ case $1 in
         with_jaeger=1
         echo "with_jaeger $with_jaeger"
         ;;
-    --osds-per-host)
-        osds_per_host="$2"
-        shift
-        echo "osds_per_host $osds_per_host"
-        ;;
-    --require-osd-and-client-version)
-        require_osd_and_client_version="$2"
-        shift
-        echo "require_osd_and_client_version $require_osd_and_client_version"
-        ;;
-    --use-crush-tunables)
-        use_crush_tunables="$2"
-        shift
-        echo "use_crush_tunables $use_crush_tunables"
-        ;;
     *)
         usage_exit
 esac
@@ -1113,15 +1095,6 @@ EOF
     if [ "$crimson" -eq 1 ]; then
         $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it
     fi
-
-    if [ -n "$require_osd_and_client_version" ]; then
-        $CEPH_BIN/ceph osd set-require-min-compat-client $require_osd_and_client_version
-        $CEPH_BIN/ceph osd require-osd-release $require_osd_and_client_version --yes-i-really-mean-it
-    fi
-
-    if [ -n "$use_crush_tunables" ]; then
-        $CEPH_BIN/ceph osd crush tunables $use_crush_tunables
-    fi
 }
 
 start_osd() {
@@ -1155,13 +1128,6 @@ start_osd() {
 [osd.$osd]
         host = $HOSTNAME
 EOF
-
-            if [ "$osds_per_host" -gt 0 ]; then
-                wconf <<EOF
-        crush location = root=default host=$HOSTNAME-$(echo "$osd / $osds_per_host" | bc)
-EOF
-            fi
-
             if [ "$spdk_enabled" -eq 1 ]; then
                 wconf <<EOF
         bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}

From 0985e201342fa53c014a811156aed661b4b8f994 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 8 Nov 2023 16:43:46 +0000
Subject: [PATCH 1629/2492] ceph-volume: use 'no workqueue' options with
 dmcrypt

CloudFlare engineers made some testing and realized that using
workqueues with encryption on flash devices has a bad effect.

See [1] for details.

With this patch it will make ceph-volume call crypsetup with
`--perf-no_read_workqueue` and `--perf-no_write_workqueue` options
when the device is not a rotational.

[1] https://blog.cloudflare.com/speeding-up-linux-disk-encryption/

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
Co-Authored-by: Stefan Kooman <stefan@kooman.org>
Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 ceph.spec.in                                  |  1 +
 src/ceph-volume/ceph_volume/__init__.py       |  3 ++-
 .../ceph_volume/devices/lvm/activate.py       |  2 ++
 .../ceph_volume/devices/lvm/batch.py          |  2 +-
 .../ceph_volume/devices/lvm/common.py         |  2 +-
 .../ceph_volume/devices/raw/common.py         |  2 +-
 .../ceph_volume/util/arg_validators.py        | 11 +++++++-
 src/ceph-volume/ceph_volume/util/device.py    | 10 +++----
 .../ceph_volume/util/encryption.py            | 27 +++++++++++++++++++
 9 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 058c79c5e7f8..58ccf067997a 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -902,6 +902,7 @@ Requires: parted
 Requires: util-linux
 Requires: xfsprogs
 Requires: python%{python3_pkgversion}-setuptools
+Requires: python%{python3_pkgversion}-packaging
 Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release}
 %description volume
 This package contains a tool to deploy OSD with different devices like
diff --git a/src/ceph-volume/ceph_volume/__init__.py b/src/ceph-volume/ceph_volume/__init__.py
index 500de2237365..598f0b27ef9b 100644
--- a/src/ceph-volume/ceph_volume/__init__.py
+++ b/src/ceph-volume/ceph_volume/__init__.py
@@ -14,8 +14,9 @@ class UnloadedConfig(object):
     def __getattr__(self, *a):
         raise RuntimeError("No valid ceph configuration file was loaded.")
 
-conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path'])
+conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path', 'dmcrypt_no_workqueue'])
 conf.ceph = UnloadedConfig()
+conf.dmcrypt_no_workqueue = None
 
 __version__ = "1.0.0"
 
diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
index feb91053b447..17c66194c67e 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py
@@ -69,6 +69,8 @@ def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False):
         raise RuntimeError('could not find a bluestore OSD to activate')
 
     is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
+    if is_encrypted and conf.dmcrypt_no_workqueue is None:
+        encryption_utils.set_dmcrypt_no_workqueue()
     dmcrypt_secret = None
     osd_id = osd_block_lv.tags['ceph.osd_id']
     conf.cluster = osd_block_lv.tags['ceph.cluster_name']
diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
index 69a3f672b482..2118ce47aeea 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py
@@ -256,7 +256,7 @@ def __init__(self, argv):
         )
         parser.add_argument(
             '--dmcrypt',
-            action='store_true',
+            action=arg_validators.DmcryptAction,
             help='Enable device encryption via dm-crypt',
         )
         parser.add_argument(
diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py
index 35e53181aff0..198ba9417a1b 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/common.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py
@@ -73,7 +73,7 @@ def rollback_osd(args, osd_id=None):
         'default': "",
     },
     '--dmcrypt': {
-        'action': 'store_true',
+        'action': arg_validators.DmcryptAction,
         'help': 'Enable device encryption via dm-crypt',
     },
     '--no-systemd': {
diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py
index 89ee285be5b4..4863b9e18e05 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/common.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/common.py
@@ -46,7 +46,7 @@ def create_parser(prog, description):
     )
     parser.add_argument(
         '--dmcrypt',
-        action='store_true',
+        action=arg_validators.DmcryptAction,
         help='Enable device encryption via dm-crypt',
     )
     parser.add_argument(
diff --git a/src/ceph-volume/ceph_volume/util/arg_validators.py b/src/ceph-volume/ceph_volume/util/arg_validators.py
index 1abb5165ec00..e936cab895e3 100644
--- a/src/ceph-volume/ceph_volume/util/arg_validators.py
+++ b/src/ceph-volume/ceph_volume/util/arg_validators.py
@@ -4,11 +4,20 @@
 from ceph_volume import terminal, decorators, process
 from ceph_volume.util.device import Device
 from ceph_volume.util import disk
-
+from ceph_volume.util.encryption import set_dmcrypt_no_workqueue
+from ceph_volume import process, conf
 
 def valid_osd_id(val):
     return str(int(val))
 
+class DmcryptAction(argparse._StoreTrueAction):
+    def __init__(self, *args, **kwargs):
+        super(DmcryptAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        set_dmcrypt_no_workqueue()
+        super(DmcryptAction, self).__call__(*args, **kwargs)
+
 class ValidDevice(object):
 
     def __init__(self, as_string=False, gpt_ok=False):
diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index c3de2a97fd99..c29821d02e45 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -121,13 +121,8 @@ def __init__(self, path, with_lsm=False, lvs=None, lsblk_all=None, all_devices_v
             # check if we are not a device mapper
             if "dm-" not in real_path:
                 self.path = real_path
-        if not sys_info.devices:
-            if self.path:
-                sys_info.devices = disk.get_devices(device=self.path)
-            else:
-                sys_info.devices = disk.get_devices()
-        if sys_info.devices.get(self.path, {}):
-            self.device_nodes = sys_info.devices[self.path]['device_nodes']
+        if not sys_info.devices.get(self.path):
+            sys_info.devices = disk.get_devices()
         self.sys_api = sys_info.devices.get(self.path, {})
         self.partitions = self._get_partitions()
         self.lv_api = None
@@ -143,6 +138,7 @@ def __init__(self, path, with_lsm=False, lvs=None, lsblk_all=None, all_devices_v
         self._is_lvm_member = None
         self.ceph_device = False
         self._parse()
+        self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.lsm_data = self.fetch_lsm(with_lsm)
 
         self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons()
diff --git a/src/ceph-volume/ceph_volume/util/encryption.py b/src/ceph-volume/ceph_volume/util/encryption.py
index f8aea80b4935..844a81620d22 100644
--- a/src/ceph-volume/ceph_volume/util/encryption.py
+++ b/src/ceph-volume/ceph_volume/util/encryption.py
@@ -6,10 +6,28 @@
 from ceph_volume.util.device import Device
 from .prepare import write_keyring
 from .disk import lsblk, device_family, get_part_entry_type
+from packaging import version
 
 logger = logging.getLogger(__name__)
 mlogger = terminal.MultiLogger(__name__)
 
+def set_dmcrypt_no_workqueue(target_version: str = '2.3.4') -> None:
+    """
+    set `conf.dmcrypt_no_workqueue` to `True` if the available
+    version of `cryptsetup` is greater or equal to `version`
+    """
+    command = ["cryptsetup", "--version"]
+    out, err, rc = process.call(command)
+    try:
+        if version.parse(out[0]) >= version.parse(f'cryptsetup {target_version}'):
+            conf.dmcrypt_no_workqueue = True
+    except IndexError:
+        mlogger.debug(f'cryptsetup version check: rc={rc} out={out} err={err}')
+        raise RuntimeError("Couldn't check the cryptsetup version.")
+
+def bypass_workqueue(device: str) -> bool:
+    return not Device(device).rotational and conf.dmcrypt_no_workqueue
+
 def get_key_size_from_conf():
     """
     Return the osd dmcrypt key size from config file.
@@ -79,6 +97,10 @@ def plain_open(key, device, mapping):
         '--key-size', '256',
     ]
 
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)
 
 
@@ -103,6 +125,11 @@ def luks_open(key, device, mapping):
         device,
         mapping,
     ]
+
+    if bypass_workqueue(device):
+        command.extend(['--perf-no_read_workqueue',
+                        '--perf-no_write_workqueue'])
+
     process.call(command, stdin=key, terminal_verbose=True, show_command=True)
 
 

From 15bd38eecee66670ad6eb41fac3099bfad69c196 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Thu, 18 Jan 2024 18:55:28 +0000
Subject: [PATCH 1630/2492] mgr: add CephReleases class to sustainably compare
 releases

Changes how the upmap balancer compares min_mon_release
to account for release names eventually wrapping around the alphabet.

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/pybind/mgr/balancer/module.py | 18 ++++++++++++------
 src/pybind/mgr/mgr_module.py      | 21 +++++++++++++++++++++
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 590f24653f0c..2cbaf10c09fe 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -374,13 +374,19 @@ def set_mode(self, mode: Mode) -> Tuple[int, str, str]:
         """
         Set balancer mode
         """
+        min_compat_client = self.get_osdmap().dump().get('require_min_compat_client', '')
         if mode == Mode.upmap:
-            min_compat_client = self.get_osdmap().dump().get('require_min_compat_client', '')
-            if min_compat_client < 'luminous':  # works well because version is alphabetized..
-                warn = ('min_compat_client "%s" '
-                        '< "luminous", which is required for pg-upmap. '
-                        'Try "ceph osd set-require-min-compat-client luminous" '
-                        'before enabling this mode' % min_compat_client)
+            try:
+                release = CephReleases[min_compat_client]
+                if release.value < CephReleases.luminous.value:
+                    warn = ('min_compat_client "%s" '
+                            '< "luminous", which is required for pg-upmap. '
+                            'Try "ceph osd set-require-min-compat-client luminous" '
+                            'before enabling this mode' % min_compat_client)
+                    return (-errno.EPERM, '', warn)
+            except KeyError:
+                self.log.error('Unable to apply mode {} due to unknown min_compat_client {}'.format(mode, min_compat_client))
+                warn = ('Unable to apply mode {} due to unknown min_compat_client {}.'.format(mode, min_compat_client))
                 return (-errno.EPERM, '', warn)
         elif mode == Mode.crush_compat:
             ms = MappingState(self.get_osdmap(),
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 6c83f2771619..ef908bd81d38 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -83,6 +83,27 @@ def get_origin(tp: Any) -> Any:
 NFS_GANESHA_SUPPORTED_FSALS = ['CEPH', 'RGW']
 NFS_POOL_NAME = '.nfs'
 
+class CephReleases(IntEnum):
+    argonaut = 1
+    bobtail = 2
+    cuttlefish = 3
+    dumpling = 4
+    emperor = 5
+    firefly = 6
+    giant = 7
+    hammer = 8
+    infernalis = 9
+    jewel = 10
+    kraken = 11
+    luminous = 12
+    mimic = 13
+    nautilus = 14
+    octopus = 15
+    pacific = 16
+    quincy = 17
+    reef = 18
+    squid = 19
+    maximum = 20
 
 class NotifyType(str, Enum):
     mon_map = 'mon_map'

From f72100bbd17539d9774ae72215afefee16f20775 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 26 Jan 2024 21:35:18 +0100
Subject: [PATCH 1631/2492] ceph-volume: fix partitions support in
 disk.get_devices()

The following:
```
is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1"
```
assumes any `/sys/dev/block/x:y/partition` contains '1' which is wrong.
This file actually contains the corresponding partition number.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-volume/ceph_volume/util/device.py |  3 +-
 src/ceph-volume/ceph_volume/util/disk.py   | 59 ++++++++++------------
 2 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py
index c29821d02e45..1b52774d1a1b 100644
--- a/src/ceph-volume/ceph_volume/util/device.py
+++ b/src/ceph-volume/ceph_volume/util/device.py
@@ -138,7 +138,8 @@ def __init__(self, path, with_lsm=False, lvs=None, lsblk_all=None, all_devices_v
         self._is_lvm_member = None
         self.ceph_device = False
         self._parse()
-        self.device_nodes = sys_info.devices[self.path]['device_nodes']
+        if self.path in sys_info.devices.keys():
+            self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.lsm_data = self.fetch_lsm(with_lsm)
 
         self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons()
diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index 0bbd5336d5aa..3965d576d6e0 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -6,6 +6,7 @@
 from ceph_volume import process
 from ceph_volume.api import lvm
 from ceph_volume.util.system import get_file_contents
+from typing import Dict, List
 
 
 logger = logging.getLogger(__name__)
@@ -751,34 +752,34 @@ def __call__(cls):
 allow_loop_devices = AllowLoopDevices()
 
 
-def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block', device=''):
-    def holder_inner_loop():
+def get_block_devs_sysfs(_sys_block_path: str = '/sys/block', _sys_dev_block_path: str = '/sys/dev/block', device: str = '') -> List[List[str]]:
+    def holder_inner_loop() -> bool:
         for holder in holders:
             # /sys/block/sdy/holders/dm-8/dm/uuid
-            holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower()
+            holder_dm_type: str = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower()
             if holder_dm_type == 'mpath':
                 return True
 
     # First, get devices that are _not_ partitions
-    result = list()
+    result: List[List[str]] = list()
     if not device:
-        dev_names = os.listdir(_sys_block_path)
+        dev_names: List[str] = os.listdir(_sys_block_path)
     else:
         dev_names = [device]
     for dev in dev_names:
-        name = kname = os.path.join("/dev", dev)
+        name = kname = pname = os.path.join("/dev", dev)
         if not os.path.exists(name):
             continue
-        type_ = 'disk'
-        holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
+        type_: str = 'disk'
+        holders: List[str] = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
         if holder_inner_loop():
             continue
-        dm_dir_path = os.path.join(_sys_block_path, dev, 'dm')
+        dm_dir_path: str = os.path.join(_sys_block_path, dev, 'dm')
         if os.path.isdir(dm_dir_path):
-            dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid'))
-            type_ = dm_type.split('-')[0].lower()
-            basename = get_file_contents(os.path.join(dm_dir_path, 'name'))
-            name = os.path.join("/dev/mapper", basename)
+            dm_type: str = get_file_contents(os.path.join(dm_dir_path, 'uuid'))
+            type_: List[str] = dm_type.split('-')[0].lower()
+            basename: str = get_file_contents(os.path.join(dm_dir_path, 'name'))
+            name: str = os.path.join("/dev/mapper", basename)
         if dev.startswith('loop'):
             if not allow_loop_devices():
                 continue
@@ -786,28 +787,25 @@ def holder_inner_loop():
             if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')):
                 continue
             type_ = 'loop'
-        result.append([kname, name, type_])
+        result.append([kname, name, type_, pname])
     # Next, look for devices that _are_ partitions
-    for item in os.listdir(_sys_dev_block_path):
-        is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1"
-        dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item)))
-        if not is_part:
-            continue
-        name = kname = os.path.join("/dev", dev)
-        result.append([name, kname, "part"])
+    partitions: Dict[str, str] = get_partitions()
+    for partition in partitions.keys():
+        name = kname = os.path.join("/dev", partition)
+        result.append([name, kname, "part", partitions[partition]])
     return sorted(result, key=lambda x: x[0])
 
-def get_partitions(_sys_dev_block_path ='/sys/dev/block'):
-    devices = os.listdir(_sys_dev_block_path)
-    result = dict()
+def get_partitions(_sys_dev_block_path ='/sys/dev/block') -> List[str]:
+    devices: List[str] = os.listdir(_sys_dev_block_path)
+    result: Dict[str, str] = dict()
     for device in devices:
-        device_path = os.path.join(_sys_dev_block_path, device)
-        is_partition = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0
+        device_path: str = os.path.join(_sys_dev_block_path, device)
+        is_partition: bool = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0
         if not is_partition:
             continue
 
-        partition_sys_name = os.path.basename(os.readlink(device_path))
-        parent_device_sys_name = os.readlink(device_path).split('/')[-2:-1][0]
+        partition_sys_name: str = os.path.basename(os.path.realpath(device_path))
+        parent_device_sys_name: str = os.path.realpath(device_path).split('/')[-2:-1][0]
         result[partition_sys_name] = parent_device_sys_name
     return result
 
@@ -825,7 +823,6 @@ def get_devices(_sys_block_path='/sys/block', device=''):
     device_facts = {}
 
     block_devs = get_block_devs_sysfs(_sys_block_path)
-    partitions = get_partitions()
 
     block_types = ['disk', 'mpath', 'lvm', 'part']
     if allow_loop_devices():
@@ -840,7 +837,7 @@ def get_devices(_sys_block_path='/sys/block', device=''):
             continue
         sysdir = os.path.join(_sys_block_path, devname)
         if block[2] == 'part':
-            sysdir = os.path.join(_sys_block_path, partitions[devname], devname)
+            sysdir = os.path.join(_sys_block_path, block[3], devname)
         metadata = {}
 
         # If the device is ceph rbd it gets excluded
@@ -877,7 +874,7 @@ def get_devices(_sys_block_path='/sys/block', device=''):
             metadata['device_nodes'] = ','.join(device_slaves)
         else:
             if block[2] == 'part':
-                metadata['device_nodes'] = partitions[devname]
+                metadata['device_nodes'] = block[3]
             else:
                 metadata['device_nodes'] = devname
 

From 9f271093f4331381dc024cb4309f9f486d366818 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 27 Jan 2024 18:40:20 +1000
Subject: [PATCH 1632/2492] doc/radosgw: fix verb disagreement - index.html

Fix a tricky verb disagreement and rewrite a few sentences for what I
hope is greater clarity.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/index.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst
index ed67413646d8..f53917ad59bc 100644
--- a/doc/radosgw/index.rst
+++ b/doc/radosgw/index.rst
@@ -15,13 +15,13 @@ Storage Clusters. :term:`Ceph Object Storage` supports two interfaces:
    that is compatible with a large subset of the OpenStack Swift API.
 
 Ceph Object Storage uses the Ceph Object Gateway daemon (``radosgw``), an HTTP
-server designed for interacting with a Ceph Storage Cluster. The Ceph Object
+server designed to interact with a Ceph Storage Cluster. The Ceph Object
 Gateway provides interfaces that are compatible with both Amazon S3 and
 OpenStack Swift, and it has its own user management. Ceph Object Gateway can
-store data in the same Ceph Storage Cluster in which data from Ceph File System
-clients and Ceph Block Device clients is stored. The S3 API and the Swift API
-share a common namespace, which makes it possible to write data to a Ceph
-Storage Cluster with one API and then retrieve that data with the other API.
+use a single Ceph Storage cluster to store data from Ceph File System and from
+Ceph Block device clients. The S3 API and the Swift API share a common
+namespace, which means that it is possible to write data to a Ceph Storage
+Cluster with one API and then retrieve that data with the other API.
 
 .. ditaa::
 

From c6c05ab639f90d86bc65d5324fcfc87077b9a62b Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 4 Jan 2024 10:18:44 -0600
Subject: [PATCH 1633/2492] osd/scrub: add synchronous request to AsyncReserver
 API

To be used when handling replica reservation requests from "old"
primaries, that expect an immediate grant/deny reply.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/AsyncReserver.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/common/AsyncReserver.h b/src/common/AsyncReserver.h
index b80f9e7df8f6..1a19dc70d1bb 100644
--- a/src/common/AsyncReserver.h
+++ b/src/common/AsyncReserver.h
@@ -264,6 +264,39 @@ class AsyncReserver {
     do_queues();
   }
 
+  /**
+   * The synchronous version of request_reservation
+   * Used to handle requests from OSDs that do not support the async interface
+   * to scrub replica reservations, but still must count towards the max
+   * active reservations.
+   */
+  bool request_reservation_or_fail(
+      T item,		     ///< [in] reservation key
+      Context *on_reserved   ///< [in] callback to be called on reservation
+  )
+  {
+    std::lock_guard l(lock);
+    ceph_assert(!queue_pointers.count(item) && !in_progress.count(item));
+
+    if (in_progress.size() >= max_allowed) {
+      rdout(10) << fmt::format("{}: request: {} denied", __func__, item)
+		<< dendl;
+      return false;
+    }
+
+    const unsigned prio = UINT_MAX;
+    Reservation r(item, prio, on_reserved, nullptr);
+    queues[prio].push_back(r);
+    queue_pointers.insert(std::make_pair(
+	item, std::make_pair(prio, --(queues[prio]).end())));
+    do_queues();
+    // the new request should be in_progress now
+    ceph_assert(in_progress.count(item));
+    rdout(10) << fmt::format("{}: request: {} granted", __func__, item)
+	      << dendl;
+    return true;
+  }
+
   /**
    * Cancels reservation
    *

From c61bca6d6bac33e6d5c5af8b0f9ec77a229179d3 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Fri, 5 Jan 2024 09:07:19 -0600
Subject: [PATCH 1634/2492] osd/scrub: add "queue my request" flag to replica
 reservation messages

Up-to-date primaries will set this flag when sending a reservation
request. The replica OSD, if too busy to handle the request immediately, will queue
it until such time that the number of concurrent reservations is below the
configured limit. The queued requests are honored in FIFO order.

Old primaries will not set this flag, and will receive the expected
grant or deny reply immediately.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/messages/MOSDScrubReserve.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/messages/MOSDScrubReserve.h b/src/messages/MOSDScrubReserve.h
index 97b6dff4dd00..a2670a8e110f 100644
--- a/src/messages/MOSDScrubReserve.h
+++ b/src/messages/MOSDScrubReserve.h
@@ -19,7 +19,7 @@
 
 class MOSDScrubReserve : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 2;
+  static constexpr int HEAD_VERSION = 3;
   static constexpr int COMPAT_VERSION = 1;
 public:
   using reservation_nonce_t = uint32_t;
@@ -35,6 +35,9 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   int32_t type;
   pg_shard_t from;
   reservation_nonce_t reservation_nonce{0};
+  /// 'false' if the (legacy) primary is expecting an immediate
+  /// granted / denied response
+  bool wait_for_resources{false};
 
   epoch_t get_map_epoch() const override {
     return map_epoch;
@@ -48,12 +51,12 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
       map_epoch(0), type(-1) {}
   MOSDScrubReserve(spg_t pgid,
 		   epoch_t map_epoch,
-		   int type,
+		   ReserveMsgOp type_code,
 		   pg_shard_t from,
 		   reservation_nonce_t nonce)
     : MOSDFastDispatchOp{MSG_OSD_SCRUB_RESERVE, HEAD_VERSION, COMPAT_VERSION},
       pgid(pgid), map_epoch(map_epoch),
-      type(type), from(from), reservation_nonce{nonce} {}
+      type(static_cast<int32_t>(type_code)), from(from), reservation_nonce{nonce} {}
 
   std::string_view get_type_name() const {
     return "MOSDScrubReserve";
@@ -63,7 +66,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     out << "MOSDScrubReserve(" << pgid << " ";
     switch (type) {
     case REQUEST:
-      out << "REQUEST ";
+      out << (wait_for_resources ? "QREQUEST " : "REQUEST ");
       break;
     case GRANT:
       out << "GRANT ";
@@ -89,10 +92,16 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     decode(from, p);
     if (header.version >= 2) {
       decode(reservation_nonce, p);
+      if (header.version >= 3) {
+        decode(wait_for_resources, p);
+      } else {
+        wait_for_resources = false;
+      }
     } else {
       // a zero nonce (identifying legacy senders) is ignored when
       // checking the request for obsolescence
       reservation_nonce = 0;
+      wait_for_resources = false;
     }
   }
 
@@ -103,6 +112,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     encode(type, payload);
     encode(from, payload);
     encode(reservation_nonce, payload);
+    encode(wait_for_resources, payload);
   }
 private:
   template<class T, typename... Args>

From 5970ff6637014deb8eb24fdcad8517a8e06f3ab0 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 7 Jan 2024 08:20:46 -0600
Subject: [PATCH 1635/2492] osd/scrub: add required sub-states to handle queued
 reservation requests

The scrub async reserver is not yet used. All requests are treated as
'legacy' requests, i.e. requests that expect an immediate grant/deny
reply.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc      |   5 +
 src/osd/scrubber/osd_scrub.h       |   1 +
 src/osd/scrubber/scrub_machine.cc  | 321 +++++++++++++++++++++++------
 src/osd/scrubber/scrub_machine.h   | 207 ++++++++++++++-----
 src/osd/scrubber/scrub_resources.h |  30 +++
 5 files changed, 451 insertions(+), 113 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index bc65d64bb422..2ff70dee56d1 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -473,6 +473,11 @@ bool OsdScrub::inc_scrubs_remote(pg_t pgid)
   return m_resource_bookkeeper.inc_scrubs_remote(pgid);
 }
 
+void OsdScrub::enqueue_remote_reservation(pg_t pgid)
+{
+  m_resource_bookkeeper.enqueue_remote_reservation(pgid);
+}
+
 void OsdScrub::dec_scrubs_remote(pg_t pgid)
 {
   m_resource_bookkeeper.dec_scrubs_remote(pgid);
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 12e9db312312..2701a762f798 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -70,6 +70,7 @@ class OsdScrub {
       bool is_high_priority);
   void dec_scrubs_local();
   bool inc_scrubs_remote(pg_t pgid);
+  void enqueue_remote_reservation(pg_t pgid);
   void dec_scrubs_remote(pg_t pgid);
 
   // counting the number of PGs stuck while scrubbing, waiting for objects
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index d9d03fe68896..fc5238186868 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -331,8 +331,8 @@ ActiveScrubbing::ActiveScrubbing(my_context ctx)
   auto& session = context<Session>();
 
   session.m_perf_set->inc(scrbcnt_active_started);
-  scrbr->get_clog()->debug() << fmt::format(
-    "{} {} starts", machine.m_pg_id, scrbr->get_op_mode_text());
+  scrbr->get_clog()->debug()
+      << fmt::format("{} {} starts", pg_id, scrbr->get_op_mode_text());
 
   scrbr->on_init();
 }
@@ -747,6 +747,7 @@ ScrubMachine::ScrubMachine(PG* pg, ScrubMachineListener* pg_scrub)
 
 ScrubMachine::~ScrubMachine() = default;
 
+
 // -------- for replicas -----------------------------------------------------
 
 // ----------------------- ReplicaActive --------------------------------
@@ -759,6 +760,8 @@ ReplicaActive::ReplicaActive(my_context ctx)
   dout(10) << "-- state -->> ReplicaActive" << dendl;
   m_pg = scrbr->get_pg();
   m_osds = m_pg->get_pg_osd(ScrubberPasskey());
+  clear_shallow_history<ReplicaIdle, 0>();
+  clear_shallow_history<ReplicaActive, 0>();
 }
 
 ReplicaActive::~ReplicaActive()
@@ -767,11 +770,10 @@ ReplicaActive::~ReplicaActive()
   if (reserved_by_my_primary) {
     dout(10) << "ReplicaActive::~ReplicaActive(): clearing reservation"
 	     << dendl;
-    clear_reservation_by_remote_primary();
+    clear_reservation_by_remote_primary(false);
   }
 }
 
-
 /*
  * Note: we are expected to be in the initial internal state (Idle) when
  * receiving any registration request. Our other internal states, the
@@ -780,105 +782,299 @@ ReplicaActive::~ReplicaActive()
  *
  * Process:
  * - if already reserved: clear existing reservation, then continue
- * - ask the OSD for the "reservation resource"
- * - if granted: mark it internally and notify the Primary.
- * - otherwise: just notify the requesting primary.
+ * - for async requests:
+ *   - enqueue the request with reserver;
+ *   - move to the ReplicaWaitingReservation state
+ *   - no reply is expected by the caller
+ * - for legacy requests:
+ *   - ask the OSD for the "reservation resource"
+ *   - if granted: move to ReplicaReserved and notify the Primary.
+ *   - otherwise: just notify the requesting primary.
+ *
+ * implementation note: sc::result objects cannot be copied or moved. Thus,
+ * we've resorted to returning a code indicating the next action.
  */
-void ReplicaActive::on_reserve_req(const ReplicaReserveReq& ev)
+ReplicaReactCode ReplicaActive::on_reserve_request(
+    const ReplicaReserveReq& ev,
+    bool async_request)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   const auto m = ev.m_op->get_req<MOSDScrubReserve>();
   const auto msg_nonce = m->reservation_nonce;
-  dout(10)
-      << fmt::format(
-	     "ReplicaActive::on_reserve_req() from {} (reservation_nonce:{})",
-	     ev.m_from, msg_nonce)
-      << dendl;
+  dout(10) << fmt::format(
+		  "ReplicaActive::on_reserve_req() from {} request:{} is "
+		  "async?{} (reservation_nonce:{})",
+		  ev.m_from, ev, async_request, msg_nonce)
+	   << dendl;
+  auto& svc = m_osds->get_scrub_services();  // shorthand
 
   if (reserved_by_my_primary) {
-    dout(10) << "ReplicaActive::on_reserve_req(): already reserved" << dendl;
-    // clear the existing reservation
-    clear_reservation_by_remote_primary();  // clears the flag, too
+    dout(10) << "ReplicaActive::on_reserve_request(): already reserved"
+	     << dendl;
+    // clear the existing reservation. Clears the flag, too
+    clear_reservation_by_remote_primary(false);
   }
 
-  // ask the OSD for the reservation
-  const auto ret = get_remote_reservation();
-  if (ret.granted) {
-    reserved_by_my_primary = true;
-    dout(10) << fmt::format("{}: reserved? yes", __func__) << dendl;
+  Message* reply{nullptr};
+  ReplicaReactCode next_action{ReplicaReactCode::discard};
+
+  if (async_request) {
+    // the request is to be handled asynchronously
+    svc.enqueue_remote_reservation(pg_id.pgid);
+    next_action = ReplicaReactCode::goto_waiting_reservation;
+
   } else {
-    dout(10) << fmt::format("{}: reserved? no ({})", __func__, ret.error_msg)
-	     << dendl;
+    // an immediate yes/no is required
+    const auto granted = svc.inc_scrubs_remote(scrbr->get_spgid().pgid);
+    if (granted) {
+      reserved_by_my_primary = true;
+      dout(10) << fmt::format("{}: reserved? yes", __func__) << dendl;
+      reply = new MOSDScrubReserve(
+	  spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch,
+	  MOSDScrubReserve::GRANT, m_pg->pg_whoami, msg_nonce);
+      next_action = ReplicaReactCode::goto_replica_reserved;
+
+    } else {
+      dout(10) << fmt::format("{}: reserved? no", __func__) << dendl;
+      reply = new MOSDScrubReserve(
+	  spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch,
+	  MOSDScrubReserve::REJECT, m_pg->pg_whoami, msg_nonce);
+      // the event is discarded
+      next_action = ReplicaReactCode::discard;
+    }
   }
 
-  Message* reply = new MOSDScrubReserve(
-      spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch, ret.op,
-      m_pg->pg_whoami, msg_nonce);
-  m_osds->send_message_osd_cluster(reply, ev.m_op->get_req()->get_connection());
+  if (reply) {
+    m_osds->send_message_osd_cluster(
+	reply, ev.m_op->get_req()->get_connection());
+  }
+  return next_action;
 }
 
-
 void ReplicaActive::on_release(const ReplicaRelease& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  if (!reserved_by_my_primary) {
-    dout(5) << fmt::format(
-		   "ReplicaActive::on_release() from {}: not reserved!",
-		   ev.m_from)
-	    << dendl;
-    return;
-  }
   dout(10) << fmt::format("ReplicaActive::on_release() from {}", ev.m_from)
 	   << dendl;
-  clear_reservation_by_remote_primary();
+  clear_reservation_by_remote_primary(true);
 }
 
-
-ReplicaActive::ReservationAttemptRes ReplicaActive::get_remote_reservation()
+void ReplicaActive::clear_reservation_by_remote_primary(bool log_failure)
 {
-  using ReservationAttemptRes = ReplicaActive::ReservationAttemptRes;
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << fmt::format(
+		  "ReplicaActive::clear_reservation_by_remote_primary(): was "
+		  "reserved? {}",
+		  (reserved_by_my_primary ? "yes" : "no"))
+	   << dendl;
+  if (reserved_by_my_primary) {
+    m_osds->get_scrub_services().dec_scrubs_remote(scrbr->get_spgid().pgid);
+    reserved_by_my_primary = false;
+  } else if (log_failure) {
+    const auto msg = fmt::format(
+	"ReplicaActive::clear_reservation_by_remote_primary(): "
+	"not reserved!");
+    dout(5) << msg << dendl;
+    scrbr->get_clog()->warn() << msg;
+  }
+}
 
-  if (m_osds->get_scrub_services().inc_scrubs_remote(scrbr->get_spgid().pgid)) {
-    return ReservationAttemptRes{MOSDScrubReserve::GRANT, "", true};
-  } else {
-    return ReservationAttemptRes{
-	MOSDScrubReserve::REJECT, "failed to reserve remotely", false};
+
+// ---------------- ReplicaActive/ReplicaIdle ---------------------------
+
+ReplicaIdle::ReplicaIdle(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActive/ReplicaIdle")
+{
+  dout(10) << "-- state -->> ReplicaActive/ReplicaIdle" << dendl;
+}
+
+void ReplicaIdle::reset_ignored(const FullReset&)
+{
+  dout(10) << "ReplicaIdle::react(const FullReset&): FullReset ignored"
+	   << dendl;
+}
+
+
+// ---------------- ReplicaIdle/ReplicaUnreserved ---------------------------
+
+ReplicaUnreserved::ReplicaUnreserved(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReplicaActive/ReplicaIdle/ReplicaUnreserved")
+{
+  dout(10) << "-- state -->> ReplicaActive/ReplicaIdle/ReplicaUnreserved"
+	   << dendl;
+}
+
+sc::result ReplicaUnreserved::react(const ReplicaReserveReq& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaUnreserved::react(const ReplicaReserveReq&)" << dendl;
+
+  switch (context<ReplicaActive>().on_reserve_request(ev, false)) {
+    case ReplicaReactCode::discard:
+      return discard_event();
+    case ReplicaReactCode::goto_waiting_reservation:
+      return transit<ReplicaWaitingReservation>();
+    case ReplicaReactCode::goto_replica_reserved:
+      return transit<ReplicaReserved>();
+    default:
+      ceph_abort_msg("unexpected return value");
   }
+  // can't happen, but some compilers complain:
+  return transit<ReplicaReserved>();
 }
 
+sc::result ReplicaUnreserved::react(const StartReplica& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaUnreserved::react(const StartReplica&)" << dendl;
+  post_event(ReplicaPushesUpd{});
+  return transit<ReplicaActiveOp>();
+}
 
-void ReplicaActive::clear_reservation_by_remote_primary()
+sc::result ReplicaUnreserved::react(const ReplicaRelease&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "ReplicaActive::clear_reservation_by_remote_primary()" << dendl;
-  m_osds->get_scrub_services().dec_scrubs_remote(scrbr->get_spgid().pgid);
-  reserved_by_my_primary = false;
+  dout(10) << "ReplicaUnreserved::react(const ReplicaRelease&)" << dendl;
+  // shouldn't happen. Possible (faulty) sequence: getting an op
+  // command while in ReplicaWaitingReservation (as we would just
+  // treat that as a regular op request, but will stop waiting for
+  // reservation).
+  // must cancel the queued reservation request
+  scrbr->get_clog()->error() << fmt::format(
+      "osd.{} pg[{}]: reservation released while not reserved",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  return discard_event();
 }
 
+sc::result ReplicaUnreserved::react(const ReserverGranted&)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaUnreserved::react(const ReserverGranted&)" << dendl;
+  // shouldn't happen. Possible (faulty) sequence: getting an op
+  // command while in ReplicaWaitingReservation (as we would just
+  // treat that as a regular op request, but will stop waiting for
+  // reservation).
+  // must unreserve
+  scrbr->get_clog()->error() << fmt::format(
+      "osd.{} pg[{}]: reservation granted while not being waited for",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  return discard_event();
+}
 
-void ReplicaActive::check_for_updates(const StartReplica& ev)
+
+// ---------------- ReplicaIdle/ReplicaWaitingReservation ---------------------------
+
+ReplicaWaitingReservation::ReplicaWaitingReservation(my_context ctx)
+    : my_base(ctx)
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReplicaActive/ReplicaIdle/ReplicaWaitingReservation")
+{
+  dout(10)
+      << "-- state -->> ReplicaActive/ReplicaIdle/ReplicaWaitingReservation"
+      << dendl;
+}
+
+sc::result ReplicaWaitingReservation::react(const ReserverGranted&)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaWaitingReservation::react(const ReserverGranted&)"
+	   << dendl;
+
+  /// \todo complete the handling of the granted reservation
+  ceph_abort_msg("not implemented yet");
+  return discard_event();
+}
+
+sc::result ReplicaWaitingReservation::react(const ReplicaRelease& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaWaitingReservation::react(const ReplicaRelease&)"
+	   << dendl;
+  // must cancel the queued reservation request
+  context<ReplicaActive>().on_release(ev);
+  return transit<ReplicaUnreserved>();
+}
+
+sc::result ReplicaWaitingReservation::react(const StartReplica& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "ReplicaActive::check_for_updates()" << dendl;
+  dout(10) << "ReplicaWaitingReservation::react(const StartReplica&)" << dendl;
+
+  // this shouldn't happen. We will handle it, but will also log an error.
+  scrbr->get_clog()->error() << fmt::format(
+      "osd.{} pg[{}]: new chunk request while still waiting for "
+      "reservation",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  clear_shallow_history<ReplicaIdle, 0>();
   post_event(ReplicaPushesUpd{});
+  return transit<ReplicaActiveOp>();
 }
 
-// ---------------- ReplicaActive/ReplicaIdle ---------------------------
+sc::result ReplicaWaitingReservation::react(const ReplicaReserveReq& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaWaitingReservation::react(const ReplicaReserveReq&)"
+	   << dendl;
+  // this shouldn't happen. We will handle it, but will also log an error.
+  scrbr->get_clog()->error() << fmt::format(
+      "osd.{} pg[{}]: reservation requested while previous is pending",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  // cancel the existing reservation, and re-request
+  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  post_event(ev);
+  return transit<ReplicaUnreserved>();
+}
 
-ReplicaIdle::ReplicaIdle(my_context ctx)
+
+// ---------------- ReplicaIdle/ReplicaReserved ---------------------------
+
+ReplicaReserved::ReplicaReserved(my_context ctx)
     : my_base(ctx)
-    , NamedSimply(context<ScrubMachine>().m_scrbr, "ReplicaActive/ReplicaIdle")
+    , NamedSimply(
+	  context<ScrubMachine>().m_scrbr,
+	  "ReplicaActive/ReplicaIdle/ReplicaReserved")
 {
-  dout(10) << "-- state -->> ReplicaActive/ReplicaIdle" << dendl;
+  dout(10) << "-- state -->> ReplicaActive/ReplicaIdle/ReplicaReserved"
+	   << dendl;
 }
 
-void ReplicaIdle::reset_ignored(const FullReset&)
+sc::result ReplicaReserved::react(const ReplicaRelease& ev)
 {
-  dout(10) << "ReplicaIdle::react(const FullReset&): FullReset ignored"
-	   << dendl;
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaReserved::react(const ReplicaRelease&)" << dendl;
+  context<ReplicaActive>().on_release(ev);
+  return transit<ReplicaUnreserved>();
+}
+
+sc::result ReplicaReserved::react(const ReplicaReserveReq& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaReserved::react(const ReplicaReserveReq&)" << dendl;
+  scrbr->get_clog()->error() << fmt::format(
+      "osd.{} pg[{}]: reservation requested while still reserved",
+      scrbr->get_whoami(), scrbr->get_spgid());
+  // cancel the existing reservation, and re-request
+  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  post_event(ev);
+  return transit<ReplicaUnreserved>();
 }
 
+sc::result ReplicaReserved::react(const StartReplica& ev)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << "ReplicaReserved::react(const StartReplica&)" << dendl;
+  post_event(ReplicaPushesUpd{});
+  return transit<ReplicaActiveOp>();
+}
+
+
 // ------------- ReplicaActive/ReplicaActiveOp --------------------------
 
 ReplicaActiveOp::ReplicaActiveOp(my_context ctx)
@@ -909,9 +1105,8 @@ sc::result ReplicaActiveOp::react(const StartReplica&)
   dout(1) << msg << dendl;
   scrbr->get_clog()->warn() << msg;
 
-  post_event(ReplicaPushesUpd{});
-
   // exit & re-enter the state
+  post_event(ReplicaPushesUpd{});
   return transit<ReplicaActiveOp>();
 }
 
@@ -970,15 +1165,19 @@ sc::result ReplicaBuildingMap::react(const SchedReplica&)
     dout(10) << "replica scrub job preempted" << dendl;
 
     scrbr->send_preempted_replica();
-    return transit<ReplicaIdle>();
+    return transit<sc::shallow_history<ReplicaReserved>>();
   }
 
   // start or check progress of build_replica_map_chunk()
   auto ret_init = scrbr->build_replica_map_chunk();
   if (ret_init != -EINPROGRESS) {
-    return transit<ReplicaIdle>();
+    dout(10) << "ReplicaBuildingMap::react(const SchedReplica&): back to idle"
+	     << dendl;
+    return transit<sc::shallow_history<ReplicaReserved>>();
   }
 
+  dout(20) << "ReplicaBuildingMap::react(const SchedReplica&): discarded"
+	   << dendl;
   return discard_event();
 }
 
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index beb4d7a4c0fa..9b2a800d1882 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -10,10 +10,12 @@
 #include <boost/statechart/event_base.hpp>
 #include <boost/statechart/in_state_reaction.hpp>
 #include <boost/statechart/simple_state.hpp>
+#include <boost/statechart/shallow_history.hpp>
 #include <boost/statechart/state.hpp>
 #include <boost/statechart/state_machine.hpp>
 #include <boost/statechart/transition.hpp>
 
+#include "common/fmt_common.h"
 #include "common/version.h"
 #include "messages/MOSDOp.h"
 #include "messages/MOSDRepScrub.h"
@@ -68,6 +70,10 @@ struct OpCarryingEvent : sc::event<EV> {
   {
     *out << fmt::format("{} (from: {})", EV::event_name, m_from);
   }
+  std::string fmt_print() const
+  {
+    return fmt::format("{} (from: {})", EV::event_name, m_from);
+  }
   std::string_view print() const { return EV::event_name; }
   ~OpCarryingEvent() { on_event_discard(EV::event_name); }
 };
@@ -96,6 +102,8 @@ OP_EV(ReplicaReserveReq);
 /// explicit release request from the Primary
 OP_EV(ReplicaRelease);
 
+/// the async-reserver granted our reservation request
+OP_EV(ReserverGranted);
 
 #define MEV(E)                                          \
   struct E : sc::event<E> {                             \
@@ -222,9 +230,19 @@ struct PrimaryActive;	   ///< base state for a Primary
 struct PrimaryIdle;	   ///< ready for a new scrub request
 struct Session;            ///< either reserving or actively scrubbing
 
-// the active states for a replica:
-struct ReplicaActive;    ///< the quiescent state for a replica
+// the Replica states:
+struct ReplicaActive;  ///< base state for when peered as a replica
+
+/// Inactive replica state. Handles reservation requests
+struct ReplicaIdle;
+// its sub-states:
+struct ReplicaUnreserved;      ///< not reserved by a primary
+struct ReplicaWaitingReservation;  ///< a reservation request was received from
+struct ReplicaReserved;	       ///< we are reserved by our primary
+
+// and when handling a single chunk scrub request op:
 struct ReplicaActiveOp;
+// its sub-states:
 struct ReplicaWaitUpdates;
 struct ReplicaBuildingMap;
 
@@ -710,42 +728,63 @@ struct WaitDigestUpdate : sc::state<WaitDigestUpdate, ActiveScrubbing>,
  *   - maintain the "I am reserved by a primary" state;
  *   - handles reservation requests
  *
- *     - ReplicaIdle - ready for a new scrub request
- *          * initial state of ReplicaActive
+ *  - ReplicaIdle - ready for a new scrub request
+ *
+ *    - initial state of ReplicaActive
+ *    - No scrubbing is performed in this state, but reservation-related
+ *      events are handled.
+ *    - uses 'shallow history', so that when returning from ReplicaActiveOp, we
+ *       return to where we were - either reserved by our primary, or unreserved.
  *
- *     - ReplicaActiveOp - handling a single map request op
- *          * ReplicaWaitUpdates
- *  	    * ReplicaBuildingMap
+ *    - sub-states:
+ *      * ReplicaUnreserved - not reserved by a primary. In this state we
+ *        are waiting for either a reservation request, or a chunk scrub op.
+ *
+ *      * ReplicaWaitingReservation - a reservation request was received from
+ *        our primary. We expect a ' go ahead' from the reserver, or a
+ *        cancellation command from the primary (or an interval change).
+ *
+ *      * ReplicaReserved - we are reserved by a primary.
+ *
+ *  - ReplicaActiveOp - handling a single map request op
+ *      * ReplicaWaitUpdates
+ *      * ReplicaBuildingMap
  */
 
 struct ReplicaIdle;
 
-struct ReplicaActive : sc::state<ReplicaActive, ScrubMachine, ReplicaIdle>,
-			 NamedSimply {
+// sc::result cannot be copied or moved, so we need to postpone
+// the creation of such objects to the moment where they are
+// returned from the react() function.
+enum class ReplicaReactCode {
+  discard,
+  goto_waiting_reservation,
+  goto_replica_reserved
+};
+
+struct ReplicaActive : sc::state<
+			   ReplicaActive,
+			   ScrubMachine,
+			   mpl::list<sc::shallow_history<ReplicaIdle>>,
+			   sc::has_shallow_history>,
+		       NamedSimply {
   explicit ReplicaActive(my_context ctx);
   ~ReplicaActive();
 
   /// handle a reservation request from a primary
-  void on_reserve_req(const ReplicaReserveReq&);
+  ReplicaReactCode on_reserve_request(
+      const ReplicaReserveReq&,
+      bool async_request);
 
   /// handle a 'release' from a primary
-  void on_release(const ReplicaRelease&);
+  void on_release(const ReplicaRelease& ev);
 
-  void check_for_updates(const StartReplica&);
+  /// cancel the reserver request.
+  /// The 'failure' re 'log_failure' is logged if we are not reserved to
+  /// begin with.
+  void clear_reservation_by_remote_primary(bool log_failure);
 
-  using reactions = mpl::list<
-      // a reservation request from the primary
-      sc::in_state_reaction<
-	  ReplicaReserveReq,
-	  ReplicaActive,
-	  &ReplicaActive::on_reserve_req>,
-      // an explicit release request from the primary
-      sc::in_state_reaction<
-	  ReplicaRelease,
-	  ReplicaActive,
-	  &ReplicaActive::on_release>,
-      // when the interval ends - we may not be a replica anymore
-      sc::transition<IntervalChanged, NotActive>>;
+  using reactions = mpl::list<sc::transition<IntervalChanged, NotActive>>;
 
  private:
   bool reserved_by_my_primary{false};
@@ -753,40 +792,105 @@ struct ReplicaActive : sc::state<ReplicaActive, ScrubMachine, ReplicaIdle>,
   // shortcuts:
   PG* m_pg;
   OSDService* m_osds;
-
-  /// a convenience internal result structure
-  struct ReservationAttemptRes {
-    MOSDScrubReserve::ReserveMsgOp op;	// GRANT or REJECT
-    std::string_view error_msg;
-    bool granted;
-  };
-
-  /// request a scrub resource from our local OSD
-  /// (after performing some checks)
-  ReservationAttemptRes get_remote_reservation();
-
-  void clear_reservation_by_remote_primary();
 };
 
 
-struct ReplicaIdle : sc::state<ReplicaIdle, ReplicaActive>, NamedSimply {
+struct ReplicaIdle : sc::state<
+			 ReplicaIdle,
+			 ReplicaActive,
+			 ReplicaUnreserved,
+			 sc::has_shallow_history>,
+		     NamedSimply {
   explicit ReplicaIdle(my_context ctx);
   ~ReplicaIdle() = default;
   void reset_ignored(const FullReset&);
+  using reactions = mpl::list<sc::in_state_reaction<
+      FullReset,
+      ReplicaIdle,
+      &ReplicaIdle::reset_ignored>>;
+};
+
+/*
+ * ReplicaUnreserved
+ *
+ * Possible events:
+ * - a reservation request from a legacy primary (i.e. a primary that does not
+ *   support queued reservations). We either deny or grant, transitioning to
+ *   ReplicaReserved directly.
+ * - a reservation request from a primary that supports queued reservations.
+ *   We transition to ReplicaWaitingReservation, and wait for the Reserver's
+ *   response.
+ * - (handled by our parent state) a chunk scrub request. We transition to
+ *   ReplicaActiveOp.
+ */
+struct ReplicaUnreserved : sc::state<ReplicaUnreserved, ReplicaIdle>,
+			   NamedSimply {
+  explicit ReplicaUnreserved(my_context ctx);
 
-  // note the execution of check_for_updates() when transitioning to
-  // ReplicaActiveOp/ReplicaWaitUpdates. That would trigger a ReplicaPushesUpd
-  // event, which will be handled by ReplicaWaitUpdates.
   using reactions = mpl::list<
-      sc::transition<
-	  StartReplica,
-	  ReplicaWaitUpdates,
-	  ReplicaActive,
-	  &ReplicaActive::check_for_updates>,
-      sc::in_state_reaction<
-	  FullReset,
-	  ReplicaIdle,
-	  &ReplicaIdle::reset_ignored>>;
+      sc::custom_reaction<ReplicaReserveReq>,
+      sc::custom_reaction<ReplicaRelease>,
+      sc::custom_reaction<ReserverGranted>,
+      sc::custom_reaction<StartReplica>>;
+
+  sc::result react(const ReplicaReserveReq& ev);
+  sc::result react(const StartReplica& ev);
+  sc::result react(const ReserverGranted&);
+  sc::result react(const ReplicaRelease&);
+};
+
+/**
+ * ReplicaWaitingReservation
+ *
+ * Possible events:
+ * - 'go ahead' from the async reserver. We send a GRANT message to the
+ *   primary & transition to ReplicaReserved.
+ * - 'cancel' from the primary. We clear our reservation state, and transition
+ *   back to ReplicaUnreserved.
+ * - a chunk request: shouldn't happen, but we handle it anyway. An error
+ *   is logged (to trigger test failures).
+ * - on interval change: handled by our parent state.
+ */
+struct ReplicaWaitingReservation
+    : sc::state<ReplicaWaitingReservation, ReplicaIdle>,
+      NamedSimply {
+  explicit ReplicaWaitingReservation(my_context ctx);
+
+  using reactions = mpl::list<
+      // the 'normal' (expected) events:
+      sc::custom_reaction<ReplicaRelease>,
+      sc::custom_reaction<StartReplica>,
+      // unexpected (bug-induced) events:
+      sc::custom_reaction<ReplicaReserveReq>,
+      sc::custom_reaction<ReserverGranted>>;
+
+  sc::result react(const ReplicaRelease& ev);
+  sc::result react(const StartReplica& ev);
+  sc::result react(const ReserverGranted&);
+  sc::result react(const ReplicaReserveReq& ev);
+};
+
+/**
+ * ReplicaReserved
+ *
+ * Possible events:
+ * - 'cancel' from the primary. We clear our reservation state, and transition
+ *   back to ReplicaUnreserved.
+ * - a chunk scrub request. We transition to ReplicaActiveOp.
+ * - on interval change: we clear our reservation state, and transition
+ *   back to ReplicaUnreserved.
+ */
+struct ReplicaReserved : sc::state<ReplicaReserved, ReplicaIdle>, NamedSimply {
+  explicit ReplicaReserved(my_context ctx);
+
+  using reactions = mpl::list<
+      sc::custom_reaction<ReplicaReserveReq>,
+      sc::custom_reaction<StartReplica>,
+      sc::custom_reaction<ReplicaRelease>>;
+
+  sc::result react(const ReplicaReserveReq&);
+  sc::result react(const ReplicaRelease&);
+  sc::result react(const StartReplica& eq);
 };
 
 
@@ -833,7 +937,6 @@ struct ReplicaWaitUpdates : sc::state<ReplicaWaitUpdates, ReplicaActiveOp>,
   sc::result react(const ReplicaPushesUpd&);
 };
 
-
 struct ReplicaBuildingMap : sc::state<ReplicaBuildingMap, ReplicaActiveOp>,
 			    NamedSimply {
   explicit ReplicaBuildingMap(my_context ctx);
diff --git a/src/osd/scrubber/scrub_resources.h b/src/osd/scrubber/scrub_resources.h
index ff9587bb60b0..afb154e3e162 100644
--- a/src/osd/scrubber/scrub_resources.h
+++ b/src/osd/scrubber/scrub_resources.h
@@ -10,6 +10,33 @@
 #include "common/Formatter.h"
 #include "osd/osd_types.h"
 
+/*
+ * AsyncReserver for scrub 'remote' reservations
+ * -----------------------------------------------
+ *
+ * On the replica side, all reservations are treated as having the same priority.
+ * Note that 'high priority' scrubs, e.g. user-initiated scrubs, are not required
+ * to perform any reservations, and are never handled by the replicas' OSD.
+ *
+ * A queued scrub reservation request is cancelled by any of the following events:
+ *
+ * - a new interval: in this case, we do not expect to see a cancellation request
+ *   from the primary, and we can simply remove the request from the queue;
+ *
+ * - a cancellation request from the primary: probably a result of timing out on
+ *   the reservation process. Here, we can simply remove the request from the queue.
+ *
+ * - a new reservation request for the same PG: which means we had missed the
+ *   previous cancellation request. We cancel the previous request, and replace
+ *   it with the new one. We would also issue an error log message.
+ *
+ * Primary/Replica with differing versions:
+ *
+ * The updated version of MOSDScrubReserve contains a new 'OK to queue' field.
+ * For legacy Primary OSDs, this field is decoded as 'false', and the replica
+ * responds immediately, with grant/rejection.
+*/
+
 namespace Scrub {
 
 /**
@@ -73,6 +100,9 @@ class ScrubResources {
   /// increments the number of scrubs acting as a Replica
   bool inc_scrubs_remote(pg_t pgid);
 
+  /// queue a request with the scrub reserver
+  void enqueue_remote_reservation(pg_t pgid) {}
+
   /// decrements the number of scrubs acting as a Replica
   void dec_scrubs_remote(pg_t pgid);
 

From e2ce8ed1ff9846d4efa7de434a56edc66305e767 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Thu, 18 Jan 2024 18:57:24 +0000
Subject: [PATCH 1636/2492] mgr: add read balancer support inside the balancer
 module

Read balancing may now be managed automatically via the balancer
manager module. Users may choose between two new modes: ``upmap-read``, which
offers upmap and read optimization simultaneously, or ``read``, which may be used
to only optimize reads. Existing balancer commands have also been added to
contain more information about read balancing.

Run the following commands to test the new automatic behavior:
`ceph balancer on` (on by default)
`ceph balancer mode <read|upmap-read>`
`ceph balancer status`

Run the following commands to test the new supervised behavior:
`ceph balancer off`
`ceph balancer mode <read|upmap-read>`
`ceph balancer eval` | `ceph balancer eval <pool-name>`
`ceph balancer eval-verbose` | `ceph balancer eval-verbose <pool-name>`
`ceph balancer optimize <plan-name>`
`ceph balancer show <plan-name>`
`ceph balancer eval <plan-name>`
`ceph balancer execute <plan-name>`

In the balancer module, there is also a new "self_test" function which tests
the module's basic functionality. This test can be triggered with the following
commands:
`ceph mgr module enable selftest`
`ceph mgr self-test module balancer`

Related Trello: https://trello.com/c/sWoKctzL/859-add-read-balancer-support-inside-the-balancer-module
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 doc/dev/balancer-design.rst                   |   1 -
 doc/rados/operations/balancer.rst             |  48 ++++-
 doc/rados/operations/read-balancer.rst        |  47 ++++-
 src/mgr/PyOSDMap.cc                           |  32 +++
 src/pybind/mgr/balancer/module.py             | 190 +++++++++++++++++-
 src/pybind/mgr/ceph_module.pyi                |   1 +
 src/pybind/mgr/mgr_module.py                  |   4 +
 src/test/cli-integration/balancer/misplaced.t |   2 +
 8 files changed, 311 insertions(+), 14 deletions(-)

diff --git a/doc/dev/balancer-design.rst b/doc/dev/balancer-design.rst
index 684d163528f3..cf45473f851c 100644
--- a/doc/dev/balancer-design.rst
+++ b/doc/dev/balancer-design.rst
@@ -55,4 +55,3 @@ Plans for the Next Version
 --------------------------
 
 1. Improve behavior for heterogeneous OSDs in a pool
-2. Offer read balancing as an online option to the balancer manager module
diff --git a/doc/rados/operations/balancer.rst b/doc/rados/operations/balancer.rst
index 5f87865bcf32..949ff17c24ab 100644
--- a/doc/rados/operations/balancer.rst
+++ b/doc/rados/operations/balancer.rst
@@ -21,9 +21,9 @@ To check the current status of the balancer, run the following command:
 Automatic balancing
 -------------------
 
-When the balancer is in ``upmap`` mode, the automatic balancing feature is
-enabled by default. For more details, see :ref:`upmap`.  To disable the
-balancer, run the following command:
+When the balancer is in ``upmap`` mode, which is the default, the automatic
+upmap balancing feature is enabled.  For more details, see :ref:`upmap`.
+To disable the balancer, run the following command:
 
    .. prompt:: bash $
 
@@ -34,6 +34,10 @@ The balancer mode can be changed from ``upmap`` mode to ``crush-compat`` mode.
 ``crush-compat`` mode, the balancer automatically makes small changes to the
 data distribution in order to ensure that OSDs are utilized equally.
 
+Additional modes include ``upmap-read`` and ``read``. ``upmap-read`` mode
+combines the upmap balancer with the read balancer so that both writes
+and reads are optimized. ``read`` mode can be used when only read optimization
+is desired. For more details, see :ref:`read_balancer`.
 
 Throttling
 ----------
@@ -102,7 +106,7 @@ and then run the following command:
 Modes
 -----
 
-There are two supported balancer modes:
+There are four supported balancer modes:
 
 #. **crush-compat**. This mode uses the compat weight-set feature (introduced
    in Luminous) to manage an alternative set of weights for devices in the
@@ -135,13 +139,45 @@ There are two supported balancer modes:
 
    To use ``upmap``, all clients must be Luminous or newer.
 
-The default mode is ``upmap``. The mode can be changed to ``crush-compat`` by
-running the following command:
+#. **read**. In Reef and later releases, the OSDMap can store explicit
+   mappings for individual primary OSDs as exceptions to the normal CRUSH
+   placement calculation. These ``pg-upmap-primary`` entries provide fine-grained
+   control over primary PG mappings. This mode optimizes the placement of individual
+   primary PGs in order to achieve balanced reads, or primary PGs, in a cluster.
+   In ``read`` mode, upmap behavior is not excercised, so this mode is best for
+   uses cases in which only read balancing is desired.
+
+   To use ``pg-upmap-primary``, all clients must be Reef or newer. For more
+   details about client compatibility, see :ref:`read_balancer`.
+
+#. **upmap-read**. This balancer mode combines optimization benefits of
+   both ``upmap`` and ``read`` mode. Like in ``read`` mode, ``upmap-read``
+   makes use of ``pg-upmap-primary``. As such, only Reef and later clients
+   are compatible. For more details about client compatibility, see
+   :ref:`read_balancer`.
+
+   ``upmap-read`` is highly recommended for achieving the ``upmap`` mode's
+   offering of balanced PG distribution as well as the ``read`` mode's
+   offering of balanced reads.
+
+The default mode is ``upmap``. The mode can be changed to ``crush-compat`` by running the following command:
 
    .. prompt:: bash $
 
       ceph balancer mode crush-compat
 
+The mode can be changed to ``read`` by running the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer mode read
+
+The mode can be changed to ``upmap-read`` by running the following command:
+
+   .. prompt:: bash $
+
+      ceph balancer mode upmap-read
+
 Supervised optimization
 -----------------------
 
diff --git a/doc/rados/operations/read-balancer.rst b/doc/rados/operations/read-balancer.rst
index 0833e4326c9b..a2c189dbb16f 100644
--- a/doc/rados/operations/read-balancer.rst
+++ b/doc/rados/operations/read-balancer.rst
@@ -17,9 +17,50 @@ you may want to try improving your read performance with the read balancer.
 Online Optimization
 ===================
 
-At present, there is no online option for the read balancer. However, we plan to add
-the read balancer as an option to the :ref:`balancer` in the next Ceph version
-so it can be enabled to run automatically in the background like the upmap balancer.
+Enabling
+--------
+
+To enable automatic read balancing, you must turn on the *balancer module*
+(enabled by default in new clusters) and set the mode to ``read`` or ``upmap-read``:
+
+.. prompt:: bash $
+
+   ceph balancer on
+   ceph balancer mode <read|upmap-read>
+
+Both ``read`` and ``upmap-read`` mode make use of ``pg-upmap-primary``. In order
+to use ``pg-upmap-primary``, the cluster cannot have any pre-Reef clients.
+
+If you want to use a different balancer or if you want to make your
+own custom ``pg-upmap-primary`` entries, you might want to turn off the balancer in
+order to avoid conflict:
+
+.. prompt:: bash $
+
+   ceph balancer off
+
+To allow use of the new feature on an existing cluster, you must restrict the
+cluster to supporting only Reef (and newer) clients.  To do so, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph osd set-require-min-compat-client reef
+
+This command will fail if any pre-Reef clients or daemons are connected to
+the monitors. To see which client versions are in use, run the following
+command:
+
+.. prompt:: bash $
+
+   ceph features
+
+Balancer Module
+---------------
+
+The `balancer` module for ``ceph-mgr`` will automatically balance the number of
+primary PGs per OSD if set to ``read`` or ``upmap-read`` mode. See :ref:`balancer`
+for more information.
 
 Offline Optimization
 ====================
diff --git a/src/mgr/PyOSDMap.cc b/src/mgr/PyOSDMap.cc
index 83475f5ee5f3..58d2da41b9be 100644
--- a/src/mgr/PyOSDMap.cc
+++ b/src/mgr/PyOSDMap.cc
@@ -162,6 +162,36 @@ static PyObject *osdmap_calc_pg_upmaps(BasePyOSDMap* self, PyObject *args)
   return PyLong_FromLong(r);
 }
 
+static PyObject *osdmap_balance_primaries(BasePyOSDMap* self, PyObject *args)
+{
+  int pool_id;
+  BasePyOSDMapIncremental *incobj;
+  if (!PyArg_ParseTuple(args, "iO:balance_primaries",
+                        &pool_id, &incobj)) {
+    return nullptr;
+  }
+  auto check_pool = self->osdmap->get_pg_pool(pool_id);
+  if (!check_pool) {
+    derr << __func__ << " pool '" << pool_id
+         << "' does not exist" << dendl;
+    return nullptr;
+  }
+  dout(10) << __func__ << " osdmap " << self->osdmap
+           << " pool_id " << pool_id
+           << " inc " << incobj->inc
+           << dendl;
+  PyThreadState *tstate = PyEval_SaveThread();
+  OSDMap tmp_osd_map;
+  tmp_osd_map.deepish_copy_from(*(self->osdmap));
+  int r = self->osdmap->balance_primaries(g_ceph_context,
+                                 pool_id,
+                                 incobj->inc,
+				 tmp_osd_map);
+  PyEval_RestoreThread(tstate);
+  dout(10) << __func__ << " r = " << r << dendl;
+  return PyLong_FromLong(r);
+}
+
 static PyObject *osdmap_map_pool_pgs_up(BasePyOSDMap* self, PyObject *args)
 {
   int poolid;
@@ -324,6 +354,8 @@ PyMethodDef BasePyOSDMap_methods[] = {
    "Get pools that have CRUSH rules that TAKE the given root"},
   {"_calc_pg_upmaps", (PyCFunction)osdmap_calc_pg_upmaps, METH_VARARGS,
    "Calculate new pg-upmap values"},
+  {"_balance_primaries", (PyCFunction)osdmap_balance_primaries, METH_VARARGS,
+   "Calculate new pg-upmap-primary values"},
   {"_map_pool_pgs_up", (PyCFunction)osdmap_map_pool_pgs_up, METH_VARARGS,
    "Calculate up set mappings for all PGs in a pool"},
   {"_pg_to_up_acting_osds", (PyCFunction)osdmap_pg_to_up_acting_osds, METH_VARARGS,
diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 2cbaf10c09fe..54780d1eac82 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -9,7 +9,7 @@
 import math
 import random
 import time
-from mgr_module import CLIReadCommand, CLICommand, CommandResult, MgrModule, Option, OSDMap
+from mgr_module import CLIReadCommand, CLICommand, CommandResult, MgrModule, Option, OSDMap, CephReleases
 from threading import Event
 from typing import cast, Any, Dict, List, Optional, Sequence, Tuple, Union
 from mgr_module import CRUSHMap
@@ -55,6 +55,8 @@ class Mode(enum.Enum):
     none = 'none'
     crush_compat = 'crush-compat'
     upmap = 'upmap'
+    read = 'read'
+    upmap_read = 'upmap-read'
 
 
 class Plan(object):
@@ -116,6 +118,10 @@ def show(self) -> str:
                 osdlist += [m['from'], m['to']]
             ls.append('ceph osd pg-upmap-items %s %s' %
                       (item['pgid'], ' '.join([str(a) for a in osdlist])))
+        for item in incdump.get('new_pg_upmap_primaries', []):
+            ls.append('ceph osd pg-upmap-primary %s %s' % (item['pgid'], item['primary_osd']))
+        for item in incdump.get('old_pg_upmap_primaries', []):
+            ls.append('ceph osd rm-pg-upmap-primary %s' % item['pgid'])
         return '\n'.join(ls)
 
 
@@ -142,6 +148,9 @@ def __init__(self, ms: MappingState):
 
         self.score = 0.0
 
+        self.read_balance_score_by_pool: Dict[str, Dict[str, float]] = {}
+        self.read_balance_score_acting_by_pool: Dict[str, float] = {}
+
     def show(self, verbose: bool = False) -> str:
         if verbose:
             r = self.ms.desc + '\n'
@@ -155,9 +164,12 @@ def show(self, verbose: bool = False) -> str:
             r += 'stats_by_root %s\n' % self.stats_by_root
             r += 'score_by_pool %s\n' % self.score_by_pool
             r += 'score_by_root %s\n' % self.score_by_root
+            r += 'score %f (lower is better)\n' % self.score
+            r += 'read_balance_score_by_pool %s\n' % self.read_balance_score_by_pool
         else:
             r = self.ms.desc + ' '
-        r += 'score %f (lower is better)\n' % self.score
+            r += 'score %f (lower is better)\n' % self.score
+            r += 'read_balance_scores (lower is better) %s\n' % self.read_balance_score_acting_by_pool
         return r
 
     def calc_stats(self, count, target, total):
@@ -290,7 +302,7 @@ class Module(MgrModule):
         Option(name='mode',
                desc='Balancer mode',
                default='upmap',
-               enum_allowed=['none', 'crush-compat', 'upmap'],
+               enum_allowed=['none', 'crush-compat', 'upmap', 'read', 'upmap-read'],
                runtime=True),
         Option(name='sleep_interval',
                type='secs',
@@ -394,6 +406,19 @@ def set_mode(self, mode: Mode) -> Tuple[int, str, str]:
                               self.get("pool_stats"),
                               'initialize compat weight-set')
             self.get_compat_weight_set_weights(ms)  # ignore error
+        elif (mode == Mode.read) or (mode == Mode.upmap_read):
+            try:
+                release = CephReleases[min_compat_client]
+                if release.value < CephReleases.reef.value:
+                    warn = ('min_compat_client "%s" '
+                            '< "reef", which is required for pg-upmap-primary. '
+                            'Try "ceph osd set-require-min-compat-client reef" '
+                            'before enabling this mode' % min_compat_client)
+                    return (-errno.EPERM, '', warn)
+            except KeyError:
+                self.log.error('Unable to apply mode {} due to unknown min_compat_client {}'.format(mode, min_compat_client))
+                warn = ('Unable to apply mode {} due to unknown min_compat_client {}.'.format(mode, min_compat_client))
+                return (-errno.EPERM, '', warn)
         self.set_module_option('mode', mode.value)
         return (0, '', '')
 
@@ -894,6 +919,23 @@ def calc_eval(self, ms: MappingState, pools: List[str]) -> Eval:
                 'objects': objects,
                 'bytes': bytes,
             }
+            try:
+                read_balance_scores = pi['read_balance']
+                pe.read_balance_score_acting_by_pool[pool] = read_balance_scores['score_acting']
+                pe.read_balance_score_by_pool[pool] = {
+                    'score_acting': read_balance_scores['score_acting'],
+                    'score_stable': read_balance_scores['score_stable'],
+                    'optimal_score': read_balance_scores['optimal_score'],
+                    'raw_score_acting': read_balance_scores['raw_score_acting'],
+                    'raw_score_stable': read_balance_scores['raw_score_stable'],
+                    'primary_affinity_weighted': read_balance_scores['primary_affinity_weighted'],
+                    'average_primary_affinity': read_balance_scores['average_primary_affinity'],
+                    'average_primary_affinity_weighted': read_balance_scores['average_primary_affinity_weighted']
+                }
+            except KeyError:
+                self.log.debug("Skipping pool '{}' since it does not have a read_balance_score, "
+                               "likely because it is not replicated.".format(pool))
+
         for root in pe.total_by_root:
             pe.count_by_root[root] = {
                 'pgs': {
@@ -998,6 +1040,14 @@ def optimize(self, plan: Plan) -> Tuple[int, str]:
                 return self.do_upmap(plan)
             elif plan.mode == 'crush-compat':
                 return self.do_crush_compat(cast(MsPlan, plan))
+            elif plan.mode == 'read':
+                return self.do_read_balancing(plan)
+            elif plan.mode == 'upmap-read':
+                r_upmap, detail_upmap = self.do_upmap(plan)
+                r_read, detail_read = self.do_read_balancing(plan)
+                if (r_upmap < 0) and (r_read < 0):
+                    return r_upmap, detail_upmap
+                return 0, ''
             elif plan.mode == 'none':
                 detail = 'Please do "ceph balancer mode" to choose a valid mode first'
                 self.log.info('Idle')
@@ -1007,6 +1057,83 @@ def optimize(self, plan: Plan) -> Tuple[int, str]:
                 self.log.info(detail)
                 return -errno.EINVAL, detail
 
+    def do_read_balancing(self, plan: Plan) -> Tuple[int, str]:
+        self.log.info('do_read_balancing')
+        osdmap_dump = plan.osdmap_dump
+        msg = 'Unable to find further optimization, ' \
+              'or distribution is already perfect'
+
+        if len(plan.pools):
+            pools = plan.pools
+        else:  # all
+            pools = [str(i['pool_name']) for i in osdmap_dump.get('pools', [])]
+        if len(pools) == 0:
+            detail = 'No pools available'
+            self.log.info(detail)
+            return -errno.ENOENT, detail
+        self.log.info('pools %s' % pools)
+
+        adjusted_pools = []
+        inc = plan.inc
+        total_num_changes = 0
+        pools_with_pg_merge = []
+        crush_rule_by_pool_name = {}
+        no_read_balance_info = []
+        replicated_pools_with_optimal_score = []
+        rb_error_message = {}
+        for p in osdmap_dump.get('pools', []):
+            for pool_pg_status in plan.pg_status.get('pgs_by_pool_state', []):
+                if pool_pg_status['pool_id'] != p['pool']:
+                    continue
+                for state in pool_pg_status['pg_state_counts']:
+                    if state['state_name'] != 'active+clean':
+                        msg = "Not all PGs are active+clean; try again later."
+                        return -errno.EALREADY, msg
+            if p['pg_num'] > p['pg_num_target']:
+                pools_with_pg_merge.append(p['pool_name'])
+            crush_rule_by_pool_name[p['pool_name']] = p['crush_rule']
+            if 'read_balance' not in p:
+                no_read_balance_info.append(p['pool_name'])
+            if 'read_balance' in p:
+                if 'error_message' in p['read_balance']:
+                    rb_error_message[p['pool_name']] = p['read_balance']['error_message']
+                elif p['read_balance']['score_acting'] == p['read_balance']['optimal_score']:
+                    replicated_pools_with_optimal_score.append(p['pool_name'])
+        for pool in pools:
+            if pool not in crush_rule_by_pool_name:
+                self.log.debug('pool %s does not exist' % pool)
+                continue
+            if pool in pools_with_pg_merge:
+                self.log.debug('pool %s has pending PG(s) for merging, skipping for now' % pool)
+                continue
+            if pool in no_read_balance_info:
+                self.log.debug('pool %s has no read_balance information, skipping' % pool)
+                continue
+            if pool in replicated_pools_with_optimal_score:
+                self.log.debug('pool %s is already balanced, skipping' % pool)
+                continue
+            if pool in rb_error_message:
+                self.log.error(rb_error_message[pool])
+                continue
+            adjusted_pools.append(pool)
+        pool_dump = osdmap_dump.get('pools', [])
+        for pool in adjusted_pools:
+            for p in pool_dump:
+                if p['pool_name'] == pool:
+                    pool_id = p['pool']
+                    break
+            num_changes = plan.osdmap.balance_primaries(pool_id, inc)
+            total_num_changes += num_changes
+        if total_num_changes < 0:
+            self.no_optimization_needed = True
+            self.log.debug('unable to balance reads.')
+            return -errno.EALREADY, msg
+        self.log.info('prepared {} read changes'.format(total_num_changes))
+        if total_num_changes == 0:
+            self.no_optimization_needed = True
+            return -errno.EALREADY, msg
+        return 0, ''
+
     def do_upmap(self, plan: Plan) -> Tuple[int, str]:
         self.log.info('do_upmap')
         max_optimizations = cast(float, self.get_module_option('upmap_max_optimizations'))
@@ -1067,7 +1194,7 @@ def do_upmap(self, plan: Plan) -> Tuple[int, str]:
             left -= did
             if left <= 0:
                 break
-        self.log.info('prepared %d/%d changes' % (total_did, max_optimizations))
+        self.log.info('prepared %d/%d upmap changes' % (total_did, max_optimizations))
         if total_did == 0:
             self.no_optimization_needed = True
             return -errno.EALREADY, 'Unable to find further optimization, ' \
@@ -1422,6 +1549,19 @@ def execute(self, plan: Plan) -> Tuple[int, str]:
             }), 'foo')
             commands.append(result)
 
+        # read
+        for item in incdump.get('new_pg_upmap_primaries', []):
+            self.log.info('ceph osd pg-upmap-primary %s primary_osd %s', item['pgid'],
+                          item['primary_osd'])
+            result = CommandResult('foo')
+            self.send_command(result, 'mon', '', json.dumps({
+                'prefix': 'osd pg-upmap-primary',
+                'format': 'json',
+                'pgid': item['pgid'],
+                'id': item['primary_osd'],
+            }), 'foo')
+            commands.append(result)
+
         # wait for commands
         self.log.debug('commands %s' % commands)
         for result in commands:
@@ -1437,3 +1577,45 @@ def gather_telemetry(self) -> Dict[str, Any]:
             'active': self.active,
             'mode': self.mode,
         }
+
+    def self_test(self) -> None:
+        # turn balancer on
+        self.on()
+
+        # Get min-compat-client
+        min_compat_client = self.get_osdmap().dump().get('require_min_compat_client', '')
+        release = CephReleases[min_compat_client]
+
+        # Check upmap mode warning
+        r, _, warn = self.set_mode(Mode.upmap)
+        if release.value < CephReleases.luminous.value:
+            if r >= 0:
+                raise RuntimeError('upmap mode did not properly warn about min_compat_client')
+            if warn == '':
+                raise RuntimeError('upmap mode warning is empty when it should not be.')
+
+        # Check read mode warning
+        r, _, warn = self.set_mode(Mode.read)
+        if release.value < CephReleases.reef.value:
+            if r >= 0:
+                raise RuntimeError('read mode did not properly warn about min_compat_client')
+            if warn == '':
+                raise RuntimeError('read mode warning is empty when it should not be.')
+        r, _, warn = self.set_mode(Mode.upmap_read)
+
+        # Check upmap-read mode warning
+        if release.value < CephReleases.reef.value:
+            if r >= 0:
+                raise RuntimeError('upmap-read mode did not properly warn about min_compat_client')
+            if warn == '':
+                raise RuntimeError('upmap-read mode warning is empty when it should not be.')
+
+        # Check status
+        r, status, _ = self.show_status()
+        if r < 0:
+            raise RuntimeError('Balancer status was unsuccessful')
+        if status == '':
+            raise RuntimeError('Balancer status was empty')
+
+        # Turn off
+        self.off()
diff --git a/src/pybind/mgr/ceph_module.pyi b/src/pybind/mgr/ceph_module.pyi
index 50147f08f30d..df4a3782a0c9 100644
--- a/src/pybind/mgr/ceph_module.pyi
+++ b/src/pybind/mgr/ceph_module.pyi
@@ -19,6 +19,7 @@ class BasePyOSDMap(object):
     def _get_crush(self):...
     def _get_pools_by_take(self, take):...
     def _calc_pg_upmaps(self, inc, max_deviation, max_iterations, pool):...
+    def _balance_primaries(self, pool_id, inc):...
     def _map_pool_pgs_up(self, poolid):...
     def _pg_to_up_acting_osds(self, pool_id, ps):...
     def _pool_raw_used_rate(self, pool_id):...
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index ef908bd81d38..28fd69738eb9 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -210,6 +210,10 @@ def calc_pg_upmaps(self, inc: 'OSDMapIncremental',
             inc,
             max_deviation, max_iterations, pools)
 
+    def balance_primaries(self, pool_id: int,
+                          inc: 'OSDMapIncremental') -> int:
+        return self._balance_primaries(pool_id, inc)
+
     def map_pool_pgs_up(self, poolid: int) -> List[int]:
         return self._map_pool_pgs_up(poolid)
 
diff --git a/src/test/cli-integration/balancer/misplaced.t b/src/test/cli-integration/balancer/misplaced.t
index 050cceb6402b..b9510caa4d09 100644
--- a/src/test/cli-integration/balancer/misplaced.t
+++ b/src/test/cli-integration/balancer/misplaced.t
@@ -12,6 +12,7 @@
   $ ceph config set osd.* target_max_misplaced_ratio .07
   $ ceph balancer eval
   current cluster score [0-9]*\.?[0-9]+.* (re)
+  read_balance_scores \(lower is better\) {'rbd': [0-9]*\.?[0-9]+.*, 'balancer_opt': [0-9]*\.?[0-9]+.*} (re)
 # Turn off active balancer to use manual commands
   $ ceph balancer off
   $ ceph balancer optimize test_plan balancer_opt
@@ -22,6 +23,7 @@
   $ ceph balancer execute test_plan
   $ ceph balancer eval
   current cluster score [0-9]*\.?[0-9]+.* (re)
+  read_balance_scores \(lower is better\) {'rbd': [0-9]*\.?[0-9]+.*, 'balancer_opt': [0-9]*\.?[0-9]+.*} (re)
 # Plan is gone after execution ?
   $ ceph balancer execute test_plan
   Error ENOENT: plan test_plan not found

From 61e721c9f1af4d56918a26940f185dfa6607f2b9 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Fri, 22 Dec 2023 19:23:41 +0000
Subject: [PATCH 1637/2492] PendingReleaseNotes: add note about read balancer
 mgr module integration

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 PendingReleaseNotes | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index f11772762728..b374a1743f87 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -113,6 +113,10 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
 * RBD: The ``try-netlink`` mapping option for rbd-nbd has become the default
   and is now deprecated. If the NBD netlink interface is not supported by the
   kernel, then the mapping is retried using the legacy ioctl interface.
+* RADOS: Read balancing may now be managed automatically via the balancer
+  manager module. Users may choose between two new modes: ``upmap-read``, which
+  offers upmap and read optimization simultaneously, or ``read``, which may be used
+  to only optimize reads. For more detailed information see https://docs.ceph.com/en/latest/rados/operations/read-balancer/#online-optimization.
 
 >=18.0.0
 

From f3c74a7be4bb7502dc63aaeba8b0bac683c27dc0 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 2 Jan 2024 21:28:03 +0000
Subject: [PATCH 1638/2492] mgr/balancer: add pg_upmap_primaries to `balancer
 status detail`

Followup to https://github.com/ceph/ceph/pull/54801/commits/8a5553597ca6a428cb8ffc9fc5bebde048fbd068.
Streamlines some of the logic so pg upmap activity is properly
initalized, and updated in offline mode as well as online.

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/pybind/mgr/balancer/module.py | 37 ++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 54780d1eac82..000b0a3a6cc9 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -340,8 +340,11 @@ class Module(MgrModule):
     success_string = 'Optimization plan created successfully'
     in_progress_string = 'in progress'
     last_pg_upmap: List[Dict[str, Any]] = []
-    added_pg_upmap_items: List[Dict[str, Any]] = []
-    removed_pg_upmap_items: List[Dict[str, Any]] = []
+    pg_upmap_items_added: List[Dict[str, Any]] = []
+    pg_upmap_items_removed: List[Dict[str, Any]] = []
+    last_pg_upmap_primaries: List[Dict[str, Any]] = []
+    pg_upmap_primaries_added: List[Dict[str, Any]] = []
+    pg_upmap_activity_initalized = False
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super(Module, self).__init__(*args, **kwargs)
@@ -376,10 +379,12 @@ def show_status_detail(self) -> Tuple[int, str, str]:
             'optimize_result': self.optimize_result,
             'no_optimization_needed': self.no_optimization_needed,
             'mode': self.get_module_option('mode'),
-            'added_pg_upmap_items': self.added_pg_upmap_items,
-            'removed_pg_upmap_items': self.removed_pg_upmap_items,
+            'pg_upmap_items_added': self.pg_upmap_items_added,
+            'pg_upmap_items_removed': self.pg_upmap_items_removed,
+            'pg_upmap_primaries_added': self.pg_upmap_primaries_added,
+            'pg_upmap_primaries_removed': self.pg_upmap_primaries_removed
         }
-        return (0, json.dumps(s, indent=4), '')
+        return (0, json.dumps(s, indent=4, sort_keys=True), '')
 
     @CLICommand('balancer mode')
     def set_mode(self, mode: Mode) -> Tuple[int, str, str]:
@@ -660,6 +665,7 @@ def plan_execute(self, plan: str) -> Tuple[int, str, str]:
         if not plan_:
             return (-errno.ENOENT, '', f'plan {plan} not found')
         r, detail = self.execute(plan_)
+        self.update_pg_upmap_activity()  # update pg activity in `balancer status detail`
         self.plan_rm(plan)
         return (r, '', detail)
 
@@ -745,15 +751,13 @@ def serve(self) -> None:
                 start = time.time()
                 r, detail = self.optimize(plan)
                 end = time.time()
-                self.added_pg_upmap_items = [pg for pg in osdmap.dump().get('pg_upmap_items', '') if pg not in self.last_pg_upmap]
-                self.removed_pg_upmap_items = [pg for pg in self.last_pg_upmap if pg not in osdmap.dump().get('pg_upmap_items', '')]
-                self.last_pg_upmap = osdmap.dump().get('pg_upmap_items', '')
                 self.last_optimize_duration = str(datetime.timedelta(seconds=(end - start)))
                 if r == 0:
                     self.optimize_result = self.success_string
                     self.execute(plan)
                 else:
                     self.optimize_result = detail
+                self.update_pg_upmap_activity()  # update pg activity in `balancer status detail`
                 self.optimizing = False
             self.log.debug('Sleeping for %d', sleep_interval)
             self.event.wait(sleep_interval)
@@ -1578,6 +1582,23 @@ def gather_telemetry(self) -> Dict[str, Any]:
             'mode': self.mode,
         }
 
+    def update_pg_upmap_activity(self) -> None:
+        osdmap = self.get_osdmap()
+        if not self.pg_upmap_activity_initalized:
+            self.last_pg_upmap = osdmap.dump().get('pg_upmap_items', '')
+            self.last_pg_upmap_primaries = osdmap.dump().get('pg_upmap_primaries', '')
+            self.pg_upmap_activity_initalized = True
+
+        # update pg_upmap_items
+        self.pg_upmap_items_added = [pg for pg in osdmap.dump().get('pg_upmap_items', '') if pg not in self.last_pg_upmap]
+        self.pg_upmap_items_removed = [pg for pg in self.last_pg_upmap if pg not in osdmap.dump().get('pg_upmap_items', '')]
+        self.last_pg_upmap = osdmap.dump().get('pg_upmap_items', '')
+
+        # update pg_upmap_primaries
+        self.pg_upmap_primaries_added = [pg for pg in osdmap.dump().get('pg_upmap_primaries', '') if pg not in self.last_pg_upmap_primaries]
+        self.pg_upmap_primaries_removed = [pg for pg in self.last_pg_upmap_primaries if pg not in osdmap.dump().get('pg_upmap_primaries', '')]
+        self.last_pg_upmap_primaries = osdmap.dump().get('pg_upmap_primaries', '')
+
     def self_test(self) -> None:
         # turn balancer on
         self.on()

From 33b5f3ddbb46c823d481e65a9761278da18292e1 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Wed, 3 Jan 2024 20:52:00 +0000
Subject: [PATCH 1639/2492] qa/suites/rados/thrash/d-balancer: add read and
 upmap-read modes to balancer testing

Here, we turn the balancer mgr module to read and upmap read mode
during various thrashing scenarios.

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/suites/rados/thrash/d-balancer/read.yaml       | 10 ++++++++++
 qa/suites/rados/thrash/d-balancer/upmap-read.yaml | 10 ++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 qa/suites/rados/thrash/d-balancer/read.yaml
 create mode 100644 qa/suites/rados/thrash/d-balancer/upmap-read.yaml

diff --git a/qa/suites/rados/thrash/d-balancer/read.yaml b/qa/suites/rados/thrash/d-balancer/read.yaml
new file mode 100644
index 000000000000..4a6194a3a94f
--- /dev/null
+++ b/qa/suites/rados/thrash/d-balancer/read.yaml
@@ -0,0 +1,10 @@
+tasks:
+- exec:
+    mon.a:
+      - ceph config set mgr mgr/balancer/log_level debug
+      - ceph balancer status
+      - ceph osd set-require-min-compat-client reef
+      - ceph balancer mode read
+      - ceph balancer on
+      - ceph balancer status
+      - ceph balancer status detail
diff --git a/qa/suites/rados/thrash/d-balancer/upmap-read.yaml b/qa/suites/rados/thrash/d-balancer/upmap-read.yaml
new file mode 100644
index 000000000000..b425105ea56d
--- /dev/null
+++ b/qa/suites/rados/thrash/d-balancer/upmap-read.yaml
@@ -0,0 +1,10 @@
+tasks:
+- exec:
+    mon.a:
+      - ceph config set mgr mgr/balancer/log_level debug
+      - ceph balancer status
+      - ceph osd set-require-min-compat-client reef
+      - ceph balancer mode upmap-read
+      - ceph balancer on
+      - ceph balancer status
+      - ceph balancer status detail

From cb10c0d82127eb46529283b71f5e04b2f72e2cda Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Fri, 26 Jan 2024 17:32:43 +0000
Subject: [PATCH 1640/2492] osd: clear out unneeded pending pg-upmap-primary
 mappings

If the score did not improve, we should clear out any
pending pg-upmap-primary mappings so they don't execute
in situations where the same incremental is used to balance
multiple pools (i.e. in the balancer mgr module).

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/osd/OSDMap.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 5773695b77ad..4cd9bff44416 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5143,6 +5143,12 @@ int OSDMap::balance_primaries(
         num_changes++;
       }
     }
+  } else { // clear out any mappings that were made since the score didn't improve
+    for (auto [pg, mapped] : prim_pgs_to_check) {
+      if (mapped) {
+	pending_inc->new_pg_upmap_primary.erase(pg);
+      }
+    }
   }
 
   ldout(cct, 10) << __func__ << " num_changes " << num_changes << dendl;

From 763f6b7a22e846962f388d58fd7e699cbf16ffe7 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 29 Jan 2024 05:33:58 +1000
Subject: [PATCH 1641/2492] doc/architecture.rst: improve rados definition

Improve the definition of RADOS, and link to information about RADOS.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index 1be58f682925..148e495ff1b3 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -19,9 +19,14 @@ The Ceph Storage Cluster
 ========================
 
 Ceph provides an infinitely scalable :term:`Ceph Storage Cluster` based upon
-:abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, which you can read
-about in `RADOS - A Scalable, Reliable Storage Service for Petabyte-scale
-Storage Clusters`_.
+:abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, a reliable,
+distributed storage service that uses the intelligence in each of its nodes to
+secure the data it stores and to provide that data to client. See Sage Weil's
+"`The RADOS Object Store
+<https://ceph.io/en/news/blog/2009/the-rados-distributed-object-store/>`_" blog
+post for a brief explanation of RADOS and see `RADOS - A Scalable, Reliable
+Storage Service for Petabyte-scale Storage Clusters`_ for an exhaustive
+explanation of :term:`RADOS`.
 
 A Ceph Storage Cluster consists of multiple types of daemons:
 

From 5554e565ca7ca9c9d6bc70e245be63d947722eda Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Mon, 29 Jan 2024 00:58:25 +0000
Subject: [PATCH 1642/2492] mgr: pin pytest to version 7.4.4

On 2024-01-27, pytest updated to 8.0.0,
which broke run-tox-mgr.

https://docs.pytest.org/en/stable/changelog.html

==================================== ERRORS ====================================
_____________________ ERROR collecting alerts/__init__.py ______________________
alerts/__init__.py:2: in <module>
    from .module import Alerts
alerts/module.py:6: in <module>
    from mgr_module import CLIReadCommand, HandleCommandResult, MgrModule, Option
mgr_module.py:1: in <module>
    import ceph_module  # noqa
E   ModuleNotFoundError: No module named 'ceph_module'
______________________ ERROR collecting alerts/module.py _______________________
alerts/module.py:6: in <module>
    from mgr_module import CLIReadCommand, HandleCommandResult, MgrModule, Option
mgr_module.py:1: in <module>
    import ceph_module  # noqa
E   ModuleNotFoundError: No module named 'ceph_module'
____________________ ERROR collecting balancer/__init__.py _____________________
balancer/__init__.py:2: in <module>
    from .module import Module
balancer/module.py:12: in <module>
    from mgr_module import CLIReadCommand, CLICommand, CommandResult, MgrModule, Option, OSDMap, CephReleases
mgr_module.py:1: in <module>
    import ceph_module  # noqa
E   ModuleNotFoundError: No module named 'ceph_module'
_____________________ ERROR collecting balancer/module.py ______________________
balancer/module.py:12: in <module>
    from mgr_module import CLIReadCommand, CLICommand, CommandResult, MgrModule, Option, OSDMap, CephReleases
mgr_module.py:1: in <module>
    import ceph_module  # noqa
E   ModuleNotFoundError: No module named 'ceph_module'

Fixes: https://tracker.ceph.com/issues/64200
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/pybind/mgr/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pybind/mgr/requirements.txt b/src/pybind/mgr/requirements.txt
index 1c7b326a70f0..7daee3323b8c 100644
--- a/src/pybind/mgr/requirements.txt
+++ b/src/pybind/mgr/requirements.txt
@@ -2,3 +2,4 @@
 asyncssh==2.9
 kubernetes==11.0.0
 urllib3==1.26.15
+pytest==7.4.4

From 05a85848a01c443d67564a9904a778acfe19cee5 Mon Sep 17 00:00:00 2001
From: Wei Wang <lightmelodies@outlook.com>
Date: Mon, 29 Jan 2024 08:26:24 +0000
Subject: [PATCH 1643/2492] mon: fix health store size growing infinitely

The `check_mutes` wrongly marks `changed` to true, trigger `propose_pending` and block following `maybe_trim` logic (`have_pending` will be always be false); as a result, the health store will never be trimmed.

Signed-off-by: Wei Wang <lightmelodies@outlook.com>
---
 src/mon/HealthMonitor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 2a21b99111a5..4d2303d09fbc 100644
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -400,7 +400,7 @@ void HealthMonitor::tick()
 
 bool HealthMonitor::check_mutes()
 {
-  bool changed = true;
+  bool changed = false;
   auto now = ceph_clock_now();
   health_check_map_t all;
   gather_all_health_checks(&all);

From 4b514af39fce88b426e8e35582c3c7e63373f21b Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 23 Jan 2024 15:02:09 +0000
Subject: [PATCH 1644/2492] crimson/osd/pg: do_delete_work use claim_and_reset

otherwise, the txn will be committed twice.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 966ca8a101e7..1c997b71c5fc 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -434,7 +434,7 @@ PG::do_delete_work(ceph::os::Transaction &t, ghobject_t _next)
     t.remove(coll_ref->get_cid(), pgmeta_oid);
     t.remove_collection(coll_ref->get_cid());
     (void) shard_services.get_store().do_transaction(
-      coll_ref, std::move(t)).then([this] {
+      coll_ref, t.claim_and_reset()).then([this] {
       return shard_services.remove_pg(pgid);
     });
     return {next, false};

From 2fa8a07f2074bf364f7e6300d88bc5f84e22fc5b Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 28 Jan 2024 16:17:35 +0000
Subject: [PATCH 1645/2492] crimson/osd/shard_services:
 dispatch_context_messages use claim_and_reset

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/shard_services.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 0a6a739af274..604c045eb9b8 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -764,8 +764,7 @@ seastar::future<> ShardServices::dispatch_context_transaction(
   DEBUG("do_transaction ...");
   auto ret = get_store().do_transaction(
     col,
-    std::move(ctx.transaction));
-  ctx.reset_transaction();
+    ctx.transaction.claim_and_reset());
   return ret;
 }
 

From fb79df410d39ad01c9192433e03d8a0a33c81568 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Mon, 29 Jan 2024 15:42:10 +0530
Subject: [PATCH 1646/2492] mgr/dashboard: Create subvol of same name in
 different group

Fixes https://tracker.ceph.com/issues/64112

Issue:
Currently, we are unable to create subvolume of same name in different
subvolume group

Fix:
We are validating only the filesystem name of subvolume
which is stopping the creation a subvolume of same name.
Added more granularity , by adding the subvolumegroup name.

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../cephfs-subvolume-form/cephfs-subvolume-form.component.ts  | 3 ++-
 .../frontend/src/app/shared/api/cephfs-subvolume.service.ts   | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-form/cephfs-subvolume-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-form/cephfs-subvolume-form.component.ts
index 2c2fe8f9fa0a..0d7df7c5225b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-form/cephfs-subvolume-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-form/cephfs-subvolume-form.component.ts
@@ -108,7 +108,8 @@ export class CephfsSubvolumeFormComponent extends CdForm implements OnInit {
             this.cephFsSubvolumeService,
             null,
             null,
-            this.fsName
+            this.fsName,
+            this.subVolumeGroupName
           )
         ]
       }),
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
index ad0ce248064d..e95503ff56b9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-subvolume.service.ts
@@ -75,8 +75,8 @@ export class CephfsSubvolumeService {
     });
   }
 
-  exists(subVolumeName: string, fsName: string) {
-    return this.info(fsName, subVolumeName).pipe(
+  exists(subVolumeName: string, fsName: string, subVolumeGroupName: string = '') {
+    return this.info(fsName, subVolumeName, subVolumeGroupName).pipe(
       mapTo(true),
       catchError((error: Event) => {
         if (_.isFunction(error.preventDefault)) {

From c0431a0bb1a134af968e4b505cfbd36aca87ecd4 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sat, 27 Jan 2024 07:07:47 -0600
Subject: [PATCH 1647/2492] common: add some missing include files to
 AsyncReserver.h

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/AsyncReserver.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/common/AsyncReserver.h b/src/common/AsyncReserver.h
index 1a19dc70d1bb..32b558b308ba 100644
--- a/src/common/AsyncReserver.h
+++ b/src/common/AsyncReserver.h
@@ -16,6 +16,9 @@
 #define ASYNC_RESERVER_H
 
 #include "common/Formatter.h"
+#include "common/ceph_context.h"
+#include "common/ceph_mutex.h"
+#include "include/Context.h"
 
 #define rdout(x) lgeneric_subdout(cct,reserver,x)
 

From 0efcbe33a1da4f9ea35bdf46fa5a70404c42bd28 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 01:04:41 -0600
Subject: [PATCH 1648/2492] osd: improve MOSDScrubReserver log formatting

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/messages/MOSDScrubReserve.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/messages/MOSDScrubReserve.h b/src/messages/MOSDScrubReserve.h
index a2670a8e110f..2bb13ed74f82 100644
--- a/src/messages/MOSDScrubReserve.h
+++ b/src/messages/MOSDScrubReserve.h
@@ -37,7 +37,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   reservation_nonce_t reservation_nonce{0};
   /// 'false' if the (legacy) primary is expecting an immediate
   /// granted / denied response
-  bool wait_for_resources{false};
+  bool wait_for_resources{true};
 
   epoch_t get_map_epoch() const override {
     return map_epoch;
@@ -63,23 +63,23 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   }
 
   void print(std::ostream& out) const {
-    out << "MOSDScrubReserve(" << pgid << " ";
+    out << "MOSDScrubReserve(" << pgid << ",";
     switch (type) {
     case REQUEST:
-      out << (wait_for_resources ? "QREQUEST " : "REQUEST ");
+      out << (wait_for_resources ? "QREQUEST" : "REQUEST");
       break;
     case GRANT:
-      out << "GRANT ";
+      out << "GRANT";
       break;
     case REJECT:
-      out << "REJECT ";
+      out << "REJECT";
       break;
     case RELEASE:
-      out << "RELEASE ";
+      out << "RELEASE";
       break;
     }
-    out << "e" << map_epoch << " from: " << from
-	<< " reservation_nonce: " << reservation_nonce << ")";
+    out << ",e:" << map_epoch << ",from:" << from
+	<< ",reservation_nonce:" << reservation_nonce << ")";
     return;
   }
 

From a57036a829674f01d27c6fa13ac0c8c826d21b34 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Sun, 10 Dec 2023 17:44:33 +0000
Subject: [PATCH 1649/2492] mgr/dashboard: snap schedule module detect/eneable

Fixes: https://tracker.ceph.com/issues/63768
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 ...ephfs-snapshotschedule-list.component.html |  74 +++++++-----
 .../cephfs-snapshotschedule-list.component.ts | 107 ++++++++++++++++--
 .../rgw-multisite-details.component.html      |   8 +-
 3 files changed, 149 insertions(+), 40 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
index 2e270057d579..0346de67f941 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
@@ -1,44 +1,66 @@
-<ng-container *ngIf="isLoading$ | async">
-  <cd-loading-panel>
-    <span i18n>Loading snapshot schedules...</span>
-  </cd-loading-panel>
-</ng-container>
+<cd-alert-panel
+  *ngIf="(snapScheduleModuleStatus$ | async) === false"
+  type="info"
+  spacingClass="mb-3"
+  i18n
+  class="align-items-center"
+>
+  In order to access the snapshot scheduler feature, the snap_scheduler module must be enabled
+  <button
+    class="btn btn-light mx-2"
+    type="button"
+    (click)="enableSnapshotSchedule()">
+    Enable
+  </button>
+</cd-alert-panel>
 
-<ng-template #pathTpl
-             let-row="row">
+<ng-template
+  #pathTpl
+  let-row="row">
   <span
     class="fw-bold"
     [ngbTooltip]="fullpathTpl"
-    triggers="click:blur">{{row.path | path}}</span>
+    triggers="click:blur">
+    {{ row.path | path }}
+  </span>
 
-  <span *ngIf="row.active; else inactiveStatusTpl">
-    <i [ngClass]="[icons.success, icons.large]"
-       ngbTooltip="{{row.path}} is active"
-       class="text-success"></i>
+  <span
+  *ngIf="row.active; else inactiveStatusTpl">
+    <i
+      [ngClass]="[icons.success, icons.large]"
+      ngbTooltip="{{ row.path }} is active"
+      class="text-success"
+    ></i>
   </span>
 
   <ng-template #inactiveStatusTpl>
-    <i [ngClass]="[icons.warning, icons.large]"
-       class="text-warning"
-       ngbTooltip="{{row.path}} has been deactivated"></i>
+    <i
+      [ngClass]="[icons.warning, icons.large]"
+      class="text-warning"
+      ngbTooltip="{{ row.path }} has been deactivated"
+    ></i>
   </ng-template>
 
   <ng-template #fullpathTpl>
-  <span data-toggle="tooltip"
-        [title]="row.path"
-        class="font-monospace">{{ row.path }}
-    <cd-copy-2-clipboard-button *ngIf="row.path"
-                                [source]="row.path"
-                                [byId]="false"
-                                [showIconOnly]="true">
-    </cd-copy-2-clipboard-button>
-  </span>
-</ng-template>
-
+    <span
+      data-toggle="tooltip"
+      [title]="row.path"
+      class="font-monospace"
+      >{{ row.path }}
+      <cd-copy-2-clipboard-button
+        *ngIf="row.path"
+        [source]="row.path"
+        [byId]="false"
+        [showIconOnly]="true"
+      >
+      </cd-copy-2-clipboard-button>
+    </span>
+  </ng-template>
 </ng-template>
 
 <cd-table
   [data]="snapshotSchedules$ | async"
+  *ngIf="snapScheduleModuleStatus$ | async"
   columnMode="flex"
   [columns]="columns"
   selectionType="single"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index d5f24e897284..14752b7e2a8f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -1,7 +1,15 @@
-import { Component, Input, OnChanges, OnInit, SimpleChanges, ViewChild } from '@angular/core';
+import {
+  Component,
+  Input,
+  OnChanges,
+  OnDestroy,
+  OnInit,
+  SimpleChanges,
+  ViewChild
+} from '@angular/core';
 import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
-import { BehaviorSubject, Observable } from 'rxjs';
-import { finalize, shareReplay, switchMap } from 'rxjs/operators';
+import { BehaviorSubject, Observable, Subscription, of, timer } from 'rxjs';
+import { finalize, map, shareReplay, switchMap } from 'rxjs/operators';
 import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
 import { CdForm } from '~/app/shared/forms/cd-form';
 import { CdTableAction } from '~/app/shared/models/cd-table-action';
@@ -14,21 +22,31 @@ import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
 import { ModalService } from '~/app/shared/services/modal.service';
 import { Icons } from '~/app/shared/enum/icons.enum';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+import { MgrModuleService } from '~/app/shared/api/mgr-module.service';
+import { NotificationService } from '~/app/shared/services/notification.service';
+import { BlockUI, NgBlockUI } from 'ng-block-ui';
+import { NotificationType } from '~/app/shared/enum/notification-type.enum';
 
 @Component({
   selector: 'cd-cephfs-snapshotschedule-list',
   templateUrl: './cephfs-snapshotschedule-list.component.html',
   styleUrls: ['./cephfs-snapshotschedule-list.component.scss']
 })
-export class CephfsSnapshotscheduleListComponent extends CdForm implements OnInit, OnChanges {
+export class CephfsSnapshotscheduleListComponent
+  extends CdForm
+  implements OnInit, OnChanges, OnDestroy {
   @Input() fsName!: string;
 
   @ViewChild('pathTpl', { static: true })
   pathTpl: any;
 
+  @BlockUI()
+  blockUI: NgBlockUI;
+
   snapshotSchedules$!: Observable<SnapshotSchedule[]>;
   subject$ = new BehaviorSubject<SnapshotSchedule[]>([]);
-  isLoading$ = new BehaviorSubject<boolean>(true);
+  snapScheduleModuleStatus$ = new BehaviorSubject<boolean>(false);
+  moduleServiceListSub!: Subscription;
   columns: CdTableColumn[] = [];
   tableActions: CdTableAction[] = [];
   context!: CdTableFetchDataContext;
@@ -39,10 +57,15 @@ export class CephfsSnapshotscheduleListComponent extends CdForm implements OnIni
   selectedName: string = '';
   icons = Icons;
 
+  MODULE_NAME = 'snap_schedule';
+  ENABLE_MODULE_TIMER = 2 * 1000;
+
   constructor(
     private snapshotScheduleService: CephfsSnapshotScheduleService,
     private authStorageService: AuthStorageService,
-    private modalService: ModalService
+    private modalService: ModalService,
+    private mgrModuleService: MgrModuleService,
+    private notificationService: NotificationService
   ) {
     super();
     this.permissions = this.authStorageService.getPermissions();
@@ -55,13 +78,27 @@ export class CephfsSnapshotscheduleListComponent extends CdForm implements OnIni
   }
 
   ngOnInit(): void {
+    this.moduleServiceListSub = this.mgrModuleService
+      .list()
+      .pipe(
+        map((modules: any[]) => modules.find((module) => module?.['name'] === this.MODULE_NAME))
+      )
+      .subscribe({
+        next: (module: any) => this.snapScheduleModuleStatus$.next(module?.enabled)
+      });
+
     this.snapshotSchedules$ = this.subject$.pipe(
       switchMap(() =>
-        this.snapshotScheduleService
-          .getSnapshotScheduleList('/', this.fsName)
-          .pipe(finalize(() => this.isLoading$.next(false)))
-      ),
-      shareReplay(1)
+        this.snapScheduleModuleStatus$.pipe(
+          switchMap((status) => {
+            if (!status) {
+              return of([]);
+            }
+            return this.snapshotScheduleService.getSnapshotScheduleList('/', this.fsName);
+          }),
+          shareReplay(1)
+        )
+      )
     );
 
     this.columns = [
@@ -78,6 +115,10 @@ export class CephfsSnapshotscheduleListComponent extends CdForm implements OnIni
     this.tableActions = [];
   }
 
+  ngOnDestroy(): void {
+    this.moduleServiceListSub.unsubscribe();
+  }
+
   fetchData() {
     this.subject$.next([]);
   }
@@ -96,4 +137,48 @@ export class CephfsSnapshotscheduleListComponent extends CdForm implements OnIni
       { size: 'lg' }
     );
   }
+
+  enableSnapshotSchedule() {
+    let $obs;
+    const fnWaitUntilReconnected = () => {
+      timer(this.ENABLE_MODULE_TIMER).subscribe(() => {
+        // Trigger an API request to check if the connection is
+        // re-established.
+        this.mgrModuleService.list().subscribe(
+          () => {
+            // Resume showing the notification toasties.
+            this.notificationService.suspendToasties(false);
+            // Unblock the whole UI.
+            this.blockUI.stop();
+            // Reload the data table content.
+            this.notificationService.show(
+              NotificationType.success,
+              $localize`Enabled Snapshot Schedule Module`
+            );
+            // Reload the data table content.
+          },
+          () => {
+            fnWaitUntilReconnected();
+          }
+        );
+      });
+    };
+
+    if (!this.snapScheduleModuleStatus$.value) {
+      $obs = this.mgrModuleService
+        .enable(this.MODULE_NAME)
+        .pipe(finalize(() => this.snapScheduleModuleStatus$.next(true)));
+    }
+    $obs.subscribe(
+      () => undefined,
+      () => {
+        // Suspend showing the notification toasties.
+        this.notificationService.suspendToasties(true);
+        // Block the whole UI to prevent user interactions until
+        // the connection to the backend is reestablished
+        this.blockUI.start($localize`Reconnecting, please wait ...`);
+        fnWaitUntilReconnected();
+      }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-details/rgw-multisite-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-details/rgw-multisite-details.component.html
index 5274cf73a556..291013a5ce2f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-details/rgw-multisite-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-details/rgw-multisite-details.component.html
@@ -4,10 +4,12 @@
       <cd-alert-panel   *ngIf="!rgwModuleStatus"
                         type="info"
                         spacingClass="mb-3"
+                        class="d-flex align-items-center"
                         i18n>In order to access the import/export feature, the rgw module must be enabled
-        <a class="text-decoration-underline"
-           (click)="enableRgwModule()">
-           Enable the Object Gateway Module</a>
+
+        <button class="btn btn-light mx-2"
+                type="button"
+                (click)="enableRgwModule()">Enable</button>
       </cd-alert-panel>
       <cd-alert-panel   *ngIf="restartGatewayMessage"
                         type="warning"

From d304cbb06e3d8d8a4e54a893214fc4e20c8457ab Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Mon, 29 Jan 2024 19:25:00 +0800
Subject: [PATCH 1650/2492] script/run-make: install lvm2 for make check
 cephadm test

The make check test `run-tox-cephdam` reports error:
```
        if errors:
>           raise Error('\nERROR: '.join(errors))
E           cephadmlib.exceptions.Error: lvcreate binary does not appear to be installed

cephadm.py:4434: Error
```
So let's installing lvm2 for make check cephadm test.

Fixes: https://tracker.ceph.com/issues/64122

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 src/script/run-make.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 42d8a94aee1f..938beb9ec58a 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -56,7 +56,7 @@ function prepare() {
 
     if test -f ./install-deps.sh ; then
         ci_debug "Running install-deps.sh"
-        INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang"
+        INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang lvm2"
         $DRY_RUN source ./install-deps.sh || return 1
         trap clean_up_after_myself EXIT
     fi

From d4d08047f77b16136a4c2e0f4a28c052b26a7224 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 29 Jan 2024 09:50:28 +0000
Subject: [PATCH 1651/2492] crimson/osd/pg_map: increase verbosity

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 14f2b9f03e84..45f3bab2558e 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -47,14 +47,16 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
       DEBUG("mapping pg {} to core: {} with num_pgs of: {}",
             pgid, insert_iter->second, core_iter->second);
       return primary_mapping.container().invoke_on_others(
-        [pgid = insert_iter->first, core = insert_iter->second]
+        [pgid = insert_iter->first, core = insert_iter->second, FNAME]
         (auto &other_mapping) {
         ceph_assert_always(core != NULL_CORE);
         auto [insert_iter, inserted] = other_mapping.pg_to_core.emplace(pgid, core);
         ceph_assert_always(inserted);
+        DEBUG("mapping pg {} to core: {}", pgid, core);
       });
-    }).then([this, pgid] {
+    }).then([this, pgid, FNAME] {
       auto find_iter = pg_to_core.find(pgid);
+      DEBUG("returning pg {} mapping to core {}", pgid, find_iter->second);
       return seastar::make_ready_future<core_id_t>(find_iter->second);
     });
   }
@@ -63,7 +65,7 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
 seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
   LOG_PREFIX(PGShardMapping::remove_pg_mapping);
   DEBUG("{}", pgid);
-  return container().invoke_on(0, [pgid](auto &primary_mapping) {
+  return container().invoke_on(0, [pgid, FNAME](auto &primary_mapping) {
     auto iter = primary_mapping.pg_to_core.find(pgid);
     ceph_assert_always(iter != primary_mapping.pg_to_core.end());
     ceph_assert_always(iter->second != NULL_CORE);
@@ -72,12 +74,14 @@ seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
     ceph_assert_always(count_iter->second > 0);
     --(count_iter->second);
     primary_mapping.pg_to_core.erase(iter);
+    DEBUG("pg {} mapping erased", pgid);
     return primary_mapping.container().invoke_on_others(
-      [pgid](auto &other_mapping) {
+      [pgid, FNAME](auto &other_mapping) {
       auto iter = other_mapping.pg_to_core.find(pgid);
       ceph_assert_always(iter != other_mapping.pg_to_core.end());
       ceph_assert_always(iter->second != NULL_CORE);
       other_mapping.pg_to_core.erase(iter);
+      DEBUG("pg {} mapping erased", pgid);
     });
   });
 }

From 28afcff384a4075b372209fcfed9356d764b51eb Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 29 Jan 2024 09:57:45 +0000
Subject: [PATCH 1652/2492] crimson/osd/pg_map: add assertion

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/pg_map.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index 45f3bab2558e..c0a81f300b3d 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -56,6 +56,7 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
       });
     }).then([this, pgid, FNAME] {
       auto find_iter = pg_to_core.find(pgid);
+      ceph_assert_always(find_iter != pg_to_core.end());
       DEBUG("returning pg {} mapping to core {}", pgid, find_iter->second);
       return seastar::make_ready_future<core_id_t>(find_iter->second);
     });

From 21195114d4852ce4fba3b2f1bcc1985ddf9a6ac6 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Thu, 25 Jan 2024 13:57:09 +0100
Subject: [PATCH 1653/2492] mgr/rook: adding metrics monitoring e2e testing
 Fixes: https://tracker.ceph.com/issues/64183

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 .../rook/ci/scripts/bootstrap-rook-cluster.sh | 17 +++-
 .../cluster-prometheus-monitoring.feature     | 14 +++
 .../ci/tests/features/steps/implementation.py | 93 +++++++++++++++++++
 3 files changed, 122 insertions(+), 2 deletions(-)
 create mode 100644 src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature

diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index d3c35544131e..dcd22fce9a75 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -120,7 +120,7 @@ configure_libvirt(){
 	echo "User added to libvirt group successfully."
 	sudo systemctl enable --now libvirtd
 	sudo systemctl restart libvirtd
-	sleep 10 # wait some time for libvirtd service to restart
+	sleep 30 # wait some time for libvirtd service to restart
 	newgrp libvirt
     else
 	echo "Error adding user to libvirt group."
@@ -146,7 +146,7 @@ recreate_default_network(){
 
     # restart libvirtd service and wait a little bit for the service
     sudo systemctl restart libvirtd
-    sleep 10
+    sleep 30
 
     # Just some debugging information
     all_networks=$(virsh net-list --all)
@@ -161,6 +161,17 @@ enable_rook_orchestrator() {
     $KUBECTL -n "$ROOK_CLUSTER_NS" exec -it deploy/rook-ceph-tools -- ceph orch status
 }
 
+enable_monitoring() {
+    echo "Enabling monitoring"
+    $KUBECTL apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml
+    $KUBECTL wait --for=condition=ready pod -l app.kubernetes.io/name=prometheus-operator --timeout=90s
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/rbac.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/service-monitor.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/exporter-service-monitor.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus.yaml
+    $KUBECTL apply -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/monitoring/prometheus-service.yaml
+}
+
 ####################################################################
 ####################################################################
 
@@ -174,6 +185,8 @@ create_rook_cluster
 wait_for_rook_operator
 wait_for_ceph_cluster
 enable_rook_orchestrator
+enable_monitoring
+sleep 30 # wait for the metrics cache warmup
 
 ####################################################################
 ####################################################################
diff --git a/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature b/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature
new file mode 100644
index 000000000000..5180c72939de
--- /dev/null
+++ b/src/pybind/mgr/rook/ci/tests/features/cluster-prometheus-monitoring.feature
@@ -0,0 +1,14 @@
+Feature: Testing Rook orchestrator commands
+    Ceph has been installed using the cluster CRD available in deploy/examples/cluster-test.yaml
+
+    Scenario: Verify Prometheus metrics endpoint is working properly
+      Given I can get prometheus server configuration
+      Given the prometheus server is serving metrics
+
+    Scenario: Verify some basic metrics are working properly
+      Given I can get prometheus server configuration
+      Given the prometheus server is serving metrics
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.0" and value equal to 1
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.1" and value equal to 1
+      Then the response contains the metric "ceph_osd_in" where "ceph_daemon" is "osd.2" and value equal to 1
+      Then the response contains the metric "ceph_mon_quorum_status" where "ceph_daemon" is "mon.a" and value equal to 1
diff --git a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
index 69dcde458855..59cb117c8b1e 100644
--- a/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
+++ b/src/pybind/mgr/rook/ci/tests/features/steps/implementation.py
@@ -1,7 +1,22 @@
+import requests
+from behave import given, when, then
 from behave import *
 from utils import *
+import subprocess
 import re
 
+PROMETHEUS_SERVER_URL = None
+
+def get_prometheus_pod_host_ip():
+    try:
+        command = "minikube --profile minikube kubectl -- -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0"
+        result = subprocess.run(command, shell=True, capture_output=True, text=True, check=True)
+        host_ip = result.stdout.strip()
+        return host_ip
+    except subprocess.CalledProcessError as e:
+        print(f"Error running command: {e}")
+        return None
+
 @when("I run ceph command")
 def run_step(context):
     context.output = run_ceph_commands(context.text)
@@ -25,3 +40,81 @@ def verify_fuzzy_result_step(context):
         if not re.match(expected_lines[n], output_lines[n]):
             display_side_by_side(expected_lines[n], output_lines[n])
             assert False, ""
+
+@given('I can get prometheus server configuration')
+def step_get_prometheus_server_ip(context):
+    global PROMETHEUS_SERVER_URL
+    try:
+        PROMETHEUS_SERVER_URL = f"http://{get_prometheus_pod_host_ip()}:30900"
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to Prometheus server: {e}")
+        assert False, f"Error connecting to Prometheus server: {e}"
+
+@given('the prometheus server is serving metrics')
+def step_given_server_running(context):
+    try:
+        params = {'match[]': '{__name__!=""}'}
+        response = requests.get(f"{PROMETHEUS_SERVER_URL}/federate", params)
+        # Check if the response status code is successful (2xx)
+        response.raise_for_status()
+        # Store the response object in the context for later use
+        context.response = response
+        print(f"Prometheus server is running. Status code: {response.status_code}")
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to Prometheus server: {e}")
+        assert False, f"Error connecting to Prometheus server: {e}"
+
+@when('I query the Prometheus metrics endpoint')
+def step_when_query_metrics_endpoint(context):
+    params = {'match[]': '{__name__!=""}'}
+    context.response = requests.get(f"{PROMETHEUS_SERVER_URL}/federate", params)
+    context.response.raise_for_status()
+
+@then('the response contains the metric "{metric_name}"')
+def step_then_check_metric_value(context, metric_name):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+
+@then('the response contains the metric "{metric_name}" with value equal to {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value == float(expected_value), f"Metric '{metric_name}' value {metric_value} is not equal to {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value greater than {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value > float(expected_value), f"Metric '{metric_name}' value {metric_value} is not greater than {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value less than {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value < float(expected_value), f"Metric '{metric_name}' value {metric_value} is not less than {expected_value}"
+
+@then('the response contains the metric "{metric_name}" with value in the range {min_value}-{max_value}')
+def step_then_check_metric_value(context, metric_name, min_value, max_value):
+    metric_value = parse_metric_value(context.response.text, metric_name)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value >= float(min_value) and metric_value <= float(max_value), f"Metric '{metric_name}' value {metric_value} is not in the range {min_value}-{max_value}"
+
+@then('the response contains the metric "{metric_name}" where "{filter_by_field}" is "{field_value}" and value equal to {expected_value}')
+def step_then_check_metric_value(context, metric_name, expected_value, filter_by_field, field_value):
+    metric_value = parse_metric_value(context.response.text, metric_name, filter_by_field, field_value)
+    assert metric_value is not None, f"Metric '{metric_name}' not found in the response"
+    assert metric_value == float(expected_value), f"Metric '{metric_name}' value {metric_value} is not equal to {expected_value}"
+
+
+def parse_metric_value(metrics_text, metric_name, filter_by_field=None, field_value=None):
+    filter_condition = f'{filter_by_field}="{field_value}"' if filter_by_field and field_value else ''
+    pattern_str = rf'^{metric_name}\{{[^}}]*{filter_condition}[^}}]*\}} (\d+) (\d+)'
+    pattern = re.compile(pattern_str, re.MULTILINE)
+    match = pattern.search(metrics_text)
+    if match:
+        # Extract the values and timestamp from the matched groups
+        metric_value, _ = match.groups()
+        return float(metric_value)
+    else:
+        # Metric not found
+        return None

From 2817d8e25d84bba47951bd68cb3e8651cdb51b56 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 4 Jan 2024 23:16:10 +0100
Subject: [PATCH 1654/2492] mgr/dashboard: set bucket policies

- Adds support to set bucket policies through the Dashboard.
- Rename rgw bucket policy from 'policy' to 'bucket policy' and tab 'Permissions' to 'Policies'
- Fix: hide Tags when none are present on bucket list details and sets bucket form dirty after deleting a tag
- Added service to manage the formatting of a textArea that works with json

Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Fixes: https://tracker.ceph.com/issues/63942
---
 src/pybind/mgr/dashboard/controllers/rgw.py   | 16 +++++--
 .../rgw-bucket-details.component.html         |  8 ++--
 .../rgw-bucket-details.component.ts           |  2 +-
 .../rgw-bucket-form.component.html            | 44 ++++++++++++++++++
 .../rgw-bucket-form.component.ts              | 46 +++++++++++++++++--
 .../app/shared/api/rgw-bucket.service.spec.ts |  6 ++-
 .../src/app/shared/api/rgw-bucket.service.ts  | 10 ++--
 .../src/app/shared/enum/icons.enum.ts         |  1 +
 .../src/app/shared/forms/cd-validators.ts     | 12 +++++
 .../formly-textarea-type.component.ts         | 16 +++----
 .../text-area-json-formatter.service.spec.ts  | 16 +++++++
 .../text-area-json-formatter.service.ts       | 21 +++++++++
 src/pybind/mgr/dashboard/openapi.yaml         |  4 ++
 .../mgr/dashboard/services/rgw_client.py      | 22 +++++++++
 14 files changed, 197 insertions(+), 27 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.ts

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 661dc8afb6da..fc6080691ac5 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -291,6 +291,10 @@ def _get_policy(self, bucket: str):
         rgw_client = RgwClient.admin_instance()
         return rgw_client.get_bucket_policy(bucket)
 
+    def _set_policy(self, bucket_name: str, policy: str, daemon_name, owner):
+        rgw_client = RgwClient.instance(owner, daemon_name)
+        return rgw_client.set_bucket_policy(bucket_name, policy)
+
     def _set_tags(self, bucket_name, tags, daemon_name, owner):
         rgw_client = RgwClient.instance(owner, daemon_name)
         return rgw_client.set_tags(bucket_name, tags)
@@ -347,7 +351,7 @@ def get(self, bucket, daemon_name=None):
         result['encryption'] = encryption['Status']
         result['versioning'] = versioning['Status']
         result['mfa_delete'] = versioning['MfaDelete']
-        result['policy'] = self._get_policy(bucket_name)
+        result['bucket_policy'] = self._get_policy(bucket_name)
 
         # Append the locking configuration.
         locking = self._get_locking(result['owner'], daemon_name, bucket_name)
@@ -360,7 +364,8 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
                lock_enabled='false', lock_mode=None,
                lock_retention_period_days=None,
                lock_retention_period_years=None, encryption_state='false',
-               encryption_type=None, key_id=None, tags=None, daemon_name=None):
+               encryption_type=None, key_id=None, tags=None,
+               bucket_policy=None, daemon_name=None):
         lock_enabled = str_to_bool(lock_enabled)
         encryption_state = str_to_bool(encryption_state)
         try:
@@ -379,6 +384,9 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
             if tags:
                 self._set_tags(bucket, tags, daemon_name, uid)
 
+            if bucket_policy:
+                self._set_policy(bucket, bucket_policy, daemon_name, uid)
+
             return result
         except RequestException as e:  # pragma: no cover - handling is too obvious
             raise DashboardException(e, http_status_code=500, component='rgw')
@@ -388,7 +396,7 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
             encryption_state='false', encryption_type=None, key_id=None,
             mfa_delete=None, mfa_token_serial=None, mfa_token_pin=None,
             lock_mode=None, lock_retention_period_days=None,
-            lock_retention_period_years=None, tags=None, daemon_name=None):
+            lock_retention_period_years=None, tags=None, bucket_policy=None, daemon_name=None):
         encryption_state = str_to_bool(encryption_state)
         # When linking a non-tenant-user owned bucket to a tenanted user, we
         # need to prefix bucket name with '/'. e.g. photos -> /photos
@@ -430,6 +438,8 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
             self._delete_encryption(bucket_name, daemon_name, uid)
         if tags:
             self._set_tags(bucket_name, tags, daemon_name, uid)
+        if bucket_policy:
+            self._set_policy(bucket, bucket_policy, daemon_name, uid)
         return self._append_bid(result)
 
     def delete(self, bucket, purge_objects='true', daemon_name=None):
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
index e96a89b234f9..1732a7fb3b4b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
@@ -102,7 +102,7 @@
         </table>
 
       <!-- Tags -->
-      <ng-container *ngIf="selection.tagset">
+      <ng-container *ngIf="(selection.tagset | keyvalue)?.length">
         <legend i18n>Tags</legend>
         <table class="table table-striped table-bordered">
           <tbody>
@@ -120,15 +120,15 @@
 
     <ng-container ngbNavItem="permissions">
       <a ngbNavLink
-         i18n>Permissions</a>
+         i18n>Policies</a>
       <ng-template ngbNavContent>
 
         <table class="table table-striped table-bordered">
           <tbody>
             <tr>
               <td i18n
-                  class="bold w-25">Policy</td>
-              <td><pre>{{ selection.policy | json}}</pre></td>
+                  class="bold w-25">Bucket policy</td>
+              <td><pre>{{ selection.bucket_policy | json}}</pre></td>
             </tr>
           </tbody>
         </table>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
index 0ecbe0536dff..d731a323818e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
@@ -18,7 +18,7 @@ export class RgwBucketDetailsComponent implements OnChanges {
       this.rgwBucketService.get(this.selection.bid).subscribe((bucket: object) => {
         bucket['lock_retention_period_days'] = this.rgwBucketService.getLockDays(bucket);
         this.selection = bucket;
-        this.selection.policy = JSON.parse(this.selection.policy) || {};
+        this.selection.bucket_policy = JSON.parse(this.selection.bucket_policy) || {};
       });
     }
   }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index a9704c0bdc89..eef66a6da60b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -410,6 +410,50 @@
           </div>
         </div>
 
+        <!-- Policies -->
+        <legend class="cd-header"
+                i18n>Policies
+        </legend>
+        <div class="row">
+          <div class="col-12">
+            <div class="form-group row">
+              <label i18n
+                     class="cd-col-form-label"
+                     for="id">Bucket policy</label>
+              <div class="cd-col-form-input">
+                <textarea #bucketPolicyTextArea
+                          class="form-control resize-vertical"
+                          id="bucket_policy"
+                          formControlName="bucket_policy"
+                          (change)="bucketPolicyOnChange()">
+                </textarea>
+                <span class="invalid-feedback"
+                      *ngIf="bucketForm.showError('bucket_policy', frm, 'invalidJson')"
+                      i18n>Invalid json text</span>
+                <div class="btn-group float-end"
+                     role="group"
+                     aria-label="bucket-policy-helpers">
+                  <button type="button"
+                          id="example-generator-button"
+                          class="btn btn-light my-3"
+                          (click)="openUrl('https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-bucket-policies.html?icmpid=docs_amazons3_console')"
+                          i18n>
+                    <i [ngClass]="[icons.externalUrl]"></i>
+                    Policy examples
+                  </button>
+                  <button type="button"
+                          id="example-generator-button"
+                          class="btn btn-light my-3"
+                          (click)="openUrl('https://awspolicygen.s3.amazonaws.com/policygen.html')"
+                          i18n>
+                    <i [ngClass]="[icons.externalUrl]"></i>
+                    Policy generator
+                  </button>
+                </div>
+              </div>
+            </div>
+          </div>
+        </div>
 
       </div>
       <div class="card-footer">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index 6b90b45e16a6..f7015f609f68 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -1,4 +1,11 @@
-import { AfterViewChecked, ChangeDetectorRef, Component, OnInit } from '@angular/core';
+import {
+  AfterViewChecked,
+  ChangeDetectorRef,
+  Component,
+  OnInit,
+  ViewChild,
+  ElementRef
+} from '@angular/core';
 import { AbstractControl, Validators } from '@angular/forms';
 import { ActivatedRoute, Router } from '@angular/router';
 
@@ -22,6 +29,7 @@ import { RgwBucketMfaDelete } from '../models/rgw-bucket-mfa-delete';
 import { RgwBucketVersioning } from '../models/rgw-bucket-versioning';
 import { RgwConfigModalComponent } from '../rgw-config-modal/rgw-config-modal.component';
 import { BucketTagModalComponent } from '../bucket-tag-modal/bucket-tag-modal.component';
+import { TextAreaJsonFormatterService } from '~/app/shared/services/text-area-json-formatter.service';
 
 @Component({
   selector: 'cd-rgw-bucket-form',
@@ -30,6 +38,9 @@ import { BucketTagModalComponent } from '../bucket-tag-modal/bucket-tag-modal.co
   providers: [RgwBucketEncryptionModel]
 })
 export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewChecked {
+  @ViewChild('bucketPolicyTextArea')
+  public bucketPolicyTextArea: ElementRef<any>;
+
   bucketForm: CdFormGroup;
   editing = false;
   owners: string[] = null;
@@ -70,6 +81,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     private rgwUserService: RgwUserService,
     private notificationService: NotificationService,
     private rgwEncryptionModal: RgwBucketEncryptionModel,
+    private textAreaJsonFormatterService: TextAreaJsonFormatterService,
     public actionLabels: ActionLabelsI18n,
     private readonly changeDetectorRef: ChangeDetectorRef
   ) {
@@ -82,6 +94,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
 
   ngAfterViewChecked(): void {
     this.changeDetectorRef.detectChanges();
+    this.bucketPolicyOnChange();
   }
 
   createForm() {
@@ -129,7 +142,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
         ]
       ],
       lock_mode: ['COMPLIANCE'],
-      lock_retention_period_days: [0, [CdValidators.number(false), lockDaysValidator]]
+      lock_retention_period_days: [0, [CdValidators.number(false), lockDaysValidator]],
+      bucket_policy: ['{}', CdValidators.json()]
     });
   }
 
@@ -217,6 +231,11 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
             if (value['lock_enabled']) {
               this.bucketForm.controls['versioning'].disable();
             }
+            if (value['bucket_policy']) {
+              this.bucketForm
+                .get('bucket_policy')
+                .setValue(JSON.stringify(value['bucket_policy'], null, 2));
+            }
           }
         }
         this.loadingReady();
@@ -240,6 +259,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     }
     const values = this.bucketForm.value;
     const xmlStrTags = this.tagsToXML(this.tags);
+    const bucketPolicy = this.getBucketPolicy();
     if (this.editing) {
       // Edit
       const versioning = this.getVersioningStatus();
@@ -258,7 +278,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['mfa-token-pin'],
           values['lock_mode'],
           values['lock_retention_period_days'],
-          xmlStrTags
+          xmlStrTags,
+          bucketPolicy
         )
         .subscribe(
           () => {
@@ -287,7 +308,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['encryption_enabled'],
           values['encryption_type'],
           values['keyId'],
-          xmlStrTags
+          xmlStrTags,
+          bucketPolicy
         )
         .subscribe(
           () => {
@@ -337,6 +359,10 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     return this.isMfaDeleteEnabled ? RgwBucketMfaDelete.ENABLED : RgwBucketMfaDelete.DISABLED;
   }
 
+  getBucketPolicy() {
+    return this.bucketForm.getValue('bucket_policy') || '{}';
+  }
+
   fileUpload(files: FileList, controlName: string) {
     const file: File = files[0];
     const reader = new FileReader();
@@ -349,6 +375,16 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     });
   }
 
+  bucketPolicyOnChange() {
+    if (this.bucketPolicyTextArea) {
+      this.textAreaJsonFormatterService.format(this.bucketPolicyTextArea);
+    }
+  }
+
+  openUrl(url: string) {
+    window.open(url, '_blank');
+  }
+
   openConfigModal() {
     const modalRef = this.modalService.show(RgwConfigModalComponent, null, { size: 'lg' });
     modalRef.componentInstance.configForm
@@ -374,6 +410,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
 
   deleteTag(index: number) {
     this.tags.splice(index, 1);
+    this.bucketForm.markAsDirty();
+    this.bucketForm.updateValueAndValidity();
   }
 
   private setTag(tag: Record<string, string>, index?: number) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
index 15821c3b6265..e1002373806c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
@@ -60,11 +60,12 @@ describe('RgwBucketService', () => {
         true,
         'aws:kms',
         'qwerty1',
+        null,
         null
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&${RgwHelper.DAEMON_QUERY_PARAM}`
+      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&bucket_policy=null&${RgwHelper.DAEMON_QUERY_PARAM}`
     );
     expect(req.request.method).toBe('POST');
   });
@@ -84,11 +85,12 @@ describe('RgwBucketService', () => {
         '223344',
         'GOVERNANCE',
         '10',
+        null,
         null
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10&tags=null`
+      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10&tags=null&bucket_policy=null`
     );
     expect(req.request.method).toBe('PUT');
   });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
index 87561d92d899..a43b2616f933 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
@@ -60,7 +60,8 @@ export class RgwBucketService extends ApiClient {
     encryption_state: boolean,
     encryption_type: string,
     key_id: string,
-    tags: string
+    tags: string,
+    bucketPolicy: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       return this.http.post(this.url, null, {
@@ -77,6 +78,7 @@ export class RgwBucketService extends ApiClient {
             encryption_type,
             key_id,
             tags: tags,
+            bucket_policy: bucketPolicy,
             daemon_name: params.get('daemon_name')
           }
         })
@@ -97,7 +99,8 @@ export class RgwBucketService extends ApiClient {
     mfaTokenPin: string,
     lockMode: 'GOVERNANCE' | 'COMPLIANCE',
     lockRetentionPeriodDays: string,
-    tags: string
+    tags: string,
+    bucketPolicy: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       params = params.appendAll({
@@ -112,7 +115,8 @@ export class RgwBucketService extends ApiClient {
         mfa_token_pin: mfaTokenPin,
         lock_mode: lockMode,
         lock_retention_period_days: lockRetentionPeriodDays,
-        tags: tags
+        tags: tags,
+        bucket_policy: bucketPolicy
       });
       return this.http.put(`${this.url}/${bucket}`, null, { params: params });
     });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
index c851e1505da3..d3515fb87e4b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
@@ -82,6 +82,7 @@ export enum Icons {
   navicon = 'fa fa-navicon', // Navigation
   areaChart = 'fa fa-area-chart', // Area Chart, dashboard
   eye = 'fa fa-eye', // Observability
+  externalUrl = 'fa fa-external-link', // links to external page
 
   /* Icons for special effect */
   large = 'fa fa-lg', // icon becomes 33% larger
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
index 602a11e7343c..78171f650f5f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/cd-validators.ts
@@ -610,4 +610,16 @@ export class CdValidators {
         );
     };
   }
+
+  static json(): ValidatorFn {
+    return (control: AbstractControl): Record<string, any> | null => {
+      if (!control.value) return null;
+      try {
+        JSON.parse(control.value);
+        return null;
+      } catch (e) {
+        return { invalidJson: true };
+      }
+    };
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/formly-textarea-type/formly-textarea-type.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/formly-textarea-type/formly-textarea-type.component.ts
index a3139f0e2649..654220596ad8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/formly-textarea-type/formly-textarea-type.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/forms/crud-form/formly-textarea-type/formly-textarea-type.component.ts
@@ -1,5 +1,6 @@
 import { Component, ViewChild, ElementRef } from '@angular/core';
 import { FieldType, FieldTypeConfig } from '@ngx-formly/core';
+import { TextAreaJsonFormatterService } from '~/app/shared/services/text-area-json-formatter.service';
 
 @Component({
   selector: 'cd-formly-textarea-type',
@@ -10,16 +11,11 @@ export class FormlyTextareaTypeComponent extends FieldType<FieldTypeConfig> {
   @ViewChild('textArea')
   public textArea: ElementRef<any>;
 
+  constructor(private textAreaJsonFormatterService: TextAreaJsonFormatterService) {
+    super();
+  }
+
   onChange() {
-    const value = this.textArea.nativeElement.value;
-    try {
-      const formatted = JSON.stringify(JSON.parse(value), null, 2);
-      this.textArea.nativeElement.value = formatted;
-      this.textArea.nativeElement.style.height = 'auto';
-      const lineNumber = formatted.split('\n').length;
-      const pixelPerLine = 25;
-      const pixels = lineNumber * pixelPerLine;
-      this.textArea.nativeElement.style.height = pixels + 'px';
-    } catch (e) {}
+    this.textAreaJsonFormatterService.format(this.textArea);
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.spec.ts
new file mode 100644
index 000000000000..fc428f9841a1
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.spec.ts
@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { TextAreaJsonFormatterService } from './text-area-json-formatter.service';
+
+describe('TextAreaJsonFormatterService', () => {
+  let service: TextAreaJsonFormatterService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(TextAreaJsonFormatterService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.ts
new file mode 100644
index 000000000000..0e696022affd
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-area-json-formatter.service.ts
@@ -0,0 +1,21 @@
+import { ElementRef, Injectable } from '@angular/core';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class TextAreaJsonFormatterService {
+  constructor() {}
+
+  format(textArea: ElementRef<any>): void {
+    const value = textArea.nativeElement.value;
+    try {
+      const formatted = JSON.stringify(JSON.parse(value), null, 2);
+      textArea.nativeElement.value = formatted;
+      textArea.nativeElement.style.height = 'auto';
+      const lineNumber = formatted.split('\n').length;
+      const pixelPerLine = 20;
+      const pixels = lineNumber * pixelPerLine;
+      textArea.nativeElement.style.height = pixels + 'px';
+    } catch (e) {}
+  }
+}
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index fb2b7059a53a..ab725637b102 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -9057,6 +9057,8 @@ paths:
               properties:
                 bucket:
                   type: string
+                bucket_policy:
+                  type: string
                 daemon_name:
                   type: string
                 encryption_state:
@@ -9370,6 +9372,8 @@ paths:
               properties:
                 bucket_id:
                   type: string
+                bucket_policy:
+                  type: string
                 daemon_name:
                   type: string
                 encryption_state:
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index aec3b89b64c6..77e3bf5d7b16 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -911,6 +911,28 @@ def get_bucket_policy(self, bucket_name: str, request=None):
                     return None
             raise e
 
+    @RestClient.api_put('/{bucket_name}?policy')
+    def set_bucket_policy(self, bucket_name: str, policy: str, request=None):
+        """
+        Sets the bucket policy for a bucket.
+        :param bucket_name: The name of the bucket.
+        :type bucket_name: str
+        :param policy: The bucket policy.
+        :type policy: JSON Structured Document
+        :return: The bucket policy.
+        :rtype: Dict
+        """
+        # pylint: disable=unused-argument
+        try:
+            request = request(data=policy)
+        except RequestException as e:
+            if e.content:
+                content = json_str_to_object(e.content)
+                if content.get("Code") == "InvalidArgument":
+                    msg = "Invalid JSON document"
+                    raise DashboardException(msg=msg, component='rgw')
+            raise DashboardException(e)
+
     def perform_validations(self, retention_period_days, retention_period_years, mode):
         try:
             retention_period_days = int(retention_period_days) if retention_period_days else 0

From 70c00e8ba787d9e9106934cfee0e0afa606ce326 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 29 Jan 2024 11:23:54 -0500
Subject: [PATCH 1655/2492] cephadm: fix get_version for nvmeof

This needed to be using the container id it was
passed, instead of ctx.image which is likely to
be `None` when this is run.

Fixes: https://tracker.ceph.com/issues/64229

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadmlib/daemons/nvmeof.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
index 39488406bc8e..673761e8e6eb 100644
--- a/src/cephadm/cephadmlib/daemons/nvmeof.py
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -101,7 +101,7 @@ def get_version(ctx: CephadmContext, container_id: str) -> Optional[str]:
                 'inspect',
                 '--format',
                 '{{index .Config.Labels "io.ceph.version"}}',
-                ctx.image,
+                container_id,
             ],
         )
         version = None

From ae367352c7467f6ae3135fed154e89007e848347 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Fri, 19 Jan 2024 06:40:05 +0100
Subject: [PATCH 1656/2492] mgr/dashboard: remove bucket policy add a button on
 the bucket form to clear the bucket policy

Fixes: https://tracker.ceph.com/issues/64096
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../rgw/rgw-bucket-form/rgw-bucket-form.component.html    | 8 ++++++++
 .../ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index eef66a6da60b..77cd69934dc2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -430,6 +430,14 @@
                 <span class="invalid-feedback"
                       *ngIf="bucketForm.showError('bucket_policy', frm, 'invalidJson')"
                       i18n>Invalid json text</span>
+                <button type="button"
+                        id="clear-bucket-policy"
+                        class="btn btn-light my-3"
+                        (click)="clearBucketPolicy()"
+                        i18n>
+                  <i [ngClass]="[icons.destroy]"></i>
+                  Clear
+                </button>
                 <div class="btn-group float-end"
                      role="group"
                      aria-label="bucket-policy-helpers">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index f7015f609f68..e5267e357c1c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -385,6 +385,12 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     window.open(url, '_blank');
   }
 
+  clearBucketPolicy() {
+    this.bucketForm.get('bucket_policy').setValue('{}');
+    this.bucketForm.markAsDirty();
+    this.bucketForm.updateValueAndValidity();
+  }
+
   openConfigModal() {
     const modalRef = this.modalService.show(RgwConfigModalComponent, null, { size: 'lg' });
     modalRef.componentInstance.configForm

From 6cca0158228d1b64672a00d15f2902d6cd98bd4f Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Mon, 29 Jan 2024 17:40:13 -0600
Subject: [PATCH 1657/2492] .github/workflows: disable sync-labels in pr-triage

`sync-labels` is defined as "Whether or not to remove labels
when matching files are reverted or no longer changed by the PR".

We should disable this in case someone manually adds a label,
so the label doesn't get removed.

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 .github/workflows/pr-triage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml
index 24b1c97db708..2edb25f3ad89 100644
--- a/.github/workflows/pr-triage.yml
+++ b/.github/workflows/pr-triage.yml
@@ -11,7 +11,7 @@ jobs:
         # https://github.com/marketplace/actions/labeler?version=v4.0.2
         uses: actions/labeler@5c7539237e04b714afd8ad9b4aed733815b9fab4
         with:
-          sync-labels: ''
+          sync-labels: false
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Assign to Dashboard project
         # https://github.com/marketplace/actions/add-to-github-projects?version=v0.5.0

From b63e086fd56069c36fd792b63457e0894e34c13c Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Thu, 18 Jan 2024 13:28:46 +0530
Subject: [PATCH 1658/2492] mgr/dashboard: Export rgw multisite status via the
 API

Fixes: https://tracker.ceph.com/issues/64081

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   |  5 ++++
 .../app/shared/api/rgw-multisite.service.ts   |  5 ++--
 src/pybind/mgr/dashboard/openapi.yaml         | 24 +++++++++++++++++++
 3 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 661dc8afb6da..0a347a44d888 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -86,6 +86,7 @@ def status(self) -> dict:
 class RgwMultisiteStatus(RESTController):
     @Endpoint()
     @ReadPermission
+    @EndpointDoc("Get the multisite sync status")
     # pylint: disable=R0801
     def status(self):
         status = {'available': True, 'message': None}
@@ -110,6 +111,10 @@ def migrate(self, daemon_name=None, realm_name=None, zonegroup_name=None, zone_n
                                                          secret_key)
         return result
 
+
+@APIRouter('rgw/multisite', Scope.RGW)
+@APIDoc("RGW Multisite Management API", "RgwMultisite")
+class RgwMultisiteSyncStatus(RESTController):
     @RESTController.Collection(method='GET', path='/sync_status')
     @allow_empty_body
     # pylint: disable=W0102,W0613
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-multisite.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-multisite.service.ts
index d36c3a29e1a9..da789d29fc49 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-multisite.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-multisite.service.ts
@@ -7,7 +7,8 @@ import { RgwDaemonService } from './rgw-daemon.service';
   providedIn: 'root'
 })
 export class RgwMultisiteService {
-  private url = 'ui-api/rgw/multisite';
+  private uiUrl = 'ui-api/rgw/multisite';
+  private url = 'api/rgw/multisite';
 
   constructor(private http: HttpClient, public rgwDaemonService: RgwDaemonService) {}
 
@@ -22,7 +23,7 @@ export class RgwMultisiteService {
         access_key: zone.system_key.access_key,
         secret_key: zone.system_key.secret_key
       });
-      return this.http.put(`${this.url}/migrate`, null, { params: params });
+      return this.http.put(`${this.uiUrl}/migrate`, null, { params: params });
     });
   }
 
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index fb2b7059a53a..73ad27c36e67 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -9545,6 +9545,28 @@ paths:
       - jwt: []
       tags:
       - RgwDaemon
+  /api/rgw/multisite/sync_status:
+    get:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - RgwMultisite
   /api/rgw/realm:
     get:
       parameters: []
@@ -13194,6 +13216,8 @@ tags:
   name: RgwDaemon
 - description: Rgw Mirroring Perf Counters Management API
   name: RgwMirrorPerfCounter
+- description: RGW Multisite Management API
+  name: RgwMultisite
 - description: Rgw Perf Counters Management API
   name: RgwPerfCounter
 - description: '*No description available*'

From 7cd17931b1690029d182954b0382acc45b9ccb2b Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Thu, 25 Jan 2024 15:02:33 +0530
Subject: [PATCH 1659/2492] qa: remove error string checks and check w/ return
 value

I ran into this failure once #54972 was merged. The test is validating
the error string returned due to the failed mount. There aren't any
return value checks - which is a _more_ important check. Generic error
string checks will fail once a (error) string is changed (typo, etc..).

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_multifs_auth.py | 86 +++++++---------------------
 1 file changed, 21 insertions(+), 65 deletions(-)

diff --git a/qa/tasks/cephfs/test_multifs_auth.py b/qa/tasks/cephfs/test_multifs_auth.py
index e40ccb79af45..cbce40658682 100644
--- a/qa/tasks/cephfs/test_multifs_auth.py
+++ b/qa/tasks/cephfs/test_multifs_auth.py
@@ -226,54 +226,16 @@ def remount_with_new_client(self, keyring, cephfs_mntpt='/'):
 
 
 class TestClientsWithoutAuth(TestMultiFS):
+    # c.f., src/mount/mtab.c: EX_FAIL
+    RETVAL_KCLIENT = 32
+    # c.f., src/ceph_fuse.cc: (cpp EXIT_FAILURE). Normally the check for this
+    # case should be anything-except-0, but EXIT_FAILURE is 1 in most systems.
+    RETVAL_USER_SPACE_CLIENT = 1
 
     def setUp(self):
         super(TestClientsWithoutAuth, self).setUp()
-
-        # TODO: When MON and OSD caps for a Ceph FS are assigned to a
-        # client but MDS caps are not, mount.ceph prints "permission
-        # denied". But when MON caps are not assigned and MDS and OSD
-        # caps are, mount.ceph prints "no mds server or cluster laggy"
-        # instead of "permission denied".
-        #
-        # Before uncommenting the following line a fix would be required
-        # for latter case to change "no mds server is up or the cluster is
-        #  laggy" to "permission denied".
-        self.kernel_errmsgs = ('permission denied', 'no mds server is up or '
-                               'the cluster is laggy', 'no such file or '
-                               'directory',
-                               'input/output error')
-
-        # TODO: When MON and OSD caps are assigned for a Ceph FS to a
-        # client but MDS caps are not, ceph-fuse prints "operation not
-        # permitted". But when MON caps are not assigned and MDS and OSD
-        # caps are, ceph-fuse prints "no such file or directory" instead
-        # of "operation not permitted".
-        #
-        # Before uncommenting the following line a fix would be required
-        # for the latter case to change "no such file or directory" to
-        # "operation not permitted".
-        #self.assertIn('operation not permitted', retval[2].lower())
-        self.fuse_errmsgs = ('operation not permitted', 'no such file or '
-                             'directory')
-
-        if 'kernel' in str(type(self.mount_a)).lower():
-            self.errmsgs = self.kernel_errmsgs
-        elif 'fuse' in str(type(self.mount_a)).lower():
-            self.errmsgs = self.fuse_errmsgs
-        else:
-            raise RuntimeError('strange, the client was neither based on '
-                               'kernel nor FUSE.')
-
-    def check_that_mount_failed_for_right_reason(self, stderr):
-        stderr = stderr.lower()
-        for errmsg in self.errmsgs:
-            if errmsg in stderr:
-                break
-        else:
-            raise AssertionError('can\'t find expected set of words in the '
-                                 f'stderr\nself.errmsgs - {self.errmsgs}\n'
-                                 f'stderr - {stderr}')
+        self.retval = self.RETVAL_KCLIENT if 'kernel' in str(type(self.mount_a)).lower() \
+            else self.RETVAL_USER_SPACE_CLIENT
 
     def test_mount_all_caps_absent(self):
         # setup part...
@@ -281,16 +243,13 @@ def test_mount_all_caps_absent(self):
         keyring_path = self.mount_a.client_remote.mktemp(data=keyring)
 
         # mount the FS for which client has no auth...
-        retval = self.mount_a.remount(client_id=self.client_id,
-                                      client_keyring_path=keyring_path,
-                                      cephfs_name=self.fs2.name,
-                                      check_status=False)
-
-        # tests...
-        self.assertIsInstance(retval, tuple)
-        self.assertEqual(len(retval), 3)
-        self.assertIsInstance(retval[0], CommandFailedError)
-        self.check_that_mount_failed_for_right_reason(retval[2])
+        try:
+            self.mount_a.remount(client_id=self.client_id,
+                                 client_keyring_path=keyring_path,
+                                 cephfs_name=self.fs2.name,
+                                 check_status=False)
+        except CommandFailedError as e:
+            self.assertEqual(e.exitstatus, self.retval)
 
     def test_mount_mon_and_osd_caps_present_mds_caps_absent(self):
         # setup part...
@@ -303,13 +262,10 @@ def test_mount_mon_and_osd_caps_present_mds_caps_absent(self):
         keyring_path = self.mount_a.client_remote.mktemp(data=keyring)
 
         # mount the FS for which client has no auth...
-        retval = self.mount_a.remount(client_id=self.client_id,
-                                      client_keyring_path=keyring_path,
-                                      cephfs_name=self.fs2.name,
-                                      check_status=False)
-
-        # tests...
-        self.assertIsInstance(retval, tuple)
-        self.assertEqual(len(retval), 3)
-        self.assertIsInstance(retval[0], CommandFailedError)
-        self.check_that_mount_failed_for_right_reason(retval[2])
+        try:
+            self.mount_a.remount(client_id=self.client_id,
+                                 client_keyring_path=keyring_path,
+                                 cephfs_name=self.fs2.name,
+                                 check_status=False)
+        except CommandFailedError as e:
+            self.assertEqual(e.exitstatus, self.retval)

From 29610577eece04c028c412f112a66fafa8f70316 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 24 Jul 2023 00:33:47 -0400
Subject: [PATCH 1660/2492] mds: add mdlog trimming threshold and decay counter

Fixes: http://tracker.ceph.com/issues/61908
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/common/options/mds.yaml.in | 28 ++++++++++++++++++++++++
 src/mds/MDLog.cc               | 40 ++++++++++++++++++++++++++--------
 src/mds/MDLog.h                |  5 +++++
 3 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 2599b6532b5d..08d221835716 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -1597,3 +1597,31 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_log_trim_threshold
+  type: size
+  level: advanced
+  desc: MDS log trim threshold
+  long_desc: The threshold of the number of log segment that can be trimmed.
+  default: 128
+  min: 1
+  services:
+  - mds
+  see_also:
+  - mds_log_max_events
+  - mds_log_max_segments
+  flags:
+  - runtime
+- name: mds_log_trim_decay_rate
+  type: float
+  level: advanced
+  desc: MDS log trim decay rate
+  long_desc: The decay rate for trimming the MDS log. Increasing this value leads to the MDS spending less time in trimming the log.
+  default: 1.0
+  min: 0.01
+  services:
+  - mds
+  see_also:
+  - mds_log_max_events
+  - mds_log_max_segments
+  flags:
+  - runtime
diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 14b998850da5..c174428a1f92 100644
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -24,6 +24,7 @@
 #include "common/entity_name.h"
 #include "common/perf_counters.h"
 #include "common/Cond.h"
+#include "common/ceph_time.h"
 
 #include "events/ESubtreeMap.h"
 #include "events/ESegment.h"
@@ -45,7 +46,8 @@ MDLog::MDLog(MDSRank* m)
     mds(m),
     replay_thread(this),
     recovery_thread(this),
-    submit_thread(this)
+    submit_thread(this),
+    log_trim_counter(DecayCounter(g_conf().get_val<double>("mds_log_trim_decay_rate")))
 {
   debug_subtrees = g_conf().get_val<bool>("mds_debug_subtrees");
   event_large_threshold = g_conf().get_val<uint64_t>("mds_log_event_large_threshold");
@@ -68,7 +70,6 @@ MDLog::~MDLog()
   }
 }
 
-
 void MDLog::create_logger()
 {
   PerfCountersBuilder plb(g_ceph_context, "mds_log", l_mdl_first, l_mdl_last);
@@ -636,10 +637,6 @@ void MDLog::trim(int m)
     return;
   }
 
-  // hack: only trim for a few seconds at a time
-  utime_t stop = ceph_clock_now();
-  stop += 2.0;
-
   int op_prio = CEPH_MSG_PRIO_LOW +
 		(CEPH_MSG_PRIO_HIGH - CEPH_MSG_PRIO_LOW) *
 		expiring_segments.size() / max_segments;
@@ -654,16 +651,39 @@ void MDLog::trim(int m)
     ceph_assert(segments.size() >= pre_segments_size);
     max_expiring_segments = std::max<unsigned>(max_expiring_segments,segments.size() - pre_segments_size);
   }
-  
+
   map<uint64_t,LogSegment*>::iterator p = segments.begin();
+
+  auto trim_start = ceph::coarse_mono_clock::now();
+  std::optional<ceph::coarse_mono_time> trim_end;
+
+  auto log_trim_counter_start = log_trim_counter.get();
+  auto log_trim_threshold = g_conf().get_val<Option::size_t>("mds_log_trim_threshold");
+
   while (p != segments.end()) {
-    if (stop < ceph_clock_now())
+    // throttle - break out of trimmming if we've hit the threshold
+    if (log_trim_counter_start + new_expiring_segments >= log_trim_threshold) {
+      auto time_spent = std::chrono::duration<double>::zero();
+      if (trim_end) {
+	time_spent = std::chrono::duration<double>(*trim_end - trim_start);
+      }
+      dout(10) << __func__ << ": breaking out of trim loop - trimmed "
+	       << new_expiring_segments << " segment(s) in " << time_spent.count()
+	       << "s" << dendl;
       break;
+    }
 
     unsigned num_remaining_segments = (segments.size() - expired_segments.size() - expiring_segments.size());
+    dout(10) << __func__ << ": new_expiring_segments=" << new_expiring_segments
+	     << ", num_remaining_segments=" << num_remaining_segments
+	     << ", max_segments=" << max_segments << dendl;
+
     if ((num_remaining_segments <= max_segments) &&
-	(max_ev < 0 || (num_events - expiring_events - expired_events) <= (uint64_t)max_ev))
+	(max_ev < 0 || (num_events - expiring_events - expired_events) <= (uint64_t)max_ev)) {
+      dout(10) << __func__ << ": breaking out of trim loop - segments/events fell below ceiling"
+	       << " max_segments/max_ev" << dendl;
       break;
+    }
 
     // Do not trim too many segments at once for peak workload. If mds keeps creating N segments each tick,
     // the upper bound of 'num_remaining_segments - max_segments' is '2 * N'
@@ -699,6 +719,8 @@ void MDLog::trim(int m)
 
       uint64_t last_seq = ls->seq;
       try_expire(ls, op_prio);
+      log_trim_counter.hit();
+      trim_end = ceph::coarse_mono_clock::now();
 
       submit_mutex.lock();
       p = segments.lower_bound(last_seq + 1);
diff --git a/src/mds/MDLog.h b/src/mds/MDLog.h
index 5f8b78620ef1..c12a4b8d5338 100644
--- a/src/mds/MDLog.h
+++ b/src/mds/MDLog.h
@@ -46,6 +46,7 @@ enum {
 
 #include "MDSContext.h"
 #include "common/Cond.h"
+#include "common/DecayCounter.h"
 #include "common/Finisher.h"
 #include "common/Thread.h"
 
@@ -65,6 +66,7 @@ class ESubtreeMap;
 
 class MDLog {
 public:
+
   MDLog(MDSRank *m);
   ~MDLog();
 
@@ -301,5 +303,8 @@ class MDLog {
   std::set<LogSegment*> expired_segments;
   std::set<LogSegment*> expiring_segments;
   uint64_t events_since_last_major_segment = 0;
+
+  // log trimming decay counter
+  DecayCounter log_trim_counter;
 };
 #endif

From 3c3bc24ca3e58f1b3c225aa275cdd9c320a6dccc Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 31 Jul 2023 01:59:24 -0400
Subject: [PATCH 1661/2492] mds: remove a bunch of heuristics from
 MDLog::trim()

These were probbaly introduced to workaround some sort of
resource overusage by the MDS during trimming, but now it
looks like they are not really neeeded, especially if we
introduce a dedicated thread for log trimming.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDLog.cc | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index c174428a1f92..2ade813c22f2 100644
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -645,11 +645,8 @@ void MDLog::trim(int m)
 
   unsigned new_expiring_segments = 0;
 
-  unsigned max_expiring_segments = 0;
-  if (pre_segments_size > 0){
-    max_expiring_segments = max_segments/2;
+  if (pre_segments_size > 0) {
     ceph_assert(segments.size() >= pre_segments_size);
-    max_expiring_segments = std::max<unsigned>(max_expiring_segments,segments.size() - pre_segments_size);
   }
 
   map<uint64_t,LogSegment*>::iterator p = segments.begin();
@@ -685,15 +682,6 @@ void MDLog::trim(int m)
       break;
     }
 
-    // Do not trim too many segments at once for peak workload. If mds keeps creating N segments each tick,
-    // the upper bound of 'num_remaining_segments - max_segments' is '2 * N'
-    if (new_expiring_segments * 2 > num_remaining_segments)
-      break;
-
-    if (max_expiring_segments > 0 &&
-	expiring_segments.size() >= max_expiring_segments)
-      break;
-    
     // look at first segment
     LogSegment *ls = p->second;
     ceph_assert(ls);

From f9f3aef31739bf3fe31cb8b052c74aa921952947 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 24 Jul 2023 00:34:47 -0400
Subject: [PATCH 1662/2492] mds: allow runtime modification of mdlog trimming
 configuration

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDLog.cc   | 3 +++
 src/mds/MDSRank.cc | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 2ade813c22f2..544ddde8e05f 100644
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -1591,4 +1591,7 @@ void MDLog::handle_conf_change(const std::set<std::string>& changed, const MDSMa
   if (changed.count("mds_log_skip_unbounded_events")) {
     skip_unbounded_events = g_conf().get_val<bool>("mds_log_skip_unbounded_events");
   }
+  if (changed.count("mds_log_trim_decay_rate")){
+    log_trim_counter = DecayCounter(g_conf().get_val<double>("mds_log_trim_decay_rate"));
+  }
 }
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 02c4053bd008..9bbfc0433fa2 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3880,6 +3880,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_session_max_caps_throttle_ratio",
     "mds_symlink_recovery",
     "mds_session_metadata_threshold",
+    "mds_log_trim_threshold",
+    "mds_log_trim_decay_rate",
     NULL
   };
   return KEYS;

From 9c999db9c6c051739f3815fbe3a6d95f4757b1f7 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 8 Aug 2023 01:48:45 -0400
Subject: [PATCH 1663/2492] mds: drive mdlog trimming via a separate thread

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/common/options/mds.yaml.in | 10 ++++++++++
 src/mds/MDLog.cc               | 27 ++++++++++++++++++++++++---
 src/mds/MDLog.h                | 10 +++++++++-
 src/mds/MDSRank.cc             |  5 -----
 4 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 08d221835716..77d5fe253d55 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -1625,3 +1625,13 @@ options:
   - mds_log_max_segments
   flags:
   - runtime
+- name: mds_log_trim_upkeep_interval
+  type: millisecs
+  level: advanced
+  desc: MDS log trimming interval
+  long_desc: Interval in milliseconds to trim MDS logs.
+  default: 1000
+  services:
+  - mds
+  flags:
+  - runtime
diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 544ddde8e05f..5b1146ccd9d5 100644
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -58,6 +58,7 @@ MDLog::MDLog(MDSRank* m)
   max_events = g_conf().get_val<int64_t>("mds_log_max_events");
   skip_corrupt_events = g_conf().get_val<bool>("mds_log_skip_corrupt_events");
   skip_unbounded_events = g_conf().get_val<bool>("mds_log_skip_unbounded_events");
+  upkeep_thread = std::thread(&MDLog::log_trim_upkeep, this);
 }
 
 MDLog::~MDLog()
@@ -556,6 +557,13 @@ void MDLog::shutdown()
     }
   }
 
+  upkeep_log_trim_shutdown = true;
+  cond.notify_one();
+
+  mds->mds_lock.unlock();
+  upkeep_thread.join();
+  mds->mds_lock.lock();
+
   // Replay thread can be stuck inside e.g. Journaler::wait_for_readable,
   // so we need to shutdown the journaler first.
   if (journaler) {
@@ -606,11 +614,23 @@ void MDLog::try_to_commit_open_file_table(uint64_t last_seq)
   }
 }
 
-void MDLog::trim(int m)
+void MDLog::log_trim_upkeep(void) {
+  dout(10) << dendl;
+
+  std::unique_lock mds_lock(mds->mds_lock);
+  while (!upkeep_log_trim_shutdown.load()) {
+    if (mds->is_active() || mds->is_stopping()) {
+      trim();
+    }
+
+    cond.wait_for(mds_lock, g_conf().get_val<std::chrono::milliseconds>("mds_log_trim_upkeep_interval"));
+  }
+  dout(10) << __func__ << ": finished" << dendl;
+}
+
+void MDLog::trim()
 {
   int max_ev = max_events;
-  if (m >= 0)
-    max_ev = m;
 
   if (mds->mdcache->is_readonly()) {
     dout(10) << "trim, ignoring read-only FS" <<  dendl;
@@ -794,6 +814,7 @@ int MDLog::trim_all()
 
 void MDLog::try_expire(LogSegment *ls, int op_prio)
 {
+  ceph_assert(ceph_mutex_is_locked(mds->mds_lock));
   MDSGatherBuilder gather_bld(g_ceph_context);
   ls->try_to_expire(mds, gather_bld, op_prio);
 
diff --git a/src/mds/MDLog.h b/src/mds/MDLog.h
index c12a4b8d5338..33eba835c402 100644
--- a/src/mds/MDLog.h
+++ b/src/mds/MDLog.h
@@ -147,7 +147,6 @@ class MDLog {
   }
 
   void trim_expired_segments();
-  void trim(int max=-1);
   int trim_all();
 
   void create(MDSContext *onfinish);  // fresh, empty log! 
@@ -289,6 +288,9 @@ class MDLog {
   void _trim_expired_segments();
   void write_head(MDSContext *onfinish);
 
+  void trim();
+  void log_trim_upkeep(void);
+
   bool debug_subtrees;
   std::atomic_uint64_t event_large_threshold; // accessed by submit thread
   uint64_t events_per_segment;
@@ -306,5 +308,11 @@ class MDLog {
 
   // log trimming decay counter
   DecayCounter log_trim_counter;
+
+  // log trimming upkeeper thread
+  std::thread upkeep_thread;
+  // guarded by mds_lock
+  std::condition_variable_any cond;
+  std::atomic<bool> upkeep_log_trim_shutdown{false};
 };
 #endif
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 9bbfc0433fa2..6f8439470a65 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -743,10 +743,6 @@ void MDSRankDispatcher::tick()
   // update average session uptime
   sessionmap.update_average_session_age();
 
-  if (is_active() || is_stopping()) {
-    mdlog->trim();  // NOT during recovery!
-  }
-
   // ...
   if (is_clientreplay() || is_active() || is_stopping()) {
     server->clear_laggy_clients();
@@ -789,7 +785,6 @@ void MDSRankDispatcher::tick()
 
   // shut down?
   if (is_stopping()) {
-    mdlog->trim();
     if (mdcache->shutdown_pass()) {
       uint64_t pq_progress = 0 ;
       uint64_t pq_total = 0;

From e579ac55b58a724de147bd202150c4d2f97321af Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 26 Sep 2023 08:22:03 -0400
Subject: [PATCH 1664/2492] PendingReleaseNotes: add note about new mdlog
 trimming configurations

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 PendingReleaseNotes | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index b374a1743f87..dfc6294b36a2 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -117,6 +117,10 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   manager module. Users may choose between two new modes: ``upmap-read``, which
   offers upmap and read optimization simultaneously, or ``read``, which may be used
   to only optimize reads. For more detailed information see https://docs.ceph.com/en/latest/rados/operations/read-balancer/#online-optimization.
+* CephFS: MDS log trimming is now driven by a separate thread which tries to
+  trim the log every second (`mds_log_trim_upkeep_interval` config). Also,
+  a couple of configs govern how much time the MDS spends in trimming its
+  logs. These configs are `mds_log_trim_threshold` and `mds_log_trim_decay_rate`.
 
 >=18.0.0
 

From e699963c9bfa10ec65b8a4fa016377c4ce85c0bc Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 01:42:09 -0600
Subject: [PATCH 1665/2492] osd/scrub: define the scrub_reserver to track
 remote reservations

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc |  5 +++++
 src/osd/OSD.h  | 10 ++++++++++
 2 files changed, 15 insertions(+)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 9e6b3fd9d929..61af98924c74 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -273,6 +273,8 @@ OSDService::OSDService(OSD *osd, ceph::async::io_context_pool& poolctx) :
 		  cct->_conf->osd_min_recovery_priority),
   snap_reserver(cct, &reserver_finisher,
 		cct->_conf->osd_max_trimming_pgs),
+  scrub_reserver(cct, &reserver_finisher,
+		cct->_conf->osd_max_scrubs),
   recovery_ops_active(0),
   recovery_ops_reserved(0),
   recovery_paused(false),
@@ -9889,6 +9891,9 @@ void OSD::handle_conf_change(const ConfigProxy& conf,
   if (changed.count("osd_max_trimming_pgs")) {
     service.snap_reserver.set_max(cct->_conf->osd_max_trimming_pgs);
   }
+  if (changed.count("osd_max_scrubs")) {
+    service.scrub_reserver.set_max(cct->_conf->osd_max_scrubs);
+  }
   if (changed.count("osd_op_complaint_time") ||
       changed.count("osd_op_log_threshold")) {
     op_tracker.set_complaint_and_threshold(cct->_conf->osd_op_complaint_time,
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 5f561e643182..8c03e0192033 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -251,6 +251,14 @@ class OSDService : public Scrub::ScrubSchedListener {
    */
   std::optional<PGLockWrapper> get_locked_pg(spg_t pgid) final;
 
+  /**
+   * the entity that counts the number of active replica scrub
+   * operations, and grant scrub reservation requests asynchronously.
+   */
+  AsyncReserver<spg_t, Finisher>& get_scrub_reserver() {
+    return scrub_reserver;
+  }
+
  private:
   // -- agent shared state --
   ceph::mutex agent_lock = ceph::make_mutex("OSDService::agent_lock");
@@ -494,6 +502,8 @@ class OSDService : public Scrub::ScrubSchedListener {
   void send_pg_created();
 
   AsyncReserver<spg_t, Finisher> snap_reserver;
+  /// keeping track of replicas being reserved for scrubbing
+  AsyncReserver<spg_t, Finisher> scrub_reserver;
   void queue_recovery_context(PG *pg,
                               GenContext<ThreadPool::TPHandle&> *c,
                               uint64_t cost,

From a4f10d329b6f457ad107e4a88bdc32a0fb7d9919 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 01:25:38 -0600
Subject: [PATCH 1666/2492] osd/scrub: add a general data-carrying event type
 to the FSM

Based on Sam's Crimson FSM's 'value-event'.

Co-authored-by: Sam Just <sjust@redhat.com>
Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_machine.h | 33 ++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 9b2a800d1882..6eadb109cd54 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -102,6 +102,39 @@ OP_EV(ReplicaReserveReq);
 /// explicit release request from the Primary
 OP_EV(ReplicaRelease);
 
+template <typename T, has_formatter V>
+struct value_event_t : sc::event<T> {
+  const V value;
+
+  template <typename... Args>
+  value_event_t(Args&&... args) : value(std::forward<Args>(args)...)
+  {
+    on_event_creation(T::event_name);
+  }
+
+  value_event_t(const value_event_t&) = default;
+  value_event_t(value_event_t&&) = default;
+  value_event_t& operator=(const value_event_t&) = default;
+  value_event_t& operator=(value_event_t&&) = default;
+  ~value_event_t() { on_event_discard(T::event_name); }
+
+  template <typename FormatContext>
+  auto fmt_print_ctx(FormatContext& ctx) const
+  {
+    return fmt::format_to(ctx.out(), "{}({})", T::event_name, value);
+  }
+};
+
+#define VALUE_EVENT(T, V)                                          \
+  struct T : value_event_t<T, V> {                                 \
+    static constexpr const char* event_name = #T;                  \
+    template <typename... Args>                                    \
+    T(Args&&... args) : value_event_t(std::forward<Args>(args)...) \
+    {                                                              \
+    }                                                              \
+  };
+
+
 /// the async-reserver granted our reservation request
 OP_EV(ReserverGranted);
 

From a70f96229df508385c339d62a5b6f46b1bf9dd07 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 30 Jan 2024 02:55:13 -0600
Subject: [PATCH 1667/2492] common: allow AsyncReserver requests with no
 callbacks

As no callbacks are needed for request_reservation_or_fail(),
the synchronous request API.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/AsyncReserver.h | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/common/AsyncReserver.h b/src/common/AsyncReserver.h
index 32b558b308ba..b98e54ef767c 100644
--- a/src/common/AsyncReserver.h
+++ b/src/common/AsyncReserver.h
@@ -113,8 +113,10 @@ class AsyncReserver {
       if (it->second.empty()) {
 	queues.erase(it);
       }
-      f->queue(p.grant);
-      p.grant = nullptr;
+      if (p.grant) {
+	f->queue(p.grant);
+	p.grant = nullptr;
+      }
       in_progress[p.item] = p;
       if (p.preempt) {
 	preempt_by_prio.insert(std::make_pair(p.prio, p.item));
@@ -274,8 +276,7 @@ class AsyncReserver {
    * active reservations.
    */
   bool request_reservation_or_fail(
-      T item,		     ///< [in] reservation key
-      Context *on_reserved   ///< [in] callback to be called on reservation
+      T item		     ///< [in] reservation key
   )
   {
     std::lock_guard l(lock);
@@ -288,7 +289,7 @@ class AsyncReserver {
     }
 
     const unsigned prio = UINT_MAX;
-    Reservation r(item, prio, on_reserved, nullptr);
+    Reservation r(item, prio, nullptr, nullptr);
     queues[prio].push_back(r);
     queue_pointers.insert(std::make_pair(
 	item, std::make_pair(prio, --(queues[prio]).end())));

From 92bbd73c7cbf0fa4acf88fb5becac91271e8ec07 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 05:02:31 -0600
Subject: [PATCH 1668/2492] osd/scrub: remove remote reservation code from
 ScrubResources

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc       | 15 --------
 src/osd/scrubber/osd_scrub.h        |  3 --
 src/osd/scrubber/scrub_resources.cc | 54 +++--------------------------
 src/osd/scrubber/scrub_resources.h  | 44 ++---------------------
 4 files changed, 6 insertions(+), 110 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index 2ff70dee56d1..b4e1ec8cef23 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -468,21 +468,6 @@ void OsdScrub::dec_scrubs_local()
   m_resource_bookkeeper.dec_scrubs_local();
 }
 
-bool OsdScrub::inc_scrubs_remote(pg_t pgid)
-{
-  return m_resource_bookkeeper.inc_scrubs_remote(pgid);
-}
-
-void OsdScrub::enqueue_remote_reservation(pg_t pgid)
-{
-  m_resource_bookkeeper.enqueue_remote_reservation(pgid);
-}
-
-void OsdScrub::dec_scrubs_remote(pg_t pgid)
-{
-  m_resource_bookkeeper.dec_scrubs_remote(pgid);
-}
-
 void OsdScrub::mark_pg_scrub_blocked(spg_t blocked_pg)
 {
   m_queue.mark_pg_scrub_blocked(blocked_pg);
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 2701a762f798..9aca86ad798f 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -69,9 +69,6 @@ class OsdScrub {
   std::unique_ptr<Scrub::LocalResourceWrapper> inc_scrubs_local(
       bool is_high_priority);
   void dec_scrubs_local();
-  bool inc_scrubs_remote(pg_t pgid);
-  void enqueue_remote_reservation(pg_t pgid);
-  void dec_scrubs_remote(pg_t pgid);
 
   // counting the number of PGs stuck while scrubbing, waiting for objects
   void mark_pg_scrub_blocked(spg_t blocked_pg);
diff --git a/src/osd/scrubber/scrub_resources.cc b/src/osd/scrubber/scrub_resources.cc
index a69c6f06aca9..c47048c2a020 100644
--- a/src/osd/scrubber/scrub_resources.cc
+++ b/src/osd/scrubber/scrub_resources.cc
@@ -40,8 +40,8 @@ std::unique_ptr<LocalResourceWrapper> ScrubResources::inc_scrubs_local(
   if (is_high_priority || can_inc_local_scrubs_unlocked()) {
     ++scrubs_local;
     log_upwards(fmt::format(
-	"{}: {} -> {} (max {}, remote {})", __func__, (scrubs_local - 1),
-	scrubs_local, conf->osd_max_scrubs, granted_reservations.size()));
+	"{}: {} -> {} (max {})", __func__, (scrubs_local - 1), scrubs_local,
+	conf->osd_max_scrubs));
     return std::make_unique<LocalResourceWrapper>(*this);
   }
   return nullptr;
@@ -62,63 +62,17 @@ void ScrubResources::dec_scrubs_local()
 {
   std::lock_guard lck{resource_lock};
   log_upwards(fmt::format(
-      "{}:  {} -> {} (max {}, remote {})",
-      __func__, scrubs_local, (scrubs_local - 1), conf->osd_max_scrubs,
-      granted_reservations.size()));
+      "{}:  {} -> {} (max {})", __func__, scrubs_local, (scrubs_local - 1),
+      conf->osd_max_scrubs));
   --scrubs_local;
   ceph_assert(scrubs_local >= 0);
 }
 
-// ------------------------- scrubbing on this OSD as replicas ----------------
-
-bool ScrubResources::inc_scrubs_remote(pg_t pgid)
-{
-  std::lock_guard lck{resource_lock};
-
-  // if this PG is already reserved - it's probably a benign bug.
-  // report it, but do not fail the reservation.
-  if (granted_reservations.contains(pgid)) {
-    log_upwards(fmt::format("{}: pg[{}] already reserved", __func__, pgid));
-    return true;
-  }
-
-  auto pre_op_cnt = granted_reservations.size();
-  if (std::cmp_less(pre_op_cnt, conf->osd_max_scrubs)) {
-    granted_reservations.insert(pgid);
-    log_upwards(fmt::format(
-	"{}: pg[{}] reserved. Remote scrubs count changed from {} -> {} (max "
-	"{}, local {})",
-	__func__, pgid, pre_op_cnt, granted_reservations.size(),
-	conf->osd_max_scrubs, scrubs_local));
-    return true;
-  }
-
-  log_upwards(fmt::format(
-      "{}: pg[{}] failed. Too many concurrent replica scrubs ({} >= max ({}))",
-      __func__, pgid, pre_op_cnt, conf->osd_max_scrubs));
-  return false;
-}
-
-void ScrubResources::dec_scrubs_remote(pg_t pgid)
-{
-  std::lock_guard lck{resource_lock};
-  // we might not have this PG in the set (e.g. if we are concluding a
-  // high priority scrub, one that does not require reservations)
-  auto cnt = granted_reservations.erase(pgid);
-  if (cnt) {
-    log_upwards(fmt::format(
-	"{}: remote reservation for {} removed -> {} (max {}, local {})",
-	__func__, pgid, granted_reservations.size(), conf->osd_max_scrubs,
-	scrubs_local));
-  }
-}
 
 void ScrubResources::dump_scrub_reservations(ceph::Formatter* f) const
 {
   std::lock_guard lck{resource_lock};
   f->dump_int("scrubs_local", scrubs_local);
-  f->dump_int("granted_reservations", granted_reservations.size());
-  f->dump_string("PGs being served", fmt::format("{}", granted_reservations));
   f->dump_int("osd_max_scrubs", conf->osd_max_scrubs);
 }
 
diff --git a/src/osd/scrubber/scrub_resources.h b/src/osd/scrubber/scrub_resources.h
index afb154e3e162..0aaa2f4bd0f7 100644
--- a/src/osd/scrubber/scrub_resources.h
+++ b/src/osd/scrubber/scrub_resources.h
@@ -10,33 +10,6 @@
 #include "common/Formatter.h"
 #include "osd/osd_types.h"
 
-/*
- * AsyncReserver for scrub 'remote' reservations
- * -----------------------------------------------
- *
- * On the replica side, all reservations are treated as having the same priority.
- * Note that 'high priority' scrubs, e.g. user-initiated scrubs, are not required
- * to perform any reservations, and are never handled by the replicas' OSD.
- *
- * A queued scrub reservation request is cancelled by any of the following events:
- *
- * - a new interval: in this case, we do not expect to see a cancellation request
- *   from the primary, and we can simply remove the request from the queue;
- *
- * - a cancellation request from the primary: probably a result of timing out on
- *   the reservation process. Here, we can simply remove the request from the queue.
- *
- * - a new reservation request for the same PG: which means we had missed the
- *   previous cancellation request. We cancel the previous request, and replace
- *   it with the new one. We would also issue an error log message.
- *
- * Primary/Replica with differing versions:
- *
- * The updated version of MOSDScrubReserve contains a new 'OK to queue' field.
- * For legacy Primary OSDs, this field is decoded as 'false', and the replica
- * responds immediately, with grant/rejection.
-*/
-
 namespace Scrub {
 
 /**
@@ -50,8 +23,8 @@ class LocalResourceWrapper;
 /**
  * The number of concurrent scrub operations performed on an OSD is limited
  * by a configuration parameter. The 'ScrubResources' class is responsible for
- * maintaining a count of the number of scrubs currently performed, both
- * acting as primary and acting as a replica, and for enforcing the limit.
+ * maintaining a count of the number of scrubs currently performed by primary
+ * PGs on this OSD, and for enforcing the limit.
  */
 class ScrubResources {
   friend class LocalResourceWrapper;
@@ -65,10 +38,6 @@ class ScrubResources {
    */
   int scrubs_local{0};
 
-  /// the set of PGs that have active scrub reservations as replicas
-  /// \todo come C++23 - consider std::flat_set<pg_t>
-  std::set<pg_t> granted_reservations;
-
   mutable ceph::mutex resource_lock =
       ceph::make_mutex("ScrubQueue::resource_lock");
 
@@ -97,15 +66,6 @@ class ScrubResources {
   /// decrements the number of scrubs acting as a Primary
   void dec_scrubs_local();
 
-  /// increments the number of scrubs acting as a Replica
-  bool inc_scrubs_remote(pg_t pgid);
-
-  /// queue a request with the scrub reserver
-  void enqueue_remote_reservation(pg_t pgid) {}
-
-  /// decrements the number of scrubs acting as a Replica
-  void dec_scrubs_remote(pg_t pgid);
-
   void dump_scrub_reservations(ceph::Formatter* f) const;
 };
 

From 3bb35d5729f4c723083fccfe55a3e3f81b070cff Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 27 Nov 2023 16:58:24 +0530
Subject: [PATCH 1669/2492] src/test: test async I/O with a file created with
 O_PATH

Fixes: https://tracker.ceph.com/issues/63632
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 64 ++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index 787e022d4922..c50d299ec848 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -208,3 +208,67 @@ TEST_F(TestClient, LlreadvLlwritevNullContext) {
   client->ll_release(fh);
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
 }
+
+TEST_F(TestClient, LlreadvLlwritevOPathFileHandle) {
+  /* Test that async I/O fails if the file has been created with O_PATH flag;
+  EBADF is returned and the callback is finished*/
+
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevopathfilehandlefile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+          O_RDWR | O_CREAT | O_PATH,
+          &file, &fh, &stx, 0, 0, myperm));
+
+  char out0[] = "hello ";
+  char out1[] = "world\n";  
+  struct iovec iov_out[2] = {
+    {out0, sizeof(out0)},
+    {out1, sizeof(out1)}
+  };
+
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+    {in0, sizeof(in0)},
+    {in1, sizeof(in1)}
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-opath-filehandle"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-opath-filehandle"));
+
+  int64_t rc;
+  bufferlist bl;
+
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, writefinish.get(),
+                                 nullptr);
+  ASSERT_EQ(rc, 0);
+  rc = writefinish->wait();
+  ASSERT_EQ(rc, -CEPHFS_EBADF);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, readfinish.get(),
+                                 &bl);
+  ASSERT_EQ(rc, 0);
+  rc = readfinish->wait();
+  ASSERT_EQ(rc, -CEPHFS_EBADF);
+  ASSERT_EQ(bl.length(), 0);
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+}
\ No newline at end of file

From 86050710f7242a3d4dfb4e57268a970a384f99fc Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 6 Dec 2023 15:18:55 +0530
Subject: [PATCH 1670/2492] src/test: test async I/O with read only file

Fixes: https://tracker.ceph.com/issues/63648
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 65 +++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index c50d299ec848..4b689dd86879 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -271,4 +271,67 @@ TEST_F(TestClient, LlreadvLlwritevOPathFileHandle) {
 
   client->ll_release(fh);
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
-}
\ No newline at end of file
+}
+
+TEST_F(TestClient, LlreadvLlwritevReadOnlyFile) {
+  /* Test async I/O with read only file*/
+
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevreadonlyfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+          O_RDONLY | O_CREAT | O_TRUNC,
+          &file, &fh, &stx, 0, 0, myperm));
+
+  char out_buf_0[] = "hello ";
+  char out_buf_1[] = "world\n";
+  struct iovec iov_out[2] = {
+    {out_buf_0, sizeof(out_buf_0)},
+    {out_buf_1, sizeof(out_buf_1)},
+  };
+
+  char in_buf_0[sizeof(out_buf_0)];
+  char in_buf_1[sizeof(out_buf_1)];
+  struct iovec iov_in[2] = {
+    {in_buf_0, sizeof(in_buf_0)},
+    {in_buf_1, sizeof(in_buf_1)},
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  int64_t rc;
+  bufferlist bl;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-read-only"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-read-only"));
+
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, writefinish.get(),
+                                 nullptr);
+  ASSERT_EQ(rc, 0);
+  rc = writefinish->wait();
+  ASSERT_EQ(rc, -CEPHFS_EBADF);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, readfinish.get(),
+                                 &bl);
+  ASSERT_EQ(rc, 0);
+  rc = readfinish->wait();
+  ASSERT_EQ(rc, 0);
+  ASSERT_EQ(bl.length(), 0);
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+}

From 6c734e33db709f0fd710b0112a90cd53c18caab9 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 6 Dec 2023 15:21:59 +0530
Subject: [PATCH 1671/2492] src/test: test async I/O if the client is not
 mounted

Fixes: https://tracker.ceph.com/issues/63629
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 60 ++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index 4b689dd86879..edb5c547396f 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -335,3 +335,63 @@ TEST_F(TestClient, LlreadvLlwritevReadOnlyFile) {
   client->ll_release(fh);
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
 }
+
+TEST_F(TestClient, LlreadvLlwritevIOClientNotMounted) {
+  /* Test that performing async I/O if the client is not mounted returns
+  ENOTCONN; callback is finished and thus the caller is not stalled .*/
+
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevioclientnotmountedfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+				  O_RDWR | O_CREAT | O_TRUNC,
+				  &file, &fh, &stx, 0, 0, myperm));
+
+  char out0[] = "hello ";
+  char out1[] = "world\n";
+  struct iovec iov_out[2] = {
+	  {out0, sizeof(out0)},
+	  {out1, sizeof(out1)},
+  };
+
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+	  {in0, sizeof(in0)},
+	  {in1, sizeof(in1)},
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-io-client-not-mounted"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-io-client-not-mounted"));
+
+  int64_t rc;
+  bufferlist bl;
+
+  ASSERT_EQ(client->ll_release(fh), 0);
+  client->unmount();
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, writefinish.get(), nullptr);
+  ASSERT_EQ(rc, 0);
+  rc = writefinish->wait();
+  ASSERT_EQ(rc, -CEPHFS_ENOTCONN);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, readfinish.get(), &bl);
+  ASSERT_EQ(rc, 0);
+  rc = readfinish->wait();
+  ASSERT_EQ(rc, -CEPHFS_ENOTCONN);
+}

From 45e0f890a2ccb91c38c04438a840bfde19a56d3c Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 6 Dec 2023 15:33:07 +0530
Subject: [PATCH 1672/2492] src/test: test async I/O with negative iov
 structures count

Fixes: https://tracker.ceph.com/issues/63619
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 62 ++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index edb5c547396f..1ea7c61bf89a 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -395,3 +395,65 @@ TEST_F(TestClient, LlreadvLlwritevIOClientNotMounted) {
   rc = readfinish->wait();
   ASSERT_EQ(rc, -CEPHFS_ENOTCONN);
 }
+
+TEST_F(TestClient, LlreadvLlwritevNegativeIOVCount) {
+  /* Test function handles negative iovcnt and returns EINVAL */
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevnegativeiovcountfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+				  O_RDWR | O_CREAT | O_TRUNC,
+				  &file, &fh, &stx, 0, 0, myperm));
+
+  char out0[] = "hello ";
+  char out1[] = "world\n";  
+  struct iovec iov_out[2] = {
+	  {out0, sizeof(out0)},
+	  {out1, sizeof(out1)}
+  };
+
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+	  {in0, sizeof(in0)},
+	  {in1, sizeof(in1)}
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-negative-iovcnt"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-negative-iovcnt"));
+
+  int64_t rc;
+  bufferlist bl;
+
+  rc = client->ll_preadv_pwritev(fh, iov_out, -2, 0, true, writefinish.get(),
+                                 nullptr);
+  ASSERT_EQ(rc, 0);
+  ssize_t bytes_written = writefinish->wait();
+  ASSERT_EQ(bytes_written, -CEPHFS_EINVAL);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, -2, 0, false, readfinish.get(),
+                                 &bl);
+  ASSERT_EQ(rc, 0);
+  ssize_t bytes_read = readfinish->wait();
+  ASSERT_EQ(bytes_read, -CEPHFS_EINVAL);
+  ASSERT_EQ(bl.length(), 0);
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+}

From ab8462443fc2831bc24df4822749bd289be6aacf Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 7 Dec 2023 23:17:20 +0530
Subject: [PATCH 1673/2492] src/test: test zero bytes async i/o

Fixes: https://tracker.ceph.com/issues/63734
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 70 ++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index 1ea7c61bf89a..427bf0723ab7 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -457,3 +457,73 @@ TEST_F(TestClient, LlreadvLlwritevNegativeIOVCount) {
   client->ll_release(fh);
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
 }
+
+TEST_F(TestClient, LlreadvLlwritevZeroBytes) {
+  /* Test async i/o with empty input/output buffers*/
+
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevzerobytesfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+				  O_RDWR | O_CREAT | O_TRUNC,
+				  &file, &fh, &stx, 0, 0, myperm));
+
+  char out_empty_buf_0[0];
+  char out_empty_buf_1[0];
+  struct iovec iov_out[2] = {
+    {out_empty_buf_0, sizeof(out_empty_buf_0)},
+    {out_empty_buf_1, sizeof(out_empty_buf_1)}
+  };
+
+  char in_empty_buf_0[sizeof(out_empty_buf_0)];
+  char in_empty_buf_1[sizeof(out_empty_buf_1)];
+  struct iovec iov_in[2] = {
+    {in_empty_buf_0, sizeof(in_empty_buf_0)},
+    {in_empty_buf_1, sizeof(in_empty_buf_1)}
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-zero-bytes"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-zero-bytes"));
+
+  int64_t rc;
+  bufferlist bl;
+
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, writefinish.get(),
+                                 nullptr);
+  ASSERT_EQ(rc, 0);
+  ssize_t bytes_written = writefinish->wait();
+  ASSERT_EQ(bytes_written, -CEPHFS_EINVAL);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, readfinish.get(),
+                                 &bl);
+  ASSERT_EQ(rc, 0);
+  ssize_t bytes_read = readfinish->wait();
+  ASSERT_EQ(bytes_read, 0);
+
+  copy_bufferlist_to_iovec(iov_in, 2, &bl, bytes_read);
+  ASSERT_EQ(0, strncmp((const char*)iov_in[0].iov_base,
+                       (const char*)iov_out[0].iov_base,
+                       iov_out[0].iov_len));
+  ASSERT_EQ(0, strncmp((const char*)iov_in[1].iov_base,
+                       (const char*)iov_out[1].iov_base, 
+                       iov_out[1].iov_len));
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+}
\ No newline at end of file

From b13fab272d0c089b10544244982037898f612a4c Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Fri, 24 Nov 2023 02:13:52 +0530
Subject: [PATCH 1674/2492] client: check for negative value of iovcnt

this is intentionally kept in Client::_preadv_pwritev_locked since all its
callers will be providing iovec structures and thus handling this case in
all the callers will lead to boilerplate code.

Fixes: https://tracker.ceph.com/issues/63619
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 13 ++++++-------
 src/client/Client.h  |  4 ++--
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8966bf60adb0..d8b6c32090e4 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -10655,8 +10655,6 @@ int Client::read(int fd, char *buf, loff_t size, loff_t offset)
 
 int Client::preadv(int fd, const struct iovec *iov, int iovcnt, loff_t offset)
 {
-  if (iovcnt < 0)
-    return -CEPHFS_EINVAL;
   return _preadv_pwritev(fd, iov, iovcnt, offset, false);
 }
 
@@ -11219,13 +11217,11 @@ int Client::write(int fd, const char *buf, loff_t size, loff_t offset)
 
 int Client::pwritev(int fd, const struct iovec *iov, int iovcnt, int64_t offset)
 {
-  if (iovcnt < 0)
-    return -CEPHFS_EINVAL;
   return _preadv_pwritev(fd, iov, iovcnt, offset, true);
 }
 
 int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
-                                       unsigned iovcnt, int64_t offset,
+                                       int iovcnt, int64_t offset,
                                        bool write, bool clamp_to_int,
                                        Context *onfinish, bufferlist *blp,
                                        bool do_fsync, bool syncdataonly)
@@ -11236,8 +11232,11 @@ int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
     if (fh->flags & O_PATH)
         return -CEPHFS_EBADF;
 #endif
+    if(iovcnt < 0) {
+      return -CEPHFS_EINVAL;
+    }
     loff_t totallen = 0;
-    for (unsigned i = 0; i < iovcnt; i++) {
+    for (int i = 0; i < iovcnt; i++) {
         totallen += iov[i].iov_len;
     }
 
@@ -11275,7 +11274,7 @@ int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
     }
 }
 
-int Client::_preadv_pwritev(int fd, const struct iovec *iov, unsigned iovcnt,
+int Client::_preadv_pwritev(int fd, const struct iovec *iov, int iovcnt,
                             int64_t offset, bool write, Context *onfinish,
                             bufferlist *blp)
 {
diff --git a/src/client/Client.h b/src/client/Client.h
index 9c1303384be8..c0f4af964315 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -1678,12 +1678,12 @@ class Client : public Dispatcher, public md_config_obs_t {
           const struct iovec *iov, int iovcnt, Context *onfinish = nullptr,
           bool do_fsync = false, bool syncdataonly = false);
   int64_t _preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
-                                 unsigned iovcnt, int64_t offset,
+                                 int iovcnt, int64_t offset,
                                  bool write, bool clamp_to_int,
                                  Context *onfinish = nullptr,
                                  bufferlist *blp = nullptr,
                                  bool do_fsync = false, bool syncdataonly = false);
-  int _preadv_pwritev(int fd, const struct iovec *iov, unsigned iovcnt,
+  int _preadv_pwritev(int fd, const struct iovec *iov, int iovcnt,
                       int64_t offset, bool write, Context *onfinish = nullptr,
                       bufferlist *blp = nullptr);
   int _flush(Fh *fh);

From a67d081148ba0680539fbf76ae13c6484bb730de Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 7 Dec 2023 23:14:01 +0530
Subject: [PATCH 1675/2492] client: do not accept zero byte write request

Fixes: https://tracker.ceph.com/issues/63734
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index d8b6c32090e4..58a289d1a23a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -11441,6 +11441,10 @@ int64_t Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf,
   CWF_iofinish *cwf_iofinish = NULL;
   C_SaferCond *cond_iofinish = NULL;
 
+  if (size < 1) { // zero bytes write is not supported by osd
+    return -CEPHFS_EINVAL;
+  }
+
   if ( (uint64_t)(offset+size) > mdsmap->get_max_filesize() && //exceeds config
        (uint64_t)(offset+size) > in->size ) { //exceeds filesize 
       return -CEPHFS_EFBIG;              

From bbde5258dec2562b710af31f8a842bac93b5d4cc Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Fri, 24 Nov 2023 19:23:25 +0530
Subject: [PATCH 1676/2492] client: make sure the callback is finished when
 returning ENOTCONN

when the client is not mounted

Fixes: https://tracker.ceph.com/issues/63629
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 58a289d1a23a..8682423ea1e1 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -15948,12 +15948,21 @@ int64_t Client::ll_preadv_pwritev(struct Fh *fh, const struct iovec *iov,
                                   bool do_fsync, bool syncdataonly)
 {
     RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
-    if (!mref_reader.is_state_satisfied())
-      return -CEPHFS_ENOTCONN;
+    if (!mref_reader.is_state_satisfied()) {
+      int64_t rc = -CEPHFS_ENOTCONN;
+      if (onfinish != nullptr) {
+        onfinish->complete(rc);
+        /* async call should always return zero to caller and allow the
+        caller to wait on callback for the actual errno. */
+        rc = 0;
+      }
+      return rc;
+    }
 
     std::scoped_lock cl(client_lock);
     return _preadv_pwritev_locked(fh, iov, iovcnt, offset, write, true,
     				  onfinish, bl, do_fsync, syncdataonly);
+
 }
 
 int Client::ll_flush(Fh *fh)

From 9d4ae970bc3c95ad9a097f2b3c99dae58354800f Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 5 Dec 2023 17:10:56 +0530
Subject: [PATCH 1677/2492] client: handle callback completion if the async I/O
 failed

and remove the context completion code from Client::_preadv_pwritev_locked since
this case will indeed be handled in the Client::ll_preadv_pwritev along with all
other cases

Fixes: https://tracker.ceph.com/issues/63734
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 37 ++++++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8682423ea1e1..6ee5e742b9b2 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -11259,11 +11259,6 @@ int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
                           onfinish);
         ldout(cct, 3) << "preadv(" << fh << ", " <<  offset << ") = " << r << dendl;
         if (r <= 0) {
-          if (r < 0 && onfinish != nullptr) {
-            client_lock.unlock();
-            onfinish->complete(r);
-            client_lock.lock();
-          }
           return r;
         }
 
@@ -15960,9 +15955,37 @@ int64_t Client::ll_preadv_pwritev(struct Fh *fh, const struct iovec *iov,
     }
 
     std::scoped_lock cl(client_lock);
-    return _preadv_pwritev_locked(fh, iov, iovcnt, offset, write, true,
-    				  onfinish, bl, do_fsync, syncdataonly);
 
+    int64_t retval = _preadv_pwritev_locked(fh, iov, iovcnt, offset, write,
+                                            true, onfinish, bl, do_fsync,
+                                            syncdataonly);
+    /* There are two scenarios with each having two cases to handle here
+    1) async io
+      1.a) r == 0:
+        async call in progress, the context will be automatically invoked,
+        so just return the retval (i.e. zero).
+      1.b) r < 0:
+        There was an error; no context completion should've took place so
+        complete the context with retval followed by returning zero to the
+        caller.
+    2) sync io
+      2.a) r >= 0:
+        sync call success; return the no. of bytes read/written.
+      2.b) r < 0:
+        sync call failed; return the errno. */
+
+    if (retval < 0) {
+      if (onfinish != nullptr) {
+        //async io failed
+        client_lock.unlock();
+        onfinish->complete(retval);
+        client_lock.lock();
+        /* async call should always return zero to caller and allow the
+        caller to wait on callback for the actual errno/retval. */
+        retval = 0;
+      }
+    }
+    return retval;
 }
 
 int Client::ll_flush(Fh *fh)

From cc7b69f90387d774c150ab5644b89804cbd9c190 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 7 Aug 2023 16:42:20 +0530
Subject: [PATCH 1678/2492] mgr/nfs: add a helper to validate cephfs path

Fixes: https://tracker.ceph.com/issues/62077
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/nfs/export.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py
index 2d07cd6eab60..ed3bc1be3a44 100644
--- a/src/pybind/mgr/nfs/export.py
+++ b/src/pybind/mgr/nfs/export.py
@@ -71,6 +71,17 @@ def normalize_path(path: str) -> str:
     return path
 
 
+def validate_cephfs_path(mgr: 'Module', fs_name: str, path: str) -> None:
+    try:
+        cephfs_path_is_dir(mgr, fs_name, path)
+    except NotADirectoryError:
+        raise NFSException(f"path {path} is not a dir", -errno.ENOTDIR)
+    except cephfs.ObjectNotFound:
+        raise NFSObjectNotFound(f"path {path} does not exist")
+    except cephfs.Error as e:
+        raise NFSException(e.args[1], -e.args[0])
+
+
 class NFSRados:
     def __init__(self, rados: 'Rados', namespace: str) -> None:
         self.rados = rados

From 52ef1f1abb259dca0c33519c49aabedd926c6cb2 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 7 Aug 2023 16:43:13 +0530
Subject: [PATCH 1679/2492] mgr/nfs: validate path before updating a cephfs
 export

Fixes: https://tracker.ceph.com/issues/62077
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/nfs/export.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py
index ed3bc1be3a44..8e6dcff7fcb5 100644
--- a/src/pybind/mgr/nfs/export.py
+++ b/src/pybind/mgr/nfs/export.py
@@ -686,6 +686,8 @@ def create_export_from_dict(self,
             if not check_fs(self.mgr, fs_name):
                 raise FSNotFound(fs_name)
 
+            validate_cephfs_path(self.mgr, fs_name, path)
+
             user_id = f"nfs.{cluster_id}.{ex_id}"
             if "user_id" in fsal and fsal["user_id"] != user_id:
                 raise NFSInvalidOperation(f"export FSAL user_id must be '{user_id}'")

From 7ce3ceabe1e1be46b7aa96d9dadc956748427487 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 23 Aug 2023 17:19:10 +0530
Subject: [PATCH 1680/2492] mgr/nfs: use helper to validate cephfs path

Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/pybind/mgr/nfs/export.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py
index 8e6dcff7fcb5..31020ee10e2b 100644
--- a/src/pybind/mgr/nfs/export.py
+++ b/src/pybind/mgr/nfs/export.py
@@ -714,14 +714,7 @@ def create_cephfs_export(self,
                              clients: list = [],
                              sectype: Optional[List[str]] = None) -> Dict[str, Any]:
 
-        try:
-            cephfs_path_is_dir(self.mgr, fs_name, path)
-        except NotADirectoryError:
-            raise NFSException(f"path {path} is not a dir", -errno.ENOTDIR)
-        except cephfs.ObjectNotFound:
-            raise NFSObjectNotFound(f"path {path} does not exist")
-        except cephfs.Error as e:
-            raise NFSException(e.args[1], -e.args[0])
+        validate_cephfs_path(self.mgr, fs_name, path)
 
         pseudo_path = normalize_path(pseudo_path)
 

From a2fae7b0e267045fc6de30b21db7c43ab940f76a Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 5 Dec 2023 15:41:01 +0530
Subject: [PATCH 1681/2492] mgr/cephadm: configure the dashboard gateways

cephadm configures the nvmeof gateways and add the gateways to a config
store which dashboard will later on fetch to make the grpc calls.

Fixes: https://tracker.ceph.com/issues/64201
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .gitmodules                                   |  3 +
 src/pybind/mgr/cephadm/serve.py               |  2 +-
 src/pybind/mgr/cephadm/services/nvmeof.py     | 45 ++++++++++++++-
 src/pybind/mgr/dashboard/ceph-nvmeof          |  1 +
 src/pybind/mgr/dashboard/constraints.txt      |  2 +
 .../mgr/dashboard/services/nvmeof_cli.py      | 44 +++++++++++++++
 .../mgr/dashboard/services/nvmeof_conf.py     | 55 +++++++++++++++++++
 7 files changed, 148 insertions(+), 4 deletions(-)
 create mode 160000 src/pybind/mgr/dashboard/ceph-nvmeof
 create mode 100644 src/pybind/mgr/dashboard/services/nvmeof_cli.py
 create mode 100644 src/pybind/mgr/dashboard/services/nvmeof_conf.py

diff --git a/.gitmodules b/.gitmodules
index 088ae3b577ce..e47fe6495f5f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -75,3 +75,6 @@
 [submodule "src/jaegertracing/opentelemetry-cpp"]
 	path = src/jaegertracing/opentelemetry-cpp
 	url = https://github.com/open-telemetry/opentelemetry-cpp.git
+[submodule "nvmeof"]
+	path = src/pybind/mgr/dashboard/ceph-nvmeof
+	url = https://github.com/ceph/ceph-nvmeof.git
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 116e97238691..4afe0a3b7aca 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -40,7 +40,7 @@
 
 logger = logging.getLogger(__name__)
 
-REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw']
+REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw', 'nvmeof']
 
 
 class CephadmServe:
diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index 7d2dd16cf0d6..32cb71b67447 100644
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -2,6 +2,7 @@
 import logging
 import json
 from typing import List, cast, Optional
+from ipaddress import ip_address, IPv6Address
 
 from mgr_module import HandleCommandResult
 from ceph.deployment.service_spec import NvmeofServiceSpec
@@ -55,8 +56,39 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         return daemon_spec
 
     def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None:
-        # TODO: what integration do we need with the dashboard?
-        pass
+        def get_set_cmd_dicts(out: str) -> List[dict]:
+            gateways = json.loads(out)['gateways']
+            cmd_dicts = []
+
+            spec = cast(NvmeofServiceSpec,
+                        self.mgr.spec_store.all_specs.get(daemon_descrs[0].service_name(), None))
+
+            for dd in daemon_descrs:
+                assert dd.hostname is not None
+
+                if not spec:
+                    logger.warning(f'No ServiceSpec found for {dd.service_name()}')
+                    continue
+
+                ip = utils.resolve_ip(self.mgr.inventory.get_addr(dd.hostname))
+                if type(ip_address(ip)) is IPv6Address:
+                    ip = f'[{ip}]'
+                service_url = '{}:{}'.format(ip, spec.port or '5500')
+                gw = gateways.get(dd.hostname)
+                if not gw or gw['service_url'] != service_url:
+                    logger.info(f'Adding NVMeoF gateway {service_url} to Dashboard')
+                    cmd_dicts.append({
+                        'prefix': 'dashboard nvmeof-gateway-add',
+                        'inbuf': service_url,
+                        'name': dd.hostname
+                    })
+            return cmd_dicts
+
+        self._check_and_set_dashboard(
+            service_name='nvmeof',
+            get_cmd='dashboard nvmeof-gateway-list',
+            get_set_cmd_dicts=get_set_cmd_dicts
+        )
 
     def ok_to_stop(self,
                    daemon_ids: List[str],
@@ -83,7 +115,14 @@ def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None
         Called after the daemon is removed.
         """
         logger.debug(f'Post remove daemon {self.TYPE}.{daemon.daemon_id}')
-        # TODO: remove config for dashboard nvmeof gateways if any
+        # remove config for dashboard nvmeof gateways if any
+        ret, out, err = self.mgr.mon_command({
+            'prefix': 'dashboard nvmeof-gateway-rm',
+            'name': daemon.hostname,
+        })
+        if not ret:
+            logger.info(f'{daemon.hostname} removed from iscsi gateways dashboard config')
+
         # and any certificates being used for mTLS
 
     def purge(self, service_name: str) -> None:
diff --git a/src/pybind/mgr/dashboard/ceph-nvmeof b/src/pybind/mgr/dashboard/ceph-nvmeof
new file mode 160000
index 000000000000..c6f6ce77863f
--- /dev/null
+++ b/src/pybind/mgr/dashboard/ceph-nvmeof
@@ -0,0 +1 @@
+Subproject commit c6f6ce77863f854444dee3d2a59d360f3b4f2255
diff --git a/src/pybind/mgr/dashboard/constraints.txt b/src/pybind/mgr/dashboard/constraints.txt
index fd6141048800..590a8c1e1c42 100644
--- a/src/pybind/mgr/dashboard/constraints.txt
+++ b/src/pybind/mgr/dashboard/constraints.txt
@@ -4,3 +4,5 @@ bcrypt~=3.1
 python3-saml~=1.4
 requests~=2.26
 Routes~=2.4
+grpcio~=1.48
+grpcio-tools~=1.48
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_cli.py b/src/pybind/mgr/dashboard/services/nvmeof_cli.py
new file mode 100644
index 000000000000..5921ab48ea8b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/nvmeof_cli.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import errno
+import json
+
+from mgr_module import CLICheckNonemptyFileInput, CLIReadCommand, CLIWriteCommand
+
+from ..rest_client import RequestException
+from .nvmeof_conf import NvmeofGatewaysConfig, NvmeofGatewayAlreadyExists, \
+    ManagedByOrchestratorException
+
+@CLIReadCommand('dashboard nvmeof-gateway-list')
+def list_nvmeof_gateways(_):
+    '''
+    List NVMe-oF gateways
+    '''
+    return 0, json.dumps(NvmeofGatewaysConfig.get_gateways_config()), ''
+
+@CLIWriteCommand('dashboard nvmeof-gateway-add')
+@CLICheckNonemptyFileInput(desc='NVMe-oF gateway configuration')
+def add_nvmeof_gateway(_, inbuf, name: str):
+    '''
+    Add NVMe-oF gateway configuration. Gateway URL read from -i <file>
+    '''
+    service_url = inbuf
+    try:
+        NvmeofGatewaysConfig.add_gateway(name, service_url)
+        return 0, 'Success', ''
+    except NvmeofGatewayAlreadyExists as ex:
+        return -errno.EEXIST, '', str(ex)
+    except ManagedByOrchestratorException as ex:
+        return -errno.EINVAL, '', str(ex)
+    except RequestException as ex:
+        return -errno.EINVAL, '', str(ex)
+
+@CLIWriteCommand('dashboard nvmeof-gateway-rm')
+def remove_nvmeof_gateway(_, name: str):
+    '''
+    Remove NVMe-oF gateway configuration
+    '''
+    try:
+        NvmeofGatewaysConfig.remove_gateway(name)
+        return 0, 'Success', ''
+    except ManagedByOrchestratorException as ex:
+        return -errno.EINVAL, '', str(ex)
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_conf.py b/src/pybind/mgr/dashboard/services/nvmeof_conf.py
new file mode 100644
index 000000000000..709635dc7db9
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/nvmeof_conf.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+import json
+
+from .. import mgr
+
+class NvmeofGatewayAlreadyExists(Exception):
+    def __init__(self, gateway_name):
+        super(NvmeofGatewayAlreadyExists, self).__init__(
+            "NVMe-oF gateway '{}' already exists".format(gateway_name))
+
+class NvmeofGatewayDoesNotExist(Exception):
+    def __init__(self, hostname):
+        super(NvmeofGatewayDoesNotExist, self).__init__(
+            "NVMe-oF gateway '{}' does not exist".format(hostname))
+
+class ManagedByOrchestratorException(Exception):
+    def __init__(self):
+        super(ManagedByOrchestratorException, self).__init__(
+            "NVMe-oF configuration is managed by the orchestrator")
+
+_NVMEOF_STORE_KEY = "_nvmeof_config"
+
+class NvmeofGatewaysConfig(object):
+    @classmethod
+    def _load_config_from_store(cls):
+        json_db = mgr.get_store(_NVMEOF_STORE_KEY,
+                                '{"gateways": {}}')
+        config = json.loads(json_db)
+        cls._save_config(config)
+        return config
+
+    @classmethod
+    def _save_config(cls, config):
+        mgr.set_store(_NVMEOF_STORE_KEY, json.dumps(config))
+    
+    @classmethod
+    def get_gateways_config(cls):
+        return cls._load_config_from_store()
+    
+    @classmethod
+    def add_gateway(cls, name, service_url):
+        config = cls.get_gateways_config()
+        if name in config:
+            raise NvmeofGatewayAlreadyExists(name)
+        config['gateways'][name] = {'service_url': service_url}
+        cls._save_config(config)
+    
+    @classmethod
+    def remove_gateway(cls, name):
+        config = cls.get_gateways_config()
+        if name not in config['gateways']:
+            raise NvmeofGatewayDoesNotExist(name)
+        del config['gateways'][name]
+        cls._save_config(config)

From 24015310b12ef131278e45d17222d53b15fa5570 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 5 Dec 2023 15:46:12 +0530
Subject: [PATCH 1682/2492] mgr/dashboard: gRPC client for nvmeof communication

Created a gRPC client by utilising the protobuf file available in nvmeof
repo

Copied the file to this repo and generated its output.

Fixes: https://tracker.ceph.com/issues/64201
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .gitmodules                                   |   3 -
 src/pybind/mgr/dashboard/ceph-nvmeof          |   1 -
 .../mgr/dashboard/controllers/nvmeof.py       |  23 +
 src/pybind/mgr/dashboard/module.py            |   3 +
 .../mgr/dashboard/services/nvmeof_client.py   |  29 +
 .../dashboard/services/proto/gateway.proto    | 247 ++++++++
 .../dashboard/services/proto/gateway_pb2.py   | 318 ++++++++++
 .../services/proto/gateway_pb2_grpc.py        | 577 ++++++++++++++++++
 8 files changed, 1197 insertions(+), 4 deletions(-)
 delete mode 160000 src/pybind/mgr/dashboard/ceph-nvmeof
 create mode 100644 src/pybind/mgr/dashboard/controllers/nvmeof.py
 create mode 100644 src/pybind/mgr/dashboard/services/nvmeof_client.py
 create mode 100644 src/pybind/mgr/dashboard/services/proto/gateway.proto
 create mode 100644 src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
 create mode 100644 src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py

diff --git a/.gitmodules b/.gitmodules
index e47fe6495f5f..088ae3b577ce 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -75,6 +75,3 @@
 [submodule "src/jaegertracing/opentelemetry-cpp"]
 	path = src/jaegertracing/opentelemetry-cpp
 	url = https://github.com/open-telemetry/opentelemetry-cpp.git
-[submodule "nvmeof"]
-	path = src/pybind/mgr/dashboard/ceph-nvmeof
-	url = https://github.com/ceph/ceph-nvmeof.git
diff --git a/src/pybind/mgr/dashboard/ceph-nvmeof b/src/pybind/mgr/dashboard/ceph-nvmeof
deleted file mode 160000
index c6f6ce77863f..000000000000
--- a/src/pybind/mgr/dashboard/ceph-nvmeof
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c6f6ce77863f854444dee3d2a59d360f3b4f2255
diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
new file mode 100644
index 000000000000..274985245fe3
--- /dev/null
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -0,0 +1,23 @@
+# # import grpc
+
+# from .proto import gateway_pb2 as pb2
+# from .proto import gateway_pb2_grpc as pb2_grpc
+
+
+# class NVMeoFClient(object):
+#     def __init__(self):
+#         self.host = '192.168.100.102'
+
+# from ..cephnvmeof.control.cli import GatewayClient
+
+from ..security import Scope
+from ..services.nvmeof_client import NVMeoFClient
+from . import APIDoc, APIRouter, RESTController, Endpoint, ReadPermission, CreatePermission
+
+@APIRouter('/nvmeof', Scope.ISCSI)
+@APIDoc('NVMe-oF Management API', 'NVMe-oF')
+class Nvmeof(RESTController):
+    @ReadPermission
+    def list(self):
+        """List all NVMeoF gateways"""
+        return NVMeoFClient().get_subsystems()
diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index efef273bda06..24e947c7066c 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -37,6 +37,9 @@
 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
     prepare_url_prefix, str_to_bool
 
+# pylint: disable=unused-import
+from .services import nvmeof_cli
+
 try:
     import cherrypy
     from cherrypy._cptools import HandlerWrapperTool
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
new file mode 100644
index 000000000000..ec8911abc20b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -0,0 +1,29 @@
+import grpc
+import json
+
+import logging
+
+from .proto import gateway_pb2 as pb2
+from .proto import gateway_pb2_grpc as pb2_grpc
+
+from google.protobuf.json_format import MessageToJson
+
+from .nvmeof_conf import NvmeofGatewaysConfig
+
+logger = logging.getLogger('nvmeof_client')
+
+
+class NVMeoFClient(object):
+    def __init__(self):
+        logger.info('Initiating nvmeof gateway connection...')
+
+        self.gateway_addr = list(NvmeofGatewaysConfig.get_gateways_config()['gateways'].values())[0]['service_url']
+        self.channel = grpc.insecure_channel(
+            '{}'.format(self.gateway_addr)
+        )
+        logger.info('Found nvmeof gateway at {}'.format(self.gateway_addr))
+        self.stub = pb2_grpc.GatewayStub(self.channel)
+
+    def get_subsystems(self):
+        response = self.stub.get_subsystems(pb2.get_subsystems_req())
+        return json.loads(MessageToJson(response))
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway.proto b/src/pybind/mgr/dashboard/services/proto/gateway.proto
new file mode 100644
index 000000000000..85238c8210a3
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/proto/gateway.proto
@@ -0,0 +1,247 @@
+//
+//  Copyright (c) 2021 International Business Machines
+//  All rights reserved.
+//
+//  SPDX-License-Identifier: MIT
+//
+//  Authors: anita.shekar@ibm.com, sandy.kaur@ibm.com
+//
+
+
+syntax = "proto3";
+
+enum TransportType {
+    INVALID = 0;
+    RDMA = 1;
+    FC = 2;
+    TCP = 3;
+    PCIE = 256;
+    VFIOUSER = 1024;
+    CUSTOM = 4096;
+}
+
+enum AddressFamily {
+    invalid = 0;
+    ipv4 = 1;
+    ipv6 = 2;
+    ib = 3;
+    fc = 4;
+}
+
+enum LogLevel {
+    DISABLED = 0;
+    ERROR = 1;
+    WARN = 2;
+    NOTICE = 3;
+    INFO = 4;
+    DEBUG = 5;
+}
+
+service Gateway {
+	// Creates a bdev from an RBD image
+	rpc create_bdev(create_bdev_req) returns (bdev) {}
+
+	// Resizes a bdev
+	rpc resize_bdev(resize_bdev_req) returns (req_status) {}
+
+	// Deletes a bdev
+	rpc delete_bdev(delete_bdev_req) returns (req_status) {}
+
+	// Creates a subsystem
+	rpc create_subsystem(create_subsystem_req) returns(req_status) {}
+
+	// Deletes a subsystem
+	rpc delete_subsystem(delete_subsystem_req) returns(req_status) {}
+
+	// Adds a namespace to a subsystem
+	rpc add_namespace(add_namespace_req) returns(nsid_status) {}
+
+	// Removes a namespace from a subsystem
+	rpc remove_namespace(remove_namespace_req) returns(req_status) {}
+
+	// Adds a host to a subsystem
+	rpc add_host(add_host_req) returns (req_status) {}
+
+	// Removes a host from a subsystem
+	rpc remove_host(remove_host_req) returns (req_status) {}
+
+	// Creates a listener for a subsystem at a given IP/Port
+	rpc create_listener(create_listener_req) returns(req_status) {}
+
+	// Deletes a listener from a subsystem at a given IP/Port
+	rpc delete_listener(delete_listener_req) returns(req_status) {}
+
+	// Gets subsystems
+	rpc get_subsystems(get_subsystems_req) returns(subsystems_info) {}
+
+	// Gets spdk nvmf log flags and level
+	rpc get_spdk_nvmf_log_flags_and_level(get_spdk_nvmf_log_flags_and_level_req) returns(spdk_nvmf_log_flags_and_level_info) {}
+
+	// Disables spdk nvmf logs
+	rpc disable_spdk_nvmf_logs(disable_spdk_nvmf_logs_req) returns(req_status) {}
+
+	// Set spdk nvmf logs
+	rpc set_spdk_nvmf_logs(set_spdk_nvmf_logs_req) returns(req_status) {}
+
+	// Set spdk nvmf logs
+	rpc get_gateway_info(get_gateway_info_req) returns(gateway_info) {}
+}
+
+// Request messages
+
+message create_bdev_req {
+	string bdev_name = 1;
+	string rbd_pool_name = 2;
+	string rbd_image_name = 3;
+	int32 block_size = 4;
+	optional string uuid = 5;
+}
+
+message resize_bdev_req {
+	string bdev_name = 1;
+	int32 new_size = 2;
+}
+
+message delete_bdev_req {
+	string bdev_name = 1;
+	bool force = 2;
+}
+
+message create_subsystem_req {
+	string subsystem_nqn = 1;
+	string serial_number = 2;
+	int32 max_namespaces = 3;
+	bool  ana_reporting  = 4;
+	bool  enable_ha      = 5;
+}
+
+message delete_subsystem_req {
+	string subsystem_nqn = 1;
+}
+
+message add_namespace_req {
+	string subsystem_nqn = 1;
+	string bdev_name = 2;
+	optional uint32 nsid = 3;
+	optional int32 anagrpid = 4;
+}
+
+message remove_namespace_req {
+	string subsystem_nqn = 1;
+	uint32 nsid = 2;
+}
+
+message add_host_req {
+	string subsystem_nqn = 1;
+	string host_nqn = 2;
+}
+
+message remove_host_req {
+	string subsystem_nqn = 1;
+	string host_nqn = 2;
+}
+
+message create_listener_req {
+	string nqn = 1;
+	string gateway_name = 2;
+	TransportType trtype = 3;
+	AddressFamily adrfam = 4;
+	string traddr = 5;
+	string trsvcid = 6;
+}
+
+message delete_listener_req {
+	string nqn = 1;
+	string gateway_name = 2;
+	TransportType trtype = 3;
+	AddressFamily adrfam = 4;
+	string traddr = 5;
+	string trsvcid = 6;
+}
+
+message get_subsystems_req {
+}
+
+message get_spdk_nvmf_log_flags_and_level_req {
+}
+
+message disable_spdk_nvmf_logs_req {
+}
+
+message set_spdk_nvmf_logs_req {
+	optional LogLevel log_level = 1;
+	optional LogLevel print_level = 2;
+}
+
+message get_gateway_info_req {
+	string cli_version = 1;
+}
+
+// Return messages 
+
+message bdev {
+	string bdev_name = 1;
+	bool status = 2;
+}
+
+message req_status {
+	bool status = 1;
+}
+
+message nsid_status {
+	uint32 nsid = 1;
+	bool status = 2;
+}
+
+message subsystems_info {
+	repeated subsystem subsystems = 1;
+}
+
+message subsystem {
+	string nqn = 1;
+	string subtype = 2;
+	repeated listen_address listen_addresses = 3;
+	repeated host hosts = 4;
+	bool allow_any_host = 5;
+	optional string serial_number = 6;
+	optional string model_number = 7;
+	optional uint32 max_namespaces = 8;
+	optional uint32 min_cntlid = 9;
+	optional uint32 max_cntlid = 10;
+	repeated namespace namespaces = 11;
+}
+
+message gateway_info {
+	string cli_version = 1;
+	string gateway_version = 2;
+	string gateway_name = 3;
+	string gateway_group = 4;
+	string gateway_addr = 5;
+	string gateway_port = 6;
+	bool status = 7;
+}
+
+message listen_address {
+	string transport = 1;
+	TransportType trtype = 2;
+	AddressFamily adrfam = 3;
+	string traddr = 4;
+	string trsvcid = 5;
+}
+
+message host {
+    string nqn = 1;
+}
+
+message namespace {
+    uint32 nsid = 1;
+    string name = 2;
+    optional string bdev_name = 3;
+    optional string nguid = 4;
+    optional string uuid = 5;
+    optional uint32 anagrpid = 6;
+}
+
+message spdk_nvmf_log_flags_and_level_info {
+	string flags_level =1;
+}
\ No newline at end of file
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
new file mode 100644
index 000000000000..2cd4d0c5fe14
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: gateway.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rgateway.proto\"\x83\x01\n\x0f\x63reate_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x12\n\nblock_size\x18\x04 \x01(\x05\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x00\x88\x01\x01\x42\x07\n\x05_uuid\"6\n\x0fresize_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x10\n\x08new_size\x18\x02 \x01(\x05\"3\n\x0f\x64\x65lete_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\"\x86\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x16\n\x0emax_namespaces\x18\x03 \x01(\x05\x12\x15\n\rana_reporting\x18\x04 \x01(\x08\x12\x11\n\tenable_ha\x18\x05 \x01(\x08\"-\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\"}\n\x11\x61\x64\x64_namespace_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x11\n\x04nsid\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x0b\n\t_anagrpid\";\n\x14remove_namespace_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x0c\n\x04nsid\x18\x02 \x01(\r\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"\x99\x01\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x1e\n\x06trtype\x18\x03 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x04 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x05 \x01(\t\x12\x0f\n\x07trsvcid\x18\x06 \x01(\t\"\x99\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x1e\n\x06trtype\x18\x03 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x04 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x05 \x01(\t\x12\x0f\n\x07trsvcid\x18\x06 \x01(\t\"\x14\n\x12get_subsystems_req\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"+\n\x14get_gateway_info_req\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\")\n\x04\x62\x64\x65v\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x08\"\x1c\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x08\"+\n\x0bnsid_status\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0e\n\x06status\x18\x02 \x01(\x08\"1\n\x0fsubsystems_info\x12\x1e\n\nsubsystems\x18\x01 \x03(\x0b\x32\n.subsystem\"\xfc\x02\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0f\n\x07subtype\x18\x02 \x01(\t\x12)\n\x10listen_addresses\x18\x03 \x03(\x0b\x32\x0f.listen_address\x12\x14\n\x05hosts\x18\x04 \x03(\x0b\x32\x05.host\x12\x16\n\x0e\x61llow_any_host\x18\x05 \x01(\x08\x12\x1a\n\rserial_number\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0cmodel_number\x18\x07 \x01(\tH\x01\x88\x01\x01\x12\x1b\n\x0emax_namespaces\x18\x08 \x01(\rH\x02\x88\x01\x01\x12\x17\n\nmin_cntlid\x18\t \x01(\rH\x03\x88\x01\x01\x12\x17\n\nmax_cntlid\x18\n \x01(\rH\x04\x88\x01\x01\x12\x1e\n\nnamespaces\x18\x0b \x03(\x0b\x32\n.namespaceB\x10\n\x0e_serial_numberB\x0f\n\r_model_numberB\x11\n\x0f_max_namespacesB\r\n\x0b_min_cntlidB\r\n\x0b_max_cntlid\"\xa5\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x17\n\x0fgateway_version\x18\x02 \x01(\t\x12\x14\n\x0cgateway_name\x18\x03 \x01(\t\x12\x15\n\rgateway_group\x18\x04 \x01(\t\x12\x14\n\x0cgateway_addr\x18\x05 \x01(\t\x12\x14\n\x0cgateway_port\x18\x06 \x01(\t\x12\x0e\n\x06status\x18\x07 \x01(\x08\"\x84\x01\n\x0elisten_address\x12\x11\n\ttransport\x18\x01 \x01(\t\x12\x1e\n\x06trtype\x18\x02 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\t\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"\xab\x01\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\tbdev_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05nguid\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x06 \x01(\rH\x03\x88\x01\x01\x42\x0c\n\n_bdev_nameB\x08\n\x06_nguidB\x07\n\x05_uuidB\x0b\n\t_anagrpid\"9\n\"spdk_nvmf_log_flags_and_level_info\x12\x13\n\x0b\x66lags_level\x18\x01 \x01(\t*^\n\rTransportType\x12\x0b\n\x07INVALID\x10\x00\x12\x08\n\x04RDMA\x10\x01\x12\x06\n\x02\x46\x43\x10\x02\x12\x07\n\x03TCP\x10\x03\x12\t\n\x04PCIE\x10\x80\x02\x12\r\n\x08VFIOUSER\x10\x80\x08\x12\x0b\n\x06\x43USTOM\x10\x80 *@\n\rAddressFamily\x12\x0b\n\x07invalid\x10\x00\x12\x08\n\x04ipv4\x10\x01\x12\x08\n\x04ipv6\x10\x02\x12\x06\n\x02ib\x10\x03\x12\x06\n\x02\x66\x63\x10\x04*N\n\x08LogLevel\x12\x0c\n\x08\x44ISABLED\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x08\n\x04WARN\x10\x02\x12\n\n\x06NOTICE\x10\x03\x12\x08\n\x04INFO\x10\x04\x12\t\n\x05\x44\x45\x42UG\x10\x05\x32\xaf\x07\n\x07Gateway\x12(\n\x0b\x63reate_bdev\x12\x10.create_bdev_req\x1a\x05.bdev\"\x00\x12.\n\x0bresize_bdev\x12\x10.resize_bdev_req\x1a\x0b.req_status\"\x00\x12.\n\x0b\x64\x65lete_bdev\x12\x10.delete_bdev_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12\x33\n\radd_namespace\x12\x12.add_namespace_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10remove_namespace\x12\x15.remove_namespace_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x39\n\x0eget_subsystems\x12\x13.get_subsystems_req\x1a\x10.subsystems_info\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3')
+
+_TRANSPORTTYPE = DESCRIPTOR.enum_types_by_name['TransportType']
+TransportType = enum_type_wrapper.EnumTypeWrapper(_TRANSPORTTYPE)
+_ADDRESSFAMILY = DESCRIPTOR.enum_types_by_name['AddressFamily']
+AddressFamily = enum_type_wrapper.EnumTypeWrapper(_ADDRESSFAMILY)
+_LOGLEVEL = DESCRIPTOR.enum_types_by_name['LogLevel']
+LogLevel = enum_type_wrapper.EnumTypeWrapper(_LOGLEVEL)
+INVALID = 0
+RDMA = 1
+FC = 2
+TCP = 3
+PCIE = 256
+VFIOUSER = 1024
+CUSTOM = 4096
+invalid = 0
+ipv4 = 1
+ipv6 = 2
+ib = 3
+fc = 4
+DISABLED = 0
+ERROR = 1
+WARN = 2
+NOTICE = 3
+INFO = 4
+DEBUG = 5
+
+
+_CREATE_BDEV_REQ = DESCRIPTOR.message_types_by_name['create_bdev_req']
+_RESIZE_BDEV_REQ = DESCRIPTOR.message_types_by_name['resize_bdev_req']
+_DELETE_BDEV_REQ = DESCRIPTOR.message_types_by_name['delete_bdev_req']
+_CREATE_SUBSYSTEM_REQ = DESCRIPTOR.message_types_by_name['create_subsystem_req']
+_DELETE_SUBSYSTEM_REQ = DESCRIPTOR.message_types_by_name['delete_subsystem_req']
+_ADD_NAMESPACE_REQ = DESCRIPTOR.message_types_by_name['add_namespace_req']
+_REMOVE_NAMESPACE_REQ = DESCRIPTOR.message_types_by_name['remove_namespace_req']
+_ADD_HOST_REQ = DESCRIPTOR.message_types_by_name['add_host_req']
+_REMOVE_HOST_REQ = DESCRIPTOR.message_types_by_name['remove_host_req']
+_CREATE_LISTENER_REQ = DESCRIPTOR.message_types_by_name['create_listener_req']
+_DELETE_LISTENER_REQ = DESCRIPTOR.message_types_by_name['delete_listener_req']
+_GET_SUBSYSTEMS_REQ = DESCRIPTOR.message_types_by_name['get_subsystems_req']
+_GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ = DESCRIPTOR.message_types_by_name['get_spdk_nvmf_log_flags_and_level_req']
+_DISABLE_SPDK_NVMF_LOGS_REQ = DESCRIPTOR.message_types_by_name['disable_spdk_nvmf_logs_req']
+_SET_SPDK_NVMF_LOGS_REQ = DESCRIPTOR.message_types_by_name['set_spdk_nvmf_logs_req']
+_GET_GATEWAY_INFO_REQ = DESCRIPTOR.message_types_by_name['get_gateway_info_req']
+_BDEV = DESCRIPTOR.message_types_by_name['bdev']
+_REQ_STATUS = DESCRIPTOR.message_types_by_name['req_status']
+_NSID_STATUS = DESCRIPTOR.message_types_by_name['nsid_status']
+_SUBSYSTEMS_INFO = DESCRIPTOR.message_types_by_name['subsystems_info']
+_SUBSYSTEM = DESCRIPTOR.message_types_by_name['subsystem']
+_GATEWAY_INFO = DESCRIPTOR.message_types_by_name['gateway_info']
+_LISTEN_ADDRESS = DESCRIPTOR.message_types_by_name['listen_address']
+_HOST = DESCRIPTOR.message_types_by_name['host']
+_NAMESPACE = DESCRIPTOR.message_types_by_name['namespace']
+_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO = DESCRIPTOR.message_types_by_name['spdk_nvmf_log_flags_and_level_info']
+create_bdev_req = _reflection.GeneratedProtocolMessageType('create_bdev_req', (_message.Message,), {
+  'DESCRIPTOR' : _CREATE_BDEV_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:create_bdev_req)
+  })
+_sym_db.RegisterMessage(create_bdev_req)
+
+resize_bdev_req = _reflection.GeneratedProtocolMessageType('resize_bdev_req', (_message.Message,), {
+  'DESCRIPTOR' : _RESIZE_BDEV_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:resize_bdev_req)
+  })
+_sym_db.RegisterMessage(resize_bdev_req)
+
+delete_bdev_req = _reflection.GeneratedProtocolMessageType('delete_bdev_req', (_message.Message,), {
+  'DESCRIPTOR' : _DELETE_BDEV_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:delete_bdev_req)
+  })
+_sym_db.RegisterMessage(delete_bdev_req)
+
+create_subsystem_req = _reflection.GeneratedProtocolMessageType('create_subsystem_req', (_message.Message,), {
+  'DESCRIPTOR' : _CREATE_SUBSYSTEM_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:create_subsystem_req)
+  })
+_sym_db.RegisterMessage(create_subsystem_req)
+
+delete_subsystem_req = _reflection.GeneratedProtocolMessageType('delete_subsystem_req', (_message.Message,), {
+  'DESCRIPTOR' : _DELETE_SUBSYSTEM_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:delete_subsystem_req)
+  })
+_sym_db.RegisterMessage(delete_subsystem_req)
+
+add_namespace_req = _reflection.GeneratedProtocolMessageType('add_namespace_req', (_message.Message,), {
+  'DESCRIPTOR' : _ADD_NAMESPACE_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:add_namespace_req)
+  })
+_sym_db.RegisterMessage(add_namespace_req)
+
+remove_namespace_req = _reflection.GeneratedProtocolMessageType('remove_namespace_req', (_message.Message,), {
+  'DESCRIPTOR' : _REMOVE_NAMESPACE_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:remove_namespace_req)
+  })
+_sym_db.RegisterMessage(remove_namespace_req)
+
+add_host_req = _reflection.GeneratedProtocolMessageType('add_host_req', (_message.Message,), {
+  'DESCRIPTOR' : _ADD_HOST_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:add_host_req)
+  })
+_sym_db.RegisterMessage(add_host_req)
+
+remove_host_req = _reflection.GeneratedProtocolMessageType('remove_host_req', (_message.Message,), {
+  'DESCRIPTOR' : _REMOVE_HOST_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:remove_host_req)
+  })
+_sym_db.RegisterMessage(remove_host_req)
+
+create_listener_req = _reflection.GeneratedProtocolMessageType('create_listener_req', (_message.Message,), {
+  'DESCRIPTOR' : _CREATE_LISTENER_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:create_listener_req)
+  })
+_sym_db.RegisterMessage(create_listener_req)
+
+delete_listener_req = _reflection.GeneratedProtocolMessageType('delete_listener_req', (_message.Message,), {
+  'DESCRIPTOR' : _DELETE_LISTENER_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:delete_listener_req)
+  })
+_sym_db.RegisterMessage(delete_listener_req)
+
+get_subsystems_req = _reflection.GeneratedProtocolMessageType('get_subsystems_req', (_message.Message,), {
+  'DESCRIPTOR' : _GET_SUBSYSTEMS_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:get_subsystems_req)
+  })
+_sym_db.RegisterMessage(get_subsystems_req)
+
+get_spdk_nvmf_log_flags_and_level_req = _reflection.GeneratedProtocolMessageType('get_spdk_nvmf_log_flags_and_level_req', (_message.Message,), {
+  'DESCRIPTOR' : _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:get_spdk_nvmf_log_flags_and_level_req)
+  })
+_sym_db.RegisterMessage(get_spdk_nvmf_log_flags_and_level_req)
+
+disable_spdk_nvmf_logs_req = _reflection.GeneratedProtocolMessageType('disable_spdk_nvmf_logs_req', (_message.Message,), {
+  'DESCRIPTOR' : _DISABLE_SPDK_NVMF_LOGS_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:disable_spdk_nvmf_logs_req)
+  })
+_sym_db.RegisterMessage(disable_spdk_nvmf_logs_req)
+
+set_spdk_nvmf_logs_req = _reflection.GeneratedProtocolMessageType('set_spdk_nvmf_logs_req', (_message.Message,), {
+  'DESCRIPTOR' : _SET_SPDK_NVMF_LOGS_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:set_spdk_nvmf_logs_req)
+  })
+_sym_db.RegisterMessage(set_spdk_nvmf_logs_req)
+
+get_gateway_info_req = _reflection.GeneratedProtocolMessageType('get_gateway_info_req', (_message.Message,), {
+  'DESCRIPTOR' : _GET_GATEWAY_INFO_REQ,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:get_gateway_info_req)
+  })
+_sym_db.RegisterMessage(get_gateway_info_req)
+
+bdev = _reflection.GeneratedProtocolMessageType('bdev', (_message.Message,), {
+  'DESCRIPTOR' : _BDEV,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:bdev)
+  })
+_sym_db.RegisterMessage(bdev)
+
+req_status = _reflection.GeneratedProtocolMessageType('req_status', (_message.Message,), {
+  'DESCRIPTOR' : _REQ_STATUS,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:req_status)
+  })
+_sym_db.RegisterMessage(req_status)
+
+nsid_status = _reflection.GeneratedProtocolMessageType('nsid_status', (_message.Message,), {
+  'DESCRIPTOR' : _NSID_STATUS,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:nsid_status)
+  })
+_sym_db.RegisterMessage(nsid_status)
+
+subsystems_info = _reflection.GeneratedProtocolMessageType('subsystems_info', (_message.Message,), {
+  'DESCRIPTOR' : _SUBSYSTEMS_INFO,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:subsystems_info)
+  })
+_sym_db.RegisterMessage(subsystems_info)
+
+subsystem = _reflection.GeneratedProtocolMessageType('subsystem', (_message.Message,), {
+  'DESCRIPTOR' : _SUBSYSTEM,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:subsystem)
+  })
+_sym_db.RegisterMessage(subsystem)
+
+gateway_info = _reflection.GeneratedProtocolMessageType('gateway_info', (_message.Message,), {
+  'DESCRIPTOR' : _GATEWAY_INFO,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:gateway_info)
+  })
+_sym_db.RegisterMessage(gateway_info)
+
+listen_address = _reflection.GeneratedProtocolMessageType('listen_address', (_message.Message,), {
+  'DESCRIPTOR' : _LISTEN_ADDRESS,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:listen_address)
+  })
+_sym_db.RegisterMessage(listen_address)
+
+host = _reflection.GeneratedProtocolMessageType('host', (_message.Message,), {
+  'DESCRIPTOR' : _HOST,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:host)
+  })
+_sym_db.RegisterMessage(host)
+
+namespace = _reflection.GeneratedProtocolMessageType('namespace', (_message.Message,), {
+  'DESCRIPTOR' : _NAMESPACE,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:namespace)
+  })
+_sym_db.RegisterMessage(namespace)
+
+spdk_nvmf_log_flags_and_level_info = _reflection.GeneratedProtocolMessageType('spdk_nvmf_log_flags_and_level_info', (_message.Message,), {
+  'DESCRIPTOR' : _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO,
+  '__module__' : 'gateway_pb2'
+  # @@protoc_insertion_point(class_scope:spdk_nvmf_log_flags_and_level_info)
+  })
+_sym_db.RegisterMessage(spdk_nvmf_log_flags_and_level_info)
+
+_GATEWAY = DESCRIPTOR.services_by_name['Gateway']
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _TRANSPORTTYPE._serialized_start=2436
+  _TRANSPORTTYPE._serialized_end=2530
+  _ADDRESSFAMILY._serialized_start=2532
+  _ADDRESSFAMILY._serialized_end=2596
+  _LOGLEVEL._serialized_start=2598
+  _LOGLEVEL._serialized_end=2676
+  _CREATE_BDEV_REQ._serialized_start=18
+  _CREATE_BDEV_REQ._serialized_end=149
+  _RESIZE_BDEV_REQ._serialized_start=151
+  _RESIZE_BDEV_REQ._serialized_end=205
+  _DELETE_BDEV_REQ._serialized_start=207
+  _DELETE_BDEV_REQ._serialized_end=258
+  _CREATE_SUBSYSTEM_REQ._serialized_start=261
+  _CREATE_SUBSYSTEM_REQ._serialized_end=395
+  _DELETE_SUBSYSTEM_REQ._serialized_start=397
+  _DELETE_SUBSYSTEM_REQ._serialized_end=442
+  _ADD_NAMESPACE_REQ._serialized_start=444
+  _ADD_NAMESPACE_REQ._serialized_end=569
+  _REMOVE_NAMESPACE_REQ._serialized_start=571
+  _REMOVE_NAMESPACE_REQ._serialized_end=630
+  _ADD_HOST_REQ._serialized_start=632
+  _ADD_HOST_REQ._serialized_end=687
+  _REMOVE_HOST_REQ._serialized_start=689
+  _REMOVE_HOST_REQ._serialized_end=747
+  _CREATE_LISTENER_REQ._serialized_start=750
+  _CREATE_LISTENER_REQ._serialized_end=903
+  _DELETE_LISTENER_REQ._serialized_start=906
+  _DELETE_LISTENER_REQ._serialized_end=1059
+  _GET_SUBSYSTEMS_REQ._serialized_start=1061
+  _GET_SUBSYSTEMS_REQ._serialized_end=1081
+  _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ._serialized_start=1083
+  _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ._serialized_end=1122
+  _DISABLE_SPDK_NVMF_LOGS_REQ._serialized_start=1124
+  _DISABLE_SPDK_NVMF_LOGS_REQ._serialized_end=1152
+  _SET_SPDK_NVMF_LOGS_REQ._serialized_start=1154
+  _SET_SPDK_NVMF_LOGS_REQ._serialized_end=1280
+  _GET_GATEWAY_INFO_REQ._serialized_start=1282
+  _GET_GATEWAY_INFO_REQ._serialized_end=1325
+  _BDEV._serialized_start=1327
+  _BDEV._serialized_end=1368
+  _REQ_STATUS._serialized_start=1370
+  _REQ_STATUS._serialized_end=1398
+  _NSID_STATUS._serialized_start=1400
+  _NSID_STATUS._serialized_end=1443
+  _SUBSYSTEMS_INFO._serialized_start=1445
+  _SUBSYSTEMS_INFO._serialized_end=1494
+  _SUBSYSTEM._serialized_start=1497
+  _SUBSYSTEM._serialized_end=1877
+  _GATEWAY_INFO._serialized_start=1880
+  _GATEWAY_INFO._serialized_end=2045
+  _LISTEN_ADDRESS._serialized_start=2048
+  _LISTEN_ADDRESS._serialized_end=2180
+  _HOST._serialized_start=2182
+  _HOST._serialized_end=2201
+  _NAMESPACE._serialized_start=2204
+  _NAMESPACE._serialized_end=2375
+  _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO._serialized_start=2377
+  _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO._serialized_end=2434
+  _GATEWAY._serialized_start=2679
+  _GATEWAY._serialized_end=3622
+# @@protoc_insertion_point(module_scope)
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
new file mode 100644
index 000000000000..b3d9c2dcc76b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
@@ -0,0 +1,577 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from ..proto import gateway_pb2 as gateway__pb2
+
+
+class GatewayStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.create_bdev = channel.unary_unary(
+                '/Gateway/create_bdev',
+                request_serializer=gateway__pb2.create_bdev_req.SerializeToString,
+                response_deserializer=gateway__pb2.bdev.FromString,
+                )
+        self.resize_bdev = channel.unary_unary(
+                '/Gateway/resize_bdev',
+                request_serializer=gateway__pb2.resize_bdev_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.delete_bdev = channel.unary_unary(
+                '/Gateway/delete_bdev',
+                request_serializer=gateway__pb2.delete_bdev_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.create_subsystem = channel.unary_unary(
+                '/Gateway/create_subsystem',
+                request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.delete_subsystem = channel.unary_unary(
+                '/Gateway/delete_subsystem',
+                request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.add_namespace = channel.unary_unary(
+                '/Gateway/add_namespace',
+                request_serializer=gateway__pb2.add_namespace_req.SerializeToString,
+                response_deserializer=gateway__pb2.nsid_status.FromString,
+                )
+        self.remove_namespace = channel.unary_unary(
+                '/Gateway/remove_namespace',
+                request_serializer=gateway__pb2.remove_namespace_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.add_host = channel.unary_unary(
+                '/Gateway/add_host',
+                request_serializer=gateway__pb2.add_host_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.remove_host = channel.unary_unary(
+                '/Gateway/remove_host',
+                request_serializer=gateway__pb2.remove_host_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.create_listener = channel.unary_unary(
+                '/Gateway/create_listener',
+                request_serializer=gateway__pb2.create_listener_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.delete_listener = channel.unary_unary(
+                '/Gateway/delete_listener',
+                request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.get_subsystems = channel.unary_unary(
+                '/Gateway/get_subsystems',
+                request_serializer=gateway__pb2.get_subsystems_req.SerializeToString,
+                response_deserializer=gateway__pb2.subsystems_info.FromString,
+                )
+        self.get_spdk_nvmf_log_flags_and_level = channel.unary_unary(
+                '/Gateway/get_spdk_nvmf_log_flags_and_level',
+                request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+                response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+                )
+        self.disable_spdk_nvmf_logs = channel.unary_unary(
+                '/Gateway/disable_spdk_nvmf_logs',
+                request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.set_spdk_nvmf_logs = channel.unary_unary(
+                '/Gateway/set_spdk_nvmf_logs',
+                request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
+        self.get_gateway_info = channel.unary_unary(
+                '/Gateway/get_gateway_info',
+                request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
+                response_deserializer=gateway__pb2.gateway_info.FromString,
+                )
+
+
+class GatewayServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def create_bdev(self, request, context):
+        """Creates a bdev from an RBD image
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def resize_bdev(self, request, context):
+        """Resizes a bdev
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def delete_bdev(self, request, context):
+        """Deletes a bdev
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def create_subsystem(self, request, context):
+        """Creates a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def delete_subsystem(self, request, context):
+        """Deletes a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def add_namespace(self, request, context):
+        """Adds a namespace to a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def remove_namespace(self, request, context):
+        """Removes a namespace from a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def add_host(self, request, context):
+        """Adds a host to a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def remove_host(self, request, context):
+        """Removes a host from a subsystem
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def create_listener(self, request, context):
+        """Creates a listener for a subsystem at a given IP/Port
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def delete_listener(self, request, context):
+        """Deletes a listener from a subsystem at a given IP/Port
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def get_subsystems(self, request, context):
+        """Gets subsystems
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def get_spdk_nvmf_log_flags_and_level(self, request, context):
+        """Gets spdk nvmf log flags and level
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def disable_spdk_nvmf_logs(self, request, context):
+        """Disables spdk nvmf logs
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def set_spdk_nvmf_logs(self, request, context):
+        """Set spdk nvmf logs
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def get_gateway_info(self, request, context):
+        """Set spdk nvmf logs
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_GatewayServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'create_bdev': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_bdev,
+                    request_deserializer=gateway__pb2.create_bdev_req.FromString,
+                    response_serializer=gateway__pb2.bdev.SerializeToString,
+            ),
+            'resize_bdev': grpc.unary_unary_rpc_method_handler(
+                    servicer.resize_bdev,
+                    request_deserializer=gateway__pb2.resize_bdev_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'delete_bdev': grpc.unary_unary_rpc_method_handler(
+                    servicer.delete_bdev,
+                    request_deserializer=gateway__pb2.delete_bdev_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'create_subsystem': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_subsystem,
+                    request_deserializer=gateway__pb2.create_subsystem_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'delete_subsystem': grpc.unary_unary_rpc_method_handler(
+                    servicer.delete_subsystem,
+                    request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'add_namespace': grpc.unary_unary_rpc_method_handler(
+                    servicer.add_namespace,
+                    request_deserializer=gateway__pb2.add_namespace_req.FromString,
+                    response_serializer=gateway__pb2.nsid_status.SerializeToString,
+            ),
+            'remove_namespace': grpc.unary_unary_rpc_method_handler(
+                    servicer.remove_namespace,
+                    request_deserializer=gateway__pb2.remove_namespace_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'add_host': grpc.unary_unary_rpc_method_handler(
+                    servicer.add_host,
+                    request_deserializer=gateway__pb2.add_host_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'remove_host': grpc.unary_unary_rpc_method_handler(
+                    servicer.remove_host,
+                    request_deserializer=gateway__pb2.remove_host_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'create_listener': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_listener,
+                    request_deserializer=gateway__pb2.create_listener_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'delete_listener': grpc.unary_unary_rpc_method_handler(
+                    servicer.delete_listener,
+                    request_deserializer=gateway__pb2.delete_listener_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'get_subsystems': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_subsystems,
+                    request_deserializer=gateway__pb2.get_subsystems_req.FromString,
+                    response_serializer=gateway__pb2.subsystems_info.SerializeToString,
+            ),
+            'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_spdk_nvmf_log_flags_and_level,
+                    request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
+                    response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
+            ),
+            'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+                    servicer.disable_spdk_nvmf_logs,
+                    request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+                    servicer.set_spdk_nvmf_logs,
+                    request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'get_gateway_info': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_gateway_info,
+                    request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
+                    response_serializer=gateway__pb2.gateway_info.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'Gateway', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Gateway(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def create_bdev(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/create_bdev',
+            gateway__pb2.create_bdev_req.SerializeToString,
+            gateway__pb2.bdev.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def resize_bdev(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/resize_bdev',
+            gateway__pb2.resize_bdev_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def delete_bdev(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/delete_bdev',
+            gateway__pb2.delete_bdev_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def create_subsystem(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/create_subsystem',
+            gateway__pb2.create_subsystem_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def delete_subsystem(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/delete_subsystem',
+            gateway__pb2.delete_subsystem_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def add_namespace(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/add_namespace',
+            gateway__pb2.add_namespace_req.SerializeToString,
+            gateway__pb2.nsid_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def remove_namespace(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/remove_namespace',
+            gateway__pb2.remove_namespace_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def add_host(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/add_host',
+            gateway__pb2.add_host_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def remove_host(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/remove_host',
+            gateway__pb2.remove_host_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def create_listener(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/create_listener',
+            gateway__pb2.create_listener_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def delete_listener(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/delete_listener',
+            gateway__pb2.delete_listener_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def get_subsystems(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/get_subsystems',
+            gateway__pb2.get_subsystems_req.SerializeToString,
+            gateway__pb2.subsystems_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def get_spdk_nvmf_log_flags_and_level(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/get_spdk_nvmf_log_flags_and_level',
+            gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+            gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def disable_spdk_nvmf_logs(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/disable_spdk_nvmf_logs',
+            gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def set_spdk_nvmf_logs(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/set_spdk_nvmf_logs',
+            gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def get_gateway_info(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/get_gateway_info',
+            gateway__pb2.get_gateway_info_req.SerializeToString,
+            gateway__pb2.gateway_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

From 1b768332b16f9d0735bf4217375c96ca1ad0881b Mon Sep 17 00:00:00 2001
From: avanthakkar <avanjohn@gmail.com>
Date: Mon, 11 Dec 2023 18:50:54 +0530
Subject: [PATCH 1683/2492] mgr/dashboard: introduce APIs for NvmeOf management

Fixes: https://tracker.ceph.com/issues/64201
Signed-off-by: avanthakkar <avanjohn@gmail.com>
---
 .../mgr/dashboard/controllers/nvmeof.py       | 89 ++++++++++++++++-
 src/pybind/mgr/dashboard/security.py          |  1 +
 .../mgr/dashboard/services/access_control.py  |  1 +
 .../mgr/dashboard/services/nvmeof_client.py   | 99 +++++++++++++++++++
 4 files changed, 188 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
index 274985245fe3..6458c47a8856 100644
--- a/src/pybind/mgr/dashboard/controllers/nvmeof.py
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -10,14 +10,99 @@
 
 # from ..cephnvmeof.control.cli import GatewayClient
 
+from typing import Optional
 from ..security import Scope
 from ..services.nvmeof_client import NVMeoFClient
-from . import APIDoc, APIRouter, RESTController, Endpoint, ReadPermission, CreatePermission
+# from ..services.proto import gateway_pb2 as pb2
+from . import APIDoc, APIRouter, RESTController, Endpoint, ReadPermission, CreatePermission, \
+    DeletePermission, allow_empty_body, UpdatePermission
+    
 
-@APIRouter('/nvmeof', Scope.ISCSI)
+@APIRouter('/nvmeof', Scope.NVME_OF)
 @APIDoc('NVMe-oF Management API', 'NVMe-oF')
 class Nvmeof(RESTController):
     @ReadPermission
     def list(self):
         """List all NVMeoF gateways"""
         return NVMeoFClient().get_subsystems()
+
+
+@APIRouter('/nvmeof/bdev', Scope.NVME_OF)
+@APIDoc('NVMe-oF Block Device Management API', 'NVMe-oF')
+class NvmeofBdev(RESTController):
+    @CreatePermission
+    def create(self, name: str, rbd_pool: str, rbd_image: str, block_size: int, uuid: Optional[str] = None):
+        """Create a new NVMeoF block device"""
+        return NVMeoFClient().create_bdev(name, rbd_pool, rbd_image, block_size, uuid)
+    
+    @DeletePermission
+    @allow_empty_body
+    def delete(self, name: str, force: bool):
+        """Delete an existing NVMeoF block device"""
+        return NVMeoFClient().delete_bdev(name, force)
+    
+    @Endpoint('PUT')
+    @UpdatePermission
+    @allow_empty_body
+    def resize(self, name: str, size: int):
+        """Resize an existing NVMeoF block device"""
+        return NVMeoFClient().resize_bdev(name, size)
+
+
+@APIRouter('/nvmeof/namespace', Scope.NVME_OF)
+@APIDoc('NVMe-oF Namespace Management API', 'NVMe-oF')
+class NvmeofNamespace(RESTController):
+    @CreatePermission
+    def create(self, subsystem_nqn: str, bdev_name: str, nsid: int, anagrpid: Optional[str] = None):
+        """Create a new NVMeoF namespace"""
+        return NVMeoFClient().create_namespace(subsystem_nqn, bdev_name, nsid, anagrpid)
+    
+    @Endpoint('DELETE', path='{subsystem_nqn}')
+    def delete(self, subsystem_nqn: str, nsid: int):
+        """Delete an existing NVMeoF namespace"""
+        return NVMeoFClient().delete_namespace(subsystem_nqn, nsid)
+    
+@APIRouter('/nvmeof/subsystem', Scope.NVME_OF)
+@APIDoc('NVMe-oF Subsystem Management API', 'NVMe-oF')
+class NvmeofSubsystem(RESTController):
+    @CreatePermission
+    def create(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
+                         ana_reporting: bool, enable_ha: bool) :
+        """Create a new NVMeoF subsystem"""
+        return NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces,
+                                               ana_reporting, enable_ha)
+    
+    @Endpoint('DELETE', path='{subsystem_nqn}')
+    def delete(self, subsystem_nqn: str):
+        """Delete an existing NVMeoF subsystem"""
+        return NVMeoFClient().delete_subsystem(subsystem_nqn)
+
+
+@APIRouter('/nvmeof/hosts', Scope.NVME_OF)
+@APIDoc('NVMe-oF Host Management API', 'NVMe-oF')
+class NvmeofHost(RESTController):
+    @CreatePermission
+    def create(self, subsystem_nqn: str, host_nqn: str):
+        """Create a new NVMeoF host"""
+        return NVMeoFClient().add_host(subsystem_nqn, host_nqn)
+    
+    @Endpoint('DELETE')
+    def delete(self, subsystem_nqn: str, host_nqn: str):
+        """Delete an existing NVMeoF host"""
+        return NVMeoFClient().remove_host(subsystem_nqn, host_nqn)
+
+
+@APIRouter('/nvmeof/listener', Scope.NVME_OF)
+@APIDoc('NVMe-oF Listener Management API', 'NVMe-oF')
+class NvmeofListener(RESTController):
+    @CreatePermission
+    def create(self, nqn: str, gateway: str, trtype: str, adrfam: str,
+               traddr: str, trsvcid: str):
+        """Create a new NVMeoF listener"""
+        return NVMeoFClient().create_listener(nqn, gateway, trtype, adrfam, traddr, trsvcid)
+    
+    @Endpoint('DELETE')
+    def delete(self, nqn: str, gateway: str, trtype, adrfam,
+               traddr: str, trsvcid: str):
+        """Delete an existing NVMeoF listener"""
+        return NVMeoFClient().delete_listener(nqn, gateway, trtype, adrfam, traddr, trsvcid)
\ No newline at end of file
diff --git a/src/pybind/mgr/dashboard/security.py b/src/pybind/mgr/dashboard/security.py
index 4c6e5c564af3..2b624aabcc72 100644
--- a/src/pybind/mgr/dashboard/security.py
+++ b/src/pybind/mgr/dashboard/security.py
@@ -26,6 +26,7 @@ class Scope(object):
     USER = "user"
     DASHBOARD_SETTINGS = "dashboard-settings"
     NFS_GANESHA = "nfs-ganesha"
+    NVME_OF = "nvme-of"
 
     @classmethod
     def all_scopes(cls):
diff --git a/src/pybind/mgr/dashboard/services/access_control.py b/src/pybind/mgr/dashboard/services/access_control.py
index 0cbe49bb160a..b45f81fb9b1d 100644
--- a/src/pybind/mgr/dashboard/services/access_control.py
+++ b/src/pybind/mgr/dashboard/services/access_control.py
@@ -222,6 +222,7 @@ def from_dict(cls, r_dict):
         Scope.ISCSI: [_P.READ, _P.CREATE, _P.UPDATE, _P.DELETE],
         Scope.RBD_MIRRORING: [_P.READ, _P.CREATE, _P.UPDATE, _P.DELETE],
         Scope.GRAFANA: [_P.READ],
+        Scope.NVME_OF: [_P.READ, _P.CREATE, _P.UPDATE, _P.DELETE],
     })
 
 
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index ec8911abc20b..ff7c6dfc8ced 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -1,3 +1,5 @@
+from enum import Enum
+from typing import Optional
 import grpc
 import json
 
@@ -9,6 +11,7 @@
 from google.protobuf.json_format import MessageToJson
 
 from .nvmeof_conf import NvmeofGatewaysConfig
+from ..tools import str_to_bool
 
 logger = logging.getLogger('nvmeof_client')
 
@@ -27,3 +30,99 @@ def __init__(self):
     def get_subsystems(self):
         response = self.stub.get_subsystems(pb2.get_subsystems_req())
         return json.loads(MessageToJson(response))
+    
+    def create_bdev(self, name: str, rbd_pool: str, rbd_image: str, block_size: int, uuid: Optional[str] = None):
+        response = self.stub.create_bdev(pb2.create_bdev_req(
+            bdev_name=name,
+            rbd_pool_name=rbd_pool,
+            rbd_image_name=rbd_image,
+            block_size=block_size,
+            uuid=uuid
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def resize_bdev(self, name: str, size: int):
+        response = self.stub.resize_bdev(pb2.resize_bdev_req(
+            bdev_name=name,
+            new_size=size
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def delete_bdev(self, name: str, force: bool):
+        response = self.stub.delete_bdev(pb2.delete_bdev_req(
+            bdev_name=name,
+            force=str_to_bool(force)
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
+                         ana_reporting: bool, enable_ha: bool) :
+        response = self.stub.create_subsystem(pb2.create_subsystem_req(
+            subsystem_nqn=subsystem_nqn,
+            serial_number=serial_number,
+            max_namespaces=int(max_namespaces),
+            ana_reporting=str_to_bool(ana_reporting),
+            enable_ha=str_to_bool(enable_ha)
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def delete_subsystem(self, subsystem_nqn: str):
+        response = self.stub.delete_subsystem(pb2.delete_subsystem_req(
+            subsystem_nqn=subsystem_nqn
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def create_namespace(self, subsystem_nqn: str, bdev_name: str, nsid: int, anagrpid: Optional[str] = None):
+        response = self.stub.add_namespace(pb2.add_namespace_req(
+            subsystem_nqn=subsystem_nqn,
+            bdev_name=bdev_name,
+            nsid=int(nsid),
+            anagrpid=anagrpid
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def delete_namespace(self, subsystem_nqn: str, nsid: int):
+        response = self.stub.remove_namespace(pb2.remove_namespace_req(
+            subsystem_nqn=subsystem_nqn,
+            nsid=nsid
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def add_host(self, subsystem_nqn: str, host_nqn: str):
+        response = self.stub.add_host(pb2.add_host_req(
+            subsystem_nqn=subsystem_nqn,
+            host_nqn=host_nqn
+        ))
+        return json.loads(MessageToJson(response))
+    
+    def remove_host(self, subsystem_nqn: str, host_nqn: str):
+        response = self.stub.remove_host(pb2.remove_host_req(
+            subsystem_nqn=subsystem_nqn,
+            host_nqn=host_nqn
+        ))
+        return json.loads(MessageToJson(response))
+
+    def create_listener(self, nqn: str, gateway: str, trtype: str, adrfam: str,
+                        traddr: str, trsvcid: str):
+        req = pb2.create_listener_req(
+                nqn=nqn,
+                gateway_name=gateway,
+                trtype=pb2.TransportType.Value(trtype.upper()),
+                adrfam=pb2.AddressFamily.Value(adrfam.lower()),
+                traddr=traddr,
+                trsvcid=trsvcid,
+            )
+        ret = self.stub.create_listener(req)
+        return json.loads(MessageToJson(ret))
+    
+    def delete_listener(self, nqn: str, gateway: str, trttype, adrfam,
+                        traddr: str, trsvcid: str):
+        response = self.stub.delete_listener(pb2.delete_listener_req(
+            nqn=nqn,
+            gateway_name=gateway,
+            trtype=trttype,
+            adrfam=adrfam,
+            traddr=traddr,
+            trsvcid=trsvcid
+        ))
+        return json.loads(MessageToJson(response))

From 35e40c4e8ff293bf18cdf78a372732f31d36467f Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 12 Jan 2024 14:11:18 +0530
Subject: [PATCH 1684/2492] mgr/dashboard: adapt to new nvmeof APIs

Fixes: https://tracker.ceph.com/issues/64201
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 ceph.spec.in                                  |    2 +
 debian/control                                |    1 +
 src/pybind/mgr/cephadm/services/nvmeof.py     |    2 +-
 src/pybind/mgr/dashboard/constraints.txt      |    2 -
 .../mgr/dashboard/controllers/nvmeof.py       |  290 +-
 src/pybind/mgr/dashboard/module.py            |    4 +-
 src/pybind/mgr/dashboard/openapi.yaml         |  546 +++
 src/pybind/mgr/dashboard/requirements.txt     |    2 +
 .../mgr/dashboard/services/nvmeof_cli.py      |    7 +-
 .../mgr/dashboard/services/nvmeof_client.py   |  256 +-
 .../mgr/dashboard/services/nvmeof_conf.py     |   11 +-
 .../dashboard/services/proto/gateway.proto    |  332 +-
 .../dashboard/services/proto/gateway_pb2.py   | 3425 +++++++++++++++--
 .../services/proto/gateway_pb2_grpc.py        |  959 +++--
 src/pybind/mgr/dashboard/tox.ini              |    2 +
 15 files changed, 4911 insertions(+), 930 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 58ccf067997a..d061803099ea 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -625,6 +625,8 @@ Requires:       ceph-mgr = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-grafana-dashboards = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-prometheus-alerts = %{_epoch_prefix}%{version}-%{release}
 Requires:       python%{python3_pkgversion}-setuptools
+Requires:       python%{python3_pkgversion}-grpcio
+Requires:       python%{python3_pkgversion}-grpcio-tools
 %if 0%{?fedora} || 0%{?rhel} || 0%{?openEuler}
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-routes
diff --git a/debian/control b/debian/control
index f1bdec95a2ef..b4967b398f4d 100644
--- a/debian/control
+++ b/debian/control
@@ -96,6 +96,7 @@ Build-Depends: automake,
                tox <pkg.ceph.check>,
                python3-coverage <pkg.ceph.check>,
                python3-dateutil <pkg.ceph.check>,
+               python3-grpcio <pkg.ceph.check>,
                python3-openssl <pkg.ceph.check>,
                python3-prettytable <pkg.ceph.check>,
                python3-requests <pkg.ceph.check>,
diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index 32cb71b67447..7d2de75f67f9 100644
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -121,7 +121,7 @@ def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None
             'name': daemon.hostname,
         })
         if not ret:
-            logger.info(f'{daemon.hostname} removed from iscsi gateways dashboard config')
+            logger.info(f'{daemon.hostname} removed from nvmeof gateways dashboard config')
 
         # and any certificates being used for mTLS
 
diff --git a/src/pybind/mgr/dashboard/constraints.txt b/src/pybind/mgr/dashboard/constraints.txt
index 590a8c1e1c42..fd6141048800 100644
--- a/src/pybind/mgr/dashboard/constraints.txt
+++ b/src/pybind/mgr/dashboard/constraints.txt
@@ -4,5 +4,3 @@ bcrypt~=3.1
 python3-saml~=1.4
 requests~=2.26
 Routes~=2.4
-grpcio~=1.48
-grpcio-tools~=1.48
diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
index 6458c47a8856..125b93dc3cc7 100644
--- a/src/pybind/mgr/dashboard/controllers/nvmeof.py
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -1,108 +1,192 @@
-# # import grpc
+# -*- coding: utf-8 -*-
+import json
+from typing import Optional
 
-# from .proto import gateway_pb2 as pb2
-# from .proto import gateway_pb2_grpc as pb2_grpc
+from ..security import Scope
+from . import APIDoc, APIRouter, CreatePermission, DeletePermission, Endpoint, \
+    EndpointDoc, ReadPermission, RESTController
 
+try:
+    from google.protobuf.json_format import MessageToJson
 
-# class NVMeoFClient(object):
-#     def __init__(self):
-#         self.host = '192.168.100.102'
+    from ..services.nvmeof_client import NVMeoFClient
+except ImportError:
+    MessageToJson = None
+else:
+    @APIRouter('/nvmeof/namespace', Scope.NVME_OF)
+    @APIDoc('NVMe-oF Namespace Management API', 'NVMe-oF')
+    class NvmeofNamespace(RESTController):
+        @ReadPermission
+        def list(self, subsystem_nqn: str):
+            """
+            List all NVMeoF namespaces
+            """
+            response = MessageToJson(NVMeoFClient().list_namespaces(subsystem_nqn))
+            return json.loads(response)
 
-# from ..cephnvmeof.control.cli import GatewayClient
+        @CreatePermission
+        def create(self, rbd_pool: str, rbd_image: str, subsystem_nqn: str,
+                   create_image: Optional[bool] = True, image_size: Optional[int] = 1024,
+                   block_size: int = 512, nsid: Optional[int] = 1,
+                   uuid: Optional[str] = None, anagrpid: Optional[int] = 1):
+            """
+            Create a new NVMeoF namespace
+            :param rbd_pool: RBD pool name
+            :param rbd_image: RBD image name
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            :param create_image: Create RBD image
+            :param image_size: RBD image size
+            :param block_size: NVMeoF namespace block size
+            :param nsid: NVMeoF namespace ID
+            :param uuid: NVMeoF namespace UUID
+            :param anagrpid: NVMeoF namespace ANA group ID
+            """
+            response = NVMeoFClient().create_namespace(rbd_pool, rbd_image,
+                                                       subsystem_nqn, block_size,
+                                                       nsid, uuid, anagrpid,
+                                                       create_image, image_size)
+            return json.loads(MessageToJson(response))
 
-from typing import Optional
-from ..security import Scope
-from ..services.nvmeof_client import NVMeoFClient
-# from ..services.proto import gateway_pb2 as pb2
-from . import APIDoc, APIRouter, RESTController, Endpoint, ReadPermission, CreatePermission, \
-    DeletePermission, allow_empty_body, UpdatePermission
-    
-
-@APIRouter('/nvmeof', Scope.NVME_OF)
-@APIDoc('NVMe-oF Management API', 'NVMe-oF')
-class Nvmeof(RESTController):
-    @ReadPermission
-    def list(self):
-        """List all NVMeoF gateways"""
-        return NVMeoFClient().get_subsystems()
-
-
-@APIRouter('/nvmeof/bdev', Scope.NVME_OF)
-@APIDoc('NVMe-oF Block Device Management API', 'NVMe-oF')
-class NvmeofBdev(RESTController):
-    @CreatePermission
-    def create(self, name: str, rbd_pool: str, rbd_image: str, block_size: int, uuid: Optional[str] = None):
-        """Create a new NVMeoF block device"""
-        return NVMeoFClient().create_bdev(name, rbd_pool, rbd_image, block_size, uuid)
-    
-    @DeletePermission
-    @allow_empty_body
-    def delete(self, name: str, force: bool):
-        """Delete an existing NVMeoF block device"""
-        return NVMeoFClient().delete_bdev(name, force)
-    
-    @Endpoint('PUT')
-    @UpdatePermission
-    @allow_empty_body
-    def resize(self, name: str, size: int):
-        """Resize an existing NVMeoF block device"""
-        return NVMeoFClient().resize_bdev(name, size)
-
-
-@APIRouter('/nvmeof/namespace', Scope.NVME_OF)
-@APIDoc('NVMe-oF Namespace Management API', 'NVMe-oF')
-class NvmeofNamespace(RESTController):
-    @CreatePermission
-    def create(self, subsystem_nqn: str, bdev_name: str, nsid: int, anagrpid: Optional[str] = None):
-        """Create a new NVMeoF namespace"""
-        return NVMeoFClient().create_namespace(subsystem_nqn, bdev_name, nsid, anagrpid)
-    
-    @Endpoint('DELETE', path='{subsystem_nqn}')
-    def delete(self, subsystem_nqn: str, nsid: int):
-        """Delete an existing NVMeoF namespace"""
-        return NVMeoFClient().delete_namespace(subsystem_nqn, nsid)
-    
-@APIRouter('/nvmeof/subsystem', Scope.NVME_OF)
-@APIDoc('NVMe-oF Subsystem Management API', 'NVMe-oF')
-class NvmeofSubsystem(RESTController):
-    @CreatePermission
-    def create(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
-                         ana_reporting: bool, enable_ha: bool) :
-        """Create a new NVMeoF subsystem"""
-        return NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces,
-                                               ana_reporting, enable_ha)
-    
-    @Endpoint('DELETE', path='{subsystem_nqn}')
-    def delete(self, subsystem_nqn: str):
-        """Delete an existing NVMeoF subsystem"""
-        return NVMeoFClient().delete_subsystem(subsystem_nqn)
-
-
-@APIRouter('/nvmeof/hosts', Scope.NVME_OF)
-@APIDoc('NVMe-oF Host Management API', 'NVMe-oF')
-class NvmeofHost(RESTController):
-    @CreatePermission
-    def create(self, subsystem_nqn: str, host_nqn: str):
-        """Create a new NVMeoF host"""
-        return NVMeoFClient().add_host(subsystem_nqn, host_nqn)
-    
-    @Endpoint('DELETE')
-    def delete(self, subsystem_nqn: str, host_nqn: str):
-        """Delete an existing NVMeoF host"""
-        return NVMeoFClient().remove_host(subsystem_nqn, host_nqn)
-
-
-@APIRouter('/nvmeof/listener', Scope.NVME_OF)
-@APIDoc('NVMe-oF Listener Management API', 'NVMe-oF')
-class NvmeofListener(RESTController):
-    @CreatePermission
-    def create(self, nqn: str, gateway: str, trtype: str, adrfam: str,
-               traddr: str, trsvcid: str):
-        """Create a new NVMeoF listener"""
-        return NVMeoFClient().create_listener(nqn, gateway, trtype, adrfam, traddr, trsvcid)
-    
-    @Endpoint('DELETE')
-    def delete(self, nqn: str, gateway: str, trtype, adrfam,
-               traddr: str, trsvcid: str):
-        """Delete an existing NVMeoF listener"""
-        return NVMeoFClient().delete_listener(nqn, gateway, trtype, adrfam, traddr, trsvcid)
\ No newline at end of file
+        @Endpoint('DELETE', path='{subsystem_nqn}')
+        def delete(self, subsystem_nqn: str, nsid: int):
+            """
+            Delete an existing NVMeoF namespace
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            :param nsid: NVMeoF namespace ID
+            """
+            response = NVMeoFClient().delete_namespace(subsystem_nqn, nsid)
+            return json.loads(MessageToJson(response))
+
+    @APIRouter('/nvmeof/subsystem', Scope.NVME_OF)
+    @APIDoc('NVMe-oF Subsystem Management API', 'NVMe-oF')
+    class NvmeofSubsystem(RESTController):
+        @ReadPermission
+        @EndpointDoc("List all NVMeoF gateways",
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'serial_number': (str, 'NVMeoF subsystem serial number')
+                     })
+        def list(self, subsystem_nqn: Optional[str] = None, serial_number: Optional[str] = None):
+            response = MessageToJson(NVMeoFClient().list_subsystems(
+                subsystem_nqn=subsystem_nqn, serial_number=serial_number))
+
+            return json.loads(response)
+
+        @CreatePermission
+        def create(self, subsystem_nqn: str, serial_number: Optional[str] = None,
+                   max_namespaces: Optional[int] = 256, ana_reporting: Optional[bool] = False,
+                   enable_ha: Optional[bool] = False):
+            """
+            Create a new NVMeoF subsystem
+
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            :param serial_number: NVMeoF subsystem serial number
+            :param max_namespaces: NVMeoF subsystem maximum namespaces
+            :param ana_reporting: NVMeoF subsystem ANA reporting
+            :param enable_ha: NVMeoF subsystem enable HA
+            """
+            response = NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces,
+                                                       ana_reporting, enable_ha)
+            return json.loads(MessageToJson(response))
+
+        @DeletePermission
+        @Endpoint('DELETE', path='{subsystem_nqn}')
+        def delete(self, subsystem_nqn: str):
+            """
+            Delete an existing NVMeoF subsystem
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            """
+            response = NVMeoFClient().delete_subsystem(subsystem_nqn)
+            return json.loads(MessageToJson(response))
+
+    @APIRouter('/nvmeof/hosts', Scope.NVME_OF)
+    @APIDoc('NVMe-oF Host Management API', 'NVMe-oF')
+    class NvmeofHost(RESTController):
+        @ReadPermission
+        def list(self, subsystem_nqn: str):
+            """
+            List all NVMeoF hosts
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            """
+            response = MessageToJson(NVMeoFClient().list_hosts(subsystem_nqn))
+            return json.loads(response)
+
+        @CreatePermission
+        def create(self, subsystem_nqn: str, host_nqn: str):
+            """
+            Create a new NVMeoF host
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            :param host_nqn: NVMeoF host NQN
+            """
+            response = NVMeoFClient().add_host(subsystem_nqn, host_nqn)
+            return json.loads(MessageToJson(response))
+
+        @DeletePermission
+        def delete(self, subsystem_nqn: str, host_nqn: str):
+            """
+            Delete an existing NVMeoF host
+            :param subsystem_nqn: NVMeoF subsystem NQN
+            :param host_nqn: NVMeoF host NQN
+            """
+            response = NVMeoFClient().remove_host(subsystem_nqn, host_nqn)
+            return json.loads(MessageToJson(response))
+
+    @APIRouter('/nvmeof/listener', Scope.NVME_OF)
+    @APIDoc('NVMe-oF Listener Management API', 'NVMe-oF')
+    class NvmeofListener(RESTController):
+        @ReadPermission
+        def list(self, subsystem_nqn: str):
+            """
+            List all NVMeoF listeners
+            :param nqn: NVMeoF subsystem NQN
+            """
+            response = MessageToJson(NVMeoFClient().list_listeners(subsystem_nqn))
+            return json.loads(response)
+
+        @CreatePermission
+        def create(self, nqn: str, gateway: str, traddr: Optional[str] = None,
+                   trtype: Optional[str] = 'TCP', adrfam: Optional[str] = 'IPV4',
+                   trsvcid: Optional[int] = 4420,
+                   auto_ha_state: Optional[str] = 'AUTO_HA_UNSET'):
+            """
+            Create a new NVMeoF listener
+            :param nqn: NVMeoF subsystem NQN
+            :param gateway: NVMeoF gateway
+            :param traddr: NVMeoF transport address
+            :param trtype: NVMeoF transport type
+            :param adrfam: NVMeoF address family
+            :param trsvcid: NVMeoF transport service ID
+            :param auto_ha_state: NVMeoF auto HA state
+            """
+            response = NVMeoFClient().create_listener(nqn, gateway, traddr,
+                                                      trtype, adrfam, trsvcid, auto_ha_state)
+            return json.loads(MessageToJson(response))
+
+        @DeletePermission
+        def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None,
+                   transport_type: Optional[str] = 'TCP', addr_family: Optional[str] = 'IPV4',
+                   transport_svc_id: Optional[int] = 4420):
+            """
+            Delete an existing NVMeoF listener
+            :param nqn: NVMeoF subsystem NQN
+            :param gateway: NVMeoF gateway
+            :param traddr: NVMeoF transport address
+            :param transport_type: NVMeoF transport type
+            :param addr_family: NVMeoF address family
+            :param transport_svc_id: NVMeoF transport service ID
+            """
+            response = NVMeoFClient().delete_listener(nqn, gateway, traddr, transport_type,
+                                                      addr_family, transport_svc_id)
+            return json.loads(MessageToJson(response))
+
+    @APIRouter('/nvmeof/gateway', Scope.NVME_OF)
+    @APIDoc('NVMe-oF Gateway Management API', 'NVMe-oF')
+    class NvmeofGateway(RESTController):
+        @ReadPermission
+        @Endpoint()
+        def info(self):
+            """
+            Get NVMeoF gateway information
+            """
+            response = MessageToJson(NVMeoFClient().gateway_info())
+            return json.loads(response)
diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index 24e947c7066c..41160b698aae 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -29,6 +29,7 @@
 from . import mgr
 from .controllers import Router, json_error_page
 from .grafana import push_local_dashboards
+from .services import nvmeof_cli  # noqa # pylint: disable=unused-import
 from .services.auth import AuthManager, AuthManagerTool, JwtManager
 from .services.exception import dashboard_exception_handler
 from .services.rgw_client import configure_rgw_credentials
@@ -37,9 +38,6 @@
 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
     prepare_url_prefix, str_to_bool
 
-# pylint: disable=unused-import
-from .services import nvmeof_cli
-
 try:
     import cherrypy
     from cherrypy._cptools import HandlerWrapperTool
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 55a38701eb6d..b7bb688520c4 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -7318,6 +7318,550 @@ paths:
       summary: Updates an NFS-Ganesha export
       tags:
       - NFS-Ganesha
+  /api/nvmeof/gateway/info:
+    get:
+      description: "\n            Get NVMeoF gateway information\n            "
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/hosts:
+    post:
+      description: "\n            Create a new NVMeoF host\n            :param subsystem_nqn:\
+        \ NVMeoF subsystem NQN\n            :param host_nqn: NVMeoF host NQN\n   \
+        \         "
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                host_nqn:
+                  type: string
+                subsystem_nqn:
+                  type: string
+              required:
+              - subsystem_nqn
+              - host_nqn
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/hosts/{subsystem_nqn}:
+    get:
+      description: "\n            List all NVMeoF hosts\n            :param subsystem_nqn:\
+        \ NVMeoF subsystem NQN\n            "
+      parameters:
+      - in: path
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/hosts/{subsystem_nqn}/{host_nqn}:
+    delete:
+      description: "\n            Delete an existing NVMeoF host\n            :param\
+        \ subsystem_nqn: NVMeoF subsystem NQN\n            :param host_nqn: NVMeoF\
+        \ host NQN\n            "
+      parameters:
+      - in: path
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: host_nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/listener:
+    get:
+      description: "\n            List all NVMeoF listeners\n            :param nqn:\
+        \ NVMeoF subsystem NQN\n            "
+      parameters:
+      - in: query
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+    post:
+      description: "\n            Create a new NVMeoF listener\n            :param\
+        \ nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
+        \            :param traddr: NVMeoF transport address\n            :param trtype:\
+        \ NVMeoF transport type\n            :param adrfam: NVMeoF address family\n\
+        \            :param trsvcid: NVMeoF transport service ID\n            :param\
+        \ auto_ha_state: NVMeoF auto HA state\n            "
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                adrfam:
+                  default: IPV4
+                  type: string
+                auto_ha_state:
+                  default: AUTO_HA_UNSET
+                  type: string
+                gateway:
+                  type: string
+                nqn:
+                  type: string
+                traddr:
+                  type: string
+                trsvcid:
+                  default: 4420
+                  type: integer
+                trtype:
+                  default: TCP
+                  type: string
+              required:
+              - nqn
+              - gateway
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/listener/{nqn}/{gateway}:
+    delete:
+      description: "\n            Delete an existing NVMeoF listener\n           \
+        \ :param nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
+        \            :param traddr: NVMeoF transport address\n            :param transport_type:\
+        \ NVMeoF transport type\n            :param addr_family: NVMeoF address family\n\
+        \            :param transport_svc_id: NVMeoF transport service ID\n      \
+        \      "
+      parameters:
+      - in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: gateway
+        required: true
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: traddr
+        schema:
+          type: string
+      - default: TCP
+        in: query
+        name: transport_type
+        schema:
+          type: string
+      - default: IPV4
+        in: query
+        name: addr_family
+        schema:
+          type: string
+      - default: 4420
+        in: query
+        name: transport_svc_id
+        schema:
+          type: integer
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/namespace:
+    post:
+      description: "\n            Create a new NVMeoF namespace\n            :param\
+        \ rbd_pool: RBD pool name\n            :param rbd_image: RBD image name\n\
+        \            :param subsystem_nqn: NVMeoF subsystem NQN\n            :param\
+        \ create_image: Create RBD image\n            :param image_size: RBD image\
+        \ size\n            :param block_size: NVMeoF namespace block size\n     \
+        \       :param nsid: NVMeoF namespace ID\n            :param uuid: NVMeoF\
+        \ namespace UUID\n            :param anagrpid: NVMeoF namespace ANA group\
+        \ ID\n            "
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                anagrpid:
+                  default: 1
+                  type: integer
+                block_size:
+                  default: 512
+                  type: integer
+                create_image:
+                  default: true
+                  type: boolean
+                image_size:
+                  default: 1024
+                  type: integer
+                nsid:
+                  default: 1
+                  type: integer
+                rbd_image:
+                  type: string
+                rbd_pool:
+                  type: string
+                subsystem_nqn:
+                  type: string
+                uuid:
+                  type: string
+              required:
+              - rbd_pool
+              - rbd_image
+              - subsystem_nqn
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/namespace/{subsystem_nqn}:
+    get:
+      description: "\n            List all NVMeoF namespaces\n            "
+      parameters:
+      - in: path
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/namespace/{subsystem_nqn}/{nsid}:
+    delete:
+      description: "\n            Delete an existing NVMeoF namespace\n          \
+        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            :param nsid: NVMeoF\
+        \ namespace ID\n            "
+      parameters:
+      - in: path
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: nsid
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/subsystem:
+    get:
+      parameters:
+      - allowEmptyValue: true
+        description: NVMeoF subsystem NQN
+        in: query
+        name: subsystem_nqn
+        schema:
+          type: string
+      - allowEmptyValue: true
+        description: NVMeoF subsystem serial number
+        in: query
+        name: serial_number
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: List all NVMeoF gateways
+      tags:
+      - NVMe-oF
+    post:
+      description: "\n            Create a new NVMeoF subsystem\n\n            :param\
+        \ subsystem_nqn: NVMeoF subsystem NQN\n            :param serial_number: NVMeoF\
+        \ subsystem serial number\n            :param max_namespaces: NVMeoF subsystem\
+        \ maximum namespaces\n            :param ana_reporting: NVMeoF subsystem ANA\
+        \ reporting\n            :param enable_ha: NVMeoF subsystem enable HA\n  \
+        \          "
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                ana_reporting:
+                  default: false
+                  type: boolean
+                enable_ha:
+                  default: false
+                  type: boolean
+                max_namespaces:
+                  default: 256
+                  type: integer
+                serial_number:
+                  type: integer
+                subsystem_nqn:
+                  type: string
+              required:
+              - subsystem_nqn
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
+  /api/nvmeof/subsystem/{subsystem_nqn}:
+    delete:
+      description: "\n            Delete an existing NVMeoF subsystem\n          \
+        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            "
+      parameters:
+      - in: path
+        name: subsystem_nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - NVMe-oF
   /api/osd:
     get:
       parameters: []
@@ -13180,6 +13724,8 @@ tags:
   name: Monitor
 - description: NFS-Ganesha Cluster Management API
   name: NFS-Ganesha
+- description: NVMe-oF Gateway Management API
+  name: NVMe-oF
 - description: OSD management API
   name: OSD
 - description: OSD Perf Counters Management API
diff --git a/src/pybind/mgr/dashboard/requirements.txt b/src/pybind/mgr/dashboard/requirements.txt
index 292971819c9c..5643b55f9647 100644
--- a/src/pybind/mgr/dashboard/requirements.txt
+++ b/src/pybind/mgr/dashboard/requirements.txt
@@ -11,3 +11,5 @@ pyyaml
 natsort
 setuptools
 jsonpatch
+grpcio==1.46.5
+grpcio-tools==1.46.5
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_cli.py b/src/pybind/mgr/dashboard/services/nvmeof_cli.py
index 5921ab48ea8b..75a121a48d6e 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_cli.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_cli.py
@@ -5,8 +5,9 @@
 from mgr_module import CLICheckNonemptyFileInput, CLIReadCommand, CLIWriteCommand
 
 from ..rest_client import RequestException
-from .nvmeof_conf import NvmeofGatewaysConfig, NvmeofGatewayAlreadyExists, \
-    ManagedByOrchestratorException
+from .nvmeof_conf import ManagedByOrchestratorException, \
+    NvmeofGatewayAlreadyExists, NvmeofGatewaysConfig
+
 
 @CLIReadCommand('dashboard nvmeof-gateway-list')
 def list_nvmeof_gateways(_):
@@ -15,6 +16,7 @@ def list_nvmeof_gateways(_):
     '''
     return 0, json.dumps(NvmeofGatewaysConfig.get_gateways_config()), ''
 
+
 @CLIWriteCommand('dashboard nvmeof-gateway-add')
 @CLICheckNonemptyFileInput(desc='NVMe-oF gateway configuration')
 def add_nvmeof_gateway(_, inbuf, name: str):
@@ -32,6 +34,7 @@ def add_nvmeof_gateway(_, inbuf, name: str):
     except RequestException as ex:
         return -errno.EINVAL, '', str(ex)
 
+
 @CLIWriteCommand('dashboard nvmeof-gateway-rm')
 def remove_nvmeof_gateway(_, name: str):
     '''
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index ff7c6dfc8ced..bebaef7d9ecb 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -1,128 +1,152 @@
-from enum import Enum
+import logging
 from typing import Optional
-import grpc
-import json
 
-import logging
+from ..tools import str_to_bool
+from .nvmeof_conf import NvmeofGatewaysConfig
 
-from .proto import gateway_pb2 as pb2
-from .proto import gateway_pb2_grpc as pb2_grpc
+logger = logging.getLogger('nvmeof_client')
 
-from google.protobuf.json_format import MessageToJson
+try:
+    import grpc
 
-from .nvmeof_conf import NvmeofGatewaysConfig
-from ..tools import str_to_bool
+    from .proto import gateway_pb2 as pb2
+    from .proto import gateway_pb2_grpc as pb2_grpc
+except ImportError:
+    grpc = None
+else:
+    class NVMeoFClient(object):
+        def __init__(self):
+            logger.info('Initiating nvmeof gateway connection...')
 
-logger = logging.getLogger('nvmeof_client')
+            self.gateway_addr = list(NvmeofGatewaysConfig.get_gateways_config()[
+                                     'gateways'].values())[0]['service_url']
+            self.channel = grpc.insecure_channel(
+                '{}'.format(self.gateway_addr)
+            )
+            logger.info('Found nvmeof gateway at %s', self.gateway_addr)
+            self.stub = pb2_grpc.GatewayStub(self.channel)
+
+        def list_subsystems(self, subsystem_nqn: Optional[str] = None,
+                            serial_number: Optional[str] = None):
+            return self.stub.list_subsystems(pb2.list_subsystems_req(
+                subsystem_nqn=subsystem_nqn,
+                serial_number=serial_number
+            ))
+
+        def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
+                             ana_reporting: bool, enable_ha: bool):
+            return self.stub.create_subsystem(pb2.create_subsystem_req(
+                subsystem_nqn=subsystem_nqn,
+                serial_number=serial_number,
+                max_namespaces=int(max_namespaces),
+                ana_reporting=str_to_bool(ana_reporting),
+                enable_ha=str_to_bool(enable_ha)
+            ))
+
+        def delete_subsystem(self, subsystem_nqn: str):
+            return self.stub.delete_subsystem(pb2.delete_subsystem_req(
+                subsystem_nqn=subsystem_nqn
+            ))
+
+        def list_namespaces(self, subsystem_nqn: str, nsid: Optional[int] = 1,
+                            uuid: Optional[str] = None):
+            return self.stub.list_namespaces(pb2.list_namespaces_req(
+                subsystem=subsystem_nqn,
+                nsid=int(nsid),
+                uuid=uuid
+            ))
 
+        def create_namespace(self, rbd_pool_name: str, rbd_image_name: str,
+                             subsystem_nqn: str, block_size: int = 512,
+                             nsid: Optional[int] = 1, uuid: Optional[str] = None,
+                             anagrpid: Optional[int] = 1, create_image: Optional[bool] = True,
+                             size: Optional[int] = 1024):
+            return self.stub.namespace_add(pb2.namespace_add_req(
+                rbd_pool_name=rbd_pool_name,
+                rbd_image_name=rbd_image_name,
+                subsystem_nqn=subsystem_nqn,
+                nsid=int(nsid),
+                block_size=block_size,
+                uuid=uuid,
+                anagrpid=anagrpid,
+                create_image=create_image,
+                size=size
+            ))
 
-class NVMeoFClient(object):
-    def __init__(self):
-        logger.info('Initiating nvmeof gateway connection...')
-
-        self.gateway_addr = list(NvmeofGatewaysConfig.get_gateways_config()['gateways'].values())[0]['service_url']
-        self.channel = grpc.insecure_channel(
-            '{}'.format(self.gateway_addr)
-        )
-        logger.info('Found nvmeof gateway at {}'.format(self.gateway_addr))
-        self.stub = pb2_grpc.GatewayStub(self.channel)
-
-    def get_subsystems(self):
-        response = self.stub.get_subsystems(pb2.get_subsystems_req())
-        return json.loads(MessageToJson(response))
-    
-    def create_bdev(self, name: str, rbd_pool: str, rbd_image: str, block_size: int, uuid: Optional[str] = None):
-        response = self.stub.create_bdev(pb2.create_bdev_req(
-            bdev_name=name,
-            rbd_pool_name=rbd_pool,
-            rbd_image_name=rbd_image,
-            block_size=block_size,
-            uuid=uuid
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def resize_bdev(self, name: str, size: int):
-        response = self.stub.resize_bdev(pb2.resize_bdev_req(
-            bdev_name=name,
-            new_size=size
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def delete_bdev(self, name: str, force: bool):
-        response = self.stub.delete_bdev(pb2.delete_bdev_req(
-            bdev_name=name,
-            force=str_to_bool(force)
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
-                         ana_reporting: bool, enable_ha: bool) :
-        response = self.stub.create_subsystem(pb2.create_subsystem_req(
-            subsystem_nqn=subsystem_nqn,
-            serial_number=serial_number,
-            max_namespaces=int(max_namespaces),
-            ana_reporting=str_to_bool(ana_reporting),
-            enable_ha=str_to_bool(enable_ha)
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def delete_subsystem(self, subsystem_nqn: str):
-        response = self.stub.delete_subsystem(pb2.delete_subsystem_req(
-            subsystem_nqn=subsystem_nqn
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def create_namespace(self, subsystem_nqn: str, bdev_name: str, nsid: int, anagrpid: Optional[str] = None):
-        response = self.stub.add_namespace(pb2.add_namespace_req(
-            subsystem_nqn=subsystem_nqn,
-            bdev_name=bdev_name,
-            nsid=int(nsid),
-            anagrpid=anagrpid
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def delete_namespace(self, subsystem_nqn: str, nsid: int):
-        response = self.stub.remove_namespace(pb2.remove_namespace_req(
-            subsystem_nqn=subsystem_nqn,
-            nsid=nsid
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def add_host(self, subsystem_nqn: str, host_nqn: str):
-        response = self.stub.add_host(pb2.add_host_req(
-            subsystem_nqn=subsystem_nqn,
-            host_nqn=host_nqn
-        ))
-        return json.loads(MessageToJson(response))
-    
-    def remove_host(self, subsystem_nqn: str, host_nqn: str):
-        response = self.stub.remove_host(pb2.remove_host_req(
-            subsystem_nqn=subsystem_nqn,
-            host_nqn=host_nqn
-        ))
-        return json.loads(MessageToJson(response))
-
-    def create_listener(self, nqn: str, gateway: str, trtype: str, adrfam: str,
-                        traddr: str, trsvcid: str):
-        req = pb2.create_listener_req(
+        def delete_namespace(self, subsystem_nqn: str, nsid: int):
+            return self.stub.remove_namespace(pb2.remove_namespace_req(
+                subsystem_nqn=subsystem_nqn,
+                nsid=nsid
+            ))
+
+        def list_hosts(self, subsystem_nqn: str):
+            return self.stub.list_hosts(pb2.list_hosts_req(
+                subsystem=subsystem_nqn
+            ))
+
+        def add_host(self, subsystem_nqn: str, host_nqn: str):
+            return self.stub.add_host(pb2.add_host_req(
+                subsystem_nqn=subsystem_nqn,
+                host_nqn=host_nqn
+            ))
+
+        def remove_host(self, subsystem_nqn: str, host_nqn: str):
+            return self.stub.remove_host(pb2.remove_host_req(
+                subsystem_nqn=subsystem_nqn,
+                host_nqn=host_nqn
+            ))
+
+        def list_listeners(self, subsystem_nqn: str):
+            return self.stub.list_listeners(pb2.list_listeners_req(
+                subsystem=subsystem_nqn
+            ))
+
+        def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
+                            transport_type: Optional[str] = 'TCP',
+                            addr_family: Optional[str] = 'IPV4',
+                            transport_svc_id: Optional[int] = 4420,
+                            auto_ha_state: Optional[str] = 'AUTO_HA_UNSET'):
+            traddr = None
+            if traddr is None:
+                addr = self.gateway_addr
+                ip_address, _ = addr.split(':')
+                traddr = self._escape_address_if_ipv6(ip_address)
+
+            req = pb2.create_listener_req(
                 nqn=nqn,
                 gateway_name=gateway,
-                trtype=pb2.TransportType.Value(trtype.upper()),
-                adrfam=pb2.AddressFamily.Value(adrfam.lower()),
                 traddr=traddr,
-                trsvcid=trsvcid,
+                trtype=pb2.TransportType.Value(transport_type.upper()),
+                adrfam=pb2.AddressFamily.Value(addr_family.lower()),
+                trsvcid=transport_svc_id,
+                auto_ha_state=pb2.AutoHAState.Value(auto_ha_state.upper())
             )
-        ret = self.stub.create_listener(req)
-        return json.loads(MessageToJson(ret))
-    
-    def delete_listener(self, nqn: str, gateway: str, trttype, adrfam,
-                        traddr: str, trsvcid: str):
-        response = self.stub.delete_listener(pb2.delete_listener_req(
-            nqn=nqn,
-            gateway_name=gateway,
-            trtype=trttype,
-            adrfam=adrfam,
-            traddr=traddr,
-            trsvcid=trsvcid
-        ))
-        return json.loads(MessageToJson(response))
+            return self.stub.create_listener(req)
+
+        def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
+                            transport_type: Optional[str] = 'TCP',
+                            addr_family: Optional[str] = 'IPV4',
+                            transport_svc_id: Optional[int] = 4420):
+            traddr = None
+            if traddr is None:
+                addr = self.gateway_addr
+                ip_address, _ = addr.split(':')
+                traddr = self._escape_address_if_ipv6(ip_address)
+
+            return self.stub.delete_listener(pb2.delete_listener_req(
+                nqn=nqn,
+                gateway_name=gateway,
+                traddr=traddr,
+                trtype=pb2.TransportType.Value(transport_type.upper()),
+                adrfam=pb2.AddressFamily.Value(addr_family.lower()),
+                trsvcid=int(transport_svc_id)
+            ))
+
+        def gateway_info(self):
+            return self.stub.get_gateway_info(pb2.get_gateway_info_req())
+
+        def _escape_address_if_ipv6(self, addr):
+            ret_addr = addr
+            if ":" in addr and not addr.strip().startswith("["):
+                ret_addr = f"[{addr}]"
+            return ret_addr
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_conf.py b/src/pybind/mgr/dashboard/services/nvmeof_conf.py
index 709635dc7db9..901098ea5665 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_conf.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_conf.py
@@ -4,23 +4,28 @@
 
 from .. import mgr
 
+
 class NvmeofGatewayAlreadyExists(Exception):
     def __init__(self, gateway_name):
         super(NvmeofGatewayAlreadyExists, self).__init__(
             "NVMe-oF gateway '{}' already exists".format(gateway_name))
 
+
 class NvmeofGatewayDoesNotExist(Exception):
     def __init__(self, hostname):
         super(NvmeofGatewayDoesNotExist, self).__init__(
             "NVMe-oF gateway '{}' does not exist".format(hostname))
 
+
 class ManagedByOrchestratorException(Exception):
     def __init__(self):
         super(ManagedByOrchestratorException, self).__init__(
             "NVMe-oF configuration is managed by the orchestrator")
 
+
 _NVMEOF_STORE_KEY = "_nvmeof_config"
 
+
 class NvmeofGatewaysConfig(object):
     @classmethod
     def _load_config_from_store(cls):
@@ -33,11 +38,11 @@ def _load_config_from_store(cls):
     @classmethod
     def _save_config(cls, config):
         mgr.set_store(_NVMEOF_STORE_KEY, json.dumps(config))
-    
+
     @classmethod
     def get_gateways_config(cls):
         return cls._load_config_from_store()
-    
+
     @classmethod
     def add_gateway(cls, name, service_url):
         config = cls.get_gateways_config()
@@ -45,7 +50,7 @@ def add_gateway(cls, name, service_url):
             raise NvmeofGatewayAlreadyExists(name)
         config['gateways'][name] = {'service_url': service_url}
         cls._save_config(config)
-    
+
     @classmethod
     def remove_gateway(cls, name):
         config = cls.get_gateways_config()
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway.proto b/src/pybind/mgr/dashboard/services/proto/gateway.proto
index 85238c8210a3..787aefb1ce9f 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway.proto
+++ b/src/pybind/mgr/dashboard/services/proto/gateway.proto
@@ -31,21 +31,21 @@ enum AddressFamily {
 enum LogLevel {
     DISABLED = 0;
     ERROR = 1;
-    WARN = 2;
+    WARNING = 2;
     NOTICE = 3;
     INFO = 4;
     DEBUG = 5;
 }
 
-service Gateway {
-	// Creates a bdev from an RBD image
-	rpc create_bdev(create_bdev_req) returns (bdev) {}
-
-	// Resizes a bdev
-	rpc resize_bdev(resize_bdev_req) returns (req_status) {}
+enum AutoHAState {
+    AUTO_HA_UNSET = 0;
+    AUTO_HA_OFF = 1;
+    AUTO_HA_ON = 2;
+}
 
-	// Deletes a bdev
-	rpc delete_bdev(delete_bdev_req) returns (req_status) {}
+service Gateway {
+	// Creates a namespace from an RBD image
+	rpc namespace_add(namespace_add_req) returns (nsid_status) {}
 
 	// Creates a subsystem
 	rpc create_subsystem(create_subsystem_req) returns(req_status) {}
@@ -53,11 +53,23 @@ service Gateway {
 	// Deletes a subsystem
 	rpc delete_subsystem(delete_subsystem_req) returns(req_status) {}
 
-	// Adds a namespace to a subsystem
-	rpc add_namespace(add_namespace_req) returns(nsid_status) {}
+	// List namespaces
+	rpc list_namespaces(list_namespaces_req) returns(namespaces_info) {}
+
+	// Resizes a namespace
+	rpc namespace_resize(namespace_resize_req) returns (req_status) {}
+
+	// Gets namespace's IO stats
+	rpc namespace_get_io_stats(namespace_get_io_stats_req) returns (namespace_io_stats_info) {}
+
+	// Sets namespace's qos limits
+	rpc namespace_set_qos_limits(namespace_set_qos_req) returns (req_status) {}
+
+	// Changes namespace's load balancing group
+	rpc namespace_change_load_balancing_group(namespace_change_load_balancing_group_req) returns (req_status) {}
 
-	// Removes a namespace from a subsystem
-	rpc remove_namespace(remove_namespace_req) returns(req_status) {}
+	// Deletes a namespace
+	rpc namespace_delete(namespace_delete_req) returns (req_status) {}
 
 	// Adds a host to a subsystem
 	rpc add_host(add_host_req) returns (req_status) {}
@@ -65,70 +77,104 @@ service Gateway {
 	// Removes a host from a subsystem
 	rpc remove_host(remove_host_req) returns (req_status) {}
 
+	// List hosts
+	rpc list_hosts(list_hosts_req) returns(hosts_info) {}
+
+	// List connections
+	rpc list_connections(list_connections_req) returns(connections_info) {}
+
 	// Creates a listener for a subsystem at a given IP/Port
 	rpc create_listener(create_listener_req) returns(req_status) {}
 
 	// Deletes a listener from a subsystem at a given IP/Port
 	rpc delete_listener(delete_listener_req) returns(req_status) {}
 
-	// Gets subsystems
-	rpc get_subsystems(get_subsystems_req) returns(subsystems_info) {}
+	// List listeners
+	rpc list_listeners(list_listeners_req) returns(listeners_info) {}
+
+	// List subsystems
+	rpc list_subsystems(list_subsystems_req) returns(subsystems_info) {}
 
 	// Gets spdk nvmf log flags and level
 	rpc get_spdk_nvmf_log_flags_and_level(get_spdk_nvmf_log_flags_and_level_req) returns(spdk_nvmf_log_flags_and_level_info) {}
 
-	// Disables spdk nvmf logs
-	rpc disable_spdk_nvmf_logs(disable_spdk_nvmf_logs_req) returns(req_status) {}
+        // Disables spdk nvmf logs
+        rpc disable_spdk_nvmf_logs(disable_spdk_nvmf_logs_req) returns(req_status) {}
 
 	// Set spdk nvmf logs
 	rpc set_spdk_nvmf_logs(set_spdk_nvmf_logs_req) returns(req_status) {}
 
-	// Set spdk nvmf logs
+	// Get gateway info
 	rpc get_gateway_info(get_gateway_info_req) returns(gateway_info) {}
 }
 
 // Request messages
 
-message create_bdev_req {
-	string bdev_name = 1;
-	string rbd_pool_name = 2;
-	string rbd_image_name = 3;
-	int32 block_size = 4;
-	optional string uuid = 5;
+message namespace_add_req {
+	string rbd_pool_name = 1;
+	string rbd_image_name = 2;
+	string subsystem_nqn = 3;
+	optional uint32 nsid = 4;
+	uint32 block_size = 5;
+	optional string uuid = 6;
+	optional int32 anagrpid = 7;
+	optional bool create_image = 8;
+	optional uint32 size = 9;
 }
 
-message resize_bdev_req {
-	string bdev_name = 1;
-	int32 new_size = 2;
+message namespace_resize_req {
+	string subsystem_nqn = 1;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
+	uint32 new_size = 4;
 }
 
-message delete_bdev_req {
-	string bdev_name = 1;
-	bool force = 2;
+message namespace_get_io_stats_req {
+	string subsystem_nqn = 1;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
 }
 
-message create_subsystem_req {
+message namespace_set_qos_req {
 	string subsystem_nqn = 1;
-	string serial_number = 2;
-	int32 max_namespaces = 3;
-	bool  ana_reporting  = 4;
-	bool  enable_ha      = 5;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
+	optional uint64 rw_ios_per_second = 4;
+	optional uint64 rw_mbytes_per_second = 5;
+	optional uint64 r_mbytes_per_second = 6;
+	optional uint64 w_mbytes_per_second = 7;
 }
 
-message delete_subsystem_req {
+message namespace_change_load_balancing_group_req {
 	string subsystem_nqn = 1;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
+	int32 anagrpid = 4;
 }
 
-message add_namespace_req {
+message namespace_delete_req {
 	string subsystem_nqn = 1;
-	string bdev_name = 2;
-	optional uint32 nsid = 3;
-	optional int32 anagrpid = 4;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
+}
+
+message create_subsystem_req {
+	string subsystem_nqn = 1;
+	string serial_number = 2;
+	optional uint32 max_namespaces = 3;
+	bool ana_reporting = 4;
+	bool enable_ha = 5;
 }
 
-message remove_namespace_req {
+message delete_subsystem_req {
 	string subsystem_nqn = 1;
-	uint32 nsid = 2;
+	optional bool force = 2;
+}
+
+message list_namespaces_req {
+	string subsystem = 1;
+	optional uint32 nsid = 2;
+	optional string uuid = 3;
 }
 
 message add_host_req {
@@ -141,25 +187,40 @@ message remove_host_req {
 	string host_nqn = 2;
 }
 
+message list_hosts_req {
+	string subsystem = 1;
+}
+
+message list_connections_req {
+	string subsystem = 1;
+}
+
 message create_listener_req {
 	string nqn = 1;
 	string gateway_name = 2;
-	TransportType trtype = 3;
-	AddressFamily adrfam = 4;
-	string traddr = 5;
-	string trsvcid = 6;
+	string traddr = 3;
+	optional TransportType trtype = 4;
+	optional AddressFamily adrfam = 5;
+	optional uint32 trsvcid = 6;
+	optional AutoHAState auto_ha_state = 7;
 }
 
 message delete_listener_req {
 	string nqn = 1;
 	string gateway_name = 2;
-	TransportType trtype = 3;
-	AddressFamily adrfam = 4;
-	string traddr = 5;
-	string trsvcid = 6;
+	string traddr = 3;
+	optional TransportType trtype = 4;
+	optional AddressFamily adrfam = 5;
+	optional uint32 trsvcid = 6;
+}
+
+message list_listeners_req {
+	string subsystem = 1;
 }
 
-message get_subsystems_req {
+message list_subsystems_req {
+	optional string subsystem_nqn = 1;
+	optional string serial_number = 2;
 }
 
 message get_spdk_nvmf_log_flags_and_level_req {
@@ -174,74 +235,175 @@ message set_spdk_nvmf_logs_req {
 }
 
 message get_gateway_info_req {
-	string cli_version = 1;
+	optional string cli_version = 1;
 }
 
 // Return messages 
 
-message bdev {
-	string bdev_name = 1;
-	bool status = 2;
+message bdev_status {
+	int32 status = 1;
+	string error_message = 2;
+	string bdev_name = 3;
 }
 
 message req_status {
-	bool status = 1;
+	int32 status = 1;
+	string error_message = 2;
 }
 
 message nsid_status {
-	uint32 nsid = 1;
-	bool status = 2;
+	int32 status = 1;
+	string error_message = 2;
+	uint32 nsid = 3;
 }
 
 message subsystems_info {
-	repeated subsystem subsystems = 1;
+	int32 status = 1;
+	string error_message = 2;
+	repeated subsystem subsystems = 3;
 }
 
 message subsystem {
 	string nqn = 1;
-	string subtype = 2;
-	repeated listen_address listen_addresses = 3;
-	repeated host hosts = 4;
-	bool allow_any_host = 5;
-	optional string serial_number = 6;
-	optional string model_number = 7;
-	optional uint32 max_namespaces = 8;
-	optional uint32 min_cntlid = 9;
-	optional uint32 max_cntlid = 10;
-	repeated namespace namespaces = 11;
+	bool enable_ha = 2;
+	string serial_number = 3;
+	string model_number = 4;
+	uint32 min_cntlid = 5;
+	uint32 max_cntlid = 6;
+	uint32 namespace_count = 7;
+	string subtype = 8;
 }
 
 message gateway_info {
 	string cli_version = 1;
-	string gateway_version = 2;
-	string gateway_name = 3;
-	string gateway_group = 4;
-	string gateway_addr = 5;
-	string gateway_port = 6;
-	bool status = 7;
+	string version = 2;
+	string name = 3;
+	string group = 4;
+	string addr = 5;
+	string port = 6;
+	bool bool_status = 7;
+	int32 status = 8;
+	string error_message = 9;
+	string spdk_version = 10;
+}
+
+message cli_version {
+	int32 status = 1;
+	string error_message = 2;
+	string version = 3;
+}
+
+message gw_version {
+	int32 status = 1;
+	string error_message = 2;
+	string version = 3;
 }
 
-message listen_address {
-	string transport = 1;
+message listener_info {
+	string gateway_name = 1;
 	TransportType trtype = 2;
 	AddressFamily adrfam = 3;
 	string traddr = 4;
-	string trsvcid = 5;
+	uint32 trsvcid = 5;
+}
+
+message listeners_info {
+	int32 status = 1;
+	string error_message = 2;
+	repeated listener_info listeners = 3;
 }
 
 message host {
     string nqn = 1;
 }
 
+message hosts_info {
+	int32 status = 1;
+	string error_message = 2;
+	bool allow_any_host = 3;
+	string subsystem_nqn = 4;
+	repeated host hosts = 5;
+}
+
+message connection {
+	string nqn = 1;
+	string traddr = 2;
+	uint32 trsvcid = 3;
+	TransportType trtype = 4;
+	AddressFamily adrfam = 5;
+	bool connected = 6;
+	int32 qpairs_count = 7;
+	int32 controller_id = 8;
+}
+
+message connections_info {
+	int32 status = 1;
+	string error_message = 2;
+	string subsystem_nqn = 3;
+	repeated connection connections = 4;
+}
+
 message namespace {
-    uint32 nsid = 1;
-    string name = 2;
-    optional string bdev_name = 3;
-    optional string nguid = 4;
-    optional string uuid = 5;
-    optional uint32 anagrpid = 6;
+	uint32 nsid = 1;
+	string bdev_name = 2;
+	string rbd_image_name = 3;
+	string rbd_pool_name = 4;
+	uint32 load_balancing_group = 5;
+	uint32 block_size = 6;
+	uint64 rbd_image_size = 7;
+	string uuid = 8;
+	uint64 rw_ios_per_second = 9;
+	uint64 rw_mbytes_per_second = 10;
+	uint64 r_mbytes_per_second = 11;
+	uint64 w_mbytes_per_second = 12;
+}
+
+message namespaces_info {
+	int32 status = 1;
+	string error_message = 2;
+	string subsystem_nqn = 3;
+	repeated namespace namespaces = 4;
+}
+
+message namespace_io_stats_info {
+	int32 status = 1;
+	string error_message = 2;
+	string subsystem_nqn = 3;
+	uint32 nsid = 4;
+	string uuid = 5;
+	string bdev_name = 6;
+	uint64 tick_rate = 7;
+	uint64 ticks = 8;
+	uint64 bytes_read = 9;
+	uint64 num_read_ops = 10;
+	uint64 bytes_written = 11;
+	uint64 num_write_ops = 12;
+	uint64 bytes_unmapped = 13;
+	uint64 num_unmap_ops = 14;
+	uint64 read_latency_ticks = 15;
+	uint64 max_read_latency_ticks = 16;
+	uint64 min_read_latency_ticks = 17;
+	uint64 write_latency_ticks = 18;
+	uint64 max_write_latency_ticks = 19;
+	uint64 min_write_latency_ticks = 20;
+	uint64 unmap_latency_ticks = 21;
+	uint64 max_unmap_latency_ticks = 22;
+	uint64 min_unmap_latency_ticks = 23;
+	uint64 copy_latency_ticks = 24;
+	uint64 max_copy_latency_ticks = 25;
+	uint64 min_copy_latency_ticks = 26;
+	repeated uint32 io_error = 27;
+}
+
+message spdk_log_flag_info {
+    string name = 1;
+    bool enabled = 2;
 }
 
 message spdk_nvmf_log_flags_and_level_info {
-	string flags_level =1;
+	int32 status = 1;
+	string error_message = 2;
+	repeated spdk_log_flag_info nvmf_log_flags = 3;
+	LogLevel log_level = 4;
+	LogLevel log_print_level = 5;
 }
\ No newline at end of file
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
index 2cd4d0c5fe14..00349b5256cd 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
@@ -2,27 +2,195 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: gateway.proto
 """Generated protocol buffer code."""
-from google.protobuf.internal import enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import message as _message
 from google.protobuf import reflection as _reflection
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import enum_type_wrapper
+
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
 
 
+DESCRIPTOR = _descriptor.FileDescriptor(
+    name='gateway.proto',
+    package='',
+    syntax='proto3',
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+    serialized_pb=b'\n\rgateway.proto\"\x91\x02\n\x11namespace_add_req\x12\x15\n\rrbd_pool_name\x18\x01 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x11\n\x04nsid\x18\x04 \x01(\rH\x00\x88\x01\x01\x12\x12\n\nblock_size\x18\x05 \x01(\r\x12\x11\n\x04uuid\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x07 \x01(\x05H\x02\x88\x01\x01\x12\x19\n\x0c\x63reate_image\x18\x08 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04size\x18\t \x01(\rH\x04\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x0f\n\r_create_imageB\x07\n\x05_size\"w\n\x14namespace_resize_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08new_size\x18\x04 \x01(\rB\x07\n\x05_nsidB\x07\n\x05_uuid\"k\n\x1anamespace_get_io_stats_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\xcc\x02\n\x15namespace_set_qos_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x1e\n\x11rw_ios_per_second\x18\x04 \x01(\x04H\x02\x88\x01\x01\x12!\n\x14rw_mbytes_per_second\x18\x05 \x01(\x04H\x03\x88\x01\x01\x12 \n\x13r_mbytes_per_second\x18\x06 \x01(\x04H\x04\x88\x01\x01\x12 \n\x13w_mbytes_per_second\x18\x07 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x14\n\x12_rw_ios_per_secondB\x17\n\x15_rw_mbytes_per_secondB\x16\n\x14_r_mbytes_per_secondB\x16\n\x14_w_mbytes_per_second\"\x8c\x01\n)namespace_change_load_balancing_group_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08\x61nagrpid\x18\x04 \x01(\x05\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"e\n\x14namespace_delete_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\x9e\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x1b\n\x0emax_namespaces\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x15\n\rana_reporting\x18\x04 \x01(\x08\x12\x11\n\tenable_ha\x18\x05 \x01(\x08\x42\x11\n\x0f_max_namespaces\"K\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x12\n\x05\x66orce\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x08\n\x06_force\"`\n\x13list_namespaces_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"#\n\x0elist_hosts_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\")\n\x14list_connections_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"\x86\x02\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportTypeH\x00\x88\x01\x01\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x01\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x02\x88\x01\x01\x12(\n\rauto_ha_state\x18\x07 \x01(\x0e\x32\x0c.AutoHAStateH\x03\x88\x01\x01\x42\t\n\x07_trtypeB\t\n\x07_adrfamB\n\n\x08_trsvcidB\x10\n\x0e_auto_ha_state\"\xca\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportTypeH\x00\x88\x01\x01\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x01\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x02\x88\x01\x01\x42\t\n\x07_trtypeB\t\n\x07_adrfamB\n\n\x08_trsvcid\"\'\n\x12list_listeners_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"q\n\x13list_subsystems_req\x12\x1a\n\rsubsystem_nqn\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rserial_number\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x10\n\x0e_subsystem_nqnB\x10\n\x0e_serial_number\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"@\n\x14get_gateway_info_req\x12\x18\n\x0b\x63li_version\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_cli_version\"G\n\x0b\x62\x64\x65v_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x11\n\tbdev_name\x18\x03 \x01(\t\"3\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\"B\n\x0bnsid_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0c\n\x04nsid\x18\x03 \x01(\r\"X\n\x0fsubsystems_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x1e\n\nsubsystems\x18\x03 \x03(\x0b\x32\n.subsystem\"\xaa\x01\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x11\n\tenable_ha\x18\x02 \x01(\x08\x12\x15\n\rserial_number\x18\x03 \x01(\t\x12\x14\n\x0cmodel_number\x18\x04 \x01(\t\x12\x12\n\nmin_cntlid\x18\x05 \x01(\r\x12\x12\n\nmax_cntlid\x18\x06 \x01(\r\x12\x17\n\x0fnamespace_count\x18\x07 \x01(\r\x12\x0f\n\x07subtype\x18\x08 \x01(\t\"\xbf\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05group\x18\x04 \x01(\t\x12\x0c\n\x04\x61\x64\x64r\x18\x05 \x01(\t\x12\x0c\n\x04port\x18\x06 \x01(\t\x12\x13\n\x0b\x62ool_status\x18\x07 \x01(\x08\x12\x0e\n\x06status\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\x12\x14\n\x0cspdk_version\x18\n \x01(\t\"E\n\x0b\x63li_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"D\n\ngw_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"\x86\x01\n\rlistener_info\x12\x14\n\x0cgateway_name\x18\x01 \x01(\t\x12\x1e\n\x06trtype\x18\x02 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\r\"Z\n\x0elisteners_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12!\n\tlisteners\x18\x03 \x03(\x0b\x32\x0e.listener_info\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"x\n\nhosts_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x16\n\x0e\x61llow_any_host\x18\x03 \x01(\x08\x12\x15\n\rsubsystem_nqn\x18\x04 \x01(\t\x12\x14\n\x05hosts\x18\x05 \x03(\x0b\x32\x05.host\"\xba\x01\n\nconnection\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0e\n\x06traddr\x18\x02 \x01(\t\x12\x0f\n\x07trsvcid\x18\x03 \x01(\r\x12\x1e\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamily\x12\x11\n\tconnected\x18\x06 \x01(\x08\x12\x14\n\x0cqpairs_count\x18\x07 \x01(\x05\x12\x15\n\rcontroller_id\x18\x08 \x01(\x05\"r\n\x10\x63onnections_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12 \n\x0b\x63onnections\x18\x04 \x03(\x0b\x32\x0b.connection\"\xa6\x02\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x04 \x01(\t\x12\x1c\n\x14load_balancing_group\x18\x05 \x01(\r\x12\x12\n\nblock_size\x18\x06 \x01(\r\x12\x16\n\x0erbd_image_size\x18\x07 \x01(\x04\x12\x0c\n\x04uuid\x18\x08 \x01(\t\x12\x19\n\x11rw_ios_per_second\x18\t \x01(\x04\x12\x1c\n\x14rw_mbytes_per_second\x18\n \x01(\x04\x12\x1b\n\x13r_mbytes_per_second\x18\x0b \x01(\x04\x12\x1b\n\x13w_mbytes_per_second\x18\x0c \x01(\x04\"o\n\x0fnamespaces_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x1e\n\nnamespaces\x18\x04 \x03(\x0b\x32\n.namespace\"\xb7\x05\n\x17namespace_io_stats_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x0c\n\x04nsid\x18\x04 \x01(\r\x12\x0c\n\x04uuid\x18\x05 \x01(\t\x12\x11\n\tbdev_name\x18\x06 \x01(\t\x12\x11\n\ttick_rate\x18\x07 \x01(\x04\x12\r\n\x05ticks\x18\x08 \x01(\x04\x12\x12\n\nbytes_read\x18\t \x01(\x04\x12\x14\n\x0cnum_read_ops\x18\n \x01(\x04\x12\x15\n\rbytes_written\x18\x0b \x01(\x04\x12\x15\n\rnum_write_ops\x18\x0c \x01(\x04\x12\x16\n\x0e\x62ytes_unmapped\x18\r \x01(\x04\x12\x15\n\rnum_unmap_ops\x18\x0e \x01(\x04\x12\x1a\n\x12read_latency_ticks\x18\x0f \x01(\x04\x12\x1e\n\x16max_read_latency_ticks\x18\x10 \x01(\x04\x12\x1e\n\x16min_read_latency_ticks\x18\x11 \x01(\x04\x12\x1b\n\x13write_latency_ticks\x18\x12 \x01(\x04\x12\x1f\n\x17max_write_latency_ticks\x18\x13 \x01(\x04\x12\x1f\n\x17min_write_latency_ticks\x18\x14 \x01(\x04\x12\x1b\n\x13unmap_latency_ticks\x18\x15 \x01(\x04\x12\x1f\n\x17max_unmap_latency_ticks\x18\x16 \x01(\x04\x12\x1f\n\x17min_unmap_latency_ticks\x18\x17 \x01(\x04\x12\x1a\n\x12\x63opy_latency_ticks\x18\x18 \x01(\x04\x12\x1e\n\x16max_copy_latency_ticks\x18\x19 \x01(\x04\x12\x1e\n\x16min_copy_latency_ticks\x18\x1a \x01(\x04\x12\x10\n\x08io_error\x18\x1b \x03(\r\"3\n\x12spdk_log_flag_info\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x02 \x01(\x08\"\xba\x01\n\"spdk_nvmf_log_flags_and_level_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12+\n\x0envmf_log_flags\x18\x03 \x03(\x0b\x32\x13.spdk_log_flag_info\x12\x1c\n\tlog_level\x18\x04 \x01(\x0e\x32\t.LogLevel\x12\"\n\x0flog_print_level\x18\x05 \x01(\x0e\x32\t.LogLevel*^\n\rTransportType\x12\x0b\n\x07INVALID\x10\x00\x12\x08\n\x04RDMA\x10\x01\x12\x06\n\x02\x46\x43\x10\x02\x12\x07\n\x03TCP\x10\x03\x12\t\n\x04PCIE\x10\x80\x02\x12\r\n\x08VFIOUSER\x10\x80\x08\x12\x0b\n\x06\x43USTOM\x10\x80 *@\n\rAddressFamily\x12\x0b\n\x07invalid\x10\x00\x12\x08\n\x04ipv4\x10\x01\x12\x08\n\x04ipv6\x10\x02\x12\x06\n\x02ib\x10\x03\x12\x06\n\x02\x66\x63\x10\x04*Q\n\x08LogLevel\x12\x0c\n\x08\x44ISABLED\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\n\n\x06NOTICE\x10\x03\x12\x08\n\x04INFO\x10\x04\x12\t\n\x05\x44\x45\x42UG\x10\x05*A\n\x0b\x41utoHAState\x12\x11\n\rAUTO_HA_UNSET\x10\x00\x12\x0f\n\x0b\x41UTO_HA_OFF\x10\x01\x12\x0e\n\nAUTO_HA_ON\x10\x02\x32\xc0\n\n\x07Gateway\x12\x33\n\rnamespace_add\x12\x12.namespace_add_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12;\n\x0flist_namespaces\x12\x14.list_namespaces_req\x1a\x10.namespaces_info\"\x00\x12\x38\n\x10namespace_resize\x12\x15.namespace_resize_req\x1a\x0b.req_status\"\x00\x12Q\n\x16namespace_get_io_stats\x12\x1b.namespace_get_io_stats_req\x1a\x18.namespace_io_stats_info\"\x00\x12\x41\n\x18namespace_set_qos_limits\x12\x16.namespace_set_qos_req\x1a\x0b.req_status\"\x00\x12\x62\n%namespace_change_load_balancing_group\x12*.namespace_change_load_balancing_group_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10namespace_delete\x12\x15.namespace_delete_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12,\n\nlist_hosts\x12\x0f.list_hosts_req\x1a\x0b.hosts_info\"\x00\x12>\n\x10list_connections\x12\x15.list_connections_req\x1a\x11.connections_info\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x38\n\x0elist_listeners\x12\x13.list_listeners_req\x1a\x0f.listeners_info\"\x00\x12;\n\x0flist_subsystems\x12\x14.list_subsystems_req\x1a\x10.subsystems_info\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3'
+)
 
+_TRANSPORTTYPE = _descriptor.EnumDescriptor(
+    name='TransportType',
+    full_name='TransportType',
+    filename=None,
+    file=DESCRIPTOR,
+    create_key=_descriptor._internal_create_key,
+    values=[
+        _descriptor.EnumValueDescriptor(
+            name='INVALID', index=0, number=0,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='RDMA', index=1, number=1,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='FC', index=2, number=2,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='TCP', index=3, number=3,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='PCIE', index=4, number=256,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='VFIOUSER', index=5, number=1024,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='CUSTOM', index=6, number=4096,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+    ],
+    containing_type=None,
+    serialized_options=None,
+    serialized_start=5347,
+    serialized_end=5441,
+)
+_sym_db.RegisterEnumDescriptor(_TRANSPORTTYPE)
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rgateway.proto\"\x83\x01\n\x0f\x63reate_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x12\n\nblock_size\x18\x04 \x01(\x05\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x00\x88\x01\x01\x42\x07\n\x05_uuid\"6\n\x0fresize_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x10\n\x08new_size\x18\x02 \x01(\x05\"3\n\x0f\x64\x65lete_bdev_req\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\r\n\x05\x66orce\x18\x02 \x01(\x08\"\x86\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x16\n\x0emax_namespaces\x18\x03 \x01(\x05\x12\x15\n\rana_reporting\x18\x04 \x01(\x08\x12\x11\n\tenable_ha\x18\x05 \x01(\x08\"-\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\"}\n\x11\x61\x64\x64_namespace_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x11\n\x04nsid\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x0b\n\t_anagrpid\";\n\x14remove_namespace_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x0c\n\x04nsid\x18\x02 \x01(\r\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"\x99\x01\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x1e\n\x06trtype\x18\x03 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x04 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x05 \x01(\t\x12\x0f\n\x07trsvcid\x18\x06 \x01(\t\"\x99\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x1e\n\x06trtype\x18\x03 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x04 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x05 \x01(\t\x12\x0f\n\x07trsvcid\x18\x06 \x01(\t\"\x14\n\x12get_subsystems_req\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"+\n\x14get_gateway_info_req\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\")\n\x04\x62\x64\x65v\x12\x11\n\tbdev_name\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x08\"\x1c\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x08\"+\n\x0bnsid_status\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0e\n\x06status\x18\x02 \x01(\x08\"1\n\x0fsubsystems_info\x12\x1e\n\nsubsystems\x18\x01 \x03(\x0b\x32\n.subsystem\"\xfc\x02\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0f\n\x07subtype\x18\x02 \x01(\t\x12)\n\x10listen_addresses\x18\x03 \x03(\x0b\x32\x0f.listen_address\x12\x14\n\x05hosts\x18\x04 \x03(\x0b\x32\x05.host\x12\x16\n\x0e\x61llow_any_host\x18\x05 \x01(\x08\x12\x1a\n\rserial_number\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0cmodel_number\x18\x07 \x01(\tH\x01\x88\x01\x01\x12\x1b\n\x0emax_namespaces\x18\x08 \x01(\rH\x02\x88\x01\x01\x12\x17\n\nmin_cntlid\x18\t \x01(\rH\x03\x88\x01\x01\x12\x17\n\nmax_cntlid\x18\n \x01(\rH\x04\x88\x01\x01\x12\x1e\n\nnamespaces\x18\x0b \x03(\x0b\x32\n.namespaceB\x10\n\x0e_serial_numberB\x0f\n\r_model_numberB\x11\n\x0f_max_namespacesB\r\n\x0b_min_cntlidB\r\n\x0b_max_cntlid\"\xa5\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x17\n\x0fgateway_version\x18\x02 \x01(\t\x12\x14\n\x0cgateway_name\x18\x03 \x01(\t\x12\x15\n\rgateway_group\x18\x04 \x01(\t\x12\x14\n\x0cgateway_addr\x18\x05 \x01(\t\x12\x14\n\x0cgateway_port\x18\x06 \x01(\t\x12\x0e\n\x06status\x18\x07 \x01(\x08\"\x84\x01\n\x0elisten_address\x12\x11\n\ttransport\x18\x01 \x01(\t\x12\x1e\n\x06trtype\x18\x02 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\t\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"\xab\x01\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\tbdev_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05nguid\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x06 \x01(\rH\x03\x88\x01\x01\x42\x0c\n\n_bdev_nameB\x08\n\x06_nguidB\x07\n\x05_uuidB\x0b\n\t_anagrpid\"9\n\"spdk_nvmf_log_flags_and_level_info\x12\x13\n\x0b\x66lags_level\x18\x01 \x01(\t*^\n\rTransportType\x12\x0b\n\x07INVALID\x10\x00\x12\x08\n\x04RDMA\x10\x01\x12\x06\n\x02\x46\x43\x10\x02\x12\x07\n\x03TCP\x10\x03\x12\t\n\x04PCIE\x10\x80\x02\x12\r\n\x08VFIOUSER\x10\x80\x08\x12\x0b\n\x06\x43USTOM\x10\x80 *@\n\rAddressFamily\x12\x0b\n\x07invalid\x10\x00\x12\x08\n\x04ipv4\x10\x01\x12\x08\n\x04ipv6\x10\x02\x12\x06\n\x02ib\x10\x03\x12\x06\n\x02\x66\x63\x10\x04*N\n\x08LogLevel\x12\x0c\n\x08\x44ISABLED\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x08\n\x04WARN\x10\x02\x12\n\n\x06NOTICE\x10\x03\x12\x08\n\x04INFO\x10\x04\x12\t\n\x05\x44\x45\x42UG\x10\x05\x32\xaf\x07\n\x07Gateway\x12(\n\x0b\x63reate_bdev\x12\x10.create_bdev_req\x1a\x05.bdev\"\x00\x12.\n\x0bresize_bdev\x12\x10.resize_bdev_req\x1a\x0b.req_status\"\x00\x12.\n\x0b\x64\x65lete_bdev\x12\x10.delete_bdev_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12\x33\n\radd_namespace\x12\x12.add_namespace_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10remove_namespace\x12\x15.remove_namespace_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x39\n\x0eget_subsystems\x12\x13.get_subsystems_req\x1a\x10.subsystems_info\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3')
-
-_TRANSPORTTYPE = DESCRIPTOR.enum_types_by_name['TransportType']
 TransportType = enum_type_wrapper.EnumTypeWrapper(_TRANSPORTTYPE)
-_ADDRESSFAMILY = DESCRIPTOR.enum_types_by_name['AddressFamily']
+_ADDRESSFAMILY = _descriptor.EnumDescriptor(
+    name='AddressFamily',
+    full_name='AddressFamily',
+    filename=None,
+    file=DESCRIPTOR,
+    create_key=_descriptor._internal_create_key,
+    values=[
+        _descriptor.EnumValueDescriptor(
+            name='invalid', index=0, number=0,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='ipv4', index=1, number=1,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='ipv6', index=2, number=2,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='ib', index=3, number=3,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='fc', index=4, number=4,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+    ],
+    containing_type=None,
+    serialized_options=None,
+    serialized_start=5443,
+    serialized_end=5507,
+)
+_sym_db.RegisterEnumDescriptor(_ADDRESSFAMILY)
+
 AddressFamily = enum_type_wrapper.EnumTypeWrapper(_ADDRESSFAMILY)
-_LOGLEVEL = DESCRIPTOR.enum_types_by_name['LogLevel']
+_LOGLEVEL = _descriptor.EnumDescriptor(
+    name='LogLevel',
+    full_name='LogLevel',
+    filename=None,
+    file=DESCRIPTOR,
+    create_key=_descriptor._internal_create_key,
+    values=[
+        _descriptor.EnumValueDescriptor(
+            name='DISABLED', index=0, number=0,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='ERROR', index=1, number=1,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='WARNING', index=2, number=2,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='NOTICE', index=3, number=3,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='INFO', index=4, number=4,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='DEBUG', index=5, number=5,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+    ],
+    containing_type=None,
+    serialized_options=None,
+    serialized_start=5509,
+    serialized_end=5590,
+)
+_sym_db.RegisterEnumDescriptor(_LOGLEVEL)
+
 LogLevel = enum_type_wrapper.EnumTypeWrapper(_LOGLEVEL)
+_AUTOHASTATE = _descriptor.EnumDescriptor(
+    name='AutoHAState',
+    full_name='AutoHAState',
+    filename=None,
+    file=DESCRIPTOR,
+    create_key=_descriptor._internal_create_key,
+    values=[
+        _descriptor.EnumValueDescriptor(
+            name='AUTO_HA_UNSET', index=0, number=0,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='AUTO_HA_OFF', index=1, number=1,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='AUTO_HA_ON', index=2, number=2,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+    ],
+    containing_type=None,
+    serialized_options=None,
+    serialized_start=5592,
+    serialized_end=5657,
+)
+_sym_db.RegisterEnumDescriptor(_AUTOHASTATE)
+
+AutoHAState = enum_type_wrapper.EnumTypeWrapper(_AUTOHASTATE)
 INVALID = 0
 RDMA = 1
 FC = 2
@@ -37,282 +205,3097 @@
 fc = 4
 DISABLED = 0
 ERROR = 1
-WARN = 2
+WARNING = 2
 NOTICE = 3
 INFO = 4
 DEBUG = 5
+AUTO_HA_UNSET = 0
+AUTO_HA_OFF = 1
+AUTO_HA_ON = 2
+
+
+_NAMESPACE_ADD_REQ = _descriptor.Descriptor(
+    name='namespace_add_req',
+    full_name='namespace_add_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='rbd_pool_name', full_name='namespace_add_req.rbd_pool_name', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rbd_image_name', full_name='namespace_add_req.rbd_image_name', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_add_req.subsystem_nqn', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_add_req.nsid', index=3,
+            number=4, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='block_size', full_name='namespace_add_req.block_size', index=4,
+            number=5, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_add_req.uuid', index=5,
+            number=6, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='anagrpid', full_name='namespace_add_req.anagrpid', index=6,
+            number=7, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='create_image', full_name='namespace_add_req.create_image', index=7,
+            number=8, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='size', full_name='namespace_add_req.size', index=8,
+            number=9, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_add_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_add_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_anagrpid', full_name='namespace_add_req._anagrpid',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_create_image', full_name='namespace_add_req._create_image',
+            index=3, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_size', full_name='namespace_add_req._size',
+            index=4, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=18,
+    serialized_end=291,
+)
+
+
+_NAMESPACE_RESIZE_REQ = _descriptor.Descriptor(
+    name='namespace_resize_req',
+    full_name='namespace_resize_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_resize_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_resize_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_resize_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='new_size', full_name='namespace_resize_req.new_size', index=3,
+            number=4, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_resize_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_resize_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=293,
+    serialized_end=412,
+)
+
+
+_NAMESPACE_GET_IO_STATS_REQ = _descriptor.Descriptor(
+    name='namespace_get_io_stats_req',
+    full_name='namespace_get_io_stats_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_get_io_stats_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_get_io_stats_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_get_io_stats_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_get_io_stats_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_get_io_stats_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=414,
+    serialized_end=521,
+)
+
+
+_NAMESPACE_SET_QOS_REQ = _descriptor.Descriptor(
+    name='namespace_set_qos_req',
+    full_name='namespace_set_qos_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_set_qos_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_set_qos_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_set_qos_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rw_ios_per_second', full_name='namespace_set_qos_req.rw_ios_per_second', index=3,
+            number=4, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rw_mbytes_per_second', full_name='namespace_set_qos_req.rw_mbytes_per_second', index=4,
+            number=5, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='r_mbytes_per_second', full_name='namespace_set_qos_req.r_mbytes_per_second', index=5,
+            number=6, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='w_mbytes_per_second', full_name='namespace_set_qos_req.w_mbytes_per_second', index=6,
+            number=7, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_set_qos_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_set_qos_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_rw_ios_per_second', full_name='namespace_set_qos_req._rw_ios_per_second',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_rw_mbytes_per_second', full_name='namespace_set_qos_req._rw_mbytes_per_second',
+            index=3, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_r_mbytes_per_second', full_name='namespace_set_qos_req._r_mbytes_per_second',
+            index=4, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_w_mbytes_per_second', full_name='namespace_set_qos_req._w_mbytes_per_second',
+            index=5, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=524,
+    serialized_end=856,
+)
+
+
+_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ = _descriptor.Descriptor(
+    name='namespace_change_load_balancing_group_req',
+    full_name='namespace_change_load_balancing_group_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_change_load_balancing_group_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_change_load_balancing_group_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_change_load_balancing_group_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='anagrpid', full_name='namespace_change_load_balancing_group_req.anagrpid', index=3,
+            number=4, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_change_load_balancing_group_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_change_load_balancing_group_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=859,
+    serialized_end=999,
+)
+
+
+_NAMESPACE_DELETE_REQ = _descriptor.Descriptor(
+    name='namespace_delete_req',
+    full_name='namespace_delete_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_delete_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_delete_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_delete_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='namespace_delete_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace_delete_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1001,
+    serialized_end=1102,
+)
+
+
+_CREATE_SUBSYSTEM_REQ = _descriptor.Descriptor(
+    name='create_subsystem_req',
+    full_name='create_subsystem_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='create_subsystem_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='serial_number', full_name='create_subsystem_req.serial_number', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_namespaces', full_name='create_subsystem_req.max_namespaces', index=2,
+            number=3, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='ana_reporting', full_name='create_subsystem_req.ana_reporting', index=3,
+            number=4, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='enable_ha', full_name='create_subsystem_req.enable_ha', index=4,
+            number=5, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_max_namespaces', full_name='create_subsystem_req._max_namespaces',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1105,
+    serialized_end=1263,
+)
+
+
+_DELETE_SUBSYSTEM_REQ = _descriptor.Descriptor(
+    name='delete_subsystem_req',
+    full_name='delete_subsystem_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='delete_subsystem_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='force', full_name='delete_subsystem_req.force', index=1,
+            number=2, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_force', full_name='delete_subsystem_req._force',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1265,
+    serialized_end=1340,
+)
+
+
+_LIST_NAMESPACES_REQ = _descriptor.Descriptor(
+    name='list_namespaces_req',
+    full_name='list_namespaces_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem', full_name='list_namespaces_req.subsystem', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='list_namespaces_req.nsid', index=1,
+            number=2, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='list_namespaces_req.uuid', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_nsid', full_name='list_namespaces_req._nsid',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='list_namespaces_req._uuid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1342,
+    serialized_end=1438,
+)
+
+
+_ADD_HOST_REQ = _descriptor.Descriptor(
+    name='add_host_req',
+    full_name='add_host_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='add_host_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='host_nqn', full_name='add_host_req.host_nqn', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=1440,
+    serialized_end=1495,
+)
+
+
+_REMOVE_HOST_REQ = _descriptor.Descriptor(
+    name='remove_host_req',
+    full_name='remove_host_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='remove_host_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='host_nqn', full_name='remove_host_req.host_nqn', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=1497,
+    serialized_end=1555,
+)
+
+
+_LIST_HOSTS_REQ = _descriptor.Descriptor(
+    name='list_hosts_req',
+    full_name='list_hosts_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem', full_name='list_hosts_req.subsystem', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=1557,
+    serialized_end=1592,
+)
+
+
+_LIST_CONNECTIONS_REQ = _descriptor.Descriptor(
+    name='list_connections_req',
+    full_name='list_connections_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem', full_name='list_connections_req.subsystem', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=1594,
+    serialized_end=1635,
+)
+
+
+_CREATE_LISTENER_REQ = _descriptor.Descriptor(
+    name='create_listener_req',
+    full_name='create_listener_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='create_listener_req.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='gateway_name', full_name='create_listener_req.gateway_name', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='traddr', full_name='create_listener_req.traddr', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trtype', full_name='create_listener_req.trtype', index=3,
+            number=4, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='adrfam', full_name='create_listener_req.adrfam', index=4,
+            number=5, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trsvcid', full_name='create_listener_req.trsvcid', index=5,
+            number=6, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='auto_ha_state', full_name='create_listener_req.auto_ha_state', index=6,
+            number=7, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_trtype', full_name='create_listener_req._trtype',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_adrfam', full_name='create_listener_req._adrfam',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_trsvcid', full_name='create_listener_req._trsvcid',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_auto_ha_state', full_name='create_listener_req._auto_ha_state',
+            index=3, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1638,
+    serialized_end=1900,
+)
+
+
+_DELETE_LISTENER_REQ = _descriptor.Descriptor(
+    name='delete_listener_req',
+    full_name='delete_listener_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='delete_listener_req.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='gateway_name', full_name='delete_listener_req.gateway_name', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='traddr', full_name='delete_listener_req.traddr', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trtype', full_name='delete_listener_req.trtype', index=3,
+            number=4, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='adrfam', full_name='delete_listener_req.adrfam', index=4,
+            number=5, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trsvcid', full_name='delete_listener_req.trsvcid', index=5,
+            number=6, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_trtype', full_name='delete_listener_req._trtype',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_adrfam', full_name='delete_listener_req._adrfam',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_trsvcid', full_name='delete_listener_req._trsvcid',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=1903,
+    serialized_end=2105,
+)
+
+
+_LIST_LISTENERS_REQ = _descriptor.Descriptor(
+    name='list_listeners_req',
+    full_name='list_listeners_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem', full_name='list_listeners_req.subsystem', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2107,
+    serialized_end=2146,
+)
+
+
+_LIST_SUBSYSTEMS_REQ = _descriptor.Descriptor(
+    name='list_subsystems_req',
+    full_name='list_subsystems_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='list_subsystems_req.subsystem_nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='serial_number', full_name='list_subsystems_req.serial_number', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_subsystem_nqn', full_name='list_subsystems_req._subsystem_nqn',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_serial_number', full_name='list_subsystems_req._serial_number',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=2148,
+    serialized_end=2261,
+)
+
+
+_GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ = _descriptor.Descriptor(
+    name='get_spdk_nvmf_log_flags_and_level_req',
+    full_name='get_spdk_nvmf_log_flags_and_level_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2263,
+    serialized_end=2302,
+)
+
+
+_DISABLE_SPDK_NVMF_LOGS_REQ = _descriptor.Descriptor(
+    name='disable_spdk_nvmf_logs_req',
+    full_name='disable_spdk_nvmf_logs_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2304,
+    serialized_end=2332,
+)
+
+
+_SET_SPDK_NVMF_LOGS_REQ = _descriptor.Descriptor(
+    name='set_spdk_nvmf_logs_req',
+    full_name='set_spdk_nvmf_logs_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='log_level', full_name='set_spdk_nvmf_logs_req.log_level', index=0,
+            number=1, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='print_level', full_name='set_spdk_nvmf_logs_req.print_level', index=1,
+            number=2, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_log_level', full_name='set_spdk_nvmf_logs_req._log_level',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_print_level', full_name='set_spdk_nvmf_logs_req._print_level',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=2334,
+    serialized_end=2460,
+)
+
+
+_GET_GATEWAY_INFO_REQ = _descriptor.Descriptor(
+    name='get_gateway_info_req',
+    full_name='get_gateway_info_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='cli_version', full_name='get_gateway_info_req.cli_version', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_cli_version', full_name='get_gateway_info_req._cli_version',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=2462,
+    serialized_end=2526,
+)
+
+
+_BDEV_STATUS = _descriptor.Descriptor(
+    name='bdev_status',
+    full_name='bdev_status',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='bdev_status.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='bdev_status.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bdev_name', full_name='bdev_status.bdev_name', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2528,
+    serialized_end=2599,
+)
+
+
+_REQ_STATUS = _descriptor.Descriptor(
+    name='req_status',
+    full_name='req_status',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='req_status.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='req_status.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2601,
+    serialized_end=2652,
+)
+
 
+_NSID_STATUS = _descriptor.Descriptor(
+    name='nsid_status',
+    full_name='nsid_status',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='nsid_status.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='nsid_status.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='nsid_status.nsid', index=2,
+            number=3, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2654,
+    serialized_end=2720,
+)
 
-_CREATE_BDEV_REQ = DESCRIPTOR.message_types_by_name['create_bdev_req']
-_RESIZE_BDEV_REQ = DESCRIPTOR.message_types_by_name['resize_bdev_req']
-_DELETE_BDEV_REQ = DESCRIPTOR.message_types_by_name['delete_bdev_req']
-_CREATE_SUBSYSTEM_REQ = DESCRIPTOR.message_types_by_name['create_subsystem_req']
-_DELETE_SUBSYSTEM_REQ = DESCRIPTOR.message_types_by_name['delete_subsystem_req']
-_ADD_NAMESPACE_REQ = DESCRIPTOR.message_types_by_name['add_namespace_req']
-_REMOVE_NAMESPACE_REQ = DESCRIPTOR.message_types_by_name['remove_namespace_req']
-_ADD_HOST_REQ = DESCRIPTOR.message_types_by_name['add_host_req']
-_REMOVE_HOST_REQ = DESCRIPTOR.message_types_by_name['remove_host_req']
-_CREATE_LISTENER_REQ = DESCRIPTOR.message_types_by_name['create_listener_req']
-_DELETE_LISTENER_REQ = DESCRIPTOR.message_types_by_name['delete_listener_req']
-_GET_SUBSYSTEMS_REQ = DESCRIPTOR.message_types_by_name['get_subsystems_req']
-_GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ = DESCRIPTOR.message_types_by_name['get_spdk_nvmf_log_flags_and_level_req']
-_DISABLE_SPDK_NVMF_LOGS_REQ = DESCRIPTOR.message_types_by_name['disable_spdk_nvmf_logs_req']
-_SET_SPDK_NVMF_LOGS_REQ = DESCRIPTOR.message_types_by_name['set_spdk_nvmf_logs_req']
-_GET_GATEWAY_INFO_REQ = DESCRIPTOR.message_types_by_name['get_gateway_info_req']
-_BDEV = DESCRIPTOR.message_types_by_name['bdev']
-_REQ_STATUS = DESCRIPTOR.message_types_by_name['req_status']
-_NSID_STATUS = DESCRIPTOR.message_types_by_name['nsid_status']
-_SUBSYSTEMS_INFO = DESCRIPTOR.message_types_by_name['subsystems_info']
-_SUBSYSTEM = DESCRIPTOR.message_types_by_name['subsystem']
-_GATEWAY_INFO = DESCRIPTOR.message_types_by_name['gateway_info']
-_LISTEN_ADDRESS = DESCRIPTOR.message_types_by_name['listen_address']
-_HOST = DESCRIPTOR.message_types_by_name['host']
-_NAMESPACE = DESCRIPTOR.message_types_by_name['namespace']
-_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO = DESCRIPTOR.message_types_by_name['spdk_nvmf_log_flags_and_level_info']
-create_bdev_req = _reflection.GeneratedProtocolMessageType('create_bdev_req', (_message.Message,), {
-  'DESCRIPTOR' : _CREATE_BDEV_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:create_bdev_req)
-  })
-_sym_db.RegisterMessage(create_bdev_req)
-
-resize_bdev_req = _reflection.GeneratedProtocolMessageType('resize_bdev_req', (_message.Message,), {
-  'DESCRIPTOR' : _RESIZE_BDEV_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:resize_bdev_req)
-  })
-_sym_db.RegisterMessage(resize_bdev_req)
-
-delete_bdev_req = _reflection.GeneratedProtocolMessageType('delete_bdev_req', (_message.Message,), {
-  'DESCRIPTOR' : _DELETE_BDEV_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:delete_bdev_req)
-  })
-_sym_db.RegisterMessage(delete_bdev_req)
+
+_SUBSYSTEMS_INFO = _descriptor.Descriptor(
+    name='subsystems_info',
+    full_name='subsystems_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='subsystems_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='subsystems_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystems', full_name='subsystems_info.subsystems', index=2,
+            number=3, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2722,
+    serialized_end=2810,
+)
+
+
+_SUBSYSTEM = _descriptor.Descriptor(
+    name='subsystem',
+    full_name='subsystem',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='subsystem.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='enable_ha', full_name='subsystem.enable_ha', index=1,
+            number=2, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='serial_number', full_name='subsystem.serial_number', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='model_number', full_name='subsystem.model_number', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_cntlid', full_name='subsystem.min_cntlid', index=4,
+            number=5, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_cntlid', full_name='subsystem.max_cntlid', index=5,
+            number=6, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='namespace_count', full_name='subsystem.namespace_count', index=6,
+            number=7, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subtype', full_name='subsystem.subtype', index=7,
+            number=8, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2813,
+    serialized_end=2983,
+)
+
+
+_GATEWAY_INFO = _descriptor.Descriptor(
+    name='gateway_info',
+    full_name='gateway_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='cli_version', full_name='gateway_info.cli_version', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='version', full_name='gateway_info.version', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='name', full_name='gateway_info.name', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='group', full_name='gateway_info.group', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='addr', full_name='gateway_info.addr', index=4,
+            number=5, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='port', full_name='gateway_info.port', index=5,
+            number=6, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bool_status', full_name='gateway_info.bool_status', index=6,
+            number=7, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='status', full_name='gateway_info.status', index=7,
+            number=8, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='gateway_info.error_message', index=8,
+            number=9, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='spdk_version', full_name='gateway_info.spdk_version', index=9,
+            number=10, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2986,
+    serialized_end=3177,
+)
+
+
+_CLI_VERSION = _descriptor.Descriptor(
+    name='cli_version',
+    full_name='cli_version',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='cli_version.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='cli_version.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='version', full_name='cli_version.version', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3179,
+    serialized_end=3248,
+)
+
+
+_GW_VERSION = _descriptor.Descriptor(
+    name='gw_version',
+    full_name='gw_version',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='gw_version.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='gw_version.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='version', full_name='gw_version.version', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3250,
+    serialized_end=3318,
+)
+
+
+_LISTENER_INFO = _descriptor.Descriptor(
+    name='listener_info',
+    full_name='listener_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='gateway_name', full_name='listener_info.gateway_name', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trtype', full_name='listener_info.trtype', index=1,
+            number=2, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='adrfam', full_name='listener_info.adrfam', index=2,
+            number=3, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='traddr', full_name='listener_info.traddr', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trsvcid', full_name='listener_info.trsvcid', index=4,
+            number=5, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3321,
+    serialized_end=3455,
+)
+
+
+_LISTENERS_INFO = _descriptor.Descriptor(
+    name='listeners_info',
+    full_name='listeners_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='listeners_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='listeners_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='listeners', full_name='listeners_info.listeners', index=2,
+            number=3, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3457,
+    serialized_end=3547,
+)
+
+
+_HOST = _descriptor.Descriptor(
+    name='host',
+    full_name='host',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='host.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3549,
+    serialized_end=3568,
+)
+
+
+_HOSTS_INFO = _descriptor.Descriptor(
+    name='hosts_info',
+    full_name='hosts_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='hosts_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='hosts_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='allow_any_host', full_name='hosts_info.allow_any_host', index=2,
+            number=3, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='hosts_info.subsystem_nqn', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='hosts', full_name='hosts_info.hosts', index=4,
+            number=5, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3570,
+    serialized_end=3690,
+)
+
+
+_CONNECTION = _descriptor.Descriptor(
+    name='connection',
+    full_name='connection',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='connection.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='traddr', full_name='connection.traddr', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trsvcid', full_name='connection.trsvcid', index=2,
+            number=3, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trtype', full_name='connection.trtype', index=3,
+            number=4, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='adrfam', full_name='connection.adrfam', index=4,
+            number=5, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='connected', full_name='connection.connected', index=5,
+            number=6, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='qpairs_count', full_name='connection.qpairs_count', index=6,
+            number=7, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='controller_id', full_name='connection.controller_id', index=7,
+            number=8, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3693,
+    serialized_end=3879,
+)
+
+
+_CONNECTIONS_INFO = _descriptor.Descriptor(
+    name='connections_info',
+    full_name='connections_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='connections_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='connections_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='connections_info.subsystem_nqn', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='connections', full_name='connections_info.connections', index=3,
+            number=4, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3881,
+    serialized_end=3995,
+)
+
+
+_NAMESPACE = _descriptor.Descriptor(
+    name='namespace',
+    full_name='namespace',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace.nsid', index=0,
+            number=1, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bdev_name', full_name='namespace.bdev_name', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rbd_image_name', full_name='namespace.rbd_image_name', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rbd_pool_name', full_name='namespace.rbd_pool_name', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='load_balancing_group', full_name='namespace.load_balancing_group', index=4,
+            number=5, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='block_size', full_name='namespace.block_size', index=5,
+            number=6, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rbd_image_size', full_name='namespace.rbd_image_size', index=6,
+            number=7, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace.uuid', index=7,
+            number=8, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rw_ios_per_second', full_name='namespace.rw_ios_per_second', index=8,
+            number=9, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='rw_mbytes_per_second', full_name='namespace.rw_mbytes_per_second', index=9,
+            number=10, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='r_mbytes_per_second', full_name='namespace.r_mbytes_per_second', index=10,
+            number=11, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='w_mbytes_per_second', full_name='namespace.w_mbytes_per_second', index=11,
+            number=12, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=3998,
+    serialized_end=4292,
+)
+
+
+_NAMESPACES_INFO = _descriptor.Descriptor(
+    name='namespaces_info',
+    full_name='namespaces_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='namespaces_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='namespaces_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespaces_info.subsystem_nqn', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='namespaces', full_name='namespaces_info.namespaces', index=3,
+            number=4, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=4294,
+    serialized_end=4405,
+)
+
+
+_NAMESPACE_IO_STATS_INFO = _descriptor.Descriptor(
+    name='namespace_io_stats_info',
+    full_name='namespace_io_stats_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='namespace_io_stats_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='namespace_io_stats_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='subsystem_nqn', full_name='namespace_io_stats_info.subsystem_nqn', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='namespace_io_stats_info.nsid', index=3,
+            number=4, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace_io_stats_info.uuid', index=4,
+            number=5, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bdev_name', full_name='namespace_io_stats_info.bdev_name', index=5,
+            number=6, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='tick_rate', full_name='namespace_io_stats_info.tick_rate', index=6,
+            number=7, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='ticks', full_name='namespace_io_stats_info.ticks', index=7,
+            number=8, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bytes_read', full_name='namespace_io_stats_info.bytes_read', index=8,
+            number=9, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='num_read_ops', full_name='namespace_io_stats_info.num_read_ops', index=9,
+            number=10, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bytes_written', full_name='namespace_io_stats_info.bytes_written', index=10,
+            number=11, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='num_write_ops', full_name='namespace_io_stats_info.num_write_ops', index=11,
+            number=12, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='bytes_unmapped', full_name='namespace_io_stats_info.bytes_unmapped', index=12,
+            number=13, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='num_unmap_ops', full_name='namespace_io_stats_info.num_unmap_ops', index=13,
+            number=14, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='read_latency_ticks', full_name='namespace_io_stats_info.read_latency_ticks', index=14,
+            number=15, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_read_latency_ticks', full_name='namespace_io_stats_info.max_read_latency_ticks', index=15,
+            number=16, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_read_latency_ticks', full_name='namespace_io_stats_info.min_read_latency_ticks', index=16,
+            number=17, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='write_latency_ticks', full_name='namespace_io_stats_info.write_latency_ticks', index=17,
+            number=18, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_write_latency_ticks', full_name='namespace_io_stats_info.max_write_latency_ticks', index=18,
+            number=19, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_write_latency_ticks', full_name='namespace_io_stats_info.min_write_latency_ticks', index=19,
+            number=20, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='unmap_latency_ticks', full_name='namespace_io_stats_info.unmap_latency_ticks', index=20,
+            number=21, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_unmap_latency_ticks', full_name='namespace_io_stats_info.max_unmap_latency_ticks', index=21,
+            number=22, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_unmap_latency_ticks', full_name='namespace_io_stats_info.min_unmap_latency_ticks', index=22,
+            number=23, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='copy_latency_ticks', full_name='namespace_io_stats_info.copy_latency_ticks', index=23,
+            number=24, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_copy_latency_ticks', full_name='namespace_io_stats_info.max_copy_latency_ticks', index=24,
+            number=25, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_copy_latency_ticks', full_name='namespace_io_stats_info.min_copy_latency_ticks', index=25,
+            number=26, type=4, cpp_type=4, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='io_error', full_name='namespace_io_stats_info.io_error', index=26,
+            number=27, type=13, cpp_type=3, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=4408,
+    serialized_end=5103,
+)
+
+
+_SPDK_LOG_FLAG_INFO = _descriptor.Descriptor(
+    name='spdk_log_flag_info',
+    full_name='spdk_log_flag_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='name', full_name='spdk_log_flag_info.name', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='enabled', full_name='spdk_log_flag_info.enabled', index=1,
+            number=2, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=5105,
+    serialized_end=5156,
+)
+
+
+_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO = _descriptor.Descriptor(
+    name='spdk_nvmf_log_flags_and_level_info',
+    full_name='spdk_nvmf_log_flags_and_level_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='spdk_nvmf_log_flags_and_level_info.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='spdk_nvmf_log_flags_and_level_info.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nvmf_log_flags', full_name='spdk_nvmf_log_flags_and_level_info.nvmf_log_flags', index=2,
+            number=3, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='log_level', full_name='spdk_nvmf_log_flags_and_level_info.log_level', index=3,
+            number=4, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='log_print_level', full_name='spdk_nvmf_log_flags_and_level_info.log_print_level', index=4,
+            number=5, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=5159,
+    serialized_end=5345,
+)
+
+_NAMESPACE_ADD_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['nsid'])
+_NAMESPACE_ADD_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_ADD_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['uuid'])
+_NAMESPACE_ADD_REQ.fields_by_name['uuid'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_uuid']
+_NAMESPACE_ADD_REQ.oneofs_by_name['_anagrpid'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['anagrpid'])
+_NAMESPACE_ADD_REQ.fields_by_name['anagrpid'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_anagrpid']
+_NAMESPACE_ADD_REQ.oneofs_by_name['_create_image'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['create_image'])
+_NAMESPACE_ADD_REQ.fields_by_name['create_image'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_create_image']
+_NAMESPACE_ADD_REQ.oneofs_by_name['_size'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['size'])
+_NAMESPACE_ADD_REQ.fields_by_name['size'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_size']
+_NAMESPACE_RESIZE_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_RESIZE_REQ.fields_by_name['nsid'])
+_NAMESPACE_RESIZE_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_RESIZE_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_RESIZE_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_RESIZE_REQ.fields_by_name['uuid'])
+_NAMESPACE_RESIZE_REQ.fields_by_name['uuid'].containing_oneof = _NAMESPACE_RESIZE_REQ.oneofs_by_name['_uuid']
+_NAMESPACE_GET_IO_STATS_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_GET_IO_STATS_REQ.fields_by_name['nsid'])
+_NAMESPACE_GET_IO_STATS_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_GET_IO_STATS_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_GET_IO_STATS_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_GET_IO_STATS_REQ.fields_by_name['uuid'])
+_NAMESPACE_GET_IO_STATS_REQ.fields_by_name['uuid'].containing_oneof = _NAMESPACE_GET_IO_STATS_REQ.oneofs_by_name['_uuid']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['nsid'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['uuid'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['uuid'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_uuid']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_rw_ios_per_second'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['rw_ios_per_second'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['rw_ios_per_second'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_rw_ios_per_second']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_rw_mbytes_per_second'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['rw_mbytes_per_second'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['rw_mbytes_per_second'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_rw_mbytes_per_second']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_r_mbytes_per_second'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['r_mbytes_per_second'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['r_mbytes_per_second'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_r_mbytes_per_second']
+_NAMESPACE_SET_QOS_REQ.oneofs_by_name['_w_mbytes_per_second'].fields.append(
+    _NAMESPACE_SET_QOS_REQ.fields_by_name['w_mbytes_per_second'])
+_NAMESPACE_SET_QOS_REQ.fields_by_name['w_mbytes_per_second'].containing_oneof = _NAMESPACE_SET_QOS_REQ.oneofs_by_name['_w_mbytes_per_second']
+_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.fields_by_name['nsid'])
+_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.fields_by_name[
+    'nsid'].containing_oneof = _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.fields_by_name['uuid'])
+_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.fields_by_name[
+    'uuid'].containing_oneof = _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ.oneofs_by_name['_uuid']
+_NAMESPACE_DELETE_REQ.oneofs_by_name['_nsid'].fields.append(
+    _NAMESPACE_DELETE_REQ.fields_by_name['nsid'])
+_NAMESPACE_DELETE_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_DELETE_REQ.oneofs_by_name['_nsid']
+_NAMESPACE_DELETE_REQ.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE_DELETE_REQ.fields_by_name['uuid'])
+_NAMESPACE_DELETE_REQ.fields_by_name['uuid'].containing_oneof = _NAMESPACE_DELETE_REQ.oneofs_by_name['_uuid']
+_CREATE_SUBSYSTEM_REQ.oneofs_by_name['_max_namespaces'].fields.append(
+    _CREATE_SUBSYSTEM_REQ.fields_by_name['max_namespaces'])
+_CREATE_SUBSYSTEM_REQ.fields_by_name['max_namespaces'].containing_oneof = _CREATE_SUBSYSTEM_REQ.oneofs_by_name['_max_namespaces']
+_DELETE_SUBSYSTEM_REQ.oneofs_by_name['_force'].fields.append(
+    _DELETE_SUBSYSTEM_REQ.fields_by_name['force'])
+_DELETE_SUBSYSTEM_REQ.fields_by_name['force'].containing_oneof = _DELETE_SUBSYSTEM_REQ.oneofs_by_name['_force']
+_LIST_NAMESPACES_REQ.oneofs_by_name['_nsid'].fields.append(
+    _LIST_NAMESPACES_REQ.fields_by_name['nsid'])
+_LIST_NAMESPACES_REQ.fields_by_name['nsid'].containing_oneof = _LIST_NAMESPACES_REQ.oneofs_by_name['_nsid']
+_LIST_NAMESPACES_REQ.oneofs_by_name['_uuid'].fields.append(
+    _LIST_NAMESPACES_REQ.fields_by_name['uuid'])
+_LIST_NAMESPACES_REQ.fields_by_name['uuid'].containing_oneof = _LIST_NAMESPACES_REQ.oneofs_by_name['_uuid']
+_CREATE_LISTENER_REQ.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
+_CREATE_LISTENER_REQ.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
+_CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'].enum_type = _AUTOHASTATE
+_CREATE_LISTENER_REQ.oneofs_by_name['_trtype'].fields.append(
+    _CREATE_LISTENER_REQ.fields_by_name['trtype'])
+_CREATE_LISTENER_REQ.fields_by_name['trtype'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_trtype']
+_CREATE_LISTENER_REQ.oneofs_by_name['_adrfam'].fields.append(
+    _CREATE_LISTENER_REQ.fields_by_name['adrfam'])
+_CREATE_LISTENER_REQ.fields_by_name['adrfam'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_adrfam']
+_CREATE_LISTENER_REQ.oneofs_by_name['_trsvcid'].fields.append(
+    _CREATE_LISTENER_REQ.fields_by_name['trsvcid'])
+_CREATE_LISTENER_REQ.fields_by_name['trsvcid'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_trsvcid']
+_CREATE_LISTENER_REQ.oneofs_by_name['_auto_ha_state'].fields.append(
+    _CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'])
+_CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_auto_ha_state']
+_DELETE_LISTENER_REQ.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
+_DELETE_LISTENER_REQ.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
+_DELETE_LISTENER_REQ.oneofs_by_name['_trtype'].fields.append(
+    _DELETE_LISTENER_REQ.fields_by_name['trtype'])
+_DELETE_LISTENER_REQ.fields_by_name['trtype'].containing_oneof = _DELETE_LISTENER_REQ.oneofs_by_name['_trtype']
+_DELETE_LISTENER_REQ.oneofs_by_name['_adrfam'].fields.append(
+    _DELETE_LISTENER_REQ.fields_by_name['adrfam'])
+_DELETE_LISTENER_REQ.fields_by_name['adrfam'].containing_oneof = _DELETE_LISTENER_REQ.oneofs_by_name['_adrfam']
+_DELETE_LISTENER_REQ.oneofs_by_name['_trsvcid'].fields.append(
+    _DELETE_LISTENER_REQ.fields_by_name['trsvcid'])
+_DELETE_LISTENER_REQ.fields_by_name['trsvcid'].containing_oneof = _DELETE_LISTENER_REQ.oneofs_by_name['_trsvcid']
+_LIST_SUBSYSTEMS_REQ.oneofs_by_name['_subsystem_nqn'].fields.append(
+    _LIST_SUBSYSTEMS_REQ.fields_by_name['subsystem_nqn'])
+_LIST_SUBSYSTEMS_REQ.fields_by_name['subsystem_nqn'].containing_oneof = _LIST_SUBSYSTEMS_REQ.oneofs_by_name['_subsystem_nqn']
+_LIST_SUBSYSTEMS_REQ.oneofs_by_name['_serial_number'].fields.append(
+    _LIST_SUBSYSTEMS_REQ.fields_by_name['serial_number'])
+_LIST_SUBSYSTEMS_REQ.fields_by_name['serial_number'].containing_oneof = _LIST_SUBSYSTEMS_REQ.oneofs_by_name['_serial_number']
+_SET_SPDK_NVMF_LOGS_REQ.fields_by_name['log_level'].enum_type = _LOGLEVEL
+_SET_SPDK_NVMF_LOGS_REQ.fields_by_name['print_level'].enum_type = _LOGLEVEL
+_SET_SPDK_NVMF_LOGS_REQ.oneofs_by_name['_log_level'].fields.append(
+    _SET_SPDK_NVMF_LOGS_REQ.fields_by_name['log_level'])
+_SET_SPDK_NVMF_LOGS_REQ.fields_by_name['log_level'].containing_oneof = _SET_SPDK_NVMF_LOGS_REQ.oneofs_by_name['_log_level']
+_SET_SPDK_NVMF_LOGS_REQ.oneofs_by_name['_print_level'].fields.append(
+    _SET_SPDK_NVMF_LOGS_REQ.fields_by_name['print_level'])
+_SET_SPDK_NVMF_LOGS_REQ.fields_by_name['print_level'].containing_oneof = _SET_SPDK_NVMF_LOGS_REQ.oneofs_by_name['_print_level']
+_GET_GATEWAY_INFO_REQ.oneofs_by_name['_cli_version'].fields.append(
+    _GET_GATEWAY_INFO_REQ.fields_by_name['cli_version'])
+_GET_GATEWAY_INFO_REQ.fields_by_name['cli_version'].containing_oneof = _GET_GATEWAY_INFO_REQ.oneofs_by_name['_cli_version']
+_SUBSYSTEMS_INFO.fields_by_name['subsystems'].message_type = _SUBSYSTEM
+_LISTENER_INFO.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
+_LISTENER_INFO.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
+_LISTENERS_INFO.fields_by_name['listeners'].message_type = _LISTENER_INFO
+_HOSTS_INFO.fields_by_name['hosts'].message_type = _HOST
+_CONNECTION.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
+_CONNECTION.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
+_CONNECTIONS_INFO.fields_by_name['connections'].message_type = _CONNECTION
+_NAMESPACES_INFO.fields_by_name['namespaces'].message_type = _NAMESPACE
+_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['nvmf_log_flags'].message_type = _SPDK_LOG_FLAG_INFO
+_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['log_level'].enum_type = _LOGLEVEL
+_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['log_print_level'].enum_type = _LOGLEVEL
+DESCRIPTOR.message_types_by_name['namespace_add_req'] = _NAMESPACE_ADD_REQ
+DESCRIPTOR.message_types_by_name['namespace_resize_req'] = _NAMESPACE_RESIZE_REQ
+DESCRIPTOR.message_types_by_name['namespace_get_io_stats_req'] = _NAMESPACE_GET_IO_STATS_REQ
+DESCRIPTOR.message_types_by_name['namespace_set_qos_req'] = _NAMESPACE_SET_QOS_REQ
+DESCRIPTOR.message_types_by_name['namespace_change_load_balancing_group_req'] = _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ
+DESCRIPTOR.message_types_by_name['namespace_delete_req'] = _NAMESPACE_DELETE_REQ
+DESCRIPTOR.message_types_by_name['create_subsystem_req'] = _CREATE_SUBSYSTEM_REQ
+DESCRIPTOR.message_types_by_name['delete_subsystem_req'] = _DELETE_SUBSYSTEM_REQ
+DESCRIPTOR.message_types_by_name['list_namespaces_req'] = _LIST_NAMESPACES_REQ
+DESCRIPTOR.message_types_by_name['add_host_req'] = _ADD_HOST_REQ
+DESCRIPTOR.message_types_by_name['remove_host_req'] = _REMOVE_HOST_REQ
+DESCRIPTOR.message_types_by_name['list_hosts_req'] = _LIST_HOSTS_REQ
+DESCRIPTOR.message_types_by_name['list_connections_req'] = _LIST_CONNECTIONS_REQ
+DESCRIPTOR.message_types_by_name['create_listener_req'] = _CREATE_LISTENER_REQ
+DESCRIPTOR.message_types_by_name['delete_listener_req'] = _DELETE_LISTENER_REQ
+DESCRIPTOR.message_types_by_name['list_listeners_req'] = _LIST_LISTENERS_REQ
+DESCRIPTOR.message_types_by_name['list_subsystems_req'] = _LIST_SUBSYSTEMS_REQ
+DESCRIPTOR.message_types_by_name['get_spdk_nvmf_log_flags_and_level_req'] = _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ
+DESCRIPTOR.message_types_by_name['disable_spdk_nvmf_logs_req'] = _DISABLE_SPDK_NVMF_LOGS_REQ
+DESCRIPTOR.message_types_by_name['set_spdk_nvmf_logs_req'] = _SET_SPDK_NVMF_LOGS_REQ
+DESCRIPTOR.message_types_by_name['get_gateway_info_req'] = _GET_GATEWAY_INFO_REQ
+DESCRIPTOR.message_types_by_name['bdev_status'] = _BDEV_STATUS
+DESCRIPTOR.message_types_by_name['req_status'] = _REQ_STATUS
+DESCRIPTOR.message_types_by_name['nsid_status'] = _NSID_STATUS
+DESCRIPTOR.message_types_by_name['subsystems_info'] = _SUBSYSTEMS_INFO
+DESCRIPTOR.message_types_by_name['subsystem'] = _SUBSYSTEM
+DESCRIPTOR.message_types_by_name['gateway_info'] = _GATEWAY_INFO
+DESCRIPTOR.message_types_by_name['cli_version'] = _CLI_VERSION
+DESCRIPTOR.message_types_by_name['gw_version'] = _GW_VERSION
+DESCRIPTOR.message_types_by_name['listener_info'] = _LISTENER_INFO
+DESCRIPTOR.message_types_by_name['listeners_info'] = _LISTENERS_INFO
+DESCRIPTOR.message_types_by_name['host'] = _HOST
+DESCRIPTOR.message_types_by_name['hosts_info'] = _HOSTS_INFO
+DESCRIPTOR.message_types_by_name['connection'] = _CONNECTION
+DESCRIPTOR.message_types_by_name['connections_info'] = _CONNECTIONS_INFO
+DESCRIPTOR.message_types_by_name['namespace'] = _NAMESPACE
+DESCRIPTOR.message_types_by_name['namespaces_info'] = _NAMESPACES_INFO
+DESCRIPTOR.message_types_by_name['namespace_io_stats_info'] = _NAMESPACE_IO_STATS_INFO
+DESCRIPTOR.message_types_by_name['spdk_log_flag_info'] = _SPDK_LOG_FLAG_INFO
+DESCRIPTOR.message_types_by_name['spdk_nvmf_log_flags_and_level_info'] = _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO
+DESCRIPTOR.enum_types_by_name['TransportType'] = _TRANSPORTTYPE
+DESCRIPTOR.enum_types_by_name['AddressFamily'] = _ADDRESSFAMILY
+DESCRIPTOR.enum_types_by_name['LogLevel'] = _LOGLEVEL
+DESCRIPTOR.enum_types_by_name['AutoHAState'] = _AUTOHASTATE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+namespace_add_req = _reflection.GeneratedProtocolMessageType('namespace_add_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_ADD_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_add_req)
+})
+_sym_db.RegisterMessage(namespace_add_req)
+
+namespace_resize_req = _reflection.GeneratedProtocolMessageType('namespace_resize_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_RESIZE_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_resize_req)
+})
+_sym_db.RegisterMessage(namespace_resize_req)
+
+namespace_get_io_stats_req = _reflection.GeneratedProtocolMessageType('namespace_get_io_stats_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_GET_IO_STATS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_get_io_stats_req)
+})
+_sym_db.RegisterMessage(namespace_get_io_stats_req)
+
+namespace_set_qos_req = _reflection.GeneratedProtocolMessageType('namespace_set_qos_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_SET_QOS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_set_qos_req)
+})
+_sym_db.RegisterMessage(namespace_set_qos_req)
+
+namespace_change_load_balancing_group_req = _reflection.GeneratedProtocolMessageType('namespace_change_load_balancing_group_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_change_load_balancing_group_req)
+})
+_sym_db.RegisterMessage(namespace_change_load_balancing_group_req)
+
+namespace_delete_req = _reflection.GeneratedProtocolMessageType('namespace_delete_req', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_DELETE_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_delete_req)
+})
+_sym_db.RegisterMessage(namespace_delete_req)
 
 create_subsystem_req = _reflection.GeneratedProtocolMessageType('create_subsystem_req', (_message.Message,), {
-  'DESCRIPTOR' : _CREATE_SUBSYSTEM_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:create_subsystem_req)
-  })
+    'DESCRIPTOR': _CREATE_SUBSYSTEM_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:create_subsystem_req)
+})
 _sym_db.RegisterMessage(create_subsystem_req)
 
 delete_subsystem_req = _reflection.GeneratedProtocolMessageType('delete_subsystem_req', (_message.Message,), {
-  'DESCRIPTOR' : _DELETE_SUBSYSTEM_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:delete_subsystem_req)
-  })
+    'DESCRIPTOR': _DELETE_SUBSYSTEM_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:delete_subsystem_req)
+})
 _sym_db.RegisterMessage(delete_subsystem_req)
 
-add_namespace_req = _reflection.GeneratedProtocolMessageType('add_namespace_req', (_message.Message,), {
-  'DESCRIPTOR' : _ADD_NAMESPACE_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:add_namespace_req)
-  })
-_sym_db.RegisterMessage(add_namespace_req)
-
-remove_namespace_req = _reflection.GeneratedProtocolMessageType('remove_namespace_req', (_message.Message,), {
-  'DESCRIPTOR' : _REMOVE_NAMESPACE_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:remove_namespace_req)
-  })
-_sym_db.RegisterMessage(remove_namespace_req)
+list_namespaces_req = _reflection.GeneratedProtocolMessageType('list_namespaces_req', (_message.Message,), {
+    'DESCRIPTOR': _LIST_NAMESPACES_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:list_namespaces_req)
+})
+_sym_db.RegisterMessage(list_namespaces_req)
 
 add_host_req = _reflection.GeneratedProtocolMessageType('add_host_req', (_message.Message,), {
-  'DESCRIPTOR' : _ADD_HOST_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:add_host_req)
-  })
+    'DESCRIPTOR': _ADD_HOST_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:add_host_req)
+})
 _sym_db.RegisterMessage(add_host_req)
 
 remove_host_req = _reflection.GeneratedProtocolMessageType('remove_host_req', (_message.Message,), {
-  'DESCRIPTOR' : _REMOVE_HOST_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:remove_host_req)
-  })
+    'DESCRIPTOR': _REMOVE_HOST_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:remove_host_req)
+})
 _sym_db.RegisterMessage(remove_host_req)
 
+list_hosts_req = _reflection.GeneratedProtocolMessageType('list_hosts_req', (_message.Message,), {
+    'DESCRIPTOR': _LIST_HOSTS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:list_hosts_req)
+})
+_sym_db.RegisterMessage(list_hosts_req)
+
+list_connections_req = _reflection.GeneratedProtocolMessageType('list_connections_req', (_message.Message,), {
+    'DESCRIPTOR': _LIST_CONNECTIONS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:list_connections_req)
+})
+_sym_db.RegisterMessage(list_connections_req)
+
 create_listener_req = _reflection.GeneratedProtocolMessageType('create_listener_req', (_message.Message,), {
-  'DESCRIPTOR' : _CREATE_LISTENER_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:create_listener_req)
-  })
+    'DESCRIPTOR': _CREATE_LISTENER_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:create_listener_req)
+})
 _sym_db.RegisterMessage(create_listener_req)
 
 delete_listener_req = _reflection.GeneratedProtocolMessageType('delete_listener_req', (_message.Message,), {
-  'DESCRIPTOR' : _DELETE_LISTENER_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:delete_listener_req)
-  })
+    'DESCRIPTOR': _DELETE_LISTENER_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:delete_listener_req)
+})
 _sym_db.RegisterMessage(delete_listener_req)
 
-get_subsystems_req = _reflection.GeneratedProtocolMessageType('get_subsystems_req', (_message.Message,), {
-  'DESCRIPTOR' : _GET_SUBSYSTEMS_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:get_subsystems_req)
-  })
-_sym_db.RegisterMessage(get_subsystems_req)
+list_listeners_req = _reflection.GeneratedProtocolMessageType('list_listeners_req', (_message.Message,), {
+    'DESCRIPTOR': _LIST_LISTENERS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:list_listeners_req)
+})
+_sym_db.RegisterMessage(list_listeners_req)
+
+list_subsystems_req = _reflection.GeneratedProtocolMessageType('list_subsystems_req', (_message.Message,), {
+    'DESCRIPTOR': _LIST_SUBSYSTEMS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:list_subsystems_req)
+})
+_sym_db.RegisterMessage(list_subsystems_req)
 
 get_spdk_nvmf_log_flags_and_level_req = _reflection.GeneratedProtocolMessageType('get_spdk_nvmf_log_flags_and_level_req', (_message.Message,), {
-  'DESCRIPTOR' : _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:get_spdk_nvmf_log_flags_and_level_req)
-  })
+    'DESCRIPTOR': _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:get_spdk_nvmf_log_flags_and_level_req)
+})
 _sym_db.RegisterMessage(get_spdk_nvmf_log_flags_and_level_req)
 
 disable_spdk_nvmf_logs_req = _reflection.GeneratedProtocolMessageType('disable_spdk_nvmf_logs_req', (_message.Message,), {
-  'DESCRIPTOR' : _DISABLE_SPDK_NVMF_LOGS_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:disable_spdk_nvmf_logs_req)
-  })
+    'DESCRIPTOR': _DISABLE_SPDK_NVMF_LOGS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:disable_spdk_nvmf_logs_req)
+})
 _sym_db.RegisterMessage(disable_spdk_nvmf_logs_req)
 
 set_spdk_nvmf_logs_req = _reflection.GeneratedProtocolMessageType('set_spdk_nvmf_logs_req', (_message.Message,), {
-  'DESCRIPTOR' : _SET_SPDK_NVMF_LOGS_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:set_spdk_nvmf_logs_req)
-  })
+    'DESCRIPTOR': _SET_SPDK_NVMF_LOGS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:set_spdk_nvmf_logs_req)
+})
 _sym_db.RegisterMessage(set_spdk_nvmf_logs_req)
 
 get_gateway_info_req = _reflection.GeneratedProtocolMessageType('get_gateway_info_req', (_message.Message,), {
-  'DESCRIPTOR' : _GET_GATEWAY_INFO_REQ,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:get_gateway_info_req)
-  })
+    'DESCRIPTOR': _GET_GATEWAY_INFO_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:get_gateway_info_req)
+})
 _sym_db.RegisterMessage(get_gateway_info_req)
 
-bdev = _reflection.GeneratedProtocolMessageType('bdev', (_message.Message,), {
-  'DESCRIPTOR' : _BDEV,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:bdev)
-  })
-_sym_db.RegisterMessage(bdev)
+bdev_status = _reflection.GeneratedProtocolMessageType('bdev_status', (_message.Message,), {
+    'DESCRIPTOR': _BDEV_STATUS,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:bdev_status)
+})
+_sym_db.RegisterMessage(bdev_status)
 
 req_status = _reflection.GeneratedProtocolMessageType('req_status', (_message.Message,), {
-  'DESCRIPTOR' : _REQ_STATUS,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:req_status)
-  })
+    'DESCRIPTOR': _REQ_STATUS,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:req_status)
+})
 _sym_db.RegisterMessage(req_status)
 
 nsid_status = _reflection.GeneratedProtocolMessageType('nsid_status', (_message.Message,), {
-  'DESCRIPTOR' : _NSID_STATUS,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:nsid_status)
-  })
+    'DESCRIPTOR': _NSID_STATUS,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:nsid_status)
+})
 _sym_db.RegisterMessage(nsid_status)
 
 subsystems_info = _reflection.GeneratedProtocolMessageType('subsystems_info', (_message.Message,), {
-  'DESCRIPTOR' : _SUBSYSTEMS_INFO,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:subsystems_info)
-  })
+    'DESCRIPTOR': _SUBSYSTEMS_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:subsystems_info)
+})
 _sym_db.RegisterMessage(subsystems_info)
 
 subsystem = _reflection.GeneratedProtocolMessageType('subsystem', (_message.Message,), {
-  'DESCRIPTOR' : _SUBSYSTEM,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:subsystem)
-  })
+    'DESCRIPTOR': _SUBSYSTEM,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:subsystem)
+})
 _sym_db.RegisterMessage(subsystem)
 
 gateway_info = _reflection.GeneratedProtocolMessageType('gateway_info', (_message.Message,), {
-  'DESCRIPTOR' : _GATEWAY_INFO,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:gateway_info)
-  })
+    'DESCRIPTOR': _GATEWAY_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:gateway_info)
+})
 _sym_db.RegisterMessage(gateway_info)
 
-listen_address = _reflection.GeneratedProtocolMessageType('listen_address', (_message.Message,), {
-  'DESCRIPTOR' : _LISTEN_ADDRESS,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:listen_address)
-  })
-_sym_db.RegisterMessage(listen_address)
+cli_version = _reflection.GeneratedProtocolMessageType('cli_version', (_message.Message,), {
+    'DESCRIPTOR': _CLI_VERSION,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:cli_version)
+})
+_sym_db.RegisterMessage(cli_version)
+
+gw_version = _reflection.GeneratedProtocolMessageType('gw_version', (_message.Message,), {
+    'DESCRIPTOR': _GW_VERSION,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:gw_version)
+})
+_sym_db.RegisterMessage(gw_version)
+
+listener_info = _reflection.GeneratedProtocolMessageType('listener_info', (_message.Message,), {
+    'DESCRIPTOR': _LISTENER_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:listener_info)
+})
+_sym_db.RegisterMessage(listener_info)
+
+listeners_info = _reflection.GeneratedProtocolMessageType('listeners_info', (_message.Message,), {
+    'DESCRIPTOR': _LISTENERS_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:listeners_info)
+})
+_sym_db.RegisterMessage(listeners_info)
 
 host = _reflection.GeneratedProtocolMessageType('host', (_message.Message,), {
-  'DESCRIPTOR' : _HOST,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:host)
-  })
+    'DESCRIPTOR': _HOST,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:host)
+})
 _sym_db.RegisterMessage(host)
 
+hosts_info = _reflection.GeneratedProtocolMessageType('hosts_info', (_message.Message,), {
+    'DESCRIPTOR': _HOSTS_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:hosts_info)
+})
+_sym_db.RegisterMessage(hosts_info)
+
+connection = _reflection.GeneratedProtocolMessageType('connection', (_message.Message,), {
+    'DESCRIPTOR': _CONNECTION,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:connection)
+})
+_sym_db.RegisterMessage(connection)
+
+connections_info = _reflection.GeneratedProtocolMessageType('connections_info', (_message.Message,), {
+    'DESCRIPTOR': _CONNECTIONS_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:connections_info)
+})
+_sym_db.RegisterMessage(connections_info)
+
 namespace = _reflection.GeneratedProtocolMessageType('namespace', (_message.Message,), {
-  'DESCRIPTOR' : _NAMESPACE,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:namespace)
-  })
+    'DESCRIPTOR': _NAMESPACE,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace)
+})
 _sym_db.RegisterMessage(namespace)
 
+namespaces_info = _reflection.GeneratedProtocolMessageType('namespaces_info', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACES_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespaces_info)
+})
+_sym_db.RegisterMessage(namespaces_info)
+
+namespace_io_stats_info = _reflection.GeneratedProtocolMessageType('namespace_io_stats_info', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_IO_STATS_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace_io_stats_info)
+})
+_sym_db.RegisterMessage(namespace_io_stats_info)
+
+spdk_log_flag_info = _reflection.GeneratedProtocolMessageType('spdk_log_flag_info', (_message.Message,), {
+    'DESCRIPTOR': _SPDK_LOG_FLAG_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:spdk_log_flag_info)
+})
+_sym_db.RegisterMessage(spdk_log_flag_info)
+
 spdk_nvmf_log_flags_and_level_info = _reflection.GeneratedProtocolMessageType('spdk_nvmf_log_flags_and_level_info', (_message.Message,), {
-  'DESCRIPTOR' : _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO,
-  '__module__' : 'gateway_pb2'
-  # @@protoc_insertion_point(class_scope:spdk_nvmf_log_flags_and_level_info)
-  })
+    'DESCRIPTOR': _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:spdk_nvmf_log_flags_and_level_info)
+})
 _sym_db.RegisterMessage(spdk_nvmf_log_flags_and_level_info)
 
-_GATEWAY = DESCRIPTOR.services_by_name['Gateway']
-if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  _TRANSPORTTYPE._serialized_start=2436
-  _TRANSPORTTYPE._serialized_end=2530
-  _ADDRESSFAMILY._serialized_start=2532
-  _ADDRESSFAMILY._serialized_end=2596
-  _LOGLEVEL._serialized_start=2598
-  _LOGLEVEL._serialized_end=2676
-  _CREATE_BDEV_REQ._serialized_start=18
-  _CREATE_BDEV_REQ._serialized_end=149
-  _RESIZE_BDEV_REQ._serialized_start=151
-  _RESIZE_BDEV_REQ._serialized_end=205
-  _DELETE_BDEV_REQ._serialized_start=207
-  _DELETE_BDEV_REQ._serialized_end=258
-  _CREATE_SUBSYSTEM_REQ._serialized_start=261
-  _CREATE_SUBSYSTEM_REQ._serialized_end=395
-  _DELETE_SUBSYSTEM_REQ._serialized_start=397
-  _DELETE_SUBSYSTEM_REQ._serialized_end=442
-  _ADD_NAMESPACE_REQ._serialized_start=444
-  _ADD_NAMESPACE_REQ._serialized_end=569
-  _REMOVE_NAMESPACE_REQ._serialized_start=571
-  _REMOVE_NAMESPACE_REQ._serialized_end=630
-  _ADD_HOST_REQ._serialized_start=632
-  _ADD_HOST_REQ._serialized_end=687
-  _REMOVE_HOST_REQ._serialized_start=689
-  _REMOVE_HOST_REQ._serialized_end=747
-  _CREATE_LISTENER_REQ._serialized_start=750
-  _CREATE_LISTENER_REQ._serialized_end=903
-  _DELETE_LISTENER_REQ._serialized_start=906
-  _DELETE_LISTENER_REQ._serialized_end=1059
-  _GET_SUBSYSTEMS_REQ._serialized_start=1061
-  _GET_SUBSYSTEMS_REQ._serialized_end=1081
-  _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ._serialized_start=1083
-  _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ._serialized_end=1122
-  _DISABLE_SPDK_NVMF_LOGS_REQ._serialized_start=1124
-  _DISABLE_SPDK_NVMF_LOGS_REQ._serialized_end=1152
-  _SET_SPDK_NVMF_LOGS_REQ._serialized_start=1154
-  _SET_SPDK_NVMF_LOGS_REQ._serialized_end=1280
-  _GET_GATEWAY_INFO_REQ._serialized_start=1282
-  _GET_GATEWAY_INFO_REQ._serialized_end=1325
-  _BDEV._serialized_start=1327
-  _BDEV._serialized_end=1368
-  _REQ_STATUS._serialized_start=1370
-  _REQ_STATUS._serialized_end=1398
-  _NSID_STATUS._serialized_start=1400
-  _NSID_STATUS._serialized_end=1443
-  _SUBSYSTEMS_INFO._serialized_start=1445
-  _SUBSYSTEMS_INFO._serialized_end=1494
-  _SUBSYSTEM._serialized_start=1497
-  _SUBSYSTEM._serialized_end=1877
-  _GATEWAY_INFO._serialized_start=1880
-  _GATEWAY_INFO._serialized_end=2045
-  _LISTEN_ADDRESS._serialized_start=2048
-  _LISTEN_ADDRESS._serialized_end=2180
-  _HOST._serialized_start=2182
-  _HOST._serialized_end=2201
-  _NAMESPACE._serialized_start=2204
-  _NAMESPACE._serialized_end=2375
-  _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO._serialized_start=2377
-  _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO._serialized_end=2434
-  _GATEWAY._serialized_start=2679
-  _GATEWAY._serialized_end=3622
+
+_GATEWAY = _descriptor.ServiceDescriptor(
+    name='Gateway',
+    full_name='Gateway',
+    file=DESCRIPTOR,
+    index=0,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+    serialized_start=5660,
+    serialized_end=7004,
+    methods=[
+        _descriptor.MethodDescriptor(
+            name='namespace_add',
+            full_name='Gateway.namespace_add',
+            index=0,
+            containing_service=None,
+            input_type=_NAMESPACE_ADD_REQ,
+            output_type=_NSID_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='create_subsystem',
+            full_name='Gateway.create_subsystem',
+            index=1,
+            containing_service=None,
+            input_type=_CREATE_SUBSYSTEM_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='delete_subsystem',
+            full_name='Gateway.delete_subsystem',
+            index=2,
+            containing_service=None,
+            input_type=_DELETE_SUBSYSTEM_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='list_namespaces',
+            full_name='Gateway.list_namespaces',
+            index=3,
+            containing_service=None,
+            input_type=_LIST_NAMESPACES_REQ,
+            output_type=_NAMESPACES_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='namespace_resize',
+            full_name='Gateway.namespace_resize',
+            index=4,
+            containing_service=None,
+            input_type=_NAMESPACE_RESIZE_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='namespace_get_io_stats',
+            full_name='Gateway.namespace_get_io_stats',
+            index=5,
+            containing_service=None,
+            input_type=_NAMESPACE_GET_IO_STATS_REQ,
+            output_type=_NAMESPACE_IO_STATS_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='namespace_set_qos_limits',
+            full_name='Gateway.namespace_set_qos_limits',
+            index=6,
+            containing_service=None,
+            input_type=_NAMESPACE_SET_QOS_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='namespace_change_load_balancing_group',
+            full_name='Gateway.namespace_change_load_balancing_group',
+            index=7,
+            containing_service=None,
+            input_type=_NAMESPACE_CHANGE_LOAD_BALANCING_GROUP_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='namespace_delete',
+            full_name='Gateway.namespace_delete',
+            index=8,
+            containing_service=None,
+            input_type=_NAMESPACE_DELETE_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='add_host',
+            full_name='Gateway.add_host',
+            index=9,
+            containing_service=None,
+            input_type=_ADD_HOST_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='remove_host',
+            full_name='Gateway.remove_host',
+            index=10,
+            containing_service=None,
+            input_type=_REMOVE_HOST_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='list_hosts',
+            full_name='Gateway.list_hosts',
+            index=11,
+            containing_service=None,
+            input_type=_LIST_HOSTS_REQ,
+            output_type=_HOSTS_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='list_connections',
+            full_name='Gateway.list_connections',
+            index=12,
+            containing_service=None,
+            input_type=_LIST_CONNECTIONS_REQ,
+            output_type=_CONNECTIONS_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='create_listener',
+            full_name='Gateway.create_listener',
+            index=13,
+            containing_service=None,
+            input_type=_CREATE_LISTENER_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='delete_listener',
+            full_name='Gateway.delete_listener',
+            index=14,
+            containing_service=None,
+            input_type=_DELETE_LISTENER_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='list_listeners',
+            full_name='Gateway.list_listeners',
+            index=15,
+            containing_service=None,
+            input_type=_LIST_LISTENERS_REQ,
+            output_type=_LISTENERS_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='list_subsystems',
+            full_name='Gateway.list_subsystems',
+            index=16,
+            containing_service=None,
+            input_type=_LIST_SUBSYSTEMS_REQ,
+            output_type=_SUBSYSTEMS_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='get_spdk_nvmf_log_flags_and_level',
+            full_name='Gateway.get_spdk_nvmf_log_flags_and_level',
+            index=17,
+            containing_service=None,
+            input_type=_GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
+            output_type=_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='disable_spdk_nvmf_logs',
+            full_name='Gateway.disable_spdk_nvmf_logs',
+            index=18,
+            containing_service=None,
+            input_type=_DISABLE_SPDK_NVMF_LOGS_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='set_spdk_nvmf_logs',
+            full_name='Gateway.set_spdk_nvmf_logs',
+            index=19,
+            containing_service=None,
+            input_type=_SET_SPDK_NVMF_LOGS_REQ,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='get_gateway_info',
+            full_name='Gateway.get_gateway_info',
+            index=20,
+            containing_service=None,
+            input_type=_GET_GATEWAY_INFO_REQ,
+            output_type=_GATEWAY_INFO,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+    ])
+_sym_db.RegisterServiceDescriptor(_GATEWAY)
+
+DESCRIPTOR.services_by_name['Gateway'] = _GATEWAY
+
 # @@protoc_insertion_point(module_scope)
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
index b3d9c2dcc76b..9993ca4cf6b9 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
@@ -2,7 +2,7 @@
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
 
-from ..proto import gateway_pb2 as gateway__pb2
+from . import gateway_pb2 as gateway__pb2
 
 
 class GatewayStub(object):
@@ -14,135 +14,174 @@ def __init__(self, channel):
         Args:
             channel: A grpc.Channel.
         """
-        self.create_bdev = channel.unary_unary(
-                '/Gateway/create_bdev',
-                request_serializer=gateway__pb2.create_bdev_req.SerializeToString,
-                response_deserializer=gateway__pb2.bdev.FromString,
-                )
-        self.resize_bdev = channel.unary_unary(
-                '/Gateway/resize_bdev',
-                request_serializer=gateway__pb2.resize_bdev_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
-        self.delete_bdev = channel.unary_unary(
-                '/Gateway/delete_bdev',
-                request_serializer=gateway__pb2.delete_bdev_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+        self.namespace_add = channel.unary_unary(
+            '/Gateway/namespace_add',
+            request_serializer=gateway__pb2.namespace_add_req.SerializeToString,
+            response_deserializer=gateway__pb2.nsid_status.FromString,
+        )
         self.create_subsystem = channel.unary_unary(
-                '/Gateway/create_subsystem',
-                request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/create_subsystem',
+            request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.delete_subsystem = channel.unary_unary(
-                '/Gateway/delete_subsystem',
-                request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
-        self.add_namespace = channel.unary_unary(
-                '/Gateway/add_namespace',
-                request_serializer=gateway__pb2.add_namespace_req.SerializeToString,
-                response_deserializer=gateway__pb2.nsid_status.FromString,
-                )
-        self.remove_namespace = channel.unary_unary(
-                '/Gateway/remove_namespace',
-                request_serializer=gateway__pb2.remove_namespace_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/delete_subsystem',
+            request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.list_namespaces = channel.unary_unary(
+            '/Gateway/list_namespaces',
+            request_serializer=gateway__pb2.list_namespaces_req.SerializeToString,
+            response_deserializer=gateway__pb2.namespaces_info.FromString,
+        )
+        self.namespace_resize = channel.unary_unary(
+            '/Gateway/namespace_resize',
+            request_serializer=gateway__pb2.namespace_resize_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.namespace_get_io_stats = channel.unary_unary(
+            '/Gateway/namespace_get_io_stats',
+            request_serializer=gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+            response_deserializer=gateway__pb2.namespace_io_stats_info.FromString,
+        )
+        self.namespace_set_qos_limits = channel.unary_unary(
+            '/Gateway/namespace_set_qos_limits',
+            request_serializer=gateway__pb2.namespace_set_qos_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.namespace_change_load_balancing_group = channel.unary_unary(
+            '/Gateway/namespace_change_load_balancing_group',
+            request_serializer=gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.namespace_delete = channel.unary_unary(
+            '/Gateway/namespace_delete',
+            request_serializer=gateway__pb2.namespace_delete_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.add_host = channel.unary_unary(
-                '/Gateway/add_host',
-                request_serializer=gateway__pb2.add_host_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/add_host',
+            request_serializer=gateway__pb2.add_host_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.remove_host = channel.unary_unary(
-                '/Gateway/remove_host',
-                request_serializer=gateway__pb2.remove_host_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/remove_host',
+            request_serializer=gateway__pb2.remove_host_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.list_hosts = channel.unary_unary(
+            '/Gateway/list_hosts',
+            request_serializer=gateway__pb2.list_hosts_req.SerializeToString,
+            response_deserializer=gateway__pb2.hosts_info.FromString,
+        )
+        self.list_connections = channel.unary_unary(
+            '/Gateway/list_connections',
+            request_serializer=gateway__pb2.list_connections_req.SerializeToString,
+            response_deserializer=gateway__pb2.connections_info.FromString,
+        )
         self.create_listener = channel.unary_unary(
-                '/Gateway/create_listener',
-                request_serializer=gateway__pb2.create_listener_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/create_listener',
+            request_serializer=gateway__pb2.create_listener_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.delete_listener = channel.unary_unary(
-                '/Gateway/delete_listener',
-                request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
-        self.get_subsystems = channel.unary_unary(
-                '/Gateway/get_subsystems',
-                request_serializer=gateway__pb2.get_subsystems_req.SerializeToString,
-                response_deserializer=gateway__pb2.subsystems_info.FromString,
-                )
+            '/Gateway/delete_listener',
+            request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
+        self.list_listeners = channel.unary_unary(
+            '/Gateway/list_listeners',
+            request_serializer=gateway__pb2.list_listeners_req.SerializeToString,
+            response_deserializer=gateway__pb2.listeners_info.FromString,
+        )
+        self.list_subsystems = channel.unary_unary(
+            '/Gateway/list_subsystems',
+            request_serializer=gateway__pb2.list_subsystems_req.SerializeToString,
+            response_deserializer=gateway__pb2.subsystems_info.FromString,
+        )
         self.get_spdk_nvmf_log_flags_and_level = channel.unary_unary(
-                '/Gateway/get_spdk_nvmf_log_flags_and_level',
-                request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-                response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-                )
+            '/Gateway/get_spdk_nvmf_log_flags_and_level',
+            request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+            response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+        )
         self.disable_spdk_nvmf_logs = channel.unary_unary(
-                '/Gateway/disable_spdk_nvmf_logs',
-                request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/disable_spdk_nvmf_logs',
+            request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.set_spdk_nvmf_logs = channel.unary_unary(
-                '/Gateway/set_spdk_nvmf_logs',
-                request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/set_spdk_nvmf_logs',
+            request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.get_gateway_info = channel.unary_unary(
-                '/Gateway/get_gateway_info',
-                request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
-                response_deserializer=gateway__pb2.gateway_info.FromString,
-                )
+            '/Gateway/get_gateway_info',
+            request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
+            response_deserializer=gateway__pb2.gateway_info.FromString,
+        )
 
 
 class GatewayServicer(object):
     """Missing associated documentation comment in .proto file."""
 
-    def create_bdev(self, request, context):
-        """Creates a bdev from an RBD image
+    def namespace_add(self, request, context):
+        """Creates a namespace from an RBD image
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def resize_bdev(self, request, context):
-        """Resizes a bdev
+    def create_subsystem(self, request, context):
+        """Creates a subsystem
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def delete_bdev(self, request, context):
-        """Deletes a bdev
+    def delete_subsystem(self, request, context):
+        """Deletes a subsystem
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def create_subsystem(self, request, context):
-        """Creates a subsystem
+    def list_namespaces(self, request, context):
+        """List namespaces
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def delete_subsystem(self, request, context):
-        """Deletes a subsystem
+    def namespace_resize(self, request, context):
+        """Resizes a namespace
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def namespace_get_io_stats(self, request, context):
+        """Gets namespace's IO stats
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def namespace_set_qos_limits(self, request, context):
+        """Sets namespace's qos limits
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def add_namespace(self, request, context):
-        """Adds a namespace to a subsystem
+    def namespace_change_load_balancing_group(self, request, context):
+        """Changes namespace's load balancing group
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def remove_namespace(self, request, context):
-        """Removes a namespace from a subsystem
+    def namespace_delete(self, request, context):
+        """Deletes a namespace
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -162,6 +201,20 @@ def remove_host(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def list_hosts(self, request, context):
+        """List hosts
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def list_connections(self, request, context):
+        """List connections
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
     def create_listener(self, request, context):
         """Creates a listener for a subsystem at a given IP/Port
         """
@@ -176,8 +229,15 @@ def delete_listener(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def get_subsystems(self, request, context):
-        """Gets subsystems
+    def list_listeners(self, request, context):
+        """List listeners
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def list_subsystems(self, request, context):
+        """List subsystems
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -205,7 +265,7 @@ def set_spdk_nvmf_logs(self, request, context):
         raise NotImplementedError('Method not implemented!')
 
     def get_gateway_info(self, request, context):
-        """Set spdk nvmf logs
+        """Get gateway info
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -214,364 +274,475 @@ def get_gateway_info(self, request, context):
 
 def add_GatewayServicer_to_server(servicer, server):
     rpc_method_handlers = {
-            'create_bdev': grpc.unary_unary_rpc_method_handler(
-                    servicer.create_bdev,
-                    request_deserializer=gateway__pb2.create_bdev_req.FromString,
-                    response_serializer=gateway__pb2.bdev.SerializeToString,
-            ),
-            'resize_bdev': grpc.unary_unary_rpc_method_handler(
-                    servicer.resize_bdev,
-                    request_deserializer=gateway__pb2.resize_bdev_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'delete_bdev': grpc.unary_unary_rpc_method_handler(
-                    servicer.delete_bdev,
-                    request_deserializer=gateway__pb2.delete_bdev_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'create_subsystem': grpc.unary_unary_rpc_method_handler(
-                    servicer.create_subsystem,
-                    request_deserializer=gateway__pb2.create_subsystem_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'delete_subsystem': grpc.unary_unary_rpc_method_handler(
-                    servicer.delete_subsystem,
-                    request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'add_namespace': grpc.unary_unary_rpc_method_handler(
-                    servicer.add_namespace,
-                    request_deserializer=gateway__pb2.add_namespace_req.FromString,
-                    response_serializer=gateway__pb2.nsid_status.SerializeToString,
-            ),
-            'remove_namespace': grpc.unary_unary_rpc_method_handler(
-                    servicer.remove_namespace,
-                    request_deserializer=gateway__pb2.remove_namespace_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'add_host': grpc.unary_unary_rpc_method_handler(
-                    servicer.add_host,
-                    request_deserializer=gateway__pb2.add_host_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'remove_host': grpc.unary_unary_rpc_method_handler(
-                    servicer.remove_host,
-                    request_deserializer=gateway__pb2.remove_host_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'create_listener': grpc.unary_unary_rpc_method_handler(
-                    servicer.create_listener,
-                    request_deserializer=gateway__pb2.create_listener_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'delete_listener': grpc.unary_unary_rpc_method_handler(
-                    servicer.delete_listener,
-                    request_deserializer=gateway__pb2.delete_listener_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'get_subsystems': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_subsystems,
-                    request_deserializer=gateway__pb2.get_subsystems_req.FromString,
-                    response_serializer=gateway__pb2.subsystems_info.SerializeToString,
-            ),
-            'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_spdk_nvmf_log_flags_and_level,
-                    request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
-                    response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
-            ),
-            'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-                    servicer.disable_spdk_nvmf_logs,
-                    request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-                    servicer.set_spdk_nvmf_logs,
-                    request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'get_gateway_info': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_gateway_info,
-                    request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
-                    response_serializer=gateway__pb2.gateway_info.SerializeToString,
-            ),
+        'namespace_add': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_add,
+            request_deserializer=gateway__pb2.namespace_add_req.FromString,
+            response_serializer=gateway__pb2.nsid_status.SerializeToString,
+        ),
+        'create_subsystem': grpc.unary_unary_rpc_method_handler(
+            servicer.create_subsystem,
+            request_deserializer=gateway__pb2.create_subsystem_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'delete_subsystem': grpc.unary_unary_rpc_method_handler(
+            servicer.delete_subsystem,
+            request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_namespaces': grpc.unary_unary_rpc_method_handler(
+            servicer.list_namespaces,
+            request_deserializer=gateway__pb2.list_namespaces_req.FromString,
+            response_serializer=gateway__pb2.namespaces_info.SerializeToString,
+        ),
+        'namespace_resize': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_resize,
+            request_deserializer=gateway__pb2.namespace_resize_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_get_io_stats': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_get_io_stats,
+            request_deserializer=gateway__pb2.namespace_get_io_stats_req.FromString,
+            response_serializer=gateway__pb2.namespace_io_stats_info.SerializeToString,
+        ),
+        'namespace_set_qos_limits': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_set_qos_limits,
+            request_deserializer=gateway__pb2.namespace_set_qos_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_change_load_balancing_group': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_change_load_balancing_group,
+            request_deserializer=gateway__pb2.namespace_change_load_balancing_group_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_delete': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_delete,
+            request_deserializer=gateway__pb2.namespace_delete_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'add_host': grpc.unary_unary_rpc_method_handler(
+            servicer.add_host,
+            request_deserializer=gateway__pb2.add_host_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'remove_host': grpc.unary_unary_rpc_method_handler(
+            servicer.remove_host,
+            request_deserializer=gateway__pb2.remove_host_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_hosts': grpc.unary_unary_rpc_method_handler(
+            servicer.list_hosts,
+            request_deserializer=gateway__pb2.list_hosts_req.FromString,
+            response_serializer=gateway__pb2.hosts_info.SerializeToString,
+        ),
+        'list_connections': grpc.unary_unary_rpc_method_handler(
+            servicer.list_connections,
+            request_deserializer=gateway__pb2.list_connections_req.FromString,
+            response_serializer=gateway__pb2.connections_info.SerializeToString,
+        ),
+        'create_listener': grpc.unary_unary_rpc_method_handler(
+            servicer.create_listener,
+            request_deserializer=gateway__pb2.create_listener_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'delete_listener': grpc.unary_unary_rpc_method_handler(
+            servicer.delete_listener,
+            request_deserializer=gateway__pb2.delete_listener_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_listeners': grpc.unary_unary_rpc_method_handler(
+            servicer.list_listeners,
+            request_deserializer=gateway__pb2.list_listeners_req.FromString,
+            response_serializer=gateway__pb2.listeners_info.SerializeToString,
+        ),
+        'list_subsystems': grpc.unary_unary_rpc_method_handler(
+            servicer.list_subsystems,
+            request_deserializer=gateway__pb2.list_subsystems_req.FromString,
+            response_serializer=gateway__pb2.subsystems_info.SerializeToString,
+        ),
+        'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
+            servicer.get_spdk_nvmf_log_flags_and_level,
+            request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
+            response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
+        ),
+        'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+            servicer.disable_spdk_nvmf_logs,
+            request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+            servicer.set_spdk_nvmf_logs,
+            request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'get_gateway_info': grpc.unary_unary_rpc_method_handler(
+            servicer.get_gateway_info,
+            request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
+            response_serializer=gateway__pb2.gateway_info.SerializeToString,
+        ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-            'Gateway', rpc_method_handlers)
+        'Gateway', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
-
  # This class is part of an EXPERIMENTAL API.
-class Gateway(object):
-    """Missing associated documentation comment in .proto file."""
 
-    @staticmethod
-    def create_bdev(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/create_bdev',
-            gateway__pb2.create_bdev_req.SerializeToString,
-            gateway__pb2.bdev.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
-    @staticmethod
-    def resize_bdev(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/resize_bdev',
-            gateway__pb2.resize_bdev_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+class Gateway(object):
+    """Missing associated documentation comment in .proto file."""
 
     @staticmethod
-    def delete_bdev(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/delete_bdev',
-            gateway__pb2.delete_bdev_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+    def namespace_add(request,
+                      target,
+                      options=(),
+                      channel_credentials=None,
+                      call_credentials=None,
+                      insecure=False,
+                      compression=None,
+                      wait_for_ready=None,
+                      timeout=None,
+                      metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_add',
+                                             gateway__pb2.namespace_add_req.SerializeToString,
+                                             gateway__pb2.nsid_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_subsystem(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_subsystem',
-            gateway__pb2.create_subsystem_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.create_subsystem_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_subsystem(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_subsystem',
-            gateway__pb2.delete_subsystem_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.delete_subsystem_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def add_namespace(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/add_namespace',
-            gateway__pb2.add_namespace_req.SerializeToString,
-            gateway__pb2.nsid_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+    def list_namespaces(request,
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/list_namespaces',
+                                             gateway__pb2.list_namespaces_req.SerializeToString,
+                                             gateway__pb2.namespaces_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def remove_namespace(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/remove_namespace',
-            gateway__pb2.remove_namespace_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+    def namespace_resize(request,
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_resize',
+                                             gateway__pb2.namespace_resize_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def namespace_get_io_stats(request,
+                               target,
+                               options=(),
+                               channel_credentials=None,
+                               call_credentials=None,
+                               insecure=False,
+                               compression=None,
+                               wait_for_ready=None,
+                               timeout=None,
+                               metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_get_io_stats',
+                                             gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+                                             gateway__pb2.namespace_io_stats_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def namespace_set_qos_limits(request,
+                                 target,
+                                 options=(),
+                                 channel_credentials=None,
+                                 call_credentials=None,
+                                 insecure=False,
+                                 compression=None,
+                                 wait_for_ready=None,
+                                 timeout=None,
+                                 metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_set_qos_limits',
+                                             gateway__pb2.namespace_set_qos_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def namespace_change_load_balancing_group(request,
+                                              target,
+                                              options=(),
+                                              channel_credentials=None,
+                                              call_credentials=None,
+                                              insecure=False,
+                                              compression=None,
+                                              wait_for_ready=None,
+                                              timeout=None,
+                                              metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_change_load_balancing_group',
+                                             gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def namespace_delete(request,
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_delete',
+                                             gateway__pb2.namespace_delete_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def add_host(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                 target,
+                 options=(),
+                 channel_credentials=None,
+                 call_credentials=None,
+                 insecure=False,
+                 compression=None,
+                 wait_for_ready=None,
+                 timeout=None,
+                 metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/add_host',
-            gateway__pb2.add_host_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.add_host_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def remove_host(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                    target,
+                    options=(),
+                    channel_credentials=None,
+                    call_credentials=None,
+                    insecure=False,
+                    compression=None,
+                    wait_for_ready=None,
+                    timeout=None,
+                    metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/remove_host',
-            gateway__pb2.remove_host_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.remove_host_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def list_hosts(request,
+                   target,
+                   options=(),
+                   channel_credentials=None,
+                   call_credentials=None,
+                   insecure=False,
+                   compression=None,
+                   wait_for_ready=None,
+                   timeout=None,
+                   metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/list_hosts',
+                                             gateway__pb2.list_hosts_req.SerializeToString,
+                                             gateway__pb2.hosts_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def list_connections(request,
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/list_connections',
+                                             gateway__pb2.list_connections_req.SerializeToString,
+                                             gateway__pb2.connections_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_listener(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_listener',
-            gateway__pb2.create_listener_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.create_listener_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_listener(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_listener',
-            gateway__pb2.delete_listener_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.delete_listener_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def list_listeners(request,
+                       target,
+                       options=(),
+                       channel_credentials=None,
+                       call_credentials=None,
+                       insecure=False,
+                       compression=None,
+                       wait_for_ready=None,
+                       timeout=None,
+                       metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/list_listeners',
+                                             gateway__pb2.list_listeners_req.SerializeToString,
+                                             gateway__pb2.listeners_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def get_subsystems(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/Gateway/get_subsystems',
-            gateway__pb2.get_subsystems_req.SerializeToString,
-            gateway__pb2.subsystems_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+    def list_subsystems(request,
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/list_subsystems',
+                                             gateway__pb2.list_subsystems_req.SerializeToString,
+                                             gateway__pb2.subsystems_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_spdk_nvmf_log_flags_and_level(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                                          target,
+                                          options=(),
+                                          channel_credentials=None,
+                                          call_credentials=None,
+                                          insecure=False,
+                                          compression=None,
+                                          wait_for_ready=None,
+                                          timeout=None,
+                                          metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_spdk_nvmf_log_flags_and_level',
-            gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-            gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+                                             gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def disable_spdk_nvmf_logs(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                               target,
+                               options=(),
+                               channel_credentials=None,
+                               call_credentials=None,
+                               insecure=False,
+                               compression=None,
+                               wait_for_ready=None,
+                               timeout=None,
+                               metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/disable_spdk_nvmf_logs',
-            gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def set_spdk_nvmf_logs(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                           target,
+                           options=(),
+                           channel_credentials=None,
+                           call_credentials=None,
+                           insecure=False,
+                           compression=None,
+                           wait_for_ready=None,
+                           timeout=None,
+                           metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/set_spdk_nvmf_logs',
-            gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_gateway_info(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_gateway_info',
-            gateway__pb2.get_gateway_info_req.SerializeToString,
-            gateway__pb2.gateway_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.get_gateway_info_req.SerializeToString,
+                                             gateway__pb2.gateway_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/src/pybind/mgr/dashboard/tox.ini b/src/pybind/mgr/dashboard/tox.ini
index 47756e946e12..c40ab440f1dc 100644
--- a/src/pybind/mgr/dashboard/tox.ini
+++ b/src/pybind/mgr/dashboard/tox.ini
@@ -14,6 +14,7 @@ addopts =
     --cov --cov-append --cov-report=term
     --doctest-modules
     --ignore=frontend/ --ignore=module.py
+    --ignore=services/proto/
     --instafail
 
 [base]
@@ -70,6 +71,7 @@ exclude =
     .eggs,
     venv,
     frontend,
+    services/proto
 statistics = True
 #TODO: Uncomment and refactor (https://tracker.ceph.com/issues/41221)
 #max-complexity = 10

From 6c0417fbcbe6b9760b3836e5166d6bd929578096 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 30 Jan 2024 22:24:11 +1000
Subject: [PATCH 1685/2492] doc/architecture.rst - fix typo

s/requies/requires

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index 148e495ff1b3..7380d74a72a5 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -592,7 +592,7 @@ cluster map, the client doesn't know anything about object locations.**
 
 **Object locations must be computed.**
 
-The client requies only the object ID and the name of the pool in order to
+The client requires only the object ID and the name of the pool in order to
 compute the object location.
 
 Ceph stores data in named pools (for example,  "liverpool"). When a client

From 2128fd09dc3115c5bb6a92e2e900344f8faf0db2 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 8 Aug 2023 17:10:05 +0530
Subject: [PATCH 1686/2492] qa: test cases to make sure invalid paths don't get
 updated

Fixes: https://tracker.ceph.com/issues/62077
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 qa/tasks/cephfs/test_nfs.py | 73 +++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/qa/tasks/cephfs/test_nfs.py b/qa/tasks/cephfs/test_nfs.py
index 2d06cbac7baa..6d1c65dfb7df 100644
--- a/qa/tasks/cephfs/test_nfs.py
+++ b/qa/tasks/cephfs/test_nfs.py
@@ -411,6 +411,21 @@ def _nfs_export_apply(self, cluster, exports, raise_on_error=False):
                                     stdin=json.dumps(exports),
                                     stdout=StringIO(), stderr=StringIO())
 
+    def update_export(self, cluster_id, path, pseudo, fs_name):
+        self.ctx.cluster.run(args=['ceph', 'nfs', 'export', 'apply',
+                                   cluster_id, '-i', '-'],
+                             stdin=json.dumps({
+                                 "path": path,
+                                 "pseudo": pseudo,
+                                 "squash": "none",
+                                 "access_type": "rw",
+                                 "protocols": [4],
+                                 "fsal": {
+                                     "name": "CEPH",
+                                     "fs_name": fs_name
+                                 }
+                             }))
+
     def test_create_and_delete_cluster(self):
         '''
         Test successful creation and deletion of the nfs cluster.
@@ -1138,3 +1153,61 @@ def test_pseudo_path_in_json_response_when_updating_exports_failed(self):
         finally:
             self._delete_cluster_with_fs(self.fs_name, mnt_pt)
             self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}'])
+
+    def test_cephfs_export_update_with_nonexistent_dir(self):
+        """
+        Test that invalid path is not allowed while updating a CephFS
+        export.
+        """
+        self._create_cluster_with_fs(self.fs_name)
+        self._create_export(export_id=1)
+
+        try:
+            self.update_export(self.cluster_id, "/not_existent_dir",
+                               self.pseudo_path, self.fs_name)
+        except CommandFailedError as e:
+            if e.exitstatus != errno.ENOENT:
+                raise
+
+        self._delete_export()
+        self._delete_cluster_with_fs(self.fs_name)
+
+    def test_cephfs_export_update_at_non_dir_path(self):
+        """
+        Test that non-directory path are not allowed while updating a CephFS
+        export.
+        """
+        mnt_pt = '/mnt'
+        preserve_mode = self._sys_cmd(['stat', '-c', '%a', mnt_pt])
+        self._create_cluster_with_fs(self.fs_name, mnt_pt)
+        try:
+            self.ctx.cluster.run(args=['touch', f'{mnt_pt}/testfile'])
+            self._create_export(export_id=1)
+
+            # test at a file path
+            try:
+                self.update_export(self.cluster_id, "/testfile",
+                                   self.pseudo_path, self.fs_name)
+            except CommandFailedError as e:
+                if e.exitstatus != errno.ENOTDIR:
+                    raise
+
+            # test at a symlink path
+            self.ctx.cluster.run(args=['mkdir', f'{mnt_pt}/testdir'])
+            self.ctx.cluster.run(args=['ln', '-s', f'{mnt_pt}/testdir',
+                                       f'{mnt_pt}/testdir_symlink'])
+            try:
+                self.update_export(self.cluster_id, "/testdir_symlink",
+                                   self.pseudo_path, self.fs_name)
+            except CommandFailedError as e:
+                if e.exitstatus != errno.ENOTDIR:
+                    raise
+
+            # verify the path wasn't changed
+            export = json.loads(self._nfs_cmd("export", "ls",
+                                              self.cluster_id, "--detailed"))
+            self.assertEqual(export[0]["pseudo"], "/cephfs")
+
+        finally:
+            self.ctx.cluster.run(args=['rm', '-rf', f'{mnt_pt}/*'])
+            self._delete_cluster_with_fs(self.fs_name, mnt_pt, preserve_mode)

From 849a58b0873154a6713b4c8d838634f13b0b806d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 25 Jan 2024 12:41:39 -0500
Subject: [PATCH 1687/2492] qa/distros: add supported-container-hosts for
 cephadm and upgrade suites

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/distros/supported-container-hosts/centos_9.stream.yaml      | 1 +
 qa/distros/supported-container-hosts/centos_9.stream_runc.yaml | 1 +
 qa/distros/supported-container-hosts/ubuntu_22.04.yaml         | 1 +
 qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$       | 2 +-
 qa/suites/orch/cephadm/osds/0-distro                           | 2 +-
 qa/suites/orch/cephadm/smoke-roleless/0-distro                 | 2 +-
 qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$       | 2 +-
 qa/suites/orch/cephadm/smoke/0-distro                          | 2 +-
 qa/suites/orch/cephadm/thrash/0-distro                         | 2 +-
 qa/suites/orch/cephadm/with-work/0-distro                      | 2 +-
 qa/suites/orch/cephadm/workunits/0-distro                      | 2 +-
 qa/suites/upgrade/quincy-x/parallel/0-random-distro$           | 2 +-
 qa/suites/upgrade/quincy-x/stress-split/0-distro               | 2 +-
 qa/suites/upgrade/reef-x/parallel/0-random-distro$             | 2 +-
 qa/suites/upgrade/reef-x/stress-split/0-distro                 | 2 +-
 qa/suites/upgrade/telemetry-upgrade/quincy-x/0-random-distro$  | 2 +-
 16 files changed, 16 insertions(+), 13 deletions(-)
 create mode 120000 qa/distros/supported-container-hosts/centos_9.stream.yaml
 create mode 120000 qa/distros/supported-container-hosts/centos_9.stream_runc.yaml
 create mode 120000 qa/distros/supported-container-hosts/ubuntu_22.04.yaml

diff --git a/qa/distros/supported-container-hosts/centos_9.stream.yaml b/qa/distros/supported-container-hosts/centos_9.stream.yaml
new file mode 120000
index 000000000000..584cad6777d0
--- /dev/null
+++ b/qa/distros/supported-container-hosts/centos_9.stream.yaml
@@ -0,0 +1 @@
+../container-hosts/centos_9.stream.yaml
\ No newline at end of file
diff --git a/qa/distros/supported-container-hosts/centos_9.stream_runc.yaml b/qa/distros/supported-container-hosts/centos_9.stream_runc.yaml
new file mode 120000
index 000000000000..81b17ea14508
--- /dev/null
+++ b/qa/distros/supported-container-hosts/centos_9.stream_runc.yaml
@@ -0,0 +1 @@
+../container-hosts/centos_9.stream_runc.yaml
\ No newline at end of file
diff --git a/qa/distros/supported-container-hosts/ubuntu_22.04.yaml b/qa/distros/supported-container-hosts/ubuntu_22.04.yaml
new file mode 120000
index 000000000000..01afa8a63a24
--- /dev/null
+++ b/qa/distros/supported-container-hosts/ubuntu_22.04.yaml
@@ -0,0 +1 @@
+../container-hosts/ubuntu_22.04.yaml
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
+++ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/0-distro b/qa/suites/orch/cephadm/osds/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/osds/0-distro
+++ b/qa/suites/orch/cephadm/osds/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/0-distro b/qa/suites/orch/cephadm/smoke-roleless/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/smoke-roleless/0-distro
+++ b/qa/suites/orch/cephadm/smoke-roleless/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$ b/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$
+++ b/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/0-distro b/qa/suites/orch/cephadm/smoke/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/smoke/0-distro
+++ b/qa/suites/orch/cephadm/smoke/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/0-distro b/qa/suites/orch/cephadm/thrash/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/thrash/0-distro
+++ b/qa/suites/orch/cephadm/thrash/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/0-distro b/qa/suites/orch/cephadm/with-work/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/with-work/0-distro
+++ b/qa/suites/orch/cephadm/with-work/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/0-distro b/qa/suites/orch/cephadm/workunits/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/orch/cephadm/workunits/0-distro
+++ b/qa/suites/orch/cephadm/workunits/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/parallel/0-random-distro$ b/qa/suites/upgrade/quincy-x/parallel/0-random-distro$
index 4b341719de40..66187855738e 120000
--- a/qa/suites/upgrade/quincy-x/parallel/0-random-distro$
+++ b/qa/suites/upgrade/quincy-x/parallel/0-random-distro$
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/stress-split/0-distro b/qa/suites/upgrade/quincy-x/stress-split/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/upgrade/quincy-x/stress-split/0-distro
+++ b/qa/suites/upgrade/quincy-x/stress-split/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/parallel/0-random-distro$ b/qa/suites/upgrade/reef-x/parallel/0-random-distro$
index 4b341719de40..66187855738e 120000
--- a/qa/suites/upgrade/reef-x/parallel/0-random-distro$
+++ b/qa/suites/upgrade/reef-x/parallel/0-random-distro$
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/reef-x/stress-split/0-distro b/qa/suites/upgrade/reef-x/stress-split/0-distro
index 4b341719de40..66187855738e 120000
--- a/qa/suites/upgrade/reef-x/stress-split/0-distro
+++ b/qa/suites/upgrade/reef-x/stress-split/0-distro
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/upgrade/telemetry-upgrade/quincy-x/0-random-distro$ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/0-random-distro$
index 4b341719de40..66187855738e 120000
--- a/qa/suites/upgrade/telemetry-upgrade/quincy-x/0-random-distro$
+++ b/qa/suites/upgrade/telemetry-upgrade/quincy-x/0-random-distro$
@@ -1 +1 @@
-.qa/distros/container-hosts
\ No newline at end of file
+.qa/distros/supported-container-hosts
\ No newline at end of file

From 4da590009dbb1c153a47e7032f14bd95e53aec7d Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Thu, 23 Nov 2023 16:47:57 +0100
Subject: [PATCH 1688/2492] os/bluestore: use values instead of Buffer pointers
 in buffer_map

buffer_map is a map<int, unique_ptr<Buffer>>, therefore, when reading a
Buffer we will have 2 cache misses. This commit refactors buffer_map to
be a map<int, Buffer> so that Buffers are contigous in memory to improve
data locality and reduce cache misses

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc | 117 +++++++++++++++++-----------------
 src/os/bluestore/BlueStore.h  |  72 +++++++++++++--------
 2 files changed, 101 insertions(+), 88 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 3f33ff4eab01..32de7981f8b0 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -627,8 +627,8 @@ void _dump_extent_map(CephContext *cct, const BlueStore::ExtentMap &em)
     std::lock_guard l(e.blob->shared_blob->get_cache()->lock);
     for (auto& i : e.blob->get_bc().buffer_map) {
       dout(LogLevelV) << __func__ << "       0x" << std::hex << i.first
-		      << "~" << i.second->length << std::dec
-		      << " " << *i.second << dendl;
+		      << "~" << i.second.length << std::dec
+		      << " " << i.second << dendl;
     }
   }
 }
@@ -1714,7 +1714,7 @@ int BlueStore::BufferSpace::_discard(BufferCacheShard* cache, uint32_t offset, u
   auto i = _data_lower_bound(offset);
   uint32_t end = offset + length;
   while (i != buffer_map.end()) {
-    Buffer *b = i->second.get();
+    Buffer *b = &i->second;
     if (b->offset >= end) {
       break;
     }
@@ -1729,13 +1729,9 @@ int BlueStore::BufferSpace::_discard(BufferCacheShard* cache, uint32_t offset, u
 	if (b->data.length()) {
 	  bufferlist bl;
 	  bl.substr_of(b->data, b->length - tail, tail);
-	  Buffer *nb = new Buffer(this, b->state, b->seq, end, bl, b->flags);
-	  nb->maybe_rebuild();
-	  _add_buffer(cache, nb, 0, b);
+	  _add_buffer(cache, this, Buffer(this, b->state, b->seq, end, bl, b->flags), 0, 0, b);
 	} else {
-	  _add_buffer(cache, new Buffer(this, b->state, b->seq, end, tail,
-                                        b->flags),
-	              0, b);
+	  _add_buffer(cache, this, Buffer(this, b->state, b->seq, end, tail, b->flags), 0, 0, b);
 	}
 	if (!b->is_writing()) {
 	  cache->_adjust_size(b, front - (int64_t)b->length);
@@ -1765,13 +1761,11 @@ int BlueStore::BufferSpace::_discard(BufferCacheShard* cache, uint32_t offset, u
     if (b->data.length()) {
       bufferlist bl;
       bl.substr_of(b->data, b->length - keep, keep);
-      Buffer *nb = new Buffer(this, b->state, b->seq, end, bl, b->flags);
-      nb->maybe_rebuild();
-      _add_buffer(cache, nb, 0, b);
+      _add_buffer(cache, this,
+                  Buffer(this, b->state, b->seq, end, bl, b->flags), 0, 0, b);
     } else {
-      _add_buffer(cache, new Buffer(this, b->state, b->seq, end, keep,
-                                    b->flags),
-                  0, b);
+      _add_buffer(cache, this,
+                  Buffer(this, b->state, b->seq, end, keep, b->flags), 0, 0, b);
     }
     _rm_buffer(cache, i);
     cache->_audit("discard end 2");
@@ -1796,9 +1790,8 @@ void BlueStore::BufferSpace::read(
   {
     std::lock_guard l(cache->lock);
     for (auto i = _data_lower_bound(offset);
-         i != buffer_map.end() && offset < end && i->first < end;
-         ++i) {
-      Buffer *b = i->second.get();
+         i != buffer_map.end() && offset < end && i->first < end; ++i) {
+      Buffer *b = &i->second;
       ceph_assert(b->end() > offset);
 
       bool val = false;
@@ -1898,9 +1891,10 @@ bool BlueStore::BufferSpace::_dup_writing(BufferCacheShard* cache, BufferSpace*
     copied = true;
     for (auto it = writing.begin(); it != writing.end(); ++it) {
       Buffer& b = *it;
-      Buffer* to_b = new Buffer(to, b.state, b.seq, b.offset, b.data, b.flags);
-      ceph_assert(to_b->is_writing());
-      to->_add_buffer(cache, to_b, 0, nullptr);
+      ceph_assert(b.is_writing());
+      to->_add_buffer(cache, to,
+                      Buffer(to, b.state, b.seq, b.offset, b.data, b.flags), 0,
+                      0, nullptr);
     }
   }
   return copied;
@@ -1914,39 +1908,42 @@ void BlueStore::BufferSpace::split(BufferCacheShard* cache, size_t pos, BlueStor
 
   auto p = --buffer_map.end();
   while (true) {
-    if (p->second->end() <= pos)
-      break;
+    if (p->second.end() <= pos) break;
 
-    if (p->second->offset < pos) {
-      ldout(cache->cct, 30) << __func__ << " cut " << *p->second << dendl;
-      size_t left = pos - p->second->offset;
-      size_t right = p->second->length - left;
-      if (p->second->data.length()) {
-	bufferlist bl;
-	bl.substr_of(p->second->data, left, right);
-	r._add_buffer(cache, new Buffer(&r, p->second->state, p->second->seq,
-                                        0, bl, p->second->flags),
-		      0, p->second.get());
+    if (p->second.offset < pos) {
+      ldout(cache->cct, 30) << __func__ << " cut " << p->second << dendl;
+      size_t left = pos - p->second.offset;
+      size_t right = p->second.length - left;
+      if (p->second.data.length()) {
+        bufferlist bl;
+        bl.substr_of(p->second.data, left, right);
+        r._add_buffer(
+            cache, &r,
+            Buffer(&r, p->second.state, p->second.seq, 0, bl, p->second.flags),
+            0, 0, &p->second);
       } else {
-	r._add_buffer(cache, new Buffer(&r, p->second->state, p->second->seq,
-                                        0, right, p->second->flags),
-		      0, p->second.get());
+        r._add_buffer(cache, &r, Buffer(&r, p->second.state, p->second.seq, 0, right,
+                      p->second.flags), 0, 0, &p->second);
       }
-      cache->_adjust_size(p->second.get(), -right);
-      p->second->truncate(left);
+      cache->_adjust_size(&p->second, -right);
+      p->second.truncate(left);
       break;
     }
 
-    ceph_assert(p->second->end() > pos);
-    ldout(cache->cct, 30) << __func__ << " move " << *p->second << dendl;
-    if (p->second->data.length()) {
-      r._add_buffer(cache, new Buffer(&r, p->second->state, p->second->seq,
-                               p->second->offset - pos, p->second->data, p->second->flags),
-                    0, p->second.get());
+    ceph_assert(p->second.end() > pos);
+    ldout(cache->cct, 30) << __func__ << " move " << p->second << dendl;
+    if (p->second.data.length()) {
+      r._add_buffer(cache, &r,
+                    Buffer(&r, p->second.state, p->second.seq,
+                                     p->second.offset - pos, p->second.data,
+                                     p->second.flags),
+                    0, 0, &p->second);
     } else {
-      r._add_buffer(cache, new Buffer(&r, p->second->state, p->second->seq,
-                               p->second->offset - pos, p->second->length, p->second->flags),
-                    0, p->second.get());
+      r._add_buffer(cache, &r,
+                    Buffer(&r, p->second.state, p->second.seq,
+                                     p->second.offset - pos, p->second.length,
+                                     p->second.flags),
+                    0, 0, &p->second);
     }
     if (p == buffer_map.begin()) {
       _rm_buffer(cache, p);
@@ -1964,7 +1961,7 @@ void BlueStore::BufferSpace::split(BufferCacheShard* cache, size_t pos, BlueStor
 std::ostream& operator<<(std::ostream& out, const BlueStore::BufferSpace& bc)
 {
   for (auto& [i, j] : bc.buffer_map) {
-    out << " [0x" << std::hex << i << "]=" << *j << std::dec;
+    out << " [0x" << std::hex << i << "]=" << j << std::dec;
   }
   if (!bc.writing.empty()) {
     out << " writing:";
@@ -2908,9 +2905,9 @@ uint32_t BlueStore::Blob::merge_blob(CephContext* cct, Blob* blob_to_dissolve)
   // move BufferSpace buffers
   while(!src->bc.buffer_map.empty()) {
     auto buf = src->bc.buffer_map.extract(src->bc.buffer_map.cbegin());
-    buf.mapped()->space = &dst->bc;
+    buf.mapped().space = &dst->bc;
     if (dst->bc.buffer_map.count(buf.key()) == 0) {
-      dst->bc.buffer_map[buf.key()] = std::move(buf.mapped());
+      dst->bc.buffer_map.insert({buf.key(), std::move(buf.mapped())});
     }
   }
   // move BufferSpace writing
@@ -5202,12 +5199,12 @@ void BlueStore::Collection::split_cache(
 
       auto rehome_blob = [&](Blob* b) {
 	for (auto& i : b->bc.buffer_map) {
-	  if (!i.second->is_writing()) {
-	    ldout(store->cct, 1) << __func__ << "   moving " << *i.second
+	  if (!i.second.is_writing()) {
+	    ldout(store->cct, 1) << __func__ << "   moving " << i.second
 				 << dendl;
-	    dest->cache->_move(cache, i.second.get());
+	    dest->cache->_move(cache, &i.second);
 	  } else {
-	    ldout(store->cct, 1) << __func__ << "   not moving " << *i.second
+	    ldout(store->cct, 1) << __func__ << "   not moving " << i.second
 				 << dendl;
 	  }
 	}
@@ -5236,13 +5233,13 @@ void BlueStore::Collection::split_cache(
 	b.second->last_encoded_id = -1;
       }
       for (auto& e : o->extent_map.extent_map) {
-	cache->rm_extent();
-	dest->cache->add_extent();
+        cache->rm_extent();
+        dest->cache->add_extent();
 	Blob* tb = e.blob.get();
-	if (tb->last_encoded_id == -1) {
-	  rehome_blob(tb);
-	  tb->last_encoded_id = 0;
-	}
+        if (tb->last_encoded_id == -1) {
+          rehome_blob(tb);
+          tb->last_encoded_id = 0;
+        }
       }
       for (auto& b : o->extent_map.spanning_blob_map) {
 	Blob* tb = b.second.get();
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 44fd8b2e809b..4b157f86502c 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -17,6 +17,7 @@
 
 #include "acconfig.h"
 
+#include <tuple>
 #include <unistd.h>
 
 #include <atomic>
@@ -32,6 +33,7 @@
 #include <boost/functional/hash.hpp>
 #include <boost/dynamic_bitset.hpp>
 #include <boost/circular_buffer.hpp>
+#include <utility>
 
 #include "include/cpp-btree/btree_set.h"
 
@@ -366,7 +368,7 @@ class BlueStore : public ObjectStore,
 	boost::intrusive::list_member_hook<>,
 	&Buffer::state_item> > state_list_t;
 
-    mempool::bluestore_cache_meta::map<uint32_t, std::unique_ptr<Buffer>>
+    mempool::bluestore_cache_meta::map<uint32_t, Buffer>
       buffer_map;
 
     // we use a bare intrusive list here instead of std::map because
@@ -379,56 +381,63 @@ class BlueStore : public ObjectStore,
       ceph_assert(writing.empty());
     }
 
-    void _add_buffer(BufferCacheShard* cache, Buffer* b, int level, Buffer* near) {
+    void _add_buffer(BufferCacheShard *cache, BufferSpace *space, Buffer&& buffer,
+                     uint16_t cache_private, int level, Buffer *near) {
+      auto it = buffer_map.emplace(buffer.offset, std::move(buffer));
+      Buffer *cached_buffer = &it.first->second;
+      cached_buffer->cache_private = cache_private;
+      _add_buffer(cache, space, cached_buffer, level, near);
+    }
+
+    void _add_buffer(BufferCacheShard *cache, BufferSpace *space,
+                     Buffer *buffer, int level, Buffer *near) {
       cache->_audit("_add_buffer start");
-      buffer_map[b->offset].reset(b);
-      if (b->is_writing()) {
+      if (buffer->is_writing()) {
         // we might get already cached data for which resetting mempool is inppropriate
         // hence calling try_assign_to_mempool
-        b->data.try_assign_to_mempool(mempool::mempool_bluestore_writing);
-        if (writing.empty() || writing.rbegin()->seq <= b->seq) {
-          writing.push_back(*b);
+        buffer->data.try_assign_to_mempool(mempool::mempool_bluestore_writing);
+        if (writing.empty() || writing.rbegin()->seq <= buffer->seq) {
+          writing.push_back(*buffer);
         } else {
           auto it = writing.begin();
-          while (it->seq < b->seq) {
+          while (it->seq < buffer->seq) {
             ++it;
           }
 
-          ceph_assert(it->seq >= b->seq);
+          ceph_assert(it->seq >= buffer->seq);
           // note that this will insert b before it
           // hence the order is maintained
-          writing.insert(it, *b);
+          writing.insert(it, *buffer);
         }
       } else {
-        b->data.reassign_to_mempool(mempool::mempool_bluestore_cache_data);
-        cache->_add(b, level, near);
+        buffer->data.reassign_to_mempool(mempool::mempool_bluestore_cache_data);
+        cache->_add(buffer, level, near);
       }
       cache->_audit("_add_buffer end");
     }
     void _rm_buffer(BufferCacheShard* cache, Buffer *b) {
       _rm_buffer(cache, buffer_map.find(b->offset));
     }
-    std::map<uint32_t, std::unique_ptr<Buffer>>::iterator
+    std::map<uint32_t, Buffer>::iterator
     _rm_buffer(BufferCacheShard* cache,
-		    std::map<uint32_t, std::unique_ptr<Buffer>>::iterator p) {
+		    std::map<uint32_t, Buffer>::iterator p) {
       ceph_assert(p != buffer_map.end());
       cache->_audit("_rm_buffer start");
-      if (p->second->is_writing()) {
-        writing.erase(writing.iterator_to(*p->second));
+      if (p->second.is_writing()) {
+        writing.erase(writing.iterator_to(p->second));
       } else {
-	cache->_rm(p->second.get());
+	cache->_rm(&p->second);
       }
       p = buffer_map.erase(p);
       cache->_audit("_rm_buffer end");
       return p;
     }
 
-    std::map<uint32_t,std::unique_ptr<Buffer>>::iterator _data_lower_bound(
-      uint32_t offset) {
+    std::map<uint32_t, Buffer>::iterator _data_lower_bound(uint32_t offset) {
       auto i = buffer_map.lower_bound(offset);
       if (i != buffer_map.begin()) {
 	--i;
-	if (i->first + i->second->length <= offset)
+	if (i->first + i->second.length <= offset)
 	  ++i;
       }
       return i;
@@ -449,18 +458,25 @@ class BlueStore : public ObjectStore,
     void write(BufferCacheShard* cache, uint64_t seq, uint32_t offset, ceph::buffer::list& bl,
 	       unsigned flags) {
       std::lock_guard l(cache->lock);
-      Buffer *b = new Buffer(this, Buffer::STATE_WRITING, seq, offset, bl,
+      Buffer b(this, Buffer::STATE_WRITING, seq, offset, bl,
 			     flags);
-      b->cache_private = _discard(cache, offset, bl.length());
-      _add_buffer(cache, b, (flags & Buffer::FLAG_NOCACHE) ? 0 : 1, nullptr);
+      uint16_t cache_private = _discard(cache, offset, bl.length());
+      _add_buffer(cache, this,
+                  Buffer(this, Buffer::STATE_WRITING, seq, offset, bl,
+                                   flags),
+                  cache_private, (flags & Buffer::FLAG_NOCACHE) ? 0 : 1,
+                  nullptr);
       cache->_trim();
     }
     void _finish_write(BufferCacheShard* cache, uint64_t seq);
     void did_read(BufferCacheShard* cache, uint32_t offset, ceph::buffer::list& bl) {
       std::lock_guard l(cache->lock);
-      Buffer *b = new Buffer(this, Buffer::STATE_CLEAN, 0, offset, bl);
-      b->cache_private = _discard(cache, offset, bl.length());
-      _add_buffer(cache, b, 1, nullptr);
+      Buffer b(this, Buffer::STATE_CLEAN, 0, offset, bl);
+      uint16_t cache_private = _discard(cache, offset, bl.length());
+      _add_buffer(
+          cache, this,
+          Buffer(this, Buffer::STATE_CLEAN, 0, offset, bl, 0),
+          cache_private, 1, nullptr);
       cache->_trim();
     }
 
@@ -481,8 +497,8 @@ class BlueStore : public ObjectStore,
       f->open_array_section("buffers");
       for (auto& i : buffer_map) {
 	f->open_object_section("buffer");
-	ceph_assert(i.first == i.second->offset);
-	i.second->dump(f);
+	ceph_assert(i.first == i.second.offset);
+	i.second.dump(f);
 	f->close_section();
       }
       f->close_section();

From efbb1ac90668c2e726ba008e4d79810a78ac4d3b Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Tue, 2 Jan 2024 12:08:45 +0100
Subject: [PATCH 1689/2492] os/bluestore: Buffer move ctor

Introduce move constructor to Buffers.

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 4b157f86502c..44dace5f4906 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -313,6 +313,20 @@ class BlueStore : public ObjectStore,
       : space(space), state(s), flags(f), seq(q), offset(o),
 	length(b.length()), data(b) {}
 
+    Buffer(Buffer &&other) {
+      std::swap(space, other.space);
+      std::swap(state, other.state);
+      std::swap(cache_private, other.cache_private);
+      std::swap(flags, other.flags);
+      std::swap(seq, other.seq);
+      std::swap(offset, other.offset);
+      std::swap(length, other.length);
+      std::swap(data, other.data);
+      std::swap(cache_age_bin, other.cache_age_bin);
+      lru_item.swap_nodes(other.lru_item);
+      state_item.swap_nodes(other.state_item);
+    }
+
     bool is_empty() const {
       return state == STATE_EMPTY;
     }

From 8d915fe1e79d7797baf8318075e2bd8969f75826 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Tue, 2 Jan 2024 12:28:21 +0100
Subject: [PATCH 1690/2492] os/bluestore: remove no-op buffer creation

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 44dace5f4906..f40c636e072f 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -472,8 +472,6 @@ class BlueStore : public ObjectStore,
     void write(BufferCacheShard* cache, uint64_t seq, uint32_t offset, ceph::buffer::list& bl,
 	       unsigned flags) {
       std::lock_guard l(cache->lock);
-      Buffer b(this, Buffer::STATE_WRITING, seq, offset, bl,
-			     flags);
       uint16_t cache_private = _discard(cache, offset, bl.length());
       _add_buffer(cache, this,
                   Buffer(this, Buffer::STATE_WRITING, seq, offset, bl,
@@ -485,7 +483,6 @@ class BlueStore : public ObjectStore,
     void _finish_write(BufferCacheShard* cache, uint64_t seq);
     void did_read(BufferCacheShard* cache, uint32_t offset, ceph::buffer::list& bl) {
       std::lock_guard l(cache->lock);
-      Buffer b(this, Buffer::STATE_CLEAN, 0, offset, bl);
       uint16_t cache_private = _discard(cache, offset, bl.length());
       _add_buffer(
           cache, this,

From 51bda540d63059bc8053a1c0f6f7a4315291f46f Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 24 Jan 2024 15:08:14 +0000
Subject: [PATCH 1691/2492] mgr/cephadm: add a new config option
 'oob_default_addr'

So there's a default value (169.254.1.1) which is the default
address for the 'OS to iDrac pass-through' interface.
Given that node-proxy will reach the RedFish API through this interface,
we can make users avoid to pass that addr when providing the host spec
at bootstrap time.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 5c5e602e944f..1451943ab28a 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -495,6 +495,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             desc="Destination for cephadm command's persistent logging",
             enum_allowed=['file', 'syslog', 'file,syslog'],
         ),
+        Option(
+            'oob_default_addr',
+            type='str',
+            default='169.254.1.1',
+            desc="Default address for RedFish API (oob management)."
+        ),
     ]
 
     def __init__(self, *args: Any, **kwargs: Any):
@@ -579,6 +585,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.log_refresh_metadata = False
             self.default_cephadm_command_timeout = 0
             self.cephadm_log_destination = ''
+            self.oob_default_addr = ''
 
         self.notify(NotifyType.mon_map, None)
         self.config_notify()
@@ -1631,7 +1638,7 @@ def _add_host(self, spec):
 
         if spec.oob:
             if not spec.oob.get('addr'):
-                spec.oob['addr'] = spec.hostname
+                spec.oob['addr'] = self.oob_default_addr
             if not spec.oob.get('port'):
                 spec.oob['port'] = '443'
             host_oob_info = dict()

From 8ebd93da1b8f147da7e261fdbf352a867bc8b0dd Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Thu, 25 Jan 2024 14:34:26 +0000
Subject: [PATCH 1692/2492] mgr/cephadm: update node-proxy unit tests

The recent migration to a separate daemon implied
some changes which have broken these tests.
This commit fixes them.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 .../mgr/cephadm/tests/test_node_proxy.py      | 100 ++++++------------
 1 file changed, 33 insertions(+), 67 deletions(-)

diff --git a/src/pybind/mgr/cephadm/tests/test_node_proxy.py b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
index 0c9ee127547c..b19bb5dbc50c 100644
--- a/src/pybind/mgr/cephadm/tests/test_node_proxy.py
+++ b/src/pybind/mgr/cephadm/tests/test_node_proxy.py
@@ -1,6 +1,7 @@
 import cherrypy
 import json
 from _pytest.monkeypatch import MonkeyPatch
+from urllib.error import URLError
 from cherrypy.test import helper
 from cephadm.agent import NodeProxyEndpoint
 from unittest.mock import MagicMock, call, patch
@@ -23,6 +24,7 @@ def __init__(self) -> None:
         self.agent_cache.agent_ports = {"host01": 1234}
         self.node_proxy_cache = NodeProxyCache(self)
         self.node_proxy_cache.save = MagicMock()
+        self.node_proxy = MagicMock()
         self.http_server = MagicMock()
         self.http_server.agent = MagicMock()
         self.http_server.agent.ssl_certs = SSLCerts()
@@ -35,13 +37,13 @@ def get_mgr_ip(self) -> str:
 class TestNodeProxyEndpoint(helper.CPWebCase):
     mgr = FakeMgr()
     app = NodeProxyEndpoint(mgr)
-    mgr.node_proxy.keyrings = {"host01": "fake-secret01",
-                               "host02": "fake-secret02"}
-    mgr.node_proxy.oob = {"host01": {"username": "oob-user01",
-                                     "password": "oob-pass01"},
-                          "host02": {"username": "oob-user02",
-                                     "password": "oob-pass02"}}
-    mgr.node_proxy.data = node_proxy_data.full_set
+    mgr.node_proxy_cache.keyrings = {"host01": "fake-secret01",
+                                     "host02": "fake-secret02"}
+    mgr.node_proxy_cache.oob = {"host01": {"username": "oob-user01",
+                                           "password": "oob-pass01"},
+                                "host02": {"username": "oob-user02",
+                                           "password": "oob-pass02"}}
+    mgr.node_proxy_cache.data = node_proxy_data.full_set
 
     @classmethod
     def setup_server(cls):
@@ -117,11 +119,6 @@ def test_data_raises_alert(self):
 
         assert TestNodeProxyEndpoint.mgr.set_health_warning.mock_calls == calls
 
-    # @pytest.mark.parametrize("method", ["GET", "PATCH"])
-    # def test_led_no_hostname(self, method):
-    #     self.getPage("/led", method=method)
-    #     self.assertStatus('501 Not Implemented')
-
     def test_led_GET_no_hostname(self):
         self.getPage("/led", method="GET")
         self.assertStatus('501 Not Implemented')
@@ -132,20 +129,16 @@ def test_led_PATCH_no_hostname(self):
                                                                  ('Content-Length', str(len(data)))])
         self.assertStatus('501 Not Implemented')
 
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_set_led_no_type(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
-        data = '{"IndicatorLED": "Blinking"}'
+    def test_set_led_no_type(self):
+        data = '{"state": "on", "keyring": "fake-secret01"}'
         self.getPage("/host01/led", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
                                                                         ('Content-Length', str(len(data)))])
         self.assertStatus('400 Bad Request')
 
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_set_chassis_led(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
-        data = '{"IndicatorLED": "Blinking"}'
-        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
-            a.return_value = '{"http_code": 200}'
+    def test_set_chassis_led(self):
+        data = '{"state": "on", "keyring": "fake-secret01"}'
+        with patch('cephadm.agent.http_req') as p:
+            p.return_value = [], '{}', 200
             self.getPage("/host01/led/chassis", method="PATCH", body=data, headers=[('Content-Type', 'application/json'),
                                                                                     ('Content-Length', str(len(data)))])
             self.assertStatus('200 OK')
@@ -170,70 +163,43 @@ def test_get_led_type_drive_missing_id(self):
         self.getPage("/host01/led/drive", method="GET")
         self.assertStatus('400 Bad Request')
 
-    def test_get_led_type_chassis_answer_invalid_json(self):
-        self.getPage("/host01/led/chassis", method="GET")
-        self.assertStatus('503 Service Unavailable')
+    def test_get_led_url_error(self):
+        with patch('cephadm.agent.http_req') as p:
+            p.side_effect = URLError('fake error')
+            self.getPage("/host02/led/chassis", method="GET")
+            self.assertStatus('502 Bad Gateway')
 
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_get_led_type_chassis_answer_no_http_code(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
-        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
-            a.return_value = '{"foo": "bar"}'
-            self.getPage("/host01/led/chassis", method="GET")
-            self.assertStatus('503 Service Unavailable')
-
-    def test_get_led_status_not_200(self):
-        self.getPage("/host01/led/chassis", method="GET")
-        self.assertStatus('503 Service Unavailable')
-
-    def test_get_led_key_error(self):
-        self.getPage("/host02/led/chassis", method="GET")
-        self.assertStatus('502 Bad Gateway')
-
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_get_chassis_led_ok(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
-        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
-            a.return_value = '{"http_code": 200}'
+    def test_get_chassis_led_ok(self):
+        with patch('cephadm.agent.http_req', return_value=MagicMock()) as p:
+            p.return_value = [], '{}', 200
             self.getPage("/host01/led/chassis", method="GET")
             self.assertStatus('200 OK')
 
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_get_drive_led_without_id(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
+    def test_get_drive_led_without_id(self):
         self.getPage("/host01/led/drive", method="GET")
         self.assertStatus('400 Bad Request')
 
-    @patch('cephadm.agent.AgentMessageThread.join', return_value=MagicMock)
-    @patch('cephadm.agent.AgentMessageThread.start', return_value=MagicMock)
-    def test_get_drive_led_with_id(self, m_agent_msg_thread_start, m_agent_msg_thread_join):
-        with patch('cephadm.agent.AgentMessageThread.get_agent_response') as a:
-            a.return_value = '{"http_code": 200}'
+    def test_get_drive_led_with_id(self):
+        with patch('cephadm.agent.http_req', return_value=MagicMock()) as p:
+            p.return_value = [], '{}', 200
             self.getPage("/host01/led/drive/123", method="GET")
             self.assertStatus('200 OK')
 
-    # def test_led_endpoint_unreachable(self):
-    #     TestNodeProxyEndpoint.app.query_endpoint = MagicMock(side_effect=URLError("fake-error"))
-    #     self.getPage("/host02/led", method="GET")
-    #     calls = [call(addr='10.10.10.12',
-    #                   data=None,
-    #                   endpoint='/led',
-    #                   headers={},
-    #                   method='GET',
-    #                   port=8080,
-    #                   ssl_ctx=TestNodeProxyEndpoint.app.ssl_ctx)]
-    #     self.assertStatus('502 Bad Gateway')
-    #     assert TestNodeProxyEndpoint.app.query_endpoint.mock_calls == calls
-
     def test_fullreport_with_valid_hostname(self):
+        # data = '{"cephx": {"name": "node-proxy.host01", "secret": "fake-secret01"}}'
+        # self.getPage("/host02/fullreport", method="POST", body=data, headers=[('Content-Type', 'application/json'), ('Content-Length', str(len(data)))])
         self.getPage("/host02/fullreport", method="GET")
         self.assertStatus('200 OK')
 
     def test_fullreport_no_hostname(self):
+        # data = '{"cephx": {"name": "node-proxy.host01", "secret": "fake-secret01"}}'
+        # self.getPage("/fullreport", method="POST", body=data, headers=[('Content-Type', 'application/json'), ('Content-Length', str(len(data)))])
         self.getPage("/fullreport", method="GET")
         self.assertStatus('200 OK')
 
     def test_fullreport_with_invalid_hostname(self):
+        # data = '{"cephx": {"name": "node-proxy.host03", "secret": "fake-secret03"}}'
+        # self.getPage("/host03/fullreport", method="POST", body=data, headers=[('Content-Type', 'application/json'), ('Content-Length', str(len(data)))])
         self.getPage("/host03/fullreport", method="GET")
         self.assertStatus('404 Not Found')
 

From 9a949f1ad74124aa5519c115bc2cf8cbdadc68e9 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 30 Jan 2024 16:38:31 +0000
Subject: [PATCH 1693/2492] agent/node-proxy: fix wrong host name used in data
 endpoint

data['cephx']['name'] will return something like:

node-proxy.hostname123

the prefix "node-proxy." has the be removed otherwise there will be
a mismatch between what is actually expected.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/agent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index 68495d3bc061..5fbf309c544c 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -264,7 +264,7 @@ def data(self) -> None:
         self.validate_node_proxy_data(data)
         if 'patch' not in data.keys():
             raise cherrypy.HTTPError(400, 'Malformed data received.')
-        host = data['cephx']['name']
+        host = data['cephx']['name'][11:]
         self.mgr.node_proxy_cache.save(host, data['patch'])
         self.raise_alert(data)
 

From ff6c339cfafa7feee2b1fe58f89e880a11aa213e Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 30 Jan 2024 11:32:20 -0600
Subject: [PATCH 1694/2492] .github/workflows: revert version bump to labler
 action

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 .github/workflows/pr-triage.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml
index c41795900e47..24b1c97db708 100644
--- a/.github/workflows/pr-triage.yml
+++ b/.github/workflows/pr-triage.yml
@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Assign labels based on modified files
-        # https://github.com/marketplace/actions/labeler?version=v5.0.0
-        uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9
+        # https://github.com/marketplace/actions/labeler?version=v4.0.2
+        uses: actions/labeler@5c7539237e04b714afd8ad9b4aed733815b9fab4
         with:
-          sync-labels: false
+          sync-labels: ''
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Assign to Dashboard project
         # https://github.com/marketplace/actions/add-to-github-projects?version=v0.5.0

From 8d7eb9f7e07bc91d0e887ffa3baefafa71bccd10 Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Tue, 26 Dec 2023 10:41:18 +0200
Subject: [PATCH 1695/2492] osd: Add 'read_ratio' pool parameterr

This parameter is used for better read balancing with non identical
devices.
- This parameter is controlled using the commands 'ceph osd pool set/get'
- This parameter is applicable only for replicated pools
- Valid values are integers in the range [0..100] and represent the
  percentage of read IOs out of all IOs in the pool
  - Value of 0 unsets this parameter and the value will be the default
    value (this is the generic behavior of the command 'ceph osd pool
    set'
  - default value can be set by config parameter
    `osd_pool_default_read_ratio`

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/common/options/global.yaml.in | 12 ++++++++++
 src/mon/MonCommands.h             |  4 ++--
 src/mon/OSDMonitor.cc             | 37 +++++++++++++++++++++++++++++--
 src/osd/osd_types.cc              |  4 +++-
 src/osd/osd_types.h               |  1 +
 5 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 88c896f90262..cebf59304a6e 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -2550,6 +2550,18 @@ options:
   - mon
   flags:
   - runtime
+- name: osd_pool_default_read_ratio
+  type: uint
+  level: advanced
+  desc: Default read ratio (the percent of read IOs out of all IOs) for a pool.
+  long_desc: Default read ratio (the percent of read IOs out of all IOs) for a pool.
+    applicable to replicated pools only. This value is used to improve read balancing
+    when OSDs have different weights.
+  default: 70
+  services:
+  - mon
+  flags:
+  - runtime
 - name: osd_erasure_code_plugins
   type: str
   level: advanced
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 954dcb077fbc..14bb3602c9b5 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1137,11 +1137,11 @@ COMMAND("osd pool rename "
 	"rename <srcpool> to <destpool>", "osd", "rw")
 COMMAND("osd pool get "
 	"name=pool,type=CephPoolname "
-	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|eio|bulk",
+	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|erasure_code_profile|min_read_recency_for_promote|all|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|eio|bulk|read_ratio",
 	"get pool parameter <var>", "osd", "r")
 COMMAND("osd pool set "
 	"name=pool,type=CephPoolname "
-	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|eio|bulk "
+	"name=var,type=CephChoices,strings=size|min_size|pg_num|pgp_num|pgp_num_actual|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|write_fadvise_dontneed|noscrub|nodeep-scrub|hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|use_gmt_hitset|target_max_bytes|target_max_objects|cache_target_dirty_ratio|cache_target_dirty_high_ratio|cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|min_read_recency_for_promote|min_write_recency_for_promote|fast_read|hit_set_grade_decay_rate|hit_set_search_last_n|scrub_min_interval|scrub_max_interval|deep_scrub_interval|recovery_priority|recovery_op_priority|scrub_priority|compression_mode|compression_algorithm|compression_required_ratio|compression_max_blob_size|compression_min_blob_size|csum_type|csum_min_block|csum_max_block|allow_ec_overwrites|fingerprint_algorithm|pg_autoscale_mode|pg_autoscale_bias|pg_num_min|pg_num_max|target_size_bytes|target_size_ratio|dedup_tier|dedup_chunk_algorithm|dedup_cdc_chunk_size|eio|bulk|read_ratio "
 	"name=val,type=CephString "
 	"name=yes_i_really_mean_it,type=CephBool,req=false",
 	"set pool parameter <var> to <val>", "osd", "rw")
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f8e379326f25..37cb3033189f 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -5398,7 +5398,7 @@ namespace {
     CSUM_TYPE, CSUM_MAX_BLOCK, CSUM_MIN_BLOCK, FINGERPRINT_ALGORITHM,
     PG_AUTOSCALE_MODE, PG_NUM_MIN, TARGET_SIZE_BYTES, TARGET_SIZE_RATIO,
     PG_AUTOSCALE_BIAS, DEDUP_TIER, DEDUP_CHUNK_ALGORITHM, 
-    DEDUP_CDC_CHUNK_SIZE, POOL_EIO, BULK, PG_NUM_MAX };
+    DEDUP_CDC_CHUNK_SIZE, POOL_EIO, BULK, PG_NUM_MAX, READ_RATIO };
 
   std::set<osd_pool_get_choices>
     subtract_second_from_first(const std::set<osd_pool_get_choices>& first,
@@ -6148,7 +6148,8 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
       {"dedup_tier", DEDUP_TIER},
       {"dedup_chunk_algorithm", DEDUP_CHUNK_ALGORITHM},
       {"dedup_cdc_chunk_size", DEDUP_CDC_CHUNK_SIZE},
-      {"bulk", BULK}
+      {"bulk", BULK},
+      {"read_ratio", READ_RATIO}
     };
 
     typedef std::set<osd_pool_get_choices> choices_set_t;
@@ -6165,6 +6166,9 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
     const choices_set_t ONLY_ERASURE_CHOICES = {
       EC_OVERWRITES, ERASURE_CODE_PROFILE
     };
+    const choices_set_t ONLY_REPLICA_CHOICES = {
+      READ_RATIO
+    };
 
     choices_set_t selected_choices;
     if (var == "all") {
@@ -6182,6 +6186,10 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
 	selected_choices = subtract_second_from_first(selected_choices,
 						      ONLY_ERASURE_CHOICES);
       }
+      if(!p->is_replicated()) {
+        selected_choices = subtract_second_from_first(selected_choices,
+						      ONLY_REPLICA_CHOICES);
+      }
     } else /* var != "all" */  {
       choices_map_t::const_iterator found = ALL_CHOICES.find(var);
       if (found == ALL_CHOICES.end()) {
@@ -6210,6 +6218,15 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
 	goto reply;
       }
 
+      if (!p->is_replicated() &&
+	  ONLY_REPLICA_CHOICES.find(selected)
+	  != ONLY_REPLICA_CHOICES.end()) {
+	ss << "pool '" << poolstr
+	   << "' is not a replicated pool: variable not applicable";
+	r = -EACCES;
+	goto reply;
+      }
+
       if (pool_opts_t::is_opt_name(var) &&
 	  !p->opts.is_set(pool_opts_t::get_opt_desc(var).key)) {
 	ss << "option '" << var << "' is not set on pool '" << poolstr << "'";
@@ -6378,6 +6395,7 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
 	  case DEDUP_TIER:
 	  case DEDUP_CHUNK_ALGORITHM:
 	  case DEDUP_CDC_CHUNK_SIZE:
+          case READ_RATIO:
             pool_opts_t::key_t key = pool_opts_t::get_opt_desc(i->first).key;
             if (p->opts.is_set(key)) {
               if(*it == CSUM_TYPE) {
@@ -6541,6 +6559,7 @@ bool OSDMonitor::preprocess_command(MonOpRequestRef op)
 	  case DEDUP_TIER:
 	  case DEDUP_CHUNK_ALGORITHM:
 	  case DEDUP_CDC_CHUNK_SIZE:
+          case READ_RATIO:
 	    for (i = ALL_CHOICES.begin(); i != ALL_CHOICES.end(); ++i) {
 	      if (i->second == *it)
 		break;
@@ -8314,6 +8333,11 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
     return -EACCES;
   }
 
+  if (!p.is_replicated() &&
+      (var == "read_ratio")) {
+    return -EACCES;
+  }
+
   if (var == "size") {
     if (p.has_flag(pg_pool_t::FLAG_NOSIZECHANGE)) {
       ss << "pool size change is disabled; you must unset nosizechange flag for the pool first";
@@ -8948,6 +8972,15 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
         ss << "error parsing int value '" << val << "': " << interr;
         return -EINVAL;
       }
+    } else if (var == "read_ratio") {
+      if (interr.length()) {
+        ss << "error parsing int value '" << val << "': " << interr;
+        return -EINVAL;
+      }
+      if (n < 0 || n > 100) {
+        ss << "read_ratio must be between 0 and 100";
+        return -ERANGE;
+      }
     }
 
     pool_opts_t::opt_desc_t desc = pool_opts_t::get_opt_desc(var);
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 7596723a0e30..2af3894f1e7d 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -1376,7 +1376,9 @@ static opt_mapping_t opt_mapping = boost::assign::map_list_of
            ("dedup_cdc_chunk_size", pool_opts_t::opt_desc_t(
 	     pool_opts_t::DEDUP_CDC_CHUNK_SIZE, pool_opts_t::INT))
 	   ("pg_num_max", pool_opts_t::opt_desc_t(
-             pool_opts_t::PG_NUM_MAX, pool_opts_t::INT));
+             pool_opts_t::PG_NUM_MAX, pool_opts_t::INT))
+	   ("read_ratio", pool_opts_t::opt_desc_t(
+             pool_opts_t::READ_RATIO, pool_opts_t::INT));
 
 bool pool_opts_t::is_opt_name(const std::string& name)
 {
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 66f39a91ac4d..8f08e298ee53 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -1101,6 +1101,7 @@ class pool_opts_t {
     DEDUP_CHUNK_ALGORITHM,
     DEDUP_CDC_CHUNK_SIZE,
     PG_NUM_MAX, // max pg_num
+    READ_RATIO, // read ration for the read balancer work [0-100]
   };
 
   enum type_t {

From 2953db8b58535605882dff2e1d4ff36e6075e122 Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Tue, 16 Jan 2024 20:33:47 +0200
Subject: [PATCH 1696/2492] osd: Read balancer for OSDs with different sizes

This commit adds calculation for desired primary distribution which
takes into account the osd size. This way smaller OSDs can take more
read operations (by adding more primaries) and the larger OSDs take less
primaries and the load of the cluater can increase. (This feature offset
a bit the weakest link in the chain effect under some conditions). In
order to calculate the loads correctly there is a need to know the
read/write ratio for the pool, and this commit assumes the read_ratio
parameter is available for the pool.

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/osd/OSDMap.cc | 277 +++++++++++++++++++++++++++++++++++++++++++++-
 src/osd/OSDMap.h  |  28 ++++-
 2 files changed, 299 insertions(+), 6 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 4cd9bff44416..be18f9e9a962 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5000,7 +5000,8 @@ int OSDMap::balance_primaries(
   CephContext *cct,
   int64_t pid,
   OSDMap::Incremental *pending_inc,
-  OSDMap& tmp_osd_map) const
+  OSDMap& tmp_osd_map,
+  const std::optional<rb_policy>& rbp) const
 {
   // This function only handles replicated pools.
   const pg_pool_t* pool = get_pg_pool(pid);
@@ -5037,7 +5038,7 @@ int OSDMap::balance_primaries(
   // calculate desired primary distribution for each osd
   map<uint64_t,float> desired_prim_dist;
   int rc = 0;
-  rc = calc_desired_primary_distribution(cct, pid, osds_to_check, desired_prim_dist);
+  rc = calc_desired_primary_distribution(cct, pid, osds_to_check, desired_prim_dist, rbp);
   if (rc < 0) {
     ldout(cct, 10) << __func__ << " Error in calculating desired primary distribution" << dendl;
     return -EINVAL;
@@ -5155,11 +5156,56 @@ int OSDMap::balance_primaries(
   return num_changes;
 }
 
+void OSDMap::rm_all_upmap_prims(CephContext *cct, OSDMap::Incremental *pending_inc, uint64_t pid) {
+  map<uint64_t,set<pg_t>> prim_pgs_by_osd;
+  get_pgs_by_osd(cct, pid, &prim_pgs_by_osd);
+  for (auto &[_, pgs] : prim_pgs_by_osd) {
+    for (auto &pg : pgs) {
+      if (pending_inc->new_pg_upmap_primary.contains(pg)) {
+        ldout(cct,30) << __func__ << "Removing pending pg_upmap_prim for pg " << pg << dendl;
+        pending_inc->new_pg_upmap_primary.erase(pg);
+      }
+      if (pg_upmap_primaries.contains(pg)) {
+        ldout(cct, 30) << __func__ << "Removing pg_upmap_prim for pg " << pg << dendl;
+        pending_inc->old_pg_upmap_primary.insert(pg);
+      }
+    }
+  }
+}
+
 int OSDMap::calc_desired_primary_distribution(
   CephContext *cct,
   int64_t pid,
   const vector<uint64_t> &osds,
-  std::map<uint64_t, float>& desired_primary_distribution) const
+  map<uint64_t, float>& desired_primary_distribution,
+  const std::optional<rb_policy>& rbp) const
+{
+  rb_policy policy;
+  if (rbp) {
+    policy = rbp.value();
+  }
+  else {
+    //TODO: Change this to support policy parameters in the future
+    policy = RB_SIMPLE;
+  }
+
+  switch (policy) {
+    case RB_SIMPLE:
+      return calc_desired_primary_distribution_simple(cct, pid, osds, desired_primary_distribution);
+    case RB_OSDSIZEOPT:
+      return calc_desired_primary_distribution_osdsize_opt(cct, pid, osds, desired_primary_distribution);
+    default:
+      ldout(cct, 10) << __func__ << " invalid read balance policy" << int(policy) << dendl;
+      return -EINVAL;
+  }
+
+}
+
+int OSDMap::calc_desired_primary_distribution_simple(
+  CephContext *cct,
+  int64_t pid,
+  const vector<uint64_t> &osds,
+  map<uint64_t, float>& desired_primary_distribution) const
 {
   // will return a perfect distribution of floats
   // without calculating the floor of each value
@@ -5167,7 +5213,7 @@ int OSDMap::calc_desired_primary_distribution(
   // This function only handles replicated pools.
   const pg_pool_t* pool = get_pg_pool(pid);
   if (pool->is_replicated()) {
-    ldout(cct, 20) << __func__ << " calculating distribution for replicated pool "
+    ldout(cct, 20) << __func__ << " calculating simple distribution for replicated pool "
                    << get_pool_name(pid) << dendl;
     uint64_t replica_count = pool->get_size();
     
@@ -5206,6 +5252,229 @@ int OSDMap::calc_desired_primary_distribution(
   return 0;
 }
 
+//
+// For read balancing with different osd sizes - calculate the desired number of primaries
+// per OSD for a given pool with constraints (forced*) that are set by previous
+// assignments.
+//
+float OSDMap::calc_desired_prims_for_osdsizeopt(int npgs, int forced_primaries,
+                                                int forced_secondaries, int iops_per_osd,
+                                                int write_ratio) const {
+  int forced_iops_per_pg = forced_primaries * 100 + forced_secondaries * write_ratio;
+  int pgs_left = npgs - forced_primaries - forced_secondaries;
+  int iops_left = iops_per_osd - forced_iops_per_pg;
+  if (pgs_left <= 0)
+    return float(forced_primaries);
+  else
+    return float(forced_primaries) + float(iops_left - pgs_left * write_ratio) / float(100 - write_ratio);
+}
+
+int OSDMap::calc_desired_primary_distribution_osdsize_opt(
+  CephContext *cct,
+  int64_t pid,
+  const vector<uint64_t> &osds,
+  map<uint64_t, float>& desired_primary_distribution) const
+{
+  const pg_pool_t* pool = get_pg_pool(pid);
+  if (!pool->is_replicated()) {   // read balancing works only for replicated pools
+    ldout(cct, 10) << __func__ <<" skipping erasure pool "
+                   << get_pool_name(pid) << dendl;
+    return -EINVAL;
+  } else if (pool->get_size() <= 1) {
+    ldout(cct, 10) << __func__ << " skipping replicated pool "
+                   << get_pool_name(pid) <<  " with a single replica" << dendl;
+    return -EINVAL;
+  } else {
+    int replica_count = pool->get_size();
+    int pg_num = pool->get_pg_num();
+    map<uint64_t,set<pg_t>> pgs_by_osd;
+    pgs_by_osd = get_pgs_by_osd(cct, pid);
+    int sum_pgs = 0;
+    for (auto& [_, pgs] : pgs_by_osd) {
+      sum_pgs += pgs.size();
+    }
+    if (sum_pgs != pg_num * replica_count) {
+      ldout(cct, 10) << __func__ << " Some of the PGs for pool '"
+                     << get_pool_name(pid) << "' don't have" << replica_count << " replicas "
+                     << "- can't perform osd-size-optimized read balancing " << dendl;
+      return -EINVAL;
+    }
+    ldout(cct, 20) << __func__ << " calculating OSD size optimized primary distribution for replicated pool "
+                   << get_pool_name(pid) << dendl;
+
+    int64_t read_ratio = 0;
+    {
+      // new scope since it is not allowed to use def_read_ratio after std::move
+      uint64_t def_read_ratio = cct->_conf.get_val<uint64_t>("osd_pool_default_read_ratio");
+      read_ratio = pool->opts.value_or<int64_t>(pool_opts_t::key_t::READ_RATIO, std::move(def_read_ratio));
+    }
+    int write_ratio = 100 - read_ratio;
+    ldout(cct, 30) << __func__ << " Pool: " << pid << " read ratio: " << read_ratio << " write ratio: " << write_ratio << dendl;
+    int num_osds = osds.size();
+    if (pg_num != (pool->get_pgp_num_mask() + 1)) {
+      // TODO: handle pgs with different sizes
+      //pool_t op:  unsigned get_pg_num_divisor(pg_t pgid) const;
+      ldout(cct, 10) << __func__ << " number of PGs for pool '"
+                     << get_pool_name(pid) << "' is not a power of 2 "
+                     << "- read balance calculation is not optinmal" << dendl;
+    }
+
+    vector<bool> osds_set(num_osds, false);
+    int osd_set_count = 0;
+    set<pg_t> pgs_used;
+    int osds_left = num_osds;
+    int iops_left = pg_num * (100 + ((replica_count - 1) * write_ratio));
+    int primaries_left = pg_num;
+    bool cont = true;
+    int iops_per_osd;
+
+    // first loop - mark OSDs in which all PGs should be primary (very small one, if exist)
+    // for OSDs selected in this loop we "force" the primary to be on these OSDs so that there
+    // is only little freedom to the balancer (in cases where one PG is mapped to more than 1 of these OSDs).
+    //
+    while (cont) {
+      cont = false;
+      if (osds_left <= 0 || iops_left <= 0 || primaries_left <= 0)
+        break;    // We are done here
+      iops_per_osd = iops_left / osds_left;
+      int p_pgs;
+
+      for (int i = 0 ; i < num_osds ; i++) {
+        int npgs = pgs_by_osd[osds[i]].size();
+        p_pgs = npgs;
+        if (osds_set[i])
+          continue;   // We are already done with this OSD
+        for (auto &p : pgs_by_osd[osds[i]]) {
+          if (pgs_used.contains(p))
+            p_pgs--;   // We already force primary on this pg
+        }
+        int osd_max_io_load = p_pgs * 100 + ((npgs - p_pgs) * write_ratio);
+        //
+        // Check if for this OSD we can mark all non-set PGs to primaries
+        //
+        if (osd_max_io_load < iops_per_osd) {
+          osds_set[i] = true;
+          osd_set_count++;
+          desired_primary_distribution.insert({osds[i], float(p_pgs)});
+          iops_left -= osd_max_io_load;
+          osds_left--;
+          primaries_left -= p_pgs;
+          cont = true;
+          for (auto &p : pgs_by_osd[osds[i]]) {
+            // Doesn't matter that we count some PGs twice - if we are here all
+            // PGs of this OSD are already forced to choose a primary
+            if (!pgs_used.contains(p))
+              pgs_used.insert(p);
+          }
+          if (osd_set_count == num_osds) {
+            break;    // We are done here
+          }
+        }
+      }
+    }
+    ldout(cct, 30) << __func__ << " read-balancer: All primaries OSDs for pool '"
+                   << get_pool_name(pid) << "': " << osds_set << dendl;
+    ldout(cct, 30) << __func__ << " read-balancer: PGs with primaries fixes " << pgs_used << dendl;
+    //
+    // Second loop - mark all OSDs which should have no primaries (if any) - these
+    // are the large OSDs which will make the disk fully loaded only with write operations
+    //
+    cont = true;
+    map <pg_t, int> wo_pgs;  // count PGs which are not yet marked as primaries
+    while (cont) {
+      cont = false;
+      if (osds_left <= 0 || iops_left <= 0 || primaries_left <= 0)
+        break;    // We are done here
+      iops_per_osd = iops_left / osds_left;
+      int p_pgs;
+
+      for (int i = 0 ; i < num_osds; i++) {
+        int npgs = pgs_by_osd[osds[i]].size();
+        p_pgs = 0;
+        if (osds_set[i])
+          continue;   // We are already done with this OSD
+        // Check minimal load on this device - all the pgs are secondary but those
+        // which are already marked asecondaries (size - 1) times.
+        for (auto &p : pgs_by_osd[osds[i]]) {
+          if (!pgs_used.contains(p)) {   // no mark for the primary of this PG yet
+            // so we count that no more than (size - 1) PGs are marked as secondaries
+            if (wo_pgs.contains(p)) {
+              if (wo_pgs[p] >= (replica_count - 1)) {
+                // the >= instead of == will allow us to later increase the number of wo_pgs[p] without
+                // knowing which PGs secondaries and which are primaries
+                p_pgs++;   // We force primary on this pg since all other OSDs in this PG are secondaries
+              }
+            }
+          }
+        }
+        int osd_min_io_load = p_pgs * 100 + (npgs - p_pgs) * write_ratio;
+        if (osd_min_io_load > iops_per_osd) {
+          osds_set[i] = true;
+          osd_set_count++;
+          desired_primary_distribution.insert({osds[i], float(p_pgs)});
+          //iops_left -= iops_per_osd;  //TODO: consider replaceing with -= osd_min_io_load
+          iops_left -= osd_min_io_load;
+          osds_left--;
+          primaries_left -= p_pgs;
+          cont = true;
+          for (auto &p : pgs_by_osd[osds[i]]) {
+            if (!pgs_used.contains(p)) {
+              if (wo_pgs.contains(p)) {
+                wo_pgs[p]++;
+              } else {
+                wo_pgs.insert({p, 1});
+              }
+            }
+          }
+          if (osds_left == 0) {
+            break;    // We are done here
+          }
+        }
+      }
+    }
+    ldout(cct, 30) << __func__ << " read-balancer: OSDs with all primaries or no primaries '"
+                   << get_pool_name(pid) << "': " << osds_set << dendl;
+    ldout(cct, 30) << __func__ << " read-balancer: wo_pgs " << wo_pgs << dendl;
+
+    //TODO: should we iterate over the above 2 loops or one iteration is enough? ^^
+    //
+    // Now split the rest of the primaries between the remaining OSDs (if any) based on the
+    // number of PGs mapped to them
+    //
+    if (osds_left > 0) {
+      iops_per_osd = iops_left / osds_left;
+
+      ldout(cct, 30) << __func__
+                    << " read-balancer: primaries_left [" << primaries_left
+                    << "] osds_left [" << osds_left
+                    << "] iops_left [" << iops_left
+                    << "] iops_per_osd [" << iops_per_osd << "]" << dendl;
+
+      for (int i = 0 ; i < num_osds ; i++) {
+        if (desired_primary_distribution.contains(osds[i]))
+          continue;   // We are already done with this OSD
+        int npgs = pgs_by_osd[osds[i]].size();
+        int forced_primaries = 0;
+        int forced_secondaries = 0;
+        for (auto &p : pgs_by_osd[osds[i]]) {
+          if (pgs_used.contains(p)) {
+            forced_secondaries++;
+          } else if (wo_pgs.contains(p) && wo_pgs[p] >= (replica_count - 1)) {
+            forced_primaries++;
+          }
+        }
+        float nprims = calc_desired_prims_for_osdsizeopt(npgs, forced_primaries, forced_secondaries, iops_per_osd, write_ratio);
+        desired_primary_distribution.insert({osds[i], nprims});
+      }
+    }
+  }
+
+  ldout(cct, 30) << __func__ << " read-balancer: desired_primary_distribution: "
+                 << desired_primary_distribution << dendl;
+
+  return 0;
+
+}
 int OSDMap::calc_pg_upmaps(
   CephContext *cct,
   uint32_t max_deviation,
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index e37aeafc4312..31da6e863743 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -1479,17 +1479,41 @@ class OSDMap {
     std::vector<int> *orig,
     std::vector<int> *out);             ///< resulting alternative mapping
 
+  enum rb_policy {
+    RB_SIMPLE = 0,
+    RB_OSDSIZEOPT
+  };
+
   int balance_primaries(
     CephContext *cct,
     int64_t pid,
     Incremental *pending_inc,
-    OSDMap& tmp_osd_map) const;
+    OSDMap& tmp_osd_map,
+    const std::optional<rb_policy>& rbp = std::nullopt) const;
+
+  void rm_all_upmap_prims(CephContext *cct, Incremental *pending_inc, uint64_t pid);
 
   int calc_desired_primary_distribution(
     CephContext *cct,
     int64_t pid, // pool id
     const std::vector<uint64_t> &osds,
-    std::map<uint64_t, float>& desired_primary_distribution) const; // vector of osd ids
+    std::map<uint64_t, float>& desired_primary_distribution, // vector of osd ids
+    const std::optional<rb_policy>& rbp = std::nullopt) const;
+
+  int calc_desired_primary_distribution_simple(
+    CephContext *cct,
+    int64_t pid, // pool id
+    const std::vector<uint64_t> &osds,
+    std::map<uint64_t, float>& desired_primary_distribution /* vector of osd ids */) const;
+
+  int calc_desired_primary_distribution_osdsize_opt(
+    CephContext *cct,
+    int64_t pid, // pool id
+    const std::vector<uint64_t> &osds,
+    std::map<uint64_t, float>& desired_primary_distribution /* vector of osd ids */) const;
+
+  float calc_desired_prims_for_osdsizeopt(int npgs, int forced_primaries, int forced_secondaries,
+                                          int iops_per_osd, int write_ratio) const;
 
   int calc_pg_upmaps(
     CephContext *cct,

From b13ffd280959c3ec4894f99c6584651f2b8b5110 Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Tue, 16 Jan 2024 20:45:32 +0200
Subject: [PATCH 1697/2492] osd: Unit tests for read balancer on OSDs with
 different sizes

Test cases for the read balancer which takes osd sizes into account.
Some balancing code refactoring and reorg for code that is used in
multiple tests

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/test/osd/TestOSDMap.cc | 318 ++++++++++++++++++++++++++++++++-----
 1 file changed, 280 insertions(+), 38 deletions(-)

diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc
index c3c8a153158f..6dd9309968e9 100644
--- a/src/test/osd/TestOSDMap.cc
+++ b/src/test/osd/TestOSDMap.cc
@@ -35,6 +35,7 @@ int main(int argc, char **argv) {
 class OSDMapTest : public testing::Test,
                    public ::testing::WithParamInterface<std::pair<int, int>> {
   int num_osds = 6;
+  bool verbose = false;
 public:
   OSDMap osdmap;
   OSDMapMapping mapping;
@@ -50,7 +51,8 @@ class OSDMapTest : public testing::Test,
   const string EC_RULE_NAME = "erasure";
 
   OSDMapTest() {}
-
+  void set_verbose(bool v) { verbose = v; }
+  bool is_verbose() const {return verbose; }
   void set_up_map(int new_num_osds = 6, bool no_default_pools = false) {
     num_osds = new_num_osds;
     uuid_d fsid;
@@ -84,6 +86,61 @@ class OSDMapTest : public testing::Test,
     set_rep_pool("reppool",new_pool_inc);
     osdmap.apply_incremental(new_pool_inc);
   }
+  //
+  // The following function is currently used only for read balancer which works
+  // only on replicated pools. EC pool is created just to keep pool numbers consistent
+  // weights are integers in [0..100]
+  // [ods#i_weight, osd#i_host] = weights[i]
+  // *Note:* hosts are not yet implemented.
+  //
+  void set_up_map_heterogeneous(const vector<pair<int, int>> &weights, int new_num_hosts) {
+
+    num_osds = weights.size();
+    uuid_d fsid;
+    osdmap.build_simple(g_ceph_context, 0, fsid, num_osds);
+    OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+    pending_inc.fsid = osdmap.get_fsid();
+    entity_addrvec_t sample_addrs;
+    sample_addrs.v.push_back(entity_addr_t());
+    uuid_d sample_uuid;
+    for (int i = 0; i < num_osds; ++i) {
+      int ow = weights[i].first;
+      ceph_assert(ow >= 0 && ow <= 100);
+      ow = ow * CEPH_OSD_IN / 100;
+      sample_uuid.generate_random();
+      sample_addrs.v[0].nonce = i;
+      pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW;
+      pending_inc.new_up_client[i] = sample_addrs;
+      pending_inc.new_up_cluster[i] = sample_addrs;
+      pending_inc.new_hb_back_up[i] = sample_addrs;
+      pending_inc.new_hb_front_up[i] = sample_addrs;
+      pending_inc.new_weight[i] = ow;
+      pending_inc.new_uuid[i] = sample_uuid;
+    }
+    osdmap.apply_incremental(pending_inc);
+
+    OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1);
+    new_pool_inc.new_pool_max = osdmap.get_pool_max();
+    new_pool_inc.fsid = osdmap.get_fsid();
+    // make a not needed ec pool just to keep pool numbers consistent
+    set_ec_pool("ec", new_pool_inc);
+    // and a replicated pool
+    set_rep_pool("reppool",new_pool_inc);
+
+    //
+    // Force npgs to be a power of 2
+    //
+    ceph_assert(new_pool_inc.new_pools.contains(my_rep_pool));
+    int npgs = new_pool_inc.new_pools[my_rep_pool].get_pg_num();
+    float lg = log2(npgs);
+    if (ceil(lg) != floor(lg)) {  // npgs is not a power of 2
+      cout << "******* Fix npgs from " << npgs << " to " << (1 << (int)ceil(lg)) << std::endl;
+      npgs = 1 << (int)ceil(lg);
+      new_pool_inc.new_pools[my_rep_pool].set_pg_num(1 << (int)ceil(lg));
+      new_pool_inc.new_pools[my_rep_pool].set_pgp_num(1 << (int)ceil(lg));
+    }
+    osdmap.apply_incremental(new_pool_inc);
+  }
   int get_ec_crush_rule() {
     int r = osdmap.crush->get_rule_id(EC_RULE_NAME);
     if (r < 0) {
@@ -126,6 +183,35 @@ class OSDMapTest : public testing::Test,
     return pool_id;
   }
 
+  void balance_capacity(int64_t pid) {
+    set<int64_t> only_pools;
+    only_pools.insert(pid);
+    OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
+    osdmap.calc_pg_upmaps(g_ceph_context,
+                          0,
+                          100,
+                          only_pools,
+                          &pending_inc);
+    osdmap.apply_incremental(pending_inc);
+  }
+
+  void set_pool_read_ratio(uint64_t pid, int64_t ratio) {
+    ceph_assert(ratio >= 0 && ratio <= 100);
+    OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
+    pg_pool_t *np = nullptr;
+    {
+      const pg_pool_t *p = osdmap.get_pg_pool(pid);
+      np = pending_inc.get_new_pool(pid, p);
+      ceph_assert(np != nullptr);
+    }
+    if (ratio == 0) {
+      np->opts.unset(pool_opts_t::READ_RATIO);
+    } else {
+      np->opts.set(pool_opts_t::READ_RATIO, ratio);
+    }
+    osdmap.apply_incremental(pending_inc);
+  }
+
   unsigned int get_num_osds() { return num_osds; }
   void get_crush(const OSDMap& tmap, CrushWrapper& newcrush) {
     bufferlist bl;
@@ -242,6 +328,130 @@ class OSDMapTest : public testing::Test,
     }
     return score >= 1.0 && score <= float(nosds);
   }
+
+  void test_rb_osd_size(const vector <pair<int, int>> &weights, const vector<int> &read_ratios, bool expect_failure, bool reset_rb = false, bool build = true) {
+
+    if (build) {
+      set_up_map_heterogeneous(weights, 1);
+    }
+    int num_osds = osdmap.get_num_osds();
+
+    map<uint64_t,set<pg_t>> orig_prim_pgs_by_osd;
+    map<uint64_t,set<pg_t>> pgs_by_osd = osdmap.get_pgs_by_osd(g_ceph_context, my_rep_pool, &orig_prim_pgs_by_osd);
+    if (verbose) {
+      cout << "PGs distribution:" << std::endl;
+      for (auto i = 0 ; i < num_osds ; i++) {
+        cout << "osd." << i << ": " << pgs_by_osd[i].size() << std::endl;
+      }
+    }
+    if (reset_rb) {
+      OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
+      osdmap.rm_all_upmap_prims(g_ceph_context, &pending_inc, my_rep_pool);
+      osdmap.apply_incremental(pending_inc);
+    }
+
+    // Make sure capacity is balanced first
+    balance_capacity(my_rep_pool);
+    pgs_by_osd.clear();
+    orig_prim_pgs_by_osd.clear();
+    pgs_by_osd = osdmap.get_pgs_by_osd(g_ceph_context, my_rep_pool, &orig_prim_pgs_by_osd);
+
+    map<uint64_t,set<pg_t>> prim_pgs_by_osd;
+    if (verbose) {
+      cout << "Balanced PG distribution:" << std::endl;
+      for (auto i = 0 ; i < num_osds ; i++) {
+        cout << "osd." << i << ": " << pgs_by_osd[i].size() << "/" << orig_prim_pgs_by_osd[i].size() << std::endl;
+      }
+    }
+
+    vector<uint64_t> osds_to_check(num_osds);
+    for (auto i = 0 ; i < num_osds ; i++) {
+      osds_to_check[i] = i;
+    }
+
+    const pg_pool_t *p = osdmap.get_pg_pool(my_rep_pool);
+    auto pg_num = p->get_pg_num();
+
+    for (auto rr : read_ratios) {
+      map<uint64_t, float> desired_prims;
+      set_pool_read_ratio(my_rep_pool, rr);
+      if (reset_rb) {
+        OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
+        osdmap.rm_all_upmap_prims(g_ceph_context, &pending_inc, my_rep_pool);
+        osdmap.apply_incremental(pending_inc);
+      }
+      int rc = osdmap.calc_desired_primary_distribution_osdsize_opt(g_ceph_context, my_rep_pool,
+                                                                    osds_to_check, desired_prims);
+      if (expect_failure) {
+        ASSERT_TRUE(rc < 0);
+        if (verbose) {
+          cout << "Can't calculate osd-size-optimized read balancing - this is expected" << std::endl;
+        }
+        return;
+      } else {
+        ASSERT_TRUE(rc >= 0);
+      }
+      if (verbose) {
+        cout << ">>>>>Desired primary distribution for read ratio: " << rr << std::endl;
+      }
+      float total_prims = 0.0;
+      map<uint64_t,set<pg_t>> prim_pgs_by_osd;
+      osdmap.get_pgs_by_osd(g_ceph_context, my_rep_pool, &prim_pgs_by_osd);
+      int high_load_before = 0;
+      for (auto& [oid, des] : desired_prims) {
+        int pgs = pgs_by_osd[oid].size();
+        int oprims = orig_prim_pgs_by_osd[oid].size();
+        int cur_load = oprims * 100 + (pgs - oprims) * (100 - rr);
+        if (verbose) {
+          cout << "osd." << oid << ": " << pgs << "/" << des
+              <<  " Load = current/desired "
+              <<  cur_load << "/"
+              << des * 100 + (pgs - des) * (100 - rr) << std::endl;
+        }
+        ASSERT_LE(des, float(pgs));
+        total_prims += des;
+        if (cur_load > high_load_before) {
+          high_load_before = cur_load;
+        }
+      }
+      ASSERT_TRUE(total_prims < (pg_num + 0.4) && total_prims > (pg_num - 0.4));    // handle rounding errors
+
+      // Balance reads
+      OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
+      osdmap.balance_primaries(g_ceph_context, my_rep_pool, &pending_inc, osdmap, OSDMap::RB_OSDSIZEOPT);
+      osdmap.apply_incremental(pending_inc);
+      prim_pgs_by_osd.clear();
+      osdmap.get_pgs_by_osd(g_ceph_context, my_rep_pool, &prim_pgs_by_osd);
+      if (verbose) {
+        cout << "<<<<<PGs distribution:" << std::endl;
+      }
+      int high_load_after = 0;
+      for (auto i = 0 ; i < weights.size() ; i++) {
+        int pgs = pgs_by_osd[i].size();
+        int prims = prim_pgs_by_osd[i].size();
+        int cur_load = prims * 100 + (pgs - prims) * (100 - rr);
+        ASSERT_GE(pgs, prims);
+        if (verbose) {
+          cout << "osd." << i << ": " << pgs << "/" << prims
+              << " Load = " << cur_load << std::endl;
+        }
+        //ASSERT_LE(prim_pgs_by_osd[i].size(), (desired_prims[i] + 1.0));
+        //ASSERT_GE(prim_pgs_by_osd[i].size(), (desired_prims[i] - 1.0));
+        if (cur_load > high_load_after) {
+          high_load_after = cur_load;
+        }
+      }
+      if (verbose) {
+        cout << "=== Read ratio: " << rr << " High load before: " << high_load_before << " High load after: " << high_load_after << std::endl;
+      }
+      ASSERT_LE(high_load_after, high_load_before);
+      if (verbose) {
+        cout << " ====== end of iteration for read ratio " << rr << std::endl;
+      }
+    }
+
+    return;
+  }
 };
 
 TEST_F(OSDMapTest, Create) {
@@ -1357,17 +1567,8 @@ TEST_F(OSDMapTest, BUG_38897) {
 
   // ready to go
   {
-    set<int64_t> only_pools;
     ASSERT_TRUE(pool_1_id >= 0);
-    only_pools.insert(pool_1_id);
-    OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
-    // require perfect distribution! (max deviation 0)
-    osdmap.calc_pg_upmaps(g_ceph_context,
-                          0, // so we can force optimizing
-                          100,
-                          only_pools,
-                          &pending_inc);
-    osdmap.apply_incremental(pending_inc);
+    balance_capacity(pool_1_id);
   }
 }
 
@@ -2466,15 +2667,7 @@ TEST_F(OSDMapTest, read_balance_small_map) {
     }
 
     // Make sure capacity is balanced first
-    set<int64_t> only_pools;
-    only_pools.insert(my_rep_pool);
-    OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
-    osdmap.calc_pg_upmaps(g_ceph_context,
-                          0,
-                          100,
-                          only_pools,
-                          &pending_inc);
-    osdmap.apply_incremental(pending_inc);
+    balance_capacity(my_rep_pool);
 
     // Get read balance score before balancing
     OSDMap::read_balance_info_t rb_info;
@@ -2550,15 +2743,7 @@ TEST_F(OSDMapTest, read_balance_large_map) {
     }
 
     // Make sure capacity is balanced first
-    set<int64_t> only_pools;
-    only_pools.insert(my_rep_pool);
-    OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
-    osdmap.calc_pg_upmaps(g_ceph_context,
-                          0,
-                          100,
-                          only_pools,
-                          &pending_inc);
-    osdmap.apply_incremental(pending_inc);
+    balance_capacity(my_rep_pool);
   
     // Get read balance score before balancing
     OSDMap::read_balance_info_t rb_info;
@@ -2639,15 +2824,7 @@ TEST_F(OSDMapTest, read_balance_random_map) {
     }
 
     // Make sure capacity is balanced first
-    set<int64_t> only_pools;
-    only_pools.insert(my_rep_pool);
-    OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
-    osdmap.calc_pg_upmaps(g_ceph_context,
-                          0,
-                          100,
-                          only_pools,
-                          &pending_inc);
-    osdmap.apply_incremental(pending_inc);
+    balance_capacity(my_rep_pool);
 
     // Get read balance score before balancing
     OSDMap::read_balance_info_t rb_info;
@@ -2707,6 +2884,71 @@ TEST_F(OSDMapTest, read_balance_random_map) {
   }
 }
 
+TEST_F(OSDMapTest, rb_osdsize_opt_1small_osd) {
+  //TO-REMOVE (the comment) - look ar 43124 for examples
+  vector <pair<int, int>> weights = {
+      {57, 0}, {100, 0}, {100, 0}, {100, 0}, {100, 0},
+  };
+  vector<int> read_ratios = {10, 50, 70, 80, 100};
+  if (is_verbose()) {
+    cout << " First iteration - incremental, no reset" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false);
+  if (is_verbose()) {
+    cout << std::endl << " Second iteration - reset between read ratio tests" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false, true, false);
+
+  return;
+}
+
+TEST_F(OSDMapTest, rb_osdsize_opt_mixed_osds) {
+  vector <pair<int, int>> weights = {
+    {50, 0}, {50, 0}, {100, 0},  {100, 0}, {50, 0}, {100, 0},
+  };
+  vector<int> read_ratios = {10, 50, 70, 80, 100};
+  if (is_verbose()) {
+    cout << " First iteration - incremental, no reset" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false);
+  if (is_verbose()) {
+    cout << std::endl << " Second iteration - reset between read ratio tests" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false, true, false);
+
+  return;
+}
+
+TEST_F(OSDMapTest, rb_osdsize_opt_1large_osd) {
+  vector <pair<int, int>> weights = {
+      {25, 0}, {25, 0}, {25, 0}, {25, 0}, {25, 0}, {25, 0}, {100, 0},
+  };
+  vector<int> read_ratios = {70};
+  //expect failure here, so no need for multiple runs
+  test_rb_osd_size(weights, read_ratios, true);
+
+  return;
+}
+
+TEST_F(OSDMapTest, rb_osdsize_opt_1large_mixed_osds) {
+  //TO_REMOVE (the comment) - look ar 43124 for examples
+  vector <pair<int, int>> weights = {
+      {50, 0}, {70, 0}, {50, 0}, {35, 0}, {35, 0}, {35, 0},
+      {50, 0}, {75, 0}, {35, 0}, {50, 0}, {100, 0},
+  };
+  vector<int> read_ratios = {10, 30, 50, 70, 80, 90, 100};
+  if (is_verbose()) {
+    cout << " First iteration - incremental, no reset" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false);
+  if (is_verbose()) {
+    cout << std::endl << " Second iteration - reset between read ratio tests" << std::endl << "====================" << std::endl << std::endl;
+  }
+  test_rb_osd_size(weights, read_ratios, false, true, false);
+
+  return;
+}
+
 INSTANTIATE_TEST_SUITE_P(
   OSDMap,
   OSDMapTest,

From c1cb08689fc54e0203d8330494bd6c248219c316 Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Wed, 24 Jan 2024 14:40:53 +0200
Subject: [PATCH 1698/2492] osd: Add score for read balance osd size aware
 policy

This score works for pools in which the read_ratio
value is set.
Current limitations:
- This mechanism ignores osd read affinty
  - There is a plan adding support for read affinity 0
    in the next version.
- This mechanism works only when all PGs are full
- If read_ration is not set - the existing mechanism (named
  fair score) is used.

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/osd/OSDMap.cc | 264 +++++++++++++++++++++++++++++++++++++---------
 src/osd/OSDMap.h  |  53 +++++++---
 2 files changed, 255 insertions(+), 62 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index be18f9e9a962..b35bc592eec0 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -3852,14 +3852,45 @@ void OSDMap::dump_read_balance_score(CephContext *cct,
     auto rc = calc_read_balance_score(cct, pid, &rb_info);
     if (rc >= 0) {
       f->open_object_section("read_balance");
+      string score_type_str;
+      switch (rb_info.score_type) {
+      case RBS_FAIR:
+        score_type_str = "Fair distribution";
+        break;
+      case RBS_SIZE_OPTIMAL:
+        score_type_str = "OSD size aware";
+        break;
+      default:
+        score_type_str = "Unknown";
+        break;
+      }
+      f->dump_string("score_type", score_type_str);
       f->dump_float("score_acting", rb_info.acting_adj_score);
       f->dump_float("score_stable", rb_info.adjusted_score);
-      f->dump_float("optimal_score", rb_info.optimal_score);
-      f->dump_float("raw_score_acting", rb_info.acting_raw_score);
-      f->dump_float("raw_score_stable", rb_info.raw_score);
-      f->dump_float("primary_affinity_weighted", rb_info.pa_weighted);
-      f->dump_float("average_primary_affinity", rb_info.pa_avg);
-      f->dump_float("average_primary_affinity_weighted", rb_info.pa_weighted_avg);
+      if (rb_info.score_type == RBS_FAIR) {
+        f->dump_float("optimal_score", rb_info.optimal_score);
+        f->dump_float("raw_score_acting", rb_info.acting_raw_score);
+        f->dump_float("raw_score_stable", rb_info.raw_score);
+        f->dump_float("primary_affinity_weighted", rb_info.pa_weighted);
+        f->dump_float("average_primary_affinity", rb_info.pa_avg);
+        f->dump_float("average_primary_affinity_weighted", rb_info.pa_weighted_avg);
+      } else if (rb_info.score_type == RBS_SIZE_OPTIMAL) {
+        f->dump_int("average_osd_load", rb_info.avg_osd_load);
+        f->open_object_section("most_loaded_osd");
+        f->dump_int("osd", rb_info.max_osd);
+        f->dump_int("pgs", rb_info.max_osd_pgs);
+        f->dump_int("primaries", rb_info.max_osd_prims);
+        f->dump_int("load", rb_info.max_osd_load);
+        f->close_section();
+        if (rb_info.max_osd != rb_info.max_acting_osd) {
+          f->open_object_section("most_loaded_acting_osd");
+          f->dump_int("osd", rb_info.max_acting_osd);
+          f->dump_int("pgs", rb_info.max_acting_osd_pgs);
+          f->dump_int("primaries", rb_info.max_acting_osd_prims);
+          f->dump_int("load", rb_info.max_acting_osd_load);
+          f->close_section();
+        }
+      }
       if (rb_info.err_msg.length() > 0) {
         f->dump_string("error_message", rb_info.err_msg);
       }
@@ -5310,7 +5341,7 @@ int OSDMap::calc_desired_primary_distribution_osdsize_opt(
     }
     int write_ratio = 100 - read_ratio;
     ldout(cct, 30) << __func__ << " Pool: " << pid << " read ratio: " << read_ratio << " write ratio: " << write_ratio << dendl;
-    int num_osds = osds.size();
+    auto num_osds = osds.size();
     if (pg_num != (pool->get_pgp_num_mask() + 1)) {
       // TODO: handle pgs with different sizes
       //pool_t op:  unsigned get_pg_num_divisor(pg_t pgid) const;
@@ -6286,6 +6317,7 @@ int64_t OSDMap::has_zero_pa_pgs(CephContext *cct, int64_t pool_id) const
 }
 
 void OSDMap::zero_rbi(read_balance_info_t &rbi) const {
+  rbi.score_type = RBS_FAIR;
   rbi.pa_avg = 0.;
   rbi.pa_weighted = 0.;
   rbi.pa_weighted_avg = 0.;
@@ -6294,10 +6326,18 @@ void OSDMap::zero_rbi(read_balance_info_t &rbi) const {
   rbi.adjusted_score = 0.;
   rbi.acting_raw_score = 0.;
   rbi.acting_adj_score = 0.;
+  rbi.max_osd = 0;
+  rbi.max_osd_load = 0;
+  rbi.max_osd_pgs = 0;
+  rbi.max_osd_prims = 0;
+  rbi.max_acting_osd = 0;
+  rbi.max_acting_osd_load = 0;
+  rbi.max_acting_osd_pgs = 0;
+  rbi.max_acting_osd_prims = 0;
   rbi.err_msg = "";
 }
 
-int OSDMap::set_rbi(
+int OSDMap::set_rbi_fair(
     CephContext *cct,
     read_balance_info_t &rbi,
     int64_t pool_id,
@@ -6314,9 +6354,13 @@ int OSDMap::set_rbi(
     float max_osd_score) const
 {
   // put all the ugly code here, so rest of code is nicer.
+  //TODO: split this function to 2 (or just rename) according to score type
+
   const pg_pool_t* pool = get_pg_pool(pool_id);
   zero_rbi(rbi);
 
+  rbi.score_type = RBS_FAIR;
+
   if (total_w_pa / total_osd_weight < 1. / float(pool->get_size())) {
     ldout(cct, 20) << __func__ << " pool " << pool_id << " average primary affinity is lower than"
                     << 1. / float(pool->get_size()) << dendl;
@@ -6379,32 +6423,10 @@ int OSDMap::set_rbi(
   return 0;
 }
 
-int OSDMap::calc_read_balance_score(CephContext *cct, int64_t pool_id,
-				    read_balance_info_t *p_rbi) const
+int OSDMap::calc_rbs_fair(CephContext *cct, OSDMap& tmp_osd_map, int64_t pool_id,
+                          const pg_pool_t *pgpool, read_balance_info_t &rbi) const
 {
-  //BUG: wrong score with one PG replica 3 and 4 OSDs
-  if (cct != nullptr)
-    ldout(cct,20) << __func__ << " pool " << get_pool_name(pool_id) << dendl;
-
-  OSDMap tmp_osd_map;
-  tmp_osd_map.deepish_copy_from(*this);
-  if (p_rbi == nullptr) {
-    // The only case where error message is not set - this is not tested in the unit test.
-    if (cct != nullptr)
-      ldout(cct,30) << __func__ << " p_rbi is nullptr." << dendl;
-    return -EINVAL;
-  }
-
-  if (tmp_osd_map.pools.count(pool_id) == 0) {
-    if (cct != nullptr)
-      ldout(cct,30) << __func__ << " pool " << pool_id << " not found." << dendl;
-    zero_rbi(*p_rbi);
-    p_rbi->err_msg = fmt::format("pool {} not found", pool_id);
-    return -ENOENT;
-  }
-  int rc = 0;
-  const pg_pool_t* pool = tmp_osd_map.get_pg_pool(pool_id);
-  auto num_pgs = pool->get_pg_num();
+  auto num_pgs = pgpool->get_pg_num();
 
   map<uint64_t,set<pg_t>> pgs_by_osd;
   map<uint64_t,set<pg_t>> prim_pgs_by_osd;
@@ -6417,7 +6439,7 @@ int OSDMap::calc_read_balance_score(CephContext *cct, int64_t pool_id,
 		  << prim_pgs_by_osd << dendl;
 
   if (pgs_by_osd.empty()) {
-    //p_rbi->err_msg = fmt::format("pool {} has no PGs mapped to OSDs", pool_id);
+    //rbi.err_msg = fmt::format("pool {} has no PGs mapped to OSDs", pool_id);
     return -EINVAL;
   }
   if (cct != nullptr) {
@@ -6505,8 +6527,8 @@ int OSDMap::calc_read_balance_score(CephContext *cct, int64_t pool_id,
       ldout(cct, 10) << __func__ << " pool " << pool_id
 	         << " has primary_affinity set to zero on all OSDs" << dendl;
     }
-    zero_rbi(*p_rbi);
-    p_rbi->err_msg = fmt::format("pool {} has primary_affinity set to zero on all OSDs", pool_id);
+    zero_rbi(rbi);
+    rbi.err_msg = fmt::format("pool {} has primary_affinity set to zero on all OSDs", pool_id);
 
     return -ERANGE;   // score has a different meaning now.
   }
@@ -6514,30 +6536,172 @@ int OSDMap::calc_read_balance_score(CephContext *cct, int64_t pool_id,
     max_osd_score *= prim_affinity_sum / num_osds;
   }
 
-  rc = tmp_osd_map.set_rbi(cct, *p_rbi, pool_id, total_weighted_pa,
-                           prim_affinity_sum, num_osds, osd_pa_count,
-                           total_osd_weight, max_prims_per_osd,
-                           max_acting_prims_per_osd, avg_prims_per_osd,
-                           prim_on_zero_pa, acting_on_zero_pa, max_osd_score);
+  int rc = tmp_osd_map.set_rbi_fair(cct, rbi, pool_id, total_weighted_pa,
+                                    prim_affinity_sum, num_osds, osd_pa_count,
+                                    total_osd_weight, max_prims_per_osd,
+                                    max_acting_prims_per_osd, avg_prims_per_osd,
+                                    prim_on_zero_pa, acting_on_zero_pa, max_osd_score);
 
   if (cct != nullptr) {
     ldout(cct,30) << __func__ << " pool " << get_pool_name(pool_id)
-                  << " pa_avg " << p_rbi->pa_avg
-                  << " pa_weighted " << p_rbi->pa_weighted
-                  << " pa_weighted_avg " << p_rbi->pa_weighted_avg
-                  << " optimal_score " << p_rbi->optimal_score
-                  << " adjusted_score " << p_rbi->adjusted_score
-                  << " acting_adj_score " << p_rbi->acting_adj_score
+                  << " pa_avg " << rbi.pa_avg
+                  << " pa_weighted " << rbi.pa_weighted
+                  << " pa_weighted_avg " << rbi.pa_weighted_avg
+                  << " optimal_score " << rbi.optimal_score
+                  << " adjusted_score " << rbi.adjusted_score
+                  << " acting_adj_score " << rbi.acting_adj_score
                   << dendl;
     ldout(cct,20) << __func__ << " pool " << get_pool_name(pool_id)
-		  << " raw_score: " << p_rbi->raw_score
-		  << " acting_raw_score: " << p_rbi->acting_raw_score
+		  << " raw_score: " << rbi.raw_score
+		  << " acting_raw_score: " << rbi.acting_raw_score
 		  << dendl;
     ldout(cct,10) << __func__ << " pool " << get_pool_name(pool_id)
-		  << " wl_score: " << p_rbi->acting_adj_score << dendl;
+		  << " wl_score: " << rbi.acting_adj_score << dendl;
   }
 
   return rc;
+
+}
+
+int OSDMap::calc_rbs_size_optimal(CephContext *cct, OSDMap& tmp_osd_map, int64_t pool_id,
+                                  const pg_pool_t *pgpool, read_balance_info_t &rbi) const
+{
+  zero_rbi(rbi);
+  rbi.score_type = RBS_SIZE_OPTIMAL;
+
+  auto num_pgs = pgpool->get_pg_num();
+
+  if (num_pgs == 0) {
+    rbi.err_msg = fmt::format("ERROR: pool {} has no PGs", pool_id);
+    if (cct != nullptr) {
+      ldout(cct, 20) << __func__ << " pool " << pool_id
+                     << " has no PGs - can't calculate size-optimal read balancer score" << dendl;
+    }
+  }
+
+  map<uint64_t,set<pg_t>> pgs_by_osd;
+  map<uint64_t,set<pg_t>> prim_pgs_by_osd;
+  map<uint64_t,set<pg_t>> acting_prims_by_osd;
+
+  pgs_by_osd = tmp_osd_map.get_pgs_by_osd(cct, pool_id, &prim_pgs_by_osd, &acting_prims_by_osd);
+  auto num_osds = pgs_by_osd.size();
+  int64_t num_pg_osd_legs = 0;
+  for (auto i = 0 ; i < num_osds ; i++) {
+    if (get_primary_affinity(i) != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+      if (cct != nullptr) {
+        ldout(cct, 30) << __func__ << " pool " << pool_id
+                           << " has primary_affinity set to non-default value on some OSDs" << dendl;
+      }
+      if (rbi.err_msg.empty()) {
+        rbi.err_msg = fmt::format("Warning: pool {} has primary_affinity set to non-default value on some OSDs, "
+                                  "this is ignored by the size-optimal read balancer", pool_id);
+      }
+    }
+    num_pg_osd_legs += pgs_by_osd[i].size();
+  }
+  if (num_pg_osd_legs != num_pgs * pgpool->get_size()) {
+    if (cct != nullptr) {
+      ldout(cct, 30) << __func__ << " pool " << pool_id
+                     << " has " << num_pg_osd_legs << " PG OSD legs, expected " << num_pgs * pgpool->get_size()
+                     << " - can't calculate size-optimal read balancer score" << dendl;
+    }
+    rbi.err_msg = fmt::format("ERROR: pool {} has {} PG OSD legs, expected {} - Can't calculate size-optimal read balancer score",
+                              pool_id, num_pg_osd_legs, num_pgs * pgpool->get_size());
+    return -EINVAL;
+  }
+  int64_t rr = 0;
+  pgpool->opts.get(pool_opts_t::READ_RATIO, &rr);
+  if (rr <= 0 || rr > 100) {
+    if (cct != nullptr) {
+      ldout(cct, 30) << __func__ << " pool " << pool_id
+                      << " has invalid read_ratio " << rr << " - can't calculate size-optimal read balancer score" << dendl;
+    }
+    rbi.err_msg = fmt::format("ERROR: pool {} has invalid read_ratio {} - Can't calculate size-optimal read balancer score",
+                              pool_id, rr);
+    return -EINVAL;
+  }
+  uint64_t load_per_pg = 100 + (pgpool->get_size() - 1) * (100 - rr);
+  uint64_t total_load = load_per_pg * num_pgs;
+  if (num_osds == 0) {
+    rbi.err_msg = fmt::format("ERROR: pool {} has no active OSDs, can't calculate loads and read balance score", pool_id);
+    return -EINVAL;
+  }
+  float load_per_osd = total_load / num_osds;
+  rbi.max_osd = -1;
+  rbi.max_acting_osd = -1;
+  rbi.avg_osd_load = int(load_per_osd);
+  for (auto &[o, pgs] : pgs_by_osd) {
+    int64_t npgs = pgs.size();
+    int64_t nprims = prim_pgs_by_osd.contains(o) ? prim_pgs_by_osd.at(o).size() : 0;
+    int64_t nacting_prims = acting_prims_by_osd.contains(o) ? acting_prims_by_osd.at(o).size() : 0;
+    int64_t load = nprims * 100 + (npgs - nprims) * (100 - rr);
+    int64_t acting_load = nacting_prims * 100 + (npgs - nacting_prims) * (100 - rr);
+    if (load > rbi.max_osd_load) {
+      rbi.max_osd_load = load;
+      rbi.max_osd = o;
+      rbi.max_osd_pgs = npgs;
+      rbi.max_osd_prims = nprims;
+    }
+    if (acting_load > rbi.max_acting_osd_load) {
+      rbi.max_acting_osd_load = acting_load;
+      rbi.max_acting_osd = o;
+      rbi.max_acting_osd_pgs = npgs;
+      rbi.max_acting_osd_prims = nacting_prims;
+    }
+  }
+  if (rbi.max_acting_osd < 0) {
+    rbi.err_msg = fmt::format("ERROR: Could not find max_acting_load for pool {}", pool_id);
+    return -EINVAL;
+  }
+  // All conditions that can cause load_per_osd to be 0 were checked before this point.
+  ceph_assert(load_per_osd != 0.0);
+  rbi.acting_adj_score = rbi_round(float(rbi.max_acting_osd_load / load_per_osd));
+  if (rbi.max_osd < 0) {
+    // This is just a warning since the important value is the rbi.acting_adj_score
+    rbi.err_msg = fmt::format("Warning: Could not find max_load for pool {}", pool_id);
+  } else {
+    rbi.adjusted_score = rbi_round(float(rbi.max_osd_load / load_per_osd));
+  }
+  return 0;
+}
+
+int OSDMap::calc_read_balance_score(CephContext *cct, int64_t pool_id,
+				    read_balance_info_t *p_rbi) const
+{
+  //BUG: wrong score with one PG replica 3 and 4 OSDs
+  if (cct != nullptr)
+    ldout(cct,20) << __func__ << " pool " << get_pool_name(pool_id) << dendl;
+
+  OSDMap tmp_osd_map;
+  tmp_osd_map.deepish_copy_from(*this);
+  if (p_rbi == nullptr) {
+    // The only case where error message is not set - this is not tested in the unit test.
+    if (cct != nullptr)
+      ldout(cct,30) << __func__ << " p_rbi is nullptr." << dendl;
+    return -EINVAL;
+  }
+
+  if (tmp_osd_map.pools.count(pool_id) == 0) {
+    if (cct != nullptr)
+      ldout(cct,30) << __func__ << " pool " << pool_id << " not found." << dendl;
+    zero_rbi(*p_rbi);
+    p_rbi->err_msg = fmt::format("pool {} not found", pool_id);
+    return -ENOENT;
+  }
+
+  const pg_pool_t* pool = tmp_osd_map.get_pg_pool(pool_id);
+  if (!pool->is_replicated()) {
+    zero_rbi(*p_rbi);
+    p_rbi->err_msg = fmt::format("pool {} is not a replicated pool, read balance score is meaningless", pool_id);
+    return -EPERM;
+  }
+  if (pool->opts.is_set(pool_opts_t::READ_RATIO)) {
+    // if read_ratio is set use osd-size-aware read balance score
+    return calc_rbs_size_optimal(cct, tmp_osd_map, pool_id, pool, *p_rbi);
+  } else {
+    // if read ratio is not set use fair read balance score
+    return calc_rbs_fair(cct, tmp_osd_map, pool_id, pool, *p_rbi);
+  }
 }
 
 int OSDMap::get_osds_by_bucket_name(const string &name, set<int> *osds) const
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 31da6e863743..0378afdc186d 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -1635,17 +1635,32 @@ bool try_drop_remap_underfull(
   );
 
 public:
-    typedef struct {
-      float pa_avg;
-      float pa_weighted;
-      float pa_weighted_avg;
-      float raw_score;
-      float optimal_score;  	// based on primary_affinity values
-      float adjusted_score; 	// based on raw_score and pa_avg 1 is optimal
-      float acting_raw_score;   // based on active_primaries (temporary)
-      float acting_adj_score;   // based on raw_active_score and pa_avg 1 is optimal
-      std::string  err_msg;
-    } read_balance_info_t;
+  typedef enum {
+    RBS_FAIR = 0,
+    RBS_SIZE_OPTIMAL,
+  } read_balance_score_t;
+
+  typedef struct {
+    read_balance_score_t score_type;
+    float pa_avg;
+    float pa_weighted;
+    float pa_weighted_avg;
+    float raw_score;
+    float optimal_score;  	// based on primary_affinity values
+    float adjusted_score; 	// based on raw_score and pa_avg 1 is optimal
+    float acting_raw_score;   // based on active_primaries (temporary)
+    float acting_adj_score;   // based on raw_active_score and pa_avg 1 is optimal
+    int64_t max_osd;
+    int64_t max_osd_load;
+    int64_t max_osd_pgs;
+    int64_t max_osd_prims;
+    int64_t max_acting_osd;
+    int64_t max_acting_osd_load;
+    int64_t max_acting_osd_pgs;
+    int64_t max_acting_osd_prims;
+    int64_t avg_osd_load;
+    std::string  err_msg;
+  } read_balance_info_t;
   //
   // This function calculates scores about the cluster read balance state
   // p_rb_info->acting_adj_score is the current read balance score (acting)
@@ -1659,6 +1674,20 @@ bool try_drop_remap_underfull(
     read_balance_info_t *p_rb_info) const;
 
 private:
+  int calc_rbs_fair(
+    CephContext *cct,
+    OSDMap& tmp_osd_map,
+    int64_t pool_id,
+    const pg_pool_t *pgpool,
+    read_balance_info_t &rb_info) const;
+
+  int calc_rbs_size_optimal(
+    CephContext *cct,
+    OSDMap& tmp_osd_map,
+    int64_t pool_id,
+    const pg_pool_t *pgpool,
+    read_balance_info_t &p_rb_info) const;
+
   float rbi_round(float f) const {
     return (f > 0.0) ? floor(f * 100 + 0.5) / 100 : ceil(f * 100 - 0.5) / 100;
   }
@@ -1671,7 +1700,7 @@ bool try_drop_remap_underfull(
     read_balance_info_t &rbi
   ) const;
 
-  int set_rbi(
+  int set_rbi_fair(
     CephContext *cct,
     read_balance_info_t &rbi,
     int64_t pool_id,

From 6d27bbf77ee8330861028baaa6f3895151db2e2f Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Wed, 24 Jan 2024 14:46:07 +0200
Subject: [PATCH 1699/2492] osd: unit tests for read balance osd size aware
 score

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/test/osd/TestOSDMap.cc | 67 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc
index 6dd9309968e9..31e34b5e2815 100644
--- a/src/test/osd/TestOSDMap.cc
+++ b/src/test/osd/TestOSDMap.cc
@@ -2949,6 +2949,73 @@ TEST_F(OSDMapTest, rb_osdsize_opt_1large_mixed_osds) {
   return;
 }
 
+TEST_F(OSDMapTest, rb_osdsize_opt_score) {
+  //TO-REMOVE (the comment) - look ar 43124 for examples
+  vector <pair<int, int>> weights = {
+      {57, 0}, {90, 0}, {100, 0}, {90, 0}, {100, 0},
+  };
+  set_up_map_heterogeneous(weights, 1);
+  //set_verbose(true);
+
+  // Make sure capacity is balanced first
+  balance_capacity(my_rep_pool);
+  map<uint64_t,set<pg_t>> prim_pgs_by_osd;
+  map<uint64_t,set<pg_t>> pgs_by_osd = osdmap.get_pgs_by_osd(g_ceph_context, my_rep_pool, &prim_pgs_by_osd);
+  OSDMap::read_balance_info_t rbi;
+
+  vector<int> read_ratios = {10, 25, 50, 70, 75, 80, 90, 100};
+
+  // calc fair score
+  auto rc = osdmap.calc_read_balance_score(g_ceph_context, my_rep_pool, &rbi);
+  ASSERT_GE(rc, 0);
+  // assert type is fair (no more checks needed since other tests check this)
+  ASSERT_EQ(rbi.score_type, OSDMap::RBS_FAIR);
+  float fair_score = rbi.acting_adj_score;
+  // set read_ratio
+  for (int rr : read_ratios) {
+    set_pool_read_ratio(my_rep_pool, rr);
+    // calc score
+    rc = osdmap.calc_read_balance_score(g_ceph_context, my_rep_pool, &rbi);
+    ASSERT_GE(rc, 0);
+    // assert type is size-optimal
+    ASSERT_EQ(rbi.score_type, OSDMap::RBS_SIZE_OPTIMAL);
+    // check that osd info is correct
+    if (is_verbose()) {
+      cout << "max_osd: " << rbi.max_osd << std::endl
+          << "max_osd_load: " << rbi.max_osd_load << std::endl
+          << "max_osd_pgs: " << rbi.max_osd_pgs << std::endl
+          << "max_osd_prims: " << rbi.max_osd_prims << std::endl
+          << "max_acting_osd: " << rbi.max_acting_osd << std::endl
+          << "max_acting_osd_load: " << rbi.max_acting_osd_load << std::endl
+          << "max_acting_osd_pgs: " << rbi.max_acting_osd_pgs << std::endl
+          << "max_acting_osd_prims: " << rbi.max_acting_osd_prims << std::endl
+          << "avg_osd_load: " << rbi.avg_osd_load << std::endl
+          << "acting_adj_score: " << rbi.acting_adj_score << std::endl
+          << "adjusted_score: " << rbi.adjusted_score << std::endl
+          << "fair_score: " << fair_score << std::endl;
+    }
+    ASSERT_TRUE(pgs_by_osd.contains(rbi.max_osd));
+    ASSERT_EQ(rbi.max_osd_pgs, pgs_by_osd.at(rbi.max_osd).size());
+    ASSERT_TRUE(prim_pgs_by_osd.contains(rbi.max_osd));
+    ASSERT_EQ(rbi.max_osd_prims, prim_pgs_by_osd.at(rbi.max_osd).size());
+    ASSERT_GE(rbi.acting_adj_score, 1.0);
+    if (rr <= 85) {
+      // with high read ratios, scores can be equal
+      ASSERT_NE(rbi.acting_adj_score, fair_score);
+    }
+    ASSERT_FLOAT_EQ(rbi.acting_adj_score, rbi.adjusted_score);
+    auto total_load = rbi.avg_osd_load * weights.size();
+    const pg_pool_t *p = osdmap.get_pg_pool(my_rep_pool);
+    auto pg_load = total_load / p->get_pg_num();
+    ASSERT_GE(rbi.max_osd_load, pg_load);
+    if (p->get_size() > 1) {
+      int wr = (pg_load - 100) / (p->get_size() - 1);
+      ASSERT_EQ(wr, 100 - rr);
+    }
+  }
+  return;
+}
+
 INSTANTIATE_TEST_SUITE_P(
   OSDMap,
   OSDMapTest,

From 7386d0a9d1a69a58076d46fdbb53d41f8b4b9807 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 30 Jan 2024 20:38:40 +0000
Subject: [PATCH 1700/2492] mgr/balancer: add new keys to balancer mgr module

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/pybind/mgr/balancer/module.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 000b0a3a6cc9..c98ce9aec41c 100644
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -926,16 +926,15 @@ def calc_eval(self, ms: MappingState, pools: List[str]) -> Eval:
             try:
                 read_balance_scores = pi['read_balance']
                 pe.read_balance_score_acting_by_pool[pool] = read_balance_scores['score_acting']
-                pe.read_balance_score_by_pool[pool] = {
-                    'score_acting': read_balance_scores['score_acting'],
-                    'score_stable': read_balance_scores['score_stable'],
-                    'optimal_score': read_balance_scores['optimal_score'],
-                    'raw_score_acting': read_balance_scores['raw_score_acting'],
-                    'raw_score_stable': read_balance_scores['raw_score_stable'],
-                    'primary_affinity_weighted': read_balance_scores['primary_affinity_weighted'],
-                    'average_primary_affinity': read_balance_scores['average_primary_affinity'],
-                    'average_primary_affinity_weighted': read_balance_scores['average_primary_affinity_weighted']
-                }
+                score_keys = ['score_type', 'score_acting', 'score_stable',
+                              'optimal_score', 'raw_score_acting', 'raw_score_stable',
+                              'primary_affinity_weighted', 'average_primary_affinity',
+                              'average_primary_affinity_weighted', 'average_osd_load',
+                              'most_loaded_osd', 'most_loaded_acting_osd']
+                pe.read_balance_score_by_pool[pool] = {}
+                for key in score_keys:
+                    if key in read_balance_scores:
+                        pe.read_balance_score_by_pool[pool][key] = read_balance_scores[key]
             except KeyError:
                 self.log.debug("Skipping pool '{}' since it does not have a read_balance_score, "
                                "likely because it is not replicated.".format(pool))
@@ -1101,8 +1100,9 @@ def do_read_balancing(self, plan: Plan) -> Tuple[int, str]:
             if 'read_balance' in p:
                 if 'error_message' in p['read_balance']:
                     rb_error_message[p['pool_name']] = p['read_balance']['error_message']
-                elif p['read_balance']['score_acting'] == p['read_balance']['optimal_score']:
-                    replicated_pools_with_optimal_score.append(p['pool_name'])
+                elif 'optimal_score' in p['read_balance']:
+                    if p['read_balance']['score_acting'] == p['read_balance']['optimal_score']:
+                        replicated_pools_with_optimal_score.append(p['pool_name'])
         for pool in pools:
             if pool not in crush_rule_by_pool_name:
                 self.log.debug('pool %s does not exist' % pool)

From 00ee28b427c327aa6dbaa31e70f5cbf9c387f597 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 30 Jan 2024 22:06:03 +0000
Subject: [PATCH 1701/2492] tools: add 'osd-size-aware' mode to osdmaptool

Sample command:
`osdmaptool om --read out.txt --read-pool cephfs.a.data --osd-size-aware`

Signed-off-by: Laura Flores <lflores@ibm.com>
---
 src/test/cli/osdmaptool/help.t |  1 +
 src/tools/osdmaptool.cc        | 28 +++++++++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/test/cli/osdmaptool/help.t b/src/test/cli/osdmaptool/help.t
index 624fe9102e69..327d0b183caf 100644
--- a/src/test/cli/osdmaptool/help.t
+++ b/src/test/cli/osdmaptool/help.t
@@ -38,5 +38,6 @@
      --save                  write modified osdmap with upmap or crush-adjust changes
      --read <file>           calculate pg upmap entries to balance pg primaries
      --read-pool <poolname>  specify which pool the read balancer should adjust
+     --osd-size-aware        account for devices of different sizes, applicable to read mode only
      --vstart                prefix upmap and read output with './bin/'
   [1]
diff --git a/src/tools/osdmaptool.cc b/src/tools/osdmaptool.cc
index ddaf2e65a0c5..56e86c845911 100644
--- a/src/tools/osdmaptool.cc
+++ b/src/tools/osdmaptool.cc
@@ -69,6 +69,7 @@ void usage()
   cout << "   --save                  write modified osdmap with upmap or crush-adjust changes" << std::endl;
   cout << "   --read <file>           calculate pg upmap entries to balance pg primaries" << std::endl;
   cout << "   --read-pool <poolname>  specify which pool the read balancer should adjust" << std::endl;
+  cout << "   --osd-size-aware        account for devices of different sizes, applicable to read mode only" << std::endl;
   cout << "   --vstart                prefix upmap and read output with './bin/'" << std::endl;
   exit(1);
 }
@@ -181,6 +182,7 @@ int main(int argc, const char **argv)
   bool test_map_pgs_dump_all = false;
   bool save = false;
   bool vstart = false;
+  bool osd_size_aware = false;
 
   std::string val;
   std::ostringstream err;
@@ -292,6 +294,8 @@ int main(int argc, const char **argv)
       save = true;
     } else if (ceph_argparse_flag(args, i, "--vstart", (char*)NULL)) {
       vstart = true;
+    } else if (ceph_argparse_flag(args, i, "--osd-size-aware", (char*)NULL)) {
+      osd_size_aware = true;
     } else {
       ++i;
     }
@@ -308,6 +312,10 @@ int main(int argc, const char **argv)
     cerr << me << ": upmap-deviation must be >= 1" << std::endl;
     usage();
   }
+  if (!read && osd_size_aware) {
+    cerr << me << ": osd-size-aware is only applicable to read mode" << std::endl;
+    usage();
+  }
   fn = args[0];
 
   if (range_first >= 0 && range_last >= 0) {
@@ -484,6 +492,19 @@ int main(int argc, const char **argv)
       exit(1);
     }
 
+    int64_t read_ratio = 0;
+    if (osd_size_aware) {
+      pool->opts.get(pool_opts_t::READ_RATIO, &read_ratio);
+      if (read_ratio <= 0 || read_ratio > 100) {
+	cerr << "The read ratio for pool " << read_pool << " is unset or invalid."
+	     << " To set read ratio, please run 'ceph osd pool set <pool name> read_ratio <value>'." << std::endl;
+	exit(1);
+      } else {
+	cout << "Accounting for devices of different sizes on pool " << read_pool
+	     << " with a read ratio of " << read_ratio << "." << std::endl;
+      }
+    }
+
     OSDMap tmp_osd_map;
     tmp_osd_map.deepish_copy_from(osdmap);
 
@@ -498,8 +519,13 @@ int main(int argc, const char **argv)
     ceph_assert(read_balance_score_before >= 0);
 
     // Calculate read balancer
+    int num_changes = 0;
     OSDMap::Incremental pending_inc(osdmap.get_epoch()+1);
-    int num_changes = osdmap.balance_primaries(g_ceph_context, pid, &pending_inc, tmp_osd_map);
+    if (osd_size_aware) { // account for different device sizes
+      num_changes = osdmap.balance_primaries(g_ceph_context, pid, &pending_inc, tmp_osd_map, OSDMap::RB_OSDSIZEOPT);
+    } else { // default
+      num_changes = osdmap.balance_primaries(g_ceph_context, pid, &pending_inc, tmp_osd_map);
+    }
 
     if (num_changes < 0) {
       cerr << "Error balancing primaries. Rerun with at least --debug-osd=10 for more details." << std::endl;

From 538f94cf168cbccab2331197c1a7a1fe8b954847 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 25 Jan 2024 13:13:41 +0530
Subject: [PATCH 1702/2492] vstart: add nvmeof_gw to the vstart script

so that it can be later used by the dashboard to configure the nvmeof
through UI

and create rbd pool in UI

Fixes: https://tracker.ceph.com/issues/64201
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 ceph.spec.in  |  6 ++++++
 src/vstart.sh | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index d061803099ea..139c462eef7d 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -625,14 +625,20 @@ Requires:       ceph-mgr = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-grafana-dashboards = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-prometheus-alerts = %{_epoch_prefix}%{version}-%{release}
 Requires:       python%{python3_pkgversion}-setuptools
+%if 0%{?fedora} || 0%{?rhel} >= 9
 Requires:       python%{python3_pkgversion}-grpcio
 Requires:       python%{python3_pkgversion}-grpcio-tools
+%endif
 %if 0%{?fedora} || 0%{?rhel} || 0%{?openEuler}
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-routes
 Requires:       python%{python3_pkgversion}-werkzeug
 %if 0%{?weak_deps}
 Recommends:     python%{python3_pkgversion}-saml
+%if 0%{?fedora} || 0%{?rhel} <= 8
+Recommends:     python%{python3_pkgversion}-grpcio
+Recommends:     python%{python3_pkgversion}-grpcio-tools
+%endif
 %endif
 %endif
 %if 0%{?suse_version}
diff --git a/src/vstart.sh b/src/vstart.sh
index 0c9ef3237791..2317d5da6053 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -206,7 +206,7 @@ inc_osd_num=0
 msgr="21"
 
 read -r -d '' usage <<EOF || true
-usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 $0 -n -d
+usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 NVMEOF_GW=ceph:5500 $0 -n -d
 options:
 	-d, --debug
 	-t, --trace
@@ -1302,6 +1302,14 @@ EOF
                     debug echo dashboard module not working correctly!
                 fi
             fi
+
+            ceph_adm osd pool create rbd
+            ceph_adm osd pool application enable rbd rbd
+
+            if [ -n "${NVMEOF_GW}" ]; then
+                echo "Adding nvmeof-gateway ${NVMEOF_GW} to dashboard"
+                ceph_adm dashboard nvmeof-gateway-add -i <(echo "${NVMEOF_GW}") "${NVMEOF_GW/:/_}"
+            fi
         fi
         if $with_mgr_restful; then
             create_mgr_restful_secret

From 8516c0ebc5a83eb647b482df7b977f41bd894b32 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 25 Jan 2024 13:05:01 -0600
Subject: [PATCH 1703/2492] osd/scrub: directly manage remote reservations in
 the FSM

The FSM now interacts with the scrub_reserver directly.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc   |   9 ++
 src/osd/scrubber/pg_scrubber.h    |   2 +
 src/osd/scrubber/scrub_machine.cc | 152 ++++++++++++++++++------------
 src/osd/scrubber/scrub_machine.h  | 135 +++++++++++++++++++++++---
 src/osd/scrubber_common.h         |  31 +++++-
 5 files changed, 253 insertions(+), 76 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 3ef5a2ef567d..9266a54d7858 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -397,6 +397,15 @@ void PgScrubber::send_scrub_is_finished(epoch_t epoch_queued)
   dout(10) << "scrubber event --<< " << __func__ << dendl;
 }
 
+void PgScrubber::send_granted_by_reserver(const AsyncScrubResData& req)
+{
+  dout(10) << "scrubber event -->> granted_by_reserver" << dendl;
+  if (check_interval(req.request_epoch)) {
+    m_fsm->process_event(Scrub::ReserverGranted{req});
+  }
+  dout(10) << "scrubber event --<< granted_by_reserver" << dendl;
+}
+
 // -----------------
 
 bool PgScrubber::is_reserving() const
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index 9c29d5fdedb9..bcab24cddfa3 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -227,6 +227,8 @@ class PgScrubber : public ScrubPgIF,
 
   void send_scrub_is_finished(epoch_t epoch_queued) final;
 
+  void send_granted_by_reserver(const AsyncScrubResData& req) final;
+
   /**
    *  we allow some number of preemptions of the scrub, which mean we do
    *  not block.  Then we start to block.  Once we start blocking, we do
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index fc5238186868..57c0492ec2e4 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -766,12 +766,7 @@ ReplicaActive::ReplicaActive(my_context ctx)
 
 ReplicaActive::~ReplicaActive()
 {
-  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  if (reserved_by_my_primary) {
-    dout(10) << "ReplicaActive::~ReplicaActive(): clearing reservation"
-	     << dendl;
-    clear_reservation_by_remote_primary(false);
-  }
+  clear_remote_reservation(false);
 }
 
 /*
@@ -800,85 +795,118 @@ ReplicaReactCode ReplicaActive::on_reserve_request(
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   const auto m = ev.m_op->get_req<MOSDScrubReserve>();
-  const auto msg_nonce = m->reservation_nonce;
   dout(10) << fmt::format(
-		  "ReplicaActive::on_reserve_req() from {} request:{} is "
-		  "async?{} (reservation_nonce:{})",
-		  ev.m_from, ev, async_request, msg_nonce)
+		  "ReplicaActive::on_reserve_req() request:{} async_request:{} "
+		  "reservation_nonce:{}",
+		  ev, async_request, m->reservation_nonce)
 	   << dendl;
-  auto& svc = m_osds->get_scrub_services();  // shorthand
-
-  if (reserved_by_my_primary) {
-    dout(10) << "ReplicaActive::on_reserve_request(): already reserved"
-	     << dendl;
-    // clear the existing reservation. Clears the flag, too
-    clear_reservation_by_remote_primary(false);
-  }
 
-  Message* reply{nullptr};
+  ceph_assert(!reservation_granted);
+  ceph_assert(!pending_reservation_nonce);
   ReplicaReactCode next_action{ReplicaReactCode::discard};
+  AsyncScrubResData request_details{
+      pg_id, ev.m_from, ev.m_op->sent_epoch, m->reservation_nonce};
+  auto& reserver = m_osds->get_scrub_reserver();
 
   if (async_request) {
     // the request is to be handled asynchronously
-    svc.enqueue_remote_reservation(pg_id.pgid);
+    dout(20) << fmt::format(
+		    "{}: async request: {} details:{}", __func__, ev,
+		    request_details)
+	     << dendl;
+    pending_reservation_nonce = m->reservation_nonce;
+    const auto reservation_cb = new RtReservationCB(m_pg, request_details);
+    reserver.request_reservation(pg_id, reservation_cb, 0, nullptr);
     next_action = ReplicaReactCode::goto_waiting_reservation;
 
   } else {
     // an immediate yes/no is required
-    const auto granted = svc.inc_scrubs_remote(scrbr->get_spgid().pgid);
-    if (granted) {
-      reserved_by_my_primary = true;
+    Message* reply{nullptr};
+    reservation_granted = reserver.request_reservation_or_fail(pg_id);
+    if (reservation_granted) {
       dout(10) << fmt::format("{}: reserved? yes", __func__) << dendl;
       reply = new MOSDScrubReserve(
 	  spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch,
-	  MOSDScrubReserve::GRANT, m_pg->pg_whoami, msg_nonce);
+	  MOSDScrubReserve::GRANT, m_pg->pg_whoami, m->reservation_nonce);
       next_action = ReplicaReactCode::goto_replica_reserved;
 
     } else {
       dout(10) << fmt::format("{}: reserved? no", __func__) << dendl;
       reply = new MOSDScrubReserve(
 	  spg_t(pg_id.pgid, m_pg->get_primary().shard), ev.m_op->sent_epoch,
-	  MOSDScrubReserve::REJECT, m_pg->pg_whoami, msg_nonce);
+	  MOSDScrubReserve::REJECT, m_pg->pg_whoami, m->reservation_nonce);
       // the event is discarded
       next_action = ReplicaReactCode::discard;
     }
-  }
 
-  if (reply) {
     m_osds->send_message_osd_cluster(
 	reply, ev.m_op->get_req()->get_connection());
   }
+
   return next_action;
 }
 
+bool ReplicaActive::granted_by_reserver(const AsyncScrubResData& reservation)
+{
+  DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
+  dout(10) << fmt::format("{}: reservation granted: {}", __func__, reservation)
+	   << dendl;
+
+  /// verify that the granted reservation is the one we were waiting for
+  if (reservation.nonce != pending_reservation_nonce) {
+    dout(5) << fmt::format(
+	"{}: reservation_nonce mismatch: {} != {}", __func__, reservation.nonce,
+	pending_reservation_nonce) << dendl;
+    return false;
+  }
+
+  reservation_granted = true;
+  pending_reservation_nonce = 0;  // no longer pending
+
+  // notify the primary
+  auto grant_msg = make_message<MOSDScrubReserve>(
+      spg_t(pg_id.pgid, m_pg->get_primary().shard), reservation.request_epoch,
+      MOSDScrubReserve::GRANT, m_pg->pg_whoami, pending_reservation_nonce);
+  m_pg->send_cluster_message(
+      m_pg->get_primary().osd, grant_msg, reservation.request_epoch, false);
+  return true;
+}
+
 void ReplicaActive::on_release(const ReplicaRelease& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << fmt::format("ReplicaActive::on_release() from {}", ev.m_from)
 	   << dendl;
-  clear_reservation_by_remote_primary(true);
+  clear_remote_reservation(true);
 }
 
-void ReplicaActive::clear_reservation_by_remote_primary(bool log_failure)
+void ReplicaActive::clear_remote_reservation(bool warn_if_no_reservation)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << fmt::format(
-		  "ReplicaActive::clear_reservation_by_remote_primary(): was "
-		  "reserved? {}",
-		  (reserved_by_my_primary ? "yes" : "no"))
+		  "ReplicaActive::clear_remote_reservation(): "
+		  "pending_reservation_nonce {}, reservation_granted {}",
+		  reservation_granted, pending_reservation_nonce)
 	   << dendl;
-  if (reserved_by_my_primary) {
-    m_osds->get_scrub_services().dec_scrubs_remote(scrbr->get_spgid().pgid);
-    reserved_by_my_primary = false;
-  } else if (log_failure) {
-    const auto msg = fmt::format(
-	"ReplicaActive::clear_reservation_by_remote_primary(): "
-	"not reserved!");
+  if (reservation_granted || pending_reservation_nonce) {
+    m_osds->get_scrub_reserver().cancel_reservation(pg_id);
+    reservation_granted = false;
+    pending_reservation_nonce = 0;
+  } else if (warn_if_no_reservation) {
+    const auto msg =
+	"ReplicaActive::clear_remote_reservation(): "
+	"not reserved!";
     dout(5) << msg << dendl;
     scrbr->get_clog()->warn() << msg;
   }
 }
 
+void ReplicaActive::ignore_unhandled_grant(const ReserverGranted&)
+{
+  dout(10) << "ReplicaActive::react(const ReserverGranted&): ignored"
+	   << dendl;
+}
+
 
 // ---------------- ReplicaActive/ReplicaIdle ---------------------------
 
@@ -913,7 +941,8 @@ sc::result ReplicaUnreserved::react(const ReplicaReserveReq& ev)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReplicaUnreserved::react(const ReplicaReserveReq&)" << dendl;
 
-  switch (context<ReplicaActive>().on_reserve_request(ev, false)) {
+  bool async_request = true && ev.m_op->get_req<MOSDScrubReserve>()->wait_for_resources /* && a config */;
+  switch (context<ReplicaActive>().on_reserve_request(ev, async_request)) {
     case ReplicaReactCode::discard:
       return discard_event();
     case ReplicaReactCode::goto_waiting_reservation:
@@ -939,15 +968,11 @@ sc::result ReplicaUnreserved::react(const ReplicaRelease&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReplicaUnreserved::react(const ReplicaRelease&)" << dendl;
-  // shouldn't happen. Possible (faulty) sequence: getting an op
-  // command while in ReplicaWaitingReservation (as we would just
-  // treat that as a regular op request, but will stop waiting for
-  // reservation).
-  // must cancel the queued reservation request
+  // this is a bug. We should never receive a release request unless we
+  // are reserved or have a pending reservation.
   scrbr->get_clog()->error() << fmt::format(
       "osd.{} pg[{}]: reservation released while not reserved",
       scrbr->get_whoami(), scrbr->get_spgid());
-  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
   return discard_event();
 }
 
@@ -955,15 +980,13 @@ sc::result ReplicaUnreserved::react(const ReserverGranted&)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReplicaUnreserved::react(const ReserverGranted&)" << dendl;
-  // shouldn't happen. Possible (faulty) sequence: getting an op
-  // command while in ReplicaWaitingReservation (as we would just
-  // treat that as a regular op request, but will stop waiting for
-  // reservation).
+  // shouldn't happen. Might be a result of a cancelled reservation
+  // that was still delivered.
   // must unreserve
-  scrbr->get_clog()->error() << fmt::format(
-      "osd.{} pg[{}]: reservation granted while not being waited for",
-      scrbr->get_whoami(), scrbr->get_spgid());
-  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  dout(5) << "ReplicaUnreserved::react(const ReserverGranted&): reservation "
+	     "granted while not being waited for"
+	  << dendl;
+  context<ReplicaActive>().clear_remote_reservation(false);
   return discard_event();
 }
 
@@ -981,14 +1004,17 @@ ReplicaWaitingReservation::ReplicaWaitingReservation(my_context ctx)
       << dendl;
 }
 
-sc::result ReplicaWaitingReservation::react(const ReserverGranted&)
+sc::result ReplicaWaitingReservation::react(const ReserverGranted& ev)
 {
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
-  dout(10) << "ReplicaWaitingReservation::react(const ReserverGranted&)"
+  dout(10) << fmt::format(
+		  "ReplicaWaitingReservation::react(const ReserverGranted&): "
+		  "event:{}",
+		  ev)
 	   << dendl;
-
-  /// \todo complete the handling of the granted reservation
-  ceph_abort_msg("not implemented yet");
+  if (context<ReplicaActive>().granted_by_reserver(ev.value)) {
+    return transit<ReplicaReserved>();
+  }
   return discard_event();
 }
 
@@ -1012,6 +1038,7 @@ sc::result ReplicaWaitingReservation::react(const StartReplica& ev)
       "osd.{} pg[{}]: new chunk request while still waiting for "
       "reservation",
       scrbr->get_whoami(), scrbr->get_spgid());
+  context<ReplicaActive>().clear_remote_reservation(true);
   clear_shallow_history<ReplicaIdle, 0>();
   post_event(ReplicaPushesUpd{});
   return transit<ReplicaActiveOp>();
@@ -1027,7 +1054,7 @@ sc::result ReplicaWaitingReservation::react(const ReplicaReserveReq& ev)
       "osd.{} pg[{}]: reservation requested while previous is pending",
       scrbr->get_whoami(), scrbr->get_spgid());
   // cancel the existing reservation, and re-request
-  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  context<ReplicaActive>().clear_remote_reservation(true);
   post_event(ev);
   return transit<ReplicaUnreserved>();
 }
@@ -1060,8 +1087,11 @@ sc::result ReplicaReserved::react(const ReplicaReserveReq& ev)
   scrbr->get_clog()->error() << fmt::format(
       "osd.{} pg[{}]: reservation requested while still reserved",
       scrbr->get_whoami(), scrbr->get_spgid());
+  // This is a bug. We should never receive a new request unless the
+  // previous one was cancelled - either by the primary, or on interval
+  // change.
   // cancel the existing reservation, and re-request
-  context<ReplicaActive>().clear_reservation_by_remote_primary(true);
+  context<ReplicaActive>().clear_remote_reservation(true);
   post_event(ev);
   return transit<ReplicaUnreserved>();
 }
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 6eadb109cd54..254e7861ed95 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -16,13 +16,11 @@
 #include <boost/statechart/transition.hpp>
 
 #include "common/fmt_common.h"
+#include "include/Context.h"
 #include "common/version.h"
 #include "messages/MOSDOp.h"
 #include "messages/MOSDRepScrub.h"
 #include "messages/MOSDRepScrubMap.h"
-#include "messages/MOSDScrubReserve.h"
-
-#include "include/Context.h"
 #include "osd/scrubber_common.h"
 
 #include "scrub_machine_lstnr.h"
@@ -136,7 +134,7 @@ struct value_event_t : sc::event<T> {
 
 
 /// the async-reserver granted our reservation request
-OP_EV(ReserverGranted);
+VALUE_EVENT(ReserverGranted, AsyncScrubResData);
 
 #define MEV(E)                                          \
   struct E : sc::event<E> {                             \
@@ -783,6 +781,41 @@ struct WaitDigestUpdate : sc::state<WaitDigestUpdate, ActiveScrubbing>,
  *      * ReplicaWaitUpdates
  *      * ReplicaBuildingMap
  */
+/*
+ * AsyncReserver for scrub 'remote' reservations
+ * -----------------------------------------------
+ *
+ * Unless disabled by 'osd_scrub_disable_reservation_queuing' (*), scrub
+ * reservation requests are handled by an async reserver: they are queued,
+ * until the number of concurrent scrubs is below the configured limit.
+
+ * (*) Note: the 'osd_scrub_disable_reservation_queuing' option is a temporary
+ * debug measure, and will be removed without deprecation in a future release.
+ *
+ * On the replica side, all reservations are treated as having the same priority.
+ * Note that 'high priority' scrubs, e.g. user-initiated scrubs, do not perform
+ * reservations on replicas at all.
+ *
+ * A queued scrub reservation request is cancelled by any of the following events:
+ *
+ * - a new interval: in this case, we do not expect to see a cancellation request
+ *   from the primary, and we can simply remove the request from the queue;
+ *
+ * - a cancellation request from the primary: probably a result of timing out on
+ *   the reservation process. Here, we can simply remove the request from the queue.
+ *
+ * - a new reservation request for the same PG: this is a bug. We had missed the
+ *   previous cancellation request, which could never happen.
+ *   We cancel the previous request, and replace
+ *   it with the new one. We would also issue an error log message.
+ *
+ * Primary/Replica with differing versions:
+ *
+ * The updated version of MOSDScrubReserve contains a new 'wait_for_resources'
+ * field. For legacy Primary OSDs, this field is decoded as 'false', and the
+ * replica responds immediately, with grant/rejection.
+*/
+
 
 struct ReplicaIdle;
 
@@ -809,22 +842,95 @@ struct ReplicaActive : sc::state<
       const ReplicaReserveReq&,
       bool async_request);
 
+  /**
+   * the queued reservation request was granted by the async reserver.
+   * Notify the Primary.
+   * Returns 'false' if the reservation is not the last one to be received
+   * by this replica.
+   */
+  bool granted_by_reserver(const AsyncScrubResData& resevation);
+
   /// handle a 'release' from a primary
   void on_release(const ReplicaRelease& ev);
 
-  /// cancel the reserver request.
-  /// The 'failure' re 'log_failure' is logged if we are not reserved to
-  /// begin with.
-  void clear_reservation_by_remote_primary(bool log_failure);
+  /**
+   * cancel a granted or pending reservation
+   *
+   * warn_if_no_reservation is set to true if the call is in response to a
+   * cancellation from the primary.  In that event, we *must* find a
+   * a granted or pending reservation and failing to do so warrants
+   * a warning to clog as it is a bug.
+   */
+  void clear_remote_reservation(bool warn_if_no_reservation);
+
+  /**
+   * discard (and log) unhandled 'reservation granted' messages
+   * from the async reserver.
+   * As canceled reservations may still be triggered, this is not
+   * necessarily a bug.
+   */
+  void ignore_unhandled_grant(const ReserverGranted&);
 
-  using reactions = mpl::list<sc::transition<IntervalChanged, NotActive>>;
+  using reactions = mpl::list<
+      sc::transition<IntervalChanged, NotActive>,
+      sc::in_state_reaction<
+	  ReserverGranted,
+	  ReplicaActive,
+	  &ReplicaActive::ignore_unhandled_grant>>;
 
  private:
-  bool reserved_by_my_primary{false};
-
-  // shortcuts:
   PG* m_pg;
   OSDService* m_osds;
+
+  // --- remote reservation machinery
+
+  /*
+   * 'reservation_granted' is set to 'true' when we have grant confirmation
+   *  to the primary, and the reservation has not yet been canceled (either
+   *  by the primary or following an interval change).
+   *
+   * Note the interaction with 'pending_reservation_nonce': the combination
+   * of these two variables is used to track the state of the reservation
+   * with the scrub_reserver. The possible combinations:
+   * - pending_reservation_nonce == 0 && !reservation_granted -- no reservation
+   *    was granted, and none is pending;
+   * - pending_reservation_nonce != 0 && !reservation_granted -- we have a
+   *   pending cb in the AsyncReserver for a request with nonce
+   *   'pending_reservation_nonce'
+   * - pending_reservation_nonce == 0 && reservation_granted -- we have sent
+   *   a response to the primary granting the reservation
+   * (invariant: !((pending_reservation_nonce != 0) && reservation_granted)
+   *
+   * Note that in the event that the primary is too old to support asynchronous
+   * reservation, MOSDScrubReserve::wait_for_resources will be set to false by
+   * the decoder and we bypass the 2'nd case above.
+   * See ReplicaActive::on_reserve_request().
+   */
+  bool reservation_granted{false};
+
+  /**
+   * a reservation request with this nonce is queued at the scrub_reserver,
+   * and was not yet granted.
+   */
+  MOSDScrubReserve::reservation_nonce_t pending_reservation_nonce{0};
+
+  // clang-format off
+  struct RtReservationCB : public Context {
+    PGRef pg;
+    AsyncScrubResData res_data;
+
+    explicit RtReservationCB(PGRef pg, AsyncScrubResData request_details)
+	: pg{pg}
+	, res_data{request_details}
+    {}
+
+    void finish(int) override {
+      pg->lock();
+      pg->m_scrubber->send_granted_by_reserver(res_data);
+      pg->unlock();
+    }
+  };
+  // clang-format on
 };
 
 
@@ -862,9 +968,10 @@ struct ReplicaUnreserved : sc::state<ReplicaUnreserved, ReplicaIdle>,
 
   using reactions = mpl::list<
       sc::custom_reaction<ReplicaReserveReq>,
+      sc::custom_reaction<StartReplica>,
+      // unexpected (bug-induced) events:
       sc::custom_reaction<ReplicaRelease>,
-      sc::custom_reaction<ReserverGranted>,
-      sc::custom_reaction<StartReplica>>;
+      sc::custom_reaction<ReserverGranted>>;
 
   sc::result react(const ReplicaReserveReq& ev);
   sc::result react(const StartReplica& ev);
diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 067b9754c110..66e61d856cd4 100644
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -3,10 +3,11 @@
 #pragma once
 
 #include <fmt/ranges.h>
-
 #include "common/ceph_time.h"
+#include "common/fmt_common.h"
 #include "common/scrub_types.h"
 #include "include/types.h"
+#include "messages/MOSDScrubReserve.h"
 #include "os/ObjectStore.h"
 
 #include "OpRequest.h"
@@ -24,6 +25,32 @@ namespace Scrub {
   struct ReplicaActive;
 }
 
+/// reservation-related data sent by the primary to the replicas,
+/// and used to match the responses to the requests
+struct AsyncScrubResData {
+  spg_t pgid;
+  pg_shard_t from;
+  epoch_t request_epoch;
+  MOSDScrubReserve::reservation_nonce_t nonce;
+  AsyncScrubResData(
+      spg_t pgid,
+      pg_shard_t from,
+      epoch_t request_epoch,
+      MOSDScrubReserve::reservation_nonce_t nonce)
+      : pgid{pgid}
+      , from{from}
+      , request_epoch{request_epoch}
+      , nonce{nonce}
+  {}
+  template <typename FormatContext>
+  auto fmt_print_ctx(FormatContext& ctx) const
+  {
+    return fmt::format_to(
+	ctx.out(), "pg[{}],f:{},ep:{},n:{}", pgid, from, request_epoch, nonce);
+  }
+};
+
+
 /// Facilitating scrub-related object access to private PG data
 class ScrubberPasskey {
 private:
@@ -317,6 +344,8 @@ struct ScrubPgIF {
 
   virtual void send_scrub_is_finished(epoch_t epoch_queued) = 0;
 
+  virtual void send_granted_by_reserver(const AsyncScrubResData& req) = 0;
+
   virtual void on_applied_when_primary(const eversion_t& applied_version) = 0;
 
   // --------------------------------------------------

From 46e18b9a2d224c6b65ab46a945bcbe26fcd078f6 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 05:45:50 -0600
Subject: [PATCH 1704/2492] osd/scrub: control reservation queueing using a
 config option

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/options/osd.yaml.in    | 10 ++++++++++
 src/osd/scrubber/scrub_machine.cc | 11 ++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index c2281c00ece4..2a7bad84e7b1 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -535,6 +535,16 @@ options:
   see_also:
   - osd_scrub_slow_reservation_response
   with_legacy: false
+- name: osd_scrub_disable_reservation_queuing
+  type: bool
+  level: advanced
+  desc: Disable queuing of scrub reservations
+  long_desc: When set - scrub replica reservations are responded to immediately, with
+    either success or failure (the pre-Squid version behaviour). This configuration
+    option is introduced to support mixed-version clusters and debugging, and will
+    be removed in the next release.
+  default: false
+  with_legacy: false
 # where rados plugins are stored
 - name: osd_class_dir
   type: str
diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index 57c0492ec2e4..ce4196e1ebbb 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -941,7 +941,16 @@ sc::result ReplicaUnreserved::react(const ReplicaReserveReq& ev)
   DECLARE_LOCALS;  // 'scrbr' & 'pg_id' aliases
   dout(10) << "ReplicaUnreserved::react(const ReplicaReserveReq&)" << dendl;
 
-  bool async_request = true && ev.m_op->get_req<MOSDScrubReserve>()->wait_for_resources /* && a config */;
+  const auto& m = *(ev.m_op->get_req<MOSDScrubReserve>());
+  const auto async_disabled = scrbr->get_pg_cct()->_conf.get_val<bool>(
+      "osd_scrub_disable_reservation_queuing");
+  const bool async_request = !async_disabled && m.wait_for_resources;
+  dout(15) << fmt::format(
+		  "ReplicaUnreserved::react(const ReplicaReserveReq&): "
+		  "request:{} disabled?:{} -> async? {}", m.wait_for_resources,
+		  async_disabled, async_request)
+	   << dendl;
+
   switch (context<ReplicaActive>().on_reserve_request(ev, async_request)) {
     case ReplicaReactCode::discard:
       return discard_event();

From cf9ed68c816b513d952e75399828ca306fb852a1 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 09:35:07 -0600
Subject: [PATCH 1705/2492] osd/scrub: restore 'dump_scrub_reservations'
 functionality

As ScrubResources is no longer involved in remote reservations, some
of the data listed by 'dump_scrub_reservations' is now collected by
OsdScrub itself (prior to this change, OsdScrub just forwarded the
request to ScrubResources).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/OSD.cc                     | 2 +-
 src/osd/scrubber/osd_scrub.cc      | 8 ++++++++
 src/osd/scrubber/osd_scrub.h       | 2 ++
 src/osd/scrubber/osd_scrub_sched.h | 7 +++++++
 4 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 61af98924c74..b0ff7883fabf 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2893,7 +2893,7 @@ will start to track new ops received afterwards.";
     f->close_section();
   } else if (prefix == "dump_scrub_reservations") {
     f->open_object_section("scrub_reservations");
-    service.get_scrub_services().resource_bookkeeper().dump_scrub_reservations(f);
+    service.get_scrub_services().dump_scrub_reservations(f);
     f->close_section();
   } else if (prefix == "get_latest_osdmap") {
     get_latest_osdmap();
diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index b4e1ec8cef23..a74e1ae5c30d 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -57,6 +57,14 @@ void OsdScrub::dump_scrubs(ceph::Formatter* f) const
   m_queue.dump_scrubs(f);
 }
 
+void OsdScrub::dump_scrub_reservations(ceph::Formatter* f) const
+{
+  m_resource_bookkeeper.dump_scrub_reservations(f);
+  f->open_array_section("remote_scrub_reservations");
+  m_osd_svc.get_scrub_reserver().dump(f);
+  f->close_section();
+}
+
 void OsdScrub::log_fwd(std::string_view text)
 {
   dout(20) << text << dendl;
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 9aca86ad798f..64709cc7aab8 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -51,6 +51,8 @@ class OsdScrub {
 
   void dump_scrubs(ceph::Formatter* f) const;  ///< fwd to the queue
 
+  void dump_scrub_reservations(ceph::Formatter* f) const;
+
   /**
    * on_config_change() (the refactored "OSD::sched_all_scrubs()")
    *
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index 0eddb22ea7dd..95f1680d403d 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -108,6 +108,7 @@ ScrubQueue interfaces (main functions):
 // clang-format on
 
 #include <optional>
+#include "common/AsyncReserver.h"
 #include "utime.h"
 #include "osd/scrubber/scrub_job.h"
 #include "osd/PG.h"
@@ -135,6 +136,12 @@ class ScrubSchedListener {
    */
   virtual std::optional<PGLockWrapper> get_locked_pg(spg_t pgid) = 0;
 
+  /**
+   * allow access to the scrub_reserver, the AsyncReserver that keeps track
+   * of 'remote replica reservations'.
+   */
+  virtual AsyncReserver<spg_t, Finisher>& get_scrub_reserver() = 0;
+
   virtual ~ScrubSchedListener() {}
 };
 

From 6fa0fa3153522622fdc2128b5d6f4e76ca160ef9 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 28 Jan 2024 10:48:18 -0600
Subject: [PATCH 1706/2492] test: fix test_scrub_sched following recent scrub
 changes

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/osd/test_scrub_sched.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/test/osd/test_scrub_sched.cc b/src/test/osd/test_scrub_sched.cc
index afeb32b21d9d..b6c069c4b5f4 100644
--- a/src/test/osd/test_scrub_sched.cc
+++ b/src/test/osd/test_scrub_sched.cc
@@ -10,6 +10,7 @@
 
 #include "common/async/context_pool.h"
 #include "common/ceph_argparse.h"
+#include "common/Finisher.h"
 #include "global/global_context.h"
 #include "global/global_init.h"
 #include "include/utime_fmt.h"
@@ -109,9 +110,17 @@ class FakeOsd : public Scrub::ScrubSchedListener {
     return std::nullopt;
   }
 
+  AsyncReserver<spg_t, Finisher>& get_scrub_reserver() final
+  {
+    return m_scrub_reserver;
+  }
+
  private:
   int m_osd_num;
   std::map<spg_t, schedule_result_t> m_next_response;
+  Finisher reserver_finisher{g_ceph_context};
+  AsyncReserver<spg_t, Finisher> m_scrub_reserver{
+      g_ceph_context, &reserver_finisher, 1};
 };
 
 

From 7aeac65dc48b9a368d75e7a6af6bfb153425a1f9 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 31 Jan 2024 00:55:45 +0200
Subject: [PATCH 1707/2492] cpp_redis: update to a version with the
 CMAKE_CXX_STANDARD fixed

This updates the cpp_redis submodule to a version
which contains https://github.com/ceph/cpp_redis/pull/5

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/cpp_redis | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cpp_redis b/src/cpp_redis
index c659475ea43b..72d992fff2a9 160000
--- a/src/cpp_redis
+++ b/src/cpp_redis
@@ -1 +1 @@
-Subproject commit c659475ea43bc77850018aa1433d55cad902ea85
+Subproject commit 72d992fff2a95edb37430a75909a844637549331

From d7c9691623542f2bcc238c307d2679a6d51fa129 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Wed, 13 Dec 2023 01:08:52 +0000
Subject: [PATCH 1708/2492] mgr/dashboard: added snap schedule form

Fixes: https://tracker.ceph.com/issues/63827
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  42 ++-
 ...ephfs-snapshotschedule-form.component.html | 179 ++++++++++++
 ...ephfs-snapshotschedule-form.component.scss |   0
 ...fs-snapshotschedule-form.component.spec.ts |  79 ++++++
 .../cephfs-snapshotschedule-form.component.ts | 257 ++++++++++++++++++
 .../cephfs-snapshotschedule-list.component.ts |  21 +-
 .../cephfs-tabs/cephfs-tabs.component.html    |   1 +
 .../src/app/ceph/cephfs/cephfs.module.ts      |  14 +-
 .../api/cephfs-snapshot-schedule.service.ts   |  79 +++++-
 .../src/app/shared/api/cephfs.service.ts      |   4 +-
 .../src/app/shared/enum/icons.enum.ts         |   1 +
 .../app/shared/enum/repeat-frequency.enum.ts  |   5 +
 .../shared/enum/retention-frequency.enum.ts   |   8 +
 .../app/shared/models/snapshot-schedule.ts    |  16 ++
 .../shared/services/task-message.service.ts   |   6 +
 src/pybind/mgr/dashboard/openapi.yaml         |  49 +++-
 16 files changed, 736 insertions(+), 25 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 01827e3ef0ec..86f112522c8f 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -941,25 +941,24 @@ def create(self, vol_name: str, subvol_name: str, snap_name: str, clone_name: st
         return f'Clone {clone_name} created successfully'
 
 
-@APIRouter('/cephfs/snaphost/schedule', Scope.CEPHFS)
+@APIRouter('/cephfs/snapshot/schedule', Scope.CEPHFS)
 @APIDoc("Cephfs Snapshot Scheduling API", "CephFSSnapshotSchedule")
 class CephFSSnapshotSchedule(RESTController):
 
     def list(self, fs: str, path: str = '/', recursive: bool = True):
         error_code, out, err = mgr.remote('snap_schedule', 'snap_schedule_list',
-                                          path, recursive, fs, 'plain')
-
+                                          path, recursive, fs, None, None, 'plain')
         if len(out) == 0:
             return []
 
         snapshot_schedule_list = out.split('\n')
-        output = []
+        output: list[Any] = []
 
         for snap in snapshot_schedule_list:
             current_path = snap.strip().split(' ')[0]
             error_code, status_out, err = mgr.remote('snap_schedule', 'snap_schedule_get',
-                                                     current_path, fs, 'plain')
-            output.append(json.loads(status_out))
+                                                     current_path, fs, None, None, 'json')
+            output = output + json.loads(status_out)
 
         output_json = json.dumps(output)
 
@@ -967,5 +966,34 @@ def list(self, fs: str, path: str = '/', recursive: bool = True):
             raise DashboardException(
                 f'Failed to get list of snapshot schedules for path {path}: {err}'
             )
-
         return json.loads(output_json)
+
+    def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_policy=None):
+        error_code, _, err = mgr.remote('snap_schedule',
+                                        'snap_schedule_add',
+                                        path,
+                                        snap_schedule,
+                                        start,
+                                        fs)
+
+        if retention_policy:
+            retention_policies = retention_policy.split('|')
+            for retention in retention_policies:
+                retention_count = retention.split('-')[0]
+                retention_spec_or_period = retention.split('-')[1]
+                error_code_retention, _, err_retention = mgr.remote('snap_schedule',
+                                                                    'snap_schedule_retention_add',
+                                                                    path,
+                                                                    retention_spec_or_period,
+                                                                    retention_count,
+                                                                    fs)
+                if error_code_retention != 0:
+                    raise DashboardException(
+                        f'Failed to add retention policy for path {path}: {err_retention}'
+                    )
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to create snapshot schedule for path {path}: {err}'
+            )
+
+        return f'Snapshot schedule for path {path} created successfully'
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
new file mode 100644
index 000000000000..9e9cde86b325
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
@@ -0,0 +1,179 @@
+<cd-modal [modalRef]="activeModal">
+  <ng-container i18n="form title"
+                class="modal-title">{{ action | titlecase }} {{ resource | upperFirst }}</ng-container>
+    <ng-container class="modal-content"
+                  *cdFormLoading="loading">
+    <form name="snapScheduleForm"
+          #formDir="ngForm"
+          [formGroup]="snapScheduleForm"
+          novalidate>
+      <div class="modal-body">
+        <!-- Directory -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="directory"
+                 i18n>Directory
+          </label>
+          <div class="cd-col-form-input">
+            <ng-template #loading>
+              <i [ngClass]="[icons.spinner, icons.spin, 'mt-2', 'me-2']"></i>
+              <span i18n>Loading directories</span>
+            </ng-template>
+            <select class="form-select"
+                    id="directory"
+                    name="directory"
+                    formControlName="directory"
+                    *ngIf="directories$ | async as directories; else loading">
+              <option [ngValue]="null"
+                      i18n>--Select a directory--</option>
+              <option *ngFor="let dir of directories"
+                      [value]="dir.path">{{ dir.path }}</option>
+            </select>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('directory', formDir, 'required')"
+                  i18n>This field is required.</span>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('directory', formDir, 'notUnique')"
+                  i18n>A snapshot schedule for this path already exists.</span>
+          </div>
+        </div>
+        <!--Start date -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="startDate"
+                 i18n>Start date
+          </label>
+          <div class="cd-col-form-input">
+            <div class="input-group">
+              <input class="form-control"
+                     placeholder="yyyy-mm-dd"
+                     name="startDate"
+                     id="startDate"
+                     formControlName="startDate"
+                     [minDate]="minDate"
+                     ngbDatepicker
+                     #d="ngbDatepicker"
+                     (click)="d.open()">
+              <button type="button"
+                      class="btn btn-light"
+                      (click)="d.toggle()"
+                      title="Open">
+                <i [ngClass]="icons.calendar"></i>
+              </button>
+            </div>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('startDate', formDir, 'required')"
+                  i18n>This field is required.</span>
+          </div>
+        </div>
+        <!-- Start time -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="startTime"
+                 i18n>Start time
+            <cd-helper>The time zone is assumed to be UTC.</cd-helper>
+          </label>
+          <div class="cd-col-form-input">
+            <ngb-timepicker [spinners]="false"
+                            [seconds]="false"
+                            [meridian]="true"
+                            formControlName="startTime"
+                            id="startTime"
+                            name="startTime"></ngb-timepicker>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('startTime', formDir, 'required')"
+                  i18n>This field is required.</span>
+          </div>
+        </div>
+        <!-- Repeat interval -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="repeatInterval"
+                 i18n>Schedule
+          </label>
+          <div class="cd-col-form-input">
+            <div class="input-group">
+              <input class="form-control"
+                     type="number"
+                     min="1"
+                     id="repeatInterval"
+                     name="repeatInterval"
+                     formControlName="repeatInterval">
+              <select [ngClass]="['form-select', 'me-5']"
+                      id="repeatFrequency"
+                      name="repeatFrequency"
+                      formControlName="repeatFrequency"
+                      *ngIf="repeatFrequencies">
+                <option *ngFor="let freq of repeatFrequencies"
+                        [value]="freq[1]"
+                        i18n>{{ freq[0] }}</option>
+              </select>
+            </div>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('repeatFrequency', formDir, 'notUnique')"
+                  i18n>This schedule already exists for the selected directory.</span>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('repeatInterval', formDir, 'required')"
+                  i18n>This field is required.</span>
+            <span class="invalid-feedback"
+                  *ngIf="snapScheduleForm.showError('repeatInterval', formDir, 'min')"
+                  i18n>Choose a value greater than 0.</span>
+          </div>
+        </div>
+        <!-- Retention policies -->
+        <ng-container formArrayName="retentionPolicies"
+                      *ngFor="let retentionPolicy of retentionPolicies.controls; index as i">
+          <ng-container [formGroupName]="i">
+            <div class="form-group row">
+              <label [ngClass]="{'cd-col-form-label': true, 'visible': i == 0, 'invisible': i > 0}"
+                    for="retentionInterval"
+                    i18n>Retention policy
+              </label>
+              <div class="cd-col-form-input">
+                <div class="input-group">
+                  <input class="form-control"
+                         type="number"
+                         min="1"
+                         id="retentionInterval"
+                         name="retentionInterval"
+                         formControlName="retentionInterval">
+                  <select class="form-select"
+                          id="retentionFrequency"
+                          name="retentionFrequency"
+                          formControlName="retentionFrequency"
+                          *ngIf="retentionFrequencies">
+                    <option *ngFor="let freq of retentionFrequencies"
+                            [value]="freq[1]"
+                            i18n>{{ freq[0] }}</option>
+                  </select>
+                  <button class="btn btn-light"
+                          type="button"
+                          (click)="removeRetentionPolicy(i)">
+                    <i [ngClass]="[icons.trash]"></i>
+                  </button>
+                </div>
+                <span class="invalid-feedback"
+                      *ngIf="snapScheduleForm.controls['retentionPolicies'].controls[i].invalid"
+                      i18n>This retention policy already exists for the selected directory.</span>
+              </div>
+            </div>
+          </ng-container>
+        </ng-container>
+        <div class="d-flex flex-row align-content-center justify-content-end">
+          <button class="btn btn-light"
+                  type="button"
+                  (click)="addRetentionPolicy()">
+            <i [ngClass]="[icons.add, 'me-2']"></i>
+            <span i18n>Add retention policy</span>
+          </button>
+        </div>
+      </div>
+
+      <div class="modal-footer">
+        <cd-form-button-panel (submitActionEvent)="submit()"
+                              [form]="snapScheduleForm"
+                              [submitText]="(action | titlecase) + ' ' + (resource | upperFirst)"></cd-form-button-panel>
+      </div>
+    </form>
+  </ng-container>
+</cd-modal>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.spec.ts
new file mode 100644
index 000000000000..6a9fbcb942a6
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.spec.ts
@@ -0,0 +1,79 @@
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-form.component';
+import {
+  NgbActiveModal,
+  NgbDatepickerModule,
+  NgbTimepickerModule
+} from '@ng-bootstrap/ng-bootstrap';
+import { ToastrModule } from 'ngx-toastr';
+import { SharedModule } from '~/app/shared/shared.module';
+import { RouterTestingModule } from '@angular/router/testing';
+import { ReactiveFormsModule } from '@angular/forms';
+import { FormHelper, configureTestBed } from '~/testing/unit-test-helper';
+import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
+
+describe('CephfsSnapshotscheduleFormComponent', () => {
+  let component: CephfsSnapshotscheduleFormComponent;
+  let fixture: ComponentFixture<CephfsSnapshotscheduleFormComponent>;
+  let formHelper: FormHelper;
+  let createSpy: jasmine.Spy;
+
+  configureTestBed({
+    declarations: [CephfsSnapshotscheduleFormComponent],
+    providers: [NgbActiveModal],
+    imports: [
+      SharedModule,
+      ToastrModule.forRoot(),
+      ReactiveFormsModule,
+      HttpClientTestingModule,
+      RouterTestingModule,
+      NgbDatepickerModule,
+      NgbTimepickerModule
+    ]
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(CephfsSnapshotscheduleFormComponent);
+    component = fixture.componentInstance;
+    component.fsName = 'test_fs';
+    component.ngOnInit();
+    formHelper = new FormHelper(component.snapScheduleForm);
+    createSpy = spyOn(TestBed.inject(CephfsSnapshotScheduleService), 'create').and.stub();
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+
+  it('should have a form open in modal', () => {
+    const nativeEl = fixture.debugElement.nativeElement;
+    expect(nativeEl.querySelector('cd-modal')).not.toBe(null);
+  });
+
+  it('should submit the form', () => {
+    const input = {
+      directory: '/test',
+      startDate: {
+        year: 2023,
+        month: 11,
+        day: 14
+      },
+      startTime: {
+        hour: 0,
+        minute: 6,
+        second: 22
+      },
+      repeatInterval: 4,
+      repeatFrequency: 'h'
+    };
+
+    formHelper.setMultipleValues(input);
+    component.snapScheduleForm.get('directory').setValue('/test');
+    component.submit();
+
+    expect(createSpy).toHaveBeenCalled();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
new file mode 100644
index 000000000000..5b6d900e7520
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -0,0 +1,257 @@
+import { ChangeDetectorRef, Component, OnInit } from '@angular/core';
+import { AbstractControl, FormArray, FormControl, FormGroup, Validators } from '@angular/forms';
+import { NgbActiveModal, NgbDateStruct, NgbTimeStruct } from '@ng-bootstrap/ng-bootstrap';
+import { uniq } from 'lodash';
+import { Observable, timer } from 'rxjs';
+import { map, switchMap } from 'rxjs/operators';
+import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
+import { CephfsService } from '~/app/shared/api/cephfs.service';
+import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { RepeatFrequency } from '~/app/shared/enum/repeat-frequency.enum';
+import { RetentionFrequency } from '~/app/shared/enum/retention-frequency.enum';
+import { CdForm } from '~/app/shared/forms/cd-form';
+import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
+import { CdTableColumn } from '~/app/shared/models/cd-table-column';
+import { CephfsDir } from '~/app/shared/models/cephfs-directory-models';
+import { FinishedTask } from '~/app/shared/models/finished-task';
+import { RetentionPolicy, SnapshotScheduleFormValue } from '~/app/shared/models/snapshot-schedule';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+
+const VALIDATON_TIMER = 300;
+
+@Component({
+  selector: 'cd-cephfs-snapshotschedule-form',
+  templateUrl: './cephfs-snapshotschedule-form.component.html',
+  styleUrls: ['./cephfs-snapshotschedule-form.component.scss']
+})
+export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnInit {
+  fsName!: string;
+  id!: number;
+  isEdit = false;
+  icons = Icons;
+  repeatFrequencies = Object.entries(RepeatFrequency);
+  retentionFrequencies = Object.entries(RetentionFrequency);
+
+  currentTime!: NgbTimeStruct;
+  minDate!: NgbDateStruct;
+
+  snapScheduleForm!: CdFormGroup;
+
+  action!: string;
+  resource!: string;
+
+  columns!: CdTableColumn[];
+  directories$!: Observable<CephfsDir[]>;
+
+  constructor(
+    public activeModal: NgbActiveModal,
+    private actionLabels: ActionLabelsI18n,
+    private cephfsService: CephfsService,
+    private snapScheduleService: CephfsSnapshotScheduleService,
+    private taskWrapper: TaskWrapperService,
+    private cd: ChangeDetectorRef
+  ) {
+    super();
+    this.resource = $localize`Snapshot schedule`;
+
+    const currentDatetime = new Date();
+    this.minDate = {
+      year: currentDatetime.getUTCFullYear(),
+      month: currentDatetime.getUTCMonth() + 1,
+      day: currentDatetime.getUTCDate()
+    };
+    this.currentTime = {
+      hour: currentDatetime.getUTCHours(),
+      minute: currentDatetime.getUTCMinutes(),
+      second: currentDatetime.getUTCSeconds()
+    };
+  }
+
+  ngOnInit(): void {
+    this.action = this.actionLabels.CREATE;
+    this.directories$ = this.cephfsService.lsDir(this.id, '/', 3);
+    this.createForm();
+    this.loadingReady();
+  }
+
+  get retentionPolicies() {
+    return this.snapScheduleForm.get('retentionPolicies') as FormArray;
+  }
+
+  createForm() {
+    this.snapScheduleForm = new CdFormGroup(
+      {
+        directory: new FormControl(undefined, {
+          validators: [Validators.required]
+        }),
+        startDate: new FormControl(this.minDate, {
+          validators: [Validators.required]
+        }),
+        startTime: new FormControl(this.currentTime, {
+          validators: [Validators.required]
+        }),
+        repeatInterval: new FormControl(1, {
+          validators: [Validators.required, Validators.min(1)]
+        }),
+        repeatFrequency: new FormControl(RepeatFrequency.Daily, {
+          validators: [Validators.required]
+        }),
+        retentionPolicies: new FormArray([])
+      },
+      {
+        asyncValidators: [this.validateSchedule(), this.validateRetention()]
+      }
+    );
+  }
+
+  addRetentionPolicy() {
+    this.retentionPolicies.push(
+      new FormGroup({
+        retentionInterval: new FormControl(1),
+        retentionFrequency: new FormControl(RetentionFrequency.Daily)
+      })
+    );
+    this.cd.detectChanges();
+  }
+
+  removeRetentionPolicy(idx: number) {
+    this.retentionPolicies.removeAt(idx);
+    this.cd.detectChanges();
+  }
+
+  parseDatetime(date: NgbDateStruct, time?: NgbTimeStruct): string {
+    return `${date.year}-${date.month}-${date.day}T${time.hour || '00'}:${time.minute || '00'}:${
+      time.second || '00'
+    }`;
+  }
+  parseSchedule(interval: number, frequency: string): string {
+    return `${interval}${frequency}`;
+  }
+
+  parseRetentionPolicies(retentionPolicies: RetentionPolicy[]) {
+    return retentionPolicies
+      ?.filter((r) => r?.retentionInterval !== null && r?.retentionFrequency !== null)
+      ?.map?.((r) => `${r.retentionInterval}-${r.retentionFrequency}`)
+      .join('|');
+  }
+
+  submit() {
+    if (this.snapScheduleForm.invalid) {
+      this.snapScheduleForm.setErrors({ cdSubmitButton: true });
+      return;
+    }
+
+    const values = this.snapScheduleForm.value as SnapshotScheduleFormValue;
+
+    const snapScheduleObj = {
+      fs: this.fsName,
+      path: values.directory,
+      snap_schedule: this.parseSchedule(values.repeatInterval, values.repeatFrequency),
+      start: this.parseDatetime(values.startDate, values.startTime)
+    };
+
+    const retentionPoliciesValues = this.parseRetentionPolicies(values?.retentionPolicies);
+    if (retentionPoliciesValues) {
+      snapScheduleObj['retention_policy'] = retentionPoliciesValues;
+    }
+
+    this.taskWrapper
+      .wrapTaskAroundCall({
+        task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.CREATE, {
+          path: snapScheduleObj.path
+        }),
+        call: this.snapScheduleService.create(snapScheduleObj)
+      })
+      .subscribe({
+        error: () => {
+          this.snapScheduleForm.setErrors({ cdSubmitButton: true });
+        },
+        complete: () => {
+          this.activeModal.close();
+        }
+      });
+  }
+
+  validateSchedule() {
+    return (frm: AbstractControl) => {
+      const directory = frm.get('directory');
+      const repeatFrequency = frm.get('repeatFrequency');
+      const repeatInterval = frm.get('repeatInterval');
+      return timer(VALIDATON_TIMER).pipe(
+        switchMap(() =>
+          this.snapScheduleService
+            .checkScheduleExists(
+              directory?.value,
+              this.fsName,
+              repeatInterval?.value,
+              repeatFrequency?.value
+            )
+            .pipe(
+              map((exists: boolean) => {
+                if (exists) {
+                  repeatFrequency?.setErrors({ notUnique: true }, { emitEvent: true });
+                } else {
+                  repeatFrequency?.setErrors(null);
+                }
+                return null;
+              })
+            )
+        )
+      );
+    };
+  }
+
+  getFormArrayItem(frm: FormGroup, frmArrayName: string, ctrl: string, idx: number) {
+    return (frm.get(frmArrayName) as FormArray)?.controls?.[idx]?.get?.(ctrl);
+  }
+
+  validateRetention() {
+    return (frm: FormGroup) => {
+      return timer(VALIDATON_TIMER).pipe(
+        switchMap(() => {
+          const retentionList = (frm.get('retentionPolicies') as FormArray).controls?.map(
+            (ctrl) => {
+              return ctrl.get('retentionFrequency').value;
+            }
+          );
+          if (uniq(retentionList)?.length !== retentionList?.length) {
+            this.getFormArrayItem(
+              frm,
+              'retentionPolicies',
+              'retentionFrequency',
+              retentionList.length - 1
+            )?.setErrors?.({
+              notUnique: true
+            });
+            return null;
+          }
+          return this.snapScheduleService
+            .checkRetentionPolicyExists(frm.get('directory').value, this.fsName, retentionList)
+            .pipe(
+              map(({ exists, errorIndex }) => {
+                if (exists) {
+                  this.getFormArrayItem(
+                    frm,
+                    'retentionPolicies',
+                    'retentionFrequency',
+                    errorIndex
+                  )?.setErrors?.({ notUnique: true });
+                } else {
+                  (frm.get('retentionPolicies') as FormArray).controls?.forEach?.((_, i) => {
+                    this.getFormArrayItem(
+                      frm,
+                      'retentionPolicies',
+                      'retentionFrequency',
+                      i
+                    )?.setErrors?.(null);
+                  });
+                }
+                return null;
+              })
+            );
+        })
+      );
+    };
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index 14752b7e2a8f..6b406cfc1712 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -26,6 +26,8 @@ import { MgrModuleService } from '~/app/shared/api/mgr-module.service';
 import { NotificationService } from '~/app/shared/services/notification.service';
 import { BlockUI, NgBlockUI } from 'ng-block-ui';
 import { NotificationType } from '~/app/shared/enum/notification-type.enum';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { CephfsSnapshotscheduleFormComponent } from '../cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component';
 
 @Component({
   selector: 'cd-cephfs-snapshotschedule-list',
@@ -36,6 +38,7 @@ export class CephfsSnapshotscheduleListComponent
   extends CdForm
   implements OnInit, OnChanges, OnDestroy {
   @Input() fsName!: string;
+  @Input() id!: number;
 
   @ViewChild('pathTpl', { static: true })
   pathTpl: any;
@@ -65,7 +68,8 @@ export class CephfsSnapshotscheduleListComponent
     private authStorageService: AuthStorageService,
     private modalService: ModalService,
     private mgrModuleService: MgrModuleService,
-    private notificationService: NotificationService
+    private notificationService: NotificationService,
+    private actionLables: ActionLabelsI18n
   ) {
     super();
     this.permissions = this.authStorageService.getPermissions();
@@ -112,7 +116,14 @@ export class CephfsSnapshotscheduleListComponent
       { prop: 'created', name: $localize`Created`, cellTransformation: CellTemplate.timeAgo }
     ];
 
-    this.tableActions = [];
+    this.tableActions = [
+      {
+        name: this.actionLables.CREATE,
+        permission: 'create',
+        icon: Icons.add,
+        click: () => this.openModal(true)
+      }
+    ];
   }
 
   ngOnDestroy(): void {
@@ -129,9 +140,11 @@ export class CephfsSnapshotscheduleListComponent
 
   openModal(edit = false) {
     this.modalService.show(
-      {},
+      CephfsSnapshotscheduleFormComponent,
       {
-        fsName: 'fs1',
+        fsName: this.fsName,
+        id: this.id,
+        path: this.selection?.first()?.path,
         isEdit: edit
       },
       { size: 'lg' }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
index d21d47034f80..a840692ed767 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
@@ -59,6 +59,7 @@
       <ng-template ngbNavContent>
         <cd-cephfs-snapshotschedule-list
           [fsName]="selection.mdsmap.fs_name"
+          [id]="id"
         ></cd-cephfs-snapshotschedule-list>
       </ng-template>
     </ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index 36ad6ddcb874..b13273dc4a21 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -3,7 +3,13 @@ import { NgModule } from '@angular/core';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 
 import { TreeModule } from '@circlon/angular-tree-component';
-import { NgbNavModule, NgbTooltipModule, NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap';
+import {
+  NgbDatepickerModule,
+  NgbNavModule,
+  NgbTimepickerModule,
+  NgbTooltipModule,
+  NgbTypeaheadModule
+} from '@ng-bootstrap/ng-bootstrap';
 import { NgChartsModule } from 'ng2-charts';
 
 import { AppRoutingModule } from '~/app/app-routing.module';
@@ -23,6 +29,7 @@ import { CephfsSubvolumeSnapshotsListComponent } from './cephfs-subvolume-snapsh
 import { CephfsSnapshotscheduleListComponent } from './cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component';
 import { DataTableModule } from '../../shared/datatable/datatable.module';
 import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component';
+import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component';
 
 @NgModule({
   imports: [
@@ -36,7 +43,9 @@ import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapsh
     ReactiveFormsModule,
     NgbTypeaheadModule,
     NgbTooltipModule,
-    DataTableModule
+    DataTableModule,
+    NgbDatepickerModule,
+    NgbTimepickerModule
   ],
   declarations: [
     CephfsDetailComponent,
@@ -53,6 +62,7 @@ import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapsh
     CephfsSubvolumegroupFormComponent,
     CephfsSubvolumeSnapshotsListComponent,
     CephfsSnapshotscheduleListComponent,
+    CephfsSnapshotscheduleFormComponent,
     CephfsSubvolumeSnapshotsFormComponent
   ]
 })
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
index ec9f58c0feec..0666bb179e83 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -1,8 +1,11 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { Observable } from 'rxjs/internal/Observable';
+import { catchError, map } from 'rxjs/operators';
+import { intersection, isEqual, uniqWith } from 'lodash';
 import { SnapshotSchedule } from '../models/snapshot-schedule';
-import { map } from 'rxjs/operators';
+import { of } from 'rxjs';
+import { RepeatFrequency } from '../enum/repeat-frequency.enum';
 
 @Injectable({
   providedIn: 'root'
@@ -12,28 +15,86 @@ export class CephfsSnapshotScheduleService {
 
   constructor(private http: HttpClient) {}
 
-  getSnapshotScheduleList(
+  create(data: Record<string, any>): Observable<any> {
+    return this.http.post(`${this.baseURL}/snapshot/schedule`, data, { observe: 'response' });
+  }
+
+  checkScheduleExists(
     path: string,
     fs: string,
-    recursive = true
-  ): Observable<SnapshotSchedule[]> {
+    interval: number,
+    frequency: RepeatFrequency
+  ): Observable<boolean> {
+    return this.getSnapshotScheduleList(path, fs, false).pipe(
+      map((response) => {
+        const index = response.findIndex(
+          (x) => x.path === path && x.schedule === `${interval}${frequency}`
+        );
+        return index > -1;
+      }),
+      catchError(() => {
+        return of(false);
+      })
+    );
+  }
+
+  checkRetentionPolicyExists(
+    path: string,
+    fs: string,
+    retentionFrequencies: string[]
+  ): Observable<{ exists: boolean; errorIndex: number }> {
+    return this.getList(path, fs, false).pipe(
+      map((response) => {
+        let errorIndex = -1;
+        let exists = false;
+        const index = response.findIndex((x) => x.path === path);
+        const result = retentionFrequencies?.length
+          ? intersection(Object.keys(response?.[index]?.retention), retentionFrequencies)
+          : [];
+        exists = !!result?.length;
+        result?.forEach((r) => (errorIndex = retentionFrequencies.indexOf(r)));
+
+        return { exists, errorIndex };
+      }),
+      catchError(() => {
+        return of({ exists: false, errorIndex: -1 });
+      })
+    );
+  }
+
+  private getList(path: string, fs: string, recursive = true): Observable<SnapshotSchedule[]> {
     return this.http
       .get<SnapshotSchedule[]>(
-        `${this.baseURL}/snaphost/schedule?path=${path}&fs=${fs}&recursive=${recursive}`
+        `${this.baseURL}/snapshot/schedule?path=${path}&fs=${fs}&recursive=${recursive}`
       )
       .pipe(
-        map((snapList: SnapshotSchedule[]) =>
+        catchError(() => {
+          return of([]);
+        })
+      );
+  }
+
+  getSnapshotScheduleList(
+    path: string,
+    fs: string,
+    recursive = true
+  ): Observable<SnapshotSchedule[]> {
+    return this.getList(path, fs, recursive).pipe(
+      map((snapList: SnapshotSchedule[]) =>
+        uniqWith(
           snapList.map((snapItem: SnapshotSchedule) => ({
             ...snapItem,
             status: snapItem.active ? 'Active' : 'Inactive',
             subvol: snapItem?.subvol || ' - ',
-            retention: Object.values(snapItem.retention)?.length
+            retention: Object.values(snapItem?.retention || [])?.length
               ? Object.entries(snapItem.retention)
                   ?.map?.(([frequency, interval]) => `${interval}${frequency.toLocaleUpperCase()}`)
                   .join(' ')
               : '-'
-          }))
+          })),
+          isEqual
         )
-      );
+      )
+    );
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
index 6142d7359de2..a265ae7a2658 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
@@ -21,8 +21,8 @@ export class CephfsService {
     return this.http.get(`${this.baseURL}`);
   }
 
-  lsDir(id: number, path?: string): Observable<CephfsDir[]> {
-    let apiPath = `${this.baseUiURL}/${id}/ls_dir?depth=2`;
+  lsDir(id: number, path?: string, depth: number = 2): Observable<CephfsDir[]> {
+    let apiPath = `${this.baseUiURL}/${id}/ls_dir?depth=${depth}`;
     if (path) {
       apiPath += `&path=${encodeURIComponent(path)}`;
     }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
index d3515fb87e4b..be454076b862 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/icons.enum.ts
@@ -82,6 +82,7 @@ export enum Icons {
   navicon = 'fa fa-navicon', // Navigation
   areaChart = 'fa fa-area-chart', // Area Chart, dashboard
   eye = 'fa fa-eye', // Observability
+  calendar = 'fa fa-calendar',
   externalUrl = 'fa fa-external-link', // links to external page
 
   /* Icons for special effect */
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
new file mode 100644
index 000000000000..db3563ed2b47
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
@@ -0,0 +1,5 @@
+export enum RepeatFrequency {
+  Hourly = 'h',
+  Daily = 'd',
+  Weekly = 'w'
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
new file mode 100644
index 000000000000..44714dac9463
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
@@ -0,0 +1,8 @@
+export enum RetentionFrequency {
+  Hourly = 'h',
+  Daily = 'd',
+  Weekly = 'w',
+  Monthly = 'm',
+  Yearly = 'y',
+  'lastest snapshots' = 'n'
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts
index b1cea7466f67..af3b0f7c5e6b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/snapshot-schedule.ts
@@ -1,3 +1,5 @@
+import { NgbDateStruct, NgbTimeStruct } from '@ng-bootstrap/ng-bootstrap';
+
 export interface SnapshotSchedule {
   fs?: string;
   subvol?: string;
@@ -15,3 +17,17 @@ export interface SnapshotSchedule {
   active: boolean;
   status: 'Active' | 'Inactive';
 }
+
+export interface SnapshotScheduleFormValue {
+  directory: string;
+  startDate: NgbDateStruct;
+  startTime: NgbTimeStruct;
+  repeatInterval: number;
+  repeatFrequency: string;
+  retentionPolicies: RetentionPolicy[];
+}
+
+export interface RetentionPolicy {
+  retentionInterval: number;
+  retentionFrequency: string;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index dba742fbf783..f631842919c0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -387,6 +387,9 @@ export class TaskMessageService {
     'cephfs/subvolume/snapshot/delete': this.newTaskMessage(
       this.commonOperations.delete,
       (metadata) => this.snapshot(metadata)
+    ),
+    'cephfs/snapshot/schedule/create': this.newTaskMessage(this.commonOperations.add, (metadata) =>
+      this.snapshotSchedule(metadata)
     )
   };
 
@@ -459,6 +462,9 @@ export class TaskMessageService {
     return $localize`snapshot '${metadata.snapshotName}'`;
   }
 
+  snapshotSchedule(metadata: any) {
+    return $localize`snapshot schedule for path '${metadata?.path}'`;
+  }
   crudMessageId(id: string) {
     return $localize`${id}`;
   }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 54f21e9484ae..ad681795dc07 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1758,7 +1758,7 @@ paths:
       summary: Rename CephFS Volume
       tags:
       - Cephfs
-  /api/cephfs/snaphost/schedule:
+  /api/cephfs/snapshot/schedule:
     get:
       parameters:
       - in: query
@@ -1795,6 +1795,53 @@ paths:
       - jwt: []
       tags:
       - CephFSSnapshotSchedule
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                fs:
+                  type: string
+                path:
+                  type: string
+                retention_policy:
+                  type: string
+                snap_schedule:
+                  type: string
+                start:
+                  type: string
+              required:
+              - fs
+              - path
+              - snap_schedule
+              - start
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
   /api/cephfs/subvolume:
     post:
       parameters: []

From 2f0542d66901295cf875893de0ac15304578d917 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 30 Jan 2024 19:51:53 +1000
Subject: [PATCH 1709/2492] doc/architecture: improve some paragraphs

Improve paragraphs under the heading "The Ceph Storage Cluster". Remove
a sentence that was pleonastic in its context in the paragraph.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index 7380d74a72a5..fc9e06da8cd8 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -38,9 +38,8 @@ A Ceph Storage Cluster consists of multiple types of daemons:
 .. _arch_monitor:
 
 Ceph Monitors maintain the master copy of the cluster map, which they provide
-to Ceph clients. Provisioning multiple monitors within the Ceph cluster ensures
-availability in the event that one of the monitor daemons or its host fails.
-The Ceph monitor provides copies of the cluster map to storage cluster clients.
+to Ceph clients. The existence of multiple monitors in the Ceph cluster ensures
+availability if one of the monitor daemons or its host fails.
 
 A Ceph OSD Daemon checks its own state and the state of other OSDs and reports
 back to monitors.
@@ -52,10 +51,11 @@ A Ceph Metadata Server (MDS) manages file metadata when CephFS is used to
 provide file services.
 
 Storage cluster clients and :term:`Ceph OSD Daemon`\s use the CRUSH algorithm
-to compute information about data location. This means that clients and OSDs
-are not bottlenecked by a central lookup table. Ceph's high-level features
-include a native interface to the Ceph Storage Cluster via ``librados``, and a
-number of service interfaces built on top of ``librados``.
+to compute information about the location of data. Use of the CRUSH algoritm
+means that clients and OSDs are not bottlenecked by a central lookup table.
+Ceph's high-level features include a native interface to the Ceph Storage
+Cluster via ``librados``, and a number of service interfaces built on top of
+``librados``.
 
 Storing Data
 ------------

From 2a2d3d4c6b38a917f465680e9321e810426555ea Mon Sep 17 00:00:00 2001
From: jagombar <48735467+jagombar@users.noreply.github.com>
Date: Tue, 30 Jan 2024 18:25:53 +0000
Subject: [PATCH 1710/2492] librbd: return ENOENT from Snapshot::get_timestamp
 for nonexistent snap_id

Fixes: https://tracker.ceph.com/issues/47287

Signed-off-by: John Agombar <agombar@uk.ibm.com>
---
 src/librbd/api/Snapshot.cc     | 4 +++-
 src/test/librbd/test_librbd.cc | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/librbd/api/Snapshot.cc b/src/librbd/api/Snapshot.cc
index 03cefbd1c0a0..306ddb593da8 100644
--- a/src/librbd/api/Snapshot.cc
+++ b/src/librbd/api/Snapshot.cc
@@ -378,7 +378,9 @@ int Snapshot<I>::remove(I *ictx, const char *snap_name, uint32_t flags,
 template <typename I>
 int Snapshot<I>::get_timestamp(I *ictx, uint64_t snap_id, struct timespec *timestamp) {
   auto snap_it = ictx->snap_info.find(snap_id);
-  ceph_assert(snap_it != ictx->snap_info.end());
+  if (snap_it == ictx->snap_info.end()) {
+    return -ENOENT;
+  }
   utime_t time = snap_it->second.timestamp;
   time.to_timespec(timestamp);
   return 0;
diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index 2b9557795348..94420a64286a 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -1902,6 +1902,8 @@ TEST_F(TestLibRBD, TestGetSnapShotTimeStamp)
   ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
   ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
 
+  ASSERT_EQ(-ENOENT, rbd_snap_get_timestamp(image, 0, NULL));
+
   ASSERT_EQ(0, rbd_snap_create(image, "snap1"));
   num_snaps = rbd_snap_list(image, snaps, &max_size);
   ASSERT_EQ(1, num_snaps);

From 36c83bf5424308214228b1c86d8e1d215b8519c5 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 30 Jan 2024 14:17:35 +0000
Subject: [PATCH 1711/2492] ceph-volume: fix zap_partitions() in
 devices.lvm.zap

The current logic is unable to find the holders for the
partition being zapped.

This commit fixes this issue.

Fixes: https://tracker.ceph.com/issues/64248

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-volume/ceph_volume/devices/lvm/zap.py | 7 +++++--
 src/ceph-volume/ceph_volume/util/disk.py       | 6 ++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/zap.py b/src/ceph-volume/ceph_volume/devices/lvm/zap.py
index d4d78ad01810..7d808a5f4fc3 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/zap.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/zap.py
@@ -201,8 +201,11 @@ def zap_partition(self, device):
         """
         if device.is_encrypted:
             # find the holder
-            holders = [
-                '/dev/%s' % holder for holder in device.sys_api.get('holders', [])
+            pname = device.sys_api.get('parent')
+            devname = device.sys_api.get('devname')
+            parent_device = Device(f'/dev/{pname}')
+            holders: List[str] = [
+                f'/dev/{holder}' for holder in parent_device.sys_api['partitions'][devname]['holders']
             ]
             for mapper_uuid in os.listdir('/dev/mapper'):
                 mapper_path = os.path.join('/dev/mapper', mapper_uuid)
diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
index 3965d576d6e0..a0bc39b9d179 100644
--- a/src/ceph-volume/ceph_volume/util/disk.py
+++ b/src/ceph-volume/ceph_volume/util/disk.py
@@ -6,7 +6,7 @@
 from ceph_volume import process
 from ceph_volume.api import lvm
 from ceph_volume.util.system import get_file_contents
-from typing import Dict, List
+from typing import Dict, List, Any
 
 
 logger = logging.getLogger(__name__)
@@ -829,6 +829,7 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         block_types.append('loop')
 
     for block in block_devs:
+        metadata: Dict[str, Any] = {}
         if block[2] == 'lvm':
             block[1] = lvm.get_lv_path_from_mapper(block[1])
         devname = os.path.basename(block[0])
@@ -838,7 +839,6 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         sysdir = os.path.join(_sys_block_path, devname)
         if block[2] == 'part':
             sysdir = os.path.join(_sys_block_path, block[3], devname)
-        metadata = {}
 
         # If the device is ceph rbd it gets excluded
         if is_ceph_rbd(diskname):
@@ -904,7 +904,9 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         metadata['size'] = float(size) * 512
         metadata['human_readable_size'] = human_readable_size(metadata['size'])
         metadata['path'] = diskname
+        metadata['devname'] = devname
         metadata['type'] = block[2]
+        metadata['parent'] = block[3]
 
         # some facts from udevadm
         p = udevadm_property(sysdir)

From dd1d30b7b209eaff376620ebd6b9a65df5ee7361 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 24 Dec 2023 14:16:12 +0000
Subject: [PATCH 1712/2492] mon/OSDMonitor: fix osd_epochs insertion on
 prepare_beacon()

Previously, an OSD marked out could have been added back to the map upon new beacon sent by the OSD.
If the OSD goes down (after being added back to map) - it won't be marked as OUT again.
Therefore, `osd_epochs` would remain with a "stale" OSD, as a result OSDMonitor::get_min_last_epoch_clean() will always return the "stale" OSD epoch.

When osd is marked out, it will be erased from `osd_epochs` map. Correspondingly, it should be added back to the map **only** once it's marked in.

Fixes: https://tracker.ceph.com/issues/63883

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f8e379326f25..d8ea74c8ae08 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -4399,8 +4399,9 @@ bool OSDMonitor::prepare_beacon(MonOpRequestRef op)
 
   last_osd_report[from].first = ceph_clock_now();
   last_osd_report[from].second = beacon->osd_beacon_report_interval;
-  osd_epochs[from] = beacon->version;
-
+  if (osdmap.is_in(from)) {
+    osd_epochs[from] = beacon->version;
+  }
   for (const auto& pg : beacon->pgs) {
     if (auto* pool = osdmap.get_pg_pool(pg.pool()); pool != nullptr) {
       unsigned pg_num = pool->get_pg_num();

From 88466b3e7daad51a93ef1ccdbced1f33337c4eec Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 31 Jan 2024 15:45:26 +0300
Subject: [PATCH 1713/2492] ceph-volume: Fix migration from WAL to data with no
 DB

Fixes: https://tracker.ceph.com/issues/64260

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 .../ceph_volume/devices/lvm/migrate.py        |  6 +-
 .../tests/devices/lvm/test_migrate.py         | 87 +++++++++++++++++++
 2 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py
index 64589a2d6284..474b479dee96 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py
@@ -167,7 +167,11 @@ def update_tags_when_lv_create(self, create_type):
             aux_dev.lv_api.set_tags(tags)
 
     def remove_lvs(self, source_devices, target_type):
-        remaining_devices = [self.data_device, self.db_device, self.wal_device]
+        remaining_devices = [self.data_device]
+        if self.db_device:
+            remaining_devices.append(self.db_device)
+        if self.wal_device:
+            remaining_devices.append(self.wal_device)
 
         outdated_tags = []
         for device, type in source_devices:
diff --git a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py
index 7e516f3d23bd..91e6155f38cd 100644
--- a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py
+++ b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py
@@ -2465,6 +2465,93 @@ def test_migrate_data_wal_to_db(self,
             '--devs-source', '/var/lib/ceph/osd/ceph-2/block',
             '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal']
 
+    @patch('os.getuid')
+    def test_migrate_wal_to_db(self,
+                                m_getuid,
+                                monkeypatch,
+                                capsys):
+        m_getuid.return_value = 0
+
+        source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \
+        'ceph.cluster_name=ceph,' \
+        'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev'
+        source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \
+        'ceph.cluster_name=ceph,' \
+        'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev'
+
+        data_vol = api.Volume(lv_name='volume1',
+                              lv_uuid='datauuid',
+                              vg_name='vg',
+                              lv_path='/dev/VolGroup/lv1',
+                              lv_tags=source_tags)
+
+        wal_vol = api.Volume(lv_name='volume3',
+                             lv_uuid='waluuid',
+                             vg_name='vg',
+                             lv_path='/dev/VolGroup/lv3',
+                             lv_tags=source_wal_tags)
+
+        self.mock_single_volumes = {
+            '/dev/VolGroup/lv1': data_vol,
+            '/dev/VolGroup/lv3': wal_vol,
+        }
+        monkeypatch.setattr(migrate.api, 'get_single_lv',
+            self.mock_get_single_lv)
+
+        self.mock_volume = data_vol
+        monkeypatch.setattr(api, 'get_lv_by_fullname',
+            self.mock_get_lv_by_fullname)
+
+        self.mock_process_input = []
+        monkeypatch.setattr(process, 'call', self.mock_process)
+
+        devices = []
+        devices.append([Device('/dev/VolGroup/lv1'), 'block'])
+        devices.append([Device('/dev/VolGroup/lv3'), 'wal'])
+
+        monkeypatch.setattr(migrate, 'find_associated_devices',
+            lambda osd_id, osd_fsid: devices)
+
+        monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active",
+            lambda id: False)
+
+        monkeypatch.setattr(migrate, 'get_cluster_name',
+            lambda osd_id, osd_fsid: 'ceph')
+        monkeypatch.setattr(system, 'chown', lambda path: 0)
+        m = migrate.Migrate(argv=[
+            '--osd-id', '2',
+            '--osd-fsid', '1234',
+            '--from', 'wal',
+            '--target', 'vgname/data'])
+
+        m.main()
+
+        n = len(self.mock_process_input)
+        assert n >= 1
+        for s in self.mock_process_input:
+            print(s)
+
+        assert self. mock_process_input[n-3] == [
+            'lvchange',
+            '--deltag', 'ceph.osd_id=2',
+            '--deltag', 'ceph.type=wal',
+            '--deltag', 'ceph.osd_fsid=1234',
+            '--deltag', 'ceph.cluster_name=ceph',
+            '--deltag', 'ceph.wal_uuid=waluuid',
+            '--deltag', 'ceph.wal_device=wal_dev',
+            '/dev/VolGroup/lv3']
+        assert self. mock_process_input[n-2] == [
+            'lvchange',
+            '--deltag', 'ceph.wal_uuid=waluuid',
+            '--deltag', 'ceph.wal_device=wal_dev',
+            '/dev/VolGroup/lv1']
+        assert self. mock_process_input[n-1] == [
+            'ceph-bluestore-tool',
+            '--path', '/var/lib/ceph/osd/ceph-2',
+            '--dev-target', '/var/lib/ceph/osd/ceph-2/block',
+            '--command', 'bluefs-bdev-migrate',
+            '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal']
+
     @patch('os.getuid')
     def test_migrate_data_wal_to_db_encrypted(self,
                                               m_getuid,

From 1f89b685c0c5c4891b5587ff4e480a615e6f5a48 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 31 Jan 2024 12:46:38 +0000
Subject: [PATCH 1714/2492] mon/OSDMonitor: rename get_lower_bound to
 get_lower_bound_by_pool

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 4 ++--
 src/mon/OSDMonitor.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index d8ea74c8ae08..fe6024a82ea1 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -395,7 +395,7 @@ void LastEpochClean::report(unsigned pg_num, const pg_t& pg,
   return lec.report(pg_num, pg.ps(), last_epoch_clean);
 }
 
-epoch_t LastEpochClean::get_lower_bound(const OSDMap& latest) const
+epoch_t LastEpochClean::get_lower_bound_by_pool(const OSDMap& latest) const
 {
   auto floor = latest.get_epoch();
   for (auto& pool : latest.get_pools()) {
@@ -2287,7 +2287,7 @@ version_t OSDMonitor::get_trim_to() const
 
 epoch_t OSDMonitor::get_min_last_epoch_clean() const
 {
-  auto floor = last_epoch_clean.get_lower_bound(osdmap);
+  auto floor = last_epoch_clean.get_lower_bound_by_pool(osdmap);
   // also scan osd epochs
   // don't trim past the oldest reported osd epoch
   for (auto [osd, epoch] : osd_epochs) {
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 08aa4f61783b..e3b91417650f 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -114,7 +114,7 @@ class LastEpochClean {
 public:
   void report(unsigned pg_num, const pg_t& pg, epoch_t last_epoch_clean);
   void remove_pool(uint64_t pool);
-  epoch_t get_lower_bound(const OSDMap& latest) const;
+  epoch_t get_lower_bound_by_pool(const OSDMap& latest) const;
 
   void dump(Formatter *f) const;
 };

From ba750f45ab06546df261ab2bb7be993ecc0b5d22 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 31 Jan 2024 12:48:04 +0000
Subject: [PATCH 1715/2492] mon/OSDMonitor: osd_epochs to store only UP osds

OUT OSDs can be the primary for a PG via pg_temp. Therfore, we should
consider OUT as valid beacons.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index fe6024a82ea1..7ec7b6169052 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -906,12 +906,7 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap)
       if (state & CEPH_OSD_UP) {
 	// could be marked up *or* down, but we're too lazy to check which
 	last_osd_report.erase(osd);
-      }
-    }
-    for (auto [osd, weight] : inc.new_weight) {
-      if (weight == CEPH_OSD_OUT) {
-        // manually marked out, so drop it
-        osd_epochs.erase(osd);
+	osd_epochs.erase(osd);
       }
     }
   }
@@ -2292,6 +2287,7 @@ epoch_t OSDMonitor::get_min_last_epoch_clean() const
   // don't trim past the oldest reported osd epoch
   for (auto [osd, epoch] : osd_epochs) {
     if (epoch < floor) {
+      ceph_assert(osdmap.is_up(osd));
       floor = epoch;
     }
   }
@@ -4399,9 +4395,8 @@ bool OSDMonitor::prepare_beacon(MonOpRequestRef op)
 
   last_osd_report[from].first = ceph_clock_now();
   last_osd_report[from].second = beacon->osd_beacon_report_interval;
-  if (osdmap.is_in(from)) {
-    osd_epochs[from] = beacon->version;
-  }
+  ceph_assert(osdmap.is_up(from));
+  osd_epochs[from] = beacon->version;
   for (const auto& pg : beacon->pgs) {
     if (auto* pool = osdmap.get_pg_pool(pg.pool()); pool != nullptr) {
       unsigned pg_num = pool->get_pg_num();

From b7c0a6a5b0e7d6ba063e1dd1715f938ecf7ec55d Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 31 Jan 2024 15:23:44 +0100
Subject: [PATCH 1716/2492] doc: add node-proxy documentation

This commit adds some documentation about the
'hardware inventory / monitoring' feature (node-proxy agent).

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 doc/hardware-monitoring/index.rst | 183 ++++++++++++++++++++++++++++++
 doc/index.rst                     |   1 +
 doc/monitoring/index.rst          |   3 +
 3 files changed, 187 insertions(+)
 create mode 100644 doc/hardware-monitoring/index.rst

diff --git a/doc/hardware-monitoring/index.rst b/doc/hardware-monitoring/index.rst
new file mode 100644
index 000000000000..dcafa82303f5
--- /dev/null
+++ b/doc/hardware-monitoring/index.rst
@@ -0,0 +1,183 @@
+.. _hardware-monitoring:
+
+Hardware monitoring
+===================
+
+`node-proxy` is the internal name to designate the running agent which inventories a machine's hardware, provides the different statuses and enable the operator to perform some actions.
+It gathers details from the RedFish API, processes and pushes data to agent endpoint in the Ceph manager daemon.
+
+.. graphviz::
+
+     digraph G {
+         node [shape=record];
+         mgr [label="{<mgr> ceph manager}"];
+         dashboard [label="<dashboard> ceph dashboard"];
+         agent [label="<agent> agent"];
+         redfish [label="<redfish> redfish"];
+     
+         agent -> redfish [label=" 1." color=green];
+         agent -> mgr [label=" 2." color=orange];
+         dashboard:dashboard -> mgr [label=" 3."color=lightgreen];
+         node [shape=plaintext];
+         legend [label=<<table border="0" cellborder="1" cellspacing="0">
+             <tr><td bgcolor="lightgrey">Legend</td></tr>
+             <tr><td align="center">1. Collects data from redfish API</td></tr>
+             <tr><td align="left">2. Pushes data to ceph mgr</td></tr>
+             <tr><td align="left">3. Query ceph mgr</td></tr>
+         </table>>];
+     }
+
+
+Limitations
+-----------
+
+For the time being, the `node-proxy` agent relies on the RedFish API.
+It implies both `node-proxy` agent and `ceph-mgr` daemon need to be able to access the Out-Of-Band network to work.
+
+
+Deploying the agent
+-------------------
+
+| The first step is to provide the out of band management tool credentials.
+| This can be done when adding the host with a service spec file:
+
+.. code-block:: bash
+
+  # cat host.yml
+  ---
+  service_type: host
+  hostname: node-10
+  addr: 10.10.10.10
+  oob:
+    addr: 20.20.20.10
+    username: admin
+    password: p@ssword
+
+Apply the spec:
+
+.. code-block:: bash
+
+  # ceph orch apply -i host.yml
+  Added host 'node-10' with addr '10.10.10.10'
+
+Deploy the agent:
+
+.. code-block:: bash
+
+  # ceph config set mgr mgr/cephadm/hw_monitoring true
+
+CLI
+---
+
+| **orch** **hardware** **status** [hostname] [--category CATEGORY] [--format plain | json]
+
+supported categories are:
+
+* summary (default)
+* memory
+* storage
+* processors
+* network
+* power
+* fans
+* firmwares
+* criticals
+
+Examples
+********
+
+
+hardware health statuses summary 
+++++++++++++++++++++++++++++++++
+
+.. code-block:: bash
+
+  # ceph orch hardware status
+  +------------+---------+-----+-----+--------+-------+------+
+  |    HOST    | STORAGE | CPU | NET | MEMORY | POWER | FANS |
+  +------------+---------+-----+-----+--------+-------+------+
+  |   node-10  |    ok   |  ok |  ok |   ok   |   ok  |  ok  |
+  +------------+---------+-----+-----+--------+-------+------+
+
+
+storage devices report
+++++++++++++++++++++++
+
+.. code-block:: bash
+
+  # ceph orch hardware status IBM-Ceph-1 --category storage
+  +------------+--------------------------------------------------------+------------------+----------------+----------+----------------+--------+---------+
+  |    HOST    |                          NAME                          |      MODEL       |      SIZE      | PROTOCOL |       SN       | STATUS |  STATE  |
+  +------------+--------------------------------------------------------+------------------+----------------+----------+----------------+--------+---------+
+  |   node-10  | Disk 8 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT99QLL    |   OK   | Enabled |
+  |   node-10  | Disk 10 in Backplane 1 of Storage Controller in Slot 2 | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZYX    |   OK   | Enabled |
+  |   node-10  | Disk 11 in Backplane 1 of Storage Controller in Slot 2 | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZWB    |   OK   | Enabled |
+  |   node-10  | Disk 9 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZC9    |   OK   | Enabled |
+  |   node-10  | Disk 3 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT9903Y    |   OK   | Enabled |
+  |   node-10  | Disk 1 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT9901E    |   OK   | Enabled |
+  |   node-10  | Disk 7 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZQJ    |   OK   | Enabled |
+  |   node-10  | Disk 2 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT99PA2    |   OK   | Enabled |
+  |   node-10  | Disk 4 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT99PFG    |   OK   | Enabled |
+  |   node-10  | Disk 0 in Backplane 0 of Storage Controller in Slot 2  | MZ7L33T8HBNAAD3  | 3840755981824  |   SATA   | S6M5NE0T800539 |   OK   | Enabled |
+  |   node-10  | Disk 1 in Backplane 0 of Storage Controller in Slot 2  | MZ7L33T8HBNAAD3  | 3840755981824  |   SATA   | S6M5NE0T800554 |   OK   | Enabled |
+  |   node-10  | Disk 6 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZER    |   OK   | Enabled |
+  |   node-10  | Disk 0 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT98ZEJ    |   OK   | Enabled |
+  |   node-10  | Disk 5 in Backplane 1 of Storage Controller in Slot 2  | ST20000NM008D-3D | 20000588955136 |   SATA   |    ZVT99QMH    |   OK   | Enabled |
+  |   node-10  |           Disk 0 on AHCI Controller in SL 6            |  MTFDDAV240TDU   |  240057409536  |   SATA   |  22373BB1E0F8  |   OK   | Enabled |
+  |   node-10  |           Disk 1 on AHCI Controller in SL 6            |  MTFDDAV240TDU   |  240057409536  |   SATA   |  22373BB1E0D5  |   OK   | Enabled |
+  +------------+--------------------------------------------------------+------------------+----------------+----------+----------------+--------+---------+
+
+
+
+firmwares details
++++++++++++++++++
+
+.. code-block:: bash
+
+  # ceph orch hardware status node-10 --category firmwares
+  +------------+----------------------------------------------------------------------------+--------------------------------------------------------------+----------------------+-------------+--------+
+  |    HOST    |                                 COMPONENT                                  |                             NAME                             |         DATE         |   VERSION   | STATUS |
+  +------------+----------------------------------------------------------------------------+--------------------------------------------------------------+----------------------+-------------+--------+
+  |   node-10  |               current-107649-7.03__raid.backplane.firmware.0               |                         Backplane 0                          | 2022-12-05T00:00:00Z |     7.03    |   OK   |
+  
+  
+  ... omitted output ...
+  
+  
+  |   node-10  |               previous-25227-6.10.30.20__idrac.embedded.1-1                |             Integrated Remote Access Controller              |      00:00:00Z       |  6.10.30.20 |   OK   |
+  +------------+----------------------------------------------------------------------------+--------------------------------------------------------------+----------------------+-------------+--------+
+
+
+hardware critical warnings report
++++++++++++++++++++++++++++++++++
+
+.. code-block:: bash
+
+  # ceph orch hardware status --category criticals
+  +------------+-----------+------------+----------+-----------------+
+  |    HOST    | COMPONENT |    NAME    |  STATUS  |      STATE      |
+  +------------+-----------+------------+----------+-----------------+
+  |   node-10  |   power   | PS2 Status | critical |    unplugged    |
+  +------------+-----------+------------+----------+-----------------+
+
+
+Developpers
+-----------
+
+.. py:currentmodule:: cephadm.agent
+.. autoclass:: NodeProxyEndpoint
+.. automethod:: NodeProxyEndpoint.__init__
+.. automethod:: NodeProxyEndpoint.oob
+.. automethod:: NodeProxyEndpoint.data
+.. automethod:: NodeProxyEndpoint.fullreport
+.. automethod:: NodeProxyEndpoint.summary
+.. automethod:: NodeProxyEndpoint.criticals
+.. automethod:: NodeProxyEndpoint.memory
+.. automethod:: NodeProxyEndpoint.storage
+.. automethod:: NodeProxyEndpoint.network
+.. automethod:: NodeProxyEndpoint.power
+.. automethod:: NodeProxyEndpoint.processors
+.. automethod:: NodeProxyEndpoint.fans
+.. automethod:: NodeProxyEndpoint.firmwares
+.. automethod:: NodeProxyEndpoint.led
+
diff --git a/doc/index.rst b/doc/index.rst
index 8edc2cb09e0e..df8652dc065d 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -121,5 +121,6 @@ about Ceph, see our `Architecture`_ section.
    releases/general
    releases/index
    security/index
+   hardware-monitoring/index
    Glossary <glossary>
    Tracing <jaegertracing/index>
diff --git a/doc/monitoring/index.rst b/doc/monitoring/index.rst
index 2bf2aca90f2f..3f0bb29194b1 100644
--- a/doc/monitoring/index.rst
+++ b/doc/monitoring/index.rst
@@ -470,5 +470,8 @@ Useful queries
   rate(ceph_rbd_read_latency_sum[30s]) / rate(ceph_rbd_read_latency_count[30s]) * on (instance) group_left (ceph_daemon) ceph_rgw_metadata
 
 
+Hardware monitoring
+===================
 
+See :ref:`hardware-monitoring`
 

From f9202a9afbb7346689dc380b14146f34c396bd5f Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 31 Jan 2024 14:47:11 +0000
Subject: [PATCH 1717/2492] fixup: create in get_hash_info()

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECCommon.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc
index 2d7acf15d18e..8752b54b4629 100644
--- a/src/osd/ECCommon.cc
+++ b/src/osd/ECCommon.cc
@@ -998,6 +998,8 @@ ECUtil::HashInfoRef ECCommon::UnstableHashInfoRegistry::get_hash_info(
         dout(0) << __func__ << ": Mismatch of total_chunk_size "
       		       << hinfo.get_total_chunk_size() << dendl;
         return ECUtil::HashInfoRef();
+      } else {
+        create = true;
       }
     } else if (size == 0) { // If empty object and no hinfo, create it
       create = true;

From a3bdffb4e0366af8288258539f17bafeae17e3c4 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Wed, 31 Jan 2024 15:25:10 +0000
Subject: [PATCH 1718/2492] src/osd/OSDMap.cc: Fix encoder to produce same
 bytestream

Fixes: https://tracker.ceph.com/issues/63389

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/osd/OSDMap.cc | 11 +++++++----
 src/osd/OSDMap.h  |  3 ++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 4cd9bff44416..aaa7a66a4974 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -588,9 +588,9 @@ void OSDMap::Incremental::encode(ceph::buffer::list& bl, uint64_t features) cons
       v = 5;
     } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
       v = 6;
-    } /* else if (!HAVE_FEATURE(features, SERVER_REEF)) {
+    } else if (!HAVE_FEATURE(features, SERVER_REEF)) {
       v = 8;
-    } */
+    }
     ENCODE_START(v, 1, bl); // client-usable data
     encode(fsid, bl);
     encode(epoch, bl);
@@ -3015,6 +3015,9 @@ bool OSDMap::primary_changed_broken(
 uint64_t OSDMap::get_encoding_features() const
 {
   uint64_t f = SIGNIFICANT_FEATURES;
+  if (require_osd_release < ceph_release_t::reef) {
+    f &= ~CEPH_FEATURE_SERVER_REEF;
+  }
   if (require_osd_release < ceph_release_t::octopus) {
     f &= ~CEPH_FEATURE_SERVER_OCTOPUS;
   }
@@ -3194,9 +3197,9 @@ void OSDMap::encode(ceph::buffer::list& bl, uint64_t features) const
       v = 6;
     } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
       v = 7;
-    } /* else if (!HAVE_FEATURE(features, SERVER_REEF)) {
+    } else if (!HAVE_FEATURE(features, SERVER_REEF)) {
       v = 9;
-    } */
+    }
     ENCODE_START(v, 1, bl); // client-usable data
     // base
     encode(fsid, bl);
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index e37aeafc4312..f54667cbc0ed 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -570,7 +570,8 @@ class OSDMap {
     CEPH_FEATUREMASK_SERVER_LUMINOUS |
     CEPH_FEATUREMASK_SERVER_MIMIC |
     CEPH_FEATUREMASK_SERVER_NAUTILUS |
-    CEPH_FEATUREMASK_SERVER_OCTOPUS;
+    CEPH_FEATUREMASK_SERVER_OCTOPUS |
+    CEPH_FEATUREMASK_SERVER_REEF;
 
   struct addrs_s {
     mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > client_addrs;

From 61222e11ab493c3ccb3c188816a3747bec5012e0 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 29 Jan 2024 10:58:43 +0000
Subject: [PATCH 1719/2492] mgr/dashboard: added dir search to snap schdl form

Fixes: https://tracker.ceph.com/issues/64246
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 ...ephfs-snapshotschedule-form.component.html | 25 ++++----
 .../cephfs-snapshotschedule-form.component.ts | 28 +++++---
 .../src/app/ceph/cephfs/cephfs.module.ts      |  3 +-
 .../src/app/shared/api/cephfs.service.ts      |  3 +-
 .../api/directory-store.service.spec.ts       | 24 +++++++
 .../app/shared/api/directory-store.service.ts | 64 +++++++++++++++++++
 6 files changed, 124 insertions(+), 23 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.ts

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
index 9e9cde86b325..e315e8ab7671 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
@@ -15,20 +15,19 @@
                  i18n>Directory
           </label>
           <div class="cd-col-form-input">
-            <ng-template #loading>
+            <div class="input-group">
+            <input id="typeahead-http"
+                   i18n
+                   type="text"
+                   class="form-control"
+                   disabled="directoryStore.isLoading"
+                   formControlName="directory"
+                   [ngbTypeahead]="search"
+                   [placeholder]="directoryStore.isLoading ? 'Loading directories' : 'Directory search'" />
+            <div *ngIf="directoryStore.isLoading">
               <i [ngClass]="[icons.spinner, icons.spin, 'mt-2', 'me-2']"></i>
-              <span i18n>Loading directories</span>
-            </ng-template>
-            <select class="form-select"
-                    id="directory"
-                    name="directory"
-                    formControlName="directory"
-                    *ngIf="directories$ | async as directories; else loading">
-              <option [ngValue]="null"
-                      i18n>--Select a directory--</option>
-              <option *ngFor="let dir of directories"
-                      [value]="dir.path">{{ dir.path }}</option>
-            </select>
+            </div>
+          </div>
             <span class="invalid-feedback"
                   *ngIf="snapScheduleForm.showError('directory', formDir, 'required')"
                   i18n>This field is required.</span>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 5b6d900e7520..41e55c72599a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -2,10 +2,10 @@ import { ChangeDetectorRef, Component, OnInit } from '@angular/core';
 import { AbstractControl, FormArray, FormControl, FormGroup, Validators } from '@angular/forms';
 import { NgbActiveModal, NgbDateStruct, NgbTimeStruct } from '@ng-bootstrap/ng-bootstrap';
 import { uniq } from 'lodash';
-import { Observable, timer } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { Observable, OperatorFunction, of, timer } from 'rxjs';
+import { catchError, debounceTime, distinctUntilChanged, map, switchMap } from 'rxjs/operators';
 import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
-import { CephfsService } from '~/app/shared/api/cephfs.service';
+import { DirectoryStoreService } from '~/app/shared/api/directory-store.service';
 import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants';
 import { Icons } from '~/app/shared/enum/icons.enum';
 import { RepeatFrequency } from '~/app/shared/enum/repeat-frequency.enum';
@@ -13,12 +13,12 @@ import { RetentionFrequency } from '~/app/shared/enum/retention-frequency.enum';
 import { CdForm } from '~/app/shared/forms/cd-form';
 import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
-import { CephfsDir } from '~/app/shared/models/cephfs-directory-models';
 import { FinishedTask } from '~/app/shared/models/finished-task';
 import { RetentionPolicy, SnapshotScheduleFormValue } from '~/app/shared/models/snapshot-schedule';
 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 
 const VALIDATON_TIMER = 300;
+const DEBOUNCE_TIMER = 300;
 
 @Component({
   selector: 'cd-cephfs-snapshotschedule-form',
@@ -42,15 +42,14 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
   resource!: string;
 
   columns!: CdTableColumn[];
-  directories$!: Observable<CephfsDir[]>;
 
   constructor(
     public activeModal: NgbActiveModal,
     private actionLabels: ActionLabelsI18n,
-    private cephfsService: CephfsService,
     private snapScheduleService: CephfsSnapshotScheduleService,
     private taskWrapper: TaskWrapperService,
-    private cd: ChangeDetectorRef
+    private cd: ChangeDetectorRef,
+    public directoryStore: DirectoryStoreService
   ) {
     super();
     this.resource = $localize`Snapshot schedule`;
@@ -70,7 +69,7 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
 
   ngOnInit(): void {
     this.action = this.actionLabels.CREATE;
-    this.directories$ = this.cephfsService.lsDir(this.id, '/', 3);
+    this.directoryStore.loadDirectories(this.id, '/', 3);
     this.createForm();
     this.loadingReady();
   }
@@ -79,6 +78,19 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     return this.snapScheduleForm.get('retentionPolicies') as FormArray;
   }
 
+  search: OperatorFunction<string, readonly string[]> = (input: Observable<string>) =>
+    input.pipe(
+      debounceTime(DEBOUNCE_TIMER),
+      distinctUntilChanged(),
+      switchMap((term) =>
+        this.directoryStore.search(term, this.id).pipe(
+          catchError(() => {
+            return of([]);
+          })
+        )
+      )
+    );
+
   createForm() {
     this.snapScheduleForm = new CdFormGroup(
       {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index b13273dc4a21..687dd0b93ee9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -45,7 +45,8 @@ import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-f
     NgbTooltipModule,
     DataTableModule,
     NgbDatepickerModule,
-    NgbTimepickerModule
+    NgbTimepickerModule,
+    NgbTypeaheadModule
   ],
   declarations: [
     CephfsDetailComponent,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
index a265ae7a2658..d2dfbc0e2a76 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
@@ -6,6 +6,7 @@ import { Observable } from 'rxjs';
 
 import { cdEncode } from '../decorators/cd-encode';
 import { CephfsDir, CephfsQuotas } from '../models/cephfs-directory-models';
+import { shareReplay } from 'rxjs/operators';
 
 @cdEncode
 @Injectable({
@@ -26,7 +27,7 @@ export class CephfsService {
     if (path) {
       apiPath += `&path=${encodeURIComponent(path)}`;
     }
-    return this.http.get<CephfsDir[]>(apiPath);
+    return this.http.get<CephfsDir[]>(apiPath).pipe(shareReplay());
   }
 
   getCephfs(id: number) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.spec.ts
new file mode 100644
index 000000000000..78590c89f53f
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.spec.ts
@@ -0,0 +1,24 @@
+import { TestBed } from '@angular/core/testing';
+
+import { DirectoryStoreService } from './directory-store.service';
+import { configureTestBed } from '~/testing/unit-test-helper';
+import { CephfsService } from './cephfs.service';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+
+describe('DirectoryStoreService', () => {
+  let service: DirectoryStoreService;
+
+  configureTestBed({
+    imports: [HttpClientTestingModule],
+    providers: [CephfsService]
+  });
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(DirectoryStoreService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.ts
new file mode 100644
index 000000000000..cdc5337ac121
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/directory-store.service.ts
@@ -0,0 +1,64 @@
+import { Injectable } from '@angular/core';
+import { CephfsService } from './cephfs.service';
+import { BehaviorSubject, Observable, Subject, timer } from 'rxjs';
+import { CephfsDir } from '../models/cephfs-directory-models';
+import { filter, map, retry, share, switchMap, takeUntil, tap } from 'rxjs/operators';
+
+type DirectoryStore = Record<number, CephfsDir[]>;
+
+const POLLING_INTERVAL = 600 * 1000;
+
+@Injectable({
+  providedIn: 'root'
+})
+export class DirectoryStoreService {
+  private _directoryStoreSubject = new BehaviorSubject<DirectoryStore>({});
+
+  readonly directoryStore$: Observable<DirectoryStore> = this._directoryStoreSubject.asObservable();
+
+  stopDirectoryPolling = new Subject();
+
+  isLoading = true;
+
+  constructor(private cephFsService: CephfsService) {}
+
+  loadDirectories(id: number, path = '/', depth = 3) {
+    this.directoryStore$
+      .pipe(
+        filter((store: DirectoryStore) => !Boolean(store[id])),
+        switchMap(() =>
+          timer(0, POLLING_INTERVAL).pipe(
+            switchMap(() =>
+              this.cephFsService.lsDir(id, path, depth).pipe(
+                tap((response) => {
+                  this.isLoading = false;
+                  this._directoryStoreSubject.next({ [id]: response });
+                })
+              )
+            ),
+            retry(),
+            share(),
+            takeUntil(this.stopDirectoryPolling)
+          )
+        )
+      )
+      .subscribe();
+  }
+
+  search(term: string, id: number, limit = 5) {
+    return this.directoryStore$.pipe(
+      map((store: DirectoryStore) => {
+        const regEx = new RegExp(term, 'gi');
+        const results = store[id]
+          .filter((x) => regEx.test(x.path))
+          .map((x) => x.path)
+          .slice(0, limit);
+        return results;
+      })
+    );
+  }
+
+  stopPollingDictories() {
+    this.stopDirectoryPolling.next();
+  }
+}

From ecbd8376675c39893c55ceb7fa8653458d3fc6ed Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 31 Jan 2024 12:31:02 -0500
Subject: [PATCH 1720/2492] doc/rgw: fix Attributes index in CreateTopic
 example

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/notifications.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
index 314e0ba6f167..655f3bbc39e3 100644
--- a/doc/radosgw/notifications.rst
+++ b/doc/radosgw/notifications.rst
@@ -159,7 +159,7 @@ updating, use the name of an existing topic and different endpoint values).
    [&Attributes.entry.12.key=time_to_live&Attributes.entry.12.value=<seconds to live>]
    [&Attributes.entry.13.key=max_retries&Attributes.entry.13.value=<retries number>]
    [&Attributes.entry.14.key=retry_sleep_duration&Attributes.entry.14.value=<sleep seconds>]
-   [&Attributes.entry.14.key=Policy&Attributes.entry.14.value=<policy-JSON-string>]
+   [&Attributes.entry.15.key=Policy&Attributes.entry.15.value=<policy-JSON-string>]
 
 Request parameters:
 

From eaa4ccd729c9937b998924b75f0e1a1d39b4af36 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Wed, 22 Nov 2023 19:31:39 -0500
Subject: [PATCH 1721/2492] rgw: change section name of rgw_op counters

The rgw_op section of `counter dump/schema` becomes:
- rgw_op_global for the global op counters
- rgw_op_per_user for the user labeled counters
- rgw_op_per_bucket for the bucket labeled counters

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 doc/radosgw/metrics.rst      | 13 ++++++++++---
 src/rgw/rgw_perf_counters.cc | 17 ++++++++++-------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/doc/radosgw/metrics.rst b/doc/radosgw/metrics.rst
index dd5c56c1752a..ee9e7225bc2b 100644
--- a/doc/radosgw/metrics.rst
+++ b/doc/radosgw/metrics.rst
@@ -75,8 +75,15 @@ The following metrics related to S3 or Swift operations are tracked per Ceph Obj
      - Guage
      - Total latency of list bucket operations
 
-More information about op metrics can be seen in the ``rgw_op`` section of the output of the ``counter schema`` command.
-To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` section of the output of the ``counter dump`` command::
+There are three different sections in the output of the ``counter dump`` and ``counter schema`` commands that show the op metrics and their information.
+The sections are ``rgw_op``, ``rgw_op_per_user``, and ``rgw_op_per_bucket``.
+
+The counters in the ``rgw_op`` section reflect the totals of each op metric for a given Ceph Object Gateway.
+The counters in the ``rgw_op_per_user`` and ``rgw_op_per_bucket`` sections are labeled counters of op metrics for a user or bucket respectively.
+
+Information about op metrics can be seen in the ``rgw_op`` sections of the output of the ``counter schema`` command.
+
+To view op metrics in the Ceph Object Gateway go to the ``rgw_op`` sections of the output of the ``counter dump`` command::
 
     "rgw_op": [
         {
@@ -112,7 +119,7 @@ Op Metrics Labels
 
 Op metrics can also be tracked per-user or per-bucket. These metrics are exported to Prometheus with labels like Bucket = {name} or User = {userid}::
 
-    "rgw_op": [
+    "rgw_op_per_bucket": [
         ...
         {
             "labels": {
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index 125bace14450..40668c3d8227 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -112,11 +112,14 @@ namespace rgw::op_counters {
 ceph::perf_counters::PerfCountersCache *user_counters_cache = NULL;
 ceph::perf_counters::PerfCountersCache *bucket_counters_cache = NULL;
 PerfCounters *global_op_counters = NULL;
-const std::string rgw_op_counters_key = "rgw_op";
+const std::string rgw_global_op_counters_key = "rgw_op";
+const std::string rgw_user_op_counters_key = "rgw_op_per_user";
+const std::string rgw_bucket_op_counters_key = "rgw_op_per_bucket";
 
 std::shared_ptr<PerfCounters> create_rgw_op_counters(const std::string& name, CephContext *cct) {
   std::string_view key = ceph::perf_counters::key_name(name);
-  ceph_assert(rgw_op_counters_key == key);
+  ceph_assert(rgw_global_op_counters_key == key ||
+              rgw_user_op_counters_key == key || rgw_bucket_op_counters_key == key);
   PerfCountersBuilder pcb(cct, name, l_rgw_op_first, l_rgw_op_last);
   add_rgw_op_counters(&pcb);
   std::shared_ptr<PerfCounters> new_counters(pcb.create_perf_counters());
@@ -125,7 +128,7 @@ std::shared_ptr<PerfCounters> create_rgw_op_counters(const std::string& name, Ce
 }
 
 void global_op_counters_init(CephContext *cct) {
-  PerfCountersBuilder pcb(cct, rgw_op_counters_key, l_rgw_op_first, l_rgw_op_last);
+  PerfCountersBuilder pcb(cct, rgw_global_op_counters_key, l_rgw_op_first, l_rgw_op_last);
   add_rgw_op_counters(&pcb);
   PerfCounters *new_counters = pcb.create_perf_counters();
   cct->get_perfcounters_collection()->add(new_counters);
@@ -138,18 +141,18 @@ CountersContainer get(req_state *s) {
 
   if (user_counters_cache && !s->user->get_id().id.empty()) {
     if (s->user->get_tenant().empty()) {
-      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}});
+      key = ceph::perf_counters::key_create(rgw_user_op_counters_key, {{"user", s->user->get_id().id}});
     } else {
-      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"User", s->user->get_id().id}, {"Tenant", s->user->get_tenant()}});
+      key = ceph::perf_counters::key_create(rgw_user_op_counters_key, {{"user", s->user->get_id().id}, {"tenant", s->user->get_tenant()}});
     }
     counters.user_counters = user_counters_cache->get(key);
   }
 
   if (bucket_counters_cache && !s->bucket_name.empty()) {
     if (s->bucket_tenant.empty()) {
-      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}});
+      key = ceph::perf_counters::key_create(rgw_bucket_op_counters_key, {{"bucket", s->bucket_name}});
     } else {
-      key = ceph::perf_counters::key_create(rgw_op_counters_key, {{"Bucket", s->bucket_name}, {"Tenant", s->bucket_tenant}});
+      key = ceph::perf_counters::key_create(rgw_bucket_op_counters_key, {{"bucket", s->bucket_name}, {"tenant", s->bucket_tenant}});
     }
     counters.bucket_counters = bucket_counters_cache->get(key);
   }

From 6a0960bf82f07185ffccc08528e518124500fc3e Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Wed, 29 Nov 2023 13:06:01 -0500
Subject: [PATCH 1722/2492] rgw: change topic label to lowercase

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 src/rgw/rgw_perf_counters.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index 40668c3d8227..d316476a27b7 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -211,7 +211,7 @@ const std::string rgw_topic_counters_key = "rgw_topic";
 CountersManager::CountersManager(const std::string& topic_name, CephContext *cct)
     : cct(cct)
 {
-  const std::string topic_key = ceph::perf_counters::key_create(rgw_topic_counters_key, {{"Topic", topic_name}});
+  const std::string topic_key = ceph::perf_counters::key_create(rgw_topic_counters_key, {{"topic", topic_name}});
   PerfCountersBuilder pcb(cct, topic_key, l_rgw_topic_first, l_rgw_topic_last);
   add_rgw_topic_counters(&pcb);
   topic_counters = std::unique_ptr<PerfCounters>(pcb.create_perf_counters());

From 0fc875a6416171386057b167c6d7681463773e60 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 31 Jan 2024 14:47:23 +0000
Subject: [PATCH 1723/2492] fixup: don't use obc.attr_cache; move back to disk

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 33 ++++++++++++++++++++++++++++-----
 src/osd/ECBackend.h  |  8 +++++++-
 src/osd/PGBackend.h  |  1 +
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 318b1aca1f2f..e455c5062b74 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1444,6 +1444,25 @@ struct ECClassicalOp : ECCommon::RMWPipeline::Op {
   }
 };
 
+std::tuple<
+  int,
+  map<string, bufferlist, less<>>,
+  size_t
+> ECBackend::get_attrs_n_size_from_disk(const hobject_t& hoid)
+{
+  struct stat st;
+  if (int r = object_stat(hoid, &st); r < 0) {
+    dout(10) << __func__ << ": stat error " << r << " on" << hoid << dendl;
+    return { r, {}, 0 };
+  }
+  map<string, bufferlist, less<>> real_attrs;
+  if (int r = PGBackend::objects_get_attrs(hoid, &real_attrs); r < 0) {
+    dout(10) << __func__ << ": get attr error " << r << " on" << hoid << dendl;
+    return { r, {}, 0 };
+  }
+  return { 0, real_attrs, st.st_size };
+}
+
 void ECBackend::submit_transaction(
   const hobject_t &hoid,
   const object_stat_sum_t &delta_stats,
@@ -1479,11 +1498,15 @@ void ECBackend::submit_transaction(
     sinfo,
     *(op->t),
     [&](const hobject_t &i) {
-      ECUtil::HashInfoRef ref = unstable_hashinfo_registry.get_hash_info(
-	i,
-	true,
-	op->t->obc_map[hoid]->attr_cache,
-	op->t->obc_map[hoid]->obs.oi.size);
+      dout(10) << "submit_transaction: obtaining hash info for get_write_plan" << dendl;
+      ECUtil::HashInfoRef ref;
+      if (auto [r, attrs, size] = get_attrs_n_size_from_disk(i); r >= 0 || r == -ENOENT) {
+        ref = unstable_hashinfo_registry.get_hash_info(
+	  i,
+	  true,
+	  attrs, //op->t->obc_map[hoid]->attr_cache,
+	  size); //op->t->obc_map[hoid]->obs.oi.size);
+      }
       if (!ref) {
 	derr << __func__ << ": get_hash_info(" << i << ")"
 	     << " returned a null pointer and there is no "
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index 56e1c972fe24..db003f0adcb9 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -400,9 +400,15 @@ class ECBackend : public PGBackend, public ECCommon {
 
   ECCommon::UnstableHashInfoRegistry unstable_hashinfo_registry;
 
-  int object_stat(const hobject_t &hoid, struct stat* st);
+
+  std::tuple<
+    int,
+    std::map<std::string, ceph::bufferlist, std::less<>>,
+    size_t
+  > get_attrs_n_size_from_disk(const hobject_t& hoid);
 
 public:
+  int object_stat(const hobject_t &hoid, struct stat* st);
   ECBackend(
     PGBackend::Listener *pg,
     const coll_t &coll,
diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h
index 274d1fb874bc..c44c90514a39 100644
--- a/src/osd/PGBackend.h
+++ b/src/osd/PGBackend.h
@@ -52,6 +52,7 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
   */
  class PGBackend {
  public:
+  virtual int object_stat(const hobject_t &hoid, struct stat* st) { return -1;};
    CephContext* cct;
  protected:
    ObjectStore *store;

From 7000cbf82f40e0997c815a35a3a53f32e1afcef1 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 1 Feb 2024 12:55:44 +0800
Subject: [PATCH 1724/2492] crimson/osd: correct variable naming

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/shard_services.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 604c045eb9b8..ff35df280ad8 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -708,9 +708,9 @@ ShardServices::get_or_create_pg(
   std::unique_ptr<PGCreateInfo> info)
 {
   if (info) {
-    auto [fut, creating] = local_state.pg_map.wait_for_pg(
+    auto [fut, existed] = local_state.pg_map.wait_for_pg(
       std::move(trigger), pgid);
-    if (!creating) {
+    if (!existed) {
       local_state.pg_map.set_creating(pgid);
       (void)handle_pg_create_info(
 	std::move(info));

From cde33fcae7526303b6f9eba30c49d46b404da087 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 18 Jan 2024 11:21:19 +0800
Subject: [PATCH 1725/2492] crimson/os/seastore/avlallocator: add the interface
 that allocate multiple regions

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../os/seastore/random_block_manager.h        |  9 +++
 .../random_block_manager/avlallocator.cc      | 71 +++++++++++++++++++
 .../random_block_manager/avlallocator.h       |  3 +
 .../random_block_manager/block_rb_manager.cc  | 28 ++++++++
 .../random_block_manager/block_rb_manager.h   |  4 +-
 .../random_block_manager/extent_allocator.h   | 11 +++
 6 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h
index d9be1b5e6d9c..f0e0f018bd10 100644
--- a/src/crimson/os/seastore/random_block_manager.h
+++ b/src/crimson/os/seastore/random_block_manager.h
@@ -22,6 +22,11 @@
 
 namespace crimson::os::seastore {
 
+struct alloc_paddr_result {
+  paddr_t start;
+  extent_len_t len;
+};
+
 struct rbm_shard_info_t {
   std::size_t size = 0;
   uint64_t start_offset = 0;
@@ -125,6 +130,10 @@ class RandomBlockManager {
   // allocator, return start addr of allocated blocks
   virtual paddr_t alloc_extent(size_t size) = 0;
 
+  using allocate_ret_bare = std::list<alloc_paddr_result>;
+  using allo_extents_ret = allocate_ertr::future<allocate_ret_bare>;
+  virtual allocate_ret_bare alloc_extents(size_t size) = 0;
+
   virtual void mark_space_used(paddr_t paddr, size_t len) = 0;
   virtual void mark_space_free(paddr_t paddr, size_t len) = 0;
 
diff --git a/src/crimson/os/seastore/random_block_manager/avlallocator.cc b/src/crimson/os/seastore/random_block_manager/avlallocator.cc
index 28137a23d798..51624b1c8b29 100644
--- a/src/crimson/os/seastore/random_block_manager/avlallocator.cc
+++ b/src/crimson/os/seastore/random_block_manager/avlallocator.cc
@@ -87,6 +87,33 @@ rbm_abs_addr AvlAllocator::find_block(size_t size)
   return total_size;
 }
 
+extent_len_t AvlAllocator::find_block(
+  size_t size,
+  rbm_abs_addr &start)
+{
+  uint64_t max_size = 0;
+  auto p = extent_size_tree.rbegin();
+  if (p != extent_size_tree.rend()) {
+    max_size = p->end - p->start;
+  }
+
+  assert(max_size);
+  if (max_size <= size) {
+    start = p->start;
+    return max_size;
+  }
+
+  const auto comp = extent_size_tree.key_comp();
+  auto iter = extent_size_tree.lower_bound(
+    extent_range_t{base_addr, base_addr + size}, comp);
+  ceph_assert(iter != extent_size_tree.end());
+  ceph_assert(is_aligned(iter->start, block_size));
+  ceph_assert(size <= iter->length());
+  start = iter->start;
+  return size;
+}
+
+
 void AvlAllocator::_add_to_tree(rbm_abs_addr start, rbm_abs_addr size)
 {
   LOG_PREFIX(AvlAllocator::_add_to_tree);
@@ -174,6 +201,50 @@ std::optional<interval_set<rbm_abs_addr>> AvlAllocator::alloc_extent(
   return result;
 }
 
+std::optional<interval_set<rbm_abs_addr>> AvlAllocator::alloc_extents(
+  size_t size)
+{
+  LOG_PREFIX(AvlAllocator::alloc_extents);
+  if (available_size < size) {
+    return std::nullopt;
+  }
+  if (extent_size_tree.empty()) {
+    return std::nullopt;
+  }
+  ceph_assert(size > 0);
+  ceph_assert(is_aligned(size, block_size));
+
+  interval_set<rbm_abs_addr> result;
+
+  auto try_to_alloc_block = [this, &result, FNAME] (uint64_t alloc_size)
+  {
+    while (alloc_size) {
+      rbm_abs_addr start = 0;
+      extent_len_t len = find_block(alloc_size, start);
+      ceph_assert(len);
+      _remove_from_tree(start, len);
+      DEBUG("allocate addr: {}, allocate size: {}, available size: {}",
+	start, len, available_size);
+      result.insert(start, len);
+      alloc_size -= len;
+    }
+    return 0;
+  };
+  
+  auto alloc = std::min(max_alloc_size, size);
+  try_to_alloc_block(alloc);
+
+  assert(!result.empty());
+  for (auto p : result) {
+    INFO("result start: {}, end: {}", p.first, p.first + p.second);
+    if (detailed) {
+      assert(!reserved_extent_tracker.contains(p.first, p.second));
+      reserved_extent_tracker.insert(p.first, p.second);
+    }
+  }
+  return result;
+}
+
 void AvlAllocator::free_extent(rbm_abs_addr addr, size_t size)
 {
   assert(total_size);
diff --git a/src/crimson/os/seastore/random_block_manager/avlallocator.h b/src/crimson/os/seastore/random_block_manager/avlallocator.h
index d1a4fabca5a4..fa703994e3f1 100644
--- a/src/crimson/os/seastore/random_block_manager/avlallocator.h
+++ b/src/crimson/os/seastore/random_block_manager/avlallocator.h
@@ -64,6 +64,8 @@ class AvlAllocator : public ExtentAllocator {
     detailed(detailed) {}
   std::optional<interval_set<rbm_abs_addr>> alloc_extent(
     size_t size) final;
+  std::optional<interval_set<rbm_abs_addr>> alloc_extents(
+    size_t size) final;
 
   void free_extent(rbm_abs_addr addr, size_t size) final;
   void mark_extent_used(rbm_abs_addr addr, size_t size) final;
@@ -141,6 +143,7 @@ class AvlAllocator : public ExtentAllocator {
 
   void _remove_from_tree(rbm_abs_addr start, rbm_abs_addr size);
   rbm_abs_addr find_block(size_t size);
+  extent_len_t find_block(size_t size, rbm_abs_addr &start);
 
   using extent_tree_t = 
     boost::intrusive::avl_set<
diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
index 511b70a2eec9..6bae96f84a5a 100644
--- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
+++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
@@ -62,6 +62,34 @@ paddr_t BlockRBManager::alloc_extent(size_t size)
   return paddr;
 }
 
+BlockRBManager::allocate_ret_bare
+BlockRBManager::alloc_extents(size_t size)
+{
+  LOG_PREFIX(BlockRBManager::alloc_extents);
+  assert(allocator);
+  auto alloc = allocator->alloc_extents(size);
+  if (!alloc) {
+    return {};
+  }
+  allocate_ret_bare ret;
+  size_t len = 0;
+  for (auto extent = (*alloc).begin();
+       extent != (*alloc).end();
+       extent++) {
+    len += extent.get_len();
+    paddr_t paddr = convert_abs_addr_to_paddr(
+      extent.get_start(),
+      device->get_device_id());
+    DEBUG("allocated addr: {}, size: {}, requested size: {}",
+         paddr, extent.get_len(), size);
+    ret.push_back(
+      {std::move(paddr),
+      static_cast<extent_len_t>(extent.get_len())});
+  }
+  ceph_assert(size == len);
+  return ret;
+}
+
 void BlockRBManager::complete_allocation(
     paddr_t paddr, size_t size)
 {
diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
index b686820d066e..14d2a28a3146 100644
--- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
+++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
@@ -54,11 +54,11 @@ class BlockRBManager final : public RandomBlockManager {
    * To do so, alloc_extent() looks into both in-memory allocator
    * and freebitmap blocks.
    *
-   * TODO: multiple allocation
-   *
    */
   paddr_t alloc_extent(size_t size) final; // allocator, return blocks
 
+  allocate_ret_bare alloc_extents(size_t size) final; // allocator, return blocks
+
   void complete_allocation(paddr_t addr, size_t size) final;
 
   size_t get_start_rbm_addr() const {
diff --git a/src/crimson/os/seastore/random_block_manager/extent_allocator.h b/src/crimson/os/seastore/random_block_manager/extent_allocator.h
index 8a3e62c6d050..e3d3c8003486 100644
--- a/src/crimson/os/seastore/random_block_manager/extent_allocator.h
+++ b/src/crimson/os/seastore/random_block_manager/extent_allocator.h
@@ -26,6 +26,17 @@ class ExtentAllocator {
    */
   virtual std::optional<interval_set<rbm_abs_addr>> alloc_extent(
     size_t size) = 0;
+
+  /**
+   * alloc_extents
+   *
+   * Allocate regions for the given size. A continuous region is returned
+   * if possible.
+   *
+   */
+  virtual std::optional<interval_set<rbm_abs_addr>> alloc_extents(
+    size_t size) = 0;
+
   /**
    * free_extent
    *

From fb7221749b7f03043673f8a72f7ad3c31b52d083 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 31 Jan 2024 10:48:35 +0100
Subject: [PATCH 1726/2492] mgr/rook: adding deployment to ceph image built for
 rook e2e testing Fixes: https://tracker.ceph.com/issues/64266

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/ci/Dockerfile                        | 1 +
 src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/pybind/mgr/rook/ci/Dockerfile b/src/pybind/mgr/rook/ci/Dockerfile
index 30ebea5749c8..a338188195ae 100644
--- a/src/pybind/mgr/rook/ci/Dockerfile
+++ b/src/pybind/mgr/rook/ci/Dockerfile
@@ -1,3 +1,4 @@
 FROM quay.io/ceph/daemon-base:latest-main
 COPY ./tmp_build/orchestrator /usr/share/ceph/mgr/orchestrator
 COPY ./tmp_build/rook /usr/share/ceph/mgr/rook
+COPY ./tmp_build/ceph/ /usr/lib/python3.6/site-packages/ceph/
diff --git a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
index dcd22fce9a75..24a6a5da23f4 100755
--- a/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
+++ b/src/pybind/mgr/rook/ci/scripts/bootstrap-rook-cluster.sh
@@ -44,6 +44,7 @@ build_ceph_image() {
     mkdir -p tmp_build/orchestrator
     cp ./../../orchestrator/*.py tmp_build/orchestrator
     cp ../*.py tmp_build/rook
+    cp -r ../../../../../src/python-common/ceph/ tmp_build/
 
     # we use the following tag to trick the Docker
     # running inside minikube so it uses this image instead

From 3c69808379a189b2c157d97d03d34a8af2b16820 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 23 Jan 2024 14:06:01 +0000
Subject: [PATCH 1727/2492] crimson/os/alienstore: cleanup N_CORES_FOR_SEASTAR
 constant

No CPU pinning if crimson_alien_thread_cpu_cores is not set.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/os/alienstore/alien_store.cc | 11 +++++------
 src/crimson/os/alienstore/alien_store.h  |  3 ---
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index d53dacb18246..31dedbcf893f 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -103,14 +103,13 @@ seastar::future<> AlienStore::start()
   }
   auto cpu_cores = seastar::resource::parse_cpuset(
     get_conf<std::string>("crimson_alien_thread_cpu_cores"));
-  // cores except the first "N_CORES_FOR_SEASTAR" ones will
-  // be used for alien threads scheduling:
-  // 	[0, N_CORES_FOR_SEASTAR) are reserved for seastar reactors
-  // 	[N_CORES_FOR_SEASTAR, ..] are assigned to alien threads.
+  //  crimson_alien_thread_cpu_cores are assigned to alien threads.
   if (!cpu_cores.has_value()) {
+    // no core isolation by default, cores [0, seastar::smp::count)
+    // will be shared between both alien and seastar reactor threads.
     seastar::resource::cpuset cpuset;
-    std::copy(boost::counting_iterator<unsigned>(N_CORES_FOR_SEASTAR),
-	      boost::counting_iterator<unsigned>(sysconf(_SC_NPROCESSORS_ONLN)),
+    std::copy(boost::counting_iterator<unsigned>(0),
+	      boost::counting_iterator<unsigned>(seastar::smp::count - 1),
 	      std::inserter(cpuset, cpuset.end()));
     if (cpuset.empty()) {
       logger().error("{}: unable to get nproc: {}", __func__, errno);
diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h
index 79c19b29896c..7138da955aa1 100644
--- a/src/crimson/os/alienstore/alien_store.h
+++ b/src/crimson/os/alienstore/alien_store.h
@@ -117,9 +117,6 @@ class AlienStore final : public FuturizedStore,
     });
   }
 
-  // number of cores that are PREVENTED from being scheduled
-  // to run alien store threads.
-  static constexpr int N_CORES_FOR_SEASTAR = 3;
   mutable std::unique_ptr<crimson::os::ThreadPool> tp;
   const std::string type;
   const std::string path;

From 66a07cb14c4a885b464d4d8af51ad40f2fb96347 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 23 Jan 2024 14:08:48 +0000
Subject: [PATCH 1728/2492] vstarh.sh: Crimson - set
 crimson_alien_thread_cpu_cores

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/vstart.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index 2317d5da6053..f9b2f3ce4cf2 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1666,6 +1666,12 @@ fi
 
 if [ "$ceph_osd" == "crimson-osd" ]; then
     $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp
+    if [ "$(expr $(nproc) - 1)" -gt "$(($CEPH_NUM_OSD * crimson_smp))" ]; then
+      echo "crimson_alien_thread_cpu_cores:" $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"
+      $CEPH_BIN/ceph -c $conf_fn config set osd crimson_alien_thread_cpu_cores $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"
+    else
+      echo "No alien thread cpu core isolation"
+    fi
 fi
 
 if [ $CEPH_NUM_MGR -gt 0 ]; then

From d79f30099c8471d2543b6ea09e744a6c3fcc1701 Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Sun, 28 Jan 2024 17:00:56 +0530
Subject: [PATCH 1729/2492] mds: Add debug messages in replica handling

Fixes: https://tracker.ceph.com/issues/64241
Signed-off-by: Kotresh HR <khiremat@redhat.com>
---
 src/mds/MDCache.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 325da1ae4c97..f79e806494bf 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -11139,12 +11139,16 @@ void MDCache::send_dentry_link(CDentry *dn, const MDRequestRef& mdr)
   CDir *subtree = get_subtree_root(dn->get_dir());
   for (const auto &p : dn->get_replicas()) {
     // don't tell (rename) witnesses; they already know
-    if (mdr.get() && mdr->more()->witnessed.count(p.first))
+    if (mdr.get() && mdr->more()->witnessed.count(p.first)) {
+      dout(20) << __func__ << " witnesses already know, skip notifying replica for the dentry " << *dn << dendl;
       continue;
+    }
     if (mds->mdsmap->get_state(p.first) < MDSMap::STATE_REJOIN ||
 	(mds->mdsmap->get_state(p.first) == MDSMap::STATE_REJOIN &&
-	 rejoin_gather.count(p.first)))
+	 rejoin_gather.count(p.first))) {
+      dout(20) << __func__ << " mds is not ready, skip notifying replica for the dentry " << *dn << dendl;
       continue;
+    }
     CDentry::linkage_t *dnl = dn->get_linkage();
     auto m = make_message<MDentryLink>(subtree->dirfrag(), dn->get_dir()->dirfrag(), dn->get_name(), dnl->is_primary());
     if (dnl->is_primary()) {
@@ -11218,13 +11222,17 @@ void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, const MDRequestR
        it != replicas.end();
        ++it) {
     // don't tell (rmdir) witnesses; they already know
-    if (mdr.get() && mdr->more()->witnessed.count(*it))
+    if (mdr.get() && mdr->more()->witnessed.count(*it)) {
+      dout(20) << __func__ << " witnesses already know, skip notifying replica for the dentry " << *dn << dendl;
       continue;
+    }
 
     if (mds->mdsmap->get_state(*it) < MDSMap::STATE_REJOIN ||
 	(mds->mdsmap->get_state(*it) == MDSMap::STATE_REJOIN &&
-	 rejoin_gather.count(*it)))
+	 rejoin_gather.count(*it))) {
+      dout(20) << __func__ << " mds is not ready, skip notifying replica for the dentry " << *dn << dendl;
       continue;
+    }
 
     auto unlink = make_message<MDentryUnlink>(dn->get_dir()->dirfrag(), dn->get_name());
     if (straydn) {

From 8f50048dd148aca0f06d88614cf990a884675690 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Thu, 1 Feb 2024 18:46:41 +0530
Subject: [PATCH 1730/2492] qa/cephfs: declare f-string to get values to
 substitute variable names

Fixes: https://tracker.ceph.com/issues/64289
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 qa/tasks/cephfs/caps_helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/caps_helper.py b/qa/tasks/cephfs/caps_helper.py
index 75a40ac83fe8..2b02ea71ea04 100644
--- a/qa/tasks/cephfs/caps_helper.py
+++ b/qa/tasks/cephfs/caps_helper.py
@@ -167,10 +167,10 @@ def run_mon_cap_tests(self, fs, client_id):
         fsnames = get_fsnames_from_moncap(moncap)
         if fsnames == []:
             log.info('no FS name is mentioned in moncap, client has '
-                     'permission to list all files. moncap -\n{moncap}')
+                     f'permission to list all files. moncap -\n{moncap}')
             return
 
-        log.info('FS names are mentioned in moncap. moncap -\n{moncap}')
+        log.info(f'FS names are mentioned in moncap. moncap -\n{moncap}')
         log.info('testing for presence of these FS names in output of '
                  '"fs ls" command run by client.')
         for fsname in fsnames:

From 0bab1e6b690ec4a12e8dc27229c435d64ca0d0be Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Thu, 1 Feb 2024 14:40:03 +0000
Subject: [PATCH 1731/2492] msg: update MOSDOp() to use ceph_tid_t instead of
 long

The MOSDOp constructor receives the the transaction ID as a long
instead of ceph_tid_t.

The issue is that "long" uses 32b on Windows instead of 64 bits,
so it flips after about 2 billion requests. At that point, the OSD
replies are dropped because of transaction ID mismatches.

We'll solve the issue by using the correct type for the transaction
id, specifically ceph_tid_t.

Fixes: https://tracker.ceph.com/issues/64291

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/messages/MOSDOp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 6edc590e51f1..61908d802cd7 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -178,7 +178,7 @@ class MOSDOp final : public MOSDFastDispatchOp {
       partial_decode_needed(true),
       final_decode_needed(true),
       bdata_encode(false) { }
-  MOSDOp(int inc, long tid, const hobject_t& ho, spg_t& _pgid,
+  MOSDOp(int inc, ceph_tid_t tid, const hobject_t& ho, spg_t& _pgid,
 	 epoch_t _osdmap_epoch,
 	 int _flags, uint64_t feat)
     : MOSDFastDispatchOp(CEPH_MSG_OSD_OP, HEAD_VERSION, COMPAT_VERSION),

From a23b424f74f848927a7fe77d827f4c12d6fb38c2 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 1 Feb 2024 15:55:59 +0000
Subject: [PATCH 1732/2492] rgw/rest: fix url decode of post params passed as
 attributes

this is fixing a regression with the SNS policies
cause by: 4bdc5d18dd68b95c6ccd4c0e77a1bd04ad86dbb8

the changes to the test code is to accomodate different
boto3 versions

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/rgw_rest_s3.cc                      |  3 ++-
 src/test/rgw/bucket_notification/test_bn.py | 16 +++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index b3d3891b0ea9..4433422e1263 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5169,7 +5169,8 @@ void update_attribute_map(const std::string& input, AttributeMap& map) {
   auto pos = key_or_value.find("=");
   if (pos != std::string::npos) {
     const auto key_or_value_lhs = key_or_value.substr(0, pos);
-    const auto key_or_value_rhs = url_decode(key_or_value.substr(pos + 1, key_or_value.size() - 1));
+    constexpr bool in_query = true; // replace '+' with ' '
+    const auto key_or_value_rhs = url_decode(key_or_value.substr(pos + 1, key_or_value.size() - 1), in_query);
     const auto map_it = map.find(idx);
     if (map_it == map.end()) {
       // new entry
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index d493a57e004e..6e9248a3669b 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -4429,8 +4429,13 @@ def test_ps_s3_topic_permissions():
         # 2nd user tries to set the attribute
         status = topic_conf2.set_attributes(attribute_name="persistent", attribute_val="false", topic_arn=topic_arn)
         assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
     except Exception as err:
-        print(err)
+        print('unexpected error type: '+type(err).__name__)
 
     # create bucket for conn2 and try publishing notification to topic
     _ = conn2.create_bucket(bucket_name)
@@ -4442,8 +4447,13 @@ def test_ps_s3_topic_permissions():
         s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
         _, status = s3_notification_conf2.set_config()
         assert False, "'AccessDenied' error is expected"
-    except ClientError as error:
-        assert_equal(error.response['Error']['Code'], 'AccessDenied')
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
 
     # Topic policy is now added by the 1st user to allow 2nd user.
     topic_policy  = topic_policy.replace("Deny", "Allow")

From e547768fedd02d631dc59cd32e9f026c1743fd31 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 30 Jan 2024 17:53:16 +0000
Subject: [PATCH 1733/2492] crimson: introduce crimson_seastar_cpu_cores

seastar::smp::count is deduced from the cpuset provided.

from seastar/src/core/reactor:
```
   if (smp_opts.smp) {
        smp::count = smp_opts.smp.get_value();
    } else {
        smp::count = cpu_set.size();
    }
```

Co-authored-by: Yingxin Cheng <yingxin.cheng@intel.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 qa/config/crimson_qa_overrides.yaml           |  1 -
 .../crimson-rados/basic/clusters/fixed-2.yaml |  9 +++++++
 .../crimson-rados/perf/clusters/fixed-2.yaml  |  9 +++++++
 .../crimson-rados/rbd/clusters/fixed-1.yaml   |  7 +++++
 .../singleton/all/osd-backfill.yaml           |  8 ++++++
 .../thrash/clusters/fixed-2.yaml              |  9 +++++++
 src/common/options/crimson.yaml.in            |  9 ++++---
 src/crimson/os/alienstore/alien_store.cc      | 16 ++++--------
 .../osd/main_config_bootstrap_helpers.cc      | 26 ++++++++++++-------
 src/crimson/osd/osd.cc                        |  2 +-
 src/vstart.sh                                 | 19 +++++++-------
 11 files changed, 78 insertions(+), 37 deletions(-)

diff --git a/qa/config/crimson_qa_overrides.yaml b/qa/config/crimson_qa_overrides.yaml
index 8720103dab62..fa8f49a49868 100644
--- a/qa/config/crimson_qa_overrides.yaml
+++ b/qa/config/crimson_qa_overrides.yaml
@@ -8,7 +8,6 @@ overrides:
       mon:
         osd pool default crimson: true
       osd:
-        crimson seastar smp: 3
         crimson osd obc lru size: 10
     flavor: crimson
   workunit:
diff --git a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
index 9774de6887bb..79641f695ab6 100644
--- a/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/basic/clusters/fixed-2.yaml
@@ -6,6 +6,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc
diff --git a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
index 3f322f371496..3409ab6d6345 100644
--- a/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/perf/clusters/fixed-2.yaml
@@ -10,6 +10,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc
diff --git a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
index d204f3eb2d0e..2bde4afcda5d 100644
--- a/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
+++ b/qa/suites/crimson-rados/rbd/clusters/fixed-1.yaml
@@ -10,6 +10,13 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-1
+      osd.1:
+        crimson seastar cpu cores: 2-3
+      osd.2:
+        crimson seastar cpu cores: 4-5
       global:
         ms cluster mode: crc
         ms service mode: crc
diff --git a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
index f475d5dc39b7..56e4b7d657b8 100644
--- a/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
+++ b/qa/suites/crimson-rados/singleton/all/osd-backfill.yaml
@@ -26,4 +26,12 @@ tasks:
     conf:
       osd:
         osd min pg log entries: 5
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-1
+      osd.1:
+        crimson seastar cpu cores: 2-3
+      osd.2:
+        crimson seastar cpu cores: 4-5
+      global:
 - osd_backfill:
diff --git a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
index 9774de6887bb..79641f695ab6 100644
--- a/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
+++ b/qa/suites/crimson-rados/thrash/clusters/fixed-2.yaml
@@ -6,6 +6,15 @@ overrides:
     conf:
       osd:
         osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
       global:
         ms cluster mode: crc
         ms service mode: crc
diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index d3c2c57a674c..c52c54d5250b 100644
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -19,17 +19,18 @@ options:
   default: 6
   flags:
   - startup
-- name: crimson_seastar_smp
-  type: uint
+- name: crimson_seastar_cpu_cores
+  type: str
   level: advanced
-  desc: Number of seastar reactor threads to use for the osd
-  default: 1
+  desc: CPU cores on which seastar reactor threads will run in cpuset(7) format, smp::count is deduced from this option
   flags:
   - startup
 - name: crimson_alien_thread_cpu_cores
   type: str
   level: advanced
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
+  flags:
+  - startup
 - name: seastore_segment_size
   type: size
   desc: Segment size to use for SegmentManager
diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 31dedbcf893f..32696775bb09 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -105,17 +105,11 @@ seastar::future<> AlienStore::start()
     get_conf<std::string>("crimson_alien_thread_cpu_cores"));
   //  crimson_alien_thread_cpu_cores are assigned to alien threads.
   if (!cpu_cores.has_value()) {
-    // no core isolation by default, cores [0, seastar::smp::count)
-    // will be shared between both alien and seastar reactor threads.
-    seastar::resource::cpuset cpuset;
-    std::copy(boost::counting_iterator<unsigned>(0),
-	      boost::counting_iterator<unsigned>(seastar::smp::count - 1),
-	      std::inserter(cpuset, cpuset.end()));
-    if (cpuset.empty()) {
-      logger().error("{}: unable to get nproc: {}", __func__, errno);
-    } else {
-      cpu_cores = cpuset;
-    }
+    // no core isolation by default, seastar_cpu_cores will be
+    // shared between both alien and seastar reactor threads.
+    cpu_cores = seastar::resource::parse_cpuset(
+      get_conf<std::string>("crimson_seastar_cpu_cores"));
+    ceph_assert(cpu_cores.has_value());
   }
   const auto num_threads =
     get_conf<uint64_t>("crimson_alien_op_num_threads");
diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc
index 807fd1591aed..cbb22ec0e6b0 100644
--- a/src/crimson/osd/main_config_bootstrap_helpers.cc
+++ b/src/crimson/osd/main_config_bootstrap_helpers.cc
@@ -148,17 +148,23 @@ _get_early_config(int argc, const char *argv[])
 	if (auto found = std::find_if(
 	      std::begin(early_args),
 	      std::end(early_args),
-	      [](auto* arg) { return "--smp"sv == arg; });
+	      [](auto* arg) { return "--cpuset"sv == arg; });
 	    found == std::end(early_args)) {
-
-	  // Set --smp based on crimson_seastar_smp config option
-	  ret.early_args.emplace_back("--smp");
-
-	  auto smp_config = local_conf().get_val<uint64_t>(
-	    "crimson_seastar_smp");
-
-	  ret.early_args.emplace_back(fmt::format("{}", smp_config));
-	  logger().info("get_early_config: set --smp {}", smp_config);
+	  auto smp_config = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
+	  if (!smp_config.empty()) {
+	    // Set --cpuset based on crimson_seastar_cpu_cores config option
+	    // --smp default is one per CPU
+	    ret.early_args.emplace_back("--cpuset");
+	    ret.early_args.emplace_back(smp_config);
+	    logger().info("get_early_config: set --cpuset {}", smp_config);
+	  } else {
+	    logger().warn("get_early_config: no cpuset specified, falling back"
+	                  " to seastar's default of: all");
+	  }
+	} else {
+	  logger().error("get_early_config: --cpuset can be "
+	                 "set only using crimson_seastar_cpu_cores");
+	  ceph_abort();
 	}
 	return 0;
       });
diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 063f9d86ea77..5c02b2895c2d 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -362,7 +362,7 @@ namespace {
 seastar::future<> OSD::start()
 {
   LOG_PREFIX(OSD::start);
-  INFO();
+  INFO("seastar::smp::count {}", seastar::smp::count);
 
   startup_time = ceph::mono_clock::now();
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
diff --git a/src/vstart.sh b/src/vstart.sh
index f9b2f3ce4cf2..2a0e2a615a13 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1110,18 +1110,12 @@ start_osd() {
     local osds_wait
     for osd in `seq $start $end`
     do
-	local extra_seastar_args
 	if [ "$ceph_osd" == "crimson-osd" ]; then
         bottom_cpu=$(( osd * crimson_smp ))
         top_cpu=$(( bottom_cpu + crimson_smp - 1 ))
-	    # set a single CPU nodes for each osd
-	    extra_seastar_args="--cpuset $bottom_cpu-$top_cpu"
-	    if [ "$debug" -ne 0 ]; then
-		extra_seastar_args+=" --debug"
-	    fi
-            if [ "$trace" -ne 0 ]; then
-                extra_seastar_args+=" --trace"
-            fi
+	    # set exclusive CPU nodes for each osd
+	    echo "$CEPH_BIN/ceph -c $conf_fn config set osd.$osd crimson_seastar_cpu_cores $bottom_cpu-$top_cpu"
+	    $CEPH_BIN/ceph -c $conf_fn config set "osd.$osd" crimson_seastar_cpu_cores "$bottom_cpu-$top_cpu"
 	fi
 	if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then
             wconf <<EOF
@@ -1665,7 +1659,12 @@ EOF
 fi
 
 if [ "$ceph_osd" == "crimson-osd" ]; then
-    $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp
+     if [ "$debug" -ne 0 ]; then
+        extra_seastar_args=" --debug"
+    fi
+    if [ "$trace" -ne 0 ]; then
+        extra_seastar_args=" --trace"
+    fi
     if [ "$(expr $(nproc) - 1)" -gt "$(($CEPH_NUM_OSD * crimson_smp))" ]; then
       echo "crimson_alien_thread_cpu_cores:" $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"
       $CEPH_BIN/ceph -c $conf_fn config set osd crimson_alien_thread_cpu_cores $(($CEPH_NUM_OSD * crimson_smp))-"$(expr $(nproc) - 1)"

From a809d48fc13de3d69c2bbb3245860a9c454ac199 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 19:59:10 -0800
Subject: [PATCH 1734/2492] src/crush: add a rule type enum

We're going to add more rule types that don't map directly onto
pool types.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc | 8 ++++----
 src/crush/crush.h          | 7 ++++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 5e51aad8dba4..f7df0c6670ce 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -363,10 +363,10 @@ int CrushCompiler::decompile(ostream &out)
     out << "\tid " << i << "\n";
 
     switch (crush.get_rule_type(i)) {
-    case CEPH_PG_TYPE_REPLICATED:
+    case CRUSH_RULE_TYPE_REPLICATED:
       out << "\ttype replicated\n";
       break;
-    case CEPH_PG_TYPE_ERASURE:
+    case CRUSH_RULE_TYPE_ERASURE:
       out << "\ttype erasure\n";
       break;
     default:
@@ -781,9 +781,9 @@ int CrushCompiler::parse_rule(iter_t const& i)
   string tname = string_node(i->children[start+2]);
   int type;
   if (tname == "replicated")
-    type = CEPH_PG_TYPE_REPLICATED;
+    type = CRUSH_RULE_TYPE_REPLICATED;
   else if (tname == "erasure")
-    type = CEPH_PG_TYPE_ERASURE;
+    type = CRUSH_RULE_TYPE_ERASURE;
   else 
     ceph_abort();
 
diff --git a/src/crush/crush.h b/src/crush/crush.h
index fde2df6a8a3e..263922615b63 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -87,7 +87,12 @@ struct crush_rule {
 #define crush_rule_size(len) (sizeof(struct crush_rule) + \
 			      (len)*sizeof(struct crush_rule_step))
 
-
+enum crush_rule_type {
+	CRUSH_RULE_TYPE_REPLICATED = 1,
+	CRUSH_RULE_TYPE_ERASURE = 3,
+	CRUSH_RULE_TYPE_MSR_FIRSTN = 4,
+	CRUSH_RULE_TYPE_MSR_INDEP = 5
+};
 
 /*
  * A bucket is a named container of other items (either devices or

From caa9e7a45edc74a7e8eaecb0f300db9c0e3b68bc Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:09:45 -0800
Subject: [PATCH 1735/2492] crush: add msr rule types and CrushWrapper
 infrastructure

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc  | 10 ++++++++++
 src/crush/CrushWrapper.cc   | 24 ++++++++++++++++++++++++
 src/crush/CrushWrapper.h    |  6 +++++-
 src/crush/grammar.h         |  2 +-
 src/include/ceph_features.h |  6 ++++--
 src/osd/OSDMap.cc           |  5 +++++
 6 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index f7df0c6670ce..740f003005b9 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -369,6 +369,12 @@ int CrushCompiler::decompile(ostream &out)
     case CRUSH_RULE_TYPE_ERASURE:
       out << "\ttype erasure\n";
       break;
+    case CRUSH_RULE_TYPE_MSR_FIRSTN:
+      out << "\ttype msr_firstn\n";
+      break;
+    case CRUSH_RULE_TYPE_MSR_INDEP:
+      out << "\ttype msr_indep\n";
+      break;
     default:
       out << "\ttype " << crush.get_rule_type(i) << "\n";
     }
@@ -784,6 +790,10 @@ int CrushCompiler::parse_rule(iter_t const& i)
     type = CRUSH_RULE_TYPE_REPLICATED;
   else if (tname == "erasure")
     type = CRUSH_RULE_TYPE_ERASURE;
+  else if (tname == "msr_firstn")
+    type = CRUSH_RULE_TYPE_MSR_FIRSTN;
+  else if (tname == "msr_indep")
+    type = CRUSH_RULE_TYPE_MSR_INDEP;
   else 
     ceph_abort();
 
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 0f40e6875e1b..1471056f1fef 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -135,6 +135,29 @@ bool CrushWrapper::is_v5_rule(unsigned ruleid) const
   return false;
 }
 
+bool CrushWrapper::has_msr_rules() const
+{
+  for (unsigned i=0; i<crush->max_rules; i++) {
+    if (is_msr_rule(i)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool CrushWrapper::is_msr_rule(unsigned ruleid) const
+{
+  if (ruleid >= crush->max_rules)
+    return false;
+
+  crush_rule *r = crush->rules[ruleid];
+  if (!r)
+    return false;
+
+  return r->type == CRUSH_RULE_TYPE_MSR_INDEP ||
+    r->type == CRUSH_RULE_TYPE_MSR_FIRSTN;
+}
+
 bool CrushWrapper::has_choose_args() const
 {
   return !choose_args.empty();
@@ -3515,6 +3538,7 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("has_v4_buckets", (int)has_v4_buckets());
   f->dump_int("require_feature_tunables5", (int)has_nondefault_tunables5());
   f->dump_int("has_v5_rules", (int)has_v5_rules());
+  f->dump_int("has_msr_rules", (int)has_msr_rules());
 }
 
 void CrushWrapper::dump_choose_args(Formatter *f) const
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index b8caa24ce621..e84a829a2d3d 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -329,13 +329,17 @@ class CrushWrapper {
   bool has_v5_rules() const;
   bool has_choose_args() const;          // any choose_args
   bool has_incompat_choose_args() const; // choose_args that can't be made compat
+  bool has_msr_rules() const;
 
   bool is_v2_rule(unsigned ruleid) const;
   bool is_v3_rule(unsigned ruleid) const;
   bool is_v5_rule(unsigned ruleid) const;
+  bool is_msr_rule(unsigned ruleid) const;
 
   std::string get_min_required_version() const {
-    if (has_v5_rules() || has_nondefault_tunables5())
+    if (has_msr_rules())
+      return "squid";
+    else if (has_v5_rules() || has_nondefault_tunables5())
       return "jewel";
     else if (has_v4_buckets())
       return "hammer";
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index b5c356a49e17..072b36b073aa 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -170,7 +170,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_emit );
       crushrule = str_p("rule") >> !name >> '{'
 				>> (str_p("id") | str_p("ruleset")) >> posint
-				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") )
+				>> str_p("type") >> ( str_p("replicated") | str_p("erasure") | str_p("msr_firstn") | str_p("msr_indep") )
 				>> !(str_p("min_size") >> posint)
 				>> !(str_p("max_size") >> posint)
 			   >> +step
diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 1937eeb4c698..0ff0c7ff1089 100644
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -137,7 +137,7 @@ DEFINE_CEPH_FEATURE(34, 3, RANGE_BLOCKLIST)
 DEFINE_CEPH_FEATURE(35, 1, OSD_CACHEPOOL)    // 3.14
 DEFINE_CEPH_FEATURE(36, 1, CRUSH_V2)         // 3.14
 DEFINE_CEPH_FEATURE(37, 1, EXPORT_PEER)      // 3.14
-DEFINE_CEPH_FEATURE_RETIRED(38, 1, OSD_ERASURE_CODES, MIMIC, OCTOPUS)
+DEFINE_CEPH_FEATURE(38, 2, CRUSH_MSR)        // X.XX kernel version once in a release
 // available
 DEFINE_CEPH_FEATURE(39, 1, OSDMAP_ENC)       // 3.15
 DEFINE_CEPH_FEATURE(40, 1, MDS_INLINE_DATA)  // 3.19
@@ -218,6 +218,7 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_OSD_CACHEPOOL |	    \
 	 CEPH_FEATURE_CRUSH_V2 |	    \
 	 CEPH_FEATURE_EXPORT_PEER |	    \
+	 CEPH_FEATURE_CRUSH_MSR |	    \
 	 CEPH_FEATURE_OSDMAP_ENC |          \
 	 CEPH_FEATURE_MDS_INLINE_DATA |	    \
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |	    \
@@ -265,9 +266,10 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_CRUSH_TUNABLES2 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES3 |		\
 	 CEPH_FEATURE_CRUSH_TUNABLES5 |		\
+	 CEPH_FEATURE_CRUSH_MSR |		\
 	 CEPH_FEATURE_CRUSH_V2 |		\
 	 CEPH_FEATURE_CRUSH_V4 |		\
-	 CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS)
+	 CEPH_FEATUREMASK_CRUSH_MSR)
 
 /*
  * make sure we don't try to use the reserved features
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 5773695b77ad..e88b9b6bfea8 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1789,6 +1789,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
 	features |= CEPH_FEATURE_CRUSH_TUNABLES3;
       if (crush->is_v5_rule(ruleid))
 	features |= CEPH_FEATURE_CRUSH_TUNABLES5;
+      if (crush->is_msr_rule(ruleid))
+	features |= CEPH_FEATURE_CRUSH_MSR;
     }
   }
   mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;
@@ -1843,6 +1845,9 @@ ceph_release_t OSDMap::get_min_compat_client() const
 {
   uint64_t f = get_features(CEPH_ENTITY_TYPE_CLIENT, nullptr);
 
+  if (HAVE_FEATURE(f, CRUSH_MSR)) {
+    return ceph_release_t::squid;        // v19.2.0
+  }
   if (HAVE_FEATURE(f, OSDMAP_PG_UPMAP) ||      // v12.0.0-1733-g27d6f43
       HAVE_FEATURE(f, CRUSH_CHOOSE_ARGS)) {    // v12.0.1-2172-gef1ef28
     return ceph_release_t::luminous;  // v12.2.0

From cbfff015e4242b891e64d929a8cbc173c2cd248a Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:33:54 -0800
Subject: [PATCH 1736/2492] osd/OSDMap: minor formatting nit

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/osd/OSDMap.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index e88b9b6bfea8..8abb35dc1cbf 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1764,9 +1764,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_V4;
   if (crush->has_nondefault_tunables5())
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
-  if (crush->has_incompat_choose_args()) {
+  if (crush->has_incompat_choose_args())
     features |= CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS;
-  }
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty() || !pg_upmap_primaries.empty())

From 3a2517f2a4a6c67f5c615b981700c8ee703a543c Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:20:07 -0800
Subject: [PATCH 1737/2492] crush: add msr tunables

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushCompiler.cc | 50 +++++++++++++++++++++++++++++++++++
 src/crush/CrushWrapper.cc  | 25 ++++++++++++++++++
 src/crush/CrushWrapper.h   | 53 +++++++++++++++++++++++++++++++++-----
 src/crush/crush.h          | 11 +++++++-
 src/crush/grammar.h        |  8 ++++++
 src/osd/OSDMap.cc          |  2 ++
 6 files changed, 142 insertions(+), 7 deletions(-)

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 740f003005b9..bc1d82be79b1 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -321,6 +321,13 @@ int CrushCompiler::decompile(ostream &out)
   if (crush.get_allowed_bucket_algs() != CRUSH_LEGACY_ALLOWED_BUCKET_ALGS)
     out << "tunable allowed_bucket_algs " << crush.get_allowed_bucket_algs()
 	<< "\n";
+  if (crush.has_nondefault_tunables_msr()) {
+    out << "tunable msr_descents " << crush.get_msr_descents()
+	<< "\n";
+    out << "tunable msr_collision_tries "
+	<< crush.get_msr_collision_tries()
+	<< "\n";
+  }
 
   out << "\n# devices\n";
   for (int i=0; i<crush.get_max_devices(); i++) {
@@ -428,6 +435,15 @@ int CrushCompiler::decompile(ostream &out)
 	out << "\tstep set_chooseleaf_stable " << crush.get_rule_arg1(i, j)
 	    << "\n";
 	break;
+      case CRUSH_RULE_SET_MSR_DESCENTS:
+	out << "\tstep set_msr_descents " << crush.get_rule_arg1(i, j)
+	    << "\n";
+	break;
+      case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+	out << "\tstep set_msr_collision_tries "
+	    << crush.get_rule_arg1(i, j)
+	    << "\n";
+	break;
       case CRUSH_RULE_CHOOSE_FIRSTN:
 	out << "\tstep choose firstn "
 	    << crush.get_rule_arg1(i, j) 
@@ -456,6 +472,13 @@ int CrushCompiler::decompile(ostream &out)
 	print_type_name(out, crush.get_rule_arg2(i, j), crush);
 	out << "\n";
 	break;
+      case CRUSH_RULE_CHOOSE_MSR:
+	out << "\tstep choosemsr "
+	    << crush.get_rule_arg1(i, j)
+	    << " type ";
+	print_type_name(out, crush.get_rule_arg2(i, j), crush);
+	out << "\n";
+	break;
       }
     }
     out << "}\n";
@@ -538,6 +561,10 @@ int CrushCompiler::parse_tunable(iter_t const& i)
     crush.set_straw_calc_version(val);
   else if (name == "allowed_bucket_algs")
     crush.set_allowed_bucket_algs(val);
+  else if (name == "msr_descents")
+    crush.set_msr_descents(val);
+  else if (name == "msr_collision_tries")
+    crush.set_msr_collision_tries(val);
   else {
     err << "tunable " << name << " not recognized" << std::endl;
     return -1;
@@ -915,6 +942,18 @@ int CrushCompiler::parse_rule(iter_t const& i)
 	crush.set_rule_step_set_chooseleaf_stable(ruleno, step++, val);
       }
       break;
+    case crush_grammar::_step_set_msr_descents:
+      {
+	int val = int_node(s->children[1]);
+	crush.set_rule_step_set_msr_descents(ruleno, step++, val);
+      }
+      break;
+    case crush_grammar::_step_set_msr_collision_tries:
+      {
+	int val = int_node(s->children[1]);
+	crush.set_rule_step_set_msr_collision_tries(ruleno, step++, val);
+      }
+      break;
 
     case crush_grammar::_step_choose:
     case crush_grammar::_step_chooseleaf:
@@ -942,6 +981,17 @@ int CrushCompiler::parse_rule(iter_t const& i)
       }
       break;
 
+    case crush_grammar::_step_choose_msr:
+      {
+	string type = string_node(s->children[3]);
+	if (!type_id.count(type)) {
+	  err << "in rule '" << rname << "' type '" << type << "' not defined" << std::endl;
+	  return -1;
+	}
+	crush.set_rule_step_choose_msr(ruleno, step++, int_node(s->children[1]), type_id[type]);
+      }
+      break;
+
     case crush_grammar::_step_emit:
       crush.set_rule_step_emit(ruleno, step++);
       break;
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 1471056f1fef..573e22b6e56a 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -3103,6 +3103,10 @@ void CrushWrapper::encode(bufferlist& bl, uint64_t features) const
       }
     }
   }
+  if (HAVE_FEATURE(features, CRUSH_MSR)) {
+    encode(crush->msr_descents, bl);
+    encode(crush->msr_collision_tries, bl);
+  }
 }
 
 static void decode_32_or_64_string_map(map<int32_t,string>& m, bufferlist::const_iterator& blp)
@@ -3253,6 +3257,12 @@ void CrushWrapper::decode(bufferlist::const_iterator& blp)
 	choose_args[choose_args_index] = arg_map;
       }
     }
+    if (!blp.end()) {
+      decode(crush->msr_descents, blp);
+      decode(crush->msr_collision_tries, blp);
+    } else {
+      set_default_msr_tunables();
+    }
     update_choose_args(nullptr); // in case we decode a legacy "corrupted" map
     finalize();
   }
@@ -3508,6 +3518,8 @@ void CrushWrapper::dump_tunables(Formatter *f) const
   f->dump_int("chooseleaf_descend_once", get_chooseleaf_descend_once());
   f->dump_int("chooseleaf_vary_r", get_chooseleaf_vary_r());
   f->dump_int("chooseleaf_stable", get_chooseleaf_stable());
+  f->dump_int("msr_descents", get_msr_descents());
+  f->dump_int("msr_collision_tries", get_msr_collision_tries());
   f->dump_int("straw_calc_version", get_straw_calc_version());
   f->dump_int("allowed_bucket_algs", get_allowed_bucket_algs());
 
@@ -3637,6 +3649,11 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_int("num", get_rule_arg1(rule_id, j));
       f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
       break;
+    case CRUSH_RULE_CHOOSE_MSR:
+      f->dump_string("op", "choosemsr");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      f->dump_string("type", get_type_name(get_rule_arg2(rule_id, j)));
+      break;
     case CRUSH_RULE_SET_CHOOSE_TRIES:
       f->dump_string("op", "set_choose_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
@@ -3645,6 +3662,14 @@ void CrushWrapper::dump_rule(int rule_id, Formatter *f) const
       f->dump_string("op", "set_chooseleaf_tries");
       f->dump_int("num", get_rule_arg1(rule_id, j));
       break;
+    case CRUSH_RULE_SET_MSR_DESCENTS:
+      f->dump_string("op", "set_msr_descents");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      break;
+    case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+      f->dump_string("op", "set_msr_collision_tries");
+      f->dump_int("num", get_rule_arg1(rule_id, j));
+      break;
     default:
       f->dump_int("opcode", get_rule_op(rule_id, j));
       f->dump_int("arg1", get_rule_arg1(rule_id, j));
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index e84a829a2d3d..4425103ff6f2 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -125,6 +125,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_bobtail() {
     crush->choose_local_tries = 0;
@@ -134,6 +135,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 0;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_firefly() {
     crush->choose_local_tries = 0;
@@ -143,6 +145,7 @@ class CrushWrapper {
     crush->chooseleaf_vary_r = 1;
     crush->chooseleaf_stable = 0;
     crush->allowed_bucket_algs = CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+    set_default_msr_tunables();
   }
   void set_tunables_hammer() {
     crush->choose_local_tries = 0;
@@ -156,6 +159,7 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
+    set_default_msr_tunables();
   }
   void set_tunables_jewel() {
     crush->choose_local_tries = 0;
@@ -169,6 +173,7 @@ class CrushWrapper {
       (1 << CRUSH_BUCKET_LIST) |
       (1 << CRUSH_BUCKET_STRAW) |
       (1 << CRUSH_BUCKET_STRAW2);
+    set_default_msr_tunables();
   }
 
   void set_tunables_legacy() {
@@ -233,6 +238,24 @@ class CrushWrapper {
     crush->straw_calc_version = n;
   }
 
+  unsigned get_msr_descents() const {
+    return crush->msr_descents;
+  }
+  void set_msr_descents(unsigned n) {
+    crush->msr_descents = n;
+  }
+
+  unsigned get_msr_collision_tries() const {
+    return crush->msr_collision_tries;
+  }
+  void set_msr_collision_tries(unsigned n) {
+    crush->msr_collision_tries = n;
+  }
+  void set_default_msr_tunables() {
+    set_msr_descents(100);
+    set_msr_collision_tries(100);
+  }
+
   unsigned get_allowed_bucket_algs() const {
     return crush->allowed_bucket_algs;
   }
@@ -248,7 +271,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 0 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_bobtail_tunables() const {
     return
@@ -258,7 +282,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 0 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_firefly_tunables() const {
     return
@@ -268,7 +293,8 @@ class CrushWrapper {
       crush->chooseleaf_descend_once == 1 &&
       crush->chooseleaf_vary_r == 1 &&
       crush->chooseleaf_stable == 0 &&
-      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS;
+      crush->allowed_bucket_algs == CRUSH_LEGACY_ALLOWED_BUCKET_ALGS &&
+      !has_nondefault_tunables_msr();
   }
   bool has_hammer_tunables() const {
     return
@@ -281,7 +307,8 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2));
+				      (1 << CRUSH_BUCKET_STRAW2)) &&
+      !has_nondefault_tunables_msr();
   }
   bool has_jewel_tunables() const {
     return
@@ -294,7 +321,8 @@ class CrushWrapper {
       crush->allowed_bucket_algs == ((1 << CRUSH_BUCKET_UNIFORM) |
 				      (1 << CRUSH_BUCKET_LIST) |
 				      (1 << CRUSH_BUCKET_STRAW) |
-				      (1 << CRUSH_BUCKET_STRAW2));
+				      (1 << CRUSH_BUCKET_STRAW2)) &&
+      !has_nondefault_tunables_msr();
   }
 
   bool has_optimal_tunables() const {
@@ -322,6 +350,11 @@ class CrushWrapper {
     return
         crush->chooseleaf_stable != 0;
   }
+  bool has_nondefault_tunables_msr() const {
+    return
+      crush->msr_descents != 100 ||
+      crush->msr_collision_tries != 100;
+  }
 
   bool has_v2_rules() const;
   bool has_v3_rules() const;
@@ -337,7 +370,7 @@ class CrushWrapper {
   bool is_msr_rule(unsigned ruleid) const;
 
   std::string get_min_required_version() const {
-    if (has_msr_rules())
+    if (has_msr_rules() || has_nondefault_tunables_msr())
       return "squid";
     else if (has_v5_rules() || has_nondefault_tunables5())
       return "jewel";
@@ -1155,6 +1188,14 @@ class CrushWrapper {
   int set_rule_step_set_chooseleaf_stable(unsigned ruleno, unsigned step, int val) {
     return set_rule_step(ruleno, step, CRUSH_RULE_SET_CHOOSELEAF_STABLE, val, 0);
   }
+
+  int set_rule_step_set_msr_descents(unsigned ruleno, unsigned step, int val) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_DESCENTS, val, 0);
+  }
+  int set_rule_step_set_msr_collision_tries(unsigned ruleno, unsigned step, int val) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_SET_MSR_COLLISION_TRIES, val, 0);
+  }
+
   int set_rule_step_choose_firstn(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_FIRSTN, val, type);
   }
diff --git a/src/crush/crush.h b/src/crush/crush.h
index 263922615b63..a2ec2e1de69f 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -65,7 +65,10 @@ enum crush_opcodes {
 	CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES = 10,
 	CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES = 11,
 	CRUSH_RULE_SET_CHOOSELEAF_VARY_R = 12,
-	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13
+	CRUSH_RULE_SET_CHOOSELEAF_STABLE = 13,
+
+	CRUSH_RULE_SET_MSR_DESCENTS = 14,
+	CRUSH_RULE_SET_MSR_COLLISION_TRIES = 15,
 };
 
 /*
@@ -415,6 +418,12 @@ struct crush_map {
          */
 	__u8 chooseleaf_stable;
 
+	/*! Sets total descents for MSR rules */
+	__u32 msr_descents;
+
+	/*! Sets local collision retries for MSR rules */
+	__u32 msr_collision_tries;
+
         /*! @cond INTERNAL */
 	/* This value is calculated after decode or construction by
 	   the builder. It is exposed here (rather than having a
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 072b36b073aa..9e955091c6a8 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -50,6 +50,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     _step_set_choose_tries,
     _step_set_choose_local_tries,
     _step_set_choose_local_fallback_tries,
+    _step_set_msr_descents,
+    _step_set_msr_collision_tries,
     _step_choose,
     _step_chooseleaf,
     _step_emit,
@@ -91,6 +93,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_tries> >    step_set_chooseleaf_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_vary_r> >    step_set_chooseleaf_vary_r;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_chooseleaf_stable> >    step_set_chooseleaf_stable;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_descents> >    step_set_msr_descents;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_collision_tries> >    step_set_msr_collision_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose> >    step_choose;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_chooseleaf> >      step_chooseleaf;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_emit> >      step_emit;
@@ -149,6 +153,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
       step_set_chooseleaf_tries = str_p("set_chooseleaf_tries") >> posint;
       step_set_chooseleaf_vary_r = str_p("set_chooseleaf_vary_r") >> posint;
       step_set_chooseleaf_stable = str_p("set_chooseleaf_stable") >> posint;
+      step_set_msr_descents = str_p("set_msr_descents") >> posint;
+      step_set_msr_collision_tries = str_p("set_msr_collision_tries") >> posint;
       step_choose = str_p("choose")
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
@@ -165,6 +171,8 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_set_chooseleaf_tries |
 				step_set_chooseleaf_vary_r |
 				step_set_chooseleaf_stable |
+				step_set_msr_descents |
+				step_set_msr_collision_tries |
 				step_choose |
 				step_chooseleaf |
 				step_emit );
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 8abb35dc1cbf..a6dd32c554d3 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1766,6 +1766,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
     features |= CEPH_FEATURE_CRUSH_TUNABLES5;
   if (crush->has_incompat_choose_args())
     features |= CEPH_FEATUREMASK_CRUSH_CHOOSE_ARGS;
+  if (crush->has_nondefault_tunables_msr())
+    features |= CEPH_FEATURE_CRUSH_MSR;
   mask |= CEPH_FEATURES_CRUSH;
 
   if (!pg_upmap.empty() || !pg_upmap_items.empty() || !pg_upmap_primaries.empty())

From 9db2e6b24cd24af8cce5625b1cd7727c9cb320c4 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 18 Dec 2023 12:58:15 -0800
Subject: [PATCH 1738/2492] crush: add choosemsr rule

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.h | 3 +++
 src/crush/crush.h        | 3 +++
 src/crush/grammar.h      | 6 ++++++
 3 files changed, 12 insertions(+)

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 4425103ff6f2..f71f6d6ea79b 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -1208,6 +1208,9 @@ class CrushWrapper {
   int set_rule_step_choose_leaf_indep(unsigned ruleno, unsigned step, int val, int type) {
     return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSELEAF_INDEP, val, type);
   }
+  int set_rule_step_choose_msr(unsigned ruleno, unsigned step, int val, int type) {
+    return set_rule_step(ruleno, step, CRUSH_RULE_CHOOSE_MSR, val, type);
+  }
   int set_rule_step_emit(unsigned ruleno, unsigned step) {
     return set_rule_step(ruleno, step, CRUSH_RULE_EMIT, 0, 0);
   }
diff --git a/src/crush/crush.h b/src/crush/crush.h
index a2ec2e1de69f..7c6ce7fdf453 100644
--- a/src/crush/crush.h
+++ b/src/crush/crush.h
@@ -69,6 +69,9 @@ enum crush_opcodes {
 
 	CRUSH_RULE_SET_MSR_DESCENTS = 14,
 	CRUSH_RULE_SET_MSR_COLLISION_TRIES = 15,
+
+	/* choose variant without FIRSTN|INDEP */
+	CRUSH_RULE_CHOOSE_MSR = 16
 };
 
 /*
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 9e955091c6a8..0c9a2da7d770 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -54,6 +54,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     _step_set_msr_collision_tries,
     _step_choose,
     _step_chooseleaf,
+    _step_choose_msr,
     _step_emit,
     _step,
     _crushrule,
@@ -97,6 +98,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_set_msr_collision_tries> >    step_set_msr_collision_tries;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose> >    step_choose;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_chooseleaf> >      step_chooseleaf;
+    boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_choose_msr> >      step_choose_msr;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step_emit> >      step_emit;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_step> >      step;
     boost::spirit::rule<ScannerT, boost::spirit::parser_context<>, boost::spirit::parser_tag<_crushrule> >      crushrule;
@@ -163,6 +165,9 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 	>> ( str_p("indep") | str_p("firstn") )
 	>> integer
 	>> str_p("type") >> name;
+      step_choose_msr = str_p("choosemsr")
+	>> integer
+	>> str_p("type") >> name;
       step_emit = str_p("emit");
       step = str_p("step") >> ( step_take |
 				step_set_choose_tries |
@@ -175,6 +180,7 @@ struct crush_grammar : public boost::spirit::grammar<crush_grammar>
 				step_set_msr_collision_tries |
 				step_choose |
 				step_chooseleaf |
+				step_choose_msr |
 				step_emit );
       crushrule = str_p("rule") >> !name >> '{'
 				>> (str_p("id") | str_p("ruleset")) >> posint

From ddac5d1cb85f182e5984929b667f4dc56ea97984 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 1 Feb 2024 13:18:38 +0800
Subject: [PATCH 1739/2492] crimson/osd/shard_services: make sure that only
 up/acting members can create pgs

Fixes: https://tracker.ceph.com/issues/64282
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/shard_services.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index ff35df280ad8..fffb7f8d3cdf 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -610,15 +610,23 @@ seastar::future<Ref<PG>> ShardServices::make_pg(
 
 seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
   std::unique_ptr<PGCreateInfo> info) {
-  LOG_PREFIX(OSDSingletonState::trim_maps);
   return seastar::do_with(
     std::move(info),
-    [FNAME, this](auto &info)
+    [this](auto &info)
     -> seastar::future<Ref<PG>> {
       return get_map(info->epoch).then(
-	[&info, FNAME, this](cached_map_t startmap)
+	[&info, this](cached_map_t startmap)
 	-> seastar::future<std::tuple<Ref<PG>, cached_map_t>> {
+	  LOG_PREFIX(ShardServices::handle_pg_create_info);
 	  const spg_t &pgid = info->pgid;
+	  if (!get_map()->is_up_acting_osd_shard(pgid, local_state.whoami)
+	      || !startmap->is_up_acting_osd_shard(pgid, local_state.whoami)) {
+	    DEBUG("ignore pgid {}, doesn't exist anymore, discarding");
+	    local_state.pg_map.pg_creation_canceled(pgid);
+	    return seastar::make_ready_future<
+	      std::tuple<Ref<PG>, OSDMapService::cached_map_t>
+	      >(std::make_tuple(Ref<PG>(), startmap));
+	  }
 	  if (info->by_mon) {
 	    int64_t pool_id = pgid.pgid.pool();
 	    const pg_pool_t *pool = get_map()->get_pg_pool(pool_id);

From e99486edc59b2df9e7c31504d0b4d45539a49f0b Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Wed, 17 Jan 2024 18:12:50 +0530
Subject: [PATCH 1740/2492] mgr/dashboard: allow sending back error status code
 fetching clients fails

Fixes: https://tracker.ceph.com/issues/64089
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
(cherry picked from commit 564dba38c49864e71958ddcd970856b834e9c6b7)
---
 .../mgr/dashboard/controllers/cephfs.py       | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 86f112522c8f..fca5ddc95ee1 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -105,10 +105,17 @@ def get(self, fs_id):
         return self.fs_status(fs_id)
 
     @RESTController.Resource('GET')
-    def clients(self, fs_id):
+    def clients(self, fs_id, **kwargs):
+        flag = kwargs.pop('suppress_client_ls_errors', 'True')
+        if flag not in ('True', 'False'):
+            raise DashboardException(msg='suppress_client_ls_errors value '
+                                         'needs to be either True or False '
+                                         f'but provided {flag}',
+                                     component='cephfs')
+
         fs_id = self.fs_id_to_int(fs_id)
 
-        return self._clients(fs_id)
+        return self._clients(fs_id, suppress_client_ls_errors=flag)
 
     @RESTController.Resource('DELETE', path='/client/{client_id}')
     def evict(self, fs_id, client_id):
@@ -351,17 +358,23 @@ def fs_status(self, fs_id):
             "versions": mds_versions
         }
 
-    def _clients(self, fs_id):
+    def _clients(self, fs_id, **kwargs):
+        suppress_get_errors = kwargs.pop('suppress_client_ls_errors', 'True')
         cephfs_clients = self.cephfs_clients.get(fs_id, None)
         if cephfs_clients is None:
             cephfs_clients = CephFSClients(mgr, fs_id)
             self.cephfs_clients[fs_id] = cephfs_clients
 
         try:
-            status, clients = cephfs_clients.get()
+            status, clients = cephfs_clients.get(suppress_get_errors)
         except AttributeError:
             raise cherrypy.HTTPError(404,
                                      "No cephfs with id {0}".format(fs_id))
+        except RuntimeError:
+            raise cherrypy.HTTPError(500,
+                                     f"Could not fetch client(s), maybe there "
+                                     f"is no active MDS on CephFS {fs_id} or "
+                                     "the FS is in failed state.")
 
         if clients is None:
             raise cherrypy.HTTPError(404,
@@ -609,11 +622,14 @@ def __init__(self, module_inst, fscid):
         self.fscid = fscid
 
     @ViewCache()
-    def get(self):
+    def get(self, suppress_errors='True'):
         try:
             ret = CephService.send_command('mds', 'session ls', srv_spec='{0}:0'.format(self.fscid))
         except RuntimeError:
-            ret = []
+            if suppress_errors == 'True':
+                ret = []
+            else:
+                raise
         return ret
 
 

From d45bb4f94905d3061cefc4d1f20b8a35dc6e1493 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Fri, 2 Feb 2024 13:44:44 +0100
Subject: [PATCH 1741/2492] mgr/rook: removing all the code related to OSDs
 creation/removal Fixes: https://tracker.ceph.com/issues/64211

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py       | 150 +-----------
 src/pybind/mgr/rook/rook_cluster.py | 358 +---------------------------
 2 files changed, 2 insertions(+), 506 deletions(-)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index ca2d168db636..91099710c4b3 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -82,12 +82,6 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
             default='local',
             desc='storage class name for LSO-discovered PVs',
         ),
-        Option(
-            'drive_group_interval',
-            type='float',
-            default=300.0,
-            desc='interval in seconds between re-application of applied drive_groups',
-        ),
     ]
 
     @staticmethod
@@ -126,9 +120,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self.config_notify()
         if TYPE_CHECKING:
             self.storage_class = 'foo'
-            self.drive_group_interval = 10.0
 
-        self._load_drive_groups()
         self._shutdown = threading.Event()
 
     def config_notify(self) -> None:
@@ -144,7 +136,6 @@ def config_notify(self) -> None:
             self.log.debug(' mgr option %s = %s',
                            opt['name'], getattr(self, opt['name']))  # type: ignore
         assert isinstance(self.storage_class, str)
-        assert isinstance(self.drive_group_interval, float)
 
         if self._rook_cluster:
             self._rook_cluster.storage_class_name = self.storage_class
@@ -211,10 +202,6 @@ def serve(self) -> None:
         self._initialized.set()
         self.config_notify()
 
-        while not self._shutdown.is_set():
-            self._apply_drivegroups(list(self._drive_group_map.values()))
-            self._shutdown.wait(self.drive_group_interval)
-
     @handle_orch_error
     def get_inventory(self, host_filter: Optional[orchestrator.InventoryFilter] = None, refresh: bool = False) -> List[orchestrator.InventoryHost]:
         host_list = None
@@ -415,15 +402,6 @@ def sum_running_pods(service_type: str, service_name: Optional[str] = None) -> i
                 running=sum_running_pods('osd')
             )
 
-            # drivegroups
-            for name, dg in self._drive_group_map.items():
-                spec[f'osd.{name}'] = orchestrator.ServiceDescription(
-                    spec=dg,
-                    last_refresh=now,
-                    size=0,
-                    running=0,
-                )
-
         if service_type == 'rbd-mirror' or service_type is None:
             # rbd-mirrors
             all_mirrors = self.rook_cluster.get_resource("cephrbdmirrors")
@@ -576,9 +554,6 @@ def remove_service(self, service_name: str, force: bool = False) -> str:
         elif service_type == 'rbd-mirror':
             return self.rook_cluster.rm_service('cephrbdmirrors', service_id)
         elif service_type == 'osd':
-            if service_id in self._drive_group_map:
-                del self._drive_group_map[service_id]
-                self._save_drive_groups()
             return f'Removed {service_name}'
         elif service_type == 'ingress':
             self.log.info("{0} service '{1}' does not exist".format('ingress', service_id))
@@ -634,134 +609,11 @@ def apply_nfs(self, spec):
     def remove_daemons(self, names: List[str]) -> List[str]:
         return self.rook_cluster.remove_pods(names)
 
-    def apply_drivegroups(self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]:
-        for drive_group in specs:
-            self._drive_group_map[str(drive_group.service_id)] = drive_group
-        self._save_drive_groups()
-        return OrchResult(self._apply_drivegroups(specs))
-
-    def _apply_drivegroups(self, ls: List[DriveGroupSpec]) -> List[str]:
-        all_hosts = raise_if_exception(self.get_hosts())
-        result_list: List[str] = []
-        for drive_group in ls:
-            matching_hosts = drive_group.placement.filter_matching_hosts(
-                lambda label=None, as_hostspec=None: all_hosts
-            )
-
-            if not self.rook_cluster.node_exists(matching_hosts[0]):
-                raise RuntimeError("Node '{0}' is not in the Kubernetes "
-                               "cluster".format(matching_hosts))
-
-            # Validate whether cluster CRD can accept individual OSD
-            # creations (i.e. not useAllDevices)
-            if not self.rook_cluster.can_create_osd():
-                raise RuntimeError("Rook cluster configuration does not "
-                                "support OSD creation.")
-            result_list.append(self.rook_cluster.add_osds(drive_group, matching_hosts))
-        return result_list
-
-    def _load_drive_groups(self) -> None:
-        stored_drive_group = self.get_store("drive_group_map")
-        self._drive_group_map: Dict[str, DriveGroupSpec] = {}
-        if stored_drive_group:
-            for name, dg in json.loads(stored_drive_group).items():
-                try:
-                    self._drive_group_map[name] = DriveGroupSpec.from_json(dg)
-                except ValueError as e:
-                    self.log.error(f'Failed to load drive group {name} ({dg}): {e}')
-
-    def _save_drive_groups(self) -> None:
-        json_drive_group_map = {
-            name: dg.to_json() for name, dg in self._drive_group_map.items()
-        }
-        self.set_store("drive_group_map", json.dumps(json_drive_group_map))
-
-    def remove_osds(self,
-                    osd_ids: List[str],
-                    replace: bool = False,
-                    force: bool = False,
-                    zap: bool = False,
-                    no_destroy: bool = False) -> OrchResult[str]:
-        assert self._rook_cluster is not None
-        if zap:
-            raise RuntimeError("Rook does not support zapping devices during OSD removal.")
-        res = self._rook_cluster.remove_osds(osd_ids, replace, force, self.mon_command)
-        return OrchResult(res)
-
     def add_host_label(self, host: str, label: str) -> OrchResult[str]:
         return self.rook_cluster.add_host_label(host, label)
-    
+
     def remove_host_label(self, host: str, label: str, force: bool = False) -> OrchResult[str]:
         return self.rook_cluster.remove_host_label(host, label)
-    """
-    @handle_orch_error
-    def create_osds(self, drive_group):
-        # type: (DriveGroupSpec) -> str
-        # Creates OSDs from a drive group specification.
-
-        # $: ceph orch osd create -i <dg.file>
-
-        # The drivegroup file must only contain one spec at a time.
-        # 
-
-        targets = []  # type: List[str]
-        if drive_group.data_devices and drive_group.data_devices.paths:
-            targets += [d.path for d in drive_group.data_devices.paths]
-        if drive_group.data_directories:
-            targets += drive_group.data_directories
-
-        all_hosts = raise_if_exception(self.get_hosts())
-
-        matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts)
-
-        assert len(matching_hosts) == 1
-
-        if not self.rook_cluster.node_exists(matching_hosts[0]):
-            raise RuntimeError("Node '{0}' is not in the Kubernetes "
-                               "cluster".format(matching_hosts))
-
-        # Validate whether cluster CRD can accept individual OSD
-        # creations (i.e. not useAllDevices)
-        if not self.rook_cluster.can_create_osd():
-            raise RuntimeError("Rook cluster configuration does not "
-                               "support OSD creation.")
-
-        return self.rook_cluster.add_osds(drive_group, matching_hosts)
-
-        # TODO: this was the code to update the progress reference:
-        
-        @handle_orch_error
-        def has_osds(matching_hosts: List[str]) -> bool:
-
-            # Find OSD pods on this host
-            pod_osd_ids = set()
-            pods = self.k8s.list_namespaced_pod(self._rook_env.namespace,
-                                                label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name),
-                                                field_selector="spec.nodeName={0}".format(
-                                                    matching_hosts[0]
-                                                )).items
-            for p in pods:
-                pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id']))
-
-            self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids))
-
-            found = []
-            osdmap = self.get("osd_map")
-            for osd in osdmap['osds']:
-                osd_id = osd['osd']
-                if osd_id not in pod_osd_ids:
-                    continue
-
-                metadata = self.get_metadata('osd', "%s" % osd_id)
-                if metadata and metadata['devices'] in targets:
-                    found.append(osd_id)
-                else:
-                    self.log.info("ignoring osd {0} {1}".format(
-                        osd_id, metadata['devices'] if metadata else 'DNE'
-                    ))
-
-            return found is not None        
-    """
 
     @handle_orch_error
     def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
index ff8af5f38df4..16d498a70e30 100644
--- a/src/pybind/mgr/rook/rook_cluster.py
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -383,324 +383,6 @@ def get_item_name(self, item: Any) -> Any:
                         "{} doesn't contain a metadata.name. Unable to track changes".format(
                             self.api_func))
 
-class DefaultCreator():
-    def __init__(self, inventory: 'Dict[str, List[Device]]', coreV1_api: 'client.CoreV1Api', storage_class_name: 'str'):
-        self.coreV1_api = coreV1_api
-        self.storage_class_name = storage_class_name
-        self.inventory = inventory
-
-    def device_to_device_set(self, drive_group: DriveGroupSpec, d: Device) -> ccl.StorageClassDeviceSetsItem:
-        device_set = ccl.StorageClassDeviceSetsItem(
-                    name=d.sys_api['pv_name'],
-                    volumeClaimTemplates= ccl.VolumeClaimTemplatesList(),
-                    count=1,
-                    encrypted=drive_group.encrypted,
-                    portable=False
-                )
-        device_set.volumeClaimTemplates.append(
-            ccl.VolumeClaimTemplatesItem(
-                metadata=ccl.Metadata(
-                    name="data"
-                ),
-                spec=ccl.Spec(
-                    storageClassName=self.storage_class_name,
-                    volumeMode="Block",
-                    accessModes=ccl.CrdObjectList(["ReadWriteOnce"]),
-                    resources={
-                        "requests":{
-                                "storage": 1
-                        }
-                    },
-                    volumeName=d.sys_api['pv_name']
-                )
-            )
-        )
-        return device_set
-
-    def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupSpec, matching_hosts: List[str]) -> List[Device]:
-        device_list = []
-        assert drive_group.data_devices is not None
-        sizematcher: Matcher = AllMatcher('', None)
-        if drive_group.data_devices.size:
-            sizematcher = SizeMatcher('size', drive_group.data_devices.size)
-        limit = getattr(drive_group.data_devices, 'limit', None)
-        count = 0
-        _all = getattr(drive_group.data_devices, 'all', None)
-        paths = [device.path for device in drive_group.data_devices.paths]
-        osd_list = []
-        for pod in rook_pods.items:
-            if (
-                hasattr(pod, 'metadata') 
-                and hasattr(pod.metadata, 'labels') 
-                and 'osd' in pod.metadata.labels 
-                and 'ceph.rook.io/DeviceSet' in pod.metadata.labels
-            ):
-                osd_list.append(pod.metadata.labels['ceph.rook.io/DeviceSet'])
-        for _, node in self.inventory.items():
-            for device in node:
-                if device.sys_api['pv_name'] in osd_list:
-                    count += 1
-        for _, node in self.inventory.items():
-            for device in node:
-                if not limit or (count < limit):
-                    if device.available:
-                        if (
-                            _all
-                            or (
-                                device.sys_api['node'] in matching_hosts
-                                and sizematcher.compare(device)
-                                and (
-                                    not drive_group.data_devices.paths
-                                    or (device.path in paths)
-                                )
-                            )
-                        ):
-                            device_list.append(device)
-                            count += 1
-        
-        return device_list
-
-    def add_osds(self, rook_pods: KubernetesResource, drive_group: DriveGroupSpec, matching_hosts: List[str]) -> Any:
-        to_create = self.filter_devices(rook_pods, drive_group,matching_hosts)
-        assert drive_group.data_devices is not None
-        def _add_osds(current_cluster, new_cluster):
-            # type: (ccl.CephCluster, ccl.CephCluster) -> ccl.CephCluster
-            if not hasattr(new_cluster.spec, 'storage') or not new_cluster.spec.storage:
-                new_cluster.spec.storage = ccl.Storage()
-
-            if not hasattr(new_cluster.spec.storage, 'storageClassDeviceSets') or not new_cluster.spec.storage.storageClassDeviceSets:
-                new_cluster.spec.storage.storageClassDeviceSets = ccl.StorageClassDeviceSetsList()
-
-            existing_scds = [
-                scds.name for scds in new_cluster.spec.storage.storageClassDeviceSets
-            ]
-            for device in to_create:
-                new_scds = self.device_to_device_set(drive_group, device)
-                if new_scds.name not in existing_scds:
-                    new_cluster.spec.storage.storageClassDeviceSets.append(new_scds)
-            return new_cluster
-        return _add_osds
-
-class LSOCreator(DefaultCreator):
-    def filter_devices(self, rook_pods: KubernetesResource, drive_group: DriveGroupSpec, matching_hosts: List[str]) -> List[Device]:
-        device_list = []
-        assert drive_group.data_devices is not None
-        sizematcher: Matcher = AllMatcher('', None)
-        if drive_group.data_devices.size:
-            sizematcher = SizeMatcher('size', drive_group.data_devices.size)
-        limit = getattr(drive_group.data_devices, 'limit', None)
-        _all = getattr(drive_group.data_devices, 'all', None)
-        paths = [device.path for device in drive_group.data_devices.paths]
-        vendor = getattr(drive_group.data_devices, 'vendor', None)
-        model = getattr(drive_group.data_devices, 'model', None)
-        count = 0
-        osd_list = []
-        for pod in rook_pods.items:
-            if (
-                hasattr(pod, 'metadata') 
-                and hasattr(pod.metadata, 'labels') 
-                and 'osd' in pod.metadata.labels 
-                and 'ceph.rook.io/DeviceSet' in pod.metadata.labels
-            ):
-                osd_list.append(pod.metadata.labels['ceph.rook.io/DeviceSet'])
-        for _, node in self.inventory.items():
-            for device in node:
-                if device.sys_api['pv_name'] in osd_list:
-                    count += 1
-        for _, node in self.inventory.items():
-            for device in node:
-                if not limit or (count < limit):
-                    if device.available:
-                        if (
-                            _all
-                            or (
-                                device.sys_api['node'] in matching_hosts
-                                and sizematcher.compare(device)
-                                and (
-                                    not drive_group.data_devices.paths
-                                    or device.path in paths
-                                ) 
-                                and (
-                                    not vendor 
-                                    or device.sys_api['vendor'] == vendor
-                                )
-                                and (
-                                    not model 
-                                    or device.sys_api['model'].startsWith(model)
-                                )
-                            )
-                        ):
-                            device_list.append(device)
-                            count += 1
-        return device_list
-
-class DefaultRemover():
-    def __init__(
-        self,
-        coreV1_api: 'client.CoreV1Api', 
-        batchV1_api: 'client.BatchV1Api', 
-        appsV1_api: 'client.AppsV1Api', 
-        osd_ids: List[str], 
-        replace_flag: bool, 
-        force_flag: bool, 
-        mon_command: Callable, 
-        patch: Callable, 
-        rook_env: 'RookEnv',
-        inventory: Dict[str, List[Device]]
-    ):
-        self.batchV1_api = batchV1_api
-        self.appsV1_api = appsV1_api
-        self.coreV1_api = coreV1_api
-
-        self.osd_ids = osd_ids
-        self.replace_flag = replace_flag
-        self.force_flag = force_flag
-
-        self.mon_command = mon_command
-
-        self.patch = patch
-        self.rook_env = rook_env
-
-        self.inventory = inventory
-        self.osd_pods: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_pod,
-                                                               namespace=self.rook_env.namespace,
-                                                               label_selector='app=rook-ceph-osd')
-        self.jobs: KubernetesResource = KubernetesResource(self.batchV1_api.list_namespaced_job,
-                                                           namespace=self.rook_env.namespace,
-                                                           label_selector='app=rook-ceph-osd-prepare')
-        self.pvcs: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_persistent_volume_claim,
-                                                           namespace=self.rook_env.namespace)
-
-
-    def remove_device_sets(self) -> str:
-        self.to_remove: Dict[str, int] = {}
-        self.pvc_to_remove: List[str] = []
-        for pod in self.osd_pods.items:
-            if (
-                hasattr(pod, 'metadata') 
-                and hasattr(pod.metadata, 'labels') 
-                and 'osd' in pod.metadata.labels 
-                and pod.metadata.labels['osd'] in self.osd_ids
-            ):
-                if pod.metadata.labels['ceph.rook.io/DeviceSet'] in self.to_remove:
-                    self.to_remove[pod.metadata.labels['ceph.rook.io/DeviceSet']] = self.to_remove[pod.metadata.labels['ceph.rook.io/DeviceSet']] + 1
-                else:
-                    self.to_remove[pod.metadata.labels['ceph.rook.io/DeviceSet']] = 1
-                self.pvc_to_remove.append(pod.metadata.labels['ceph.rook.io/pvc'])
-        def _remove_osds(current_cluster, new_cluster):
-            # type: (ccl.CephCluster, ccl.CephCluster) -> ccl.CephCluster
-            assert new_cluster.spec.storage is not None and new_cluster.spec.storage.storageClassDeviceSets is not None
-            for _set in new_cluster.spec.storage.storageClassDeviceSets:
-                    if _set.name in self.to_remove:
-                        if _set.count == self.to_remove[_set.name]:
-                            new_cluster.spec.storage.storageClassDeviceSets.remove(_set)
-                        else:
-                            _set.count = _set.count - self.to_remove[_set.name]
-            return new_cluster
-        return self.patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, _remove_osds)
-
-    def check_force(self) -> None:
-        if not self.force_flag:
-            safe_args = {'prefix': 'osd safe-to-destroy',
-                        'ids': [str(x) for x in self.osd_ids]}
-            ret, out, err = self.mon_command(safe_args)
-            if ret != 0:
-                raise RuntimeError(err)
-
-    def set_osds_down(self) -> None:
-        down_flag_args = {
-            'prefix': 'osd down',
-            'ids': [str(x) for x in self.osd_ids]
-        }
-        ret, out, err = self.mon_command(down_flag_args)
-        if ret != 0:
-            raise RuntimeError(err)
-
-    def scale_deployments(self) -> None:
-        for osd_id in self.osd_ids:
-            self.appsV1_api.patch_namespaced_deployment_scale(namespace=self.rook_env.namespace,
-                                                              name='rook-ceph-osd-{}'.format(osd_id),
-                                                              body=client.V1Scale(spec=client.V1ScaleSpec(replicas=0)))
-
-    def set_osds_out(self) -> None:
-        out_flag_args = {
-            'prefix': 'osd out',
-            'ids': [str(x) for x in self.osd_ids]
-        }
-        ret, out, err = self.mon_command(out_flag_args)
-        if ret != 0:
-            raise RuntimeError(err)
-            
-    def delete_deployments(self) -> None:
-        for osd_id in self.osd_ids:
-            self.appsV1_api.delete_namespaced_deployment(namespace=self.rook_env.namespace,
-                                                         name='rook-ceph-osd-{}'.format(osd_id),
-                                                         propagation_policy='Foreground')
-
-    def clean_up_prepare_jobs_and_pvc(self) -> None:
-        for job in self.jobs.items:
-            if job.metadata.labels['ceph.rook.io/pvc'] in self.pvc_to_remove:
-                self.batchV1_api.delete_namespaced_job(name=job.metadata.name, namespace=self.rook_env.namespace,
-                                                       propagation_policy='Foreground')
-                self.coreV1_api.delete_namespaced_persistent_volume_claim(name=job.metadata.labels['ceph.rook.io/pvc'],
-                                                                          namespace=self.rook_env.namespace,
-                                                                          propagation_policy='Foreground')
-
-    def purge_osds(self) -> None:
-        for id in self.osd_ids:
-            purge_args = {
-                'prefix': 'osd purge-actual',
-                'id': int(id),
-                'yes_i_really_mean_it': True
-            }
-            ret, out, err = self.mon_command(purge_args)
-            if ret != 0:
-                raise RuntimeError(err)
-
-    def destroy_osds(self) -> None:
-        for id in self.osd_ids:
-            destroy_args = {
-                'prefix': 'osd destroy-actual',
-                'id': int(id),
-                'yes_i_really_mean_it': True
-            }
-            ret, out, err = self.mon_command(destroy_args)
-            if ret != 0:
-                raise RuntimeError(err)
-
-    def remove(self) -> str:
-        try:
-            self.check_force()
-        except Exception as e:
-            log.exception("Error checking if OSDs are safe to destroy")
-            return f"OSDs not safe to destroy or unable to check if they are safe to destroy: {e}"
-        try:
-            remove_result = self.remove_device_sets()
-        except Exception as e:
-            log.exception("Error patching ceph cluster CRD")
-            return f"Not possible to modify Ceph cluster CRD: {e}"
-        try:
-            self.scale_deployments()
-            self.delete_deployments()
-            self.clean_up_prepare_jobs_and_pvc()
-        except Exception as e:
-            log.exception("Ceph cluster CRD patched, but error cleaning environment")
-            return f"Error cleaning environment after removing OSDs from Ceph cluster CRD: {e}"
-        try:
-            self.set_osds_down()
-            self.set_osds_out()
-            if self.replace_flag:
-                self.destroy_osds()
-            else:
-                self.purge_osds()
-        except Exception as e:
-            log.exception("OSDs removed from environment, but not able to remove OSDs from Ceph cluster")
-            return f"Error removing OSDs from Ceph cluster: {e}"
-
-        return remove_result
-
-
-
 class RookCluster(object):
     # import of client.CoreV1Api must be optional at import time.
     # Instead allow mgr/rook to be imported anyway.
@@ -1112,7 +794,6 @@ def _create_zone() -> cos.CephObjectStore:
                             name=spec.rgw_zone
                         )
             return object_store
-                
 
         def _update_zone(new: cos.CephObjectStore) -> cos.CephObjectStore:
             if new.spec.gateway:
@@ -1204,48 +885,11 @@ def _update_mon_count(current, new):
             return new
         return self._patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, _update_mon_count)
 
-    def add_osds(self, drive_group, matching_hosts):
-        # type: (DriveGroupSpec, List[str]) -> str
-        assert drive_group.objectstore in ("bluestore", "filestore")
-        assert drive_group.service_id
-        storage_class = self.get_storage_class()
-        inventory = self.get_discovered_devices()
-        creator: Optional[DefaultCreator] = None
-        if (
-            storage_class.metadata.labels
-            and 'local.storage.openshift.io/owner-name' in storage_class.metadata.labels
-        ):
-            creator = LSOCreator(inventory, self.coreV1_api, self.storage_class_name)
-        else:
-            creator = DefaultCreator(inventory, self.coreV1_api, self.storage_class_name)
-        return self._patch(
-            ccl.CephCluster,
-            'cephclusters',
-            self.rook_env.cluster_name,
-            creator.add_osds(self.rook_pods, drive_group, matching_hosts)
-        )
-
-    def remove_osds(self, osd_ids: List[str], replace: bool, force: bool, mon_command: Callable) -> str:
-        inventory = self.get_discovered_devices()
-        self.remover = DefaultRemover(
-            self.coreV1_api,
-            self.batchV1_api, 
-            self.appsV1_api, 
-            osd_ids, 
-            replace, 
-            force, 
-            mon_command, 
-            self._patch, 
-            self.rook_env,
-            inventory
-        )
-        return self.remover.remove()
-
     def get_hosts(self) -> List[orchestrator.HostSpec]:
         ret = []
         for node in self.nodes.items:
             spec = orchestrator.HostSpec(
-                node.metadata.name, 
+                node.metadata.name,
                 addr='/'.join([addr.address for addr in node.status.addresses]), 
                 labels=[label.split('/')[1] for label in node.metadata.labels if label.startswith('ceph-label')],
             )

From 1183db4146defaaa8371d65e2e65c99b21e106e9 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 31 Jan 2024 14:29:43 -0500
Subject: [PATCH 1742/2492] rgw: SiteConfig::load() falls back to local
 zonegroup

allow radosgw-admin commands like 'user create' to operate on a new zone
that hasn't been committed to the period yet. this follows similar logic
in RGWSI_Zone::do_start()

Fixes: https://tracker.ceph.com/issues/64255

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_zone.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index c8ebb79f542f..b93f319ac176 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -1222,12 +1222,15 @@ int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y,
   if (realm && !force_local_zonegroup) {
     // try to load the realm's period
     r = load_period_zonegroup(dpp, y, cfgstore, *realm, zone_params.id);
-  } else {
-    // fall back to a local zonegroup
-    r = load_local_zonegroup(dpp, y, cfgstore, zone_params.id);
+    if (r != -ENOENT) {
+      return r;
+    }
+    ldpp_dout(dpp, 10) << "cannot find current period zonegroup, "
+        "using local zonegroup configuration" << dendl;
   }
 
-  return r;
+  // fall back to a local zonegroup
+  return load_local_zonegroup(dpp, y, cfgstore, zone_params.id);
 }
 
 std::unique_ptr<SiteConfig> SiteConfig::make_fake() {
@@ -1285,6 +1288,9 @@ int SiteConfig::load_local_zonegroup(const DoutPrefixProvider* dpp,
   if (!zonegroup_name.empty()) {
     r = cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name,
                                          *local_zonegroup, nullptr);
+  } else if (realm) {
+    r = cfgstore->read_default_zonegroup(dpp, y, realm->id,
+                                         *local_zonegroup, nullptr);
   } else {
     r = read_or_create_default_zonegroup(dpp, y, cfgstore, zone_params,
                                          *local_zonegroup);

From 4dc12092be584da44baca14e31ca33231164235f Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 2 Feb 2024 11:53:45 +1000
Subject: [PATCH 1743/2492] doc/rados: update config for autoscaler

Update doc/rados/configuration/pool-pg-config-ref.rst to account for the
behavior of autoscaler.

Previously, this file was last meaningfully altered in 2013, prior to
the invention of autoscaler. A recent confusion was brought to my
attention on the Ceph Slack whereby a user attempted to alter the
default values of a Quincy cluster, as suggested in this documentation.
That alteration caused Ceph to throw the error "Error ERANGE: 'pgp_num'
must be greater than 0 and lower or equal than 'pg_num', which in this
case is one" and a related "rgw_init_ioctx ERROR" reading in part
"Numerical result out of range". The user removed the
"osd_pool_default_pgp_num" configuration line from ceph.conf and the
cluster worked as expected. I presume that this is because the removal
of this configuration line allowed autoscaler to work as intended.

Fixes: https://tracker.ceph.com/issues/64259

Co-authored-by: David Orman <ormandj@corenode.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../configuration/pool-pg-config-ref.rst      | 41 ++++++++++++++++---
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/doc/rados/configuration/pool-pg-config-ref.rst b/doc/rados/configuration/pool-pg-config-ref.rst
index 902c80346bbe..c3a25a3e74f4 100644
--- a/doc/rados/configuration/pool-pg-config-ref.rst
+++ b/doc/rados/configuration/pool-pg-config-ref.rst
@@ -6,12 +6,41 @@
 
 .. index:: pools; configuration
 
-Ceph uses default values to determine how many placement groups (PGs) will be
-assigned to each pool. We recommend overriding some of the defaults.
-Specifically, we recommend setting a pool's replica size and overriding the
-default number of placement groups. You can set these values when running
-`pool`_ commands. You can also override the defaults by adding new ones in the
-``[global]`` section of your Ceph configuration file.
+The number of placement groups that the CRUSH algorithm assigns to each pool is
+determined by the values of variables in the centralized configuration database
+in the monitor cluster. 
+
+Both containerized deployments of Ceph (deployments made using ``cephadm`` or
+Rook) and non-containerized deployments of Ceph rely on the values in the
+central configuration database in the monitor cluster to assign placement
+groups to pools. 
+
+Example Commands
+----------------
+
+To see the value of the variable that governs the number of placement groups in a given pool, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph config get osd osd_pool_default_pg_num
+
+To set the value of the variable that governs the number of placement groups in a given pool, run a command of the following form:
+
+.. prompt:: bash
+
+   ceph config set osd osd_pool_default_pg_num
+
+Manual Tuning
+-------------
+In some cases, it might be advisable to override some of the defaults. For
+example, you might determine that it is wise to set a pool's replica size and
+to override the default number of placement groups in the pool. You can set
+these values when running `pool`_ commands. 
+
+See Also
+--------
+
+See :ref:`pg-autoscaler`.
 
 
 .. literalinclude:: pool-pg.conf

From 1796022aa4778226c2d7e04c1cb989ab9d025272 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sat, 3 Feb 2024 02:18:25 -0500
Subject: [PATCH 1744/2492] src/common/options: Correct typo in rgw.yaml.in

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 src/common/options/rgw.yaml.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index b0f442366f4e..f4ef0079dd6b 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -233,7 +233,7 @@ options:
   long_desc: The lifecycle maintenance thread is responsible for lifecycle related
     maintenance work. The thread itself can be disabled, but in order for lifecycle
     to work correctly, at least one RGW in each zone needs to have this thread running.
-    Havingthe thread enabled on multiple RGW processes within the same zone can spread
+    Having the thread enabled on multiple RGW processes within the same zone can spread
     some of the maintenance work between them.
   default: true
   services:

From 27e9a15a53633b0c77bb0fc60387e58b46c569dd Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Sat, 3 Feb 2024 13:01:37 +0530
Subject: [PATCH 1745/2492] qa: Fix fs/full suite

Fix fs/full suite. The tests in the suite were failing
in release branches, quincy and pacific because of change
in the rados bluefs pre allocation (PR 51774). The change
affected main/quincy differently than quincy/pacific.

Fixes: https://tracker.ceph.com/issues/63132
Signed-off-by: Kotresh HR <khiremat@redhat.com>
---
 qa/suites/fs/full/tasks/mgr-osd-full.yaml     | 2 +-
 qa/workunits/fs/full/subvolume_clone.sh       | 6 +++---
 qa/workunits/fs/full/subvolume_rm.sh          | 6 +++---
 qa/workunits/fs/full/subvolume_snapshot_rm.sh | 8 ++++----
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/qa/suites/fs/full/tasks/mgr-osd-full.yaml b/qa/suites/fs/full/tasks/mgr-osd-full.yaml
index b4f673e39579..a005f52037c7 100644
--- a/qa/suites/fs/full/tasks/mgr-osd-full.yaml
+++ b/qa/suites/fs/full/tasks/mgr-osd-full.yaml
@@ -12,7 +12,7 @@ overrides:
         debug mds: 20
       osd: # force bluestore since it's required for ec overwrites
         osd objectstore: bluestore
-        bluestore block size: 1073741824
+        bluestore block size: 2147483648
 tasks:
 - workunit:
     cleanup: true
diff --git a/qa/workunits/fs/full/subvolume_clone.sh b/qa/workunits/fs/full/subvolume_clone.sh
index 75648f306f8b..edaea195deff 100755
--- a/qa/workunits/fs/full/subvolume_clone.sh
+++ b/qa/workunits/fs/full/subvolume_clone.sh
@@ -7,8 +7,8 @@ set -ex
 # Hence the subsequent subvolume commands on the clone fails with
 # 'MetadataMgrException: -2 (section 'GLOBAL' does not exist)' traceback.
 
-# The osd is of the size 1GB. The full-ratios are set so that osd is treated full
-# at around 600MB. The subvolume is created and 100MB is written.
+# The osd is of the size 2GiB. The full-ratios are set so that osd is treated full
+# at around 1.2GB. The subvolume is created and 200MB is written.
 # The subvolume is snapshotted and cloned ten times. Since the clone delay is set to 15 seconds,
 # all the clones reach pending state for sure. Among ten clones, only few succeed and rest fails
 # with ENOSPACE.
@@ -46,7 +46,7 @@ echo "After ratios are set"
 df -h
 ceph osd df
 
-for i in {1..100};do sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path_0/1MB_file-$i status=progress bs=1M count=1 conv=fdatasync;done
+for i in {1..100};do sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path_0/2MB_file-$i status=progress bs=1M count=2 conv=fdatasync;done
 
 # For debugging
 echo "After subvolumes are written"
diff --git a/qa/workunits/fs/full/subvolume_rm.sh b/qa/workunits/fs/full/subvolume_rm.sh
index a464e30f56e9..2a3bf956df33 100755
--- a/qa/workunits/fs/full/subvolume_rm.sh
+++ b/qa/workunits/fs/full/subvolume_rm.sh
@@ -2,8 +2,8 @@
 set -ex
 
 # This testcase tests the scenario of the 'ceph fs subvolume rm' mgr command
-# when the osd is full. The command used to hang. The osd is of the size 1GB.
-# The subvolume is created and 500MB file is written. The full-ratios are
+# when the osd is full. The command used to hang. The osd is of the size 2GiB.
+# The subvolume is created and 1GB file is written. The full-ratios are
 # set below 500MB such that the osd is treated as full. Now the subvolume is
 # is removed. This should be successful with the introduction of FULL
 # capabilities which the mgr holds.
@@ -21,7 +21,7 @@ echo "Before write"
 df -h
 ceph osd df
 
-sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/500MB_file-1 status=progress bs=1M count=500
+sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/1GB_file-1 status=progress bs=1M count=1000
 
 ceph osd set-full-ratio 0.2
 ceph osd set-nearfull-ratio 0.16
diff --git a/qa/workunits/fs/full/subvolume_snapshot_rm.sh b/qa/workunits/fs/full/subvolume_snapshot_rm.sh
index f6d0add9fda4..8df89d3c7a3b 100755
--- a/qa/workunits/fs/full/subvolume_snapshot_rm.sh
+++ b/qa/workunits/fs/full/subvolume_snapshot_rm.sh
@@ -7,8 +7,8 @@ set -ex
 # snapshot rm of the same snapshot fails with 'MetadataMgrException: -2 (section 'GLOBAL' does not exist)'
 # traceback.
 
-# The osd is of the size 1GB. The subvolume is created and 800MB file is written.
-# Then full-ratios are set below 500MB such that the osd is treated as full.
+# The osd is of the size 2GiB. The subvolume is created and 1.6GB file is written.
+# Then full-ratios are set below 1GiB such that the osd is treated as full.
 # The subvolume snapshot is taken which succeeds as no extra space is required
 # for snapshot. Now, the removal of the snapshot fails with ENOSPACE as it
 # fails to remove the snapshot metadata set. The snapshot removal fails
@@ -31,8 +31,8 @@ echo "Before write"
 df $CEPH_MNT
 ceph osd df
 
-# Write 800MB file and set full ratio to around 200MB
-ignore_failure sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/800MB_file-1 status=progress bs=1M count=800 conv=fdatasync
+# Write 1.6GB file and set full ratio to around 400MB
+ignore_failure sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/1.6GB_file-1 status=progress bs=1M count=1600 conv=fdatasync
 
 ceph osd set-full-ratio 0.2
 ceph osd set-nearfull-ratio 0.16

From 2a30ddccd04b204821f0496856847e8ffc07835d Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 17 Jan 2024 23:09:02 +0800
Subject: [PATCH 1746/2492] debian/control: add adduser to Depends of cephadm
 and ceph-common

in `debian/ceph-common.postinst` and `debian/cephadm.postinst`, we
use `adduser --system` to create the system user when configuring
the corresponding package.

before this change, the dependency is not listed in the runtime
`Depends` section of ceph-common and cephadm.

in this change, the dependency is added. this is also suggested
by Securing Debian Manual, see
https://www.debian.org/doc/manuals/securing-debian-manual/bpp-lower-privs.en.html

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/control | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/debian/control b/debian/control
index b4967b398f4d..98e76fceba8b 100644
--- a/debian/control
+++ b/debian/control
@@ -184,7 +184,8 @@ Description: debugging symbols for ceph-base
 Package: cephadm
 Architecture: linux-any
 Recommends: podman (>= 2.0.2) | docker.io | docker-ce
-Depends: lvm2,
+Depends: adduser (>= 3.11),
+	 lvm2,
 	 python3,
 	 ${python3:Depends},
 Description: cephadm utility to bootstrap ceph daemons with systemd and containers
@@ -610,7 +611,8 @@ Description: debugging symbols for rbd-nbd
 
 Package: ceph-common
 Architecture: linux-any
-Depends: librbd1 (= ${binary:Version}),
+Depends: adduser (>= 3.11),
+         librbd1 (= ${binary:Version}),
          python3-cephfs (= ${binary:Version}),
          python3-ceph-argparse (= ${binary:Version}),
          python3-ceph-common (= ${binary:Version}),

From a1a857698cad4f2feeea0b56fd781350e44078b7 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 17 Jan 2024 23:36:12 +0800
Subject: [PATCH 1747/2492] debian/ceph-common.postinst: set user directory
 using adduser

now that adduser allows us to set its home directory, we can do
this using adduser instead of using usermod. this change also
silences the warning from lintian
"maintainer-script-lacks-home-in-adduser". lintian complains if
`adduser --system` is called without passing `--home` option.

also, take this opportunity to s/-c/--comment/ in the command line
of `usermod`, for better readability.

Fixes: https://tracker.ceph.com/issues/64069
Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/ceph-common.postinst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/debian/ceph-common.postinst b/debian/ceph-common.postinst
index d147de5386b2..e058d096ee1a 100644
--- a/debian/ceph-common.postinst
+++ b/debian/ceph-common.postinst
@@ -52,16 +52,20 @@ case "$1" in
                  --system \
                  --no-create-home \
                  --disabled-password \
+                 --home $SERVER_HOME \
                  --uid $SERVER_UID \
                  --gid $SERVER_GID \
                  $SERVER_USER 2>/dev/null || true
          echo "..done"
        fi
        # 3. adjust passwd entry
+       # NOTE: we should use "adduser --comment" if we don't need to
+       # support adduser <3.136. "adduser --gecos" is deprecated,
+       # and will be removed, so we don't use it. the first distro
+       # using --comment is debian/trixie or ubuntu/mantic.
        echo -n "Setting system user $SERVER_USER properties.."
-       usermod -c "$SERVER_NAME" \
-               -d $SERVER_HOME   \
-               -g $SERVER_GROUP  \
+       usermod --comment "$SERVER_NAME" \
+               --gid $SERVER_GROUP      \
                $SERVER_USER
        # Unlock $SERVER_USER in case it is locked from an uninstall
        if [ -f /etc/shadow ]; then

From c517d0b0e95b20cd9bd979405d0248b9419e5c85 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 17 Jan 2024 23:44:23 +0800
Subject: [PATCH 1748/2492] debian/cephadm.postinst: wrap command line lines
 which is too long

for better readability.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/cephadm.postinst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/cephadm.postinst b/debian/cephadm.postinst
index 53d503e1eaa0..090677ad992e 100644
--- a/debian/cephadm.postinst
+++ b/debian/cephadm.postinst
@@ -25,7 +25,11 @@ case "$1" in
        # 1. create user if not existing
        if ! getent passwd | grep -q "^cephadm:"; then
          echo -n "Adding system user cephadm.."
-         adduser --quiet --system --disabled-password --gecos 'cephadm user for mgr/cephadm' --shell /bin/bash cephadm 2>/dev/null || true
+         adduser --quiet \
+                 --system \
+                 --disabled-password \
+                 --gecos 'cephadm user for mgr/cephadm' \
+                 --shell /bin/bash cephadm 2>/dev/null || true
          echo "..done"
        fi
 

From 90bc0369243077c2aaf67f0de2bab5810b217f4e Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 17 Jan 2024 23:47:39 +0800
Subject: [PATCH 1749/2492] debian/cephadm.postinst: specify --home when
 adduser

quote from adduser/NEWS.Debian.gz:

>  System user home defaults to /nonexistent if --home is not specified.
>  Packages that call adduser to create system accounts should explicitly
>  specify a location for /home (see Lintian check
>  maintainer-script-lacks-home-in-adduser).

so let's follow this change in adduser. otherwise "cephadm"
would have a $HOME at `/nonexistent`.

Fixes: https://tracker.ceph.com/issues/64069
Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/cephadm.postinst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/cephadm.postinst b/debian/cephadm.postinst
index 090677ad992e..eaa574cbb634 100644
--- a/debian/cephadm.postinst
+++ b/debian/cephadm.postinst
@@ -28,6 +28,7 @@ case "$1" in
          adduser --quiet \
                  --system \
                  --disabled-password \
+                 --home /home/cephadm \
                  --gecos 'cephadm user for mgr/cephadm' \
                  --shell /bin/bash cephadm 2>/dev/null || true
          echo "..done"

From 013eebf5357666cf2509a37eecf69ef4ac349321 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 17 Jan 2024 23:52:48 +0800
Subject: [PATCH 1750/2492] debian/cephadm.postinst: replace tab with spaces

for better readability, and to be more consistent with the rest
of this file, and other .postinst scripts of this project.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/cephadm.postinst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/debian/cephadm.postinst b/debian/cephadm.postinst
index eaa574cbb634..274b7f7846a6 100644
--- a/debian/cephadm.postinst
+++ b/debian/cephadm.postinst
@@ -43,19 +43,19 @@ case "$1" in
 
        # set up (initially empty) .ssh/authorized_keys file
        if ! test -d /home/cephadm/.ssh; then
-	   mkdir /home/cephadm/.ssh
-	   chown --reference /home/cephadm /home/cephadm/.ssh
-	   chmod 0700 /home/cephadm/.ssh
+           mkdir /home/cephadm/.ssh
+           chown --reference /home/cephadm /home/cephadm/.ssh
+           chmod 0700 /home/cephadm/.ssh
        fi
        if ! test -e /home/cephadm/.ssh/authorized_keys; then
-	   touch /home/cephadm/.ssh/authorized_keys
-	   chown --reference /home/cephadm /home/cephadm/.ssh/authorized_keys
-	   chmod 0600 /home/cephadm/.ssh/authorized_keys
+           touch /home/cephadm/.ssh/authorized_keys
+           chown --reference /home/cephadm /home/cephadm/.ssh/authorized_keys
+           chmod 0600 /home/cephadm/.ssh/authorized_keys
        fi
 
     ;;
     abort-upgrade|abort-remove|abort-deconfigure)
-	:
+       :
     ;;
 
     *)

From e74ec0b27dfb480a2951f85bf00072e95d944772 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sat, 3 Feb 2024 13:46:05 +0800
Subject: [PATCH 1751/2492] debian/cephadm.postinst: stop using adduser --gecos

--gecos option of adduser is deprecated in debian/bookworm, and
will be removed in debian/trixie,
see https://manpages.debian.org/bookworm/adduser/adduser.8.en.html.
so to be future-proof, let's switch to `usermod --comment`. please
note, since we still need to support ubuntu/jammy which is used in
our CI, and `adduser` shipped by ubuntu/jammy does not support
`--comment` yet, so we cannot use this option.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/cephadm.postinst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/cephadm.postinst b/debian/cephadm.postinst
index 274b7f7846a6..50aa6f8dd510 100644
--- a/debian/cephadm.postinst
+++ b/debian/cephadm.postinst
@@ -29,8 +29,8 @@ case "$1" in
                  --system \
                  --disabled-password \
                  --home /home/cephadm \
-                 --gecos 'cephadm user for mgr/cephadm' \
                  --shell /bin/bash cephadm 2>/dev/null || true
+         usermod --comment "cephadm user for mgr/cephadm" cephadm
          echo "..done"
        fi
 

From bbfa5bac97e624e042ba67bab4d0b8bcafefaa0c Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:22:19 -0800
Subject: [PATCH 1752/2492] crush/mapper: add support for MSR types

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/mapper.c | 1087 +++++++++++++++++++++++++++++++++++++++++---
 src/crush/mapper.h |   31 +-
 2 files changed, 1050 insertions(+), 68 deletions(-)

diff --git a/src/crush/mapper.c b/src/crush/mapper.c
index 736cc6162c97..021777ef0b2b 100644
--- a/src/crush/mapper.c
+++ b/src/crush/mapper.c
@@ -27,6 +27,9 @@
 
 #define dprintk(args...) /* printf(args) */
 
+#define MIN(x, y) ((x) > (y) ? (y) : (x))
+#define MAX(y, x) ((x) < (y) ? (y) : (x))
+
 /*
  * Implement the core CRUSH mapping algorithm.
  */
@@ -820,65 +823,11 @@ static void crush_choose_indep(const struct crush_map *map,
 #endif
 }
 
-
-/* This takes a chunk of memory and sets it up to be a shiny new
-   working area for a CRUSH placement computation. It must be called
-   on any newly allocated memory before passing it in to
-   crush_do_rule. It may be used repeatedly after that, so long as the
-   map has not changed. If the map /has/ changed, you must make sure
-   the working size is no smaller than what was allocated and re-run
-   crush_init_workspace.
-
-   If you do retain the working space between calls to crush, make it
-   thread-local. If you reinstitute the locking I've spent so much
-   time getting rid of, I will be very unhappy with you. */
-
-void crush_init_workspace(const struct crush_map *m, void *v) {
-	/* We work by moving through the available space and setting
-	   values and pointers as we go.
-
-	   It's a bit like Forth's use of the 'allot' word since we
-	   set the pointer first and then reserve the space for it to
-	   point to by incrementing the point. */
-	struct crush_work *w = (struct crush_work *)v;
-	char *point = (char *)v;
-	__s32 b;
-	point += sizeof(struct crush_work);
-	w->work = (struct crush_work_bucket **)point;
-	point += m->max_buckets * sizeof(struct crush_work_bucket *);
-	for (b = 0; b < m->max_buckets; ++b) {
-		if (m->buckets[b] == 0)
-			continue;
-
-		w->work[b] = (struct crush_work_bucket *) point;
-		switch (m->buckets[b]->alg) {
-		default:
-			point += sizeof(struct crush_work_bucket);
-			break;
-		}
-		w->work[b]->perm_x = 0;
-		w->work[b]->perm_n = 0;
-		w->work[b]->perm = (__u32 *)point;
-		point += m->buckets[b]->size * sizeof(__u32);
-	}
-	BUG_ON((char *)point - (char *)w != m->working_size);
-}
-
-/**
- * crush_do_rule - calculate a mapping with the given input and rule
- * @map: the crush_map
- * @ruleno: the rule id
- * @x: hash input
- * @result: pointer to result vector
- * @result_max: maximum result size
- * @weight: weight vector (for map leaves)
- * @weight_max: size of weight vector
- * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
- */
-int crush_do_rule(const struct crush_map *map,
-		  int ruleno, int x, int *result, int result_max,
-		  const __u32 *weight, int weight_max,
-		  void *cwin, const struct crush_choose_arg *choose_args)
+static int crush_do_rule_no_retry(
+	const struct crush_map *map,
+	int ruleno, int x, int *result, int result_max,
+	const __u32 *weight, int weight_max,
+	void *cwin, const struct crush_choose_arg *choose_args)
 {
 	int result_len;
 	struct crush_work *cw = cwin;
@@ -1081,3 +1030,1023 @@ int crush_do_rule(const struct crush_map *map,
 
 	return result_len;
 }
+
+/// invariant through crush_msr_do_rule invocation
+struct crush_msr_input {
+	const struct crush_map *map;
+	const struct crush_rule *rule;
+
+	const unsigned result_max;
+
+	const unsigned weight_len;
+	const __u32 *weights;
+
+	const int map_input;
+	const struct crush_choose_arg *choose_args;
+
+	const unsigned msr_descents;
+	const unsigned msr_collision_tries;
+};
+
+/// encapsulates work space, invariant within an EMIT block
+struct crush_msr_workspace {
+	const unsigned start_stepno;
+	const unsigned end_stepno;
+
+	const unsigned result_len;
+
+	const struct crush_work *crush_work;
+
+	// step_vecs has shape int[end_stepno - start_stepno][result_len]
+	int **step_vecs;
+};
+
+/**
+ * crush_msr_output
+ *
+ * Encapsulates output space.  Successive results through a crush_msr_do_rule
+ * invocation are placed into *out.
+ */
+struct crush_msr_output {
+	const unsigned result_len;
+	unsigned returned_so_far;
+	int *out;
+};
+
+/**
+ * crush_msr_scan_config_steps
+ *
+ * Scans possibly empty sequence of CRUSH_RULE_SET_MSR_*_TRIES
+ * steps at the start of the rule.  Returns index of next step.
+ * Populates *msr_descents and *msr_collision_tries (if non-null) with
+ * last matching rule.
+ * @steps: steps to scan
+ * @step_len: length of steps
+ * @msr_descents: out param for CRUSH_RULE_SET_MSR_DESCENTS
+ * @msr_collision_tries: out param for CRUSH_RULE_SET_MSR_COLLISION_TRIES
+ */
+static unsigned crush_msr_scan_config_steps(
+	const struct crush_rule_step *steps,
+	unsigned step_len,
+	unsigned *msr_descents,
+	unsigned *msr_collision_tries)
+{
+	unsigned stepno = 0;
+	for (; stepno < step_len; ++stepno) {
+		const struct crush_rule_step *step = &steps[stepno];
+		switch (step->op) {
+		case CRUSH_RULE_SET_MSR_DESCENTS:
+			if (msr_descents) *msr_descents = step->arg1;
+			break;
+		case CRUSH_RULE_SET_MSR_COLLISION_TRIES:
+			if (msr_collision_tries) *msr_collision_tries = step->arg1;
+			break;
+		default:
+			return stepno;
+		}
+	}
+	return stepno;
+}
+
+/// clear workspace represented by *ws
+static void crush_msr_clear_workspace(
+	struct crush_msr_workspace *ws)
+{
+	for (unsigned stepno = ws->start_stepno; stepno < ws->end_stepno;
+	     ++stepno) {
+		for (unsigned i = 0; i < ws->result_len; ++i) {
+			ws->step_vecs[stepno - ws->start_stepno][i] =
+				CRUSH_ITEM_UNDEF;
+		}
+	}
+}
+
+/**
+ * crush_msr_scan_next
+ *
+ * Validates an EMIT block of the form (TAKE CHOOSE_MSR* EMIT)
+ * If sequence is valid, populates total_children with the width
+ * of the mapping from the choose steps and next_emit with the
+ * index of the next EMIT step.
+ *
+ * @rule: rule to scan
+ * @result_max: max number of results to return
+ * @stepno: points at step at which to begin scanning, must be CRUSH_RULE_TAKE
+ * @total_children: output param for total fanout of EMIT block
+ * @next_emit: output param for ending EMIT step
+ * @return 0 if valid, -1 if there were validation errors
+ */
+static int crush_msr_scan_next(
+	const struct crush_rule *rule,
+	unsigned result_max,
+	unsigned stepno,
+	unsigned *total_children,
+	unsigned *next_emit)
+{
+	if (stepno + 1 >= rule->len) {
+		dprintk("stepno too large\n");
+		return -1;
+	}
+	if (rule->steps[stepno].op != CRUSH_RULE_TAKE) {
+		dprintk("first rule not CRUSH_RULE_TAKE\n");
+		return -1;
+	}
+	++stepno;
+
+	if (total_children) *total_children = 1;
+	for (; stepno < rule->len; ++stepno) {
+		const struct crush_rule_step *curstep =
+			&(rule->steps[stepno]);
+		if (curstep->op == CRUSH_RULE_EMIT) {
+			break;
+		}
+		if (rule->steps[stepno].op != CRUSH_RULE_CHOOSE_MSR) {
+			dprintk("found non-choose non-emit step %d\n", stepno);
+			return -1;
+		}
+		if (total_children) {
+			*total_children *= curstep->arg1 ? curstep->arg1
+				: result_max;
+		}
+	}
+	if (stepno >= rule->len) {
+		dprintk("did not find emit\n");
+		return -1;
+	}
+	if (next_emit) {
+		*next_emit = stepno;
+	}
+	return 0;
+}
+
+/**
+ * crush_msr_scan_rule
+ *
+ * MSR rules must have the form:
+ * 1) Possibly empty sequence of CRUSH_RULE_SET_MSR_.*_TRIES steps
+ * 2) A sequence of EMIT blocks of the form
+ *   (TAKE CHOOSE_MSR* EMIT)*
+ *
+ * crush_msr_scan_rule validates that the form obeys the above form and
+ * populates max_steps with the length of the longest sequence of CHOOSE_MSR
+ * steps.
+ *
+ * crush_msr_scan_rule replicates the scan behavior of crush_msr_do_rule.
+ *
+ * @rule: rule to scan
+ * @result_max: max number of results to return
+ * @max_steps: length of longest string of choosemsr steps
+ * @return 0 if valid, -1 otherwise
+ */
+static int crush_msr_scan_rule(
+	const struct crush_rule *rule,
+	unsigned result_max,
+	unsigned *max_steps)
+{
+	if (max_steps) *max_steps = 0;
+	unsigned next_stepno = crush_msr_scan_config_steps(
+		rule->steps,
+		rule->len,
+		NULL, NULL);
+	while (next_stepno < rule->len) {
+		unsigned next_emit_stepno;
+		int r = crush_msr_scan_next(
+			rule, result_max, next_stepno,
+			NULL, &next_emit_stepno);
+		if (r < 0) return r;
+
+		if (max_steps) {
+			*max_steps = MAX(
+				*max_steps,
+				next_emit_stepno - (next_stepno + 1));
+		}
+		next_stepno = next_emit_stepno + 1;
+	}
+	return 0;
+}
+
+/// Returns true if all leaf slots in [start, end) are mapped
+static int crush_msr_leaf_vec_populated(
+	const struct crush_msr_workspace *workspace,
+	const unsigned start, const unsigned end)
+{
+	BUG_ON(start >= end);
+	BUG_ON(end > workspace->result_len);
+	BUG_ON(workspace->end_stepno <= workspace->start_stepno);
+	// we check the last step vector here because output
+	// won't be ordered by index for FIRSTN rules
+	int *leaf_vec = workspace->step_vecs[
+	  workspace->end_stepno - workspace->start_stepno - 1];
+	for (unsigned i = start; i < end; ++i) {
+		if (leaf_vec[i] == CRUSH_ITEM_UNDEF) {
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/// Returns try value to pass to crush based on index, tries, and local_tries
+static unsigned crush_msr_get_retry_value(
+	const unsigned result_max,
+	const unsigned index,
+	const unsigned msr_descents,
+	const unsigned msr_collision_tries)
+{
+	const unsigned total_index = (msr_descents * result_max) + index;
+	return (total_index << 16) + msr_collision_tries;
+}
+
+/**
+ * crush_msr_descend
+ *
+ * Descend recursively from bucket until we either hit a leaf or an
+ * interior node of type type.
+ * @input: crush input information
+ * @workspace: struct with working space
+ * @bucket: bucket from which to descend
+ * @type: target node type
+ * @tryno: top level try number, incremented with each call into crush_msr_choose
+ *         from crush_msr_do_rule
+ * @local_tryno: local collision try number, incremented with each call into
+ *               crush_msr_descend from crush_msr_choose after collision
+ * @index: mapping index
+ */
+static int crush_msr_descend(
+	const struct crush_msr_input *input,
+	const struct crush_msr_workspace *workspace,
+	const struct crush_bucket *bucket,
+	const int type,
+	const unsigned tryno,
+	const unsigned local_tryno,
+	const unsigned index)
+{
+	dprintk(" crush_msr_descend type %d tryno %d local_tryno %d index %d\n",
+		type, tryno, local_tryno, index);
+	while (1) {
+		const int child_bucket_candidate = crush_bucket_choose(
+			bucket,
+			workspace->crush_work->work[-1 - bucket->id],
+			input->map_input,
+			crush_msr_get_retry_value(
+				input->result_max,
+				index, tryno, local_tryno),
+			(input->choose_args ?
+			 &(input->choose_args[-1 - bucket->id]) : 0),
+			index);
+
+		if (child_bucket_candidate >= 0) {
+			return child_bucket_candidate;
+		}
+
+		bucket = input->map->buckets[-1 - child_bucket_candidate];
+		if (bucket->type == type) {
+			return child_bucket_candidate;
+		}
+	}
+}
+
+/**
+ * crush_msr_valid_candidate
+ *
+ * Checks whether candidate is a valid choice given buckets already
+ * mapped for step stepno.
+ *
+ * If candidate has already been mapped for a position in
+ * [include_start, include_end), candidate is valid.
+ *
+ * Else, if candidate has already been mapped for a position in
+ * [exclude_start, exclude_end), candidate is invalid.
+ *
+ * Otherwise, candidate is valid.
+ *
+ * @stepno: step to check
+ * @exclude_start: start of exclusion range
+ * @exclude_end: end of exlusion range
+ * @include_start: start of inclusion range
+ * @include_end: end of inclusion range
+ * @candidate: bucket to check
+ *
+ * Note, [exclude_start, exclude_end) must contain [include_start, include_end).
+ */
+static int crush_msr_valid_candidate(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned exclude_start,
+	unsigned exclude_end,
+	unsigned include_start,
+	unsigned include_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(exclude_end <= exclude_start);
+	BUG_ON(include_end <= include_start);
+
+	BUG_ON(exclude_start > include_start);
+	BUG_ON(exclude_end < include_end);
+
+	BUG_ON(exclude_end > workspace->result_len);
+
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = exclude_start; i < exclude_end; ++i) {
+		if (vec[i] == candidate) {
+			if (i >= include_start && i < include_end) {
+				dprintk(" crush_msr_valid_candidate: "
+					"candidate %d already chosen for "
+					"stride\n",
+					candidate);
+				return 1;
+			} else {
+				dprintk(" crush_msr_valid_candidate: "
+					"candidate %d collision\n",
+					candidate);
+				return 0;
+			}
+		}
+	}
+	dprintk(" crush_msr_valid_candidate: candidate %d no collision\n",
+		candidate);
+	return 1;
+}
+
+/**
+ * crush_msr_push_used
+ *
+ * See crush_msr_choose for details, used to push bucket indicies onto collision
+ * set for specified stride.  User is responsible for ensuring that
+ * [stride_start, stride_end) never holds more than stride_end - stride_start
+ * entries.
+ * @workspace: holds working space information
+ * @stepno: index of step
+ * @stride_start: start of stride
+ * @stride_end: one past end of stride
+ * @candidate: element to add to set
+ * @return 1 if added (not already present), 0 if not added due to already
+ *           being present
+ */
+static int crush_msr_push_used(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned stride_start,
+	unsigned stride_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(stride_end <= stride_start);
+	BUG_ON(stride_end > workspace->result_len);
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = stride_start; i < stride_end; ++i) {
+		if (vec[i] == candidate) {
+			return 0;
+		} else if (vec[i] == CRUSH_ITEM_UNDEF) {
+			vec[i] = candidate;
+			return 1;
+		}
+	}
+	BUG_ON("impossible");
+	return 0;
+}
+
+/**
+ * crush_msr_pop_used
+ *
+ * See crush_msr_choose for details, used to pop bucket indicies from collision
+ * set for specified stride.  If an element is to be popped, crush_msr_pop_used
+ * must be called prior to pushing another element.
+ * @workspace: holds working space information
+ * @stepno: index of step
+ * @stride_start: start of stride
+ * @stride_end: one past end of stride
+ * @candidate: element to pop from set
+ */
+static void crush_msr_pop_used(
+	const struct crush_msr_workspace *workspace,
+	unsigned stepno,
+	unsigned stride_start,
+	unsigned stride_end,
+	int candidate)
+{
+	BUG_ON(stepno >= workspace->end_stepno);
+	BUG_ON(stepno < workspace->start_stepno);
+
+	BUG_ON(stride_end <= stride_start);
+	BUG_ON(stride_end > workspace->result_len);
+	int *vec = workspace->step_vecs[stepno - workspace->start_stepno];
+	for (unsigned i = stride_end; i > stride_start;) {
+		--i;
+		if (vec[i] != CRUSH_ITEM_UNDEF) {
+			BUG_ON(vec[i] != candidate);
+			vec[i] = CRUSH_ITEM_UNDEF;
+			return;
+		}
+	}
+	BUG_ON(0 == "impossible");
+}
+
+/**
+ * crush_msr_emit_result
+ *
+ * Outputs mapping result from specified position.  Position in output
+ * buffer depends on rule type -- FIRSTN outputs in output order, INDEP
+ * outputs into specified position.
+ * @output: output buffer
+ * @rule_type: CRUSH_RULE_TYPE_MSR_FIRSTN or CRUSH_RULE_TYPE_MSR_INDEP
+ * @position: mapping position
+ * @result: mapping value to output
+ */
+static void crush_msr_emit_result(
+	struct crush_msr_output *output,
+	int rule_type,
+	unsigned position,
+	int result)
+{
+	BUG_ON(position >= output->result_len);
+	BUG_ON(output->returned_so_far >= output->result_len);
+	if (rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
+		BUG_ON(output->out[output->returned_so_far] != CRUSH_ITEM_NONE);
+		output->out[(output->returned_so_far)++] = result;
+	} else {
+		BUG_ON(output->out[position] != CRUSH_ITEM_NONE);
+		output->out[position] = result;
+		++output->returned_so_far;
+	}
+	dprintk(" emit: %d, returned_so_far: %d\n",
+		result, output->returned_so_far);
+}
+
+/**
+ * crush_msr_choose
+ *
+ * Performs mapping for a single EMIT block containing CHOOSE steps
+ * [current_stepno, end_stepno) into mapping indices [start_index, end_index).
+ *
+ * Like chooseleaf, crush_msr_choose is essentially depth-first -- it chooses
+ * an item and all of the descendents under that item before moving to the
+ * next item.  Each choose step in the block gets its own workspace for
+ * collision detection.
+ *
+ * crush_msr_choose (and its recursive calls) will locally retry any bucket
+ * selections that produce a collision (up to msr_collision_tries times), but
+ * won't retry if it hits an out osd -- that's handled by calling back into
+ * crush_msr_choose up to msr_descents times.
+ *
+ * @input: crush input information
+ * @workspace: working space for this EMIT block
+ * @output: crush mapping output buffer specification
+ * @total_children: total number of children implied by the step sequence, may
+ *                  be larger than end_index - start_index.
+ * @start_index: start mapping index
+ * @end_index: end mapping index
+ * @current_stepno: first choose step
+ * @end_stepno: one past last choose step, must be an EMIT
+ * @tryno: try number, see crush_msr_do_rule
+ */
+static unsigned crush_msr_choose(
+	const struct crush_msr_input *input,
+	const struct crush_msr_workspace *workspace,
+	struct crush_msr_output *output,
+	const struct crush_bucket *bucket,
+	const unsigned total_descendants,
+	const unsigned start_index, const unsigned end_index,
+	const unsigned current_stepno, const unsigned end_stepno,
+	const unsigned tryno)
+{
+	dprintk("crush_msr_choose: bucket %d, start_index %d, end_index %d\n",
+		bucket->id, start_index, end_index);
+
+	BUG_ON(current_stepno >= input->rule->len);
+	const struct crush_rule_step *curstep =
+		&(input->rule->steps[current_stepno]);
+	BUG_ON(curstep->op != CRUSH_RULE_CHOOSE_MSR);
+
+	/* This call into crush_msr_choose is responsible, ultimately, for
+	 * populating indices [start_index, end_index).  We do this by
+	 * dividing that range into a set of strides specified in the
+	 * step -- choosemsr 4 host would dictate that the range be divided
+	 * into 4 strides.
+	 *
+	 * If the full rule is
+	 *
+	 * ...
+	 * step take root
+	 * step choosemsr 4 host
+	 * step choosemsr 2 osd
+	 * step emit
+	 *
+	 * total_descendants for the initial call would be 8 (4*2) with
+	 * num_stride=4 (4 hosts) and stride_length = 2 (2 osds per host).
+	 * For the recursive calls, total_descendants would be 2 (8 / 4),
+	 * stride_length would be 1 and num_strides would be 2.
+	 */
+
+	// choosemsr 0 host should select result_max hosts
+	const unsigned num_strides = curstep->arg1 ? curstep->arg1
+		: input->result_max;
+
+	// total_descendants is the product of the steps in the block
+	BUG_ON(total_descendants % num_strides != 0);
+	const unsigned stride_length = total_descendants / num_strides;
+
+	/* MSR steps like
+	 *
+	 * step choosemsr 4 host
+	 *
+	 * guarantee that the output mapping will be divided into at least
+	 * 4 hosts, not exactly 4 hosts.  We achieve this by ensuring that
+	 * the sets of hosts for each stride are disjoint -- a host selected
+	 * for stride 0 will not be used for any other stride.
+	 *
+	 * However, a single stride might end up using more than one host.
+	 * If an OSD on a host is marked out, crush_msr_choose will simply
+	 * skip that index when it hits it.  crush_msr_do_rule will then
+	 * call back into crush_msr_choose and eventually find another OSD
+	 * either on the same host or on another one not already used in
+	 * another stride. For this reason, a single stride may need to
+	 * remember up to stride_length entries for collision detection
+	 * purposes.
+	 *
+	 * Unfortunately, we only have stride_length entries to work with
+	 * in workspace.  Thus, prior to returning from crush_msr_choose,
+	 * we remove entries that didn't actually result in a mapping.  We
+	 * use the following undo vector to achieve this -- any strides that
+	 * didn't result in a successful mapping are set in undo to be undone
+	 * immediately prior to returning.
+	 *
+	 * Why prior to returning and not immediately?  Selecting a bucket in
+	 * a stride impacts subsequent choices as they may have collided.  In
+	 * order to limit the impact of marking an OSD out, we treat it as
+	 * collidable until the next pass.
+	 */
+	int undo[num_strides];
+	for (unsigned stride = 0; stride < num_strides; ++stride) {
+		undo[stride] = CRUSH_ITEM_UNDEF;
+	}
+
+	dprintk("crush_msr_choose: bucket %d, start_index %d, "
+		"end_index %d, stride_length %d\n",
+		bucket->id, start_index, end_index, stride_length);
+
+	unsigned mapped = 0;
+	unsigned stride_index = 0;
+	for (unsigned stride_start = start_index;
+	     stride_start < end_index;
+	     stride_start += stride_length, ++stride_index) {
+		const unsigned stride_end =
+		  MIN(stride_start + stride_length, end_index);
+
+		// all descendants for this stride have been mapped already
+		if (crush_msr_leaf_vec_populated(
+		      workspace, stride_start, stride_end)) {
+		  continue;
+		}
+
+		int found = 0;
+		int child_bucket_candidate;
+		for (unsigned local_tryno = 0;
+		     local_tryno < input->msr_collision_tries;
+		     ++local_tryno) {
+			child_bucket_candidate = crush_msr_descend(
+				input, workspace, bucket,
+				curstep->arg2, tryno, local_tryno,
+				stride_index);
+
+			/* candidate is valid if:
+			 * - we already chose it for this stride
+			 * - it hasn't been chosen for any stride */
+			if (crush_msr_valid_candidate(
+				    workspace,
+				    current_stepno,
+				    // Collision on elements in [start_index, end_index)
+				    start_index, end_index,
+				    // ...unless in [stride_start, stride_end)
+				    stride_start, stride_end,
+				    child_bucket_candidate)) {
+				found = 1;
+				break;
+			}
+		}
+
+		/* failed to find non-colliding choice after msr_collision_tries
+		 * attempts */
+		if (!found) continue;
+
+		if (curstep->arg2 == 0 /* leaf */) {
+			if (stride_length != 1 ||
+			    (current_stepno + 1 != end_stepno)) {
+				/* Either condition above implies that there's
+				 * another step after a choosemsr step for the
+				 * leaf type, rule is malformed, bail */
+				continue;
+			}
+			if (is_out(input->map, input->weights,
+				   input->weight_len,
+				   child_bucket_candidate, input->map_input)) {
+				dprintk(" crush_msr_choose: item %d out\n",
+					child_bucket_candidate);
+				/* crush_msr_do_rule will try again,
+				 * msr_descents permitting */
+				continue;
+			}
+			// for collision detection
+			int pushed = crush_msr_push_used(
+				workspace, current_stepno, stride_start, stride_end,
+				child_bucket_candidate);
+			/* stride_length == 1, can't already be there */
+			BUG_ON(!pushed);
+			// final output, ordering depending on input->rule->type
+			crush_msr_emit_result(
+				output, input->rule->type,
+				stride_start, child_bucket_candidate);
+			mapped++;
+		} else /* not leaf */ {
+			if (current_stepno + 1 >= end_stepno) {
+				/* Type isn't leaf, rule is malformed since there
+				 * isn't another step */
+				continue;
+			}
+			struct crush_bucket *child_bucket = input->map->buckets[
+				-1 - child_bucket_candidate];
+			unsigned child_mapped = crush_msr_choose(
+				input, workspace, output,
+				child_bucket,
+				// total_descendants for recursive call
+				stride_length,
+				// recursive call populates
+				// [stride_start, stride_end)
+				stride_start, stride_end,
+				// next step
+				current_stepno + 1, end_stepno,
+				tryno);
+			int pushed = crush_msr_push_used(
+				workspace,
+				current_stepno,
+				stride_start,
+				stride_end,
+				child_bucket_candidate);
+			/* pushed may be false if we already chose this bucket
+			 * for this stride.  If so, child_mapped must have been
+			 * != 0 at the time, so we still retain it */
+			if (pushed && (child_mapped == 0)) {
+				// no child mapped, and we didn't choose it
+				// before
+				undo[stride_index] = child_bucket_candidate;
+			} else {
+				mapped += child_mapped;
+			}
+		}
+	}
+
+	// pop unused buckets
+	stride_index = 0;
+	for (unsigned stride_start = start_index;
+	     stride_start < end_index;
+	     stride_start += stride_length, ++stride_index) {
+		if (undo[stride_index] != CRUSH_ITEM_UNDEF) {
+			unsigned stride_end =
+			  MIN(stride_start + stride_length, end_index);
+			crush_msr_pop_used(
+				workspace,
+				current_stepno,
+				stride_start,
+				stride_end,
+				undo[stride_index]);
+		}
+	}
+
+	return mapped;
+}
+
+/**
+ * crush_msr_do_rule - calculate a mapping with the given input and msr rule
+ *
+ * msr_firstn and msr_indep rules are intended to address a limitation of
+ * conventional crush rules in that they do not retry steps outside of
+ * a CHOOSELEAF step.  In the case of a crush rule like
+ *
+ * rule replicated_rule_1 {
+ *   ...
+ *   step take default class hdd
+ *   step chooseleaf firstn 3 type host
+ *   step emit
+ * }
+ *
+ * the chooseleaf step will ensure that if all of the osds on a
+ * particular host are marked out, mappings including those OSDs would
+ * end up on another host (provided that there are enough hosts).
+ *
+ * However, if the rule used two choose steps instead
+ *
+ * rule replicated_rule_1 {
+ *   ...
+ *   step take default class hdd
+ *   step choose firstn 3 type host
+ *   step choose firstn 1 type osd
+ *   step emit
+ * }
+ *
+ * marking an OSD down could cause it to be remapped to another on the same
+ * host, but not to another host.  If all of the OSDs on a host are marked
+ * down, the PGs will simply be degraded and unable to remap until the host
+ * is removed from the CRUSH heirarchy or reweighted to 0.
+ *
+ * Normally, we can comfortably work around this by using a chooseleaf
+ * step as in the first example, but there are cases where we want to map
+ * multiple OSDs to each host (wide EC codes on small clusters, for
+ * example) which can't be handled with chooseleaf as it currently
+ * exists.
+ *
+ * rule ecpool-86 {
+ *   type msr_indep
+ *   ...
+ *   step choosemsr 4 type host
+ *   step choosemsr 4 type osd
+ *   step emit
+ * }
+ *
+ * With an 8+6 code, this rule can tolerate a host and a single OSD down without
+ * becoming unavailable on 4 hosts.  It relies on ensuring that no more than 4
+ * OSDs are mapped to any single host, however, which can't be done with a
+ * conventional CRUSH rule without the drawback described above.  By using
+ * msr_indep, this rule can deal with an OSD failure by remapping to another
+ * host.
+ *
+ * MSR rules have some structural differences from conventional rules:
+ * - The rule type determines whether the mapping is FIRSTN or INDEP.  Because
+ *   the descent can retry steps, it doesn't really make sense for steps to
+ *   individually specify output order and I'm not really aware of any use cases
+ *   that would benefit from it.
+ * - MSR rules *must* be structured as a (possibly empty) prefix of config
+ *   steps (CRUSH_RULE_SET_MSR*) followed by a sequence of EMIT blocks
+ *   each comprised of a TAKE step, a sequence of CHOOSE_MSR steps, and
+ *   ended by an EMIT step.
+ * - MSR choose steps must be choosemsr.  choose and chooseleaf are not permitted.
+ *
+ * MSR rules also have different requirements for working space.  Conventional
+ * CRUSH requires 3 vectors of size result_max to use for working space -- two
+ * to alternate as it processes each rule and one, additionally, for chooseleaf.
+ * MSR rules need N vectors where N is the number of choosemsr in the longest
+ * EMIT block since it needs to retain all of the choices made as part of each
+ * descent.
+ *
+ * See crush_msr_choose for details.
+ *
+ * doc/dev/crush-msr.rst has an overview of the motivation behind CRUSH MSR
+ * rules and should be kept up to date with any changes to implementation or
+ * documentation in this file.
+ *
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
+ */
+static int crush_msr_do_rule(
+	const struct crush_map *map,
+	int ruleno, int map_input, int *result, int result_max,
+	const __u32 *weight, int weight_max,
+	void *cwin, const struct crush_choose_arg *choose_args)
+{
+	unsigned msr_descents = map->msr_descents;
+	unsigned msr_collision_tries = map->msr_collision_tries;
+	struct crush_rule *rule = map->rules[ruleno];
+	unsigned start_stepno = crush_msr_scan_config_steps(
+		rule->steps, rule->len,
+		&msr_descents, &msr_collision_tries);
+
+	struct crush_msr_input input = {
+		.map = map,
+		.rule = map->rules[ruleno],
+		.result_max = result_max,
+		.weight_len = weight_max,
+		.weights = weight,
+		.map_input = map_input,
+		.choose_args = choose_args,
+		.msr_descents = msr_descents,
+		.msr_collision_tries = msr_collision_tries
+	};
+
+	struct crush_msr_output output = {
+		.result_len = result_max,
+		.returned_so_far = 0,
+		.out = result
+	};
+	for (unsigned i = 0; i < output.result_len; ++i) {
+		output.out[i] = CRUSH_ITEM_NONE;
+	}
+
+	unsigned start_index = 0;
+	while (start_stepno < input.rule->len) {
+		unsigned emit_stepno, total_children;
+		if (crush_msr_scan_next(
+			    input.rule, input.result_max,
+			    start_stepno, &total_children,
+			    &emit_stepno) != 0) {
+			// invalid rule, return whatever we have
+			dprintk("crush_msr_scan_returned -1\n");
+			return 0;
+		}
+
+		const struct crush_rule_step *take_step =
+			&(input.rule->steps[start_stepno]);
+		BUG_ON(take_step->op != CRUSH_RULE_TAKE);
+
+		if (take_step->arg1 >= 0) {
+			if (start_stepno + 1 != emit_stepno) {
+				// invalid rule
+				dprintk("take step specifies osd, but "
+					"there are subsequent choose steps\n");
+				return 0;
+			} else {
+				crush_msr_emit_result(
+					&output, input.rule->type,
+					start_index, take_step->arg1);
+			}
+		} else {
+			dprintk("start_stepno %d\n", start_stepno);
+			dprintk("root bucket: %d\n",
+				input.rule->steps[start_stepno].arg1);
+			struct crush_bucket *root_bucket = input.map->buckets[
+				-1 - input.rule->steps[start_stepno].arg1];
+			dprintk(
+				"root bucket: %d %p\n",
+				input.rule->steps[start_stepno].arg1,
+				root_bucket);
+
+			++start_stepno;
+			BUG_ON(emit_stepno >= input.rule->len);
+			BUG_ON(emit_stepno < start_stepno);
+			BUG_ON(start_stepno >= input.rule->len);
+
+			struct crush_work *cw = cwin;
+			int *out_vecs[input.rule->len];
+			for (unsigned stepno = 0;
+			     stepno < input.rule->len; ++stepno) {
+				out_vecs[stepno] =
+					(int*)((char*)cw + map->working_size) +
+					(stepno * result_max);
+			}
+			struct crush_msr_workspace workspace = {
+				.start_stepno = start_stepno,
+				.end_stepno = emit_stepno,
+				.result_len = result_max,
+				.crush_work = cw,
+				.step_vecs = out_vecs
+			};
+			crush_msr_clear_workspace(&workspace);
+
+
+			unsigned tries_so_far = 0;
+			unsigned end_index = MIN(start_index + total_children,
+						 input.result_max);
+			unsigned return_limit_for_block =
+				output.returned_so_far + (end_index - start_index);
+			while (tries_so_far < input.msr_descents &&
+			       output.returned_so_far < return_limit_for_block) {
+				crush_msr_choose(
+					&input, &workspace, &output,
+					root_bucket,
+					total_children,
+					start_index,
+					end_index,
+					start_stepno, emit_stepno,
+					tries_so_far);
+				dprintk("returned_so_far: %d\n",
+					output.returned_so_far);
+				++tries_so_far;
+			}
+			start_index = end_index;
+			start_stepno = emit_stepno + 1;
+		}
+	}
+
+	if (rule->type == CRUSH_RULE_TYPE_MSR_FIRSTN) {
+		return output.returned_so_far;
+	} else {
+		return input.result_max;
+	}
+}
+
+/// Return 1 if msr, 0 otherwise
+static int rule_type_is_msr(int type)
+{
+	return type == CRUSH_RULE_TYPE_MSR_FIRSTN ||
+		type == CRUSH_RULE_TYPE_MSR_INDEP;
+}
+
+size_t crush_work_size(const struct crush_map *map,
+		       int result_max)
+{
+	unsigned ruleno;
+	unsigned out_vecs = 3; /* normal do_rule needs 3 outvecs */
+	for (ruleno = 0; ruleno < map->max_rules; ++ruleno) {
+		const struct crush_rule *rule = map->rules[ruleno];
+		if (!rule) continue;
+		if (!rule_type_is_msr(rule->type))
+			continue;
+		unsigned rule_max_msr_steps;
+		// we ignore the return value because rule_max_msr_steps will be
+		// populated with the longest step sequence before hitting
+		// the error
+		(void)crush_msr_scan_rule(rule, result_max, &rule_max_msr_steps);
+		out_vecs = MAX(rule_max_msr_steps, out_vecs);
+	}
+	return map->working_size + result_max * out_vecs * sizeof(__u32);
+}
+
+/* This takes a chunk of memory and sets it up to be a shiny new
+   working area for a CRUSH placement computation. It must be called
+   on any newly allocated memory before passing it in to
+   crush_do_rule. It may be used repeatedly after that, so long as the
+   map has not changed. If the map /has/ changed, you must make sure
+   the working size is no smaller than what was allocated and re-run
+   crush_init_workspace.
+
+   If you do retain the working space between calls to crush, make it
+   thread-local. If you reinstitute the locking I've spent so much
+   time getting rid of, I will be very unhappy with you. */
+
+void crush_init_workspace(const struct crush_map *m, void *v) {
+	/* We work by moving through the available space and setting
+	   values and pointers as we go.
+
+	   It's a bit like Forth's use of the 'allot' word since we
+	   set the pointer first and then reserve the space for it to
+	   point to by incrementing the point. */
+	struct crush_work *w = (struct crush_work *)v;
+	char *point = (char *)v;
+	__s32 b;
+	point += sizeof(struct crush_work);
+	w->work = (struct crush_work_bucket **)point;
+	point += m->max_buckets * sizeof(struct crush_work_bucket *);
+	for (b = 0; b < m->max_buckets; ++b) {
+		if (m->buckets[b] == 0)
+			continue;
+
+		w->work[b] = (struct crush_work_bucket *) point;
+		switch (m->buckets[b]->alg) {
+		default:
+			point += sizeof(struct crush_work_bucket);
+			break;
+		}
+		w->work[b]->perm_x = 0;
+		w->work[b]->perm_n = 0;
+		w->work[b]->perm = (__u32 *)point;
+		point += m->buckets[b]->size * sizeof(__u32);
+	}
+	BUG_ON((char *)point - (char *)w != m->working_size);
+}
+
+/**
+ * crush_do_rule - calculate a mapping with the given input and rule
+ * @map: the crush_map
+ * @ruleno: the rule id
+ * @x: hash input
+ * @result: pointer to result vector
+ * @result_max: maximum result size
+ * @weight: weight vector (for map leaves)
+ * @weight_max: size of weight vector
+ * @cwin: Pointer to at least map->working_size bytes of memory or NULL.
+ */
+int crush_do_rule(const struct crush_map *map,
+		  int ruleno, int x, int *result, int result_max,
+		  const __u32 *weight, int weight_max,
+		  void *cwin, const struct crush_choose_arg *choose_args)
+{
+	const struct crush_rule *rule;
+
+	if ((__u32)ruleno >= map->max_rules) {
+		dprintk(" bad ruleno %d\n", ruleno);
+		return 0;
+	}
+
+	rule = map->rules[ruleno];
+	if (rule_type_is_msr(rule->type)) {
+		return crush_msr_do_rule(
+			map,
+			ruleno,
+			x,
+			result,
+			result_max,
+			weight,
+			weight_max,
+			cwin,
+			choose_args);
+	} else {
+		return crush_do_rule_no_retry(
+			map,
+			ruleno,
+			x,
+			result,
+			result_max,
+			weight,
+			weight_max,
+			cwin,
+			choose_args);
+	}
+}
diff --git a/src/crush/mapper.h b/src/crush/mapper.h
index 0ec927d9e616..840449620fbc 100644
--- a/src/crush/mapper.h
+++ b/src/crush/mapper.h
@@ -59,6 +59,23 @@
  *         char __cwin__[crush_work_size(__map__, __result_max__)];
  *         crush_init_workspace(__map__, __cwin__);
  *
+ * There are two CRUSH variants implemented.  Rules of type
+ * - CRUSH_RULE_TYPE_REPLICATED
+ * - CRUSH_RULE_TYPE_ERASURE
+ * use crush_do_rule_no_retry.  The crush descent algorithm implemented
+ * there cannot retry prior steps upon hitting an out osd, so such rules
+ * rely on the chooseleaf variant to implement failure domains and have
+ * important limitations when mapping multiple OSDs per failure domain.
+ * See crush_msr_do_rule in mapper.c for a more detailed explanation.
+ *
+ * Rules of type
+ * - CRUSH_RULE_TYPE_MSR_FIRSTN
+ * - CRUSH_RULE_TYPE_MSR_INDEP
+ * use crush_msr_do_rule, which retries the full descent when it hits an
+ * out OSD.  This extra flexibility allows it to more effectively map multiple
+ * OSDs per failure domain.  See the comment on crush_msr_do_rule in mapper.c
+ * for more details.
+ *
  * @param map the crush_map
  * @param ruleno a positive integer < __CRUSH_MAX_RULES__
  * @param x the value to map to __result_max__ items
@@ -77,15 +94,11 @@ extern int crush_do_rule(const struct crush_map *map,
 			 const __u32 *weights, int weight_max,
 			 void *cwin, const struct crush_choose_arg *choose_args);
 
-/* Returns the exact amount of workspace that will need to be used
-   for a given combination of crush_map and result_max. The caller can
-   then allocate this much on its own, either on the stack, in a
-   per-thread long-lived buffer, or however it likes. */
-
-static inline size_t crush_work_size(const struct crush_map *map,
-				     int result_max) {
-	return map->working_size + result_max * 3 * sizeof(__u32);
-}
+/* Returns enough workspace for any crush rule within map to generate
+   result_max outputs. The caller can then allocate this much on its own,
+   either on the stack, in a per-thread long-lived buffer, or however it likes.*/
+extern size_t crush_work_size(const struct crush_map *map,
+			      int result_max);
 
 extern void crush_init_workspace(const struct crush_map *m, void *v);
 

From 8eb68353963e0ab16f4a3e0afb9d250e4bf7fb08 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 1 Feb 2024 11:38:13 -0800
Subject: [PATCH 1753/2492] doc/dev/crush-msr.rst: add developer summary of
 crush msr

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 doc/dev/crush-msr.rst | 140 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 doc/dev/crush-msr.rst

diff --git a/doc/dev/crush-msr.rst b/doc/dev/crush-msr.rst
new file mode 100644
index 000000000000..a18b56740c27
--- /dev/null
+++ b/doc/dev/crush-msr.rst
@@ -0,0 +1,140 @@
+============================
+CRUSH MSR (Multi-step Retry)
+============================
+
+Motivation
+----------
+
+Conventional CRUSH has an important limitation: rules with
+multiple `choose` steps which hit an `out` osd cannot retry
+prior steps.  As an example, with a rule like
+::
+
+    rule replicated_rule_1 {
+        ...
+        step take default class hdd
+        step chooseleaf firstn 3 type host
+        step emit
+    }
+
+one might expect that if all of the OSDs on a particular host
+are marked out, mappings including those OSDs would end up
+on another host (provided that there are enough hosts).  Indeed,
+that's what will happen.  Moreover, if 1/8 OSDs on a host are
+marked out, roughly 1/8 of the PGs mapped to that host will end
+up remapped to some other host keeping overall per-OSD utilization
+even.
+
+Suppose, instead, the rule were written like this:
+::
+
+    rule replicated_rule_1 {
+        ...
+        step take default class hdd
+        step choose firstn 3 type host
+        step choose firstn 1 type osd
+        step emit
+    }
+
+The behavior would be very similar as long as no OSDs are marked
+out.  However, if an OSD is marked out, any PGs mapped to that
+OSD will be remapped to other OSDs on the same host resulting in
+those OSDs being over-utilized relative to OSDs on other hosts.
+Moreover, if all of the OSDs on a host are marked out, mappings
+that happen to hit that host will fail resulting in undersized PGs.
+
+As long as the goal is to split N OSDs between N failure domains,
+the solution is simply to use the `chooseleaf` variant above.  However,
+consider a use case where we want to split an 8+6 EC encoding over 4
+hosts in order to tolerate the loss of a host and an OSD on another
+host with 1.75x storage overhead.  The rule would have to look
+something like:
+::
+
+    rule ecpool-86 {
+        ...
+        step take default class hdd
+        step choose indep 4 type host
+        step choose indep 4 type osd
+        step emit
+    }
+
+This does split up to 16 OSDs between 4 hosts (with an 8+6 code,
+it would put 4 OSDs on each of the first 3 and 2 on the last) and
+meets our failure requirements.  However, for the reasons outlined
+above, it will behave poorly as OSDs are marked out if there are
+other hosts to rebalance to.  `chooseleaf` is not a solution here
+because it does not support mapping more than one leaf below the
+specified type.
+
+MSR
+---
+
+CRUSH MSR (Multi-step Retry) rules solve the above problem by using a
+different descent algorithm which retries all of the steps upon
+hitting an out OSD.  Where classic CRUSH is breadth first (for each
+step, it fully populates the vector before proceeding to the next
+step), MSR rules are depth first -- for each choice, we recursively
+descend through all of the steps before continuing with the next
+choice.  The above use case can be satisfied with the following rule:
+
+::
+
+    rule ecpool-86 {
+        type msr_indep
+        ...
+        step take default class hdd
+        step choosemsr 4 type host
+        step choosemsr 4 type osd
+        step emit
+    }
+
+As with the `chooseleaf` example at the top, as OSDs are marked out,
+those OSDs are be remapped proportionately to other hosts so long as
+there are extras available.  For details on how that works while
+still preserving failure domain isolation, see the comments in
+mapper.c:crush_msr_choose.
+
+Rule Structure
+--------------
+
+CRUSH MSR rules are crush rules with type CRUSH_RULE_TYPE_MSR_FIRSTN
+or CRUSH_RULE_TYPE_MSR_INDEP (see mapper.c: rule_type_is_msr).  Unlike
+with classic crush rules, individual steps do not specify firstn or
+indep.  The output order is instead defined by the rule type for the
+whole rule.
+
+MSR rules have some structural differences from conventional rules:
+
+- The rule type determines whether the mapping is FIRSTN or INDEP.
+  Because the descent can retry steps, it doesn't really make sense
+  for steps to individually specify output order and I'm not really
+  aware of any use cases that would benefit from it.
+- MSR rules *must* be structured as a (possibly empty) prefix of
+  config steps (CRUSH_RULE_SET_CHOOSE_MSR*) followed by a sequence of
+  EMIT blocks each comprised of a TAKE step, a sequence of CHOOSE_MSR
+  steps, and ended by an EMIT step.
+- MSR steps must be `choosemsr`.  `choose` and `chooseleaf` are not
+  permitted.
+
+Working Space
+-------------
+
+MSR rules also have different requirements for working space.
+Conventional CRUSH requires 3 vectors of size result_max to use for
+working space -- two to alternate as it processes each rule and one,
+additionally, for `chooseleaf`.  MSR rules need N vectors where N is the
+number of `choosemsr` steps in the longest EMIT block since it needs to
+retain all of the choices made as part of each descent.
+
+See mapper.h/c:crush_work_size, crush_msr_scan_rule for details.
+
+Implementation
+--------------
+
+mapper.h/c:crush_do_rule internally branches to
+mapper.c:crush_msr_do_rule for rules of type CRUSH_RULE_TYPE_MSR_*
+(see mapper.c:rule_type_is_msr).
+
+MSR related functions in mapper.c are annotated with more details
+about the algorithm.

From 8fba03fe42a4981cb40c58c5f8b8c6f7524f6178 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 21 Nov 2023 15:39:23 -0800
Subject: [PATCH 1754/2492] mon/OSDMonitor: generalize rule type check for
 pools

Add rule_valid_for_pool_type to CrushWrapper to generalize
rule type <-> pool type mapping to include the new MSR
types.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.h | 14 ++++++++++++++
 src/mon/OSDMonitor.cc    |  6 +++---
 src/osd/OSDMap.cc        |  2 +-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index f71f6d6ea79b..77f70d10049e 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -602,6 +602,20 @@ class CrushWrapper {
     if (have_rmaps)
       rule_name_rmap[name] = i;
   }
+  bool rule_valid_for_pool_type(int rule_id, int ptype) const {
+    auto rule_type = get_rule_type(rule_id);
+    switch (ptype) {
+    case CEPH_PG_TYPE_REPLICATED:
+      return rule_type == CRUSH_RULE_TYPE_REPLICATED ||
+	rule_type == CRUSH_RULE_TYPE_MSR_FIRSTN;
+    case CEPH_PG_TYPE_ERASURE:
+      return rule_type == CRUSH_RULE_TYPE_ERASURE ||
+	rule_type == CRUSH_RULE_TYPE_MSR_INDEP;
+    default:
+      ceph_abort_msg("impossible");
+    }
+  }
+
   bool is_shadow_item(int id) const {
     const char *name = get_item_name(id);
     return name && !is_valid_crush_name(name);
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f8e379326f25..5e5cd0bfbebc 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -8072,7 +8072,7 @@ int OSDMonitor::prepare_new_pool(string& name,
     return r;
   }
 
-  if (osdmap.crush->get_rule_type(crush_rule) != (int)pool_type) {
+  if (!osdmap.crush->rule_valid_for_pool_type(crush_rule, pool_type)) {
     *ss << "crush rule " << crush_rule << " type does not match pool";
     return -EINVAL;
   }
@@ -8344,7 +8344,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
 	return -EPERM;
       }
     }
-    if (osdmap.crush->get_rule_type(p.get_crush_rule()) != (int)p.type) {
+    if (!osdmap.crush->rule_valid_for_pool_type(p.get_crush_rule(), p.type)) {
       ss << "crush rule " << p.get_crush_rule() << " type does not match pool";
       return -EINVAL;
     }
@@ -8577,7 +8577,7 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
       ss << cpp_strerror(id);
       return -ENOENT;
     }
-    if (osdmap.crush->get_rule_type(id) != (int)p.get_type()) {
+    if (!osdmap.crush->rule_valid_for_pool_type(id, p.get_type())) {
       ss << "crush rule " << id << " type does not match pool";
       return -EINVAL;
     }
diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index a6dd32c554d3..9853a0c8b2df 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -4530,7 +4530,7 @@ int OSDMap::validate_crush_rules(CrushWrapper *newcrush,
 	  << " but it is not present";
       return -EINVAL;
     }
-    if (newcrush->get_rule_type(ruleno) != (int)pool.get_type()) {
+    if (!newcrush->rule_valid_for_pool_type(ruleno, pool.get_type())) {
       *ss << "pool " << i.first << " type does not match rule " << ruleno;
       return -EINVAL;
     }

From ab2b62c12099027fe9147da1b50bae0c81a4cc3c Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 1 Feb 2024 13:00:55 -0800
Subject: [PATCH 1755/2492] test/crush/crush.cc: s/NULL/nullptr/g

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 1f53084a70af..16153f1692f4 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -37,7 +37,7 @@ std::unique_ptr<CrushWrapper> build_indep_map(CephContext *cct, int num_rack,
 
   int rootno;
   c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
-		5, 0, NULL, NULL, &rootno);
+		5, 0, nullptr, nullptr, &rootno);
   c->set_item_name(rootno, "default");
 
   map<string,string> loc;
@@ -115,7 +115,7 @@ class CRUSHTest : public ::testing::Test
 TEST_F(CRUSHTest, indep_toosmall) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 1, 3, 1));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
+  c->dump_tree(&cout, nullptr);
 
   for (int x = 0; x < 100; ++x) {
     vector<int> out;
@@ -134,7 +134,7 @@ TEST_F(CRUSHTest, indep_toosmall) {
 TEST_F(CRUSHTest, indep_basic) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
+  c->dump_tree(&cout, nullptr);
 
   for (int x = 0; x < 100; ++x) {
     vector<int> out;
@@ -158,7 +158,7 @@ TEST_F(CRUSHTest, indep_out_alt) {
   int num = 3*3*3;
   for (int i=0; i<num / 2; ++i)
     weight[i*2] = 0;
-  c->dump_tree(&cout, NULL);
+  c->dump_tree(&cout, nullptr);
 
   // need more retries to get 9/9 hosts for x in 0..99
   c->set_choose_total_tries(100);
@@ -184,7 +184,7 @@ TEST_F(CRUSHTest, indep_out_contig) {
   int num = 3*3*3;
   for (int i=0; i<num / 3; ++i)
     weight[i] = 0;
-  c->dump_tree(&cout, NULL);
+  c->dump_tree(&cout, nullptr);
 
   c->set_choose_total_tries(100);
   for (int x = 0; x < 100; ++x) {
@@ -206,7 +206,7 @@ TEST_F(CRUSHTest, indep_out_progressive) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   c->set_choose_total_tries(100);
   vector<__u32> tweight(c->get_max_devices(), 0x10000);
-  c->dump_tree(&cout, NULL);
+  c->dump_tree(&cout, nullptr);
 
   int tchanged = 0;
   for (int x = 1; x < 5; ++x) {

From 28989d08e7a533d1317bf66c125c9efbc974b432 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:24:30 -0800
Subject: [PATCH 1756/2492] test/crush/crush.cc: convert indep test cases to
 test MSR as well

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 285 ++++++++++++++++++++++++++++++----------
 1 file changed, 216 insertions(+), 69 deletions(-)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 16153f1692f4..ce92afcfb42f 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -18,69 +18,11 @@
 #include "include/stringify.h"
 
 #include "crush/CrushWrapper.h"
+#include "crush/CrushCompiler.h"
 #include "osd/osd_types.h"
 
 using namespace std;
 
-std::unique_ptr<CrushWrapper> build_indep_map(CephContext *cct, int num_rack,
-                              int num_host, int num_osd)
-{
-  std::unique_ptr<CrushWrapper> c(new CrushWrapper);
-  c->create();
-
-  c->set_type_name(5, "root");
-  c->set_type_name(4, "row");
-  c->set_type_name(3, "rack");
-  c->set_type_name(2, "chasis");
-  c->set_type_name(1, "host");
-  c->set_type_name(0, "osd");
-
-  int rootno;
-  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
-		5, 0, nullptr, nullptr, &rootno);
-  c->set_item_name(rootno, "default");
-
-  map<string,string> loc;
-  loc["root"] = "default";
-
-  int osd = 0;
-  for (int r=0; r<num_rack; ++r) {
-    loc["rack"] = string("rack-") + stringify(r);
-    for (int h=0; h<num_host; ++h) {
-      loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
-      for (int o=0; o<num_osd; ++o, ++osd) {
-	c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
-      }
-    }
-  }
-  int ret;
-  int ruleno = 0;
-  ret = c->add_rule(ruleno, 4, 123);
-  ceph_assert(ret == ruleno);
-  ret = c->set_rule_step(ruleno, 0, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 1, CRUSH_RULE_TAKE, rootno, 0);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 2, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
-  ceph_assert(ret == 0);
-  ret = c->set_rule_step(ruleno, 3, CRUSH_RULE_EMIT, 0, 0);
-  ceph_assert(ret == 0);
-  c->set_rule_name(ruleno, "data");
-
-  c->finalize();
-
-  if (false) {
-    Formatter *f = Formatter::create("json-pretty");
-    f->open_object_section("crush_map");
-    c->dump(f);
-    f->close_section();
-    f->flush(cout);
-    delete f;
-  }
-
-  return c;
-}
-
 int get_num_dups(const vector<int>& v)
 {
   std::set<int> s;
@@ -94,7 +36,21 @@ int get_num_dups(const vector<int>& v)
   return dups;
 }
 
-class CRUSHTest : public ::testing::Test
+class RuleType {
+  bool msr;
+
+public:
+  RuleType(bool msr) : msr(msr) {}
+
+  bool is_msr() const { return msr; }
+
+  friend std::ostream &operator<<(std::ostream &, RuleType);
+};
+std::ostream &operator<<(std::ostream &lhs, RuleType rhs) {
+  return lhs << (rhs.msr ? "MSR" : "NORMAL");
+}
+
+class IndepTest : public ::testing::TestWithParam<RuleType>
 {
 public:
   void SetUp() final
@@ -108,11 +64,91 @@ class CRUSHTest : public ::testing::Test
     cct->put();
     cct = nullptr;
   }
+
+  std::unique_ptr<CrushWrapper> build_indep_map(
+    CephContext *cct, int num_rack, int num_host, int num_osd)
+  {
+    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
+    c->create();
+    c->set_tunables_optimal();
+
+    c->set_type_name(5, "root");
+    c->set_type_name(4, "row");
+    c->set_type_name(3, "rack");
+    c->set_type_name(2, "chasis");
+    c->set_type_name(1, "host");
+    c->set_type_name(0, "osd");
+
+    int rootno;
+    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		  5, 0, nullptr, nullptr, &rootno);
+    c->set_item_name(rootno, "default");
+
+    map<string,string> loc;
+    loc["root"] = "default";
+
+    int osd = 0;
+    for (int r=0; r<num_rack; ++r) {
+      loc["rack"] = string("rack-") + stringify(r);
+      for (int h=0; h<num_host; ++h) {
+	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+	for (int o=0; o<num_osd; ++o, ++osd) {
+	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+	}
+      }
+    }
+    int ret;
+    int ruleno = 0;
+
+    if (GetParam().is_msr()) {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    } else {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 10, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_INDEP, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    }
+
+    c->set_rule_name(ruleno, "data");
+    c->finalize();
+
+    if (false) {
+      Formatter *f = Formatter::create("json-pretty");
+      f->open_object_section("crush_map");
+      c->dump(f);
+      f->close_section();
+      f->flush(cout);
+      delete f;
+    }
+
+    return c;
+  }
+
 protected:
   CephContext *cct = nullptr;
 };
 
-TEST_F(CRUSHTest, indep_toosmall) {
+TEST_P(IndepTest, toosmall) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 1, 3, 1));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, nullptr);
@@ -131,7 +167,7 @@ TEST_F(CRUSHTest, indep_toosmall) {
   }
 }
 
-TEST_F(CRUSHTest, indep_basic) {
+TEST_P(IndepTest, basic) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
   c->dump_tree(&cout, nullptr);
@@ -150,7 +186,88 @@ TEST_F(CRUSHTest, indep_basic) {
   }
 }
 
-TEST_F(CRUSHTest, indep_out_alt) {
+TEST_P(IndepTest, single_out_first) {
+  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+
+    int num_none = 0;
+    for (unsigned i=0; i<out.size(); ++i) {
+      if (out[i] == CRUSH_ITEM_NONE)
+	num_none++;
+    }
+    ASSERT_EQ(0, num_none);
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[0]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 5, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    // First item should have been remapped
+    ASSERT_NE(CRUSH_ITEM_NONE, out2[0]);
+    ASSERT_NE(out[0], out2[0]);
+    for (unsigned i=1; i<out.size(); ++i) {
+      // but none of the others
+      ASSERT_EQ(out[i], out2[i]);
+    }
+    ASSERT_EQ(0, get_num_dups(out2));
+  }
+}
+
+TEST_P(IndepTest, single_out_last) {
+  std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+
+    int num_none = 0;
+    for (unsigned i=0; i<out.size(); ++i) {
+      if (out[i] == CRUSH_ITEM_NONE)
+	num_none++;
+    }
+    ASSERT_EQ(0, num_none);
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    unsigned last = out.size() - 1;
+    weight[out[last]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 5, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    // Last
+    ASSERT_NE(CRUSH_ITEM_NONE, out2[last]);
+    ASSERT_NE(out[last], out2[last]);
+    for (unsigned i=0; i<last; ++i) {
+      // but none of the others
+      ASSERT_EQ(out[i], out2[i]);
+    }
+    ASSERT_EQ(0, get_num_dups(out2));
+  }
+}
+
+TEST_P(IndepTest, out_alt) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -176,7 +293,7 @@ TEST_F(CRUSHTest, indep_out_alt) {
   }
 }
 
-TEST_F(CRUSHTest, indep_out_contig) {
+TEST_P(IndepTest, out_contig) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   vector<__u32> weight(c->get_max_devices(), 0x10000);
 
@@ -201,8 +318,7 @@ TEST_F(CRUSHTest, indep_out_contig) {
   }
 }
 
-
-TEST_F(CRUSHTest, indep_out_progressive) {
+TEST_P(IndepTest, out_progressive) {
   std::unique_ptr<CrushWrapper> c(build_indep_map(cct, 3, 3, 3));
   c->set_choose_total_tries(100);
   vector<__u32> tweight(c->get_max_devices(), 0x10000);
@@ -217,8 +333,15 @@ TEST_F(CRUSHTest, indep_out_progressive) {
     for (unsigned i=0; i<weight.size(); ++i) {
       vector<int> out;
       c->do_rule(0, x, out, 7, weight, 0);
-      cout << "(" << i << "/" << weight.size() << " out) "
-	   << x << " -> " << out << std::endl;
+      cout << "(" << i << "/" << weight.size() << " out) ";
+      if (i > 0) cout << "marked out " << i - 1 << " ";
+      cout << x << " -> " << out << std::endl;
+
+      int num_none = 0;
+      for (unsigned k=0; k<out.size(); ++k) {
+	if (out[k] == CRUSH_ITEM_NONE)
+	  num_none++;
+      }
       ASSERT_EQ(0, get_num_dups(out));
 
       // make sure nothing moved
@@ -238,7 +361,6 @@ TEST_F(CRUSHTest, indep_out_progressive) {
 	    cout << " " << out[j] << " moved from " << pos[out[j]] << " to " << j << std::endl;
 	    ++moved;
 	  }
-	  //ASSERT_EQ(j, pos[out[j]]);
 	}
       }
       if (moved || changed)
@@ -260,6 +382,31 @@ TEST_F(CRUSHTest, indep_out_progressive) {
 
 }
 
+INSTANTIATE_TEST_SUITE_P(
+  IndepTest,
+  IndepTest,
+  ::testing::Values(RuleType(true), RuleType(false)),
+  testing::PrintToStringParamName());
+
+
+class CRUSHTest : public ::testing::Test
+{
+public:
+  void SetUp() final
+  {
+    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
+    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  }
+  void TearDown() final
+  {
+    cct->put();
+    cct = nullptr;
+  }
+protected:
+  CephContext *cct = nullptr;
+};
+
 TEST_F(CRUSHTest, straw_zero) {
   // zero weight items should have no effect on placement.
 

From 0445c3347e4ad825ec05dd4623806f3a65fcbb89 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Dec 2023 17:00:44 -0800
Subject: [PATCH 1757/2492] test/crush/crush.cc: add test variants for firstn
 rules

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 303 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 303 insertions(+)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index ce92afcfb42f..03bbca97e32f 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -388,6 +388,309 @@ INSTANTIATE_TEST_SUITE_P(
   ::testing::Values(RuleType(true), RuleType(false)),
   testing::PrintToStringParamName());
 
+class FirstnTest : public ::testing::TestWithParam<RuleType>
+{
+public:
+  void SetUp() final
+  {
+    CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
+    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  }
+  void TearDown() final
+  {
+    cct->put();
+    cct = nullptr;
+  }
+
+  std::unique_ptr<CrushWrapper> build_firstn_map(
+    CephContext *cct, int num_rack, int num_host, int num_osd)
+  {
+    std::unique_ptr<CrushWrapper> c(new CrushWrapper);
+    c->create();
+    c->set_tunables_optimal();
+
+    c->set_type_name(5, "root");
+    c->set_type_name(4, "row");
+    c->set_type_name(3, "rack");
+    c->set_type_name(2, "chasis");
+    c->set_type_name(1, "host");
+    c->set_type_name(0, "osd");
+
+    int rootno;
+    c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+		  5, 0, nullptr, nullptr, &rootno);
+    c->set_item_name(rootno, "default");
+
+    map<string,string> loc;
+    loc["root"] = "default";
+
+    int osd = 0;
+    for (int r=0; r<num_rack; ++r) {
+      loc["rack"] = string("rack-") + stringify(r);
+      for (int h=0; h<num_host; ++h) {
+	loc["host"] = string("host-") + stringify(r) + string("-") + stringify(h);
+	for (int o=0; o<num_osd; ++o, ++osd) {
+	  c->insert_item(cct, osd, 1.0, string("osd.") + stringify(osd), loc);
+	}
+      }
+    }
+    int ret;
+    int ruleno = 0;
+
+    if (GetParam().is_msr()) {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_MSR_FIRSTN);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 1, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    } else {
+      unsigned step_id = 0;
+      ret = c->add_rule(ruleno, 4, CRUSH_RULE_TYPE_ERASURE);
+      ceph_assert(ret == ruleno);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 0, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_TAKE, rootno, 0);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(
+	ruleno, step_id++, CRUSH_RULE_CHOOSELEAF_FIRSTN, CRUSH_CHOOSE_N, 1);
+      ceph_assert(ret == 0);
+      ret = c->set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+      ceph_assert(ret == 0);
+    }
+
+    c->set_rule_name(ruleno, "data");
+    c->finalize();
+
+    if (false) {
+      Formatter *f = Formatter::create("json-pretty");
+      f->open_object_section("crush_map");
+      c->dump(f);
+      f->close_section();
+      f->flush(cout);
+      delete f;
+    }
+
+    return c;
+  }
+
+protected:
+  CephContext *cct = nullptr;
+};
+
+TEST_P(FirstnTest, basic) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, toosmall) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 1, 3, 1));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 5, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, single_out_first) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[0]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 3, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    ASSERT_EQ(3, out2.size());
+    ASSERT_EQ(0, get_num_dups(out2));
+    for (unsigned i=0; i<out2.size(); ++i) {
+      EXPECT_NE(out2[i], out[0]);
+    }
+    if (GetParam().is_msr()) {
+      // normal crush doesn't guarantee this reliably
+      ASSERT_EQ(out2[0], out[1]);
+      ASSERT_EQ(out2[1], out[2]);
+      ASSERT_NE(out2[2], out[0]);
+    }
+  }
+}
+
+TEST_P(FirstnTest, single_out_last) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  c->dump_tree(&cout, nullptr);
+
+  for (int x = 0; x < 1000; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+    vector<int> out;
+    c->do_rule(0, x, out, 3, weight, 0);
+
+    for (unsigned i=0; i<out.size(); ++i) {
+      EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+    }
+    ASSERT_EQ(3, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+
+    // mark first osd out
+    weight[out[2]] = 0;
+
+    vector<int> out2;
+    c->do_rule(0, x, out2, 3, weight, 0);
+
+    cout << "input " << x
+	 << " marked out " << out[0]
+	 << " out " << out
+	 << " -> out2 " << out2
+	 << std::endl;
+
+    ASSERT_EQ(3, out2.size());
+    ASSERT_EQ(0, get_num_dups(out2));
+    for (unsigned i=0; i<out2.size(); ++i) {
+      EXPECT_NE(out2[i], out[2]);
+    }
+    ASSERT_EQ(out2[0], out[0]);
+    ASSERT_EQ(out2[1], out[1]);
+    ASSERT_NE(out2[2], out[2]);
+  }
+}
+
+TEST_P(FirstnTest, out_alt) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; i<num / 2; ++i)
+    weight[i*2] = 0;
+  c->dump_tree(&cout, nullptr);
+
+  // need more retries to get 9/9 hosts for x in 0..99
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 9, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    ASSERT_EQ(9, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, out_contig) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+  // mark a bunch of osds out
+  int num = 3*3*3;
+  for (int i=0; i<num / 3; ++i)
+    weight[i] = 0;
+  c->dump_tree(&cout, nullptr);
+
+  // need more retries to get 7/7 hosts for x in 0..99
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  for (int x = 0; x < 100; ++x) {
+    vector<int> out;
+    c->do_rule(0, x, out, 7, weight, 0);
+    cout << x << " -> " << out << std::endl;
+    ASSERT_EQ(6, out.size());
+    ASSERT_EQ(0, get_num_dups(out));
+  }
+}
+
+TEST_P(FirstnTest, out_progressive) {
+  std::unique_ptr<CrushWrapper> c(build_firstn_map(cct, 3, 3, 3));
+  if (!GetParam().is_msr()) {
+    c->set_choose_total_tries(500);
+  }
+  vector<__u32> tweight(c->get_max_devices(), 0x10000);
+  c->dump_tree(&cout, nullptr);
+
+  int tchanged = 0;
+  for (int x = 1; x < 5; ++x) {
+    vector<__u32> weight(c->get_max_devices(), 0x10000);
+
+    std::set<int> prev;
+    for (unsigned i=0; i<weight.size(); ++i) {
+      vector<int> out;
+      c->do_rule(0, x, out, 7, weight, 0);
+      cout << "(" << i << "/" << weight.size() << " out) ";
+      if (i > 0) cout << "marked out " << i - 1 << " ";
+      cout << x << " -> " << out << std::endl;
+
+      ASSERT_EQ(0, get_num_dups(out));
+
+      int changed = 0;
+      for (unsigned j=0; j<out.size(); ++j) {
+	if (i && prev.count(out[j]) == 0) {
+	  ++changed;
+	  ++tchanged;
+	}
+      }
+      if (changed)
+	cout << " " << changed << " changed" << std::endl;
+      ASSERT_LE(changed, 3);
+
+      // mark another osd out
+      weight[i] = 0;
+      prev = std::set<int>{out.begin(), out.end()};
+    }
+  }
+  cout << tchanged << " total changed" << std::endl;
+}
+
+INSTANTIATE_TEST_SUITE_P(
+  FirstnTest,
+  FirstnTest,
+  ::testing::Values(RuleType(true), RuleType(false)),
+  testing::PrintToStringParamName());
 
 class CRUSHTest : public ::testing::Test
 {

From 4b4eb17d328612284af22870908ca93de869a911 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Nov 2023 20:26:07 -0800
Subject: [PATCH 1758/2492] test/crush/crush.cc: add tests specifically for MSR

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crush/crush.cc | 456 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 456 insertions(+)

diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 03bbca97e32f..2be7d5540d8d 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -1103,3 +1103,459 @@ TEST_F(CRUSHTest, straw2_reweight) {
     cout << "     vs " << estddev << std::endl;
   }
 }
+
+struct cluster_test_spec_t {
+  const int num_osds_per_host;
+  const int num_hosts;
+
+  const int num_hosts_mapped;
+  const int num_mapped_per_host;
+  const int num_mapped_size;
+
+  const int num_osds;
+
+  cluster_test_spec_t(
+    int num_osds_per_host, int num_hosts,
+    int num_hosts_mapped, int num_mapped_per_host, int num_mapped_size)
+    : num_osds_per_host(num_osds_per_host), num_hosts(num_hosts),
+      num_hosts_mapped(num_hosts_mapped),
+      num_mapped_per_host(num_mapped_per_host),
+      num_mapped_size(num_mapped_size),
+      num_osds(num_osds_per_host * num_hosts) {}
+
+  void validate_osd(int osd) const {
+    EXPECT_GE(osd, 0);
+    EXPECT_LT(osd, num_osds);
+  }
+
+  bool check_osd(int osd) const {
+    return osd >= 0 && osd < num_osds;
+  }
+
+  void validate_host(int host) const {
+    assert(host >= 0);
+    assert(host < num_hosts);
+  }
+
+  std::pair<int, int> host_to_osd_range(int host) const {
+    validate_host(host);
+    auto first = host * num_osds_per_host;
+    return std::make_pair(first, first + num_osds_per_host);
+  }
+
+  int osd_to_host(int osd) const {
+    validate_osd(osd);
+    return osd / num_osds_per_host;
+  }
+};
+
+static constexpr int ROOT_TYPE = 2;
+static constexpr int HOST_TYPE = 1;
+static constexpr int OSD_TYPE = 0;
+std::pair<int, std::unique_ptr<CrushWrapper>> create_crush_heirarchy(
+  CephContext *cct,
+  const cluster_test_spec_t &spec)
+{
+  auto c = std::make_unique<CrushWrapper>();
+  c->create();
+  c->set_tunables_optimal();
+
+
+  c->set_type_name(ROOT_TYPE, "root");
+  c->set_type_name(HOST_TYPE, "host");
+  c->set_type_name(OSD_TYPE, "osd");
+
+  int rootno;
+  c->add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
+	       ROOT_TYPE, 0, nullptr, nullptr, &rootno);
+  c->set_item_name(rootno, "default");
+
+  for (auto host_id = 0; host_id < spec.num_hosts; ++host_id) {
+    const std::string host_name = fmt::format("host{}", host_id);
+    const auto first_host_osd = host_id * spec.num_osds_per_host;
+    const auto next_first_host_osd = first_host_osd + spec.num_osds_per_host;
+    for (auto osd_id = first_host_osd; osd_id < next_first_host_osd; ++osd_id) {
+      const std::string osd_name = fmt::format("osd{}", osd_id);
+      auto ret = c->insert_item(
+	cct, osd_id, 1.0, osd_name,
+	{{ "root", "default"}, {"host", host_name}});
+      EXPECT_EQ(ret, 0);
+    }
+  }
+
+  c->finalize();
+  return std::make_pair(rootno, std::move(c));
+}
+
+std::vector<uint32_t> create_weight_vector(
+  const cluster_test_spec_t &spec)
+{
+  return std::vector<uint32_t>(spec.num_osds, CEPH_OSD_IN);
+}
+
+std::vector<uint32_t> create_weight_vector_first_osd_out(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &mapping)
+{
+  auto weights = create_weight_vector(spec);
+  spec.validate_osd(mapping[0]);
+  weights[mapping[0]] = CEPH_OSD_OUT;
+  return weights;
+}
+
+std::vector<uint32_t> create_weight_vector_first_host_out(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &mapping)
+{
+  auto weights = create_weight_vector(spec);
+  const auto [first, end] = spec.host_to_osd_range(spec.osd_to_host(mapping[0]));
+  for (auto i = first; i < end; ++i) {
+    weights[i] = CEPH_OSD_OUT;
+  }
+  return weights;
+}
+
+enum class mapping_change_t {
+  SAME,
+  FAILURE,
+  SAME_HOST,
+  NEW_HOST
+};
+void compare_mappings(
+  const cluster_test_spec_t &spec,
+  const std::vector<int> &before,
+  const std::vector<int> &after,
+  mapping_change_t expectation,
+  const std::pair<int, int> &range)
+{
+  const auto &[begin, end] = range;
+  for (auto i = begin; i < end; ++i) {
+    switch (expectation) {
+    case mapping_change_t::SAME:
+      EXPECT_EQ(before[i], after[i]);
+      break;
+    case mapping_change_t::FAILURE:
+      EXPECT_EQ(CRUSH_ITEM_NONE, after[i]);
+      break;
+    case mapping_change_t::SAME_HOST:
+      EXPECT_NE(before[i], after[i]);
+      if (!spec.check_osd(after[i])) {
+	spec.validate_osd(after[i]);
+      } else {
+	EXPECT_EQ(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
+      }
+      break;
+    case mapping_change_t::NEW_HOST:
+      EXPECT_NE(before[i], after[i]);
+      if (!spec.check_osd(after[i])) {
+	spec.validate_osd(after[i]);
+      } else {
+	EXPECT_NE(spec.osd_to_host(before[i]), spec.osd_to_host(after[i]));
+      }
+      break;
+    }
+  }
+}
+
+std::vector<int> get_mapping(
+  const cluster_test_spec_t &spec,
+  CrushWrapper &c,
+  const std::vector<uint32_t> &weights,
+  int ruleno)
+{
+  std::vector<int> out;
+  c.do_rule(
+    ruleno, 0 /* seed */, out, spec.num_mapped_size,
+    weights,
+    0);
+  EXPECT_EQ(std::size(out), spec.num_mapped_size);
+  return out;
+}
+
+unsigned count_mapped(const auto &v) {
+  unsigned ret = 0;
+  for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
+  return ret;
+}
+
+TEST_F(CRUSHTest, msr_4_host_2_choose_rule) {
+  cluster_test_spec_t spec{3, 4, 3, 1, 3};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
+  EXPECT_EQ(
+    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, 2, 1, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  auto count_mapped = [](const auto &v) {
+    unsigned ret = 0;
+    for (const auto &i : v) ret += (i != CRUSH_ITEM_NONE);
+    return ret;
+  };
+
+  EXPECT_EQ(count_mapped(before), count_mapped(after_host_out));
+
+  auto weights_osd_out = create_weight_vector_first_osd_out(spec, before);
+  auto after_osd_out = get_mapping(spec, *c, weights_osd_out, ruleno);
+  EXPECT_EQ(count_mapped(before), count_mapped(after_osd_out));
+}
+
+TEST_F(CRUSHTest, msr_2_host_2_osd) {
+  cluster_test_spec_t spec{2, 3, 2, 2, 3};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, 0, rootno));
+  EXPECT_EQ(
+    0, c->set_rule_step_choose_msr(ruleno, 1, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, 2, spec.num_mapped_per_host, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, 3));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  ASSERT_EQ(count_mapped(before), 3);
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::NEW_HOST,
+    {0, spec.num_mapped_per_host});
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::SAME,
+    {spec.num_mapped_per_host, spec.num_mapped_size});
+}
+
+TEST_F(CRUSHTest, msr_5_host_8_6_ec_choose) {
+  cluster_test_spec_t spec{4, 5, 4, 4, 14};
+  auto [rootno, c] = create_crush_heirarchy(cct, spec);
+
+  auto ruleno = c->add_rule(-1, 4, CRUSH_RULE_TYPE_MSR_INDEP);
+  unsigned step_id = 0;
+  EXPECT_EQ(0, c->set_rule_step_take(ruleno, step_id++, rootno));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, step_id++, spec.num_hosts_mapped, HOST_TYPE));
+  EXPECT_EQ(
+    0,
+    c->set_rule_step_choose_msr(
+      ruleno, step_id++, spec.num_mapped_per_host, OSD_TYPE));
+  EXPECT_EQ(0, c->set_rule_step_emit(ruleno, step_id++));
+
+  auto weights_all_in = create_weight_vector(spec);
+  auto before = get_mapping(spec, *c, weights_all_in, ruleno);
+  for (auto i : before) { spec.validate_osd(i); }
+
+  /* MSR test case.  With normal CRUSH, hitting an out osd won't cause
+   * a retry of the previous step, so marking all of the osds on a host
+   * out will not cause positions mapped to that pg to remap.
+   * However, because the above is an MSR rule type, hitting an out osd
+   * will cause a retry of the previous steps as well.
+   * See https://tracker.ceph.com/issues/62214 for the original motivation */
+  auto weights_host_out = create_weight_vector_first_host_out(spec, before);
+  auto after_host_out = get_mapping(spec, *c, weights_host_out, ruleno);
+
+  CrushCompiler cc{*c, std::cout};
+  cc.decompile(std::cout);
+
+  fmt::print("weights_all_in: {}\n", fmt::join(weights_all_in, ", "));
+  fmt::print("weights_host_out: {}\n", fmt::join(weights_host_out, ", "));
+  fmt::print("before        : {}\n", fmt::join(before, ", "));
+  fmt::print("after_host_out: {}\n", fmt::join(after_host_out, ", "));
+
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::NEW_HOST,
+    {0, spec.num_mapped_per_host});
+  compare_mappings(
+    spec, before, after_host_out, mapping_change_t::SAME,
+    {spec.num_mapped_per_host, spec.num_mapped_size});
+}
+
+TEST_F(CRUSHTest, msr_multi_root) {
+  constexpr unsigned NUM_HOSTS = 4;
+  constexpr unsigned NUM_OSDS_PER_HOST = 3;
+
+  auto c = CrushWrapper();
+  c.create();
+  c.set_tunables_optimal();
+
+  c.set_type_name(ROOT_TYPE, "root");
+  c.set_type_name(HOST_TYPE, "host");
+  c.set_type_name(OSD_TYPE, "osd");
+
+  std::map<int, std::pair<std::string, std::string>> osd_id_to_host_root;
+  std::map<std::string, int> root_name_to_id;
+  std::map<std::string, std::vector<int>> host_name_to_osds;
+  unsigned next_osd_id = 0;
+
+  auto populate_root = [&](const auto &root_name) {
+    int rootno;
+    c.add_bucket(0, CRUSH_BUCKET_STRAW2, CRUSH_HASH_RJENKINS1,
+		 ROOT_TYPE, 0, nullptr, nullptr, &rootno);
+    c.set_item_name(rootno, root_name);
+    root_name_to_id[root_name] = rootno;
+
+    for (unsigned host_id = 0; host_id < NUM_HOSTS; ++host_id) {
+      const std::string host_name =
+	fmt::format("{}-host{}", root_name, host_id);
+      for (unsigned osd = 0; osd < NUM_OSDS_PER_HOST; ++osd) {
+	const int osd_id = next_osd_id++;
+	const std::string osd_name = fmt::format("{}-osd{}", root_name, osd_id);
+	auto ret = c.insert_item(
+	  cct, osd_id, 1.0, osd_name,
+	  {{ "root", root_name }, { "host", host_name }});
+	osd_id_to_host_root[osd_id] = std::make_pair(host_name, root_name);
+	host_name_to_osds[host_name].push_back(osd_id);
+	EXPECT_EQ(ret, 0);
+      }
+    }
+  };
+
+  int ruleno = 0;
+  int ret = c.add_rule(ruleno, 8, CRUSH_RULE_TYPE_MSR_INDEP);
+  ceph_assert(ret == ruleno);
+
+  unsigned step_id = 0;
+  auto populate_rule = [&](const auto &rule_name) {
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_TAKE, root_name_to_id[rule_name], 0);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, HOST_TYPE);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(
+      ruleno, step_id++, CRUSH_RULE_CHOOSE_MSR, 2, OSD_TYPE);
+    ceph_assert(ret == 0);
+    ret = c.set_rule_step(ruleno, step_id++, CRUSH_RULE_EMIT, 0, 0);
+    ceph_assert(ret == 0);
+  };
+
+  for (const auto &root_name : { "ssd", "hdd" }) {
+    populate_root(root_name);
+    populate_rule(root_name);
+  }
+  c.set_rule_name(ruleno, "rule_name");
+  c.finalize();
+
+  constexpr unsigned ACTING_SIZE = 8;
+  constexpr unsigned OSDS_PER_ROOT = 4;
+  constexpr unsigned OSDS_PER_HOST = 2;
+  auto validate_output = [&](const auto &out) {
+    std::set<std::string> hosts;
+    for (unsigned host = 0; host < (ACTING_SIZE / OSDS_PER_HOST); ++host) {
+      std::set<std::string> hosts_this_failure_domain;
+      unsigned start = host * OSDS_PER_HOST;
+      unsigned end = (host + 1) * OSDS_PER_HOST;
+      for (unsigned i = start; i < end; ++i) {
+	EXPECT_NE(out[i], CRUSH_ITEM_NONE);
+	EXPECT_EQ(osd_id_to_host_root.count(out[i]), 1);
+	const auto &[host_name, root_name] = osd_id_to_host_root[out[start]];
+	EXPECT_EQ(i < OSDS_PER_ROOT ? "ssd" : "hdd", root_name);
+	hosts_this_failure_domain.insert(host_name);
+      }
+      for (const auto &i: hosts_this_failure_domain) {
+	EXPECT_EQ(hosts.count(i), 0);
+	hosts.insert(i);
+      }
+    }
+  };
+
+  const std::vector<uint32_t> all_in(next_osd_id, CEPH_OSD_IN);
+  for (int x = 0; x < 1000; ++x) {
+    std::vector<int> out;
+    c.do_rule(ruleno, x, out, 8, all_in, 0);
+    EXPECT_EQ(count_mapped(out), 8);
+    validate_output(out);
+
+    {
+      std::vector<uint32_t> osds_out_weight = all_in;
+      std::set<unsigned> osd_idx_out{{1, 5}};
+      for (const auto &i: osd_idx_out) {
+	osds_out_weight[out[i]] = CEPH_OSD_OUT;
+      }
+      std::vector<int> osds_out;
+      c.do_rule(ruleno, x, osds_out, 8, osds_out_weight, 0);
+      EXPECT_EQ(count_mapped(osds_out), 8);
+      validate_output(osds_out);
+      for (unsigned i = 0; i < osds_out.size(); ++i) {
+	if (osd_idx_out.count(i)) {
+	  EXPECT_NE(osds_out[i], out[i]);
+	} else {
+	  EXPECT_EQ(osds_out[i], out[i]);
+	}
+      }
+    }
+
+    {
+      std::vector<uint32_t> hosts_out_weight = all_in;
+      std::set<unsigned> osd_ids_out;
+
+      for (const auto &i : {2, 6}) {
+	const auto &[host_name, _] = osd_id_to_host_root[out[i]];
+	for (const auto &osd_id: host_name_to_osds[host_name]) {
+	  osd_ids_out.insert(osd_id);
+	  hosts_out_weight[osd_id] = CEPH_OSD_OUT;
+	}
+      }
+
+      std::vector<int> hosts_out;
+      c.do_rule(ruleno, x, hosts_out, 8, hosts_out_weight, 0);
+      EXPECT_EQ(count_mapped(hosts_out), 8);
+      validate_output(hosts_out);
+      for (unsigned i = 0; i < hosts_out.size(); ++i) {
+	if (osd_ids_out.count(out[i])) {
+	  EXPECT_NE(hosts_out[i], out[i]);
+	} else {
+	  EXPECT_EQ(hosts_out[i], out[i]);
+	}
+      }
+    }
+  }
+}

From f58d4e85664a4d99af7d89e087bf8bbf40652b67 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 20 Nov 2023 20:43:18 -0800
Subject: [PATCH 1759/2492] vstart.sh: add --osds-per-host

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/vstart.sh | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index 0c9ef3237791..4e776def9717 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -176,6 +176,7 @@ zoned_enabled=0
 io_uring_enabled=0
 with_jaeger=0
 force_addr=0
+osds_per_host=0
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -265,6 +266,7 @@ options:
 	--seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore
 	--seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
 	--crimson-smp: number of cores to use for crimson
+	--osds-per-host: populate crush_location as each host holds the specified number of osds if set
 \n
 EOF
 
@@ -599,6 +601,11 @@ case $1 in
         with_jaeger=1
         echo "with_jaeger $with_jaeger"
         ;;
+    --osds-per-host)
+        osds_per_host="$2"
+        shift
+        echo "osds_per_host $osds_per_host"
+        ;;
     *)
         usage_exit
 esac
@@ -1128,6 +1135,13 @@ start_osd() {
 [osd.$osd]
         host = $HOSTNAME
 EOF
+
+            if [ "$osds_per_host" -gt 0 ]; then
+                wconf <<EOF
+        crush location = root=default host=$HOSTNAME-$(echo "$osd / $osds_per_host" | bc)
+EOF
+            fi
+
             if [ "$spdk_enabled" -eq 1 ]; then
                 wconf <<EOF
         bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]}

From d9f463e650483d7d01c715431e0ec5a1ea31b7af Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 30 Nov 2023 22:12:19 -0800
Subject: [PATCH 1760/2492] vstart.sh: add --require-osd-and-client-version and
 --use-crush-tunables flags

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/vstart.sh | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/vstart.sh b/src/vstart.sh
index 4e776def9717..20a35729c8e7 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -177,6 +177,8 @@ io_uring_enabled=0
 with_jaeger=0
 force_addr=0
 osds_per_host=0
+require_osd_and_client_version=""
+use_crush_tunables=""
 
 with_mgr_dashboard=true
 if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] ||
@@ -267,6 +269,8 @@ options:
 	--seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD
 	--crimson-smp: number of cores to use for crimson
 	--osds-per-host: populate crush_location as each host holds the specified number of osds if set
+	--require-osd-and-client-version: if supplied, do set-require-min-compat-client and require-osd-release to specified value
+	--use-crush-tunables: if supplied, set tunables to specified value
 \n
 EOF
 
@@ -606,6 +610,16 @@ case $1 in
         shift
         echo "osds_per_host $osds_per_host"
         ;;
+    --require-osd-and-client-version)
+        require_osd_and_client_version="$2"
+        shift
+        echo "require_osd_and_client_version $require_osd_and_client_version"
+        ;;
+    --use-crush-tunables)
+        use_crush_tunables="$2"
+        shift
+        echo "use_crush_tunables $use_crush_tunables"
+        ;;
     *)
         usage_exit
 esac
@@ -1102,6 +1116,15 @@ EOF
     if [ "$crimson" -eq 1 ]; then
         $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it
     fi
+
+    if [ -n "$require_osd_and_client_version" ]; then
+        $CEPH_BIN/ceph osd set-require-min-compat-client $require_osd_and_client_version
+        $CEPH_BIN/ceph osd require-osd-release $require_osd_and_client_version --yes-i-really-mean-it
+    fi
+
+    if [ -n "$use_crush_tunables" ]; then
+        $CEPH_BIN/ceph osd crush tunables $use_crush_tunables
+    fi
 }
 
 start_osd() {

From b398c548fdba46a1fc546e5654c269fd5de1917f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 12 Dec 2023 18:31:46 -0800
Subject: [PATCH 1761/2492] erasure-code: add support for multiple osds in a
 single failure domain

Adds support for crush-osds-per-failure-domain and
crush-num-failure-domains via MSR rules.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crush/CrushWrapper.cc       | 141 +++++++++++++++++++++++++++++---
 src/crush/CrushWrapper.h        |  38 ++++++++-
 src/erasure-code/ErasureCode.cc |  46 ++++++++---
 src/erasure-code/ErasureCode.h  |   2 +
 4 files changed, 200 insertions(+), 27 deletions(-)

diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index 573e22b6e56a..e434d1a17d85 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -2261,6 +2261,7 @@ void CrushWrapper::reweight_bucket(
 int CrushWrapper::add_simple_rule_at(
   string name, string root_name,
   string failure_domain_name,
+  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   int rno,
@@ -2332,17 +2333,19 @@ int CrushWrapper::add_simple_rule_at(
   }
   crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
   if (type)
-    crush_rule_set_step(rule, step++,
-			mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
-			CRUSH_RULE_CHOOSELEAF_INDEP,
-			CRUSH_CHOOSE_N,
-			type);
+    crush_rule_set_step(
+      rule, step++,
+      mode == "firstn" ? CRUSH_RULE_CHOOSELEAF_FIRSTN :
+      CRUSH_RULE_CHOOSELEAF_INDEP,
+      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
+      type);
   else
-    crush_rule_set_step(rule, step++,
-			mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
-			CRUSH_RULE_CHOOSE_INDEP,
-			CRUSH_CHOOSE_N,
-			0);
+    crush_rule_set_step(
+      rule, step++,
+      mode == "firstn" ? CRUSH_RULE_CHOOSE_FIRSTN :
+      CRUSH_RULE_CHOOSE_INDEP,
+      num_failure_domains <= 0 ? CRUSH_CHOOSE_N : num_failure_domains,
+      0);
   crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
 
   int ret = crush_add_rule(crush, rule, rno);
@@ -2358,13 +2361,125 @@ int CrushWrapper::add_simple_rule_at(
 int CrushWrapper::add_simple_rule(
   string name, string root_name,
   string failure_domain_name,
+  int num_failure_domains,
   string device_class,
   string mode, int rule_type,
   ostream *err)
 {
-  return add_simple_rule_at(name, root_name, failure_domain_name, device_class,
-			    mode,
-			    rule_type, -1, err);
+  return add_simple_rule_at(
+    name, root_name, failure_domain_name, num_failure_domains,
+    device_class,
+    mode,
+    rule_type, -1, err);
+}
+
+int CrushWrapper::add_multi_osd_per_failure_domain_rule_at(
+  string name, string root_name, string failure_domain_name,
+  int num_failure_domains,
+  int osds_per_failure_domain,
+  string device_class,
+  crush_rule_type rule_type,
+  int rno,
+  ostream *err)
+{
+  if (rule_exists(name)) {
+    if (err)
+      *err << "rule " << name << " exists";
+    return -EEXIST;
+  }
+  if (rno >= 0) {
+    if (rule_exists(rno)) {
+      if (err)
+        *err << "rule with ruleno " << rno << " exists";
+      return -EEXIST;
+    }
+  } else {
+    for (rno = 0; rno < get_max_rules(); rno++) {
+      if (!rule_exists(rno))
+        break;
+    }
+  }
+  if (!name_exists(root_name)) {
+    if (err)
+      *err << "root item " << root_name << " does not exist";
+    return -ENOENT;
+  }
+  int root = get_item_id(root_name);
+  int type = 0;
+  if (failure_domain_name.length()) {
+    type = get_type_id(failure_domain_name);
+    if (type < 0) {
+      if (err)
+	*err << "unknown type " << failure_domain_name;
+      return -EINVAL;
+    }
+  }
+  if (device_class.size()) {
+    if (!class_exists(device_class)) {
+      if (err)
+	*err << "device class " << device_class << " does not exist";
+      return -EINVAL;
+    }
+    int c = get_class_id(device_class);
+    if (class_bucket.count(root) == 0 ||
+	class_bucket[root].count(c) == 0) {
+      if (err)
+	*err << "root " << root_name << " has no devices with class "
+	     << device_class;
+      return -EINVAL;
+    }
+    root = class_bucket[root][c];
+  }
+  if (rule_type != CRUSH_RULE_TYPE_MSR_INDEP &&
+      rule_type != CRUSH_RULE_TYPE_MSR_FIRSTN) {
+    if (err)
+      *err << "unknown rule_type " << rule_type;
+    return -EINVAL;
+  }
+
+  int steps = 4;
+  crush_rule *rule = crush_make_rule(steps, rule_type);
+  ceph_assert(rule);
+  int step = 0;
+  crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
+  crush_rule_set_step(rule, step++,
+		      CRUSH_RULE_CHOOSE_MSR,
+		      num_failure_domains,
+		      type);
+  crush_rule_set_step(rule, step++,
+		      CRUSH_RULE_CHOOSE_MSR,
+		      osds_per_failure_domain,
+		      0);
+  crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+
+  int ret = crush_add_rule(crush, rule, rno);
+  if(ret < 0) {
+    *err << "failed to add rule " << rno << " because " << cpp_strerror(ret);
+    return ret;
+  }
+  set_rule_name(rno, name);
+  have_rmaps = false;
+  return rno;
+}
+
+
+int CrushWrapper::add_indep_multi_osd_per_failure_domain_rule(
+  string name, string root_name,
+  string failure_domain_name,
+  int num_failure_domains,
+  int osds_per_failure_domain,
+  string device_class,
+  ostream *err)
+{
+  return add_multi_osd_per_failure_domain_rule_at(
+    name, root_name,
+    failure_domain_name,
+    num_failure_domains,
+    osds_per_failure_domain,
+    device_class,
+    CRUSH_RULE_TYPE_MSR_INDEP,
+    -1,
+    err);
 }
 
 float CrushWrapper::_get_take_weight_osd_map(int root,
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 77f70d10049e..7018ca498c67 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -1231,16 +1231,52 @@ class CrushWrapper {
 
   int add_simple_rule(
     std::string name, std::string root_name, std::string failure_domain_type,
+    int num_failure_domains,
     std::string device_class, std::string mode, int rule_type,
     std::ostream *err = 0);
+  int add_simple_rule(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    std::string device_class, std::string mode, int rule_type,
+    std::ostream *err = 0) {
+    return add_simple_rule(
+      name, root_name, failure_domain_type, -1,
+      device_class, mode, rule_type, err);
+  }
+
+  int add_indep_multi_osd_per_failure_domain_rule(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    int osds_per_failure_domain,
+    int num_failure_domains,
+    std::string device_class,
+    std::ostream *err = 0);
 
   /**
    * @param rno rule[set] id to use, -1 to pick the lowest available
    */
   int add_simple_rule_at(
     std::string name, std::string root_name,
-    std::string failure_domain_type, std::string device_class, std::string mode,
+    std::string failure_domain_type,
+    int num_failure_domains,
+    std::string device_class, std::string mode,
     int rule_type, int rno, std::ostream *err = 0);
+  int add_simple_rule_at(
+    std::string name, std::string root_name,
+    std::string failure_domain_type,
+    std::string device_class, std::string mode,
+    int rule_type, int rno, std::ostream *err = 0) {
+    return add_simple_rule_at(
+      name, root_name, failure_domain_type, -1,
+      device_class, mode, rule_type, rno, err);
+  }
+
+  int add_multi_osd_per_failure_domain_rule_at(
+    std::string name, std::string root_name, std::string failure_domain_type,
+    int osds_per_failure_domain,
+    int num_failure_domains,
+    std::string device_class,
+    crush_rule_type rule_type,
+    int rno,
+    std::ostream *err = 0);
 
   int remove_rule(int ruleno);
 
diff --git a/src/erasure-code/ErasureCode.cc b/src/erasure-code/ErasureCode.cc
index 5212baee2518..6784fa355cf7 100644
--- a/src/erasure-code/ErasureCode.cc
+++ b/src/erasure-code/ErasureCode.cc
@@ -52,6 +52,12 @@ int ErasureCode::init(
   err |= to_string("crush-failure-domain", profile,
 		   &rule_failure_domain,
 		   DEFAULT_RULE_FAILURE_DOMAIN, ss);
+  err |= to_int("crush-osds-per-failure-domain", profile,
+		&rule_osds_per_failure_domain,
+		"0", ss);
+  err |= to_int("crush-num-failure-domains", profile,
+		&rule_num_failure_domains,
+		"0", ss);
   err |= to_string("crush-device-class", profile,
 		   &rule_device_class,
 		   "", ss);
@@ -66,19 +72,33 @@ int ErasureCode::create_rule(
   CrushWrapper &crush,
   std::ostream *ss) const
 {
-  int ruleid = crush.add_simple_rule(
-    name,
-    rule_root,
-    rule_failure_domain,
-    rule_device_class,
-    "indep",
-    pg_pool_t::TYPE_ERASURE,
-    ss);
-
-  if (ruleid < 0)
-    return ruleid;
-
-  return ruleid;
+  if (rule_osds_per_failure_domain <= 1) {
+    return crush.add_simple_rule(
+      name,
+      rule_root,
+      rule_failure_domain,
+      rule_num_failure_domains,
+      rule_device_class,
+      "indep",
+      pg_pool_t::TYPE_ERASURE,
+      ss);
+  } else {
+    if (rule_num_failure_domains < 1)  {
+      if (ss) {
+	*ss << "crush-num-failure-domains " << rule_num_failure_domains
+	    << " must be >= 1 if crush-osds-per-failure-domain specified";
+	return -EINVAL;
+      }
+    }
+    return crush.add_indep_multi_osd_per_failure_domain_rule(
+      name,
+      rule_root,
+      rule_failure_domain,
+      rule_num_failure_domains,
+      rule_osds_per_failure_domain,
+      rule_device_class,
+      ss);
+  }
 }
 
 int ErasureCode::sanity_check_k_m(int k, int m, ostream *ss)
diff --git a/src/erasure-code/ErasureCode.h b/src/erasure-code/ErasureCode.h
index c246d5dc6b67..fd6d1a41f714 100644
--- a/src/erasure-code/ErasureCode.h
+++ b/src/erasure-code/ErasureCode.h
@@ -37,6 +37,8 @@ namespace ceph {
     std::string rule_root;
     std::string rule_failure_domain;
     std::string rule_device_class;
+    int rule_osds_per_failure_domain = -1;
+    int rule_num_failure_domains = -1;
 
     ~ErasureCode() override {}
 

From aa88dfac4e6618cf5d71b6f9983572b13f2c4e12 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 11 Dec 2023 13:06:42 -0800
Subject: [PATCH 1762/2492] doc/rados/operations: add CRUSH MSR documentation

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 doc/rados/operations/crush-map-edits.rst | 31 +++++++++++++++++++++---
 doc/rados/operations/crush-map.rst       | 22 +++++++++++++++++
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/doc/rados/operations/crush-map-edits.rst b/doc/rados/operations/crush-map-edits.rst
index 46a4a4f74e87..84fd85dc2c01 100644
--- a/doc/rados/operations/crush-map-edits.rst
+++ b/doc/rados/operations/crush-map-edits.rst
@@ -419,7 +419,7 @@ centers for three-way replication, and yet another rule for erasure coding acros
 six storage devices. For a detailed discussion of CRUSH rules, see **Section 3.2**
 of `CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_.
 
-A rule takes the following form::
+A normal CRUSH rule takes the following form::
 
     rule <rulename> {
 
@@ -430,6 +430,19 @@ A rule takes the following form::
         step emit
     }
 
+CRUSH MSR (Multi-Step Retry) rules are a distinct type of CRUSH rule which
+supports retrying steps and provides better support for configurations that
+require multiple OSDs within each failure domain. MSR rules take the following
+form::
+
+    rule <rulename> {
+
+        id [a unique integer ID]
+        type [msr_indep|msr_firstn]
+        step take <bucket-name> [class <device-class>]
+        step choosemsr <N> type <bucket-type>
+        step emit
+    }
 
 ``id``
    :Description: A unique integer that identifies the rule.
@@ -441,12 +454,14 @@ A rule takes the following form::
 
 ``type``
    :Description: Denotes the type of replication strategy to be enforced by the
-                 rule.
+                 rule.  msr_firstn and msr_indep are a distinct descent algorithm
+		 which supports retrying steps within the rule and therefore
+		 multiple OSDs per failure domain.
    :Purpose: A component of the rule mask.
    :Type: String
    :Required: Yes
    :Default: ``replicated``
-   :Valid Values: ``replicated`` or ``erasure``
+   :Valid Values: ``replicated``, ``erasure``, ``msr_firstn``, ``msr_indep``
 
 
 ``step take <bucket-name> [class <device-class>]``
@@ -525,6 +540,16 @@ A rule takes the following form::
                  final CRUSH mapping transformation is therefore 1, 2, 3, 4, 5
                  → 1, 2, 6, 4, 5.
 
+``step choosemsr {num} type {bucket-type}``
+   :Description: Selects a num buckets of type bucket-type.  msr_firstn and msr_indep
+		 must use choosemsr rather than choose or chooseleaf.
+
+                 - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (as many buckets as are available).
+                 - If ``pool-num-replicas > {num} > 0``, choose that many buckets.
+   :Purpose: Choose step required for msr_firstn and msr_indep rules.
+   :Prerequisite: Follows ``step take`` and precedes ``step emit``
+   :Example: ``step choosemsr 3 type host``
+
 .. _crush-reclassify:
 
 Migrating from a legacy SSD rule to device classes
diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst
index 39151e6d4a76..8f833d28dacb 100644
--- a/doc/rados/operations/crush-map.rst
+++ b/doc/rados/operations/crush-map.rst
@@ -709,6 +709,13 @@ The relevant erasure-code profile properties are as follows:
    [default: ``default``].
  * **crush-failure-domain**: the CRUSH bucket type used in the distribution of
    erasure-coded shards [default: ``host``].
+ * **crush-osds-per-failure-domain**: Maximum number of OSDs to place in each
+   failure domain -- defaults to 1.  Using a value greater than one will
+   cause a CRUSH MSR rule to be created, see below.  Must be specified if
+   ``crush-num-failure-domains`` is specified.
+ * **crush-num-failure-domains**: Number of failure domains to map.  Must be
+   specified if ``crush-osds-per-failure-domain`` is specified.  Results in
+   a CRUSH MSR rule being created.
  * **crush-device-class**: the device class on which to place data [default:
    none, which means that all devices are used].
  * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the
@@ -726,6 +733,21 @@ The relevant erasure-code profile properties are as follows:
    argument is omitted, then Ceph will create the CRUSH rule automatically.
 
 
+CRUSH MSR Rules
+---------------
+
+Creating an erasure-code profile with a ``crush-osds-per-failure-domain``
+value greater than one will cause a CRUSH MSR rule type to be created
+instead of a normal CRUSH rule.  Normal crush rules cannot retry prior
+steps when an out OSD is encountered and rely on CHOOSELEAF steps to
+permit moving OSDs to new hosts.  However, CHOOSELEAF rules don't
+support more than a single OSD per failure domain.  MSR rules, new in
+squid, support multiple OSDs per failure domain by retrying all prior
+steps when an out OSD is encountered.  Using MSR rules requires that
+OSDs and clients be required to support the CRUSH_MSR feature bit
+(squid or newer).
+
+
 Deleting rules
 --------------
 

From e56e1bb536af7073969a134a57a113860eb892c7 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 20 Dec 2023 19:20:32 -0800
Subject: [PATCH 1763/2492] qa/erasure-code: modify jerasure 4/2 ec test case
 to use msr

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
index dfcc61607a7d..a0cd68a55f53 100644
--- a/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
+++ b/qa/erasure-code/ec-rados-plugin=jerasure-k=4-m=2.yaml
@@ -11,7 +11,9 @@ tasks:
       k: 4
       m: 2
       technique: reed_sol_van
-      crush-failure-domain: osd
+      crush-failure-domain: host
+      crush-osds-per-failure-domain: 2
+      crush-num-failure-domains: 3
     op_weights:
       read: 100
       write: 0

From 39ad2e7168a5f2cb14c3eeee76a5641607e254b4 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 8 Jan 2024 19:24:41 -0800
Subject: [PATCH 1764/2492] test/cli/crushtool/choose-args.t: add msr related
 json output

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/cli/crushtool/choose-args.t | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/cli/crushtool/choose-args.t b/src/test/cli/crushtool/choose-args.t
index e0956ec0a754..99120f0f211f 100644
--- a/src/test/cli/crushtool/choose-args.t
+++ b/src/test/cli/crushtool/choose-args.t
@@ -159,6 +159,8 @@
           "chooseleaf_descend_once": 0,
           "chooseleaf_vary_r": 0,
           "chooseleaf_stable": 0,
+          "msr_descents": 100,
+          "msr_collision_tries": 100,
           "straw_calc_version": 0,
           "allowed_bucket_algs": 22,
           "profile": "argonaut",
@@ -172,7 +174,8 @@
           "has_v3_rules": 0,
           "has_v4_buckets": 1,
           "require_feature_tunables5": 0,
-          "has_v5_rules": 0
+          "has_v5_rules": 0,
+          "has_msr_rules": 0
       },
       "choose_args": {
           "1": [],

From c52aabfb45f816b89ae7148a126de1df9dfa9bcd Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 8 Jan 2024 19:25:05 -0800
Subject: [PATCH 1765/2492] test/cli/osdmaptool/crush.t: adjust --import-crush
 size output

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/cli/osdmaptool/crush.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/cli/osdmaptool/crush.t b/src/test/cli/osdmaptool/crush.t
index 520f11e50d5c..2c4c1bb1ad57 100644
--- a/src/test/cli/osdmaptool/crush.t
+++ b/src/test/cli/osdmaptool/crush.t
@@ -6,7 +6,7 @@
   osdmaptool: exported crush map to oc
   $ osdmaptool --import-crush oc myosdmap
   osdmaptool: osdmap file 'myosdmap'
-  osdmaptool: imported 497 byte crush map from oc
+  osdmaptool: imported 505 byte crush map from oc
   osdmaptool: writing epoch 3 to myosdmap
   $ osdmaptool --adjust-crush-weight 0:5 myosdmap
   osdmaptool: osdmap file 'myosdmap'

From 0736d5d0a1c227d571d591b9d565c02f5f4b9a58 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 9 Jan 2024 11:35:45 -0800
Subject: [PATCH 1766/2492] tasks/.../test_erasure_code_profile: assertSubset
 in test_create_plugin

Newly added profile options may break this test otherwise.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 qa/tasks/mgr/dashboard/test_erasure_code_profile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
index 7fb7c1c8270f..a50914008934 100644
--- a/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
+++ b/qa/tasks/mgr/dashboard/test_erasure_code_profile.py
@@ -79,7 +79,7 @@ def test_create_plugin(self):
         self.assertStatus(201)
 
         self._get('/api/erasure_code_profile/lrc')
-        self.assertJsonBody({
+        self.assertJsonSubset({
             'crush-device-class': '',
             'crush-failure-domain': 'host',
             'crush-root': 'default',

From 67464df04539ec0a9993854cb81777f0d1645997 Mon Sep 17 00:00:00 2001
From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
Date: Wed, 31 Jan 2024 20:09:33 +0200
Subject: [PATCH 1767/2492] osd: Fix compilation warnings for PR
 https://github.com/ceph/ceph/pull/55198

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>

Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com>
---
 src/osd/OSDMap.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index b35bc592eec0..2d0f2bb604f2 100644
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5341,8 +5341,8 @@ int OSDMap::calc_desired_primary_distribution_osdsize_opt(
     }
     int write_ratio = 100 - read_ratio;
     ldout(cct, 30) << __func__ << " Pool: " << pid << " read ratio: " << read_ratio << " write ratio: " << write_ratio << dendl;
-    auto num_osds = osds.size();
-    if (pg_num != (pool->get_pgp_num_mask() + 1)) {
+    int num_osds = osds.size();
+    if (pg_num != int((pool->get_pgp_num_mask() + 1))) {
       // TODO: handle pgs with different sizes
       //pool_t op:  unsigned get_pg_num_divisor(pg_t pgid) const;
       ldout(cct, 10) << __func__ << " number of PGs for pool '"
@@ -6451,7 +6451,7 @@ int OSDMap::calc_rbs_fair(CephContext *cct, OSDMap& tmp_osd_map, int64_t pool_id
     }
   }
 
-  auto num_osds = pgs_by_osd.size();
+  int num_osds = pgs_by_osd.size();
 
   float avg_prims_per_osd = (float)num_pgs / (float)num_osds;
   uint64_t max_prims_per_osd = 0;
@@ -6586,8 +6586,8 @@ int OSDMap::calc_rbs_size_optimal(CephContext *cct, OSDMap& tmp_osd_map, int64_t
   pgs_by_osd = tmp_osd_map.get_pgs_by_osd(cct, pool_id, &prim_pgs_by_osd, &acting_prims_by_osd);
   auto num_osds = pgs_by_osd.size();
   int64_t num_pg_osd_legs = 0;
-  for (auto i = 0 ; i < num_osds ; i++) {
-    if (get_primary_affinity(i) != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
+  for (uint64_t i = 0 ; i < num_osds ; i++) {
+    if (get_primary_affinity(int(i)) != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
       if (cct != nullptr) {
         ldout(cct, 30) << __func__ << " pool " << pool_id
                            << " has primary_affinity set to non-default value on some OSDs" << dendl;

From 2226e0c942eb3f19699bd507c73aa165044a9afc Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 19 Jan 2024 10:51:38 +0800
Subject: [PATCH 1768/2492] crimson/os/seastore: add new interfaces to alloc
 multiple extents for a single request

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/async_cleaner.h       |  8 +++
 .../os/seastore/btree/fixed_kv_btree.h        |  6 +-
 src/crimson/os/seastore/btree/fixed_kv_node.h | 20 +++---
 src/crimson/os/seastore/cache.cc              | 55 ++++++++++++----
 src/crimson/os/seastore/cache.h               | 66 +++++++++++++++++--
 .../flat_collection_manager.cc                |  4 +-
 .../os/seastore/extent_placement_manager.h    | 66 +++++++++++++++++--
 .../omap_manager/btree/btree_omap_manager.cc  |  4 +-
 .../btree/omap_btree_node_impl.cc             |  4 +-
 .../node_extent_manager/seastore.h            |  2 +-
 src/crimson/os/seastore/transaction_manager.h | 64 ++++++++++++++++--
 src/crimson/tools/store_nbd/tm_driver.cc      | 20 ++++--
 .../seastore/test_btree_lba_manager.cc        | 10 ++-
 .../crimson/seastore/test_seastore_cache.cc   |  4 +-
 .../seastore/test_transaction_manager.cc      | 30 ++++++---
 15 files changed, 294 insertions(+), 69 deletions(-)

diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h
index fb8e03bb4bcf..8f69d742d741 100644
--- a/src/crimson/os/seastore/async_cleaner.h
+++ b/src/crimson/os/seastore/async_cleaner.h
@@ -1695,6 +1695,14 @@ class RBMCleaner : public AsyncCleaner {
     return paddr;
   }
 
+  std::list<alloc_paddr_result> alloc_paddrs(extent_len_t length) {
+    // TODO: implement allocation strategy (dirty metadata and multiple devices)
+    auto rbs = rb_group->get_rb_managers();
+    auto ret = rbs[0]->alloc_extents(length);
+    stats.used_bytes += length;
+    return ret;
+  }
+
   size_t get_total_bytes() const {
     auto rbs = rb_group->get_rb_managers();
     size_t total = 0;
diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h
index a2a97e30a503..b91a4e2d07bf 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_btree.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h
@@ -356,7 +356,7 @@ class FixedKVBtree {
   using mkfs_ret = phy_tree_root_t;
   static mkfs_ret mkfs(RootBlockRef &root_block, op_context_t<node_key_t> c) {
     assert(root_block->is_mutation_pending());
-    auto root_leaf = c.cache.template alloc_new_extent<leaf_node_t>(
+    auto root_leaf = c.cache.template alloc_new_non_data_extent<leaf_node_t>(
       c.trans,
       node_size,
       placement_hint_t::HOT,
@@ -1022,7 +1022,7 @@ class FixedKVBtree {
     assert(is_lba_backref_node(e->get_type()));
     
     auto do_rewrite = [&](auto &fixed_kv_extent) {
-      auto n_fixed_kv_extent = c.cache.template alloc_new_extent<
+      auto n_fixed_kv_extent = c.cache.template alloc_new_non_data_extent<
         std::remove_reference_t<decltype(fixed_kv_extent)>
         >(
         c.trans,
@@ -1781,7 +1781,7 @@ class FixedKVBtree {
     SUBTRACET(seastore_fixedkv_tree, "split_from {}, depth {}", c.trans, split_from, iter.get_depth());
 
     if (split_from == iter.get_depth()) {
-      auto nroot = c.cache.template alloc_new_extent<internal_node_t>(
+      auto nroot = c.cache.template alloc_new_non_data_extent<internal_node_t>(
         c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
       fixed_kv_node_meta_t<node_key_t> meta{
         min_max_t<node_key_t>::min, min_max_t<node_key_t>::max, iter.get_depth() + 1};
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index 4d32475ee24d..6ad55e3b9b32 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -757,9 +757,9 @@ struct FixedKVInternalNode
 
   std::tuple<Ref, Ref, NODE_KEY>
   make_split_children(op_context_t<NODE_KEY> c) {
-    auto left = c.cache.template alloc_new_extent<node_type_t>(
+    auto left = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
-    auto right = c.cache.template alloc_new_extent<node_type_t>(
+    auto right = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
     this->split_child_ptrs(*left, *right);
     auto pivot = this->split_into(*left, *right);
@@ -774,7 +774,7 @@ struct FixedKVInternalNode
   Ref make_full_merge(
     op_context_t<NODE_KEY> c,
     Ref &right) {
-    auto replacement = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
     replacement->merge_child_ptrs(*this, *right);
     replacement->merge_from(*this, *right->template cast<node_type_t>());
@@ -789,9 +789,9 @@ struct FixedKVInternalNode
     bool prefer_left) {
     ceph_assert(_right->get_type() == this->get_type());
     auto &right = *_right->template cast<node_type_t>();
-    auto replacement_left = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement_left = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
-    auto replacement_right = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement_right = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
 
     auto pivot = this->balance_into_new_nodes(
@@ -1127,9 +1127,9 @@ struct FixedKVLeafNode
 
   std::tuple<Ref, Ref, NODE_KEY>
   make_split_children(op_context_t<NODE_KEY> c) {
-    auto left = c.cache.template alloc_new_extent<node_type_t>(
+    auto left = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
-    auto right = c.cache.template alloc_new_extent<node_type_t>(
+    auto right = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
     if constexpr (has_children) {
       this->split_child_ptrs(*left, *right);
@@ -1146,7 +1146,7 @@ struct FixedKVLeafNode
   Ref make_full_merge(
     op_context_t<NODE_KEY> c,
     Ref &right) {
-    auto replacement = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
     if constexpr (has_children) {
       replacement->merge_child_ptrs(*this, *right);
@@ -1163,9 +1163,9 @@ struct FixedKVLeafNode
     bool prefer_left) {
     ceph_assert(_right->get_type() == this->get_type());
     auto &right = *_right->template cast<node_type_t>();
-    auto replacement_left = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement_left = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
-    auto replacement_right = c.cache.template alloc_new_extent<node_type_t>(
+    auto replacement_right = c.cache.template alloc_new_non_data_extent<node_type_t>(
       c.trans, node_size, placement_hint_t::HOT, INIT_GENERATION);
 
     auto pivot = this->balance_into_new_nodes(
diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index b81fe62906c7..0e5eeb96c17e 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -962,32 +962,33 @@ CachedExtentRef Cache::alloc_new_extent_by_type(
   LOG_PREFIX(Cache::alloc_new_extent_by_type);
   SUBDEBUGT(seastore_cache, "allocate {} {}B, hint={}, gen={}",
             t, type, length, hint, rewrite_gen_printer_t{gen});
+  ceph_assert(get_extent_category(type) == data_category_t::METADATA);
   switch (type) {
   case extent_types_t::ROOT:
     ceph_assert(0 == "ROOT is never directly alloc'd");
     return CachedExtentRef();
   case extent_types_t::LADDR_INTERNAL:
-    return alloc_new_extent<lba_manager::btree::LBAInternalNode>(t, length, hint, gen);
+    return alloc_new_non_data_extent<lba_manager::btree::LBAInternalNode>(t, length, hint, gen);
   case extent_types_t::LADDR_LEAF:
-    return alloc_new_extent<lba_manager::btree::LBALeafNode>(
+    return alloc_new_non_data_extent<lba_manager::btree::LBALeafNode>(
       t, length, hint, gen);
   case extent_types_t::ONODE_BLOCK_STAGED:
-    return alloc_new_extent<onode::SeastoreNodeExtent>(t, length, hint, gen);
+    return alloc_new_non_data_extent<onode::SeastoreNodeExtent>(
+      t, length, hint, gen);
   case extent_types_t::OMAP_INNER:
-    return alloc_new_extent<omap_manager::OMapInnerNode>(t, length, hint, gen);
+    return alloc_new_non_data_extent<omap_manager::OMapInnerNode>(
+      t, length, hint, gen);
   case extent_types_t::OMAP_LEAF:
-    return alloc_new_extent<omap_manager::OMapLeafNode>(t, length, hint, gen);
+    return alloc_new_non_data_extent<omap_manager::OMapLeafNode>(
+      t, length, hint, gen);
   case extent_types_t::COLL_BLOCK:
-    return alloc_new_extent<collection_manager::CollectionNode>(t, length, hint, gen);
-  case extent_types_t::OBJECT_DATA_BLOCK:
-    return alloc_new_extent<ObjectDataBlock>(t, length, hint, gen);
+    return alloc_new_non_data_extent<collection_manager::CollectionNode>(
+      t, length, hint, gen);
   case extent_types_t::RETIRED_PLACEHOLDER:
     ceph_assert(0 == "impossible");
     return CachedExtentRef();
-  case extent_types_t::TEST_BLOCK:
-    return alloc_new_extent<TestBlock>(t, length, hint, gen);
   case extent_types_t::TEST_BLOCK_PHYSICAL:
-    return alloc_new_extent<TestBlockPhysical>(t, length, hint, gen);
+    return alloc_new_non_data_extent<TestBlockPhysical>(t, length, hint, gen);
   case extent_types_t::NONE: {
     ceph_assert(0 == "NONE is an invalid extent type");
     return CachedExtentRef();
@@ -998,6 +999,38 @@ CachedExtentRef Cache::alloc_new_extent_by_type(
   }
 }
 
+std::vector<CachedExtentRef> Cache::alloc_new_data_extents_by_type(
+  Transaction &t,        ///< [in, out] current transaction
+  extent_types_t type,   ///< [in] type tag
+  extent_len_t length,   ///< [in] length
+  placement_hint_t hint, ///< [in] user hint
+  rewrite_gen_t gen      ///< [in] rewrite generation
+)
+{
+  LOG_PREFIX(Cache::alloc_new_data_extents_by_type);
+  SUBDEBUGT(seastore_cache, "allocate {} {}B, hint={}, gen={}",
+            t, type, length, hint, rewrite_gen_printer_t{gen});
+  ceph_assert(get_extent_category(type) == data_category_t::DATA);
+  std::vector<CachedExtentRef> res;
+  switch (type) {
+  case extent_types_t::OBJECT_DATA_BLOCK:
+    {
+      auto extents = alloc_new_data_extents<ObjectDataBlock>(t, length, hint, gen);
+      res.insert(res.begin(), extents.begin(), extents.end());
+    }
+    return res;
+  case extent_types_t::TEST_BLOCK:
+    {
+      auto extents = alloc_new_data_extents<TestBlock>(t, length, hint, gen);
+      res.insert(res.begin(), extents.begin(), extents.end());
+    }
+    return res;
+  default:
+    ceph_assert(0 == "impossible");
+    return res;
+  }
+}
+
 CachedExtentRef Cache::duplicate_for_write(
   Transaction &t,
   CachedExtentRef i) {
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 0f32bfe87e66..a6aeb357148f 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -859,13 +859,13 @@ class Cache {
   }
 
   /**
-   * alloc_new_extent
+   * alloc_new_non_data_extent
    *
    * Allocates a fresh extent. if delayed is true, addr will be alloc'd later.
    * Note that epaddr can only be fed by the btree lba unittest for now
    */
   template <typename T>
-  TCachedExtentRef<T> alloc_new_extent(
+  TCachedExtentRef<T> alloc_new_non_data_extent(
     Transaction &t,         ///< [in, out] current transaction
     extent_len_t length,    ///< [in] length
     placement_hint_t hint,  ///< [in] user hint
@@ -876,13 +876,13 @@ class Cache {
     rewrite_gen_t gen
 #endif
   ) {
-    LOG_PREFIX(Cache::alloc_new_extent);
+    LOG_PREFIX(Cache::alloc_new_non_data_extent);
     SUBTRACET(seastore_cache, "allocate {} {}B, hint={}, gen={}",
               t, T::TYPE, length, hint, rewrite_gen_printer_t{gen});
 #ifdef UNIT_TESTS_BUILT
-    auto result = epm.alloc_new_extent(t, T::TYPE, length, hint, gen, epaddr);
+    auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, hint, gen, epaddr);
 #else
-    auto result = epm.alloc_new_extent(t, T::TYPE, length, hint, gen);
+    auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, hint, gen);
 #endif
     auto ret = CachedExtent::make_cached_extent_ref<T>(std::move(result.bp));
     ret->init(CachedExtent::extent_state_t::INITIAL_WRITE_PENDING,
@@ -897,6 +897,49 @@ class Cache {
               hint, rewrite_gen_printer_t{result.gen}, *ret);
     return ret;
   }
+  /**
+   * alloc_new_data_extents
+   *
+   * Allocates a fresh extent. if delayed is true, addr will be alloc'd later.
+   * Note that epaddr can only be fed by the btree lba unittest for now
+   */
+  template <typename T>
+  std::vector<TCachedExtentRef<T>> alloc_new_data_extents(
+    Transaction &t,         ///< [in, out] current transaction
+    extent_len_t length,    ///< [in] length
+    placement_hint_t hint,  ///< [in] user hint
+#ifdef UNIT_TESTS_BUILT
+    rewrite_gen_t gen,      ///< [in] rewrite generation
+    std::optional<paddr_t> epaddr = std::nullopt ///< [in] paddr fed by callers
+#else
+    rewrite_gen_t gen
+#endif
+  ) {
+    LOG_PREFIX(Cache::alloc_new_data_extents);
+    SUBTRACET(seastore_cache, "allocate {} {}B, hint={}, gen={}",
+              t, T::TYPE, length, hint, rewrite_gen_printer_t{gen});
+#ifdef UNIT_TESTS_BUILT
+    auto results = epm.alloc_new_data_extents(t, T::TYPE, length, hint, gen, epaddr);
+#else
+    auto results = epm.alloc_new_data_extents(t, T::TYPE, length, hint, gen);
+#endif
+    std::vector<TCachedExtentRef<T>> extents;
+    for (auto &result : results) {
+      auto ret = CachedExtent::make_cached_extent_ref<T>(std::move(result.bp));
+      ret->init(CachedExtent::extent_state_t::INITIAL_WRITE_PENDING,
+                result.paddr,
+                hint,
+                result.gen,
+                t.get_trans_id());
+      t.add_fresh_extent(ret);
+      SUBDEBUGT(seastore_cache,
+                "allocated {} {}B extent at {}, hint={}, gen={} -- {}",
+                t, T::TYPE, length, result.paddr,
+                hint, rewrite_gen_printer_t{result.gen}, *ret);
+      extents.emplace_back(std::move(ret));
+    }
+    return extents;
+  }
 
   /**
    * alloc_remapped_extent
@@ -953,6 +996,19 @@ class Cache {
     rewrite_gen_t gen      ///< [in] rewrite generation
     );
 
+  /**
+   * alloc_new_extent
+   *
+   * Allocates a fresh extent.  addr will be relative until commit.
+   */
+  std::vector<CachedExtentRef> alloc_new_data_extents_by_type(
+    Transaction &t,        ///< [in, out] current transaction
+    extent_types_t type,   ///< [in] type tag
+    extent_len_t length,   ///< [in] length
+    placement_hint_t hint, ///< [in] user hint
+    rewrite_gen_t gen      ///< [in] rewrite generation
+    );
+
   /**
    * Allocates mutable buffer from extent_set on offset~len
    *
diff --git a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
index 3c65ed0e2c18..81353655694e 100644
--- a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
+++ b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
@@ -30,7 +30,7 @@ FlatCollectionManager::mkfs(Transaction &t)
 {
 
   logger().debug("FlatCollectionManager: {}", __func__);
-  return tm.alloc_extent<CollectionNode>(
+  return tm.alloc_non_data_extent<CollectionNode>(
     t, L_ADDR_MIN, MIN_FLAT_BLOCK_SIZE
   ).si_then([](auto&& root_extent) {
     coll_root_t coll_root = coll_root_t(
@@ -74,7 +74,7 @@ FlatCollectionManager::create(coll_root_t &coll_root, Transaction &t,
 	// TODO return error probably, but such a nonsensically large number of
 	// collections would create a ton of other problems as well
 	assert(new_size < MAX_FLAT_BLOCK_SIZE);
-        return tm.alloc_extent<CollectionNode>(
+        return tm.alloc_non_data_extent<CollectionNode>(
 	  t, L_ADDR_MIN, new_size
 	).si_then([=, this, &coll_root, &t] (auto &&root_extent) {
           coll_root.update(root_extent->get_laddr(), root_extent->get_length());
diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h
index 371b49e07414..70dbc428d10b 100644
--- a/src/crimson/os/seastore/extent_placement_manager.h
+++ b/src/crimson/os/seastore/extent_placement_manager.h
@@ -35,6 +35,8 @@ class ExtentOolWriter {
 
   virtual paddr_t alloc_paddr(extent_len_t length) = 0;
 
+  virtual std::list<alloc_paddr_result> alloc_paddrs(extent_len_t length) = 0;
+
   using alloc_write_ertr = base_ertr;
   using alloc_write_iertr = trans_iertr<alloc_write_ertr>;
   virtual alloc_write_iertr::future<> alloc_write_ool_extents(
@@ -82,6 +84,10 @@ class SegmentedOolWriter : public ExtentOolWriter {
     return make_delayed_temp_paddr(0);
   }
 
+  std::list<alloc_paddr_result> alloc_paddrs(extent_len_t length) final {
+    return {alloc_paddr_result{make_delayed_temp_paddr(0), length}};
+  }
+
   bool can_inplace_rewrite(Transaction& t,
     CachedExtentRef extent) final {
     return false;
@@ -130,6 +136,11 @@ class RandomBlockOolWriter : public ExtentOolWriter {
     return rb_cleaner->alloc_paddr(length);
   }
 
+  std::list<alloc_paddr_result> alloc_paddrs(extent_len_t length) final {
+    assert(rb_cleaner);
+    return rb_cleaner->alloc_paddrs(length);
+  }
+
   bool can_inplace_rewrite(Transaction& t,
     CachedExtentRef extent) final {
     if (!extent->is_dirty()) {
@@ -268,7 +279,7 @@ class ExtentPlacementManager {
     bufferptr bp;
     rewrite_gen_t gen;
   };
-  alloc_result_t alloc_new_extent(
+  alloc_result_t alloc_new_non_data_extent(
     Transaction& t,
     extent_types_t type,
     extent_len_t length,
@@ -302,19 +313,64 @@ class ExtentPlacementManager {
     if (gen == INLINE_GENERATION) {
 #endif
       addr = make_record_relative_paddr(0);
-    } else if (category == data_category_t::DATA) {
-      assert(data_writers_by_gen[generation_to_writer(gen)]);
-      addr = data_writers_by_gen[
-	  generation_to_writer(gen)]->alloc_paddr(length);
     } else {
       assert(category == data_category_t::METADATA);
       assert(md_writers_by_gen[generation_to_writer(gen)]);
       addr = md_writers_by_gen[
 	  generation_to_writer(gen)]->alloc_paddr(length);
     }
+    assert(!(category == data_category_t::DATA));
     return {addr, std::move(bp), gen};
   }
 
+  std::list<alloc_result_t> alloc_new_data_extents(
+    Transaction& t,
+    extent_types_t type,
+    extent_len_t length,
+    placement_hint_t hint,
+#ifdef UNIT_TESTS_BUILT
+    rewrite_gen_t gen,
+    std::optional<paddr_t> external_paddr = std::nullopt
+#else
+    rewrite_gen_t gen
+#endif
+  ) {
+    assert(hint < placement_hint_t::NUM_HINTS);
+    assert(is_target_rewrite_generation(gen));
+    assert(gen == INIT_GENERATION || hint == placement_hint_t::REWRITE);
+
+    data_category_t category = get_extent_category(type);
+    gen = adjust_generation(category, type, hint, gen);
+    assert(gen != INLINE_GENERATION);
+
+    // XXX: bp might be extended to point to different memory (e.g. PMem)
+    // according to the allocator.
+    std::list<alloc_result_t> allocs;
+#ifdef UNIT_TESTS_BUILT
+    if (unlikely(external_paddr.has_value())) {
+      assert(external_paddr->is_fake());
+      auto bp = ceph::bufferptr(
+        buffer::create_page_aligned(length));
+      bp.zero();
+      allocs.emplace_back(alloc_result_t{*external_paddr, std::move(bp), gen});
+    } else {
+#else
+    {
+#endif
+      assert(category == data_category_t::DATA);
+      assert(data_writers_by_gen[generation_to_writer(gen)]);
+      auto addrs = data_writers_by_gen[
+          generation_to_writer(gen)]->alloc_paddrs(length);
+      for (auto &ext : addrs) {
+        auto bp = ceph::bufferptr(
+          buffer::create_page_aligned(ext.len));
+        bp.zero();
+        allocs.emplace_back(alloc_result_t{ext.start, std::move(bp), gen});
+      }
+    }
+    return allocs;
+  }
+
   /**
    * dispatch_result_t
    *
diff --git a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
index 77dc270a5323..c182965764ee 100644
--- a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
@@ -22,7 +22,7 @@ BtreeOMapManager::initialize_omap(Transaction &t, laddr_t hint)
 {
   LOG_PREFIX(BtreeOMapManager::initialize_omap);
   DEBUGT("hint: {}", t, hint);
-  return tm.alloc_extent<OMapLeafNode>(t, hint, OMAP_LEAF_BLOCK_SIZE)
+  return tm.alloc_non_data_extent<OMapLeafNode>(t, hint, OMAP_LEAF_BLOCK_SIZE)
     .si_then([hint, &t](auto&& root_extent) {
       root_extent->set_size(0);
       omap_node_meta_t meta{1};
@@ -51,7 +51,7 @@ BtreeOMapManager::handle_root_split(
 {
   LOG_PREFIX(BtreeOMapManager::handle_root_split);
   DEBUGT("{}", oc.t, omap_root);
-  return oc.tm.alloc_extent<OMapInnerNode>(oc.t, omap_root.hint,
+  return oc.tm.alloc_non_data_extent<OMapInnerNode>(oc.t, omap_root.hint,
                                            OMAP_INNER_BLOCK_SIZE)
     .si_then([&omap_root, mresult, oc](auto&& nroot) -> handle_root_split_ret {
     auto [left, right, pivot] = *(mresult.split_tuple);
diff --git a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
index 96115f13237c..c932f6d590c1 100644
--- a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
@@ -346,7 +346,7 @@ OMapInnerNode::make_full_merge(omap_context_t oc, OMapNodeRef right)
 {
   LOG_PREFIX(OMapInnerNode::make_full_merge);
   DEBUGT("", oc.t);
-  return oc.tm.alloc_extent<OMapInnerNode>(oc.t, oc.hint,
+  return oc.tm.alloc_non_data_extent<OMapInnerNode>(oc.t, oc.hint,
     OMAP_INNER_BLOCK_SIZE)
     .si_then([this, right] (auto &&replacement) {
       replacement->merge_from(*this, *right->cast<OMapInnerNode>());
@@ -679,7 +679,7 @@ OMapLeafNode::make_full_merge(omap_context_t oc, OMapNodeRef right)
   ceph_assert(right->get_type() == TYPE);
   LOG_PREFIX(OMapLeafNode::make_full_merge);
   DEBUGT("this: {}", oc.t, *this);
-  return oc.tm.alloc_extent<OMapLeafNode>(oc.t, oc.hint, OMAP_LEAF_BLOCK_SIZE)
+  return oc.tm.alloc_non_data_extent<OMapLeafNode>(oc.t, oc.hint, OMAP_LEAF_BLOCK_SIZE)
     .si_then([this, right] (auto &&replacement) {
       replacement->merge_from(*this, *right->cast<OMapLeafNode>());
       return full_merge_ret(
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
index c12e583bd566..cff67f9cf51f 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
@@ -131,7 +131,7 @@ class SeastoreNodeExtentManager final: public TransactionManagerHandle {
         return alloc_iertr::make_ready_future<NodeExtentRef>();
       }
     }
-    return tm.alloc_extent<SeastoreNodeExtent>(t, hint, len
+    return tm.alloc_non_data_extent<SeastoreNodeExtent>(t, hint, len
     ).si_then([len, &t](auto extent) {
       SUBDEBUGT(seastore_onode,
           "allocated {}B at {:#x} -- {}",
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 2bfed6ac9f66..4bacd78fe60b 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -279,7 +279,7 @@ class TransactionManager : public ExtentCallbackInterface {
     std::vector<laddr_t> offsets);
 
   /**
-   * alloc_extent
+   * alloc_non_data_extent
    *
    * Allocates a new block of type T with the minimum lba range of size len
    * greater than laddr_hint.
@@ -288,16 +288,16 @@ class TransactionManager : public ExtentCallbackInterface {
   template <typename T>
   using alloc_extent_ret = alloc_extent_iertr::future<TCachedExtentRef<T>>;
   template <typename T>
-  alloc_extent_ret<T> alloc_extent(
+  alloc_extent_ret<T> alloc_non_data_extent(
     Transaction &t,
     laddr_t laddr_hint,
     extent_len_t len,
     placement_hint_t placement_hint = placement_hint_t::HOT) {
-    LOG_PREFIX(TransactionManager::alloc_extent);
+    LOG_PREFIX(TransactionManager::alloc_non_data_extent);
     SUBTRACET(seastore_tm, "{} len={}, placement_hint={}, laddr_hint={}",
               t, T::TYPE, len, placement_hint, laddr_hint);
     ceph_assert(is_aligned(laddr_hint, epm->get_block_size()));
-    auto ext = cache->alloc_new_extent<T>(
+    auto ext = cache->alloc_new_non_data_extent<T>(
       t,
       len,
       placement_hint,
@@ -309,13 +309,64 @@ class TransactionManager : public ExtentCallbackInterface {
       ext->get_paddr(),
       *ext
     ).si_then([ext=std::move(ext), laddr_hint, &t](auto &&) mutable {
-      LOG_PREFIX(TransactionManager::alloc_extent);
+      LOG_PREFIX(TransactionManager::alloc_non_data_extent);
       SUBDEBUGT(seastore_tm, "new extent: {}, laddr_hint: {}", t, *ext, laddr_hint);
       return alloc_extent_iertr::make_ready_future<TCachedExtentRef<T>>(
 	std::move(ext));
     });
   }
 
+  /**
+   * alloc_data_extents
+   *
+   * Allocates a new block of type T with the minimum lba range of size len
+   * greater than laddr_hint.
+   */
+  using alloc_extents_iertr = alloc_extent_iertr;
+  template <typename T>
+  using alloc_extents_ret = alloc_extents_iertr::future<
+    std::vector<TCachedExtentRef<T>>>;
+  template <typename T>
+  alloc_extents_ret<T> alloc_data_extents(
+    Transaction &t,
+    laddr_t laddr_hint,
+    extent_len_t len,
+    placement_hint_t placement_hint = placement_hint_t::HOT) {
+    LOG_PREFIX(TransactionManager::alloc_data_extents);
+    SUBTRACET(seastore_tm, "{} len={}, placement_hint={}, laddr_hint={}",
+              t, T::TYPE, len, placement_hint, laddr_hint);
+    ceph_assert(is_aligned(laddr_hint, epm->get_block_size()));
+    auto exts = cache->alloc_new_data_extents<T>(
+      t,
+      len,
+      placement_hint,
+      INIT_GENERATION);
+    return seastar::do_with(
+      std::move(exts),
+      laddr_hint,
+      [this, &t](auto &exts, auto &laddr_hint) {
+      return trans_intr::do_for_each(
+        exts,
+        [this, &t, &laddr_hint](auto &ext) {
+        return lba_manager->alloc_extent(
+          t,
+          laddr_hint,
+          ext->get_length(),
+          ext->get_paddr(),
+          *ext
+        ).si_then([&ext, &laddr_hint, &t](auto &&) mutable {
+          LOG_PREFIX(TransactionManager::alloc_extents);
+          SUBDEBUGT(seastore_tm, "new extent: {}, laddr_hint: {}", t, *ext, laddr_hint);
+          laddr_hint += ext->get_length();
+          return alloc_extent_iertr::now();
+        });
+      }).si_then([&exts] {
+        return alloc_extent_iertr::make_ready_future<
+          std::vector<TCachedExtentRef<T>>>(std::move(exts));
+      });
+    });
+  }
+
   template <typename T>
   read_extent_ret<T> get_mutable_extent_by_laddr(Transaction &t, laddr_t laddr, extent_len_t len) {
     return get_pin(t, laddr
@@ -540,7 +591,6 @@ class TransactionManager : public ExtentCallbackInterface {
    *
    * allocates more than one new blocks of type T.
    */
-   using alloc_extents_iertr = alloc_extent_iertr;
    template<class T>
    alloc_extents_iertr::future<std::vector<TCachedExtentRef<T>>>
    alloc_extents(
@@ -557,7 +607,7 @@ class TransactionManager : public ExtentCallbackInterface {
                        boost::make_counting_iterator(0),
                        boost::make_counting_iterator(num),
          [this, &t, len, hint, &extents] (auto i) {
-         return alloc_extent<T>(t, hint, len).si_then(
+         return alloc_non_data_extent<T>(t, hint, len).si_then(
            [&extents](auto &&node) {
            extents.push_back(node);
          });
diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc
index 62891aa864c8..01658c4f0637 100644
--- a/src/crimson/tools/store_nbd/tm_driver.cc
+++ b/src/crimson/tools/store_nbd/tm_driver.cc
@@ -33,12 +33,20 @@ seastar::future<> TMDriver::write(
           crimson::ct_error::pass_further_all{}
         ).si_then([this, offset, &t, &ptr] {
           logger().debug("dec_ref complete");
-          return tm->alloc_extent<TestBlock>(t, offset, ptr.length());
-        }).si_then([this, offset, &t, &ptr](auto ext) {
-          boost::ignore_unused(offset);  // avoid clang warning;
-          assert(ext->get_laddr() == (size_t)offset);
-          assert(ext->get_bptr().length() == ptr.length());
-          ext->get_bptr().swap(ptr);
+          return tm->alloc_data_extents<TestBlock>(t, offset, ptr.length());
+        }).si_then([this, offset, &t, &ptr](auto extents) mutable {
+	  boost::ignore_unused(offset);  // avoid clang warning;
+	  auto off = offset;
+	  auto left = ptr.length();
+	  size_t written = 0;
+	  for (auto &ext : extents) {
+	    assert(ext->get_laddr() == (size_t)off);
+	    assert(ext->get_bptr().length() <= left);
+	    ptr.copy_out(written, ext->get_length(), ext->get_bptr().c_str());
+	    off += ext->get_length();
+	    left -= ext->get_length();
+	  }
+	  assert(!left);
           logger().debug("submitting transaction");
           return tm->submit_transaction(t);
         });
diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc
index f18c3ac670ad..5199f963dc95 100644
--- a/src/test/crimson/seastore/test_btree_lba_manager.cc
+++ b/src/test/crimson/seastore/test_btree_lba_manager.cc
@@ -262,12 +262,14 @@ struct lba_btree_test : btree_test_base {
     ceph_assert(check.count(addr) == 0);
     check.emplace(addr, get_map_val(len));
     lba_btree_update([=, this](auto &btree, auto &t) {
-      auto extent = cache->alloc_new_extent<TestBlock>(
+      auto extents = cache->alloc_new_data_extents<TestBlock>(
 	  t,
 	  TestBlock::SIZE,
 	  placement_hint_t::HOT,
 	  0,
 	  get_paddr());
+      assert(extents.size() == 1);
+      auto extent = extents.front();
       return btree.insert(
 	get_op_context(t), addr, get_map_val(len), extent.get()
       ).si_then([addr, extent](auto p){
@@ -374,7 +376,7 @@ struct btree_lba_manager_test : btree_test_base {
       test_lba_mappings
     };
     if (create_fake_extent) {
-      cache->alloc_new_extent<TestBlockPhysical>(
+      cache->alloc_new_non_data_extent<TestBlockPhysical>(
           *t.t,
           TestBlockPhysical::SIZE,
           placement_hint_t::HOT,
@@ -425,12 +427,14 @@ struct btree_lba_manager_test : btree_test_base {
     auto ret = with_trans_intr(
       *t.t,
       [=, this](auto &t) {
-	auto extent = cache->alloc_new_extent<TestBlock>(
+	auto extents = cache->alloc_new_data_extents<TestBlock>(
 	    t,
 	    TestBlock::SIZE,
 	    placement_hint_t::HOT,
 	    0,
 	    get_paddr());
+	assert(extents.size() == 1);
+	auto extent = extents.front();
 	return lba_manager->alloc_extent(
 	  t, hint, len, extent->get_paddr(), *extent);
       }).unsafe_get0();
diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc
index b249d27e4617..66c9899538c8 100644
--- a/src/test/crimson/seastore/test_seastore_cache.cc
+++ b/src/test/crimson/seastore/test_seastore_cache.cc
@@ -129,7 +129,7 @@ TEST_F(cache_test_t, test_addr_fixup)
     int csum = 0;
     {
       auto t = get_transaction();
-      auto extent = cache->alloc_new_extent<TestBlockPhysical>(
+      auto extent = cache->alloc_new_non_data_extent<TestBlockPhysical>(
 	*t,
 	TestBlockPhysical::SIZE,
 	placement_hint_t::HOT,
@@ -160,7 +160,7 @@ TEST_F(cache_test_t, test_dirty_extent)
     {
       // write out initial test block
       auto t = get_transaction();
-      auto extent = cache->alloc_new_extent<TestBlockPhysical>(
+      auto extent = cache->alloc_new_non_data_extent<TestBlockPhysical>(
 	*t,
 	TestBlockPhysical::SIZE,
 	placement_hint_t::HOT,
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index dfe599177056..c33c9ffce4bc 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -386,9 +386,11 @@ struct transaction_manager_test_t :
     laddr_t hint,
     extent_len_t len,
     char contents) {
-    auto extent = with_trans_intr(*(t.t), [&](auto& trans) {
-      return tm->alloc_extent<TestBlock>(trans, hint, len);
+    auto extents = with_trans_intr(*(t.t), [&](auto& trans) {
+      return tm->alloc_data_extents<TestBlock>(trans, hint, len);
     }).unsafe_get0();
+    assert(extents.size() == 1);
+    auto extent = extents.front();
     extent->set_contents(contents);
     EXPECT_FALSE(test_mappings.contains(extent->get_laddr(), t.mapping_delta));
     EXPECT_EQ(len, extent->get_length());
@@ -723,9 +725,11 @@ struct transaction_manager_test_t :
 		boost::make_counting_iterator(0),
 		boost::make_counting_iterator(num),
 		[&t, this, size](auto) {
-		  return tm->alloc_extent<TestBlock>(
+		  return tm->alloc_data_extents<TestBlock>(
 		    *(t.t), L_ADDR_MIN, size
-		  ).si_then([&t, this, size](auto extent) {
+		  ).si_then([&t, this, size](auto extents) {
+		    assert(extents.size() == 1);
+		    auto extent = extents.front();
 		    extent->set_contents(get_random_contents());
 		    EXPECT_FALSE(
 		      test_mappings.contains(extent->get_laddr(), t.mapping_delta));
@@ -1110,9 +1114,11 @@ struct transaction_manager_test_t :
             o_len - new_offset - new_len)
         }
       ).si_then([this, new_offset, new_len, o_laddr, &t, &bl](auto ret) {
-        return tm->alloc_extent<TestBlock>(t, o_laddr + new_offset, new_len
+        return tm->alloc_data_extents<TestBlock>(t, o_laddr + new_offset, new_len
         ).si_then([this, ret = std::move(ret), new_len,
-                   new_offset, o_laddr, &t, &bl](auto ext) mutable {
+                   new_offset, o_laddr, &t, &bl](auto extents) mutable {
+	  assert(extents.size() == 1);
+	  auto ext = extents.front();
           ceph_assert(ret.size() == 2);
           auto iter = bl.cbegin();
           iter.copy(new_len, ext->get_bptr().c_str());
@@ -1141,9 +1147,11 @@ struct transaction_manager_test_t :
             o_len - new_offset - new_len)
         }
       ).si_then([this, new_offset, new_len, o_laddr, &t, &bl](auto ret) {
-        return tm->alloc_extent<TestBlock>(t, o_laddr + new_offset, new_len
+        return tm->alloc_data_extents<TestBlock>(t, o_laddr + new_offset, new_len
         ).si_then([this, ret = std::move(ret), new_offset, new_len,
-                   o_laddr, &t, &bl](auto ext) mutable {
+                   o_laddr, &t, &bl](auto extents) mutable {
+	  assert(extents.size() == 1);
+	  auto ext = extents.front();
           ceph_assert(ret.size() == 1);
           auto iter = bl.cbegin();
           iter.copy(new_len, ext->get_bptr().c_str());
@@ -1167,9 +1175,11 @@ struct transaction_manager_test_t :
             new_offset)
         }
       ).si_then([this, new_offset, new_len, o_laddr, &t, &bl](auto ret) {
-        return tm->alloc_extent<TestBlock>(t, o_laddr + new_offset, new_len
+        return tm->alloc_data_extents<TestBlock>(t, o_laddr + new_offset, new_len
         ).si_then([this, ret = std::move(ret), new_len, o_laddr, &t, &bl]
-          (auto ext) mutable {
+          (auto extents) mutable {
+	  assert(extents.size() == 1);
+	  auto ext = extents.front();
           ceph_assert(ret.size() == 1);
           auto iter = bl.cbegin();
           iter.copy(new_len, ext->get_bptr().c_str());

From 554db8d88dc4149a1698cb31c64b2a5c6f29f4ee Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 6 Nov 2023 18:33:01 +0800
Subject: [PATCH 1769/2492] crimson/os/seastore/transaction_manager:
 TransactionManager::alloc_extent(s) may return enospc

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/common/errorator.h                | 17 +++++++++++
 src/crimson/os/seastore/cache.h               | 13 ++++----
 .../flat_collection_manager.cc                |  5 +++-
 .../os/seastore/extent_placement_manager.h    | 20 ++++++++-----
 .../os/seastore/object_data_handler.cc        | 21 +++++++++----
 .../omap_manager/btree/btree_omap_manager.cc  | 10 +++++--
 .../btree/omap_btree_node_impl.cc             | 30 +++++++++++++++----
 .../node_extent_manager/seastore.h            |  5 +++-
 .../random_block_manager/block_rb_manager.cc  |  3 ++
 src/crimson/os/seastore/transaction_manager.h |  9 +++++-
 .../seastore/test_transaction_manager.cc      | 15 ++++++++--
 11 files changed, 116 insertions(+), 32 deletions(-)

diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h
index c5d63d5b9c1d..656af5813560 100644
--- a/src/crimson/common/errorator.h
+++ b/src/crimson/common/errorator.h
@@ -200,6 +200,23 @@ struct unthrowable_wrapper : error_t<unthrowable_wrapper<ErrorT, ErrorV>> {
     }
   };
 
+  class assert_failure {
+    const char* const msg = nullptr;
+  public:
+    template <std::size_t N>
+    assert_failure(const char (&msg)[N])
+      : msg(msg) {
+    }
+    assert_failure() = default;
+
+    void operator()(const unthrowable_wrapper&) {
+      if (msg) {
+        ceph_abort(msg);
+      } else {
+        ceph_abort();
+      }
+    }
+  };
 
 private:
   // can be used only to initialize the `instance` member
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index a6aeb357148f..75463f465fe1 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -884,17 +884,20 @@ class Cache {
 #else
     auto result = epm.alloc_new_non_data_extent(t, T::TYPE, length, hint, gen);
 #endif
-    auto ret = CachedExtent::make_cached_extent_ref<T>(std::move(result.bp));
+    if (!result) {
+      return nullptr;
+    }
+    auto ret = CachedExtent::make_cached_extent_ref<T>(std::move(result->bp));
     ret->init(CachedExtent::extent_state_t::INITIAL_WRITE_PENDING,
-              result.paddr,
+              result->paddr,
               hint,
-              result.gen,
+              result->gen,
 	      t.get_trans_id());
     t.add_fresh_extent(ret);
     SUBDEBUGT(seastore_cache,
               "allocated {} {}B extent at {}, hint={}, gen={} -- {}",
-              t, T::TYPE, length, result.paddr,
-              hint, rewrite_gen_printer_t{result.gen}, *ret);
+              t, T::TYPE, length, result->paddr,
+              hint, rewrite_gen_printer_t{result->gen}, *ret);
     return ret;
   }
   /**
diff --git a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
index 81353655694e..c32dc66619a0 100644
--- a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
+++ b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc
@@ -88,7 +88,10 @@ FlatCollectionManager::create(coll_root_t &coll_root, Transaction &t,
 	  }).si_then([] (auto) {
             return create_iertr::make_ready_future<>();
           });
-        });
+        }).handle_error_interruptible(
+	  crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	  create_iertr::pass_further{}
+	);
       }
       case CollectionNode::create_result_t::SUCCESS: {
         return create_iertr::make_ready_future<>();
diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h
index 70dbc428d10b..b1a4de2ef457 100644
--- a/src/crimson/os/seastore/extent_placement_manager.h
+++ b/src/crimson/os/seastore/extent_placement_manager.h
@@ -279,7 +279,7 @@ class ExtentPlacementManager {
     bufferptr bp;
     rewrite_gen_t gen;
   };
-  alloc_result_t alloc_new_non_data_extent(
+  std::optional<alloc_result_t> alloc_new_non_data_extent(
     Transaction& t,
     extent_types_t type,
     extent_len_t length,
@@ -298,11 +298,6 @@ class ExtentPlacementManager {
     data_category_t category = get_extent_category(type);
     gen = adjust_generation(category, type, hint, gen);
 
-    // XXX: bp might be extended to point to different memory (e.g. PMem)
-    // according to the allocator.
-    auto bp = ceph::bufferptr(
-      buffer::create_page_aligned(length));
-    bp.zero();
     paddr_t addr;
 #ifdef UNIT_TESTS_BUILT
     if (unlikely(external_paddr.has_value())) {
@@ -320,7 +315,18 @@ class ExtentPlacementManager {
 	  generation_to_writer(gen)]->alloc_paddr(length);
     }
     assert(!(category == data_category_t::DATA));
-    return {addr, std::move(bp), gen};
+
+    if (addr.is_null()) {
+      return std::nullopt;
+    }
+
+    // XXX: bp might be extended to point to different memory (e.g. PMem)
+    // according to the allocator.
+    auto bp = ceph::bufferptr(
+      buffer::create_page_aligned(length));
+    bp.zero();
+
+    return alloc_result_t{addr, std::move(bp), gen};
   }
 
   std::list<alloc_result_t> alloc_new_data_extents(
diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 5e49a48a1880..39c0be33d6de 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -543,7 +543,10 @@ ObjectDataHandler::write_ret do_insertions(
 	  auto iter = region.bl->cbegin();
 	  iter.copy(region.len, extent->get_bptr().c_str());
 	  return ObjectDataHandler::write_iertr::now();
-	});
+	}).handle_error_interruptible(
+	  crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	  ObjectDataHandler::write_iertr::pass_further{}
+	);
       } else if (region.is_zero()) {
 	DEBUGT("reserving: {}~{}",
 	       ctx.t,
@@ -564,7 +567,10 @@ ObjectDataHandler::write_ret do_insertions(
 	  }
 	  ceph_assert(pin->get_key() == region.addr);
 	  return ObjectDataHandler::write_iertr::now();
-	});
+	}).handle_error_interruptible(
+	  crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	  ObjectDataHandler::write_iertr::pass_further{}
+	);
       } else {
 	ceph_abort("impossible");
 	return ObjectDataHandler::write_iertr::now();
@@ -1057,7 +1063,10 @@ ObjectDataHandler::write_ret ObjectDataHandler::prepare_data_reservation(
 	pin->get_key(),
 	pin->get_length());
       return write_iertr::now();
-    });
+    }).handle_error_interruptible(
+      crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+      write_iertr::pass_further{}
+    );
   }
 }
 
@@ -1677,12 +1686,12 @@ ObjectDataHandler::clone_ret ObjectDataHandler::clone_extents(
 	  return TransactionManager::reserve_extent_iertr::now();
 	});
       });
-    },
+    }
+  ).handle_error_interruptible(
     ObjectDataHandler::write_iertr::pass_further{},
     crimson::ct_error::assert_all{
       "object_data_handler::clone invalid error"
-    }
-  );
+  });
 }
 
 ObjectDataHandler::clone_ret ObjectDataHandler::clone(
diff --git a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
index c182965764ee..046cc286208b 100644
--- a/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/btree_omap_manager.cc
@@ -32,7 +32,10 @@ BtreeOMapManager::initialize_omap(Transaction &t, laddr_t hint)
       t.get_omap_tree_stats().depth = 1u;
       t.get_omap_tree_stats().extents_num_delta++;
       return initialize_omap_iertr::make_ready_future<omap_root_t>(omap_root);
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    TransactionManager::alloc_extent_iertr::pass_further{}
+  );
 }
 
 BtreeOMapManager::get_root_ret
@@ -65,7 +68,10 @@ BtreeOMapManager::handle_root_split(
     oc.t.get_omap_tree_stats().depth = omap_root.depth;
     ++(oc.t.get_omap_tree_stats().extents_num_delta);
     return seastar::now();
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    TransactionManager::alloc_extent_iertr::pass_further{}
+  );
 }
 
 BtreeOMapManager::handle_root_merge_ret
diff --git a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
index c932f6d590c1..96a1e3bdefe7 100644
--- a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
@@ -338,7 +338,10 @@ OMapInnerNode:: make_split_children(omap_context_t oc)
       return split_children_ret(
              interruptible::ready_future_marker{},
              std::make_tuple(left, right, split_into(*left, *right)));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    split_children_iertr::pass_further{}
+  );
 }
 
 OMapInnerNode::full_merge_ret
@@ -353,7 +356,10 @@ OMapInnerNode::make_full_merge(omap_context_t oc, OMapNodeRef right)
       return full_merge_ret(
         interruptible::ready_future_marker{},
         std::move(replacement));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    full_merge_iertr::pass_further{}
+  );
 }
 
 OMapInnerNode::make_balanced_ret
@@ -373,7 +379,10 @@ OMapInnerNode::make_balanced(omap_context_t oc, OMapNodeRef _right)
              std::make_tuple(replacement_left, replacement_right,
                              balance_into_new_nodes(*this, right,
                                *replacement_left, *replacement_right)));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    make_balanced_iertr::pass_further{}
+  );
 }
 
 OMapInnerNode::merge_entry_ret
@@ -670,7 +679,10 @@ OMapLeafNode::make_split_children(omap_context_t oc)
       return split_children_ret(
              interruptible::ready_future_marker{},
              std::make_tuple(left, right, split_into(*left, *right)));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    split_children_iertr::pass_further{}
+  );
 }
 
 OMapLeafNode::full_merge_ret
@@ -685,7 +697,10 @@ OMapLeafNode::make_full_merge(omap_context_t oc, OMapNodeRef right)
       return full_merge_ret(
         interruptible::ready_future_marker{},
         std::move(replacement));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    full_merge_iertr::pass_further{}
+  );
 }
 
 OMapLeafNode::make_balanced_ret
@@ -706,7 +721,10 @@ OMapLeafNode::make_balanced(omap_context_t oc, OMapNodeRef _right)
                balance_into_new_nodes(
                  *this, right,
                  *replacement_left, *replacement_right)));
-  });
+  }).handle_error_interruptible(
+    crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+    make_balanced_iertr::pass_further{}
+  );
 }
 
 
diff --git a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
index cff67f9cf51f..fb8fe54271d3 100644
--- a/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
+++ b/src/crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager/seastore.h
@@ -145,7 +145,10 @@ class SeastoreNodeExtentManager final: public TransactionManagerHandle {
       assert(extent->get_length() == len);
       std::ignore = len;
       return NodeExtentRef(extent);
-    });
+    }).handle_error_interruptible(
+      crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+      alloc_iertr::pass_further{}
+    );
   }
 
   retire_iertr::future<> retire_extent(
diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
index 6bae96f84a5a..cca520a9bfea 100644
--- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
+++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
@@ -51,6 +51,9 @@ paddr_t BlockRBManager::alloc_extent(size_t size)
   LOG_PREFIX(BlockRBManager::alloc_extent);
   assert(allocator);
   auto alloc = allocator->alloc_extent(size);
+  if (!alloc) {
+    return P_ADDR_NULL;
+  }
   ceph_assert((*alloc).num_intervals() == 1);
   auto extent = (*alloc).begin();
   ceph_assert(size == extent.get_len());
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 4bacd78fe60b..7720469c2821 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -284,7 +284,8 @@ class TransactionManager : public ExtentCallbackInterface {
    * Allocates a new block of type T with the minimum lba range of size len
    * greater than laddr_hint.
    */
-  using alloc_extent_iertr = LBAManager::alloc_extent_iertr;
+  using alloc_extent_iertr = LBAManager::alloc_extent_iertr::extend<
+    crimson::ct_error::enospc>;
   template <typename T>
   using alloc_extent_ret = alloc_extent_iertr::future<TCachedExtentRef<T>>;
   template <typename T>
@@ -302,6 +303,9 @@ class TransactionManager : public ExtentCallbackInterface {
       len,
       placement_hint,
       INIT_GENERATION);
+    if (!ext) {
+      return crimson::ct_error::enospc::make();
+    }
     return lba_manager->alloc_extent(
       t,
       laddr_hint,
@@ -341,6 +345,9 @@ class TransactionManager : public ExtentCallbackInterface {
       len,
       placement_hint,
       INIT_GENERATION);
+    if (exts.empty()) {
+      return crimson::ct_error::enospc::make();
+    }
     return seastar::do_with(
       std::move(exts),
       laddr_hint,
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index c33c9ffce4bc..378644dafa84 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -1135,7 +1135,10 @@ struct transaction_manager_test_t :
                     std::move(lpin), std::move(ext), std::move(rpin)));
             });
           });
-        });
+        }).handle_error_interruptible(
+	  crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	  crimson::ct_error::pass_further_all{}
+	);
       });
     } else if (new_offset == 0 && o_len != new_offset + new_len) {
       return tm->remap_pin<TestBlock, 1>(
@@ -1164,7 +1167,10 @@ struct transaction_manager_test_t :
                   nullptr, std::move(ext), std::move(rpin)));
           });
         });
-      });
+      }).handle_error_interruptible(
+	crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	crimson::ct_error::pass_further_all{}
+      );
     } else if (new_offset != 0 && o_len == new_offset + new_len) {
       return tm->remap_pin<TestBlock, 1>(
         t,
@@ -1191,7 +1197,10 @@ struct transaction_manager_test_t :
                   std::move(lpin), std::move(ext), nullptr));
           });
         });
-      });
+      }).handle_error_interruptible(
+	crimson::ct_error::enospc::assert_failure{"unexpected enospc"},
+	crimson::ct_error::pass_further_all{}
+      );
     } else {
       ceph_abort("impossible");
         return _overwrite_pin_iertr::make_ready_future<

From e418cfd6f58f3135a6fa8145a844791df4b8e24c Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 1 Nov 2023 14:50:34 +0800
Subject: [PATCH 1770/2492] test/crimson/seastore: test avlallocator's
 scattered allocation

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/test/crimson/seastore/test_extent_allocator.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/test/crimson/seastore/test_extent_allocator.cc b/src/test/crimson/seastore/test_extent_allocator.cc
index 8217e5a660f1..87a5d0feb584 100644
--- a/src/test/crimson/seastore/test_extent_allocator.cc
+++ b/src/test/crimson/seastore/test_extent_allocator.cc
@@ -57,6 +57,9 @@ struct allocator_test_t :
   auto allocate(size_t size) {
     return allocator->alloc_extent(size);
   }
+  auto allocates(size_t size) {
+    return allocator->alloc_extents(size);
+  }
   void free(uint64_t start, uint64_t length) {
     allocator->free_extent(start, length);
   }
@@ -98,7 +101,7 @@ TEST_P(allocator_test_t, test_init_alloc_free)
   }
 }
 
-TEST_P(allocator_test_t, test_alloc_failure)
+TEST_P(allocator_test_t, test_scattered_alloc)
 {
   uint64_t block_size = 8192;
   uint64_t capacity = 1024 * block_size;
@@ -108,13 +111,12 @@ TEST_P(allocator_test_t, test_alloc_failure)
     allocator->mark_extent_used(0, block_size * 256);
     allocator->mark_extent_used(block_size * 512, block_size * 256);
 
-    auto result = allocate(block_size * 512);
-    ASSERT_EQ(false, result.has_value());
+    auto result = allocates(block_size * 512);
+    ASSERT_EQ(true, result.has_value());
 
     free(0, block_size * 256);
-    allocator->mark_extent_used(0, block_size * 512);
 
-    result = allocate(block_size * 512);
+    result = allocates(block_size * 512);
     ASSERT_EQ(false, result.has_value());
   }
 }
@@ -142,9 +144,9 @@ TEST_P(allocator_test_t, test_random_alloc_verify)
     for (auto p : alloc_map) {
       free(p.first, p.second);
       avail += p.second;
-      alloc_map.erase(p.first, p.second);
       ASSERT_EQ(avail, allocator->get_available_size());
     }
+    alloc_map.clear();
     ASSERT_EQ(capacity, allocator->get_available_size());
 
     for (int i = 0; i < 100; i++) {

From 63b87f70ce98d5ce1db73b85115ae038dc9a9a58 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 6 Nov 2023 18:34:28 +0800
Subject: [PATCH 1771/2492] test/crimson/seastore: add unittest cases for
 scattered allocation

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/common/errorator.h                |  6 ++
 src/crimson/os/seastore/async_cleaner.h       | 15 +++++
 .../os/seastore/extent_placement_manager.h    | 21 ++++++
 .../os/seastore/random_block_manager.h        |  3 +
 .../random_block_manager/block_rb_manager.cc  | 19 ++++++
 .../random_block_manager/block_rb_manager.h   |  4 ++
 .../seastore/test_transaction_manager.cc      | 66 +++++++++++++++++++
 7 files changed, 134 insertions(+)

diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h
index 656af5813560..ebc5a95bb3e6 100644
--- a/src/crimson/common/errorator.h
+++ b/src/crimson/common/errorator.h
@@ -11,6 +11,8 @@
 #include "crimson/common/utility.h"
 #include "include/ceph_assert.h"
 
+class transaction_manager_test_t;
+
 namespace crimson::interruptible {
 
 template <typename, typename>
@@ -528,6 +530,7 @@ struct errorator {
 
   protected:
     using base_t::get_exception;
+    friend class ::transaction_manager_test_t;
   public:
     using errorator_type = ::crimson::errorator<AllowedErrors...>;
     using promise_type = seastar::promise<ValueT>;
@@ -708,6 +711,9 @@ struct errorator {
     auto unsafe_get0() {
       return seastar::future<ValueT>::get0();
     }
+    void unsafe_wait() {
+      seastar::future<ValueT>::wait();
+    }
 
     template <class FuncT>
     _future finally(FuncT &&func) {
diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h
index 8f69d742d741..1c94745d820c 100644
--- a/src/crimson/os/seastore/async_cleaner.h
+++ b/src/crimson/os/seastore/async_cleaner.h
@@ -1156,6 +1156,10 @@ class AsyncCleaner {
 
   virtual std::size_t get_reclaim_size_per_cycle() const = 0;
 
+#ifdef UNIT_TESTS_BUILT
+  virtual void prefill_fragmented_devices() {}
+#endif
+
   // test only
   virtual bool check_usage() = 0;
 
@@ -1677,6 +1681,17 @@ class RBMCleaner : public AsyncCleaner {
     return 0;
   }
 
+#ifdef UNIT_TESTS_BUILT
+  void prefill_fragmented_devices() final {
+    LOG_PREFIX(RBMCleaner::prefill_fragmented_devices);
+    SUBDEBUG(seastore_cleaner, "");
+    auto rbs = rb_group->get_rb_managers();
+    for (auto p : rbs) {
+      p->prefill_fragmented_device();
+    }
+  }
+#endif
+
   RandomBlockManager* get_rbm(paddr_t paddr) {
     auto rbs = rb_group->get_rb_managers();
     for (auto p : rbs) {
diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h
index b1a4de2ef457..6a71a4ca9864 100644
--- a/src/crimson/os/seastore/extent_placement_manager.h
+++ b/src/crimson/os/seastore/extent_placement_manager.h
@@ -48,6 +48,10 @@ class ExtentOolWriter {
 
   virtual bool can_inplace_rewrite(Transaction& t,
     CachedExtentRef extent) = 0;
+
+#ifdef UNIT_TESTS_BUILT
+  virtual void prefill_fragmented_devices() {}
+#endif
 };
 using ExtentOolWriterRef = std::unique_ptr<ExtentOolWriter>;
 
@@ -152,6 +156,13 @@ class RandomBlockOolWriter : public ExtentOolWriter {
     return crimson::os::seastore::can_inplace_rewrite(extent->get_type());
   }
 
+#ifdef UNIT_TESTS_BUILT
+  void prefill_fragmented_devices() final {
+    LOG_PREFIX(RandomBlockOolWriter::prefill_fragmented_devices);
+    SUBDEBUG(seastore_epm, "");
+    return rb_cleaner->prefill_fragmented_devices();
+  }
+#endif
 private:
   alloc_write_iertr::future<> do_write(
     Transaction& t,
@@ -377,6 +388,16 @@ class ExtentPlacementManager {
     return allocs;
   }
 
+#ifdef UNIT_TESTS_BUILT
+  void prefill_fragmented_devices() {
+    LOG_PREFIX(ExtentPlacementManager::prefill_fragmented_devices);
+    SUBDEBUG(seastore_epm, "");
+    for (auto &writer : writer_refs) {
+      writer->prefill_fragmented_devices();
+    }
+  }
+#endif
+
   /**
    * dispatch_result_t
    *
diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h
index f0e0f018bd10..fc45af3b2f55 100644
--- a/src/crimson/os/seastore/random_block_manager.h
+++ b/src/crimson/os/seastore/random_block_manager.h
@@ -149,6 +149,9 @@ class RandomBlockManager {
   virtual rbm_extent_state_t get_extent_state(paddr_t addr, size_t size) = 0;
   virtual size_t get_journal_size() const = 0;
   virtual ~RandomBlockManager() {}
+#ifdef UNIT_TESTS_BUILT
+  virtual void prefill_fragmented_device() = 0;
+#endif
 };
 using RandomBlockManagerRef = std::unique_ptr<RandomBlockManager>;
 
diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
index cca520a9bfea..715aafbc8a8e 100644
--- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
+++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.cc
@@ -182,6 +182,25 @@ BlockRBManager::write_ertr::future<> BlockRBManager::write(
     std::move(bptr));
 }
 
+#ifdef UNIT_TESTS_BUILT
+void BlockRBManager::prefill_fragmented_device()
+{
+  LOG_PREFIX(BlockRBManager::prefill_fragmented_device);
+  // the first 2 blocks must be allocated to lba root
+  // and backref root during mkfs
+  for (size_t block = get_block_size() * 2;
+      block <= get_size() - get_block_size() * 2;
+      block += get_block_size() * 2) {
+    DEBUG("marking {}~{} used",
+      get_start_rbm_addr() + block,
+      get_block_size());
+    allocator->mark_extent_used(
+      get_start_rbm_addr() + block,
+      get_block_size());
+  }
+}
+#endif
+
 std::ostream &operator<<(std::ostream &out, const rbm_metadata_header_t &header)
 {
   out << " rbm_metadata_header_t(size=" << header.size
diff --git a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
index 14d2a28a3146..bc33710eee02 100644
--- a/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
+++ b/src/crimson/os/seastore/random_block_manager/block_rb_manager.h
@@ -127,6 +127,10 @@ class BlockRBManager final : public RandomBlockManager {
     return device->get_journal_size();
   }
 
+#ifdef UNIT_TESTS_BUILT
+  void prefill_fragmented_device() final;
+#endif
+
 private:
   /*
    * this contains the number of bitmap blocks, free blocks and
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index 378644dafa84..869eae716cd6 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -398,6 +398,40 @@ struct transaction_manager_test_t :
     return extent;
   }
 
+  std::vector<TestBlockRef> alloc_extents(
+    test_transaction_t &t,
+    laddr_t hint,
+    extent_len_t len,
+    char contents) {
+    auto extents = with_trans_intr(*(t.t), [&](auto& trans) {
+      return tm->alloc_data_extents<TestBlock>(trans, hint, len);
+    }).unsafe_get0();
+    size_t length = 0;
+    for (auto &extent : extents) {
+      extent->set_contents(contents);
+      length += extent->get_length();
+      EXPECT_FALSE(test_mappings.contains(extent->get_laddr(), t.mapping_delta));
+      test_mappings.alloced(hint, *extent, t.mapping_delta);
+    }
+    EXPECT_EQ(len, length);
+    return extents;
+  }
+
+  void alloc_extents_deemed_fail(
+    test_transaction_t &t,
+    laddr_t hint,
+    extent_len_t len,
+    char contents)
+  {
+    std::cout << __func__ << std::endl;
+    auto fut = with_trans_intr(*(t.t), [&](auto& trans) {
+      return tm->alloc_data_extents<TestBlock>(trans, hint, len);
+    });
+    fut.unsafe_wait();
+    assert(fut.failed());
+    (void)fut.get_exception();
+  }
+
   TestBlockRef alloc_extent(
     test_transaction_t &t,
     laddr_t hint,
@@ -1675,6 +1709,30 @@ struct tm_multi_tier_device_test_t :
   tm_multi_tier_device_test_t() : transaction_manager_test_t(1, 2) {}
 };
 
+struct tm_random_block_device_test_t :
+  public transaction_manager_test_t {
+
+  tm_random_block_device_test_t() : transaction_manager_test_t(1, 0) {}
+};
+
+TEST_P(tm_random_block_device_test_t, scatter_allocation)
+{
+  run_async([this] {
+    constexpr laddr_t ADDR = 0xFF * 4096;
+    epm->prefill_fragmented_devices();
+    auto t = create_transaction();
+    for (int i = 0; i < 1991; i++) {
+      auto extents = alloc_extents(t, ADDR + i * 16384, 16384, 'a');
+      std::cout << "num of extents: " << extents.size() << std::endl;
+    }
+    alloc_extents_deemed_fail(t, ADDR + 1991 * 16384, 16384, 'a');
+    check_mappings(t);
+    check();
+    submit_transaction(std::move(t));
+    check();
+  });
+}
+
 TEST_P(tm_single_device_test_t, basic)
 {
   constexpr laddr_t SIZE = 4096;
@@ -2133,3 +2191,11 @@ INSTANTIATE_TEST_SUITE_P(
     "segmented"
   )
 );
+
+INSTANTIATE_TEST_SUITE_P(
+  transaction_manager_test,
+  tm_random_block_device_test_t,
+  ::testing::Values (
+    "circularbounded"
+  )
+);

From cf509b1933bdba9028501cedf45d9ce757d9995e Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 11 Dec 2023 10:23:36 +0800
Subject: [PATCH 1772/2492] crimson/os/seastore/transaction_manager: fix
 errorator mismatch

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/transaction_manager.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 7720469c2821..640b98f79426 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -384,7 +384,7 @@ class TransactionManager : public ExtentCallbackInterface {
       return this->read_pin<T>(t, std::move(pin));
     }).si_then([this, &t](auto extent) {
       auto ext = get_mutable_extent(t, extent)->template cast<T>();
-      return alloc_extent_iertr::make_ready_future<TCachedExtentRef<T>>(
+      return read_extent_iertr::make_ready_future<TCachedExtentRef<T>>(
 	std::move(ext));
     });
   }

From b35549d714f7e02facdc5536d28ee520221576e1 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 19 Jan 2024 11:14:39 +0800
Subject: [PATCH 1773/2492] crimson/os/seastore/object_data_handler: allow
 alloc multiple extents for a single request

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../os/seastore/object_data_handler.cc        | 29 +++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 39c0be33d6de..96963ea96f7a 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -526,22 +526,27 @@ ObjectDataHandler::write_ret do_insertions(
 	       ctx.t,
 	       region.addr,
 	       region.len);
-	return ctx.tm.alloc_extent<ObjectDataBlock>(
+	return ctx.tm.alloc_data_extents<ObjectDataBlock>(
 	  ctx.t,
 	  region.addr,
 	  region.len
-	).si_then([&region](auto extent) {
-	  if (extent->get_laddr() != region.addr) {
-	    logger().debug(
-	      "object_data_handler::do_insertions alloc got addr {},"
-	      " should have been {}",
-	      extent->get_laddr(),
-	      region.addr);
-	  }
-	  ceph_assert(extent->get_laddr() == region.addr);
-	  ceph_assert(extent->get_length() == region.len);
+        ).si_then([&region](auto extents) {
+          auto off = region.addr;
+          auto left = region.len;
 	  auto iter = region.bl->cbegin();
-	  iter.copy(region.len, extent->get_bptr().c_str());
+          for (auto &extent : extents) {
+            ceph_assert(left >= extent->get_length());
+            if (extent->get_laddr() != off) {
+              logger().debug(
+                "object_data_handler::do_insertions alloc got addr {},"
+                " should have been {}",
+                extent->get_laddr(),
+                off);
+            }
+            iter.copy(extent->get_length(), extent->get_bptr().c_str());
+            off += extent->get_length();
+            left -= extent->get_length();
+          }
 	  return ObjectDataHandler::write_iertr::now();
 	}).handle_error_interruptible(
 	  crimson::ct_error::enospc::assert_failure{"unexpected enospc"},

From c35866f3716715b2e8c35ac76c8c14582db1f7f2 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 19 Jan 2024 13:26:03 +0800
Subject: [PATCH 1774/2492] crimson/os/seastore/transaction_manager: allow
 allocating multiple extents when rewriting data extents

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/lba_manager.cc        |   2 +
 src/crimson/os/seastore/lba_manager.h         |   2 +
 .../lba_manager/btree/btree_lba_manager.cc    |   6 +-
 .../lba_manager/btree/btree_lba_manager.h     |   2 +
 .../os/seastore/transaction_manager.cc        | 119 ++++++++++++++----
 5 files changed, 104 insertions(+), 27 deletions(-)

diff --git a/src/crimson/os/seastore/lba_manager.cc b/src/crimson/os/seastore/lba_manager.cc
index d113bbd1e957..c3473bc483eb 100644
--- a/src/crimson/os/seastore/lba_manager.cc
+++ b/src/crimson/os/seastore/lba_manager.cc
@@ -16,7 +16,9 @@ LBAManager::update_mappings(
     return update_mapping(
       t,
       extent->get_laddr(),
+      extent->get_length(),
       extent->get_prior_paddr_and_reset(),
+      extent->get_length(),
       extent->get_paddr(),
       nullptr	// all the extents should have already been
 		// added to the fixed_kv_btree
diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h
index d4d1826bcf90..0c02e1bd61bb 100644
--- a/src/crimson/os/seastore/lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager.h
@@ -191,7 +191,9 @@ class LBAManager {
   virtual update_mapping_ret update_mapping(
     Transaction& t,
     laddr_t laddr,
+    extent_len_t prev_len,
     paddr_t prev_addr,
+    extent_len_t len,
     paddr_t paddr,
     LogicalCachedExtent *nextent) = 0;
 
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
index 1b7f927ec0fe..b9fa0685a80a 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc
@@ -530,7 +530,9 @@ BtreeLBAManager::update_mapping_ret
 BtreeLBAManager::update_mapping(
   Transaction& t,
   laddr_t laddr,
+  extent_len_t prev_len,
   paddr_t prev_addr,
+  extent_len_t len,
   paddr_t addr,
   LogicalCachedExtent *nextent)
 {
@@ -539,13 +541,15 @@ BtreeLBAManager::update_mapping(
   return _update_mapping(
     t,
     laddr,
-    [prev_addr, addr](
+    [prev_addr, addr, prev_len, len](
       const lba_map_val_t &in) {
       assert(!addr.is_null());
       lba_map_val_t ret = in;
       ceph_assert(in.pladdr.is_paddr());
       ceph_assert(in.pladdr.get_paddr() == prev_addr);
+      ceph_assert(in.len == prev_len);
       ret.pladdr = addr;
+      ret.len = len;
       return ret;
     },
     nextent
diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
index 1c907f76d311..626b9e02c0c5 100644
--- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
+++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h
@@ -333,7 +333,9 @@ class BtreeLBAManager : public LBAManager {
   update_mapping_ret update_mapping(
     Transaction& t,
     laddr_t laddr,
+    extent_len_t prev_len,
     paddr_t prev_addr,
+    extent_len_t len,
     paddr_t paddr,
     LogicalCachedExtent*) final;
 
diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index e45224412057..348c43e3c825 100644
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -439,32 +439,99 @@ TransactionManager::rewrite_logical_extent(
 
   auto lextent = extent->cast<LogicalCachedExtent>();
   cache->retire_extent(t, extent);
-  auto nlextent = cache->alloc_new_extent_by_type(
-    t,
-    lextent->get_type(),
-    lextent->get_length(),
-    lextent->get_user_hint(),
-    // get target rewrite generation
-    lextent->get_rewrite_generation())->cast<LogicalCachedExtent>();
-  lextent->get_bptr().copy_out(
-    0,
-    lextent->get_length(),
-    nlextent->get_bptr().c_str());
-  nlextent->set_laddr(lextent->get_laddr());
-  nlextent->set_modify_time(lextent->get_modify_time());
-
-  DEBUGT("rewriting logical extent -- {} to {}", t, *lextent, *nlextent);
-
-  /* This update_mapping is, strictly speaking, unnecessary for delayed_alloc
-   * extents since we're going to do it again once we either do the ool write
-   * or allocate a relative inline addr.  TODO: refactor AsyncCleaner to
-   * avoid this complication. */
-  return lba_manager->update_mapping(
-    t,
-    lextent->get_laddr(),
-    lextent->get_paddr(),
-    nlextent->get_paddr(),
-    nlextent.get());
+  if (get_extent_category(lextent->get_type()) == data_category_t::METADATA) {
+    auto nlextent = cache->alloc_new_extent_by_type(
+      t,
+      lextent->get_type(),
+      lextent->get_length(),
+      lextent->get_user_hint(),
+      // get target rewrite generation
+      lextent->get_rewrite_generation())->cast<LogicalCachedExtent>();
+    lextent->get_bptr().copy_out(
+      0,
+      lextent->get_length(),
+      nlextent->get_bptr().c_str());
+    nlextent->set_laddr(lextent->get_laddr());
+    nlextent->set_modify_time(lextent->get_modify_time());
+
+    DEBUGT("rewriting logical extent -- {} to {}", t, *lextent, *nlextent);
+
+    /* This update_mapping is, strictly speaking, unnecessary for delayed_alloc
+     * extents since we're going to do it again once we either do the ool write
+     * or allocate a relative inline addr.  TODO: refactor AsyncCleaner to
+     * avoid this complication. */
+    return lba_manager->update_mapping(
+      t,
+      lextent->get_laddr(),
+      lextent->get_length(),
+      lextent->get_paddr(),
+      nlextent->get_length(),
+      nlextent->get_paddr(),
+      nlextent.get());
+  } else {
+    assert(get_extent_category(lextent->get_type()) == data_category_t::DATA);
+    auto extents = cache->alloc_new_data_extents_by_type(
+      t,
+      lextent->get_type(),
+      lextent->get_length(),
+      lextent->get_user_hint(),
+      // get target rewrite generation
+      lextent->get_rewrite_generation());
+    return seastar::do_with(
+      std::move(extents),
+      0,
+      lextent->get_length(),
+      [this, lextent, &t](auto &extents, auto &off, auto &left) {
+      return trans_intr::do_for_each(
+        extents,
+        [lextent, this, &t, &off, &left](auto &nextent) {
+        LOG_PREFIX(TransactionManager::rewrite_logical_extent);
+        bool first_extent = (off == 0);
+        ceph_assert(left >= nextent->get_length());
+        auto nlextent = nextent->template cast<LogicalCachedExtent>();
+        lextent->get_bptr().copy_out(
+          0,
+          nlextent->get_length(),
+          nlextent->get_bptr().c_str());
+        nlextent->set_laddr(lextent->get_laddr() + off);
+        nlextent->set_modify_time(lextent->get_modify_time());
+        DEBUGT("rewriting logical extent -- {} to {}", t, *lextent, *nlextent);
+
+        /* This update_mapping is, strictly speaking, unnecessary for delayed_alloc
+         * extents since we're going to do it again once we either do the ool write
+         * or allocate a relative inline addr.  TODO: refactor AsyncCleaner to
+         * avoid this complication. */
+        auto fut = base_iertr::now();
+        if (first_extent) {
+          fut = lba_manager->update_mapping(
+            t,
+            lextent->get_laddr() + off,
+            lextent->get_length(),
+            lextent->get_paddr(),
+            nlextent->get_length(),
+            nlextent->get_paddr(),
+            nlextent.get());
+        } else {
+          fut = lba_manager->alloc_extent(
+            t,
+            lextent->get_laddr() + off,
+            nlextent->get_length(),
+            nlextent->get_paddr(),
+            *nlextent
+          ).si_then([lextent, nlextent, off](auto mapping) {
+            ceph_assert(mapping->get_key() == lextent->get_laddr() + off);
+            ceph_assert(mapping->get_val() == nlextent->get_paddr());
+            return seastar::now();
+          });
+        }
+        return fut.si_then([&off, &left, nlextent] {
+          off += nlextent->get_length();
+          left -= nlextent->get_length();
+          return seastar::now();
+        });
+      });
+    });
+  }
 }
 
 TransactionManager::rewrite_extent_ret TransactionManager::rewrite_extent(

From 5ad241442d2c141ba508faba61f39d70f3f09679 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 5 Feb 2024 01:36:10 +1000
Subject: [PATCH 1775/2492] doc/rados: update PG guidance

Update the "Creating a Pool" section of doc/rados/operations/pools.rst
so that the documentation no longer insists that the user change the
values of "osd_pool_default_pg_num" and "osd_pool_default_pgp_num".

See also: https://github.com/ceph/ceph/pull/55419

Tracker: https://tracker.ceph.com/issues/64259

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/pools.rst | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index dda9e844e965..afe9926589a4 100644
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -94,19 +94,12 @@ To get even more information, you can execute this command with the ``--format``
 Creating a Pool
 ===============
 
-Before creating a pool, consult `Pool, PG and CRUSH Config Reference`_.  Your
-Ceph configuration file contains a setting (namely, ``pg_num``) that determines
-the number of PGs.  However, this setting's default value is NOT appropriate
-for most systems.  In most cases, you should override this default value when
-creating your pool.  For details on PG numbers, see `setting the number of
-placement groups`_
-
-For example:
-
-.. prompt:: bash $
-
-    osd_pool_default_pg_num = 128
-    osd_pool_default_pgp_num = 128
+Before creating a pool, consult `Pool, PG and CRUSH Config Reference`_. The
+Ceph central configuration database in the monitor cluster contains a setting
+(namely, ``pg_num``) that determines the number of PGs per pool when a pool has
+been created and no per-pool value has been specified. It is possible to change
+this value from its default. For more on the subject of setting the number of
+PGs per pool, see `setting the number of placement groups`_.
 
 .. note:: In Luminous and later releases, each pool must be associated with the
    application that will be using the pool. For more information, see

From ebf05d5d6cf0532d321a668a4444d2069f995d4a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Sun, 4 Feb 2024 19:11:41 +0000
Subject: [PATCH 1776/2492] node-proxy: refactor entrypoint

This commit introduces a major refactor of the main
entrypoint.

- subclass threading.Thread:
  - Introduce a new class `BaseThread()` that is a
    `threading.Thread()` abstraction class in order
    to monitor the different threads.
  - `BaseSystem()` inherits from `BaseThread()`.
  - Handle `SIGTERM` signal in order to gracefully shutdown
    node-proxy (make threads exit gracefully, log out from RedFish API, etc.)

Additionally, this:
  - drops the class `Logger()` from util.py which
was not adding value. It is now replaced with a simple `get_logger()`
function.
  - changes the node-proxy API port from 8080 to 9456
    (8080 being widely used for frontend apps...)
  - changes the container entrypoint in order to use the
    `ceph-node-proxy` binary from the packaging

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/api.py    |  68 +++--
 .../ceph_node_proxy/baseredfishsystem.py      | 113 ++++----
 .../ceph_node_proxy/basesystem.py             |  21 +-
 src/ceph-node-proxy/ceph_node_proxy/main.py   | 241 +++++++-----------
 .../ceph_node_proxy/redfish_client.py         |  28 +-
 .../ceph_node_proxy/redfishdellsystem.py      |  24 +-
 .../ceph_node_proxy/reporter.py               |  87 +++----
 src/ceph-node-proxy/ceph_node_proxy/util.py   | 119 ++++++---
 src/cephadm/cephadmlib/daemons/node_proxy.py  |   6 +-
 src/pybind/mgr/cephadm/agent.py               |   2 +-
 src/pybind/mgr/cephadm/services/node_proxy.py |   6 +-
 11 files changed, 341 insertions(+), 374 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/api.py b/src/ceph-node-proxy/ceph_node_proxy/api.py
index 93e41def3bfe..25ae03e51952 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/api.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/api.py
@@ -1,15 +1,15 @@
-import cherrypy
+import cherrypy  # type: ignore
 from urllib.error import HTTPError
-from cherrypy._cpserver import Server
+from cherrypy._cpserver import Server  # type: ignore
 from threading import Thread, Event
 from typing import Dict, Any, List
-from ceph_node_proxy.util import Config, Logger, write_tmp_file
+from ceph_node_proxy.util import Config, get_logger, write_tmp_file
 from ceph_node_proxy.basesystem import BaseSystem
 from ceph_node_proxy.reporter import Reporter
 from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
-    from ceph_node_proxy.main import NodeProxy
+    from ceph_node_proxy.main import NodeProxyManager
 
 
 @cherrypy.tools.auth_basic(on=True)
@@ -21,8 +21,7 @@ def __init__(self, api: 'API') -> None:
 
     @cherrypy.expose
     def start(self) -> Dict[str, str]:
-        self.api.backend.start_client()
-        # self.backend.start_update_loop()
+        self.api.backend.start()
         self.api.reporter.run()
         return {'ok': 'node-proxy daemon started'}
 
@@ -32,9 +31,8 @@ def reload(self) -> Dict[str, str]:
         return {'ok': 'node-proxy config reloaded'}
 
     def _stop(self) -> None:
-        self.api.backend.stop_update_loop()
-        self.api.backend.client.logout()
-        self.api.reporter.stop()
+        self.api.backend.shutdown()
+        self.api.reporter.shutdown()
 
     @cherrypy.expose
     def stop(self) -> Dict[str, str]:
@@ -61,11 +59,11 @@ def __init__(self,
                  addr: str = '0.0.0.0',
                  port: int = 0) -> None:
         super().__init__()
-        self.log = Logger(__name__)
+        self.log = get_logger(__name__)
         self.backend = backend
         self.reporter = reporter
         self.config = config
-        self.socket_port = self.config.__dict__['server']['port'] if not port else port
+        self.socket_port = self.config.__dict__['api']['port'] if not port else port
         self.socket_host = addr
         self.subscribe()
 
@@ -134,10 +132,10 @@ def shutdown(self, **kw: Any) -> int:
 
         if 'force' not in data.keys():
             msg = "The key 'force' wasn't passed."
-            self.log.logger.debug(msg)
+            self.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
         try:
-            result: int = self.backend.shutdown(force=data['force'])
+            result: int = self.backend.shutdown_host(force=data['force'])
         except HTTPError as e:
             raise cherrypy.HTTPError(e.code, e.reason)
         return result
@@ -167,14 +165,14 @@ def _led(self, **kw: Any) -> Dict[str, Any]:
 
         if not led_type:
             msg = "the led type must be provided (either 'chassis' or 'drive')."
-            self.log.logger.debug(msg)
+            self.log.debug(msg)
             raise cherrypy.HTTPError(400, msg)
 
         if led_type == 'drive':
             id_drive_required = not id_drive
             if id_drive_required or id_drive not in self.backend.get_storage():
                 msg = 'A valid device ID must be provided.'
-                self.log.logger.debug(msg)
+                self.log.debug(msg)
                 raise cherrypy.HTTPError(400, msg)
 
         try:
@@ -183,7 +181,7 @@ def _led(self, **kw: Any) -> Dict[str, Any]:
 
                 if 'state' not in data or data['state'] not in ['on', 'off']:
                     msg = "Invalid data. 'state' must be provided and have a valid value (on|off)."
-                    self.log.logger.error(msg)
+                    self.log.error(msg)
                     raise cherrypy.HTTPError(400, msg)
 
                 func: Any = (self.backend.device_led_on if led_type == 'drive' and data['state'] == 'on' else
@@ -228,34 +226,30 @@ def stop(self) -> None:
 
 
 class NodeProxyApi(Thread):
-    def __init__(self,
-                 node_proxy: 'NodeProxy',
-                 username: str,
-                 password: str,
-                 ssl_crt: str,
-                 ssl_key: str) -> None:
+    def __init__(self, node_proxy_mgr: 'NodeProxyManager') -> None:
         super().__init__()
-        self.log = Logger(__name__)
+        self.log = get_logger(__name__)
         self.cp_shutdown_event = Event()
-        self.node_proxy = node_proxy
-        self.username = username
-        self.password = password
-        self.ssl_crt = ssl_crt
-        self.ssl_key = ssl_key
-        self.api = API(self.node_proxy.system,
-                       self.node_proxy.reporter_agent,
-                       self.node_proxy.config)
+        self.node_proxy_mgr = node_proxy_mgr
+        self.username = self.node_proxy_mgr.username
+        self.password = self.node_proxy_mgr.password
+        self.ssl_crt = self.node_proxy_mgr.api_ssl_crt
+        self.ssl_key = self.node_proxy_mgr.api_ssl_key
+        self.system = self.node_proxy_mgr.system
+        self.reporter_agent = self.node_proxy_mgr.reporter_agent
+        self.config = self.node_proxy_mgr.config
+        self.api = API(self.system, self.reporter_agent, self.config)
 
     def check_auth(self, realm: str, username: str, password: str) -> bool:
         return self.username == username and \
             self.password == password
 
     def shutdown(self) -> None:
-        self.log.logger.info('Stopping node-proxy API...')
+        self.log.info('Stopping node-proxy API...')
         self.cp_shutdown_event.set()
 
     def run(self) -> None:
-        self.log.logger.info('node-proxy API configuration...')
+        self.log.info('node-proxy API configuration...')
         cherrypy.config.update({
             'environment': 'production',
             'engine.autoreload.on': False,
@@ -281,11 +275,11 @@ def run(self) -> None:
         cherrypy.server.unsubscribe()
         try:
             cherrypy.engine.start()
-            self.log.logger.info('node-proxy API started.')
+            self.log.info('node-proxy API started.')
             self.cp_shutdown_event.wait()
             self.cp_shutdown_event.clear()
-            cherrypy.engine.stop()
+            cherrypy.engine.exit()
             cherrypy.server.httpserver = None
-            self.log.logger.info('node-proxy API shutdown.')
+            self.log.info('node-proxy API shutdown.')
         except Exception as e:
-            self.log.logger.error(f'node-proxy API error: {e}')
+            self.log.error(f'node-proxy API error: {e}')
diff --git a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
index e80523fed185..ea4e65cc6ac6 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/baseredfishsystem.py
@@ -2,9 +2,8 @@
 import json
 from ceph_node_proxy.basesystem import BaseSystem
 from ceph_node_proxy.redfish_client import RedFishClient
-from threading import Thread, Lock
 from time import sleep
-from ceph_node_proxy.util import Logger, retry
+from ceph_node_proxy.util import get_logger
 from typing import Dict, Any, List, Callable, Union
 from urllib.error import HTTPError, URLError
 
@@ -15,20 +14,16 @@ def __init__(self, **kw: Any) -> None:
         self.common_endpoints: List[str] = kw.get('common_endpoints', ['/Systems/System.Embedded.1',
                                                                        '/UpdateService'])
         self.chassis_endpoint: str = kw.get('chassis_endpoint', '/Chassis/System.Embedded.1')
-        self.log = Logger(__name__)
+        self.log = get_logger(__name__)
         self.host: str = kw['host']
         self.port: str = kw['port']
         self.username: str = kw['username']
         self.password: str = kw['password']
         # move the following line (class attribute?)
         self.client: RedFishClient = RedFishClient(host=self.host, port=self.port, username=self.username, password=self.password)
-        self.log.logger.info(f'redfish system initialization, host: {self.host}, user: {self.username}')
-
-        self.run: bool = False
-        self.thread: Thread
+        self.log.info(f'redfish system initialization, host: {self.host}, user: {self.username}')
         self.data_ready: bool = False
         self.previous_data: Dict = {}
-        self.lock: Lock = Lock()
         self.data: Dict[str, Dict[str, Any]] = {}
         self._system: Dict[str, Dict[str, Any]] = {}
         self._sys: Dict[str, Any] = {}
@@ -44,71 +39,63 @@ def __init__(self, **kw: Any) -> None:
                                                                    'firmwares'])
         self.update_funcs: List[Callable] = []
         for component in self.component_list:
-            self.log.logger.debug(f'adding: {component} to hw component gathered list.')
+            self.log.debug(f'adding: {component} to hw component gathered list.')
             func = f'_update_{component}'
             if hasattr(self, func):
                 f = getattr(self, func)
                 self.update_funcs.append(f)
 
-        self.start_client()
-
-    def start_client(self) -> None:
+    def main(self) -> None:
+        self.stop = False
         self.client.login()
-        self.start_update_loop()
-
-    def start_update_loop(self) -> None:
-        self.run = True
-        self.thread = Thread(target=self.update)
-        self.thread.start()
-
-    def stop_update_loop(self) -> None:
-        self.run = False
-        self.thread.join()
-
-    def update(self) -> None:
-        #  this loop can have:
-        #  - caching logic
-        while self.run:
-            self.log.logger.debug('waiting for a lock in the update loop.')
-            self.lock.acquire()
-            self.log.logger.debug('lock acquired in the update loop.')
-            try:
-                self._update_system()
-                self._update_sn()
-
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                    executor.map(lambda f: f(), self.update_funcs)
-
-                self.data_ready = True
-            except RuntimeError as e:
-                self.run = False
-                self.log.logger.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
-                self.client.logout()
-            finally:
-                self.lock.release()
-                sleep(5)
-                self.log.logger.debug('lock released in the update loop.')
+        while not self.stop:
+            self.log.debug('waiting for a lock in the update loop.')
+            with self.lock:
+                if not self.pending_shutdown:
+                    self.log.debug('lock acquired in the update loop.')
+                    try:
+                        self._update_system()
+                        self._update_sn()
+
+                        with concurrent.futures.ThreadPoolExecutor() as executor:
+                            executor.map(lambda f: f(), self.update_funcs)
+
+                        self.data_ready = True
+                    except RuntimeError as e:
+                        self.stop = True
+                        self.log.error(f'Error detected, trying to gracefully log out from redfish api.\n{e}')
+                        self.client.logout()
+                        raise
+                    sleep(5)
+            self.log.debug('lock released in the update loop.')
+        self.log.debug('exiting update loop.')
+        raise SystemExit(0)
 
     def flush(self) -> None:
-        self.log.logger.debug('Acquiring lock to flush data.')
+        self.log.debug('Acquiring lock to flush data.')
         self.lock.acquire()
-        self.log.logger.debug('Lock acquired, flushing data.')
+        self.log.debug('Lock acquired, flushing data.')
         self._system = {}
         self.previous_data = {}
-        self.log.logger.info('Data flushed.')
+        self.log.info('Data flushed.')
         self.data_ready = False
-        self.log.logger.debug('Data marked as not ready.')
+        self.log.debug('Data marked as not ready.')
         self.lock.release()
-        self.log.logger.debug('Released the lock after flushing data.')
+        self.log.debug('Released the lock after flushing data.')
 
-    @retry(retries=10, delay=2)
+    # @retry(retries=10, delay=2)
     def _get_path(self, path: str) -> Dict:
+        result: Dict[str, Any] = {}
         try:
-            result = self.client.get_path(path)
+            if not self.pending_shutdown:
+                self.log.debug(f'Getting path: {path}')
+                result = self.client.get_path(path)
+            else:
+                self.log.debug(f'Pending shutdown, aborting query to {path}')
         except RuntimeError:
             raise
         if result is None:
-            self.log.logger.error(f'The client reported an error when getting path: {path}')
+            self.log.error(f'The client reported an error when getting path: {path}')
             raise RuntimeError(f'Could not get path: {path}')
         return result
 
@@ -197,7 +184,7 @@ def get_device_led(self, device: str) -> Dict[str, Any]:
                                        endpoint=endpoint,
                                        timeout=10)
         except HTTPError as e:
-            self.log.logger.error(f"Couldn't get the ident device LED status for device '{device}': {e}")
+            self.log.error(f"Couldn't get the ident device LED status for device '{device}': {e}")
             raise
         response_json = json.loads(result[1])
         _result: Dict[str, Any] = {'http_code': result[2]}
@@ -215,7 +202,7 @@ def set_device_led(self, device: str, data: Dict[str, bool]) -> int:
                 endpoint=self._sys['storage'][device]['redfish_endpoint']
             )
         except (HTTPError, KeyError) as e:
-            self.log.logger.error(f"Couldn't set the ident device LED for device '{device}': {e}")
+            self.log.error(f"Couldn't set the ident device LED for device '{device}': {e}")
             raise
         return status
 
@@ -226,7 +213,7 @@ def get_chassis_led(self) -> Dict[str, Any]:
                                        endpoint=endpoint,
                                        timeout=10)
         except HTTPError as e:
-            self.log.logger.error(f"Couldn't get the ident chassis LED status: {e}")
+            self.log.error(f"Couldn't get the ident chassis LED status: {e}")
             raise
         response_json = json.loads(result[1])
         _result: Dict[str, Any] = {'http_code': result[2]}
@@ -246,18 +233,18 @@ def set_chassis_led(self, data: Dict[str, str]) -> int:
                 endpoint=f'/redfish/v1{self.chassis_endpoint}'
             )
         except HTTPError as e:
-            self.log.logger.error(f"Couldn't set the ident chassis LED: {e}")
+            self.log.error(f"Couldn't set the ident chassis LED: {e}")
             raise
         return status
 
-    def shutdown(self, force: bool = False) -> int:
+    def shutdown_host(self, force: bool = False) -> int:
         reboot_type: str = 'GracefulRebootWithForcedShutdown' if force else 'GracefulRebootWithoutForcedShutdown'
 
         try:
             job_id: str = self.create_reboot_job(reboot_type)
             status = self.schedule_reboot_job(job_id)
         except (HTTPError, KeyError) as e:
-            self.log.logger.error(f"Couldn't create the reboot job: {e}")
+            self.log.error(f"Couldn't create the reboot job: {e}")
             raise
         return status
 
@@ -266,7 +253,7 @@ def powercycle(self) -> int:
             job_id: str = self.create_reboot_job('PowerCycle')
             status = self.schedule_reboot_job(job_id)
         except (HTTPError, URLError) as e:
-            self.log.logger.error(f"Couldn't perform power cycle: {e}")
+            self.log.error(f"Couldn't perform power cycle: {e}")
             raise
         return status
 
@@ -279,7 +266,7 @@ def create_reboot_job(self, reboot_type: str) -> str:
             )
             job_id: str = headers['Location'].split('/')[-1]
         except (HTTPError, URLError) as e:
-            self.log.logger.error(f"Couldn't create the reboot job: {e}")
+            self.log.error(f"Couldn't create the reboot job: {e}")
             raise
         return job_id
 
@@ -291,6 +278,6 @@ def schedule_reboot_job(self, job_id: str) -> int:
                 endpoint=self.setup_job_queue_endpoint
             )
         except (HTTPError, KeyError) as e:
-            self.log.logger.error(f"Couldn't schedule the reboot job: {e}")
+            self.log.error(f"Couldn't schedule the reboot job: {e}")
             raise
         return status
diff --git a/src/ceph-node-proxy/ceph_node_proxy/basesystem.py b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
index c2389d8dc85e..65eca55af1f0 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/basesystem.py
@@ -1,14 +1,21 @@
 import socket
-from ceph_node_proxy.util import Config
+from threading import Lock
+from ceph_node_proxy.util import Config, get_logger, BaseThread
 from typing import Dict, Any
 from ceph_node_proxy.baseclient import BaseClient
 
 
-class BaseSystem:
+class BaseSystem(BaseThread):
     def __init__(self, **kw: Any) -> None:
+        super().__init__()
+        self.lock: Lock = Lock()
         self._system: Dict = {}
-        self.config: Config = kw['config']
+        self.config: Config = kw.get('config', {})
         self.client: BaseClient
+        self.log = get_logger(__name__)
+
+    def main(self) -> None:
+        raise NotImplementedError()
 
     def get_system(self) -> Dict[str, Any]:
         raise NotImplementedError()
@@ -76,19 +83,13 @@ def chassis_led_off(self) -> int:
     def get_host(self) -> str:
         return socket.gethostname()
 
-    def start_update_loop(self) -> None:
-        raise NotImplementedError()
-
     def stop_update_loop(self) -> None:
         raise NotImplementedError()
 
-    def start_client(self) -> None:
-        raise NotImplementedError()
-
     def flush(self) -> None:
         raise NotImplementedError()
 
-    def shutdown(self, force: bool = False) -> int:
+    def shutdown_host(self, force: bool = False) -> int:
         raise NotImplementedError()
 
     def powercycle(self) -> int:
diff --git a/src/ceph-node-proxy/ceph_node_proxy/main.py b/src/ceph-node-proxy/ceph_node_proxy/main.py
index 689089aa609f..2a6479c4238e 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/main.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/main.py
@@ -1,176 +1,84 @@
-from threading import Thread
 from ceph_node_proxy.redfishdellsystem import RedfishDellSystem
 from ceph_node_proxy.api import NodeProxyApi
 from ceph_node_proxy.reporter import Reporter
-from ceph_node_proxy.util import Config, Logger, http_req, write_tmp_file
+from ceph_node_proxy.util import Config, get_logger, http_req, write_tmp_file, CONFIG
 from typing import Dict, Any, Optional
 
 import argparse
-import traceback
-import logging
 import os
 import ssl
 import json
 import time
+import signal
 
-logger = logging.getLogger(__name__)
-
-DEFAULT_CONFIG = {
-    'reporter': {
-        'check_interval': 5,
-        'push_data_max_retries': 30,
-        'endpoint': 'https://127.0.0.1:7150/node-proxy/data',
-    },
-    'system': {
-        'refresh_interval': 5
-    },
-    'server': {
-        'port': 8080,
-    },
-    'logging': {
-        'level': 20,
-    }
-}
-
-
-class NodeProxyManager(Thread):
-    def __init__(self,
-                 mgr_host: str,
-                 cephx_name: str,
-                 cephx_secret: str,
-                 ca_path: str,
-                 api_ssl_crt: str,
-                 api_ssl_key: str,
-                 mgr_agent_port: int = 7150):
-        super().__init__()
-        self.mgr_host = mgr_host
-        self.cephx_name = cephx_name
-        self.cephx_secret = cephx_secret
-        self.ca_path = ca_path
-        self.api_ssl_crt = api_ssl_crt
-        self.api_ssl_key = api_ssl_key
-        self.mgr_agent_port = str(mgr_agent_port)
-        self.stop = False
+
+class NodeProxyManager:
+    def __init__(self, **kw: Any) -> None:
+        self.exc: Optional[Exception] = None
+        self.log = get_logger(__name__)
+        self.mgr_host: str = kw['mgr_host']
+        self.cephx_name: str = kw['cephx_name']
+        self.cephx_secret: str = kw['cephx_secret']
+        self.ca_path: str = kw['ca_path']
+        self.api_ssl_crt: str = kw['api_ssl_crt']
+        self.api_ssl_key: str = kw['api_ssl_key']
+        self.mgr_agent_port: str = str(kw['mgr_agent_port'])
+        self.stop: bool = False
         self.ssl_ctx = ssl.create_default_context()
         self.ssl_ctx.check_hostname = True
         self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
         self.ssl_ctx.load_verify_locations(self.ca_path)
+        self.reporter_scheme: str = kw.get('reporter_scheme', 'https')
+        self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
+        self.cephx = {'cephx': {'name': self.cephx_name,
+                                'secret': self.cephx_secret}}
+        self.config = Config('/etc/ceph/node-proxy.yml', config=CONFIG)
 
     def run(self) -> None:
         self.init()
         self.loop()
 
     def init(self) -> None:
-        node_proxy_meta = {
-            'cephx': {
-                'name': self.cephx_name,
-                'secret': self.cephx_secret
-            }
-        }
+        self.init_system()
+        self.init_reporter()
+        self.init_api()
+
+    def fetch_oob_details(self) -> Dict[str, str]:
         headers, result, status = http_req(hostname=self.mgr_host,
                                            port=self.mgr_agent_port,
-                                           data=json.dumps(node_proxy_meta),
+                                           data=json.dumps(self.cephx),
                                            endpoint='/node-proxy/oob',
                                            ssl_ctx=self.ssl_ctx)
         if status != 200:
             msg = f'No out of band tool details could be loaded: {status}, {result}'
-            logger.debug(msg)
+            self.log.debug(msg)
             raise RuntimeError(msg)
 
         result_json = json.loads(result)
-        kwargs = {
+        oob_details: Dict[str, str] = {
             'host': result_json['result']['addr'],
             'username': result_json['result']['username'],
             'password': result_json['result']['password'],
-            'cephx': node_proxy_meta['cephx'],
-            'mgr_host': self.mgr_host,
-            'mgr_agent_port': self.mgr_agent_port,
-            'api_ssl_crt': self.api_ssl_crt,
-            'api_ssl_key': self.api_ssl_key
+            'port': result_json['result'].get('port', '443')
         }
-        if result_json['result'].get('port'):
-            kwargs['port'] = result_json['result']['port']
-
-        self.node_proxy: NodeProxy = NodeProxy(**kwargs)
-        self.node_proxy.start()
-
-    def loop(self) -> None:
-        while not self.stop:
-            try:
-                status = self.node_proxy.check_status()
-                label = 'Ok' if status else 'Critical'
-                logger.debug(f'node-proxy status: {label}')
-            except Exception as e:
-                logger.error(f'node-proxy not running: {e.__class__.__name__}: {e}')
-                time.sleep(120)
-                self.init()
-            else:
-                logger.debug('node-proxy alive, next check in 60sec.')
-                time.sleep(60)
-
-    def shutdown(self) -> None:
-        self.stop = True
-        # if `self.node_proxy.shutdown()` is called before self.start(), it will fail.
-        if self.__dict__.get('node_proxy'):
-            self.node_proxy.shutdown()
+        return oob_details
 
-
-class NodeProxy(Thread):
-    def __init__(self, **kw: Any) -> None:
-        super().__init__()
-        self.username: str = kw.get('username', '')
-        self.password: str = kw.get('password', '')
-        self.host: str = kw.get('host', '')
-        self.port: int = kw.get('port', 443)
-        self.cephx: Dict[str, Any] = kw.get('cephx', {})
-        self.reporter_scheme: str = kw.get('reporter_scheme', 'https')
-        self.mgr_host: str = kw.get('mgr_host', '')
-        self.mgr_agent_port: str = kw.get('mgr_agent_port', '')
-        self.reporter_endpoint: str = kw.get('reporter_endpoint', '/node-proxy/data')
-        self.api_ssl_crt: str = kw.get('api_ssl_crt', '')
-        self.api_ssl_key: str = kw.get('api_ssl_key', '')
-        self.exc: Optional[Exception] = None
-        self.log = Logger(__name__)
-
-    def run(self) -> None:
+    def init_system(self) -> None:
+        oob_details = self.fetch_oob_details()
+        self.username: str = oob_details['username']
+        self.password: str = oob_details['password']
         try:
-            self.main()
-        except Exception as e:
-            self.exc = e
-            return
-
-    def shutdown(self) -> None:
-        self.log.logger.info('Shutting down node-proxy...')
-        self.system.client.logout()
-        self.system.stop_update_loop()
-        self.reporter_agent.stop()
-
-    def check_status(self) -> bool:
-        if self.__dict__.get('system') and not self.system.run:
-            raise RuntimeError('node-proxy encountered an error.')
-        if self.exc:
-            traceback.print_tb(self.exc.__traceback__)
-            self.log.logger.error(f'{self.exc.__class__.__name__}: {self.exc}')
-            raise self.exc
-        return True
-
-    def main(self) -> None:
-        # TODO: add a check and fail if host/username/password/data aren't passed
-        self.config = Config('/etc/ceph/node-proxy.yml', default_config=DEFAULT_CONFIG)
-        self.log = Logger(__name__, level=self.config.__dict__['logging']['level'])
-
-        # create the redfish system and the obsever
-        self.log.logger.info('Server initialization...')
-        try:
-            self.system = RedfishDellSystem(host=self.host,
-                                            port=self.port,
-                                            username=self.username,
-                                            password=self.password,
+            self.system = RedfishDellSystem(host=oob_details['host'],
+                                            port=oob_details['port'],
+                                            username=oob_details['username'],
+                                            password=oob_details['password'],
                                             config=self.config)
+            self.system.start()
         except RuntimeError:
-            self.log.logger.error("Can't initialize the redfish system.")
+            self.log.error("Can't initialize the redfish system.")
             raise
 
+    def init_reporter(self) -> None:
         try:
             self.reporter_agent = Reporter(self.system,
                                            self.cephx,
@@ -178,23 +86,54 @@ def main(self) -> None:
                                            reporter_hostname=self.mgr_host,
                                            reporter_port=self.mgr_agent_port,
                                            reporter_endpoint=self.reporter_endpoint)
-            self.reporter_agent.run()
+            self.reporter_agent.start()
         except RuntimeError:
-            self.log.logger.error("Can't initialize the reporter.")
+            self.log.error("Can't initialize the reporter.")
             raise
 
+    def init_api(self) -> None:
         try:
-            self.log.logger.info('Starting node-proxy API...')
-            self.api = NodeProxyApi(self,
-                                    username=self.username,
-                                    password=self.password,
-                                    ssl_crt=self.api_ssl_crt,
-                                    ssl_key=self.api_ssl_key)
+            self.log.info('Starting node-proxy API...')
+            self.api = NodeProxyApi(self)
             self.api.start()
         except Exception as e:
-            self.log.logger.error(f"Can't start node-proxy API: {e}")
+            self.log.error(f"Can't start node-proxy API: {e}")
             raise
 
+    def loop(self) -> None:
+        while not self.stop:
+            for thread in [self.system, self.reporter_agent]:
+                try:
+                    status = thread.check_status()
+                    label = 'Ok' if status else 'Critical'
+                    self.log.debug(f'{thread} status: {label}')
+                except Exception as e:
+                    self.log.error(f'{thread} not running: {e.__class__.__name__}: {e}')
+                    thread.shutdown()
+                    self.init_system()
+                    self.init_reporter()
+            self.log.debug('All threads are alive, next check in 20sec.')
+            time.sleep(20)
+
+    def shutdown(self) -> None:
+        self.stop = True
+        # if `self.system.shutdown()` is called before self.start(), it will fail.
+        if hasattr(self, 'api'):
+            self.api.shutdown()
+        if hasattr(self, 'reporter_agent'):
+            self.reporter_agent.shutdown()
+        if hasattr(self, 'system'):
+            self.system.shutdown()
+
+
+def handler(signum: Any, frame: Any, t_mgr: 'NodeProxyManager') -> None:
+    t_mgr.system.pending_shutdown = True
+    t_mgr.log.info('SIGTERM caught, shutting down threads...')
+    t_mgr.shutdown()
+    t_mgr.log.info('Logging out from RedFish API')
+    t_mgr.system.client.logout()
+    raise SystemExit(0)
+
 
 def main() -> None:
     parser = argparse.ArgumentParser(
@@ -205,8 +144,15 @@ def main() -> None:
         help='path of config file in json format',
         required=True
     )
+    parser.add_argument(
+        '--debug',
+        help='increase logging verbosity (debug level)',
+        action='store_true',
+    )
 
     args = parser.parse_args()
+    if args.debug:
+        CONFIG['logging']['level'] = 10
 
     if not os.path.exists(args.config):
         raise Exception(f'No config file found at provided config path: {args.config}')
@@ -226,18 +172,19 @@ def main() -> None:
     listener_key = config['listener.key']
     name = config['name']
 
-    f = write_tmp_file(root_cert,
-                       prefix_name='cephadm-endpoint-root-cert')
+    ca_file = write_tmp_file(root_cert,
+                             prefix_name='cephadm-endpoint-root-cert')
 
     node_proxy_mgr = NodeProxyManager(mgr_host=target_ip,
                                       cephx_name=name,
                                       cephx_secret=keyring,
                                       mgr_agent_port=target_port,
-                                      ca_path=f.name,
+                                      ca_path=ca_file.name,
                                       api_ssl_crt=listener_cert,
                                       api_ssl_key=listener_key)
-    if not node_proxy_mgr.is_alive():
-        node_proxy_mgr.start()
+    signal.signal(signal.SIGTERM,
+                  lambda signum, frame: handler(signum, frame, node_proxy_mgr))
+    node_proxy_mgr.run()
 
 
 if __name__ == '__main__':
diff --git a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
index eeca2e5ba217..08ee4170dcc6 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
@@ -1,7 +1,7 @@
 import json
 from urllib.error import HTTPError, URLError
 from ceph_node_proxy.baseclient import BaseClient
-from ceph_node_proxy.util import Logger, http_req
+from ceph_node_proxy.util import get_logger, http_req
 from typing import Dict, Any, Tuple, Optional
 from http.client import HTTPMessage
 
@@ -15,8 +15,8 @@ def __init__(self,
                  username: str = '',
                  password: str = ''):
         super().__init__(host, username, password)
-        self.log: Logger = Logger(__name__)
-        self.log.logger.info(f'Initializing redfish client {__name__}')
+        self.log = get_logger(__name__)
+        self.log.info(f'Initializing redfish client {__name__}')
         self.host: str = host
         self.port: str = port
         self.url: str = f'https://{self.host}:{self.port}'
@@ -25,8 +25,8 @@ def __init__(self,
 
     def login(self) -> None:
         if not self.is_logged_in():
-            self.log.logger.info('Logging in to '
-                                 f"{self.url} as '{self.username}'")
+            self.log.info('Logging in to '
+                          f"{self.url} as '{self.username}'")
             oob_credentials = json.dumps({'UserName': self.username,
                                           'Password': self.password})
             headers = {'Content-Type': 'application/json'}
@@ -36,27 +36,27 @@ def login(self) -> None:
                                                            headers=headers,
                                                            endpoint='/redfish/v1/SessionService/Sessions/')
                 if _status_code != 201:
-                    self.log.logger.error(f"Can't log in to {self.url} as '{self.username}': {_status_code}")
+                    self.log.error(f"Can't log in to {self.url} as '{self.username}': {_status_code}")
                     raise RuntimeError
             except URLError as e:
                 msg = f"Can't log in to {self.url} as '{self.username}': {e}"
-                self.log.logger.error(msg)
+                self.log.error(msg)
                 raise RuntimeError
             self.token = _headers['X-Auth-Token']
             self.location = _headers['Location']
 
     def is_logged_in(self) -> bool:
-        self.log.logger.debug(f'Checking token validity for {self.url}')
+        self.log.debug(f'Checking token validity for {self.url}')
         if not self.location or not self.token:
-            self.log.logger.debug(f'No token found for {self.url}.')
+            self.log.debug(f'No token found for {self.url}.')
             return False
         headers = {'X-Auth-Token': self.token}
         try:
             _headers, _data, _status_code = self.query(headers=headers,
                                                        endpoint=self.location)
         except URLError as e:
-            self.log.logger.error("Can't check token "
-                                  f'validity for {self.url}: {e}')
+            self.log.error("Can't check token "
+                           f'validity for {self.url}: {e}')
             raise
         return _status_code == 200
 
@@ -69,7 +69,7 @@ def logout(self) -> Dict[str, Any]:
                                                     endpoint=self.location)
                 result = json.loads(_data)
         except URLError:
-            self.log.logger.error(f"Can't log out from {self.url}")
+            self.log.error(f"Can't log out from {self.url}")
 
         self.location = ''
         self.token = ''
@@ -84,7 +84,7 @@ def get_path(self, path: str) -> Dict[str, Any]:
             result_json = json.loads(result)
             return result_json
         except URLError as e:
-            self.log.logger.error(f"Can't get path {path}:\n{e}")
+            self.log.error(f"Can't get path {path}:\n{e}")
             raise RuntimeError
 
     def query(self,
@@ -111,5 +111,5 @@ def query(self,
 
             return response_headers, response_str, response_status
         except (HTTPError, URLError) as e:
-            self.log.logger.debug(f'{e}')
+            self.log.debug(f'{e}')
             raise
diff --git a/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
index 83b73657b1aa..f0d24c667c96 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfishdellsystem.py
@@ -1,12 +1,12 @@
 from ceph_node_proxy.baseredfishsystem import BaseRedfishSystem
-from ceph_node_proxy.util import Logger, normalize_dict, to_snake_case
+from ceph_node_proxy.util import get_logger, normalize_dict, to_snake_case
 from typing import Dict, Any, List
 
 
 class RedfishDellSystem(BaseRedfishSystem):
     def __init__(self, **kw: Any) -> None:
         super().__init__(**kw)
-        self.log = Logger(__name__)
+        self.log = get_logger(__name__)
         self.job_service_endpoint: str = '/redfish/v1/Managers/iDRAC.Embedded.1/Oem/Dell/DellJobService'
         self.create_reboot_job_endpoint: str = f'{self.job_service_endpoint}/Actions/DellJobService.CreateRebootJob'
         self.setup_job_queue_endpoint: str = f'{self.job_service_endpoint}/Actions/DellJobService.SetupJobQueue'
@@ -23,7 +23,7 @@ def build_common_data(self,
                 try:
                     result[member_id][to_snake_case(field)] = member_info[field]
                 except KeyError:
-                    self.log.logger.warning(f'Could not find field: {field} in member_info: {member_info}')
+                    self.log.warning(f'Could not find field: {field} in member_info: {member_info}')
 
         return normalize_dict(result)
 
@@ -41,7 +41,7 @@ def build_chassis_data(self,
                     try:
                         result[_id][to_snake_case(field)] = member_elt[field]
                     except KeyError:
-                        self.log.logger.warning(f'Could not find field: {field} in data: {data[elt]}')
+                        self.log.warning(f'Could not find field: {field} in data: {data[elt]}')
         return normalize_dict(result)
 
     def get_sn(self) -> str:
@@ -73,7 +73,7 @@ def get_fans(self) -> Dict[str, Dict[str, Dict]]:
 
     def _update_network(self) -> None:
         fields = ['Description', 'Name', 'SpeedMbps', 'Status']
-        self.log.logger.debug('Updating network')
+        self.log.debug('Updating network')
         self._sys['network'] = self.build_common_data(data=self._system['Systems'],
                                                       fields=fields,
                                                       path='EthernetInterfaces')
@@ -86,7 +86,7 @@ def _update_processors(self) -> None:
                   'Model',
                   'Status',
                   'Manufacturer']
-        self.log.logger.debug('Updating processors')
+        self.log.debug('Updating processors')
         self._sys['processors'] = self.build_common_data(data=self._system['Systems'],
                                                          fields=fields,
                                                          path='Processors')
@@ -100,7 +100,7 @@ def _update_storage(self) -> None:
                   'PhysicalLocation']
         entities = self.get_members(data=self._system['Systems'],
                                     path='Storage')
-        self.log.logger.debug('Updating storage')
+        self.log.debug('Updating storage')
         result: Dict[str, Dict[str, Dict]] = dict()
         for entity in entities:
             for drive in entity['Drives']:
@@ -115,7 +115,7 @@ def _update_storage(self) -> None:
         self._sys['storage'] = normalize_dict(result)
 
     def _update_sn(self) -> None:
-        self.log.logger.debug('Updating serial number')
+        self.log.debug('Updating serial number')
         self._sys['SKU'] = self._system['Systems']['SKU']
 
     def _update_memory(self) -> None:
@@ -123,7 +123,7 @@ def _update_memory(self) -> None:
                   'MemoryDeviceType',
                   'CapacityMiB',
                   'Status']
-        self.log.logger.debug('Updating memory')
+        self.log.debug('Updating memory')
         self._sys['memory'] = self.build_common_data(data=self._system['Systems'],
                                                      fields=fields,
                                                      path='Memory')
@@ -137,7 +137,7 @@ def _update_power(self) -> None:
                 'Status'
             ]
         }
-        self.log.logger.debug('Updating powersupplies')
+        self.log.debug('Updating powersupplies')
         self._sys['power'] = self.build_chassis_data(fields, 'Power')
 
     def _update_fans(self) -> None:
@@ -148,7 +148,7 @@ def _update_fans(self) -> None:
                 'Status'
             ],
         }
-        self.log.logger.debug('Updating fans')
+        self.log.debug('Updating fans')
         self._sys['fans'] = self.build_chassis_data(fields, 'Thermal')
 
     def _update_firmwares(self) -> None:
@@ -160,7 +160,7 @@ def _update_firmwares(self) -> None:
             'Updateable',
             'Status',
         ]
-        self.log.logger.debug('Updating firmwares')
+        self.log.debug('Updating firmwares')
         self._sys['firmwares'] = self.build_common_data(data=self._system['UpdateService'],
                                                         fields=fields,
                                                         path='FirmwareInventory')
diff --git a/src/ceph-node-proxy/ceph_node_proxy/reporter.py b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
index aa16d83421f3..4e9c4e2798b3 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/reporter.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
@@ -1,12 +1,11 @@
-from threading import Thread
 import time
 import json
-from ceph_node_proxy.util import Logger, http_req
+from ceph_node_proxy.util import get_logger, http_req, BaseThread
 from urllib.error import HTTPError, URLError
 from typing import Dict, Any
 
 
-class Reporter:
+class Reporter(BaseThread):
     def __init__(self,
                  system: Any,
                  cephx: Dict[str, Any],
@@ -14,61 +13,57 @@ def __init__(self,
                  reporter_hostname: str = '',
                  reporter_port: str = '443',
                  reporter_endpoint: str = '/node-proxy/data') -> None:
+        super().__init__()
         self.system = system
         self.data: Dict[str, Any] = {}
-        self.finish = False
+        self.stop: bool = False
         self.cephx = cephx
-        self.data['cephx'] = self.cephx
+        self.data['cephx'] = self.cephx['cephx']
         self.reporter_scheme: str = reporter_scheme
         self.reporter_hostname: str = reporter_hostname
         self.reporter_port: str = reporter_port
         self.reporter_endpoint: str = reporter_endpoint
-        self.log = Logger(__name__)
+        self.log = get_logger(__name__)
         self.reporter_url: str = (f'{reporter_scheme}://{reporter_hostname}:'
                                   f'{reporter_port}{reporter_endpoint}')
-        self.log.logger.info(f'Reporter url set to {self.reporter_url}')
+        self.log.info(f'Reporter url set to {self.reporter_url}')
 
-    def stop(self) -> None:
-        self.finish = True
-        self.thread.join()
-
-    def run(self) -> None:
-        self.thread = Thread(target=self.loop)
-        self.thread.start()
-
-    def loop(self) -> None:
-        while not self.finish:
+    def main(self) -> None:
+        while not self.stop:
             # Any logic to avoid sending the all the system
             # information every loop can go here. In a real
             # scenario probably we should just send the sub-parts
             # that have changed to minimize the traffic in
             # dense clusters
-            self.log.logger.debug('waiting for a lock in reporter loop.')
-            self.system.lock.acquire()
-            self.log.logger.debug('lock acquired in reporter loop.')
-            if self.system.data_ready:
-                self.log.logger.debug('data ready to be sent to the mgr.')
-                if not self.system.get_system() == self.system.previous_data:
-                    self.log.logger.info('data has changed since last iteration.')
-                    self.data['patch'] = self.system.get_system()
-                    try:
-                        # TODO: add a timeout parameter to the reporter in the config file
-                        self.log.logger.info(f'sending data to {self.reporter_url}')
-                        http_req(hostname=self.reporter_hostname,
-                                 port=self.reporter_port,
-                                 method='POST',
-                                 headers={'Content-Type': 'application/json'},
-                                 endpoint=self.reporter_endpoint,
-                                 scheme=self.reporter_scheme,
-                                 data=json.dumps(self.data))
-                    except (HTTPError, URLError) as e:
-                        self.log.logger.error(f"The reporter couldn't send data to the mgr: {e}")
-                        # Need to add a new parameter 'max_retries' to the reporter if it can't
-                        # send the data for more than x times, maybe the daemon should stop altogether
-                    else:
-                        self.system.previous_data = self.system.get_system()
-                else:
-                    self.log.logger.debug('no diff, not sending data to the mgr.')
-            self.system.lock.release()
-            self.log.logger.debug('lock released in reporter loop.')
-            time.sleep(5)
+            self.log.debug('waiting for a lock in reporter loop.')
+            with self.system.lock:
+                if not self.system.pending_shutdown:
+                    self.log.debug('lock acquired in reporter loop.')
+                    if self.system.data_ready:
+                        self.log.debug('data ready to be sent to the mgr.')
+                        if not self.system.get_system() == self.system.previous_data:
+                            self.log.info('data has changed since last iteration.')
+                            self.data['patch'] = self.system.get_system()
+                            try:
+                                # TODO: add a timeout parameter to the reporter in the config file
+                                self.log.info(f'sending data to {self.reporter_url}')
+                                http_req(hostname=self.reporter_hostname,
+                                         port=self.reporter_port,
+                                         method='POST',
+                                         headers={'Content-Type': 'application/json'},
+                                         endpoint=self.reporter_endpoint,
+                                         scheme=self.reporter_scheme,
+                                         data=json.dumps(self.data))
+                            except (HTTPError, URLError) as e:
+                                self.log.error(f"The reporter couldn't send data to the mgr: {e}")
+                                raise
+                                # Need to add a new parameter 'max_retries' to the reporter if it can't
+                                # send the data for more than x times, maybe the daemon should stop altogether
+                            else:
+                                self.system.previous_data = self.system.get_system()
+                        else:
+                            self.log.debug('no diff, not sending data to the mgr.')
+                    time.sleep(5)
+            self.log.debug('lock released in reporter loop.')
+        self.log.debug('exiting reporter loop.')
+        raise SystemExit(0)
diff --git a/src/ceph-node-proxy/ceph_node_proxy/util.py b/src/ceph-node-proxy/ceph_node_proxy/util.py
index a94acc9ed556..f6ed0fb483d6 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/util.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/util.py
@@ -4,43 +4,57 @@
 import time
 import re
 import ssl
+import traceback
+import threading
 from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 from urllib.error import HTTPError, URLError
 from urllib.request import urlopen, Request
-from typing import Dict, List, Callable, Any, Optional, MutableMapping, Tuple
-
-
-class Logger:
-    _Logger: List['Logger'] = []
-
-    def __init__(self, name: str, level: int = logging.INFO):
-        self.name = name
-        self.level = level
-
-        Logger._Logger.append(self)
-        self.logger = self.get_logger()
-
-    def get_logger(self) -> logging.Logger:
-        logger = logging.getLogger(self.name)
-        logger.setLevel(self.level)
-        handler = logging.StreamHandler()
-        handler.setLevel(self.level)
-        fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        handler.setFormatter(fmt)
-        logger.handlers.clear()
-        logger.addHandler(handler)
-        logger.propagate = False
-
-        return logger
+from typing import Dict, Callable, Any, Optional, MutableMapping, Tuple, Union
+
+
+CONFIG: Dict[str, Any] = {
+    'reporter': {
+        'check_interval': 5,
+        'push_data_max_retries': 30,
+        'endpoint': 'https://%(mgr_host):%(mgr_port)/node-proxy/data',
+    },
+    'system': {
+        'refresh_interval': 5
+    },
+    'api': {
+        'port': 9456,
+    },
+    'logging': {
+        'level': logging.INFO,
+    }
+}
+
+
+def get_logger(name: str, level: Union[int, str] = logging.NOTSET) -> logging.Logger:
+    if level == logging.NOTSET:
+        log_level = CONFIG['logging']['level']
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level)
+    handler = logging.StreamHandler()
+    handler.setLevel(log_level)
+    fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler.setFormatter(fmt)
+    logger.handlers.clear()
+    logger.addHandler(handler)
+    logger.propagate = False
+
+    return logger
+
+
+logger = get_logger(__name__)
 
 
 class Config:
-
     def __init__(self,
                  config_file: str = '/etc/ceph/node-proxy.yaml',
-                 default_config: Dict[str, Any] = {}) -> None:
+                 config: Dict[str, Any] = {}) -> None:
         self.config_file = config_file
-        self.default_config = default_config
+        self.config = config
 
         self.load_config()
 
@@ -49,27 +63,56 @@ def load_config(self) -> None:
             with open(self.config_file, 'r') as f:
                 self.config = yaml.safe_load(f)
         else:
-            self.config = self.default_config
+            self.config = self.config
 
-        for k, v in self.default_config.items():
+        for k, v in self.config.items():
             if k not in self.config.keys():
                 self.config[k] = v
 
         for k, v in self.config.items():
             setattr(self, k, v)
 
-        # TODO: need to be improved
-        for _l in Logger._Logger:
-            _l.logger.setLevel(self.logging['level'])  # type: ignore
-            _l.logger.handlers[0].setLevel(self.logging['level'])  # type: ignore
-
     def reload(self, config_file: str = '') -> None:
         if config_file != '':
             self.config_file = config_file
         self.load_config()
 
 
-log = Logger(__name__)
+class BaseThread(threading.Thread):
+    def __init__(self) -> None:
+        super().__init__()
+        self.exc: Optional[Exception] = None
+        self.stop: bool = False
+        self.daemon = True
+        self.name = self.__class__.__name__
+        self.log: logging.Logger = get_logger(__name__)
+        self.pending_shutdown: bool = False
+
+    def run(self) -> None:
+        logger.info(f'Starting {self.name}')
+        try:
+            self.main()
+        except Exception as e:
+            self.exc = e
+            return
+
+    def shutdown(self) -> None:
+        self.stop = True
+        self.pending_shutdown = True
+
+    def check_status(self) -> bool:
+        logger.debug(f'Checking status of {self.name}')
+        if self.exc:
+            traceback.print_tb(self.exc.__traceback__)
+            logger.error(f'Caught exception: {self.exc.__class__.__name__}')
+            raise self.exc
+        if not self.is_alive():
+            logger.info(f'{self.name} not alive')
+            self.start()
+        return True
+
+    def main(self) -> None:
+        raise NotImplementedError()
 
 
 def to_snake_case(name: str) -> str:
@@ -93,12 +136,12 @@ def _retry(*args: str, **kwargs: Any) -> Callable:
             _tries = retries
             while _tries > 1:
                 try:
-                    log.logger.debug('{} {} attempt(s) left.'.format(f, _tries - 1))
+                    logger.debug('{} {} attempt(s) left.'.format(f, _tries - 1))
                     return f(*args, **kwargs)
                 except exceptions:
                     time.sleep(delay)
                     _tries -= 1
-            log.logger.warn('{} has failed after {} tries'.format(f, retries))
+            logger.warn('{} has failed after {} tries'.format(f, retries))
             return f(*args, **kwargs)
         return _retry
     return decorator
diff --git a/src/cephadm/cephadmlib/daemons/node_proxy.py b/src/cephadm/cephadmlib/daemons/node_proxy.py
index a4cce11a53ca..c197ded3ff6c 100644
--- a/src/cephadm/cephadmlib/daemons/node_proxy.py
+++ b/src/cephadm/cephadmlib/daemons/node_proxy.py
@@ -24,7 +24,7 @@ class NodeProxy(ContainerDaemonForm):
 
     daemon_type = 'node-proxy'
     # TODO: update this if we make node-proxy an executable
-    entrypoint = 'python3'
+    entrypoint = '/usr/sbin/ceph-node-proxy'
     required_files = ['node-proxy.json']
 
     @classmethod
@@ -88,7 +88,7 @@ def customize_process_args(self, ctx: CephadmContext, args: List[str]) -> None:
         # the config in _get_container_mounts above. They
         # will both need to be updated when we have a proper
         # location in the container for node-proxy
-        args.extend(['/usr/share/ceph/ceph_node_proxy/main.py', '--config', '/usr/share/ceph/node-proxy.json'])
+        args.extend(['--config', '/usr/share/ceph/node-proxy.json'])
 
     def validate(self):
         # type: () -> None
@@ -130,7 +130,7 @@ def create_daemon_dirs(self, data_dir, uid, gid):
     def container(self, ctx: CephadmContext) -> CephContainer:
         # So the container can modprobe iscsi_target_mod and have write perms
         # to configfs we need to make this a privileged container.
-        ctr = daemon_to_container(ctx, self, privileged=True, envs=['PYTHONPATH=$PYTHONPATH:/usr/share/ceph'])
+        ctr = daemon_to_container(ctx, self, privileged=True)
         return to_deployment_container(ctx, ctr)
 
     def config_and_keyring(
diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py
index e09b98d0e4d7..12c03901de8d 100644
--- a/src/pybind/mgr/cephadm/agent.py
+++ b/src/pybind/mgr/cephadm/agent.py
@@ -361,7 +361,7 @@ def led(self, **kw: Any) -> Dict[str, Any]:
 
         try:
             headers, result, status = http_req(hostname=addr,
-                                               port='8080',
+                                               port='9456',
                                                headers=header,
                                                method=method,
                                                data=json.dumps(payload),
diff --git a/src/pybind/mgr/cephadm/services/node_proxy.py b/src/pybind/mgr/cephadm/services/node_proxy.py
index ebbbaf212c74..e5608ca42b51 100644
--- a/src/pybind/mgr/cephadm/services/node_proxy.py
+++ b/src/pybind/mgr/cephadm/services/node_proxy.py
@@ -105,7 +105,7 @@ def led(self, light_type: str, action: str, hostname: str, device: Optional[str]
 
         try:
             headers, result, status = http_req(hostname=addr,
-                                               port='8080',
+                                               port='9456',
                                                headers=header,
                                                method=method,
                                                data=json.dumps(payload),
@@ -142,7 +142,7 @@ def shutdown(self, hostname: str, force: Optional[bool] = False) -> Dict[str, An
 
         try:
             headers, result, status = http_req(hostname=addr,
-                                               port='8080',
+                                               port='9456',
                                                headers=header,
                                                data=json.dumps(payload),
                                                endpoint=endpoint,
@@ -165,7 +165,7 @@ def powercycle(self, hostname: str) -> Dict[str, Any]:
 
         try:
             headers, result, status = http_req(hostname=addr,
-                                               port='8080',
+                                               port='9456',
                                                headers=header,
                                                data="{}",
                                                endpoint=endpoint,

From 1e9b8f0e76a423225854941582a0e06143791c69 Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 5 Feb 2024 15:16:17 +0530
Subject: [PATCH 1777/2492] rbd-mirror: fixed a typo in instance_watcher

Fixes a typo in NotifyAckPayload::dump().

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 src/tools/rbd_mirror/instance_watcher/Types.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/rbd_mirror/instance_watcher/Types.cc b/src/tools/rbd_mirror/instance_watcher/Types.cc
index 0e9922733724..225cd7407336 100644
--- a/src/tools/rbd_mirror/instance_watcher/Types.cc
+++ b/src/tools/rbd_mirror/instance_watcher/Types.cc
@@ -237,7 +237,7 @@ void NotifyAckPayload::decode(bufferlist::const_iterator &iter) {
 void NotifyAckPayload::dump(Formatter *f) const {
   f->dump_string("instance_id", instance_id);
   f->dump_unsigned("request_id", request_id);
-  f->dump_int("request_id", ret_val);
+  f->dump_int("ret_val", ret_val);
 }
 
 } // namespace instance_watcher

From 6ee4416d80d17f57460ebe044bd9cc8d43dc26e8 Mon Sep 17 00:00:00 2001
From: Pritha Srivastava <prsrivas@redhat.com>
Date: Tue, 30 Jan 2024 16:48:42 +0530
Subject: [PATCH 1778/2492] rgw/auth: correcting the error message to indicate
 clearly that a pre-signed URL has expired.

Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
---
 src/rgw/rgw_auth.cc    | 5 +++++
 src/rgw/rgw_auth_s3.cc | 6 +++---
 src/rgw/rgw_common.h   | 1 +
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_auth.cc b/src/rgw/rgw_auth.cc
index 7be6518514e8..4b3f33e9c70e 100644
--- a/src/rgw/rgw_auth.cc
+++ b/src/rgw/rgw_auth.cc
@@ -299,6 +299,11 @@ rgw::auth::Strategy::apply(const DoutPrefixProvider *dpp, const rgw::auth::Strat
        * nullptr inside. */
       ldpp_dout(dpp, 5) << "Failed the auth strategy, reason="
                        << result.get_reason() << dendl;
+      //Special handling for expired pre-signed URL
+      if (result.get_reason() == ERR_PRESIGNED_URL_EXPIRED) {
+        result = result_t::deny(-EPERM);
+        set_req_state_err(s, -EPERM, "The pre-signed URL has expired");
+      }
       return result.get_reason();
     }
 
diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 519060f93fb2..6f0141ef9415 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -298,15 +298,15 @@ static inline int parse_v4_query_string(const req_info& info,              /* in
      you can set is 1, and the maximum is 604800 (seven days) */
   time_t exp = atoll(expires.data());
   if ((exp < 1) || (exp > 7*24*60*60)) {
-    dout(10) << "NOTICE: exp out of range, exp = " << exp << dendl;
+    dout(10) << "ERROR: exp out of range, exp = " << exp << dendl;
     return -EPERM;
   }
   /* handle expiration in epoch time */
   uint64_t req_sec = (uint64_t)internal_timegm(&date_t);
   uint64_t now = ceph_clock_now();
   if (now >= req_sec + exp) {
-    dout(10) << "NOTICE: now = " << now << ", req_sec = " << req_sec << ", exp = " << exp << dendl;
-    return -EPERM;
+    dout(10) << "ERROR: presigned URL has expired, now = " << now << ", req_sec = " << req_sec << ", exp = " << exp << dendl;
+    return -ERR_PRESIGNED_URL_EXPIRED;
   }
 
   signedheaders = info.args.get("x-amz-signedheaders");
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index ef9aa5420558..3bf6c4dfac8e 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -306,6 +306,7 @@ static inline const char* to_mime_type(const RGWFormat f)
 #define ERR_OBJECT_NOT_APPENDABLE                        2220
 #define ERR_INVALID_BUCKET_STATE                         2221
 #define ERR_INVALID_OBJECT_STATE			 2222
+#define ERR_PRESIGNED_URL_EXPIRED			 2223
 
 #define ERR_BUSY_RESHARDING      2300
 #define ERR_NO_SUCH_ENTITY       2301

From 9a03029b10bf99a053063bd1631503c2a6f6fab3 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 5 Feb 2024 05:49:26 -0600
Subject: [PATCH 1779/2492] osd/scrub: improve scheduling decisions logs

And, to that end, relocated determine_scrub_time(), a
helper function that examines pg 'info' vs the planned
scrub details to determine the initial scrub time,
to the scrubber itself (where it belongs, and where
the logs are more expressive).

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/osd_scrub.cc       |  8 ------
 src/osd/scrubber/osd_scrub.h        |  6 ----
 src/osd/scrubber/osd_scrub_sched.cc | 36 ------------------------
 src/osd/scrubber/osd_scrub_sched.h  |  4 ---
 src/osd/scrubber/pg_scrubber.cc     | 43 ++++++++++++++++++++++++++---
 src/osd/scrubber/pg_scrubber.h      |  9 ++++++
 6 files changed, 48 insertions(+), 58 deletions(-)

diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc
index a74e1ae5c30d..d2d2db3ff725 100644
--- a/src/osd/scrubber/osd_scrub.cc
+++ b/src/osd/scrubber/osd_scrub.cc
@@ -427,14 +427,6 @@ PerfCounters* OsdScrub::get_perf_counters(int pool_type, scrub_level_t level)
 // ////////////////////////////////////////////////////////////////////////// //
 // forwarders to the queue
 
-Scrub::sched_params_t OsdScrub::determine_scrub_time(
-    const requested_scrub_t& request_flags,
-    const pg_info_t& pg_info,
-    const pool_opts_t& pool_conf) const
-{
-  return m_queue.determine_scrub_time(request_flags, pg_info, pool_conf);
-}
-
 void OsdScrub::update_job(
     Scrub::ScrubJobRef sjob,
     const Scrub::sched_params_t& suggested,
diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h
index 64709cc7aab8..cd1158d47236 100644
--- a/src/osd/scrubber/osd_scrub.h
+++ b/src/osd/scrubber/osd_scrub.h
@@ -76,12 +76,6 @@ class OsdScrub {
   void mark_pg_scrub_blocked(spg_t blocked_pg);
   void clear_pg_scrub_blocked(spg_t blocked_pg);
 
-  // updating scheduling information for a specific PG
-  Scrub::sched_params_t determine_scrub_time(
-      const requested_scrub_t& request_flags,
-      const pg_info_t& pg_info,
-      const pool_opts_t& pool_conf) const;
-
   /**
    * modify a scrub-job's scheduled time and deadline
    *
diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc
index 36644211ec39..1d0bf614c9b9 100644
--- a/src/osd/scrubber/osd_scrub_sched.cc
+++ b/src/osd/scrubber/osd_scrub_sched.cc
@@ -168,42 +168,6 @@ void ScrubQueue::delay_on_failure(
 }
 
 
-sched_params_t ScrubQueue::determine_scrub_time(
-  const requested_scrub_t& request_flags,
-  const pg_info_t& pg_info,
-  const pool_opts_t& pool_conf) const
-{
-  sched_params_t res;
-
-  if (request_flags.must_scrub || request_flags.need_auto) {
-
-    // Set the smallest time that isn't utime_t()
-    res.proposed_time = PgScrubber::scrub_must_stamp();
-    res.is_must = Scrub::must_scrub_t::mandatory;
-    // we do not need the interval data in this case
-
-  } else if (pg_info.stats.stats_invalid && conf()->osd_scrub_invalid_stats) {
-    res.proposed_time = time_now();
-    res.is_must = Scrub::must_scrub_t::mandatory;
-
-  } else {
-    res.proposed_time = pg_info.history.last_scrub_stamp;
-    res.min_interval = pool_conf.value_or(pool_opts_t::SCRUB_MIN_INTERVAL, 0.0);
-    res.max_interval = pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0);
-  }
-
-  dout(15) << fmt::format(
-		"suggested: {:s} hist: {:s} v:{}/{} must:{} pool-min:{} {}",
-		res.proposed_time, pg_info.history.last_scrub_stamp,
-		(bool)pg_info.stats.stats_invalid,
-		conf()->osd_scrub_invalid_stats,
-		(res.is_must == must_scrub_t::mandatory ? "y" : "n"),
-		res.min_interval, request_flags)
-	   << dendl;
-  return res;
-}
-
-
 std::vector<ScrubTargetId> ScrubQueue::ready_to_scrub(
     OSDRestrictions restrictions,  // note: 4B in size! (copy)
     utime_t scrub_tick)
diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h
index 95f1680d403d..140c1428889c 100644
--- a/src/osd/scrubber/osd_scrub_sched.h
+++ b/src/osd/scrubber/osd_scrub_sched.h
@@ -233,10 +233,6 @@ class ScrubQueue {
       Scrub::delay_cause_t delay_cause,
       utime_t now_is);
 
-  sched_params_t determine_scrub_time(const requested_scrub_t& request_flags,
-				      const pg_info_t& pg_info,
-				      const pool_opts_t& pool_conf) const;
-
   std::ostream& gen_prefix(std::ostream& out, std::string_view fn) const;
 
  public:
diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 9266a54d7858..1c87c3b88a25 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -501,6 +501,43 @@ void PgScrubber::rm_from_osd_scrubbing()
   }
 }
 
+sched_params_t PgScrubber::determine_scrub_time(
+    const pool_opts_t& pool_conf) const
+{
+  sched_params_t res;
+
+  if (m_planned_scrub.must_scrub || m_planned_scrub.need_auto) {
+
+    // Set the smallest time that isn't utime_t()
+    res.proposed_time = PgScrubber::scrub_must_stamp();
+    res.is_must = Scrub::must_scrub_t::mandatory;
+    // we do not need the interval data in this case
+
+  } else if (
+      m_pg->info.stats.stats_invalid &&
+      get_pg_cct()->_conf->osd_scrub_invalid_stats) {
+    res.proposed_time = ceph_clock_now();
+    res.is_must = Scrub::must_scrub_t::mandatory;
+
+  } else {
+    res.proposed_time = m_pg->info.history.last_scrub_stamp;
+    res.min_interval = pool_conf.value_or(pool_opts_t::SCRUB_MIN_INTERVAL, 0.0);
+    res.max_interval = pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0);
+  }
+
+  dout(15)
+      << fmt::format(
+	     "{}: suggested: {:s} hist: {:s} v:{}/{} must:{} pool-min:{} {}",
+	     __func__, res.proposed_time, m_pg->info.history.last_scrub_stamp,
+	     (bool)m_pg->info.stats.stats_invalid,
+	     get_pg_cct()->_conf->osd_scrub_invalid_stats,
+	     (res.is_must == must_scrub_t::mandatory ? "y" : "n"),
+	     res.min_interval, m_planned_scrub)
+      << dendl;
+  return res;
+}
+
+
 /*
  * Note: referring to m_planned_scrub here is temporary, as this set of
  * scheduling flags will be removed in a followup PR.
@@ -513,8 +550,7 @@ void PgScrubber::schedule_scrub_with_osd()
   auto pre_state = m_scrub_job->state_desc();
   auto pre_reg = registration_state();
 
-  auto suggested = m_osds->get_scrub_services().determine_scrub_time(
-      m_planned_scrub, m_pg->info, m_pg->get_pgpool().info.opts);
+  auto suggested = determine_scrub_time(m_pg->get_pgpool().info.opts);
   m_osds->get_scrub_services().register_with_osd(m_scrub_job, suggested);
 
   dout(10) << fmt::format(
@@ -554,8 +590,7 @@ void PgScrubber::update_scrub_job(const requested_scrub_t& request_flags)
 
   if (is_primary() && m_scrub_job) {
     ceph_assert(m_pg->is_locked());
-    auto suggested = m_osds->get_scrub_services().determine_scrub_time(
-	request_flags, m_pg->info, m_pg->get_pgpool().info.opts);
+    auto suggested = determine_scrub_time(m_pg->get_pgpool().info.opts);
     m_osds->get_scrub_services().update_job(m_scrub_job, suggested, true);
     m_pg->publish_stats_to_osd();
   }
diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h
index bcab24cddfa3..78e8ba90d449 100644
--- a/src/osd/scrubber/pg_scrubber.h
+++ b/src/osd/scrubber/pg_scrubber.h
@@ -784,6 +784,15 @@ class PgScrubber : public ScrubPgIF,
    */
   Scrub::sched_conf_t populate_config_params() const;
 
+  /**
+   * determine the time when the next scrub should be scheduled
+   *
+   * based on the planned scrub's flags, time of last scrub, and
+   * the pool's scrub configuration.
+   */
+  Scrub::sched_params_t determine_scrub_time(
+      const pool_opts_t& pool_conf) const;
+
   /*
    * Select a range of objects to scrub.
    *

From 0bd01b9c1a78c13d404c667e49ed3b9cb931fb47 Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Tue, 19 Dec 2023 20:35:50 +0530
Subject: [PATCH 1780/2492] common/LogEntry: Add log level to str helper for
 fmt::formatter<LogEntry>

The Ceph cluster logs were missing the string equivalent [INF|WRN|ERR|DBG]
representation of the 'prio' field. This was broken since the introduction
of commit 2901943618f071060f72d69cf632a4097e439df4 of
PR: https://github.com/ceph/ceph/pull/47830. This probably caused false
positives in teuthology testing and particularly for those tests that check
for cluster badness by parsing the cluster logs.

The fix involves adding a static helper function to the LogEntry struct.
This function returns the string appropriate representation of the log
level similar to the operator<<() for LogEntry.

Fixes: https://tracker.ceph.com/issues/64314
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/common/LogEntry.h | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/common/LogEntry.h b/src/common/LogEntry.h
index 3ddebbd3043c..5de0b71efba2 100644
--- a/src/common/LogEntry.h
+++ b/src/common/LogEntry.h
@@ -125,6 +125,23 @@ struct LogEntry {
   void dump(ceph::Formatter *f) const;
   static void generate_test_instances(std::list<LogEntry*>& o);
   static clog_type str_to_level(std::string const &str);
+  static std::string_view level_to_str(clog_type t) {
+    switch (t) {
+    case CLOG_DEBUG:
+      return "DBG";
+    case CLOG_INFO:
+      return "INF";
+    case CLOG_SEC:
+      return "SEC";
+    case CLOG_WARN:
+      return "WRN";
+    case CLOG_ERROR:
+      return "ERR";
+    case CLOG_UNKNOWN:
+      return "UNKNOWN";
+    }
+    return "???";
+  }
 };
 WRITE_CLASS_ENCODER_FEATURES(LogEntry)
 
@@ -204,8 +221,9 @@ template <> struct fmt::formatter<EntityName> : fmt::formatter<std::string_view>
 template <> struct fmt::formatter<LogEntry> : fmt::formatter<std::string_view> {
   template <typename FormatContext>
   auto format(const LogEntry& e, FormatContext& ctx) {
-    return fmt::format_to(ctx.out(), "{} {} ({}) {} : {} {} {}",
-			  e.stamp, e.name, e.rank, e.seq, e.channel, e.prio, e.msg);
+    return fmt::format_to(ctx.out(), "{} {} ({}) {} : {} [{}] {}",
+                          e.stamp, e.name, e.rank, e.seq, e.channel,
+                          LogEntry::level_to_str(e.prio), e.msg);
   }
 };
 

From f0987e5151b43b70f62e823abd65aa4fa8306e3f Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Fri, 5 Jan 2024 19:21:17 +0000
Subject: [PATCH 1781/2492] rgw: add new cap user-info-without-keys

This new cap allows users to run the admin api op
`get user info` without the S3 keys and Swift keys
in the response.

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 doc/radosgw/admin.rst                 |   2 +-
 doc/radosgw/adminops.rst              |   7 +-
 qa/tasks/radosgw_admin_rest.py        | 172 +++++++++++++++++++++++++-
 src/rgw/driver/rados/rgw_rest_user.cc |  16 ++-
 src/rgw/driver/rados/rgw_user.cc      |  16 ++-
 src/rgw/driver/rados/rgw_user.h       |   2 +-
 src/rgw/rgw_common.cc                 |   1 +
 7 files changed, 200 insertions(+), 16 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 57d38c97ab2f..501f161c1df9 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -375,7 +375,7 @@ form:
 
 .. prompt:: bash
 
-   --caps="[users|buckets|metadata|usage|zone|amz-cache|info|bilog|mdlog|datalog|user-policy|oidc-provider|roles|ratelimit]=[\*|read|write|read, write]"
+   --caps="[users|buckets|metadata|usage|zone|amz-cache|info|bilog|mdlog|datalog|user-policy|oidc-provider|roles|ratelimit|user-info-without-keys]=[\*|read|write|read, write]"
 
 For example:
 
diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst
index f1faac6b1c2f..f9e501515d0b 100644
--- a/doc/radosgw/adminops.rst
+++ b/doc/radosgw/adminops.rst
@@ -273,13 +273,14 @@ TBD.
 Get User Info
 =============
 
-Get user information.
+Get user information. Cap ``users`` or ``user-info-without-keys`` must be set to ``read`` to run this operation.
+If cap ``user-info-without-keys`` is set to ``read`` or ``*``, S3 keys and Swift keys will not be
+included in the response unless the user running this operation is the system user, an admin user, or the cap ``users`` is set to ``read``.
 
 Either a ``uid`` or ``access-key`` must be supplied as a request parameter. We recommend supplying uid.
 If both are provided but correspond to different users, the info for the user specified with ``uid`` will be returned.
 
-:caps: users=read
-
+:caps: users=read or user-info-without-keys=read
 
 Syntax
 ~~~~~~
diff --git a/qa/tasks/radosgw_admin_rest.py b/qa/tasks/radosgw_admin_rest.py
index 3de4d6bc9258..4b07ad330d34 100644
--- a/qa/tasks/radosgw_admin_rest.py
+++ b/qa/tasks/radosgw_admin_rest.py
@@ -28,13 +28,13 @@ def rgwadmin_rest(connection, cmd, params=None, headers=None, raw=False):
     perform a rest command
     """
     log.info('radosgw-admin-rest: %s %s' % (cmd, params))
-    put_cmds = ['create', 'link', 'add']
+    put_cmds = ['create', 'link', 'add', 'set']
     post_cmds = ['unlink', 'modify']
     delete_cmds = ['trim', 'rm', 'process']
-    get_cmds = ['check', 'info', 'show', 'list', '']
+    get_cmds = ['check', 'info', 'show', 'list', 'get', '']
 
     bucket_sub_resources = ['object', 'policy', 'index']
-    user_sub_resources = ['subuser', 'key', 'caps']
+    user_sub_resources = ['subuser', 'key', 'caps', 'quota']
     zone_sub_resources = ['pool', 'log', 'garbage']
 
     def get_cmd_method_and_handler(cmd):
@@ -117,6 +117,167 @@ def build_admin_request(conn, method, resource = '', headers=None, data='',
         log.info(' json result: %s' % result.json())
         return result.status_code, result.json()
 
+def test_cap_user_info_without_keys_get_user_info_privileged_users(ctx, client, op, op_args, uid, display_name, access_key, secret_key, user_type):
+    user_caps = 'user-info-without-keys=read'
+
+    (err, out) = rgwadmin(ctx, client, [
+            'user', 'create',
+            '--uid', uid,
+            '--display-name', display_name,
+            '--access-key', access_key,
+            '--secret', secret_key,
+            '--caps', user_caps,
+            user_type
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    endpoint = ctx.rgw.role_endpoints.get(client)
+
+    privileged_user_conn = boto.s3.connection.S3Connection(
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        is_secure=True if endpoint.cert else False,
+        port=endpoint.port,
+        host=endpoint.hostname,
+        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
+        )
+
+    (ret, out) = rgwadmin_rest(privileged_user_conn, op, op_args)
+    # show that even though the cap is set, since the user is privileged the user can still see keys
+    assert len(out['keys']) == 1
+    assert out['swift_keys'] == []
+
+    (err, out) = rgwadmin(ctx, client, [
+            'user', 'rm',
+            '--uid', uid,
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+def test_cap_user_info_without_keys_get_user_info(ctx, client, admin_conn, admin_user, op, op_args):
+    true_admin_uid = 'a_user'
+    true_admin_display_name = 'True Admin User'
+    true_admin_access_key = 'true_admin_akey'
+    true_admin_secret_key = 'true_admin_skey'
+
+    system_uid = 'system_user'
+    system_display_name = 'System User'
+    system_access_key = 'system_akey'
+    system_secret_key = 'system_skey'
+
+    test_cap_user_info_without_keys_get_user_info_privileged_users(ctx, client, op, op_args, system_uid, system_display_name, system_access_key, system_secret_key, '--system')
+    test_cap_user_info_without_keys_get_user_info_privileged_users(ctx, client, op, op_args, true_admin_uid, true_admin_display_name, true_admin_access_key, true_admin_secret_key, '--admin')
+
+    # TESTCASE 'info-existing','user','info','existing user','returns no keys with user-info-without-keys cap set to read'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'user-info-without-keys=read'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    (ret, out) = rgwadmin_rest(admin_conn, op, op_args)
+    assert 'keys' not in out
+    assert 'swift_keys' not in out
+
+    # TESTCASE 'info-existing','user','info','existing user','returns no keys with user-info-without-keys cap set to read'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'user-info-without-keys=*'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    (ret, out) = rgwadmin_rest(admin_conn, op, op_args)
+    assert 'keys' not in out
+    assert 'swift_keys' not in out
+
+    # TESTCASE 'info-existing','user','info','existing user','returns keys with user-info-without-keys cap set to read but cap users is set to read'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'users=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    (ret, out) = rgwadmin_rest(admin_conn, op, op_args)
+    assert 'keys' in out
+    assert 'swift_keys' in out
+
+    # TESTCASE 'info-existing','user','info','existing user','returns 403 with user-info-without-keys cap set to write'
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'rm',
+            '--uid', admin_user,
+            '--caps', 'users=read, write; user-info-without-keys=*'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'user-info-without-keys=write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    (ret, out) = rgwadmin_rest(admin_conn, op, op_args)
+    assert ret == 403
+
+    # remove cap user-info-without-keys permenantly for future testing
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'rm',
+            '--uid', admin_user,
+            '--caps', 'user-info-without-keys=write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    # reset cap users permenantly for future testing
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'users=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+def test_cap_user_info_without_keys(ctx, client, admin_conn, admin_user, user1):
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'rm',
+            '--uid', admin_user,
+            '--caps', 'users=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
+
+    op = ['user', 'info']
+    op_args = {'uid' : user1}
+    test_cap_user_info_without_keys_get_user_info(ctx, client, admin_conn, admin_user, op, op_args)
+
+    # add caps that were removed earlier in the function back in
+    (err, out) = rgwadmin(ctx, client, [
+            'caps', 'add',
+            '--uid', admin_user,
+            '--caps', 'users=read, write'
+            ])
+    logging.error(out)
+    logging.error(err)
+    assert not err
 
 def task(ctx, config):
     """
@@ -291,6 +452,9 @@ def task(ctx, config):
     assert out['type'] == 'rgw'
     assert out['mfa_ids'] == []
 
+    # TESTCASES for cap user-info-without-keys
+    test_cap_user_info_without_keys(ctx, client, admin_conn, admin_user, user1)
+
     # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
     (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
     assert ret == 200
@@ -812,4 +976,4 @@ def task(ctx, config):
 
     # TESTCASE 'ratelimit' 'global' 'modify' 'anonymous' 'enabled' 'succeeds'
     (ret, out) = rgwadmin_rest(admin_conn, ['ratelimit', 'modify'], {'ratelimit-scope' : 'bucket', 'global': 'true', 'enabled' : 'true'})
-    assert ret == 200
\ No newline at end of file
+    assert ret == 200
diff --git a/src/rgw/driver/rados/rgw_rest_user.cc b/src/rgw/driver/rados/rgw_rest_user.cc
index 200f1c03d2e1..89712970fb2d 100644
--- a/src/rgw/driver/rados/rgw_rest_user.cc
+++ b/src/rgw/driver/rados/rgw_rest_user.cc
@@ -73,6 +73,10 @@ class RGWOp_User_Info : public RGWRESTOp {
   RGWOp_User_Info() {}
 
   int check_caps(const RGWUserCaps& caps) override {
+    int r = caps.check_cap("user-info-without-keys", RGW_CAP_READ);
+    if (r == 0) {
+      return r;
+    }
     return caps.check_cap("users", RGW_CAP_READ);
   }
 
@@ -84,10 +88,12 @@ class RGWOp_User_Info : public RGWRESTOp {
 void RGWOp_User_Info::execute(optional_yield y)
 {
   RGWUserAdminOpState op_state(driver);
+  op_state.set_system(s->system_request);
 
   std::string uid_str, access_key_str;
   bool fetch_stats;
   bool sync_stats;
+  bool dump_keys = false;
 
   RESTArgs::get_string(s, "uid", uid_str, &uid_str);
   RESTArgs::get_string(s, "access-key", access_key_str, &access_key_str);
@@ -111,7 +117,15 @@ void RGWOp_User_Info::execute(optional_yield y)
   op_state.set_fetch_stats(fetch_stats);
   op_state.set_sync_stats(sync_stats);
 
-  op_ret = RGWUserAdminOp_User::info(s, driver, op_state, flusher, y);
+  // dump_keys is false if user-info-without-keys is 'read' and
+  // the user is not the system user or an admin user
+  int keys_perm = s->user->get_info().caps.check_cap("users", RGW_CAP_READ);
+  if (keys_perm == 0 || op_state.system || s->auth.identity->is_admin_of(uid)) {
+    dump_keys = true;
+    ldpp_dout(s, 20) << "dump_keys is set to true" << dendl;
+  }
+
+  op_ret = RGWUserAdminOp_User::info(s, driver, op_state, flusher, dump_keys, y);
 }
 
 class RGWOp_User_Create : public RGWRESTOp {
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index b5569e481c53..f13e087c4364 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -126,7 +126,7 @@ static void dump_swift_keys_info(Formatter *f, RGWUserInfo &info)
 }
 
 static void dump_user_info(Formatter *f, RGWUserInfo &info,
-                           RGWStorageStats *stats = NULL)
+                           bool dump_keys, RGWStorageStats *stats = NULL)
 {
   f->open_object_section("user_info");
   encode_json("tenant", info.user_id.tenant, f);
@@ -137,8 +137,11 @@ static void dump_user_info(Formatter *f, RGWUserInfo &info,
   encode_json("max_buckets", (int)info.max_buckets, f);
 
   dump_subusers_info(f, info);
-  dump_access_keys_info(f, info);
-  dump_swift_keys_info(f, info);
+
+  if (dump_keys) {
+    dump_access_keys_info(f, info);
+    dump_swift_keys_info(f, info);
+  }
 
   encode_json("caps", info.caps, f);
 
@@ -2098,6 +2101,7 @@ int RGWUserAdminOp_User::list(const DoutPrefixProvider *dpp, rgw::sal::Driver* d
 int RGWUserAdminOp_User::info(const DoutPrefixProvider *dpp,
 			      rgw::sal::Driver* driver, RGWUserAdminOpState& op_state,
 			      RGWFormatterFlusher& flusher,
+                              bool dump_keys,
 			      optional_yield y)
 {
   RGWUserInfo info;
@@ -2140,7 +2144,7 @@ int RGWUserAdminOp_User::info(const DoutPrefixProvider *dpp,
   if (formatter) {
     flusher.start(0);
 
-    dump_user_info(formatter, info, arg_stats);
+    dump_user_info(formatter, info, dump_keys, arg_stats);
     flusher.flush();
   }
 
@@ -2174,7 +2178,7 @@ int RGWUserAdminOp_User::create(const DoutPrefixProvider *dpp,
   if (formatter) {
     flusher.start(0);
 
-    dump_user_info(formatter, info);
+    dump_user_info(formatter, info, true);
     flusher.flush();
   }
 
@@ -2207,7 +2211,7 @@ int RGWUserAdminOp_User::modify(const DoutPrefixProvider *dpp,
   if (formatter) {
     flusher.start(0);
 
-    dump_user_info(formatter, info);
+    dump_user_info(formatter, info, true);
     flusher.flush();
   }
 
diff --git a/src/rgw/driver/rados/rgw_user.h b/src/rgw/driver/rados/rgw_user.h
index a0cd7ed84fee..91c859feafca 100644
--- a/src/rgw/driver/rados/rgw_user.h
+++ b/src/rgw/driver/rados/rgw_user.h
@@ -645,7 +645,7 @@ class RGWUserAdminOp_User
   static int info(const DoutPrefixProvider *dpp,
 		  rgw::sal::Driver* driver,
                   RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher,
-		  optional_yield y);
+		  bool dump_keys, optional_yield y);
 
   static int create(const DoutPrefixProvider *dpp,
 		    rgw::sal::Driver* driver,
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index f75ed2aca016..125d27801838 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -2091,6 +2091,7 @@ bool RGWUserCaps::is_valid_cap_type(const string& tp)
                                     "user-policy",
                                     "amz-cache",
                                     "oidc-provider",
+                                    "user-info-without-keys",
 				                            "ratelimit"};
 
   for (unsigned int i = 0; i < sizeof(cap_type) / sizeof(char *); ++i) {

From 1a33cefa1980a18b11479275c1f708c1de24ee37 Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Thu, 16 Mar 2023 09:43:58 +0000
Subject: [PATCH 1782/2492] common: use close_range on Linux

Fix rook/rook#10110, which occurs when _SC_OPEN_MAX/RLIMIT_NOFILE is
set to very large values (2^30), leaving fork_function pegging a core
busylooping.

The glibc wrappers closefrom(3)/close_range(3) are not available before
glibc 2.34, so we invoke the syscall directly. When glibc 2.34 is old
enough to be a reasonable hard minimum dependency, we should switch to
using closefrom.

If we're not running on (recent enough) Linux, we fall back to the
existing approach.

Fixes: https://tracker.ceph.com/issues/59125
Signed-off-by: edef <edef@edef.eu>
---
 src/common/SubProcess.cc   |  9 +++++++++
 src/common/fork_function.h | 31 ++++++++++++++++++++-----------
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/common/SubProcess.cc b/src/common/SubProcess.cc
index 1faf33e36eee..8f28ff25ca21 100644
--- a/src/common/SubProcess.cc
+++ b/src/common/SubProcess.cc
@@ -4,6 +4,9 @@
 #include <sys/types.h>
 #include <signal.h>
 #endif
+#ifdef __linux__
+#include <sys/syscall.h>
+#endif
 #include <stdarg.h>
 #include <fcntl.h>
 #include <unistd.h>
@@ -200,6 +203,12 @@ int SubProcess::spawn() {
     int maxfd = sysconf(_SC_OPEN_MAX);
     if (maxfd == -1)
       maxfd = 16384;
+
+#if defined(__linux__) && defined(SYS_close_range)
+    if (::syscall(SYS_close_range, STDERR_FILENO + 1, ~0U, 0) == 0)
+      maxfd = STDERR_FILENO;
+#endif
+
     for (int fd = 0; fd <= maxfd; fd++) {
       if (fd == STDIN_FILENO && stdin_op != CLOSE)
 	continue;
diff --git a/src/common/fork_function.h b/src/common/fork_function.h
index 3a4f2f29c08a..5c94be4dcaf7 100644
--- a/src/common/fork_function.h
+++ b/src/common/fork_function.h
@@ -13,6 +13,9 @@
 #ifndef _WIN32
 #include <sys/wait.h>
 #endif
+#ifdef __linux__
+#include <sys/syscall.h>
+#endif
 #include <sys/types.h>
 
 #include "include/ceph_assert.h"
@@ -53,17 +56,23 @@ static inline int fork_function(
   // we are forker (first child)
 
   // close all fds
-  int maxfd = sysconf(_SC_OPEN_MAX);
-  if (maxfd == -1)
-    maxfd = 16384;
-  for (int fd = 0; fd <= maxfd; fd++) {
-    if (fd == STDIN_FILENO)
-      continue;
-    if (fd == STDOUT_FILENO)
-      continue;
-    if (fd == STDERR_FILENO)
-      continue;
-    ::close(fd);
+#if defined(__linux__) && defined(SYS_close_range)
+  if (::syscall(SYS_close_range, STDERR_FILENO + 1, ~0U, 0))
+#endif
+  {
+    // fall back to manually closing
+    int maxfd = sysconf(_SC_OPEN_MAX);
+    if (maxfd == -1)
+      maxfd = 16384;
+    for (int fd = 0; fd <= maxfd; fd++) {
+      if (fd == STDIN_FILENO)
+        continue;
+      if (fd == STDOUT_FILENO)
+        continue;
+      if (fd == STDERR_FILENO)
+        continue;
+      ::close(fd);
+    }
   }
 
   sigset_t mask, oldmask;

From 9b31e9330fe2e67687399479ee4924b4ad671d1f Mon Sep 17 00:00:00 2001
From: Pritha Srivastava <prsrivas@redhat.com>
Date: Mon, 5 Feb 2024 14:00:47 +0530
Subject: [PATCH 1783/2492] rgw: code to display the complete user id that
 includes tenant, namespace, and id for a GET request to admin ops for user
 info. added code to display namespace also.

Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
---
 src/rgw/driver/rados/rgw_user.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index b5569e481c53..c4c47049f939 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -129,7 +129,11 @@ static void dump_user_info(Formatter *f, RGWUserInfo &info,
                            RGWStorageStats *stats = NULL)
 {
   f->open_object_section("user_info");
+  encode_json("full_user_id", info.user_id, f);
   encode_json("tenant", info.user_id.tenant, f);
+  if (!info.user_id.ns.empty()) {
+    encode_json("namespace", info.user_id.ns, f);
+  }
   encode_json("user_id", info.user_id.id, f);
   encode_json("display_name", info.display_name, f);
   encode_json("email", info.user_email, f);

From 4581d0dafa8806531cc6ce18971b2aba7017bc02 Mon Sep 17 00:00:00 2001
From: Zhipeng Li <qiuxinyidian@gmail.com>
Date: Tue, 23 Jan 2024 14:50:52 +0800
Subject: [PATCH 1784/2492] rgw: modify topic owner check when creating

add tests to cover topic policies
as well as behavior when no policies are defined

Fixes: https://tracker.ceph.com/issues/64124

Signed-off-by: Zhipeng Li <qiuxinyidian@gmail.com>
---
 PendingReleaseNotes                         |   9 ++
 src/common/options/rgw.yaml.in              |   8 +-
 src/rgw/rgw_iam_policy.cc                   |   4 +
 src/rgw/rgw_iam_policy.h                    |   3 +-
 src/rgw/rgw_rest_pubsub.cc                  |  55 +++++-----
 src/test/rgw/bucket_notification/test_bn.py | 107 +++++++++++++++++++-
 6 files changed, 156 insertions(+), 30 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index dfc6294b36a2..0ecec83ec81f 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -121,6 +121,15 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   trim the log every second (`mds_log_trim_upkeep_interval` config). Also,
   a couple of configs govern how much time the MDS spends in trimming its
   logs. These configs are `mds_log_trim_threshold` and `mds_log_trim_decay_rate`.
+* RGW: Notification topics are now owned by the user that created them. 
+  By default, only the owner can read/write their topics. Topic policy documents
+  are now supported to grant these permissions to other users. Preexisting topics
+  are treated as if they have no owner, and any user can read/write them using the SNS API. 
+  If such a topic is recreated with CreateTopic, the issuing user becomes the new owner.
+  For backward compatibility, all users still have permission to publish bucket 
+  notifications to topics owned by other users. A new configuration parameter:
+  ``rgw_topic_require_publish_policy`` can be enabled to deny ``sns:Publish``
+  permissions unless explicitly granted by topic policy.
 
 >=18.0.0
 
diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index f4ef0079dd6b..6fab43e5589f 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3917,15 +3917,15 @@ options:
   services:
   - rgw
   with_legacy: true
-- name: mandatory_topic_permissions
+- name: rgw_topic_require_publish_policy
   type: bool
   level: basic
-  desc: Whether to validate user permissions to access notification topics.
+  desc: Whether to validate user permissions to publish notifications to topics.
   long_desc: If true, all users (other then the owner of the topic) will need
-    to have a policy to access topics.
+    to have a policy to publish notifications to topics.
     The topic policy can be set by owner via CreateTopic() or SetTopicAttribute().
     Following permissions can be granted "sns:Publish", "sns:GetTopicAttributes",
-    "sns:SetTopicAttributes" and "sns:DeleteTopic" via Policy.
+    "sns:SetTopicAttributes", "sns:DeleteTopic" and "sns:CreateTopic" via Policy.
     NOTE that even if set to "false" topics will still follow the policies if set on them.
   default: false
   services:
diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc
index 76b24034d614..813b78f161e1 100644
--- a/src/rgw/rgw_iam_policy.cc
+++ b/src/rgw/rgw_iam_policy.cc
@@ -161,6 +161,7 @@ static const actpair actpairs[] =
  { "sns:DeleteTopic", snsDeleteTopic},
  { "sns:Publish", snsPublish},
  { "sns:SetTopicAttributes", snsSetTopicAttributes},
+ { "sns:CreateTopic", snsCreateTopic},
 };
 
 struct PolicyParser;
@@ -1476,6 +1477,9 @@ const char* action_bit_string(uint64_t action) {
 
   case snsPublish:
     return "sns:Publish";
+
+  case snsCreateTopic:
+    return "sns:CreateTopic";
   }
   return "s3Invalid";
 }
diff --git a/src/rgw/rgw_iam_policy.h b/src/rgw/rgw_iam_policy.h
index e528d1515c77..5d6f334c176e 100644
--- a/src/rgw/rgw_iam_policy.h
+++ b/src/rgw/rgw_iam_policy.h
@@ -145,7 +145,8 @@ static constexpr std::uint64_t snsGetTopicAttributes = stsAll + 1;
 static constexpr std::uint64_t snsDeleteTopic = stsAll + 2;
 static constexpr std::uint64_t snsPublish = stsAll + 3;
 static constexpr std::uint64_t snsSetTopicAttributes = stsAll + 4;
-static constexpr std::uint64_t snsAll = stsAll + 5;
+static constexpr std::uint64_t snsCreateTopic = stsAll + 5;
+static constexpr std::uint64_t snsAll = stsAll + 6;
 
 static constexpr std::uint64_t s3Count = s3All;
 static constexpr std::uint64_t allCount = snsAll + 1;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 611589d721f3..191f535d82bd 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <boost/tokenizer.hpp>
 #include <optional>
+#include "rgw_iam_policy.h"
 #include "rgw_rest_pubsub.h"
 #include "rgw_pubsub_push.h"
 #include "rgw_pubsub.h"
@@ -75,8 +76,8 @@ std::optional<rgw::IAM::Policy> get_policy_from_text(req_state* const s,
         s->cct, s->owner.id.tenant, bl,
         s->cct->_conf.get_val<bool>("rgw_policy_reject_invalid_principals"));
   } catch (rgw::IAM::PolicyParseException& e) {
-    ldout(s->cct, 1) << "failed to parse policy:' " << policy_text
-                     << " ' with error: " << e.what() << dendl;
+    ldout(s->cct, 1) << "failed to parse policy: '" << policy_text
+                     << "' with error: " << e.what() << dendl;
     s->err.message = e.what();
     return std::nullopt;
   }
@@ -91,13 +92,18 @@ int verify_topic_owner_or_policy(req_state* const s,
   }
   // no policy set.
   if (topic.policy_text.empty()) {
-    // if mandatory_topic_permissions is true, then validate all users for
-    // permission.
-    if (s->cct->_conf->mandatory_topic_permissions) {
-      return -EACCES;
-    } else {
+    // if rgw_topic_require_publish_policy is "false" dont validate "publish" policies
+    if (op == rgw::IAM::snsPublish && !s->cct->_conf->rgw_topic_require_publish_policy) {
+      return 0;
+    }
+    if (topic.user.empty()) {
+      // if we don't know the original user and there is no policy
+      // we will not reject the request.
+      // this is for compatibility with versions that did not store the user in the topic
       return 0;
     }
+    s->err.message = "Topic was created by another user.";
+    return -EACCES;
   }
   // bufferlist::static_from_string wants non const string
   std::string policy_text(topic.policy_text);
@@ -107,7 +113,7 @@ int verify_topic_owner_or_policy(req_state* const s,
                      s->user->get_tenant(), topic.name);
   if (!p || p->eval(s->env, *s->auth.identity, op, arn, princ_type) !=
                 rgw::IAM::Effect::Allow) {
-    ldout(s->cct, 1) << "topic_policy failed validation, topic_policy: " << p
+    ldout(s->cct, 1) << "topic policy failed validation, topic policy: " << p
                      << dendl;
     return -EACCES;
   }
@@ -195,13 +201,17 @@ class RGWPSCreateTopicOp : public RGWOp {
       return 0;
     }
     if (ret == 0) {
-      if (result.user == s->owner.id ||
-          !s->cct->_conf->mandatory_topic_permissions) {
+      ret = verify_topic_owner_or_policy(
+          s, result, driver->get_zone()->get_zonegroup().get_name(),
+          rgw::IAM::snsCreateTopic);
+      if (ret == 0)
+      {
         return 0;
       }
-      ldpp_dout(this, 1) << "failed to create topic '" << topic_name
+
+      ldpp_dout(this, 1) << "no permission to modify topic '" << topic_name
                          << "', topic already exist." << dendl;
-      return -EPERM;
+      return -EACCES;
     }
     ldpp_dout(this, 1) << "failed to read topic '" << topic_name
                        << "', with error:" << ret << dendl;
@@ -408,8 +418,8 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
       s, result, driver->get_zone()->get_zonegroup().get_name(),
       rgw::IAM::snsGetTopicAttributes);
   if (op_ret != 0) {
-    ldpp_dout(this, 1) << "failed to get topic '" << topic_name
-                       << "', topic owned by other user" << dendl;
+    ldpp_dout(this, 1) << "no permission to get topic '" << topic_name
+                       << "'" << dendl;
     return;
   }
   ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
@@ -492,8 +502,8 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
       s, result, driver->get_zone()->get_zonegroup().get_name(),
       rgw::IAM::snsGetTopicAttributes);
   if (op_ret != 0) {
-    ldpp_dout(this, 1) << "failed to get topic '" << topic_name
-                       << "', topic owned by other user" << dendl;
+    ldpp_dout(this, 1) << "no permission to get topic '" << topic_name
+                       << "'" << dendl;
     return;
   }
   ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
@@ -617,8 +627,8 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
         s, result, driver->get_zone()->get_zonegroup().get_name(),
         rgw::IAM::snsSetTopicAttributes);
     if (ret != 0) {
-      ldpp_dout(this, 1) << "failed to set attributes for topic '" << topic_name
-                         << "', topic owned by other user" << dendl;
+      ldpp_dout(this, 1) << "no permission to set attributes for topic '" << topic_name
+                         << "'" << dendl;
       return ret;
     }
 
@@ -750,8 +760,8 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
         s, result, driver->get_zone()->get_zonegroup().get_name(),
         rgw::IAM::snsDeleteTopic);
     if (op_ret != 0) {
-      ldpp_dout(this, 1) << "failed to remove topic '" << topic_name
-                         << "' topic owned by other user" << dendl;
+      ldpp_dout(this, 1) << "no permission to remove topic '" << topic_name
+                         << "'" << dendl;
       return;
     }
   } else {
@@ -1025,9 +1035,8 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
         s, topic_info, driver->get_zone()->get_zonegroup().get_name(),
         rgw::IAM::snsPublish);
     if (op_ret != 0) {
-      ldpp_dout(this, 1) << "failed to create notification for topic '"
-                         << topic_name << "' topic owned by other user"
-                         << dendl;
+      ldpp_dout(this, 1) << "no permission to create notification for topic '"
+                         << topic_name << "'" << dendl;
       return;
     }
     // make sure that full topic configuration match
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 6e9248a3669b..30cbfdfe7865 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -4395,6 +4395,7 @@ def test_ps_s3_multiple_topics_notification():
     conn.delete_bucket(bucket_name)
     http_server.close()
 
+
 @attr('basic_test')
 def test_ps_s3_topic_permissions():
     """ test s3 topic set/get/delete permissions """
@@ -4410,7 +4411,7 @@ def test_ps_s3_topic_permissions():
                 "Sid": "Statement",
                 "Effect": "Deny",
                 "Principal": "*",
-                "Action": ["sns:Publish", "sns:SetTopicAttributes", "sns:GetTopicAttributes"],
+                "Action": ["sns:Publish", "sns:SetTopicAttributes", "sns:GetTopicAttributes", "sns:DeleteTopic", "sns:CreateTopic"],
                 "Resource": f"arn:aws:sns:{zonegroup}::{topic_name}"
             }
         ]
@@ -4421,10 +4422,23 @@ def test_ps_s3_topic_permissions():
     topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args, policy_text=topic_policy)
     topic_arn = topic_conf.set_config()
 
-    # 2nd user tries to fetch the topic
     topic_conf2 = PSTopicS3(conn2, topic_name, zonegroup, endpoint_args=endpoint_args)
+    try:
+        # 2nd user tries to override the topic
+        topic_arn = topic_conf2.set_config()
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
+
+    # 2nd user tries to fetch the topic
     _, status = topic_conf2.get_config(topic_arn=topic_arn)
     assert_equal(status, 403)
+
     try:
         # 2nd user tries to set the attribute
         status = topic_conf2.set_attributes(attribute_name="persistent", attribute_val="false", topic_arn=topic_arn)
@@ -4455,6 +4469,18 @@ def test_ps_s3_topic_permissions():
     except Exception as err:
         print('unexpected error type: '+type(err).__name__)
 
+    try:
+        # 2nd user tries to delete the topic
+        status = topic_conf2.del_config(topic_arn=topic_arn)
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
+
     # Topic policy is now added by the 1st user to allow 2nd user.
     topic_policy  = topic_policy.replace("Deny", "Allow")
     topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args, policy_text=topic_policy)
@@ -4469,6 +4495,82 @@ def test_ps_s3_topic_permissions():
     s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
     _, status = s3_notification_conf2.set_config()
     assert_equal(status, 200)
+    # 2nd user tries to delete the topic again
+    status = topic_conf2.del_config(topic_arn=topic_arn)
+    assert_equal(status, 200)
+
+    # cleanup
+    s3_notification_conf2.del_config()
+    # delete the bucket
+    conn2.delete_bucket(bucket_name)
+
+
+@attr('basic_test')
+def test_ps_s3_topic_no_permissions():
+    """ test s3 topic set/get/delete permissions """
+    conn1 = connection()
+    conn2 = another_user()
+    zonegroup = 'default'
+    bucket_name = gen_bucket_name()
+    topic_name = bucket_name + TOPIC_SUFFIX
+    
+    # create s3 topic without policy
+    endpoint_address = 'amqp://127.0.0.1:7001'
+    endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
+    topic_conf = PSTopicS3(conn1, topic_name, zonegroup, endpoint_args=endpoint_args)
+    topic_arn = topic_conf.set_config()
+
+    topic_conf2 = PSTopicS3(conn2, topic_name, zonegroup, endpoint_args=endpoint_args)
+    try:
+        # 2nd user tries to override the topic
+        topic_arn = topic_conf2.set_config()
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
+
+    # 2nd user tries to fetch the topic
+    _, status = topic_conf2.get_config(topic_arn=topic_arn)
+    assert_equal(status, 403)
+
+    try:
+        # 2nd user tries to set the attribute
+        status = topic_conf2.set_attributes(attribute_name="persistent", attribute_val="false", topic_arn=topic_arn)
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
+
+    # create bucket for conn2 publish notification to topic
+    # should be allowed based on the default value of rgw_topic_require_publish_policy=false
+    _ = conn2.create_bucket(bucket_name)
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
+                         'Events': []
+                       }]
+    s3_notification_conf2 = PSNotificationS3(conn2, bucket_name, topic_conf_list)
+    _, status = s3_notification_conf2.set_config()
+    assert_equal(status, 200)
+    
+    try:
+        # 2nd user tries to delete the topic
+        status = topic_conf2.del_config(topic_arn=topic_arn)
+        assert False, "'AccessDenied' error is expected"
+    except ClientError as err:
+        if 'Error' in err.response:
+            assert_equal(err.response['Error']['Code'], 'AccessDenied')
+        else:
+            assert_equal(err.response['Code'], 'AccessDenied')
+    except Exception as err:
+        print('unexpected error type: '+type(err).__name__)
 
     # cleanup
     s3_notification_conf2.del_config()
@@ -4476,6 +4578,7 @@ def test_ps_s3_topic_permissions():
     # delete the bucket
     conn2.delete_bucket(bucket_name)
 
+
 def kafka_security(security_type, mechanism='PLAIN'):
     """ test pushing kafka s3 notification securly to master """
     conn = connection()

From 320a2179a3c6c1981a0fd2494938515997c1bfad Mon Sep 17 00:00:00 2001
From: Omri Zeneva <ozeneva@redhat.com>
Date: Wed, 24 Aug 2022 09:57:11 -0400
Subject: [PATCH 1785/2492] tracer/osd/librados/build/rgw: rgw and osd end2end
 tracing using opentelemetry

* build: add opentelemetry to cmake system
crimson targets that uses Message.cc/h are built before opentelemetry (o-tel), so we need to build o-tel eralier so we also add the library to the include path earlier
this shoud work for WITH_JAEGER flag both the ON/OFF cases, and for librados where the compilation flag is ignored

* msg/tracer: add o-tel trace to Messages with decode/encode function in tracer.h
some files that uses Message.cc/h just need the encode/decode functions  and not all others functions.
some crimson targets does not link with ceph_context (common) which is required for tracer.cc file. so we just need to include that functions

* librados: Add opentelemtry trace param for aio_operate and operate methods
in order to propagate the trace info I added the otel-trace as an extra param.
in some places, there already was a blkin trace info, and since it is not used in other places we can safely change it to o-tel trace info.
this will be done in another commit, so the cleanup of blkin trace will be in a dedicated commit

* osd: use the o-tel trace of the msg as a parent span of the osd trace
if there is a valid span in the msg, we will add this op to the request
trace, otherwise it will start a new trace for the OSD op

* rgw: pass put obj trace info to librados
in order to make it possible, I saved the trace info inside the sal::Object, so we can use it later when writing the object to rados
it could be used also later for read ops.
note the trace field of req_state is initalized only in rgw_process, so it's also required in librgw request flow

* prevent breaking channges to kSize. make sure that changes between components built with
different versions of OTEL do not break message compatibility

Signed-off-by: Omri Zeneva <ozeneva@redhat.com>
---
 cmake/modules/BuildOpentelemetry.cmake       |  1 +
 src/CMakeLists.txt                           | 15 +++--
 src/common/tracer.cc                         | 38 +-----------
 src/common/tracer.h                          | 63 +++++++++++++++++---
 src/include/rados/librados.hpp               |  2 +
 src/include/rados/librados_fwd.hpp           | 12 ++++
 src/librados/IoCtxImpl.cc                    |  8 +--
 src/librados/IoCtxImpl.h                     |  4 +-
 src/librados/librados_asio.h                 |  6 +-
 src/librados/librados_cxx.cc                 | 22 ++++++-
 src/messages/MOSDOp.h                        | 46 +++++++++++++-
 src/msg/Message.cc                           |  9 +++
 src/msg/Message.h                            |  6 ++
 src/osd/OSD.cc                               |  7 ++-
 src/osdc/Objecter.cc                         |  4 ++
 src/osdc/Objecter.h                          | 12 ++--
 src/rgw/driver/rados/rgw_putobj_processor.cc | 10 ++--
 src/rgw/driver/rados/rgw_putobj_processor.h  | 24 ++++----
 src/rgw/driver/rados/rgw_rados.cc            | 31 ++++++----
 src/rgw/driver/rados/rgw_rados.h             |  6 +-
 src/rgw/driver/rados/rgw_sal_rados.cc        | 15 ++---
 src/rgw/driver/rados/rgw_sal_rados.h         | 14 +++--
 src/rgw/driver/rados/rgw_tools.cc            |  8 +--
 src/rgw/driver/rados/rgw_tools.h             |  4 +-
 src/rgw/rgw_aio.cc                           | 23 +++----
 src/rgw/rgw_aio.h                            |  2 +-
 src/rgw/rgw_lib.cc                           |  2 +
 src/rgw/rgw_op.cc                            |  2 +
 src/rgw/rgw_sal.h                            |  5 +-
 src/rgw/rgw_sal_filter.h                     |  5 +-
 src/rgw/rgw_sal_store.h                      |  5 +-
 31 files changed, 281 insertions(+), 130 deletions(-)

diff --git a/cmake/modules/BuildOpentelemetry.cmake b/cmake/modules/BuildOpentelemetry.cmake
index ba2edaa09329..48b219e9c0fc 100644
--- a/cmake/modules/BuildOpentelemetry.cmake
+++ b/cmake/modules/BuildOpentelemetry.cmake
@@ -82,4 +82,5 @@ function(build_opentelemetry)
     PROPERTIES
       INTERFACE_LINK_LIBRARIES "${opentelemetry_deps}"
       INTERFACE_INCLUDE_DIRECTORIES "${opentelemetry_include_dir}")
+  include_directories(SYSTEM "${opentelemetry_include_dir}")
 endfunction()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index afa2442d5709..0b88801d3b15 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -301,6 +301,15 @@ if (WITH_BLKIN)
   add_subdirectory(blkin/blkin-lib)
 endif(WITH_BLKIN)
 
+if(WITH_JAEGER)
+  find_package(thrift 0.13.0 REQUIRED)
+  include(BuildOpentelemetry)
+  build_opentelemetry()
+  add_library(jaeger_base INTERFACE)
+  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
+    thrift::libthrift)
+endif()
+
 set(mds_files)
 list(APPEND mds_files
   mds/MDSMap.cc
@@ -441,12 +450,6 @@ target_compile_definitions(common-objs PRIVATE
 add_dependencies(common-objs legacy-option-headers)
 
 if(WITH_JAEGER)
-  find_package(thrift 0.13.0 REQUIRED)
-  include(BuildOpentelemetry)
-  build_opentelemetry()
-  add_library(jaeger_base INTERFACE)
-  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
-    thrift::libthrift)
   add_dependencies(common-objs jaeger_base)
   target_link_libraries(common-objs jaeger_base)
 endif()
diff --git a/src/common/tracer.cc b/src/common/tracer.cc
index e98053735b48..6a84480d60b5 100644
--- a/src/common/tracer.cc
+++ b/src/common/tracer.cc
@@ -60,8 +60,7 @@ jspan_ptr Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool
 }
 
 jspan_ptr Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan_ptr& parent_span) {
-  if (parent_span && parent_span->IsRecording()) {
-    ceph_assert(tracer);
+  if (is_enabled() && parent_span && parent_span->IsRecording()) {
     opentelemetry::trace::StartSpanOptions span_opts;
     span_opts.parent = parent_span->GetContext();
     ldout(cct, 20) << "adding span " << span_name << " " << dendl;
@@ -85,41 +84,6 @@ bool Tracer::is_enabled() const {
   return cct->_conf->jaeger_tracing_enable;
 }
 
-void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f) {
-  ENCODE_START(1, 1, bl);
-  using namespace opentelemetry;
-  using namespace trace;
-  auto is_valid = span_ctx.IsValid();
-  encode(is_valid, bl);
-  if (is_valid) {
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceId::kSize), bl);
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanId::kSize), bl);
-    encode(span_ctx.trace_flags().flags(), bl);
-  }
-  ENCODE_FINISH(bl);
-}
-
-void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
-  using namespace opentelemetry;
-  using namespace trace;
-  DECODE_START(1, bl);
-  bool is_valid;
-  decode(is_valid, bl);
-  if (is_valid) {
-    std::array<uint8_t, TraceId::kSize> trace_id;
-    std::array<uint8_t, SpanId::kSize> span_id;
-    uint8_t flags;
-    decode(trace_id, bl);
-    decode(span_id, bl);
-    decode(flags, bl);
-    span_ctx = SpanContext(
-      TraceId(nostd::span<uint8_t, TraceId::kSize>(trace_id)),
-      SpanId(nostd::span<uint8_t, SpanId::kSize>(span_id)),
-      TraceFlags(flags),
-      true);
-  }
-  DECODE_FINISH(bl);
-}
 } // namespace tracing
 
 #endif // HAVE_JAEGER
diff --git a/src/common/tracer.h b/src/common/tracer.h
index 291ff9de25a7..b039d304a79a 100644
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -4,7 +4,7 @@
 #pragma once
 
 #include "acconfig.h"
-#include "include/buffer.h"
+#include "include/encoding.h"
 
 #ifdef HAVE_JAEGER
 #include "opentelemetry/trace/provider.h"
@@ -16,6 +16,11 @@ using jspan_attribute = opentelemetry::common::AttributeValue;
 
 namespace tracing {
 
+static constexpr int TraceIdkSize = 16;
+static constexpr int SpanIdkSize = 8;
+static_assert(TraceIdkSize == opentelemetry::trace::TraceId::kSize);
+static_assert(SpanIdkSize == opentelemetry::trace::SpanId::kSize);
+
 class Tracer {
  private:
   const static opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> noop_tracer;
@@ -24,6 +29,7 @@ class Tracer {
   opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> tracer;
 
  public:
+
   Tracer() = default;
 
   void init(CephContext* _cct, opentelemetry::nostd::string_view service_name);
@@ -46,8 +52,41 @@ class Tracer {
 
 };
 
-void encode(const jspan_context& span, ceph::buffer::list& bl, uint64_t f = 0);
-void decode(jspan_context& span_ctx, ceph::buffer::list::const_iterator& bl);
+inline void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f = 0) {
+  ENCODE_START(1, 1, bl);
+  using namespace opentelemetry;
+  using namespace trace;
+  auto is_valid = span_ctx.IsValid();
+  encode(is_valid, bl);
+  if (is_valid) {
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceIdkSize), bl);
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanIdkSize), bl);
+    encode(span_ctx.trace_flags().flags(), bl);
+  }
+  ENCODE_FINISH(bl);
+}
+
+inline void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
+  using namespace opentelemetry;
+  using namespace trace;
+  DECODE_START(1, bl);
+  bool is_valid;
+  decode(is_valid, bl);
+  if (is_valid) {
+    std::array<uint8_t, TraceIdkSize> trace_id;
+    std::array<uint8_t, SpanIdkSize> span_id;
+    uint8_t flags;
+    decode(trace_id, bl);
+    decode(span_id, bl);
+    decode(flags, bl);
+    span_ctx = SpanContext(
+      TraceId(nostd::span<uint8_t, TraceIdkSize>(trace_id)),
+      SpanId(nostd::span<uint8_t, SpanIdkSize>(span_id)),
+      TraceFlags(flags),
+      true);
+  }
+  DECODE_FINISH(bl);
+}
 
 } // namespace tracing
 
@@ -63,10 +102,20 @@ class Value {
 
 using jspan_attribute = Value;
 
-struct jspan_context {
-  jspan_context() {}
-  jspan_context(bool sampled_flag, bool is_remote) {}
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+class SpanContext {
+public:
+  SpanContext() = default;
+  SpanContext(bool sampled_flag, bool is_remote) {}
+  bool IsValid() const { return false;}
 };
+} // namespace trace
+} // namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
 
 class jspan {
   jspan_context _ctx;
@@ -76,7 +125,7 @@ class jspan {
   void AddEvent(std::string_view) {}
   void AddEvent(std::string_view, std::initializer_list<std::pair<std::string_view, jspan_attribute>> fields) {}
   template <typename T> void AddEvent(std::string_view name, const T& fields = {}) {}
-  const jspan_context& GetContext() { return _ctx; }
+  jspan_context GetContext() const { return _ctx; }
   void UpdateName(std::string_view) {}
   bool IsRecording() { return false; }
 };
diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index 2cd418627be9..4a7ac3ea6e0a 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -1169,10 +1169,12 @@ inline namespace v14_2_0 {
     // compound object operations
     int operate(const std::string& oid, ObjectWriteOperation *op);
     int operate(const std::string& oid, ObjectWriteOperation *op, int flags);
+    int operate(const std::string& oid, ObjectWriteOperation *op, int flags, const jspan_context *trace_info);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl, int flags);
     int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op);
     int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags);
+    int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags, const jspan_context *trace_info);
     /**
      * Schedule an async write operation with explicit snapshot parameters
      *
diff --git a/src/include/rados/librados_fwd.hpp b/src/include/rados/librados_fwd.hpp
index 396f3a838757..d9a455adb38a 100644
--- a/src/include/rados/librados_fwd.hpp
+++ b/src/include/rados/librados_fwd.hpp
@@ -3,6 +3,18 @@
 
 struct blkin_trace_info;
 
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+
+class SpanContext;
+
+} // namespace trace
+} // inline namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
+
 namespace libradosstriper {
 
 class RadosStriper;
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index d66b56560f9c..b6be9050b1a9 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -637,7 +637,7 @@ int librados::IoCtxImpl::writesame(const object_t& oid, bufferlist& bl,
 }
 
 int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
-				 ceph::real_time *pmtime, int flags)
+				 ceph::real_time *pmtime, int flags, const jspan_context* otel_trace)
 {
   ceph::real_time ut = (pmtime ? *pmtime :
     ceph::real_clock::now());
@@ -664,7 +664,7 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
     oid, oloc,
     *o, snapc, ut,
     flags | extra_op_flags,
-    oncommit, &ver);
+    oncommit, &ver, osd_reqid_t(), nullptr, otel_trace);
   objecter->op_submit(objecter_op);
 
   {
@@ -753,7 +753,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
 				     ::ObjectOperation *o, AioCompletionImpl *c,
 				     const SnapContext& snap_context,
 				     const ceph::real_time *pmtime, int flags,
-                                     const blkin_trace_info *trace_info)
+                                     const blkin_trace_info *trace_info, const jspan_context *otel_trace)
 {
   FUNCTRACE(client->cct);
   OID_EVENT_TRACE(oid.name.c_str(), "RADOS_WRITE_OP_BEGIN");
@@ -779,7 +779,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
   trace.event("init root span");
   Objecter::Op *op = objecter->prepare_mutate_op(
     oid, oloc, *o, snap_context, ut, flags | extra_op_flags,
-    oncomplete, &c->objver, osd_reqid_t(), &trace);
+    oncomplete, &c->objver, osd_reqid_t(), &trace, otel_trace);
   objecter->op_submit(op, &c->tid);
   trace.event("rados operate op submitted");
 
diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index 477768ef78a4..23c402d7b5df 100644
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -154,12 +154,12 @@ struct librados::IoCtxImpl {
   int getxattrs(const object_t& oid, std::map<std::string, bufferlist>& attrset);
   int rmxattr(const object_t& oid, const char *name);
 
-  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0);
+  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0, const jspan_context *otel_trace = nullptr);
   int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
   int aio_operate(const object_t& oid, ::ObjectOperation *o,
 		  AioCompletionImpl *c, const SnapContext& snap_context,
 		  const ceph::real_time *pmtime, int flags,
-		  const blkin_trace_info *trace_info = nullptr);
+		  const blkin_trace_info *trace_info = nullptr, const jspan_context *otel_trace = nullptr);
   int aio_operate_read(const object_t& oid, ::ObjectOperation *o,
 		       AioCompletionImpl *c, int flags, bufferlist *pbl, const blkin_trace_info *trace_info = nullptr);
 
diff --git a/src/librados/librados_asio.h b/src/librados/librados_asio.h
index bd672d951f73..2eae1c268f6c 100644
--- a/src/librados/librados_asio.h
+++ b/src/librados/librados_asio.h
@@ -152,7 +152,7 @@ auto async_write(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectReadOperation *read_op, int flags,
-                   CompletionToken&& token)
+                   CompletionToken&& token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<bufferlist>;
   using Signature = typename Op::Signature;
@@ -176,7 +176,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectWriteOperation *write_op, int flags,
-                   CompletionToken &&token)
+                   CompletionToken &&token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<void>;
   using Signature = typename Op::Signature;
@@ -184,7 +184,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
   auto p = Op::create(ctx.get_executor(), init.completion_handler);
   auto& op = p->user_data;
 
-  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags);
+  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags, trace_ctx);
   if (ret < 0) {
     auto ec = boost::system::error_code{-ret, librados::detail::err_category()};
     ceph::async::post(std::move(p), ec);
diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc
index 926ddf86dab4..f9bc3b8fd044 100644
--- a/src/librados/librados_cxx.cc
+++ b/src/librados/librados_cxx.cc
@@ -1525,6 +1525,14 @@ int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperat
   return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags));
 }
 
+int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
+{
+  object_t obj(oid);
+  if (unlikely(!o->impl))
+    return -EINVAL;
+  return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags), otel_trace);
+}
+
 int librados::IoCtx::operate(const std::string& oid, librados::ObjectReadOperation *o, bufferlist *pbl)
 {
   object_t obj(oid);
@@ -1550,6 +1558,7 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
   return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
 				  io_ctx_impl->snapc, o->impl->prt, 0);
 }
+
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
 				 ObjectWriteOperation *o, int flags)
 {
@@ -1558,7 +1567,18 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
     return -EINVAL;
   return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
 				  io_ctx_impl->snapc, o->impl->prt,
-				  translate_flags(flags));
+				  translate_flags(flags), nullptr);
+}
+
+int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
+				 ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
+{
+  object_t obj(oid);
+  if (unlikely(!o->impl))
+    return -EINVAL;
+  return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
+				  io_ctx_impl->snapc, o->impl->prt,
+				  translate_flags(flags), nullptr, otel_trace);
 }
 
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 61908d802cd7..0153fdbb2723 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -36,7 +36,7 @@ namespace _mosdop {
 template<typename V>
 class MOSDOp final : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 8;
+  static constexpr int HEAD_VERSION = 9;
   static constexpr int COMPAT_VERSION = 3;
 
 private:
@@ -364,9 +364,38 @@ struct ceph_osd_request_head {
 
       encode(retry_attempt, payload);
       encode(features, payload);
-    } else {
-      // latest v8 encoding with hobject_t hash separate from pgid, no
+    } else if (!HAVE_FEATURE(features, SERVER_SQUID)) {
+      // v8 encoding with hobject_t hash separate from pgid, no
       // reassert version
+      header.version = 8;
+
+      encode(pgid, payload);
+      encode(hobj.get_hash(), payload);
+      encode(osdmap_epoch, payload);
+      encode(flags, payload);
+      encode(reqid, payload);
+      encode_trace(payload, features);
+
+      // -- above decoded up front; below decoded post-dispatch thread --
+
+      encode(client_inc, payload);
+      encode(mtime, payload);
+      encode(get_object_locator(), payload);
+      encode(hobj.oid, payload);
+
+      __u16 num_ops = ops.size();
+      encode(num_ops, payload);
+      for (unsigned i = 0; i < ops.size(); i++)
+	encode(ops[i].op, payload);
+
+      encode(hobj.snap, payload);
+      encode(snap_seq, payload);
+      encode(snaps, payload);
+
+      encode(retry_attempt, payload);
+      encode(features, payload);
+    } else {
+      // latest v9 opentelemetry trace
       header.version = HEAD_VERSION;
 
       encode(pgid, payload);
@@ -375,6 +404,7 @@ struct ceph_osd_request_head {
       encode(flags, payload);
       encode(reqid, payload);
       encode_trace(payload, features);
+      encode_otel_trace(payload, features);
 
       // -- above decoded up front; below decoded post-dispatch thread --
 
@@ -404,6 +434,16 @@ struct ceph_osd_request_head {
 
     // Always keep here the newest version of decoding order/rule
     if (header.version == HEAD_VERSION) {
+      decode(pgid, p);
+      uint32_t hash;
+      decode(hash, p);
+      hobj.set_hash(hash);
+      decode(osdmap_epoch, p);
+      decode(flags, p);
+      decode(reqid, p);
+      decode_trace(p);
+      decode_otel_trace(p);
+    } else if (header.version == 8) {
       decode(pgid, p);      // actual pgid
       uint32_t hash;
       decode(hash, p); // raw hash value
diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 70ac4ad13389..1faadb22a1cd 100644
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -1033,6 +1033,15 @@ void Message::decode_trace(ceph::bufferlist::const_iterator &p, bool create)
 #endif
 }
 
+void Message::encode_otel_trace(ceph::bufferlist &bl, uint64_t features) const
+{
+  tracing::encode(otel_trace, bl);
+}
+
+void Message::decode_otel_trace(ceph::bufferlist::const_iterator &p, bool create)
+{
+  tracing::decode(otel_trace, p);
+}
 
 // This routine is not used for ordinary messages, but only when encapsulating a message
 // for forwarding and routing.  It's also used in a backward compatibility test, which only
diff --git a/src/msg/Message.h b/src/msg/Message.h
index 40833744b67d..e56e4bb1b6d0 100644
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -32,6 +32,7 @@
 #include "common/ref.h"
 #include "common/debug.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "include/ceph_assert.h" // Because intrusive_ptr clobbers our assert...
 #include "include/buffer.h"
 #include "include/types.h"
@@ -282,6 +283,11 @@ class Message : public RefCountedObject {
   void encode_trace(ceph::buffer::list &bl, uint64_t features) const;
   void decode_trace(ceph::buffer::list::const_iterator &p, bool create = false);
 
+  // otel tracing
+  jspan_context otel_trace{false, false};
+  void encode_otel_trace(ceph::buffer::list &bl, uint64_t features) const;
+  void decode_otel_trace(ceph::buffer::list::const_iterator &p, bool create = false);
+
   class CompletionHook : public Context {
   protected:
     Message *m;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b0ff7883fabf..5913dd9fde08 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7566,7 +7566,12 @@ void OSD::ms_fast_dispatch(Message *m)
     tracepoint(osd, ms_fast_dispatch, reqid.name._type,
         reqid.name._num, reqid.tid, reqid.inc);
   }
-  op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+
+  if (m->otel_trace.IsValid()) {
+    op->osd_parent_span = tracing::osd::tracer.add_span("op-request-created", m->otel_trace);
+  } else {
+    op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+  }
 
   if (m->trace)
     op->osd_trace.init("osd op", &trace_endpoint, &m->trace);
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 969b486d8997..d881c6e1dc38 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -3239,6 +3239,10 @@ Objecter::MOSDOp *Objecter::_prepare_osd_op(Op *op)
     m->set_reqid(op->reqid);
   }
 
+  if (op->otel_trace && op->otel_trace->IsValid()) {
+     m->otel_trace = jspan_context(*op->otel_trace);
+  }
+
   logger->inc(l_osdc_op_send);
   ssize_t sum = 0;
   for (unsigned i = 0; i < m->ops.size(); i++) {
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 6daf57b928d3..68bd76268ae9 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -56,6 +56,7 @@
 #include "common/config_obs.h"
 #include "common/shunique_lock.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "common/Throttle.h"
 
 #include "mon/MonClient.h"
@@ -2057,6 +2058,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
     osd_reqid_t reqid; // explicitly setting reqid
     ZTracer::Trace trace;
+    const jspan_context* otel_trace = nullptr;
 
     static bool has_completion(decltype(onfinish)& f) {
       return std::visit([](auto&& arg) { return bool(arg);}, f);
@@ -2107,7 +2109,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
 
     Op(const object_t& o, const object_locator_t& ol, osdc_opvec&& _ops,
        int f, Context* fin, version_t *ov, int *offset = nullptr,
-       ZTracer::Trace *parent_trace = nullptr) :
+       ZTracer::Trace *parent_trace = nullptr, const jspan_context *otel_trace = nullptr) :
       target(o, ol, f),
       ops(std::move(_ops)),
       out_bl(ops.size(), nullptr),
@@ -2116,7 +2118,8 @@ class Objecter : public md_config_obs_t, public Dispatcher {
       out_ec(ops.size(), nullptr),
       onfinish(fin),
       objver(ov),
-      data_offset(offset) {
+      data_offset(offset),
+      otel_trace(otel_trace) {
       if (target.base_oloc.key == o)
 	target.base_oloc.key.clear();
       if (parent_trace && parent_trace->valid()) {
@@ -3041,10 +3044,11 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     ceph::real_time mtime, int flags,
     Context *oncommit, version_t *objver = NULL,
     osd_reqid_t reqid = osd_reqid_t(),
-    ZTracer::Trace *parent_trace = nullptr) {
+    ZTracer::Trace *parent_trace = nullptr,
+    const jspan_context *otel_trace = nullptr) {
     Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver,
-		   nullptr, parent_trace);
+		   nullptr, nullptr, otel_trace);
     o->priority = op.priority;
     o->mtime = mtime;
     o->snapc = snapc;
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index d7462587c87c..8a97e08248af 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -146,7 +146,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset)
   }
   constexpr uint64_t id = 0; // unused
   auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
-						     std::move(op), y),
+						     std::move(op), y, &trace),
 		    cost, id);
   return process_completed(c, &written);
 }
@@ -162,7 +162,7 @@ int RadosWriter::write_exclusive(const bufferlist& data)
 
   constexpr uint64_t id = 0; // unused
   auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
-						     std::move(op), y),
+						     std::move(op), y, &trace),
 		    cost, id);
   auto d = aio->drain();
   c.splice(c.end(), d);
@@ -383,7 +383,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
   read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
 
   r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
-                        flags & rgw::sal::FLAG_LOG_OP);
+                        writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -514,7 +514,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.modify_tail = true;
 
   r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
-                        flags & rgw::sal::FLAG_LOG_OP);
+                        writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0)
     return r;
 
@@ -753,7 +753,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
   }
   r = obj_op.write_meta(actual_size + cur_size,
 			accounted_size + *cur_accounted_size,
-			attrs, rctx, flags & rgw::sal::FLAG_LOG_OP);
+			attrs, rctx, writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index 53b14c41ee08..62be10870144 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -73,14 +73,15 @@ class RadosWriter : public rgw::sal::DataProcessor {
   RawObjSet written; // set of written objects for deletion
   const DoutPrefixProvider *dpp;
   optional_yield y;
+  jspan_context& trace;
 
  public:
   RadosWriter(Aio *aio, RGWRados *store,
               const RGWBucketInfo& bucket_info,
               RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
-              const DoutPrefixProvider *dpp, optional_yield y)
+              const DoutPrefixProvider *dpp, optional_yield y, jspan_context& _trace)
     : aio(aio), store(store), bucket_info(bucket_info),
-      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y)
+      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y), trace(_trace)
   {}
   ~RadosWriter();
 
@@ -102,6 +103,7 @@ class RadosWriter : public rgw::sal::DataProcessor {
   // so they aren't deleted on destruction
   void clear_written() { written.clear(); }
 
+  jspan_context& get_trace() { return trace; }
 };
 
 
@@ -132,12 +134,14 @@ class ManifestObjectProcessor : public HeadObjectProcessor,
                           const rgw_placement_rule *ptail_placement_rule,
                           const rgw_user& owner, RGWObjectCtx& _obj_ctx,
                           const rgw_obj& _head_obj,
-                          const DoutPrefixProvider* dpp, optional_yield y)
+                          const DoutPrefixProvider* dpp,
+                          optional_yield y,
+                          jspan_context& trace)
     : HeadObjectProcessor(0),
       store(store), bucket_info(bucket_info),
       owner(owner),
       obj_ctx(_obj_ctx), head_obj(_head_obj),
-      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y),
+      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y, trace),
       chunk(&writer, 0), stripe(&chunk, this, 0), dpp(dpp) {
         if (ptail_placement_rule) {
           tail_placement_rule = *ptail_placement_rule;
@@ -174,9 +178,9 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                         RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
                         std::optional<uint64_t> olh_epoch,
                         const std::string& unique_tag,
-                        const DoutPrefixProvider *dpp, optional_yield y)
+                        const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       olh_epoch(olh_epoch), unique_tag(unique_tag)
   {}
 
@@ -219,9 +223,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                            const rgw_obj& _head_obj,
                            const std::string& upload_id, uint64_t part_num,
                            const std::string& part_num_str,
-                           const DoutPrefixProvider *dpp, optional_yield y)
+                           const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       target_obj(head_obj), upload_id(upload_id),
       part_num(part_num), part_num_str(part_num_str),
       mp(head_obj.key.name, upload_id)
@@ -263,9 +267,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                           const rgw_obj& _head_obj,
                           const std::string& unique_tag, uint64_t position,
                           uint64_t *cur_accounted_size,
-                          const DoutPrefixProvider *dpp, optional_yield y)
+                          const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
             : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                                      owner, obj_ctx, _head_obj, dpp, y),
+                                      owner, obj_ctx, _head_obj, dpp, y, trace),
               position(position), cur_size(0), cur_accounted_size(cur_accounted_size),
               unique_tag(unique_tag), cur_manifest(nullptr)
     {}
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b802bb114bd7..09342951592a 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2897,6 +2897,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
 
   rgw_zone_id no_zone;
 
+  jspan_context no_trace{false, false};
+
   r = copy_obj(obj_ctx,
                user,
                NULL, /* req_info *info */
@@ -2925,7 +2927,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
                NULL, /* void (*progress_cb)(off_t, void *) */
                NULL, /* void *progress_data */
                dpp,
-               y);
+               y,
+               no_trace);
   if (r == -ECANCELED || r == -ENOENT) {
     /* Has already been overwritten, meaning another rgw process already
      * copied it out */
@@ -2991,6 +2994,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
     obj_ctx.set_atomic(archive_obj);
     obj_ctx.set_atomic(obj);
 
+    jspan_context no_trace{false, false};
+
     int ret = copy_obj(obj_ctx,
                        user,
                        nullptr,       /* req_info *info */
@@ -3019,7 +3024,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
                        nullptr,       /* void (*progress_cb)(off_t, void *) */
                        nullptr,       /* void *progress_data */
                        dpp,
-                       y);
+                       y,
+                       no_trace);
     if (ret == -ECANCELED || ret == -ENOENT) {
       /* Has already been overwritten, meaning another rgw process already
        * copied it out */
@@ -3049,7 +3055,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
                                            map<string, bufferlist>& attrs,
                                            bool assume_noent, bool modify_tail,
                                            void *_index_op, const req_context& rctx,
-                                           bool log_op)
+                                           jspan_context& trace, bool log_op)
 {
   RGWRados::Bucket::UpdateIndex *index_op = static_cast<RGWRados::Bucket::UpdateIndex *>(_index_op);
   RGWRados *store = target->get_store();
@@ -3224,7 +3230,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   auto& ioctx = ref.ioctx;
 
   tracepoint(rgw_rados, operate_enter, req_id.c_str());
-  r = rgw_rados_operate(rctx.dpp, ref.ioctx, ref.obj.oid, &op, rctx.y);
+  r = rgw_rados_operate(rctx.dpp, ref.ioctx, ref.obj.oid, &op, rctx.y, 0, &trace);
   tracepoint(rgw_rados, operate_exit, req_id.c_str());
   if (r < 0) { /* we can expect to get -ECANCELED if object was replaced under,
                 or -ENOENT if was removed, or -EEXIST if it did not exist
@@ -3340,7 +3346,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
 
 int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
                                         map<string, bufferlist>& attrs, const req_context& rctx,
-                                        bool log_op)
+                                        jspan_context& trace, bool log_op)
 {
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -3351,13 +3357,13 @@ int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
   bool assume_noent = (meta.if_match == NULL && meta.if_nomatch == NULL);
   int r;
   if (assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, trace, log_op);
     if (r == -EEXIST) {
       assume_noent = false;
     }
   }
   if (!assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, trace, log_op);
   }
   return r;
 }
@@ -4166,9 +4172,10 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   rgw::BlockingAioThrottle aio(cct->_conf->rgw_put_obj_min_window_size);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(&aio, this, dest_bucket_info, nullptr,
                                   user_id, obj_ctx, dest_obj, olh_epoch,
-				  tag, rctx.dpp, rctx.y);
+				  tag, rctx.dpp, rctx.y, no_trace);
   RGWRESTConn *conn;
   auto& zone_conn_map = svc.zone->get_zone_conn_map();
   auto& zonegroup_conn_map = svc.zone->get_zonegroup_conn_map();
@@ -4601,7 +4608,8 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y)
+               optional_yield y,
+               jspan_context& trace)
 {
   int ret;
   uint64_t obj_size;
@@ -4876,7 +4884,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   write_op.meta.delete_at = delete_at;
   write_op.meta.modify_tail = !copy_itself;
 
-  ret = write_op.write_meta(obj_size, astate->accounted_size, attrs, rctx);
+  ret = write_op.write_meta(obj_size, astate->accounted_size, attrs, rctx, trace);
   if (ret < 0) {
     goto done_ret;
   }
@@ -4946,9 +4954,10 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 
   auto aio = rgw::make_throttle(cct->_conf->rgw_put_obj_min_window_size, y);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(aio.get(), this, dest_bucket_info,
                                   &dest_placement, dest_bucket_info.owner,
-                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y);
+                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y, no_trace);
   int ret = processor.prepare(y);
   if (ret < 0)
     return ret;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 9245dc1512bd..264f5eb4f33a 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -823,10 +823,11 @@ class RGWRados
                      std::map<std::string, bufferlist>& attrs,
                      bool modify_tail, bool assume_noent,
                      void *index_op, const req_context& rctx,
+                     jspan_context& trace,
                      bool log_op = true);
       int write_meta(uint64_t size, uint64_t accounted_size,
                      std::map<std::string, bufferlist>& attrs,
-                     const req_context& rctx, bool log_op = true);
+                     const req_context& rctx, jspan_context& trace, bool log_op = true);
       int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive);
       const req_state* get_req_state() {
         return nullptr;  /* XXX dang Only used by LTTng, and it handles null anyway */
@@ -1199,7 +1200,8 @@ class RGWRados
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y);
+               optional_yield y,
+               jspan_context& trace);
 
   int copy_obj_data(RGWObjectCtx& obj_ctx,
                RGWBucketInfo& dest_bucket_info,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 299f46e3c9df..24aba37d430c 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1427,7 +1427,7 @@ std::unique_ptr<Writer> RadosStore::get_append_writer(const DoutPrefixProvider *
 				 this, std::move(aio), owner,
 				 ptail_placement_rule,
 				 unique_tag, position,
-				 cur_accounted_size);
+				 cur_accounted_size, obj->get_trace());
 }
 
 std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *dpp,
@@ -1445,7 +1445,7 @@ std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *
 				 bucket_info, obj_ctx, obj->get_obj(),
 				 this, std::move(aio), owner,
 				 ptail_placement_rule,
-				 olh_epoch, unique_tag);
+				 olh_epoch, unique_tag, obj->get_trace());
 }
 
 const std::string& RadosStore::get_compression_type(const rgw_placement_rule& rule)
@@ -1854,7 +1854,7 @@ int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp,
   attrs.erase(RGW_ATTR_TAIL_TAG);
 
   const req_context rctx{dpp, y, nullptr};
-  return obj_op.write_meta(0, 0, attrs, rctx);
+  return obj_op.write_meta(0, 0, attrs, rctx, head_obj->get_trace());
 }
 
 int RadosObject::get_max_chunk_size(const DoutPrefixProvider* dpp, rgw_placement_rule placement_rule, uint64_t* max_chunk_size, uint64_t* alignment)
@@ -2100,7 +2100,8 @@ int RadosObject::copy_object(User* user,
 				     progress_cb,
 				     progress_data,
 				     dpp,
-				     y);
+				     y,
+                                     dest_object->get_trace());
 }
 
 int RadosObject::RadosReadOp::iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y)
@@ -2304,7 +2305,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     encode(upload_info, bl);
     obj_op.meta.data = &bl;
 
-    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx, false);
+    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx, get_trace(), false);
   } while (ret == -EEXIST);
 
   return ret;
@@ -2588,7 +2589,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   obj_op.meta.olh_epoch = olh_epoch;
 
   const req_context rctx{dpp, y, nullptr};
-  ret = obj_op.write_meta(ofs, accounted_size, attrs, rctx);
+  ret = obj_op.write_meta(ofs, accounted_size, attrs, rctx, get_trace());
   if (ret < 0)
     return ret;
 
@@ -2686,7 +2687,7 @@ std::unique_ptr<Writer> RadosMultipartUpload::get_writer(
   return std::make_unique<RadosMultipartWriter>(dpp, y, get_upload_id(),
 				 bucket_info, obj_ctx,
 				 obj->get_obj(), store, std::move(aio), owner,
-				 ptail_placement_rule, part_num, part_num_str);
+				 ptail_placement_rule, part_num, part_num_str, obj->get_trace());
 }
 
 MPRadosSerializer::MPRadosSerializer(const DoutPrefixProvider *dpp, RadosStore* store, RadosObject* obj, const std::string& lock_name) :
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index d4bd19a55fe1..a4f4809a2f05 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -739,7 +739,8 @@ class RadosAtomicWriter : public StoreWriter {
 		    const rgw_user& owner,
 		    const rgw_placement_rule *ptail_placement_rule,
 		    uint64_t olh_epoch,
-		    const std::string& unique_tag) :
+		    const std::string& unique_tag,
+                    jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -747,7 +748,7 @@ class RadosAtomicWriter : public StoreWriter {
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, olh_epoch, unique_tag,
-				  dpp, y)
+				  dpp, y, trace)
   {}
   ~RadosAtomicWriter() = default;
 
@@ -787,7 +788,8 @@ class RadosAppendWriter : public StoreWriter {
 		    const rgw_placement_rule *ptail_placement_rule,
 		    const std::string& unique_tag,
 		    uint64_t position,
-		    uint64_t *cur_accounted_size) :
+		    uint64_t *cur_accounted_size,
+                    jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -795,7 +797,7 @@ class RadosAppendWriter : public StoreWriter {
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, unique_tag, position,
-				  cur_accounted_size, dpp, y)
+				  cur_accounted_size, dpp, y, trace)
   {}
   ~RadosAppendWriter() = default;
 
@@ -833,7 +835,7 @@ class RadosMultipartWriter : public StoreWriter {
 		       RadosStore* _store, std::unique_ptr<Aio> _aio,
 		       const rgw_user& owner,
 		       const rgw_placement_rule *ptail_placement_rule,
-		       uint64_t part_num, const std::string& part_num_str) :
+		       uint64_t part_num, const std::string& part_num_str, jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -841,7 +843,7 @@ class RadosMultipartWriter : public StoreWriter {
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, upload_id,
-				  part_num, part_num_str, dpp, y)
+				  part_num, part_num_str, dpp, y, trace)
   {}
   ~RadosMultipartWriter() = default;
 
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index e4bd34aa868f..c143875538aa 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -198,7 +198,7 @@ int rgw_delete_system_obj(const DoutPrefixProvider *dpp,
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags)
+                      optional_yield y, int flags, const jspan_context* trace_info)
 {
   // given a yield_context, call async_operate() to yield the coroutine instead
   // of blocking
@@ -225,13 +225,13 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-		      int flags)
+		      int flags, const jspan_context* trace_info)
 {
   if (y) {
     auto& context = y.get_io_context();
     auto& yield = y.get_yield_context();
     boost::system::error_code ec;
-    librados::async_operate(context, ioctx, oid, op, flags, yield[ec]);
+    librados::async_operate(context, ioctx, oid, op, flags, yield[ec], trace_info);
     return -ec.value();
   }
   if (is_asio_thread) {
@@ -240,7 +240,7 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
     ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl;
 #endif
   }
-  return ioctx.operate(oid, op, flags);
+  return ioctx.operate(oid, op, flags, trace_info);
 }
 
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 27bc6f0c4daa..aa365deb42aa 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -97,10 +97,10 @@ extern thread_local bool is_asio_thread;
 /// perform the rados operation, using the yield context when given
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags = 0);
+                      optional_yield y, int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-		      int flags = 0);
+		      int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                      bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl,
                      optional_yield y);
diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index c70acae79e95..1bc50c89229c 100644
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -50,15 +50,16 @@ void cb(librados::completion_t, void* arg) {
 }
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op) {
-  return [ctx = std::move(ctx), op = std::move(op)] (Aio* aio, AioResult& r) mutable {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), trace_ctx] (Aio* aio, AioResult& r) mutable {
       constexpr bool read = std::is_same_v<std::decay_t<Op>, librados::ObjectReadOperation>;
       // use placement new to construct the rados state inside of user_data
       auto s = new (&r.user_data) state(aio, ctx, r);
       if constexpr (read) {
+        (void)trace_ctx; // suppress unused trace_ctx warning. until we will support the read op trace
         r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data);
       } else {
-        r.result = ctx.aio_operate(r.obj.oid, s->c, &op);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, trace_ctx);
       }
       if (r.result < 0) {
         // cb() won't be called, so release everything here
@@ -89,8 +90,8 @@ struct Handler {
 template <typename Op>
 Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
                          boost::asio::io_context& context,
-                         spawn::yield_context yield) {
-  return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
+                         spawn::yield_context yield, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), &context, yield, trace_ctx] (Aio* aio, AioResult& r) mutable {
       // arrange for the completion Handler to run on the yield_context's strand
       // executor so it can safely call back into Aio without locking
       using namespace boost::asio;
@@ -98,7 +99,7 @@ Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
       auto ex = get_associated_executor(init.completion_handler);
 
       librados::async_operate(context, ctx, r.obj.oid, &op, 0,
-                              bind_executor(ex, Handler{aio, ctx, r}));
+                              bind_executor(ex, Handler{aio, ctx, r}), trace_ctx);
     };
 }
 
@@ -115,15 +116,15 @@ Aio::OpFunc d3n_cache_aio_abstract(const DoutPrefixProvider *dpp, optional_yield
 
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y) {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y, jspan_context *trace_ctx = nullptr) {
   static_assert(std::is_base_of_v<librados::ObjectOperation, std::decay_t<Op>>);
   static_assert(!std::is_lvalue_reference_v<Op>);
   static_assert(!std::is_const_v<Op>);
   if (y) {
     return aio_abstract(std::move(ctx), std::forward<Op>(op),
-                        y.get_io_context(), y.get_yield_context());
+                        y.get_io_context(), y.get_yield_context(), trace_ctx);
   }
-  return aio_abstract(std::move(ctx), std::forward<Op>(op));
+  return aio_abstract(std::move(ctx), std::forward<Op>(op), null_yield, trace_ctx);
 }
 
 } // anonymous namespace
@@ -135,8 +136,8 @@ Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
 }
 Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
                              librados::ObjectWriteOperation&& op,
-                             optional_yield y) {
-  return aio_abstract(std::move(ctx), std::move(op), y);
+                             optional_yield y, jspan_context *trace_ctx) {
+  return aio_abstract(std::move(ctx), std::move(op), y, trace_ctx);
 }
 
 Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
diff --git a/src/rgw/rgw_aio.h b/src/rgw/rgw_aio.h
index 0070346327b7..9be144f607f2 100644
--- a/src/rgw/rgw_aio.h
+++ b/src/rgw/rgw_aio.h
@@ -96,7 +96,7 @@ class Aio {
                             optional_yield y);
   static OpFunc librados_op(librados::IoCtx ctx,
                             librados::ObjectWriteOperation&& op,
-                            optional_yield y);
+                            optional_yield y, jspan_context *trace_ctx = nullptr);
   static OpFunc d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
                              off_t read_ofs, off_t read_len, std::string& location);
 };
diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index 3168abadb0b4..5a8fc14a8047 100644
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -218,6 +218,8 @@ namespace rgw {
       goto done;
     }
 
+    s->trace = tracing::rgw::tracer.start_trace(op->name());
+
     /* req is-a RGWOp, currently initialized separately */
     ret = req->op_init();
     if (ret < 0) {
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index f6011dbe5c9d..b7e50e3421f7 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4230,6 +4230,8 @@ void RGWPutObj::execute(optional_yield y)
 
   rgw_placement_rule *pdest_placement = &s->dest_placement;
 
+  s->object->set_trace(s->trace->GetContext());
+
   if (multipart) {
     std::unique_ptr<rgw::sal::MultipartUpload> upload;
     upload = s->bucket->get_multipart_upload(s->object->get_name(),
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 6ee02be0b67f..278ed1965bb4 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1056,6 +1056,9 @@ class Object {
     /** Get a unique copy of this object */
     virtual std::unique_ptr<Object> clone() = 0;
 
+    virtual jspan_context& get_trace() = 0;
+    virtual void set_trace (jspan_context&& _trace_ctx) = 0;
+
     /* dang - This is temporary, until the API is completed */
     /** Get the key for this object */
     virtual rgw_obj_key& get_key() = 0;
@@ -1135,7 +1138,7 @@ class MultipartUpload {
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() = 0;
 
   /** Get the trace context of this upload */
-  virtual const jspan_context& get_trace() = 0;
+  virtual jspan_context& get_trace() = 0;
 
   /** Get the Object that represents this upload */
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() = 0;
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 80e236a25d94..b5c4c4dfc681 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -649,6 +649,9 @@ class FilterObject : public Object {
     return std::make_unique<FilterObject>(*this);
   }
 
+  virtual jspan_context& get_trace() { return next->get_trace(); }
+  virtual void set_trace (jspan_context&& _trace_ctx) { next->set_trace(std::move(_trace_ctx)); }
+
   virtual void print(std::ostream& out) const override { return next->print(out); }
 
   /* Internal to Filters */
@@ -688,7 +691,7 @@ class FilterMultipartUpload : public MultipartUpload {
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return next->get_trace(); }
+  virtual jspan_context& get_trace() override { return next->get_trace(); }
 
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override;
 
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index b1592413d782..7c35258dd5dc 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -145,6 +145,7 @@ class StoreObject : public Object {
     RGWObjState state;
     Bucket* bucket = nullptr;
     bool delete_marker{false};
+    jspan_context trace_ctx{false, false};
 
   public:
     StoreObject() = default;
@@ -217,6 +218,8 @@ class StoreObject : public Object {
        * work with lifecycle */
       return -1;
     }
+    jspan_context& get_trace() override { return trace_ctx; }
+    void set_trace (jspan_context&& _trace_ctx) override { trace_ctx = std::move(_trace_ctx); }
 
     virtual int get_torrent_info(const DoutPrefixProvider* dpp,
                                  optional_yield y, bufferlist& bl) override {
@@ -254,7 +257,7 @@ class StoreMultipartUpload : public MultipartUpload {
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return trace_ctx; }
+  virtual jspan_context& get_trace() override { return trace_ctx; }
 
   virtual void print(std::ostream& out) const override {
     out << get_meta();

From fe15b52edb5d228d2ed56679c62cf48493ae2d54 Mon Sep 17 00:00:00 2001
From: Tobias Urdin <tobias.urdin@binero.se>
Date: Tue, 6 Feb 2024 07:50:55 +0000
Subject: [PATCH 1786/2492] rgw/auth: ignoring signatures for HTTP OPTIONS
 calls

Before [1] we always sent all HTTP OPTIONS requests to
the S3AnonymousEngine and ignored any provided AWSv4
credentials sent in the request.

That PR changed so that if we got credentials in the
request we instead sent it through the authentication
code in order to solve HTTP OPTIONS requests on tenanted
users to start working (because we need to resolve the
tenant, also called bucket tenant in the code, and we can't
only rely on the bucket name since it will not be found).

We solved this by modifying the canonical HTTP method used
when calculating the AWSv4 signature by instead using the
access-control-request-method header which worked good.

This change did not take into account that when you generated
a presigned URL for a put_object request you can also pass in
extra parameters like a canned ACL [2] to the Params variable
in for example boto3's generated_presigned_url().

Doing that will cause the client to add the x-amz-acl header
to x-amz-signedheaders and also use that in their signature
calculation.

When doing a HTTP OPTIONS calls for CORS on that presigned URL
the browser will never send a x-amz-acl header with the correct
data since that is something that the actual PUT request should
include later, so that HTTP OPTIONS call should pass even though
the signature can never be calculated correctly server-side like
verified against AWS S3 in tracker [3].

This patch as a result skips the signature calculation when doing
EC2 auth using the LocalEngine but we still need to pass the request
there in order to lookup the user to support buckets in a tenant.

For the Keystone EC2 auth we're pretty out of luck in the sense that
Keystone's API itself requires us to send the AWSv4 signature in the
request with the access_key in order to obtain a token, and we cannot
leave the signature out, we also cannot spoof the signature from
rgw -> keystone since we don't have access to the secret_key if it's
not in our cache.

For that approach we simply pass on to get_access_token() that if it's
an HTTP OPTIONS and we find the access_key in the cache we pull that
and ignore verifying signature and pass it on for validation. This means
that the cache must be warm if using Keystone auth and adding extra
params to a presigned URL.

This partly makes some of the commits in [1] redundant for EC2
LocalEngine auth but we still need it for tenanted bucket support.

[1] https://github.com/ceph/ceph/pull/52673
[2] https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
[3] https://tracker.ceph.com/issues/64308

Fixes: https://tracker.ceph.com/issues/64308
Signed-off-by: Tobias Urdin <tobias.urdin@binero.se>
---
 src/rgw/rgw_auth_keystone.cc | 22 ++++++++++++++++------
 src/rgw/rgw_auth_keystone.h  |  1 +
 src/rgw/rgw_rest_s3.cc       |  7 +++++++
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc
index 552159823bb4..e088578a9c3d 100644
--- a/src/rgw/rgw_auth_keystone.cc
+++ b/src/rgw/rgw_auth_keystone.cc
@@ -570,6 +570,7 @@ auto EC2Engine::get_access_token(const DoutPrefixProvider* dpp,
                                  const std::string& string_to_sign,
                                  const std::string_view& signature,
                                  const signature_factory_t& signature_factory,
+                                 bool ignore_signature,
                                  optional_yield y) const
     -> access_token_result
 {
@@ -584,12 +585,19 @@ auto EC2Engine::get_access_token(const DoutPrefixProvider* dpp,
 
   /* Check that credentials can correctly be used to sign data */
   if (t) {
-    std::string sig(signature);
-    server_signature_t server_signature = signature_factory(cct, t->get<1>(), string_to_sign);
-    if (sig.compare(server_signature) == 0) {
+    /* We should ignore checking signature in cache if caller tells us to which
+     * means we're handling a HTTP OPTIONS call. */
+    if (ignore_signature) {
+      ldpp_dout(dpp, 20) << "ignore_signature set and found in cache" << dendl;
       return {t->get<0>(), t->get<1>(), 0};
     } else {
-      ldpp_dout(dpp, 0) << "Secret string does not correctly sign payload, cache miss" << dendl;
+      std::string sig(signature);
+      server_signature_t server_signature = signature_factory(cct, t->get<1>(), string_to_sign);
+      if (sig.compare(server_signature) == 0) {
+        return {t->get<0>(), t->get<1>(), 0};
+      } else {
+        ldpp_dout(dpp, 0) << "Secret string does not correctly sign payload, cache miss" << dendl;
+      }
     }
   } else {
     ldpp_dout(dpp, 0) << "No stored secret string, cache miss" << dendl;
@@ -662,7 +670,6 @@ rgw::auth::Engine::result_t EC2Engine::authenticate(
   const string_to_sign_t& string_to_sign,
   const signature_factory_t& signature_factory,
   const completer_factory_t& completer_factory,
-  /* Passthrough only! */
   const req_state* s,
   optional_yield y) const
 {
@@ -681,9 +688,12 @@ rgw::auth::Engine::result_t EC2Engine::authenticate(
     std::vector<std::string> admin;
   } accepted_roles(cct);
 
+  /* When we handle a HTTP OPTIONS call we must ignore the signature */
+  bool ignore_signature = (s->op_type == RGW_OP_OPTIONS_CORS);
+
   auto [t, secret_key, failure_reason] =
     get_access_token(dpp, access_key_id, string_to_sign,
-                     signature, signature_factory, y);
+                     signature, signature_factory, ignore_signature, y);
   if (! t) {
     if (failure_reason == -ERR_SIGNATURE_NO_MATCH) {
       // we looked up a secret but it didn't generate the same signature as
diff --git a/src/rgw/rgw_auth_keystone.h b/src/rgw/rgw_auth_keystone.h
index c6852d639783..de8f98c6c5a6 100644
--- a/src/rgw/rgw_auth_keystone.h
+++ b/src/rgw/rgw_auth_keystone.h
@@ -164,6 +164,7 @@ class EC2Engine : public rgw::auth::s3::AWSEngine {
                    const std::string& string_to_sign,
                    const std::string_view& signature,
 		   const signature_factory_t& signature_factory,
+                   bool ignore_signature,
                    optional_yield y) const;
   result_t authenticate(const DoutPrefixProvider* dpp,
                         const std::string_view& access_key_id,
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 2117af94d70d..32b0e60d5d68 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -6213,6 +6213,13 @@ rgw::auth::s3::LocalEngine::authenticate(
   }
   const RGWAccessKey& k = iter->second;
 
+  /* Ignore signature for HTTP OPTIONS */
+  if (s->op_type == RGW_OP_OPTIONS_CORS) {
+    auto apl = apl_factory->create_apl_local(cct, s, user->get_info(),
+                                             k.subuser, std::nullopt, access_key_id);
+    return result_t::grant(std::move(apl), completer_factory(k.key));
+  }
+
   const VersionAbstractor::server_signature_t server_signature = \
     signature_factory(cct, k.key, string_to_sign);
   auto compare = signature.compare(server_signature);

From fb802ca7248159945fc2517f5b5674f87c2d8503 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 4 Feb 2024 09:17:11 +0000
Subject: [PATCH 1787/2492] seastar: bump up seastar submodule version

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/seastar | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/seastar b/src/seastar
index 50c6790df3db..f3a1484c83b8 160000
--- a/src/seastar
+++ b/src/seastar
@@ -1 +1 @@
-Subproject commit 50c6790df3db6ab38b5fea7e03494828dc2aafdc
+Subproject commit f3a1484c83b8b8ed774582f85fe33bf957390400

From 42ead863ecab39bbb6b01ef5ea9cf4b3b4e7cef6 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 4 Feb 2024 10:34:56 +0000
Subject: [PATCH 1788/2492] CMakeLists.txt: fix Seastar expected dirs

```
  # create the directory so cmake won't complain when looking at the imported
  # target: Seastar exports this directory created at build-time
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index afa2442d5709..d266eb72598b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -392,6 +392,7 @@ if(WITH_SEASTAR)
   # create the directory so cmake won't complain when looking at the imported
   # target: Seastar exports this directory created at build-time
   file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/seastar/gen/include")
+  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/seastar/gen/src")
   add_subdirectory(crimson)
 endif()
 

From 40c1978b690641636213baa2ea98000fe0e9e4dc Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 4 Feb 2024 10:36:22 +0000
Subject: [PATCH 1789/2492] crimson/crush/CrushLocation: fix compilation errors

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/crush/CrushLocation.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/crush/CrushLocation.cc b/src/crimson/crush/CrushLocation.cc
index d45264000bee..0d66e5587e88 100644
--- a/src/crimson/crush/CrushLocation.cc
+++ b/src/crimson/crush/CrushLocation.cc
@@ -92,8 +92,8 @@ seastar::future<> CrushLocation::update_from_hook()
         local_conf().get_val<std::string>("crush_location_hook"),
         std::move(params)
       ).then([this] (auto process) {
-        auto stdout = process.stdout();
-        return do_with(
+        auto stdout = process.cout();
+        return seastar::do_with(
           std::move(process),
           std::move(stdout),
           [this](auto& process, auto& stdout)

From d49990f246462f07832f010b9ad61943bb864459 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 5 Feb 2024 09:35:24 +0000
Subject: [PATCH 1790/2492] ceph.spec.in: crimson add protobuf

Following https://github.com/scylladb/seastar/commit/eea4102091bb4d689e0b5a769512966b716ee8a2

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 ceph.spec.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 5318b3865d1a..94c0464515cf 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -346,6 +346,8 @@ BuildRequires:  ragel
 BuildRequires:  systemtap-sdt-devel
 BuildRequires:  libubsan
 BuildRequires:  libasan
+BuildRequires:  protobuf-devel
+BuildRequires:  protobuf-compiler
 %if 0%{?rhel} == 8
 BuildRequires:  %{gts_prefix}-annobin
 BuildRequires:  %{gts_prefix}-annobin-plugin-gcc

From 6e453e2dc2cb4254bf4919cf87da5683173ca4e7 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 6 Feb 2024 17:52:56 +0000
Subject: [PATCH 1791/2492] fixup: ECRcoveryBackend don't goes to disk as well

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 16 +++++++++++-----
 src/osd/ECBackend.h  |  9 ++++++---
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index e455c5062b74..13d1317f272c 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -130,7 +130,7 @@ ECBackend::ECBackend(
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
-    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent()),
+    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width),
     unstable_hashinfo_registry(cct, ec_impl) {
@@ -150,14 +150,16 @@ ECBackend::RecoveryBackend::RecoveryBackend(
   const ECUtil::stripe_info_t& sinfo,
   ReadPipeline& read_pipeline,
   UnstableHashInfoRegistry& unstable_hashinfo_registry,
-  ECListener* parent)
+  ECListener* parent,
+  ECBackend* ecbackend)
   : cct(cct),
     coll(coll),
     ec_impl(std::move(ec_impl)),
     sinfo(sinfo),
     read_pipeline(read_pipeline),
     unstable_hashinfo_registry(unstable_hashinfo_registry),
-    parent(parent) {
+    parent(parent),
+    ecbackend(ecbackend) {
 }
 
 PGBackend::RecoveryHandle *ECBackend::RecoveryBackend::open_recovery_op()
@@ -575,8 +577,12 @@ void ECBackend::RecoveryBackend::continue_recovery_op(
       uint64_t amount = get_recovery_chunk_size();
 
       if (op.recovery_progress.first && op.obc) {
-	/* We've got the attrs and the hinfo, might as well use them */
-	op.hinfo = unstable_hashinfo_registry.get_hash_info(op.hoid, false, op.obc->attr_cache, op.obc->obs.oi.size);
+        if (auto [r, attrs, size] = ecbackend->get_attrs_n_size_from_disk(op.hoid);
+	    r >= 0 || r == -ENOENT) {
+          op.hinfo = unstable_hashinfo_registry.get_hash_info(op.hoid, false, attrs, size);
+        } else {
+          derr << __func__ << ": can't stat-or-getattr on " << op.hoid << dendl;
+	}
 	if (!op.hinfo) {
           derr << __func__ << ": " << op.hoid << " has inconsistent hinfo"
                << dendl;
diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index db003f0adcb9..e61ec6a0cc5c 100644
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -198,6 +198,7 @@ class ECBackend : public PGBackend, public ECCommon {
     UnstableHashInfoRegistry& unstable_hashinfo_registry;
     // TODO: lay an interface down here
     ECListener* parent;
+    ECBackend* ecbackend;
 
     ECListener *get_parent() const { return parent; }
     const OSDMapRef& get_osdmap() const { return get_parent()->pgb_get_osdmap(); }
@@ -212,7 +213,8 @@ class ECBackend : public PGBackend, public ECCommon {
 		    const ECUtil::stripe_info_t& sinfo,
 		    ReadPipeline& read_pipeline,
 		    UnstableHashInfoRegistry& unstable_hashinfo_registry,
-		    ECListener* parent);
+		    ECListener* parent,
+		    ECBackend* ecbackend);
   struct RecoveryOp {
     hobject_t hoid;
     eversion_t v;
@@ -307,8 +309,9 @@ class ECBackend : public PGBackend, public ECCommon {
 		      const ECUtil::stripe_info_t& sinfo,
 		      ReadPipeline& read_pipeline,
 		      UnstableHashInfoRegistry& unstable_hashinfo_registry,
-		      Listener* parent)
-      : RecoveryBackend(cct, coll, std::move(ec_impl), sinfo, read_pipeline, unstable_hashinfo_registry, parent->get_eclistener()),
+		      Listener* parent,
+		      ECBackend* ecbackend)
+      : RecoveryBackend(cct, coll, std::move(ec_impl), sinfo, read_pipeline, unstable_hashinfo_registry, parent->get_eclistener(), ecbackend),
 	parent(parent) {
     }
 

From e5a85982f08532db11ea912bac1f2eaaff01ab24 Mon Sep 17 00:00:00 2001
From: Shreyansh Sancheti <ssanchet@redhat.com>
Date: Mon, 29 Jan 2024 13:03:07 -0500
Subject: [PATCH 1792/2492] RGW: Getting an RGW service Segfault when assigning
 an attribute to an IAM role

The current implementation of RGWTagRole, which inherits RGWRestRole::verify_permission() from its base class, encounters a critical issue when loading RGWRole from storage and initializing the RGWRestRole::_role member variable.To address this issue and ensure that errors in initialization are appropriately handled, it is proposed to separate the initialization logic from the permission-checking logic.

Fixes: https://tracker.ceph.com/issues/64232
Signed-off-by: Shreyansh Sancheti <ssanchet@redhat.com>
---
 src/rgw/rgw_rest_role.cc | 44 ++++++++++++++++++++++++++++------------
 src/rgw/rgw_rest_role.h  |  4 ++++
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc
index fd537c0c9946..6132b1111787 100644
--- a/src/rgw/rgw_rest_role.cc
+++ b/src/rgw/rgw_rest_role.cc
@@ -80,23 +80,13 @@ int RGWRestRole::verify_permission(optional_yield y)
   if (s->auth.identity->is_anonymous()) {
     return -EACCES;
   }
-
+  
   string role_name = s->info.args.get("RoleName");
-  std::unique_ptr<rgw::sal::RGWRole> role = driver->get_role(role_name,
-							    s->user->get_tenant());
-  if (op_ret = role->get(s, y); op_ret < 0) {
-    if (op_ret == -ENOENT) {
-      op_ret = -ERR_NO_ROLE_FOUND;
-    }
-    return op_ret;
-  }
-
   if (int ret = check_caps(s->user->get_caps()); ret == 0) {
-    _role = std::move(role);
     return ret;
   }
 
-  string resource_name = role->get_path() + role_name;
+  string resource_name = _role->get_path() + role_name;
   uint64_t op = get_op();
   if (!verify_user_permission(this,
                               s,
@@ -107,8 +97,21 @@ int RGWRestRole::verify_permission(optional_yield y)
     return -EACCES;
   }
 
-  _role = std::move(role);
+  return 0;
+}
 
+int RGWRestRole::init_processing(optional_yield y)
+{
+  string role_name = s->info.args.get("RoleName");
+  std::unique_ptr<rgw::sal::RGWRole> role = driver->get_role(role_name,
+                                                             s->user->get_tenant());
+  if (int ret = role->get(s, y); ret < 0) {
+    if (ret == -ENOENT) {
+      return -ERR_NO_ROLE_FOUND;
+    }
+    return ret;
+  }
+  _role = std::move(role);
   return 0;
 }
 
@@ -202,6 +205,11 @@ int RGWCreateRole::verify_permission(optional_yield y)
   return 0;
 }
 
+int RGWCreateRole::init_processing(optional_yield y)
+{
+  return 0; // avoid calling RGWRestRole::init_processing()
+}
+
 int RGWCreateRole::get_params()
 {
   role_name = s->info.args.get("RoleName");
@@ -437,6 +445,11 @@ int RGWGetRole::_verify_permission(const rgw::sal::RGWRole* role)
   return 0;
 }
 
+int RGWGetRole::init_processing(optional_yield y)
+{
+  return 0; // avoid calling RGWRestRole::init_processing()
+}
+
 int RGWGetRole::get_params()
 {
   role_name = s->info.args.get("RoleName");
@@ -558,6 +571,11 @@ int RGWListRoles::verify_permission(optional_yield y)
   return 0;
 }
 
+int RGWListRoles::init_processing(optional_yield y)
+{
+  return 0; // avoid calling RGWRestRole::init_processing()
+}
+
 int RGWListRoles::get_params()
 {
   path_prefix = s->info.args.get("PathPrefix");
diff --git a/src/rgw/rgw_rest_role.h b/src/rgw/rgw_rest_role.h
index 98a08833bf72..a93c418001f9 100644
--- a/src/rgw/rgw_rest_role.h
+++ b/src/rgw/rgw_rest_role.h
@@ -21,6 +21,7 @@ class RGWRestRole : public RGWRESTOp {
   std::vector<std::string> tagKeys;
   std::unique_ptr<rgw::sal::RGWRole> _role;
   int verify_permission(optional_yield y) override;
+  int init_processing(optional_yield y) override;
   void send_response() override;
   virtual uint64_t get_op() = 0;
   int parse_tags();
@@ -43,6 +44,7 @@ class RGWCreateRole : public RGWRoleWrite {
 public:
   RGWCreateRole(const bufferlist& bl_post_body) : bl_post_body(bl_post_body) {};
   int verify_permission(optional_yield y) override;
+  int init_processing(optional_yield y) override;
   void execute(optional_yield y) override;
   int get_params();
   const char* name() const override { return "create_role"; }
@@ -66,6 +68,7 @@ class RGWGetRole : public RGWRoleRead {
 public:
   RGWGetRole() = default;
   int verify_permission(optional_yield y) override;
+  int init_processing(optional_yield y) override; 
   void execute(optional_yield y) override;
   int get_params();
   const char* name() const override { return "get_role"; }
@@ -88,6 +91,7 @@ class RGWListRoles : public RGWRoleRead {
 public:
   RGWListRoles() = default;
   int verify_permission(optional_yield y) override;
+  int init_processing(optional_yield y) override;
   void execute(optional_yield y) override;
   int get_params();
   const char* name() const override { return "list_roles"; }

From 65e671472088fcd63bb8870c8d8bd2e947d1887c Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Thu, 25 Jan 2024 19:22:35 +0530
Subject: [PATCH 1793/2492] mgr/dashboards: add generated json files

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../dashboards_out/radosgw-detail.json        |  8 +++----
 .../dashboards_out/radosgw-overview.json      | 14 +++++------
 .../features/radosgw-detail.feature           | 12 +++++-----
 .../features/radosgw_overview.feature         | 24 +++++++++----------
 4 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
index 18ba369460a3..e7c6f91e226b 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-detail.json
@@ -114,14 +114,14 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "sum by (instance_id) (\n  rate(ceph_rgw_get_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_get_initial_lat_count{job=~\"$job\"}[$__rate_interval])\n) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
+               "expr": "sum by (instance_id) (\n  rate(ceph_rgw_op_get_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_get_obj_lat_count{job=~\"$job\"}[$__rate_interval])\n) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "GET {{ceph_daemon}}",
                "refId": "A"
             },
             {
-               "expr": "sum by (instance_id) (\n  rate(ceph_rgw_put_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_put_initial_lat_count{job=~\"$job\"}[$__rate_interval])\n) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
+               "expr": "sum by (instance_id) (\n  rate(ceph_rgw_op_put_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_put_obj_lat_count{job=~\"$job\"}[$__rate_interval])\n) * on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "PUT {{ceph_daemon}}",
@@ -216,14 +216,14 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "rate(ceph_rgw_get_b{job=~\"$job\"}[$__rate_interval]) *\n  on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
+               "expr": "rate(ceph_rgw_op_get_obj_bytes{job=~\"$job\"}[$__rate_interval]) *\n  on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "GETs {{ceph_daemon}}",
                "refId": "A"
             },
             {
-               "expr": "rate(ceph_rgw_put_b{job=~\"$job\"}[$__rate_interval]) *\n  on (instance_id) group_left (ceph_daemon)\n  ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
+               "expr": "rate(ceph_rgw_op_put_obj_bytes{job=~\"$job\"}[$__rate_interval]) *\n  on (instance_id) group_left (ceph_daemon)\n  ceph_rgw_metadata{job=~\"$job\", ceph_daemon=~\"$rgw_servers\"}\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "PUTs {{ceph_daemon}}",
diff --git a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
index 92e416764e7c..3cb8a7c76acc 100644
--- a/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/radosgw-overview.json
@@ -108,14 +108,14 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "label_replace(\n  rate(ceph_rgw_get_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_get_initial_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
+               "expr": "label_replace(\n  rate(ceph_rgw_op_get_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_get_obj_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "GET {{rgw_host}}",
                "refId": "A"
             },
             {
-               "expr": "label_replace(\n  rate(ceph_rgw_put_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_put_initial_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
+               "expr": "label_replace(\n  rate(ceph_rgw_op_put_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_put_obj_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "PUT {{rgw_host}}",
@@ -305,7 +305,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "label_replace(\n  rate(ceph_rgw_get_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_get_initial_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
+               "expr": "label_replace(\n  rate(ceph_rgw_op_get_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_get_obj_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "{{rgw_host}}",
@@ -400,14 +400,14 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "sum(rate(ceph_rgw_get_b{job=~\"$job\"}[$__rate_interval]))",
+               "expr": "sum(rate(ceph_rgw_op_get_obj_bytes{job=~\"$job\"}[$__rate_interval]))",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "GETs",
                "refId": "A"
             },
             {
-               "expr": "sum(rate(ceph_rgw_put_b{job=~\"$job\"}[$__rate_interval]))",
+               "expr": "sum(rate(ceph_rgw_op_put_obj_bytes{job=~\"$job\"}[$__rate_interval]))",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "PUTs",
@@ -502,7 +502,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "label_replace(sum by (instance_id) (\n  rate(ceph_rgw_get_b{job=~\"$job\"}[$__rate_interval]) +\n    rate(ceph_rgw_put_b{job=~\"$job\"}[$__rate_interval])) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
+               "expr": "label_replace(sum by (instance_id) (\n  rate(ceph_rgw_op_get_obj_bytes{job=~\"$job\"}[$__rate_interval]) +\n    rate(ceph_rgw_op_put_obj_bytes{job=~\"$job\"}[$__rate_interval])) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "{{rgw_host}}",
@@ -597,7 +597,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "label_replace(\n  rate(ceph_rgw_put_initial_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_put_initial_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
+               "expr": "label_replace(\n  rate(ceph_rgw_op_put_obj_lat_sum{job=~\"$job\"}[$__rate_interval]) /\n    rate(ceph_rgw_op_put_obj_lat_count{job=~\"$job\"}[$__rate_interval]) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{job=~\"$job\"},\n  \"rgw_host\", \"$1\", \"ceph_daemon\", \"rgw.(.*)\"\n)\n",
                "format": "time_series",
                "intervalFactor": 1,
                "legendFormat": "{{rgw_host}}",
diff --git a/monitoring/ceph-mixin/tests_dashboards/features/radosgw-detail.feature b/monitoring/ceph-mixin/tests_dashboards/features/radosgw-detail.feature
index e0016c5077d1..f819fc6dc412 100644
--- a/monitoring/ceph-mixin/tests_dashboards/features/radosgw-detail.feature
+++ b/monitoring/ceph-mixin/tests_dashboards/features/radosgw-detail.feature
@@ -3,8 +3,8 @@ Feature: RGW Host Detail Dashboard
 Scenario: "Test $rgw_servers GET/PUT Latencies - GET"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
-    | ceph_rgw_get_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
+    | ceph_rgw_op_get_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_get_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When interval is `30s`
   And variable `rgw_servers` is `rgw.foo`
@@ -15,8 +15,8 @@ Scenario: "Test $rgw_servers GET/PUT Latencies - GET"
 Scenario: "Test $rgw_servers GET/PUT Latencies - PUT"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_put_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
-    | ceph_rgw_put_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
+    | ceph_rgw_op_put_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
+    | ceph_rgw_op_put_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When interval is `30s`
   And variable `rgw_servers` is `rgw.foo`
@@ -27,7 +27,7 @@ Scenario: "Test $rgw_servers GET/PUT Latencies - PUT"
 Scenario: "Test Bandwidth by HTTP Operation - GET"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_get_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
     | ceph_rgw_metadata{ceph_daemon="rgw.1", hostname="localhost", instance="127.0.0.1", instance_id="92806566", job="ceph"} | 1 1 1 |
   When interval is `30s`
   And variable `rgw_servers` is `rgw.1`
@@ -38,7 +38,7 @@ Scenario: "Test Bandwidth by HTTP Operation - GET"
 Scenario: "Test Bandwidth by HTTP Operation - PUT"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_put_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
+    | ceph_rgw_op_put_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
     | ceph_rgw_metadata{ceph_daemon="rgw.1", hostname="localhost", instance="127.0.0.1", instance_id="92806566", job="ceph"} | 1 1 1 |
   When interval is `30s`
   And variable `rgw_servers` is `rgw.1`
diff --git a/monitoring/ceph-mixin/tests_dashboards/features/radosgw_overview.feature b/monitoring/ceph-mixin/tests_dashboards/features/radosgw_overview.feature
index 642e43978782..14a6e5543a43 100644
--- a/monitoring/ceph-mixin/tests_dashboards/features/radosgw_overview.feature
+++ b/monitoring/ceph-mixin/tests_dashboards/features/radosgw_overview.feature
@@ -3,8 +3,8 @@ Feature: RGW Overview Dashboard
 Scenario: "Test Average GET Latencies"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
-    | ceph_rgw_get_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
+    | ceph_rgw_op_get_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_get_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When interval is `30s`
   Then Grafana panel `Average GET/PUT Latencies by RGW Instance` with legend `GET {{rgw_host}}` shows:
@@ -14,8 +14,8 @@ Scenario: "Test Average GET Latencies"
 Scenario: "Test Average PUT Latencies"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_put_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
-    | ceph_rgw_put_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
+    | ceph_rgw_op_put_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
+    | ceph_rgw_op_put_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When interval is `30s`
   Then Grafana panel `Average GET/PUT Latencies by RGW Instance` with legend `PUT {{rgw_host}}` shows:
@@ -35,8 +35,8 @@ Scenario: "Test Total Requests/sec by RGW Instance"
 Scenario: "Test GET Latencies by RGW Instance"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
-    | ceph_rgw_get_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
+    | ceph_rgw_op_get_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_get_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 20 60 80 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When interval is `30s`
   Then Grafana panel `GET Latencies by RGW Instance` with legend `{{rgw_host}}` shows:
@@ -46,7 +46,7 @@ Scenario: "Test GET Latencies by RGW Instance"
 Scenario: "Test Bandwidth Consumed by Type- GET"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_get_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
   When evaluation time is `1m`
   And interval is `30s`
   Then Grafana panel `Bandwidth Consumed by Type` with legend `GETs` shows:
@@ -56,7 +56,7 @@ Scenario: "Test Bandwidth Consumed by Type- GET"
 Scenario: "Test Bandwidth Consumed by Type- PUT"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_put_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
+    | ceph_rgw_op_put_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
   When evaluation time is `1m`
   And interval is `30s`
   Then Grafana panel `Bandwidth Consumed by Type` with legend `PUTs` shows:
@@ -66,8 +66,8 @@ Scenario: "Test Bandwidth Consumed by Type- PUT"
 Scenario: "Test Bandwidth by RGW Instance"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_get_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
-    | ceph_rgw_put_b{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
+    | ceph_rgw_op_get_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 10 50 100 |
+    | ceph_rgw_op_put_obj_bytes{instance="127.0.0.1", instance_id="92806566", job="ceph"} | 5 20 50 |
     | ceph_rgw_metadata{ceph_daemon="rgw.1", hostname="localhost", instance="127.0.0.1", instance_id="92806566", job="ceph"} | 1 1 1 |
   When evaluation time is `1m`
   And interval is `30s`
@@ -78,8 +78,8 @@ Scenario: "Test Bandwidth by RGW Instance"
 Scenario: "Test PUT Latencies by RGW Instance"
   Given the following series:
     | metrics | values |
-    | ceph_rgw_put_initial_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
-    | ceph_rgw_put_initial_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
+    | ceph_rgw_op_put_obj_lat_sum{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 15 35 55 |
+    | ceph_rgw_op_put_obj_lat_count{instance="127.0.0.1", instance_id="58892247", job="ceph"} | 10 30 50 |
     | ceph_rgw_metadata{ceph_daemon="rgw.foo", hostname="localhost", instance="127.0.0.1", instance_id="58892247", job="ceph"} | 1 1 1 |
   When evaluation time is `1m`
   And interval is `30s`

From 685047b035368666c09d654f0e862b9d2f114701 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 7 Feb 2024 09:59:38 +0000
Subject: [PATCH 1794/2492] mon/OSDMonitor: add comments

Co-authored-by: Samuel Just <sjust@redhat.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 11 +++++++++--
 src/mon/OSDMonitor.h  | 20 ++++++++++++++++++--
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 7ec7b6169052..e853a35a4470 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -2280,11 +2280,18 @@ version_t OSDMonitor::get_trim_to() const
   return 0;
 }
 
+/* There are two constraints on trimming:
+ * 1. we must not trim past the last_epoch_clean for any pg
+ * 2. we must not trim past the last reported epoch for any up
+ *    osds.
+ *
+ * LastEpochClean::get_lower_bound_by_pool gives a value <= constraint 1.
+ * For constraint 2, we take the min over osd_epochs, which is populated with
+ * MOSDBeacon::version, see OSDMonitor::prepare_beacon
+ */
 epoch_t OSDMonitor::get_min_last_epoch_clean() const
 {
   auto floor = last_epoch_clean.get_lower_bound_by_pool(osdmap);
-  // also scan osd epochs
-  // don't trim past the oldest reported osd epoch
   for (auto [osd, epoch] : osd_epochs) {
     if (epoch < floor) {
       ceph_assert(osdmap.is_up(osd));
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index e3b91417650f..15e41a9b1798 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -114,6 +114,12 @@ class LastEpochClean {
 public:
   void report(unsigned pg_num, const pg_t& pg, epoch_t last_epoch_clean);
   void remove_pool(uint64_t pool);
+  /**
+   * get_lower_bound_by_pool
+   *
+   * Returns epoch e such that e <= pg.last_epoch_clean for all pgs in cluster.
+   * May return 0 if any pool does not have comprehensive values for all pgs.
+  */
   epoch_t get_lower_bound_by_pool(const OSDMap& latest) const;
 
   void dump(Formatter *f) const;
@@ -639,8 +645,18 @@ class OSDMonitor : public PaxosService,
 
   // when we last received PG stats from each osd and the osd's osd_beacon_report_interval
   std::map<int, std::pair<utime_t, int>> last_osd_report;
-  // TODO: use last_osd_report to store the osd report epochs, once we don't
-  //       need to upgrade from pre-luminous releases.
+  /**
+    * osd_epochs
+    *
+    * Records the MOSDBeacon::version (the osd epoch at which the OSD sent the
+    * beacon) of the most recent beacon recevied from each currently up OSD.
+    * Used in OSDMonitor::get_min_last_epoch_clean().
+    * Down osds are trimmed upon commit of each map
+    *  (OSDMonitor::update_from_paxos).
+    *
+    * TODO: use last_osd_report to store the osd report epochs, once we don't
+    * need to upgrade from pre-luminous releases.
+    */
   std::map<int,epoch_t> osd_epochs;
   LastEpochClean last_epoch_clean;
   bool preprocess_beacon(MonOpRequestRef op);

From 956f644f86c0790c17976b98df112747d66837df Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Tue, 6 Feb 2024 17:42:50 +0530
Subject: [PATCH 1795/2492] doc: add description of metric fields for
 cephfs-top

Fixes: https://tracker.ceph.com/issues/63991
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 doc/cephfs/cephfs-top.rst | 56 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/doc/cephfs/cephfs-top.rst b/doc/cephfs/cephfs-top.rst
index 49439a4bd18b..1588c4f5ced7 100644
--- a/doc/cephfs/cephfs-top.rst
+++ b/doc/cephfs/cephfs-top.rst
@@ -63,6 +63,62 @@ By default, `cephfs-top` uses `client.fstop` user to connect to a Ceph cluster::
   $ ceph auth get-or-create client.fstop mon 'allow r' mds 'allow r' osd 'allow r' mgr 'allow r'
   $ cephfs-top
 
+Description of Fields
+---------------------
+
+1. chit     : Cap hit
+             Percentage of file capability hits over total number of caps
+
+2. dlease   : Dentry lease
+             Percentage of dentry leases handed out over the total dentry lease requests
+
+3. ofiles   : Opened files
+             Number of opened files
+
+4. oicaps   : Pinned caps
+             Number of pinned caps
+
+5. oinodes  : Opened inodes
+             Number of opened inodes
+
+6. rtio     : Total size of read IOs
+             Number of bytes read in input/output operations generated by all process
+
+7. wtio     : Total size of write IOs
+             Number of bytes written in input/output operations generated by all processes
+
+8. raio     : Average size of read IOs
+             Mean of number of bytes read in input/output operations generated by all 
+             process over total IO done
+
+9. waio     : Average size of write IOs
+             Mean of number of bytes written in input/output operations generated by all 
+             process over total IO done
+
+10. rsp     : Read speed
+             Speed of read IOs with respect to the duration since the last refresh of clients
+
+11. wsp     : Write speed 
+             Speed of write IOs with respect to the duration since the last refresh of clients
+
+12. rlatavg : Average read latency
+             Mean value of the read latencies
+
+13. rlatsd  : Standard deviation (variance) for read latency
+             Dispersion of the metric for the read latency relative to its mean
+
+14. wlatavg : Average write latency
+             Mean value of the write latencies
+
+15. wlatsd  : Standard deviation (variance) for write latency
+             Dispersion of the metric for the write latency relative to its mean
+
+16. mlatavg : Average metadata latency
+             Mean value of the metadata latencies
+
+17. mlatsd  : Standard deviation (variance) for metadata latency
+             Dispersion of the metric for the metadata latency relative to its mean
+
 Command-Line Options
 --------------------
 

From f5b8afe981650f5072294303b0e488b91fc65fe8 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 7 Feb 2024 23:32:53 +1000
Subject: [PATCH 1796/2492] doc/cephadm - edit t11ing.

Edit "Running Repeated Debugging Sessions" in
doc/cephadm/troubleshooting.rst. Improve grammar. Fix a misspelling.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/cephadm/troubleshooting.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index d891ebaf2e03..c0bf6f9a00a6 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -444,11 +444,11 @@ Running repeated debugging sessions
 When using ``cephadm shell``, as in the example above, any changes made to the
 container that is spawned by the shell command are ephemeral. After the shell
 session exits, the files that were downloaded and installed cease to be
-available. You can simply re-run the same commands every time ``cephadm
-shell`` is invoked, but in order to save time and resources one can create a
-new container image and use it for repeated debugging sessions.
+available. You can simply re-run the same commands every time ``cephadm shell``
+is invoked, but to save time and resources you can create a new container image
+and use it for repeated debugging sessions.
 
-In the following example, we create a simple file that will construct the
+In the following example, we create a simple file that constructs the
 container image. The command below uses podman but it is expected to work
 correctly even if ``podman`` is replaced with ``docker``::
 
@@ -463,14 +463,14 @@ correctly even if ``podman`` is replaced with ``docker``::
 
 The above file creates a new local image named ``ceph:debugging``. This image
 can be used on the same machine that built it. The image can also be pushed to
-a container repository or saved and copied to a node runing other Ceph
-containers. Consult the ``podman`` or ``docker`` documentation for more
+a container repository or saved and copied to a node that is running other Ceph
+containers. See the ``podman`` or ``docker`` documentation for more
 information about the container workflow.
 
 After the image has been built, it can be used to initiate repeat debugging
 sessions. By using an image in this way, you avoid the trouble of having to
-re-install the debug tools and debuginfo packages every time you need to run a
-debug session. To debug a core file using this image, in the same way as
+re-install the debug tools and the debuginfo packages every time you need to
+run a debug session. To debug a core file using this image, in the same way as
 previously described, run:
 
 .. prompt:: bash #

From a6763cbc942368e895d90e999466dc724cc46062 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Thu, 1 Feb 2024 18:15:19 +0000
Subject: [PATCH 1797/2492] mgr/dashboard: snapshot schedule edit form

Fixes: https://tracker.ceph.com/issues/64331
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  32 +++-
 .../cephfs-snapshotschedule-form.component.ts | 164 +++++++++++++++---
 .../cephfs-snapshotschedule-list.component.ts |  10 ++
 .../api/cephfs-snapshot-schedule.service.ts   |  26 ++-
 .../shared/services/task-message.service.ts   |   3 +
 src/pybind/mgr/dashboard/openapi.yaml         |  84 +++++++--
 6 files changed, 265 insertions(+), 54 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 86f112522c8f..1437962723ff 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=too-many-lines
 import errno
 import json
 import logging
 import os
 from collections import defaultdict
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import cephfs
 import cherrypy
@@ -952,7 +953,7 @@ def list(self, fs: str, path: str = '/', recursive: bool = True):
             return []
 
         snapshot_schedule_list = out.split('\n')
-        output: list[Any] = []
+        output: List[Any] = []
 
         for snap in snapshot_schedule_list:
             current_path = snap.strip().split(' ')[0]
@@ -997,3 +998,30 @@ def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_p
             )
 
         return f'Snapshot schedule for path {path} created successfully'
+
+    def set(self, fs: str, path: str, retention_to_add=None, retention_to_remove=None):
+        def editRetentionPolicies(method, retention_policy):
+            if not retention_policy:
+                return
+
+            retention_policies = retention_policy.split('|')
+            for retention in retention_policies:
+                retention_count = retention.split('-')[0]
+                retention_spec_or_period = retention.split('-')[1]
+                error_code_retention, _, err_retention = mgr.remote('snap_schedule',
+                                                                    method,
+                                                                    path,
+                                                                    retention_spec_or_period,
+                                                                    retention_count,
+                                                                    fs,
+                                                                    None,
+                                                                    None)
+                if error_code_retention != 0:
+                    raise DashboardException(
+                        f'Failed to add/remove retention policy for path {path}: {err_retention}'
+                    )
+
+        editRetentionPolicies('snap_schedule_retention_rm', retention_to_remove)
+        editRetentionPolicies('snap_schedule_retention_add', retention_to_add)
+
+        return f'Retention policies for snapshot schedule on path {path} updated successfully'
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 41e55c72599a..7459a7472c5f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -14,7 +14,11 @@ import { CdForm } from '~/app/shared/forms/cd-form';
 import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
 import { FinishedTask } from '~/app/shared/models/finished-task';
-import { RetentionPolicy, SnapshotScheduleFormValue } from '~/app/shared/models/snapshot-schedule';
+import {
+  RetentionPolicy,
+  SnapshotSchedule,
+  SnapshotScheduleFormValue
+} from '~/app/shared/models/snapshot-schedule';
 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 
 const VALIDATON_TIMER = 300;
@@ -27,11 +31,17 @@ const DEBOUNCE_TIMER = 300;
 })
 export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnInit {
   fsName!: string;
+  path!: string;
+  schedule!: string;
+  retention!: string;
+  start!: string;
+  status!: string;
   id!: number;
   isEdit = false;
   icons = Icons;
   repeatFrequencies = Object.entries(RepeatFrequency);
   retentionFrequencies = Object.entries(RetentionFrequency);
+  retentionPoliciesToRemove: RetentionPolicy[] = [];
 
   currentTime!: NgbTimeStruct;
   minDate!: NgbDateStruct;
@@ -71,7 +81,7 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     this.action = this.actionLabels.CREATE;
     this.directoryStore.loadDirectories(this.id, '/', 3);
     this.createForm();
-    this.loadingReady();
+    this.isEdit ? this.populateForm() : this.loadingReady();
   }
 
   get retentionPolicies() {
@@ -91,6 +101,50 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
       )
     );
 
+  populateForm() {
+    this.action = this.actionLabels.EDIT;
+    this.snapScheduleService.getSnapshotSchedule(this.path, this.fsName, false).subscribe({
+      next: (response: SnapshotSchedule[]) => {
+        const first = response.find((x) => x.path === this.path);
+        this.snapScheduleForm.get('directory').disable();
+        this.snapScheduleForm.get('directory').setValue(first.path);
+        this.snapScheduleForm.get('startDate').disable();
+        this.snapScheduleForm.get('startDate').setValue({
+          year: new Date(first.start).getUTCFullYear(),
+          month: new Date(first.start).getUTCMonth() + 1,
+          day: new Date(first.start).getUTCDate()
+        });
+        this.snapScheduleForm.get('startTime').disable();
+        this.snapScheduleForm.get('startTime').setValue({
+          hour: new Date(first.start).getUTCHours(),
+          minute: new Date(first.start).getUTCMinutes(),
+          second: new Date(first.start).getUTCSeconds()
+        });
+        this.snapScheduleForm.get('repeatInterval').disable();
+        this.snapScheduleForm.get('repeatInterval').setValue(first.schedule.split('')?.[0]);
+        this.snapScheduleForm.get('repeatFrequency').disable();
+        this.snapScheduleForm.get('repeatFrequency').setValue(first.schedule.split('')?.[1]);
+
+        // retention policies
+        first.retention &&
+          Object.entries(first.retention).forEach(([frequency, interval], idx) => {
+            const freqKey = Object.keys(RetentionFrequency)[
+              Object.values(RetentionFrequency).indexOf(frequency as any)
+            ];
+            this.retentionPolicies.push(
+              new FormGroup({
+                retentionInterval: new FormControl(interval),
+                retentionFrequency: new FormControl(RetentionFrequency[freqKey])
+              })
+            );
+            this.retentionPolicies.controls[idx].get('retentionInterval').disable();
+            this.retentionPolicies.controls[idx].get('retentionFrequency').disable();
+          });
+        this.loadingReady();
+      }
+    });
+  }
+
   createForm() {
     this.snapScheduleForm = new CdFormGroup(
       {
@@ -128,11 +182,19 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
   }
 
   removeRetentionPolicy(idx: number) {
+    if (this.isEdit && this.retentionPolicies.at(idx).disabled) {
+      const values = this.retentionPolicies.at(idx).value as RetentionPolicy;
+      this.retentionPoliciesToRemove.push(values);
+    }
     this.retentionPolicies.removeAt(idx);
+    this.retentionPolicies.controls.forEach((x) =>
+      x.get('retentionFrequency').updateValueAndValidity()
+    );
     this.cd.detectChanges();
   }
 
   parseDatetime(date: NgbDateStruct, time?: NgbTimeStruct): string {
+    if (!date || !time) return null;
     return `${date.year}-${date.month}-${date.day}T${time.hour || '00'}:${time.minute || '00'}:${
       time.second || '00'
     }`;
@@ -156,33 +218,69 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
 
     const values = this.snapScheduleForm.value as SnapshotScheduleFormValue;
 
-    const snapScheduleObj = {
-      fs: this.fsName,
-      path: values.directory,
-      snap_schedule: this.parseSchedule(values.repeatInterval, values.repeatFrequency),
-      start: this.parseDatetime(values.startDate, values.startTime)
-    };
+    if (this.isEdit) {
+      const retentionPoliciesToAdd = (this.snapScheduleForm.get(
+        'retentionPolicies'
+      ) as FormArray).controls
+        ?.filter(
+          (ctrl) =>
+            !ctrl.get('retentionInterval').disabled && !ctrl.get('retentionFrequency').disabled
+        )
+        .map((ctrl) => ({
+          retentionInterval: ctrl.get('retentionInterval').value,
+          retentionFrequency: ctrl.get('retentionFrequency').value
+        }));
 
-    const retentionPoliciesValues = this.parseRetentionPolicies(values?.retentionPolicies);
-    if (retentionPoliciesValues) {
-      snapScheduleObj['retention_policy'] = retentionPoliciesValues;
-    }
+      const updateObj = {
+        fs: this.fsName,
+        path: this.path,
+        retention_to_add: this.parseRetentionPolicies(retentionPoliciesToAdd) || null,
+        retention_to_remove: this.parseRetentionPolicies(this.retentionPoliciesToRemove) || null
+      };
 
-    this.taskWrapper
-      .wrapTaskAroundCall({
-        task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.CREATE, {
-          path: snapScheduleObj.path
-        }),
-        call: this.snapScheduleService.create(snapScheduleObj)
-      })
-      .subscribe({
-        error: () => {
-          this.snapScheduleForm.setErrors({ cdSubmitButton: true });
-        },
-        complete: () => {
-          this.activeModal.close();
-        }
-      });
+      this.taskWrapper
+        .wrapTaskAroundCall({
+          task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.EDIT, {
+            path: this.path
+          }),
+          call: this.snapScheduleService.update(updateObj)
+        })
+        .subscribe({
+          error: () => {
+            this.snapScheduleForm.setErrors({ cdSubmitButton: true });
+          },
+          complete: () => {
+            this.activeModal.close();
+          }
+        });
+    } else {
+      const snapScheduleObj = {
+        fs: this.fsName,
+        path: values.directory,
+        snap_schedule: this.parseSchedule(values?.repeatInterval, values?.repeatFrequency),
+        start: this.parseDatetime(values?.startDate, values?.startTime)
+      };
+
+      const retentionPoliciesValues = this.parseRetentionPolicies(values?.retentionPolicies);
+      if (retentionPoliciesValues) {
+        snapScheduleObj['retention_policy'] = retentionPoliciesValues;
+      }
+      this.taskWrapper
+        .wrapTaskAroundCall({
+          task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.CREATE, {
+            path: snapScheduleObj.path
+          }),
+          call: this.snapScheduleService.create(snapScheduleObj)
+        })
+        .subscribe({
+          error: () => {
+            this.snapScheduleForm.setErrors({ cdSubmitButton: true });
+          },
+          complete: () => {
+            this.activeModal.close();
+          }
+        });
+    }
   }
 
   validateSchedule() {
@@ -190,6 +288,11 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
       const directory = frm.get('directory');
       const repeatFrequency = frm.get('repeatFrequency');
       const repeatInterval = frm.get('repeatInterval');
+
+      if (this.isEdit) {
+        return of(null);
+      }
+
       return timer(VALIDATON_TIMER).pipe(
         switchMap(() =>
           this.snapScheduleService
@@ -239,7 +342,12 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
             return null;
           }
           return this.snapScheduleService
-            .checkRetentionPolicyExists(frm.get('directory').value, this.fsName, retentionList)
+            .checkRetentionPolicyExists(
+              frm.get('directory').value,
+              this.fsName,
+              retentionList,
+              this.retentionPoliciesToRemove?.map?.((rp) => rp.retentionFrequency) || []
+            )
             .pipe(
               map(({ exists, errorIndex }) => {
                 if (exists) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index 6b406cfc1712..b6b52a15c99d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -121,6 +121,12 @@ export class CephfsSnapshotscheduleListComponent
         name: this.actionLables.CREATE,
         permission: 'create',
         icon: Icons.add,
+        click: () => this.openModal(false)
+      },
+      {
+        name: this.actionLables.EDIT,
+        permission: 'update',
+        icon: Icons.edit,
         click: () => this.openModal(true)
       }
     ];
@@ -145,6 +151,10 @@ export class CephfsSnapshotscheduleListComponent
         fsName: this.fsName,
         id: this.id,
         path: this.selection?.first()?.path,
+        schedule: this.selection?.first()?.schedule,
+        retention: this.selection?.first()?.retention,
+        start: this.selection?.first()?.start,
+        status: this.selection?.first()?.status,
         isEdit: edit
       },
       { size: 'lg' }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
index 0666bb179e83..0719089c249f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -19,6 +19,14 @@ export class CephfsSnapshotScheduleService {
     return this.http.post(`${this.baseURL}/snapshot/schedule`, data, { observe: 'response' });
   }
 
+  update(data: Record<string, any>): Observable<any> {
+    return this.http.put(
+      `${this.baseURL}/snapshot/schedule/${data.fs}/${encodeURIComponent(data.path)}`,
+      data,
+      { observe: 'response' }
+    );
+  }
+
   checkScheduleExists(
     path: string,
     fs: string,
@@ -41,15 +49,21 @@ export class CephfsSnapshotScheduleService {
   checkRetentionPolicyExists(
     path: string,
     fs: string,
-    retentionFrequencies: string[]
+    retentionFrequencies: string[],
+    retentionFrequenciesRemoved: string[] = []
   ): Observable<{ exists: boolean; errorIndex: number }> {
-    return this.getList(path, fs, false).pipe(
+    return this.getSnapshotSchedule(path, fs, false).pipe(
       map((response) => {
         let errorIndex = -1;
         let exists = false;
         const index = response.findIndex((x) => x.path === path);
         const result = retentionFrequencies?.length
-          ? intersection(Object.keys(response?.[index]?.retention), retentionFrequencies)
+          ? intersection(
+              Object.keys(response?.[index]?.retention).filter(
+                (v) => !retentionFrequenciesRemoved.includes(v)
+              ),
+              retentionFrequencies
+            )
           : [];
         exists = !!result?.length;
         result?.forEach((r) => (errorIndex = retentionFrequencies.indexOf(r)));
@@ -62,10 +76,10 @@ export class CephfsSnapshotScheduleService {
     );
   }
 
-  private getList(path: string, fs: string, recursive = true): Observable<SnapshotSchedule[]> {
+  getSnapshotSchedule(path: string, fs: string, recursive = true): Observable<SnapshotSchedule[]> {
     return this.http
       .get<SnapshotSchedule[]>(
-        `${this.baseURL}/snapshot/schedule?path=${path}&fs=${fs}&recursive=${recursive}`
+        `${this.baseURL}/snapshot/schedule/${fs}?path=${path}&recursive=${recursive}`
       )
       .pipe(
         catchError(() => {
@@ -79,7 +93,7 @@ export class CephfsSnapshotScheduleService {
     fs: string,
     recursive = true
   ): Observable<SnapshotSchedule[]> {
-    return this.getList(path, fs, recursive).pipe(
+    return this.getSnapshotSchedule(path, fs, recursive).pipe(
       map((snapList: SnapshotSchedule[]) =>
         uniqWith(
           snapList.map((snapItem: SnapshotSchedule) => ({
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index f631842919c0..84e31efea013 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -390,6 +390,9 @@ export class TaskMessageService {
     ),
     'cephfs/snapshot/schedule/create': this.newTaskMessage(this.commonOperations.add, (metadata) =>
       this.snapshotSchedule(metadata)
+    ),
+    'cephfs/snapshot/schedule/edit': this.newTaskMessage(this.commonOperations.update, (metadata) =>
+      this.snapshotSchedule(metadata)
     )
   };
 
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 1aaffcb637c3..8457ecb65c6a 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1759,9 +1759,57 @@ paths:
       tags:
       - Cephfs
   /api/cephfs/snapshot/schedule:
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                fs:
+                  type: string
+                path:
+                  type: string
+                retention_policy:
+                  type: string
+                snap_schedule:
+                  type: string
+                start:
+                  type: string
+              required:
+              - fs
+              - path
+              - snap_schedule
+              - start
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
+  /api/cephfs/snapshot/schedule/{fs}:
     get:
       parameters:
-      - in: query
+      - in: path
         name: fs
         required: true
         schema:
@@ -1795,35 +1843,35 @@ paths:
       - jwt: []
       tags:
       - CephFSSnapshotSchedule
-    post:
-      parameters: []
+  /api/cephfs/snapshot/schedule/{fs}/{path}:
+    put:
+      parameters:
+      - in: path
+        name: fs
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: path
+        required: true
+        schema:
+          type: string
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                fs:
-                  type: string
-                path:
-                  type: string
-                retention_policy:
-                  type: string
-                snap_schedule:
+                retention_to_add:
                   type: string
-                start:
+                retention_to_remove:
                   type: string
-              required:
-              - fs
-              - path
-              - snap_schedule
-              - start
               type: object
       responses:
-        '201':
+        '200':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: Resource created.
+          description: Resource updated.
         '202':
           content:
             application/vnd.ceph.api.v1.0+json:

From 5172ad074669d88fea455827a22a7f30a01d93d6 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 7 Feb 2024 09:00:29 -0600
Subject: [PATCH 1798/2492] rgw: mark maybe_unused variables

Specifically, variables that are only used in debug builds.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/rgw/driver/rados/cls_fifo_legacy.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/rgw/driver/rados/cls_fifo_legacy.cc b/src/rgw/driver/rados/cls_fifo_legacy.cc
index 4f2c0738b396..fad9404fdb39 100644
--- a/src/rgw/driver/rados/cls_fifo_legacy.cc
+++ b/src/rgw/driver/rados/cls_fifo_legacy.cc
@@ -547,7 +547,8 @@ void FIFO::_update_meta(const DoutPrefixProvider *dpp, const fifo::update& updat
   update_meta(&op, info.version, update);
   auto updater = std::make_unique<Updater>(dpp, this, c, update, version, pcanceled,
 					   tid);
-  auto r = ioctx.aio_operate(oid, Updater::call(std::move(updater)), &op);
+  [[maybe_unused]] auto r =
+      ioctx.aio_operate(oid, Updater::call(std::move(updater)), &op);
   assert(r >= 0);
 }
 
@@ -1296,8 +1297,9 @@ void FIFO::read_meta(const DoutPrefixProvider *dpp, std::uint64_t tid, lr::AioCo
   encode(gm, in);
   auto reader = std::make_unique<Reader>(dpp, this, c, tid);
   auto rp = reader.get();
-  auto r = ioctx.aio_exec(oid, Reader::call(std::move(reader)), fifo::op::CLASS,
-			  fifo::op::GET_META, in, &rp->bl);
+  [[maybe_unused]] auto r = ioctx.aio_exec(
+      oid, Reader::call(std::move(reader)), fifo::op::CLASS, fifo::op::GET_META,
+      in, &rp->bl);
   assert(r >= 0);
 }
 

From 6504732e1b5b34756a119bc1547d22c07bd6458a Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 7 Feb 2024 23:18:35 +1000
Subject: [PATCH 1799/2492] doc/radosgw: add confval directives

Add confval directives to the documentation of "quota cache" options.
This addresses a request made by Antony D'Atri in https://github.com/ceph/ceph/pull/55075/files#r1444006246.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/admin.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 5717e367018a..fe29849cbec5 100644
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -541,11 +541,11 @@ Quota statistics are cached by each RGW instance. If multiple RGW instances are
 deployed, then this cache may prevent quotas from being perfectly enforced,
 because each instance may have a different set of quota settings.  
 
-The options that control this behavior are 
+Here are the options that control this behavior: 
 
-#. ``rgw_bucket_quota_ttl``
-#. ``rgw_user_quota_bucket_sync_interval`` and 
-#. ``rgw_user_quota_sync_interval``.
+:confval:`rgw_bucket_quota_ttl`
+:confval:`rgw_user_quota_bucket_sync_interval`
+:confval:`rgw_user_quota_sync_interval`
 
 Increasing these values will make quota operations more efficient at the cost
 of increasing the likelihood that the multiple RGW instances may not

From 85562f9fce436691c3905bc5bc5955d1348ef468 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 6 Feb 2024 09:43:50 -0500
Subject: [PATCH 1800/2492] crypto/qat: qualify spawn::yield_context

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/crypto/qat/qcccrypto.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc
index 35bb5d3459f7..d99912d9c229 100644
--- a/src/crypto/qat/qcccrypto.cc
+++ b/src/crypto/qat/qcccrypto.cc
@@ -330,7 +330,7 @@ bool QccCrypto::perform_op_batch(unsigned char* out, const unsigned char* in, si
   int avail_inst = NON_INSTANCE;
 
   if (y) {
-    yield_context yield = y.get_yield_context();
+    spawn::yield_context yield = y.get_yield_context();
     avail_inst = async_get_instance(yield);
   } else {
     auto result = async_get_instance(boost::asio::use_future);
@@ -540,7 +540,7 @@ bool QccCrypto::symPerformOp(int avail_inst,
     do {
       poll_retry_num = RETRY_MAX_NUM;
       if (y) {
-        yield_context yield = y.get_yield_context();
+        spawn::yield_context yield = y.get_yield_context();
         status = helper.async_perform_op(avail_inst, std::span<CpaCySymDpOpData*>(pOpDataVec), yield);
       } else {
         auto result = helper.async_perform_op(avail_inst, std::span<CpaCySymDpOpData*>(pOpDataVec), boost::asio::use_future);

From 484fdea52805a0e0b1c4631977cfcd7fc2afb6f8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 12:48:15 -0500
Subject: [PATCH 1801/2492] crypto/qat: add missing include use_future.hpp

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/crypto/qat/qcccrypto.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/crypto/qat/qcccrypto.h b/src/crypto/qat/qcccrypto.h
index 04cd4d9cafa3..973688fbf0c5 100644
--- a/src/crypto/qat/qcccrypto.h
+++ b/src/crypto/qat/qcccrypto.h
@@ -16,8 +16,9 @@
 #include <vector>
 #include <functional>
 #include <span>
-#include "boost/circular_buffer.hpp"
-#include "boost/asio/thread_pool.hpp"
+#include <boost/circular_buffer.hpp>
+#include <boost/asio/thread_pool.hpp>
+#include <boost/asio/use_future.hpp>
 extern "C" {
 #include "cpa.h"
 #include "cpa_cy_sym_dp.h"

From 10a5cb84f41495f8f5228333354a3d269dc286d3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 13:53:58 -0500
Subject: [PATCH 1802/2492] crypto/qat: remove unused captures of 'this'

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/crypto/qat/qcccrypto.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc
index d99912d9c229..98910426f99b 100644
--- a/src/crypto/qat/qcccrypto.cc
+++ b/src/crypto/qat/qcccrypto.cc
@@ -66,7 +66,7 @@ auto QccCrypto::async_get_instance(CompletionToken&& token) {
       // keep a few objects to wait QAT instance to make sure qat full utilization as much as possible,
       // that is, QAT don't need to wait for new objects to ensure
       // that QAT will not be in a free state as much as possible
-      instance_completions.push_back([this, ex, handler2 = std::move(handler1)](int inst)mutable{
+      instance_completions.push_back([ex, handler2 = std::move(handler1)](int inst)mutable{
         boost::asio::post(ex, std::bind(handler2, inst));
       });
     } else {
@@ -479,7 +479,7 @@ auto QatCrypto::async_perform_op(int avail_inst, std::span<CpaCySymDpOpData*> pO
   using Signature = void(CpaStatus);
   async_completion<CompletionToken, Signature> init(token);
   auto ex = boost::asio::get_associated_executor(init.completion_handler);
-  completion_handler = [this, ex, handler = init.completion_handler](CpaStatus stat) {
+  completion_handler = [ex, handler = init.completion_handler](CpaStatus stat) {
     boost::asio::post(ex, std::bind(handler, stat));
   };
 

From 8f0d1788aa216bd9a84482e321753fa1ab9ba6d5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 14:58:37 -0500
Subject: [PATCH 1803/2492] crypto/qat: remove unused variable qcc_alloc_mutex

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/crypto/qat/qcccrypto.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc
index 98910426f99b..6110e28c7921 100644
--- a/src/crypto/qat/qcccrypto.cc
+++ b/src/crypto/qat/qcccrypto.cc
@@ -39,7 +39,6 @@ static void symDpCallback(CpaCySymDpOpData *pOpData,
   }
 }
 
-static std::mutex qcc_alloc_mutex;
 static std::mutex qcc_eng_mutex;
 static std::atomic<bool> init_called = { false };
 static std::mutex poll_inst_mutex;

From 80eae5de523dc796a4bd6ab0e9cccc2b9b2f6287 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 12:06:28 -0500
Subject: [PATCH 1804/2492] cmake: rename WITH_QAT to WITH_QATDRV

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt            | 6 +++---
 src/crypto/CMakeLists.txt | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 582eda5c74dc..50c8905ca0a0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -307,10 +307,10 @@ endif()
 
 option(WITH_BLUEFS "libbluefs library" OFF)
 
-option(WITH_QAT "Enable Qat driver" OFF)
-if(WITH_QAT)
+option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
+if(WITH_QATDRV)
   find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
-  set(HAVE_QATDRV $(QatDrv_FOUND))
+  set(HAVE_QATDRV TRUE)
 endif()
 
 option(WITH_QATZIP "Enable QATZIP" OFF)
diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt
index f930886d3d7c..33daee114821 100644
--- a/src/crypto/CMakeLists.txt
+++ b/src/crypto/CMakeLists.txt
@@ -7,6 +7,6 @@ if(HAVE_INTEL AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
   add_subdirectory(isa-l)
 endif()
 
-if(WITH_QAT)
+if(WITH_QATDRV)
   add_subdirectory(qat)
 endif()

From d6663c389b7959327d77a4278098aa8333cb4ffd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 12:23:57 -0500
Subject: [PATCH 1805/2492] cmake: add WITH_QATLIB to build against qatlib

FindQAT.cmake adds library targets QAT::qat and QAT::usdm for qatlib,
and FindQatDrv.cmake adds alias targets to the same

src/compressor and src/crypto/qat link against these common QAT targets

src/crypto/qat/qcccrypto.h uses new HAVE_QATDRV define to determine
which headers to include

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt                 |  7 +++++-
 cmake/modules/FindQAT.cmake    | 40 ++++++++++++++++++++++++++++++++++
 cmake/modules/FindQatDrv.cmake |  8 +++++++
 src/compressor/CMakeLists.txt  |  6 ++---
 src/crypto/CMakeLists.txt      |  2 +-
 src/crypto/qat/CMakeLists.txt  |  4 ++--
 src/crypto/qat/qcccrypto.cc    |  7 +++++-
 src/crypto/qat/qcccrypto.h     | 10 +++++++++
 8 files changed, 76 insertions(+), 8 deletions(-)
 create mode 100644 cmake/modules/FindQAT.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 50c8905ca0a0..213c6b694731 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -307,10 +307,15 @@ endif()
 
 option(WITH_BLUEFS "libbluefs library" OFF)
 
+option(WITH_QATLIB "Enable QAT with qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
+
 if(WITH_QATDRV)
   find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
-  set(HAVE_QATDRV TRUE)
+  set(HAVE_QAT TRUE)
+elseif(WITH_QATLIB)
+  find_package(QAT REQUIRED)
+  set(HAVE_QAT TRUE)
 endif()
 
 option(WITH_QATZIP "Enable QATZIP" OFF)
diff --git a/cmake/modules/FindQAT.cmake b/cmake/modules/FindQAT.cmake
new file mode 100644
index 000000000000..9044e5493922
--- /dev/null
+++ b/cmake/modules/FindQAT.cmake
@@ -0,0 +1,40 @@
+find_package(PkgConfig)
+pkg_search_module(PC_QAT libqat qatlib QUIET)
+
+find_path(QAT_INCLUDE_DIR
+  NAMES qat/cpa.h
+  HINTS ${PC_QAT_INCLUDE_DIRS})
+
+find_library(QAT_LIBRARY
+  NAMES qat
+  HINTS ${PC_QAT_LIBRARY_DIRS})
+
+find_library(QAT_USDM_LIBRARY
+  NAMES usdm
+  HINTS ${PC_QAT_LIBRARY_DIRS})
+
+set(QAT_VERSION ${PC_QAT_VERSION})
+set(QAT_LIBRARIES ${QAT_LIBRARY} ${QAT_USDM_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(QAT
+  REQUIRED_VARS QAT_LIBRARY QAT_USDM_LIBRARY QAT_INCLUDE_DIR
+  VERSION_VAR QAT_VERSION)
+
+mark_as_advanced(QAT_LIBRARY QAT_USDM_LIBRARY QAT_LIBRARIES QAT_INCLUDE_DIR QAT_VERSION)
+
+if(QAT_FOUND AND NOT (TARGET QAT::qat))
+  add_library(QAT::qat UNKNOWN IMPORTED)
+  set_target_properties(QAT::qat PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${QAT_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${QAT_LIBRARY}")
+endif()
+
+if(QAT_FOUND AND NOT (TARGET QAT::usdm))
+  add_library(QAT::usdm UNKNOWN IMPORTED)
+  set_target_properties(QAT::usdm PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${QAT_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${QAT_USDM_LIBRARY}")
+endif()
diff --git a/cmake/modules/FindQatDrv.cmake b/cmake/modules/FindQatDrv.cmake
index 3305a38c06cb..e2849d9c89be 100644
--- a/cmake/modules/FindQatDrv.cmake
+++ b/cmake/modules/FindQatDrv.cmake
@@ -74,7 +74,15 @@ foreach(component ${QatDrv_FIND_COMPONENTS})
     add_library(QatDrv::${component} STATIC IMPORTED GLOBAL)
     set_target_properties(QatDrv::${component} PROPERTIES
                           INTERFACE_INCLUDE_DIRECTORIES "${QatDrv_INCLUDE_DIRS}"
+                          INTERFACE_COMPILE_OPTIONS "-DHAVE_QATDRV"
                           IMPORTED_LINK_INTERFACE_LANGUAGES "C"
                           IMPORTED_LOCATION "${QatDrv_${component}_LIBRARIES}")
   endif()
+
+  # add alias targets to match FindQAT.cmake
+  if(component STREQUAL "qat_s")
+    add_library(QAT::qat ALIAS QatDrv::qat_s)
+  elseif(component STREQUAL "usdm_drv_s")
+    add_library(QAT::usdm ALIAS QatDrv::usdm_drv_s)
+  endif()
 endforeach()
diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt
index 3e99f8b73875..1f9cab5f3c70 100644
--- a/src/compressor/CMakeLists.txt
+++ b/src/compressor/CMakeLists.txt
@@ -6,10 +6,10 @@ if (HAVE_QATZIP)
 endif()
 add_library(compressor_objs OBJECT ${compressor_srcs})
 add_dependencies(compressor_objs common-objs)
-if(HAVE_QATZIP AND HAVE_QATDRV)
+if(HAVE_QATZIP AND HAVE_QAT)
   target_link_libraries(compressor_objs PRIVATE
-                        QatDrv::qat_s
-                        QatDrv::usdm_drv_s
+                        QAT::qat
+                        QAT::usdm
                         qatzip::qatzip
                        )
 endif()
diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt
index 33daee114821..d1ba381999f2 100644
--- a/src/crypto/CMakeLists.txt
+++ b/src/crypto/CMakeLists.txt
@@ -7,6 +7,6 @@ if(HAVE_INTEL AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
   add_subdirectory(isa-l)
 endif()
 
-if(WITH_QATDRV)
+if(HAVE_QAT)
   add_subdirectory(qat)
 endif()
diff --git a/src/crypto/qat/CMakeLists.txt b/src/crypto/qat/CMakeLists.txt
index 77791cacf79b..04bc0b7e7f43 100644
--- a/src/crypto/qat/CMakeLists.txt
+++ b/src/crypto/qat/CMakeLists.txt
@@ -12,8 +12,8 @@ add_library(ceph_crypto_qat SHARED ${qat_crypto_plugin_srcs})
 add_dependencies(crypto_plugins ceph_crypto_qat)
 
 target_link_libraries(ceph_crypto_qat PRIVATE
-                      QatDrv::qat_s
-                      QatDrv::usdm_drv_s
+                      QAT::qat
+                      QAT::usdm
                       spawn)
 
 add_dependencies(crypto_plugins ceph_crypto_qat)
diff --git a/src/crypto/qat/qcccrypto.cc b/src/crypto/qat/qcccrypto.cc
index 6110e28c7921..56681c74517c 100644
--- a/src/crypto/qat/qcccrypto.cc
+++ b/src/crypto/qat/qcccrypto.cc
@@ -88,7 +88,9 @@ void QccCrypto::QccFreeInstance(int entry) {
 
 void QccCrypto::cleanup() {
   icp_sal_userStop();
+#ifdef HAVE_QATDRV
   qaeMemDestroy();
+#endif
   is_init = false;
   init_called = false;
   derr << "Failure during QAT init sequence. Quitting" << dendl;
@@ -139,6 +141,7 @@ bool QccCrypto::init(const size_t chunk_size, const size_t max_requests) {
   dout(15) << "First init for QAT" << dendl;
   init_called = true;
 
+#ifdef HAVE_QATDRV
   // Find if the usermode memory driver is available. We need to this to
   // create contiguous memory needed by QAT.
   stat = qaeMemInit();
@@ -147,7 +150,7 @@ bool QccCrypto::init(const size_t chunk_size, const size_t max_requests) {
     this->cleanup();
     return false;
   }
-
+#endif
   stat = icp_sal_userStart("CEPH");
   if (stat != CPA_STATUS_SUCCESS) {
     derr << "Unable to start qat device" << dendl;
@@ -300,7 +303,9 @@ bool QccCrypto::destroy() {
 
   //Un-init memory driver and QAT HW
   icp_sal_userStop();
+#ifdef HAVE_QATDRV
   qaeMemDestroy();
+#endif
   init_called = false;
   is_init = false;
   return true;
diff --git a/src/crypto/qat/qcccrypto.h b/src/crypto/qat/qcccrypto.h
index 973688fbf0c5..cd17a909e2d6 100644
--- a/src/crypto/qat/qcccrypto.h
+++ b/src/crypto/qat/qcccrypto.h
@@ -20,6 +20,7 @@
 #include <boost/asio/thread_pool.hpp>
 #include <boost/asio/use_future.hpp>
 extern "C" {
+#ifdef HAVE_QATDRV
 #include "cpa.h"
 #include "cpa_cy_sym_dp.h"
 #include "cpa_cy_im.h"
@@ -29,6 +30,15 @@ extern "C" {
 #include "icp_sal_user.h"
 #include "icp_sal_poll.h"
 #include "qae_mem_utils.h"
+#else
+#include <qat/cpa.h>
+#include <qat/cpa_cy_im.h>
+#include <qat/cpa_cy_sym_dp.h>
+#include <qat/cpa_cy_sym.h>
+#include <qat/qae_mem.h>
+#include <qat/icp_sal_user.h>
+#include <qat/icp_sal_poll.h>
+#endif
 }
 
 class QccCrypto {

From 127059e7c600c3381fd53298cae764d55938e427 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 12:41:44 -0500
Subject: [PATCH 1806/2492] cmake: rename qatzip->QATzip and support pkg-config

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt                 |  6 +++---
 cmake/modules/FindQATzip.cmake | 32 ++++++++++++++++++++++++++++++++
 cmake/modules/Findqatzip.cmake | 24 ------------------------
 src/CMakeLists.txt             |  3 ++-
 src/compressor/CMakeLists.txt  |  2 +-
 5 files changed, 38 insertions(+), 29 deletions(-)
 create mode 100644 cmake/modules/FindQATzip.cmake
 delete mode 100644 cmake/modules/Findqatzip.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 213c6b694731..edb653a68fda 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -309,6 +309,7 @@ option(WITH_BLUEFS "libbluefs library" OFF)
 
 option(WITH_QATLIB "Enable QAT with qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
+option(WITH_QATZIP "Enable QATzip" OFF)
 
 if(WITH_QATDRV)
   find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
@@ -318,10 +319,9 @@ elseif(WITH_QATLIB)
   set(HAVE_QAT TRUE)
 endif()
 
-option(WITH_QATZIP "Enable QATZIP" OFF)
 if(WITH_QATZIP)
-  find_package(qatzip REQUIRED)
-  set(HAVE_QATZIP ${qatzip_FOUND})
+  find_package(QATzip REQUIRED)
+  set(HAVE_QATZIP TRUE)
 endif(WITH_QATZIP)
 
 # needs mds and? XXX
diff --git a/cmake/modules/FindQATzip.cmake b/cmake/modules/FindQATzip.cmake
new file mode 100644
index 000000000000..e61f5f7f4a19
--- /dev/null
+++ b/cmake/modules/FindQATzip.cmake
@@ -0,0 +1,32 @@
+# - Find QATzip
+# Find the QATzip compression library and includes
+#
+# QATzip_INCLUDE_DIR - where to find QATzip.h, etc.
+# QATzip_LIBRARIES - List of libraries when using QATzip.
+# QATzip_FOUND - True if QATzip found.
+
+find_package(PkgConfig QUIET)
+pkg_search_module(PC_QATzip qatzip QUIET)
+
+find_path(QATzip_INCLUDE_DIR
+  NAMES qatzip.h
+  HINTS ${PC_QATzip_INCLUDE_DIRS})
+
+find_library(QATzip_LIBRARIES
+  NAMES qatzip
+  HINTS ${PC_QATzip_LIBRARY_DIRS})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(QATzip DEFAULT_MSG QATzip_LIBRARIES QATzip_INCLUDE_DIR)
+
+mark_as_advanced(
+  QATzip_LIBRARIES
+  QATzip_INCLUDE_DIR)
+
+if(QATzip_FOUND AND NOT TARGET QAT::zip)
+  add_library(QAT::zip SHARED IMPORTED)
+  set_target_properties(QAT::zip PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${QATzip_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${QATzip_LIBRARIES}")
+endif()
diff --git a/cmake/modules/Findqatzip.cmake b/cmake/modules/Findqatzip.cmake
deleted file mode 100644
index 2d0f2ace3887..000000000000
--- a/cmake/modules/Findqatzip.cmake
+++ /dev/null
@@ -1,24 +0,0 @@
-# - Find qatzip
-# Find the qatzip compression library and includes
-#
-# qatzip_INCLUDE_DIR - where to find qatzip.h, etc.
-# qatzip_LIBRARIES - List of libraries when using qatzip.
-# qatzip_FOUND - True if qatzip found.
-
-find_path(qatzip_INCLUDE_DIR NAMES qatzip.h)
-find_library(qatzip_LIBRARIES NAMES qatzip HINTS /usr/local/lib64/)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(qatzip DEFAULT_MSG qatzip_LIBRARIES qatzip_INCLUDE_DIR)
-
-mark_as_advanced(
-  qatzip_LIBRARIES
-  qatzip_INCLUDE_DIR)
-
-if(qatzip_FOUND AND NOT TARGET qatzip::qatzip)
-  add_library(qatzip::qatzip SHARED IMPORTED)
-  set_target_properties(qatzip::qatzip PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES "${qatzip_INCLUDE_DIR}"
-    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
-    IMPORTED_LOCATION "${qatzip_LIBRARIES}")
-endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index d266eb72598b..23196619eb8f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -506,7 +506,8 @@ if(NOT WITH_SYSTEM_BOOST)
 endif()
 
 if(HAVE_QATZIP)
-  list(APPEND ceph_common_deps ${qatzip_LIBRARIES})
+  # TODO: only the compression plugins should depend on QAT
+  list(APPEND ceph_common_deps QAT::zip)
 endif()
 
 if(WITH_DPDK)
diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt
index 1f9cab5f3c70..d9512e87408e 100644
--- a/src/compressor/CMakeLists.txt
+++ b/src/compressor/CMakeLists.txt
@@ -10,7 +10,7 @@ if(HAVE_QATZIP AND HAVE_QAT)
   target_link_libraries(compressor_objs PRIVATE
                         QAT::qat
                         QAT::usdm
-                        qatzip::qatzip
+                        QAT::zip
                        )
 endif()
 add_dependencies(compressor_objs legacy-option-headers)

From 6d550ffae6dfeebf007f4ea8d48ef4ac0328e232 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Thu, 11 Jan 2024 17:12:07 +0100
Subject: [PATCH 1807/2492] mgr/prometheus: fix orch check to prevent
 Prometheus from crashing Fixes: https://tracker.ceph.com/issues/63992

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/prometheus/module.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
index e6d4860efff3..45a6c9f00137 100644
--- a/src/pybind/mgr/prometheus/module.py
+++ b/src/pybind/mgr/prometheus/module.py
@@ -13,7 +13,7 @@
 
 from mgr_module import CLIReadCommand, MgrModule, MgrStandbyModule, PG_STATES, Option, ServiceInfoT, HandleCommandResult, CLIWriteCommand
 from mgr_util import get_default_addr, profile_method, build_url
-from orchestrator import OrchestratorClientMixin, raise_if_exception, NoOrchestrator
+from orchestrator import OrchestratorClientMixin, raise_if_exception, OrchestratorError
 from rbd import RBD
 
 from typing import DefaultDict, Optional, Dict, Any, Set, cast, Tuple, Union, List, Callable
@@ -646,8 +646,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         _global_instance = self
         self.metrics_thread = MetricCollectionThread(_global_instance)
         self.health_history = HealthHistory(self)
-        self.modify_instance_id = self.get_orch_status() and self.get_module_option(
-            'exclude_perf_counters')
 
     def _setup_static_metrics(self) -> Dict[str, Metric]:
         metrics = {}
@@ -864,10 +862,12 @@ def _setup_static_metrics(self) -> Dict[str, Metric]:
 
         return metrics
 
-    def get_orch_status(self) -> bool:
+    def orch_is_available(self) -> bool:
         try:
             return self.available()[0]
-        except NoOrchestrator:
+        except (RuntimeError, OrchestratorError, ImportError):
+            # import error could happend during startup in case
+            # orchestrator has not been loaded yet by the mgr
             return False
 
     def get_server_addr(self) -> str:
@@ -1292,18 +1292,22 @@ def _get_pool_info(pool: Dict[str, Any]) -> Tuple[str, str]:
         # Populate other servers metadata
         # If orchestrator is available and ceph-exporter is running modify rgw instance id
         # to match the one from exporter
-        if self.modify_instance_id:
+        modify_instance_id = self.orch_is_available() and self.get_module_option('exclude_perf_counters')
+        if modify_instance_id:
             daemons = raise_if_exception(self.list_daemons(daemon_type='rgw'))
             for daemon in daemons:
+                if daemon.daemon_id and '.' in daemon.daemon_id:
+                    instance_id = daemon.daemon_id.split(".")[2]
+                else:
+                    instance_id = daemon.daemon_id if daemon.daemon_id else ""
                 self.metrics['rgw_metadata'].set(1,
-                                                 ('{}.{}'.format(str(daemon.daemon_type),
-                                                                 str(daemon.daemon_id)),
+                                                 (f"{daemon.daemon_type}.{daemon.daemon_id}",
                                                   str(daemon.hostname),
                                                   str(daemon.version),
-                                                  str(daemon.daemon_id).split(".")[2]))
+                                                  instance_id))
         for key, value in servers.items():
             service_id, service_type = key
-            if service_type == 'rgw' and not self.modify_instance_id:
+            if service_type == 'rgw' and not modify_instance_id:
                 hostname, version, name = value
                 self.metrics['rgw_metadata'].set(
                     1,

From 30681236678c7ee006a699b658233388b0f884c8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 19:10:59 -0500
Subject: [PATCH 1808/2492] cmake: build qat from source when
 WITH_SYSTEM_QATLIB=OFF

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt               |  9 ++++++
 cmake/modules/BuildQAT.cmake | 56 ++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 cmake/modules/BuildQAT.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index edb653a68fda..1f9bc746f884 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -308,6 +308,7 @@ endif()
 option(WITH_BLUEFS "libbluefs library" OFF)
 
 option(WITH_QATLIB "Enable QAT with qatlib" OFF)
+option(WITH_SYSTEM_QATLIB "Use system packages for qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
 option(WITH_QATZIP "Enable QATzip" OFF)
 
@@ -315,7 +316,15 @@ if(WITH_QATDRV)
   find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
   set(HAVE_QAT TRUE)
 elseif(WITH_QATLIB)
+  if(NOT WITH_SYSTEM_QAT)
+    include(BuildQAT)
+    build_qat()
+  endif()
   find_package(QAT REQUIRED)
+  if(NOT WITH_SYSTEM_QAT)
+    add_dependencies(QAT::qat qatlib_ext)
+    add_dependencies(QAT::usdm qatlib_ext)
+  endif()
   set(HAVE_QAT TRUE)
 endif()
 
diff --git a/cmake/modules/BuildQAT.cmake b/cmake/modules/BuildQAT.cmake
new file mode 100644
index 000000000000..9296a0584a2a
--- /dev/null
+++ b/cmake/modules/BuildQAT.cmake
@@ -0,0 +1,56 @@
+function(build_qat)
+  set(QAT_REPO https://github.com/intel/qatlib.git)
+  set(QAT_TAG "23.11.0")
+
+  set(QAT_SOURCE_DIR ${CMAKE_BINARY_DIR}/src/qatlib)
+  set(QAT_INSTALL_DIR ${QAT_SOURCE_DIR}/install)
+  set(QAT_INCLUDE_DIR ${QAT_INSTALL_DIR}/include)
+  set(QAT_LIBRARY_DIR ${QAT_INSTALL_DIR}/lib)
+  set(QAT_LIBRARY ${QAT_LIBRARY_DIR}/libqat.a)
+  set(QAT_USDM_LIBRARY ${QAT_LIBRARY_DIR}/libusdm.a)
+
+  # this include directory won't exist until the install step, but the
+  # imported targets need it early for INTERFACE_INCLUDE_DIRECTORIES
+  file(MAKE_DIRECTORY "${QAT_INCLUDE_DIR}")
+
+  set(configure_cmd env CC=${CMAKE_C_COMPILER} ./configure --prefix=${QAT_INSTALL_DIR})
+  # disable systemd or 'make install' tries to write /usr/lib/systemd/system/qat.service
+  list(APPEND configure_cmd --disable-systemd)
+  # samples don't build on arm64
+  list(APPEND configure_cmd --disable-samples)
+  # build a static library with -fPIC that we can link into crypto/compressor plugins
+  list(APPEND configure_cmd --with-pic --enable-static --disable-shared)
+
+  set(source_dir_args
+    SOURCE_DIR ${QAT_SOURCE_DIR}
+    GIT_REPOSITORY ${QAT_REPO}
+    GIT_TAG ${QAT_TAG}
+    GIT_SHALLOW TRUE
+    GIT_CONFIG advice.detachedHead=false)
+
+  # clear the DESTDIR environment variable from debian/rules,
+  # because it messes with the internal install paths of arrow's bundled deps
+  set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
+
+  include(ExternalProject)
+  ExternalProject_Add(qatlib_ext
+    ${source_dir_args}
+    CONFIGURE_COMMAND ./autogen.sh COMMAND ${configure_cmd}
+    BUILD_COMMAND ${NO_DESTDIR_COMMAND} make -j3
+    BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${QAT_LIBRARY} ${QAT_USDM_LIBRARY}
+    INSTALL_COMMAND ${NO_DESTDIR_COMMAND} make install
+    UPDATE_COMMAND ""
+    LOG_CONFIGURE ON
+    LOG_BUILD ON
+    LOG_INSTALL ON
+    LOG_MERGED_STDOUTERR ON
+    LOG_OUTPUT_ON_FAILURE ON)
+
+  # export vars for find_package(QAT)
+  set(QAT_LIBRARY ${QAT_LIBRARY} PARENT_SCOPE)
+  set(QAT_USDM_LIBRARY ${QAT_USDM_LIBRARY} PARENT_SCOPE)
+  set(QAT_INCLUDE_DIR ${QAT_INCLUDE_DIR} PARENT_SCOPE)
+  # library dir for BuildQATzip.cmake
+  set(QAT_LIBRARY_DIR ${QAT_LIBRARY_DIR} PARENT_SCOPE)
+endfunction()

From 74c437a0c8d650f4a3351fd9960e4bd695d64cf7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 14:10:55 -0500
Subject: [PATCH 1809/2492] cmake: build from qatlib submodule

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 .gitmodules                  |  3 +++
 cmake/modules/BuildQAT.cmake | 16 +++-------------
 src/qatlib                   |  1 +
 3 files changed, 7 insertions(+), 13 deletions(-)
 create mode 160000 src/qatlib

diff --git a/.gitmodules b/.gitmodules
index 088ae3b577ce..fbe2690c9ee5 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -75,3 +75,6 @@
 [submodule "src/jaegertracing/opentelemetry-cpp"]
 	path = src/jaegertracing/opentelemetry-cpp
 	url = https://github.com/open-telemetry/opentelemetry-cpp.git
+[submodule "src/qatlib"]
+	path = src/qatlib
+	url = https://github.com/intel/qatlib.git
diff --git a/cmake/modules/BuildQAT.cmake b/cmake/modules/BuildQAT.cmake
index 9296a0584a2a..737c63c3ea8c 100644
--- a/cmake/modules/BuildQAT.cmake
+++ b/cmake/modules/BuildQAT.cmake
@@ -1,9 +1,6 @@
 function(build_qat)
-  set(QAT_REPO https://github.com/intel/qatlib.git)
-  set(QAT_TAG "23.11.0")
-
-  set(QAT_SOURCE_DIR ${CMAKE_BINARY_DIR}/src/qatlib)
-  set(QAT_INSTALL_DIR ${QAT_SOURCE_DIR}/install)
+  set(QAT_BINARY_DIR ${CMAKE_BINARY_DIR}/src/qatlib)
+  set(QAT_INSTALL_DIR ${QAT_BINARY_DIR}/install)
   set(QAT_INCLUDE_DIR ${QAT_INSTALL_DIR}/include)
   set(QAT_LIBRARY_DIR ${QAT_INSTALL_DIR}/lib)
   set(QAT_LIBRARY ${QAT_LIBRARY_DIR}/libqat.a)
@@ -21,20 +18,13 @@ function(build_qat)
   # build a static library with -fPIC that we can link into crypto/compressor plugins
   list(APPEND configure_cmd --with-pic --enable-static --disable-shared)
 
-  set(source_dir_args
-    SOURCE_DIR ${QAT_SOURCE_DIR}
-    GIT_REPOSITORY ${QAT_REPO}
-    GIT_TAG ${QAT_TAG}
-    GIT_SHALLOW TRUE
-    GIT_CONFIG advice.detachedHead=false)
-
   # clear the DESTDIR environment variable from debian/rules,
   # because it messes with the internal install paths of arrow's bundled deps
   set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
 
   include(ExternalProject)
   ExternalProject_Add(qatlib_ext
-    ${source_dir_args}
+    SOURCE_DIR "${PROJECT_SOURCE_DIR}/src/qatlib"
     CONFIGURE_COMMAND ./autogen.sh COMMAND ${configure_cmd}
     BUILD_COMMAND ${NO_DESTDIR_COMMAND} make -j3
     BUILD_IN_SOURCE 1
diff --git a/src/qatlib b/src/qatlib
new file mode 160000
index 000000000000..142e305970ec
--- /dev/null
+++ b/src/qatlib
@@ -0,0 +1 @@
+Subproject commit 142e305970ec66a860945d20bb7c330f99ed900b

From e4e7dfd4728013c54c0af48c531a4ecaaab13ae3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 19:27:01 -0500
Subject: [PATCH 1810/2492] cmake: build qatzip from source when
 WITH_SYSTEM_QATZIP=OFF

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt                  | 10 ++++++
 cmake/modules/BuildQATzip.cmake | 56 +++++++++++++++++++++++++++++++++
 cmake/modules/FindQATzip.cmake  |  1 +
 3 files changed, 67 insertions(+)
 create mode 100644 cmake/modules/BuildQATzip.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1f9bc746f884..c5d708f69de0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -311,6 +311,7 @@ option(WITH_QATLIB "Enable QAT with qatlib" OFF)
 option(WITH_SYSTEM_QATLIB "Use system packages for qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
 option(WITH_QATZIP "Enable QATzip" OFF)
+option(WITH_SYSTEM_QATZIP "Use system packages for QATzip" OFF)
 
 if(WITH_QATDRV)
   find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
@@ -329,7 +330,16 @@ elseif(WITH_QATLIB)
 endif()
 
 if(WITH_QATZIP)
+  if(NOT WITH_SYSTEM_QATZIP)
+    include(BuildQATzip)
+    build_qatzip()
+    # qatzip build depends on qatlib
+    add_dependencies(qatzip_ext QAT::qat)
+  endif()
   find_package(QATzip REQUIRED)
+  if(NOT WITH_SYSTEM_QATZIP)
+    add_dependencies(QAT::zip qatzip_ext)
+  endif()
   set(HAVE_QATZIP TRUE)
 endif(WITH_QATZIP)
 
diff --git a/cmake/modules/BuildQATzip.cmake b/cmake/modules/BuildQATzip.cmake
new file mode 100644
index 000000000000..005288e873b2
--- /dev/null
+++ b/cmake/modules/BuildQATzip.cmake
@@ -0,0 +1,56 @@
+function(build_qatzip)
+  set(QATzip_REPO https://github.com/intel/qatzip.git)
+  set(QATzip_TAG "v1.1.2")
+
+  set(QATzip_SOURCE_DIR ${CMAKE_BINARY_DIR}/src/qatzip)
+  set(QATzip_INSTALL_DIR ${QATzip_SOURCE_DIR}/install)
+  set(QATzip_INCLUDE_DIR ${QATzip_INSTALL_DIR}/include)
+  set(QATzip_LIBRARY ${QATzip_INSTALL_DIR}/lib/libqatzip.a)
+
+  # this include directory won't exist until the install step, but the
+  # imported targets need it early for INTERFACE_INCLUDE_DIRECTORIES
+  file(MAKE_DIRECTORY "${QATzip_INCLUDE_DIR}")
+
+  set(configure_cmd env CC=${CMAKE_C_COMPILER} ./configure --prefix=${QATzip_INSTALL_DIR})
+  # build a static library with -fPIC that we can link into crypto/compressor plugins
+  list(APPEND configure_cmd --with-pic --enable-static --disable-shared)
+  if(QATDRV_INCLUDE_DIR)
+    list(APPEND configure_cmd --with-ICP_ROOT=${QATDRV_INCLUDE_DIR})
+  endif()
+  if(QAT_INCLUDE_DIR)
+    list(APPEND configure_cmd CFLAGS=-I${QAT_INCLUDE_DIR})
+  endif()
+  if(QAT_LIBRARY_DIR)
+    list(APPEND configure_cmd LDFLAGS=-L${QAT_LIBRARY_DIR})
+  endif()
+
+  # clear the DESTDIR environment variable from debian/rules,
+  # because it messes with the internal install paths of arrow's bundled deps
+  set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
+
+  set(source_dir_args
+    SOURCE_DIR ${QATzip_SOURCE_DIR}
+    GIT_REPOSITORY ${QATzip_REPO}
+    GIT_TAG ${QATzip_TAG}
+    GIT_SHALLOW TRUE
+    GIT_CONFIG advice.detachedHead=false)
+
+  include(ExternalProject)
+  ExternalProject_Add(qatzip_ext
+    ${source_dir_args}
+    CONFIGURE_COMMAND ./autogen.sh COMMAND ${configure_cmd}
+    BUILD_COMMAND ${NO_DESTDIR_COMMAND} make -j3
+    BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${QATzip_LIBRARY}
+    UPDATE_COMMAND ""
+    LOG_CONFIGURE ON
+    LOG_BUILD ON
+    LOG_INSTALL ON
+    LOG_MERGED_STDOUTERR ON
+    LOG_OUTPUT_ON_FAILURE ON)
+
+  # export vars for find_package(QATzip)
+  set(QATzip_LIBRARIES ${QATzip_LIBRARY} PARENT_SCOPE)
+  set(QATzip_INCLUDE_DIR ${QATzip_INCLUDE_DIR} PARENT_SCOPE)
+  set(QATzip_INTERFACE_LINK_LIBRARIES QAT::qat QAT::usdm LZ4::LZ4 PARENT_SCOPE)
+endfunction()
diff --git a/cmake/modules/FindQATzip.cmake b/cmake/modules/FindQATzip.cmake
index e61f5f7f4a19..364cb7359148 100644
--- a/cmake/modules/FindQATzip.cmake
+++ b/cmake/modules/FindQATzip.cmake
@@ -27,6 +27,7 @@ if(QATzip_FOUND AND NOT TARGET QAT::zip)
   add_library(QAT::zip SHARED IMPORTED)
   set_target_properties(QAT::zip PROPERTIES
     INTERFACE_INCLUDE_DIRECTORIES "${QATzip_INCLUDE_DIR}"
+    INTERFACE_LINK_LIBRARIES "${QATzip_INTERFACE_LINK_LIBRARIES}"
     IMPORTED_LINK_INTERFACE_LANGUAGES "C"
     IMPORTED_LOCATION "${QATzip_LIBRARIES}")
 endif()

From 1df11fd17c5ac471915df32b223d5fcaf2effdaa Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 14:15:12 -0500
Subject: [PATCH 1811/2492] cmake: build from qatzip submodule

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 .gitmodules                     |  3 +++
 cmake/modules/BuildQATzip.cmake | 17 ++++-------------
 src/qatzip                      |  1 +
 3 files changed, 8 insertions(+), 13 deletions(-)
 create mode 160000 src/qatzip

diff --git a/.gitmodules b/.gitmodules
index fbe2690c9ee5..30307592b565 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -78,3 +78,6 @@
 [submodule "src/qatlib"]
 	path = src/qatlib
 	url = https://github.com/intel/qatlib.git
+[submodule "src/qatzip"]
+	path = src/qatzip
+	url = https://github.com/intel/qatzip.git
diff --git a/cmake/modules/BuildQATzip.cmake b/cmake/modules/BuildQATzip.cmake
index 005288e873b2..91cb43c822de 100644
--- a/cmake/modules/BuildQATzip.cmake
+++ b/cmake/modules/BuildQATzip.cmake
@@ -1,9 +1,6 @@
 function(build_qatzip)
-  set(QATzip_REPO https://github.com/intel/qatzip.git)
-  set(QATzip_TAG "v1.1.2")
-
-  set(QATzip_SOURCE_DIR ${CMAKE_BINARY_DIR}/src/qatzip)
-  set(QATzip_INSTALL_DIR ${QATzip_SOURCE_DIR}/install)
+  set(QATzip_BINARY_DIR ${CMAKE_BINARY_DIR}/src/qatzip)
+  set(QATzip_INSTALL_DIR ${QATzip_BINARY_DIR}/install)
   set(QATzip_INCLUDE_DIR ${QATzip_INSTALL_DIR}/include)
   set(QATzip_LIBRARY ${QATzip_INSTALL_DIR}/lib/libqatzip.a)
 
@@ -28,20 +25,14 @@ function(build_qatzip)
   # because it messes with the internal install paths of arrow's bundled deps
   set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
 
-  set(source_dir_args
-    SOURCE_DIR ${QATzip_SOURCE_DIR}
-    GIT_REPOSITORY ${QATzip_REPO}
-    GIT_TAG ${QATzip_TAG}
-    GIT_SHALLOW TRUE
-    GIT_CONFIG advice.detachedHead=false)
-
   include(ExternalProject)
   ExternalProject_Add(qatzip_ext
-    ${source_dir_args}
+    SOURCE_DIR "${PROJECT_SOURCE_DIR}/src/qatzip"
     CONFIGURE_COMMAND ./autogen.sh COMMAND ${configure_cmd}
     BUILD_COMMAND ${NO_DESTDIR_COMMAND} make -j3
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS ${QATzip_LIBRARY}
+    INSTALL_COMMAND ${NO_DESTDIR_COMMAND} make install
     UPDATE_COMMAND ""
     LOG_CONFIGURE ON
     LOG_BUILD ON
diff --git a/src/qatzip b/src/qatzip
new file mode 160000
index 000000000000..fdee557b5bb6
--- /dev/null
+++ b/src/qatzip
@@ -0,0 +1 @@
+Subproject commit fdee557b5bb640827758f121102dcf3583292b7a

From 4e159853d25bd8b15490f81d210c2dbb0b8d0f87 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 13:42:45 -0500
Subject: [PATCH 1812/2492] cmake: WITH_QATZIP requires WITH_QATLIB or
 WITH_QATDRV

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c5d708f69de0..6bc48ee524fb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -330,6 +330,9 @@ elseif(WITH_QATLIB)
 endif()
 
 if(WITH_QATZIP)
+  if(NOT HAVE_QAT)
+    message(FATAL_ERROR "WITH_QATZIP requires WITH_QATLIB or WITH_QATDRV")
+  endif()
   if(NOT WITH_SYSTEM_QATZIP)
     include(BuildQATzip)
     build_qatzip()

From 15046cf1adbd0a530e353532b5ead12ccf712e7a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 24 Nov 2023 13:42:59 -0500
Subject: [PATCH 1813/2492] cmake: enable WITH_QATLIB and WITH_QATZIP by
 default

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6bc48ee524fb..b40a47d4c132 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -307,10 +307,10 @@ endif()
 
 option(WITH_BLUEFS "libbluefs library" OFF)
 
-option(WITH_QATLIB "Enable QAT with qatlib" OFF)
+option(WITH_QATLIB "Enable QAT with qatlib" ON)
 option(WITH_SYSTEM_QATLIB "Use system packages for qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
-option(WITH_QATZIP "Enable QATzip" OFF)
+option(WITH_QATZIP "Enable QATzip" ON)
 option(WITH_SYSTEM_QATZIP "Use system packages for QATzip" OFF)
 
 if(WITH_QATDRV)

From 7710719decea7d66f02dbedacbf45a5427611b4e Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 20:08:19 -0500
Subject: [PATCH 1814/2492] rpm: add system_qat option, default on for centos9

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 ceph.spec.in | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 1cb63f4c9db5..ed3674655277 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -111,6 +111,11 @@
 # this is tracked in https://bugzilla.redhat.com/2152265
 %bcond_with system_arrow
 %endif
+%if 0%{?fedora} || 0%{?rhel} >= 9
+%bcond_without system_qat
+%else
+%bcond_with system_qat
+%endif
 %if 0%{?fedora} || 0%{?suse_version} || 0%{?rhel} >= 8 || 0%{?openEuler}
 %global weak_deps 1
 %endif
@@ -332,6 +337,10 @@ BuildRequires:  libarrow-devel
 BuildRequires:  parquet-libs-devel
 BuildRequires:  utf8proc-devel
 %endif
+%if 0%{with system_qat}
+BuildRequires:  qatlib-devel
+BuildRequires:  qatzip-devel
+%endif
 %if 0%{with seastar}
 BuildRequires:  c-ares-devel
 BuildRequires:  gnutls-devel
@@ -1447,6 +1456,10 @@ cmake .. \
     -DWITH_SYSTEM_ARROW:BOOL=ON \
     -DWITH_SYSTEM_UTF8PROC:BOOL=ON \
 %endif
+%if 0%{with system_qat}
+    -DWITH_SYSTEM_QATLIB:BOOL=ON \
+    -DWITH_SYSTEM_QATZIP:BOOL=ON \
+%endif
 %if 0%{with seastar}
     -DWITH_SEASTAR:BOOL=ON \
     -DWITH_JAEGER:BOOL=OFF \

From ec2e4b62a932e084d3ff369dbb49b310c3bad886 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 27 Nov 2023 08:32:38 -0500
Subject: [PATCH 1815/2492] build/win32: disable qat options

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 win32_build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/win32_build.sh b/win32_build.sh
index 4c65e7ae2600..788f1db2381b 100755
--- a/win32_build.sh
+++ b/win32_build.sh
@@ -183,6 +183,7 @@ cmake -D CMAKE_PREFIX_PATH=$depsDirs \
       -D CMAKE_TOOLCHAIN_FILE="$MINGW_CMAKE_FILE" \
       -D WITH_FMT_HEADER_ONLY=ON \
       -D WITH_LIBCEPHSQLITE=OFF \
+      -D WITH_QATLIB=OFF -D WITH_QATZIP=OFF \
       -D WITH_RDMA=OFF -D WITH_OPENLDAP=OFF \
       -D WITH_GSSAPI=OFF -D WITH_XFS=OFF \
       -D WITH_FUSE=OFF -D WITH_DOKAN=ON \

From 5d2560762edca4a9006878b4e654b812f1d7d04d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 22 Nov 2023 23:32:40 -0500
Subject: [PATCH 1816/2492] compressor: move QatAccel out of common

move the QatAccel instance out of the Compressor base class and into
the zlib and lz4 compressors that can use it

this avoids having to link QAT into the ceph-common library, and only
the plugins where it's necessary

had to add LZ4Compressor.cc to store the new static variable

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/CMakeLists.txt                       |   5 -
 src/compressor/CMakeLists.txt            |  18 ++-
 src/compressor/Compressor.cc             |   4 -
 src/compressor/Compressor.h              |   8 --
 src/compressor/lz4/CMakeLists.txt        |   4 +
 src/compressor/lz4/LZ4Compressor.cc      | 149 +++++++++++++++++++++++
 src/compressor/lz4/LZ4Compressor.h       | 120 ++----------------
 src/compressor/snappy/SnappyCompressor.h |  18 ---
 src/compressor/zlib/CMakeLists.txt       |   3 +
 src/compressor/zlib/ZlibCompressor.cc    |  18 +++
 src/compressor/zlib/ZlibCompressor.h     |  15 ++-
 11 files changed, 200 insertions(+), 162 deletions(-)
 create mode 100644 src/compressor/lz4/LZ4Compressor.cc

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 23196619eb8f..df2f3b6c9735 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -505,11 +505,6 @@ if(NOT WITH_SYSTEM_BOOST)
   list(APPEND ceph_common_deps ${ZLIB_LIBRARIES})
 endif()
 
-if(HAVE_QATZIP)
-  # TODO: only the compression plugins should depend on QAT
-  list(APPEND ceph_common_deps QAT::zip)
-endif()
-
 if(WITH_DPDK)
   list(APPEND ceph_common_deps common_async_dpdk)
 endif()
diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt
index d9512e87408e..0da71aa1f1b4 100644
--- a/src/compressor/CMakeLists.txt
+++ b/src/compressor/CMakeLists.txt
@@ -1,19 +1,15 @@
-
-set(compressor_srcs
-  Compressor.cc)
-if (HAVE_QATZIP)
-  list(APPEND compressor_srcs QatAccel.cc)
-endif()
-add_library(compressor_objs OBJECT ${compressor_srcs})
+add_library(compressor_objs OBJECT Compressor.cc)
 add_dependencies(compressor_objs common-objs)
+add_dependencies(compressor_objs legacy-option-headers)
+
 if(HAVE_QATZIP AND HAVE_QAT)
-  target_link_libraries(compressor_objs PRIVATE
+  add_library(qat_compressor OBJECT QatAccel.cc)
+  target_link_libraries(qat_compressor PUBLIC
                         QAT::qat
                         QAT::usdm
                         QAT::zip
                        )
 endif()
-add_dependencies(compressor_objs legacy-option-headers)
 
 ## compressor plugins
 
@@ -31,8 +27,8 @@ if(HAVE_BROTLI)
   add_subdirectory(brotli)
 endif()
 
-add_library(compressor STATIC $<TARGET_OBJECTS:compressor_objs>)
-target_link_libraries(compressor PRIVATE compressor_objs)
+add_library(compressor STATIC)
+target_link_libraries(compressor PUBLIC compressor_objs)
 
 set(ceph_compressor_libs
     ceph_snappy
diff --git a/src/compressor/Compressor.cc b/src/compressor/Compressor.cc
index 43d34c8eb01e..a13dfb30ddc7 100644
--- a/src/compressor/Compressor.cc
+++ b/src/compressor/Compressor.cc
@@ -26,10 +26,6 @@
 
 namespace TOPNSPC {
 
-#ifdef HAVE_QATZIP
-  QatAccel Compressor::qat_accel;
-#endif
-
 const char* Compressor::get_comp_alg_name(int a) {
 
   auto p = std::find_if(std::cbegin(compression_algorithms), std::cend(compression_algorithms),
diff --git a/src/compressor/Compressor.h b/src/compressor/Compressor.h
index 276cd875a9a8..11f020a0dd24 100644
--- a/src/compressor/Compressor.h
+++ b/src/compressor/Compressor.h
@@ -23,9 +23,6 @@
 #include "include/common_fwd.h"
 #include "include/buffer.h"
 #include "include/int_types.h"
-#ifdef HAVE_QATZIP
-  #include "QatAccel.h"
-#endif
 
 namespace TOPNSPC {
 
@@ -70,11 +67,6 @@ class Compressor {
     COMP_FORCE                  ///< compress always
   };
 
-#ifdef HAVE_QATZIP
-  bool qat_enabled;
-  static QatAccel qat_accel;
-#endif
-
   static const char* get_comp_alg_name(int a);
   static std::optional<CompressionAlgorithm> get_comp_alg_type(std::string_view s);
 
diff --git a/src/compressor/lz4/CMakeLists.txt b/src/compressor/lz4/CMakeLists.txt
index ff8e14c298c7..316493435aa6 100644
--- a/src/compressor/lz4/CMakeLists.txt
+++ b/src/compressor/lz4/CMakeLists.txt
@@ -2,11 +2,15 @@
 
 set(lz4_sources
   CompressionPluginLZ4.cc
+  LZ4Compressor.cc
 )
 
 add_library(ceph_lz4 SHARED ${lz4_sources})
 target_link_libraries(ceph_lz4
   PRIVATE LZ4::LZ4 compressor $<$<PLATFORM_ID:Windows>:ceph-common>)
+if(HAVE_QATZIP AND HAVE_QAT)
+  target_link_libraries(ceph_lz4 PRIVATE qat_compressor)
+endif()
 set_target_properties(ceph_lz4 PROPERTIES
   VERSION 2.0.0
   SOVERSION 2
diff --git a/src/compressor/lz4/LZ4Compressor.cc b/src/compressor/lz4/LZ4Compressor.cc
new file mode 100644
index 000000000000..a209a5ac149f
--- /dev/null
+++ b/src/compressor/lz4/LZ4Compressor.cc
@@ -0,0 +1,149 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "LZ4Compressor.h"
+#include "common/ceph_context.h"
+#ifdef HAVE_QATZIP
+  #include "compressor/QatAccel.h"
+#endif
+
+#ifdef HAVE_QATZIP
+QatAccel LZ4Compressor::qat_accel;
+#endif
+
+LZ4Compressor::LZ4Compressor(CephContext* cct)
+  : Compressor(COMP_ALG_LZ4, "lz4")
+{
+#ifdef HAVE_QATZIP
+  if (cct->_conf->qat_compressor_enabled && qat_accel.init("lz4"))
+    qat_enabled = true;
+  else
+    qat_enabled = false;
+#endif
+}
+
+int LZ4Compressor::compress(const ceph::buffer::list &src,
+                            ceph::buffer::list &dst,
+                            std::optional<int32_t> &compressor_message)
+{
+  // older versions of liblz4 introduce bit errors when compressing
+  // fragmented buffers.  this was fixed in lz4 commit
+  // af127334670a5e7b710bbd6adb71aa7c3ef0cd72, which first
+  // appeared in v1.8.2.
+  //
+  // workaround: rebuild if not contiguous.
+  if (!src.is_contiguous()) {
+    ceph::buffer::list new_src = src;
+    new_src.rebuild();
+    return compress(new_src, dst, compressor_message);
+  }
+
+#ifdef HAVE_QATZIP
+  if (qat_enabled)
+    return qat_accel.compress(src, dst, compressor_message);
+#endif
+  ceph::buffer::ptr outptr = ceph::buffer::create_small_page_aligned(
+    LZ4_compressBound(src.length()));
+  LZ4_stream_t lz4_stream;
+  LZ4_resetStream(&lz4_stream);
+
+  using ceph::encode;
+
+  auto p = src.begin();
+  size_t left = src.length();
+  int pos = 0;
+  const char *data;
+  unsigned num = src.get_num_buffers();
+  encode((uint32_t)num, dst);
+  while (left) {
+    uint32_t origin_len = p.get_ptr_and_advance(left, &data);
+    int compressed_len = LZ4_compress_fast_continue(
+      &lz4_stream, data, outptr.c_str()+pos, origin_len,
+      outptr.length()-pos, 1);
+    if (compressed_len <= 0)
+      return -1;
+    pos += compressed_len;
+    left -= origin_len;
+    encode(origin_len, dst);
+    encode((uint32_t)compressed_len, dst);
+  }
+  ceph_assert(p.end());
+
+  dst.append(outptr, 0, pos);
+  return 0;
+}
+
+int LZ4Compressor::decompress(const ceph::buffer::list &src,
+                              ceph::buffer::list &dst,
+                              std::optional<int32_t> compressor_message)
+{
+#ifdef HAVE_QATZIP
+  if (qat_enabled)
+    return qat_accel.decompress(src, dst, compressor_message);
+#endif
+  auto i = std::cbegin(src);
+  return decompress(i, src.length(), dst, compressor_message);
+}
+
+int LZ4Compressor::decompress(ceph::buffer::list::const_iterator &p,
+                              size_t compressed_len,
+                              ceph::buffer::list &dst,
+                              std::optional<int32_t> compressor_message)
+{
+#ifdef HAVE_QATZIP
+  if (qat_enabled)
+    return qat_accel.decompress(p, compressed_len, dst, compressor_message);
+#endif
+  using ceph::decode;
+  uint32_t count;
+  decode(count, p);
+  std::vector<std::pair<uint32_t, uint32_t> > compressed_pairs(count);
+  uint32_t total_origin = 0;
+  for (auto& [dst_size, src_size] : compressed_pairs) {
+    decode(dst_size, p);
+    decode(src_size, p);
+    total_origin += dst_size;
+  }
+  compressed_len -= (sizeof(uint32_t) + sizeof(uint32_t) * count * 2);
+
+  ceph::buffer::ptr dstptr(total_origin);
+  LZ4_streamDecode_t lz4_stream_decode;
+  LZ4_setStreamDecode(&lz4_stream_decode, nullptr, 0);
+
+  ceph::buffer::ptr cur_ptr = p.get_current_ptr();
+  ceph::buffer::ptr *ptr = &cur_ptr;
+  std::optional<ceph::buffer::ptr> data_holder;
+  if (compressed_len != cur_ptr.length()) {
+    data_holder.emplace(compressed_len);
+    p.copy_deep(compressed_len, *data_holder);
+    ptr = &*data_holder;
+  }
+
+  char *c_in = ptr->c_str();
+  char *c_out = dstptr.c_str();
+  for (unsigned i = 0; i < count; ++i) {
+    int r = LZ4_decompress_safe_continue(
+        &lz4_stream_decode, c_in, c_out, compressed_pairs[i].second, compressed_pairs[i].first);
+    if (r == (int)compressed_pairs[i].first) {
+      c_in += compressed_pairs[i].second;
+      c_out += compressed_pairs[i].first;
+    } else if (r < 0) {
+      return -1;
+    } else {
+      return -2;
+    }
+  }
+  dst.push_back(std::move(dstptr));
+  return 0;
+}
diff --git a/src/compressor/lz4/LZ4Compressor.h b/src/compressor/lz4/LZ4Compressor.h
index eca08e1a57ac..6939aae7609a 100644
--- a/src/compressor/lz4/LZ4Compressor.h
+++ b/src/compressor/lz4/LZ4Compressor.h
@@ -23,125 +23,29 @@
 #include "include/encoding.h"
 #include "common/config.h"
 
+class QatAccel;
 
 class LZ4Compressor : public Compressor {
- public:
-  LZ4Compressor(CephContext* cct) : Compressor(COMP_ALG_LZ4, "lz4") {
 #ifdef HAVE_QATZIP
-    if (cct->_conf->qat_compressor_enabled && qat_accel.init("lz4"))
-      qat_enabled = true;
-    else
-      qat_enabled = false;
+  bool qat_enabled;
+  static QatAccel qat_accel;
 #endif
-  }
-
-  int compress(const ceph::buffer::list &src, ceph::buffer::list &dst, std::optional<int32_t> &compressor_message) override {
-    // older versions of liblz4 introduce bit errors when compressing
-    // fragmented buffers.  this was fixed in lz4 commit
-    // af127334670a5e7b710bbd6adb71aa7c3ef0cd72, which first
-    // appeared in v1.8.2.
-    //
-    // workaround: rebuild if not contiguous.
-    if (!src.is_contiguous()) {
-      ceph::buffer::list new_src = src;
-      new_src.rebuild();
-      return compress(new_src, dst, compressor_message);
-    }
 
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.compress(src, dst, compressor_message);
-#endif
-    ceph::buffer::ptr outptr = ceph::buffer::create_small_page_aligned(
-      LZ4_compressBound(src.length()));
-    LZ4_stream_t lz4_stream;
-    LZ4_resetStream(&lz4_stream);
-
-    using ceph::encode;
-
-    auto p = src.begin();
-    size_t left = src.length();
-    int pos = 0;
-    const char *data;
-    unsigned num = src.get_num_buffers();
-    encode((uint32_t)num, dst);
-    while (left) {
-      uint32_t origin_len = p.get_ptr_and_advance(left, &data);
-      int compressed_len = LZ4_compress_fast_continue(
-        &lz4_stream, data, outptr.c_str()+pos, origin_len,
-        outptr.length()-pos, 1);
-      if (compressed_len <= 0)
-        return -1;
-      pos += compressed_len;
-      left -= origin_len;
-      encode(origin_len, dst);
-      encode((uint32_t)compressed_len, dst);
-    }
-    ceph_assert(p.end());
+ public:
+  explicit LZ4Compressor(CephContext* cct);
 
-    dst.append(outptr, 0, pos);
-    return 0;
-  }
+  int compress(const ceph::buffer::list &src,
+               ceph::buffer::list &dst,
+               std::optional<int32_t> &compressor_message) override;
 
-  int decompress(const ceph::buffer::list &src, ceph::buffer::list &dst, std::optional<int32_t> compressor_message) override {
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.decompress(src, dst, compressor_message);
-#endif
-    auto i = std::cbegin(src);
-    return decompress(i, src.length(), dst, compressor_message);
-  }
+  int decompress(const ceph::buffer::list &src,
+                 ceph::buffer::list &dst,
+                 std::optional<int32_t> compressor_message) override;
 
   int decompress(ceph::buffer::list::const_iterator &p,
 		 size_t compressed_len,
 		 ceph::buffer::list &dst,
-		 std::optional<int32_t> compressor_message) override {
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.decompress(p, compressed_len, dst, compressor_message);
-#endif
-    using ceph::decode;
-    uint32_t count;
-    decode(count, p);
-    std::vector<std::pair<uint32_t, uint32_t> > compressed_pairs(count);
-    uint32_t total_origin = 0;
-    for (auto& [dst_size, src_size] : compressed_pairs) {
-      decode(dst_size, p);
-      decode(src_size, p);
-      total_origin += dst_size;
-    }
-    compressed_len -= (sizeof(uint32_t) + sizeof(uint32_t) * count * 2);
-
-    ceph::buffer::ptr dstptr(total_origin);
-    LZ4_streamDecode_t lz4_stream_decode;
-    LZ4_setStreamDecode(&lz4_stream_decode, nullptr, 0);
-
-    ceph::buffer::ptr cur_ptr = p.get_current_ptr();
-    ceph::buffer::ptr *ptr = &cur_ptr;
-    std::optional<ceph::buffer::ptr> data_holder;
-    if (compressed_len != cur_ptr.length()) {
-      data_holder.emplace(compressed_len);
-      p.copy_deep(compressed_len, *data_holder);
-      ptr = &*data_holder;
-    }
-
-    char *c_in = ptr->c_str();
-    char *c_out = dstptr.c_str();
-    for (unsigned i = 0; i < count; ++i) {
-      int r = LZ4_decompress_safe_continue(
-          &lz4_stream_decode, c_in, c_out, compressed_pairs[i].second, compressed_pairs[i].first);
-      if (r == (int)compressed_pairs[i].first) {
-        c_in += compressed_pairs[i].second;
-        c_out += compressed_pairs[i].first;
-      } else if (r < 0) {
-        return -1;
-      } else {
-        return -2;
-      }
-    }
-    dst.push_back(std::move(dstptr));
-    return 0;
-  }
+		 std::optional<int32_t> compressor_message) override;
 };
 
 #endif
diff --git a/src/compressor/snappy/SnappyCompressor.h b/src/compressor/snappy/SnappyCompressor.h
index 8150f783c157..b635581068ae 100644
--- a/src/compressor/snappy/SnappyCompressor.h
+++ b/src/compressor/snappy/SnappyCompressor.h
@@ -58,19 +58,9 @@ class CEPH_BUFFER_API BufferlistSource : public snappy::Source {
 class SnappyCompressor : public Compressor {
  public:
   SnappyCompressor(CephContext* cct) : Compressor(COMP_ALG_SNAPPY, "snappy") {
-#ifdef HAVE_QATZIP
-    if (cct->_conf->qat_compressor_enabled && qat_accel.init("snappy"))
-      qat_enabled = true;
-    else
-      qat_enabled = false;
-#endif
   }
 
   int compress(const ceph::bufferlist &src, ceph::bufferlist &dst, std::optional<int32_t> &compressor_message) override {
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.compress(src, dst, compressor_message);
-#endif
     BufferlistSource source(const_cast<ceph::bufferlist&>(src).begin(), src.length());
     ceph::bufferptr ptr = ceph::buffer::create_small_page_aligned(
       snappy::MaxCompressedLength(src.length()));
@@ -81,10 +71,6 @@ class SnappyCompressor : public Compressor {
   }
 
   int decompress(const ceph::bufferlist &src, ceph::bufferlist &dst, std::optional<int32_t> compressor_message) override {
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.decompress(src, dst, compressor_message);
-#endif
     auto i = src.begin();
     return decompress(i, src.length(), dst, compressor_message);
   }
@@ -93,10 +79,6 @@ class SnappyCompressor : public Compressor {
 		 size_t compressed_len,
 		 ceph::bufferlist &dst,
 		 std::optional<int32_t> compressor_message) override {
-#ifdef HAVE_QATZIP
-    if (qat_enabled)
-      return qat_accel.decompress(p, compressed_len, dst, compressor_message);
-#endif
     BufferlistSource source_1(p, compressed_len);
     uint32_t res_len = 0;
     if (!snappy::GetUncompressedLength(&source_1, &res_len)) {
diff --git a/src/compressor/zlib/CMakeLists.txt b/src/compressor/zlib/CMakeLists.txt
index 050ff03fa28f..3480ab068c96 100644
--- a/src/compressor/zlib/CMakeLists.txt
+++ b/src/compressor/zlib/CMakeLists.txt
@@ -91,6 +91,9 @@ endif()
 
 add_library(ceph_zlib SHARED ${zlib_sources})
 target_link_libraries(ceph_zlib ZLIB::ZLIB compressor $<$<PLATFORM_ID:Windows>:ceph-common>)
+if(HAVE_QATZIP AND HAVE_QAT)
+  target_link_libraries(ceph_zlib qat_compressor)
+endif()
 target_include_directories(ceph_zlib SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/isa-l/include")
 set_target_properties(ceph_zlib PROPERTIES
   VERSION 2.0.0
diff --git a/src/compressor/zlib/ZlibCompressor.cc b/src/compressor/zlib/ZlibCompressor.cc
index 9795d79b3ba7..2a0aa006901e 100644
--- a/src/compressor/zlib/ZlibCompressor.cc
+++ b/src/compressor/zlib/ZlibCompressor.cc
@@ -17,6 +17,9 @@
 #include "ZlibCompressor.h"
 #include "osd/osd_types.h"
 #include "isa-l/include/igzip_lib.h"
+#ifdef HAVE_QATZIP
+  #include "compressor/QatAccel.h"
+#endif
 // -----------------------------------------------------------------------------
 
 #include <zlib.h>
@@ -52,6 +55,21 @@ _prefix(std::ostream* _dout)
 // compression ratio.
 #define ZLIB_MEMORY_LEVEL 8
 
+#ifdef HAVE_QATZIP
+QatAccel ZlibCompressor::qat_accel;
+#endif
+
+ZlibCompressor::ZlibCompressor(CephContext *cct, bool isal)
+  : Compressor(COMP_ALG_ZLIB, "zlib"), isal_enabled(isal), cct(cct)
+{
+#ifdef HAVE_QATZIP
+  if (cct->_conf->qat_compressor_enabled && qat_accel.init("zlib"))
+    qat_enabled = true;
+  else
+    qat_enabled = false;
+#endif
+}
+
 int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out, std::optional<int32_t> &compressor_message)
 {
   int ret;
diff --git a/src/compressor/zlib/ZlibCompressor.h b/src/compressor/zlib/ZlibCompressor.h
index da1c8117e882..33b3ea4d4603 100644
--- a/src/compressor/zlib/ZlibCompressor.h
+++ b/src/compressor/zlib/ZlibCompressor.h
@@ -20,19 +20,18 @@
 #include "common/config.h"
 #include "compressor/Compressor.h"
 
+class QatAccel;
+
 class ZlibCompressor : public Compressor {
   bool isal_enabled;
   CephContext *const cct;
-public:
-  ZlibCompressor(CephContext *cct, bool isal)
-    : Compressor(COMP_ALG_ZLIB, "zlib"), isal_enabled(isal), cct(cct) {
 #ifdef HAVE_QATZIP
-    if (cct->_conf->qat_compressor_enabled && qat_accel.init("zlib"))
-      qat_enabled = true;
-    else
-      qat_enabled = false;
+  bool qat_enabled;
+  static QatAccel qat_accel;
 #endif
-  }
+
+ public:
+  ZlibCompressor(CephContext *cct, bool isal);
 
   int compress(const ceph::buffer::list &in, ceph::buffer::list &out, std::optional<int32_t> &compressor_message) override;
   int decompress(const ceph::buffer::list &in, ceph::buffer::list &out, std::optional<int32_t> compressor_message) override;

From 84af8357d1505c1b52cada7e4cdd4ad6e79e2382 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 27 Nov 2023 10:42:06 -0500
Subject: [PATCH 1817/2492] cmake: WITH_QATLIB/ZIP disabled on aarch64

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 CMakeLists.txt               | 6 ++++--
 cmake/modules/BuildQAT.cmake | 2 --
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b40a47d4c132..d70d471ab819 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -307,10 +307,12 @@ endif()
 
 option(WITH_BLUEFS "libbluefs library" OFF)
 
-option(WITH_QATLIB "Enable QAT with qatlib" ON)
+CMAKE_DEPENDENT_OPTION(WITH_QATLIB "Enable QAT with qatlib" ON
+  "NOT CMAKE_SYSTEM_PROCESSOR MATCHES aarch64" OFF)
 option(WITH_SYSTEM_QATLIB "Use system packages for qatlib" OFF)
 option(WITH_QATDRV "Enable QAT with out-of-tree driver" OFF)
-option(WITH_QATZIP "Enable QATzip" ON)
+CMAKE_DEPENDENT_OPTION(WITH_QATZIP "Enable QATzip" ON
+  "NOT CMAKE_SYSTEM_PROCESSOR MATCHES aarch64" OFF)
 option(WITH_SYSTEM_QATZIP "Use system packages for QATzip" OFF)
 
 if(WITH_QATDRV)
diff --git a/cmake/modules/BuildQAT.cmake b/cmake/modules/BuildQAT.cmake
index 737c63c3ea8c..d65d07639dc1 100644
--- a/cmake/modules/BuildQAT.cmake
+++ b/cmake/modules/BuildQAT.cmake
@@ -13,8 +13,6 @@ function(build_qat)
   set(configure_cmd env CC=${CMAKE_C_COMPILER} ./configure --prefix=${QAT_INSTALL_DIR})
   # disable systemd or 'make install' tries to write /usr/lib/systemd/system/qat.service
   list(APPEND configure_cmd --disable-systemd)
-  # samples don't build on arm64
-  list(APPEND configure_cmd --disable-samples)
   # build a static library with -fPIC that we can link into crypto/compressor plugins
   list(APPEND configure_cmd --with-pic --enable-static --disable-shared)
 

From c7ad4c1e7020767abd889664d15716182d081680 Mon Sep 17 00:00:00 2001
From: Jane Zhu <jzhu116@bloomberg.net>
Date: Wed, 7 Feb 2024 14:12:23 -0500
Subject: [PATCH 1818/2492] rgw: do not copy olh attributes in versioning
 suspended bucket

Signed-off-by: Juan Zhu <jzhu4@dev-10-34-20-139.pw1.bcc.bloomberg.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b802bb114bd7..7925cddf86fc 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -4680,6 +4680,12 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   if (lh != attrs.end())
     src_attrs[RGW_ATTR_OBJECT_LEGAL_HOLD] = lh->second;
 
+  if (dest_bucket_info.flags & BUCKET_VERSIONS_SUSPENDED) {
+    src_attrs.erase(RGW_ATTR_OLH_ID_TAG);
+    src_attrs.erase(RGW_ATTR_OLH_INFO);
+    src_attrs.erase(RGW_ATTR_OLH_VER);
+  }
+
   set_copy_attrs(src_attrs, attrs, attrs_mod);
   attrs.erase(RGW_ATTR_ID_TAG);
   attrs.erase(RGW_ATTR_PG_VER);

From 05d54ca52c07ca61f584de7b307af546135f8f4b Mon Sep 17 00:00:00 2001
From: Prashant D <pdhange@redhat.com>
Date: Tue, 17 Jan 2023 19:31:54 -0500
Subject: [PATCH 1819/2492] common/LogEntry: Reintroduce human readable log
 level for clog messages

In {fmt} v9, fmt::formatter<> is not getting defined automatically for
the clog_type with operator<<, we need to define it explictly.
Allow fmt::formatter<> to print clog prio using {fmt}.

Fixes: https://tracker.ceph.com/issues/58436

Signed-off-by: Prashant D <pdhange@redhat.com>
---
 src/common/LogEntry.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/common/LogEntry.h b/src/common/LogEntry.h
index 3ddebbd3043c..15d83fb956e6 100644
--- a/src/common/LogEntry.h
+++ b/src/common/LogEntry.h
@@ -194,6 +194,10 @@ inline std::ostream& operator<<(std::ostream& out, const LogEntry& e)
              << e.channel << " " << e.prio << " " << e.msg;
 }
 
+#if FMT_VERSION >= 90000
+template <> struct fmt::formatter<clog_type> : fmt::ostream_formatter {};
+#endif
+
 template <> struct fmt::formatter<EntityName> : fmt::formatter<std::string_view> {
   template <typename FormatContext>
   auto format(const EntityName& e, FormatContext& ctx) {

From 6fce88edb0d524142b16182b0e74f9c9348b278d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 7 Feb 2024 16:08:49 -0500
Subject: [PATCH 1820/2492] qa/rgw: hadoop-s3a uses supported-random-distro$

without this, teuthology-suite tries to schedule against centos8 which
is not supported for squid

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/hadoop-s3a/supported-random-distro$ | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 qa/suites/rgw/hadoop-s3a/supported-random-distro$

diff --git a/qa/suites/rgw/hadoop-s3a/supported-random-distro$ b/qa/suites/rgw/hadoop-s3a/supported-random-distro$
new file mode 120000
index 000000000000..0862b4457b37
--- /dev/null
+++ b/qa/suites/rgw/hadoop-s3a/supported-random-distro$
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$
\ No newline at end of file

From 079f722c37ef6cc8bd3cc26c49ae119dd83431f9 Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Wed, 2 Aug 2023 08:08:39 +0530
Subject: [PATCH 1821/2492] mgr/volumes: support to reject CephFS clones if
 cloner threads are not available

CephFS clone creation have a limit of 4 parallel clones by default at a time and rest
of the clone create requests are queued. This makes CephFS cloning very slow when
there is large amount of clones being created.After this patch clone requests won't be accepeted
when the requests exceed the `max_concurrent_clones` config value.

Fixes:  https://tracker.ceph.com/issues/59714
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 src/pybind/mgr/volumes/fs/async_cloner.py     |  6 ++++-
 .../mgr/volumes/fs/operations/volume.py       | 27 ++++++++++++++++++-
 src/pybind/mgr/volumes/fs/volume.py           | 13 ++++++---
 src/pybind/mgr/volumes/module.py              | 10 ++++++-
 4 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py
index 146d2e755902..685b2f03c78c 100644
--- a/src/pybind/mgr/volumes/fs/async_cloner.py
+++ b/src/pybind/mgr/volumes/fs/async_cloner.py
@@ -337,9 +337,10 @@ class Cloner(AsyncJobs):
     this relies on a simple state machine (which mimics states from SubvolumeOpSm class) as
     the driver. file types supported are directories, symbolic links and regular files.
     """
-    def __init__(self, volume_client, tp_size, snapshot_clone_delay):
+    def __init__(self, volume_client, tp_size, snapshot_clone_delay, clone_no_wait):
         self.vc = volume_client
         self.snapshot_clone_delay = snapshot_clone_delay
+        self.snapshot_clone_no_wait = clone_no_wait
         self.state_table = {
             SubvolumeStates.STATE_PENDING      : handle_clone_pending,
             SubvolumeStates.STATE_INPROGRESS   : handle_clone_in_progress,
@@ -355,6 +356,9 @@ def reconfigure_max_concurrent_clones(self, tp_size):
     def reconfigure_snapshot_clone_delay(self, timeout):
         self.snapshot_clone_delay = timeout
 
+    def reconfigure_reject_clones(self, clone_no_wait):
+        self.snapshot_clone_no_wait = clone_no_wait
+
     def is_clone_cancelable(self, clone_state):
         return not (SubvolumeOpSm.is_complete_state(clone_state) or SubvolumeOpSm.is_failed_state(clone_state))
 
diff --git a/src/pybind/mgr/volumes/fs/operations/volume.py b/src/pybind/mgr/volumes/fs/operations/volume.py
index 67fbb891cefb..0bf42827161e 100644
--- a/src/pybind/mgr/volumes/fs/operations/volume.py
+++ b/src/pybind/mgr/volumes/fs/operations/volume.py
@@ -9,11 +9,12 @@
 import orchestrator
 
 from .lock import GlobalLock
-from ..exception import VolumeException
+from ..exception import VolumeException, IndexException
 from ..fs_util import create_pool, remove_pool, rename_pool, create_filesystem, \
     remove_filesystem, rename_filesystem, create_mds, volume_exists, listdir
 from .trash import Trash
 from mgr_util import open_filesystem, CephfsConnectionException
+from .clone_index import open_clone_index
 
 log = logging.getLogger(__name__)
 
@@ -260,6 +261,30 @@ def get_pending_subvol_deletions_count(fs, path):
     return {'pending_subvolume_deletions': num_pending_subvol_del}
 
 
+def get_all_pending_clones_count(self, mgr, vol_spec):
+    pending_clones_cnt = 0
+    index_path = ""
+    fs_map = mgr.get('fs_map')
+    for fs in fs_map['filesystems']:
+        volname = fs['mdsmap']['fs_name']
+        try:
+            with open_volume(self, volname) as fs_handle:
+                with open_clone_index(fs_handle, vol_spec) as index:
+                    index_path = index.path.decode('utf-8')
+                    pending_clones_cnt = pending_clones_cnt \
+                                            + len(listdir(fs_handle, index_path,
+                                                          filter_entries=None, filter_files=False))
+        except IndexException as e:
+            if e.errno == -errno.ENOENT:
+                continue
+            raise VolumeException(-e.args[0], e.args[1])
+        except VolumeException as ve:
+            log.error("error fetching clone entry for volume '{0}' ({1})".format(volname, ve))
+            raise ve
+
+    return pending_clones_cnt
+
+
 @contextmanager
 def open_volume(vc, volname):
     """
diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index c896fd73d0b0..0c4a07598054 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -13,12 +13,14 @@
 from .operations.group import open_group, create_group, remove_group, \
     open_group_unique, set_group_attrs
 from .operations.volume import create_volume, delete_volume, rename_volume, \
-    list_volumes, open_volume, get_pool_names, get_pool_ids, get_pending_subvol_deletions_count
+    list_volumes, open_volume, get_pool_names, get_pool_ids, \
+    get_pending_subvol_deletions_count, get_all_pending_clones_count
 from .operations.subvolume import open_subvol, create_subvol, remove_subvol, \
     create_clone
 
 from .vol_spec import VolSpec
-from .exception import VolumeException, ClusterError, ClusterTimeout, EvictionError
+from .exception import VolumeException, ClusterError, ClusterTimeout, \
+    EvictionError, IndexException
 from .async_cloner import Cloner
 from .purge_queue import ThreadPoolPurgeQueueMixin
 from .operations.template import SubvolumeOpType
@@ -53,7 +55,8 @@ def __init__(self, mgr):
         super().__init__(mgr)
         # volume specification
         self.volspec = VolSpec(mgr.rados.conf_get('client_snapdir'))
-        self.cloner = Cloner(self, self.mgr.max_concurrent_clones, self.mgr.snapshot_clone_delay)
+        self.cloner = Cloner(self, self.mgr.max_concurrent_clones, self.mgr.snapshot_clone_delay,
+                             self.mgr.snapshot_clone_no_wait)
         self.purge_queue = ThreadPoolPurgeQueueMixin(self, 4)
         # on startup, queue purge job for available volumes to kickstart
         # purge for leftover subvolume entries in trash. note that, if the
@@ -764,6 +767,10 @@ def clone_subvolume_snapshot(self, **kwargs):
         s_groupname  = kwargs['group_name']
 
         try:
+            if self.mgr.snapshot_clone_no_wait and \
+               get_all_pending_clones_count(self, self.mgr, self.volspec) >= self.mgr.max_concurrent_clones:
+                raise(VolumeException(-errno.EAGAIN, "all cloner threads are busy, please try again later"))
+            
             with open_volume(self, volname) as fs_handle:
                 with open_group(fs_handle, self.volspec, s_groupname) as s_group:
                     with open_subvol(self.mgr, fs_handle, self.volspec, s_group, s_subvolname, SubvolumeOpType.CLONE_SOURCE) as s_subvolume:
diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index 68031ed55a3b..4a28fdc869ea 100644
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -489,7 +489,12 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
             'periodic_async_work',
             type='bool',
             default=False,
-            desc='Periodically check for async work')
+            desc='Periodically check for async work'),
+        Option(
+            'snapshot_clone_no_wait',
+            type='bool',
+            default=True,
+            desc='Reject subvolume clone request when cloner threads are busy')
     ]
 
     def __init__(self, *args, **kwargs):
@@ -498,6 +503,7 @@ def __init__(self, *args, **kwargs):
         self.max_concurrent_clones = None
         self.snapshot_clone_delay = None
         self.periodic_async_work = False
+        self.snapshot_clone_no_wait = None
         self.lock = threading.Lock()
         super(Module, self).__init__(*args, **kwargs)
         # Initialize config option members
@@ -532,6 +538,8 @@ def config_notify(self):
                         else:
                             self.vc.cloner.unset_wakeup_timeout()
                             self.vc.purge_queue.unset_wakeup_timeout()
+                    elif opt['name'] == "snapshot_clone_no_wait":
+                        self.vc.cloner.reconfigure_reject_clones(self.snapshot_clone_no_wait)
 
     def handle_command(self, inbuf, cmd):
         handler_name = "_cmd_" + cmd['prefix'].replace(" ", "_")

From 5247d7d19455a5ab7d1bb56b9586c6b08843b15d Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Tue, 6 Feb 2024 15:13:58 +0530
Subject: [PATCH 1822/2492] mgr/dashboard: fix error while accessing roles tab
 when policy attached

Fixes https://tracker.ceph.com/issues/64270

Issue:
======
Accessing Object->Users-Roles tab causing 500 internal servor error.
This is due to the "PermissionPolicies" which are attached to role and
backend was not handling this field for rgw roles.

Fix:
====
Added "PermissionPolicies" as the valid field in backend and updated
frontend to render the attached policy in formatted JSON

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   |  6 ++--
 .../crud-table/crud-table.component.html      |  2 +-
 .../crud-table/crud-table.component.ts        | 34 +++++++++++++++++--
 .../app/shared/models/crud-table-metadata.ts  |  1 +
 4 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index e7e0da181a6f..92148aaf3ac7 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -850,9 +850,10 @@ def model(role_name: str):
         "CreateDate": {'cellTemplate': 'date'},
         "MaxSessionDuration": {'cellTemplate': 'duration'},
         "RoleId": {'isHidden': True},
-        "AssumeRolePolicyDocument": {'isHidden': True}
+        "AssumeRolePolicyDocument": {'isHidden': True},
+        "PermissionPolicies": {'isHidden': True}
     },
-    detail_columns=['RoleId', 'AssumeRolePolicyDocument'],
+    detail_columns=['RoleId', 'AssumeRolePolicyDocument', 'PermissionPolicies'],
     meta=CRUDMeta()
 )
 class RgwUserRole(NamedTuple):
@@ -863,6 +864,7 @@ class RgwUserRole(NamedTuple):
     CreateDate: str
     MaxSessionDuration: int
     AssumeRolePolicyDocument: str
+    PermissionPolicies: List
 
 
 @APIRouter('/rgw/realm', Scope.RGW)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.html
index 7e1a7f2b3418..a1edf253c012 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.html
@@ -36,7 +36,7 @@
           <tr *ngFor="let column of meta.detail_columns">
             <td i18n
                 class="bold">{{ column }}</td>
-            <td> {{ expandedRow[column] }} </td>
+            <td><pre>{{ expandedRow[column] }}</pre></td>
           </tr>
         </tbody>
       </table>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
index 6881e373b588..098a454b1d7b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
@@ -39,7 +39,7 @@ export class CRUDTableComponent implements OnInit {
   permissions: Permissions;
   permission: Permission;
   selection = new CdTableSelection();
-  expandedRow: any = null;
+  expandedRow: { [key: string]: any } = {};
   modalRef: NgbModalRef;
   tabs = {};
   resource: string;
@@ -145,7 +145,11 @@ export class CRUDTableComponent implements OnInit {
   }
 
   setExpandedRow(event: any) {
-    this.expandedRow = event;
+    for (let i = 0; i < this.meta.detail_columns.length; i++) {
+      let column = this.meta.detail_columns[i];
+      let columnDetail = event[column];
+      this.expandedRow[column] = this.formatColumnDetails(columnDetail);
+    }
   }
 
   edit() {
@@ -176,4 +180,30 @@ export class CRUDTableComponent implements OnInit {
       this.modalRef = this.modalService.show(ConfirmationModalComponent, modalVariables);
     });
   }
+
+  /**
+   * Custom string replacer function for JSON.stringify
+   *
+   * This is specifically for objects inside an array.
+   * The custom replacer recursively stringifies deep nested objects
+   **/
+  stringReplacer(_key: string, value: any) {
+    try {
+      const parsedValue = JSON.parse(value);
+      return parsedValue;
+    } catch (e) {
+      return value;
+    }
+  }
+
+  /**
+   * returns a json string for arrays and string
+   * returns the same value for the rest
+   **/
+  formatColumnDetails(details: any) {
+    if (Array.isArray(details) || typeof details === 'string') {
+      return JSON.stringify(details, this.stringReplacer, 2);
+    }
+    return details;
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
index dc33e6236ae2..fb6970d1ccbb 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/crud-table-metadata.ts
@@ -15,4 +15,5 @@ export class CrudMetadata {
   forms: any;
   columnKey: string;
   resource: string;
+  detail_columns: string[];
 }

From 798d64d0ab5367241a9b8d904b4cef30184f2fe1 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Fri, 22 Sep 2023 12:24:07 +0200
Subject: [PATCH 1823/2492] os/bluestore: dont make SharedBlob always

Instead of creating a SharedBlob for each Blob instance, we just move
necesary fields like Collection to Blob and create SharedBlob only when
necessary.

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 53 ++++++++++++++++++-----------------
 src/os/bluestore/BlueStore.h  | 33 ++++++++++++++++------
 2 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index a27b18e84196..e3e07f43e57c 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -582,7 +582,7 @@ void _dump_extent_map(CephContext *cct, const BlueStore::ExtentMap &em)
       dout(LogLevelV) << __func__ << "      csum: " << std::hex << v << std::dec
 		      << dendl;
     }
-    std::lock_guard l(e.blob->shared_blob->get_cache()->lock);
+    std::lock_guard l(e.blob->get_cache()->lock);
     for (auto& i : e.blob->get_bc().buffer_map) {
       dout(LogLevelV) << __func__ << "       0x" << std::hex << i.first
 		      << "~" << i.second->length << std::dec
@@ -2228,7 +2228,7 @@ void BlueStore::Blob::discard_unallocated(Collection *coll)
     ceph_assert(discard == all_invalid); // in case of compressed blob all
 				    // or none pextents are invalid.
     if (discard) {
-      dirty_bc().discard(shared_blob->get_cache(), 0,
+      dirty_bc().discard(get_cache(), 0,
                               get_blob().get_logical_length());
     }
   } else {
@@ -2238,7 +2238,7 @@ void BlueStore::Blob::discard_unallocated(Collection *coll)
 	dout(20) << __func__ << " 0x" << std::hex << pos
 		 << "~" << e.length
 		 << std::dec << dendl;
-	dirty_bc().discard(shared_blob->get_cache(), pos, e.length);
+	dirty_bc().discard(get_cache(), pos, e.length);
       }
       pos += e.length;
     }
@@ -2891,7 +2891,7 @@ uint32_t BlueStore::Blob::merge_blob(CephContext* cct, Blob* blob_to_dissolve)
 void BlueStore::Blob::finish_write(uint64_t seq)
 {
   while (true) {
-    auto coll = shared_blob->coll;
+    auto coll = get_collection();
     BufferCacheShard *cache = coll->cache;
     std::lock_guard l(cache->lock);
     if (coll->cache != cache) {
@@ -2920,7 +2920,7 @@ void BlueStore::Blob::split(Collection *coll, uint32_t blob_offset, Blob *r)
     &(r->used_in_blob));
 
   lb.split(blob_offset, rb);
-  dirty_bc().split(shared_blob->get_cache(), blob_offset, r->dirty_bc());
+  dirty_bc().split(get_cache(), blob_offset, r->dirty_bc());
 
   dout(10) << __func__ << " 0x" << std::hex << blob_offset << std::dec
 	   << " finish " << *this << dendl;
@@ -3237,7 +3237,7 @@ void BlueStore::ExtentMap::dup(BlueStore* b, TransContext* txc,
       e.blob->dup(*cb);
       // By default do not copy buffers to clones, and let them read data by themselves.
       // The exception are 'writing' buffers, which are not yet stable on device.
-      bool some_copied = e.blob->bc._dup_writing(cb->shared_blob->get_cache(), &cb->bc);
+      bool some_copied = e.blob->bc._dup_writing(cb->get_cache(), &cb->bc);
       if (some_copied) {
 	// Pretend we just wrote those buffers;
 	// we need to get _finish_write called, so we can clear then from writing list.
@@ -3343,7 +3343,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
       // dup the blob
       const bluestore_blob_t& blob = e.blob->get_blob();
       ceph_assert(blob.is_shared());
-      ceph_assert(e.blob->shared_blob->is_loaded());
+      ceph_assert(e.blob->is_loaded());
       ceph_assert(!blob.has_unused());
       cb = new Blob();
       e.blob->last_encoded_id = n;
@@ -3364,7 +3364,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
       }
       // By default do not copy buffers to clones, and let them read data by themselves.
       // The exception are 'writing' buffers, which are not yet stable on device.
-      bool some_copied = e.blob->bc._dup_writing(cb->shared_blob->get_cache(), &cb->bc);
+      bool some_copied = e.blob->bc._dup_writing(cb->get_cache(), &cb->bc);
       if (some_copied) {
 	// Pretend we just wrote those buffers;
 	// we need to get _finish_write called, so we can clear then from writing list.
@@ -3920,10 +3920,11 @@ bool BlueStore::ExtentMap::encode_some(
       denc_varint(0, bound); // len
       denc_varint(0, bound); // blob_offset
 
+      dout(1) << "Pere blob " << p->blob << dendl;
       p->blob->bound_encode(
         bound,
         struct_v,
-        p->blob->shared_blob->get_sbid(),
+        p->blob->get_sbid(),
         false);
     }
   }
@@ -3983,7 +3984,7 @@ bool BlueStore::ExtentMap::encode_some(
       }
       pos = p->logical_end();
       if (include_blob) {
-	p->blob->encode(app, struct_v, p->blob->shared_blob->get_sbid(), false);
+	p->blob->encode(app, struct_v, p->blob->get_sbid(), false);
       }
     }
   }
@@ -4148,7 +4149,7 @@ void BlueStore::ExtentMap::bound_encode_spanning_blobs(size_t& p)
   denc_varint((uint32_t)0, key_size);
   p += spanning_blob_map.size() * key_size;
   for (const auto& i : spanning_blob_map) {
-    i.second->bound_encode(p, struct_v, i.second->shared_blob->get_sbid(), true);
+    i.second->bound_encode(p, struct_v, i.second->get_sbid(), true);
   }
 }
 
@@ -4164,7 +4165,7 @@ void BlueStore::ExtentMap::encode_spanning_blobs(
   denc_varint(spanning_blob_map.size(), p);
   for (auto& i : spanning_blob_map) {
     denc_varint(i.second->id, p);
-    i.second->encode(p, struct_v, i.second->shared_blob->get_sbid(), true);
+    i.second->encode(p, struct_v, i.second->get_sbid(), true);
   }
 }
 
@@ -4513,7 +4514,7 @@ BlueStore::ExtentMap::debug_list_disk_layout()
 
     bluestore_extent_ref_map_t* ref_map = nullptr;
     if (bblob.is_shared()) {
-      ceph_assert(ep->blob->shared_blob->is_loaded());
+      ceph_assert(ep->blob->is_loaded());
       bluestore_shared_blob_t* bsblob = ep->blob->shared_blob->persistent;
       ref_map = &bsblob->ref_map;
     }
@@ -4984,7 +4985,7 @@ void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
   ceph_assert(!b->shared_blob);
   const bluestore_blob_t& blob = b->get_blob();
   if (!blob.is_shared()) {
-    b->set_shared_blob(new SharedBlob(this));
+    b->collection = this;
     return;
   }
 
@@ -5030,7 +5031,8 @@ void BlueStore::Collection::load_shared_blob(SharedBlobRef sb)
 void BlueStore::Collection::make_blob_shared(uint64_t sbid, BlobRef b)
 {
   ldout(store->cct, 10) << __func__ << " " << *b << dendl;
-  ceph_assert(!b->shared_blob->is_loaded());
+  ceph_assert(!b->is_loaded());
+  ceph_assert(!b->shared_blob);
 
   // update blob
   bluestore_blob_t& blob = b->dirty_blob();
@@ -5038,6 +5040,7 @@ void BlueStore::Collection::make_blob_shared(uint64_t sbid, BlobRef b)
   // drop any unused parts, unlikely we could use them in future
   blob.clear_flag(bluestore_blob_t::FLAG_HAS_UNUSED);
   // update shared blob
+  b->shared_blob = new SharedBlob(sbid, this);
   b->shared_blob->loaded = true;
   b->shared_blob->persistent = new bluestore_shared_blob_t(sbid);
   shared_blob_set.add(this, b->shared_blob.get());
@@ -8988,7 +8991,7 @@ void BlueStore::_fsck_foreach_shared_blob(
       for (auto& e : o->extent_map.extent_map) {
 	auto& b = e.blob->get_blob();
 	if (b.is_shared() && passed_sbs.count(e.blob) == 0) {
-	  auto sbid = e.blob->shared_blob->get_sbid();
+	  auto sbid = e.blob->get_sbid();
 	  if (cb(c->cid, oid, sbid, b) == false) {
 	    goto stop_iterating;
 	  }
@@ -9244,12 +9247,12 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow(
       }
     }
     if (blob.is_shared()) {
-      if (i.first->shared_blob->get_sbid() > blobid_max) {
+      if (i.first->get_sbid() > blobid_max) {
         derr << "fsck error: " << oid << " blob " << blob
-          << " sbid " << i.first->shared_blob->get_sbid() << " > blobid_max "
+          << " sbid " << i.first->get_sbid() << " > blobid_max "
           << blobid_max << dendl;
         ++errors;
-      } else if (i.first->shared_blob->get_sbid() == 0) {
+      } else if (i.first->get_sbid() == 0) {
         derr << "fsck error: " << oid << " blob " << blob
           << " marked as shared but has uninitialized sbid"
           << dendl;
@@ -9259,8 +9262,8 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow(
       if (sb_info_lock) {
         sb_info_lock->lock();
       }
-      auto sbid = i.first->shared_blob->get_sbid();
-      sb_info_t& sbi = sb_info.add_or_adopt(i.first->shared_blob->get_sbid());
+      auto sbid = i.first->get_sbid();
+      sb_info_t& sbi = sb_info.add_or_adopt(i.first->get_sbid());
       ceph_assert(sbi.pool_id == sb_info_t::INVALID_POOL_ID ||
         sbi.pool_id == oid.hobj.get_logical_pool());
       sbi.pool_id = oid.hobj.get_logical_pool();
@@ -10521,7 +10524,7 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair)
 	  if (b->get_blob().is_shared()) {
             b->dirty_blob().clear_flag(bluestore_blob_t::FLAG_SHARED);
 
-	    auto sbid = b->shared_blob->get_sbid();
+	    auto sbid = b->get_sbid();
 	    auto sb_it = sb_info.find(sbid);
 	    ceph_assert(sb_it != sb_info.end());
 	    sb_info_t& sbi = *sb_it;
@@ -11629,7 +11632,7 @@ void BlueStore::_read_cache(
     ready_regions_t cache_res;
     interval_set<uint32_t> cache_interval;
     bptr->dirty_bc().read(
-      bptr->shared_blob->get_cache(), b_off, b_len, cache_res, cache_interval,
+      bptr->get_cache(), b_off, b_len, cache_res, cache_interval,
       read_cache_policy);
     dout(20) << __func__ << "  blob " << *bptr << std::hex
              << " need 0x" << b_off << "~" << b_len
@@ -11796,7 +11799,7 @@ int BlueStore::_generate_read_result_bl(
       if (r < 0)
         return r;
       if (buffered) {
-        bptr->dirty_bc().did_read(bptr->shared_blob->get_cache(), 0,
+        bptr->dirty_bc().did_read(bptr->get_cache(), 0,
                                        raw_bl);
       }
       for (auto& req : r2r) {
@@ -11813,7 +11816,7 @@ int BlueStore::_generate_read_result_bl(
           return -EIO;
         }
         if (buffered) {
-          bptr->dirty_bc().did_read(bptr->shared_blob->get_cache(),
+          bptr->dirty_bc().did_read(bptr->get_cache(),
                                          req.r_off, req.bl);
         }
 
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 9104f4d07f44..7407963cd39a 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -606,13 +606,17 @@ class BlueStore : public ObjectStore,
     int16_t id = -1;                ///< id, for spanning blobs only, >= 0
     int16_t last_encoded_id = -1;   ///< (ephemeral) used during encoding only
     SharedBlobRef shared_blob;      ///< shared blob state (if any)
+    bool loaded = false;
+    CollectionRef collection;
+    uint64_t sbid_unloaded = 0;
 
     void set_shared_blob(SharedBlobRef sb) {
       ceph_assert((bool)sb);
       ceph_assert(!shared_blob);
+      sbid_unloaded = sb->get_sbid();
       shared_blob = sb;
-      ceph_assert(shared_blob->get_cache());
-      shared_blob->get_cache()->add_blob();
+      ceph_assert(get_cache());
+      get_cache()->add_blob();
     }
     BufferSpace bc;
   private:
@@ -648,8 +652,8 @@ class BlueStore : public ObjectStore,
       return id >= 0;
     }
 
-    bool can_split() const {
-      std::lock_guard l(shared_blob->get_cache()->lock);
+    bool can_split() {
+      std::lock_guard l(get_cache()->lock);
       // splitting a BufferSpace writing list is too hard; don't try.
       return get_bc().writing.empty() &&
              used_in_blob.can_split() &&
@@ -722,6 +726,19 @@ class BlueStore : public ObjectStore,
       if (--nref == 0)
 	delete this;
     }
+    bool is_loaded() const {
+      return shared_blob && shared_blob->is_loaded();
+    }
+    inline BufferCacheShard* get_cache() {
+      return shared_blob ? shared_blob->get_cache() : collection->cache;
+    }
+    uint64_t get_sbid() const {
+      return shared_blob ? shared_blob->get_sbid() : sbid_unloaded;
+    }
+    CollectionRef get_collection() const {
+      return shared_blob ? shared_blob->coll : collection;
+    }
+
     ~Blob();
 
 #ifdef CACHE_BLOB_BL
@@ -824,7 +841,7 @@ class BlueStore : public ObjectStore,
     }
     ~Extent() {
       if (blob) {
-	blob->shared_blob->get_cache()->rm_extent();
+	blob->get_cache()->rm_extent();
       }
     }
 
@@ -833,7 +850,7 @@ class BlueStore : public ObjectStore,
     void assign_blob(const BlobRef& b) {
       ceph_assert(!blob);
       blob = b;
-      blob->shared_blob->get_cache()->add_extent();
+      blob->get_cache()->add_extent();
     }
 
     // comparators for intrusive_set
@@ -1610,7 +1627,7 @@ class BlueStore : public ObjectStore,
 
     BlobRef new_blob() {
       BlobRef b = new Blob();
-      b->set_shared_blob(new SharedBlob(this));
+      b->collection = this;
       return b;
     }
 
@@ -2832,7 +2849,7 @@ class BlueStore : public ObjectStore,
     uint64_t offset,
     ceph::buffer::list& bl,
     unsigned flags) {
-    b->dirty_bc().write(b->shared_blob->get_cache(), txc->seq, offset, bl,
+    b->dirty_bc().write(b->get_cache(), txc->seq, offset, bl,
 			     flags);
     txc->blobs_written.insert(b);
   }

From 4515c51f63b54921223c45d48c9443a202f15003 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Sun, 24 Sep 2023 21:49:09 +0200
Subject: [PATCH 1824/2492] os/bluestore: rehome Blob AND SharedBlob

split_cache requires moving from one collection to another in a
sharedblob. Therefore, with the new refactor we need collection inside
blob to be moved too.

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index e3e07f43e57c..8eed5329b484 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -5175,19 +5175,23 @@ void BlueStore::Collection::split_cache(
 	cache->rm_blob();
 	dest->cache->add_blob();
 	SharedBlob* sb = b->shared_blob.get();
-	if (sb->coll == dest) {
+	if (sb && sb->coll == dest && b->collection) {
 	  ldout(store->cct, 20) << __func__ << "  already moved " << *sb
 				<< dendl;
 	  return;
 	}
-	ldout(store->cct, 20) << __func__ << "  moving " << *sb << dendl;
-	if (sb->get_sbid()) {
+	ldout(store->cct, 20) << __func__ << "  moving " << *b << dendl;
+	if (b->get_sbid()) {
+          ldout(store->cct, 20) << __func__ << "  moving " << *sb << dendl;
 	  ldout(store->cct, 20) << __func__
 				<< "   moving registration " << *sb << dendl;
 	  shared_blob_set.remove(sb);
 	  dest->shared_blob_set.add(dest, sb);
 	}
-	sb->coll = dest;
+        if (sb) {
+          sb->coll = dest;
+        }
+        b->collection = dest;
       };
 
       for (auto& e : o->extent_map.extent_map) {

From 8961e2cca30afefaada7ba57b1e7458e3a29c754 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Mon, 25 Sep 2023 12:11:12 +0200
Subject: [PATCH 1825/2492] os/bluestore: add missing add_blobs

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 6 +++---
 src/os/bluestore/BlueStore.h  | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 8eed5329b484..b96757a1e2be 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2172,10 +2172,10 @@ BlueStore::Blob::~Blob()
     return;
   }
  again:
-  auto coll_cache = sb->get_cache();
+  auto coll_cache = get_cache();
   if (coll_cache) {
     std::lock_guard l(coll_cache->lock);
-    if (coll_cache != sb->get_cache()) {
+    if (coll_cache != get_cache()) {
       goto again;
     }
     bc._clear(coll_cache);
@@ -3920,7 +3920,6 @@ bool BlueStore::ExtentMap::encode_some(
       denc_varint(0, bound); // len
       denc_varint(0, bound); // blob_offset
 
-      dout(1) << "Pere blob " << p->blob << dendl;
       p->blob->bound_encode(
         bound,
         struct_v,
@@ -4986,6 +4985,7 @@ void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
   const bluestore_blob_t& blob = b->get_blob();
   if (!blob.is_shared()) {
     b->collection = this;
+    b->get_cache()->add_blob();
     return;
   }
 
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 7407963cd39a..cc2ef5e430ed 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -615,6 +615,7 @@ class BlueStore : public ObjectStore,
       ceph_assert(!shared_blob);
       sbid_unloaded = sb->get_sbid();
       shared_blob = sb;
+      collection = sb->coll;
       ceph_assert(get_cache());
       get_cache()->add_blob();
     }
@@ -1628,6 +1629,7 @@ class BlueStore : public ObjectStore,
     BlobRef new_blob() {
       BlobRef b = new Blob();
       b->collection = this;
+      b->get_cache()->add_blob();
       return b;
     }
 

From e56fd0eaa944a107ccee633972ecda7459e206a2 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Mon, 25 Sep 2023 15:21:29 +0200
Subject: [PATCH 1826/2492] os/bluestore: run rm_blob before shared_blob check

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 19 +++++++++----------
 src/os/bluestore/BlueStore.h  |  1 -
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index b96757a1e2be..e6818dd371a5 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2166,11 +2166,6 @@ void BlueStore::SharedBlobSet::dump(CephContext *cct)
 
 BlueStore::Blob::~Blob()
 {
-  SharedBlob* sb = shared_blob.get();
-  if (!sb) {
-    ceph_assert(bc.buffer_map.empty());
-    return;
-  }
  again:
   auto coll_cache = get_cache();
   if (coll_cache) {
@@ -2181,6 +2176,11 @@ BlueStore::Blob::~Blob()
     bc._clear(coll_cache);
     coll_cache->rm_blob();
   }
+  SharedBlob* sb = shared_blob.get();
+  if (!sb) {
+    ceph_assert(bc.buffer_map.empty());
+    return;
+  }
 }
 
 void BlueStore::Blob::dump(Formatter* f) const
@@ -3231,7 +3231,7 @@ void BlueStore::ExtentMap::dup(BlueStore* b, TransContext* txc,
       } else {
         c->load_shared_blob(e.blob->shared_blob);
       }
-      cb = new Blob();
+      cb = c->new_blob();
       e.blob->last_encoded_id = n;
       id_to_blob[n] = cb;
       e.blob->dup(*cb);
@@ -3345,7 +3345,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
       ceph_assert(blob.is_shared());
       ceph_assert(e.blob->is_loaded());
       ceph_assert(!blob.has_unused());
-      cb = new Blob();
+      cb = c->new_blob();
       e.blob->last_encoded_id = n;
       id_to_blob[n] = cb;
       ceph_assert(ep->blob_start() < end);
@@ -4026,7 +4026,7 @@ void BlueStore::ExtentMap::ExtentDecoder::decode_extent(
     if (blobid) {
       consume_blobid(le, false, blobid - 1);
     } else {
-      Blob *b = new Blob();
+      BlobRef b = c->new_blob();
       uint64_t sbid = 0;
       b->decode(p, struct_v, &sbid, false, c);
       consume_blob(le, extent_pos, sbid, b);
@@ -4074,7 +4074,7 @@ void BlueStore::ExtentMap::ExtentDecoder::decode_spanning_blobs(
   unsigned n;
   denc_varint(n, p);
   while (n--) {
-    BlueStore::BlobRef b(new Blob());
+    BlueStore::BlobRef b(c->new_blob());
     denc_varint(b->id, p);
     uint64_t sbid = 0;
     b->decode(p, struct_v, &sbid, true, c);
@@ -4985,7 +4985,6 @@ void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
   const bluestore_blob_t& blob = b->get_blob();
   if (!blob.is_shared()) {
     b->collection = this;
-    b->get_cache()->add_blob();
     return;
   }
 
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index cc2ef5e430ed..32164abcf37d 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -617,7 +617,6 @@ class BlueStore : public ObjectStore,
       shared_blob = sb;
       collection = sb->coll;
       ceph_assert(get_cache());
-      get_cache()->add_blob();
     }
     BufferSpace bc;
   private:

From d98ca007e45751979ca5e13a863f1a7741f700fb Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Mon, 25 Sep 2023 15:50:21 +0200
Subject: [PATCH 1827/2492] os/bluestore: rename SharedBlob coll to collection

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 14 +++++++-------
 src/os/bluestore/BlueStore.h  | 19 ++++++++-----------
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index e6818dd371a5..30bb8733238e 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2071,7 +2071,7 @@ void BlueStore::OnodeSpace::dump(CephContext *cct)
 #undef dout_prefix
 #define dout_prefix *_dout << "bluestore.sharedblob(" << this << ") "
 #undef dout_context
-#define dout_context coll->store->cct
+#define dout_context collection->store->cct
 
 void BlueStore::SharedBlob::dump(Formatter* f) const
 {
@@ -2096,7 +2096,7 @@ ostream& operator<<(ostream& out, const BlueStore::SharedBlob& sb)
 }
 
 BlueStore::SharedBlob::SharedBlob(uint64_t i, Collection *_coll)
-  : coll(_coll), sbid_unloaded(i)
+  : collection(_coll), sbid_unloaded(i)
 {
   ceph_assert(sbid_unloaded > 0);
 }
@@ -2115,10 +2115,10 @@ void BlueStore::SharedBlob::put()
 	     << " removing self from set " << get_parent()
 	     << dendl;
   again:
-    auto coll_snap = coll;
+    auto coll_snap = collection;
     if (coll_snap) {
       std::lock_guard l(coll_snap->cache->lock);
-      if (coll_snap != coll) {
+      if (coll_snap != collection) {
 	goto again;
       }
       if (!coll_snap->shared_blob_set.remove(this, true)) {
@@ -2886,7 +2886,7 @@ uint32_t BlueStore::Blob::merge_blob(CephContext* cct, Blob* blob_to_dissolve)
 }
 
 #undef dout_context
-#define dout_context coll->store->cct
+#define dout_context collection->store->cct
 
 void BlueStore::Blob::finish_write(uint64_t seq)
 {
@@ -5174,7 +5174,7 @@ void BlueStore::Collection::split_cache(
 	cache->rm_blob();
 	dest->cache->add_blob();
 	SharedBlob* sb = b->shared_blob.get();
-	if (sb && sb->coll == dest && b->collection) {
+	if (sb && sb->collection == dest && b->collection) {
 	  ldout(store->cct, 20) << __func__ << "  already moved " << *sb
 				<< dendl;
 	  return;
@@ -5188,7 +5188,7 @@ void BlueStore::Collection::split_cache(
 	  dest->shared_blob_set.add(dest, sb);
 	}
         if (sb) {
-          sb->coll = dest;
+          sb->collection = dest;
         }
         b->collection = dest;
       };
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 32164abcf37d..96256a48c1db 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -499,13 +499,13 @@ class BlueStore : public ObjectStore,
     std::atomic_int nref = {0}; ///< reference count
     bool loaded = false;
 
-    CollectionRef coll;
+    CollectionRef collection;
     union {
       uint64_t sbid_unloaded;              ///< sbid if persistent isn't loaded
       bluestore_shared_blob_t *persistent; ///< persistent part of the shared blob if any
     };
 
-    SharedBlob(Collection *_coll) : coll(_coll), sbid_unloaded(0) {
+    SharedBlob(Collection *_coll) : collection(_coll), sbid_unloaded(0) {
     }
     SharedBlob(uint64_t i, Collection *_coll);
     ~SharedBlob();
@@ -535,10 +535,10 @@ class BlueStore : public ObjectStore,
       return l.get_sbid() == r.get_sbid();
     }
     inline BufferCacheShard* get_cache() {
-      return coll ? coll->cache : nullptr;
+      return collection ? collection->cache : nullptr;
     }
     inline SharedBlobSet* get_parent() {
-      return coll ? &(coll->shared_blob_set) : nullptr;
+      return collection ? &(collection->shared_blob_set) : nullptr;
     }
     inline bool is_loaded() const {
       return loaded;
@@ -569,7 +569,7 @@ class BlueStore : public ObjectStore,
     void add(Collection* coll, SharedBlob *sb) {
       std::lock_guard l(lock);
       sb_map[sb->get_sbid()] = sb;
-      sb->coll = coll;
+      sb->collection = coll;
     }
 
     bool remove(SharedBlob *sb, bool verify_nref_is_zero=false) {
@@ -606,16 +606,13 @@ class BlueStore : public ObjectStore,
     int16_t id = -1;                ///< id, for spanning blobs only, >= 0
     int16_t last_encoded_id = -1;   ///< (ephemeral) used during encoding only
     SharedBlobRef shared_blob;      ///< shared blob state (if any)
-    bool loaded = false;
     CollectionRef collection;
-    uint64_t sbid_unloaded = 0;
 
     void set_shared_blob(SharedBlobRef sb) {
       ceph_assert((bool)sb);
       ceph_assert(!shared_blob);
-      sbid_unloaded = sb->get_sbid();
       shared_blob = sb;
-      collection = sb->coll;
+      collection = sb->collection;
       ceph_assert(get_cache());
     }
     BufferSpace bc;
@@ -733,10 +730,10 @@ class BlueStore : public ObjectStore,
       return shared_blob ? shared_blob->get_cache() : collection->cache;
     }
     uint64_t get_sbid() const {
-      return shared_blob ? shared_blob->get_sbid() : sbid_unloaded;
+      return shared_blob ? shared_blob->get_sbid() : 0;
     }
     CollectionRef get_collection() const {
-      return shared_blob ? shared_blob->coll : collection;
+      return shared_blob ? shared_blob->collection : collection;
     }
 
     ~Blob();

From dd9be33a0493be9e115ead716f35a6d9d867fcdd Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Tue, 26 Sep 2023 09:40:34 +0200
Subject: [PATCH 1828/2492] os/bluestore: use add_blob and set collection
 bluestore_types_unittest

Use new_blob and set collection in Blob in bluestore unittests as per
latest changes

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/test/objectstore/test_bluestore_types.cc | 154 +++++++++++--------
 1 file changed, 88 insertions(+), 66 deletions(-)

diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc
index 2d10d3839ae6..6e047a09c251 100644
--- a/src/test/objectstore/test_bluestore_types.cc
+++ b/src/test/objectstore/test_bluestore_types.cc
@@ -434,7 +434,8 @@ TEST(Blob, put_ref)
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
     BlueStore::Blob b;
-    b.shared_blob = new BlueStore::SharedBlob(coll.get());
+    b.collection = coll;
+    b.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b.dirty_blob().allocated_test(bluestore_pextent_t(0x40715000, 0x2000));
     b.dirty_blob().allocated_test(
       bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x8000));
@@ -467,7 +468,8 @@ TEST(Blob, put_ref)
 
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(0, mas * 2));
@@ -488,7 +490,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(123, mas * 2));
@@ -512,7 +515,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas));
@@ -550,7 +554,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas));
@@ -591,7 +596,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 6));
@@ -623,7 +629,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -661,7 +668,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -716,7 +724,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -771,7 +780,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 8));
@@ -814,7 +824,8 @@ TEST(Blob, put_ref)
   // verify csum chunk size if factored in properly
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(0, mas*4));
@@ -832,7 +843,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(0x40101000, 0x4000));
     b.allocated_test(bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET,
@@ -854,7 +866,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
     b.allocated_test(bluestore_pextent_t(2, 0x5000));
@@ -872,7 +885,8 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x7000));
     b.allocated_test(bluestore_pextent_t(2, 0x7000));
@@ -901,7 +915,8 @@ TEST(Blob, put_ref)
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
     BlueStore::Blob B;
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.collection = coll;
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
     b.allocated_test(bluestore_pextent_t(2, 0x7000));
@@ -990,8 +1005,10 @@ TEST(Blob, split)
   auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
   {
     BlueStore::Blob L, R;
-    L.shared_blob = new BlueStore::SharedBlob(coll.get());
-    R.shared_blob = new BlueStore::SharedBlob(coll.get());
+    L.collection = coll;
+    R.collection = coll;
+    L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x2000));
     L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
     L.get_ref(coll.get(), 0, 0x2000);
@@ -1011,8 +1028,10 @@ TEST(Blob, split)
   }
   {
     BlueStore::Blob L, R;
-    L.shared_blob = new BlueStore::SharedBlob(coll.get());
-    R.shared_blob = new BlueStore::SharedBlob(coll.get());
+    L.collection = coll;
+    R.collection = coll;
+    L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x1000));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x12000, 0x1000));
     L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
@@ -1045,8 +1064,9 @@ TEST(Blob, legacy_decode)
   bufferlist bl, bl2;
   {
     BlueStore::Blob B;
+    B.collection = coll;
 
-    B.shared_blob = new BlueStore::SharedBlob(coll.get());
+    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     B.dirty_blob().allocated_test(bluestore_pextent_t(0x1, 0x2000));
     B.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
     B.get_ref(coll.get(), 0, 0xff0);
@@ -1091,8 +1111,10 @@ TEST(Blob, legacy_decode)
     auto p = bl.front().begin_deep();
     auto p2 = bl2.front().begin_deep();
     BlueStore::Blob Bres, Bres2;
-    Bres.shared_blob = new BlueStore::SharedBlob(coll.get());
-    Bres2.shared_blob = new BlueStore::SharedBlob(coll.get());
+    Bres.collection = coll;
+    Bres2.collection = coll;
+    Bres.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    Bres2.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
     uint64_t sbid, sbid2;
     Bres.decode(
@@ -1126,8 +1148,8 @@ TEST(ExtentMap, seek_lextent)
   BlueStore::Onode onode(coll.get(), ghobject_t(), "");
   BlueStore::ExtentMap em(&onode,
     g_ceph_context->_conf->bluestore_extent_map_inline_shard_prealloc_size);
-  BlueStore::BlobRef br(new BlueStore::Blob);
-  br->shared_blob = new BlueStore::SharedBlob(coll.get());
+  BlueStore::BlobRef br(coll->new_blob());
+  br->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
   ASSERT_EQ(em.extent_map.end(), em.seek_lextent(0));
   ASSERT_EQ(em.extent_map.end(), em.seek_lextent(100));
@@ -1179,8 +1201,8 @@ TEST(ExtentMap, has_any_lextents)
   BlueStore::Onode onode(coll.get(), ghobject_t(), "");
   BlueStore::ExtentMap em(&onode,
     g_ceph_context->_conf->bluestore_extent_map_inline_shard_prealloc_size);
-  BlueStore::BlobRef b(new BlueStore::Blob);
-  b->shared_blob = new BlueStore::SharedBlob(coll.get());
+  BlueStore::BlobRef b(coll->new_blob());
+  b->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
   ASSERT_FALSE(em.has_any_lextents(0, 0));
   ASSERT_FALSE(em.has_any_lextents(0, 1000));
@@ -1239,12 +1261,12 @@ TEST(ExtentMap, compress_extent_map)
   BlueStore::Onode onode(coll.get(), ghobject_t(), "");
   BlueStore::ExtentMap em(&onode,
     g_ceph_context->_conf->bluestore_extent_map_inline_shard_prealloc_size);
-  BlueStore::BlobRef b1(new BlueStore::Blob);
-  BlueStore::BlobRef b2(new BlueStore::Blob);
-  BlueStore::BlobRef b3(new BlueStore::Blob);
-  b1->shared_blob = new BlueStore::SharedBlob(coll.get());
-  b2->shared_blob = new BlueStore::SharedBlob(coll.get());
-  b3->shared_blob = new BlueStore::SharedBlob(coll.get());
+  BlueStore::BlobRef b1(coll->new_blob());
+  BlueStore::BlobRef b2(coll->new_blob());
+  BlueStore::BlobRef b3(coll->new_blob());
+  b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+  b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+  b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
   em.extent_map.insert(*new BlueStore::Extent(0, 0, 100, b1));
   em.extent_map.insert(*new BlueStore::Extent(100, 0, 100, b2));
@@ -1417,8 +1439,8 @@ class ExtentMapFixture : virtual public ::testing::Test {
       uint32_t num_aus     // number of aus, first, first+1.. first+num_au-1
     ) {
       uint32_t blob_length = (empty_aus + num_aus) * au_size;
-      BlueStore::BlobRef b(new BlueStore::Blob);
-      b->shared_blob = new BlueStore::SharedBlob(coll.get());
+      BlueStore::BlobRef b(coll->new_blob());
+      b->collection = coll;
       bluestore_blob_t& bb = b->dirty_blob();
       bb.init_csum(Checksummer::CSUM_CRC32C, csum_order, blob_length);
       for(size_t i = 0; i < num_aus; ++i) {
@@ -1683,8 +1705,8 @@ TEST(ExtentMap, dup_extent_map)
   size_t ext1_offs = 0x0;
   size_t ext1_len = 0x2000;
   size_t ext1_boffs = 0x0;
-  BlueStore::BlobRef b1(new BlueStore::Blob);
-  b1->shared_blob = new BlueStore::SharedBlob(coll.get());
+  BlueStore::BlobRef b1(coll->new_blob());
+  b1->collection = coll;
   auto &_b1 = b1->dirty_blob();
   _b1.init_csum(Checksummer::CSUM_CRC32C, csum_order, ext1_len);
   for(size_t i = 0; i < _b1.get_csum_count(); i++) {
@@ -1875,14 +1897,14 @@ TEST(GarbageCollector, BasicTest)
   {
     BlueStore::GarbageCollector gc(g_ceph_context);
     int64_t saving;
-    BlueStore::BlobRef b1(new BlueStore::Blob);
-    BlueStore::BlobRef b2(new BlueStore::Blob);
-    BlueStore::BlobRef b3(new BlueStore::Blob);
-    BlueStore::BlobRef b4(new BlueStore::Blob);
-    b1->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b2->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b3->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b4->shared_blob = new BlueStore::SharedBlob(coll.get());
+    BlueStore::BlobRef b1(coll->new_blob());
+    BlueStore::BlobRef b2(coll->new_blob());
+    BlueStore::BlobRef b3(coll->new_blob());
+    BlueStore::BlobRef b4(coll->new_blob());
+    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x2000, 0x1000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x1000));
     b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x1000));
@@ -1941,14 +1963,14 @@ TEST(GarbageCollector, BasicTest)
     BlueStore::old_extent_map_t old_extents;
     BlueStore::GarbageCollector gc(g_ceph_context);
     int64_t saving;
-    BlueStore::BlobRef b1(new BlueStore::Blob);
-    BlueStore::BlobRef b2(new BlueStore::Blob);
-    BlueStore::BlobRef b3(new BlueStore::Blob);
-    BlueStore::BlobRef b4(new BlueStore::Blob);
-    b1->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b2->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b3->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b4->shared_blob = new BlueStore::SharedBlob(coll.get());
+    BlueStore::BlobRef b1(coll->new_blob());
+    BlueStore::BlobRef b2(coll->new_blob());
+    BlueStore::BlobRef b3(coll->new_blob());
+    BlueStore::BlobRef b4(coll->new_blob());
+    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x40000, 0x20000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x20000));
     b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x10000));
@@ -2014,10 +2036,10 @@ TEST(GarbageCollector, BasicTest)
   {
     BlueStore::GarbageCollector gc(g_ceph_context);
     int64_t saving;
-    BlueStore::BlobRef b1(new BlueStore::Blob);
-    BlueStore::BlobRef b2(new BlueStore::Blob);
-    b1->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b2->shared_blob = new BlueStore::SharedBlob(coll.get());
+    BlueStore::BlobRef b1(coll->new_blob());
+    BlueStore::BlobRef b2(coll->new_blob());
+    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x4000, 0x2000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x2000));
     b2->dirty_blob().set_compressed(0x4000, 0x2000);
@@ -2069,16 +2091,16 @@ TEST(GarbageCollector, BasicTest)
     BlueStore::old_extent_map_t old_extents;
     BlueStore::GarbageCollector gc(g_ceph_context);
     int64_t saving;
-    BlueStore::BlobRef b0(new BlueStore::Blob);
-    BlueStore::BlobRef b1(new BlueStore::Blob);
-    BlueStore::BlobRef b2(new BlueStore::Blob);
-    BlueStore::BlobRef b3(new BlueStore::Blob);
-    BlueStore::BlobRef b4(new BlueStore::Blob);
-    b0->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b1->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b2->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b3->shared_blob = new BlueStore::SharedBlob(coll.get());
-    b4->shared_blob = new BlueStore::SharedBlob(coll.get());
+    BlueStore::BlobRef b0(coll->new_blob());
+    BlueStore::BlobRef b1(coll->new_blob());
+    BlueStore::BlobRef b2(coll->new_blob());
+    BlueStore::BlobRef b3(coll->new_blob());
+    BlueStore::BlobRef b4(coll->new_blob());
+    b0->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
+    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b0->dirty_blob().set_compressed(0x2000, 0x1000);
     b0->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x10000));
     b1->dirty_blob().set_compressed(0x20000, 0x10000);

From 13b378b6cca33f5c6d39c3f016191930132516c9 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Wed, 27 Sep 2023 10:33:49 +0200
Subject: [PATCH 1829/2492] os/bluestore: assert shared_blob->collection ==
 blob->collection

Instead of setting blob collection to the same as shared blob, we ensure
we were in the same collection instead.

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 96256a48c1db..1119a1531872 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -611,8 +611,8 @@ class BlueStore : public ObjectStore,
     void set_shared_blob(SharedBlobRef sb) {
       ceph_assert((bool)sb);
       ceph_assert(!shared_blob);
+      ceph_assert(sb->collection = collection);
       shared_blob = sb;
-      collection = sb->collection;
       ceph_assert(get_cache());
     }
     BufferSpace bc;
@@ -727,13 +727,13 @@ class BlueStore : public ObjectStore,
       return shared_blob && shared_blob->is_loaded();
     }
     inline BufferCacheShard* get_cache() {
-      return shared_blob ? shared_blob->get_cache() : collection->cache;
+      return collection->cache;
     }
     uint64_t get_sbid() const {
       return shared_blob ? shared_blob->get_sbid() : 0;
     }
     CollectionRef get_collection() const {
-      return shared_blob ? shared_blob->collection : collection;
+      return collection;
     }
 
     ~Blob();

From 0dacebf5635181b9ab582f44f67839361987ad17 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Thu, 28 Sep 2023 11:42:21 +0200
Subject: [PATCH 1830/2492] os/bluestore: move blob and sharedblob on cache
 splitting

Collection is a member of Blob after previous changes, making it necessary for rehome_blob
to move both Blob and SharedBlob

Signed-off-by: Pere Diaz Bou <pdiabou@redhat.com>
---
 src/os/bluestore/BlueStore.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 30bb8733238e..626147dff026 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -5174,7 +5174,8 @@ void BlueStore::Collection::split_cache(
 	cache->rm_blob();
 	dest->cache->add_blob();
 	SharedBlob* sb = b->shared_blob.get();
-	if (sb && sb->collection == dest && b->collection) {
+        b->collection = dest;
+	if (sb && sb->collection == dest) {
 	  ldout(store->cct, 20) << __func__ << "  already moved " << *sb
 				<< dendl;
 	  return;
@@ -5190,7 +5191,6 @@ void BlueStore::Collection::split_cache(
         if (sb) {
           sb->collection = dest;
         }
-        b->collection = dest;
       };
 
       for (auto& e : o->extent_map.extent_map) {

From 17e5e3ad5cb932d4301720ac0f12ab5d1a8e76be Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pdiabou@redhat.com>
Date: Tue, 3 Oct 2023 14:26:10 +0200
Subject: [PATCH 1831/2492] os/bluestore: blob's collection initialization in
 ctor

Set collection on blob creation instead of setting it manually after
creation

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc                | 38 ++++-------
 src/os/bluestore/BlueStore.h                 |  6 +-
 src/test/objectstore/test_bluestore_types.cc | 67 +++++++-------------
 3 files changed, 40 insertions(+), 71 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 626147dff026..9e3c72f6e3d8 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2177,10 +2177,7 @@ BlueStore::Blob::~Blob()
     coll_cache->rm_blob();
   }
   SharedBlob* sb = shared_blob.get();
-  if (!sb) {
-    ceph_assert(bc.buffer_map.empty());
-    return;
-  }
+  ceph_assert(sb || (!sb && bc.buffer_map.empty()));
 }
 
 void BlueStore::Blob::dump(Formatter* f) const
@@ -3343,7 +3340,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
       // dup the blob
       const bluestore_blob_t& blob = e.blob->get_blob();
       ceph_assert(blob.is_shared());
-      ceph_assert(e.blob->is_loaded());
+      ceph_assert(e.blob->is_shared_loaded());
       ceph_assert(!blob.has_unused());
       cb = c->new_blob();
       e.blob->last_encoded_id = n;
@@ -4074,7 +4071,7 @@ void BlueStore::ExtentMap::ExtentDecoder::decode_spanning_blobs(
   unsigned n;
   denc_varint(n, p);
   while (n--) {
-    BlueStore::BlobRef b(c->new_blob());
+    BlueStore::BlobRef b = c->new_blob();
     denc_varint(b->id, p);
     uint64_t sbid = 0;
     b->decode(p, struct_v, &sbid, true, c);
@@ -4513,7 +4510,7 @@ BlueStore::ExtentMap::debug_list_disk_layout()
 
     bluestore_extent_ref_map_t* ref_map = nullptr;
     if (bblob.is_shared()) {
-      ceph_assert(ep->blob->is_loaded());
+      ceph_assert(ep->blob->is_shared_loaded());
       bluestore_shared_blob_t* bsblob = ep->blob->shared_blob->persistent;
       ref_map = &bsblob->ref_map;
     }
@@ -4984,7 +4981,6 @@ void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
   ceph_assert(!b->shared_blob);
   const bluestore_blob_t& blob = b->get_blob();
   if (!blob.is_shared()) {
-    b->collection = this;
     return;
   }
 
@@ -5030,8 +5026,6 @@ void BlueStore::Collection::load_shared_blob(SharedBlobRef sb)
 void BlueStore::Collection::make_blob_shared(uint64_t sbid, BlobRef b)
 {
   ldout(store->cct, 10) << __func__ << " " << *b << dendl;
-  ceph_assert(!b->is_loaded());
-  ceph_assert(!b->shared_blob);
 
   // update blob
   bluestore_blob_t& blob = b->dirty_blob();
@@ -5039,7 +5033,7 @@ void BlueStore::Collection::make_blob_shared(uint64_t sbid, BlobRef b)
   // drop any unused parts, unlikely we could use them in future
   blob.clear_flag(bluestore_blob_t::FLAG_HAS_UNUSED);
   // update shared blob
-  b->shared_blob = new SharedBlob(sbid, this);
+  b->set_shared_blob(new SharedBlob(sbid, this));
   b->shared_blob->loaded = true;
   b->shared_blob->persistent = new bluestore_shared_blob_t(sbid);
   shared_blob_set.add(this, b->shared_blob.get());
@@ -5175,20 +5169,16 @@ void BlueStore::Collection::split_cache(
 	dest->cache->add_blob();
 	SharedBlob* sb = b->shared_blob.get();
         b->collection = dest;
-	if (sb && sb->collection == dest) {
-	  ldout(store->cct, 20) << __func__ << "  already moved " << *sb
-				<< dendl;
-	  return;
-	}
-	ldout(store->cct, 20) << __func__ << "  moving " << *b << dendl;
-	if (b->get_sbid()) {
-          ldout(store->cct, 20) << __func__ << "  moving " << *sb << dendl;
-	  ldout(store->cct, 20) << __func__
-				<< "   moving registration " << *sb << dendl;
-	  shared_blob_set.remove(sb);
-	  dest->shared_blob_set.add(dest, sb);
-	}
         if (sb) {
+          if (sb->collection == dest) {
+            ldout(store->cct, 20) << __func__ << "  already moved " << *sb
+              << dendl;
+            return;
+          }
+          ldout(store->cct, 20) << __func__ << "  moving " << *b << dendl;
+          ldout(store->cct, 20) << __func__ << "  moving " << *sb << dendl;
+          shared_blob_set.remove(sb);
+          dest->shared_blob_set.add(dest, sb);
           sb->collection = dest;
         }
       };
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 1119a1531872..7baaf503e39a 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -615,6 +615,7 @@ class BlueStore : public ObjectStore,
       shared_blob = sb;
       ceph_assert(get_cache());
     }
+    Blob(CollectionRef collection) : collection(collection) {}
     BufferSpace bc;
   private:
     mutable bluestore_blob_t blob;  ///< decoded blob metadata
@@ -723,7 +724,7 @@ class BlueStore : public ObjectStore,
       if (--nref == 0)
 	delete this;
     }
-    bool is_loaded() const {
+    bool is_shared_loaded() const {
       return shared_blob && shared_blob->is_loaded();
     }
     inline BufferCacheShard* get_cache() {
@@ -1623,8 +1624,7 @@ class BlueStore : public ObjectStore,
     uint64_t make_blob_unshared(SharedBlob *sb);
 
     BlobRef new_blob() {
-      BlobRef b = new Blob();
-      b->collection = this;
+      BlobRef b = new Blob(this);
       b->get_cache()->add_blob();
       return b;
     }
diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc
index 6e047a09c251..2d3b6ed92693 100644
--- a/src/test/objectstore/test_bluestore_types.cc
+++ b/src/test/objectstore/test_bluestore_types.cc
@@ -433,8 +433,7 @@ TEST(Blob, put_ref)
       g_ceph_context, "lru", NULL);
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
-    BlueStore::Blob b;
-    b.collection = coll;
+    BlueStore::Blob b(coll.get());
     b.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b.dirty_blob().allocated_test(bluestore_pextent_t(0x40715000, 0x2000));
     b.dirty_blob().allocated_test(
@@ -467,8 +466,7 @@ TEST(Blob, put_ref)
   auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
 
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -489,8 +487,7 @@ TEST(Blob, put_ref)
     ASSERT_EQ(mas*2, b.get_extents()[0].length);
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -514,8 +511,7 @@ TEST(Blob, put_ref)
     ASSERT_EQ(mas*2, b.get_extents()[0].length);
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -553,8 +549,7 @@ TEST(Blob, put_ref)
     ASSERT_EQ(3u, b.get_extents().size());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -595,8 +590,7 @@ TEST(Blob, put_ref)
     ASSERT_TRUE(b.get_extents()[4].is_valid());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll);
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -628,8 +622,7 @@ TEST(Blob, put_ref)
     ASSERT_TRUE(b.get_extents()[2].is_valid());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll);
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -667,8 +660,7 @@ TEST(Blob, put_ref)
     ASSERT_TRUE(b.get_extents()[2].is_valid());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll);
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -723,8 +715,7 @@ TEST(Blob, put_ref)
     ASSERT_FALSE(b.get_extents()[0].is_valid());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll);
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -779,8 +770,7 @@ TEST(Blob, put_ref)
     ASSERT_FALSE(b.get_extents()[0].is_valid());
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -823,8 +813,7 @@ TEST(Blob, put_ref)
   }
   // verify csum chunk size if factored in properly
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
@@ -842,8 +831,7 @@ TEST(Blob, put_ref)
     ASSERT_EQ(mas*4, b.get_extents()[0].length);
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(0x40101000, 0x4000));
@@ -865,8 +853,7 @@ TEST(Blob, put_ref)
     cout << "r " << r << std::endl;
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
@@ -884,8 +871,7 @@ TEST(Blob, put_ref)
     ASSERT_EQ(0x2000u, r[0].length);
   }
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x7000));
@@ -914,8 +900,7 @@ TEST(Blob, put_ref)
       g_ceph_context, "lru", NULL);
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
@@ -1004,9 +989,8 @@ TEST(Blob, split)
       g_ceph_context, "lru", NULL);
   auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
   {
-    BlueStore::Blob L, R;
-    L.collection = coll;
-    R.collection = coll;
+    BlueStore::Blob L(coll.get());
+    BlueStore::Blob R(coll.get());
     L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x2000));
@@ -1027,9 +1011,8 @@ TEST(Blob, split)
     ASSERT_EQ(0x1000u, R.get_referenced_bytes());
   }
   {
-    BlueStore::Blob L, R;
-    L.collection = coll;
-    R.collection = coll;
+    BlueStore::Blob L(coll.get());
+    BlueStore::Blob R(coll.get());
     L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x1000));
@@ -1063,8 +1046,7 @@ TEST(Blob, legacy_decode)
   auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
   bufferlist bl, bl2;
   {
-    BlueStore::Blob B;
-    B.collection = coll;
+    BlueStore::Blob B(coll.get());
 
     B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     B.dirty_blob().allocated_test(bluestore_pextent_t(0x1, 0x2000));
@@ -1110,9 +1092,8 @@ TEST(Blob, legacy_decode)
 
     auto p = bl.front().begin_deep();
     auto p2 = bl2.front().begin_deep();
-    BlueStore::Blob Bres, Bres2;
-    Bres.collection = coll;
-    Bres2.collection = coll;
+    BlueStore::Blob Bres(coll.get());
+    BlueStore::Blob Bres2(coll.get());
     Bres.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     Bres2.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
@@ -1440,7 +1421,6 @@ class ExtentMapFixture : virtual public ::testing::Test {
     ) {
       uint32_t blob_length = (empty_aus + num_aus) * au_size;
       BlueStore::BlobRef b(coll->new_blob());
-      b->collection = coll;
       bluestore_blob_t& bb = b->dirty_blob();
       bb.init_csum(Checksummer::CSUM_CRC32C, csum_order, blob_length);
       for(size_t i = 0; i < num_aus; ++i) {
@@ -1705,8 +1685,7 @@ TEST(ExtentMap, dup_extent_map)
   size_t ext1_offs = 0x0;
   size_t ext1_len = 0x2000;
   size_t ext1_boffs = 0x0;
-  BlueStore::BlobRef b1(coll->new_blob());
-  b1->collection = coll;
+  BlueStore::BlobRef b1 = coll->new_blob();
   auto &_b1 = b1->dirty_blob();
   _b1.init_csum(Checksummer::CSUM_CRC32C, csum_order, ext1_len);
   for(size_t i = 0; i < _b1.get_csum_count(); i++) {

From e3bd4527a1af80c06534557f32b7da8957072e6f Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Tue, 24 Oct 2023 12:35:58 +0200
Subject: [PATCH 1832/2492] os/bluestore: remove unnecessary set_shared_blob in
 tests

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/test/objectstore/test_bluestore_types.cc | 39 --------------------
 1 file changed, 39 deletions(-)

diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc
index 2d3b6ed92693..cf41b504259f 100644
--- a/src/test/objectstore/test_bluestore_types.cc
+++ b/src/test/objectstore/test_bluestore_types.cc
@@ -434,7 +434,6 @@ TEST(Blob, put_ref)
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
     BlueStore::Blob b(coll.get());
-    b.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b.dirty_blob().allocated_test(bluestore_pextent_t(0x40715000, 0x2000));
     b.dirty_blob().allocated_test(
       bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x8000));
@@ -467,7 +466,6 @@ TEST(Blob, put_ref)
 
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(0, mas * 2));
@@ -488,7 +486,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(123, mas * 2));
@@ -512,7 +509,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas));
@@ -550,7 +546,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas));
@@ -591,7 +586,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll);
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 6));
@@ -623,7 +617,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll);
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -661,7 +654,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll);
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -716,7 +708,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll);
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 4));
@@ -771,7 +762,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(1, mas * 8));
@@ -814,7 +804,6 @@ TEST(Blob, put_ref)
   // verify csum chunk size if factored in properly
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     PExtentVector r;
     b.allocated_test(bluestore_pextent_t(0, mas*4));
@@ -832,7 +821,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(0x40101000, 0x4000));
     b.allocated_test(bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET,
@@ -854,7 +842,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
     b.allocated_test(bluestore_pextent_t(2, 0x5000));
@@ -872,7 +859,6 @@ TEST(Blob, put_ref)
   }
   {
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x7000));
     b.allocated_test(bluestore_pextent_t(2, 0x7000));
@@ -901,7 +887,6 @@ TEST(Blob, put_ref)
 
     auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
     BlueStore::Blob B(coll.get());
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     bluestore_blob_t& b = B.dirty_blob();
     b.allocated_test(bluestore_pextent_t(1, 0x5000));
     b.allocated_test(bluestore_pextent_t(2, 0x7000));
@@ -991,8 +976,6 @@ TEST(Blob, split)
   {
     BlueStore::Blob L(coll.get());
     BlueStore::Blob R(coll.get());
-    L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x2000));
     L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
     L.get_ref(coll.get(), 0, 0x2000);
@@ -1013,8 +996,6 @@ TEST(Blob, split)
   {
     BlueStore::Blob L(coll.get());
     BlueStore::Blob R(coll.get());
-    L.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    R.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x1000));
     L.dirty_blob().allocated_test(bluestore_pextent_t(0x12000, 0x1000));
     L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
@@ -1048,7 +1029,6 @@ TEST(Blob, legacy_decode)
   {
     BlueStore::Blob B(coll.get());
 
-    B.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     B.dirty_blob().allocated_test(bluestore_pextent_t(0x1, 0x2000));
     B.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
     B.get_ref(coll.get(), 0, 0xff0);
@@ -1094,8 +1074,6 @@ TEST(Blob, legacy_decode)
     auto p2 = bl2.front().begin_deep();
     BlueStore::Blob Bres(coll.get());
     BlueStore::Blob Bres2(coll.get());
-    Bres.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    Bres2.set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
     uint64_t sbid, sbid2;
     Bres.decode(
@@ -1130,7 +1108,6 @@ TEST(ExtentMap, seek_lextent)
   BlueStore::ExtentMap em(&onode,
     g_ceph_context->_conf->bluestore_extent_map_inline_shard_prealloc_size);
   BlueStore::BlobRef br(coll->new_blob());
-  br->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
   ASSERT_EQ(em.extent_map.end(), em.seek_lextent(0));
   ASSERT_EQ(em.extent_map.end(), em.seek_lextent(100));
@@ -1183,7 +1160,6 @@ TEST(ExtentMap, has_any_lextents)
   BlueStore::ExtentMap em(&onode,
     g_ceph_context->_conf->bluestore_extent_map_inline_shard_prealloc_size);
   BlueStore::BlobRef b(coll->new_blob());
-  b->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
 
   ASSERT_FALSE(em.has_any_lextents(0, 0));
   ASSERT_FALSE(em.has_any_lextents(0, 1000));
@@ -1880,10 +1856,6 @@ TEST(GarbageCollector, BasicTest)
     BlueStore::BlobRef b2(coll->new_blob());
     BlueStore::BlobRef b3(coll->new_blob());
     BlueStore::BlobRef b4(coll->new_blob());
-    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x2000, 0x1000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x1000));
     b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x1000));
@@ -1946,10 +1918,6 @@ TEST(GarbageCollector, BasicTest)
     BlueStore::BlobRef b2(coll->new_blob());
     BlueStore::BlobRef b3(coll->new_blob());
     BlueStore::BlobRef b4(coll->new_blob());
-    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x40000, 0x20000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x20000));
     b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x10000));
@@ -2017,8 +1985,6 @@ TEST(GarbageCollector, BasicTest)
     int64_t saving;
     BlueStore::BlobRef b1(coll->new_blob());
     BlueStore::BlobRef b2(coll->new_blob());
-    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b1->dirty_blob().set_compressed(0x4000, 0x2000);
     b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x2000));
     b2->dirty_blob().set_compressed(0x4000, 0x2000);
@@ -2075,11 +2041,6 @@ TEST(GarbageCollector, BasicTest)
     BlueStore::BlobRef b2(coll->new_blob());
     BlueStore::BlobRef b3(coll->new_blob());
     BlueStore::BlobRef b4(coll->new_blob());
-    b0->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b1->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b2->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b3->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
-    b4->set_shared_blob(new BlueStore::SharedBlob(coll.get()));
     b0->dirty_blob().set_compressed(0x2000, 0x1000);
     b0->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x10000));
     b1->dirty_blob().set_compressed(0x20000, 0x10000);

From 07f08183a963aa7c001f9a7b9fdeb2e8f00837e0 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Mon, 27 Nov 2023 15:26:25 +0100
Subject: [PATCH 1833/2492] os/bluestore: create dummy blob on dummy onodes

dummy onodes don't have a collection, consequentially, we need to create
a blob without a collection.

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc | 7 ++++---
 src/os/bluestore/BlueStore.h  | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 9e3c72f6e3d8..3e08056bf935 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4023,7 +4023,8 @@ void BlueStore::ExtentMap::ExtentDecoder::decode_extent(
     if (blobid) {
       consume_blobid(le, false, blobid - 1);
     } else {
-      BlobRef b = c->new_blob();
+      // dummy onodes might not have collections, we need a check for it.
+      BlobRef b = c ? c->new_blob() : new Blob(nullptr);
       uint64_t sbid = 0;
       b->decode(p, struct_v, &sbid, false, c);
       consume_blob(le, extent_pos, sbid, b);
@@ -4071,7 +4072,7 @@ void BlueStore::ExtentMap::ExtentDecoder::decode_spanning_blobs(
   unsigned n;
   denc_varint(n, p);
   while (n--) {
-    BlueStore::BlobRef b = c->new_blob();
+    BlobRef b = c ? c->new_blob() : new Blob(nullptr);
     denc_varint(b->id, p);
     uint64_t sbid = 0;
     b->decode(p, struct_v, &sbid, true, c);
@@ -11051,7 +11052,7 @@ void BlueStore::inject_zombie_spanning_blob(coll_t cid, ghobject_t oid,
     o->extent_map.fault_range(db, 0, OBJECT_MAX_SIZE);
   }
 
-  BlobRef b = c->new_blob();
+  BlobRef b = c ? c->new_blob() : new Blob(nullptr);
   b->id = blob_id;
   o->extent_map.spanning_blob_map[blob_id] = b;
 
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 7baaf503e39a..9985ea5752ae 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -728,7 +728,7 @@ class BlueStore : public ObjectStore,
       return shared_blob && shared_blob->is_loaded();
     }
     inline BufferCacheShard* get_cache() {
-      return collection->cache;
+      return collection ? collection->cache : nullptr;
     }
     uint64_t get_sbid() const {
       return shared_blob ? shared_blob->get_sbid() : 0;

From 6574407890400897db1897d1de6091ccd5497add Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Wed, 13 Dec 2023 13:14:31 +0100
Subject: [PATCH 1834/2492] os/bluestore: make shared_blob access private

introduce const and non const accessors:
  const SharedblobRef& get_shared_blob
  SharedblobRef& get_dirty_shared_blob

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc | 64 +++++++++++++++++------------------
 src/os/bluestore/BlueStore.h  | 13 +++++--
 2 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 3e08056bf935..2c76964e588b 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3034,12 +3034,12 @@ void BlueStore::ExtentMap::scan_shared_blobs(
     if (ep->blob->last_encoded_id == -1) {
       const bluestore_blob_t& blob = ep->blob->get_blob();
       if (blob.is_shared()) {
-	// excellent time to load the blob
-	c->load_shared_blob(ep->blob->shared_blob);
-	if (!blob.is_compressed()) {
-	  // Restrict elastic shared blobs to non-compressed blobs.
-	  // Fsck cannot handle case when one shared blob contains refs to
-	  // both shared and non-shared blobs.
+        // excellent time to load the blob
+        c->load_shared_blob(ep->blob->get_shared_blob());
+        if (!blob.is_compressed()) {
+          // Restrict elastic shared blobs to non-compressed blobs.
+          // Fsck cannot handle case when one shared blob contains refs to
+          // both shared and non-shared blobs.
 
 	  // todo consider change to emplace_hint
 	  candidates.emplace(ep->blob_start(), ep->blob.get());
@@ -3147,7 +3147,7 @@ void BlueStore::ExtentMap::make_range_shared_maybe_merge(
 	uint32_t b_logical_length = b->merge_blob(store->cct, e.blob.get());
 	for (auto p : blob.get_extents()) {
 	  if (p.is_valid()) {
-	    b->shared_blob->get_ref(p.offset, p.length);
+	    b->get_shared_blob()->get_ref(p.offset, p.length);
 	  }
 	}
 	// reblob extents might erase e
@@ -3165,7 +3165,7 @@ void BlueStore::ExtentMap::make_range_shared_maybe_merge(
 	++ep;
       }
     } else {
-      c->load_shared_blob(e.blob->shared_blob);
+      c->load_shared_blob(e.blob->get_shared_blob());
       ++ep;
     }
   }
@@ -3226,7 +3226,7 @@ void BlueStore::ExtentMap::dup(BlueStore* b, TransContext* txc,
         // -1 to exclude next potential shard
         dirty_range_end = e.logical_end() - 1;
       } else {
-        c->load_shared_blob(e.blob->shared_blob);
+        c->load_shared_blob(e.blob->get_shared_blob());
       }
       cb = c->new_blob();
       e.blob->last_encoded_id = n;
@@ -3245,10 +3245,10 @@ void BlueStore::ExtentMap::dup(BlueStore* b, TransContext* txc,
       // bump the extent refs on the copied blob's extents
       for (auto p : blob.get_extents()) {
         if (p.is_valid()) {
-          e.blob->shared_blob->get_ref(p.offset, p.length);
+          e.blob->get_shared_blob()->get_ref(p.offset, p.length);
         }
       }
-      txc->write_shared_blob(e.blob->shared_blob);
+      txc->write_shared_blob(e.blob->get_shared_blob());
       dout(20) << __func__ << "    new " << *cb << dendl;
     }
 
@@ -3357,7 +3357,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
 	// we must copy source blob diligently region-by-region
 	// initialize shared_blob
 	cb->dirty_blob().set_flag(bluestore_blob_t::FLAG_SHARED);
-	cb->set_shared_blob(e.blob->shared_blob);
+	cb->set_shared_blob(e.blob->get_shared_blob());
       }
       // By default do not copy buffers to clones, and let them read data by themselves.
       // The exception are 'writing' buffers, which are not yet stable on device.
@@ -3369,7 +3369,7 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
 	txc->blobs_written.insert(cb);
       }
 
-      txc->write_shared_blob(e.blob->shared_blob);
+      txc->write_shared_blob(e.blob->get_shared_blob());
       dout(20) << __func__ << "    new " << *cb << dendl;
     }
 
@@ -4512,7 +4512,7 @@ BlueStore::ExtentMap::debug_list_disk_layout()
     bluestore_extent_ref_map_t* ref_map = nullptr;
     if (bblob.is_shared()) {
       ceph_assert(ep->blob->is_shared_loaded());
-      bluestore_shared_blob_t* bsblob = ep->blob->shared_blob->persistent;
+      bluestore_shared_blob_t* bsblob = ep->blob->get_shared_blob()->persistent;
       ref_map = &bsblob->ref_map;
     }
 
@@ -4979,7 +4979,7 @@ void BlueStore::Collection::flush_all_but_last()
 
 void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
 {
-  ceph_assert(!b->shared_blob);
+  ceph_assert(!b->get_shared_blob());
   const bluestore_blob_t& blob = b->get_blob();
   if (!blob.is_shared()) {
     return;
@@ -4989,12 +4989,12 @@ void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
   if (sb) {
     b->set_shared_blob(sb);
     ldout(store->cct, 10) << __func__ << " sbid 0x" << std::hex << sbid
-			  << std::dec << " had " << *b->shared_blob << dendl;
+			  << std::dec << " had " << *b->get_shared_blob() << dendl;
   } else {
     b->set_shared_blob(new SharedBlob(sbid, this));
-    shared_blob_set.add(this, b->shared_blob.get());
+    shared_blob_set.add(this, b->get_shared_blob().get());
     ldout(store->cct, 10) << __func__ << " sbid 0x" << std::hex << sbid
-			  << std::dec << " opened " << *b->shared_blob
+			  << std::dec << " opened " << *b->get_shared_blob()
 			  << dendl;
   }
 }
@@ -5035,12 +5035,12 @@ void BlueStore::Collection::make_blob_shared(uint64_t sbid, BlobRef b)
   blob.clear_flag(bluestore_blob_t::FLAG_HAS_UNUSED);
   // update shared blob
   b->set_shared_blob(new SharedBlob(sbid, this));
-  b->shared_blob->loaded = true;
-  b->shared_blob->persistent = new bluestore_shared_blob_t(sbid);
-  shared_blob_set.add(this, b->shared_blob.get());
+  b->get_shared_blob()->loaded = true;
+  b->get_shared_blob()->persistent = new bluestore_shared_blob_t(sbid);
+  shared_blob_set.add(this, b->get_shared_blob().get());
   for (auto p : blob.get_extents()) {
     if (p.is_valid()) {
-      b->shared_blob->get_ref(
+      b->get_shared_blob()->get_ref(
 	p.offset,
 	p.length);
     }
@@ -5168,7 +5168,7 @@ void BlueStore::Collection::split_cache(
 	}
 	cache->rm_blob();
 	dest->cache->add_blob();
-	SharedBlob* sb = b->shared_blob.get();
+	SharedBlob* sb = b->get_shared_blob().get();
         b->collection = dest;
         if (sb) {
           if (sb->collection == dest) {
@@ -16304,22 +16304,22 @@ void BlueStore::_wctx_finish(
       dout(20) << __func__ << "  blob " << *b << " release " << r << dendl;
       if (blob.is_shared()) {
 	PExtentVector final;
-        c->load_shared_blob(b->shared_blob);
+        c->load_shared_blob(b->get_shared_blob());
 	bool unshare = false;
 	bool* unshare_ptr =
 	  !maybe_unshared_blobs || b->is_referenced() ? nullptr : &unshare;
 	for (auto e : r) {
-	  b->shared_blob->put_ref(
+	  b->get_shared_blob()->put_ref(
 	    e.offset, e.length, &final,
 	    unshare_ptr);
 	}
 	if (unshare) {
 	  ceph_assert(maybe_unshared_blobs);
-	  maybe_unshared_blobs->insert(b->shared_blob.get());
+	  maybe_unshared_blobs->insert(b->get_shared_blob().get());
 	}
 	dout(20) << __func__ << "  shared_blob release " << final
-		 << " from " << *b->shared_blob << dendl;
-	txc->write_shared_blob(b->shared_blob);
+		 << " from " << *b->get_shared_blob() << dendl;
+	txc->write_shared_blob(b->get_shared_blob());
 	r.clear();
 	r.swap(final);
       }
@@ -16851,7 +16851,7 @@ int BlueStore::_do_remove(
   map<SharedBlob*,bluestore_extent_ref_map_t> expect;
   for (auto& e : h->extent_map.extent_map) {
     const bluestore_blob_t& b = e.blob->get_blob();
-    SharedBlob *sb = e.blob->shared_blob.get();
+    SharedBlob *sb = e.blob->get_shared_blob().get();
     if (b.is_shared() &&
 	sb->loaded &&
 	maybe_unshared_blobs.count(sb)) {
@@ -16892,14 +16892,14 @@ int BlueStore::_do_remove(
   // And now a run through .head extents to clear up freshly unshared blobs.
   for (auto& e : h->extent_map.extent_map) {
     const bluestore_blob_t& b = e.blob->get_blob();
-    SharedBlob *sb = e.blob->shared_blob.get();
+    SharedBlob *sb = e.blob->get_shared_blob().get();
     if (b.is_shared() &&
         std::find(unshared_blobs.begin(), unshared_blobs.end(),
                   sb) != unshared_blobs.end()) {
       dout(20) << __func__ << "  unsharing " << e << dendl;
       bluestore_blob_t& blob = e.blob->dirty_blob();
       blob.clear_flag(bluestore_blob_t::FLAG_SHARED);
-      if (e.blob->shared_blob->nref > 1) {
+      if (e.blob->get_shared_blob()->nref > 1) {
 	// Each blob on creation gets its own unique (empty) shared_blob.
 	// In function ExtentMap::dup() we sometimes merge 2 blobs,
 	// so they share common shared_blob used for ref counting.
@@ -16911,7 +16911,7 @@ int BlueStore::_do_remove(
 
 	// Here we skip set_shared_blob() because e.blob is already in BufferCacheShard
 	// and cannot do add_blob() twice
-	e.blob->shared_blob = new SharedBlob(c.get());
+        e.blob->get_dirty_shared_blob() = new SharedBlob(c.get());
       }
       h->extent_map.dirty_range(e.logical_offset, 1);
     }
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 9985ea5752ae..caaa787b0b90 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -605,7 +605,6 @@ class BlueStore : public ObjectStore,
     std::atomic_int nref = {0};     ///< reference count
     int16_t id = -1;                ///< id, for spanning blobs only, >= 0
     int16_t last_encoded_id = -1;   ///< (ephemeral) used during encoding only
-    SharedBlobRef shared_blob;      ///< shared blob state (if any)
     CollectionRef collection;
 
     void set_shared_blob(SharedBlobRef sb) {
@@ -618,6 +617,7 @@ class BlueStore : public ObjectStore,
     Blob(CollectionRef collection) : collection(collection) {}
     BufferSpace bc;
   private:
+    SharedBlobRef shared_blob;      ///< shared blob state (if any)
     mutable bluestore_blob_t blob;  ///< decoded blob metadata
 #ifdef CACHE_BLOB_BL
     mutable ceph::buffer::list blob_bl;     ///< cached encoded blob, blob is dirty if empty
@@ -639,6 +639,15 @@ class BlueStore : public ObjectStore,
     bluestore_blob_use_tracker_t& dirty_blob_use_tracker() {
       return used_in_blob;
     }
+
+    const SharedBlobRef& get_shared_blob() const {
+      return shared_blob;
+    }
+
+    SharedBlobRef& get_dirty_shared_blob() {
+      return shared_blob;
+    }
+
     bool is_referenced() const {
       return used_in_blob.is_not_empty();
     }
@@ -1906,7 +1915,7 @@ class BlueStore : public ObjectStore,
     void write_onode(OnodeRef& o) {
       onodes.insert(o);
     }
-    void write_shared_blob(SharedBlobRef &sb) {
+    void write_shared_blob(const SharedBlobRef &sb) {
       shared_blobs.insert(sb);
     }
     void unshare_blob(SharedBlob *sb) {

From d5510b0f62677641b642b304b6a99ad2239a71f2 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Thu, 14 Dec 2023 19:38:58 +0100
Subject: [PATCH 1835/2492] os/bluestore: test unshare blob + clone errors

This will ensure deleting and then cloning the same blob works.

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/test/objectstore/store_test.cc | 67 ++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 61ac86a8feeb..a923ede5c12c 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -3769,6 +3769,73 @@ TEST_P(StoreTest, SimpleCloneRangeTest) {
 }
 
 #if defined(WITH_BLUESTORE)
+TEST_P(StoreTest, BlueStoreUnshareBlobSimple) {
+  if (string(GetParam()) != "bluestore")
+    return;
+  int r;
+  coll_t cid;
+  auto ch = store->create_new_collection(cid);
+  {
+    ObjectStore::Transaction t;
+    t.create_collection(cid, 0);
+    cerr << "Creating collection " << cid << std::endl;
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+  ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
+  hoid.hobj.pool = -1;
+  ghobject_t hoid2(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
+  hoid2.hobj.pool = -1;
+  hoid2.generation = 2;
+  {
+    // multiples of unit_size are necesary so that ref_map in sharedblob is equal to the one in maybe_shared_blob
+    bufferlist data;
+    data.append(string(4096, 'a'));
+
+    ObjectStore::Transaction t;
+    t.write(cid, hoid, 0, data.length(), data);
+    cerr << "Creating object and write 4K " << hoid << std::endl;
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+
+    ObjectStore::Transaction t2;
+    t2.clone(cid, hoid, hoid2);
+    cerr << "Clone object" << std::endl;
+    r = queue_transaction(store, ch, std::move(t2));
+    ASSERT_EQ(r, 0);
+
+
+  }
+
+  {
+    // This should unshare previous Blob, only works with generations
+    ObjectStore::Transaction t;
+    t.remove(cid, hoid2);
+    cerr << "Removing gen 2" << std::endl;
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+
+  {
+    ObjectStore::Transaction t;
+    cerr << "Cloning again" << std::endl;
+    // expect share blob to work again
+    t.clone(cid, hoid, hoid2);
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+
+  {
+    ObjectStore::Transaction t;
+    // clean up
+    t.remove(cid, hoid);
+    t.remove(cid, hoid2);
+    t.remove_collection(cid);
+    cerr << "Cleaning" << std::endl;
+    r = queue_transaction(store, ch, std::move(t));
+    ASSERT_EQ(r, 0);
+  }
+}
 TEST_P(StoreTest, BlueStoreUnshareBlobTest) {
   if (string(GetParam()) != "bluestore")
     return;

From 4f6245fe1bde3690a31cf2c626fbaf3141af6c26 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Thu, 14 Dec 2023 19:43:07 +0100
Subject: [PATCH 1836/2492] os/bluestore: set to nullptr sharedblob on unshare

set_shared_blob requires shared_blob to be a nullptr to ensure
it's correct operation.

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 2c76964e588b..99d12f3f22f8 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -16908,10 +16908,7 @@ int BlueStore::_do_remove(
 	// but now those 2 blobs share it.
 	// This is illegal, as empty shared blobs should be unique.
 	// Fixing by re-creation.
-
-	// Here we skip set_shared_blob() because e.blob is already in BufferCacheShard
-	// and cannot do add_blob() twice
-        e.blob->get_dirty_shared_blob() = new SharedBlob(c.get());
+        e.blob->get_dirty_shared_blob() = nullptr;
       }
       h->extent_map.dirty_range(e.logical_offset, 1);
     }

From ed64db21104fa36c296c8c72691f9c24afaa9227 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Wed, 10 Jan 2024 19:18:03 +0100
Subject: [PATCH 1837/2492] os/bluestore: get_shared_blob in
 test_bluestore_types

private shared_blob member access needed to be changed

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/test/objectstore/test_bluestore_types.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc
index cf41b504259f..db0ee3808911 100644
--- a/src/test/objectstore/test_bluestore_types.cc
+++ b/src/test/objectstore/test_bluestore_types.cc
@@ -1701,7 +1701,7 @@ TEST(ExtentMap, dup_extent_map)
     BlueStore::BlobRef b2 = em2.seek_lextent(ext1_offs)->blob;
     ASSERT_TRUE(b2->get_blob().is_shared());
     ASSERT_EQ(b2->get_referenced_bytes(), ext1_len);
-    ASSERT_EQ(b1->shared_blob, b2->shared_blob);
+    ASSERT_EQ(b1->get_shared_blob(), b2->get_shared_blob());
     auto &_b2 = b2->get_blob();
     ASSERT_EQ(_b1.get_csum_count(), _b2.get_csum_count());
     for(size_t i = 0; i < _b2.get_csum_count(); i++) {
@@ -1733,14 +1733,14 @@ TEST(ExtentMap, dup_extent_map)
     // make sure (basically) onode1&2 are unmodified
     BlueStore::BlobRef b1 = em1.seek_lextent(ext1_offs)->blob;
     BlueStore::BlobRef b2 = em2.seek_lextent(ext1_offs)->blob;
-    ASSERT_EQ(b1->shared_blob, b2->shared_blob);
+    ASSERT_EQ(b1->get_shared_blob(), b2->get_shared_blob());
 
     BlueStore::Extent &ext3 = *em3.seek_lextent(clone_offs);
     ASSERT_EQ(ext3.blob_offset, clone_shift);
     ASSERT_EQ(ext3.length, clone_len);
     BlueStore::BlobRef b3 = ext3.blob;
     ASSERT_TRUE(b3->get_blob().is_shared());
-    ASSERT_EQ(b3->shared_blob, b1->shared_blob);
+    ASSERT_EQ(b3->get_shared_blob(), b1->get_shared_blob());
     ASSERT_EQ(b3->get_referenced_bytes(), clone_len);
     auto ll = b3->get_blob().get_logical_length();
     ASSERT_EQ(ll, ext1_len);
@@ -1777,8 +1777,8 @@ TEST(ExtentMap, dup_extent_map)
     BlueStore::BlobRef b1 = em1.seek_lextent(ext1_offs)->blob;
     BlueStore::BlobRef b2 = em2.seek_lextent(ext1_offs)->blob;
     BlueStore::BlobRef b3 = em3.seek_lextent(ext1_offs)->blob;
-    ASSERT_EQ(b1->shared_blob, b2->shared_blob);
-    ASSERT_EQ(b1->shared_blob, b3->shared_blob);
+    ASSERT_EQ(b1->get_shared_blob(), b2->get_shared_blob());
+    ASSERT_EQ(b1->get_shared_blob(), b3->get_shared_blob());
     auto &_b2 = b2->get_blob();
 
     BlueStore::Extent &ext4 = *em4.seek_lextent(clone_offs);
@@ -1786,7 +1786,7 @@ TEST(ExtentMap, dup_extent_map)
     ASSERT_EQ(ext4.length, clone_len);
     BlueStore::BlobRef b4 = ext4.blob;
     ASSERT_TRUE(b4->get_blob().is_shared());
-    ASSERT_EQ(b4->shared_blob, b2->shared_blob);
+    ASSERT_EQ(b4->get_shared_blob(), b2->get_shared_blob());
     ASSERT_EQ(b4->get_referenced_bytes(), clone_len);
     auto &_b4 = b4->get_blob();
     auto ll = _b4.get_logical_length();

From de6840ce37be5f195fa987b6f2db9785c9da2619 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Fri, 2 Feb 2024 13:15:50 +0100
Subject: [PATCH 1838/2492] os/bluestore: set shared_blob to nullptr on unshare

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
---
 src/os/bluestore/BlueStore.cc | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 99d12f3f22f8..7714caaad44b 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -16899,17 +16899,7 @@ int BlueStore::_do_remove(
       dout(20) << __func__ << "  unsharing " << e << dendl;
       bluestore_blob_t& blob = e.blob->dirty_blob();
       blob.clear_flag(bluestore_blob_t::FLAG_SHARED);
-      if (e.blob->get_shared_blob()->nref > 1) {
-	// Each blob on creation gets its own unique (empty) shared_blob.
-	// In function ExtentMap::dup() we sometimes merge 2 blobs,
-	// so they share common shared_blob used for ref counting.
-	// Imagine 2 blobs having same shared_blob, and shared blob gets just unshared.
-	// We cleared shared_blob content so it is now logically empty,
-	// but now those 2 blobs share it.
-	// This is illegal, as empty shared blobs should be unique.
-	// Fixing by re-creation.
-        e.blob->get_dirty_shared_blob() = nullptr;
-      }
+      e.blob->get_dirty_shared_blob() = nullptr;
       h->extent_map.dirty_range(e.logical_offset, 1);
     }
   }

From 7bcd0b03544eada041beec4bd0911160497b55d5 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Thu, 8 Feb 2024 17:58:42 +0530
Subject: [PATCH 1839/2492] client: remove unused/undefined fake_write_size()

there are no usages of this function in entire codebase

Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/client/Client.h b/src/client/Client.h
index 9c1303384be8..d49fd46c1764 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -485,7 +485,6 @@ class Client : public Dispatcher, public md_config_obs_t {
   int preadv(int fd, const struct iovec *iov, int iovcnt, loff_t offset=-1);
   int write(int fd, const char *buf, loff_t size, loff_t offset=-1);
   int pwritev(int fd, const struct iovec *iov, int iovcnt, loff_t offset=-1);
-  int fake_write_size(int fd, loff_t size);
   int ftruncate(int fd, loff_t size, const UserPerm& perms);
   int fsync(int fd, bool syncdataonly);
   int fstat(int fd, struct stat *stbuf, const UserPerm& perms,

From de578245198e0a91628384a0759c73710dbf4b98 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 8 Feb 2024 10:56:03 -0500
Subject: [PATCH 1840/2492] doc: deprecate blkin tracing

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 PendingReleaseNotes | 3 +++
 doc/dev/blkin.rst   | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 0ecec83ec81f..a413b6276f2f 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -10,6 +10,9 @@
   For multi-site deployments that make any use of Server-Side Encryption, we
   recommended running this command against every bucket in every zone after all
   zones have upgraded.
+* Tracing: The blkin tracing feature (see https://docs.ceph.com/en/reef/dev/blkin/)
+  is now deprecated in favor of Opentracing (https://docs.ceph.com/en/reef/dev/developer_guide/jaegertracing/)
+  and will be removed in a later release.
 * CEPHFS: MDS evicts clients which are not advancing their request tids which causes
   a large buildup of session metadata resulting in the MDS going read-only due to
   the RADOS operation exceeding the size threshold. `mds_session_metadata_threshold`
diff --git a/doc/dev/blkin.rst b/doc/dev/blkin.rst
index 989cddcd7ee6..f4c045e19efb 100644
--- a/doc/dev/blkin.rst
+++ b/doc/dev/blkin.rst
@@ -72,6 +72,9 @@ Destroy tracing session::
  Tracing Ceph With Blkin
 =========================
 
+.. deprecated:: This feature was deprecated in the Squid release and will
+   be removed in a later release.
+
 Ceph can use Blkin, a library created by Marios Kogias and others,
 which enables tracking a specific request from the time it enters
 the system at higher levels till it is finally served by RADOS.

From 5c613b3788d9ae686b4dc29d9414674ecb6f6adb Mon Sep 17 00:00:00 2001
From: Roy Sahar <royswi@gmail.com>
Date: Thu, 8 Feb 2024 17:58:43 +0200
Subject: [PATCH 1841/2492] nvmeof: Add mount for log location

Signed-off-by: Roy Sahar <royswi@gmail.com>
---
 src/cephadm/cephadmlib/daemons/nvmeof.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
index 39488406bc8e..5fb13f63faeb 100644
--- a/src/cephadm/cephadmlib/daemons/nvmeof.py
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -63,7 +63,7 @@ def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
     @staticmethod
-    def _get_container_mounts(data_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(data_dir: str, log_dir: str) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
         mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
@@ -73,13 +73,15 @@ def _get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
         mounts['/dev/hugepages'] = '/dev/hugepages'
         mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
+        mounts[log_dir] = '/var/log/ceph:z'
         return mounts
 
     def customize_container_mounts(
         self, ctx: CephadmContext, mounts: Dict[str, str]
     ) -> None:
         data_dir = self.identity.data_dir(ctx.data_dir)
-        mounts.update(self._get_container_mounts(data_dir))
+        log_dir = os.path.join(ctx.log_dir, self.identity.fsid)
+        mounts.update(self._get_container_mounts(data_dir, log_dir))
 
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]

From b80c12d080275e3c221a1e305c345fbc1caa3c8e Mon Sep 17 00:00:00 2001
From: Eduardo Roldan <trazor@gmail.com>
Date: Thu, 8 Feb 2024 03:40:21 -0300
Subject: [PATCH 1842/2492] doc:start.rst fix typo in hw-recs

s/power less/power loss/

Signed-off-by: Eduardo Roldan <trazor@gmail.com>
---
 doc/start/hardware-recommendations.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index e41c2eaa45e1..76dce78013ce 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -249,7 +249,7 @@ once a limited cache is filled declines considerably.  Consider also durability:
 a drive rated for 0.3 Drive Writes Per Day (DWPD or equivalent) may be fine for
 OSDs dedicated to certain types of sequentially-written read-mostly data, but
 are not a good choice for Ceph Monitor duty.  Enterprise-class SSDs are best
-for Ceph:  they almost always feature power less protection (PLP) and do
+for Ceph:  they almost always feature power loss protection (PLP) and do
 not suffer the dramatic cliffing that client (desktop) models may experience.
 
 When using a single (or mirrored pair) SSD for both operating system boot

From 4ae47bd7af6f6e1c4218c3e84db438dc0a9c404b Mon Sep 17 00:00:00 2001
From: Matt Vandermeulen <matt@reenigne.net>
Date: Thu, 8 Feb 2024 13:54:22 -0400
Subject: [PATCH 1843/2492] common: add discard threads option, descriptions
 and flags

Signed-off-by: Matt Vandermeulen <matt@reenigne.net>
---
 src/common/options/global.yaml.in | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 88c896f90262..6f255f9d5768 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4000,15 +4000,39 @@ options:
   default: false
   with_legacy: true
 - name: bdev_enable_discard
+  desc: send discards to the block device
   type: bool
   level: advanced
   default: false
   with_legacy: true
+  flags:
+  - runtime
+  see_also:
+  - bdev_async_discard
+  - bdev_async_discard_threads
 - name: bdev_async_discard
-  type: bool
+  desc: send discards to the block device in one or more threads
+  type: uint
   level: advanced
   default: false
-  with_legacy: true
+  with_legacy: false
+  flags:
+  - runtime
+  see_also:
+  - bdev_enable_discard
+  - bdev_async_discard_threads
+- name: bdev_async_discard_threads
+  desc: number of discard threads used to issue discards to the device
+  type: uint
+  level: advanced
+  default: 1
+  min: 1
+  with_legacy: false
+  flags:
+  - runtime
+  see_also:
+  - bdev_enable_discard
+  - bdev_async_discard
 - name: bdev_flock_retry_interval
   type: float
   level: advanced

From d8815e18b7e2c75db0f55dd4497ea7285bd4ae25 Mon Sep 17 00:00:00 2001
From: Matt Vandermeulen <matt@reenigne.net>
Date: Thu, 8 Feb 2024 13:54:37 -0400
Subject: [PATCH 1844/2492] blk: add threaded discard support to kernel devices

Signed-off-by: Matt Vandermeulen <matt@reenigne.net>
---
 src/blk/kernel/KernelDevice.cc | 152 ++++++++++++++++++++++++++-------
 src/blk/kernel/KernelDevice.h  |  26 ++++--
 2 files changed, 141 insertions(+), 37 deletions(-)

diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc
index 754b44d32a69..6ce5b3a92933 100644
--- a/src/blk/kernel/KernelDevice.cc
+++ b/src/blk/kernel/KernelDevice.cc
@@ -65,12 +65,10 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
     discard_callback(d_cb),
     discard_callback_priv(d_cbpriv),
     aio_stop(false),
-    discard_started(false),
-    discard_stop(false),
     aio_thread(this),
-    discard_thread(this),
     injecting_crash(0)
 {
+  cct->_conf.add_observer(this);
   fd_directs.resize(WRITE_LIFE_MAX, -1);
   fd_buffereds.resize(WRITE_LIFE_MAX, -1);
 
@@ -92,6 +90,11 @@ KernelDevice::KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, ai
   }
 }
 
+KernelDevice::~KernelDevice()
+{
+  cct->_conf.remove_observer(this);
+}
+
 int KernelDevice::_lock()
 {
   // When the block changes, systemd-udevd will open the block,
@@ -131,6 +134,7 @@ int KernelDevice::open(const string& p)
 {
   path = p;
   int r = 0, i = 0;
+  uint64_t num_discard_threads = 0;
   dout(1) << __func__ << " path " << path << dendl;
 
   struct stat statbuf;
@@ -281,7 +285,9 @@ int KernelDevice::open(const string& p)
   if (r < 0) {
     goto out_fail;
   }
-  if (support_discard && cct->_conf->bdev_enable_discard && cct->_conf->bdev_async_discard) {
+
+  num_discard_threads = cct->_conf.get_val<uint64_t>("bdev_async_discard_threads");
+  if (support_discard && cct->_conf->bdev_enable_discard && num_discard_threads > 0) {
     _discard_start();
   }
 
@@ -330,7 +336,7 @@ void KernelDevice::close()
 {
   dout(1) << __func__ << dendl;
   _aio_stop();
-  if (discard_thread.is_started()) {
+  if (_discard_started()) {
     _discard_stop();
   }
   _pre_close();
@@ -532,28 +538,55 @@ void KernelDevice::_aio_stop()
 
 void KernelDevice::_discard_start()
 {
-    discard_thread.create("bstore_discard");
+  uint64_t num = cct->_conf.get_val<uint64_t>("bdev_async_discard_threads");
+  dout(10) << __func__ << " starting " << num << " threads" << dendl;
+
+  std::unique_lock l(discard_lock);
+
+  target_discard_threads = num;
+  discard_threads.reserve(num);
+  for(uint64_t i = 0; i < num; i++)
+  {
+    // All threads created with the same name
+    discard_threads.emplace_back(new DiscardThread(this, i));
+    discard_threads.back()->create("bstore_discard");
+  }
+
+  dout(10) << __func__ << " started " << num << " threads" << dendl;
 }
 
 void KernelDevice::_discard_stop()
 {
   dout(10) << __func__ << dendl;
+
+  // Signal threads to stop, then wait for them to join
   {
     std::unique_lock l(discard_lock);
-    while (!discard_started) {
+    while (discard_threads.empty()) {
       discard_cond.wait(l);
     }
-    discard_stop = true;
+
+    for(auto &t : discard_threads) {
+      t->stop = true;
+    }
+
     discard_cond.notify_all();
   }
-  discard_thread.join();
-  {
-    std::lock_guard l(discard_lock);
-    discard_stop = false;
-  }
+
+  // Threads are shared pointers and are cleaned up for us
+  for(auto &t : discard_threads)
+    t->join();
+  discard_threads.clear();
+
   dout(10) << __func__ << " stopped" << dendl;
 }
 
+bool KernelDevice::_discard_started()
+{
+  std::unique_lock l(discard_lock);
+  return !discard_threads.empty();
+}
+
 void KernelDevice::discard_drain()
 {
   dout(10) << __func__ << dendl;
@@ -567,7 +600,7 @@ static bool is_expected_ioerr(const int r)
 {
   // https://lxr.missinglinkelectronics.com/linux+v4.15/block/blk-core.c#L135
   return (r == -EOPNOTSUPP || r == -ETIMEDOUT || r == -ENOSPC ||
-	  r == -ENOLINK || r == -EREMOTEIO  || r == -EAGAIN || r == -EIO ||
+	  r == -ENOLINK || r == -EREMOTEIO || r == -EAGAIN || r == -EIO ||
 	  r == -ENODATA || r == -EILSEQ || r == -ENOMEM ||
 #if defined(__linux__)
 	  r == -EREMCHG || r == -EBADE
@@ -698,44 +731,57 @@ void KernelDevice::_aio_thread()
   dout(10) << __func__ << " end" << dendl;
 }
 
-void KernelDevice::_discard_thread()
+void KernelDevice::_discard_thread(uint64_t tid)
 {
+  dout(10) << __func__ << " thread " << tid << " start" << dendl;
+
+  // Thread-local list of processing discards
+  interval_set<uint64_t> discard_processing;
+
   std::unique_lock l(discard_lock);
-  ceph_assert(!discard_started);
-  discard_started = true;
   discard_cond.notify_all();
+
+  // Keeps the shared pointer around until erased from the vector
+  // and until we leave this function
+  auto thr = discard_threads[tid];
+
   while (true) {
-    ceph_assert(discard_finishing.empty());
+    ceph_assert(discard_processing.empty());
     if (discard_queued.empty()) {
-      if (discard_stop)
+      if (thr->stop)
 	break;
       dout(20) << __func__ << " sleep" << dendl;
       discard_cond.notify_all(); // for the thread trying to drain...
       discard_cond.wait(l);
       dout(20) << __func__ << " wake" << dendl;
     } else {
-      discard_finishing.swap(discard_queued);
+      // Swap the queued discards for a local list we'll process here
+      // without caring about thread fairness.  This allows the current
+      // thread to wait on the discard running while other threads pick
+      // up the next-in-queue, and do the same, ultimately issuing more
+      // discards in parallel, which is the goal.
+      discard_processing.swap(discard_queued);
       discard_running = true;
       l.unlock();
       dout(20) << __func__ << " finishing" << dendl;
-      for (auto p = discard_finishing.begin();p != discard_finishing.end(); ++p) {
-	_discard(p.get_start(), p.get_len());
+      for (auto p = discard_processing.begin(); p != discard_processing.end(); ++p) {
+        _discard(p.get_start(), p.get_len());
       }
 
-      discard_callback(discard_callback_priv, static_cast<void*>(&discard_finishing));
-      discard_finishing.clear();
+      discard_callback(discard_callback_priv, static_cast<void*>(&discard_processing));
+      discard_processing.clear();
       l.lock();
       discard_running = false;
     }
   }
-  dout(10) << __func__ << " finish" << dendl;
-  discard_started = false;
+
+  dout(10) << __func__ << " thread " << tid << " finish" << dendl;
 }
 
 int KernelDevice::_queue_discard(interval_set<uint64_t> &to_release)
 {
   // if bdev_async_discard enabled on the fly, discard_thread is not started here, fallback to sync discard
-  if (!discard_thread.is_started())
+  if (!_discard_started())
     return -1;
 
   if (to_release.empty())
@@ -743,7 +789,7 @@ int KernelDevice::_queue_discard(interval_set<uint64_t> &to_release)
 
   std::lock_guard l(discard_lock);
   discard_queued.insert(to_release);
-  discard_cond.notify_all();
+  discard_cond.notify_one();
   return 0;
 }
 
@@ -754,7 +800,7 @@ bool KernelDevice::try_discard(interval_set<uint64_t> &to_release, bool async)
   if (!support_discard || !cct->_conf->bdev_enable_discard)
     return false;
 
-  if (async && discard_thread.is_started()) {
+  if (async) {
     return 0 == _queue_discard(to_release);
   } else {
     for (auto p = to_release.begin(); p != to_release.end(); ++p) {
@@ -1447,3 +1493,51 @@ int KernelDevice::invalidate_cache(uint64_t off, uint64_t len)
   }
   return r;
 }
+
+const char** KernelDevice::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "bdev_async_discard_threads",
+    NULL
+  };
+  return KEYS;
+}
+
+void KernelDevice::handle_conf_change(const ConfigProxy& conf,
+			     const std::set <std::string> &changed)
+{
+  if (changed.count("bdev_async_discard_threads")) {
+    std::unique_lock l(discard_lock);
+
+    uint64_t oldval = target_discard_threads;
+    uint64_t newval = cct->_conf.get_val<uint64_t>("bdev_async_discard_threads");
+
+    target_discard_threads = newval;
+
+    // Increase? Spawn now, it's quick
+    if (newval > oldval) {
+      dout(10) << __func__ << " starting " << (newval - oldval) << " additional discard threads" << dendl;
+      discard_threads.reserve(target_discard_threads);
+      for(uint64_t i = oldval; i < newval; i++)
+      {
+        // All threads created with the same name
+        discard_threads.emplace_back(new DiscardThread(this, i));
+        discard_threads.back()->create("bstore_discard");
+      }
+    } else {
+      // Decrease? Signal threads after telling them to stop
+      dout(10) << __func__ << " stopping " << (oldval - newval) << " existing discard threads" << dendl;
+
+      // Signal the last threads to quit, and stop tracking them
+      for(uint64_t i = oldval - 1; i >= newval; i--)
+      {
+        // Also detach the thread so we no longer need to join
+        discard_threads[i]->stop = true;
+        discard_threads[i]->detach();
+        discard_threads.erase(discard_threads.begin() + i);
+      }
+
+      discard_cond.notify_all();
+    }
+  }
+}
diff --git a/src/blk/kernel/KernelDevice.h b/src/blk/kernel/KernelDevice.h
index e00e31f10b17..326a94339915 100644
--- a/src/blk/kernel/KernelDevice.h
+++ b/src/blk/kernel/KernelDevice.h
@@ -19,6 +19,7 @@
 
 #include "include/types.h"
 #include "include/interval_set.h"
+#include "common/config_obs.h"
 #include "common/Thread.h"
 #include "include/utime.h"
 
@@ -28,7 +29,8 @@
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
 
-class KernelDevice : public BlockDevice {
+class KernelDevice : public BlockDevice,
+                     public md_config_obs_t {
 protected:
   std::string path;
 private:
@@ -50,14 +52,11 @@ class KernelDevice : public BlockDevice {
   aio_callback_t discard_callback;
   void *discard_callback_priv;
   bool aio_stop;
-  bool discard_started;
-  bool discard_stop;
 
   ceph::mutex discard_lock = ceph::make_mutex("KernelDevice::discard_lock");
   ceph::condition_variable discard_cond;
   bool discard_running = false;
   interval_set<uint64_t> discard_queued;
-  interval_set<uint64_t> discard_finishing;
 
   struct AioCompletionThread : public Thread {
     KernelDevice *bdev;
@@ -70,12 +69,16 @@ class KernelDevice : public BlockDevice {
 
   struct DiscardThread : public Thread {
     KernelDevice *bdev;
-    explicit DiscardThread(KernelDevice *b) : bdev(b) {}
+    const uint64_t id;
+    bool stop = false;
+    explicit DiscardThread(KernelDevice *b, uint64_t id) : bdev(b), id(id) {}
     void *entry() override {
-      bdev->_discard_thread();
+      bdev->_discard_thread(id);
       return NULL;
     }
-  } discard_thread;
+  };
+  std::vector<std::shared_ptr<DiscardThread>> discard_threads;
+  uint64_t target_discard_threads = 0;
 
   std::atomic_int injecting_crash;
 
@@ -83,7 +86,7 @@ class KernelDevice : public BlockDevice {
   virtual void  _pre_close() { }  // hook for child implementations
 
   void _aio_thread();
-  void _discard_thread();
+  void _discard_thread(uint64_t tid);
   int _queue_discard(interval_set<uint64_t> &to_release);
   bool try_discard(interval_set<uint64_t> &to_release, bool async = true) override;
 
@@ -92,6 +95,7 @@ class KernelDevice : public BlockDevice {
 
   void _discard_start();
   void _discard_stop();
+  bool _discard_started();
 
   void _aio_log_start(IOContext *ioc, uint64_t offset, uint64_t length);
   void _aio_log_finish(IOContext *ioc, uint64_t offset, uint64_t length);
@@ -116,6 +120,7 @@ class KernelDevice : public BlockDevice {
 
 public:
   KernelDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv);
+  ~KernelDevice();
 
   void aio_submit(IOContext *ioc) override;
   void discard_drain() override;
@@ -151,6 +156,11 @@ class KernelDevice : public BlockDevice {
   int invalidate_cache(uint64_t off, uint64_t len) override;
   int open(const std::string& path) override;
   void close() override;
+
+  // config observer bits
+  const char** get_tracked_conf_keys() const override;
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set <std::string> &changed) override;
 };
 
 #endif

From 8c4768ecb3ec38c8fce209eae9fe931e974d0495 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Wed, 18 Oct 2023 22:52:20 +0000
Subject: [PATCH 1845/2492] qa/tasks/ceph_manager.py: Rewrite
 test_pool_min_size

Problem:

Failed the test in EC Pool configuration because PGs are
not going into active+clean (our fault for over thrashing and checking the wrong thing).
Also, PG would not go into active because we thrash below min_size
in an EC pool config, not enough shards in the acting set.
Therefore, failed the wait_for_recovery check.
Moreover, When we revive osds, we didn't add the osd back in the cluster,
this messes up true count for live_osds in the test.

Solution:

Instead of randomly choosing OSDs to thrash,
we randomly select a PG from each pool and
thrash the OSDs in the PG's acting set until
we reach min_size, then we check to see if the
PG is still active. After that we revive all
the OSDs to see if the PG recovered cleanly.

We removed some of the unnecessary part such
as `min_dead`, `min_live`, `min_out` and etc.

Also, we refractored the part of where we are
assigning k,m for the EC pools so that we get
better code readablility.

Fixes: Fixes: https://tracker.ceph.com/issues/59172

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 qa/tasks/ceph_manager.py | 180 ++++++++++++++++++++-------------------
 1 file changed, 94 insertions(+), 86 deletions(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index a12d8c750116..4a8867a44e26 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -863,10 +863,61 @@ def fix_pgp_num(self, pool=None):
         if self.ceph_manager.set_pool_pgpnum(pool, force):
             self.pools_to_fix_pgp_num.discard(pool)
 
+    def get_rand_pg_acting_set(self, pool_id=None):
+        """
+        Return an acting set of a random PG, you
+        have the option to specify which pool you
+        want the PG from.
+        """
+        pgs = self.ceph_manager.get_pg_stats()
+        if not pgs:
+            self.log('No pgs; doing nothing')
+            return
+        if pool_id:
+           pgs_in_pool = [pg for pg in pgs if int(pg['pgid'].split('.')[0]) == pool_id]
+           pg = random.choice(pgs_in_pool)
+        else:
+            pg = random.choice(pgs)
+        self.log('Choosing PG {id} with acting set {act}'.format(id=pg['pgid'],act=pg['acting']))
+        return pg['acting']
+
+    def get_k_m_ec_pool(self, pool, pool_json):
+        """
+        Returns k and m
+        """
+        k = 0
+        m = 99
+        try:
+            ec_profile = self.ceph_manager.get_pool_property(pool, 'erasure_code_profile')
+            ec_profile = pool_json['erasure_code_profile']
+            ec_profile_json = self.ceph_manager.raw_cluster_cmd(
+                'osd',
+                'erasure-code-profile',
+                'get',
+                ec_profile,
+                '--format=json')
+            ec_json = json.loads(ec_profile_json)
+            local_k = int(ec_json['k'])
+            local_m = int(ec_json['m'])
+            self.log("pool {pool} local_k={k} local_m={m}".format(pool=pool,
+                                                                  k=local_k, m=local_m))
+            if local_k > k:
+                self.log("setting k={local_k} from previous {k}".format(local_k=local_k, k=k))
+                k = local_k
+            if local_m < m:
+                self.log("setting m={local_m} from previous {m}".format(local_m=local_m, m=m))
+                m = local_m
+        except CommandFailedError:
+            self.log("failed to read erasure_code_profile. %s was likely removed", pool)
+            return None, None
+
+        return k, m
+
     def test_pool_min_size(self):
         """
-        Loop to selectively push PGs below their min_size and test that recovery
-        still occurs.
+        Loop to selectively push PGs to their min_size and test that recovery
+        still occurs. We achieve this by randomly picking a PG and fail the OSDs
+        according to the PG's acting set.
         """
         self.log("test_pool_min_size")
         self.all_up()
@@ -874,9 +925,6 @@ def test_pool_min_size(self):
         self.ceph_manager.wait_for_recovery(
             timeout=self.config.get('timeout')
             )
-        minout = int(self.config.get("min_out", 1))
-        minlive = int(self.config.get("min_live", 2))
-        mindead = int(self.config.get("min_dead", 1))
         self.log("doing min_size thrashing")
         self.ceph_manager.wait_for_clean(timeout=180)
         assert self.ceph_manager.is_clean(), \
@@ -885,94 +933,54 @@ def test_pool_min_size(self):
         while time.time() - start < self.config.get("test_min_size_duration", 1800):
             # look up k and m from all the pools on each loop, in case it
             # changes as the cluster runs
-            k = 0
-            m = 99
-            has_pools = False
             pools_json = self.ceph_manager.get_osd_dump_json()['pools']
-
+            if len(pools_json) == 0:
+                self.log("No pools yet, waiting")
+                time.sleep(5)
+                continue
             for pool_json in pools_json:
                 pool = pool_json['pool_name']
-                has_pools = True
+                pool_id = pool_json['pool']
                 pool_type = pool_json['type']  # 1 for rep, 3 for ec
                 min_size = pool_json['min_size']
                 self.log("pool {pool} min_size is {min_size}".format(pool=pool,min_size=min_size))
-                try:
-                    ec_profile = self.ceph_manager.get_pool_property(pool, 'erasure_code_profile')
-                    if pool_type != PoolType.ERASURE_CODED:
-                        continue
-                    ec_profile = pool_json['erasure_code_profile']
-                    ec_profile_json = self.ceph_manager.raw_cluster_cmd(
-                        'osd',
-                        'erasure-code-profile',
-                        'get',
-                        ec_profile,
-                        '--format=json')
-                    ec_json = json.loads(ec_profile_json)
-                    local_k = int(ec_json['k'])
-                    local_m = int(ec_json['m'])
-                    self.log("pool {pool} local_k={k} local_m={m}".format(pool=pool,
-                                                                          k=local_k, m=local_m))
-                    if local_k > k:
-                        self.log("setting k={local_k} from previous {k}".format(local_k=local_k, k=k))
-                        k = local_k
-                    if local_m < m:
-                        self.log("setting m={local_m} from previous {m}".format(local_m=local_m, m=m))
-                        m = local_m
-                except CommandFailedError:
-                    self.log("failed to read erasure_code_profile. %s was likely removed", pool)
+                if pool_type != PoolType.ERASURE_CODED:
                     continue
-
-            if has_pools :
-                self.log("using k={k}, m={m}".format(k=k,m=m))
-            else:
-                self.log("No pools yet, waiting")
-                time.sleep(5)
-                continue
-                
-            if minout > len(self.out_osds): # kill OSDs and mark out
-                self.log("forced to out an osd")
-                self.kill_osd(mark_out=True)
-                continue
-            elif mindead > len(self.dead_osds): # kill OSDs but force timeout
-                self.log("forced to kill an osd")
-                self.kill_osd()
-                continue
-            else: # make mostly-random choice to kill or revive OSDs
-                minup = max(minlive, k)
-                rand_val = random.uniform(0, 1)
-                self.log("choosing based on number of live OSDs and rand val {rand}".\
-                         format(rand=rand_val))
-                if len(self.live_osds) > minup+1 and rand_val < 0.5:
-                    # chose to knock out as many OSDs as we can w/out downing PGs
-                    
-                    most_killable = min(len(self.live_osds) - minup, m)
-                    self.log("chose to kill {n} OSDs".format(n=most_killable))
-                    for i in range(1, most_killable):
-                        self.kill_osd(mark_out=True)
-                    time.sleep(10)
-                    # try a few times since there might be a concurrent pool
-                    # creation or deletion
-                    with safe_while(
-                            sleep=25, tries=5,
-                            action='check for active or peered') as proceed:
-                        while proceed():
-                            if self.ceph_manager.all_active_or_peered():
-                                break
-                            self.log('not all PGs are active or peered')
-                else: # chose to revive OSDs, bring up a random fraction of the dead ones
-                    self.log("chose to revive osds")
-                    for i in range(1, int(rand_val * len(self.dead_osds))):
-                        self.revive_osd(i)
-
-            # let PGs repair themselves or our next knockout might kill one
-            self.ceph_manager.wait_for_clean(timeout=self.config.get('timeout'))
- 
-        # / while not self.stopping
-        self.all_up_in()
- 
-        self.ceph_manager.wait_for_recovery(
-            timeout=self.config.get('timeout')
-            )
+                else:
+                    k, m = self.get_k_m_ec_pool(pool, pool_json)
+                    if k == None and m == None:
+                        continue
+                    self.log("using k={k}, m={m}".format(k=k,m=m))
+
+                self.log("dead_osds={d}, live_osds={ld}".format(d=self.dead_osds, ld=self.live_osds))
+                minup = max(min_size, k)
+                # Choose a random PG and kill OSDs until only min_size remain
+                most_killable = min(len(self.live_osds) - minup, m)
+                self.log("chose to kill {n} OSDs".format(n=most_killable))
+                acting_set = self.get_rand_pg_acting_set(pool_id)
+                assert most_killable < len(acting_set)
+                for i in range(0, most_killable):
+                    self.kill_osd(osd=acting_set[i], mark_out=True)
+                self.log("dead_osds={d}, live_osds={ld}".format(d=self.dead_osds, ld=self.live_osds))
+                self.log("check for active or peered")
+                with safe_while(
+                    sleep=25, tries=5,
+                    action='check for active or peered') as proceed:
+                    while proceed():
+                        if self.ceph_manager.all_active_or_peered():
+                            break
+                        self.log('not all PGs are active or peered')
+                self.all_up_in() # revive all OSDs
+                # let PGs repair themselves or our next knockout might kill one
+                # wait_for_recovery since some workloads won't be able to go clean
+                self.ceph_manager.wait_for_recovery(
+                    timeout=self.config.get('timeout')
+                )
+        # while not self.stopping
+        self.all_up_in() # revive all OSDs
+
+        # Wait until all PGs are active+clean after we have revived all the OSDs
+        self.ceph_manager.wait_for_clean(timeout=self.config.get('timeout'))
 
     def inject_pause(self, conf_key, duration, check_after, should_be_down):
         """

From 9762656160c9ae12d06b29a3e8a8d0dd13847328 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Thu, 19 Oct 2023 15:57:39 +0000
Subject: [PATCH 1846/2492] qa/tasks/ceph_manager.py: Added more loggings for
 all_active_or_peered()

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 qa/tasks/ceph_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 4a8867a44e26..b112b5acc5ef 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -962,7 +962,6 @@ def test_pool_min_size(self):
                 for i in range(0, most_killable):
                     self.kill_osd(osd=acting_set[i], mark_out=True)
                 self.log("dead_osds={d}, live_osds={ld}".format(d=self.dead_osds, ld=self.live_osds))
-                self.log("check for active or peered")
                 with safe_while(
                     sleep=25, tries=5,
                     action='check for active or peered') as proceed:
@@ -2926,8 +2925,10 @@ def all_active_or_peered(self):
         """
         Wrapper to check if all PGs are active or peered
         """
+        self.log("checking for active or peered")
         pgs = self.get_pg_stats()
         if self._get_num_active(pgs) + self._get_num_peered(pgs) == len(pgs):
+            self.log("all pgs are active or peered!")
             return True
         else:
             self.dump_pgs_not_active_peered(pgs)

From 56c681c39a35686368e43e3baabcc7e7c5bd8b72 Mon Sep 17 00:00:00 2001
From: Aishwarya Mathuria <amathuri@redhat.com>
Date: Fri, 9 Feb 2024 07:01:35 +0000
Subject: [PATCH 1847/2492] osd: Remove check on osd_pg_delete_cost change

With the new cost estimation technique, we don't need osd_pg_delete_cost to be hard-coded to one particular value.
Hence we can now remove the check we had previously added to prevent users from modifying deletion cost with mClock.

Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
---
 src/osd/OSD.cc | 13 -------------
 src/osd/OSD.h  |  1 -
 2 files changed, 14 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 46e8eb5e3513..19db3f35cd6b 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4071,7 +4071,6 @@ int OSD::init()
 
   // Override a few options if mclock scheduler is enabled.
   maybe_override_sleep_options_for_qos();
-  maybe_override_cost_for_qos();
   maybe_override_options_for_qos();
   maybe_override_max_osd_capacity_for_qos();
 
@@ -9886,9 +9885,6 @@ void OSD::handle_conf_change(const ConfigProxy& conf,
       changed.count("osd_recovery_sleep_hybrid")) {
     maybe_override_sleep_options_for_qos();
   }
-  if (changed.count("osd_pg_delete_cost")) {
-    maybe_override_cost_for_qos();
-  }
   if (changed.count("osd_min_recovery_priority")) {
     service.local_reserver.set_min_priority(cct->_conf->osd_min_recovery_priority);
     service.remote_reserver.set_min_priority(cct->_conf->osd_min_recovery_priority);
@@ -10222,15 +10218,6 @@ void OSD::maybe_override_sleep_options_for_qos()
   }
 }
 
-void OSD::maybe_override_cost_for_qos()
-{
-  // If the scheduler enabled is mclock, override the default PG deletion cost
-  // so that mclock can meet the QoS goals.
-  if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
-    uint64_t pg_delete_cost = 1048576;
-    cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost));
-  }
-}
 
 /**
  * A context for receiving status from a background mon command to set
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 6aface4b3ee0..11c1d32a622d 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1984,7 +1984,6 @@ class OSD : public Dispatcher,
   void maybe_override_sleep_options_for_qos();
   bool maybe_override_options_for_qos(
     const std::set<std::string> *changed = nullptr);
-  void maybe_override_cost_for_qos();
   int run_osd_bench_test(int64_t count,
                          int64_t bsize,
                          int64_t osize,

From d828847b51edc664c381555c56b8c72356b49f49 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 7 Feb 2024 07:25:18 -0600
Subject: [PATCH 1848/2492] osd: restoring timely collection of PG stats

500 seconds is way too long, e.g. when compared to the 5s
default configuration option for the manager collection of the OSD data.

Fixes tracker issue 53342 note 5 (a specific scenario leading
to 'not all pgs scrubbed')

Fixes: https://tracker.ceph.com/issues/53342 - partial fix

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/options/global.yaml.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 072f94a9d796..feb9aeb7b5e5 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -2920,8 +2920,8 @@ options:
   level: advanced
   desc: The maximum interval seconds for update pg's reported_epoch,
         benefit for osdmap trim when osdmap not change frequently.
-  default: 500
   with_legacy: true
+  default: 5
 - name: osd_pg_stat_report_interval_max_epochs
   type: int
   level: advanced

From d706cecb4177e4f86ab3ad2bb717bedeaecc66c3 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 7 Feb 2024 07:40:32 -0600
Subject: [PATCH 1849/2492] osd: no 'legacy' form for two configuration options

also - fixing review comments not addressed in the original PR.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/options/global.yaml.in | 12 +++++-------
 src/osd/PeeringState.cc           | 10 ++++++----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index feb9aeb7b5e5..ead0d0caedd3 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -2918,18 +2918,16 @@ options:
 - name: osd_pg_stat_report_interval_max_seconds
   type: int
   level: advanced
-  desc: The maximum interval seconds for update pg's reported_epoch,
-        benefit for osdmap trim when osdmap not change frequently.
-  with_legacy: true
+  desc: How often (in seconds) should PGs stats be collected.
+  with_legacy: false
   default: 5
 - name: osd_pg_stat_report_interval_max_epochs
   type: int
   level: advanced
-  desc: The maximum interval by which pg's reported_epoch lags behind,
-        otherwise, pg's reported_epoch must be updated,
-        benefit for osdmap trim when osdmap changes frequently.
+  desc: The maximum number of epochs allowed to pass before PG stats
+        are collected.
   default: 500
-  with_legacy: true
+  with_legacy: false
 # Max number of snap intervals to report to mgr in pg_stat_t
 - name: osd_max_snap_prune_intervals_per_epoch
   type: uint
diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc
index 4da75b2385da..38ff7722b8b2 100644
--- a/src/osd/PeeringState.cc
+++ b/src/osd/PeeringState.cc
@@ -3909,16 +3909,18 @@ std::optional<pg_stat_t> PeeringState::prepare_stats_for_publish(
   // when there is no change in osdmap,
   // update info.stats.reported_epoch by the number of time seconds.
   utime_t cutoff_time = now;
-  cutoff_time -= cct->_conf->osd_pg_stat_report_interval_max_seconds;
-  bool is_time_expired = cutoff_time > info.stats.last_fresh ? true : false;
+  cutoff_time -=
+      cct->_conf.get_val<int64_t>("osd_pg_stat_report_interval_max_seconds");
+  const bool is_time_expired = cutoff_time > info.stats.last_fresh;
 
   // 500 epoch osdmaps are also the minimum number of osdmaps that mon must retain.
   // if info.stats.reported_epoch less than current osdmap epoch exceeds 500 osdmaps,
   // it can be considered that the one reported by pgid is too old and needs to be updated.
   // to facilitate mon trim osdmaps
   epoch_t cutoff_epoch = info.stats.reported_epoch;
-  cutoff_epoch += cct->_conf->osd_pg_stat_report_interval_max_epochs;
-  bool is_epoch_behind = cutoff_epoch < get_osdmap_epoch() ? true : false;
+  cutoff_epoch +=
+      cct->_conf.get_val<int64_t>("osd_pg_stat_report_interval_max_epochs");
+  const bool is_epoch_behind = cutoff_epoch < get_osdmap_epoch();
 
   if (pg_stats_publish && pre_publish == *pg_stats_publish &&
       (!is_epoch_behind && !is_time_expired)) {

From 463f463d50943c6fe17c39e41f64b1df27944ff6 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 20 Dec 2023 11:25:03 -0500
Subject: [PATCH 1850/2492] rgw/user: add 'active' flag to RGWAccessKey

inactive keys are removed from the key pool so can't be used to
authenticate. the 'key create' admin op now takes an 'active' option

Fixes: https://tracker.ceph.com/issues/59186

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/adminops.rst              |  7 ++
 src/rgw/driver/rados/rgw_rest_user.cc | 10 ++-
 src/rgw/driver/rados/rgw_user.cc      | 26 +++-----
 src/rgw/driver/rados/rgw_user.h       |  5 ++
 src/rgw/rgw_acl_types.h               |  9 ++-
 src/rgw/rgw_admin.cc                  |  9 +++
 src/rgw/rgw_common.cc                 |  4 ++
 src/rgw/services/svc_user_rados.cc    | 95 ++++++++++++++++-----------
 src/test/cli/radosgw-admin/help.t     |  1 +
 9 files changed, 108 insertions(+), 58 deletions(-)

diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst
index f1faac6b1c2f..a7bdec431f51 100644
--- a/doc/radosgw/adminops.rst
+++ b/doc/radosgw/adminops.rst
@@ -1163,6 +1163,13 @@ Request Parameters
 :Example: True [True]
 :Required: No
 
+``active``
+
+:Description: Activate or deactivate a key.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
 
 Response Entities
 ~~~~~~~~~~~~~~~~~
diff --git a/src/rgw/driver/rados/rgw_rest_user.cc b/src/rgw/driver/rados/rgw_rest_user.cc
index 200f1c03d2e1..bd87e4e27e02 100644
--- a/src/rgw/driver/rados/rgw_rest_user.cc
+++ b/src/rgw/driver/rados/rgw_rest_user.cc
@@ -651,7 +651,9 @@ void RGWOp_Key_Create::execute(optional_yield y)
   std::string secret_key;
   std::string key_type_str;
 
-  bool gen_key;
+  bool gen_key = true;
+  bool active = true;
+  bool active_specified = false;
 
   RGWUserAdminOpState op_state(driver);
 
@@ -662,12 +664,16 @@ void RGWOp_Key_Create::execute(optional_yield y)
   RESTArgs::get_string(s, "access-key", access_key, &access_key);
   RESTArgs::get_string(s, "secret-key", secret_key, &secret_key);
   RESTArgs::get_string(s, "key-type", key_type_str, &key_type_str);
-  RESTArgs::get_bool(s, "generate-key", true, &gen_key);
+  RESTArgs::get_bool(s, "generate-key", gen_key, &gen_key);
+  RESTArgs::get_bool(s, "active", active, &active, &active_specified);
 
   op_state.set_user_id(uid);
   op_state.set_subuser(subuser);
   op_state.set_access_key(access_key);
   op_state.set_secret_key(secret_key);
+  if (active_specified) {
+    op_state.access_key_active = active;
+  }
 
   if (gen_key)
     op_state.set_generate_key();
diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc
index b5569e481c53..c2ae4d318088 100644
--- a/src/rgw/driver/rados/rgw_user.cc
+++ b/src/rgw/driver/rados/rgw_user.cc
@@ -102,6 +102,7 @@ static void dump_access_keys_info(Formatter *f, RGWUserInfo &info)
     f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser);
     f->dump_string("access_key", k.id);
     f->dump_string("secret_key", k.key);
+    f->dump_bool("active", k.active);
     f->close_section();
   }
   f->close_section();
@@ -120,6 +121,7 @@ static void dump_swift_keys_info(Formatter *f, RGWUserInfo &info)
     info.user_id.to_str(s);
     f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser);
     f->dump_string("secret_key", k.key);
+    f->dump_bool("active", k.active);
     f->close_section();
   }
   f->close_section();
@@ -600,11 +602,6 @@ int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err
   std::string key = op_state.get_secret_key();
   int key_type = op_state.get_key_type();
 
-  RGWAccessKey modify_key;
-
-  pair<string, RGWAccessKey> key_pair;
-  map<std::string, RGWAccessKey>::iterator kiter;
-
   switch (key_type) {
   case KEY_TYPE_S3:
     id = op_state.get_access_key();
@@ -630,8 +627,8 @@ int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err
     return -ERR_INVALID_ACCESS_KEY;
   }
 
-  key_pair.first = id;
-
+  RGWAccessKey modify_key;
+  map<std::string, RGWAccessKey>::iterator kiter;
   if (key_type == KEY_TYPE_SWIFT) {
     modify_key.id = id;
     modify_key.subuser = op_state.get_subuser();
@@ -649,16 +646,13 @@ int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err
     key = secret_key_buf;
   }
 
-  if (key.empty()) {
-      set_err_msg(err_msg, "empty secret key");
-      return -ERR_INVALID_SECRET_KEY;
+  if (!key.empty()) {
+    // update the access key with the new secret key
+    modify_key.key = key;
+  }
+  if (op_state.access_key_active) {
+    modify_key.active = *op_state.access_key_active;
   }
-
-  // update the access key with the new secret key
-  modify_key.key = key;
-
-  key_pair.second = modify_key;
-
 
   if (key_type == KEY_TYPE_S3) {
     (*access_keys)[id] = modify_key;
diff --git a/src/rgw/driver/rados/rgw_user.h b/src/rgw/driver/rados/rgw_user.h
index a0cd7ed84fee..50ad28af24cf 100644
--- a/src/rgw/driver/rados/rgw_user.h
+++ b/src/rgw/driver/rados/rgw_user.h
@@ -135,6 +135,7 @@ struct RGWUserAdminOpState {
   std::map<std::string, RGWAccessKey> op_access_keys;
   int32_t key_type{-1};
   bool access_key_exist = false;
+  std::optional<bool> access_key_active;
 
   std::set<std::string> mfa_ids;
 
@@ -275,6 +276,10 @@ struct RGWUserAdminOpState {
     access_key_exist = true;
   }
 
+  void set_access_key_active(bool active) {
+    access_key_active = active;
+  }
+
   void set_suspension(__u8 is_suspended) {
     suspended = is_suspended;
     suspension_op = true;
diff --git a/src/rgw/rgw_acl_types.h b/src/rgw/rgw_acl_types.h
index b9866e9b289a..3f9f1715aba4 100644
--- a/src/rgw/rgw_acl_types.h
+++ b/src/rgw/rgw_acl_types.h
@@ -45,24 +45,29 @@ struct RGWAccessKey {
   std::string id; // AccessKey
   std::string key; // SecretKey
   std::string subuser;
+  bool active = true;
 
   RGWAccessKey() {}
   RGWAccessKey(std::string _id, std::string _key)
     : id(std::move(_id)), key(std::move(_key)) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(2, 2, bl);
+    ENCODE_START(3, 2, bl);
     encode(id, bl);
     encode(key, bl);
     encode(subuser, bl);
+    encode(active, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-     DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
+     DECODE_START_LEGACY_COMPAT_LEN_32(3, 2, 2, bl);
      decode(id, bl);
      decode(key, bl);
      decode(subuser, bl);
+     if (struct_v >= 3) {
+       decode(active, bl);
+     }
      DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8265852973f9..2977933b2e1c 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -336,6 +336,7 @@ void usage()
   cout << "   --gen-access-key                  generate random access key (for S3)\n";
   cout << "   --gen-secret                      generate random secret key\n";
   cout << "   --key-type=<type>                 key type, options are: swift, s3\n";
+  cout << "   --key-active=<bool>               activate or deactivate a key\n";
   cout << "   --temp-url-key[-2]=<key>          temp url key\n";
   cout << "   --access=<access>                 Set access permissions for sub-user, should be one\n";
   cout << "                                     of read, write, readwrite, full\n";
@@ -3357,6 +3358,8 @@ int main(int argc, const char **argv)
   int commit = false;
   int staging = false;
   int key_type = KEY_TYPE_UNDEFINED;
+  int key_active = true;
+  bool key_active_specified = false;
   std::unique_ptr<rgw::sal::Bucket> bucket;
   uint32_t perm_mask = 0;
   RGWUserInfo info;
@@ -3610,6 +3613,8 @@ int main(int argc, const char **argv)
         cerr << "bad key type: " << key_type_str << std::endl;
         exit(1);
       }
+    } else if (ceph_argparse_binary_flag(args, i, &key_active, NULL, "--key-active", (char*)NULL)) {
+      key_active_specified = true;
     } else if (ceph_argparse_witharg(args, i, &val, "--job-id", (char*)NULL)) {
       job_id = val;
     } else if (ceph_argparse_binary_flag(args, i, &gen_access_key, NULL, "--gen-access-key", (char*)NULL)) {
@@ -6443,6 +6448,10 @@ int main(int argc, const char **argv)
   if (key_type != KEY_TYPE_UNDEFINED)
     user_op.set_key_type(key_type);
 
+  if (key_active_specified) {
+    user_op.access_key_active = key_active;
+  }
+
   // set suspension operation parameters
   if (opt_cmd == OPT::USER_ENABLE)
     user_op.set_suspension(false);
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index f75ed2aca016..b1fa2fe64985 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -2912,6 +2912,7 @@ void RGWAccessKey::dump(Formatter *f) const
   encode_json("access_key", id, f);
   encode_json("secret_key", key, f);
   encode_json("subuser", subuser, f);
+  encode_json("active", active, f);
 }
 
 void RGWAccessKey::dump_plain(Formatter *f) const
@@ -2932,6 +2933,7 @@ void RGWAccessKey::dump(Formatter *f, const string& user, bool swift) const
     encode_json("access_key", id, f);
   }
   encode_json("secret_key", key, f);
+  encode_json("active", active, f);
 }
 
 void RGWAccessKey::decode_json(JSONObj *obj) {
@@ -2945,6 +2947,7 @@ void RGWAccessKey::decode_json(JSONObj *obj) {
       subuser = user.substr(pos + 1);
     }
   }
+  JSONDecoder::decode_json("active", active, obj);
 }
 
 void RGWAccessKey::decode_json(JSONObj *obj, bool swift) {
@@ -2961,6 +2964,7 @@ void RGWAccessKey::decode_json(JSONObj *obj, bool swift) {
     }
   }
   JSONDecoder::decode_json("secret_key", key, obj, true);
+  JSONDecoder::decode_json("active", active, obj);
 }
 
 void RGWStorageStats::dump(Formatter *f) const
diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc
index dd48bc4c01ef..0d01c96d481f 100644
--- a/src/rgw/services/svc_user_rados.cc
+++ b/src/rgw/services/svc_user_rados.cc
@@ -151,6 +151,22 @@ int RGWSI_User_RADOS::read_user_info(RGWSI_MetaBackend::Context *ctx,
   return 0;
 }
 
+static bool s3_key_active(const RGWUserInfo* info, const std::string& id) {
+  if (!info) {
+    return false;
+  }
+  auto i = info->access_keys.find(id);
+  return i != info->access_keys.end() && i->second.active;
+}
+
+static bool swift_key_active(const RGWUserInfo* info, const std::string& id) {
+  if (!info) {
+    return false;
+  }
+  auto i = info->swift_keys.find(id);
+  return i != info->swift_keys.end() && i->second.active;
+}
+
 class PutOperation
 {
   RGWSI_User_RADOS::Svc& svc;
@@ -203,28 +219,28 @@ class PutOperation
       }
     }
 
-    for (auto iter = info.swift_keys.begin(); iter != info.swift_keys.end(); ++iter) {
-      if (old_info && old_info->swift_keys.count(iter->first) != 0)
+    for (const auto& [id, key] : info.swift_keys) {
+      if (!key.active || swift_key_active(old_info, id))
         continue;
-      auto& k = iter->second;
       /* check if swift mapping exists */
       RGWUserInfo inf;
-      int r = svc.user->get_user_info_by_swift(ctx, k.id, &inf, nullptr, nullptr, y, dpp);
+      int r = svc.user->get_user_info_by_swift(ctx, id, &inf, nullptr, nullptr, y, dpp);
       if (r >= 0 && inf.user_id != info.user_id &&
           (!old_info || inf.user_id != old_info->user_id)) {
-        ldpp_dout(dpp, 0) << "WARNING: can't store user info, swift id (" << k.id
+        ldpp_dout(dpp, 0) << "WARNING: can't store user info, swift id (" << id
           << ") already mapped to another user (" << info.user_id << ")" << dendl;
         return -EEXIST;
       }
     }
 
     /* check if access keys already exist */
-    for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) {
-      if (old_info && old_info->access_keys.count(iter->first) != 0)
+    for (const auto& [id, key] : info.access_keys) {
+      if (!key.active) // new key not active
+        continue;
+      if (s3_key_active(old_info, id)) // old key already active
         continue;
-      auto& k = iter->second;
       RGWUserInfo inf;
-      int r = svc.user->get_user_info_by_access_key(ctx, k.id, &inf, nullptr, nullptr, y, dpp);
+      int r = svc.user->get_user_info_by_access_key(ctx, id, &inf, nullptr, nullptr, y, dpp);
       if (r >= 0 && inf.user_id != info.user_id &&
           (!old_info || inf.user_id != old_info->user_id)) {
         ldpp_dout(dpp, 0) << "WARNING: can't store user info, access key already mapped to another user" << dendl;
@@ -266,23 +282,25 @@ class PutOperation
     }
 
     const bool renamed = old_info && old_info->user_id != info.user_id;
-    for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) {
-      auto& k = iter->second;
-      if (old_info && old_info->access_keys.count(iter->first) != 0 && !renamed)
+    for (const auto& [id, key] : info.access_keys) {
+      if (!key.active)
+        continue;
+      if (s3_key_active(old_info, id) && !renamed)
         continue;
 
-      ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_keys_pool, k.id,
+      ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_keys_pool, id,
                                link_bl, exclusive, NULL, real_time(), y);
       if (ret < 0)
         return ret;
     }
 
-    for (auto siter = info.swift_keys.begin(); siter != info.swift_keys.end(); ++siter) {
-      auto& k = siter->second;
-      if (old_info && old_info->swift_keys.count(siter->first) != 0 && !renamed)
+    for (const auto& [id, key] : info.swift_keys) {
+      if (!key.active)
+        continue;
+      if (swift_key_active(old_info, id) && !renamed)
         continue;
 
-      ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_swift_pool, k.id,
+      ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_swift_pool, id,
                                link_bl, exclusive, NULL, real_time(), y);
       if (ret < 0)
         return ret;
@@ -323,23 +341,21 @@ class PutOperation
       }
     }
 
-    for ([[maybe_unused]] const auto& [name, access_key] : old_info.access_keys) {
-      if (!new_info.access_keys.count(access_key.id)) {
-        ret = svc.user->remove_key_index(dpp, access_key, y);
+    for (const auto& [id, key] : old_info.access_keys) {
+      if (key.active && !s3_key_active(&new_info, id)) {
+        ret = svc.user->remove_key_index(dpp, key, y);
         if (ret < 0 && ret != -ENOENT) {
-          set_err_msg("ERROR: could not remove index for key " + access_key.id);
+          set_err_msg("ERROR: could not remove index for key " + id);
           return ret;
         }
       }
     }
 
-    for (auto old_iter = old_info.swift_keys.begin(); old_iter != old_info.swift_keys.end(); ++old_iter) {
-      const auto& swift_key = old_iter->second;
-      auto new_iter = new_info.swift_keys.find(swift_key.id);
-      if (new_iter == new_info.swift_keys.end()) {
-        ret = svc.user->remove_swift_name_index(dpp, swift_key.id, y);
+    for (const auto& [id, key] : old_info.swift_keys) {
+      if (key.active && !swift_key_active(&new_info, id)) {
+        ret = svc.user->remove_swift_name_index(dpp, id, y);
         if (ret < 0 && ret != -ENOENT) {
-          set_err_msg("ERROR: could not remove index for swift_name " + swift_key.id);
+          set_err_msg("ERROR: could not remove index for swift_name " + id);
           return ret;
         }
       }
@@ -432,24 +448,27 @@ int RGWSI_User_RADOS::remove_user_info(RGWSI_MetaBackend::Context *ctx,
 {
   int ret;
 
-  auto kiter = info.access_keys.begin();
-  for (; kiter != info.access_keys.end(); ++kiter) {
-    ldpp_dout(dpp, 10) << "removing key index: " << kiter->first << dendl;
-    ret = remove_key_index(dpp, kiter->second, y);
+  for (const auto& [id, key] : info.access_keys) {
+    if (!key.active) {
+      continue;
+    }
+    ldpp_dout(dpp, 10) << "removing key index: " << id << dendl;
+    ret = remove_key_index(dpp, key, y);
     if (ret < 0 && ret != -ENOENT) {
-      ldpp_dout(dpp, 0) << "ERROR: could not remove " << kiter->first << " (access key object), should be fixed (err=" << ret << ")" << dendl;
+      ldpp_dout(dpp, 0) << "ERROR: could not remove " << id << " (access key object), should be fixed (err=" << ret << ")" << dendl;
       return ret;
     }
   }
 
-  auto siter = info.swift_keys.begin();
-  for (; siter != info.swift_keys.end(); ++siter) {
-    auto& k = siter->second;
-    ldpp_dout(dpp, 10) << "removing swift subuser index: " << k.id << dendl;
+  for (const auto& [id, key] : info.swift_keys) {
+    if (!key.active) {
+      continue;
+    }
+    ldpp_dout(dpp, 10) << "removing swift subuser index: " << id << dendl;
     /* check if swift mapping exists */
-    ret = remove_swift_name_index(dpp, k.id, y);
+    ret = remove_swift_name_index(dpp, id, y);
     if (ret < 0 && ret != -ENOENT) {
-      ldpp_dout(dpp, 0) << "ERROR: could not remove " << k.id << " (swift name object), should be fixed (err=" << ret << ")" << dendl;
+      ldpp_dout(dpp, 0) << "ERROR: could not remove " << id << " (swift name object), should be fixed (err=" << ret << ")" << dendl;
       return ret;
     }
   }
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 8f6fc36190cc..cedc2e4bbf0d 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -204,6 +204,7 @@
      --gen-access-key                  generate random access key (for S3)
      --gen-secret                      generate random secret key
      --key-type=<type>                 key type, options are: swift, s3
+     --key-active=<bool>               activate or deactivate a key
      --temp-url-key[-2]=<key>          temp url key
      --access=<access>                 Set access permissions for sub-user, should be one
                                        of read, write, readwrite, full

From a572a0c167cddc50902665f38e4cb937b695668d Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Tue, 6 Feb 2024 18:38:31 +0530
Subject: [PATCH 1851/2492] mgr/dashboard: Add RGW per user/bucket panels in 
 grafana

Fixes: https://tracker.ceph.com/issues/64359

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 monitoring/ceph-mixin/dashboards.libsonnet    |    1 +
 .../dashboards/rgw-s3-analytics.libsonnet     | 2446 +++++++++
 .../ceph-mixin/dashboards/utils.libsonnet     |   10 +
 .../dashboards_out/rgw-s3-analytics.json      | 4695 +++++++++++++++++
 .../ceph-mixin/tests_dashboards/util.py       |    2 +-
 5 files changed, 7153 insertions(+), 1 deletion(-)
 create mode 100644 monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet
 create mode 100644 monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json

diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet
index 4c67f21c039e..8cb0bcceaf79 100644
--- a/monitoring/ceph-mixin/dashboards.libsonnet
+++ b/monitoring/ceph-mixin/dashboards.libsonnet
@@ -7,5 +7,6 @@
     (import 'dashboards/rbd.libsonnet') +
     (import 'dashboards/rgw.libsonnet') +
     (import 'dashboards/ceph-cluster.libsonnet') +
+    (import 'dashboards/rgw-s3-analytics.libsonnet') +
     { _config:: $._config },
 }
diff --git a/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet b/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet
new file mode 100644
index 000000000000..c470dbe5f195
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet
@@ -0,0 +1,2446 @@
+local g = import 'grafonnet/grafana.libsonnet';
+
+(import 'utils.libsonnet') {
+  'rgw-s3-analytics.json':
+    $.dashboardSchema(
+      'RGW S3 Analytics',
+      '',
+      'BnxelG7Sz',
+      'now-1h',
+      '30s',
+      22,
+      $._config.dashboardTags,
+      ''
+    )
+    .addAnnotation(
+      $.addAnnotationSchema(
+        1,
+        '-- Grafana --',
+        true,
+        true,
+        'rgba(0, 211, 255, 1)',
+        'Annotations & Alerts',
+        'dashboard'
+      )
+    )
+
+    .addTemplate(
+      g.template.datasource('datasource', 'prometheus', 'default', label='Data Source')
+    )
+
+    .addTemplate(
+      $.addTemplateSchema('rgw_servers',
+                          '$datasource',
+                          'label_values(ceph_rgw_metadata{}, ceph_daemon)',
+                          2,
+                          true,
+                          0,
+                          null,
+                          '')
+    )
+
+    .addTemplate(
+      g.template.adhoc('Filters', '$datasource', 'filters', 0)
+    )
+
+
+    .addPanels([
+      $.addRowSchema(false, true, 'Overview') + { gridPos: { x: 0, y: 0, w: 24, h: 1 } },
+      $.addStatPanel(
+        title='Total PUTs',
+        datasource='${datasource}',
+        gridPosition={ x: 0, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Total GETs',
+        datasource='${datasource}',
+        gridPosition={ x: 6, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum\n(ceph_rgw_op_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Total Objects',
+        datasource='${datasource}',
+        gridPosition={ x: 12, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Average Object Size',
+        datasource='${datasource}',
+        gridPosition={ x: 18, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum\n((sum by(instance_id)(ceph_rgw_op_put_obj_bytes) > 0) / (sum by(instance_id)(ceph_rgw_op_put_obj_ops) > 0) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addBarGaugePanel(
+        title='Total Operations',
+        datasource='${datasource}',
+        gridPosition={ x: 0, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_buckets_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Buckets',
+          range=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_bucket_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Buckets',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Objects',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } } } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+      $.addBarGaugePanel(
+        title='Total Size',
+        datasource='${datasource}',
+        gridPosition={ x: 8, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Objects',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'unit', value: 'decbytes' }] }] } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+      $.addBarGaugePanel(
+        title='Total Latencies',
+        datasource='${datasource}',
+        gridPosition={ x: 16, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_buckets_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Bucket',
+          range=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_bucket_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Bucket',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Object',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'unit', value: 'ms' }] }] } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Summary Per Bucket by Bandwidth',
+        gridPosition={ h: 8, w: 12, x: 0, y: 12 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'color-text' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'decbytes' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              Bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': false,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #D': 4,
+              'Value #F': 5,
+              bucket: 1,
+              ceph_daemon: 0,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              bucket: 'Bucket',
+              ceph_daemon: 'Daemon',
+              'ceph_daemon 1': 'Daemon',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Latency(ms) Per Bucket',
+        gridPosition={ h: 8, w: 12, x: 12, y: 12 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'ms' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'joinByField',
+          options: {
+            byField: 'Bucket',
+            mode: 'outer',
+          },
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              Bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #C': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'ceph_daemon 5': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #C': 4,
+              'Value #D': 5,
+              'Value #F': 6,
+              bucket: 1,
+              ceph_daemon: 0,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              bucket: 'Bucket',
+              ceph_daemon: 'Daemon',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='List Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Summary Per User By Bandwidth',
+        gridPosition={ h: 8, w: 12, x: 0, y: 20 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'decbytes' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              User: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              instance: {
+                aggregations: [],
+              },
+              user: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #D': 4,
+              'Value #F': 5,
+              ceph_daemon: 0,
+              user: 1,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              ceph_daemon: 'Daemon',
+              user: 'User',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Latency(ms) Per User',
+        gridPosition={ h: 8, w: 12, x: 12, y: 20 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'ms' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'joinByField',
+          options: {
+            byField: 'User',
+            mode: 'outer',
+          },
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              User: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #C': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              user: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'ceph_daemon 5': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #C': 4,
+              'Value #D': 5,
+              'Value #F': 6,
+              ceph_daemon: 0,
+              user: 1,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              ceph_daemon: 'Daemon',
+              user: 'User',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+      ]),
+
+
+      $.addRowSchema(false, true, 'Buckets', collapsed=true)
+      .addPanels([
+        $.addBarGaugePanel(
+          title='Top 5 Bucket PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 0, y: 29, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Bucket GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 6, y: 29, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Buckets PUTs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 12, y: 29, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Buckets GETs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 18, y: 29, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket GETs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Copy by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Delete by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket List by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Delete by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Copy by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 53 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.addTableExtended(
+          datasource='${datasource}',
+          title='Summary Per Bucket by Operations',
+          gridPosition={ h: 8, w: 12, x: 12, y: 53 },
+          options={
+            footer: {
+              fields: '',
+              reducer: ['sum'],
+              countRows: false,
+              enablePagination: false,
+              show: false,
+            },
+            frameIndex: 1,
+            showHeader: true,
+            sortBy: [
+              {
+                desc: true,
+                displayName: 'PUTs',
+              },
+            ],
+          },
+          custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+          thresholds={
+            mode: 'absolute',
+            steps: [
+              { color: 'green' },
+            ],
+          },
+          overrides=[{
+            matcher: { id: 'byType', options: 'number' },
+            properties: [
+              { id: 'unit', value: 'none' },
+            ],
+          }],
+          pluginVersion='9.4.7'
+        )
+        .addTransformations([
+          {
+            id: 'merge',
+            options: {},
+          },
+          {
+            id: 'joinByField',
+            options: {
+              byField: 'Bucket',
+              mode: 'outer',
+            },
+          },
+          {
+            id: 'groupBy',
+            options: {
+              fields: {
+                Bucket: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #A': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #B': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #C': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #D': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #F': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                bucket: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                ceph_daemon: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+              },
+            },
+          },
+          {
+            id: 'organize',
+            options: {
+              excludeByName: {
+                'Time 1': true,
+                'Time 2': true,
+                'Time 3': true,
+                'Time 4': true,
+                'Time 5': true,
+                'Time 6': true,
+                'Time 7': true,
+                __name__: true,
+                '__name__ 1': true,
+                '__name__ 2': true,
+                '__name__ 3': true,
+                '__name__ 4': true,
+                '__name__ 5': true,
+                '__name__ 6': true,
+                '__name__ 7': true,
+                'ceph_daemon 1': true,
+                'ceph_daemon 2': true,
+                'ceph_daemon 3': true,
+                'ceph_daemon 4': true,
+                'instance 1': true,
+                'instance 2': true,
+                'instance 3': true,
+                'instance 4': true,
+                'instance 5': true,
+                'instance 6': true,
+                'instance 7': true,
+                'instance_id 1': true,
+                'instance_id 2': true,
+                'instance_id 3': true,
+                'instance_id 4': true,
+                'instance_id 5': true,
+                'instance_id 6': true,
+                'instance_id 7': true,
+                'job 1': true,
+                'job 2': true,
+                'job 3': true,
+                'job 4': true,
+                'job 5': true,
+                'job 6': true,
+                'job 7': true,
+              },
+              indexByName: {
+                'Value #A': 2,
+                'Value #B': 3,
+                'Value #C': 4,
+                'Value #D': 5,
+                'Value #F': 6,
+                bucket: 1,
+                ceph_daemon: 0,
+              },
+              renameByName: {
+                Bucket: '',
+                'Value #A': 'PUTs',
+                'Value #B': 'GETs',
+                'Value #C': 'List',
+                'Value #D': 'Delete',
+                'Value #E': 'Copy',
+                'Value #F': 'Copy',
+                'Value #G': '',
+                bucket: 'Bucket',
+                ceph_daemon: 'Daemon',
+              },
+            },
+          },
+        ]).addTargets([
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+        ]),
+      ]) + { gridPos: { x: 0, y: 28, w: 24, h: 1 } },
+
+
+      $.addRowSchema(false, true, 'Users', collapsed=true)
+      .addPanels([
+        $.addBarGaugePanel(
+          title='Top 5 Users PUTs By Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 0, y: 62, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green' }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)\n',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 6, y: 62, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)\n',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ x: 12, y: 62, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users GETs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 18, y: 62, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User GETs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Delete by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User COPY by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User List by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Delete by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Copy by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 86 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.addTableExtended(
+          datasource='${datasource}',
+          title='Summary Per User By Operations',
+          gridPosition={ h: 8, w: 12, x: 12, y: 86 },
+          options={
+            footer: {
+              fields: '',
+              reducer: ['sum'],
+              countRows: false,
+              enablePagination: false,
+              show: false,
+            },
+            frameIndex: 1,
+            showHeader: true,
+            sortBy: [
+              {
+                desc: true,
+                displayName: 'PUTs',
+              },
+            ],
+          },
+          custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+          thresholds={
+            mode: 'absolute',
+            steps: [
+              { color: 'green' },
+            ],
+          },
+          overrides=[{
+            matcher: { id: 'byType', options: 'number' },
+            properties: [
+              { id: 'unit', value: 'none' },
+            ],
+          }],
+          pluginVersion='9.4.7'
+        )
+        .addTransformations([
+          {
+            id: 'merge',
+            options: {},
+          },
+          {
+            id: 'joinByField',
+            options: {
+              byField: 'User',
+              mode: 'outer',
+            },
+          },
+          {
+            id: 'groupBy',
+            options: {
+              fields: {
+                User: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #A': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #B': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #C': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #D': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #F': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                ceph_daemon: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                user: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+              },
+            },
+          },
+          {
+            id: 'organize',
+            options: {
+              excludeByName: {},
+              indexByName: {
+                'Value #A': 2,
+                'Value #B': 3,
+                'Value #C': 4,
+                'Value #D': 5,
+                'Value #F': 6,
+                ceph_daemon: 0,
+                user: 1,
+              },
+              renameByName: {
+                'Value #A': 'PUTs',
+                'Value #B': 'GETs',
+                'Value #C': 'LIST',
+                'Value #D': 'DELETE',
+                'Value #F': 'COPY',
+                ceph_daemon: 'Daemon',
+                user: 'User',
+              },
+            },
+          },
+        ]).addTargets([
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+        ]),
+      ]) + { gridPos: { x: 0, y: 29, w: 24, h: 1 } },
+    ]),
+}
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index 16ba771cb3a0..397c00fe53ad 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -524,6 +524,16 @@ local timeSeries = import 'timeseries_panel.libsonnet';
       maxDataPoints: maxDataPoints,
       interval: interval,
     },
+
+  addBarGaugePanel(title='',
+                   description='',
+                   datasource='${DS_PROMETHEUS}',
+                   gridPosition={},
+                   unit='percentunit',
+                   thresholds={})::
+    g.barGaugePanel.new(title, description, datasource, unit, thresholds) + {
+      gridPos: gridPosition,
+    },
   addTableExtended(
     title='',
     datasource=null,
diff --git a/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json b/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json
new file mode 100644
index 000000000000..6a41df1cfaa9
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json
@@ -0,0 +1,4695 @@
+{
+   "__inputs": [ ],
+   "__requires": [ ],
+   "annotations": {
+      "list": [
+         {
+            "builtIn": 1,
+            "datasource": "-- Grafana --",
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "showIn": 0,
+            "tags": [ ],
+            "type": "dashboard"
+         }
+      ]
+   },
+   "description": "",
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 2,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Overview",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 0,
+            "y": 1
+         },
+         "id": 3,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total PUTs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 6,
+            "y": 1
+         },
+         "id": 4,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum\n(ceph_rgw_op_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total GETs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 12,
+            "y": 1
+         },
+         "id": 5,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total Objects",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 18,
+            "y": 1
+         },
+         "id": 6,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum\n((sum by(instance_id)(ceph_rgw_op_put_obj_bytes) > 0) / (sum by(instance_id)(ceph_rgw_op_put_obj_ops) > 0) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Average Object Size",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            }
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 0,
+            "y": 4
+         },
+         "id": 7,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "List Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_buckets_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "List Buckets",
+               "range": true,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "E"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_bucket_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Buckets",
+               "range": false,
+               "refId": "F"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": true,
+               "refId": "G"
+            }
+         ],
+         "title": "Total Operations",
+         "type": "bargauge"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "unit": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 8,
+            "y": 4
+         },
+         "id": 8,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": true,
+               "refId": "D"
+            }
+         ],
+         "title": "Total Size",
+         "type": "bargauge"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "unit": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 4
+         },
+         "id": 9,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "List Object",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_buckets_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "List Bucket",
+               "range": true,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Object",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Object",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Object",
+               "range": false,
+               "refId": "E"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_bucket_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Bucket",
+               "range": false,
+               "refId": "F"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Object",
+               "range": true,
+               "refId": "G"
+            }
+         ],
+         "title": "Total Latencies",
+         "type": "bargauge"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "color-text"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 12
+         },
+         "id": 10,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "D"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Summary Per Bucket by Bandwidth",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "Bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": false,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #D": 4,
+                     "Value #F": 5,
+                     "bucket": 1,
+                     "ceph_daemon": 0
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "bucket": "Bucket",
+                     "ceph_daemon": "Daemon",
+                     "ceph_daemon 1": "Daemon"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 12
+         },
+         "id": 11,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "List Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "E"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency(ms) Per Bucket",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "Bucket",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "Bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #C": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "ceph_daemon 5": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #C": 4,
+                     "Value #D": 5,
+                     "Value #F": 6,
+                     "bucket": 1,
+                     "ceph_daemon": 0
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "bucket": "Bucket",
+                     "ceph_daemon": "Daemon"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 20
+         },
+         "id": 12,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "D"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Summary Per User By Bandwidth",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "User": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "instance": {
+                        "aggregations": [ ]
+                     },
+                     "user": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #D": 4,
+                     "Value #F": 5,
+                     "ceph_daemon": 0,
+                     "user": 1
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "ceph_daemon": "Daemon",
+                     "user": "User"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 20
+         },
+         "id": 13,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "E"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency(ms) Per User",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "User",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "User": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #C": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "user": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "ceph_daemon 5": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #C": 4,
+                     "Value #D": 5,
+                     "Value #F": 6,
+                     "ceph_daemon": 0,
+                     "user": 1
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "ceph_daemon": "Daemon",
+                     "user": "User"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 28
+         },
+         "id": 14,
+         "panels": [
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 29
+               },
+               "id": 15,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Bucket PUTs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 29
+               },
+               "id": 16,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Bucket GETs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 29
+               },
+               "id": 17,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Buckets PUTs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 29
+               },
+               "id": 18,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Buckets GETs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 37
+               },
+               "id": 19,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket PUTs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 37
+               },
+               "id": 20,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket GETs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 37
+               },
+               "id": 21,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Copy by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 37
+               },
+               "id": 22,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Delete by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 45
+               },
+               "id": 23,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket GETs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 45
+               },
+               "id": 24,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket PUTs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 45
+               },
+               "id": 25,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket List by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 45
+               },
+               "id": 26,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Delete by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 53
+               },
+               "id": 27,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Copy by Operations",
+               "type": "timeseries"
+            },
+            {
+               "columns": [ ],
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "align": "auto",
+                        "cellOptions": {
+                           "type": "auto"
+                        },
+                        "filterable": false,
+                        "inspect": false
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "options": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "unit",
+                              "value": "none"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 53
+               },
+               "id": 28,
+               "links": [ ],
+               "options": {
+                  "footer": {
+                     "countRows": false,
+                     "enablePagination": false,
+                     "fields": "",
+                     "reducer": [
+                        "sum"
+                     ],
+                     "show": false
+                  },
+                  "frameIndex": 1,
+                  "showHeader": true,
+                  "sortBy": [
+                     {
+                        "desc": true,
+                        "displayName": "PUTs"
+                     }
+                  ]
+               },
+               "pluginVersion": "9.4.7",
+               "styles": "",
+               "targets": [
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "B"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "C"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "D"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "E"
+                  }
+               ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Summary Per Bucket by Operations",
+               "transformations": [
+                  {
+                     "id": "merge",
+                     "options": { }
+                  },
+                  {
+                     "id": "joinByField",
+                     "options": {
+                        "byField": "Bucket",
+                        "mode": "outer"
+                     }
+                  },
+                  {
+                     "id": "groupBy",
+                     "options": {
+                        "fields": {
+                           "Bucket": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #A": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #B": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #C": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #D": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #F": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "bucket": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "ceph_daemon": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           }
+                        }
+                     }
+                  },
+                  {
+                     "id": "organize",
+                     "options": {
+                        "excludeByName": {
+                           "Time 1": true,
+                           "Time 2": true,
+                           "Time 3": true,
+                           "Time 4": true,
+                           "Time 5": true,
+                           "Time 6": true,
+                           "Time 7": true,
+                           "__name__": true,
+                           "__name__ 1": true,
+                           "__name__ 2": true,
+                           "__name__ 3": true,
+                           "__name__ 4": true,
+                           "__name__ 5": true,
+                           "__name__ 6": true,
+                           "__name__ 7": true,
+                           "ceph_daemon 1": true,
+                           "ceph_daemon 2": true,
+                           "ceph_daemon 3": true,
+                           "ceph_daemon 4": true,
+                           "instance 1": true,
+                           "instance 2": true,
+                           "instance 3": true,
+                           "instance 4": true,
+                           "instance 5": true,
+                           "instance 6": true,
+                           "instance 7": true,
+                           "instance_id 1": true,
+                           "instance_id 2": true,
+                           "instance_id 3": true,
+                           "instance_id 4": true,
+                           "instance_id 5": true,
+                           "instance_id 6": true,
+                           "instance_id 7": true,
+                           "job 1": true,
+                           "job 2": true,
+                           "job 3": true,
+                           "job 4": true,
+                           "job 5": true,
+                           "job 6": true,
+                           "job 7": true
+                        },
+                        "indexByName": {
+                           "Value #A": 2,
+                           "Value #B": 3,
+                           "Value #C": 4,
+                           "Value #D": 5,
+                           "Value #F": 6,
+                           "bucket": 1,
+                           "ceph_daemon": 0
+                        },
+                        "renameByName": {
+                           "Bucket": "",
+                           "Value #A": "PUTs",
+                           "Value #B": "GETs",
+                           "Value #C": "List",
+                           "Value #D": "Delete",
+                           "Value #E": "Copy",
+                           "Value #F": "Copy",
+                           "Value #G": "",
+                           "bucket": "Bucket",
+                           "ceph_daemon": "Daemon"
+                        }
+                     }
+                  }
+               ],
+               "type": "table"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Buckets",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 29
+         },
+         "id": 29,
+         "panels": [
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 62
+               },
+               "id": 30,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)\n",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users PUTs By Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 62
+               },
+               "id": 31,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)\n",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users GETs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 62
+               },
+               "id": 32,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users PUTs by Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 62
+               },
+               "id": 33,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users GETs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 70
+               },
+               "id": 34,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User PUTs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 70
+               },
+               "id": 35,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User GETs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 70
+               },
+               "id": 36,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Delete by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 70
+               },
+               "id": 37,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User COPY by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 78
+               },
+               "id": 38,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User GETs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 78
+               },
+               "id": 39,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User PUTs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 78
+               },
+               "id": 40,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User List by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 78
+               },
+               "id": 41,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Delete by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 86
+               },
+               "id": 42,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Copy by Operations",
+               "type": "timeseries"
+            },
+            {
+               "columns": [ ],
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "align": "auto",
+                        "cellOptions": {
+                           "type": "auto"
+                        },
+                        "filterable": false,
+                        "inspect": false
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "options": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "unit",
+                              "value": "none"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 86
+               },
+               "id": 43,
+               "links": [ ],
+               "options": {
+                  "footer": {
+                     "countRows": false,
+                     "enablePagination": false,
+                     "fields": "",
+                     "reducer": [
+                        "sum"
+                     ],
+                     "show": false
+                  },
+                  "frameIndex": 1,
+                  "showHeader": true,
+                  "sortBy": [
+                     {
+                        "desc": true,
+                        "displayName": "PUTs"
+                     }
+                  ]
+               },
+               "pluginVersion": "9.4.7",
+               "styles": "",
+               "targets": [
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "B"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "C"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "D"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "E"
+                  }
+               ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Summary Per User By Operations",
+               "transformations": [
+                  {
+                     "id": "merge",
+                     "options": { }
+                  },
+                  {
+                     "id": "joinByField",
+                     "options": {
+                        "byField": "User",
+                        "mode": "outer"
+                     }
+                  },
+                  {
+                     "id": "groupBy",
+                     "options": {
+                        "fields": {
+                           "User": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #A": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #B": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #C": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #D": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #F": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "ceph_daemon": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "user": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           }
+                        }
+                     }
+                  },
+                  {
+                     "id": "organize",
+                     "options": {
+                        "excludeByName": { },
+                        "indexByName": {
+                           "Value #A": 2,
+                           "Value #B": 3,
+                           "Value #C": 4,
+                           "Value #D": 5,
+                           "Value #F": 6,
+                           "ceph_daemon": 0,
+                           "user": 1
+                        },
+                        "renameByName": {
+                           "Value #A": "PUTs",
+                           "Value #B": "GETs",
+                           "Value #C": "LIST",
+                           "Value #D": "DELETE",
+                           "Value #F": "COPY",
+                           "ceph_daemon": "Daemon",
+                           "user": "User"
+                        }
+                     }
+                  }
+               ],
+               "type": "table"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Users",
+         "titleSize": "h6",
+         "type": "row"
+      }
+   ],
+   "refresh": "30s",
+   "rows": [ ],
+   "schemaVersion": 22,
+   "style": "dark",
+   "tags": [
+      "ceph-mixin"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data Source",
+            "name": "datasource",
+            "options": [ ],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "allValue": null,
+            "current": { },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": false,
+            "name": "rgw_servers",
+            "options": [ ],
+            "query": "label_values(ceph_rgw_metadata{}, ceph_daemon)",
+            "refresh": 2,
+            "regex": "",
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         },
+         {
+            "datasource": "$datasource",
+            "hide": 2,
+            "label": "filters",
+            "name": "Filters",
+            "type": "adhoc"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "",
+   "title": "RGW S3 Analytics",
+   "uid": "BnxelG7Sz",
+   "version": 0
+}
diff --git a/monitoring/ceph-mixin/tests_dashboards/util.py b/monitoring/ceph-mixin/tests_dashboards/util.py
index 4213ff6d5a1e..c6369c5f8344 100644
--- a/monitoring/ceph-mixin/tests_dashboards/util.py
+++ b/monitoring/ceph-mixin/tests_dashboards/util.py
@@ -55,7 +55,7 @@ def add_dashboard_queries(data: Dict[str, Any], dashboard_data: Dict[str, Any],
                 title = panel['title']
                 legend_format = target['legendFormat'] if 'legendFormat' in target else ""
                 query_id = f'{title}-{legend_format}'
-                if query_id in panel_ids_in_file:
+                if query_id in panel_ids_in_file and legend_format != '__auto':
                     cprint((f'ERROR: Query in panel "{title}" with legend "{legend_format}"'
                             f' already exists in the same file: "{path}"'), 'red')
                     error = 1

From cd068b831dc8cf665577db40563a2a6a03b6c4c0 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Thu, 8 Feb 2024 22:46:37 -0500
Subject: [PATCH 1852/2492] rgw: add new `object manifest` sub-command

Adds a new sub-command to radosgw-admin to list the manifest, which
are the rados objects that contain the RGW object's data.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 doc/man/8/radosgw-admin.rst |  3 ++
 src/rgw/rgw_admin.cc        | 76 ++++++++++++++++++++++++++++++++++++-
 src/rgw/rgw_obj_types.h     |  2 +-
 3 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst
index 778cef410293..dda2de57291b 100644
--- a/doc/man/8/radosgw-admin.rst
+++ b/doc/man/8/radosgw-admin.rst
@@ -140,6 +140,9 @@ as follows:
 :command:`object stat`
   Stat an object for its metadata.
 
+:command:`object manifest`
+  Display the manifest of RADOS objects containing the data.
+
 :command:`object unlink`
   Unlink object from bucket index.
 
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8265852973f9..081204c24f44 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -682,6 +682,7 @@ enum class OPT {
   OBJECT_RM,
   OBJECT_UNLINK,
   OBJECT_STAT,
+  OBJECT_MANIFEST,
   OBJECT_REWRITE,
   OBJECT_REINDEX,
   OBJECTS_EXPIRE,
@@ -903,6 +904,7 @@ static SimpleCmd::Commands all_cmds = {
   { "object rm", OPT::OBJECT_RM },
   { "object unlink", OPT::OBJECT_UNLINK },
   { "object stat", OPT::OBJECT_STAT },
+  { "object manifest", OPT::OBJECT_MANIFEST },
   { "object rewrite", OPT::OBJECT_REWRITE },
   { "object reindex", OPT::OBJECT_REINDEX },
   { "objects expire", OPT::OBJECTS_EXPIRE },
@@ -4180,6 +4182,7 @@ int main(int argc, const char **argv)
 			 OPT::LOG_SHOW,
 			 OPT::USAGE_SHOW,
 			 OPT::OBJECT_STAT,
+			 OPT::OBJECT_MANIFEST,
 			 OPT::BI_GET,
 			 OPT::BI_LIST,
 			 OPT::OLH_GET,
@@ -8327,7 +8330,7 @@ int main(int argc, const char **argv)
       bufferlist& bl = iter->second;
       bool handled = false;
       if (iter->first == RGW_ATTR_MANIFEST) {
-        handled = decode_dump<RGWObjManifest>("manifest", bl, formatter.get());
+	handled = decode_dump<RGWObjManifest>("manifest", bl, formatter.get());
       } else if (iter->first == RGW_ATTR_ACL) {
         handled = decode_dump<RGWAccessControlPolicy>("policy", bl, formatter.get());
       } else if (iter->first == RGW_ATTR_ID_TAG) {
@@ -8360,7 +8363,76 @@ int main(int argc, const char **argv)
     formatter->close_section();
     formatter->close_section();
     formatter->flush(cout);
-  }
+  } // OPT::OBJECT_STAT
+
+  if (opt_cmd == OPT::OBJECT_MANIFEST) {
+    int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
+    if (ret < 0) {
+      cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) <<
+	std::endl;
+      return -ret;
+    }
+
+    std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(object);
+    obj->set_instance(object_version);
+
+    ret = obj->get_obj_attrs(null_yield, dpp());
+    if (ret < 0) {
+      cerr << "ERROR: failed to retrieve object metadata, returned error: " <<
+	cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
+
+    formatter->open_object_section("outer");  // name not displayed since top level
+    formatter->dump_unsigned("size", obj->get_obj_size());
+
+    auto attr_iter = obj->get_attrs().find(RGW_ATTR_MANIFEST);
+    if (attr_iter == obj->get_attrs().end()) {
+      cerr << "ERROR: unable to find object manifest" << std::endl;
+      return ENOENT;
+    }
+
+    RGWObjManifest m;
+    try {
+      auto part_iter = attr_iter->second.cbegin();
+      decode(m, part_iter);
+    } catch (buffer::error& err) {
+      cerr << "ERROR: unable to decode manifest" << std::endl;
+      return EIO;
+    }
+
+    rgw::sal::RadosStore* store =
+      dynamic_cast<rgw::sal::RadosStore*>(driver);
+    if (!store) {
+      cerr << "ERROR: this command (currently) only works with "
+	"RADOS back-ends" << std::endl;
+      return EINVAL;
+    }
+
+    RGWRados* rados = store->getRados();
+
+    formatter->open_array_section("objects");
+    unsigned index = 0;
+    for (auto p = m.obj_begin(dpp()); p != m.obj_end(dpp()); ++p, ++index) {
+      formatter->open_object_section("object"); // name not displayed since in array
+
+      formatter->dump_unsigned("index", index);
+      formatter->dump_unsigned("part_id", p.get_cur_part_id());
+      formatter->dump_unsigned("stripe_id", p.get_cur_stripe());
+      formatter->dump_unsigned("offset", p.get_ofs());
+      formatter->dump_unsigned("size", p.get_stripe_size());
+
+      formatter->open_object_section("raw_obj");
+      p.get_location().get_raw_obj(rados).dump(formatter.get());
+      formatter->close_section(); // raw_obj
+
+      formatter->close_section(); // object
+    }
+    formatter->close_section(); // objects array
+
+    formatter->close_section(); // outer
+    formatter->flush(cout);
+  } // OPT::OBJECT_MANIFEST
 
   if (opt_cmd == OPT::BUCKET_CHECK) {
     if (check_head_obj_locator) {
diff --git a/src/rgw/rgw_obj_types.h b/src/rgw/rgw_obj_types.h
index 70f9ddf5adf3..a092e5ccdab6 100644
--- a/src/rgw/rgw_obj_types.h
+++ b/src/rgw/rgw_obj_types.h
@@ -432,7 +432,7 @@ struct rgw_raw_obj {
   }
 
   void encode(bufferlist& bl) const {
-     ENCODE_START(6, 6, bl);
+    ENCODE_START(6, 6, bl);
     encode(pool, bl);
     encode(oid, bl);
     encode(loc, bl);

From 575362d826f8f58832a2bb865b9dc73c474f4fe1 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Fri, 9 Feb 2024 12:27:00 -0500
Subject: [PATCH 1853/2492] rgw: if head object does not have data, still
 include in `object manifest`

In order to enable users who might like to use `radosgw-admin object
manifest ...` to get a complete list of raw objects that comprise the
RGW object, we'll include the head object when it includes not
data. It will be indicated with an "index" of -1.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/rgw_admin.cc | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 081204c24f44..5fecee0dc023 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -8411,9 +8411,30 @@ int main(int argc, const char **argv)
 
     RGWRados* rados = store->getRados();
 
+    rgw_obj head_obj = obj->get_obj();
+    rgw_raw_obj raw_head_obj;
+    store->get_raw_obj(m.get_head_placement_rule(), head_obj, &raw_head_obj);
+    
     formatter->open_array_section("objects");
     unsigned index = 0;
     for (auto p = m.obj_begin(dpp()); p != m.obj_end(dpp()); ++p, ++index) {
+      rgw_raw_obj raw_obj =  p.get_location().get_raw_obj(rados);
+
+      if (index == 0 && raw_obj != raw_head_obj) {
+	// we have a head object without data, so let's include it
+	formatter->open_object_section("object"); // name not displayed since in array
+
+	formatter->dump_int("index", -1);
+	formatter->dump_unsigned("offset", 0);
+	formatter->dump_unsigned("size", 0);
+	
+	formatter->open_object_section("raw_obj");
+	raw_head_obj.dump(formatter.get());
+	formatter->close_section(); // raw_obj
+
+	formatter->close_section(); // object
+      }
+
       formatter->open_object_section("object"); // name not displayed since in array
 
       formatter->dump_unsigned("index", index);
@@ -8423,7 +8444,7 @@ int main(int argc, const char **argv)
       formatter->dump_unsigned("size", p.get_stripe_size());
 
       formatter->open_object_section("raw_obj");
-      p.get_location().get_raw_obj(rados).dump(formatter.get());
+      raw_obj.dump(formatter.get());
       formatter->close_section(); // raw_obj
 
       formatter->close_section(); // object

From 3e6e36c9cc5662f4816ee47f4db4e47d168fc2b4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 9 Feb 2024 12:49:10 -0500
Subject: [PATCH 1854/2492] test/rgw: add multisite test case for DeleteObjects

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/rgw/rgw_multi/tests.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 3f5ae0fea5d3..923bea588886 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -666,6 +666,39 @@ def test_object_delete():
             zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
             check_bucket_eq(source_conn, target_conn, bucket)
 
+def test_multi_object_delete():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objnames = [f'obj{i}' for i in range(1,50)]
+    content = 'asdasd'
+
+    # don't wait for meta sync just yet
+    for zone, bucket in zone_bucket:
+        create_objects(zone, bucket, objnames, content)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # check objects exist
+    for source_conn, bucket in zone_bucket:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
+    # check object removal
+    for source_conn, bucket in zone_bucket:
+        bucket.delete_keys(objnames)
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
 def get_latest_object_version(key):
     for k in key.bucket.list_versions(key.name):
         if k.is_latest:

From 692c2e4d555dd0afe8db6a40634caf446bbc13a7 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 9 Feb 2024 13:38:49 -0500
Subject: [PATCH 1855/2492] rgw/datalog: RGWDataChangesLog::add_entry() uses
 null_yield

work around a deadlock under RGWDataChangesLog::add_entry() with
multi-object delete by making the push() call synchronous

this is a partial revert of d38a69b6e18cb798528d64a2c6c3e439212fe6d2

Fixes: https://tracker.ceph.com/issues/63373

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_datalog.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_datalog.cc b/src/rgw/driver/rados/rgw_datalog.cc
index 9274dc2ae9be..4c9503071ef1 100644
--- a/src/rgw/driver/rados/rgw_datalog.cc
+++ b/src/rgw/driver/rados/rgw_datalog.cc
@@ -737,7 +737,8 @@ int RGWDataChangesLog::add_entry(const DoutPrefixProvider *dpp,
     ldpp_dout(dpp, 20) << "RGWDataChangesLog::add_entry() sending update with now=" << now << " cur_expiration=" << expiration << dendl;
 
     auto be = bes->head();
-    ret = be->push(dpp, index, now, change.key, std::move(bl), y);
+    // TODO: pass y once we fix the deadlock from https://tracker.ceph.com/issues/63373
+    ret = be->push(dpp, index, now, change.key, std::move(bl), null_yield);
 
     now = real_clock::now();
 

From 00fc7960f682a16aa7a02aeb0300780a1bf56476 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Thu, 8 Feb 2024 16:14:03 -0600
Subject: [PATCH 1856/2492] qa/suites/orch: whitelist warnings that are
 expected in test environments

Fixes: https://tracker.ceph.com/issues/64343
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/suites/orch/cephadm/smoke/start.yaml                     | 4 ++++
 qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml  | 4 ++++
 .../cephadm/workunits/task/test_monitoring_stack_basic.yaml | 4 ++++
 .../orch/cephadm/workunits/task/test_orch_cli_mon.yaml      | 4 ++++
 .../orch/cephadm/workunits/task/test_rgw_multisite.yaml     | 4 ++++
 .../workunits/task/test_set_mon_crush_locations.yaml        | 5 +++++
 qa/tasks/thrashosds-health.yaml                             | 6 ++++++
 7 files changed, 31 insertions(+)

diff --git a/qa/suites/orch/cephadm/smoke/start.yaml b/qa/suites/orch/cephadm/smoke/start.yaml
index 77f493ca1b9f..8cec0860a2f7 100644
--- a/qa/suites/orch/cephadm/smoke/start.yaml
+++ b/qa/suites/orch/cephadm/smoke/start.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
 tasks:
 - cephadm:
     conf:
diff --git a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
index c195bc052b7e..74f72c057ded 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_host_drain.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml b/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
index 62947ef65d9c..b7f00481963d 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_monitoring_stack_basic.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
index 2a33dc8399c1..c85711420a3f 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
 roles:
 - - host.a
   - osd.0
diff --git a/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml b/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
index 976e3730ce32..dc982364b319 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
@@ -1,3 +1,7 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
 roles:
 - - host.a
   - mon.a
diff --git a/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml b/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
index 6d9bd1525d83..7379f88ee984 100644
--- a/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
+++ b/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
@@ -1,3 +1,8 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - MON_DOWN
+      - POOL_APP_NOT_ENABLED
 roles:
 - - host.a
   - osd.0
diff --git a/qa/tasks/thrashosds-health.yaml b/qa/tasks/thrashosds-health.yaml
index 1405f4740845..6654836b6359 100644
--- a/qa/tasks/thrashosds-health.yaml
+++ b/qa/tasks/thrashosds-health.yaml
@@ -19,3 +19,9 @@ overrides:
       - slow request
       - timeout on replica
       - late reservation from
+      - MON_DOWN
+      - OSDMAP_FLAGS
+      - OSD_DOWN
+      - PG_DEGRADED
+      - PG_AVAILABILITY
+      - POOL_APP_NOT_ENABLED

From 8bfe2d1ae6e24590651bfde3854b60d950677b37 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 10 Feb 2024 13:14:59 +1000
Subject: [PATCH 1857/2492] doc/radosgw: remove invalid LUA context options

Remove "background", "getdata", and "putdata" from the list of LUA
context options. Passing these options throws the following error:
"ERROR: invalid script context: background. must be one of: preRequest,
postRequest".

Fixes: https://tracker.ceph.com/issues/64327

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/lua-scripting.rst | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst
index 8f95aea3c14a..a663c8ed04d5 100644
--- a/doc/radosgw/lua-scripting.rst
+++ b/doc/radosgw/lua-scripting.rst
@@ -8,11 +8,8 @@ Lua Scripting
 
 This feature allows users to assign execution context to Lua scripts. The supported contexts are:
 
- - ``prerequest`` which will execute a script before each operation is performed
- - ``postrequest`` which will execute after each operation is performed
- - ``background`` which will execute within a specified time interval
- - ``getdata`` which will execute on objects' data when objects are downloaded
- - ``putdata`` which will execute on objects' data when objects are uploaded
+ - ``preRequest`` which will execute a script before each operation is performed
+ - ``postRequest`` which will execute after each operation is performed
 
 A request (pre or post) or data (get or put) context script may be constrained to operations belonging to a specific tenant's users.
 The request context script can also access fields in the request and modify certain fields, as well as the `Global RGW Table`_.
@@ -43,29 +40,28 @@ To upload a script:
 
 ::
    
-   # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+   # radosgw-admin script put --infile={lua-file-path} --context={preRequest|postRequest} [--tenant={tenant-name}]
 
 
-* When uploading a script with the ``background`` context, a tenant name should not be specified.
 * When uploading a script into a cluster deployed with cephadm, use the following command:
 
 ::
 
-  # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+  # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest} [--tenant={tenant-name}]
 
 
 To print the content of the script to standard output:
 
 ::
    
-   # radosgw-admin script get --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+   # radosgw-admin script get --context={preRequest|postRequest} [--tenant={tenant-name}]
 
 
 To remove the script:
 
 ::
    
-   # radosgw-admin script rm --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+   # radosgw-admin script rm --context={preRequest|postRequest} [--tenant={tenant-name}]
 
 
 Package Management via CLI

From 284e99ebc67ee1caf0623f216fc7c959bba57835 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 11 Feb 2024 00:36:29 +1000
Subject: [PATCH 1858/2492] doc/radosgw: undo 55524

Roll back the docs changes made in
https://github.com/ceph/ceph/pull/55524, in accordance with Casey
Bodley's instructions to me here:
https://github.com/ceph/ceph/pull/55524#issuecomment-1937020543.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/lua-scripting.rst | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst
index a663c8ed04d5..eafde154e25c 100644
--- a/doc/radosgw/lua-scripting.rst
+++ b/doc/radosgw/lua-scripting.rst
@@ -8,6 +8,11 @@ Lua Scripting
 
 This feature allows users to assign execution context to Lua scripts. The supported contexts are:
 
+ - ``prerequest`` which will execute a script before each operation is performed
+ - ``postrequest`` which will execute after each operation is performed
+ - ``background`` which will execute within a specified time interval
+ - ``getdata`` which will execute on objects' data when objects are downloaded
+ - ``putdata`` which will execute on objects' data when objects are uploaded
  - ``preRequest`` which will execute a script before each operation is performed
  - ``postRequest`` which will execute after each operation is performed
 
@@ -39,29 +44,28 @@ To upload a script:
    
 
 ::
-   
-   # radosgw-admin script put --infile={lua-file-path} --context={preRequest|postRequest} [--tenant={tenant-name}]
-
 
-* When uploading a script into a cluster deployed with cephadm, use the following command:
+   # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]   
+   
+* When uploading a script with the ``background`` context, a tenant name should not be specified.
 
 ::
 
-  # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest} [--tenant={tenant-name}]
+  # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
 
 
 To print the content of the script to standard output:
 
 ::
    
-   # radosgw-admin script get --context={preRequest|postRequest} [--tenant={tenant-name}]
+   # radosgw-admin script get --context={preRequest|postRequest|background|getdata|putdata} [--tenant={tenant-name}]
 
 
 To remove the script:
 
 ::
    
-   # radosgw-admin script rm --context={preRequest|postRequest} [--tenant={tenant-name}]
+   # radosgw-admin script rm --context={preRequest|postRequest|background|getdata|putdata} [--tenant={tenant-name}]
 
 
 Package Management via CLI

From c7155c3db7365e8b25d15af6c2fd2b2871aa552f Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 11 Feb 2024 15:42:14 +0800
Subject: [PATCH 1859/2492] cmake: build boost debug variant when
 CMAKE_BUILD_TYPE is Debug

boost has some different predefined  build variants. they are quite
like CMake's CMAKE_BUILD_TYPE. in which, "debug" enables some
features related features. so it would be nice if we can have it
enabled for the Debug build, if the boost is built from source.
see also
https://www.boost.org/build/doc/html/bbv2/overview/builtins/features.html

before this change, we always build the "release" variant.  in this
change, "debug" variant is built if Ceph's is built with
CMAKE_BUILD_TYPE=Debug. please note, this change does not change
the way how boost is built when packaging Ceph, as our debian/rpm
receipts do not define CMAKE_BUILD_TYPE and respect the distros'
settings, in that case, the "release" variant is still built.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/BuildBoost.cmake | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmake/modules/BuildBoost.cmake b/cmake/modules/BuildBoost.cmake
index a22578795908..47cd702ca225 100644
--- a/cmake/modules/BuildBoost.cmake
+++ b/cmake/modules/BuildBoost.cmake
@@ -47,7 +47,11 @@ endmacro()
 
 function(do_build_boost root_dir version)
   cmake_parse_arguments(Boost_BUILD "" "" COMPONENTS ${ARGN})
-  set(boost_features "variant=release")
+  if(CMAKE_BUILD_TYPE STREQUAL Debug)
+    set(boost_features "variant=debug")
+  else()
+    set(boost_features "variant=release")
+  endif()
   if(Boost_USE_MULTITHREADED)
     list(APPEND boost_features "threading=multi")
   else()

From 0de5755531e890e26c37a184db98b40538d25286 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 11 Feb 2024 16:53:26 +0800
Subject: [PATCH 1860/2492] cmake: find_package(cap) before linking against it

before this change, we link against libcap without finding it. this
works fine as long as libcap-devel or libcap-dev is installed in the
system. but if it is not, the source would fail to build due to missing
`sys/capability.h`. this is not a great developer experience.

in this change, a `Findcap.cmake` is added to find the capability
library. which would fail the build at the configure phase.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/Findcap.cmake  | 35 +++++++++++++++++++++++++++++++++++
 src/extblkdev/CMakeLists.txt |  1 +
 2 files changed, 36 insertions(+)
 create mode 100644 cmake/modules/Findcap.cmake

diff --git a/cmake/modules/Findcap.cmake b/cmake/modules/Findcap.cmake
new file mode 100644
index 000000000000..f33b22d2c29e
--- /dev/null
+++ b/cmake/modules/Findcap.cmake
@@ -0,0 +1,35 @@
+# Try to find libcap
+#
+find_package(PkgConfig QUIET REQUIRED)
+
+pkg_check_modules(PC_cap QUIET cap)
+
+find_library(cap_LIBRARY
+  NAMES cap
+  HINTS
+    ${PC_cap_LIBDIR}
+    ${PC_cap_LIBRARY_DIRS})
+
+find_path(cap_INCLUDE_DIR
+  NAMES sys/capability.h
+  HINTS
+    ${PC_cap_INCLUDEDIR}
+    ${PC_cap_INCLUDE_DIRS})
+
+mark_as_advanced(
+  cap_LIBRARY
+  cap_INCLUDE_DIR)
+
+include (FindPackageHandleStandardArgs)
+find_package_handle_standard_args (cap
+  REQUIRED_VARS
+    cap_LIBRARY
+    cap_INCLUDE_DIR)
+
+if(cap_FOUND AND NOT TARGET cap::cap)
+  add_library(cap::cap UNKNOWN IMPORTED)
+  set_target_properties(cap::cap
+    PROPERTIES
+      IMPORTED_LOCATION ${cap_LIBRARY}
+      INTERFACE_INCLUDE_DIRECTORIES ${cap_INCLUDE_DIR})
+endif()
diff --git a/src/extblkdev/CMakeLists.txt b/src/extblkdev/CMakeLists.txt
index 64010f31cf3e..27e7c23e4942 100644
--- a/src/extblkdev/CMakeLists.txt
+++ b/src/extblkdev/CMakeLists.txt
@@ -7,6 +7,7 @@ add_subdirectory(vdo)
 add_library(extblkdev STATIC ExtBlkDevPlugin.cc)
 
 if(NOT WIN32)
+find_package(cap)
 target_link_libraries(extblkdev cap)
 endif()
 

From 5ad6f16562aec7086ca6aefdfb88c1059eb0586d Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 11 Feb 2024 19:32:19 +0200
Subject: [PATCH 1861/2492] doc/osd: update the documentation of several
 scrub-related options

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 doc/glossary.rst                           |  2 +-
 doc/rados/configuration/osd-config-ref.rst |  2 ++
 src/common/options/osd.yaml.in             | 16 ++++++++++++----
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 12e40f59c98b..8c7dfe03ce80 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -422,7 +422,7 @@
                 "inconsistent" (that is, the PG is marked "inconsistent"). 
 
                 There are two kinds of scrubbing: light scrubbing and deep
-                scrubbing (also called "normal scrubbing" and "deep scrubbing",
+                scrubbing (also called "shallow scrubbing" and "deep scrubbing",
                 respectively). Light scrubbing is performed daily and does
                 nothing more than confirm that a given object exists and that
                 its metadata is correct. Deep scrubbing is performed weekly and
diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
index 060121200838..634690b88c06 100644
--- a/doc/rados/configuration/osd-config-ref.rst
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -171,7 +171,9 @@ increase or decrease the frequency and depth of scrubbing operations.
 .. confval:: osd_scrub_min_interval
 .. confval:: osd_scrub_max_interval
 .. confval:: osd_scrub_chunk_min
+.. confval:: osd_shallow_scrub_chunk_min
 .. confval:: osd_scrub_chunk_max
+.. confval:: osd_shallow_scrub_chunk_max
 .. confval:: osd_scrub_sleep
 .. confval:: osd_deep_scrub_interval
 .. confval:: osd_scrub_interval_randomize_ratio
diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 2a7bad84e7b1..598cf4f50fd0 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -381,9 +381,11 @@ options:
 - name: osd_scrub_sleep
   type: float
   level: advanced
-  desc: Duration to inject a delay during scrubbing
-  fmt_desc: Time to sleep before scrubbing the next group of chunks (seconds). Increasing this value will slow
-    down the overall rate of scrubbing so that client operations will be less impacted.
+  desc: Duration (in seconds) of delay injected between chunks when scrubbing
+  fmt_desc: Sleep time in seconds before scrubbing the next group of objects (the next chunk).
+    Increasing this value will slow down the overall rate of scrubbing, reducing scrub
+    impact on client operations.
+    This setting is ignored when the mClock scheduler is used.
   default: 0
   flags:
   - runtime
@@ -392,7 +394,13 @@ options:
 - name: osd_scrub_extended_sleep
   type: float
   level: advanced
-  desc: Duration to inject a delay during scrubbing out of scrubbing hours (seconds)
+  desc: Duration (in seconds) of delay injected between chunks when scrubbing out
+    of scrubbing hours
+  fmt_desc: Sleep time in seconds before scrubbing the next group of objects (the next chunk).
+    This configuration value is used for scrubbing out of scrubbing hours.
+    Increasing this value will slow down the overall rate of scrubbing, reducing scrub
+    impact on client operations.
+    This setting is ignored when the mClock scheduler is used.
   default: 0
   see_also:
   - osd_scrub_begin_hour

From e0bb7117031408b50416031d9adb9e99e16b2100 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Wed, 7 Feb 2024 14:03:45 +0000
Subject: [PATCH 1862/2492] mgr/dashboard: added delete and activation actions

Fixes: https://tracker.ceph.com/issues/64355
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  51 ++++++
 ...ephfs-snapshotschedule-list.component.html |   2 +-
 .../cephfs-snapshotschedule-list.component.ts | 120 ++++++++++++--
 .../api/cephfs-snapshot-schedule.service.ts   |  30 +++-
 .../src/app/shared/constants/app.constants.ts |   5 +
 .../shared/services/task-message.service.ts   |  24 ++-
 src/pybind/mgr/dashboard/openapi.yaml         | 147 ++++++++++++++++++
 7 files changed, 357 insertions(+), 22 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 1437962723ff..83e9cbfd8533 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -1025,3 +1025,54 @@ def editRetentionPolicies(method, retention_policy):
         editRetentionPolicies('snap_schedule_retention_add', retention_to_add)
 
         return f'Retention policies for snapshot schedule on path {path} updated successfully'
+
+    @RESTController.Resource('DELETE')
+    def delete_snapshot(self, fs: str, path: str, schedule: str, start: str):
+        error_code, _, err = mgr.remote('snap_schedule',
+                                        'snap_schedule_rm',
+                                        path,
+                                        schedule,
+                                        start,
+                                        fs,
+                                        None,
+                                        None)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to delete snapshot schedule for path {path}: {err}'
+            )
+
+        return f'Snapshot schedule for path {path} deleted successfully'
+
+    @RESTController.Resource('POST')
+    def deactivate(self, fs: str, path: str, schedule: str, start: str):
+        error_code, _, err = mgr.remote('snap_schedule',
+                                        'snap_schedule_deactivate',
+                                        path,
+                                        schedule,
+                                        start,
+                                        fs,
+                                        None,
+                                        None)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to deactivate snapshot schedule for path {path}: {err}'
+            )
+
+        return f'Snapshot schedule for path {path} deactivated successfully'
+
+    @RESTController.Resource('POST')
+    def activate(self, fs: str, path: str, schedule: str, start: str):
+        error_code, _, err = mgr.remote('snap_schedule',
+                                        'snap_schedule_activate',
+                                        path,
+                                        schedule,
+                                        start,
+                                        fs,
+                                        None,
+                                        None)
+        if error_code != 0:
+            raise DashboardException(
+                f'Failed to activate snapshot schedule for path {path}: {err}'
+            )
+
+        return f'Snapshot schedule for path {path} activated successfully'
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
index 0346de67f941..f2e93cc742c7 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
@@ -73,7 +73,7 @@
       [permission]="permissions.cephfs"
       [selection]="selection"
       class="btn-group"
-      [tableActions]="tableActions"
+      [tableActions]="tableActions$ | async"
     >
     </cd-table-actions>
   </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index b6b52a15c99d..58d3a0cc056a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -26,8 +26,11 @@ import { MgrModuleService } from '~/app/shared/api/mgr-module.service';
 import { NotificationService } from '~/app/shared/services/notification.service';
 import { BlockUI, NgBlockUI } from 'ng-block-ui';
 import { NotificationType } from '~/app/shared/enum/notification-type.enum';
-import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants';
 import { CephfsSnapshotscheduleFormComponent } from '../cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component';
+import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+import { FinishedTask } from '~/app/shared/models/finished-task';
 
 @Component({
   selector: 'cd-cephfs-snapshotschedule-list',
@@ -51,7 +54,7 @@ export class CephfsSnapshotscheduleListComponent
   snapScheduleModuleStatus$ = new BehaviorSubject<boolean>(false);
   moduleServiceListSub!: Subscription;
   columns: CdTableColumn[] = [];
-  tableActions: CdTableAction[] = [];
+  tableActions$ = new BehaviorSubject<CdTableAction[]>([]);
   context!: CdTableFetchDataContext;
   selection = new CdTableSelection();
   permissions!: Permissions;
@@ -59,6 +62,26 @@ export class CephfsSnapshotscheduleListComponent
   errorMessage: string = '';
   selectedName: string = '';
   icons = Icons;
+  tableActions: CdTableAction[] = [
+    {
+      name: this.actionLabels.CREATE,
+      permission: 'create',
+      icon: Icons.add,
+      click: () => this.openModal(false)
+    },
+    {
+      name: this.actionLabels.EDIT,
+      permission: 'update',
+      icon: Icons.edit,
+      click: () => this.openModal(true)
+    },
+    {
+      name: this.actionLabels.DELETE,
+      permission: 'delete',
+      icon: Icons.trash,
+      click: () => this.deleteSnapshotSchedule()
+    }
+  ];
 
   MODULE_NAME = 'snap_schedule';
   ENABLE_MODULE_TIMER = 2 * 1000;
@@ -69,7 +92,8 @@ export class CephfsSnapshotscheduleListComponent
     private modalService: ModalService,
     private mgrModuleService: MgrModuleService,
     private notificationService: NotificationService,
-    private actionLables: ActionLabelsI18n
+    private actionLabels: ActionLabelsI18n,
+    private taskWrapper: TaskWrapperService
   ) {
     super();
     this.permissions = this.authStorageService.getPermissions();
@@ -116,20 +140,7 @@ export class CephfsSnapshotscheduleListComponent
       { prop: 'created', name: $localize`Created`, cellTransformation: CellTemplate.timeAgo }
     ];
 
-    this.tableActions = [
-      {
-        name: this.actionLables.CREATE,
-        permission: 'create',
-        icon: Icons.add,
-        click: () => this.openModal(false)
-      },
-      {
-        name: this.actionLables.EDIT,
-        permission: 'update',
-        icon: Icons.edit,
-        click: () => this.openModal(true)
-      }
-    ];
+    this.tableActions$.next(this.tableActions);
   }
 
   ngOnDestroy(): void {
@@ -142,6 +153,19 @@ export class CephfsSnapshotscheduleListComponent
 
   updateSelection(selection: CdTableSelection) {
     this.selection = selection;
+    if (!this.selection.hasSelection) return;
+    const isActive = this.selection.first()?.active;
+
+    this.tableActions$.next([
+      ...this.tableActions,
+      {
+        name: isActive ? this.actionLabels.DEACTIVATE : this.actionLabels.ACTIVATE,
+        permission: 'update',
+        icon: isActive ? Icons.warning : Icons.success,
+        click: () =>
+          isActive ? this.deactivateSnapshotSchedule() : this.activateSnapshotSchedule()
+      }
+    ]);
   }
 
   openModal(edit = false) {
@@ -204,4 +228,66 @@ export class CephfsSnapshotscheduleListComponent
       }
     );
   }
+
+  deactivateSnapshotSchedule() {
+    const { path, start, fs, schedule } = this.selection.first();
+
+    this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      itemDescription: $localize`snapshot schedule`,
+      actionDescription: this.actionLabels.DEACTIVATE,
+      submitActionObservable: () =>
+        this.taskWrapper.wrapTaskAroundCall({
+          task: new FinishedTask('cephfs/snapshot/schedule/deactivate', {
+            path
+          }),
+          call: this.snapshotScheduleService.deactivate({
+            path,
+            schedule,
+            start,
+            fs
+          })
+        })
+    });
+  }
+
+  activateSnapshotSchedule() {
+    const { path, start, fs, schedule } = this.selection.first();
+
+    this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      itemDescription: $localize`snapshot schedule`,
+      actionDescription: this.actionLabels.ACTIVATE,
+      submitActionObservable: () =>
+        this.taskWrapper.wrapTaskAroundCall({
+          task: new FinishedTask('cephfs/snapshot/schedule/activate', {
+            path
+          }),
+          call: this.snapshotScheduleService.activate({
+            path,
+            schedule,
+            start,
+            fs
+          })
+        })
+    });
+  }
+
+  deleteSnapshotSchedule() {
+    const { path, start, fs, schedule } = this.selection.first();
+
+    this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      itemDescription: $localize`snapshot schedule`,
+      submitActionObservable: () =>
+        this.taskWrapper.wrapTaskAroundCall({
+          task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.DELETE, {
+            path
+          }),
+          call: this.snapshotScheduleService.delete({
+            path,
+            schedule,
+            start,
+            fs
+          })
+        })
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
index 0719089c249f..9e07f6057ac1 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -19,14 +19,38 @@ export class CephfsSnapshotScheduleService {
     return this.http.post(`${this.baseURL}/snapshot/schedule`, data, { observe: 'response' });
   }
 
-  update(data: Record<string, any>): Observable<any> {
+  update({ fs, path, ...rest }: Record<string, any>): Observable<any> {
     return this.http.put(
-      `${this.baseURL}/snapshot/schedule/${data.fs}/${encodeURIComponent(data.path)}`,
-      data,
+      `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(path)}`,
+      rest,
       { observe: 'response' }
     );
   }
 
+  activate({ fs, path, ...rest }: Record<string, any>): Observable<any> {
+    return this.http.post(
+      `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(path)}/activate`,
+      rest,
+      { observe: 'response' }
+    );
+  }
+
+  deactivate({ fs, path, ...rest }: Record<string, any>): Observable<any> {
+    return this.http.post(
+      `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(path)}/deactivate`,
+      rest,
+      { observe: 'response' }
+    );
+  }
+
+  delete({ fs, path, schedule, start }: Record<string, any>): Observable<any> {
+    return this.http.delete(
+      `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(
+        path
+      )}/delete_snapshot?schedule=${schedule}&start=${encodeURIComponent(start)}`
+    );
+  }
+
   checkScheduleExists(
     path: string,
     fs: string,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index 7edce5ff6671..2cf3f1047bab 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -141,6 +141,8 @@ export class ActionLabelsI18n {
   IMPORT: any;
   MIGRATE: string;
   START_UPGRADE: string;
+  ACTIVATE: string;
+  DEACTIVATE: string;
 
   constructor() {
     /* Create a new item */
@@ -218,6 +220,9 @@ export class ActionLabelsI18n {
     this.DEMOTE = $localize`Demote`;
 
     this.START_UPGRADE = $localize`Start Upgrade`;
+
+    this.ACTIVATE = $localize`Activate`;
+    this.DEACTIVATE = $localize`Deactivate`;
   }
 }
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index 84e31efea013..4fbcc09d0902 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -69,7 +69,17 @@ export class TaskMessageService {
     delete: new TaskMessageOperation($localize`Deleting`, $localize`delete`, $localize`Deleted`),
     add: new TaskMessageOperation($localize`Adding`, $localize`add`, $localize`Added`),
     remove: new TaskMessageOperation($localize`Removing`, $localize`remove`, $localize`Removed`),
-    import: new TaskMessageOperation($localize`Importing`, $localize`import`, $localize`Imported`)
+    import: new TaskMessageOperation($localize`Importing`, $localize`import`, $localize`Imported`),
+    activate: new TaskMessageOperation(
+      $localize`Importing`,
+      $localize`activate`,
+      $localize`Activated`
+    ),
+    deactivate: new TaskMessageOperation(
+      $localize`Importing`,
+      $localize`deactivate`,
+      $localize`Deactivated`
+    )
   };
 
   rbd = {
@@ -393,6 +403,18 @@ export class TaskMessageService {
     ),
     'cephfs/snapshot/schedule/edit': this.newTaskMessage(this.commonOperations.update, (metadata) =>
       this.snapshotSchedule(metadata)
+    ),
+    'cephfs/snapshot/schedule/delete': this.newTaskMessage(
+      this.commonOperations.delete,
+      (metadata) => this.snapshotSchedule(metadata)
+    ),
+    'cephfs/snapshot/schedule/activate': this.newTaskMessage(
+      this.commonOperations.activate,
+      (metadata) => this.snapshotSchedule(metadata)
+    ),
+    'cephfs/snapshot/schedule/deactivate': this.newTaskMessage(
+      this.commonOperations.deactivate,
+      (metadata) => this.snapshotSchedule(metadata)
     )
   };
 
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 8457ecb65c6a..1fbaac399aaa 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1890,6 +1890,153 @@ paths:
       - jwt: []
       tags:
       - CephFSSnapshotSchedule
+  /api/cephfs/snapshot/schedule/{fs}/{path}/activate:
+    post:
+      parameters:
+      - in: path
+        name: fs
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: path
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                schedule:
+                  type: string
+                start:
+                  type: string
+              required:
+              - schedule
+              - start
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
+  /api/cephfs/snapshot/schedule/{fs}/{path}/deactivate:
+    post:
+      parameters:
+      - in: path
+        name: fs
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: path
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                schedule:
+                  type: string
+                start:
+                  type: string
+              required:
+              - schedule
+              - start
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
+  /api/cephfs/snapshot/schedule/{fs}/{path}/delete_snapshot:
+    delete:
+      parameters:
+      - in: path
+        name: fs
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: path
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: schedule
+        required: true
+        schema:
+          type: string
+      - in: query
+        name: start
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - CephFSSnapshotSchedule
   /api/cephfs/subvolume:
     post:
       parameters: []

From 1713c4852ca66f9ec5023515a26f1f06902a21bc Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Thu, 1 Feb 2024 18:37:27 +0530
Subject: [PATCH 1863/2492] qa: add qa/tasks/nvmeof.py and
 rbd/nvmeof_basic_task and fio workunits

This is v2 of the rbd/nvmeof test: It deploys 1 gateway and 1 initiator.
Then does basic verification on nvme commands and runs fio.

This commit creates:
1. qa/tasks/nvmeof.py: adds a new 'Nvmeof' task which deploys
    the gateway and shares config with the initiator hosts.
    Sharing config was previously done by 'nvmeof_gateway_cfg' task
    in qa/tasks/cephadm.py (that task is removed in this commit).
2. qa/workunits/rbd/nvmeof_basic_tests.sh:
    Runs nvme commands (discovery, connect, connect-all, disconnect-all,
    and list-subsys) and does basic verification of the output.
3. qa/workunits/rbd/nvmeof_fio_test.sh:
    Runs fio command. Also runs iostat in parallel if IOSTAT_INTERVAL
    variable is set. This variable configures the delay between each iostat
    print.

nvmeof-cli upgrade from v0.0.6 to v0.0.7 introduced major changes
to all nvmeof commands. This commit changes v0.0.6 commands to
v0.0.7 in qa/workunits/rbd/nvmeof_initiator.sh

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 qa/suites/rbd/nvmeof/base/install.yaml        |  19 --
 qa/suites/rbd/nvmeof/cluster/fixed-3.yaml     |   2 +
 .../nvmeof/workloads/nvmeof_initiator.yaml    |  23 ++-
 qa/tasks/cephadm.py                           |  38 ----
 qa/tasks/nvmeof.py                            | 168 ++++++++++++++++++
 qa/workunits/rbd/nvmeof_basic_tests.sh        |  72 ++++++++
 qa/workunits/rbd/nvmeof_fio_test.sh           |  36 ++++
 qa/workunits/rbd/nvmeof_initiator.sh          |  74 ++------
 8 files changed, 311 insertions(+), 121 deletions(-)
 create mode 100644 qa/tasks/nvmeof.py
 create mode 100755 qa/workunits/rbd/nvmeof_basic_tests.sh
 create mode 100755 qa/workunits/rbd/nvmeof_fio_test.sh

diff --git a/qa/suites/rbd/nvmeof/base/install.yaml b/qa/suites/rbd/nvmeof/base/install.yaml
index 5a852f14dbe1..6fc91d8f359f 100644
--- a/qa/suites/rbd/nvmeof/base/install.yaml
+++ b/qa/suites/rbd/nvmeof/base/install.yaml
@@ -10,23 +10,4 @@ tasks:
     - ceph orch host ls
     - ceph orch device ls
     - ceph osd lspools
-    # create pool
-    - ceph osd pool create mypool
-    - rbd pool init mypool
-    # deploy nvmeof
-    ## Uncomment to test specific nvmeof images
-    ## - ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:latest
-    - ceph orch apply nvmeof mypool --placement="1 $(hostname)"
-    - ceph orch ps --refresh
 
-- cephadm.wait_for_service:
-    service: nvmeof.mypool
-
-- cephadm.nvmeof_gateway_cfg:
-    source: host.a 
-    target: client.1
-    service: nvmeof.mypool
-
-- exec:
-    client.0:
-      - journalctl -u $(systemctl list-units | grep nvmeof.mypool | awk '{print $1}')
diff --git a/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml b/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
index 42e696cd2f14..f417079e31a1 100644
--- a/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
+++ b/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
@@ -5,9 +5,11 @@ roles:
   - osd.0
   - osd.1
   - client.0
+  - ceph.nvmeof.nvmeof.a
 - - host.b
   - mon.b
   - osd.2
   - osd.3
   - osd.4
   - client.1
+- - client.2
diff --git a/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
index 4c947c1f787f..bbb9b0ab5f2a 100644
--- a/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
+++ b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
@@ -1,6 +1,27 @@
 tasks:
+- nvmeof:
+    client: client.0
+    version: latest  # "default" uses packaged version; change to test specific nvmeof images, example "latest"
+    rbd:
+        pool_name: mypool
+        image_name: myimage
+    gateway_config:
+        source: host.a 
+        target: client.2
+        vars:
+            cli_version: latest
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool
+
 - workunit:
     no_coverage_and_limits: true
     clients:
-      client.1:
+      client.2:
         - rbd/nvmeof_initiator.sh
+        - rbd/nvmeof_basic_tests.sh
+        - rbd/nvmeof_fio_test.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE: myimage
+      IOSTAT_INTERVAL: '10'
diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 9e386bbd5c29..f60aabc84359 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -21,7 +21,6 @@
 from teuthology.orchestra import run
 from teuthology.orchestra.daemon import DaemonGroup
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import ConfigError
 from textwrap import dedent
 from tasks.cephfs.filesystem import MDSCluster, Filesystem
 from tasks.util import chacra
@@ -101,43 +100,6 @@ def update_archive_setting(ctx, key, value):
         yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
 
 
-@contextlib.contextmanager
-def nvmeof_gateway_cfg(ctx, config):
-    source_host = config.get('source')
-    target_host = config.get('target')
-    nvmeof_service = config.get('service')
-    if not (source_host and target_host and nvmeof_service):
-        raise ConfigError('nvmeof_gateway_cfg requires "source", "target", and "service"')
-    remote = list(ctx.cluster.only(source_host).remotes.keys())[0]
-    ip_address = remote.ip_address
-    gateway_name = ""
-    r = remote.run(args=[
-        'systemctl', 'list-units',
-        run.Raw('|'), 'grep', nvmeof_service
-    ], stdout=StringIO())
-    output = r.stdout.getvalue()
-    pattern_str = f"{re.escape(nvmeof_service)}(.*?)(?=\.service)"
-    pattern = re.compile(pattern_str)
-    match = pattern.search(output)
-    if match:
-        gateway_name = match.group()
-    conf_data = dedent(f"""
-        NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
-        NVMEOF_GATEWAY_NAME={gateway_name}
-        """)
-    target_remote = list(ctx.cluster.only(target_host).remotes.keys())[0]
-    target_remote.write_file(
-        path='/etc/ceph/nvmeof.env',
-        data=conf_data,
-        sudo=True
-    )
-
-    try:
-        yield
-    finally:
-        pass
-
-
 @contextlib.contextmanager
 def normalize_hostnames(ctx):
     """
diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py
new file mode 100644
index 000000000000..b75d00d93ae8
--- /dev/null
+++ b/qa/tasks/nvmeof.py
@@ -0,0 +1,168 @@
+import logging
+from textwrap import dedent
+from teuthology.task import Task
+from teuthology import misc
+from teuthology.exceptions import ConfigError
+from tasks.util import get_remote_for_role
+from tasks.cephadm import _shell
+
+log = logging.getLogger(__name__)
+
+conf_file = '/etc/ceph/nvmeof.env'
+
+
+class Nvmeof(Task):
+    """
+    Setup nvmeof gateway on client and then share gateway config to target host.
+
+        - nvmeof:
+            client: client.0
+            version: default
+            rbd:
+                pool_name: mypool
+                image_name: myimage
+                rbd_size: 1024
+            gateway_config:
+                source: host.a 
+                target: client.2
+                vars:
+                    cli_version: latest
+                    
+    """
+
+    def setup(self):
+        super(Nvmeof, self).setup()
+        try:
+            self.client = self.config['client']
+        except KeyError:
+            raise ConfigError('nvmeof requires a client to connect with')
+
+        self.cluster_name, type_, self.client_id = misc.split_role(self.client)
+        if type_ != 'client':
+            msg = 'client role ({0}) must be a client'.format(self.client)
+            raise ConfigError(msg)
+        self.remote = get_remote_for_role(self.ctx, self.client)
+
+    def begin(self):
+        super(Nvmeof, self).begin()
+        self._set_defaults()
+        self.deploy_nvmeof()
+        self.set_gateway_cfg()
+
+    def _set_defaults(self):
+        self.gateway_image = self.config.get('version', 'default')
+
+        rbd_config = self.config.get('rbd', {})
+        self.poolname = rbd_config.get('pool_name', 'mypool')
+        self.rbd_image_name = rbd_config.get('image_name', 'myimage')
+        self.rbd_size = rbd_config.get('rbd_size', 1024*8)
+
+        gateway_config = self.config.get('gateway_config', {})
+        conf_vars = gateway_config.get('vars', {})
+        self.cli_image = conf_vars.get('cli_version', 'latest')
+        self.bdev = conf_vars.get('bdev', 'mybdev')
+        self.serial = conf_vars.get('serial', 'SPDK00000000000001')
+        self.nqn = conf_vars.get('nqn', 'nqn.2016-06.io.spdk:cnode1')
+        self.port = conf_vars.get('port', '4420')
+        self.srport = conf_vars.get('srport', '5500')
+
+    def deploy_nvmeof(self):
+        """
+        Deploy nvmeof gateway.
+        """
+        log.info('[nvmeof]: deploying nvmeof gateway...')
+        if not hasattr(self.ctx, 'ceph'):
+            self.ctx.ceph = {}
+        fsid = self.ctx.ceph[self.cluster_name].fsid
+
+        nodes = []
+        daemons = {}
+
+        for remote, roles in self.ctx.cluster.remotes.items():
+            for role in [r for r in roles
+                         if misc.is_type('nvmeof', self.cluster_name)(r)]:
+                c_, _, id_ = misc.split_role(role)
+                log.info('Adding %s on %s' % (role, remote.shortname))
+                nodes.append(remote.shortname + '=' + id_)
+                daemons[role] = (remote, id_)
+
+        if nodes:
+            image = self.gateway_image
+            if (image != "default"):
+                log.info(f'[nvmeof]: ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:{image}')
+                _shell(self.ctx, self.cluster_name, self.remote, [
+                    'ceph', 'config', 'set', 'mgr', 
+                    'mgr/cephadm/container_image_nvmeof',
+                    f'quay.io/ceph/nvmeof:{image}'
+                ])
+
+            poolname = self.poolname
+            imagename = self.rbd_image_name
+
+            log.info(f'[nvmeof]: ceph osd pool create {poolname}')
+            _shell(self.ctx, self.cluster_name, self.remote, [
+                'ceph', 'osd', 'pool', 'create', poolname
+            ])
+
+            log.info(f'[nvmeof]: rbd pool init {poolname}')
+            _shell(self.ctx, self.cluster_name, self.remote, [
+                'rbd', 'pool', 'init', poolname
+            ])
+
+            log.info(f'[nvmeof]: ceph orch apply nvmeof {poolname}')
+            _shell(self.ctx, self.cluster_name, self.remote, [
+                'ceph', 'orch', 'apply', 'nvmeof', poolname, 
+                '--placement', str(len(nodes)) + ';' + ';'.join(nodes)
+            ])
+
+            log.info(f'[nvmeof]: rbd create {poolname}/{imagename} --size {self.rbd_size}')
+            _shell(self.ctx, self.cluster_name, self.remote, [
+                'rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}'
+            ])
+
+        for role, i in daemons.items():
+            remote, id_ = i
+            self.ctx.daemons.register_daemon(
+                remote, 'nvmeof', id_,
+                cluster=self.cluster_name,
+                fsid=fsid,
+                logger=log.getChild(role),
+                wait=False,
+                started=True,
+            )
+        log.info("[nvmeof]: executed deploy_nvmeof successfully!")
+        
+    def set_gateway_cfg(self):
+        log.info('[nvmeof]: running set_gateway_cfg...')
+        gateway_config = self.config.get('gateway_config', {})
+        source_host = gateway_config.get('source')
+        target_host = gateway_config.get('target')
+        if not (source_host and target_host):
+            raise ConfigError('gateway_config requires "source" and "target"')
+        remote = list(self.ctx.cluster.only(source_host).remotes.keys())[0]
+        ip_address = remote.ip_address
+        gateway_name = ""
+        nvmeof_daemons = self.ctx.daemons.iter_daemons_of_role('nvmeof', cluster=self.cluster_name)
+        for daemon in nvmeof_daemons:
+            if ip_address == daemon.remote.ip_address:
+                gateway_name = daemon.name()
+        conf_data = dedent(f"""
+            NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
+            NVMEOF_GATEWAY_NAME={gateway_name}
+            NVMEOF_CLI_IMAGE="quay.io/ceph/nvmeof-cli:{self.cli_image}"
+            NVMEOF_BDEV={self.bdev}
+            NVMEOF_SERIAL={self.serial}
+            NVMEOF_NQN={self.nqn}
+            NVMEOF_PORT={self.port}
+            NVMEOF_SRPORT={self.srport}
+            """)
+        target_remote = list(self.ctx.cluster.only(target_host).remotes.keys())[0]
+        target_remote.write_file(
+            path=conf_file,
+            data=conf_data,
+            sudo=True
+        )
+        log.info("[nvmeof]: executed set_gateway_cfg successfully!")
+
+
+task = Nvmeof
diff --git a/qa/workunits/rbd/nvmeof_basic_tests.sh b/qa/workunits/rbd/nvmeof_basic_tests.sh
new file mode 100755
index 000000000000..878e043fbeb5
--- /dev/null
+++ b/qa/workunits/rbd/nvmeof_basic_tests.sh
@@ -0,0 +1,72 @@
+#!/bin/bash -x
+
+source /etc/ceph/nvmeof.env
+SPDK_CONTROLLER="SPDK bdev Controller"
+DISCOVERY_PORT="8009"
+
+discovery() {
+    output=$(sudo nvme discover -t tcp -a $NVMEOF_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT)
+    expected_discovery_stdout="subtype: nvme subsystem"
+    if ! echo "$output" | grep -q "$expected_discovery_stdout"; then
+        return 1
+    fi
+}
+
+connect() {
+    sudo nvme connect -t tcp --traddr $NVMEOF_GATEWAY_IP_ADDRESS -s $NVMEOF_PORT -n $NVMEOF_NQN
+    output=$(sudo nvme list)
+    if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
+        return 1
+    fi
+}
+
+disconnect_all() {
+    sudo nvme disconnect-all
+    output=$(sudo nvme list)
+    if echo "$output" | grep -q "$SPDK_CONTROLLER"; then
+        return 1
+    fi
+}
+
+connect_all() {
+    sudo nvme connect-all --traddr=$NVMEOF_GATEWAY_IP_ADDRESS --transport=tcp
+    output=$(sudo nvme list)
+    if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
+        return 1
+    fi
+}
+
+list_subsys() {
+    expected_count=$1
+    output=$(sudo nvme list-subsys --output-format=json)
+    multipath=$(echo $output | grep -c '"tcp"')
+    if [ "$multipath" -ne "$expected_count" ]; then
+        return 1
+    fi
+}
+
+
+test_run() {
+    echo "[nvmeof] Running test: $1"
+    $1 "${@:2}" # execute func
+    if [ $? -eq 0 ]; then
+        echo "[nvmeof] $1 test passed!"
+    else
+        echo "[nvmeof] $1 test failed!"
+        exit 1
+    fi
+}
+
+
+test_run disconnect_all
+test_run discovery 
+test_run connect
+test_run list_subsys 1
+test_run disconnect_all
+test_run list_subsys 0
+test_run connect_all
+test_run list_subsys 1
+
+
+echo "-------------Test Summary-------------"
+echo "[nvmeof] All nvmeof basic tests passed!"
diff --git a/qa/workunits/rbd/nvmeof_fio_test.sh b/qa/workunits/rbd/nvmeof_fio_test.sh
new file mode 100755
index 000000000000..bacc15e83eb7
--- /dev/null
+++ b/qa/workunits/rbd/nvmeof_fio_test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash -ex
+
+sudo yum -y install fio
+sudo yum -y install sysstat
+
+fio_file=$(mktemp -t nvmeof-fio-XXXX)
+drives_list=$(sudo nvme list --output-format=json | jq -r '.Devices | .[] | select(.ModelNumber == "SPDK bdev Controller") | .DevicePath')
+
+RUNTIME=${RUNTIME:-600}
+# IOSTAT_INTERVAL=10
+
+
+cat >> $fio_file <<EOF
+[nvmeof-fio-test]
+ioengine=${IO_ENGINE:-sync}
+bsrange=${BS_RANGE:-4k-64k}
+numjobs=${NUM_OF_JOBS:-1}
+size=${SIZE:-1G}
+time_based=1
+runtime=$RUNTIME
+rw=${RW:-randrw}
+filename=$(echo "$drives_list" | tr '\n' ':' | sed 's/:$//')
+verify=md5
+verify_fatal=1
+EOF
+
+fio --showcmd $fio_file
+sudo fio $fio_file &
+
+if [ -n "$IOSTAT_INTERVAL" ]; then
+    iostat_count=$(( RUNTIME / IOSTAT_INTERVAL ))
+    iostat -d $IOSTAT_INTERVAL $iostat_count -h 
+fi
+wait
+
+echo "[nvmeof] fio test successful!"
diff --git a/qa/workunits/rbd/nvmeof_initiator.sh b/qa/workunits/rbd/nvmeof_initiator.sh
index c3f3c1082bfc..8f79238083b2 100755
--- a/qa/workunits/rbd/nvmeof_initiator.sh
+++ b/qa/workunits/rbd/nvmeof_initiator.sh
@@ -6,74 +6,22 @@ sudo modprobe nvme-fabrics
 sudo modprobe nvme-tcp
 sudo dnf install nvme-cli -y
 
-# import NVMEOF_GATEWAY_IP_ADDRESS and NVMEOF_GATEWAY_NAME=nvmeof.poolname.smithiXXX.abcde
 source /etc/ceph/nvmeof.env
 
-HOSTNAME=$(hostname)
-IMAGE="quay.io/ceph/nvmeof-cli:latest"
-RBD_POOL=$(awk -F'.' '{print $2}' <<< "$NVMEOF_GATEWAY_NAME")
-RBD_IMAGE="myimage"
-RBD_SIZE=$((1024*8)) #8GiB
-BDEV="mybdev"
-SERIAL="SPDK00000000000001"
-NQN="nqn.2016-06.io.spdk:cnode1"
-PORT="4420"
-SRPORT="5500"
-DISCOVERY_PORT="8009"
+# RBD_POOL and RBD_IMAGE is intended to be set from yaml, 'mypool' and 'myimage' are defaults
+RBD_POOL="${RBD_POOL:-mypool}"
+RBD_IMAGE="${RBD_IMAGE:-myimage}"
 
-rbd create $RBD_POOL/$RBD_IMAGE --size $RBD_SIZE
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_bdev --pool $RBD_POOL --image $RBD_IMAGE --bdev $BDEV
+HOSTNAME=$(hostname)
 sudo podman images
 sudo podman ps
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_subsystem --subnqn $NQN --serial $SERIAL
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT add_namespace --subnqn $NQN --bdev $BDEV
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT create_listener -n $NQN -g client.$NVMEOF_GATEWAY_NAME -a $NVMEOF_GATEWAY_IP_ADDRESS -s $PORT
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT add_host --subnqn $NQN --host "*"
-sudo podman run -it $IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $SRPORT get_subsystems
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem list
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem add --subsystem $NVMEOF_NQN
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT namespace add --subsystem $NVMEOF_NQN --rbd-pool $RBD_POOL --rbd-image $RBD_IMAGE
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT listener add --subsystem $NVMEOF_NQN --gateway-name client.$NVMEOF_GATEWAY_NAME --traddr $NVMEOF_GATEWAY_IP_ADDRESS --trsvcid $NVMEOF_PORT
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT host add --subsystem $NVMEOF_NQN --host "*"
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem list
 sudo lsmod | grep nvme
 sudo nvme list
-sudo nvme discover -t tcp -a $NVMEOF_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT
-sudo nvme connect -t tcp --traddr $NVMEOF_GATEWAY_IP_ADDRESS -s $PORT -n $NQN
-sudo nvme list
-
-echo "testing nvmeof initiator..."
-
-nvme_model="SPDK bdev Controller"
-
-echo "Test 1: create initiator - starting"
-if ! sudo nvme list | grep -q "$nvme_model"; then
-  echo "nvmeof initiator not created!"
-  exit 1
-fi
-echo "Test 1: create initiator - passed!"
-
-
-echo "Test 2: device size - starting"
-image_size_in_bytes=$(($RBD_SIZE * 1024 * 1024))
-nvme_size=$(sudo nvme list --output-format=json | \
-        jq -r ".Devices | .[] | select(.ModelNumber == \"$nvme_model\") | .PhysicalSize")
-if [ "$image_size_in_bytes" != "$nvme_size" ]; then
-  echo "block device size do not match!"
-  exit 1
-fi
-echo "Test 2: device size - passed!"
-
-
-echo "Test 3: basic IO - starting"
-nvme_drive=$(sudo nvme list --output-format=json | \
-        jq -r ".Devices | .[] | select(.ModelNumber == \"$nvme_model\") | .DevicePath")
-io_input_file="/tmp/nvmeof_test_input"
-echo "Hello world" > $io_input_file
-truncate -s 2k $io_input_file
-sudo dd if=$io_input_file of=$nvme_drive oflag=direct count=1 bs=2k #write
-io_output_file="/tmp/nvmeof_test_output"
-sudo dd if=$nvme_drive of=$io_output_file iflag=direct count=1 bs=2k #read
-if ! cmp $io_input_file $io_output_file; then
-  echo "nvmeof initiator - io test failed!"
-  exit 1
-fi
-sudo rm -f $io_input_file $io_output_file
-echo "Test 3: basic IO - passed!"
-
 
-echo "nvmeof initiator tests passed!"
+echo "[nvmeof] Initiator setup done"
\ No newline at end of file

From 5999196f37bc5cb12de26d5f0aa077229e3ffc42 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Sun, 11 Feb 2024 22:50:05 +0100
Subject: [PATCH 1864/2492] cephadm: remove restriction for crush device
 classes

A restriction has been introduced here (https://github.com/ceph/ceph/commit/6c6cb2f5130dbcf8e42cf03666173948411fc92b) which doesn't let OSDs be created with custom crush device classes.
Crush Device Class is the key that helps the crush distinguish between multiple storage classes, so it must accept any custom names.

Fixes: https://tracker.ceph.com/issues/64382
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 .../ceph/deployment/translate.py              | 64 +++----------------
 .../ceph/tests/test_drive_group.py            | 16 +++--
 2 files changed, 21 insertions(+), 59 deletions(-)

diff --git a/src/python-common/ceph/deployment/translate.py b/src/python-common/ceph/deployment/translate.py
index 86243b8aefda..dd91b33e986b 100644
--- a/src/python-common/ceph/deployment/translate.py
+++ b/src/python-common/ceph/deployment/translate.py
@@ -13,9 +13,7 @@
 # TODO refactor this to a DriveSelection method
 class to_ceph_volume(object):
 
-    _supported_device_classes = [
-        "hdd", "ssd", "nvme"
-    ]
+    NO_CRUSH = '_NO_CRUSH'
 
     def __init__(self,
                  selection,  # type: DriveSelection
@@ -34,20 +32,6 @@ def prepare_devices(self):
 
         lvcount: Dict[str, List[str]] = dict()
 
-        """
-        Default entry for the global crush_device_class definition;
-        if there's no global definition at spec level, we do not want
-        to apply anything to the provided devices, hence we need to run
-        a ceph-volume command without that option, otherwise we init an
-        entry for the globally defined crush_device_class.
-        """
-        if self.spec.crush_device_class:
-            lvcount[self.spec.crush_device_class] = []
-
-        # entry where the drives that don't require a crush_device_class
-        # option are collected
-        lvcount["no_crush"] = []
-
         """
         for each device, check if it's just a path or it has a crush_device
         class definition, and append an entry to the right crush_device_
@@ -57,35 +41,16 @@ class group
             # iterate on List[Device], containing both path and
             # crush_device_class
             path = device.path
-            crush_device_class = device.crush_device_class
+            crush_device_class = (
+                device.crush_device_class
+                or self.spec.crush_device_class
+                or self.NO_CRUSH
+            )
 
             if path is None:
                 raise ValueError("Device path can't be empty")
 
-            """
-            if crush_device_class is specified for the current Device path
-            we should either init the array for this group or append the
-            drive path to the existing entry
-            """
-            if crush_device_class:
-                if crush_device_class in lvcount.keys():
-                    lvcount[crush_device_class].append(path)
-                else:
-                    lvcount[crush_device_class] = [path]
-                continue
-
-            """
-            if no crush_device_class is specified for the current path
-            but a global definition is present in the spec, so we group
-            the drives together
-            """
-            if crush_device_class is None and self.spec.crush_device_class:
-                lvcount[self.spec.crush_device_class].append(path)
-                continue
-            else:
-                # default use case
-                lvcount["no_crush"].append(path)
-                continue
+            lvcount.setdefault(crush_device_class, []).append(path)
 
         return lvcount
 
@@ -136,7 +101,7 @@ def run(self):
                         cmd += " --block.db {}".format(db_devices.pop())
                     if wal_devices:
                         cmd += " --block.wal {}".format(wal_devices.pop())
-                    if d in self._supported_device_classes:
+                    if d != self.NO_CRUSH:
                         cmd += " --crush-device-class {}".format(d)
 
                     cmds.append(cmd)
@@ -159,7 +124,7 @@ def run(self):
                 if self.spec.block_db_size:
                     cmd += " --block-db-size {}".format(self.spec.block_db_size)
 
-                if d in self._supported_device_classes:
+                if d != self.NO_CRUSH:
                     cmd += " --crush-device-class {}".format(d)
                 cmds.append(cmd)
 
@@ -180,17 +145,6 @@ def run(self):
                 cmds[i] += " --yes"
                 cmds[i] += " --no-systemd"
 
-            # set the --crush-device-class option when:
-            # - crush_device_class is specified at spec level (global for all the osds)  # noqa E501
-            # - crush_device_class is allowed
-            # - there's no override at osd level
-            if (
-                    self.spec.crush_device_class and
-                    self.spec.crush_device_class in self._supported_device_classes and  # noqa E501
-                    "crush-device-class" not in cmds[i]
-               ):
-                cmds[i] += " --crush-device-class {}".format(self.spec.crush_device_class)  # noqa E501
-
             if self.preview:
                 cmds[i] += " --report"
                 cmds[i] += " --format json"
diff --git a/src/python-common/ceph/tests/test_drive_group.py b/src/python-common/ceph/tests/test_drive_group.py
index 77e9b4083d49..cd4a238af0d5 100644
--- a/src/python-common/ceph/tests/test_drive_group.py
+++ b/src/python-common/ceph/tests/test_drive_group.py
@@ -392,8 +392,12 @@ def test_ceph_volume_command_12(test_input2):
     drive = drive_selection.DriveSelection(spec, spec.data_devices.paths)
     cmds = translate.to_ceph_volume(drive, []).run()
 
-    assert (cmds[0] == 'lvm batch --no-auto /dev/sdb --crush-device-class ssd --yes --no-systemd')  # noqa E501
-    assert (cmds[1] == 'lvm batch --no-auto /dev/sda --crush-device-class hdd --yes --no-systemd')  # noqa E501
+    expected_cmds = [
+        'lvm batch --no-auto /dev/sdb --crush-device-class ssd --yes --no-systemd',
+        'lvm batch --no-auto /dev/sda --crush-device-class hdd --yes --no-systemd',
+    ]
+    assert len(cmds) == len(expected_cmds), f"Expected {expected_cmds} got {cmds}"
+    assert all(cmd in cmds for cmd in expected_cmds), f'Expected {expected_cmds} got {cmds}'
 
 
 @pytest.mark.parametrize("test_input3",
@@ -418,8 +422,12 @@ def test_ceph_volume_command_13(test_input3):
     drive = drive_selection.DriveSelection(spec, spec.data_devices.paths)
     cmds = translate.to_ceph_volume(drive, []).run()
 
-    assert (cmds[0] == 'lvm batch --no-auto /dev/sdb --yes --no-systemd')  # noqa E501
-    assert (cmds[1] == 'lvm batch --no-auto /dev/sda --crush-device-class hdd --yes --no-systemd')  # noqa E501
+    expected_cmds = [
+        'lvm batch --no-auto /dev/sdb --yes --no-systemd',
+        'lvm batch --no-auto /dev/sda --crush-device-class hdd --yes --no-systemd',
+    ]
+    assert len(cmds) == len(expected_cmds), f"Expected {expected_cmds} got {cmds}"
+    assert all(cmd in cmds for cmd in expected_cmds), f'Expected {expected_cmds} got {cmds}'
 
 
 @pytest.mark.parametrize("test_input4",

From 7a9cc2dcce50fb1f4c04421b53c83030f3087f85 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 12 Feb 2024 09:24:13 +0000
Subject: [PATCH 1865/2492] osd/SnapMapper: Crimson - remove LINE log lines

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/SnapMapper.cc | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index e79bc5d450a2..40611f7a646a 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -95,7 +95,7 @@ int OSDriver::get_keys(
   const std::set<std::string> &keys,
   std::map<std::string, ceph::buffer::list> *out)
 {
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
+  CRIMSON_DEBUG("OSDriver::{}", __func__);
   using crimson::os::FuturizedStore;
   return interruptor::green_get(os->omap_get_values(
     ch, hoid, keys
@@ -107,54 +107,53 @@ int OSDriver::get_keys(
     assert(e.value() > 0);
     return -e.value();
   }))); // this requires seastar::thread
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
 }
 
 int OSDriver::get_next(
   const std::string &key,
   std::pair<std::string, ceph::buffer::list> *next)
 {
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
+  CRIMSON_DEBUG("OSDriver::{} key {}", __func__, key);
   using crimson::os::FuturizedStore;
   return interruptor::green_get(os->omap_get_values(
     ch, hoid, key
   ).safe_then_unpack([&key, next] (bool, FuturizedStore::Shard::omap_values_t&& vals) {
-    CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__);
+    CRIMSON_DEBUG("OSDriver::get_next key {} got omap values", key);
     if (auto nit = std::begin(vals); nit == std::end(vals)) {
-      CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__);
+      CRIMSON_DEBUG("OSDriver::get_next key {} no more values", key);
       return -ENOENT;
     } else {
-      CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__);
+      CRIMSON_DEBUG("OSDriver::get_next returning next: {}, ", nit->first);
       assert(nit->first > key);
       *next = *nit;
       return 0;
     }
   }, FuturizedStore::Shard::read_errorator::all_same_way([] {
-    CRIMSON_DEBUG("OSDriver::{}:{}", "get_next", __LINE__);
+    CRIMSON_DEBUG("OSDriver::get_next saw error returning EINVAL");
     return -EINVAL;
   }))); // this requires seastar::thread
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
 }
 
 int OSDriver::get_next_or_current(
   const std::string &key,
   std::pair<std::string, ceph::buffer::list> *next_or_current)
 {
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
+  CRIMSON_DEBUG("OSDriver::{} key {}", __func__, key);
   using crimson::os::FuturizedStore;
   // let's try to get current first
   return interruptor::green_get(os->omap_get_values(
     ch, hoid, FuturizedStore::Shard::omap_keys_t{key}
   ).safe_then([&key, next_or_current] (FuturizedStore::Shard::omap_values_t&& vals) {
+    CRIMSON_DEBUG("OSDriver::get_next_or_current returning {}", key);
     assert(vals.size() == 1);
     *next_or_current = std::make_pair(key, std::move(vals[0]));
     return 0;
   }, FuturizedStore::Shard::read_errorator::all_same_way(
     [next_or_current, &key, this] {
+    CRIMSON_DEBUG("OSDriver::get_next_or_current no current, try next {}", key);
     // no current, try next
     return get_next(key, next_or_current);
   }))); // this requires seastar::thread
-  CRIMSON_DEBUG("OSDriver::{}:{}", __func__, __LINE__);
 }
 #else
 int OSDriver::get_keys(

From d489704acdc3e6abaa337245de8ea312cefe6516 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 12 Feb 2024 09:32:46 +0000
Subject: [PATCH 1866/2492] osd/SnapMapper: is_mapping() to be public

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/SnapMapper.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index a0a4cd3221b9..70d1b6c39add 100644
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -256,8 +256,6 @@ class SnapMapper : public Scrub::SnapMapReaderI {
   std::pair<std::string, ceph::buffer::list> to_raw(
     const std::pair<snapid_t, hobject_t> &to_map) const;
 
-  static bool is_mapping(const std::string &to_test);
-
   static std::pair<snapid_t, hobject_t> from_raw(
     const std::pair<std::string, ceph::buffer::list> &image);
 
@@ -317,6 +315,8 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     return std::string(buf, r) + '_';
   }
 
+  static bool is_mapping(const std::string &to_test);
+
   uint32_t mask_bits;
   const uint32_t match;
   std::string last_key_checked;

From 8fd9b037f5802ec8dab4d69f11435e4217dff76a Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 12 Feb 2024 09:26:36 +0000
Subject: [PATCH 1867/2492] osd/SnapMapper: Crimson - fix OSDriver::get_next

There are other omap_keys that we may return which are not
SnapMapper entries (_biginfo, _epoch, _fastinfo etc.).

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/SnapMapper.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index 40611f7a646a..8f1be7052693 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -119,7 +119,8 @@ int OSDriver::get_next(
     ch, hoid, key
   ).safe_then_unpack([&key, next] (bool, FuturizedStore::Shard::omap_values_t&& vals) {
     CRIMSON_DEBUG("OSDriver::get_next key {} got omap values", key);
-    if (auto nit = std::begin(vals); nit == std::end(vals)) {
+    if (auto nit = std::begin(vals);
+        nit == std::end(vals) || !SnapMapper::is_mapping(nit->first)) {
       CRIMSON_DEBUG("OSDriver::get_next key {} no more values", key);
       return -ENOENT;
     } else {

From aa1e6352d0c2c77c299a16f93e16e83903fb9d54 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 13 Feb 2024 01:06:05 +1000
Subject: [PATCH 1868/2492] doc/glossary: Add link to CRUSH paper

Add a link in the CRUSH definition to the original CRUSH paper.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 12e40f59c98b..67cd9a4408b4 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -216,7 +216,9 @@
 	CRUSH
                 **C**\ontrolled **R**\eplication **U**\nder **S**\calable
                 **H**\ashing. The algorithm that Ceph uses to compute object
-                storage locations.
+                storage locations. See `CRUSH: Controlled, Scalable,
+                Decentralized Placement of Replicated Data
+                <https://ceph.com/assets/pdfs/weil-crush-sc06.pdf>`_.
 
 	CRUSH rule
                 The CRUSH data placement rule that applies to a particular

From 48e41aaf4f266567d014c9f80106b2b973a89124 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Thu, 8 Feb 2024 15:23:42 +0000
Subject: [PATCH 1869/2492] mgr/dashboard: added subvolume and translations

This commit adds support for subvolume snap scheduling and translations for snap schedule repeat schedules and retention policies

Fixes: https://tracker.ceph.com/issues/64372
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       | 33 ++++++++------
 ...ephfs-snapshotschedule-form.component.html |  1 +
 .../cephfs-snapshotschedule-form.component.ts | 43 +++++++++++++++++--
 ...ephfs-snapshotschedule-list.component.html | 30 ++++++++++---
 ...ephfs-snapshotschedule-list.component.scss |  4 ++
 .../cephfs-snapshotschedule-list.component.ts | 36 +++++++++++-----
 .../api/cephfs-snapshot-schedule.service.ts   | 41 ++++++++++++++----
 .../app/shared/enum/repeat-frequency.enum.ts  | 12 ++++++
 .../shared/enum/retention-frequency.enum.ts   |  9 ++++
 src/pybind/mgr/dashboard/openapi.yaml         | 26 +++++++++++
 10 files changed, 195 insertions(+), 40 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 83e9cbfd8533..ad2cafe0ba8e 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -969,13 +969,16 @@ def list(self, fs: str, path: str = '/', recursive: bool = True):
             )
         return json.loads(output_json)
 
-    def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_policy=None):
+    def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_policy=None,
+               subvol=None, group=None):
         error_code, _, err = mgr.remote('snap_schedule',
                                         'snap_schedule_add',
                                         path,
                                         snap_schedule,
                                         start,
-                                        fs)
+                                        fs,
+                                        subvol,
+                                        group)
 
         if retention_policy:
             retention_policies = retention_policy.split('|')
@@ -999,7 +1002,8 @@ def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_p
 
         return f'Snapshot schedule for path {path} created successfully'
 
-    def set(self, fs: str, path: str, retention_to_add=None, retention_to_remove=None):
+    def set(self, fs: str, path: str, retention_to_add=None, retention_to_remove=None,
+            subvol=None, group=None):
         def editRetentionPolicies(method, retention_policy):
             if not retention_policy:
                 return
@@ -1014,8 +1018,8 @@ def editRetentionPolicies(method, retention_policy):
                                                                     retention_spec_or_period,
                                                                     retention_count,
                                                                     fs,
-                                                                    None,
-                                                                    None)
+                                                                    subvol,
+                                                                    group)
                 if error_code_retention != 0:
                     raise DashboardException(
                         f'Failed to add/remove retention policy for path {path}: {err_retention}'
@@ -1027,15 +1031,16 @@ def editRetentionPolicies(method, retention_policy):
         return f'Retention policies for snapshot schedule on path {path} updated successfully'
 
     @RESTController.Resource('DELETE')
-    def delete_snapshot(self, fs: str, path: str, schedule: str, start: str):
+    def delete_snapshot(self, fs: str, path: str, schedule: str, start: str,
+                        subvol=None, group=None):
         error_code, _, err = mgr.remote('snap_schedule',
                                         'snap_schedule_rm',
                                         path,
                                         schedule,
                                         start,
                                         fs,
-                                        None,
-                                        None)
+                                        subvol,
+                                        group)
         if error_code != 0:
             raise DashboardException(
                 f'Failed to delete snapshot schedule for path {path}: {err}'
@@ -1044,15 +1049,15 @@ def delete_snapshot(self, fs: str, path: str, schedule: str, start: str):
         return f'Snapshot schedule for path {path} deleted successfully'
 
     @RESTController.Resource('POST')
-    def deactivate(self, fs: str, path: str, schedule: str, start: str):
+    def deactivate(self, fs: str, path: str, schedule: str, start: str, subvol=None, group=None):
         error_code, _, err = mgr.remote('snap_schedule',
                                         'snap_schedule_deactivate',
                                         path,
                                         schedule,
                                         start,
                                         fs,
-                                        None,
-                                        None)
+                                        subvol,
+                                        group)
         if error_code != 0:
             raise DashboardException(
                 f'Failed to deactivate snapshot schedule for path {path}: {err}'
@@ -1061,15 +1066,15 @@ def deactivate(self, fs: str, path: str, schedule: str, start: str):
         return f'Snapshot schedule for path {path} deactivated successfully'
 
     @RESTController.Resource('POST')
-    def activate(self, fs: str, path: str, schedule: str, start: str):
+    def activate(self, fs: str, path: str, schedule: str, start: str, subvol=None, group=None):
         error_code, _, err = mgr.remote('snap_schedule',
                                         'snap_schedule_activate',
                                         path,
                                         schedule,
                                         start,
                                         fs,
-                                        None,
-                                        None)
+                                        subvol,
+                                        group)
         if error_code != 0:
             raise DashboardException(
                 f'Failed to activate snapshot schedule for path {path}: {err}'
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
index e315e8ab7671..a67293e1138b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.html
@@ -36,6 +36,7 @@
                   i18n>A snapshot schedule for this path already exists.</span>
           </div>
         </div>
+
         <!--Start date -->
         <div class="form-group row">
           <label class="cd-col-form-label required"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 7459a7472c5f..d14d7debcce9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -5,6 +5,7 @@ import { uniq } from 'lodash';
 import { Observable, OperatorFunction, of, timer } from 'rxjs';
 import { catchError, debounceTime, distinctUntilChanged, map, switchMap } from 'rxjs/operators';
 import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
+import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
 import { DirectoryStoreService } from '~/app/shared/api/directory-store.service';
 import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants';
 import { Icons } from '~/app/shared/enum/icons.enum';
@@ -23,6 +24,7 @@ import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 
 const VALIDATON_TIMER = 300;
 const DEBOUNCE_TIMER = 300;
+const DEFAULT_SUBVOLUME_GROUP = '_nogroup';
 
 @Component({
   selector: 'cd-cephfs-snapshotschedule-form',
@@ -36,12 +38,18 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
   retention!: string;
   start!: string;
   status!: string;
+  subvol!: string;
+  group!: string;
   id!: number;
   isEdit = false;
   icons = Icons;
   repeatFrequencies = Object.entries(RepeatFrequency);
   retentionFrequencies = Object.entries(RetentionFrequency);
   retentionPoliciesToRemove: RetentionPolicy[] = [];
+  isDefaultSubvolumeGroup = false;
+  subvolumeGroup!: string;
+  subvolume!: string;
+  isSubvolume = false;
 
   currentTime!: NgbTimeStruct;
   minDate!: NgbDateStruct;
@@ -59,7 +67,8 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     private snapScheduleService: CephfsSnapshotScheduleService,
     private taskWrapper: TaskWrapperService,
     private cd: ChangeDetectorRef,
-    public directoryStore: DirectoryStoreService
+    public directoryStore: DirectoryStoreService,
+    private subvolumeService: CephfsSubvolumeService
   ) {
     super();
     this.resource = $localize`Snapshot schedule`;
@@ -82,6 +91,25 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     this.directoryStore.loadDirectories(this.id, '/', 3);
     this.createForm();
     this.isEdit ? this.populateForm() : this.loadingReady();
+    this.snapScheduleForm.get('directory').valueChanges.subscribe({
+      next: (value: string) => {
+        this.subvolumeGroup = value?.split?.('/')?.[2];
+        this.subvolume = value?.split?.('/')?.[3];
+        this.subvolumeService
+          .exists(
+            this.subvolume,
+            this.fsName,
+            this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP ? '' : this.subvolumeGroup
+          )
+          .subscribe({
+            next: (exists: boolean) => {
+              this.isSubvolume = exists;
+              this.isDefaultSubvolumeGroup =
+                exists && this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP;
+            }
+          });
+      }
+    });
   }
 
   get retentionPolicies() {
@@ -149,6 +177,7 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     this.snapScheduleForm = new CdFormGroup(
       {
         directory: new FormControl(undefined, {
+          updateOn: 'blur',
           validators: [Validators.required]
         }),
         startDate: new FormControl(this.minDate, {
@@ -234,6 +263,8 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
       const updateObj = {
         fs: this.fsName,
         path: this.path,
+        subvol: this.subvol,
+        group: this.group,
         retention_to_add: this.parseRetentionPolicies(retentionPoliciesToAdd) || null,
         retention_to_remove: this.parseRetentionPolicies(this.retentionPoliciesToRemove) || null
       };
@@ -262,9 +293,15 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
       };
 
       const retentionPoliciesValues = this.parseRetentionPolicies(values?.retentionPolicies);
-      if (retentionPoliciesValues) {
-        snapScheduleObj['retention_policy'] = retentionPoliciesValues;
+
+      if (retentionPoliciesValues) snapScheduleObj['retention_policy'] = retentionPoliciesValues;
+
+      if (this.isSubvolume) snapScheduleObj['subvol'] = this.subvolume;
+
+      if (this.isSubvolume && !this.isDefaultSubvolumeGroup) {
+        snapScheduleObj['group'] = this.subvolumeGroup;
       }
+
       this.taskWrapper
         .wrapTaskAroundCall({
           task: new FinishedTask('cephfs/snapshot/schedule/' + URLVerbs.CREATE, {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
index f2e93cc742c7..f26f63e755a6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
@@ -21,14 +21,14 @@
     class="fw-bold"
     [ngbTooltip]="fullpathTpl"
     triggers="click:blur">
-    {{ row.path | path }}
+    {{ row.path?.split?.("@")?.[0] | path }}
   </span>
 
   <span
   *ngIf="row.active; else inactiveStatusTpl">
     <i
       [ngClass]="[icons.success, icons.large]"
-      ngbTooltip="{{ row.path }} is active"
+      ngbTooltip="{{ row.path?.split?.('@')?.[0] }} is active"
       class="text-success"
     ></i>
   </span>
@@ -37,7 +37,7 @@
     <i
       [ngClass]="[icons.warning, icons.large]"
       class="text-warning"
-      ngbTooltip="{{ row.path }} has been deactivated"
+      ngbTooltip="{{ row.path?.split?.('@')?.[0] }} has been deactivated"
     ></i>
   </ng-template>
 
@@ -46,10 +46,10 @@
       data-toggle="tooltip"
       [title]="row.path"
       class="font-monospace"
-      >{{ row.path }}
+      >{{ row.path?.split?.("@")?.[0] }}
       <cd-copy-2-clipboard-button
         *ngIf="row.path"
-        [source]="row.path"
+        [source]="row.path?.split?.('@')?.[0]"
         [byId]="false"
         [showIconOnly]="true"
       >
@@ -58,6 +58,26 @@
   </ng-template>
 </ng-template>
 
+<ng-template
+  #retentionTpl
+  let-row="row">
+  <ul *ngIf="row.retentionCopy.length; else noDataTpl">
+    <li *ngFor="let ret of row.retentionCopy">{{ ret }}</li>
+  </ul>
+</ng-template>
+
+<ng-template
+  #subvolTpl
+  let-row="row">
+  <span *ngIf="row.subvol; else noDataTpl">
+    {{row.subvol}}
+  </span>
+</ng-template>
+
+
+
+<ng-template #noDataTpl>-</ng-template>
+
 <cd-table
   [data]="snapshotSchedules$ | async"
   *ngIf="snapScheduleModuleStatus$ | async"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss
index e69de29bb2d1..96b1c02c8985 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.scss
@@ -0,0 +1,4 @@
+ul {
+  list-style: none;
+  padding: 0;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index 58d3a0cc056a..581ee6e2fa3a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -46,6 +46,12 @@ export class CephfsSnapshotscheduleListComponent
   @ViewChild('pathTpl', { static: true })
   pathTpl: any;
 
+  @ViewChild('retentionTpl', { static: true })
+  retentionTpl: any;
+
+  @ViewChild('subvolTpl', { static: true })
+  subvolTpl: any;
+
   @BlockUI()
   blockUI: NgBlockUI;
 
@@ -122,7 +128,9 @@ export class CephfsSnapshotscheduleListComponent
             if (!status) {
               return of([]);
             }
-            return this.snapshotScheduleService.getSnapshotScheduleList('/', this.fsName);
+            return this.snapshotScheduleService
+              .getSnapshotScheduleList('/', this.fsName)
+              .pipe(map((list) => list.map((l) => ({ ...l, path: `${l.path}@${l.schedule}` }))));
           }),
           shareReplay(1)
         )
@@ -131,9 +139,11 @@ export class CephfsSnapshotscheduleListComponent
 
     this.columns = [
       { prop: 'path', name: $localize`Path`, flexGrow: 3, cellTemplate: this.pathTpl },
-      { prop: 'subvol', name: $localize`Subvolume` },
-      { prop: 'schedule', name: $localize`Repeat interval` },
-      { prop: 'retention', name: $localize`Retention policy` },
+      { prop: 'subvol', name: $localize`Subvolume`, cellTemplate: this.subvolTpl },
+      { prop: 'scheduleCopy', name: $localize`Repeat interval` },
+      { prop: 'schedule', isHidden: true },
+      { prop: 'retentionCopy', name: $localize`Retention policy`, cellTemplate: this.retentionTpl },
+      { prop: 'retention', isHidden: true },
       { prop: 'created_count', name: $localize`Created Count` },
       { prop: 'pruned_count', name: $localize`Deleted Count` },
       { prop: 'start', name: $localize`Start time`, cellTransformation: CellTemplate.timeAgo },
@@ -230,7 +240,7 @@ export class CephfsSnapshotscheduleListComponent
   }
 
   deactivateSnapshotSchedule() {
-    const { path, start, fs, schedule } = this.selection.first();
+    const { path, start, fs, schedule, subvol, group } = this.selection.first();
 
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
       itemDescription: $localize`snapshot schedule`,
@@ -244,14 +254,16 @@ export class CephfsSnapshotscheduleListComponent
             path,
             schedule,
             start,
-            fs
+            fs,
+            subvol,
+            group
           })
         })
     });
   }
 
   activateSnapshotSchedule() {
-    const { path, start, fs, schedule } = this.selection.first();
+    const { path, start, fs, schedule, subvol, group } = this.selection.first();
 
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
       itemDescription: $localize`snapshot schedule`,
@@ -265,14 +277,16 @@ export class CephfsSnapshotscheduleListComponent
             path,
             schedule,
             start,
-            fs
+            fs,
+            subvol,
+            group
           })
         })
     });
   }
 
   deleteSnapshotSchedule() {
-    const { path, start, fs, schedule } = this.selection.first();
+    const { path, start, fs, schedule, subvol, group } = this.selection.first();
 
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
       itemDescription: $localize`snapshot schedule`,
@@ -285,7 +299,9 @@ export class CephfsSnapshotscheduleListComponent
             path,
             schedule,
             start,
-            fs
+            fs,
+            subvol,
+            group
           })
         })
     });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
index 9e07f6057ac1..93c04dc38ed9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -5,7 +5,12 @@ import { catchError, map } from 'rxjs/operators';
 import { intersection, isEqual, uniqWith } from 'lodash';
 import { SnapshotSchedule } from '../models/snapshot-schedule';
 import { of } from 'rxjs';
-import { RepeatFrequency } from '../enum/repeat-frequency.enum';
+import {
+  RepeaFrequencyPlural,
+  RepeaFrequencySingular,
+  RepeatFrequency
+} from '../enum/repeat-frequency.enum';
+import { RetentionFrequencyCopy } from '../enum/retention-frequency.enum';
 
 @Injectable({
   providedIn: 'root'
@@ -43,12 +48,14 @@ export class CephfsSnapshotScheduleService {
     );
   }
 
-  delete({ fs, path, schedule, start }: Record<string, any>): Observable<any> {
-    return this.http.delete(
-      `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(
-        path
-      )}/delete_snapshot?schedule=${schedule}&start=${encodeURIComponent(start)}`
-    );
+  delete({ fs, path, schedule, start, subvol, group }: Record<string, any>): Observable<any> {
+    let deleteUrl = `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(
+      path
+    )}/delete_snapshot?schedule=${schedule}&start=${encodeURIComponent(start)}`;
+    if (subvol && group) {
+      deleteUrl += `&subvol=${encodeURIComponent(subvol)}&group=${encodeURIComponent(group)}`;
+    }
+    return this.http.delete(deleteUrl);
   }
 
   checkScheduleExists(
@@ -122,8 +129,10 @@ export class CephfsSnapshotScheduleService {
         uniqWith(
           snapList.map((snapItem: SnapshotSchedule) => ({
             ...snapItem,
+            scheduleCopy: this.parseScheduleCopy(snapItem.schedule),
             status: snapItem.active ? 'Active' : 'Inactive',
-            subvol: snapItem?.subvol || ' - ',
+            subvol: snapItem?.subvol,
+            retentionCopy: this.parseRetentionCopy(snapItem?.retention),
             retention: Object.values(snapItem?.retention || [])?.length
               ? Object.entries(snapItem.retention)
                   ?.map?.(([frequency, interval]) => `${interval}${frequency.toLocaleUpperCase()}`)
@@ -135,4 +144,20 @@ export class CephfsSnapshotScheduleService {
       )
     );
   }
+
+  parseScheduleCopy(schedule: string): string {
+    const scheduleArr = schedule.split('');
+    const interval = Number(scheduleArr.filter((x) => !isNaN(Number(x))).join(''));
+    const frequencyUnit = scheduleArr[scheduleArr.length - 1];
+    const frequency =
+      interval > 1 ? RepeaFrequencyPlural[frequencyUnit] : RepeaFrequencySingular[frequencyUnit];
+    return $localize`Every ${interval > 1 ? interval + ' ' : ''}${frequency}`;
+  }
+
+  parseRetentionCopy(retention: string | Record<string, number>): string[] {
+    if (!retention) return ['-'];
+    return Object.entries(retention).map(([frequency, interval]) =>
+      $localize`${interval} ${RetentionFrequencyCopy[frequency]}`.toLocaleLowerCase()
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
index db3563ed2b47..06fb1c3afc0c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
@@ -3,3 +3,15 @@ export enum RepeatFrequency {
   Daily = 'd',
   Weekly = 'w'
 }
+
+export enum RepeaFrequencySingular {
+  h = 'hour',
+  d = 'day',
+  w = 'week'
+}
+
+export enum RepeaFrequencyPlural {
+  h = 'hours',
+  d = 'days',
+  w = 'weeks'
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
index 44714dac9463..193418a1783c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
@@ -6,3 +6,12 @@ export enum RetentionFrequency {
   Yearly = 'y',
   'lastest snapshots' = 'n'
 }
+
+export enum RetentionFrequencyCopy {
+  h = 'Hourly',
+  d = 'Daily',
+  w = 'Weekly',
+  m = 'Monthly',
+  y = 'Yearly',
+  n = 'lastest snapshots'
+}
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 1fbaac399aaa..3f6e22c57c5e 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1768,6 +1768,8 @@ paths:
               properties:
                 fs:
                   type: string
+                group:
+                  type: string
                 path:
                   type: string
                 retention_policy:
@@ -1776,6 +1778,8 @@ paths:
                   type: string
                 start:
                   type: string
+                subvol:
+                  type: string
               required:
               - fs
               - path
@@ -1861,10 +1865,14 @@ paths:
           application/json:
             schema:
               properties:
+                group:
+                  type: string
                 retention_to_add:
                   type: string
                 retention_to_remove:
                   type: string
+                subvol:
+                  type: string
               type: object
       responses:
         '200':
@@ -1908,10 +1916,14 @@ paths:
           application/json:
             schema:
               properties:
+                group:
+                  type: string
                 schedule:
                   type: string
                 start:
                   type: string
+                subvol:
+                  type: string
               required:
               - schedule
               - start
@@ -1958,10 +1970,14 @@ paths:
           application/json:
             schema:
               properties:
+                group:
+                  type: string
                 schedule:
                   type: string
                 start:
                   type: string
+                subvol:
+                  type: string
               required:
               - schedule
               - start
@@ -2013,6 +2029,16 @@ paths:
         required: true
         schema:
           type: string
+      - allowEmptyValue: true
+        in: query
+        name: subvol
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: group
+        schema:
+          type: string
       responses:
         '202':
           content:

From b2fcc556a2973128e80f971fdc1126b6f255afc2 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Mon, 12 Feb 2024 17:40:06 +0000
Subject: [PATCH 1870/2492] fixup: don't store ref-to-on-stack-temporary in
 RecoveryBackend::coll

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/ECBackend.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 13d1317f272c..fa3b895b2761 100644
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -130,7 +130,7 @@ ECBackend::ECBackend(
   : PGBackend(cct, pg, store, coll, ch),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
-    recovery_backend(cct, coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this),
+    recovery_backend(cct, this->coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this),
     ec_impl(ec_impl),
     sinfo(ec_impl->get_data_chunk_count(), stripe_width),
     unstable_hashinfo_registry(cct, ec_impl) {

From a53c0651fbd8991e258ce7d0e3a80865dabfea88 Mon Sep 17 00:00:00 2001
From: Shinya Hayashi <shinya-hayashi@cybozu.co.jp>
Date: Tue, 30 Jan 2024 10:32:50 +0000
Subject: [PATCH 1871/2492] debian: add ceph-exporter package

It is hard for Debian/Ubuntu users to use ceph-exporter
because it is not included in any deb packages.

This commit adds a new deb package for ceph-exporter.

Fixes: https://tracker.ceph.com/issues/64095
Signed-off-by: Shinya Hayashi <shinya-hayashi@cybozu.co.jp>
---
 debian/ceph-exporter.install |  1 +
 debian/control               | 24 ++++++++++++++++++++++++
 debian/rules                 |  1 +
 3 files changed, 26 insertions(+)
 create mode 100644 debian/ceph-exporter.install

diff --git a/debian/ceph-exporter.install b/debian/ceph-exporter.install
new file mode 100644
index 000000000000..618cf9eadcc1
--- /dev/null
+++ b/debian/ceph-exporter.install
@@ -0,0 +1 @@
+usr/bin/ceph-exporter
diff --git a/debian/control b/debian/control
index f1bdec95a2ef..bb8d681929c3 100644
--- a/debian/control
+++ b/debian/control
@@ -351,6 +351,30 @@ Description: debugging symbols for ceph-mgr
  .
  This package contains the debugging symbols for ceph-mgr.
 
+Package: ceph-exporter
+Architecture: linux-any
+Depends: ceph-base (= ${binary:Version}),
+Description: metrics exporter for the ceph distributed storage system
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the metrics exporter daemon, which is used to expose
+ the performance metrics.
+
+Package: ceph-exporter-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: ceph-exporter (= ${binary:Version}),
+         ${misc:Depends},
+Description: debugging symbols for ceph-exporter
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.
+ .
+ This package contains the debugging symbols for ceph-exporter.
+
 Package: ceph-mon
 Architecture: linux-any
 Depends: ceph-base (= ${binary:Version}),
diff --git a/debian/rules b/debian/rules
index ed7f4a255ed4..68c69e044749 100755
--- a/debian/rules
+++ b/debian/rules
@@ -105,6 +105,7 @@ override_dh_strip:
 	dh_strip -pceph-mds --dbg-package=ceph-mds-dbg
 	dh_strip -pceph-fuse --dbg-package=ceph-fuse-dbg
 	dh_strip -pceph-mgr --dbg-package=ceph-mgr-dbg
+	dh_strip -pceph-exporter --dbg-package=ceph-exporter-dbg
 	dh_strip -pceph-mon --dbg-package=ceph-mon-dbg
 	dh_strip -pceph-osd --dbg-package=ceph-osd-dbg
 	dh_strip -pceph-base --dbg-package=ceph-base-dbg

From 7a31137ec70147b5f5e187e9ef98ccb13bf2e359 Mon Sep 17 00:00:00 2001
From: Pritha Srivastava <prsrivas@redhat.com>
Date: Mon, 12 Feb 2024 12:32:11 +0530
Subject: [PATCH 1872/2492] dashboard/rgw: changes to use 'full_user_id' from
 rgw user info as 'uid' in dashboard. 'full_user_id' includes tenant and user
 id, and also namespace wherever it is non empty.

Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
Reviewed-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py | 24 ++++++---------------
 src/pybind/mgr/dashboard/tests/test_rgw.py  |  2 ++
 2 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 661dc8afb6da..28e2754afcf9 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -500,21 +500,6 @@ def buckets_and_users_count(self, daemon_name=None):
 @APIRouter('/rgw/user', Scope.RGW)
 @APIDoc("RGW User Management API", "RgwUser")
 class RgwUser(RgwRESTController):
-    def _append_uid(self, user):
-        """
-        Append the user identifier that looks like [<tenant>$]<user>.
-        See http://docs.ceph.com/docs/jewel/radosgw/multitenancy/ for
-        more information.
-        :param user: The user parameters.
-        :type user: dict
-        :return: The modified user parameters including the 'uid' parameter.
-        :rtype: dict
-        """
-        if isinstance(user, dict):
-            user['uid'] = '{}${}'.format(user['tenant'], user['user_id']) \
-                if user['tenant'] else user['user_id']
-        return user
-
     @staticmethod
     def _keys_allowed():
         permissions = AuthManager.get_user(JwtManager.get_username()).permissions_dict()
@@ -550,7 +535,8 @@ def get(self, uid, daemon_name=None, stats=True) -> dict:
         if not self._keys_allowed():
             del result['keys']
             del result['swift_keys']
-        return self._append_uid(result)
+        result['uid'] = result['full_user_id']
+        return result
 
     @Endpoint()
     @ReadPermission
@@ -583,7 +569,8 @@ def create(self, uid, display_name, email=None, max_buckets=None,
         if secret_key is not None:
             params['secret-key'] = secret_key
         result = self.proxy(daemon_name, 'PUT', 'user', params)
-        return self._append_uid(result)
+        result['uid'] = result['full_user_id']
+        return result
 
     @allow_empty_body
     def set(self, uid, display_name=None, email=None, max_buckets=None,
@@ -598,7 +585,8 @@ def set(self, uid, display_name=None, email=None, max_buckets=None,
         if suspended is not None:
             params['suspended'] = suspended
         result = self.proxy(daemon_name, 'POST', 'user', params)
-        return self._append_uid(result)
+        result['uid'] = result['full_user_id']
+        return result
 
     def delete(self, uid, daemon_name=None):
         try:
diff --git a/src/pybind/mgr/dashboard/tests/test_rgw.py b/src/pybind/mgr/dashboard/tests/test_rgw.py
index d9b8593a7d76..b84838d10167 100644
--- a/src/pybind/mgr/dashboard/tests/test_rgw.py
+++ b/src/pybind/mgr/dashboard/tests/test_rgw.py
@@ -282,6 +282,7 @@ def test_user_get_with_keys(self, keys_allowed, mock_proxy):
         mock_proxy.return_value = {
             'tenant': '',
             'user_id': 'my_user_id',
+            'full_user_id': 'my_user_id',
             'keys': [],
             'swift_keys': []
         }
@@ -297,6 +298,7 @@ def test_user_get_without_keys(self, keys_allowed, mock_proxy):
         mock_proxy.return_value = {
             'tenant': '',
             'user_id': 'my_user_id',
+            'full_user_id': 'my_user_id',
             'keys': [],
             'swift_keys': []
         }

From 218f3632ca951b2524c7945ac37975b27b1c01f7 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:19:07 +0800
Subject: [PATCH 1873/2492] exporter: mark BlockTimer::get_ms() const

this function does not mutate the state of `BlockTimer`, so mark
it with `const` specifier, for better readability.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 2 +-
 src/exporter/util.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index 06a8338b8a50..9b5108390cb8 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -22,7 +22,7 @@ BlockTimer::~BlockTimer() {
 }
 
 // useful with stop
-double BlockTimer::get_ms() {
+double BlockTimer::get_ms() const {
 	return ms.count();
 }
 
diff --git a/src/exporter/util.h b/src/exporter/util.h
index 2628864d5158..243568b14d21 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -9,7 +9,7 @@ class BlockTimer {
 	BlockTimer(std::string file, std::string function);
 	~BlockTimer();
 	void stop();
-	double get_ms();
+	double get_ms() const;
  private:
 	std::chrono::duration<double, std::milli> ms;
 	std::string file, function;

From 20b1a197a841c29ccae587cb77d4ee16698236f4 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:21:11 +0800
Subject: [PATCH 1874/2492] exporter: drop unused function

string_is_digit() is not used, so drop it. `string_is_digit()` is
not defined or used, so drop it as well.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 8 --------
 src/exporter/util.h  | 2 --
 2 files changed, 10 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index 9b5108390cb8..e611c2a732e4 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -35,14 +35,6 @@ void BlockTimer::stop() {
 	}
 }
 
-bool string_is_digit(std::string s) {
-	size_t i = 0;
-	while (std::isdigit(s[i]) && i < s.size()) {
-		i++;
-	}
-	return i >= s.size();
-}
-
 std::string read_file_to_string(std::string path) {
 	std::ifstream is(path);
 	std::stringstream buffer;
diff --git a/src/exporter/util.h b/src/exporter/util.h
index 243568b14d21..66692c1a5b76 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -17,8 +17,6 @@ class BlockTimer {
 	std::chrono::time_point<std::chrono::high_resolution_clock> t1, t2;
 };
 
-bool string_is_digit(std::string s);
 std::string read_file_to_string(std::string path);
-std::string get_hostname(std::string path);
 
 void promethize(std::string &name);

From 74e985be15e0a742bbfece3ff94e9d92e6c70cd4 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:23:01 +0800
Subject: [PATCH 1875/2492] export: drop unused macro

TIMED_FUNCTION is not used, so drop it.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/exporter/util.h b/src/exporter/util.h
index 66692c1a5b76..c92b7ede1ee5 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -2,8 +2,6 @@
 #include <chrono>
 #include <string>
 
-#define TIMED_FUNCTION() BlockTimer timer(__FILE__, __FUNCTION__) 
-
 class BlockTimer {
  public:
 	BlockTimer(std::string file, std::string function);

From b68957db50d10e46b25f8bdcf8d782ec2aac1664 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:34:19 +0800
Subject: [PATCH 1876/2492] exporter: use std::chrono::high_resolution_clock
 for timing

we do not care about the wall time here, we are just timing the
duration of a certain period. so use std::chrono::steady_clock here.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 4 ++--
 src/exporter/util.h  | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index e611c2a732e4..d4453ba797bc 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -15,7 +15,7 @@
 
 BlockTimer::BlockTimer(std::string file, std::string function)
 	: file(file), function(function), stopped(false) {
-	t1 = std::chrono::high_resolution_clock::now();
+	t1 = clock_t::now();
 }
 BlockTimer::~BlockTimer() {
   dout(20) << file << ":" << function << ": " << ms.count() << "ms" << dendl;
@@ -30,7 +30,7 @@ double BlockTimer::get_ms() const {
 void BlockTimer::stop() {
 	if (!stopped) {
 		stopped = true;
-		t2 = std::chrono::high_resolution_clock::now();
+		t2 = clock_t::now();
 		ms = t2 - t1;
 	}
 }
diff --git a/src/exporter/util.h b/src/exporter/util.h
index c92b7ede1ee5..07598d9e8255 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -12,7 +12,9 @@ class BlockTimer {
 	std::chrono::duration<double, std::milli> ms;
 	std::string file, function;
 	bool stopped;
-	std::chrono::time_point<std::chrono::high_resolution_clock> t1, t2;
+	using clock_t = std::chrono::steady_clock;
+	clock_t::time_point t1;
+	clock_t::time_point t2;
 };
 
 std::string read_file_to_string(std::string path);

From 555f0f2ba30da214041a401989486bb9edd2e964 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:38:17 +0800
Subject: [PATCH 1877/2492] exporter: remove BlockTimer::ms

this member variable is redundant.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 6 +++---
 src/exporter/util.h  | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index d4453ba797bc..6def113a5d4d 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -18,12 +18,13 @@ BlockTimer::BlockTimer(std::string file, std::string function)
 	t1 = clock_t::now();
 }
 BlockTimer::~BlockTimer() {
-  dout(20) << file << ":" << function << ": " << ms.count() << "ms" << dendl;
+  dout(20) << file << ":" << function << ": " << get_ms() << "ms" << dendl;
 }
 
 // useful with stop
 double BlockTimer::get_ms() const {
-	return ms.count();
+	using milliseconds_t = std::chrono::duration<double, std::milli>;
+	return std::chrono::duration_cast<milliseconds_t>(t2 - t1).count();
 }
 
 // Manually stop the timer as you might want to get the time
@@ -31,7 +32,6 @@ void BlockTimer::stop() {
 	if (!stopped) {
 		stopped = true;
 		t2 = clock_t::now();
-		ms = t2 - t1;
 	}
 }
 
diff --git a/src/exporter/util.h b/src/exporter/util.h
index 07598d9e8255..2f4eaebb6152 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -9,7 +9,6 @@ class BlockTimer {
 	void stop();
 	double get_ms() const;
  private:
-	std::chrono::duration<double, std::milli> ms;
 	std::string file, function;
 	bool stopped;
 	using clock_t = std::chrono::steady_clock;

From 9de7f0fd7efdb92944335378de0589bcca9f15a8 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:41:00 +0800
Subject: [PATCH 1878/2492] exporter: use string_view for file and function
 name

according to
https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html,
both __FILE__ and __FUNCTION__ expand to C string constants. so no
need to store them as a std::string. let's use std::string_view instead.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 6 ++++--
 src/exporter/util.h  | 7 ++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index 6def113a5d4d..d6b4ba01ca76 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -13,8 +13,10 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_ceph_exporter
 
-BlockTimer::BlockTimer(std::string file, std::string function)
-	: file(file), function(function), stopped(false) {
+BlockTimer::BlockTimer(std::string_view file, std::string_view function)
+	: file(file),
+    function(function),
+    stopped(false) {
 	t1 = clock_t::now();
 }
 BlockTimer::~BlockTimer() {
diff --git a/src/exporter/util.h b/src/exporter/util.h
index 2f4eaebb6152..9adc128d2b4a 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -1,15 +1,16 @@
 #include "common/hostname.h"
 #include <chrono>
-#include <string>
+#include <string_view>
 
 class BlockTimer {
  public:
-	BlockTimer(std::string file, std::string function);
+	BlockTimer(std::string_view file, std::string_view function);
 	~BlockTimer();
 	void stop();
 	double get_ms() const;
  private:
-	std::string file, function;
+	const std::string_view file;
+	const std::string_view function;
 	bool stopped;
 	using clock_t = std::chrono::steady_clock;
 	clock_t::time_point t1;

From 850c2a1a5566d5db965998efe1a601ad4c6c7ef2 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 13 Feb 2024 14:43:23 +0800
Subject: [PATCH 1879/2492] exporter: initialize member variable in-class

for better readability.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/exporter/util.cc | 3 +--
 src/exporter/util.h  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/exporter/util.cc b/src/exporter/util.cc
index d6b4ba01ca76..451867be277c 100644
--- a/src/exporter/util.cc
+++ b/src/exporter/util.cc
@@ -15,8 +15,7 @@
 
 BlockTimer::BlockTimer(std::string_view file, std::string_view function)
 	: file(file),
-    function(function),
-    stopped(false) {
+    function(function) {
 	t1 = clock_t::now();
 }
 BlockTimer::~BlockTimer() {
diff --git a/src/exporter/util.h b/src/exporter/util.h
index 9adc128d2b4a..cd5448812e36 100644
--- a/src/exporter/util.h
+++ b/src/exporter/util.h
@@ -11,7 +11,7 @@ class BlockTimer {
  private:
 	const std::string_view file;
 	const std::string_view function;
-	bool stopped;
+	bool stopped = false;
 	using clock_t = std::chrono::steady_clock;
 	clock_t::time_point t1;
 	clock_t::time_point t2;

From dbdc161f7473b6445a46f2cd69efb09f99e8193d Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Tue, 22 Aug 2023 12:42:21 +0530
Subject: [PATCH 1880/2492] qa: add test cases for the support to reject clones
 feature

Fixes: https://tracker.ceph.com/issues/59714
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py         | 158 ++++++++++++++++++++++++
 qa/workunits/fs/full/subvolume_clone.sh |   3 +
 2 files changed, 161 insertions(+)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index d86af3412dcf..650ebcf26c8f 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -7000,6 +7000,11 @@ def test_subvolume_snapshot_clone_cancel_pending(self):
         # snapshot subvolume
         self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot)
 
+        # Disable the snapshot_clone_no_wait config option
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', False)
+        threads_available = self.config_get('mgr', 'mgr/volumes/snapshot_clone_no_wait')
+        self.assertEqual(threads_available, 'false')
+
         # schedule clones
         for clone in clones:
             self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone)
@@ -7485,6 +7490,159 @@ def test_subvolume_under_group_snapshot_clone(self):
         # verify trash dir is clean
         self._wait_for_trash_empty()
 
+    def test_subvolume_snapshot_clone_with_no_wait_enabled(self):
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1, clone2, clone3 = self._gen_subvol_clone_name(3)
+
+        # create subvolume
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
+
+        # do some IO
+        self._do_subvolume_io(subvolume, number_of_files=10)
+
+        # snapshot subvolume
+        self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot)
+
+        # Decrease number of cloner threads
+        self.config_set('mgr', 'mgr/volumes/max_concurrent_clones', 2)
+        max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones'))
+        self.assertEqual(max_concurrent_clones, 2)
+
+        # Enable the snapshot_clone_no_wait config option
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', True)
+        threads_available = self.config_get('mgr', 'mgr/volumes/snapshot_clone_no_wait')
+        self.assertEqual(threads_available, 'true')
+
+        # Insert delay of 15 seconds at the beginning of the snapshot clone
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 15)
+
+        # schedule a clone1
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone1)
+
+        # schedule a clone2
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone2)
+
+        # schedule a clone3
+        cmd_ret = self.run_ceph_cmd(
+            args=["fs", "subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone3], check_status=False, stdout=StringIO(),
+            stderr=StringIO())
+        self.assertEqual(cmd_ret.returncode, errno.EAGAIN, "Expecting EAGAIN error")
+
+        # check clone1 status
+        self._wait_for_clone_to_complete(clone1)
+
+        # verify clone1
+        self._verify_clone(subvolume, snapshot, clone1)
+
+        # check clone2 status
+        self._wait_for_clone_to_complete(clone2)
+
+        # verify clone2
+        self._verify_clone(subvolume, snapshot, clone2)
+
+        # schedule clone3 , it should be successful this time
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone3)
+
+        # check clone3 status
+        self._wait_for_clone_to_complete(clone3)
+
+        # verify clone3
+        self._verify_clone(subvolume, snapshot, clone3)
+
+        # set number of cloner threads to default
+        self.config_set('mgr', 'mgr/volumes/max_concurrent_clones', 4)
+        max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones'))
+        self.assertEqual(max_concurrent_clones, 4)
+
+        # set the snapshot_clone_delay to default
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 0)
+
+        # remove snapshot
+        self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot)
+
+        # remove subvolumes
+        self._fs_cmd("subvolume", "rm", self.volname, subvolume)
+        self._fs_cmd("subvolume", "rm", self.volname, clone1)
+        self._fs_cmd("subvolume", "rm", self.volname, clone2)
+        self._fs_cmd("subvolume", "rm", self.volname, clone3)
+
+        # verify trash dir is clean
+        self._wait_for_trash_empty()
+
+    def test_subvolume_snapshot_clone_with_no_wait_not_enabled(self):
+        subvolume = self._gen_subvol_name()
+        snapshot = self._gen_subvol_snap_name()
+        clone1, clone2, clone3 = self._gen_subvol_clone_name(3)
+
+        # create subvolume
+        self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
+
+        # do some IO
+        self._do_subvolume_io(subvolume, number_of_files=10)
+
+        # snapshot subvolume
+        self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot)
+
+        # Disable the snapshot_clone_no_wait config option
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', False)
+        threads_available = self.config_get('mgr', 'mgr/volumes/snapshot_clone_no_wait')
+        self.assertEqual(threads_available, 'false')
+
+        # Decrease number of cloner threads
+        self.config_set('mgr', 'mgr/volumes/max_concurrent_clones', 2)
+        max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones'))
+        self.assertEqual(max_concurrent_clones, 2)
+
+        # schedule a clone1
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone1)
+
+        # schedule a clone2
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone2)
+
+        # schedule a clone3
+        self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone3)
+
+        # check clone1 status
+        self._wait_for_clone_to_complete(clone1)
+
+        # verify clone1
+        self._verify_clone(subvolume, snapshot, clone1)
+
+        # check clone2 status
+        self._wait_for_clone_to_complete(clone2)
+
+        # verify clone2
+        self._verify_clone(subvolume, snapshot, clone2)
+
+        # check clone3 status
+        self._wait_for_clone_to_complete(clone3)
+
+        # verify clone3
+        self._verify_clone(subvolume, snapshot, clone3)
+
+        # set the snapshot_clone_no_wait config option to default
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', True)
+        threads_available = self.config_get('mgr', 'mgr/volumes/snapshot_clone_no_wait')
+        self.assertEqual(threads_available, 'true')
+
+        # set number of cloner threads to default
+        self.config_set('mgr', 'mgr/volumes/max_concurrent_clones', 4)
+        max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones'))
+        self.assertEqual(max_concurrent_clones, 4)
+
+        # remove snapshot
+        self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot)
+
+        # remove subvolumes
+        self._fs_cmd("subvolume", "rm", self.volname, subvolume)
+        self._fs_cmd("subvolume", "rm", self.volname, clone1)
+        self._fs_cmd("subvolume", "rm", self.volname, clone2)
+        self._fs_cmd("subvolume", "rm", self.volname, clone3)
+
+        # verify trash dir is clean
+        self._wait_for_trash_empty()
+
 
 class TestMisc(TestVolumesHelper):
     """Miscellaneous tests related to FS volume, subvolume group, and subvolume operations."""
diff --git a/qa/workunits/fs/full/subvolume_clone.sh b/qa/workunits/fs/full/subvolume_clone.sh
index 75648f306f8b..4fe60be84838 100755
--- a/qa/workunits/fs/full/subvolume_clone.sh
+++ b/qa/workunits/fs/full/subvolume_clone.sh
@@ -59,6 +59,9 @@ ceph fs subvolume snapshot create cephfs sub_0 snap_0
 # Set clone snapshot delay
 ceph config set mgr mgr/volumes/snapshot_clone_delay 15
 
+# Disable the snapshot_clone_no_wait config option
+ceph config set mgr mgr/volumes/snapshot_clone_no_wait false
+
 # Schedule few clones, some would fail with no space
 for i in $(eval echo {1..$NUM_CLONES});do ceph fs subvolume snapshot clone cephfs sub_0 snap_0 clone_$i;done
 

From 6a44322d59e5542731dd450b383046a13bc28a66 Mon Sep 17 00:00:00 2001
From: neeraj pratap singh
 <neerajpratapsingh@li-ff7f0d4c-3462-11b2-a85c-d4004c0fa1a0.ibm.com>
Date: Tue, 22 Aug 2023 13:32:58 +0530
Subject: [PATCH 1881/2492] doc: add the reject the clone when threads are not
 available feature in the document

Fixes: https://tracker.ceph.com/issues/59714
Signed-off-by: Neeraj Pratap Singh <neesingh@redhat.com>
---
 PendingReleaseNotes       |  9 +++++++++
 doc/cephfs/fs-volumes.rst | 31 +++++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 0ecec83ec81f..54ac2d7e192c 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -131,6 +131,15 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   ``rgw_topic_require_publish_policy`` can be enabled to deny ``sns:Publish``
   permissions unless explicitly granted by topic policy.
 
+* CephFS: The `subvolume snapshot clone` command now depends on the config option
+  `snapshot_clone_no_wait` which is used to reject the clone operation when
+  all the cloner threads are busy. This config option is enabled by default which means 
+  that if no cloner threads are free, the clone request errors out with EAGAIN.
+  The value of the config option can be fetched by using:
+   `ceph config get mgr mgr/volumes/snapshot_clone_no_wait`
+  and it can be disabled by using:
+   `ceph config set mgr mgr/volumes/snapshot_clone_no_wait false`
+
 >=18.0.0
 
 * The RGW policy parser now rejects unknown principals by default. If you are
diff --git a/doc/cephfs/fs-volumes.rst b/doc/cephfs/fs-volumes.rst
index 1e7adf3a03d4..dc9be8379aa3 100644
--- a/doc/cephfs/fs-volumes.rst
+++ b/doc/cephfs/fs-volumes.rst
@@ -579,6 +579,8 @@ To initiate a clone operation use:
 
    ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name>
 
+.. note:: ``subvolume snapshot clone`` command depends upon the above mentioned config option ``snapshot_clone_no_wait``
+
 If a snapshot (source subvolume) is a part of non-default group, the group name needs to be specified:
 
 .. prompt:: bash #
@@ -597,12 +599,6 @@ Similar to specifying a pool layout when creating a subvolume, pool layout can b
 
    ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --pool_layout <pool_layout>
 
-Configure the maximum number of concurrent clones. The default is 4:
-
-.. prompt:: bash #
-
-   ceph config set mgr mgr/volumes/max_concurrent_clones <value>
-
 To check the status of a clone operation use:
 
 .. prompt:: bash #
@@ -728,6 +724,29 @@ On successful cancellation, the cloned subvolume is moved to the ``canceled`` st
 
 .. note:: The canceled cloned may be deleted by supplying the ``--force`` option to the `fs subvolume rm` command.
 
+Configurables
+~~~~~~~~~~~~~
+
+Configure the maximum number of concurrent clone operations. The default is 4:
+
+.. prompt:: bash #
+
+   ceph config set mgr mgr/volumes/max_concurrent_clones <value>
+
+Configure the snapshot_clone_no_wait option :
+
+.. prompt:: bash #
+
+   ``snapshot_clone_no_wait`` config option is used to reject the clone creation request when the cloner threads 
+   ( which can be configured using above option i.e. ``max_concurrent_clones``) are not available.
+   It is enabled by default i.e. the value set is True, whereas it can be configured by using below command.
+
+   ceph config set mgr mgr/volumes/snapshot_clone_no_wait <bool>
+
+   The current value of ``snapshot_clone_no_wait`` can be fetched by using below command.
+
+   ceph config get mgr mgr/volumes/snapshot_clone_no_wait
+
 
 .. _subvol-pinning:
 

From c3251fbf873b9781287ae937db614f115b7674ac Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Wed, 1 Nov 2023 16:06:49 +0100
Subject: [PATCH 1882/2492] mgr/dashboard: add ACLs

Fixes: https://tracker.ceph.com/issues/63547
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py   |    21 +-
 .../mgr/dashboard/frontend/package-lock.json  | 31799 +++++++++-------
 .../mgr/dashboard/frontend/package.json       |     4 +
 .../rgw-bucket-details.component.html         |    28 +
 .../rgw-bucket-details.component.ts           |    41 +-
 .../rgw-bucket-acl-permissions.enum.ts        |    14 +
 .../rgw-bucket-form.component.html            |    51 +
 .../rgw-bucket-form.component.ts              |   104 +-
 .../app/shared/api/rgw-bucket.service.spec.ts |    10 +-
 .../src/app/shared/api/rgw-bucket.service.ts  |    10 +-
 .../mgr/dashboard/frontend/tsconfig.json      |     8 +-
 src/pybind/mgr/dashboard/openapi.yaml         |     4 +
 .../mgr/dashboard/services/rgw_client.py      |    24 +
 13 files changed, 18149 insertions(+), 13969 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-acl-permissions.enum.ts

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index e7e0da181a6f..d0065dabb53b 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -304,6 +304,14 @@ def _set_tags(self, bucket_name, tags, daemon_name, owner):
         rgw_client = RgwClient.instance(owner, daemon_name)
         return rgw_client.set_tags(bucket_name, tags)
 
+    def _get_acl(self, bucket_name, daemon_name, owner):
+        rgw_client = RgwClient.instance(owner, daemon_name)
+        return str(rgw_client.get_acl(bucket_name))
+
+    def _set_acl(self, bucket_name: str, acl: str, owner, daemon_name):
+        rgw_client = RgwClient.instance(owner, daemon_name)
+        return rgw_client.set_acl(bucket_name, acl)
+
     @staticmethod
     def strip_tenant_from_bucket_name(bucket_name):
         # type (str) -> str
@@ -357,6 +365,7 @@ def get(self, bucket, daemon_name=None):
         result['versioning'] = versioning['Status']
         result['mfa_delete'] = versioning['MfaDelete']
         result['bucket_policy'] = self._get_policy(bucket_name)
+        result['acl'] = self._get_acl(bucket_name, daemon_name, result['owner'])
 
         # Append the locking configuration.
         locking = self._get_locking(result['owner'], daemon_name, bucket_name)
@@ -370,7 +379,7 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
                lock_retention_period_days=None,
                lock_retention_period_years=None, encryption_state='false',
                encryption_type=None, key_id=None, tags=None,
-               bucket_policy=None, daemon_name=None):
+               bucket_policy=None, canned_acl=None, daemon_name=None):
         lock_enabled = str_to_bool(lock_enabled)
         encryption_state = str_to_bool(encryption_state)
         try:
@@ -392,6 +401,9 @@ def create(self, bucket, uid, zonegroup=None, placement_target=None,
             if bucket_policy:
                 self._set_policy(bucket, bucket_policy, daemon_name, uid)
 
+            if canned_acl:
+                self._set_acl(bucket, canned_acl, uid, daemon_name)
+
             return result
         except RequestException as e:  # pragma: no cover - handling is too obvious
             raise DashboardException(e, http_status_code=500, component='rgw')
@@ -401,7 +413,8 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
             encryption_state='false', encryption_type=None, key_id=None,
             mfa_delete=None, mfa_token_serial=None, mfa_token_pin=None,
             lock_mode=None, lock_retention_period_days=None,
-            lock_retention_period_years=None, tags=None, bucket_policy=None, daemon_name=None):
+            lock_retention_period_years=None, tags=None, bucket_policy=None,
+            canned_acl=None, daemon_name=None):
         encryption_state = str_to_bool(encryption_state)
         # When linking a non-tenant-user owned bucket to a tenanted user, we
         # need to prefix bucket name with '/'. e.g. photos -> /photos
@@ -444,7 +457,9 @@ def set(self, bucket, bucket_id, uid, versioning_state=None,
         if tags:
             self._set_tags(bucket_name, tags, daemon_name, uid)
         if bucket_policy:
-            self._set_policy(bucket, bucket_policy, daemon_name, uid)
+            self._set_policy(bucket_name, bucket_policy, daemon_name, uid)
+        if canned_acl:
+            self._set_acl(bucket_name, canned_acl, uid, daemon_name)
         return self._append_bid(result)
 
     def delete(self, bucket, purge_objects='true', daemon_name=None):
diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json
index ac7de1018365..8e6a9c4018fd 100644
--- a/src/pybind/mgr/dashboard/frontend/package-lock.json
+++ b/src/pybind/mgr/dashboard/frontend/package-lock.json
@@ -42,8 +42,11 @@
         "ngx-toastr": "17.0.2",
         "rxjs": "6.6.3",
         "simplebar-angular": "2.3.6",
+        "stream-browserify": "3.0.0",
         "swagger-ui": "4.12.0",
+        "timers-browserify": "2.0.12",
         "tslib": "2.3.1",
+        "xml2js": "0.6.2",
         "zone.js": "0.11.8"
       },
       "devDependencies": {
@@ -65,6 +68,7 @@
         "@types/lodash": "4.14.161",
         "@types/node": "18.17.12",
         "@types/swagger-ui": "3.52.0",
+        "@types/xml2js": "0.4.14",
         "@typescript-eslint/eslint-plugin": "5.27.1",
         "@typescript-eslint/parser": "5.27.1",
         "axe-core": "4.4.3",
@@ -115,11 +119,11 @@
       "dev": true
     },
     "node_modules/@ampproject/remapping": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
-      "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz",
+      "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==",
       "dependencies": {
-        "@jridgewell/gen-mapping": "^0.1.0",
+        "@jridgewell/gen-mapping": "^0.3.0",
         "@jridgewell/trace-mapping": "^0.3.9"
       },
       "engines": {
@@ -269,6 +273,221 @@
         }
       }
     },
+    "node_modules/@angular-devkit/build-angular/node_modules/@ampproject/remapping": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
+      "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==",
+      "dev": true,
+      "dependencies": {
+        "@jridgewell/gen-mapping": "^0.1.0",
+        "@jridgewell/trace-mapping": "^0.3.9"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/core": {
+      "version": "7.20.12",
+      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
+      "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
+      "dev": true,
+      "dependencies": {
+        "@ampproject/remapping": "^2.1.0",
+        "@babel/code-frame": "^7.18.6",
+        "@babel/generator": "^7.20.7",
+        "@babel/helper-compilation-targets": "^7.20.7",
+        "@babel/helper-module-transforms": "^7.20.11",
+        "@babel/helpers": "^7.20.7",
+        "@babel/parser": "^7.20.7",
+        "@babel/template": "^7.20.7",
+        "@babel/traverse": "^7.20.12",
+        "@babel/types": "^7.20.7",
+        "convert-source-map": "^1.7.0",
+        "debug": "^4.1.0",
+        "gensync": "^1.0.0-beta.2",
+        "json5": "^2.2.2",
+        "semver": "^6.3.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/babel"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/core/node_modules/semver": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/generator": {
+      "version": "7.20.14",
+      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.20.14.tgz",
+      "integrity": "sha512-AEmuXHdcD3A52HHXxaTmYlb8q/xMEhoRP67B3T4Oq7lbmSoqroMZzjnGj3+i1io3pdnF8iBYVu4Ilj+c4hBxYg==",
+      "dev": true,
+      "dependencies": {
+        "@babel/types": "^7.20.7",
+        "@jridgewell/gen-mapping": "^0.3.2",
+        "jsesc": "^2.5.1"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
+      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+      "dev": true,
+      "dependencies": {
+        "@jridgewell/set-array": "^1.0.1",
+        "@jridgewell/sourcemap-codec": "^1.4.10",
+        "@jridgewell/trace-mapping": "^0.3.9"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/helper-split-export-declaration": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz",
+      "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==",
+      "dev": true,
+      "dependencies": {
+        "@babel/types": "^7.18.6"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/runtime": {
+      "version": "7.20.13",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.13.tgz",
+      "integrity": "sha512-gt3PKXs0DBoL9xCvOIIZ2NEqAGZqHjAnmVbfQtB620V0uReIQutpel14KcneZuer7UioY8ALKZ7iocavvzTNFA==",
+      "dev": true,
+      "dependencies": {
+        "regenerator-runtime": "^0.13.11"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@babel/template": {
+      "version": "7.20.7",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.20.7.tgz",
+      "integrity": "sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==",
+      "dev": true,
+      "dependencies": {
+        "@babel/code-frame": "^7.18.6",
+        "@babel/parser": "^7.20.7",
+        "@babel/types": "^7.20.7"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/@jridgewell/gen-mapping": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz",
+      "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==",
+      "dev": true,
+      "dependencies": {
+        "@jridgewell/set-array": "^1.0.0",
+        "@jridgewell/sourcemap-codec": "^1.4.10"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/browserslist": {
+      "version": "4.21.5",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz",
+      "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/browserslist"
+        }
+      ],
+      "dependencies": {
+        "caniuse-lite": "^1.0.30001449",
+        "electron-to-chromium": "^1.4.284",
+        "node-releases": "^2.0.8",
+        "update-browserslist-db": "^1.0.10"
+      },
+      "bin": {
+        "browserslist": "cli.js"
+      },
+      "engines": {
+        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/esbuild": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.8.tgz",
+      "integrity": "sha512-g24ybC3fWhZddZK6R3uD2iF/RIPnRpwJAqLov6ouX3hMbY4+tKolP0VMF3zuIYCaXun+yHwS5IPQ91N2BT191g==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "bin": {
+        "esbuild": "bin/esbuild"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "optionalDependencies": {
+        "@esbuild/android-arm": "0.17.8",
+        "@esbuild/android-arm64": "0.17.8",
+        "@esbuild/android-x64": "0.17.8",
+        "@esbuild/darwin-arm64": "0.17.8",
+        "@esbuild/darwin-x64": "0.17.8",
+        "@esbuild/freebsd-arm64": "0.17.8",
+        "@esbuild/freebsd-x64": "0.17.8",
+        "@esbuild/linux-arm": "0.17.8",
+        "@esbuild/linux-arm64": "0.17.8",
+        "@esbuild/linux-ia32": "0.17.8",
+        "@esbuild/linux-loong64": "0.17.8",
+        "@esbuild/linux-mips64el": "0.17.8",
+        "@esbuild/linux-ppc64": "0.17.8",
+        "@esbuild/linux-riscv64": "0.17.8",
+        "@esbuild/linux-s390x": "0.17.8",
+        "@esbuild/linux-x64": "0.17.8",
+        "@esbuild/netbsd-x64": "0.17.8",
+        "@esbuild/openbsd-x64": "0.17.8",
+        "@esbuild/sunos-x64": "0.17.8",
+        "@esbuild/win32-arm64": "0.17.8",
+        "@esbuild/win32-ia32": "0.17.8",
+        "@esbuild/win32-x64": "0.17.8"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@angular-devkit/build-angular/node_modules/regenerator-runtime": {
+      "version": "0.13.11",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
+      "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==",
+      "dev": true
+    },
     "node_modules/@angular-devkit/build-angular/node_modules/rxjs": {
       "version": "6.6.7",
       "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz",
@@ -287,12 +506,33 @@
       "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
       "dev": true
     },
+    "node_modules/@angular-devkit/build-angular/node_modules/semver": {
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/@angular-devkit/build-angular/node_modules/tslib": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==",
       "dev": true
     },
+    "node_modules/@angular-devkit/build-angular/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/@angular-devkit/build-webpack": {
       "version": "0.1502.9",
       "resolved": "https://registry.npmjs.org/@angular-devkit/build-webpack/-/build-webpack-0.1502.9.tgz",
@@ -475,6 +715,39 @@
         "@angular/cli": ">= 13.0.0 < 14.0.0"
       }
     },
+    "node_modules/@angular-eslint/schematics/node_modules/ignore": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz",
+      "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 4"
+      }
+    },
+    "node_modules/@angular-eslint/schematics/node_modules/strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/@angular-eslint/schematics/node_modules/tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "dev": true,
+      "dependencies": {
+        "rimraf": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.17.0"
+      }
+    },
     "node_modules/@angular-eslint/template-parser": {
       "version": "13.5.0",
       "resolved": "https://registry.npmjs.org/@angular-eslint/template-parser/-/template-parser-13.5.0.tgz",
@@ -551,6 +824,83 @@
         "yarn": ">= 1.13.0"
       }
     },
+    "node_modules/@angular/cli/node_modules/ini": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-3.0.1.tgz",
+      "integrity": "sha512-it4HyVAUTKBc6m8e1iXWvXSTdndF7HbdN713+kvLrymxTaU4AUBWrJ4vEooP+V7fexnVD3LKcBshjGGPefSMUQ==",
+      "dev": true,
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/@angular/cli/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@angular/cli/node_modules/resolve": {
+      "version": "1.22.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz",
+      "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==",
+      "dev": true,
+      "dependencies": {
+        "is-core-module": "^2.9.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/@angular/cli/node_modules/semver": {
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@angular/cli/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@angular/cli/node_modules/yargs": {
+      "version": "17.6.2",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz",
+      "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==",
+      "dev": true,
+      "dependencies": {
+        "cliui": "^8.0.1",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "require-directory": "^2.1.1",
+        "string-width": "^4.2.3",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^21.1.1"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/@angular/common": {
       "version": "15.2.9",
       "resolved": "https://registry.npmjs.org/@angular/common/-/common-15.2.9.tgz",
@@ -615,43 +965,16 @@
         "typescript": ">=4.8.2 <5.0"
       }
     },
-    "node_modules/@angular/compiler-cli/node_modules/@babel/core": {
-      "version": "7.19.3",
-      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.19.3.tgz",
-      "integrity": "sha512-WneDJxdsjEvyKtXKsaBGbDeiyOjR5vYq4HcShxnIbG0qixpoHjI3MqeZM9NDvsojNCEBItQE4juOo/bU6e72gQ==",
+    "node_modules/@angular/compiler-cli/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
       "dependencies": {
-        "@ampproject/remapping": "^2.1.0",
-        "@babel/code-frame": "^7.18.6",
-        "@babel/generator": "^7.19.3",
-        "@babel/helper-compilation-targets": "^7.19.3",
-        "@babel/helper-module-transforms": "^7.19.0",
-        "@babel/helpers": "^7.19.0",
-        "@babel/parser": "^7.19.3",
-        "@babel/template": "^7.18.10",
-        "@babel/traverse": "^7.19.3",
-        "@babel/types": "^7.19.3",
-        "convert-source-map": "^1.7.0",
-        "debug": "^4.1.0",
-        "gensync": "^1.0.0-beta.2",
-        "json5": "^2.2.1",
-        "semver": "^6.3.0"
+        "yallist": "^4.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/babel"
-      }
-    },
-    "node_modules/@angular/compiler-cli/node_modules/@babel/core/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+        "node": ">=10"
       }
     },
     "node_modules/@angular/compiler-cli/node_modules/magic-string": {
@@ -666,6 +989,27 @@
         "node": ">=12"
       }
     },
+    "node_modules/@angular/compiler-cli/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@angular/compiler-cli/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/@angular/core": {
       "version": "15.2.9",
       "resolved": "https://registry.npmjs.org/@angular/core/-/core-15.2.9.tgz",
@@ -729,43 +1073,6 @@
         "@angular/compiler-cli": "15.2.9"
       }
     },
-    "node_modules/@angular/localize/node_modules/@babel/core": {
-      "version": "7.19.3",
-      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.19.3.tgz",
-      "integrity": "sha512-WneDJxdsjEvyKtXKsaBGbDeiyOjR5vYq4HcShxnIbG0qixpoHjI3MqeZM9NDvsojNCEBItQE4juOo/bU6e72gQ==",
-      "dependencies": {
-        "@ampproject/remapping": "^2.1.0",
-        "@babel/code-frame": "^7.18.6",
-        "@babel/generator": "^7.19.3",
-        "@babel/helper-compilation-targets": "^7.19.3",
-        "@babel/helper-module-transforms": "^7.19.0",
-        "@babel/helpers": "^7.19.0",
-        "@babel/parser": "^7.19.3",
-        "@babel/template": "^7.18.10",
-        "@babel/traverse": "^7.19.3",
-        "@babel/types": "^7.19.3",
-        "convert-source-map": "^1.7.0",
-        "debug": "^4.1.0",
-        "gensync": "^1.0.0-beta.2",
-        "json5": "^2.2.1",
-        "semver": "^6.3.0"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/babel"
-      }
-    },
-    "node_modules/@angular/localize/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
     "node_modules/@angular/platform-browser": {
       "version": "15.2.9",
       "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-15.2.9.tgz",
@@ -903,908 +1210,1026 @@
         "node": ">=8.0.0"
       }
     },
-    "node_modules/@applitools/eyes-sdk-core": {
-      "version": "12.23.12",
-      "resolved": "https://registry.npmjs.org/@applitools/eyes-sdk-core/-/eyes-sdk-core-12.23.12.tgz",
-      "integrity": "sha512-nVDWYe1VvUpCQrvjUgDASIDDgHMtIjxXrI52Tvdov1ya/jhyNn6vMYHW9ZgSmjwdrYAoq+p7B90qIdy/HEV5DA==",
+    "node_modules/@applitools/eyes-cypress/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@applitools/dom-capture": "11.0.1",
-        "@applitools/dom-snapshot": "4.5.8",
-        "@applitools/driver": "1.2.4",
-        "@applitools/isomorphic-fetch": "3.0.0",
-        "@applitools/logger": "1.0.4",
-        "@applitools/screenshoter": "3.2.4",
-        "@applitools/snippets": "2.1.7",
-        "@applitools/types": "1.0.14",
-        "@applitools/utils": "1.2.3",
-        "axios": "0.21.4",
-        "chalk": "3.0.0",
-        "cosmiconfig": "6.0.0",
-        "dateformat": "3.0.3",
-        "debug": "4.2.0",
-        "deepmerge": "4.2.2",
-        "stack-trace": "0.0.10",
-        "tunnel": "0.0.6"
-      },
-      "bin": {
-        "eyes-check-network": "bin/runCheckNetwork.js"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">= 8.9.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@applitools/functional-commons": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/@applitools/functional-commons/-/functional-commons-1.6.0.tgz",
-      "integrity": "sha512-fwiF0CbeYHDEOTD/NKaFgaI8LvRcGYG2GaJJiRwcedKko16sQ8F3TK5wXfj2Ytjf+8gjwHwsEEX550z3yvDWxA==",
+    "node_modules/@applitools/eyes-cypress/node_modules/array-flatten": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/body-parser": {
+      "version": "1.19.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
+      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
       "dev": true,
+      "dependencies": {
+        "bytes": "3.1.0",
+        "content-type": "~1.0.4",
+        "debug": "2.6.9",
+        "depd": "~1.1.2",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
+        "on-finished": "~2.3.0",
+        "qs": "6.7.0",
+        "raw-body": "2.4.0",
+        "type-is": "~1.6.17"
+      },
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@applitools/http-commons": {
-      "version": "2.4.3",
-      "resolved": "https://registry.npmjs.org/@applitools/http-commons/-/http-commons-2.4.3.tgz",
-      "integrity": "sha512-MBhrHcjDxhkyiw9bRlwe6uhjYg7IsNrsAbDf3w8wfB6iFrCABxsTnPKbnLoT4pyf//s2NFJqFUmcJpwJeEjJmw==",
+    "node_modules/@applitools/eyes-cypress/node_modules/bytes": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
       "dev": true,
-      "dependencies": {
-        "@applitools/functional-commons": "^1.5.5",
-        "@applitools/monitoring-commons": "^1.0.19",
-        "agentkeepalive": "^4.1.0",
-        "debug": "^4.1.1",
-        "lodash.merge": "^4.6.2",
-        "node-fetch": "^2.6.0"
-      },
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@applitools/isomorphic-fetch": {
+    "node_modules/@applitools/eyes-cypress/node_modules/chalk": {
       "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@applitools/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz",
-      "integrity": "sha512-7rutaN/2M5wYjOIOTKS/Zuc1Na90fJNEAqvo/jCxt7nSD1kYscHV3aCk9t7RD59gmzLMvUTIxFbjl4RUMV8qfg==",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
       "dev": true,
       "dependencies": {
-        "node-fetch": "^2.3.0",
-        "whatwg-fetch": ">=0.10.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@applitools/jsdom": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@applitools/jsdom/-/jsdom-1.0.3.tgz",
-      "integrity": "sha512-Iy/Sw7GCzbVYdhg/yveTnss6mNma1Gr+MtYQhcd4ahG7aDyiHXYAAx8XSqO56dozaRz5trJVjQ4K8vf7NS+R+A==",
+    "node_modules/@applitools/eyes-cypress/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "abab": "^2.0.0",
-        "acorn": "^7.4.1",
-        "acorn-globals": "^4.3.2",
-        "array-equal": "^1.0.0",
-        "cssom": "^0.4.1",
-        "cssstyle": "^2.0.0",
-        "data-urls": "^1.1.0",
-        "domexception": "^1.0.1",
-        "escodegen": "^1.11.1",
-        "html-encoding-sniffer": "^1.0.2",
-        "nwsapi": "^2.2.0",
-        "parse5": "5.1.0",
-        "pn": "^1.1.0",
-        "request": "^2.88.0",
-        "request-promise-native": "^1.0.7",
-        "saxes": "^3.1.9",
-        "symbol-tree": "^3.2.2",
-        "tough-cookie": "^3.0.1",
-        "w3c-hr-time": "^1.0.1",
-        "w3c-xmlserializer": "^1.1.2",
-        "webidl-conversions": "^4.0.2",
-        "whatwg-encoding": "^1.0.5",
-        "whatwg-mimetype": "^2.3.0",
-        "whatwg-url": "^7.0.0",
-        "ws": "^7.0.0",
-        "xml-name-validator": "^3.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@applitools/logger": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@applitools/logger/-/logger-1.0.4.tgz",
-      "integrity": "sha512-GQ/OdEVUY4fnkNLXVswSxaBncIfQqsfy+H1JyI85XGTjWIH4LqV/GQqnaiet2o4SHabGe8vHs0eyoWyT0WgAnQ==",
+    "node_modules/@applitools/eyes-cypress/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/content-disposition": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
+      "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
       "dev": true,
       "dependencies": {
-        "@applitools/utils": "1.2.3",
-        "chalk": "3.0.0"
+        "safe-buffer": "5.1.2"
       },
       "engines": {
-        "node": ">= 8.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@applitools/monitoring-commons": {
-      "version": "1.0.19",
-      "resolved": "https://registry.npmjs.org/@applitools/monitoring-commons/-/monitoring-commons-1.0.19.tgz",
-      "integrity": "sha512-rzEOvGoiEF4KnK0PJ9I0btdwnaNlIPLYhjF1vTEG15PoucbbKpix9fYusxWlDG7kMiZya8ZycVPc0woVlNaHRQ==",
+    "node_modules/@applitools/eyes-cypress/node_modules/cookie": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
+      "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==",
       "dev": true,
-      "dependencies": {
-        "debug": "^4.1.0"
-      },
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@applitools/screenshoter": {
-      "version": "3.2.4",
-      "resolved": "https://registry.npmjs.org/@applitools/screenshoter/-/screenshoter-3.2.4.tgz",
-      "integrity": "sha512-v1lhUVMZYvjP2aFm2XTZmschRaPv0xJiVmi4tc7aWcWk4Gu4LgrmcV0yI6QqPkl5fGuK/VwvRBg9tyvX1eWZhA==",
+    "node_modules/@applitools/eyes-cypress/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "@applitools/snippets": "2.1.7",
-        "@applitools/utils": "1.2.3",
-        "png-async": "0.9.4"
-      },
+        "ms": "2.0.0"
+      }
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/depd": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+      "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
+      "dev": true,
       "engines": {
-        "node": ">= 8.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@applitools/snippets": {
-      "version": "2.1.7",
-      "resolved": "https://registry.npmjs.org/@applitools/snippets/-/snippets-2.1.7.tgz",
-      "integrity": "sha512-Tr4Gj7Qov/oPy+8WI4oVmmubxqpOzr8P3Wjzpl6rA57xKLg6/TiIg5oZNb4+jEmO2ShjNYLaEwRWHl7kPgb4fw==",
+    "node_modules/@applitools/eyes-cypress/node_modules/destroy": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
+      "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/express": {
+      "version": "4.17.1",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
+      "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
       "dev": true,
+      "dependencies": {
+        "accepts": "~1.3.7",
+        "array-flatten": "1.1.1",
+        "body-parser": "1.19.0",
+        "content-disposition": "0.5.3",
+        "content-type": "~1.0.4",
+        "cookie": "0.4.0",
+        "cookie-signature": "1.0.6",
+        "debug": "2.6.9",
+        "depd": "~1.1.2",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "finalhandler": "~1.1.2",
+        "fresh": "0.5.2",
+        "merge-descriptors": "1.0.1",
+        "methods": "~1.1.2",
+        "on-finished": "~2.3.0",
+        "parseurl": "~1.3.3",
+        "path-to-regexp": "0.1.7",
+        "proxy-addr": "~2.0.5",
+        "qs": "6.7.0",
+        "range-parser": "~1.2.1",
+        "safe-buffer": "5.1.2",
+        "send": "0.17.1",
+        "serve-static": "1.14.1",
+        "setprototypeof": "1.1.1",
+        "statuses": "~1.5.0",
+        "type-is": "~1.6.18",
+        "utils-merge": "1.0.1",
+        "vary": "~1.1.2"
+      },
       "engines": {
-        "node": ">=8.9.0"
+        "node": ">= 0.10.0"
       }
     },
-    "node_modules/@applitools/types": {
-      "version": "1.0.14",
-      "resolved": "https://registry.npmjs.org/@applitools/types/-/types-1.0.14.tgz",
-      "integrity": "sha512-Exfi8EOGGBwpRpaLizXmsUXInXGXNU26qup7AjLJSJLYETwn3Q3W+0aLHBUbdFA87w9fPCQjXJuZxJgqsNpVig==",
+    "node_modules/@applitools/eyes-cypress/node_modules/finalhandler": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
+      "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
       "dev": true,
+      "dependencies": {
+        "debug": "2.6.9",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "on-finished": "~2.3.0",
+        "parseurl": "~1.3.3",
+        "statuses": "~1.5.0",
+        "unpipe": "~1.0.0"
+      },
       "engines": {
-        "node": ">= 8.9.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@applitools/utils": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/@applitools/utils/-/utils-1.2.3.tgz",
-      "integrity": "sha512-MZXsrzeHTvjFLzpfyKRDUmZWzNxH3gWd3reqYf+1kYimALKB3CO82VDNmkaGJykrRbxEP03Yqha7fHJj9eKslQ==",
+    "node_modules/@applitools/eyes-cypress/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">= 8.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@applitools/visual-grid-client": {
-      "version": "15.8.31",
-      "resolved": "https://registry.npmjs.org/@applitools/visual-grid-client/-/visual-grid-client-15.8.31.tgz",
-      "integrity": "sha512-DPkZ5ynlPcBKx8XMXGOtKjaxJkJs11Ui2SPRPwzGD4Soilq/ijcOfBbNBx89KC7TkntOVlInD/rZBIAfoElO/Q==",
+    "node_modules/@applitools/eyes-cypress/node_modules/http-errors": {
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
+      "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
       "dev": true,
       "dependencies": {
-        "@applitools/eyes-sdk-core": "12.23.12",
-        "@applitools/functional-commons": "1.6.0",
-        "@applitools/http-commons": "2.4.3",
-        "@applitools/isomorphic-fetch": "3.0.0",
-        "@applitools/jsdom": "1.0.3",
-        "abort-controller": "3.0.0",
-        "chalk": "3.0.0",
-        "he": "1.2.0",
-        "lodash.mapvalues": "4.6.0",
-        "mime-types": "2.1.27",
-        "mkdirp": "0.5.5",
-        "postcss-value-parser": "4.1.0",
-        "throat": "5.0.0"
+        "depd": "~1.1.2",
+        "inherits": "2.0.3",
+        "setprototypeof": "1.1.1",
+        "statuses": ">= 1.5.0 < 2",
+        "toidentifier": "1.0.0"
       },
       "engines": {
-        "node": ">=8.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@assemblyscript/loader": {
-      "version": "0.10.1",
-      "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.10.1.tgz",
-      "integrity": "sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==",
+    "node_modules/@applitools/eyes-cypress/node_modules/inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==",
       "dev": true
     },
-    "node_modules/@babel/code-frame": {
-      "version": "7.22.13",
-      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz",
-      "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==",
+    "node_modules/@applitools/eyes-cypress/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/on-finished": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+      "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==",
+      "dev": true,
       "dependencies": {
-        "@babel/highlight": "^7.22.13",
-        "chalk": "^2.4.2"
+        "ee-first": "1.1.1"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@babel/code-frame/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-      "dependencies": {
-        "color-convert": "^1.9.0"
-      },
+    "node_modules/@applitools/eyes-cypress/node_modules/qs": {
+      "version": "6.7.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
+      "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==",
+      "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=0.6"
       }
     },
-    "node_modules/@babel/code-frame/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "node_modules/@applitools/eyes-cypress/node_modules/raw-body": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
+      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "dev": true,
       "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "bytes": "3.1.0",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
+        "unpipe": "1.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@babel/code-frame/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+    "node_modules/@applitools/eyes-cypress/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/send": {
+      "version": "0.17.1",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
+      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+      "dev": true,
       "dependencies": {
-        "color-name": "1.1.3"
+        "debug": "2.6.9",
+        "depd": "~1.1.2",
+        "destroy": "~1.0.4",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "fresh": "0.5.2",
+        "http-errors": "~1.7.2",
+        "mime": "1.6.0",
+        "ms": "2.1.1",
+        "on-finished": "~2.3.0",
+        "range-parser": "~1.2.1",
+        "statuses": "~1.5.0"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/@babel/code-frame/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
-    },
-    "node_modules/@babel/code-frame/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-      "engines": {
-        "node": ">=4"
-      }
+    "node_modules/@applitools/eyes-cypress/node_modules/send/node_modules/ms": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+      "dev": true
     },
-    "node_modules/@babel/code-frame/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+    "node_modules/@applitools/eyes-cypress/node_modules/serve-static": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
+      "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
+      "dev": true,
       "dependencies": {
-        "has-flag": "^3.0.0"
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "parseurl": "~1.3.3",
+        "send": "0.17.1"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/@babel/compat-data": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.22.20.tgz",
-      "integrity": "sha512-BQYjKbpXjoXwFW5jGqiizJQQT/aC7pFm9Ok1OWssonuguICi264lbgMzRp2ZMmRSlfkX6DsWDDcsrctK8Rwfiw==",
+    "node_modules/@applitools/eyes-cypress/node_modules/setprototypeof": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
+      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==",
+      "dev": true
+    },
+    "node_modules/@applitools/eyes-cypress/node_modules/statuses": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
+      "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@babel/core": {
-      "version": "7.20.12",
-      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
-      "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
+    "node_modules/@applitools/eyes-cypress/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "@ampproject/remapping": "^2.1.0",
-        "@babel/code-frame": "^7.18.6",
-        "@babel/generator": "^7.20.7",
-        "@babel/helper-compilation-targets": "^7.20.7",
-        "@babel/helper-module-transforms": "^7.20.11",
-        "@babel/helpers": "^7.20.7",
-        "@babel/parser": "^7.20.7",
-        "@babel/template": "^7.20.7",
-        "@babel/traverse": "^7.20.12",
-        "@babel/types": "^7.20.7",
-        "convert-source-map": "^1.7.0",
-        "debug": "^4.1.0",
-        "gensync": "^1.0.0-beta.2",
-        "json5": "^2.2.2",
-        "semver": "^6.3.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/babel"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/core/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/@applitools/eyes-cypress/node_modules/toidentifier": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
+      "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
-    "node_modules/@babel/generator": {
-      "version": "7.20.14",
-      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.20.14.tgz",
-      "integrity": "sha512-AEmuXHdcD3A52HHXxaTmYlb8q/xMEhoRP67B3T4Oq7lbmSoqroMZzjnGj3+i1io3pdnF8iBYVu4Ilj+c4hBxYg==",
-      "dependencies": {
-        "@babel/types": "^7.20.7",
-        "@jridgewell/gen-mapping": "^0.3.2",
-        "jsesc": "^2.5.1"
-      },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=0.6"
       }
     },
-    "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
-      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+    "node_modules/@applitools/eyes-sdk-core": {
+      "version": "12.23.12",
+      "resolved": "https://registry.npmjs.org/@applitools/eyes-sdk-core/-/eyes-sdk-core-12.23.12.tgz",
+      "integrity": "sha512-nVDWYe1VvUpCQrvjUgDASIDDgHMtIjxXrI52Tvdov1ya/jhyNn6vMYHW9ZgSmjwdrYAoq+p7B90qIdy/HEV5DA==",
+      "dev": true,
       "dependencies": {
-        "@jridgewell/set-array": "^1.0.1",
-        "@jridgewell/sourcemap-codec": "^1.4.10",
-        "@jridgewell/trace-mapping": "^0.3.9"
+        "@applitools/dom-capture": "11.0.1",
+        "@applitools/dom-snapshot": "4.5.8",
+        "@applitools/driver": "1.2.4",
+        "@applitools/isomorphic-fetch": "3.0.0",
+        "@applitools/logger": "1.0.4",
+        "@applitools/screenshoter": "3.2.4",
+        "@applitools/snippets": "2.1.7",
+        "@applitools/types": "1.0.14",
+        "@applitools/utils": "1.2.3",
+        "axios": "0.21.4",
+        "chalk": "3.0.0",
+        "cosmiconfig": "6.0.0",
+        "dateformat": "3.0.3",
+        "debug": "4.2.0",
+        "deepmerge": "4.2.2",
+        "stack-trace": "0.0.10",
+        "tunnel": "0.0.6"
+      },
+      "bin": {
+        "eyes-check-network": "bin/runCheckNetwork.js"
       },
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">= 8.9.0"
       }
     },
-    "node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz",
-      "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.18.6"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.22.15.tgz",
-      "integrity": "sha512-QkBXwGgaoC2GtGZRoma6kv7Szfv06khvhFav67ZExau2RaXzy8MpHSMO2PNoP2XtmQphJQRHFfg77Bq731Yizw==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.15"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "follow-redirects": "^1.14.0"
       }
     },
-    "node_modules/@babel/helper-compilation-targets": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz",
-      "integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/chalk": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+      "dev": true,
       "dependencies": {
-        "@babel/compat-data": "^7.22.9",
-        "@babel/helper-validator-option": "^7.22.15",
-        "browserslist": "^4.21.9",
-        "lru-cache": "^5.1.1",
-        "semver": "^6.3.1"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-compilation-targets/node_modules/browserslist": {
-      "version": "4.21.11",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.11.tgz",
-      "integrity": "sha512-xn1UXOKUz7DjdGlg9RrUr0GGiWzI97UQJnugHtH0OLDfJB7jMgoIkYvRIEO1l9EeEERVqeqLYOcFBW9ldjypbQ==",
-      "funding": [
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/browserslist"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/browserslist"
-        },
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/ai"
-        }
-      ],
+    "node_modules/@applitools/eyes-sdk-core/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
       "dependencies": {
-        "caniuse-lite": "^1.0.30001538",
-        "electron-to-chromium": "^1.4.526",
-        "node-releases": "^2.0.13",
-        "update-browserslist-db": "^1.0.13"
-      },
-      "bin": {
-        "browserslist": "cli.js"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@babel/helper-compilation-targets/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "bin": {
-        "semver": "bin/semver.js"
-      }
+    "node_modules/@applitools/eyes-sdk-core/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
     },
-    "node_modules/@babel/helper-create-class-features-plugin": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.15.tgz",
-      "integrity": "sha512-jKkwA59IXcvSaiK2UN45kKwSC9o+KuoXsBDvHvU/7BecYIp8GQ2UwrVvFgJASUT+hBnwJx6MhvMCuMzwZZ7jlg==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/cosmiconfig": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
+      "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
-        "@babel/helper-environment-visitor": "^7.22.5",
-        "@babel/helper-function-name": "^7.22.5",
-        "@babel/helper-member-expression-to-functions": "^7.22.15",
-        "@babel/helper-optimise-call-expression": "^7.22.5",
-        "@babel/helper-replace-supers": "^7.22.9",
-        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
-        "@babel/helper-split-export-declaration": "^7.22.6",
-        "semver": "^6.3.1"
+        "@types/parse-json": "^4.0.0",
+        "import-fresh": "^3.1.0",
+        "parse-json": "^5.0.0",
+        "path-type": "^4.0.0",
+        "yaml": "^1.7.2"
       },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-create-class-features-plugin/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/debug": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+      "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
+      "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "ms": "2.1.2"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@babel/helper-create-class-features-plugin/node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.22.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
-      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/deepmerge": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
+      "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
       "dev": true,
-      "dependencies": {
-        "@babel/types": "^7.22.5"
-      },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@babel/helper-create-class-features-plugin/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
-    "node_modules/@babel/helper-create-regexp-features-plugin": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.22.15.tgz",
-      "integrity": "sha512-29FkPLFjn4TPEa3RE7GpW+qbE8tlsu3jntNYNfcGsc49LphF1PQIiD+vMZ1z1xVOKt+93khA9tc2JBs3kBjA7w==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
-        "regexpu-core": "^5.3.1",
-        "semver": "^6.3.1"
-      },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-create-regexp-features-plugin/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@applitools/eyes-sdk-core/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-create-regexp-features-plugin/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/@applitools/functional-commons": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@applitools/functional-commons/-/functional-commons-1.6.0.tgz",
+      "integrity": "sha512-fwiF0CbeYHDEOTD/NKaFgaI8LvRcGYG2GaJJiRwcedKko16sQ8F3TK5wXfj2Ytjf+8gjwHwsEEX550z3yvDWxA==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+      "engines": {
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/@babel/helper-define-polyfill-provider": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.3.tgz",
-      "integrity": "sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==",
+    "node_modules/@applitools/http-commons": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/@applitools/http-commons/-/http-commons-2.4.3.tgz",
+      "integrity": "sha512-MBhrHcjDxhkyiw9bRlwe6uhjYg7IsNrsAbDf3w8wfB6iFrCABxsTnPKbnLoT4pyf//s2NFJqFUmcJpwJeEjJmw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-compilation-targets": "^7.17.7",
-        "@babel/helper-plugin-utils": "^7.16.7",
+        "@applitools/functional-commons": "^1.5.5",
+        "@applitools/monitoring-commons": "^1.0.19",
+        "agentkeepalive": "^4.1.0",
         "debug": "^4.1.1",
-        "lodash.debounce": "^4.0.8",
-        "resolve": "^1.14.2",
-        "semver": "^6.1.2"
+        "lodash.merge": "^4.6.2",
+        "node-fetch": "^2.6.0"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.4.0-0"
+      "engines": {
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/@babel/helper-define-polyfill-provider/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/@applitools/isomorphic-fetch": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@applitools/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz",
+      "integrity": "sha512-7rutaN/2M5wYjOIOTKS/Zuc1Na90fJNEAqvo/jCxt7nSD1kYscHV3aCk9t7RD59gmzLMvUTIxFbjl4RUMV8qfg==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
-    "node_modules/@babel/helper-environment-visitor": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
-      "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
-      "engines": {
-        "node": ">=6.9.0"
+      "dependencies": {
+        "node-fetch": "^2.3.0",
+        "whatwg-fetch": ">=0.10.0"
       }
     },
-    "node_modules/@babel/helper-function-name": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
-      "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
+    "node_modules/@applitools/jsdom": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@applitools/jsdom/-/jsdom-1.0.3.tgz",
+      "integrity": "sha512-Iy/Sw7GCzbVYdhg/yveTnss6mNma1Gr+MtYQhcd4ahG7aDyiHXYAAx8XSqO56dozaRz5trJVjQ4K8vf7NS+R+A==",
+      "dev": true,
       "dependencies": {
-        "@babel/template": "^7.22.15",
-        "@babel/types": "^7.23.0"
+        "abab": "^2.0.0",
+        "acorn": "^7.4.1",
+        "acorn-globals": "^4.3.2",
+        "array-equal": "^1.0.0",
+        "cssom": "^0.4.1",
+        "cssstyle": "^2.0.0",
+        "data-urls": "^1.1.0",
+        "domexception": "^1.0.1",
+        "escodegen": "^1.11.1",
+        "html-encoding-sniffer": "^1.0.2",
+        "nwsapi": "^2.2.0",
+        "parse5": "5.1.0",
+        "pn": "^1.1.0",
+        "request": "^2.88.0",
+        "request-promise-native": "^1.0.7",
+        "saxes": "^3.1.9",
+        "symbol-tree": "^3.2.2",
+        "tough-cookie": "^3.0.1",
+        "w3c-hr-time": "^1.0.1",
+        "w3c-xmlserializer": "^1.1.2",
+        "webidl-conversions": "^4.0.2",
+        "whatwg-encoding": "^1.0.5",
+        "whatwg-mimetype": "^2.3.0",
+        "whatwg-url": "^7.0.0",
+        "ws": "^7.0.0",
+        "xml-name-validator": "^3.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-function-name/node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
-      "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+    "node_modules/@applitools/jsdom/node_modules/acorn": {
+      "version": "7.4.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
+      "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
+      "dev": true,
+      "bin": {
+        "acorn": "bin/acorn"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@babel/helper-hoist-variables": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz",
-      "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==",
+    "node_modules/@applitools/jsdom/node_modules/parse5": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz",
+      "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==",
+      "dev": true
+    },
+    "node_modules/@applitools/jsdom/node_modules/tr46": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
+      "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==",
+      "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "punycode": "^2.1.0"
       }
     },
-    "node_modules/@babel/helper-member-expression-to-functions": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.23.0.tgz",
-      "integrity": "sha512-6gfrPwh7OuT6gZyJZvd6WbTfrqAo7vm4xCzAXOusKqq/vWdKXphTpj5klHKNmRUU6/QRGlBsyU9mAIPaWHlqJA==",
+    "node_modules/@applitools/jsdom/node_modules/webidl-conversions": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "dev": true
+    },
+    "node_modules/@applitools/jsdom/node_modules/whatwg-url": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
+      "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.23.0"
-      },
+        "lodash.sortby": "^4.7.0",
+        "tr46": "^1.0.1",
+        "webidl-conversions": "^4.0.2"
+      }
+    },
+    "node_modules/@applitools/jsdom/node_modules/ws": {
+      "version": "7.5.9",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+      "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8.3.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@babel/helper-module-imports": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz",
-      "integrity": "sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==",
+    "node_modules/@applitools/logger": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@applitools/logger/-/logger-1.0.4.tgz",
+      "integrity": "sha512-GQ/OdEVUY4fnkNLXVswSxaBncIfQqsfy+H1JyI85XGTjWIH4LqV/GQqnaiet2o4SHabGe8vHs0eyoWyT0WgAnQ==",
+      "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.15"
+        "@applitools/utils": "1.2.3",
+        "chalk": "3.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">= 8.9.0"
       }
     },
-    "node_modules/@babel/helper-module-transforms": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.0.tgz",
-      "integrity": "sha512-WhDWw1tdrlT0gMgUJSlX0IQvoO1eN279zrAUbVB+KpV2c3Tylz8+GnKOLllCS6Z/iZQEyVYxhZVUdPTqs2YYPw==",
+    "node_modules/@applitools/logger/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
       "dependencies": {
-        "@babel/helper-environment-visitor": "^7.22.20",
-        "@babel/helper-module-imports": "^7.22.15",
-        "@babel/helper-simple-access": "^7.22.5",
-        "@babel/helper-split-export-declaration": "^7.22.6",
-        "@babel/helper-validator-identifier": "^7.22.20"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@babel/helper-module-transforms/node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.22.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
-      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+    "node_modules/@applitools/logger/node_modules/chalk": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+      "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-optimise-call-expression": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.22.5.tgz",
-      "integrity": "sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==",
+    "node_modules/@applitools/logger/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@babel/helper-plugin-utils": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz",
-      "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==",
+    "node_modules/@applitools/logger/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@applitools/logger/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-remap-async-to-generator": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.22.20.tgz",
-      "integrity": "sha512-pBGyV4uBqOns+0UvhsTO8qgl8hO89PmiDYv+/COyp1aeMcmfrfruz+/nCMFiYyFF/Knn0yfrC85ZzNFjembFTw==",
+    "node_modules/@applitools/logger/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
-        "@babel/helper-environment-visitor": "^7.22.20",
-        "@babel/helper-wrap-function": "^7.22.20"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-remap-async-to-generator/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@applitools/monitoring-commons": {
+      "version": "1.0.19",
+      "resolved": "https://registry.npmjs.org/@applitools/monitoring-commons/-/monitoring-commons-1.0.19.tgz",
+      "integrity": "sha512-rzEOvGoiEF4KnK0PJ9I0btdwnaNlIPLYhjF1vTEG15PoucbbKpix9fYusxWlDG7kMiZya8ZycVPc0woVlNaHRQ==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "debug": "^4.1.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/@babel/helper-replace-supers": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.22.20.tgz",
-      "integrity": "sha512-qsW0In3dbwQUbK8kejJ4R7IHVGwHJlV6lpG6UA7a9hSa2YEiAib+N1T2kr6PEeUT+Fl7najmSOS6SmAwCHK6Tw==",
+    "node_modules/@applitools/screenshoter": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/@applitools/screenshoter/-/screenshoter-3.2.4.tgz",
+      "integrity": "sha512-v1lhUVMZYvjP2aFm2XTZmschRaPv0xJiVmi4tc7aWcWk4Gu4LgrmcV0yI6QqPkl5fGuK/VwvRBg9tyvX1eWZhA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-environment-visitor": "^7.22.20",
-        "@babel/helper-member-expression-to-functions": "^7.22.15",
-        "@babel/helper-optimise-call-expression": "^7.22.5"
+        "@applitools/snippets": "2.1.7",
+        "@applitools/utils": "1.2.3",
+        "png-async": "0.9.4"
       },
       "engines": {
-        "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "node": ">= 8.9.0"
       }
     },
-    "node_modules/@babel/helper-simple-access": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz",
-      "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==",
+    "node_modules/@applitools/snippets": {
+      "version": "2.1.7",
+      "resolved": "https://registry.npmjs.org/@applitools/snippets/-/snippets-2.1.7.tgz",
+      "integrity": "sha512-Tr4Gj7Qov/oPy+8WI4oVmmubxqpOzr8P3Wjzpl6rA57xKLg6/TiIg5oZNb4+jEmO2ShjNYLaEwRWHl7kPgb4fw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.9.0"
+      }
+    },
+    "node_modules/@applitools/types": {
+      "version": "1.0.14",
+      "resolved": "https://registry.npmjs.org/@applitools/types/-/types-1.0.14.tgz",
+      "integrity": "sha512-Exfi8EOGGBwpRpaLizXmsUXInXGXNU26qup7AjLJSJLYETwn3Q3W+0aLHBUbdFA87w9fPCQjXJuZxJgqsNpVig==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8.9.0"
+      }
+    },
+    "node_modules/@applitools/utils": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/@applitools/utils/-/utils-1.2.3.tgz",
+      "integrity": "sha512-MZXsrzeHTvjFLzpfyKRDUmZWzNxH3gWd3reqYf+1kYimALKB3CO82VDNmkaGJykrRbxEP03Yqha7fHJj9eKslQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8.9.0"
+      }
+    },
+    "node_modules/@applitools/visual-grid-client": {
+      "version": "15.8.31",
+      "resolved": "https://registry.npmjs.org/@applitools/visual-grid-client/-/visual-grid-client-15.8.31.tgz",
+      "integrity": "sha512-DPkZ5ynlPcBKx8XMXGOtKjaxJkJs11Ui2SPRPwzGD4Soilq/ijcOfBbNBx89KC7TkntOVlInD/rZBIAfoElO/Q==",
+      "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@applitools/eyes-sdk-core": "12.23.12",
+        "@applitools/functional-commons": "1.6.0",
+        "@applitools/http-commons": "2.4.3",
+        "@applitools/isomorphic-fetch": "3.0.0",
+        "@applitools/jsdom": "1.0.3",
+        "abort-controller": "3.0.0",
+        "chalk": "3.0.0",
+        "he": "1.2.0",
+        "lodash.mapvalues": "4.6.0",
+        "mime-types": "2.1.27",
+        "mkdirp": "0.5.5",
+        "postcss-value-parser": "4.1.0",
+        "throat": "5.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8.9.0"
       }
     },
-    "node_modules/@babel/helper-skip-transparent-expression-wrappers": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.22.5.tgz",
-      "integrity": "sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==",
+    "node_modules/@applitools/visual-grid-client/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz",
-      "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==",
+    "node_modules/@applitools/visual-grid-client/node_modules/chalk": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.18.6"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-string-parser": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz",
-      "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==",
+    "node_modules/@applitools/visual-grid-client/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@babel/helper-validator-identifier": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
-      "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
+    "node_modules/@applitools/visual-grid-client/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@applitools/visual-grid-client/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helper-validator-option": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz",
-      "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==",
+    "node_modules/@applitools/visual-grid-client/node_modules/mime-db": {
+      "version": "1.44.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
+      "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==",
+      "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@babel/helper-wrap-function": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.22.20.tgz",
-      "integrity": "sha512-pms/UwkOpnQe/PDAEdV/d7dVCoBbB+R4FvYoHGZz+4VPcg7RtYy2KP7S2lbuWM6FCSgob5wshfGESbC/hzNXZw==",
+    "node_modules/@applitools/visual-grid-client/node_modules/mime-types": {
+      "version": "2.1.27",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
+      "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-function-name": "^7.22.5",
-        "@babel/template": "^7.22.15",
-        "@babel/types": "^7.22.19"
+        "mime-db": "1.44.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@babel/helper-wrap-function/node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+    "node_modules/@applitools/visual-grid-client/node_modules/mkdirp": {
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
       "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+        "minimist": "^1.2.5"
       },
-      "engines": {
-        "node": ">=6.9.0"
+      "bin": {
+        "mkdirp": "bin/cmd.js"
       }
     },
-    "node_modules/@babel/helpers": {
-      "version": "7.23.1",
-      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.1.tgz",
-      "integrity": "sha512-chNpneuK18yW5Oxsr+t553UZzzAs3aZnFm4bxhebsNTeshrC95yA7l5yl7GBAG+JG1rF0F7zzD2EixK9mWSDoA==",
+    "node_modules/@applitools/visual-grid-client/node_modules/postcss-value-parser": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz",
+      "integrity": "sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==",
+      "dev": true
+    },
+    "node_modules/@applitools/visual-grid-client/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
       "dependencies": {
-        "@babel/template": "^7.22.15",
-        "@babel/traverse": "^7.23.0",
-        "@babel/types": "^7.23.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@babel/helpers/node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+    "node_modules/@assemblyscript/loader": {
+      "version": "0.10.1",
+      "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.10.1.tgz",
+      "integrity": "sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==",
+      "dev": true
+    },
+    "node_modules/@babel/code-frame": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.4.tgz",
+      "integrity": "sha512-r1IONyb6Ia+jYR2vvIDhdWdlTGhqbBoFqLTQidzZ4kepUFH15ejXvFHxCVbtl7BOXIudsIubf4E81xeA3h3IXA==",
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+        "@babel/highlight": "^7.23.4",
+        "chalk": "^2.4.2"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz",
-      "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==",
-      "dependencies": {
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "chalk": "^2.4.2",
-        "js-tokens": "^4.0.0"
-      },
+    "node_modules/@babel/compat-data": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.3.tgz",
+      "integrity": "sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ==",
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+    "node_modules/@babel/core": {
+      "version": "7.19.3",
+      "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.19.3.tgz",
+      "integrity": "sha512-WneDJxdsjEvyKtXKsaBGbDeiyOjR5vYq4HcShxnIbG0qixpoHjI3MqeZM9NDvsojNCEBItQE4juOo/bU6e72gQ==",
       "dependencies": {
-        "color-convert": "^1.9.0"
+        "@ampproject/remapping": "^2.1.0",
+        "@babel/code-frame": "^7.18.6",
+        "@babel/generator": "^7.19.3",
+        "@babel/helper-compilation-targets": "^7.19.3",
+        "@babel/helper-module-transforms": "^7.19.0",
+        "@babel/helpers": "^7.19.0",
+        "@babel/parser": "^7.19.3",
+        "@babel/template": "^7.18.10",
+        "@babel/traverse": "^7.19.3",
+        "@babel/types": "^7.19.3",
+        "convert-source-map": "^1.7.0",
+        "debug": "^4.1.0",
+        "gensync": "^1.0.0-beta.2",
+        "json5": "^2.2.1",
+        "semver": "^6.3.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=6.9.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/babel"
       }
     },
-    "node_modules/@babel/highlight/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "node_modules/@babel/generator": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.4.tgz",
+      "integrity": "sha512-esuS49Cga3HcThFNebGhlgsrVLkvhqvYDTzgjfFFlHJcIfLe5jFmRRfCQ1KuBfc4Jrtn3ndLgKWAKjBE+IraYQ==",
       "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "@babel/types": "^7.23.4",
+        "@jridgewell/gen-mapping": "^0.3.2",
+        "@jridgewell/trace-mapping": "^0.3.17",
+        "jsesc": "^2.5.1"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+    "node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz",
+      "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==",
+      "dev": true,
       "dependencies": {
-        "color-name": "1.1.3"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
-    },
-    "node_modules/@babel/highlight/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+        "@babel/types": "^7.18.6"
+      },
       "engines": {
-        "node": ">=4"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+    "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.22.15.tgz",
+      "integrity": "sha512-QkBXwGgaoC2GtGZRoma6kv7Szfv06khvhFav67ZExau2RaXzy8MpHSMO2PNoP2XtmQphJQRHFfg77Bq731Yizw==",
+      "dev": true,
       "dependencies": {
-        "has-flag": "^3.0.0"
+        "@babel/types": "^7.22.15"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/parser": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz",
-      "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==",
-      "bin": {
-        "parser": "bin/babel-parser.js"
+    "node_modules/@babel/helper-compilation-targets": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz",
+      "integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==",
+      "dependencies": {
+        "@babel/compat-data": "^7.22.9",
+        "@babel/helper-validator-option": "^7.22.15",
+        "browserslist": "^4.21.9",
+        "lru-cache": "^5.1.1",
+        "semver": "^6.3.1"
       },
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": {
+    "node_modules/@babel/helper-create-class-features-plugin": {
       "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.22.15.tgz",
-      "integrity": "sha512-FB9iYlz7rURmRJyXRKEnalYPPdn87H5no108cyuQQyMwlpJ2SJtpIUBI27kdTin956pz+LPypkPVPUTlxOmrsg==",
+      "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.22.15.tgz",
+      "integrity": "sha512-jKkwA59IXcvSaiK2UN45kKwSC9o+KuoXsBDvHvU/7BecYIp8GQ2UwrVvFgJASUT+hBnwJx6MhvMCuMzwZZ7jlg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "@babel/helper-environment-visitor": "^7.22.5",
+        "@babel/helper-function-name": "^7.22.5",
+        "@babel/helper-member-expression-to-functions": "^7.22.15",
+        "@babel/helper-optimise-call-expression": "^7.22.5",
+        "@babel/helper-replace-supers": "^7.22.9",
+        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
+        "@babel/helper-split-export-declaration": "^7.22.6",
+        "semver": "^6.3.1"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -1813,366 +2238,316 @@
         "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.22.15.tgz",
-      "integrity": "sha512-Hyph9LseGvAeeXzikV88bczhsrLrIZqDPxO+sSmAunMPaGrBGhfMWzCPYTtiW9t+HzSE2wtV8e5cc5P6r1xMDQ==",
+    "node_modules/@babel/helper-create-class-features-plugin/node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
-        "@babel/plugin-transform-optional-chaining": "^7.22.15"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.13.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-async-generator-functions": {
-      "version": "7.20.7",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.20.7.tgz",
-      "integrity": "sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-async-generator-functions instead.",
+    "node_modules/@babel/helper-create-regexp-features-plugin": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.22.15.tgz",
+      "integrity": "sha512-29FkPLFjn4TPEa3RE7GpW+qbE8tlsu3jntNYNfcGsc49LphF1PQIiD+vMZ1z1xVOKt+93khA9tc2JBs3kBjA7w==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-environment-visitor": "^7.18.9",
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/helper-remap-async-to-generator": "^7.18.9",
-        "@babel/plugin-syntax-async-generators": "^7.8.4"
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "regexpu-core": "^5.3.1",
+        "semver": "^6.3.1"
       },
       "engines": {
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-class-properties": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz",
-      "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-properties instead.",
+    "node_modules/@babel/helper-create-regexp-features-plugin/node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-class-features-plugin": "^7.18.6",
-        "@babel/helper-plugin-utils": "^7.18.6"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-class-static-block": {
-      "version": "7.21.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.21.0.tgz",
-      "integrity": "sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-static-block instead.",
+    "node_modules/@babel/helper-define-polyfill-provider": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.3.tgz",
+      "integrity": "sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-class-features-plugin": "^7.21.0",
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/plugin-syntax-class-static-block": "^7.14.5"
+        "@babel/helper-compilation-targets": "^7.17.7",
+        "@babel/helper-plugin-utils": "^7.16.7",
+        "debug": "^4.1.1",
+        "lodash.debounce": "^4.0.8",
+        "resolve": "^1.14.2",
+        "semver": "^6.1.2"
       },
+      "peerDependencies": {
+        "@babel/core": "^7.4.0-0"
+      }
+    },
+    "node_modules/@babel/helper-environment-visitor": {
+      "version": "7.22.20",
+      "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
+      "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.12.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-dynamic-import": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz",
-      "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-dynamic-import instead.",
-      "dev": true,
+    "node_modules/@babel/helper-function-name": {
+      "version": "7.23.0",
+      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
+      "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6",
-        "@babel/plugin-syntax-dynamic-import": "^7.8.3"
+        "@babel/template": "^7.22.15",
+        "@babel/types": "^7.23.0"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-export-namespace-from": {
-      "version": "7.18.9",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz",
-      "integrity": "sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-export-namespace-from instead.",
-      "dev": true,
+    "node_modules/@babel/helper-hoist-variables": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz",
+      "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.9",
-        "@babel/plugin-syntax-export-namespace-from": "^7.8.3"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-json-strings": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz",
-      "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-json-strings instead.",
+    "node_modules/@babel/helper-member-expression-to-functions": {
+      "version": "7.23.0",
+      "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.23.0.tgz",
+      "integrity": "sha512-6gfrPwh7OuT6gZyJZvd6WbTfrqAo7vm4xCzAXOusKqq/vWdKXphTpj5klHKNmRUU6/QRGlBsyU9mAIPaWHlqJA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6",
-        "@babel/plugin-syntax-json-strings": "^7.8.3"
+        "@babel/types": "^7.23.0"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-logical-assignment-operators": {
-      "version": "7.20.7",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.20.7.tgz",
-      "integrity": "sha512-y7C7cZgpMIjWlKE5T7eJwp+tnRYM89HmRvWM5EQuB5BoHEONjmQ8lSNmBUwOyy/GFRsohJED51YBF79hE1djug==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-logical-assignment-operators instead.",
-      "dev": true,
+    "node_modules/@babel/helper-module-imports": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz",
+      "integrity": "sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4"
+        "@babel/types": "^7.22.15"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz",
-      "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-nullish-coalescing-operator instead.",
-      "dev": true,
+    "node_modules/@babel/helper-module-transforms": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz",
+      "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6",
-        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3"
+        "@babel/helper-environment-visitor": "^7.22.20",
+        "@babel/helper-module-imports": "^7.22.15",
+        "@babel/helper-simple-access": "^7.22.5",
+        "@babel/helper-split-export-declaration": "^7.22.6",
+        "@babel/helper-validator-identifier": "^7.22.20"
       },
       "engines": {
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-numeric-separator": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz",
-      "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-numeric-separator instead.",
+    "node_modules/@babel/helper-optimise-call-expression": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.22.5.tgz",
+      "integrity": "sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6",
-        "@babel/plugin-syntax-numeric-separator": "^7.10.4"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-object-rest-spread": {
-      "version": "7.20.7",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.7.tgz",
-      "integrity": "sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-object-rest-spread instead.",
+    "node_modules/@babel/helper-plugin-utils": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz",
+      "integrity": "sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==",
       "dev": true,
-      "dependencies": {
-        "@babel/compat-data": "^7.20.5",
-        "@babel/helper-compilation-targets": "^7.20.7",
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
-        "@babel/plugin-transform-parameters": "^7.20.7"
-      },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-optional-catch-binding": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz",
-      "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-catch-binding instead.",
+    "node_modules/@babel/helper-remap-async-to-generator": {
+      "version": "7.22.20",
+      "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.22.20.tgz",
+      "integrity": "sha512-pBGyV4uBqOns+0UvhsTO8qgl8hO89PmiDYv+/COyp1aeMcmfrfruz+/nCMFiYyFF/Knn0yfrC85ZzNFjembFTw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.18.6",
-        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3"
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "@babel/helper-environment-visitor": "^7.22.20",
+        "@babel/helper-wrap-function": "^7.22.20"
       },
       "engines": {
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-optional-chaining": {
-      "version": "7.21.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.21.0.tgz",
-      "integrity": "sha512-p4zeefM72gpmEe2fkUr/OnOXpWEf8nAgk7ZYVqqfFiyIG7oFfVZcCrU64hWn5xp4tQ9LkV4bTIa5rD0KANpKNA==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-chaining instead.",
+    "node_modules/@babel/helper-remap-async-to-generator/node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/helper-skip-transparent-expression-wrappers": "^7.20.0",
-        "@babel/plugin-syntax-optional-chaining": "^7.8.3"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-private-methods": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz",
-      "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-methods instead.",
+    "node_modules/@babel/helper-replace-supers": {
+      "version": "7.22.20",
+      "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.22.20.tgz",
+      "integrity": "sha512-qsW0In3dbwQUbK8kejJ4R7IHVGwHJlV6lpG6UA7a9hSa2YEiAib+N1T2kr6PEeUT+Fl7najmSOS6SmAwCHK6Tw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-class-features-plugin": "^7.18.6",
-        "@babel/helper-plugin-utils": "^7.18.6"
+        "@babel/helper-environment-visitor": "^7.22.20",
+        "@babel/helper-member-expression-to-functions": "^7.22.15",
+        "@babel/helper-optimise-call-expression": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-proposal-private-property-in-object": {
-      "version": "7.21.11",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.11.tgz",
-      "integrity": "sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-property-in-object instead.",
-      "dev": true,
+    "node_modules/@babel/helper-simple-access": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz",
+      "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==",
       "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.18.6",
-        "@babel/helper-create-class-features-plugin": "^7.21.0",
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/plugin-syntax-private-property-in-object": "^7.14.5"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-proposal-unicode-property-regex": {
-      "version": "7.18.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz",
-      "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==",
-      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-unicode-property-regex instead.",
+    "node_modules/@babel/helper-skip-transparent-expression-wrappers": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.22.5.tgz",
+      "integrity": "sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-regexp-features-plugin": "^7.18.6",
-        "@babel/helper-plugin-utils": "^7.18.6"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
-        "node": ">=4"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-async-generators": {
-      "version": "7.8.4",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz",
-      "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==",
-      "dev": true,
+    "node_modules/@babel/helper-split-export-declaration": {
+      "version": "7.22.6",
+      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
+      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/types": "^7.22.5"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-bigint": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz",
-      "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==",
-      "dev": true,
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+    "node_modules/@babel/helper-string-parser": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz",
+      "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==",
+      "engines": {
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-class-properties": {
-      "version": "7.12.13",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz",
-      "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==",
+    "node_modules/@babel/helper-validator-identifier": {
+      "version": "7.22.20",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
+      "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@babel/helper-validator-option": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz",
+      "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/@babel/helper-wrap-function": {
+      "version": "7.22.20",
+      "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.22.20.tgz",
+      "integrity": "sha512-pms/UwkOpnQe/PDAEdV/d7dVCoBbB+R4FvYoHGZz+4VPcg7RtYy2KP7S2lbuWM6FCSgob5wshfGESbC/hzNXZw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.12.13"
+        "@babel/helper-function-name": "^7.22.5",
+        "@babel/template": "^7.22.15",
+        "@babel/types": "^7.22.19"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-class-static-block": {
-      "version": "7.14.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz",
-      "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==",
-      "dev": true,
+    "node_modules/@babel/helpers": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.4.tgz",
+      "integrity": "sha512-HfcMizYz10cr3h29VqyfGL6ZWIjTwWfvYBMsBVGwpcbhNGe3wQ1ZXZRPzZoAHhd9OqHadHqjQ89iVKINXnbzuw==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.14.5"
+        "@babel/template": "^7.22.15",
+        "@babel/traverse": "^7.23.4",
+        "@babel/types": "^7.23.4"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-dynamic-import": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz",
-      "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==",
-      "dev": true,
+    "node_modules/@babel/highlight": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz",
+      "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/helper-validator-identifier": "^7.22.20",
+        "chalk": "^2.4.2",
+        "js-tokens": "^4.0.0"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-export-namespace-from": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz",
-      "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==",
-      "dev": true,
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.3"
+    "node_modules/@babel/parser": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.4.tgz",
+      "integrity": "sha512-vf3Xna6UEprW+7t6EtOmFpHNAuxw3xqPZghy+brsnusscJRW5BMUzzHZc5ICjULee81WeUV2jjakG09MDglJXQ==",
+      "bin": {
+        "parser": "bin/babel-parser.js"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=6.0.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-import-assertions": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.22.5.tgz",
-      "integrity": "sha512-rdV97N7KqsRzeNGoWUOK6yUsWarLjE5Su/Snk9IYPU9CwkWHs4t+rTGOvffTR8XGkJMTAdLfO0xVnXm8wugIJg==",
+    "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.23.3.tgz",
+      "integrity": "sha512-iRkKcCqb7iGnq9+3G6rZ+Ciz5VywC4XNRHe57lKM+jOeYAoR0lVqdeeDRfh0tQcTfw/+vBhHn926FmQhLtlFLQ==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
@@ -2181,40 +2556,54 @@
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-import-meta": {
-      "version": "7.10.4",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz",
-      "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==",
+    "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.23.3.tgz",
+      "integrity": "sha512-WwlxbfMNdVEpQjZmK5mhm7oSwD3dS6eU+Iwsi4Knl9wAletWem7kaRsGOG+8UEbRyqxY4SS5zvtfXwX+jMxUwQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.10.4"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
+        "@babel/plugin-transform-optional-chaining": "^7.23.3"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.13.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-json-strings": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz",
-      "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==",
+    "node_modules/@babel/plugin-proposal-async-generator-functions": {
+      "version": "7.20.7",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.20.7.tgz",
+      "integrity": "sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-async-generator-functions instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/helper-environment-visitor": "^7.18.9",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/helper-remap-async-to-generator": "^7.18.9",
+        "@babel/plugin-syntax-async-generators": "^7.8.4"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-jsx": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz",
-      "integrity": "sha512-gvyP4hZrgrs/wWMaocvxZ44Hw0b3W8Pe+cMxc8V1ULQ07oh8VNbIRaoD1LRZVTvD+0nieDKjfgKg89sD7rrKrg==",
+    "node_modules/@babel/plugin-proposal-class-properties": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz",
+      "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-properties instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-create-class-features-plugin": "^7.18.6",
+        "@babel/helper-plugin-utils": "^7.18.6"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2223,85 +2612,101 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-logical-assignment-operators": {
-      "version": "7.10.4",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz",
-      "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==",
+    "node_modules/@babel/plugin-proposal-class-static-block": {
+      "version": "7.21.0",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.21.0.tgz",
+      "integrity": "sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-static-block instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.10.4"
+        "@babel/helper-create-class-features-plugin": "^7.21.0",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/plugin-syntax-class-static-block": "^7.14.5"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
-      }
-    },
-    "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz",
-      "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==",
-      "dev": true,
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.12.0"
       }
     },
-    "node_modules/@babel/plugin-syntax-numeric-separator": {
-      "version": "7.10.4",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz",
-      "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==",
+    "node_modules/@babel/plugin-proposal-dynamic-import": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz",
+      "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-dynamic-import instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.10.4"
+        "@babel/helper-plugin-utils": "^7.18.6",
+        "@babel/plugin-syntax-dynamic-import": "^7.8.3"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-object-rest-spread": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz",
-      "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==",
+    "node_modules/@babel/plugin-proposal-export-namespace-from": {
+      "version": "7.18.9",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz",
+      "integrity": "sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-export-namespace-from instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/helper-plugin-utils": "^7.18.9",
+        "@babel/plugin-syntax-export-namespace-from": "^7.8.3"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-optional-catch-binding": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz",
-      "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==",
+    "node_modules/@babel/plugin-proposal-json-strings": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz",
+      "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-json-strings instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/helper-plugin-utils": "^7.18.6",
+        "@babel/plugin-syntax-json-strings": "^7.8.3"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-optional-chaining": {
-      "version": "7.8.3",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz",
-      "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==",
+    "node_modules/@babel/plugin-proposal-logical-assignment-operators": {
+      "version": "7.20.7",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.20.7.tgz",
+      "integrity": "sha512-y7C7cZgpMIjWlKE5T7eJwp+tnRYM89HmRvWM5EQuB5BoHEONjmQ8lSNmBUwOyy/GFRsohJED51YBF79hE1djug==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-logical-assignment-operators instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.8.0"
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-private-property-in-object": {
-      "version": "7.14.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz",
-      "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==",
+    "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz",
+      "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-nullish-coalescing-operator instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.14.5"
+        "@babel/helper-plugin-utils": "^7.18.6",
+        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2310,13 +2715,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-top-level-await": {
-      "version": "7.14.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz",
-      "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==",
+    "node_modules/@babel/plugin-proposal-numeric-separator": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz",
+      "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-numeric-separator instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.14.5"
+        "@babel/helper-plugin-utils": "^7.18.6",
+        "@babel/plugin-syntax-numeric-separator": "^7.10.4"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2325,13 +2732,18 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-syntax-typescript": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.22.5.tgz",
-      "integrity": "sha512-1mS2o03i7t1c6VzH6fdQ3OA8tcEIxwG18zIPRp+UY1Ihv6W+XZzBCVxExF9upussPXJ0xE9XRHwMoNs1ep/nRQ==",
+    "node_modules/@babel/plugin-proposal-object-rest-spread": {
+      "version": "7.20.7",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.7.tgz",
+      "integrity": "sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-object-rest-spread instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/compat-data": "^7.20.5",
+        "@babel/helper-compilation-targets": "^7.20.7",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+        "@babel/plugin-transform-parameters": "^7.20.7"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2340,13 +2752,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-arrow-functions": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.22.5.tgz",
-      "integrity": "sha512-26lTNXoVRdAnsaDXPpvCNUq+OVWEVC6bx7Vvz9rC53F2bagUWW4u4ii2+h8Fejfh7RYqPxn+libeFBBck9muEw==",
+    "node_modules/@babel/plugin-proposal-optional-catch-binding": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz",
+      "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-catch-binding instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.18.6",
+        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2355,15 +2769,16 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-async-to-generator": {
-      "version": "7.20.7",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.20.7.tgz",
-      "integrity": "sha512-Uo5gwHPT9vgnSXQxqGtpdufUiWp96gk7yiP4Mp5bm1QMkEmLXBO7PAGYbKoJ6DhAwiNkcHFBol/x5zZZkL/t0Q==",
+    "node_modules/@babel/plugin-proposal-optional-chaining": {
+      "version": "7.21.0",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.21.0.tgz",
+      "integrity": "sha512-p4zeefM72gpmEe2fkUr/OnOXpWEf8nAgk7ZYVqqfFiyIG7oFfVZcCrU64hWn5xp4tQ9LkV4bTIa5rD0KANpKNA==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-chaining instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-module-imports": "^7.18.6",
         "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/helper-remap-async-to-generator": "^7.18.9"
+        "@babel/helper-skip-transparent-expression-wrappers": "^7.20.0",
+        "@babel/plugin-syntax-optional-chaining": "^7.8.3"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2372,13 +2787,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-block-scoped-functions": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.22.5.tgz",
-      "integrity": "sha512-tdXZ2UdknEKQWKJP1KMNmuF5Lx3MymtMN/pvA+p/VEkhK8jVcQ1fzSy8KM9qRYhAf2/lV33hoMPKI/xaI9sADA==",
+    "node_modules/@babel/plugin-proposal-private-methods": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz",
+      "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-methods instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-create-class-features-plugin": "^7.18.6",
+        "@babel/helper-plugin-utils": "^7.18.6"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2387,13 +2804,17 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-block-scoping": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.23.0.tgz",
-      "integrity": "sha512-cOsrbmIOXmf+5YbL99/S49Y3j46k/T16b9ml8bm9lP6N9US5iQ2yBK7gpui1pg0V/WMcXdkfKbTb7HXq9u+v4g==",
+    "node_modules/@babel/plugin-proposal-private-property-in-object": {
+      "version": "7.21.11",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.11.tgz",
+      "integrity": "sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-property-in-object instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-annotate-as-pure": "^7.18.6",
+        "@babel/helper-create-class-features-plugin": "^7.21.0",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/plugin-syntax-private-property-in-object": "^7.14.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2402,118 +2823,102 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-classes": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.22.15.tgz",
-      "integrity": "sha512-VbbC3PGjBdE0wAWDdHM9G8Gm977pnYI0XpqMd6LrKISj8/DJXEsWqgRuTYaNE9Bv0JGhTZUzHDlMk18IpOuoqw==",
+    "node_modules/@babel/plugin-proposal-unicode-property-regex": {
+      "version": "7.18.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz",
+      "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==",
+      "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-unicode-property-regex instead.",
       "dev": true,
       "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
-        "@babel/helper-compilation-targets": "^7.22.15",
-        "@babel/helper-environment-visitor": "^7.22.5",
-        "@babel/helper-function-name": "^7.22.5",
-        "@babel/helper-optimise-call-expression": "^7.22.5",
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-replace-supers": "^7.22.9",
-        "@babel/helper-split-export-declaration": "^7.22.6",
-        "globals": "^11.1.0"
+        "@babel/helper-create-regexp-features-plugin": "^7.18.6",
+        "@babel/helper-plugin-utils": "^7.18.6"
       },
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=4"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-classes/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@babel/plugin-syntax-async-generators": {
+      "version": "7.8.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz",
+      "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
-      "engines": {
-        "node": ">=6.9.0"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-classes/node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.22.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
-      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+    "node_modules/@babel/plugin-syntax-bigint": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz",
+      "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
-      "engines": {
-        "node": ">=6.9.0"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-computed-properties": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.22.5.tgz",
-      "integrity": "sha512-4GHWBgRf0krxPX+AaPtgBAlTgTeZmqDynokHOX7aqqAB4tHs3U2Y02zH6ETFdLZGcg9UQSD1WCmkVrE9ErHeOg==",
+    "node_modules/@babel/plugin-syntax-class-properties": {
+      "version": "7.12.13",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz",
+      "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/template": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.12.13"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-computed-properties/node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+    "node_modules/@babel/plugin-syntax-class-static-block": {
+      "version": "7.14.5",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz",
+      "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==",
       "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+        "@babel/helper-plugin-utils": "^7.14.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-destructuring": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.23.0.tgz",
-      "integrity": "sha512-vaMdgNXFkYrB+8lbgniSYWHsgqK5gjaMNcc84bMIOMRLH0L9AqYq3hwMdvnyqj1OPqea8UtjPEuS/DCenah1wg==",
+    "node_modules/@babel/plugin-syntax-dynamic-import": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz",
+      "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-dotall-regex": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.22.5.tgz",
-      "integrity": "sha512-5/Yk9QxCQCl+sOIB1WelKnVRxTJDSAIxtJLL2/pqL14ZVlbH0fUQUZa/T5/UnQtBNgghR7mfB8ERBKyKPCi7Vw==",
+    "node_modules/@babel/plugin-syntax-export-namespace-from": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz",
+      "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-regexp-features-plugin": "^7.22.5",
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.3"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-duplicate-keys": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.22.5.tgz",
-      "integrity": "sha512-dEnYD+9BBgld5VBXHnF/DbYGp3fqGMsyxKbtD1mDyIA7AkTSpKXFhCVuj/oQVOoALfBs77DudA0BE4d5mcpmqw==",
+    "node_modules/@babel/plugin-syntax-import-assertions": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.23.3.tgz",
+      "integrity": "sha512-lPgDSU+SJLK3xmFDTV2ZRQAiM7UuUjGidwBywFavObCiZc1BeAAcMtHJKUya92hPHO+at63JJPLygilZard8jw==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
@@ -2525,45 +2930,36 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-exponentiation-operator": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.22.5.tgz",
-      "integrity": "sha512-vIpJFNM/FjZ4rh1myqIya9jXwrwwgFRHPjT3DkUA9ZLHuzox8jiXkOLvwm1H+PQIP3CqfC++WPKeuDi0Sjdj1g==",
-      "dev": true,
+    "node_modules/@babel/plugin-syntax-import-meta": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz",
+      "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==",
+      "dev": true,
       "dependencies": {
-        "@babel/helper-builder-binary-assignment-operator-visitor": "^7.22.5",
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.10.4"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-for-of": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.22.15.tgz",
-      "integrity": "sha512-me6VGeHsx30+xh9fbDLLPi0J1HzmeIIyenoOQHuw2D4m2SAU3NrspX5XxJLBpqn5yrLzrlw2Iy3RA//Bx27iOA==",
+    "node_modules/@babel/plugin-syntax-json-strings": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz",
+      "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-function-name": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.22.5.tgz",
-      "integrity": "sha512-UIzQNMS0p0HHiQm3oelztj+ECwFnj+ZRV4KnguvlsD2of1whUeM6o7wGNj6oLwcDoAXQ8gEqfgC24D+VdIcevg==",
+    "node_modules/@babel/plugin-syntax-jsx": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.23.3.tgz",
+      "integrity": "sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-compilation-targets": "^7.22.5",
-        "@babel/helper-function-name": "^7.22.5",
         "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
@@ -2573,126 +2969,100 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-literals": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.22.5.tgz",
-      "integrity": "sha512-fTLj4D79M+mepcw3dgFBTIDYpbcB9Sm0bpm4ppXPaO+U+PKFFyV9MGRvS0gvGw62sd10kT5lRMKXAADb9pWy8g==",
+    "node_modules/@babel/plugin-syntax-logical-assignment-operators": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz",
+      "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.10.4"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-member-expression-literals": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.22.5.tgz",
-      "integrity": "sha512-RZEdkNtzzYCFl9SE9ATaUMTj2hqMb4StarOJLrZRbqqU4HSBE7UlBw9WBWQiDzrJZJdUWiMTVDI6Gv/8DPvfew==",
+    "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz",
+      "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-modules-amd": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.23.0.tgz",
-      "integrity": "sha512-xWT5gefv2HGSm4QHtgc1sYPbseOyf+FFDo2JbpE25GWl5BqTGO9IMwTYJRoIdjsF85GE+VegHxSCUt5EvoYTAw==",
+    "node_modules/@babel/plugin-syntax-numeric-separator": {
+      "version": "7.10.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz",
+      "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-module-transforms": "^7.23.0",
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.10.4"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-modules-commonjs": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.23.0.tgz",
-      "integrity": "sha512-32Xzss14/UVc7k9g775yMIvkVK8xwKE0DPdP5JTapr3+Z9w4tzeOuLNY6BXDQR6BdnzIlXnCGAzsk/ICHBLVWQ==",
+    "node_modules/@babel/plugin-syntax-object-rest-spread": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz",
+      "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-module-transforms": "^7.23.0",
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-simple-access": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-modules-systemjs": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.23.0.tgz",
-      "integrity": "sha512-qBej6ctXZD2f+DhlOC9yO47yEYgUh5CZNz/aBoH4j/3NOlRfJXJbY7xDQCqQVf9KbrqGzIWER1f23doHGrIHFg==",
+    "node_modules/@babel/plugin-syntax-optional-catch-binding": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz",
+      "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-hoist-variables": "^7.22.5",
-        "@babel/helper-module-transforms": "^7.23.0",
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-validator-identifier": "^7.22.20"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-modules-umd": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.22.5.tgz",
-      "integrity": "sha512-+S6kzefN/E1vkSsKx8kmQuqeQsvCKCd1fraCM7zXm4SFoggI099Tr4G8U81+5gtMdUeMQ4ipdQffbKLX0/7dBQ==",
+    "node_modules/@babel/plugin-syntax-optional-chaining": {
+      "version": "7.8.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz",
+      "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-module-transforms": "^7.22.5",
-        "@babel/helper-plugin-utils": "^7.22.5"
-      },
-      "engines": {
-        "node": ">=6.9.0"
+        "@babel/helper-plugin-utils": "^7.8.0"
       },
       "peerDependencies": {
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-named-capturing-groups-regex": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.22.5.tgz",
-      "integrity": "sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==",
+    "node_modules/@babel/plugin-syntax-private-property-in-object": {
+      "version": "7.14.5",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz",
+      "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-create-regexp-features-plugin": "^7.22.5",
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.14.5"
       },
       "engines": {
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-new-target": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.22.5.tgz",
-      "integrity": "sha512-AsF7K0Fx/cNKVyk3a+DW0JLo+Ua598/NxMRvxDnkpCIGFh43+h/v2xyhRUYf6oD8gE4QtL83C7zZVghMjHd+iw==",
+    "node_modules/@babel/plugin-syntax-top-level-await": {
+      "version": "7.14.5",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz",
+      "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.14.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2701,14 +3071,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-object-super": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.22.5.tgz",
-      "integrity": "sha512-klXqyaT9trSjIUrcsYIfETAzmOEZL3cBYqOYLJxBHfMFFggmXOv+NYSX/Jbs9mzMVESw/WycLFPRx8ba/b2Ipw==",
+    "node_modules/@babel/plugin-syntax-typescript": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.23.3.tgz",
+      "integrity": "sha512-9EiNjVJOMwCO+43TqoTrgQ8jMwcAd0sWyXi9RPfIsLTj4R2MADDDQXELhffaUx/uJv2AYcxBgPwH6j4TIA4ytQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-replace-supers": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2717,15 +3086,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-optional-chaining": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.23.0.tgz",
-      "integrity": "sha512-sBBGXbLJjxTzLBF5rFWaikMnOGOk/BmK6vVByIdEggZ7Vn6CvWXZyRkkLFK6WE0IF8jSliyOkUN6SScFgzCM0g==",
+    "node_modules/@babel/plugin-transform-arrow-functions": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.23.3.tgz",
+      "integrity": "sha512-NzQcQrzaQPkaEwoTm4Mhyl8jI1huEL/WWIEvudjTCMJ9aBZNpsJbMASx7EQECtQQPS/DcnFpo0FIh3LvEO9cxQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
-        "@babel/plugin-syntax-optional-chaining": "^7.8.3"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2734,13 +3101,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-parameters": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.22.15.tgz",
-      "integrity": "sha512-hjk7qKIqhyzhhUvRT683TYQOFa/4cQKwQy7ALvTpODswN40MljzNDa0YldevS6tGbxwaEKVn502JmY0dP7qEtQ==",
+    "node_modules/@babel/plugin-transform-async-to-generator": {
+      "version": "7.20.7",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.20.7.tgz",
+      "integrity": "sha512-Uo5gwHPT9vgnSXQxqGtpdufUiWp96gk7yiP4Mp5bm1QMkEmLXBO7PAGYbKoJ6DhAwiNkcHFBol/x5zZZkL/t0Q==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-module-imports": "^7.18.6",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/helper-remap-async-to-generator": "^7.18.9"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2749,10 +3118,10 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-property-literals": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.22.5.tgz",
-      "integrity": "sha512-TiOArgddK3mK/x1Qwf5hay2pxI6wCZnvQqrFSqbtg1GLl2JcNMitVH/YnqjP+M31pLUeTfzY1HAXFDnUBV30rQ==",
+    "node_modules/@babel/plugin-transform-block-scoped-functions": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.23.3.tgz",
+      "integrity": "sha512-vI+0sIaPIO6CNuM9Kk5VmXcMVRiOpDh7w2zZt9GXzmE/9KD70CUEVhvPR/etAeNK/FAEkhxQtXOzVF3EuRL41A==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
@@ -2764,10 +3133,10 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-display-name": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.22.5.tgz",
-      "integrity": "sha512-PVk3WPYudRF5z4GKMEYUrLjPl38fJSKNaEOkFuoprioowGuWN6w2RKznuFNSlJx7pzzXXStPUnNSOEO0jL5EVw==",
+    "node_modules/@babel/plugin-transform-block-scoping": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.23.4.tgz",
+      "integrity": "sha512-0QqbP6B6HOh7/8iNR4CQU2Th/bbRtBp4KS9vcaZd1fZ0wSh5Fyssg0UCIHwxh+ka+pNDREbVLQnHCMHKZfPwfw==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
@@ -2779,17 +3148,21 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-jsx": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.22.15.tgz",
-      "integrity": "sha512-oKckg2eZFa8771O/5vi7XeTvmM6+O9cxZu+kanTU7tD4sin5nO/G8jGJhq8Hvt2Z0kUoEDRayuZLaUlYl8QuGA==",
+    "node_modules/@babel/plugin-transform-classes": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.23.3.tgz",
+      "integrity": "sha512-FGEQmugvAEu2QtgtU0uTASXevfLMFfBeVCIIdcQhn/uBQsMTjBajdnAtanQlOcuihWh10PZ7+HWvc7NtBwP74w==",
       "dev": true,
       "dependencies": {
         "@babel/helper-annotate-as-pure": "^7.22.5",
-        "@babel/helper-module-imports": "^7.22.15",
+        "@babel/helper-compilation-targets": "^7.22.15",
+        "@babel/helper-environment-visitor": "^7.22.20",
+        "@babel/helper-function-name": "^7.23.0",
+        "@babel/helper-optimise-call-expression": "^7.22.5",
         "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/plugin-syntax-jsx": "^7.22.5",
-        "@babel/types": "^7.22.15"
+        "@babel/helper-replace-supers": "^7.22.20",
+        "@babel/helper-split-export-declaration": "^7.22.6",
+        "globals": "^11.1.0"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2798,40 +3171,40 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-jsx-development": {
+    "node_modules/@babel/plugin-transform-classes/node_modules/@babel/helper-annotate-as-pure": {
       "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.22.5.tgz",
-      "integrity": "sha512-bDhuzwWMuInwCYeDeMzyi7TaBgRQei6DqxhbyniL7/VG4RSS7HtSL2QbY4eESy1KJqlWt8g3xeEBGPuo+XqC8A==",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
       "dev": true,
       "dependencies": {
-        "@babel/plugin-transform-react-jsx": "^7.22.5"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-jsx/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@babel/plugin-transform-computed-properties": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.23.3.tgz",
+      "integrity": "sha512-dTj83UVTLw/+nbiHqQSFdwO9CbTtwq1DsDqm3CUEtDrZNET5rT5E6bIdTlOftDTDLMYxvxHNEYO4B9SLl8SLZw==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/template": "^7.22.15"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-pure-annotations": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.22.5.tgz",
-      "integrity": "sha512-gP4k85wx09q+brArVinTXhWiyzLl9UpmGva0+mWyKxk6JZequ05x3eUcIUE+FyttPKJFRRVtAvQaJ6YF9h1ZpA==",
+    "node_modules/@babel/plugin-transform-destructuring": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.23.3.tgz",
+      "integrity": "sha512-n225npDqjDIr967cMScVKHXJs7rout1q+tt50inyBCPkyZ8KxeI6d+GIbSBTT/w/9WdlWDOej3V9HE5Lgk57gw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
         "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
@@ -2841,26 +3214,29 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-react-pure-annotations/node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+    "node_modules/@babel/plugin-transform-dotall-regex": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.23.3.tgz",
+      "integrity": "sha512-vgnFYDHAKzFaTVp+mneDsIEbnJ2Np/9ng9iviHw3P/KVcgONxpNULEW/51Z/BaFojG2GI2GwwXck5uV1+1NOYQ==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@babel/helper-create-regexp-features-plugin": "^7.22.15",
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-regenerator": {
-      "version": "7.22.10",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.22.10.tgz",
-      "integrity": "sha512-F28b1mDt8KcT5bUyJc/U9nwzw6cV+UmTeRlXYIl2TNqMMJif0Jeey9/RQ3C4NOd2zp0/TRsDns9ttj2L523rsw==",
+    "node_modules/@babel/plugin-transform-duplicate-keys": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.23.3.tgz",
+      "integrity": "sha512-RrqQ+BQmU3Oyav3J+7/myfvRCq7Tbz+kKLLshUmMwNlDHExbGL7ARhajvoBJEvc+fCguPPu887N+3RRXBVKZUA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "regenerator-transform": "^0.15.2"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2869,12 +3245,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-reserved-words": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.22.5.tgz",
-      "integrity": "sha512-DTtGKFRQUDm8svigJzZHzb/2xatPc6TzNvAIJ5GqOKDsGFYgAskjRulbR/vGsPKq3OPqtexnz327qYpP57RFyA==",
+    "node_modules/@babel/plugin-transform-exponentiation-operator": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.23.3.tgz",
+      "integrity": "sha512-5fhCsl1odX96u7ILKHBj4/Y8vipoqwsJMh4csSA8qFfxrZDEA4Ssku2DyNvMJSmZNOEBT750LfFPbtrnTP90BQ==",
       "dev": true,
       "dependencies": {
+        "@babel/helper-builder-binary-assignment-operator-visitor": "^7.22.15",
         "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
@@ -2884,18 +3261,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-runtime": {
-      "version": "7.19.6",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.19.6.tgz",
-      "integrity": "sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw==",
+    "node_modules/@babel/plugin-transform-for-of": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.23.3.tgz",
+      "integrity": "sha512-X8jSm8X1CMwxmK878qsUGJRmbysKNbdpTv/O1/v0LuY/ZkZrng5WYiekYSdg9m09OTmDDUWeEDsTE+17WYbAZw==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-module-imports": "^7.18.6",
-        "@babel/helper-plugin-utils": "^7.19.0",
-        "babel-plugin-polyfill-corejs2": "^0.3.3",
-        "babel-plugin-polyfill-corejs3": "^0.6.0",
-        "babel-plugin-polyfill-regenerator": "^0.4.1",
-        "semver": "^6.3.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2904,19 +3276,27 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-runtime/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/@babel/plugin-transform-function-name": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.23.3.tgz",
+      "integrity": "sha512-I1QXp1LxIvt8yLaib49dRW5Okt7Q4oaxao6tFVKS/anCdEOMtYwWVKoiOA1p34GOWIZjUK0E+zCp7+l1pfQyiw==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+      "dependencies": {
+        "@babel/helper-compilation-targets": "^7.22.15",
+        "@babel/helper-function-name": "^7.23.0",
+        "@babel/helper-plugin-utils": "^7.22.5"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-shorthand-properties": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.22.5.tgz",
-      "integrity": "sha512-vM4fq9IXHscXVKzDv5itkO1X52SmdFBFcMIBZ2FRn2nqVYqw6dBexUgMvAjHW+KXpPPViD/Yo3GrDEBaRC0QYA==",
+    "node_modules/@babel/plugin-transform-literals": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.23.3.tgz",
+      "integrity": "sha512-wZ0PIXRxnwZvl9AYpqNUxpZ5BiTGrYt7kueGQ+N5FiQ7RCOD4cm8iShd6S6ggfVIWaJf2EMk8eRzAh52RfP4rQ==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5"
@@ -2928,14 +3308,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-spread": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.22.5.tgz",
-      "integrity": "sha512-5ZzDQIGyvN4w8+dMmpohL6MBo+l2G7tfC/O2Dg7/hjpgeWvUx8FzfeOKxGog9IimPa4YekaQ9PlDqTLOljkcxg==",
+    "node_modules/@babel/plugin-transform-member-expression-literals": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.23.3.tgz",
+      "integrity": "sha512-sC3LdDBDi5x96LA+Ytekz2ZPk8i/Ck+DEuDbRAll5rknJ5XRTSaPKEYwomLcs1AA8wg9b3KjIQRsnApj+q51Ag==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2944,12 +3323,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-sticky-regex": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.22.5.tgz",
-      "integrity": "sha512-zf7LuNpHG0iEeiyCNwX4j3gDg1jgt1k3ZdXBKbZSoA3BbGQGvMiSvfbZRR3Dr3aeJe3ooWFZxOOG3IRStYp2Bw==",
+    "node_modules/@babel/plugin-transform-modules-amd": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.23.3.tgz",
+      "integrity": "sha512-vJYQGxeKM4t8hYCKVBlZX/gtIY2I7mRGFNcm85sgXGMTBcoV3QdVtdpbcWEbzbfUIUZKwvgFT82mRvaQIebZzw==",
       "dev": true,
       "dependencies": {
+        "@babel/helper-module-transforms": "^7.23.3",
         "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
@@ -2959,13 +3339,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-template-literals": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.22.5.tgz",
-      "integrity": "sha512-5ciOehRNf+EyUeewo8NkbQiUs4d6ZxiHo6BcBcnFlgiJfu16q0bQUw9Jvo0b0gBKFG1SMhDSjeKXSYuJLeFSMA==",
+    "node_modules/@babel/plugin-transform-modules-commonjs": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.23.3.tgz",
+      "integrity": "sha512-aVS0F65LKsdNOtcz6FRCpE4OgsP2OFnW46qNxNIX9h3wuzaNcSQsJysuMwqSibC98HPrf2vCgtxKNwS0DAlgcA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-module-transforms": "^7.23.3",
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-simple-access": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2974,13 +3356,16 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-typeof-symbol": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.22.5.tgz",
-      "integrity": "sha512-bYkI5lMzL4kPii4HHEEChkD0rkc+nvnlR6+o/qdqR6zrm0Sv/nodmyLhlq2DO0YKLUNd2VePmPRjJXSBh9OIdA==",
+    "node_modules/@babel/plugin-transform-modules-systemjs": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.23.3.tgz",
+      "integrity": "sha512-ZxyKGTkF9xT9YJuKQRo19ewf3pXpopuYQd8cDXqNzc3mUNbOME0RKMoZxviQk74hwzfQsEe66dE92MaZbdHKNQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.22.5"
+        "@babel/helper-hoist-variables": "^7.22.5",
+        "@babel/helper-module-transforms": "^7.23.3",
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-validator-identifier": "^7.22.20"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2989,12 +3374,13 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-unicode-escapes": {
-      "version": "7.22.10",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.22.10.tgz",
-      "integrity": "sha512-lRfaRKGZCBqDlRU3UIFovdp9c9mEvlylmpod0/OatICsSfuQ9YFthRo1tpTkGsklEefZdqlEFdY4A2dwTb6ohg==",
+    "node_modules/@babel/plugin-transform-modules-umd": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.23.3.tgz",
+      "integrity": "sha512-zHsy9iXX2nIsCBFPud3jKn1IRPWg3Ing1qOZgeKV39m1ZgIdpJqvlWVeiHBZC6ITRG0MfskhYe9cLgntfSFPIg==",
       "dev": true,
       "dependencies": {
+        "@babel/helper-module-transforms": "^7.23.3",
         "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
@@ -3004,10 +3390,10 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/plugin-transform-unicode-regex": {
+    "node_modules/@babel/plugin-transform-named-capturing-groups-regex": {
       "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.22.5.tgz",
-      "integrity": "sha512-028laaOKptN5vHJf9/Arr/HiJekMd41hOEZYvNsrsXqJ7YPYuX2bQxh31fkZzGmq3YqHRJzYFFAVYvKfMPKqyg==",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.22.5.tgz",
+      "integrity": "sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==",
       "dev": true,
       "dependencies": {
         "@babel/helper-create-regexp-features-plugin": "^7.22.5",
@@ -3017,90 +3403,16 @@
         "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@babel/preset-env": {
-      "version": "7.20.2",
-      "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.20.2.tgz",
-      "integrity": "sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==",
+    "node_modules/@babel/plugin-transform-new-target": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.23.3.tgz",
+      "integrity": "sha512-YJ3xKqtJMAT5/TIZnpAR3I+K+WaDowYbN3xyxI8zxx/Gsypwf9B9h0VB+1Nh6ACAAPRS5NSRje0uVv5i79HYGQ==",
       "dev": true,
       "dependencies": {
-        "@babel/compat-data": "^7.20.1",
-        "@babel/helper-compilation-targets": "^7.20.0",
-        "@babel/helper-plugin-utils": "^7.20.2",
-        "@babel/helper-validator-option": "^7.18.6",
-        "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6",
-        "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.9",
-        "@babel/plugin-proposal-async-generator-functions": "^7.20.1",
-        "@babel/plugin-proposal-class-properties": "^7.18.6",
-        "@babel/plugin-proposal-class-static-block": "^7.18.6",
-        "@babel/plugin-proposal-dynamic-import": "^7.18.6",
-        "@babel/plugin-proposal-export-namespace-from": "^7.18.9",
-        "@babel/plugin-proposal-json-strings": "^7.18.6",
-        "@babel/plugin-proposal-logical-assignment-operators": "^7.18.9",
-        "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
-        "@babel/plugin-proposal-numeric-separator": "^7.18.6",
-        "@babel/plugin-proposal-object-rest-spread": "^7.20.2",
-        "@babel/plugin-proposal-optional-catch-binding": "^7.18.6",
-        "@babel/plugin-proposal-optional-chaining": "^7.18.9",
-        "@babel/plugin-proposal-private-methods": "^7.18.6",
-        "@babel/plugin-proposal-private-property-in-object": "^7.18.6",
-        "@babel/plugin-proposal-unicode-property-regex": "^7.18.6",
-        "@babel/plugin-syntax-async-generators": "^7.8.4",
-        "@babel/plugin-syntax-class-properties": "^7.12.13",
-        "@babel/plugin-syntax-class-static-block": "^7.14.5",
-        "@babel/plugin-syntax-dynamic-import": "^7.8.3",
-        "@babel/plugin-syntax-export-namespace-from": "^7.8.3",
-        "@babel/plugin-syntax-import-assertions": "^7.20.0",
-        "@babel/plugin-syntax-json-strings": "^7.8.3",
-        "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4",
-        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3",
-        "@babel/plugin-syntax-numeric-separator": "^7.10.4",
-        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
-        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
-        "@babel/plugin-syntax-optional-chaining": "^7.8.3",
-        "@babel/plugin-syntax-private-property-in-object": "^7.14.5",
-        "@babel/plugin-syntax-top-level-await": "^7.14.5",
-        "@babel/plugin-transform-arrow-functions": "^7.18.6",
-        "@babel/plugin-transform-async-to-generator": "^7.18.6",
-        "@babel/plugin-transform-block-scoped-functions": "^7.18.6",
-        "@babel/plugin-transform-block-scoping": "^7.20.2",
-        "@babel/plugin-transform-classes": "^7.20.2",
-        "@babel/plugin-transform-computed-properties": "^7.18.9",
-        "@babel/plugin-transform-destructuring": "^7.20.2",
-        "@babel/plugin-transform-dotall-regex": "^7.18.6",
-        "@babel/plugin-transform-duplicate-keys": "^7.18.9",
-        "@babel/plugin-transform-exponentiation-operator": "^7.18.6",
-        "@babel/plugin-transform-for-of": "^7.18.8",
-        "@babel/plugin-transform-function-name": "^7.18.9",
-        "@babel/plugin-transform-literals": "^7.18.9",
-        "@babel/plugin-transform-member-expression-literals": "^7.18.6",
-        "@babel/plugin-transform-modules-amd": "^7.19.6",
-        "@babel/plugin-transform-modules-commonjs": "^7.19.6",
-        "@babel/plugin-transform-modules-systemjs": "^7.19.6",
-        "@babel/plugin-transform-modules-umd": "^7.18.6",
-        "@babel/plugin-transform-named-capturing-groups-regex": "^7.19.1",
-        "@babel/plugin-transform-new-target": "^7.18.6",
-        "@babel/plugin-transform-object-super": "^7.18.6",
-        "@babel/plugin-transform-parameters": "^7.20.1",
-        "@babel/plugin-transform-property-literals": "^7.18.6",
-        "@babel/plugin-transform-regenerator": "^7.18.6",
-        "@babel/plugin-transform-reserved-words": "^7.18.6",
-        "@babel/plugin-transform-shorthand-properties": "^7.18.6",
-        "@babel/plugin-transform-spread": "^7.19.0",
-        "@babel/plugin-transform-sticky-regex": "^7.18.6",
-        "@babel/plugin-transform-template-literals": "^7.18.9",
-        "@babel/plugin-transform-typeof-symbol": "^7.18.9",
-        "@babel/plugin-transform-unicode-escapes": "^7.18.10",
-        "@babel/plugin-transform-unicode-regex": "^7.18.6",
-        "@babel/preset-modules": "^0.1.5",
-        "@babel/types": "^7.20.2",
-        "babel-plugin-polyfill-corejs2": "^0.3.3",
-        "babel-plugin-polyfill-corejs3": "^0.6.0",
-        "babel-plugin-polyfill-regenerator": "^0.4.1",
-        "core-js-compat": "^3.25.1",
-        "semver": "^6.3.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -3109,43 +3421,31 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/preset-env/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
-    "node_modules/@babel/preset-modules": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.6.tgz",
-      "integrity": "sha512-ID2yj6K/4lKfhuU3+EX4UvNbIt7eACFbHmNUjzA+ep+B5971CknnA/9DEWKbRokfbbtblxxxXFJJrH47UEAMVg==",
+    "node_modules/@babel/plugin-transform-object-super": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.23.3.tgz",
+      "integrity": "sha512-BwQ8q0x2JG+3lxCVFohg+KbQM7plfpBwThdW9A6TMtWwLsbDA01Ek2Zb/AgDN39BiZsExm4qrXxjk+P1/fzGrA==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.0.0",
-        "@babel/plugin-proposal-unicode-property-regex": "^7.4.4",
-        "@babel/plugin-transform-dotall-regex": "^7.4.4",
-        "@babel/types": "^7.4.4",
-        "esutils": "^2.0.2"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-replace-supers": "^7.22.20"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0-0 || ^8.0.0-0 <8.0.0"
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/preset-react": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.22.15.tgz",
-      "integrity": "sha512-Csy1IJ2uEh/PecCBXXoZGAZBeCATTuePzCSB7dLYWS0vOEj6CNpjxIhW4duWwZodBNueH7QO14WbGn8YyeuN9w==",
+    "node_modules/@babel/plugin-transform-optional-chaining": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.23.4.tgz",
+      "integrity": "sha512-ZU8y5zWOfjM5vZ+asjgAPwDaBjJzgufjES89Rs4Lpq63O300R/kOz30WCLo6BxxX6QVEilwSlpClnG5cZaikTA==",
       "dev": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.22.5",
-        "@babel/helper-validator-option": "^7.22.15",
-        "@babel/plugin-transform-react-display-name": "^7.22.5",
-        "@babel/plugin-transform-react-jsx": "^7.22.15",
-        "@babel/plugin-transform-react-jsx-development": "^7.22.5",
-        "@babel/plugin-transform-react-pure-annotations": "^7.22.5"
+        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5",
+        "@babel/plugin-syntax-optional-chaining": "^7.8.3"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -3154,91 +3454,90 @@
         "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/regjsgen": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/@babel/regjsgen/-/regjsgen-0.8.0.tgz",
-      "integrity": "sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==",
-      "dev": true
-    },
-    "node_modules/@babel/runtime": {
-      "version": "7.20.13",
-      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.13.tgz",
-      "integrity": "sha512-gt3PKXs0DBoL9xCvOIIZ2NEqAGZqHjAnmVbfQtB620V0uReIQutpel14KcneZuer7UioY8ALKZ7iocavvzTNFA==",
+    "node_modules/@babel/plugin-transform-parameters": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.23.3.tgz",
+      "integrity": "sha512-09lMt6UsUb3/34BbECKVbVwrT9bO6lILWln237z7sLaWnMsTi7Yc9fhX5DLpkJzAGfaReXI22wP41SZmnAA3Vw==",
+      "dev": true,
       "dependencies": {
-        "regenerator-runtime": "^0.13.11"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/runtime-corejs3": {
-      "version": "7.23.1",
-      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.23.1.tgz",
-      "integrity": "sha512-OKKfytwoc0tr7cDHwQm0RLVR3y+hDGFz3EPuvLNU/0fOeXJeKNIHj7ffNVFnncWt3sC58uyUCRSzf8nBQbyF6A==",
+    "node_modules/@babel/plugin-transform-property-literals": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.23.3.tgz",
+      "integrity": "sha512-jR3Jn3y7cZp4oEWPFAlRsSWjxKe4PZILGBSd4nis1TsC5qeSpb+nrtihJuDhNI7QHiVbUaiXa0X2RZY3/TI6Nw==",
+      "dev": true,
       "dependencies": {
-        "core-js-pure": "^3.30.2",
-        "regenerator-runtime": "^0.14.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/runtime-corejs3/node_modules/regenerator-runtime": {
-      "version": "0.14.0",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz",
-      "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA=="
-    },
-    "node_modules/@babel/template": {
-      "version": "7.20.7",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.20.7.tgz",
-      "integrity": "sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==",
+    "node_modules/@babel/plugin-transform-react-display-name": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.23.3.tgz",
+      "integrity": "sha512-GnvhtVfA2OAtzdX58FJxU19rhoGeQzyVndw3GgtdECQvQFXPEZIOVULHVZGAYmOgmqjXpVpfocAbSjh99V/Fqw==",
+      "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.18.6",
-        "@babel/parser": "^7.20.7",
-        "@babel/types": "^7.20.7"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/traverse": {
-      "version": "7.23.2",
-      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz",
-      "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==",
+    "node_modules/@babel/plugin-transform-react-jsx": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.23.4.tgz",
+      "integrity": "sha512-5xOpoPguCZCRbo/JeHlloSkTA8Bld1J/E1/kLfD1nsuiW1m8tduTA1ERCgIZokDflX/IBzKcqR3l7VlRgiIfHA==",
+      "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/generator": "^7.23.0",
-        "@babel/helper-environment-visitor": "^7.22.20",
-        "@babel/helper-function-name": "^7.23.0",
-        "@babel/helper-hoist-variables": "^7.22.5",
-        "@babel/helper-split-export-declaration": "^7.22.6",
-        "@babel/parser": "^7.23.0",
-        "@babel/types": "^7.23.0",
-        "debug": "^4.1.0",
-        "globals": "^11.1.0"
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "@babel/helper-module-imports": "^7.22.15",
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/plugin-syntax-jsx": "^7.23.3",
+        "@babel/types": "^7.23.4"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/traverse/node_modules/@babel/generator": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz",
-      "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==",
+    "node_modules/@babel/plugin-transform-react-jsx-development": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.22.5.tgz",
+      "integrity": "sha512-bDhuzwWMuInwCYeDeMzyi7TaBgRQei6DqxhbyniL7/VG4RSS7HtSL2QbY4eESy1KJqlWt8g3xeEBGPuo+XqC8A==",
+      "dev": true,
       "dependencies": {
-        "@babel/types": "^7.23.0",
-        "@jridgewell/gen-mapping": "^0.3.2",
-        "@jridgewell/trace-mapping": "^0.3.17",
-        "jsesc": "^2.5.1"
+        "@babel/plugin-transform-react-jsx": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/traverse/node_modules/@babel/helper-split-export-declaration": {
-      "version": "7.22.6",
-      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz",
-      "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==",
+    "node_modules/@babel/plugin-transform-react-jsx/node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+      "dev": true,
       "dependencies": {
         "@babel/types": "^7.22.5"
       },
@@ -3246,1990 +3545,1956 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/traverse/node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
-      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+    "node_modules/@babel/plugin-transform-react-pure-annotations": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.23.3.tgz",
+      "integrity": "sha512-qMFdSS+TUhB7Q/3HVPnEdYJDQIk57jkntAwSuz9xfSE4n+3I+vHYCli3HoHawN1Z3RfCz/y1zXA/JXjG6cVImQ==",
+      "dev": true,
       "dependencies": {
-        "@jridgewell/set-array": "^1.0.1",
-        "@jridgewell/sourcemap-codec": "^1.4.10",
-        "@jridgewell/trace-mapping": "^0.3.9"
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@babel/types": {
-      "version": "7.23.0",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz",
-      "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==",
+    "node_modules/@babel/plugin-transform-react-pure-annotations/node_modules/@babel/helper-annotate-as-pure": {
+      "version": "7.22.5",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
+      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+      "dev": true,
       "dependencies": {
-        "@babel/helper-string-parser": "^7.22.5",
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "to-fast-properties": "^2.0.0"
+        "@babel/types": "^7.22.5"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@bcoe/v8-coverage": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz",
-      "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
-      "dev": true
-    },
-    "node_modules/@braintree/sanitize-url": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.0.tgz",
-      "integrity": "sha512-mgmE7XBYY/21erpzhexk4Cj1cyTQ9LzvnTxtzM17BJ7ERMNE6W72mQRo0I1Ud8eFJ+RVVIcBNhLFZ3GX4XFz5w=="
-    },
-    "node_modules/@circlon/angular-tree-component": {
-      "version": "10.0.0",
-      "resolved": "https://registry.npmjs.org/@circlon/angular-tree-component/-/angular-tree-component-10.0.0.tgz",
-      "integrity": "sha512-3dRWLbOdMfIuvZjX6AMHmvzPtqhNFECMWMpNVXrZfZtTAa0n+Y4lxbuLST85q5QiedBZuC720p/7kkZ78PJ+iw==",
+    "node_modules/@babel/plugin-transform-regenerator": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.23.3.tgz",
+      "integrity": "sha512-KP+75h0KghBMcVpuKisx3XTu9Ncut8Q8TuvGO4IhY+9D5DFEckQefOuIsB/gQ2tG71lCke4NMrtIPS8pOj18BQ==",
+      "dev": true,
       "dependencies": {
-        "lodash-es": "^4.17.15",
-        "mobx": "~4.14.1",
-        "tslib": "^2.0.0"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "regenerator-transform": "^0.15.2"
+      },
+      "engines": {
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "@angular/common": ">=10.0.0 <11.0.0",
-        "@angular/core": ">=10.0.0 <11.0.0"
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@colors/colors": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
-      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==",
+    "node_modules/@babel/plugin-transform-reserved-words": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.23.3.tgz",
+      "integrity": "sha512-QnNTazY54YqgGxwIexMZva9gqbPa15t/x9VS+0fsEFWplwVpXYZivtgl43Z1vMpc1bdPP2PP8siFeVcnFvA3Cg==",
       "dev": true,
-      "optional": true,
-      "engines": {
-        "node": ">=0.1.90"
-      }
-    },
-    "node_modules/@compodoc/compodoc": {
-      "version": "1.1.18",
-      "resolved": "https://registry.npmjs.org/@compodoc/compodoc/-/compodoc-1.1.18.tgz",
-      "integrity": "sha512-+AFtcj2U3AJq6r8a2+PTdajIlS7m3pgvDhqgoYZJ4Rg/Zp9xvuDvUJU+5oHu8iHCAWwda3NoLUDjOZMNR8uIKg==",
-      "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@angular-devkit/schematics": "^13.1.2",
-        "@babel/core": "^7.16.7",
-        "@babel/preset-env": "^7.16.7",
-        "@compodoc/live-server": "^1.2.3",
-        "@compodoc/ngd-transformer": "^2.1.0",
-        "chalk": "^4.1.2",
-        "cheerio": "^1.0.0-rc.10",
-        "chokidar": "^3.5.2",
-        "colors": "1.4.0",
-        "commander": "^8.3.0",
-        "cosmiconfig": "^7.0.1",
-        "decache": "^4.6.1",
-        "fancy-log": "^2.0.0",
-        "findit2": "^2.2.3",
-        "fs-extra": "^10.0.0",
-        "glob": "^7.2.0",
-        "handlebars": "^4.7.7",
-        "html-entities": "^2.3.2",
-        "i18next": "^21.6.5",
-        "inside": "^1.0.0",
-        "json5": "^2.2.0",
-        "lodash": "^4.17.21",
-        "loglevel": "^1.8.0",
-        "loglevel-plugin-prefix": "^0.8.4",
-        "lunr": "^2.3.9",
-        "marked": "^4.0.9",
-        "minimist": "^1.2.5",
-        "opencollective-postinstall": "^2.0.3",
-        "os-name": "4.0.1",
-        "pdfjs-dist": "^2.12.313",
-        "pdfmake": "^0.2.4",
-        "semver": "^7.3.5",
-        "traverse": "^0.6.6",
-        "ts-morph": "^13.0.2",
-        "uuid": "^8.3.2"
-      },
-      "bin": {
-        "compodoc": "bin/index-cli.js"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": ">= 12.0.0"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/@angular-devkit/core": {
-      "version": "13.3.11",
-      "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-13.3.11.tgz",
-      "integrity": "sha512-rfqoLMRYhlz0wzKlHx7FfyIyQq8dKTsmbCoIVU1cEIH0gyTMVY7PbVzwRRcO6xp5waY+0hA+0Brriujpuhkm4w==",
+    "node_modules/@babel/plugin-transform-runtime": {
+      "version": "7.19.6",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.19.6.tgz",
+      "integrity": "sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw==",
       "dev": true,
       "dependencies": {
-        "ajv": "8.9.0",
-        "ajv-formats": "2.1.1",
-        "fast-json-stable-stringify": "2.1.0",
-        "magic-string": "0.25.7",
-        "rxjs": "6.6.7",
-        "source-map": "0.7.3"
+        "@babel/helper-module-imports": "^7.18.6",
+        "@babel/helper-plugin-utils": "^7.19.0",
+        "babel-plugin-polyfill-corejs2": "^0.3.3",
+        "babel-plugin-polyfill-corejs3": "^0.6.0",
+        "babel-plugin-polyfill-regenerator": "^0.4.1",
+        "semver": "^6.3.0"
       },
       "engines": {
-        "node": "^12.20.0 || ^14.15.0 || >=16.10.0",
-        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
-        "yarn": ">= 1.13.0"
+        "node": ">=6.9.0"
       },
       "peerDependencies": {
-        "chokidar": "^3.5.2"
-      },
-      "peerDependenciesMeta": {
-        "chokidar": {
-          "optional": true
-        }
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/@angular-devkit/schematics": {
-      "version": "13.3.11",
-      "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-13.3.11.tgz",
-      "integrity": "sha512-ben+EGXpCrClnIVAAnEQmhQdKmnnqFhMp5BqMxgOslSYBAmCutLA6rBu5vsc8kZcGian1wt+lueF7G1Uk5cGBg==",
+    "node_modules/@babel/plugin-transform-shorthand-properties": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.23.3.tgz",
+      "integrity": "sha512-ED2fgqZLmexWiN+YNFX26fx4gh5qHDhn1O2gvEhreLW2iI63Sqm4llRLCXALKrCnbN4Jy0VcMQZl/SAzqug/jg==",
       "dev": true,
       "dependencies": {
-        "@angular-devkit/core": "13.3.11",
-        "jsonc-parser": "3.0.0",
-        "magic-string": "0.25.7",
-        "ora": "5.4.1",
-        "rxjs": "6.6.7"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": "^12.20.0 || ^14.15.0 || >=16.10.0",
-        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
-        "yarn": ">= 1.13.0"
-      }
-    },
-    "node_modules/@compodoc/compodoc/node_modules/ajv": {
-      "version": "8.9.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
-      "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
-      "dev": true,
-      "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "json-schema-traverse": "^1.0.0",
-        "require-from-string": "^2.0.2",
-        "uri-js": "^4.2.2"
+        "node": ">=6.9.0"
       },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@babel/plugin-transform-spread": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.23.3.tgz",
+      "integrity": "sha512-VvfVYlrlBVu+77xVTOAoxQ6mZbnIq5FM0aGBSFEcIh03qHf+zNqA4DC/3XMUozTg7bZV3e3mZQ0i13VB6v5yUg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-skip-transparent-expression-wrappers": "^7.22.5"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=6.9.0"
       },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/cosmiconfig": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
-      "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==",
+    "node_modules/@babel/plugin-transform-sticky-regex": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.23.3.tgz",
+      "integrity": "sha512-HZOyN9g+rtvnOU3Yh7kSxXrKbzgrm5X4GncPY1QOquu7epga5MxKHVpYu2hvQnry/H+JjckSYRb93iNfsioAGg==",
       "dev": true,
       "dependencies": {
-        "@types/parse-json": "^4.0.0",
-        "import-fresh": "^3.2.1",
-        "parse-json": "^5.0.0",
-        "path-type": "^4.0.0",
-        "yaml": "^1.10.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/@babel/plugin-transform-template-literals": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.23.3.tgz",
+      "integrity": "sha512-Flok06AYNp7GV2oJPZZcP9vZdszev6vPBkHLwxwSpaIqx75wn6mUd3UFWsSsA0l8nXAKkyCmL/sR02m8RYGeHg==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": "*"
+        "node": ">=6.9.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
-    "node_modules/@compodoc/compodoc/node_modules/jsonc-parser": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz",
-      "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==",
-      "dev": true
-    },
-    "node_modules/@compodoc/compodoc/node_modules/magic-string": {
-      "version": "0.25.7",
-      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz",
-      "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==",
-      "dev": true,
-      "dependencies": {
-        "sourcemap-codec": "^1.4.4"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/rxjs": {
-      "version": "6.6.7",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz",
-      "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==",
+    "node_modules/@babel/plugin-transform-typeof-symbol": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.23.3.tgz",
+      "integrity": "sha512-4t15ViVnaFdrPC74be1gXBSMzXk3B4Us9lP7uLRQHTFpV5Dvt33pn+2MyyNxmN3VTTm3oTrZVMUmuw3oBnQ2oQ==",
       "dev": true,
       "dependencies": {
-        "tslib": "^1.9.0"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "npm": ">=2.0.0"
-      }
-    },
-    "node_modules/@compodoc/compodoc/node_modules/source-map": {
-      "version": "0.7.3",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
-      "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 8"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/compodoc/node_modules/tslib": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
-      "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
-      "dev": true
-    },
-    "node_modules/@compodoc/live-server": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/@compodoc/live-server/-/live-server-1.2.3.tgz",
-      "integrity": "sha512-hDmntVCyjjaxuJzPzBx68orNZ7TW4BtHWMnXlIVn5dqhK7vuFF/11hspO1cMmc+2QTYgqde1TBcb3127S7Zrow==",
+    "node_modules/@babel/plugin-transform-unicode-escapes": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.23.3.tgz",
+      "integrity": "sha512-OMCUx/bU6ChE3r4+ZdylEqAjaQgHAgipgW8nsCfu5pGqDcFytVd91AwRvUJSBZDz0exPGgnjoqhgRYLRjFZc9Q==",
       "dev": true,
       "dependencies": {
-        "chokidar": "^3.5.2",
-        "colors": "1.4.0",
-        "connect": "^3.7.0",
-        "cors": "latest",
-        "event-stream": "4.0.1",
-        "faye-websocket": "0.11.x",
-        "http-auth": "4.1.9",
-        "http-auth-connect": "^1.0.5",
-        "morgan": "^1.10.0",
-        "object-assign": "latest",
-        "open": "8.4.0",
-        "proxy-middleware": "latest",
-        "send": "latest",
-        "serve-index": "^1.9.1"
-      },
-      "bin": {
-        "live-server": "live-server.js"
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/live-server/node_modules/open": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
-      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
+    "node_modules/@babel/plugin-transform-unicode-regex": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.23.3.tgz",
+      "integrity": "sha512-wMHpNA4x2cIA32b/ci3AfwNgheiva2W0WUKWTK7vBHBhDKfPsc5cFGNWm69WBqpwd86u1qwZ9PWevKqm1A3yAw==",
       "dev": true,
       "dependencies": {
-        "define-lazy-prop": "^2.0.0",
-        "is-docker": "^2.1.1",
-        "is-wsl": "^2.2.0"
+        "@babel/helper-create-regexp-features-plugin": "^7.22.15",
+        "@babel/helper-plugin-utils": "^7.22.5"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=6.9.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/ngd-core": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/@compodoc/ngd-core/-/ngd-core-2.1.1.tgz",
-      "integrity": "sha512-Z+wE6wWZYVnudRYg6qunDlyh3Orw39Ib66Gvrz5kX5u7So+iu3tr6sQJdqH6yGS3hAjig5avlfhWLlgsb6/x1Q==",
+    "node_modules/@babel/preset-env": {
+      "version": "7.20.2",
+      "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.20.2.tgz",
+      "integrity": "sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==",
       "dev": true,
       "dependencies": {
-        "ansi-colors": "^4.1.3",
-        "fancy-log": "^2.0.0",
-        "typescript": "^5.0.4"
+        "@babel/compat-data": "^7.20.1",
+        "@babel/helper-compilation-targets": "^7.20.0",
+        "@babel/helper-plugin-utils": "^7.20.2",
+        "@babel/helper-validator-option": "^7.18.6",
+        "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6",
+        "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.9",
+        "@babel/plugin-proposal-async-generator-functions": "^7.20.1",
+        "@babel/plugin-proposal-class-properties": "^7.18.6",
+        "@babel/plugin-proposal-class-static-block": "^7.18.6",
+        "@babel/plugin-proposal-dynamic-import": "^7.18.6",
+        "@babel/plugin-proposal-export-namespace-from": "^7.18.9",
+        "@babel/plugin-proposal-json-strings": "^7.18.6",
+        "@babel/plugin-proposal-logical-assignment-operators": "^7.18.9",
+        "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
+        "@babel/plugin-proposal-numeric-separator": "^7.18.6",
+        "@babel/plugin-proposal-object-rest-spread": "^7.20.2",
+        "@babel/plugin-proposal-optional-catch-binding": "^7.18.6",
+        "@babel/plugin-proposal-optional-chaining": "^7.18.9",
+        "@babel/plugin-proposal-private-methods": "^7.18.6",
+        "@babel/plugin-proposal-private-property-in-object": "^7.18.6",
+        "@babel/plugin-proposal-unicode-property-regex": "^7.18.6",
+        "@babel/plugin-syntax-async-generators": "^7.8.4",
+        "@babel/plugin-syntax-class-properties": "^7.12.13",
+        "@babel/plugin-syntax-class-static-block": "^7.14.5",
+        "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+        "@babel/plugin-syntax-export-namespace-from": "^7.8.3",
+        "@babel/plugin-syntax-import-assertions": "^7.20.0",
+        "@babel/plugin-syntax-json-strings": "^7.8.3",
+        "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4",
+        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3",
+        "@babel/plugin-syntax-numeric-separator": "^7.10.4",
+        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
+        "@babel/plugin-syntax-optional-chaining": "^7.8.3",
+        "@babel/plugin-syntax-private-property-in-object": "^7.14.5",
+        "@babel/plugin-syntax-top-level-await": "^7.14.5",
+        "@babel/plugin-transform-arrow-functions": "^7.18.6",
+        "@babel/plugin-transform-async-to-generator": "^7.18.6",
+        "@babel/plugin-transform-block-scoped-functions": "^7.18.6",
+        "@babel/plugin-transform-block-scoping": "^7.20.2",
+        "@babel/plugin-transform-classes": "^7.20.2",
+        "@babel/plugin-transform-computed-properties": "^7.18.9",
+        "@babel/plugin-transform-destructuring": "^7.20.2",
+        "@babel/plugin-transform-dotall-regex": "^7.18.6",
+        "@babel/plugin-transform-duplicate-keys": "^7.18.9",
+        "@babel/plugin-transform-exponentiation-operator": "^7.18.6",
+        "@babel/plugin-transform-for-of": "^7.18.8",
+        "@babel/plugin-transform-function-name": "^7.18.9",
+        "@babel/plugin-transform-literals": "^7.18.9",
+        "@babel/plugin-transform-member-expression-literals": "^7.18.6",
+        "@babel/plugin-transform-modules-amd": "^7.19.6",
+        "@babel/plugin-transform-modules-commonjs": "^7.19.6",
+        "@babel/plugin-transform-modules-systemjs": "^7.19.6",
+        "@babel/plugin-transform-modules-umd": "^7.18.6",
+        "@babel/plugin-transform-named-capturing-groups-regex": "^7.19.1",
+        "@babel/plugin-transform-new-target": "^7.18.6",
+        "@babel/plugin-transform-object-super": "^7.18.6",
+        "@babel/plugin-transform-parameters": "^7.20.1",
+        "@babel/plugin-transform-property-literals": "^7.18.6",
+        "@babel/plugin-transform-regenerator": "^7.18.6",
+        "@babel/plugin-transform-reserved-words": "^7.18.6",
+        "@babel/plugin-transform-shorthand-properties": "^7.18.6",
+        "@babel/plugin-transform-spread": "^7.19.0",
+        "@babel/plugin-transform-sticky-regex": "^7.18.6",
+        "@babel/plugin-transform-template-literals": "^7.18.9",
+        "@babel/plugin-transform-typeof-symbol": "^7.18.9",
+        "@babel/plugin-transform-unicode-escapes": "^7.18.10",
+        "@babel/plugin-transform-unicode-regex": "^7.18.6",
+        "@babel/preset-modules": "^0.1.5",
+        "@babel/types": "^7.20.2",
+        "babel-plugin-polyfill-corejs2": "^0.3.3",
+        "babel-plugin-polyfill-corejs3": "^0.6.0",
+        "babel-plugin-polyfill-regenerator": "^0.4.1",
+        "core-js-compat": "^3.25.1",
+        "semver": "^6.3.0"
       },
       "engines": {
-        "node": ">= 10.0.0"
-      }
-    },
-    "node_modules/@compodoc/ngd-core/node_modules/typescript": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
-      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
+        "node": ">=6.9.0"
       },
-      "engines": {
-        "node": ">=14.17"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@compodoc/ngd-transformer": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/@compodoc/ngd-transformer/-/ngd-transformer-2.1.3.tgz",
-      "integrity": "sha512-oWxJza7CpWR8/FeWYfE6j+jgncnGBsTWnZLt5rD2GUpsGSQTuGrsFPnmbbaVLgRS5QIVWBJYke7QFBr/7qVMWg==",
+    "node_modules/@babel/preset-modules": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.6.tgz",
+      "integrity": "sha512-ID2yj6K/4lKfhuU3+EX4UvNbIt7eACFbHmNUjzA+ep+B5971CknnA/9DEWKbRokfbbtblxxxXFJJrH47UEAMVg==",
       "dev": true,
       "dependencies": {
-        "@aduh95/viz.js": "3.4.0",
-        "@compodoc/ngd-core": "~2.1.1",
-        "dot": "^2.0.0-beta.1",
-        "fs-extra": "^11.1.1"
+        "@babel/helper-plugin-utils": "^7.0.0",
+        "@babel/plugin-proposal-unicode-property-regex": "^7.4.4",
+        "@babel/plugin-transform-dotall-regex": "^7.4.4",
+        "@babel/types": "^7.4.4",
+        "esutils": "^2.0.2"
       },
-      "engines": {
-        "node": ">= 10.0.0"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0 || ^8.0.0-0 <8.0.0"
       }
     },
-    "node_modules/@compodoc/ngd-transformer/node_modules/fs-extra": {
-      "version": "11.1.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz",
-      "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==",
+    "node_modules/@babel/preset-react": {
+      "version": "7.23.3",
+      "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.23.3.tgz",
+      "integrity": "sha512-tbkHOS9axH6Ysf2OUEqoSZ6T3Fa2SrNH6WTWSPBboxKzdxNc9qOICeLXkNG0ZEwbQ1HY8liwOce4aN/Ceyuq6w==",
       "dev": true,
       "dependencies": {
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
+        "@babel/helper-plugin-utils": "^7.22.5",
+        "@babel/helper-validator-option": "^7.22.15",
+        "@babel/plugin-transform-react-display-name": "^7.23.3",
+        "@babel/plugin-transform-react-jsx": "^7.22.15",
+        "@babel/plugin-transform-react-jsx-development": "^7.22.5",
+        "@babel/plugin-transform-react-pure-annotations": "^7.23.3"
       },
       "engines": {
-        "node": ">=14.14"
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@cypress/browserify-preprocessor": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@cypress/browserify-preprocessor/-/browserify-preprocessor-3.0.2.tgz",
-      "integrity": "sha512-y6mlFR+IR2cqcm3HabSp7AEcX9QfF1EUL4eOaw/7xexdhmdQU8ez6piyRopZQob4BK8oKTsc9PkupsU2rzjqMA==",
-      "dev": true,
+    "node_modules/@babel/regjsgen": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/@babel/regjsgen/-/regjsgen-0.8.0.tgz",
+      "integrity": "sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==",
+      "dev": true
+    },
+    "node_modules/@babel/runtime": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.4.tgz",
+      "integrity": "sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==",
       "dependencies": {
-        "@babel/core": "^7.16.0",
-        "@babel/plugin-proposal-class-properties": "^7.16.0",
-        "@babel/plugin-proposal-object-rest-spread": "^7.16.0",
-        "@babel/plugin-transform-runtime": "^7.16.0",
-        "@babel/preset-env": "^7.16.0",
-        "@babel/preset-react": "^7.16.0",
-        "@babel/runtime": "^7.16.0",
-        "babel-plugin-add-module-exports": "^1.0.4",
-        "babelify": "^10.0.0",
-        "bluebird": "^3.7.2",
-        "browserify": "^16.2.3",
-        "coffeeify": "^3.0.1",
-        "coffeescript": "^1.12.7",
-        "debug": "^4.3.2",
-        "fs-extra": "^9.0.0",
-        "lodash.clonedeep": "^4.5.0",
-        "through2": "^2.0.0",
-        "watchify": "^4.0.0"
+        "regenerator-runtime": "^0.14.0"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@cypress/browserify-preprocessor/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "dev": true,
+    "node_modules/@babel/runtime-corejs3": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.23.4.tgz",
+      "integrity": "sha512-zQyB4MJGM+rvd4pM58n26kf3xbiitw9MHzL8oLiBMKb8MCtVDfV5nDzzJWWzLMtbvKI9wN6XwJYl479qF4JluQ==",
       "dependencies": {
-        "ms": "2.1.2"
+        "core-js-pure": "^3.30.2",
+        "regenerator-runtime": "^0.14.0"
       },
       "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@cypress/browserify-preprocessor/node_modules/fs-extra": {
-      "version": "9.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
-      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
-      "dev": true,
+    "node_modules/@babel/template": {
+      "version": "7.22.15",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
+      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
       "dependencies": {
-        "at-least-node": "^1.0.0",
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
+        "@babel/code-frame": "^7.22.13",
+        "@babel/parser": "^7.22.15",
+        "@babel/types": "^7.22.15"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@cypress/request": {
-      "version": "2.88.12",
-      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.12.tgz",
-      "integrity": "sha512-tOn+0mDZxASFM+cuAP9szGUGPI1HwWVSvdzm7V4cCsPdFTx6qMj29CwaQmRAMIEhORIUBFBsYROYJcveK4uOjA==",
-      "dev": true,
+    "node_modules/@babel/traverse": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.4.tgz",
+      "integrity": "sha512-IYM8wSUwunWTB6tFC2dkKZhxbIjHoWemdK+3f8/wq8aKhbUscxD5MX72ubd90fxvFknaLPeGw5ycU84V1obHJg==",
       "dependencies": {
-        "aws-sign2": "~0.7.0",
-        "aws4": "^1.8.0",
-        "caseless": "~0.12.0",
-        "combined-stream": "~1.0.6",
-        "extend": "~3.0.2",
-        "forever-agent": "~0.6.1",
-        "form-data": "~2.3.2",
-        "http-signature": "~1.3.6",
-        "is-typedarray": "~1.0.0",
-        "isstream": "~0.1.2",
-        "json-stringify-safe": "~5.0.1",
-        "mime-types": "~2.1.19",
-        "performance-now": "^2.1.0",
-        "qs": "~6.10.3",
-        "safe-buffer": "^5.1.2",
-        "tough-cookie": "^4.1.3",
-        "tunnel-agent": "^0.6.0",
-        "uuid": "^8.3.2"
+        "@babel/code-frame": "^7.23.4",
+        "@babel/generator": "^7.23.4",
+        "@babel/helper-environment-visitor": "^7.22.20",
+        "@babel/helper-function-name": "^7.23.0",
+        "@babel/helper-hoist-variables": "^7.22.5",
+        "@babel/helper-split-export-declaration": "^7.22.6",
+        "@babel/parser": "^7.23.4",
+        "@babel/types": "^7.23.4",
+        "debug": "^4.1.0",
+        "globals": "^11.1.0"
       },
       "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/@cypress/request/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@cypress/request/node_modules/qs": {
-      "version": "6.10.4",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.4.tgz",
-      "integrity": "sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g==",
-      "dev": true,
+    "node_modules/@babel/types": {
+      "version": "7.23.4",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.4.tgz",
+      "integrity": "sha512-7uIFwVYpoplT5jp/kVv6EF93VaJ8H+Yn5IczYiaAi98ajzjfoZfslet/e0sLh+wVBjb2qqIut1b0S26VSafsSQ==",
       "dependencies": {
-        "side-channel": "^1.0.4"
+        "@babel/helper-string-parser": "^7.23.4",
+        "@babel/helper-validator-identifier": "^7.22.20",
+        "to-fast-properties": "^2.0.0"
       },
       "engines": {
-        "node": ">=0.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/@cypress/request/node_modules/tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
-      "dev": true,
-      "dependencies": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
+    "node_modules/@bcoe/v8-coverage": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz",
+      "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==",
+      "dev": true
+    },
+    "node_modules/@braintree/sanitize-url": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.0.tgz",
+      "integrity": "sha512-mgmE7XBYY/21erpzhexk4Cj1cyTQ9LzvnTxtzM17BJ7ERMNE6W72mQRo0I1Ud8eFJ+RVVIcBNhLFZ3GX4XFz5w=="
+    },
+    "node_modules/@circlon/angular-tree-component": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/@circlon/angular-tree-component/-/angular-tree-component-10.0.0.tgz",
+      "integrity": "sha512-3dRWLbOdMfIuvZjX6AMHmvzPtqhNFECMWMpNVXrZfZtTAa0n+Y4lxbuLST85q5QiedBZuC720p/7kkZ78PJ+iw==",
+      "dependencies": {
+        "lodash-es": "^4.17.15",
+        "mobx": "~4.14.1",
+        "tslib": "^2.0.0"
       },
-      "engines": {
-        "node": ">=6"
+      "peerDependencies": {
+        "@angular/common": ">=10.0.0 <11.0.0",
+        "@angular/core": ">=10.0.0 <11.0.0"
       }
     },
-    "node_modules/@cypress/request/node_modules/universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
+    "node_modules/@colors/colors": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz",
+      "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==",
       "dev": true,
+      "optional": true,
       "engines": {
-        "node": ">= 4.0.0"
+        "node": ">=0.1.90"
       }
     },
-    "node_modules/@cypress/xvfb": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz",
-      "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==",
+    "node_modules/@compodoc/compodoc": {
+      "version": "1.1.18",
+      "resolved": "https://registry.npmjs.org/@compodoc/compodoc/-/compodoc-1.1.18.tgz",
+      "integrity": "sha512-+AFtcj2U3AJq6r8a2+PTdajIlS7m3pgvDhqgoYZJ4Rg/Zp9xvuDvUJU+5oHu8iHCAWwda3NoLUDjOZMNR8uIKg==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "debug": "^3.1.0",
-        "lodash.once": "^4.1.1"
+        "@angular-devkit/schematics": "^13.1.2",
+        "@babel/core": "^7.16.7",
+        "@babel/preset-env": "^7.16.7",
+        "@compodoc/live-server": "^1.2.3",
+        "@compodoc/ngd-transformer": "^2.1.0",
+        "chalk": "^4.1.2",
+        "cheerio": "^1.0.0-rc.10",
+        "chokidar": "^3.5.2",
+        "colors": "1.4.0",
+        "commander": "^8.3.0",
+        "cosmiconfig": "^7.0.1",
+        "decache": "^4.6.1",
+        "fancy-log": "^2.0.0",
+        "findit2": "^2.2.3",
+        "fs-extra": "^10.0.0",
+        "glob": "^7.2.0",
+        "handlebars": "^4.7.7",
+        "html-entities": "^2.3.2",
+        "i18next": "^21.6.5",
+        "inside": "^1.0.0",
+        "json5": "^2.2.0",
+        "lodash": "^4.17.21",
+        "loglevel": "^1.8.0",
+        "loglevel-plugin-prefix": "^0.8.4",
+        "lunr": "^2.3.9",
+        "marked": "^4.0.9",
+        "minimist": "^1.2.5",
+        "opencollective-postinstall": "^2.0.3",
+        "os-name": "4.0.1",
+        "pdfjs-dist": "^2.12.313",
+        "pdfmake": "^0.2.4",
+        "semver": "^7.3.5",
+        "traverse": "^0.6.6",
+        "ts-morph": "^13.0.2",
+        "uuid": "^8.3.2"
+      },
+      "bin": {
+        "compodoc": "bin/index-cli.js"
+      },
+      "engines": {
+        "node": ">= 12.0.0"
       }
     },
-    "node_modules/@cypress/xvfb/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+    "node_modules/@compodoc/compodoc/node_modules/@angular-devkit/core": {
+      "version": "13.3.11",
+      "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-13.3.11.tgz",
+      "integrity": "sha512-rfqoLMRYhlz0wzKlHx7FfyIyQq8dKTsmbCoIVU1cEIH0gyTMVY7PbVzwRRcO6xp5waY+0hA+0Brriujpuhkm4w==",
       "dev": true,
       "dependencies": {
-        "ms": "^2.1.1"
+        "ajv": "8.9.0",
+        "ajv-formats": "2.1.1",
+        "fast-json-stable-stringify": "2.1.0",
+        "magic-string": "0.25.7",
+        "rxjs": "6.6.7",
+        "source-map": "0.7.3"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.15.0 || >=16.10.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
+      },
+      "peerDependencies": {
+        "chokidar": "^3.5.2"
+      },
+      "peerDependenciesMeta": {
+        "chokidar": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@discoveryjs/json-ext": {
-      "version": "0.5.7",
-      "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz",
-      "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==",
+    "node_modules/@compodoc/compodoc/node_modules/@angular-devkit/schematics": {
+      "version": "13.3.11",
+      "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-13.3.11.tgz",
+      "integrity": "sha512-ben+EGXpCrClnIVAAnEQmhQdKmnnqFhMp5BqMxgOslSYBAmCutLA6rBu5vsc8kZcGian1wt+lueF7G1Uk5cGBg==",
       "dev": true,
+      "dependencies": {
+        "@angular-devkit/core": "13.3.11",
+        "jsonc-parser": "3.0.0",
+        "magic-string": "0.25.7",
+        "ora": "5.4.1",
+        "rxjs": "6.6.7"
+      },
       "engines": {
-        "node": ">=10.0.0"
+        "node": "^12.20.0 || ^14.15.0 || >=16.10.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
       }
     },
-    "node_modules/@esbuild/android-arm": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.8.tgz",
-      "integrity": "sha512-0/rb91GYKhrtbeglJXOhAv9RuYimgI8h623TplY2X+vA4EXnk3Zj1fXZreJ0J3OJJu1bwmb0W7g+2cT/d8/l/w==",
-      "cpu": [
-        "arm"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/ajv": {
+      "version": "8.9.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
+      "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=12"
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
       }
     },
-    "node_modules/@esbuild/android-arm64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.8.tgz",
-      "integrity": "sha512-oa/N5j6v1svZQs7EIRPqR8f+Bf8g6HBDjD/xHC02radE/NjKHK7oQmtmLxPs1iVwYyvE+Kolo6lbpfEQ9xnhxQ==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "android"
-      ],
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@esbuild/android-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.8.tgz",
-      "integrity": "sha512-bTliMLqD7pTOoPg4zZkXqCDuzIUguEWLpeqkNfC41ODBHwoUgZ2w5JBeYimv4oP6TDVocoYmEhZrCLQTrH89bg==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "android"
-      ],
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/@compodoc/compodoc/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@esbuild/darwin-arm64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.8.tgz",
-      "integrity": "sha512-ghAbV3ia2zybEefXRRm7+lx8J/rnupZT0gp9CaGy/3iolEXkJ6LYRq4IpQVI9zR97ID80KJVoUlo3LSeA/sMAg==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@esbuild/darwin-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.8.tgz",
-      "integrity": "sha512-n5WOpyvZ9TIdv2V1K3/iIkkJeKmUpKaCTdun9buhGRWfH//osmUjlv4Z5mmWdPWind/VGcVxTHtLfLCOohsOXw==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@compodoc/compodoc/node_modules/commander": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz",
+      "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
       "engines": {
-        "node": ">=12"
+        "node": ">= 12"
       }
     },
-    "node_modules/@esbuild/freebsd-arm64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.8.tgz",
-      "integrity": "sha512-a/SATTaOhPIPFWvHZDoZYgxaZRVHn0/LX1fHLGfZ6C13JqFUZ3K6SMD6/HCtwOQ8HnsNaEeokdiDSFLuizqv5A==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/fs-extra": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
+      "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
       "engines": {
         "node": ">=12"
       }
     },
-    "node_modules/@esbuild/freebsd-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.8.tgz",
-      "integrity": "sha512-xpFJb08dfXr5+rZc4E+ooZmayBW6R3q59daCpKZ/cDU96/kvDM+vkYzNeTJCGd8rtO6fHWMq5Rcv/1cY6p6/0Q==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
       "engines": {
-        "node": ">=12"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/@esbuild/linux-arm": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.8.tgz",
-      "integrity": "sha512-6Ij8gfuGszcEwZpi5jQIJCVIACLS8Tz2chnEBfYjlmMzVsfqBP1iGmHQPp7JSnZg5xxK9tjCc+pJ2WtAmPRFVA==",
-      "cpu": [
-        "arm"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
       "engines": {
-        "node": ">=12"
+        "node": ">=8"
       }
     },
-    "node_modules/@esbuild/linux-arm64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.8.tgz",
-      "integrity": "sha512-v3iwDQuDljLTxpsqQDl3fl/yihjPAyOguxuloON9kFHYwopeJEf1BkDXODzYyXEI19gisEsQlG1bM65YqKSIww==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/jsonc-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz",
+      "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==",
+      "dev": true
+    },
+    "node_modules/@compodoc/compodoc/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=10"
       }
     },
-    "node_modules/@esbuild/linux-ia32": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.8.tgz",
-      "integrity": "sha512-8svILYKhE5XetuFk/B6raFYIyIqydQi+GngEXJgdPdI7OMKUbSd7uzR02wSY4kb53xBrClLkhH4Xs8P61Q2BaA==",
-      "cpu": [
-        "ia32"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/magic-string": {
+      "version": "0.25.7",
+      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz",
+      "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=12"
+      "dependencies": {
+        "sourcemap-codec": "^1.4.4"
       }
     },
-    "node_modules/@esbuild/linux-loong64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.8.tgz",
-      "integrity": "sha512-B6FyMeRJeV0NpyEOYlm5qtQfxbdlgmiGdD+QsipzKfFky0K5HW5Td6dyK3L3ypu1eY4kOmo7wW0o94SBqlqBSA==",
-      "cpu": [
-        "loong64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
       "engines": {
-        "node": ">=12"
+        "node": "*"
       }
     },
-    "node_modules/@esbuild/linux-mips64el": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.8.tgz",
-      "integrity": "sha512-CCb67RKahNobjm/eeEqeD/oJfJlrWyw29fgiyB6vcgyq97YAf3gCOuP6qMShYSPXgnlZe/i4a8WFHBw6N8bYAA==",
-      "cpu": [
-        "mips64el"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/rxjs": {
+      "version": "6.6.7",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz",
+      "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "tslib": "^1.9.0"
+      },
       "engines": {
-        "node": ">=12"
+        "npm": ">=2.0.0"
       }
     },
-    "node_modules/@esbuild/linux-ppc64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.8.tgz",
-      "integrity": "sha512-bytLJOi55y55+mGSdgwZ5qBm0K9WOCh0rx+vavVPx+gqLLhxtSFU0XbeYy/dsAAD6xECGEv4IQeFILaSS2auXw==",
-      "cpu": [
-        "ppc64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=10"
       }
     },
-    "node_modules/@esbuild/linux-riscv64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.8.tgz",
-      "integrity": "sha512-2YpRyQJmKVBEHSBLa8kBAtbhucaclb6ex4wchfY0Tj3Kg39kpjeJ9vhRU7x4mUpq8ISLXRXH1L0dBYjAeqzZAw==",
-      "cpu": [
-        "riscv64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/source-map": {
+      "version": "0.7.3",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
+      "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
       "engines": {
-        "node": ">=12"
+        "node": ">= 8"
       }
     },
-    "node_modules/@esbuild/linux-s390x": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.8.tgz",
-      "integrity": "sha512-QgbNY/V3IFXvNf11SS6exkpVcX0LJcob+0RWCgV9OiDAmVElnxciHIisoSix9uzYzScPmS6dJFbZULdSAEkQVw==",
-      "cpu": [
-        "s390x"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=8"
       }
     },
-    "node_modules/@esbuild/linux-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.8.tgz",
-      "integrity": "sha512-mM/9S0SbAFDBc4OPoyP6SEOo5324LpUxdpeIUUSrSTOfhHU9hEfqRngmKgqILqwx/0DVJBzeNW7HmLEWp9vcOA==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/compodoc/node_modules/tslib": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
+      "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
+      "dev": true
+    },
+    "node_modules/@compodoc/compodoc/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@compodoc/live-server": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/@compodoc/live-server/-/live-server-1.2.3.tgz",
+      "integrity": "sha512-hDmntVCyjjaxuJzPzBx68orNZ7TW4BtHWMnXlIVn5dqhK7vuFF/11hspO1cMmc+2QTYgqde1TBcb3127S7Zrow==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "chokidar": "^3.5.2",
+        "colors": "1.4.0",
+        "connect": "^3.7.0",
+        "cors": "latest",
+        "event-stream": "4.0.1",
+        "faye-websocket": "0.11.x",
+        "http-auth": "4.1.9",
+        "http-auth-connect": "^1.0.5",
+        "morgan": "^1.10.0",
+        "object-assign": "latest",
+        "open": "8.4.0",
+        "proxy-middleware": "latest",
+        "send": "latest",
+        "serve-index": "^1.9.1"
+      },
+      "bin": {
+        "live-server": "live-server.js"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@esbuild/netbsd-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.8.tgz",
-      "integrity": "sha512-eKUYcWaWTaYr9zbj8GertdVtlt1DTS1gNBWov+iQfWuWyuu59YN6gSEJvFzC5ESJ4kMcKR0uqWThKUn5o8We6Q==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/live-server/node_modules/cors": {
+      "version": "2.8.5",
+      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "netbsd"
-      ],
+      "dependencies": {
+        "object-assign": "^4",
+        "vary": "^1"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">= 0.10"
       }
     },
-    "node_modules/@esbuild/openbsd-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.8.tgz",
-      "integrity": "sha512-Vc9J4dXOboDyMXKD0eCeW0SIeEzr8K9oTHJU+Ci1mZc5njPfhKAqkRt3B/fUNU7dP+mRyralPu8QUkiaQn7iIg==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/live-server/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "openbsd"
-      ],
-      "engines": {
-        "node": ">=12"
+      "dependencies": {
+        "ms": "2.0.0"
       }
     },
-    "node_modules/@esbuild/sunos-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.8.tgz",
-      "integrity": "sha512-0xvOTNuPXI7ft1LYUgiaXtpCEjp90RuBBYovdd2lqAFxje4sEucurg30M1WIm03+3jxByd3mfo+VUmPtRSVuOw==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/live-server/node_modules/debug/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/@compodoc/live-server/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "dev": true
+    },
+    "node_modules/@compodoc/live-server/node_modules/object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "sunos"
-      ],
       "engines": {
-        "node": ">=12"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@esbuild/win32-arm64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.8.tgz",
-      "integrity": "sha512-G0JQwUI5WdEFEnYNKzklxtBheCPkuDdu1YrtRrjuQv30WsYbkkoixKxLLv8qhJmNI+ATEWquZe/N0d0rpr55Mg==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@compodoc/live-server/node_modules/open": {
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
+      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ],
+      "dependencies": {
+        "define-lazy-prop": "^2.0.0",
+        "is-docker": "^2.1.1",
+        "is-wsl": "^2.2.0"
+      },
       "engines": {
         "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@esbuild/win32-ia32": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.8.tgz",
-      "integrity": "sha512-Fqy63515xl20OHGFykjJsMnoIWS+38fqfg88ClvPXyDbLtgXal2DTlhb1TfTX34qWi3u4I7Cq563QcHpqgLx8w==",
-      "cpu": [
-        "ia32"
-      ],
+    "node_modules/@compodoc/live-server/node_modules/send": {
+      "version": "0.18.0",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
+      "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ],
+      "dependencies": {
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "fresh": "0.5.2",
+        "http-errors": "2.0.0",
+        "mime": "1.6.0",
+        "ms": "2.1.3",
+        "on-finished": "2.4.1",
+        "range-parser": "~1.2.1",
+        "statuses": "2.0.1"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/@esbuild/win32-x64": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.8.tgz",
-      "integrity": "sha512-1iuezdyDNngPnz8rLRDO2C/ZZ/emJLb72OsZeqQ6gL6Avko/XCXZw+NuxBSNhBAP13Hie418V7VMt9et1FMvpg==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@compodoc/ngd-core": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@compodoc/ngd-core/-/ngd-core-2.1.1.tgz",
+      "integrity": "sha512-Z+wE6wWZYVnudRYg6qunDlyh3Orw39Ib66Gvrz5kX5u7So+iu3tr6sQJdqH6yGS3hAjig5avlfhWLlgsb6/x1Q==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ],
+      "dependencies": {
+        "ansi-colors": "^4.1.3",
+        "fancy-log": "^2.0.0",
+        "typescript": "^5.0.4"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">= 10.0.0"
       }
     },
-    "node_modules/@eslint/eslintrc": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz",
-      "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==",
+    "node_modules/@compodoc/ngd-core/node_modules/typescript": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.2.tgz",
+      "integrity": "sha512-6l+RyNy7oAHDfxC4FzSJcz9vnjTKxrLpDG5M2Vu4SHRVNg6xzqZp6LYSR9zjqQTu8DU/f5xwxUdADOkbrIX2gQ==",
       "dev": true,
-      "dependencies": {
-        "ajv": "^6.12.4",
-        "debug": "^4.3.2",
-        "espree": "^9.4.0",
-        "globals": "^13.19.0",
-        "ignore": "^5.2.0",
-        "import-fresh": "^3.2.1",
-        "js-yaml": "^4.1.0",
-        "minimatch": "^3.1.2",
-        "strip-json-comments": "^3.1.1"
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://opencollective.com/eslint"
+        "node": ">=14.17"
       }
     },
-    "node_modules/@eslint/eslintrc/node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+    "node_modules/@compodoc/ngd-transformer": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@compodoc/ngd-transformer/-/ngd-transformer-2.1.3.tgz",
+      "integrity": "sha512-oWxJza7CpWR8/FeWYfE6j+jgncnGBsTWnZLt5rD2GUpsGSQTuGrsFPnmbbaVLgRS5QIVWBJYke7QFBr/7qVMWg==",
       "dev": true,
       "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
-        "uri-js": "^4.2.2"
+        "@aduh95/viz.js": "3.4.0",
+        "@compodoc/ngd-core": "~2.1.1",
+        "dot": "^2.0.0-beta.1",
+        "fs-extra": "^11.1.1"
       },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
+      "engines": {
+        "node": ">= 10.0.0"
       }
     },
-    "node_modules/@eslint/eslintrc/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
-    },
-    "node_modules/@eslint/eslintrc/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/@compodoc/ngd-transformer/node_modules/fs-extra": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz",
+      "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
       },
       "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "node": ">=14.14"
       }
     },
-    "node_modules/@eslint/eslintrc/node_modules/globals": {
-      "version": "13.22.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz",
-      "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==",
+    "node_modules/@cypress/browserify-preprocessor": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@cypress/browserify-preprocessor/-/browserify-preprocessor-3.0.2.tgz",
+      "integrity": "sha512-y6mlFR+IR2cqcm3HabSp7AEcX9QfF1EUL4eOaw/7xexdhmdQU8ez6piyRopZQob4BK8oKTsc9PkupsU2rzjqMA==",
       "dev": true,
       "dependencies": {
-        "type-fest": "^0.20.2"
+        "@babel/core": "^7.16.0",
+        "@babel/plugin-proposal-class-properties": "^7.16.0",
+        "@babel/plugin-proposal-object-rest-spread": "^7.16.0",
+        "@babel/plugin-transform-runtime": "^7.16.0",
+        "@babel/preset-env": "^7.16.0",
+        "@babel/preset-react": "^7.16.0",
+        "@babel/runtime": "^7.16.0",
+        "babel-plugin-add-module-exports": "^1.0.4",
+        "babelify": "^10.0.0",
+        "bluebird": "^3.7.2",
+        "browserify": "^16.2.3",
+        "coffeeify": "^3.0.1",
+        "coffeescript": "^1.12.7",
+        "debug": "^4.3.2",
+        "fs-extra": "^9.0.0",
+        "lodash.clonedeep": "^4.5.0",
+        "through2": "^2.0.0",
+        "watchify": "^4.0.0"
       },
       "engines": {
         "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@eslint/eslintrc/node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+    "node_modules/@cypress/request": {
+      "version": "2.88.12",
+      "resolved": "https://registry.npmjs.org/@cypress/request/-/request-2.88.12.tgz",
+      "integrity": "sha512-tOn+0mDZxASFM+cuAP9szGUGPI1HwWVSvdzm7V4cCsPdFTx6qMj29CwaQmRAMIEhORIUBFBsYROYJcveK4uOjA==",
       "dev": true,
       "dependencies": {
-        "argparse": "^2.0.1"
+        "aws-sign2": "~0.7.0",
+        "aws4": "^1.8.0",
+        "caseless": "~0.12.0",
+        "combined-stream": "~1.0.6",
+        "extend": "~3.0.2",
+        "forever-agent": "~0.6.1",
+        "form-data": "~2.3.2",
+        "http-signature": "~1.3.6",
+        "is-typedarray": "~1.0.0",
+        "isstream": "~0.1.2",
+        "json-stringify-safe": "~5.0.1",
+        "mime-types": "~2.1.19",
+        "performance-now": "^2.1.0",
+        "qs": "~6.10.3",
+        "safe-buffer": "^5.1.2",
+        "tough-cookie": "^4.1.3",
+        "tunnel-agent": "^0.6.0",
+        "uuid": "^8.3.2"
       },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
-      }
-    },
-    "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
-      "dev": true
-    },
-    "node_modules/@eslint/eslintrc/node_modules/type-fest": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
-      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
-      "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/@foliojs-fork/fontkit": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/@foliojs-fork/fontkit/-/fontkit-1.9.1.tgz",
-      "integrity": "sha512-U589voc2/ROnvx1CyH9aNzOQWJp127JGU1QAylXGQ7LoEAF6hMmahZLQ4eqAcgHUw+uyW4PjtCItq9qudPkK3A==",
-      "dev": true,
-      "dependencies": {
-        "@foliojs-fork/restructure": "^2.0.2",
-        "brfs": "^2.0.0",
-        "brotli": "^1.2.0",
-        "browserify-optional": "^1.0.1",
-        "clone": "^1.0.4",
-        "deep-equal": "^1.0.0",
-        "dfa": "^1.2.0",
-        "tiny-inflate": "^1.0.2",
-        "unicode-properties": "^1.2.2",
-        "unicode-trie": "^2.0.0"
+        "node": ">= 6"
       }
     },
-    "node_modules/@foliojs-fork/linebreak": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@foliojs-fork/linebreak/-/linebreak-1.1.1.tgz",
-      "integrity": "sha512-pgY/+53GqGQI+mvDiyprvPWgkTlVBS8cxqee03ejm6gKAQNsR1tCYCIvN9FHy7otZajzMqCgPOgC4cHdt4JPig==",
+    "node_modules/@cypress/request/node_modules/form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
       "dev": true,
       "dependencies": {
-        "base64-js": "1.3.1",
-        "brfs": "^2.0.2",
-        "unicode-trie": "^2.0.0"
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 0.12"
       }
     },
-    "node_modules/@foliojs-fork/linebreak/node_modules/base64-js": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
-      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
-      "dev": true
-    },
-    "node_modules/@foliojs-fork/pdfkit": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/@foliojs-fork/pdfkit/-/pdfkit-0.13.0.tgz",
-      "integrity": "sha512-YXeG1fml9k97YNC9K8e292Pj2JzGt9uOIiBFuQFxHsdQ45BlxW+JU3RQK6JAvXU7kjhjP8rCcYvpk36JLD33sQ==",
+    "node_modules/@cypress/request/node_modules/http-signature": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz",
+      "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==",
       "dev": true,
       "dependencies": {
-        "@foliojs-fork/fontkit": "^1.9.1",
-        "@foliojs-fork/linebreak": "^1.1.1",
-        "crypto-js": "^4.0.0",
-        "png-js": "^1.0.0"
+        "assert-plus": "^1.0.0",
+        "jsprim": "^2.0.2",
+        "sshpk": "^1.14.1"
+      },
+      "engines": {
+        "node": ">=0.10"
       }
     },
-    "node_modules/@foliojs-fork/restructure": {
+    "node_modules/@cypress/request/node_modules/jsprim": {
       "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@foliojs-fork/restructure/-/restructure-2.0.2.tgz",
-      "integrity": "sha512-59SgoZ3EXbkfSX7b63tsou/SDGzwUEK6MuB5sKqgVK1/XE0fxmpsOb9DQI8LXW3KfGnAjImCGhhEb7uPPAUVNA==",
-      "dev": true
-    },
-    "node_modules/@hapi/hoek": {
-      "version": "9.3.0",
-      "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
-      "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==",
-      "dev": true
-    },
-    "node_modules/@hapi/topo": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz",
-      "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz",
+      "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==",
       "dev": true,
+      "engines": [
+        "node >=0.6.0"
+      ],
       "dependencies": {
-        "@hapi/hoek": "^9.0.0"
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.4.0",
+        "verror": "1.10.0"
       }
     },
-    "node_modules/@humanwhocodes/config-array": {
-      "version": "0.9.5",
-      "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz",
-      "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==",
+    "node_modules/@cypress/request/node_modules/qs": {
+      "version": "6.10.4",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.4.tgz",
+      "integrity": "sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g==",
       "dev": true,
       "dependencies": {
-        "@humanwhocodes/object-schema": "^1.2.1",
-        "debug": "^4.1.1",
-        "minimatch": "^3.0.4"
+        "side-channel": "^1.0.4"
       },
       "engines": {
-        "node": ">=10.10.0"
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@humanwhocodes/object-schema": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
-      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
-      "dev": true
-    },
-    "node_modules/@isaacs/cliui": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
-      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+    "node_modules/@cypress/request/node_modules/tough-cookie": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
+      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
       "dev": true,
       "dependencies": {
-        "string-width": "^5.1.2",
-        "string-width-cjs": "npm:string-width@^4.2.0",
-        "strip-ansi": "^7.0.1",
-        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
-        "wrap-ansi": "^8.1.0",
-        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+        "psl": "^1.1.33",
+        "punycode": "^2.1.1",
+        "universalify": "^0.2.0",
+        "url-parse": "^1.5.3"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=6"
       }
     },
-    "node_modules/@isaacs/cliui/node_modules/ansi-regex": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
-      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
+    "node_modules/@cypress/request/node_modules/universalify": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
+      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
       "dev": true,
       "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+        "node": ">= 4.0.0"
       }
     },
-    "node_modules/@isaacs/cliui/node_modules/ansi-styles": {
-      "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
-      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+    "node_modules/@cypress/xvfb": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz",
+      "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==",
       "dev": true,
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      "dependencies": {
+        "debug": "^3.1.0",
+        "lodash.once": "^4.1.1"
       }
     },
-    "node_modules/@isaacs/cliui/node_modules/emoji-regex": {
-      "version": "9.2.2",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
-      "dev": true
-    },
-    "node_modules/@isaacs/cliui/node_modules/string-width": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
-      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+    "node_modules/@cypress/xvfb/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
       "dev": true,
       "dependencies": {
-        "eastasianwidth": "^0.2.0",
-        "emoji-regex": "^9.2.2",
-        "strip-ansi": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "ms": "^2.1.1"
       }
     },
-    "node_modules/@isaacs/cliui/node_modules/strip-ansi": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+    "node_modules/@discoveryjs/json-ext": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz",
+      "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==",
       "dev": true,
-      "dependencies": {
-        "ansi-regex": "^6.0.1"
-      },
       "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+        "node": ">=10.0.0"
       }
     },
-    "node_modules/@isaacs/cliui/node_modules/wrap-ansi": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
-      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+    "node_modules/@esbuild/android-arm": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.8.tgz",
+      "integrity": "sha512-0/rb91GYKhrtbeglJXOhAv9RuYimgI8h623TplY2X+vA4EXnk3Zj1fXZreJ0J3OJJu1bwmb0W7g+2cT/d8/l/w==",
+      "cpu": [
+        "arm"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^6.1.0",
-        "string-width": "^5.0.1",
-        "strip-ansi": "^7.0.1"
-      },
+      "optional": true,
+      "os": [
+        "android"
+      ],
       "engines": {
         "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
-    "node_modules/@istanbuljs/load-nyc-config": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
-      "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==",
+    "node_modules/@esbuild/android-arm64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.8.tgz",
+      "integrity": "sha512-oa/N5j6v1svZQs7EIRPqR8f+Bf8g6HBDjD/xHC02radE/NjKHK7oQmtmLxPs1iVwYyvE+Kolo6lbpfEQ9xnhxQ==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "camelcase": "^5.3.1",
-        "find-up": "^4.1.0",
-        "get-package-type": "^0.1.0",
-        "js-yaml": "^3.13.1",
-        "resolve-from": "^5.0.0"
-      },
+      "optional": true,
+      "os": [
+        "android"
+      ],
       "engines": {
-        "node": ">=8"
+        "node": ">=12"
       }
     },
-    "node_modules/@istanbuljs/schema": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz",
-      "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==",
+    "node_modules/@esbuild/android-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.8.tgz",
+      "integrity": "sha512-bTliMLqD7pTOoPg4zZkXqCDuzIUguEWLpeqkNfC41ODBHwoUgZ2w5JBeYimv4oP6TDVocoYmEhZrCLQTrH89bg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
       "engines": {
-        "node": ">=8"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/console": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz",
-      "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==",
+    "node_modules/@esbuild/darwin-arm64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.8.tgz",
+      "integrity": "sha512-ghAbV3ia2zybEefXRRm7+lx8J/rnupZT0gp9CaGy/3iolEXkJ6LYRq4IpQVI9zR97ID80KJVoUlo3LSeA/sMAg==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "jest-message-util": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "slash": "^3.0.0"
-      },
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/console/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@esbuild/darwin-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.8.tgz",
+      "integrity": "sha512-n5WOpyvZ9TIdv2V1K3/iIkkJeKmUpKaCTdun9buhGRWfH//osmUjlv4Z5mmWdPWind/VGcVxTHtLfLCOohsOXw==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/core": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz",
-      "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==",
+    "node_modules/@esbuild/freebsd-arm64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.8.tgz",
+      "integrity": "sha512-a/SATTaOhPIPFWvHZDoZYgxaZRVHn0/LX1fHLGfZ6C13JqFUZ3K6SMD6/HCtwOQ8HnsNaEeokdiDSFLuizqv5A==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/console": "^29.7.0",
-        "@jest/reporters": "^29.7.0",
-        "@jest/test-result": "^29.7.0",
-        "@jest/transform": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "ansi-escapes": "^4.2.1",
-        "chalk": "^4.0.0",
-        "ci-info": "^3.2.0",
-        "exit": "^0.1.2",
-        "graceful-fs": "^4.2.9",
-        "jest-changed-files": "^29.7.0",
-        "jest-config": "^29.7.0",
-        "jest-haste-map": "^29.7.0",
-        "jest-message-util": "^29.7.0",
-        "jest-regex-util": "^29.6.3",
-        "jest-resolve": "^29.7.0",
-        "jest-resolve-dependencies": "^29.7.0",
-        "jest-runner": "^29.7.0",
-        "jest-runtime": "^29.7.0",
-        "jest-snapshot": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jest-validate": "^29.7.0",
-        "jest-watcher": "^29.7.0",
-        "micromatch": "^4.0.4",
-        "pretty-format": "^29.7.0",
-        "slash": "^3.0.0",
-        "strip-ansi": "^6.0.0"
-      },
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
-      },
-      "peerDependenciesMeta": {
-        "node-notifier": {
-          "optional": true
-        }
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/core/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@esbuild/freebsd-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.8.tgz",
+      "integrity": "sha512-xpFJb08dfXr5+rZc4E+ooZmayBW6R3q59daCpKZ/cDU96/kvDM+vkYzNeTJCGd8rtO6fHWMq5Rcv/1cY6p6/0Q==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/environment": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz",
-      "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==",
+    "node_modules/@esbuild/linux-arm": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.8.tgz",
+      "integrity": "sha512-6Ij8gfuGszcEwZpi5jQIJCVIACLS8Tz2chnEBfYjlmMzVsfqBP1iGmHQPp7JSnZg5xxK9tjCc+pJ2WtAmPRFVA==",
+      "cpu": [
+        "arm"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/fake-timers": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "jest-mock": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/expect": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz",
-      "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==",
+    "node_modules/@esbuild/linux-arm64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.8.tgz",
+      "integrity": "sha512-v3iwDQuDljLTxpsqQDl3fl/yihjPAyOguxuloON9kFHYwopeJEf1BkDXODzYyXEI19gisEsQlG1bM65YqKSIww==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "expect": "^29.7.0",
-        "jest-snapshot": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/expect-utils": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz",
-      "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==",
+    "node_modules/@esbuild/linux-ia32": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.8.tgz",
+      "integrity": "sha512-8svILYKhE5XetuFk/B6raFYIyIqydQi+GngEXJgdPdI7OMKUbSd7uzR02wSY4kb53xBrClLkhH4Xs8P61Q2BaA==",
+      "cpu": [
+        "ia32"
+      ],
       "dev": true,
-      "dependencies": {
-        "jest-get-type": "^29.6.3"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/fake-timers": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz",
-      "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==",
+    "node_modules/@esbuild/linux-loong64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.8.tgz",
+      "integrity": "sha512-B6FyMeRJeV0NpyEOYlm5qtQfxbdlgmiGdD+QsipzKfFky0K5HW5Td6dyK3L3ypu1eY4kOmo7wW0o94SBqlqBSA==",
+      "cpu": [
+        "loong64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "@sinonjs/fake-timers": "^10.0.2",
-        "@types/node": "*",
-        "jest-message-util": "^29.7.0",
-        "jest-mock": "^29.7.0",
-        "jest-util": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/globals": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz",
-      "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==",
+    "node_modules/@esbuild/linux-mips64el": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.8.tgz",
+      "integrity": "sha512-CCb67RKahNobjm/eeEqeD/oJfJlrWyw29fgiyB6vcgyq97YAf3gCOuP6qMShYSPXgnlZe/i4a8WFHBw6N8bYAA==",
+      "cpu": [
+        "mips64el"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/environment": "^29.7.0",
-        "@jest/expect": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "jest-mock": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz",
-      "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==",
+    "node_modules/@esbuild/linux-ppc64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.8.tgz",
+      "integrity": "sha512-bytLJOi55y55+mGSdgwZ5qBm0K9WOCh0rx+vavVPx+gqLLhxtSFU0XbeYy/dsAAD6xECGEv4IQeFILaSS2auXw==",
+      "cpu": [
+        "ppc64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@bcoe/v8-coverage": "^0.2.3",
-        "@jest/console": "^29.7.0",
-        "@jest/test-result": "^29.7.0",
-        "@jest/transform": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@jridgewell/trace-mapping": "^0.3.18",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "collect-v8-coverage": "^1.0.0",
-        "exit": "^0.1.2",
-        "glob": "^7.1.3",
-        "graceful-fs": "^4.2.9",
-        "istanbul-lib-coverage": "^3.0.0",
-        "istanbul-lib-instrument": "^6.0.0",
-        "istanbul-lib-report": "^3.0.0",
-        "istanbul-lib-source-maps": "^4.0.0",
-        "istanbul-reports": "^3.1.3",
-        "jest-message-util": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jest-worker": "^29.7.0",
-        "slash": "^3.0.0",
-        "string-length": "^4.0.1",
-        "strip-ansi": "^6.0.0",
-        "v8-to-istanbul": "^9.0.1"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
-      },
-      "peerDependenciesMeta": {
-        "node-notifier": {
-          "optional": true
-        }
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@esbuild/linux-riscv64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.8.tgz",
+      "integrity": "sha512-2YpRyQJmKVBEHSBLa8kBAtbhucaclb6ex4wchfY0Tj3Kg39kpjeJ9vhRU7x4mUpq8ISLXRXH1L0dBYjAeqzZAw==",
+      "cpu": [
+        "riscv64"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/@esbuild/linux-s390x": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.8.tgz",
+      "integrity": "sha512-QgbNY/V3IFXvNf11SS6exkpVcX0LJcob+0RWCgV9OiDAmVElnxciHIisoSix9uzYzScPmS6dJFbZULdSAEkQVw==",
+      "cpu": [
+        "s390x"
+      ],
       "dev": true,
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/istanbul-lib-instrument": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.0.tgz",
-      "integrity": "sha512-x58orMzEVfzPUKqlbLd1hXCnySCxKdDKa6Rjg97CwuLLRI4g3FHTdnExu1OqffVFay6zeMW+T6/DowFLndWnIw==",
+    "node_modules/@esbuild/linux-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.8.tgz",
+      "integrity": "sha512-mM/9S0SbAFDBc4OPoyP6SEOo5324LpUxdpeIUUSrSTOfhHU9hEfqRngmKgqILqwx/0DVJBzeNW7HmLEWp9vcOA==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@babel/core": "^7.12.3",
-        "@babel/parser": "^7.14.7",
-        "@istanbuljs/schema": "^0.1.2",
-        "istanbul-lib-coverage": "^3.2.0",
-        "semver": "^7.5.4"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": ">=10"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/lru-cache": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+    "node_modules/@esbuild/netbsd-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.8.tgz",
+      "integrity": "sha512-eKUYcWaWTaYr9zbj8GertdVtlt1DTS1gNBWov+iQfWuWyuu59YN6gSEJvFzC5ESJ4kMcKR0uqWThKUn5o8We6Q==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "yallist": "^4.0.0"
-      },
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
       "engines": {
-        "node": ">=10"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/semver": {
-      "version": "7.5.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+    "node_modules/@esbuild/openbsd-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.8.tgz",
+      "integrity": "sha512-Vc9J4dXOboDyMXKD0eCeW0SIeEzr8K9oTHJU+Ci1mZc5njPfhKAqkRt3B/fUNU7dP+mRyralPu8QUkiaQn7iIg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "lru-cache": "^6.0.0"
-      },
-      "bin": {
-        "semver": "bin/semver.js"
-      },
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
       "engines": {
-        "node": ">=10"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/reporters/node_modules/yallist": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-      "dev": true
-    },
-    "node_modules/@jest/schemas": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz",
-      "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==",
+    "node_modules/@esbuild/sunos-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.8.tgz",
+      "integrity": "sha512-0xvOTNuPXI7ft1LYUgiaXtpCEjp90RuBBYovdd2lqAFxje4sEucurg30M1WIm03+3jxByd3mfo+VUmPtRSVuOw==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@sinclair/typebox": "^0.27.8"
-      },
+      "optional": true,
+      "os": [
+        "sunos"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/source-map": {
-      "version": "28.1.2",
-      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-28.1.2.tgz",
-      "integrity": "sha512-cV8Lx3BeStJb8ipPHnqVw/IM2VCMWO3crWZzYodSIkxXnRcXJipCdx1JCK0K5MsJJouZQTH73mzf4vgxRaH9ww==",
+    "node_modules/@esbuild/win32-arm64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.8.tgz",
+      "integrity": "sha512-G0JQwUI5WdEFEnYNKzklxtBheCPkuDdu1YrtRrjuQv30WsYbkkoixKxLLv8qhJmNI+ATEWquZe/N0d0rpr55Mg==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jridgewell/trace-mapping": "^0.3.13",
-        "callsites": "^3.0.0",
-        "graceful-fs": "^4.2.9"
-      },
+      "optional": true,
+      "os": [
+        "win32"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/test-result": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz",
-      "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==",
+    "node_modules/@esbuild/win32-ia32": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.8.tgz",
+      "integrity": "sha512-Fqy63515xl20OHGFykjJsMnoIWS+38fqfg88ClvPXyDbLtgXal2DTlhb1TfTX34qWi3u4I7Cq563QcHpqgLx8w==",
+      "cpu": [
+        "ia32"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/console": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "collect-v8-coverage": "^1.0.0"
-      },
+      "optional": true,
+      "os": [
+        "win32"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/test-sequencer": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz",
-      "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==",
+    "node_modules/@esbuild/win32-x64": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.8.tgz",
+      "integrity": "sha512-1iuezdyDNngPnz8rLRDO2C/ZZ/emJLb72OsZeqQ6gL6Avko/XCXZw+NuxBSNhBAP13Hie418V7VMt9et1FMvpg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/test-result": "^29.7.0",
-        "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^29.7.0",
-        "slash": "^3.0.0"
-      },
-      "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=12"
       }
     },
-    "node_modules/@jest/transform": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz",
-      "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==",
+    "node_modules/@eslint/eslintrc": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz",
+      "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==",
       "dev": true,
       "dependencies": {
-        "@babel/core": "^7.11.6",
-        "@jest/types": "^29.6.3",
-        "@jridgewell/trace-mapping": "^0.3.18",
-        "babel-plugin-istanbul": "^6.1.1",
-        "chalk": "^4.0.0",
-        "convert-source-map": "^2.0.0",
-        "fast-json-stable-stringify": "^2.1.0",
-        "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^29.7.0",
-        "jest-regex-util": "^29.6.3",
-        "jest-util": "^29.7.0",
-        "micromatch": "^4.0.4",
-        "pirates": "^4.0.4",
-        "slash": "^3.0.0",
-        "write-file-atomic": "^4.0.2"
+        "ajv": "^6.12.4",
+        "debug": "^4.3.2",
+        "espree": "^9.4.0",
+        "globals": "^13.19.0",
+        "ignore": "^5.2.0",
+        "import-fresh": "^3.2.1",
+        "js-yaml": "^4.1.0",
+        "minimatch": "^3.1.2",
+        "strip-json-comments": "^3.1.1"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/@jest/transform/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@eslint/eslintrc/node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=10"
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
       }
     },
-    "node_modules/@jest/transform/node_modules/convert-source-map": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
-      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
-      "dev": true
-    },
-    "node_modules/@jest/types": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz",
-      "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==",
+    "node_modules/@eslint/eslintrc/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "@jest/schemas": "^29.6.3",
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "@types/istanbul-reports": "^3.0.0",
-        "@types/node": "*",
-        "@types/yargs": "^17.0.8",
-        "chalk": "^4.0.0"
-      },
-      "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/@jest/types/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/@eslint/eslintrc/node_modules/globals": {
+      "version": "13.23.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz",
+      "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "type-fest": "^0.20.2"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@jridgewell/gen-mapping": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz",
-      "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==",
+    "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "node_modules/@eslint/eslintrc/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
       "dependencies": {
-        "@jridgewell/set-array": "^1.0.0",
-        "@jridgewell/sourcemap-codec": "^1.4.10"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=6.0.0"
+        "node": "*"
       }
     },
-    "node_modules/@jridgewell/resolve-uri": {
+    "node_modules/@eslint/eslintrc/node_modules/strip-json-comments": {
       "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz",
-      "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+      "dev": true,
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@jridgewell/set-array": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
-      "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
+      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=14"
       }
     },
-    "node_modules/@jridgewell/source-map": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.5.tgz",
-      "integrity": "sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==",
+    "node_modules/@foliojs-fork/fontkit": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/@foliojs-fork/fontkit/-/fontkit-1.9.1.tgz",
+      "integrity": "sha512-U589voc2/ROnvx1CyH9aNzOQWJp127JGU1QAylXGQ7LoEAF6hMmahZLQ4eqAcgHUw+uyW4PjtCItq9qudPkK3A==",
       "dev": true,
       "dependencies": {
-        "@jridgewell/gen-mapping": "^0.3.0",
-        "@jridgewell/trace-mapping": "^0.3.9"
+        "@foliojs-fork/restructure": "^2.0.2",
+        "brfs": "^2.0.0",
+        "brotli": "^1.2.0",
+        "browserify-optional": "^1.0.1",
+        "clone": "^1.0.4",
+        "deep-equal": "^1.0.0",
+        "dfa": "^1.2.0",
+        "tiny-inflate": "^1.0.2",
+        "unicode-properties": "^1.2.2",
+        "unicode-trie": "^2.0.0"
       }
     },
-    "node_modules/@jridgewell/source-map/node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
-      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+    "node_modules/@foliojs-fork/linebreak": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@foliojs-fork/linebreak/-/linebreak-1.1.1.tgz",
+      "integrity": "sha512-pgY/+53GqGQI+mvDiyprvPWgkTlVBS8cxqee03ejm6gKAQNsR1tCYCIvN9FHy7otZajzMqCgPOgC4cHdt4JPig==",
       "dev": true,
       "dependencies": {
-        "@jridgewell/set-array": "^1.0.1",
-        "@jridgewell/sourcemap-codec": "^1.4.10",
-        "@jridgewell/trace-mapping": "^0.3.9"
-      },
-      "engines": {
-        "node": ">=6.0.0"
+        "base64-js": "1.3.1",
+        "brfs": "^2.0.2",
+        "unicode-trie": "^2.0.0"
       }
     },
-    "node_modules/@jridgewell/sourcemap-codec": {
-      "version": "1.4.15",
-      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
-      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+    "node_modules/@foliojs-fork/linebreak/node_modules/base64-js": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
+      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
+      "dev": true
     },
-    "node_modules/@jridgewell/trace-mapping": {
-      "version": "0.3.19",
-      "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz",
-      "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==",
+    "node_modules/@foliojs-fork/pdfkit": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@foliojs-fork/pdfkit/-/pdfkit-0.14.0.tgz",
+      "integrity": "sha512-nMOiQAv6id89MT3tVTCgc7HxD5ZMANwio2o5yvs5sexQkC0KI3BLaLakpsrHmFfeGFAhqPmZATZGbJGXTUebpg==",
+      "dev": true,
       "dependencies": {
-        "@jridgewell/resolve-uri": "^3.1.0",
-        "@jridgewell/sourcemap-codec": "^1.4.14"
+        "@foliojs-fork/fontkit": "^1.9.1",
+        "@foliojs-fork/linebreak": "^1.1.1",
+        "crypto-js": "^4.2.0",
+        "png-js": "^1.0.0"
       }
     },
-    "node_modules/@juggle/resize-observer": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.4.0.tgz",
-      "integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA=="
+    "node_modules/@foliojs-fork/restructure": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@foliojs-fork/restructure/-/restructure-2.0.2.tgz",
+      "integrity": "sha512-59SgoZ3EXbkfSX7b63tsou/SDGzwUEK6MuB5sKqgVK1/XE0fxmpsOb9DQI8LXW3KfGnAjImCGhhEb7uPPAUVNA==",
+      "dev": true
     },
-    "node_modules/@kurkle/color": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz",
-      "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw=="
+    "node_modules/@gar/promisify": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz",
+      "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==",
+      "dev": true
     },
-    "node_modules/@leichtgewicht/ip-codec": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz",
-      "integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==",
+    "node_modules/@hapi/hoek": {
+      "version": "9.3.0",
+      "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz",
+      "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==",
       "dev": true
     },
-    "node_modules/@ng-bootstrap/ng-bootstrap": {
-      "version": "14.2.0",
-      "resolved": "https://registry.npmjs.org/@ng-bootstrap/ng-bootstrap/-/ng-bootstrap-14.2.0.tgz",
-      "integrity": "sha512-nqEKVXauSontGKqC5WSKpch5TiAGDZB3hluvxkINS0r9LUE6sBQRP3qeYOe7Uwu+UbQcj28NG3qFHhpfnG8KHw==",
+    "node_modules/@hapi/topo": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz",
+      "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==",
+      "dev": true,
       "dependencies": {
-        "tslib": "^2.3.0"
-      },
-      "peerDependencies": {
-        "@angular/common": "^15.0.0",
-        "@angular/core": "^15.0.0",
-        "@angular/forms": "^15.0.0",
-        "@angular/localize": "^15.0.0",
-        "@popperjs/core": "^2.11.6",
-        "rxjs": "^6.5.3 || ^7.4.0"
+        "@hapi/hoek": "^9.0.0"
       }
     },
-    "node_modules/@ngtools/webpack": {
-      "version": "15.2.9",
-      "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-15.2.9.tgz",
-      "integrity": "sha512-nOXUGqKkAEMlCcrhkDwWDzcVdKNH7MNRUXfNzsFc9zdeR/5p3qt6SVMN7OOE3NREyI7P6nzARc3S+6QDBjf3Jg==",
+    "node_modules/@humanwhocodes/config-array": {
+      "version": "0.9.5",
+      "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz",
+      "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==",
       "dev": true,
-      "engines": {
-        "node": "^14.20.0 || ^16.13.0 || >=18.10.0",
-        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
-        "yarn": ">= 1.13.0"
+      "dependencies": {
+        "@humanwhocodes/object-schema": "^1.2.1",
+        "debug": "^4.1.1",
+        "minimatch": "^3.0.4"
       },
-      "peerDependencies": {
-        "@angular/compiler-cli": "^15.0.0",
-        "typescript": ">=4.8.2 <5.0",
-        "webpack": "^5.54.0"
+      "engines": {
+        "node": ">=10.10.0"
       }
     },
-    "node_modules/@ngx-formly/bootstrap": {
-      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/@ngx-formly/bootstrap/-/bootstrap-6.1.1.tgz",
-      "integrity": "sha512-yNzASqUrzvhMndERzoTBCvj1rtsgsmMXiXsqIP7PRJ4AdGtsTZvpxNYZAltdKEgJvc1hS/lDMJdS7IHg2qFN9Q==",
+    "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
       "dependencies": {
-        "tslib": "^2.0.0"
-      },
-      "peerDependencies": {
-        "@ngx-formly/core": "6.1.1",
-        "bootstrap": "^5.0.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/@ngx-formly/core": {
-      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/@ngx-formly/core/-/core-6.1.1.tgz",
-      "integrity": "sha512-6Fg9TBcXXrnUkHqVlpCQbVE5BWJQBvCitQRngW7kiA/+86rhH5mkL19enULWKq7fEMi54uCVvWsz7l6VOaJhLA==",
+    "node_modules/@humanwhocodes/config-array/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
       "dependencies": {
-        "tslib": "^2.0.0"
+        "brace-expansion": "^1.1.7"
       },
-      "peerDependencies": {
-        "@angular/forms": ">=13.2.0",
-        "rxjs": "^6.5.3 || ^7.0.0"
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/@nodelib/fs.scandir": {
-      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
-      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+    "node_modules/@humanwhocodes/object-schema": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz",
+      "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==",
+      "dev": true
+    },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
       "dev": true,
       "dependencies": {
-        "@nodelib/fs.stat": "2.0.5",
-        "run-parallel": "^1.1.9"
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
       },
       "engines": {
-        "node": ">= 8"
+        "node": ">=12"
       }
     },
-    "node_modules/@nodelib/fs.stat": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
-      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+    "node_modules/@isaacs/cliui/node_modules/ansi-regex": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
       "dev": true,
       "engines": {
-        "node": ">= 8"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
       }
     },
-    "node_modules/@nodelib/fs.walk": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
-      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+    "node_modules/@isaacs/cliui/node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
       "dev": true,
-      "dependencies": {
-        "@nodelib/fs.scandir": "2.1.5",
-        "fastq": "^1.6.0"
-      },
       "engines": {
-        "node": ">= 8"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@npmcli/fs": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.0.tgz",
-      "integrity": "sha512-7kZUAaLscfgbwBQRbvdMYaZOWyMEcPTH/tJjnyAWJ/dvvs9Ef+CERx/qJb9GExJpl1qipaDGn7KqHnFGGixd0w==",
+    "node_modules/@isaacs/cliui/node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
+      "dev": true
+    },
+    "node_modules/@isaacs/cliui/node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
       "dev": true,
       "dependencies": {
-        "semver": "^7.3.5"
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@npmcli/git": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-4.1.0.tgz",
-      "integrity": "sha512-9hwoB3gStVfa0N31ymBmrX+GuDGdVA/QWShZVqE0HK2Af+7QGGrCTbZia/SW0ImUTjTne7SP91qxDmtXvDHRPQ==",
+    "node_modules/@isaacs/cliui/node_modules/strip-ansi": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
+      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
       "dev": true,
       "dependencies": {
-        "@npmcli/promise-spawn": "^6.0.0",
-        "lru-cache": "^7.4.4",
-        "npm-pick-manifest": "^8.0.0",
-        "proc-log": "^3.0.0",
-        "promise-inflight": "^1.0.1",
-        "promise-retry": "^2.0.1",
-        "semver": "^7.3.5",
-        "which": "^3.0.0"
+        "ansi-regex": "^6.0.1"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
       }
     },
-    "node_modules/@npmcli/git/node_modules/lru-cache": {
-      "version": "7.18.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
-      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+    "node_modules/@isaacs/cliui/node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
       "dev": true,
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
       "engines": {
         "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
-    "node_modules/@npmcli/git/node_modules/which": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
-      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
+    "node_modules/@istanbuljs/load-nyc-config": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
+      "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==",
       "dev": true,
       "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/which.js"
+        "camelcase": "^5.3.1",
+        "find-up": "^4.1.0",
+        "get-package-type": "^0.1.0",
+        "js-yaml": "^3.13.1",
+        "resolve-from": "^5.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@npmcli/installed-package-contents": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@npmcli/installed-package-contents/-/installed-package-contents-2.0.2.tgz",
-      "integrity": "sha512-xACzLPhnfD51GKvTOOuNX2/V4G4mz9/1I2MfDoye9kBM3RYe5g2YbscsaGoTlaWqkxeiapBWyseULVKpSVHtKQ==",
+    "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "dev": true,
       "dependencies": {
-        "npm-bundled": "^3.0.0",
-        "npm-normalize-package-bin": "^3.0.0"
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
       },
       "bin": {
-        "installed-package-contents": "lib/index.js"
-      },
-      "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "js-yaml": "bin/js-yaml.js"
       }
     },
-    "node_modules/@npmcli/node-gyp": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-3.0.0.tgz",
-      "integrity": "sha512-gp8pRXC2oOxu0DUE1/M3bYtb1b3/DbJ5aM113+XJBgfXdussRAsX0YOrOhdd8WvnAR6auDBvJomGAkLKA5ydxA==",
+    "node_modules/@istanbuljs/schema": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz",
+      "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==",
       "dev": true,
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@npmcli/promise-spawn": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-6.0.2.tgz",
-      "integrity": "sha512-gGq0NJkIGSwdbUt4yhdF8ZrmkGKVz9vAdVzpOfnom+V8PLSmSOVhZwbNvZZS1EYcJN5hzzKBxmmVVAInM6HQLg==",
+    "node_modules/@jest/console": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz",
+      "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==",
       "dev": true,
       "dependencies": {
-        "which": "^3.0.0"
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "jest-message-util": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "slash": "^3.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@npmcli/promise-spawn/node_modules/which": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
-      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
+    "node_modules/@jest/console/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/which.js"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@npmcli/run-script": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-6.0.2.tgz",
-      "integrity": "sha512-NCcr1uQo1k5U+SYlnIrbAh3cxy+OQT1VtqiAbxdymSlptbzBb62AjH2xXgjNCoP073hoa1CfCAcwoZ8k96C4nA==",
+    "node_modules/@jest/console/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "@npmcli/node-gyp": "^3.0.0",
-        "@npmcli/promise-spawn": "^6.0.0",
-        "node-gyp": "^9.0.0",
-        "read-package-json-fast": "^3.0.0",
-        "which": "^3.0.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@npmcli/run-script/node_modules/which": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
-      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
-      "dev": true,
+    "node_modules/@jest/console/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
       "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/which.js"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@nrwl/cli": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/cli/-/cli-15.9.3.tgz",
-      "integrity": "sha512-qiAKHkov3iBx6hroPTitUrkRSUZFQqVgNJiF9gXRFC6pNJe9RS4rlmcIaoUFOboi9CnH5jwblNJVcz8YSVYOvA==",
+    "node_modules/@jest/console/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@jest/console/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "nx": "15.9.3"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/@nrwl/tao": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/tao/-/tao-15.9.3.tgz",
-      "integrity": "sha512-NcjFCbuMa53C3fBrK7qLUImUBySyr9EVwmiZuAv9sZZtm4eILK8w3qihjrB4FFUuLjPU/SViriYXi+hF2tbP4w==",
+    "node_modules/@jest/console/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/console/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "nx": "15.9.3"
+        "has-flag": "^4.0.0"
       },
-      "bin": {
-        "tao": "index.js"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
+    "node_modules/@jest/core": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz",
+      "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==",
+      "dev": true,
+      "dependencies": {
+        "@jest/console": "^29.7.0",
+        "@jest/reporters": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
+        "@jest/transform": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "ansi-escapes": "^4.2.1",
+        "chalk": "^4.0.0",
+        "ci-info": "^3.2.0",
+        "exit": "^0.1.2",
+        "graceful-fs": "^4.2.9",
+        "jest-changed-files": "^29.7.0",
+        "jest-config": "^29.7.0",
+        "jest-haste-map": "^29.7.0",
+        "jest-message-util": "^29.7.0",
+        "jest-regex-util": "^29.6.3",
+        "jest-resolve": "^29.7.0",
+        "jest-resolve-dependencies": "^29.7.0",
+        "jest-runner": "^29.7.0",
+        "jest-runtime": "^29.7.0",
+        "jest-snapshot": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "jest-validate": "^29.7.0",
+        "jest-watcher": "^29.7.0",
+        "micromatch": "^4.0.4",
+        "pretty-format": "^29.7.0",
+        "slash": "^3.0.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      },
+      "peerDependencies": {
+        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
+      },
+      "peerDependenciesMeta": {
+        "node-notifier": {
+          "optional": true
+        }
+      }
     },
-    "node_modules/@nrwl/cli/node_modules/axios": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz",
-      "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==",
+    "node_modules/@jest/core/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/chalk": {
+    "node_modules/@jest/core/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -5245,262 +5510,3035 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/cli-spinners": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.6.1.tgz",
-      "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==",
+    "node_modules/@jest/core/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
-      "engines": {
-        "node": ">=6"
+      "dependencies": {
+        "color-name": "~1.1.4"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/cliui": {
-      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
-      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+    "node_modules/@jest/core/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@jest/core/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "string-width": "^4.2.0",
-        "strip-ansi": "^6.0.0",
-        "wrap-ansi": "^7.0.0"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/fast-glob": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.7.tgz",
-      "integrity": "sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==",
+    "node_modules/@jest/core/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
-      "dependencies": {
-        "@nodelib/fs.stat": "^2.0.2",
-        "@nodelib/fs.walk": "^1.2.3",
-        "glob-parent": "^5.1.2",
-        "merge2": "^1.3.0",
-        "micromatch": "^4.0.4"
-      },
       "engines": {
         "node": ">=8"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+    "node_modules/@jest/core/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=8"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/fs-extra": {
-      "version": "11.1.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz",
-      "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==",
+    "node_modules/@jest/environment": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz",
+      "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==",
       "dev": true,
       "dependencies": {
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
+        "@jest/fake-timers": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "jest-mock": "^29.7.0"
       },
       "engines": {
-        "node": ">=14.14"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/glob": {
-      "version": "7.1.4",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
-      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+    "node_modules/@jest/expect": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz",
+      "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.0.4",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "expect": "^29.7.0",
+        "jest-snapshot": "^29.7.0"
       },
       "engines": {
-        "node": "*"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+    "node_modules/@jest/expect-utils": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz",
+      "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==",
       "dev": true,
       "dependencies": {
-        "argparse": "^2.0.1"
+        "jest-get-type": "^29.6.3"
       },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
-      }
-    },
-    "node_modules/@nrwl/cli/node_modules/lines-and-columns": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-2.0.3.tgz",
-      "integrity": "sha512-cNOjgCnLB+FnvWWtyRTzmB3POJ+cXxTA81LoW7u8JdmhfXzriropYwpjShnz1QLLWsQwY7nIxoDmcPTwphDK9w==",
-      "dev": true,
       "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/lru-cache": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+    "node_modules/@jest/fake-timers": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz",
+      "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==",
       "dev": true,
       "dependencies": {
-        "yallist": "^4.0.0"
+        "@jest/types": "^29.6.3",
+        "@sinonjs/fake-timers": "^10.0.2",
+        "@types/node": "*",
+        "jest-message-util": "^29.7.0",
+        "jest-mock": "^29.7.0",
+        "jest-util": "^29.7.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/minimatch": {
-      "version": "3.0.5",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz",
-      "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==",
+    "node_modules/@jest/globals": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz",
+      "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==",
       "dev": true,
       "dependencies": {
-        "brace-expansion": "^1.1.7"
+        "@jest/environment": "^29.7.0",
+        "@jest/expect": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "jest-mock": "^29.7.0"
       },
       "engines": {
-        "node": "*"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/nx": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/nx/-/nx-15.9.3.tgz",
-      "integrity": "sha512-GLwbykfTABc7/UZjQEEnV1bQbTVC53W+Zj4xWY640/45I4iZf/TUqKMBCgtLZ9v89gEsKOM4zsx55CqHT3bekA==",
+    "node_modules/@jest/reporters": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz",
+      "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@nrwl/cli": "15.9.3",
-        "@nrwl/tao": "15.9.3",
-        "@parcel/watcher": "2.0.4",
-        "@yarnpkg/lockfile": "^1.1.0",
-        "@yarnpkg/parsers": "^3.0.0-rc.18",
-        "@zkochan/js-yaml": "0.0.6",
-        "axios": "^1.0.0",
-        "chalk": "^4.1.0",
-        "cli-cursor": "3.1.0",
-        "cli-spinners": "2.6.1",
-        "cliui": "^7.0.2",
-        "dotenv": "~10.0.0",
-        "enquirer": "~2.3.6",
-        "fast-glob": "3.2.7",
-        "figures": "3.2.0",
-        "flat": "^5.0.2",
-        "fs-extra": "^11.1.0",
-        "glob": "7.1.4",
-        "ignore": "^5.0.4",
-        "js-yaml": "4.1.0",
-        "jsonc-parser": "3.2.0",
-        "lines-and-columns": "~2.0.3",
-        "minimatch": "3.0.5",
-        "npm-run-path": "^4.0.1",
-        "open": "^8.4.0",
-        "semver": "7.3.4",
-        "string-width": "^4.2.3",
-        "strong-log-transformer": "^2.1.0",
-        "tar-stream": "~2.2.0",
-        "tmp": "~0.2.1",
-        "tsconfig-paths": "^4.1.2",
-        "tslib": "^2.3.0",
-        "v8-compile-cache": "2.3.0",
-        "yargs": "^17.6.2",
-        "yargs-parser": "21.1.1"
-      },
-      "bin": {
-        "nx": "bin/nx.js"
+        "@bcoe/v8-coverage": "^0.2.3",
+        "@jest/console": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
+        "@jest/transform": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@jridgewell/trace-mapping": "^0.3.18",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "collect-v8-coverage": "^1.0.0",
+        "exit": "^0.1.2",
+        "glob": "^7.1.3",
+        "graceful-fs": "^4.2.9",
+        "istanbul-lib-coverage": "^3.0.0",
+        "istanbul-lib-instrument": "^6.0.0",
+        "istanbul-lib-report": "^3.0.0",
+        "istanbul-lib-source-maps": "^4.0.0",
+        "istanbul-reports": "^3.1.3",
+        "jest-message-util": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "jest-worker": "^29.7.0",
+        "slash": "^3.0.0",
+        "string-length": "^4.0.1",
+        "strip-ansi": "^6.0.0",
+        "v8-to-istanbul": "^9.0.1"
       },
-      "optionalDependencies": {
-        "@nrwl/nx-darwin-arm64": "15.9.3",
-        "@nrwl/nx-darwin-x64": "15.9.3",
-        "@nrwl/nx-linux-arm-gnueabihf": "15.9.3",
-        "@nrwl/nx-linux-arm64-gnu": "15.9.3",
-        "@nrwl/nx-linux-arm64-musl": "15.9.3",
-        "@nrwl/nx-linux-x64-gnu": "15.9.3",
-        "@nrwl/nx-linux-x64-musl": "15.9.3",
-        "@nrwl/nx-win32-arm64-msvc": "15.9.3",
-        "@nrwl/nx-win32-x64-msvc": "15.9.3"
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       },
       "peerDependencies": {
-        "@swc-node/register": "^1.4.2",
-        "@swc/core": "^1.2.173"
+        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
       },
       "peerDependenciesMeta": {
-        "@swc-node/register": {
-          "optional": true
-        },
-        "@swc/core": {
+        "node-notifier": {
           "optional": true
         }
       }
     },
-    "node_modules/@nrwl/cli/node_modules/proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
+    "node_modules/@jest/reporters/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@jest/reporters/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/istanbul-lib-instrument": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.1.tgz",
+      "integrity": "sha512-EAMEJBsYuyyztxMxW3g7ugGPkrZsV57v0Hmv3mm1uQsmB+QnZuepg731CRaIgeUVSdmsTngOkSnauNF8p7FIhA==",
+      "dev": true,
+      "dependencies": {
+        "@babel/core": "^7.12.3",
+        "@babel/parser": "^7.14.7",
+        "@istanbuljs/schema": "^0.1.2",
+        "istanbul-lib-coverage": "^3.2.0",
+        "semver": "^7.5.4"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/jest-worker": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz",
+      "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "jest-util": "^29.7.0",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/reporters/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@jest/schemas": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz",
+      "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==",
+      "dev": true,
+      "dependencies": {
+        "@sinclair/typebox": "^0.27.8"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/source-map": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz",
+      "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==",
+      "dev": true,
+      "dependencies": {
+        "@jridgewell/trace-mapping": "^0.3.18",
+        "callsites": "^3.0.0",
+        "graceful-fs": "^4.2.9"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/test-result": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz",
+      "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==",
+      "dev": true,
+      "dependencies": {
+        "@jest/console": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "collect-v8-coverage": "^1.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/test-sequencer": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz",
+      "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==",
+      "dev": true,
+      "dependencies": {
+        "@jest/test-result": "^29.7.0",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^29.7.0",
+        "slash": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/test-sequencer/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/transform": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz",
+      "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==",
+      "dev": true,
+      "dependencies": {
+        "@babel/core": "^7.11.6",
+        "@jest/types": "^29.6.3",
+        "@jridgewell/trace-mapping": "^0.3.18",
+        "babel-plugin-istanbul": "^6.1.1",
+        "chalk": "^4.0.0",
+        "convert-source-map": "^2.0.0",
+        "fast-json-stable-stringify": "^2.1.0",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^29.7.0",
+        "jest-regex-util": "^29.6.3",
+        "jest-util": "^29.7.0",
+        "micromatch": "^4.0.4",
+        "pirates": "^4.0.4",
+        "slash": "^3.0.0",
+        "write-file-atomic": "^4.0.2"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@jest/transform/node_modules/convert-source-map": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
+      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
+      "dev": true
+    },
+    "node_modules/@jest/transform/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/transform/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/types": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz",
+      "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==",
+      "dev": true,
+      "dependencies": {
+        "@jest/schemas": "^29.6.3",
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "@types/istanbul-reports": "^3.0.0",
+        "@types/node": "*",
+        "@types/yargs": "^17.0.8",
+        "chalk": "^4.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@jest/types/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@jest/types/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@jest/types/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/@jest/types/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@jest/types/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jest/types/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@jridgewell/gen-mapping": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
+      "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
+      "dependencies": {
+        "@jridgewell/set-array": "^1.0.1",
+        "@jridgewell/sourcemap-codec": "^1.4.10",
+        "@jridgewell/trace-mapping": "^0.3.9"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@jridgewell/resolve-uri": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz",
+      "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==",
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@jridgewell/set-array": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
+      "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/@jridgewell/source-map": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.5.tgz",
+      "integrity": "sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==",
+      "dev": true,
+      "dependencies": {
+        "@jridgewell/gen-mapping": "^0.3.0",
+        "@jridgewell/trace-mapping": "^0.3.9"
+      }
+    },
+    "node_modules/@jridgewell/sourcemap-codec": {
+      "version": "1.4.15",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
+      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+    },
+    "node_modules/@jridgewell/trace-mapping": {
+      "version": "0.3.20",
+      "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz",
+      "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==",
+      "dependencies": {
+        "@jridgewell/resolve-uri": "^3.1.0",
+        "@jridgewell/sourcemap-codec": "^1.4.14"
+      }
+    },
+    "node_modules/@juggle/resize-observer": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.4.0.tgz",
+      "integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA=="
+    },
+    "node_modules/@kurkle/color": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz",
+      "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw=="
+    },
+    "node_modules/@leichtgewicht/ip-codec": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz",
+      "integrity": "sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==",
+      "dev": true
+    },
+    "node_modules/@ng-bootstrap/ng-bootstrap": {
+      "version": "14.2.0",
+      "resolved": "https://registry.npmjs.org/@ng-bootstrap/ng-bootstrap/-/ng-bootstrap-14.2.0.tgz",
+      "integrity": "sha512-nqEKVXauSontGKqC5WSKpch5TiAGDZB3hluvxkINS0r9LUE6sBQRP3qeYOe7Uwu+UbQcj28NG3qFHhpfnG8KHw==",
+      "dependencies": {
+        "tslib": "^2.3.0"
+      },
+      "peerDependencies": {
+        "@angular/common": "^15.0.0",
+        "@angular/core": "^15.0.0",
+        "@angular/forms": "^15.0.0",
+        "@angular/localize": "^15.0.0",
+        "@popperjs/core": "^2.11.6",
+        "rxjs": "^6.5.3 || ^7.4.0"
+      }
+    },
+    "node_modules/@ngtools/webpack": {
+      "version": "15.2.9",
+      "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-15.2.9.tgz",
+      "integrity": "sha512-nOXUGqKkAEMlCcrhkDwWDzcVdKNH7MNRUXfNzsFc9zdeR/5p3qt6SVMN7OOE3NREyI7P6nzARc3S+6QDBjf3Jg==",
+      "dev": true,
+      "engines": {
+        "node": "^14.20.0 || ^16.13.0 || >=18.10.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
+      },
+      "peerDependencies": {
+        "@angular/compiler-cli": "^15.0.0",
+        "typescript": ">=4.8.2 <5.0",
+        "webpack": "^5.54.0"
+      }
+    },
+    "node_modules/@ngx-formly/bootstrap": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/@ngx-formly/bootstrap/-/bootstrap-6.1.1.tgz",
+      "integrity": "sha512-yNzASqUrzvhMndERzoTBCvj1rtsgsmMXiXsqIP7PRJ4AdGtsTZvpxNYZAltdKEgJvc1hS/lDMJdS7IHg2qFN9Q==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "@ngx-formly/core": "6.1.1",
+        "bootstrap": "^5.0.0"
+      }
+    },
+    "node_modules/@ngx-formly/core": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/@ngx-formly/core/-/core-6.1.1.tgz",
+      "integrity": "sha512-6Fg9TBcXXrnUkHqVlpCQbVE5BWJQBvCitQRngW7kiA/+86rhH5mkL19enULWKq7fEMi54uCVvWsz7l6VOaJhLA==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "@angular/forms": ">=13.2.0",
+        "rxjs": "^6.5.3 || ^7.0.0"
+      }
+    },
+    "node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@npmcli/fs": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.0.tgz",
+      "integrity": "sha512-7kZUAaLscfgbwBQRbvdMYaZOWyMEcPTH/tJjnyAWJ/dvvs9Ef+CERx/qJb9GExJpl1qipaDGn7KqHnFGGixd0w==",
+      "dev": true,
+      "dependencies": {
+        "semver": "^7.3.5"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/fs/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@npmcli/fs/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@npmcli/fs/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@npmcli/git": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-4.1.0.tgz",
+      "integrity": "sha512-9hwoB3gStVfa0N31ymBmrX+GuDGdVA/QWShZVqE0HK2Af+7QGGrCTbZia/SW0ImUTjTne7SP91qxDmtXvDHRPQ==",
+      "dev": true,
+      "dependencies": {
+        "@npmcli/promise-spawn": "^6.0.0",
+        "lru-cache": "^7.4.4",
+        "npm-pick-manifest": "^8.0.0",
+        "proc-log": "^3.0.0",
+        "promise-inflight": "^1.0.1",
+        "promise-retry": "^2.0.1",
+        "semver": "^7.3.5",
+        "which": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/git/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@npmcli/git/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@npmcli/git/node_modules/semver/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@npmcli/git/node_modules/which": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
+      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
+      "dev": true,
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/which.js"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/git/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@npmcli/installed-package-contents": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@npmcli/installed-package-contents/-/installed-package-contents-2.0.2.tgz",
+      "integrity": "sha512-xACzLPhnfD51GKvTOOuNX2/V4G4mz9/1I2MfDoye9kBM3RYe5g2YbscsaGoTlaWqkxeiapBWyseULVKpSVHtKQ==",
+      "dev": true,
+      "dependencies": {
+        "npm-bundled": "^3.0.0",
+        "npm-normalize-package-bin": "^3.0.0"
+      },
+      "bin": {
+        "installed-package-contents": "lib/index.js"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/move-file": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz",
+      "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==",
+      "deprecated": "This functionality has been moved to @npmcli/fs",
+      "dev": true,
+      "dependencies": {
+        "mkdirp": "^1.0.4",
+        "rimraf": "^3.0.2"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/@npmcli/node-gyp": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-3.0.0.tgz",
+      "integrity": "sha512-gp8pRXC2oOxu0DUE1/M3bYtb1b3/DbJ5aM113+XJBgfXdussRAsX0YOrOhdd8WvnAR6auDBvJomGAkLKA5ydxA==",
+      "dev": true,
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/promise-spawn": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-6.0.2.tgz",
+      "integrity": "sha512-gGq0NJkIGSwdbUt4yhdF8ZrmkGKVz9vAdVzpOfnom+V8PLSmSOVhZwbNvZZS1EYcJN5hzzKBxmmVVAInM6HQLg==",
+      "dev": true,
+      "dependencies": {
+        "which": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/promise-spawn/node_modules/which": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
+      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
+      "dev": true,
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/which.js"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/run-script": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-6.0.2.tgz",
+      "integrity": "sha512-NCcr1uQo1k5U+SYlnIrbAh3cxy+OQT1VtqiAbxdymSlptbzBb62AjH2xXgjNCoP073hoa1CfCAcwoZ8k96C4nA==",
+      "dev": true,
+      "dependencies": {
+        "@npmcli/node-gyp": "^3.0.0",
+        "@npmcli/promise-spawn": "^6.0.0",
+        "node-gyp": "^9.0.0",
+        "read-package-json-fast": "^3.0.0",
+        "which": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@npmcli/run-script/node_modules/which": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-3.0.1.tgz",
+      "integrity": "sha512-XA1b62dzQzLfaEOSQFTCOd5KFf/1VSzZo7/7TUjnya6u0vGGKzU96UQBZTAThCb2j4/xjBAyii1OhRLJEivHvg==",
+      "dev": true,
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/which.js"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@nrwl/cli": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/cli/-/cli-15.9.3.tgz",
+      "integrity": "sha512-qiAKHkov3iBx6hroPTitUrkRSUZFQqVgNJiF9gXRFC6pNJe9RS4rlmcIaoUFOboi9CnH5jwblNJVcz8YSVYOvA==",
+      "dev": true,
+      "dependencies": {
+        "nx": "15.9.3"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/@nrwl/tao": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/tao/-/tao-15.9.3.tgz",
+      "integrity": "sha512-NcjFCbuMa53C3fBrK7qLUImUBySyr9EVwmiZuAv9sZZtm4eILK8w3qihjrB4FFUuLjPU/SViriYXi+hF2tbP4w==",
+      "dev": true,
+      "dependencies": {
+        "nx": "15.9.3"
+      },
+      "bin": {
+        "tao": "index.js"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/@zkochan/js-yaml": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/@zkochan/js-yaml/-/js-yaml-0.0.6.tgz",
+      "integrity": "sha512-nzvgl3VfhcELQ8LyVrYOru+UtAy1nrygk2+AGbTm8a5YcO6o8lSjAT+pfg3vJWxIoZKOUhrK6UU7xW/+00kQrg==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/cli-spinners": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.6.1.tgz",
+      "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/cliui": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
+      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+      "dev": true,
+      "dependencies": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0",
+        "wrap-ansi": "^7.0.0"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@nrwl/cli/node_modules/fast-glob": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.7.tgz",
+      "integrity": "sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.4"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/fs-extra": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz",
+      "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==",
+      "dev": true,
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.14"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/glob": {
+      "version": "7.1.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/lines-and-columns": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-2.0.4.tgz",
+      "integrity": "sha512-wM1+Z03eypVAVUCE7QdSqpVIvelbOakn1M0bPDoA4SGWPx3sNDVUiMo3L6To6WWGClB7VyXnhQ4Sn7gxiJbE6A==",
+      "dev": true,
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/minimatch": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz",
+      "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/nx": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/nx/-/nx-15.9.3.tgz",
+      "integrity": "sha512-GLwbykfTABc7/UZjQEEnV1bQbTVC53W+Zj4xWY640/45I4iZf/TUqKMBCgtLZ9v89gEsKOM4zsx55CqHT3bekA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@nrwl/cli": "15.9.3",
+        "@nrwl/tao": "15.9.3",
+        "@parcel/watcher": "2.0.4",
+        "@yarnpkg/lockfile": "^1.1.0",
+        "@yarnpkg/parsers": "^3.0.0-rc.18",
+        "@zkochan/js-yaml": "0.0.6",
+        "axios": "^1.0.0",
+        "chalk": "^4.1.0",
+        "cli-cursor": "3.1.0",
+        "cli-spinners": "2.6.1",
+        "cliui": "^7.0.2",
+        "dotenv": "~10.0.0",
+        "enquirer": "~2.3.6",
+        "fast-glob": "3.2.7",
+        "figures": "3.2.0",
+        "flat": "^5.0.2",
+        "fs-extra": "^11.1.0",
+        "glob": "7.1.4",
+        "ignore": "^5.0.4",
+        "js-yaml": "4.1.0",
+        "jsonc-parser": "3.2.0",
+        "lines-and-columns": "~2.0.3",
+        "minimatch": "3.0.5",
+        "npm-run-path": "^4.0.1",
+        "open": "^8.4.0",
+        "semver": "7.3.4",
+        "string-width": "^4.2.3",
+        "strong-log-transformer": "^2.1.0",
+        "tar-stream": "~2.2.0",
+        "tmp": "~0.2.1",
+        "tsconfig-paths": "^4.1.2",
+        "tslib": "^2.3.0",
+        "v8-compile-cache": "2.3.0",
+        "yargs": "^17.6.2",
+        "yargs-parser": "21.1.1"
+      },
+      "bin": {
+        "nx": "bin/nx.js"
+      },
+      "optionalDependencies": {
+        "@nrwl/nx-darwin-arm64": "15.9.3",
+        "@nrwl/nx-darwin-x64": "15.9.3",
+        "@nrwl/nx-linux-arm-gnueabihf": "15.9.3",
+        "@nrwl/nx-linux-arm64-gnu": "15.9.3",
+        "@nrwl/nx-linux-arm64-musl": "15.9.3",
+        "@nrwl/nx-linux-x64-gnu": "15.9.3",
+        "@nrwl/nx-linux-x64-musl": "15.9.3",
+        "@nrwl/nx-win32-arm64-msvc": "15.9.3",
+        "@nrwl/nx-win32-x64-msvc": "15.9.3"
+      },
+      "peerDependencies": {
+        "@swc-node/register": "^1.4.2",
+        "@swc/core": "^1.2.173"
+      },
+      "peerDependenciesMeta": {
+        "@swc-node/register": {
+          "optional": true
+        },
+        "@swc/core": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/semver": {
+      "version": "7.3.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
+      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "dev": true,
+      "dependencies": {
+        "rimraf": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.17.0"
+      }
+    },
+    "node_modules/@nrwl/cli/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@nrwl/devkit": {
+      "version": "13.1.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/devkit/-/devkit-13.1.3.tgz",
+      "integrity": "sha512-TAAsZJvVc/obeH0rZKY6miVhyM2GHGb8qIWp9MAIdLlXf4VDcNC7rxwb5OrGVSwuTTjqGYBGPUx0yEogOOJthA==",
+      "dev": true,
+      "dependencies": {
+        "@nrwl/tao": "13.1.3",
+        "ejs": "^3.1.5",
+        "ignore": "^5.0.4",
+        "rxjs": "^6.5.4",
+        "semver": "7.3.4",
+        "tslib": "^2.0.0"
+      }
+    },
+    "node_modules/@nrwl/devkit/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/devkit/node_modules/semver": {
+      "version": "7.3.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
+      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/devkit/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@nrwl/nx-darwin-arm64": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-darwin-arm64/-/nx-darwin-arm64-15.9.3.tgz",
+      "integrity": "sha512-2htJzVa+S/uLg5tj4nbO/tRz2SRMQIpT6EeWMgDGuEKQdpuRLVj2ez9hMpkRn9tl1tBUwR05hbV28DnOLRESVA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-darwin-x64": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-darwin-x64/-/nx-darwin-x64-15.9.3.tgz",
+      "integrity": "sha512-p+8UkfC6KTLOX4XRt7NSP8DoTzEgs73+SN0csoXT9VsNO35+F0Z5zMZxpEc7RVo5Wen/4PGh2OWA+8gtgntsJQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-linux-arm-gnueabihf": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-15.9.3.tgz",
+      "integrity": "sha512-xwW7bZtggrxhFbYvvWWArtcSWwoxWzi/4wNgP3wPbcZFNZiraahVQSpIyJXrS9aajGbdvuDBM8cbDsMj9v7mwg==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-linux-arm64-gnu": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-15.9.3.tgz",
+      "integrity": "sha512-KNxDL2OAHxhFqztEjv2mNwXD6xrzoUury7NsYZYqlxJUNc3YYBfRSLEatnw491crvMBndbxfGVTWEO9S4YmRuw==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-linux-arm64-musl": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm64-musl/-/nx-linux-arm64-musl-15.9.3.tgz",
+      "integrity": "sha512-AxoZzfsXH7ZqDE+WrQtRumufIcSIBw4U/LikiDLaWWoGtNpAfKLkD/PHirZiNxHIeGy1Toi4ccMUolXbafLVFw==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-linux-x64-gnu": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-x64-gnu/-/nx-linux-x64-gnu-15.9.3.tgz",
+      "integrity": "sha512-P8AOPRufvV4a5cSczNsw84zFAI7NgAiEBTybYcyymdNJmo0iArJXEmvj/G4mB20O8VCsCkwqMYAu6nQEnES1Kw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-linux-x64-musl": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-x64-musl/-/nx-linux-x64-musl-15.9.3.tgz",
+      "integrity": "sha512-4ZYDp7T319+xbw7Z7KVtRefzaXJipZfgrM49r+Y1FAfYDc8y18zvKz3slK26wfWz+EUZwKsa/DfA2KmyRG3DvQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-win32-arm64-msvc": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-15.9.3.tgz",
+      "integrity": "sha512-UhgxIPgTZBKN1oxlLPSklkSzVL3hA4lAiVc9A0Utumpbp0ob/Xx+2vHzg3cnmNH3jWkZ+9OsC2dKyeMB6gAbSw==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/nx-win32-x64-msvc": {
+      "version": "15.9.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/nx-win32-x64-msvc/-/nx-win32-x64-msvc-15.9.3.tgz",
+      "integrity": "sha512-gdnvqURKnu0EQGOFJ6NUKq6wSB+viNb7Z8qtKhzSmFwVjT8akOnLWn7ZhL9v28TAjLM7/s1Mwvmz/IMj1PGlcQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@nrwl/tao": {
+      "version": "13.1.3",
+      "resolved": "https://registry.npmjs.org/@nrwl/tao/-/tao-13.1.3.tgz",
+      "integrity": "sha512-/IwJgSgCBD1SaF+n8RuXX2OxDAh8ut/+P8pMswjm8063ac30UlAHjQ4XTYyskLH8uoUmNi2hNaGgHUrkwt7tQA==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "4.1.0",
+        "enquirer": "~2.3.6",
+        "fs-extra": "^9.1.0",
+        "jsonc-parser": "3.0.0",
+        "nx": "13.1.3",
+        "rxjs": "^6.5.4",
+        "rxjs-for-await": "0.0.2",
+        "semver": "7.3.4",
+        "tmp": "~0.2.1",
+        "tslib": "^2.0.0",
+        "yargs-parser": "20.0.0"
+      },
+      "bin": {
+        "tao": "index.js"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/chalk": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+      "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/@nrwl/tao/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/jsonc-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz",
+      "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==",
+      "dev": true
+    },
+    "node_modules/@nrwl/tao/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/semver": {
+      "version": "7.3.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
+      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "dev": true,
+      "dependencies": {
+        "rimraf": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.17.0"
+      }
+    },
+    "node_modules/@nrwl/tao/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@nrwl/tao/node_modules/yargs-parser": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.0.0.tgz",
+      "integrity": "sha512-8eblPHTL7ZWRkyjIZJjnGf+TijiKJSwA24svzLRVvtgoi/RZiKa9fFQTrlx0OKLnyHSdt/enrdadji6WFfESVA==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@parcel/watcher": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.0.4.tgz",
+      "integrity": "sha512-cTDi+FUDBIUOBKEtj+nhiJ71AZVlkAsQFuGQTun5tV9mwQBQgZvhCzG+URPQc8myeN32yRVZEfVAPCs1RW+Jvg==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "node-addon-api": "^3.2.1",
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">= 10.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/parcel"
+      }
+    },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "dev": true,
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@popperjs/core": {
+      "version": "2.10.2",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.10.2.tgz",
+      "integrity": "sha512-IXf3XA7+XyN7CP9gGh/XB0UxVMlvARGEgGXLubFICsUMGz6Q+DU+i4gGlpOxTjKvXjkJDJC8YdqdKkDj9qZHEQ==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/popperjs"
+      }
+    },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "dev": true,
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+      "dev": true
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "dev": true
+    },
+    "node_modules/@schematics/angular": {
+      "version": "15.2.9",
+      "resolved": "https://registry.npmjs.org/@schematics/angular/-/angular-15.2.9.tgz",
+      "integrity": "sha512-0Lit6TLNUwcAYiEkXgZp3vY9xAO1cnZCBXuUcp+6v+Ddnrt2w/YOiGe74p21cYe0StkTpTljsqsKBTiX7TMjQg==",
+      "dev": true,
+      "dependencies": {
+        "@angular-devkit/core": "15.2.9",
+        "@angular-devkit/schematics": "15.2.9",
+        "jsonc-parser": "3.2.0"
+      },
+      "engines": {
+        "node": "^14.20.0 || ^16.13.0 || >=18.10.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
+      }
+    },
+    "node_modules/@sideway/address": {
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz",
+      "integrity": "sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==",
+      "dev": true,
+      "dependencies": {
+        "@hapi/hoek": "^9.0.0"
+      }
+    },
+    "node_modules/@sideway/formula": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz",
+      "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==",
+      "dev": true
+    },
+    "node_modules/@sideway/pinpoint": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz",
+      "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==",
+      "dev": true
+    },
+    "node_modules/@sigstore/bundle": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-1.1.0.tgz",
+      "integrity": "sha512-PFutXEy0SmQxYI4texPw3dd2KewuNqv7OuK1ZFtY2fM754yhvG2KdgwIhRnoEE2uHdtdGNQ8s0lb94dW9sELog==",
+      "dev": true,
+      "dependencies": {
+        "@sigstore/protobuf-specs": "^0.2.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@sigstore/protobuf-specs": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@sigstore/protobuf-specs/-/protobuf-specs-0.2.1.tgz",
+      "integrity": "sha512-XTWVxnWJu+c1oCshMLwnKvz8ZQJJDVOlciMfgpJBQbThVjKTCG8dwyhgLngBD2KN0ap9F/gOV8rFDEx8uh7R2A==",
+      "dev": true,
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@sigstore/sign": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@sigstore/sign/-/sign-1.0.0.tgz",
+      "integrity": "sha512-INxFVNQteLtcfGmcoldzV6Je0sbbfh9I16DM4yJPw3j5+TFP8X6uIiA18mvpEa9yyeycAKgPmOA3X9hVdVTPUA==",
+      "dev": true,
+      "dependencies": {
+        "@sigstore/bundle": "^1.1.0",
+        "@sigstore/protobuf-specs": "^0.2.0",
+        "make-fetch-happen": "^11.0.1"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@sigstore/sign/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@sigstore/sign/node_modules/make-fetch-happen": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-11.1.1.tgz",
+      "integrity": "sha512-rLWS7GCSTcEujjVBs2YqG7Y4643u8ucvCJeSRqiLYhesrDuzeuFIk37xREzAsfQaqzl8b9rNCE4m6J8tvX4Q8w==",
+      "dev": true,
+      "dependencies": {
+        "agentkeepalive": "^4.2.1",
+        "cacache": "^17.0.0",
+        "http-cache-semantics": "^4.1.1",
+        "http-proxy-agent": "^5.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "is-lambda": "^1.0.1",
+        "lru-cache": "^7.7.1",
+        "minipass": "^5.0.0",
+        "minipass-fetch": "^3.0.0",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "negotiator": "^0.6.3",
+        "promise-retry": "^2.0.1",
+        "socks-proxy-agent": "^7.0.0",
+        "ssri": "^10.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@sigstore/sign/node_modules/minipass": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+      "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@sigstore/sign/node_modules/minipass-fetch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.4.tgz",
+      "integrity": "sha512-jHAqnA728uUpIaFm7NWsCnqKT6UqZz7GcI/bDpPATuwYyKwJwW0remxSCxUlKiEty+eopHGa3oc8WxgQ1FFJqg==",
+      "dev": true,
+      "dependencies": {
+        "minipass": "^7.0.3",
+        "minipass-sized": "^1.0.3",
+        "minizlib": "^2.1.2"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      },
+      "optionalDependencies": {
+        "encoding": "^0.1.13"
+      }
+    },
+    "node_modules/@sigstore/sign/node_modules/minipass-fetch/node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/@sigstore/tuf": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@sigstore/tuf/-/tuf-1.0.3.tgz",
+      "integrity": "sha512-2bRovzs0nJZFlCN3rXirE4gwxCn97JNjMmwpecqlbgV9WcxX7WRuIrgzx/X7Ib7MYRbyUTpBYE0s2x6AmZXnlg==",
+      "dev": true,
+      "dependencies": {
+        "@sigstore/protobuf-specs": "^0.2.0",
+        "tuf-js": "^1.1.7"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@sinclair/typebox": {
+      "version": "0.27.8",
+      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
+      "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==",
+      "dev": true
+    },
+    "node_modules/@sinonjs/commons": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
+      "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
+      "dev": true,
+      "dependencies": {
+        "type-detect": "4.0.8"
+      }
+    },
+    "node_modules/@sinonjs/fake-timers": {
+      "version": "10.3.0",
+      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz",
+      "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==",
+      "dev": true,
+      "dependencies": {
+        "@sinonjs/commons": "^3.0.0"
+      }
+    },
+    "node_modules/@stylelint/postcss-css-in-js": {
+      "version": "0.37.3",
+      "resolved": "https://registry.npmjs.org/@stylelint/postcss-css-in-js/-/postcss-css-in-js-0.37.3.tgz",
+      "integrity": "sha512-scLk3cSH1H9KggSniseb2KNAU5D9FWc3H7BxCSAIdtU9OWIyw0zkEZ9qEKHryRM+SExYXRKNb7tOOVNAsQ3iwg==",
+      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+      "dev": true,
+      "dependencies": {
+        "@babel/core": "^7.17.9"
+      },
+      "peerDependencies": {
+        "postcss": ">=7.0.0",
+        "postcss-syntax": ">=0.36.2"
+      }
+    },
+    "node_modules/@stylelint/postcss-markdown": {
+      "version": "0.36.2",
+      "resolved": "https://registry.npmjs.org/@stylelint/postcss-markdown/-/postcss-markdown-0.36.2.tgz",
+      "integrity": "sha512-2kGbqUVJUGE8dM+bMzXG/PYUWKkjLIkRLWNh39OaADkiabDRdw8ATFCgbMz5xdIcvwspPAluSL7uY+ZiTWdWmQ==",
+      "deprecated": "Use the original unforked package instead: postcss-markdown",
+      "dev": true,
+      "dependencies": {
+        "remark": "^13.0.0",
+        "unist-util-find-all-after": "^3.0.2"
+      },
+      "peerDependencies": {
+        "postcss": ">=7.0.0",
+        "postcss-syntax": ">=0.36.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ast": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ast/-/apidom-ast-0.83.0.tgz",
+      "integrity": "sha512-zAn9kHFi2JmEldYxzw6x7rbKxL4NVWvOeCWQL0AlwcWHPRhW+16/1VeHNhoWeiWm6QMERNT8z0o5frg+2czb6g==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2",
+        "unraw": "^3.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-core": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-core/-/apidom-core-0.83.0.tgz",
+      "integrity": "sha512-4pWzSbxfYrS5rH7tl4WLO5nyR7pF+aAIymwsyV2Xrec44p6d4UZaJEn1iI3r9PBBdlmOHPKgr3QiOxn71Q3XUA==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-ast": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "minim": "~0.23.8",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "short-unique-id": "^5.0.2",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-error": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-error/-/apidom-error-0.83.0.tgz",
+      "integrity": "sha512-0T3B+5Q2cApW0EkcMAqpgvsj+ab46HPvkVsYClA9/L0suRvyPiI5XDkHsw26qPGsmuB5nCH4hveZHlbWwRINMg==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7"
+      }
+    },
+    "node_modules/@swagger-api/apidom-json-pointer": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-json-pointer/-/apidom-json-pointer-0.83.0.tgz",
+      "integrity": "sha512-mT60Dfqfym9LisGcFEUV/ZwCWrcd/sI24ACAUr7D/gCMX2GuJHC7qrRwWVjGDaaDMVhDM5eCi6GKPjQhs0Ckmw==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-api-design-systems": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-api-design-systems/-/apidom-ns-api-design-systems-0.83.0.tgz",
+      "integrity": "sha512-ahkhB8QIQhos0g2WRAPb7d3HRPP4FgaPTq81Fd3IeCy1pqsRrMhBOHBt3aksOmSvCrHScXHiIU0OBsGA+vt1CA==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-1": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-asyncapi-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-asyncapi-2/-/apidom-ns-asyncapi-2-0.83.0.tgz",
+      "integrity": "sha512-A53C93GXcB9D7XSZRzEHv2k+GSa7nl7agN364sFFxS4Q/CtwNQiKVkpMCc5nG7/jUJOgj9BgevBR2p5kgYzH8Q==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-json-schema-draft-7": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-json-schema-draft-4": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-4/-/apidom-ns-json-schema-draft-4-0.83.0.tgz",
+      "integrity": "sha512-boknhIfrXF1k9IxLV0CkO1EoeXed4mzDNbFNKTkIv7UAdFwAa7NiQLVlEehNY3Ufm3/PjVMzYVQ80tUbyQE2Sw==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-ast": "^0.83.0",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-json-schema-draft-6": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-6/-/apidom-ns-json-schema-draft-6-0.83.0.tgz",
+      "integrity": "sha512-QP5MJh8hB5eK1+lZlZvUk7H02Oa+Qaq+BPNpAbmV4oG8YLUg98NxyKt+BFVhtfHWa1/i/Cpr3muiNdVIClduxw==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-ns-json-schema-draft-4": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-json-schema-draft-7": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-7/-/apidom-ns-json-schema-draft-7-0.83.0.tgz",
+      "integrity": "sha512-+91iNJQ1Oe7Hx7Q306O2JUyp7I1s0FvoZ/8FxiVYtcohGQW21CQ0j8kLv4NrQjHuHRgOquPPUXOEJGcX7s8Zsw==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-ns-json-schema-draft-6": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-openapi-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-openapi-2/-/apidom-ns-openapi-2-0.83.0.tgz",
+      "integrity": "sha512-05/IsGs1dJffvbyaxCXGA5r+tVMJpL+LOwqiKl7hGqUWOC4ku2sA0fLhxiu7fhedxq/Kbqi7ahQMihQhEP0cDQ==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-ns-json-schema-draft-4": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-openapi-3-0": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-openapi-3-0/-/apidom-ns-openapi-3-0-0.83.0.tgz",
+      "integrity": "sha512-OAN6buySWrWSvnctKVSxkG5HyUOVc8F87zHy8mxcKn91AaHPC6h8LBxIXcmXFDfZNvORZYTi7GFw3W+mnIMTwg==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-ns-json-schema-draft-4": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-ns-openapi-3-1": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-openapi-3-1/-/apidom-ns-openapi-3-1-0.83.0.tgz",
+      "integrity": "sha512-xD/T5f9Phqk4/FN5iaH8OM+5AbUqXQV92zdN5twrLCgCCA3l/1PMA7g9qEBTCG3f6UmyJ/6TTFOJyz7utye7Hg==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-ast": "^0.83.0",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-0": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-api-design-systems-json": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-api-design-systems-json/-/apidom-parser-adapter-api-design-systems-json-0.83.0.tgz",
+      "integrity": "sha512-GeMW5pamup8KeaYSbyV2/zMilslIPhQLMf9h9le9JJGJ233ugiBf/y5Vguyj1w1TQXniXztXF43B3A+RNArkmg==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-api-design-systems": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-api-design-systems-yaml": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-api-design-systems-yaml/-/apidom-parser-adapter-api-design-systems-yaml-0.83.0.tgz",
+      "integrity": "sha512-KYpW/gVfz4SQ4YPmC3x9wnUcOlwah7D4r/S2+FLvEQhf6LoEmKHL1ljcZ1Ma3seWCqMhmS1sKXHWNcYyNtY49A==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-api-design-systems": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-asyncapi-json-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-asyncapi-json-2/-/apidom-parser-adapter-asyncapi-json-2-0.83.0.tgz",
+      "integrity": "sha512-iQPDH6uIGRvJTQt6olkVUwndT91fVNrlBH3LybwHbFVLs1CKcQGJQ4lLENGw97YBVp83VO78P20Av5CiGEu80Q==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-asyncapi-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2/-/apidom-parser-adapter-asyncapi-yaml-2-0.83.0.tgz",
+      "integrity": "sha512-Q5UuatTIpYTzdCZH6ZcbT9Pw0MCLzaYzrFM6hdBWusbUriuwT12nTyt3Wer7/6nOcg+ysPTX7lUpxfUMPwT6xA==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-asyncapi-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-json": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-json/-/apidom-parser-adapter-json-0.83.0.tgz",
+      "integrity": "sha512-V6KDWP4JuLYaTpd9J8n76kiFP09trJ6PmeVERioPoZn0HpaNh7eFcIFkejFGamQADYPrF6aW6b3A2MmJjTqbMg==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-ast": "^0.83.0",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2",
+        "tree-sitter": "=0.20.4",
+        "tree-sitter-json": "=0.20.1",
+        "web-tree-sitter": "=0.20.3"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-json-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-json-2/-/apidom-parser-adapter-openapi-json-2-0.83.0.tgz",
+      "integrity": "sha512-bNrD+hpmQINU+hhzgc5VEFp04UJXRf4tKq4XpPrtVBOvZ4uJwmqLVVVNfZqes8OfLt/7ijgxNju6IwruvLeylQ==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-json-3-0": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-json-3-0/-/apidom-parser-adapter-openapi-json-3-0-0.83.0.tgz",
+      "integrity": "sha512-UbtCsg+OBbWE1vYXPeNHeLSj+79YHhDtNNPai5NFTcXgPlNhuEOKBeCqq+VBA7sos3amk0lHYUz/UFCDIcR29w==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-0": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-json-3-1": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-json-3-1/-/apidom-parser-adapter-openapi-json-3-1-0.83.0.tgz",
+      "integrity": "sha512-+O2m00jNtESw1y+KCubcte61S1SN9Nxda/KaA6yXLsZgjiYAs0HXcPEyjwGbhjHtm6NfexbOdT0poHOYbsvWfQ==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-1": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-yaml-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-yaml-2/-/apidom-parser-adapter-openapi-yaml-2-0.83.0.tgz",
+      "integrity": "sha512-YtU1wSE57yucov8A179TSB5WMJ4X5pxF5ccxW8yNxwVPH3tYkVgh5mPI8zVXQsjWLCSpyhZbiLWT5reYl5Onqw==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0/-/apidom-parser-adapter-openapi-yaml-3-0-0.83.0.tgz",
+      "integrity": "sha512-3he5fFM3GS6/WtcVldvWQgW2TFO7S2rWqYMHGASdLLm8E9pzfRw2T30ZymkDuMlC4rqH9zscbJnRFMXQV9OylQ==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-0": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1/-/apidom-parser-adapter-openapi-yaml-3-1-0.83.0.tgz",
+      "integrity": "sha512-m8SAWw8fD0QH3SR70NiDzFsJnQjzEREY5v8O8brqs5c/Rz/JtJ2WCDrLHK7eVq/Myapl/ZRJx+/xJbPZckzE0g==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-1": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.0.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-parser-adapter-yaml-1-2": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-yaml-1-2/-/apidom-parser-adapter-yaml-1-2-0.83.0.tgz",
+      "integrity": "sha512-3Pgtz88rxaiW2qg1RC8BUhusHAXe/a+FDNscfa9GHzHMEVZSmeZ13tfhzOW6a4TINmWyO7DNcKtdvlVQAPlmXQ==",
+      "optional": true,
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-ast": "^0.83.0",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2",
+        "tree-sitter": "=0.20.4",
+        "tree-sitter-yaml": "=0.5.0",
+        "web-tree-sitter": "=0.20.3"
+      }
+    },
+    "node_modules/@swagger-api/apidom-reference": {
+      "version": "0.83.0",
+      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-reference/-/apidom-reference-0.83.0.tgz",
+      "integrity": "sha512-f7Pm3fQwjf1pqniV+9abkC+oYUAbL/31GCg58r8ou4Cx+5hGTpUg81caMjdeg5Y4+Txj2ZUaAaUYyigEV25i4w==",
+      "dependencies": {
+        "@babel/runtime-corejs3": "^7.20.7",
+        "@swagger-api/apidom-core": "^0.83.0",
+        "@types/ramda": "~0.29.6",
+        "axios": "^1.4.0",
+        "minimatch": "^7.4.3",
+        "process": "^0.11.10",
+        "ramda": "~0.29.0",
+        "ramda-adjunct": "^4.1.1",
+        "stampit": "^4.3.2"
+      },
+      "optionalDependencies": {
+        "@swagger-api/apidom-error": "^0.83.0",
+        "@swagger-api/apidom-json-pointer": "^0.83.0",
+        "@swagger-api/apidom-ns-asyncapi-2": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-2": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-0": "^0.83.0",
+        "@swagger-api/apidom-ns-openapi-3-1": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-api-design-systems-json": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-api-design-systems-yaml": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-asyncapi-json-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-asyncapi-yaml-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-json": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-json-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-json-3-0": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-json-3-1": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-yaml-2": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-yaml-3-0": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-openapi-yaml-3-1": "^0.83.0",
+        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.83.0"
+      }
+    },
+    "node_modules/@swagger-api/apidom-reference/node_modules/minimatch": {
+      "version": "7.4.6",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.6.tgz",
+      "integrity": "sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@swimlane/ngx-datatable": {
+      "version": "18.0.0",
+      "resolved": "https://registry.npmjs.org/@swimlane/ngx-datatable/-/ngx-datatable-18.0.0.tgz",
+      "integrity": "sha512-secqjzlLpGJqoXjcoCoTf8ClnVlZAENJcXvuBfseGenOD+evGNXc4UTZhwCPDUBlJ4xnMZHUWK6IVk5sXe+WlQ==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "@angular/common": "^10.0.0",
+        "@angular/core": "^10.0.0",
+        "@angular/platform-browser": "^10.0.0",
+        "rxjs": "^6.5.5"
+      }
+    },
+    "node_modules/@tootallnate/once": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
+      "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
+      "dev": true,
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@ts-morph/common": {
+      "version": "0.12.3",
+      "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.12.3.tgz",
+      "integrity": "sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==",
+      "dev": true,
+      "dependencies": {
+        "fast-glob": "^3.2.7",
+        "minimatch": "^3.0.4",
+        "mkdirp": "^1.0.4",
+        "path-browserify": "^1.0.1"
+      }
+    },
+    "node_modules/@ts-morph/common/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/@ts-morph/common/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/@tufjs/canonical-json": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-1.0.0.tgz",
+      "integrity": "sha512-QTnf++uxunWvG2z3UFNzAoQPHxnSXOwtaI3iJ+AohhV+5vONuArPjJE7aPXPVXfXJsqrVbZBu9b81AJoSd09IQ==",
+      "dev": true,
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@tufjs/models": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@tufjs/models/-/models-1.0.4.tgz",
+      "integrity": "sha512-qaGV9ltJP0EO25YfFUPhxRVK0evXFIAGicsVXuRim4Ed9cjPxYhNnNJ49SFmbeLgtxpslIkX317IgpfcHPVj/A==",
+      "dev": true,
+      "dependencies": {
+        "@tufjs/canonical-json": "1.0.0",
+        "minimatch": "^9.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/@tufjs/models/node_modules/minimatch": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
+      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@types/babel__core": {
+      "version": "7.20.4",
+      "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.4.tgz",
+      "integrity": "sha512-mLnSC22IC4vcWiuObSRjrLd9XcBTGf59vUSoq2jkQDJ/QQ8PMI9rSuzE+aEV8karUMbskw07bKYoUJCKTUaygg==",
+      "dev": true,
+      "dependencies": {
+        "@babel/parser": "^7.20.7",
+        "@babel/types": "^7.20.7",
+        "@types/babel__generator": "*",
+        "@types/babel__template": "*",
+        "@types/babel__traverse": "*"
+      }
+    },
+    "node_modules/@types/babel__generator": {
+      "version": "7.6.7",
+      "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.7.tgz",
+      "integrity": "sha512-6Sfsq+EaaLrw4RmdFWE9Onp63TOUue71AWb4Gpa6JxzgTYtimbM086WnYTy2U67AofR++QKCo08ZP6pwx8YFHQ==",
+      "dev": true,
+      "dependencies": {
+        "@babel/types": "^7.0.0"
+      }
+    },
+    "node_modules/@types/babel__template": {
+      "version": "7.4.4",
+      "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz",
+      "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==",
+      "dev": true,
+      "dependencies": {
+        "@babel/parser": "^7.1.0",
+        "@babel/types": "^7.0.0"
+      }
+    },
+    "node_modules/@types/babel__traverse": {
+      "version": "7.20.4",
+      "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.4.tgz",
+      "integrity": "sha512-mSM/iKUk5fDDrEV/e83qY+Cr3I1+Q3qqTuEn++HAWYjEa1+NxZr6CNrcJGf2ZTnq4HoFGC3zaTPZTobCzCFukA==",
+      "dev": true,
+      "dependencies": {
+        "@babel/types": "^7.20.7"
+      }
+    },
+    "node_modules/@types/body-parser": {
+      "version": "1.19.5",
+      "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz",
+      "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==",
+      "dev": true,
+      "dependencies": {
+        "@types/connect": "*",
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/bonjour": {
+      "version": "3.5.13",
+      "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.13.tgz",
+      "integrity": "sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/brace-expansion": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@types/brace-expansion/-/brace-expansion-1.1.0.tgz",
+      "integrity": "sha512-SaU/Kgp6z40CiF9JxlsrSrBEa+8YIry9IiCPhhYSNekeEhIAkY7iyu9aZ+5dSQIdo7mf86MUVvxWYm5GAzB/0g==",
+      "dev": true
+    },
+    "node_modules/@types/connect": {
+      "version": "3.4.38",
+      "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
+      "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/connect-history-api-fallback": {
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.3.tgz",
+      "integrity": "sha512-6mfQ6iNvhSKCZJoY6sIG3m0pKkdUcweVNOLuBBKvoWGzl2yRxOJcYOTRyLKt3nxXvBLJWa6QkW//tgbIwJehmA==",
+      "dev": true,
+      "dependencies": {
+        "@types/express-serve-static-core": "*",
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/cypress-cucumber-preprocessor": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-4.0.1.tgz",
+      "integrity": "sha512-sK2/uU5CtmJ51zo0JF2Lc4iSw9Fy3xn9ewfewuooV5Qmeb5O+brAHuoXKMV7UWwRbBmd+txhAXAJoi4S5QLDRQ==",
+      "dev": true
+    },
+    "node_modules/@types/eslint": {
+      "version": "8.44.7",
+      "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.7.tgz",
+      "integrity": "sha512-f5ORu2hcBbKei97U73mf+l9t4zTGl74IqZ0GQk4oVea/VS8tQZYkUveSYojk+frraAVYId0V2WC9O4PTNru2FQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/estree": "*",
+        "@types/json-schema": "*"
+      }
+    },
+    "node_modules/@types/eslint-scope": {
+      "version": "3.7.7",
+      "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz",
+      "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==",
+      "dev": true,
+      "dependencies": {
+        "@types/eslint": "*",
+        "@types/estree": "*"
+      }
+    },
+    "node_modules/@types/estree": {
+      "version": "0.0.51",
+      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz",
+      "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==",
+      "dev": true
+    },
+    "node_modules/@types/express": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz",
+      "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/body-parser": "*",
+        "@types/express-serve-static-core": "^4.17.33",
+        "@types/qs": "*",
+        "@types/serve-static": "*"
+      }
+    },
+    "node_modules/@types/express-serve-static-core": {
+      "version": "4.17.41",
+      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.41.tgz",
+      "integrity": "sha512-OaJ7XLaelTgrvlZD8/aa0vvvxZdUmlCn6MtWeB7TkiKW70BQLc9XEPpDLPdbo52ZhXUCrznlWdCHWxJWtdyajA==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "@types/qs": "*",
+        "@types/range-parser": "*",
+        "@types/send": "*"
+      }
+    },
+    "node_modules/@types/file-saver": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@types/file-saver/-/file-saver-2.0.1.tgz",
+      "integrity": "sha512-g1QUuhYVVAamfCifK7oB7G3aIl4BbOyzDOqVyUfEr4tfBKrXfeH+M+Tg7HKCXSrbzxYdhyCP7z9WbKo0R2hBCw=="
+    },
+    "node_modules/@types/graceful-fs": {
+      "version": "4.1.9",
+      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz",
+      "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/hast": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.8.tgz",
+      "integrity": "sha512-aMIqAlFd2wTIDZuvLbhUT+TGvMxrNC8ECUIVtH6xxy0sQLs3iu6NO8Kp/VT5je7i5ufnebXzdV1dNDMnvaH6IQ==",
+      "dependencies": {
+        "@types/unist": "^2"
+      }
+    },
+    "node_modules/@types/hoist-non-react-statics": {
+      "version": "3.3.5",
+      "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.5.tgz",
+      "integrity": "sha512-SbcrWzkKBw2cdwRTwQAswfpB9g9LJWfjtUeW/jvNwbhC8cpmmNYVePa+ncbUe0rGTQ7G3Ff6mYUN2VMfLVr+Sg==",
+      "dependencies": {
+        "@types/react": "*",
+        "hoist-non-react-statics": "^3.3.0"
+      }
+    },
+    "node_modules/@types/http-errors": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz",
+      "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==",
+      "dev": true
+    },
+    "node_modules/@types/http-proxy": {
+      "version": "1.17.14",
+      "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.14.tgz",
+      "integrity": "sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/istanbul-lib-coverage": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz",
+      "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==",
+      "dev": true
+    },
+    "node_modules/@types/istanbul-lib-report": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz",
+      "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==",
+      "dev": true,
+      "dependencies": {
+        "@types/istanbul-lib-coverage": "*"
+      }
+    },
+    "node_modules/@types/istanbul-reports": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz",
+      "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/istanbul-lib-report": "*"
+      }
+    },
+    "node_modules/@types/jest": {
+      "version": "29.5.4",
+      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.4.tgz",
+      "integrity": "sha512-PhglGmhWeD46FYOVLt3X7TiWjzwuVGW9wG/4qocPevXMjCmrIc5b6db9WjeGE4QYVpUAWMDv3v0IiBwObY289A==",
+      "dev": true,
+      "dependencies": {
+        "expect": "^29.0.0",
+        "pretty-format": "^29.0.0"
+      }
+    },
+    "node_modules/@types/jsdom": {
+      "version": "20.0.1",
+      "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz",
+      "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "@types/tough-cookie": "*",
+        "parse5": "^7.0.0"
+      }
+    },
+    "node_modules/@types/jsdom/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
+    "node_modules/@types/jsdom/node_modules/parse5": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
+      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
+      "dev": true,
+      "dependencies": {
+        "entities": "^4.4.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      }
+    },
+    "node_modules/@types/json-schema": {
+      "version": "7.0.15",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
+      "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
+      "dev": true
+    },
+    "node_modules/@types/lodash": {
+      "version": "4.14.161",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.161.tgz",
+      "integrity": "sha512-EP6O3Jkr7bXvZZSZYlsgt5DIjiGr0dXP1/jVEwVLTFgg0d+3lWVQkRavYVQszV7dYUwvg0B8R0MBDpcmXg7XIA==",
+      "dev": true
+    },
+    "node_modules/@types/long": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
+      "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",
+      "dev": true
+    },
+    "node_modules/@types/mdast": {
+      "version": "3.0.15",
+      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.15.tgz",
+      "integrity": "sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/unist": "^2"
+      }
+    },
+    "node_modules/@types/mime": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
+      "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==",
+      "dev": true
+    },
+    "node_modules/@types/minimatch": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz",
+      "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==",
+      "dev": true
+    },
+    "node_modules/@types/minimist": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
+      "dev": true
+    },
+    "node_modules/@types/node": {
+      "version": "18.17.12",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.12.tgz",
+      "integrity": "sha512-d6xjC9fJ/nSnfDeU0AMDsaJyb1iHsqCSOdi84w4u+SlN/UgQdY5tRhpMzaFYsI4mnpvgTivEaQd0yOUhAtOnEQ==",
+      "dev": true
+    },
+    "node_modules/@types/node-forge": {
+      "version": "1.3.9",
+      "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.9.tgz",
+      "integrity": "sha512-meK88cx/sTalPSLSoCzkiUB4VPIFHmxtXm5FaaqRDqBX2i/Sy8bJ4odsan0b20RBjPh06dAQ+OTTdnyQyhJZyQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/normalize-package-data": {
+      "version": "2.4.4",
+      "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz",
+      "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==",
+      "dev": true
+    },
+    "node_modules/@types/parse-json": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
+      "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==",
+      "dev": true
+    },
+    "node_modules/@types/prettier": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.3.tgz",
+      "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==",
+      "dev": true
+    },
+    "node_modules/@types/prop-types": {
+      "version": "15.7.10",
+      "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.10.tgz",
+      "integrity": "sha512-mxSnDQxPqsZxmeShFH+uwQ4kO4gcJcGahjjMFeLbKE95IAZiiZyiEepGZjtXJ7hN/yfu0bu9xN2ajcU0JcxX6A=="
+    },
+    "node_modules/@types/qs": {
+      "version": "6.9.10",
+      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.10.tgz",
+      "integrity": "sha512-3Gnx08Ns1sEoCrWssEgTSJs/rsT2vhGP+Ja9cnnk9k4ALxinORlQneLXFeFKOTJMOeZUFD1s7w+w2AphTpvzZw==",
+      "dev": true
+    },
+    "node_modules/@types/ramda": {
+      "version": "0.29.9",
+      "resolved": "https://registry.npmjs.org/@types/ramda/-/ramda-0.29.9.tgz",
+      "integrity": "sha512-X3yEG6tQCWBcUAql+RPC/O1Hm9BSU+MXu2wJnCETuAgUlrEDwTA1kIOdEEE4YXDtf0zfQLHa9CCE7WYp9kqPIQ==",
+      "dependencies": {
+        "types-ramda": "^0.29.6"
+      }
+    },
+    "node_modules/@types/range-parser": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
+      "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
+      "dev": true
+    },
+    "node_modules/@types/react": {
+      "version": "18.2.37",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.37.tgz",
+      "integrity": "sha512-RGAYMi2bhRgEXT3f4B92WTohopH6bIXw05FuGlmJEnv/omEn190+QYEIYxIAuIBdKgboYYdVved2p1AxZVQnaw==",
+      "dependencies": {
+        "@types/prop-types": "*",
+        "@types/scheduler": "*",
+        "csstype": "^3.0.2"
+      }
+    },
+    "node_modules/@types/react-redux": {
+      "version": "7.1.30",
+      "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.30.tgz",
+      "integrity": "sha512-i2kqM6YaUwFKduamV6QM/uHbb0eCP8f8ZQ/0yWf+BsAVVsZPRYJ9eeGWZ3uxLfWwwA0SrPRMTPTqsPFkY3HZdA==",
+      "dependencies": {
+        "@types/hoist-non-react-statics": "^3.3.0",
+        "@types/react": "*",
+        "hoist-non-react-statics": "^3.3.0",
+        "redux": "^4.0.0"
+      }
+    },
+    "node_modules/@types/retry": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
+      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
+      "dev": true
+    },
+    "node_modules/@types/scheduler": {
+      "version": "0.16.6",
+      "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.6.tgz",
+      "integrity": "sha512-Vlktnchmkylvc9SnwwwozTv04L/e1NykF5vgoQ0XTmI8DD+wxfjQuHuvHS3p0r2jz2x2ghPs2h1FVeDirIteWA=="
+    },
+    "node_modules/@types/send": {
+      "version": "0.17.4",
+      "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
+      "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==",
+      "dev": true,
+      "dependencies": {
+        "@types/mime": "^1",
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/serve-index": {
+      "version": "1.9.4",
+      "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.4.tgz",
+      "integrity": "sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*"
+      }
+    },
+    "node_modules/@types/serve-static": {
+      "version": "1.15.5",
+      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.5.tgz",
+      "integrity": "sha512-PDRk21MnK70hja/YF8AHfC7yIsiQHn1rcXx7ijCFBX/k+XQJhQT/gw3xekXKJvx+5SXaMMS8oqQy09Mzvz2TuQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/http-errors": "*",
+        "@types/mime": "*",
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/sinonjs__fake-timers": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz",
+      "integrity": "sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==",
+      "dev": true
+    },
+    "node_modules/@types/sizzle": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.6.tgz",
+      "integrity": "sha512-m04Om5Gz6kbjUwAQ7XJJQ30OdEFsSmAVsvn4NYwcTRyMVpKKa1aPuESw1n2CxS5fYkOQv3nHgDKeNa8e76fUkw==",
+      "dev": true
+    },
+    "node_modules/@types/sockjs": {
+      "version": "0.3.36",
+      "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz",
+      "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/stack-utils": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz",
+      "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==",
       "dev": true
     },
-    "node_modules/@nrwl/cli/node_modules/semver": {
-      "version": "7.3.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+    "node_modules/@types/swagger-ui": {
+      "version": "3.52.0",
+      "resolved": "https://registry.npmjs.org/@types/swagger-ui/-/swagger-ui-3.52.0.tgz",
+      "integrity": "sha512-SlufixEmh+8CLHNgTfAfCT1icNOF7bXboWabhHr1+hIolqlvfwYJGe7HgRcpI3ChE7HWASmEKLkMu34rxseJjQ==",
+      "dev": true
+    },
+    "node_modules/@types/tough-cookie": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+      "dev": true
+    },
+    "node_modules/@types/unist": {
+      "version": "2.0.10",
+      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz",
+      "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA=="
+    },
+    "node_modules/@types/uuid": {
+      "version": "3.4.13",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.13.tgz",
+      "integrity": "sha512-pAeZeUbLE4Z9Vi9wsWV2bYPTweEHeJJy0G4pEjOA/FSvy1Ad5U5Km8iDV6TKre1mjBiVNfAdVHKruP8bAh4Q5A==",
+      "dev": true
+    },
+    "node_modules/@types/ws": {
+      "version": "8.5.9",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.9.tgz",
+      "integrity": "sha512-jbdrY0a8lxfdTp/+r7Z4CkycbOFN8WX+IOchLJr3juT/xzbJ8URyTVSJ/hvNdadTgM1mnedb47n+Y31GsFnQlg==",
       "dev": true,
       "dependencies": {
-        "lru-cache": "^6.0.0"
-      },
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
+        "@types/node": "*"
       }
     },
-    "node_modules/@nrwl/cli/node_modules/v8-compile-cache": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
-      "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
-      "dev": true
+    "node_modules/@types/xml2js": {
+      "version": "0.4.14",
+      "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.14.tgz",
+      "integrity": "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
     },
-    "node_modules/@nrwl/cli/node_modules/yallist": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+    "node_modules/@types/yargs": {
+      "version": "17.0.31",
+      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.31.tgz",
+      "integrity": "sha512-bocYSx4DI8TmdlvxqGpVNXOgCNR1Jj0gNPhhAY+iz1rgKDAaYrAYdFYnhDV1IFuiuVc9HkOwyDcFxaTElF3/wg==",
+      "dev": true,
+      "dependencies": {
+        "@types/yargs-parser": "*"
+      }
+    },
+    "node_modules/@types/yargs-parser": {
+      "version": "21.0.3",
+      "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz",
+      "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==",
       "dev": true
     },
-    "node_modules/@nrwl/cli/node_modules/yargs-parser": {
-      "version": "21.1.1",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
-      "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
+    "node_modules/@types/yauzl": {
+      "version": "2.10.3",
+      "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz",
+      "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==",
       "dev": true,
-      "engines": {
-        "node": ">=12"
+      "optional": true,
+      "dependencies": {
+        "@types/node": "*"
       }
     },
-    "node_modules/@nrwl/devkit": {
-      "version": "13.1.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/devkit/-/devkit-13.1.3.tgz",
-      "integrity": "sha512-TAAsZJvVc/obeH0rZKY6miVhyM2GHGb8qIWp9MAIdLlXf4VDcNC7rxwb5OrGVSwuTTjqGYBGPUx0yEogOOJthA==",
+    "node_modules/@typescript-eslint/eslint-plugin": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.27.1.tgz",
+      "integrity": "sha512-6dM5NKT57ZduNnJfpY81Phe9nc9wolnMCnknb1im6brWi1RYv84nbMS3olJa27B6+irUVV1X/Wb+Am0FjJdGFw==",
       "dev": true,
       "dependencies": {
-        "@nrwl/tao": "13.1.3",
-        "ejs": "^3.1.5",
-        "ignore": "^5.0.4",
-        "rxjs": "^6.5.4",
-        "semver": "7.3.4",
-        "tslib": "^2.0.0"
+        "@typescript-eslint/scope-manager": "5.27.1",
+        "@typescript-eslint/type-utils": "5.27.1",
+        "@typescript-eslint/utils": "5.27.1",
+        "debug": "^4.3.4",
+        "functional-red-black-tree": "^1.0.1",
+        "ignore": "^5.2.0",
+        "regexpp": "^3.2.0",
+        "semver": "^7.3.7",
+        "tsutils": "^3.21.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "@typescript-eslint/parser": "^5.0.0",
+        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@nrwl/devkit/node_modules/lru-cache": {
+    "node_modules/@typescript-eslint/eslint-plugin/node_modules/lru-cache": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
       "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
@@ -5512,10 +8550,10 @@
         "node": ">=10"
       }
     },
-    "node_modules/@nrwl/devkit/node_modules/semver": {
-      "version": "7.3.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+    "node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -5527,2319 +8565,2277 @@
         "node": ">=10"
       }
     },
-    "node_modules/@nrwl/devkit/node_modules/yallist": {
+    "node_modules/@typescript-eslint/eslint-plugin/node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "dev": true
     },
-    "node_modules/@nrwl/nx-darwin-arm64": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-darwin-arm64/-/nx-darwin-arm64-15.9.3.tgz",
-      "integrity": "sha512-2htJzVa+S/uLg5tj4nbO/tRz2SRMQIpT6EeWMgDGuEKQdpuRLVj2ez9hMpkRn9tl1tBUwR05hbV28DnOLRESVA==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@typescript-eslint/experimental-utils": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-5.27.1.tgz",
+      "integrity": "sha512-Vd8uewIixGP93sEnmTRIH6jHZYRQRkGPDPpapACMvitJKX8335VHNyqKTE+mZ+m3E2c5VznTZfSsSsS5IF7vUA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
+      "dependencies": {
+        "@typescript-eslint/utils": "5.27.1"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
       }
     },
-    "node_modules/@nrwl/nx-darwin-x64": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-darwin-x64/-/nx-darwin-x64-15.9.3.tgz",
-      "integrity": "sha512-p+8UkfC6KTLOX4XRt7NSP8DoTzEgs73+SN0csoXT9VsNO35+F0Z5zMZxpEc7RVo5Wen/4PGh2OWA+8gtgntsJQ==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@typescript-eslint/parser": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.27.1.tgz",
+      "integrity": "sha512-7Va2ZOkHi5NP+AZwb5ReLgNF6nWLGTeUJfxdkVUAPPSaAdbWNnFZzLZ4EGGmmiCTg+AwlbE1KyUYTBglosSLHQ==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
+      "dependencies": {
+        "@typescript-eslint/scope-manager": "5.27.1",
+        "@typescript-eslint/types": "5.27.1",
+        "@typescript-eslint/typescript-estree": "5.27.1",
+        "debug": "^4.3.4"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@nrwl/nx-linux-arm-gnueabihf": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm-gnueabihf/-/nx-linux-arm-gnueabihf-15.9.3.tgz",
-      "integrity": "sha512-xwW7bZtggrxhFbYvvWWArtcSWwoxWzi/4wNgP3wPbcZFNZiraahVQSpIyJXrS9aajGbdvuDBM8cbDsMj9v7mwg==",
-      "cpu": [
-        "arm"
-      ],
+    "node_modules/@typescript-eslint/scope-manager": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.27.1.tgz",
+      "integrity": "sha512-fQEOSa/QroWE6fAEg+bJxtRZJTH8NTskggybogHt4H9Da8zd4cJji76gA5SBlR0MgtwF7rebxTbDKB49YUCpAg==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "@typescript-eslint/types": "5.27.1",
+        "@typescript-eslint/visitor-keys": "5.27.1"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
       }
     },
-    "node_modules/@nrwl/nx-linux-arm64-gnu": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-15.9.3.tgz",
-      "integrity": "sha512-KNxDL2OAHxhFqztEjv2mNwXD6xrzoUury7NsYZYqlxJUNc3YYBfRSLEatnw491crvMBndbxfGVTWEO9S4YmRuw==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@typescript-eslint/type-utils": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.27.1.tgz",
+      "integrity": "sha512-+UC1vVUWaDHRnC2cQrCJ4QtVjpjjCgjNFpg8b03nERmkHv9JV9X5M19D7UFMd+/G7T/sgFwX2pGmWK38rqyvXw==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "@typescript-eslint/utils": "5.27.1",
+        "debug": "^4.3.4",
+        "tsutils": "^3.21.0"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "eslint": "*"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@nrwl/nx-linux-arm64-musl": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-arm64-musl/-/nx-linux-arm64-musl-15.9.3.tgz",
-      "integrity": "sha512-AxoZzfsXH7ZqDE+WrQtRumufIcSIBw4U/LikiDLaWWoGtNpAfKLkD/PHirZiNxHIeGy1Toi4ccMUolXbafLVFw==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@typescript-eslint/types": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.27.1.tgz",
+      "integrity": "sha512-LgogNVkBhCTZU/m8XgEYIWICD6m4dmEDbKXESCbqOXfKZxRKeqpiJXQIErv66sdopRKZPo5l32ymNqibYEH/xg==",
+      "dev": true,
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      }
+    },
+    "node_modules/@typescript-eslint/typescript-estree": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.27.1.tgz",
+      "integrity": "sha512-DnZvvq3TAJ5ke+hk0LklvxwYsnXpRdqUY5gaVS0D4raKtbznPz71UJGnPTHEFo0GDxqLOLdMkkmVZjSpET1hFw==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "@typescript-eslint/types": "5.27.1",
+        "@typescript-eslint/visitor-keys": "5.27.1",
+        "debug": "^4.3.4",
+        "globby": "^11.1.0",
+        "is-glob": "^4.0.3",
+        "semver": "^7.3.7",
+        "tsutils": "^3.21.0"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@nrwl/nx-linux-x64-gnu": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-x64-gnu/-/nx-linux-x64-gnu-15.9.3.tgz",
-      "integrity": "sha512-P8AOPRufvV4a5cSczNsw84zFAI7NgAiEBTybYcyymdNJmo0iArJXEmvj/G4mB20O8VCsCkwqMYAu6nQEnES1Kw==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/globby": {
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz",
+      "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "array-union": "^2.1.0",
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.2.9",
+        "ignore": "^5.2.0",
+        "merge2": "^1.4.1",
+        "slash": "^3.0.0"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@nrwl/nx-linux-x64-musl": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-linux-x64-musl/-/nx-linux-x64-musl-15.9.3.tgz",
-      "integrity": "sha512-4ZYDp7T319+xbw7Z7KVtRefzaXJipZfgrM49r+Y1FAfYDc8y18zvKz3slK26wfWz+EUZwKsa/DfA2KmyRG3DvQ==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "linux"
-      ],
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": ">=10"
       }
     },
-    "node_modules/@nrwl/nx-win32-arm64-msvc": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-15.9.3.tgz",
-      "integrity": "sha512-UhgxIPgTZBKN1oxlLPSklkSzVL3hA4lAiVc9A0Utumpbp0ob/Xx+2vHzg3cnmNH3jWkZ+9OsC2dKyeMB6gAbSw==",
-      "cpu": [
-        "arm64"
-      ],
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ],
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": ">=10"
       }
     },
-    "node_modules/@nrwl/nx-win32-x64-msvc": {
-      "version": "15.9.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/nx-win32-x64-msvc/-/nx-win32-x64-msvc-15.9.3.tgz",
-      "integrity": "sha512-gdnvqURKnu0EQGOFJ6NUKq6wSB+viNb7Z8qtKhzSmFwVjT8akOnLWn7ZhL9v28TAjLM7/s1Mwvmz/IMj1PGlcQ==",
-      "cpu": [
-        "x64"
-      ],
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
-      "optional": true,
-      "os": [
-        "win32"
-      ],
       "engines": {
-        "node": ">= 10"
+        "node": ">=8"
       }
     },
-    "node_modules/@nrwl/tao": {
-      "version": "13.1.3",
-      "resolved": "https://registry.npmjs.org/@nrwl/tao/-/tao-13.1.3.tgz",
-      "integrity": "sha512-/IwJgSgCBD1SaF+n8RuXX2OxDAh8ut/+P8pMswjm8063ac30UlAHjQ4XTYyskLH8uoUmNi2hNaGgHUrkwt7tQA==",
+    "node_modules/@typescript-eslint/typescript-estree/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/@typescript-eslint/utils": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.27.1.tgz",
+      "integrity": "sha512-mZ9WEn1ZLDaVrhRaYgzbkXBkTPghPFsup8zDbbsYTxC5OmqrFE7skkKS/sraVsLP3TcT3Ki5CSyEFBRkLH/H/w==",
       "dev": true,
       "dependencies": {
-        "chalk": "4.1.0",
-        "enquirer": "~2.3.6",
-        "fs-extra": "^9.1.0",
-        "jsonc-parser": "3.0.0",
-        "nx": "13.1.3",
-        "rxjs": "^6.5.4",
-        "rxjs-for-await": "0.0.2",
-        "semver": "7.3.4",
-        "tmp": "~0.2.1",
-        "tslib": "^2.0.0",
-        "yargs-parser": "20.0.0"
+        "@types/json-schema": "^7.0.9",
+        "@typescript-eslint/scope-manager": "5.27.1",
+        "@typescript-eslint/types": "5.27.1",
+        "@typescript-eslint/typescript-estree": "5.27.1",
+        "eslint-scope": "^5.1.1",
+        "eslint-utils": "^3.0.0"
       },
-      "bin": {
-        "tao": "index.js"
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
       }
     },
-    "node_modules/@nrwl/tao/node_modules/chalk": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
-      "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+    "node_modules/@typescript-eslint/visitor-keys": {
+      "version": "5.27.1",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.27.1.tgz",
+      "integrity": "sha512-xYs6ffo01nhdJgPieyk7HAOpjhTsx7r/oB9LWEhwAXgwn33tkr+W8DI2ChboqhZlC4q3TC6geDYPoiX8ROqyOQ==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "@typescript-eslint/types": "5.27.1",
+        "eslint-visitor-keys": "^3.3.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
       }
     },
-    "node_modules/@nrwl/tao/node_modules/fs-extra": {
-      "version": "9.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
-      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
+    "node_modules/@webassemblyjs/ast": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz",
+      "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==",
       "dev": true,
       "dependencies": {
-        "at-least-node": "^1.0.0",
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
+        "@webassemblyjs/helper-numbers": "1.11.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.1"
       }
     },
-    "node_modules/@nrwl/tao/node_modules/jsonc-parser": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz",
-      "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==",
+    "node_modules/@webassemblyjs/floating-point-hex-parser": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz",
+      "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==",
       "dev": true
     },
-    "node_modules/@nrwl/tao/node_modules/lru-cache": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+    "node_modules/@webassemblyjs/helper-api-error": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz",
+      "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==",
+      "dev": true
+    },
+    "node_modules/@webassemblyjs/helper-buffer": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz",
+      "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==",
+      "dev": true
+    },
+    "node_modules/@webassemblyjs/helper-numbers": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz",
+      "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==",
       "dev": true,
       "dependencies": {
-        "yallist": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=10"
+        "@webassemblyjs/floating-point-hex-parser": "1.11.1",
+        "@webassemblyjs/helper-api-error": "1.11.1",
+        "@xtuc/long": "4.2.2"
       }
     },
-    "node_modules/@nrwl/tao/node_modules/semver": {
-      "version": "7.3.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+    "node_modules/@webassemblyjs/helper-wasm-bytecode": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz",
+      "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==",
+      "dev": true
+    },
+    "node_modules/@webassemblyjs/helper-wasm-section": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz",
+      "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==",
+      "dev": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.11.1",
+        "@webassemblyjs/helper-buffer": "1.11.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
+        "@webassemblyjs/wasm-gen": "1.11.1"
+      }
+    },
+    "node_modules/@webassemblyjs/ieee754": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz",
+      "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==",
+      "dev": true,
+      "dependencies": {
+        "@xtuc/ieee754": "^1.2.0"
+      }
+    },
+    "node_modules/@webassemblyjs/leb128": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz",
+      "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==",
       "dev": true,
       "dependencies": {
-        "lru-cache": "^6.0.0"
-      },
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
+        "@xtuc/long": "4.2.2"
       }
     },
-    "node_modules/@nrwl/tao/node_modules/yallist": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+    "node_modules/@webassemblyjs/utf8": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz",
+      "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==",
       "dev": true
     },
-    "node_modules/@parcel/watcher": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.0.4.tgz",
-      "integrity": "sha512-cTDi+FUDBIUOBKEtj+nhiJ71AZVlkAsQFuGQTun5tV9mwQBQgZvhCzG+URPQc8myeN32yRVZEfVAPCs1RW+Jvg==",
+    "node_modules/@webassemblyjs/wasm-edit": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz",
+      "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "node-addon-api": "^3.2.1",
-        "node-gyp-build": "^4.3.0"
-      },
-      "engines": {
-        "node": ">= 10.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/parcel"
+        "@webassemblyjs/ast": "1.11.1",
+        "@webassemblyjs/helper-buffer": "1.11.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
+        "@webassemblyjs/helper-wasm-section": "1.11.1",
+        "@webassemblyjs/wasm-gen": "1.11.1",
+        "@webassemblyjs/wasm-opt": "1.11.1",
+        "@webassemblyjs/wasm-parser": "1.11.1",
+        "@webassemblyjs/wast-printer": "1.11.1"
       }
     },
-    "node_modules/@pkgjs/parseargs": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+    "node_modules/@webassemblyjs/wasm-gen": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz",
+      "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==",
       "dev": true,
-      "optional": true,
-      "engines": {
-        "node": ">=14"
+      "dependencies": {
+        "@webassemblyjs/ast": "1.11.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
+        "@webassemblyjs/ieee754": "1.11.1",
+        "@webassemblyjs/leb128": "1.11.1",
+        "@webassemblyjs/utf8": "1.11.1"
       }
     },
-    "node_modules/@popperjs/core": {
-      "version": "2.10.2",
-      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.10.2.tgz",
-      "integrity": "sha512-IXf3XA7+XyN7CP9gGh/XB0UxVMlvARGEgGXLubFICsUMGz6Q+DU+i4gGlpOxTjKvXjkJDJC8YdqdKkDj9qZHEQ==",
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/popperjs"
+    "node_modules/@webassemblyjs/wasm-opt": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
+      "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
+      "dev": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.11.1",
+        "@webassemblyjs/helper-buffer": "1.11.1",
+        "@webassemblyjs/wasm-gen": "1.11.1",
+        "@webassemblyjs/wasm-parser": "1.11.1"
       }
     },
-    "node_modules/@protobufjs/aspromise": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
-      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
-      "dev": true
-    },
-    "node_modules/@protobufjs/base64": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
-      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
-      "dev": true
-    },
-    "node_modules/@protobufjs/codegen": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
-      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
-      "dev": true
-    },
-    "node_modules/@protobufjs/eventemitter": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
-      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
-      "dev": true
-    },
-    "node_modules/@protobufjs/fetch": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
-      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+    "node_modules/@webassemblyjs/wasm-parser": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz",
+      "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==",
       "dev": true,
       "dependencies": {
-        "@protobufjs/aspromise": "^1.1.1",
-        "@protobufjs/inquire": "^1.1.0"
+        "@webassemblyjs/ast": "1.11.1",
+        "@webassemblyjs/helper-api-error": "1.11.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
+        "@webassemblyjs/ieee754": "1.11.1",
+        "@webassemblyjs/leb128": "1.11.1",
+        "@webassemblyjs/utf8": "1.11.1"
       }
     },
-    "node_modules/@protobufjs/float": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
-      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
-      "dev": true
-    },
-    "node_modules/@protobufjs/inquire": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
-      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
-      "dev": true
+    "node_modules/@webassemblyjs/wast-printer": {
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz",
+      "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==",
+      "dev": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.11.1",
+        "@xtuc/long": "4.2.2"
+      }
     },
-    "node_modules/@protobufjs/path": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
-      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+    "node_modules/@xtuc/ieee754": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
+      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
       "dev": true
     },
-    "node_modules/@protobufjs/pool": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
-      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+    "node_modules/@xtuc/long": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz",
+      "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==",
       "dev": true
     },
-    "node_modules/@protobufjs/utf8": {
+    "node_modules/@yarnpkg/lockfile": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
-      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz",
+      "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==",
       "dev": true
     },
-    "node_modules/@schematics/angular": {
-      "version": "15.2.9",
-      "resolved": "https://registry.npmjs.org/@schematics/angular/-/angular-15.2.9.tgz",
-      "integrity": "sha512-0Lit6TLNUwcAYiEkXgZp3vY9xAO1cnZCBXuUcp+6v+Ddnrt2w/YOiGe74p21cYe0StkTpTljsqsKBTiX7TMjQg==",
+    "node_modules/@yarnpkg/parsers": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@yarnpkg/parsers/-/parsers-3.0.0.tgz",
+      "integrity": "sha512-jVZa3njBv6tcOUw34nlUdUM/40wwtm/gnVF8rtk0tA6vNcokqYI8CFU1BZjlpFwUSZaXxYkrtuPE/f2MMFlTxQ==",
       "dev": true,
       "dependencies": {
-        "@angular-devkit/core": "15.2.9",
-        "@angular-devkit/schematics": "15.2.9",
-        "jsonc-parser": "3.2.0"
+        "js-yaml": "^3.10.0",
+        "tslib": "^2.4.0"
       },
       "engines": {
-        "node": "^14.20.0 || ^16.13.0 || >=18.10.0",
-        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
-        "yarn": ">= 1.13.0"
+        "node": ">=18.12.0"
       }
     },
-    "node_modules/@sideway/address": {
-      "version": "4.1.4",
-      "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz",
-      "integrity": "sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==",
+    "node_modules/@yarnpkg/parsers/node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "dev": true,
       "dependencies": {
-        "@hapi/hoek": "^9.0.0"
+        "sprintf-js": "~1.0.2"
       }
     },
-    "node_modules/@sideway/formula": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz",
-      "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==",
+    "node_modules/@yarnpkg/parsers/node_modules/js-yaml": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/@yarnpkg/parsers/node_modules/tslib": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==",
       "dev": true
     },
-    "node_modules/@sideway/pinpoint": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz",
-      "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==",
+    "node_modules/abab": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
+      "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
+      "deprecated": "Use your platform's native atob() and btoa() methods instead",
       "dev": true
     },
-    "node_modules/@sigstore/bundle": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-1.1.0.tgz",
-      "integrity": "sha512-PFutXEy0SmQxYI4texPw3dd2KewuNqv7OuK1ZFtY2fM754yhvG2KdgwIhRnoEE2uHdtdGNQ8s0lb94dW9sELog==",
+    "node_modules/abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
+    "node_modules/abort-controller": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
       "dev": true,
       "dependencies": {
-        "@sigstore/protobuf-specs": "^0.2.0"
+        "event-target-shim": "^5.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
-      }
-    },
-    "node_modules/@sigstore/protobuf-specs": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/@sigstore/protobuf-specs/-/protobuf-specs-0.2.1.tgz",
-      "integrity": "sha512-XTWVxnWJu+c1oCshMLwnKvz8ZQJJDVOlciMfgpJBQbThVjKTCG8dwyhgLngBD2KN0ap9F/gOV8rFDEx8uh7R2A==",
-      "dev": true,
-      "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=6.5"
       }
     },
-    "node_modules/@sigstore/sign": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@sigstore/sign/-/sign-1.0.0.tgz",
-      "integrity": "sha512-INxFVNQteLtcfGmcoldzV6Je0sbbfh9I16DM4yJPw3j5+TFP8X6uIiA18mvpEa9yyeycAKgPmOA3X9hVdVTPUA==",
+    "node_modules/accepts": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
       "dev": true,
       "dependencies": {
-        "@sigstore/bundle": "^1.1.0",
-        "@sigstore/protobuf-specs": "^0.2.0",
-        "make-fetch-happen": "^11.0.1"
+        "mime-types": "~2.1.34",
+        "negotiator": "0.6.3"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/@sigstore/tuf": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@sigstore/tuf/-/tuf-1.0.3.tgz",
-      "integrity": "sha512-2bRovzs0nJZFlCN3rXirE4gwxCn97JNjMmwpecqlbgV9WcxX7WRuIrgzx/X7Ib7MYRbyUTpBYE0s2x6AmZXnlg==",
+    "node_modules/acorn": {
+      "version": "8.11.2",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz",
+      "integrity": "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==",
       "dev": true,
-      "dependencies": {
-        "@sigstore/protobuf-specs": "^0.2.0",
-        "tuf-js": "^1.1.7"
+      "bin": {
+        "acorn": "bin/acorn"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@sinclair/typebox": {
-      "version": "0.27.8",
-      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
-      "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==",
-      "dev": true
-    },
-    "node_modules/@sinonjs/commons": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
-      "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
+    "node_modules/acorn-globals": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz",
+      "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==",
       "dev": true,
       "dependencies": {
-        "type-detect": "4.0.8"
+        "acorn": "^6.0.1",
+        "acorn-walk": "^6.0.1"
       }
     },
-    "node_modules/@sinonjs/fake-timers": {
-      "version": "10.3.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz",
-      "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==",
+    "node_modules/acorn-globals/node_modules/acorn": {
+      "version": "6.4.2",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz",
+      "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==",
       "dev": true,
-      "dependencies": {
-        "@sinonjs/commons": "^3.0.0"
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@stylelint/postcss-css-in-js": {
-      "version": "0.37.3",
-      "resolved": "https://registry.npmjs.org/@stylelint/postcss-css-in-js/-/postcss-css-in-js-0.37.3.tgz",
-      "integrity": "sha512-scLk3cSH1H9KggSniseb2KNAU5D9FWc3H7BxCSAIdtU9OWIyw0zkEZ9qEKHryRM+SExYXRKNb7tOOVNAsQ3iwg==",
-      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+    "node_modules/acorn-import-assertions": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz",
+      "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==",
       "dev": true,
-      "dependencies": {
-        "@babel/core": "^7.17.9"
-      },
       "peerDependencies": {
-        "postcss": ">=7.0.0",
-        "postcss-syntax": ">=0.36.2"
+        "acorn": "^8"
       }
     },
-    "node_modules/@stylelint/postcss-markdown": {
-      "version": "0.36.2",
-      "resolved": "https://registry.npmjs.org/@stylelint/postcss-markdown/-/postcss-markdown-0.36.2.tgz",
-      "integrity": "sha512-2kGbqUVJUGE8dM+bMzXG/PYUWKkjLIkRLWNh39OaADkiabDRdw8ATFCgbMz5xdIcvwspPAluSL7uY+ZiTWdWmQ==",
-      "deprecated": "Use the original unforked package instead: postcss-markdown",
+    "node_modules/acorn-jsx": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
+      "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
       "dev": true,
-      "dependencies": {
-        "remark": "^13.0.0",
-        "unist-util-find-all-after": "^3.0.2"
-      },
       "peerDependencies": {
-        "postcss": ">=7.0.0",
-        "postcss-syntax": ">=0.36.2"
+        "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
       }
     },
-    "node_modules/@swagger-api/apidom-ast": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ast/-/apidom-ast-0.76.2.tgz",
-      "integrity": "sha512-yLSeI3KtfpR7tI/misqTeasFonssj9GGhCOJfSHBuRAZkrPCJf0eU8vh3pL7YPa8lqFWcPT+z/arZoMcC9VLnQ==",
+    "node_modules/acorn-node": {
+      "version": "1.8.2",
+      "resolved": "https://registry.npmjs.org/acorn-node/-/acorn-node-1.8.2.tgz",
+      "integrity": "sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2",
-        "unraw": "^3.0.0"
+        "acorn": "^7.0.0",
+        "acorn-walk": "^7.0.0",
+        "xtend": "^4.0.2"
       }
     },
-    "node_modules/@swagger-api/apidom-core": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-core/-/apidom-core-0.76.2.tgz",
-      "integrity": "sha512-366dJJM7DFONlO3nUQfQRMJpJzZjPpWZldbHJZCcvy+aCyrNYI3Waauas7fm29UXRliPirGrd9e/ZsnW3Jimag==",
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-ast": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "minim": "~0.23.8",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "short-unique-id": "^5.0.2",
-        "stampit": "^4.3.2"
+    "node_modules/acorn-node/node_modules/acorn": {
+      "version": "7.4.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
+      "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
+      "dev": true,
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@swagger-api/apidom-error": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-error/-/apidom-error-0.76.2.tgz",
-      "integrity": "sha512-QxoWL+qGzwftqXSJaYLZ1Nrdtro+U1zX5Q4OLK+Ggg8Hi6Kn1SGXcHhn4JZ9J1rwrP85XCabilL3z9mhdebqWg==",
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+    "node_modules/acorn-node/node_modules/acorn-walk": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",
+      "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@swagger-api/apidom-json-pointer": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-json-pointer/-/apidom-json-pointer-0.76.2.tgz",
-      "integrity": "sha512-2XCgA4bn8vB1VMDbSiP+6SHUTiBxx1EVLW2pgqFolhLPMdiI/QBVmoW+jEkvTPo4d5gwj/vP5WDs5QnnC9VwEA==",
+    "node_modules/acorn-walk": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz",
+      "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/adjust-sourcemap-loader": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz",
+      "integrity": "sha512-OXwN5b9pCUXNQHJpwwD2qP40byEmSgzj8B4ydSN0uMNYWiFmJ6x6KwUllMmfk8Rwu/HJDFR7U8ubsWBoN0Xp0A==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+        "loader-utils": "^2.0.0",
+        "regex-parser": "^2.2.11"
+      },
+      "engines": {
+        "node": ">=8.9"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-api-design-systems": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-api-design-systems/-/apidom-ns-api-design-systems-0.76.2.tgz",
-      "integrity": "sha512-ct83R5Pvc08jeOuGShO4N0ty7VO8f46WedTDCbzT4edMRhd9Xdr5UFxkwWDuliy4uLzl9ZayHygSxfnyZKQb8g==",
-      "optional": true,
+    "node_modules/adjust-sourcemap-loader/node_modules/loader-utils": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz",
+      "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-1": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "big.js": "^5.2.2",
+        "emojis-list": "^3.0.0",
+        "json5": "^2.1.2"
+      },
+      "engines": {
+        "node": ">=8.9.0"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-asyncapi-2": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-asyncapi-2/-/apidom-ns-asyncapi-2-0.76.2.tgz",
-      "integrity": "sha512-ffV2AhF7jTBbYl2vX0nYSDufs70CmC/kNMWHkgwR2Vq86lgadUc6S/NK/djpWY8+oAU3EYmHwTqu07hpSOUb4A==",
-      "optional": true,
+    "node_modules/agent-base": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+      "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-json-schema-draft-7": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-json-schema-draft-4": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-4/-/apidom-ns-json-schema-draft-4-0.76.2.tgz",
-      "integrity": "sha512-0Y32CQE6tIt4IPsoCzWAUskZSyGkfw87IIsH5Bcm3D1qIlAhPAokQbe1212MmZoLVUvqrXDqZHXnOxxMaHZvYw==",
+    "node_modules/agentkeepalive": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
+      "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-ast": "^0.76.2",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "humanize-ms": "^1.2.1"
+      },
+      "engines": {
+        "node": ">= 8.0.0"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-json-schema-draft-6": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-6/-/apidom-ns-json-schema-draft-6-0.76.2.tgz",
-      "integrity": "sha512-i6nZtj3ie6SP1LhRtBeZNJuBppWkuC/+AsVfUzXkH5pM+3B7Puklc77hHdLtmvUTpd/iRBdlfsklvBVXJYPtUA==",
-      "optional": true,
+    "node_modules/aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@swagger-api/apidom-ns-json-schema-draft-4": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-json-schema-draft-7": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-json-schema-draft-7/-/apidom-ns-json-schema-draft-7-0.76.2.tgz",
-      "integrity": "sha512-Klyfi/1XkJVUZa1nJP87HPMjklmB3IxE+TSD27aZIEi7GKASu96euan0gflZaegexUBA9hsAngk98USbdpHpgQ==",
-      "optional": true,
+    "node_modules/ajv": {
+      "version": "8.12.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
+      "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@swagger-api/apidom-ns-json-schema-draft-6": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "fast-deep-equal": "^3.1.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
       }
     },
-    "node_modules/@swagger-api/apidom-ns-openapi-3-0": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-openapi-3-0/-/apidom-ns-openapi-3-0-0.76.2.tgz",
-      "integrity": "sha512-tV7dfbAZjX4HHul6JzmWsipMIVHCX5fAsBwLTltq8qmF9X9m6kZwg7fb4pD+cGK2KVlZl/ucDDDIQLDRWpOAog==",
+    "node_modules/ajv-formats": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
+      "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-json-schema-draft-4": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "ajv": "^8.0.0"
+      },
+      "peerDependencies": {
+        "ajv": "^8.0.0"
+      },
+      "peerDependenciesMeta": {
+        "ajv": {
+          "optional": true
+        }
       }
     },
-    "node_modules/@swagger-api/apidom-ns-openapi-3-1": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-ns-openapi-3-1/-/apidom-ns-openapi-3-1-0.76.2.tgz",
-      "integrity": "sha512-Mb9VhVacoWvQcBqxO4j0eweyM6PGupAOt7XcOL5CzID0dOU+P4BbAv6kHD++0bTqRgXk1O31HkS/yPJmPaTCrw==",
+    "node_modules/ajv-keywords": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz",
+      "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-ast": "^0.76.2",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-0": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "fast-deep-equal": "^3.1.3"
+      },
+      "peerDependencies": {
+        "ajv": "^8.8.2"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-api-design-systems-json": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-api-design-systems-json/-/apidom-parser-adapter-api-design-systems-json-0.76.2.tgz",
-      "integrity": "sha512-mJ4HLVIR9YHgWu0SiHykFQ9Sz1f3eV5Wqhrff8sH2Qll+4QSSdOOs0tW4Gp56F0HIcrU66uvrrTy1tpkO943aw==",
+    "node_modules/amdefine": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz",
+      "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==",
+      "dev": true,
       "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-api-design-systems": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-json": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+      "engines": {
+        "node": ">=0.4.2"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-api-design-systems-yaml": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-api-design-systems-yaml/-/apidom-parser-adapter-api-design-systems-yaml-0.76.2.tgz",
-      "integrity": "sha512-ot0F8Pw9/oWce6daDK+3srhNad/Iva/OlkVtN0S9cR58Zcn8p1F3s6RcN7ZG97i8EdBuyQj6Bm0jzXnOX+lvtQ==",
-      "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-api-design-systems": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+    "node_modules/ansi-colors": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz",
+      "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-asyncapi-json-2": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-asyncapi-json-2/-/apidom-parser-adapter-asyncapi-json-2-0.76.2.tgz",
-      "integrity": "sha512-FK06pb4w5E8RQ65Nh1FHHM8aWzPL7fHr2HeuXZkbSeKu4j0xyzwYkxZVGwZJOT6YPJR0Yrkb/2rD89CNXsLctA==",
-      "optional": true,
+    "node_modules/ansi-escapes": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
+      "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-asyncapi-2": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-json": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+        "type-fest": "^0.21.3"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-asyncapi-yaml-2/-/apidom-parser-adapter-asyncapi-yaml-2-0.76.2.tgz",
-      "integrity": "sha512-7TGhZgHZ9nmBJnFA7YhDWbNDbKoUOGVkBqx563ExHr2FewaohiQ/wagXAhKZzOK+HS+KHvob09uROtqOWGdIew==",
-      "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-asyncapi-2": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+    "node_modules/ansi-escapes/node_modules/type-fest": {
+      "version": "0.21.3",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
+      "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-json": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-json/-/apidom-parser-adapter-json-0.76.2.tgz",
-      "integrity": "sha512-vbH7EcldZ/gSK9FnGUW1cpibM5+hiJPQcoyLmzLZe8YBxX73qzd2WAd77v+uI56eO9Z0G4KMCRCF9PDZT/tz5Q==",
-      "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-ast": "^0.76.2",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2",
-        "tree-sitter": "=0.20.4",
-        "tree-sitter-json": "=0.20.0",
-        "web-tree-sitter": "=0.20.3"
+    "node_modules/ansi-html-community": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz",
+      "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==",
+      "dev": true,
+      "engines": [
+        "node >= 0.8.0"
+      ],
+      "bin": {
+        "ansi-html": "bin/ansi-html"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-openapi-json-3-0": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-json-3-0/-/apidom-parser-adapter-openapi-json-3-0-0.76.2.tgz",
-      "integrity": "sha512-Kqcq5QUgz1TcCuPaL+zU+wmdAEo7YM0LR5jyWQo3FAT3BhAsmeVv2wRZMiz9RMDrPyxzHzbJhjMZxCqL8r2G0g==",
-      "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-0": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-json": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+    "node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-openapi-json-3-1": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-json-3-1/-/apidom-parser-adapter-openapi-json-3-1-0.76.2.tgz",
-      "integrity": "sha512-kfZ4BBxww5afiIIeFT6l0/Kuob72dnYAP+Qnmp2zQB3GQUTilKqv+ddj4blCF19n8RGNERVv2RDHLTZhjg+1AA==",
-      "optional": true,
+    "node_modules/ansi-styles": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-1": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-json": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+        "color-convert": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-yaml-3-0/-/apidom-parser-adapter-openapi-yaml-3-0-0.76.2.tgz",
-      "integrity": "sha512-spXabhd0sgX87QaYUDou22KduSL5GHCmLNuPDpPykYelB/zZnE8aPsrjBMIgK9CPZoQCDoWYYmtRTPfJjKwf3Q==",
-      "optional": true,
+    "node_modules/any-promise": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
+      "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==",
+      "dev": true
+    },
+    "node_modules/anymatch": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
+      "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-0": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+        "normalize-path": "^3.0.0",
+        "picomatch": "^2.0.4"
+      },
+      "engines": {
+        "node": ">= 8"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-openapi-yaml-3-1/-/apidom-parser-adapter-openapi-yaml-3-1-0.76.2.tgz",
-      "integrity": "sha512-KIEg9QWeiMMKQ9VtftK+1Rc7irKQjj0VTsoEtraun9N2MWLVt7g+xZKqbqtQ4/ovv5J8JBHE+hFGLdm2qZalsg==",
-      "optional": true,
+    "node_modules/apache-crypt": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/apache-crypt/-/apache-crypt-1.2.6.tgz",
+      "integrity": "sha512-072WetlM4blL8PREJVeY+WHiUh1R5VNt2HfceGS8aKqttPHcmqE5pkKuXPz/ULmJOFkc8Hw3kfKl6vy7Qka6DA==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-1": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.0.0"
+        "unix-crypt-td-js": "^1.1.4"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@swagger-api/apidom-parser-adapter-yaml-1-2": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-parser-adapter-yaml-1-2/-/apidom-parser-adapter-yaml-1-2-0.76.2.tgz",
-      "integrity": "sha512-nmEDYOfqeB8yCHbQ5yEQkJ09zIDOeX61KXTUktP4yErm96WVjIUk5YTTAkO7QbAEND9JHE+BAnS25cBC8BxFFA==",
-      "optional": true,
-      "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-ast": "^0.76.2",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2",
-        "tree-sitter": "=0.20.4",
-        "tree-sitter-yaml": "=0.5.0",
-        "web-tree-sitter": "=0.20.3"
+    "node_modules/apache-md5": {
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/apache-md5/-/apache-md5-1.1.8.tgz",
+      "integrity": "sha512-FCAJojipPn0bXjuEpjOOOMN8FZDkxfWWp4JGN9mifU2IhxvKyXZYqpzPHdnTSUpmPDy+tsslB6Z1g+Vg6nVbYA==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@swagger-api/apidom-reference": {
-      "version": "0.76.2",
-      "resolved": "https://registry.npmjs.org/@swagger-api/apidom-reference/-/apidom-reference-0.76.2.tgz",
-      "integrity": "sha512-O1qX6Tql+B18Em/ERyqCzuhcvOG3JeRq4QIHfebzS3lNxpxX6si/z0DrL5K1azBldmnXx7UGqt/fvwq8GQJmIA==",
+    "node_modules/aproba": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
+      "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
+      "dev": true
+    },
+    "node_modules/arch": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz",
+      "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/are-we-there-yet": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz",
+      "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==",
+      "dev": true,
       "dependencies": {
-        "@babel/runtime-corejs3": "^7.20.7",
-        "@swagger-api/apidom-core": "^0.76.2",
-        "@types/ramda": "~0.29.3",
-        "axios": "^1.4.0",
-        "minimatch": "^7.4.3",
-        "process": "^0.11.10",
-        "ramda": "~0.29.0",
-        "ramda-adjunct": "^4.1.1",
-        "stampit": "^4.3.2"
+        "delegates": "^1.0.0",
+        "readable-stream": "^3.6.0"
       },
-      "optionalDependencies": {
-        "@swagger-api/apidom-error": "^0.76.2",
-        "@swagger-api/apidom-json-pointer": "^0.76.2",
-        "@swagger-api/apidom-ns-asyncapi-2": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-0": "^0.76.2",
-        "@swagger-api/apidom-ns-openapi-3-1": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-api-design-systems-json": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-api-design-systems-yaml": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-asyncapi-json-2": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-asyncapi-yaml-2": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-json": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-openapi-json-3-0": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-openapi-json-3-1": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-openapi-yaml-3-0": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-openapi-yaml-3-1": "^0.76.2",
-        "@swagger-api/apidom-parser-adapter-yaml-1-2": "^0.76.2"
-      }
-    },
-    "node_modules/@swagger-api/apidom-reference/node_modules/axios": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz",
-      "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==",
-      "dependencies": {
-        "follow-redirects": "^1.15.0",
-        "form-data": "^4.0.0",
-        "proxy-from-env": "^1.1.0"
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
       }
     },
-    "node_modules/@swagger-api/apidom-reference/node_modules/brace-expansion": {
+    "node_modules/arg": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+      "dev": true
+    },
+    "node_modules/argparse": {
       "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+    },
+    "node_modules/aria-query": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz",
+      "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==",
+      "dev": true,
       "dependencies": {
-        "balanced-match": "^1.0.0"
+        "@babel/runtime": "^7.10.2",
+        "@babel/runtime-corejs3": "^7.10.2"
+      },
+      "engines": {
+        "node": ">=6.0"
       }
     },
-    "node_modules/@swagger-api/apidom-reference/node_modules/form-data": {
+    "node_modules/arr-diff": {
       "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
+      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
+      "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==",
+      "dev": true,
       "engines": {
-        "node": ">= 6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@swagger-api/apidom-reference/node_modules/minimatch": {
-      "version": "7.4.6",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.6.tgz",
-      "integrity": "sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw==",
-      "dependencies": {
-        "brace-expansion": "^2.0.1"
-      },
+    "node_modules/arr-flatten": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
+      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
+      "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@swagger-api/apidom-reference/node_modules/proxy-from-env": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
-    },
-    "node_modules/@swimlane/ngx-datatable": {
-      "version": "18.0.0",
-      "resolved": "https://registry.npmjs.org/@swimlane/ngx-datatable/-/ngx-datatable-18.0.0.tgz",
-      "integrity": "sha512-secqjzlLpGJqoXjcoCoTf8ClnVlZAENJcXvuBfseGenOD+evGNXc4UTZhwCPDUBlJ4xnMZHUWK6IVk5sXe+WlQ==",
+    "node_modules/array-buffer-byte-length": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz",
+      "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==",
+      "dev": true,
       "dependencies": {
-        "tslib": "^2.0.0"
+        "call-bind": "^1.0.2",
+        "is-array-buffer": "^3.0.1"
       },
-      "peerDependencies": {
-        "@angular/common": "^10.0.0",
-        "@angular/core": "^10.0.0",
-        "@angular/platform-browser": "^10.0.0",
-        "rxjs": "^6.5.5"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@tootallnate/once": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
-      "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
+    "node_modules/array-differ": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz",
+      "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==",
       "dev": true,
       "engines": {
-        "node": ">= 10"
+        "node": ">=8"
       }
     },
-    "node_modules/@ts-morph/common": {
-      "version": "0.12.3",
-      "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.12.3.tgz",
-      "integrity": "sha512-4tUmeLyXJnJWvTFOKtcNJ1yh0a3SsTLi2MUoyj8iUNznFRN1ZquaNe7Oukqrnki2FzZkm0J9adCNLDZxUzvj+w==",
+    "node_modules/array-each": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz",
+      "integrity": "sha512-zHjL5SZa68hkKHBFBK6DJCTtr9sfTCPCaph/L7tMSLcTFgy+zX7E+6q5UArbtOtMBCtxdICpfTCspRse+ywyXA==",
       "dev": true,
-      "dependencies": {
-        "fast-glob": "^3.2.7",
-        "minimatch": "^3.0.4",
-        "mkdirp": "^1.0.4",
-        "path-browserify": "^1.0.1"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@ts-morph/common/node_modules/mkdirp": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
-      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+    "node_modules/array-equal": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.2.tgz",
+      "integrity": "sha512-gUHx76KtnhEgB3HOuFYiCm3FIdEs6ocM2asHvNTkfu/Y09qQVrrVVaOKENmS2KkSaGoxgXNqC+ZVtR/n0MOkSA==",
       "dev": true,
-      "bin": {
-        "mkdirp": "bin/cmd.js"
-      },
-      "engines": {
-        "node": ">=10"
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@ts-morph/common/node_modules/path-browserify": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz",
-      "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==",
+    "node_modules/array-flatten": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz",
+      "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==",
       "dev": true
     },
-    "node_modules/@tufjs/canonical-json": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-1.0.0.tgz",
-      "integrity": "sha512-QTnf++uxunWvG2z3UFNzAoQPHxnSXOwtaI3iJ+AohhV+5vONuArPjJE7aPXPVXfXJsqrVbZBu9b81AJoSd09IQ==",
+    "node_modules/array-from": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz",
+      "integrity": "sha512-GQTc6Uupx1FCavi5mPzBvVT7nEOeWMmUA9P95wpfpW1XwMSKs+KaymD5C2Up7KAUKg/mYwbsUYzdZWcoajlNZg==",
+      "dev": true
+    },
+    "node_modules/array-slice": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
+      "integrity": "sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w==",
       "dev": true,
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@tufjs/models": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@tufjs/models/-/models-1.0.4.tgz",
-      "integrity": "sha512-qaGV9ltJP0EO25YfFUPhxRVK0evXFIAGicsVXuRim4Ed9cjPxYhNnNJ49SFmbeLgtxpslIkX317IgpfcHPVj/A==",
+    "node_modules/array-union": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+      "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
       "dev": true,
-      "dependencies": {
-        "@tufjs/canonical-json": "1.0.0",
-        "minimatch": "^9.0.0"
-      },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@tufjs/models/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+    "node_modules/array-unique": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
+      "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==",
       "dev": true,
-      "dependencies": {
-        "balanced-match": "^1.0.0"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@tufjs/models/node_modules/minimatch": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
-      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+    "node_modules/arraybuffer.prototype.slice": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz",
+      "integrity": "sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==",
       "dev": true,
       "dependencies": {
-        "brace-expansion": "^2.0.1"
+        "array-buffer-byte-length": "^1.0.0",
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.2.0",
+        "es-abstract": "^1.22.1",
+        "get-intrinsic": "^1.2.1",
+        "is-array-buffer": "^3.0.2",
+        "is-shared-array-buffer": "^1.0.2"
       },
       "engines": {
-        "node": ">=16 || 14 >=14.17"
+        "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@types/babel__core": {
-      "version": "7.20.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz",
-      "integrity": "sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA==",
+    "node_modules/arrify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
+      "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==",
       "dev": true,
-      "dependencies": {
-        "@babel/parser": "^7.20.7",
-        "@babel/types": "^7.20.7",
-        "@types/babel__generator": "*",
-        "@types/babel__template": "*",
-        "@types/babel__traverse": "*"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@types/babel__generator": {
-      "version": "7.6.5",
-      "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.5.tgz",
-      "integrity": "sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w==",
-      "dev": true,
-      "dependencies": {
-        "@babel/types": "^7.0.0"
-      }
+    "node_modules/asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
+      "dev": true
     },
-    "node_modules/@types/babel__template": {
-      "version": "7.4.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.2.tgz",
-      "integrity": "sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ==",
+    "node_modules/asn1": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
+      "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
       "dev": true,
       "dependencies": {
-        "@babel/parser": "^7.1.0",
-        "@babel/types": "^7.0.0"
+        "safer-buffer": "~2.1.0"
       }
     },
-    "node_modules/@types/babel__traverse": {
-      "version": "7.20.2",
-      "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.2.tgz",
-      "integrity": "sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw==",
+    "node_modules/asn1.js": {
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
+      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
       "dev": true,
       "dependencies": {
-        "@babel/types": "^7.20.7"
+        "bn.js": "^4.0.0",
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0",
+        "safer-buffer": "^2.1.0"
       }
     },
-    "node_modules/@types/body-parser": {
-      "version": "1.19.3",
-      "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.3.tgz",
-      "integrity": "sha512-oyl4jvAfTGX9Bt6Or4H9ni1Z447/tQuxnZsytsCaExKlmJiU8sFgnIBRzJUpKwB5eWn9HuBYlUlVA74q/yN0eQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/connect": "*",
-        "@types/node": "*"
-      }
+    "node_modules/asn1.js/node_modules/bn.js": {
+      "version": "4.12.0",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
+      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
+      "dev": true
     },
-    "node_modules/@types/bonjour": {
-      "version": "3.5.11",
-      "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.11.tgz",
-      "integrity": "sha512-isGhjmBtLIxdHBDl2xGwUzEM8AOyOvWsADWq7rqirdi/ZQoHnLWErHvsThcEzTX8juDRiZtzp2Qkv5bgNh6mAg==",
+    "node_modules/assert": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.1.tgz",
+      "integrity": "sha512-zzw1uCAgLbsKwBfFc8CX78DDg+xZeBksSO3vwVIDDN5i94eOrPsSSyiVhmsSABFDM/OcpE2aagCat9dnWQLG1A==",
       "dev": true,
       "dependencies": {
-        "@types/node": "*"
+        "object.assign": "^4.1.4",
+        "util": "^0.10.4"
       }
     },
-    "node_modules/@types/brace-expansion": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@types/brace-expansion/-/brace-expansion-1.1.0.tgz",
-      "integrity": "sha512-SaU/Kgp6z40CiF9JxlsrSrBEa+8YIry9IiCPhhYSNekeEhIAkY7iyu9aZ+5dSQIdo7mf86MUVvxWYm5GAzB/0g==",
-      "dev": true
-    },
-    "node_modules/@types/connect": {
-      "version": "3.4.36",
-      "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz",
-      "integrity": "sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==",
+    "node_modules/assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==",
       "dev": true,
-      "dependencies": {
-        "@types/node": "*"
+      "engines": {
+        "node": ">=0.8"
       }
     },
-    "node_modules/@types/connect-history-api-fallback": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.1.tgz",
-      "integrity": "sha512-iaQslNbARe8fctL5Lk+DsmgWOM83lM+7FzP0eQUJs1jd3kBE8NWqBTIT2S8SqQOJjxvt2eyIjpOuYeRXq2AdMw==",
+    "node_modules/assertion-error": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
+      "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
       "dev": true,
-      "dependencies": {
-        "@types/express-serve-static-core": "*",
-        "@types/node": "*"
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/@types/cypress-cucumber-preprocessor": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-4.0.1.tgz",
-      "integrity": "sha512-sK2/uU5CtmJ51zo0JF2Lc4iSw9Fy3xn9ewfewuooV5Qmeb5O+brAHuoXKMV7UWwRbBmd+txhAXAJoi4S5QLDRQ==",
-      "dev": true
-    },
-    "node_modules/@types/eslint": {
-      "version": "8.44.3",
-      "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.3.tgz",
-      "integrity": "sha512-iM/WfkwAhwmPff3wZuPLYiHX18HI24jU8k1ZSH7P8FHwxTjZ2P6CoX2wnF43oprR+YXJM6UUxATkNvyv/JHd+g==",
+    "node_modules/assertion-error-formatter": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/assertion-error-formatter/-/assertion-error-formatter-2.0.1.tgz",
+      "integrity": "sha512-cjC3jUCh9spkroKue5PDSKH5RFQ/KNuZJhk3GwHYmB/8qqETxLOmMdLH+ohi/VukNzxDlMvIe7zScvLoOdhb6Q==",
       "dev": true,
       "dependencies": {
-        "@types/estree": "*",
-        "@types/json-schema": "*"
+        "diff": "^3.0.0",
+        "pad-right": "^0.2.2",
+        "repeat-string": "^1.6.1"
       }
     },
-    "node_modules/@types/eslint-scope": {
-      "version": "3.7.5",
-      "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.5.tgz",
-      "integrity": "sha512-JNvhIEyxVW6EoMIFIvj93ZOywYFatlpu9deeH6eSx6PE3WHYvHaQtmHmQeNw7aA81bYGBPPQqdtBm6b1SsQMmA==",
+    "node_modules/assign-symbols": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
+      "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==",
       "dev": true,
-      "dependencies": {
-        "@types/eslint": "*",
-        "@types/estree": "*"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@types/estree": {
-      "version": "0.0.51",
-      "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz",
-      "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==",
-      "dev": true
-    },
-    "node_modules/@types/express": {
-      "version": "4.17.18",
-      "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.18.tgz",
-      "integrity": "sha512-Sxv8BSLLgsBYmcnGdGjjEjqET2U+AKAdCRODmMiq02FgjwuV75Ut85DRpvFjyw/Mk0vgUOliGRU0UUmuuZHByQ==",
+    "node_modules/ast-transform": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/ast-transform/-/ast-transform-0.0.0.tgz",
+      "integrity": "sha512-e/JfLiSoakfmL4wmTGPjv0HpTICVmxwXgYOB8x+mzozHL8v+dSfCbrJ8J8hJ0YBP0XcYu1aLZ6b/3TnxNK3P2A==",
       "dev": true,
       "dependencies": {
-        "@types/body-parser": "*",
-        "@types/express-serve-static-core": "^4.17.33",
-        "@types/qs": "*",
-        "@types/serve-static": "*"
+        "escodegen": "~1.2.0",
+        "esprima": "~1.0.4",
+        "through": "~2.3.4"
       }
     },
-    "node_modules/@types/express-serve-static-core": {
-      "version": "4.17.37",
-      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.37.tgz",
-      "integrity": "sha512-ZohaCYTgGFcOP7u6aJOhY9uIZQgZ2vxC2yWoArY+FeDXlqeH66ZVBjgvg+RLVAS/DWNq4Ap9ZXu1+SUQiiWYMg==",
+    "node_modules/ast-transform/node_modules/escodegen": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.2.0.tgz",
+      "integrity": "sha512-yLy3Cc+zAC0WSmoT2fig3J87TpQ8UaZGx8ahCAs9FL8qNbyV7CVyPKS74DG4bsHiL5ew9sxdYx131OkBQMFnvA==",
       "dev": true,
       "dependencies": {
-        "@types/node": "*",
-        "@types/qs": "*",
-        "@types/range-parser": "*",
-        "@types/send": "*"
+        "esprima": "~1.0.4",
+        "estraverse": "~1.5.0",
+        "esutils": "~1.0.0"
+      },
+      "bin": {
+        "escodegen": "bin/escodegen.js",
+        "esgenerate": "bin/esgenerate.js"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      },
+      "optionalDependencies": {
+        "source-map": "~0.1.30"
       }
     },
-    "node_modules/@types/file-saver": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@types/file-saver/-/file-saver-2.0.1.tgz",
-      "integrity": "sha512-g1QUuhYVVAamfCifK7oB7G3aIl4BbOyzDOqVyUfEr4tfBKrXfeH+M+Tg7HKCXSrbzxYdhyCP7z9WbKo0R2hBCw=="
+    "node_modules/ast-transform/node_modules/esprima": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz",
+      "integrity": "sha512-rp5dMKN8zEs9dfi9g0X1ClLmV//WRyk/R15mppFNICIFRG5P92VP7Z04p8pk++gABo9W2tY+kHyu6P1mEHgmTA==",
+      "dev": true,
+      "bin": {
+        "esparse": "bin/esparse.js",
+        "esvalidate": "bin/esvalidate.js"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
     },
-    "node_modules/@types/graceful-fs": {
-      "version": "4.1.7",
-      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.7.tgz",
-      "integrity": "sha512-MhzcwU8aUygZroVwL2jeYk6JisJrPl/oov/gsgGCue9mkgl9wjGbzReYQClxiUgFDnib9FuHqTndccKeZKxTRw==",
+    "node_modules/ast-transform/node_modules/estraverse": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.5.1.tgz",
+      "integrity": "sha512-FpCjJDfmo3vsc/1zKSeqR5k42tcIhxFIlvq+h9j0fO2q/h2uLKyweq7rYJ+0CoVvrGQOxIS5wyBrW/+vF58BUQ==",
       "dev": true,
-      "dependencies": {
-        "@types/node": "*"
+      "engines": {
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/@types/hast": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.6.tgz",
-      "integrity": "sha512-47rJE80oqPmFdVDCD7IheXBrVdwuBgsYwoczFvKmwfo2Mzsnt+V9OONsYauFmICb6lQPpCuXYJWejBNs4pDJRg==",
-      "dependencies": {
-        "@types/unist": "^2"
+    "node_modules/ast-transform/node_modules/esutils": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-1.0.0.tgz",
+      "integrity": "sha512-x/iYH53X3quDwfHRz4y8rn4XcEwwCJeWsul9pF1zldMbGtgOtMNBEOuYWwB1EQlK2LRa1fev3YAgym/RElp5Cg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@types/hoist-non-react-statics": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz",
-      "integrity": "sha512-YIQtIg4PKr7ZyqNPZObpxfHsHEmuB8dXCxd6qVcGuQVDK2bpsF7bYNnBJ4Nn7giuACZg+WewExgrtAJ3XnA4Xw==",
+    "node_modules/ast-transform/node_modules/source-map": {
+      "version": "0.1.43",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz",
+      "integrity": "sha512-VtCvB9SIQhk3aF6h+N85EaqIaBFIAfZ9Cu+NJHHVvc8BbEcnvDcFw6sqQ2dQrT6SlOrZq3tIvyD9+EGq/lJryQ==",
+      "dev": true,
+      "optional": true,
       "dependencies": {
-        "@types/react": "*",
-        "hoist-non-react-statics": "^3.3.0"
+        "amdefine": ">=0.0.4"
+      },
+      "engines": {
+        "node": ">=0.8.0"
       }
     },
-    "node_modules/@types/http-errors": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.2.tgz",
-      "integrity": "sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg==",
-      "dev": true
+    "node_modules/ast-types": {
+      "version": "0.7.8",
+      "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.7.8.tgz",
+      "integrity": "sha512-RIOpVnVlltB6PcBJ5BMLx+H+6JJ/zjDGU0t7f0L6c2M1dqcK92VQopLBlPQ9R80AVXelfqYgjcPLtHtDbNFg0Q==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.6"
+      }
     },
-    "node_modules/@types/http-proxy": {
-      "version": "1.17.12",
-      "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.12.tgz",
-      "integrity": "sha512-kQtujO08dVtQ2wXAuSFfk9ASy3sug4+ogFR8Kd8UgP8PEuc1/G/8yjYRmp//PcDNJEUKOza/MrQu15bouEUCiw==",
+    "node_modules/astral-regex": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+      "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
       "dev": true,
-      "dependencies": {
-        "@types/node": "*"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@types/istanbul-lib-coverage": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz",
-      "integrity": "sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==",
+    "node_modules/async": {
+      "version": "3.2.5",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz",
+      "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==",
       "dev": true
     },
-    "node_modules/@types/istanbul-lib-report": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz",
-      "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==",
-      "dev": true,
+    "node_modules/async-mutex": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.2.4.tgz",
+      "integrity": "sha512-fcQKOXUKMQc57JlmjBCHtkKNrfGpHyR7vu18RfuLfeTAf4hK9PgOadPR5cDrBQ682zasrLUhJFe7EKAHJOduDg==",
       "dependencies": {
-        "@types/istanbul-lib-coverage": "*"
+        "tslib": "^2.0.0"
       }
     },
-    "node_modules/@types/istanbul-reports": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz",
-      "integrity": "sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==",
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/at-least-node": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
+      "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==",
       "dev": true,
-      "dependencies": {
-        "@types/istanbul-lib-report": "*"
+      "engines": {
+        "node": ">= 4.0.0"
       }
     },
-    "node_modules/@types/jest": {
-      "version": "29.5.4",
-      "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.4.tgz",
-      "integrity": "sha512-PhglGmhWeD46FYOVLt3X7TiWjzwuVGW9wG/4qocPevXMjCmrIc5b6db9WjeGE4QYVpUAWMDv3v0IiBwObY289A==",
+    "node_modules/atob": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
+      "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
       "dev": true,
-      "dependencies": {
-        "expect": "^29.0.0",
-        "pretty-format": "^29.0.0"
+      "bin": {
+        "atob": "bin/atob.js"
+      },
+      "engines": {
+        "node": ">= 4.5.0"
       }
     },
-    "node_modules/@types/jsdom": {
-      "version": "20.0.1",
-      "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz",
-      "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==",
-      "dev": true,
+    "node_modules/autolinker": {
+      "version": "3.16.2",
+      "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-3.16.2.tgz",
+      "integrity": "sha512-JiYl7j2Z19F9NdTmirENSUUIIL/9MytEWtmzhfmsKPCp9E+G35Y0UNCMoM9tFigxT59qSc8Ml2dlZXOCVTYwuA==",
       "dependencies": {
-        "@types/node": "*",
-        "@types/tough-cookie": "*",
-        "parse5": "^7.0.0"
+        "tslib": "^2.3.0"
       }
     },
-    "node_modules/@types/jsdom/node_modules/parse5": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
-      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
+    "node_modules/autoprefixer": {
+      "version": "10.4.13",
+      "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
+      "integrity": "sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==",
       "dev": true,
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/postcss/"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/autoprefixer"
+        }
+      ],
       "dependencies": {
-        "entities": "^4.4.0"
+        "browserslist": "^4.21.4",
+        "caniuse-lite": "^1.0.30001426",
+        "fraction.js": "^4.2.0",
+        "normalize-range": "^0.1.2",
+        "picocolors": "^1.0.0",
+        "postcss-value-parser": "^4.2.0"
       },
-      "funding": {
-        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      "bin": {
+        "autoprefixer": "bin/autoprefixer"
+      },
+      "engines": {
+        "node": "^10 || ^12 || >=14"
+      },
+      "peerDependencies": {
+        "postcss": "^8.1.0"
       }
     },
-    "node_modules/@types/json-schema": {
-      "version": "7.0.13",
-      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.13.tgz",
-      "integrity": "sha512-RbSSoHliUbnXj3ny0CNFOoxrIDV6SUGyStHsvDqosw6CkdPV8TtWGlfecuK4ToyMEAql6pzNxgCFKanovUzlgQ==",
-      "dev": true
-    },
-    "node_modules/@types/lodash": {
-      "version": "4.14.161",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.161.tgz",
-      "integrity": "sha512-EP6O3Jkr7bXvZZSZYlsgt5DIjiGr0dXP1/jVEwVLTFgg0d+3lWVQkRavYVQszV7dYUwvg0B8R0MBDpcmXg7XIA==",
-      "dev": true
-    },
-    "node_modules/@types/long": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
-      "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",
-      "dev": true
-    },
-    "node_modules/@types/mdast": {
-      "version": "3.0.12",
-      "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.12.tgz",
-      "integrity": "sha512-DT+iNIRNX884cx0/Q1ja7NyUPpZuv0KPyL5rGNxm1WC1OtHstl7n4Jb7nk+xacNShQMbczJjt8uFzznpp6kYBg==",
+    "node_modules/available-typed-arrays": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
+      "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
       "dev": true,
-      "dependencies": {
-        "@types/unist": "^2"
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@types/mime": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
-      "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==",
-      "dev": true
-    },
-    "node_modules/@types/minimatch": {
-      "version": "3.0.5",
-      "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz",
-      "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==",
-      "dev": true
-    },
-    "node_modules/@types/minimist": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz",
-      "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==",
-      "dev": true
-    },
-    "node_modules/@types/node": {
-      "version": "18.17.12",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.12.tgz",
-      "integrity": "sha512-d6xjC9fJ/nSnfDeU0AMDsaJyb1iHsqCSOdi84w4u+SlN/UgQdY5tRhpMzaFYsI4mnpvgTivEaQd0yOUhAtOnEQ==",
-      "dev": true
-    },
-    "node_modules/@types/normalize-package-data": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.2.tgz",
-      "integrity": "sha512-lqa4UEhhv/2sjjIQgjX8B+RBjj47eo0mzGasklVJ78UKGQY1r0VpB9XHDaZZO9qzEFDdy4MrXLuEaSmPrPSe/A==",
-      "dev": true
-    },
-    "node_modules/@types/parse-json": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
-      "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
-      "dev": true
-    },
-    "node_modules/@types/prettier": {
-      "version": "2.7.3",
-      "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.3.tgz",
-      "integrity": "sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA==",
-      "dev": true
-    },
-    "node_modules/@types/prop-types": {
-      "version": "15.7.7",
-      "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.7.tgz",
-      "integrity": "sha512-FbtmBWCcSa2J4zL781Zf1p5YUBXQomPEcep9QZCfRfQgTxz3pJWiDFLebohZ9fFntX5ibzOkSsrJ0TEew8cAog=="
-    },
-    "node_modules/@types/qs": {
-      "version": "6.9.8",
-      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz",
-      "integrity": "sha512-u95svzDlTysU5xecFNTgfFG5RUWu1A9P0VzgpcIiGZA9iraHOdSzcxMxQ55DyeRaGCSxQi7LxXDI4rzq/MYfdg==",
-      "dev": true
-    },
-    "node_modules/@types/ramda": {
-      "version": "0.29.4",
-      "resolved": "https://registry.npmjs.org/@types/ramda/-/ramda-0.29.4.tgz",
-      "integrity": "sha512-bd3nyfkZd5EVxuBf1kW6wvFz61SvAEfXXISIEIePJOj2XRjCHyro1ikvDXTXIlpRtuC6lwTMfYdkXCD+oiXQfw==",
-      "dependencies": {
-        "types-ramda": "^0.29.4"
+    "node_modules/aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==",
+      "dev": true,
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/@types/range-parser": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz",
-      "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
+    "node_modules/aws4": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
+      "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==",
       "dev": true
     },
-    "node_modules/@types/react": {
-      "version": "18.2.22",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.22.tgz",
-      "integrity": "sha512-60fLTOLqzarLED2O3UQImc/lsNRgG0jE/a1mPW9KjMemY0LMITWEsbS4VvZ4p6rorEHd5YKxxmMKSDK505GHpA==",
-      "dependencies": {
-        "@types/prop-types": "*",
-        "@types/scheduler": "*",
-        "csstype": "^3.0.2"
+    "node_modules/axe-core": {
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz",
+      "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/@types/react-redux": {
-      "version": "7.1.26",
-      "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.26.tgz",
-      "integrity": "sha512-UKPo7Cm7rswYU6PH6CmTNCRv5NYF3HrgKuHEYTK8g/3czYLrUux50gQ2pkxc9c7ZpQZi+PNhgmI8oNIRoiVIxg==",
+    "node_modules/axios": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz",
+      "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==",
       "dependencies": {
-        "@types/hoist-non-react-statics": "^3.3.0",
-        "@types/react": "*",
-        "hoist-non-react-statics": "^3.3.0",
-        "redux": "^4.0.0"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
-    "node_modules/@types/retry": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
+    "node_modules/axobject-query": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz",
+      "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==",
       "dev": true
     },
-    "node_modules/@types/scheduler": {
-      "version": "0.16.4",
-      "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.4.tgz",
-      "integrity": "sha512-2L9ifAGl7wmXwP4v3pN4p2FLhD0O1qsJpvKmNin5VA8+UvNVb447UDaAEV6UdrkA+m/Xs58U1RFps44x6TFsVQ=="
-    },
-    "node_modules/@types/send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==",
+    "node_modules/babel-jest": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz",
+      "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==",
       "dev": true,
       "dependencies": {
-        "@types/mime": "^1",
-        "@types/node": "*"
+        "@jest/transform": "^29.7.0",
+        "@types/babel__core": "^7.1.14",
+        "babel-plugin-istanbul": "^6.1.1",
+        "babel-preset-jest": "^29.6.3",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.9",
+        "slash": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.8.0"
       }
     },
-    "node_modules/@types/serve-index": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz",
-      "integrity": "sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg==",
+    "node_modules/babel-jest/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@types/express": "*"
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/@types/serve-static": {
-      "version": "1.15.2",
-      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.2.tgz",
-      "integrity": "sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==",
+    "node_modules/babel-jest/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "@types/http-errors": "*",
-        "@types/mime": "*",
-        "@types/node": "*"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@types/sinonjs__fake-timers": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz",
-      "integrity": "sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==",
-      "dev": true
-    },
-    "node_modules/@types/sizzle": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.3.tgz",
-      "integrity": "sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==",
-      "dev": true
-    },
-    "node_modules/@types/sockjs": {
-      "version": "0.3.33",
-      "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz",
-      "integrity": "sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw==",
+    "node_modules/babel-jest/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@types/node": "*"
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/@types/stack-utils": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz",
-      "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
-      "dev": true
-    },
-    "node_modules/@types/swagger-ui": {
-      "version": "3.52.0",
-      "resolved": "https://registry.npmjs.org/@types/swagger-ui/-/swagger-ui-3.52.0.tgz",
-      "integrity": "sha512-SlufixEmh+8CLHNgTfAfCT1icNOF7bXboWabhHr1+hIolqlvfwYJGe7HgRcpI3ChE7HWASmEKLkMu34rxseJjQ==",
-      "dev": true
-    },
-    "node_modules/@types/tough-cookie": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.3.tgz",
-      "integrity": "sha512-THo502dA5PzG/sfQH+42Lw3fvmYkceefOspdCwpHRul8ik2Jv1K8I5OZz1AT3/rs46kwgMCe9bSBmDLYkkOMGg==",
-      "dev": true
-    },
-    "node_modules/@types/unist": {
-      "version": "2.0.8",
-      "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.8.tgz",
-      "integrity": "sha512-d0XxK3YTObnWVp6rZuev3c49+j4Lo8g4L1ZRm9z5L0xpoZycUPshHgczK5gsUMaZOstjVYYi09p5gYvUtfChYw=="
-    },
-    "node_modules/@types/uuid": {
-      "version": "3.4.11",
-      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.11.tgz",
-      "integrity": "sha512-CJNkbEu4IdVuBMRVaNC2GjASgJK7ziqDlVXWuJ1pvhOLADl7nzxhTKjHRdOmo2SuXuygcWBmzgYgn9foTX0UiA==",
+    "node_modules/babel-jest/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/@types/ws": {
-      "version": "8.5.5",
-      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
-      "integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==",
+    "node_modules/babel-jest/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "@types/node": "*"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@types/yargs": {
-      "version": "17.0.25",
-      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.25.tgz",
-      "integrity": "sha512-gy7iPgwnzNvxgAEi2bXOHWCVOG6f7xsprVJH4MjlAWeBmJ7vh/Y1kwMtUrs64ztf24zVIRCpr3n/z6gm9QIkgg==",
+    "node_modules/babel-jest/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
-      "dependencies": {
-        "@types/yargs-parser": "*"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@types/yargs-parser": {
-      "version": "21.0.1",
-      "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.1.tgz",
-      "integrity": "sha512-axdPBuLuEJt0c4yI5OZssC19K2Mq1uKdrfZBzuxLvaztgqUtFYZUNw7lETExPYJR9jdEoIg4mb7RQKRQzOkeGQ==",
-      "dev": true
-    },
-    "node_modules/@types/yauzl": {
-      "version": "2.10.1",
-      "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.1.tgz",
-      "integrity": "sha512-CHzgNU3qYBnp/O4S3yv2tXPlvMTq0YWSTVg2/JYLqWZGHwwgJGAwd00poay/11asPq8wLFwHzubyInqHIFmmiw==",
+    "node_modules/babel-jest/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
-      "optional": true,
       "dependencies": {
-        "@types/node": "*"
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@typescript-eslint/eslint-plugin": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.27.1.tgz",
-      "integrity": "sha512-6dM5NKT57ZduNnJfpY81Phe9nc9wolnMCnknb1im6brWi1RYv84nbMS3olJa27B6+irUVV1X/Wb+Am0FjJdGFw==",
+    "node_modules/babel-loader": {
+      "version": "9.1.2",
+      "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.1.2.tgz",
+      "integrity": "sha512-mN14niXW43tddohGl8HPu5yfQq70iUThvFL/4QzESA7GcZoC0eVOhvWdQ8+3UlSjaDE9MVtsW9mxDY07W7VpVA==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/scope-manager": "5.27.1",
-        "@typescript-eslint/type-utils": "5.27.1",
-        "@typescript-eslint/utils": "5.27.1",
-        "debug": "^4.3.4",
-        "functional-red-black-tree": "^1.0.1",
-        "ignore": "^5.2.0",
-        "regexpp": "^3.2.0",
-        "semver": "^7.3.7",
-        "tsutils": "^3.21.0"
+        "find-cache-dir": "^3.3.2",
+        "schema-utils": "^4.0.0"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+        "node": ">= 14.15.0"
       },
       "peerDependencies": {
-        "@typescript-eslint/parser": "^5.0.0",
-        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
-      },
-      "peerDependenciesMeta": {
-        "typescript": {
-          "optional": true
-        }
+        "@babel/core": "^7.12.0",
+        "webpack": ">=5"
       }
     },
-    "node_modules/@typescript-eslint/eslint-plugin/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/babel-plugin-add-module-exports": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/babel-plugin-add-module-exports/-/babel-plugin-add-module-exports-1.0.4.tgz",
+      "integrity": "sha512-g+8yxHUZ60RcyaUpfNzy56OtWW+x9cyEe9j+CranqLiqbju2yf/Cy6ZtYK40EZxtrdHllzlVZgLmcOUCTlJ7Jg==",
+      "dev": true
+    },
+    "node_modules/babel-plugin-istanbul": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz",
+      "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "@babel/helper-plugin-utils": "^7.0.0",
+        "@istanbuljs/load-nyc-config": "^1.0.0",
+        "@istanbuljs/schema": "^0.1.2",
+        "istanbul-lib-instrument": "^5.0.4",
+        "test-exclude": "^6.0.0"
       },
       "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "node": ">=8"
       }
     },
-    "node_modules/@typescript-eslint/experimental-utils": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-5.27.1.tgz",
-      "integrity": "sha512-Vd8uewIixGP93sEnmTRIH6jHZYRQRkGPDPpapACMvitJKX8335VHNyqKTE+mZ+m3E2c5VznTZfSsSsS5IF7vUA==",
+    "node_modules/babel-plugin-jest-hoist": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz",
+      "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/utils": "5.27.1"
+        "@babel/template": "^7.3.3",
+        "@babel/types": "^7.3.3",
+        "@types/babel__core": "^7.1.14",
+        "@types/babel__traverse": "^7.0.6"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/babel-plugin-polyfill-corejs2": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz",
+      "integrity": "sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q==",
+      "dev": true,
+      "dependencies": {
+        "@babel/compat-data": "^7.17.7",
+        "@babel/helper-define-polyfill-provider": "^0.3.3",
+        "semver": "^6.1.1"
       },
       "peerDependencies": {
-        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@typescript-eslint/parser": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.27.1.tgz",
-      "integrity": "sha512-7Va2ZOkHi5NP+AZwb5ReLgNF6nWLGTeUJfxdkVUAPPSaAdbWNnFZzLZ4EGGmmiCTg+AwlbE1KyUYTBglosSLHQ==",
+    "node_modules/babel-plugin-polyfill-corejs3": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.6.0.tgz",
+      "integrity": "sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/scope-manager": "5.27.1",
-        "@typescript-eslint/types": "5.27.1",
-        "@typescript-eslint/typescript-estree": "5.27.1",
-        "debug": "^4.3.4"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+        "@babel/helper-define-polyfill-provider": "^0.3.3",
+        "core-js-compat": "^3.25.1"
       },
       "peerDependencies": {
-        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
-      },
-      "peerDependenciesMeta": {
-        "typescript": {
-          "optional": true
-        }
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@typescript-eslint/parser/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/babel-plugin-polyfill-regenerator": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.4.1.tgz",
+      "integrity": "sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
-      },
-      "engines": {
-        "node": ">=6.0"
+        "@babel/helper-define-polyfill-provider": "^0.3.3"
       },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
       }
     },
-    "node_modules/@typescript-eslint/scope-manager": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.27.1.tgz",
-      "integrity": "sha512-fQEOSa/QroWE6fAEg+bJxtRZJTH8NTskggybogHt4H9Da8zd4cJji76gA5SBlR0MgtwF7rebxTbDKB49YUCpAg==",
+    "node_modules/babel-preset-current-node-syntax": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz",
+      "integrity": "sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/types": "5.27.1",
-        "@typescript-eslint/visitor-keys": "5.27.1"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+        "@babel/plugin-syntax-async-generators": "^7.8.4",
+        "@babel/plugin-syntax-bigint": "^7.8.3",
+        "@babel/plugin-syntax-class-properties": "^7.8.3",
+        "@babel/plugin-syntax-import-meta": "^7.8.3",
+        "@babel/plugin-syntax-json-strings": "^7.8.3",
+        "@babel/plugin-syntax-logical-assignment-operators": "^7.8.3",
+        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3",
+        "@babel/plugin-syntax-numeric-separator": "^7.8.3",
+        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
+        "@babel/plugin-syntax-optional-chaining": "^7.8.3",
+        "@babel/plugin-syntax-top-level-await": "^7.8.3"
       },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@typescript-eslint/type-utils": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.27.1.tgz",
-      "integrity": "sha512-+UC1vVUWaDHRnC2cQrCJ4QtVjpjjCgjNFpg8b03nERmkHv9JV9X5M19D7UFMd+/G7T/sgFwX2pGmWK38rqyvXw==",
+    "node_modules/babel-preset-jest": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz",
+      "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/utils": "5.27.1",
-        "debug": "^4.3.4",
-        "tsutils": "^3.21.0"
+        "babel-plugin-jest-hoist": "^29.6.3",
+        "babel-preset-current-node-syntax": "^1.0.0"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       },
       "peerDependencies": {
-        "eslint": "*"
-      },
-      "peerDependenciesMeta": {
-        "typescript": {
-          "optional": true
-        }
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@typescript-eslint/type-utils/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/babel-runtime": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
+      "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
-      },
-      "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "core-js": "^2.4.0",
+        "regenerator-runtime": "^0.11.0"
       }
     },
-    "node_modules/@typescript-eslint/types": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.27.1.tgz",
-      "integrity": "sha512-LgogNVkBhCTZU/m8XgEYIWICD6m4dmEDbKXESCbqOXfKZxRKeqpiJXQIErv66sdopRKZPo5l32ymNqibYEH/xg==",
+    "node_modules/babel-runtime/node_modules/core-js": {
+      "version": "2.6.12",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
+      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
+      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
+      "dev": true,
+      "hasInstallScript": true
+    },
+    "node_modules/babel-runtime/node_modules/regenerator-runtime": {
+      "version": "0.11.1",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
+      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==",
+      "dev": true
+    },
+    "node_modules/babelify": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/babelify/-/babelify-10.0.0.tgz",
+      "integrity": "sha512-X40FaxyH7t3X+JFAKvb1H9wooWKLRCi8pg3m8poqtdZaIng+bjzp9RvKQCvRjF9isHiPkXspbbXT/zwXLtwgwg==",
       "dev": true,
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+        "node": ">=6.9.0"
       },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+      "peerDependencies": {
+        "@babel/core": "^7.0.0"
       }
     },
-    "node_modules/@typescript-eslint/typescript-estree": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.27.1.tgz",
-      "integrity": "sha512-DnZvvq3TAJ5ke+hk0LklvxwYsnXpRdqUY5gaVS0D4raKtbznPz71UJGnPTHEFo0GDxqLOLdMkkmVZjSpET1hFw==",
+    "node_modules/bail": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/bail/-/bail-1.0.5.tgz",
+      "integrity": "sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ==",
       "dev": true,
-      "dependencies": {
-        "@typescript-eslint/types": "5.27.1",
-        "@typescript-eslint/visitor-keys": "5.27.1",
-        "debug": "^4.3.4",
-        "globby": "^11.1.0",
-        "is-glob": "^4.0.3",
-        "semver": "^7.3.7",
-        "tsutils": "^3.21.0"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
       "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      },
-      "peerDependenciesMeta": {
-        "typescript": {
-          "optional": true
-        }
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/@typescript-eslint/typescript-estree/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+    },
+    "node_modules/base": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
+      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "cache-base": "^1.0.1",
+        "class-utils": "^0.3.5",
+        "component-emitter": "^1.2.1",
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.1",
+        "mixin-deep": "^1.2.0",
+        "pascalcase": "^0.1.1"
       },
       "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@typescript-eslint/utils": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.27.1.tgz",
-      "integrity": "sha512-mZ9WEn1ZLDaVrhRaYgzbkXBkTPghPFsup8zDbbsYTxC5OmqrFE7skkKS/sraVsLP3TcT3Ki5CSyEFBRkLH/H/w==",
+    "node_modules/base/node_modules/define-property": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+      "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==",
       "dev": true,
       "dependencies": {
-        "@types/json-schema": "^7.0.9",
-        "@typescript-eslint/scope-manager": "5.27.1",
-        "@typescript-eslint/types": "5.27.1",
-        "@typescript-eslint/typescript-estree": "5.27.1",
-        "eslint-scope": "^5.1.1",
-        "eslint-utils": "^3.0.0"
+        "is-descriptor": "^1.0.0"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      },
-      "peerDependencies": {
-        "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/@typescript-eslint/visitor-keys": {
-      "version": "5.27.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.27.1.tgz",
-      "integrity": "sha512-xYs6ffo01nhdJgPieyk7HAOpjhTsx7r/oB9LWEhwAXgwn33tkr+W8DI2ChboqhZlC4q3TC6geDYPoiX8ROqyOQ==",
+    "node_modules/base/node_modules/is-descriptor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.3.tgz",
+      "integrity": "sha512-JCNNGbwWZEVaSPtS45mdtrneRWJFp07LLmykxeFV5F6oBvNF8vHSfJuJgoT472pSfk+Mf8VnlrspaFBHWM8JAw==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/types": "5.27.1",
-        "eslint-visitor-keys": "^3.3.0"
+        "is-accessor-descriptor": "^1.0.1",
+        "is-data-descriptor": "^1.0.1"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
+        "node": ">= 0.4"
       }
     },
-    "node_modules/@webassemblyjs/ast": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz",
-      "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==",
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/basic-auth": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz",
+      "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/helper-numbers": "1.11.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.1"
+        "safe-buffer": "5.1.2"
+      },
+      "engines": {
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@webassemblyjs/floating-point-hex-parser": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz",
-      "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==",
-      "dev": true
-    },
-    "node_modules/@webassemblyjs/helper-api-error": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz",
-      "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==",
+    "node_modules/basic-auth/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
       "dev": true
     },
-    "node_modules/@webassemblyjs/helper-buffer": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz",
-      "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==",
+    "node_modules/batch": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz",
+      "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==",
       "dev": true
     },
-    "node_modules/@webassemblyjs/helper-numbers": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz",
-      "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==",
+    "node_modules/bcrypt-pbkdf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/floating-point-hex-parser": "1.11.1",
-        "@webassemblyjs/helper-api-error": "1.11.1",
-        "@xtuc/long": "4.2.2"
+        "tweetnacl": "^0.14.3"
       }
     },
-    "node_modules/@webassemblyjs/helper-wasm-bytecode": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz",
-      "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==",
+    "node_modules/bcryptjs": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz",
+      "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==",
       "dev": true
     },
-    "node_modules/@webassemblyjs/helper-wasm-section": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz",
-      "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==",
+    "node_modules/becke-ch--regex--s0-0-v1--base--pl--lib": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/becke-ch--regex--s0-0-v1--base--pl--lib/-/becke-ch--regex--s0-0-v1--base--pl--lib-1.4.0.tgz",
+      "integrity": "sha512-FnWonOyaw7Vivg5nIkrUll9HSS5TjFbyuURAiDssuL6VxrBe3ERzudRxOcWRhZYlP89UArMDikz7SapRPQpmZQ==",
+      "dev": true
+    },
+    "node_modules/big.js": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
+      "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==",
       "dev": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@webassemblyjs/helper-buffer": "1.11.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
-        "@webassemblyjs/wasm-gen": "1.11.1"
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/@webassemblyjs/ieee754": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz",
-      "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==",
+    "node_modules/binary-extensions": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
+      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
       "dev": true,
-      "dependencies": {
-        "@xtuc/ieee754": "^1.2.0"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/@webassemblyjs/leb128": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz",
-      "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==",
-      "dev": true,
+    "node_modules/bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "devOptional": true,
       "dependencies": {
-        "@xtuc/long": "4.2.2"
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
       }
     },
-    "node_modules/@webassemblyjs/utf8": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz",
-      "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==",
+    "node_modules/blob-util": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz",
+      "integrity": "sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==",
       "dev": true
     },
-    "node_modules/@webassemblyjs/wasm-edit": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz",
-      "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==",
+    "node_modules/bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
+      "dev": true
+    },
+    "node_modules/bn.js": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz",
+      "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==",
+      "dev": true
+    },
+    "node_modules/body-parser": {
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@webassemblyjs/helper-buffer": "1.11.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
-        "@webassemblyjs/helper-wasm-section": "1.11.1",
-        "@webassemblyjs/wasm-gen": "1.11.1",
-        "@webassemblyjs/wasm-opt": "1.11.1",
-        "@webassemblyjs/wasm-parser": "1.11.1",
-        "@webassemblyjs/wast-printer": "1.11.1"
+        "bytes": "3.1.2",
+        "content-type": "~1.0.4",
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "http-errors": "2.0.0",
+        "iconv-lite": "0.4.24",
+        "on-finished": "2.4.1",
+        "qs": "6.11.0",
+        "raw-body": "2.5.1",
+        "type-is": "~1.6.18",
+        "unpipe": "1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
       }
     },
-    "node_modules/@webassemblyjs/wasm-gen": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz",
-      "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==",
+    "node_modules/body-parser/node_modules/bytes": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
       "dev": true,
-      "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
-        "@webassemblyjs/ieee754": "1.11.1",
-        "@webassemblyjs/leb128": "1.11.1",
-        "@webassemblyjs/utf8": "1.11.1"
+      "engines": {
+        "node": ">= 0.8"
       }
     },
-    "node_modules/@webassemblyjs/wasm-opt": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
-      "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
+    "node_modules/body-parser/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@webassemblyjs/helper-buffer": "1.11.1",
-        "@webassemblyjs/wasm-gen": "1.11.1",
-        "@webassemblyjs/wasm-parser": "1.11.1"
+        "ms": "2.0.0"
       }
     },
-    "node_modules/@webassemblyjs/wasm-parser": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz",
-      "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==",
+    "node_modules/body-parser/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/body-parser/node_modules/qs": {
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@webassemblyjs/helper-api-error": "1.11.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
-        "@webassemblyjs/ieee754": "1.11.1",
-        "@webassemblyjs/leb128": "1.11.1",
-        "@webassemblyjs/utf8": "1.11.1"
+        "side-channel": "^1.0.4"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@webassemblyjs/wast-printer": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz",
-      "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==",
+    "node_modules/bonjour-service": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.1.1.tgz",
+      "integrity": "sha512-Z/5lQRMOG9k7W+FkeGTNjh7htqn/2LMnfOvBZ8pynNZCM9MwkQkI3zeI4oz09uWdcgmgHugVvBqxGg4VQJ5PCg==",
       "dev": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.11.1",
-        "@xtuc/long": "4.2.2"
+        "array-flatten": "^2.1.2",
+        "dns-equal": "^1.0.0",
+        "fast-deep-equal": "^3.1.3",
+        "multicast-dns": "^7.2.5"
       }
     },
-    "node_modules/@xtuc/ieee754": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
-      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
+    "node_modules/boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
       "dev": true
     },
-    "node_modules/@xtuc/long": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz",
-      "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==",
-      "dev": true
+    "node_modules/bootstrap": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.2.3.tgz",
+      "integrity": "sha512-cEKPM+fwb3cT8NzQZYEu4HilJ3anCrWqh3CHAok1p9jXqMPsPTBhU25fBckEJHJ/p+tTxTFTsFQGM+gaHpi3QQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/twbs"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/bootstrap"
+        }
+      ],
+      "peerDependencies": {
+        "@popperjs/core": "^2.11.6"
+      }
     },
-    "node_modules/@yarnpkg/lockfile": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz",
-      "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==",
-      "dev": true
+    "node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
     },
-    "node_modules/@yarnpkg/parsers": {
-      "version": "3.0.0-rc.51",
-      "resolved": "https://registry.npmjs.org/@yarnpkg/parsers/-/parsers-3.0.0-rc.51.tgz",
-      "integrity": "sha512-E3sHAnUcyvGbXDCY1YJTVTwovF1UFP8IVJ6CFGgTST8RGD2gRQqqmW8dZykpaDWKqO6RBqwJcfmJMyqYBBpixw==",
+    "node_modules/braces": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
       "dev": true,
       "dependencies": {
-        "js-yaml": "^3.10.0",
-        "tslib": "^2.4.0"
+        "fill-range": "^7.0.1"
       },
       "engines": {
-        "node": ">=18.12.0"
+        "node": ">=8"
       }
     },
-    "node_modules/@yarnpkg/parsers/node_modules/tslib": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
-      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==",
-      "dev": true
-    },
-    "node_modules/@zkochan/js-yaml": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/@zkochan/js-yaml/-/js-yaml-0.0.6.tgz",
-      "integrity": "sha512-nzvgl3VfhcELQ8LyVrYOru+UtAy1nrygk2+AGbTm8a5YcO6o8lSjAT+pfg3vJWxIoZKOUhrK6UU7xW/+00kQrg==",
+    "node_modules/brfs": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brfs/-/brfs-2.0.2.tgz",
+      "integrity": "sha512-IrFjVtwu4eTJZyu8w/V2gxU7iLTtcHih67sgEdzrhjLBMHp2uYefUBfdM4k2UvcuWMgV7PQDZHSLeNWnLFKWVQ==",
       "dev": true,
       "dependencies": {
-        "argparse": "^2.0.1"
+        "quote-stream": "^1.0.1",
+        "resolve": "^1.1.5",
+        "static-module": "^3.0.2",
+        "through2": "^2.0.0"
       },
       "bin": {
-        "js-yaml": "bin/js-yaml.js"
+        "brfs": "bin/cmd.js"
       }
     },
-    "node_modules/@zkochan/js-yaml/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
-    },
-    "node_modules/abab": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
-      "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
-      "dev": true
-    },
-    "node_modules/abbrev": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
-      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+    "node_modules/brorand": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
+      "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==",
       "dev": true
     },
-    "node_modules/abort-controller": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
-      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+    "node_modules/brotli": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz",
+      "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==",
       "dev": true,
       "dependencies": {
-        "event-target-shim": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=6.5"
+        "base64-js": "^1.1.2"
       }
     },
-    "node_modules/accepts": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
-      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
+    "node_modules/browser-pack": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz",
+      "integrity": "sha512-erYug8XoqzU3IfcU8fUgyHqyOXqIE4tUTTQ+7mqUjQlvnXkOO6OlT9c/ZoJVHYoAaqGxr09CN53G7XIsO4KtWA==",
       "dev": true,
       "dependencies": {
-        "mime-types": "~2.1.34",
-        "negotiator": "0.6.3"
+        "combine-source-map": "~0.8.0",
+        "defined": "^1.0.0",
+        "JSONStream": "^1.0.3",
+        "safe-buffer": "^5.1.1",
+        "through2": "^2.0.0",
+        "umd": "^3.0.0"
       },
-      "engines": {
-        "node": ">= 0.6"
+      "bin": {
+        "browser-pack": "bin/cmd.js"
       }
     },
-    "node_modules/accepts/node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
+    "node_modules/browser-process-hrtime": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
+      "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
+      "dev": true
     },
-    "node_modules/accepts/node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+    "node_modules/browser-resolve": {
+      "version": "1.11.3",
+      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.3.tgz",
+      "integrity": "sha512-exDi1BYWB/6raKHmDTCicQfTkqwN5fioMFV4j8BsfMU4R2DK/QfZfK7kOVkmWCNANf0snkBzqGqAJBao9gZMdQ==",
       "dev": true,
       "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
+        "resolve": "1.1.7"
       }
     },
-    "node_modules/acorn": {
-      "version": "7.4.1",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
-      "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
-      "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
+    "node_modules/browser-resolve/node_modules/resolve": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz",
+      "integrity": "sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg==",
+      "dev": true
     },
-    "node_modules/acorn-globals": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz",
-      "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==",
+    "node_modules/browserify": {
+      "version": "16.5.2",
+      "resolved": "https://registry.npmjs.org/browserify/-/browserify-16.5.2.tgz",
+      "integrity": "sha512-TkOR1cQGdmXU9zW4YukWzWVSJwrxmNdADFbqbE3HFgQWe5wqZmOawqZ7J/8MPCwk/W8yY7Y0h+7mOtcZxLP23g==",
       "dev": true,
       "dependencies": {
-        "acorn": "^6.0.1",
-        "acorn-walk": "^6.0.1"
-      }
-    },
-    "node_modules/acorn-globals/node_modules/acorn": {
-      "version": "6.4.2",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz",
-      "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==",
-      "dev": true,
+        "assert": "^1.4.0",
+        "browser-pack": "^6.0.1",
+        "browser-resolve": "^2.0.0",
+        "browserify-zlib": "~0.2.0",
+        "buffer": "~5.2.1",
+        "cached-path-relative": "^1.0.0",
+        "concat-stream": "^1.6.0",
+        "console-browserify": "^1.1.0",
+        "constants-browserify": "~1.0.0",
+        "crypto-browserify": "^3.0.0",
+        "defined": "^1.0.0",
+        "deps-sort": "^2.0.0",
+        "domain-browser": "^1.2.0",
+        "duplexer2": "~0.1.2",
+        "events": "^2.0.0",
+        "glob": "^7.1.0",
+        "has": "^1.0.0",
+        "htmlescape": "^1.1.0",
+        "https-browserify": "^1.0.0",
+        "inherits": "~2.0.1",
+        "insert-module-globals": "^7.0.0",
+        "JSONStream": "^1.0.3",
+        "labeled-stream-splicer": "^2.0.0",
+        "mkdirp-classic": "^0.5.2",
+        "module-deps": "^6.2.3",
+        "os-browserify": "~0.3.0",
+        "parents": "^1.0.1",
+        "path-browserify": "~0.0.0",
+        "process": "~0.11.0",
+        "punycode": "^1.3.2",
+        "querystring-es3": "~0.2.0",
+        "read-only-stream": "^2.0.0",
+        "readable-stream": "^2.0.2",
+        "resolve": "^1.1.4",
+        "shasum": "^1.0.0",
+        "shell-quote": "^1.6.1",
+        "stream-browserify": "^2.0.0",
+        "stream-http": "^3.0.0",
+        "string_decoder": "^1.1.1",
+        "subarg": "^1.0.0",
+        "syntax-error": "^1.1.1",
+        "through2": "^2.0.0",
+        "timers-browserify": "^1.0.1",
+        "tty-browserify": "0.0.1",
+        "url": "~0.11.0",
+        "util": "~0.10.1",
+        "vm-browserify": "^1.0.0",
+        "xtend": "^4.0.0"
+      },
       "bin": {
-        "acorn": "bin/acorn"
+        "browserify": "bin/cmd.js"
       },
       "engines": {
-        "node": ">=0.4.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/acorn-import-assertions": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz",
-      "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==",
+    "node_modules/browserify-aes": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
+      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
       "dev": true,
-      "peerDependencies": {
-        "acorn": "^8"
+      "dependencies": {
+        "buffer-xor": "^1.0.3",
+        "cipher-base": "^1.0.0",
+        "create-hash": "^1.1.0",
+        "evp_bytestokey": "^1.0.3",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
       }
     },
-    "node_modules/acorn-jsx": {
-      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
-      "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
+    "node_modules/browserify-cipher": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
+      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
       "dev": true,
-      "peerDependencies": {
-        "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
+      "dependencies": {
+        "browserify-aes": "^1.0.4",
+        "browserify-des": "^1.0.0",
+        "evp_bytestokey": "^1.0.0"
       }
     },
-    "node_modules/acorn-node": {
-      "version": "1.8.2",
-      "resolved": "https://registry.npmjs.org/acorn-node/-/acorn-node-1.8.2.tgz",
-      "integrity": "sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A==",
+    "node_modules/browserify-des": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz",
+      "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==",
       "dev": true,
       "dependencies": {
-        "acorn": "^7.0.0",
-        "acorn-walk": "^7.0.0",
-        "xtend": "^4.0.2"
+        "cipher-base": "^1.0.1",
+        "des.js": "^1.0.0",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.1.2"
       }
     },
-    "node_modules/acorn-node/node_modules/acorn-walk": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",
-      "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==",
+    "node_modules/browserify-optional": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-optional/-/browserify-optional-1.0.1.tgz",
+      "integrity": "sha512-VrhjbZ+Ba5mDiSYEuPelekQMfTbhcA2DhLk2VQWqdcCROWeFqlTcXZ7yfRkXCIl8E+g4gINJYJiRB7WEtfomAQ==",
       "dev": true,
-      "engines": {
-        "node": ">=0.4.0"
+      "dependencies": {
+        "ast-transform": "0.0.0",
+        "ast-types": "^0.7.0",
+        "browser-resolve": "^1.8.1"
       }
     },
-    "node_modules/acorn-walk": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz",
-      "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==",
+    "node_modules/browserify-rsa": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz",
+      "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==",
       "dev": true,
-      "engines": {
-        "node": ">=0.4.0"
+      "dependencies": {
+        "bn.js": "^5.0.0",
+        "randombytes": "^2.0.1"
       }
     },
-    "node_modules/adjust-sourcemap-loader": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz",
-      "integrity": "sha512-OXwN5b9pCUXNQHJpwwD2qP40byEmSgzj8B4ydSN0uMNYWiFmJ6x6KwUllMmfk8Rwu/HJDFR7U8ubsWBoN0Xp0A==",
+    "node_modules/browserify-sign": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.2.tgz",
+      "integrity": "sha512-1rudGyeYY42Dk6texmv7c4VcQ0EsvVbLwZkA+AQB7SxvXxmcD93jcHie8bzecJ+ChDlmAm2Qyu0+Ccg5uhZXCg==",
       "dev": true,
       "dependencies": {
-        "loader-utils": "^2.0.0",
-        "regex-parser": "^2.2.11"
+        "bn.js": "^5.2.1",
+        "browserify-rsa": "^4.1.0",
+        "create-hash": "^1.2.0",
+        "create-hmac": "^1.1.7",
+        "elliptic": "^6.5.4",
+        "inherits": "^2.0.4",
+        "parse-asn1": "^5.1.6",
+        "readable-stream": "^3.6.2",
+        "safe-buffer": "^5.2.1"
       },
       "engines": {
-        "node": ">=8.9"
+        "node": ">= 4"
       }
     },
-    "node_modules/adjust-sourcemap-loader/node_modules/loader-utils": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz",
-      "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==",
+    "node_modules/browserify-zlib": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
+      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
       "dev": true,
       "dependencies": {
-        "big.js": "^5.2.2",
-        "emojis-list": "^3.0.0",
-        "json5": "^2.1.2"
-      },
-      "engines": {
-        "node": ">=8.9.0"
+        "pako": "~1.0.5"
       }
     },
-    "node_modules/agent-base": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
-      "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+    "node_modules/browserify/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "debug": "4"
-      },
-      "engines": {
-        "node": ">= 6.0.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/agentkeepalive": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
-      "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
+    "node_modules/browserify/node_modules/browser-resolve": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz",
+      "integrity": "sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==",
       "dev": true,
       "dependencies": {
-        "humanize-ms": "^1.2.1"
-      },
-      "engines": {
-        "node": ">= 8.0.0"
+        "resolve": "^1.17.0"
       }
     },
-    "node_modules/aggregate-error": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
-      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+    "node_modules/browserify/node_modules/buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz",
+      "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==",
       "dev": true,
       "dependencies": {
-        "clean-stack": "^2.0.0",
-        "indent-string": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=8"
+        "base64-js": "^1.0.2",
+        "ieee754": "^1.1.4"
       }
     },
-    "node_modules/aggregate-error/node_modules/indent-string": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
-      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+    "node_modules/browserify/node_modules/events": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-2.1.0.tgz",
+      "integrity": "sha512-3Zmiobend8P9DjmKAty0Era4jV8oJ0yGYe2nJJAxgymF9+N8F2m0hhZiMoWtcfepExzNKZumFU3ksdQbInGWCg==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=0.4.x"
       }
     },
-    "node_modules/ajv": {
-      "version": "8.12.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
-      "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
+    "node_modules/browserify/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "json-schema-traverse": "^1.0.0",
-        "require-from-string": "^2.0.2",
-        "uri-js": "^4.2.2"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
       },
       "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/ajv-formats": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
-      "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
+    "node_modules/browserify/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "ajv": "^8.0.0"
-      },
-      "peerDependencies": {
-        "ajv": "^8.0.0"
+        "brace-expansion": "^1.1.7"
       },
-      "peerDependenciesMeta": {
-        "ajv": {
-          "optional": true
-        }
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/ajv-keywords": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz",
-      "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==",
+    "node_modules/browserify/node_modules/path-browserify": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz",
+      "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==",
+      "dev": true
+    },
+    "node_modules/browserify/node_modules/punycode": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+      "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==",
+      "dev": true
+    },
+    "node_modules/browserify/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
       "dev": true,
       "dependencies": {
-        "fast-deep-equal": "^3.1.3"
-      },
-      "peerDependencies": {
-        "ajv": "^8.8.2"
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
       }
     },
-    "node_modules/amdefine": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz",
-      "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==",
+    "node_modules/browserify/node_modules/readable-stream/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
       "dev": true,
-      "optional": true,
-      "engines": {
-        "node": ">=0.4.2"
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/ansi-colors": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz",
-      "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
+    "node_modules/browserify/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
     },
-    "node_modules/ansi-escapes": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
-      "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==",
+    "node_modules/browserify/node_modules/stream-browserify": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
+      "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==",
       "dev": true,
       "dependencies": {
-        "type-fest": "^0.21.3"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "inherits": "~2.0.1",
+        "readable-stream": "^2.0.2"
       }
     },
-    "node_modules/ansi-html-community": {
-      "version": "0.0.8",
-      "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz",
-      "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==",
+    "node_modules/browserify/node_modules/timers-browserify": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz",
+      "integrity": "sha512-PIxwAupJZiYU4JmVZYwXp9FKsHMXb5h0ZEFyuXTAn8WLHOlcij+FEcbrvDsom1o5dr1YggEtFbECvGCW2sT53Q==",
       "dev": true,
-      "engines": [
-        "node >= 0.8.0"
-      ],
-      "bin": {
-        "ansi-html": "bin/ansi-html"
-      }
-    },
-    "node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dependencies": {
-        "color-convert": "^2.0.1"
+        "process": "~0.11.0"
       },
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+        "node": ">=0.6.0"
       }
     },
-    "node_modules/any-promise": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==",
-      "dev": true
-    },
-    "node_modules/anymatch": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
-      "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
-      "dev": true,
+    "node_modules/browserslist": {
+      "version": "4.22.1",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.22.1.tgz",
+      "integrity": "sha512-FEVc202+2iuClEhZhrWy6ZiAcRLvNMyYcxZ8raemul1DYVOVdFsbqckWLdsixQZCpJlwe77Z3UTalE7jsjnKfQ==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/browserslist"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
       "dependencies": {
-        "normalize-path": "^3.0.0",
-        "picomatch": "^2.0.4"
+        "caniuse-lite": "^1.0.30001541",
+        "electron-to-chromium": "^1.4.535",
+        "node-releases": "^2.0.13",
+        "update-browserslist-db": "^1.0.13"
+      },
+      "bin": {
+        "browserslist": "cli.js"
       },
       "engines": {
-        "node": ">= 8"
+        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
-    "node_modules/apache-crypt": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/apache-crypt/-/apache-crypt-1.2.6.tgz",
-      "integrity": "sha512-072WetlM4blL8PREJVeY+WHiUh1R5VNt2HfceGS8aKqttPHcmqE5pkKuXPz/ULmJOFkc8Hw3kfKl6vy7Qka6DA==",
+    "node_modules/bs-logger": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz",
+      "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==",
       "dev": true,
       "dependencies": {
-        "unix-crypt-td-js": "^1.1.4"
+        "fast-json-stable-stringify": "2.x"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">= 6"
       }
     },
-    "node_modules/apache-md5": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/apache-md5/-/apache-md5-1.1.8.tgz",
-      "integrity": "sha512-FCAJojipPn0bXjuEpjOOOMN8FZDkxfWWp4JGN9mifU2IhxvKyXZYqpzPHdnTSUpmPDy+tsslB6Z1g+Vg6nVbYA==",
+    "node_modules/bser": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz",
+      "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==",
       "dev": true,
-      "engines": {
-        "node": ">=8"
+      "dependencies": {
+        "node-int64": "^0.4.0"
       }
     },
-    "node_modules/aproba": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
-      "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
-      "dev": true
-    },
-    "node_modules/arch": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz",
-      "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==",
-      "dev": true,
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "devOptional": true,
       "funding": [
         {
           "type": "github",
@@ -7853,930 +10849,1109 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
     },
-    "node_modules/are-we-there-yet": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz",
-      "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==",
+    "node_modules/buffer-crc32": {
+      "version": "0.2.13",
+      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
+      "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==",
+      "dev": true,
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/buffer-equal": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz",
+      "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "dev": true
+    },
+    "node_modules/buffer-xor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
+      "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==",
+      "dev": true
+    },
+    "node_modules/builtin-status-codes": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
+      "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==",
+      "dev": true
+    },
+    "node_modules/builtins": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz",
+      "integrity": "sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==",
       "dev": true,
       "dependencies": {
-        "delegates": "^1.0.0",
-        "readable-stream": "^3.6.0"
+        "semver": "^7.0.0"
+      }
+    },
+    "node_modules/builtins/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+        "node": ">=10"
       }
     },
-    "node_modules/are-we-there-yet/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+    "node_modules/builtins/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=10"
       }
     },
-    "node_modules/arg": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
-      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+    "node_modules/builtins/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "dev": true
     },
-    "node_modules/argparse": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
-      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+    "node_modules/bulk-require": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/bulk-require/-/bulk-require-1.0.1.tgz",
+      "integrity": "sha512-BLU9AMnm1FMr68fR0sYvOkMew4x2ZJ8YztshITlGArl6aLtfAazOWiFj/bwJJixRO8C0wXx9PnRNRWeoR03e8Q==",
+      "dev": true,
       "dependencies": {
-        "sprintf-js": "~1.0.2"
+        "glob": "^7.1.1"
       }
     },
-    "node_modules/aria-query": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz",
-      "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==",
+    "node_modules/bulk-require/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "@babel/runtime": "^7.10.2",
-        "@babel/runtime-corejs3": "^7.10.2"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/bulk-require/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">=6.0"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/arr-diff": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-      "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==",
+    "node_modules/bulk-require/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/bytes": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
+      "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/arr-flatten": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
-      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
+    "node_modules/cacache": {
+      "version": "17.0.4",
+      "resolved": "https://registry.npmjs.org/cacache/-/cacache-17.0.4.tgz",
+      "integrity": "sha512-Z/nL3gU+zTUjz5pCA5vVjYM8pmaw2kxM7JEiE0fv3w77Wj+sFbi70CrBruUWH0uNcEdvLDixFpgA2JM4F4DBjA==",
+      "dev": true,
+      "dependencies": {
+        "@npmcli/fs": "^3.1.0",
+        "fs-minipass": "^3.0.0",
+        "glob": "^8.0.1",
+        "lru-cache": "^7.7.1",
+        "minipass": "^4.0.0",
+        "minipass-collect": "^1.0.2",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "p-map": "^4.0.0",
+        "promise-inflight": "^1.0.1",
+        "ssri": "^10.0.0",
+        "tar": "^6.1.11",
+        "unique-filename": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/cacache/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/cache-base": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
+      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
       "dev": true,
+      "dependencies": {
+        "collection-visit": "^1.0.0",
+        "component-emitter": "^1.2.1",
+        "get-value": "^2.0.6",
+        "has-value": "^1.0.0",
+        "isobject": "^3.0.1",
+        "set-value": "^2.0.0",
+        "to-object-path": "^0.3.0",
+        "union-value": "^1.0.0",
+        "unset-value": "^1.0.0"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/arr-union": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
-      "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==",
+    "node_modules/cached-path-relative": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.1.0.tgz",
+      "integrity": "sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==",
+      "dev": true
+    },
+    "node_modules/cachedir": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz",
+      "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=6"
       }
     },
-    "node_modules/array-buffer-byte-length": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz",
-      "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==",
-      "dev": true,
+    "node_modules/call-bind": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz",
+      "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==",
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "is-array-buffer": "^3.0.1"
+        "function-bind": "^1.1.2",
+        "get-intrinsic": "^1.2.1",
+        "set-function-length": "^1.1.1"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/array-differ": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz",
-      "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==",
+    "node_modules/callsite": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
+      "integrity": "sha512-0vdNRFXn5q+dtOqjfFtmtlI9N2eVZ7LMyEV2iKC5mEEFvSg/69Ml6b/WU2qF8W1nLRa0wiSrDT3Y5jOHZCwKPQ==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": "*"
       }
     },
-    "node_modules/array-each": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz",
-      "integrity": "sha512-zHjL5SZa68hkKHBFBK6DJCTtr9sfTCPCaph/L7tMSLcTFgy+zX7E+6q5UArbtOtMBCtxdICpfTCspRse+ywyXA==",
+    "node_modules/callsites": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=6"
       }
     },
-    "node_modules/array-equal": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
-      "integrity": "sha512-H3LU5RLiSsGXPhN+Nipar0iR0IofH+8r89G2y1tBKxQ/agagKyAjhkAFDRBfodP2caPrNKHpAWNIM/c9yeL7uA==",
-      "dev": true
-    },
-    "node_modules/array-flatten": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
-      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
-      "dev": true
-    },
-    "node_modules/array-from": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz",
-      "integrity": "sha512-GQTc6Uupx1FCavi5mPzBvVT7nEOeWMmUA9P95wpfpW1XwMSKs+KaymD5C2Up7KAUKg/mYwbsUYzdZWcoajlNZg==",
-      "dev": true
-    },
-    "node_modules/array-slice": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
-      "integrity": "sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w==",
+    "node_modules/camelcase": {
+      "version": "5.3.1",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
+      "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=6"
       }
     },
-    "node_modules/array-union": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
-      "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
+    "node_modules/camelcase-keys": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz",
+      "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==",
       "dev": true,
+      "dependencies": {
+        "camelcase": "^5.3.1",
+        "map-obj": "^4.0.0",
+        "quick-lru": "^4.0.1"
+      },
       "engines": {
         "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/array-unique": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-      "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==",
+    "node_modules/can-use-dom": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/can-use-dom/-/can-use-dom-0.1.0.tgz",
+      "integrity": "sha512-ceOhN1DL7Y4O6M0j9ICgmTYziV89WMd96SvSl0REd8PMgrY0B/WBOPoed5S1KUmJqXgUXh8gzSe6E3ae27upsQ=="
+    },
+    "node_modules/caniuse-lite": {
+      "version": "1.0.30001563",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001563.tgz",
+      "integrity": "sha512-na2WUmOxnwIZtwnFI2CZ/3er0wdNzU7hN+cPYz/z2ajHThnkWjNBOpEPP4n+4r2WPM847JaMotaJE3bnfzjyKw==",
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ]
+    },
+    "node_modules/caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
+      "dev": true
+    },
+    "node_modules/chai": {
+      "version": "4.3.10",
+      "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.10.tgz",
+      "integrity": "sha512-0UXG04VuVbruMUYbJ6JctvH0YnC/4q3/AkT18q4NaITo91CUm0liMS9VqzT9vZhVQ/1eqPanMWjBM+Juhfb/9g==",
       "dev": true,
+      "dependencies": {
+        "assertion-error": "^1.1.0",
+        "check-error": "^1.0.3",
+        "deep-eql": "^4.1.3",
+        "get-func-name": "^2.0.2",
+        "loupe": "^2.3.6",
+        "pathval": "^1.1.1",
+        "type-detect": "^4.0.8"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4"
       }
     },
-    "node_modules/arraybuffer.prototype.slice": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz",
-      "integrity": "sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==",
-      "dev": true,
+    "node_modules/chalk": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
       "dependencies": {
-        "array-buffer-byte-length": "^1.0.0",
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.2.0",
-        "es-abstract": "^1.22.1",
-        "get-intrinsic": "^1.2.1",
-        "is-array-buffer": "^3.0.2",
-        "is-shared-array-buffer": "^1.0.2"
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=4"
       }
     },
-    "node_modules/arrify": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
-      "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==",
+    "node_modules/char-regex": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz",
+      "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
       }
     },
-    "node_modules/asap": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
+    "node_modules/character-entities": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz",
+      "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
+    "node_modules/character-entities-legacy": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz",
+      "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
+    "node_modules/character-reference-invalid": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz",
+      "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
+    "node_modules/chardet": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
+      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
       "dev": true
     },
-    "node_modules/asn1": {
-      "version": "0.2.6",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
-      "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
       "dev": true,
-      "dependencies": {
-        "safer-buffer": "~2.1.0"
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/asn1.js": {
-      "version": "5.4.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
-      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
-      "dev": true,
+    "node_modules/chart.js": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.0.tgz",
+      "integrity": "sha512-vQEj6d+z0dcsKLlQvbKIMYFHd3t8W/7L2vfJIbYcfyPcRx92CsHqECpueN8qVGNlKyDcr5wBrYAYKnfu/9Q1hQ==",
       "dependencies": {
-        "bn.js": "^4.0.0",
-        "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0",
-        "safer-buffer": "^2.1.0"
+        "@kurkle/color": "^0.3.0"
+      },
+      "engines": {
+        "pnpm": ">=7"
       }
     },
-    "node_modules/asn1.js/node_modules/bn.js": {
-      "version": "4.12.0",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
-      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
-      "dev": true
+    "node_modules/chartjs-adapter-moment": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/chartjs-adapter-moment/-/chartjs-adapter-moment-1.0.1.tgz",
+      "integrity": "sha512-Uz+nTX/GxocuqXpGylxK19YG4R3OSVf8326D+HwSTsNw1LgzyIGRo+Qujwro1wy6X+soNSnfj5t2vZ+r6EaDmA==",
+      "peerDependencies": {
+        "chart.js": ">=3.0.0",
+        "moment": "^2.10.2"
+      }
     },
-    "node_modules/assert": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.1.tgz",
-      "integrity": "sha512-zzw1uCAgLbsKwBfFc8CX78DDg+xZeBksSO3vwVIDDN5i94eOrPsSSyiVhmsSABFDM/OcpE2aagCat9dnWQLG1A==",
+    "node_modules/check-error": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz",
+      "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==",
       "dev": true,
       "dependencies": {
-        "object.assign": "^4.1.4",
-        "util": "^0.10.4"
+        "get-func-name": "^2.0.2"
+      },
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/assert-plus": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-      "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==",
+    "node_modules/check-more-types": {
+      "version": "2.24.0",
+      "resolved": "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz",
+      "integrity": "sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA==",
       "dev": true,
       "engines": {
-        "node": ">=0.8"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/assertion-error": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
-      "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
+    "node_modules/cheerio": {
+      "version": "1.0.0-rc.12",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz",
+      "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==",
       "dev": true,
+      "dependencies": {
+        "cheerio-select": "^2.1.0",
+        "dom-serializer": "^2.0.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1",
+        "htmlparser2": "^8.0.1",
+        "parse5": "^7.0.0",
+        "parse5-htmlparser2-tree-adapter": "^7.0.0"
+      },
       "engines": {
-        "node": "*"
+        "node": ">= 6"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
       }
     },
-    "node_modules/assertion-error-formatter": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/assertion-error-formatter/-/assertion-error-formatter-2.0.1.tgz",
-      "integrity": "sha512-cjC3jUCh9spkroKue5PDSKH5RFQ/KNuZJhk3GwHYmB/8qqETxLOmMdLH+ohi/VukNzxDlMvIe7zScvLoOdhb6Q==",
+    "node_modules/cheerio-select": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+      "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
       "dev": true,
       "dependencies": {
-        "diff": "^3.0.0",
-        "pad-right": "^0.2.2",
-        "repeat-string": "^1.6.1"
+        "boolbase": "^1.0.0",
+        "css-select": "^5.1.0",
+        "css-what": "^6.1.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/assign-symbols": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
-      "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==",
+    "node_modules/cheerio-select/node_modules/css-select": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
+      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
       "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+      "dependencies": {
+        "boolbase": "^1.0.0",
+        "css-what": "^6.1.0",
+        "domhandler": "^5.0.2",
+        "domutils": "^3.0.1",
+        "nth-check": "^2.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/ast-transform": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/ast-transform/-/ast-transform-0.0.0.tgz",
-      "integrity": "sha512-e/JfLiSoakfmL4wmTGPjv0HpTICVmxwXgYOB8x+mzozHL8v+dSfCbrJ8J8hJ0YBP0XcYu1aLZ6b/3TnxNK3P2A==",
+    "node_modules/cheerio-select/node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
       "dev": true,
       "dependencies": {
-        "escodegen": "~1.2.0",
-        "esprima": "~1.0.4",
-        "through": "~2.3.4"
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
+      },
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
       }
     },
-    "node_modules/ast-transform/node_modules/escodegen": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.2.0.tgz",
-      "integrity": "sha512-yLy3Cc+zAC0WSmoT2fig3J87TpQ8UaZGx8ahCAs9FL8qNbyV7CVyPKS74DG4bsHiL5ew9sxdYx131OkBQMFnvA==",
+    "node_modules/cheerio-select/node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
       "dev": true,
       "dependencies": {
-        "esprima": "~1.0.4",
-        "estraverse": "~1.5.0",
-        "esutils": "~1.0.0"
-      },
-      "bin": {
-        "escodegen": "bin/escodegen.js",
-        "esgenerate": "bin/esgenerate.js"
+        "domelementtype": "^2.3.0"
       },
       "engines": {
-        "node": ">=0.4.0"
+        "node": ">= 4"
       },
-      "optionalDependencies": {
-        "source-map": "~0.1.30"
+      "funding": {
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
       }
     },
-    "node_modules/ast-transform/node_modules/esprima": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz",
-      "integrity": "sha512-rp5dMKN8zEs9dfi9g0X1ClLmV//WRyk/R15mppFNICIFRG5P92VP7Z04p8pk++gABo9W2tY+kHyu6P1mEHgmTA==",
+    "node_modules/cheerio-select/node_modules/domutils": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
+      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
       "dev": true,
-      "bin": {
-        "esparse": "bin/esparse.js",
-        "esvalidate": "bin/esvalidate.js"
+      "dependencies": {
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
       },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
-    "node_modules/ast-transform/node_modules/estraverse": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.5.1.tgz",
-      "integrity": "sha512-FpCjJDfmo3vsc/1zKSeqR5k42tcIhxFIlvq+h9j0fO2q/h2uLKyweq7rYJ+0CoVvrGQOxIS5wyBrW/+vF58BUQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.4.0"
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
-    "node_modules/ast-transform/node_modules/esutils": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/esutils/-/esutils-1.0.0.tgz",
-      "integrity": "sha512-x/iYH53X3quDwfHRz4y8rn4XcEwwCJeWsul9pF1zldMbGtgOtMNBEOuYWwB1EQlK2LRa1fev3YAgym/RElp5Cg==",
+    "node_modules/cheerio-select/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/ast-transform/node_modules/source-map": {
-      "version": "0.1.43",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz",
-      "integrity": "sha512-VtCvB9SIQhk3aF6h+N85EaqIaBFIAfZ9Cu+NJHHVvc8BbEcnvDcFw6sqQ2dQrT6SlOrZq3tIvyD9+EGq/lJryQ==",
+    "node_modules/cheerio/node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
       "dev": true,
-      "optional": true,
       "dependencies": {
-        "amdefine": ">=0.0.4"
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
       },
-      "engines": {
-        "node": ">=0.8.0"
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
       }
     },
-    "node_modules/ast-types": {
-      "version": "0.7.8",
-      "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.7.8.tgz",
-      "integrity": "sha512-RIOpVnVlltB6PcBJ5BMLx+H+6JJ/zjDGU0t7f0L6c2M1dqcK92VQopLBlPQ9R80AVXelfqYgjcPLtHtDbNFg0Q==",
+    "node_modules/cheerio/node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
       "dev": true,
+      "dependencies": {
+        "domelementtype": "^2.3.0"
+      },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 4"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
       }
     },
-    "node_modules/astral-regex": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
-      "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+    "node_modules/cheerio/node_modules/domutils": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
+      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
       "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/async": {
-      "version": "3.2.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
-      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==",
-      "dev": true
-    },
-    "node_modules/async-mutex": {
-      "version": "0.2.4",
-      "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.2.4.tgz",
-      "integrity": "sha512-fcQKOXUKMQc57JlmjBCHtkKNrfGpHyR7vu18RfuLfeTAf4hK9PgOadPR5cDrBQ682zasrLUhJFe7EKAHJOduDg==",
       "dependencies": {
-        "tslib": "^2.0.0"
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
-    "node_modules/asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-    },
-    "node_modules/at-least-node": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
-      "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==",
+    "node_modules/cheerio/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
       "dev": true,
       "engines": {
-        "node": ">= 4.0.0"
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/atob": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
-      "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
+    "node_modules/cheerio/node_modules/parse5": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
+      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
       "dev": true,
-      "bin": {
-        "atob": "bin/atob.js"
+      "dependencies": {
+        "entities": "^4.4.0"
       },
-      "engines": {
-        "node": ">= 4.5.0"
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/autolinker": {
-      "version": "3.16.2",
-      "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-3.16.2.tgz",
-      "integrity": "sha512-JiYl7j2Z19F9NdTmirENSUUIIL/9MytEWtmzhfmsKPCp9E+G35Y0UNCMoM9tFigxT59qSc8Ml2dlZXOCVTYwuA==",
+    "node_modules/cheerio/node_modules/parse5-htmlparser2-tree-adapter": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz",
+      "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==",
+      "dev": true,
       "dependencies": {
-        "tslib": "^2.3.0"
+        "domhandler": "^5.0.2",
+        "parse5": "^7.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/autoprefixer": {
-      "version": "10.4.13",
-      "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
-      "integrity": "sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==",
+    "node_modules/chokidar": {
+      "version": "3.5.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
+      "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
       "dev": true,
       "funding": [
         {
-          "type": "opencollective",
-          "url": "https://opencollective.com/postcss/"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/autoprefixer"
+          "type": "individual",
+          "url": "https://paulmillr.com/funding/"
         }
       ],
       "dependencies": {
-        "browserslist": "^4.21.4",
-        "caniuse-lite": "^1.0.30001426",
-        "fraction.js": "^4.2.0",
-        "normalize-range": "^0.1.2",
-        "picocolors": "^1.0.0",
-        "postcss-value-parser": "^4.2.0"
-      },
-      "bin": {
-        "autoprefixer": "bin/autoprefixer"
+        "anymatch": "~3.1.2",
+        "braces": "~3.0.2",
+        "glob-parent": "~5.1.2",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.6.0"
       },
       "engines": {
-        "node": "^10 || ^12 || >=14"
+        "node": ">= 8.10.0"
       },
-      "peerDependencies": {
-        "postcss": "^8.1.0"
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
       }
     },
-    "node_modules/autoprefixer/node_modules/postcss-value-parser": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
-      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
-      "dev": true
+    "node_modules/chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+      "optional": true
     },
-    "node_modules/available-typed-arrays": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
-      "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
+    "node_modules/chrome-trace-event": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
+      "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==",
       "dev": true,
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=6.0"
       }
     },
-    "node_modules/aws-sign2": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
-      "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==",
+    "node_modules/ci-info": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz",
+      "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==",
       "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/sibiraj-s"
+        }
+      ],
       "engines": {
-        "node": "*"
+        "node": ">=8"
       }
     },
-    "node_modules/aws4": {
-      "version": "1.12.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
-      "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==",
+    "node_modules/cipher-base": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
+      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
+      "dev": true,
+      "dependencies": {
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "node_modules/cjs-module-lexer": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz",
+      "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==",
       "dev": true
     },
-    "node_modules/axe-core": {
-      "version": "4.4.3",
-      "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz",
-      "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==",
+    "node_modules/cjson": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/cjson/-/cjson-0.5.0.tgz",
+      "integrity": "sha512-D3CKJU9YnZNyerUQ1IzNUvMnToP3MGC2XbIAPi/7yqunJJW3rBwCVapousoFtaR9IbejeEM0KIshxC1n4HQcXw==",
       "dev": true,
+      "dependencies": {
+        "json-parse-helpfulerror": "^1.0.3"
+      },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.3.0"
       }
     },
-    "node_modules/axios": {
-      "version": "0.21.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+    "node_modules/class-utils": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
+      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
       "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.14.0"
+        "arr-union": "^3.1.0",
+        "define-property": "^0.2.5",
+        "isobject": "^3.0.0",
+        "static-extend": "^0.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/axobject-query": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz",
-      "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==",
-      "dev": true
-    },
-    "node_modules/babel-jest": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz",
-      "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==",
+    "node_modules/class-utils/node_modules/define-property": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+      "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==",
       "dev": true,
       "dependencies": {
-        "@jest/transform": "^29.7.0",
-        "@types/babel__core": "^7.1.14",
-        "babel-plugin-istanbul": "^6.1.1",
-        "babel-preset-jest": "^29.6.3",
-        "chalk": "^4.0.0",
-        "graceful-fs": "^4.2.9",
-        "slash": "^3.0.0"
+        "is-descriptor": "^0.1.0"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.8.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/babel-jest/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/classnames": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz",
+      "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw=="
+    },
+    "node_modules/clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/cli-cursor": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz",
+      "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "restore-cursor": "^3.1.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
+      }
+    },
+    "node_modules/cli-spinners": {
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.1.tgz",
+      "integrity": "sha512-jHgecW0pxkonBJdrKsqxgRX9AcG+u/5k0Q7WPDfi8AogLAdwxEkyYYNWwZ5GvVFoFx2uiY1eNcSK00fh+1+FyQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/babel-loader": {
-      "version": "9.1.2",
-      "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.1.2.tgz",
-      "integrity": "sha512-mN14niXW43tddohGl8HPu5yfQq70iUThvFL/4QzESA7GcZoC0eVOhvWdQ8+3UlSjaDE9MVtsW9mxDY07W7VpVA==",
+    "node_modules/cli-table": {
+      "version": "0.3.11",
+      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.11.tgz",
+      "integrity": "sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ==",
       "dev": true,
       "dependencies": {
-        "find-cache-dir": "^3.3.2",
-        "schema-utils": "^4.0.0"
+        "colors": "1.0.3"
       },
       "engines": {
-        "node": ">= 14.15.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.12.0",
-        "webpack": ">=5"
+        "node": ">= 0.2.0"
       }
     },
-    "node_modules/babel-plugin-add-module-exports": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/babel-plugin-add-module-exports/-/babel-plugin-add-module-exports-1.0.4.tgz",
-      "integrity": "sha512-g+8yxHUZ60RcyaUpfNzy56OtWW+x9cyEe9j+CranqLiqbju2yf/Cy6ZtYK40EZxtrdHllzlVZgLmcOUCTlJ7Jg==",
-      "dev": true
-    },
-    "node_modules/babel-plugin-istanbul": {
-      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz",
-      "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==",
+    "node_modules/cli-table/node_modules/colors": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
+      "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==",
       "dev": true,
-      "dependencies": {
-        "@babel/helper-plugin-utils": "^7.0.0",
-        "@istanbuljs/load-nyc-config": "^1.0.0",
-        "@istanbuljs/schema": "^0.1.2",
-        "istanbul-lib-instrument": "^5.0.4",
-        "test-exclude": "^6.0.0"
-      },
       "engines": {
-        "node": ">=8"
+        "node": ">=0.1.90"
       }
     },
-    "node_modules/babel-plugin-jest-hoist": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz",
-      "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==",
+    "node_modules/cli-table3": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz",
+      "integrity": "sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==",
       "dev": true,
       "dependencies": {
-        "@babel/template": "^7.3.3",
-        "@babel/types": "^7.3.3",
-        "@types/babel__core": "^7.1.14",
-        "@types/babel__traverse": "^7.0.6"
+        "string-width": "^4.2.0"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": "10.* || >= 12.*"
+      },
+      "optionalDependencies": {
+        "@colors/colors": "1.5.0"
       }
     },
-    "node_modules/babel-plugin-polyfill-corejs2": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz",
-      "integrity": "sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q==",
+    "node_modules/cli-truncate": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz",
+      "integrity": "sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==",
       "dev": true,
       "dependencies": {
-        "@babel/compat-data": "^7.17.7",
-        "@babel/helper-define-polyfill-provider": "^0.3.3",
-        "semver": "^6.1.1"
+        "slice-ansi": "^3.0.0",
+        "string-width": "^4.2.0"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/babel-plugin-polyfill-corejs2/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/cli-width": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz",
+      "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+      "engines": {
+        "node": ">= 10"
       }
     },
-    "node_modules/babel-plugin-polyfill-corejs3": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.6.0.tgz",
-      "integrity": "sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==",
-      "dev": true,
+    "node_modules/cliui": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
+      "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
       "dependencies": {
-        "@babel/helper-define-polyfill-provider": "^0.3.3",
-        "core-js-compat": "^3.25.1"
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.1",
+        "wrap-ansi": "^7.0.0"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=12"
       }
     },
-    "node_modules/babel-plugin-polyfill-regenerator": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.4.1.tgz",
-      "integrity": "sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==",
+    "node_modules/clone": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
+      "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/clone-deep": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
+      "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
       "dev": true,
       "dependencies": {
-        "@babel/helper-define-polyfill-provider": "^0.3.3"
+        "is-plain-object": "^2.0.4",
+        "kind-of": "^6.0.2",
+        "shallow-clone": "^3.0.0"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0-0"
+      "engines": {
+        "node": ">=6"
       }
     },
-    "node_modules/babel-preset-current-node-syntax": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz",
-      "integrity": "sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ==",
+    "node_modules/clone-deep/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
       "dev": true,
       "dependencies": {
-        "@babel/plugin-syntax-async-generators": "^7.8.4",
-        "@babel/plugin-syntax-bigint": "^7.8.3",
-        "@babel/plugin-syntax-class-properties": "^7.8.3",
-        "@babel/plugin-syntax-import-meta": "^7.8.3",
-        "@babel/plugin-syntax-json-strings": "^7.8.3",
-        "@babel/plugin-syntax-logical-assignment-operators": "^7.8.3",
-        "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3",
-        "@babel/plugin-syntax-numeric-separator": "^7.8.3",
-        "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
-        "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
-        "@babel/plugin-syntax-optional-chaining": "^7.8.3",
-        "@babel/plugin-syntax-top-level-await": "^7.8.3"
+        "isobject": "^3.0.1"
       },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/babel-preset-jest": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz",
-      "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==",
+    "node_modules/clone-regexp": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clone-regexp/-/clone-regexp-2.2.0.tgz",
+      "integrity": "sha512-beMpP7BOtTipFuW8hrJvREQ2DrRu3BE7by0ZpibtfBA+qfHYvMGTc2Yb1JMYPKg/JUw0CHYvpg796aNTSW9z7Q==",
       "dev": true,
       "dependencies": {
-        "babel-plugin-jest-hoist": "^29.6.3",
-        "babel-preset-current-node-syntax": "^1.0.0"
+        "is-regexp": "^2.0.0"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "node": ">=6"
       }
     },
-    "node_modules/babel-runtime": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
-      "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==",
+    "node_modules/co": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
+      "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==",
       "dev": true,
-      "dependencies": {
-        "core-js": "^2.4.0",
-        "regenerator-runtime": "^0.11.0"
+      "engines": {
+        "iojs": ">= 1.0.0",
+        "node": ">= 0.12.0"
       }
     },
-    "node_modules/babel-runtime/node_modules/regenerator-runtime": {
-      "version": "0.11.1",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
-      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==",
+    "node_modules/code-block-writer": {
+      "version": "11.0.3",
+      "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-11.0.3.tgz",
+      "integrity": "sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==",
       "dev": true
     },
-    "node_modules/babelify": {
-      "version": "10.0.0",
-      "resolved": "https://registry.npmjs.org/babelify/-/babelify-10.0.0.tgz",
-      "integrity": "sha512-X40FaxyH7t3X+JFAKvb1H9wooWKLRCi8pg3m8poqtdZaIng+bjzp9RvKQCvRjF9isHiPkXspbbXT/zwXLtwgwg==",
+    "node_modules/code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==",
       "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/coffeeify": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/coffeeify/-/coffeeify-3.0.1.tgz",
+      "integrity": "sha512-Qjnr7UX6ldK1PHV7wCnv7AuCd4q19KTUtwJnu/6JRJB4rfm12zvcXtKdacUoePOKr1I4ka/ydKiwWpNAdsQb0g==",
+      "dev": true,
+      "dependencies": {
+        "convert-source-map": "^1.3.0",
+        "through2": "^2.0.0"
       },
       "peerDependencies": {
-        "@babel/core": "^7.0.0"
+        "coffeescript": ">1.9.2 <3"
       }
     },
-    "node_modules/bail": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/bail/-/bail-1.0.5.tgz",
-      "integrity": "sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ==",
+    "node_modules/coffeescript": {
+      "version": "1.12.7",
+      "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-1.12.7.tgz",
+      "integrity": "sha512-pLXHFxQMPklVoEekowk8b3erNynC+DVJzChxS/LCBBgR6/8AJkHivkm//zbowcfc7BTCAjryuhx6gPqPRfsFoA==",
       "dev": true,
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+      "bin": {
+        "cake": "bin/cake",
+        "coffee": "bin/coffee"
+      },
+      "engines": {
+        "node": ">=0.8.0"
       }
     },
-    "node_modules/balanced-match": {
+    "node_modules/collect-v8-coverage": {
       "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+      "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz",
+      "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==",
+      "dev": true
     },
-    "node_modules/base": {
-      "version": "0.11.2",
-      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
-      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
+    "node_modules/collection-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
+      "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==",
       "dev": true,
       "dependencies": {
-        "cache-base": "^1.0.1",
-        "class-utils": "^0.3.5",
-        "component-emitter": "^1.2.1",
-        "define-property": "^1.0.0",
-        "isobject": "^3.0.1",
-        "mixin-deep": "^1.2.0",
-        "pascalcase": "^0.1.1"
+        "map-visit": "^1.0.0",
+        "object-visit": "^1.0.0"
       },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/base/node_modules/define-property": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-      "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==",
-      "dev": true,
+    "node_modules/color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
       "dependencies": {
-        "is-descriptor": "^1.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "color-name": "1.1.3"
       }
     },
-    "node_modules/base64-js": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
+    "node_modules/color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
     },
-    "node_modules/basic-auth": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz",
-      "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==",
+    "node_modules/color-support": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
       "dev": true,
-      "dependencies": {
-        "safe-buffer": "5.1.2"
-      },
-      "engines": {
-        "node": ">= 0.8"
+      "bin": {
+        "color-support": "bin.js"
       }
     },
-    "node_modules/basic-auth/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+    "node_modules/colorette": {
+      "version": "2.0.20",
+      "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
+      "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==",
       "dev": true
     },
-    "node_modules/batch": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz",
-      "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==",
-      "dev": true
+    "node_modules/colors": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
+      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.1.90"
+      }
     },
-    "node_modules/bcrypt-pbkdf": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
-      "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
+    "node_modules/combine-source-map": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz",
+      "integrity": "sha512-UlxQ9Vw0b/Bt/KYwCFqdEwsQ1eL8d1gibiFb7lxQJFdvTgc2hIZi6ugsg+kyhzhPV+QEpUiEIwInIAIrgoEkrg==",
       "dev": true,
       "dependencies": {
-        "tweetnacl": "^0.14.3"
+        "convert-source-map": "~1.1.0",
+        "inline-source-map": "~0.6.0",
+        "lodash.memoize": "~3.0.3",
+        "source-map": "~0.5.3"
       }
     },
-    "node_modules/bcryptjs": {
-      "version": "2.4.3",
-      "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz",
-      "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==",
+    "node_modules/combine-source-map/node_modules/convert-source-map": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz",
+      "integrity": "sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg==",
       "dev": true
     },
-    "node_modules/becke-ch--regex--s0-0-v1--base--pl--lib": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/becke-ch--regex--s0-0-v1--base--pl--lib/-/becke-ch--regex--s0-0-v1--base--pl--lib-1.4.0.tgz",
-      "integrity": "sha512-FnWonOyaw7Vivg5nIkrUll9HSS5TjFbyuURAiDssuL6VxrBe3ERzudRxOcWRhZYlP89UArMDikz7SapRPQpmZQ==",
+    "node_modules/combine-source-map/node_modules/lodash.memoize": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz",
+      "integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A==",
       "dev": true
     },
-    "node_modules/big.js": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
-      "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==",
+    "node_modules/combine-source-map/node_modules/source-map": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
+      "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==",
       "dev": true,
       "engines": {
-        "node": "*"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/binary-extensions": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
-      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
-      "dev": true,
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
       "engines": {
-        "node": ">=8"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/bl": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-      "devOptional": true,
-      "dependencies": {
-        "buffer": "^5.5.0",
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.4.0"
+    "node_modules/comma-separated-tokens": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz",
+      "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/bl/node_modules/buffer": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-      "devOptional": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.1.13"
-      }
+    "node_modules/commander": {
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+      "dev": true
     },
-    "node_modules/bl/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "devOptional": true,
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
+    "node_modules/common-tags": {
+      "version": "1.8.2",
+      "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz",
+      "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==",
+      "dev": true,
       "engines": {
-        "node": ">= 6"
+        "node": ">=4.0.0"
       }
     },
-    "node_modules/blob-util": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz",
-      "integrity": "sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==",
+    "node_modules/commondir": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
+      "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==",
       "dev": true
     },
-    "node_modules/bluebird": {
-      "version": "3.7.2",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
-      "dev": true
+    "node_modules/component-emitter": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz",
+      "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
     },
-    "node_modules/bn.js": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz",
-      "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==",
-      "dev": true
+    "node_modules/compressible": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
+      "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
+      "dev": true,
+      "dependencies": {
+        "mime-db": ">= 1.43.0 < 2"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
     },
-    "node_modules/body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+    "node_modules/compression": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz",
+      "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==",
       "dev": true,
       "dependencies": {
-        "bytes": "3.1.0",
-        "content-type": "~1.0.4",
+        "accepts": "~1.3.5",
+        "bytes": "3.0.0",
+        "compressible": "~2.0.16",
         "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "http-errors": "1.7.2",
-        "iconv-lite": "0.4.24",
-        "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "on-headers": "~1.0.2",
+        "safe-buffer": "5.1.2",
+        "vary": "~1.1.2"
       },
       "engines": {
-        "node": ">= 0.8"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/body-parser/node_modules/debug": {
+    "node_modules/compression/node_modules/debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
       "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
@@ -8785,1845 +11960,1932 @@
         "ms": "2.0.0"
       }
     },
-    "node_modules/body-parser/node_modules/ms": {
+    "node_modules/compression/node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "dev": true
     },
-    "node_modules/bonjour-service": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.1.1.tgz",
-      "integrity": "sha512-Z/5lQRMOG9k7W+FkeGTNjh7htqn/2LMnfOvBZ8pynNZCM9MwkQkI3zeI4oz09uWdcgmgHugVvBqxGg4VQJ5PCg==",
-      "dev": true,
-      "dependencies": {
-        "array-flatten": "^2.1.2",
-        "dns-equal": "^1.0.0",
-        "fast-deep-equal": "^3.1.3",
-        "multicast-dns": "^7.2.5"
-      }
-    },
-    "node_modules/bonjour-service/node_modules/array-flatten": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz",
-      "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==",
+    "node_modules/compression/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
       "dev": true
     },
-    "node_modules/boolbase": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
-      "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
       "dev": true
     },
-    "node_modules/bootstrap": {
-      "version": "5.2.3",
-      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.2.3.tgz",
-      "integrity": "sha512-cEKPM+fwb3cT8NzQZYEu4HilJ3anCrWqh3CHAok1p9jXqMPsPTBhU25fBckEJHJ/p+tTxTFTsFQGM+gaHpi3QQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/twbs"
-        },
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/bootstrap"
-        }
-      ],
-      "peerDependencies": {
-        "@popperjs/core": "^2.11.6"
-      }
-    },
-    "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
-      "dependencies": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
-      }
-    },
-    "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+    "node_modules/concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
       "dev": true,
+      "engines": [
+        "node >= 0.8"
+      ],
       "dependencies": {
-        "fill-range": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=8"
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.2.2",
+        "typedarray": "^0.0.6"
       }
     },
-    "node_modules/brfs": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/brfs/-/brfs-2.0.2.tgz",
-      "integrity": "sha512-IrFjVtwu4eTJZyu8w/V2gxU7iLTtcHih67sgEdzrhjLBMHp2uYefUBfdM4k2UvcuWMgV7PQDZHSLeNWnLFKWVQ==",
+    "node_modules/concat-stream/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
       "dev": true,
       "dependencies": {
-        "quote-stream": "^1.0.1",
-        "resolve": "^1.1.5",
-        "static-module": "^3.0.2",
-        "through2": "^2.0.0"
-      },
-      "bin": {
-        "brfs": "bin/cmd.js"
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
       }
     },
-    "node_modules/brorand": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
-      "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==",
+    "node_modules/concat-stream/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
       "dev": true
     },
-    "node_modules/brotli": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz",
-      "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==",
+    "node_modules/concat-stream/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
       "dev": true,
       "dependencies": {
-        "base64-js": "^1.1.2"
+        "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/browser-pack": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz",
-      "integrity": "sha512-erYug8XoqzU3IfcU8fUgyHqyOXqIE4tUTTQ+7mqUjQlvnXkOO6OlT9c/ZoJVHYoAaqGxr09CN53G7XIsO4KtWA==",
+    "node_modules/connect": {
+      "version": "3.7.0",
+      "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz",
+      "integrity": "sha512-ZqRXc+tZukToSNmh5C2iWMSoV3X1YUcPbqEM4DkEG5tNQXrQUZCNVGGv3IuicnkMtPfGf3Xtp8WCXs295iQ1pQ==",
       "dev": true,
       "dependencies": {
-        "combine-source-map": "~0.8.0",
-        "defined": "^1.0.0",
-        "JSONStream": "^1.0.3",
-        "safe-buffer": "^5.1.1",
-        "through2": "^2.0.0",
-        "umd": "^3.0.0"
+        "debug": "2.6.9",
+        "finalhandler": "1.1.2",
+        "parseurl": "~1.3.3",
+        "utils-merge": "1.0.1"
       },
-      "bin": {
-        "browser-pack": "bin/cmd.js"
+      "engines": {
+        "node": ">= 0.10.0"
       }
     },
-    "node_modules/browser-process-hrtime": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
-      "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
-      "dev": true
-    },
-    "node_modules/browser-resolve": {
+    "node_modules/connect-history-api-fallback": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz",
-      "integrity": "sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==",
+      "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz",
+      "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/connect/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "resolve": "^1.17.0"
+        "ms": "2.0.0"
       }
     },
-    "node_modules/browserify": {
-      "version": "16.5.2",
-      "resolved": "https://registry.npmjs.org/browserify/-/browserify-16.5.2.tgz",
-      "integrity": "sha512-TkOR1cQGdmXU9zW4YukWzWVSJwrxmNdADFbqbE3HFgQWe5wqZmOawqZ7J/8MPCwk/W8yY7Y0h+7mOtcZxLP23g==",
+    "node_modules/connect/node_modules/finalhandler": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
+      "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
       "dev": true,
       "dependencies": {
-        "assert": "^1.4.0",
-        "browser-pack": "^6.0.1",
-        "browser-resolve": "^2.0.0",
-        "browserify-zlib": "~0.2.0",
-        "buffer": "~5.2.1",
-        "cached-path-relative": "^1.0.0",
-        "concat-stream": "^1.6.0",
-        "console-browserify": "^1.1.0",
-        "constants-browserify": "~1.0.0",
-        "crypto-browserify": "^3.0.0",
-        "defined": "^1.0.0",
-        "deps-sort": "^2.0.0",
-        "domain-browser": "^1.2.0",
-        "duplexer2": "~0.1.2",
-        "events": "^2.0.0",
-        "glob": "^7.1.0",
-        "has": "^1.0.0",
-        "htmlescape": "^1.1.0",
-        "https-browserify": "^1.0.0",
-        "inherits": "~2.0.1",
-        "insert-module-globals": "^7.0.0",
-        "JSONStream": "^1.0.3",
-        "labeled-stream-splicer": "^2.0.0",
-        "mkdirp-classic": "^0.5.2",
-        "module-deps": "^6.2.3",
-        "os-browserify": "~0.3.0",
-        "parents": "^1.0.1",
-        "path-browserify": "~0.0.0",
-        "process": "~0.11.0",
-        "punycode": "^1.3.2",
-        "querystring-es3": "~0.2.0",
-        "read-only-stream": "^2.0.0",
-        "readable-stream": "^2.0.2",
-        "resolve": "^1.1.4",
-        "shasum": "^1.0.0",
-        "shell-quote": "^1.6.1",
-        "stream-browserify": "^2.0.0",
-        "stream-http": "^3.0.0",
-        "string_decoder": "^1.1.1",
-        "subarg": "^1.0.0",
-        "syntax-error": "^1.1.1",
-        "through2": "^2.0.0",
-        "timers-browserify": "^1.0.1",
-        "tty-browserify": "0.0.1",
-        "url": "~0.11.0",
-        "util": "~0.10.1",
-        "vm-browserify": "^1.0.0",
-        "xtend": "^4.0.0"
-      },
-      "bin": {
-        "browserify": "bin/cmd.js"
+        "debug": "2.6.9",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "on-finished": "~2.3.0",
+        "parseurl": "~1.3.3",
+        "statuses": "~1.5.0",
+        "unpipe": "~1.0.0"
       },
       "engines": {
         "node": ">= 0.8"
       }
     },
-    "node_modules/browserify-aes": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
-      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
+    "node_modules/connect/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/connect/node_modules/on-finished": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+      "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==",
       "dev": true,
       "dependencies": {
-        "buffer-xor": "^1.0.3",
-        "cipher-base": "^1.0.0",
-        "create-hash": "^1.1.0",
-        "evp_bytestokey": "^1.0.3",
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.0.1"
+        "ee-first": "1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
       }
     },
-    "node_modules/browserify-cipher": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
-      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
+    "node_modules/connect/node_modules/statuses": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
       "dev": true,
-      "dependencies": {
-        "browserify-aes": "^1.0.4",
-        "browserify-des": "^1.0.0",
-        "evp_bytestokey": "^1.0.0"
+      "engines": {
+        "node": ">= 0.6"
       }
     },
-    "node_modules/browserify-des": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz",
-      "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==",
+    "node_modules/console-browserify": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz",
+      "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==",
+      "dev": true
+    },
+    "node_modules/console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==",
+      "dev": true
+    },
+    "node_modules/constants-browserify": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
+      "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==",
+      "dev": true
+    },
+    "node_modules/content-disposition": {
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
       "dev": true,
       "dependencies": {
-        "cipher-base": "^1.0.1",
-        "des.js": "^1.0.0",
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.1.2"
+        "safe-buffer": "5.2.1"
+      },
+      "engines": {
+        "node": ">= 0.6"
       }
     },
-    "node_modules/browserify-optional": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-optional/-/browserify-optional-1.0.1.tgz",
-      "integrity": "sha512-VrhjbZ+Ba5mDiSYEuPelekQMfTbhcA2DhLk2VQWqdcCROWeFqlTcXZ7yfRkXCIl8E+g4gINJYJiRB7WEtfomAQ==",
+    "node_modules/content-type": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+      "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
       "dev": true,
-      "dependencies": {
-        "ast-transform": "0.0.0",
-        "ast-types": "^0.7.0",
-        "browser-resolve": "^1.8.1"
+      "engines": {
+        "node": ">= 0.6"
       }
     },
-    "node_modules/browserify-optional/node_modules/browser-resolve": {
-      "version": "1.11.3",
-      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.3.tgz",
-      "integrity": "sha512-exDi1BYWB/6raKHmDTCicQfTkqwN5fioMFV4j8BsfMU4R2DK/QfZfK7kOVkmWCNANf0snkBzqGqAJBao9gZMdQ==",
-      "dev": true,
-      "dependencies": {
-        "resolve": "1.1.7"
+    "node_modules/convert-source-map": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
+      "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
+    },
+    "node_modules/cookie": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
+      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+      "engines": {
+        "node": ">= 0.6"
       }
     },
-    "node_modules/browserify-optional/node_modules/resolve": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz",
-      "integrity": "sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg==",
+    "node_modules/cookie-signature": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==",
       "dev": true
     },
-    "node_modules/browserify-rsa": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz",
-      "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==",
+    "node_modules/copy-anything": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz",
+      "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==",
       "dev": true,
       "dependencies": {
-        "bn.js": "^5.0.0",
-        "randombytes": "^2.0.1"
+        "is-what": "^3.14.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/mesqueeb"
+      }
+    },
+    "node_modules/copy-descriptor": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
+      "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/browserify-sign": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz",
-      "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==",
-      "dev": true,
+    "node_modules/copy-to-clipboard": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz",
+      "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==",
       "dependencies": {
-        "bn.js": "^5.1.1",
-        "browserify-rsa": "^4.0.1",
-        "create-hash": "^1.2.0",
-        "create-hmac": "^1.1.7",
-        "elliptic": "^6.5.3",
-        "inherits": "^2.0.4",
-        "parse-asn1": "^5.1.5",
-        "readable-stream": "^3.6.0",
-        "safe-buffer": "^5.2.0"
+        "toggle-selection": "^1.0.6"
       }
     },
-    "node_modules/browserify-sign/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+    "node_modules/copy-webpack-plugin": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz",
+      "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
+        "fast-glob": "^3.2.11",
+        "glob-parent": "^6.0.1",
+        "globby": "^13.1.1",
+        "normalize-path": "^3.0.0",
+        "schema-utils": "^4.0.0",
+        "serialize-javascript": "^6.0.0"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">= 14.15.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
       }
     },
-    "node_modules/browserify-zlib": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
-      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
+    "node_modules/copy-webpack-plugin/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
       "dev": true,
       "dependencies": {
-        "pako": "~1.0.5"
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
       }
     },
-    "node_modules/browserify/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/core-js": {
+      "version": "3.33.3",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.33.3.tgz",
+      "integrity": "sha512-lo0kOocUlLKmm6kv/FswQL8zbkH7mVsLJ/FULClOhv8WRVmKLVcs6XPNQAzstfeJTCHMyButEwG+z1kHxHoDZw==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
+    },
+    "node_modules/core-js-compat": {
+      "version": "3.33.3",
+      "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.33.3.tgz",
+      "integrity": "sha512-cNzGqFsh3Ot+529GIXacjTJ7kegdt5fPXxCBVS1G0iaZpuo/tBz399ymceLJveQhFFZ8qThHiP3fzuoQjKN2ow==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
-      "engines": {
-        "node": "*"
+        "browserslist": "^4.22.1"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
       }
     },
-    "node_modules/browserslist": {
-      "version": "4.21.5",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz",
-      "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==",
+    "node_modules/core-js-pure": {
+      "version": "3.33.3",
+      "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.33.3.tgz",
+      "integrity": "sha512-taJ00IDOP+XYQEA2dAe4ESkmHt1fL8wzYDo3mRWQey8uO9UojlBFMneA65kMyxfYP7106c6LzWaq7/haDT6BCQ==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
+    },
+    "node_modules/core-util-is": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
+      "dev": true
+    },
+    "node_modules/cors": {
+      "version": "2.8.5",
+      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
       "dev": true,
-      "funding": [
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/browserslist"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/browserslist"
-        }
-      ],
       "dependencies": {
-        "caniuse-lite": "^1.0.30001449",
-        "electron-to-chromium": "^1.4.284",
-        "node-releases": "^2.0.8",
-        "update-browserslist-db": "^1.0.10"
-      },
-      "bin": {
-        "browserslist": "cli.js"
+        "object-assign": "^4",
+        "vary": "^1"
       },
       "engines": {
-        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
+        "node": ">= 0.10"
       }
     },
-    "node_modules/bs-logger": {
-      "version": "0.2.6",
-      "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz",
-      "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==",
+    "node_modules/cosmiconfig": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
+      "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==",
       "dev": true,
       "dependencies": {
-        "fast-json-stable-stringify": "2.x"
+        "@types/parse-json": "^4.0.0",
+        "import-fresh": "^3.2.1",
+        "parse-json": "^5.0.0",
+        "path-type": "^4.0.0",
+        "yaml": "^1.10.0"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=10"
       }
     },
-    "node_modules/bser": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz",
-      "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==",
+    "node_modules/create-ecdh": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
+      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
       "dev": true,
       "dependencies": {
-        "node-int64": "^0.4.0"
+        "bn.js": "^4.1.0",
+        "elliptic": "^6.5.3"
       }
     },
-    "node_modules/buffer": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz",
-      "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==",
+    "node_modules/create-ecdh/node_modules/bn.js": {
+      "version": "4.12.0",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
+      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
+      "dev": true
+    },
+    "node_modules/create-hash": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
+      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
       "dev": true,
       "dependencies": {
-        "base64-js": "^1.0.2",
-        "ieee754": "^1.1.4"
+        "cipher-base": "^1.0.1",
+        "inherits": "^2.0.1",
+        "md5.js": "^1.3.4",
+        "ripemd160": "^2.0.1",
+        "sha.js": "^2.4.0"
       }
     },
-    "node_modules/buffer-crc32": {
-      "version": "0.2.13",
-      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
-      "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==",
+    "node_modules/create-hmac": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
+      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
       "dev": true,
-      "engines": {
-        "node": "*"
+      "dependencies": {
+        "cipher-base": "^1.0.3",
+        "create-hash": "^1.1.0",
+        "inherits": "^2.0.1",
+        "ripemd160": "^2.0.0",
+        "safe-buffer": "^5.0.1",
+        "sha.js": "^2.4.8"
       }
     },
-    "node_modules/buffer-equal": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz",
-      "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==",
+    "node_modules/create-jest": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz",
+      "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==",
       "dev": true,
+      "dependencies": {
+        "@jest/types": "^29.6.3",
+        "chalk": "^4.0.0",
+        "exit": "^0.1.2",
+        "graceful-fs": "^4.2.9",
+        "jest-config": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "prompts": "^2.0.1"
+      },
+      "bin": {
+        "create-jest": "bin/create-jest.js"
+      },
       "engines": {
-        "node": ">=0.4.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/buffer-from": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
-      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
-      "dev": true
-    },
-    "node_modules/buffer-xor": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
-      "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==",
-      "dev": true
-    },
-    "node_modules/builtin-status-codes": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
-      "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==",
-      "dev": true
-    },
-    "node_modules/builtins": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz",
-      "integrity": "sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==",
+    "node_modules/create-jest/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "semver": "^7.0.0"
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/bulk-require": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/bulk-require/-/bulk-require-1.0.1.tgz",
-      "integrity": "sha512-BLU9AMnm1FMr68fR0sYvOkMew4x2ZJ8YztshITlGArl6aLtfAazOWiFj/bwJJixRO8C0wXx9PnRNRWeoR03e8Q==",
+    "node_modules/create-jest/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "glob": "^7.1.1"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/bulk-require/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/create-jest/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/busboy": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
-      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+    "node_modules/create-jest/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/create-jest/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/create-jest/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
       "dependencies": {
-        "streamsearch": "^1.1.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=10.16.0"
+        "node": ">=8"
       }
     },
-    "node_modules/bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
+    "node_modules/critters": {
+      "version": "0.0.16",
+      "resolved": "https://registry.npmjs.org/critters/-/critters-0.0.16.tgz",
+      "integrity": "sha512-JwjgmO6i3y6RWtLYmXwO5jMd+maZt8Tnfu7VVISmEWyQqfLpB8soBswf8/2bu6SBXxtKA68Al3c+qIG1ApT68A==",
       "dev": true,
-      "engines": {
-        "node": ">= 0.8"
+      "dependencies": {
+        "chalk": "^4.1.0",
+        "css-select": "^4.2.0",
+        "parse5": "^6.0.1",
+        "parse5-htmlparser2-tree-adapter": "^6.0.1",
+        "postcss": "^8.3.7",
+        "pretty-bytes": "^5.3.0"
       }
     },
-    "node_modules/cacache": {
-      "version": "17.0.4",
-      "resolved": "https://registry.npmjs.org/cacache/-/cacache-17.0.4.tgz",
-      "integrity": "sha512-Z/nL3gU+zTUjz5pCA5vVjYM8pmaw2kxM7JEiE0fv3w77Wj+sFbi70CrBruUWH0uNcEdvLDixFpgA2JM4F4DBjA==",
+    "node_modules/critters/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@npmcli/fs": "^3.1.0",
-        "fs-minipass": "^3.0.0",
-        "glob": "^8.0.1",
-        "lru-cache": "^7.7.1",
-        "minipass": "^4.0.0",
-        "minipass-collect": "^1.0.2",
-        "minipass-flush": "^1.0.5",
-        "minipass-pipeline": "^1.2.4",
-        "p-map": "^4.0.0",
-        "promise-inflight": "^1.0.1",
-        "ssri": "^10.0.0",
-        "tar": "^6.1.11",
-        "unique-filename": "^3.0.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/cacache/node_modules/lru-cache": {
-      "version": "7.18.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
-      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+    "node_modules/critters/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/cache-base": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
-      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
+    "node_modules/critters/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "collection-visit": "^1.0.0",
-        "component-emitter": "^1.2.1",
-        "get-value": "^2.0.6",
-        "has-value": "^1.0.0",
-        "isobject": "^3.0.1",
-        "set-value": "^2.0.0",
-        "to-object-path": "^0.3.0",
-        "union-value": "^1.0.0",
-        "unset-value": "^1.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/cached-path-relative": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.1.0.tgz",
-      "integrity": "sha512-WF0LihfemtesFcJgO7xfOoOcnWzY/QHR4qeDqV44jPU3HTI54+LnfXK3SA27AVVGCdZFgjjFFaqUA9Jx7dMJZA==",
+    "node_modules/critters/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/cachedir": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz",
-      "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==",
+    "node_modules/critters/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=6"
+        "node": ">=8"
       }
     },
-    "node_modules/call-bind": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
-      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+    "node_modules/critters/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
       "dependencies": {
-        "function-bind": "^1.1.1",
-        "get-intrinsic": "^1.0.2"
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/callsite": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
-      "integrity": "sha512-0vdNRFXn5q+dtOqjfFtmtlI9N2eVZ7LMyEV2iKC5mEEFvSg/69Ml6b/WU2qF8W1nLRa0wiSrDT3Y5jOHZCwKPQ==",
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
       "dev": true,
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
       "engines": {
-        "node": "*"
+        "node": ">= 8"
       }
     },
-    "node_modules/callsites": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-      "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
       "dev": true,
       "engines": {
-        "node": ">=6"
+        "node": "*"
       }
     },
-    "node_modules/camelcase": {
-      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
-      "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
+    "node_modules/crypto-browserify": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
+      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
       "dev": true,
+      "dependencies": {
+        "browserify-cipher": "^1.0.0",
+        "browserify-sign": "^4.0.0",
+        "create-ecdh": "^4.0.0",
+        "create-hash": "^1.1.0",
+        "create-hmac": "^1.1.0",
+        "diffie-hellman": "^5.0.0",
+        "inherits": "^2.0.1",
+        "pbkdf2": "^3.0.3",
+        "public-encrypt": "^4.0.0",
+        "randombytes": "^2.0.0",
+        "randomfill": "^1.0.3"
+      },
       "engines": {
-        "node": ">=6"
+        "node": "*"
       }
     },
-    "node_modules/camelcase-keys": {
-      "version": "6.2.2",
-      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz",
-      "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==",
+    "node_modules/crypto-js": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz",
+      "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==",
+      "dev": true
+    },
+    "node_modules/css-loader": {
+      "version": "6.7.3",
+      "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.7.3.tgz",
+      "integrity": "sha512-qhOH1KlBMnZP8FzRO6YCH9UHXQhVMcEGLyNdb7Hv2cpcmJbW0YrddO+tG1ab5nT41KpHIYGsbeHqxB9xPu1pKQ==",
       "dev": true,
       "dependencies": {
-        "camelcase": "^5.3.1",
-        "map-obj": "^4.0.0",
-        "quick-lru": "^4.0.1"
+        "icss-utils": "^5.1.0",
+        "postcss": "^8.4.19",
+        "postcss-modules-extract-imports": "^3.0.0",
+        "postcss-modules-local-by-default": "^4.0.0",
+        "postcss-modules-scope": "^3.0.0",
+        "postcss-modules-values": "^4.0.0",
+        "postcss-value-parser": "^4.2.0",
+        "semver": "^7.3.8"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">= 12.13.0"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.0.0"
       }
     },
-    "node_modules/can-use-dom": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/can-use-dom/-/can-use-dom-0.1.0.tgz",
-      "integrity": "sha512-ceOhN1DL7Y4O6M0j9ICgmTYziV89WMd96SvSl0REd8PMgrY0B/WBOPoed5S1KUmJqXgUXh8gzSe6E3ae27upsQ=="
+    "node_modules/css-loader/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
     },
-    "node_modules/caniuse-lite": {
-      "version": "1.0.30001539",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001539.tgz",
-      "integrity": "sha512-hfS5tE8bnNiNvEOEkm8HElUHroYwlqMMENEzELymy77+tJ6m+gA2krtHl5hxJaj71OlpC2cHZbdSMX1/YEqEkA==",
-      "funding": [
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/browserslist"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
-        },
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/ai"
-        }
-      ]
+    "node_modules/css-loader/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
     },
-    "node_modules/caseless": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
-      "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
+    "node_modules/css-loader/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "dev": true
     },
-    "node_modules/chai": {
-      "version": "4.3.8",
-      "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.8.tgz",
-      "integrity": "sha512-vX4YvVVtxlfSZ2VecZgFUTU5qPCYsobVI2O9FmwEXBhDigYGQA6jRXCycIs1yJnnWbZ6/+a2zNIF5DfVCcJBFQ==",
+    "node_modules/css-select": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz",
+      "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==",
       "dev": true,
       "dependencies": {
-        "assertion-error": "^1.1.0",
-        "check-error": "^1.0.2",
-        "deep-eql": "^4.1.2",
-        "get-func-name": "^2.0.0",
-        "loupe": "^2.3.1",
-        "pathval": "^1.1.1",
-        "type-detect": "^4.0.5"
+        "boolbase": "^1.0.0",
+        "css-what": "^6.0.1",
+        "domhandler": "^4.3.1",
+        "domutils": "^2.8.0",
+        "nth-check": "^2.0.1"
       },
-      "engines": {
-        "node": ">=4"
+      "funding": {
+        "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/chalk": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
-      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+    "node_modules/css-tree": {
+      "version": "1.0.0-alpha.39",
+      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.39.tgz",
+      "integrity": "sha512-7UvkEYgBAHRG9Nt980lYxjsTrCyHFN53ky3wVsDkiMdVqylqRt+Zc+jm5qw7/qyOvN2dHSYtX0e4MbCCExSvnA==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "mdn-data": "2.0.6",
+        "source-map": "^0.6.1"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/char-regex": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz",
-      "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==",
+    "node_modules/css-tree/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true,
       "engines": {
-        "node": ">=10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/character-entities": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz",
-      "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==",
+    "node_modules/css-what": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
+      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
+      "dev": true,
+      "engines": {
+        "node": ">= 6"
+      },
       "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+        "url": "https://github.com/sponsors/fb55"
       }
     },
-    "node_modules/character-entities-legacy": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz",
-      "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
-      }
+    "node_modules/css.escape": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz",
+      "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg=="
     },
-    "node_modules/character-reference-invalid": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz",
-      "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+    "node_modules/cssesc": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
+      "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==",
+      "dev": true,
+      "bin": {
+        "cssesc": "bin/cssesc"
+      },
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/chardet": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
-      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
+    "node_modules/cssfontparser": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/cssfontparser/-/cssfontparser-1.2.1.tgz",
+      "integrity": "sha512-6tun4LoZnj7VN6YeegOVb67KBX/7JJsqvj+pv3ZA7F878/eN33AbGa5b/S/wXxS/tcp8nc40xRUrsPlxIyNUPg==",
       "dev": true
     },
-    "node_modules/charenc": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
-      "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
+    "node_modules/cssom": {
+      "version": "0.4.4",
+      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
+      "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==",
+      "dev": true
+    },
+    "node_modules/cssstyle": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
+      "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
       "dev": true,
+      "dependencies": {
+        "cssom": "~0.3.6"
+      },
       "engines": {
-        "node": "*"
+        "node": ">=8"
       }
     },
-    "node_modules/chart.js": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.0.tgz",
-      "integrity": "sha512-vQEj6d+z0dcsKLlQvbKIMYFHd3t8W/7L2vfJIbYcfyPcRx92CsHqECpueN8qVGNlKyDcr5wBrYAYKnfu/9Q1hQ==",
+    "node_modules/cssstyle/node_modules/cssom": {
+      "version": "0.3.8",
+      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
+      "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
+      "dev": true
+    },
+    "node_modules/csstype": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz",
+      "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ=="
+    },
+    "node_modules/cucumber": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/cucumber/-/cucumber-4.2.1.tgz",
+      "integrity": "sha512-3gQ0Vv4kSHsvXEFC6b1c+TfLRDzWD1/kU7e5vm8Kh8j35b95k6favan9/4ixcBNqd7UsU1T6FYcawC87+DlNKw==",
+      "deprecated": "Cucumber is publishing new releases under @cucumber/cucumber",
+      "dev": true,
       "dependencies": {
-        "@kurkle/color": "^0.3.0"
+        "assertion-error-formatter": "^2.0.1",
+        "babel-runtime": "^6.11.6",
+        "bluebird": "^3.4.1",
+        "cli-table": "^0.3.1",
+        "colors": "^1.1.2",
+        "commander": "^2.9.0",
+        "cucumber-expressions": "^5.0.13",
+        "cucumber-tag-expressions": "^1.1.1",
+        "duration": "^0.2.0",
+        "escape-string-regexp": "^1.0.5",
+        "figures": "2.0.0",
+        "gherkin": "^5.0.0",
+        "glob": "^7.0.0",
+        "indent-string": "^3.1.0",
+        "is-generator": "^1.0.2",
+        "is-stream": "^1.1.0",
+        "knuth-shuffle-seeded": "^1.0.6",
+        "lodash": "^4.17.4",
+        "mz": "^2.4.0",
+        "progress": "^2.0.0",
+        "resolve": "^1.3.3",
+        "serialize-error": "^2.1.0",
+        "stack-chain": "^2.0.0",
+        "stacktrace-js": "^2.0.0",
+        "string-argv": "0.0.2",
+        "title-case": "^2.1.1",
+        "util-arity": "^1.0.2",
+        "verror": "^1.9.0"
+      },
+      "bin": {
+        "cucumber-js": "bin/cucumber-js"
       },
       "engines": {
-        "pnpm": ">=7"
+        "node": ">=0.10"
       }
     },
-    "node_modules/chartjs-adapter-moment": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/chartjs-adapter-moment/-/chartjs-adapter-moment-1.0.1.tgz",
-      "integrity": "sha512-Uz+nTX/GxocuqXpGylxK19YG4R3OSVf8326D+HwSTsNw1LgzyIGRo+Qujwro1wy6X+soNSnfj5t2vZ+r6EaDmA==",
-      "peerDependencies": {
-        "chart.js": ">=3.0.0",
-        "moment": "^2.10.2"
+    "node_modules/cucumber-expressions": {
+      "version": "6.6.2",
+      "resolved": "https://registry.npmjs.org/cucumber-expressions/-/cucumber-expressions-6.6.2.tgz",
+      "integrity": "sha512-WcFSVBiWNLJbIcAAC3t/ACU46vaOKfe1UIF5H3qveoq+Y4XQm9j3YwHurQNufRKBBg8nCnpU7Ttsx7egjS3hwA==",
+      "deprecated": "This package is now published under @cucumber/cucumber-expressions",
+      "dev": true,
+      "dependencies": {
+        "becke-ch--regex--s0-0-v1--base--pl--lib": "^1.2.0"
       }
     },
-    "node_modules/check-error": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz",
-      "integrity": "sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==",
+    "node_modules/cucumber-messages": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/cucumber-messages/-/cucumber-messages-8.0.0.tgz",
+      "integrity": "sha512-lUnWRMjwA9+KhDec/5xRZV3Du67ISumHnVLywWQXyvzmc4P+Eqx8CoeQrBQoau3Pw1hs4kJLTDyV85hFBF00SQ==",
+      "deprecated": "This package is now published under @cucumber/messages",
       "dev": true,
-      "engines": {
-        "node": "*"
+      "dependencies": {
+        "@types/uuid": "^3.4.6",
+        "protobufjs": "^6.8.8",
+        "uuid": "^3.3.3"
       }
     },
-    "node_modules/check-more-types": {
-      "version": "2.24.0",
-      "resolved": "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz",
-      "integrity": "sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA==",
+    "node_modules/cucumber-messages/node_modules/uuid": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
       "dev": true,
-      "engines": {
-        "node": ">= 0.8.0"
+      "bin": {
+        "uuid": "bin/uuid"
       }
     },
-    "node_modules/cheerio": {
-      "version": "1.0.0-rc.12",
-      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz",
-      "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==",
+    "node_modules/cucumber-tag-expressions": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/cucumber-tag-expressions/-/cucumber-tag-expressions-1.1.1.tgz",
+      "integrity": "sha512-V9jv81sR/HaJ87FoidrvHkviXId7KmBcUi7aQPfi+W3nRO30N6GqH6lcp8K+nyiT1DgemRJBPDDeBMS93xJqMQ==",
+      "dev": true
+    },
+    "node_modules/cucumber/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "cheerio-select": "^2.1.0",
-        "dom-serializer": "^2.0.0",
-        "domhandler": "^5.0.3",
-        "domutils": "^3.0.1",
-        "htmlparser2": "^8.0.1",
-        "parse5": "^7.0.0",
-        "parse5-htmlparser2-tree-adapter": "^7.0.0"
-      },
-      "engines": {
-        "node": ">= 6"
-      },
-      "funding": {
-        "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
-    },
-    "node_modules/cheerio-select": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
-      "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+    },
+    "node_modules/cucumber/node_modules/cucumber-expressions": {
+      "version": "5.0.18",
+      "resolved": "https://registry.npmjs.org/cucumber-expressions/-/cucumber-expressions-5.0.18.tgz",
+      "integrity": "sha512-cj9UKCEvsB7bN97THmowcZt8I3rYFbTAFBNeDpKmWW3vr43CLZeWBmbk7NlHijndLwPJ7+uiF72xWrRU+RLyZA==",
+      "deprecated": "This package is now published under @cucumber/cucumber-expressions",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "boolbase": "^1.0.0",
-        "css-select": "^5.1.0",
-        "css-what": "^6.1.0",
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.3",
-        "domutils": "^3.0.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
+        "becke-ch--regex--s0-0-v1--base--pl--lib": "^1.2.0"
       }
     },
-    "node_modules/cheerio/node_modules/parse5": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
-      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
+    "node_modules/cucumber/node_modules/figures": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
+      "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==",
       "dev": true,
       "dependencies": {
-        "entities": "^4.4.0"
+        "escape-string-regexp": "^1.0.5"
       },
-      "funding": {
-        "url": "https://github.com/inikulin/parse5?sponsor=1"
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/chokidar": {
-      "version": "3.5.3",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
-      "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
+    "node_modules/cucumber/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
-      "funding": [
-        {
-          "type": "individual",
-          "url": "https://paulmillr.com/funding/"
-        }
-      ],
       "dependencies": {
-        "anymatch": "~3.1.2",
-        "braces": "~3.0.2",
-        "glob-parent": "~5.1.2",
-        "is-binary-path": "~2.1.0",
-        "is-glob": "~4.0.1",
-        "normalize-path": "~3.0.0",
-        "readdirp": "~3.6.0"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">= 8.10.0"
+        "node": "*"
       },
-      "optionalDependencies": {
-        "fsevents": "~2.3.2"
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/chownr": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
-      "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+    "node_modules/cucumber/node_modules/indent-string": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
+      "integrity": "sha512-BYqTHXTGUIvg7t1r4sJNKcbDZkL92nkXA8YtRpbjFHRHGDL/NtUeiBJMeE60kIFN/Mg8ESaWQvftaYMGJzQZCQ==",
       "dev": true,
       "engines": {
-        "node": ">=10"
+        "node": ">=4"
       }
     },
-    "node_modules/chrome-trace-event": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
-      "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==",
+    "node_modules/cucumber/node_modules/is-stream": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+      "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==",
       "dev": true,
       "engines": {
-        "node": ">=6.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/ci-info": {
-      "version": "3.8.0",
-      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz",
-      "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==",
+    "node_modules/cucumber/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/sibiraj-s"
-        }
-      ],
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
       "engines": {
-        "node": ">=8"
+        "node": "*"
       }
     },
-    "node_modules/cipher-base": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
-      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
+    "node_modules/cucumber/node_modules/serialize-error": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-2.1.0.tgz",
+      "integrity": "sha512-ghgmKt5o4Tly5yEG/UJp8qTd0AN7Xalw4XBtDEKP655B699qMEtra1WlXeE6WIvdEG481JvRxULKsInq/iNysw==",
       "dev": true,
-      "dependencies": {
-        "inherits": "^2.0.1",
-        "safe-buffer": "^5.0.1"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/cjs-module-lexer": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz",
-      "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==",
-      "dev": true
-    },
-    "node_modules/cjson": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cjson/-/cjson-0.5.0.tgz",
-      "integrity": "sha512-D3CKJU9YnZNyerUQ1IzNUvMnToP3MGC2XbIAPi/7yqunJJW3rBwCVapousoFtaR9IbejeEM0KIshxC1n4HQcXw==",
+    "node_modules/cypress": {
+      "version": "12.17.4",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.4.tgz",
+      "integrity": "sha512-gAN8Pmns9MA5eCDFSDJXWKUpaL3IDd89N9TtIupjYnzLSmlpVr+ZR+vb4U/qaMp+lB6tBvAmt7504c3Z4RU5KQ==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "json-parse-helpfulerror": "^1.0.3"
+        "@cypress/request": "2.88.12",
+        "@cypress/xvfb": "^1.2.4",
+        "@types/node": "^16.18.39",
+        "@types/sinonjs__fake-timers": "8.1.1",
+        "@types/sizzle": "^2.3.2",
+        "arch": "^2.2.0",
+        "blob-util": "^2.0.2",
+        "bluebird": "^3.7.2",
+        "buffer": "^5.6.0",
+        "cachedir": "^2.3.0",
+        "chalk": "^4.1.0",
+        "check-more-types": "^2.24.0",
+        "cli-cursor": "^3.1.0",
+        "cli-table3": "~0.6.1",
+        "commander": "^6.2.1",
+        "common-tags": "^1.8.0",
+        "dayjs": "^1.10.4",
+        "debug": "^4.3.4",
+        "enquirer": "^2.3.6",
+        "eventemitter2": "6.4.7",
+        "execa": "4.1.0",
+        "executable": "^4.1.1",
+        "extract-zip": "2.0.1",
+        "figures": "^3.2.0",
+        "fs-extra": "^9.1.0",
+        "getos": "^3.2.1",
+        "is-ci": "^3.0.0",
+        "is-installed-globally": "~0.4.0",
+        "lazy-ass": "^1.6.0",
+        "listr2": "^3.8.3",
+        "lodash": "^4.17.21",
+        "log-symbols": "^4.0.0",
+        "minimist": "^1.2.8",
+        "ospath": "^1.2.2",
+        "pretty-bytes": "^5.6.0",
+        "process": "^0.11.10",
+        "proxy-from-env": "1.0.0",
+        "request-progress": "^3.0.0",
+        "semver": "^7.5.3",
+        "supports-color": "^8.1.1",
+        "tmp": "~0.2.1",
+        "untildify": "^4.0.0",
+        "yauzl": "^2.10.0"
+      },
+      "bin": {
+        "cypress": "bin/cypress"
       },
       "engines": {
-        "node": ">= 0.3.0"
+        "node": "^14.0.0 || ^16.0.0 || >=18.0.0"
       }
     },
-    "node_modules/class-utils": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
-      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
+    "node_modules/cypress-axe": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/cypress-axe/-/cypress-axe-1.5.0.tgz",
+      "integrity": "sha512-Hy/owCjfj+25KMsecvDgo4fC/781ccL+e8p+UUYoadGVM2ogZF9XIKbiM6KI8Y3cEaSreymdD6ZzccbI2bY0lQ==",
       "dev": true,
-      "dependencies": {
-        "arr-union": "^3.1.0",
-        "define-property": "^0.2.5",
-        "isobject": "^3.0.0",
-        "static-extend": "^0.1.1"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "axe-core": "^3 || ^4",
+        "cypress": "^10 || ^11 || ^12 || ^13"
       }
     },
-    "node_modules/class-utils/node_modules/define-property": {
-      "version": "0.2.5",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-      "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==",
+    "node_modules/cypress-cucumber-preprocessor": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-4.3.1.tgz",
+      "integrity": "sha512-BKUYXqoTeKzkPXohEczDtpAwRCY0ZPtIpfRwJut16yNLqdXQMV+aItwanxe3cbJTFlwg562NAjL4LMdiOhkAjg==",
       "dev": true,
       "dependencies": {
-        "is-descriptor": "^0.1.0"
+        "@cypress/browserify-preprocessor": "^3.0.2",
+        "chai": "^4.2.0",
+        "chokidar": "3.5.2",
+        "cosmiconfig": "^4.0.0",
+        "cucumber": "^4.2.1",
+        "cucumber-expressions": "^6.0.1",
+        "cucumber-tag-expressions": "^1.1.1",
+        "dargs": "^7.0.0",
+        "debug": "^3.0.1",
+        "gherkin": "^5.1.0",
+        "glob": "^7.1.2",
+        "js-string-escape": "^1.0.1",
+        "minimist": "^1.2.5",
+        "through": "^2.3.8"
       },
-      "engines": {
-        "node": ">=0.10.0"
+      "bin": {
+        "cypress-tags": "cypress-tags.js"
       }
     },
-    "node_modules/class-utils/node_modules/is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "sprintf-js": "~1.0.2"
       }
     },
-    "node_modules/class-utils/node_modules/is-accessor-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/cypress-cucumber-preprocessor/node_modules/chokidar": {
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.2.tgz",
+      "integrity": "sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==",
+      "dev": true,
+      "dependencies": {
+        "anymatch": "~3.1.2",
+        "braces": "~3.0.2",
+        "glob-parent": "~5.1.2",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.6.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 8.10.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
       }
     },
-    "node_modules/class-utils/node_modules/is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/cosmiconfig": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-4.0.0.tgz",
+      "integrity": "sha512-6e5vDdrXZD+t5v0L8CrurPeybg4Fmf+FCSYxXKYVAqLUtyCSbuyqE059d0kDthTNRzKVjL7QMgNpEUlsoYH3iQ==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^3.0.2"
+        "is-directory": "^0.3.1",
+        "js-yaml": "^3.9.0",
+        "parse-json": "^4.0.0",
+        "require-from-string": "^2.0.1"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4"
       }
     },
-    "node_modules/class-utils/node_modules/is-data-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "ms": "^2.1.1"
       }
     },
-    "node_modules/class-utils/node_modules/is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/class-utils/node_modules/kind-of": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-      "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/js-yaml": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
       "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+      "dependencies": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
       }
     },
-    "node_modules/classnames": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz",
-      "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw=="
-    },
-    "node_modules/clean-stack": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
-      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
       "engines": {
-        "node": ">=6"
+        "node": "*"
       }
     },
-    "node_modules/cli-cursor": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz",
-      "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==",
+    "node_modules/cypress-cucumber-preprocessor/node_modules/parse-json": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
+      "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==",
       "dev": true,
       "dependencies": {
-        "restore-cursor": "^3.1.0"
+        "error-ex": "^1.3.1",
+        "json-parse-better-errors": "^1.0.1"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=4"
       }
     },
-    "node_modules/cli-spinners": {
-      "version": "2.9.1",
-      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.1.tgz",
-      "integrity": "sha512-jHgecW0pxkonBJdrKsqxgRX9AcG+u/5k0Q7WPDfi8AogLAdwxEkyYYNWwZ5GvVFoFx2uiY1eNcSK00fh+1+FyQ==",
+    "node_modules/cypress-iframe": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cypress-iframe/-/cypress-iframe-1.0.1.tgz",
+      "integrity": "sha512-Ne+xkZmWMhfq3x6wbfzK/SzsVTCrJru3R3cLXsoSAZyfUtJDamXyaIieHXeea3pQDXF4wE2w4iUuvCYHhoD31g==",
       "dev": true,
-      "engines": {
-        "node": ">=6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "peerDependencies": {
+        "@types/cypress": "^1.1.0"
       }
     },
-    "node_modules/cli-table": {
-      "version": "0.3.11",
-      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.11.tgz",
-      "integrity": "sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ==",
+    "node_modules/cypress-multi-reporters": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/cypress-multi-reporters/-/cypress-multi-reporters-1.5.0.tgz",
+      "integrity": "sha512-6rJ1rk1RpjZwTeydCDc8r3iOmWj2ZEYo++oDTJHNEu7eetb3W1cYDNo5CdxF/r0bo7TLQsOEpBHOCYBZfPVt/g==",
       "dev": true,
       "dependencies": {
-        "colors": "1.0.3"
+        "debug": "^4.1.1",
+        "lodash": "^4.17.15"
       },
       "engines": {
-        "node": ">= 0.2.0"
+        "node": ">=6.0.0"
+      },
+      "peerDependencies": {
+        "mocha": ">=3.1.2"
       }
     },
-    "node_modules/cli-table/node_modules/colors": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-      "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.1.90"
-      }
+    "node_modules/cypress/node_modules/@types/node": {
+      "version": "16.18.62",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.62.tgz",
+      "integrity": "sha512-/zbPnIBkef8sT+6vw6BxdvU3dCxRI0v6rBu/6IvXnRNtOPILucigqhUBPYxtQ/8JdAna0JLTAcNTCDmQ77QYkQ==",
+      "dev": true
     },
-    "node_modules/cli-table3": {
-      "version": "0.6.3",
-      "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz",
-      "integrity": "sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==",
+    "node_modules/cypress/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "string-width": "^4.2.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "10.* || >= 12.*"
+        "node": ">=8"
       },
-      "optionalDependencies": {
-        "@colors/colors": "1.5.0"
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/cli-truncate": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz",
-      "integrity": "sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==",
+    "node_modules/cypress/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "slice-ansi": "^3.0.0",
-        "string-width": "^4.2.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/cli-width": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz",
-      "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==",
+    "node_modules/cypress/node_modules/chalk/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": ">= 10"
+        "node": ">=8"
       }
     },
-    "node_modules/cliui": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
-      "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
+    "node_modules/cypress/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
       "dependencies": {
-        "string-width": "^4.2.0",
-        "strip-ansi": "^6.0.1",
-        "wrap-ansi": "^7.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/clone": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
-      "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==",
+    "node_modules/cypress/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/cypress/node_modules/commander": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
+      "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==",
       "dev": true,
       "engines": {
-        "node": ">=0.8"
+        "node": ">= 6"
       }
     },
-    "node_modules/clone-deep": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
-      "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
+    "node_modules/cypress/node_modules/execa": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
+      "integrity": "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==",
       "dev": true,
       "dependencies": {
-        "is-plain-object": "^2.0.4",
-        "kind-of": "^6.0.2",
-        "shallow-clone": "^3.0.0"
+        "cross-spawn": "^7.0.0",
+        "get-stream": "^5.0.0",
+        "human-signals": "^1.1.1",
+        "is-stream": "^2.0.0",
+        "merge-stream": "^2.0.0",
+        "npm-run-path": "^4.0.0",
+        "onetime": "^5.1.0",
+        "signal-exit": "^3.0.2",
+        "strip-final-newline": "^2.0.0"
       },
       "engines": {
-        "node": ">=6"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/execa?sponsor=1"
       }
     },
-    "node_modules/clone-regexp": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/clone-regexp/-/clone-regexp-2.2.0.tgz",
-      "integrity": "sha512-beMpP7BOtTipFuW8hrJvREQ2DrRu3BE7by0ZpibtfBA+qfHYvMGTc2Yb1JMYPKg/JUw0CHYvpg796aNTSW9z7Q==",
+    "node_modules/cypress/node_modules/get-stream": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
       "dev": true,
       "dependencies": {
-        "is-regexp": "^2.0.0"
+        "pump": "^3.0.0"
       },
       "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/co": {
-      "version": "4.6.0",
-      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
-      "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==",
-      "dev": true,
-      "engines": {
-        "iojs": ">= 1.0.0",
-        "node": ">= 0.12.0"
-      }
-    },
-    "node_modules/code-block-writer": {
-      "version": "11.0.3",
-      "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-11.0.3.tgz",
-      "integrity": "sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==",
-      "dev": true
-    },
-    "node_modules/code-point-at": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
-      "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==",
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/cypress/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/coffeeify": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/coffeeify/-/coffeeify-3.0.1.tgz",
-      "integrity": "sha512-Qjnr7UX6ldK1PHV7wCnv7AuCd4q19KTUtwJnu/6JRJB4rfm12zvcXtKdacUoePOKr1I4ka/ydKiwWpNAdsQb0g==",
+    "node_modules/cypress/node_modules/human-signals": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
+      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
       "dev": true,
-      "dependencies": {
-        "convert-source-map": "^1.3.0",
-        "through2": "^2.0.0"
-      },
-      "peerDependencies": {
-        "coffeescript": ">1.9.2 <3"
+      "engines": {
+        "node": ">=8.12.0"
       }
     },
-    "node_modules/coffeescript": {
-      "version": "1.12.7",
-      "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-1.12.7.tgz",
-      "integrity": "sha512-pLXHFxQMPklVoEekowk8b3erNynC+DVJzChxS/LCBBgR6/8AJkHivkm//zbowcfc7BTCAjryuhx6gPqPRfsFoA==",
+    "node_modules/cypress/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
-      "bin": {
-        "cake": "bin/cake",
-        "coffee": "bin/coffee"
+      "dependencies": {
+        "yallist": "^4.0.0"
       },
       "engines": {
-        "node": ">=0.8.0"
+        "node": ">=10"
       }
     },
-    "node_modules/collect-v8-coverage": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz",
-      "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==",
+    "node_modules/cypress/node_modules/proxy-from-env": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
+      "integrity": "sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A==",
       "dev": true
     },
-    "node_modules/collection-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
-      "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==",
+    "node_modules/cypress/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
-        "map-visit": "^1.0.0",
-        "object-visit": "^1.0.0"
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
       }
     },
-    "node_modules/color-convert": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+    "node_modules/cypress/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+      "dev": true,
       "dependencies": {
-        "color-name": "~1.1.4"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=7.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
       }
     },
-    "node_modules/color-name": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
-    },
-    "node_modules/color-support": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
-      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+    "node_modules/cypress/node_modules/tmp": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
+      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
       "dev": true,
-      "bin": {
-        "color-support": "bin.js"
+      "dependencies": {
+        "rimraf": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8.17.0"
       }
     },
-    "node_modules/colorette": {
-      "version": "2.0.20",
-      "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
-      "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==",
+    "node_modules/cypress/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "dev": true
     },
-    "node_modules/colors": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-      "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==",
+    "node_modules/d": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
+      "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
       "dev": true,
-      "engines": {
-        "node": ">=0.1.90"
+      "dependencies": {
+        "es5-ext": "^0.10.50",
+        "type": "^1.0.1"
       }
     },
-    "node_modules/combine-source-map": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz",
-      "integrity": "sha512-UlxQ9Vw0b/Bt/KYwCFqdEwsQ1eL8d1gibiFb7lxQJFdvTgc2hIZi6ugsg+kyhzhPV+QEpUiEIwInIAIrgoEkrg==",
+    "node_modules/dargs": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/dargs/-/dargs-7.0.0.tgz",
+      "integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==",
       "dev": true,
-      "dependencies": {
-        "convert-source-map": "~1.1.0",
-        "inline-source-map": "~0.6.0",
-        "lodash.memoize": "~3.0.3",
-        "source-map": "~0.5.3"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/combine-source-map/node_modules/convert-source-map": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz",
-      "integrity": "sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg==",
+    "node_modules/dash-ast": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/dash-ast/-/dash-ast-2.0.1.tgz",
+      "integrity": "sha512-5TXltWJGc+RdnabUGzhRae1TRq6m4gr+3K2wQX0is5/F2yS6MJXJvLyI3ErAnsAXuJoGqvfVD5icRgim07DrxQ==",
       "dev": true
     },
-    "node_modules/combine-source-map/node_modules/source-map": {
-      "version": "0.5.7",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
-      "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==",
+    "node_modules/dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==",
       "dev": true,
+      "dependencies": {
+        "assert-plus": "^1.0.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=0.10"
       }
     },
-    "node_modules/combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+    "node_modules/data-urls": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz",
+      "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==",
+      "dev": true,
       "dependencies": {
-        "delayed-stream": "~1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
+        "abab": "^2.0.0",
+        "whatwg-mimetype": "^2.2.0",
+        "whatwg-url": "^7.0.0"
       }
     },
-    "node_modules/comma-separated-tokens": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz",
-      "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+    "node_modules/data-urls/node_modules/tr46": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
+      "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==",
+      "dev": true,
+      "dependencies": {
+        "punycode": "^2.1.0"
       }
     },
-    "node_modules/commander": {
-      "version": "8.3.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz",
-      "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==",
+    "node_modules/data-urls/node_modules/webidl-conversions": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "dev": true
+    },
+    "node_modules/data-urls/node_modules/whatwg-url": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
+      "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==",
       "dev": true,
-      "engines": {
-        "node": ">= 12"
+      "dependencies": {
+        "lodash.sortby": "^4.7.0",
+        "tr46": "^1.0.1",
+        "webidl-conversions": "^4.0.2"
       }
     },
-    "node_modules/common-tags": {
-      "version": "1.8.2",
-      "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz",
-      "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==",
+    "node_modules/dateformat": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
+      "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==",
       "dev": true,
       "engines": {
-        "node": ">=4.0.0"
+        "node": "*"
       }
     },
-    "node_modules/commondir": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
-      "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==",
-      "dev": true
-    },
-    "node_modules/component-emitter": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
-      "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==",
+    "node_modules/dayjs": {
+      "version": "1.11.10",
+      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz",
+      "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==",
       "dev": true
     },
-    "node_modules/compressible": {
-      "version": "2.0.18",
-      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
-      "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
-      "dev": true,
+    "node_modules/debug": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
       "dependencies": {
-        "mime-db": ">= 1.43.0 < 2"
+        "ms": "2.1.2"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
       }
     },
-    "node_modules/compression": {
-      "version": "1.7.4",
-      "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz",
-      "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==",
+    "node_modules/decache": {
+      "version": "4.6.2",
+      "resolved": "https://registry.npmjs.org/decache/-/decache-4.6.2.tgz",
+      "integrity": "sha512-2LPqkLeu8XWHU8qNCS3kcF6sCcb5zIzvWaAHYSvPfwhdd7mHuah29NssMzrTYyHN4F5oFy2ko9OBYxegtU0FEw==",
       "dev": true,
       "dependencies": {
-        "accepts": "~1.3.5",
-        "bytes": "3.0.0",
-        "compressible": "~2.0.16",
-        "debug": "2.6.9",
-        "on-headers": "~1.0.2",
-        "safe-buffer": "5.1.2",
-        "vary": "~1.1.2"
-      },
-      "engines": {
-        "node": ">= 0.8.0"
+        "callsite": "^1.0.0"
       }
     },
-    "node_modules/compression/node_modules/bytes": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
-      "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==",
+    "node_modules/decamelize": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
+      "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
       "dev": true,
       "engines": {
-        "node": ">= 0.8"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/compression/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+    "node_modules/decamelize-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.1.tgz",
+      "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==",
       "dev": true,
       "dependencies": {
-        "ms": "2.0.0"
+        "decamelize": "^1.1.0",
+        "map-obj": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/compression/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/compression/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-      "dev": true
+    "node_modules/decamelize-keys/node_modules/map-obj": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
+      "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
     },
-    "node_modules/concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+    "node_modules/decimal.js": {
+      "version": "10.4.3",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
+      "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==",
       "dev": true
     },
-    "node_modules/concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+    "node_modules/decode-uri-component": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz",
+      "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==",
       "dev": true,
-      "engines": [
-        "node >= 0.8"
-      ],
-      "dependencies": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^2.2.2",
-        "typedarray": "^0.0.6"
+      "engines": {
+        "node": ">=0.10"
       }
     },
-    "node_modules/connect": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz",
-      "integrity": "sha512-ZqRXc+tZukToSNmh5C2iWMSoV3X1YUcPbqEM4DkEG5tNQXrQUZCNVGGv3IuicnkMtPfGf3Xtp8WCXs295iQ1pQ==",
-      "dev": true,
+    "node_modules/decompress-response": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+      "optional": true,
       "dependencies": {
-        "debug": "2.6.9",
-        "finalhandler": "1.1.2",
-        "parseurl": "~1.3.3",
-        "utils-merge": "1.0.1"
+        "mimic-response": "^3.1.0"
       },
       "engines": {
-        "node": ">= 0.10.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/connect-history-api-fallback": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz",
-      "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==",
+    "node_modules/dedent": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.1.tgz",
+      "integrity": "sha512-+LxW+KLWxu3HW3M2w2ympwtqPrqYRzU8fqi6Fhd18fBALe15blJPI/I4+UHveMVG6lJqB4JNd4UG0S5cnVHwIg==",
       "dev": true,
-      "engines": {
-        "node": ">=0.8"
+      "peerDependencies": {
+        "babel-plugin-macros": "^3.1.0"
+      },
+      "peerDependenciesMeta": {
+        "babel-plugin-macros": {
+          "optional": true
+        }
       }
     },
-    "node_modules/connect/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+    "node_modules/deep-eql": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz",
+      "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==",
       "dev": true,
       "dependencies": {
-        "ms": "2.0.0"
+        "type-detect": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=6"
       }
     },
-    "node_modules/connect/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/console-browserify": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz",
-      "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==",
-      "dev": true
-    },
-    "node_modules/console-control-strings": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==",
-      "dev": true
-    },
-    "node_modules/constants-browserify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
-      "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==",
-      "dev": true
-    },
-    "node_modules/content-disposition": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
-      "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
+    "node_modules/deep-equal": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz",
+      "integrity": "sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==",
       "dev": true,
       "dependencies": {
-        "safe-buffer": "5.1.2"
+        "is-arguments": "^1.1.1",
+        "is-date-object": "^1.0.5",
+        "is-regex": "^1.1.4",
+        "object-is": "^1.1.5",
+        "object-keys": "^1.1.1",
+        "regexp.prototype.flags": "^1.5.1"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/content-disposition/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+    "node_modules/deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/deep-is": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
       "dev": true
     },
-    "node_modules/content-type": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
-      "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
-      "dev": true,
+    "node_modules/deepmerge": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
+      "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/convert-source-map": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
-      "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
-    },
-    "node_modules/cookie": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
-      "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==",
+    "node_modules/default-gateway": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz",
+      "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==",
       "dev": true,
+      "dependencies": {
+        "execa": "^5.0.0"
+      },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 10"
       }
     },
-    "node_modules/cookie-signature": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
-      "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==",
-      "dev": true
-    },
-    "node_modules/copy-anything": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz",
-      "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==",
+    "node_modules/defaults": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
+      "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==",
       "dev": true,
       "dependencies": {
-        "is-what": "^3.14.1"
+        "clone": "^1.0.2"
       },
       "funding": {
-        "url": "https://github.com/sponsors/mesqueeb"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/copy-descriptor": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
-      "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==",
-      "dev": true,
+    "node_modules/define-data-property": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz",
+      "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==",
+      "dependencies": {
+        "get-intrinsic": "^1.2.1",
+        "gopd": "^1.0.1",
+        "has-property-descriptors": "^1.0.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 0.4"
       }
     },
-    "node_modules/copy-to-clipboard": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz",
-      "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==",
-      "dependencies": {
-        "toggle-selection": "^1.0.6"
+    "node_modules/define-lazy-prop": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
+      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/copy-webpack-plugin": {
-      "version": "11.0.0",
-      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz",
-      "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==",
+    "node_modules/define-properties": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz",
+      "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==",
       "dev": true,
       "dependencies": {
-        "fast-glob": "^3.2.11",
-        "glob-parent": "^6.0.1",
-        "globby": "^13.1.1",
-        "normalize-path": "^3.0.0",
-        "schema-utils": "^4.0.0",
-        "serialize-javascript": "^6.0.0"
+        "define-data-property": "^1.0.1",
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
       },
       "engines": {
-        "node": ">= 14.15.0"
+        "node": ">= 0.4"
       },
       "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      },
-      "peerDependencies": {
-        "webpack": "^5.1.0"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/copy-webpack-plugin/node_modules/glob-parent": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
-      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+    "node_modules/define-property": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
+      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
       "dev": true,
       "dependencies": {
-        "is-glob": "^4.0.3"
+        "is-descriptor": "^1.0.2",
+        "isobject": "^3.0.1"
       },
       "engines": {
-        "node": ">=10.13.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/copy-webpack-plugin/node_modules/globby": {
-      "version": "13.2.2",
-      "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz",
-      "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==",
+    "node_modules/define-property/node_modules/is-descriptor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.3.tgz",
+      "integrity": "sha512-JCNNGbwWZEVaSPtS45mdtrneRWJFp07LLmykxeFV5F6oBvNF8vHSfJuJgoT472pSfk+Mf8VnlrspaFBHWM8JAw==",
       "dev": true,
       "dependencies": {
-        "dir-glob": "^3.0.1",
-        "fast-glob": "^3.3.0",
-        "ignore": "^5.2.4",
-        "merge2": "^1.4.1",
-        "slash": "^4.0.0"
+        "is-accessor-descriptor": "^1.0.1",
+        "is-data-descriptor": "^1.0.1"
       },
       "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">= 0.4"
       }
     },
-    "node_modules/copy-webpack-plugin/node_modules/ignore": {
-      "version": "5.2.4",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",
-      "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==",
+    "node_modules/defined": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.1.tgz",
+      "integrity": "sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==",
       "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
       "engines": {
-        "node": ">= 4"
+        "node": ">=0.4.0"
       }
     },
-    "node_modules/copy-webpack-plugin/node_modules/slash": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
-      "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==",
+    "node_modules/delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+      "dev": true
+    },
+    "node_modules/depd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
       "dev": true,
       "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/core-js": {
-      "version": "2.6.12",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
-      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==",
-      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
+    "node_modules/dependency-graph": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz",
+      "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==",
       "dev": true,
-      "hasInstallScript": true
+      "engines": {
+        "node": ">= 0.6.0"
+      }
     },
-    "node_modules/core-js-compat": {
-      "version": "3.32.2",
-      "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.32.2.tgz",
-      "integrity": "sha512-+GjlguTDINOijtVRUxrQOv3kfu9rl+qPNdX2LTbJ/ZyVTuxK+ksVSAGX1nHstu4hrv1En/uPTtWgq2gI5wt4AQ==",
+    "node_modules/deps-sort": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/deps-sort/-/deps-sort-2.0.1.tgz",
+      "integrity": "sha512-1orqXQr5po+3KI6kQb9A4jnXT1PBwggGl2d7Sq2xsnOeI9GPcE/tGcF9UiSZtZBM7MukY4cAh7MemS6tZYipfw==",
       "dev": true,
       "dependencies": {
-        "browserslist": "^4.21.10"
+        "JSONStream": "^1.0.3",
+        "shasum-object": "^1.0.0",
+        "subarg": "^1.0.0",
+        "through2": "^2.0.0"
       },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
+      "bin": {
+        "deps-sort": "bin/cmd.js"
       }
     },
-    "node_modules/core-js-compat/node_modules/browserslist": {
-      "version": "4.21.11",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.11.tgz",
-      "integrity": "sha512-xn1UXOKUz7DjdGlg9RrUr0GGiWzI97UQJnugHtH0OLDfJB7jMgoIkYvRIEO1l9EeEERVqeqLYOcFBW9ldjypbQ==",
+    "node_modules/des.js": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz",
+      "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==",
       "dev": true,
-      "funding": [
-        {
-          "type": "opencollective",
-          "url": "https://opencollective.com/browserslist"
-        },
-        {
-          "type": "tidelift",
-          "url": "https://tidelift.com/funding/github/npm/browserslist"
-        },
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/ai"
-        }
-      ],
       "dependencies": {
-        "caniuse-lite": "^1.0.30001538",
-        "electron-to-chromium": "^1.4.526",
-        "node-releases": "^2.0.13",
-        "update-browserslist-db": "^1.0.13"
-      },
-      "bin": {
-        "browserslist": "cli.js"
-      },
-      "engines": {
-        "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0"
       }
     },
-    "node_modules/core-js-pure": {
-      "version": "3.32.2",
-      "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.32.2.tgz",
-      "integrity": "sha512-Y2rxThOuNywTjnX/PgA5vWM6CZ9QB9sz9oGeCixV8MqXZO70z/5SHzf9EeBrEBK0PN36DnEBBu9O/aGWzKuMZQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
+    "node_modules/destroy": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
       }
     },
-    "node_modules/core-util-is": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
-      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
-      "dev": true
+    "node_modules/detect-browser": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/detect-browser/-/detect-browser-5.2.0.tgz",
+      "integrity": "sha512-tr7XntDAu50BVENgQfajMLzacmSe34D+qZc4zjnniz0ZVuw/TZcLcyxHQjYpJTM36sGEkZZlYLnIM1hH7alTMA=="
     },
-    "node_modules/cors": {
-      "version": "2.8.5",
-      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
-      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
+    "node_modules/detect-file": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
+      "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==",
       "dev": true,
-      "dependencies": {
-        "object-assign": "^4",
-        "vary": "^1"
-      },
       "engines": {
-        "node": ">= 0.10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/cosmiconfig": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
-      "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
-      "dev": true,
-      "dependencies": {
-        "@types/parse-json": "^4.0.0",
-        "import-fresh": "^3.1.0",
-        "parse-json": "^5.0.0",
-        "path-type": "^4.0.0",
-        "yaml": "^1.7.2"
-      },
+    "node_modules/detect-libc": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
+      "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
+      "optional": true,
       "engines": {
         "node": ">=8"
       }
     },
-    "node_modules/create-ecdh": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
-      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
+    "node_modules/detect-newline": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
+      "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==",
       "dev": true,
-      "dependencies": {
-        "bn.js": "^4.1.0",
-        "elliptic": "^6.5.3"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/create-ecdh/node_modules/bn.js": {
-      "version": "4.12.0",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
-      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
+    "node_modules/detect-node": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
+      "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
       "dev": true
     },
-    "node_modules/create-hash": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
-      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
+    "node_modules/detective": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/detective/-/detective-5.2.1.tgz",
+      "integrity": "sha512-v9XE1zRnz1wRtgurGu0Bs8uHKFSTdteYZNbIPFVhUZ39L/S79ppMpdmVOZAnoz1jfEFodc48n6MX483Xo3t1yw==",
       "dev": true,
       "dependencies": {
-        "cipher-base": "^1.0.1",
-        "inherits": "^2.0.1",
-        "md5.js": "^1.3.4",
-        "ripemd160": "^2.0.1",
-        "sha.js": "^2.4.0"
+        "acorn-node": "^1.8.2",
+        "defined": "^1.0.0",
+        "minimist": "^1.2.6"
+      },
+      "bin": {
+        "detective": "bin/detective.js"
+      },
+      "engines": {
+        "node": ">=0.8.0"
       }
     },
-    "node_modules/create-hmac": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
-      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
-      "dev": true,
-      "dependencies": {
-        "cipher-base": "^1.0.3",
-        "create-hash": "^1.1.0",
-        "inherits": "^2.0.1",
-        "ripemd160": "^2.0.0",
-        "safe-buffer": "^5.0.1",
-        "sha.js": "^2.4.8"
-      }
+    "node_modules/dfa": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/dfa/-/dfa-1.2.0.tgz",
+      "integrity": "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q==",
+      "dev": true
     },
-    "node_modules/create-jest": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz",
-      "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==",
+    "node_modules/diff": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
+      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.3.1"
+      }
+    },
+    "node_modules/diff-sequences": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz",
+      "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==",
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "chalk": "^4.0.0",
-        "exit": "^0.1.2",
-        "graceful-fs": "^4.2.9",
-        "jest-config": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "prompts": "^2.0.1"
-      },
-      "bin": {
-        "create-jest": "bin/create-jest.js"
-      },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/create-jest/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/diffie-hellman": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
+      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "bn.js": "^4.1.0",
+        "miller-rabin": "^4.0.0",
+        "randombytes": "^2.0.0"
       }
     },
-    "node_modules/critters": {
-      "version": "0.0.16",
-      "resolved": "https://registry.npmjs.org/critters/-/critters-0.0.16.tgz",
-      "integrity": "sha512-JwjgmO6i3y6RWtLYmXwO5jMd+maZt8Tnfu7VVISmEWyQqfLpB8soBswf8/2bu6SBXxtKA68Al3c+qIG1ApT68A==",
+    "node_modules/diffie-hellman/node_modules/bn.js": {
+      "version": "4.12.0",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
+      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
+      "dev": true
+    },
+    "node_modules/dir-glob": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+      "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
       "dev": true,
       "dependencies": {
-        "chalk": "^4.1.0",
-        "css-select": "^4.2.0",
-        "parse5": "^6.0.1",
-        "parse5-htmlparser2-tree-adapter": "^6.0.1",
-        "postcss": "^8.3.7",
-        "pretty-bytes": "^5.3.0"
+        "path-type": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/critters/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/dns-equal": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz",
+      "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==",
+      "dev": true
+    },
+    "node_modules/dns-packet": {
+      "version": "5.6.1",
+      "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
+      "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "@leichtgewicht/ip-codec": "^2.0.1"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=6"
       }
     },
-    "node_modules/critters/node_modules/css-select": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz",
-      "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==",
+    "node_modules/doctrine": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+      "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
       "dev": true,
       "dependencies": {
-        "boolbase": "^1.0.0",
-        "css-what": "^6.0.1",
-        "domhandler": "^4.3.1",
-        "domutils": "^2.8.0",
-        "nth-check": "^2.0.1"
+        "esutils": "^2.0.2"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
+      "engines": {
+        "node": ">=6.0.0"
       }
     },
-    "node_modules/critters/node_modules/dom-serializer": {
+    "node_modules/dom-serializer": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz",
       "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==",
@@ -10637,7 +13899,45 @@
         "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
       }
     },
-    "node_modules/critters/node_modules/domhandler": {
+    "node_modules/domain-browser": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
+      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.4",
+        "npm": ">=1.2"
+      }
+    },
+    "node_modules/domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ]
+    },
+    "node_modules/domexception": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz",
+      "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==",
+      "deprecated": "Use your platform's native DOMException instead",
+      "dev": true,
+      "dependencies": {
+        "webidl-conversions": "^4.0.2"
+      }
+    },
+    "node_modules/domexception/node_modules/webidl-conversions": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "dev": true
+    },
+    "node_modules/domhandler": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz",
       "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==",
@@ -10652,7 +13952,19 @@
         "url": "https://github.com/fb55/domhandler?sponsor=1"
       }
     },
-    "node_modules/critters/node_modules/domutils": {
+    "node_modules/dommatrix": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/dommatrix/-/dommatrix-1.0.3.tgz",
+      "integrity": "sha512-l32Xp/TLgWb8ReqbVJAFIvXmY7go4nTxxlWiAFyhoQw9RKEOHBZNnyGvJWqDVSPmq3Y9HlM4npqF/T6VMOXhww==",
+      "deprecated": "dommatrix is no longer maintained. Please use @thednp/dommatrix.",
+      "dev": true
+    },
+    "node_modules/dompurify": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.3.tgz",
+      "integrity": "sha512-dqnqRkPMAjOZE0FogZ+ceJNM2dZ3V/yNOuFB7+39qpO93hHhfRpHw3heYQC7DPK9FqbQTfBKUJhiSfz4MvXYwg=="
+    },
+    "node_modules/domutils": {
       "version": "2.8.0",
       "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz",
       "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==",
@@ -10666,937 +13978,836 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
-    "node_modules/critters/node_modules/entities": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
-      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
-      "dev": true,
-      "funding": {
-        "url": "https://github.com/fb55/entities?sponsor=1"
-      }
-    },
-    "node_modules/critters/node_modules/parse5": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
-      "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==",
+    "node_modules/dot": {
+      "version": "2.0.0-beta.1",
+      "resolved": "https://registry.npmjs.org/dot/-/dot-2.0.0-beta.1.tgz",
+      "integrity": "sha512-kxM7fSnNQTXOmaeGuBSXM8O3fEsBb7XSDBllkGbRwa0lJSJTxxDE/4eSNGLKZUmlFw0f1vJ5qSV2BljrgQtgIA==",
       "dev": true
     },
-    "node_modules/critters/node_modules/parse5-htmlparser2-tree-adapter": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz",
-      "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==",
-      "dev": true,
-      "dependencies": {
-        "parse5": "^6.0.1"
-      }
-    },
-    "node_modules/cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
-      "dev": true,
-      "dependencies": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/crypt": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
-      "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-      "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/crypto-browserify": {
-      "version": "3.12.0",
-      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
-      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
+    "node_modules/dotenv": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz",
+      "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==",
       "dev": true,
-      "dependencies": {
-        "browserify-cipher": "^1.0.0",
-        "browserify-sign": "^4.0.0",
-        "create-ecdh": "^4.0.0",
-        "create-hash": "^1.1.0",
-        "create-hmac": "^1.1.0",
-        "diffie-hellman": "^5.0.0",
-        "inherits": "^2.0.1",
-        "pbkdf2": "^3.0.3",
-        "public-encrypt": "^4.0.0",
-        "randombytes": "^2.0.0",
-        "randomfill": "^1.0.3"
-      },
       "engines": {
-        "node": "*"
+        "node": ">=10"
       }
     },
-    "node_modules/crypto-js": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz",
-      "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==",
-      "dev": true
-    },
-    "node_modules/css-loader": {
-      "version": "6.7.3",
-      "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.7.3.tgz",
-      "integrity": "sha512-qhOH1KlBMnZP8FzRO6YCH9UHXQhVMcEGLyNdb7Hv2cpcmJbW0YrddO+tG1ab5nT41KpHIYGsbeHqxB9xPu1pKQ==",
-      "dev": true,
-      "dependencies": {
-        "icss-utils": "^5.1.0",
-        "postcss": "^8.4.19",
-        "postcss-modules-extract-imports": "^3.0.0",
-        "postcss-modules-local-by-default": "^4.0.0",
-        "postcss-modules-scope": "^3.0.0",
-        "postcss-modules-values": "^4.0.0",
-        "postcss-value-parser": "^4.2.0",
-        "semver": "^7.3.8"
-      },
+    "node_modules/drange": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/drange/-/drange-1.1.1.tgz",
+      "integrity": "sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA==",
       "engines": {
-        "node": ">= 12.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      },
-      "peerDependencies": {
-        "webpack": "^5.0.0"
+        "node": ">=4"
       }
     },
-    "node_modules/css-loader/node_modules/postcss-value-parser": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
-      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
+    "node_modules/duplexer": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
+      "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
       "dev": true
     },
-    "node_modules/css-select": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
-      "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
+    "node_modules/duplexer2": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
+      "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==",
       "dev": true,
       "dependencies": {
-        "boolbase": "^1.0.0",
-        "css-what": "^6.1.0",
-        "domhandler": "^5.0.2",
-        "domutils": "^3.0.1",
-        "nth-check": "^2.0.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
+        "readable-stream": "^2.0.2"
       }
     },
-    "node_modules/css-tree": {
-      "version": "1.0.0-alpha.39",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.39.tgz",
-      "integrity": "sha512-7UvkEYgBAHRG9Nt980lYxjsTrCyHFN53ky3wVsDkiMdVqylqRt+Zc+jm5qw7/qyOvN2dHSYtX0e4MbCCExSvnA==",
+    "node_modules/duplexer2/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
       "dev": true,
       "dependencies": {
-        "mdn-data": "2.0.6",
-        "source-map": "^0.6.1"
-      },
-      "engines": {
-        "node": ">=8.0.0"
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
       }
     },
-    "node_modules/css-tree/node_modules/source-map": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
+    "node_modules/duplexer2/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
     },
-    "node_modules/css-what": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
-      "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
+    "node_modules/duplexer2/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
       "dev": true,
-      "engines": {
-        "node": ">= 6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/fb55"
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/css.escape": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz",
-      "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg=="
-    },
-    "node_modules/cssesc": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
-      "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==",
+    "node_modules/duration": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/duration/-/duration-0.2.2.tgz",
+      "integrity": "sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==",
       "dev": true,
-      "bin": {
-        "cssesc": "bin/cssesc"
-      },
-      "engines": {
-        "node": ">=4"
+      "dependencies": {
+        "d": "1",
+        "es5-ext": "~0.10.46"
       }
     },
-    "node_modules/cssfontparser": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/cssfontparser/-/cssfontparser-1.2.1.tgz",
-      "integrity": "sha512-6tun4LoZnj7VN6YeegOVb67KBX/7JJsqvj+pv3ZA7F878/eN33AbGa5b/S/wXxS/tcp8nc40xRUrsPlxIyNUPg==",
-      "dev": true
-    },
-    "node_modules/cssom": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
-      "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==",
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
       "dev": true
     },
-    "node_modules/cssstyle": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
-      "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
+    "node_modules/ecc-jsbn": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
+      "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==",
       "dev": true,
       "dependencies": {
-        "cssom": "~0.3.6"
-      },
-      "engines": {
-        "node": ">=8"
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.1.0"
       }
     },
-    "node_modules/cssstyle/node_modules/cssom": {
-      "version": "0.3.8",
-      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
-      "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
+    "node_modules/ee-first": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+      "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
       "dev": true
     },
-    "node_modules/csstype": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz",
-      "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ=="
-    },
-    "node_modules/cucumber": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/cucumber/-/cucumber-4.2.1.tgz",
-      "integrity": "sha512-3gQ0Vv4kSHsvXEFC6b1c+TfLRDzWD1/kU7e5vm8Kh8j35b95k6favan9/4ixcBNqd7UsU1T6FYcawC87+DlNKw==",
-      "deprecated": "Cucumber is publishing new releases under @cucumber/cucumber",
+    "node_modules/ejs": {
+      "version": "3.1.9",
+      "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz",
+      "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==",
       "dev": true,
       "dependencies": {
-        "assertion-error-formatter": "^2.0.1",
-        "babel-runtime": "^6.11.6",
-        "bluebird": "^3.4.1",
-        "cli-table": "^0.3.1",
-        "colors": "^1.1.2",
-        "commander": "^2.9.0",
-        "cucumber-expressions": "^5.0.13",
-        "cucumber-tag-expressions": "^1.1.1",
-        "duration": "^0.2.0",
-        "escape-string-regexp": "^1.0.5",
-        "figures": "2.0.0",
-        "gherkin": "^5.0.0",
-        "glob": "^7.0.0",
-        "indent-string": "^3.1.0",
-        "is-generator": "^1.0.2",
-        "is-stream": "^1.1.0",
-        "knuth-shuffle-seeded": "^1.0.6",
-        "lodash": "^4.17.4",
-        "mz": "^2.4.0",
-        "progress": "^2.0.0",
-        "resolve": "^1.3.3",
-        "serialize-error": "^2.1.0",
-        "stack-chain": "^2.0.0",
-        "stacktrace-js": "^2.0.0",
-        "string-argv": "0.0.2",
-        "title-case": "^2.1.1",
-        "util-arity": "^1.0.2",
-        "verror": "^1.9.0"
+        "jake": "^10.8.5"
       },
       "bin": {
-        "cucumber-js": "bin/cucumber-js"
+        "ejs": "bin/cli.js"
       },
       "engines": {
-        "node": ">=0.10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/cucumber-expressions": {
-      "version": "6.6.2",
-      "resolved": "https://registry.npmjs.org/cucumber-expressions/-/cucumber-expressions-6.6.2.tgz",
-      "integrity": "sha512-WcFSVBiWNLJbIcAAC3t/ACU46vaOKfe1UIF5H3qveoq+Y4XQm9j3YwHurQNufRKBBg8nCnpU7Ttsx7egjS3hwA==",
-      "deprecated": "This package is now published under @cucumber/cucumber-expressions",
+    "node_modules/electron-to-chromium": {
+      "version": "1.4.589",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.589.tgz",
+      "integrity": "sha512-zF6y5v/YfoFIgwf2dDfAqVlPPsyQeWNpEWXbAlDUS8Ax4Z2VoiiZpAPC0Jm9hXEkJm2vIZpwB6rc4KnLTQffbQ=="
+    },
+    "node_modules/elliptic": {
+      "version": "6.5.4",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz",
+      "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==",
       "dev": true,
       "dependencies": {
-        "becke-ch--regex--s0-0-v1--base--pl--lib": "^1.2.0"
+        "bn.js": "^4.11.9",
+        "brorand": "^1.1.0",
+        "hash.js": "^1.0.0",
+        "hmac-drbg": "^1.0.1",
+        "inherits": "^2.0.4",
+        "minimalistic-assert": "^1.0.1",
+        "minimalistic-crypto-utils": "^1.0.1"
       }
     },
-    "node_modules/cucumber-messages": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/cucumber-messages/-/cucumber-messages-8.0.0.tgz",
-      "integrity": "sha512-lUnWRMjwA9+KhDec/5xRZV3Du67ISumHnVLywWQXyvzmc4P+Eqx8CoeQrBQoau3Pw1hs4kJLTDyV85hFBF00SQ==",
-      "deprecated": "This package is now published under @cucumber/messages",
+    "node_modules/elliptic/node_modules/bn.js": {
+      "version": "4.12.0",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
+      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
+      "dev": true
+    },
+    "node_modules/emittery": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz",
+      "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==",
       "dev": true,
-      "dependencies": {
-        "@types/uuid": "^3.4.6",
-        "protobufjs": "^6.8.8",
-        "uuid": "^3.3.3"
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/emittery?sponsor=1"
       }
     },
-    "node_modules/cucumber-messages/node_modules/uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
-      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
+    "node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/emojis-list": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz",
+      "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==",
       "dev": true,
-      "bin": {
-        "uuid": "bin/uuid"
+      "engines": {
+        "node": ">= 4"
       }
     },
-    "node_modules/cucumber-tag-expressions": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/cucumber-tag-expressions/-/cucumber-tag-expressions-1.1.1.tgz",
-      "integrity": "sha512-V9jv81sR/HaJ87FoidrvHkviXId7KmBcUi7aQPfi+W3nRO30N6GqH6lcp8K+nyiT1DgemRJBPDDeBMS93xJqMQ==",
-      "dev": true
-    },
-    "node_modules/cucumber/node_modules/commander": {
-      "version": "2.20.3",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-      "dev": true
+    "node_modules/encodeurl": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+      "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.8"
+      }
     },
-    "node_modules/cucumber/node_modules/cucumber-expressions": {
-      "version": "5.0.18",
-      "resolved": "https://registry.npmjs.org/cucumber-expressions/-/cucumber-expressions-5.0.18.tgz",
-      "integrity": "sha512-cj9UKCEvsB7bN97THmowcZt8I3rYFbTAFBNeDpKmWW3vr43CLZeWBmbk7NlHijndLwPJ7+uiF72xWrRU+RLyZA==",
-      "deprecated": "This package is now published under @cucumber/cucumber-expressions",
+    "node_modules/encoding": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
+      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
       "dev": true,
-      "hasInstallScript": true,
+      "optional": true,
       "dependencies": {
-        "becke-ch--regex--s0-0-v1--base--pl--lib": "^1.2.0"
+        "iconv-lite": "^0.6.2"
       }
     },
-    "node_modules/cucumber/node_modules/figures": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
-      "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==",
+    "node_modules/encoding/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
       "dev": true,
+      "optional": true,
       "dependencies": {
-        "escape-string-regexp": "^1.0.5"
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/cucumber/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/end-of-stream": {
+      "version": "1.4.4",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
+      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "devOptional": true,
+      "dependencies": {
+        "once": "^1.4.0"
+      }
+    },
+    "node_modules/enhanced-resolve": {
+      "version": "5.15.0",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz",
+      "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "graceful-fs": "^4.2.4",
+        "tapable": "^2.2.0"
       },
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=10.13.0"
       }
     },
-    "node_modules/cypress": {
-      "version": "12.17.4",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.4.tgz",
-      "integrity": "sha512-gAN8Pmns9MA5eCDFSDJXWKUpaL3IDd89N9TtIupjYnzLSmlpVr+ZR+vb4U/qaMp+lB6tBvAmt7504c3Z4RU5KQ==",
+    "node_modules/enquirer": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
+      "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@cypress/request": "2.88.12",
-        "@cypress/xvfb": "^1.2.4",
-        "@types/node": "^16.18.39",
-        "@types/sinonjs__fake-timers": "8.1.1",
-        "@types/sizzle": "^2.3.2",
-        "arch": "^2.2.0",
-        "blob-util": "^2.0.2",
-        "bluebird": "^3.7.2",
-        "buffer": "^5.6.0",
-        "cachedir": "^2.3.0",
-        "chalk": "^4.1.0",
-        "check-more-types": "^2.24.0",
-        "cli-cursor": "^3.1.0",
-        "cli-table3": "~0.6.1",
-        "commander": "^6.2.1",
-        "common-tags": "^1.8.0",
-        "dayjs": "^1.10.4",
-        "debug": "^4.3.4",
-        "enquirer": "^2.3.6",
-        "eventemitter2": "6.4.7",
-        "execa": "4.1.0",
-        "executable": "^4.1.1",
-        "extract-zip": "2.0.1",
-        "figures": "^3.2.0",
-        "fs-extra": "^9.1.0",
-        "getos": "^3.2.1",
-        "is-ci": "^3.0.0",
-        "is-installed-globally": "~0.4.0",
-        "lazy-ass": "^1.6.0",
-        "listr2": "^3.8.3",
-        "lodash": "^4.17.21",
-        "log-symbols": "^4.0.0",
-        "minimist": "^1.2.8",
-        "ospath": "^1.2.2",
-        "pretty-bytes": "^5.6.0",
-        "process": "^0.11.10",
-        "proxy-from-env": "1.0.0",
-        "request-progress": "^3.0.0",
-        "semver": "^7.5.3",
-        "supports-color": "^8.1.1",
-        "tmp": "~0.2.1",
-        "untildify": "^4.0.0",
-        "yauzl": "^2.10.0"
-      },
-      "bin": {
-        "cypress": "bin/cypress"
+        "ansi-colors": "^4.1.1"
       },
       "engines": {
-        "node": "^14.0.0 || ^16.0.0 || >=18.0.0"
+        "node": ">=8.6"
       }
     },
-    "node_modules/cypress-axe": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/cypress-axe/-/cypress-axe-1.5.0.tgz",
-      "integrity": "sha512-Hy/owCjfj+25KMsecvDgo4fC/781ccL+e8p+UUYoadGVM2ogZF9XIKbiM6KI8Y3cEaSreymdD6ZzccbI2bY0lQ==",
+    "node_modules/entities": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
+    "node_modules/env-paths": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
+      "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
       "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "axe-core": "^3 || ^4",
-        "cypress": "^10 || ^11 || ^12 || ^13"
+        "node": ">=6"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-4.3.1.tgz",
-      "integrity": "sha512-BKUYXqoTeKzkPXohEczDtpAwRCY0ZPtIpfRwJut16yNLqdXQMV+aItwanxe3cbJTFlwg562NAjL4LMdiOhkAjg==",
+    "node_modules/err-code": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
+      "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
+      "dev": true
+    },
+    "node_modules/errno": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz",
+      "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==",
       "dev": true,
+      "optional": true,
       "dependencies": {
-        "@cypress/browserify-preprocessor": "^3.0.2",
-        "chai": "^4.2.0",
-        "chokidar": "3.5.2",
-        "cosmiconfig": "^4.0.0",
-        "cucumber": "^4.2.1",
-        "cucumber-expressions": "^6.0.1",
-        "cucumber-tag-expressions": "^1.1.1",
-        "dargs": "^7.0.0",
-        "debug": "^3.0.1",
-        "gherkin": "^5.1.0",
-        "glob": "^7.1.2",
-        "js-string-escape": "^1.0.1",
-        "minimist": "^1.2.5",
-        "through": "^2.3.8"
+        "prr": "~1.0.1"
       },
       "bin": {
-        "cypress-tags": "cypress-tags.js"
+        "errno": "cli.js"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor/node_modules/chokidar": {
-      "version": "3.5.2",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.2.tgz",
-      "integrity": "sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==",
+    "node_modules/error-ex": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
       "dev": true,
       "dependencies": {
-        "anymatch": "~3.1.2",
-        "braces": "~3.0.2",
-        "glob-parent": "~5.1.2",
-        "is-binary-path": "~2.1.0",
-        "is-glob": "~4.0.1",
-        "normalize-path": "~3.0.0",
-        "readdirp": "~3.6.0"
-      },
-      "engines": {
-        "node": ">= 8.10.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.2"
+        "is-arrayish": "^0.2.1"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor/node_modules/cosmiconfig": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-4.0.0.tgz",
-      "integrity": "sha512-6e5vDdrXZD+t5v0L8CrurPeybg4Fmf+FCSYxXKYVAqLUtyCSbuyqE059d0kDthTNRzKVjL7QMgNpEUlsoYH3iQ==",
+    "node_modules/error-stack-parser": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz",
+      "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==",
       "dev": true,
       "dependencies": {
-        "is-directory": "^0.3.1",
-        "js-yaml": "^3.9.0",
-        "parse-json": "^4.0.0",
-        "require-from-string": "^2.0.1"
+        "stackframe": "^1.3.4"
+      }
+    },
+    "node_modules/es-abstract": {
+      "version": "1.22.3",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.3.tgz",
+      "integrity": "sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==",
+      "dev": true,
+      "dependencies": {
+        "array-buffer-byte-length": "^1.0.0",
+        "arraybuffer.prototype.slice": "^1.0.2",
+        "available-typed-arrays": "^1.0.5",
+        "call-bind": "^1.0.5",
+        "es-set-tostringtag": "^2.0.1",
+        "es-to-primitive": "^1.2.1",
+        "function.prototype.name": "^1.1.6",
+        "get-intrinsic": "^1.2.2",
+        "get-symbol-description": "^1.0.0",
+        "globalthis": "^1.0.3",
+        "gopd": "^1.0.1",
+        "has-property-descriptors": "^1.0.0",
+        "has-proto": "^1.0.1",
+        "has-symbols": "^1.0.3",
+        "hasown": "^2.0.0",
+        "internal-slot": "^1.0.5",
+        "is-array-buffer": "^3.0.2",
+        "is-callable": "^1.2.7",
+        "is-negative-zero": "^2.0.2",
+        "is-regex": "^1.1.4",
+        "is-shared-array-buffer": "^1.0.2",
+        "is-string": "^1.0.7",
+        "is-typed-array": "^1.1.12",
+        "is-weakref": "^1.0.2",
+        "object-inspect": "^1.13.1",
+        "object-keys": "^1.1.1",
+        "object.assign": "^4.1.4",
+        "regexp.prototype.flags": "^1.5.1",
+        "safe-array-concat": "^1.0.1",
+        "safe-regex-test": "^1.0.0",
+        "string.prototype.trim": "^1.2.8",
+        "string.prototype.trimend": "^1.0.7",
+        "string.prototype.trimstart": "^1.0.7",
+        "typed-array-buffer": "^1.0.0",
+        "typed-array-byte-length": "^1.0.0",
+        "typed-array-byte-offset": "^1.0.0",
+        "typed-array-length": "^1.0.4",
+        "unbox-primitive": "^1.0.2",
+        "which-typed-array": "^1.1.13"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+    "node_modules/es-module-lexer": {
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz",
+      "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==",
+      "dev": true
+    },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz",
+      "integrity": "sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==",
       "dev": true,
       "dependencies": {
-        "ms": "^2.1.1"
+        "get-intrinsic": "^1.2.2",
+        "has-tostringtag": "^1.0.0",
+        "hasown": "^2.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/es-to-primitive": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
+      "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "is-callable": "^1.1.4",
+        "is-date-object": "^1.0.1",
+        "is-symbol": "^1.0.2"
       },
       "engines": {
-        "node": "*"
+        "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/cypress-cucumber-preprocessor/node_modules/parse-json": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-      "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==",
+    "node_modules/es5-ext": {
+      "version": "0.10.62",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz",
+      "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "error-ex": "^1.3.1",
-        "json-parse-better-errors": "^1.0.1"
+        "es6-iterator": "^2.0.3",
+        "es6-symbol": "^3.1.3",
+        "next-tick": "^1.1.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10"
       }
     },
-    "node_modules/cypress-iframe": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cypress-iframe/-/cypress-iframe-1.0.1.tgz",
-      "integrity": "sha512-Ne+xkZmWMhfq3x6wbfzK/SzsVTCrJru3R3cLXsoSAZyfUtJDamXyaIieHXeea3pQDXF4wE2w4iUuvCYHhoD31g==",
+    "node_modules/es6-iterator": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
+      "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==",
       "dev": true,
-      "peerDependencies": {
-        "@types/cypress": "^1.1.0"
+      "dependencies": {
+        "d": "1",
+        "es5-ext": "^0.10.35",
+        "es6-symbol": "^3.1.1"
       }
     },
-    "node_modules/cypress-multi-reporters": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/cypress-multi-reporters/-/cypress-multi-reporters-1.5.0.tgz",
-      "integrity": "sha512-6rJ1rk1RpjZwTeydCDc8r3iOmWj2ZEYo++oDTJHNEu7eetb3W1cYDNo5CdxF/r0bo7TLQsOEpBHOCYBZfPVt/g==",
+    "node_modules/es6-map": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/es6-map/-/es6-map-0.1.5.tgz",
+      "integrity": "sha512-mz3UqCh0uPCIqsw1SSAkB/p0rOzF/M0V++vyN7JqlPtSW/VsYgQBvVvqMLmfBuyMzTpLnNqi6JmcSizs4jy19A==",
       "dev": true,
       "dependencies": {
-        "debug": "^4.1.1",
-        "lodash": "^4.17.15"
+        "d": "1",
+        "es5-ext": "~0.10.14",
+        "es6-iterator": "~2.0.1",
+        "es6-set": "~0.1.5",
+        "es6-symbol": "~3.1.1",
+        "event-emitter": "~0.3.5"
+      }
+    },
+    "node_modules/es6-set": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/es6-set/-/es6-set-0.1.6.tgz",
+      "integrity": "sha512-TE3LgGLDIBX332jq3ypv6bcOpkLO0AslAQo7p2VqX/1N46YNsvIWgvjojjSEnWEGWMhr1qUbYeTSir5J6mFHOw==",
+      "dev": true,
+      "dependencies": {
+        "d": "^1.0.1",
+        "es5-ext": "^0.10.62",
+        "es6-iterator": "~2.0.3",
+        "es6-symbol": "^3.1.3",
+        "event-emitter": "^0.3.5",
+        "type": "^2.7.2"
       },
       "engines": {
-        "node": ">=6.0.0"
-      },
-      "peerDependencies": {
-        "mocha": ">=3.1.2"
+        "node": ">=0.12"
       }
     },
-    "node_modules/cypress/node_modules/@types/node": {
-      "version": "16.18.54",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.54.tgz",
-      "integrity": "sha512-oTmGy68gxZZ21FhTJVVvZBYpQHEBZxHKTsGshobMqm9qWpbqdZsA5jvsuPZcHu0KwpmLrOHWPdEfg7XDpNT9UA==",
+    "node_modules/es6-set/node_modules/type": {
+      "version": "2.7.2",
+      "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
+      "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==",
       "dev": true
     },
-    "node_modules/cypress/node_modules/buffer": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+    "node_modules/es6-symbol": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
+      "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
       "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
       "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.1.13"
+        "d": "^1.0.1",
+        "ext": "^1.1.2"
       }
     },
-    "node_modules/cypress/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/esbuild-wasm": {
+      "version": "0.17.8",
+      "resolved": "https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-0.17.8.tgz",
+      "integrity": "sha512-zCmpxv95E0FuCmvdw1K836UHnj4EdiQnFfjTby35y3LAjRPtXMj3sbHDRHjbD8Mqg5lTwq3knacr/1qIFU51CQ==",
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+      "bin": {
+        "esbuild": "bin/esbuild"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/cypress/node_modules/chalk/node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+    "node_modules/escalade": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/escape-html": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
+      "dev": true
+    },
+    "node_modules/escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/escodegen": {
+      "version": "1.14.3",
+      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz",
+      "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==",
       "dev": true,
       "dependencies": {
-        "has-flag": "^4.0.0"
+        "esprima": "^4.0.1",
+        "estraverse": "^4.2.0",
+        "esutils": "^2.0.2",
+        "optionator": "^0.8.1"
+      },
+      "bin": {
+        "escodegen": "bin/escodegen.js",
+        "esgenerate": "bin/esgenerate.js"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=4.0"
+      },
+      "optionalDependencies": {
+        "source-map": "~0.6.1"
       }
     },
-    "node_modules/cypress/node_modules/commander": {
-      "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
-      "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==",
+    "node_modules/escodegen/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true,
+      "optional": true,
       "engines": {
-        "node": ">= 6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/cypress/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/eslint": {
+      "version": "8.17.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.17.0.tgz",
+      "integrity": "sha512-gq0m0BTJfci60Fz4nczYxNAlED+sMcihltndR8t9t1evnU/azx53x3t2UHXC/uRjcbvRw/XctpaNygSTcQD+Iw==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "@eslint/eslintrc": "^1.3.0",
+        "@humanwhocodes/config-array": "^0.9.2",
+        "ajv": "^6.10.0",
+        "chalk": "^4.0.0",
+        "cross-spawn": "^7.0.2",
+        "debug": "^4.3.2",
+        "doctrine": "^3.0.0",
+        "escape-string-regexp": "^4.0.0",
+        "eslint-scope": "^7.1.1",
+        "eslint-utils": "^3.0.0",
+        "eslint-visitor-keys": "^3.3.0",
+        "espree": "^9.3.2",
+        "esquery": "^1.4.0",
+        "esutils": "^2.0.2",
+        "fast-deep-equal": "^3.1.3",
+        "file-entry-cache": "^6.0.1",
+        "functional-red-black-tree": "^1.0.1",
+        "glob-parent": "^6.0.1",
+        "globals": "^13.15.0",
+        "ignore": "^5.2.0",
+        "import-fresh": "^3.0.0",
+        "imurmurhash": "^0.1.4",
+        "is-glob": "^4.0.0",
+        "js-yaml": "^4.1.0",
+        "json-stable-stringify-without-jsonify": "^1.0.1",
+        "levn": "^0.4.1",
+        "lodash.merge": "^4.6.2",
+        "minimatch": "^3.1.2",
+        "natural-compare": "^1.4.0",
+        "optionator": "^0.9.1",
+        "regexpp": "^3.2.0",
+        "strip-ansi": "^6.0.1",
+        "strip-json-comments": "^3.1.0",
+        "text-table": "^0.2.0",
+        "v8-compile-cache": "^2.0.3"
+      },
+      "bin": {
+        "eslint": "bin/eslint.js"
       },
       "engines": {
-        "node": ">=6.0"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+      "funding": {
+        "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/cypress/node_modules/fs-extra": {
-      "version": "9.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
-      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
+    "node_modules/eslint-scope": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+      "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
       "dev": true,
       "dependencies": {
-        "at-least-node": "^1.0.0",
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
+        "esrecurse": "^4.3.0",
+        "estraverse": "^4.1.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/cypress/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+    "node_modules/eslint-utils": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz",
+      "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==",
       "dev": true,
       "dependencies": {
-        "has-flag": "^4.0.0"
+        "eslint-visitor-keys": "^2.0.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^10.0.0 || ^12.0.0 || >= 14.0.0"
       },
       "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
-      }
-    },
-    "node_modules/d": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
-      "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
-      "dev": true,
-      "dependencies": {
-        "es5-ext": "^0.10.50",
-        "type": "^1.0.1"
+        "url": "https://github.com/sponsors/mysticatea"
+      },
+      "peerDependencies": {
+        "eslint": ">=5"
       }
     },
-    "node_modules/dargs": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/dargs/-/dargs-7.0.0.tgz",
-      "integrity": "sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg==",
+    "node_modules/eslint-utils/node_modules/eslint-visitor-keys": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
+      "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
       }
     },
-    "node_modules/dash-ast": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/dash-ast/-/dash-ast-2.0.1.tgz",
-      "integrity": "sha512-5TXltWJGc+RdnabUGzhRae1TRq6m4gr+3K2wQX0is5/F2yS6MJXJvLyI3ErAnsAXuJoGqvfVD5icRgim07DrxQ==",
-      "dev": true
-    },
-    "node_modules/dashdash": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
-      "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==",
+    "node_modules/eslint-visitor-keys": {
+      "version": "3.4.3",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz",
+      "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
       "dev": true,
-      "dependencies": {
-        "assert-plus": "^1.0.0"
-      },
       "engines": {
-        "node": ">=0.10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/data-urls": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz",
-      "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==",
+    "node_modules/eslint/node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
       "dev": true,
       "dependencies": {
-        "abab": "^2.0.0",
-        "whatwg-mimetype": "^2.2.0",
-        "whatwg-url": "^7.0.0"
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
       }
     },
-    "node_modules/dateformat": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
-      "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==",
+    "node_modules/eslint/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/dayjs": {
-      "version": "1.11.10",
-      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz",
-      "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==",
-      "dev": true
-    },
-    "node_modules/debug": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
-      "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
-      "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)",
       "dependencies": {
-        "ms": "2.1.2"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=6.0"
+        "node": ">=8"
       },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/decache": {
-      "version": "4.6.2",
-      "resolved": "https://registry.npmjs.org/decache/-/decache-4.6.2.tgz",
-      "integrity": "sha512-2LPqkLeu8XWHU8qNCS3kcF6sCcb5zIzvWaAHYSvPfwhdd7mHuah29NssMzrTYyHN4F5oFy2ko9OBYxegtU0FEw==",
+    "node_modules/eslint/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "callsite": "^1.0.0"
-      }
-    },
-    "node_modules/decamelize": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
-      "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/decamelize-keys": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.1.tgz",
-      "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==",
+    "node_modules/eslint/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "decamelize": "^1.1.0",
-        "map-obj": "^1.0.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/decamelize-keys/node_modules/map-obj": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
-      "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==",
+    "node_modules/eslint/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/decimal.js": {
-      "version": "10.4.3",
-      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
-      "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==",
+    "node_modules/eslint/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/decode-uri-component": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz",
-      "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==",
+    "node_modules/eslint/node_modules/escape-string-regexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
       "dev": true,
       "engines": {
-        "node": ">=0.10"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/decompress-response": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-      "optional": true,
+    "node_modules/eslint/node_modules/eslint-scope": {
+      "version": "7.2.2",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz",
+      "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==",
+      "dev": true,
       "dependencies": {
-        "mimic-response": "^3.1.0"
+        "esrecurse": "^4.3.0",
+        "estraverse": "^5.2.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/dedent": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.1.tgz",
-      "integrity": "sha512-+LxW+KLWxu3HW3M2w2ympwtqPrqYRzU8fqi6Fhd18fBALe15blJPI/I4+UHveMVG6lJqB4JNd4UG0S5cnVHwIg==",
+    "node_modules/eslint/node_modules/estraverse": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
       "dev": true,
-      "peerDependencies": {
-        "babel-plugin-macros": "^3.1.0"
-      },
-      "peerDependenciesMeta": {
-        "babel-plugin-macros": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=4.0"
       }
     },
-    "node_modules/deep-eql": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz",
-      "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==",
+    "node_modules/eslint/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
       "dev": true,
       "dependencies": {
-        "type-detect": "^4.0.0"
+        "is-glob": "^4.0.3"
       },
       "engines": {
-        "node": ">=6"
+        "node": ">=10.13.0"
       }
     },
-    "node_modules/deep-equal": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz",
-      "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==",
+    "node_modules/eslint/node_modules/globals": {
+      "version": "13.23.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz",
+      "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==",
       "dev": true,
       "dependencies": {
-        "is-arguments": "^1.0.4",
-        "is-date-object": "^1.0.1",
-        "is-regex": "^1.0.4",
-        "object-is": "^1.0.1",
-        "object-keys": "^1.1.1",
-        "regexp.prototype.flags": "^1.2.0"
+        "type-fest": "^0.20.2"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/deep-extend": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
       "engines": {
-        "node": ">=4.0.0"
-      }
-    },
-    "node_modules/deep-is": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
-      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
-      "dev": true
-    },
-    "node_modules/deepmerge": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
-      "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/eslint/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/default-gateway": {
-      "version": "6.0.3",
-      "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz",
-      "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==",
+    "node_modules/eslint/node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "node_modules/eslint/node_modules/levn": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+      "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
       "dev": true,
       "dependencies": {
-        "execa": "^5.0.0"
+        "prelude-ls": "^1.2.1",
+        "type-check": "~0.4.0"
       },
       "engines": {
-        "node": ">= 10"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/default-gateway/node_modules/execa": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
+    "node_modules/eslint/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "cross-spawn": "^7.0.3",
-        "get-stream": "^6.0.0",
-        "human-signals": "^2.1.0",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.1",
-        "onetime": "^5.1.2",
-        "signal-exit": "^3.0.3",
-        "strip-final-newline": "^2.0.0"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+        "node": "*"
       }
     },
-    "node_modules/default-gateway/node_modules/get-stream": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+    "node_modules/eslint/node_modules/optionator": {
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
+      "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
       "dev": true,
-      "engines": {
-        "node": ">=10"
+      "dependencies": {
+        "@aashutoshrathi/word-wrap": "^1.2.3",
+        "deep-is": "^0.1.3",
+        "fast-levenshtein": "^2.0.6",
+        "levn": "^0.4.1",
+        "prelude-ls": "^1.2.1",
+        "type-check": "^0.4.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "engines": {
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/default-gateway/node_modules/human-signals": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
+    "node_modules/eslint/node_modules/prelude-ls": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+      "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
       "dev": true,
       "engines": {
-        "node": ">=10.17.0"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/default-gateway/node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+    "node_modules/eslint/node_modules/strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
       "dev": true,
       "engines": {
         "node": ">=8"
@@ -11605,4193 +14816,4402 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/defaults": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
-      "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==",
+    "node_modules/eslint/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "clone": "^1.0.2"
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/define-data-property": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.0.tgz",
-      "integrity": "sha512-UzGwzcjyv3OtAvolTj1GoyNYzfFR+iqbGjcnBEENZVCpM4/Ng1yhGNvS3lR/xDS74Tb2wGG9WzNSNIOS9UVb2g==",
+    "node_modules/eslint/node_modules/type-check": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+      "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
       "dev": true,
       "dependencies": {
-        "get-intrinsic": "^1.2.1",
-        "gopd": "^1.0.1",
-        "has-property-descriptors": "^1.0.0"
+        "prelude-ls": "^1.2.1"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/define-lazy-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
-      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==",
+    "node_modules/espree": {
+      "version": "9.6.1",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
+      "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==",
       "dev": true,
+      "dependencies": {
+        "acorn": "^8.9.0",
+        "acorn-jsx": "^5.3.2",
+        "eslint-visitor-keys": "^3.4.1"
+      },
       "engines": {
-        "node": ">=8"
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/define-properties": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz",
-      "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==",
+    "node_modules/esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
       "dev": true,
-      "dependencies": {
-        "define-data-property": "^1.0.1",
-        "has-property-descriptors": "^1.0.0",
-        "object-keys": "^1.1.1"
+      "bin": {
+        "esparse": "bin/esparse.js",
+        "esvalidate": "bin/esvalidate.js"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=4"
+      }
+    },
+    "node_modules/esquery": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz",
+      "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==",
+      "dev": true,
+      "dependencies": {
+        "estraverse": "^5.1.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": ">=0.10"
       }
     },
-    "node_modules/define-property": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
+    "node_modules/esquery/node_modules/estraverse": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+      "dev": true,
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/esrecurse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
       "dev": true,
       "dependencies": {
-        "is-descriptor": "^1.0.2",
-        "isobject": "^3.0.1"
+        "estraverse": "^5.2.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4.0"
       }
     },
-    "node_modules/defined": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.1.tgz",
-      "integrity": "sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==",
+    "node_modules/esrecurse/node_modules/estraverse": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
       "dev": true,
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": ">=4.0"
       }
     },
-    "node_modules/delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+    "node_modules/estraverse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+      "dev": true,
       "engines": {
-        "node": ">=0.4.0"
+        "node": ">=4.0"
       }
     },
-    "node_modules/delegates": {
+    "node_modules/estree-is-function": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
-      "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+      "resolved": "https://registry.npmjs.org/estree-is-function/-/estree-is-function-1.0.0.tgz",
+      "integrity": "sha512-nSCWn1jkSq2QAtkaVLJZY2ezwcFO161HVc174zL1KPW3RJ+O6C3eJb8Nx7OXzvhoEv+nLgSR1g71oWUHUDTrJA==",
       "dev": true
     },
-    "node_modules/depd": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
-      "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
+    "node_modules/esutils": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
       "dev": true,
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/dependency-graph": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz",
-      "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==",
+    "node_modules/etag": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
+      "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
       "dev": true,
       "engines": {
-        "node": ">= 0.6.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/deps-sort": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/deps-sort/-/deps-sort-2.0.1.tgz",
-      "integrity": "sha512-1orqXQr5po+3KI6kQb9A4jnXT1PBwggGl2d7Sq2xsnOeI9GPcE/tGcF9UiSZtZBM7MukY4cAh7MemS6tZYipfw==",
+    "node_modules/event-emitter": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
+      "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==",
       "dev": true,
       "dependencies": {
-        "JSONStream": "^1.0.3",
-        "shasum-object": "^1.0.0",
-        "subarg": "^1.0.0",
-        "through2": "^2.0.0"
-      },
-      "bin": {
-        "deps-sort": "bin/cmd.js"
+        "d": "1",
+        "es5-ext": "~0.10.14"
       }
     },
-    "node_modules/des.js": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz",
-      "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==",
+    "node_modules/event-stream": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz",
+      "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "duplexer": "^0.1.1",
+        "from": "^0.1.7",
+        "map-stream": "0.0.7",
+        "pause-stream": "^0.0.11",
+        "split": "^1.0.1",
+        "stream-combiner": "^0.2.2",
+        "through": "^2.3.8"
       }
     },
-    "node_modules/destroy": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
-      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+    "node_modules/event-target-shim": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
       "dev": true,
       "engines": {
-        "node": ">= 0.8",
-        "npm": "1.2.8000 || >= 1.4.16"
+        "node": ">=6"
       }
     },
-    "node_modules/detect-browser": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/detect-browser/-/detect-browser-5.2.0.tgz",
-      "integrity": "sha512-tr7XntDAu50BVENgQfajMLzacmSe34D+qZc4zjnniz0ZVuw/TZcLcyxHQjYpJTM36sGEkZZlYLnIM1hH7alTMA=="
-    },
-    "node_modules/detect-file": {
+    "node_modules/eventemitter-asyncresource": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
-      "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==",
+      "resolved": "https://registry.npmjs.org/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz",
+      "integrity": "sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==",
+      "dev": true
+    },
+    "node_modules/eventemitter2": {
+      "version": "6.4.7",
+      "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.7.tgz",
+      "integrity": "sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg==",
+      "dev": true
+    },
+    "node_modules/eventemitter3": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
+      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
+      "dev": true
+    },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=0.8.x"
       }
     },
-    "node_modules/detect-libc": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
-      "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
-      "optional": true,
-      "engines": {
-        "node": ">=8"
+    "node_modules/evp_bytestokey": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
+      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
+      "dev": true,
+      "dependencies": {
+        "md5.js": "^1.3.4",
+        "safe-buffer": "^5.1.1"
       }
     },
-    "node_modules/detect-newline": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
-      "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==",
+    "node_modules/execa": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
+      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
       "dev": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.3",
+        "get-stream": "^6.0.0",
+        "human-signals": "^2.1.0",
+        "is-stream": "^2.0.0",
+        "merge-stream": "^2.0.0",
+        "npm-run-path": "^4.0.1",
+        "onetime": "^5.1.2",
+        "signal-exit": "^3.0.3",
+        "strip-final-newline": "^2.0.0"
+      },
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/execa?sponsor=1"
       }
     },
-    "node_modules/detect-node": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
-      "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
-      "dev": true
-    },
-    "node_modules/detective": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/detective/-/detective-5.2.1.tgz",
-      "integrity": "sha512-v9XE1zRnz1wRtgurGu0Bs8uHKFSTdteYZNbIPFVhUZ39L/S79ppMpdmVOZAnoz1jfEFodc48n6MX483Xo3t1yw==",
+    "node_modules/execall": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/execall/-/execall-2.0.0.tgz",
+      "integrity": "sha512-0FU2hZ5Hh6iQnarpRtQurM/aAvp3RIbfvgLHrcqJYzhXyV2KFruhuChf9NC6waAhiUR7FFtlugkI4p7f2Fqlow==",
       "dev": true,
       "dependencies": {
-        "acorn-node": "^1.8.2",
-        "defined": "^1.0.0",
-        "minimist": "^1.2.6"
-      },
-      "bin": {
-        "detective": "bin/detective.js"
+        "clone-regexp": "^2.1.0"
       },
       "engines": {
-        "node": ">=0.8.0"
+        "node": ">=8"
       }
     },
-    "node_modules/dfa": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/dfa/-/dfa-1.2.0.tgz",
-      "integrity": "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q==",
-      "dev": true
-    },
-    "node_modules/diff": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
-      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
+    "node_modules/executable": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/executable/-/executable-4.1.1.tgz",
+      "integrity": "sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg==",
       "dev": true,
+      "dependencies": {
+        "pify": "^2.2.0"
+      },
       "engines": {
-        "node": ">=0.3.1"
+        "node": ">=4"
       }
     },
-    "node_modules/diff-sequences": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz",
-      "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==",
+    "node_modules/executable/node_modules/pify": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+      "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
       "dev": true,
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/diffie-hellman": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
-      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
+    "node_modules/exit": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
+      "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==",
       "dev": true,
-      "dependencies": {
-        "bn.js": "^4.1.0",
-        "miller-rabin": "^4.0.0",
-        "randombytes": "^2.0.0"
+      "engines": {
+        "node": ">= 0.8.0"
       }
     },
-    "node_modules/diffie-hellman/node_modules/bn.js": {
-      "version": "4.12.0",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
-      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
-      "dev": true
-    },
-    "node_modules/dir-glob": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
-      "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+    "node_modules/expand-brackets": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
+      "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==",
       "dev": true,
       "dependencies": {
-        "path-type": "^4.0.0"
+        "debug": "^2.3.3",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "posix-character-classes": "^0.1.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/dns-equal": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz",
-      "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==",
-      "dev": true
-    },
-    "node_modules/dns-packet": {
-      "version": "5.6.1",
-      "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
-      "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==",
+    "node_modules/expand-brackets/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "@leichtgewicht/ip-codec": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=6"
+        "ms": "2.0.0"
       }
     },
-    "node_modules/doctrine": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
-      "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+    "node_modules/expand-brackets/node_modules/define-property": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+      "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==",
       "dev": true,
       "dependencies": {
-        "esutils": "^2.0.2"
+        "is-descriptor": "^0.1.0"
       },
       "engines": {
-        "node": ">=6.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/dom-serializer": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
-      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+    "node_modules/expand-brackets/node_modules/extend-shallow": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
       "dev": true,
       "dependencies": {
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.2",
-        "entities": "^4.2.0"
+        "is-extendable": "^0.1.0"
       },
-      "funding": {
-        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/domain-browser": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
-      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
-      "dev": true,
+    "node_modules/expand-brackets/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+      "optional": true,
       "engines": {
-        "node": ">=0.4",
-        "npm": ">=1.2"
+        "node": ">=6"
       }
     },
-    "node_modules/domelementtype": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
-      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
-      "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fb55"
-        }
-      ]
-    },
-    "node_modules/domexception": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz",
-      "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==",
+    "node_modules/expand-tilde": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
+      "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==",
       "dev": true,
       "dependencies": {
-        "webidl-conversions": "^4.0.2"
+        "homedir-polyfill": "^1.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/domhandler": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
-      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+    "node_modules/expect": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz",
+      "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==",
       "dev": true,
       "dependencies": {
-        "domelementtype": "^2.3.0"
+        "@jest/expect-utils": "^29.7.0",
+        "jest-get-type": "^29.6.3",
+        "jest-matcher-utils": "^29.7.0",
+        "jest-message-util": "^29.7.0",
+        "jest-util": "^29.7.0"
       },
       "engines": {
-        "node": ">= 4"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/domhandler?sponsor=1"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/dommatrix": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/dommatrix/-/dommatrix-1.0.3.tgz",
-      "integrity": "sha512-l32Xp/TLgWb8ReqbVJAFIvXmY7go4nTxxlWiAFyhoQw9RKEOHBZNnyGvJWqDVSPmq3Y9HlM4npqF/T6VMOXhww==",
-      "deprecated": "dommatrix is no longer maintained. Please use @thednp/dommatrix.",
+    "node_modules/exponential-backoff": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz",
+      "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==",
       "dev": true
     },
-    "node_modules/dompurify": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.3.3.tgz",
-      "integrity": "sha512-dqnqRkPMAjOZE0FogZ+ceJNM2dZ3V/yNOuFB7+39qpO93hHhfRpHw3heYQC7DPK9FqbQTfBKUJhiSfz4MvXYwg=="
-    },
-    "node_modules/domutils": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
-      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
+    "node_modules/express": {
+      "version": "4.18.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
       "dev": true,
       "dependencies": {
-        "dom-serializer": "^2.0.0",
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.3"
+        "accepts": "~1.3.8",
+        "array-flatten": "1.1.1",
+        "body-parser": "1.20.1",
+        "content-disposition": "0.5.4",
+        "content-type": "~1.0.4",
+        "cookie": "0.5.0",
+        "cookie-signature": "1.0.6",
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "finalhandler": "1.2.0",
+        "fresh": "0.5.2",
+        "http-errors": "2.0.0",
+        "merge-descriptors": "1.0.1",
+        "methods": "~1.1.2",
+        "on-finished": "2.4.1",
+        "parseurl": "~1.3.3",
+        "path-to-regexp": "0.1.7",
+        "proxy-addr": "~2.0.7",
+        "qs": "6.11.0",
+        "range-parser": "~1.2.1",
+        "safe-buffer": "5.2.1",
+        "send": "0.18.0",
+        "serve-static": "1.15.0",
+        "setprototypeof": "1.2.0",
+        "statuses": "2.0.1",
+        "type-is": "~1.6.18",
+        "utils-merge": "1.0.1",
+        "vary": "~1.1.2"
       },
-      "funding": {
-        "url": "https://github.com/fb55/domutils?sponsor=1"
+      "engines": {
+        "node": ">= 0.10.0"
       }
     },
-    "node_modules/dot": {
-      "version": "2.0.0-beta.1",
-      "resolved": "https://registry.npmjs.org/dot/-/dot-2.0.0-beta.1.tgz",
-      "integrity": "sha512-kxM7fSnNQTXOmaeGuBSXM8O3fEsBb7XSDBllkGbRwa0lJSJTxxDE/4eSNGLKZUmlFw0f1vJ5qSV2BljrgQtgIA==",
+    "node_modules/express/node_modules/array-flatten": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
       "dev": true
     },
-    "node_modules/dotenv": {
-      "version": "10.0.0",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz",
-      "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==",
+    "node_modules/express/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/drange": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/drange/-/drange-1.1.1.tgz",
-      "integrity": "sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA==",
-      "engines": {
-        "node": ">=4"
+      "dependencies": {
+        "ms": "2.0.0"
       }
     },
-    "node_modules/duplexer": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
-      "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
+    "node_modules/express/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "dev": true
     },
-    "node_modules/duplexer2": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
-      "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==",
+    "node_modules/express/node_modules/qs": {
+      "version": "6.11.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
       "dev": true,
       "dependencies": {
-        "readable-stream": "^2.0.2"
+        "side-channel": "^1.0.4"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/duration": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/duration/-/duration-0.2.2.tgz",
-      "integrity": "sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==",
+    "node_modules/ext": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz",
+      "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==",
       "dev": true,
       "dependencies": {
-        "d": "1",
-        "es5-ext": "~0.10.46"
+        "type": "^2.7.2"
       }
     },
-    "node_modules/eastasianwidth": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
+    "node_modules/ext/node_modules/type": {
+      "version": "2.7.2",
+      "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
+      "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==",
       "dev": true
     },
-    "node_modules/ecc-jsbn": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
-      "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==",
+    "node_modules/extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
+      "dev": true
+    },
+    "node_modules/extend-shallow": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
+      "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==",
       "dev": true,
       "dependencies": {
-        "jsbn": "~0.1.0",
-        "safer-buffer": "^2.1.0"
+        "assign-symbols": "^1.0.0",
+        "is-extendable": "^1.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/ee-first": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-      "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
-      "dev": true
-    },
-    "node_modules/ejs": {
-      "version": "3.1.9",
-      "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz",
-      "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==",
+    "node_modules/extend-shallow/node_modules/is-extendable": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+      "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
       "dev": true,
       "dependencies": {
-        "jake": "^10.8.5"
-      },
-      "bin": {
-        "ejs": "bin/cli.js"
+        "is-plain-object": "^2.0.4"
       },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/electron-to-chromium": {
-      "version": "1.4.528",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.528.tgz",
-      "integrity": "sha512-UdREXMXzLkREF4jA8t89FQjA8WHI6ssP38PMY4/4KhXFQbtImnghh4GkCgrtiZwLKUKVD2iTVXvDVQjfomEQuA=="
-    },
-    "node_modules/elliptic": {
-      "version": "6.5.4",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz",
-      "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==",
+    "node_modules/extend-shallow/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
       "dev": true,
       "dependencies": {
-        "bn.js": "^4.11.9",
-        "brorand": "^1.1.0",
-        "hash.js": "^1.0.0",
-        "hmac-drbg": "^1.0.1",
-        "inherits": "^2.0.4",
-        "minimalistic-assert": "^1.0.1",
-        "minimalistic-crypto-utils": "^1.0.1"
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/elliptic/node_modules/bn.js": {
-      "version": "4.12.0",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz",
-      "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==",
-      "dev": true
-    },
-    "node_modules/emittery": {
-      "version": "0.13.1",
-      "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz",
-      "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==",
+    "node_modules/external-editor": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
+      "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==",
       "dev": true,
-      "engines": {
-        "node": ">=12"
+      "dependencies": {
+        "chardet": "^0.7.0",
+        "iconv-lite": "^0.4.24",
+        "tmp": "^0.0.33"
       },
-      "funding": {
-        "url": "https://github.com/sindresorhus/emittery?sponsor=1"
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/emoji-regex": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    "node_modules/extglob": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
+      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+      "dev": true,
+      "dependencies": {
+        "array-unique": "^0.3.2",
+        "define-property": "^1.0.0",
+        "expand-brackets": "^2.1.4",
+        "extend-shallow": "^2.0.1",
+        "fragment-cache": "^0.2.1",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
     },
-    "node_modules/emojis-list": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz",
-      "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==",
+    "node_modules/extglob/node_modules/define-property": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+      "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==",
       "dev": true,
+      "dependencies": {
+        "is-descriptor": "^1.0.0"
+      },
       "engines": {
-        "node": ">= 4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/encodeurl": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
-      "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+    "node_modules/extglob/node_modules/extend-shallow": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
       "dev": true,
+      "dependencies": {
+        "is-extendable": "^0.1.0"
+      },
       "engines": {
-        "node": ">= 0.8"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/encoding": {
-      "version": "0.1.13",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
-      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
+    "node_modules/extglob/node_modules/is-descriptor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.3.tgz",
+      "integrity": "sha512-JCNNGbwWZEVaSPtS45mdtrneRWJFp07LLmykxeFV5F6oBvNF8vHSfJuJgoT472pSfk+Mf8VnlrspaFBHWM8JAw==",
       "dev": true,
-      "optional": true,
       "dependencies": {
-        "iconv-lite": "^0.6.2"
+        "is-accessor-descriptor": "^1.0.1",
+        "is-data-descriptor": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
       }
     },
-    "node_modules/encoding/node_modules/iconv-lite": {
-      "version": "0.6.3",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
-      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+    "node_modules/extract-zip": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
+      "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
       "dev": true,
-      "optional": true,
       "dependencies": {
-        "safer-buffer": ">= 2.1.2 < 3.0.0"
+        "debug": "^4.1.1",
+        "get-stream": "^5.1.0",
+        "yauzl": "^2.10.0"
+      },
+      "bin": {
+        "extract-zip": "cli.js"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 10.17.0"
+      },
+      "optionalDependencies": {
+        "@types/yauzl": "^2.9.1"
       }
     },
-    "node_modules/end-of-stream": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-      "devOptional": true,
+    "node_modules/extract-zip/node_modules/get-stream": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+      "dev": true,
       "dependencies": {
-        "once": "^1.4.0"
+        "pump": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/enhanced-resolve": {
-      "version": "5.15.0",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz",
-      "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==",
+    "node_modules/extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==",
+      "dev": true,
+      "engines": [
+        "node >=0.6.0"
+      ]
+    },
+    "node_modules/fancy-log": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/fancy-log/-/fancy-log-2.0.0.tgz",
+      "integrity": "sha512-9CzxZbACXMUXW13tS0tI8XsGGmxWzO2DmYrGuBJOJ8k8q2K7hwfJA5qHjuPPe8wtsco33YR9wc+Rlr5wYFvhSA==",
       "dev": true,
       "dependencies": {
-        "graceful-fs": "^4.2.4",
-        "tapable": "^2.2.0"
+        "color-support": "^1.1.3"
       },
       "engines": {
         "node": ">=10.13.0"
       }
     },
-    "node_modules/enquirer": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
-      "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
+    },
+    "node_modules/fast-glob": {
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
+      "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
       "dev": true,
       "dependencies": {
-        "ansi-colors": "^4.1.1"
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.4"
       },
       "engines": {
-        "node": ">=8.6"
+        "node": ">=8.6.0"
       }
     },
-    "node_modules/entities": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
-      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+    "node_modules/fast-json-patch": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz",
+      "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ=="
+    },
+    "node_modules/fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
+    },
+    "node_modules/fast-levenshtein": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+      "dev": true
+    },
+    "node_modules/fast-safe-stringify": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
+      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==",
+      "dev": true
+    },
+    "node_modules/fastest-levenshtein": {
+      "version": "1.0.16",
+      "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz",
+      "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==",
       "dev": true,
       "engines": {
-        "node": ">=0.12"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/entities?sponsor=1"
+        "node": ">= 4.9.1"
       }
     },
-    "node_modules/env-paths": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
-      "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
+    "node_modules/fastq": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
+      "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
       "dev": true,
-      "engines": {
-        "node": ">=6"
+      "dependencies": {
+        "reusify": "^1.0.4"
       }
     },
-    "node_modules/err-code": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
-      "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
-      "dev": true
-    },
-    "node_modules/errno": {
-      "version": "0.1.8",
-      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz",
-      "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==",
-      "dev": true,
-      "optional": true,
+    "node_modules/fault": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz",
+      "integrity": "sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==",
       "dependencies": {
-        "prr": "~1.0.1"
+        "format": "^0.2.0"
       },
-      "bin": {
-        "errno": "cli.js"
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/error-ex": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
-      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+    "node_modules/faye-websocket": {
+      "version": "0.11.4",
+      "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz",
+      "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==",
       "dev": true,
       "dependencies": {
-        "is-arrayish": "^0.2.1"
+        "websocket-driver": ">=0.5.1"
+      },
+      "engines": {
+        "node": ">=0.8.0"
       }
     },
-    "node_modules/error-stack-parser": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz",
-      "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==",
+    "node_modules/fb-watchman": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz",
+      "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==",
       "dev": true,
       "dependencies": {
-        "stackframe": "^1.3.4"
+        "bser": "2.1.1"
       }
     },
-    "node_modules/es-abstract": {
-      "version": "1.22.2",
-      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.2.tgz",
-      "integrity": "sha512-YoxfFcDmhjOgWPWsV13+2RNjq1F6UQnfs+8TftwNqtzlmFzEXvlUwdrNrYeaizfjQzRMxkZ6ElWMOJIFKdVqwA==",
+    "node_modules/fd-slicer": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
+      "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==",
       "dev": true,
       "dependencies": {
-        "array-buffer-byte-length": "^1.0.0",
-        "arraybuffer.prototype.slice": "^1.0.2",
-        "available-typed-arrays": "^1.0.5",
-        "call-bind": "^1.0.2",
-        "es-set-tostringtag": "^2.0.1",
-        "es-to-primitive": "^1.2.1",
-        "function.prototype.name": "^1.1.6",
-        "get-intrinsic": "^1.2.1",
-        "get-symbol-description": "^1.0.0",
-        "globalthis": "^1.0.3",
-        "gopd": "^1.0.1",
-        "has": "^1.0.3",
-        "has-property-descriptors": "^1.0.0",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3",
-        "internal-slot": "^1.0.5",
-        "is-array-buffer": "^3.0.2",
-        "is-callable": "^1.2.7",
-        "is-negative-zero": "^2.0.2",
-        "is-regex": "^1.1.4",
-        "is-shared-array-buffer": "^1.0.2",
-        "is-string": "^1.0.7",
-        "is-typed-array": "^1.1.12",
-        "is-weakref": "^1.0.2",
-        "object-inspect": "^1.12.3",
-        "object-keys": "^1.1.1",
-        "object.assign": "^4.1.4",
-        "regexp.prototype.flags": "^1.5.1",
-        "safe-array-concat": "^1.0.1",
-        "safe-regex-test": "^1.0.0",
-        "string.prototype.trim": "^1.2.8",
-        "string.prototype.trimend": "^1.0.7",
-        "string.prototype.trimstart": "^1.0.7",
-        "typed-array-buffer": "^1.0.0",
-        "typed-array-byte-length": "^1.0.0",
-        "typed-array-byte-offset": "^1.0.0",
-        "typed-array-length": "^1.0.4",
-        "unbox-primitive": "^1.0.2",
-        "which-typed-array": "^1.1.11"
+        "pend": "~1.2.0"
+      }
+    },
+    "node_modules/figures": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
+      "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==",
+      "dev": true,
+      "dependencies": {
+        "escape-string-regexp": "^1.0.5"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/es-module-lexer": {
-      "version": "0.9.3",
-      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz",
-      "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==",
-      "dev": true
-    },
-    "node_modules/es-set-tostringtag": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
-      "integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
+    "node_modules/file-entry-cache": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+      "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
       "dev": true,
       "dependencies": {
-        "get-intrinsic": "^1.1.3",
-        "has": "^1.0.3",
-        "has-tostringtag": "^1.0.0"
+        "flat-cache": "^3.0.4"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": "^10.12.0 || >=12.0.0"
       }
     },
-    "node_modules/es-to-primitive": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz",
-      "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==",
+    "node_modules/file-saver": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.2.tgz",
+      "integrity": "sha512-Wz3c3XQ5xroCxd1G8b7yL0Ehkf0TC9oYC6buPFkNnU9EnaPlifeAFCyCh+iewXTyFRcg0a6j3J7FmJsIhlhBdw=="
+    },
+    "node_modules/filelist": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
+      "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==",
       "dev": true,
       "dependencies": {
-        "is-callable": "^1.1.4",
-        "is-date-object": "^1.0.1",
-        "is-symbol": "^1.0.2"
+        "minimatch": "^5.0.1"
+      }
+    },
+    "node_modules/fill-range": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "dev": true,
+      "dependencies": {
+        "to-regex-range": "^5.0.1"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=8"
       }
     },
-    "node_modules/es5-ext": {
-      "version": "0.10.62",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz",
-      "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==",
+    "node_modules/finalhandler": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
+      "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "es6-iterator": "^2.0.3",
-        "es6-symbol": "^3.1.3",
-        "next-tick": "^1.1.0"
+        "debug": "2.6.9",
+        "encodeurl": "~1.0.2",
+        "escape-html": "~1.0.3",
+        "on-finished": "2.4.1",
+        "parseurl": "~1.3.3",
+        "statuses": "2.0.1",
+        "unpipe": "~1.0.0"
       },
       "engines": {
-        "node": ">=0.10"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/es6-iterator": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
-      "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==",
+    "node_modules/finalhandler/node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "d": "1",
-        "es5-ext": "^0.10.35",
-        "es6-symbol": "^3.1.1"
+        "ms": "2.0.0"
       }
     },
-    "node_modules/es6-map": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/es6-map/-/es6-map-0.1.5.tgz",
-      "integrity": "sha512-mz3UqCh0uPCIqsw1SSAkB/p0rOzF/M0V++vyN7JqlPtSW/VsYgQBvVvqMLmfBuyMzTpLnNqi6JmcSizs4jy19A==",
+    "node_modules/finalhandler/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
+    },
+    "node_modules/find-cache-dir": {
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
+      "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==",
       "dev": true,
       "dependencies": {
-        "d": "1",
-        "es5-ext": "~0.10.14",
-        "es6-iterator": "~2.0.1",
-        "es6-set": "~0.1.5",
-        "es6-symbol": "~3.1.1",
-        "event-emitter": "~0.3.5"
+        "commondir": "^1.0.1",
+        "make-dir": "^3.0.2",
+        "pkg-dir": "^4.1.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/avajs/find-cache-dir?sponsor=1"
       }
     },
-    "node_modules/es6-set": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/es6-set/-/es6-set-0.1.6.tgz",
-      "integrity": "sha512-TE3LgGLDIBX332jq3ypv6bcOpkLO0AslAQo7p2VqX/1N46YNsvIWgvjojjSEnWEGWMhr1qUbYeTSir5J6mFHOw==",
+    "node_modules/find-up": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+      "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
       "dev": true,
       "dependencies": {
-        "d": "^1.0.1",
-        "es5-ext": "^0.10.62",
-        "es6-iterator": "~2.0.3",
-        "es6-symbol": "^3.1.3",
-        "event-emitter": "^0.3.5",
-        "type": "^2.7.2"
+        "locate-path": "^5.0.0",
+        "path-exists": "^4.0.0"
       },
       "engines": {
-        "node": ">=0.12"
+        "node": ">=8"
       }
     },
-    "node_modules/es6-set/node_modules/type": {
-      "version": "2.7.2",
-      "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
-      "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==",
-      "dev": true
-    },
-    "node_modules/es6-symbol": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
-      "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
+    "node_modules/findit2": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz",
+      "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==",
       "dev": true,
-      "dependencies": {
-        "d": "^1.0.1",
-        "ext": "^1.1.2"
+      "engines": {
+        "node": ">=0.8.22"
       }
     },
-    "node_modules/esbuild": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.8.tgz",
-      "integrity": "sha512-g24ybC3fWhZddZK6R3uD2iF/RIPnRpwJAqLov6ouX3hMbY4+tKolP0VMF3zuIYCaXun+yHwS5IPQ91N2BT191g==",
+    "node_modules/findup-sync": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-2.0.0.tgz",
+      "integrity": "sha512-vs+3unmJT45eczmcAZ6zMJtxN3l/QXeccaXQx5cu/MeJMhewVfoWZqibRkOxPnmoR59+Zy5hjabfQc6JLSah4g==",
       "dev": true,
-      "hasInstallScript": true,
-      "optional": true,
-      "bin": {
-        "esbuild": "bin/esbuild"
+      "dependencies": {
+        "detect-file": "^1.0.0",
+        "is-glob": "^3.1.0",
+        "micromatch": "^3.0.4",
+        "resolve-dir": "^1.0.1"
       },
       "engines": {
-        "node": ">=12"
-      },
-      "optionalDependencies": {
-        "@esbuild/android-arm": "0.17.8",
-        "@esbuild/android-arm64": "0.17.8",
-        "@esbuild/android-x64": "0.17.8",
-        "@esbuild/darwin-arm64": "0.17.8",
-        "@esbuild/darwin-x64": "0.17.8",
-        "@esbuild/freebsd-arm64": "0.17.8",
-        "@esbuild/freebsd-x64": "0.17.8",
-        "@esbuild/linux-arm": "0.17.8",
-        "@esbuild/linux-arm64": "0.17.8",
-        "@esbuild/linux-ia32": "0.17.8",
-        "@esbuild/linux-loong64": "0.17.8",
-        "@esbuild/linux-mips64el": "0.17.8",
-        "@esbuild/linux-ppc64": "0.17.8",
-        "@esbuild/linux-riscv64": "0.17.8",
-        "@esbuild/linux-s390x": "0.17.8",
-        "@esbuild/linux-x64": "0.17.8",
-        "@esbuild/netbsd-x64": "0.17.8",
-        "@esbuild/openbsd-x64": "0.17.8",
-        "@esbuild/sunos-x64": "0.17.8",
-        "@esbuild/win32-arm64": "0.17.8",
-        "@esbuild/win32-ia32": "0.17.8",
-        "@esbuild/win32-x64": "0.17.8"
+        "node": ">= 0.10"
       }
     },
-    "node_modules/esbuild-wasm": {
-      "version": "0.17.8",
-      "resolved": "https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-0.17.8.tgz",
-      "integrity": "sha512-zCmpxv95E0FuCmvdw1K836UHnj4EdiQnFfjTby35y3LAjRPtXMj3sbHDRHjbD8Mqg5lTwq3knacr/1qIFU51CQ==",
+    "node_modules/findup-sync/node_modules/braces": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
       "dev": true,
-      "bin": {
-        "esbuild": "bin/esbuild"
+      "dependencies": {
+        "arr-flatten": "^1.1.0",
+        "array-unique": "^0.3.2",
+        "extend-shallow": "^2.0.1",
+        "fill-range": "^4.0.0",
+        "isobject": "^3.0.1",
+        "repeat-element": "^1.1.2",
+        "snapdragon": "^0.8.1",
+        "snapdragon-node": "^2.0.1",
+        "split-string": "^3.0.2",
+        "to-regex": "^3.0.1"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/escalade": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
-      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+    "node_modules/findup-sync/node_modules/braces/node_modules/extend-shallow": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+      "dev": true,
+      "dependencies": {
+        "is-extendable": "^0.1.0"
+      },
       "engines": {
-        "node": ">=6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/escape-html": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
-      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
-      "dev": true
-    },
-    "node_modules/escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
+    "node_modules/findup-sync/node_modules/fill-range": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
+      "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==",
+      "dev": true,
+      "dependencies": {
+        "extend-shallow": "^2.0.1",
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1",
+        "to-regex-range": "^2.1.0"
+      },
       "engines": {
-        "node": ">=0.8.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/escodegen": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz",
-      "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==",
+    "node_modules/findup-sync/node_modules/fill-range/node_modules/extend-shallow": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
       "dev": true,
       "dependencies": {
-        "esprima": "^4.0.1",
-        "estraverse": "^4.2.0",
-        "esutils": "^2.0.2",
-        "optionator": "^0.8.1"
-      },
-      "bin": {
-        "escodegen": "bin/escodegen.js",
-        "esgenerate": "bin/esgenerate.js"
+        "is-extendable": "^0.1.0"
       },
       "engines": {
-        "node": ">=4.0"
-      },
-      "optionalDependencies": {
-        "source-map": "~0.6.1"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/escodegen/node_modules/source-map": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+    "node_modules/findup-sync/node_modules/is-glob": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
+      "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==",
       "dev": true,
-      "optional": true,
+      "dependencies": {
+        "is-extglob": "^2.1.0"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint": {
-      "version": "8.17.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.17.0.tgz",
-      "integrity": "sha512-gq0m0BTJfci60Fz4nczYxNAlED+sMcihltndR8t9t1evnU/azx53x3t2UHXC/uRjcbvRw/XctpaNygSTcQD+Iw==",
+    "node_modules/findup-sync/node_modules/is-number": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
+      "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==",
       "dev": true,
       "dependencies": {
-        "@eslint/eslintrc": "^1.3.0",
-        "@humanwhocodes/config-array": "^0.9.2",
-        "ajv": "^6.10.0",
-        "chalk": "^4.0.0",
-        "cross-spawn": "^7.0.2",
-        "debug": "^4.3.2",
-        "doctrine": "^3.0.0",
-        "escape-string-regexp": "^4.0.0",
-        "eslint-scope": "^7.1.1",
-        "eslint-utils": "^3.0.0",
-        "eslint-visitor-keys": "^3.3.0",
-        "espree": "^9.3.2",
-        "esquery": "^1.4.0",
-        "esutils": "^2.0.2",
-        "fast-deep-equal": "^3.1.3",
-        "file-entry-cache": "^6.0.1",
-        "functional-red-black-tree": "^1.0.1",
-        "glob-parent": "^6.0.1",
-        "globals": "^13.15.0",
-        "ignore": "^5.2.0",
-        "import-fresh": "^3.0.0",
-        "imurmurhash": "^0.1.4",
-        "is-glob": "^4.0.0",
-        "js-yaml": "^4.1.0",
-        "json-stable-stringify-without-jsonify": "^1.0.1",
-        "levn": "^0.4.1",
-        "lodash.merge": "^4.6.2",
-        "minimatch": "^3.1.2",
-        "natural-compare": "^1.4.0",
-        "optionator": "^0.9.1",
-        "regexpp": "^3.2.0",
-        "strip-ansi": "^6.0.1",
-        "strip-json-comments": "^3.1.0",
-        "text-table": "^0.2.0",
-        "v8-compile-cache": "^2.0.3"
-      },
-      "bin": {
-        "eslint": "bin/eslint.js"
+        "kind-of": "^3.0.2"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://opencollective.com/eslint"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint-scope": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
-      "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+    "node_modules/findup-sync/node_modules/is-number/node_modules/kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
       "dev": true,
       "dependencies": {
-        "esrecurse": "^4.3.0",
-        "estraverse": "^4.1.1"
+        "is-buffer": "^1.1.5"
       },
       "engines": {
-        "node": ">=8.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint-utils": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz",
-      "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==",
+    "node_modules/findup-sync/node_modules/micromatch": {
+      "version": "3.1.10",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
+      "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
       "dev": true,
       "dependencies": {
-        "eslint-visitor-keys": "^2.0.0"
+        "arr-diff": "^4.0.0",
+        "array-unique": "^0.3.2",
+        "braces": "^2.3.1",
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "extglob": "^2.0.4",
+        "fragment-cache": "^0.2.1",
+        "kind-of": "^6.0.2",
+        "nanomatch": "^1.2.9",
+        "object.pick": "^1.3.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.2"
       },
       "engines": {
-        "node": "^10.0.0 || ^12.0.0 || >= 14.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/mysticatea"
-      },
-      "peerDependencies": {
-        "eslint": ">=5"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint-utils/node_modules/eslint-visitor-keys": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
-      "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
+    "node_modules/findup-sync/node_modules/to-regex-range": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz",
+      "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==",
       "dev": true,
+      "dependencies": {
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1"
+      },
       "engines": {
-        "node": ">=10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint-visitor-keys": {
-      "version": "3.4.3",
-      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz",
-      "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
+    "node_modules/fined": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/fined/-/fined-1.2.0.tgz",
+      "integrity": "sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng==",
       "dev": true,
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      "dependencies": {
+        "expand-tilde": "^2.0.2",
+        "is-plain-object": "^2.0.3",
+        "object.defaults": "^1.1.0",
+        "object.pick": "^1.2.0",
+        "parse-filepath": "^1.0.1"
       },
-      "funding": {
-        "url": "https://opencollective.com/eslint"
+      "engines": {
+        "node": ">= 0.10"
       }
     },
-    "node_modules/eslint/node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+    "node_modules/fined/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
       "dev": true,
       "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
-        "uri-js": "^4.2.2"
+        "isobject": "^3.0.1"
       },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
-    },
-    "node_modules/eslint/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/flagged-respawn": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz",
+      "integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q==",
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">= 0.10"
       }
     },
-    "node_modules/eslint/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/flat": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
+      "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
+      "dev": true,
+      "bin": {
+        "flat": "cli.js"
+      }
+    },
+    "node_modules/flat-cache": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz",
+      "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "flatted": "^3.2.9",
+        "keyv": "^4.5.3",
+        "rimraf": "^3.0.2"
       },
       "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+        "node": "^10.12.0 || >=12.0.0"
       }
     },
-    "node_modules/eslint/node_modules/escape-string-regexp": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
-      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
-      "dev": true,
+    "node_modules/flatted": {
+      "version": "3.2.9",
+      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz",
+      "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==",
+      "dev": true
+    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.3",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz",
+      "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
       "engines": {
-        "node": ">=10"
+        "node": ">=4.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
       }
     },
-    "node_modules/eslint/node_modules/eslint-scope": {
-      "version": "7.2.2",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz",
-      "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==",
+    "node_modules/for-each": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
+      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
       "dev": true,
       "dependencies": {
-        "esrecurse": "^4.3.0",
-        "estraverse": "^5.2.0"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://opencollective.com/eslint"
+        "is-callable": "^1.1.3"
       }
     },
-    "node_modules/eslint/node_modules/estraverse": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
-      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+    "node_modules/for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
       "dev": true,
       "engines": {
-        "node": ">=4.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint/node_modules/glob-parent": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
-      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+    "node_modules/for-own": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
+      "integrity": "sha512-0OABksIGrxKK8K4kynWkQ7y1zounQxP+CWnyclVwj81KW3vlLlGUx57DKGcP/LH216GzqnstnPocF16Nxs0Ycg==",
       "dev": true,
       "dependencies": {
-        "is-glob": "^4.0.3"
+        "for-in": "^1.0.1"
       },
       "engines": {
-        "node": ">=10.13.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint/node_modules/globals": {
-      "version": "13.22.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz",
-      "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==",
+    "node_modules/foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
       "dev": true,
       "dependencies": {
-        "type-fest": "^0.20.2"
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=14"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/eslint/node_modules/js-yaml": {
+    "node_modules/foreground-child/node_modules/signal-exit": {
       "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
       "dev": true,
-      "dependencies": {
-        "argparse": "^2.0.1"
+      "engines": {
+        "node": ">=14"
       },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/eslint/node_modules/json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
-      "dev": true
-    },
-    "node_modules/eslint/node_modules/optionator": {
-      "version": "0.9.3",
-      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
-      "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
+    "node_modules/forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==",
       "dev": true,
-      "dependencies": {
-        "@aashutoshrathi/word-wrap": "^1.2.3",
-        "deep-is": "^0.1.3",
-        "fast-levenshtein": "^2.0.6",
-        "levn": "^0.4.1",
-        "prelude-ls": "^1.2.1",
-        "type-check": "^0.4.0"
-      },
       "engines": {
-        "node": ">= 0.8.0"
+        "node": "*"
       }
     },
-    "node_modules/eslint/node_modules/type-fest": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
-      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
-      "dev": true,
+    "node_modules/fork-awesome": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/fork-awesome/-/fork-awesome-1.1.7.tgz",
+      "integrity": "sha512-IHI7XCSXrKfUIWslse8c/PaaVDT1oBaYge+ju40ihL2ooiQeBpTr4wvIXhgTd2NuhntlvX+M5jYHAPTzNlmv0g==",
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">=0.10.3"
       }
     },
-    "node_modules/espree": {
-      "version": "9.6.1",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
-      "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==",
-      "dev": true,
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
       "dependencies": {
-        "acorn": "^8.9.0",
-        "acorn-jsx": "^5.3.2",
-        "eslint-visitor-keys": "^3.4.1"
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
       },
       "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://opencollective.com/eslint"
+        "node": ">= 6"
+      }
+    },
+    "node_modules/format": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz",
+      "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==",
+      "engines": {
+        "node": ">=0.4.x"
       }
     },
-    "node_modules/espree/node_modules/acorn": {
-      "version": "8.10.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
-      "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
+    "node_modules/forwarded": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
       "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
       "engines": {
-        "node": ">=0.4.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/esprima": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
-      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+    "node_modules/fraction.js": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
+      "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==",
       "dev": true,
-      "bin": {
-        "esparse": "bin/esparse.js",
-        "esvalidate": "bin/esvalidate.js"
-      },
       "engines": {
-        "node": ">=4"
+        "node": "*"
+      },
+      "funding": {
+        "type": "patreon",
+        "url": "https://github.com/sponsors/rawify"
       }
     },
-    "node_modules/esquery": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz",
-      "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==",
+    "node_modules/fragment-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
+      "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==",
       "dev": true,
       "dependencies": {
-        "estraverse": "^5.1.0"
+        "map-cache": "^0.2.2"
       },
       "engines": {
-        "node": ">=0.10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/esquery/node_modules/estraverse": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
-      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+    "node_modules/fresh": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
+      "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
       "dev": true,
       "engines": {
-        "node": ">=4.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/esrecurse": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
-      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+    "node_modules/from": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
+      "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==",
+      "dev": true
+    },
+    "node_modules/fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "devOptional": true
+    },
+    "node_modules/fs-extra": {
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
+      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
       "dev": true,
       "dependencies": {
-        "estraverse": "^5.2.0"
+        "at-least-node": "^1.0.0",
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
       },
       "engines": {
-        "node": ">=4.0"
+        "node": ">=10"
       }
     },
-    "node_modules/esrecurse/node_modules/estraverse": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
-      "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+    "node_modules/fs-minipass": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz",
+      "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==",
       "dev": true,
+      "dependencies": {
+        "minipass": "^7.0.3"
+      },
       "engines": {
-        "node": ">=4.0"
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/estraverse": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
-      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+    "node_modules/fs-minipass/node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "dev": true,
       "engines": {
-        "node": ">=4.0"
+        "node": ">=16 || 14 >=14.17"
       }
     },
-    "node_modules/estree-is-function": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/estree-is-function/-/estree-is-function-1.0.0.tgz",
-      "integrity": "sha512-nSCWn1jkSq2QAtkaVLJZY2ezwcFO161HVc174zL1KPW3RJ+O6C3eJb8Nx7OXzvhoEv+nLgSR1g71oWUHUDTrJA==",
+    "node_modules/fs-monkey": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.5.tgz",
+      "integrity": "sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew==",
       "dev": true
     },
-    "node_modules/esutils": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
-      "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
+      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
       "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/function-bind": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/etag": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
-      "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
+    "node_modules/function.prototype.name": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz",
+      "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==",
       "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2",
+        "define-properties": "^1.2.0",
+        "es-abstract": "^1.22.1",
+        "functions-have-names": "^1.2.3"
+      },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/event-emitter": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
-      "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==",
+    "node_modules/functional-red-black-tree": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
+      "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==",
+      "dev": true
+    },
+    "node_modules/functions-have-names": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz",
+      "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==",
       "dev": true,
-      "dependencies": {
-        "d": "1",
-        "es5-ext": "~0.10.14"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/event-stream": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz",
-      "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==",
+    "node_modules/gauge": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz",
+      "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==",
       "dev": true,
       "dependencies": {
-        "duplexer": "^0.1.1",
-        "from": "^0.1.7",
-        "map-stream": "0.0.7",
-        "pause-stream": "^0.0.11",
-        "split": "^1.0.1",
-        "stream-combiner": "^0.2.2",
-        "through": "^2.3.8"
+        "aproba": "^1.0.3 || ^2.0.0",
+        "color-support": "^1.1.3",
+        "console-control-strings": "^1.1.0",
+        "has-unicode": "^2.0.1",
+        "signal-exit": "^3.0.7",
+        "string-width": "^4.2.3",
+        "strip-ansi": "^6.0.1",
+        "wide-align": "^1.1.5"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
       }
     },
-    "node_modules/event-target-shim": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
-      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-      "dev": true,
+    "node_modules/gensync": {
+      "version": "1.0.0-beta.2",
+      "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
+      "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
       "engines": {
-        "node": ">=6"
+        "node": ">=6.9.0"
       }
     },
-    "node_modules/eventemitter-asyncresource": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz",
-      "integrity": "sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==",
-      "dev": true
-    },
-    "node_modules/eventemitter2": {
-      "version": "6.4.7",
-      "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.7.tgz",
-      "integrity": "sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg==",
+    "node_modules/get-assigned-identifiers": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/get-assigned-identifiers/-/get-assigned-identifiers-1.2.0.tgz",
+      "integrity": "sha512-mBBwmeGTrxEMO4pMaaf/uUEFHnYtwr8FTe8Y/mer4rcV/bye0qGm6pw1bGZFGStxC5O76c5ZAVBGnqHmOaJpdQ==",
       "dev": true
     },
-    "node_modules/eventemitter3": {
-      "version": "4.0.7",
-      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
-      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
-      "dev": true
+    "node_modules/get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "engines": {
+        "node": "6.* || 8.* || >= 10.*"
+      }
     },
-    "node_modules/events": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-2.1.0.tgz",
-      "integrity": "sha512-3Zmiobend8P9DjmKAty0Era4jV8oJ0yGYe2nJJAxgymF9+N8F2m0hhZiMoWtcfepExzNKZumFU3ksdQbInGWCg==",
+    "node_modules/get-func-name": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
+      "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==",
       "dev": true,
       "engines": {
-        "node": ">=0.4.x"
+        "node": "*"
       }
     },
-    "node_modules/evp_bytestokey": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
-      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
-      "dev": true,
+    "node_modules/get-intrinsic": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
+      "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
       "dependencies": {
-        "md5.js": "^1.3.4",
-        "safe-buffer": "^5.1.1"
+        "function-bind": "^1.1.2",
+        "has-proto": "^1.0.1",
+        "has-symbols": "^1.0.3",
+        "hasown": "^2.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/execa": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
-      "integrity": "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==",
+    "node_modules/get-package-type": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
+      "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
       "dev": true,
-      "dependencies": {
-        "cross-spawn": "^7.0.0",
-        "get-stream": "^5.0.0",
-        "human-signals": "^1.1.1",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.0",
-        "onetime": "^5.1.0",
-        "signal-exit": "^3.0.2",
-        "strip-final-newline": "^2.0.0"
-      },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/execa/node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+    "node_modules/get-stdin": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
+      "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/execall": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/execall/-/execall-2.0.0.tgz",
-      "integrity": "sha512-0FU2hZ5Hh6iQnarpRtQurM/aAvp3RIbfvgLHrcqJYzhXyV2KFruhuChf9NC6waAhiUR7FFtlugkI4p7f2Fqlow==",
+    "node_modules/get-stream": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
+      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
       "dev": true,
-      "dependencies": {
-        "clone-regexp": "^2.1.0"
-      },
       "engines": {
-        "node": ">=8"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/executable": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/executable/-/executable-4.1.1.tgz",
-      "integrity": "sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg==",
+    "node_modules/get-symbol-description": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz",
+      "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==",
       "dev": true,
       "dependencies": {
-        "pify": "^2.2.0"
+        "call-bind": "^1.0.2",
+        "get-intrinsic": "^1.1.1"
       },
       "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/exit": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
-      "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8.0"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/expand-brackets": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-      "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==",
+    "node_modules/get-value": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
+      "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==",
       "dev": true,
-      "dependencies": {
-        "debug": "^2.3.3",
-        "define-property": "^0.2.5",
-        "extend-shallow": "^2.0.1",
-        "posix-character-classes": "^0.1.0",
-        "regex-not": "^1.0.0",
-        "snapdragon": "^0.8.1",
-        "to-regex": "^3.0.1"
-      },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/expand-brackets/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+    "node_modules/getos": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/getos/-/getos-3.2.1.tgz",
+      "integrity": "sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q==",
       "dev": true,
       "dependencies": {
-        "ms": "2.0.0"
+        "async": "^3.2.0"
       }
     },
-    "node_modules/expand-brackets/node_modules/define-property": {
-      "version": "0.2.5",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-      "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==",
+    "node_modules/getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==",
       "dev": true,
       "dependencies": {
-        "is-descriptor": "^0.1.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "assert-plus": "^1.0.0"
       }
     },
-    "node_modules/expand-brackets/node_modules/extend-shallow": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+    "node_modules/gherkin": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/gherkin/-/gherkin-5.1.0.tgz",
+      "integrity": "sha512-axTCsxH0m0cixijLvo7s9591h5pMb8ifQxFDun5FnfFhVsUhxgdnH0H7TSK7q8I4ASUU18DJ/tmlnMegMuLUUQ==",
+      "deprecated": "This package is now published under @cucumber/gherkin",
       "dev": true,
-      "dependencies": {
-        "is-extendable": "^0.1.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+      "bin": {
+        "gherkin-javascript": "bin/gherkin"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
+    "node_modules/gherkin-lint": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/gherkin-lint/-/gherkin-lint-4.2.2.tgz",
+      "integrity": "sha512-+vu0wbrwxaaEdrheU9pH2MYR6zk38u2IkrCIg6IETUw1lkrNVAfIfOCihwrrL2NTJv5Iia/C7hZEBNwjGSkL2Q==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^3.0.2"
+        "commander": "5.0.0",
+        "core-js": "3.6.4",
+        "gherkin": "9.0.0",
+        "glob": "7.1.6",
+        "lodash": "4.17.21",
+        "strip-json-comments": "3.0.1",
+        "xml-js": "^1.6.11"
+      },
+      "bin": {
+        "gherkin-lint": "dist/main.js"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10.0.0"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-accessor-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/gherkin-lint/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
+    "node_modules/gherkin-lint/node_modules/commander": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-5.0.0.tgz",
+      "integrity": "sha512-JrDGPAKjMGSP1G0DUoaceEJ3DZgAfr/q6X7FVk4+U5KxUSKviYGM2k6zWkfyyBHy5rAtzgYJFa1ro2O9PtoxwQ==",
       "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 6"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-data-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/gherkin-lint/node_modules/core-js": {
+      "version": "3.6.4",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.4.tgz",
+      "integrity": "sha512-4paDGScNgZP2IXXilaffL9X7968RuvwlkK3xWtZRVqgd8SYNiVKRJvkFd1aqqEuPfN7E68ZHEp9hDj6lHj4Hyw==",
+      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
       "dev": true,
-      "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
+    "node_modules/gherkin-lint/node_modules/gherkin": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/gherkin/-/gherkin-9.0.0.tgz",
+      "integrity": "sha512-6xoAepoxo5vhkBXjB4RCfVnSKHu5z9SqXIQVUyj+Jw8BQX8odATlee5otXgdN8llZvyvHokuvNiBeB3naEnnIQ==",
+      "deprecated": "This package is now published under @cucumber/gherkin",
       "dev": true,
       "dependencies": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
+        "commander": "^4.0.1",
+        "cucumber-messages": "8.0.0",
+        "source-map-support": "^0.5.16"
       },
-      "engines": {
-        "node": ">=0.10.0"
+      "bin": {
+        "gherkin-javascript": "bin/gherkin"
       }
     },
-    "node_modules/expand-brackets/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+    "node_modules/gherkin-lint/node_modules/gherkin/node_modules/commander": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
+      "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 6"
       }
     },
-    "node_modules/expand-brackets/node_modules/kind-of": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-      "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+    "node_modules/gherkin-lint/node_modules/glob": {
+      "version": "7.1.6",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+      "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
       "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/expand-brackets/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/expand-template": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
-      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
-      "optional": true,
-      "engines": {
-        "node": ">=6"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/expand-tilde": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
-      "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==",
+    "node_modules/gherkin-lint/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "homedir-polyfill": "^1.0.1"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "*"
       }
     },
-    "node_modules/expect": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz",
-      "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==",
+    "node_modules/gherkin-lint/node_modules/strip-json-comments": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz",
+      "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==",
       "dev": true,
-      "dependencies": {
-        "@jest/expect-utils": "^29.7.0",
-        "jest-get-type": "^29.6.3",
-        "jest-matcher-utils": "^29.7.0",
-        "jest-message-util": "^29.7.0",
-        "jest-util": "^29.7.0"
-      },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/exponential-backoff": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz",
-      "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==",
-      "dev": true
+    "node_modules/github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
+      "optional": true
     },
-    "node_modules/express": {
-      "version": "4.17.1",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
-      "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
-      "dev": true,
+    "node_modules/glob": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
+      "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
       "dependencies": {
-        "accepts": "~1.3.7",
-        "array-flatten": "1.1.1",
-        "body-parser": "1.19.0",
-        "content-disposition": "0.5.3",
-        "content-type": "~1.0.4",
-        "cookie": "0.4.0",
-        "cookie-signature": "1.0.6",
-        "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "etag": "~1.8.1",
-        "finalhandler": "~1.1.2",
-        "fresh": "0.5.2",
-        "merge-descriptors": "1.0.1",
-        "methods": "~1.1.2",
-        "on-finished": "~2.3.0",
-        "parseurl": "~1.3.3",
-        "path-to-regexp": "0.1.7",
-        "proxy-addr": "~2.0.5",
-        "qs": "6.7.0",
-        "range-parser": "~1.2.1",
-        "safe-buffer": "5.1.2",
-        "send": "0.17.1",
-        "serve-static": "1.14.1",
-        "setprototypeof": "1.1.1",
-        "statuses": "~1.5.0",
-        "type-is": "~1.6.18",
-        "utils-merge": "1.0.1",
-        "vary": "~1.1.2"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^5.0.1",
+        "once": "^1.3.0"
       },
       "engines": {
-        "node": ">= 0.10.0"
-      }
-    },
-    "node_modules/express/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.0.0"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/express/node_modules/destroy": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
-      "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==",
-      "dev": true
-    },
-    "node_modules/express/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/express/node_modules/safe-buffer": {
+    "node_modules/glob-parent": {
       "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-      "dev": true
-    },
-    "node_modules/express/node_modules/send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "dependencies": {
-        "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "destroy": "~1.0.4",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "etag": "~1.8.1",
-        "fresh": "0.5.2",
-        "http-errors": "~1.7.2",
-        "mime": "1.6.0",
-        "ms": "2.1.1",
-        "on-finished": "~2.3.0",
-        "range-parser": "~1.2.1",
-        "statuses": "~1.5.0"
+        "is-glob": "^4.0.1"
       },
       "engines": {
-        "node": ">= 0.8.0"
+        "node": ">= 6"
       }
     },
-    "node_modules/express/node_modules/send/node_modules/ms": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
+    "node_modules/glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
       "dev": true
     },
-    "node_modules/ext": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz",
-      "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==",
+    "node_modules/global-dirs": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.1.tgz",
+      "integrity": "sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==",
       "dev": true,
       "dependencies": {
-        "type": "^2.7.2"
+        "ini": "2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/ext/node_modules/type": {
-      "version": "2.7.2",
-      "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
-      "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==",
-      "dev": true
-    },
-    "node_modules/extend": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
-      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
-      "dev": true
+    "node_modules/global-dirs/node_modules/ini": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
+      "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
     },
-    "node_modules/extend-shallow": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-      "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==",
+    "node_modules/global-modules": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz",
+      "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==",
       "dev": true,
       "dependencies": {
-        "assign-symbols": "^1.0.0",
-        "is-extendable": "^1.0.1"
+        "global-prefix": "^1.0.1",
+        "is-windows": "^1.0.1",
+        "resolve-dir": "^1.0.0"
       },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/external-editor": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
-      "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==",
+    "node_modules/global-prefix": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
+      "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==",
       "dev": true,
       "dependencies": {
-        "chardet": "^0.7.0",
-        "iconv-lite": "^0.4.24",
-        "tmp": "^0.0.33"
+        "expand-tilde": "^2.0.2",
+        "homedir-polyfill": "^1.0.1",
+        "ini": "^1.3.4",
+        "is-windows": "^1.0.1",
+        "which": "^1.2.14"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/external-editor/node_modules/tmp": {
-      "version": "0.0.33",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
-      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
+    "node_modules/global-prefix/node_modules/which": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
       "dev": true,
       "dependencies": {
-        "os-tmpdir": "~1.0.2"
+        "isexe": "^2.0.0"
       },
+      "bin": {
+        "which": "bin/which"
+      }
+    },
+    "node_modules/globals": {
+      "version": "11.12.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
+      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
       "engines": {
-        "node": ">=0.6.0"
+        "node": ">=4"
       }
     },
-    "node_modules/extglob": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+    "node_modules/globalthis": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz",
+      "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==",
       "dev": true,
       "dependencies": {
-        "array-unique": "^0.3.2",
-        "define-property": "^1.0.0",
-        "expand-brackets": "^2.1.4",
-        "extend-shallow": "^2.0.1",
-        "fragment-cache": "^0.2.1",
-        "regex-not": "^1.0.0",
-        "snapdragon": "^0.8.1",
-        "to-regex": "^3.0.1"
+        "define-properties": "^1.1.3"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/extglob/node_modules/define-property": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-      "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==",
+    "node_modules/globby": {
+      "version": "13.2.2",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz",
+      "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==",
       "dev": true,
       "dependencies": {
-        "is-descriptor": "^1.0.0"
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.3.0",
+        "ignore": "^5.2.4",
+        "merge2": "^1.4.1",
+        "slash": "^4.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/extglob/node_modules/extend-shallow": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+    "node_modules/globjoin": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz",
+      "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==",
+      "dev": true
+    },
+    "node_modules/gonzales-pe": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz",
+      "integrity": "sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ==",
       "dev": true,
       "dependencies": {
-        "is-extendable": "^0.1.0"
+        "minimist": "^1.2.5"
+      },
+      "bin": {
+        "gonzales": "bin/gonzales.js"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=0.6.0"
       }
     },
-    "node_modules/extglob/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+    "node_modules/gopd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+      "dependencies": {
+        "get-intrinsic": "^1.1.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/extract-zip": {
+    "node_modules/graceful-fs": {
+      "version": "4.2.11",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+      "dev": true
+    },
+    "node_modules/handle-thing": {
       "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
-      "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
+      "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz",
+      "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==",
+      "dev": true
+    },
+    "node_modules/handlebars": {
+      "version": "4.7.8",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
+      "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==",
       "dev": true,
       "dependencies": {
-        "debug": "^4.1.1",
-        "get-stream": "^5.1.0",
-        "yauzl": "^2.10.0"
+        "minimist": "^1.2.5",
+        "neo-async": "^2.6.2",
+        "source-map": "^0.6.1",
+        "wordwrap": "^1.0.0"
       },
       "bin": {
-        "extract-zip": "cli.js"
+        "handlebars": "bin/handlebars"
       },
       "engines": {
-        "node": ">= 10.17.0"
+        "node": ">=0.4.7"
       },
       "optionalDependencies": {
-        "@types/yauzl": "^2.9.1"
+        "uglify-js": "^3.1.4"
       }
     },
-    "node_modules/extsprintf": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
-      "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==",
+    "node_modules/handlebars/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true,
-      "engines": [
-        "node >=0.6.0"
-      ]
+      "engines": {
+        "node": ">=0.10.0"
+      }
     },
-    "node_modules/fancy-log": {
+    "node_modules/har-schema": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/fancy-log/-/fancy-log-2.0.0.tgz",
-      "integrity": "sha512-9CzxZbACXMUXW13tS0tI8XsGGmxWzO2DmYrGuBJOJ8k8q2K7hwfJA5qHjuPPe8wtsco33YR9wc+Rlr5wYFvhSA==",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==",
       "dev": true,
-      "dependencies": {
-        "color-support": "^1.1.3"
-      },
       "engines": {
-        "node": ">=10.13.0"
+        "node": ">=4"
       }
     },
-    "node_modules/fast-deep-equal": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
-      "dev": true
-    },
-    "node_modules/fast-glob": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
-      "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==",
+    "node_modules/har-validator": {
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
+      "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
       "dev": true,
       "dependencies": {
-        "@nodelib/fs.stat": "^2.0.2",
-        "@nodelib/fs.walk": "^1.2.3",
-        "glob-parent": "^5.1.2",
-        "merge2": "^1.3.0",
-        "micromatch": "^4.0.4"
+        "ajv": "^6.12.3",
+        "har-schema": "^2.0.0"
       },
       "engines": {
-        "node": ">=8.6.0"
+        "node": ">=6"
       }
     },
-    "node_modules/fast-json-patch": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz",
-      "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ=="
-    },
-    "node_modules/fast-json-stable-stringify": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
-      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
-      "dev": true
+    "node_modules/har-validator/node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
     },
-    "node_modules/fast-levenshtein": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-      "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+    "node_modules/har-validator/node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
       "dev": true
     },
-    "node_modules/fast-safe-stringify": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
-      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==",
+    "node_modules/hard-rejection": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz",
+      "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/harmony-reflect": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.2.tgz",
+      "integrity": "sha512-HIp/n38R9kQjDEziXyDTuW3vvoxxyxjxFzXLrBr18uB47GnSt+G9D29fqrpM5ZkspMcPICud3XsBJQ4Y2URg8g==",
       "dev": true
     },
-    "node_modules/fastest-levenshtein": {
-      "version": "1.0.16",
-      "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz",
-      "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==",
+    "node_modules/has": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.4.tgz",
+      "integrity": "sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ==",
       "dev": true,
       "engines": {
-        "node": ">= 4.9.1"
+        "node": ">= 0.4.0"
       }
     },
-    "node_modules/fastq": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
-      "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
+    "node_modules/has-bigints": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
+      "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==",
       "dev": true,
-      "dependencies": {
-        "reusify": "^1.0.4"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/fault": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz",
-      "integrity": "sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==",
-      "dependencies": {
-        "format": "^0.2.0"
-      },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+    "node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/faye-websocket": {
-      "version": "0.11.4",
-      "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz",
-      "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==",
-      "dev": true,
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz",
+      "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==",
       "dependencies": {
-        "websocket-driver": ">=0.5.1"
+        "get-intrinsic": "^1.2.2"
       },
-      "engines": {
-        "node": ">=0.8.0"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/fb-watchman": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz",
-      "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==",
-      "dev": true,
-      "dependencies": {
-        "bser": "2.1.1"
+    "node_modules/has-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
+      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/fd-slicer": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
-      "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==",
-      "dev": true,
-      "dependencies": {
-        "pend": "~1.2.0"
+    "node_modules/has-symbols": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/figures": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
-      "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==",
+    "node_modules/has-tostringtag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
+      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
       "dev": true,
       "dependencies": {
-        "escape-string-regexp": "^1.0.5"
+        "has-symbols": "^1.0.2"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/file-entry-cache": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
-      "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+    "node_modules/has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==",
+      "dev": true
+    },
+    "node_modules/has-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
+      "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==",
       "dev": true,
       "dependencies": {
-        "flat-cache": "^3.0.4"
+        "get-value": "^2.0.6",
+        "has-values": "^1.0.0",
+        "isobject": "^3.0.0"
       },
       "engines": {
-        "node": "^10.12.0 || >=12.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/file-saver": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.2.tgz",
-      "integrity": "sha512-Wz3c3XQ5xroCxd1G8b7yL0Ehkf0TC9oYC6buPFkNnU9EnaPlifeAFCyCh+iewXTyFRcg0a6j3J7FmJsIhlhBdw=="
-    },
-    "node_modules/filelist": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
-      "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==",
+    "node_modules/has-values": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
+      "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==",
       "dev": true,
       "dependencies": {
-        "minimatch": "^5.0.1"
+        "is-number": "^3.0.0",
+        "kind-of": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/filelist/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+    "node_modules/has-values/node_modules/is-number": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
+      "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==",
       "dev": true,
       "dependencies": {
-        "balanced-match": "^1.0.0"
+        "kind-of": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/filelist/node_modules/minimatch": {
-      "version": "5.1.6",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
-      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+    "node_modules/has-values/node_modules/is-number/node_modules/kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
       "dev": true,
       "dependencies": {
-        "brace-expansion": "^2.0.1"
+        "is-buffer": "^1.1.5"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+    "node_modules/has-values/node_modules/kind-of": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
+      "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==",
       "dev": true,
       "dependencies": {
-        "to-regex-range": "^5.0.1"
+        "is-buffer": "^1.1.5"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/finalhandler": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
-      "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
+    "node_modules/hash-base": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz",
+      "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==",
       "dev": true,
       "dependencies": {
-        "debug": "2.6.9",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "on-finished": "~2.3.0",
-        "parseurl": "~1.3.3",
-        "statuses": "~1.5.0",
-        "unpipe": "~1.0.0"
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.6.0",
+        "safe-buffer": "^5.2.0"
       },
       "engines": {
-        "node": ">= 0.8"
+        "node": ">=4"
       }
     },
-    "node_modules/finalhandler/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+    "node_modules/hash.js": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz",
+      "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==",
       "dev": true,
       "dependencies": {
-        "ms": "2.0.0"
+        "inherits": "^2.0.3",
+        "minimalistic-assert": "^1.0.1"
       }
     },
-    "node_modules/finalhandler/node_modules/ms": {
+    "node_modules/hasown": {
       "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/find-cache-dir": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
-      "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==",
-      "dev": true,
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
+      "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
       "dependencies": {
-        "commondir": "^1.0.1",
-        "make-dir": "^3.0.2",
-        "pkg-dir": "^4.1.0"
+        "function-bind": "^1.1.2"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/hast-util-parse-selector": {
+      "version": "2.2.5",
+      "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz",
+      "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
+      }
+    },
+    "node_modules/hastscript": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz",
+      "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==",
+      "dependencies": {
+        "@types/hast": "^2.0.0",
+        "comma-separated-tokens": "^1.0.0",
+        "hast-util-parse-selector": "^2.0.0",
+        "property-information": "^5.0.0",
+        "space-separated-tokens": "^1.0.0"
       },
       "funding": {
-        "url": "https://github.com/avajs/find-cache-dir?sponsor=1"
+        "type": "opencollective",
+        "url": "https://opencollective.com/unified"
       }
     },
-    "node_modules/find-up": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
-      "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+    "node_modules/hdr-histogram-js": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-2.0.3.tgz",
+      "integrity": "sha512-Hkn78wwzWHNCp2uarhzQ2SGFLU3JY8SBDDd3TAABK4fc30wm+MuPOrg5QVFVfkKOQd6Bfz3ukJEI+q9sXEkK1g==",
       "dev": true,
       "dependencies": {
-        "locate-path": "^5.0.0",
-        "path-exists": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=8"
+        "@assemblyscript/loader": "^0.10.1",
+        "base64-js": "^1.2.0",
+        "pako": "^1.0.3"
       }
     },
-    "node_modules/findit2": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz",
-      "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==",
+    "node_modules/hdr-histogram-percentiles-obj": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz",
+      "integrity": "sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw==",
+      "dev": true
+    },
+    "node_modules/he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
       "dev": true,
+      "bin": {
+        "he": "bin/he"
+      }
+    },
+    "node_modules/highlight.js": {
+      "version": "10.7.3",
+      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
+      "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==",
       "engines": {
-        "node": ">=0.8.22"
+        "node": "*"
       }
     },
-    "node_modules/findup-sync": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-2.0.0.tgz",
-      "integrity": "sha512-vs+3unmJT45eczmcAZ6zMJtxN3l/QXeccaXQx5cu/MeJMhewVfoWZqibRkOxPnmoR59+Zy5hjabfQc6JLSah4g==",
+    "node_modules/hmac-drbg": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
+      "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==",
       "dev": true,
       "dependencies": {
-        "detect-file": "^1.0.0",
-        "is-glob": "^3.1.0",
-        "micromatch": "^3.0.4",
-        "resolve-dir": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 0.10"
+        "hash.js": "^1.0.3",
+        "minimalistic-assert": "^1.0.0",
+        "minimalistic-crypto-utils": "^1.0.1"
       }
     },
-    "node_modules/findup-sync/node_modules/braces": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
+    "node_modules/hoist-non-react-statics": {
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz",
+      "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==",
+      "dependencies": {
+        "react-is": "^16.7.0"
+      }
+    },
+    "node_modules/homedir-polyfill": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz",
+      "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==",
       "dev": true,
       "dependencies": {
-        "arr-flatten": "^1.1.0",
-        "array-unique": "^0.3.2",
-        "extend-shallow": "^2.0.1",
-        "fill-range": "^4.0.0",
-        "isobject": "^3.0.1",
-        "repeat-element": "^1.1.2",
-        "snapdragon": "^0.8.1",
-        "snapdragon-node": "^2.0.1",
-        "split-string": "^3.0.2",
-        "to-regex": "^3.0.1"
+        "parse-passwd": "^1.0.0"
       },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/findup-sync/node_modules/braces/node_modules/extend-shallow": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+    "node_modules/hosted-git-info": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-6.1.1.tgz",
+      "integrity": "sha512-r0EI+HBMcXadMrugk0GCQ+6BQV39PiWAZVfq7oIckeGiN7sjRGyQxPdft3nQekFTCQbYxLBH+/axZMeH8UX6+w==",
       "dev": true,
       "dependencies": {
-        "is-extendable": "^0.1.0"
+        "lru-cache": "^7.5.1"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/findup-sync/node_modules/fill-range": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-      "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==",
+    "node_modules/hosted-git-info/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
       "dev": true,
-      "dependencies": {
-        "extend-shallow": "^2.0.1",
-        "is-number": "^3.0.0",
-        "repeat-string": "^1.6.1",
-        "to-regex-range": "^2.1.0"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=12"
       }
     },
-    "node_modules/findup-sync/node_modules/fill-range/node_modules/extend-shallow": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+    "node_modules/hpack.js": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",
+      "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==",
       "dev": true,
       "dependencies": {
-        "is-extendable": "^0.1.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "inherits": "^2.0.1",
+        "obuf": "^1.0.0",
+        "readable-stream": "^2.0.1",
+        "wbuf": "^1.1.0"
       }
     },
-    "node_modules/findup-sync/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+    "node_modules/hpack.js/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
       "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
       }
     },
-    "node_modules/findup-sync/node_modules/is-glob": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
-      "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==",
+    "node_modules/hpack.js/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/hpack.js/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
       "dev": true,
       "dependencies": {
-        "is-extglob": "^2.1.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/findup-sync/node_modules/is-number": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-      "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==",
+    "node_modules/html-encoding-sniffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz",
+      "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "whatwg-encoding": "^1.0.1"
       }
     },
-    "node_modules/findup-sync/node_modules/is-number/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/html-entities": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.4.0.tgz",
+      "integrity": "sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/mdevils"
+        },
+        {
+          "type": "patreon",
+          "url": "https://patreon.com/mdevils"
+        }
+      ]
+    },
+    "node_modules/html-escaper": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
+      "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
+      "dev": true
+    },
+    "node_modules/html-linter": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/html-linter/-/html-linter-1.1.1.tgz",
+      "integrity": "sha512-DJfLevdq+YmY4R7yMdD0zaCAJOvWihg+eCe5o/jeTw86grvgmvSCIwBv1mDi+UdeJP9mDoi5rB6KtcFOn8StrA==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
+        "chalk": "^2.4.1",
+        "commander": "^2.12.2",
+        "glob": "^7.1.2"
       },
-      "engines": {
-        "node": ">=0.10.0"
+      "bin": {
+        "html-linter": "bin/html-linter.js"
       }
     },
-    "node_modules/findup-sync/node_modules/micromatch": {
-      "version": "3.1.10",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
-      "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
+    "node_modules/html-linter/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "arr-diff": "^4.0.0",
-        "array-unique": "^0.3.2",
-        "braces": "^2.3.1",
-        "define-property": "^2.0.2",
-        "extend-shallow": "^3.0.2",
-        "extglob": "^2.0.4",
-        "fragment-cache": "^0.2.1",
-        "kind-of": "^6.0.2",
-        "nanomatch": "^1.2.9",
-        "object.pick": "^1.3.0",
-        "regex-not": "^1.0.0",
-        "snapdragon": "^0.8.1",
-        "to-regex": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/findup-sync/node_modules/to-regex-range": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz",
-      "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==",
+    "node_modules/html-linter/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "is-number": "^3.0.0",
-        "repeat-string": "^1.6.1"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/fined": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/fined/-/fined-1.2.0.tgz",
-      "integrity": "sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng==",
+    "node_modules/html-linter/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "expand-tilde": "^2.0.2",
-        "is-plain-object": "^2.0.3",
-        "object.defaults": "^1.1.0",
-        "object.pick": "^1.2.0",
-        "parse-filepath": "^1.0.1"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">= 0.10"
+        "node": "*"
       }
     },
-    "node_modules/flagged-respawn": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz",
-      "integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q==",
+    "node_modules/html-tags": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.3.1.tgz",
+      "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==",
       "dev": true,
       "engines": {
-        "node": ">= 0.10"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/flat": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
-      "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
+    "node_modules/htmlescape": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/htmlescape/-/htmlescape-1.1.1.tgz",
+      "integrity": "sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg==",
       "dev": true,
-      "bin": {
-        "flat": "cli.js"
+      "engines": {
+        "node": ">=0.10"
       }
     },
-    "node_modules/flat-cache": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.0.tgz",
-      "integrity": "sha512-OHx4Qwrrt0E4jEIcI5/Xb+f+QmJYNj2rrK8wiIdQOIrB9WrrJL8cjZvXdXuBTkkEwEqLycb5BeZDV1o2i9bTew==",
+    "node_modules/htmllint": {
+      "version": "0.7.3",
+      "resolved": "https://registry.npmjs.org/htmllint/-/htmllint-0.7.3.tgz",
+      "integrity": "sha512-h8wfCu0CC0FVo18Jkygv7xqj0fa23Xlv4QsR2n34LDr8eqpf4glfbNg1HTbiCqpT3ONioMOfk8EkFUbZgrO1KA==",
       "dev": true,
       "dependencies": {
-        "flatted": "^3.2.7",
-        "keyv": "^4.5.3",
-        "rimraf": "^3.0.2"
+        "bulk-require": "^1.0.1",
+        "htmlparser2": "^3.10.0",
+        "lodash": "^4.17.11",
+        "promise": "^8.0.2"
       },
       "engines": {
-        "node": ">=12.0.0"
-      }
-    },
-    "node_modules/flatted": {
-      "version": "3.2.9",
-      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz",
-      "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==",
-      "dev": true
-    },
-    "node_modules/follow-redirects": {
-      "version": "1.15.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz",
-      "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==",
-      "funding": [
-        {
-          "type": "individual",
-          "url": "https://github.com/sponsors/RubenVerborgh"
-        }
-      ],
-      "engines": {
-        "node": ">=4.0"
-      },
-      "peerDependenciesMeta": {
-        "debug": {
-          "optional": true
-        }
+        "node": ">=4"
       }
     },
-    "node_modules/for-each": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
-      "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==",
+    "node_modules/htmllint-cli": {
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/htmllint-cli/-/htmllint-cli-0.0.7.tgz",
+      "integrity": "sha512-JREMzimj1HGrOUcA/FMMweLpHD5YRD1E5RnzXa4dj5m1zQ+6YYLklAGdaTar+qAGaj1Jasb3mGhH6FuKuLGWAQ==",
       "dev": true,
       "dependencies": {
-        "is-callable": "^1.1.3"
-      }
-    },
-    "node_modules/for-in": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
-      "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==",
-      "dev": true,
+        "bluebird": "^3.5.1",
+        "chalk": "^2.4.0",
+        "cjson": "^0.5.0",
+        "glob": "^7.1.2",
+        "htmllint": "^0.7.2",
+        "liftoff": "^2.5.0",
+        "semver": "^5.5.0",
+        "yargs": "^11.0.0"
+      },
+      "bin": {
+        "htmllint": "bin/cli.js"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4"
       }
     },
-    "node_modules/for-own": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
-      "integrity": "sha512-0OABksIGrxKK8K4kynWkQ7y1zounQxP+CWnyclVwj81KW3vlLlGUx57DKGcP/LH216GzqnstnPocF16Nxs0Ycg==",
+    "node_modules/htmllint-cli/node_modules/ansi-regex": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz",
+      "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==",
       "dev": true,
-      "dependencies": {
-        "for-in": "^1.0.1"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4"
       }
     },
-    "node_modules/foreground-child": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
-      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
+    "node_modules/htmllint-cli/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "cross-spawn": "^7.0.0",
-        "signal-exit": "^4.0.1"
-      },
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/foreground-child/node_modules/signal-exit": {
+    "node_modules/htmllint-cli/node_modules/camelcase": {
       "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
-      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
+      "integrity": "sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==",
       "dev": true,
       "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=4"
       }
     },
-    "node_modules/forever-agent": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
-      "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==",
+    "node_modules/htmllint-cli/node_modules/cliui": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
+      "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
       "dev": true,
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/fork-awesome": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/fork-awesome/-/fork-awesome-1.1.7.tgz",
-      "integrity": "sha512-IHI7XCSXrKfUIWslse8c/PaaVDT1oBaYge+ju40ihL2ooiQeBpTr4wvIXhgTd2NuhntlvX+M5jYHAPTzNlmv0g==",
-      "engines": {
-        "node": ">=0.10.3"
+      "dependencies": {
+        "string-width": "^2.1.1",
+        "strip-ansi": "^4.0.0",
+        "wrap-ansi": "^2.0.0"
       }
     },
-    "node_modules/form-data": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
-      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+    "node_modules/htmllint-cli/node_modules/find-up": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+      "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==",
       "dev": true,
       "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.6",
-        "mime-types": "^2.1.12"
+        "locate-path": "^2.0.0"
       },
       "engines": {
-        "node": ">= 0.12"
+        "node": ">=4"
       }
     },
-    "node_modules/format": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz",
-      "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==",
-      "engines": {
-        "node": ">=0.4.x"
-      }
+    "node_modules/htmllint-cli/node_modules/get-caller-file": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
+      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
+      "dev": true
     },
-    "node_modules/forwarded": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
-      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
+    "node_modules/htmllint-cli/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
       "engines": {
-        "node": ">= 0.6"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/fraction.js": {
-      "version": "4.3.6",
-      "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.6.tgz",
-      "integrity": "sha512-n2aZ9tNfYDwaHhvFTkhFErqOMIb8uyzSQ+vGJBjZyanAKZVbGUQ1sngfk9FdkBw7G26O7AgNjLcecLffD1c7eg==",
+    "node_modules/htmllint-cli/node_modules/is-fullwidth-code-point": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+      "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==",
       "dev": true,
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "type": "patreon",
-        "url": "https://github.com/sponsors/rawify"
+        "node": ">=4"
       }
     },
-    "node_modules/fragment-cache": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
-      "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==",
+    "node_modules/htmllint-cli/node_modules/locate-path": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+      "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==",
       "dev": true,
       "dependencies": {
-        "map-cache": "^0.2.2"
+        "p-locate": "^2.0.0",
+        "path-exists": "^3.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=4"
       }
     },
-    "node_modules/fresh": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
-      "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
+    "node_modules/htmllint-cli/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
       "engines": {
-        "node": ">= 0.6"
+        "node": "*"
       }
     },
-    "node_modules/from": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
-      "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==",
-      "dev": true
-    },
-    "node_modules/fs-constants": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
-      "devOptional": true
-    },
-    "node_modules/fs-extra": {
-      "version": "10.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
-      "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+    "node_modules/htmllint-cli/node_modules/p-limit": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+      "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
       "dev": true,
       "dependencies": {
-        "graceful-fs": "^4.2.0",
-        "jsonfile": "^6.0.1",
-        "universalify": "^2.0.0"
+        "p-try": "^1.0.0"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=4"
       }
     },
-    "node_modules/fs-minipass": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz",
-      "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==",
+    "node_modules/htmllint-cli/node_modules/p-locate": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+      "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==",
       "dev": true,
       "dependencies": {
-        "minipass": "^7.0.3"
+        "p-limit": "^1.1.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=4"
       }
     },
-    "node_modules/fs-minipass/node_modules/minipass": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+    "node_modules/htmllint-cli/node_modules/p-try": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+      "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==",
       "dev": true,
       "engines": {
-        "node": ">=16 || 14 >=14.17"
+        "node": ">=4"
       }
     },
-    "node_modules/fs-monkey": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.4.tgz",
-      "integrity": "sha512-INM/fWAxMICjttnD0DX1rBvinKskj5G1w+oy/pnm9u/tSlnBrzFonJMcalKJ30P8RRsPzKcCG7Q8l0jx5Fh9YQ==",
-      "dev": true
-    },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
-    },
-    "node_modules/fsevents": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
-      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
+    "node_modules/htmllint-cli/node_modules/path-exists": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+      "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==",
       "dev": true,
-      "hasInstallScript": true,
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
       "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+        "node": ">=4"
       }
     },
-    "node_modules/function-bind": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+    "node_modules/htmllint-cli/node_modules/semver": {
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver"
+      }
     },
-    "node_modules/function.prototype.name": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz",
-      "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==",
+    "node_modules/htmllint-cli/node_modules/string-width": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
+      "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.2.0",
-        "es-abstract": "^1.22.1",
-        "functions-have-names": "^1.2.3"
+        "is-fullwidth-code-point": "^2.0.0",
+        "strip-ansi": "^4.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=4"
       }
     },
-    "node_modules/functional-red-black-tree": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
-      "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==",
-      "dev": true
-    },
-    "node_modules/functions-have-names": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz",
-      "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==",
+    "node_modules/htmllint-cli/node_modules/strip-ansi": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+      "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==",
       "dev": true,
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "dependencies": {
+        "ansi-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/gauge": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz",
-      "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==",
+    "node_modules/htmllint-cli/node_modules/wrap-ansi": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
+      "integrity": "sha512-vAaEaDM946gbNpH5pLVNR+vX2ht6n0Bt3GXwVB1AuAqZosOvHNF3P7wDnh8KLkSqgUh0uh77le7Owgoz+Z9XBw==",
       "dev": true,
       "dependencies": {
-        "aproba": "^1.0.3 || ^2.0.0",
-        "color-support": "^1.1.3",
-        "console-control-strings": "^1.1.0",
-        "has-unicode": "^2.0.1",
-        "signal-exit": "^3.0.7",
-        "string-width": "^4.2.3",
-        "strip-ansi": "^6.0.1",
-        "wide-align": "^1.1.5"
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/gensync": {
-      "version": "1.0.0-beta.2",
-      "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
-      "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
+    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/ansi-regex": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+      "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==",
+      "dev": true,
       "engines": {
-        "node": ">=6.9.0"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/get-assigned-identifiers": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/get-assigned-identifiers/-/get-assigned-identifiers-1.2.0.tgz",
-      "integrity": "sha512-mBBwmeGTrxEMO4pMaaf/uUEFHnYtwr8FTe8Y/mer4rcV/bye0qGm6pw1bGZFGStxC5O76c5ZAVBGnqHmOaJpdQ==",
-      "dev": true
-    },
-    "node_modules/get-caller-file": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
-      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==",
+      "dev": true,
+      "dependencies": {
+        "number-is-nan": "^1.0.0"
+      },
       "engines": {
-        "node": "6.* || 8.* || >= 10.*"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/get-func-name": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz",
-      "integrity": "sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==",
+    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/string-width": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+      "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==",
       "dev": true,
+      "dependencies": {
+        "code-point-at": "^1.0.0",
+        "is-fullwidth-code-point": "^1.0.0",
+        "strip-ansi": "^3.0.0"
+      },
       "engines": {
-        "node": "*"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/get-intrinsic": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz",
-      "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==",
+    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==",
+      "dev": true,
       "dependencies": {
-        "function-bind": "^1.1.1",
-        "has": "^1.0.3",
-        "has-proto": "^1.0.1",
-        "has-symbols": "^1.0.3"
+        "ansi-regex": "^2.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/get-package-type": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
-      "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
+    "node_modules/htmllint-cli/node_modules/y18n": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz",
+      "integrity": "sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ==",
+      "dev": true
+    },
+    "node_modules/htmllint-cli/node_modules/yargs": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.1.tgz",
+      "integrity": "sha512-PRU7gJrJaXv3q3yQZ/+/X6KBswZiaQ+zOmdprZcouPYtQgvNU35i+68M4b1ZHLZtYFT5QObFLV+ZkmJYcwKdiw==",
       "dev": true,
-      "engines": {
-        "node": ">=8.0.0"
+      "dependencies": {
+        "cliui": "^4.0.0",
+        "decamelize": "^1.1.1",
+        "find-up": "^2.1.0",
+        "get-caller-file": "^1.0.1",
+        "os-locale": "^3.1.0",
+        "require-directory": "^2.1.1",
+        "require-main-filename": "^1.0.1",
+        "set-blocking": "^2.0.0",
+        "string-width": "^2.0.0",
+        "which-module": "^2.0.0",
+        "y18n": "^3.2.1",
+        "yargs-parser": "^9.0.2"
       }
     },
-    "node_modules/get-stdin": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz",
-      "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==",
+    "node_modules/htmllint-cli/node_modules/yargs-parser": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz",
+      "integrity": "sha512-CswCfdOgCr4MMsT1GzbEJ7Z2uYudWyrGX8Bgh/0eyCzj/DXWdKq6a/ADufkzI1WAOIW6jYaXJvRyLhDO0kfqBw==",
       "dev": true,
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "dependencies": {
+        "camelcase": "^4.1.0"
       }
     },
-    "node_modules/get-stream": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
-      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+    "node_modules/htmllint/node_modules/dom-serializer": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz",
+      "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==",
       "dev": true,
       "dependencies": {
-        "pump": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "domelementtype": "^2.0.1",
+        "entities": "^2.0.0"
       }
     },
-    "node_modules/get-symbol-description": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz",
-      "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==",
+    "node_modules/htmllint/node_modules/dom-serializer/node_modules/domelementtype": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ]
+    },
+    "node_modules/htmllint/node_modules/dom-serializer/node_modules/entities": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
       "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "get-intrinsic": "^1.1.1"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/get-value": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
-      "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==",
+    "node_modules/htmllint/node_modules/domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==",
+      "dev": true
+    },
+    "node_modules/htmllint/node_modules/domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
       "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+      "dependencies": {
+        "domelementtype": "1"
       }
     },
-    "node_modules/getos": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/getos/-/getos-3.2.1.tgz",
-      "integrity": "sha512-U56CfOK17OKgTVqozZjUKNdkfEv6jk5WISBJ8SHoagjE6L69zOwl3Z+O8myjY9MEW3i2HPWQBt/LTbCgcC973Q==",
+    "node_modules/htmllint/node_modules/domutils": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz",
+      "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==",
       "dev": true,
       "dependencies": {
-        "async": "^3.2.0"
+        "dom-serializer": "0",
+        "domelementtype": "1"
       }
     },
-    "node_modules/getpass": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
-      "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==",
+    "node_modules/htmllint/node_modules/entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==",
+      "dev": true
+    },
+    "node_modules/htmllint/node_modules/htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
       "dev": true,
       "dependencies": {
-        "assert-plus": "^1.0.0"
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
       }
     },
-    "node_modules/gherkin": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/gherkin/-/gherkin-5.1.0.tgz",
-      "integrity": "sha512-axTCsxH0m0cixijLvo7s9591h5pMb8ifQxFDun5FnfFhVsUhxgdnH0H7TSK7q8I4ASUU18DJ/tmlnMegMuLUUQ==",
-      "deprecated": "This package is now published under @cucumber/gherkin",
+    "node_modules/htmlparser2": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
+      "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==",
       "dev": true,
-      "bin": {
-        "gherkin-javascript": "bin/gherkin"
+      "funding": [
+        "https://github.com/fb55/htmlparser2?sponsor=1",
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.0.1",
+        "entities": "^4.4.0"
       }
     },
-    "node_modules/gherkin-lint": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/gherkin-lint/-/gherkin-lint-4.2.2.tgz",
-      "integrity": "sha512-+vu0wbrwxaaEdrheU9pH2MYR6zk38u2IkrCIg6IETUw1lkrNVAfIfOCihwrrL2NTJv5Iia/C7hZEBNwjGSkL2Q==",
+    "node_modules/htmlparser2/node_modules/dom-serializer": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+      "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
       "dev": true,
       "dependencies": {
-        "commander": "5.0.0",
-        "core-js": "3.6.4",
-        "gherkin": "9.0.0",
-        "glob": "7.1.6",
-        "lodash": "4.17.21",
-        "strip-json-comments": "3.0.1",
-        "xml-js": "^1.6.11"
-      },
-      "bin": {
-        "gherkin-lint": "dist/main.js"
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.2",
+        "entities": "^4.2.0"
       },
-      "engines": {
-        "node": ">=10.0.0"
+      "funding": {
+        "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
       }
     },
-    "node_modules/gherkin-lint/node_modules/commander": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-5.0.0.tgz",
-      "integrity": "sha512-JrDGPAKjMGSP1G0DUoaceEJ3DZgAfr/q6X7FVk4+U5KxUSKviYGM2k6zWkfyyBHy5rAtzgYJFa1ro2O9PtoxwQ==",
+    "node_modules/htmlparser2/node_modules/domhandler": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+      "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
       "dev": true,
+      "dependencies": {
+        "domelementtype": "^2.3.0"
+      },
       "engines": {
-        "node": ">= 6"
-      }
-    },
-    "node_modules/gherkin-lint/node_modules/core-js": {
-      "version": "3.6.4",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.4.tgz",
-      "integrity": "sha512-4paDGScNgZP2IXXilaffL9X7968RuvwlkK3xWtZRVqgd8SYNiVKRJvkFd1aqqEuPfN7E68ZHEp9hDj6lHj4Hyw==",
-      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
-      "dev": true,
-      "hasInstallScript": true,
+        "node": ">= 4"
+      },
       "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
+        "url": "https://github.com/fb55/domhandler?sponsor=1"
       }
     },
-    "node_modules/gherkin-lint/node_modules/gherkin": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/gherkin/-/gherkin-9.0.0.tgz",
-      "integrity": "sha512-6xoAepoxo5vhkBXjB4RCfVnSKHu5z9SqXIQVUyj+Jw8BQX8odATlee5otXgdN8llZvyvHokuvNiBeB3naEnnIQ==",
-      "deprecated": "This package is now published under @cucumber/gherkin",
+    "node_modules/htmlparser2/node_modules/domutils": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
+      "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
       "dev": true,
       "dependencies": {
-        "commander": "^4.0.1",
-        "cucumber-messages": "8.0.0",
-        "source-map-support": "^0.5.16"
+        "dom-serializer": "^2.0.0",
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3"
       },
-      "bin": {
-        "gherkin-javascript": "bin/gherkin"
+      "funding": {
+        "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
-    "node_modules/gherkin-lint/node_modules/gherkin/node_modules/commander": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
-      "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
+    "node_modules/htmlparser2/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
       "dev": true,
       "engines": {
-        "node": ">= 6"
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/gherkin-lint/node_modules/glob": {
-      "version": "7.1.6",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
-      "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+    "node_modules/http-auth": {
+      "version": "4.1.9",
+      "resolved": "https://registry.npmjs.org/http-auth/-/http-auth-4.1.9.tgz",
+      "integrity": "sha512-kvPYxNGc9EKGTXvOMnTBQw2RZfuiSihK/mLw/a4pbtRueTE45S55Lw/3k5CktIf7Ak0veMKEIteDj4YkNmCzmQ==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.0.4",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "apache-crypt": "^1.1.2",
+        "apache-md5": "^1.0.6",
+        "bcryptjs": "^2.4.3",
+        "uuid": "^8.3.2"
       },
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=8"
       }
     },
-    "node_modules/gherkin-lint/node_modules/strip-json-comments": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz",
-      "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==",
+    "node_modules/http-auth-connect": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/http-auth-connect/-/http-auth-connect-1.0.6.tgz",
+      "integrity": "sha512-yaO0QSCPqGCjPrl3qEEHjJP+lwZ6gMpXLuCBE06eWwcXomkI5TARtu0kxf9teFuBj6iaV3Ybr15jaWUvbzNzHw==",
       "dev": true,
       "engines": {
         "node": ">=8"
       }
     },
-    "node_modules/github-from-package": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
-      "optional": true
+    "node_modules/http-cache-semantics": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
+      "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==",
+      "dev": true
     },
-    "node_modules/glob": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
-      "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^5.0.1",
-        "once": "^1.3.0"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
+    "node_modules/http-deceiver": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz",
+      "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==",
+      "dev": true
     },
-    "node_modules/glob-parent": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
-      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+    "node_modules/http-errors": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
       "dev": true,
       "dependencies": {
-        "is-glob": "^4.0.1"
+        "depd": "2.0.0",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
+        "statuses": "2.0.1",
+        "toidentifier": "1.0.1"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">= 0.8"
       }
     },
-    "node_modules/glob-to-regexp": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
-      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
+    "node_modules/http-parser-js": {
+      "version": "0.5.8",
+      "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz",
+      "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==",
       "dev": true
     },
-    "node_modules/glob/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+    "node_modules/http-proxy": {
+      "version": "1.18.1",
+      "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz",
+      "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==",
+      "dev": true,
       "dependencies": {
-        "balanced-match": "^1.0.0"
+        "eventemitter3": "^4.0.0",
+        "follow-redirects": "^1.0.0",
+        "requires-port": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
       }
     },
-    "node_modules/glob/node_modules/minimatch": {
-      "version": "5.1.6",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
-      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+    "node_modules/http-proxy-agent": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
+      "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
+      "dev": true,
       "dependencies": {
-        "brace-expansion": "^2.0.1"
+        "@tootallnate/once": "2",
+        "agent-base": "6",
+        "debug": "4"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">= 6"
       }
     },
-    "node_modules/global-dirs": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.1.tgz",
-      "integrity": "sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==",
+    "node_modules/http-proxy-middleware": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz",
+      "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==",
       "dev": true,
       "dependencies": {
-        "ini": "2.0.0"
+        "@types/http-proxy": "^1.17.8",
+        "http-proxy": "^1.18.1",
+        "is-glob": "^4.0.1",
+        "is-plain-obj": "^3.0.0",
+        "micromatch": "^4.0.2"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=12.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "peerDependencies": {
+        "@types/express": "^4.17.13"
+      },
+      "peerDependenciesMeta": {
+        "@types/express": {
+          "optional": true
+        }
       }
     },
-    "node_modules/global-dirs/node_modules/ini": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
-      "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
+    "node_modules/http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==",
       "dev": true,
+      "dependencies": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^1.2.2",
+        "sshpk": "^1.7.0"
+      },
       "engines": {
-        "node": ">=10"
+        "node": ">=0.8",
+        "npm": ">=1.3.7"
       }
     },
-    "node_modules/global-modules": {
+    "node_modules/https-browserify": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz",
-      "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==",
+      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
+      "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==",
+      "dev": true
+    },
+    "node_modules/https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
       "dev": true,
       "dependencies": {
-        "global-prefix": "^1.0.1",
-        "is-windows": "^1.0.1",
-        "resolve-dir": "^1.0.0"
+        "agent-base": "6",
+        "debug": "4"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 6"
       }
     },
-    "node_modules/global-prefix": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
-      "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==",
+    "node_modules/human-signals": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
+      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
       "dev": true,
-      "dependencies": {
-        "expand-tilde": "^2.0.2",
-        "homedir-polyfill": "^1.0.1",
-        "ini": "^1.3.4",
-        "is-windows": "^1.0.1",
-        "which": "^1.2.14"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10.17.0"
       }
     },
-    "node_modules/global-prefix/node_modules/ini": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-      "dev": true
-    },
-    "node_modules/global-prefix/node_modules/which": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
-      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+    "node_modules/humanize-ms": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
+      "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
       "dev": true,
       "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "which": "bin/which"
+        "ms": "^2.0.0"
       }
     },
-    "node_modules/globals": {
-      "version": "11.12.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
-      "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==",
-      "engines": {
-        "node": ">=4"
+    "node_modules/i18next": {
+      "version": "21.10.0",
+      "resolved": "https://registry.npmjs.org/i18next/-/i18next-21.10.0.tgz",
+      "integrity": "sha512-YeuIBmFsGjUfO3qBmMOc0rQaun4mIpGKET5WDwvu8lU7gvwpcariZLNtL0Fzj+zazcHUrlXHiptcFhBMFaxzfg==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://locize.com"
+        },
+        {
+          "type": "individual",
+          "url": "https://locize.com/i18next.html"
+        },
+        {
+          "type": "individual",
+          "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project"
+        }
+      ],
+      "dependencies": {
+        "@babel/runtime": "^7.17.2"
       }
     },
-    "node_modules/globalthis": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz",
-      "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==",
+    "node_modules/iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
       "dev": true,
       "dependencies": {
-        "define-properties": "^1.1.3"
+        "safer-buffer": ">= 2.1.2 < 3"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/icss-utils": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
+      "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==",
+      "dev": true,
+      "engines": {
+        "node": "^10 || ^12 || >= 14"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "peerDependencies": {
+        "postcss": "^8.1.0"
       }
     },
-    "node_modules/globby": {
-      "version": "11.1.0",
-      "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz",
-      "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==",
+    "node_modules/identity-obj-proxy": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz",
+      "integrity": "sha512-00n6YnVHKrinT9t0d9+5yZC6UBNJANpYEQvL2LlX6Ab9lnmxzIRcEmTPuyGScvl1+jKuCICX1Z0Ab1pPKKdikA==",
       "dev": true,
       "dependencies": {
-        "array-union": "^2.1.0",
-        "dir-glob": "^3.0.1",
-        "fast-glob": "^3.2.9",
-        "ignore": "^5.2.0",
-        "merge2": "^1.4.1",
-        "slash": "^3.0.0"
+        "harmony-reflect": "^1.4.6"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">=4"
       }
     },
-    "node_modules/globjoin": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz",
-      "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==",
-      "dev": true
+    "node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
     },
-    "node_modules/gonzales-pe": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz",
-      "integrity": "sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ==",
+    "node_modules/ignore": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz",
+      "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 4"
+      }
+    },
+    "node_modules/ignore-walk": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-6.0.3.tgz",
+      "integrity": "sha512-C7FfFoTA+bI10qfeydT8aZbvr91vAEU+2W5BZUlzPec47oNb07SsOfwYrtxuvOYdUApPP/Qlh4DtAO51Ekk2QA==",
       "dev": true,
       "dependencies": {
-        "minimist": "^1.2.5"
-      },
-      "bin": {
-        "gonzales": "bin/gonzales.js"
+        "minimatch": "^9.0.0"
       },
       "engines": {
-        "node": ">=0.6.0"
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/gopd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
-      "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+    "node_modules/ignore-walk/node_modules/minimatch": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
+      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
       "dev": true,
       "dependencies": {
-        "get-intrinsic": "^1.1.3"
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/graceful-fs": {
-      "version": "4.2.11",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
-      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
-      "dev": true
-    },
-    "node_modules/handle-thing": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz",
-      "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==",
-      "dev": true
-    },
-    "node_modules/handlebars": {
-      "version": "4.7.8",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
-      "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==",
+    "node_modules/image-size": {
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz",
+      "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==",
       "dev": true,
-      "dependencies": {
-        "minimist": "^1.2.5",
-        "neo-async": "^2.6.2",
-        "source-map": "^0.6.1",
-        "wordwrap": "^1.0.0"
-      },
+      "optional": true,
       "bin": {
-        "handlebars": "bin/handlebars"
+        "image-size": "bin/image-size.js"
       },
       "engines": {
-        "node": ">=0.4.7"
-      },
-      "optionalDependencies": {
-        "uglify-js": "^3.1.4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/handlebars/node_modules/source-map": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-      "dev": true,
+    "node_modules/immutable": {
+      "version": "3.8.2",
+      "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz",
+      "integrity": "sha512-15gZoQ38eYjEjxkorfbcgBKBL6R7T459OuK+CpcWt7O3KF4uPCx2tD0uFETlUDIyo+1789crbMhTvQBSR5yBMg==",
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/har-schema": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
-      "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==",
+    "node_modules/import-fresh": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+      "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+      "dev": true,
+      "dependencies": {
+        "parent-module": "^1.0.0",
+        "resolve-from": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/import-fresh/node_modules/resolve-from": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+      "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
       "dev": true,
       "engines": {
         "node": ">=4"
       }
     },
-    "node_modules/har-validator": {
-      "version": "5.1.5",
-      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
-      "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
-      "deprecated": "this library is no longer supported",
+    "node_modules/import-lazy": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-4.0.0.tgz",
+      "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==",
       "dev": true,
-      "dependencies": {
-        "ajv": "^6.12.3",
-        "har-schema": "^2.0.0"
-      },
       "engines": {
-        "node": ">=6"
+        "node": ">=8"
       }
     },
-    "node_modules/har-validator/node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+    "node_modules/import-local": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz",
+      "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==",
       "dev": true,
       "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
-        "uri-js": "^4.2.2"
+        "pkg-dir": "^4.2.0",
+        "resolve-cwd": "^3.0.0"
+      },
+      "bin": {
+        "import-local-fixture": "fixtures/cli.js"
+      },
+      "engines": {
+        "node": ">=8"
       },
       "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/har-validator/node_modules/json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
-      "dev": true
+    "node_modules/imurmurhash": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8.19"
+      }
     },
-    "node_modules/hard-rejection": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz",
-      "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==",
+    "node_modules/indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
       "dev": true,
       "engines": {
-        "node": ">=6"
+        "node": ">=8"
       }
     },
-    "node_modules/harmony-reflect": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.2.tgz",
-      "integrity": "sha512-HIp/n38R9kQjDEziXyDTuW3vvoxxyxjxFzXLrBr18uB47GnSt+G9D29fqrpM5ZkspMcPICud3XsBJQ4Y2URg8g==",
+    "node_modules/infer-owner": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz",
+      "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==",
       "dev": true
     },
-    "node_modules/has": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
-      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
       "dependencies": {
-        "function-bind": "^1.1.1"
-      },
-      "engines": {
-        "node": ">= 0.4.0"
+        "once": "^1.3.0",
+        "wrappy": "1"
       }
     },
-    "node_modules/has-bigints": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
-      "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==",
+    "node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    },
+    "node_modules/ini": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+      "devOptional": true
+    },
+    "node_modules/inline-source-map": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz",
+      "integrity": "sha512-0mVWSSbNDvedDWIN4wxLsdPM4a7cIPcpyMxj3QZ406QRwQ6ePGB1YIHxVPjqpcUGbWQ5C+nHTwGNWAGvt7ggVA==",
       "dev": true,
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "dependencies": {
+        "source-map": "~0.5.3"
       }
     },
-    "node_modules/has-flag": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+    "node_modules/inline-source-map/node_modules/source-map": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
+      "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/has-property-descriptors": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
-      "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
+    "node_modules/inquirer": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.4.tgz",
+      "integrity": "sha512-nn4F01dxU8VeKfq192IjLsxu0/OmMZ4Lg3xKAns148rCaXP6ntAoEkVYZThWjwON8AlzdZZi6oqnhNbxUG9hVg==",
       "dev": true,
       "dependencies": {
-        "get-intrinsic": "^1.1.1"
+        "ansi-escapes": "^4.2.1",
+        "chalk": "^4.1.1",
+        "cli-cursor": "^3.1.0",
+        "cli-width": "^3.0.0",
+        "external-editor": "^3.0.3",
+        "figures": "^3.0.0",
+        "lodash": "^4.17.21",
+        "mute-stream": "0.0.8",
+        "ora": "^5.4.1",
+        "run-async": "^2.4.0",
+        "rxjs": "^7.5.5",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0",
+        "through": "^2.3.6",
+        "wrap-ansi": "^7.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": ">=12.0.0"
       }
     },
-    "node_modules/has-proto": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
-      "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
+    "node_modules/inquirer/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/has-symbols": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
-      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+    "node_modules/inquirer/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/has-tostringtag": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
-      "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
+    "node_modules/inquirer/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "has-symbols": "^1.0.2"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/has-unicode": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
-      "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==",
+    "node_modules/inquirer/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/has-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
-      "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==",
+    "node_modules/inquirer/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "get-value": "^2.0.6",
-        "has-values": "^1.0.0",
-        "isobject": "^3.0.0"
-      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/has-values": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
-      "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==",
+    "node_modules/inquirer/node_modules/rxjs": {
+      "version": "7.8.1",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
+      "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==",
       "dev": true,
       "dependencies": {
-        "is-number": "^3.0.0",
-        "kind-of": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "tslib": "^2.1.0"
       }
     },
-    "node_modules/has-values/node_modules/is-number": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-      "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==",
+    "node_modules/inquirer/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^3.0.2"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/has-values/node_modules/is-number/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+    "node_modules/insert-module-globals": {
+      "version": "7.2.1",
+      "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.2.1.tgz",
+      "integrity": "sha512-ufS5Qq9RZN+Bu899eA9QCAYThY+gGW7oRkmb0vC93Vlyu/CFGcH0OYPEjVkDXA5FEbTt1+VWzdoOD3Ny9N+8tg==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
+        "acorn-node": "^1.5.2",
+        "combine-source-map": "^0.8.0",
+        "concat-stream": "^1.6.1",
+        "is-buffer": "^1.1.0",
+        "JSONStream": "^1.0.3",
+        "path-is-absolute": "^1.0.1",
+        "process": "~0.11.0",
+        "through2": "^2.0.0",
+        "undeclared-identifiers": "^1.1.2",
+        "xtend": "^4.0.0"
       },
-      "engines": {
-        "node": ">=0.10.0"
+      "bin": {
+        "insert-module-globals": "bin/cmd.js"
       }
     },
-    "node_modules/has-values/node_modules/kind-of": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
-      "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==",
+    "node_modules/inside": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/inside/-/inside-1.0.0.tgz",
+      "integrity": "sha512-tvFwvS4g7q6iDot/4FjtWFHwwpv6TVvEumbTdLQilk1F07ojakbXPQcvf3kMAlyNDpzKRzn+d33O3RuXODuxZQ==",
+      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+      "dev": true
+    },
+    "node_modules/internal-slot": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz",
+      "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==",
       "dev": true,
       "dependencies": {
-        "is-buffer": "^1.1.5"
+        "get-intrinsic": "^1.2.2",
+        "hasown": "^2.0.0",
+        "side-channel": "^1.0.4"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 0.4"
       }
     },
-    "node_modules/hash-base": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz",
-      "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==",
-      "dev": true,
+    "node_modules/invariant": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
+      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
       "dependencies": {
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.6.0",
-        "safe-buffer": "^5.2.0"
-      },
+        "loose-envify": "^1.0.0"
+      }
+    },
+    "node_modules/invert-kv": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
+      "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
+      "dev": true,
       "engines": {
         "node": ">=4"
       }
     },
-    "node_modules/hash-base/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+    "node_modules/ip": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
+      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+      "dev": true
+    },
+    "node_modules/ip-regex": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
+      "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/ipaddr.js": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz",
+      "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/is-absolute": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
+      "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
+        "is-relative": "^1.0.0",
+        "is-windows": "^1.0.1"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/hash.js": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz",
-      "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==",
+    "node_modules/is-accessor-descriptor": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.1.tgz",
+      "integrity": "sha512-YBUanLI8Yoihw923YeFUS5fs0fF2f5TSFTNiYAAzhhDscDa3lEqYuz1pDOEP5KvX94I9ey3vsqjJcLVFVU+3QA==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "minimalistic-assert": "^1.0.1"
+        "hasown": "^2.0.0"
+      },
+      "engines": {
+        "node": ">= 0.10"
       }
     },
-    "node_modules/hast-util-parse-selector": {
-      "version": "2.2.5",
-      "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz",
-      "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==",
+    "node_modules/is-alphabetical": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz",
+      "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==",
       "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/unified"
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/hastscript": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz",
-      "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==",
+    "node_modules/is-alphanumerical": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz",
+      "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==",
       "dependencies": {
-        "@types/hast": "^2.0.0",
-        "comma-separated-tokens": "^1.0.0",
-        "hast-util-parse-selector": "^2.0.0",
-        "property-information": "^5.0.0",
-        "space-separated-tokens": "^1.0.0"
+        "is-alphabetical": "^1.0.0",
+        "is-decimal": "^1.0.0"
       },
       "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/unified"
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/hdr-histogram-js": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-2.0.3.tgz",
-      "integrity": "sha512-Hkn78wwzWHNCp2uarhzQ2SGFLU3JY8SBDDd3TAABK4fc30wm+MuPOrg5QVFVfkKOQd6Bfz3ukJEI+q9sXEkK1g==",
+    "node_modules/is-arguments": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
+      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
       "dev": true,
       "dependencies": {
-        "@assemblyscript/loader": "^0.10.1",
-        "base64-js": "^1.2.0",
-        "pako": "^1.0.3"
-      }
-    },
-    "node_modules/hdr-histogram-percentiles-obj": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz",
-      "integrity": "sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw==",
-      "dev": true
-    },
-    "node_modules/he": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
-      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
-      "dev": true,
-      "bin": {
-        "he": "bin/he"
-      }
-    },
-    "node_modules/highlight.js": {
-      "version": "10.7.3",
-      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
-      "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==",
+        "call-bind": "^1.0.2",
+        "has-tostringtag": "^1.0.0"
+      },
       "engines": {
-        "node": "*"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/hmac-drbg": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
-      "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==",
+    "node_modules/is-array-buffer": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz",
+      "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==",
       "dev": true,
       "dependencies": {
-        "hash.js": "^1.0.3",
-        "minimalistic-assert": "^1.0.0",
-        "minimalistic-crypto-utils": "^1.0.1"
+        "call-bind": "^1.0.2",
+        "get-intrinsic": "^1.2.0",
+        "is-typed-array": "^1.1.10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/hoist-non-react-statics": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz",
-      "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==",
+    "node_modules/is-arrayish": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
+      "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
+      "dev": true
+    },
+    "node_modules/is-bigint": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
+      "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==",
+      "dev": true,
       "dependencies": {
-        "react-is": "^16.7.0"
+        "has-bigints": "^1.0.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/hoist-non-react-statics/node_modules/react-is": {
-      "version": "16.13.1",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
-      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
-    },
-    "node_modules/homedir-polyfill": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz",
-      "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==",
+    "node_modules/is-binary-path": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
       "dev": true,
       "dependencies": {
-        "parse-passwd": "^1.0.0"
+        "binary-extensions": "^2.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/hosted-git-info": {
-      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-6.1.1.tgz",
-      "integrity": "sha512-r0EI+HBMcXadMrugk0GCQ+6BQV39PiWAZVfq7oIckeGiN7sjRGyQxPdft3nQekFTCQbYxLBH+/axZMeH8UX6+w==",
+    "node_modules/is-boolean-object": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz",
+      "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==",
       "dev": true,
       "dependencies": {
-        "lru-cache": "^7.5.1"
+        "call-bind": "^1.0.2",
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/hosted-git-info/node_modules/lru-cache": {
-      "version": "7.18.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
-      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "dev": true
+    },
+    "node_modules/is-callable": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
+      "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==",
       "dev": true,
       "engines": {
-        "node": ">=12"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/hpack.js": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",
-      "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==",
+    "node_modules/is-ci": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz",
+      "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.1",
-        "obuf": "^1.0.0",
-        "readable-stream": "^2.0.1",
-        "wbuf": "^1.1.0"
+        "ci-info": "^3.2.0"
+      },
+      "bin": {
+        "is-ci": "bin.js"
       }
     },
-    "node_modules/html-encoding-sniffer": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz",
-      "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==",
+    "node_modules/is-core-module": {
+      "version": "2.13.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz",
+      "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==",
       "dev": true,
       "dependencies": {
-        "whatwg-encoding": "^1.0.1"
+        "hasown": "^2.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/html-entities": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.4.0.tgz",
-      "integrity": "sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==",
-      "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/mdevils"
-        },
-        {
-          "type": "patreon",
-          "url": "https://patreon.com/mdevils"
-        }
-      ]
-    },
-    "node_modules/html-escaper": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
-      "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
-      "dev": true
-    },
-    "node_modules/html-linter": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/html-linter/-/html-linter-1.1.1.tgz",
-      "integrity": "sha512-DJfLevdq+YmY4R7yMdD0zaCAJOvWihg+eCe5o/jeTw86grvgmvSCIwBv1mDi+UdeJP9mDoi5rB6KtcFOn8StrA==",
+    "node_modules/is-data-descriptor": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.1.tgz",
+      "integrity": "sha512-bc4NlCDiCr28U4aEsQ3Qs2491gVq4V8G7MQyws968ImqjKuYtTJXrl7Vq7jsN7Ly/C3xj5KWFrY7sHNeDkAzXw==",
       "dev": true,
       "dependencies": {
-        "chalk": "^2.4.1",
-        "commander": "^2.12.2",
-        "glob": "^7.1.2"
+        "hasown": "^2.0.0"
       },
-      "bin": {
-        "html-linter": "bin/html-linter.js"
+      "engines": {
+        "node": ">= 0.4"
       }
     },
-    "node_modules/html-linter/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+    "node_modules/is-date-object": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz",
+      "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==",
       "dev": true,
       "dependencies": {
-        "color-convert": "^1.9.0"
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/html-linter/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "node_modules/is-decimal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz",
+      "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
+    "node_modules/is-descriptor": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.7.tgz",
+      "integrity": "sha512-C3grZTvObeN1xud4cRWl366OMXZTj0+HGyk4hvfpx4ZHt1Pb60ANSXqCK7pdOTeUQpRzECBSTphqvD7U+l22Eg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "is-accessor-descriptor": "^1.0.1",
+        "is-data-descriptor": "^1.0.1"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
       }
     },
-    "node_modules/html-linter/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+    "node_modules/is-directory": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
+      "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==",
       "dev": true,
-      "dependencies": {
-        "color-name": "1.1.3"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/html-linter/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-      "dev": true
-    },
-    "node_modules/html-linter/node_modules/commander": {
-      "version": "2.20.3",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-      "dev": true
-    },
-    "node_modules/html-linter/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/is-docker": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
+      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==",
       "dev": true,
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+      "bin": {
+        "is-docker": "cli.js"
       },
       "engines": {
-        "node": "*"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/is-dom": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-dom/-/is-dom-1.1.0.tgz",
+      "integrity": "sha512-u82f6mvhYxRPKpw8V1N0W8ce1xXwOrQtgGcxl6UCL5zBmZu3is/18K0rR7uFCnMDuAsS/3W54mGL4vsaFUQlEQ==",
+      "dependencies": {
+        "is-object": "^1.0.1",
+        "is-window": "^1.0.2"
+      }
+    },
+    "node_modules/is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/html-linter/node_modules/has-flag": {
+    "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/is-generator": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-generator/-/is-generator-1.0.3.tgz",
+      "integrity": "sha512-G56jBpbJeg7ds83HW1LuShNs8J73Fv3CPz/bmROHOHlnKkN8sWb9ujiagjmxxMUywftgq48HlBZELKKqFLk0oA==",
+      "dev": true
+    },
+    "node_modules/is-generator-fn": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz",
+      "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=6"
       }
     },
-    "node_modules/html-linter/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+    "node_modules/is-generator-function": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
+      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
       "dev": true,
       "dependencies": {
-        "has-flag": "^3.0.0"
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/html-tags": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.3.1.tgz",
-      "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
+        "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/htmlescape": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/htmlescape/-/htmlescape-1.1.1.tgz",
-      "integrity": "sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint": {
-      "version": "0.7.3",
-      "resolved": "https://registry.npmjs.org/htmllint/-/htmllint-0.7.3.tgz",
-      "integrity": "sha512-h8wfCu0CC0FVo18Jkygv7xqj0fa23Xlv4QsR2n34LDr8eqpf4glfbNg1HTbiCqpT3ONioMOfk8EkFUbZgrO1KA==",
+    "node_modules/is-glob": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
       "dev": true,
       "dependencies": {
-        "bulk-require": "^1.0.1",
-        "htmlparser2": "^3.10.0",
-        "lodash": "^4.17.11",
-        "promise": "^8.0.2"
+        "is-extglob": "^2.1.1"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint-cli": {
-      "version": "0.0.7",
-      "resolved": "https://registry.npmjs.org/htmllint-cli/-/htmllint-cli-0.0.7.tgz",
-      "integrity": "sha512-JREMzimj1HGrOUcA/FMMweLpHD5YRD1E5RnzXa4dj5m1zQ+6YYLklAGdaTar+qAGaj1Jasb3mGhH6FuKuLGWAQ==",
+    "node_modules/is-hexadecimal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz",
+      "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/wooorm"
+      }
+    },
+    "node_modules/is-installed-globally": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz",
+      "integrity": "sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==",
       "dev": true,
       "dependencies": {
-        "bluebird": "^3.5.1",
-        "chalk": "^2.4.0",
-        "cjson": "^0.5.0",
-        "glob": "^7.1.2",
-        "htmllint": "^0.7.2",
-        "liftoff": "^2.5.0",
-        "semver": "^5.5.0",
-        "yargs": "^11.0.0"
-      },
-      "bin": {
-        "htmllint": "bin/cli.js"
+        "global-dirs": "^3.0.0",
+        "is-path-inside": "^3.0.2"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/htmllint-cli/node_modules/ansi-regex": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz",
-      "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==",
+    "node_modules/is-interactive": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
+      "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint-cli/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+    "node_modules/is-lambda": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz",
+      "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==",
+      "dev": true
+    },
+    "node_modules/is-negative-zero": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz",
+      "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==",
       "dev": true,
-      "dependencies": {
-        "color-convert": "^1.9.0"
-      },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/camelcase": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-      "integrity": "sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==",
+    "node_modules/is-number": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=0.12.0"
       }
     },
-    "node_modules/htmllint-cli/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "node_modules/is-number-object": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz",
+      "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/cliui": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-      "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
-      "dev": true,
-      "dependencies": {
-        "string-width": "^2.1.1",
-        "strip-ansi": "^4.0.0",
-        "wrap-ansi": "^2.0.0"
+    "node_modules/is-object": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz",
+      "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+    "node_modules/is-path-inside": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
+      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
       "dev": true,
-      "dependencies": {
-        "color-name": "1.1.3"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint-cli/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-      "dev": true
-    },
-    "node_modules/htmllint-cli/node_modules/find-up": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-      "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==",
+    "node_modules/is-plain-obj": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz",
+      "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==",
       "dev": true,
-      "dependencies": {
-        "locate-path": "^2.0.0"
+      "engines": {
+        "node": ">=10"
       },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/is-plain-object": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
+      "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint-cli/node_modules/get-caller-file": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
-      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
+    "node_modules/is-potential-custom-element-name": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
+      "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
       "dev": true
     },
-    "node_modules/htmllint-cli/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/is-regex": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
+      "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "call-bind": "^1.0.2",
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": "*"
+        "node": ">= 0.4"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+    "node_modules/is-regexp": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-2.1.0.tgz",
+      "integrity": "sha512-OZ4IlER3zmRIoB9AqNhEggVxqIH4ofDns5nRrPS6yQxXE1TPCUpFznBfRQmQa8uC+pXqjMnukiJBxCisIxiLGA==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=6"
       }
     },
-    "node_modules/htmllint-cli/node_modules/is-fullwidth-code-point": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-      "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==",
+    "node_modules/is-relative": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
+      "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==",
+      "dev": true,
+      "dependencies": {
+        "is-unc-path": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/is-shared-array-buffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz",
+      "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==",
+      "dev": true,
+      "dependencies": {
+        "call-bind": "^1.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-stream": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
+      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/htmllint-cli/node_modules/locate-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
-      "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==",
+    "node_modules/is-string": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
+      "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
       "dev": true,
       "dependencies": {
-        "p-locate": "^2.0.0",
-        "path-exists": "^3.0.0"
+        "has-tostringtag": "^1.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/p-limit": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
-      "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
+    "node_modules/is-symbol": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz",
+      "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==",
       "dev": true,
       "dependencies": {
-        "p-try": "^1.0.0"
+        "has-symbols": "^1.0.2"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/p-locate": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
-      "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==",
+    "node_modules/is-typed-array": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz",
+      "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==",
       "dev": true,
       "dependencies": {
-        "p-limit": "^1.1.0"
+        "which-typed-array": "^1.1.11"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/path-exists": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-      "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==",
+    "node_modules/is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==",
+      "dev": true
+    },
+    "node_modules/is-unc-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz",
+      "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==",
       "dev": true,
+      "dependencies": {
+        "unc-path-regex": "^0.1.2"
+      },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint-cli/node_modules/semver": {
-      "version": "5.7.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
-      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
+    "node_modules/is-unicode-supported": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
+      "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver"
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/htmllint-cli/node_modules/string-width": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-      "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+    "node_modules/is-weakref": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz",
+      "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==",
       "dev": true,
       "dependencies": {
-        "is-fullwidth-code-point": "^2.0.0",
-        "strip-ansi": "^4.0.0"
+        "call-bind": "^1.0.2"
       },
-      "engines": {
-        "node": ">=4"
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/htmllint-cli/node_modules/strip-ansi": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-      "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==",
+    "node_modules/is-what": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz",
+      "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==",
+      "dev": true
+    },
+    "node_modules/is-window": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-window/-/is-window-1.0.2.tgz",
+      "integrity": "sha512-uj00kdXyZb9t9RcAUAwMZAnkBUwdYGhYlt7djMXhfyhUCzwNba50tIiBKR7q0l7tdoBtFVw/3JmLY6fI3rmZmg=="
+    },
+    "node_modules/is-windows": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
       "dev": true,
-      "dependencies": {
-        "ansi-regex": "^3.0.0"
-      },
       "engines": {
-        "node": ">=4"
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint-cli/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+    "node_modules/is-wsl": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
+      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
       "dev": true,
       "dependencies": {
-        "has-flag": "^3.0.0"
+        "is-docker": "^2.0.0"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint-cli/node_modules/wrap-ansi": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
-      "integrity": "sha512-vAaEaDM946gbNpH5pLVNR+vX2ht6n0Bt3GXwVB1AuAqZosOvHNF3P7wDnh8KLkSqgUh0uh77le7Owgoz+Z9XBw==",
+    "node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
+      "dev": true
+    },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+      "dev": true
+    },
+    "node_modules/isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
       "dev": true,
-      "dependencies": {
-        "string-width": "^1.0.1",
-        "strip-ansi": "^3.0.1"
-      },
       "engines": {
         "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/ansi-regex": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-      "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==",
+    "node_modules/isomorphic-form-data": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isomorphic-form-data/-/isomorphic-form-data-2.0.0.tgz",
+      "integrity": "sha512-TYgVnXWeESVmQSg4GLVbalmQ+B4NPi/H4eWxqALKj63KsUrcu301YDjBqaOw3h+cbak7Na4Xyps3BiptHtxTfg==",
       "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+      "dependencies": {
+        "form-data": "^2.3.2"
       }
     },
-    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-      "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==",
+    "node_modules/isomorphic-form-data/node_modules/form-data": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz",
+      "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==",
       "dev": true,
       "dependencies": {
-        "number-is-nan": "^1.0.0"
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">= 0.12"
       }
     },
-    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/string-width": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-      "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==",
+    "node_modules/isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==",
+      "dev": true
+    },
+    "node_modules/istanbul-lib-coverage": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz",
+      "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/istanbul-lib-instrument": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz",
+      "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==",
       "dev": true,
       "dependencies": {
-        "code-point-at": "^1.0.0",
-        "is-fullwidth-code-point": "^1.0.0",
-        "strip-ansi": "^3.0.0"
+        "@babel/core": "^7.12.3",
+        "@babel/parser": "^7.14.7",
+        "@istanbuljs/schema": "^0.1.2",
+        "istanbul-lib-coverage": "^3.2.0",
+        "semver": "^6.3.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint-cli/node_modules/wrap-ansi/node_modules/strip-ansi": {
+    "node_modules/istanbul-lib-report": {
       "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-      "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz",
+      "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==",
       "dev": true,
       "dependencies": {
-        "ansi-regex": "^2.0.0"
+        "istanbul-lib-coverage": "^3.0.0",
+        "make-dir": "^4.0.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
       }
     },
-    "node_modules/htmllint-cli/node_modules/y18n": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.2.tgz",
-      "integrity": "sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ==",
-      "dev": true
-    },
-    "node_modules/htmllint-cli/node_modules/yargs": {
-      "version": "11.1.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.1.tgz",
-      "integrity": "sha512-PRU7gJrJaXv3q3yQZ/+/X6KBswZiaQ+zOmdprZcouPYtQgvNU35i+68M4b1ZHLZtYFT5QObFLV+ZkmJYcwKdiw==",
+    "node_modules/istanbul-lib-report/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "cliui": "^4.0.0",
-        "decamelize": "^1.1.1",
-        "find-up": "^2.1.0",
-        "get-caller-file": "^1.0.1",
-        "os-locale": "^3.1.0",
-        "require-directory": "^2.1.1",
-        "require-main-filename": "^1.0.1",
-        "set-blocking": "^2.0.0",
-        "string-width": "^2.0.0",
-        "which-module": "^2.0.0",
-        "y18n": "^3.2.1",
-        "yargs-parser": "^9.0.2"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint-cli/node_modules/yargs-parser": {
-      "version": "9.0.2",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz",
-      "integrity": "sha512-CswCfdOgCr4MMsT1GzbEJ7Z2uYudWyrGX8Bgh/0eyCzj/DXWdKq6a/ADufkzI1WAOIW6jYaXJvRyLhDO0kfqBw==",
+    "node_modules/istanbul-lib-report/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
       "dependencies": {
-        "camelcase": "^4.1.0"
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
-    "node_modules/htmllint/node_modules/dom-serializer": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz",
-      "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==",
+    "node_modules/istanbul-lib-report/node_modules/make-dir": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz",
+      "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==",
       "dev": true,
       "dependencies": {
-        "domelementtype": "^2.0.1",
-        "entities": "^2.0.0"
+        "semver": "^7.5.3"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/htmllint/node_modules/dom-serializer/node_modules/domelementtype": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
-      "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+    "node_modules/istanbul-lib-report/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fb55"
-        }
-      ]
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
     },
-    "node_modules/htmllint/node_modules/dom-serializer/node_modules/entities": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
-      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
+    "node_modules/istanbul-lib-report/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
-      "funding": {
-        "url": "https://github.com/fb55/entities?sponsor=1"
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint/node_modules/domelementtype": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
-      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==",
+    "node_modules/istanbul-lib-report/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "dev": true
     },
-    "node_modules/htmllint/node_modules/domhandler": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
-      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+    "node_modules/istanbul-lib-source-maps": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz",
+      "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==",
       "dev": true,
       "dependencies": {
-        "domelementtype": "1"
+        "debug": "^4.1.1",
+        "istanbul-lib-coverage": "^3.0.0",
+        "source-map": "^0.6.1"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
-    "node_modules/htmllint/node_modules/domutils": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz",
-      "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==",
+    "node_modules/istanbul-lib-source-maps/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true,
-      "dependencies": {
-        "dom-serializer": "0",
-        "domelementtype": "1"
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
-    "node_modules/htmllint/node_modules/entities": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
-      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==",
-      "dev": true
-    },
-    "node_modules/htmllint/node_modules/htmlparser2": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
-      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+    "node_modules/istanbul-reports": {
+      "version": "3.1.6",
+      "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.6.tgz",
+      "integrity": "sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==",
       "dev": true,
       "dependencies": {
-        "domelementtype": "^1.3.1",
-        "domhandler": "^2.3.0",
-        "domutils": "^1.5.1",
-        "entities": "^1.1.1",
-        "inherits": "^2.0.1",
-        "readable-stream": "^3.1.1"
+        "html-escaper": "^2.0.0",
+        "istanbul-lib-report": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/htmllint/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+    "node_modules/jackspeak": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
+      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
+        "@isaacs/cliui": "^8.0.2"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
       }
     },
-    "node_modules/htmlparser2": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
-      "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==",
+    "node_modules/jake": {
+      "version": "10.8.7",
+      "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz",
+      "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==",
       "dev": true,
-      "funding": [
-        "https://github.com/fb55/htmlparser2?sponsor=1",
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/fb55"
-        }
-      ],
       "dependencies": {
-        "domelementtype": "^2.3.0",
-        "domhandler": "^5.0.3",
-        "domutils": "^3.0.1",
-        "entities": "^4.4.0"
+        "async": "^3.2.3",
+        "chalk": "^4.0.2",
+        "filelist": "^1.0.4",
+        "minimatch": "^3.1.2"
+      },
+      "bin": {
+        "jake": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
-    "node_modules/http-auth": {
-      "version": "4.1.9",
-      "resolved": "https://registry.npmjs.org/http-auth/-/http-auth-4.1.9.tgz",
-      "integrity": "sha512-kvPYxNGc9EKGTXvOMnTBQw2RZfuiSihK/mLw/a4pbtRueTE45S55Lw/3k5CktIf7Ak0veMKEIteDj4YkNmCzmQ==",
+    "node_modules/jake/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "apache-crypt": "^1.1.2",
-        "apache-md5": "^1.0.6",
-        "bcryptjs": "^2.4.3",
-        "uuid": "^8.3.2"
+        "color-convert": "^2.0.1"
       },
       "engines": {
         "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/http-auth-connect": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/http-auth-connect/-/http-auth-connect-1.0.6.tgz",
-      "integrity": "sha512-yaO0QSCPqGCjPrl3qEEHjJP+lwZ6gMpXLuCBE06eWwcXomkI5TARtu0kxf9teFuBj6iaV3Ybr15jaWUvbzNzHw==",
+    "node_modules/jake/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
-      "engines": {
-        "node": ">=8"
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/http-cache-semantics": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
-      "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==",
-      "dev": true
-    },
-    "node_modules/http-deceiver": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz",
-      "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==",
-      "dev": true
+    "node_modules/jake/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
     },
-    "node_modules/http-errors": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
-      "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
+    "node_modules/jake/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "depd": "~1.1.2",
-        "inherits": "2.0.3",
-        "setprototypeof": "1.1.1",
-        "statuses": ">= 1.5.0 < 2",
-        "toidentifier": "1.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/http-errors/node_modules/inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==",
+    "node_modules/jake/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/http-parser-js": {
-      "version": "0.5.8",
-      "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz",
-      "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==",
-      "dev": true
+    "node_modules/jake/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
     },
-    "node_modules/http-proxy": {
-      "version": "1.18.1",
-      "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz",
-      "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==",
+    "node_modules/jake/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "eventemitter3": "^4.0.0",
-        "follow-redirects": "^1.0.0",
-        "requires-port": "^1.0.0"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=8.0.0"
+        "node": "*"
       }
     },
-    "node_modules/http-proxy-agent": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
-      "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
+    "node_modules/jake/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "@tootallnate/once": "2",
-        "agent-base": "6",
-        "debug": "4"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=8"
       }
     },
-    "node_modules/http-proxy-middleware": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz",
-      "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==",
+    "node_modules/jest": {
+      "version": "29.6.4",
+      "resolved": "https://registry.npmjs.org/jest/-/jest-29.6.4.tgz",
+      "integrity": "sha512-tEFhVQFF/bzoYV1YuGyzLPZ6vlPrdfvDmmAxudA1dLEuiztqg2Rkx20vkKY32xiDROcD2KXlgZ7Cu8RPeEHRKw==",
       "dev": true,
       "dependencies": {
-        "@types/http-proxy": "^1.17.8",
-        "http-proxy": "^1.18.1",
-        "is-glob": "^4.0.1",
-        "is-plain-obj": "^3.0.0",
-        "micromatch": "^4.0.2"
+        "@jest/core": "^29.6.4",
+        "@jest/types": "^29.6.3",
+        "import-local": "^3.0.2",
+        "jest-cli": "^29.6.4"
+      },
+      "bin": {
+        "jest": "bin/jest.js"
       },
       "engines": {
-        "node": ">=12.0.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       },
       "peerDependencies": {
-        "@types/express": "^4.17.13"
+        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
       },
       "peerDependenciesMeta": {
-        "@types/express": {
+        "node-notifier": {
           "optional": true
         }
       }
     },
-    "node_modules/http-proxy-middleware/node_modules/is-plain-obj": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz",
-      "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==",
+    "node_modules/jest-canvas-mock": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/jest-canvas-mock/-/jest-canvas-mock-2.4.0.tgz",
+      "integrity": "sha512-mmMpZzpmLzn5vepIaHk5HoH3Ka4WykbSoLuG/EKoJd0x0ID/t+INo1l8ByfcUJuDM+RIsL4QDg/gDnBbrj2/IQ==",
+      "dev": true,
+      "dependencies": {
+        "cssfontparser": "^1.2.1",
+        "moo-color": "^1.0.2"
+      }
+    },
+    "node_modules/jest-changed-files": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz",
+      "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==",
+      "dev": true,
+      "dependencies": {
+        "execa": "^5.0.0",
+        "jest-util": "^29.7.0",
+        "p-limit": "^3.1.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/jest-changed-files/node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
       "dev": true,
+      "dependencies": {
+        "yocto-queue": "^0.1.0"
+      },
       "engines": {
         "node": ">=10"
       },
@@ -15799,338 +19219,340 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/http-signature": {
-      "version": "1.3.6",
-      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.3.6.tgz",
-      "integrity": "sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw==",
+    "node_modules/jest-circus": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz",
+      "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==",
       "dev": true,
       "dependencies": {
-        "assert-plus": "^1.0.0",
-        "jsprim": "^2.0.2",
-        "sshpk": "^1.14.1"
+        "@jest/environment": "^29.7.0",
+        "@jest/expect": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "co": "^4.6.0",
+        "dedent": "^1.0.0",
+        "is-generator-fn": "^2.0.0",
+        "jest-each": "^29.7.0",
+        "jest-matcher-utils": "^29.7.0",
+        "jest-message-util": "^29.7.0",
+        "jest-runtime": "^29.7.0",
+        "jest-snapshot": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "p-limit": "^3.1.0",
+        "pretty-format": "^29.7.0",
+        "pure-rand": "^6.0.0",
+        "slash": "^3.0.0",
+        "stack-utils": "^2.0.3"
       },
       "engines": {
-        "node": ">=0.10"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/https-browserify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
-      "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==",
-      "dev": true
-    },
-    "node_modules/https-proxy-agent": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
-      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+    "node_modules/jest-circus/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "agent-base": "6",
-        "debug": "4"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">= 6"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/human-signals": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
-      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
+    "node_modules/jest-circus/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
       "engines": {
-        "node": ">=8.12.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/humanize-ms": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
-      "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
+    "node_modules/jest-circus/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "ms": "^2.0.0"
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/i18next": {
-      "version": "21.10.0",
-      "resolved": "https://registry.npmjs.org/i18next/-/i18next-21.10.0.tgz",
-      "integrity": "sha512-YeuIBmFsGjUfO3qBmMOc0rQaun4mIpGKET5WDwvu8lU7gvwpcariZLNtL0Fzj+zazcHUrlXHiptcFhBMFaxzfg==",
+    "node_modules/jest-circus/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-circus/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-circus/node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
       "dev": true,
-      "funding": [
-        {
-          "type": "individual",
-          "url": "https://locize.com"
-        },
-        {
-          "type": "individual",
-          "url": "https://locize.com/i18next.html"
-        },
-        {
-          "type": "individual",
-          "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project"
-        }
-      ],
       "dependencies": {
-        "@babel/runtime": "^7.17.2"
+        "yocto-queue": "^0.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/iconv-lite": {
-      "version": "0.4.24",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
-      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+    "node_modules/jest-circus/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-circus/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "safer-buffer": ">= 2.1.2 < 3"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
       }
     },
-    "node_modules/icss-utils": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
-      "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==",
+    "node_modules/jest-config": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz",
+      "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==",
       "dev": true,
+      "dependencies": {
+        "@babel/core": "^7.11.6",
+        "@jest/test-sequencer": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "babel-jest": "^29.7.0",
+        "chalk": "^4.0.0",
+        "ci-info": "^3.2.0",
+        "deepmerge": "^4.2.2",
+        "glob": "^7.1.3",
+        "graceful-fs": "^4.2.9",
+        "jest-circus": "^29.7.0",
+        "jest-environment-node": "^29.7.0",
+        "jest-get-type": "^29.6.3",
+        "jest-regex-util": "^29.6.3",
+        "jest-resolve": "^29.7.0",
+        "jest-runner": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "jest-validate": "^29.7.0",
+        "micromatch": "^4.0.4",
+        "parse-json": "^5.2.0",
+        "pretty-format": "^29.7.0",
+        "slash": "^3.0.0",
+        "strip-json-comments": "^3.1.1"
+      },
       "engines": {
-        "node": "^10 || ^12 || >= 14"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       },
       "peerDependencies": {
-        "postcss": "^8.1.0"
-      }
-    },
-    "node_modules/identity-obj-proxy": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz",
-      "integrity": "sha512-00n6YnVHKrinT9t0d9+5yZC6UBNJANpYEQvL2LlX6Ab9lnmxzIRcEmTPuyGScvl1+jKuCICX1Z0Ab1pPKKdikA==",
-      "dev": true,
-      "dependencies": {
-        "harmony-reflect": "^1.4.6"
+        "@types/node": "*",
+        "ts-node": ">=9.0.0"
       },
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/ieee754": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
+      "peerDependenciesMeta": {
+        "@types/node": {
+          "optional": true
         },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
+        "ts-node": {
+          "optional": true
         }
-      ]
-    },
-    "node_modules/ignore": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz",
-      "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 4"
       }
     },
-    "node_modules/ignore-walk": {
-      "version": "6.0.3",
-      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-6.0.3.tgz",
-      "integrity": "sha512-C7FfFoTA+bI10qfeydT8aZbvr91vAEU+2W5BZUlzPec47oNb07SsOfwYrtxuvOYdUApPP/Qlh4DtAO51Ekk2QA==",
+    "node_modules/jest-config/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "minimatch": "^9.0.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/ignore-walk/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+    "node_modules/jest-config/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "balanced-match": "^1.0.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/ignore-walk/node_modules/minimatch": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
-      "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+    "node_modules/jest-config/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "brace-expansion": "^2.0.1"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=16 || 14 >=14.17"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/image-size": {
-      "version": "0.5.5",
-      "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz",
-      "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==",
+    "node_modules/jest-config/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
-      "optional": true,
-      "bin": {
-        "image-size": "bin/image-size.js"
+      "dependencies": {
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/immutable": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.4.tgz",
-      "integrity": "sha512-fsXeu4J4i6WNWSikpI88v/PcVflZz+6kMhUfIwc5SY+poQRPnaf5V7qds6SUyUN3cVxEzuCab7QIoLOQ+DQ1wA==",
+    "node_modules/jest-config/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/import-fresh": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
-      "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+    "node_modules/jest-config/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "parent-module": "^1.0.0",
-        "resolve-from": "^4.0.0"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">=6"
+        "node": "*"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/import-fresh/node_modules/resolve-from": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
-      "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/import-lazy": {
+    "node_modules/jest-config/node_modules/has-flag": {
       "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-4.0.0.tgz",
-      "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
         "node": ">=8"
       }
     },
-    "node_modules/import-local": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz",
-      "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==",
+    "node_modules/jest-config/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "pkg-dir": "^4.2.0",
-        "resolve-cwd": "^3.0.0"
-      },
-      "bin": {
-        "import-local-fixture": "fixtures/cli.js"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": "*"
       }
     },
-    "node_modules/imurmurhash": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
+    "node_modules/jest-config/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
       "engines": {
-        "node": ">=0.8.19"
+        "node": ">=8"
       }
     },
-    "node_modules/indent-string": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
-      "integrity": "sha512-BYqTHXTGUIvg7t1r4sJNKcbDZkL92nkXA8YtRpbjFHRHGDL/NtUeiBJMeE60kIFN/Mg8ESaWQvftaYMGJzQZCQ==",
+    "node_modules/jest-config/node_modules/strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
       "dev": true,
       "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
-      "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/inherits": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
-    },
-    "node_modules/ini": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-3.0.1.tgz",
-      "integrity": "sha512-it4HyVAUTKBc6m8e1iXWvXSTdndF7HbdN713+kvLrymxTaU4AUBWrJ4vEooP+V7fexnVD3LKcBshjGGPefSMUQ==",
+    "node_modules/jest-config/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/inline-source-map": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz",
-      "integrity": "sha512-0mVWSSbNDvedDWIN4wxLsdPM4a7cIPcpyMxj3QZ406QRwQ6ePGB1YIHxVPjqpcUGbWQ5C+nHTwGNWAGvt7ggVA==",
+    "node_modules/jest-diff": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz",
+      "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==",
       "dev": true,
       "dependencies": {
-        "source-map": "~0.5.3"
-      }
-    },
-    "node_modules/inline-source-map/node_modules/source-map": {
-      "version": "0.5.7",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
-      "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==",
-      "dev": true,
+        "chalk": "^4.0.0",
+        "diff-sequences": "^29.6.3",
+        "jest-get-type": "^29.6.3",
+        "pretty-format": "^29.7.0"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/inquirer": {
-      "version": "8.2.4",
-      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.4.tgz",
-      "integrity": "sha512-nn4F01dxU8VeKfq192IjLsxu0/OmMZ4Lg3xKAns148rCaXP6ntAoEkVYZThWjwON8AlzdZZi6oqnhNbxUG9hVg==",
+    "node_modules/jest-diff/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "ansi-escapes": "^4.2.1",
-        "chalk": "^4.1.1",
-        "cli-cursor": "^3.1.0",
-        "cli-width": "^3.0.0",
-        "external-editor": "^3.0.3",
-        "figures": "^3.0.0",
-        "lodash": "^4.17.21",
-        "mute-stream": "0.0.8",
-        "ora": "^5.4.1",
-        "run-async": "^2.4.0",
-        "rxjs": "^7.5.5",
-        "string-width": "^4.1.0",
-        "strip-ansi": "^6.0.0",
-        "through": "^2.3.6",
-        "wrap-ansi": "^7.0.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=12.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/inquirer/node_modules/chalk": {
+    "node_modules/jest-diff/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -16146,1031 +19568,1124 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/inquirer/node_modules/rxjs": {
-      "version": "7.8.1",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
-      "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==",
-      "dev": true,
-      "dependencies": {
-        "tslib": "^2.1.0"
-      }
-    },
-    "node_modules/insert-module-globals": {
-      "version": "7.2.1",
-      "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.2.1.tgz",
-      "integrity": "sha512-ufS5Qq9RZN+Bu899eA9QCAYThY+gGW7oRkmb0vC93Vlyu/CFGcH0OYPEjVkDXA5FEbTt1+VWzdoOD3Ny9N+8tg==",
+    "node_modules/jest-diff/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "acorn-node": "^1.5.2",
-        "combine-source-map": "^0.8.0",
-        "concat-stream": "^1.6.1",
-        "is-buffer": "^1.1.0",
-        "JSONStream": "^1.0.3",
-        "path-is-absolute": "^1.0.1",
-        "process": "~0.11.0",
-        "through2": "^2.0.0",
-        "undeclared-identifiers": "^1.1.2",
-        "xtend": "^4.0.0"
+        "color-name": "~1.1.4"
       },
-      "bin": {
-        "insert-module-globals": "bin/cmd.js"
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/inside": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/inside/-/inside-1.0.0.tgz",
-      "integrity": "sha512-tvFwvS4g7q6iDot/4FjtWFHwwpv6TVvEumbTdLQilk1F07ojakbXPQcvf3kMAlyNDpzKRzn+d33O3RuXODuxZQ==",
-      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+    "node_modules/jest-diff/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/internal-slot": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz",
-      "integrity": "sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==",
+    "node_modules/jest-diff/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "get-intrinsic": "^1.2.0",
-        "has": "^1.0.3",
-        "side-channel": "^1.0.4"
-      },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=8"
       }
     },
-    "node_modules/invariant": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
-      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
+    "node_modules/jest-diff/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
       "dependencies": {
-        "loose-envify": "^1.0.0"
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/invert-kv": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
-      "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
+    "node_modules/jest-docblock": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz",
+      "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==",
       "dev": true,
+      "dependencies": {
+        "detect-newline": "^3.0.0"
+      },
       "engines": {
-        "node": ">=4"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/ip": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
-      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
-      "dev": true
-    },
-    "node_modules/ip-regex": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
-      "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==",
+    "node_modules/jest-each": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz",
+      "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==",
       "dev": true,
+      "dependencies": {
+        "@jest/types": "^29.6.3",
+        "chalk": "^4.0.0",
+        "jest-get-type": "^29.6.3",
+        "jest-util": "^29.7.0",
+        "pretty-format": "^29.7.0"
+      },
       "engines": {
-        "node": ">=4"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/ipaddr.js": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
-      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+    "node_modules/jest-each/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
-        "node": ">= 0.10"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/is-absolute": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
-      "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==",
+    "node_modules/jest-each/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "is-relative": "^1.0.0",
-        "is-windows": "^1.0.1"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/is-accessor-descriptor": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-      "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+    "node_modules/jest-each/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^6.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/is-alphabetical": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz",
-      "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+    "node_modules/jest-each/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-each/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/is-alphanumerical": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz",
-      "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==",
+    "node_modules/jest-each/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
       "dependencies": {
-        "is-alphabetical": "^1.0.0",
-        "is-decimal": "^1.0.0"
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/is-arguments": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
-      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
+    "node_modules/jest-environment-jsdom": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-29.7.0.tgz",
+      "integrity": "sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
+        "@jest/environment": "^29.7.0",
+        "@jest/fake-timers": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/jsdom": "^20.0.0",
+        "@types/node": "*",
+        "jest-mock": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "jsdom": "^20.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "peerDependencies": {
+        "canvas": "^2.5.0"
+      },
+      "peerDependenciesMeta": {
+        "canvas": {
+          "optional": true
+        }
       }
     },
-    "node_modules/is-array-buffer": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz",
-      "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==",
+    "node_modules/jest-environment-node": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz",
+      "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "get-intrinsic": "^1.2.0",
-        "is-typed-array": "^1.1.10"
+        "@jest/environment": "^29.7.0",
+        "@jest/fake-timers": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "jest-mock": "^29.7.0",
+        "jest-util": "^29.7.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/is-arrayish": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
-      "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
-      "dev": true
+    "node_modules/jest-get-type": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz",
+      "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==",
+      "dev": true,
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
     },
-    "node_modules/is-bigint": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz",
-      "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==",
+    "node_modules/jest-haste-map": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz",
+      "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==",
       "dev": true,
       "dependencies": {
-        "has-bigints": "^1.0.1"
+        "@jest/types": "^29.6.3",
+        "@types/graceful-fs": "^4.1.3",
+        "@types/node": "*",
+        "anymatch": "^3.0.3",
+        "fb-watchman": "^2.0.0",
+        "graceful-fs": "^4.2.9",
+        "jest-regex-util": "^29.6.3",
+        "jest-util": "^29.7.0",
+        "jest-worker": "^29.7.0",
+        "micromatch": "^4.0.4",
+        "walker": "^1.0.8"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "^2.3.2"
       }
     },
-    "node_modules/is-binary-path": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
-      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+    "node_modules/jest-haste-map/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "binary-extensions": "^2.0.0"
-      },
       "engines": {
         "node": ">=8"
       }
     },
-    "node_modules/is-boolean-object": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz",
-      "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==",
+    "node_modules/jest-haste-map/node_modules/jest-worker": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz",
+      "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
+        "@types/node": "*",
+        "jest-util": "^29.7.0",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/is-buffer": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
-      "dev": true
-    },
-    "node_modules/is-callable": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
-      "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==",
+    "node_modules/jest-haste-map/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
       "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
       }
     },
-    "node_modules/is-ci": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz",
-      "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==",
+    "node_modules/jest-jasmine2": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-28.1.3.tgz",
+      "integrity": "sha512-nlNWJY1u62w+WAVgnXOQTdxFdZhqlxpKvMTn1cOK1QHX2oRrkPV3JcIcJfXwcGcifttOJZhExcgDUqSHrYQ6Dw==",
       "dev": true,
       "dependencies": {
-        "ci-info": "^3.2.0"
+        "@jest/environment": "^28.1.3",
+        "@jest/expect": "^28.1.3",
+        "@jest/source-map": "^28.1.2",
+        "@jest/test-result": "^28.1.3",
+        "@jest/types": "^28.1.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "co": "^4.6.0",
+        "is-generator-fn": "^2.0.0",
+        "jest-each": "^28.1.3",
+        "jest-matcher-utils": "^28.1.3",
+        "jest-message-util": "^28.1.3",
+        "jest-runtime": "^28.1.3",
+        "jest-snapshot": "^28.1.3",
+        "jest-util": "^28.1.3",
+        "p-limit": "^3.1.0",
+        "pretty-format": "^28.1.3"
       },
-      "bin": {
-        "is-ci": "bin.js"
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-core-module": {
-      "version": "2.13.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz",
-      "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/console": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/console/-/console-28.1.3.tgz",
+      "integrity": "sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw==",
       "dev": true,
       "dependencies": {
-        "has": "^1.0.3"
+        "@jest/types": "^28.1.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "jest-message-util": "^28.1.3",
+        "jest-util": "^28.1.3",
+        "slash": "^3.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-data-descriptor": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-      "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/environment": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-28.1.3.tgz",
+      "integrity": "sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA==",
       "dev": true,
       "dependencies": {
-        "kind-of": "^6.0.0"
+        "@jest/fake-timers": "^28.1.3",
+        "@jest/types": "^28.1.3",
+        "@types/node": "*",
+        "jest-mock": "^28.1.3"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-date-object": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz",
-      "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/expect": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-28.1.3.tgz",
+      "integrity": "sha512-lzc8CpUbSoE4dqT0U+g1qODQjBRHPpCPXissXD4mS9+sWQdmmpeJ9zSH1rS1HEkrsMN0fb7nKrJ9giAR1d3wBw==",
       "dev": true,
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "expect": "^28.1.3",
+        "jest-snapshot": "^28.1.3"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/is-decimal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz",
-      "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-descriptor": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-      "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+    "node_modules/jest-jasmine2/node_modules/@jest/expect-utils": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-28.1.3.tgz",
+      "integrity": "sha512-wvbi9LUrHJLn3NlDW6wF2hvIMtd4JUl2QNVrjq+IBSHirgfrR3o9RnVtxzdEGO2n9JyIWwHnLfby5KzqBGg2YA==",
       "dev": true,
       "dependencies": {
-        "is-accessor-descriptor": "^1.0.0",
-        "is-data-descriptor": "^1.0.0",
-        "kind-of": "^6.0.2"
+        "jest-get-type": "^28.0.2"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-directory": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
-      "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==",
+    "node_modules/jest-jasmine2/node_modules/@jest/fake-timers": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-28.1.3.tgz",
+      "integrity": "sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw==",
       "dev": true,
+      "dependencies": {
+        "@jest/types": "^28.1.3",
+        "@sinonjs/fake-timers": "^9.1.2",
+        "@types/node": "*",
+        "jest-message-util": "^28.1.3",
+        "jest-mock": "^28.1.3",
+        "jest-util": "^28.1.3"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-docker": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
-      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/globals": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-28.1.3.tgz",
+      "integrity": "sha512-XFU4P4phyryCXu1pbcqMO0GSQcYe1IsalYCDzRNyhetyeyxMcIxa11qPNDpVNLeretItNqEmYYQn1UYz/5x1NA==",
       "dev": true,
-      "bin": {
-        "is-docker": "cli.js"
+      "dependencies": {
+        "@jest/environment": "^28.1.3",
+        "@jest/expect": "^28.1.3",
+        "@jest/types": "^28.1.3"
       },
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/is-dom": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-dom/-/is-dom-1.1.0.tgz",
-      "integrity": "sha512-u82f6mvhYxRPKpw8V1N0W8ce1xXwOrQtgGcxl6UCL5zBmZu3is/18K0rR7uFCnMDuAsS/3W54mGL4vsaFUQlEQ==",
-      "dependencies": {
-        "is-object": "^1.0.1",
-        "is-window": "^1.0.2"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-extendable": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-      "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+    "node_modules/jest-jasmine2/node_modules/@jest/schemas": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-28.1.3.tgz",
+      "integrity": "sha512-/l/VWsdt/aBXgjshLWOFyFt3IVdYypu5y2Wn2rOO1un6nkqIn8SLXzgIMYXFyYsRWDyF5EthmKJMIdJvk08grg==",
       "dev": true,
       "dependencies": {
-        "is-plain-object": "^2.0.4"
+        "@sinclair/typebox": "^0.24.1"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-extglob": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/source-map": {
+      "version": "28.1.2",
+      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-28.1.2.tgz",
+      "integrity": "sha512-cV8Lx3BeStJb8ipPHnqVw/IM2VCMWO3crWZzYodSIkxXnRcXJipCdx1JCK0K5MsJJouZQTH73mzf4vgxRaH9ww==",
       "dev": true,
+      "dependencies": {
+        "@jridgewell/trace-mapping": "^0.3.13",
+        "callsites": "^3.0.0",
+        "graceful-fs": "^4.2.9"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-fullwidth-code-point": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+    "node_modules/jest-jasmine2/node_modules/@jest/test-result": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-28.1.3.tgz",
+      "integrity": "sha512-kZAkxnSE+FqE8YjW8gNuoVkkC9I7S1qmenl8sGcDOLropASP+BkcGKwhXoyqQuGOGeYY0y/ixjrd/iERpEXHNg==",
+      "dev": true,
+      "dependencies": {
+        "@jest/console": "^28.1.3",
+        "@jest/types": "^28.1.3",
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "collect-v8-coverage": "^1.0.0"
+      },
       "engines": {
-        "node": ">=8"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-generator": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/is-generator/-/is-generator-1.0.3.tgz",
-      "integrity": "sha512-G56jBpbJeg7ds83HW1LuShNs8J73Fv3CPz/bmROHOHlnKkN8sWb9ujiagjmxxMUywftgq48HlBZELKKqFLk0oA==",
-      "dev": true
-    },
-    "node_modules/is-generator-fn": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz",
-      "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==",
+    "node_modules/jest-jasmine2/node_modules/@jest/transform": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-28.1.3.tgz",
+      "integrity": "sha512-u5dT5di+oFI6hfcLOHGTAfmUxFRrjK+vnaP0kkVow9Md/M7V/MxqQMOz/VV25UZO8pzeA9PjfTpOu6BDuwSPQA==",
       "dev": true,
+      "dependencies": {
+        "@babel/core": "^7.11.6",
+        "@jest/types": "^28.1.3",
+        "@jridgewell/trace-mapping": "^0.3.13",
+        "babel-plugin-istanbul": "^6.1.1",
+        "chalk": "^4.0.0",
+        "convert-source-map": "^1.4.0",
+        "fast-json-stable-stringify": "^2.0.0",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^28.1.3",
+        "jest-regex-util": "^28.0.2",
+        "jest-util": "^28.1.3",
+        "micromatch": "^4.0.4",
+        "pirates": "^4.0.4",
+        "slash": "^3.0.0",
+        "write-file-atomic": "^4.0.1"
+      },
       "engines": {
-        "node": ">=6"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-generator-function": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
-      "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==",
+    "node_modules/jest-jasmine2/node_modules/@jest/types": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/@jest/types/-/types-28.1.3.tgz",
+      "integrity": "sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ==",
       "dev": true,
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "@jest/schemas": "^28.1.3",
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "@types/istanbul-reports": "^3.0.0",
+        "@types/node": "*",
+        "@types/yargs": "^17.0.8",
+        "chalk": "^4.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-glob": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
-      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+    "node_modules/jest-jasmine2/node_modules/@sinclair/typebox": {
+      "version": "0.24.51",
+      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz",
+      "integrity": "sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA==",
+      "dev": true
+    },
+    "node_modules/jest-jasmine2/node_modules/@sinonjs/commons": {
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz",
+      "integrity": "sha512-Ky+XkAkqPZSm3NLBeUng77EBQl3cmeJhITaGHdYH8kjVB+aun3S4XBRti2zt17mtt0mIUDiNxYeoJm6drVvBJQ==",
       "dev": true,
       "dependencies": {
-        "is-extglob": "^2.1.1"
-      },
-      "engines": {
-        "node": ">=0.10.0"
+        "type-detect": "4.0.8"
       }
     },
-    "node_modules/is-hexadecimal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz",
-      "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/wooorm"
+    "node_modules/jest-jasmine2/node_modules/@sinonjs/fake-timers": {
+      "version": "9.1.2",
+      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-9.1.2.tgz",
+      "integrity": "sha512-BPS4ynJW/o92PUR4wgriz2Ud5gpST5vz6GQfMixEDK0Z8ZCUv2M7SkBLykH56T++Xs+8ln9zTGbOvNGIe02/jw==",
+      "dev": true,
+      "dependencies": {
+        "@sinonjs/commons": "^1.7.0"
       }
     },
-    "node_modules/is-installed-globally": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz",
-      "integrity": "sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==",
+    "node_modules/jest-jasmine2/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "global-dirs": "^3.0.0",
-        "is-path-inside": "^3.0.2"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/is-interactive": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
-      "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==",
+    "node_modules/jest-jasmine2/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
-      "engines": {
-        "node": ">=8"
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/is-lambda": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz",
-      "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==",
-      "dev": true
-    },
-    "node_modules/is-negative-zero": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz",
-      "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==",
+    "node_modules/jest-jasmine2/node_modules/camelcase": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
       "dev": true,
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/is-number": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.12.0"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/is-number-object": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz",
-      "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==",
+    "node_modules/jest-jasmine2/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/is-object": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz",
-      "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==",
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/is-path-inside": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
-      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+    "node_modules/jest-jasmine2/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
       "engines": {
-        "node": ">=8"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/is-plain-obj": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
-      "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==",
+    "node_modules/jest-jasmine2/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-jasmine2/node_modules/diff-sequences": {
+      "version": "28.1.1",
+      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-28.1.1.tgz",
+      "integrity": "sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-plain-object": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
-      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+    "node_modules/jest-jasmine2/node_modules/expect": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/expect/-/expect-28.1.3.tgz",
+      "integrity": "sha512-eEh0xn8HlsuOBxFgIss+2mX85VAS4Qy3OSkjV7rlBWljtA4oWH37glVGyOZSZvErDT/yBywZdPGwCXuTvSG85g==",
       "dev": true,
       "dependencies": {
-        "isobject": "^3.0.1"
+        "@jest/expect-utils": "^28.1.3",
+        "jest-get-type": "^28.0.2",
+        "jest-matcher-utils": "^28.1.3",
+        "jest-message-util": "^28.1.3",
+        "jest-util": "^28.1.3"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-potential-custom-element-name": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
-      "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
-      "dev": true
-    },
-    "node_modules/is-regex": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
-      "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==",
+    "node_modules/jest-jasmine2/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": "*"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/is-regexp": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-2.1.0.tgz",
-      "integrity": "sha512-OZ4IlER3zmRIoB9AqNhEggVxqIH4ofDns5nRrPS6yQxXE1TPCUpFznBfRQmQa8uC+pXqjMnukiJBxCisIxiLGA==",
+    "node_modules/jest-jasmine2/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=6"
+        "node": ">=8"
       }
     },
-    "node_modules/is-relative": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
-      "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==",
+    "node_modules/jest-jasmine2/node_modules/jest-diff": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-28.1.3.tgz",
+      "integrity": "sha512-8RqP1B/OXzjjTWkqMX67iqgwBVJRgCyKD3L9nq+6ZqJMdvjE8RgHktqZ6jNrkdMT+dJuYNI3rhQpxaz7drJHfw==",
       "dev": true,
       "dependencies": {
-        "is-unc-path": "^1.0.0"
+        "chalk": "^4.0.0",
+        "diff-sequences": "^28.1.1",
+        "jest-get-type": "^28.0.2",
+        "pretty-format": "^28.1.3"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-shared-array-buffer": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz",
-      "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==",
+    "node_modules/jest-jasmine2/node_modules/jest-each": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-28.1.3.tgz",
+      "integrity": "sha512-arT1z4sg2yABU5uogObVPvSlSMQlDA48owx07BDPAiasW0yYpYHYOo4HHLz9q0BVzDVU4hILFjzJw0So9aCL/g==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2"
+        "@jest/types": "^28.1.3",
+        "chalk": "^4.0.0",
+        "jest-get-type": "^28.0.2",
+        "jest-util": "^28.1.3",
+        "pretty-format": "^28.1.3"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-stream": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==",
+    "node_modules/jest-jasmine2/node_modules/jest-get-type": {
+      "version": "28.0.2",
+      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-28.0.2.tgz",
+      "integrity": "sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA==",
       "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-string": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz",
-      "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==",
+    "node_modules/jest-jasmine2/node_modules/jest-haste-map": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-28.1.3.tgz",
+      "integrity": "sha512-3S+RQWDXccXDKSWnkHa/dPwt+2qwA8CJzR61w3FoYCvoo3Pn8tvGcysmMF0Bj0EX5RYvAI2EIvC57OmotfdtKA==",
       "dev": true,
       "dependencies": {
-        "has-tostringtag": "^1.0.0"
+        "@jest/types": "^28.1.3",
+        "@types/graceful-fs": "^4.1.3",
+        "@types/node": "*",
+        "anymatch": "^3.0.3",
+        "fb-watchman": "^2.0.0",
+        "graceful-fs": "^4.2.9",
+        "jest-regex-util": "^28.0.2",
+        "jest-util": "^28.1.3",
+        "jest-worker": "^28.1.3",
+        "micromatch": "^4.0.4",
+        "walker": "^1.0.8"
       },
       "engines": {
-        "node": ">= 0.4"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+      "optionalDependencies": {
+        "fsevents": "^2.3.2"
       }
     },
-    "node_modules/is-symbol": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz",
-      "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==",
+    "node_modules/jest-jasmine2/node_modules/jest-matcher-utils": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-28.1.3.tgz",
+      "integrity": "sha512-kQeJ7qHemKfbzKoGjHHrRKH6atgxMk8Enkk2iPQ3XwO6oE/KYD8lMYOziCkeSB9G4adPM4nR1DE8Tf5JeWH6Bw==",
       "dev": true,
       "dependencies": {
-        "has-symbols": "^1.0.2"
+        "chalk": "^4.0.0",
+        "jest-diff": "^28.1.3",
+        "jest-get-type": "^28.0.2",
+        "pretty-format": "^28.1.3"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-typed-array": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz",
-      "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==",
+    "node_modules/jest-jasmine2/node_modules/jest-message-util": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-28.1.3.tgz",
+      "integrity": "sha512-PFdn9Iewbt575zKPf1286Ht9EPoJmYT7P0kY+RibeYZ2XtOr53pDLEFoTWXbd1h4JiGiWpTBC84fc8xMXQMb7g==",
       "dev": true,
       "dependencies": {
-        "which-typed-array": "^1.1.11"
+        "@babel/code-frame": "^7.12.13",
+        "@jest/types": "^28.1.3",
+        "@types/stack-utils": "^2.0.0",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.9",
+        "micromatch": "^4.0.4",
+        "pretty-format": "^28.1.3",
+        "slash": "^3.0.0",
+        "stack-utils": "^2.0.3"
       },
       "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-typedarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
-      "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==",
-      "dev": true
-    },
-    "node_modules/is-unc-path": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz",
-      "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==",
+    "node_modules/jest-jasmine2/node_modules/jest-mock": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-28.1.3.tgz",
+      "integrity": "sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA==",
       "dev": true,
       "dependencies": {
-        "unc-path-regex": "^0.1.2"
+        "@jest/types": "^28.1.3",
+        "@types/node": "*"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-unicode-supported": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
-      "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==",
+    "node_modules/jest-jasmine2/node_modules/jest-regex-util": {
+      "version": "28.0.2",
+      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-28.0.2.tgz",
+      "integrity": "sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw==",
       "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-weakref": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz",
-      "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==",
+    "node_modules/jest-jasmine2/node_modules/jest-resolve": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-28.1.3.tgz",
+      "integrity": "sha512-Z1W3tTjE6QaNI90qo/BJpfnvpxtaFTFw5CDgwpyE/Kz8U/06N1Hjf4ia9quUhCh39qIGWF1ZuxFiBiJQwSEYKQ==",
       "dev": true,
       "dependencies": {
-        "call-bind": "^1.0.2"
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^28.1.3",
+        "jest-pnp-resolver": "^1.2.2",
+        "jest-util": "^28.1.3",
+        "jest-validate": "^28.1.3",
+        "resolve": "^1.20.0",
+        "resolve.exports": "^1.1.0",
+        "slash": "^3.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/is-what": {
-      "version": "3.14.1",
-      "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz",
-      "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==",
-      "dev": true
-    },
-    "node_modules/is-window": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-window/-/is-window-1.0.2.tgz",
-      "integrity": "sha512-uj00kdXyZb9t9RcAUAwMZAnkBUwdYGhYlt7djMXhfyhUCzwNba50tIiBKR7q0l7tdoBtFVw/3JmLY6fI3rmZmg=="
-    },
-    "node_modules/is-windows": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
-      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
-      "dev": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/is-wsl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
-      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
+    "node_modules/jest-jasmine2/node_modules/jest-runtime": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-28.1.3.tgz",
+      "integrity": "sha512-NU+881ScBQQLc1JHG5eJGU7Ui3kLKrmwCPPtYsJtBykixrM2OhVQlpMmFWJjMyDfdkGgBMNjXCGB/ebzsgNGQw==",
       "dev": true,
       "dependencies": {
-        "is-docker": "^2.0.0"
+        "@jest/environment": "^28.1.3",
+        "@jest/fake-timers": "^28.1.3",
+        "@jest/globals": "^28.1.3",
+        "@jest/source-map": "^28.1.2",
+        "@jest/test-result": "^28.1.3",
+        "@jest/transform": "^28.1.3",
+        "@jest/types": "^28.1.3",
+        "chalk": "^4.0.0",
+        "cjs-module-lexer": "^1.0.0",
+        "collect-v8-coverage": "^1.0.0",
+        "execa": "^5.0.0",
+        "glob": "^7.1.3",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^28.1.3",
+        "jest-message-util": "^28.1.3",
+        "jest-mock": "^28.1.3",
+        "jest-regex-util": "^28.0.2",
+        "jest-resolve": "^28.1.3",
+        "jest-snapshot": "^28.1.3",
+        "jest-util": "^28.1.3",
+        "slash": "^3.0.0",
+        "strip-bom": "^4.0.0"
       },
       "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
-      "dev": true
-    },
-    "node_modules/isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "dev": true
-    },
-    "node_modules/isobject": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/isomorphic-form-data": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isomorphic-form-data/-/isomorphic-form-data-2.0.0.tgz",
-      "integrity": "sha512-TYgVnXWeESVmQSg4GLVbalmQ+B4NPi/H4eWxqALKj63KsUrcu301YDjBqaOw3h+cbak7Na4Xyps3BiptHtxTfg==",
+    "node_modules/jest-jasmine2/node_modules/jest-snapshot": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-28.1.3.tgz",
+      "integrity": "sha512-4lzMgtiNlc3DU/8lZfmqxN3AYD6GGLbl+72rdBpXvcV+whX7mDrREzkPdp2RnmfIiWBg1YbuFSkXduF2JcafJg==",
       "dev": true,
       "dependencies": {
-        "form-data": "^2.3.2"
-      }
-    },
-    "node_modules/isstream": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
-      "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==",
-      "dev": true
-    },
-    "node_modules/istanbul-lib-coverage": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz",
-      "integrity": "sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==",
-      "dev": true,
+        "@babel/core": "^7.11.6",
+        "@babel/generator": "^7.7.2",
+        "@babel/plugin-syntax-typescript": "^7.7.2",
+        "@babel/traverse": "^7.7.2",
+        "@babel/types": "^7.3.3",
+        "@jest/expect-utils": "^28.1.3",
+        "@jest/transform": "^28.1.3",
+        "@jest/types": "^28.1.3",
+        "@types/babel__traverse": "^7.0.6",
+        "@types/prettier": "^2.1.5",
+        "babel-preset-current-node-syntax": "^1.0.0",
+        "chalk": "^4.0.0",
+        "expect": "^28.1.3",
+        "graceful-fs": "^4.2.9",
+        "jest-diff": "^28.1.3",
+        "jest-get-type": "^28.0.2",
+        "jest-haste-map": "^28.1.3",
+        "jest-matcher-utils": "^28.1.3",
+        "jest-message-util": "^28.1.3",
+        "jest-util": "^28.1.3",
+        "natural-compare": "^1.4.0",
+        "pretty-format": "^28.1.3",
+        "semver": "^7.3.5"
+      },
       "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/istanbul-lib-instrument": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz",
-      "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==",
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+      }
+    },
+    "node_modules/jest-jasmine2/node_modules/jest-util": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-28.1.3.tgz",
+      "integrity": "sha512-XdqfpHwpcSRko/C35uLYFM2emRAltIIKZiJ9eAmhjsj0CqZMa0p1ib0R5fWIqGhn1a103DebTbpqIaP1qCQ6tQ==",
       "dev": true,
       "dependencies": {
-        "@babel/core": "^7.12.3",
-        "@babel/parser": "^7.14.7",
-        "@istanbuljs/schema": "^0.1.2",
-        "istanbul-lib-coverage": "^3.2.0",
-        "semver": "^6.3.0"
+        "@jest/types": "^28.1.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "ci-info": "^3.2.0",
+        "graceful-fs": "^4.2.9",
+        "picomatch": "^2.2.3"
       },
       "engines": {
-        "node": ">=8"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/istanbul-lib-instrument/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+    "node_modules/jest-jasmine2/node_modules/jest-validate": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-28.1.3.tgz",
+      "integrity": "sha512-SZbOGBWEsaTxBGCOpsRWlXlvNkvTkY0XxRfh7zYmvd8uL5Qzyg0CHAXiXKROflh801quA6+/DsT4ODDthOC/OA==",
       "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+      "dependencies": {
+        "@jest/types": "^28.1.3",
+        "camelcase": "^6.2.0",
+        "chalk": "^4.0.0",
+        "jest-get-type": "^28.0.2",
+        "leven": "^3.1.0",
+        "pretty-format": "^28.1.3"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/istanbul-lib-report": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz",
-      "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==",
+    "node_modules/jest-jasmine2/node_modules/jest-worker": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-28.1.3.tgz",
+      "integrity": "sha512-CqRA220YV/6jCo8VWvAt1KKx6eek1VIHMPeLEbpcfSfkEeWyBNppynM/o6q+Wmw+sOhos2ml34wZbSX3G13//g==",
       "dev": true,
       "dependencies": {
-        "istanbul-lib-coverage": "^3.0.0",
-        "make-dir": "^4.0.0",
-        "supports-color": "^7.1.0"
+        "@types/node": "*",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/istanbul-lib-report/node_modules/make-dir": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz",
-      "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==",
+    "node_modules/jest-jasmine2/node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
       "dev": true,
       "dependencies": {
-        "semver": "^7.5.3"
+        "has-flag": "^4.0.0"
       },
       "engines": {
         "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
       }
     },
-    "node_modules/istanbul-lib-source-maps": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz",
-      "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==",
+    "node_modules/jest-jasmine2/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
       "dependencies": {
-        "debug": "^4.1.1",
-        "istanbul-lib-coverage": "^3.0.0",
-        "source-map": "^0.6.1"
+        "yallist": "^4.0.0"
       },
       "engines": {
         "node": ">=10"
       }
     },
-    "node_modules/istanbul-lib-source-maps/node_modules/source-map": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/istanbul-reports": {
-      "version": "3.1.6",
-      "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.6.tgz",
-      "integrity": "sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==",
+    "node_modules/jest-jasmine2/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "html-escaper": "^2.0.0",
-        "istanbul-lib-report": "^3.0.0"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": ">=8"
+        "node": "*"
       }
     },
-    "node_modules/jackspeak": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.3.tgz",
-      "integrity": "sha512-R2bUw+kVZFS/h1AZqBKrSgDmdmjApzgY0AlCPumopFiAlbUxE2gf+SCuBzQ0cP5hHmUmFYF5yw55T97Th5Kstg==",
+    "node_modules/jest-jasmine2/node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
       "dev": true,
       "dependencies": {
-        "@isaacs/cliui": "^8.0.2"
+        "yocto-queue": "^0.1.0"
       },
       "engines": {
-        "node": ">=14"
+        "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      },
-      "optionalDependencies": {
-        "@pkgjs/parseargs": "^0.11.0"
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/jake": {
-      "version": "10.8.7",
-      "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz",
-      "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==",
+    "node_modules/jest-jasmine2/node_modules/pretty-format": {
+      "version": "28.1.3",
+      "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-28.1.3.tgz",
+      "integrity": "sha512-8gFb/To0OmxHR9+ZTb14Df2vNxdGCX8g1xWGUTqUw5TiZvcQf5sHKObd5UcPyLLyowNwDAMTF3XWOG1B6mxl1Q==",
       "dev": true,
       "dependencies": {
-        "async": "^3.2.3",
-        "chalk": "^4.0.2",
-        "filelist": "^1.0.4",
-        "minimatch": "^3.1.2"
-      },
-      "bin": {
-        "jake": "bin/cli.js"
+        "@jest/schemas": "^28.1.3",
+        "ansi-regex": "^5.0.1",
+        "ansi-styles": "^5.0.0",
+        "react-is": "^18.0.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
       }
     },
-    "node_modules/jake/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-jasmine2/node_modules/pretty-format/node_modules/ansi-styles": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
+      "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
       "engines": {
         "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest": {
-      "version": "29.6.4",
-      "resolved": "https://registry.npmjs.org/jest/-/jest-29.6.4.tgz",
-      "integrity": "sha512-tEFhVQFF/bzoYV1YuGyzLPZ6vlPrdfvDmmAxudA1dLEuiztqg2Rkx20vkKY32xiDROcD2KXlgZ7Cu8RPeEHRKw==",
+    "node_modules/jest-jasmine2/node_modules/react-is": {
+      "version": "18.2.0",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
+      "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
+      "dev": true
+    },
+    "node_modules/jest-jasmine2/node_modules/resolve.exports": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-1.1.1.tgz",
+      "integrity": "sha512-/NtpHNDN7jWhAaQ9BvBUYZ6YTXsRBgfqWFWP7BZBaoMJO/I3G5OFzvTuWNlZC3aPjins1F+TNrLKsGbH4rfsRQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/jest-jasmine2/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
-        "@jest/core": "^29.6.4",
-        "@jest/types": "^29.6.3",
-        "import-local": "^3.0.2",
-        "jest-cli": "^29.6.4"
+        "lru-cache": "^6.0.0"
       },
       "bin": {
-        "jest": "bin/jest.js"
+        "semver": "bin/semver.js"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
-      },
-      "peerDependenciesMeta": {
-        "node-notifier": {
-          "optional": true
-        }
+        "node": ">=10"
       }
     },
-    "node_modules/jest-canvas-mock": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/jest-canvas-mock/-/jest-canvas-mock-2.4.0.tgz",
-      "integrity": "sha512-mmMpZzpmLzn5vepIaHk5HoH3Ka4WykbSoLuG/EKoJd0x0ID/t+INo1l8ByfcUJuDM+RIsL4QDg/gDnBbrj2/IQ==",
+    "node_modules/jest-jasmine2/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-jasmine2/node_modules/strip-bom": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz",
+      "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-jasmine2/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "cssfontparser": "^1.2.1",
-        "moo-color": "^1.0.2"
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/jest-changed-files": {
+    "node_modules/jest-jasmine2/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/jest-leak-detector": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz",
-      "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==",
+      "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz",
+      "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==",
       "dev": true,
       "dependencies": {
-        "execa": "^5.0.0",
-        "jest-util": "^29.7.0",
-        "p-limit": "^3.1.0"
+        "jest-get-type": "^29.6.3",
+        "pretty-format": "^29.7.0"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-changed-files/node_modules/execa": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
+    "node_modules/jest-matcher-utils": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz",
+      "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==",
       "dev": true,
       "dependencies": {
-        "cross-spawn": "^7.0.3",
-        "get-stream": "^6.0.0",
-        "human-signals": "^2.1.0",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.1",
-        "onetime": "^5.1.2",
-        "signal-exit": "^3.0.3",
-        "strip-final-newline": "^2.0.0"
+        "chalk": "^4.0.0",
+        "jest-diff": "^29.7.0",
+        "jest-get-type": "^29.6.3",
+        "pretty-format": "^29.7.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/jest-matcher-utils/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-changed-files/node_modules/get-stream": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+    "node_modules/jest-matcher-utils/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
       "engines": {
         "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-changed-files/node_modules/human-signals": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
+    "node_modules/jest-matcher-utils/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
       "engines": {
-        "node": ">=10.17.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-changed-files/node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+    "node_modules/jest-matcher-utils/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-matcher-utils/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
         "node": ">=8"
+      }
+    },
+    "node_modules/jest-matcher-utils/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/jest-circus": {
+    "node_modules/jest-message-util": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz",
-      "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==",
+      "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz",
+      "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==",
       "dev": true,
       "dependencies": {
-        "@jest/environment": "^29.7.0",
-        "@jest/expect": "^29.7.0",
-        "@jest/test-result": "^29.7.0",
+        "@babel/code-frame": "^7.12.13",
         "@jest/types": "^29.6.3",
-        "@types/node": "*",
+        "@types/stack-utils": "^2.0.0",
         "chalk": "^4.0.0",
-        "co": "^4.6.0",
-        "dedent": "^1.0.0",
-        "is-generator-fn": "^2.0.0",
-        "jest-each": "^29.7.0",
-        "jest-matcher-utils": "^29.7.0",
-        "jest-message-util": "^29.7.0",
-        "jest-runtime": "^29.7.0",
-        "jest-snapshot": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "p-limit": "^3.1.0",
+        "graceful-fs": "^4.2.9",
+        "micromatch": "^4.0.4",
         "pretty-format": "^29.7.0",
-        "pure-rand": "^6.0.0",
         "slash": "^3.0.0",
         "stack-utils": "^2.0.3"
       },
@@ -17178,7 +20693,22 @@
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-circus/node_modules/chalk": {
+    "node_modules/jest-message-util/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/jest-message-util/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -17194,512 +20724,561 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-cli": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz",
-      "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==",
+    "node_modules/jest-message-util/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@jest/core": "^29.7.0",
-        "@jest/test-result": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "chalk": "^4.0.0",
-        "create-jest": "^29.7.0",
-        "exit": "^0.1.2",
-        "import-local": "^3.0.2",
-        "jest-config": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jest-validate": "^29.7.0",
-        "yargs": "^17.3.1"
-      },
-      "bin": {
-        "jest": "bin/jest.js"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
-      },
-      "peerDependenciesMeta": {
-        "node-notifier": {
-          "optional": true
-        }
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-cli/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-message-util/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-message-util/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-message-util/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-message-util/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-config": {
+    "node_modules/jest-mock": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz",
-      "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==",
+      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz",
+      "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==",
       "dev": true,
       "dependencies": {
-        "@babel/core": "^7.11.6",
-        "@jest/test-sequencer": "^29.7.0",
         "@jest/types": "^29.6.3",
-        "babel-jest": "^29.7.0",
-        "chalk": "^4.0.0",
-        "ci-info": "^3.2.0",
-        "deepmerge": "^4.2.2",
-        "glob": "^7.1.3",
-        "graceful-fs": "^4.2.9",
-        "jest-circus": "^29.7.0",
-        "jest-environment-node": "^29.7.0",
-        "jest-get-type": "^29.6.3",
-        "jest-regex-util": "^29.6.3",
-        "jest-resolve": "^29.7.0",
-        "jest-runner": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jest-validate": "^29.7.0",
-        "micromatch": "^4.0.4",
-        "parse-json": "^5.2.0",
-        "pretty-format": "^29.7.0",
-        "slash": "^3.0.0",
-        "strip-json-comments": "^3.1.1"
+        "@types/node": "*",
+        "jest-util": "^29.7.0"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      }
+    },
+    "node_modules/jest-pnp-resolver": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz",
+      "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
       },
       "peerDependencies": {
-        "@types/node": "*",
-        "ts-node": ">=9.0.0"
+        "jest-resolve": "*"
       },
       "peerDependenciesMeta": {
-        "@types/node": {
-          "optional": true
-        },
-        "ts-node": {
+        "jest-resolve": {
           "optional": true
         }
       }
     },
-    "node_modules/jest-config/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-preset-angular": {
+      "version": "13.1.1",
+      "resolved": "https://registry.npmjs.org/jest-preset-angular/-/jest-preset-angular-13.1.1.tgz",
+      "integrity": "sha512-X8i7icKt9U5uhj7YKqdEZm7ZZPvNFRxfBnU+9SALdIkHYJhwtlJ5/MUk9wo4f3lX2smOkIl9LPJUu1APO+11Jg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "bs-logger": "^0.2.6",
+        "esbuild-wasm": ">=0.13.8",
+        "jest-environment-jsdom": "^29.0.0",
+        "jest-util": "^29.0.0",
+        "pretty-format": "^29.0.0",
+        "ts-jest": "^29.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || >=16.10.0"
+      },
+      "optionalDependencies": {
+        "esbuild": ">=0.13.8"
       },
+      "peerDependencies": {
+        "@angular-devkit/build-angular": ">=13.0.0 <17.0.0",
+        "@angular/compiler-cli": ">=13.0.0 <17.0.0",
+        "@angular/core": ">=13.0.0 <17.0.0",
+        "@angular/platform-browser-dynamic": ">=13.0.0 <17.0.0",
+        "jest": "^29.0.0",
+        "typescript": ">=4.4"
+      }
+    },
+    "node_modules/jest-preset-angular/node_modules/@esbuild/android-arm": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.19.6.tgz",
+      "integrity": "sha512-muPzBqXJKCbMYoNbb1JpZh/ynl0xS6/+pLjrofcR3Nad82SbsCogYzUE6Aq9QT3cLP0jR/IVK/NHC9b90mSHtg==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/jest-preset-angular/node_modules/@esbuild/android-arm64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.19.6.tgz",
+      "integrity": "sha512-KQ/hbe9SJvIJ4sR+2PcZ41IBV+LPJyYp6V1K1P1xcMRup9iYsBoQn4MzE3mhMLOld27Au2eDcLlIREeKGUXpHQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-config/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/android-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.19.6.tgz",
+      "integrity": "sha512-VVJVZQ7p5BBOKoNxd0Ly3xUM78Y4DyOoFKdkdAe2m11jbh0LEU4bPles4e/72EMl4tapko8o915UalN/5zhspg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
+      "optional": true,
+      "os": [
+        "android"
+      ],
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-diff": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz",
-      "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/darwin-arm64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.19.6.tgz",
+      "integrity": "sha512-91LoRp/uZAKx6ESNspL3I46ypwzdqyDLXZH7x2QYCLgtnaU08+AXEbabY2yExIz03/am0DivsTtbdxzGejfXpA==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "chalk": "^4.0.0",
-        "diff-sequences": "^29.6.3",
-        "jest-get-type": "^29.6.3",
-        "pretty-format": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-diff/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/darwin-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.19.6.tgz",
+      "integrity": "sha512-QCGHw770ubjBU1J3ZkFJh671MFajGTYMZumPs9E/rqU52md6lIil97BR0CbPq6U+vTh3xnTNDHKRdR8ggHnmxQ==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-docblock": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz",
-      "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/freebsd-arm64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.6.tgz",
+      "integrity": "sha512-J53d0jGsDcLzWk9d9SPmlyF+wzVxjXpOH7jVW5ae7PvrDst4kiAz6sX+E8btz0GB6oH12zC+aHRD945jdjF2Vg==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "detect-newline": "^3.0.0"
-      },
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-each": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz",
-      "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/freebsd-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.19.6.tgz",
+      "integrity": "sha512-hn9qvkjHSIB5Z9JgCCjED6YYVGCNpqB7dEGavBdG6EjBD8S/UcNUIlGcB35NCkMETkdYwfZSvD9VoDJX6VeUVA==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "chalk": "^4.0.0",
-        "jest-get-type": "^29.6.3",
-        "jest-util": "^29.7.0",
-        "pretty-format": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-each/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-arm": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.19.6.tgz",
+      "integrity": "sha512-G8IR5zFgpXad/Zp7gr7ZyTKyqZuThU6z1JjmRyN1vSF8j0bOlGzUwFSMTbctLAdd7QHpeyu0cRiuKrqK1ZTwvQ==",
+      "cpu": [
+        "arm"
+      ],
       "dev": true,
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-environment-jsdom": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-29.7.0.tgz",
-      "integrity": "sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-arm64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.19.6.tgz",
+      "integrity": "sha512-HQCOrk9XlH3KngASLaBfHpcoYEGUt829A9MyxaI8RMkfRA8SakG6YQEITAuwmtzFdEu5GU4eyhKcpv27dFaOBg==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/environment": "^29.7.0",
-        "@jest/fake-timers": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/jsdom": "^20.0.0",
-        "@types/node": "*",
-        "jest-mock": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jsdom": "^20.0.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "peerDependencies": {
-        "canvas": "^2.5.0"
-      },
-      "peerDependenciesMeta": {
-        "canvas": {
-          "optional": true
-        }
+        "node": ">=12"
       }
     },
-    "node_modules/jest-environment-node": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz",
-      "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-ia32": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.19.6.tgz",
+      "integrity": "sha512-22eOR08zL/OXkmEhxOfshfOGo8P69k8oKHkwkDrUlcB12S/sw/+COM4PhAPT0cAYW/gpqY2uXp3TpjQVJitz7w==",
+      "cpu": [
+        "ia32"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/environment": "^29.7.0",
-        "@jest/fake-timers": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "jest-mock": "^29.7.0",
-        "jest-util": "^29.7.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-get-type": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz",
-      "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-loong64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.19.6.tgz",
+      "integrity": "sha512-82RvaYAh/SUJyjWA8jDpyZCHQjmEggL//sC7F3VKYcBMumQjUL3C5WDl/tJpEiKtt7XrWmgjaLkrk205zfvwTA==",
+      "cpu": [
+        "loong64"
+      ],
       "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-haste-map": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz",
-      "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-mips64el": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.19.6.tgz",
+      "integrity": "sha512-8tvnwyYJpR618vboIv2l8tK2SuK/RqUIGMfMENkeDGo3hsEIrpGldMGYFcWxWeEILe5Fi72zoXLmhZ7PR23oQA==",
+      "cpu": [
+        "mips64el"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "@types/graceful-fs": "^4.1.3",
-        "@types/node": "*",
-        "anymatch": "^3.0.3",
-        "fb-watchman": "^2.0.0",
-        "graceful-fs": "^4.2.9",
-        "jest-regex-util": "^29.6.3",
-        "jest-util": "^29.7.0",
-        "jest-worker": "^29.7.0",
-        "micromatch": "^4.0.4",
-        "walker": "^1.0.8"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "^2.3.2"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-28.1.3.tgz",
-      "integrity": "sha512-nlNWJY1u62w+WAVgnXOQTdxFdZhqlxpKvMTn1cOK1QHX2oRrkPV3JcIcJfXwcGcifttOJZhExcgDUqSHrYQ6Dw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-ppc64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.19.6.tgz",
+      "integrity": "sha512-Qt+D7xiPajxVNk5tQiEJwhmarNnLPdjXAoA5uWMpbfStZB0+YU6a3CtbWYSy+sgAsnyx4IGZjWsTzBzrvg/fMA==",
+      "cpu": [
+        "ppc64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/environment": "^28.1.3",
-        "@jest/expect": "^28.1.3",
-        "@jest/source-map": "^28.1.2",
-        "@jest/test-result": "^28.1.3",
-        "@jest/types": "^28.1.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "co": "^4.6.0",
-        "is-generator-fn": "^2.0.0",
-        "jest-each": "^28.1.3",
-        "jest-matcher-utils": "^28.1.3",
-        "jest-message-util": "^28.1.3",
-        "jest-runtime": "^28.1.3",
-        "jest-snapshot": "^28.1.3",
-        "jest-util": "^28.1.3",
-        "p-limit": "^3.1.0",
-        "pretty-format": "^28.1.3"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-riscv64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.19.6.tgz",
+      "integrity": "sha512-lxRdk0iJ9CWYDH1Wpnnnc640ajF4RmQ+w6oHFZmAIYu577meE9Ka/DCtpOrwr9McMY11ocbp4jirgGgCi7Ls/g==",
+      "cpu": [
+        "riscv64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/console": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/console/-/console-28.1.3.tgz",
-      "integrity": "sha512-QPAkP5EwKdK/bxIr6C1I4Vs0rm2nHiANzj/Z5X2JQkrZo6IqvC4ldZ9K95tF0HdidhA8Bo6egxSzUFPYKcEXLw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-s390x": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.19.6.tgz",
+      "integrity": "sha512-MopyYV39vnfuykHanRWHGRcRC3AwU7b0QY4TI8ISLfAGfK+tMkXyFuyT1epw/lM0pflQlS53JoD22yN83DHZgA==",
+      "cpu": [
+        "s390x"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^28.1.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "jest-message-util": "^28.1.3",
-        "jest-util": "^28.1.3",
-        "slash": "^3.0.0"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/environment": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-28.1.3.tgz",
-      "integrity": "sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/linux-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.6.tgz",
+      "integrity": "sha512-UWcieaBzsN8WYbzFF5Jq7QULETPcQvlX7KL4xWGIB54OknXJjBO37sPqk7N82WU13JGWvmDzFBi1weVBajPovg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/fake-timers": "^28.1.3",
-        "@jest/types": "^28.1.3",
-        "@types/node": "*",
-        "jest-mock": "^28.1.3"
-      },
+      "optional": true,
+      "os": [
+        "linux"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/expect": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-28.1.3.tgz",
-      "integrity": "sha512-lzc8CpUbSoE4dqT0U+g1qODQjBRHPpCPXissXD4mS9+sWQdmmpeJ9zSH1rS1HEkrsMN0fb7nKrJ9giAR1d3wBw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/netbsd-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.19.6.tgz",
+      "integrity": "sha512-EpWiLX0fzvZn1wxtLxZrEW+oQED9Pwpnh+w4Ffv8ZLuMhUoqR9q9rL4+qHW8F4Mg5oQEKxAoT0G+8JYNqCiR6g==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "expect": "^28.1.3",
-        "jest-snapshot": "^28.1.3"
-      },
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/expect-utils": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-28.1.3.tgz",
-      "integrity": "sha512-wvbi9LUrHJLn3NlDW6wF2hvIMtd4JUl2QNVrjq+IBSHirgfrR3o9RnVtxzdEGO2n9JyIWwHnLfby5KzqBGg2YA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/openbsd-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.19.6.tgz",
+      "integrity": "sha512-fFqTVEktM1PGs2sLKH4M5mhAVEzGpeZJuasAMRnvDZNCV0Cjvm1Hu35moL2vC0DOrAQjNTvj4zWrol/lwQ8Deg==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "jest-get-type": "^28.0.2"
-      },
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/fake-timers": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-28.1.3.tgz",
-      "integrity": "sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/sunos-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.19.6.tgz",
+      "integrity": "sha512-M+XIAnBpaNvaVAhbe3uBXtgWyWynSdlww/JNZws0FlMPSBy+EpatPXNIlKAdtbFVII9OpX91ZfMb17TU3JKTBA==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^28.1.3",
-        "@sinonjs/fake-timers": "^9.1.2",
-        "@types/node": "*",
-        "jest-message-util": "^28.1.3",
-        "jest-mock": "^28.1.3",
-        "jest-util": "^28.1.3"
-      },
+      "optional": true,
+      "os": [
+        "sunos"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/globals": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-28.1.3.tgz",
-      "integrity": "sha512-XFU4P4phyryCXu1pbcqMO0GSQcYe1IsalYCDzRNyhetyeyxMcIxa11qPNDpVNLeretItNqEmYYQn1UYz/5x1NA==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/win32-arm64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.19.6.tgz",
+      "integrity": "sha512-2DchFXn7vp/B6Tc2eKdTsLzE0ygqKkNUhUBCNtMx2Llk4POIVMUq5rUYjdcedFlGLeRe1uLCpVvCmE+G8XYybA==",
+      "cpu": [
+        "arm64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/environment": "^28.1.3",
-        "@jest/expect": "^28.1.3",
-        "@jest/types": "^28.1.3"
-      },
+      "optional": true,
+      "os": [
+        "win32"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/schemas": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-28.1.3.tgz",
-      "integrity": "sha512-/l/VWsdt/aBXgjshLWOFyFt3IVdYypu5y2Wn2rOO1un6nkqIn8SLXzgIMYXFyYsRWDyF5EthmKJMIdJvk08grg==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/win32-ia32": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.19.6.tgz",
+      "integrity": "sha512-PBo/HPDQllyWdjwAVX+Gl2hH0dfBydL97BAH/grHKC8fubqp02aL4S63otZ25q3sBdINtOBbz1qTZQfXbP4VBg==",
+      "cpu": [
+        "ia32"
+      ],
       "dev": true,
-      "dependencies": {
-        "@sinclair/typebox": "^0.24.1"
-      },
+      "optional": true,
+      "os": [
+        "win32"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/test-result": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-28.1.3.tgz",
-      "integrity": "sha512-kZAkxnSE+FqE8YjW8gNuoVkkC9I7S1qmenl8sGcDOLropASP+BkcGKwhXoyqQuGOGeYY0y/ixjrd/iERpEXHNg==",
+    "node_modules/jest-preset-angular/node_modules/@esbuild/win32-x64": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.19.6.tgz",
+      "integrity": "sha512-OE7yIdbDif2kKfrGa+V0vx/B3FJv2L4KnIiLlvtibPyO9UkgO3rzYE0HhpREo2vmJ1Ixq1zwm9/0er+3VOSZJA==",
+      "cpu": [
+        "x64"
+      ],
       "dev": true,
-      "dependencies": {
-        "@jest/console": "^28.1.3",
-        "@jest/types": "^28.1.3",
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "collect-v8-coverage": "^1.0.0"
-      },
+      "optional": true,
+      "os": [
+        "win32"
+      ],
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/transform": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-28.1.3.tgz",
-      "integrity": "sha512-u5dT5di+oFI6hfcLOHGTAfmUxFRrjK+vnaP0kkVow9Md/M7V/MxqQMOz/VV25UZO8pzeA9PjfTpOu6BDuwSPQA==",
+    "node_modules/jest-preset-angular/node_modules/esbuild": {
+      "version": "0.19.6",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.19.6.tgz",
+      "integrity": "sha512-Xl7dntjA2OEIvpr9j0DVxxnog2fyTGnyVoQXAMQI6eR3mf9zCQds7VIKUDCotDgE/p4ncTgeRqgX8t5d6oP4Gw==",
       "dev": true,
-      "dependencies": {
-        "@babel/core": "^7.11.6",
-        "@jest/types": "^28.1.3",
-        "@jridgewell/trace-mapping": "^0.3.13",
-        "babel-plugin-istanbul": "^6.1.1",
-        "chalk": "^4.0.0",
-        "convert-source-map": "^1.4.0",
-        "fast-json-stable-stringify": "^2.0.0",
-        "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^28.1.3",
-        "jest-regex-util": "^28.0.2",
-        "jest-util": "^28.1.3",
-        "micromatch": "^4.0.4",
-        "pirates": "^4.0.4",
-        "slash": "^3.0.0",
-        "write-file-atomic": "^4.0.1"
+      "hasInstallScript": true,
+      "optional": true,
+      "bin": {
+        "esbuild": "bin/esbuild"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=12"
+      },
+      "optionalDependencies": {
+        "@esbuild/android-arm": "0.19.6",
+        "@esbuild/android-arm64": "0.19.6",
+        "@esbuild/android-x64": "0.19.6",
+        "@esbuild/darwin-arm64": "0.19.6",
+        "@esbuild/darwin-x64": "0.19.6",
+        "@esbuild/freebsd-arm64": "0.19.6",
+        "@esbuild/freebsd-x64": "0.19.6",
+        "@esbuild/linux-arm": "0.19.6",
+        "@esbuild/linux-arm64": "0.19.6",
+        "@esbuild/linux-ia32": "0.19.6",
+        "@esbuild/linux-loong64": "0.19.6",
+        "@esbuild/linux-mips64el": "0.19.6",
+        "@esbuild/linux-ppc64": "0.19.6",
+        "@esbuild/linux-riscv64": "0.19.6",
+        "@esbuild/linux-s390x": "0.19.6",
+        "@esbuild/linux-x64": "0.19.6",
+        "@esbuild/netbsd-x64": "0.19.6",
+        "@esbuild/openbsd-x64": "0.19.6",
+        "@esbuild/sunos-x64": "0.19.6",
+        "@esbuild/win32-arm64": "0.19.6",
+        "@esbuild/win32-ia32": "0.19.6",
+        "@esbuild/win32-x64": "0.19.6"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@jest/types": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/@jest/types/-/types-28.1.3.tgz",
-      "integrity": "sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ==",
+    "node_modules/jest-regex-util": {
+      "version": "29.6.3",
+      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz",
+      "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==",
       "dev": true,
-      "dependencies": {
-        "@jest/schemas": "^28.1.3",
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "@types/istanbul-reports": "^3.0.0",
-        "@types/node": "*",
-        "@types/yargs": "^17.0.8",
-        "chalk": "^4.0.0"
-      },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@sinclair/typebox": {
-      "version": "0.24.51",
-      "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz",
-      "integrity": "sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA==",
-      "dev": true
-    },
-    "node_modules/jest-jasmine2/node_modules/@sinonjs/commons": {
-      "version": "1.8.6",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz",
-      "integrity": "sha512-Ky+XkAkqPZSm3NLBeUng77EBQl3cmeJhITaGHdYH8kjVB+aun3S4XBRti2zt17mtt0mIUDiNxYeoJm6drVvBJQ==",
+    "node_modules/jest-resolve": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz",
+      "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==",
       "dev": true,
       "dependencies": {
-        "type-detect": "4.0.8"
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.9",
+        "jest-haste-map": "^29.7.0",
+        "jest-pnp-resolver": "^1.2.2",
+        "jest-util": "^29.7.0",
+        "jest-validate": "^29.7.0",
+        "resolve": "^1.20.0",
+        "resolve.exports": "^2.0.0",
+        "slash": "^3.0.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/@sinonjs/fake-timers": {
-      "version": "9.1.2",
-      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-9.1.2.tgz",
-      "integrity": "sha512-BPS4ynJW/o92PUR4wgriz2Ud5gpST5vz6GQfMixEDK0Z8ZCUv2M7SkBLykH56T++Xs+8ln9zTGbOvNGIe02/jw==",
+    "node_modules/jest-resolve-dependencies": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz",
+      "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==",
       "dev": true,
       "dependencies": {
-        "@sinonjs/commons": "^1.7.0"
+        "jest-regex-util": "^29.6.3",
+        "jest-snapshot": "^29.7.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/camelcase": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
-      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+    "node_modules/jest-resolve/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/chalk": {
+    "node_modules/jest-resolve/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -17715,480 +21294,434 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/diff-sequences": {
-      "version": "28.1.1",
-      "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-28.1.1.tgz",
-      "integrity": "sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw==",
-      "dev": true,
-      "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
-      }
-    },
-    "node_modules/jest-jasmine2/node_modules/execa": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
+    "node_modules/jest-resolve/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "cross-spawn": "^7.0.3",
-        "get-stream": "^6.0.0",
-        "human-signals": "^2.1.0",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.1",
-        "onetime": "^5.1.2",
-        "signal-exit": "^3.0.3",
-        "strip-final-newline": "^2.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/expect": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/expect/-/expect-28.1.3.tgz",
-      "integrity": "sha512-eEh0xn8HlsuOBxFgIss+2mX85VAS4Qy3OSkjV7rlBWljtA4oWH37glVGyOZSZvErDT/yBywZdPGwCXuTvSG85g==",
+    "node_modules/jest-resolve/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-resolve/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
-      "dependencies": {
-        "@jest/expect-utils": "^28.1.3",
-        "jest-get-type": "^28.0.2",
-        "jest-matcher-utils": "^28.1.3",
-        "jest-message-util": "^28.1.3",
-        "jest-util": "^28.1.3"
-      },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/get-stream": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+    "node_modules/jest-resolve/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/jest-resolve/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/human-signals": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
+    "node_modules/jest-runner": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz",
+      "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==",
       "dev": true,
+      "dependencies": {
+        "@jest/console": "^29.7.0",
+        "@jest/environment": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
+        "@jest/transform": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "emittery": "^0.13.1",
+        "graceful-fs": "^4.2.9",
+        "jest-docblock": "^29.7.0",
+        "jest-environment-node": "^29.7.0",
+        "jest-haste-map": "^29.7.0",
+        "jest-leak-detector": "^29.7.0",
+        "jest-message-util": "^29.7.0",
+        "jest-resolve": "^29.7.0",
+        "jest-runtime": "^29.7.0",
+        "jest-util": "^29.7.0",
+        "jest-watcher": "^29.7.0",
+        "jest-worker": "^29.7.0",
+        "p-limit": "^3.1.0",
+        "source-map-support": "0.5.13"
+      },
       "engines": {
-        "node": ">=10.17.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+    "node_modules/jest-runner/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
         "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-diff": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-28.1.3.tgz",
-      "integrity": "sha512-8RqP1B/OXzjjTWkqMX67iqgwBVJRgCyKD3L9nq+6ZqJMdvjE8RgHktqZ6jNrkdMT+dJuYNI3rhQpxaz7drJHfw==",
+    "node_modules/jest-runner/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "chalk": "^4.0.0",
-        "diff-sequences": "^28.1.1",
-        "jest-get-type": "^28.0.2",
-        "pretty-format": "^28.1.3"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-each": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-28.1.3.tgz",
-      "integrity": "sha512-arT1z4sg2yABU5uogObVPvSlSMQlDA48owx07BDPAiasW0yYpYHYOo4HHLz9q0BVzDVU4hILFjzJw0So9aCL/g==",
+    "node_modules/jest-runner/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^28.1.3",
-        "chalk": "^4.0.0",
-        "jest-get-type": "^28.0.2",
-        "jest-util": "^28.1.3",
-        "pretty-format": "^28.1.3"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-get-type": {
-      "version": "28.0.2",
-      "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-28.0.2.tgz",
-      "integrity": "sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA==",
+    "node_modules/jest-runner/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-runner/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-haste-map": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-28.1.3.tgz",
-      "integrity": "sha512-3S+RQWDXccXDKSWnkHa/dPwt+2qwA8CJzR61w3FoYCvoo3Pn8tvGcysmMF0Bj0EX5RYvAI2EIvC57OmotfdtKA==",
+    "node_modules/jest-runner/node_modules/jest-worker": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz",
+      "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^28.1.3",
-        "@types/graceful-fs": "^4.1.3",
         "@types/node": "*",
-        "anymatch": "^3.0.3",
-        "fb-watchman": "^2.0.0",
-        "graceful-fs": "^4.2.9",
-        "jest-regex-util": "^28.0.2",
-        "jest-util": "^28.1.3",
-        "jest-worker": "^28.1.3",
-        "micromatch": "^4.0.4",
-        "walker": "^1.0.8"
+        "jest-util": "^29.7.0",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "^2.3.2"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-matcher-utils": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-28.1.3.tgz",
-      "integrity": "sha512-kQeJ7qHemKfbzKoGjHHrRKH6atgxMk8Enkk2iPQ3XwO6oE/KYD8lMYOziCkeSB9G4adPM4nR1DE8Tf5JeWH6Bw==",
+    "node_modules/jest-runner/node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
       "dev": true,
       "dependencies": {
-        "chalk": "^4.0.0",
-        "jest-diff": "^28.1.3",
-        "jest-get-type": "^28.0.2",
-        "pretty-format": "^28.1.3"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-message-util": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-28.1.3.tgz",
-      "integrity": "sha512-PFdn9Iewbt575zKPf1286Ht9EPoJmYT7P0kY+RibeYZ2XtOr53pDLEFoTWXbd1h4JiGiWpTBC84fc8xMXQMb7g==",
+    "node_modules/jest-runner/node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
       "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.12.13",
-        "@jest/types": "^28.1.3",
-        "@types/stack-utils": "^2.0.0",
-        "chalk": "^4.0.0",
-        "graceful-fs": "^4.2.9",
-        "micromatch": "^4.0.4",
-        "pretty-format": "^28.1.3",
-        "slash": "^3.0.0",
-        "stack-utils": "^2.0.3"
+        "yocto-queue": "^0.1.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-mock": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-28.1.3.tgz",
-      "integrity": "sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA==",
+    "node_modules/jest-runner/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
       "dev": true,
-      "dependencies": {
-        "@jest/types": "^28.1.3",
-        "@types/node": "*"
-      },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
-      }
-    },
-    "node_modules/jest-jasmine2/node_modules/jest-regex-util": {
-      "version": "28.0.2",
-      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-28.0.2.tgz",
-      "integrity": "sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw==",
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/jest-runner/node_modules/source-map-support": {
+      "version": "0.5.13",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz",
+      "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==",
       "dev": true,
-      "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-resolve": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-28.1.3.tgz",
-      "integrity": "sha512-Z1W3tTjE6QaNI90qo/BJpfnvpxtaFTFw5CDgwpyE/Kz8U/06N1Hjf4ia9quUhCh39qIGWF1ZuxFiBiJQwSEYKQ==",
+    "node_modules/jest-runner/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "chalk": "^4.0.0",
-        "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^28.1.3",
-        "jest-pnp-resolver": "^1.2.2",
-        "jest-util": "^28.1.3",
-        "jest-validate": "^28.1.3",
-        "resolve": "^1.20.0",
-        "resolve.exports": "^1.1.0",
-        "slash": "^3.0.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-runtime": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-28.1.3.tgz",
-      "integrity": "sha512-NU+881ScBQQLc1JHG5eJGU7Ui3kLKrmwCPPtYsJtBykixrM2OhVQlpMmFWJjMyDfdkGgBMNjXCGB/ebzsgNGQw==",
+    "node_modules/jest-runtime": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz",
+      "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==",
       "dev": true,
       "dependencies": {
-        "@jest/environment": "^28.1.3",
-        "@jest/fake-timers": "^28.1.3",
-        "@jest/globals": "^28.1.3",
-        "@jest/source-map": "^28.1.2",
-        "@jest/test-result": "^28.1.3",
-        "@jest/transform": "^28.1.3",
-        "@jest/types": "^28.1.3",
+        "@jest/environment": "^29.7.0",
+        "@jest/fake-timers": "^29.7.0",
+        "@jest/globals": "^29.7.0",
+        "@jest/source-map": "^29.6.3",
+        "@jest/test-result": "^29.7.0",
+        "@jest/transform": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "@types/node": "*",
         "chalk": "^4.0.0",
         "cjs-module-lexer": "^1.0.0",
         "collect-v8-coverage": "^1.0.0",
-        "execa": "^5.0.0",
         "glob": "^7.1.3",
         "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^28.1.3",
-        "jest-message-util": "^28.1.3",
-        "jest-mock": "^28.1.3",
-        "jest-regex-util": "^28.0.2",
-        "jest-resolve": "^28.1.3",
-        "jest-snapshot": "^28.1.3",
-        "jest-util": "^28.1.3",
+        "jest-haste-map": "^29.7.0",
+        "jest-message-util": "^29.7.0",
+        "jest-mock": "^29.7.0",
+        "jest-regex-util": "^29.6.3",
+        "jest-resolve": "^29.7.0",
+        "jest-snapshot": "^29.7.0",
+        "jest-util": "^29.7.0",
         "slash": "^3.0.0",
         "strip-bom": "^4.0.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-snapshot": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-28.1.3.tgz",
-      "integrity": "sha512-4lzMgtiNlc3DU/8lZfmqxN3AYD6GGLbl+72rdBpXvcV+whX7mDrREzkPdp2RnmfIiWBg1YbuFSkXduF2JcafJg==",
+    "node_modules/jest-runtime/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@babel/core": "^7.11.6",
-        "@babel/generator": "^7.7.2",
-        "@babel/plugin-syntax-typescript": "^7.7.2",
-        "@babel/traverse": "^7.7.2",
-        "@babel/types": "^7.3.3",
-        "@jest/expect-utils": "^28.1.3",
-        "@jest/transform": "^28.1.3",
-        "@jest/types": "^28.1.3",
-        "@types/babel__traverse": "^7.0.6",
-        "@types/prettier": "^2.1.5",
-        "babel-preset-current-node-syntax": "^1.0.0",
-        "chalk": "^4.0.0",
-        "expect": "^28.1.3",
-        "graceful-fs": "^4.2.9",
-        "jest-diff": "^28.1.3",
-        "jest-get-type": "^28.0.2",
-        "jest-haste-map": "^28.1.3",
-        "jest-matcher-utils": "^28.1.3",
-        "jest-message-util": "^28.1.3",
-        "jest-util": "^28.1.3",
-        "natural-compare": "^1.4.0",
-        "pretty-format": "^28.1.3",
-        "semver": "^7.3.5"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-util": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-28.1.3.tgz",
-      "integrity": "sha512-XdqfpHwpcSRko/C35uLYFM2emRAltIIKZiJ9eAmhjsj0CqZMa0p1ib0R5fWIqGhn1a103DebTbpqIaP1qCQ6tQ==",
+    "node_modules/jest-runtime/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^28.1.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "ci-info": "^3.2.0",
-        "graceful-fs": "^4.2.9",
-        "picomatch": "^2.2.3"
-      },
-      "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-validate": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-28.1.3.tgz",
-      "integrity": "sha512-SZbOGBWEsaTxBGCOpsRWlXlvNkvTkY0XxRfh7zYmvd8uL5Qzyg0CHAXiXKROflh801quA6+/DsT4ODDthOC/OA==",
+    "node_modules/jest-runtime/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^28.1.3",
-        "camelcase": "^6.2.0",
-        "chalk": "^4.0.0",
-        "jest-get-type": "^28.0.2",
-        "leven": "^3.1.0",
-        "pretty-format": "^28.1.3"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-worker": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-28.1.3.tgz",
-      "integrity": "sha512-CqRA220YV/6jCo8VWvAt1KKx6eek1VIHMPeLEbpcfSfkEeWyBNppynM/o6q+Wmw+sOhos2ml34wZbSX3G13//g==",
+    "node_modules/jest-runtime/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@types/node": "*",
-        "merge-stream": "^2.0.0",
-        "supports-color": "^8.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/jest-worker/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+    "node_modules/jest-runtime/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-runtime/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
       "dev": true,
       "dependencies": {
-        "has-flag": "^4.0.0"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": "*"
       },
       "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/pretty-format": {
-      "version": "28.1.3",
-      "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-28.1.3.tgz",
-      "integrity": "sha512-8gFb/To0OmxHR9+ZTb14Df2vNxdGCX8g1xWGUTqUw5TiZvcQf5sHKObd5UcPyLLyowNwDAMTF3XWOG1B6mxl1Q==",
+    "node_modules/jest-runtime/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-runtime/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "@jest/schemas": "^28.1.3",
-        "ansi-regex": "^5.0.1",
-        "ansi-styles": "^5.0.0",
-        "react-is": "^18.0.0"
+        "brace-expansion": "^1.1.7"
       },
       "engines": {
-        "node": "^12.13.0 || ^14.15.0 || ^16.10.0 || >=17.0.0"
+        "node": "*"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/pretty-format/node_modules/ansi-styles": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
-      "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
+    "node_modules/jest-runtime/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
       "dev": true,
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-jasmine2/node_modules/resolve.exports": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-1.1.1.tgz",
-      "integrity": "sha512-/NtpHNDN7jWhAaQ9BvBUYZ6YTXsRBgfqWFWP7BZBaoMJO/I3G5OFzvTuWNlZC3aPjins1F+TNrLKsGbH4rfsRQ==",
+    "node_modules/jest-runtime/node_modules/strip-bom": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz",
+      "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==",
       "dev": true,
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-leak-detector": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz",
-      "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==",
+    "node_modules/jest-runtime/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "jest-get-type": "^29.6.3",
-        "pretty-format": "^29.7.0"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-matcher-utils": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz",
-      "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==",
+    "node_modules/jest-silent-reporter": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/jest-silent-reporter/-/jest-silent-reporter-0.5.0.tgz",
+      "integrity": "sha512-epdLt8Oj0a1AyRiR6F8zx/1SVT1Mi7VU3y4wB2uOBHs/ohIquC7v2eeja7UN54uRPyHInIKWdL+RdG228n5pJQ==",
       "dev": true,
       "dependencies": {
         "chalk": "^4.0.0",
-        "jest-diff": "^29.7.0",
-        "jest-get-type": "^29.6.3",
-        "pretty-format": "^29.7.0"
+        "jest-util": "^26.0.0"
+      }
+    },
+    "node_modules/jest-silent-reporter/node_modules/@jest/types": {
+      "version": "26.6.2",
+      "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.6.2.tgz",
+      "integrity": "sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/istanbul-lib-coverage": "^2.0.0",
+        "@types/istanbul-reports": "^3.0.0",
+        "@types/node": "*",
+        "@types/yargs": "^15.0.0",
+        "chalk": "^4.0.0"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">= 10.14.2"
       }
     },
-    "node_modules/jest-matcher-utils/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-silent-reporter/node_modules/@types/yargs": {
+      "version": "15.0.18",
+      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.18.tgz",
+      "integrity": "sha512-DDi2KmvAnNsT/EvU8jp1UR7pOJojBtJ3GLZ/uw1MUq4VbbESppPWoHUY4h0OB4BbEbGJiyEsmUcuZDZtoR+ZwQ==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "@types/yargs-parser": "*"
       }
     },
-    "node_modules/jest-message-util": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz",
-      "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==",
+    "node_modules/jest-silent-reporter/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@babel/code-frame": "^7.12.13",
-        "@jest/types": "^29.6.3",
-        "@types/stack-utils": "^2.0.0",
-        "chalk": "^4.0.0",
-        "graceful-fs": "^4.2.9",
-        "micromatch": "^4.0.4",
-        "pretty-format": "^29.7.0",
-        "slash": "^3.0.0",
-        "stack-utils": "^2.0.3"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-message-util/node_modules/chalk": {
+    "node_modules/jest-silent-reporter/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18204,108 +21737,127 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-mock": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz",
-      "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==",
+    "node_modules/jest-silent-reporter/node_modules/ci-info": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
+      "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
+      "dev": true
+    },
+    "node_modules/jest-silent-reporter/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "jest-util": "^29.7.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-pnp-resolver": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz",
-      "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==",
+    "node_modules/jest-silent-reporter/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-silent-reporter/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=6"
-      },
-      "peerDependencies": {
-        "jest-resolve": "*"
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-silent-reporter/node_modules/is-ci": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
+      "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+      "dev": true,
+      "dependencies": {
+        "ci-info": "^2.0.0"
       },
-      "peerDependenciesMeta": {
-        "jest-resolve": {
-          "optional": true
-        }
+      "bin": {
+        "is-ci": "bin.js"
       }
     },
-    "node_modules/jest-preset-angular": {
-      "version": "13.1.1",
-      "resolved": "https://registry.npmjs.org/jest-preset-angular/-/jest-preset-angular-13.1.1.tgz",
-      "integrity": "sha512-X8i7icKt9U5uhj7YKqdEZm7ZZPvNFRxfBnU+9SALdIkHYJhwtlJ5/MUk9wo4f3lX2smOkIl9LPJUu1APO+11Jg==",
+    "node_modules/jest-silent-reporter/node_modules/jest-util": {
+      "version": "26.6.2",
+      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-26.6.2.tgz",
+      "integrity": "sha512-MDW0fKfsn0OI7MS7Euz6h8HNDXVQ0gaM9uW6RjfDmd1DAFcaxX9OqIakHIqhbnmF08Cf2DLDG+ulq8YQQ0Lp0Q==",
       "dev": true,
       "dependencies": {
-        "bs-logger": "^0.2.6",
-        "esbuild-wasm": ">=0.13.8",
-        "jest-environment-jsdom": "^29.0.0",
-        "jest-util": "^29.0.0",
-        "pretty-format": "^29.0.0",
-        "ts-jest": "^29.0.0"
+        "@jest/types": "^26.6.2",
+        "@types/node": "*",
+        "chalk": "^4.0.0",
+        "graceful-fs": "^4.2.4",
+        "is-ci": "^2.0.0",
+        "micromatch": "^4.0.2"
       },
       "engines": {
-        "node": "^14.15.0 || >=16.10.0"
-      },
-      "optionalDependencies": {
-        "esbuild": ">=0.13.8"
-      },
-      "peerDependencies": {
-        "@angular-devkit/build-angular": ">=13.0.0 <17.0.0",
-        "@angular/compiler-cli": ">=13.0.0 <17.0.0",
-        "@angular/core": ">=13.0.0 <17.0.0",
-        "@angular/platform-browser-dynamic": ">=13.0.0 <17.0.0",
-        "jest": "^29.0.0",
-        "typescript": ">=4.4"
+        "node": ">= 10.14.2"
       }
     },
-    "node_modules/jest-regex-util": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz",
-      "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==",
+    "node_modules/jest-silent-reporter/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-resolve": {
+    "node_modules/jest-snapshot": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz",
-      "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==",
+      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz",
+      "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==",
       "dev": true,
       "dependencies": {
+        "@babel/core": "^7.11.6",
+        "@babel/generator": "^7.7.2",
+        "@babel/plugin-syntax-jsx": "^7.7.2",
+        "@babel/plugin-syntax-typescript": "^7.7.2",
+        "@babel/types": "^7.3.3",
+        "@jest/expect-utils": "^29.7.0",
+        "@jest/transform": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "babel-preset-current-node-syntax": "^1.0.0",
         "chalk": "^4.0.0",
+        "expect": "^29.7.0",
         "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^29.7.0",
-        "jest-pnp-resolver": "^1.2.2",
+        "jest-diff": "^29.7.0",
+        "jest-get-type": "^29.6.3",
+        "jest-matcher-utils": "^29.7.0",
+        "jest-message-util": "^29.7.0",
         "jest-util": "^29.7.0",
-        "jest-validate": "^29.7.0",
-        "resolve": "^1.20.0",
-        "resolve.exports": "^2.0.0",
-        "slash": "^3.0.0"
+        "natural-compare": "^1.4.0",
+        "pretty-format": "^29.7.0",
+        "semver": "^7.5.3"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-resolve-dependencies": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz",
-      "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==",
+    "node_modules/jest-snapshot/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "jest-regex-util": "^29.6.3",
-        "jest-snapshot": "^29.7.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-resolve/node_modules/chalk": {
+    "node_modules/jest-snapshot/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18321,121 +21873,111 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-runner": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz",
-      "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==",
+    "node_modules/jest-snapshot/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@jest/console": "^29.7.0",
-        "@jest/environment": "^29.7.0",
-        "@jest/test-result": "^29.7.0",
-        "@jest/transform": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "emittery": "^0.13.1",
-        "graceful-fs": "^4.2.9",
-        "jest-docblock": "^29.7.0",
-        "jest-environment-node": "^29.7.0",
-        "jest-haste-map": "^29.7.0",
-        "jest-leak-detector": "^29.7.0",
-        "jest-message-util": "^29.7.0",
-        "jest-resolve": "^29.7.0",
-        "jest-runtime": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "jest-watcher": "^29.7.0",
-        "jest-worker": "^29.7.0",
-        "p-limit": "^3.1.0",
-        "source-map-support": "0.5.13"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-runner/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-snapshot/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-snapshot/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-snapshot/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "yallist": "^4.0.0"
       },
       "engines": {
         "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-runner/node_modules/source-map": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+    "node_modules/jest-snapshot/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
       }
     },
-    "node_modules/jest-runner/node_modules/source-map-support": {
-      "version": "0.5.13",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz",
-      "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==",
+    "node_modules/jest-snapshot/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "buffer-from": "^1.0.0",
-        "source-map": "^0.6.0"
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/jest-runtime": {
+    "node_modules/jest-snapshot/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
+    "node_modules/jest-util": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz",
-      "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==",
+      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz",
+      "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==",
       "dev": true,
       "dependencies": {
-        "@jest/environment": "^29.7.0",
-        "@jest/fake-timers": "^29.7.0",
-        "@jest/globals": "^29.7.0",
-        "@jest/source-map": "^29.6.3",
-        "@jest/test-result": "^29.7.0",
-        "@jest/transform": "^29.7.0",
         "@jest/types": "^29.6.3",
         "@types/node": "*",
         "chalk": "^4.0.0",
-        "cjs-module-lexer": "^1.0.0",
-        "collect-v8-coverage": "^1.0.0",
-        "glob": "^7.1.3",
+        "ci-info": "^3.2.0",
         "graceful-fs": "^4.2.9",
-        "jest-haste-map": "^29.7.0",
-        "jest-message-util": "^29.7.0",
-        "jest-mock": "^29.7.0",
-        "jest-regex-util": "^29.6.3",
-        "jest-resolve": "^29.7.0",
-        "jest-snapshot": "^29.7.0",
-        "jest-util": "^29.7.0",
-        "slash": "^3.0.0",
-        "strip-bom": "^4.0.0"
+        "picomatch": "^2.2.3"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-runtime/node_modules/@jest/source-map": {
-      "version": "29.6.3",
-      "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz",
-      "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==",
+    "node_modules/jest-util/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@jridgewell/trace-mapping": "^0.3.18",
-        "callsites": "^3.0.0",
-        "graceful-fs": "^4.2.9"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-runtime/node_modules/chalk": {
+    "node_modules/jest-util/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18451,62 +21993,90 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-runtime/node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+    "node_modules/jest-util/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "*"
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/jest-util/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-util/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-util/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/jest-silent-reporter": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/jest-silent-reporter/-/jest-silent-reporter-0.5.0.tgz",
-      "integrity": "sha512-epdLt8Oj0a1AyRiR6F8zx/1SVT1Mi7VU3y4wB2uOBHs/ohIquC7v2eeja7UN54uRPyHInIKWdL+RdG228n5pJQ==",
+    "node_modules/jest-validate": {
+      "version": "29.7.0",
+      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
+      "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==",
       "dev": true,
       "dependencies": {
+        "@jest/types": "^29.6.3",
+        "camelcase": "^6.2.0",
         "chalk": "^4.0.0",
-        "jest-util": "^26.0.0"
+        "jest-get-type": "^29.6.3",
+        "leven": "^3.1.0",
+        "pretty-format": "^29.7.0"
+      },
+      "engines": {
+        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-silent-reporter/node_modules/@jest/types": {
-      "version": "26.6.2",
-      "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.6.2.tgz",
-      "integrity": "sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==",
+    "node_modules/jest-validate/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@types/istanbul-lib-coverage": "^2.0.0",
-        "@types/istanbul-reports": "^3.0.0",
-        "@types/node": "*",
-        "@types/yargs": "^15.0.0",
-        "chalk": "^4.0.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">= 10.14.2"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-silent-reporter/node_modules/@types/yargs": {
-      "version": "15.0.16",
-      "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.16.tgz",
-      "integrity": "sha512-2FeD5qezW3FvLpZ0JpfuaEWepgNLl9b2gQYiz/ce0NhoB1W/D+VZu98phITXkADYerfr/jb7JcDcVhITsc9bwg==",
+    "node_modules/jest-validate/node_modules/camelcase": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
       "dev": true,
-      "dependencies": {
-        "@types/yargs-parser": "*"
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/jest-silent-reporter/node_modules/chalk": {
+    "node_modules/jest-validate/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18522,106 +22092,80 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-silent-reporter/node_modules/ci-info": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
-      "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==",
-      "dev": true
-    },
-    "node_modules/jest-silent-reporter/node_modules/is-ci": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz",
-      "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==",
+    "node_modules/jest-validate/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "ci-info": "^2.0.0"
+        "color-name": "~1.1.4"
       },
-      "bin": {
-        "is-ci": "bin.js"
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-silent-reporter/node_modules/jest-util": {
-      "version": "26.6.2",
-      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-26.6.2.tgz",
-      "integrity": "sha512-MDW0fKfsn0OI7MS7Euz6h8HNDXVQ0gaM9uW6RjfDmd1DAFcaxX9OqIakHIqhbnmF08Cf2DLDG+ulq8YQQ0Lp0Q==",
+    "node_modules/jest-validate/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-validate/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-validate/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^26.6.2",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "graceful-fs": "^4.2.4",
-        "is-ci": "^2.0.0",
-        "micromatch": "^4.0.2"
+        "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">= 10.14.2"
+        "node": ">=8"
       }
     },
-    "node_modules/jest-snapshot": {
+    "node_modules/jest-watcher": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz",
-      "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==",
+      "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz",
+      "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==",
       "dev": true,
       "dependencies": {
-        "@babel/core": "^7.11.6",
-        "@babel/generator": "^7.7.2",
-        "@babel/plugin-syntax-jsx": "^7.7.2",
-        "@babel/plugin-syntax-typescript": "^7.7.2",
-        "@babel/types": "^7.3.3",
-        "@jest/expect-utils": "^29.7.0",
-        "@jest/transform": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
         "@jest/types": "^29.6.3",
-        "babel-preset-current-node-syntax": "^1.0.0",
+        "@types/node": "*",
+        "ansi-escapes": "^4.2.1",
         "chalk": "^4.0.0",
-        "expect": "^29.7.0",
-        "graceful-fs": "^4.2.9",
-        "jest-diff": "^29.7.0",
-        "jest-get-type": "^29.6.3",
-        "jest-matcher-utils": "^29.7.0",
-        "jest-message-util": "^29.7.0",
+        "emittery": "^0.13.1",
         "jest-util": "^29.7.0",
-        "natural-compare": "^1.4.0",
-        "pretty-format": "^29.7.0",
-        "semver": "^7.5.3"
+        "string-length": "^4.0.1"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
-    "node_modules/jest-snapshot/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-watcher/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
-      }
-    },
-    "node_modules/jest-util": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz",
-      "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==",
-      "dev": true,
-      "dependencies": {
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "chalk": "^4.0.0",
-        "ci-info": "^3.2.0",
-        "graceful-fs": "^4.2.9",
-        "picomatch": "^2.2.3"
-      },
-      "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-util/node_modules/chalk": {
+    "node_modules/jest-watcher/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18637,71 +22181,99 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-validate": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz",
-      "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==",
+    "node_modules/jest-watcher/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@jest/types": "^29.6.3",
-        "camelcase": "^6.2.0",
-        "chalk": "^4.0.0",
-        "jest-get-type": "^29.6.3",
-        "leven": "^3.1.0",
-        "pretty-format": "^29.7.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/jest-validate/node_modules/camelcase": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
-      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+    "node_modules/jest-watcher/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest-watcher/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-watcher/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
       },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+      "engines": {
+        "node": ">=8"
       }
     },
-    "node_modules/jest-validate/node_modules/chalk": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+    "node_modules/jest-worker": {
+      "version": "27.5.1",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz",
+      "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "@types/node": "*",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      }
+    },
+    "node_modules/jest-worker/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
       },
       "engines": {
         "node": ">=10"
       },
       "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
       }
     },
-    "node_modules/jest-watcher": {
-      "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz",
-      "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==",
+    "node_modules/jest/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
       "dependencies": {
-        "@jest/test-result": "^29.7.0",
-        "@jest/types": "^29.6.3",
-        "@types/node": "*",
-        "ansi-escapes": "^4.2.1",
-        "chalk": "^4.0.0",
-        "emittery": "^0.13.1",
-        "jest-util": "^29.7.0",
-        "string-length": "^4.0.1"
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/jest-watcher/node_modules/chalk": {
+    "node_modules/jest/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
@@ -18717,34 +22289,76 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/jest-worker": {
+    "node_modules/jest/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/jest/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/jest/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jest/node_modules/jest-cli": {
       "version": "29.7.0",
-      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz",
-      "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==",
+      "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz",
+      "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==",
       "dev": true,
       "dependencies": {
-        "@types/node": "*",
+        "@jest/core": "^29.7.0",
+        "@jest/test-result": "^29.7.0",
+        "@jest/types": "^29.6.3",
+        "chalk": "^4.0.0",
+        "create-jest": "^29.7.0",
+        "exit": "^0.1.2",
+        "import-local": "^3.0.2",
+        "jest-config": "^29.7.0",
         "jest-util": "^29.7.0",
-        "merge-stream": "^2.0.0",
-        "supports-color": "^8.0.0"
+        "jest-validate": "^29.7.0",
+        "yargs": "^17.3.1"
+      },
+      "bin": {
+        "jest": "bin/jest.js"
       },
       "engines": {
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+      },
+      "peerDependencies": {
+        "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0"
+      },
+      "peerDependenciesMeta": {
+        "node-notifier": {
+          "optional": true
+        }
       }
     },
-    "node_modules/jest-worker/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+    "node_modules/jest/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
       "dependencies": {
         "has-flag": "^4.0.0"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
+        "node": ">=8"
       }
     },
     "node_modules/jju": {
@@ -18754,9 +22368,9 @@
       "dev": true
     },
     "node_modules/joi": {
-      "version": "17.10.2",
-      "resolved": "https://registry.npmjs.org/joi/-/joi-17.10.2.tgz",
-      "integrity": "sha512-hcVhjBxRNW/is3nNLdGLIjkgXetkeGc2wyhydhz8KumG23Aerk4HPjU5zaPAMRqXQFc0xNqXTC7+zQjxr0GlKA==",
+      "version": "17.11.0",
+      "resolved": "https://registry.npmjs.org/joi/-/joi-17.11.0.tgz",
+      "integrity": "sha512-NgB+lZLNoqISVy1rZocE9PZI36bL/77ie924Ri43yEvi9GUUMPeyVIr8KdFTMUlby1p0PBYMk9spIxEUQYqrJQ==",
       "dev": true,
       "dependencies": {
         "@hapi/hoek": "^9.0.0",
@@ -18786,13 +22400,11 @@
       "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="
     },
     "node_modules/js-yaml": {
-      "version": "3.14.1",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
-      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
-      "dev": true,
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
       "dependencies": {
-        "argparse": "^1.0.7",
-        "esprima": "^4.0.0"
+        "argparse": "^2.0.1"
       },
       "bin": {
         "js-yaml": "bin/js-yaml.js"
@@ -18849,18 +22461,6 @@
         }
       }
     },
-    "node_modules/jsdom/node_modules/acorn": {
-      "version": "8.10.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
-      "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
-      "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
     "node_modules/jsdom/node_modules/acorn-globals": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz",
@@ -18872,9 +22472,9 @@
       }
     },
     "node_modules/jsdom/node_modules/acorn-walk": {
-      "version": "8.2.0",
-      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz",
-      "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==",
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.0.tgz",
+      "integrity": "sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA==",
       "dev": true,
       "engines": {
         "node": ">=0.4.0"
@@ -18904,6 +22504,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
       "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",
+      "deprecated": "Use your platform's native DOMException instead",
       "dev": true,
       "dependencies": {
         "webidl-conversions": "^7.0.0"
@@ -18912,6 +22513,18 @@
         "node": ">=12"
       }
     },
+    "node_modules/jsdom/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/jsdom/node_modules/escodegen": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
@@ -18942,20 +22555,6 @@
         "node": ">=4.0"
       }
     },
-    "node_modules/jsdom/node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/jsdom/node_modules/html-encoding-sniffer": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@@ -18992,15 +22591,6 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/jsdom/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/jsdom/node_modules/saxes": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
@@ -19114,27 +22704,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/jsdom/node_modules/ws": {
-      "version": "8.14.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
-      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
-      "dev": true,
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/jsdom/node_modules/xml-name-validator": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
@@ -19279,38 +22848,18 @@
       }
     },
     "node_modules/jsprim": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz",
-      "integrity": "sha512-gqXddjPqQ6G40VdnI6T6yObEC+pDNvyP95wdQhkWkg7crHH3km5qP1FsOXEkzEQwnz6gz5qGTn1c2Y52wP3OyQ==",
-      "dev": true,
-      "engines": [
-        "node >=0.6.0"
-      ],
-      "dependencies": {
-        "assert-plus": "1.0.0",
-        "extsprintf": "1.3.0",
-        "json-schema": "0.4.0",
-        "verror": "1.10.0"
-      }
-    },
-    "node_modules/jsprim/node_modules/core-util-is": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==",
-      "dev": true
-    },
-    "node_modules/jsprim/node_modules/verror": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
-      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz",
+      "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==",
       "dev": true,
-      "engines": [
-        "node >=0.6.0"
-      ],
       "dependencies": {
-        "assert-plus": "^1.0.0",
-        "core-util-is": "1.0.2",
-        "extsprintf": "^1.2.0"
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.4.0",
+        "verror": "1.10.0"
+      },
+      "engines": {
+        "node": ">=0.6.0"
       }
     },
     "node_modules/karma-source-map-support": {
@@ -19323,9 +22872,9 @@
       }
     },
     "node_modules/keyv": {
-      "version": "4.5.3",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
-      "integrity": "sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug==",
+      "version": "4.5.4",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
+      "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
       "dev": true,
       "dependencies": {
         "json-buffer": "3.0.1"
@@ -19464,16 +23013,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/less/node_modules/pify": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
-      "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
-      "dev": true,
-      "optional": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/less/node_modules/semver": {
       "version": "5.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
@@ -19504,13 +23043,13 @@
       }
     },
     "node_modules/levn": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
-      "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
+      "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==",
       "dev": true,
       "dependencies": {
-        "prelude-ls": "^1.2.1",
-        "type-check": "~0.4.0"
+        "prelude-ls": "~1.1.2",
+        "type-check": "~0.3.2"
       },
       "engines": {
         "node": ">= 0.8.0"
@@ -19552,6 +23091,18 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/liftoff/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "dev": true,
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -19631,15 +23182,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/load-json-file/node_modules/strip-bom": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/loader-runner": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
@@ -19704,10 +23246,9 @@
       "dev": true
     },
     "node_modules/lodash.memoize": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz",
-      "integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A==",
-      "dev": true
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
+      "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag=="
     },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
@@ -19754,6 +23295,21 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/log-symbols/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
     "node_modules/log-symbols/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -19770,6 +23326,45 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
+    "node_modules/log-symbols/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/log-symbols/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/log-symbols/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/log-symbols/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/log-update": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/log-update/-/log-update-4.0.0.tgz",
@@ -19788,6 +23383,39 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/log-update/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/log-update/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/log-update/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
     "node_modules/log-update/node_modules/slice-ansi": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
@@ -19866,12 +23494,12 @@
       }
     },
     "node_modules/loupe": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.6.tgz",
-      "integrity": "sha512-RaPMZKiMy8/JruncMU5Bt6na1eftNoo++R4Y+N2FrxkDVTrGvcyzFTsaGif4QTeKESheMGegbhw6iUAq+5A8zA==",
+      "version": "2.3.7",
+      "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz",
+      "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==",
       "dev": true,
       "dependencies": {
-        "get-func-name": "^2.0.0"
+        "get-func-name": "^2.0.1"
       }
     },
     "node_modules/lower-case": {
@@ -19946,15 +23574,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/make-dir/node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
     "node_modules/make-error": {
       "version": "1.3.6",
       "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
@@ -19962,29 +23581,93 @@
       "dev": true
     },
     "node_modules/make-fetch-happen": {
-      "version": "11.1.1",
-      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-11.1.1.tgz",
-      "integrity": "sha512-rLWS7GCSTcEujjVBs2YqG7Y4643u8ucvCJeSRqiLYhesrDuzeuFIk37xREzAsfQaqzl8b9rNCE4m6J8tvX4Q8w==",
+      "version": "10.2.1",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz",
+      "integrity": "sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==",
       "dev": true,
       "dependencies": {
         "agentkeepalive": "^4.2.1",
-        "cacache": "^17.0.0",
-        "http-cache-semantics": "^4.1.1",
+        "cacache": "^16.1.0",
+        "http-cache-semantics": "^4.1.0",
         "http-proxy-agent": "^5.0.0",
         "https-proxy-agent": "^5.0.0",
         "is-lambda": "^1.0.1",
         "lru-cache": "^7.7.1",
-        "minipass": "^5.0.0",
-        "minipass-fetch": "^3.0.0",
+        "minipass": "^3.1.6",
+        "minipass-collect": "^1.0.2",
+        "minipass-fetch": "^2.0.3",
         "minipass-flush": "^1.0.5",
         "minipass-pipeline": "^1.2.4",
         "negotiator": "^0.6.3",
         "promise-retry": "^2.0.1",
         "socks-proxy-agent": "^7.0.0",
-        "ssri": "^10.0.0"
+        "ssri": "^9.0.0"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/@npmcli/fs": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz",
+      "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==",
+      "dev": true,
+      "dependencies": {
+        "@gar/promisify": "^1.1.3",
+        "semver": "^7.3.5"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/cacache": {
+      "version": "16.1.3",
+      "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz",
+      "integrity": "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==",
+      "dev": true,
+      "dependencies": {
+        "@npmcli/fs": "^2.1.0",
+        "@npmcli/move-file": "^2.0.0",
+        "chownr": "^2.0.0",
+        "fs-minipass": "^2.1.0",
+        "glob": "^8.0.1",
+        "infer-owner": "^1.0.4",
+        "lru-cache": "^7.7.1",
+        "minipass": "^3.1.6",
+        "minipass-collect": "^1.0.2",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "mkdirp": "^1.0.4",
+        "p-map": "^4.0.0",
+        "promise-inflight": "^1.0.1",
+        "rimraf": "^3.0.2",
+        "ssri": "^9.0.0",
+        "tar": "^6.1.11",
+        "unique-filename": "^2.0.0"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/chownr": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+      "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/fs-minipass": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
+      "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
+      "dev": true,
+      "dependencies": {
+        "minipass": "^3.0.0"
+      },
+      "engines": {
+        "node": ">= 8"
       }
     },
     "node_modules/make-fetch-happen/node_modules/lru-cache": {
@@ -19997,14 +23680,86 @@
       }
     },
     "node_modules/make-fetch-happen/node_modules/minipass": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
-      "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+      "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/semver/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/ssri": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz",
+      "integrity": "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==",
+      "dev": true,
+      "dependencies": {
+        "minipass": "^3.1.1"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/unique-filename": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz",
+      "integrity": "sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==",
+      "dev": true,
+      "dependencies": {
+        "unique-slug": "^3.0.0"
+      },
+      "engines": {
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
+      }
+    },
+    "node_modules/make-fetch-happen/node_modules/unique-slug": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-3.0.0.tgz",
+      "integrity": "sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==",
       "dev": true,
+      "dependencies": {
+        "imurmurhash": "^0.1.4"
+      },
       "engines": {
-        "node": ">=8"
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
       }
     },
+    "node_modules/make-fetch-happen/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/make-iterator": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/make-iterator/-/make-iterator-1.0.1.tgz",
@@ -20281,6 +24036,21 @@
         "node": ">=10"
       }
     },
+    "node_modules/meow/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/meow/node_modules/type-fest": {
       "version": "0.18.1",
       "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.1.tgz",
@@ -20421,19 +24191,19 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.44.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
-      "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==",
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.27",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
-      "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
       "dependencies": {
-        "mime-db": "1.44.0"
+        "mime-db": "1.52.0"
       },
       "engines": {
         "node": ">= 0.6"
@@ -20512,15 +24282,14 @@
       "dev": true
     },
     "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-      "dev": true,
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
+      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
       "dependencies": {
-        "brace-expansion": "^1.1.7"
+        "brace-expansion": "^2.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=10"
       }
     },
     "node_modules/minimist": {
@@ -20555,6 +24324,15 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/minimist-options/node_modules/is-plain-obj": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
+      "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/minipass": {
       "version": "4.2.8",
       "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz",
@@ -20595,31 +24373,40 @@
       "dev": true
     },
     "node_modules/minipass-fetch": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.4.tgz",
-      "integrity": "sha512-jHAqnA728uUpIaFm7NWsCnqKT6UqZz7GcI/bDpPATuwYyKwJwW0remxSCxUlKiEty+eopHGa3oc8WxgQ1FFJqg==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-2.1.2.tgz",
+      "integrity": "sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==",
       "dev": true,
       "dependencies": {
-        "minipass": "^7.0.3",
+        "minipass": "^3.1.6",
         "minipass-sized": "^1.0.3",
         "minizlib": "^2.1.2"
       },
       "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+        "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
       },
       "optionalDependencies": {
         "encoding": "^0.1.13"
       }
     },
     "node_modules/minipass-fetch/node_modules/minipass": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "version": "3.3.6",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+      "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
       "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
       "engines": {
-        "node": ">=16 || 14 >=14.17"
+        "node": ">=8"
       }
     },
+    "node_modules/minipass-fetch/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/minipass-flush": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
@@ -20782,16 +24569,40 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/mkdirp": {
-      "version": "0.5.5",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
-      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+    "node_modules/mixin-deep/node_modules/is-extendable": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+      "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
       "dev": true,
       "dependencies": {
-        "minimist": "^1.2.5"
+        "is-plain-object": "^2.0.4"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/mixin-deep/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "dev": true,
+      "dependencies": {
+        "isobject": "^3.0.1"
       },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/mkdirp": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+      "dev": true,
       "bin": {
         "mkdirp": "bin/cmd.js"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
     "node_modules/mkdirp-classic": {
@@ -20830,6 +24641,18 @@
         "ms": "2.0.0"
       }
     },
+    "node_modules/mocha-junit-reporter/node_modules/mkdirp": {
+      "version": "0.5.6",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
+      "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
+      "dev": true,
+      "dependencies": {
+        "minimist": "^1.2.6"
+      },
+      "bin": {
+        "mkdirp": "bin/cmd.js"
+      }
+    },
     "node_modules/mocha-junit-reporter/node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -20865,6 +24688,45 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/module-deps/node_modules/browser-resolve": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz",
+      "integrity": "sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==",
+      "dev": true,
+      "dependencies": {
+        "resolve": "^1.17.0"
+      }
+    },
+    "node_modules/module-deps/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/module-deps/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/module-deps/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/moment": {
       "version": "2.29.4",
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
@@ -20882,6 +24744,12 @@
         "color-name": "^1.1.4"
       }
     },
+    "node_modules/moo-color/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
     "node_modules/morgan": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz",
@@ -20907,21 +24775,24 @@
         "ms": "2.0.0"
       }
     },
-    "node_modules/morgan/node_modules/depd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
-      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
     "node_modules/morgan/node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "dev": true
     },
+    "node_modules/morgan/node_modules/on-finished": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+      "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==",
+      "dev": true,
+      "dependencies": {
+        "ee-first": "1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/mri": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@@ -20965,6 +24836,28 @@
         "node": ">=8"
       }
     },
+    "node_modules/multimatch/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/multimatch/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/mute-stream": {
       "version": "0.0.8",
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
@@ -20989,9 +24882,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "3.3.6",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
-      "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==",
+      "version": "3.3.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
+      "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
       "dev": true,
       "funding": [
         {
@@ -21216,9 +25109,9 @@
       }
     },
     "node_modules/node-abi": {
-      "version": "3.47.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.47.0.tgz",
-      "integrity": "sha512-2s6B2CWZM//kPgwnuI0KrYwNjfdByE25zvAaEpq9IH4zcNsarH8Ihu/UuX6XMPEogDAxkuUFeZn60pXNHAqn3A==",
+      "version": "3.51.0",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.51.0.tgz",
+      "integrity": "sha512-SQkEP4hmNWjlniS5zdnfIXTk1x7Ome85RDzHlTbBtzE97Gfwz/Ipw4v/Ryk20DWIy3yCNVLVlGKApCnmvYoJbA==",
       "optional": true,
       "dependencies": {
         "semver": "^7.3.5"
@@ -21227,6 +25120,39 @@
         "node": ">=10"
       }
     },
+    "node_modules/node-abi/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "optional": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-abi/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "optional": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-abi/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "optional": true
+    },
     "node_modules/node-abort-controller": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
@@ -21285,33 +25211,11 @@
         "web-streams-polyfill": "^3.0.3"
       },
       "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/node-fetch"
-      }
-    },
-    "node_modules/node-fetch/node_modules/tr46": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
-      "dev": true
-    },
-    "node_modules/node-fetch/node_modules/webidl-conversions": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
-      "dev": true
-    },
-    "node_modules/node-fetch/node_modules/whatwg-url": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-      "dev": true,
-      "dependencies": {
-        "tr46": "~0.0.3",
-        "webidl-conversions": "^3.0.0"
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/node-fetch"
       }
     },
     "node_modules/node-forge": {
@@ -21324,16 +25228,16 @@
       }
     },
     "node_modules/node-gyp": {
-      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.0.tgz",
-      "integrity": "sha512-dMXsYP6gc9rRbejLXmTbVRYjAHw7ppswsKyMxuxJxxOHzluIO1rGp9TOQgjFJ+2MCqcOcQTOPB/8Xwhr+7s4Eg==",
+      "version": "9.4.1",
+      "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz",
+      "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==",
       "dev": true,
       "dependencies": {
         "env-paths": "^2.2.0",
         "exponential-backoff": "^3.1.1",
         "glob": "^7.1.4",
         "graceful-fs": "^4.2.6",
-        "make-fetch-happen": "^11.0.3",
+        "make-fetch-happen": "^10.0.3",
         "nopt": "^6.0.0",
         "npmlog": "^6.0.0",
         "rimraf": "^3.0.2",
@@ -21349,9 +25253,9 @@
       }
     },
     "node_modules/node-gyp-build": {
-      "version": "4.6.1",
-      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz",
-      "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==",
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.7.0.tgz",
+      "integrity": "sha512-PbZERfeFdrHQOOXiAKOY0VPbykZy90ndPKk0d+CFDegTKmWp1VgOTz2xACVbr1BjCWxrQp68CXtvNsveFhqDJg==",
       "dev": true,
       "bin": {
         "node-gyp-build": "bin.js",
@@ -21359,6 +25263,16 @@
         "node-gyp-build-test": "build-test.js"
       }
     },
+    "node_modules/node-gyp/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
     "node_modules/node-gyp/node_modules/glob": {
       "version": "7.2.3",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -21379,6 +25293,51 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/node-gyp/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-gyp/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/node-gyp/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-gyp/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/node-int64": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@@ -21420,6 +25379,39 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/normalize-package-data/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/normalize-package-data/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/normalize-package-data/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -21457,9 +25449,9 @@
       }
     },
     "node_modules/npm-install-checks": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-6.2.0.tgz",
-      "integrity": "sha512-744wat5wAAHsxa4590mWO0tJ8PKxR8ORZsH9wGpQc3nWTzozMAgBN/XyqYw7mg3yqLM8dLwEnwSfKMmXAjF69g==",
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-6.3.0.tgz",
+      "integrity": "sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==",
       "dev": true,
       "dependencies": {
         "semver": "^7.1.1"
@@ -21468,6 +25460,39 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/npm-install-checks/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-install-checks/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-install-checks/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/npm-normalize-package-bin": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-3.0.1.tgz",
@@ -21492,6 +25517,39 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/npm-package-arg/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-package-arg/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-package-arg/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/npm-packlist": {
       "version": "7.0.4",
       "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-7.0.4.tgz",
@@ -21519,6 +25577,39 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/npm-pick-manifest/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-pick-manifest/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/npm-pick-manifest/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/npm-registry-fetch": {
       "version": "14.0.5",
       "resolved": "https://registry.npmjs.org/npm-registry-fetch/-/npm-registry-fetch-14.0.5.tgz",
@@ -21537,6 +25628,41 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/npm-registry-fetch/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/npm-registry-fetch/node_modules/make-fetch-happen": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-11.1.1.tgz",
+      "integrity": "sha512-rLWS7GCSTcEujjVBs2YqG7Y4643u8ucvCJeSRqiLYhesrDuzeuFIk37xREzAsfQaqzl8b9rNCE4m6J8tvX4Q8w==",
+      "dev": true,
+      "dependencies": {
+        "agentkeepalive": "^4.2.1",
+        "cacache": "^17.0.0",
+        "http-cache-semantics": "^4.1.1",
+        "http-proxy-agent": "^5.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "is-lambda": "^1.0.1",
+        "lru-cache": "^7.7.1",
+        "minipass": "^5.0.0",
+        "minipass-fetch": "^3.0.0",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "negotiator": "^0.6.3",
+        "promise-retry": "^2.0.1",
+        "socks-proxy-agent": "^7.0.0",
+        "ssri": "^10.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
     "node_modules/npm-registry-fetch/node_modules/minipass": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
@@ -21546,72 +25672,67 @@
         "node": ">=8"
       }
     },
-    "node_modules/npm-run-all": {
-      "version": "4.1.5",
-      "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz",
-      "integrity": "sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==",
+    "node_modules/npm-registry-fetch/node_modules/minipass-fetch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.4.tgz",
+      "integrity": "sha512-jHAqnA728uUpIaFm7NWsCnqKT6UqZz7GcI/bDpPATuwYyKwJwW0remxSCxUlKiEty+eopHGa3oc8WxgQ1FFJqg==",
       "dev": true,
       "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "chalk": "^2.4.1",
-        "cross-spawn": "^6.0.5",
-        "memorystream": "^0.3.1",
-        "minimatch": "^3.0.4",
-        "pidtree": "^0.3.0",
-        "read-pkg": "^3.0.0",
-        "shell-quote": "^1.6.1",
-        "string.prototype.padend": "^3.0.0"
-      },
-      "bin": {
-        "npm-run-all": "bin/npm-run-all/index.js",
-        "run-p": "bin/run-p/index.js",
-        "run-s": "bin/run-s/index.js"
+        "minipass": "^7.0.3",
+        "minipass-sized": "^1.0.3",
+        "minizlib": "^2.1.2"
       },
       "engines": {
-        "node": ">= 4"
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      },
+      "optionalDependencies": {
+        "encoding": "^0.1.13"
       }
     },
-    "node_modules/npm-run-all/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+    "node_modules/npm-registry-fetch/node_modules/minipass-fetch/node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "dev": true,
-      "dependencies": {
-        "color-convert": "^1.9.0"
-      },
       "engines": {
-        "node": ">=4"
+        "node": ">=16 || 14 >=14.17"
       }
     },
-    "node_modules/npm-run-all/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+    "node_modules/npm-run-all": {
+      "version": "4.1.5",
+      "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz",
+      "integrity": "sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==",
       "dev": true,
       "dependencies": {
         "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
+        "chalk": "^2.4.1",
+        "cross-spawn": "^6.0.5",
+        "memorystream": "^0.3.1",
+        "minimatch": "^3.0.4",
+        "pidtree": "^0.3.0",
+        "read-pkg": "^3.0.0",
+        "shell-quote": "^1.6.1",
+        "string.prototype.padend": "^3.0.0"
+      },
+      "bin": {
+        "npm-run-all": "bin/npm-run-all/index.js",
+        "run-p": "bin/run-p/index.js",
+        "run-s": "bin/run-s/index.js"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">= 4"
       }
     },
-    "node_modules/npm-run-all/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+    "node_modules/npm-run-all/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "dev": true,
       "dependencies": {
-        "color-name": "1.1.3"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
-    "node_modules/npm-run-all/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-      "dev": true
-    },
     "node_modules/npm-run-all/node_modules/cross-spawn": {
       "version": "6.0.5",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
@@ -21628,13 +25749,16 @@
         "node": ">=4.8"
       }
     },
-    "node_modules/npm-run-all/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+    "node_modules/npm-run-all/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
       "engines": {
-        "node": ">=4"
+        "node": "*"
       }
     },
     "node_modules/npm-run-all/node_modules/path-key": {
@@ -21676,18 +25800,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/npm-run-all/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/npm-run-all/node_modules/which": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -21815,53 +25927,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/object-copy/node_modules/is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/object-copy/node_modules/is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/object-copy/node_modules/is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-      "dev": true,
-      "dependencies": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/object-copy/node_modules/is-descriptor/node_modules/kind-of": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-      "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/object-copy/node_modules/kind-of": {
       "version": "3.2.2",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
@@ -21875,9 +25940,9 @@
       }
     },
     "node_modules/object-inspect": {
-      "version": "1.12.3",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
-      "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==",
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+      "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -21984,9 +26049,9 @@
       "dev": true
     },
     "node_modules/on-finished": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-      "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==",
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+      "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
       "dev": true,
       "dependencies": {
         "ee-first": "1.1.1"
@@ -22070,40 +26135,6 @@
         "node": ">= 0.8.0"
       }
     },
-    "node_modules/optionator/node_modules/levn": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
-      "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==",
-      "dev": true,
-      "dependencies": {
-        "prelude-ls": "~1.1.2",
-        "type-check": "~0.3.2"
-      },
-      "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
-    "node_modules/optionator/node_modules/prelude-ls": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
-      "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
-    "node_modules/optionator/node_modules/type-check": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
-      "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==",
-      "dev": true,
-      "dependencies": {
-        "prelude-ls": "~1.1.2"
-      },
-      "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
     "node_modules/ora": {
       "version": "5.4.1",
       "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz",
@@ -22127,6 +26158,21 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/ora/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
     "node_modules/ora/node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -22143,6 +26189,45 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
+    "node_modules/ora/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/ora/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/ora/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/ora/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/os-browserify": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz",
@@ -22209,6 +26294,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/os-locale/node_modules/is-stream": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+      "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/os-locale/node_modules/npm-run-path": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
@@ -22340,15 +26434,15 @@
       }
     },
     "node_modules/p-limit": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
-      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
+      "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
       "dev": true,
       "dependencies": {
-        "yocto-queue": "^0.1.0"
+        "p-try": "^2.0.0"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=6"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
@@ -22366,30 +26460,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/p-locate/node_modules/p-limit": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
-      "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
-      "dev": true,
-      "dependencies": {
-        "p-try": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/p-locate/node_modules/p-try": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/p-map": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
@@ -22418,22 +26488,13 @@
         "node": ">=8"
       }
     },
-    "node_modules/p-retry/node_modules/retry": {
-      "version": "0.13.1",
-      "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
-      "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 4"
-      }
-    },
     "node_modules/p-try": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
-      "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
+      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
       "dev": true,
       "engines": {
-        "node": ">=4"
+        "node": ">=6"
       }
     },
     "node_modules/pacote": {
@@ -22588,9 +26649,9 @@
       }
     },
     "node_modules/parse5": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz",
-      "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
+      "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==",
       "dev": true
     },
     "node_modules/parse5-html-rewriting-stream": {
@@ -22607,32 +26668,19 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/parse5-html-rewriting-stream/node_modules/parse5": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
-      "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
-      "dev": true,
-      "dependencies": {
-        "entities": "^4.4.0"
-      },
-      "funding": {
-        "url": "https://github.com/inikulin/parse5?sponsor=1"
-      }
-    },
-    "node_modules/parse5-htmlparser2-tree-adapter": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz",
-      "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==",
+    "node_modules/parse5-html-rewriting-stream/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
       "dev": true,
-      "dependencies": {
-        "domhandler": "^5.0.2",
-        "parse5": "^7.0.0"
+      "engines": {
+        "node": ">=0.12"
       },
       "funding": {
-        "url": "https://github.com/inikulin/parse5?sponsor=1"
+        "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": {
+    "node_modules/parse5-html-rewriting-stream/node_modules/parse5": {
       "version": "7.1.2",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
       "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
@@ -22644,6 +26692,15 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
+    "node_modules/parse5-htmlparser2-tree-adapter": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz",
+      "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==",
+      "dev": true,
+      "dependencies": {
+        "parse5": "^6.0.1"
+      }
+    },
     "node_modules/parse5-sax-parser": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/parse5-sax-parser/-/parse5-sax-parser-7.0.0.tgz",
@@ -22656,6 +26713,18 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
+    "node_modules/parse5-sax-parser/node_modules/entities": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/parse5-sax-parser/node_modules/parse5": {
       "version": "7.1.2",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
@@ -22687,9 +26756,9 @@
       }
     },
     "node_modules/path-browserify": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz",
-      "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz",
+      "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==",
       "dev": true
     },
     "node_modules/path-exists": {
@@ -22772,18 +26841,18 @@
       }
     },
     "node_modules/path-scurry/node_modules/lru-cache": {
-      "version": "10.0.1",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz",
-      "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==",
+      "version": "10.0.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.3.tgz",
+      "integrity": "sha512-B7gr+F6MkqB3uzINHXNctGieGsRTMwIBgxkp0yq/5BwcuDzD4A8wQpHQW6vDAm1uKSLQghmRdD9sKqf2vJ1cEg==",
       "dev": true,
       "engines": {
         "node": "14 || >=16.14"
       }
     },
     "node_modules/path-scurry/node_modules/minipass": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "dev": true,
       "engines": {
         "node": ">=16 || 14 >=14.17"
@@ -22857,13 +26926,13 @@
       }
     },
     "node_modules/pdfmake": {
-      "version": "0.2.7",
-      "resolved": "https://registry.npmjs.org/pdfmake/-/pdfmake-0.2.7.tgz",
-      "integrity": "sha512-ClLpgx30H5G3EDvRW1MrA1Xih6YxEaSgIVFrOyBMgAAt62V+hxsyWAi6JNP7u1Fc5JKYAbpb4RRVw8Rhvmz5cQ==",
+      "version": "0.2.8",
+      "resolved": "https://registry.npmjs.org/pdfmake/-/pdfmake-0.2.8.tgz",
+      "integrity": "sha512-lI+amfIaUL8CrPhndxFdhIgMj9JB49Sj4DARltKC1gLm/5NsPohZqfB+D+II8HymtPB6eugUFD5oBxmzO57qHA==",
       "dev": true,
       "dependencies": {
         "@foliojs-fork/linebreak": "^1.1.1",
-        "@foliojs-fork/pdfkit": "^0.13.0",
+        "@foliojs-fork/pdfkit": "^0.14.0",
         "iconv-lite": "^0.6.3",
         "xmldoc": "^1.1.2"
       },
@@ -22925,12 +26994,13 @@
       }
     },
     "node_modules/pify": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-      "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
+      "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
       "dev": true,
+      "optional": true,
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=6"
       }
     },
     "node_modules/pirates": {
@@ -23108,20 +27178,6 @@
         "readable-stream": "^3.1.1"
       }
     },
-    "node_modules/postcss-html/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "dev": true,
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/postcss-less": {
       "version": "3.1.4",
       "resolved": "https://registry.npmjs.org/postcss-less/-/postcss-less-3.1.4.tgz",
@@ -23188,22 +27244,39 @@
         "webpack": "^5.0.0"
       }
     },
-    "node_modules/postcss-loader/node_modules/cosmiconfig": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
-      "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==",
+    "node_modules/postcss-loader/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
       "dev": true,
       "dependencies": {
-        "@types/parse-json": "^4.0.0",
-        "import-fresh": "^3.2.1",
-        "parse-json": "^5.0.0",
-        "path-type": "^4.0.0",
-        "yaml": "^1.10.0"
+        "yallist": "^4.0.0"
       },
       "engines": {
         "node": ">=10"
       }
     },
+    "node_modules/postcss-loader/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/postcss-loader/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/postcss-media-query-parser": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/postcss-media-query-parser/-/postcss-media-query-parser-0.2.3.tgz",
@@ -23473,9 +27546,9 @@
       }
     },
     "node_modules/postcss-value-parser": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz",
-      "integrity": "sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
+      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
       "dev": true
     },
     "node_modules/prebuild-install": {
@@ -23505,9 +27578,9 @@
       }
     },
     "node_modules/prelude-ls": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
-      "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
+      "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==",
       "dev": true,
       "engines": {
         "node": ">= 0.8.0"
@@ -23563,6 +27636,12 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
+    "node_modules/pretty-format/node_modules/react-is": {
+      "version": "18.2.0",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
+      "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
+      "dev": true
+    },
     "node_modules/pretty-quick": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/pretty-quick/-/pretty-quick-3.0.2.tgz",
@@ -23586,6 +27665,120 @@
         "prettier": ">=2.0.0"
       }
     },
+    "node_modules/pretty-quick/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/chalk": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/pretty-quick/node_modules/execa": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
+      "integrity": "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==",
+      "dev": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.0",
+        "get-stream": "^5.0.0",
+        "human-signals": "^1.1.1",
+        "is-stream": "^2.0.0",
+        "merge-stream": "^2.0.0",
+        "npm-run-path": "^4.0.0",
+        "onetime": "^5.1.0",
+        "signal-exit": "^3.0.2",
+        "strip-final-newline": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/get-stream": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+      "dev": true,
+      "dependencies": {
+        "pump": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/human-signals": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
+      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.12.0"
+      }
+    },
+    "node_modules/pretty-quick/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/prismjs": {
       "version": "1.29.0",
       "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
@@ -23654,6 +27847,15 @@
         "node": ">=10"
       }
     },
+    "node_modules/promise-retry/node_modules/retry": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
+      "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
+      "dev": true,
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/prompts": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -23677,11 +27879,6 @@
         "react-is": "^16.13.1"
       }
     },
-    "node_modules/prop-types/node_modules/react-is": {
-      "version": "16.13.1",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
-      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
-    },
     "node_modules/property-information": {
       "version": "5.6.0",
       "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz",
@@ -23733,11 +27930,19 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/proxy-addr/node_modules/ipaddr.js": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/proxy-from-env": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
-      "integrity": "sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A==",
-      "dev": true
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
     "node_modules/proxy-middleware": {
       "version": "0.15.0",
@@ -23849,15 +28054,18 @@
       }
     },
     "node_modules/punycode": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-      "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==",
-      "dev": true
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
     },
     "node_modules/pure-rand": {
-      "version": "6.0.3",
-      "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.0.3.tgz",
-      "integrity": "sha512-KddyFewCsO0j3+np81IQ+SweXLDnDQTs5s67BOnrYmYe/yNmUhttQyGsYzy8yUnoljGAQ9sl38YB4vH8ur7Y+w==",
+      "version": "6.0.4",
+      "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.0.4.tgz",
+      "integrity": "sha512-LA0Y9kxMYv47GIPJy6MI84fqTd2HmYZI83W/kM/SkKfDlajnZYfmXFTxkbY+xSBPkLJxltMa9hIkmdc29eguMA==",
       "dev": true,
       "funding": [
         {
@@ -23871,12 +28079,17 @@
       ]
     },
     "node_modules/qs": {
-      "version": "6.7.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
-      "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==",
-      "dev": true,
+      "version": "6.11.2",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
+      "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
+      "dependencies": {
+        "side-channel": "^1.0.4"
+      },
       "engines": {
         "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/querystring-es3": {
@@ -23937,9 +28150,9 @@
       }
     },
     "node_modules/ramda": {
-      "version": "0.29.0",
-      "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.29.0.tgz",
-      "integrity": "sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==",
+      "version": "0.29.1",
+      "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.29.1.tgz",
+      "integrity": "sha512-OfxIeWzd4xdUNxlWhgFazxsA/nl3mS4/jGZI5n00uWOoSSFRhC1b6gl6xvmzUamgmqELraWp0J/qqVlXYPDPyA==",
       "funding": {
         "type": "opencollective",
         "url": "https://opencollective.com/ramda"
@@ -24000,13 +28213,13 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
+      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
       "dev": true,
       "dependencies": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
       },
@@ -24014,6 +28227,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/raw-body/node_modules/bytes": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/rc": {
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
@@ -24029,21 +28251,6 @@
         "rc": "cli.js"
       }
     },
-    "node_modules/rc/node_modules/ini": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-      "optional": true
-    },
-    "node_modules/rc/node_modules/strip-json-comments": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-      "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
-      "optional": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/react": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz",
@@ -24128,10 +28335,9 @@
       }
     },
     "node_modules/react-is": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
-      "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
-      "dev": true
+      "version": "16.13.1",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
+      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
     },
     "node_modules/react-redux": {
       "version": "7.2.9",
@@ -24186,6 +28392,36 @@
         "readable-stream": "^2.0.2"
       }
     },
+    "node_modules/read-only-stream/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/read-only-stream/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/read-only-stream/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/read-package-json": {
       "version": "6.0.4",
       "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-6.0.4.tgz",
@@ -24223,23 +28459,14 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/read-package-json/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
-      "dev": true,
-      "dependencies": {
-        "balanced-match": "^1.0.0"
-      }
-    },
     "node_modules/read-package-json/node_modules/glob": {
-      "version": "10.3.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.7.tgz",
-      "integrity": "sha512-wCMbE1m9Nx5yD9LYtgsVWq5VhHlk5WzJirw594qZR6AIvQYuHrdDtIktUVjQItalD53y7dqoedu9xP0u0WaxIQ==",
+      "version": "10.3.10",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
+      "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
       "dev": true,
       "dependencies": {
         "foreground-child": "^3.1.0",
-        "jackspeak": "^2.0.3",
+        "jackspeak": "^2.3.5",
         "minimatch": "^9.0.1",
         "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0",
         "path-scurry": "^1.10.1"
@@ -24279,9 +28506,9 @@
       }
     },
     "node_modules/read-package-json/node_modules/minipass": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "dev": true,
       "engines": {
         "node": ">=16 || 14 >=14.17"
@@ -24427,33 +28654,16 @@
       }
     },
     "node_modules/readable-stream": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-      "dev": true,
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/readable-stream/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-      "dev": true
-    },
-    "node_modules/readable-stream/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dev": true,
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
       "dependencies": {
-        "safe-buffer": "~5.1.0"
+        "inherits": "^2.0.3",
+        "string_decoder": "^1.1.1",
+        "util-deprecate": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
       }
     },
     "node_modules/readdirp": {
@@ -24493,15 +28703,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/redent/node_modules/indent-string": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
-      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/redux": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/redux/-/redux-4.2.1.tgz",
@@ -24565,9 +28766,9 @@
       }
     },
     "node_modules/regenerator-runtime": {
-      "version": "0.13.11",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
-      "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz",
+      "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA=="
     },
     "node_modules/regenerator-transform": {
       "version": "0.15.2",
@@ -24720,6 +28921,14 @@
         "node": ">= 6.0.0"
       }
     },
+    "node_modules/remarkable/node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "dependencies": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
     "node_modules/repeat-element": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz",
@@ -24811,15 +29020,6 @@
         "request": "^2.34"
       }
     },
-    "node_modules/request-promise-native/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/request-promise-native/node_modules/tough-cookie": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
@@ -24833,49 +29033,18 @@
         "node": ">=0.8"
       }
     },
-    "node_modules/request/node_modules/core-util-is": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==",
-      "dev": true
-    },
-    "node_modules/request/node_modules/http-signature": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
-      "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==",
-      "dev": true,
-      "dependencies": {
-        "assert-plus": "^1.0.0",
-        "jsprim": "^1.2.2",
-        "sshpk": "^1.7.0"
-      },
-      "engines": {
-        "node": ">=0.8",
-        "npm": ">=1.3.7"
-      }
-    },
-    "node_modules/request/node_modules/jsprim": {
-      "version": "1.4.2",
-      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz",
-      "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==",
+    "node_modules/request/node_modules/form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
       "dev": true,
       "dependencies": {
-        "assert-plus": "1.0.0",
-        "extsprintf": "1.3.0",
-        "json-schema": "0.4.0",
-        "verror": "1.10.0"
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
       },
       "engines": {
-        "node": ">=0.6.0"
-      }
-    },
-    "node_modules/request/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
+        "node": ">= 0.12"
       }
     },
     "node_modules/request/node_modules/qs": {
@@ -24910,20 +29079,6 @@
         "uuid": "bin/uuid"
       }
     },
-    "node_modules/request/node_modules/verror": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
-      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
-      "dev": true,
-      "engines": [
-        "node >=0.6.0"
-      ],
-      "dependencies": {
-        "assert-plus": "^1.0.0",
-        "core-util-is": "1.0.2",
-        "extsprintf": "^1.2.0"
-      }
-    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -24958,12 +29113,12 @@
       "integrity": "sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ=="
     },
     "node_modules/resolve": {
-      "version": "1.22.1",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz",
-      "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==",
+      "version": "1.22.8",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
+      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
       "dev": true,
       "dependencies": {
-        "is-core-module": "^2.9.0",
+        "is-core-module": "^2.13.0",
         "path-parse": "^1.0.7",
         "supports-preserve-symlinks-flag": "^1.0.0"
       },
@@ -25085,9 +29240,9 @@
       }
     },
     "node_modules/retry": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
+      "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
       "dev": true,
       "engines": {
         "node": ">= 4"
@@ -25124,6 +29279,16 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/rimraf/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
     "node_modules/rimraf/node_modules/glob": {
       "version": "7.2.3",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -25144,6 +29309,18 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/rimraf/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/ripemd160": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz",
@@ -25347,12 +29524,17 @@
         }
       }
     },
-    "node_modules/sax": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
-      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
+    "node_modules/sass/node_modules/immutable": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.4.tgz",
+      "integrity": "sha512-fsXeu4J4i6WNWSikpI88v/PcVflZz+6kMhUfIwc5SY+poQRPnaf5V7qds6SUyUN3cVxEzuCab7QIoLOQ+DQ1wA==",
       "dev": true
     },
+    "node_modules/sax": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz",
+      "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA=="
+    },
     "node_modules/saxes": {
       "version": "3.1.11",
       "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz",
@@ -25418,52 +29600,28 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz",
       "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==",
-      "dev": true
-    },
-    "node_modules/selfsigned": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz",
-      "integrity": "sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==",
-      "dev": true,
-      "dependencies": {
-        "node-forge": "^1"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/semver": {
-      "version": "7.5.3",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
-      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
-      "devOptional": true,
-      "dependencies": {
-        "lru-cache": "^6.0.0"
-      },
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/semver/node_modules/lru-cache": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
-      "devOptional": true,
+      "dev": true
+    },
+    "node_modules/selfsigned": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz",
+      "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==",
+      "dev": true,
       "dependencies": {
-        "yallist": "^4.0.0"
+        "@types/node-forge": "^1.3.0",
+        "node-forge": "^1"
       },
       "engines": {
         "node": ">=10"
       }
     },
-    "node_modules/semver/node_modules/yallist": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-      "devOptional": true
+    "node_modules/semver": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "bin": {
+        "semver": "bin/semver.js"
+      }
     },
     "node_modules/send": {
       "version": "0.18.0",
@@ -25504,80 +29662,24 @@
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "dev": true
     },
-    "node_modules/send/node_modules/depd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
-      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/send/node_modules/http-errors": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
-      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
-      "dev": true,
-      "dependencies": {
-        "depd": "2.0.0",
-        "inherits": "2.0.4",
-        "setprototypeof": "1.2.0",
-        "statuses": "2.0.1",
-        "toidentifier": "1.0.1"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
     "node_modules/send/node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "dev": true
     },
-    "node_modules/send/node_modules/on-finished": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
-      "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
-      "dev": true,
+    "node_modules/serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
       "dependencies": {
-        "ee-first": "1.1.1"
+        "type-fest": "^0.20.2"
       },
       "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/send/node_modules/setprototypeof": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
-      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
-      "dev": true
-    },
-    "node_modules/send/node_modules/statuses": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
-      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/send/node_modules/toidentifier": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
-      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.6"
-      }
-    },
-    "node_modules/serialize-error": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-2.1.0.tgz",
-      "integrity": "sha512-ghgmKt5o4Tly5yEG/UJp8qTd0AN7Xalw4XBtDEKP655B699qMEtra1WlXeE6WIvdEG481JvRxULKsInq/iNysw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
     "node_modules/serialize-javascript": {
@@ -25616,6 +29718,15 @@
         "ms": "2.0.0"
       }
     },
+    "node_modules/serve-index/node_modules/depd": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+      "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/serve-index/node_modules/http-errors": {
       "version": "1.6.3",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
@@ -25649,67 +29760,25 @@
       "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==",
       "dev": true
     },
-    "node_modules/serve-static": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
-      "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
+    "node_modules/serve-index/node_modules/statuses": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
       "dev": true,
-      "dependencies": {
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "parseurl": "~1.3.3",
-        "send": "0.17.1"
-      },
       "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
-    "node_modules/serve-static/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.0.0"
+        "node": ">= 0.6"
       }
     },
-    "node_modules/serve-static/node_modules/debug/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/serve-static/node_modules/destroy": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
-      "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==",
-      "dev": true
-    },
-    "node_modules/serve-static/node_modules/ms": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
-      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
-      "dev": true
-    },
-    "node_modules/serve-static/node_modules/send": {
-      "version": "0.17.1",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
-      "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
+    "node_modules/serve-static": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
+      "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
       "dev": true,
       "dependencies": {
-        "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "destroy": "~1.0.4",
         "encodeurl": "~1.0.2",
         "escape-html": "~1.0.3",
-        "etag": "~1.8.1",
-        "fresh": "0.5.2",
-        "http-errors": "~1.7.2",
-        "mime": "1.6.0",
-        "ms": "2.1.1",
-        "on-finished": "~2.3.0",
-        "range-parser": "~1.2.1",
-        "statuses": "~1.5.0"
+        "parseurl": "~1.3.3",
+        "send": "0.18.0"
       },
       "engines": {
         "node": ">= 0.8.0"
@@ -25721,6 +29790,20 @@
       "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==",
       "dev": true
     },
+    "node_modules/set-function-length": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
+      "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==",
+      "dependencies": {
+        "define-data-property": "^1.1.1",
+        "get-intrinsic": "^1.2.1",
+        "gopd": "^1.0.1",
+        "has-property-descriptors": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/set-function-name": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz",
@@ -25762,19 +29845,27 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/set-value/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+    "node_modules/set-value/node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
       "dev": true,
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
       "engines": {
         "node": ">=0.10.0"
       }
     },
+    "node_modules/setimmediate": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+      "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
+    },
     "node_modules/setprototypeof": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
-      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
       "dev": true
     },
     "node_modules/sha.js": {
@@ -25903,6 +29994,76 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/sigstore/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/sigstore/node_modules/make-fetch-happen": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-11.1.1.tgz",
+      "integrity": "sha512-rLWS7GCSTcEujjVBs2YqG7Y4643u8ucvCJeSRqiLYhesrDuzeuFIk37xREzAsfQaqzl8b9rNCE4m6J8tvX4Q8w==",
+      "dev": true,
+      "dependencies": {
+        "agentkeepalive": "^4.2.1",
+        "cacache": "^17.0.0",
+        "http-cache-semantics": "^4.1.1",
+        "http-proxy-agent": "^5.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "is-lambda": "^1.0.1",
+        "lru-cache": "^7.7.1",
+        "minipass": "^5.0.0",
+        "minipass-fetch": "^3.0.0",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "negotiator": "^0.6.3",
+        "promise-retry": "^2.0.1",
+        "socks-proxy-agent": "^7.0.0",
+        "ssri": "^10.0.0"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/sigstore/node_modules/minipass": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+      "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/sigstore/node_modules/minipass-fetch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.4.tgz",
+      "integrity": "sha512-jHAqnA728uUpIaFm7NWsCnqKT6UqZz7GcI/bDpPATuwYyKwJwW0remxSCxUlKiEty+eopHGa3oc8WxgQ1FFJqg==",
+      "dev": true,
+      "dependencies": {
+        "minipass": "^7.0.3",
+        "minipass-sized": "^1.0.3",
+        "minizlib": "^2.1.2"
+      },
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      },
+      "optionalDependencies": {
+        "encoding": "^0.1.13"
+      }
+    },
+    "node_modules/sigstore/node_modules/minipass-fetch/node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
     "node_modules/simple-concat": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
@@ -25979,34 +30140,22 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
       "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
     },
-    "node_modules/simplebar/node_modules/core-js": {
-      "version": "3.32.2",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.32.2.tgz",
-      "integrity": "sha512-pxXSw1mYZPDGvTQqEc5vgIb83jGQKFGYWY76z4a7weZXUolw3G+OvpZqSRcfYOoOVUQJYEPsWeQK8pKEnUtWxQ==",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
-    "node_modules/simplebar/node_modules/lodash.memoize": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
-      "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag=="
-    },
     "node_modules/sisteransi": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
       "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
       "dev": true
     },
-    "node_modules/slash": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
-      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+    "node_modules/slash": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
+      "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
       }
     },
     "node_modules/slice-ansi": {
@@ -26023,6 +30172,39 @@
         "node": ">=8"
       }
     },
+    "node_modules/slice-ansi/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/slice-ansi/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/slice-ansi/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
     "node_modules/smart-buffer": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
@@ -26078,6 +30260,19 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/snapdragon-node/node_modules/is-descriptor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.3.tgz",
+      "integrity": "sha512-JCNNGbwWZEVaSPtS45mdtrneRWJFp07LLmykxeFV5F6oBvNF8vHSfJuJgoT472pSfk+Mf8VnlrspaFBHWM8JAw==",
+      "dev": true,
+      "dependencies": {
+        "is-accessor-descriptor": "^1.0.1",
+        "is-data-descriptor": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/snapdragon-util": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz",
@@ -26135,86 +30330,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/snapdragon/node_modules/is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/is-accessor-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
-      "dev": true,
-      "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/is-data-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
-      "dev": true,
-      "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-      "dev": true,
-      "dependencies": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/snapdragon/node_modules/kind-of": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-      "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/snapdragon/node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -26269,23 +30384,6 @@
         "node": ">= 10"
       }
     },
-    "node_modules/socks-proxy-agent/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.1.2"
-      },
-      "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/source-map": {
       "version": "0.7.4",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz",
@@ -26420,9 +30518,9 @@
       }
     },
     "node_modules/spdx-license-ids": {
-      "version": "3.0.15",
-      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.15.tgz",
-      "integrity": "sha512-lpT8hSQp9jAKp9mhtBU4Xjon8LPGBvLIuBiSVhMEtmLecTh2mO0tlqrAMp47tBXzMr13NJMQ2lf7RpQGLJ3HsQ==",
+      "version": "3.0.16",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.16.tgz",
+      "integrity": "sha512-eWN+LnM3GR6gPu35WxNgbGl8rmY1AEmoMDvL/QD6zYmPWgywxWqJWNdLGT+ke8dKNWrcYgYjPpG5gbTfghP8rw==",
       "dev": true
     },
     "node_modules/spdy": {
@@ -26455,20 +30553,6 @@
         "wbuf": "^1.7.3"
       }
     },
-    "node_modules/spdy-transport/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "dev": true,
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/specificity": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/specificity/-/specificity-0.4.1.tgz",
@@ -26508,9 +30592,9 @@
       "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
     },
     "node_modules/sshpk": {
-      "version": "1.17.0",
-      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz",
-      "integrity": "sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==",
+      "version": "1.18.0",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
+      "integrity": "sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==",
       "dev": true,
       "dependencies": {
         "asn1": "~0.2.3",
@@ -26545,9 +30629,9 @@
       }
     },
     "node_modules/ssri/node_modules/minipass": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "dev": true,
       "engines": {
         "node": ">=16 || 14 >=14.17"
@@ -26700,11 +30784,14 @@
         "node": "^8.12.0 || >=9.7.0"
       }
     },
-    "node_modules/start-server-and-test/node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+    "node_modules/start-server-and-test/node_modules/get-stream": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
       "dev": true,
+      "dependencies": {
+        "pump": "^3.0.0"
+      },
       "engines": {
         "node": ">=8"
       },
@@ -26712,6 +30799,15 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/start-server-and-test/node_modules/human-signals": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
+      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.12.0"
+      }
+    },
     "node_modules/start-server-and-test/node_modules/p-finally": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz",
@@ -26755,77 +30851,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/static-extend/node_modules/is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/static-extend/node_modules/is-accessor-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
-      "dev": true,
-      "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/static-extend/node_modules/is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
-      "dev": true,
-      "dependencies": {
-        "kind-of": "^3.0.2"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/static-extend/node_modules/is-data-descriptor/node_modules/kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
-      "dev": true,
-      "dependencies": {
-        "is-buffer": "^1.1.5"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/static-extend/node_modules/is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-      "dev": true,
-      "dependencies": {
-        "is-accessor-descriptor": "^0.1.6",
-        "is-data-descriptor": "^0.1.4",
-        "kind-of": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/static-extend/node_modules/kind-of": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-      "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/static-module": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/static-module/-/static-module-3.0.4.tgz",
@@ -26857,13 +30882,43 @@
         "sourcemap-codec": "^1.4.1"
       }
     },
+    "node_modules/static-module/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/static-module/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/static-module/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/statuses": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
-      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
       "dev": true,
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 0.8"
       }
     },
     "node_modules/stealthy-require": {
@@ -26876,13 +30931,12 @@
       }
     },
     "node_modules/stream-browserify": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
-      "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==",
-      "dev": true,
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",
+      "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==",
       "dependencies": {
-        "inherits": "~2.0.1",
-        "readable-stream": "^2.0.2"
+        "inherits": "~2.0.4",
+        "readable-stream": "^3.5.0"
       }
     },
     "node_modules/stream-combiner": {
@@ -26905,6 +30959,36 @@
         "readable-stream": "^2.0.2"
       }
     },
+    "node_modules/stream-combiner2/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/stream-combiner2/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/stream-combiner2/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/stream-http": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-3.2.0.tgz",
@@ -26917,20 +31001,6 @@
         "xtend": "^4.0.2"
       }
     },
-    "node_modules/stream-http/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "dev": true,
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/stream-splicer": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/stream-splicer/-/stream-splicer-2.0.1.tgz",
@@ -26941,19 +31011,40 @@
         "readable-stream": "^2.0.2"
       }
     },
-    "node_modules/streamsearch": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
-      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
-      "engines": {
-        "node": ">=10.0.0"
+    "node_modules/stream-splicer/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/stream-splicer/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/stream-splicer/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
       }
     },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
       "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
-      "devOptional": true,
       "dependencies": {
         "safe-buffer": "~5.2.0"
       }
@@ -27003,9 +31094,6 @@
         "emoji-regex": "^8.0.0",
         "is-fullwidth-code-point": "^3.0.0",
         "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
       }
     },
     "node_modules/string.prototype.padend": {
@@ -27089,18 +31177,15 @@
       "dev": true,
       "dependencies": {
         "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
       }
     },
     "node_modules/strip-bom": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz",
-      "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
       "dev": true,
       "engines": {
-        "node": ">=8"
+        "node": ">=4"
       }
     },
     "node_modules/strip-eof": {
@@ -27134,15 +31219,12 @@
       }
     },
     "node_modules/strip-json-comments": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
-      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
-      "dev": true,
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+      "optional": true,
       "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
+        "node": ">=0.10.0"
       }
     },
     "node_modules/strong-log-transformer": {
@@ -27321,6 +31403,21 @@
         "stylelint": "^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0 || ^12.0.0 || ^13.0.0"
       }
     },
+    "node_modules/stylelint/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
     "node_modules/stylelint/node_modules/autoprefixer": {
       "version": "9.8.8",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-9.8.8.tgz",
@@ -27365,38 +31462,23 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/stylelint/node_modules/cosmiconfig": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
-      "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==",
+    "node_modules/stylelint/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
       "dependencies": {
-        "@types/parse-json": "^4.0.0",
-        "import-fresh": "^3.2.1",
-        "parse-json": "^5.0.0",
-        "path-type": "^4.0.0",
-        "yaml": "^1.10.0"
+        "color-name": "~1.1.4"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=7.0.0"
       }
     },
-    "node_modules/stylelint/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.1.2"
-      },
-      "engines": {
-        "node": ">=6.0"
-      },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
-      }
+    "node_modules/stylelint/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
     },
     "node_modules/stylelint/node_modules/global-modules": {
       "version": "2.0.0",
@@ -27424,11 +31506,34 @@
         "node": ">=6"
       }
     },
-    "node_modules/stylelint/node_modules/ini": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-      "dev": true
+    "node_modules/stylelint/node_modules/globby": {
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz",
+      "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==",
+      "dev": true,
+      "dependencies": {
+        "array-union": "^2.1.0",
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.2.9",
+        "ignore": "^5.2.0",
+        "merge2": "^1.4.1",
+        "slash": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/stylelint/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
     },
     "node_modules/stylelint/node_modules/picocolors": {
       "version": "0.2.1",
@@ -27453,6 +31558,15 @@
         "url": "https://opencollective.com/postcss/"
       }
     },
+    "node_modules/stylelint/node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/stylelint/node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -27462,6 +31576,18 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/stylelint/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/stylelint/node_modules/which": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -27537,15 +31663,14 @@
       }
     },
     "node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dev": true,
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
       "dependencies": {
-        "has-flag": "^4.0.0"
+        "has-flag": "^3.0.0"
       },
       "engines": {
-        "node": ">=8"
+        "node": ">=4"
       }
     },
     "node_modules/supports-preserve-symlinks-flag": {
@@ -27567,79 +31692,26 @@
       "dev": true
     },
     "node_modules/swagger-client": {
-      "version": "3.22.3",
-      "resolved": "https://registry.npmjs.org/swagger-client/-/swagger-client-3.22.3.tgz",
-      "integrity": "sha512-9I3BGD/6LItBzvJoKaRZ+QQ7IcEKq+iVlvvvcfZz65WgnXkORM1uj5+M+Oa5d8Tu5qABuOXd1UnlClBPuTITBA==",
+      "version": "3.24.5",
+      "resolved": "https://registry.npmjs.org/swagger-client/-/swagger-client-3.24.5.tgz",
+      "integrity": "sha512-qb4Rr9LpWs7o2AO4KdiIK+dz0GbrRLyD+UyN24h6AcNcDUnwfkb6LgFE4e6bXwVXWJzMp27w1QvSQ4hQNMPnoQ==",
       "dependencies": {
         "@babel/runtime-corejs3": "^7.22.15",
-        "@swagger-api/apidom-core": ">=0.76.2 <1.0.0",
-        "@swagger-api/apidom-json-pointer": ">=0.76.2 <1.0.0",
-        "@swagger-api/apidom-ns-openapi-3-1": ">=0.76.2 <1.0.0",
-        "@swagger-api/apidom-reference": ">=0.76.2 <1.0.0",
+        "@swagger-api/apidom-core": ">=0.83.0 <1.0.0",
+        "@swagger-api/apidom-error": ">=0.83.0 <1.0.0",
+        "@swagger-api/apidom-json-pointer": ">=0.83.0 <1.0.0",
+        "@swagger-api/apidom-ns-openapi-3-1": ">=0.83.0 <1.0.0",
+        "@swagger-api/apidom-reference": ">=0.83.0 <1.0.0",
         "cookie": "~0.5.0",
         "deepmerge": "~4.3.0",
         "fast-json-patch": "^3.0.0-1",
         "is-plain-object": "^5.0.0",
         "js-yaml": "^4.1.0",
-        "node-abort-controller": "^3.1.1",
-        "node-fetch-commonjs": "^3.3.1",
-        "qs": "^6.10.2",
-        "traverse": "~0.6.6",
-        "undici": "^5.24.0"
-      }
-    },
-    "node_modules/swagger-client/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
-    },
-    "node_modules/swagger-client/node_modules/cookie": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
-      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/swagger-client/node_modules/deepmerge": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
-      "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/swagger-client/node_modules/is-plain-object": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
-      "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/swagger-client/node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
-      "dependencies": {
-        "argparse": "^2.0.1"
-      },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
-      }
-    },
-    "node_modules/swagger-client/node_modules/qs": {
-      "version": "6.11.2",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
-      "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
-      "dependencies": {
-        "side-channel": "^1.0.4"
-      },
-      "engines": {
-        "node": ">=0.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
+        "node-abort-controller": "^3.1.1",
+        "node-fetch-commonjs": "^3.3.1",
+        "qs": "^6.10.2",
+        "traverse": "~0.6.6",
+        "undici": "^5.24.0"
       }
     },
     "node_modules/swagger-ui": {
@@ -27684,55 +31756,6 @@
         "zenscroll": "^4.0.2"
       }
     },
-    "node_modules/swagger-ui/node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
-    },
-    "node_modules/swagger-ui/node_modules/immutable": {
-      "version": "3.8.2",
-      "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz",
-      "integrity": "sha512-15gZoQ38eYjEjxkorfbcgBKBL6R7T459OuK+CpcWt7O3KF4uPCx2tD0uFETlUDIyo+1789crbMhTvQBSR5yBMg==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/swagger-ui/node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
-      "dependencies": {
-        "argparse": "^2.0.1"
-      },
-      "bin": {
-        "js-yaml": "bin/js-yaml.js"
-      }
-    },
-    "node_modules/swagger-ui/node_modules/serialize-error": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
-      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
-      "dependencies": {
-        "type-fest": "^0.20.2"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/swagger-ui/node_modules/type-fest": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
-      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/symbol-observable": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-4.0.0.tgz",
@@ -27773,6 +31796,39 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/table/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/table/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/table/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
     "node_modules/table/node_modules/slice-ansi": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
@@ -27828,12 +31884,6 @@
         "tar-stream": "^2.1.4"
       }
     },
-    "node_modules/tar-fs/node_modules/chownr": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
-      "optional": true
-    },
     "node_modules/tar-stream": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
@@ -27850,18 +31900,13 @@
         "node": ">=6"
       }
     },
-    "node_modules/tar-stream/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "devOptional": true,
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
+    "node_modules/tar/node_modules/chownr": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+      "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+      "dev": true,
       "engines": {
-        "node": ">= 6"
+        "node": ">=10"
       }
     },
     "node_modules/tar/node_modules/fs-minipass": {
@@ -27897,18 +31942,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/tar/node_modules/mkdirp": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
-      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
-      "dev": true,
-      "bin": {
-        "mkdirp": "bin/cmd.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/tar/node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -27967,18 +32000,6 @@
         }
       }
     },
-    "node_modules/terser-webpack-plugin/node_modules/acorn": {
-      "version": "8.10.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
-      "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
-      "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
     "node_modules/terser-webpack-plugin/node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -28004,26 +32025,6 @@
         "ajv": "^6.9.1"
       }
     },
-    "node_modules/terser-webpack-plugin/node_modules/commander": {
-      "version": "2.20.3",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-      "dev": true
-    },
-    "node_modules/terser-webpack-plugin/node_modules/jest-worker": {
-      "version": "27.5.1",
-      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz",
-      "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==",
-      "dev": true,
-      "dependencies": {
-        "@types/node": "*",
-        "merge-stream": "^2.0.0",
-        "supports-color": "^8.0.0"
-      },
-      "engines": {
-        "node": ">= 10.13.0"
-      }
-    },
     "node_modules/terser-webpack-plugin/node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -28048,25 +32049,10 @@
         "url": "https://opencollective.com/webpack"
       }
     },
-    "node_modules/terser-webpack-plugin/node_modules/supports-color": {
-      "version": "8.1.1",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
-      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/supports-color?sponsor=1"
-      }
-    },
     "node_modules/terser-webpack-plugin/node_modules/terser": {
-      "version": "5.20.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.20.0.tgz",
-      "integrity": "sha512-e56ETryaQDyebBwJIWYB2TT6f2EZ0fL0sW/JRXNMN26zZdKi2u/E/5my5lG6jNxym6qsrVXfFRmOdV42zlAgLQ==",
+      "version": "5.24.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.24.0.tgz",
+      "integrity": "sha512-ZpGR4Hy3+wBEzVEnHvstMvqpD/nABNelQn/z2r0fjVWGQsN3bpOLzQlqDxmb4CDZnXq5lpjnQ+mHQLAOpfM5iw==",
       "dev": true,
       "dependencies": {
         "@jridgewell/source-map": "^0.3.3",
@@ -28081,24 +32067,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/terser/node_modules/acorn": {
-      "version": "8.10.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
-      "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
-      "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
-    "node_modules/terser/node_modules/commander": {
-      "version": "2.20.3",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-      "dev": true
-    },
     "node_modules/test-exclude": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
@@ -28113,6 +32081,16 @@
         "node": ">=8"
       }
     },
+    "node_modules/test-exclude/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
     "node_modules/test-exclude/node_modules/glob": {
       "version": "7.2.3",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -28133,6 +32111,18 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/test-exclude/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -28167,10 +32157,13 @@
       "dev": true
     },
     "node_modules/throttleit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz",
-      "integrity": "sha512-rkTVqu6IjfQ/6+uNuuc3sZek4CEYxTJom3IktzgdSxcZqdARuebbA/f4QmAxMQIxqq9ZLEUkSYqvuk1I6VKq4g==",
-      "dev": true
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.1.tgz",
+      "integrity": "sha512-vDZpf9Chs9mAdfY046mcPt8fg5QSZr37hEH4TXYBnDF+izxgrbRGUAAaBvIk/fJm9aOFCGFd1EsNg5AZCbnQCQ==",
+      "dev": true,
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
     },
     "node_modules/through": {
       "version": "2.3.8",
@@ -28188,6 +32181,36 @@
         "xtend": "~4.0.1"
       }
     },
+    "node_modules/through2/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/through2/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/through2/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/thunky": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",
@@ -28195,12 +32218,11 @@
       "dev": true
     },
     "node_modules/timers-browserify": {
-      "version": "1.4.2",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz",
-      "integrity": "sha512-PIxwAupJZiYU4JmVZYwXp9FKsHMXb5h0ZEFyuXTAn8WLHOlcij+FEcbrvDsom1o5dr1YggEtFbECvGCW2sT53Q==",
-      "dev": true,
+      "version": "2.0.12",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz",
+      "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==",
       "dependencies": {
-        "process": "~0.11.0"
+        "setimmediate": "^1.0.4"
       },
       "engines": {
         "node": ">=0.6.0"
@@ -28223,15 +32245,15 @@
       }
     },
     "node_modules/tmp": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
-      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
+      "version": "0.0.33",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
+      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
       "dev": true,
       "dependencies": {
-        "rimraf": "^3.0.0"
+        "os-tmpdir": "~1.0.2"
       },
       "engines": {
-        "node": ">=8.17.0"
+        "node": ">=0.6.0"
       }
     },
     "node_modules/tmpl": {
@@ -28305,9 +32327,9 @@
       "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ=="
     },
     "node_modules/toidentifier": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
-      "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
       "dev": true,
       "engines": {
         "node": ">=0.6"
@@ -28327,32 +32349,11 @@
         "node": ">=6"
       }
     },
-    "node_modules/tough-cookie/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/tr46": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
-      "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==",
-      "dev": true,
-      "dependencies": {
-        "punycode": "^2.1.0"
-      }
-    },
-    "node_modules/tr46/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "dev": true
     },
     "node_modules/transifex-i18ntool": {
       "version": "1.1.0",
@@ -28400,13 +32401,13 @@
       }
     },
     "node_modules/tree-sitter-json": {
-      "version": "0.20.0",
-      "resolved": "https://registry.npmjs.org/tree-sitter-json/-/tree-sitter-json-0.20.0.tgz",
-      "integrity": "sha512-PteOLH+Tx6Bz4ZA/d40/DbkiSXXRM/gKahhHI8hQ1lWNfFvdknnz9k3Mz84ol5srRyLboJ8wp8GSkhZ6ht9EGQ==",
+      "version": "0.20.1",
+      "resolved": "https://registry.npmjs.org/tree-sitter-json/-/tree-sitter-json-0.20.1.tgz",
+      "integrity": "sha512-482hf7J+aBwhksSw8yWaqI8nyP1DrSwnS4IMBShsnkFWD3SE8oalHnsEik59fEVi3orcTCUtMzSjZx+0Tpa6Vw==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
-        "nan": "^2.14.1"
+        "nan": "^2.18.0"
       }
     },
     "node_modules/tree-sitter-yaml": {
@@ -28481,21 +32482,39 @@
         }
       }
     },
-    "node_modules/ts-jest/node_modules/lodash.memoize": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
-      "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==",
-      "dev": true
+    "node_modules/ts-jest/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
     },
-    "node_modules/ts-jest/node_modules/yargs-parser": {
-      "version": "21.1.1",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
-      "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
+    "node_modules/ts-jest/node_modules/semver": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
       "engines": {
-        "node": ">=12"
+        "node": ">=10"
       }
     },
+    "node_modules/ts-jest/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/ts-morph": {
       "version": "13.0.3",
       "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-13.0.3.tgz",
@@ -28559,15 +32578,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/tsconfig-paths/node_modules/strip-bom": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/tslib": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz",
@@ -28614,21 +32624,74 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/tuf-js/node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+    "node_modules/tuf-js/node_modules/lru-cache": {
+      "version": "7.18.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+      "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+      "dev": true,
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/tuf-js/node_modules/make-fetch-happen": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-11.1.1.tgz",
+      "integrity": "sha512-rLWS7GCSTcEujjVBs2YqG7Y4643u8ucvCJeSRqiLYhesrDuzeuFIk37xREzAsfQaqzl8b9rNCE4m6J8tvX4Q8w==",
       "dev": true,
       "dependencies": {
-        "ms": "2.1.2"
+        "agentkeepalive": "^4.2.1",
+        "cacache": "^17.0.0",
+        "http-cache-semantics": "^4.1.1",
+        "http-proxy-agent": "^5.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "is-lambda": "^1.0.1",
+        "lru-cache": "^7.7.1",
+        "minipass": "^5.0.0",
+        "minipass-fetch": "^3.0.0",
+        "minipass-flush": "^1.0.5",
+        "minipass-pipeline": "^1.2.4",
+        "negotiator": "^0.6.3",
+        "promise-retry": "^2.0.1",
+        "socks-proxy-agent": "^7.0.0",
+        "ssri": "^10.0.0"
       },
       "engines": {
-        "node": ">=6.0"
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      }
+    },
+    "node_modules/tuf-js/node_modules/minipass": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+      "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/tuf-js/node_modules/minipass-fetch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.4.tgz",
+      "integrity": "sha512-jHAqnA728uUpIaFm7NWsCnqKT6UqZz7GcI/bDpPATuwYyKwJwW0remxSCxUlKiEty+eopHGa3oc8WxgQ1FFJqg==",
+      "dev": true,
+      "dependencies": {
+        "minipass": "^7.0.3",
+        "minipass-sized": "^1.0.3",
+        "minizlib": "^2.1.2"
       },
-      "peerDependenciesMeta": {
-        "supports-color": {
-          "optional": true
-        }
+      "engines": {
+        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+      },
+      "optionalDependencies": {
+        "encoding": "^0.1.13"
+      }
+    },
+    "node_modules/tuf-js/node_modules/minipass-fetch/node_modules/minipass": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
+      "dev": true,
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
       }
     },
     "node_modules/tunnel": {
@@ -28665,12 +32728,12 @@
       "dev": true
     },
     "node_modules/type-check": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
-      "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
+      "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==",
       "dev": true,
       "dependencies": {
-        "prelude-ls": "^1.2.1"
+        "prelude-ls": "~1.1.2"
       },
       "engines": {
         "node": ">= 0.8.0"
@@ -28686,10 +32749,9 @@
       }
     },
     "node_modules/type-fest": {
-      "version": "0.21.3",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
-      "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
-      "dev": true,
+      "version": "0.20.2",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+      "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
       "engines": {
         "node": ">=10"
       },
@@ -28797,9 +32859,9 @@
       }
     },
     "node_modules/types-ramda": {
-      "version": "0.29.4",
-      "resolved": "https://registry.npmjs.org/types-ramda/-/types-ramda-0.29.4.tgz",
-      "integrity": "sha512-XO/820iRsCDwqLjE8XE+b57cVGPyk1h+U9lBGpDWvbEky+NQChvHVwaKM05WnW1c5z3EVQh8NhXFmh2E/1YazQ==",
+      "version": "0.29.6",
+      "resolved": "https://registry.npmjs.org/types-ramda/-/types-ramda-0.29.6.tgz",
+      "integrity": "sha512-VJoOk1uYNh9ZguGd3eZvqkdhD4hTGtnjRBUx5Zc0U9ftmnCgiWcSj/lsahzKunbiwRje1MxxNkEy1UdcXRCpYw==",
       "dependencies": {
         "ts-toolbelt": "^9.6.0"
       }
@@ -28886,11 +32948,11 @@
       "dev": true
     },
     "node_modules/undici": {
-      "version": "5.25.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.25.2.tgz",
-      "integrity": "sha512-tch8RbCfn1UUH1PeVCXva4V8gDpGAud/w0WubD6sHC46vYQ3KDxL+xv1A2UxK0N6jrVedutuPHxe1XIoqerwMw==",
+      "version": "5.27.2",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.2.tgz",
+      "integrity": "sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==",
       "dependencies": {
-        "busboy": "^1.6.0"
+        "@fastify/busboy": "^2.0.0"
       },
       "engines": {
         "node": ">=14.0"
@@ -29027,15 +33089,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/union-value/node_modules/is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/unique-filename": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz",
@@ -29097,9 +33150,9 @@
       }
     },
     "node_modules/universalify": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
-      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
       "dev": true,
       "engines": {
         "node": ">= 10.0.0"
@@ -29226,15 +33279,6 @@
         "punycode": "^2.1.0"
       }
     },
-    "node_modules/uri-js/node_modules/punycode": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
-      "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
-      "dev": true,
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/urix": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
@@ -29261,20 +33305,11 @@
         "requires-port": "^1.0.0"
       }
     },
-    "node_modules/url/node_modules/qs": {
-      "version": "6.11.2",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
-      "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
-      "dev": true,
-      "dependencies": {
-        "side-channel": "^1.0.4"
-      },
-      "engines": {
-        "node": ">=0.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
+    "node_modules/url/node_modules/punycode": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+      "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==",
+      "dev": true
     },
     "node_modules/use": {
       "version": "3.1.1",
@@ -29303,8 +33338,7 @@
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-      "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
-      "devOptional": true
+      "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
     "node_modules/util/node_modules/inherits": {
       "version": "2.0.3",
@@ -29331,25 +33365,31 @@
       }
     },
     "node_modules/v8-compile-cache": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz",
-      "integrity": "sha512-ocyWc3bAHBB/guyqJQVI5o4BZkPhznPYUG2ea80Gond/BgNWpap8TOmLSeeQG7bnh2KMISxskdADG59j7zruhw==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
+      "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
       "dev": true
     },
     "node_modules/v8-to-istanbul": {
-      "version": "9.1.0",
-      "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz",
-      "integrity": "sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA==",
+      "version": "9.1.3",
+      "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.3.tgz",
+      "integrity": "sha512-9lDD+EVI2fjFsMWXc6dy5JJzBsVTcQ2fVkfBvncZ6xJWG9wtBhOldG+mHkSL0+V1K/xgZz0JDO5UT5hFwHUghg==",
       "dev": true,
       "dependencies": {
         "@jridgewell/trace-mapping": "^0.3.12",
         "@types/istanbul-lib-coverage": "^2.0.1",
-        "convert-source-map": "^1.6.0"
+        "convert-source-map": "^2.0.0"
       },
       "engines": {
         "node": ">=10.12.0"
       }
     },
+    "node_modules/v8-to-istanbul/node_modules/convert-source-map": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
+      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
+      "dev": true
+    },
     "node_modules/validate-npm-package-license": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",
@@ -29382,17 +33422,17 @@
       }
     },
     "node_modules/verror": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.1.tgz",
-      "integrity": "sha512-veufcmxri4e3XSrT0xwfUR7kguIkaxBeosDg00yDWhk49wdwkSUrvvsm7nc75e1PUyvIeZj6nS8VQRYz2/S4Xg==",
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
       "dev": true,
+      "engines": [
+        "node >=0.6.0"
+      ],
       "dependencies": {
         "assert-plus": "^1.0.0",
         "core-util-is": "1.0.2",
         "extsprintf": "^1.2.0"
-      },
-      "engines": {
-        "node": ">=0.6.0"
       }
     },
     "node_modules/verror/node_modules/core-util-is": {
@@ -29481,6 +33521,12 @@
         "xml-name-validator": "^3.0.0"
       }
     },
+    "node_modules/w3c-xmlserializer/node_modules/webidl-conversions": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
+      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "dev": true
+    },
     "node_modules/wait-on": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-5.3.0.tgz",
@@ -29500,6 +33546,15 @@
         "node": ">=8.9.0"
       }
     },
+    "node_modules/wait-on/node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
     "node_modules/walker": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz",
@@ -29530,6 +33585,25 @@
         "node": ">= 8.10.0"
       }
     },
+    "node_modules/watchify/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/watchify/node_modules/browser-resolve": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz",
+      "integrity": "sha512-7sWsQlYL2rGLy2IWm8WL8DCTJvYLc/qlOnsakDac87SOoCd16WLsaAMdCiAqsTNHIe+SXfaqyxyo6THoWqs8WQ==",
+      "dev": true,
+      "dependencies": {
+        "resolve": "^1.17.0"
+      }
+    },
     "node_modules/watchify/node_modules/browserify": {
       "version": "17.0.0",
       "resolved": "https://registry.npmjs.org/browserify/-/browserify-17.0.0.tgz",
@@ -29602,13 +33676,14 @@
         "xtend": "~4.0.1"
       }
     },
-    "node_modules/watchify/node_modules/events": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
-      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+    "node_modules/watchify/node_modules/buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz",
+      "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==",
       "dev": true,
-      "engines": {
-        "node": ">=0.8.x"
+      "dependencies": {
+        "base64-js": "^1.0.2",
+        "ieee754": "^1.1.4"
       }
     },
     "node_modules/watchify/node_modules/glob": {
@@ -29631,36 +33706,54 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/watchify/node_modules/path-browserify": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz",
-      "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==",
-      "dev": true
-    },
-    "node_modules/watchify/node_modules/stream-browserify": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",
-      "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==",
+    "node_modules/watchify/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "dependencies": {
-        "inherits": "~2.0.4",
-        "readable-stream": "^3.5.0"
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
       }
     },
-    "node_modules/watchify/node_modules/stream-browserify/node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+    "node_modules/watchify/node_modules/punycode": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+      "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==",
+      "dev": true
+    },
+    "node_modules/watchify/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/watchify/node_modules/readable-stream/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
       "dev": true,
       "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
+        "safe-buffer": "~5.1.0"
       }
     },
+    "node_modules/watchify/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
     "node_modules/watchify/node_modules/through2": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz",
@@ -29684,6 +33777,18 @@
         "node": ">= 6"
       }
     },
+    "node_modules/watchify/node_modules/timers-browserify": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz",
+      "integrity": "sha512-PIxwAupJZiYU4JmVZYwXp9FKsHMXb5h0ZEFyuXTAn8WLHOlcij+FEcbrvDsom1o5dr1YggEtFbECvGCW2sT53Q==",
+      "dev": true,
+      "dependencies": {
+        "process": "~0.11.0"
+      },
+      "engines": {
+        "node": ">=0.6.0"
+      }
+    },
     "node_modules/watchify/node_modules/util": {
       "version": "0.12.5",
       "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",
@@ -29743,9 +33848,9 @@
       "optional": true
     },
     "node_modules/webidl-conversions": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
-      "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
       "dev": true
     },
     "node_modules/webpack": {
@@ -29818,27 +33923,6 @@
         "webpack": "^5.0.0"
       }
     },
-    "node_modules/webpack-dev-middleware/node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/webpack-dev-middleware/node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
     "node_modules/webpack-dev-server": {
       "version": "4.11.1",
       "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz",
@@ -29868,295 +33952,30 @@
         "p-retry": "^4.5.0",
         "rimraf": "^3.0.2",
         "schema-utils": "^4.0.0",
-        "selfsigned": "^2.1.1",
-        "serve-index": "^1.9.1",
-        "sockjs": "^0.3.24",
-        "spdy": "^4.0.2",
-        "webpack-dev-middleware": "^5.3.1",
-        "ws": "^8.4.2"
-      },
-      "bin": {
-        "webpack-dev-server": "bin/webpack-dev-server.js"
-      },
-      "engines": {
-        "node": ">= 12.13.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/webpack"
-      },
-      "peerDependencies": {
-        "webpack": "^4.37.0 || ^5.0.0"
-      },
-      "peerDependenciesMeta": {
-        "webpack-cli": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/body-parser": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
-      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
-      "dev": true,
-      "dependencies": {
-        "bytes": "3.1.2",
-        "content-type": "~1.0.4",
-        "debug": "2.6.9",
-        "depd": "2.0.0",
-        "destroy": "1.2.0",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "on-finished": "2.4.1",
-        "qs": "6.11.0",
-        "raw-body": "2.5.1",
-        "type-is": "~1.6.18",
-        "unpipe": "1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8",
-        "npm": "1.2.8000 || >= 1.4.16"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/bytes": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
-      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/content-disposition": {
-      "version": "0.5.4",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
-      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
-      "dev": true,
-      "dependencies": {
-        "safe-buffer": "5.2.1"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/cookie": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
-      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.0.0"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/depd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
-      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/express": {
-      "version": "4.18.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
-      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
-      "dev": true,
-      "dependencies": {
-        "accepts": "~1.3.8",
-        "array-flatten": "1.1.1",
-        "body-parser": "1.20.1",
-        "content-disposition": "0.5.4",
-        "content-type": "~1.0.4",
-        "cookie": "0.5.0",
-        "cookie-signature": "1.0.6",
-        "debug": "2.6.9",
-        "depd": "2.0.0",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "etag": "~1.8.1",
-        "finalhandler": "1.2.0",
-        "fresh": "0.5.2",
-        "http-errors": "2.0.0",
-        "merge-descriptors": "1.0.1",
-        "methods": "~1.1.2",
-        "on-finished": "2.4.1",
-        "parseurl": "~1.3.3",
-        "path-to-regexp": "0.1.7",
-        "proxy-addr": "~2.0.7",
-        "qs": "6.11.0",
-        "range-parser": "~1.2.1",
-        "safe-buffer": "5.2.1",
-        "send": "0.18.0",
-        "serve-static": "1.15.0",
-        "setprototypeof": "1.2.0",
-        "statuses": "2.0.1",
-        "type-is": "~1.6.18",
-        "utils-merge": "1.0.1",
-        "vary": "~1.1.2"
-      },
-      "engines": {
-        "node": ">= 0.10.0"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/finalhandler": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
-      "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
-      "dev": true,
-      "dependencies": {
-        "debug": "2.6.9",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "on-finished": "2.4.1",
-        "parseurl": "~1.3.3",
-        "statuses": "2.0.1",
-        "unpipe": "~1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/http-errors": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
-      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
-      "dev": true,
-      "dependencies": {
-        "depd": "2.0.0",
-        "inherits": "2.0.4",
-        "setprototypeof": "1.2.0",
-        "statuses": "2.0.1",
-        "toidentifier": "1.0.1"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/ipaddr.js": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz",
-      "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/webpack-dev-server/node_modules/on-finished": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
-      "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
-      "dev": true,
-      "dependencies": {
-        "ee-first": "1.1.1"
+        "selfsigned": "^2.1.1",
+        "serve-index": "^1.9.1",
+        "sockjs": "^0.3.24",
+        "spdy": "^4.0.2",
+        "webpack-dev-middleware": "^5.3.1",
+        "ws": "^8.4.2"
       },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/qs": {
-      "version": "6.11.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
-      "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
-      "dev": true,
-      "dependencies": {
-        "side-channel": "^1.0.4"
+      "bin": {
+        "webpack-dev-server": "bin/webpack-dev-server.js"
       },
       "engines": {
-        "node": ">=0.6"
+        "node": ">= 12.13.0"
       },
       "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/raw-body": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
-      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
-      "dev": true,
-      "dependencies": {
-        "bytes": "3.1.2",
-        "http-errors": "2.0.0",
-        "iconv-lite": "0.4.24",
-        "unpipe": "1.0.0"
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
       },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/serve-static": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
-      "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
-      "dev": true,
-      "dependencies": {
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "parseurl": "~1.3.3",
-        "send": "0.18.0"
+      "peerDependencies": {
+        "webpack": "^4.37.0 || ^5.0.0"
       },
-      "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/setprototypeof": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
-      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
-      "dev": true
-    },
-    "node_modules/webpack-dev-server/node_modules/statuses": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
-      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
-    "node_modules/webpack-dev-server/node_modules/toidentifier": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
-      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.6"
+      "peerDependenciesMeta": {
+        "webpack-cli": {
+          "optional": true
+        }
       }
     },
     "node_modules/webpack-dev-server/node_modules/webpack-dev-middleware": {
@@ -30182,27 +34001,6 @@
         "webpack": "^4.0.0 || ^5.0.0"
       }
     },
-    "node_modules/webpack-dev-server/node_modules/ws": {
-      "version": "8.14.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
-      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
-      "dev": true,
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/webpack-merge": {
       "version": "5.8.0",
       "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz",
@@ -30246,18 +34044,6 @@
         }
       }
     },
-    "node_modules/webpack/node_modules/acorn": {
-      "version": "8.10.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
-      "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
-      "dev": true,
-      "bin": {
-        "acorn": "bin/acorn"
-      },
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
     "node_modules/webpack/node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -30283,15 +34069,6 @@
         "ajv": "^6.9.1"
       }
     },
-    "node_modules/webpack/node_modules/events": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
-      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.8.x"
-      }
-    },
     "node_modules/webpack/node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
@@ -30361,14 +34138,13 @@
       "dev": true
     },
     "node_modules/whatwg-url": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
-      "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
       "dev": true,
       "dependencies": {
-        "lodash.sortby": "^4.7.0",
-        "tr46": "^1.0.1",
-        "webidl-conversions": "^4.0.2"
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
       }
     },
     "node_modules/which": {
@@ -30409,13 +34185,13 @@
       "dev": true
     },
     "node_modules/which-typed-array": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz",
-      "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==",
+      "version": "1.1.13",
+      "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz",
+      "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==",
       "dev": true,
       "dependencies": {
         "available-typed-arrays": "^1.0.5",
-        "call-bind": "^1.0.2",
+        "call-bind": "^1.0.4",
         "for-each": "^0.3.3",
         "gopd": "^1.0.1",
         "has-tostringtag": "^1.0.0"
@@ -30457,6 +34233,53 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/windows-release/node_modules/execa": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-4.1.0.tgz",
+      "integrity": "sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==",
+      "dev": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.0",
+        "get-stream": "^5.0.0",
+        "human-signals": "^1.1.1",
+        "is-stream": "^2.0.0",
+        "merge-stream": "^2.0.0",
+        "npm-run-path": "^4.0.0",
+        "onetime": "^5.1.0",
+        "signal-exit": "^3.0.2",
+        "strip-final-newline": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+      }
+    },
+    "node_modules/windows-release/node_modules/get-stream": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
+      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+      "dev": true,
+      "dependencies": {
+        "pump": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/windows-release/node_modules/human-signals": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
+      "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.12.0"
+      }
+    },
     "node_modules/word-wrap": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@@ -30498,14 +34321,71 @@
         "ansi-styles": "^4.0.0",
         "string-width": "^4.1.0",
         "strip-ansi": "^6.0.0"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^2.0.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=8"
       },
       "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
       }
     },
+    "node_modules/wrap-ansi-cjs/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true
+    },
+    "node_modules/wrap-ansi/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/wrap-ansi/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -30525,16 +34405,16 @@
       }
     },
     "node_modules/ws": {
-      "version": "7.5.9",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
-      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+      "version": "8.14.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
+      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
       "dev": true,
       "engines": {
-        "node": ">=8.3.0"
+        "node": ">=10.0.0"
       },
       "peerDependencies": {
         "bufferutil": "^4.0.1",
-        "utf-8-validate": "^5.0.2"
+        "utf-8-validate": ">=5.0.2"
       },
       "peerDependenciesMeta": {
         "bufferutil": {
@@ -30588,6 +34468,26 @@
       "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==",
       "dev": true
     },
+    "node_modules/xml2js": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz",
+      "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==",
+      "dependencies": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      },
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
     "node_modules/xmlchars": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
@@ -30634,9 +34534,9 @@
       }
     },
     "node_modules/yargs": {
-      "version": "17.6.2",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz",
-      "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==",
+      "version": "17.7.2",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
+      "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
       "dependencies": {
         "cliui": "^8.0.1",
         "escalade": "^3.1.1",
@@ -30651,15 +34551,6 @@
       }
     },
     "node_modules/yargs-parser": {
-      "version": "20.0.0",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.0.0.tgz",
-      "integrity": "sha512-8eblPHTL7ZWRkyjIZJjnGf+TijiKJSwA24svzLRVvtgoi/RZiKa9fFQTrlx0OKLnyHSdt/enrdadji6WFfESVA==",
-      "dev": true,
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/yargs/node_modules/yargs-parser": {
       "version": "21.1.1",
       "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
       "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
diff --git a/src/pybind/mgr/dashboard/frontend/package.json b/src/pybind/mgr/dashboard/frontend/package.json
index 6e19fff3aaa8..fd4f7a0bff6b 100644
--- a/src/pybind/mgr/dashboard/frontend/package.json
+++ b/src/pybind/mgr/dashboard/frontend/package.json
@@ -76,8 +76,11 @@
     "ngx-toastr": "17.0.2",
     "rxjs": "6.6.3",
     "simplebar-angular": "2.3.6",
+    "stream-browserify": "3.0.0",
     "swagger-ui": "4.12.0",
+    "timers-browserify": "2.0.12",
     "tslib": "2.3.1",
+    "xml2js": "0.6.2",
     "zone.js": "0.11.8"
   },
   "devDependencies": {
@@ -99,6 +102,7 @@
     "@types/lodash": "4.14.161",
     "@types/node": "18.17.12",
     "@types/swagger-ui": "3.52.0",
+    "@types/xml2js": "0.4.14",
     "@typescript-eslint/eslint-plugin": "5.27.1",
     "@typescript-eslint/parser": "5.27.1",
     "axe-core": "4.4.3",
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
index 1732a7fb3b4b..a68ea5661e20 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.html
@@ -130,6 +130,34 @@
                   class="bold w-25">Bucket policy</td>
               <td><pre>{{ selection.bucket_policy | json}}</pre></td>
             </tr>
+            <tr>
+              <td i18n
+                  class="bold w-25">ACL</td>
+              <td>
+                <table class="table">
+                  <thead>
+                    <tr i18n>
+                      <th>Grantee</th>
+                      <th>Permissions</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    <tr i18n>
+                      <td>Bucket Owner</td>
+                      <td>{{ aclPermissions.Owner || '-'}}</td>
+                    </tr>
+                    <tr i18n>
+                      <td>Everyone</td>
+                      <td>{{ aclPermissions.AllUsers || '-'}}</td>
+                    </tr>
+                    <tr i18n>
+                      <td>Authenticated users group</td>
+                      <td>{{ aclPermissions.AuthenticatedUsers || '-'}}</td>
+                    </tr>
+                  </tbody>
+                </table>
+              </td>
+            </tr>
           </tbody>
         </table>
       </ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
index d731a323818e..451a7dd38111 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-details/rgw-bucket-details.component.ts
@@ -2,6 +2,8 @@ import { Component, Input, OnChanges } from '@angular/core';
 
 import { RgwBucketService } from '~/app/shared/api/rgw-bucket.service';
 
+import * as xml2js from 'xml2js';
+
 @Component({
   selector: 'cd-rgw-bucket-details',
   templateUrl: './rgw-bucket-details.component.html',
@@ -11,6 +13,8 @@ export class RgwBucketDetailsComponent implements OnChanges {
   @Input()
   selection: any;
 
+  aclPermissions: Record<string, string[]> = {};
+
   constructor(private rgwBucketService: RgwBucketService) {}
 
   ngOnChanges() {
@@ -18,8 +22,43 @@ export class RgwBucketDetailsComponent implements OnChanges {
       this.rgwBucketService.get(this.selection.bid).subscribe((bucket: object) => {
         bucket['lock_retention_period_days'] = this.rgwBucketService.getLockDays(bucket);
         this.selection = bucket;
-        this.selection.bucket_policy = JSON.parse(this.selection.bucket_policy) || {};
+        this.aclPermissions = this.parseXmlAcl(this.selection.acl, this.selection.owner);
       });
     }
   }
+
+  parseXmlAcl(xml: any, bucketOwner: string): Record<string, string[]> {
+    const parser = new xml2js.Parser({ explicitArray: false, trim: true });
+    let data: Record<string, string[]> = {
+      Owner: ['-'],
+      AllUsers: ['-'],
+      AuthenticatedUsers: ['-']
+    };
+    parser.parseString(xml, (err, result) => {
+      if (err) return null;
+
+      const xmlGrantees: any = result['AccessControlPolicy']['AccessControlList']['Grant'];
+      if (Array.isArray(xmlGrantees)) {
+        for (let i = 0; i < xmlGrantees.length; i++) {
+          const grantee = xmlGrantees[i];
+          if (grantee?.Grantee?.URI) {
+            const granteeGroup = grantee.Grantee.URI.split('/').pop();
+            if (data[granteeGroup].includes('-')) {
+              data[granteeGroup] = [grantee?.Permission];
+            } else {
+              data[granteeGroup].push(grantee?.Permission);
+            }
+          }
+          if (grantee?.Grantee?.ID && bucketOwner === grantee?.Grantee?.ID) {
+            data['Owner'] = grantee?.Permission;
+          }
+        }
+      } else {
+        if (xmlGrantees?.Grantee?.ID && bucketOwner === xmlGrantees?.Grantee?.ID) {
+          data['Owner'] = xmlGrantees?.Permission;
+        }
+      }
+    });
+    return data;
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-acl-permissions.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-acl-permissions.enum.ts
new file mode 100644
index 000000000000..a392f4ab103e
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-acl-permissions.enum.ts
@@ -0,0 +1,14 @@
+export enum RgwBucketAclPermissions {
+  Read = 'Read',
+  Write = 'Write',
+  All = 'Read and write',
+  FullControl = 'Full control'
+}
+
+export enum RgwBucketAclGrantee {
+  Owner = 'Owner',
+  Everyone = 'Everyone',
+  AuthenticatedUsers = 'Authenticated Users'
+}
+
+export type AclPermissionsType = RgwBucketAclPermissions[keyof RgwBucketAclPermissions];
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index 77cd69934dc2..fd085a74d2d0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -417,6 +417,8 @@
         <div class="row">
           <div class="col-12">
             <div class="form-group row">
+
+              <!-- Bucket policy -->
               <label i18n
                      class="cd-col-form-label"
                      for="id">Bucket policy</label>
@@ -460,6 +462,55 @@
                 </div>
               </div>
             </div>
+
+            <div class="form-group row">
+
+              <!-- ACL -->
+              <label class="cd-col-form-label"
+                     i18n>ACL
+                <cd-helper>Any changes to the ACL will overwrite previous one.
+                  You can choose any of the available options to modify the spcified user group.</cd-helper>
+              </label>
+              <div class="cd-col-form-input">
+                <div class="input-group">
+                  <span class="input-group-text"
+                        for="grantee"
+                        i18n>Grantee
+                    <cd-helper>Select a grantee (user group) to modify it's permisions</cd-helper>
+                  </span>
+                  <select id="grantee"
+                          name="grantee"
+                          class="form-input form-select"
+                          formControlName="grantee"
+                          (change)="onSelectionFilter()">
+                    <option *ngFor="let item of grantees"
+                            [value]="item"
+                            i18n>{{ item }}</option>
+                  </select>
+                  <span class="invalid-feedback"
+                        *ngIf="bucketForm.showError('grantee', frm, 'required')"
+                        i18n>This field is required.</span>
+                  <span class="input-group-text"
+                        for="aclPermission"
+                        i18n>Permissions
+                    <cd-helper>Select the permision to give to the selected grantee.
+                        Regardless, the owner of the bucket will always have
+                          FULL CONTROL access</cd-helper>
+                    </span>
+                  <select id="aclPermission"
+                          name="aclPermission"
+                          class="form-input form-select"
+                          formControlName="aclPermission">
+                    <option *ngFor="let permission of aclPermissions"
+                            [value]="permission"
+                            i18n>{{ permission }}</option>
+                  </select>
+                  <span class="invalid-feedback"
+                        *ngIf="bucketForm.showError('aclPermission', frm, 'required')"
+                        i18n>This field is required.</span>
+                </div>
+              </div>
+            </div>
           </div>
         </div>
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index e5267e357c1c..3b3a50e88f82 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -11,6 +11,7 @@ import { ActivatedRoute, Router } from '@angular/router';
 
 import _ from 'lodash';
 import { forkJoin } from 'rxjs';
+import * as xml2js from 'xml2js';
 
 import { RgwBucketService } from '~/app/shared/api/rgw-bucket.service';
 import { RgwSiteService } from '~/app/shared/api/rgw-site.service';
@@ -26,6 +27,11 @@ import { ModalService } from '~/app/shared/services/modal.service';
 import { NotificationService } from '~/app/shared/services/notification.service';
 import { RgwBucketEncryptionModel } from '../models/rgw-bucket-encryption';
 import { RgwBucketMfaDelete } from '../models/rgw-bucket-mfa-delete';
+import {
+  AclPermissionsType,
+  RgwBucketAclPermissions as aclPermission,
+  RgwBucketAclGrantee as Grantee
+} from './rgw-bucket-acl-permissions.enum';
 import { RgwBucketVersioning } from '../models/rgw-bucket-versioning';
 import { RgwConfigModalComponent } from '../rgw-config-modal/rgw-config-modal.component';
 import { BucketTagModalComponent } from '../bucket-tag-modal/bucket-tag-modal.component';
@@ -63,6 +69,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
       attribute: 'value'
     }
   ];
+  grantees: string[] = [Grantee.Owner, Grantee.Everyone, Grantee.AuthenticatedUsers];
+  aclPermissions: AclPermissionsType[] = [aclPermission.FullControl];
 
   get isVersioningEnabled(): boolean {
     return this.bucketForm.getValue('versioning');
@@ -143,7 +151,9 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
       ],
       lock_mode: ['COMPLIANCE'],
       lock_retention_period_days: [0, [CdValidators.number(false), lockDaysValidator]],
-      bucket_policy: ['{}', CdValidators.json()]
+      bucket_policy: ['{}', CdValidators.json()],
+      grantee: [Grantee.Owner, [Validators.required]],
+      aclPermission: [[aclPermission.FullControl], [Validators.required]]
     });
   }
 
@@ -223,6 +233,12 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           // Append default values.
           value = _.merge(defaults, value);
           // Update the form.
+          if (this.editing) {
+            [value['grantee'], value['aclPermission']] = this.aclXmlToFormValues(
+              bidResp['acl'],
+              bidResp['owner']
+            );
+          }
           this.bucketForm.setValue(value);
           if (this.editing) {
             this.isVersioningAlreadyEnabled = this.isVersioningEnabled;
@@ -236,6 +252,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
                 .get('bucket_policy')
                 .setValue(JSON.stringify(value['bucket_policy'], null, 2));
             }
+            this.filterAclPermissions();
           }
         }
         this.loadingReady();
@@ -260,6 +277,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     const values = this.bucketForm.value;
     const xmlStrTags = this.tagsToXML(this.tags);
     const bucketPolicy = this.getBucketPolicy();
+    const cannedAcl = this.permissionToCannedAcl();
+
     if (this.editing) {
       // Edit
       const versioning = this.getVersioningStatus();
@@ -279,7 +298,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['lock_mode'],
           values['lock_retention_period_days'],
           xmlStrTags,
-          bucketPolicy
+          bucketPolicy,
+          cannedAcl
         )
         .subscribe(
           () => {
@@ -309,7 +329,8 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
           values['encryption_type'],
           values['keyId'],
           xmlStrTags,
-          bucketPolicy
+          bucketPolicy,
+          cannedAcl
         )
         .subscribe(
           () => {
@@ -446,4 +467,81 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     xml += '</TagSet></Tagging>';
     return xml;
   }
+
+  aclXmlToFormValues(xml: any, bucketOwner: string): [Grantee, AclPermissionsType] {
+    const parser = new xml2js.Parser({ explicitArray: false, trim: true });
+    let selectedAclPermission: AclPermissionsType = aclPermission.FullControl;
+    let selectedGrantee: Grantee = Grantee.Owner;
+    parser.parseString(xml, (err, result) => {
+      if (err) return null;
+
+      const xmlGrantees: any = result['AccessControlPolicy']['AccessControlList']['Grant'];
+      for (let i = 0; i < xmlGrantees.length; i++) {
+        if (xmlGrantees[i]['Grantee']['ID'] === bucketOwner) continue;
+        if (
+          xmlGrantees[i]['Grantee']['URI'] &&
+          xmlGrantees[i]['Grantee']['URI'].includes('AllUsers')
+        ) {
+          selectedGrantee = Grantee.Everyone;
+          if (
+            xmlGrantees[i]['Permission'] === 'READ' &&
+            selectedAclPermission !== aclPermission.Write
+          ) {
+            selectedAclPermission = aclPermission.Read;
+          } else if (
+            xmlGrantees[i]['Permission'] === ' WRITE' &&
+            selectedAclPermission !== aclPermission.Read
+          ) {
+            selectedAclPermission = aclPermission.Write;
+          } else {
+            selectedAclPermission = aclPermission.All;
+          }
+        } else if (
+          xmlGrantees[i]['Grantee']['URI'] &&
+          xmlGrantees[i]['Grantee']['URI'].includes('AuthenticatedUsers')
+        ) {
+          selectedGrantee = Grantee.AuthenticatedUsers;
+          selectedAclPermission = aclPermission.Read;
+        }
+      }
+    });
+    return [selectedGrantee, selectedAclPermission];
+  }
+
+  /*
+   Set the selector's options to the available options depending
+   on the selected Grantee and reset it's value
+   */
+  onSelectionFilter() {
+    this.filterAclPermissions();
+    this.bucketForm.get('aclPermission').setValue(this.aclPermissions[0]);
+  }
+
+  filterAclPermissions() {
+    const selectedGrantee: Grantee = this.bucketForm.get('grantee').value;
+    switch (selectedGrantee) {
+      case Grantee.Owner:
+        this.aclPermissions = [aclPermission.FullControl];
+        break;
+      case Grantee.Everyone:
+        this.aclPermissions = [aclPermission.Read, aclPermission.All];
+        break;
+      case Grantee.AuthenticatedUsers:
+        this.aclPermissions = [aclPermission.Read];
+        break;
+    }
+  }
+
+  permissionToCannedAcl(): string {
+    const selectedGrantee: Grantee = this.bucketForm.get('grantee').value;
+    const selectedAclPermission = this.bucketForm.get('aclPermission').value;
+    switch (selectedGrantee) {
+      case Grantee.Everyone:
+        return selectedAclPermission === aclPermission.Read ? 'public-read' : 'public-read-write';
+      case Grantee.AuthenticatedUsers:
+        return 'authenticated-read';
+      default:
+        return 'private';
+    }
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
index e1002373806c..f40fd17cfc08 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
@@ -61,11 +61,12 @@ describe('RgwBucketService', () => {
         'aws:kms',
         'qwerty1',
         null,
-        null
+        null,
+        'private'
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&bucket_policy=null&${RgwHelper.DAEMON_QUERY_PARAM}`
+      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&bucket_policy=null&canned_acl=private&${RgwHelper.DAEMON_QUERY_PARAM}`
     );
     expect(req.request.method).toBe('POST');
   });
@@ -86,11 +87,12 @@ describe('RgwBucketService', () => {
         'GOVERNANCE',
         '10',
         null,
-        null
+        null,
+        'private'
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10&tags=null&bucket_policy=null`
+      `api/rgw/bucket/foo?${RgwHelper.DAEMON_QUERY_PARAM}&bucket_id=bar&uid=baz&versioning_state=Enabled&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&mfa_delete=Enabled&mfa_token_serial=1&mfa_token_pin=223344&lock_mode=GOVERNANCE&lock_retention_period_days=10&tags=null&bucket_policy=null&canned_acl=private`
     );
     expect(req.request.method).toBe('PUT');
   });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
index a43b2616f933..e8e123bf9f25 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
@@ -61,7 +61,8 @@ export class RgwBucketService extends ApiClient {
     encryption_type: string,
     key_id: string,
     tags: string,
-    bucketPolicy: string
+    bucketPolicy: string,
+    cannedAcl: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       return this.http.post(this.url, null, {
@@ -79,6 +80,7 @@ export class RgwBucketService extends ApiClient {
             key_id,
             tags: tags,
             bucket_policy: bucketPolicy,
+            canned_acl: cannedAcl,
             daemon_name: params.get('daemon_name')
           }
         })
@@ -100,7 +102,8 @@ export class RgwBucketService extends ApiClient {
     lockMode: 'GOVERNANCE' | 'COMPLIANCE',
     lockRetentionPeriodDays: string,
     tags: string,
-    bucketPolicy: string
+    bucketPolicy: string,
+    cannedAcl: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
       params = params.appendAll({
@@ -116,7 +119,8 @@ export class RgwBucketService extends ApiClient {
         lock_mode: lockMode,
         lock_retention_period_days: lockRetentionPeriodDays,
         tags: tags,
-        bucket_policy: bucketPolicy
+        bucket_policy: bucketPolicy,
+        canned_acl: cannedAcl
       });
       return this.http.put(`${this.url}/${bucket}`, null, { params: params });
     });
diff --git a/src/pybind/mgr/dashboard/frontend/tsconfig.json b/src/pybind/mgr/dashboard/frontend/tsconfig.json
index e0cf323fd54f..dd2162a1f722 100644
--- a/src/pybind/mgr/dashboard/frontend/tsconfig.json
+++ b/src/pybind/mgr/dashboard/frontend/tsconfig.json
@@ -22,7 +22,13 @@
     "baseUrl": "./",
     "resolveJsonModule": true,
     "paths": {
-      "~/*": ["src/*"]
+      "~/*": ["src/*"],
+      "timers": [
+          "node_modules/timers-browserify"
+        ],
+        "stream": [
+          "node_modules/stream-browserify"
+        ]
     },
     "typeRoots": ["node_modules/@types"],
     "lib": ["es2017", "dom"],
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 55a38701eb6d..1c1da60895f5 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -9059,6 +9059,8 @@ paths:
                   type: string
                 bucket_policy:
                   type: string
+                canned_acl:
+                  type: string
                 daemon_name:
                   type: string
                 encryption_state:
@@ -9374,6 +9376,8 @@ paths:
                   type: string
                 bucket_policy:
                   type: string
+                canned_acl:
+                  type: string
                 daemon_name:
                   type: string
                 encryption_state:
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index 77e3bf5d7b16..ea79c86c3fe8 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -658,6 +658,30 @@ def set_bucket_versioning(self, bucket_name, versioning_state, mfa_delete,
                                      http_status_code=error.status_code,
                                      component='rgw')
 
+    @RestClient.api_get('/{bucket_name}?acl')
+    def get_acl(self, bucket_name, request=None):
+        # pylint: disable=unused-argument
+        try:
+            result = request(raw_content=True)  # type: ignore
+            return result.decode("utf-8")
+        except RequestException as error:
+            msg = 'Error getting ACLs'
+            if error.status_code == 404:
+                msg = '{}: {}'.format(msg, str(error))
+            raise DashboardException(msg=msg,
+                                     http_status_code=error.status_code,
+                                     component='rgw')
+
+    @RestClient.api_put('/{bucket_name}?acl')
+    def set_acl(self, bucket_name, acl, request=None):
+        # pylint: disable=unused-argument
+        headers = {'x-amz-acl': acl}
+        try:
+            result = request(headers=headers)  # type: ignore
+        except RequestException as e:
+            raise DashboardException(msg=str(e), component='rgw')
+        return result
+
     @RestClient.api_get('/{bucket_name}?encryption')
     def get_bucket_encryption(self, bucket_name, request=None):
         # pylint: disable=unused-argument

From 8f2ee228af0fa0723328c0cf5c7e85906be8e287 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 13 Feb 2024 12:02:27 +0000
Subject: [PATCH 1883/2492] osd/SnapMapper: fix _lookup_purged_snap

We should verify that the snap we are looking for matches the
pool also.
Same as we are checking in OSDMonitor:
https://github.com/ceph/ceph/pull/28865/commits/0a48392ce066471233cc1e81e957b2999b9c411c

Fixes: https://tracker.ceph.com/issues/64347

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/osd/SnapMapper.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index e79bc5d450a2..1d3bfa95b9a6 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -764,6 +764,10 @@ int SnapMapper::_lookup_purged_snap(
   decode(gotpool, p);
   decode(*begin, p);
   decode(*end, p);
+  if (gotpool != pool) {
+    dout(20) << __func__ << " got wrong pool " << gotpool << dendl;
+    return -ENOENT;
+  }
   if (snap < *begin || snap >= *end) {
     dout(20) << __func__ << " pool " << pool << " snap " << snap
 	     << " found [" << *begin << "," << *end << "), no overlap" << dendl;

From 5442f7eb218ff9024365f3a28e4affb5add1ee1e Mon Sep 17 00:00:00 2001
From: Mykola Golub <mykola.golub@clyso.com>
Date: Sun, 11 Feb 2024 09:43:30 +0000
Subject: [PATCH 1884/2492] tools/rbd: make 'children' command support
 --image-id

Fixes: https://tracker.ceph.com/issues/64376
Signed-off-by: Mykola Golub <mykola.golub@clyso.com>
---
 PendingReleaseNotes              |  2 ++
 qa/workunits/rbd/cli_generic.sh  |  5 +++++
 src/test/cli/rbd/help.t          |  7 ++++---
 src/tools/rbd/action/Children.cc | 21 +++++++++++++++++----
 4 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index a413b6276f2f..2e41462a1eb6 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -133,6 +133,8 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   notifications to topics owned by other users. A new configuration parameter:
   ``rgw_topic_require_publish_policy`` can be enabled to deny ``sns:Publish``
   permissions unless explicitly granted by topic policy.
+* RBD: The option ``--image-id`` has been added to `rbd children` CLI command,
+  so it can be run for images in the trash.
 
 >=18.0.0
 
diff --git a/qa/workunits/rbd/cli_generic.sh b/qa/workunits/rbd/cli_generic.sh
index c35bbe8f83e6..15c47074db5f 100755
--- a/qa/workunits/rbd/cli_generic.sh
+++ b/qa/workunits/rbd/cli_generic.sh
@@ -432,6 +432,7 @@ test_trash() {
     rbd trash mv test2
     ID=`rbd trash ls | cut -d ' ' -f 1`
     rbd info --image-id $ID | grep "rbd image 'test2'"
+    rbd children --image-id $ID | wc -l | grep 0
 
     rbd trash restore $ID
     rbd ls | grep test2
@@ -449,6 +450,7 @@ test_trash() {
     rbd create $RBD_CREATE_ARGS -s 1 test1
     rbd snap create test1@snap1
     rbd snap protect test1@snap1
+    rbd clone test1@snap1 clone
     rbd trash mv test1
 
     rbd trash ls | grep test1
@@ -459,7 +461,10 @@ test_trash() {
     ID=`rbd trash ls | cut -d ' ' -f 1`
     rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 1
     rbd snap ls --image-id $ID | grep '.*snap1.*'
+    rbd children --image-id $ID | wc -l | grep 1
+    rbd children --image-id $ID | grep 'clone'
 
+    rbd rm clone
     rbd snap unprotect --image-id $ID --snap snap1
     rbd snap rm --image-id $ID --snap snap1
     rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 0
diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index 2d1c901462c7..866bd8f11c85 100644
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -195,9 +195,9 @@
   
   rbd help children
   usage: rbd children [--pool <pool>] [--namespace <namespace>] 
-                      [--image <image>] [--snap <snap>] [--snap-id <snap-id>] 
-                      [--all] [--descendants] [--format <format>] 
-                      [--pretty-format] 
+                      [--image <image>] [--snap <snap>] [--image-id <image-id>] 
+                      [--snap-id <snap-id>] [--all] [--descendants] 
+                      [--format <format>] [--pretty-format] 
                       <image-or-snap-spec> 
   
   Display children of an image or its snapshot.
@@ -212,6 +212,7 @@
     --namespace arg       namespace name
     --image arg           image name
     --snap arg            snapshot name
+    --image-id arg        image id
     --snap-id arg         snapshot id
     -a [ --all ]          list all children (include trash)
     --descendants         include all descendants
diff --git a/src/tools/rbd/action/Children.cc b/src/tools/rbd/action/Children.cc
index 58e861b6928b..6881989abb25 100644
--- a/src/tools/rbd/action/Children.cc
+++ b/src/tools/rbd/action/Children.cc
@@ -84,6 +84,7 @@ void get_arguments(po::options_description *positional,
                    po::options_description *options) {
   at::add_image_or_snap_spec_options(positional, options,
                                      at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
   at::add_snap_id_option(options);
   options->add_options()
     ("all,a", po::bool_switch(), "list all children (include trash)");
@@ -104,14 +105,26 @@ int execute(const po::variables_map &vm,
   std::string namespace_name;
   std::string image_name;
   std::string snap_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
   int r = utils::get_pool_image_snapshot_names(
     vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
-    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
-    utils::SPEC_VALIDATION_NONE);
+    &image_name, &snap_name, image_id.empty(),
+    utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_NONE);
   if (r < 0) {
     return r;
   }
 
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id."
+              << std::endl;
+    return -EINVAL;
+  }
+
   if (snap_id != LIBRADOS_SNAP_HEAD && !snap_name.empty()) {
     std::cerr << "rbd: trying to access snapshot using both name and id."
               << std::endl;
@@ -127,8 +140,8 @@ int execute(const po::variables_map &vm,
   librados::Rados rados;
   librados::IoCtx io_ctx;
   librbd::Image image;
-  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
-                                 true, &rados, &io_ctx, &image);
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name,
+				 image_id, "", true, &rados, &io_ctx, &image);
   if (r < 0) {
     return r;
   }

From 8fca9cdb4d95f54449a87cedddacf3167eba744d Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Wed, 7 Feb 2024 17:56:40 -0500
Subject: [PATCH 1885/2492] rgw/multisite: add x-amx-replicated-at header to
 replicated objects

fixes: https://tracker.ceph.com/issues/64365
Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 8 ++++++++
 src/rgw/rgw_common.h              | 1 +
 src/rgw/rgw_rest_s3.cc            | 9 +++++++++
 3 files changed, 18 insertions(+)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b802bb114bd7..714e7702b1a0 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -21,6 +21,7 @@
 #include "common/Formatter.h"
 #include "common/Throttle.h"
 #include "common/BackTrace.h"
+#include "common/ceph_time.h"
 
 #include "rgw_sal.h"
 #include "rgw_zone.h"
@@ -4412,6 +4413,13 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     encode(trace, bl);
     cb.get_attrs()[RGW_ATTR_OBJ_REPLICATION_TRACE] = std::move(bl);
   }
+  {
+    // add x-amz-replicated-at
+    bufferlist bl;
+    ceph::real_time timestamp = real_clock::now();
+    encode(timestamp, bl);
+    cb.get_attrs()[RGW_ATTR_OBJ_REPLICATION_TIMESTAMP] = std::move(bl);
+  }
 
   if (source_zone.empty()) {
     set_copy_attrs(cb.get_attrs(), attrs, attrs_mod);
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index ef9aa5420558..f4ab07f97de1 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -145,6 +145,7 @@ using ceph::crypto::MD5;
 
 #define RGW_ATTR_OBJ_REPLICATION_STATUS RGW_ATTR_PREFIX "amz-replication-status"
 #define RGW_ATTR_OBJ_REPLICATION_TRACE RGW_ATTR_PREFIX "replication-trace"
+#define RGW_ATTR_OBJ_REPLICATION_TIMESTAMP RGW_ATTR_PREFIX "replicated-at"
 
 /* IAM Policy */
 #define RGW_ATTR_IAM_POLICY	RGW_ATTR_PREFIX "iam-policy"
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 2117af94d70d..60cba8a17f2d 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -463,6 +463,15 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
       }
     } catch (const buffer::error&) {} // omit x-rgw-replicated-from headers
   }
+  if (auto i = attrs.find(RGW_ATTR_OBJ_REPLICATION_TIMESTAMP);
+      i != attrs.end()) {
+    try {
+      ceph::real_time replicated_time;
+      decode(replicated_time, i->second);
+      dump_time(s, "x-rgw-replicated-at", replicated_time);
+    } catch (const buffer::error&) {}
+  }
+
   if (multipart_parts_count) {
     dump_header(s, "x-amz-mp-parts-count", *multipart_parts_count);
   }

From 574421aeec3bdd502c64e87102846e5181b18415 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 14 Feb 2024 03:04:26 +1000
Subject: [PATCH 1886/2492] doc/cephadm: correct nfs config pool name

s/nfs-ganesha/.nfs/ in doc/cephadm/services/nfs, in accordance with
information given to me by John Mulligan over Slack.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/cephadm/services/nfs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephadm/services/nfs.rst b/doc/cephadm/services/nfs.rst
index 2f12c591631f..ab616ddcb130 100644
--- a/doc/cephadm/services/nfs.rst
+++ b/doc/cephadm/services/nfs.rst
@@ -15,7 +15,7 @@ Deploying NFS ganesha
 =====================
 
 Cephadm deploys NFS Ganesha daemon (or set of daemons).  The configuration for
-NFS is stored in the ``nfs-ganesha`` pool and exports are managed via the
+NFS is stored in the ``.nfs`` pool and exports are managed via the
 ``ceph nfs export ...`` commands and via the dashboard.
 
 To deploy a NFS Ganesha gateway, run the following command:

From 5afa3fc52f088d03ddd4ab42c968815af69160cb Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 27 Nov 2023 12:00:43 -0500
Subject: [PATCH 1887/2492] rgw: cumulatively fix 6 AWS SigV4 request failure
 cases

These changes address checksum header identification and signing
algorithm selection, including checksum trailer verification
for signed- and unsigned-payload cases.

These changes address all the actual S3 request failures I have
so far been able to reproduce, with and without content checksums
and/or new trailing checksum headers, and with and without
SSL.

Fixes: https://tracker.ceph.com/issues/63153

Specifically, it fixes the request failures that motivated the
initial tracker filing.  It extracts but does not validate new client
content checksums if present.  Validation and management of new
S3 content-checksum headers will follow in a subsequent change.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

squashed commits:

* wip chunk meta parsing--seem to have first AWSv4ComplMulti::ChunkMeta::create_next sort of parsing
* use constexpr sarlen(...) for static array lengths throughout rgw_auth_s3.cc
* link AWSv4CompleMulti::ChunkMeta to its enclosing completer
* capture original content-length header before AWSv4ComplMulti overwrites it
* mostly extract the trailer
* fix misordered content-length, experiment w/exbuf
* save leftover bytes between calls to AWSv4ComplMulti::recv_chunk()
* propagate data_offset_in_stream from AWSv4ComplMulti::recv_chunk()
* clean up trailer section extract
* trailer section cleanup and introduce extract_helper
* unrolled checksum extract--fixup
* fix sv_trailer end pos, and cleanup
* add proplist interface to rgw::auth::Completer and AWSv4ComplMulti
* spliterate trailers
* check completer props
* redefine prop_map to point into already-allocated trailer_vec
* hax: thread a counter onto AWSv4ComplMulti recv_body() and recv_chunk path
* fix apparent bug where due to reads less than chunk_size induce a final, zero-length read that was skipped before forcing recognition of the last chunk in the stream
* check only for a trailing checksum named in x-amz-trailer
* don't try to match signatures when no signature provided (because streaming unsigned)
* oops, fix content_length decl
* fix recognition of next chunk envelope in unsigned aws-chunk case
* clean up AWSv4CompMulti flags and correctly detect aws unsigned chunked
* rework checksum-trailer extraction and introduce AWSv4ComplMulti::calc_v4_trailing_signature
* thread const struct req_state* into AWSv4ComplMulti
* large cleanup of trailer parsing, no regression
* fix trailer signature calculation--checks
* correctly generate final chunk hmac
* typo in comment
* verify trailing signature when expected (using expected final chunk signature)
* move trailer_vec back onto recv_body()'s stack
* remove strange completer comment
* remove last_frag (now points into parsing_buf)
* remove implied dependency on content_length
* move trailer recognition to AWSv4ComplMulti::complete()
* remove now-unused is_last_chunk() predicate
* remove unused ChunkMeta::completer
* responses to review comments
* when trailer is sig expected, fail (only) if none present or if it does not match calculated
* remove stale parse_content_length(...) decl
* remove now-unused AWSv4ComplMulti::content_length
* fix extract_helper end search position as in mut_extract_helper
* change "\n" reserve term in get_canon_amz_hdrs() part of the sum (review)
    and initialize length to 0
* remove debugging code

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_auth.h     |   2 +
 src/rgw/rgw_auth_s3.cc | 498 ++++++++++++++++++++++++++++++++++-------
 src/rgw/rgw_auth_s3.h  | 134 +++++++++--
 src/rgw/rgw_common.h   |  16 +-
 src/rgw/rgw_file_int.h |   2 +-
 src/rgw/rgw_rest.cc    |  19 +-
 src/rgw/rgw_rest.h     |  19 +-
 src/rgw/rgw_rest_s3.cc |  59 +++--
 8 files changed, 604 insertions(+), 145 deletions(-)

diff --git a/src/rgw/rgw_auth.h b/src/rgw/rgw_auth.h
index 2668af3a3c1d..e2de0afb7265 100644
--- a/src/rgw/rgw_auth.h
+++ b/src/rgw/rgw_auth.h
@@ -10,6 +10,8 @@
 #include <system_error>
 #include <utility>
 
+#include "include/function2.hpp"
+
 #include "rgw_common.h"
 #include "rgw_web_idp.h"
 
diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 6f0141ef9415..765d19bfa689 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include <algorithm>
+#include <boost/algorithm/string/predicate.hpp>
 #include <map>
 #include <iterator>
 #include <string>
@@ -10,6 +11,7 @@
 
 #include "common/armor.h"
 #include "common/utf8.h"
+#include "common/split.h"
 #include "rgw_rest_s3.h"
 #include "rgw_auth_s3.h"
 #include "rgw_common.h"
@@ -61,12 +63,19 @@ static const auto signed_subresources = {
  * ?get the canonical amazon-style header for something?
  */
 
+template<typename M>
 static std::string
-get_canon_amz_hdr(const meta_map_t& meta_map)
+get_canon_amz_hdrs(const M& map)
 {
-  std::string dest;
-
-  for (const auto& kv : meta_map) {
+  size_t length = 0;
+  std::string dest; // why dest?
+  std::for_each(map.begin(), map.end(), [&length] (const auto& elt) -> void {
+    length += elt.first.size() + sarrlen(":") + elt.second.size() +
+      sarrlen("\n");
+  });
+  dest.reserve(length);
+
+  for (const auto& kv : map) {
     dest.append(kv.first);
     dest.append(":");
     dest.append(kv.second);
@@ -74,7 +83,7 @@ get_canon_amz_hdr(const meta_map_t& meta_map)
   }
 
   return dest;
-}
+} /* get_canon_amz_hdrs */
 
 /*
  * ?get the canonical representation of the object's location
@@ -152,8 +161,8 @@ void rgw_create_s3_canonical_header(
   }
   dest.append("\n");
 
-  dest.append(get_canon_amz_hdr(meta_map));
-  dest.append(get_canon_amz_hdr(qs_map));
+  dest.append(get_canon_amz_hdrs(meta_map));
+  dest.append(get_canon_amz_hdrs(qs_map));
   dest.append(get_canon_resource(dpp, request_uri, sub_resources));
 
   dest_str = dest;
@@ -604,13 +613,13 @@ std::string get_v4_canonical_qs(const req_info& info, const bool using_qs)
   auto iter = std::begin(canonical_qs_map);
   std::string canonical_qs;
   canonical_qs.append(iter->first)
-              .append("=", ::strlen("="))
+              .append("=", sarrlen("="))
               .append(iter->second);
 
   for (iter++; iter != std::end(canonical_qs_map); iter++) {
-    canonical_qs.append("&", ::strlen("&"))
+    canonical_qs.append("&", sarrlen("&"))
                 .append(iter->first)
-                .append("=", ::strlen("="))
+                .append("=", sarrlen("="))
                 .append(iter->second);
   }
 
@@ -647,13 +656,13 @@ std::string gen_v4_canonical_qs(const req_info& info, bool is_non_s3_op)
   auto iter = std::begin(canonical_qs_map);
   std::string canonical_qs;
   canonical_qs.append(iter->first)
-              .append("=", ::strlen("="))
+              .append("=", sarrlen("="))
               .append(iter->second);
 
   for (iter++; iter != std::end(canonical_qs_map); iter++) {
-    canonical_qs.append("&", ::strlen("&"))
+    canonical_qs.append("&", sarrlen("&"))
                 .append(iter->first)
-                .append("=", ::strlen("="))
+                .append("=", sarrlen("="))
                 .append(iter->second);
   }
 
@@ -700,8 +709,9 @@ get_v4_canonical_headers(const req_info& info,
     /* TODO(rzarzynski): we'd like to switch to sstring here but it should
      * get push_back() and reserve() first. */
     std::string token_env = "HTTP_";
-    token_env.reserve(token.length() + std::strlen("HTTP_") + 1);
+    token_env.reserve(token.length() + sarrlen("HTTP_") + 1);
 
+    /* XXX can we please stop doing this? */
     std::transform(std::begin(token), std::end(token),
                    std::back_inserter(token_env), [](const int c) {
                      return c == '-' ? '_' : c == '_' ? '-' : std::toupper(c);
@@ -733,11 +743,11 @@ get_v4_canonical_headers(const req_info& info,
 
       if (!secure_port.empty()) {
 	if (secure_port != "443")
-	  token_value.append(":", std::strlen(":"))
+	  token_value.append(":", sarrlen(":"))
                      .append(secure_port.data(), secure_port.length());
       } else if (!port.empty()) {
 	if (port != "80")
-	  token_value.append(":", std::strlen(":"))
+	  token_value.append(":", sarrlen(":"))
                      .append(port.data(), port.length());
       }
     }
@@ -752,9 +762,9 @@ get_v4_canonical_headers(const req_info& info,
     boost::trim_all<std::string>(value);
 
     canonical_hdrs.append(name.data(), name.length())
-                  .append(":", std::strlen(":"))
+                  .append(":", sarrlen(":"))
                   .append(value)
-                  .append("\n", std::strlen("\n"));
+                  .append("\n", sarrlen("\n"));
   }
   return canonical_hdrs;
 }
@@ -812,9 +822,9 @@ std::string gen_v4_canonical_headers(const req_info& info,
     signed_hdrs->append(name);
 
     canonical_hdrs.append(name.data(), name.length())
-                  .append(":", std::strlen(":"))
+                  .append(":", sarrlen(":"))
                   .append(value)
-                  .append("\n", std::strlen("\n"));
+                  .append("\n", sarrlen("\n"));
   }
 
   return canonical_hdrs;
@@ -1021,7 +1031,7 @@ get_v2_signature(CephContext* const cct,
 bool AWSv4ComplMulti::ChunkMeta::is_new_chunk_in_stream(size_t stream_pos) const
 {
   return stream_pos >= (data_offset_in_stream + data_length);
-}
+} /* ChunkMeta::is_new_chunk_in_stream */
 
 size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos) const
 {
@@ -1031,24 +1041,22 @@ size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos) const
   } else {
     return data_offset_in_stream + data_length - stream_pos;
   }
-}
-
+} /* ChunkMeta::get_data_size */
 
 /* AWSv4 completers begin. */
 std::pair<AWSv4ComplMulti::ChunkMeta, size_t /* consumed */>
 AWSv4ComplMulti::ChunkMeta::create_next(CephContext* const cct,
                                         ChunkMeta&& old,
                                         const char* const metabuf,
-                                        const size_t metabuf_len)
+                                        const size_t metabuf_len,
+					uint32_t flags)
 {
   std::string_view metastr(metabuf, metabuf_len);
 
-  const size_t semicolon_pos = metastr.find(";");
-  if (semicolon_pos == std::string_view::npos) {
-    ldout(cct, 20) << "AWSv4ComplMulti cannot find the ';' separator"
-                   << dendl;
-    throw rgw::io::Exception(EINVAL, std::system_category());
-  }
+  bool unsigned_chunked = flags & AWSv4ComplMulti::FLAG_UNSIGNED_CHUNKED;
+  bool expect_chunk_signature = !unsigned_chunked; // for now
+
+  ldout(cct, 20) << "AWSv4ComplMulti::create_next() old.cnt: " << old.cnt << dendl;
 
   char* data_field_end;
   /* strtoull ignores the "\r\n" sequence after each non-first chunk. */
@@ -1059,45 +1067,81 @@ AWSv4ComplMulti::ChunkMeta::create_next(CephContext* const cct,
     throw rgw::io::Exception(EINVAL, std::system_category());
   }
 
-  /* Parse the chunk_signature=... part. */
-  const auto signature_part = metastr.substr(semicolon_pos + 1);
-  const size_t eq_sign_pos = signature_part.find("=");
-  if (eq_sign_pos == std::string_view::npos) {
-    ldout(cct, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
-                   << dendl;
-    throw rgw::io::Exception(EINVAL, std::system_category());
-  }
+  if (expect_chunk_signature) {
 
-  /* OK, we have at least the beginning of a signature. */
-  const size_t data_sep_pos = signature_part.find("\r\n");
-  if (data_sep_pos == std::string_view::npos) {
-    ldout(cct, 20) << "AWSv4ComplMulti: no new line at signature end"
-                   << dendl;
-    throw rgw::io::Exception(EINVAL, std::system_category());
-  }
+    /* traditional parse looks for
+       string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
+       cf. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html */
 
-  const auto signature = \
-    signature_part.substr(eq_sign_pos + 1, data_sep_pos - 1 - eq_sign_pos);
-  if (signature.length() != SIG_SIZE) {
-    ldout(cct, 20) << "AWSv4ComplMulti: signature.length() != 64"
-                   << dendl;
-    throw rgw::io::Exception(EINVAL, std::system_category());
-  }
+    const size_t semicolon_pos = metastr.find(";");
+    if (semicolon_pos == std::string_view::npos) {
+      ldout(cct, 20) << "AWSv4ComplMulti cannot find the ';' separator"
+		     << dendl;
+      throw rgw::io::Exception(EINVAL, std::system_category());
+    }
 
-  const size_t data_starts_in_stream = \
-    + semicolon_pos + strlen(";") + data_sep_pos  + strlen("\r\n")
-    + old.data_offset_in_stream + old.data_length;
+    /* Parse the chunk_signature=... part. */
+    const auto signature_part = metastr.substr(semicolon_pos + 1);
+    const size_t eq_sign_pos = signature_part.find("=");
+    if (eq_sign_pos == std::string_view::npos) {
+      ldout(cct, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
+                     << dendl;
+      throw rgw::io::Exception(EINVAL, std::system_category());
+    }
 
-  ldout(cct, 20) << "parsed new chunk; signature=" << signature
-                 << ", data_length=" << data_length
-                 << ", data_starts_in_stream=" << data_starts_in_stream
-                 << dendl;
+    /* OK, we have at least the beginning of a signature. */
+    const size_t data_sep_pos = signature_part.find("\r\n");
+    if (data_sep_pos == std::string_view::npos) {
+      ldout(cct, 20) << "AWSv4ComplMulti: no new line at signature end"
+                     << dendl;
+      throw rgw::io::Exception(EINVAL, std::system_category());
+    }
 
-  return std::make_pair(ChunkMeta(data_starts_in_stream,
-                                  data_length,
-                                  signature),
-                        semicolon_pos + 83);
-}
+    const auto signature =
+        signature_part.substr(eq_sign_pos + 1, data_sep_pos - 1 - eq_sign_pos);
+    if (signature.length() != SIG_SIZE) {
+      ldout(cct, 20) << "AWSv4ComplMulti: signature.length() != 64" << dendl;
+      throw rgw::io::Exception(EINVAL, std::system_category());
+    }
+
+    const size_t data_starts_in_stream =
+        +semicolon_pos + sarrlen(";") + data_sep_pos + sarrlen("\r\n") +
+        old.data_offset_in_stream + old.data_length;
+
+    ldout(cct, 20) << "parsed new chunk; signature=" << signature
+                   << ", data_length=" << data_length
+                   << ", data_starts_in_stream=" << data_starts_in_stream
+                   << dendl;
+
+    return std::make_pair(
+	     ChunkMeta(data_starts_in_stream, data_length, signature, flags,
+		       ++old.cnt),
+	     semicolon_pos + 83);
+  } else {
+    /* no-chunk-signature aws-chunked */
+    ldout(cct, 20) << "AWSv4ComplMulti: non-signature meta chunk; data_length "
+		   << data_length
+		   << dendl;
+
+    /* currently we might see "\r\n20000\r\nreate the directory..." */
+    size_t crlf_pos = metastr.find("\r\n");
+    if (crlf_pos == 0) [[likely]] {
+      crlf_pos = metastr.find("\r\n", 2); // skip to next one
+    }
+    if (crlf_pos == std::string_view::npos) {
+      ldout(cct, 20) << "AWSv4ComplMulti: no new line at expected chunk end"
+                     << dendl;
+      throw rgw::io::Exception(EINVAL, std::system_category());
+    }
+
+    const size_t consumed = crlf_pos + sarrlen("\r\n");
+    const size_t data_starts_in_stream =
+        consumed + old.data_offset_in_stream + old.data_length;
+    return std::make_pair(ChunkMeta(data_starts_in_stream, data_length,
+			    "" /* signature */, flags, ++old.cnt),
+                          consumed);
+  } /* no-signature */
+} /* AWSv4ComplMulti::ChunkMeta::create_next */
 
 std::string
 AWSv4ComplMulti::calc_chunk_signature(const std::string& payload_hash) const
@@ -1110,7 +1154,7 @@ AWSv4ComplMulti::calc_chunk_signature(const std::string& payload_hash) const
     AWS4_EMPTY_PAYLOAD_HASH,
     payload_hash);
 
-  ldout(cct, 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
+  ldout(cct(), 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
                  << dendl;
 
   /* new chunk signature */
@@ -1119,22 +1163,36 @@ AWSv4ComplMulti::calc_chunk_signature(const std::string& payload_hash) const
   return sig.to_str();
 }
 
-
 bool AWSv4ComplMulti::is_signature_mismatched()
 {
+  /* in streaming unsigned payload, there are no chunk signatures nor trailer
+   * signature; there may be a trailing checksum
+   *
+   * if (flags & FLAG_UNSIGNED_CHUNKED), then we assert chunk_meta.signature.empty(),
+   * and conversely
+   */
+  if (flags & FLAG_UNSIGNED_CHUNKED) {
+    return false;
+  }
+
   /* The validity of previous chunk can be verified only after getting meta-
    * data of the next one. */
   const auto payload_hash = calc_hash_sha256_restart_stream(&sha256_hash);
   const auto calc_signature = calc_chunk_signature(payload_hash);
 
+  if (cct()->_conf->subsys.should_gather(ceph_subsys_rgw, 16)) [[unlikely]] {
+    ldout(cct(), 16) << "AWSv4ComplMulti: declared signature="
+		     << chunk_meta.get_signature()
+		     << "\nAWSv4ComplMulti: calculated signature="
+		     << calc_signature << dendl;
+    ldout(cct(), 16) << "AWSv4ComplMulti: prev_chunk_signature="
+		     << prev_chunk_signature << dendl;
+
+  }
+
   if (chunk_meta.get_signature() != calc_signature) {
-    ldout(cct, 20) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
+    ldout(cct(), 16) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
                    << dendl;
-    ldout(cct, 20) << "AWSv4ComplMulti: declared signature="
-                   << chunk_meta.get_signature() << dendl;
-    ldout(cct, 20) << "AWSv4ComplMulti: calculated signature="
-                   << calc_signature << dendl;
-
     return true;
   } else {
     prev_chunk_signature = chunk_meta.get_signature();
@@ -1142,16 +1200,19 @@ bool AWSv4ComplMulti::is_signature_mismatched()
   }
 }
 
-size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool& eof)
+AWSv4ComplMulti::ReceiveChunkResult AWSv4ComplMulti::recv_chunk(
+  char* const buf, const size_t buf_max, uint32_t cnt, bool& eof)
 {
   /* Buffer stores only parsed stream. Raw values reflect the stream
    * we're getting from a client. */
   size_t buf_pos = 0;
 
+  ldout(cct(), 20) << "AWSv4ComplMulti::recv_chunk() cnt: " << cnt << dendl;
+
   if (chunk_meta.is_new_chunk_in_stream(stream_pos)) {
     /* Verify signature of the previous chunk. We aren't doing that for new
      * one as the procedure requires calculation of payload hash. This code
-     * won't be triggered for the last, zero-length chunk. Instead, is will
+     * won't be triggered for the last, zero-length chunk. Instead, it will
      * be checked in the complete() method.  */
     if (stream_pos >= ChunkMeta::META_MAX_SIZE && is_signature_mismatched()) {
       throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH, std::system_category());
@@ -1163,8 +1224,24 @@ size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool&
     do {
       const size_t orig_size = parsing_buf.size();
       parsing_buf.resize(parsing_buf.size() + to_extract);
+
+      auto pb_size = parsing_buf.size();
+      auto pb_capacity = parsing_buf.capacity();
+
       const size_t received = io_base_t::recv_body(parsing_buf.data() + orig_size,
                                                    to_extract);
+
+      ldout(cct(), 20) << "AWSv4ComplMulti::recv_chunk() "
+		     << "after io_base_t::recv_body recv pb_size: "
+		     << pb_size
+		     << " pb_capacity "
+		     << pb_capacity
+		     << " to_extract: "
+		     << to_extract
+		     << " received: "
+		     << received
+		     << dendl;
+
       parsing_buf.resize(parsing_buf.size() - (to_extract - received));
       if (received == 0) {
         eof = true;
@@ -1177,14 +1254,14 @@ size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool&
 
     size_t consumed;
     std::tie(chunk_meta, consumed) = \
-      ChunkMeta::create_next(cct, std::move(chunk_meta),
-                             parsing_buf.data(), parsing_buf.size());
+      ChunkMeta::create_next(cct(), std::move(chunk_meta),
+	 parsing_buf.data(), parsing_buf.size(), flags);
 
     /* We can drop the bytes consumed during metadata parsing. The remainder
      * can be chunk's data plus possibly beginning of next chunks' metadata. */
     parsing_buf.erase(std::begin(parsing_buf),
                       std::begin(parsing_buf) + consumed);
-  }
+  } /* if (chunk_meta.is_new_chunk_in_stream(stream_pos)) */
 
   size_t stream_pos_was = stream_pos - parsing_buf.size();
 
@@ -1197,9 +1274,16 @@ size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool&
    * the final buffer. This is a trade-off between frontend's read overhead
    * and memcpy. */
   if (to_extract > 0 && parsing_buf.size() > 0) {
+
     const auto data_len = std::min(to_extract, parsing_buf.size());
     const auto data_end_iter = std::begin(parsing_buf) + data_len;
-    dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract << ", data_len=" << data_len << dendl;
+
+    dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
+	     << ", data_len=" << data_len
+	     << dendl;
+
+    /* if is-last-frag, then */
+    lf_bytes = stream_pos - stream_pos_was - data_len;
 
     std::copy(std::begin(parsing_buf), data_end_iter, buf);
     parsing_buf.erase(std::begin(parsing_buf), data_end_iter);
@@ -1229,18 +1313,146 @@ size_t AWSv4ComplMulti::recv_chunk(char* const buf, const size_t buf_max, bool&
   }
 
   dout(20) << "AWSv4ComplMulti: filled=" << buf_pos << dendl;
-  return buf_pos;
-}
+  return ReceiveChunkResult(buf_pos, chunk_meta.get_offset());
+} /* AWSv4Complmulti::recv_chunk */
+
+std::string
+AWSv4ComplMulti::calc_v4_trailer_signature(const trailer_map_t& trailer_map,
+					   const std::string_view last_chunk_sig)
+{
+  const auto headers = get_canon_amz_hdrs(trailer_map);
+  const auto canon_header_hash = calc_hash_sha256(headers);
+
+  const auto string_to_sign = string_join_reserve("\n",
+    "AWS4-HMAC-SHA256-TRAILER",
+    date,
+    credential_scope,
+    last_chunk_sig,
+    canon_header_hash.to_str());
+
+  const auto trailer_signature =
+    calc_hmac_sha256(signing_key, string_to_sign).to_str();
+
+  ldout(cct(), 10) << "trailer headers = " << headers
+		 << "\ntrailers string to sign = "
+                 << rgw::crypt_sanitize::log_content{string_to_sign}
+		 << "\ncalc trailer signature = "
+		 << trailer_signature
+		 << "\nexpected last-chunk-sig = "
+		 << last_chunk_sig
+		 << dendl;
+
+  return trailer_signature;
+} /* calc_v4_trailer_signature */
+
+
+/* the following templates capture the start (and for extract_helper)
+ * end boundaries of a substring match as constant strings, moving
+ * a small amount of work to compile time */
+
+using ExtractResult = std::tuple<bool, std::string_view, size_t>;
+
+/* adapted from here: https://ctrpeach.io/posts/cpp20-string-literal-template-parameters/ */
+template<size_t N>
+struct StringLiteral {
+    constexpr StringLiteral(const char (&str)[N]) {
+        std::copy_n(str, N, val);
+    }
+    char val[N];
+};
+
+template <StringLiteral start, StringLiteral end>
+static inline ExtractResult mut_extract_helper(std::string_view& region) {
+  if (auto spos = region.find(start.val);
+      spos != std::string_view::npos) {
+    if (auto epos = region.find(end.val, spos + sarrlen(start.val));
+	epos != std::string_view::npos) {
+      std::string_view matched = region.substr(spos, epos - spos);
+      auto consumed = matched.size() + sarrlen(end.val);
+      region.remove_prefix(consumed);
+      return ExtractResult(true, matched, spos + consumed);
+    }
+  }
+  return ExtractResult(false, "", 0);
+} /* mut_extract_helper <begin, end> */
+
+template <StringLiteral end>
+static inline ExtractResult extract_helper(const std::string_view& region,
+					   const std::string_view start) {
+  if (auto spos = region.find(start);
+      spos != std::string_view::npos) {
+    if (auto epos = region.find(end.val, spos + start.length());
+	epos != std::string_view::npos) {
+      std::string_view matched = region.substr(spos, epos - spos);
+      auto consumed = matched.size() + sarrlen(end.val);
+      return ExtractResult(true, matched, consumed);
+    }
+  }
+  return ExtractResult(false, "", 0);
+} /* extract_helper <end> */
+
+using split_func_t =
+  const fu2::unique_function<void(const std::string_view k,
+				  const std::string_view v) const>;
+
+static inline void split_header(const std::string_view hdr, split_func_t f)
+{
+  auto kv = ceph::split(hdr, ":");
+  auto k = kv.begin();
+  if (k != kv.end()) {
+    auto v = std::next(k);
+    if (v != kv.end()) [[likely]] {
+      f(*k, *v);
+    }
+  }
+} /* split_header */
+
+inline void AWSv4ComplMulti::extract_trailing_headers(
+  std::string_view x_amz_trailer, std::string_view& mut_sv_trailer,
+  AWSv4ComplMulti::trailer_map_t& trailer_map)
+{
+  using std::get;
+  size_t consumed = 0;
+  /* spliterate x_amz_trailer */
+  auto kv = ceph::split(x_amz_trailer, ",");
+  for (auto k = kv.begin(); k != kv.end(); k = std::next(k)) {
+    /* extract trailer;  if there's more than one trailer, don't rely on their
+     * order in x-amz-trailer */
+    auto ex_header = extract_helper<"\r\n">(mut_sv_trailer, *k);
+    if (get<0>(ex_header)) {
+      auto header = get<1>(ex_header);
+      split_header(header, [&](const std::string_view k, const std::string_view v) -> void {
+	if (cct()->_conf->subsys.should_gather(ceph_subsys_rgw, 10)) [[unlikely]] {
+	  ldout(cct(), 10) << fmt::format("\nextracted trailing header {}={}", k, v) << dendl;
+	}
+	/* populate trailer map with expected headers and their values, if sent */
+	trailer_map.insert(trailer_map_t::value_type(k, v));
+	/* populate to req_info.env as well */
+	put_prop(ys_header_mangle(k), v);
+      });
+      consumed += get<2>(ex_header);
+    } /* one trailer */
+  } /* foreach trailer */
+  /* advance mut_sv_trailer */
+  mut_sv_trailer.remove_prefix(consumed);
+} /* AWSv4complmulti::extract_trailing_headers */
 
 size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max)
 {
+  using std::get;
+
   bool eof = false;
   size_t total = 0;
 
+  ldout(cct(), 20) << "AWSv4ComplMulti::recv_body() buf_max: " << buf_max << dendl;
+
+  uint32_t cnt = 0;
   while (total < buf_max && !eof) {
-    const size_t received = recv_chunk(buf + total, buf_max - total, eof);
-    total += received;
+    ReceiveChunkResult rcr =
+      recv_chunk(buf + total, buf_max - total, cnt++, eof);
+    total += rcr.received;
   }
+
   dout(20) << "AWSv4ComplMulti: received=" << total << dendl;
   return total;
 }
@@ -1253,6 +1465,7 @@ void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider* dpp, req_st
   if (!decoded_length) {
     throw -EINVAL;
   } else {
+    /* XXXX oh my, we forget the original content length */
     s_rw->length = decoded_length;
     s_rw->content_length = parse_content_length(decoded_length);
 
@@ -1269,21 +1482,135 @@ void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider* dpp, req_st
 
 bool AWSv4ComplMulti::complete()
 {
-  /* Now it's time to verify the signature of the last, zero-length chunk. */
+  /* historically, this code has been validating not the final zero-length
+   * chunk, but the one before that; we'll do that as before, and then
+   * consume the last chunk signature and the trailer section */
   if (is_signature_mismatched()) {
-    ldout(cct, 10) << "ERROR: signature of last chunk does not match"
+    ldout(cct(), 10) << "ERROR: signature of last payload chunk does not match"
                    << dendl;
     return false;
   } else {
+    /* now it's time to verify the signature of the last, zero-length chunk */
+    const auto string_to_sign = string_join_reserve("\n",
+    "AWS4-HMAC-SHA256-PAYLOAD",
+    date,
+    credential_scope,
+    prev_chunk_signature,
+    AWS4_EMPTY_PAYLOAD_HASH,
+    AWS4_EMPTY_PAYLOAD_HASH);
+
+    const auto final_chunk_signature =
+      calc_hmac_sha256(signing_key, string_to_sign).to_str();
+
+    ldout(cct(), 10) << "final chunk signature = "
+		     << final_chunk_signature
+		     << "\nprev_chunk_signature was "
+		     << prev_chunk_signature
+		     << dendl;
+
+    /* in the last-chunk case, parsing_buf potentially holds unconsumed
+     * data, including the final chunk boundary */
+    std::string_view last_frag(parsing_buf.begin().get_ptr(), lf_bytes);
+
+    size_t tbuf_pos = 0;
+
+    static constexpr size_t trailer_buf_size = 256;
+    boost::container::static_vector<char, trailer_buf_size> trailer_vec;
+
+    std::copy(parsing_buf.begin(), parsing_buf.begin() + lf_bytes,
+              trailer_vec.begin());
+    tbuf_pos += lf_bytes;
+
+    while (tbuf_pos < trailer_buf_size) {
+      const size_t received =
+          io_base_t::recv_body(trailer_vec.data() + tbuf_pos,
+			       trailer_buf_size - tbuf_pos - 1);
+      dout(30) << "AWSv4ComplMulti: recv trailer received=" << received
+               << dendl;
+      if (received == 0) {
+        break;
+      }
+      tbuf_pos += received;
+    }
+
+    if (tbuf_pos == trailer_buf_size) {
+      ldout(cct(), 10) << "AWSv4ComplMulti:: recv trailer exceeded size limit of "
+		       << trailer_buf_size - 1
+		       << " bytes"
+		       << dendl;
+      throw rgw::io::Exception(ERR_LIMIT_EXCEEDED, std::system_category());
+    }
+
+    std::string_view expected_trailer_signature;
+    std::string calculated_trailer_signature;
+
+    if (tbuf_pos > sarrlen("\r\n0;")) {
+      const std::string_view sv_trailer(trailer_vec.data() + sarrlen("\r\n0;"),
+                                        tbuf_pos - sarrlen("\r\n0;"));
+
+      if (cct()->_conf->subsys.should_gather(ceph_subsys_rgw, 10)) [[unlikely]] {
+        ldout(cct(), 10) << "trailer_section: " << sv_trailer << dendl;
+      }
+
+      std::string_view mut_sv_trailer(sv_trailer);
+
+      auto chunk_signature =
+          mut_extract_helper<"chunk-signature=", "\r\n">(mut_sv_trailer);
+
+      std::string_view sig_sv;
+      if (get<0>(chunk_signature)) {
+        sig_sv = get<1>(chunk_signature);
+        sig_sv.remove_prefix(sarrlen("chunk-signature="));
+
+        ldout(cct(), 10) << "expected last chunk signature: " << sig_sv
+                         << " remaining: " << mut_sv_trailer << dendl;
+      }
+
+      trailer_map_t trailer_map;
+
+      if (x_amz_trailer) {
+        extract_trailing_headers(*x_amz_trailer, mut_sv_trailer, trailer_map);
+      }
+
+      auto trailer_signature =
+          mut_extract_helper<"x-amz-trailer-signature:", "\r\n">(
+              mut_sv_trailer);
+      if (get<0>(trailer_signature)) {
+        auto trailing_sig = get<1>(trailer_signature);
+        split_header(
+            trailing_sig,
+            [&](const std::string_view k, const std::string_view v) -> void {
+              expected_trailer_signature = v;
+            });
+
+        calculated_trailer_signature =
+            calc_v4_trailer_signature(trailer_map, final_chunk_signature);
+
+        ldout(cct(), 10) << "expected trailer signature="
+                         << expected_trailer_signature
+                         << "\n calculated trailer signature="
+                         << calculated_trailer_signature
+                         << "\n trailer bytes remaining/not consumed: "
+                         << mut_sv_trailer << dendl;
+      } /* matched trailer signature */
+    } /* have trailer */
+
+    if (expect_trailer_signature() &&
+	(expected_trailer_signature.empty() ||
+	 (calculated_trailer_signature != expected_trailer_signature))) {
+      throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH, std::system_category());
+    }
+
     return true;
   }
-}
+} /* AWSv4Complmulti:: complete */
 
 rgw::auth::Completer::cmplptr_t
 AWSv4ComplMulti::create(const req_state* const s,
                         std::string_view date,
                         std::string_view credential_scope,
                         std::string_view seed_signature,
+			uint32_t flags,
                         const boost::optional<std::string>& secret_key)
 {
   if (!secret_key) {
@@ -1300,6 +1627,7 @@ AWSv4ComplMulti::create(const req_state* const s,
                                            std::move(date),
                                            std::move(credential_scope),
                                            std::move(seed_signature),
+					   flags,
                                            signing_key);
 }
 
diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h
index c03dfad825dd..f68768200565 100644
--- a/src/rgw/rgw_auth_s3.h
+++ b/src/rgw/rgw_auth_s3.h
@@ -4,6 +4,8 @@
 #pragma once
 
 #include <array>
+#include <boost/algorithm/string/predicate.hpp>
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <string_view>
@@ -11,6 +13,7 @@
 
 #include <boost/algorithm/string.hpp>
 #include <boost/container/static_vector.hpp>
+#include <boost/container/flat_map.hpp>
 
 #include "common/sstring.hh"
 #include "rgw_common.h"
@@ -256,37 +259,47 @@ class AWSAuthStrategy : public rgw::auth::Strategy,
   const char* get_name() const noexcept override {
     return "rgw::auth::s3::AWSAuthStrategy";
   }
-};
-
+}; /* AWSAuthstrategy */
 
 class AWSv4ComplMulti : public rgw::auth::Completer,
                         public rgw::io::DecoratedRestfulClient<rgw::io::RestfulClient*>,
                         public std::enable_shared_from_this<AWSv4ComplMulti> {
+
   using io_base_t = rgw::io::DecoratedRestfulClient<rgw::io::RestfulClient*>;
   using signing_key_t = sha256_digest_t;
 
-  CephContext* const cct;
+  using trailer_map_t = boost::container::flat_map<std::string_view, std::string_view>;
+
+  const req_state* const s;
 
   const std::string_view date;
   const std::string_view credential_scope;
+  const uint32_t flags;
   const signing_key_t signing_key;
 
   class ChunkMeta {
     size_t data_offset_in_stream = 0;
     size_t data_length = 0;
     std::string signature;
+    uint32_t flags{FLAG_NONE};
+    uint32_t cnt;
 
     ChunkMeta(const size_t data_starts_in_stream,
               const size_t data_length,
-              const std::string_view signature)
+              const std::string_view signature,
+	      uint32_t _flags,
+	      uint32_t _cnt)
       : data_offset_in_stream(data_starts_in_stream),
         data_length(data_length),
-        signature(std::string(signature)) {
-    }
+        signature(std::string(signature)),
+	flags(_flags),
+	cnt(_cnt)
+    {}
 
-    explicit ChunkMeta(const std::string_view& signature)
-      : signature(std::string(signature)) {
-    }
+    explicit ChunkMeta(const std::string_view& signature, uint32_t _flags,
+		       uint32_t _cnt)
+      : signature(std::string(signature)), flags(_flags), cnt(_cnt)
+    {}
 
   public:
     static constexpr size_t SIG_SIZE = 64;
@@ -309,10 +322,14 @@ class AWSv4ComplMulti : public rgw::auth::Completer,
       return signature;
     }
 
+    size_t get_offset() { return data_offset_in_stream; }
+
     /* Factory: create an object representing metadata of first, initial chunk
      * in a stream. */
-    static ChunkMeta create_first(const std::string_view& seed_signature) {
-      return ChunkMeta(seed_signature);
+    static ChunkMeta create_first(const std::string_view& seed_signature,
+				  uint32_t flags,
+				  uint32_t cnt) {
+      return ChunkMeta(seed_signature, flags, cnt);
     }
 
     /* Factory: parse a block of META_MAX_SIZE bytes and creates an object
@@ -321,38 +338,87 @@ class AWSv4ComplMulti : public rgw::auth::Completer,
     static std::pair<ChunkMeta, size_t> create_next(CephContext* cct,
                                                     ChunkMeta&& prev,
                                                     const char* metabuf,
-                                                    size_t metabuf_len);
+                                                    size_t metabuf_len,
+						    uint32_t flags);
   } chunk_meta;
 
+  uint16_t lf_bytes;
   size_t stream_pos;
   boost::container::static_vector<char, ChunkMeta::META_MAX_SIZE> parsing_buf;
+  boost::optional<std::string_view> x_amz_trailer;
   ceph::crypto::SHA256* sha256_hash;
   std::string prev_chunk_signature;
 
   bool is_signature_mismatched();
   std::string calc_chunk_signature(const std::string& payload_hash) const;
-  size_t recv_chunk(char* buf, size_t max, bool& eof);
 
-public:
+  struct ReceiveChunkResult {
+    size_t received;
+    size_t data_offset_in_stream;
+
+    ReceiveChunkResult(size_t x, size_t y)
+      : received(x), data_offset_in_stream(y)
+    {}
+  }; /* ReceiveChunkResult */
+
+  inline CephContext* cct() const {
+    return s->cct;
+  }
+
+  inline bool expect_trailer_signature() const {
+    return flags & AWSv4ComplMulti::FLAG_TRAILER_SIGNATURE;
+  }
+
+  inline void put_prop(const std::string_view k, const std::string_view v) {
+    /* assume the caller will mangle the key name, if required */
+    auto& map = const_cast<env_map_t&>(s->info.env->get_map());
+    map.insert(env_map_t::value_type(k, v));
+  }
+
+  inline void extract_trailing_headers(std::string_view x_amz_trailer,
+				       std::string_view& mut_sv_trailer,
+				       trailer_map_t& trailer_map);
+
+  std::string calc_v4_trailer_signature(const trailer_map_t& trailer_map,
+					const std::string_view last_chunk_sig);
+
+  ReceiveChunkResult recv_chunk(char* buf, size_t max, uint32_t rc_cnt, bool& eof);
+
+  public:
+
+  static constexpr uint32_t FLAG_NONE =              0x00;
+  static constexpr uint32_t FLAG_TRAILING_CHECKSUM = 0x01;
+  static constexpr uint32_t FLAG_UNSIGNED_PAYLOAD =  0x02;
+  static constexpr uint32_t FLAG_UNSIGNED_CHUNKED =  0x04;
+  static constexpr uint32_t FLAG_TRAILER_SIGNATURE = 0x08;
+
   /* We need the constructor to be public because of the std::make_shared that
    * is employed by the create() method. */
   AWSv4ComplMulti(const req_state* const s,
                   std::string_view date,
                   std::string_view credential_scope,
                   std::string_view seed_signature,
+		  uint32_t _flags,
                   const signing_key_t& signing_key)
     : io_base_t(nullptr),
-      cct(s->cct),
+      s(s),
       date(std::move(date)),
       credential_scope(std::move(credential_scope)),
+      flags(_flags),
       signing_key(signing_key),
 
       /* The evolving state. */
-      chunk_meta(ChunkMeta::create_first(seed_signature)),
+      chunk_meta(ChunkMeta::create_first(
+		   seed_signature, flags, 0 /* first call in cycle */)),
       stream_pos(0),
       sha256_hash(calc_hash_sha256_open_stream()),
-      prev_chunk_signature(std::move(seed_signature)) {
-  }
+      prev_chunk_signature(std::move(seed_signature))
+  {
+    auto cksum = s->info.env->get("HTTP_X_AMZ_TRAILER");
+    if (!! cksum) {
+      x_amz_trailer = std::string_view(cksum, std::strlen(cksum));
+    }
+  } /* AWSv4ComplMulti */
 
   ~AWSv4ComplMulti() {
     if (sha256_hash) {
@@ -372,6 +438,7 @@ class AWSv4ComplMulti : public rgw::auth::Completer,
                           std::string_view date,
                           std::string_view credential_scope,
                           std::string_view seed_signature,
+			  uint32_t flags,
                           const boost::optional<std::string>& secret_key);
 
 };
@@ -454,6 +521,13 @@ static constexpr char AWS4_UNSIGNED_PAYLOAD_HASH[] = "UNSIGNED-PAYLOAD";
 static constexpr char AWS4_STREAMING_PAYLOAD_HASH[] = \
   "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
 
+/* trailing header forms */
+static constexpr char AWS4_STREAMING_UNSIGNED_PAYLOAD_TRAILER[] = \
+  "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+static constexpr char AWS4_STREAMING_HMAC_SHA256_PAYLOAD_TRAILER[] = \
+  "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
+
 bool is_non_s3_op(RGWOpType op_type);
 
 int parse_v4_credentials(const req_info& info,                     /* in */
@@ -578,11 +652,31 @@ static inline const char* get_v4_exp_payload_hash(const req_info& info)
   return expected_request_payload_hash;
 }
 
-static inline bool is_v4_payload_unsigned(const char* const exp_payload_hash)
+static inline bool is_traditional_v4_unsigned_payload(const char* const exp_payload_hash)
 {
   return boost::equals(exp_payload_hash, AWS4_UNSIGNED_PAYLOAD_HASH);
 }
 
+static inline bool is_v4_payload_unsigned_chunked(const char* const exp_payload_hash)
+{
+  return boost::equals(exp_payload_hash, AWS4_STREAMING_UNSIGNED_PAYLOAD_TRAILER);
+}
+
+static inline bool is_v4_payload_unsigned(const char* const exp_payload_hash)
+{
+  return boost::contains(exp_payload_hash, "UNSIGNED-PAYLOAD");
+}
+
+static inline bool have_checksum_trailer(const char* const exp_payload_hash)
+{
+  return boost::algorithm::ends_with(exp_payload_hash, "TRAILER");
+}
+
+static inline bool expect_trailer_signature(const char* const exp_payload_hash)
+{
+  return boost::equals(exp_payload_hash, AWS4_STREAMING_HMAC_SHA256_PAYLOAD_TRAILER);
+}
+
 static inline bool is_v4_payload_empty(const req_state* const s)
 {
   /* from rfc2616 - 4.3 Message Body
@@ -596,7 +690,7 @@ static inline bool is_v4_payload_empty(const req_state* const s)
 
 static inline bool is_v4_payload_streamed(const char* const exp_payload_hash)
 {
-  return boost::equals(exp_payload_hash, AWS4_STREAMING_PAYLOAD_HASH);
+  return boost::algorithm::starts_with(exp_payload_hash, "STREAMING-");
 }
 
 std::string get_v4_canonical_qs(const req_info& info, bool using_qs);
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 3bf6c4dfac8e..ea91859f5e68 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -446,8 +446,10 @@ class RGWConf {
   }
 };
 
+using env_map_t = std::map<std::string, std::string, ltstr_nocase>;
+
 class RGWEnv {
-  std::map<std::string, std::string, ltstr_nocase> env_map;
+  env_map_t env_map;
   RGWConf conf;
 public:
   void init(CephContext *cct);
@@ -1849,6 +1851,18 @@ int decode_bl(bufferlist& bl, T& t)
   return 0;
 }
 
+static inline std::string ys_header_mangle(std::string_view name)
+{
+  /* can we please stop doing this? */
+  std::string out;
+  out.reserve(name.length());
+  std::transform(std::begin(name), std::end(name),
+		 std::back_inserter(out), [](const int c) {
+		   return c == '-' ? '_' : c == '_' ? '-' : std::toupper(c);
+		 });
+  return out;
+} /* ys_header_mangle */
+
 extern int rgw_bucket_parse_bucket_instance(const std::string& bucket_instance, std::string *bucket_name, std::string *bucket_id, int *shard_id);
 
 boost::intrusive_ptr<CephContext>
diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h
index 5658003adb8a..d50846a255cb 100644
--- a/src/rgw/rgw_file_int.h
+++ b/src/rgw/rgw_file_int.h
@@ -260,7 +260,7 @@ namespace rgw {
     static constexpr uint32_t FLAG_ROOT =    0x0002;
     static constexpr uint32_t FLAG_CREATE =  0x0004;
     static constexpr uint32_t FLAG_CREATING =  0x0008;
-    static constexpr uint32_t FLAG_SYMBOLIC_LINK = 0x0009;
+    static constexpr uint32_t FLAG_SYMBOLIC_LINK = 0x0009; // XXXX bug?
     static constexpr uint32_t FLAG_DIRECTORY = 0x0010;
     static constexpr uint32_t FLAG_BUCKET = 0x0020;
     static constexpr uint32_t FLAG_LOCK =   0x0040;
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 6b827f983fb6..a7c35f3773cc 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1080,7 +1080,7 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl)
   }
 
   return len;
-}
+} /* RGWPutObj_ObjStore::get_data(bufferlist& bl) */
 
 
 /*
@@ -2010,23 +2010,6 @@ RGWRESTMgr::~RGWRESTMgr()
   delete default_mgr;
 }
 
-int64_t parse_content_length(const char *content_length)
-{
-  int64_t len = -1;
-
-  if (*content_length == '\0') {
-    len = 0;
-  } else {
-    string err;
-    len = strict_strtoll(content_length, 10, &err);
-    if (!err.empty()) {
-      len = -1;
-    }
-  }
-
-  return len;
-}
-
 int RGWREST::preprocess(req_state *s, rgw::io::BasicClient* cio)
 {
   req_info& info = s->info;
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 0b6cf62ed2c7..8ee587e7c7b4 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -8,6 +8,7 @@
 #include <string_view>
 #include <boost/container/flat_set.hpp>
 #include "common/sstring.hh"
+#include "common/strtol.h"
 #include "common/ceph_json.h"
 #include "include/ceph_assert.h" /* needed because of common/ceph_json.h */
 #include "rgw_op.h"
@@ -782,6 +783,23 @@ inline void dump_header_if_nonempty(req_state* s,
   }
 }
 
+static inline int64_t parse_content_length(const char *content_length)
+{
+  int64_t len = -1;
+
+  if (*content_length == '\0') {
+    len = 0;
+  } else {
+    std::string err;
+    len = strict_strtoll(content_length, 10, &err);
+    if (!err.empty()) {
+      len = -1;
+    }
+  }
+
+  return len;
+} /* parse_content_length */
+
 inline std::string compute_domain_uri(const req_state *s) {
   std::string uri = (!s->info.domain.empty()) ? s->info.domain :
     [&s]() -> std::string {
@@ -799,7 +817,6 @@ inline std::string compute_domain_uri(const req_state *s) {
 }
 
 extern void dump_content_length(req_state *s, uint64_t len);
-extern int64_t parse_content_length(const char *content_length);
 extern void dump_etag(req_state *s,
                       const std::string_view& etag,
                       bool quoted = false);
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 63bdaeff5859..fddb3f092c1a 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#include <cstdint>
 #include <errno.h>
 #include <array>
 #include <string.h>
@@ -5628,9 +5629,9 @@ AWSSignerV4::prepare(const DoutPrefixProvider *dpp,
    *   because the URL is used to upload an arbitrary payload. Instead, you
    *   use a constant string UNSIGNED-PAYLOAD.
    *
-   * This means we have absolutely no business in spawning completer. Both
-   * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
-   * by default. We don't need to change that. */
+   * This means that, in the absence of a trailer, we don't need to spawn
+   * a completer. Both aws4_auth_needs_complete and aws4_auth_streaming_mode
+   * are set to false by default. We don't need to change that. */
   return {
     access_key_id,
     date,
@@ -5768,19 +5769,22 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
     throw -ERR_AMZ_CONTENT_SHA256_MISMATCH;
   }
 
-  /* Requests authenticated with the Query Parameters are treated as unsigned.
-   * From "Authenticating Requests: Using Query Parameters (AWS Signature
-   * Version 4)":
-   *
-   *   You don't include a payload hash in the Canonical Request, because
-   *   when you create a presigned URL, you don't know the payload content
-   *   because the URL is used to upload an arbitrary payload. Instead, you
-   *   use a constant string UNSIGNED-PAYLOAD.
-   *
-   * This means we have absolutely no business in spawning completer. Both
-   * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
-   * by default. We don't need to change that. */
-  if (is_v4_payload_unsigned(exp_payload_hash) || is_v4_payload_empty(s) || is_non_s3_op) {
+  /* Traditional UNSIGNED-PAYLOAD requests do not require a completer, but since
+   * 2022, even unsigned payload requests can be sent as aws-chunked and may
+   * have a checksum trailer */
+
+  auto traditional_v4_unsigned =
+    is_traditional_v4_unsigned_payload(exp_payload_hash);
+  auto v4_unsigned = is_v4_payload_unsigned(exp_payload_hash);
+  auto v4_unsigned_chunked = is_v4_payload_unsigned_chunked(exp_payload_hash);
+  auto checksum_trailer = have_checksum_trailer(exp_payload_hash);
+  auto trailer_signature = expect_trailer_signature(exp_payload_hash);
+
+  if (traditional_v4_unsigned ||
+      (v4_unsigned && !checksum_trailer) ||
+      is_v4_payload_empty(s) ||
+      is_non_s3_op) {
+    ldpp_dout(s, 10) << __func__ << ": UNSIGNED-PAYLOAD or other v4 no-completer case" << dendl;
     return {
       access_key_id,
       client_signature,
@@ -5855,9 +5859,6 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
         cmpl_factory
       };
     } else {
-      /* IMHO "streamed" doesn't fit too good here. I would prefer to call
-       * it "chunked" but let's be coherent with Amazon's terminology. */
-
       ldpp_dout(s, 10) << "body content detected in multiple chunks" << dendl;
 
       /* payload in multiple chunks */
@@ -5883,11 +5884,31 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
       /* In the case of query string-based authentication there should be no
        * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
        * for CanonReq. */
+
+      uint32_t flags{AWSv4ComplMulti::FLAG_NONE};
+
+      if (v4_unsigned) {
+	flags |= AWSv4ComplMulti::FLAG_UNSIGNED_PAYLOAD;
+      }
+
+      if (checksum_trailer) {
+	flags |= AWSv4ComplMulti::FLAG_TRAILING_CHECKSUM;
+      }
+
+      if (v4_unsigned_chunked) {
+	flags |= AWSv4ComplMulti::FLAG_UNSIGNED_CHUNKED;
+      }
+
+      if (trailer_signature) {
+	flags |= AWSv4ComplMulti::FLAG_TRAILER_SIGNATURE;
+      }
+
       const auto cmpl_factory = std::bind(AWSv4ComplMulti::create,
                                           s,
                                           date,
                                           credential_scope,
                                           client_signature,
+					  flags,
                                           std::placeholders::_1);
       return {
         access_key_id,

From e2a65f1d07e00c9fd5686196047724b23722b9bb Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Fri, 5 Jan 2024 10:40:49 -0500
Subject: [PATCH 1888/2492] rgw_cksum: add junit5-based unit test suite

This can be run by hand with:

./mvnw clean package
./mvnw test -Dtest=PutObjects

The following properties are sourced from the environment:

    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY

    RGW_HTTP_ENDPOINT_URL
    RGW_HTTPS_ENDPOINT_URL

Then adds:

qa/workunits/rgw: add test driver script for maven suite

Launch it fromn cls.yaml, as with test_librgw_file.h.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 qa/suites/rgw/verify/0-install.yaml           |     5 +-
 qa/suites/rgw/verify/tasks/cls.yaml           |     1 +
 qa/workunits/rgw/jcksum/.gitignore            |    20 +
 .../rgw/jcksum/.mvn/wrapper/maven-wrapper.jar |   Bin 0 -> 62547 bytes
 .../.mvn/wrapper/maven-wrapper.properties     |    18 +
 qa/workunits/rgw/jcksum/README.md             |     0
 qa/workunits/rgw/jcksum/file-1m               | 30819 ++++++++++++++++
 qa/workunits/rgw/jcksum/file-200b             |     5 +
 qa/workunits/rgw/jcksum/file-21983b           |   568 +
 qa/workunits/rgw/jcksum/file-256k             |  7216 ++++
 qa/workunits/rgw/jcksum/file-8b               |     1 +
 qa/workunits/rgw/jcksum/mvnw                  |   308 +
 qa/workunits/rgw/jcksum/mvnw.cmd              |   205 +
 qa/workunits/rgw/jcksum/pom-SNAPSHOT.xml      |    56 +
 qa/workunits/rgw/jcksum/pom.xml               |   145 +
 .../src/main/java/io/ceph/jcksum/jcksum.java  |   370 +
 .../test/java/io/ceph/jcksum/PutObjects.java  |   308 +
 qa/workunits/rgw/test_awssdkv4_sig.sh         |    42 +
 18 files changed, 40085 insertions(+), 2 deletions(-)
 create mode 100644 qa/workunits/rgw/jcksum/.gitignore
 create mode 100644 qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.jar
 create mode 100644 qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.properties
 create mode 100644 qa/workunits/rgw/jcksum/README.md
 create mode 100644 qa/workunits/rgw/jcksum/file-1m
 create mode 100644 qa/workunits/rgw/jcksum/file-200b
 create mode 100644 qa/workunits/rgw/jcksum/file-21983b
 create mode 100644 qa/workunits/rgw/jcksum/file-256k
 create mode 100644 qa/workunits/rgw/jcksum/file-8b
 create mode 100755 qa/workunits/rgw/jcksum/mvnw
 create mode 100644 qa/workunits/rgw/jcksum/mvnw.cmd
 create mode 100644 qa/workunits/rgw/jcksum/pom-SNAPSHOT.xml
 create mode 100644 qa/workunits/rgw/jcksum/pom.xml
 create mode 100644 qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
 create mode 100644 qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
 create mode 100755 qa/workunits/rgw/test_awssdkv4_sig.sh

diff --git a/qa/suites/rgw/verify/0-install.yaml b/qa/suites/rgw/verify/0-install.yaml
index 18779671634d..85b25e7ef869 100644
--- a/qa/suites/rgw/verify/0-install.yaml
+++ b/qa/suites/rgw/verify/0-install.yaml
@@ -1,9 +1,10 @@
 tasks:
 - install:
       # extra packages added for the rgw-datacache task
+      # java and maven needed for S3 trailer signature tests
       extra_system_packages:
-        deb: ['s3cmd']
-        rpm: ['s3cmd']
+        deb: ['s3cmd', 'maven']
+        rpm: ['s3cmd', 'maven']
 - ceph:
 - openssl_keys:
 - rgw:
diff --git a/qa/suites/rgw/verify/tasks/cls.yaml b/qa/suites/rgw/verify/tasks/cls.yaml
index e185796bc53c..71cca69d603e 100644
--- a/qa/suites/rgw/verify/tasks/cls.yaml
+++ b/qa/suites/rgw/verify/tasks/cls.yaml
@@ -13,3 +13,4 @@ tasks:
         - rgw/test_rgw_gc_log.sh
         - rgw/test_rgw_obj.sh
         - rgw/test_librgw_file.sh
+	- rgw/test_awssdkv4_sig.sh
diff --git a/qa/workunits/rgw/jcksum/.gitignore b/qa/workunits/rgw/jcksum/.gitignore
new file mode 100644
index 000000000000..19b9f97248cf
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/.gitignore
@@ -0,0 +1,20 @@
+# Maven
+target/
+
+# Ignore Gradle GUI config
+gradle-app.setting
+
+# Eclipse
+/.classpath
+/.settings/
+/.project
+/bin/
+
+# IntelliJ
+.idea
+*.iml
+*.ipr
+*.iws
+
+# Misc
+*.log
diff --git a/qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.jar b/qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.jar
new file mode 100644
index 0000000000000000000000000000000000000000..cb28b0e37c7d206feb564310fdeec0927af4123a
GIT binary patch
literal 62547
zcmb5V1CS=sk~Z9!wr$(CZEL#U=Co~N+O}=mwr$(Cds^S@-Tij=#=rmlVk@E|Dyp8$
z$UKz?`Q$l@GN3=8fq)=^fVx`E)Pern1@-q?PE1vZPD);!LGdpP^)C$aAFx&{CzjH`
zpQV9;fd0PyFPNN=yp*_@iYmRFcvOrKbU!1a*o)t$0ex(~3z5?bw11HQYW_uDngyer
za60w&wz^`W&Z!0XSH^cLNR&k>%)Vr|$}(wfBzmSbuK^)dy#xr@_NZVszJASn12dw;
z-KbI5yz=2awY0>OUF)&crfPu&tVl|!>g*#ur@K=$@8N05<_Mldg}X`N6O<~3|Dpk3
z<UIRcbhf^wCy5@WX~(j;#aN);In?VFeoy+)_ZgQb$iI2^KR3XSzn<ATnEu1}{}}=E
z9|!|G14A?8e*r}LXP~u#i?PkWfMfkna90NdJ3C{Ce<6nVufz<k%>RWb!e7z<{Mr96
z^C<tn;J@ue|L?6p{|{;U|F^a5{|hbu&`I~-b^cc{W=ISH+yB)C^EVjL{<}_5F-0+X
zQ89TDDKV9*Xd!42CRni-J~8^-u8>{%ROigEIapRGbFA5g4XoQAe_Y1ii3Ci!KV`?$
zZ2Hy<C%~!g%>1VP#hVp>OOqe~m|lo@^276Ik<~*6eRSOe;$wn_0@St#cJy}qI#RP=
zHVMXyFYYX%T_k3MNbtOX{<*_6Htq*o|7~MkS|A|A|8AqKl!%zTirAJGz;R<3&F7_N
z)uC9$9K1M-)g0#}tnM(lO2k~W&4xT7gshgZ1-y2Yo-q9Li7%zguh7W#kGfnjo7Cl6
z!^wTtP392HU0aVB!$cPHjdK}yi7xNMp+KVZy3_u}+lBCloJ&C?#NE@y$_{Uv83*iV
zhDOcv`=|CiyQ5)C4fghUmxmwBP0fvuR>aV`bZ3{Q4&6-(M@5sHt0M(}WetqItGB1C
z<I^6UkIt-$#>CU-)_n-VD;(6T1%0(@6%U`UgUwgJCCdXvI#f%79Elbg4^yucgfW1^
zNF!|C39SaXsqU9kIimX0vZ`U29)>O|Kfs*hXBXC;Cs9_Zos3%8lu)JGm~c19+j8Va
z)~kFfHouwMbfRHJ``%9mLj_bCx!<)O9XNq&uH(>(Q0V7-gom7$kxSpjpPiYGG{IT8
zKdjoDkkMTL9-|vXDuUL=B-K)nVaSFd5TsX0v1C$ETE1Ajnhe9ept?d;xVCWMc$MbR
zL{-oP*vjp_3%f0b8h!Qija6rzq~E!#7X~8^ZUb#@rnF~sG0hx^Ok?G9dwmit494OT
z_WQzm_sR_#%|I`jx5(6aJYTLv;3U#e@*^jms9#~U`eHOZZEB~yn=4UA(=_U#pYn5e
zeeaDmq-$-)&)5Y}h1zDbftv>|?GjQ=)qUw*^<?4Qug_064CXwb#|8rsLI=CxwcLMz
zHjR>CkcAG#o%I8i186AbS@;qrezPCQYWHe=q-5zF>xO*Kk|VTZD<z}3d$|FhBQ-9;
zqXBmTu5iJ4hp;Gs+m1*Vmumq~RFG#&_!3&SYE_j%M~{|r=-+{W#^QN9Rw+$1+{%u6
ztTEKsz<ibfGp7oB*V_V?r`nNQhejgKB9M!aO%fqpSOLYrkD4c!#z&skA|))ZO{*`k
zJiGGtOZX4GRzoXF0Sim5*SgIy_(pgr5T!XTFXdq0d|T}AI&|e)cS&s~zT6ahCt1l1
zT|;3k<!J&(m^dI)-x>;t={XqrKfR|{itr~k71VS?cBc=9zgeFbpeQf*Wad-tAW7(o
ze6RbNeu31Uebi}b0>|=7ZjH*J+zSj8fy|+T)+X{N8Vv^d+USG3arWZ?pz)WD)VW}P
z0!D>}01W#e@VWTL8w1m|h`D(EnHc*C5#1WK4<w{2hR-Qu&>G|C5ViXO$YzKfJkda#
z2c2*qXI-StLW*7_c-%Dws+D#Kkv^gL!_=GMn?Y^0J7*3le!!fTzSux%=1T$O8oy8j
z%)PQ9!O+>+y+Dw*r`*}y4SpUa21pWJ$gEDXCZg8L+B!pYWd8X;jRBQkN_b=#tb6Nx
zVodM4k?gF&R&P=s`B3d@M5Qvr;1;i_w1AI=*rH(G1kVRMC`_nohm~Ie5^YW<LklCW
zh?^^=kj`^XEK69~Ax?b9{=IxK!iA{9zQKS!OQBs_q=bOVeg`#Vca$a>YqZMV2<`J*
z`i)p799U_mcUjKYn!^T&hu7`Lw$PkddV&W(ni)y|9f}rGr|i-7nnfH6nyB$Q{(*Nv
zZz@~rzWM#V@sjT3ewv9c`pP@xM6D!StnV@qCdO${loe(4Gy00NDF5&@Ku;h2P+Vh7
z(X6De$cX5@V}DH<wT@;7^h<^tnD`tVgF}!?NK3t*7|j8KqjdQQ5uVLe^{23OdE6J2
zi-0=62~C{kkvIc0W}2vwiFG56-KK++>XG?K^6mV>XiT768Ee^ye&Cs=2yefVcFn|G
zBz$~J(ld&1j@%`sBK^^0Gs$I$q9{R}!HhVu|B@Bhb29PF(%U6#P|T|{ughrfjB@s-
zZ)nWbT=6f6aVyk86h(0{NqFg#_d-&q^A@E2l0Iu0(C1@^s6Y-G0r32qll>aW3cHP#
zyH`KWu&2?XrIGVB6LOgb+$1zrsW>c2!a(2Y!TnGSAg(|akb#ROpk$~$h}jiY&nWEz
zmMxk4&H$8yk(6GKOLQCx$Ji-5H%$Oo4l7~@gbHzNj;iC%_g-+`hCf=YA>Z&F)I1sI
z%?Mm27>#i5b5x*U%#QE0wgsN|L73Qf%Mq)<FQ&Mo1iYC<$9{B|SppV_E$JUy-=H_z
zz06Apsrs`HuztjT{?R!3GII}_EH}V95C%MNxuHj3L8US5&QZJw;l36?K5xH)l_K$0
znUElOm7XhO^~p0~<$C#{JI1?hYqwbOII}7vSC7<aaLW3&Kdf9hYl)%8s&MQZ0xq59
zwdB?`0fITRbCh@%dfBfS*U0Po3sibYz+b*`qUs70(HMcC_8}J)bfq(-w&j^SzhbH(
zn{4Ql+5QMT3&Y1k8*+0S&jQqiY{<T32~TidB2;UMTbhU4Qj8dR#m&VJ6+W<d8N^6l
zY!*TBo0I_bAo_K=Ui&{JN?&n%mE`n?vTU0YGMTwf5TnsP=8#pum}G~F<1i^z%*4Hm
z1I0;>QW@O+)a;#mQN?b8e#X%wHbZyA_F+`P%-1SZVnTPPMermk1Rpm#(;z^tMJqwt
zDMHw=^c9%?#BcjyPGZFlGOC12RN(i`QAez>VM4#BK&Tm~MZ_!#U8PR->|l+38rIqk
zap{3_ei_txm=KL<4p_ukI`9GAEZ+--)Z%)I+9LYO!c|rF=Da5DE@8%g-Zb*O-z8Tv
zzbvTzeUcYFgy{b)<cdkL$=7b>8Q6+BPl*C}p~DiX%RHMlZf;NmCH;xy=D6Ii;tGU~
zM?k;9X_E?)-wP|VRChb4LrAL*?XD6R2L(MxRFolr6GJ$C>Ihr*nv#lBU>Yklt`-bQ
zr;5c(o}R!m4PRz=CnYcQv}m?O=CA(PWBW0?)UY)5d4Kf;8-HU@=xMnA#uw{g`hK{U
zB-EQG%T-7FMuUQ;r2xgBi1w69b-Jk8Kujr>`C#&kw-kx_R_GLRC}oum#c{je^h&x9
zoEe)8uUX|SahpME4SEog-5X^wQE0^I!YEHlwawJ|l^^0kD)z{o4^I$Eha$5tzD*A8
zR<*lss4U5N*JCYl;sxBaQkB3M8VT|gXibxFR-NH4Hsmw|{={*Xk)%!$IeqpW&($DQ
zuf$~fL+;QIaK?EUfKSX;Gpbm8{<=v#$SrH~P-it--v1kL>3SbJS@>hAE2x_k1-iK#
zRN~My-v@dGN3E#c!V1(nOH>vJ{rcOVCx$5s7B?7EKe%B`bb<ObEfr$*j`M4kQDF;^
z$)<Ep9ojr!n+?@Fw@c#)27Pxjsx};#DZb2Y&}NG1g?*LP#{7$EO((8w#!RKKv8+s!
zmv=7waG@<#niQdBy4YRHFJYrx3N5-4$8=1zh$))fFm1IA*BJyQ#7kx>x(8}km#t2a
z1A~COG(S4C7~h~k+3;NkxdA4gbB7bRVbm%$DXK0TSBI=Ph6f+PA@$t){_NrRLb`jp
zn1u=O0C8%&`rdQgO3kEi#QqiBQcBcbG3<hi%oScTlJZoPB79~!s@@L+Y$N(R9Hx$4
zV5*-?&7y$QGs;^8evwiVm6-BL!fam-?7{M?Mz0#DP0!a3UjblLaFBgObQXLaEa7QC
zDpwbhz~C1Q3t7E>wqPrJ8+0r<`L0Co-n8y-NbWbx;}DTq@F<bOLiVNd9q}vMnA%*P
zTc~6w2P#ybh5aX13s80NZKZzTE}CBpehKRdZ&stlyBvCO(}_UMi#aFz(mWPfxAV>D
z1b)B$b>Nwx^2;+oIcgW(4I`5DeLE$mWYYc7#tishbd;Y!oQLxI>?6_zq7Ej)92xAZ
z!D0mfl|v4EC<3(06V8m+BS)Vx90b=xBSTwTznptIbt5u5KD54$vwl|kp#RpZuJ*k)
z>jw52JS&x<m<yy)V<(rx5_MF@vO5QoqfZORCiaA`)1Q1l`h33tsqFS>)9&g3RDXGV
zE<BxgBMRv3bbTT<YJb#v^>lux37>A=`#5(UuRx&d4qxrV<38_w?#plbw03l9>Nz$Y
zZS;fNq6>cGvoASa2y(D&qR9_{@tVrnvduek+riBR#VCG|4Ne^w@mf2<b0Xn$^ftK!
zUQQ=+H;=)de`O~X*S(2dn6pgOy`*q?!IKY(-85~U2Gh@^{B#GK&9Ncm`w8x~G+T?}
z6e!Pw(-fFN3z<tqfaPSdVWzKF)K0r^?aG98G(0wAdhqEV1~UDK(?ujKF>Y;-k90%V
zpA6dVw|naH;pM~VAwLcQZ|pyTEr;_S2GpkB?7)+?cW{0yE$G43`viTn+^}IPNlDo3
zmE`*)*tFe^=p+a{a5xR;H0r=&!u9y)kYUv@;NU<!rIJfL66+0T%iYwZHC+LgKD!GA
zL2IuLX8^c^t$6){JO|ImKi}$)?tUU7S6^uOiZVyRkp*dn$0c%G9<kf89AYO}@9xx9
z9}W$KUx0n+k54zm6XQXH_DXzKo%rRiC%91{+rV3g0E`)|+bwNc=}IWGAm7f`2V%5y
zS}OkQ*mi}5r`P)1wTtFn;fX9JUa%smlf(Kflj`0L3o$nkMG5TjYE`e$?2%mjPk2<<
z@~hjWcveKi26iD=1NNvKVI*7kpxS`Rlp3vF-c*L}o_Sy)!+^VjHA9NQDkE0RH5Qc2
zX$jj#Cpi}!WUiJT^gN>KZ)`u-KFTv0S&FTEQc;D3d|KEKSxirI9TtAWe#h<v>vOXV
z>807~TWI~^rL?)WMmi!T!j-vjsw@f11?#jNTu^cmjp!+A1f__Dw!7oqF>&r$V7gc<
z?6D92h~Y?faUD+I8V!w~8Z%ws5<LtywtI0$0z0`d=2}3Z9=A0Nhc(xGwuR&T+hlz^
zpBVI%AC{;K=EAv4;Pu18IQ)TS;&I$>S{20(AkaTZc>=z`ZK=>ik1td7Op#vAnD;8S
zh<>2tmEZiSm-nEjua<oYdF>WVE)aUXp$BumSS;qw#Xy7-yeq)(<{2G#ap8z)+lTi(
ziMb<AO!rU+vI9Y9hi52#H5Z_Oqm-X<*c418GCXRIFf$<tw{O+jDt#P>-iig6!==yk
zb6{;1hs`#qO5OJQlcJ|62g!?fbI^6v-(`tAQ%Drjcm!`-$%Q#@yw3pf`mXjN>=BSH
z(Nftnf50zUUTK;htPt0ONKJq1_d0!a^g>DeNCNpoyZhsnch+s|jXg1!NnEv%li2yw
zL}Y=P3u`S%Fj)lhWv0vF4}R;rh4&}2YB8B!|7^}a{#O<YYC_ba{El6}Ou&a<3Fhm!
zWDn>ac|%oFdMToRrWxEIEN<0CG@_j#R4%R4i0$*6xzzr}^`rI!#y9Xkr{+Rt9G$*@
zQ}XJ+_dl^9@(QYdlXLIMI_Q2uSl>N9g*YXMjddFvVouadTFwyNOT0uG$p!rGF5*`1
z&xsKPj&;t10m&pdPv+LpZd$pyI_v1IJnMD%kWn{vY=O3k1sJRYwPoDV1S4OfVz4FB
z$^ygjgHCW=ySKSsoSA&wSlq83JB+O-)s>>e@a{_FjB{@=AlrX7wq>JE=n@<VNDUqc
zh~=-r|3**!2l@2xq|N`ONQ7-|ogAJ1GDdO+<~Ax$1`bZf4)lgr29AzN8kU|oLufuD
z$&K}$P+@i;(ygU(QN|<5(uAyWa738GnWHWiN6I6!PGnV8a)&y`P4!vLsPn~CvoI^o
zKg{wpJQ(t-7z+J-&O^Qme5kxli`1H)xp|*RZZ0#qPA0i-+9of1z8>}@fba(;n4EG|
zge1i<k^HPrn!xB(<|f-FjrKZqoqiCL1e<flKspuwQ5ZoKOYu()w<!e?N#;wLE7z*4
zG(Bzt+nS_Z*J>)?NE@M@DC5eEv<yyjW!;#w_S-OlYe-x;CkMMU#HIo*%j_sJ|Cy#)
zkZ&q)Chy)V5L&8V(Tf)awl$WSJ|3mss6}ULPPV09D<^2JcPWnBLUXEA2g(DskHZEX
zh<A}_J$sedQb5DQjysSgNx53|i{tv92H!GN?ub3~h|&e&S}j)f4$Dhjf8=}CibXBg
zRNnPI#Uh&jhJ=FG`GQQlD@SFHCdozVHPGau7-azcajduK#~&3I9~JgFQV~wyE@*Hy
zSfZ<l*RlF4OEN#Yp?10_&j<V*9k}ya)dzR97ej*2%CqTGW9~ok2aaYM7AiFPZi>4;
z#R~0aNssmFHANL@-eDq2_jFn=MXE9y>1FZH4&v<}vEdB6Kz^l)X%%X@E#4)ahB(KY
zx8RH+1*6b|o1$_lRqi^)qoLs;eV5zkKSN;HDwJIx#ceKS!A$ZJ-BpJSc*zl+D~EM2
zm@Kpq2M*kX`;gES_Dd1Y#UH`i!#1HdehqP^{DA-AW^dV(UPu|O<jvzCHjx?S&$X_@
zI63*jpF(JzqsSD70uo-eumPq+*u>@Hvr>?X3^~=1i<W7`R1=`lXYRLQ>aRa~AVXbj
z-yGL<(5}*)su2Tj#oIt+c6Gh}$0|sUYGGDzNMX+$Oi$e&UJt3&kwu)HX+XP{es(S3
z%9C9y({_fu>^BKjI7k;mZ4DKrdqxw`IM#8{Sh?X(6WE4S6-9M}U0&e32fV$2w{`19
zd=9JfCaYm@J$;nSG3(|byYDqh>c%`JW)W*Y0&K~g6)W?AvVP&DsF_6!fG3i%j^Q>R
zR_j5@NguaZB{&XjXF+~6m|utO*pxq$8?0GjW0J-e6Lnf0c@}hvom8KOnirhjOM7!n
zP#Iv^0_BqJI?hR5+Dl}p!7X}^NvFOCGvh9y*hgik<&X)3UcEBCdUr$Dt8?0f&LSur
ze*n!(V(7umZ%UCS>Hf(g=}39OcvGbf2+D;OZ089m_nUbdCE0PXJfnyrIlLXGh2D!m
zK=C#{JmoHY1ws47L0zeWkxxV=A%V8a&E^w%;fBp`PN_ndicD@oN?p?Bu~20>;h;W`
ztV=hI*Ts$6JXOwOY?sOk_1xjzNYA#40dD}|js#3V{SLhPEkn5>Ma+cGQ<o%qL<Q%n
z)p-m&QK15=GqmN<;E8=EA~$;A7zz7Gvv;^(c7@{n0c?w(sxQfHbV|Xpndn6B*C-VZ
zEgAOnx3+LF)@Qm`l532w37Q*TT(kKzuIq(^E9dw?fI0f$1gWd$=7wA+rX4l87liYB
z-Dx?;<Z)WFw+M|B0>i*#`g-*g56Q&@!dg)|1YpLai3Bu8a;l2fnD6&)MZ~hS%&J}k
z2p-wG=S|5YGy*Rcnm<9VIVq%~`Q{g(Vq4V)CP257v06=M2W|8AgZO0CC_}HVQ>`VU
zy;2LDlG1iwIeMj?l40_`21Qsm?d=1~6f4@_&`lp~pIeXnR)wF0z7FH&wu~L~mfmMr
zY4_w6tc{ZP&sa&Ui@UxZ*!UovRT})(p!GtQh~+AMZ6wcqMXM*4r@EaUdt>;Qs2Nt8
zDCJi#^Rwx|T|j_kZi6K!X>Ir%%UxaH>m6I9Yp;Sr;DKJ@{)dz4hpG>jX?>iiXzVQ0
zR$IzL8q11KPvIWIT{hU`TrFyI0YQh`#>J4XE*3;v^07C004~FC7TlRVVC}<}LC4h_
zZjZ)2*#)JyXPHcwte!}{y%i_!{^KwF9qzIRst@oUu~4m;1J_qR;Pz1KSI{rXY5_I_
z%gWC*%bNsb;v?>+TbM$qT`_U8{-g@egY=7+SN#(?RE<2nfrWrOn2OXK!ek7v`aDrH
zxCoFHyA&@^@m+#Y(*cohQ4B76me;)(t}{#7?E$_u#1fv)vUE5K;jmlgYI0$Mo!*EA
zf?dx$4L(?nyFbv|AF1kB!$P_q)wk1*@L<tI7)z|y;PAN-6?~)dmYAN5!m0%1M~LEF
z_<N}*>0>m<Jl7mrsl1+U-CsdkE{<0<uQ+i9Ep&l3?XC6Qx3CRj#I$i6V|7Gr^YPk6
zhuACdME6iF5tO0Sp&bG~!gP0FuN>SC(A8f4Rgmv1HG;QDWFj<(1oz)JHr+cP|EPET
zSD~QW&W(W?1PF-iZ()b|UrnB(#wG^NR!*X}t~OS-21dpXq)h)YcdA(1A`2n<G!shH
zgorijds{aB^i!X<=rORMTpWa4MGZ>zVFax9rx~WuN=SVt`OIR=eE@$^9&Gx_HCfN=
zI(V`)Jn+tJPF~mS?ED7#InwS&6<j%wm+|6WL34>OfH;qDzI_8@t>In6<x;*Gw*Yw%
z?iUIMsuaBB=x1Cv6H=|API5-Qjn3UQjCj{RP52pTmlOdCLoPdsSxY-;mqhdyM_>nl
zo}q{Ds*cTG*w3CH{Mw9*Zs|iDH^KqmhlLp_+wfw<q}t*Kwoi;C?nA6#x0ph>IS24G
z{c@fdgqy^Y)RNpI7va^nYr9;18t|j=AYDMpj)j1oNE;8+QQ)ap8O??lv%jbrb*a;}
z?OvnGXbtE9zt;TOyWc|$9BeSGQbfNZR`o_C!kMr|mzFvN+5;g2TgFo8DzgS2kkuw@
z=`Gq?xbAPzyf3MQ^ZXp>Gx4GwPD))qv<1EreWT!S@H-IpO{TPP1se8Yv8f@Xw>B}Y
z@#;egDL_+0WDA)AuP5@5Dyefuu&0g;P>ro9Qr>@2-VDrb(-whYxmWgkRGE(KC2LwS
z;ya>ASBlDMtcZCCD8h+Awq1%A|Hbx)rpn`REck#(J^SbjiHXe-jBp!?>~DC7Wb?mC
z_AN+^nOt;3tPnaRZBEpB6s|hCcFouWlA{3QJHP!EPBq1``CIsgMC<oiD!Y(*FZo;w
zV*qp3$&_F}b^rx3Q6+fj$mjwGERLlcI%ETo!VJMbFW)lV>YD#80(bsKpvwO)0#)1{
zos6v&9c=%W0G-T@9sfSLxeGZvnHk$SnHw57+5X4!u1dvH0YwOvuZ7M^2YOKra0dqR
zD`K@MTs(k@h>V<EOiB@Xpd=4(gUz@lE$jYOjyzm1&}<*!Rs!qBN&`PnJiRsJa<k3x
zn#=TN^5f&_3c?qRw>eI5UYI%n7#3L_WXVnpu$Vr-g}gEE>Y8ZQQsj_w<T+=4cBDUI
z3zkp6^ti5DdbC^)?WXC;d}!usTUL-UzoIL5Du@K0<KA78gAyOK-yYRC*Dg5fjkwYO
zROF(tM+b{E9J8!0hI##?+y~?U%m;fQFs^v{N_m5_8?LCHDZDZ$0*kX%bW|LbZjWQL
zTB=`B<b3rU-=J8L4|s0;0ViUpp^UW}G1IV2Ab5ThMPGCza-Xd~0YC60zSbU;gKl$N
z9~Y%0O9IHHIzU>bl&t6nj{;ga4q8SN#Z<H$(mh`y2lbG1u)2RcX{0)zUiZ0`we*v?
zRWs*-YCyVK70gkZd#3DJF$S`(#`gu(PtwS=>6cBZepMoyv7MF-tnnZp*(8jq848yZ
zsG_fP$Y-rtCAPPI7QC^nzQjlk;p3tk88!1dJuEFZ!BoB;c!T>L>xSD<#+4X%*;_IB
z0bZ%-SLOi5DV7uo{z}YLKHsOHfFIYlu8h(?gRs9@bbzk&dkvw*CWnV;GTAKOZfbY9
z(nKOTQ?fRRs(pr@KsUDq@*P`YUk4j=m?FIoIr)pHUCSE84|Qcf6GucZBRt;6oq_8Z
zP^R{LRMo?8>5oaye)Jgg9?H}q?%m@2bBI!XOOP1B0s$%htwA&XuR`=chDc2)ebgna
zFWvevD|V882V)@vt|>eeB+@<-L0^6NN%B5BREi8K=GwHVh6X>kCN+R3l{%oJw5g>F
zrj$rp$<x5{%Dd&c{zb=gqx|vjc$u+fQ~<=xa?|zwdGhl8?Rl}lAn?rTtF})J<_=<P
zZ<#BOvz#B`F%`?6pTYJ{XYKa0#X{@Itb@mF36=$j02&pwTyYm8wM_$O&f-@(m*T>9
zhepggNYDlBLM;Q*C<EOEy)e)av?`jZL9pOkqXYZ-rxvq`ZR8Dyp?#G?%a7?}>B&%w
zW+aY{Mj{=;Rc0dkUw~k)SwgT$RVEn+1QV;%<*FZg!1OcfOcLiF@~k$`IG|E8J0?R2
zk?iDGLR*b|9#WhNLtavx0&=Nx2NII{!@1T78VEA*I#65C`b5)8cGclxKQoVFM$P({
zLw<xqIAwsC7xGcp);pEM8LO0_(juAhdE3&t7P&jX@-eLqa<ai0ronDF_^r<;v+U|@
zC@$mmN@tejs)}KagB^2WXSC}ikckk+&RWL=^a{_cQaKr&N)+{KR&7~tyf}%abFX6q
zG71?g)D|Tr(C01Nx%MTj!z}=Yr1f20h6VmGKd5I!KY`Be@*2IG1gFw|%Cn(qlPd0O
z84OzfiAvR7_?7y;9-_0j7*MBGvi}S5uA$+K{Vpz->JKo9!9xN4Q8a2F`xL&_>KZfN
zOK?5jP%CT{^m4_jZ<Ofk;wOk0vVOYQ2{V0>ahnn4DrqgTr%(e_({|z2`C2NrR6=v9
z*|55wrjpExm3M&wQ^P?rQPmkI9Z9jlcB~4IfYuLaBV95OGm#E|YwBvj5Z}L~f`&wc
zrFo!zLX*C{d2}OGE{YCxyPDNV(%RZ7;;6oM*5a>5LmLy~_NIuhXTy-*>*^oo1L;`o
zlY#igc#sXmsfGHA{Vu$lCq$&Ok|9~pSl5Q3csNqZc-!a;O@R$G28a@Sg#&gnrYFsk
z&OjZtfIdsr%RV)bh>{>f883aoWuYCPDP{_)%yQhVdYh;6(EOO=;ztX1>n-LcOvCIr
zKPLkb`WG2;>r)LTp!~AlXjf-Oe3k`Chvw$l7SB2bA=x3s$;;VTFL0QcHliysKd^*n
zg-SNbtPnMAIBX7uiwi&vS)`dunX$}x)f=iwHH;OS6jZ9dYJ^w<lfOFp$uO|PqoKe=
zJgz#|0g$+Tl)t5#cE|+_jV8KY{V%c$Laj$p@i%@32}FkwW7Hb8JMPZxY9fn_slCD}
zH625V)Eo!6f!MlU+AeeHsx?FceCG-wq-|dg>Q=F#j9U{wJ9eGH<w^dj^Nd_mNS^fO
zCRQ7>^#vzm$HIm->xSO>WQ~nwLYQ8FS|?l!vWL<%j1~P<+07ZMKkTqE0F*Oy1FchM
z2(Nx-db%$WC~|loN~e!U`A4)V4@A|gPZh`TA18`y<vaS#sbRre(fEOZ<ZQ`PS>O1{
z(?VA_M6SYp-A#%JEppNHsV~kgW+*Ez=?H?GV!<$F^nOd+SZX(f0IoC#@A=TDv4B2M
z%G-laS}yqR0f+qnYW_e7E;5$Q!eO-%XWZML++hz$Xaq@c%2&ognqB2%k;Cs!WA6vl
z{6s3fwj*0Q_odHNXd(8234^=Asmc0#8ChzaSyIeCkO(wxqC=R`cZY1|TSK)EYx{W9
z!YXa8GER#Hx<^$eY>{d;u8*+0ocvY0f#D-}KO!`zyDD$%z1*2KI>T+Xmp)%%7c$P<
zvTF;ea#Zfzz51>&s<=tS74(t=Hm0dIncn~&zaxiohmQn>6x`R+%vT%~Dhc%RQ=Cj^
z&%gxxQo!zAsu6Z+Ud#P!%3is<<Hm1si9=xqaKmm4T>%*dJXe!*wZ-yidw|zw|C`cR
z`fiF^(yZt?p{ZX|8Ita<vJx0h&Nk}<hSUDZRD<|&3qOD7?uWd_MHD?1H1opJQfK+d
ztW7^b`(!2!%0cA+sr8KiD%0qLS0%s<S9`%W(R&h~TE3h!$P#l4TvGq@cp|%E&_3^y
z)wRvYW~ti8P<ruWC7N_&I5xZQLj$BkxrB4gt)%%>)UC$=fg6wOve?w+8ww|^7OQ0d
zN<D?s`3%O<hBdpfAJ73<>(3dmJ@mV8>74I$kQl8NM%aC+2l?<ziJOP3Nj$NBO$Zm5
za7tbYQO&zQsL8Y=lnnT@|2VtTDL)By(tj`_(D_k?kCmSKk>ZQ2pqkMs{&q(|4hwNM
z^xYnjj)q6uAK@m|H$g2ARS2($e9aqGYlEED9sT?~{isH3Sk}kjmZ05Atkgh^M6VNP
zX7@!i@k$yRsDK8RA1iqi0}#Phs7y(bKYAQbO9y=~10?8cXtIC4@gF#xZS;y3mAI`h
zZ^VmqwJ%W>kisQ!J6R?Zjcgar;Il%$jI*@y)B+fn^53<Z3}PY#ocRlS*4m6#eoc*D
z^G4bdDB%8R-xxSKw5X5N!@e@w<Cs)^$joR^dZ5p4{FPwT#LD@(RTvw82CuOhFPl}+
z-t<#hW3VmFyMwO^TrJ!LH~VAE$u^5iDMQ8BrteWL+0G@xEL`0_ME&_{X8`Ca!4abw
zAeWBDLb|k>jQd0`)=C~w%Lo?qw!q3fVi{~2arObUM{s=q)hgBn64~)W0tyi?(vlFb
z>tCE=B1c<v0(}7W0uhz(M(kMsw*YQ}w47E-&mhm6Ef1{TWF*PKwy=?0P*XW5qxT5`
zMC-?k44FrWAoLah+xtR$qBCfvVaKNt!O1KcH}mpwZ#c?O<Mu=BC@E&wD%$^p;{`J1
zUetF6jPkG{-O6%w->bfyY=V38fUGN(#vmn1aY!@v_c70}pa(Lrle-(-SH8Nd!emQF
zf3kz0cE~KzB%37B24|e=l4)<zlUVRGOPfvL4Ry<L1?}(JT*BC*@(p8NKIw11BVO*c
zzi`zk>L}g1AF@v%J*A;5F7li!>I0`lfO9TR+ak<Y$te4Zeg@NF38$FC=!n~!7Q}qT
z3E%7{E;d&)fB#sHG`!-w#VhkdO+nVCm}QNg+6YL|h|F__jT5<r@Eopd=*+*zY=lBX
zXJQnD37?aWyLx;>`xyqWnj5iwJ$>t_vp(bet2p(jRD;5Q9x2*`|FA4#5cfo8SF@cW
zeO{H7C0_YJ*P@_BEvm2dB}pUDYXq@G1^Ee#NY9Q`l`$BUXb01#lmQk^{g3?aaP~(*
zD;INgi#8TDZ&*@ZKhx$jA^H-H1Lp`%`O{Y{@_o!+7ST}{Ng^P;X>~Bci{|Qdf1{}p
z_kK+z<tDd@=j>L;>D30r6~R?|h!5NKYOi6X&I5)|ME+NG>d9^`hxKpU^)KBOpZiU^
z;|SzGWtbaclC-%9(zR-|q}kB8H&($nsB1LPAkgcm+Qs@cAov{IXxo5PHrH(8DuEMb
z3_R#>7^jjGeS7$!`}m8!8$z|)I~{dhd)SvoH9oR9#LjO{{8O&r7w{d9V1z^syn&E6
z{DG0vlQF_Yb3*|>RzVop^{$mWp|%NDYj@4{d*-@O^<(=L=DMFIQHEp-dtz@1Rumd;
zadt^4B#(uUyM6aeUJkGl0GfaULpR!2Ql&q$nEV^+SiDptdPbuJ=VJ)`czZ@&HPUuj
zc5dSRB&xk)dI~;6N?wkzI}}4K3i%I=EnlKGpPJ9hu?mNzH7|H0j(mN3(ubdaps3GM
z1i+9gk=!$mH=L#LRDf4!mXw0;uxSUIXhl|#h*uK+fQPilJc8RCK9GNPt=X^8`*;3$
zBBo77gkGB5F8a8)*OR10nK&~8CEMPVQyhY>i`PS{L^-*WAz$ljtU%zlG1lm%%U4Zw
zms0oZR8b|`>4U1X*9JLQQ>m9MF5%ppoafz^;`7DbmmIENrc$hucekkE4I83WhT%(9
zMaE;f7`g4B#vl(#tNP8$3q{$&oY*oa0HLX6D?xTW3M6f<^{%CK4OE1Pmfue`M6Dh=
z&Z-zrq$^xhP%|hU&)(+2KSSpeHgX^0?gRZ5wA8@%%9~@|*Ylux1M{WQ4ekG(T+_b`
zb6<oz^lbvev~V++$rC}$*BvgYAXG{?KcIztrX(y~IHHGjmZtf0C<OUlY*_4T&#6N5
z9I`6{eOqqp+TfjYP;y%aa_8?>I)QRGp%fRF)^T?i^j&JDBhfNU9?>Sl6WVMM%S?7<
ze|4ga<QQ%CFfC2!(F<jzks4u#PD$-vhPNI`buJHWN_8GSmKZP&!ezm)`pb_ufq%|e
z65VysTK{G$XMb<1f0?oTC!h20sft+iltsTHV$jShMg}xAzdvOAnbl^SH4Io@{;7|w
zcmm=LTua$mgYz83sR&vPS*oAGv0XRT^v%u-f-e|2uvTE0Gexp+zUUh*2Aeh)6}0n*
z6-@#lYzgcXQ9MmM(2<G4N5BX>DbPooB=F4Y=>~_+y~Q1{Ox@%q>v+_ZIOfnz5y+qy
zhi+^!CE*Lv-}>g^%G=bGLqD(aTN;yHDBH#<M$qg){m{{Q?t%NZ{J?a__4EZDzviFa
zm*MAJF9ZVu^8C9+ne|_}FKOUtCS+x5>tOC=X02}QU~Xdme``Wn>N>6{VwgU~Z>g+0
zxv0`>>iSfu$baHMw8(^FL6QWe;}(U>@;8j)t)yHAOj?SdeH;evFx-kpU@nT>lsrUt
zqhV}2pD^5bC4786guG1`5|fK@pE6xcT#ns)vR|^?A08G62teHaE&<GZV+LvneF;S#
z68Wn}!rRkgqGaX6A_ow8EmDIWBl|GIqJxlxNhE#>p`ZrCBj_Swt*~dVt=5*RK6Y{%
zABqK$X59BnrK3r3u=wxklRnA1uh+q`<b)RM<`VYR#Tj&jFv8I!vxF0~E2AGvt})TF
zM4DI~n<>?T0kE1YhvDWF4OY#<(+V|R@R%tdkq2huF(!Ip+EpZF3zr*|9pmKHPo)Cu
z;H+^s&`Ql}u=Jt~ZWj`bAw|i-3#7(2WuRU3DU{BW8`?!O?YO1M$*MMTsaEM!5Jyp~
z!gp6yR4$O%wQ8%dyz43ZPeoJwy;o;yg=S0^Y}%|)to>=N^`!3VMf1~}OZ`Dl$q&|w
z9$!i3!i1uAgPTuKSWdBrDr*N$g=E#mdqfj*h;Z}OG`{n245+g;IKfdn!&gF2OtHaD
zyGDzj@@d2!P(_Ux)3v;1ABTj__{w*kaRF-1YVU`})Acgk?(T*1YqEve3=5)8bkZK*
z!Tu<iq|P2`%F?J$)p|je2QMGIMZ2uQp_h@y-2G1Sdah5zN$MlgauhJ_S?DV|LPgo3
zy5@seq+HX~oFS@o{3}f1b%{xH5rZ+``=g_jA~NT11)`8ASoww?bFpmkOYPKJLN%zY
z#G=)BG8w4pvGQhd)zYf%ZbJ93=mTw|TF&1&U993<+v<$LGO!)AJOm399>s*e$h@^u
z>#zV0771Bix~r&h2FJ9)%N{>s>?2tk1$bId)1#G;OKgn-U8jUo^<QlyOD0m1h~yc9
zYT#r5TFa%e#Znr=VS)J8%MMo_req|4=S?B33f1PSc%hek1)1YQU-d5MwWR)q{_5Dw
z5=9oVB{#GZF0eecabm;6NDFL9BCyNJ=OBBmn{iM3umir}Zs#^GkuV;mNx)+(nzUhm
za=8_JmcdIqGaDw8$Nac*nUyqFG3QM(c@Q7<O+zJi+6L1W`{B!c@eUQHYVv2<_pzKm
z{Yazs3^VMCH}1;u!_KAXh~!;i#>AK;Hu)hQEi}swD(264kAS-SBCD$R(Ro0rh8~Le
zzRwxbz_JHDbD+hTX15AWmVw!#rC)-zeZahQQmo6F<p_)?;#yLR0`(}T(+Mw<^A<!v
zc35M9X*Jx1F#{d03NAsMf@PKoBty#Ypo0aqna_Kuym{4XET%)T9bJwPZ{^tb5jBfr
z*c%no6c#ooY>G1)ah3uuyIuTMof}RO!`Y3^Fxn_-G$23RDOh(@NU?r6`*S?#E50)w
zpcsgDZ-iO{;EesgDQq9;p*C#QH(sp~2w^zAJWaUL%@yo)iIL6y8;e_}=dwQc%k%;H
zFt5lenH*`}LWd+fPqi;exJeRZgl&nLR%|a!%1x0RQ54cgyWBYrL>sskcAtPxi&8c(
zw_K?sI*3n%S;lKiYpveBN08{rgV&-B1NN5Jiu07~%n#%&f!(R(z1)xsxtRBkg#+Lv
zh21zX?aYDd_f}qdA`Os<SbTR6gd5&=nL3UG#AuZUynK!i-YNc=2BCEbZ?L)$b<a8R
ztX7zJC>*j!eC<5)iUJ&Twj7?*p%vEOGElGhpRZsccM!<<I|PU&)~CLHRCdx^CfXu#
zsW!u!iTd$hJWe)wvI32v??s)XOU|ph)!DfqAs>k}DeC;TY;rULQs3e}lZyP#UVb=6
zB$Dkm2FaHWUXr7<{R&46sfZ)&(HXxB_=e`%LZci`s7L6c-L7iF&wdmTJz`*^=jD~*
zpOZ@<L`81Cqd*Yjy{6K>jcq8LezVkE^M6D9^QgZqnX&x*mr1_<yBUD`Uyb@wHjYjP
zR#yL2!>Cf#R9R3&{i3%v#}V$UZzGC;Or*=Dw5SXBC6NV|sGZp^#%RTimyaj@!ZuyJ
z6C+r}O1TsAzV9PAa*Gd!9#FQMl)ZLHzTr99biAqA(dz-m9LeIeKny3YB=*+|#-Gq#
zaErUR5Z*Wh^e<+wcm70eW;f-g=YTbMiDX)AznDM6B73)T4r%nq+*hKcKF?)#vbv?K
zPMe=sFCuC*ZqsBP<kLDXB+;25f0Z(q<`_UcS2#i?okbKY2v?D0X1#iP{=mJbgcND@
z&#MyXSlI=3U0Ep8#+@<?%U2V_k!NYVk+@qUfG#m=g{?a)8cJ##l4wA+d6wL8LN&#T
zf0sflzjQ~tY85=sKWgn>h-?g!m*O`}6<}Pfj}&#3Y1n9|Y@cUdD5GX_)6Sx9pPfS7
zxkt?g6ZwJ+50C7qrh6dMFmr7qah`FskT_H=GC92vkVh$WfZa2%5L99_DxyM{$#6HQ
zx$VR-Wwt!q9JL2{ybEGJr$^?!V4m_BqDqt!mbs=QjHf340+^a{)waVvP0+98(BA$M
ztWr&sM=juyYgvf`(SC}+y@QtYgU>0ghJ6VbU}|kEraR&&W%#;!#KI?le%g`e>ZV<h
zy-v0ocOwQIK`Y=pL1GCjC0E5)ML4*UQ=f3`r>PiDrneh#&1(Y?uiMo^f5qo@{JE<b
zbyE=O{D7Qvv(BAPE3PepyXNc{qS%VT<a4X6oAW#|L(A)d#}HD$tYR*#$<xHb{D5S?
z#hQC*tmq@dwj2X@i`K;kOsJoV2<+AncW1k=WIExam*|8Z5AI?tB7BM(oxVe`c9BSp
zj>r(p9>8GhDa+PC9yG;lX+D?hQ^fZB&Sdox219zUj_5;+n<0@Wi3@DK`MU8FM!OFJ
z8*_mTA-u!Ab#95FRVWTIqAL#BVQGxE_s?>Ql|@0o9vos&r<_4d!+Q6(_270)6#lu$
zV!j$a?_V0I<(3Z=J7C-K0a^Kc1Go9p&T6yQeAD+)dG-$a&%Fo0AOte~_Z&_m2@ue~
z9cKFf-A41Dz31Ooj9FSR`l?H5UtdP?JS=UU$jF#znE1k@0g%K?KQuwZkfDI3Ai)(q
z#x_Yo6WR_Y@#6I_02S&NpcP<%sw!!M_3#*8qa+*4rS@x=i{-2K#*Qr)*Q$-{<_(<|
z0730e+rubnT38*m;|$-4!1r6u&Ua2kO_s-(7*NGgDTe##%I>_9uW;X__b_k)xlv$;
zW%K2hsmr>5e^Z~`tS-eUgWmSF9}Yg8E}qydSVX0nYZMX_x94QK?tw2>^;raVTqstR
zIrNAX2`X~|h->dTOb9IrA!i5INpLV}99ES|i0ldzC`;R$FBY5&7+TIy8%GO8SZ37_
zw=^Swk?z+j-&0-cTE|LU0q@IKRa&C6ZlXbSa2vN5r-)*f<3{wLV*uJUw980AFkWN7
zKh{?97GmVu-0rs9FB6ludy|n`gN5p~?y51aJzBg6#+-=0pWdZ2n4xTiQ=&3As-!-6
zFlb|ssAJEJL#s8(=odfz8^9b#@RrvNE4gjuEITzAd7R4+rq$yEJKXP?6D@yM7xZ&^
z@%jnE3}bteJo{p(l`hu`Yvzg9I#~>(T;>c;ufeLfc!m3D&RaQS=gAtEO-WbI+f_#|
zaVpq-<%~=27U8*qlVCuI6z9@j)#R!z3{jc>&I(qT-8IBW57_$z5Qm3gVC1TcWJNc%
zDk?H3%QHno@fu9nT%L^K)=<y5j8-3OLoCwml<m#@a2$TO^5(_-we401|7KH(F{xle
zu=0_Sg!kbGYy25rD<nZeZ{kAp9*E_*bc7OM{Of}>#sRiRNg|=%<E|Ca3Q|0x7HSHc
z{ld(ilKgxux4G(17ZVL?iyT{C;CgvryHzGF6mGg|va{LJT=9ILH6TokVZBad;E0>M
zR;8BE)QA<ZwMA{E(`T1LMmc2??84y!^Qh`i-9QajvsfolrXdPS`Q41!Rg4pyt1mVf
zx*lGrA^{XxdZv_6j?V}P@D!0xPC8+KS#?u1SgI<35ORP@C0bz5Xjdj;rLcsg+tCdC
zgfu@@W1QO6<po<L<4lPGE32&BU=t-L?E|M_jtx~p$PYEIbsn@A+$E4%vB25$MS_c*
zNAYWBv$3w;nQbk(yDwgDE>4#Dsg^EakzttRg9pkfIrF3iVYVM#*_<jJZhSj+=C5$}
z+9-V?yuj3z;(V-yx3cFktR2b%>+#3X+~qeZc^WQJvEyVlO@9=0pl!ayNOh|{j0j^a
z+zi_$_0QKhwArW)sJ$wji;A`?$ecbr?(4x5%2pLgh#wggbt)#T^2R3a9m+>GcrUxU
z*u-WTgHAN*e!0;Wa%1k)J_P(Vdp>vwrROTVae@6Wn04q4JL-)g&bWO6PWGuN2Q*s9
zn47Q2bIn4=!P1k0jN_U#+`Ah59zRD??jY?s;<z4_rwa|@e<wNXd%o&3A}eKXx<yjF
zTTkpcsgA7NIv=}eBSBJ^;M!1|c2jX0s@_}Q)#6>U;k@%q87=dM*_yvLN0->qswJWb
z<x!&rY&2O<B(q~zx_=-q{T8<qLr>Imaj{Ah&`)C$u#E0mfZh;iyyWNyEg;w0v%QS5
zGXqad{`>!XZJ%+nT+DiVm;<DrU}w%S2N4HKC&HhxABT$X!|G*WV|7%39mrQX81uPj
zv#2wpFJ2uxd+9@K%YoXcA{LS)<WFCt@2@sf$I>lahOGmZyeqJ-;D&!S3d%CQS4ZFM
zkzq5U^O|vIsU_erz_^^$|D0E3(i*&fF-fN}8!k3ugsUmW1{&dgnk!|>z2At?h^^T@
zWN_|`?#UM!FwqmSAgD6Hw%VM|fEAlhIA~^S@d@o<`-sxtE(|<><#76_5^l)Xr|l}Q
zd@7Fa8Bj1ICqcy2fKl1rD4TYd84)PG5Ee2W4Nt@NNmpJWvc3q@@*c;~%^Vasf2H`y
z+~U-19wtFT?@yIFc4SE_ab?s@wEUfSkOED}+qVjjy>=eac2^S^+|_3%cjH%EUTJ&r
znp9q?RbStJcT*Vi{3KDa^jr4>{5x+?!1)8c2SqiCEzE$TQ+`3KPQQnG8_Qk<^)y_o
zt1Q^f{#yCUt!1e(3;E6y?>p+7sGAYLp`lA3c~Y`re9q&`c6>0?c0E2Ap5seFv92#X
z1Vldj!7A8@8tWr&?%;EBQ_Fwd)8A3!wIx`V!~~h(!$pCy7=&*+*uIzG@*d%*{qG#4
zX0^}}sRN^N=p{w(+yjv%xwb!%lnVTE7l1l6gJwQmq_G83J&Y98$S!r*L8}IiIa2E=
zE!0tbOuEDb*No0-KB{zjo1k#_4FHtr{!)>o+Y@bll}Sa6D^xktI0H&l{jKAK)A(iz
zB-N00F?~Z}Y7tG+vp)-q*v71(C}65$-=uXx^|R$xx9zZip-V>Hqeyfd(wteM)+!!H
z$s+>g4I@+`h2>C|J;PhvtOq)`xm4;CyF}R<)!ma3T{Vf_5|zo;D4YI4ZDBkE(vMeE
zb#ZV;n}CgA0w8x!UC2&5Z(K)9bibj#?~>R(72lFx_Am~jS?;7mo~p+05~XGD+(wV4
zEVYnf0N5+-<tP?1gB|glI&`j;EH@4!XKT+aAk1Z@L9<=6LSY9eH82@H$6Lx;1Bxcv
z%5^A}uY?_hP^bayLLK7J%)3cT_fR_18XOQ&cV*aaMGjT;&gDDL^BhVUh(<4F4hH3c
zF$M|KB70L-z@{I4cBYj0`fF_cxCFUjnxu>7O+Gc1L!sPGUHv<6=cV8}*m$m`kBs@z
zy;goR(?J^JrB7uXXpD00+SD0luk!vK3wwp(N%|X!HmO{xC#OMYQ&a7Yqv-54iEUK4
zVH;)rY6)pUX~ESvQK^w|&}>J{I?YlvOhpMgt-JB}m5Br`Q9X+^8+Xa%S81hO<1t#h
zbS+MljFP1J0GGN<u*_}U>R1}KwE=cfey%;@n&@Kli+Z5d>daJjbvuO3dW{r$1FT0j
zR$c9$t~P50P+NhG^krLH%k}wsQ%mm+@#c;-c9>rYy;8#(jZ|KA8RrmnN2~>w0ciU7
zGiLC?Q^{^Ox-9F()RE^>Xq(MAbGaT0^6jc>M5^*&uc@YGt5Iw4i{6_z5}H$oO`arY
z4BT(POK%DnxbH>P$A;OWPb@gYS96F7`j<s%D8sEa?)~9qI9NM3KMj#n>Tn6JO@hdM
za>_p!1mf?ULJZb1w-+HamqN__2CtI%VK`k^(++Ga0%z*z@k0wYJDqT^)~%|4O299;
zh1_iRtc7you(kOK8?Q$R7v-@Qk4+i=8GD2_zI0%{Ra`_prF{+UPW^m5MCA&4ZUpZb
z2*!)KA8b--Upp~U%f+rsmCmV~!Y>Gzl#yVvZER2h;f&rkdx{r#9mc8DZMJaQXs?SL
zCg3#>xR6ve8&YkP*`Z=lng|Ow+h@t*!Ial*XQg3P;VS8@E1C)VS`?L9N+rxlD7bxC
z3@Ag)Vu?#ykY`ND+GvRYTUP&-KDMiqly$Z~uFXt^)4Jjk9RIs*&$?-UPM*d7&m${m
zm12kaN3mV1J|c6f$>V+{lvHp~XVW3DU0;cBR>7|)4bo{xa1-ts<jLa&I4xP(P^UY(
zK^=O2Qw>-lYU-Q-b)_fVVl`EP5X}+J9Ez<Ex`Qpu&R?yHFbL<2c$Q;V!e@I{V!wKR
z7!xcCG-NprVU;C3WX5sx<d|M#WS7cM_k_jEf*uBp3nkhd&!k<vHt0u1^hAKyJwmds
zD7D_NpsMmlXl7Pc(mkTV?`LXRZzE08JLC1i<|KcEEW;jr4E%J$z2FYEQ?magWT$MG
z5w|PTM+AV^Sz~TIthAm)ZPhU}8DhAq7D0JyIc%3i?Lvo$+)+sNEg@Y_iByf8V2Q85
zOBboV@sg~jAmogcwCVaT#eK6@Ti}WBL@#n_)Y$X%FM3dWWfN(uT+lfa2-U^=q|fkx
zfXo?m>T20x8XIv=m7witdu7!3Lh=KE#OyKpT1GWk{YAo^ny|fvZt<+jmsFs=l*%e&
zmRkBt5ccv4O7!HAyv2~rsq*(FmMTm?@TX3&1`nu|7C^F{ad%GLuoX}R<meu_@4fM<
z-!2h@6;xk4-$>l}6`)uHF_xlx^gVca+mGH4T8u8;q{S*x3=j;kelz^atO~)v!Q_BT
z4H6%IA}bvfuk0_vweELeEl8N5w-Q1GF!@f{VKnbyYB2?}d&QvI-j}~RI_+9t9$tC2
z94<uJXj74`(inSXhxsnvUvEd#uSMCc1>m=3eLi=sQb^S5;fqP?3aaXc&<O}>`}`lq
z&M8dOXvxx9Y1^u_ZQHhO+qP}nwkvJhwoz$Mp6Q<HKIeJn^xWlLUhMzd5i4S?cZUYb
z6zvSUafe`@8T+`8>c<DM&nrWh7UU9263B#n^4BDg74A^>q^7M#eWm}!3U@s07hop`
zW24|J{t$aB`W>uBTssEvYMyi$hkaOqWh+^(RV_1MYn<UF!6R+6a*Lj<8=h{-?ucW$
zthY`+xpL?cA9;vun}VrgvC8xfggo(n`UW5QQnEM9{=2t&ulrf<ou?+PTg}FaPVkUK
z!ULy9w0m6I{74gD^)2=Q1#||EMU|xRm9B##^iD(a5&SE|{_Jmif_=MGbb`J5E~Qx9
z_bJ^2g>E0XPgW?7sBDk=Cqs(;$qrPEflqa0ZE?A3cBfW%0RPA235Wb6@=R_d>Sez;
z`spwa50bq?-zh+id~Q!T`AYn`$GHzs;jxIw(A1_Ql&f|qP}|bon#H;sjKmSDM!nyn
z>bU8l%3DB3F+$}|J^da!!pN|DO!Ndc2J)wMk!+Rr1hes#V}5o(?(yQSphn|9_aU<-
zn|nsDS{^x&tweP;Ft`2ur>Koo2IdXJDsr6<Dg_v>IN)7vB41Yy-^Wbo9*2th2QA@C
zE0-0Gk12YOO?d_Gu<sRH1Tl4a&{tqUwG+=b{V(|ew{P^Bv|#t!gRP>u6b3&(PIL`d
zh4{`k54hu9o%v1K3PGuccez-wdC<&2fp)>`qIIaf)R{5un7-vwm=>LD7ibnJ$|KyE
zzw`X*tM0S|V(I3vf454PY{yA5lbE+36_<1kd=&0Xy4jfvUKZ<i&B?KmWgU&frLdAP
zWQ!={R`S{V1#z7QsU$DOBZ}@t9L25+0o>0$Jq!AG4KS7DrE9rph;dK^6*#CIU9qu7
z?)6O`TN&MCWGmUVd1@E2ow2`vZ1A#nGo8_n!dmX77DCgAP1va*ILU+!a&$zdm6Pa6
z4#|*&3dM+r_RJb%!0}7X!An&T4a4@ejqNJ;=1YVQ{J6|oURuj8MBZ8i7l=zz%S4-;
zL}=M^wU43lZVwNJgN|#xIfo$aZfY#odZ6~z?aNn=oR1@zDb=a(o3w`IGu&j>6lYxL
z&MtqINe4Z>bdsHNkVIu$Dbq0wc#<Qh#Z1Sj<sL6gy91C$%5L#9E+9Z38PSLBc_wZ#
zw|DPH+@N@+NzI`Yk=!!wo}kHdjI&@6w?ord2!kF@AT~x2`XWz9Ahe=y_FcM!rgunh
zQpY}tyZU`~2PzLgpnZaWvrVr}q4NYSIrMP#qVu3VkD&#L;Pqe~h5=>X-xev221e~L
zbm8kJ(Xzij$gF4Ij0(yuR?H1hShSy@{WXsHyKtAedk4O!IdpR{E32Oqp{1TD{usJi
zGG@{3A$x%R*pp8b$RQo4w&eDhN`&b~iZ2m3U>@9p1o5kXoEVmHX7I6Uw4dn((mFw`
zilWrqFd=F5sH$&*<FBNV=|$b8j|g-!@cv81Z2hQ&!)rw%dhEENOk>(eJB52zaLwRe
zz`sruIc=Ck75>v5P5kd>B2u=drvGPg6s&k5^W!%CDxtRO)V6_Y_QP{%7B>E~vyMLG
zhrfn8kijyK&bX+rZsnSJ26!j$1x+V!Pyn|ph%sXWr9^f&lf|C;+I^Fi_4;`-LJI&F
zr;5O@#4jZX=Yaw0`pUy<vDqB`SgkGph~aVnBU5HD3ev5}{n-#9m0-lppcQkPV77Oc
zaP${nyX~TZ#;{=UVHO&#@iCI-$A|xRH23$iAf1l*1@OZN&VKJT@UVKzlS10YqAQxD
z*>fF4J8A9wE#7_9!X|_s8~<rFKR+K|kGm@OA?G1Qn4|Qw^xGx~6?VY8tV6tpFL&?B
zElwzoUtJv6YBv4<Lck>YUzWu&#E^%4NxUA3*jK-F5R3LP2|msHBLmiMIzVpPAEX)2
zLKYjm3VI4r#7|nP^}-}rL+Q4?LqlmBnbL+R8P%8VmV{`wP0=~2)LptW_i682*sUR#
z+EifOk_cWVKg-iWr^Qf4cs^3&@BFRC6n0vu{HqZzNqW1{m)3K@gi$i}O(hT`f#bT-
z8PqCdSj~FncPNmMKl9i9QPH1OMhvd42zLL~qWVup#nIJRg_?7KQ-g3jGTt5ywN;Qx
zwmz4dddJYIOsC8VqC2R%NQ>zm=PJH70kS|EsEB>2Otmtf-18`jUGA6kMZL3vEASDN
zNX%?0+=vgsUz!dxZ@~)eU17m4pN3xGC0T;#a@b9Iu0g_v*a3|ck^s_DVA^%yH-wt=
zm1)7&q6&Rq#)nc9PQ6DKD{NU=&ul10rTiIe!)x^PS~=K(wX9|?k&{Mv&S$iL9@H7=
zG0w~UxKXLF003zJ-H%fGA4Db9{~#p&Bl7ki^SWwv2sfoAlrLMvza)uh;7Aa_@FL4b
z4G>`j5Mn9e5JrrN#R$wiB(!6@lU@49(tawM&oma6lB$-^!Pmmo;&j57CDmKi)y<h(
zXqL}YElO`Y&m@UsMkK{MCtzLA*X<`cp4aT(>esg~P;lJPy9D(!;n;^1ql)$5uYf~f
z&GywSWx=ABov_%8pCx=g-gww_u26?5st=rdeExu?5dvj^C?ZZxDv@Si^nX~2qA&K=
z2jr;{=L(x~9GLXrIGXs>dehU^D}_NMCMegdtNVWyx)8xHT6Qu!R>?%@RvAD<fTWPT
zbQXf<n()O!xQ#Pbk5h~n7YIct6olxrfE6UPNC}gtKvBtr-K-SW`|*EHXUdOp{F#XY
z?JsnO90yJ*O~g9z->s9er;NMkweUBFNrBm1F5e0_>^%CwM6ui}K_MpRqLS0*@lAcj
zB6TTCBv>w2qh)qU3*kN+6tPmMQx|5Z0A4n67U-nss90Ec_rDF}r)IR4PE{$8;BSt=
zT%6|jyD^(w6a*A5>_|TkMqx~e$n@8{`q?|)Q&Y4UWcI!yP-8AwBQ#P`%M&ib;}pli
z9KAPU_9txQ3zOM#(x}*lN8q$2(Tq1yT4RN0!t~|&RdQMXfm!81d0ZuyD}aG3r4+g`
z8Aevs3E_ssRAMR+&*Q30M!J5&o%^(3$ZJ=PLZ9<@x^0nb>dm17;8EQJE>hLgR(Wc%
zn_LXw|5=<mgh;A%n|KRFf=!tm*s+<J31t9T4~m;{d98+{UM8O9Y3n(H4jT2x1<3ek
zRCP96)YpgXmLb5T8RCO#=@H=-mI|8WPy(|UYm-$kz(=0t1MzP~=;TJ|q`O5>b$6%X
zS~ClDAZ?wdQrtKcV9>_v1_IXqy)?<@cGGq#!H`DNOE1hb4*P_@tGbMy6r@iCN=NiA
zL1jLwuMw&N-e9H(v7>HGwqegSgD{GSzZ@sZ?g5Y`fuZ^X2hL=qeFO(;u|QZl1|HmW
zYv+kq#fq_Kzr_<t;$1-t3`$OEY~3n_q{T)Rou@A)3;+fK`61#)M28uxu-pQIoi3k&
z@@D|N0vw;}fCrrk_Bpr~YT-f~A@MB7zNgvCzi#3OND*De1`PU2K-mM~#&}W>LaezT
zqIkG6R+ve#k6!xy*}@Kz@jcRaG9g|~<nqgOn62L}JrNDZjU4F@aV<%LlNo-^2n?i7
z%|1$7{FswZK|-=pEJmMwPA8WqjwJ0M>j5fAYegGOE0k8+qtF?E<xze>gI99h*W}Cw
z7TP&T0tz4QxiW<gb)txWm7#_bTxNntXapy5bDUCo7H4xD&DjOE8;<3;GBa57DJYnZ
zYz#6XyFIk6(k%&!?Xc#tisS?+poLN_Fmul~V%YHM0M1E@z)4~ejjf`6?8DH0(3-bD
z5P#*bHnbbLTN#v_{!WmT$r@`&i~2Nc`t|XmZ-^%oLL+Cr3zKt~sMp~37c*-Pxa>!r
zF4?|!WiNo=$ZCyrom-ep7y}(MVWOWxL+9?AlhX<>p||=VzvX`lUX(EdR^e5m%Rp_q
zim6JL6{>S%OKoX(0FS>c1zY|;&!%i-sSE>ybYX3&^>zb`NPj7?N^ydh=s=0fpyyz%
zraFILQ17_9<<sDR30@KDNwyojVnat(lC)gJgvUt1uzobNU;2o@xr#!#7^%#jK1iuD
z7r}e067z-?(;GTSgBQ+#sj?SrF6s4ypglFZq~$VVzLvf5FWezR$nLUzNjEQ0AJo<2
zcB9bvm4&6HiBoIY+zQ8CO2E#{Cc9O-tpSb&=P>ettJJt~I+sl=&CPH<r$$NF*gAE`
z_p=lr4JlXKZS)N-Lgmklh}r`zrQ{F%X!spf9^b}Sx9!|LuKV_)@Z30(8oPaQFU>wz
zC9dEb#QFQcY?bk11Y=tEl{t+2IG`QFmYS>ECl;kv=N6&_xJLQt>}ZQiFSf+!D*4Ar
zGJ~LFB7e_2AQaxg*h{$!eJ6=smO(d2ZNmwzcy3OG@)kNymCWS44|>fP^<x&)TY2iy
zd8=u_8UI{JryjD7&f;g$C^xsXX)jROQB+P<9-s1k)feVpMDGRaUZv%fcb`-jy=~ss
z2~a=oT+WYe*ma=*p7E0FtZVj%@!udd*G!ub{?}?k&l5G4t2p4Dx<t007{2niswln#
zJMOhvLlrOsY00{9LqPI>7QkJHkE9JmLryhcxFASKb4GYkJ|u^Fj=VdF0%6kgKllkt
zC|_ov2R4cJ2QjjYjT6jE#J1J<<H4Z-B>xaNC>Xm;0SX<`LuW*}*{yQ3c9{Zl=<9NP
z^2g5rAdO!-b4XfeBrXa4f{M0&VDrq+ps&2C8FYl@S59?edhp~7ee>GR$zQI4r8ONi
zP^OA+8zrTAxOMx5ZBS03RS@J_V`3{QsOxznx6Yt*$IuEd3%R|Ki&zZkjNvrxlPD$m
z%K+rwM!`E&Z46ogX<Y)q$<-O!`Ps?Y)6-+4tJ~;H3Ps$sZq7IohvmIlPfMkf>Cu!3
z8use`FJJ?g_xi?~?MxZYXEu=F=XTC8P3{W*CbG3Wk)^31nD~W>*cJ@W4xg%Qqo7rq
z`pUu8wL!6Cm~@niI*YmQ+NbldAlQRh?L!)upVZ)|1{2;0gh38FD&8h#V{7tR&&J}I
zX1?;dBqK}5XVyv;l(%?@IVMYj3lL4r)Wx9$<99}{B92UthUfHW3DvGth^Q0-=kcJ1
z!*I9xYAc$5N$~rXV>_VzPVv`6CeX(A_j3*ZkeB~lor#8O-k+0OOYzTkri@PVRRpOP
zmB<rRnO}WV)476D7x6~ub!@M-2Qd&zYv%Y%g+p`hDE8{Z4h*5&$sOT^IWG0SZcgjg
z`O&$?XW-TG7xurfM}NI&k*kC+!S8#;INUOgNU?v=AX%E6_4nPaC)wkVTqoUuVAo1w
z)SkErj=K?#WGW12?ghp5<R>V|NKlJT?y4Q82er)@l<zX`5TroC<yK7LQ3#{#fD~vf
z#zjU7|ANemL(L4B*I^%$>K&P%CeLbRw8f+ZC9R)twg5ayJ-Va!hbpPlhs?>297lC8
zvD*WtsmSS{t{}hMPS;JjNf)`_WzqoEt~Pd0T;+_0g*?p=dEQ0#Aemzg_czxPUspzI
z^H5oelpi$Z{#zG$emQJ#$q#|K%a0_x5`|;7XGMuQ7lQB9zsnh6b75B9@>ZatHR_6c
z0(k}`kfHic{V|@;ghTu>UOZ_jFClp>UT#piDniL(5ZNYXWeW0VRfBerxamg4su5<;
z(}Ct2AhR@I-ro0}DdZLRtgI@dm+V`cRZjgV-H+aXm5|Mgz`aZX63i<|oHk-E)cABn
z0$NR?(>fla7)Ong28FZSi9Yk0LtYl5lZw5wT!K5=fYT$avgkMKJWx~V#i@7~6_{dM
zxDDPIW2l{O2Elv#i^cjYg~lGHRj(W*9gD`(FILKY$R`tL2qo&rtU*c;li!V`O$aV{
z!m|n!FAB2>MR_FVN*Ktv5+2dW4rr3YmfEheyD+48%USM#q6)w%#2}~=5yZE1LLcth
zF%VtefH&#AcMx7)JNC$P>~OFuG6sK}F7V$D7m!{ixz&inpAVpFXiu^QruAw@Sc7Y2
z_A^V(2W_+KTGRp2aQSMAg<x6%12zeS27=o$@PonK4GuYkkc8_oNyZ_fx!DpPP7rH)
zq%QZ$f)JMRFv;DSDXcQg2>yV#b3@{?5q@hPEP6oF3^}|@8GuD6iKbX;!LI!L=P#Za
zL$Zuv#=x3fseRMZ()#SQcXv->xW`C|6<d<qjTnlR-zB%D{uMGag>quwqL1M&KByBj
z2V`}(uL4JB-hUs6304@%QL~S6VF^6ZI=e-Nm9Tc^7gWLd*HM-^S&0d1NuObw-Y3e>
zqSXR3>u^~aDQx>tHzn9x?XRk}+__h_LvS~3Fa`#+m*MB9qG(g(GY-^;wO|i#x^?CR
zVsOitW{)5m7YV<iB~ZA&0a?gaYj)AUbCPcfrFo{aih&&1c1xLcoR<G$zDt-`y;52O
zV(4ER?6nKAl>{kb&Z!eXmI}pxP_^kI{}#<BeMXLv4!<Y5yd3DDTG;6mav){xh&<>_
zgjaG)(y7RO*u`io)9E{kXo@kDHrbP;mO`v2Hei32u~HxyuS)acL!R(MUiOKsKCRtv
z#H4&dEtrDz|MLy<&(dV!`Pr-J2RVuX1OUME@1%*GzLOchqoc94!9QF$QnrTrRzl`K
zYz}h+XD4&p|5Pg33fh+ch;6#w*H5`@6xA;;S5)H>i$}ii2d*l_1qHxY<N-feyD&<Z
zSh);bmV8lzMD7Oi=KVNpO%WrkpZUGof1Cd3k?r*Mj;}|oUdl^1J2?mWCwczlSvE8@
zf-qqQ$nDS*hH;1`!sN1#G&zfAFlMU!<No`4=-gY5r9wU_VWyNmA+1rOmx>`L3g=t?
z!-H0J5>kDt$4DQ{@V3$htxCI;N+$d^K^ad8q~&)NCV6wa5(D${P!Y2w(XF!8d0GpJ
zRa=xLRQ;=8`J2+A334};LOIhU`HQ*0v4Upn?w|sciL|{AJSrG_(%-(W9EZb%>EAGG
zpDY?z1rQLps`nbC<D;Ix7t&)6xy-i{Jj_>tzqJ#@wxU4}(j!ZQ{`g`g*SXlLah*W9
zyuh)UWoRCknQtd~Lk#BT_qjwj&Kw8U)w=owaJ;A5ae}3)y>{neYNS`|VHJdcSEBF#
zBJ6a;T)u;^i#L~LVF-X7!E$SggILXMlsEy~v}K*DM2)f@U~g|Q6I-Pss@)`>fgFWx
zsq&7pe!|VA-h;@=fBF{(mR1^{1>ukTYUdyF^#A+(|I<gJqOQ6jg8T)FMHHpBPgGK5
zE}58wj%Al%5^NPj%w`M)s*q5kGz>_&nm{<l*KzZnQFE4XeMQM#I+V|Dz4Na6hUVen
zA*vA7=OQ}~W-e0b+4H_-d_9@_{qe@>_xaKn3h4&yMyym2k-wMFg(s@ez=DPmuB%`|
z6;e@HQKB(|!PU1sW)W6~x|=8m6rL~4dQ9LTk|RzL-_(_77B4I~ZG=q7K%qHiv!FD8
zmt;V<xb90THH1F906CDtzY$k*9<R<^yt=Q=vzV{L7Z@ac%h;3(!g~_M-N8~hZ49%f
z;2fRrBUA68yuh&NwybUr-qPWMOvylxN(@ENBDNX<AP5tln;>nhb{ymaydv2V;X-5p
zTt2ln?kaB9&(dH_X70^@rrCfz)nwfa9LYTHXO(IPcTEf$QiEhTpl??L+`Eetyqof8
zzl=q)?KdYni!C_9b8Z3xm7r5<5ZG-0uA`u^7Dm7k4mAsQ(rkoWy*^DZJa~#y6+hNG
zh?7{D9$a9LS`a@SvZ5?C{JUHovWU9KI}z8YV4pWftx21v*Q;MpU{+b@<PdxGK$c*n
zpG?R=tLjB3=pzQZ@@XZUs9n-cts(SYf{`Q#-58^tK)<+772q6Z+*FaD;sr}NlJ9Si
zInq4cesgHKy2w4L^={z4S>>Or(}pwO^fu0qA3_k_Bo2}lIxvmMhucG-o>O=+R6YxZ
zjs!o%K1AA*q#&bs@~%YA@C;}?!7yIml1`%lT3Cvq4)%A)U0o1)7HM;mm4-ZZK2`Lj
zLo?!Kq1G1y1lk>$U~_tOW=%XFoyIui^Cdk511&V}x#n4JeB7>bpQkYIkpGQRHxH$L
z%tS=WHC~upIXSem>=TTv?BLsQ37AO88(X+L1bI<;Bt>eY!}w<J%2~a3^@;GQk-L-C
zp_J}STi66}cCpD?yd2DLDK)?bL{zsEa;Yzr1%-LoPI<vng{B&CHG|a_62_DzDrV_u
z3z4x~1LRva!GmFdtnrHSAc2yaUc!Cyf{ABgQ%2pNT*ePKCB?1Y65@QKb?Hn?Okx*m
zuuh#hvvIwvgw_JvaXD1DYw~%J(wIR^9nK2;#L3Ewio+`dQ=pzJn{E+OT5`EcoISZ1
zv&u0redRAlvpR2?#kd^{j%t*BItx|k0jG-si_+H>jYoBn#2RGEP49&ZH-Z_}R_JK_
z>o*_y!pOI6?Vf*{x-XT;^(<Sm4wo-g1O%XK31IFzsULSCiFjjG{JwSgescYW!`7Q(
z(lWE=TH<W^E1d;WGreR^`tJlrn>_0}2twfk`*)_lLl0H-g|}BC?dm7CU|^-gNJ~rx
z($>97WTKf71$?2|V$Ybpf~Aj@ZZOcb3#uRq51%4^ts-#RMrJhgm|K3QpCsPGW=2dZ
zAr5-HYX!D*o#Q&2;jL%X?0{}yH}j*(JC4ck;u%=a_D6CrXyBIM&O#7QWgc?@7MCsY
zfH6&xgQm<Z9PS~NG{?F{__J>G$U6Miu$iF(*6d8Mq3Z+en_Fi`6VFF=i6L8+;Hr6J
zmT=k0A2T{9Ghh9@)|G5R-<3A|qe_a#ipsFs6Yd!}Lcdl8k)I22-)F^4O&GP&1ljl~
z!REpRoer@}YTSWM&mueNci|^H?GbJcfC_Y@?Y+e4Yw?Qoy@VLy_8u2d#0W~C6j(pe
zyO6SqpGhB-;)%3lwMGseMkWH0EgErnd9a_pLaxbWJug8$meJoY@o-5kNv&A$MJZ=U
z^fXPLqV6m3#x%4V*OYD<mz!OaL}G{(x(Zn78ch1cePsE^2e~@JWbe|e#aji@$i+vw
z*8J;y4M0v}tb1U|FW0&Qpm_VSbXqDMqhHM<w*8{FF3nKrzBf&1<UYE4ofE$1=iab1
zSH=CO(;$8FovU-~BWd|QO1Y8ABjzdgp)2#Qn&vY50Bs?ueU3Z?TY1r*DTWdyAKwy#
zyv{HOXJ_D3J~Qr@c4s#I_SQAK!-vW<+Zd7ukiJM&Z<)rpu6%QJDBb8oQ@=*vaq}(%
zXdsu8J~^V~wFm2<kU*+D?yEfRnsLCcx$Vn0B))s<KVMOI))4fSq6xZ1AAA0}*-#*>
zUPS&WHikdN<{#Yj|EFQ`UojD4`Zh*CZO4C<on#erRU}n(9~z=`cu;wGrU+`(CL${P
z6~rcrAqg0N#KD3V7qOZ#F$N@3r1<N1n`h|_3s;_whb+9KrYA+un3IQx=cdQSZdU;@
zgMwn+<8Aln_pXUQE8nkorU06BR{|gu#H&WDdwOaf0_?>v`w^&*FfqBi`iXsWg%%a<
zk@*c%j1+xib(4q^nHHO^y5d8iNkvczbqZ5;^ZVu%*PJ!O?X-CoNP*&tOU!5%bwUEw
zQN?P*a=KKlu{`7<n9R(Rg=cLB;D7@qEqwKxaR$R5qJXWA`_I@Zax-;YzxU$uN!#@+
zxQ+y}!5iv{M*E0X{_@u(?G00e5M$<Et|XW<8ewXQ(_LyR(YYJ?$fgKq0do2vh$Kca
zw?+1$t|WsGEvTr>GoA}DE=#nDibRgecw>-*da~7&wgow}|DyCJq!-Lp8a~(zR@tO1
zgu(4s4HptPGn(HmN2ayYs@g+yx1n`nU3KM{tQHhMHBw7f#gwru$=C()`aKZAl^dYc
ze7fC)8EZEXOryk6AD&-4L+4cJ&M@3;;{R)mi4=`ti7IZByr^|_HNsjcNFu?mIE)jD
za2j)FPwRY!R_YR-<hoOsQFHS<|15PbFdJG%afYG1Ze2oItOe*f5_ElPml|ounhrI%
zPE~{!7Y|K~F}fVS)JHSLQ9qeB{$pu16pZ*XbuKoPEL>P?URm0Pti*<n3H9u0W?_sE
z00=RF0hE4FPL-k`ZzxGAE|;a6hnI>e#5jmfK)6EvaKCT{h)kbJl{AGr1Ekt}pG?^e
z*botRf-RsB8q10BTroj{ZP**)2zkXTF+{9<4@$aNDreO7%tttKkR3z`3lj<k1`ZYn
zYn_o{o)z234lQeHSKdd%2@i*_5D0hBkRc}A4oBVwcURG%PTph~Aq76Z&z?&)wfN2H
zEyzH@>d?heAJEe<0%4zYK?};Ur*!a>PbGYFFi(OF-%wyzbKeBdbkjv^i9mn@UocSS
z4;J%-Q$l`zb&r*Pb`U;3@qkc=8QaPE9KwmlVwAf01sa*uI2*N`9U^3*1lLsM9dJ(4
zZBkU}os|5YT#Z;PD8xVv!yo$-n{-n4JM5ukjnTciniiT`(cZ6sD6~67e5_?8am%!w
zeCLUxq~7x-!Xg#PgKV&caC@7mu<86am{WaXo(lAemt4~I$utSp(URWpYNo$RvU*$N
z#%iiA+h`(E;BUg;=I!#EaxO89bUK3*v5Nc3GPmURC5TqzC|))DsFNtJICH6oBW6#q
z+B(<B&`b9YiYge3DOuymRDK2Ze%Ya;wCKX+X%fftd$edeGX@n_Xz5!TRa=fD=^U~M
z9?bZP5VCP0D8WHc?fR-t9w8NRm`liv)^bH)sENnIqPv$6BPTnOKd>N{ey+^mk_{!@
z)VhAWXG=_0j|0f9iJ;c404PiIFqK)(AD05Xh`Fk`r$^b`v+>*g+_+h@r)e+ELJ45)
z?20~u<}HQyQ5AsBz(teF9!!_GLXnm{5Z0e{Ki*@!=&3x4-RcjBn##DDzHJ|KSZ5(E
z9=tFZ)p~-}x%9sCY27)2i>(E-^OiYT?<djgmQHIMPtc347Rv=Q*y(&jD_CtKeZj3^
zXckY#)f|4;6KEIhi&4E9syFn2>_)a;yXAGR$y+E`myMd;xDA#_Q49t*E}&ql#H~|x
z2J2R1_#2lt91NnF!uqW%_=HlbF?A{B{n>}9$g5QF!bh_a7LTU~Jyz}7>W5{_LAov{
zy<f8Mj}O*?NVx0nb9^h$m`({QXlJitjfsb>2_dmGy)d)&7^bJyUjEw%3xj{cuG0Eo
zwL*XQB*Oi=r&HIIecC1%lbE;Y-*5|cL955S+2@uR18JDL<0;;Uc2Q9JEyo1R!!sz_
z#BqnkGfbLP#oQJk3y}nwMd(3Tt^PVA#zXnYF7D0W1)#+`i?<h^`5@8s0};`La7O&3
zhp1*He|{36n4=20msPq<-nH)Vr-1P9p?^Sgf1RhutsNa%fqHCSrog4h>@cm}fBkKD
z+Mpcuim53|v7;8Tv(KraEyOK`HvJq^;rlNzOjIbW&HJDFqW>doN&j7)`RDv#v|PQ+
z03WnB4Y4X@Fe-@%3;He*FjY1MFmkyv0>64Cp~FIDKQTwmFP~_CxZOf{8gPy}I<=JC
zo%_bmue&$UU0|GG%%99eI!m#5Y1MD3AsJqG#gt3u{%sj5&tQ&xZpP%fcKdYPtr<3$
zAeqgZ=vdjA;Xi##r%!J+yhK)TDP3%C7Y#J|&N^))dRk&qJSU*b;1W%t1;j#2{l~#{
zo8QYEny2AY>N{z4S6|uBzYp>7nP_tqX#!DfgQfeY6CO7ZRJ10&$5Rc+BEPb{ns!Bi
z`y;v{>LQheel`}&OniUiNtQv@;EQP5iR&MitbPCYvoZgL76Tqu#lruAI`#g9F#j!=
z^FLRVg0?m$=BCaL`u{ZnNKV>N`O$<S-}rug$^e;;5bSaiQ44v9GSU!DfSLQltFSzB
zn<Uo&fJvC)#$$D-jE@CZ6qM?JPJtfsI~qh0W2lGG{?v(jn*8p5HJ$l*-`V*FP#c`l
zciNQ!&xZWTvC!S7=-M@pPa!dT-m36Ap?Bz_m+VzBZiY{#2%PG>SuDvY`AoyfIzL9~
zo|bs1ADo<cNN?Q_OHXjI$ec4@qo=F|(hUs8s7RPlcy?;gq;6K5opD?9>XMr{tRGL%
zA#cLu%kuMrYQX<hZ7Iw}=nua>Jq8(&qS|UYUxdCla(;SJLYIdQp)1luCxniVg~duy
zUTPo9%ev2~W}Vbm-*=!DKv$%TktO$2rF~7-W-{ODp{sL%yQY_tcupR@HlA0f#^1l8
zbi><!?|KPxHdmqcLGCA9m(j#`++<&}(&XhVJ8#DI8lJsTrZ_I2ba5dickr#z1C14L
zARHh$4(yoq4BX%le0cKlppSF|mqqsSFNIdLjx3eh*i)vMq{|VrF&}cuPY^qT>MV~o
zz)zl1a?sGv)E}kP$4v3CQgTjpSJo?s>_$e>s2i+M^D5EfrwjFAo(8E%(^ROV0vz0o
z-cg0jIk24n!wxZainfH)+?MGu@kg$XgaMY-^H}z^vG~X<yMuIo-a6T+WY8{EBZ<YF
zgIK3JQ7h0t@xF_LwVF`lQWmQur#u(uv>C7z2;p2Kv`b^3S#b5ssMOJ7724v>S36dD
zeypxJ<=E~<BYo6*3!=tpc)T%a*%irgA?_1qXBc`bLnzV>sD4f5wX060RIF-AR0#{Z
z=&y$r8A-e6q18lIF{@O9Mi%dYSYT6erw!@zrl=u<Z9I?E*@RAMygn<|c0_8g?p;(p
zfPx}z^qO^Qzk8()=VB}L9MnNWVGdH1n-KK|f2?iVcCs@U=m4vcb6)DDZRRD!+uX!~
z5AUsL2g0>j>o(3=M*Bg4E$#bLhNUPO+Mn}>+IVN-`>5gM7tT7jre|&*_t;Tpk%PJL
z%$qScr*q7OJ6?p&;VjEZ&*A;wHv2GdJ+fE;d(Qj#pmf2WL5#s^ZrXYC8x7)>5vq_7
zMCL}T{jNMA5`}6P5#PaMJDB2~TVt;!yEP)WEDAoi9PUt89S2Cj?+E0V(=_sv4Vn6b
z_kS6~X!G;PKK>vZF@gWpg8Zuh%YX^2UYPdCg7?EH#^gkdOWpy(%RnXyyrhmJT~UJw
zAR;%Zgb6z(mS+o9MT|Sc6O({!i0pzk;s9?Dq)%tTW3*XdM3zhPn*`z45$Bg!P4xfy
zD*{>30*JsSk?bQ-DgG62v>Vw-w`SA}{*Za7%N(d-mr@~xq5&OvPa*F2Q3Mqzzf%Oe
z4N$`+<=;f5_$9nBd=PhPRU>9_2N8M`tT<-fcvc&!qkoAo4J{e3&;6(YoF8Wd&A+>;
z|MSKXb~83~{=byCWHm57tRs{!AI<5papN(zKssb_p_WT@0kL0T0Z5#KLbz%zfk?f7
zR!vXBs36XaNcq5usS7<>skM_*P$e*^8y1ksiuokbsGFQ_{-8BAMfu!Z6G=88;>Fxt
z|F-RU{=9i6obkTa0k~L#g;9ot8GCSxjAsyeN~1;^E=o5`m%u7dO1C*nn1gklHCBUw
z;R(LgZ}sHld`c%&=S+Vx%;_I1*36P`WYx%&AboA1W@P;BvuFW+ng*wh?^aH4-b7So
zG?9kFs_6ma85@wo!Z`L)B#zQAZz{Mc7S%d<*_4cKYaKRSY`#<{w?}4*Z>f2gvK`P1
zfT~v?LkvzaxnV|3^^P5UZa1I@u*4>TdXADYkent$d1q;jzE~%v?@rFYC~<iuzNW%}
z&<lxOtzR5tXjdhmeb!y=WrS>jB;IM5n_U0;r>5Xmdu{;2%zCwa&n>vnRC^&+dUZKy
zt=@Lfsb$dsMP}Bn;3sb+u76jBKX(|0P-^P!&CUJ!;M?R?z7)$0DXkMG*ccBLj+xI)
zYP=jIl88MY5Jyf@wKN--x@We~_^#kM2#Xg$0yD+2Tu^MZ1w%AIpCToT-qQbctHpc_
z>Z97ECB%ak;R<4hEt6bVqgYm(!~^Yx9?6_FUDqQQVk=HETy<x)ZU@wf8)=!&Y<kLu
zWGerSV5;uV(Ss~f-EUMF)7Ym3J?WvQZ+?`N_<3?}HeeDQ-1ttMn)aC7AevC8EgEXH
z%S9@IG4_1#WR>Wpi!O^<IK-?FeeY}~nnY1FZ!#{Hc~vB?+1Xfd%eQU-Fz`Cgc$MPc
zJ*{#l+8TrVKtVO0=0YFbpI~=(9m3E1Hfr2twqinP2$}j5075saj!Jcicg94oK|i+8
zDl-J<L^)2JWHjmN|3SU_s3DUZx8tInwW%?b(lDwb@rF!1ps7M7=Nqz$9G~Cxug%*D
zk0KH=IY))>`EZ_5AoSv@VbUzsqusIZ;yX<ySVYL1RqLJ9ph2V-I0=X%mbaI5NYGY-
zOCSz51^BclM<BdRc>!4CsMiznO}S{4e>^0`c<)c~mC#*{90@<TgXxqXpzQO-^6-9A
z;MAg`aC@Xb9WYm2?BfkSp<l@Cz)#4$+-ZP-Z(TTR5jv!%rmHkPa#-jOmPUQ1`vYIN
z*i!$1v@F~bV^|NUC%4skZDdFeO8TN|*49y@ooDsu*dv`zvGm3dt~j>+T@%<X$qZ4=
zC<*3d<{q~=wgaC#P!?5+;PE-S6hA(BQYtIWJsl|UgDH(zPhb4vsLA7wRGGcVa@k^<
z`tx^3M8PjL6V_hJfVLahnUsH>EQ~>bovc8n_$bvqkOU7CrYe8uI5~{3O7EijeX`js
z-$LNz4pJA7_V5~JA_Wl*uSrQYSh9Wm($%@jowv^fSPW<~kK&M*hAleywHd?7v{`;Y
zBhL2+-O+7QK_)7XOJAbdTV-S`!I)t~GE8z+fV<jIMC<~}(~#V85VKLFZ3MMr=8TOf
z!O}{i>7y;wp#!wj75drv;R*UdSh(}u$%{V<Wr71S(c#5gegx<RaRY;Lfj%D5%wVIh
z@Eu=8wLLhqQN?Rf3hWjS_5BVdSd5dK$sn^Z-#{FnV@iS)32Bj=$-IdA&P}YWyW+s|
zh2uz)4XLn^dMcj#V$2!^b#h1ajv8iX`7TXq<}Q?bv0&`ZK9Oy=5z$bbx4SS;V{o6;
z$xAk2M;3;GlZt#F+$ZlK^aX)AGBe!XAJrpY5!I*Qp!5Y}Ab#w6+Kswz-p<r?=w&BH
z_w=QBP~5SARP5IBSfxFt|D~0~Cv+$B0IBe}JEwjiDYtO&w{SS`Sj@Ns;Y(yp4@dlw
zIj_O0&qwwg1~og*!DSet<t4o9>Sd0gLeFp;h6FkiVz%g=EY3G#>RU;alRy;vQmk*|
z@x-ba0XKE%IyL4OYw6IXzMiS(q^UDk=t(#XgkuF`{P?=k8k3r)rmhkv`vg@kiWd34
z-~t+1aV3SabTbG=nQYs>3~E<}{5@0g**LAWi*~SfRZhGcgP{e5T!0M7CU}`f@r8xI
z0bx%sI!?5);-wG+Mx&S=NRfIi>V-wP(n&$X0Bhd)qI^ch%96s6&u7qpiK8ijA=X_R
zk&|9f$GXf-;VgnrxV83Cp-Q!!sHH`5O^o~qZu!xny1t?(Au(EAn)D??v<1Uo;#<u<
zt6DFiiRcr7k_5;zi{u5+#zM)GD;ml~eC2M%zm-TR>m7-M@ovk|()C(`o>QMTp}F?>
zakm3bHBKUjH-MHXDow7#Z|@wea1X9ePH;%YA)fCZ9-MD)p^(p!2E`aU9nmJlm;CXQ
zkx~$WQ`Yq{1h5k>E>Ex{Z=P=)<Ej#49vg41$|RJU>N*0b8_O({IeKg?vqQ)hk=JHe
z5iqUKm!~mLP0fnRwkCO<t=1DRCN#BFPI+29>(xxTV@&p+o8wdSP$jZofYP}yEkvSc
z5yD-^>04{zTP7X44q9Af&-wgt7k|XtncO&L@y-wFFR44RsPu57FRvIBaI^Pqy_*DV
z@i13CsaR5@X@xH=NT3}T`_vsy!a02n80eQqya=-p7#YW`Jc0z!QglGg`1zeg6uXwI
zsB~hlNMo)kFL(V3Q1<%8yoI6X7ncn-&&Uh3rL@S(6@wKAXt6Wr=a2ObI7}8$D-FoI
z>AJA>WsBEMi5ba6JhJ%9EAi&ocd(ZsD|MsXwu@X;2h#|(bSWu@2{+c7soC`<N-us%
zwC+KRPSF!$&8)ag6}`g_W=Y-FR2io?sYDG><oM_V+}8_KtB~@SC$d-cD`MA@W9s(W
zx0!iijKByVNV3^bVcwjuDKK?M9F!E?{ffzq99|(HQ+4zKI~u?0a^E@e{AIDEvDBYk
zYPT4IdE4)A94g1Xh+f4GUL6s=ACPZOl_9c)(=U}he!^wCLDC!}yG1Nt${uC8^aj|;
z#3`-LSzZ5zOO@UC7f8(=d<14ISnk$nhZ`JDnRVD?mMvQG+L0(FqQXT8WbeQc%6m1&
z(Cw2K;RGd|d#YJ9o8~=V=J037p#6G|cLa5pm}G~UUQ2op<9c8mE$v#^Oz*(TBCIvk
zg57HvUxHy}{XwN#I+re(dXLe?r^7y2QcoDFeZ5J1y*U9F0LG3sO2`m+hfT8iII;N{
zCCGR-edT&mlO}jC;rdyzyEOf@(mr5$)Bnicm8{6<rU`rFU<#Ks-TiHXy^;NV5~}Sh
zGMGEH(013&d6v862FTkB#x1K8@RR<KO|=>%uo{sMYq&Vyufb)?OI59ds)O+kyE8@G
z@tlpNr0<P9!P}mChFSI3w8y_T<$MOFAvwvu40PR-Z43LeTZfb^;_!{%XdCE(alpS)
z$+(yv=PSz*uP~i@XpoH5SX}0j6IH?ddb5%iD=`438V7ZQtAm*>UO~}qd0HQve6njJ
zda2+l$gdX7AvvGhxM6OToCuQ|Zw|9!g1)O+7>~{KNvASjp9#Cqc<Jjn%6{ep4~BAW
z2JOQW>e-or+y5xdzWL3g<qZzU;EfCG!yYQi_9zCslh((Y84!DNy9K1Ss5CWp^Dg4}
zVvpwj+11JZQi<);*r$4PzZIl+dw1-i3+p?&H?zfhkjB;66CZy$eg8g!zsYk*9T8{Z
zs*fqVjgU?qZnJpDE@C;X4vLC;px~iLTiarv5<28ZvU)^X|BGD17eg>L<X0xLdX3ay
zOGs5DA)8(lJx~r@Em<2wsQ~@A>Wt2oa+T(I+{j(&bF1laUsJB{<yrlcGHK}8AUhC?
zoIVSfR^Gs9OoAB$lw6C~g~<TN_RE77mV})YIN<3x_HML@`slvWlDpsZbyRh#Xv(hw
zs&)uHejiIS2hRZQl6tZTX|FfXIcE_}N66@%@H^d^^#z$b{Lh|W>fOgE-B}qslaS>C
z)TjzG8XecbS%a+?yT!0QmTex?E478;D|sL*oS4C<IT~42EoCkv<fU$56uJp3HHsR$
zo;gN33YHOV73J$E;RfCP$fQ~7SLTRE(igk5nZC4-D|24<emnkf&1IY7H)IY8JCQL8
zC*{_--jSlZgbzo9tjGu}$YGpIuQ|F0O-di|Ru08rXB}ZLHm}(rGEmtR-*g9e2!J=y
z!Ab<m`RIWboe>-g0Tq(YoH|eyxJ#1j088C|U-w5id`%Sz7X_w#l+U9+)$|2no<}5J
zRb_9@0esSr?n}HvVGbD5@$p$8k4?qOe-GNOk3-K^Mw>Xg+drCKi5@$GTeijpI;;IG
ziD<&go`ptLC&^<0jw^l0aY?_pUUK+xp#0Bk66iQ29vpR)VBE{JOJ&OL^gKsN<&t<|
zCMLTYM<?a@L*q&V6@aF{`o8}~9937t!vg&HGL*mp0C@kMIQqw<xcmnHcqqltNzm5X
z`e$1rWo~2qGoEqz!EFDZy-BvRzU&V&g*(>SDG5Ie9O>6Dl#T{@cscz<F*(G9a3U(n
z4-?gMvbFW%3%;+Q9*~|WAtYZv-f=fw&5_mN)epX#i!JYyEYEH4hv(eiJpfgPngYKC
z2!ehXQKsnZOVdovW$GZ0hF^=b+96*}=IPfgf?_czLPF>%)}?tC#?rj>iwQ0!YUk~R
z$rB-k=fa9x&631Z9Mfqj_GRoS1MzqSMEdaZ2!isP19Sr>qG8!yL(WWF)_&{F)r<r7
zFroT<t<|(9`RC}+mJTJX^yO|b7wx}?$Z+X`Cb%Z*NKI=vcStd*b(s|Irgw4o?SX_N
zTwm!T?4{_!I%w1Pwr~hE?f27<9dGk~3)6>>KnJGSciSp!P0fqHr+G=fGO02Q#9gHK
zpwz+yhpC4w*<9JO@#(MdkZcWbdCO5B!H`Z|nV?UtcBo96$BgX+7VYMwp@b-%;BrJu
zMd*K!{1txv{kHKPDs9?WZrz_^o1Tq2P=+=|E=Oy4#WE{>9}*9(apqhmE`&AeBzQgQ
zELFLCmb~q|6y0FCt|B}*uI*ayZ#6=$BpGtF{Jfye#Q>FZ?BPnk)*Qmd?rNG^tvFUU
z_b&antYsZnUR6<MWPiRKgXZ@POHas$PRDijxVN<aT5RsZTD8jV)w0rx;2cB#e7v@^
zli|!E67Pz6Nh*6Ko#;2$Tkma9H-dh|gL&Qw_Ma;`Bd$s<fA3OZ@cW0$=Y&KziJ3B8
zKT|gu`#lkpcmi#a(Wck|Z8>Q9tQUy81r$&ovT#fy;(Db4F&M*C=KxQgHDrRcVR#d+
z0(D<YzCl%cGT`5RbSZIpuxzhv!aWxCI>|*9#u`w_%2o3faI{?dNd9$#5nj1PROHNq
z7HJ(;7B1ThyM>a@Fo^lJb2ls2lD`}ocREH|5pKN;$>gFyM6k)kZG;lA;@kSJIqUhf
zX%dhcN(Jtomz4(rNng&1br3Xx33EvCWz%o8s;<SUZU<|9r5~p(SV8=KJ;)AvZ0B-&
zB=E0>SpRiKEUFd+KJ+u|gn|J85dZ)Exc&=V|Ns8Xs#P>qv6PX&VAJXJ(ILZO!WJd0
z`+|f5HrEj~isRN7?dBHotcPI7;6W48*%<fsA6?s2cD$ET7tiOHOuu8{I=ZeHF>J(9
zftl1Tr`bKH*WNdFx+h;BZ+`p!qKl~|Zt5izh}#pU9FQKE97#$@*pf38Hr8A+<SbF3
zC^w<foHbyyOtGd=_gp6XQF^KKPR`4-t;!23(jm*{(kqu6&^dB!g#9D4&E-~<^#KJZ
zhUOcK4_&|zR|G~FIBmUj8%fQ?3wZI><tm1qUtJwNrndD)mLb!2^vE(xn6Zu*+6*6?
zPPSY5>`N+50U3$6h%^!4<EJ8U1GnMmv3ME^QWN=ocJxF7vFG$bic{y61q8Ey+=PX4
zqa@uPq3xd1<5-a`hbL1bD{M5O3-cZE;%+mdd}KC?L7UXo6&*{k&o3Bi_Av)Q7(5SA
z2DDCn@dO9)hBoFDTbczfa}2B}DTyI~ZpFY2#e{-Js68;{W$GM6q3JexIeR&%!J=RY
zNeqOBHPMaHd1DbnSoS#zk;f^5sq1oQVm8R98V;_Mx{(M-jI~B!2tHmcL32fx@>fBN
zjh^cl#8qW5OZbvxCfYzKHuyeKLF4z^@~+oqlz9(Hx8vypIiUlt!(vs}_t#4@nh$s;
z>FYERg*KD#Xs+W4q-V-IBQK!)M1)Aa+h+V+is)z!_=gEn&^ci7<<lSp%cV$cgU+d!
zB{6&V-FV{Shk*=MDDJ-mhS_<?j0Sx02?kne9cQCm#nj^O;A7F(ot;U`HCqZy$umDj
z$VhLew_}Z~_K^q(HLZ1_CG#0e)Vf5ht1%!atKtcPv%Q`L+S^Jp;F6jF#R16)E0Gx@
z%I}&fkuPDQ5vlAMqK-Z2F>DEEmYcoSh?WdXUsP7O4)&lQXA(BVM5jI8s6;m<h<nYc
zYbjfRUOcR4OD+U;Xxnj4!p-5ssl<tF%ROb5NL(aKEYxMl!Omsn5tVq6m&@xe9R2ty
zaD)OvJ6OOvVnaP@n|@!1J94bXw<KSpX4^fAg>O}94AC0gG(`>|T)yuV1l~i-ejCCt
zoejDhX0nrZDP|x9u4zp%S2UeDzV`o#pBGu1tZ-$<9TIbN=ALwhQ0=9S{8#}Uu8n-~
z5~xIvUhLSz@c@0|me$CdZCpZl(vQw@a0Y4^{T0w_>pOkwI^x4KkBf3qGmm)nG|Ps5
z_XTY~^b^mL&_*yjl~RRIi&eS(>y?y}O4-)nWyTEPpQAb#Xz8SnnfIL+nAcNL9nqV9
zRL|eyF)RKI5-kJO6}>Q89XmgY@b1&!JI>g3ryZ@jN2v3vm7O`AL!BTWNouJzV+$+Y
zYY}u%i>K6=IYU2O$2TAyVjGt?wgF9xCj;?EK(8fWu!!~48`3u^W$eUlCh*91PLxu1
zR<rN@kH9P?SNX2s^Aazw5IaX7ziWE8THeG60EjU<A$lRyC^WY!o&QLhlUyuPiN25=
zU%UxW)3QEWRW)v1qr`{rxmznF66&k*@0ST;;{>Y(F7Q3s7h$Q-p&L$ucN}it*-9KR
z_<<V9aZ_zaq3i4pukzhO4HK+G$rb3ekIm>wHu?!dav0$P+PI3{J8?{+l|n&2YMLV2
z+hRta$A5WpCXl1RNbYBsX8IGX{2v>U|8_I-JD56K|GexW>}F_e_g_1r?08v8Kz{V$
zT=6aGMk>ibvRO@Yrc@ezaD0%ydHkXGHrR{7>q~~tO7ChJflwa4-xL|@#YIJejC5VT
zInU4CjQ9V0+lClQY=vh^s4MadwQmk7li{54Y;Ht}gkZOIh9(vfK?3kXLoD72!lHD#
zwI-Jg|IhT=Y#s|tso1PWp;|aJ2}M?Y{ETyYG<86woO_b+WVRh<9eJu#i5jxKu(s~3
z4mz+@3=aNl^xt{E2_xewFIsHJfCzEkqQ0<7e|{vT>{;WlICA|DW4c@^A*osWudRAP
zJut4A^wh@}XW4*&iFq|rOUqg*x%1F+hu3U6Am;CLXMF&({;q0uEWG2w2lZtg)prt`
z=5@!oRH~lpncz1yO4+)?>NkO4NEgP4U~VPmfw~CEWo`!#AeTySp3qOE<yqk==w-Fh
zZoMZ&lPYg)Bf{l|^Ec9eHrZHNCIgHA99HK~=g9g$6}JC&SpU*3CM#@74A3F-q;H%b
z?|h7~p3*QyKz79VOM&F)%{IJOaV|%(P^+tYv(Wwoh$JQz(gr}bJQ5GRK7C{yczU*Z
z0d@glsmT!7B#p$>#{<xc=qKc7RmbL$iD3w2I>oUW>FwHkZ3rBaFeISHfiVSB7%}M)
z=10EZ1Ec&<kbU9nte?Sr<dBzRXU#i$*kS$Owlw|Bhmq(T@<et$=Ga&nut&%m$aDFo
z4)XQZ)Cl|y6@#9nrY4Bc@)ztgx@#SpqT)}Gurk@SSEbxhd|F%HO-p5^V(jP^YS&&J
z+%c?oG&5Rvm9?m;x&9<%WaOISo}X}ji<*e=QA(7qDLSq^Y2N2^47cAW)I;Cqh>l;4
zG98m5sU!pVqojGEFh8P{2|!ReQ&hfDEH2dmTVkrS;$dN~G2v-qnxn^A2VeHqY@;P}
zudZD5vHtVvB*loIDF1M7AEEvS&h0;X`u}!1vj6S-NmdbeL=r{*T2J6^VA7F`S`CDd
zY|=AA6|9Tu8>ND6fQhfK4;L3vAdJPBA}d6YOyKP&ZVi%z6{lbkE|VyB*p1_julR^k
zqBwjkqmFK=u&e8MfArjW-(Ei8{rWso1vt5NhUdN|zpXqK{ylJ8@}wq-nV~L4bIjtt
zt$&(1FTIs+aw}{&0SO4*sa0H2h&7g}VN5uYjfed5h7eGp$2Wu*@<SPMp)XHv{s5It
zX&Tv-;;C<TNZ%Oi$|qe=DKAP7j<WeX)KI8N5@^t#2{|BFvgbdm__-Qf7x1hUrN=s;
z7kJoEjE4RpS=m~PP%CvnU~QHxAtAS3Xi!_Hz}%@uZNH%Ip>m9WIr0kxOc}fX9eOWh
zFKfV>+SD$@kESKYm{F*J9<G?3TsD|4f)J8N&K`NhMJ_{$tOBOGp*NSq=4&}<2m<9d
zfkc(GDpy0`)tH9_e^)uW`pEK>0XQjr$!<~v(J%&RMuQM+6CkmnYZDGlOUdq}%)VA&
zl#acS%XE2KuX~7IamK`og@C`21~*cEEc#PZM6HT*Veb_l&Ej~j0zL7p0Eo`mMu(=X
zJ$v;&Lya75I4C^saKROgfi(fdP0C$GM3WyZn%mm3yEI>|S&O(u{{S<}ihUp#`X&_z
zmQBma;82#`C;dR5Sx09e07FvtJLhZ{9R~|$FCdU6TDNUwTc9kNct?8e@o2MpQDrkg
zN?G+aYtTjiUPA=RX5o{4RYu}6;)ET>TcgL^VpfIpluJ|lQR(_)>6k%L^FZmoK-Wm-
zR5qy0P)hm8yvqOL>>Z;k4U}!s?%1~7v7K~m+gh=0c9Ip_9UC3nwr$%^I>yU6`;2kV
z-uJ%y-afzA7;BC7jc-=XnpHK+Kf*tcOS>f5ab2&J&5hIOfXzs=&cz|Qmrpu6Z);`R
z0%3^dioK5x?o7t~SK7u5m{dyUZ#QUPqBHYn@jETeG>VU=ieZuJ;mm<dNK^GW-iRN;
zJMLxWuu(|x_mSRGSd*~}JU=A=3LvF_&rALrxn(a&qn=Fr_0RX(+v|)95jZ9I%y5OJ
z8k#%`EfqvwTjiiNk#8u<7<<OCg8%m8PJOi6XCeKq?fIF~GZ;IgeNbzjX{I!)rOGo^
zb(S|7*C-(e9n2||up+Eq!v`F|goG<|KJCGIpNY`t-zw(!Oj>^j>dZM7))cw?a`w8R
z%3M0R=kdOt^W^$Kq5Z%aJ(a$<BgXoK?$Zt>(*qFpy^W}Ij$h+Jnmc9eaP(vB@{@8t
zz=RQ$x4XYC#enS$fxh@;cSZ|D%7ug;0z{C8I8h{KocN-cyv3UG_nk99UNS<TfllVb
z-!E;EJV-QhI;5;*do3PgAKGT_k_}E!36!e!iSgcK`5T5K3hKY>4ki^OFkYea`q`rs
zG@qdMI;4ogcd5Tr`di1JBg4I*6CFvCID_2SN5&)DZG&wXW{|c+BdQ4)G9_{YGA@A*
zaf}o^hQFJCFtzt&*ua~%3NylCjLtqWTfmA-@zw;@*?d&RE3O8G&d;AVC|rZrU}jx#
zC-9SF`9;CbQ(?07o8Q9E12vi)EP@tOIYKEKnO@-o!ggkC)^#L-c40iZtb4Y-cS>$I
zTn~+>rn*Ts>*y*z^b3-fAlne+M-*%ecrI^rmKAVv23cB`aWD?JDJ5NIafRvRr*~~C
z)99Afs`BPK!5BFT)b_^8GyH*{22}yDq;be`GnPl=vW+ITnaqzl(uYOHhXi}S!P+QZ
z4SwfEPuu&z4t#?6Zaw}bvN{;|80DfxCTuOdz-}iY%AO}SBj1nx1(*F%3A-zdxU0aj
z`zzw9-l?C(2H7rtBA*_)*rea>G?SnBgv#L)17oe57KFyDgzE36&tlDunHKKW$?}ta
ztJc>6h<^^#x1@iTYrc}__pe0yf1OnQmoTjWaCG`#Cbdb?g5kXaXd-7;tfx?>Y-gI|
zt7_K}yT5WM-2?bD<z<eYLu_3x*<lg4bNkg&D$DcqDqv0>-}ym*?~sZ{FgkQ9tXFSF
zls=QGy?fZ=+(@M>P3Y>@O{f<p(Y+FVc%_jL8>44yU^fP>zNzIQ0(&O$JCd_!p?2;}
zI6E1j@`DxzgJv<TxXyL6^5n6)9{m}0v}k;7Il-V;FrjRC7sP(l2GP$hLwm*ric@k^
zpKW~m=Z9d3LPKuy#Ucv7a@GH5iTvMk)qgz%LDMhkNzB60$ywRO@V`F(#{}<$+Lzyk
zBGN|@S`&Q)ohmZ&m1+)n<8~&A63wF&lPan&L2^TRqD`G{H|Ez8+V?0N6_h=G)a((g
zqk8bNicvM$w>qcE@zgapQ?tophO14`=14DUZ*#@%rRi``pi0lkNgidSs<J+2dQpWP
zSDpqd{>HGjXK8gO{drQoNqR&tRjM4>^DtW`)fiRFO4LE=Z+nCBS~|B3gZsh`Y?-$g
z@8@Z$D7C!L9l=SWoE;(+*YirPLWvBd$5Ztn3J3EaGM+#pW#@{3%yksGqy(2Bt5PVE
zf*fICtPp77%}5j#0G8<=v=)LR>-a3dxja8cy3m$=MZ2#$8mbLvxE%NptMd+L?mG`v
zF1cANFv17DqP^P5)AYH<qPZ>DQWHk*s~HFq6OaJ3h#BUqUOMkh)~!(ptZ2WP!<pQs
z&7F8EdRVinVjx8S$;n{FFXXP7?z`wh%X8F%_j6OZfP!YKwNK{_hTgMk`8l|R`k0g~
zjrs#dgW3~0Nn;R7I;#3U)OIgYRi``Wo>_$TBV}!@>Ta#eQS_{ffgpfiRbyw1f)X4S
z_iU`lNuTy86;%!sF3yh<Px`V<rt7>?$5zjW4F?6<RtQC7ry0E-s_eBi_2aOz8iVH2
zt#@RE?5z9_ib|#LgwePCOZ+O-0?8nMWnF9@Qxo)lnG!d2D%|!$Ny!F0V-rWZH}wkB
zXfNBtIgQ4Lxh`g6$J?XUYp$_ps^}$3QKZbp4cdiH4mu@G|E;UZF$iW{edF)%+vl;m
z^WM@E+-}BoQ<rONOrab`qWGwfZ;?225504qe{vAdjrB&mDGlO!YXucN+JL*#{%tw4
zY(+V<+|9nKcfC23)=f;6uUS8fw`xw%H-2V}ojn9a|NQ_xM(+@rpQuWuf=~FHZ_&sx
z4W*sE`8Hkj)Ajca*g!($O~EOpNb=&m`Y5L<H+z6&)|%jzlJzgm_rSJZ|D%InCX@h+
zSmxs#QM^Jg-n_a4`#f}UuN^W6^d(NXmexrcLT-&u2(cBpfR^vmW+G_LA>E9Ts-TnA
zDyx5p1h$Z3IsHv7b*Q{5(bkPc{<PyT7!j#AF)9vzxj{PMQWZ6=RK#rka^r-LIw{y1
zl8+)PnvSw4s`Ev6nl+~E2BiyS%|0{nOgVs+!F)ee|7R?3yUB)jeU)#FUs|%@zfl$Z
zcP#&dgF5{K!-dKJd7&|QtB!7?maaaBXcrzGUCTrpi9sbQTuO#IK8G!j5O1}Cw!Y1{
zP3KMCH`bkark`*-hB<4c(j=Ei(;^`=wazKa<7j*&_5JVbtNgbmrb0nM5mDAKgnlUS
zwLhnZs);FZ9rTOv(p&cZr=>f`2Wfxg*Z#IvQ;W_q9|GqXGj<@abo)FyPtzI~i25&o
zC!cJR%0!}lLf^L2eAfZg7Z69wp{J?D6UhXr%vvAn?%)7Ngct4Hrs@LZqD9qFHYAWy
z4l=2LI?ER&$He2n`RiG&nsfLv?8$Cl)&d8a-~-N`I|&EPa@Y=v@>0Gl?jlt>AUY;H
z`**5bpS#VGhdp4pKbf3iEF*>-eXg_$bqt5Dc%q0+)R50>zd^l7sN5R5Z)Ut+oz-8_
zJ`Z9HE9(=wRTD)T=%GZTEi9K5naPzlfE$|3GYGLRCLsnqLi8Sc6y&iskqA&Z$#7Ng
z7Q@C0)6k;J$TlQ+VKZ5)-Ff_BNoIMm+~!@Cv1yAUI-U!R)LHc@+nSUzo$GlRb+8W<
zYPG%NFfr;!(RlnvBbN~~EpT6Xj5*^Z&73tdIQ$LZu`vkfzdTKa5|JJtQ_rm4g$9LO
zKtgYVdW=b<2WGM3I_j|Rd8gZ3j;)S#AT(aP^d>9wrtQS_+K>pZDX^?mN!Z>f^jP@1
zlJ;i79_MgOAJa`%S9EdVn>ip{d!k6c5%zizdIoB9Nr!n`*X#%6xP1?vHKc6*6+vKx
zmEt|f^02)S_u_wlW_<`7uLQU%{wdH0iojOf_=}2=(krE<*!~kn%==#0Zz`?8v@4gP
zPB=-O-W=OO3tD19%eX>PZj3YfrCt0sEjgTd#b$buAgBri#)wW14x7QcHf2Cneuizz
z368r7`zpf`YltXY9|2V{stf8VCHgKXVGjv$m!hdDf0gi`(Q!(Pyg~FO28Vr#!BYP|
zI)qG2?Ho=1Us9dTml}-ZOR?g5Vk)f+r=dbCN*N1=qNfG>UCLeA8pd3Ub-pRx1b3FA
zEn`CIMf`2Mt3>>#3RkE19o}aMzi^C`+Z>8iIPHSdTdmjCdJBtNmd9o0^LrJc9|U9c
zD~=FUnSyghk7jScMWT|SHkP(&DK$Z=n&lGm+FDTpGxfoIyKV)H6^nY~INQ#=OtIT!
zyB*J=(#oHf=S)MNOncW->!c0r0H#=2QzobO&f@x&Y8sYi-)Ld;83zO$9@nPPhD}yt
z{P`*fT@Z(?YAmF{1)C;o?G@dfd2$c+=Av*|;P@Yz1KnclB-Z-fJQ-=+T*g>0B7!g#
zQH{dHt_%wj=wlmT&m59)TQ~xK)gB6f^EY$=1zcbGf~Q>p_PzDCHR6lndGmqPY2)&w
z$Th^K%1v@KeY-5DpLr4zeJcHqB`HqX0A$e)AIm(Y(hNQk5uqovcuch0v=`DU5YC3y
z-5i&?5@i$icVgS3@YrU<+aBw+WUaTr5Ya9$)S>!<@Q?5PsQIz560=q4wGE3Ycs*vK
z8@ys>cpbG8Ff74#oVzfy)S@LK27V5-0h|;_<C4*KrhuTT(XjS_*82Kt`QAr82sMML
zS-68!S>~=j1TTZ9_1LrbBUHb?)F4fc)&F7hX1v160!vJc!aRI>vp*bYK=CB(Qbtw7
zDr2O^J%%#zHa7M5hGBh#<F)PMv7e0c?twyo1lAE?Qmn<A$&;r6kkp=J7ctT0G~ud(
zMc1r=k{0w~t7ygcr6;fLb?|A1(*8k>8(2IBAk}zdhAk$`=QYe^0P6Bb+j5X)Grmi$
z6YH?*kx9hX>KCI04iaM_wzSVD+%EWS)@DR&<qlaqW5;DH_L|$%=*-?eL>n<W8Ry`O
zI|rLQaLSJ*n{eNuE0(3oOg*Hqdf-ps5^iq&rbN$n&<e;(vOh5iQW!`9j?8UY(`Sd6
z;!k9YqT}tJmu<qCo*-k->WsSBc2VIZ>C(jX((ZiV0=cp}rtTO&|GMvbmE4FpBF5Rd
z6ZG=>X&>N3?ZN2^11pXEP4L?XUo`qrwxgQm4X~RCttXmZAhnhu4KDK=VkKq?@@Q_Z
za`*<UMhK^|d#jpx|2R{30s1lZpK)wnk7>x<R|(DD`xAWNFjr?s<+131A-D2OogpLo
z4@|nTJi**aW@#MR?t&wG57pY!WQ1&i^l3tfvZRtM!_&0MR1s`i*Fsp2h>yHrsAEsR
zV(7)2+|h)<aEEu;*DiK*)T&{?t@NsM`Fl3>%EHHLD3>Qg{>G|ns_%5g5aSzA#z91R
zMDKNuIt@|t?PkPsjCxUy&fu^At*yUYdBV!R_KOyVb?DO&z$GLJh9~b|3ELsysL7U6
zp24`RH+;%C(!bWHtX&*bF!l-jEXsR_|K~XL+9c+$`<11IzZ4>se?JZh1Ds60y#7sW
zoh+O!Tuqd}w)1VxzL>W?;A=$xf1Os={m;|NbvBxm+JC@H^Fj$J=?t2XqL|2KWl$3+
zz$K+#_-KW(t<BO`=w8&{5PfdIQ3)fbJ}8XZ>)MEg6zBSF8XqU$IUhHj+&VwsZqd7)
ztjz$#CZrccfmFdi_1$#&wl~A<U*BXXA3F{M{iiFJ5^Vdf9$UCACCMLJ5P*6%*8;4n
zL@GqW9u%v&$)(Z&&`OVEtKdKQe~dNEB$7rmX^UiQu!G;I`1fpM$|_$Uu2tLt7<Km6
z`*`c!>*RisBaBy~)w|txu1QrvR1?)2mb&m2N$C(5MS%hSX)VJnb@ZGXB5^%(<#1L@
zL^>fBd+dEe`&hxXM<0A9tviIs^BDkByJdc~mtTYr!%F7Q1XnK2$%h$Ob30*hSP$Bt
zDd#w{2Z%x^Wpv8!)<D=t_6B>hm>6u01mY!xmPgwZ#Q0148)SxJc3Udt!-&}eRO^LN
ze26pQB!Jhg&Z>#FD>`C`sU44><=v>O>tJdLs!HPpV#AM32^J@Za-9J(CQjKxpzXao
zQfRkWP%g9P8XV21MmoHfx{DICLSc*t4qVeQL9t}<nRGFav!f()eCpq+YW_GwK-fnV
z?jqb}{AK_5<`9$kah)(SO=agaV}Q{e-pb3ZxN)+$*i_7LT6ioBMQ_?b;=xzVyshDc
zeCnBznuBw}_vL2>&Pz0rM}YTba@XsD=XMW@FxFM{QYQJHvM(JsUSa3mcTUl9^qcVA
zBveO--fqw%{#QGR1vy;x88+qMcgzmcYc#8U`CPPt6bl?uj%w_`b~9JliftnOa|ziW
z|6(q&STs_*0{KNa(Z79@{`X&JY1^+;Xa69b|Dd7D&H!hVf6&hh4NZ5v0pt&DEsMpo
zMr0ak4U%PP5+e(ja@sKj)2IONU+B`cVR&53WbXAm5=K>~>@0Qh7kK*=iU^KaC~-ir
zYFQA7@!SSrZyYEp95i%GCj*1WgtDId*icG=rKu~O#ZtEB2^+&4+s_Tv1;2OIjh~pG
zcfHczxNp>;OeocnVoL-HyKU!i!v0vWF_jJs&O1zm%4%40S7_FVNX1;R4h^c1u9V@f
z`YzP6l>w>%a#*jk(Y82xQ@`@L(*zD&H>NY`iH(iyEU5R$qwTKC5jm4>BikQGHp^)u
z-RQ`UCa70hJaYQeA=HtU1;fyxkcB2oY&q&->r-G9pis)t$`508$?eDDueFdW=n5hJ
z08lH$dKN$y#OEE@k{#|<%GYY=_c~fHfC@pD54KSP9{Ek@T47ez$;m$}i<r~s6c)TJ
zDCEQMN<71?Zuq`iOv09SPmJw?(dhlCAwXunqM`?P$)+klct;@=2<#s1FPu_WQI#N3
za|!GhV}(&!)`Mzf2+_>wR}3?)hbkwS$@p2iVH0IM$lB*XYA+#}-re|UNzCE)SOYwy
z=Y!fkG4&I%3J(_H#UsV#SjHulRIVcpJ`utDTY{k&6?#fzt~@Om=L(vs6cxAJxkIWI
z@H7)f2h%9!jl@C!lm+X4uu;TT6o0pd<Av!LwKdbG0cihYuL4!k943i{SwfXpXc4>7
zteFQ(ND@djf#o2kTkjcgT=dHs7ukmP0&l8{f;o3JuHGd2Op*?p7?Ct=jA*tIg{MZk
z$2Lsc0e8Tdcwr<wO@`i)rYT++Z=4_u(Wpvbm=)@;vy@S|GaEZy44KcddbPYE|6M@^
z>jx|_Ok?9uB3Il|^2FF%X#ck}WoIvrzQXN%kT$9NI{79Wm~gZ3`8I+O`)`n30feZ(
zDO-fl6I<e~IF_1;5TLCuI8uZHkS_a&(7lCr>G3c^8S;Y_M-)+^CmM0tT^g0?H#<c*
z9oHF>>H8!oC8W%oU!~3|DJ?)~LT9*&GAQG13zOGq6gs*={cu<uRfLW*5SCw{NoEua
z&>|(V7{R$y@{-iV*9q@AD(#Ktb}J&3&k|5Djs$)9WM7!6#EaJ_ilvbfUvyh8c?-{n
zfu<JZD#;FE(0Hg&r_rW=qAj53>FrC0u6}UJZ7aj@(cNG_(CKgjQQTA-UK@-MVmick
zot}6F%@jhq(*}!rVFp5d6?dg|G}M*moyLriI!PQDI;E1L1eOa6>F9E6&mdLD>^0jJ
z09l?1PptuV65gm=)VYiv<5?*<+MH~*G|$~9Z3XEy@B1-M(}o&*Fr9Sv6NYAP#`h{p
zbwbUE3xeJ;vD}QMqECN)!yvDHRwb7c1s6IRmW!094`?Fm!l~45w)0X`Hg+6Y0-xf#
zSMemBdE)Q=e^58HR{kWrL5-H0X6pDu%o{0=#!KxGp0A;6{N5kI+EoY_eTE%2q|rwm
zekNeLY-R?<KePvv70Vg%1Kd{Y<xoVBHOhBx_+afLZ<)^ME+36xS{!a+M8miopa^2e
zP`@au48k)(7S$Q<@?bU?%y#yRJ0ma4zA9BgIk2?wu5Al@8}#Q*l?q3Me(vt&q2CD_
zrpldR^R#w2+=LX~h4Ne{M_J2qS#n>htk!YP2|@dbd8TWG4#G)=bXlE{^ZTb^Q$}Er
zz)Fp)ul24tBtQFIegdI37`K$VR3tVdi<(fIsu{#QMx=$&CK9M8oN%3Mk;>ZPd-;Q-
zn|sSKSnc-S0yrw#TlA$+p{J~u=u98s>IoL@cNLOxH=+1m?;t1bR$vR=M$US&Z8DO3
z_&zhQuId1$wVNsS=X?&s(ecIi#00o{kuPs6kpYkL$jMyGW8U7mlCVaZeEL=HsIxqm
zFRLxWin8B>!Dc#9Z#t0RNQiR-@5J+=;tC7|1D*~rxcwHa5iIVD@99cCFE@BukUC-S
z^iJdt?dwU)kH2VY9?|zVShMbZctzFRz5Q4tiXa^>@U%jDYq}$rSyc#p2wXr}mc0qq
z^lT>$y)N(Qg0dwmEwTopneoU(y)>Mj+f{iHM0o|>ZtCg-itPj4addYz??aE)Rp&hk
z_SI)%XeSf=SjZq18h!Cc>Xy&EynnxdHQ){(x@g|ZA%`3LU^KzX02c5N;F#tEk1)7v
z(|V9tO3>?^X|kQ*rRBf4>mWW2$-Lx})|M7z125&VHcxsCqB!<$l1F$zCrJ+nm0f3Z
z%Hq^=SKpHyV2@Y*Cu2x>fXC0SscnR*($zEB{KOniJcpn@e`PMH*_Q6*0Z^8RNCEvZ
z+UU9!927p9YZ&g=bnUvQUZc<V;^M<G%xS@R=D@0n9X`Vox8ALZT{5e&UC13mPi90i
zst7`}8|AC_dvKuSZ3kE8z|`c;8Tq+ePI!(x`ob;<PQR7NR)oke4*L{$RMcI%3^c#)
z#3#~B>disyn;-4;ACXOe-Xor9K8Qbp{ldE17+G@VQT+9ZJQ*9dZoXfU2u<x^@|i5=
z)IRH;tJ>e|mMhrrZk2R7&~YjFW4`BTq45UwVc6JORKU)wBCTa<qAkDbMtHS5cvwoc
zi-ghZoS5)0kO58#)A||SRT7?((k*G<ak?B}`^qYVVBIGQ(H~`Yn^dGO3m#nN=SECX
zFhSqSY*6eJUMX<PVP=z@QM$z6KmSF7wvl%F{vWc{;1`Sd9~9=Y7EVrI6!;$;;%ge=
zzY&OvByIUG4SMKPS*l8#lBmQTROA9&vztUKu|y}b1_T-cbqm2nd9mIsedAy^kfHN9
z<VFxBs2`I57vU(US=&6wUqBG+N+wt8U3%*0$$uzVL-OCi$0?A6#L)oonW0D+v~772
zt(11yp0ejyN?Q%09k$D!EtaQuY84dwrXz`irPcU-1U@F*(Nr`vE=%?M%5AlWFiS)t
z1gW#mM^F)|N&<$U3--LL)~-MOzmCk2sl-VNF%i{*i~UB>nITh0GD}s$`C5pb(9{b9
znwee6j%?-UV)_7opOioCf5@C<E+c!Y4u^LhuF~FEKiGi0V}<VK1xq)}xH~SM1A*jR
zm3nN6EGZ)pHplKXOgc06=u1#k(zkgR1b^!min7`9vqG}on{xqBeR@uBG3%Asv*hKC
z%J;MvD7|-!6l04OAjwTuxWVsQ)+!!_O~y&0F1pJ_x9<bhZ6@5n!W^!(O~%Qr%CJY=
zL8c@gpswPIJe!n+mg7pixWC2KatsRHsO)C?NF}Q4)K2Ti)L#<d1I4^iJR>?@w^@g&
z&6<hPh>8+oMmV;5JW@TT63&CSDrfYL2$L)pVseDtAwPwleEM3F^-Ufn3PpfxFmx6o
zQ`Wq9x#d$e`VKn5LOXNsrqhGao7~|s(u~drPrZ+;aP!C%z4NskZstCbAibD}O%8Ij
zb~C(taxco~WzJLxhL1T}3ctXMbV6}_z=IZN9L0|SxLSe`$X`<)BhM`$1&&)e_}fCh
z=idVL<+u6Vn{&ksP*ZLlMo$fC`dtzF_?~L?4Rril2G4%v5^7sUa^&8aMtMX&mtapl
zD(dW|cisM3fqMaB`8?QbkyiUl2g>hMB5EoS&IB8T<h-K}Xwcy|!mwXLSJ7_p93fG3
zWGXigT_FQk=(qVGJc`g6QPn^je?h+amV7%|{ulkKqt8iq^b02IUoiO}5bgiMha14r
z<bNaD4XXNI<s#Cj&7TfO9ShNta8*H=*?<nEettB;*<Va*WPdCI?AFdS<<{CvuV7vT
zy{}Q9b^|H$#8CbdDcUf}--yNUP29{(-F&-JGc*4Il%|_%%<uP<P?nq_NPXd?mfUuZ
zMsxGH-edKDZF<sWm5;?XW?|*>doC<zYA0Oib(G1n*?x}Iq%^f0n@bTq$^^$YS+tDM
z3O6248R>~)b$nT=`%GgU`k-)+8}`)F*~I~DXMaTP%kZftx11~?iALs5J+&Rom#p%Y
z>dH}-euH4u=_V3hc6^*2WMtL!9%yRTJ93p}@aV0zdY*?xchFI>m+UivV=;aMFp0P~
zwB8P)wvV6D-GL?6hJ#g7Hy7=2i^&Od#S=j!;Rc_yj<nF~-j#2GDgTDCWkmDd9J`gC
zLMuzQ?@%h6Wh%oGRazY5?Bm%IdDA=4q?YWF4Sv%U+9*R)9{^Kg?q?13kP7__{#wS^
zQl?T2056te!$}HnP|G(AKWbO{F(EL+<lZqJM1By>O!*4aN7{vqzg2t-R|Dav%_NDk
z`H_FVlSi==(~f-#65VmQ{EE92<E8GKieH498+7wRW#_u9DlI3Cby!Ls^b|LT9X6Ho
zv(ZKw?R4Emc2w@IpmKeQoXTCmXWS0J2uJV;5+>x<03lwo5p)s=ZJ^L7PlS>132Whr
zR6v~t(#I+(`usYLCoO;Rt8j&b^5g_xgs*98Gp|N}b>-`HtVm)Msc<Xj$}ya?vHydV
zA=vN{+|GqZ)(_1~n8M=D-M1l7KjPZJ!qzt+!W%HQk9GyyC(;R@f308pa<s6ENT@Q{
z+;q#~;OA}|)<$f6J<?qbV(6uzVLqvxucbH*lgS<A7}5AW{rdWM+sZ(%kVP1fJ$W=K
zy>D)71y?(K6DRCZV26RsHPHKk)EKKZA%C99t3$t^B0<vvo)G_8YN2#-z?}PnjKLRV
zMF0KT^dFG<S8XLz#pXZ2;``uvkb_2vR+<wEA~m+wfbP9R&A^_<YSCQ~EYd}1q8wfC
zHnHLV0p|y${{|w4(r4zE=+NjEP_N3xzdmt(emTxEH9q-B_;r<3Ly!pq92wpcPw}47
zdRb2PNLKGBl4l?xtn}yhwW%_vXM^pjXxq&8qVGz5Kkj97Z|a7HXOT@@#|ZxA3WBhF
zPrZy?CUN}a`QS87$0HLN!ZXv-*?Q>-k5@?E>A-YMbFe?>ms?J?_guHHNU(;id*>xH
zTrtam+Aq?n@-y@uY@A?hy?1qX^eLu_RaH4Ave?A8NapgQF=C%XI7wlcCf4<6BRo_%
zBXxxc*A6-3CruF?3i8HOdbc%>N=-iiOF+9HX|ht6SCkz;A^am&qi_I&qk1B(x<=(m
z>QG)nswCOLl_1{SZ@_eE#m^qb6#6DoMsB*)`17ui+XvF%(}|J4G$z2G*;E!1ERnAH
z@q%=#uV6kBddqy4=g>!VTV)9*1=i{wJ}Ep!I*?)uJdA(LwE?(!?;}_u=^M2NShWC_
z*7l4aBJ=!QVU2-iehgb`$vOI8zkm{W%QO~?xOD;NgI;Iqa3#^$^U5D&McReLe&qs#
zR<<lIr`pbL!li_7kZeK}>^@QpR4#W~Laz+QBsPt@3L#KF`Yr8}jgHe;5(<zs`u#E_
zEu6e5$x!l(wNZ7}LTip%m?o2q6g!hY@F5iq{O&$wO#ur-BIXscI+Q$Krn7PT_J<<|
z`Lpp3T8$IgW(`ZEch55Tr8R@=@5Om1nbE)Vv(292uRl^|AdrdK0t)gB<-!{CDxb0f
zOO&*2{gbqkS-~a8i9&ccGW(DS5{f7qp*Ey`T~b}d>cfpQ=;Zjtbt;c%y^#-m=hqOT
z;KAYakW+$w0&F}>K10&SiPcD9SrDOuczj@U#W})5jGU-_htU`U6Q%wdy((%?J}y+$
z=$4jw1N&#xa0nJo)qTxG{D(`3*#8tY|67hJRF;)r6F|#I`Ar6I0aafR<U5lrJPQgT
zf@nhx7Nydl1Savval4U;rfyhI`waeo|Jus4bY9r+8-MAk_74;xdxsm7!=DrBN0|$H
z@9*!B-3W_kl({-VOWk<Xz>a=kr-Z0I^}9xf^u;G5iEQCbpv3b#S#%H|HYHsQaHK$!
zU#3Fpz8*^pK%RRmX<_09eIVziB0jOgPgFnI-*QcwEBtBiO#v!>{W1cLNXyw3D9M|A
z*oGy(u8BkDA1c;MsXmpK^-~pl=We^RYnhZ4bz*)Q)C2G+E3tgx9PzU0T>c|1ilS!T
zyE=bz`=wskDiOi!@!l?Y))#%{FM`}7r~X)i1)1*c6_2Q!_1{)fp%cS|YF+Q-CB%d<
z=zYus`Vt@Mx*a7V)=mpLS$-5viaKgNB=+zN657qy0qR94!cTtX-Z%KBCg4OKw7b=t
zr=`7q5Ox=lJ%!G5WIyNQC1xpqYU0<e(q_pNb4EKEaQ9G(?Hd{anjulzgD0pMCmq-K
zWaUP)LqaTR7N^#=_Mhl76HX6c@^>{!I$hyrk!6%De$gp<_*Gc?ES(OwY8U^)Kjgc{
zSlhpXDb|;{+y9`u{EuMz54rlky2~p6xX2>MV6BZ&k`$q%q7v(xYps2wr9e8^4<;CB
zc)eA<kvQPmw4$p(IPl||fea*!g6|iiC~w^c$?``vaeC%h>T~B^rjzO6<4BDDH;il6
zFsM8jL+agQ;zazW(uiQjM%fPf2N~_p{cy29XP11_lQFpt`t#9nlk}>fv((FZt-dBa
zuMIc4HmPHW04n0TTG9ug9;&OV9euL$Ib|+M7}}L~z4e%%%<qv#JL{%>b|r<G;vD^5
zNT`!8?Zrk{nZ6$)buIANW8~=A*b*wu3t$k?ao&7EfJn;bJVP|2qiMPwswvFMtB$5O
zn_dM-#TG;u3MkS~dfv4LOY}Ke0G>~6OQj(S2d7XfYn#xp8;KQ55UYu#gY*De5j6Cc
z#R%?rqwpy7I1(kpU7B*Pq=etXeYUn<K1FAFc&lV69S`~?gz<oy+nILJtD5zKj3&_#
zQUm;smLma<(MRI}a<Uq|J=7?$(?uVPP(MseqU~8<^m5T6E(Tov(Mgb9bR`k0@KO`1
zqnA-;DxI1h?f1FtgV2r}B^qIsi`f#}IA*fUkUef+I}m9;rO~E!jBUrGsMsKmaVrz0
zh_g(z2URF{4VcpnDRnSJ+>04jg%ZPjYqQNa$==yTG=6KX+=;i2Xg+kjV2T*Gc!(ef
z`Q4fR*TA=M5-}z+s%YO+!K{k}S**ic&>o4_Tmv$EQTOp7F6TXPCj-UTXy?OQ<ls)j
z-5-M-F%GJ7RW+H-JiQZkST=O)d`;C;Pv$vP*TN$LA*=iFE@)5cNU6TX*H(hJFu@?s
zmj{E*uE6sT3uoD)4;Gml3T2#(0z)W0qb{bA@`*`&VJ=R@A<1t`Y1(qAc{ym6G|i}F
zBGDN3m^02GkM>=%*y62Qajk{rXbR%jMCOFMiVE3KekQa4xR}B%=iPtd8BXo~q$OX_
zSp910{Ew;m|GATsq_XiJ3w@s(jrj^NDtr(Dp!`Ve!Oq?|EJ9=vY2>IfrV{rT%(jiY
zi}W@jA2iqd=?q>s;3%?@oi7~Ndo3Ge-2!zX58j(w&zVlPuXm3rcHb7O0RsM|!Ys(b
zh<z+m3~TL^(&&Jtr#SnIN4~11AiI>(=*&Aywo3vuJoWZnU!u2_4bNkDTc&&bCYc%T
zM~~xYxS#3KXFzQ@OXdc%9QDOxqiTd_<fgK9y0x~~kUTV;<B&f_-Tg$t^aI+zfhy>>
zT;(DX9{5dIuC4pO_xy+3{Ov)1I7j!Z)<V!m4nK%vFD3-h_oUR-^20iy?)^5HmPW2N
z_$`l7>6&nHUvTRP>VU5dm#<il%&q#mOWTm)Cg3hydn%XT$ndu0-6fT^6QI7k@bi*q
z)|RKCdKV0iq19&pMb`8WDML8zm?!@IR3A<KpPb!i;)9wdKlob&G+{l@$M~hSH~OeT
z7wFo|ER`(5=U9Noi7zpnk7)7vZ8h;ywTE&Gjnn42wQWRBWZu%X^ll4j%Mi~f-H0%p
z>849icG)cvl0QOPkCIzG^lOp4#UcNr`VhBp(Ha%8@KPlvT*5u!v_$b#b~%sn3K{mu
zaxeD%Q~{;Lw03ZAq(Pc<UD7Y?b@Gw@1v|oz?@4dnt}kf856G<#&}pp=?lY^?C_{{c
z^{D-crn@4gL!lNERn?o-EO5nCUgeTRmMZQz+~@>-IVj>n*h3l2{sqioCMGatQY0kx
zi`1(WWDQ=;gmLSGptEQ%UFC)th@|71<8eiRtX&Mx@#1q#nMF_BMfQdS>!!Qkx2o}=
zuqRi?`UOX5P3fP%M+71Q$ctH4Av}bXED#fQ`KR4!b~60nsAv^*M7c-x`|~B}XIuq%
zlqIJOf><N=sYlp6#!uh>WvlhQ<Rt<63lPj-fDriiDa8MYJlM?%Ao(P4y4WkHf#KvR
zP%02vnCl?n5upo(3kZd;N(pk=uaTAYtF}ik+WLR?2f<3<J$-+p>@Uw$du|14)tZ?;
zPNZ|xZSwp1y+d4sut8E4*l2JWR|~o0A9vD-<?+N;GE5dY@>?<CNM5*xWUWDY>zC-w
zDc@=wE1YKb*OMSi_Kx}&w;#h3>sHp|8^hnA3w?-WK)X?@Z2dgV7`9Cupf-B2RE4x^
zwlw+~!V9C^tyb`J;m2}ksD`w}G9`yu(^--{SQ+wt^Fu4Li~Fft!3QO`upSkAU?o;#
z(1Q%GUVWbbkTK-M=T+ULkk3s6Dc9`G4CO6|=&<KbOFBlY-RRT^c{#WxwN-V_A8;G`
zGWOi;#de&MK7mY3K0Ve6X#{cw)L6Gtad%C<<Xg>-S&D+rbJQ$`Y-xL~o<EnPk6B#j
zvv@_OX%zc>l;kc(l)VbU>{&>bV+*?ua;$bnDc29RW+Ig16)Vf6=L|fMR_P2b7>6}0
zdlB#-gj|j*C~M=F^2=K*k~=tl6YM3SXXi&K-`EvEXnWz&4D-^hQRBJI3gKKDj^6|>
z*WhHSim1qAffNt60Mve9lfw^+&0bx-AM0%j>QP3%W=S@(l=(nrJ678mRQ(#+sI@d{
zdb#5fo#T;hK7xJ=M58wZf|?DHwD%!OZ3JrTGV5#{cfQwuiMvz%!CQ}CubJ7`z?<y9
z|C*`VTK!+Tj{~Zb%KtII$_<O%`|Ue1Ni&HhGc9ykZKzTrtSXvP&|&vQoJ`P+F$G)d
zH}%gLNGkAB-Y3vE<jr*iy>@rSF<+KHNV2goc)a6hP0oHB@3LLKSH2w{um&J*z1Ka2
zLIR>lvOvh>Oxe%?3A@v<_T|}${zf_&@C~^FCo#jB(W9VLO?DX{)n(BQ0(V0`mI|9Y
z#U3WwxixJkU_NTvA>5q(A@r2dnEXJp#6B=pww$XGU}~1~c``UKqQb=^*2P|4Dq*_!
zhY^i61Sy%T5$Td0O6^C>h(<nglq256xJy|$dp+t~?<5Mh)t@*id}?lo;OTh2equ6Q
z6Imgbvd~>xVvT!}Y##WeT8+s+Uuz=7)~V$>!zU;%d>H)rm*6^IrsCma%|cifwDLk_
z!^W2voQ)D;I$=v2E>iSaBw!d7aD+|LWl2iD!cBw`Q5p1~fk_xGiPi8e^mY&#viTAk
zmaKL8m;JQ4bY(n<q7FTi$+xX1_SpTqXYDqr<p3U<m*_g8vw)GG^O_YY6Bfr5lz9Ok
zr~LX;2|8o}c0Z^>6uBZt02z#noM<c^rb*}ZzFV<=yjuAzBrU`x(nWvPBDfT3PX~}}
z3Doo1IDuLDqa;A=-tiIaefFE7()?&h+rAO85y5!7w%#gWb~^GSQkKi;=7E&0JZ-v0
zU^YC`i$~It>MxTfF-RzjKre-c+@B)#J3pN-Zv7F}JtAwNk3j?OkpVCL6W1)Q$FLAj
zGI!tX;g`O<QQ*h(B+&a@%0TQLZQzC`=_1x7YISx;gcD+89G69uc@~@K2;updjESr;
zmt5o@9q12kl|}}l8^H&f>{%pt=0|q54Jyj##w*4e*|_;Us2Tn?!#^R(>u}|FAw1G_
z#wQsagnj9$TAC`2B_XgB$wNq~Sxgl?#0+QWWcB{G`c6~&SosbtRt}Tukw`TQ!oG1=
zYyL(y<;Wh+H24>=E}Gs=Hs<SQ2GhiuV3Az+{drOj5Pe_+jic!MFM*sPCAZ!k`P;X9
zs(){l=^qjNA3VQ8%gZ2fA?Y*I*tBzGA5@5Fm`G}$SW-elL_mTF3B()-W{HNQaGEln
zVr;;SYDOVIYw5FKR<W1|Uc;wVP_g(zLOiIxp;i&|ccDVN>2%fg;&Qdvr74{E!R?Bd
zIRQ?{{xkLJ_44P@y3^#(Be%(pk%$liKbUUo76wSoVfJmt9iTKL3z{uW6L&?jYg>EY
zsx{<T6R=`Zjz?z`<vL|4n47yi$F7jyysEJmnTK}xaU~PbHW2<I+_M1iV=RS$RkN*U
z*FN}AXd%$TeQ;`)qB~%Z1C23=M27-4W?-(P!P!7po~o=P`<E-T7KI^WQDLEnM=_k(
zMk>kRiW@q%<$VZvbS(TKKTO4{Ad6l^IeY(F^3}=mX9|FZmQ`~R<!Q@HG}gvv3UXBV
z<p3ea$V8wgm{kO)(bIT4cquV|iDVqonkQTy7FKdgMEE1^cW44wCHO}UzZTs!M#Xr&
zp6U$_qH1QrFs#MfH`R4EOJC+9tly;)MTJy#r5^bkLTt4W;Y(P*<)k4q?32+gib;7n
zRhd#0oFRt16e1-F5&=d2j;QFIbj12&CD0)iw%l7LqG~UsI_6q4FG+zZRn&409VBH+
z5Sg}y?1xYpnbbI7V-ZSNXU+8sc-12>ErNxlBPl3ast}W$T4V?SW=6kIGn@-^`qJv|
zZXwhK4Kl1a4E}nLI`rdOi?^pd6;LZ-|8G&INHgOeC5q{_#s+SXb0r(;5ryHFsoTJD
zx$VtNDh=-Tx3t!NTlk=hgAaSM)#U}e>_-Ex(|JoX*hWmBPPdTIa-2(BIOUJ|Iddy|
zwY*J%z%W$}*;uSoB!BIJB6N6UhQUIQE_yz_qzI>J^KBi}BY>=s6i!&Tc@qiz!=i?7
zxiX$U`wY+pL|g$eMs`>($`tgd_(wYg79#sL4Fo+aAXig?OQz2#X0Qak(8U8^&8==C
z#-0^IygzQfJG4SWwS5vko2aaOJn*kM+f1-)aG{T43VJA<p``IHHz+Q+YTtZiQ4(JS
zX+EScn94Wgz4Lsq=}&oqX5?MHq@;?i>gxdP(fJ4&U{XR90*#a)G8+clOwdF?hJ?D)
zmxu>0>M|g_QRHe_7G|q6o`C>9x4xd$Gl7lAuR~+FtNid=%DRsnf}YI*yOToWO%xnP
zY*1G5yDn<c-v!aWMHW?Ij!Go=T|IryIuWjCYE<>TGv{{x<GQ9wOOw3m0T;_tt%J`q
z74Ah7*BAD6gIzK2Kp|m4>g5FhWU65q3-|-(+-rJ2WCeSJn(7Az>ej4Jp9+l-GyZ_|
zJ8}>iA4g|}q1AhEEv#uWR&$g&Uyht?fVU<mo*~tGG*v0ZIb)5oiI%N2C04^*&oE5e
zS~q>(qk(j?^D`))s>oG08pow!f>P1u71P%oL2)UC4GeS87&G?{)NE;D=my1Q9{~;y
zJULE=bG6jXE28Y11YmoZoo945`MM*`v%5b=_02*0cwzDve#3(4M}NPt`)?SCa|7*q
z-94ks(R6WH-l9fE4m4}10WSu&O`|;ZCIT%vL$_pbABY!}s33@~gIvZ0H4co|=_-T$
zF#lC7r`89_+RL9wYN=E3YwR?2{$^ki(KKd>smX(Wh*^VmQh|Ob<Jh+3^WFPB2~REY
z*Xyw_SsEfsb(q(%pbTm5&<w>5$n_%N{!{9xP~LJO0^=V?BK8AbCEFBhDd$^yih$>U
z(o{RReCU{#zHSEavFNdc8Yt<%N9pd1flD{ZVSWQu*ea1t#$J5f6*6;<i!*-Gu<#qw
zr4Q4AO-Z6(w#=ZPw!1z_R-|)05@KW<hl5udA5yTYJkNv_eBFdl61c*?AkP7$oOc?x
z<x<!;`V=49<iDlT3SA4x&zF*$Z^XZ&!Mv^X>tCx=&;EIN^S}<jK*rsfCni57MfFoh
zaI&dwJHbZ$fc&)D{`$tAV(2+G`HsSbn!eT1PR;K&nTDU^{oe;JO_f60Ts-ll?r8Ik
zPnp?LA_75_MH)D5aT>*3s%=M#)`~=nz!&Q0&{EP|9nzWyS<#!QxP;!E8&3D}?QKh^
zqGum|+;xu9QE=F#fe2ws5+y1Igr&l`fLyLKry=1}(W+2W`waeOR`ZXlW1B{|;4sE3
zn^ZVlR11hiV~p<~TaSen8I~ay#7Ql=-_|U@$8yjZsZ=Vi+^`JV2+kn+oiSUi<eH8y
z@$I6vg3guJ-8kGpiIyCuSUy6A;a(i3;+woj=cQ2g#wgs~1lh+Z>%omO_+7}saXnJ9
z5ETil<PEat5SwKtc2Q!f*eI%Erm%4Z|LSFO>bag(g#jZPopCgJu+n@(i7g}3EK2@N
zd64$77H5a`i%b%a^iRjMaprwzWz(`=7E6QY)o)gek7H)yZ-BLw^6FAoHwTj9nJtWc
ztKaytMlWGLg29W{?gr|rx&snb@XyvR_}x3fmC>d=-nQp5ab3*whTw}DfUcKlMDDx`
z-%?ek^*|Kqooy#>2lfklZ|jN4X$&n6f)RNNPl(+0S>t(8xSeOGj~X0CGRrWmm(WXT
z))DDW_t&y$D#2`9<-+JT0x1==26*gpWPV~IF=rePVF%e-I&y$@5eo~A+>yZ&z6&7>
z*INESfBHGNegTWga&d@;n;FSCGyW?}e_Qw#GTLHo*fWxuuG@I~5VA!A1pOdRTiPA~
z^AGe(yo=9bwLJD}@oDf$d+34~=(vIuPtOKiP}obDc|?@hY}J*@V|UynBeAkYa?S{@
z_f$U=K+>deTAi&=a*xv>Ruyw$UsTWY=Yn=xjf;s)6NQu>_niQ_idmzIwuL`Scf)f=
zyzK?D5a5)^D@H&qN%F6Zd0JeXX*Knbe~VLe^gi|?JK67&mB4jrapV-$`hCQT;C{%T
z*pjxB+Y|~LD9bmMN%Iq}S$F$x1yWU7@GcR91V8h;!O2I5MN_rq*gRx(k8T!1WSDTp
zr9eJO4$~H94aG^6k5p8k<Om<4^qIVR$}7-89S^Cv*!ogfj$eEPw!O(2IX)cH)b7Uj
z(D{$6AkG~zWBFXWE-m0(g0#;Me9`&$f<u4*0Yi#94G`ohUqed@1;=Jkgm1|*U|vTh
zt&nM}VN$MI(5Pb8Io*fnt5>=kFJ>4lnY0q_Bsa$@vTRW6uY?slH|Qt)Yu6Yun&pfJ
zBi!h;6x?FDs&79#PT*HSCEUsKws#s%TFy*=2PAfb`>gEPBn+D-WdfXA?MkB=<8kb_
z1+4D11mdHG0EcAyg4dneLtfJ8)RyHQl@6hWJNe(d_EjyCHf7%Xsd)S4A-4COz{G@%
z5xQ!P>AS@H@;4W<KcB|c{cOQ*2g|H}`X;Eo??m94U0Z$av_gm_-II2mAV*#d0QEB7
zsxjDOx=tIvaYk6XFlXDDM(R)(hRc(eGSZ)Vhbz^iN`K7?4P>s)N91)3A6PleMe2<&
z!(zv#%Uc?N`(Xmm)OJPYt)BM`nRjoWA&P0Yxl@c9Y02zlPH1J5l$nhPrMwu=atkz4
z)a-1+OEL;d@ctx=s<<+3Sv1VYy0RY<X+ux=2KtT`Vu|w~vi~*GQywuIz7Q_V!Hn=y
z)ZU6YkR>miji|#hy$66#`5;u~BkH4^$EGZ-Y4xyZ=%3KuaeLYKAUr$xMtIh_5mga>
zPz<#G0mQ7IxEw-yO}BueN}RaFlg$RwCDB)vLF$wDu%qZyLYsPKdcb<wALyt@*8W0U
zqSWfMa(_N(R=CrXFJc*SPB7&bo#W1MPzwLctt;6o8MA$uNlEm#qF^&@-Pig^Y~zDf
zQ$poL$<#~N=SdF3)gUAOAiS!ip^ss&mIhwdqMwR^;cN&W&Dre3&C9-tq_YijOMQwi
z$+oHul7HFXyNU-uuiEq1^63NOXgwnU@1O+oYJm64lf`6-Eh)=6E~{BWUTOawUj1M=
ztB-#zP;`DkLY0HB3vw>HD23$qn9i#JFqIo#OK?u7db2-$GatzO!On87%}Br};~#}n
zziVB;qf_4(K$u>Qyz$ln_kBGS!CD-t4Y}9oxL@7@Sx*?NOAzdeINUD>Hl#*V%pfA;
zSA`==YatS*G*crJ3`3ll4)vKss&)UtY#7ZxiVoG%9(4<%`WWcjX2jV(^g7Yhj+h5J
z$5=?S=tuCyEt74^6jo@6y|@~N>&cVfFNtaRl=)Gm!vR;Bc$3-;ySCI$%kdmjQ|si`
z{$q_YCe6vjy6re9jGN|`43D``)1PODtz0)vhV4XV36nVpOnMx2uM%qZ<3TtcI%>BQ
zf0(J`{JqPPJxw><pYvIVsSl6PJoMGZQho;>k#&nIvoZ5e9Sno)<M+aWI>B2r+E0G}
z@&M|zf4E0Q$O*NBR2<R=T|-dFe=pgLp}i{RB)8(%@x~v%=j6^BdSt6*N6>I;?i7N}
z@2^Su#`%qeX}m3cbSojiLk#84kvW1fICNPS`OyT0SpUoA0(s^2m~J<^eKE!dhJx_N
zG_T}0&(<*an>oF=@?6?55g&IxSgY3?7|@pmDRE6gJyJNPH6un~%0hZ@?h=hI6O$b^
z)29#<4$E)cE-5IFbRpk9JVrw$$966UDyw;Iym4OY4Fc!&s1ZH4BJ1-$9<)Zt1c)N-
zU^&9hsk6z?3%<9kGKHW|6~k;&cghtWz`oz`_YjVuvy;B;T67=L2c6=8`7WyTBv*QH
zNv*bo1#KOk{O&)@&pkd*?v+kcJ8tM>AGx$~WMhH{L40_N=bkrVg+^p!H)IqXCQf2_
z0fPig=8CEo>p4vE(nc^DKbZ|9_Xo}$i<nrsc7=?}L#k*l;@(0MupF7>4zJ`jVh95;
z5%aNP3@``=EJ=Vt9U`y+$YtX;%OPzgZ_3+;+mh{p#W&y4-%%Bf`LhOy-*kB0qnB^m
z_nBTz_b?-`F$*ymByshU>D)za2g`0j^ioo;A#QeL@x3@|+_!=YXA5f6Xg(Ack&WOg
zJ<2i|Fd6OmyH!@YSMVxb;=M)ZDhBt)4`5T*>cUXWPG#%@$&*<VQf|_QEj^;1*YoCf
zJ#NA#h99M>>K&u3#|`fm2mj*FKVf?du{xZ}WKWETTFhq6_fO$PS5(ItF=3~pFp~*j
z!ys1<4EL1)#{`mz@gW|t-FpPkd%pK)n_R<XGG4d(`l+MhcMBN~`esM~IYYy6_!rFx
zi{d8*%ew{lp})MvPrpY_KPlU6Z~JBl>b)F;z7cQ6dym_>YI3&e!=!m006oS3Mjq{q
ze%hNzW=G0jpfl2K(x`CDuZCsJV*hm9T~%5n<kIXgQNQAP!`L&M6@X9sbN#(*m#<@h
z%}-Wzi=wtGP^movMlGquoiO4?o3`i5{PXH>7R_g}VFpk`G((D^MWVMAmRp--T{`P;
zwMgD<;e`fm`g3|fPns|6qnd{|FCHY*YAguXH(?%sx%4+Gu|Y)_8mk4EljxmP+MP`*
z`SUbI{TCIN2OV+$y#g->Jqv#$wL;}4xJmah#$0`v^ughM_XjTA$B}ux)JZuY5-GW4
zKy440I+w=ZtE-_i+0xImq}vyzD68?8;94-5L~_O6Ty>X3itdA-x?6P(c4jkr+f!H(
zUDeqiG>3bn^Sf8(`_YwqPeJ9&-@OCQZm4X{FfRMeBtN4E9Ca@;GVpU*L>lVb;@=PH
zTQvTr?^jKyCKh&ZVOI*<<KC*~^3XVwVfca^38Eb{nv~0qIE}HZeQ1PH6@xX9j=JR&
zV9g(^6{BGm>y%<p+?m`qz>T*Aw(XCPr<m!ia5g4Y#nP}ajxm*wuXj;<P^VUy{hE2!
z{HfFfv$G5;{*d4Nyh9w2DD45oj0yXra&b;S<;yi7y4TTHH^RF`4tmVV2(-)2zT|Uc
zBp#RP``Xi6u<*MDkF@RXW4gC*!c<^Me7Y`|BW#!50N68ocGHaqPqwv@$&c)|C!ZhL
zZa1&Cd{-{UBact(QP7|N;&zbI?F6FmH3##E^#4Q2`A_-A_`j|D|3k^iRJU+LUBdRQ
z#E7AbiK6Q^`0*VXW3AS@rC&BcMh4#BkR%<g*cI-CStF2Fgn%kPuBToy-S(u@j%<UY
z>FC=39*y$A`FSzxBiQ#W+uW10d8&gYp4{teh;^p@anft+z$5!Hv&@h0X-@xJ<hC0V
z5NQ!<23a&gVCZX7Py!Q<C!5dfrk3jph84fTdJG>G>hbTCxjDwMiWK@1b%8wYL6BrV
zT41m}tX8g-`P@vj4T!Mlk8F0S!MA`^J=SCy9-jdwDe^hVDa`WwyI^H@ryt=F5y6>b
zT8&iI6&j8edAfX^ycgWbnMZQ26Q~`LmdEScKC8|~$Jgyw(>18NAQ$9AwCRmri!96L
zp^)b0P2CR-9S%cG$#rU}MXnx21T#031o>2VrDs@sa-FpjfvgLPW>Q&LHUoNOtmkt#
zoDZ=5OGp{^vO~=p29^`aXd8K?(+f-bW`N$U;-o;%f?RcR!k02Nod2h^^8ly%Z67#E
zC3|IOuj~^YBO=Fklo@3mvd6I<BO-f+$R;_6;)m=|cJxzrgpiUwGE4Y>{Z*&FZ>iq*
zxh|JuJoo2$p8MJ3zO@dQ;%1#~Mrm<Be@2cYu0c%^GGOhc(C21=on-T=Hcb7}5n>48
zB0053<snZ_Q)+q9Am)fE)VXwukarzr;x1>{1bDi_a@jo<4!@!`w4}B(&Qb`~IeSBh
zu+_yIYl2Wgk+?x4pCmAM>x_SqBPUj#c`C`k>_fp@qPlAAwD$!zOxRkL7;=|nu(#ut
zyF^;&hm-D_;ji{d6rOloACu5*NkF4IC3@rifMG(|^Skv$H&^YnYL*rpw=UCi;JOuz
zN*NX(7wZXS4tF@6PIWAs%*j!$RoL*3sh)}iry%thDvN5AUM888q_(>|Tzt|Yea<bs
zFUsz!G2e(z4SIrgldINq>3AyMYBgm$H_<iyV88GPvpUO1LF{eir>`F^v2%)bux)3s
znFIEBDK;-JS5SH|;1?afJb<*=c5puu=w<j}*^B{~H`bf2`bj2Jt_)m{0#y^mx0(Sx
zuAP%sBaS@P+JTZYuAqVen-Wz}1l0)ac}>%tv#ihn*R!^Hd$KWAp4$#`joJ*)$kNtZ
z2Al6h>Z>(u?3tmzA4^d+jLKx{97!Pb4;CX&u;M||**7zXI7hO6nrdMx*Xa=|-`#1^
zBQ?Ha&7cd7hN=%y4yUp?zl8~Lo;%mQrDe8!ce-W_K94FFMN*g(w8q-_K5S+c<WmVG
zEEJ_sK9zmSF@5;*&Y{LdAHgZbCWE5(MGvPND(SspeWbOl$2fe-X^VCFwU<RH@A$4F
zWD{_$Q`{myCbTh+F2OF=mmBNduFRWz!P!CH?nI)kzg9K%+ND>0<Wxop{UDcr4KWc{
zw4K5(a^$*)5|QJFh@ofNM0PwBD|*gtyuDIrgZdHfXaetIrMMgQ@RRhM*i&yqB&rsq
zD3F`z`*bm>{<BbM=$*(FdA(Gj<CG^PtKUjqsWj@?x%}j+moBaC^0S87XD-EW{!Pi{
zRcb3F^pu4_<|Y(ao3>o29X&PzpV;UJE^!xnFc%b@>kvW4m#xiOj-L*DadC&2N#0Us
z;<-(m1WB7$=j6hjcPC6JB)D3T2#IC`ibu#yi!uK7W2!j|Z>~RaJ*&XXy#ytIk2DIp
z5?Qd^s90_?ILjU#>ZWk5HXts}grg_!Gmgm!d?e<PC~eDJ`^;k<9pBwEtRZ90pBYRH
zr3Ef+CHDEOF133^Q}voowOX+u5LlN`IJH4CHOliZ?(b5lFH}t6qj2E8;t>LGR7xEP
zvTCrslV~94ym5_i<5oqy(@@?wN}lIdtiY8=?|Ng!XeYnly`@9wCGx2S$3x|0x8T2h
zz7A85Vb2>s44rKpI_4Y7_Pnd2^mYj2%^jM|Du>u4`^Psda^JIP%*DK6bo`Vf&f{!%
zDTYCwF5Nhi=)QhU2$@eQv&ZzxsX+Hl+gP6kW|e!n9IU2>Vh~cioI{>4WvR}t*4Hpz
z%5z?HjLGoka}Q3AbX9AkY|Yjf^M(>@tBAI9JO5pDCQu0R3Nns>)LC#vB2p96C*?K?
zvX$un$sBDx$1=+NNj*@Oa@u*b@O*XBr_sg@8sCUq-|LK!MUmC)epklrv}5O_^<{NP
zX16|c$9Wtbks3y7geI^tF5oRZJu;<AG~5!X7b-Hose24+z1b3X{`Nh1zT%@AoU%>v
zwkW8j+8Ccxo9stEDOT_Go&j%$KCgVO7pm+^%PKEPBZqbMw%s@732XS{cX+wCSjH1s
z<N2eX3UI%~%NcIGj+WJ1-MLJ%y-DgbEUi$rLe5CSFuv7!tR>5)bc=g**<^NNsroY`
z?}fHHlg<QL0w)w?&3&%!s29Is7#D;{ac0DC<z+8iMC9rd$M|XX4}upw{OS!0afhhy
z25U;3R8OPS$P|$sRuCBSNBRV1Y;9&!rl$(VF575rTy8APLiIuy->u^B?2r{^^gQ&j
zbF~T((>|Yg&C5WKL8DCnl1}Z3!YHFW2S1|;Xr0`Uz-;=FxEwYc4QpeAtnm7^f~uzX
zl;xA!?>MLR?tL80Iudm;mi{!ewL91KhG7Hsa-XepKi<2mc6%zf0GwtbfJ1Zf-<@Xu
z#|XWDzv|04t)&9Id!UxAAkN{t5qC%%8-WV3i;3duS19%m2||Y{!3pR1=g|zQYAMqc
zff)_2nj-O4wfxy;UNM?|Uieo!^J$A*uDe>@V(N<I!snfp?W5N{&W1iY`Z(Zv1XDHj
z-7DTH_m4n%W?oI+)+wQ>KH;KS;Y_dtE8${p>RdcrW;=2*fj4~d?OG0l-(g?ik}vz}
z)5-wDppVts>K-=|@{=!53?=8)Jw#<?>RGpS_FWpbwtn}{v!JEJ$q-sr7F6&OPBuI#
zuVNFM<b6fa@+xc7C?{)>Pte79XgEu!P&qRq8u4J>r%$l-IQ00Lin90(_KtC)aR_de
zxN=pY2<1b29_^AG2WJIGmmX4rv3$!`l15{e(H!1^+x9voZ6;882YAE12q7+lgy+>)
zj|s0CyzI9=Mo!R}&LXB`&DYpZ7c?0r(&KNV+~TULd0y^e;G{KVR4nL0KvU9mr8&$^
zxrM-9<BULp1ie>P8zE`J?aZ(iB~Rz<{vvnk2HaZU#K$aVFfYnbAXVUOLU#A<S+lSO
z$`#MSd^_uYn>s5JvS%+<hPj+ns1{FB7u^@{5W2OSA;(lkBGQzyEY0Ha8n~H5Sh>26
zi$sNuPY}dLGUS$0g&;oBqhzv2dY`l3@6Na403M!Sh${B|7(y|_cONa;6BrtUe@ZzV
z7SThtHT8k?Rwc)(Z}@BP#H@JJHz&GR&M=E@P9KJ89yQKmRh&I~%vbL1L-K3E>7>CH
z)Y!=jXVb1iPrAoAZZ3}3wU*5~nrV!ZjL5zqJ<@NwjHCZC>68Cc<{&E_#<Qr;)a#5U
z-N8nc#TSx%6ok}vq^}6b$S`sUhAFXGdJN6bT-e6mKD4yfjCU=BSBz3h%adGPrv9<7
zhGGH^$AzxLlC%{f_yLj)(U<<!JP#W$VM-E%bq7P!;7QpWxeRrcbhFE$nnX7l=vHZ#
zsWZIFP3)oOH0w;bQY>S;E*jOdjtg<E19=)ZmgJ~#+wPiPod(%4MWJtYgaNl#MF68#
ziob0|PDpDrQ<pz6`J5g5Ozn^m;PmeJA(q!m_WBWYI2?j>?uKN|l`P8sjz&Qf7a^z9
z;{3-8T+H4y99_zc;JYIvs!sk$G<SQ;H#Zk<G@5RNMQ_4+lLlR)NF^9fAPYY=`c7y=
z{B4l8bd1+ODEeDYwbGRq1?f7&J%sF$Xxuex2!Tg$VR(o&gqAvnLUvLYc0L1wR(>}`
z??mt*Mm9Z@glCZb!X?!xXD-21sFDPEpZOK{sbQseQ$%6~b;n+*z0hRoR}0Pe>B|#t
z$XrVcXv8M|q*Z8MY&r9J0A=d^1bHpjrUXu)qEj~$%%=gZp`^~%O*lzxUquG^p6;n;
z^(3HL+hx4gRP?4N*b2p9!^|2~rcw3!9nQj$vmZusbXYz_x^AVc`3qBFm(jS9ueU5h
z^AnNnbswfQ2Jq=W=T+p-V|nQco@<U0PTc0<;7^LTft>bOAH$pLQZ+BKH8E$iM>IDz
z3|wc?QP`yI=X5YTlp8h}%<Zy)pZp01CF`uRauFaMQI&q5+qHc-F3Cps<+{X=y{3(Q
zt#A~eZw^4;3cu4g*yr$fD;$t%)a+Q93qyhwt-JOrDe>p6{Deq?S0QD$Ug>i<VsT{Z
zNZf&Nn9Yp98S`@4qUbBQMy*q$-3E>h1SdPZg237Rl{S~=Ha4~-ckMoIWMn+X@@`V6
z#HHZj>MQbt$Qqp*9T(cjc^lxZ7UO(>PwzF-qEr(wo`vaulxdall|KP`7p4gd`2<O(
z1N8n;6?Wd&gsWVF75@Ydo$ft|Er-=z?zxs6zd+i?P?cAPVQw{Z$*(@N^lO#~C|q(&
z@$`AIK-3^SBuU&OMr__YtzFdNu>3&Jy=#sAes*0diLB(U$Nx46<srT4sk6o6P7=%O
zhw>VQvP)8idSs8^zaV91xw*O-JMH=)FoJshRob|_)O)ojtfP))WHCr(;*2;VMQ75^
zfN@a^f#o<|*9X;3IcGodLUz-3i~FAu+zI4c5h+nW^h_!^)b*B_xw-l4O$TB(ixaqW
ziMoa%i=BeS<-F45kMO;Tw|FWa`G<G3-xOKhWGvmJx|VXIPEX-V)szfTbyZZ|3z?)U
zc^%hH(1XG2nj5Naz~yvoz^R-hpmMmJmJFYgn!JD=!p+QH0Pr}?k8nVkTAK0y|1bF6
zoe&NVW={Km=KK1S1JaHkWoKKUQ|~l%hNO0dr#zWHmgZJznoEwWm3drKdVX1U6#L5#
zQWoapXYd5EER}9eKX%8Crr;J*BU)>2c!SuOA3CbowPhF6csf1|&qqugUrj;UgGHm|
z;j^yoH?MZhR;AYOW_XW2Lg2j%%ejL)B@*bUMD`g<#Z${1+fa57r7X<h|8nBv8r>82
zcqY-cfPnK%Y^3@s<qw39I3*A%a&Yr;y*!1LsA=td!6Lk^l}EQY)stIuf#mjU!CHAN
z%_7ndcshx}XztT;N<<N+cZw*^DxGG5)9IEDE8Q*-2_bA%d-<SA8Na9gb9H8>zRner
zt)bBToYCph6Jv*W+&t?&9FG4(Iu<!o(3LVe*X2KV9iK4%?pBR-1s<PB2~1fo>2w46
z4B#AcFy_^J@f*6<{>CN}Sj969*DYV*e7<61U>GoN{tz!Do90+jApFueVY_IW(MQF;
zl?4yA_(MvMwN&pWKVyg{3uU_+y6RMdot2vu%mC?st=N0pf-~JZXE?3JFf)j<{1xsU
z`2ephz)#HzsW<ft&J}W0BCk)zh`<=a+Z~$D&vD5)aiqR=aJA>EP!inHm2hI(V(~@W
zY7gGU-lO52cHD&SY)>QHgy$=>^X%u0TQZfCizro!*weMyvZC=;MWOawdAx~`3C*W`
z%^#^$uRP;gyqEE0<(i8xcQY$oc+6mY#z{-XFxsO1(cN8<M}(uexsybCZ<}?BPuLwP
z^{{b#pPOqjey#%Qd{&nv!!bLOaw*P)s{IHOW+JvNEi{WIa(FPm7xx-IfrDgNm#{js
zM)FIJsCh$*b(5CM>Y)>p;^q9|5bk`Z*p|c!?(rErw#y;yT(%@c7trQBv6cj)$3>pI
z>tz+;IB?D=aQV=s(n)o63*yn8dX1m7#Z4G{%fF@K2o5n3jxR~mU?nzMi#;}8e#(>{
zy{Z4!AI)jZ8TY;nq1aq}tq;~=zzoTv)er06oeX3;9{uP{LWR*2%9cmE%S^`~!BW>X
zn3PZFTf3g*dG68~^1*q@#^Ge(_8puPEFLD8OS|0b2a{5e=N4S%;~f3tC>F6UxK#v9
z)N-#Mv8=ePCh1KsUKD1A8jF<m)s1*e(Ok^!*@9Xtd%5^n@@B7YEsvFcvetT!kL);|
zvZ=Rt08Glh&xQ14c~@xfh?|gsRgH|<5tr$_5)L{4e+G*M4fJTEGsd9rqlF6u-?!i&
zz1&M^b0xcHLHgk+^DNBwfpX5x!jGLySKeYx>_%$MPf|_yCN9oy%*@um6D{w*2|4GY
zb}gafrSC+f=b*W{)!a!fqwZ9)K>fk=i4qf!4M?0v{CMNTo2A9}mQzV=%3UT&i{3{W
z>ulG#M!K7%jPf6Mjff9BMslgQq3zIogY);Cv3v;&b#;^=sh#(Bn%W)H*bHNaLwdpq
z85%fUTUJJNjYO_426T2TBj<?NSs;23Ek0pi`55pxf$Vc%b@&Y?;cmh?dMI_>0D{6t
zw&S_HZ|C?pI_2q(9Fas&@uJs6nVX;P*5K#6p|#)_(8PM-{L(;2wl`ma{ZAd5gA)?y
z>0GSLoK<*FwW+G8@-M3vcffg7I(qm7lzF)n`Q9iCv<vWIg-udr&$6Fj<FG)__)$*P
zQ^aMTptP8LKS(=EGfm@w4RpEKhbV9WC&9qo%Gydx9UxiX0^YYzBv|u%Z$U6!@P0Im
zyWvV(m(%xve^3U7XyCWs0@{xg`24NshRdia$;)YK@x$d00t21?m#_!kGu#Sv_P_-2
z*GUD;ntm69^-GNX;o-oDz@@kEBC7rvaetsWh)eS|Hr8HBy?z$`>p*mn7=|CjlpG{x
z&r0n<qKJPGH4ljTR{Fos4%ZKnX@ONLDDp1{-Gh>}XLWZ!>=lynUr7D`6n`7a_ZgT<
zm!i;&?Fb0Q2QmqmCHfZ7ex=_tU~(7b)L?RIvPyEAU=gLIZ-VTAA~WR00yKyTXg^(G
zqWLZJs!FnQYMOH3*fN&Tn(IKMLf{Ki?pRo8zZJ6YVyj)y0^)-sR}2-)%mI(Aw2AgT
zbbp1T{qB(OSNJd0cVBH^tI>HR(q+#*lmi@LWe*rZz&M2h1L_=50uZ1e*n#E*`6?aw
zj`ka&JpceRGe@}Ey1)Q~O}0qHRg4K_u>4e1arvJ7Q9!=t5AuzG`n=a-f0}{+ln<V+
z1M4)Dhm5O>CE#zu$`oVn44eS&T?N*wz~t~E&<X{ANWdYcze&LUZVFg>oQDBrB_MSg
z_yVrQehWbD0xHX|v-hpselAu;O7s;P*!uAT`dr~}Lie=tknaGoiU?;*8Cwgala-65
zosOB4mATbdXJFujzgA4?UkCKE093A1KM?W&Pw>A?IACqg1z~IZYkdP70EeCfjii(n
z3k%ax?4|rY(87N&_vhsyVK1zp@uils|B%`(V4e3%sj5f|i(eIhiSg-fHK1Pb0-mS^
zeh?WA7#{hhNci5e;?n*iVy|)iJiR>|8{TN3!=VBC2dN)~^ISSW_(g<^rHr$)nVrdA
z39BMa5w<WZpdDM8xMIAifHkR~C~eIOXmNBeGe5HQ{*?9eTxf6=^ilwlO##W6?s-p_
ziAD!V2Dem2z&#z1pPw^M;L{DS0I5a*rL{K;_!`iDSmp;X4zApXTQ1S<y;yz%G#{Wg
ze+y_oQHukFeo-s1`Zu?-a0N^#S|NX-U$7hVuLQJTzRm9eKRPuB>l5q+5F@)4b%5->
zA^-P20l_e^S2PTa&HE2wf3jf)#)2ITVXzndeuMpPo8}kphQKhegB%QO+yBpDpgkcl
z1nlPp14#+^bIA7__h16pMFECzKJ3p4`;Rf$gnr%{!5#oG42AH&X8hV8061%4W91ku
z`OW_hyI+uBOqYXkVC&BqoKWmv;|{O|4d#Nay<)gkxBr^^N48(VDF7Sj#H1i3>9138
zkhxAU7;M)I18&d!Yw!V9zQA0tp(G4<8U5GX{YoYCQ?p56FxcD-2FwO5fqyx@__=$L
zeK6Sg3>XQv)qz1?zW-k$_j`-)tf+yRU_%fXrenc>$^70d1Q-W?T#vy;6#Y-Q-<2)+
z5i<sy_VBO4zb#epTUHG1aQR=;4u07RE@AK^KMdNu>Tl6MA7j9m&oBhRXTKr*$3gec
z3E;zX457RGZwUvD$l&8e42Qb^cbq>zYy@ive8`2N9vk=#6+AQlZZ7qk=?(ap1q0n0
z{B9Fte-{Gi-Tvax1)M+d1}Fyg@9X~sh1m|hsDcZuYOnxriBPN;z)q3<=-yBN2iM6V
A?*IS*

literal 0
HcmV?d00001

diff --git a/qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.properties b/qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.properties
new file mode 100644
index 000000000000..346d645fd06f
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
diff --git a/qa/workunits/rgw/jcksum/README.md b/qa/workunits/rgw/jcksum/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/workunits/rgw/jcksum/file-1m b/qa/workunits/rgw/jcksum/file-1m
new file mode 100644
index 000000000000..084ad696ac55
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/file-1m
@@ -0,0 +1,30819 @@
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// C++Builder's IDE cannot build a static library from files with hyphens
+// in their name.  See http://qc.codegear.com/wc/qcmain.aspx?d=70977 .
+// This file serves as a workaround.
+
+#include "src/gtest-all.cc"
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// Links gtest.lib and gtest_main.lib into the current project in C++Builder.
+// This means that these libraries can't be renamed, but it's the only way to
+// ensure that Debug versus Release test builds are linked against the
+// appropriate Debug or Release build of the libraries.
+
+#pragma link "gtest.lib"
+#pragma link "gtest_main.lib"
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample2.h"
+
+#include <string.h>
+
+// Clones a 0-terminated C string, allocating memory using new.
+const char* MyString::CloneCString(const char* a_c_string) {
+  if (a_c_string == NULL) return NULL;
+
+  const size_t len = strlen(a_c_string);
+  char* const clone = new char[ len + 1 ];
+  memcpy(clone, a_c_string, len + 1);
+
+  return clone;
+}
+
+// Sets the 0-terminated C string this MyString object
+// represents.
+void MyString::Set(const char* a_c_string) {
+  // Makes sure this works when c_string == c_string_
+  const char* const temp = MyString::CloneCString(a_c_string);
+  delete[] c_string_;
+  c_string_ = temp;
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include <stdio.h>
+
+#include "sample4.h"
+
+// Returns the current counter value, and increments it.
+int Counter::Increment() {
+  return counter_++;
+}
+
+// Prints the current counter value to STDOUT.
+void Counter::Print() const {
+  printf("%d", counter_);
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample1.h"
+
+// Returns n! (the factorial of n).  For negative n, n! is defined to be 1.
+int Factorial(int n) {
+  int result = 1;
+  for (int i = 1; i <= n; i++) {
+    result *= i;
+  }
+
+  return result;
+}
+
+// Returns true iff n is a prime number.
+bool IsPrime(int n) {
+  // Trivial case 1: small numbers
+  if (n <= 1) return false;
+
+  // Trivial case 2: even numbers
+  if (n % 2 == 0) return n == 2;
+
+  // Now, we have that n is odd and n >= 3.
+
+  // Try to divide n by every odd number i, starting from 3
+  for (int i = 3; ; i += 2) {
+    // We only have to try i up to the squre root of n
+    if (i > n/i) break;
+
+    // Now, we have i <= n/i < n.
+    // If n is divisible by i, n is not prime.
+    if (n % i == 0) return false;
+  }
+
+  // n has no integer factor in the range (1, n), and thus is prime.
+  return true;
+}
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to use Google Test listener API to implement
+// a primitive leak checker.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+namespace {
+
+// We will track memory used by this class.
+class Water {
+ public:
+  // Normal Water declarations go here.
+
+  // operator new and operator delete help us control water allocation.
+  void* operator new(size_t allocation_size) {
+    allocated_++;
+    return malloc(allocation_size);
+  }
+
+  void operator delete(void* block, size_t /* allocation_size */) {
+    allocated_--;
+    free(block);
+  }
+
+  static int allocated() { return allocated_; }
+
+ private:
+  static int allocated_;
+};
+
+int Water::allocated_ = 0;
+
+// This event listener monitors how many Water objects are created and
+// destroyed by each test, and reports a failure if a test leaks some Water
+// objects. It does this by comparing the number of live Water objects at
+// the beginning of a test and at the end of a test.
+class LeakChecker : public EmptyTestEventListener {
+ private:
+  // Called before a test starts.
+  virtual void OnTestStart(const TestInfo& /* test_info */) {
+    initially_allocated_ = Water::allocated();
+  }
+
+  // Called after a test ends.
+  virtual void OnTestEnd(const TestInfo& /* test_info */) {
+    int difference = Water::allocated() - initially_allocated_;
+
+    // You can generate a failure in any event handler except
+    // OnTestPartResult. Just use an appropriate Google Test assertion to do
+    // it.
+    EXPECT_LE(difference, 0) << "Leaked " << difference << " unit(s) of Water!";
+  }
+
+  int initially_allocated_;
+};
+
+TEST(ListenersTest, DoesNotLeak) {
+  Water* water = new Water;
+  delete water;
+}
+
+// This should fail when the --check_for_leaks command line flag is
+// specified.
+TEST(ListenersTest, LeaksWater) {
+  Water* water = new Water;
+  EXPECT_TRUE(water != NULL);
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  bool check_for_leaks = false;
+  if (argc > 1 && strcmp(argv[1], "--check_for_leaks") == 0 )
+    check_for_leaks = true;
+  else
+    printf("%s\n", "Run this program with --check_for_leaks to enable "
+           "custom leak checking in the tests.");
+
+  // If we are given the --check_for_leaks command line flag, installs the
+  // leak checker.
+  if (check_for_leaks) {
+    TestEventListeners& listeners = UnitTest::GetInstance()->listeners();
+
+    // Adds the leak checker to the end of the test event listener list,
+    // after the default text output printer and the default XML report
+    // generator.
+    //
+    // The order is important - it ensures that failures generated in the
+    // leak checker's OnTestEnd() method are processed by the text and XML
+    // printers *before* their OnTestEnd() methods are called, such that
+    // they are attributed to the right test. Remember that a listener
+    // receives an OnXyzStart event *after* listeners preceding it in the
+    // list received that event, and receives an OnXyzEnd event *before*
+    // listeners preceding it.
+    //
+    // We don't need to worry about deleting the new listener later, as
+    // Google Test will do it.
+    listeners.Append(new LeakChecker);
+  }
+  return RUN_ALL_TESTS();
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// This sample shows how to write a simple unit test for a function,
+// using Google C++ testing framework.
+//
+// Writing a unit test using Google C++ testing framework is easy as 1-2-3:
+
+
+// Step 1. Include necessary header files such that the stuff your
+// test logic needs is declared.
+//
+// Don't forget gtest.h, which declares the testing framework.
+
+#include <limits.h>
+#include "sample1.h"
+#include "gtest/gtest.h"
+
+
+// Step 2. Use the TEST macro to define your tests.
+//
+// TEST has two parameters: the test case name and the test name.
+// After using the macro, you should define your test logic between a
+// pair of braces.  You can use a bunch of macros to indicate the
+// success or failure of a test.  EXPECT_TRUE and EXPECT_EQ are
+// examples of such macros.  For a complete list, see gtest.h.
+//
+// <TechnicalDetails>
+//
+// In Google Test, tests are grouped into test cases.  This is how we
+// keep test code organized.  You should put logically related tests
+// into the same test case.
+//
+// The test case name and the test name should both be valid C++
+// identifiers.  And you should not use underscore (_) in the names.
+//
+// Google Test guarantees that each test you define is run exactly
+// once, but it makes no guarantee on the order the tests are
+// executed.  Therefore, you should write your tests in such a way
+// that their results don't depend on their order.
+//
+// </TechnicalDetails>
+
+
+// Tests Factorial().
+
+// Tests factorial of negative numbers.
+TEST(FactorialTest, Negative) {
+  // This test is named "Negative", and belongs to the "FactorialTest"
+  // test case.
+  EXPECT_EQ(1, Factorial(-5));
+  EXPECT_EQ(1, Factorial(-1));
+  EXPECT_GT(Factorial(-10), 0);
+
+  // <TechnicalDetails>
+  //
+  // EXPECT_EQ(expected, actual) is the same as
+  //
+  //   EXPECT_TRUE((expected) == (actual))
+  //
+  // except that it will print both the expected value and the actual
+  // value when the assertion fails.  This is very helpful for
+  // debugging.  Therefore in this case EXPECT_EQ is preferred.
+  //
+  // On the other hand, EXPECT_TRUE accepts any Boolean expression,
+  // and is thus more general.
+  //
+  // </TechnicalDetails>
+}
+
+// Tests factorial of 0.
+TEST(FactorialTest, Zero) {
+  EXPECT_EQ(1, Factorial(0));
+}
+
+// Tests factorial of positive numbers.
+TEST(FactorialTest, Positive) {
+  EXPECT_EQ(1, Factorial(1));
+  EXPECT_EQ(2, Factorial(2));
+  EXPECT_EQ(6, Factorial(3));
+  EXPECT_EQ(40320, Factorial(8));
+}
+
+
+// Tests IsPrime()
+
+// Tests negative input.
+TEST(IsPrimeTest, Negative) {
+  // This test belongs to the IsPrimeTest test case.
+
+  EXPECT_FALSE(IsPrime(-1));
+  EXPECT_FALSE(IsPrime(-2));
+  EXPECT_FALSE(IsPrime(INT_MIN));
+}
+
+// Tests some trivial cases.
+TEST(IsPrimeTest, Trivial) {
+  EXPECT_FALSE(IsPrime(0));
+  EXPECT_FALSE(IsPrime(1));
+  EXPECT_TRUE(IsPrime(2));
+  EXPECT_TRUE(IsPrime(3));
+}
+
+// Tests positive input.
+TEST(IsPrimeTest, Positive) {
+  EXPECT_FALSE(IsPrime(4));
+  EXPECT_TRUE(IsPrime(5));
+  EXPECT_FALSE(IsPrime(6));
+  EXPECT_TRUE(IsPrime(23));
+}
+
+// Step 3. Call RUN_ALL_TESTS() in main().
+//
+// We do this by linking in src/gtest_main.cc file, which consists of
+// a main() function which calls RUN_ALL_TESTS() for us.
+//
+// This runs all the tests you've defined, prints the result, and
+// returns 0 if successful, or 1 otherwise.
+//
+// Did you notice that we didn't register the tests?  The
+// RUN_ALL_TESTS() macro magically knows about all the tests we
+// defined.  Isn't this convenient?
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// This sample shows how to write a more complex unit test for a class
+// that has multiple member functions.
+//
+// Usually, it's a good idea to have one test for each method in your
+// class.  You don't have to do that exactly, but it helps to keep
+// your tests organized.  You may also throw in additional tests as
+// needed.
+
+#include "sample2.h"
+#include "gtest/gtest.h"
+
+// In this example, we test the MyString class (a simple string).
+
+// Tests the default c'tor.
+TEST(MyString, DefaultConstructor) {
+  const MyString s;
+
+  // Asserts that s.c_string() returns NULL.
+  //
+  // <TechnicalDetails>
+  //
+  // If we write NULL instead of
+  //
+  //   static_cast<const char *>(NULL)
+  //
+  // in this assertion, it will generate a warning on gcc 3.4.  The
+  // reason is that EXPECT_EQ needs to know the types of its
+  // arguments in order to print them when it fails.  Since NULL is
+  // #defined as 0, the compiler will use the formatter function for
+  // int to print it.  However, gcc thinks that NULL should be used as
+  // a pointer, not an int, and therefore complains.
+  //
+  // The root of the problem is C++'s lack of distinction between the
+  // integer number 0 and the null pointer constant.  Unfortunately,
+  // we have to live with this fact.
+  //
+  // </TechnicalDetails>
+  EXPECT_STREQ(NULL, s.c_string());
+
+  EXPECT_EQ(0u, s.Length());
+}
+
+const char kHelloString[] = "Hello, world!";
+
+// Tests the c'tor that accepts a C string.
+TEST(MyString, ConstructorFromCString) {
+  const MyString s(kHelloString);
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+  EXPECT_EQ(sizeof(kHelloString)/sizeof(kHelloString[0]) - 1,
+            s.Length());
+}
+
+// Tests the copy c'tor.
+TEST(MyString, CopyConstructor) {
+  const MyString s1(kHelloString);
+  const MyString s2 = s1;
+  EXPECT_EQ(0, strcmp(s2.c_string(), kHelloString));
+}
+
+// Tests the Set method.
+TEST(MyString, Set) {
+  MyString s;
+
+  s.Set(kHelloString);
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+
+  // Set should work when the input pointer is the same as the one
+  // already in the MyString object.
+  s.Set(s.c_string());
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+
+  // Can we set the MyString to NULL?
+  s.Set(NULL);
+  EXPECT_STREQ(NULL, s.c_string());
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// In this example, we use a more advanced feature of Google Test called
+// test fixture.
+//
+// A test fixture is a place to hold objects and functions shared by
+// all tests in a test case.  Using a test fixture avoids duplicating
+// the test code necessary to initialize and cleanup those common
+// objects for each test.  It is also useful for defining sub-routines
+// that your tests need to invoke a lot.
+//
+// <TechnicalDetails>
+//
+// The tests share the test fixture in the sense of code sharing, not
+// data sharing.  Each test is given its own fresh copy of the
+// fixture.  You cannot expect the data modified by one test to be
+// passed on to another test, which is a bad idea.
+//
+// The reason for this design is that tests should be independent and
+// repeatable.  In particular, a test should not fail as the result of
+// another test's failure.  If one test depends on info produced by
+// another test, then the two tests should really be one big test.
+//
+// The macros for indicating the success/failure of a test
+// (EXPECT_TRUE, FAIL, etc) need to know what the current test is
+// (when Google Test prints the test result, it tells you which test
+// each failure belongs to).  Technically, these macros invoke a
+// member function of the Test class.  Therefore, you cannot use them
+// in a global function.  That's why you should put test sub-routines
+// in a test fixture.
+//
+// </TechnicalDetails>
+
+#include "sample3-inl.h"
+#include "gtest/gtest.h"
+
+// To use a test fixture, derive a class from testing::Test.
+class QueueTest : public testing::Test {
+ protected:  // You should make the members protected s.t. they can be
+             // accessed from sub-classes.
+
+  // virtual void SetUp() will be called before each test is run.  You
+  // should define it if you need to initialize the varaibles.
+  // Otherwise, this can be skipped.
+  virtual void SetUp() {
+    q1_.Enqueue(1);
+    q2_.Enqueue(2);
+    q2_.Enqueue(3);
+  }
+
+  // virtual void TearDown() will be called after each test is run.
+  // You should define it if there is cleanup work to do.  Otherwise,
+  // you don't have to provide it.
+  //
+  // virtual void TearDown() {
+  // }
+
+  // A helper function that some test uses.
+  static int Double(int n) {
+    return 2*n;
+  }
+
+  // A helper function for testing Queue::Map().
+  void MapTester(const Queue<int> * q) {
+    // Creates a new queue, where each element is twice as big as the
+    // corresponding one in q.
+    const Queue<int> * const new_q = q->Map(Double);
+
+    // Verifies that the new queue has the same size as q.
+    ASSERT_EQ(q->Size(), new_q->Size());
+
+    // Verifies the relationship between the elements of the two queues.
+    for ( const QueueNode<int> * n1 = q->Head(), * n2 = new_q->Head();
+          n1 != NULL; n1 = n1->next(), n2 = n2->next() ) {
+      EXPECT_EQ(2 * n1->element(), n2->element());
+    }
+
+    delete new_q;
+  }
+
+  // Declares the variables your tests want to use.
+  Queue<int> q0_;
+  Queue<int> q1_;
+  Queue<int> q2_;
+};
+
+// When you have a test fixture, you define a test using TEST_F
+// instead of TEST.
+
+// Tests the default c'tor.
+TEST_F(QueueTest, DefaultConstructor) {
+  // You can access data in the test fixture here.
+  EXPECT_EQ(0u, q0_.Size());
+}
+
+// Tests Dequeue().
+TEST_F(QueueTest, Dequeue) {
+  int * n = q0_.Dequeue();
+  EXPECT_TRUE(n == NULL);
+
+  n = q1_.Dequeue();
+  ASSERT_TRUE(n != NULL);
+  EXPECT_EQ(1, *n);
+  EXPECT_EQ(0u, q1_.Size());
+  delete n;
+
+  n = q2_.Dequeue();
+  ASSERT_TRUE(n != NULL);
+  EXPECT_EQ(2, *n);
+  EXPECT_EQ(1u, q2_.Size());
+  delete n;
+}
+
+// Tests the Queue::Map() function.
+TEST_F(QueueTest, Map) {
+  MapTester(&q0_);
+  MapTester(&q1_);
+  MapTester(&q2_);
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest.h"
+#include "sample4.h"
+
+// Tests the Increment() method.
+TEST(Counter, Increment) {
+  Counter c;
+
+  // EXPECT_EQ() evaluates its arguments exactly once, so they
+  // can have side effects.
+
+  EXPECT_EQ(0, c.Increment());
+  EXPECT_EQ(1, c.Increment());
+  EXPECT_EQ(2, c.Increment());
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// This sample teaches how to reuse a test fixture in multiple test
+// cases by deriving sub-fixtures from it.
+//
+// When you define a test fixture, you specify the name of the test
+// case that will use this fixture.  Therefore, a test fixture can
+// be used by only one test case.
+//
+// Sometimes, more than one test cases may want to use the same or
+// slightly different test fixtures.  For example, you may want to
+// make sure that all tests for a GUI library don't leak important
+// system resources like fonts and brushes.  In Google Test, you do
+// this by putting the shared logic in a super (as in "super class")
+// test fixture, and then have each test case use a fixture derived
+// from this super fixture.
+
+#include <limits.h>
+#include <time.h>
+#include "sample3-inl.h"
+#include "gtest/gtest.h"
+#include "sample1.h"
+
+// In this sample, we want to ensure that every test finishes within
+// ~5 seconds.  If a test takes longer to run, we consider it a
+// failure.
+//
+// We put the code for timing a test in a test fixture called
+// "QuickTest".  QuickTest is intended to be the super fixture that
+// other fixtures derive from, therefore there is no test case with
+// the name "QuickTest".  This is OK.
+//
+// Later, we will derive multiple test fixtures from QuickTest.
+class QuickTest : public testing::Test {
+ protected:
+  // Remember that SetUp() is run immediately before a test starts.
+  // This is a good place to record the start time.
+  virtual void SetUp() {
+    start_time_ = time(NULL);
+  }
+
+  // TearDown() is invoked immediately after a test finishes.  Here we
+  // check if the test was too slow.
+  virtual void TearDown() {
+    // Gets the time when the test finishes
+    const time_t end_time = time(NULL);
+
+    // Asserts that the test took no more than ~5 seconds.  Did you
+    // know that you can use assertions in SetUp() and TearDown() as
+    // well?
+    EXPECT_TRUE(end_time - start_time_ <= 5) << "The test took too long.";
+  }
+
+  // The UTC time (in seconds) when the test starts
+  time_t start_time_;
+};
+
+
+// We derive a fixture named IntegerFunctionTest from the QuickTest
+// fixture.  All tests using this fixture will be automatically
+// required to be quick.
+class IntegerFunctionTest : public QuickTest {
+  // We don't need any more logic than already in the QuickTest fixture.
+  // Therefore the body is empty.
+};
+
+
+// Now we can write tests in the IntegerFunctionTest test case.
+
+// Tests Factorial()
+TEST_F(IntegerFunctionTest, Factorial) {
+  // Tests factorial of negative numbers.
+  EXPECT_EQ(1, Factorial(-5));
+  EXPECT_EQ(1, Factorial(-1));
+  EXPECT_GT(Factorial(-10), 0);
+
+  // Tests factorial of 0.
+  EXPECT_EQ(1, Factorial(0));
+
+  // Tests factorial of positive numbers.
+  EXPECT_EQ(1, Factorial(1));
+  EXPECT_EQ(2, Factorial(2));
+  EXPECT_EQ(6, Factorial(3));
+  EXPECT_EQ(40320, Factorial(8));
+}
+
+
+// Tests IsPrime()
+TEST_F(IntegerFunctionTest, IsPrime) {
+  // Tests negative input.
+  EXPECT_FALSE(IsPrime(-1));
+  EXPECT_FALSE(IsPrime(-2));
+  EXPECT_FALSE(IsPrime(INT_MIN));
+
+  // Tests some trivial cases.
+  EXPECT_FALSE(IsPrime(0));
+  EXPECT_FALSE(IsPrime(1));
+  EXPECT_TRUE(IsPrime(2));
+  EXPECT_TRUE(IsPrime(3));
+
+  // Tests positive input.
+  EXPECT_FALSE(IsPrime(4));
+  EXPECT_TRUE(IsPrime(5));
+  EXPECT_FALSE(IsPrime(6));
+  EXPECT_TRUE(IsPrime(23));
+}
+
+
+// The next test case (named "QueueTest") also needs to be quick, so
+// we derive another fixture from QuickTest.
+//
+// The QueueTest test fixture has some logic and shared objects in
+// addition to what's in QuickTest already.  We define the additional
+// stuff inside the body of the test fixture, as usual.
+class QueueTest : public QuickTest {
+ protected:
+  virtual void SetUp() {
+    // First, we need to set up the super fixture (QuickTest).
+    QuickTest::SetUp();
+
+    // Second, some additional setup for this fixture.
+    q1_.Enqueue(1);
+    q2_.Enqueue(2);
+    q2_.Enqueue(3);
+  }
+
+  // By default, TearDown() inherits the behavior of
+  // QuickTest::TearDown().  As we have no additional cleaning work
+  // for QueueTest, we omit it here.
+  //
+  // virtual void TearDown() {
+  //   QuickTest::TearDown();
+  // }
+
+  Queue<int> q0_;
+  Queue<int> q1_;
+  Queue<int> q2_;
+};
+
+
+// Now, let's write tests using the QueueTest fixture.
+
+// Tests the default constructor.
+TEST_F(QueueTest, DefaultConstructor) {
+  EXPECT_EQ(0u, q0_.Size());
+}
+
+// Tests Dequeue().
+TEST_F(QueueTest, Dequeue) {
+  int* n = q0_.Dequeue();
+  EXPECT_TRUE(n == NULL);
+
+  n = q1_.Dequeue();
+  EXPECT_TRUE(n != NULL);
+  EXPECT_EQ(1, *n);
+  EXPECT_EQ(0u, q1_.Size());
+  delete n;
+
+  n = q2_.Dequeue();
+  EXPECT_TRUE(n != NULL);
+  EXPECT_EQ(2, *n);
+  EXPECT_EQ(1u, q2_.Size());
+  delete n;
+}
+
+// If necessary, you can derive further test fixtures from a derived
+// fixture itself.  For example, you can derive another fixture from
+// QueueTest.  Google Test imposes no limit on how deep the hierarchy
+// can be.  In practice, however, you probably don't want it to be too
+// deep as to be confusing.
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// This sample shows how to test common properties of multiple
+// implementations of the same interface (aka interface tests).
+
+// The interface and its implementations are in this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+// First, we define some factory functions for creating instances of
+// the implementations.  You may be able to skip this step if all your
+// implementations can be constructed the same way.
+
+template <class T>
+PrimeTable* CreatePrimeTable();
+
+template <>
+PrimeTable* CreatePrimeTable<OnTheFlyPrimeTable>() {
+  return new OnTheFlyPrimeTable;
+}
+
+template <>
+PrimeTable* CreatePrimeTable<PreCalculatedPrimeTable>() {
+  return new PreCalculatedPrimeTable(10000);
+}
+
+// Then we define a test fixture class template.
+template <class T>
+class PrimeTableTest : public testing::Test {
+ protected:
+  // The ctor calls the factory function to create a prime table
+  // implemented by T.
+  PrimeTableTest() : table_(CreatePrimeTable<T>()) {}
+
+  virtual ~PrimeTableTest() { delete table_; }
+
+  // Note that we test an implementation via the base interface
+  // instead of the actual implementation class.  This is important
+  // for keeping the tests close to the real world scenario, where the
+  // implementation is invoked via the base interface.  It avoids
+  // got-yas where the implementation class has a method that shadows
+  // a method with the same name (but slightly different argument
+  // types) in the base interface, for example.
+  PrimeTable* const table_;
+};
+
+#if GTEST_HAS_TYPED_TEST
+
+using testing::Types;
+
+// Google Test offers two ways for reusing tests for different types.
+// The first is called "typed tests".  You should use it if you
+// already know *all* the types you are gonna exercise when you write
+// the tests.
+
+// To write a typed test case, first use
+//
+//   TYPED_TEST_CASE(TestCaseName, TypeList);
+//
+// to declare it and specify the type parameters.  As with TEST_F,
+// TestCaseName must match the test fixture name.
+
+// The list of types we want to test.
+typedef Types<OnTheFlyPrimeTable, PreCalculatedPrimeTable> Implementations;
+
+TYPED_TEST_CASE(PrimeTableTest, Implementations);
+
+// Then use TYPED_TEST(TestCaseName, TestName) to define a typed test,
+// similar to TEST_F.
+TYPED_TEST(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  // Inside the test body, you can refer to the type parameter by
+  // TypeParam, and refer to the fixture class by TestFixture.  We
+  // don't need them in this example.
+
+  // Since we are in the template world, C++ requires explicitly
+  // writing 'this->' when referring to members of the fixture class.
+  // This is something you have to learn to live with.
+  EXPECT_FALSE(this->table_->IsPrime(-5));
+  EXPECT_FALSE(this->table_->IsPrime(0));
+  EXPECT_FALSE(this->table_->IsPrime(1));
+  EXPECT_FALSE(this->table_->IsPrime(4));
+  EXPECT_FALSE(this->table_->IsPrime(6));
+  EXPECT_FALSE(this->table_->IsPrime(100));
+}
+
+TYPED_TEST(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(this->table_->IsPrime(2));
+  EXPECT_TRUE(this->table_->IsPrime(3));
+  EXPECT_TRUE(this->table_->IsPrime(5));
+  EXPECT_TRUE(this->table_->IsPrime(7));
+  EXPECT_TRUE(this->table_->IsPrime(11));
+  EXPECT_TRUE(this->table_->IsPrime(131));
+}
+
+TYPED_TEST(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, this->table_->GetNextPrime(0));
+  EXPECT_EQ(3, this->table_->GetNextPrime(2));
+  EXPECT_EQ(5, this->table_->GetNextPrime(3));
+  EXPECT_EQ(7, this->table_->GetNextPrime(5));
+  EXPECT_EQ(11, this->table_->GetNextPrime(7));
+  EXPECT_EQ(131, this->table_->GetNextPrime(128));
+}
+
+// That's it!  Google Test will repeat each TYPED_TEST for each type
+// in the type list specified in TYPED_TEST_CASE.  Sit back and be
+// happy that you don't have to define them multiple times.
+
+#endif  // GTEST_HAS_TYPED_TEST
+
+#if GTEST_HAS_TYPED_TEST_P
+
+using testing::Types;
+
+// Sometimes, however, you don't yet know all the types that you want
+// to test when you write the tests.  For example, if you are the
+// author of an interface and expect other people to implement it, you
+// might want to write a set of tests to make sure each implementation
+// conforms to some basic requirements, but you don't know what
+// implementations will be written in the future.
+//
+// How can you write the tests without committing to the type
+// parameters?  That's what "type-parameterized tests" can do for you.
+// It is a bit more involved than typed tests, but in return you get a
+// test pattern that can be reused in many contexts, which is a big
+// win.  Here's how you do it:
+
+// First, define a test fixture class template.  Here we just reuse
+// the PrimeTableTest fixture defined earlier:
+
+template <class T>
+class PrimeTableTest2 : public PrimeTableTest<T> {
+};
+
+// Then, declare the test case.  The argument is the name of the test
+// fixture, and also the name of the test case (as usual).  The _P
+// suffix is for "parameterized" or "pattern".
+TYPED_TEST_CASE_P(PrimeTableTest2);
+
+// Next, use TYPED_TEST_P(TestCaseName, TestName) to define a test,
+// similar to what you do with TEST_F.
+TYPED_TEST_P(PrimeTableTest2, ReturnsFalseForNonPrimes) {
+  EXPECT_FALSE(this->table_->IsPrime(-5));
+  EXPECT_FALSE(this->table_->IsPrime(0));
+  EXPECT_FALSE(this->table_->IsPrime(1));
+  EXPECT_FALSE(this->table_->IsPrime(4));
+  EXPECT_FALSE(this->table_->IsPrime(6));
+  EXPECT_FALSE(this->table_->IsPrime(100));
+}
+
+TYPED_TEST_P(PrimeTableTest2, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(this->table_->IsPrime(2));
+  EXPECT_TRUE(this->table_->IsPrime(3));
+  EXPECT_TRUE(this->table_->IsPrime(5));
+  EXPECT_TRUE(this->table_->IsPrime(7));
+  EXPECT_TRUE(this->table_->IsPrime(11));
+  EXPECT_TRUE(this->table_->IsPrime(131));
+}
+
+TYPED_TEST_P(PrimeTableTest2, CanGetNextPrime) {
+  EXPECT_EQ(2, this->table_->GetNextPrime(0));
+  EXPECT_EQ(3, this->table_->GetNextPrime(2));
+  EXPECT_EQ(5, this->table_->GetNextPrime(3));
+  EXPECT_EQ(7, this->table_->GetNextPrime(5));
+  EXPECT_EQ(11, this->table_->GetNextPrime(7));
+  EXPECT_EQ(131, this->table_->GetNextPrime(128));
+}
+
+// Type-parameterized tests involve one extra step: you have to
+// enumerate the tests you defined:
+REGISTER_TYPED_TEST_CASE_P(
+    PrimeTableTest2,  // The first argument is the test case name.
+    // The rest of the arguments are the test names.
+    ReturnsFalseForNonPrimes, ReturnsTrueForPrimes, CanGetNextPrime);
+
+// At this point the test pattern is done.  However, you don't have
+// any real test yet as you haven't said which types you want to run
+// the tests with.
+
+// To turn the abstract test pattern into real tests, you instantiate
+// it with a list of types.  Usually the test pattern will be defined
+// in a .h file, and anyone can #include and instantiate it.  You can
+// even instantiate it more than once in the same program.  To tell
+// different instances apart, you give each of them a name, which will
+// become part of the test case name and can be used in test filters.
+
+// The list of types we want to test.  Note that it doesn't have to be
+// defined at the time we write the TYPED_TEST_P()s.
+typedef Types<OnTheFlyPrimeTable, PreCalculatedPrimeTable>
+    PrimeTableImplementations;
+INSTANTIATE_TYPED_TEST_CASE_P(OnTheFlyAndPreCalculated,    // Instance name
+                              PrimeTableTest2,             // Test case name
+                              PrimeTableImplementations);  // Type list
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to test common properties of multiple
+// implementations of an interface (aka interface tests) using
+// value-parameterized tests. Each test in the test case has
+// a parameter that is an interface pointer to an implementation
+// tested.
+
+// The interface and its implementations are in this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_PARAM_TEST
+
+using ::testing::TestWithParam;
+using ::testing::Values;
+
+// As a general rule, to prevent a test from affecting the tests that come
+// after it, you should create and destroy the tested objects for each test
+// instead of reusing them.  In this sample we will define a simple factory
+// function for PrimeTable objects.  We will instantiate objects in test's
+// SetUp() method and delete them in TearDown() method.
+typedef PrimeTable* CreatePrimeTableFunc();
+
+PrimeTable* CreateOnTheFlyPrimeTable() {
+  return new OnTheFlyPrimeTable();
+}
+
+template <size_t max_precalculated>
+PrimeTable* CreatePreCalculatedPrimeTable() {
+  return new PreCalculatedPrimeTable(max_precalculated);
+}
+
+// Inside the test body, fixture constructor, SetUp(), and TearDown() you
+// can refer to the test parameter by GetParam().  In this case, the test
+// parameter is a factory function which we call in fixture's SetUp() to
+// create and store an instance of PrimeTable.
+class PrimeTableTest : public TestWithParam<CreatePrimeTableFunc*> {
+ public:
+  virtual ~PrimeTableTest() { delete table_; }
+  virtual void SetUp() { table_ = (*GetParam())(); }
+  virtual void TearDown() {
+    delete table_;
+    table_ = NULL;
+  }
+
+ protected:
+  PrimeTable* table_;
+};
+
+TEST_P(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  EXPECT_FALSE(table_->IsPrime(-5));
+  EXPECT_FALSE(table_->IsPrime(0));
+  EXPECT_FALSE(table_->IsPrime(1));
+  EXPECT_FALSE(table_->IsPrime(4));
+  EXPECT_FALSE(table_->IsPrime(6));
+  EXPECT_FALSE(table_->IsPrime(100));
+}
+
+TEST_P(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(table_->IsPrime(2));
+  EXPECT_TRUE(table_->IsPrime(3));
+  EXPECT_TRUE(table_->IsPrime(5));
+  EXPECT_TRUE(table_->IsPrime(7));
+  EXPECT_TRUE(table_->IsPrime(11));
+  EXPECT_TRUE(table_->IsPrime(131));
+}
+
+TEST_P(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, table_->GetNextPrime(0));
+  EXPECT_EQ(3, table_->GetNextPrime(2));
+  EXPECT_EQ(5, table_->GetNextPrime(3));
+  EXPECT_EQ(7, table_->GetNextPrime(5));
+  EXPECT_EQ(11, table_->GetNextPrime(7));
+  EXPECT_EQ(131, table_->GetNextPrime(128));
+}
+
+// In order to run value-parameterized tests, you need to instantiate them,
+// or bind them to a list of values which will be used as test parameters.
+// You can instantiate them in a different translation module, or even
+// instantiate them several times.
+//
+// Here, we instantiate our tests with a list of two PrimeTable object
+// factory functions:
+INSTANTIATE_TEST_CASE_P(
+    OnTheFlyAndPreCalculated,
+    PrimeTableTest,
+    Values(&CreateOnTheFlyPrimeTable, &CreatePreCalculatedPrimeTable<1000>));
+
+#else
+
+// Google Test may not support value-parameterized tests with some
+// compilers. If we use conditional compilation to compile out all
+// code referring to the gtest_main library, MSVC linker will not link
+// that library at all and consequently complain about missing entry
+// point defined in that library (fatal error LNK1561: entry point
+// must be defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
+
+#endif  // GTEST_HAS_PARAM_TEST
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to test code relying on some global flag variables.
+// Combine() helps with generating all possible combinations of such flags,
+// and each test is given one combination as a parameter.
+
+// Use class definitions to test from this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_COMBINE
+
+// Suppose we want to introduce a new, improved implementation of PrimeTable
+// which combines speed of PrecalcPrimeTable and versatility of
+// OnTheFlyPrimeTable (see prime_tables.h). Inside it instantiates both
+// PrecalcPrimeTable and OnTheFlyPrimeTable and uses the one that is more
+// appropriate under the circumstances. But in low memory conditions, it can be
+// told to instantiate without PrecalcPrimeTable instance at all and use only
+// OnTheFlyPrimeTable.
+class HybridPrimeTable : public PrimeTable {
+ public:
+  HybridPrimeTable(bool force_on_the_fly, int max_precalculated)
+      : on_the_fly_impl_(new OnTheFlyPrimeTable),
+        precalc_impl_(force_on_the_fly ? NULL :
+                          new PreCalculatedPrimeTable(max_precalculated)),
+        max_precalculated_(max_precalculated) {}
+  virtual ~HybridPrimeTable() {
+    delete on_the_fly_impl_;
+    delete precalc_impl_;
+  }
+
+  virtual bool IsPrime(int n) const {
+    if (precalc_impl_ != NULL && n < max_precalculated_)
+      return precalc_impl_->IsPrime(n);
+    else
+      return on_the_fly_impl_->IsPrime(n);
+  }
+
+  virtual int GetNextPrime(int p) const {
+    int next_prime = -1;
+    if (precalc_impl_ != NULL && p < max_precalculated_)
+      next_prime = precalc_impl_->GetNextPrime(p);
+
+    return next_prime != -1 ? next_prime : on_the_fly_impl_->GetNextPrime(p);
+  }
+
+ private:
+  OnTheFlyPrimeTable* on_the_fly_impl_;
+  PreCalculatedPrimeTable* precalc_impl_;
+  int max_precalculated_;
+};
+
+using ::testing::TestWithParam;
+using ::testing::Bool;
+using ::testing::Values;
+using ::testing::Combine;
+
+// To test all code paths for HybridPrimeTable we must test it with numbers
+// both within and outside PreCalculatedPrimeTable's capacity and also with
+// PreCalculatedPrimeTable disabled. We do this by defining fixture which will
+// accept different combinations of parameters for instantiating a
+// HybridPrimeTable instance.
+class PrimeTableTest : public TestWithParam< ::testing::tuple<bool, int> > {
+ protected:
+  virtual void SetUp() {
+    // This can be written as
+    //
+    // bool force_on_the_fly;
+    // int max_precalculated;
+    // tie(force_on_the_fly, max_precalculated) = GetParam();
+    //
+    // once the Google C++ Style Guide allows use of ::std::tr1::tie.
+    //
+    bool force_on_the_fly = ::testing::get<0>(GetParam());
+    int max_precalculated = ::testing::get<1>(GetParam());
+    table_ = new HybridPrimeTable(force_on_the_fly, max_precalculated);
+  }
+  virtual void TearDown() {
+    delete table_;
+    table_ = NULL;
+  }
+  HybridPrimeTable* table_;
+};
+
+TEST_P(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  // Inside the test body, you can refer to the test parameter by GetParam().
+  // In this case, the test parameter is a PrimeTable interface pointer which
+  // we can use directly.
+  // Please note that you can also save it in the fixture's SetUp() method
+  // or constructor and use saved copy in the tests.
+
+  EXPECT_FALSE(table_->IsPrime(-5));
+  EXPECT_FALSE(table_->IsPrime(0));
+  EXPECT_FALSE(table_->IsPrime(1));
+  EXPECT_FALSE(table_->IsPrime(4));
+  EXPECT_FALSE(table_->IsPrime(6));
+  EXPECT_FALSE(table_->IsPrime(100));
+}
+
+TEST_P(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(table_->IsPrime(2));
+  EXPECT_TRUE(table_->IsPrime(3));
+  EXPECT_TRUE(table_->IsPrime(5));
+  EXPECT_TRUE(table_->IsPrime(7));
+  EXPECT_TRUE(table_->IsPrime(11));
+  EXPECT_TRUE(table_->IsPrime(131));
+}
+
+TEST_P(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, table_->GetNextPrime(0));
+  EXPECT_EQ(3, table_->GetNextPrime(2));
+  EXPECT_EQ(5, table_->GetNextPrime(3));
+  EXPECT_EQ(7, table_->GetNextPrime(5));
+  EXPECT_EQ(11, table_->GetNextPrime(7));
+  EXPECT_EQ(131, table_->GetNextPrime(128));
+}
+
+// In order to run value-parameterized tests, you need to instantiate them,
+// or bind them to a list of values which will be used as test parameters.
+// You can instantiate them in a different translation module, or even
+// instantiate them several times.
+//
+// Here, we instantiate our tests with a list of parameters. We must combine
+// all variations of the boolean flag suppressing PrecalcPrimeTable and some
+// meaningful values for tests. We choose a small value (1), and a value that
+// will put some of the tested numbers beyond the capability of the
+// PrecalcPrimeTable instance and some inside it (10). Combine will produce all
+// possible combinations.
+INSTANTIATE_TEST_CASE_P(MeaningfulTestParameters,
+                        PrimeTableTest,
+                        Combine(Bool(), Values(1, 10)));
+
+#else
+
+// Google Test may not support Combine() with some compilers. If we
+// use conditional compilation to compile out all code referring to
+// the gtest_main library, MSVC linker will not link that library at
+// all and consequently complain about missing entry point defined in
+// that library (fatal error LNK1561: entry point must be
+// defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, CombineIsNotSupportedOnThisPlatform) {}
+
+#endif  // GTEST_HAS_COMBINE
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to use Google Test listener API to implement
+// an alternative console output and how to use the UnitTest reflection API
+// to enumerate test cases and tests and to inspect their results.
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+namespace {
+
+// Provides alternative output mode which produces minimal amount of
+// information about tests.
+class TersePrinter : public EmptyTestEventListener {
+ private:
+  // Called before any test activity starts.
+  virtual void OnTestProgramStart(const UnitTest& /* unit_test */) {}
+
+  // Called after all test activities have ended.
+  virtual void OnTestProgramEnd(const UnitTest& unit_test) {
+    fprintf(stdout, "TEST %s\n", unit_test.Passed() ? "PASSED" : "FAILED");
+    fflush(stdout);
+  }
+
+  // Called before a test starts.
+  virtual void OnTestStart(const TestInfo& test_info) {
+    fprintf(stdout,
+            "*** Test %s.%s starting.\n",
+            test_info.test_case_name(),
+            test_info.name());
+    fflush(stdout);
+  }
+
+  // Called after a failed assertion or a SUCCEED() invocation.
+  virtual void OnTestPartResult(const TestPartResult& test_part_result) {
+    fprintf(stdout,
+            "%s in %s:%d\n%s\n",
+            test_part_result.failed() ? "*** Failure" : "Success",
+            test_part_result.file_name(),
+            test_part_result.line_number(),
+            test_part_result.summary());
+    fflush(stdout);
+  }
+
+  // Called after a test ends.
+  virtual void OnTestEnd(const TestInfo& test_info) {
+    fprintf(stdout,
+            "*** Test %s.%s ending.\n",
+            test_info.test_case_name(),
+            test_info.name());
+    fflush(stdout);
+  }
+};  // class TersePrinter
+
+TEST(CustomOutputTest, PrintsMessage) {
+  printf("Printing something from the test body...\n");
+}
+
+TEST(CustomOutputTest, Succeeds) {
+  SUCCEED() << "SUCCEED() has been invoked from here";
+}
+
+TEST(CustomOutputTest, Fails) {
+  EXPECT_EQ(1, 2)
+      << "This test fails in order to demonstrate alternative failure messages";
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  bool terse_output = false;
+  if (argc > 1 && strcmp(argv[1], "--terse_output") == 0 )
+    terse_output = true;
+  else
+    printf("%s\n", "Run this program with --terse_output to change the way "
+           "it prints its output.");
+
+  UnitTest& unit_test = *UnitTest::GetInstance();
+
+  // If we are given the --terse_output command line flag, suppresses the
+  // standard output and attaches own result printer.
+  if (terse_output) {
+    TestEventListeners& listeners = unit_test.listeners();
+
+    // Removes the default console output listener from the list so it will
+    // not receive events from Google Test and won't print any output. Since
+    // this operation transfers ownership of the listener to the caller we
+    // have to delete it as well.
+    delete listeners.Release(listeners.default_result_printer());
+
+    // Adds the custom output listener to the list. It will now receive
+    // events from Google Test and print the alternative output. We don't
+    // have to worry about deleting it since Google Test assumes ownership
+    // over it after adding it to the list.
+    listeners.Append(new TersePrinter);
+  }
+  int ret_val = RUN_ALL_TESTS();
+
+  // This is an example of using the UnitTest reflection API to inspect test
+  // results. Here we discount failures from the tests we expected to fail.
+  int unexpectedly_failed_tests = 0;
+  for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
+    const TestCase& test_case = *unit_test.GetTestCase(i);
+    for (int j = 0; j < test_case.total_test_count(); ++j) {
+      const TestInfo& test_info = *test_case.GetTestInfo(j);
+      // Counts failed tests that were not meant to fail (those without
+      // 'Fails' in the name).
+      if (test_info.result()->Failed() &&
+          strcmp(test_info.name(), "Fails") != 0) {
+        unexpectedly_failed_tests++;
+      }
+    }
+  }
+
+  // Test that were meant to fail should not affect the test program outcome.
+  if (unexpectedly_failed_tests == 0)
+    ret_val = 0;
+
+  return ret_val;
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// Sometimes it's desirable to build Google Test by compiling a single file.
+// This file serves this purpose.
+
+// This line ensures that gtest.h can be compiled on its own, even
+// when it's fused.
+#include "gtest/gtest.h"
+
+// The following lines pull in the real gtest *.cc files.
+#include "src/gtest.cc"
+#include "src/gtest-death-test.cc"
+#include "src/gtest-filepath.cc"
+#include "src/gtest-port.cc"
+#include "src/gtest-printers.cc"
+#include "src/gtest-test-part.cc"
+#include "src/gtest-typed-test.cc"
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan), vladl@google.com (Vlad Losev)
+//
+// This file implements death tests.
+
+#include "gtest/gtest-death-test.h"
+#include "gtest/internal/gtest-port.h"
+#include "gtest/internal/custom/gtest.h"
+
+#if GTEST_HAS_DEATH_TEST
+
+# if GTEST_OS_MAC
+#  include <crt_externs.h>
+# endif  // GTEST_OS_MAC
+
+# include <errno.h>
+# include <fcntl.h>
+# include <limits.h>
+
+# if GTEST_OS_LINUX
+#  include <signal.h>
+# endif  // GTEST_OS_LINUX
+
+# include <stdarg.h>
+
+# if GTEST_OS_WINDOWS
+#  include <windows.h>
+# else
+#  include <sys/mman.h>
+#  include <sys/wait.h>
+# endif  // GTEST_OS_WINDOWS
+
+# if GTEST_OS_QNX
+#  include <spawn.h>
+# endif  // GTEST_OS_QNX
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-string.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+
+// Constants.
+
+// The default death test style.
+static const char kDefaultDeathTestStyle[] = "fast";
+
+GTEST_DEFINE_string_(
+    death_test_style,
+    internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle),
+    "Indicates how to run a death test in a forked child process: "
+    "\"threadsafe\" (child process re-executes the test binary "
+    "from the beginning, running only the specific death test) or "
+    "\"fast\" (child process runs the death test immediately "
+    "after forking).");
+
+GTEST_DEFINE_bool_(
+    death_test_use_fork,
+    internal::BoolFromGTestEnv("death_test_use_fork", false),
+    "Instructs to use fork()/_exit() instead of clone() in death tests. "
+    "Ignored and always uses fork() on POSIX systems where clone() is not "
+    "implemented. Useful when running under valgrind or similar tools if "
+    "those do not support clone(). Valgrind 3.3.1 will just fail if "
+    "it sees an unsupported combination of clone() flags. "
+    "It is not recommended to use this flag w/o valgrind though it will "
+    "work in 99% of the cases. Once valgrind is fixed, this flag will "
+    "most likely be removed.");
+
+namespace internal {
+GTEST_DEFINE_string_(
+    internal_run_death_test, "",
+    "Indicates the file, line number, temporal index of "
+    "the single death test to run, and a file descriptor to "
+    "which a success code may be sent, all separated by "
+    "the '|' characters.  This flag is specified if and only if the current "
+    "process is a sub-process launched for running a thread-safe "
+    "death test.  FOR INTERNAL USE ONLY.");
+}  // namespace internal
+
+#if GTEST_HAS_DEATH_TEST
+
+namespace internal {
+
+// Valid only for fast death tests. Indicates the code is running in the
+// child process of a fast style death test.
+# if !GTEST_OS_WINDOWS
+static bool g_in_fast_death_test_child = false;
+# endif
+
+// Returns a Boolean value indicating whether the caller is currently
+// executing in the context of the death test child process.  Tools such as
+// Valgrind heap checkers may need this to modify their behavior in death
+// tests.  IMPORTANT: This is an internal utility.  Using it may break the
+// implementation of death tests.  User code MUST NOT use it.
+bool InDeathTestChild() {
+# if GTEST_OS_WINDOWS
+
+  // On Windows, death tests are thread-safe regardless of the value of the
+  // death_test_style flag.
+  return !GTEST_FLAG(internal_run_death_test).empty();
+
+# else
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe")
+    return !GTEST_FLAG(internal_run_death_test).empty();
+  else
+    return g_in_fast_death_test_child;
+#endif
+}
+
+}  // namespace internal
+
+// ExitedWithCode constructor.
+ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) {
+}
+
+// ExitedWithCode function-call operator.
+bool ExitedWithCode::operator()(int exit_status) const {
+# if GTEST_OS_WINDOWS
+
+  return exit_status == exit_code_;
+
+# else
+
+  return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_;
+
+# endif  // GTEST_OS_WINDOWS
+}
+
+# if !GTEST_OS_WINDOWS
+// KilledBySignal constructor.
+KilledBySignal::KilledBySignal(int signum) : signum_(signum) {
+}
+
+// KilledBySignal function-call operator.
+bool KilledBySignal::operator()(int exit_status) const {
+#  if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
+  {
+    bool result;
+    if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) {
+      return result;
+    }
+  }
+#  endif  // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
+  return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_;
+}
+# endif  // !GTEST_OS_WINDOWS
+
+namespace internal {
+
+// Utilities needed for death tests.
+
+// Generates a textual description of a given exit code, in the format
+// specified by wait(2).
+static std::string ExitSummary(int exit_code) {
+  Message m;
+
+# if GTEST_OS_WINDOWS
+
+  m << "Exited with exit status " << exit_code;
+
+# else
+
+  if (WIFEXITED(exit_code)) {
+    m << "Exited with exit status " << WEXITSTATUS(exit_code);
+  } else if (WIFSIGNALED(exit_code)) {
+    m << "Terminated by signal " << WTERMSIG(exit_code);
+  }
+#  ifdef WCOREDUMP
+  if (WCOREDUMP(exit_code)) {
+    m << " (core dumped)";
+  }
+#  endif
+# endif  // GTEST_OS_WINDOWS
+
+  return m.GetString();
+}
+
+// Returns true if exit_status describes a process that was terminated
+// by a signal, or exited normally with a nonzero exit code.
+bool ExitedUnsuccessfully(int exit_status) {
+  return !ExitedWithCode(0)(exit_status);
+}
+
+# if !GTEST_OS_WINDOWS
+// Generates a textual failure message when a death test finds more than
+// one thread running, or cannot determine the number of threads, prior
+// to executing the given statement.  It is the responsibility of the
+// caller not to pass a thread_count of 1.
+static std::string DeathTestThreadWarning(size_t thread_count) {
+  Message msg;
+  msg << "Death tests use fork(), which is unsafe particularly"
+      << " in a threaded context. For this test, " << GTEST_NAME_ << " ";
+  if (thread_count == 0)
+    msg << "couldn't detect the number of threads.";
+  else
+    msg << "detected " << thread_count << " threads.";
+  return msg.GetString();
+}
+# endif  // !GTEST_OS_WINDOWS
+
+// Flag characters for reporting a death test that did not die.
+static const char kDeathTestLived = 'L';
+static const char kDeathTestReturned = 'R';
+static const char kDeathTestThrew = 'T';
+static const char kDeathTestInternalError = 'I';
+
+// An enumeration describing all of the possible ways that a death test can
+// conclude.  DIED means that the process died while executing the test
+// code; LIVED means that process lived beyond the end of the test code;
+// RETURNED means that the test statement attempted to execute a return
+// statement, which is not allowed; THREW means that the test statement
+// returned control by throwing an exception.  IN_PROGRESS means the test
+// has not yet concluded.
+// TODO(vladl@google.com): Unify names and possibly values for
+// AbortReason, DeathTestOutcome, and flag characters above.
+enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW };
+
+// Routine for aborting the program which is safe to call from an
+// exec-style death test child process, in which case the error
+// message is propagated back to the parent process.  Otherwise, the
+// message is simply printed to stderr.  In either case, the program
+// then exits with status 1.
+void DeathTestAbort(const std::string& message) {
+  // On a POSIX system, this function may be called from a threadsafe-style
+  // death test child process, which operates on a very small stack.  Use
+  // the heap for any additional non-minuscule memory requirements.
+  const InternalRunDeathTestFlag* const flag =
+      GetUnitTestImpl()->internal_run_death_test_flag();
+  if (flag != NULL) {
+    FILE* parent = posix::FDOpen(flag->write_fd(), "w");
+    fputc(kDeathTestInternalError, parent);
+    fprintf(parent, "%s", message.c_str());
+    fflush(parent);
+    _exit(1);
+  } else {
+    fprintf(stderr, "%s", message.c_str());
+    fflush(stderr);
+    posix::Abort();
+  }
+}
+
+// A replacement for CHECK that calls DeathTestAbort if the assertion
+// fails.
+# define GTEST_DEATH_TEST_CHECK_(expression) \
+  do { \
+    if (!::testing::internal::IsTrue(expression)) { \
+      DeathTestAbort( \
+          ::std::string("CHECK failed: File ") + __FILE__ +  ", line " \
+          + ::testing::internal::StreamableToString(__LINE__) + ": " \
+          + #expression); \
+    } \
+  } while (::testing::internal::AlwaysFalse())
+
+// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for
+// evaluating any system call that fulfills two conditions: it must return
+// -1 on failure, and set errno to EINTR when it is interrupted and
+// should be tried again.  The macro expands to a loop that repeatedly
+// evaluates the expression as long as it evaluates to -1 and sets
+// errno to EINTR.  If the expression evaluates to -1 but errno is
+// something other than EINTR, DeathTestAbort is called.
+# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \
+  do { \
+    int gtest_retval; \
+    do { \
+      gtest_retval = (expression); \
+    } while (gtest_retval == -1 && errno == EINTR); \
+    if (gtest_retval == -1) { \
+      DeathTestAbort( \
+          ::std::string("CHECK failed: File ") + __FILE__ + ", line " \
+          + ::testing::internal::StreamableToString(__LINE__) + ": " \
+          + #expression + " != -1"); \
+    } \
+  } while (::testing::internal::AlwaysFalse())
+
+// Returns the message describing the last system error in errno.
+std::string GetLastErrnoDescription() {
+    return errno == 0 ? "" : posix::StrError(errno);
+}
+
+// This is called from a death test parent process to read a failure
+// message from the death test child process and log it with the FATAL
+// severity. On Windows, the message is read from a pipe handle. On other
+// platforms, it is read from a file descriptor.
+static void FailFromInternalError(int fd) {
+  Message error;
+  char buffer[256];
+  int num_read;
+
+  do {
+    while ((num_read = posix::Read(fd, buffer, 255)) > 0) {
+      buffer[num_read] = '\0';
+      error << buffer;
+    }
+  } while (num_read == -1 && errno == EINTR);
+
+  if (num_read == 0) {
+    GTEST_LOG_(FATAL) << error.GetString();
+  } else {
+    const int last_error = errno;
+    GTEST_LOG_(FATAL) << "Error while reading death test internal: "
+                      << GetLastErrnoDescription() << " [" << last_error << "]";
+  }
+}
+
+// Death test constructor.  Increments the running death test count
+// for the current test.
+DeathTest::DeathTest() {
+  TestInfo* const info = GetUnitTestImpl()->current_test_info();
+  if (info == NULL) {
+    DeathTestAbort("Cannot run a death test outside of a TEST or "
+                   "TEST_F construct");
+  }
+}
+
+// Creates and returns a death test by dispatching to the current
+// death test factory.
+bool DeathTest::Create(const char* statement, const RE* regex,
+                       const char* file, int line, DeathTest** test) {
+  return GetUnitTestImpl()->death_test_factory()->Create(
+      statement, regex, file, line, test);
+}
+
+const char* DeathTest::LastMessage() {
+  return last_death_test_message_.c_str();
+}
+
+void DeathTest::set_last_death_test_message(const std::string& message) {
+  last_death_test_message_ = message;
+}
+
+std::string DeathTest::last_death_test_message_;
+
+// Provides cross platform implementation for some death functionality.
+class DeathTestImpl : public DeathTest {
+ protected:
+  DeathTestImpl(const char* a_statement, const RE* a_regex)
+      : statement_(a_statement),
+        regex_(a_regex),
+        spawned_(false),
+        status_(-1),
+        outcome_(IN_PROGRESS),
+        read_fd_(-1),
+        write_fd_(-1) {}
+
+  // read_fd_ is expected to be closed and cleared by a derived class.
+  ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); }
+
+  void Abort(AbortReason reason);
+  virtual bool Passed(bool status_ok);
+
+  const char* statement() const { return statement_; }
+  const RE* regex() const { return regex_; }
+  bool spawned() const { return spawned_; }
+  void set_spawned(bool is_spawned) { spawned_ = is_spawned; }
+  int status() const { return status_; }
+  void set_status(int a_status) { status_ = a_status; }
+  DeathTestOutcome outcome() const { return outcome_; }
+  void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; }
+  int read_fd() const { return read_fd_; }
+  void set_read_fd(int fd) { read_fd_ = fd; }
+  int write_fd() const { return write_fd_; }
+  void set_write_fd(int fd) { write_fd_ = fd; }
+
+  // Called in the parent process only. Reads the result code of the death
+  // test child process via a pipe, interprets it to set the outcome_
+  // member, and closes read_fd_.  Outputs diagnostics and terminates in
+  // case of unexpected codes.
+  void ReadAndInterpretStatusByte();
+
+ private:
+  // The textual content of the code this object is testing.  This class
+  // doesn't own this string and should not attempt to delete it.
+  const char* const statement_;
+  // The regular expression which test output must match.  DeathTestImpl
+  // doesn't own this object and should not attempt to delete it.
+  const RE* const regex_;
+  // True if the death test child process has been successfully spawned.
+  bool spawned_;
+  // The exit status of the child process.
+  int status_;
+  // How the death test concluded.
+  DeathTestOutcome outcome_;
+  // Descriptor to the read end of the pipe to the child process.  It is
+  // always -1 in the child process.  The child keeps its write end of the
+  // pipe in write_fd_.
+  int read_fd_;
+  // Descriptor to the child's write end of the pipe to the parent process.
+  // It is always -1 in the parent process.  The parent keeps its end of the
+  // pipe in read_fd_.
+  int write_fd_;
+};
+
+// Called in the parent process only. Reads the result code of the death
+// test child process via a pipe, interprets it to set the outcome_
+// member, and closes read_fd_.  Outputs diagnostics and terminates in
+// case of unexpected codes.
+void DeathTestImpl::ReadAndInterpretStatusByte() {
+  char flag;
+  int bytes_read;
+
+  // The read() here blocks until data is available (signifying the
+  // failure of the death test) or until the pipe is closed (signifying
+  // its success), so it's okay to call this in the parent before
+  // the child process has exited.
+  do {
+    bytes_read = posix::Read(read_fd(), &flag, 1);
+  } while (bytes_read == -1 && errno == EINTR);
+
+  if (bytes_read == 0) {
+    set_outcome(DIED);
+  } else if (bytes_read == 1) {
+    switch (flag) {
+      case kDeathTestReturned:
+        set_outcome(RETURNED);
+        break;
+      case kDeathTestThrew:
+        set_outcome(THREW);
+        break;
+      case kDeathTestLived:
+        set_outcome(LIVED);
+        break;
+      case kDeathTestInternalError:
+        FailFromInternalError(read_fd());  // Does not return.
+        break;
+      default:
+        GTEST_LOG_(FATAL) << "Death test child process reported "
+                          << "unexpected status byte ("
+                          << static_cast<unsigned int>(flag) << ")";
+    }
+  } else {
+    GTEST_LOG_(FATAL) << "Read from death test child process failed: "
+                      << GetLastErrnoDescription();
+  }
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd()));
+  set_read_fd(-1);
+}
+
+// Signals that the death test code which should have exited, didn't.
+// Should be called only in a death test child process.
+// Writes a status byte to the child's status file descriptor, then
+// calls _exit(1).
+void DeathTestImpl::Abort(AbortReason reason) {
+  // The parent process considers the death test to be a failure if
+  // it finds any data in our pipe.  So, here we write a single flag byte
+  // to the pipe, then exit.
+  const char status_ch =
+      reason == TEST_DID_NOT_DIE ? kDeathTestLived :
+      reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned;
+
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1));
+  // We are leaking the descriptor here because on some platforms (i.e.,
+  // when built as Windows DLL), destructors of global objects will still
+  // run after calling _exit(). On such systems, write_fd_ will be
+  // indirectly closed from the destructor of UnitTestImpl, causing double
+  // close if it is also closed here. On debug configurations, double close
+  // may assert. As there are no in-process buffers to flush here, we are
+  // relying on the OS to close the descriptor after the process terminates
+  // when the destructors are not run.
+  _exit(1);  // Exits w/o any normal exit hooks (we were supposed to crash)
+}
+
+// Returns an indented copy of stderr output for a death test.
+// This makes distinguishing death test output lines from regular log lines
+// much easier.
+static ::std::string FormatDeathTestOutput(const ::std::string& output) {
+  ::std::string ret;
+  for (size_t at = 0; ; ) {
+    const size_t line_end = output.find('\n', at);
+    ret += "[  DEATH   ] ";
+    if (line_end == ::std::string::npos) {
+      ret += output.substr(at);
+      break;
+    }
+    ret += output.substr(at, line_end + 1 - at);
+    at = line_end + 1;
+  }
+  return ret;
+}
+
+// Assesses the success or failure of a death test, using both private
+// members which have previously been set, and one argument:
+//
+// Private data members:
+//   outcome:  An enumeration describing how the death test
+//             concluded: DIED, LIVED, THREW, or RETURNED.  The death test
+//             fails in the latter three cases.
+//   status:   The exit status of the child process. On *nix, it is in the
+//             in the format specified by wait(2). On Windows, this is the
+//             value supplied to the ExitProcess() API or a numeric code
+//             of the exception that terminated the program.
+//   regex:    A regular expression object to be applied to
+//             the test's captured standard error output; the death test
+//             fails if it does not match.
+//
+// Argument:
+//   status_ok: true if exit_status is acceptable in the context of
+//              this particular death test, which fails if it is false
+//
+// Returns true iff all of the above conditions are met.  Otherwise, the
+// first failing condition, in the order given above, is the one that is
+// reported. Also sets the last death test message string.
+bool DeathTestImpl::Passed(bool status_ok) {
+  if (!spawned())
+    return false;
+
+  const std::string error_message = GetCapturedStderr();
+
+  bool success = false;
+  Message buffer;
+
+  buffer << "Death test: " << statement() << "\n";
+  switch (outcome()) {
+    case LIVED:
+      buffer << "    Result: failed to die.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case THREW:
+      buffer << "    Result: threw an exception.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case RETURNED:
+      buffer << "    Result: illegal return in test statement.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case DIED:
+      if (status_ok) {
+        const bool matched = RE::PartialMatch(error_message.c_str(), *regex());
+        if (matched) {
+          success = true;
+        } else {
+          buffer << "    Result: died but not with expected error.\n"
+                 << "  Expected: " << regex()->pattern() << "\n"
+                 << "Actual msg:\n" << FormatDeathTestOutput(error_message);
+        }
+      } else {
+        buffer << "    Result: died but not with expected exit code:\n"
+               << "            " << ExitSummary(status()) << "\n"
+               << "Actual msg:\n" << FormatDeathTestOutput(error_message);
+      }
+      break;
+    case IN_PROGRESS:
+    default:
+      GTEST_LOG_(FATAL)
+          << "DeathTest::Passed somehow called before conclusion of test";
+  }
+
+  DeathTest::set_last_death_test_message(buffer.GetString());
+  return success;
+}
+
+# if GTEST_OS_WINDOWS
+// WindowsDeathTest implements death tests on Windows. Due to the
+// specifics of starting new processes on Windows, death tests there are
+// always threadsafe, and Google Test considers the
+// --gtest_death_test_style=fast setting to be equivalent to
+// --gtest_death_test_style=threadsafe there.
+//
+// A few implementation notes:  Like the Linux version, the Windows
+// implementation uses pipes for child-to-parent communication. But due to
+// the specifics of pipes on Windows, some extra steps are required:
+//
+// 1. The parent creates a communication pipe and stores handles to both
+//    ends of it.
+// 2. The parent starts the child and provides it with the information
+//    necessary to acquire the handle to the write end of the pipe.
+// 3. The child acquires the write end of the pipe and signals the parent
+//    using a Windows event.
+// 4. Now the parent can release the write end of the pipe on its side. If
+//    this is done before step 3, the object's reference count goes down to
+//    0 and it is destroyed, preventing the child from acquiring it. The
+//    parent now has to release it, or read operations on the read end of
+//    the pipe will not return when the child terminates.
+// 5. The parent reads child's output through the pipe (outcome code and
+//    any possible error messages) from the pipe, and its stderr and then
+//    determines whether to fail the test.
+//
+// Note: to distinguish Win32 API calls from the local method and function
+// calls, the former are explicitly resolved in the global namespace.
+//
+class WindowsDeathTest : public DeathTestImpl {
+ public:
+  WindowsDeathTest(const char* a_statement,
+                   const RE* a_regex,
+                   const char* file,
+                   int line)
+      : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {}
+
+  // All of these virtual functions are inherited from DeathTest.
+  virtual int Wait();
+  virtual TestRole AssumeRole();
+
+ private:
+  // The name of the file in which the death test is located.
+  const char* const file_;
+  // The line number on which the death test is located.
+  const int line_;
+  // Handle to the write end of the pipe to the child process.
+  AutoHandle write_handle_;
+  // Child process handle.
+  AutoHandle child_handle_;
+  // Event the child process uses to signal the parent that it has
+  // acquired the handle to the write end of the pipe. After seeing this
+  // event the parent can release its own handles to make sure its
+  // ReadFile() calls return when the child terminates.
+  AutoHandle event_handle_;
+};
+
+// Waits for the child in a death test to exit, returning its exit
+// status, or 0 if no child process exists.  As a side effect, sets the
+// outcome data member.
+int WindowsDeathTest::Wait() {
+  if (!spawned())
+    return 0;
+
+  // Wait until the child either signals that it has acquired the write end
+  // of the pipe or it dies.
+  const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() };
+  switch (::WaitForMultipleObjects(2,
+                                   wait_handles,
+                                   FALSE,  // Waits for any of the handles.
+                                   INFINITE)) {
+    case WAIT_OBJECT_0:
+    case WAIT_OBJECT_0 + 1:
+      break;
+    default:
+      GTEST_DEATH_TEST_CHECK_(false);  // Should not get here.
+  }
+
+  // The child has acquired the write end of the pipe or exited.
+  // We release the handle on our side and continue.
+  write_handle_.Reset();
+  event_handle_.Reset();
+
+  ReadAndInterpretStatusByte();
+
+  // Waits for the child process to exit if it haven't already. This
+  // returns immediately if the child has already exited, regardless of
+  // whether previous calls to WaitForMultipleObjects synchronized on this
+  // handle or not.
+  GTEST_DEATH_TEST_CHECK_(
+      WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(),
+                                             INFINITE));
+  DWORD status_code;
+  GTEST_DEATH_TEST_CHECK_(
+      ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE);
+  child_handle_.Reset();
+  set_status(static_cast<int>(status_code));
+  return status();
+}
+
+// The AssumeRole process for a Windows death test.  It creates a child
+// process with the same executable as the current process to run the
+// death test.  The child process is given the --gtest_filter and
+// --gtest_internal_run_death_test flags such that it knows to run the
+// current death test only.
+DeathTest::TestRole WindowsDeathTest::AssumeRole() {
+  const UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const TestInfo* const info = impl->current_test_info();
+  const int death_test_index = info->result()->death_test_count();
+
+  if (flag != NULL) {
+    // ParseInternalRunDeathTestFlag() has performed all the necessary
+    // processing.
+    set_write_fd(flag->write_fd());
+    return EXECUTE_TEST;
+  }
+
+  // WindowsDeathTest uses an anonymous pipe to communicate results of
+  // a death test.
+  SECURITY_ATTRIBUTES handles_are_inheritable = {
+    sizeof(SECURITY_ATTRIBUTES), NULL, TRUE };
+  HANDLE read_handle, write_handle;
+  GTEST_DEATH_TEST_CHECK_(
+      ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable,
+                   0)  // Default buffer size.
+      != FALSE);
+  set_read_fd(::_open_osfhandle(reinterpret_cast<intptr_t>(read_handle),
+                                O_RDONLY));
+  write_handle_.Reset(write_handle);
+  event_handle_.Reset(::CreateEvent(
+      &handles_are_inheritable,
+      TRUE,    // The event will automatically reset to non-signaled state.
+      FALSE,   // The initial state is non-signalled.
+      NULL));  // The even is unnamed.
+  GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL);
+  const std::string filter_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" +
+      info->test_case_name() + "." + info->name();
+  const std::string internal_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag +
+      "=" + file_ + "|" + StreamableToString(line_) + "|" +
+      StreamableToString(death_test_index) + "|" +
+      StreamableToString(static_cast<unsigned int>(::GetCurrentProcessId())) +
+      // size_t has the same width as pointers on both 32-bit and 64-bit
+      // Windows platforms.
+      // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx.
+      "|" + StreamableToString(reinterpret_cast<size_t>(write_handle)) +
+      "|" + StreamableToString(reinterpret_cast<size_t>(event_handle_.Get()));
+
+  char executable_path[_MAX_PATH + 1];  // NOLINT
+  GTEST_DEATH_TEST_CHECK_(
+      _MAX_PATH + 1 != ::GetModuleFileNameA(NULL,
+                                            executable_path,
+                                            _MAX_PATH));
+
+  std::string command_line =
+      std::string(::GetCommandLineA()) + " " + filter_flag + " \"" +
+      internal_flag + "\"";
+
+  DeathTest::set_last_death_test_message("");
+
+  CaptureStderr();
+  // Flush the log buffers since the log streams are shared with the child.
+  FlushInfoLog();
+
+  // The child process will share the standard handles with the parent.
+  STARTUPINFOA startup_info;
+  memset(&startup_info, 0, sizeof(STARTUPINFO));
+  startup_info.dwFlags = STARTF_USESTDHANDLES;
+  startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE);
+  startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE);
+  startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE);
+
+  PROCESS_INFORMATION process_info;
+  GTEST_DEATH_TEST_CHECK_(::CreateProcessA(
+      executable_path,
+      const_cast<char*>(command_line.c_str()),
+      NULL,   // Retuned process handle is not inheritable.
+      NULL,   // Retuned thread handle is not inheritable.
+      TRUE,   // Child inherits all inheritable handles (for write_handle_).
+      0x0,    // Default creation flags.
+      NULL,   // Inherit the parent's environment.
+      UnitTest::GetInstance()->original_working_dir(),
+      &startup_info,
+      &process_info) != FALSE);
+  child_handle_.Reset(process_info.hProcess);
+  ::CloseHandle(process_info.hThread);
+  set_spawned(true);
+  return OVERSEE_TEST;
+}
+# else  // We are not on Windows.
+
+// ForkingDeathTest provides implementations for most of the abstract
+// methods of the DeathTest interface.  Only the AssumeRole method is
+// left undefined.
+class ForkingDeathTest : public DeathTestImpl {
+ public:
+  ForkingDeathTest(const char* statement, const RE* regex);
+
+  // All of these virtual functions are inherited from DeathTest.
+  virtual int Wait();
+
+ protected:
+  void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; }
+
+ private:
+  // PID of child process during death test; 0 in the child process itself.
+  pid_t child_pid_;
+};
+
+// Constructs a ForkingDeathTest.
+ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex)
+    : DeathTestImpl(a_statement, a_regex),
+      child_pid_(-1) {}
+
+// Waits for the child in a death test to exit, returning its exit
+// status, or 0 if no child process exists.  As a side effect, sets the
+// outcome data member.
+int ForkingDeathTest::Wait() {
+  if (!spawned())
+    return 0;
+
+  ReadAndInterpretStatusByte();
+
+  int status_value;
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0));
+  set_status(status_value);
+  return status_value;
+}
+
+// A concrete death test class that forks, then immediately runs the test
+// in the child process.
+class NoExecDeathTest : public ForkingDeathTest {
+ public:
+  NoExecDeathTest(const char* a_statement, const RE* a_regex) :
+      ForkingDeathTest(a_statement, a_regex) { }
+  virtual TestRole AssumeRole();
+};
+
+// The AssumeRole process for a fork-and-run death test.  It implements a
+// straightforward fork, with a simple pipe to transmit the status byte.
+DeathTest::TestRole NoExecDeathTest::AssumeRole() {
+  const size_t thread_count = GetThreadCount();
+  if (thread_count != 1) {
+    GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count);
+  }
+
+  int pipe_fd[2];
+  GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1);
+
+  DeathTest::set_last_death_test_message("");
+  CaptureStderr();
+  // When we fork the process below, the log file buffers are copied, but the
+  // file descriptors are shared.  We flush all log files here so that closing
+  // the file descriptors in the child process doesn't throw off the
+  // synchronization between descriptors and buffers in the parent process.
+  // This is as close to the fork as possible to avoid a race condition in case
+  // there are multiple threads running before the death test, and another
+  // thread writes to the log file.
+  FlushInfoLog();
+
+  const pid_t child_pid = fork();
+  GTEST_DEATH_TEST_CHECK_(child_pid != -1);
+  set_child_pid(child_pid);
+  if (child_pid == 0) {
+    GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0]));
+    set_write_fd(pipe_fd[1]);
+    // Redirects all logging to stderr in the child process to prevent
+    // concurrent writes to the log files.  We capture stderr in the parent
+    // process and append the child process' output to a log.
+    LogToStderr();
+    // Event forwarding to the listeners of event listener API mush be shut
+    // down in death test subprocesses.
+    GetUnitTestImpl()->listeners()->SuppressEventForwarding();
+    g_in_fast_death_test_child = true;
+    return EXECUTE_TEST;
+  } else {
+    GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1]));
+    set_read_fd(pipe_fd[0]);
+    set_spawned(true);
+    return OVERSEE_TEST;
+  }
+}
+
+// A concrete death test class that forks and re-executes the main
+// program from the beginning, with command-line flags set that cause
+// only this specific death test to be run.
+class ExecDeathTest : public ForkingDeathTest {
+ public:
+  ExecDeathTest(const char* a_statement, const RE* a_regex,
+                const char* file, int line) :
+      ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { }
+  virtual TestRole AssumeRole();
+ private:
+  static ::std::vector<testing::internal::string>
+  GetArgvsForDeathTestChildProcess() {
+    ::std::vector<testing::internal::string> args = GetInjectableArgvs();
+#  if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
+    ::std::vector<testing::internal::string> extra_args =
+        GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_();
+    args.insert(args.end(), extra_args.begin(), extra_args.end());
+#  endif  // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
+    return args;
+  }
+  // The name of the file in which the death test is located.
+  const char* const file_;
+  // The line number on which the death test is located.
+  const int line_;
+};
+
+// Utility class for accumulating command-line arguments.
+class Arguments {
+ public:
+  Arguments() {
+    args_.push_back(NULL);
+  }
+
+  ~Arguments() {
+    for (std::vector<char*>::iterator i = args_.begin(); i != args_.end();
+         ++i) {
+      free(*i);
+    }
+  }
+  void AddArgument(const char* argument) {
+    args_.insert(args_.end() - 1, posix::StrDup(argument));
+  }
+
+  template <typename Str>
+  void AddArguments(const ::std::vector<Str>& arguments) {
+    for (typename ::std::vector<Str>::const_iterator i = arguments.begin();
+         i != arguments.end();
+         ++i) {
+      args_.insert(args_.end() - 1, posix::StrDup(i->c_str()));
+    }
+  }
+  char* const* Argv() {
+    return &args_[0];
+  }
+
+ private:
+  std::vector<char*> args_;
+};
+
+// A struct that encompasses the arguments to the child process of a
+// threadsafe-style death test process.
+struct ExecDeathTestArgs {
+  char* const* argv;  // Command-line arguments for the child's call to exec
+  int close_fd;       // File descriptor to close; the read end of a pipe
+};
+
+#  if GTEST_OS_MAC
+inline char** GetEnviron() {
+  // When Google Test is built as a framework on MacOS X, the environ variable
+  // is unavailable. Apple's documentation (man environ) recommends using
+  // _NSGetEnviron() instead.
+  return *_NSGetEnviron();
+}
+#  else
+// Some POSIX platforms expect you to declare environ. extern "C" makes
+// it reside in the global namespace.
+extern "C" char** environ;
+inline char** GetEnviron() { return environ; }
+#  endif  // GTEST_OS_MAC
+
+#  if !GTEST_OS_QNX
+// The main function for a threadsafe-style death test child process.
+// This function is called in a clone()-ed process and thus must avoid
+// any potentially unsafe operations like malloc or libc functions.
+static int ExecDeathTestChildMain(void* child_arg) {
+  ExecDeathTestArgs* const args = static_cast<ExecDeathTestArgs*>(child_arg);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd));
+
+  // We need to execute the test program in the same environment where
+  // it was originally invoked.  Therefore we change to the original
+  // working directory first.
+  const char* const original_dir =
+      UnitTest::GetInstance()->original_working_dir();
+  // We can safely call chdir() as it's a direct system call.
+  if (chdir(original_dir) != 0) {
+    DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " +
+                   GetLastErrnoDescription());
+    return EXIT_FAILURE;
+  }
+
+  // We can safely call execve() as it's a direct system call.  We
+  // cannot use execvp() as it's a libc function and thus potentially
+  // unsafe.  Since execve() doesn't search the PATH, the user must
+  // invoke the test program via a valid path that contains at least
+  // one path separator.
+  execve(args->argv[0], args->argv, GetEnviron());
+  DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " +
+                 original_dir + " failed: " +
+                 GetLastErrnoDescription());
+  return EXIT_FAILURE;
+}
+#  endif  // !GTEST_OS_QNX
+
+// Two utility routines that together determine the direction the stack
+// grows.
+// This could be accomplished more elegantly by a single recursive
+// function, but we want to guard against the unlikely possibility of
+// a smart compiler optimizing the recursion away.
+//
+// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining
+// StackLowerThanAddress into StackGrowsDown, which then doesn't give
+// correct answer.
+void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_;
+void StackLowerThanAddress(const void* ptr, bool* result) {
+  int dummy;
+  *result = (&dummy < ptr);
+}
+
+// Make sure AddressSanitizer does not tamper with the stack here.
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+bool StackGrowsDown() {
+  int dummy;
+  bool result;
+  StackLowerThanAddress(&dummy, &result);
+  return result;
+}
+
+// Spawns a child process with the same executable as the current process in
+// a thread-safe manner and instructs it to run the death test.  The
+// implementation uses fork(2) + exec.  On systems where clone(2) is
+// available, it is used instead, being slightly more thread-safe.  On QNX,
+// fork supports only single-threaded environments, so this function uses
+// spawn(2) there instead.  The function dies with an error message if
+// anything goes wrong.
+static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) {
+  ExecDeathTestArgs args = { argv, close_fd };
+  pid_t child_pid = -1;
+
+#  if GTEST_OS_QNX
+  // Obtains the current directory and sets it to be closed in the child
+  // process.
+  const int cwd_fd = open(".", O_RDONLY);
+  GTEST_DEATH_TEST_CHECK_(cwd_fd != -1);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC));
+  // We need to execute the test program in the same environment where
+  // it was originally invoked.  Therefore we change to the original
+  // working directory first.
+  const char* const original_dir =
+      UnitTest::GetInstance()->original_working_dir();
+  // We can safely call chdir() as it's a direct system call.
+  if (chdir(original_dir) != 0) {
+    DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " +
+                   GetLastErrnoDescription());
+    return EXIT_FAILURE;
+  }
+
+  int fd_flags;
+  // Set close_fd to be closed after spawn.
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD));
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD,
+                                        fd_flags | FD_CLOEXEC));
+  struct inheritance inherit = {0};
+  // spawn is a system call.
+  child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron());
+  // Restores the current working directory.
+  GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd));
+
+#  else   // GTEST_OS_QNX
+#   if GTEST_OS_LINUX
+  // When a SIGPROF signal is received while fork() or clone() are executing,
+  // the process may hang. To avoid this, we ignore SIGPROF here and re-enable
+  // it after the call to fork()/clone() is complete.
+  struct sigaction saved_sigprof_action;
+  struct sigaction ignore_sigprof_action;
+  memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action));
+  sigemptyset(&ignore_sigprof_action.sa_mask);
+  ignore_sigprof_action.sa_handler = SIG_IGN;
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction(
+      SIGPROF, &ignore_sigprof_action, &saved_sigprof_action));
+#   endif  // GTEST_OS_LINUX
+
+#   if GTEST_HAS_CLONE
+  const bool use_fork = GTEST_FLAG(death_test_use_fork);
+
+  if (!use_fork) {
+    static const bool stack_grows_down = StackGrowsDown();
+    const size_t stack_size = getpagesize();
+    // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead.
+    void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
+                             MAP_ANON | MAP_PRIVATE, -1, 0);
+    GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED);
+
+    // Maximum stack alignment in bytes:  For a downward-growing stack, this
+    // amount is subtracted from size of the stack space to get an address
+    // that is within the stack space and is aligned on all systems we care
+    // about.  As far as I know there is no ABI with stack alignment greater
+    // than 64.  We assume stack and stack_size already have alignment of
+    // kMaxStackAlignment.
+    const size_t kMaxStackAlignment = 64;
+    void* const stack_top =
+        static_cast<char*>(stack) +
+            (stack_grows_down ? stack_size - kMaxStackAlignment : 0);
+    GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment &&
+        reinterpret_cast<intptr_t>(stack_top) % kMaxStackAlignment == 0);
+
+    child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args);
+
+    GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1);
+  }
+#   else
+  const bool use_fork = true;
+#   endif  // GTEST_HAS_CLONE
+
+  if (use_fork && (child_pid = fork()) == 0) {
+      ExecDeathTestChildMain(&args);
+      _exit(0);
+  }
+#  endif  // GTEST_OS_QNX
+#  if GTEST_OS_LINUX
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(
+      sigaction(SIGPROF, &saved_sigprof_action, NULL));
+#  endif  // GTEST_OS_LINUX
+
+  GTEST_DEATH_TEST_CHECK_(child_pid != -1);
+  return child_pid;
+}
+
+// The AssumeRole process for a fork-and-exec death test.  It re-executes the
+// main program from the beginning, setting the --gtest_filter
+// and --gtest_internal_run_death_test flags to cause only the current
+// death test to be re-run.
+DeathTest::TestRole ExecDeathTest::AssumeRole() {
+  const UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const TestInfo* const info = impl->current_test_info();
+  const int death_test_index = info->result()->death_test_count();
+
+  if (flag != NULL) {
+    set_write_fd(flag->write_fd());
+    return EXECUTE_TEST;
+  }
+
+  int pipe_fd[2];
+  GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1);
+  // Clear the close-on-exec flag on the write end of the pipe, lest
+  // it be closed when the child process does an exec:
+  GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1);
+
+  const std::string filter_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "="
+      + info->test_case_name() + "." + info->name();
+  const std::string internal_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "="
+      + file_ + "|" + StreamableToString(line_) + "|"
+      + StreamableToString(death_test_index) + "|"
+      + StreamableToString(pipe_fd[1]);
+  Arguments args;
+  args.AddArguments(GetArgvsForDeathTestChildProcess());
+  args.AddArgument(filter_flag.c_str());
+  args.AddArgument(internal_flag.c_str());
+
+  DeathTest::set_last_death_test_message("");
+
+  CaptureStderr();
+  // See the comment in NoExecDeathTest::AssumeRole for why the next line
+  // is necessary.
+  FlushInfoLog();
+
+  const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1]));
+  set_child_pid(child_pid);
+  set_read_fd(pipe_fd[0]);
+  set_spawned(true);
+  return OVERSEE_TEST;
+}
+
+# endif  // !GTEST_OS_WINDOWS
+
+// Creates a concrete DeathTest-derived class that depends on the
+// --gtest_death_test_style flag, and sets the pointer pointed to
+// by the "test" argument to its address.  If the test should be
+// skipped, sets that pointer to NULL.  Returns true, unless the
+// flag is set to an invalid value.
+bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex,
+                                     const char* file, int line,
+                                     DeathTest** test) {
+  UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const int death_test_index = impl->current_test_info()
+      ->increment_death_test_count();
+
+  if (flag != NULL) {
+    if (death_test_index > flag->index()) {
+      DeathTest::set_last_death_test_message(
+          "Death test count (" + StreamableToString(death_test_index)
+          + ") somehow exceeded expected maximum ("
+          + StreamableToString(flag->index()) + ")");
+      return false;
+    }
+
+    if (!(flag->file() == file && flag->line() == line &&
+          flag->index() == death_test_index)) {
+      *test = NULL;
+      return true;
+    }
+  }
+
+# if GTEST_OS_WINDOWS
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe" ||
+      GTEST_FLAG(death_test_style) == "fast") {
+    *test = new WindowsDeathTest(statement, regex, file, line);
+  }
+
+# else
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe") {
+    *test = new ExecDeathTest(statement, regex, file, line);
+  } else if (GTEST_FLAG(death_test_style) == "fast") {
+    *test = new NoExecDeathTest(statement, regex);
+  }
+
+# endif  // GTEST_OS_WINDOWS
+
+  else {  // NOLINT - this is more readable than unbalanced brackets inside #if.
+    DeathTest::set_last_death_test_message(
+        "Unknown death test style \"" + GTEST_FLAG(death_test_style)
+        + "\" encountered");
+    return false;
+  }
+
+  return true;
+}
+
+# if GTEST_OS_WINDOWS
+// Recreates the pipe and event handles from the provided parameters,
+// signals the event, and returns a file descriptor wrapped around the pipe
+// handle. This function is called in the child process only.
+int GetStatusFileDescriptor(unsigned int parent_process_id,
+                            size_t write_handle_as_size_t,
+                            size_t event_handle_as_size_t) {
+  AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE,
+                                                   FALSE,  // Non-inheritable.
+                                                   parent_process_id));
+  if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) {
+    DeathTestAbort("Unable to open parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  // TODO(vladl@google.com): Replace the following check with a
+  // compile-time assertion when available.
+  GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t));
+
+  const HANDLE write_handle =
+      reinterpret_cast<HANDLE>(write_handle_as_size_t);
+  HANDLE dup_write_handle;
+
+  // The newly initialized handle is accessible only in in the parent
+  // process. To obtain one accessible within the child, we need to use
+  // DuplicateHandle.
+  if (!::DuplicateHandle(parent_process_handle.Get(), write_handle,
+                         ::GetCurrentProcess(), &dup_write_handle,
+                         0x0,    // Requested privileges ignored since
+                                 // DUPLICATE_SAME_ACCESS is used.
+                         FALSE,  // Request non-inheritable handler.
+                         DUPLICATE_SAME_ACCESS)) {
+    DeathTestAbort("Unable to duplicate the pipe handle " +
+                   StreamableToString(write_handle_as_size_t) +
+                   " from the parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  const HANDLE event_handle = reinterpret_cast<HANDLE>(event_handle_as_size_t);
+  HANDLE dup_event_handle;
+
+  if (!::DuplicateHandle(parent_process_handle.Get(), event_handle,
+                         ::GetCurrentProcess(), &dup_event_handle,
+                         0x0,
+                         FALSE,
+                         DUPLICATE_SAME_ACCESS)) {
+    DeathTestAbort("Unable to duplicate the event handle " +
+                   StreamableToString(event_handle_as_size_t) +
+                   " from the parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  const int write_fd =
+      ::_open_osfhandle(reinterpret_cast<intptr_t>(dup_write_handle), O_APPEND);
+  if (write_fd == -1) {
+    DeathTestAbort("Unable to convert pipe handle " +
+                   StreamableToString(write_handle_as_size_t) +
+                   " to a file descriptor");
+  }
+
+  // Signals the parent that the write end of the pipe has been acquired
+  // so the parent can release its own write end.
+  ::SetEvent(dup_event_handle);
+
+  return write_fd;
+}
+# endif  // GTEST_OS_WINDOWS
+
+// Returns a newly created InternalRunDeathTestFlag object with fields
+// initialized from the GTEST_FLAG(internal_run_death_test) flag if
+// the flag is specified; otherwise returns NULL.
+InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() {
+  if (GTEST_FLAG(internal_run_death_test) == "") return NULL;
+
+  // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we
+  // can use it here.
+  int line = -1;
+  int index = -1;
+  ::std::vector< ::std::string> fields;
+  SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields);
+  int write_fd = -1;
+
+# if GTEST_OS_WINDOWS
+
+  unsigned int parent_process_id = 0;
+  size_t write_handle_as_size_t = 0;
+  size_t event_handle_as_size_t = 0;
+
+  if (fields.size() != 6
+      || !ParseNaturalNumber(fields[1], &line)
+      || !ParseNaturalNumber(fields[2], &index)
+      || !ParseNaturalNumber(fields[3], &parent_process_id)
+      || !ParseNaturalNumber(fields[4], &write_handle_as_size_t)
+      || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) {
+    DeathTestAbort("Bad --gtest_internal_run_death_test flag: " +
+                   GTEST_FLAG(internal_run_death_test));
+  }
+  write_fd = GetStatusFileDescriptor(parent_process_id,
+                                     write_handle_as_size_t,
+                                     event_handle_as_size_t);
+# else
+
+  if (fields.size() != 4
+      || !ParseNaturalNumber(fields[1], &line)
+      || !ParseNaturalNumber(fields[2], &index)
+      || !ParseNaturalNumber(fields[3], &write_fd)) {
+    DeathTestAbort("Bad --gtest_internal_run_death_test flag: "
+        + GTEST_FLAG(internal_run_death_test));
+  }
+
+# endif  // GTEST_OS_WINDOWS
+
+  return new InternalRunDeathTestFlag(fields[0], line, index, write_fd);
+}
+
+}  // namespace internal
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: keith.ray@gmail.com (Keith Ray)
+
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-filepath.h"
+#include "gtest/internal/gtest-port.h"
+
+#include <stdlib.h>
+
+#if GTEST_OS_WINDOWS_MOBILE
+# include <windows.h>
+#elif GTEST_OS_WINDOWS
+# include <direct.h>
+# include <io.h>
+#elif GTEST_OS_SYMBIAN
+// Symbian OpenC has PATH_MAX in sys/syslimits.h
+# include <sys/syslimits.h>
+#else
+# include <limits.h>
+# include <climits>  // Some Linux distributions define PATH_MAX here.
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+#if GTEST_OS_WINDOWS
+# define GTEST_PATH_MAX_ _MAX_PATH
+#elif defined(PATH_MAX)
+# define GTEST_PATH_MAX_ PATH_MAX
+#elif defined(_XOPEN_PATH_MAX)
+# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX
+#else
+# define GTEST_PATH_MAX_ _POSIX_PATH_MAX
+#endif  // GTEST_OS_WINDOWS
+
+#include "gtest/internal/gtest-string.h"
+
+namespace testing {
+namespace internal {
+
+#if GTEST_OS_WINDOWS
+// On Windows, '\\' is the standard path separator, but many tools and the
+// Windows API also accept '/' as an alternate path separator. Unless otherwise
+// noted, a file path can contain either kind of path separators, or a mixture
+// of them.
+const char kPathSeparator = '\\';
+const char kAlternatePathSeparator = '/';
+const char kAlternatePathSeparatorString[] = "/";
+# if GTEST_OS_WINDOWS_MOBILE
+// Windows CE doesn't have a current directory. You should not use
+// the current directory in tests on Windows CE, but this at least
+// provides a reasonable fallback.
+const char kCurrentDirectoryString[] = "\\";
+// Windows CE doesn't define INVALID_FILE_ATTRIBUTES
+const DWORD kInvalidFileAttributes = 0xffffffff;
+# else
+const char kCurrentDirectoryString[] = ".\\";
+# endif  // GTEST_OS_WINDOWS_MOBILE
+#else
+const char kPathSeparator = '/';
+const char kCurrentDirectoryString[] = "./";
+#endif  // GTEST_OS_WINDOWS
+
+// Returns whether the given character is a valid path separator.
+static bool IsPathSeparator(char c) {
+#if GTEST_HAS_ALT_PATH_SEP_
+  return (c == kPathSeparator) || (c == kAlternatePathSeparator);
+#else
+  return c == kPathSeparator;
+#endif
+}
+
+// Returns the current working directory, or "" if unsuccessful.
+FilePath FilePath::GetCurrentDir() {
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
+  // Windows CE doesn't have a current directory, so we just return
+  // something reasonable.
+  return FilePath(kCurrentDirectoryString);
+#elif GTEST_OS_WINDOWS
+  char cwd[GTEST_PATH_MAX_ + 1] = { '\0' };
+  return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd);
+#else
+  char cwd[GTEST_PATH_MAX_ + 1] = { '\0' };
+  char* result = getcwd(cwd, sizeof(cwd));
+# if GTEST_OS_NACL
+  // getcwd will likely fail in NaCl due to the sandbox, so return something
+  // reasonable. The user may have provided a shim implementation for getcwd,
+  // however, so fallback only when failure is detected.
+  return FilePath(result == NULL ? kCurrentDirectoryString : cwd);
+# endif  // GTEST_OS_NACL
+  return FilePath(result == NULL ? "" : cwd);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+// Returns a copy of the FilePath with the case-insensitive extension removed.
+// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns
+// FilePath("dir/file"). If a case-insensitive extension is not
+// found, returns a copy of the original FilePath.
+FilePath FilePath::RemoveExtension(const char* extension) const {
+  const std::string dot_extension = std::string(".") + extension;
+  if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) {
+    return FilePath(pathname_.substr(
+        0, pathname_.length() - dot_extension.length()));
+  }
+  return *this;
+}
+
+// Returns a pointer to the last occurence of a valid path separator in
+// the FilePath. On Windows, for example, both '/' and '\' are valid path
+// separators. Returns NULL if no path separator was found.
+const char* FilePath::FindLastPathSeparator() const {
+  const char* const last_sep = strrchr(c_str(), kPathSeparator);
+#if GTEST_HAS_ALT_PATH_SEP_
+  const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator);
+  // Comparing two pointers of which only one is NULL is undefined.
+  if (last_alt_sep != NULL &&
+      (last_sep == NULL || last_alt_sep > last_sep)) {
+    return last_alt_sep;
+  }
+#endif
+  return last_sep;
+}
+
+// Returns a copy of the FilePath with the directory part removed.
+// Example: FilePath("path/to/file").RemoveDirectoryName() returns
+// FilePath("file"). If there is no directory part ("just_a_file"), it returns
+// the FilePath unmodified. If there is no file part ("just_a_dir/") it
+// returns an empty FilePath ("").
+// On Windows platform, '\' is the path separator, otherwise it is '/'.
+FilePath FilePath::RemoveDirectoryName() const {
+  const char* const last_sep = FindLastPathSeparator();
+  return last_sep ? FilePath(last_sep + 1) : *this;
+}
+
+// RemoveFileName returns the directory path with the filename removed.
+// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/".
+// If the FilePath is "a_file" or "/a_file", RemoveFileName returns
+// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does
+// not have a file, like "just/a/dir/", it returns the FilePath unmodified.
+// On Windows platform, '\' is the path separator, otherwise it is '/'.
+FilePath FilePath::RemoveFileName() const {
+  const char* const last_sep = FindLastPathSeparator();
+  std::string dir;
+  if (last_sep) {
+    dir = std::string(c_str(), last_sep + 1 - c_str());
+  } else {
+    dir = kCurrentDirectoryString;
+  }
+  return FilePath(dir);
+}
+
+// Helper functions for naming files in a directory for xml output.
+
+// Given directory = "dir", base_name = "test", number = 0,
+// extension = "xml", returns "dir/test.xml". If number is greater
+// than zero (e.g., 12), returns "dir/test_12.xml".
+// On Windows platform, uses \ as the separator rather than /.
+FilePath FilePath::MakeFileName(const FilePath& directory,
+                                const FilePath& base_name,
+                                int number,
+                                const char* extension) {
+  std::string file;
+  if (number == 0) {
+    file = base_name.string() + "." + extension;
+  } else {
+    file = base_name.string() + "_" + StreamableToString(number)
+        + "." + extension;
+  }
+  return ConcatPaths(directory, FilePath(file));
+}
+
+// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml".
+// On Windows, uses \ as the separator rather than /.
+FilePath FilePath::ConcatPaths(const FilePath& directory,
+                               const FilePath& relative_path) {
+  if (directory.IsEmpty())
+    return relative_path;
+  const FilePath dir(directory.RemoveTrailingPathSeparator());
+  return FilePath(dir.string() + kPathSeparator + relative_path.string());
+}
+
+// Returns true if pathname describes something findable in the file-system,
+// either a file, directory, or whatever.
+bool FilePath::FileOrDirectoryExists() const {
+#if GTEST_OS_WINDOWS_MOBILE
+  LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str());
+  const DWORD attributes = GetFileAttributes(unicode);
+  delete [] unicode;
+  return attributes != kInvalidFileAttributes;
+#else
+  posix::StatStruct file_stat;
+  return posix::Stat(pathname_.c_str(), &file_stat) == 0;
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+// Returns true if pathname describes a directory in the file-system
+// that exists.
+bool FilePath::DirectoryExists() const {
+  bool result = false;
+#if GTEST_OS_WINDOWS
+  // Don't strip off trailing separator if path is a root directory on
+  // Windows (like "C:\\").
+  const FilePath& path(IsRootDirectory() ? *this :
+                                           RemoveTrailingPathSeparator());
+#else
+  const FilePath& path(*this);
+#endif
+
+#if GTEST_OS_WINDOWS_MOBILE
+  LPCWSTR unicode = String::AnsiToUtf16(path.c_str());
+  const DWORD attributes = GetFileAttributes(unicode);
+  delete [] unicode;
+  if ((attributes != kInvalidFileAttributes) &&
+      (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+    result = true;
+  }
+#else
+  posix::StatStruct file_stat;
+  result = posix::Stat(path.c_str(), &file_stat) == 0 &&
+      posix::IsDir(file_stat);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+  return result;
+}
+
+// Returns true if pathname describes a root directory. (Windows has one
+// root directory per disk drive.)
+bool FilePath::IsRootDirectory() const {
+#if GTEST_OS_WINDOWS
+  // TODO(wan@google.com): on Windows a network share like
+  // \\server\share can be a root directory, although it cannot be the
+  // current directory.  Handle this properly.
+  return pathname_.length() == 3 && IsAbsolutePath();
+#else
+  return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]);
+#endif
+}
+
+// Returns true if pathname describes an absolute path.
+bool FilePath::IsAbsolutePath() const {
+  const char* const name = pathname_.c_str();
+#if GTEST_OS_WINDOWS
+  return pathname_.length() >= 3 &&
+     ((name[0] >= 'a' && name[0] <= 'z') ||
+      (name[0] >= 'A' && name[0] <= 'Z')) &&
+     name[1] == ':' &&
+     IsPathSeparator(name[2]);
+#else
+  return IsPathSeparator(name[0]);
+#endif
+}
+
+// Returns a pathname for a file that does not currently exist. The pathname
+// will be directory/base_name.extension or
+// directory/base_name_<number>.extension if directory/base_name.extension
+// already exists. The number will be incremented until a pathname is found
+// that does not already exist.
+// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'.
+// There could be a race condition if two or more processes are calling this
+// function at the same time -- they could both pick the same filename.
+FilePath FilePath::GenerateUniqueFileName(const FilePath& directory,
+                                          const FilePath& base_name,
+                                          const char* extension) {
+  FilePath full_pathname;
+  int number = 0;
+  do {
+    full_pathname.Set(MakeFileName(directory, base_name, number++, extension));
+  } while (full_pathname.FileOrDirectoryExists());
+  return full_pathname;
+}
+
+// Returns true if FilePath ends with a path separator, which indicates that
+// it is intended to represent a directory. Returns false otherwise.
+// This does NOT check that a directory (or file) actually exists.
+bool FilePath::IsDirectory() const {
+  return !pathname_.empty() &&
+         IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]);
+}
+
+// Create directories so that path exists. Returns true if successful or if
+// the directories already exist; returns false if unable to create directories
+// for any reason.
+bool FilePath::CreateDirectoriesRecursively() const {
+  if (!this->IsDirectory()) {
+    return false;
+  }
+
+  if (pathname_.length() == 0 || this->DirectoryExists()) {
+    return true;
+  }
+
+  const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName());
+  return parent.CreateDirectoriesRecursively() && this->CreateFolder();
+}
+
+// Create the directory so that path exists. Returns true if successful or
+// if the directory already exists; returns false if unable to create the
+// directory for any reason, including if the parent directory does not
+// exist. Not named "CreateDirectory" because that's a macro on Windows.
+bool FilePath::CreateFolder() const {
+#if GTEST_OS_WINDOWS_MOBILE
+  FilePath removed_sep(this->RemoveTrailingPathSeparator());
+  LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str());
+  int result = CreateDirectory(unicode, NULL) ? 0 : -1;
+  delete [] unicode;
+#elif GTEST_OS_WINDOWS
+  int result = _mkdir(pathname_.c_str());
+#else
+  int result = mkdir(pathname_.c_str(), 0777);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+  if (result == -1) {
+    return this->DirectoryExists();  // An error is OK if the directory exists.
+  }
+  return true;  // No error.
+}
+
+// If input name has a trailing separator character, remove it and return the
+// name, otherwise return the name string unmodified.
+// On Windows platform, uses \ as the separator, other platforms use /.
+FilePath FilePath::RemoveTrailingPathSeparator() const {
+  return IsDirectory()
+      ? FilePath(pathname_.substr(0, pathname_.length() - 1))
+      : *this;
+}
+
+// Removes any redundant separators that might be in the pathname.
+// For example, "bar///foo" becomes "bar/foo". Does not eliminate other
+// redundancies that might be in a pathname involving "." or "..".
+// TODO(wan@google.com): handle Windows network shares (e.g. \\server\share).
+void FilePath::Normalize() {
+  if (pathname_.c_str() == NULL) {
+    pathname_ = "";
+    return;
+  }
+  const char* src = pathname_.c_str();
+  char* const dest = new char[pathname_.length() + 1];
+  char* dest_ptr = dest;
+  memset(dest_ptr, 0, pathname_.length() + 1);
+
+  while (*src != '\0') {
+    *dest_ptr = *src;
+    if (!IsPathSeparator(*src)) {
+      src++;
+    } else {
+#if GTEST_HAS_ALT_PATH_SEP_
+      if (*dest_ptr == kAlternatePathSeparator) {
+        *dest_ptr = kPathSeparator;
+      }
+#endif
+      while (IsPathSeparator(*src))
+        src++;
+    }
+    dest_ptr++;
+  }
+  *dest_ptr = '\0';
+  pathname_ = dest;
+  delete[] dest;
+}
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/internal/gtest-port.h"
+
+#include <limits.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fstream>
+
+#if GTEST_OS_WINDOWS
+# include <windows.h>
+# include <io.h>
+# include <sys/stat.h>
+# include <map>  // Used in ThreadLocal.
+#else
+# include <unistd.h>
+#endif  // GTEST_OS_WINDOWS
+
+#if GTEST_OS_MAC
+# include <mach/mach_init.h>
+# include <mach/task.h>
+# include <mach/vm_map.h>
+#endif  // GTEST_OS_MAC
+
+#if GTEST_OS_QNX
+# include <devctl.h>
+# include <fcntl.h>
+# include <sys/procfs.h>
+#endif  // GTEST_OS_QNX
+
+#if GTEST_OS_AIX
+# include <procinfo.h>
+# include <sys/types.h>
+#endif  // GTEST_OS_AIX
+
+#include "gtest/gtest-spi.h"
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-internal.h"
+#include "gtest/internal/gtest-string.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+namespace internal {
+
+#if defined(_MSC_VER) || defined(__BORLANDC__)
+// MSVC and C++Builder do not provide a definition of STDERR_FILENO.
+const int kStdOutFileno = 1;
+const int kStdErrFileno = 2;
+#else
+const int kStdOutFileno = STDOUT_FILENO;
+const int kStdErrFileno = STDERR_FILENO;
+#endif  // _MSC_VER
+
+#if GTEST_OS_LINUX
+
+namespace {
+template <typename T>
+T ReadProcFileField(const string& filename, int field) {
+  std::string dummy;
+  std::ifstream file(filename.c_str());
+  while (field-- > 0) {
+    file >> dummy;
+  }
+  T output = 0;
+  file >> output;
+  return output;
+}
+}  // namespace
+
+// Returns the number of active threads, or 0 when there is an error.
+size_t GetThreadCount() {
+  const string filename =
+      (Message() << "/proc/" << getpid() << "/stat").GetString();
+  return ReadProcFileField<int>(filename, 19);
+}
+
+#elif GTEST_OS_MAC
+
+size_t GetThreadCount() {
+  const task_t task = mach_task_self();
+  mach_msg_type_number_t thread_count;
+  thread_act_array_t thread_list;
+  const kern_return_t status = task_threads(task, &thread_list, &thread_count);
+  if (status == KERN_SUCCESS) {
+    // task_threads allocates resources in thread_list and we need to free them
+    // to avoid leaks.
+    vm_deallocate(task,
+                  reinterpret_cast<vm_address_t>(thread_list),
+                  sizeof(thread_t) * thread_count);
+    return static_cast<size_t>(thread_count);
+  } else {
+    return 0;
+  }
+}
+
+#elif GTEST_OS_QNX
+
+// Returns the number of threads running in the process, or 0 to indicate that
+// we cannot detect it.
+size_t GetThreadCount() {
+  const int fd = open("/proc/self/as", O_RDONLY);
+  if (fd < 0) {
+    return 0;
+  }
+  procfs_info process_info;
+  const int status =
+      devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL);
+  close(fd);
+  if (status == EOK) {
+    return static_cast<size_t>(process_info.num_threads);
+  } else {
+    return 0;
+  }
+}
+
+#elif GTEST_OS_AIX
+
+size_t GetThreadCount() {
+  struct procentry64 entry;
+  pid_t pid = getpid();
+  int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1);
+  if (status == 1) {
+    return entry.pi_thcount;
+  } else {
+    return 0;
+  }
+}
+
+#else
+
+size_t GetThreadCount() {
+  // There's no portable way to detect the number of threads, so we just
+  // return 0 to indicate that we cannot detect it.
+  return 0;
+}
+
+#endif  // GTEST_OS_LINUX
+
+#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
+
+void SleepMilliseconds(int n) {
+  ::Sleep(n);
+}
+
+AutoHandle::AutoHandle()
+    : handle_(INVALID_HANDLE_VALUE) {}
+
+AutoHandle::AutoHandle(Handle handle)
+    : handle_(handle) {}
+
+AutoHandle::~AutoHandle() {
+  Reset();
+}
+
+AutoHandle::Handle AutoHandle::Get() const {
+  return handle_;
+}
+
+void AutoHandle::Reset() {
+  Reset(INVALID_HANDLE_VALUE);
+}
+
+void AutoHandle::Reset(HANDLE handle) {
+  // Resetting with the same handle we already own is invalid.
+  if (handle_ != handle) {
+    if (IsCloseable()) {
+      ::CloseHandle(handle_);
+    }
+    handle_ = handle;
+  } else {
+    GTEST_CHECK_(!IsCloseable())
+        << "Resetting a valid handle to itself is likely a programmer error "
+            "and thus not allowed.";
+  }
+}
+
+bool AutoHandle::IsCloseable() const {
+  // Different Windows APIs may use either of these values to represent an
+  // invalid handle.
+  return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE;
+}
+
+Notification::Notification()
+    : event_(::CreateEvent(NULL,   // Default security attributes.
+                           TRUE,   // Do not reset automatically.
+                           FALSE,  // Initially unset.
+                           NULL)) {  // Anonymous event.
+  GTEST_CHECK_(event_.Get() != NULL);
+}
+
+void Notification::Notify() {
+  GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE);
+}
+
+void Notification::WaitForNotification() {
+  GTEST_CHECK_(
+      ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0);
+}
+
+Mutex::Mutex()
+    : owner_thread_id_(0),
+      type_(kDynamic),
+      critical_section_init_phase_(0),
+      critical_section_(new CRITICAL_SECTION) {
+  ::InitializeCriticalSection(critical_section_);
+}
+
+Mutex::~Mutex() {
+  // Static mutexes are leaked intentionally. It is not thread-safe to try
+  // to clean them up.
+  // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires
+  // nothing to clean it up but is available only on Vista and later.
+  // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx
+  if (type_ == kDynamic) {
+    ::DeleteCriticalSection(critical_section_);
+    delete critical_section_;
+    critical_section_ = NULL;
+  }
+}
+
+void Mutex::Lock() {
+  ThreadSafeLazyInit();
+  ::EnterCriticalSection(critical_section_);
+  owner_thread_id_ = ::GetCurrentThreadId();
+}
+
+void Mutex::Unlock() {
+  ThreadSafeLazyInit();
+  // We don't protect writing to owner_thread_id_ here, as it's the
+  // caller's responsibility to ensure that the current thread holds the
+  // mutex when this is called.
+  owner_thread_id_ = 0;
+  ::LeaveCriticalSection(critical_section_);
+}
+
+// Does nothing if the current thread holds the mutex. Otherwise, crashes
+// with high probability.
+void Mutex::AssertHeld() {
+  ThreadSafeLazyInit();
+  GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId())
+      << "The current thread is not holding the mutex @" << this;
+}
+
+// Initializes owner_thread_id_ and critical_section_ in static mutexes.
+void Mutex::ThreadSafeLazyInit() {
+  // Dynamic mutexes are initialized in the constructor.
+  if (type_ == kStatic) {
+    switch (
+        ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) {
+      case 0:
+        // If critical_section_init_phase_ was 0 before the exchange, we
+        // are the first to test it and need to perform the initialization.
+        owner_thread_id_ = 0;
+        critical_section_ = new CRITICAL_SECTION;
+        ::InitializeCriticalSection(critical_section_);
+        // Updates the critical_section_init_phase_ to 2 to signal
+        // initialization complete.
+        GTEST_CHECK_(::InterlockedCompareExchange(
+                          &critical_section_init_phase_, 2L, 1L) ==
+                      1L);
+        break;
+      case 1:
+        // Somebody else is already initializing the mutex; spin until they
+        // are done.
+        while (::InterlockedCompareExchange(&critical_section_init_phase_,
+                                            2L,
+                                            2L) != 2L) {
+          // Possibly yields the rest of the thread's time slice to other
+          // threads.
+          ::Sleep(0);
+        }
+        break;
+
+      case 2:
+        break;  // The mutex is already initialized and ready for use.
+
+      default:
+        GTEST_CHECK_(false)
+            << "Unexpected value of critical_section_init_phase_ "
+            << "while initializing a static mutex.";
+    }
+  }
+}
+
+namespace {
+
+class ThreadWithParamSupport : public ThreadWithParamBase {
+ public:
+  static HANDLE CreateThread(Runnable* runnable,
+                             Notification* thread_can_start) {
+    ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start);
+    DWORD thread_id;
+    // TODO(yukawa): Consider to use _beginthreadex instead.
+    HANDLE thread_handle = ::CreateThread(
+        NULL,    // Default security.
+        0,       // Default stack size.
+        &ThreadWithParamSupport::ThreadMain,
+        param,   // Parameter to ThreadMainStatic
+        0x0,     // Default creation flags.
+        &thread_id);  // Need a valid pointer for the call to work under Win98.
+    GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error "
+                                        << ::GetLastError() << ".";
+    if (thread_handle == NULL) {
+      delete param;
+    }
+    return thread_handle;
+  }
+
+ private:
+  struct ThreadMainParam {
+    ThreadMainParam(Runnable* runnable, Notification* thread_can_start)
+        : runnable_(runnable),
+          thread_can_start_(thread_can_start) {
+    }
+    scoped_ptr<Runnable> runnable_;
+    // Does not own.
+    Notification* thread_can_start_;
+  };
+
+  static DWORD WINAPI ThreadMain(void* ptr) {
+    // Transfers ownership.
+    scoped_ptr<ThreadMainParam> param(static_cast<ThreadMainParam*>(ptr));
+    if (param->thread_can_start_ != NULL)
+      param->thread_can_start_->WaitForNotification();
+    param->runnable_->Run();
+    return 0;
+  }
+
+  // Prohibit instantiation.
+  ThreadWithParamSupport();
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport);
+};
+
+}  // namespace
+
+ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable,
+                                         Notification* thread_can_start)
+      : thread_(ThreadWithParamSupport::CreateThread(runnable,
+                                                     thread_can_start)) {
+}
+
+ThreadWithParamBase::~ThreadWithParamBase() {
+  Join();
+}
+
+void ThreadWithParamBase::Join() {
+  GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0)
+      << "Failed to join the thread with error " << ::GetLastError() << ".";
+}
+
+// Maps a thread to a set of ThreadIdToThreadLocals that have values
+// instantiated on that thread and notifies them when the thread exits.  A
+// ThreadLocal instance is expected to persist until all threads it has
+// values on have terminated.
+class ThreadLocalRegistryImpl {
+ public:
+  // Registers thread_local_instance as having value on the current thread.
+  // Returns a value that can be used to identify the thread from other threads.
+  static ThreadLocalValueHolderBase* GetValueOnCurrentThread(
+      const ThreadLocalBase* thread_local_instance) {
+    DWORD current_thread = ::GetCurrentThreadId();
+    MutexLock lock(&mutex_);
+    ThreadIdToThreadLocals* const thread_to_thread_locals =
+        GetThreadLocalsMapLocked();
+    ThreadIdToThreadLocals::iterator thread_local_pos =
+        thread_to_thread_locals->find(current_thread);
+    if (thread_local_pos == thread_to_thread_locals->end()) {
+      thread_local_pos = thread_to_thread_locals->insert(
+          std::make_pair(current_thread, ThreadLocalValues())).first;
+      StartWatcherThreadFor(current_thread);
+    }
+    ThreadLocalValues& thread_local_values = thread_local_pos->second;
+    ThreadLocalValues::iterator value_pos =
+        thread_local_values.find(thread_local_instance);
+    if (value_pos == thread_local_values.end()) {
+      value_pos =
+          thread_local_values
+              .insert(std::make_pair(
+                  thread_local_instance,
+                  linked_ptr<ThreadLocalValueHolderBase>(
+                      thread_local_instance->NewValueForCurrentThread())))
+              .first;
+    }
+    return value_pos->second.get();
+  }
+
+  static void OnThreadLocalDestroyed(
+      const ThreadLocalBase* thread_local_instance) {
+    std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+    // Clean up the ThreadLocalValues data structure while holding the lock, but
+    // defer the destruction of the ThreadLocalValueHolderBases.
+    {
+      MutexLock lock(&mutex_);
+      ThreadIdToThreadLocals* const thread_to_thread_locals =
+          GetThreadLocalsMapLocked();
+      for (ThreadIdToThreadLocals::iterator it =
+          thread_to_thread_locals->begin();
+          it != thread_to_thread_locals->end();
+          ++it) {
+        ThreadLocalValues& thread_local_values = it->second;
+        ThreadLocalValues::iterator value_pos =
+            thread_local_values.find(thread_local_instance);
+        if (value_pos != thread_local_values.end()) {
+          value_holders.push_back(value_pos->second);
+          thread_local_values.erase(value_pos);
+          // This 'if' can only be successful at most once, so theoretically we
+          // could break out of the loop here, but we don't bother doing so.
+        }
+      }
+    }
+    // Outside the lock, let the destructor for 'value_holders' deallocate the
+    // ThreadLocalValueHolderBases.
+  }
+
+  static void OnThreadExit(DWORD thread_id) {
+    GTEST_CHECK_(thread_id != 0) << ::GetLastError();
+    std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+    // Clean up the ThreadIdToThreadLocals data structure while holding the
+    // lock, but defer the destruction of the ThreadLocalValueHolderBases.
+    {
+      MutexLock lock(&mutex_);
+      ThreadIdToThreadLocals* const thread_to_thread_locals =
+          GetThreadLocalsMapLocked();
+      ThreadIdToThreadLocals::iterator thread_local_pos =
+          thread_to_thread_locals->find(thread_id);
+      if (thread_local_pos != thread_to_thread_locals->end()) {
+        ThreadLocalValues& thread_local_values = thread_local_pos->second;
+        for (ThreadLocalValues::iterator value_pos =
+            thread_local_values.begin();
+            value_pos != thread_local_values.end();
+            ++value_pos) {
+          value_holders.push_back(value_pos->second);
+        }
+        thread_to_thread_locals->erase(thread_local_pos);
+      }
+    }
+    // Outside the lock, let the destructor for 'value_holders' deallocate the
+    // ThreadLocalValueHolderBases.
+  }
+
+ private:
+  // In a particular thread, maps a ThreadLocal object to its value.
+  typedef std::map<const ThreadLocalBase*,
+                   linked_ptr<ThreadLocalValueHolderBase> > ThreadLocalValues;
+  // Stores all ThreadIdToThreadLocals having values in a thread, indexed by
+  // thread's ID.
+  typedef std::map<DWORD, ThreadLocalValues> ThreadIdToThreadLocals;
+
+  // Holds the thread id and thread handle that we pass from
+  // StartWatcherThreadFor to WatcherThreadFunc.
+  typedef std::pair<DWORD, HANDLE> ThreadIdAndHandle;
+
+  static void StartWatcherThreadFor(DWORD thread_id) {
+    // The returned handle will be kept in thread_map and closed by
+    // watcher_thread in WatcherThreadFunc.
+    HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION,
+                                 FALSE,
+                                 thread_id);
+    GTEST_CHECK_(thread != NULL);
+    // We need to to pass a valid thread ID pointer into CreateThread for it
+    // to work correctly under Win98.
+    DWORD watcher_thread_id;
+    HANDLE watcher_thread = ::CreateThread(
+        NULL,   // Default security.
+        0,      // Default stack size
+        &ThreadLocalRegistryImpl::WatcherThreadFunc,
+        reinterpret_cast<LPVOID>(new ThreadIdAndHandle(thread_id, thread)),
+        CREATE_SUSPENDED,
+        &watcher_thread_id);
+    GTEST_CHECK_(watcher_thread != NULL);
+    // Give the watcher thread the same priority as ours to avoid being
+    // blocked by it.
+    ::SetThreadPriority(watcher_thread,
+                        ::GetThreadPriority(::GetCurrentThread()));
+    ::ResumeThread(watcher_thread);
+    ::CloseHandle(watcher_thread);
+  }
+
+  // Monitors exit from a given thread and notifies those
+  // ThreadIdToThreadLocals about thread termination.
+  static DWORD WINAPI WatcherThreadFunc(LPVOID param) {
+    const ThreadIdAndHandle* tah =
+        reinterpret_cast<const ThreadIdAndHandle*>(param);
+    GTEST_CHECK_(
+        ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0);
+    OnThreadExit(tah->first);
+    ::CloseHandle(tah->second);
+    delete tah;
+    return 0;
+  }
+
+  // Returns map of thread local instances.
+  static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() {
+    mutex_.AssertHeld();
+    static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals;
+    return map;
+  }
+
+  // Protects access to GetThreadLocalsMapLocked() and its return value.
+  static Mutex mutex_;
+  // Protects access to GetThreadMapLocked() and its return value.
+  static Mutex thread_map_mutex_;
+};
+
+Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex);
+Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex);
+
+ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread(
+      const ThreadLocalBase* thread_local_instance) {
+  return ThreadLocalRegistryImpl::GetValueOnCurrentThread(
+      thread_local_instance);
+}
+
+void ThreadLocalRegistry::OnThreadLocalDestroyed(
+      const ThreadLocalBase* thread_local_instance) {
+  ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance);
+}
+
+#endif  // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
+
+#if GTEST_USES_POSIX_RE
+
+// Implements RE.  Currently only needed for death tests.
+
+RE::~RE() {
+  if (is_valid_) {
+    // regfree'ing an invalid regex might crash because the content
+    // of the regex is undefined. Since the regex's are essentially
+    // the same, one cannot be valid (or invalid) without the other
+    // being so too.
+    regfree(&partial_regex_);
+    regfree(&full_regex_);
+  }
+  free(const_cast<char*>(pattern_));
+}
+
+// Returns true iff regular expression re matches the entire str.
+bool RE::FullMatch(const char* str, const RE& re) {
+  if (!re.is_valid_) return false;
+
+  regmatch_t match;
+  return regexec(&re.full_regex_, str, 1, &match, 0) == 0;
+}
+
+// Returns true iff regular expression re matches a substring of str
+// (including str itself).
+bool RE::PartialMatch(const char* str, const RE& re) {
+  if (!re.is_valid_) return false;
+
+  regmatch_t match;
+  return regexec(&re.partial_regex_, str, 1, &match, 0) == 0;
+}
+
+// Initializes an RE from its string representation.
+void RE::Init(const char* regex) {
+  pattern_ = posix::StrDup(regex);
+
+  // Reserves enough bytes to hold the regular expression used for a
+  // full match.
+  const size_t full_regex_len = strlen(regex) + 10;
+  char* const full_pattern = new char[full_regex_len];
+
+  snprintf(full_pattern, full_regex_len, "^(%s)$", regex);
+  is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0;
+  // We want to call regcomp(&partial_regex_, ...) even if the
+  // previous expression returns false.  Otherwise partial_regex_ may
+  // not be properly initialized can may cause trouble when it's
+  // freed.
+  //
+  // Some implementation of POSIX regex (e.g. on at least some
+  // versions of Cygwin) doesn't accept the empty string as a valid
+  // regex.  We change it to an equivalent form "()" to be safe.
+  if (is_valid_) {
+    const char* const partial_regex = (*regex == '\0') ? "()" : regex;
+    is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0;
+  }
+  EXPECT_TRUE(is_valid_)
+      << "Regular expression \"" << regex
+      << "\" is not a valid POSIX Extended regular expression.";
+
+  delete[] full_pattern;
+}
+
+#elif GTEST_USES_SIMPLE_RE
+
+// Returns true iff ch appears anywhere in str (excluding the
+// terminating '\0' character).
+bool IsInSet(char ch, const char* str) {
+  return ch != '\0' && strchr(str, ch) != NULL;
+}
+
+// Returns true iff ch belongs to the given classification.  Unlike
+// similar functions in <ctype.h>, these aren't affected by the
+// current locale.
+bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; }
+bool IsAsciiPunct(char ch) {
+  return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~");
+}
+bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); }
+bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); }
+bool IsAsciiWordChar(char ch) {
+  return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') ||
+      ('0' <= ch && ch <= '9') || ch == '_';
+}
+
+// Returns true iff "\\c" is a supported escape sequence.
+bool IsValidEscape(char c) {
+  return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW"));
+}
+
+// Returns true iff the given atom (specified by escaped and pattern)
+// matches ch.  The result is undefined if the atom is invalid.
+bool AtomMatchesChar(bool escaped, char pattern_char, char ch) {
+  if (escaped) {  // "\\p" where p is pattern_char.
+    switch (pattern_char) {
+      case 'd': return IsAsciiDigit(ch);
+      case 'D': return !IsAsciiDigit(ch);
+      case 'f': return ch == '\f';
+      case 'n': return ch == '\n';
+      case 'r': return ch == '\r';
+      case 's': return IsAsciiWhiteSpace(ch);
+      case 'S': return !IsAsciiWhiteSpace(ch);
+      case 't': return ch == '\t';
+      case 'v': return ch == '\v';
+      case 'w': return IsAsciiWordChar(ch);
+      case 'W': return !IsAsciiWordChar(ch);
+    }
+    return IsAsciiPunct(pattern_char) && pattern_char == ch;
+  }
+
+  return (pattern_char == '.' && ch != '\n') || pattern_char == ch;
+}
+
+// Helper function used by ValidateRegex() to format error messages.
+std::string FormatRegexSyntaxError(const char* regex, int index) {
+  return (Message() << "Syntax error at index " << index
+          << " in simple regular expression \"" << regex << "\": ").GetString();
+}
+
+// Generates non-fatal failures and returns false if regex is invalid;
+// otherwise returns true.
+bool ValidateRegex(const char* regex) {
+  if (regex == NULL) {
+    // TODO(wan@google.com): fix the source file location in the
+    // assertion failures to match where the regex is used in user
+    // code.
+    ADD_FAILURE() << "NULL is not a valid simple regular expression.";
+    return false;
+  }
+
+  bool is_valid = true;
+
+  // True iff ?, *, or + can follow the previous atom.
+  bool prev_repeatable = false;
+  for (int i = 0; regex[i]; i++) {
+    if (regex[i] == '\\') {  // An escape sequence
+      i++;
+      if (regex[i] == '\0') {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1)
+                      << "'\\' cannot appear at the end.";
+        return false;
+      }
+
+      if (!IsValidEscape(regex[i])) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1)
+                      << "invalid escape sequence \"\\" << regex[i] << "\".";
+        is_valid = false;
+      }
+      prev_repeatable = true;
+    } else {  // Not an escape sequence.
+      const char ch = regex[i];
+
+      if (ch == '^' && i > 0) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'^' can only appear at the beginning.";
+        is_valid = false;
+      } else if (ch == '$' && regex[i + 1] != '\0') {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'$' can only appear at the end.";
+        is_valid = false;
+      } else if (IsInSet(ch, "()[]{}|")) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'" << ch << "' is unsupported.";
+        is_valid = false;
+      } else if (IsRepeat(ch) && !prev_repeatable) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'" << ch << "' can only follow a repeatable token.";
+        is_valid = false;
+      }
+
+      prev_repeatable = !IsInSet(ch, "^$?*+");
+    }
+  }
+
+  return is_valid;
+}
+
+// Matches a repeated regex atom followed by a valid simple regular
+// expression.  The regex atom is defined as c if escaped is false,
+// or \c otherwise.  repeat is the repetition meta character (?, *,
+// or +).  The behavior is undefined if str contains too many
+// characters to be indexable by size_t, in which case the test will
+// probably time out anyway.  We are fine with this limitation as
+// std::string has it too.
+bool MatchRepetitionAndRegexAtHead(
+    bool escaped, char c, char repeat, const char* regex,
+    const char* str) {
+  const size_t min_count = (repeat == '+') ? 1 : 0;
+  const size_t max_count = (repeat == '?') ? 1 :
+      static_cast<size_t>(-1) - 1;
+  // We cannot call numeric_limits::max() as it conflicts with the
+  // max() macro on Windows.
+
+  for (size_t i = 0; i <= max_count; ++i) {
+    // We know that the atom matches each of the first i characters in str.
+    if (i >= min_count && MatchRegexAtHead(regex, str + i)) {
+      // We have enough matches at the head, and the tail matches too.
+      // Since we only care about *whether* the pattern matches str
+      // (as opposed to *how* it matches), there is no need to find a
+      // greedy match.
+      return true;
+    }
+    if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i]))
+      return false;
+  }
+  return false;
+}
+
+// Returns true iff regex matches a prefix of str.  regex must be a
+// valid simple regular expression and not start with "^", or the
+// result is undefined.
+bool MatchRegexAtHead(const char* regex, const char* str) {
+  if (*regex == '\0')  // An empty regex matches a prefix of anything.
+    return true;
+
+  // "$" only matches the end of a string.  Note that regex being
+  // valid guarantees that there's nothing after "$" in it.
+  if (*regex == '$')
+    return *str == '\0';
+
+  // Is the first thing in regex an escape sequence?
+  const bool escaped = *regex == '\\';
+  if (escaped)
+    ++regex;
+  if (IsRepeat(regex[1])) {
+    // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so
+    // here's an indirect recursion.  It terminates as the regex gets
+    // shorter in each recursion.
+    return MatchRepetitionAndRegexAtHead(
+        escaped, regex[0], regex[1], regex + 2, str);
+  } else {
+    // regex isn't empty, isn't "$", and doesn't start with a
+    // repetition.  We match the first atom of regex with the first
+    // character of str and recurse.
+    return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) &&
+        MatchRegexAtHead(regex + 1, str + 1);
+  }
+}
+
+// Returns true iff regex matches any substring of str.  regex must be
+// a valid simple regular expression, or the result is undefined.
+//
+// The algorithm is recursive, but the recursion depth doesn't exceed
+// the regex length, so we won't need to worry about running out of
+// stack space normally.  In rare cases the time complexity can be
+// exponential with respect to the regex length + the string length,
+// but usually it's must faster (often close to linear).
+bool MatchRegexAnywhere(const char* regex, const char* str) {
+  if (regex == NULL || str == NULL)
+    return false;
+
+  if (*regex == '^')
+    return MatchRegexAtHead(regex + 1, str);
+
+  // A successful match can be anywhere in str.
+  do {
+    if (MatchRegexAtHead(regex, str))
+      return true;
+  } while (*str++ != '\0');
+  return false;
+}
+
+// Implements the RE class.
+
+RE::~RE() {
+  free(const_cast<char*>(pattern_));
+  free(const_cast<char*>(full_pattern_));
+}
+
+// Returns true iff regular expression re matches the entire str.
+bool RE::FullMatch(const char* str, const RE& re) {
+  return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str);
+}
+
+// Returns true iff regular expression re matches a substring of str
+// (including str itself).
+bool RE::PartialMatch(const char* str, const RE& re) {
+  return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str);
+}
+
+// Initializes an RE from its string representation.
+void RE::Init(const char* regex) {
+  pattern_ = full_pattern_ = NULL;
+  if (regex != NULL) {
+    pattern_ = posix::StrDup(regex);
+  }
+
+  is_valid_ = ValidateRegex(regex);
+  if (!is_valid_) {
+    // No need to calculate the full pattern when the regex is invalid.
+    return;
+  }
+
+  const size_t len = strlen(regex);
+  // Reserves enough bytes to hold the regular expression used for a
+  // full match: we need space to prepend a '^', append a '$', and
+  // terminate the string with '\0'.
+  char* buffer = static_cast<char*>(malloc(len + 3));
+  full_pattern_ = buffer;
+
+  if (*regex != '^')
+    *buffer++ = '^';  // Makes sure full_pattern_ starts with '^'.
+
+  // We don't use snprintf or strncpy, as they trigger a warning when
+  // compiled with VC++ 8.0.
+  memcpy(buffer, regex, len);
+  buffer += len;
+
+  if (len == 0 || regex[len - 1] != '$')
+    *buffer++ = '$';  // Makes sure full_pattern_ ends with '$'.
+
+  *buffer = '\0';
+}
+
+#endif  // GTEST_USES_POSIX_RE
+
+const char kUnknownFile[] = "unknown file";
+
+// Formats a source file path and a line number as they would appear
+// in an error message from the compiler used to compile this code.
+GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) {
+  const std::string file_name(file == NULL ? kUnknownFile : file);
+
+  if (line < 0) {
+    return file_name + ":";
+  }
+#ifdef _MSC_VER
+  return file_name + "(" + StreamableToString(line) + "):";
+#else
+  return file_name + ":" + StreamableToString(line) + ":";
+#endif  // _MSC_VER
+}
+
+// Formats a file location for compiler-independent XML output.
+// Although this function is not platform dependent, we put it next to
+// FormatFileLocation in order to contrast the two functions.
+// Note that FormatCompilerIndependentFileLocation() does NOT append colon
+// to the file location it produces, unlike FormatFileLocation().
+GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(
+    const char* file, int line) {
+  const std::string file_name(file == NULL ? kUnknownFile : file);
+
+  if (line < 0)
+    return file_name;
+  else
+    return file_name + ":" + StreamableToString(line);
+}
+
+GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line)
+    : severity_(severity) {
+  const char* const marker =
+      severity == GTEST_INFO ?    "[  INFO ]" :
+      severity == GTEST_WARNING ? "[WARNING]" :
+      severity == GTEST_ERROR ?   "[ ERROR ]" : "[ FATAL ]";
+  GetStream() << ::std::endl << marker << " "
+              << FormatFileLocation(file, line).c_str() << ": ";
+}
+
+// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program.
+GTestLog::~GTestLog() {
+  GetStream() << ::std::endl;
+  if (severity_ == GTEST_FATAL) {
+    fflush(stderr);
+    posix::Abort();
+  }
+}
+// Disable Microsoft deprecation warnings for POSIX functions called from
+// this class (creat, dup, dup2, and close)
+GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
+
+#if GTEST_HAS_STREAM_REDIRECTION
+
+// Object that captures an output stream (stdout/stderr).
+class CapturedStream {
+ public:
+  // The ctor redirects the stream to a temporary file.
+  explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) {
+# if GTEST_OS_WINDOWS
+    char temp_dir_path[MAX_PATH + 1] = { '\0' };  // NOLINT
+    char temp_file_path[MAX_PATH + 1] = { '\0' };  // NOLINT
+
+    ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path);
+    const UINT success = ::GetTempFileNameA(temp_dir_path,
+                                            "gtest_redir",
+                                            0,  // Generate unique file name.
+                                            temp_file_path);
+    GTEST_CHECK_(success != 0)
+        << "Unable to create a temporary file in " << temp_dir_path;
+    const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE);
+    GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file "
+                                    << temp_file_path;
+    filename_ = temp_file_path;
+# else
+    // There's no guarantee that a test has write access to the current
+    // directory, so we create the temporary file in the /tmp directory
+    // instead. We use /tmp on most systems, and /sdcard on Android.
+    // That's because Android doesn't have /tmp.
+#  if GTEST_OS_LINUX_ANDROID
+    // Note: Android applications are expected to call the framework's
+    // Context.getExternalStorageDirectory() method through JNI to get
+    // the location of the world-writable SD Card directory. However,
+    // this requires a Context handle, which cannot be retrieved
+    // globally from native code. Doing so also precludes running the
+    // code as part of a regular standalone executable, which doesn't
+    // run in a Dalvik process (e.g. when running it through 'adb shell').
+    //
+    // The location /sdcard is directly accessible from native code
+    // and is the only location (unofficially) supported by the Android
+    // team. It's generally a symlink to the real SD Card mount point
+    // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or
+    // other OEM-customized locations. Never rely on these, and always
+    // use /sdcard.
+    char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX";
+#  else
+    char name_template[] = "/tmp/captured_stream.XXXXXX";
+#  endif  // GTEST_OS_LINUX_ANDROID
+    const int captured_fd = mkstemp(name_template);
+    filename_ = name_template;
+# endif  // GTEST_OS_WINDOWS
+    fflush(NULL);
+    dup2(captured_fd, fd_);
+    close(captured_fd);
+  }
+
+  ~CapturedStream() {
+    remove(filename_.c_str());
+  }
+
+  std::string GetCapturedString() {
+    if (uncaptured_fd_ != -1) {
+      // Restores the original stream.
+      fflush(NULL);
+      dup2(uncaptured_fd_, fd_);
+      close(uncaptured_fd_);
+      uncaptured_fd_ = -1;
+    }
+
+    FILE* const file = posix::FOpen(filename_.c_str(), "r");
+    const std::string content = ReadEntireFile(file);
+    posix::FClose(file);
+    return content;
+  }
+
+ private:
+  const int fd_;  // A stream to capture.
+  int uncaptured_fd_;
+  // Name of the temporary file holding the stderr output.
+  ::std::string filename_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream);
+};
+
+GTEST_DISABLE_MSC_WARNINGS_POP_()
+
+static CapturedStream* g_captured_stderr = NULL;
+static CapturedStream* g_captured_stdout = NULL;
+
+// Starts capturing an output stream (stdout/stderr).
+void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) {
+  if (*stream != NULL) {
+    GTEST_LOG_(FATAL) << "Only one " << stream_name
+                      << " capturer can exist at a time.";
+  }
+  *stream = new CapturedStream(fd);
+}
+
+// Stops capturing the output stream and returns the captured string.
+std::string GetCapturedStream(CapturedStream** captured_stream) {
+  const std::string content = (*captured_stream)->GetCapturedString();
+
+  delete *captured_stream;
+  *captured_stream = NULL;
+
+  return content;
+}
+
+// Starts capturing stdout.
+void CaptureStdout() {
+  CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout);
+}
+
+// Starts capturing stderr.
+void CaptureStderr() {
+  CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr);
+}
+
+// Stops capturing stdout and returns the captured string.
+std::string GetCapturedStdout() {
+  return GetCapturedStream(&g_captured_stdout);
+}
+
+// Stops capturing stderr and returns the captured string.
+std::string GetCapturedStderr() {
+  return GetCapturedStream(&g_captured_stderr);
+}
+
+#endif  // GTEST_HAS_STREAM_REDIRECTION
+
+std::string TempDir() {
+#if GTEST_OS_WINDOWS_MOBILE
+  return "\\temp\\";
+#elif GTEST_OS_WINDOWS
+  const char* temp_dir = posix::GetEnv("TEMP");
+  if (temp_dir == NULL || temp_dir[0] == '\0')
+    return "\\temp\\";
+  else if (temp_dir[strlen(temp_dir) - 1] == '\\')
+    return temp_dir;
+  else
+    return std::string(temp_dir) + "\\";
+#elif GTEST_OS_LINUX_ANDROID
+  return "/sdcard/";
+#else
+  return "/tmp/";
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+size_t GetFileSize(FILE* file) {
+  fseek(file, 0, SEEK_END);
+  return static_cast<size_t>(ftell(file));
+}
+
+std::string ReadEntireFile(FILE* file) {
+  const size_t file_size = GetFileSize(file);
+  char* const buffer = new char[file_size];
+
+  size_t bytes_last_read = 0;  // # of bytes read in the last fread()
+  size_t bytes_read = 0;       // # of bytes read so far
+
+  fseek(file, 0, SEEK_SET);
+
+  // Keeps reading the file until we cannot read further or the
+  // pre-determined file size is reached.
+  do {
+    bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
+    bytes_read += bytes_last_read;
+  } while (bytes_last_read > 0 && bytes_read < file_size);
+
+  const std::string content(buffer, bytes_read);
+  delete[] buffer;
+
+  return content;
+}
+
+#if GTEST_HAS_DEATH_TEST
+
+static const ::std::vector<testing::internal::string>* g_injected_test_argvs =
+                                        NULL;  // Owned.
+
+void SetInjectableArgvs(const ::std::vector<testing::internal::string>* argvs) {
+  if (g_injected_test_argvs != argvs)
+    delete g_injected_test_argvs;
+  g_injected_test_argvs = argvs;
+}
+
+const ::std::vector<testing::internal::string>& GetInjectableArgvs() {
+  if (g_injected_test_argvs != NULL) {
+    return *g_injected_test_argvs;
+  }
+  return GetArgvs();
+}
+#endif  // GTEST_HAS_DEATH_TEST
+
+#if GTEST_OS_WINDOWS_MOBILE
+namespace posix {
+void Abort() {
+  DebugBreak();
+  TerminateProcess(GetCurrentProcess(), 1);
+}
+}  // namespace posix
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+// Returns the name of the environment variable corresponding to the
+// given flag.  For example, FlagToEnvVar("foo") will return
+// "GTEST_FOO" in the open-source version.
+static std::string FlagToEnvVar(const char* flag) {
+  const std::string full_flag =
+      (Message() << GTEST_FLAG_PREFIX_ << flag).GetString();
+
+  Message env_var;
+  for (size_t i = 0; i != full_flag.length(); i++) {
+    env_var << ToUpper(full_flag.c_str()[i]);
+  }
+
+  return env_var.GetString();
+}
+
+// Parses 'str' for a 32-bit signed integer.  If successful, writes
+// the result to *value and returns true; otherwise leaves *value
+// unchanged and returns false.
+bool ParseInt32(const Message& src_text, const char* str, Int32* value) {
+  // Parses the environment variable as a decimal integer.
+  char* end = NULL;
+  const long long_value = strtol(str, &end, 10);  // NOLINT
+
+  // Has strtol() consumed all characters in the string?
+  if (*end != '\0') {
+    // No - an invalid character was encountered.
+    Message msg;
+    msg << "WARNING: " << src_text
+        << " is expected to be a 32-bit integer, but actually"
+        << " has value \"" << str << "\".\n";
+    printf("%s", msg.GetString().c_str());
+    fflush(stdout);
+    return false;
+  }
+
+  // Is the parsed value in the range of an Int32?
+  const Int32 result = static_cast<Int32>(long_value);
+  if (long_value == LONG_MAX || long_value == LONG_MIN ||
+      // The parsed value overflows as a long.  (strtol() returns
+      // LONG_MAX or LONG_MIN when the input overflows.)
+      result != long_value
+      // The parsed value overflows as an Int32.
+      ) {
+    Message msg;
+    msg << "WARNING: " << src_text
+        << " is expected to be a 32-bit integer, but actually"
+        << " has value " << str << ", which overflows.\n";
+    printf("%s", msg.GetString().c_str());
+    fflush(stdout);
+    return false;
+  }
+
+  *value = result;
+  return true;
+}
+
+// Reads and returns the Boolean environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+//
+// The value is considered true iff it's not "0".
+bool BoolFromGTestEnv(const char* flag, bool default_value) {
+#if defined(GTEST_GET_BOOL_FROM_ENV_)
+  return GTEST_GET_BOOL_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_BOOL_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* const string_value = posix::GetEnv(env_var.c_str());
+  return string_value == NULL ?
+      default_value : strcmp(string_value, "0") != 0;
+}
+
+// Reads and returns a 32-bit integer stored in the environment
+// variable corresponding to the given flag; if it isn't set or
+// doesn't represent a valid 32-bit integer, returns default_value.
+Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) {
+#if defined(GTEST_GET_INT32_FROM_ENV_)
+  return GTEST_GET_INT32_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_INT32_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* const string_value = posix::GetEnv(env_var.c_str());
+  if (string_value == NULL) {
+    // The environment variable is not set.
+    return default_value;
+  }
+
+  Int32 result = default_value;
+  if (!ParseInt32(Message() << "Environment variable " << env_var,
+                  string_value, &result)) {
+    printf("The default value %s is used.\n",
+           (Message() << default_value).GetString().c_str());
+    fflush(stdout);
+    return default_value;
+  }
+
+  return result;
+}
+
+// Reads and returns the string environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+std::string StringFromGTestEnv(const char* flag, const char* default_value) {
+#if defined(GTEST_GET_STRING_FROM_ENV_)
+  return GTEST_GET_STRING_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_STRING_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* value = posix::GetEnv(env_var.c_str());
+  if (value != NULL) {
+    return value;
+  }
+
+  // As a special case for the 'output' flag, if GTEST_OUTPUT is not
+  // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build
+  // system.  The value of XML_OUTPUT_FILE is a filename without the
+  // "xml:" prefix of GTEST_OUTPUT.
+  //
+  // The net priority order after flag processing is thus:
+  //   --gtest_output command line flag
+  //   GTEST_OUTPUT environment variable
+  //   XML_OUTPUT_FILE environment variable
+  //   'default_value'
+  if (strcmp(flag, "output") == 0) {
+    value = posix::GetEnv("XML_OUTPUT_FILE");
+    if (value != NULL) {
+      return std::string("xml:") + value;
+    }
+  }
+  return default_value;
+}
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Google Test - The Google C++ Testing Framework
+//
+// This file implements a universal value printer that can print a
+// value of any type T:
+//
+//   void ::testing::internal::UniversalPrinter<T>::Print(value, ostream_ptr);
+//
+// It uses the << operator when possible, and prints the bytes in the
+// object otherwise.  A user can override its behavior for a class
+// type Foo by defining either operator<<(::std::ostream&, const Foo&)
+// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that
+// defines Foo.
+
+#include "gtest/gtest-printers.h"
+#include <ctype.h>
+#include <stdio.h>
+#include <cwchar>
+#include <ostream>  // NOLINT
+#include <string>
+#include "gtest/internal/gtest-port.h"
+
+namespace testing {
+
+namespace {
+
+using ::std::ostream;
+
+// Prints a segment of bytes in the given object.
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start,
+                                size_t count, ostream* os) {
+  char text[5] = "";
+  for (size_t i = 0; i != count; i++) {
+    const size_t j = start + i;
+    if (i != 0) {
+      // Organizes the bytes into groups of 2 for easy parsing by
+      // human.
+      if ((j % 2) == 0)
+        *os << ' ';
+      else
+        *os << '-';
+    }
+    GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]);
+    *os << text;
+  }
+}
+
+// Prints the bytes in the given value to the given ostream.
+void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count,
+                              ostream* os) {
+  // Tells the user how big the object is.
+  *os << count << "-byte object <";
+
+  const size_t kThreshold = 132;
+  const size_t kChunkSize = 64;
+  // If the object size is bigger than kThreshold, we'll have to omit
+  // some details by printing only the first and the last kChunkSize
+  // bytes.
+  // TODO(wan): let the user control the threshold using a flag.
+  if (count < kThreshold) {
+    PrintByteSegmentInObjectTo(obj_bytes, 0, count, os);
+  } else {
+    PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os);
+    *os << " ... ";
+    // Rounds up to 2-byte boundary.
+    const size_t resume_pos = (count - kChunkSize + 1)/2*2;
+    PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os);
+  }
+  *os << ">";
+}
+
+}  // namespace
+
+namespace internal2 {
+
+// Delegates to PrintBytesInObjectToImpl() to print the bytes in the
+// given object.  The delegation simplifies the implementation, which
+// uses the << operator and thus is easier done outside of the
+// ::testing::internal namespace, which contains a << operator that
+// sometimes conflicts with the one in STL.
+void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count,
+                          ostream* os) {
+  PrintBytesInObjectToImpl(obj_bytes, count, os);
+}
+
+}  // namespace internal2
+
+namespace internal {
+
+// Depending on the value of a char (or wchar_t), we print it in one
+// of three formats:
+//   - as is if it's a printable ASCII (e.g. 'a', '2', ' '),
+//   - as a hexidecimal escape sequence (e.g. '\x7F'), or
+//   - as a special escape sequence (e.g. '\r', '\n').
+enum CharFormat {
+  kAsIs,
+  kHexEscape,
+  kSpecialEscape
+};
+
+// Returns true if c is a printable ASCII character.  We test the
+// value of c directly instead of calling isprint(), which is buggy on
+// Windows Mobile.
+inline bool IsPrintableAscii(wchar_t c) {
+  return 0x20 <= c && c <= 0x7E;
+}
+
+// Prints a wide or narrow char c as a character literal without the
+// quotes, escaping it when necessary; returns how c was formatted.
+// The template argument UnsignedChar is the unsigned version of Char,
+// which is the type of c.
+template <typename UnsignedChar, typename Char>
+static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) {
+  switch (static_cast<wchar_t>(c)) {
+    case L'\0':
+      *os << "\\0";
+      break;
+    case L'\'':
+      *os << "\\'";
+      break;
+    case L'\\':
+      *os << "\\\\";
+      break;
+    case L'\a':
+      *os << "\\a";
+      break;
+    case L'\b':
+      *os << "\\b";
+      break;
+    case L'\f':
+      *os << "\\f";
+      break;
+    case L'\n':
+      *os << "\\n";
+      break;
+    case L'\r':
+      *os << "\\r";
+      break;
+    case L'\t':
+      *os << "\\t";
+      break;
+    case L'\v':
+      *os << "\\v";
+      break;
+    default:
+      if (IsPrintableAscii(c)) {
+        *os << static_cast<char>(c);
+        return kAsIs;
+      } else {
+        *os << "\\x" + String::FormatHexInt(static_cast<UnsignedChar>(c));
+        return kHexEscape;
+      }
+  }
+  return kSpecialEscape;
+}
+
+// Prints a wchar_t c as if it's part of a string literal, escaping it when
+// necessary; returns how c was formatted.
+static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) {
+  switch (c) {
+    case L'\'':
+      *os << "'";
+      return kAsIs;
+    case L'"':
+      *os << "\\\"";
+      return kSpecialEscape;
+    default:
+      return PrintAsCharLiteralTo<wchar_t>(c, os);
+  }
+}
+
+// Prints a char c as if it's part of a string literal, escaping it when
+// necessary; returns how c was formatted.
+static CharFormat PrintAsStringLiteralTo(char c, ostream* os) {
+  return PrintAsStringLiteralTo(
+      static_cast<wchar_t>(static_cast<unsigned char>(c)), os);
+}
+
+// Prints a wide or narrow character c and its code.  '\0' is printed
+// as "'\\0'", other unprintable characters are also properly escaped
+// using the standard C++ escape sequence.  The template argument
+// UnsignedChar is the unsigned version of Char, which is the type of c.
+template <typename UnsignedChar, typename Char>
+void PrintCharAndCodeTo(Char c, ostream* os) {
+  // First, print c as a literal in the most readable form we can find.
+  *os << ((sizeof(c) > 1) ? "L'" : "'");
+  const CharFormat format = PrintAsCharLiteralTo<UnsignedChar>(c, os);
+  *os << "'";
+
+  // To aid user debugging, we also print c's code in decimal, unless
+  // it's 0 (in which case c was printed as '\\0', making the code
+  // obvious).
+  if (c == 0)
+    return;
+  *os << " (" << static_cast<int>(c);
+
+  // For more convenience, we print c's code again in hexidecimal,
+  // unless c was already printed in the form '\x##' or the code is in
+  // [1, 9].
+  if (format == kHexEscape || (1 <= c && c <= 9)) {
+    // Do nothing.
+  } else {
+    *os << ", 0x" << String::FormatHexInt(static_cast<UnsignedChar>(c));
+  }
+  *os << ")";
+}
+
+void PrintTo(unsigned char c, ::std::ostream* os) {
+  PrintCharAndCodeTo<unsigned char>(c, os);
+}
+void PrintTo(signed char c, ::std::ostream* os) {
+  PrintCharAndCodeTo<unsigned char>(c, os);
+}
+
+// Prints a wchar_t as a symbol if it is printable or as its internal
+// code otherwise and also as its code.  L'\0' is printed as "L'\\0'".
+void PrintTo(wchar_t wc, ostream* os) {
+  PrintCharAndCodeTo<wchar_t>(wc, os);
+}
+
+// Prints the given array of characters to the ostream.  CharType must be either
+// char or wchar_t.
+// The array starts at begin, the length is len, it may include '\0' characters
+// and may not be NUL-terminated.
+template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+static void PrintCharsAsStringTo(
+    const CharType* begin, size_t len, ostream* os) {
+  const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\"";
+  *os << kQuoteBegin;
+  bool is_previous_hex = false;
+  for (size_t index = 0; index < len; ++index) {
+    const CharType cur = begin[index];
+    if (is_previous_hex && IsXDigit(cur)) {
+      // Previous character is of '\x..' form and this character can be
+      // interpreted as another hexadecimal digit in its number. Break string to
+      // disambiguate.
+      *os << "\" " << kQuoteBegin;
+    }
+    is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape;
+  }
+  *os << "\"";
+}
+
+// Prints a (const) char/wchar_t array of 'len' elements, starting at address
+// 'begin'.  CharType must be either char or wchar_t.
+template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+static void UniversalPrintCharArray(
+    const CharType* begin, size_t len, ostream* os) {
+  // The code
+  //   const char kFoo[] = "foo";
+  // generates an array of 4, not 3, elements, with the last one being '\0'.
+  //
+  // Therefore when printing a char array, we don't print the last element if
+  // it's '\0', such that the output matches the string literal as it's
+  // written in the source code.
+  if (len > 0 && begin[len - 1] == '\0') {
+    PrintCharsAsStringTo(begin, len - 1, os);
+    return;
+  }
+
+  // If, however, the last element in the array is not '\0', e.g.
+  //    const char kFoo[] = { 'f', 'o', 'o' };
+  // we must print the entire array.  We also print a message to indicate
+  // that the array is not NUL-terminated.
+  PrintCharsAsStringTo(begin, len, os);
+  *os << " (no terminating NUL)";
+}
+
+// Prints a (const) char array of 'len' elements, starting at address 'begin'.
+void UniversalPrintArray(const char* begin, size_t len, ostream* os) {
+  UniversalPrintCharArray(begin, len, os);
+}
+
+// Prints a (const) wchar_t array of 'len' elements, starting at address
+// 'begin'.
+void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) {
+  UniversalPrintCharArray(begin, len, os);
+}
+
+// Prints the given C string to the ostream.
+void PrintTo(const char* s, ostream* os) {
+  if (s == NULL) {
+    *os << "NULL";
+  } else {
+    *os << ImplicitCast_<const void*>(s) << " pointing to ";
+    PrintCharsAsStringTo(s, strlen(s), os);
+  }
+}
+
+// MSVC compiler can be configured to define whar_t as a typedef
+// of unsigned short. Defining an overload for const wchar_t* in that case
+// would cause pointers to unsigned shorts be printed as wide strings,
+// possibly accessing more memory than intended and causing invalid
+// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when
+// wchar_t is implemented as a native type.
+#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED)
+// Prints the given wide C string to the ostream.
+void PrintTo(const wchar_t* s, ostream* os) {
+  if (s == NULL) {
+    *os << "NULL";
+  } else {
+    *os << ImplicitCast_<const void*>(s) << " pointing to ";
+    PrintCharsAsStringTo(s, std::wcslen(s), os);
+  }
+}
+#endif  // wchar_t is native
+
+// Prints a ::string object.
+#if GTEST_HAS_GLOBAL_STRING
+void PrintStringTo(const ::string& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_GLOBAL_STRING
+
+void PrintStringTo(const ::std::string& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+
+// Prints a ::wstring object.
+#if GTEST_HAS_GLOBAL_WSTRING
+void PrintWideStringTo(const ::wstring& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+#if GTEST_HAS_STD_WSTRING
+void PrintWideStringTo(const ::std::wstring& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+}  // namespace internal
+
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+// The Google C++ Testing Framework (Google Test)
+
+#include "gtest/gtest-test-part.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+
+using internal::GetUnitTestImpl;
+
+// Gets the summary of the failure message by omitting the stack trace
+// in it.
+std::string TestPartResult::ExtractSummary(const char* message) {
+  const char* const stack_trace = strstr(message, internal::kStackTraceMarker);
+  return stack_trace == NULL ? message :
+      std::string(message, stack_trace);
+}
+
+// Prints a TestPartResult object.
+std::ostream& operator<<(std::ostream& os, const TestPartResult& result) {
+  return os
+      << result.file_name() << ":" << result.line_number() << ": "
+      << (result.type() == TestPartResult::kSuccess ? "Success" :
+          result.type() == TestPartResult::kFatalFailure ? "Fatal failure" :
+          "Non-fatal failure") << ":\n"
+      << result.message() << std::endl;
+}
+
+// Appends a TestPartResult to the array.
+void TestPartResultArray::Append(const TestPartResult& result) {
+  array_.push_back(result);
+}
+
+// Returns the TestPartResult at the given index (0-based).
+const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const {
+  if (index < 0 || index >= size()) {
+    printf("\nInvalid index (%d) into TestPartResultArray.\n", index);
+    internal::posix::Abort();
+  }
+
+  return array_[index];
+}
+
+// Returns the number of TestPartResult objects in the array.
+int TestPartResultArray::size() const {
+  return static_cast<int>(array_.size());
+}
+
+namespace internal {
+
+HasNewFatalFailureHelper::HasNewFatalFailureHelper()
+    : has_new_fatal_failure_(false),
+      original_reporter_(GetUnitTestImpl()->
+                         GetTestPartResultReporterForCurrentThread()) {
+  GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this);
+}
+
+HasNewFatalFailureHelper::~HasNewFatalFailureHelper() {
+  GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(
+      original_reporter_);
+}
+
+void HasNewFatalFailureHelper::ReportTestPartResult(
+    const TestPartResult& result) {
+  if (result.fatally_failed())
+    has_new_fatal_failure_ = true;
+  original_reporter_->ReportTestPartResult(result);
+}
+
+}  // namespace internal
+
+}  // namespace testing
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest-typed-test.h"
+#include "gtest/gtest.h"
+
+namespace testing {
+namespace internal {
+
+#if GTEST_HAS_TYPED_TEST_P
+
+// Skips to the first non-space char in str. Returns an empty string if str
+// contains only whitespace characters.
+static const char* SkipSpaces(const char* str) {
+  while (IsSpace(*str))
+    str++;
+  return str;
+}
+
+static std::vector<std::string> SplitIntoTestNames(const char* src) {
+  std::vector<std::string> name_vec;
+  src = SkipSpaces(src);
+  for (; src != NULL; src = SkipComma(src)) {
+    name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src)));
+  }
+  return name_vec;
+}
+
+// Verifies that registered_tests match the test names in
+// registered_tests_; returns registered_tests if successful, or
+// aborts the program otherwise.
+const char* TypedTestCasePState::VerifyRegisteredTestNames(
+    const char* file, int line, const char* registered_tests) {
+  typedef RegisteredTestsMap::const_iterator RegisteredTestIter;
+  registered_ = true;
+
+  std::vector<std::string> name_vec = SplitIntoTestNames(registered_tests);
+
+  Message errors;
+
+  std::set<std::string> tests;
+  for (std::vector<std::string>::const_iterator name_it = name_vec.begin();
+       name_it != name_vec.end(); ++name_it) {
+    const std::string& name = *name_it;
+    if (tests.count(name) != 0) {
+      errors << "Test " << name << " is listed more than once.\n";
+      continue;
+    }
+
+    bool found = false;
+    for (RegisteredTestIter it = registered_tests_.begin();
+         it != registered_tests_.end();
+         ++it) {
+      if (name == it->first) {
+        found = true;
+        break;
+      }
+    }
+
+    if (found) {
+      tests.insert(name);
+    } else {
+      errors << "No test named " << name
+             << " can be found in this test case.\n";
+    }
+  }
+
+  for (RegisteredTestIter it = registered_tests_.begin();
+       it != registered_tests_.end();
+       ++it) {
+    if (tests.count(it->first) == 0) {
+      errors << "You forgot to list test " << it->first << ".\n";
+    }
+  }
+
+  const std::string& errors_str = errors.GetString();
+  if (errors_str != "") {
+    fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(),
+            errors_str.c_str());
+    fflush(stderr);
+    posix::Abort();
+  }
+
+  return registered_tests;
+}
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// The Google C++ Testing Framework (Google Test)
+
+#include "gtest/gtest.h"
+#include "gtest/internal/custom/gtest.h"
+#include "gtest/gtest-spi.h"
+
+#include <ctype.h>
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <wchar.h>
+#include <wctype.h>
+
+#include <algorithm>
+#include <iomanip>
+#include <limits>
+#include <list>
+#include <map>
+#include <ostream>  // NOLINT
+#include <sstream>
+#include <vector>
+
+#if GTEST_OS_LINUX
+
+// TODO(kenton@google.com): Use autoconf to detect availability of
+// gettimeofday().
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+
+# include <fcntl.h>  // NOLINT
+# include <limits.h>  // NOLINT
+# include <sched.h>  // NOLINT
+// Declares vsnprintf().  This header is not available on Windows.
+# include <strings.h>  // NOLINT
+# include <sys/mman.h>  // NOLINT
+# include <sys/time.h>  // NOLINT
+# include <unistd.h>  // NOLINT
+# include <string>
+
+#elif GTEST_OS_SYMBIAN
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+# include <sys/time.h>  // NOLINT
+
+#elif GTEST_OS_ZOS
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+# include <sys/time.h>  // NOLINT
+
+// On z/OS we additionally need strings.h for strcasecmp.
+# include <strings.h>  // NOLINT
+
+#elif GTEST_OS_WINDOWS_MOBILE  // We are on Windows CE.
+
+# include <windows.h>  // NOLINT
+# undef min
+
+#elif GTEST_OS_WINDOWS  // We are on Windows proper.
+
+# include <io.h>  // NOLINT
+# include <sys/timeb.h>  // NOLINT
+# include <sys/types.h>  // NOLINT
+# include <sys/stat.h>  // NOLINT
+
+# if GTEST_OS_WINDOWS_MINGW
+// MinGW has gettimeofday() but not _ftime64().
+// TODO(kenton@google.com): Use autoconf to detect availability of
+//   gettimeofday().
+// TODO(kenton@google.com): There are other ways to get the time on
+//   Windows, like GetTickCount() or GetSystemTimeAsFileTime().  MinGW
+//   supports these.  consider using them instead.
+#  define GTEST_HAS_GETTIMEOFDAY_ 1
+#  include <sys/time.h>  // NOLINT
+# endif  // GTEST_OS_WINDOWS_MINGW
+
+// cpplint thinks that the header is already included, so we want to
+// silence it.
+# include <windows.h>  // NOLINT
+# undef min
+
+#else
+
+// Assume other platforms have gettimeofday().
+// TODO(kenton@google.com): Use autoconf to detect availability of
+//   gettimeofday().
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+
+// cpplint thinks that the header is already included, so we want to
+// silence it.
+# include <sys/time.h>  // NOLINT
+# include <unistd.h>  // NOLINT
+
+#endif  // GTEST_OS_LINUX
+
+#if GTEST_HAS_EXCEPTIONS
+# include <stdexcept>
+#endif
+
+#if GTEST_CAN_STREAM_RESULTS_
+# include <arpa/inet.h>  // NOLINT
+# include <netdb.h>  // NOLINT
+# include <sys/socket.h>  // NOLINT
+# include <sys/types.h>  // NOLINT
+#endif
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+#if GTEST_OS_WINDOWS
+# define vsnprintf _vsnprintf
+#endif  // GTEST_OS_WINDOWS
+
+namespace testing {
+
+using internal::CountIf;
+using internal::ForEach;
+using internal::GetElementOr;
+using internal::Shuffle;
+
+// Constants.
+
+// A test whose test case name or test name matches this filter is
+// disabled and not run.
+static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*";
+
+// A test case whose name matches this filter is considered a death
+// test case and will be run before test cases whose name doesn't
+// match this filter.
+static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*";
+
+// A test filter that matches everything.
+static const char kUniversalFilter[] = "*";
+
+// The default output file for XML output.
+static const char kDefaultOutputFile[] = "test_detail.xml";
+
+// The environment variable name for the test shard index.
+static const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
+// The environment variable name for the total number of test shards.
+static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
+// The environment variable name for the test shard status file.
+static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE";
+
+namespace internal {
+
+// The text used in failure messages to indicate the start of the
+// stack trace.
+const char kStackTraceMarker[] = "\nStack trace:\n";
+
+// g_help_flag is true iff the --help flag or an equivalent form is
+// specified on the command line.
+bool g_help_flag = false;
+
+}  // namespace internal
+
+static const char* GetDefaultFilter() {
+#ifdef GTEST_TEST_FILTER_ENV_VAR_
+  const char* const testbridge_test_only = getenv(GTEST_TEST_FILTER_ENV_VAR_);
+  if (testbridge_test_only != NULL) {
+    return testbridge_test_only;
+  }
+#endif  // GTEST_TEST_FILTER_ENV_VAR_
+  return kUniversalFilter;
+}
+
+GTEST_DEFINE_bool_(
+    also_run_disabled_tests,
+    internal::BoolFromGTestEnv("also_run_disabled_tests", false),
+    "Run disabled tests too, in addition to the tests normally being run.");
+
+GTEST_DEFINE_bool_(
+    break_on_failure,
+    internal::BoolFromGTestEnv("break_on_failure", false),
+    "True iff a failed assertion should be a debugger break-point.");
+
+GTEST_DEFINE_bool_(
+    catch_exceptions,
+    internal::BoolFromGTestEnv("catch_exceptions", true),
+    "True iff " GTEST_NAME_
+    " should catch exceptions and treat them as test failures.");
+
+GTEST_DEFINE_string_(
+    color,
+    internal::StringFromGTestEnv("color", "auto"),
+    "Whether to use colors in the output.  Valid values: yes, no, "
+    "and auto.  'auto' means to use colors if the output is "
+    "being sent to a terminal and the TERM environment variable "
+    "is set to a terminal type that supports colors.");
+
+GTEST_DEFINE_string_(
+    filter,
+    internal::StringFromGTestEnv("filter", GetDefaultFilter()),
+    "A colon-separated list of glob (not regex) patterns "
+    "for filtering the tests to run, optionally followed by a "
+    "'-' and a : separated list of negative patterns (tests to "
+    "exclude).  A test is run if it matches one of the positive "
+    "patterns and does not match any of the negative patterns.");
+
+GTEST_DEFINE_bool_(list_tests, false,
+                   "List all tests without running them.");
+
+GTEST_DEFINE_string_(
+    output,
+    internal::StringFromGTestEnv("output", ""),
+    "A format (currently must be \"xml\"), optionally followed "
+    "by a colon and an output file name or directory. A directory "
+    "is indicated by a trailing pathname separator. "
+    "Examples: \"xml:filename.xml\", \"xml::directoryname/\". "
+    "If a directory is specified, output files will be created "
+    "within that directory, with file-names based on the test "
+    "executable's name and, if necessary, made unique by adding "
+    "digits.");
+
+GTEST_DEFINE_bool_(
+    print_time,
+    internal::BoolFromGTestEnv("print_time", true),
+    "True iff " GTEST_NAME_
+    " should display elapsed time in text output.");
+
+GTEST_DEFINE_int32_(
+    random_seed,
+    internal::Int32FromGTestEnv("random_seed", 0),
+    "Random number seed to use when shuffling test orders.  Must be in range "
+    "[1, 99999], or 0 to use a seed based on the current time.");
+
+GTEST_DEFINE_int32_(
+    repeat,
+    internal::Int32FromGTestEnv("repeat", 1),
+    "How many times to repeat each test.  Specify a negative number "
+    "for repeating forever.  Useful for shaking out flaky tests.");
+
+GTEST_DEFINE_bool_(
+    show_internal_stack_frames, false,
+    "True iff " GTEST_NAME_ " should include internal stack frames when "
+    "printing test failure stack traces.");
+
+GTEST_DEFINE_bool_(
+    shuffle,
+    internal::BoolFromGTestEnv("shuffle", false),
+    "True iff " GTEST_NAME_
+    " should randomize tests' order on every run.");
+
+GTEST_DEFINE_int32_(
+    stack_trace_depth,
+    internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth),
+    "The maximum number of stack frames to print when an "
+    "assertion fails.  The valid range is 0 through 100, inclusive.");
+
+GTEST_DEFINE_string_(
+    stream_result_to,
+    internal::StringFromGTestEnv("stream_result_to", ""),
+    "This flag specifies the host name and the port number on which to stream "
+    "test results. Example: \"localhost:555\". The flag is effective only on "
+    "Linux.");
+
+GTEST_DEFINE_bool_(
+    throw_on_failure,
+    internal::BoolFromGTestEnv("throw_on_failure", false),
+    "When this flag is specified, a failed assertion will throw an exception "
+    "if exceptions are enabled or exit the program with a non-zero code "
+    "otherwise.");
+
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+GTEST_DEFINE_string_(
+    flagfile,
+    internal::StringFromGTestEnv("flagfile", ""),
+    "This flag specifies the flagfile to read command-line flags from.");
+#endif  // GTEST_USE_OWN_FLAGFILE_FLAG_
+
+namespace internal {
+
+// Generates a random number from [0, range), using a Linear
+// Congruential Generator (LCG).  Crashes if 'range' is 0 or greater
+// than kMaxRange.
+UInt32 Random::Generate(UInt32 range) {
+  // These constants are the same as are used in glibc's rand(3).
+  state_ = (1103515245U*state_ + 12345U) % kMaxRange;
+
+  GTEST_CHECK_(range > 0)
+      << "Cannot generate a number in the range [0, 0).";
+  GTEST_CHECK_(range <= kMaxRange)
+      << "Generation of a number in [0, " << range << ") was requested, "
+      << "but this can only generate numbers in [0, " << kMaxRange << ").";
+
+  // Converting via modulus introduces a bit of downward bias, but
+  // it's simple, and a linear congruential generator isn't too good
+  // to begin with.
+  return state_ % range;
+}
+
+// GTestIsInitialized() returns true iff the user has initialized
+// Google Test.  Useful for catching the user mistake of not initializing
+// Google Test before calling RUN_ALL_TESTS().
+static bool GTestIsInitialized() { return GetArgvs().size() > 0; }
+
+// Iterates over a vector of TestCases, keeping a running sum of the
+// results of calling a given int-returning method on each.
+// Returns the sum.
+static int SumOverTestCaseList(const std::vector<TestCase*>& case_list,
+                               int (TestCase::*method)() const) {
+  int sum = 0;
+  for (size_t i = 0; i < case_list.size(); i++) {
+    sum += (case_list[i]->*method)();
+  }
+  return sum;
+}
+
+// Returns true iff the test case passed.
+static bool TestCasePassed(const TestCase* test_case) {
+  return test_case->should_run() && test_case->Passed();
+}
+
+// Returns true iff the test case failed.
+static bool TestCaseFailed(const TestCase* test_case) {
+  return test_case->should_run() && test_case->Failed();
+}
+
+// Returns true iff test_case contains at least one test that should
+// run.
+static bool ShouldRunTestCase(const TestCase* test_case) {
+  return test_case->should_run();
+}
+
+// AssertHelper constructor.
+AssertHelper::AssertHelper(TestPartResult::Type type,
+                           const char* file,
+                           int line,
+                           const char* message)
+    : data_(new AssertHelperData(type, file, line, message)) {
+}
+
+AssertHelper::~AssertHelper() {
+  delete data_;
+}
+
+// Message assignment, for assertion streaming support.
+void AssertHelper::operator=(const Message& message) const {
+  UnitTest::GetInstance()->
+    AddTestPartResult(data_->type, data_->file, data_->line,
+                      AppendUserMessage(data_->message, message),
+                      UnitTest::GetInstance()->impl()
+                      ->CurrentOsStackTraceExceptTop(1)
+                      // Skips the stack frame for this function itself.
+                      );  // NOLINT
+}
+
+// Mutex for linked pointers.
+GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex);
+
+// A copy of all command line arguments.  Set by InitGoogleTest().
+::std::vector<testing::internal::string> g_argvs;
+
+const ::std::vector<testing::internal::string>& GetArgvs() {
+#if defined(GTEST_CUSTOM_GET_ARGVS_)
+  return GTEST_CUSTOM_GET_ARGVS_();
+#else  // defined(GTEST_CUSTOM_GET_ARGVS_)
+  return g_argvs;
+#endif  // defined(GTEST_CUSTOM_GET_ARGVS_)
+}
+
+// Returns the current application's name, removing directory path if that
+// is present.
+FilePath GetCurrentExecutableName() {
+  FilePath result;
+
+#if GTEST_OS_WINDOWS
+  result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe"));
+#else
+  result.Set(FilePath(GetArgvs()[0]));
+#endif  // GTEST_OS_WINDOWS
+
+  return result.RemoveDirectoryName();
+}
+
+// Functions for processing the gtest_output flag.
+
+// Returns the output format, or "" for normal printed output.
+std::string UnitTestOptions::GetOutputFormat() {
+  const char* const gtest_output_flag = GTEST_FLAG(output).c_str();
+  if (gtest_output_flag == NULL) return std::string("");
+
+  const char* const colon = strchr(gtest_output_flag, ':');
+  return (colon == NULL) ?
+      std::string(gtest_output_flag) :
+      std::string(gtest_output_flag, colon - gtest_output_flag);
+}
+
+// Returns the name of the requested output file, or the default if none
+// was explicitly specified.
+std::string UnitTestOptions::GetAbsolutePathToOutputFile() {
+  const char* const gtest_output_flag = GTEST_FLAG(output).c_str();
+  if (gtest_output_flag == NULL)
+    return "";
+
+  const char* const colon = strchr(gtest_output_flag, ':');
+  if (colon == NULL)
+    return internal::FilePath::ConcatPaths(
+        internal::FilePath(
+            UnitTest::GetInstance()->original_working_dir()),
+        internal::FilePath(kDefaultOutputFile)).string();
+
+  internal::FilePath output_name(colon + 1);
+  if (!output_name.IsAbsolutePath())
+    // TODO(wan@google.com): on Windows \some\path is not an absolute
+    // path (as its meaning depends on the current drive), yet the
+    // following logic for turning it into an absolute path is wrong.
+    // Fix it.
+    output_name = internal::FilePath::ConcatPaths(
+        internal::FilePath(UnitTest::GetInstance()->original_working_dir()),
+        internal::FilePath(colon + 1));
+
+  if (!output_name.IsDirectory())
+    return output_name.string();
+
+  internal::FilePath result(internal::FilePath::GenerateUniqueFileName(
+      output_name, internal::GetCurrentExecutableName(),
+      GetOutputFormat().c_str()));
+  return result.string();
+}
+
+// Returns true iff the wildcard pattern matches the string.  The
+// first ':' or '\0' character in pattern marks the end of it.
+//
+// This recursive algorithm isn't very efficient, but is clear and
+// works well enough for matching test names, which are short.
+bool UnitTestOptions::PatternMatchesString(const char *pattern,
+                                           const char *str) {
+  switch (*pattern) {
+    case '\0':
+    case ':':  // Either ':' or '\0' marks the end of the pattern.
+      return *str == '\0';
+    case '?':  // Matches any single character.
+      return *str != '\0' && PatternMatchesString(pattern + 1, str + 1);
+    case '*':  // Matches any string (possibly empty) of characters.
+      return (*str != '\0' && PatternMatchesString(pattern, str + 1)) ||
+          PatternMatchesString(pattern + 1, str);
+    default:  // Non-special character.  Matches itself.
+      return *pattern == *str &&
+          PatternMatchesString(pattern + 1, str + 1);
+  }
+}
+
+bool UnitTestOptions::MatchesFilter(
+    const std::string& name, const char* filter) {
+  const char *cur_pattern = filter;
+  for (;;) {
+    if (PatternMatchesString(cur_pattern, name.c_str())) {
+      return true;
+    }
+
+    // Finds the next pattern in the filter.
+    cur_pattern = strchr(cur_pattern, ':');
+
+    // Returns if no more pattern can be found.
+    if (cur_pattern == NULL) {
+      return false;
+    }
+
+    // Skips the pattern separater (the ':' character).
+    cur_pattern++;
+  }
+}
+
+// Returns true iff the user-specified filter matches the test case
+// name and the test name.
+bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name,
+                                        const std::string &test_name) {
+  const std::string& full_name = test_case_name + "." + test_name.c_str();
+
+  // Split --gtest_filter at '-', if there is one, to separate into
+  // positive filter and negative filter portions
+  const char* const p = GTEST_FLAG(filter).c_str();
+  const char* const dash = strchr(p, '-');
+  std::string positive;
+  std::string negative;
+  if (dash == NULL) {
+    positive = GTEST_FLAG(filter).c_str();  // Whole string is a positive filter
+    negative = "";
+  } else {
+    positive = std::string(p, dash);   // Everything up to the dash
+    negative = std::string(dash + 1);  // Everything after the dash
+    if (positive.empty()) {
+      // Treat '-test1' as the same as '*-test1'
+      positive = kUniversalFilter;
+    }
+  }
+
+  // A filter is a colon-separated list of patterns.  It matches a
+  // test if any pattern in it matches the test.
+  return (MatchesFilter(full_name, positive.c_str()) &&
+          !MatchesFilter(full_name, negative.c_str()));
+}
+
+#if GTEST_HAS_SEH
+// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the
+// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise.
+// This function is useful as an __except condition.
+int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) {
+  // Google Test should handle a SEH exception if:
+  //   1. the user wants it to, AND
+  //   2. this is not a breakpoint exception, AND
+  //   3. this is not a C++ exception (VC++ implements them via SEH,
+  //      apparently).
+  //
+  // SEH exception code for C++ exceptions.
+  // (see http://support.microsoft.com/kb/185294 for more information).
+  const DWORD kCxxExceptionCode = 0xe06d7363;
+
+  bool should_handle = true;
+
+  if (!GTEST_FLAG(catch_exceptions))
+    should_handle = false;
+  else if (exception_code == EXCEPTION_BREAKPOINT)
+    should_handle = false;
+  else if (exception_code == kCxxExceptionCode)
+    should_handle = false;
+
+  return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH;
+}
+#endif  // GTEST_HAS_SEH
+
+}  // namespace internal
+
+// The c'tor sets this object as the test part result reporter used by
+// Google Test.  The 'result' parameter specifies where to report the
+// results. Intercepts only failures from the current thread.
+ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter(
+    TestPartResultArray* result)
+    : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD),
+      result_(result) {
+  Init();
+}
+
+// The c'tor sets this object as the test part result reporter used by
+// Google Test.  The 'result' parameter specifies where to report the
+// results.
+ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter(
+    InterceptMode intercept_mode, TestPartResultArray* result)
+    : intercept_mode_(intercept_mode),
+      result_(result) {
+  Init();
+}
+
+void ScopedFakeTestPartResultReporter::Init() {
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  if (intercept_mode_ == INTERCEPT_ALL_THREADS) {
+    old_reporter_ = impl->GetGlobalTestPartResultReporter();
+    impl->SetGlobalTestPartResultReporter(this);
+  } else {
+    old_reporter_ = impl->GetTestPartResultReporterForCurrentThread();
+    impl->SetTestPartResultReporterForCurrentThread(this);
+  }
+}
+
+// The d'tor restores the test part result reporter used by Google Test
+// before.
+ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() {
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  if (intercept_mode_ == INTERCEPT_ALL_THREADS) {
+    impl->SetGlobalTestPartResultReporter(old_reporter_);
+  } else {
+    impl->SetTestPartResultReporterForCurrentThread(old_reporter_);
+  }
+}
+
+// Increments the test part result count and remembers the result.
+// This method is from the TestPartResultReporterInterface interface.
+void ScopedFakeTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  result_->Append(result);
+}
+
+namespace internal {
+
+// Returns the type ID of ::testing::Test.  We should always call this
+// instead of GetTypeId< ::testing::Test>() to get the type ID of
+// testing::Test.  This is to work around a suspected linker bug when
+// using Google Test as a framework on Mac OS X.  The bug causes
+// GetTypeId< ::testing::Test>() to return different values depending
+// on whether the call is from the Google Test framework itself or
+// from user test code.  GetTestTypeId() is guaranteed to always
+// return the same value, as it always calls GetTypeId<>() from the
+// gtest.cc, which is within the Google Test framework.
+TypeId GetTestTypeId() {
+  return GetTypeId<Test>();
+}
+
+// The value of GetTestTypeId() as seen from within the Google Test
+// library.  This is solely for testing GetTestTypeId().
+extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId();
+
+// This predicate-formatter checks that 'results' contains a test part
+// failure of the given type and that the failure message contains the
+// given substring.
+AssertionResult HasOneFailure(const char* /* results_expr */,
+                              const char* /* type_expr */,
+                              const char* /* substr_expr */,
+                              const TestPartResultArray& results,
+                              TestPartResult::Type type,
+                              const string& substr) {
+  const std::string expected(type == TestPartResult::kFatalFailure ?
+                        "1 fatal failure" :
+                        "1 non-fatal failure");
+  Message msg;
+  if (results.size() != 1) {
+    msg << "Expected: " << expected << "\n"
+        << "  Actual: " << results.size() << " failures";
+    for (int i = 0; i < results.size(); i++) {
+      msg << "\n" << results.GetTestPartResult(i);
+    }
+    return AssertionFailure() << msg;
+  }
+
+  const TestPartResult& r = results.GetTestPartResult(0);
+  if (r.type() != type) {
+    return AssertionFailure() << "Expected: " << expected << "\n"
+                              << "  Actual:\n"
+                              << r;
+  }
+
+  if (strstr(r.message(), substr.c_str()) == NULL) {
+    return AssertionFailure() << "Expected: " << expected << " containing \""
+                              << substr << "\"\n"
+                              << "  Actual:\n"
+                              << r;
+  }
+
+  return AssertionSuccess();
+}
+
+// The constructor of SingleFailureChecker remembers where to look up
+// test part results, what type of failure we expect, and what
+// substring the failure message should contain.
+SingleFailureChecker:: SingleFailureChecker(
+    const TestPartResultArray* results,
+    TestPartResult::Type type,
+    const string& substr)
+    : results_(results),
+      type_(type),
+      substr_(substr) {}
+
+// The destructor of SingleFailureChecker verifies that the given
+// TestPartResultArray contains exactly one failure that has the given
+// type and contains the given substring.  If that's not the case, a
+// non-fatal failure will be generated.
+SingleFailureChecker::~SingleFailureChecker() {
+  EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_);
+}
+
+DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter(
+    UnitTestImpl* unit_test) : unit_test_(unit_test) {}
+
+void DefaultGlobalTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  unit_test_->current_test_result()->AddTestPartResult(result);
+  unit_test_->listeners()->repeater()->OnTestPartResult(result);
+}
+
+DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter(
+    UnitTestImpl* unit_test) : unit_test_(unit_test) {}
+
+void DefaultPerThreadTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result);
+}
+
+// Returns the global test part result reporter.
+TestPartResultReporterInterface*
+UnitTestImpl::GetGlobalTestPartResultReporter() {
+  internal::MutexLock lock(&global_test_part_result_reporter_mutex_);
+  return global_test_part_result_repoter_;
+}
+
+// Sets the global test part result reporter.
+void UnitTestImpl::SetGlobalTestPartResultReporter(
+    TestPartResultReporterInterface* reporter) {
+  internal::MutexLock lock(&global_test_part_result_reporter_mutex_);
+  global_test_part_result_repoter_ = reporter;
+}
+
+// Returns the test part result reporter for the current thread.
+TestPartResultReporterInterface*
+UnitTestImpl::GetTestPartResultReporterForCurrentThread() {
+  return per_thread_test_part_result_reporter_.get();
+}
+
+// Sets the test part result reporter for the current thread.
+void UnitTestImpl::SetTestPartResultReporterForCurrentThread(
+    TestPartResultReporterInterface* reporter) {
+  per_thread_test_part_result_reporter_.set(reporter);
+}
+
+// Gets the number of successful test cases.
+int UnitTestImpl::successful_test_case_count() const {
+  return CountIf(test_cases_, TestCasePassed);
+}
+
+// Gets the number of failed test cases.
+int UnitTestImpl::failed_test_case_count() const {
+  return CountIf(test_cases_, TestCaseFailed);
+}
+
+// Gets the number of all test cases.
+int UnitTestImpl::total_test_case_count() const {
+  return static_cast<int>(test_cases_.size());
+}
+
+// Gets the number of all test cases that contain at least one test
+// that should run.
+int UnitTestImpl::test_case_to_run_count() const {
+  return CountIf(test_cases_, ShouldRunTestCase);
+}
+
+// Gets the number of successful tests.
+int UnitTestImpl::successful_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count);
+}
+
+// Gets the number of failed tests.
+int UnitTestImpl::failed_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count);
+}
+
+// Gets the number of disabled tests that will be reported in the XML report.
+int UnitTestImpl::reportable_disabled_test_count() const {
+  return SumOverTestCaseList(test_cases_,
+                             &TestCase::reportable_disabled_test_count);
+}
+
+// Gets the number of disabled tests.
+int UnitTestImpl::disabled_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count);
+}
+
+// Gets the number of tests to be printed in the XML report.
+int UnitTestImpl::reportable_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count);
+}
+
+// Gets the number of all tests.
+int UnitTestImpl::total_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::total_test_count);
+}
+
+// Gets the number of tests that should run.
+int UnitTestImpl::test_to_run_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count);
+}
+
+// Returns the current OS stack trace as an std::string.
+//
+// The maximum number of stack frames to be included is specified by
+// the gtest_stack_trace_depth flag.  The skip_count parameter
+// specifies the number of top frames to be skipped, which doesn't
+// count against the number of frames to be included.
+//
+// For example, if Foo() calls Bar(), which in turn calls
+// CurrentOsStackTraceExceptTop(1), Foo() will be included in the
+// trace but Bar() and CurrentOsStackTraceExceptTop() won't.
+std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) {
+  return os_stack_trace_getter()->CurrentStackTrace(
+      static_cast<int>(GTEST_FLAG(stack_trace_depth)),
+      skip_count + 1
+      // Skips the user-specified number of frames plus this function
+      // itself.
+      );  // NOLINT
+}
+
+// Returns the current time in milliseconds.
+TimeInMillis GetTimeInMillis() {
+#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__)
+  // Difference between 1970-01-01 and 1601-01-01 in milliseconds.
+  // http://analogous.blogspot.com/2005/04/epoch.html
+  const TimeInMillis kJavaEpochToWinFileTimeDelta =
+    static_cast<TimeInMillis>(116444736UL) * 100000UL;
+  const DWORD kTenthMicrosInMilliSecond = 10000;
+
+  SYSTEMTIME now_systime;
+  FILETIME now_filetime;
+  ULARGE_INTEGER now_int64;
+  // TODO(kenton@google.com): Shouldn't this just use
+  //   GetSystemTimeAsFileTime()?
+  GetSystemTime(&now_systime);
+  if (SystemTimeToFileTime(&now_systime, &now_filetime)) {
+    now_int64.LowPart = now_filetime.dwLowDateTime;
+    now_int64.HighPart = now_filetime.dwHighDateTime;
+    now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) -
+      kJavaEpochToWinFileTimeDelta;
+    return now_int64.QuadPart;
+  }
+  return 0;
+#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_
+  __timeb64 now;
+
+  // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996
+  // (deprecated function) there.
+  // TODO(kenton@google.com): Use GetTickCount()?  Or use
+  //   SystemTimeToFileTime()
+  GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
+  _ftime64(&now);
+  GTEST_DISABLE_MSC_WARNINGS_POP_()
+
+  return static_cast<TimeInMillis>(now.time) * 1000 + now.millitm;
+#elif GTEST_HAS_GETTIMEOFDAY_
+  struct timeval now;
+  gettimeofday(&now, NULL);
+  return static_cast<TimeInMillis>(now.tv_sec) * 1000 + now.tv_usec / 1000;
+#else
+# error "Don't know how to get the current time on your system."
+#endif
+}
+
+// Utilities
+
+// class String.
+
+#if GTEST_OS_WINDOWS_MOBILE
+// Creates a UTF-16 wide string from the given ANSI string, allocating
+// memory using new. The caller is responsible for deleting the return
+// value using delete[]. Returns the wide string, or NULL if the
+// input is NULL.
+LPCWSTR String::AnsiToUtf16(const char* ansi) {
+  if (!ansi) return NULL;
+  const int length = strlen(ansi);
+  const int unicode_length =
+      MultiByteToWideChar(CP_ACP, 0, ansi, length,
+                          NULL, 0);
+  WCHAR* unicode = new WCHAR[unicode_length + 1];
+  MultiByteToWideChar(CP_ACP, 0, ansi, length,
+                      unicode, unicode_length);
+  unicode[unicode_length] = 0;
+  return unicode;
+}
+
+// Creates an ANSI string from the given wide string, allocating
+// memory using new. The caller is responsible for deleting the return
+// value using delete[]. Returns the ANSI string, or NULL if the
+// input is NULL.
+const char* String::Utf16ToAnsi(LPCWSTR utf16_str)  {
+  if (!utf16_str) return NULL;
+  const int ansi_length =
+      WideCharToMultiByte(CP_ACP, 0, utf16_str, -1,
+                          NULL, 0, NULL, NULL);
+  char* ansi = new char[ansi_length + 1];
+  WideCharToMultiByte(CP_ACP, 0, utf16_str, -1,
+                      ansi, ansi_length, NULL, NULL);
+  ansi[ansi_length] = 0;
+  return ansi;
+}
+
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+// Compares two C strings.  Returns true iff they have the same content.
+//
+// Unlike strcmp(), this function can handle NULL argument(s).  A NULL
+// C string is considered different to any non-NULL C string,
+// including the empty string.
+bool String::CStringEquals(const char * lhs, const char * rhs) {
+  if ( lhs == NULL ) return rhs == NULL;
+
+  if ( rhs == NULL ) return false;
+
+  return strcmp(lhs, rhs) == 0;
+}
+
+#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING
+
+// Converts an array of wide chars to a narrow string using the UTF-8
+// encoding, and streams the result to the given Message object.
+static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length,
+                                     Message* msg) {
+  for (size_t i = 0; i != length; ) {  // NOLINT
+    if (wstr[i] != L'\0') {
+      *msg << WideStringToUtf8(wstr + i, static_cast<int>(length - i));
+      while (i != length && wstr[i] != L'\0')
+        i++;
+    } else {
+      *msg << '\0';
+      i++;
+    }
+  }
+}
+
+#endif  // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING
+
+void SplitString(const ::std::string& str, char delimiter,
+                 ::std::vector< ::std::string>* dest) {
+  ::std::vector< ::std::string> parsed;
+  ::std::string::size_type pos = 0;
+  while (::testing::internal::AlwaysTrue()) {
+    const ::std::string::size_type colon = str.find(delimiter, pos);
+    if (colon == ::std::string::npos) {
+      parsed.push_back(str.substr(pos));
+      break;
+    } else {
+      parsed.push_back(str.substr(pos, colon - pos));
+      pos = colon + 1;
+    }
+  }
+  dest->swap(parsed);
+}
+
+}  // namespace internal
+
+// Constructs an empty Message.
+// We allocate the stringstream separately because otherwise each use of
+// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's
+// stack frame leading to huge stack frames in some cases; gcc does not reuse
+// the stack space.
+Message::Message() : ss_(new ::std::stringstream) {
+  // By default, we want there to be enough precision when printing
+  // a double to a Message.
+  *ss_ << std::setprecision(std::numeric_limits<double>::digits10 + 2);
+}
+
+// These two overloads allow streaming a wide C string to a Message
+// using the UTF-8 encoding.
+Message& Message::operator <<(const wchar_t* wide_c_str) {
+  return *this << internal::String::ShowWideCString(wide_c_str);
+}
+Message& Message::operator <<(wchar_t* wide_c_str) {
+  return *this << internal::String::ShowWideCString(wide_c_str);
+}
+
+#if GTEST_HAS_STD_WSTRING
+// Converts the given wide string to a narrow string using the UTF-8
+// encoding, and streams the result to this Message object.
+Message& Message::operator <<(const ::std::wstring& wstr) {
+  internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this);
+  return *this;
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+#if GTEST_HAS_GLOBAL_WSTRING
+// Converts the given wide string to a narrow string using the UTF-8
+// encoding, and streams the result to this Message object.
+Message& Message::operator <<(const ::wstring& wstr) {
+  internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this);
+  return *this;
+}
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+// Gets the text streamed to this object so far as an std::string.
+// Each '\0' character in the buffer is replaced with "\\0".
+std::string Message::GetString() const {
+  return internal::StringStreamToString(ss_.get());
+}
+
+// AssertionResult constructors.
+// Used in EXPECT_TRUE/FALSE(assertion_result).
+AssertionResult::AssertionResult(const AssertionResult& other)
+    : success_(other.success_),
+      message_(other.message_.get() != NULL ?
+               new ::std::string(*other.message_) :
+               static_cast< ::std::string*>(NULL)) {
+}
+
+// Swaps two AssertionResults.
+void AssertionResult::swap(AssertionResult& other) {
+  using std::swap;
+  swap(success_, other.success_);
+  swap(message_, other.message_);
+}
+
+// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE.
+AssertionResult AssertionResult::operator!() const {
+  AssertionResult negation(!success_);
+  if (message_.get() != NULL)
+    negation << *message_;
+  return negation;
+}
+
+// Makes a successful assertion result.
+AssertionResult AssertionSuccess() {
+  return AssertionResult(true);
+}
+
+// Makes a failed assertion result.
+AssertionResult AssertionFailure() {
+  return AssertionResult(false);
+}
+
+// Makes a failed assertion result with the given failure message.
+// Deprecated; use AssertionFailure() << message.
+AssertionResult AssertionFailure(const Message& message) {
+  return AssertionFailure() << message;
+}
+
+namespace internal {
+
+namespace edit_distance {
+std::vector<EditType> CalculateOptimalEdits(const std::vector<size_t>& left,
+                                            const std::vector<size_t>& right) {
+  std::vector<std::vector<double> > costs(
+      left.size() + 1, std::vector<double>(right.size() + 1));
+  std::vector<std::vector<EditType> > best_move(
+      left.size() + 1, std::vector<EditType>(right.size() + 1));
+
+  // Populate for empty right.
+  for (size_t l_i = 0; l_i < costs.size(); ++l_i) {
+    costs[l_i][0] = static_cast<double>(l_i);
+    best_move[l_i][0] = kRemove;
+  }
+  // Populate for empty left.
+  for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) {
+    costs[0][r_i] = static_cast<double>(r_i);
+    best_move[0][r_i] = kAdd;
+  }
+
+  for (size_t l_i = 0; l_i < left.size(); ++l_i) {
+    for (size_t r_i = 0; r_i < right.size(); ++r_i) {
+      if (left[l_i] == right[r_i]) {
+        // Found a match. Consume it.
+        costs[l_i + 1][r_i + 1] = costs[l_i][r_i];
+        best_move[l_i + 1][r_i + 1] = kMatch;
+        continue;
+      }
+
+      const double add = costs[l_i + 1][r_i];
+      const double remove = costs[l_i][r_i + 1];
+      const double replace = costs[l_i][r_i];
+      if (add < remove && add < replace) {
+        costs[l_i + 1][r_i + 1] = add + 1;
+        best_move[l_i + 1][r_i + 1] = kAdd;
+      } else if (remove < add && remove < replace) {
+        costs[l_i + 1][r_i + 1] = remove + 1;
+        best_move[l_i + 1][r_i + 1] = kRemove;
+      } else {
+        // We make replace a little more expensive than add/remove to lower
+        // their priority.
+        costs[l_i + 1][r_i + 1] = replace + 1.00001;
+        best_move[l_i + 1][r_i + 1] = kReplace;
+      }
+    }
+  }
+
+  // Reconstruct the best path. We do it in reverse order.
+  std::vector<EditType> best_path;
+  for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) {
+    EditType move = best_move[l_i][r_i];
+    best_path.push_back(move);
+    l_i -= move != kAdd;
+    r_i -= move != kRemove;
+  }
+  std::reverse(best_path.begin(), best_path.end());
+  return best_path;
+}
+
+namespace {
+
+// Helper class to convert string into ids with deduplication.
+class InternalStrings {
+ public:
+  size_t GetId(const std::string& str) {
+    IdMap::iterator it = ids_.find(str);
+    if (it != ids_.end()) return it->second;
+    size_t id = ids_.size();
+    return ids_[str] = id;
+  }
+
+ private:
+  typedef std::map<std::string, size_t> IdMap;
+  IdMap ids_;
+};
+
+}  // namespace
+
+std::vector<EditType> CalculateOptimalEdits(
+    const std::vector<std::string>& left,
+    const std::vector<std::string>& right) {
+  std::vector<size_t> left_ids, right_ids;
+  {
+    InternalStrings intern_table;
+    for (size_t i = 0; i < left.size(); ++i) {
+      left_ids.push_back(intern_table.GetId(left[i]));
+    }
+    for (size_t i = 0; i < right.size(); ++i) {
+      right_ids.push_back(intern_table.GetId(right[i]));
+    }
+  }
+  return CalculateOptimalEdits(left_ids, right_ids);
+}
+
+namespace {
+
+// Helper class that holds the state for one hunk and prints it out to the
+// stream.
+// It reorders adds/removes when possible to group all removes before all
+// adds. It also adds the hunk header before printint into the stream.
+class Hunk {
+ public:
+  Hunk(size_t left_start, size_t right_start)
+      : left_start_(left_start),
+        right_start_(right_start),
+        adds_(),
+        removes_(),
+        common_() {}
+
+  void PushLine(char edit, const char* line) {
+    switch (edit) {
+      case ' ':
+        ++common_;
+        FlushEdits();
+        hunk_.push_back(std::make_pair(' ', line));
+        break;
+      case '-':
+        ++removes_;
+        hunk_removes_.push_back(std::make_pair('-', line));
+        break;
+      case '+':
+        ++adds_;
+        hunk_adds_.push_back(std::make_pair('+', line));
+        break;
+    }
+  }
+
+  void PrintTo(std::ostream* os) {
+    PrintHeader(os);
+    FlushEdits();
+    for (std::list<std::pair<char, const char*> >::const_iterator it =
+             hunk_.begin();
+         it != hunk_.end(); ++it) {
+      *os << it->first << it->second << "\n";
+    }
+  }
+
+  bool has_edits() const { return adds_ || removes_; }
+
+ private:
+  void FlushEdits() {
+    hunk_.splice(hunk_.end(), hunk_removes_);
+    hunk_.splice(hunk_.end(), hunk_adds_);
+  }
+
+  // Print a unified diff header for one hunk.
+  // The format is
+  //   "@@ -<left_start>,<left_length> +<right_start>,<right_length> @@"
+  // where the left/right parts are ommitted if unnecessary.
+  void PrintHeader(std::ostream* ss) const {
+    *ss << "@@ ";
+    if (removes_) {
+      *ss << "-" << left_start_ << "," << (removes_ + common_);
+    }
+    if (removes_ && adds_) {
+      *ss << " ";
+    }
+    if (adds_) {
+      *ss << "+" << right_start_ << "," << (adds_ + common_);
+    }
+    *ss << " @@\n";
+  }
+
+  size_t left_start_, right_start_;
+  size_t adds_, removes_, common_;
+  std::list<std::pair<char, const char*> > hunk_, hunk_adds_, hunk_removes_;
+};
+
+}  // namespace
+
+// Create a list of diff hunks in Unified diff format.
+// Each hunk has a header generated by PrintHeader above plus a body with
+// lines prefixed with ' ' for no change, '-' for deletion and '+' for
+// addition.
+// 'context' represents the desired unchanged prefix/suffix around the diff.
+// If two hunks are close enough that their contexts overlap, then they are
+// joined into one hunk.
+std::string CreateUnifiedDiff(const std::vector<std::string>& left,
+                              const std::vector<std::string>& right,
+                              size_t context) {
+  const std::vector<EditType> edits = CalculateOptimalEdits(left, right);
+
+  size_t l_i = 0, r_i = 0, edit_i = 0;
+  std::stringstream ss;
+  while (edit_i < edits.size()) {
+    // Find first edit.
+    while (edit_i < edits.size() && edits[edit_i] == kMatch) {
+      ++l_i;
+      ++r_i;
+      ++edit_i;
+    }
+
+    // Find the first line to include in the hunk.
+    const size_t prefix_context = std::min(l_i, context);
+    Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1);
+    for (size_t i = prefix_context; i > 0; --i) {
+      hunk.PushLine(' ', left[l_i - i].c_str());
+    }
+
+    // Iterate the edits until we found enough suffix for the hunk or the input
+    // is over.
+    size_t n_suffix = 0;
+    for (; edit_i < edits.size(); ++edit_i) {
+      if (n_suffix >= context) {
+        // Continue only if the next hunk is very close.
+        std::vector<EditType>::const_iterator it = edits.begin() + edit_i;
+        while (it != edits.end() && *it == kMatch) ++it;
+        if (it == edits.end() || (it - edits.begin()) - edit_i >= context) {
+          // There is no next edit or it is too far away.
+          break;
+        }
+      }
+
+      EditType edit = edits[edit_i];
+      // Reset count when a non match is found.
+      n_suffix = edit == kMatch ? n_suffix + 1 : 0;
+
+      if (edit == kMatch || edit == kRemove || edit == kReplace) {
+        hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str());
+      }
+      if (edit == kAdd || edit == kReplace) {
+        hunk.PushLine('+', right[r_i].c_str());
+      }
+
+      // Advance indices, depending on edit type.
+      l_i += edit != kAdd;
+      r_i += edit != kRemove;
+    }
+
+    if (!hunk.has_edits()) {
+      // We are done. We don't want this hunk.
+      break;
+    }
+
+    hunk.PrintTo(&ss);
+  }
+  return ss.str();
+}
+
+}  // namespace edit_distance
+
+namespace {
+
+// The string representation of the values received in EqFailure() are already
+// escaped. Split them on escaped '\n' boundaries. Leave all other escaped
+// characters the same.
+std::vector<std::string> SplitEscapedString(const std::string& str) {
+  std::vector<std::string> lines;
+  size_t start = 0, end = str.size();
+  if (end > 2 && str[0] == '"' && str[end - 1] == '"') {
+    ++start;
+    --end;
+  }
+  bool escaped = false;
+  for (size_t i = start; i + 1 < end; ++i) {
+    if (escaped) {
+      escaped = false;
+      if (str[i] == 'n') {
+        lines.push_back(str.substr(start, i - start - 1));
+        start = i + 1;
+      }
+    } else {
+      escaped = str[i] == '\\';
+    }
+  }
+  lines.push_back(str.substr(start, end - start));
+  return lines;
+}
+
+}  // namespace
+
+// Constructs and returns the message for an equality assertion
+// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure.
+//
+// The first four parameters are the expressions used in the assertion
+// and their values, as strings.  For example, for ASSERT_EQ(foo, bar)
+// where foo is 5 and bar is 6, we have:
+//
+//   lhs_expression: "foo"
+//   rhs_expression: "bar"
+//   lhs_value:      "5"
+//   rhs_value:      "6"
+//
+// The ignoring_case parameter is true iff the assertion is a
+// *_STRCASEEQ*.  When it's true, the string "Ignoring case" will
+// be inserted into the message.
+AssertionResult EqFailure(const char* lhs_expression,
+                          const char* rhs_expression,
+                          const std::string& lhs_value,
+                          const std::string& rhs_value,
+                          bool ignoring_case) {
+  Message msg;
+  msg << "      Expected: " << lhs_expression;
+  if (lhs_value != lhs_expression) {
+    msg << "\n      Which is: " << lhs_value;
+  }
+  msg << "\nTo be equal to: " << rhs_expression;
+  if (rhs_value != rhs_expression) {
+    msg << "\n      Which is: " << rhs_value;
+  }
+
+  if (ignoring_case) {
+    msg << "\nIgnoring case";
+  }
+
+  if (!lhs_value.empty() && !rhs_value.empty()) {
+    const std::vector<std::string> lhs_lines =
+        SplitEscapedString(lhs_value);
+    const std::vector<std::string> rhs_lines =
+        SplitEscapedString(rhs_value);
+    if (lhs_lines.size() > 1 || rhs_lines.size() > 1) {
+      msg << "\nWith diff:\n"
+          << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines);
+    }
+  }
+
+  return AssertionFailure() << msg;
+}
+
+// Constructs a failure message for Boolean assertions such as EXPECT_TRUE.
+std::string GetBoolAssertionFailureMessage(
+    const AssertionResult& assertion_result,
+    const char* expression_text,
+    const char* actual_predicate_value,
+    const char* expected_predicate_value) {
+  const char* actual_message = assertion_result.message();
+  Message msg;
+  msg << "Value of: " << expression_text
+      << "\n  Actual: " << actual_predicate_value;
+  if (actual_message[0] != '\0')
+    msg << " (" << actual_message << ")";
+  msg << "\nExpected: " << expected_predicate_value;
+  return msg.GetString();
+}
+
+// Helper function for implementing ASSERT_NEAR.
+AssertionResult DoubleNearPredFormat(const char* expr1,
+                                     const char* expr2,
+                                     const char* abs_error_expr,
+                                     double val1,
+                                     double val2,
+                                     double abs_error) {
+  const double diff = fabs(val1 - val2);
+  if (diff <= abs_error) return AssertionSuccess();
+
+  // TODO(wan): do not print the value of an expression if it's
+  // already a literal.
+  return AssertionFailure()
+      << "The difference between " << expr1 << " and " << expr2
+      << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n"
+      << expr1 << " evaluates to " << val1 << ",\n"
+      << expr2 << " evaluates to " << val2 << ", and\n"
+      << abs_error_expr << " evaluates to " << abs_error << ".";
+}
+
+
+// Helper template for implementing FloatLE() and DoubleLE().
+template <typename RawType>
+AssertionResult FloatingPointLE(const char* expr1,
+                                const char* expr2,
+                                RawType val1,
+                                RawType val2) {
+  // Returns success if val1 is less than val2,
+  if (val1 < val2) {
+    return AssertionSuccess();
+  }
+
+  // or if val1 is almost equal to val2.
+  const FloatingPoint<RawType> lhs(val1), rhs(val2);
+  if (lhs.AlmostEquals(rhs)) {
+    return AssertionSuccess();
+  }
+
+  // Note that the above two checks will both fail if either val1 or
+  // val2 is NaN, as the IEEE floating-point standard requires that
+  // any predicate involving a NaN must return false.
+
+  ::std::stringstream val1_ss;
+  val1_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
+          << val1;
+
+  ::std::stringstream val2_ss;
+  val2_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
+          << val2;
+
+  return AssertionFailure()
+      << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n"
+      << "  Actual: " << StringStreamToString(&val1_ss) << " vs "
+      << StringStreamToString(&val2_ss);
+}
+
+}  // namespace internal
+
+// Asserts that val1 is less than, or almost equal to, val2.  Fails
+// otherwise.  In particular, it fails if either val1 or val2 is NaN.
+AssertionResult FloatLE(const char* expr1, const char* expr2,
+                        float val1, float val2) {
+  return internal::FloatingPointLE<float>(expr1, expr2, val1, val2);
+}
+
+// Asserts that val1 is less than, or almost equal to, val2.  Fails
+// otherwise.  In particular, it fails if either val1 or val2 is NaN.
+AssertionResult DoubleLE(const char* expr1, const char* expr2,
+                         double val1, double val2) {
+  return internal::FloatingPointLE<double>(expr1, expr2, val1, val2);
+}
+
+namespace internal {
+
+// The helper function for {ASSERT|EXPECT}_EQ with int or enum
+// arguments.
+AssertionResult CmpHelperEQ(const char* lhs_expression,
+                            const char* rhs_expression,
+                            BiggestInt lhs,
+                            BiggestInt rhs) {
+  if (lhs == rhs) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   FormatForComparisonFailureMessage(lhs, rhs),
+                   FormatForComparisonFailureMessage(rhs, lhs),
+                   false);
+}
+
+// A macro for implementing the helper functions needed to implement
+// ASSERT_?? and EXPECT_?? with integer or enum arguments.  It is here
+// just to avoid copy-and-paste of similar code.
+#define GTEST_IMPL_CMP_HELPER_(op_name, op)\
+AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \
+                                   BiggestInt val1, BiggestInt val2) {\
+  if (val1 op val2) {\
+    return AssertionSuccess();\
+  } else {\
+    return AssertionFailure() \
+        << "Expected: (" << expr1 << ") " #op " (" << expr2\
+        << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\
+        << " vs " << FormatForComparisonFailureMessage(val2, val1);\
+  }\
+}
+
+// Implements the helper function for {ASSERT|EXPECT}_NE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(NE, !=)
+// Implements the helper function for {ASSERT|EXPECT}_LE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(LE, <=)
+// Implements the helper function for {ASSERT|EXPECT}_LT with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(LT, < )
+// Implements the helper function for {ASSERT|EXPECT}_GE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(GE, >=)
+// Implements the helper function for {ASSERT|EXPECT}_GT with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(GT, > )
+
+#undef GTEST_IMPL_CMP_HELPER_
+
+// The helper function for {ASSERT|EXPECT}_STREQ.
+AssertionResult CmpHelperSTREQ(const char* lhs_expression,
+                               const char* rhs_expression,
+                               const char* lhs,
+                               const char* rhs) {
+  if (String::CStringEquals(lhs, rhs)) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   PrintToString(lhs),
+                   PrintToString(rhs),
+                   false);
+}
+
+// The helper function for {ASSERT|EXPECT}_STRCASEEQ.
+AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression,
+                                   const char* rhs_expression,
+                                   const char* lhs,
+                                   const char* rhs) {
+  if (String::CaseInsensitiveCStringEquals(lhs, rhs)) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   PrintToString(lhs),
+                   PrintToString(rhs),
+                   true);
+}
+
+// The helper function for {ASSERT|EXPECT}_STRNE.
+AssertionResult CmpHelperSTRNE(const char* s1_expression,
+                               const char* s2_expression,
+                               const char* s1,
+                               const char* s2) {
+  if (!String::CStringEquals(s1, s2)) {
+    return AssertionSuccess();
+  } else {
+    return AssertionFailure() << "Expected: (" << s1_expression << ") != ("
+                              << s2_expression << "), actual: \""
+                              << s1 << "\" vs \"" << s2 << "\"";
+  }
+}
+
+// The helper function for {ASSERT|EXPECT}_STRCASENE.
+AssertionResult CmpHelperSTRCASENE(const char* s1_expression,
+                                   const char* s2_expression,
+                                   const char* s1,
+                                   const char* s2) {
+  if (!String::CaseInsensitiveCStringEquals(s1, s2)) {
+    return AssertionSuccess();
+  } else {
+    return AssertionFailure()
+        << "Expected: (" << s1_expression << ") != ("
+        << s2_expression << ") (ignoring case), actual: \""
+        << s1 << "\" vs \"" << s2 << "\"";
+  }
+}
+
+}  // namespace internal
+
+namespace {
+
+// Helper functions for implementing IsSubString() and IsNotSubstring().
+
+// This group of overloaded functions return true iff needle is a
+// substring of haystack.  NULL is considered a substring of itself
+// only.
+
+bool IsSubstringPred(const char* needle, const char* haystack) {
+  if (needle == NULL || haystack == NULL)
+    return needle == haystack;
+
+  return strstr(haystack, needle) != NULL;
+}
+
+bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) {
+  if (needle == NULL || haystack == NULL)
+    return needle == haystack;
+
+  return wcsstr(haystack, needle) != NULL;
+}
+
+// StringType here can be either ::std::string or ::std::wstring.
+template <typename StringType>
+bool IsSubstringPred(const StringType& needle,
+                     const StringType& haystack) {
+  return haystack.find(needle) != StringType::npos;
+}
+
+// This function implements either IsSubstring() or IsNotSubstring(),
+// depending on the value of the expected_to_be_substring parameter.
+// StringType here can be const char*, const wchar_t*, ::std::string,
+// or ::std::wstring.
+template <typename StringType>
+AssertionResult IsSubstringImpl(
+    bool expected_to_be_substring,
+    const char* needle_expr, const char* haystack_expr,
+    const StringType& needle, const StringType& haystack) {
+  if (IsSubstringPred(needle, haystack) == expected_to_be_substring)
+    return AssertionSuccess();
+
+  const bool is_wide_string = sizeof(needle[0]) > 1;
+  const char* const begin_string_quote = is_wide_string ? "L\"" : "\"";
+  return AssertionFailure()
+      << "Value of: " << needle_expr << "\n"
+      << "  Actual: " << begin_string_quote << needle << "\"\n"
+      << "Expected: " << (expected_to_be_substring ? "" : "not ")
+      << "a substring of " << haystack_expr << "\n"
+      << "Which is: " << begin_string_quote << haystack << "\"";
+}
+
+}  // namespace
+
+// IsSubstring() and IsNotSubstring() check whether needle is a
+// substring of haystack (NULL is considered a substring of itself
+// only), and return an appropriate error message when they fail.
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const char* needle, const char* haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const wchar_t* needle, const wchar_t* haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const char* needle, const char* haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const wchar_t* needle, const wchar_t* haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::string& needle, const ::std::string& haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::string& needle, const ::std::string& haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+#if GTEST_HAS_STD_WSTRING
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::wstring& needle, const ::std::wstring& haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::wstring& needle, const ::std::wstring& haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+namespace internal {
+
+#if GTEST_OS_WINDOWS
+
+namespace {
+
+// Helper function for IsHRESULT{SuccessFailure} predicates
+AssertionResult HRESULTFailureHelper(const char* expr,
+                                     const char* expected,
+                                     long hr) {  // NOLINT
+# if GTEST_OS_WINDOWS_MOBILE
+
+  // Windows CE doesn't support FormatMessage.
+  const char error_text[] = "";
+
+# else
+
+  // Looks up the human-readable system message for the HRESULT code
+  // and since we're not passing any params to FormatMessage, we don't
+  // want inserts expanded.
+  const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS;
+  const DWORD kBufSize = 4096;
+  // Gets the system's human readable message string for this HRESULT.
+  char error_text[kBufSize] = { '\0' };
+  DWORD message_length = ::FormatMessageA(kFlags,
+                                          0,  // no source, we're asking system
+                                          hr,  // the error
+                                          0,  // no line width restrictions
+                                          error_text,  // output buffer
+                                          kBufSize,  // buf size
+                                          NULL);  // no arguments for inserts
+  // Trims tailing white space (FormatMessage leaves a trailing CR-LF)
+  for (; message_length && IsSpace(error_text[message_length - 1]);
+          --message_length) {
+    error_text[message_length - 1] = '\0';
+  }
+
+# endif  // GTEST_OS_WINDOWS_MOBILE
+
+  const std::string error_hex("0x" + String::FormatHexInt(hr));
+  return ::testing::AssertionFailure()
+      << "Expected: " << expr << " " << expected << ".\n"
+      << "  Actual: " << error_hex << " " << error_text << "\n";
+}
+
+}  // namespace
+
+AssertionResult IsHRESULTSuccess(const char* expr, long hr) {  // NOLINT
+  if (SUCCEEDED(hr)) {
+    return AssertionSuccess();
+  }
+  return HRESULTFailureHelper(expr, "succeeds", hr);
+}
+
+AssertionResult IsHRESULTFailure(const char* expr, long hr) {  // NOLINT
+  if (FAILED(hr)) {
+    return AssertionSuccess();
+  }
+  return HRESULTFailureHelper(expr, "fails", hr);
+}
+
+#endif  // GTEST_OS_WINDOWS
+
+// Utility functions for encoding Unicode text (wide strings) in
+// UTF-8.
+
+// A Unicode code-point can have upto 21 bits, and is encoded in UTF-8
+// like this:
+//
+// Code-point length   Encoding
+//   0 -  7 bits       0xxxxxxx
+//   8 - 11 bits       110xxxxx 10xxxxxx
+//  12 - 16 bits       1110xxxx 10xxxxxx 10xxxxxx
+//  17 - 21 bits       11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+
+// The maximum code-point a one-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint1 = (static_cast<UInt32>(1) <<  7) - 1;
+
+// The maximum code-point a two-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint2 = (static_cast<UInt32>(1) << (5 + 6)) - 1;
+
+// The maximum code-point a three-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint3 = (static_cast<UInt32>(1) << (4 + 2*6)) - 1;
+
+// The maximum code-point a four-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint4 = (static_cast<UInt32>(1) << (3 + 3*6)) - 1;
+
+// Chops off the n lowest bits from a bit pattern.  Returns the n
+// lowest bits.  As a side effect, the original bit pattern will be
+// shifted to the right by n bits.
+inline UInt32 ChopLowBits(UInt32* bits, int n) {
+  const UInt32 low_bits = *bits & ((static_cast<UInt32>(1) << n) - 1);
+  *bits >>= n;
+  return low_bits;
+}
+
+// Converts a Unicode code point to a narrow string in UTF-8 encoding.
+// code_point parameter is of type UInt32 because wchar_t may not be
+// wide enough to contain a code point.
+// If the code_point is not a valid Unicode code point
+// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted
+// to "(Invalid Unicode 0xXXXXXXXX)".
+std::string CodePointToUtf8(UInt32 code_point) {
+  if (code_point > kMaxCodePoint4) {
+    return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")";
+  }
+
+  char str[5];  // Big enough for the largest valid code point.
+  if (code_point <= kMaxCodePoint1) {
+    str[1] = '\0';
+    str[0] = static_cast<char>(code_point);                          // 0xxxxxxx
+  } else if (code_point <= kMaxCodePoint2) {
+    str[2] = '\0';
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xC0 | code_point);                   // 110xxxxx
+  } else if (code_point <= kMaxCodePoint3) {
+    str[3] = '\0';
+    str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xE0 | code_point);                   // 1110xxxx
+  } else {  // code_point <= kMaxCodePoint4
+    str[4] = '\0';
+    str[3] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xF0 | code_point);                   // 11110xxx
+  }
+  return str;
+}
+
+// The following two functions only make sense if the the system
+// uses UTF-16 for wide string encoding. All supported systems
+// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16.
+
+// Determines if the arguments constitute UTF-16 surrogate pair
+// and thus should be combined into a single Unicode code point
+// using CreateCodePointFromUtf16SurrogatePair.
+inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) {
+  return sizeof(wchar_t) == 2 &&
+      (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00;
+}
+
+// Creates a Unicode code point from UTF16 surrogate pair.
+inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first,
+                                                    wchar_t second) {
+  const UInt32 mask = (1 << 10) - 1;
+  return (sizeof(wchar_t) == 2) ?
+      (((first & mask) << 10) | (second & mask)) + 0x10000 :
+      // This function should not be called when the condition is
+      // false, but we provide a sensible default in case it is.
+      static_cast<UInt32>(first);
+}
+
+// Converts a wide string to a narrow string in UTF-8 encoding.
+// The wide string is assumed to have the following encoding:
+//   UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS)
+//   UTF-32 if sizeof(wchar_t) == 4 (on Linux)
+// Parameter str points to a null-terminated wide string.
+// Parameter num_chars may additionally limit the number
+// of wchar_t characters processed. -1 is used when the entire string
+// should be processed.
+// If the string contains code points that are not valid Unicode code points
+// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output
+// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding
+// and contains invalid UTF-16 surrogate pairs, values in those pairs
+// will be encoded as individual Unicode characters from Basic Normal Plane.
+std::string WideStringToUtf8(const wchar_t* str, int num_chars) {
+  if (num_chars == -1)
+    num_chars = static_cast<int>(wcslen(str));
+
+  ::std::stringstream stream;
+  for (int i = 0; i < num_chars; ++i) {
+    UInt32 unicode_code_point;
+
+    if (str[i] == L'\0') {
+      break;
+    } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) {
+      unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i],
+                                                                 str[i + 1]);
+      i++;
+    } else {
+      unicode_code_point = static_cast<UInt32>(str[i]);
+    }
+
+    stream << CodePointToUtf8(unicode_code_point);
+  }
+  return StringStreamToString(&stream);
+}
+
+// Converts a wide C string to an std::string using the UTF-8 encoding.
+// NULL will be converted to "(null)".
+std::string String::ShowWideCString(const wchar_t * wide_c_str) {
+  if (wide_c_str == NULL)  return "(null)";
+
+  return internal::WideStringToUtf8(wide_c_str, -1);
+}
+
+// Compares two wide C strings.  Returns true iff they have the same
+// content.
+//
+// Unlike wcscmp(), this function can handle NULL argument(s).  A NULL
+// C string is considered different to any non-NULL C string,
+// including the empty string.
+bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) {
+  if (lhs == NULL) return rhs == NULL;
+
+  if (rhs == NULL) return false;
+
+  return wcscmp(lhs, rhs) == 0;
+}
+
+// Helper function for *_STREQ on wide strings.
+AssertionResult CmpHelperSTREQ(const char* lhs_expression,
+                               const char* rhs_expression,
+                               const wchar_t* lhs,
+                               const wchar_t* rhs) {
+  if (String::WideCStringEquals(lhs, rhs)) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   PrintToString(lhs),
+                   PrintToString(rhs),
+                   false);
+}
+
+// Helper function for *_STRNE on wide strings.
+AssertionResult CmpHelperSTRNE(const char* s1_expression,
+                               const char* s2_expression,
+                               const wchar_t* s1,
+                               const wchar_t* s2) {
+  if (!String::WideCStringEquals(s1, s2)) {
+    return AssertionSuccess();
+  }
+
+  return AssertionFailure() << "Expected: (" << s1_expression << ") != ("
+                            << s2_expression << "), actual: "
+                            << PrintToString(s1)
+                            << " vs " << PrintToString(s2);
+}
+
+// Compares two C strings, ignoring case.  Returns true iff they have
+// the same content.
+//
+// Unlike strcasecmp(), this function can handle NULL argument(s).  A
+// NULL C string is considered different to any non-NULL C string,
+// including the empty string.
+bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) {
+  if (lhs == NULL)
+    return rhs == NULL;
+  if (rhs == NULL)
+    return false;
+  return posix::StrCaseCmp(lhs, rhs) == 0;
+}
+
+  // Compares two wide C strings, ignoring case.  Returns true iff they
+  // have the same content.
+  //
+  // Unlike wcscasecmp(), this function can handle NULL argument(s).
+  // A NULL C string is considered different to any non-NULL wide C string,
+  // including the empty string.
+  // NB: The implementations on different platforms slightly differ.
+  // On windows, this method uses _wcsicmp which compares according to LC_CTYPE
+  // environment variable. On GNU platform this method uses wcscasecmp
+  // which compares according to LC_CTYPE category of the current locale.
+  // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the
+  // current locale.
+bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs,
+                                              const wchar_t* rhs) {
+  if (lhs == NULL) return rhs == NULL;
+
+  if (rhs == NULL) return false;
+
+#if GTEST_OS_WINDOWS
+  return _wcsicmp(lhs, rhs) == 0;
+#elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID
+  return wcscasecmp(lhs, rhs) == 0;
+#else
+  // Android, Mac OS X and Cygwin don't define wcscasecmp.
+  // Other unknown OSes may not define it either.
+  wint_t left, right;
+  do {
+    left = towlower(*lhs++);
+    right = towlower(*rhs++);
+  } while (left && left == right);
+  return left == right;
+#endif  // OS selector
+}
+
+// Returns true iff str ends with the given suffix, ignoring case.
+// Any string is considered to end with an empty suffix.
+bool String::EndsWithCaseInsensitive(
+    const std::string& str, const std::string& suffix) {
+  const size_t str_len = str.length();
+  const size_t suffix_len = suffix.length();
+  return (str_len >= suffix_len) &&
+         CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len,
+                                      suffix.c_str());
+}
+
+// Formats an int value as "%02d".
+std::string String::FormatIntWidth2(int value) {
+  std::stringstream ss;
+  ss << std::setfill('0') << std::setw(2) << value;
+  return ss.str();
+}
+
+// Formats an int value as "%X".
+std::string String::FormatHexInt(int value) {
+  std::stringstream ss;
+  ss << std::hex << std::uppercase << value;
+  return ss.str();
+}
+
+// Formats a byte as "%02X".
+std::string String::FormatByte(unsigned char value) {
+  std::stringstream ss;
+  ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase
+     << static_cast<unsigned int>(value);
+  return ss.str();
+}
+
+// Converts the buffer in a stringstream to an std::string, converting NUL
+// bytes to "\\0" along the way.
+std::string StringStreamToString(::std::stringstream* ss) {
+  const ::std::string& str = ss->str();
+  const char* const start = str.c_str();
+  const char* const end = start + str.length();
+
+  std::string result;
+  result.reserve(2 * (end - start));
+  for (const char* ch = start; ch != end; ++ch) {
+    if (*ch == '\0') {
+      result += "\\0";  // Replaces NUL with "\\0";
+    } else {
+      result += *ch;
+    }
+  }
+
+  return result;
+}
+
+// Appends the user-supplied message to the Google-Test-generated message.
+std::string AppendUserMessage(const std::string& gtest_msg,
+                              const Message& user_msg) {
+  // Appends the user message if it's non-empty.
+  const std::string user_msg_string = user_msg.GetString();
+  if (user_msg_string.empty()) {
+    return gtest_msg;
+  }
+
+  return gtest_msg + "\n" + user_msg_string;
+}
+
+}  // namespace internal
+
+// class TestResult
+
+// Creates an empty TestResult.
+TestResult::TestResult()
+    : death_test_count_(0),
+      elapsed_time_(0) {
+}
+
+// D'tor.
+TestResult::~TestResult() {
+}
+
+// Returns the i-th test part result among all the results. i can
+// range from 0 to total_part_count() - 1. If i is not in that range,
+// aborts the program.
+const TestPartResult& TestResult::GetTestPartResult(int i) const {
+  if (i < 0 || i >= total_part_count())
+    internal::posix::Abort();
+  return test_part_results_.at(i);
+}
+
+// Returns the i-th test property. i can range from 0 to
+// test_property_count() - 1. If i is not in that range, aborts the
+// program.
+const TestProperty& TestResult::GetTestProperty(int i) const {
+  if (i < 0 || i >= test_property_count())
+    internal::posix::Abort();
+  return test_properties_.at(i);
+}
+
+// Clears the test part results.
+void TestResult::ClearTestPartResults() {
+  test_part_results_.clear();
+}
+
+// Adds a test part result to the list.
+void TestResult::AddTestPartResult(const TestPartResult& test_part_result) {
+  test_part_results_.push_back(test_part_result);
+}
+
+// Adds a test property to the list. If a property with the same key as the
+// supplied property is already represented, the value of this test_property
+// replaces the old value for that key.
+void TestResult::RecordProperty(const std::string& xml_element,
+                                const TestProperty& test_property) {
+  if (!ValidateTestProperty(xml_element, test_property)) {
+    return;
+  }
+  internal::MutexLock lock(&test_properites_mutex_);
+  const std::vector<TestProperty>::iterator property_with_matching_key =
+      std::find_if(test_properties_.begin(), test_properties_.end(),
+                   internal::TestPropertyKeyIs(test_property.key()));
+  if (property_with_matching_key == test_properties_.end()) {
+    test_properties_.push_back(test_property);
+    return;
+  }
+  property_with_matching_key->SetValue(test_property.value());
+}
+
+// The list of reserved attributes used in the <testsuites> element of XML
+// output.
+static const char* const kReservedTestSuitesAttributes[] = {
+  "disabled",
+  "errors",
+  "failures",
+  "name",
+  "random_seed",
+  "tests",
+  "time",
+  "timestamp"
+};
+
+// The list of reserved attributes used in the <testsuite> element of XML
+// output.
+static const char* const kReservedTestSuiteAttributes[] = {
+  "disabled",
+  "errors",
+  "failures",
+  "name",
+  "tests",
+  "time"
+};
+
+// The list of reserved attributes used in the <testcase> element of XML output.
+static const char* const kReservedTestCaseAttributes[] = {
+  "classname",
+  "name",
+  "status",
+  "time",
+  "type_param",
+  "value_param"
+};
+
+template <int kSize>
+std::vector<std::string> ArrayAsVector(const char* const (&array)[kSize]) {
+  return std::vector<std::string>(array, array + kSize);
+}
+
+static std::vector<std::string> GetReservedAttributesForElement(
+    const std::string& xml_element) {
+  if (xml_element == "testsuites") {
+    return ArrayAsVector(kReservedTestSuitesAttributes);
+  } else if (xml_element == "testsuite") {
+    return ArrayAsVector(kReservedTestSuiteAttributes);
+  } else if (xml_element == "testcase") {
+    return ArrayAsVector(kReservedTestCaseAttributes);
+  } else {
+    GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element;
+  }
+  // This code is unreachable but some compilers may not realizes that.
+  return std::vector<std::string>();
+}
+
+static std::string FormatWordList(const std::vector<std::string>& words) {
+  Message word_list;
+  for (size_t i = 0; i < words.size(); ++i) {
+    if (i > 0 && words.size() > 2) {
+      word_list << ", ";
+    }
+    if (i == words.size() - 1) {
+      word_list << "and ";
+    }
+    word_list << "'" << words[i] << "'";
+  }
+  return word_list.GetString();
+}
+
+bool ValidateTestPropertyName(const std::string& property_name,
+                              const std::vector<std::string>& reserved_names) {
+  if (std::find(reserved_names.begin(), reserved_names.end(), property_name) !=
+          reserved_names.end()) {
+    ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name
+                  << " (" << FormatWordList(reserved_names)
+                  << " are reserved by " << GTEST_NAME_ << ")";
+    return false;
+  }
+  return true;
+}
+
+// Adds a failure if the key is a reserved attribute of the element named
+// xml_element.  Returns true if the property is valid.
+bool TestResult::ValidateTestProperty(const std::string& xml_element,
+                                      const TestProperty& test_property) {
+  return ValidateTestPropertyName(test_property.key(),
+                                  GetReservedAttributesForElement(xml_element));
+}
+
+// Clears the object.
+void TestResult::Clear() {
+  test_part_results_.clear();
+  test_properties_.clear();
+  death_test_count_ = 0;
+  elapsed_time_ = 0;
+}
+
+// Returns true iff the test failed.
+bool TestResult::Failed() const {
+  for (int i = 0; i < total_part_count(); ++i) {
+    if (GetTestPartResult(i).failed())
+      return true;
+  }
+  return false;
+}
+
+// Returns true iff the test part fatally failed.
+static bool TestPartFatallyFailed(const TestPartResult& result) {
+  return result.fatally_failed();
+}
+
+// Returns true iff the test fatally failed.
+bool TestResult::HasFatalFailure() const {
+  return CountIf(test_part_results_, TestPartFatallyFailed) > 0;
+}
+
+// Returns true iff the test part non-fatally failed.
+static bool TestPartNonfatallyFailed(const TestPartResult& result) {
+  return result.nonfatally_failed();
+}
+
+// Returns true iff the test has a non-fatal failure.
+bool TestResult::HasNonfatalFailure() const {
+  return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0;
+}
+
+// Gets the number of all test parts.  This is the sum of the number
+// of successful test parts and the number of failed test parts.
+int TestResult::total_part_count() const {
+  return static_cast<int>(test_part_results_.size());
+}
+
+// Returns the number of the test properties.
+int TestResult::test_property_count() const {
+  return static_cast<int>(test_properties_.size());
+}
+
+// class Test
+
+// Creates a Test object.
+
+// The c'tor saves the states of all flags.
+Test::Test()
+    : gtest_flag_saver_(new GTEST_FLAG_SAVER_) {
+}
+
+// The d'tor restores the states of all flags.  The actual work is
+// done by the d'tor of the gtest_flag_saver_ field, and thus not
+// visible here.
+Test::~Test() {
+}
+
+// Sets up the test fixture.
+//
+// A sub-class may override this.
+void Test::SetUp() {
+}
+
+// Tears down the test fixture.
+//
+// A sub-class may override this.
+void Test::TearDown() {
+}
+
+// Allows user supplied key value pairs to be recorded for later output.
+void Test::RecordProperty(const std::string& key, const std::string& value) {
+  UnitTest::GetInstance()->RecordProperty(key, value);
+}
+
+// Allows user supplied key value pairs to be recorded for later output.
+void Test::RecordProperty(const std::string& key, int value) {
+  Message value_message;
+  value_message << value;
+  RecordProperty(key, value_message.GetString().c_str());
+}
+
+namespace internal {
+
+void ReportFailureInUnknownLocation(TestPartResult::Type result_type,
+                                    const std::string& message) {
+  // This function is a friend of UnitTest and as such has access to
+  // AddTestPartResult.
+  UnitTest::GetInstance()->AddTestPartResult(
+      result_type,
+      NULL,  // No info about the source file where the exception occurred.
+      -1,    // We have no info on which line caused the exception.
+      message,
+      "");   // No stack trace, either.
+}
+
+}  // namespace internal
+
+// Google Test requires all tests in the same test case to use the same test
+// fixture class.  This function checks if the current test has the
+// same fixture class as the first test in the current test case.  If
+// yes, it returns true; otherwise it generates a Google Test failure and
+// returns false.
+bool Test::HasSameFixtureClass() {
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  const TestCase* const test_case = impl->current_test_case();
+
+  // Info about the first test in the current test case.
+  const TestInfo* const first_test_info = test_case->test_info_list()[0];
+  const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_;
+  const char* const first_test_name = first_test_info->name();
+
+  // Info about the current test.
+  const TestInfo* const this_test_info = impl->current_test_info();
+  const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_;
+  const char* const this_test_name = this_test_info->name();
+
+  if (this_fixture_id != first_fixture_id) {
+    // Is the first test defined using TEST?
+    const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId();
+    // Is this test defined using TEST?
+    const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId();
+
+    if (first_is_TEST || this_is_TEST) {
+      // Both TEST and TEST_F appear in same test case, which is incorrect.
+      // Tell the user how to fix this.
+
+      // Gets the name of the TEST and the name of the TEST_F.  Note
+      // that first_is_TEST and this_is_TEST cannot both be true, as
+      // the fixture IDs are different for the two tests.
+      const char* const TEST_name =
+          first_is_TEST ? first_test_name : this_test_name;
+      const char* const TEST_F_name =
+          first_is_TEST ? this_test_name : first_test_name;
+
+      ADD_FAILURE()
+          << "All tests in the same test case must use the same test fixture\n"
+          << "class, so mixing TEST_F and TEST in the same test case is\n"
+          << "illegal.  In test case " << this_test_info->test_case_name()
+          << ",\n"
+          << "test " << TEST_F_name << " is defined using TEST_F but\n"
+          << "test " << TEST_name << " is defined using TEST.  You probably\n"
+          << "want to change the TEST to TEST_F or move it to another test\n"
+          << "case.";
+    } else {
+      // Two fixture classes with the same name appear in two different
+      // namespaces, which is not allowed. Tell the user how to fix this.
+      ADD_FAILURE()
+          << "All tests in the same test case must use the same test fixture\n"
+          << "class.  However, in test case "
+          << this_test_info->test_case_name() << ",\n"
+          << "you defined test " << first_test_name
+          << " and test " << this_test_name << "\n"
+          << "using two different test fixture classes.  This can happen if\n"
+          << "the two classes are from different namespaces or translation\n"
+          << "units and have the same name.  You should probably rename one\n"
+          << "of the classes to put the tests into different test cases.";
+    }
+    return false;
+  }
+
+  return true;
+}
+
+#if GTEST_HAS_SEH
+
+// Adds an "exception thrown" fatal failure to the current test.  This
+// function returns its result via an output parameter pointer because VC++
+// prohibits creation of objects with destructors on stack in functions
+// using __try (see error C2712).
+static std::string* FormatSehExceptionMessage(DWORD exception_code,
+                                              const char* location) {
+  Message message;
+  message << "SEH exception with code 0x" << std::setbase(16) <<
+    exception_code << std::setbase(10) << " thrown in " << location << ".";
+
+  return new std::string(message.GetString());
+}
+
+#endif  // GTEST_HAS_SEH
+
+namespace internal {
+
+#if GTEST_HAS_EXCEPTIONS
+
+// Adds an "exception thrown" fatal failure to the current test.
+static std::string FormatCxxExceptionMessage(const char* description,
+                                             const char* location) {
+  Message message;
+  if (description != NULL) {
+    message << "C++ exception with description \"" << description << "\"";
+  } else {
+    message << "Unknown C++ exception";
+  }
+  message << " thrown in " << location << ".";
+
+  return message.GetString();
+}
+
+static std::string PrintTestPartResultToString(
+    const TestPartResult& test_part_result);
+
+GoogleTestFailureException::GoogleTestFailureException(
+    const TestPartResult& failure)
+    : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// We put these helper functions in the internal namespace as IBM's xlC
+// compiler rejects the code if they were declared static.
+
+// Runs the given method and handles SEH exceptions it throws, when
+// SEH is supported; returns the 0-value for type Result in case of an
+// SEH exception.  (Microsoft compilers cannot handle SEH and C++
+// exceptions in the same function.  Therefore, we provide a separate
+// wrapper function for handling SEH exceptions.)
+template <class T, typename Result>
+Result HandleSehExceptionsInMethodIfSupported(
+    T* object, Result (T::*method)(), const char* location) {
+#if GTEST_HAS_SEH
+  __try {
+    return (object->*method)();
+  } __except (internal::UnitTestOptions::GTestShouldProcessSEH(  // NOLINT
+      GetExceptionCode())) {
+    // We create the exception message on the heap because VC++ prohibits
+    // creation of objects with destructors on stack in functions using __try
+    // (see error C2712).
+    std::string* exception_message = FormatSehExceptionMessage(
+        GetExceptionCode(), location);
+    internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure,
+                                             *exception_message);
+    delete exception_message;
+    return static_cast<Result>(0);
+  }
+#else
+  (void)location;
+  return (object->*method)();
+#endif  // GTEST_HAS_SEH
+}
+
+// Runs the given method and catches and reports C++ and/or SEH-style
+// exceptions, if they are supported; returns the 0-value for type
+// Result in case of an SEH exception.
+template <class T, typename Result>
+Result HandleExceptionsInMethodIfSupported(
+    T* object, Result (T::*method)(), const char* location) {
+  // NOTE: The user code can affect the way in which Google Test handles
+  // exceptions by setting GTEST_FLAG(catch_exceptions), but only before
+  // RUN_ALL_TESTS() starts. It is technically possible to check the flag
+  // after the exception is caught and either report or re-throw the
+  // exception based on the flag's value:
+  //
+  // try {
+  //   // Perform the test method.
+  // } catch (...) {
+  //   if (GTEST_FLAG(catch_exceptions))
+  //     // Report the exception as failure.
+  //   else
+  //     throw;  // Re-throws the original exception.
+  // }
+  //
+  // However, the purpose of this flag is to allow the program to drop into
+  // the debugger when the exception is thrown. On most platforms, once the
+  // control enters the catch block, the exception origin information is
+  // lost and the debugger will stop the program at the point of the
+  // re-throw in this function -- instead of at the point of the original
+  // throw statement in the code under test.  For this reason, we perform
+  // the check early, sacrificing the ability to affect Google Test's
+  // exception handling in the method where the exception is thrown.
+  if (internal::GetUnitTestImpl()->catch_exceptions()) {
+#if GTEST_HAS_EXCEPTIONS
+    try {
+      return HandleSehExceptionsInMethodIfSupported(object, method, location);
+    } catch (const internal::GoogleTestFailureException&) {  // NOLINT
+      // This exception type can only be thrown by a failed Google
+      // Test assertion with the intention of letting another testing
+      // framework catch it.  Therefore we just re-throw it.
+      throw;
+    } catch (const std::exception& e) {  // NOLINT
+      internal::ReportFailureInUnknownLocation(
+          TestPartResult::kFatalFailure,
+          FormatCxxExceptionMessage(e.what(), location));
+    } catch (...) {  // NOLINT
+      internal::ReportFailureInUnknownLocation(
+          TestPartResult::kFatalFailure,
+          FormatCxxExceptionMessage(NULL, location));
+    }
+    return static_cast<Result>(0);
+#else
+    return HandleSehExceptionsInMethodIfSupported(object, method, location);
+#endif  // GTEST_HAS_EXCEPTIONS
+  } else {
+    return (object->*method)();
+  }
+}
+
+}  // namespace internal
+
+// Runs the test and updates the test result.
+void Test::Run() {
+  if (!HasSameFixtureClass()) return;
+
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+  internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()");
+  // We will run the test only if SetUp() was successful.
+  if (!HasFatalFailure()) {
+    impl->os_stack_trace_getter()->UponLeavingGTest();
+    internal::HandleExceptionsInMethodIfSupported(
+        this, &Test::TestBody, "the test body");
+  }
+
+  // However, we want to clean up as much as possible.  Hence we will
+  // always call TearDown(), even if SetUp() or the test body has
+  // failed.
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+  internal::HandleExceptionsInMethodIfSupported(
+      this, &Test::TearDown, "TearDown()");
+}
+
+// Returns true iff the current test has a fatal failure.
+bool Test::HasFatalFailure() {
+  return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure();
+}
+
+// Returns true iff the current test has a non-fatal failure.
+bool Test::HasNonfatalFailure() {
+  return internal::GetUnitTestImpl()->current_test_result()->
+      HasNonfatalFailure();
+}
+
+// class TestInfo
+
+// Constructs a TestInfo object. It assumes ownership of the test factory
+// object.
+TestInfo::TestInfo(const std::string& a_test_case_name,
+                   const std::string& a_name,
+                   const char* a_type_param,
+                   const char* a_value_param,
+                   internal::CodeLocation a_code_location,
+                   internal::TypeId fixture_class_id,
+                   internal::TestFactoryBase* factory)
+    : test_case_name_(a_test_case_name),
+      name_(a_name),
+      type_param_(a_type_param ? new std::string(a_type_param) : NULL),
+      value_param_(a_value_param ? new std::string(a_value_param) : NULL),
+      location_(a_code_location),
+      fixture_class_id_(fixture_class_id),
+      should_run_(false),
+      is_disabled_(false),
+      matches_filter_(false),
+      factory_(factory),
+      result_() {}
+
+// Destructs a TestInfo object.
+TestInfo::~TestInfo() { delete factory_; }
+
+namespace internal {
+
+// Creates a new TestInfo object and registers it with Google Test;
+// returns the created object.
+//
+// Arguments:
+//
+//   test_case_name:   name of the test case
+//   name:             name of the test
+//   type_param:       the name of the test's type parameter, or NULL if
+//                     this is not a typed or a type-parameterized test.
+//   value_param:      text representation of the test's value parameter,
+//                     or NULL if this is not a value-parameterized test.
+//   code_location:    code location where the test is defined
+//   fixture_class_id: ID of the test fixture class
+//   set_up_tc:        pointer to the function that sets up the test case
+//   tear_down_tc:     pointer to the function that tears down the test case
+//   factory:          pointer to the factory that creates a test object.
+//                     The newly created TestInfo instance will assume
+//                     ownership of the factory object.
+TestInfo* MakeAndRegisterTestInfo(
+    const char* test_case_name,
+    const char* name,
+    const char* type_param,
+    const char* value_param,
+    CodeLocation code_location,
+    TypeId fixture_class_id,
+    SetUpTestCaseFunc set_up_tc,
+    TearDownTestCaseFunc tear_down_tc,
+    TestFactoryBase* factory) {
+  TestInfo* const test_info =
+      new TestInfo(test_case_name, name, type_param, value_param,
+                   code_location, fixture_class_id, factory);
+  GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info);
+  return test_info;
+}
+
+#if GTEST_HAS_PARAM_TEST
+void ReportInvalidTestCaseType(const char* test_case_name,
+                               CodeLocation code_location) {
+  Message errors;
+  errors
+      << "Attempted redefinition of test case " << test_case_name << ".\n"
+      << "All tests in the same test case must use the same test fixture\n"
+      << "class.  However, in test case " << test_case_name << ", you tried\n"
+      << "to define a test using a fixture class different from the one\n"
+      << "used earlier. This can happen if the two fixture classes are\n"
+      << "from different namespaces and have the same name. You should\n"
+      << "probably rename one of the classes to put the tests into different\n"
+      << "test cases.";
+
+  fprintf(stderr, "%s %s",
+          FormatFileLocation(code_location.file.c_str(),
+                             code_location.line).c_str(),
+          errors.GetString().c_str());
+}
+#endif  // GTEST_HAS_PARAM_TEST
+
+}  // namespace internal
+
+namespace {
+
+// A predicate that checks the test name of a TestInfo against a known
+// value.
+//
+// This is used for implementation of the TestCase class only.  We put
+// it in the anonymous namespace to prevent polluting the outer
+// namespace.
+//
+// TestNameIs is copyable.
+class TestNameIs {
+ public:
+  // Constructor.
+  //
+  // TestNameIs has NO default constructor.
+  explicit TestNameIs(const char* name)
+      : name_(name) {}
+
+  // Returns true iff the test name of test_info matches name_.
+  bool operator()(const TestInfo * test_info) const {
+    return test_info && test_info->name() == name_;
+  }
+
+ private:
+  std::string name_;
+};
+
+}  // namespace
+
+namespace internal {
+
+// This method expands all parameterized tests registered with macros TEST_P
+// and INSTANTIATE_TEST_CASE_P into regular tests and registers those.
+// This will be done just once during the program runtime.
+void UnitTestImpl::RegisterParameterizedTests() {
+#if GTEST_HAS_PARAM_TEST
+  if (!parameterized_tests_registered_) {
+    parameterized_test_registry_.RegisterTests();
+    parameterized_tests_registered_ = true;
+  }
+#endif
+}
+
+}  // namespace internal
+
+// Creates the test object, runs it, records its result, and then
+// deletes it.
+void TestInfo::Run() {
+  if (!should_run_) return;
+
+  // Tells UnitTest where to store test result.
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  impl->set_current_test_info(this);
+
+  TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
+
+  // Notifies the unit test event listeners that a test is about to start.
+  repeater->OnTestStart(*this);
+
+  const TimeInMillis start = internal::GetTimeInMillis();
+
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+
+  // Creates the test object.
+  Test* const test = internal::HandleExceptionsInMethodIfSupported(
+      factory_, &internal::TestFactoryBase::CreateTest,
+      "the test fixture's constructor");
+
+  // Runs the test only if the test object was created and its
+  // constructor didn't generate a fatal failure.
+  if ((test != NULL) && !Test::HasFatalFailure()) {
+    // This doesn't throw as all user code that can throw are wrapped into
+    // exception handling code.
+    test->Run();
+  }
+
+  // Deletes the test object.
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+  internal::HandleExceptionsInMethodIfSupported(
+      test, &Test::DeleteSelf_, "the test fixture's destructor");
+
+  result_.set_elapsed_time(internal::GetTimeInMillis() - start);
+
+  // Notifies the unit test event listener that a test has just finished.
+  repeater->OnTestEnd(*this);
+
+  // Tells UnitTest to stop associating assertion results to this
+  // test.
+  impl->set_current_test_info(NULL);
+}
+
+// class TestCase
+
+// Gets the number of successful tests in this test case.
+int TestCase::successful_test_count() const {
+  return CountIf(test_info_list_, TestPassed);
+}
+
+// Gets the number of failed tests in this test case.
+int TestCase::failed_test_count() const {
+  return CountIf(test_info_list_, TestFailed);
+}
+
+// Gets the number of disabled tests that will be reported in the XML report.
+int TestCase::reportable_disabled_test_count() const {
+  return CountIf(test_info_list_, TestReportableDisabled);
+}
+
+// Gets the number of disabled tests in this test case.
+int TestCase::disabled_test_count() const {
+  return CountIf(test_info_list_, TestDisabled);
+}
+
+// Gets the number of tests to be printed in the XML report.
+int TestCase::reportable_test_count() const {
+  return CountIf(test_info_list_, TestReportable);
+}
+
+// Get the number of tests in this test case that should run.
+int TestCase::test_to_run_count() const {
+  return CountIf(test_info_list_, ShouldRunTest);
+}
+
+// Gets the number of all tests.
+int TestCase::total_test_count() const {
+  return static_cast<int>(test_info_list_.size());
+}
+
+// Creates a TestCase with the given name.
+//
+// Arguments:
+//
+//   name:         name of the test case
+//   a_type_param: the name of the test case's type parameter, or NULL if
+//                 this is not a typed or a type-parameterized test case.
+//   set_up_tc:    pointer to the function that sets up the test case
+//   tear_down_tc: pointer to the function that tears down the test case
+TestCase::TestCase(const char* a_name, const char* a_type_param,
+                   Test::SetUpTestCaseFunc set_up_tc,
+                   Test::TearDownTestCaseFunc tear_down_tc)
+    : name_(a_name),
+      type_param_(a_type_param ? new std::string(a_type_param) : NULL),
+      set_up_tc_(set_up_tc),
+      tear_down_tc_(tear_down_tc),
+      should_run_(false),
+      elapsed_time_(0) {
+}
+
+// Destructor of TestCase.
+TestCase::~TestCase() {
+  // Deletes every Test in the collection.
+  ForEach(test_info_list_, internal::Delete<TestInfo>);
+}
+
+// Returns the i-th test among all the tests. i can range from 0 to
+// total_test_count() - 1. If i is not in that range, returns NULL.
+const TestInfo* TestCase::GetTestInfo(int i) const {
+  const int index = GetElementOr(test_indices_, i, -1);
+  return index < 0 ? NULL : test_info_list_[index];
+}
+
+// Returns the i-th test among all the tests. i can range from 0 to
+// total_test_count() - 1. If i is not in that range, returns NULL.
+TestInfo* TestCase::GetMutableTestInfo(int i) {
+  const int index = GetElementOr(test_indices_, i, -1);
+  return index < 0 ? NULL : test_info_list_[index];
+}
+
+// Adds a test to this test case.  Will delete the test upon
+// destruction of the TestCase object.
+void TestCase::AddTestInfo(TestInfo * test_info) {
+  test_info_list_.push_back(test_info);
+  test_indices_.push_back(static_cast<int>(test_indices_.size()));
+}
+
+// Runs every test in this TestCase.
+void TestCase::Run() {
+  if (!should_run_) return;
+
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  impl->set_current_test_case(this);
+
+  TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
+
+  repeater->OnTestCaseStart(*this);
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+  internal::HandleExceptionsInMethodIfSupported(
+      this, &TestCase::RunSetUpTestCase, "SetUpTestCase()");
+
+  const internal::TimeInMillis start = internal::GetTimeInMillis();
+  for (int i = 0; i < total_test_count(); i++) {
+    GetMutableTestInfo(i)->Run();
+  }
+  elapsed_time_ = internal::GetTimeInMillis() - start;
+
+  impl->os_stack_trace_getter()->UponLeavingGTest();
+  internal::HandleExceptionsInMethodIfSupported(
+      this, &TestCase::RunTearDownTestCase, "TearDownTestCase()");
+
+  repeater->OnTestCaseEnd(*this);
+  impl->set_current_test_case(NULL);
+}
+
+// Clears the results of all tests in this test case.
+void TestCase::ClearResult() {
+  ad_hoc_test_result_.Clear();
+  ForEach(test_info_list_, TestInfo::ClearTestResult);
+}
+
+// Shuffles the tests in this test case.
+void TestCase::ShuffleTests(internal::Random* random) {
+  Shuffle(random, &test_indices_);
+}
+
+// Restores the test order to before the first shuffle.
+void TestCase::UnshuffleTests() {
+  for (size_t i = 0; i < test_indices_.size(); i++) {
+    test_indices_[i] = static_cast<int>(i);
+  }
+}
+
+// Formats a countable noun.  Depending on its quantity, either the
+// singular form or the plural form is used. e.g.
+//
+// FormatCountableNoun(1, "formula", "formuli") returns "1 formula".
+// FormatCountableNoun(5, "book", "books") returns "5 books".
+static std::string FormatCountableNoun(int count,
+                                       const char * singular_form,
+                                       const char * plural_form) {
+  return internal::StreamableToString(count) + " " +
+      (count == 1 ? singular_form : plural_form);
+}
+
+// Formats the count of tests.
+static std::string FormatTestCount(int test_count) {
+  return FormatCountableNoun(test_count, "test", "tests");
+}
+
+// Formats the count of test cases.
+static std::string FormatTestCaseCount(int test_case_count) {
+  return FormatCountableNoun(test_case_count, "test case", "test cases");
+}
+
+// Converts a TestPartResult::Type enum to human-friendly string
+// representation.  Both kNonFatalFailure and kFatalFailure are translated
+// to "Failure", as the user usually doesn't care about the difference
+// between the two when viewing the test result.
+static const char * TestPartResultTypeToString(TestPartResult::Type type) {
+  switch (type) {
+    case TestPartResult::kSuccess:
+      return "Success";
+
+    case TestPartResult::kNonFatalFailure:
+    case TestPartResult::kFatalFailure:
+#ifdef _MSC_VER
+      return "error: ";
+#else
+      return "Failure\n";
+#endif
+    default:
+      return "Unknown result type";
+  }
+}
+
+namespace internal {
+
+// Prints a TestPartResult to an std::string.
+static std::string PrintTestPartResultToString(
+    const TestPartResult& test_part_result) {
+  return (Message()
+          << internal::FormatFileLocation(test_part_result.file_name(),
+                                          test_part_result.line_number())
+          << " " << TestPartResultTypeToString(test_part_result.type())
+          << test_part_result.message()).GetString();
+}
+
+// Prints a TestPartResult.
+static void PrintTestPartResult(const TestPartResult& test_part_result) {
+  const std::string& result =
+      PrintTestPartResultToString(test_part_result);
+  printf("%s\n", result.c_str());
+  fflush(stdout);
+  // If the test program runs in Visual Studio or a debugger, the
+  // following statements add the test part result message to the Output
+  // window such that the user can double-click on it to jump to the
+  // corresponding source code location; otherwise they do nothing.
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE
+  // We don't call OutputDebugString*() on Windows Mobile, as printing
+  // to stdout is done by OutputDebugString() there already - we don't
+  // want the same message printed twice.
+  ::OutputDebugStringA(result.c_str());
+  ::OutputDebugStringA("\n");
+#endif
+}
+
+// class PrettyUnitTestResultPrinter
+
+enum GTestColor {
+  COLOR_DEFAULT,
+  COLOR_RED,
+  COLOR_GREEN,
+  COLOR_YELLOW
+};
+
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \
+    !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+
+// Returns the character attribute for the given color.
+WORD GetColorAttribute(GTestColor color) {
+  switch (color) {
+    case COLOR_RED:    return FOREGROUND_RED;
+    case COLOR_GREEN:  return FOREGROUND_GREEN;
+    case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN;
+    default:           return 0;
+  }
+}
+
+#else
+
+// Returns the ANSI color code for the given color.  COLOR_DEFAULT is
+// an invalid input.
+const char* GetAnsiColorCode(GTestColor color) {
+  switch (color) {
+    case COLOR_RED:     return "1";
+    case COLOR_GREEN:   return "2";
+    case COLOR_YELLOW:  return "3";
+    default:            return NULL;
+  };
+}
+
+#endif  // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE
+
+// Returns true iff Google Test should use colors in the output.
+bool ShouldUseColor(bool stdout_is_tty) {
+  const char* const gtest_color = GTEST_FLAG(color).c_str();
+
+  if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) {
+#if GTEST_OS_WINDOWS
+    // On Windows the TERM variable is usually not set, but the
+    // console there does support colors.
+    return stdout_is_tty;
+#else
+    // On non-Windows platforms, we rely on the TERM variable.
+    const char* const term = posix::GetEnv("TERM");
+    const bool term_supports_color =
+        String::CStringEquals(term, "xterm") ||
+        String::CStringEquals(term, "xterm-color") ||
+        String::CStringEquals(term, "xterm-256color") ||
+        String::CStringEquals(term, "screen") ||
+        String::CStringEquals(term, "screen-256color") ||
+        String::CStringEquals(term, "tmux") ||
+        String::CStringEquals(term, "tmux-256color") ||
+        String::CStringEquals(term, "rxvt-unicode") ||
+        String::CStringEquals(term, "rxvt-unicode-256color") ||
+        String::CStringEquals(term, "linux") ||
+        String::CStringEquals(term, "cygwin");
+    return stdout_is_tty && term_supports_color;
+#endif  // GTEST_OS_WINDOWS
+  }
+
+  return String::CaseInsensitiveCStringEquals(gtest_color, "yes") ||
+      String::CaseInsensitiveCStringEquals(gtest_color, "true") ||
+      String::CaseInsensitiveCStringEquals(gtest_color, "t") ||
+      String::CStringEquals(gtest_color, "1");
+  // We take "yes", "true", "t", and "1" as meaning "yes".  If the
+  // value is neither one of these nor "auto", we treat it as "no" to
+  // be conservative.
+}
+
+// Helpers for printing colored strings to stdout. Note that on Windows, we
+// cannot simply emit special characters and have the terminal change colors.
+// This routine must actually emit the characters rather than return a string
+// that would be colored when printed, as can be done on Linux.
+void ColoredPrintf(GTestColor color, const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \
+    GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
+  const bool use_color = AlwaysFalse();
+#else
+  static const bool in_color_mode =
+      ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0);
+  const bool use_color = in_color_mode && (color != COLOR_DEFAULT);
+#endif  // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS
+  // The '!= 0' comparison is necessary to satisfy MSVC 7.1.
+
+  if (!use_color) {
+    vprintf(fmt, args);
+    va_end(args);
+    return;
+  }
+
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \
+    !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+  const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
+
+  // Gets the current text color.
+  CONSOLE_SCREEN_BUFFER_INFO buffer_info;
+  GetConsoleScreenBufferInfo(stdout_handle, &buffer_info);
+  const WORD old_color_attrs = buffer_info.wAttributes;
+
+  // We need to flush the stream buffers into the console before each
+  // SetConsoleTextAttribute call lest it affect the text that is already
+  // printed but has not yet reached the console.
+  fflush(stdout);
+  SetConsoleTextAttribute(stdout_handle,
+                          GetColorAttribute(color) | FOREGROUND_INTENSITY);
+  vprintf(fmt, args);
+
+  fflush(stdout);
+  // Restores the text color.
+  SetConsoleTextAttribute(stdout_handle, old_color_attrs);
+#else
+  printf("\033[0;3%sm", GetAnsiColorCode(color));
+  vprintf(fmt, args);
+  printf("\033[m");  // Resets the terminal to default.
+#endif  // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE
+  va_end(args);
+}
+
+// Text printed in Google Test's text output and --gunit_list_tests
+// output to label the type parameter and value parameter for a test.
+static const char kTypeParamLabel[] = "TypeParam";
+static const char kValueParamLabel[] = "GetParam()";
+
+void PrintFullTestCommentIfPresent(const TestInfo& test_info) {
+  const char* const type_param = test_info.type_param();
+  const char* const value_param = test_info.value_param();
+
+  if (type_param != NULL || value_param != NULL) {
+    printf(", where ");
+    if (type_param != NULL) {
+      printf("%s = %s", kTypeParamLabel, type_param);
+      if (value_param != NULL)
+        printf(" and ");
+    }
+    if (value_param != NULL) {
+      printf("%s = %s", kValueParamLabel, value_param);
+    }
+  }
+}
+
+// This class implements the TestEventListener interface.
+//
+// Class PrettyUnitTestResultPrinter is copyable.
+class PrettyUnitTestResultPrinter : public TestEventListener {
+ public:
+  PrettyUnitTestResultPrinter() {}
+  static void PrintTestName(const char * test_case, const char * test) {
+    printf("%s.%s", test_case, test);
+  }
+
+  // The following methods override what's in the TestEventListener class.
+  virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {}
+  virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration);
+  virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test);
+  virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {}
+  virtual void OnTestCaseStart(const TestCase& test_case);
+  virtual void OnTestStart(const TestInfo& test_info);
+  virtual void OnTestPartResult(const TestPartResult& result);
+  virtual void OnTestEnd(const TestInfo& test_info);
+  virtual void OnTestCaseEnd(const TestCase& test_case);
+  virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test);
+  virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {}
+  virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration);
+  virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {}
+
+ private:
+  static void PrintFailedTests(const UnitTest& unit_test);
+};
+
+  // Fired before each iteration of tests starts.
+void PrettyUnitTestResultPrinter::OnTestIterationStart(
+    const UnitTest& unit_test, int iteration) {
+  if (GTEST_FLAG(repeat) != 1)
+    printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1);
+
+  const char* const filter = GTEST_FLAG(filter).c_str();
+
+  // Prints the filter if it's not *.  This reminds the user that some
+  // tests may be skipped.
+  if (!String::CStringEquals(filter, kUniversalFilter)) {
+    ColoredPrintf(COLOR_YELLOW,
+                  "Note: %s filter = %s\n", GTEST_NAME_, filter);
+  }
+
+  if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) {
+    const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1);
+    ColoredPrintf(COLOR_YELLOW,
+                  "Note: This is test shard %d of %s.\n",
+                  static_cast<int>(shard_index) + 1,
+                  internal::posix::GetEnv(kTestTotalShards));
+  }
+
+  if (GTEST_FLAG(shuffle)) {
+    ColoredPrintf(COLOR_YELLOW,
+                  "Note: Randomizing tests' orders with a seed of %d .\n",
+                  unit_test.random_seed());
+  }
+
+  ColoredPrintf(COLOR_GREEN,  "[==========] ");
+  printf("Running %s from %s.\n",
+         FormatTestCount(unit_test.test_to_run_count()).c_str(),
+         FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str());
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart(
+    const UnitTest& /*unit_test*/) {
+  ColoredPrintf(COLOR_GREEN,  "[----------] ");
+  printf("Global test environment set-up.\n");
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) {
+  const std::string counts =
+      FormatCountableNoun(test_case.test_to_run_count(), "test", "tests");
+  ColoredPrintf(COLOR_GREEN, "[----------] ");
+  printf("%s from %s", counts.c_str(), test_case.name());
+  if (test_case.type_param() == NULL) {
+    printf("\n");
+  } else {
+    printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param());
+  }
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) {
+  ColoredPrintf(COLOR_GREEN,  "[ RUN      ] ");
+  PrintTestName(test_info.test_case_name(), test_info.name());
+  printf("\n");
+  fflush(stdout);
+}
+
+// Called after an assertion failure.
+void PrettyUnitTestResultPrinter::OnTestPartResult(
+    const TestPartResult& result) {
+  // If the test part succeeded, we don't need to do anything.
+  if (result.type() == TestPartResult::kSuccess)
+    return;
+
+  // Print failure message from the assertion (e.g. expected this and got that).
+  PrintTestPartResult(result);
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) {
+  if (test_info.result()->Passed()) {
+    ColoredPrintf(COLOR_GREEN, "[       OK ] ");
+  } else {
+    ColoredPrintf(COLOR_RED, "[  FAILED  ] ");
+  }
+  PrintTestName(test_info.test_case_name(), test_info.name());
+  if (test_info.result()->Failed())
+    PrintFullTestCommentIfPresent(test_info);
+
+  if (GTEST_FLAG(print_time)) {
+    printf(" (%s ms)\n", internal::StreamableToString(
+           test_info.result()->elapsed_time()).c_str());
+  } else {
+    printf("\n");
+  }
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) {
+  if (!GTEST_FLAG(print_time)) return;
+
+  const std::string counts =
+      FormatCountableNoun(test_case.test_to_run_count(), "test", "tests");
+  ColoredPrintf(COLOR_GREEN, "[----------] ");
+  printf("%s from %s (%s ms total)\n\n",
+         counts.c_str(), test_case.name(),
+         internal::StreamableToString(test_case.elapsed_time()).c_str());
+  fflush(stdout);
+}
+
+void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart(
+    const UnitTest& /*unit_test*/) {
+  ColoredPrintf(COLOR_GREEN,  "[----------] ");
+  printf("Global test environment tear-down\n");
+  fflush(stdout);
+}
+
+// Internal helper for printing the list of failed tests.
+void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) {
+  const int failed_test_count = unit_test.failed_test_count();
+  if (failed_test_count == 0) {
+    return;
+  }
+
+  for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
+    const TestCase& test_case = *unit_test.GetTestCase(i);
+    if (!test_case.should_run() || (test_case.failed_test_count() == 0)) {
+      continue;
+    }
+    for (int j = 0; j < test_case.total_test_count(); ++j) {
+      const TestInfo& test_info = *test_case.GetTestInfo(j);
+      if (!test_info.should_run() || test_info.result()->Passed()) {
+        continue;
+      }
+      ColoredPrintf(COLOR_RED, "[  FAILED  ] ");
+      printf("%s.%s", test_case.name(), test_info.name());
+      PrintFullTestCommentIfPresent(test_info);
+      printf("\n");
+    }
+  }
+}
+
+void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test,
+                                                     int /*iteration*/) {
+  ColoredPrintf(COLOR_GREEN,  "[==========] ");
+  printf("%s from %s ran.",
+         FormatTestCount(unit_test.test_to_run_count()).c_str(),
+         FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str());
+  if (GTEST_FLAG(print_time)) {
+    printf(" (%s ms total)",
+           internal::StreamableToString(unit_test.elapsed_time()).c_str());
+  }
+  printf("\n");
+  ColoredPrintf(COLOR_GREEN,  "[  PASSED  ] ");
+  printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str());
+
+  int num_failures = unit_test.failed_test_count();
+  if (!unit_test.Passed()) {
+    const int failed_test_count = unit_test.failed_test_count();
+    ColoredPrintf(COLOR_RED,  "[  FAILED  ] ");
+    printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str());
+    PrintFailedTests(unit_test);
+    printf("\n%2d FAILED %s\n", num_failures,
+                        num_failures == 1 ? "TEST" : "TESTS");
+  }
+
+  int num_disabled = unit_test.reportable_disabled_test_count();
+  if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) {
+    if (!num_failures) {
+      printf("\n");  // Add a spacer if no FAILURE banner is displayed.
+    }
+    ColoredPrintf(COLOR_YELLOW,
+                  "  YOU HAVE %d DISABLED %s\n\n",
+                  num_disabled,
+                  num_disabled == 1 ? "TEST" : "TESTS");
+  }
+  // Ensure that Google Test output is printed before, e.g., heapchecker output.
+  fflush(stdout);
+}
+
+// End PrettyUnitTestResultPrinter
+
+// class TestEventRepeater
+//
+// This class forwards events to other event listeners.
+class TestEventRepeater : public TestEventListener {
+ public:
+  TestEventRepeater() : forwarding_enabled_(true) {}
+  virtual ~TestEventRepeater();
+  void Append(TestEventListener *listener);
+  TestEventListener* Release(TestEventListener* listener);
+
+  // Controls whether events will be forwarded to listeners_. Set to false
+  // in death test child processes.
+  bool forwarding_enabled() const { return forwarding_enabled_; }
+  void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; }
+
+  virtual void OnTestProgramStart(const UnitTest& unit_test);
+  virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration);
+  virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test);
+  virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test);
+  virtual void OnTestCaseStart(const TestCase& test_case);
+  virtual void OnTestStart(const TestInfo& test_info);
+  virtual void OnTestPartResult(const TestPartResult& result);
+  virtual void OnTestEnd(const TestInfo& test_info);
+  virtual void OnTestCaseEnd(const TestCase& test_case);
+  virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test);
+  virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test);
+  virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration);
+  virtual void OnTestProgramEnd(const UnitTest& unit_test);
+
+ private:
+  // Controls whether events will be forwarded to listeners_. Set to false
+  // in death test child processes.
+  bool forwarding_enabled_;
+  // The list of listeners that receive events.
+  std::vector<TestEventListener*> listeners_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater);
+};
+
+TestEventRepeater::~TestEventRepeater() {
+  ForEach(listeners_, Delete<TestEventListener>);
+}
+
+void TestEventRepeater::Append(TestEventListener *listener) {
+  listeners_.push_back(listener);
+}
+
+// TODO(vladl@google.com): Factor the search functionality into Vector::Find.
+TestEventListener* TestEventRepeater::Release(TestEventListener *listener) {
+  for (size_t i = 0; i < listeners_.size(); ++i) {
+    if (listeners_[i] == listener) {
+      listeners_.erase(listeners_.begin() + i);
+      return listener;
+    }
+  }
+
+  return NULL;
+}
+
+// Since most methods are very similar, use macros to reduce boilerplate.
+// This defines a member that forwards the call to all listeners.
+#define GTEST_REPEATER_METHOD_(Name, Type) \
+void TestEventRepeater::Name(const Type& parameter) { \
+  if (forwarding_enabled_) { \
+    for (size_t i = 0; i < listeners_.size(); i++) { \
+      listeners_[i]->Name(parameter); \
+    } \
+  } \
+}
+// This defines a member that forwards the call to all listeners in reverse
+// order.
+#define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \
+void TestEventRepeater::Name(const Type& parameter) { \
+  if (forwarding_enabled_) { \
+    for (int i = static_cast<int>(listeners_.size()) - 1; i >= 0; i--) { \
+      listeners_[i]->Name(parameter); \
+    } \
+  } \
+}
+
+GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest)
+GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest)
+GTEST_REPEATER_METHOD_(OnTestCaseStart, TestCase)
+GTEST_REPEATER_METHOD_(OnTestStart, TestInfo)
+GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult)
+GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest)
+GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest)
+GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest)
+GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo)
+GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestCase)
+GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest)
+
+#undef GTEST_REPEATER_METHOD_
+#undef GTEST_REVERSE_REPEATER_METHOD_
+
+void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test,
+                                             int iteration) {
+  if (forwarding_enabled_) {
+    for (size_t i = 0; i < listeners_.size(); i++) {
+      listeners_[i]->OnTestIterationStart(unit_test, iteration);
+    }
+  }
+}
+
+void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test,
+                                           int iteration) {
+  if (forwarding_enabled_) {
+    for (int i = static_cast<int>(listeners_.size()) - 1; i >= 0; i--) {
+      listeners_[i]->OnTestIterationEnd(unit_test, iteration);
+    }
+  }
+}
+
+// End TestEventRepeater
+
+// This class generates an XML output file.
+class XmlUnitTestResultPrinter : public EmptyTestEventListener {
+ public:
+  explicit XmlUnitTestResultPrinter(const char* output_file);
+
+  virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration);
+
+ private:
+  // Is c a whitespace character that is normalized to a space character
+  // when it appears in an XML attribute value?
+  static bool IsNormalizableWhitespace(char c) {
+    return c == 0x9 || c == 0xA || c == 0xD;
+  }
+
+  // May c appear in a well-formed XML document?
+  static bool IsValidXmlCharacter(char c) {
+    return IsNormalizableWhitespace(c) || c >= 0x20;
+  }
+
+  // Returns an XML-escaped copy of the input string str.  If
+  // is_attribute is true, the text is meant to appear as an attribute
+  // value, and normalizable whitespace is preserved by replacing it
+  // with character references.
+  static std::string EscapeXml(const std::string& str, bool is_attribute);
+
+  // Returns the given string with all characters invalid in XML removed.
+  static std::string RemoveInvalidXmlCharacters(const std::string& str);
+
+  // Convenience wrapper around EscapeXml when str is an attribute value.
+  static std::string EscapeXmlAttribute(const std::string& str) {
+    return EscapeXml(str, true);
+  }
+
+  // Convenience wrapper around EscapeXml when str is not an attribute value.
+  static std::string EscapeXmlText(const char* str) {
+    return EscapeXml(str, false);
+  }
+
+  // Verifies that the given attribute belongs to the given element and
+  // streams the attribute as XML.
+  static void OutputXmlAttribute(std::ostream* stream,
+                                 const std::string& element_name,
+                                 const std::string& name,
+                                 const std::string& value);
+
+  // Streams an XML CDATA section, escaping invalid CDATA sequences as needed.
+  static void OutputXmlCDataSection(::std::ostream* stream, const char* data);
+
+  // Streams an XML representation of a TestInfo object.
+  static void OutputXmlTestInfo(::std::ostream* stream,
+                                const char* test_case_name,
+                                const TestInfo& test_info);
+
+  // Prints an XML representation of a TestCase object
+  static void PrintXmlTestCase(::std::ostream* stream,
+                               const TestCase& test_case);
+
+  // Prints an XML summary of unit_test to output stream out.
+  static void PrintXmlUnitTest(::std::ostream* stream,
+                               const UnitTest& unit_test);
+
+  // Produces a string representing the test properties in a result as space
+  // delimited XML attributes based on the property key="value" pairs.
+  // When the std::string is not empty, it includes a space at the beginning,
+  // to delimit this attribute from prior attributes.
+  static std::string TestPropertiesAsXmlAttributes(const TestResult& result);
+
+  // The output file.
+  const std::string output_file_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter);
+};
+
+// Creates a new XmlUnitTestResultPrinter.
+XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file)
+    : output_file_(output_file) {
+  if (output_file_.c_str() == NULL || output_file_.empty()) {
+    fprintf(stderr, "XML output file may not be null\n");
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+  }
+}
+
+// Called after the unit test ends.
+void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test,
+                                                  int /*iteration*/) {
+  FILE* xmlout = NULL;
+  FilePath output_file(output_file_);
+  FilePath output_dir(output_file.RemoveFileName());
+
+  if (output_dir.CreateDirectoriesRecursively()) {
+    xmlout = posix::FOpen(output_file_.c_str(), "w");
+  }
+  if (xmlout == NULL) {
+    // TODO(wan): report the reason of the failure.
+    //
+    // We don't do it for now as:
+    //
+    //   1. There is no urgent need for it.
+    //   2. It's a bit involved to make the errno variable thread-safe on
+    //      all three operating systems (Linux, Windows, and Mac OS).
+    //   3. To interpret the meaning of errno in a thread-safe way,
+    //      we need the strerror_r() function, which is not available on
+    //      Windows.
+    fprintf(stderr,
+            "Unable to open file \"%s\"\n",
+            output_file_.c_str());
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+  }
+  std::stringstream stream;
+  PrintXmlUnitTest(&stream, unit_test);
+  fprintf(xmlout, "%s", StringStreamToString(&stream).c_str());
+  fclose(xmlout);
+}
+
+// Returns an XML-escaped copy of the input string str.  If is_attribute
+// is true, the text is meant to appear as an attribute value, and
+// normalizable whitespace is preserved by replacing it with character
+// references.
+//
+// Invalid XML characters in str, if any, are stripped from the output.
+// It is expected that most, if not all, of the text processed by this
+// module will consist of ordinary English text.
+// If this module is ever modified to produce version 1.1 XML output,
+// most invalid characters can be retained using character references.
+// TODO(wan): It might be nice to have a minimally invasive, human-readable
+// escaping scheme for invalid characters, rather than dropping them.
+std::string XmlUnitTestResultPrinter::EscapeXml(
+    const std::string& str, bool is_attribute) {
+  Message m;
+
+  for (size_t i = 0; i < str.size(); ++i) {
+    const char ch = str[i];
+    switch (ch) {
+      case '<':
+        m << "&lt;";
+        break;
+      case '>':
+        m << "&gt;";
+        break;
+      case '&':
+        m << "&amp;";
+        break;
+      case '\'':
+        if (is_attribute)
+          m << "&apos;";
+        else
+          m << '\'';
+        break;
+      case '"':
+        if (is_attribute)
+          m << "&quot;";
+        else
+          m << '"';
+        break;
+      default:
+        if (IsValidXmlCharacter(ch)) {
+          if (is_attribute && IsNormalizableWhitespace(ch))
+            m << "&#x" << String::FormatByte(static_cast<unsigned char>(ch))
+              << ";";
+          else
+            m << ch;
+        }
+        break;
+    }
+  }
+
+  return m.GetString();
+}
+
+// Returns the given string with all characters invalid in XML removed.
+// Currently invalid characters are dropped from the string. An
+// alternative is to replace them with certain characters such as . or ?.
+std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters(
+    const std::string& str) {
+  std::string output;
+  output.reserve(str.size());
+  for (std::string::const_iterator it = str.begin(); it != str.end(); ++it)
+    if (IsValidXmlCharacter(*it))
+      output.push_back(*it);
+
+  return output;
+}
+
+// The following routines generate an XML representation of a UnitTest
+// object.
+//
+// This is how Google Test concepts map to the DTD:
+//
+// <testsuites name="AllTests">        <-- corresponds to a UnitTest object
+//   <testsuite name="testcase-name">  <-- corresponds to a TestCase object
+//     <testcase name="test-name">     <-- corresponds to a TestInfo object
+//       <failure message="...">...</failure>
+//       <failure message="...">...</failure>
+//       <failure message="...">...</failure>
+//                                     <-- individual assertion failures
+//     </testcase>
+//   </testsuite>
+// </testsuites>
+
+// Formats the given time in milliseconds as seconds.
+std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) {
+  ::std::stringstream ss;
+  ss << (static_cast<double>(ms) * 1e-3);
+  return ss.str();
+}
+
+static bool PortableLocaltime(time_t seconds, struct tm* out) {
+#if defined(_MSC_VER)
+  return localtime_s(out, &seconds) == 0;
+#elif defined(__MINGW32__) || defined(__MINGW64__)
+  // MINGW <time.h> provides neither localtime_r nor localtime_s, but uses
+  // Windows' localtime(), which has a thread-local tm buffer.
+  struct tm* tm_ptr = localtime(&seconds);  // NOLINT
+  if (tm_ptr == NULL)
+    return false;
+  *out = *tm_ptr;
+  return true;
+#else
+  return localtime_r(&seconds, out) != NULL;
+#endif
+}
+
+// Converts the given epoch time in milliseconds to a date string in the ISO
+// 8601 format, without the timezone information.
+std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) {
+  struct tm time_struct;
+  if (!PortableLocaltime(static_cast<time_t>(ms / 1000), &time_struct))
+    return "";
+  // YYYY-MM-DDThh:mm:ss
+  return StreamableToString(time_struct.tm_year + 1900) + "-" +
+      String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" +
+      String::FormatIntWidth2(time_struct.tm_mday) + "T" +
+      String::FormatIntWidth2(time_struct.tm_hour) + ":" +
+      String::FormatIntWidth2(time_struct.tm_min) + ":" +
+      String::FormatIntWidth2(time_struct.tm_sec);
+}
+
+// Streams an XML CDATA section, escaping invalid CDATA sequences as needed.
+void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream,
+                                                     const char* data) {
+  const char* segment = data;
+  *stream << "<![CDATA[";
+  for (;;) {
+    const char* const next_segment = strstr(segment, "]]>");
+    if (next_segment != NULL) {
+      stream->write(
+          segment, static_cast<std::streamsize>(next_segment - segment));
+      *stream << "]]>]]&gt;<![CDATA[";
+      segment = next_segment + strlen("]]>");
+    } else {
+      *stream << segment;
+      break;
+    }
+  }
+  *stream << "]]>";
+}
+
+void XmlUnitTestResultPrinter::OutputXmlAttribute(
+    std::ostream* stream,
+    const std::string& element_name,
+    const std::string& name,
+    const std::string& value) {
+  const std::vector<std::string>& allowed_names =
+      GetReservedAttributesForElement(element_name);
+
+  GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) !=
+                   allowed_names.end())
+      << "Attribute " << name << " is not allowed for element <" << element_name
+      << ">.";
+
+  *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\"";
+}
+
+// Prints an XML representation of a TestInfo object.
+// TODO(wan): There is also value in printing properties with the plain printer.
+void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream,
+                                                 const char* test_case_name,
+                                                 const TestInfo& test_info) {
+  const TestResult& result = *test_info.result();
+  const std::string kTestcase = "testcase";
+
+  *stream << "    <testcase";
+  OutputXmlAttribute(stream, kTestcase, "name", test_info.name());
+
+  if (test_info.value_param() != NULL) {
+    OutputXmlAttribute(stream, kTestcase, "value_param",
+                       test_info.value_param());
+  }
+  if (test_info.type_param() != NULL) {
+    OutputXmlAttribute(stream, kTestcase, "type_param", test_info.type_param());
+  }
+
+  OutputXmlAttribute(stream, kTestcase, "status",
+                     test_info.should_run() ? "run" : "notrun");
+  OutputXmlAttribute(stream, kTestcase, "time",
+                     FormatTimeInMillisAsSeconds(result.elapsed_time()));
+  OutputXmlAttribute(stream, kTestcase, "classname", test_case_name);
+  *stream << TestPropertiesAsXmlAttributes(result);
+
+  int failures = 0;
+  for (int i = 0; i < result.total_part_count(); ++i) {
+    const TestPartResult& part = result.GetTestPartResult(i);
+    if (part.failed()) {
+      if (++failures == 1) {
+        *stream << ">\n";
+      }
+      const string location = internal::FormatCompilerIndependentFileLocation(
+          part.file_name(), part.line_number());
+      const string summary = location + "\n" + part.summary();
+      *stream << "      <failure message=\""
+              << EscapeXmlAttribute(summary.c_str())
+              << "\" type=\"\">";
+      const string detail = location + "\n" + part.message();
+      OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str());
+      *stream << "</failure>\n";
+    }
+  }
+
+  if (failures == 0)
+    *stream << " />\n";
+  else
+    *stream << "    </testcase>\n";
+}
+
+// Prints an XML representation of a TestCase object
+void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream,
+                                                const TestCase& test_case) {
+  const std::string kTestsuite = "testsuite";
+  *stream << "  <" << kTestsuite;
+  OutputXmlAttribute(stream, kTestsuite, "name", test_case.name());
+  OutputXmlAttribute(stream, kTestsuite, "tests",
+                     StreamableToString(test_case.reportable_test_count()));
+  OutputXmlAttribute(stream, kTestsuite, "failures",
+                     StreamableToString(test_case.failed_test_count()));
+  OutputXmlAttribute(
+      stream, kTestsuite, "disabled",
+      StreamableToString(test_case.reportable_disabled_test_count()));
+  OutputXmlAttribute(stream, kTestsuite, "errors", "0");
+  OutputXmlAttribute(stream, kTestsuite, "time",
+                     FormatTimeInMillisAsSeconds(test_case.elapsed_time()));
+  *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result())
+          << ">\n";
+
+  for (int i = 0; i < test_case.total_test_count(); ++i) {
+    if (test_case.GetTestInfo(i)->is_reportable())
+      OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i));
+  }
+  *stream << "  </" << kTestsuite << ">\n";
+}
+
+// Prints an XML summary of unit_test to output stream out.
+void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream,
+                                                const UnitTest& unit_test) {
+  const std::string kTestsuites = "testsuites";
+
+  *stream << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+  *stream << "<" << kTestsuites;
+
+  OutputXmlAttribute(stream, kTestsuites, "tests",
+                     StreamableToString(unit_test.reportable_test_count()));
+  OutputXmlAttribute(stream, kTestsuites, "failures",
+                     StreamableToString(unit_test.failed_test_count()));
+  OutputXmlAttribute(
+      stream, kTestsuites, "disabled",
+      StreamableToString(unit_test.reportable_disabled_test_count()));
+  OutputXmlAttribute(stream, kTestsuites, "errors", "0");
+  OutputXmlAttribute(
+      stream, kTestsuites, "timestamp",
+      FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp()));
+  OutputXmlAttribute(stream, kTestsuites, "time",
+                     FormatTimeInMillisAsSeconds(unit_test.elapsed_time()));
+
+  if (GTEST_FLAG(shuffle)) {
+    OutputXmlAttribute(stream, kTestsuites, "random_seed",
+                       StreamableToString(unit_test.random_seed()));
+  }
+
+  *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result());
+
+  OutputXmlAttribute(stream, kTestsuites, "name", "AllTests");
+  *stream << ">\n";
+
+  for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
+    if (unit_test.GetTestCase(i)->reportable_test_count() > 0)
+      PrintXmlTestCase(stream, *unit_test.GetTestCase(i));
+  }
+  *stream << "</" << kTestsuites << ">\n";
+}
+
+// Produces a string representing the test properties in a result as space
+// delimited XML attributes based on the property key="value" pairs.
+std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes(
+    const TestResult& result) {
+  Message attributes;
+  for (int i = 0; i < result.test_property_count(); ++i) {
+    const TestProperty& property = result.GetTestProperty(i);
+    attributes << " " << property.key() << "="
+        << "\"" << EscapeXmlAttribute(property.value()) << "\"";
+  }
+  return attributes.GetString();
+}
+
+// End XmlUnitTestResultPrinter
+
+#if GTEST_CAN_STREAM_RESULTS_
+
+// Checks if str contains '=', '&', '%' or '\n' characters. If yes,
+// replaces them by "%xx" where xx is their hexadecimal value. For
+// example, replaces "=" with "%3D".  This algorithm is O(strlen(str))
+// in both time and space -- important as the input str may contain an
+// arbitrarily long test failure message and stack trace.
+string StreamingListener::UrlEncode(const char* str) {
+  string result;
+  result.reserve(strlen(str) + 1);
+  for (char ch = *str; ch != '\0'; ch = *++str) {
+    switch (ch) {
+      case '%':
+      case '=':
+      case '&':
+      case '\n':
+        result.append("%" + String::FormatByte(static_cast<unsigned char>(ch)));
+        break;
+      default:
+        result.push_back(ch);
+        break;
+    }
+  }
+  return result;
+}
+
+void StreamingListener::SocketWriter::MakeConnection() {
+  GTEST_CHECK_(sockfd_ == -1)
+      << "MakeConnection() can't be called when there is already a connection.";
+
+  addrinfo hints;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = AF_UNSPEC;    // To allow both IPv4 and IPv6 addresses.
+  hints.ai_socktype = SOCK_STREAM;
+  addrinfo* servinfo = NULL;
+
+  // Use the getaddrinfo() to get a linked list of IP addresses for
+  // the given host name.
+  const int error_num = getaddrinfo(
+      host_name_.c_str(), port_num_.c_str(), &hints, &servinfo);
+  if (error_num != 0) {
+    GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: "
+                        << gai_strerror(error_num);
+  }
+
+  // Loop through all the results and connect to the first we can.
+  for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != NULL;
+       cur_addr = cur_addr->ai_next) {
+    sockfd_ = socket(
+        cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol);
+    if (sockfd_ != -1) {
+      // Connect the client socket to the server socket.
+      if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) {
+        close(sockfd_);
+        sockfd_ = -1;
+      }
+    }
+  }
+
+  freeaddrinfo(servinfo);  // all done with this structure
+
+  if (sockfd_ == -1) {
+    GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to "
+                        << host_name_ << ":" << port_num_;
+  }
+}
+
+// End of class Streaming Listener
+#endif  // GTEST_CAN_STREAM_RESULTS__
+
+// Class ScopedTrace
+
+// Pushes the given source file location and message onto a per-thread
+// trace stack maintained by Google Test.
+ScopedTrace::ScopedTrace(const char* file, int line, const Message& message)
+    GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) {
+  TraceInfo trace;
+  trace.file = file;
+  trace.line = line;
+  trace.message = message.GetString();
+
+  UnitTest::GetInstance()->PushGTestTrace(trace);
+}
+
+// Pops the info pushed by the c'tor.
+ScopedTrace::~ScopedTrace()
+    GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) {
+  UnitTest::GetInstance()->PopGTestTrace();
+}
+
+
+// class OsStackTraceGetter
+
+const char* const OsStackTraceGetterInterface::kElidedFramesMarker =
+    "... " GTEST_NAME_ " internal frames ...";
+
+string OsStackTraceGetter::CurrentStackTrace(int /*max_depth*/,
+                                             int /*skip_count*/) {
+  return "";
+}
+
+void OsStackTraceGetter::UponLeavingGTest() {}
+
+// A helper class that creates the premature-exit file in its
+// constructor and deletes the file in its destructor.
+class ScopedPrematureExitFile {
+ public:
+  explicit ScopedPrematureExitFile(const char* premature_exit_filepath)
+      : premature_exit_filepath_(premature_exit_filepath) {
+    // If a path to the premature-exit file is specified...
+    if (premature_exit_filepath != NULL && *premature_exit_filepath != '\0') {
+      // create the file with a single "0" character in it.  I/O
+      // errors are ignored as there's nothing better we can do and we
+      // don't want to fail the test because of this.
+      FILE* pfile = posix::FOpen(premature_exit_filepath, "w");
+      fwrite("0", 1, 1, pfile);
+      fclose(pfile);
+    }
+  }
+
+  ~ScopedPrematureExitFile() {
+    if (premature_exit_filepath_ != NULL && *premature_exit_filepath_ != '\0') {
+      remove(premature_exit_filepath_);
+    }
+  }
+
+ private:
+  const char* const premature_exit_filepath_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile);
+};
+
+}  // namespace internal
+
+// class TestEventListeners
+
+TestEventListeners::TestEventListeners()
+    : repeater_(new internal::TestEventRepeater()),
+      default_result_printer_(NULL),
+      default_xml_generator_(NULL) {
+}
+
+TestEventListeners::~TestEventListeners() { delete repeater_; }
+
+// Returns the standard listener responsible for the default console
+// output.  Can be removed from the listeners list to shut down default
+// console output.  Note that removing this object from the listener list
+// with Release transfers its ownership to the user.
+void TestEventListeners::Append(TestEventListener* listener) {
+  repeater_->Append(listener);
+}
+
+// Removes the given event listener from the list and returns it.  It then
+// becomes the caller's responsibility to delete the listener. Returns
+// NULL if the listener is not found in the list.
+TestEventListener* TestEventListeners::Release(TestEventListener* listener) {
+  if (listener == default_result_printer_)
+    default_result_printer_ = NULL;
+  else if (listener == default_xml_generator_)
+    default_xml_generator_ = NULL;
+  return repeater_->Release(listener);
+}
+
+// Returns repeater that broadcasts the TestEventListener events to all
+// subscribers.
+TestEventListener* TestEventListeners::repeater() { return repeater_; }
+
+// Sets the default_result_printer attribute to the provided listener.
+// The listener is also added to the listener list and previous
+// default_result_printer is removed from it and deleted. The listener can
+// also be NULL in which case it will not be added to the list. Does
+// nothing if the previous and the current listener objects are the same.
+void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) {
+  if (default_result_printer_ != listener) {
+    // It is an error to pass this method a listener that is already in the
+    // list.
+    delete Release(default_result_printer_);
+    default_result_printer_ = listener;
+    if (listener != NULL)
+      Append(listener);
+  }
+}
+
+// Sets the default_xml_generator attribute to the provided listener.  The
+// listener is also added to the listener list and previous
+// default_xml_generator is removed from it and deleted. The listener can
+// also be NULL in which case it will not be added to the list. Does
+// nothing if the previous and the current listener objects are the same.
+void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) {
+  if (default_xml_generator_ != listener) {
+    // It is an error to pass this method a listener that is already in the
+    // list.
+    delete Release(default_xml_generator_);
+    default_xml_generator_ = listener;
+    if (listener != NULL)
+      Append(listener);
+  }
+}
+
+// Controls whether events will be forwarded by the repeater to the
+// listeners in the list.
+bool TestEventListeners::EventForwardingEnabled() const {
+  return repeater_->forwarding_enabled();
+}
+
+void TestEventListeners::SuppressEventForwarding() {
+  repeater_->set_forwarding_enabled(false);
+}
+
+// class UnitTest
+
+// Gets the singleton UnitTest object.  The first time this method is
+// called, a UnitTest object is constructed and returned.  Consecutive
+// calls will return the same object.
+//
+// We don't protect this under mutex_ as a user is not supposed to
+// call this before main() starts, from which point on the return
+// value will never change.
+UnitTest* UnitTest::GetInstance() {
+  // When compiled with MSVC 7.1 in optimized mode, destroying the
+  // UnitTest object upon exiting the program messes up the exit code,
+  // causing successful tests to appear failed.  We have to use a
+  // different implementation in this case to bypass the compiler bug.
+  // This implementation makes the compiler happy, at the cost of
+  // leaking the UnitTest object.
+
+  // CodeGear C++Builder insists on a public destructor for the
+  // default implementation.  Use this implementation to keep good OO
+  // design with private destructor.
+
+#if (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__)
+  static UnitTest* const instance = new UnitTest;
+  return instance;
+#else
+  static UnitTest instance;
+  return &instance;
+#endif  // (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__)
+}
+
+// Gets the number of successful test cases.
+int UnitTest::successful_test_case_count() const {
+  return impl()->successful_test_case_count();
+}
+
+// Gets the number of failed test cases.
+int UnitTest::failed_test_case_count() const {
+  return impl()->failed_test_case_count();
+}
+
+// Gets the number of all test cases.
+int UnitTest::total_test_case_count() const {
+  return impl()->total_test_case_count();
+}
+
+// Gets the number of all test cases that contain at least one test
+// that should run.
+int UnitTest::test_case_to_run_count() const {
+  return impl()->test_case_to_run_count();
+}
+
+// Gets the number of successful tests.
+int UnitTest::successful_test_count() const {
+  return impl()->successful_test_count();
+}
+
+// Gets the number of failed tests.
+int UnitTest::failed_test_count() const { return impl()->failed_test_count(); }
+
+// Gets the number of disabled tests that will be reported in the XML report.
+int UnitTest::reportable_disabled_test_count() const {
+  return impl()->reportable_disabled_test_count();
+}
+
+// Gets the number of disabled tests.
+int UnitTest::disabled_test_count() const {
+  return impl()->disabled_test_count();
+}
+
+// Gets the number of tests to be printed in the XML report.
+int UnitTest::reportable_test_count() const {
+  return impl()->reportable_test_count();
+}
+
+// Gets the number of all tests.
+int UnitTest::total_test_count() const { return impl()->total_test_count(); }
+
+// Gets the number of tests that should run.
+int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); }
+
+// Gets the time of the test program start, in ms from the start of the
+// UNIX epoch.
+internal::TimeInMillis UnitTest::start_timestamp() const {
+    return impl()->start_timestamp();
+}
+
+// Gets the elapsed time, in milliseconds.
+internal::TimeInMillis UnitTest::elapsed_time() const {
+  return impl()->elapsed_time();
+}
+
+// Returns true iff the unit test passed (i.e. all test cases passed).
+bool UnitTest::Passed() const { return impl()->Passed(); }
+
+// Returns true iff the unit test failed (i.e. some test case failed
+// or something outside of all tests failed).
+bool UnitTest::Failed() const { return impl()->Failed(); }
+
+// Gets the i-th test case among all the test cases. i can range from 0 to
+// total_test_case_count() - 1. If i is not in that range, returns NULL.
+const TestCase* UnitTest::GetTestCase(int i) const {
+  return impl()->GetTestCase(i);
+}
+
+// Returns the TestResult containing information on test failures and
+// properties logged outside of individual test cases.
+const TestResult& UnitTest::ad_hoc_test_result() const {
+  return *impl()->ad_hoc_test_result();
+}
+
+// Gets the i-th test case among all the test cases. i can range from 0 to
+// total_test_case_count() - 1. If i is not in that range, returns NULL.
+TestCase* UnitTest::GetMutableTestCase(int i) {
+  return impl()->GetMutableTestCase(i);
+}
+
+// Returns the list of event listeners that can be used to track events
+// inside Google Test.
+TestEventListeners& UnitTest::listeners() {
+  return *impl()->listeners();
+}
+
+// Registers and returns a global test environment.  When a test
+// program is run, all global test environments will be set-up in the
+// order they were registered.  After all tests in the program have
+// finished, all global test environments will be torn-down in the
+// *reverse* order they were registered.
+//
+// The UnitTest object takes ownership of the given environment.
+//
+// We don't protect this under mutex_, as we only support calling it
+// from the main thread.
+Environment* UnitTest::AddEnvironment(Environment* env) {
+  if (env == NULL) {
+    return NULL;
+  }
+
+  impl_->environments().push_back(env);
+  return env;
+}
+
+// Adds a TestPartResult to the current TestResult object.  All Google Test
+// assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call
+// this to report their results.  The user code should use the
+// assertion macros instead of calling this directly.
+void UnitTest::AddTestPartResult(
+    TestPartResult::Type result_type,
+    const char* file_name,
+    int line_number,
+    const std::string& message,
+    const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) {
+  Message msg;
+  msg << message;
+
+  internal::MutexLock lock(&mutex_);
+  if (impl_->gtest_trace_stack().size() > 0) {
+    msg << "\n" << GTEST_NAME_ << " trace:";
+
+    for (int i = static_cast<int>(impl_->gtest_trace_stack().size());
+         i > 0; --i) {
+      const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1];
+      msg << "\n" << internal::FormatFileLocation(trace.file, trace.line)
+          << " " << trace.message;
+    }
+  }
+
+  if (os_stack_trace.c_str() != NULL && !os_stack_trace.empty()) {
+    msg << internal::kStackTraceMarker << os_stack_trace;
+  }
+
+  const TestPartResult result =
+    TestPartResult(result_type, file_name, line_number,
+                   msg.GetString().c_str());
+  impl_->GetTestPartResultReporterForCurrentThread()->
+      ReportTestPartResult(result);
+
+  if (result_type != TestPartResult::kSuccess) {
+    // gtest_break_on_failure takes precedence over
+    // gtest_throw_on_failure.  This allows a user to set the latter
+    // in the code (perhaps in order to use Google Test assertions
+    // with another testing framework) and specify the former on the
+    // command line for debugging.
+    if (GTEST_FLAG(break_on_failure)) {
+#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+      // Using DebugBreak on Windows allows gtest to still break into a debugger
+      // when a failure happens and both the --gtest_break_on_failure and
+      // the --gtest_catch_exceptions flags are specified.
+      DebugBreak();
+#else
+      // Dereference NULL through a volatile pointer to prevent the compiler
+      // from removing. We use this rather than abort() or __builtin_trap() for
+      // portability: Symbian doesn't implement abort() well, and some debuggers
+      // don't correctly trap abort().
+      *static_cast<volatile int*>(NULL) = 1;
+#endif  // GTEST_OS_WINDOWS
+    } else if (GTEST_FLAG(throw_on_failure)) {
+#if GTEST_HAS_EXCEPTIONS
+      throw internal::GoogleTestFailureException(result);
+#else
+      // We cannot call abort() as it generates a pop-up in debug mode
+      // that cannot be suppressed in VC 7.1 or below.
+      exit(1);
+#endif
+    }
+  }
+}
+
+// Adds a TestProperty to the current TestResult object when invoked from
+// inside a test, to current TestCase's ad_hoc_test_result_ when invoked
+// from SetUpTestCase or TearDownTestCase, or to the global property set
+// when invoked elsewhere.  If the result already contains a property with
+// the same key, the value will be updated.
+void UnitTest::RecordProperty(const std::string& key,
+                              const std::string& value) {
+  impl_->RecordProperty(TestProperty(key, value));
+}
+
+// Runs all tests in this UnitTest object and prints the result.
+// Returns 0 if successful, or 1 otherwise.
+//
+// We don't protect this under mutex_, as we only support calling it
+// from the main thread.
+int UnitTest::Run() {
+  const bool in_death_test_child_process =
+      internal::GTEST_FLAG(internal_run_death_test).length() > 0;
+
+  // Google Test implements this protocol for catching that a test
+  // program exits before returning control to Google Test:
+  //
+  //   1. Upon start, Google Test creates a file whose absolute path
+  //      is specified by the environment variable
+  //      TEST_PREMATURE_EXIT_FILE.
+  //   2. When Google Test has finished its work, it deletes the file.
+  //
+  // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before
+  // running a Google-Test-based test program and check the existence
+  // of the file at the end of the test execution to see if it has
+  // exited prematurely.
+
+  // If we are in the child process of a death test, don't
+  // create/delete the premature exit file, as doing so is unnecessary
+  // and will confuse the parent process.  Otherwise, create/delete
+  // the file upon entering/leaving this function.  If the program
+  // somehow exits before this function has a chance to return, the
+  // premature-exit file will be left undeleted, causing a test runner
+  // that understands the premature-exit-file protocol to report the
+  // test as having failed.
+  const internal::ScopedPrematureExitFile premature_exit_file(
+      in_death_test_child_process ?
+      NULL : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE"));
+
+  // Captures the value of GTEST_FLAG(catch_exceptions).  This value will be
+  // used for the duration of the program.
+  impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions));
+
+#if GTEST_HAS_SEH
+  // Either the user wants Google Test to catch exceptions thrown by the
+  // tests or this is executing in the context of death test child
+  // process. In either case the user does not want to see pop-up dialogs
+  // about crashes - they are expected.
+  if (impl()->catch_exceptions() || in_death_test_child_process) {
+# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT
+    // SetErrorMode doesn't exist on CE.
+    SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT |
+                 SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX);
+# endif  // !GTEST_OS_WINDOWS_MOBILE
+
+# if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE
+    // Death test children can be terminated with _abort().  On Windows,
+    // _abort() can show a dialog with a warning message.  This forces the
+    // abort message to go to stderr instead.
+    _set_error_mode(_OUT_TO_STDERR);
+# endif
+
+# if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE
+    // In the debug version, Visual Studio pops up a separate dialog
+    // offering a choice to debug the aborted program. We need to suppress
+    // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement
+    // executed. Google Test will notify the user of any unexpected
+    // failure via stderr.
+    //
+    // VC++ doesn't define _set_abort_behavior() prior to the version 8.0.
+    // Users of prior VC versions shall suffer the agony and pain of
+    // clicking through the countless debug dialogs.
+    // TODO(vladl@google.com): find a way to suppress the abort dialog() in the
+    // debug mode when compiled with VC 7.1 or lower.
+    if (!GTEST_FLAG(break_on_failure))
+      _set_abort_behavior(
+          0x0,                                    // Clear the following flags:
+          _WRITE_ABORT_MSG | _CALL_REPORTFAULT);  // pop-up window, core dump.
+# endif
+  }
+#endif  // GTEST_HAS_SEH
+
+  return internal::HandleExceptionsInMethodIfSupported(
+      impl(),
+      &internal::UnitTestImpl::RunAllTests,
+      "auxiliary test code (environments or event listeners)") ? 0 : 1;
+}
+
+// Returns the working directory when the first TEST() or TEST_F() was
+// executed.
+const char* UnitTest::original_working_dir() const {
+  return impl_->original_working_dir_.c_str();
+}
+
+// Returns the TestCase object for the test that's currently running,
+// or NULL if no test is running.
+const TestCase* UnitTest::current_test_case() const
+    GTEST_LOCK_EXCLUDED_(mutex_) {
+  internal::MutexLock lock(&mutex_);
+  return impl_->current_test_case();
+}
+
+// Returns the TestInfo object for the test that's currently running,
+// or NULL if no test is running.
+const TestInfo* UnitTest::current_test_info() const
+    GTEST_LOCK_EXCLUDED_(mutex_) {
+  internal::MutexLock lock(&mutex_);
+  return impl_->current_test_info();
+}
+
+// Returns the random seed used at the start of the current test run.
+int UnitTest::random_seed() const { return impl_->random_seed(); }
+
+#if GTEST_HAS_PARAM_TEST
+// Returns ParameterizedTestCaseRegistry object used to keep track of
+// value-parameterized tests and instantiate and register them.
+internal::ParameterizedTestCaseRegistry&
+    UnitTest::parameterized_test_registry()
+        GTEST_LOCK_EXCLUDED_(mutex_) {
+  return impl_->parameterized_test_registry();
+}
+#endif  // GTEST_HAS_PARAM_TEST
+
+// Creates an empty UnitTest.
+UnitTest::UnitTest() {
+  impl_ = new internal::UnitTestImpl(this);
+}
+
+// Destructor of UnitTest.
+UnitTest::~UnitTest() {
+  delete impl_;
+}
+
+// Pushes a trace defined by SCOPED_TRACE() on to the per-thread
+// Google Test trace stack.
+void UnitTest::PushGTestTrace(const internal::TraceInfo& trace)
+    GTEST_LOCK_EXCLUDED_(mutex_) {
+  internal::MutexLock lock(&mutex_);
+  impl_->gtest_trace_stack().push_back(trace);
+}
+
+// Pops a trace from the per-thread Google Test trace stack.
+void UnitTest::PopGTestTrace()
+    GTEST_LOCK_EXCLUDED_(mutex_) {
+  internal::MutexLock lock(&mutex_);
+  impl_->gtest_trace_stack().pop_back();
+}
+
+namespace internal {
+
+UnitTestImpl::UnitTestImpl(UnitTest* parent)
+    : parent_(parent),
+      GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */)
+      default_global_test_part_result_reporter_(this),
+      default_per_thread_test_part_result_reporter_(this),
+      GTEST_DISABLE_MSC_WARNINGS_POP_()
+      global_test_part_result_repoter_(
+          &default_global_test_part_result_reporter_),
+      per_thread_test_part_result_reporter_(
+          &default_per_thread_test_part_result_reporter_),
+#if GTEST_HAS_PARAM_TEST
+      parameterized_test_registry_(),
+      parameterized_tests_registered_(false),
+#endif  // GTEST_HAS_PARAM_TEST
+      last_death_test_case_(-1),
+      current_test_case_(NULL),
+      current_test_info_(NULL),
+      ad_hoc_test_result_(),
+      os_stack_trace_getter_(NULL),
+      post_flag_parse_init_performed_(false),
+      random_seed_(0),  // Will be overridden by the flag before first use.
+      random_(0),  // Will be reseeded before first use.
+      start_timestamp_(0),
+      elapsed_time_(0),
+#if GTEST_HAS_DEATH_TEST
+      death_test_factory_(new DefaultDeathTestFactory),
+#endif
+      // Will be overridden by the flag before first use.
+      catch_exceptions_(false) {
+  listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter);
+}
+
+UnitTestImpl::~UnitTestImpl() {
+  // Deletes every TestCase.
+  ForEach(test_cases_, internal::Delete<TestCase>);
+
+  // Deletes every Environment.
+  ForEach(environments_, internal::Delete<Environment>);
+
+  delete os_stack_trace_getter_;
+}
+
+// Adds a TestProperty to the current TestResult object when invoked in a
+// context of a test, to current test case's ad_hoc_test_result when invoke
+// from SetUpTestCase/TearDownTestCase, or to the global property set
+// otherwise.  If the result already contains a property with the same key,
+// the value will be updated.
+void UnitTestImpl::RecordProperty(const TestProperty& test_property) {
+  std::string xml_element;
+  TestResult* test_result;  // TestResult appropriate for property recording.
+
+  if (current_test_info_ != NULL) {
+    xml_element = "testcase";
+    test_result = &(current_test_info_->result_);
+  } else if (current_test_case_ != NULL) {
+    xml_element = "testsuite";
+    test_result = &(current_test_case_->ad_hoc_test_result_);
+  } else {
+    xml_element = "testsuites";
+    test_result = &ad_hoc_test_result_;
+  }
+  test_result->RecordProperty(xml_element, test_property);
+}
+
+#if GTEST_HAS_DEATH_TEST
+// Disables event forwarding if the control is currently in a death test
+// subprocess. Must not be called before InitGoogleTest.
+void UnitTestImpl::SuppressTestEventsIfInSubprocess() {
+  if (internal_run_death_test_flag_.get() != NULL)
+    listeners()->SuppressEventForwarding();
+}
+#endif  // GTEST_HAS_DEATH_TEST
+
+// Initializes event listeners performing XML output as specified by
+// UnitTestOptions. Must not be called before InitGoogleTest.
+void UnitTestImpl::ConfigureXmlOutput() {
+  const std::string& output_format = UnitTestOptions::GetOutputFormat();
+  if (output_format == "xml") {
+    listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter(
+        UnitTestOptions::GetAbsolutePathToOutputFile().c_str()));
+  } else if (output_format != "") {
+    printf("WARNING: unrecognized output format \"%s\" ignored.\n",
+           output_format.c_str());
+    fflush(stdout);
+  }
+}
+
+#if GTEST_CAN_STREAM_RESULTS_
+// Initializes event listeners for streaming test results in string form.
+// Must not be called before InitGoogleTest.
+void UnitTestImpl::ConfigureStreamingOutput() {
+  const std::string& target = GTEST_FLAG(stream_result_to);
+  if (!target.empty()) {
+    const size_t pos = target.find(':');
+    if (pos != std::string::npos) {
+      listeners()->Append(new StreamingListener(target.substr(0, pos),
+                                                target.substr(pos+1)));
+    } else {
+      printf("WARNING: unrecognized streaming target \"%s\" ignored.\n",
+             target.c_str());
+      fflush(stdout);
+    }
+  }
+}
+#endif  // GTEST_CAN_STREAM_RESULTS_
+
+// Performs initialization dependent upon flag values obtained in
+// ParseGoogleTestFlagsOnly.  Is called from InitGoogleTest after the call to
+// ParseGoogleTestFlagsOnly.  In case a user neglects to call InitGoogleTest
+// this function is also called from RunAllTests.  Since this function can be
+// called more than once, it has to be idempotent.
+void UnitTestImpl::PostFlagParsingInit() {
+  // Ensures that this function does not execute more than once.
+  if (!post_flag_parse_init_performed_) {
+    post_flag_parse_init_performed_ = true;
+
+#if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_)
+    // Register to send notifications about key process state changes.
+    listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_());
+#endif  // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_)
+
+#if GTEST_HAS_DEATH_TEST
+    InitDeathTestSubprocessControlInfo();
+    SuppressTestEventsIfInSubprocess();
+#endif  // GTEST_HAS_DEATH_TEST
+
+    // Registers parameterized tests. This makes parameterized tests
+    // available to the UnitTest reflection API without running
+    // RUN_ALL_TESTS.
+    RegisterParameterizedTests();
+
+    // Configures listeners for XML output. This makes it possible for users
+    // to shut down the default XML output before invoking RUN_ALL_TESTS.
+    ConfigureXmlOutput();
+
+#if GTEST_CAN_STREAM_RESULTS_
+    // Configures listeners for streaming test results to the specified server.
+    ConfigureStreamingOutput();
+#endif  // GTEST_CAN_STREAM_RESULTS_
+  }
+}
+
+// A predicate that checks the name of a TestCase against a known
+// value.
+//
+// This is used for implementation of the UnitTest class only.  We put
+// it in the anonymous namespace to prevent polluting the outer
+// namespace.
+//
+// TestCaseNameIs is copyable.
+class TestCaseNameIs {
+ public:
+  // Constructor.
+  explicit TestCaseNameIs(const std::string& name)
+      : name_(name) {}
+
+  // Returns true iff the name of test_case matches name_.
+  bool operator()(const TestCase* test_case) const {
+    return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0;
+  }
+
+ private:
+  std::string name_;
+};
+
+// Finds and returns a TestCase with the given name.  If one doesn't
+// exist, creates one and returns it.  It's the CALLER'S
+// RESPONSIBILITY to ensure that this function is only called WHEN THE
+// TESTS ARE NOT SHUFFLED.
+//
+// Arguments:
+//
+//   test_case_name: name of the test case
+//   type_param:     the name of the test case's type parameter, or NULL if
+//                   this is not a typed or a type-parameterized test case.
+//   set_up_tc:      pointer to the function that sets up the test case
+//   tear_down_tc:   pointer to the function that tears down the test case
+TestCase* UnitTestImpl::GetTestCase(const char* test_case_name,
+                                    const char* type_param,
+                                    Test::SetUpTestCaseFunc set_up_tc,
+                                    Test::TearDownTestCaseFunc tear_down_tc) {
+  // Can we find a TestCase with the given name?
+  const std::vector<TestCase*>::const_iterator test_case =
+      std::find_if(test_cases_.begin(), test_cases_.end(),
+                   TestCaseNameIs(test_case_name));
+
+  if (test_case != test_cases_.end())
+    return *test_case;
+
+  // No.  Let's create one.
+  TestCase* const new_test_case =
+      new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc);
+
+  // Is this a death test case?
+  if (internal::UnitTestOptions::MatchesFilter(test_case_name,
+                                               kDeathTestCaseFilter)) {
+    // Yes.  Inserts the test case after the last death test case
+    // defined so far.  This only works when the test cases haven't
+    // been shuffled.  Otherwise we may end up running a death test
+    // after a non-death test.
+    ++last_death_test_case_;
+    test_cases_.insert(test_cases_.begin() + last_death_test_case_,
+                       new_test_case);
+  } else {
+    // No.  Appends to the end of the list.
+    test_cases_.push_back(new_test_case);
+  }
+
+  test_case_indices_.push_back(static_cast<int>(test_case_indices_.size()));
+  return new_test_case;
+}
+
+// Helpers for setting up / tearing down the given environment.  They
+// are for use in the ForEach() function.
+static void SetUpEnvironment(Environment* env) { env->SetUp(); }
+static void TearDownEnvironment(Environment* env) { env->TearDown(); }
+
+// Runs all tests in this UnitTest object, prints the result, and
+// returns true if all tests are successful.  If any exception is
+// thrown during a test, the test is considered to be failed, but the
+// rest of the tests will still be run.
+//
+// When parameterized tests are enabled, it expands and registers
+// parameterized tests first in RegisterParameterizedTests().
+// All other functions called from RunAllTests() may safely assume that
+// parameterized tests are ready to be counted and run.
+bool UnitTestImpl::RunAllTests() {
+  // Makes sure InitGoogleTest() was called.
+  if (!GTestIsInitialized()) {
+    printf("%s",
+           "\nThis test program did NOT call ::testing::InitGoogleTest "
+           "before calling RUN_ALL_TESTS().  Please fix it.\n");
+    return false;
+  }
+
+  // Do not run any test if the --help flag was specified.
+  if (g_help_flag)
+    return true;
+
+  // Repeats the call to the post-flag parsing initialization in case the
+  // user didn't call InitGoogleTest.
+  PostFlagParsingInit();
+
+  // Even if sharding is not on, test runners may want to use the
+  // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding
+  // protocol.
+  internal::WriteToShardStatusFileIfNeeded();
+
+  // True iff we are in a subprocess for running a thread-safe-style
+  // death test.
+  bool in_subprocess_for_death_test = false;
+
+#if GTEST_HAS_DEATH_TEST
+  in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL);
+# if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_)
+  if (in_subprocess_for_death_test) {
+    GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_();
+  }
+# endif  // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_)
+#endif  // GTEST_HAS_DEATH_TEST
+
+  const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex,
+                                        in_subprocess_for_death_test);
+
+  // Compares the full test names with the filter to decide which
+  // tests to run.
+  const bool has_tests_to_run = FilterTests(should_shard
+                                              ? HONOR_SHARDING_PROTOCOL
+                                              : IGNORE_SHARDING_PROTOCOL) > 0;
+
+  // Lists the tests and exits if the --gtest_list_tests flag was specified.
+  if (GTEST_FLAG(list_tests)) {
+    // This must be called *after* FilterTests() has been called.
+    ListTestsMatchingFilter();
+    return true;
+  }
+
+  random_seed_ = GTEST_FLAG(shuffle) ?
+      GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0;
+
+  // True iff at least one test has failed.
+  bool failed = false;
+
+  TestEventListener* repeater = listeners()->repeater();
+
+  start_timestamp_ = GetTimeInMillis();
+  repeater->OnTestProgramStart(*parent_);
+
+  // How many times to repeat the tests?  We don't want to repeat them
+  // when we are inside the subprocess of a death test.
+  const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat);
+  // Repeats forever if the repeat count is negative.
+  const bool forever = repeat < 0;
+  for (int i = 0; forever || i != repeat; i++) {
+    // We want to preserve failures generated by ad-hoc test
+    // assertions executed before RUN_ALL_TESTS().
+    ClearNonAdHocTestResult();
+
+    const TimeInMillis start = GetTimeInMillis();
+
+    // Shuffles test cases and tests if requested.
+    if (has_tests_to_run && GTEST_FLAG(shuffle)) {
+      random()->Reseed(random_seed_);
+      // This should be done before calling OnTestIterationStart(),
+      // such that a test event listener can see the actual test order
+      // in the event.
+      ShuffleTests();
+    }
+
+    // Tells the unit test event listeners that the tests are about to start.
+    repeater->OnTestIterationStart(*parent_, i);
+
+    // Runs each test case if there is at least one test to run.
+    if (has_tests_to_run) {
+      // Sets up all environments beforehand.
+      repeater->OnEnvironmentsSetUpStart(*parent_);
+      ForEach(environments_, SetUpEnvironment);
+      repeater->OnEnvironmentsSetUpEnd(*parent_);
+
+      // Runs the tests only if there was no fatal failure during global
+      // set-up.
+      if (!Test::HasFatalFailure()) {
+        for (int test_index = 0; test_index < total_test_case_count();
+             test_index++) {
+          GetMutableTestCase(test_index)->Run();
+        }
+      }
+
+      // Tears down all environments in reverse order afterwards.
+      repeater->OnEnvironmentsTearDownStart(*parent_);
+      std::for_each(environments_.rbegin(), environments_.rend(),
+                    TearDownEnvironment);
+      repeater->OnEnvironmentsTearDownEnd(*parent_);
+    }
+
+    elapsed_time_ = GetTimeInMillis() - start;
+
+    // Tells the unit test event listener that the tests have just finished.
+    repeater->OnTestIterationEnd(*parent_, i);
+
+    // Gets the result and clears it.
+    if (!Passed()) {
+      failed = true;
+    }
+
+    // Restores the original test order after the iteration.  This
+    // allows the user to quickly repro a failure that happens in the
+    // N-th iteration without repeating the first (N - 1) iterations.
+    // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in
+    // case the user somehow changes the value of the flag somewhere
+    // (it's always safe to unshuffle the tests).
+    UnshuffleTests();
+
+    if (GTEST_FLAG(shuffle)) {
+      // Picks a new random seed for each iteration.
+      random_seed_ = GetNextRandomSeed(random_seed_);
+    }
+  }
+
+  repeater->OnTestProgramEnd(*parent_);
+
+  return !failed;
+}
+
+// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file
+// if the variable is present. If a file already exists at this location, this
+// function will write over it. If the variable is present, but the file cannot
+// be created, prints an error and exits.
+void WriteToShardStatusFileIfNeeded() {
+  const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile);
+  if (test_shard_file != NULL) {
+    FILE* const file = posix::FOpen(test_shard_file, "w");
+    if (file == NULL) {
+      ColoredPrintf(COLOR_RED,
+                    "Could not write to the test shard status file \"%s\" "
+                    "specified by the %s environment variable.\n",
+                    test_shard_file, kTestShardStatusFile);
+      fflush(stdout);
+      exit(EXIT_FAILURE);
+    }
+    fclose(file);
+  }
+}
+
+// Checks whether sharding is enabled by examining the relevant
+// environment variable values. If the variables are present,
+// but inconsistent (i.e., shard_index >= total_shards), prints
+// an error and exits. If in_subprocess_for_death_test, sharding is
+// disabled because it must only be applied to the original test
+// process. Otherwise, we could filter out death tests we intended to execute.
+bool ShouldShard(const char* total_shards_env,
+                 const char* shard_index_env,
+                 bool in_subprocess_for_death_test) {
+  if (in_subprocess_for_death_test) {
+    return false;
+  }
+
+  const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1);
+  const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1);
+
+  if (total_shards == -1 && shard_index == -1) {
+    return false;
+  } else if (total_shards == -1 && shard_index != -1) {
+    const Message msg = Message()
+      << "Invalid environment variables: you have "
+      << kTestShardIndex << " = " << shard_index
+      << ", but have left " << kTestTotalShards << " unset.\n";
+    ColoredPrintf(COLOR_RED, msg.GetString().c_str());
+    fflush(stdout);
+    exit(EXIT_FAILURE);
+  } else if (total_shards != -1 && shard_index == -1) {
+    const Message msg = Message()
+      << "Invalid environment variables: you have "
+      << kTestTotalShards << " = " << total_shards
+      << ", but have left " << kTestShardIndex << " unset.\n";
+    ColoredPrintf(COLOR_RED, msg.GetString().c_str());
+    fflush(stdout);
+    exit(EXIT_FAILURE);
+  } else if (shard_index < 0 || shard_index >= total_shards) {
+    const Message msg = Message()
+      << "Invalid environment variables: we require 0 <= "
+      << kTestShardIndex << " < " << kTestTotalShards
+      << ", but you have " << kTestShardIndex << "=" << shard_index
+      << ", " << kTestTotalShards << "=" << total_shards << ".\n";
+    ColoredPrintf(COLOR_RED, msg.GetString().c_str());
+    fflush(stdout);
+    exit(EXIT_FAILURE);
+  }
+
+  return total_shards > 1;
+}
+
+// Parses the environment variable var as an Int32. If it is unset,
+// returns default_val. If it is not an Int32, prints an error
+// and aborts.
+Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) {
+  const char* str_val = posix::GetEnv(var);
+  if (str_val == NULL) {
+    return default_val;
+  }
+
+  Int32 result;
+  if (!ParseInt32(Message() << "The value of environment variable " << var,
+                  str_val, &result)) {
+    exit(EXIT_FAILURE);
+  }
+  return result;
+}
+
+// Given the total number of shards, the shard index, and the test id,
+// returns true iff the test should be run on this shard. The test id is
+// some arbitrary but unique non-negative integer assigned to each test
+// method. Assumes that 0 <= shard_index < total_shards.
+bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) {
+  return (test_id % total_shards) == shard_index;
+}
+
+// Compares the name of each test with the user-specified filter to
+// decide whether the test should be run, then records the result in
+// each TestCase and TestInfo object.
+// If shard_tests == true, further filters tests based on sharding
+// variables in the environment - see
+// http://code.google.com/p/googletest/wiki/GoogleTestAdvancedGuide.
+// Returns the number of tests that should run.
+int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) {
+  const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ?
+      Int32FromEnvOrDie(kTestTotalShards, -1) : -1;
+  const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ?
+      Int32FromEnvOrDie(kTestShardIndex, -1) : -1;
+
+  // num_runnable_tests are the number of tests that will
+  // run across all shards (i.e., match filter and are not disabled).
+  // num_selected_tests are the number of tests to be run on
+  // this shard.
+  int num_runnable_tests = 0;
+  int num_selected_tests = 0;
+  for (size_t i = 0; i < test_cases_.size(); i++) {
+    TestCase* const test_case = test_cases_[i];
+    const std::string &test_case_name = test_case->name();
+    test_case->set_should_run(false);
+
+    for (size_t j = 0; j < test_case->test_info_list().size(); j++) {
+      TestInfo* const test_info = test_case->test_info_list()[j];
+      const std::string test_name(test_info->name());
+      // A test is disabled if test case name or test name matches
+      // kDisableTestFilter.
+      const bool is_disabled =
+          internal::UnitTestOptions::MatchesFilter(test_case_name,
+                                                   kDisableTestFilter) ||
+          internal::UnitTestOptions::MatchesFilter(test_name,
+                                                   kDisableTestFilter);
+      test_info->is_disabled_ = is_disabled;
+
+      const bool matches_filter =
+          internal::UnitTestOptions::FilterMatchesTest(test_case_name,
+                                                       test_name);
+      test_info->matches_filter_ = matches_filter;
+
+      const bool is_runnable =
+          (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) &&
+          matches_filter;
+
+      const bool is_selected = is_runnable &&
+          (shard_tests == IGNORE_SHARDING_PROTOCOL ||
+           ShouldRunTestOnShard(total_shards, shard_index,
+                                num_runnable_tests));
+
+      num_runnable_tests += is_runnable;
+      num_selected_tests += is_selected;
+
+      test_info->should_run_ = is_selected;
+      test_case->set_should_run(test_case->should_run() || is_selected);
+    }
+  }
+  return num_selected_tests;
+}
+
+// Prints the given C-string on a single line by replacing all '\n'
+// characters with string "\\n".  If the output takes more than
+// max_length characters, only prints the first max_length characters
+// and "...".
+static void PrintOnOneLine(const char* str, int max_length) {
+  if (str != NULL) {
+    for (int i = 0; *str != '\0'; ++str) {
+      if (i >= max_length) {
+        printf("...");
+        break;
+      }
+      if (*str == '\n') {
+        printf("\\n");
+        i += 2;
+      } else {
+        printf("%c", *str);
+        ++i;
+      }
+    }
+  }
+}
+
+// Prints the names of the tests matching the user-specified filter flag.
+void UnitTestImpl::ListTestsMatchingFilter() {
+  // Print at most this many characters for each type/value parameter.
+  const int kMaxParamLength = 250;
+
+  for (size_t i = 0; i < test_cases_.size(); i++) {
+    const TestCase* const test_case = test_cases_[i];
+    bool printed_test_case_name = false;
+
+    for (size_t j = 0; j < test_case->test_info_list().size(); j++) {
+      const TestInfo* const test_info =
+          test_case->test_info_list()[j];
+      if (test_info->matches_filter_) {
+        if (!printed_test_case_name) {
+          printed_test_case_name = true;
+          printf("%s.", test_case->name());
+          if (test_case->type_param() != NULL) {
+            printf("  # %s = ", kTypeParamLabel);
+            // We print the type parameter on a single line to make
+            // the output easy to parse by a program.
+            PrintOnOneLine(test_case->type_param(), kMaxParamLength);
+          }
+          printf("\n");
+        }
+        printf("  %s", test_info->name());
+        if (test_info->value_param() != NULL) {
+          printf("  # %s = ", kValueParamLabel);
+          // We print the value parameter on a single line to make the
+          // output easy to parse by a program.
+          PrintOnOneLine(test_info->value_param(), kMaxParamLength);
+        }
+        printf("\n");
+      }
+    }
+  }
+  fflush(stdout);
+}
+
+// Sets the OS stack trace getter.
+//
+// Does nothing if the input and the current OS stack trace getter are
+// the same; otherwise, deletes the old getter and makes the input the
+// current getter.
+void UnitTestImpl::set_os_stack_trace_getter(
+    OsStackTraceGetterInterface* getter) {
+  if (os_stack_trace_getter_ != getter) {
+    delete os_stack_trace_getter_;
+    os_stack_trace_getter_ = getter;
+  }
+}
+
+// Returns the current OS stack trace getter if it is not NULL;
+// otherwise, creates an OsStackTraceGetter, makes it the current
+// getter, and returns it.
+OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() {
+  if (os_stack_trace_getter_ == NULL) {
+#ifdef GTEST_OS_STACK_TRACE_GETTER_
+    os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_;
+#else
+    os_stack_trace_getter_ = new OsStackTraceGetter;
+#endif  // GTEST_OS_STACK_TRACE_GETTER_
+  }
+
+  return os_stack_trace_getter_;
+}
+
+// Returns the TestResult for the test that's currently running, or
+// the TestResult for the ad hoc test if no test is running.
+TestResult* UnitTestImpl::current_test_result() {
+  return current_test_info_ ?
+      &(current_test_info_->result_) : &ad_hoc_test_result_;
+}
+
+// Shuffles all test cases, and the tests within each test case,
+// making sure that death tests are still run first.
+void UnitTestImpl::ShuffleTests() {
+  // Shuffles the death test cases.
+  ShuffleRange(random(), 0, last_death_test_case_ + 1, &test_case_indices_);
+
+  // Shuffles the non-death test cases.
+  ShuffleRange(random(), last_death_test_case_ + 1,
+               static_cast<int>(test_cases_.size()), &test_case_indices_);
+
+  // Shuffles the tests inside each test case.
+  for (size_t i = 0; i < test_cases_.size(); i++) {
+    test_cases_[i]->ShuffleTests(random());
+  }
+}
+
+// Restores the test cases and tests to their order before the first shuffle.
+void UnitTestImpl::UnshuffleTests() {
+  for (size_t i = 0; i < test_cases_.size(); i++) {
+    // Unshuffles the tests in each test case.
+    test_cases_[i]->UnshuffleTests();
+    // Resets the index of each test case.
+    test_case_indices_[i] = static_cast<int>(i);
+  }
+}
+
+// Returns the current OS stack trace as an std::string.
+//
+// The maximum number of stack frames to be included is specified by
+// the gtest_stack_trace_depth flag.  The skip_count parameter
+// specifies the number of top frames to be skipped, which doesn't
+// count against the number of frames to be included.
+//
+// For example, if Foo() calls Bar(), which in turn calls
+// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in
+// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't.
+std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/,
+                                            int skip_count) {
+  // We pass skip_count + 1 to skip this wrapper function in addition
+  // to what the user really wants to skip.
+  return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1);
+}
+
+// Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to
+// suppress unreachable code warnings.
+namespace {
+class ClassUniqueToAlwaysTrue {};
+}
+
+bool IsTrue(bool condition) { return condition; }
+
+bool AlwaysTrue() {
+#if GTEST_HAS_EXCEPTIONS
+  // This condition is always false so AlwaysTrue() never actually throws,
+  // but it makes the compiler think that it may throw.
+  if (IsTrue(false))
+    throw ClassUniqueToAlwaysTrue();
+#endif  // GTEST_HAS_EXCEPTIONS
+  return true;
+}
+
+// If *pstr starts with the given prefix, modifies *pstr to be right
+// past the prefix and returns true; otherwise leaves *pstr unchanged
+// and returns false.  None of pstr, *pstr, and prefix can be NULL.
+bool SkipPrefix(const char* prefix, const char** pstr) {
+  const size_t prefix_len = strlen(prefix);
+  if (strncmp(*pstr, prefix, prefix_len) == 0) {
+    *pstr += prefix_len;
+    return true;
+  }
+  return false;
+}
+
+// Parses a string as a command line flag.  The string should have
+// the format "--flag=value".  When def_optional is true, the "=value"
+// part can be omitted.
+//
+// Returns the value of the flag, or NULL if the parsing failed.
+const char* ParseFlagValue(const char* str,
+                           const char* flag,
+                           bool def_optional) {
+  // str and flag must not be NULL.
+  if (str == NULL || flag == NULL) return NULL;
+
+  // The flag must start with "--" followed by GTEST_FLAG_PREFIX_.
+  const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag;
+  const size_t flag_len = flag_str.length();
+  if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL;
+
+  // Skips the flag name.
+  const char* flag_end = str + flag_len;
+
+  // When def_optional is true, it's OK to not have a "=value" part.
+  if (def_optional && (flag_end[0] == '\0')) {
+    return flag_end;
+  }
+
+  // If def_optional is true and there are more characters after the
+  // flag name, or if def_optional is false, there must be a '=' after
+  // the flag name.
+  if (flag_end[0] != '=') return NULL;
+
+  // Returns the string after "=".
+  return flag_end + 1;
+}
+
+// Parses a string for a bool flag, in the form of either
+// "--flag=value" or "--flag".
+//
+// In the former case, the value is taken as true as long as it does
+// not start with '0', 'f', or 'F'.
+//
+// In the latter case, the value is taken as true.
+//
+// On success, stores the value of the flag in *value, and returns
+// true.  On failure, returns false without changing *value.
+bool ParseBoolFlag(const char* str, const char* flag, bool* value) {
+  // Gets the value of the flag as a string.
+  const char* const value_str = ParseFlagValue(str, flag, true);
+
+  // Aborts if the parsing failed.
+  if (value_str == NULL) return false;
+
+  // Converts the string value to a bool.
+  *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F');
+  return true;
+}
+
+// Parses a string for an Int32 flag, in the form of
+// "--flag=value".
+//
+// On success, stores the value of the flag in *value, and returns
+// true.  On failure, returns false without changing *value.
+bool ParseInt32Flag(const char* str, const char* flag, Int32* value) {
+  // Gets the value of the flag as a string.
+  const char* const value_str = ParseFlagValue(str, flag, false);
+
+  // Aborts if the parsing failed.
+  if (value_str == NULL) return false;
+
+  // Sets *value to the value of the flag.
+  return ParseInt32(Message() << "The value of flag --" << flag,
+                    value_str, value);
+}
+
+// Parses a string for a string flag, in the form of
+// "--flag=value".
+//
+// On success, stores the value of the flag in *value, and returns
+// true.  On failure, returns false without changing *value.
+bool ParseStringFlag(const char* str, const char* flag, std::string* value) {
+  // Gets the value of the flag as a string.
+  const char* const value_str = ParseFlagValue(str, flag, false);
+
+  // Aborts if the parsing failed.
+  if (value_str == NULL) return false;
+
+  // Sets *value to the value of the flag.
+  *value = value_str;
+  return true;
+}
+
+// Determines whether a string has a prefix that Google Test uses for its
+// flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_.
+// If Google Test detects that a command line flag has its prefix but is not
+// recognized, it will print its help message. Flags starting with
+// GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test
+// internal flags and do not trigger the help message.
+static bool HasGoogleTestFlagPrefix(const char* str) {
+  return (SkipPrefix("--", &str) ||
+          SkipPrefix("-", &str) ||
+          SkipPrefix("/", &str)) &&
+         !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) &&
+         (SkipPrefix(GTEST_FLAG_PREFIX_, &str) ||
+          SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str));
+}
+
+// Prints a string containing code-encoded text.  The following escape
+// sequences can be used in the string to control the text color:
+//
+//   @@    prints a single '@' character.
+//   @R    changes the color to red.
+//   @G    changes the color to green.
+//   @Y    changes the color to yellow.
+//   @D    changes to the default terminal text color.
+//
+// TODO(wan@google.com): Write tests for this once we add stdout
+// capturing to Google Test.
+static void PrintColorEncoded(const char* str) {
+  GTestColor color = COLOR_DEFAULT;  // The current color.
+
+  // Conceptually, we split the string into segments divided by escape
+  // sequences.  Then we print one segment at a time.  At the end of
+  // each iteration, the str pointer advances to the beginning of the
+  // next segment.
+  for (;;) {
+    const char* p = strchr(str, '@');
+    if (p == NULL) {
+      ColoredPrintf(color, "%s", str);
+      return;
+    }
+
+    ColoredPrintf(color, "%s", std::string(str, p).c_str());
+
+    const char ch = p[1];
+    str = p + 2;
+    if (ch == '@') {
+      ColoredPrintf(color, "@");
+    } else if (ch == 'D') {
+      color = COLOR_DEFAULT;
+    } else if (ch == 'R') {
+      color = COLOR_RED;
+    } else if (ch == 'G') {
+      color = COLOR_GREEN;
+    } else if (ch == 'Y') {
+      color = COLOR_YELLOW;
+    } else {
+      --str;
+    }
+  }
+}
+
+static const char kColorEncodedHelpMessage[] =
+"This program contains tests written using " GTEST_NAME_ ". You can use the\n"
+"following command line flags to control its behavior:\n"
+"\n"
+"Test Selection:\n"
+"  @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n"
+"      List the names of all tests instead of running them. The name of\n"
+"      TEST(Foo, Bar) is \"Foo.Bar\".\n"
+"  @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS"
+    "[@G-@YNEGATIVE_PATTERNS]@D\n"
+"      Run only the tests whose name matches one of the positive patterns but\n"
+"      none of the negative patterns. '?' matches any single character; '*'\n"
+"      matches any substring; ':' separates two patterns.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n"
+"      Run all disabled tests too.\n"
+"\n"
+"Test Execution:\n"
+"  @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n"
+"      Run the tests repeatedly; use a negative count to repeat forever.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n"
+"      Randomize tests' orders on every iteration.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n"
+"      Random number seed to use for shuffling test orders (between 1 and\n"
+"      99999, or 0 to use a seed based on the current time).\n"
+"\n"
+"Test Output:\n"
+"  @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n"
+"      Enable/disable colored output. The default is @Gauto@D.\n"
+"  -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n"
+"      Don't print the elapsed time of each test.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "output=xml@Y[@G:@YDIRECTORY_PATH@G"
+    GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n"
+"      Generate an XML report in the given directory or with the given file\n"
+"      name. @YFILE_PATH@D defaults to @Gtest_details.xml@D.\n"
+#if GTEST_CAN_STREAM_RESULTS_
+"  @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n"
+"      Stream test results to the given server.\n"
+#endif  // GTEST_CAN_STREAM_RESULTS_
+"\n"
+"Assertion Behavior:\n"
+#if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS
+"  @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n"
+"      Set the default death test style.\n"
+#endif  // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS
+"  @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n"
+"      Turn assertion failures into debugger break-points.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n"
+"      Turn assertion failures into C++ exceptions.\n"
+"  @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n"
+"      Do not report exceptions as test failures. Instead, allow them\n"
+"      to crash the program or throw a pop-up (on Windows).\n"
+"\n"
+"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set "
+    "the corresponding\n"
+"environment variable of a flag (all letters in upper-case). For example, to\n"
+"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_
+    "color=no@D or set\n"
+"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n"
+"\n"
+"For more information, please read the " GTEST_NAME_ " documentation at\n"
+"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n"
+"(not one in your own code or tests), please report it to\n"
+"@G<" GTEST_DEV_EMAIL_ ">@D.\n";
+
+bool ParseGoogleTestFlag(const char* const arg) {
+  return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
+                       &GTEST_FLAG(also_run_disabled_tests)) ||
+      ParseBoolFlag(arg, kBreakOnFailureFlag,
+                    &GTEST_FLAG(break_on_failure)) ||
+      ParseBoolFlag(arg, kCatchExceptionsFlag,
+                    &GTEST_FLAG(catch_exceptions)) ||
+      ParseStringFlag(arg, kColorFlag, &GTEST_FLAG(color)) ||
+      ParseStringFlag(arg, kDeathTestStyleFlag,
+                      &GTEST_FLAG(death_test_style)) ||
+      ParseBoolFlag(arg, kDeathTestUseFork,
+                    &GTEST_FLAG(death_test_use_fork)) ||
+      ParseStringFlag(arg, kFilterFlag, &GTEST_FLAG(filter)) ||
+      ParseStringFlag(arg, kInternalRunDeathTestFlag,
+                      &GTEST_FLAG(internal_run_death_test)) ||
+      ParseBoolFlag(arg, kListTestsFlag, &GTEST_FLAG(list_tests)) ||
+      ParseStringFlag(arg, kOutputFlag, &GTEST_FLAG(output)) ||
+      ParseBoolFlag(arg, kPrintTimeFlag, &GTEST_FLAG(print_time)) ||
+      ParseInt32Flag(arg, kRandomSeedFlag, &GTEST_FLAG(random_seed)) ||
+      ParseInt32Flag(arg, kRepeatFlag, &GTEST_FLAG(repeat)) ||
+      ParseBoolFlag(arg, kShuffleFlag, &GTEST_FLAG(shuffle)) ||
+      ParseInt32Flag(arg, kStackTraceDepthFlag,
+                     &GTEST_FLAG(stack_trace_depth)) ||
+      ParseStringFlag(arg, kStreamResultToFlag,
+                      &GTEST_FLAG(stream_result_to)) ||
+      ParseBoolFlag(arg, kThrowOnFailureFlag,
+                    &GTEST_FLAG(throw_on_failure));
+}
+
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+void LoadFlagsFromFile(const std::string& path) {
+  FILE* flagfile = posix::FOpen(path.c_str(), "r");
+  if (!flagfile) {
+    fprintf(stderr,
+            "Unable to open file \"%s\"\n",
+            GTEST_FLAG(flagfile).c_str());
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+  }
+  std::string contents(ReadEntireFile(flagfile));
+  posix::FClose(flagfile);
+  std::vector<std::string> lines;
+  SplitString(contents, '\n', &lines);
+  for (size_t i = 0; i < lines.size(); ++i) {
+    if (lines[i].empty())
+      continue;
+    if (!ParseGoogleTestFlag(lines[i].c_str()))
+      g_help_flag = true;
+  }
+}
+#endif  // GTEST_USE_OWN_FLAGFILE_FLAG_
+
+// Parses the command line for Google Test flags, without initializing
+// other parts of Google Test.  The type parameter CharType can be
+// instantiated to either char or wchar_t.
+template <typename CharType>
+void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
+  for (int i = 1; i < *argc; i++) {
+    const std::string arg_string = StreamableToString(argv[i]);
+    const char* const arg = arg_string.c_str();
+
+    using internal::ParseBoolFlag;
+    using internal::ParseInt32Flag;
+    using internal::ParseStringFlag;
+
+    bool remove_flag = false;
+    if (ParseGoogleTestFlag(arg)) {
+      remove_flag = true;
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+    } else if (ParseStringFlag(arg, kFlagfileFlag, &GTEST_FLAG(flagfile))) {
+      LoadFlagsFromFile(GTEST_FLAG(flagfile));
+      remove_flag = true;
+#endif  // GTEST_USE_OWN_FLAGFILE_FLAG_
+    } else if (arg_string == "--help" || arg_string == "-h" ||
+               arg_string == "-?" || arg_string == "/?" ||
+               HasGoogleTestFlagPrefix(arg)) {
+      // Both help flag and unrecognized Google Test flags (excluding
+      // internal ones) trigger help display.
+      g_help_flag = true;
+    }
+
+    if (remove_flag) {
+      // Shift the remainder of the argv list left by one.  Note
+      // that argv has (*argc + 1) elements, the last one always being
+      // NULL.  The following loop moves the trailing NULL element as
+      // well.
+      for (int j = i; j != *argc; j++) {
+        argv[j] = argv[j + 1];
+      }
+
+      // Decrements the argument count.
+      (*argc)--;
+
+      // We also need to decrement the iterator as we just removed
+      // an element.
+      i--;
+    }
+  }
+
+  if (g_help_flag) {
+    // We print the help here instead of in RUN_ALL_TESTS(), as the
+    // latter may not be called at all if the user is using Google
+    // Test with another testing framework.
+    PrintColorEncoded(kColorEncodedHelpMessage);
+  }
+}
+
+// Parses the command line for Google Test flags, without initializing
+// other parts of Google Test.
+void ParseGoogleTestFlagsOnly(int* argc, char** argv) {
+  ParseGoogleTestFlagsOnlyImpl(argc, argv);
+}
+void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) {
+  ParseGoogleTestFlagsOnlyImpl(argc, argv);
+}
+
+// The internal implementation of InitGoogleTest().
+//
+// The type parameter CharType can be instantiated to either char or
+// wchar_t.
+template <typename CharType>
+void InitGoogleTestImpl(int* argc, CharType** argv) {
+  // We don't want to run the initialization code twice.
+  if (GTestIsInitialized()) return;
+
+  if (*argc <= 0) return;
+
+  g_argvs.clear();
+  for (int i = 0; i != *argc; i++) {
+    g_argvs.push_back(StreamableToString(argv[i]));
+  }
+
+  ParseGoogleTestFlagsOnly(argc, argv);
+  GetUnitTestImpl()->PostFlagParsingInit();
+}
+
+}  // namespace internal
+
+// Initializes Google Test.  This must be called before calling
+// RUN_ALL_TESTS().  In particular, it parses a command line for the
+// flags that Google Test recognizes.  Whenever a Google Test flag is
+// seen, it is removed from argv, and *argc is decremented.
+//
+// No value is returned.  Instead, the Google Test flag variables are
+// updated.
+//
+// Calling the function for the second time has no user-visible effect.
+void InitGoogleTest(int* argc, char** argv) {
+#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+  GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv);
+#else  // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+  internal::InitGoogleTestImpl(argc, argv);
+#endif  // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+}
+
+// This overloaded version can be used in Windows programs compiled in
+// UNICODE mode.
+void InitGoogleTest(int* argc, wchar_t** argv) {
+#if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+  GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv);
+#else  // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+  internal::InitGoogleTestImpl(argc, argv);
+#endif  // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_)
+}
+
+}  // namespace testing
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+GTEST_API_ int main(int argc, char **argv) {
+  printf("Running main() from gtest_main.cc\n");
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+// Copyright 2010, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+//
+// Tests that verify interaction of exceptions and death tests.
+
+#include "gtest/gtest-death-test.h"
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_DEATH_TEST
+
+# if GTEST_HAS_SEH
+#  include <windows.h>          // For RaiseException().
+# endif
+
+# include "gtest/gtest-spi.h"
+
+# if GTEST_HAS_EXCEPTIONS
+
+#  include <exception>  // For std::exception.
+
+// Tests that death tests report thrown exceptions as failures and that the
+// exceptions do not escape death test macros.
+TEST(CxxExceptionDeathTest, ExceptionIsFailure) {
+  try {
+    EXPECT_NONFATAL_FAILURE(EXPECT_DEATH(throw 1, ""), "threw an exception");
+  } catch (...) {  // NOLINT
+    FAIL() << "An exception escaped a death test macro invocation "
+           << "with catch_exceptions "
+           << (testing::GTEST_FLAG(catch_exceptions) ? "enabled" : "disabled");
+  }
+}
+
+class TestException : public std::exception {
+ public:
+  virtual const char* what() const throw() { return "exceptional message"; }
+};
+
+TEST(CxxExceptionDeathTest, PrintsMessageForStdExceptions) {
+  // Verifies that the exception message is quoted in the failure text.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DEATH(throw TestException(), ""),
+                          "exceptional message");
+  // Verifies that the location is mentioned in the failure text.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DEATH(throw TestException(), ""),
+                          "gtest-death-test_ex_test.cc");
+}
+# endif  // GTEST_HAS_EXCEPTIONS
+
+# if GTEST_HAS_SEH
+// Tests that enabling interception of SEH exceptions with the
+// catch_exceptions flag does not interfere with SEH exceptions being
+// treated as death by death tests.
+TEST(SehExceptionDeasTest, CatchExceptionsDoesNotInterfere) {
+  EXPECT_DEATH(RaiseException(42, 0x0, 0, NULL), "")
+      << "with catch_exceptions "
+      << (testing::GTEST_FLAG(catch_exceptions) ? "enabled" : "disabled");
+}
+# endif
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  testing::GTEST_FLAG(catch_exceptions) = GTEST_ENABLE_CATCH_EXCEPTIONS_ != 0;
+  return RUN_ALL_TESTS();
+}
+// Copyright 2003, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: Dan Egnor (egnor@google.com)
+// Ported to Windows: Vadim Berman (vadimb@google.com)
+
+#include "gtest/internal/gtest-linked_ptr.h"
+
+#include <stdlib.h>
+#include "gtest/gtest.h"
+
+namespace {
+
+using testing::Message;
+using testing::internal::linked_ptr;
+
+int num;
+Message* history = NULL;
+
+// Class which tracks allocation/deallocation
+class A {
+ public:
+  A(): mynum(num++) { *history << "A" << mynum << " ctor\n"; }
+  virtual ~A() { *history << "A" << mynum << " dtor\n"; }
+  virtual void Use() { *history << "A" << mynum << " use\n"; }
+ protected:
+  int mynum;
+};
+
+// Subclass
+class B : public A {
+ public:
+  B() { *history << "B" << mynum << " ctor\n"; }
+  ~B() { *history << "B" << mynum << " dtor\n"; }
+  virtual void Use() { *history << "B" << mynum << " use\n"; }
+};
+
+class LinkedPtrTest : public testing::Test {
+ public:
+  LinkedPtrTest() {
+    num = 0;
+    history = new Message;
+  }
+
+  virtual ~LinkedPtrTest() {
+    delete history;
+    history = NULL;
+  }
+};
+
+TEST_F(LinkedPtrTest, GeneralTest) {
+  {
+    linked_ptr<A> a0, a1, a2;
+    // Use explicit function call notation here to suppress self-assign warning.
+    a0.operator=(a0);
+    a1 = a2;
+    ASSERT_EQ(a0.get(), static_cast<A*>(NULL));
+    ASSERT_EQ(a1.get(), static_cast<A*>(NULL));
+    ASSERT_EQ(a2.get(), static_cast<A*>(NULL));
+    ASSERT_TRUE(a0 == NULL);
+    ASSERT_TRUE(a1 == NULL);
+    ASSERT_TRUE(a2 == NULL);
+
+    {
+      linked_ptr<A> a3(new A);
+      a0 = a3;
+      ASSERT_TRUE(a0 == a3);
+      ASSERT_TRUE(a0 != NULL);
+      ASSERT_TRUE(a0.get() == a3);
+      ASSERT_TRUE(a0 == a3.get());
+      linked_ptr<A> a4(a0);
+      a1 = a4;
+      linked_ptr<A> a5(new A);
+      ASSERT_TRUE(a5.get() != a3);
+      ASSERT_TRUE(a5 != a3.get());
+      a2 = a5;
+      linked_ptr<B> b0(new B);
+      linked_ptr<A> a6(b0);
+      ASSERT_TRUE(b0 == a6);
+      ASSERT_TRUE(a6 == b0);
+      ASSERT_TRUE(b0 != NULL);
+      a5 = b0;
+      a5 = b0;
+      a3->Use();
+      a4->Use();
+      a5->Use();
+      a6->Use();
+      b0->Use();
+      (*b0).Use();
+      b0.get()->Use();
+    }
+
+    a0->Use();
+    a1->Use();
+    a2->Use();
+
+    a1 = a2;
+    a2.reset(new A);
+    a0.reset();
+
+    linked_ptr<A> a7;
+  }
+
+  ASSERT_STREQ(
+    "A0 ctor\n"
+    "A1 ctor\n"
+    "A2 ctor\n"
+    "B2 ctor\n"
+    "A0 use\n"
+    "A0 use\n"
+    "B2 use\n"
+    "B2 use\n"
+    "B2 use\n"
+    "B2 use\n"
+    "B2 use\n"
+    "B2 dtor\n"
+    "A2 dtor\n"
+    "A0 use\n"
+    "A0 use\n"
+    "A1 use\n"
+    "A3 ctor\n"
+    "A0 dtor\n"
+    "A3 dtor\n"
+    "A1 dtor\n",
+    history->GetString().c_str());
+}
+
+}  // Unnamed namespace
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Tests for the Message class.
+
+#include "gtest/gtest-message.h"
+
+#include "gtest/gtest.h"
+
+namespace {
+
+using ::testing::Message;
+
+// Tests the testing::Message class
+
+// Tests the default constructor.
+TEST(MessageTest, DefaultConstructor) {
+  const Message msg;
+  EXPECT_EQ("", msg.GetString());
+}
+
+// Tests the copy constructor.
+TEST(MessageTest, CopyConstructor) {
+  const Message msg1("Hello");
+  const Message msg2(msg1);
+  EXPECT_EQ("Hello", msg2.GetString());
+}
+
+// Tests constructing a Message from a C-string.
+TEST(MessageTest, ConstructsFromCString) {
+  Message msg("Hello");
+  EXPECT_EQ("Hello", msg.GetString());
+}
+
+// Tests streaming a float.
+TEST(MessageTest, StreamsFloat) {
+  const std::string s = (Message() << 1.23456F << " " << 2.34567F).GetString();
+  // Both numbers should be printed with enough precision.
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "1.234560", s.c_str());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, " 2.345669", s.c_str());
+}
+
+// Tests streaming a double.
+TEST(MessageTest, StreamsDouble) {
+  const std::string s = (Message() << 1260570880.4555497 << " "
+                                  << 1260572265.1954534).GetString();
+  // Both numbers should be printed with enough precision.
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "1260570880.45", s.c_str());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, " 1260572265.19", s.c_str());
+}
+
+// Tests streaming a non-char pointer.
+TEST(MessageTest, StreamsPointer) {
+  int n = 0;
+  int* p = &n;
+  EXPECT_NE("(null)", (Message() << p).GetString());
+}
+
+// Tests streaming a NULL non-char pointer.
+TEST(MessageTest, StreamsNullPointer) {
+  int* p = NULL;
+  EXPECT_EQ("(null)", (Message() << p).GetString());
+}
+
+// Tests streaming a C string.
+TEST(MessageTest, StreamsCString) {
+  EXPECT_EQ("Foo", (Message() << "Foo").GetString());
+}
+
+// Tests streaming a NULL C string.
+TEST(MessageTest, StreamsNullCString) {
+  char* p = NULL;
+  EXPECT_EQ("(null)", (Message() << p).GetString());
+}
+
+// Tests streaming std::string.
+TEST(MessageTest, StreamsString) {
+  const ::std::string str("Hello");
+  EXPECT_EQ("Hello", (Message() << str).GetString());
+}
+
+// Tests that we can output strings containing embedded NULs.
+TEST(MessageTest, StreamsStringWithEmbeddedNUL) {
+  const char char_array_with_nul[] =
+      "Here's a NUL\0 and some more string";
+  const ::std::string string_with_nul(char_array_with_nul,
+                                      sizeof(char_array_with_nul) - 1);
+  EXPECT_EQ("Here's a NUL\\0 and some more string",
+            (Message() << string_with_nul).GetString());
+}
+
+// Tests streaming a NUL char.
+TEST(MessageTest, StreamsNULChar) {
+  EXPECT_EQ("\\0", (Message() << '\0').GetString());
+}
+
+// Tests streaming int.
+TEST(MessageTest, StreamsInt) {
+  EXPECT_EQ("123", (Message() << 123).GetString());
+}
+
+// Tests that basic IO manipulators (endl, ends, and flush) can be
+// streamed to Message.
+TEST(MessageTest, StreamsBasicIoManip) {
+  EXPECT_EQ("Line 1.\nA NUL char \\0 in line 2.",
+               (Message() << "Line 1." << std::endl
+                         << "A NUL char " << std::ends << std::flush
+                         << " in line 2.").GetString());
+}
+
+// Tests Message::GetString()
+TEST(MessageTest, GetString) {
+  Message msg;
+  msg << 1 << " lamb";
+  EXPECT_EQ("1 lamb", msg.GetString());
+}
+
+// Tests streaming a Message object to an ostream.
+TEST(MessageTest, StreamsToOStream) {
+  Message msg("Hello");
+  ::std::stringstream ss;
+  ss << msg;
+  EXPECT_EQ("Hello", testing::internal::StringStreamToString(&ss));
+}
+
+// Tests that a Message object doesn't take up too much stack space.
+TEST(MessageTest, DoesNotTakeUpMuchStackSpace) {
+  EXPECT_LE(sizeof(Message), 16U);
+}
+
+}  // namespace
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest.h"
+
+TEST(DummyTest, Dummy) {
+  // This test doesn't verify anything.  We just need it to create a
+  // realistic stage for testing the behavior of Google Test when
+  // RUN_ALL_TESTS() is called without testing::InitGoogleTest() being
+  // called first.
+}
+
+int main() {
+  return RUN_ALL_TESTS();
+}
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+
+#include "gtest/gtest-test-part.h"
+
+#include "gtest/gtest.h"
+
+using testing::Message;
+using testing::Test;
+using testing::TestPartResult;
+using testing::TestPartResultArray;
+
+namespace {
+
+// Tests the TestPartResult class.
+
+// The test fixture for testing TestPartResult.
+class TestPartResultTest : public Test {
+ protected:
+  TestPartResultTest()
+      : r1_(TestPartResult::kSuccess, "foo/bar.cc", 10, "Success!"),
+        r2_(TestPartResult::kNonFatalFailure, "foo/bar.cc", -1, "Failure!"),
+        r3_(TestPartResult::kFatalFailure, NULL, -1, "Failure!") {}
+
+  TestPartResult r1_, r2_, r3_;
+};
+
+
+TEST_F(TestPartResultTest, ConstructorWorks) {
+  Message message;
+  message << "something is terribly wrong";
+  message << static_cast<const char*>(testing::internal::kStackTraceMarker);
+  message << "some unimportant stack trace";
+
+  const TestPartResult result(TestPartResult::kNonFatalFailure,
+                              "some_file.cc",
+                              42,
+                              message.GetString().c_str());
+
+  EXPECT_EQ(TestPartResult::kNonFatalFailure, result.type());
+  EXPECT_STREQ("some_file.cc", result.file_name());
+  EXPECT_EQ(42, result.line_number());
+  EXPECT_STREQ(message.GetString().c_str(), result.message());
+  EXPECT_STREQ("something is terribly wrong", result.summary());
+}
+
+TEST_F(TestPartResultTest, ResultAccessorsWork) {
+  const TestPartResult success(TestPartResult::kSuccess,
+                               "file.cc",
+                               42,
+                               "message");
+  EXPECT_TRUE(success.passed());
+  EXPECT_FALSE(success.failed());
+  EXPECT_FALSE(success.nonfatally_failed());
+  EXPECT_FALSE(success.fatally_failed());
+
+  const TestPartResult nonfatal_failure(TestPartResult::kNonFatalFailure,
+                                        "file.cc",
+                                        42,
+                                        "message");
+  EXPECT_FALSE(nonfatal_failure.passed());
+  EXPECT_TRUE(nonfatal_failure.failed());
+  EXPECT_TRUE(nonfatal_failure.nonfatally_failed());
+  EXPECT_FALSE(nonfatal_failure.fatally_failed());
+
+  const TestPartResult fatal_failure(TestPartResult::kFatalFailure,
+                                     "file.cc",
+                                     42,
+                                     "message");
+  EXPECT_FALSE(fatal_failure.passed());
+  EXPECT_TRUE(fatal_failure.failed());
+  EXPECT_FALSE(fatal_failure.nonfatally_failed());
+  EXPECT_TRUE(fatal_failure.fatally_failed());
+}
+
+// Tests TestPartResult::type().
+TEST_F(TestPartResultTest, type) {
+  EXPECT_EQ(TestPartResult::kSuccess, r1_.type());
+  EXPECT_EQ(TestPartResult::kNonFatalFailure, r2_.type());
+  EXPECT_EQ(TestPartResult::kFatalFailure, r3_.type());
+}
+
+// Tests TestPartResult::file_name().
+TEST_F(TestPartResultTest, file_name) {
+  EXPECT_STREQ("foo/bar.cc", r1_.file_name());
+  EXPECT_STREQ(NULL, r3_.file_name());
+}
+
+// Tests TestPartResult::line_number().
+TEST_F(TestPartResultTest, line_number) {
+  EXPECT_EQ(10, r1_.line_number());
+  EXPECT_EQ(-1, r2_.line_number());
+}
+
+// Tests TestPartResult::message().
+TEST_F(TestPartResultTest, message) {
+  EXPECT_STREQ("Success!", r1_.message());
+}
+
+// Tests TestPartResult::passed().
+TEST_F(TestPartResultTest, Passed) {
+  EXPECT_TRUE(r1_.passed());
+  EXPECT_FALSE(r2_.passed());
+  EXPECT_FALSE(r3_.passed());
+}
+
+// Tests TestPartResult::failed().
+TEST_F(TestPartResultTest, Failed) {
+  EXPECT_FALSE(r1_.failed());
+  EXPECT_TRUE(r2_.failed());
+  EXPECT_TRUE(r3_.failed());
+}
+
+// Tests TestPartResult::fatally_failed().
+TEST_F(TestPartResultTest, FatallyFailed) {
+  EXPECT_FALSE(r1_.fatally_failed());
+  EXPECT_FALSE(r2_.fatally_failed());
+  EXPECT_TRUE(r3_.fatally_failed());
+}
+
+// Tests TestPartResult::nonfatally_failed().
+TEST_F(TestPartResultTest, NonfatallyFailed) {
+  EXPECT_FALSE(r1_.nonfatally_failed());
+  EXPECT_TRUE(r2_.nonfatally_failed());
+  EXPECT_FALSE(r3_.nonfatally_failed());
+}
+
+// Tests the TestPartResultArray class.
+
+class TestPartResultArrayTest : public Test {
+ protected:
+  TestPartResultArrayTest()
+      : r1_(TestPartResult::kNonFatalFailure, "foo/bar.cc", -1, "Failure 1"),
+        r2_(TestPartResult::kFatalFailure, "foo/bar.cc", -1, "Failure 2") {}
+
+  const TestPartResult r1_, r2_;
+};
+
+// Tests that TestPartResultArray initially has size 0.
+TEST_F(TestPartResultArrayTest, InitialSizeIsZero) {
+  TestPartResultArray results;
+  EXPECT_EQ(0, results.size());
+}
+
+// Tests that TestPartResultArray contains the given TestPartResult
+// after one Append() operation.
+TEST_F(TestPartResultArrayTest, ContainsGivenResultAfterAppend) {
+  TestPartResultArray results;
+  results.Append(r1_);
+  EXPECT_EQ(1, results.size());
+  EXPECT_STREQ("Failure 1", results.GetTestPartResult(0).message());
+}
+
+// Tests that TestPartResultArray contains the given TestPartResults
+// after two Append() operations.
+TEST_F(TestPartResultArrayTest, ContainsGivenResultsAfterTwoAppends) {
+  TestPartResultArray results;
+  results.Append(r1_);
+  results.Append(r2_);
+  EXPECT_EQ(2, results.size());
+  EXPECT_STREQ("Failure 1", results.GetTestPartResult(0).message());
+  EXPECT_STREQ("Failure 2", results.GetTestPartResult(1).message());
+}
+
+typedef TestPartResultArrayTest TestPartResultArrayDeathTest;
+
+// Tests that the program dies when GetTestPartResult() is called with
+// an invalid index.
+TEST_F(TestPartResultArrayDeathTest, DiesWhenIndexIsOutOfBound) {
+  TestPartResultArray results;
+  results.Append(r1_);
+
+  EXPECT_DEATH_IF_SUPPORTED(results.GetTestPartResult(-1), "");
+  EXPECT_DEATH_IF_SUPPORTED(results.GetTestPartResult(1), "");
+}
+
+// TODO(mheule@google.com): Add a test for the class HasNewFatalFailureHelper.
+
+}  // namespace
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/internal/gtest-tuple.h"
+#include <utility>
+#include "gtest/gtest.h"
+
+namespace {
+
+using ::std::tr1::get;
+using ::std::tr1::make_tuple;
+using ::std::tr1::tuple;
+using ::std::tr1::tuple_element;
+using ::std::tr1::tuple_size;
+using ::testing::StaticAssertTypeEq;
+
+// Tests that tuple_element<K, tuple<T0, T1, ..., TN> >::type returns TK.
+TEST(tuple_element_Test, ReturnsElementType) {
+  StaticAssertTypeEq<int, tuple_element<0, tuple<int, char> >::type>();
+  StaticAssertTypeEq<int&, tuple_element<1, tuple<double, int&> >::type>();
+  StaticAssertTypeEq<bool, tuple_element<2, tuple<double, int, bool> >::type>();
+}
+
+// Tests that tuple_size<T>::value gives the number of fields in tuple
+// type T.
+TEST(tuple_size_Test, ReturnsNumberOfFields) {
+  EXPECT_EQ(0, +tuple_size<tuple<> >::value);
+  EXPECT_EQ(1, +tuple_size<tuple<void*> >::value);
+  EXPECT_EQ(1, +tuple_size<tuple<char> >::value);
+  EXPECT_EQ(1, +(tuple_size<tuple<tuple<int, double> > >::value));
+  EXPECT_EQ(2, +(tuple_size<tuple<int&, const char> >::value));
+  EXPECT_EQ(3, +(tuple_size<tuple<char*, void, const bool&> >::value));
+}
+
+// Tests comparing a tuple with itself.
+TEST(ComparisonTest, ComparesWithSelf) {
+  const tuple<int, char, bool> a(5, 'a', false);
+
+  EXPECT_TRUE(a == a);
+  EXPECT_FALSE(a != a);
+}
+
+// Tests comparing two tuples with the same value.
+TEST(ComparisonTest, ComparesEqualTuples) {
+  const tuple<int, bool> a(5, true), b(5, true);
+
+  EXPECT_TRUE(a == b);
+  EXPECT_FALSE(a != b);
+}
+
+// Tests comparing two different tuples that have no reference fields.
+TEST(ComparisonTest, ComparesUnequalTuplesWithoutReferenceFields) {
+  typedef tuple<const int, char> FooTuple;
+
+  const FooTuple a(0, 'x');
+  const FooTuple b(1, 'a');
+
+  EXPECT_TRUE(a != b);
+  EXPECT_FALSE(a == b);
+
+  const FooTuple c(1, 'b');
+
+  EXPECT_TRUE(b != c);
+  EXPECT_FALSE(b == c);
+}
+
+// Tests comparing two different tuples that have reference fields.
+TEST(ComparisonTest, ComparesUnequalTuplesWithReferenceFields) {
+  typedef tuple<int&, const char&> FooTuple;
+
+  int i = 5;
+  const char ch = 'a';
+  const FooTuple a(i, ch);
+
+  int j = 6;
+  const FooTuple b(j, ch);
+
+  EXPECT_TRUE(a != b);
+  EXPECT_FALSE(a == b);
+
+  j = 5;
+  const char ch2 = 'b';
+  const FooTuple c(j, ch2);
+
+  EXPECT_TRUE(b != c);
+  EXPECT_FALSE(b == c);
+}
+
+// Tests that a tuple field with a reference type is an alias of the
+// variable it's supposed to reference.
+TEST(ReferenceFieldTest, IsAliasOfReferencedVariable) {
+  int n = 0;
+  tuple<bool, int&> t(true, n);
+
+  n = 1;
+  EXPECT_EQ(n, get<1>(t))
+      << "Changing a underlying variable should update the reference field.";
+
+  // Makes sure that the implementation doesn't do anything funny with
+  // the & operator for the return type of get<>().
+  EXPECT_EQ(&n, &(get<1>(t)))
+      << "The address of a reference field should equal the address of "
+      << "the underlying variable.";
+
+  get<1>(t) = 2;
+  EXPECT_EQ(2, n)
+      << "Changing a reference field should update the underlying variable.";
+}
+
+// Tests that tuple's default constructor default initializes each field.
+// This test needs to compile without generating warnings.
+TEST(TupleConstructorTest, DefaultConstructorDefaultInitializesEachField) {
+  // The TR1 report requires that tuple's default constructor default
+  // initializes each field, even if it's a primitive type.  If the
+  // implementation forgets to do this, this test will catch it by
+  // generating warnings about using uninitialized variables (assuming
+  // a decent compiler).
+
+  tuple<> empty;
+
+  tuple<int> a1, b1;
+  b1 = a1;
+  EXPECT_EQ(0, get<0>(b1));
+
+  tuple<int, double> a2, b2;
+  b2 = a2;
+  EXPECT_EQ(0, get<0>(b2));
+  EXPECT_EQ(0.0, get<1>(b2));
+
+  tuple<double, char, bool*> a3, b3;
+  b3 = a3;
+  EXPECT_EQ(0.0, get<0>(b3));
+  EXPECT_EQ('\0', get<1>(b3));
+  EXPECT_TRUE(get<2>(b3) == NULL);
+
+  tuple<int, int, int, int, int, int, int, int, int, int> a10, b10;
+  b10 = a10;
+  EXPECT_EQ(0, get<0>(b10));
+  EXPECT_EQ(0, get<1>(b10));
+  EXPECT_EQ(0, get<2>(b10));
+  EXPECT_EQ(0, get<3>(b10));
+  EXPECT_EQ(0, get<4>(b10));
+  EXPECT_EQ(0, get<5>(b10));
+  EXPECT_EQ(0, get<6>(b10));
+  EXPECT_EQ(0, get<7>(b10));
+  EXPECT_EQ(0, get<8>(b10));
+  EXPECT_EQ(0, get<9>(b10));
+}
+
+// Tests constructing a tuple from its fields.
+TEST(TupleConstructorTest, ConstructsFromFields) {
+  int n = 1;
+  // Reference field.
+  tuple<int&> a(n);
+  EXPECT_EQ(&n, &(get<0>(a)));
+
+  // Non-reference fields.
+  tuple<int, char> b(5, 'a');
+  EXPECT_EQ(5, get<0>(b));
+  EXPECT_EQ('a', get<1>(b));
+
+  // Const reference field.
+  const int m = 2;
+  tuple<bool, const int&> c(true, m);
+  EXPECT_TRUE(get<0>(c));
+  EXPECT_EQ(&m, &(get<1>(c)));
+}
+
+// Tests tuple's copy constructor.
+TEST(TupleConstructorTest, CopyConstructor) {
+  tuple<double, bool> a(0.0, true);
+  tuple<double, bool> b(a);
+
+  EXPECT_DOUBLE_EQ(0.0, get<0>(b));
+  EXPECT_TRUE(get<1>(b));
+}
+
+// Tests constructing a tuple from another tuple that has a compatible
+// but different type.
+TEST(TupleConstructorTest, ConstructsFromDifferentTupleType) {
+  tuple<int, int, char> a(0, 1, 'a');
+  tuple<double, long, int> b(a);
+
+  EXPECT_DOUBLE_EQ(0.0, get<0>(b));
+  EXPECT_EQ(1, get<1>(b));
+  EXPECT_EQ('a', get<2>(b));
+}
+
+// Tests constructing a 2-tuple from an std::pair.
+TEST(TupleConstructorTest, ConstructsFromPair) {
+  ::std::pair<int, char> a(1, 'a');
+  tuple<int, char> b(a);
+  tuple<int, const char&> c(a);
+}
+
+// Tests assigning a tuple to another tuple with the same type.
+TEST(TupleAssignmentTest, AssignsToSameTupleType) {
+  const tuple<int, long> a(5, 7L);
+  tuple<int, long> b;
+  b = a;
+  EXPECT_EQ(5, get<0>(b));
+  EXPECT_EQ(7L, get<1>(b));
+}
+
+// Tests assigning a tuple to another tuple with a different but
+// compatible type.
+TEST(TupleAssignmentTest, AssignsToDifferentTupleType) {
+  const tuple<int, long, bool> a(1, 7L, true);
+  tuple<long, int, bool> b;
+  b = a;
+  EXPECT_EQ(1L, get<0>(b));
+  EXPECT_EQ(7, get<1>(b));
+  EXPECT_TRUE(get<2>(b));
+}
+
+// Tests assigning an std::pair to a 2-tuple.
+TEST(TupleAssignmentTest, AssignsFromPair) {
+  const ::std::pair<int, bool> a(5, true);
+  tuple<int, bool> b;
+  b = a;
+  EXPECT_EQ(5, get<0>(b));
+  EXPECT_TRUE(get<1>(b));
+
+  tuple<long, bool> c;
+  c = a;
+  EXPECT_EQ(5L, get<0>(c));
+  EXPECT_TRUE(get<1>(c));
+}
+
+// A fixture for testing big tuples.
+class BigTupleTest : public testing::Test {
+ protected:
+  typedef tuple<int, int, int, int, int, int, int, int, int, int> BigTuple;
+
+  BigTupleTest() :
+      a_(1, 0, 0, 0, 0, 0, 0, 0, 0, 2),
+      b_(1, 0, 0, 0, 0, 0, 0, 0, 0, 3) {}
+
+  BigTuple a_, b_;
+};
+
+// Tests constructing big tuples.
+TEST_F(BigTupleTest, Construction) {
+  BigTuple a;
+  BigTuple b(b_);
+}
+
+// Tests that get<N>(t) returns the N-th (0-based) field of tuple t.
+TEST_F(BigTupleTest, get) {
+  EXPECT_EQ(1, get<0>(a_));
+  EXPECT_EQ(2, get<9>(a_));
+
+  // Tests that get() works on a const tuple too.
+  const BigTuple a(a_);
+  EXPECT_EQ(1, get<0>(a));
+  EXPECT_EQ(2, get<9>(a));
+}
+
+// Tests comparing big tuples.
+TEST_F(BigTupleTest, Comparisons) {
+  EXPECT_TRUE(a_ == a_);
+  EXPECT_FALSE(a_ != a_);
+
+  EXPECT_TRUE(a_ != b_);
+  EXPECT_FALSE(a_ == b_);
+}
+
+TEST(MakeTupleTest, WorksForScalarTypes) {
+  tuple<bool, int> a;
+  a = make_tuple(true, 5);
+  EXPECT_TRUE(get<0>(a));
+  EXPECT_EQ(5, get<1>(a));
+
+  tuple<char, int, long> b;
+  b = make_tuple('a', 'b', 5);
+  EXPECT_EQ('a', get<0>(b));
+  EXPECT_EQ('b', get<1>(b));
+  EXPECT_EQ(5, get<2>(b));
+}
+
+TEST(MakeTupleTest, WorksForPointers) {
+  int a[] = { 1, 2, 3, 4 };
+  const char* const str = "hi";
+  int* const p = a;
+
+  tuple<const char*, int*> t;
+  t = make_tuple(str, p);
+  EXPECT_EQ(str, get<0>(t));
+  EXPECT_EQ(p, get<1>(t));
+}
+
+}  // namespace
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file is AUTOMATICALLY GENERATED on 10/31/2011 by command
+// 'gen_gtest_pred_impl.py 5'.  DO NOT EDIT BY HAND!
+
+// Regression test for gtest_pred_impl.h
+//
+// This file is generated by a script and quite long.  If you intend to
+// learn how Google Test works by reading its unit tests, read
+// gtest_unittest.cc instead.
+//
+// This is intended as a regression test for the Google Test predicate
+// assertions.  We compile it as part of the gtest_unittest target
+// only to keep the implementation tidy and compact, as it is quite
+// involved to set up the stage for testing Google Test using Google
+// Test itself.
+//
+// Currently, gtest_unittest takes ~11 seconds to run in the testing
+// daemon.  In the future, if it grows too large and needs much more
+// time to finish, we should consider separating this file into a
+// stand-alone regression test.
+
+#include <iostream>
+
+#include "gtest/gtest.h"
+#include "gtest/gtest-spi.h"
+
+// A user-defined data type.
+struct Bool {
+  explicit Bool(int val) : value(val != 0) {}
+
+  bool operator>(int n) const { return value > Bool(n).value; }
+
+  Bool operator+(const Bool& rhs) const { return Bool(value + rhs.value); }
+
+  bool operator==(const Bool& rhs) const { return value == rhs.value; }
+
+  bool value;
+};
+
+// Enables Bool to be used in assertions.
+std::ostream& operator<<(std::ostream& os, const Bool& x) {
+  return os << (x.value ? "true" : "false");
+}
+
+// Sample functions/functors for testing unary predicate assertions.
+
+// A unary predicate function.
+template <typename T1>
+bool PredFunction1(T1 v1) {
+  return v1 > 0;
+}
+
+// The following two functions are needed to circumvent a bug in
+// gcc 2.95.3, which sometimes has problem with the above template
+// function.
+bool PredFunction1Int(int v1) {
+  return v1 > 0;
+}
+bool PredFunction1Bool(Bool v1) {
+  return v1 > 0;
+}
+
+// A unary predicate functor.
+struct PredFunctor1 {
+  template <typename T1>
+  bool operator()(const T1& v1) {
+    return v1 > 0;
+  }
+};
+
+// A unary predicate-formatter function.
+template <typename T1>
+testing::AssertionResult PredFormatFunction1(const char* e1,
+                                             const T1& v1) {
+  if (PredFunction1(v1))
+    return testing::AssertionSuccess();
+
+  return testing::AssertionFailure()
+      << e1
+      << " is expected to be positive, but evaluates to "
+      << v1 << ".";
+}
+
+// A unary predicate-formatter functor.
+struct PredFormatFunctor1 {
+  template <typename T1>
+  testing::AssertionResult operator()(const char* e1,
+                                      const T1& v1) const {
+    return PredFormatFunction1(e1, v1);
+  }
+};
+
+// Tests for {EXPECT|ASSERT}_PRED_FORMAT1.
+
+class Predicate1Test : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    expected_to_finish_ = true;
+    finished_ = false;
+    n1_ = 0;
+  }
+
+  virtual void TearDown() {
+    // Verifies that each of the predicate's arguments was evaluated
+    // exactly once.
+    EXPECT_EQ(1, n1_) <<
+        "The predicate assertion didn't evaluate argument 2 "
+        "exactly once.";
+
+    // Verifies that the control flow in the test function is expected.
+    if (expected_to_finish_ && !finished_) {
+      FAIL() << "The predicate assertion unexpactedly aborted the test.";
+    } else if (!expected_to_finish_ && finished_) {
+      FAIL() << "The failed predicate assertion didn't abort the test "
+                "as expected.";
+    }
+  }
+
+  // true iff the test function is expected to run to finish.
+  static bool expected_to_finish_;
+
+  // true iff the test function did run to finish.
+  static bool finished_;
+
+  static int n1_;
+};
+
+bool Predicate1Test::expected_to_finish_;
+bool Predicate1Test::finished_;
+int Predicate1Test::n1_;
+
+typedef Predicate1Test EXPECT_PRED_FORMAT1Test;
+typedef Predicate1Test ASSERT_PRED_FORMAT1Test;
+typedef Predicate1Test EXPECT_PRED1Test;
+typedef Predicate1Test ASSERT_PRED1Test;
+
+// Tests a successful EXPECT_PRED1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED1Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED1(PredFunction1Int,
+               ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED1Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED1(PredFunction1Bool,
+               Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED1Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED1(PredFunctor1(),
+               ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED1Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED1(PredFunctor1(),
+               Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED1Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED1(PredFunction1Int,
+                 n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED1Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED1(PredFunction1Bool,
+                 Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED1Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED1(PredFunctor1(),
+                 n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED1Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED1(PredFunctor1(),
+                 Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED1Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED1(PredFunction1Int,
+               ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED1Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED1(PredFunction1Bool,
+               Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED1Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED1(PredFunctor1(),
+               ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED1Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED1(PredFunctor1(),
+               Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED1Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED1(PredFunction1Int,
+                 n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED1Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED1(PredFunction1Bool,
+                 Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED1Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED1(PredFunctor1(),
+                 n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED1Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED1(PredFunctor1(),
+                 Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT1(PredFormatFunction1,
+                      ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT1(PredFormatFunction1,
+                      Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT1(PredFormatFunctor1(),
+                      ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT1(PredFormatFunctor1(),
+                      Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT1(PredFormatFunction1,
+                        n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT1(PredFormatFunction1,
+                        Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT1(PredFormatFunctor1(),
+                        n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT1Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT1(PredFormatFunctor1(),
+                        Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT1(PredFormatFunction1,
+                      ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT1(PredFormatFunction1,
+                      Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT1(PredFormatFunctor1(),
+                      ++n1_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT1(PredFormatFunctor1(),
+                      Bool(++n1_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(PredFormatFunction1,
+                        n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(PredFormatFunction1,
+                        Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(PredFormatFunctor1(),
+                        n1_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT1 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT1Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(PredFormatFunctor1(),
+                        Bool(n1_++));
+    finished_ = true;
+  }, "");
+}
+// Sample functions/functors for testing binary predicate assertions.
+
+// A binary predicate function.
+template <typename T1, typename T2>
+bool PredFunction2(T1 v1, T2 v2) {
+  return v1 + v2 > 0;
+}
+
+// The following two functions are needed to circumvent a bug in
+// gcc 2.95.3, which sometimes has problem with the above template
+// function.
+bool PredFunction2Int(int v1, int v2) {
+  return v1 + v2 > 0;
+}
+bool PredFunction2Bool(Bool v1, Bool v2) {
+  return v1 + v2 > 0;
+}
+
+// A binary predicate functor.
+struct PredFunctor2 {
+  template <typename T1, typename T2>
+  bool operator()(const T1& v1,
+                  const T2& v2) {
+    return v1 + v2 > 0;
+  }
+};
+
+// A binary predicate-formatter function.
+template <typename T1, typename T2>
+testing::AssertionResult PredFormatFunction2(const char* e1,
+                                             const char* e2,
+                                             const T1& v1,
+                                             const T2& v2) {
+  if (PredFunction2(v1, v2))
+    return testing::AssertionSuccess();
+
+  return testing::AssertionFailure()
+      << e1 << " + " << e2
+      << " is expected to be positive, but evaluates to "
+      << v1 + v2 << ".";
+}
+
+// A binary predicate-formatter functor.
+struct PredFormatFunctor2 {
+  template <typename T1, typename T2>
+  testing::AssertionResult operator()(const char* e1,
+                                      const char* e2,
+                                      const T1& v1,
+                                      const T2& v2) const {
+    return PredFormatFunction2(e1, e2, v1, v2);
+  }
+};
+
+// Tests for {EXPECT|ASSERT}_PRED_FORMAT2.
+
+class Predicate2Test : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    expected_to_finish_ = true;
+    finished_ = false;
+    n1_ = n2_ = 0;
+  }
+
+  virtual void TearDown() {
+    // Verifies that each of the predicate's arguments was evaluated
+    // exactly once.
+    EXPECT_EQ(1, n1_) <<
+        "The predicate assertion didn't evaluate argument 2 "
+        "exactly once.";
+    EXPECT_EQ(1, n2_) <<
+        "The predicate assertion didn't evaluate argument 3 "
+        "exactly once.";
+
+    // Verifies that the control flow in the test function is expected.
+    if (expected_to_finish_ && !finished_) {
+      FAIL() << "The predicate assertion unexpactedly aborted the test.";
+    } else if (!expected_to_finish_ && finished_) {
+      FAIL() << "The failed predicate assertion didn't abort the test "
+                "as expected.";
+    }
+  }
+
+  // true iff the test function is expected to run to finish.
+  static bool expected_to_finish_;
+
+  // true iff the test function did run to finish.
+  static bool finished_;
+
+  static int n1_;
+  static int n2_;
+};
+
+bool Predicate2Test::expected_to_finish_;
+bool Predicate2Test::finished_;
+int Predicate2Test::n1_;
+int Predicate2Test::n2_;
+
+typedef Predicate2Test EXPECT_PRED_FORMAT2Test;
+typedef Predicate2Test ASSERT_PRED_FORMAT2Test;
+typedef Predicate2Test EXPECT_PRED2Test;
+typedef Predicate2Test ASSERT_PRED2Test;
+
+// Tests a successful EXPECT_PRED2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED2Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED2(PredFunction2Int,
+               ++n1_,
+               ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED2Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED2(PredFunction2Bool,
+               Bool(++n1_),
+               Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED2Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED2(PredFunctor2(),
+               ++n1_,
+               ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED2Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED2(PredFunctor2(),
+               Bool(++n1_),
+               Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED2Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED2(PredFunction2Int,
+                 n1_++,
+                 n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED2Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED2(PredFunction2Bool,
+                 Bool(n1_++),
+                 Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED2Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED2(PredFunctor2(),
+                 n1_++,
+                 n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED2Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED2(PredFunctor2(),
+                 Bool(n1_++),
+                 Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED2Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED2(PredFunction2Int,
+               ++n1_,
+               ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED2Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED2(PredFunction2Bool,
+               Bool(++n1_),
+               Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED2Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED2(PredFunctor2(),
+               ++n1_,
+               ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED2Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED2(PredFunctor2(),
+               Bool(++n1_),
+               Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED2Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED2(PredFunction2Int,
+                 n1_++,
+                 n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED2Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED2(PredFunction2Bool,
+                 Bool(n1_++),
+                 Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED2Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED2(PredFunctor2(),
+                 n1_++,
+                 n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED2Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED2(PredFunctor2(),
+                 Bool(n1_++),
+                 Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT2(PredFormatFunction2,
+                      ++n1_,
+                      ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT2(PredFormatFunction2,
+                      Bool(++n1_),
+                      Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT2(PredFormatFunctor2(),
+                      ++n1_,
+                      ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT2(PredFormatFunctor2(),
+                      Bool(++n1_),
+                      Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(PredFormatFunction2,
+                        n1_++,
+                        n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(PredFormatFunction2,
+                        Bool(n1_++),
+                        Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(PredFormatFunctor2(),
+                        n1_++,
+                        n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT2Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(PredFormatFunctor2(),
+                        Bool(n1_++),
+                        Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT2(PredFormatFunction2,
+                      ++n1_,
+                      ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT2(PredFormatFunction2,
+                      Bool(++n1_),
+                      Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT2(PredFormatFunctor2(),
+                      ++n1_,
+                      ++n2_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT2(PredFormatFunctor2(),
+                      Bool(++n1_),
+                      Bool(++n2_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(PredFormatFunction2,
+                        n1_++,
+                        n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(PredFormatFunction2,
+                        Bool(n1_++),
+                        Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(PredFormatFunctor2(),
+                        n1_++,
+                        n2_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT2 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT2Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(PredFormatFunctor2(),
+                        Bool(n1_++),
+                        Bool(n2_++));
+    finished_ = true;
+  }, "");
+}
+// Sample functions/functors for testing ternary predicate assertions.
+
+// A ternary predicate function.
+template <typename T1, typename T2, typename T3>
+bool PredFunction3(T1 v1, T2 v2, T3 v3) {
+  return v1 + v2 + v3 > 0;
+}
+
+// The following two functions are needed to circumvent a bug in
+// gcc 2.95.3, which sometimes has problem with the above template
+// function.
+bool PredFunction3Int(int v1, int v2, int v3) {
+  return v1 + v2 + v3 > 0;
+}
+bool PredFunction3Bool(Bool v1, Bool v2, Bool v3) {
+  return v1 + v2 + v3 > 0;
+}
+
+// A ternary predicate functor.
+struct PredFunctor3 {
+  template <typename T1, typename T2, typename T3>
+  bool operator()(const T1& v1,
+                  const T2& v2,
+                  const T3& v3) {
+    return v1 + v2 + v3 > 0;
+  }
+};
+
+// A ternary predicate-formatter function.
+template <typename T1, typename T2, typename T3>
+testing::AssertionResult PredFormatFunction3(const char* e1,
+                                             const char* e2,
+                                             const char* e3,
+                                             const T1& v1,
+                                             const T2& v2,
+                                             const T3& v3) {
+  if (PredFunction3(v1, v2, v3))
+    return testing::AssertionSuccess();
+
+  return testing::AssertionFailure()
+      << e1 << " + " << e2 << " + " << e3
+      << " is expected to be positive, but evaluates to "
+      << v1 + v2 + v3 << ".";
+}
+
+// A ternary predicate-formatter functor.
+struct PredFormatFunctor3 {
+  template <typename T1, typename T2, typename T3>
+  testing::AssertionResult operator()(const char* e1,
+                                      const char* e2,
+                                      const char* e3,
+                                      const T1& v1,
+                                      const T2& v2,
+                                      const T3& v3) const {
+    return PredFormatFunction3(e1, e2, e3, v1, v2, v3);
+  }
+};
+
+// Tests for {EXPECT|ASSERT}_PRED_FORMAT3.
+
+class Predicate3Test : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    expected_to_finish_ = true;
+    finished_ = false;
+    n1_ = n2_ = n3_ = 0;
+  }
+
+  virtual void TearDown() {
+    // Verifies that each of the predicate's arguments was evaluated
+    // exactly once.
+    EXPECT_EQ(1, n1_) <<
+        "The predicate assertion didn't evaluate argument 2 "
+        "exactly once.";
+    EXPECT_EQ(1, n2_) <<
+        "The predicate assertion didn't evaluate argument 3 "
+        "exactly once.";
+    EXPECT_EQ(1, n3_) <<
+        "The predicate assertion didn't evaluate argument 4 "
+        "exactly once.";
+
+    // Verifies that the control flow in the test function is expected.
+    if (expected_to_finish_ && !finished_) {
+      FAIL() << "The predicate assertion unexpactedly aborted the test.";
+    } else if (!expected_to_finish_ && finished_) {
+      FAIL() << "The failed predicate assertion didn't abort the test "
+                "as expected.";
+    }
+  }
+
+  // true iff the test function is expected to run to finish.
+  static bool expected_to_finish_;
+
+  // true iff the test function did run to finish.
+  static bool finished_;
+
+  static int n1_;
+  static int n2_;
+  static int n3_;
+};
+
+bool Predicate3Test::expected_to_finish_;
+bool Predicate3Test::finished_;
+int Predicate3Test::n1_;
+int Predicate3Test::n2_;
+int Predicate3Test::n3_;
+
+typedef Predicate3Test EXPECT_PRED_FORMAT3Test;
+typedef Predicate3Test ASSERT_PRED_FORMAT3Test;
+typedef Predicate3Test EXPECT_PRED3Test;
+typedef Predicate3Test ASSERT_PRED3Test;
+
+// Tests a successful EXPECT_PRED3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED3Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED3(PredFunction3Int,
+               ++n1_,
+               ++n2_,
+               ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED3Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED3(PredFunction3Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED3Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED3(PredFunctor3(),
+               ++n1_,
+               ++n2_,
+               ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED3Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED3(PredFunctor3(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED3Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED3(PredFunction3Int,
+                 n1_++,
+                 n2_++,
+                 n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED3Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED3(PredFunction3Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED3Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED3(PredFunctor3(),
+                 n1_++,
+                 n2_++,
+                 n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED3Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED3(PredFunctor3(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED3Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED3(PredFunction3Int,
+               ++n1_,
+               ++n2_,
+               ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED3Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED3(PredFunction3Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED3Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED3(PredFunctor3(),
+               ++n1_,
+               ++n2_,
+               ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED3Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED3(PredFunctor3(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED3Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED3(PredFunction3Int,
+                 n1_++,
+                 n2_++,
+                 n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED3Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED3(PredFunction3Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED3Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED3(PredFunctor3(),
+                 n1_++,
+                 n2_++,
+                 n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED3Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED3(PredFunctor3(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT3(PredFormatFunction3,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT3(PredFormatFunction3,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT3(PredFormatFunctor3(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT3(PredFormatFunctor3(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT3(PredFormatFunction3,
+                        n1_++,
+                        n2_++,
+                        n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT3(PredFormatFunction3,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT3(PredFormatFunctor3(),
+                        n1_++,
+                        n2_++,
+                        n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT3Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT3(PredFormatFunctor3(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT3(PredFormatFunction3,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT3(PredFormatFunction3,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT3(PredFormatFunctor3(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT3(PredFormatFunctor3(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT3(PredFormatFunction3,
+                        n1_++,
+                        n2_++,
+                        n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT3(PredFormatFunction3,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT3(PredFormatFunctor3(),
+                        n1_++,
+                        n2_++,
+                        n3_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT3 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT3Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT3(PredFormatFunctor3(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++));
+    finished_ = true;
+  }, "");
+}
+// Sample functions/functors for testing 4-ary predicate assertions.
+
+// A 4-ary predicate function.
+template <typename T1, typename T2, typename T3, typename T4>
+bool PredFunction4(T1 v1, T2 v2, T3 v3, T4 v4) {
+  return v1 + v2 + v3 + v4 > 0;
+}
+
+// The following two functions are needed to circumvent a bug in
+// gcc 2.95.3, which sometimes has problem with the above template
+// function.
+bool PredFunction4Int(int v1, int v2, int v3, int v4) {
+  return v1 + v2 + v3 + v4 > 0;
+}
+bool PredFunction4Bool(Bool v1, Bool v2, Bool v3, Bool v4) {
+  return v1 + v2 + v3 + v4 > 0;
+}
+
+// A 4-ary predicate functor.
+struct PredFunctor4 {
+  template <typename T1, typename T2, typename T3, typename T4>
+  bool operator()(const T1& v1,
+                  const T2& v2,
+                  const T3& v3,
+                  const T4& v4) {
+    return v1 + v2 + v3 + v4 > 0;
+  }
+};
+
+// A 4-ary predicate-formatter function.
+template <typename T1, typename T2, typename T3, typename T4>
+testing::AssertionResult PredFormatFunction4(const char* e1,
+                                             const char* e2,
+                                             const char* e3,
+                                             const char* e4,
+                                             const T1& v1,
+                                             const T2& v2,
+                                             const T3& v3,
+                                             const T4& v4) {
+  if (PredFunction4(v1, v2, v3, v4))
+    return testing::AssertionSuccess();
+
+  return testing::AssertionFailure()
+      << e1 << " + " << e2 << " + " << e3 << " + " << e4
+      << " is expected to be positive, but evaluates to "
+      << v1 + v2 + v3 + v4 << ".";
+}
+
+// A 4-ary predicate-formatter functor.
+struct PredFormatFunctor4 {
+  template <typename T1, typename T2, typename T3, typename T4>
+  testing::AssertionResult operator()(const char* e1,
+                                      const char* e2,
+                                      const char* e3,
+                                      const char* e4,
+                                      const T1& v1,
+                                      const T2& v2,
+                                      const T3& v3,
+                                      const T4& v4) const {
+    return PredFormatFunction4(e1, e2, e3, e4, v1, v2, v3, v4);
+  }
+};
+
+// Tests for {EXPECT|ASSERT}_PRED_FORMAT4.
+
+class Predicate4Test : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    expected_to_finish_ = true;
+    finished_ = false;
+    n1_ = n2_ = n3_ = n4_ = 0;
+  }
+
+  virtual void TearDown() {
+    // Verifies that each of the predicate's arguments was evaluated
+    // exactly once.
+    EXPECT_EQ(1, n1_) <<
+        "The predicate assertion didn't evaluate argument 2 "
+        "exactly once.";
+    EXPECT_EQ(1, n2_) <<
+        "The predicate assertion didn't evaluate argument 3 "
+        "exactly once.";
+    EXPECT_EQ(1, n3_) <<
+        "The predicate assertion didn't evaluate argument 4 "
+        "exactly once.";
+    EXPECT_EQ(1, n4_) <<
+        "The predicate assertion didn't evaluate argument 5 "
+        "exactly once.";
+
+    // Verifies that the control flow in the test function is expected.
+    if (expected_to_finish_ && !finished_) {
+      FAIL() << "The predicate assertion unexpactedly aborted the test.";
+    } else if (!expected_to_finish_ && finished_) {
+      FAIL() << "The failed predicate assertion didn't abort the test "
+                "as expected.";
+    }
+  }
+
+  // true iff the test function is expected to run to finish.
+  static bool expected_to_finish_;
+
+  // true iff the test function did run to finish.
+  static bool finished_;
+
+  static int n1_;
+  static int n2_;
+  static int n3_;
+  static int n4_;
+};
+
+bool Predicate4Test::expected_to_finish_;
+bool Predicate4Test::finished_;
+int Predicate4Test::n1_;
+int Predicate4Test::n2_;
+int Predicate4Test::n3_;
+int Predicate4Test::n4_;
+
+typedef Predicate4Test EXPECT_PRED_FORMAT4Test;
+typedef Predicate4Test ASSERT_PRED_FORMAT4Test;
+typedef Predicate4Test EXPECT_PRED4Test;
+typedef Predicate4Test ASSERT_PRED4Test;
+
+// Tests a successful EXPECT_PRED4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED4Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED4(PredFunction4Int,
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED4Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED4(PredFunction4Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED4Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED4(PredFunctor4(),
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED4Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED4(PredFunctor4(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED4Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED4(PredFunction4Int,
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED4Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED4(PredFunction4Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED4Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED4(PredFunctor4(),
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED4Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED4(PredFunctor4(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED4Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED4(PredFunction4Int,
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED4Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED4(PredFunction4Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED4Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED4(PredFunctor4(),
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED4Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED4(PredFunctor4(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED4Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED4(PredFunction4Int,
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED4Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED4(PredFunction4Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED4Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED4(PredFunctor4(),
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED4Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED4(PredFunctor4(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT4(PredFormatFunction4,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT4(PredFormatFunction4,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT4(PredFormatFunctor4(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT4(PredFormatFunctor4(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(PredFormatFunction4,
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(PredFormatFunction4,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(PredFormatFunctor4(),
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT4Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(PredFormatFunctor4(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT4(PredFormatFunction4,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT4(PredFormatFunction4,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT4(PredFormatFunctor4(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT4(PredFormatFunctor4(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT4(PredFormatFunction4,
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT4(PredFormatFunction4,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT4(PredFormatFunctor4(),
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT4 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT4Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT4(PredFormatFunctor4(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++));
+    finished_ = true;
+  }, "");
+}
+// Sample functions/functors for testing 5-ary predicate assertions.
+
+// A 5-ary predicate function.
+template <typename T1, typename T2, typename T3, typename T4, typename T5>
+bool PredFunction5(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5) {
+  return v1 + v2 + v3 + v4 + v5 > 0;
+}
+
+// The following two functions are needed to circumvent a bug in
+// gcc 2.95.3, which sometimes has problem with the above template
+// function.
+bool PredFunction5Int(int v1, int v2, int v3, int v4, int v5) {
+  return v1 + v2 + v3 + v4 + v5 > 0;
+}
+bool PredFunction5Bool(Bool v1, Bool v2, Bool v3, Bool v4, Bool v5) {
+  return v1 + v2 + v3 + v4 + v5 > 0;
+}
+
+// A 5-ary predicate functor.
+struct PredFunctor5 {
+  template <typename T1, typename T2, typename T3, typename T4, typename T5>
+  bool operator()(const T1& v1,
+                  const T2& v2,
+                  const T3& v3,
+                  const T4& v4,
+                  const T5& v5) {
+    return v1 + v2 + v3 + v4 + v5 > 0;
+  }
+};
+
+// A 5-ary predicate-formatter function.
+template <typename T1, typename T2, typename T3, typename T4, typename T5>
+testing::AssertionResult PredFormatFunction5(const char* e1,
+                                             const char* e2,
+                                             const char* e3,
+                                             const char* e4,
+                                             const char* e5,
+                                             const T1& v1,
+                                             const T2& v2,
+                                             const T3& v3,
+                                             const T4& v4,
+                                             const T5& v5) {
+  if (PredFunction5(v1, v2, v3, v4, v5))
+    return testing::AssertionSuccess();
+
+  return testing::AssertionFailure()
+      << e1 << " + " << e2 << " + " << e3 << " + " << e4 << " + " << e5
+      << " is expected to be positive, but evaluates to "
+      << v1 + v2 + v3 + v4 + v5 << ".";
+}
+
+// A 5-ary predicate-formatter functor.
+struct PredFormatFunctor5 {
+  template <typename T1, typename T2, typename T3, typename T4, typename T5>
+  testing::AssertionResult operator()(const char* e1,
+                                      const char* e2,
+                                      const char* e3,
+                                      const char* e4,
+                                      const char* e5,
+                                      const T1& v1,
+                                      const T2& v2,
+                                      const T3& v3,
+                                      const T4& v4,
+                                      const T5& v5) const {
+    return PredFormatFunction5(e1, e2, e3, e4, e5, v1, v2, v3, v4, v5);
+  }
+};
+
+// Tests for {EXPECT|ASSERT}_PRED_FORMAT5.
+
+class Predicate5Test : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    expected_to_finish_ = true;
+    finished_ = false;
+    n1_ = n2_ = n3_ = n4_ = n5_ = 0;
+  }
+
+  virtual void TearDown() {
+    // Verifies that each of the predicate's arguments was evaluated
+    // exactly once.
+    EXPECT_EQ(1, n1_) <<
+        "The predicate assertion didn't evaluate argument 2 "
+        "exactly once.";
+    EXPECT_EQ(1, n2_) <<
+        "The predicate assertion didn't evaluate argument 3 "
+        "exactly once.";
+    EXPECT_EQ(1, n3_) <<
+        "The predicate assertion didn't evaluate argument 4 "
+        "exactly once.";
+    EXPECT_EQ(1, n4_) <<
+        "The predicate assertion didn't evaluate argument 5 "
+        "exactly once.";
+    EXPECT_EQ(1, n5_) <<
+        "The predicate assertion didn't evaluate argument 6 "
+        "exactly once.";
+
+    // Verifies that the control flow in the test function is expected.
+    if (expected_to_finish_ && !finished_) {
+      FAIL() << "The predicate assertion unexpactedly aborted the test.";
+    } else if (!expected_to_finish_ && finished_) {
+      FAIL() << "The failed predicate assertion didn't abort the test "
+                "as expected.";
+    }
+  }
+
+  // true iff the test function is expected to run to finish.
+  static bool expected_to_finish_;
+
+  // true iff the test function did run to finish.
+  static bool finished_;
+
+  static int n1_;
+  static int n2_;
+  static int n3_;
+  static int n4_;
+  static int n5_;
+};
+
+bool Predicate5Test::expected_to_finish_;
+bool Predicate5Test::finished_;
+int Predicate5Test::n1_;
+int Predicate5Test::n2_;
+int Predicate5Test::n3_;
+int Predicate5Test::n4_;
+int Predicate5Test::n5_;
+
+typedef Predicate5Test EXPECT_PRED_FORMAT5Test;
+typedef Predicate5Test ASSERT_PRED_FORMAT5Test;
+typedef Predicate5Test EXPECT_PRED5Test;
+typedef Predicate5Test ASSERT_PRED5Test;
+
+// Tests a successful EXPECT_PRED5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED5Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED5(PredFunction5Int,
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_,
+               ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED5Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED5(PredFunction5Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_),
+               Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED5Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED5(PredFunctor5(),
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_,
+               ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED5Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED5(PredFunctor5(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_),
+               Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED5Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED5(PredFunction5Int,
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++,
+                 n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED5Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED5(PredFunction5Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++),
+                 Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED5Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED5(PredFunctor5(),
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++,
+                 n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED5Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED5(PredFunctor5(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++),
+                 Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED5Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED5(PredFunction5Int,
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_,
+               ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED5Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED5(PredFunction5Bool,
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_),
+               Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED5Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED5(PredFunctor5(),
+               ++n1_,
+               ++n2_,
+               ++n3_,
+               ++n4_,
+               ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED5Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED5(PredFunctor5(),
+               Bool(++n1_),
+               Bool(++n2_),
+               Bool(++n3_),
+               Bool(++n4_),
+               Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED5Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED5(PredFunction5Int,
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++,
+                 n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED5Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED5(PredFunction5Bool,
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++),
+                 Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED5Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED5(PredFunctor5(),
+                 n1_++,
+                 n2_++,
+                 n3_++,
+                 n4_++,
+                 n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED5Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED5(PredFunctor5(),
+                 Bool(n1_++),
+                 Bool(n2_++),
+                 Bool(n3_++),
+                 Bool(n4_++),
+                 Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctionOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT5(PredFormatFunction5,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_,
+                      ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctionOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT5(PredFormatFunction5,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_),
+                      Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctorOnBuiltInTypeSuccess) {
+  EXPECT_PRED_FORMAT5(PredFormatFunctor5(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_,
+                      ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctorOnUserTypeSuccess) {
+  EXPECT_PRED_FORMAT5(PredFormatFunctor5(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_),
+                      Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a failed EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctionOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT5(PredFormatFunction5,
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++,
+                        n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctionOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT5(PredFormatFunction5,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++),
+                        Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctorOnBuiltInTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT5(PredFormatFunctor5(),
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++,
+                        n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed EXPECT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(EXPECT_PRED_FORMAT5Test, FunctorOnUserTypeFailure) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT5(PredFormatFunctor5(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++),
+                        Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a successful ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctionOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT5(PredFormatFunction5,
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_,
+                      ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctionOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT5(PredFormatFunction5,
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_),
+                      Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctorOnBuiltInTypeSuccess) {
+  ASSERT_PRED_FORMAT5(PredFormatFunctor5(),
+                      ++n1_,
+                      ++n2_,
+                      ++n3_,
+                      ++n4_,
+                      ++n5_);
+  finished_ = true;
+}
+
+// Tests a successful ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctorOnUserTypeSuccess) {
+  ASSERT_PRED_FORMAT5(PredFormatFunctor5(),
+                      Bool(++n1_),
+                      Bool(++n2_),
+                      Bool(++n3_),
+                      Bool(++n4_),
+                      Bool(++n5_));
+  finished_ = true;
+}
+
+// Tests a failed ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctionOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT5(PredFormatFunction5,
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++,
+                        n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a function on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctionOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT5(PredFormatFunction5,
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++),
+                        Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a built-in type (int).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctorOnBuiltInTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT5(PredFormatFunctor5(),
+                        n1_++,
+                        n2_++,
+                        n3_++,
+                        n4_++,
+                        n5_++);
+    finished_ = true;
+  }, "");
+}
+
+// Tests a failed ASSERT_PRED_FORMAT5 where the
+// predicate-formatter is a functor on a user-defined type (Bool).
+TEST_F(ASSERT_PRED_FORMAT5Test, FunctorOnUserTypeFailure) {
+  expected_to_finish_ = false;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT5(PredFormatFunctor5(),
+                        Bool(n1_++),
+                        Bool(n2_++),
+                        Bool(n3_++),
+                        Bool(n4_++),
+                        Bool(n5_++));
+    finished_ = true;
+  }, "");
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Tests the --gtest_repeat=number flag.
+
+#include <stdlib.h>
+#include <iostream>
+#include "gtest/gtest.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+
+GTEST_DECLARE_string_(death_test_style);
+GTEST_DECLARE_string_(filter);
+GTEST_DECLARE_int32_(repeat);
+
+}  // namespace testing
+
+using testing::GTEST_FLAG(death_test_style);
+using testing::GTEST_FLAG(filter);
+using testing::GTEST_FLAG(repeat);
+
+namespace {
+
+// We need this when we are testing Google Test itself and therefore
+// cannot use Google Test assertions.
+#define GTEST_CHECK_INT_EQ_(expected, actual) \
+  do {\
+    const int expected_val = (expected);\
+    const int actual_val = (actual);\
+    if (::testing::internal::IsTrue(expected_val != actual_val)) {\
+      ::std::cout << "Value of: " #actual "\n"\
+                  << "  Actual: " << actual_val << "\n"\
+                  << "Expected: " #expected "\n"\
+                  << "Which is: " << expected_val << "\n";\
+      ::testing::internal::posix::Abort();\
+    }\
+  } while (::testing::internal::AlwaysFalse())
+
+
+// Used for verifying that global environment set-up and tear-down are
+// inside the gtest_repeat loop.
+
+int g_environment_set_up_count = 0;
+int g_environment_tear_down_count = 0;
+
+class MyEnvironment : public testing::Environment {
+ public:
+  MyEnvironment() {}
+  virtual void SetUp() { g_environment_set_up_count++; }
+  virtual void TearDown() { g_environment_tear_down_count++; }
+};
+
+// A test that should fail.
+
+int g_should_fail_count = 0;
+
+TEST(FooTest, ShouldFail) {
+  g_should_fail_count++;
+  EXPECT_EQ(0, 1) << "Expected failure.";
+}
+
+// A test that should pass.
+
+int g_should_pass_count = 0;
+
+TEST(FooTest, ShouldPass) {
+  g_should_pass_count++;
+}
+
+// A test that contains a thread-safe death test and a fast death
+// test.  It should pass.
+
+int g_death_test_count = 0;
+
+TEST(BarDeathTest, ThreadSafeAndFast) {
+  g_death_test_count++;
+
+  GTEST_FLAG(death_test_style) = "threadsafe";
+  EXPECT_DEATH_IF_SUPPORTED(::testing::internal::posix::Abort(), "");
+
+  GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_DEATH_IF_SUPPORTED(::testing::internal::posix::Abort(), "");
+}
+
+#if GTEST_HAS_PARAM_TEST
+int g_param_test_count = 0;
+
+const int kNumberOfParamTests = 10;
+
+class MyParamTest : public testing::TestWithParam<int> {};
+
+TEST_P(MyParamTest, ShouldPass) {
+  // TODO(vladl@google.com): Make parameter value checking robust
+  //                         WRT order of tests.
+  GTEST_CHECK_INT_EQ_(g_param_test_count % kNumberOfParamTests, GetParam());
+  g_param_test_count++;
+}
+INSTANTIATE_TEST_CASE_P(MyParamSequence,
+                        MyParamTest,
+                        testing::Range(0, kNumberOfParamTests));
+#endif  // GTEST_HAS_PARAM_TEST
+
+// Resets the count for each test.
+void ResetCounts() {
+  g_environment_set_up_count = 0;
+  g_environment_tear_down_count = 0;
+  g_should_fail_count = 0;
+  g_should_pass_count = 0;
+  g_death_test_count = 0;
+#if GTEST_HAS_PARAM_TEST
+  g_param_test_count = 0;
+#endif  // GTEST_HAS_PARAM_TEST
+}
+
+// Checks that the count for each test is expected.
+void CheckCounts(int expected) {
+  GTEST_CHECK_INT_EQ_(expected, g_environment_set_up_count);
+  GTEST_CHECK_INT_EQ_(expected, g_environment_tear_down_count);
+  GTEST_CHECK_INT_EQ_(expected, g_should_fail_count);
+  GTEST_CHECK_INT_EQ_(expected, g_should_pass_count);
+  GTEST_CHECK_INT_EQ_(expected, g_death_test_count);
+#if GTEST_HAS_PARAM_TEST
+  GTEST_CHECK_INT_EQ_(expected * kNumberOfParamTests, g_param_test_count);
+#endif  // GTEST_HAS_PARAM_TEST
+}
+
+// Tests the behavior of Google Test when --gtest_repeat is not specified.
+void TestRepeatUnspecified() {
+  ResetCounts();
+  GTEST_CHECK_INT_EQ_(1, RUN_ALL_TESTS());
+  CheckCounts(1);
+}
+
+// Tests the behavior of Google Test when --gtest_repeat has the given value.
+void TestRepeat(int repeat) {
+  GTEST_FLAG(repeat) = repeat;
+
+  ResetCounts();
+  GTEST_CHECK_INT_EQ_(repeat > 0 ? 1 : 0, RUN_ALL_TESTS());
+  CheckCounts(repeat);
+}
+
+// Tests using --gtest_repeat when --gtest_filter specifies an empty
+// set of tests.
+void TestRepeatWithEmptyFilter(int repeat) {
+  GTEST_FLAG(repeat) = repeat;
+  GTEST_FLAG(filter) = "None";
+
+  ResetCounts();
+  GTEST_CHECK_INT_EQ_(0, RUN_ALL_TESTS());
+  CheckCounts(0);
+}
+
+// Tests using --gtest_repeat when --gtest_filter specifies a set of
+// successful tests.
+void TestRepeatWithFilterForSuccessfulTests(int repeat) {
+  GTEST_FLAG(repeat) = repeat;
+  GTEST_FLAG(filter) = "*-*ShouldFail";
+
+  ResetCounts();
+  GTEST_CHECK_INT_EQ_(0, RUN_ALL_TESTS());
+  GTEST_CHECK_INT_EQ_(repeat, g_environment_set_up_count);
+  GTEST_CHECK_INT_EQ_(repeat, g_environment_tear_down_count);
+  GTEST_CHECK_INT_EQ_(0, g_should_fail_count);
+  GTEST_CHECK_INT_EQ_(repeat, g_should_pass_count);
+  GTEST_CHECK_INT_EQ_(repeat, g_death_test_count);
+#if GTEST_HAS_PARAM_TEST
+  GTEST_CHECK_INT_EQ_(repeat * kNumberOfParamTests, g_param_test_count);
+#endif  // GTEST_HAS_PARAM_TEST
+}
+
+// Tests using --gtest_repeat when --gtest_filter specifies a set of
+// failed tests.
+void TestRepeatWithFilterForFailedTests(int repeat) {
+  GTEST_FLAG(repeat) = repeat;
+  GTEST_FLAG(filter) = "*ShouldFail";
+
+  ResetCounts();
+  GTEST_CHECK_INT_EQ_(1, RUN_ALL_TESTS());
+  GTEST_CHECK_INT_EQ_(repeat, g_environment_set_up_count);
+  GTEST_CHECK_INT_EQ_(repeat, g_environment_tear_down_count);
+  GTEST_CHECK_INT_EQ_(repeat, g_should_fail_count);
+  GTEST_CHECK_INT_EQ_(0, g_should_pass_count);
+  GTEST_CHECK_INT_EQ_(0, g_death_test_count);
+#if GTEST_HAS_PARAM_TEST
+  GTEST_CHECK_INT_EQ_(0, g_param_test_count);
+#endif  // GTEST_HAS_PARAM_TEST
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  testing::AddGlobalTestEnvironment(new MyEnvironment);
+
+  TestRepeatUnspecified();
+  TestRepeat(0);
+  TestRepeat(1);
+  TestRepeat(5);
+
+  TestRepeatWithEmptyFilter(2);
+  TestRepeatWithEmptyFilter(3);
+
+  TestRepeatWithFilterForSuccessfulTests(3);
+
+  TestRepeatWithFilterForFailedTests(4);
+
+  // It would be nice to verify that the tests indeed loop forever
+  // when GTEST_FLAG(repeat) is negative, but this test will be quite
+  // complicated to write.  Since this flag is for interactive
+  // debugging only and doesn't affect the normal test result, such a
+  // test would be an overkill.
+
+  printf("PASS\n");
+  return 0;
+}
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Tests that SCOPED_TRACE() and various Google Test assertions can be
+// used in a large number of threads concurrently.
+
+#include "gtest/gtest.h"
+
+#include <iostream>
+#include <vector>
+
+// We must define this macro in order to #include
+// gtest-internal-inl.h.  This is how Google Test prevents a user from
+// accidentally depending on its internal implementation.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+#if GTEST_IS_THREADSAFE
+
+namespace testing {
+namespace {
+
+using internal::Notification;
+using internal::TestPropertyKeyIs;
+using internal::ThreadWithParam;
+using internal::scoped_ptr;
+
+// In order to run tests in this file, for platforms where Google Test is
+// thread safe, implement ThreadWithParam. See the description of its API
+// in gtest-port.h, where it is defined for already supported platforms.
+
+// How many threads to create?
+const int kThreadCount = 50;
+
+std::string IdToKey(int id, const char* suffix) {
+  Message key;
+  key << "key_" << id << "_" << suffix;
+  return key.GetString();
+}
+
+std::string IdToString(int id) {
+  Message id_message;
+  id_message << id;
+  return id_message.GetString();
+}
+
+void ExpectKeyAndValueWereRecordedForId(
+    const std::vector<TestProperty>& properties,
+    int id, const char* suffix) {
+  TestPropertyKeyIs matches_key(IdToKey(id, suffix).c_str());
+  const std::vector<TestProperty>::const_iterator property =
+      std::find_if(properties.begin(), properties.end(), matches_key);
+  ASSERT_TRUE(property != properties.end())
+      << "expecting " << suffix << " value for id " << id;
+  EXPECT_STREQ(IdToString(id).c_str(), property->value());
+}
+
+// Calls a large number of Google Test assertions, where exactly one of them
+// will fail.
+void ManyAsserts(int id) {
+  GTEST_LOG_(INFO) << "Thread #" << id << " running...";
+
+  SCOPED_TRACE(Message() << "Thread #" << id);
+
+  for (int i = 0; i < kThreadCount; i++) {
+    SCOPED_TRACE(Message() << "Iteration #" << i);
+
+    // A bunch of assertions that should succeed.
+    EXPECT_TRUE(true);
+    ASSERT_FALSE(false) << "This shouldn't fail.";
+    EXPECT_STREQ("a", "a");
+    ASSERT_LE(5, 6);
+    EXPECT_EQ(i, i) << "This shouldn't fail.";
+
+    // RecordProperty() should interact safely with other threads as well.
+    // The shared_key forces property updates.
+    Test::RecordProperty(IdToKey(id, "string").c_str(), IdToString(id).c_str());
+    Test::RecordProperty(IdToKey(id, "int").c_str(), id);
+    Test::RecordProperty("shared_key", IdToString(id).c_str());
+
+    // This assertion should fail kThreadCount times per thread.  It
+    // is for testing whether Google Test can handle failed assertions in a
+    // multi-threaded context.
+    EXPECT_LT(i, 0) << "This should always fail.";
+  }
+}
+
+void CheckTestFailureCount(int expected_failures) {
+  const TestInfo* const info = UnitTest::GetInstance()->current_test_info();
+  const TestResult* const result = info->result();
+  GTEST_CHECK_(expected_failures == result->total_part_count())
+      << "Logged " << result->total_part_count() << " failures "
+      << " vs. " << expected_failures << " expected";
+}
+
+// Tests using SCOPED_TRACE() and Google Test assertions in many threads
+// concurrently.
+TEST(StressTest, CanUseScopedTraceAndAssertionsInManyThreads) {
+  {
+    scoped_ptr<ThreadWithParam<int> > threads[kThreadCount];
+    Notification threads_can_start;
+    for (int i = 0; i != kThreadCount; i++)
+      threads[i].reset(new ThreadWithParam<int>(&ManyAsserts,
+                                                i,
+                                                &threads_can_start));
+
+    threads_can_start.Notify();
+
+    // Blocks until all the threads are done.
+    for (int i = 0; i != kThreadCount; i++)
+      threads[i]->Join();
+  }
+
+  // Ensures that kThreadCount*kThreadCount failures have been reported.
+  const TestInfo* const info = UnitTest::GetInstance()->current_test_info();
+  const TestResult* const result = info->result();
+
+  std::vector<TestProperty> properties;
+  // We have no access to the TestResult's list of properties but we can
+  // copy them one by one.
+  for (int i = 0; i < result->test_property_count(); ++i)
+    properties.push_back(result->GetTestProperty(i));
+
+  EXPECT_EQ(kThreadCount * 2 + 1, result->test_property_count())
+      << "String and int values recorded on each thread, "
+      << "as well as one shared_key";
+  for (int i = 0; i < kThreadCount; ++i) {
+    ExpectKeyAndValueWereRecordedForId(properties, i, "string");
+    ExpectKeyAndValueWereRecordedForId(properties, i, "int");
+  }
+  CheckTestFailureCount(kThreadCount*kThreadCount);
+}
+
+void FailingThread(bool is_fatal) {
+  if (is_fatal)
+    FAIL() << "Fatal failure in some other thread. "
+           << "(This failure is expected.)";
+  else
+    ADD_FAILURE() << "Non-fatal failure in some other thread. "
+                  << "(This failure is expected.)";
+}
+
+void GenerateFatalFailureInAnotherThread(bool is_fatal) {
+  ThreadWithParam<bool> thread(&FailingThread, is_fatal, NULL);
+  thread.Join();
+}
+
+TEST(NoFatalFailureTest, ExpectNoFatalFailureIgnoresFailuresInOtherThreads) {
+  EXPECT_NO_FATAL_FAILURE(GenerateFatalFailureInAnotherThread(true));
+  // We should only have one failure (the one from
+  // GenerateFatalFailureInAnotherThread()), since the EXPECT_NO_FATAL_FAILURE
+  // should succeed.
+  CheckTestFailureCount(1);
+}
+
+void AssertNoFatalFailureIgnoresFailuresInOtherThreads() {
+  ASSERT_NO_FATAL_FAILURE(GenerateFatalFailureInAnotherThread(true));
+}
+TEST(NoFatalFailureTest, AssertNoFatalFailureIgnoresFailuresInOtherThreads) {
+  // Using a subroutine, to make sure, that the test continues.
+  AssertNoFatalFailureIgnoresFailuresInOtherThreads();
+  // We should only have one failure (the one from
+  // GenerateFatalFailureInAnotherThread()), since the EXPECT_NO_FATAL_FAILURE
+  // should succeed.
+  CheckTestFailureCount(1);
+}
+
+TEST(FatalFailureTest, ExpectFatalFailureIgnoresFailuresInOtherThreads) {
+  // This statement should fail, since the current thread doesn't generate a
+  // fatal failure, only another one does.
+  EXPECT_FATAL_FAILURE(GenerateFatalFailureInAnotherThread(true), "expected");
+  CheckTestFailureCount(2);
+}
+
+TEST(FatalFailureOnAllThreadsTest, ExpectFatalFailureOnAllThreads) {
+  // This statement should succeed, because failures in all threads are
+  // considered.
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(
+      GenerateFatalFailureInAnotherThread(true), "expected");
+  CheckTestFailureCount(0);
+  // We need to add a failure, because main() checks that there are failures.
+  // But when only this test is run, we shouldn't have any failures.
+  ADD_FAILURE() << "This is an expected non-fatal failure.";
+}
+
+TEST(NonFatalFailureTest, ExpectNonFatalFailureIgnoresFailuresInOtherThreads) {
+  // This statement should fail, since the current thread doesn't generate a
+  // fatal failure, only another one does.
+  EXPECT_NONFATAL_FAILURE(GenerateFatalFailureInAnotherThread(false),
+                          "expected");
+  CheckTestFailureCount(2);
+}
+
+TEST(NonFatalFailureOnAllThreadsTest, ExpectNonFatalFailureOnAllThreads) {
+  // This statement should succeed, because failures in all threads are
+  // considered.
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(
+      GenerateFatalFailureInAnotherThread(false), "expected");
+  CheckTestFailureCount(0);
+  // We need to add a failure, because main() checks that there are failures,
+  // But when only this test is run, we shouldn't have any failures.
+  ADD_FAILURE() << "This is an expected non-fatal failure.";
+}
+
+}  // namespace
+}  // namespace testing
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  const int result = RUN_ALL_TESTS();  // Expected to fail.
+  GTEST_CHECK_(result == 1) << "RUN_ALL_TESTS() did not fail as expected";
+
+  printf("\nPASS\n");
+  return 0;
+}
+
+#else
+TEST(StressTest,
+     DISABLED_ThreadSafetyTestsAreSkippedWhenGoogleTestIsNotThreadSafe) {
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+#endif  // GTEST_IS_THREADSAFE
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// This program is meant to be run by gtest_help_test.py.  Do not run
+// it directly.
+
+#include "gtest/gtest.h"
+
+// When a help flag is specified, this program should skip the tests
+// and exit with 0; otherwise the following test will be executed,
+// causing this program to exit with a non-zero code.
+TEST(HelpFlagTest, ShouldNotBeRun) {
+  ASSERT_TRUE(false) << "Tests shouldn't be run when --help is specified.";
+}
+
+#if GTEST_HAS_DEATH_TEST
+TEST(DeathTest, UsedByPythonScriptToDetectSupportForDeathTestsInThisBinary) {}
+#endif
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: phanna@google.com (Patrick Hanna)
+
+// Unit test for Google Test's --gtest_list_tests flag.
+//
+// A user can ask Google Test to list all tests that will run
+// so that when using a filter, a user will know what
+// tests to look for. The tests will not be run after listing.
+//
+// This program will be invoked from a Python unit test.
+// Don't run it directly.
+
+#include "gtest/gtest.h"
+
+// Several different test cases and tests that will be listed.
+TEST(Foo, Bar1) {
+}
+
+TEST(Foo, Bar2) {
+}
+
+TEST(Foo, DISABLED_Bar3) {
+}
+
+TEST(Abc, Xyz) {
+}
+
+TEST(Abc, Def) {
+}
+
+TEST(FooBar, Baz) {
+}
+
+class FooTest : public testing::Test {
+};
+
+TEST_F(FooTest, Test1) {
+}
+
+TEST_F(FooTest, DISABLED_Test2) {
+}
+
+TEST_F(FooTest, Test3) {
+}
+
+TEST(FooDeathTest, Test1) {
+}
+
+// A group of value-parameterized tests.
+
+class MyType {
+ public:
+  explicit MyType(const std::string& a_value) : value_(a_value) {}
+
+  const std::string& value() const { return value_; }
+
+ private:
+  std::string value_;
+};
+
+// Teaches Google Test how to print a MyType.
+void PrintTo(const MyType& x, std::ostream* os) {
+  *os << x.value();
+}
+
+class ValueParamTest : public testing::TestWithParam<MyType> {
+};
+
+TEST_P(ValueParamTest, TestA) {
+}
+
+TEST_P(ValueParamTest, TestB) {
+}
+
+INSTANTIATE_TEST_CASE_P(
+    MyInstantiation, ValueParamTest,
+    testing::Values(MyType("one line"),
+                    MyType("two\nlines"),
+                    MyType("a very\nloooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong line")));  // NOLINT
+
+// A group of typed tests.
+
+// A deliberately long type name for testing the line-truncating
+// behavior when printing a type parameter.
+class VeryLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName {  // NOLINT
+};
+
+template <typename T>
+class TypedTest : public testing::Test {
+};
+
+template <typename T, int kSize>
+class MyArray {
+};
+
+typedef testing::Types<VeryLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName,  // NOLINT
+                       int*, MyArray<bool, 42> > MyTypes;
+
+TYPED_TEST_CASE(TypedTest, MyTypes);
+
+TYPED_TEST(TypedTest, TestA) {
+}
+
+TYPED_TEST(TypedTest, TestB) {
+}
+
+// A group of type-parameterized tests.
+
+template <typename T>
+class TypeParamTest : public testing::Test {
+};
+
+TYPED_TEST_CASE_P(TypeParamTest);
+
+TYPED_TEST_P(TypeParamTest, TestA) {
+}
+
+TYPED_TEST_P(TypeParamTest, TestB) {
+}
+
+REGISTER_TYPED_TEST_CASE_P(TypeParamTest, TestA, TestB);
+
+INSTANTIATE_TYPED_TEST_CASE_P(My, TypeParamTest, MyTypes);
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+
+  return RUN_ALL_TESTS();
+}
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Tests that a Google Test program that has no test defined can run
+// successfully.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  // An ad-hoc assertion outside of all tests.
+  //
+  // This serves three purposes:
+  //
+  // 1. It verifies that an ad-hoc assertion can be executed even if
+  //    no test is defined.
+  // 2. It verifies that a failed ad-hoc assertion causes the test
+  //    program to fail.
+  // 3. We had a bug where the XML output won't be generated if an
+  //    assertion is executed before RUN_ALL_TESTS() is called, even
+  //    though --gtest_output=xml is specified.  This makes sure the
+  //    bug is fixed and doesn't regress.
+  EXPECT_EQ(1, 2);
+
+  // The above EXPECT_EQ() should cause RUN_ALL_TESTS() to return non-zero.
+  return RUN_ALL_TESTS() ? 0 : 1;
+}
+// Copyright 2013, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Tests that Google Test manipulates the premature-exit-detection
+// file correctly.
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::internal::posix::GetEnv;
+using ::testing::internal::posix::Stat;
+using ::testing::internal::posix::StatStruct;
+
+namespace {
+
+class PrematureExitTest : public Test {
+ public:
+  // Returns true iff the given file exists.
+  static bool FileExists(const char* filepath) {
+    StatStruct stat;
+    return Stat(filepath, &stat) == 0;
+  }
+
+ protected:
+  PrematureExitTest() {
+    premature_exit_file_path_ = GetEnv("TEST_PREMATURE_EXIT_FILE");
+
+    // Normalize NULL to "" for ease of handling.
+    if (premature_exit_file_path_ == NULL) {
+      premature_exit_file_path_ = "";
+    }
+  }
+
+  // Returns true iff the premature-exit file exists.
+  bool PrematureExitFileExists() const {
+    return FileExists(premature_exit_file_path_);
+  }
+
+  const char* premature_exit_file_path_;
+};
+
+typedef PrematureExitTest PrematureExitDeathTest;
+
+// Tests that:
+//   - the premature-exit file exists during the execution of a
+//     death test (EXPECT_DEATH*), and
+//   - a death test doesn't interfere with the main test process's
+//     handling of the premature-exit file.
+TEST_F(PrematureExitDeathTest, FileExistsDuringExecutionOfDeathTest) {
+  if (*premature_exit_file_path_ == '\0') {
+    return;
+  }
+
+  EXPECT_DEATH_IF_SUPPORTED({
+      // If the file exists, crash the process such that the main test
+      // process will catch the (expected) crash and report a success;
+      // otherwise don't crash, which will cause the main test process
+      // to report that the death test has failed.
+      if (PrematureExitFileExists()) {
+        exit(1);
+      }
+    }, "");
+}
+
+// Tests that the premature-exit file exists during the execution of a
+// normal (non-death) test.
+TEST_F(PrematureExitTest, PrematureExitFileExistsDuringTestExecution) {
+  if (*premature_exit_file_path_ == '\0') {
+    return;
+  }
+
+  EXPECT_TRUE(PrematureExitFileExists())
+      << " file " << premature_exit_file_path_
+      << " should exist during test execution, but doesn't.";
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+  const int exit_code = RUN_ALL_TESTS();
+
+  // Test that the premature-exit file is deleted upon return from
+  // RUN_ALL_TESTS().
+  const char* const filepath = GetEnv("TEST_PREMATURE_EXIT_FILE");
+  if (filepath != NULL && *filepath != '\0') {
+    if (PrematureExitTest::FileExists(filepath)) {
+      printf(
+          "File %s shouldn't exist after the test program finishes, but does.",
+          filepath);
+      return 1;
+    }
+  }
+
+  return exit_code;
+}
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Verifies that test shuffling works.
+
+#include "gtest/gtest.h"
+
+namespace {
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Message;
+using ::testing::Test;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::UnitTest;
+using ::testing::internal::scoped_ptr;
+
+// The test methods are empty, as the sole purpose of this program is
+// to print the test names before/after shuffling.
+
+class A : public Test {};
+TEST_F(A, A) {}
+TEST_F(A, B) {}
+
+TEST(ADeathTest, A) {}
+TEST(ADeathTest, B) {}
+TEST(ADeathTest, C) {}
+
+TEST(B, A) {}
+TEST(B, B) {}
+TEST(B, C) {}
+TEST(B, DISABLED_D) {}
+TEST(B, DISABLED_E) {}
+
+TEST(BDeathTest, A) {}
+TEST(BDeathTest, B) {}
+
+TEST(C, A) {}
+TEST(C, B) {}
+TEST(C, C) {}
+TEST(C, DISABLED_D) {}
+
+TEST(CDeathTest, A) {}
+
+TEST(DISABLED_D, A) {}
+TEST(DISABLED_D, DISABLED_B) {}
+
+// This printer prints the full test names only, starting each test
+// iteration with a "----" marker.
+class TestNamePrinter : public EmptyTestEventListener {
+ public:
+  virtual void OnTestIterationStart(const UnitTest& /* unit_test */,
+                                    int /* iteration */) {
+    printf("----\n");
+  }
+
+  virtual void OnTestStart(const TestInfo& test_info) {
+    printf("%s.%s\n", test_info.test_case_name(), test_info.name());
+  }
+};
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  // Replaces the default printer with TestNamePrinter, which prints
+  // the test name only.
+  TestEventListeners& listeners = UnitTest::GetInstance()->listeners();
+  delete listeners.Release(listeners.default_result_printer());
+  listeners.Append(new TestNamePrinter);
+
+  return RUN_ALL_TESTS();
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+// This test verifies that it's possible to use Google Test by including
+// the gtest.h header file alone.
+
+#include "gtest/gtest.h"
+
+namespace {
+
+void Subroutine() {
+  EXPECT_EQ(42, 42);
+}
+
+TEST(NoFatalFailureTest, ExpectNoFatalFailure) {
+  EXPECT_NO_FATAL_FAILURE(;);
+  EXPECT_NO_FATAL_FAILURE(SUCCEED());
+  EXPECT_NO_FATAL_FAILURE(Subroutine());
+  EXPECT_NO_FATAL_FAILURE({ SUCCEED(); });
+}
+
+TEST(NoFatalFailureTest, AssertNoFatalFailure) {
+  ASSERT_NO_FATAL_FAILURE(;);
+  ASSERT_NO_FATAL_FAILURE(SUCCEED());
+  ASSERT_NO_FATAL_FAILURE(Subroutine());
+  ASSERT_NO_FATAL_FAILURE({ SUCCEED(); });
+}
+
+}  // namespace
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Tests Google Test's throw-on-failure mode with exceptions enabled.
+
+#include "gtest/gtest.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdexcept>
+
+// Prints the given failure message and exits the program with
+// non-zero.  We use this instead of a Google Test assertion to
+// indicate a failure, as the latter is been tested and cannot be
+// relied on.
+void Fail(const char* msg) {
+  printf("FAILURE: %s\n", msg);
+  fflush(stdout);
+  exit(1);
+}
+
+// Tests that an assertion failure throws a subclass of
+// std::runtime_error.
+void TestFailureThrowsRuntimeError() {
+  testing::GTEST_FLAG(throw_on_failure) = true;
+
+  // A successful assertion shouldn't throw.
+  try {
+    EXPECT_EQ(3, 3);
+  } catch(...) {
+    Fail("A successful assertion wrongfully threw.");
+  }
+
+  // A failed assertion should throw a subclass of std::runtime_error.
+  try {
+    EXPECT_EQ(2, 3) << "Expected failure";
+  } catch(const std::runtime_error& e) {
+    if (strstr(e.what(), "Expected failure") != NULL)
+      return;
+
+    printf("%s",
+           "A failed assertion did throw an exception of the right type, "
+           "but the message is incorrect.  Instead of containing \"Expected "
+           "failure\", it is:\n");
+    Fail(e.what());
+  } catch(...) {
+    Fail("A failed assertion threw the wrong type of exception.");
+  }
+  Fail("A failed assertion should've thrown but didn't.");
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  // We want to ensure that people can use Google Test assertions in
+  // other testing frameworks, as long as they initialize Google Test
+  // properly and set the thrown-on-failure mode.  Therefore, we don't
+  // use Google Test's constructs for defining and running tests
+  // (e.g. TEST and RUN_ALL_TESTS) here.
+
+  TestFailureThrowsRuntimeError();
+  return 0;
+}
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Tests Google Test's throw-on-failure mode with exceptions disabled.
+//
+// This program must be compiled with exceptions disabled.  It will be
+// invoked by gtest_throw_on_failure_test.py, and is expected to exit
+// with non-zero in the throw-on-failure mode or 0 otherwise.
+
+#include "gtest/gtest.h"
+
+#include <stdio.h>                      // for fflush, fprintf, NULL, etc.
+#include <stdlib.h>                     // for exit
+#include <exception>                    // for set_terminate
+
+// This terminate handler aborts the program using exit() rather than abort().
+// This avoids showing pop-ups on Windows systems and core dumps on Unix-like
+// ones.
+void TerminateHandler() {
+  fprintf(stderr, "%s\n", "Unhandled C++ exception terminating the program.");
+  fflush(NULL);
+  exit(1);
+}
+
+int main(int argc, char** argv) {
+#if GTEST_HAS_EXCEPTIONS
+  std::set_terminate(&TerminateHandler);
+#endif
+  testing::InitGoogleTest(&argc, argv);
+
+  // We want to ensure that people can use Google Test assertions in
+  // other testing frameworks, as long as they initialize Google Test
+  // properly and set the throw-on-failure mode.  Therefore, we don't
+  // use Google Test's constructs for defining and running tests
+  // (e.g. TEST and RUN_ALL_TESTS) here.
+
+  // In the throw-on-failure mode with exceptions disabled, this
+  // assertion will cause the program to exit with a non-zero code.
+  EXPECT_EQ(2, 3);
+
+  // When not in the throw-on-failure mode, the control will reach
+  // here.
+  return 0;
+}
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Tests using global test environments.
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "gtest/gtest.h"
+
+#define GTEST_IMPLEMENTATION_ 1  // Required for the next #include.
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+GTEST_DECLARE_string_(filter);
+}
+
+namespace {
+
+enum FailureType {
+  NO_FAILURE, NON_FATAL_FAILURE, FATAL_FAILURE
+};
+
+// For testing using global test environments.
+class MyEnvironment : public testing::Environment {
+ public:
+  MyEnvironment() { Reset(); }
+
+  // Depending on the value of failure_in_set_up_, SetUp() will
+  // generate a non-fatal failure, generate a fatal failure, or
+  // succeed.
+  virtual void SetUp() {
+    set_up_was_run_ = true;
+
+    switch (failure_in_set_up_) {
+      case NON_FATAL_FAILURE:
+        ADD_FAILURE() << "Expected non-fatal failure in global set-up.";
+        break;
+      case FATAL_FAILURE:
+        FAIL() << "Expected fatal failure in global set-up.";
+        break;
+      default:
+        break;
+    }
+  }
+
+  // Generates a non-fatal failure.
+  virtual void TearDown() {
+    tear_down_was_run_ = true;
+    ADD_FAILURE() << "Expected non-fatal failure in global tear-down.";
+  }
+
+  // Resets the state of the environment s.t. it can be reused.
+  void Reset() {
+    failure_in_set_up_ = NO_FAILURE;
+    set_up_was_run_ = false;
+    tear_down_was_run_ = false;
+  }
+
+  // We call this function to set the type of failure SetUp() should
+  // generate.
+  void set_failure_in_set_up(FailureType type) {
+    failure_in_set_up_ = type;
+  }
+
+  // Was SetUp() run?
+  bool set_up_was_run() const { return set_up_was_run_; }
+
+  // Was TearDown() run?
+  bool tear_down_was_run() const { return tear_down_was_run_; }
+
+ private:
+  FailureType failure_in_set_up_;
+  bool set_up_was_run_;
+  bool tear_down_was_run_;
+};
+
+// Was the TEST run?
+bool test_was_run;
+
+// The sole purpose of this TEST is to enable us to check whether it
+// was run.
+TEST(FooTest, Bar) {
+  test_was_run = true;
+}
+
+// Prints the message and aborts the program if condition is false.
+void Check(bool condition, const char* msg) {
+  if (!condition) {
+    printf("FAILED: %s\n", msg);
+    testing::internal::posix::Abort();
+  }
+}
+
+// Runs the tests.  Return true iff successful.
+//
+// The 'failure' parameter specifies the type of failure that should
+// be generated by the global set-up.
+int RunAllTests(MyEnvironment* env, FailureType failure) {
+  env->Reset();
+  env->set_failure_in_set_up(failure);
+  test_was_run = false;
+  testing::internal::GetUnitTestImpl()->ClearAdHocTestResult();
+  return RUN_ALL_TESTS();
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  // Registers a global test environment, and verifies that the
+  // registration function returns its argument.
+  MyEnvironment* const env = new MyEnvironment;
+  Check(testing::AddGlobalTestEnvironment(env) == env,
+        "AddGlobalTestEnvironment() should return its argument.");
+
+  // Verifies that RUN_ALL_TESTS() runs the tests when the global
+  // set-up is successful.
+  Check(RunAllTests(env, NO_FAILURE) != 0,
+        "RUN_ALL_TESTS() should return non-zero, as the global tear-down "
+        "should generate a failure.");
+  Check(test_was_run,
+        "The tests should run, as the global set-up should generate no "
+        "failure");
+  Check(env->tear_down_was_run(),
+        "The global tear-down should run, as the global set-up was run.");
+
+  // Verifies that RUN_ALL_TESTS() runs the tests when the global
+  // set-up generates no fatal failure.
+  Check(RunAllTests(env, NON_FATAL_FAILURE) != 0,
+        "RUN_ALL_TESTS() should return non-zero, as both the global set-up "
+        "and the global tear-down should generate a non-fatal failure.");
+  Check(test_was_run,
+        "The tests should run, as the global set-up should generate no "
+        "fatal failure.");
+  Check(env->tear_down_was_run(),
+        "The global tear-down should run, as the global set-up was run.");
+
+  // Verifies that RUN_ALL_TESTS() runs no test when the global set-up
+  // generates a fatal failure.
+  Check(RunAllTests(env, FATAL_FAILURE) != 0,
+        "RUN_ALL_TESTS() should return non-zero, as the global set-up "
+        "should generate a fatal failure.");
+  Check(!test_was_run,
+        "The tests should not run, as the global set-up should generate "
+        "a fatal failure.");
+  Check(env->tear_down_was_run(),
+        "The global tear-down should run, as the global set-up was run.");
+
+  // Verifies that RUN_ALL_TESTS() doesn't do global set-up or
+  // tear-down when there is no test to run.
+  testing::GTEST_FLAG(filter) = "-*";
+  Check(RunAllTests(env, NO_FAILURE) == 0,
+        "RUN_ALL_TESTS() should return zero, as there is no test to run.");
+  Check(!env->set_up_was_run(),
+        "The global set-up should not run, as there is no test to run.");
+  Check(!env->tear_down_was_run(),
+        "The global tear-down should not run, "
+        "as the global set-up was not run.");
+
+  printf("PASS\n");
+  return 0;
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Unit test for Google Test test filters.
+//
+// A user can specify which test(s) in a Google Test program to run via
+// either the GTEST_FILTER environment variable or the --gtest_filter
+// flag.  This is used for testing such functionality.
+//
+// The program will be invoked from a Python unit test.  Don't run it
+// directly.
+
+#include "gtest/gtest.h"
+
+namespace {
+
+// Test case FooTest.
+
+class FooTest : public testing::Test {
+};
+
+TEST_F(FooTest, Abc) {
+}
+
+TEST_F(FooTest, Xyz) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case BarTest.
+
+TEST(BarTest, TestOne) {
+}
+
+TEST(BarTest, TestTwo) {
+}
+
+TEST(BarTest, TestThree) {
+}
+
+TEST(BarTest, DISABLED_TestFour) {
+  FAIL() << "Expected failure.";
+}
+
+TEST(BarTest, DISABLED_TestFive) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case BazTest.
+
+TEST(BazTest, TestOne) {
+  FAIL() << "Expected failure.";
+}
+
+TEST(BazTest, TestA) {
+}
+
+TEST(BazTest, TestB) {
+}
+
+TEST(BazTest, DISABLED_TestC) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case HasDeathTest
+
+TEST(HasDeathTest, Test1) {
+  EXPECT_DEATH_IF_SUPPORTED(exit(1), ".*");
+}
+
+// We need at least two death tests to make sure that the all death tests
+// aren't on the first shard.
+TEST(HasDeathTest, Test2) {
+  EXPECT_DEATH_IF_SUPPORTED(exit(1), ".*");
+}
+
+// Test case FoobarTest
+
+TEST(DISABLED_FoobarTest, Test1) {
+  FAIL() << "Expected failure.";
+}
+
+TEST(DISABLED_FoobarTest, DISABLED_Test2) {
+  FAIL() << "Expected failure.";
+}
+
+// Test case FoobarbazTest
+
+TEST(DISABLED_FoobarbazTest, TestA) {
+  FAIL() << "Expected failure.";
+}
+
+#if GTEST_HAS_PARAM_TEST
+class ParamTest : public testing::TestWithParam<int> {
+};
+
+TEST_P(ParamTest, TestX) {
+}
+
+TEST_P(ParamTest, TestY) {
+}
+
+INSTANTIATE_TEST_CASE_P(SeqP, ParamTest, testing::Values(1, 2));
+INSTANTIATE_TEST_CASE_P(SeqQ, ParamTest, testing::Values(5, 6));
+#endif  // GTEST_HAS_PARAM_TEST
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+
+  return RUN_ALL_TESTS();
+}
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Unit test for include/gtest/gtest_prod.h.
+
+#include "gtest/gtest.h"
+#include "test/production.h"
+
+// Tests that private members can be accessed from a TEST declared as
+// a friend of the class.
+TEST(PrivateCodeTest, CanAccessPrivateMembers) {
+  PrivateCode a;
+  EXPECT_EQ(0, a.x_);
+
+  a.set_x(1);
+  EXPECT_EQ(1, a.x_);
+}
+
+typedef testing::Test PrivateCodeFixtureTest;
+
+// Tests that private members can be accessed from a TEST_F declared
+// as a friend of the class.
+TEST_F(PrivateCodeFixtureTest, CanAccessPrivateMembers) {
+  PrivateCode a;
+  EXPECT_EQ(0, a.x_);
+
+  a.set_x(2);
+  EXPECT_EQ(2, a.x_);
+}
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest.h"
+
+// Tests that we don't have to define main() when we link to
+// gtest_main instead of gtest.
+
+namespace {
+
+TEST(GTestMainTest, ShouldSucceed) {
+}
+
+}  // namespace
+
+// We are using the main() function defined in src/gtest_main.cc, so
+// we don't define it here.
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// The purpose of this file is to generate Google Test output under
+// various conditions.  The output will then be verified by
+// gtest_output_test.py to ensure that Google Test generates the
+// desired messages.  Therefore, most tests in this file are MEANT TO
+// FAIL.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest-spi.h"
+#include "gtest/gtest.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+#include <stdlib.h>
+
+#if GTEST_IS_THREADSAFE
+using testing::ScopedFakeTestPartResultReporter;
+using testing::TestPartResultArray;
+
+using testing::internal::Notification;
+using testing::internal::ThreadWithParam;
+#endif
+
+namespace posix = ::testing::internal::posix;
+
+// Tests catching fatal failures.
+
+// A subroutine used by the following test.
+void TestEq1(int x) {
+  ASSERT_EQ(1, x);
+}
+
+// This function calls a test subroutine, catches the fatal failure it
+// generates, and then returns early.
+void TryTestSubroutine() {
+  // Calls a subrountine that yields a fatal failure.
+  TestEq1(2);
+
+  // Catches the fatal failure and aborts the test.
+  //
+  // The testing::Test:: prefix is necessary when calling
+  // HasFatalFailure() outside of a TEST, TEST_F, or test fixture.
+  if (testing::Test::HasFatalFailure()) return;
+
+  // If we get here, something is wrong.
+  FAIL() << "This should never be reached.";
+}
+
+TEST(PassingTest, PassingTest1) {
+}
+
+TEST(PassingTest, PassingTest2) {
+}
+
+// Tests that parameters of failing parameterized tests are printed in the
+// failing test summary.
+class FailingParamTest : public testing::TestWithParam<int> {};
+
+TEST_P(FailingParamTest, Fails) {
+  EXPECT_EQ(1, GetParam());
+}
+
+// This generates a test which will fail. Google Test is expected to print
+// its parameter when it outputs the list of all failed tests.
+INSTANTIATE_TEST_CASE_P(PrintingFailingParams,
+                        FailingParamTest,
+                        testing::Values(2));
+
+static const char kGoldenString[] = "\"Line\0 1\"\nLine 2";
+
+TEST(NonfatalFailureTest, EscapesStringOperands) {
+  std::string actual = "actual \"string\"";
+  EXPECT_EQ(kGoldenString, actual);
+
+  const char* golden = kGoldenString;
+  EXPECT_EQ(golden, actual);
+}
+
+TEST(NonfatalFailureTest, DiffForLongStrings) {
+  std::string golden_str(kGoldenString, sizeof(kGoldenString) - 1);
+  EXPECT_EQ(golden_str, "Line 2");
+}
+
+// Tests catching a fatal failure in a subroutine.
+TEST(FatalFailureTest, FatalFailureInSubroutine) {
+  printf("(expecting a failure that x should be 1)\n");
+
+  TryTestSubroutine();
+}
+
+// Tests catching a fatal failure in a nested subroutine.
+TEST(FatalFailureTest, FatalFailureInNestedSubroutine) {
+  printf("(expecting a failure that x should be 1)\n");
+
+  // Calls a subrountine that yields a fatal failure.
+  TryTestSubroutine();
+
+  // Catches the fatal failure and aborts the test.
+  //
+  // When calling HasFatalFailure() inside a TEST, TEST_F, or test
+  // fixture, the testing::Test:: prefix is not needed.
+  if (HasFatalFailure()) return;
+
+  // If we get here, something is wrong.
+  FAIL() << "This should never be reached.";
+}
+
+// Tests HasFatalFailure() after a failed EXPECT check.
+TEST(FatalFailureTest, NonfatalFailureInSubroutine) {
+  printf("(expecting a failure on false)\n");
+  EXPECT_TRUE(false);  // Generates a nonfatal failure
+  ASSERT_FALSE(HasFatalFailure());  // This should succeed.
+}
+
+// Tests interleaving user logging and Google Test assertions.
+TEST(LoggingTest, InterleavingLoggingAndAssertions) {
+  static const int a[4] = {
+    3, 9, 2, 6
+  };
+
+  printf("(expecting 2 failures on (3) >= (a[i]))\n");
+  for (int i = 0; i < static_cast<int>(sizeof(a)/sizeof(*a)); i++) {
+    printf("i == %d\n", i);
+    EXPECT_GE(3, a[i]);
+  }
+}
+
+// Tests the SCOPED_TRACE macro.
+
+// A helper function for testing SCOPED_TRACE.
+void SubWithoutTrace(int n) {
+  EXPECT_EQ(1, n);
+  ASSERT_EQ(2, n);
+}
+
+// Another helper function for testing SCOPED_TRACE.
+void SubWithTrace(int n) {
+  SCOPED_TRACE(testing::Message() << "n = " << n);
+
+  SubWithoutTrace(n);
+}
+
+// Tests that SCOPED_TRACE() obeys lexical scopes.
+TEST(SCOPED_TRACETest, ObeysScopes) {
+  printf("(expected to fail)\n");
+
+  // There should be no trace before SCOPED_TRACE() is invoked.
+  ADD_FAILURE() << "This failure is expected, and shouldn't have a trace.";
+
+  {
+    SCOPED_TRACE("Expected trace");
+    // After SCOPED_TRACE(), a failure in the current scope should contain
+    // the trace.
+    ADD_FAILURE() << "This failure is expected, and should have a trace.";
+  }
+
+  // Once the control leaves the scope of the SCOPED_TRACE(), there
+  // should be no trace again.
+  ADD_FAILURE() << "This failure is expected, and shouldn't have a trace.";
+}
+
+// Tests that SCOPED_TRACE works inside a loop.
+TEST(SCOPED_TRACETest, WorksInLoop) {
+  printf("(expected to fail)\n");
+
+  for (int i = 1; i <= 2; i++) {
+    SCOPED_TRACE(testing::Message() << "i = " << i);
+
+    SubWithoutTrace(i);
+  }
+}
+
+// Tests that SCOPED_TRACE works in a subroutine.
+TEST(SCOPED_TRACETest, WorksInSubroutine) {
+  printf("(expected to fail)\n");
+
+  SubWithTrace(1);
+  SubWithTrace(2);
+}
+
+// Tests that SCOPED_TRACE can be nested.
+TEST(SCOPED_TRACETest, CanBeNested) {
+  printf("(expected to fail)\n");
+
+  SCOPED_TRACE("");  // A trace without a message.
+
+  SubWithTrace(2);
+}
+
+// Tests that multiple SCOPED_TRACEs can be used in the same scope.
+TEST(SCOPED_TRACETest, CanBeRepeated) {
+  printf("(expected to fail)\n");
+
+  SCOPED_TRACE("A");
+  ADD_FAILURE()
+      << "This failure is expected, and should contain trace point A.";
+
+  SCOPED_TRACE("B");
+  ADD_FAILURE()
+      << "This failure is expected, and should contain trace point A and B.";
+
+  {
+    SCOPED_TRACE("C");
+    ADD_FAILURE() << "This failure is expected, and should "
+                  << "contain trace point A, B, and C.";
+  }
+
+  SCOPED_TRACE("D");
+  ADD_FAILURE() << "This failure is expected, and should "
+                << "contain trace point A, B, and D.";
+}
+
+#if GTEST_IS_THREADSAFE
+// Tests that SCOPED_TRACE()s can be used concurrently from multiple
+// threads.  Namely, an assertion should be affected by
+// SCOPED_TRACE()s in its own thread only.
+
+// Here's the sequence of actions that happen in the test:
+//
+//   Thread A (main)                | Thread B (spawned)
+//   ===============================|================================
+//   spawns thread B                |
+//   -------------------------------+--------------------------------
+//   waits for n1                   | SCOPED_TRACE("Trace B");
+//                                  | generates failure #1
+//                                  | notifies n1
+//   -------------------------------+--------------------------------
+//   SCOPED_TRACE("Trace A");       | waits for n2
+//   generates failure #2           |
+//   notifies n2                    |
+//   -------------------------------|--------------------------------
+//   waits for n3                   | generates failure #3
+//                                  | trace B dies
+//                                  | generates failure #4
+//                                  | notifies n3
+//   -------------------------------|--------------------------------
+//   generates failure #5           | finishes
+//   trace A dies                   |
+//   generates failure #6           |
+//   -------------------------------|--------------------------------
+//   waits for thread B to finish   |
+
+struct CheckPoints {
+  Notification n1;
+  Notification n2;
+  Notification n3;
+};
+
+static void ThreadWithScopedTrace(CheckPoints* check_points) {
+  {
+    SCOPED_TRACE("Trace B");
+    ADD_FAILURE()
+        << "Expected failure #1 (in thread B, only trace B alive).";
+    check_points->n1.Notify();
+    check_points->n2.WaitForNotification();
+
+    ADD_FAILURE()
+        << "Expected failure #3 (in thread B, trace A & B both alive).";
+  }  // Trace B dies here.
+  ADD_FAILURE()
+      << "Expected failure #4 (in thread B, only trace A alive).";
+  check_points->n3.Notify();
+}
+
+TEST(SCOPED_TRACETest, WorksConcurrently) {
+  printf("(expecting 6 failures)\n");
+
+  CheckPoints check_points;
+  ThreadWithParam<CheckPoints*> thread(&ThreadWithScopedTrace,
+                                       &check_points,
+                                       NULL);
+  check_points.n1.WaitForNotification();
+
+  {
+    SCOPED_TRACE("Trace A");
+    ADD_FAILURE()
+        << "Expected failure #2 (in thread A, trace A & B both alive).";
+    check_points.n2.Notify();
+    check_points.n3.WaitForNotification();
+
+    ADD_FAILURE()
+        << "Expected failure #5 (in thread A, only trace A alive).";
+  }  // Trace A dies here.
+  ADD_FAILURE()
+      << "Expected failure #6 (in thread A, no trace alive).";
+  thread.Join();
+}
+#endif  // GTEST_IS_THREADSAFE
+
+TEST(DisabledTestsWarningTest,
+     DISABLED_AlsoRunDisabledTestsFlagSuppressesWarning) {
+  // This test body is intentionally empty.  Its sole purpose is for
+  // verifying that the --gtest_also_run_disabled_tests flag
+  // suppresses the "YOU HAVE 12 DISABLED TESTS" warning at the end of
+  // the test output.
+}
+
+// Tests using assertions outside of TEST and TEST_F.
+//
+// This function creates two failures intentionally.
+void AdHocTest() {
+  printf("The non-test part of the code is expected to have 2 failures.\n\n");
+  EXPECT_TRUE(false);
+  EXPECT_EQ(2, 3);
+}
+
+// Runs all TESTs, all TEST_Fs, and the ad hoc test.
+int RunAllTests() {
+  AdHocTest();
+  return RUN_ALL_TESTS();
+}
+
+// Tests non-fatal failures in the fixture constructor.
+class NonFatalFailureInFixtureConstructorTest : public testing::Test {
+ protected:
+  NonFatalFailureInFixtureConstructorTest() {
+    printf("(expecting 5 failures)\n");
+    ADD_FAILURE() << "Expected failure #1, in the test fixture c'tor.";
+  }
+
+  ~NonFatalFailureInFixtureConstructorTest() {
+    ADD_FAILURE() << "Expected failure #5, in the test fixture d'tor.";
+  }
+
+  virtual void SetUp() {
+    ADD_FAILURE() << "Expected failure #2, in SetUp().";
+  }
+
+  virtual void TearDown() {
+    ADD_FAILURE() << "Expected failure #4, in TearDown.";
+  }
+};
+
+TEST_F(NonFatalFailureInFixtureConstructorTest, FailureInConstructor) {
+  ADD_FAILURE() << "Expected failure #3, in the test body.";
+}
+
+// Tests fatal failures in the fixture constructor.
+class FatalFailureInFixtureConstructorTest : public testing::Test {
+ protected:
+  FatalFailureInFixtureConstructorTest() {
+    printf("(expecting 2 failures)\n");
+    Init();
+  }
+
+  ~FatalFailureInFixtureConstructorTest() {
+    ADD_FAILURE() << "Expected failure #2, in the test fixture d'tor.";
+  }
+
+  virtual void SetUp() {
+    ADD_FAILURE() << "UNEXPECTED failure in SetUp().  "
+                  << "We should never get here, as the test fixture c'tor "
+                  << "had a fatal failure.";
+  }
+
+  virtual void TearDown() {
+    ADD_FAILURE() << "UNEXPECTED failure in TearDown().  "
+                  << "We should never get here, as the test fixture c'tor "
+                  << "had a fatal failure.";
+  }
+
+ private:
+  void Init() {
+    FAIL() << "Expected failure #1, in the test fixture c'tor.";
+  }
+};
+
+TEST_F(FatalFailureInFixtureConstructorTest, FailureInConstructor) {
+  ADD_FAILURE() << "UNEXPECTED failure in the test body.  "
+                << "We should never get here, as the test fixture c'tor "
+                << "had a fatal failure.";
+}
+
+// Tests non-fatal failures in SetUp().
+class NonFatalFailureInSetUpTest : public testing::Test {
+ protected:
+  virtual ~NonFatalFailureInSetUpTest() {
+    Deinit();
+  }
+
+  virtual void SetUp() {
+    printf("(expecting 4 failures)\n");
+    ADD_FAILURE() << "Expected failure #1, in SetUp().";
+  }
+
+  virtual void TearDown() {
+    FAIL() << "Expected failure #3, in TearDown().";
+  }
+ private:
+  void Deinit() {
+    FAIL() << "Expected failure #4, in the test fixture d'tor.";
+  }
+};
+
+TEST_F(NonFatalFailureInSetUpTest, FailureInSetUp) {
+  FAIL() << "Expected failure #2, in the test function.";
+}
+
+// Tests fatal failures in SetUp().
+class FatalFailureInSetUpTest : public testing::Test {
+ protected:
+  virtual ~FatalFailureInSetUpTest() {
+    Deinit();
+  }
+
+  virtual void SetUp() {
+    printf("(expecting 3 failures)\n");
+    FAIL() << "Expected failure #1, in SetUp().";
+  }
+
+  virtual void TearDown() {
+    FAIL() << "Expected failure #2, in TearDown().";
+  }
+ private:
+  void Deinit() {
+    FAIL() << "Expected failure #3, in the test fixture d'tor.";
+  }
+};
+
+TEST_F(FatalFailureInSetUpTest, FailureInSetUp) {
+  FAIL() << "UNEXPECTED failure in the test function.  "
+         << "We should never get here, as SetUp() failed.";
+}
+
+TEST(AddFailureAtTest, MessageContainsSpecifiedFileAndLineNumber) {
+  ADD_FAILURE_AT("foo.cc", 42) << "Expected failure in foo.cc";
+}
+
+#if GTEST_IS_THREADSAFE
+
+// A unary function that may die.
+void DieIf(bool should_die) {
+  GTEST_CHECK_(!should_die) << " - death inside DieIf().";
+}
+
+// Tests running death tests in a multi-threaded context.
+
+// Used for coordination between the main and the spawn thread.
+struct SpawnThreadNotifications {
+  SpawnThreadNotifications() {}
+
+  Notification spawn_thread_started;
+  Notification spawn_thread_ok_to_terminate;
+
+ private:
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(SpawnThreadNotifications);
+};
+
+// The function to be executed in the thread spawn by the
+// MultipleThreads test (below).
+static void ThreadRoutine(SpawnThreadNotifications* notifications) {
+  // Signals the main thread that this thread has started.
+  notifications->spawn_thread_started.Notify();
+
+  // Waits for permission to finish from the main thread.
+  notifications->spawn_thread_ok_to_terminate.WaitForNotification();
+}
+
+// This is a death-test test, but it's not named with a DeathTest
+// suffix.  It starts threads which might interfere with later
+// death tests, so it must run after all other death tests.
+class DeathTestAndMultiThreadsTest : public testing::Test {
+ protected:
+  // Starts a thread and waits for it to begin.
+  virtual void SetUp() {
+    thread_.reset(new ThreadWithParam<SpawnThreadNotifications*>(
+        &ThreadRoutine, &notifications_, NULL));
+    notifications_.spawn_thread_started.WaitForNotification();
+  }
+  // Tells the thread to finish, and reaps it.
+  // Depending on the version of the thread library in use,
+  // a manager thread might still be left running that will interfere
+  // with later death tests.  This is unfortunate, but this class
+  // cleans up after itself as best it can.
+  virtual void TearDown() {
+    notifications_.spawn_thread_ok_to_terminate.Notify();
+  }
+
+ private:
+  SpawnThreadNotifications notifications_;
+  testing::internal::scoped_ptr<ThreadWithParam<SpawnThreadNotifications*> >
+      thread_;
+};
+
+#endif  // GTEST_IS_THREADSAFE
+
+// The MixedUpTestCaseTest test case verifies that Google Test will fail a
+// test if it uses a different fixture class than what other tests in
+// the same test case use.  It deliberately contains two fixture
+// classes with the same name but defined in different namespaces.
+
+// The MixedUpTestCaseWithSameTestNameTest test case verifies that
+// when the user defines two tests with the same test case name AND
+// same test name (but in different namespaces), the second test will
+// fail.
+
+namespace foo {
+
+class MixedUpTestCaseTest : public testing::Test {
+};
+
+TEST_F(MixedUpTestCaseTest, FirstTestFromNamespaceFoo) {}
+TEST_F(MixedUpTestCaseTest, SecondTestFromNamespaceFoo) {}
+
+class MixedUpTestCaseWithSameTestNameTest : public testing::Test {
+};
+
+TEST_F(MixedUpTestCaseWithSameTestNameTest,
+       TheSecondTestWithThisNameShouldFail) {}
+
+}  // namespace foo
+
+namespace bar {
+
+class MixedUpTestCaseTest : public testing::Test {
+};
+
+// The following two tests are expected to fail.  We rely on the
+// golden file to check that Google Test generates the right error message.
+TEST_F(MixedUpTestCaseTest, ThisShouldFail) {}
+TEST_F(MixedUpTestCaseTest, ThisShouldFailToo) {}
+
+class MixedUpTestCaseWithSameTestNameTest : public testing::Test {
+};
+
+// Expected to fail.  We rely on the golden file to check that Google Test
+// generates the right error message.
+TEST_F(MixedUpTestCaseWithSameTestNameTest,
+       TheSecondTestWithThisNameShouldFail) {}
+
+}  // namespace bar
+
+// The following two test cases verify that Google Test catches the user
+// error of mixing TEST and TEST_F in the same test case.  The first
+// test case checks the scenario where TEST_F appears before TEST, and
+// the second one checks where TEST appears before TEST_F.
+
+class TEST_F_before_TEST_in_same_test_case : public testing::Test {
+};
+
+TEST_F(TEST_F_before_TEST_in_same_test_case, DefinedUsingTEST_F) {}
+
+// Expected to fail.  We rely on the golden file to check that Google Test
+// generates the right error message.
+TEST(TEST_F_before_TEST_in_same_test_case, DefinedUsingTESTAndShouldFail) {}
+
+class TEST_before_TEST_F_in_same_test_case : public testing::Test {
+};
+
+TEST(TEST_before_TEST_F_in_same_test_case, DefinedUsingTEST) {}
+
+// Expected to fail.  We rely on the golden file to check that Google Test
+// generates the right error message.
+TEST_F(TEST_before_TEST_F_in_same_test_case, DefinedUsingTEST_FAndShouldFail) {
+}
+
+// Used for testing EXPECT_NONFATAL_FAILURE() and EXPECT_FATAL_FAILURE().
+int global_integer = 0;
+
+// Tests that EXPECT_NONFATAL_FAILURE() can reference global variables.
+TEST(ExpectNonfatalFailureTest, CanReferenceGlobalVariables) {
+  global_integer = 0;
+  EXPECT_NONFATAL_FAILURE({
+    EXPECT_EQ(1, global_integer) << "Expected non-fatal failure.";
+  }, "Expected non-fatal failure.");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() can reference local variables
+// (static or not).
+TEST(ExpectNonfatalFailureTest, CanReferenceLocalVariables) {
+  int m = 0;
+  static int n;
+  n = 1;
+  EXPECT_NONFATAL_FAILURE({
+    EXPECT_EQ(m, n) << "Expected non-fatal failure.";
+  }, "Expected non-fatal failure.");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() succeeds when there is exactly
+// one non-fatal failure and no fatal failure.
+TEST(ExpectNonfatalFailureTest, SucceedsWhenThereIsOneNonfatalFailure) {
+  EXPECT_NONFATAL_FAILURE({
+    ADD_FAILURE() << "Expected non-fatal failure.";
+  }, "Expected non-fatal failure.");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() fails when there is no
+// non-fatal failure.
+TEST(ExpectNonfatalFailureTest, FailsWhenThereIsNoNonfatalFailure) {
+  printf("(expecting a failure)\n");
+  EXPECT_NONFATAL_FAILURE({
+  }, "");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() fails when there are two
+// non-fatal failures.
+TEST(ExpectNonfatalFailureTest, FailsWhenThereAreTwoNonfatalFailures) {
+  printf("(expecting a failure)\n");
+  EXPECT_NONFATAL_FAILURE({
+    ADD_FAILURE() << "Expected non-fatal failure 1.";
+    ADD_FAILURE() << "Expected non-fatal failure 2.";
+  }, "");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() fails when there is one fatal
+// failure.
+TEST(ExpectNonfatalFailureTest, FailsWhenThereIsOneFatalFailure) {
+  printf("(expecting a failure)\n");
+  EXPECT_NONFATAL_FAILURE({
+    FAIL() << "Expected fatal failure.";
+  }, "");
+}
+
+// Tests that EXPECT_NONFATAL_FAILURE() fails when the statement being
+// tested returns.
+TEST(ExpectNonfatalFailureTest, FailsWhenStatementReturns) {
+  printf("(expecting a failure)\n");
+  EXPECT_NONFATAL_FAILURE({
+    return;
+  }, "");
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+// Tests that EXPECT_NONFATAL_FAILURE() fails when the statement being
+// tested throws.
+TEST(ExpectNonfatalFailureTest, FailsWhenStatementThrows) {
+  printf("(expecting a failure)\n");
+  try {
+    EXPECT_NONFATAL_FAILURE({
+      throw 0;
+    }, "");
+  } catch(int) {  // NOLINT
+  }
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// Tests that EXPECT_FATAL_FAILURE() can reference global variables.
+TEST(ExpectFatalFailureTest, CanReferenceGlobalVariables) {
+  global_integer = 0;
+  EXPECT_FATAL_FAILURE({
+    ASSERT_EQ(1, global_integer) << "Expected fatal failure.";
+  }, "Expected fatal failure.");
+}
+
+// Tests that EXPECT_FATAL_FAILURE() can reference local static
+// variables.
+TEST(ExpectFatalFailureTest, CanReferenceLocalStaticVariables) {
+  static int n;
+  n = 1;
+  EXPECT_FATAL_FAILURE({
+    ASSERT_EQ(0, n) << "Expected fatal failure.";
+  }, "Expected fatal failure.");
+}
+
+// Tests that EXPECT_FATAL_FAILURE() succeeds when there is exactly
+// one fatal failure and no non-fatal failure.
+TEST(ExpectFatalFailureTest, SucceedsWhenThereIsOneFatalFailure) {
+  EXPECT_FATAL_FAILURE({
+    FAIL() << "Expected fatal failure.";
+  }, "Expected fatal failure.");
+}
+
+// Tests that EXPECT_FATAL_FAILURE() fails when there is no fatal
+// failure.
+TEST(ExpectFatalFailureTest, FailsWhenThereIsNoFatalFailure) {
+  printf("(expecting a failure)\n");
+  EXPECT_FATAL_FAILURE({
+  }, "");
+}
+
+// A helper for generating a fatal failure.
+void FatalFailure() {
+  FAIL() << "Expected fatal failure.";
+}
+
+// Tests that EXPECT_FATAL_FAILURE() fails when there are two
+// fatal failures.
+TEST(ExpectFatalFailureTest, FailsWhenThereAreTwoFatalFailures) {
+  printf("(expecting a failure)\n");
+  EXPECT_FATAL_FAILURE({
+    FatalFailure();
+    FatalFailure();
+  }, "");
+}
+
+// Tests that EXPECT_FATAL_FAILURE() fails when there is one non-fatal
+// failure.
+TEST(ExpectFatalFailureTest, FailsWhenThereIsOneNonfatalFailure) {
+  printf("(expecting a failure)\n");
+  EXPECT_FATAL_FAILURE({
+    ADD_FAILURE() << "Expected non-fatal failure.";
+  }, "");
+}
+
+// Tests that EXPECT_FATAL_FAILURE() fails when the statement being
+// tested returns.
+TEST(ExpectFatalFailureTest, FailsWhenStatementReturns) {
+  printf("(expecting a failure)\n");
+  EXPECT_FATAL_FAILURE({
+    return;
+  }, "");
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+// Tests that EXPECT_FATAL_FAILURE() fails when the statement being
+// tested throws.
+TEST(ExpectFatalFailureTest, FailsWhenStatementThrows) {
+  printf("(expecting a failure)\n");
+  try {
+    EXPECT_FATAL_FAILURE({
+      throw 0;
+    }, "");
+  } catch(int) {  // NOLINT
+  }
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// This #ifdef block tests the output of value-parameterized tests.
+
+#if GTEST_HAS_PARAM_TEST
+
+std::string ParamNameFunc(const testing::TestParamInfo<std::string>& info) {
+  return info.param;
+}
+
+class ParamTest : public testing::TestWithParam<std::string> {
+};
+
+TEST_P(ParamTest, Success) {
+  EXPECT_EQ("a", GetParam());
+}
+
+TEST_P(ParamTest, Failure) {
+  EXPECT_EQ("b", GetParam()) << "Expected failure";
+}
+
+INSTANTIATE_TEST_CASE_P(PrintingStrings,
+                        ParamTest,
+                        testing::Values(std::string("a")),
+                        ParamNameFunc);
+
+#endif  // GTEST_HAS_PARAM_TEST
+
+// This #ifdef block tests the output of typed tests.
+#if GTEST_HAS_TYPED_TEST
+
+template <typename T>
+class TypedTest : public testing::Test {
+};
+
+TYPED_TEST_CASE(TypedTest, testing::Types<int>);
+
+TYPED_TEST(TypedTest, Success) {
+  EXPECT_EQ(0, TypeParam());
+}
+
+TYPED_TEST(TypedTest, Failure) {
+  EXPECT_EQ(1, TypeParam()) << "Expected failure";
+}
+
+#endif  // GTEST_HAS_TYPED_TEST
+
+// This #ifdef block tests the output of type-parameterized tests.
+#if GTEST_HAS_TYPED_TEST_P
+
+template <typename T>
+class TypedTestP : public testing::Test {
+};
+
+TYPED_TEST_CASE_P(TypedTestP);
+
+TYPED_TEST_P(TypedTestP, Success) {
+  EXPECT_EQ(0U, TypeParam());
+}
+
+TYPED_TEST_P(TypedTestP, Failure) {
+  EXPECT_EQ(1U, TypeParam()) << "Expected failure";
+}
+
+REGISTER_TYPED_TEST_CASE_P(TypedTestP, Success, Failure);
+
+typedef testing::Types<unsigned char, unsigned int> UnsignedTypes;
+INSTANTIATE_TYPED_TEST_CASE_P(Unsigned, TypedTestP, UnsignedTypes);
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+
+#if GTEST_HAS_DEATH_TEST
+
+// We rely on the golden file to verify that tests whose test case
+// name ends with DeathTest are run first.
+
+TEST(ADeathTest, ShouldRunFirst) {
+}
+
+# if GTEST_HAS_TYPED_TEST
+
+// We rely on the golden file to verify that typed tests whose test
+// case name ends with DeathTest are run first.
+
+template <typename T>
+class ATypedDeathTest : public testing::Test {
+};
+
+typedef testing::Types<int, double> NumericTypes;
+TYPED_TEST_CASE(ATypedDeathTest, NumericTypes);
+
+TYPED_TEST(ATypedDeathTest, ShouldRunFirst) {
+}
+
+# endif  // GTEST_HAS_TYPED_TEST
+
+# if GTEST_HAS_TYPED_TEST_P
+
+
+// We rely on the golden file to verify that type-parameterized tests
+// whose test case name ends with DeathTest are run first.
+
+template <typename T>
+class ATypeParamDeathTest : public testing::Test {
+};
+
+TYPED_TEST_CASE_P(ATypeParamDeathTest);
+
+TYPED_TEST_P(ATypeParamDeathTest, ShouldRunFirst) {
+}
+
+REGISTER_TYPED_TEST_CASE_P(ATypeParamDeathTest, ShouldRunFirst);
+
+INSTANTIATE_TYPED_TEST_CASE_P(My, ATypeParamDeathTest, NumericTypes);
+
+# endif  // GTEST_HAS_TYPED_TEST_P
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+// Tests various failure conditions of
+// EXPECT_{,NON}FATAL_FAILURE{,_ON_ALL_THREADS}.
+class ExpectFailureTest : public testing::Test {
+ public:  // Must be public and not protected due to a bug in g++ 3.4.2.
+  enum FailureMode {
+    FATAL_FAILURE,
+    NONFATAL_FAILURE
+  };
+  static void AddFailure(FailureMode failure) {
+    if (failure == FATAL_FAILURE) {
+      FAIL() << "Expected fatal failure.";
+    } else {
+      ADD_FAILURE() << "Expected non-fatal failure.";
+    }
+  }
+};
+
+TEST_F(ExpectFailureTest, ExpectFatalFailure) {
+  // Expected fatal failure, but succeeds.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE(SUCCEED(), "Expected fatal failure.");
+  // Expected fatal failure, but got a non-fatal failure.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE(AddFailure(NONFATAL_FAILURE), "Expected non-fatal "
+                       "failure.");
+  // Wrong message.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE(AddFailure(FATAL_FAILURE), "Some other fatal failure "
+                       "expected.");
+}
+
+TEST_F(ExpectFailureTest, ExpectNonFatalFailure) {
+  // Expected non-fatal failure, but succeeds.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE(SUCCEED(), "Expected non-fatal failure.");
+  // Expected non-fatal failure, but got a fatal failure.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE(AddFailure(FATAL_FAILURE), "Expected fatal failure.");
+  // Wrong message.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE(AddFailure(NONFATAL_FAILURE), "Some other non-fatal "
+                          "failure.");
+}
+
+#if GTEST_IS_THREADSAFE
+
+class ExpectFailureWithThreadsTest : public ExpectFailureTest {
+ protected:
+  static void AddFailureInOtherThread(FailureMode failure) {
+    ThreadWithParam<FailureMode> thread(&AddFailure, failure, NULL);
+    thread.Join();
+  }
+};
+
+TEST_F(ExpectFailureWithThreadsTest, ExpectFatalFailure) {
+  // We only intercept the current thread.
+  printf("(expecting 2 failures)\n");
+  EXPECT_FATAL_FAILURE(AddFailureInOtherThread(FATAL_FAILURE),
+                       "Expected fatal failure.");
+}
+
+TEST_F(ExpectFailureWithThreadsTest, ExpectNonFatalFailure) {
+  // We only intercept the current thread.
+  printf("(expecting 2 failures)\n");
+  EXPECT_NONFATAL_FAILURE(AddFailureInOtherThread(NONFATAL_FAILURE),
+                          "Expected non-fatal failure.");
+}
+
+typedef ExpectFailureWithThreadsTest ScopedFakeTestPartResultReporterTest;
+
+// Tests that the ScopedFakeTestPartResultReporter only catches failures from
+// the current thread if it is instantiated with INTERCEPT_ONLY_CURRENT_THREAD.
+TEST_F(ScopedFakeTestPartResultReporterTest, InterceptOnlyCurrentThread) {
+  printf("(expecting 2 failures)\n");
+  TestPartResultArray results;
+  {
+    ScopedFakeTestPartResultReporter reporter(
+        ScopedFakeTestPartResultReporter::INTERCEPT_ONLY_CURRENT_THREAD,
+        &results);
+    AddFailureInOtherThread(FATAL_FAILURE);
+    AddFailureInOtherThread(NONFATAL_FAILURE);
+  }
+  // The two failures should not have been intercepted.
+  EXPECT_EQ(0, results.size()) << "This shouldn't fail.";
+}
+
+#endif  // GTEST_IS_THREADSAFE
+
+TEST_F(ExpectFailureTest, ExpectFatalFailureOnAllThreads) {
+  // Expected fatal failure, but succeeds.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(SUCCEED(), "Expected fatal failure.");
+  // Expected fatal failure, but got a non-fatal failure.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(AddFailure(NONFATAL_FAILURE),
+                                      "Expected non-fatal failure.");
+  // Wrong message.
+  printf("(expecting 1 failure)\n");
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(AddFailure(FATAL_FAILURE),
+                                      "Some other fatal failure expected.");
+}
+
+TEST_F(ExpectFailureTest, ExpectNonFatalFailureOnAllThreads) {
+  // Expected non-fatal failure, but succeeds.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(SUCCEED(), "Expected non-fatal "
+                                         "failure.");
+  // Expected non-fatal failure, but got a fatal failure.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(AddFailure(FATAL_FAILURE),
+                                         "Expected fatal failure.");
+  // Wrong message.
+  printf("(expecting 1 failure)\n");
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(AddFailure(NONFATAL_FAILURE),
+                                         "Some other non-fatal failure.");
+}
+
+
+// Two test environments for testing testing::AddGlobalTestEnvironment().
+
+class FooEnvironment : public testing::Environment {
+ public:
+  virtual void SetUp() {
+    printf("%s", "FooEnvironment::SetUp() called.\n");
+  }
+
+  virtual void TearDown() {
+    printf("%s", "FooEnvironment::TearDown() called.\n");
+    FAIL() << "Expected fatal failure.";
+  }
+};
+
+class BarEnvironment : public testing::Environment {
+ public:
+  virtual void SetUp() {
+    printf("%s", "BarEnvironment::SetUp() called.\n");
+  }
+
+  virtual void TearDown() {
+    printf("%s", "BarEnvironment::TearDown() called.\n");
+    ADD_FAILURE() << "Expected non-fatal failure.";
+  }
+};
+
+// The main function.
+//
+// The idea is to use Google Test to run all the tests we have defined (some
+// of them are intended to fail), and then compare the test results
+// with the "golden" file.
+int main(int argc, char **argv) {
+  testing::GTEST_FLAG(print_time) = false;
+
+  // We just run the tests, knowing some of them are intended to fail.
+  // We will use a separate Python script to compare the output of
+  // this program with the golden file.
+
+  // It's hard to test InitGoogleTest() directly, as it has many
+  // global side effects.  The following line serves as a sanity test
+  // for it.
+  testing::InitGoogleTest(&argc, argv);
+  bool internal_skip_environment_and_ad_hoc_tests =
+      std::count(argv, argv + argc,
+                 std::string("internal_skip_environment_and_ad_hoc_tests")) > 0;
+
+#if GTEST_HAS_DEATH_TEST
+  if (testing::internal::GTEST_FLAG(internal_run_death_test) != "") {
+    // Skip the usual output capturing if we're running as the child
+    // process of an threadsafe-style death test.
+# if GTEST_OS_WINDOWS
+    posix::FReopen("nul:", "w", stdout);
+# else
+    posix::FReopen("/dev/null", "w", stdout);
+# endif  // GTEST_OS_WINDOWS
+    return RUN_ALL_TESTS();
+  }
+#endif  // GTEST_HAS_DEATH_TEST
+
+  if (internal_skip_environment_and_ad_hoc_tests)
+    return RUN_ALL_TESTS();
+
+  // Registers two global test environments.
+  // The golden file verifies that they are set up in the order they
+  // are registered, and torn down in the reverse order.
+  testing::AddGlobalTestEnvironment(new FooEnvironment);
+  testing::AddGlobalTestEnvironment(new BarEnvironment);
+
+  return RunAllTests();
+}
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: eefacm@gmail.com (Sean Mcafee)
+
+// Unit test for Google Test XML output.
+//
+// A user can specify XML output in a Google Test program to run via
+// either the GTEST_OUTPUT environment variable or the --gtest_output
+// flag.  This is used for testing such functionality.
+//
+// This program will be invoked from a Python unit test.  Don't run it
+// directly.
+
+#include "gtest/gtest.h"
+
+using ::testing::InitGoogleTest;
+using ::testing::TestEventListeners;
+using ::testing::TestWithParam;
+using ::testing::UnitTest;
+using ::testing::Test;
+using ::testing::Values;
+
+class SuccessfulTest : public Test {
+};
+
+TEST_F(SuccessfulTest, Succeeds) {
+  SUCCEED() << "This is a success.";
+  ASSERT_EQ(1, 1);
+}
+
+class FailedTest : public Test {
+};
+
+TEST_F(FailedTest, Fails) {
+  ASSERT_EQ(1, 2);
+}
+
+class DisabledTest : public Test {
+};
+
+TEST_F(DisabledTest, DISABLED_test_not_run) {
+  FAIL() << "Unexpected failure: Disabled test should not be run";
+}
+
+TEST(MixedResultTest, Succeeds) {
+  EXPECT_EQ(1, 1);
+  ASSERT_EQ(1, 1);
+}
+
+TEST(MixedResultTest, Fails) {
+  EXPECT_EQ(1, 2);
+  ASSERT_EQ(2, 3);
+}
+
+TEST(MixedResultTest, DISABLED_test) {
+  FAIL() << "Unexpected failure: Disabled test should not be run";
+}
+
+TEST(XmlQuotingTest, OutputsCData) {
+  FAIL() << "XML output: "
+            "<?xml encoding=\"utf-8\"><top><![CDATA[cdata text]]></top>";
+}
+
+// Helps to test that invalid characters produced by test code do not make
+// it into the XML file.
+TEST(InvalidCharactersTest, InvalidCharactersInMessage) {
+  FAIL() << "Invalid characters in brackets [\x1\x2]";
+}
+
+class PropertyRecordingTest : public Test {
+ public:
+  static void SetUpTestCase() { RecordProperty("SetUpTestCase", "yes"); }
+  static void TearDownTestCase() { RecordProperty("TearDownTestCase", "aye"); }
+};
+
+TEST_F(PropertyRecordingTest, OneProperty) {
+  RecordProperty("key_1", "1");
+}
+
+TEST_F(PropertyRecordingTest, IntValuedProperty) {
+  RecordProperty("key_int", 1);
+}
+
+TEST_F(PropertyRecordingTest, ThreeProperties) {
+  RecordProperty("key_1", "1");
+  RecordProperty("key_2", "2");
+  RecordProperty("key_3", "3");
+}
+
+TEST_F(PropertyRecordingTest, TwoValuesForOneKeyUsesLastValue) {
+  RecordProperty("key_1", "1");
+  RecordProperty("key_1", "2");
+}
+
+TEST(NoFixtureTest, RecordProperty) {
+  RecordProperty("key", "1");
+}
+
+void ExternalUtilityThatCallsRecordProperty(const std::string& key, int value) {
+  testing::Test::RecordProperty(key, value);
+}
+
+void ExternalUtilityThatCallsRecordProperty(const std::string& key,
+                                            const std::string& value) {
+  testing::Test::RecordProperty(key, value);
+}
+
+TEST(NoFixtureTest, ExternalUtilityThatCallsRecordIntValuedProperty) {
+  ExternalUtilityThatCallsRecordProperty("key_for_utility_int", 1);
+}
+
+TEST(NoFixtureTest, ExternalUtilityThatCallsRecordStringValuedProperty) {
+  ExternalUtilityThatCallsRecordProperty("key_for_utility_string", "1");
+}
+
+// Verifies that the test parameter value is output in the 'value_param'
+// XML attribute for value-parameterized tests.
+class ValueParamTest : public TestWithParam<int> {};
+TEST_P(ValueParamTest, HasValueParamAttribute) {}
+TEST_P(ValueParamTest, AnotherTestThatHasValueParamAttribute) {}
+INSTANTIATE_TEST_CASE_P(Single, ValueParamTest, Values(33, 42));
+
+#if GTEST_HAS_TYPED_TEST
+// Verifies that the type parameter name is output in the 'type_param'
+// XML attribute for typed tests.
+template <typename T> class TypedTest : public Test {};
+typedef testing::Types<int, long> TypedTestTypes;
+TYPED_TEST_CASE(TypedTest, TypedTestTypes);
+TYPED_TEST(TypedTest, HasTypeParamAttribute) {}
+#endif
+
+#if GTEST_HAS_TYPED_TEST_P
+// Verifies that the type parameter name is output in the 'type_param'
+// XML attribute for type-parameterized tests.
+template <typename T> class TypeParameterizedTestCase : public Test {};
+TYPED_TEST_CASE_P(TypeParameterizedTestCase);
+TYPED_TEST_P(TypeParameterizedTestCase, HasTypeParamAttribute) {}
+REGISTER_TYPED_TEST_CASE_P(TypeParameterizedTestCase, HasTypeParamAttribute);
+typedef testing::Types<int, long> TypeParameterizedTestCaseTypes;
+INSTANTIATE_TYPED_TEST_CASE_P(Single,
+                              TypeParameterizedTestCase,
+                              TypeParameterizedTestCaseTypes);
+#endif
+
+int main(int argc, char** argv) {
+  InitGoogleTest(&argc, argv);
+
+  if (argc > 1 && strcmp(argv[1], "--shut_down_xml") == 0) {
+    TestEventListeners& listeners = UnitTest::GetInstance()->listeners();
+    delete listeners.Release(listeners.default_xml_generator());
+  }
+  testing::Test::RecordProperty("ad_hoc_property", "42");
+  return RUN_ALL_TESTS();
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Tests for Google Test itself.  This verifies that the basic constructs of
+// Google Test work.
+
+#include "gtest/gtest.h"
+
+// Verifies that the command line flag variables can be accessed
+// in code once <gtest/gtest.h> has been #included.
+// Do not move it after other #includes.
+TEST(CommandLineFlagsTest, CanBeAccessedInCodeOnceGTestHIsIncluded) {
+  bool dummy = testing::GTEST_FLAG(also_run_disabled_tests)
+      || testing::GTEST_FLAG(break_on_failure)
+      || testing::GTEST_FLAG(catch_exceptions)
+      || testing::GTEST_FLAG(color) != "unknown"
+      || testing::GTEST_FLAG(filter) != "unknown"
+      || testing::GTEST_FLAG(list_tests)
+      || testing::GTEST_FLAG(output) != "unknown"
+      || testing::GTEST_FLAG(print_time)
+      || testing::GTEST_FLAG(random_seed)
+      || testing::GTEST_FLAG(repeat) > 0
+      || testing::GTEST_FLAG(show_internal_stack_frames)
+      || testing::GTEST_FLAG(shuffle)
+      || testing::GTEST_FLAG(stack_trace_depth) > 0
+      || testing::GTEST_FLAG(stream_result_to) != "unknown"
+      || testing::GTEST_FLAG(throw_on_failure);
+  EXPECT_TRUE(dummy || !dummy);  // Suppresses warning that dummy is unused.
+}
+
+#include <limits.h>  // For INT_MAX.
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include <map>
+#include <vector>
+#include <ostream>
+
+#include "gtest/gtest-spi.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+namespace internal {
+
+#if GTEST_CAN_STREAM_RESULTS_
+
+class StreamingListenerTest : public Test {
+ public:
+  class FakeSocketWriter : public StreamingListener::AbstractSocketWriter {
+   public:
+    // Sends a string to the socket.
+    virtual void Send(const string& message) { output_ += message; }
+
+    string output_;
+  };
+
+  StreamingListenerTest()
+      : fake_sock_writer_(new FakeSocketWriter),
+        streamer_(fake_sock_writer_),
+        test_info_obj_("FooTest", "Bar", NULL, NULL,
+                       CodeLocation(__FILE__, __LINE__), 0, NULL) {}
+
+ protected:
+  string* output() { return &(fake_sock_writer_->output_); }
+
+  FakeSocketWriter* const fake_sock_writer_;
+  StreamingListener streamer_;
+  UnitTest unit_test_;
+  TestInfo test_info_obj_;  // The name test_info_ was taken by testing::Test.
+};
+
+TEST_F(StreamingListenerTest, OnTestProgramEnd) {
+  *output() = "";
+  streamer_.OnTestProgramEnd(unit_test_);
+  EXPECT_EQ("event=TestProgramEnd&passed=1\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestIterationEnd) {
+  *output() = "";
+  streamer_.OnTestIterationEnd(unit_test_, 42);
+  EXPECT_EQ("event=TestIterationEnd&passed=1&elapsed_time=0ms\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestCaseStart) {
+  *output() = "";
+  streamer_.OnTestCaseStart(TestCase("FooTest", "Bar", NULL, NULL));
+  EXPECT_EQ("event=TestCaseStart&name=FooTest\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestCaseEnd) {
+  *output() = "";
+  streamer_.OnTestCaseEnd(TestCase("FooTest", "Bar", NULL, NULL));
+  EXPECT_EQ("event=TestCaseEnd&passed=1&elapsed_time=0ms\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestStart) {
+  *output() = "";
+  streamer_.OnTestStart(test_info_obj_);
+  EXPECT_EQ("event=TestStart&name=Bar\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestEnd) {
+  *output() = "";
+  streamer_.OnTestEnd(test_info_obj_);
+  EXPECT_EQ("event=TestEnd&passed=1&elapsed_time=0ms\n", *output());
+}
+
+TEST_F(StreamingListenerTest, OnTestPartResult) {
+  *output() = "";
+  streamer_.OnTestPartResult(TestPartResult(
+      TestPartResult::kFatalFailure, "foo.cc", 42, "failed=\n&%"));
+
+  // Meta characters in the failure message should be properly escaped.
+  EXPECT_EQ(
+      "event=TestPartResult&file=foo.cc&line=42&message=failed%3D%0A%26%25\n",
+      *output());
+}
+
+#endif  // GTEST_CAN_STREAM_RESULTS_
+
+// Provides access to otherwise private parts of the TestEventListeners class
+// that are needed to test it.
+class TestEventListenersAccessor {
+ public:
+  static TestEventListener* GetRepeater(TestEventListeners* listeners) {
+    return listeners->repeater();
+  }
+
+  static void SetDefaultResultPrinter(TestEventListeners* listeners,
+                                      TestEventListener* listener) {
+    listeners->SetDefaultResultPrinter(listener);
+  }
+  static void SetDefaultXmlGenerator(TestEventListeners* listeners,
+                                     TestEventListener* listener) {
+    listeners->SetDefaultXmlGenerator(listener);
+  }
+
+  static bool EventForwardingEnabled(const TestEventListeners& listeners) {
+    return listeners.EventForwardingEnabled();
+  }
+
+  static void SuppressEventForwarding(TestEventListeners* listeners) {
+    listeners->SuppressEventForwarding();
+  }
+};
+
+class UnitTestRecordPropertyTestHelper : public Test {
+ protected:
+  UnitTestRecordPropertyTestHelper() {}
+
+  // Forwards to UnitTest::RecordProperty() to bypass access controls.
+  void UnitTestRecordProperty(const char* key, const std::string& value) {
+    unit_test_.RecordProperty(key, value);
+  }
+
+  UnitTest unit_test_;
+};
+
+}  // namespace internal
+}  // namespace testing
+
+using testing::AssertionFailure;
+using testing::AssertionResult;
+using testing::AssertionSuccess;
+using testing::DoubleLE;
+using testing::EmptyTestEventListener;
+using testing::Environment;
+using testing::FloatLE;
+using testing::GTEST_FLAG(also_run_disabled_tests);
+using testing::GTEST_FLAG(break_on_failure);
+using testing::GTEST_FLAG(catch_exceptions);
+using testing::GTEST_FLAG(color);
+using testing::GTEST_FLAG(death_test_use_fork);
+using testing::GTEST_FLAG(filter);
+using testing::GTEST_FLAG(list_tests);
+using testing::GTEST_FLAG(output);
+using testing::GTEST_FLAG(print_time);
+using testing::GTEST_FLAG(random_seed);
+using testing::GTEST_FLAG(repeat);
+using testing::GTEST_FLAG(show_internal_stack_frames);
+using testing::GTEST_FLAG(shuffle);
+using testing::GTEST_FLAG(stack_trace_depth);
+using testing::GTEST_FLAG(stream_result_to);
+using testing::GTEST_FLAG(throw_on_failure);
+using testing::IsNotSubstring;
+using testing::IsSubstring;
+using testing::Message;
+using testing::ScopedFakeTestPartResultReporter;
+using testing::StaticAssertTypeEq;
+using testing::Test;
+using testing::TestCase;
+using testing::TestEventListeners;
+using testing::TestInfo;
+using testing::TestPartResult;
+using testing::TestPartResultArray;
+using testing::TestProperty;
+using testing::TestResult;
+using testing::TimeInMillis;
+using testing::UnitTest;
+using testing::internal::AddReference;
+using testing::internal::AlwaysFalse;
+using testing::internal::AlwaysTrue;
+using testing::internal::AppendUserMessage;
+using testing::internal::ArrayAwareFind;
+using testing::internal::ArrayEq;
+using testing::internal::CodePointToUtf8;
+using testing::internal::CompileAssertTypesEqual;
+using testing::internal::CopyArray;
+using testing::internal::CountIf;
+using testing::internal::EqFailure;
+using testing::internal::FloatingPoint;
+using testing::internal::ForEach;
+using testing::internal::FormatEpochTimeInMillisAsIso8601;
+using testing::internal::FormatTimeInMillisAsSeconds;
+using testing::internal::GTestFlagSaver;
+using testing::internal::GetCurrentOsStackTraceExceptTop;
+using testing::internal::GetElementOr;
+using testing::internal::GetNextRandomSeed;
+using testing::internal::GetRandomSeedFromFlag;
+using testing::internal::GetTestTypeId;
+using testing::internal::GetTimeInMillis;
+using testing::internal::GetTypeId;
+using testing::internal::GetUnitTestImpl;
+using testing::internal::ImplicitlyConvertible;
+using testing::internal::Int32;
+using testing::internal::Int32FromEnvOrDie;
+using testing::internal::IsAProtocolMessage;
+using testing::internal::IsContainer;
+using testing::internal::IsContainerTest;
+using testing::internal::IsNotContainer;
+using testing::internal::NativeArray;
+using testing::internal::ParseInt32Flag;
+using testing::internal::RelationToSourceCopy;
+using testing::internal::RelationToSourceReference;
+using testing::internal::RemoveConst;
+using testing::internal::RemoveReference;
+using testing::internal::ShouldRunTestOnShard;
+using testing::internal::ShouldShard;
+using testing::internal::ShouldUseColor;
+using testing::internal::Shuffle;
+using testing::internal::ShuffleRange;
+using testing::internal::SkipPrefix;
+using testing::internal::StreamableToString;
+using testing::internal::String;
+using testing::internal::TestEventListenersAccessor;
+using testing::internal::TestResultAccessor;
+using testing::internal::UInt32;
+using testing::internal::WideStringToUtf8;
+using testing::internal::edit_distance::CalculateOptimalEdits;
+using testing::internal::edit_distance::CreateUnifiedDiff;
+using testing::internal::edit_distance::EditType;
+using testing::internal::kMaxRandomSeed;
+using testing::internal::kTestTypeIdInGoogleTest;
+using testing::kMaxStackTraceDepth;
+
+#if GTEST_HAS_STREAM_REDIRECTION
+using testing::internal::CaptureStdout;
+using testing::internal::GetCapturedStdout;
+#endif
+
+#if GTEST_IS_THREADSAFE
+using testing::internal::ThreadWithParam;
+#endif
+
+class TestingVector : public std::vector<int> {
+};
+
+::std::ostream& operator<<(::std::ostream& os,
+                           const TestingVector& vector) {
+  os << "{ ";
+  for (size_t i = 0; i < vector.size(); i++) {
+    os << vector[i] << " ";
+  }
+  os << "}";
+  return os;
+}
+
+// This line tests that we can define tests in an unnamed namespace.
+namespace {
+
+TEST(GetRandomSeedFromFlagTest, HandlesZero) {
+  const int seed = GetRandomSeedFromFlag(0);
+  EXPECT_LE(1, seed);
+  EXPECT_LE(seed, static_cast<int>(kMaxRandomSeed));
+}
+
+TEST(GetRandomSeedFromFlagTest, PreservesValidSeed) {
+  EXPECT_EQ(1, GetRandomSeedFromFlag(1));
+  EXPECT_EQ(2, GetRandomSeedFromFlag(2));
+  EXPECT_EQ(kMaxRandomSeed - 1, GetRandomSeedFromFlag(kMaxRandomSeed - 1));
+  EXPECT_EQ(static_cast<int>(kMaxRandomSeed),
+            GetRandomSeedFromFlag(kMaxRandomSeed));
+}
+
+TEST(GetRandomSeedFromFlagTest, NormalizesInvalidSeed) {
+  const int seed1 = GetRandomSeedFromFlag(-1);
+  EXPECT_LE(1, seed1);
+  EXPECT_LE(seed1, static_cast<int>(kMaxRandomSeed));
+
+  const int seed2 = GetRandomSeedFromFlag(kMaxRandomSeed + 1);
+  EXPECT_LE(1, seed2);
+  EXPECT_LE(seed2, static_cast<int>(kMaxRandomSeed));
+}
+
+TEST(GetNextRandomSeedTest, WorksForValidInput) {
+  EXPECT_EQ(2, GetNextRandomSeed(1));
+  EXPECT_EQ(3, GetNextRandomSeed(2));
+  EXPECT_EQ(static_cast<int>(kMaxRandomSeed),
+            GetNextRandomSeed(kMaxRandomSeed - 1));
+  EXPECT_EQ(1, GetNextRandomSeed(kMaxRandomSeed));
+
+  // We deliberately don't test GetNextRandomSeed() with invalid
+  // inputs, as that requires death tests, which are expensive.  This
+  // is fine as GetNextRandomSeed() is internal and has a
+  // straightforward definition.
+}
+
+static void ClearCurrentTestPartResults() {
+  TestResultAccessor::ClearTestPartResults(
+      GetUnitTestImpl()->current_test_result());
+}
+
+// Tests GetTypeId.
+
+TEST(GetTypeIdTest, ReturnsSameValueForSameType) {
+  EXPECT_EQ(GetTypeId<int>(), GetTypeId<int>());
+  EXPECT_EQ(GetTypeId<Test>(), GetTypeId<Test>());
+}
+
+class SubClassOfTest : public Test {};
+class AnotherSubClassOfTest : public Test {};
+
+TEST(GetTypeIdTest, ReturnsDifferentValuesForDifferentTypes) {
+  EXPECT_NE(GetTypeId<int>(), GetTypeId<const int>());
+  EXPECT_NE(GetTypeId<int>(), GetTypeId<char>());
+  EXPECT_NE(GetTypeId<int>(), GetTestTypeId());
+  EXPECT_NE(GetTypeId<SubClassOfTest>(), GetTestTypeId());
+  EXPECT_NE(GetTypeId<AnotherSubClassOfTest>(), GetTestTypeId());
+  EXPECT_NE(GetTypeId<AnotherSubClassOfTest>(), GetTypeId<SubClassOfTest>());
+}
+
+// Verifies that GetTestTypeId() returns the same value, no matter it
+// is called from inside Google Test or outside of it.
+TEST(GetTestTypeIdTest, ReturnsTheSameValueInsideOrOutsideOfGoogleTest) {
+  EXPECT_EQ(kTestTypeIdInGoogleTest, GetTestTypeId());
+}
+
+// Tests FormatTimeInMillisAsSeconds().
+
+TEST(FormatTimeInMillisAsSecondsTest, FormatsZero) {
+  EXPECT_EQ("0", FormatTimeInMillisAsSeconds(0));
+}
+
+TEST(FormatTimeInMillisAsSecondsTest, FormatsPositiveNumber) {
+  EXPECT_EQ("0.003", FormatTimeInMillisAsSeconds(3));
+  EXPECT_EQ("0.01", FormatTimeInMillisAsSeconds(10));
+  EXPECT_EQ("0.2", FormatTimeInMillisAsSeconds(200));
+  EXPECT_EQ("1.2", FormatTimeInMillisAsSeconds(1200));
+  EXPECT_EQ("3", FormatTimeInMillisAsSeconds(3000));
+}
+
+TEST(FormatTimeInMillisAsSecondsTest, FormatsNegativeNumber) {
+  EXPECT_EQ("-0.003", FormatTimeInMillisAsSeconds(-3));
+  EXPECT_EQ("-0.01", FormatTimeInMillisAsSeconds(-10));
+  EXPECT_EQ("-0.2", FormatTimeInMillisAsSeconds(-200));
+  EXPECT_EQ("-1.2", FormatTimeInMillisAsSeconds(-1200));
+  EXPECT_EQ("-3", FormatTimeInMillisAsSeconds(-3000));
+}
+
+// Tests FormatEpochTimeInMillisAsIso8601().  The correctness of conversion
+// for particular dates below was verified in Python using
+// datetime.datetime.fromutctimestamp(<timetamp>/1000).
+
+// FormatEpochTimeInMillisAsIso8601 depends on the current timezone, so we
+// have to set up a particular timezone to obtain predictable results.
+class FormatEpochTimeInMillisAsIso8601Test : public Test {
+ public:
+  // On Cygwin, GCC doesn't allow unqualified integer literals to exceed
+  // 32 bits, even when 64-bit integer types are available.  We have to
+  // force the constants to have a 64-bit type here.
+  static const TimeInMillis kMillisPerSec = 1000;
+
+ private:
+  virtual void SetUp() {
+    saved_tz_ = NULL;
+
+    GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* getenv, strdup: deprecated */)
+    if (getenv("TZ"))
+      saved_tz_ = strdup(getenv("TZ"));
+    GTEST_DISABLE_MSC_WARNINGS_POP_()
+
+    // Set up the time zone for FormatEpochTimeInMillisAsIso8601 to use.  We
+    // cannot use the local time zone because the function's output depends
+    // on the time zone.
+    SetTimeZone("UTC+00");
+  }
+
+  virtual void TearDown() {
+    SetTimeZone(saved_tz_);
+    free(const_cast<char*>(saved_tz_));
+    saved_tz_ = NULL;
+  }
+
+  static void SetTimeZone(const char* time_zone) {
+    // tzset() distinguishes between the TZ variable being present and empty
+    // and not being present, so we have to consider the case of time_zone
+    // being NULL.
+#if _MSC_VER
+    // ...Unless it's MSVC, whose standard library's _putenv doesn't
+    // distinguish between an empty and a missing variable.
+    const std::string env_var =
+        std::string("TZ=") + (time_zone ? time_zone : "");
+    _putenv(env_var.c_str());
+    GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* deprecated function */)
+    tzset();
+    GTEST_DISABLE_MSC_WARNINGS_POP_()
+#else
+    if (time_zone) {
+      setenv(("TZ"), time_zone, 1);
+    } else {
+      unsetenv("TZ");
+    }
+    tzset();
+#endif
+  }
+
+  const char* saved_tz_;
+};
+
+const TimeInMillis FormatEpochTimeInMillisAsIso8601Test::kMillisPerSec;
+
+TEST_F(FormatEpochTimeInMillisAsIso8601Test, PrintsTwoDigitSegments) {
+  EXPECT_EQ("2011-10-31T18:52:42",
+            FormatEpochTimeInMillisAsIso8601(1320087162 * kMillisPerSec));
+}
+
+TEST_F(FormatEpochTimeInMillisAsIso8601Test, MillisecondsDoNotAffectResult) {
+  EXPECT_EQ(
+      "2011-10-31T18:52:42",
+      FormatEpochTimeInMillisAsIso8601(1320087162 * kMillisPerSec + 234));
+}
+
+TEST_F(FormatEpochTimeInMillisAsIso8601Test, PrintsLeadingZeroes) {
+  EXPECT_EQ("2011-09-03T05:07:02",
+            FormatEpochTimeInMillisAsIso8601(1315026422 * kMillisPerSec));
+}
+
+TEST_F(FormatEpochTimeInMillisAsIso8601Test, Prints24HourTime) {
+  EXPECT_EQ("2011-09-28T17:08:22",
+            FormatEpochTimeInMillisAsIso8601(1317229702 * kMillisPerSec));
+}
+
+TEST_F(FormatEpochTimeInMillisAsIso8601Test, PrintsEpochStart) {
+  EXPECT_EQ("1970-01-01T00:00:00", FormatEpochTimeInMillisAsIso8601(0));
+}
+
+#if GTEST_CAN_COMPARE_NULL
+
+# ifdef __BORLANDC__
+// Silences warnings: "Condition is always true", "Unreachable code"
+#  pragma option push -w-ccc -w-rch
+# endif
+
+// Tests that GTEST_IS_NULL_LITERAL_(x) is true when x is a null
+// pointer literal.
+TEST(NullLiteralTest, IsTrueForNullLiterals) {
+  EXPECT_TRUE(GTEST_IS_NULL_LITERAL_(NULL));
+  EXPECT_TRUE(GTEST_IS_NULL_LITERAL_(0));
+  EXPECT_TRUE(GTEST_IS_NULL_LITERAL_(0U));
+  EXPECT_TRUE(GTEST_IS_NULL_LITERAL_(0L));
+}
+
+// Tests that GTEST_IS_NULL_LITERAL_(x) is false when x is not a null
+// pointer literal.
+TEST(NullLiteralTest, IsFalseForNonNullLiterals) {
+  EXPECT_FALSE(GTEST_IS_NULL_LITERAL_(1));
+  EXPECT_FALSE(GTEST_IS_NULL_LITERAL_(0.0));
+  EXPECT_FALSE(GTEST_IS_NULL_LITERAL_('a'));
+  EXPECT_FALSE(GTEST_IS_NULL_LITERAL_(static_cast<void*>(NULL)));
+}
+
+# ifdef __BORLANDC__
+// Restores warnings after previous "#pragma option push" suppressed them.
+#  pragma option pop
+# endif
+
+#endif  // GTEST_CAN_COMPARE_NULL
+//
+// Tests CodePointToUtf8().
+
+// Tests that the NUL character L'\0' is encoded correctly.
+TEST(CodePointToUtf8Test, CanEncodeNul) {
+  EXPECT_EQ("", CodePointToUtf8(L'\0'));
+}
+
+// Tests that ASCII characters are encoded correctly.
+TEST(CodePointToUtf8Test, CanEncodeAscii) {
+  EXPECT_EQ("a", CodePointToUtf8(L'a'));
+  EXPECT_EQ("Z", CodePointToUtf8(L'Z'));
+  EXPECT_EQ("&", CodePointToUtf8(L'&'));
+  EXPECT_EQ("\x7F", CodePointToUtf8(L'\x7F'));
+}
+
+// Tests that Unicode code-points that have 8 to 11 bits are encoded
+// as 110xxxxx 10xxxxxx.
+TEST(CodePointToUtf8Test, CanEncode8To11Bits) {
+  // 000 1101 0011 => 110-00011 10-010011
+  EXPECT_EQ("\xC3\x93", CodePointToUtf8(L'\xD3'));
+
+  // 101 0111 0110 => 110-10101 10-110110
+  // Some compilers (e.g., GCC on MinGW) cannot handle non-ASCII codepoints
+  // in wide strings and wide chars. In order to accomodate them, we have to
+  // introduce such character constants as integers.
+  EXPECT_EQ("\xD5\xB6",
+            CodePointToUtf8(static_cast<wchar_t>(0x576)));
+}
+
+// Tests that Unicode code-points that have 12 to 16 bits are encoded
+// as 1110xxxx 10xxxxxx 10xxxxxx.
+TEST(CodePointToUtf8Test, CanEncode12To16Bits) {
+  // 0000 1000 1101 0011 => 1110-0000 10-100011 10-010011
+  EXPECT_EQ("\xE0\xA3\x93",
+            CodePointToUtf8(static_cast<wchar_t>(0x8D3)));
+
+  // 1100 0111 0100 1101 => 1110-1100 10-011101 10-001101
+  EXPECT_EQ("\xEC\x9D\x8D",
+            CodePointToUtf8(static_cast<wchar_t>(0xC74D)));
+}
+
+#if !GTEST_WIDE_STRING_USES_UTF16_
+// Tests in this group require a wchar_t to hold > 16 bits, and thus
+// are skipped on Windows, Cygwin, and Symbian, where a wchar_t is
+// 16-bit wide. This code may not compile on those systems.
+
+// Tests that Unicode code-points that have 17 to 21 bits are encoded
+// as 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx.
+TEST(CodePointToUtf8Test, CanEncode17To21Bits) {
+  // 0 0001 0000 1000 1101 0011 => 11110-000 10-010000 10-100011 10-010011
+  EXPECT_EQ("\xF0\x90\xA3\x93", CodePointToUtf8(L'\x108D3'));
+
+  // 0 0001 0000 0100 0000 0000 => 11110-000 10-010000 10-010000 10-000000
+  EXPECT_EQ("\xF0\x90\x90\x80", CodePointToUtf8(L'\x10400'));
+
+  // 1 0000 1000 0110 0011 0100 => 11110-100 10-001000 10-011000 10-110100
+  EXPECT_EQ("\xF4\x88\x98\xB4", CodePointToUtf8(L'\x108634'));
+}
+
+// Tests that encoding an invalid code-point generates the expected result.
+TEST(CodePointToUtf8Test, CanEncodeInvalidCodePoint) {
+  EXPECT_EQ("(Invalid Unicode 0x1234ABCD)", CodePointToUtf8(L'\x1234ABCD'));
+}
+
+#endif  // !GTEST_WIDE_STRING_USES_UTF16_
+
+// Tests WideStringToUtf8().
+
+// Tests that the NUL character L'\0' is encoded correctly.
+TEST(WideStringToUtf8Test, CanEncodeNul) {
+  EXPECT_STREQ("", WideStringToUtf8(L"", 0).c_str());
+  EXPECT_STREQ("", WideStringToUtf8(L"", -1).c_str());
+}
+
+// Tests that ASCII strings are encoded correctly.
+TEST(WideStringToUtf8Test, CanEncodeAscii) {
+  EXPECT_STREQ("a", WideStringToUtf8(L"a", 1).c_str());
+  EXPECT_STREQ("ab", WideStringToUtf8(L"ab", 2).c_str());
+  EXPECT_STREQ("a", WideStringToUtf8(L"a", -1).c_str());
+  EXPECT_STREQ("ab", WideStringToUtf8(L"ab", -1).c_str());
+}
+
+// Tests that Unicode code-points that have 8 to 11 bits are encoded
+// as 110xxxxx 10xxxxxx.
+TEST(WideStringToUtf8Test, CanEncode8To11Bits) {
+  // 000 1101 0011 => 110-00011 10-010011
+  EXPECT_STREQ("\xC3\x93", WideStringToUtf8(L"\xD3", 1).c_str());
+  EXPECT_STREQ("\xC3\x93", WideStringToUtf8(L"\xD3", -1).c_str());
+
+  // 101 0111 0110 => 110-10101 10-110110
+  const wchar_t s[] = { 0x576, '\0' };
+  EXPECT_STREQ("\xD5\xB6", WideStringToUtf8(s, 1).c_str());
+  EXPECT_STREQ("\xD5\xB6", WideStringToUtf8(s, -1).c_str());
+}
+
+// Tests that Unicode code-points that have 12 to 16 bits are encoded
+// as 1110xxxx 10xxxxxx 10xxxxxx.
+TEST(WideStringToUtf8Test, CanEncode12To16Bits) {
+  // 0000 1000 1101 0011 => 1110-0000 10-100011 10-010011
+  const wchar_t s1[] = { 0x8D3, '\0' };
+  EXPECT_STREQ("\xE0\xA3\x93", WideStringToUtf8(s1, 1).c_str());
+  EXPECT_STREQ("\xE0\xA3\x93", WideStringToUtf8(s1, -1).c_str());
+
+  // 1100 0111 0100 1101 => 1110-1100 10-011101 10-001101
+  const wchar_t s2[] = { 0xC74D, '\0' };
+  EXPECT_STREQ("\xEC\x9D\x8D", WideStringToUtf8(s2, 1).c_str());
+  EXPECT_STREQ("\xEC\x9D\x8D", WideStringToUtf8(s2, -1).c_str());
+}
+
+// Tests that the conversion stops when the function encounters \0 character.
+TEST(WideStringToUtf8Test, StopsOnNulCharacter) {
+  EXPECT_STREQ("ABC", WideStringToUtf8(L"ABC\0XYZ", 100).c_str());
+}
+
+// Tests that the conversion stops when the function reaches the limit
+// specified by the 'length' parameter.
+TEST(WideStringToUtf8Test, StopsWhenLengthLimitReached) {
+  EXPECT_STREQ("ABC", WideStringToUtf8(L"ABCDEF", 3).c_str());
+}
+
+#if !GTEST_WIDE_STRING_USES_UTF16_
+// Tests that Unicode code-points that have 17 to 21 bits are encoded
+// as 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx. This code may not compile
+// on the systems using UTF-16 encoding.
+TEST(WideStringToUtf8Test, CanEncode17To21Bits) {
+  // 0 0001 0000 1000 1101 0011 => 11110-000 10-010000 10-100011 10-010011
+  EXPECT_STREQ("\xF0\x90\xA3\x93", WideStringToUtf8(L"\x108D3", 1).c_str());
+  EXPECT_STREQ("\xF0\x90\xA3\x93", WideStringToUtf8(L"\x108D3", -1).c_str());
+
+  // 1 0000 1000 0110 0011 0100 => 11110-100 10-001000 10-011000 10-110100
+  EXPECT_STREQ("\xF4\x88\x98\xB4", WideStringToUtf8(L"\x108634", 1).c_str());
+  EXPECT_STREQ("\xF4\x88\x98\xB4", WideStringToUtf8(L"\x108634", -1).c_str());
+}
+
+// Tests that encoding an invalid code-point generates the expected result.
+TEST(WideStringToUtf8Test, CanEncodeInvalidCodePoint) {
+  EXPECT_STREQ("(Invalid Unicode 0xABCDFF)",
+               WideStringToUtf8(L"\xABCDFF", -1).c_str());
+}
+#else  // !GTEST_WIDE_STRING_USES_UTF16_
+// Tests that surrogate pairs are encoded correctly on the systems using
+// UTF-16 encoding in the wide strings.
+TEST(WideStringToUtf8Test, CanEncodeValidUtf16SUrrogatePairs) {
+  const wchar_t s[] = { 0xD801, 0xDC00, '\0' };
+  EXPECT_STREQ("\xF0\x90\x90\x80", WideStringToUtf8(s, -1).c_str());
+}
+
+// Tests that encoding an invalid UTF-16 surrogate pair
+// generates the expected result.
+TEST(WideStringToUtf8Test, CanEncodeInvalidUtf16SurrogatePair) {
+  // Leading surrogate is at the end of the string.
+  const wchar_t s1[] = { 0xD800, '\0' };
+  EXPECT_STREQ("\xED\xA0\x80", WideStringToUtf8(s1, -1).c_str());
+  // Leading surrogate is not followed by the trailing surrogate.
+  const wchar_t s2[] = { 0xD800, 'M', '\0' };
+  EXPECT_STREQ("\xED\xA0\x80M", WideStringToUtf8(s2, -1).c_str());
+  // Trailing surrogate appearas without a leading surrogate.
+  const wchar_t s3[] = { 0xDC00, 'P', 'Q', 'R', '\0' };
+  EXPECT_STREQ("\xED\xB0\x80PQR", WideStringToUtf8(s3, -1).c_str());
+}
+#endif  // !GTEST_WIDE_STRING_USES_UTF16_
+
+// Tests that codepoint concatenation works correctly.
+#if !GTEST_WIDE_STRING_USES_UTF16_
+TEST(WideStringToUtf8Test, ConcatenatesCodepointsCorrectly) {
+  const wchar_t s[] = { 0x108634, 0xC74D, '\n', 0x576, 0x8D3, 0x108634, '\0'};
+  EXPECT_STREQ(
+      "\xF4\x88\x98\xB4"
+          "\xEC\x9D\x8D"
+          "\n"
+          "\xD5\xB6"
+          "\xE0\xA3\x93"
+          "\xF4\x88\x98\xB4",
+      WideStringToUtf8(s, -1).c_str());
+}
+#else
+TEST(WideStringToUtf8Test, ConcatenatesCodepointsCorrectly) {
+  const wchar_t s[] = { 0xC74D, '\n', 0x576, 0x8D3, '\0'};
+  EXPECT_STREQ(
+      "\xEC\x9D\x8D" "\n" "\xD5\xB6" "\xE0\xA3\x93",
+      WideStringToUtf8(s, -1).c_str());
+}
+#endif  // !GTEST_WIDE_STRING_USES_UTF16_
+
+// Tests the Random class.
+
+TEST(RandomDeathTest, GeneratesCrashesOnInvalidRange) {
+  testing::internal::Random random(42);
+  EXPECT_DEATH_IF_SUPPORTED(
+      random.Generate(0),
+      "Cannot generate a number in the range \\[0, 0\\)");
+  EXPECT_DEATH_IF_SUPPORTED(
+      random.Generate(testing::internal::Random::kMaxRange + 1),
+      "Generation of a number in \\[0, 2147483649\\) was requested, "
+      "but this can only generate numbers in \\[0, 2147483648\\)");
+}
+
+TEST(RandomTest, GeneratesNumbersWithinRange) {
+  const UInt32 kRange = 10000;
+  testing::internal::Random random(12345);
+  for (int i = 0; i < 10; i++) {
+    EXPECT_LT(random.Generate(kRange), kRange) << " for iteration " << i;
+  }
+
+  testing::internal::Random random2(testing::internal::Random::kMaxRange);
+  for (int i = 0; i < 10; i++) {
+    EXPECT_LT(random2.Generate(kRange), kRange) << " for iteration " << i;
+  }
+}
+
+TEST(RandomTest, RepeatsWhenReseeded) {
+  const int kSeed = 123;
+  const int kArraySize = 10;
+  const UInt32 kRange = 10000;
+  UInt32 values[kArraySize];
+
+  testing::internal::Random random(kSeed);
+  for (int i = 0; i < kArraySize; i++) {
+    values[i] = random.Generate(kRange);
+  }
+
+  random.Reseed(kSeed);
+  for (int i = 0; i < kArraySize; i++) {
+    EXPECT_EQ(values[i], random.Generate(kRange)) << " for iteration " << i;
+  }
+}
+
+// Tests STL container utilities.
+
+// Tests CountIf().
+
+static bool IsPositive(int n) { return n > 0; }
+
+TEST(ContainerUtilityTest, CountIf) {
+  std::vector<int> v;
+  EXPECT_EQ(0, CountIf(v, IsPositive));  // Works for an empty container.
+
+  v.push_back(-1);
+  v.push_back(0);
+  EXPECT_EQ(0, CountIf(v, IsPositive));  // Works when no value satisfies.
+
+  v.push_back(2);
+  v.push_back(-10);
+  v.push_back(10);
+  EXPECT_EQ(2, CountIf(v, IsPositive));
+}
+
+// Tests ForEach().
+
+static int g_sum = 0;
+static void Accumulate(int n) { g_sum += n; }
+
+TEST(ContainerUtilityTest, ForEach) {
+  std::vector<int> v;
+  g_sum = 0;
+  ForEach(v, Accumulate);
+  EXPECT_EQ(0, g_sum);  // Works for an empty container;
+
+  g_sum = 0;
+  v.push_back(1);
+  ForEach(v, Accumulate);
+  EXPECT_EQ(1, g_sum);  // Works for a container with one element.
+
+  g_sum = 0;
+  v.push_back(20);
+  v.push_back(300);
+  ForEach(v, Accumulate);
+  EXPECT_EQ(321, g_sum);
+}
+
+// Tests GetElementOr().
+TEST(ContainerUtilityTest, GetElementOr) {
+  std::vector<char> a;
+  EXPECT_EQ('x', GetElementOr(a, 0, 'x'));
+
+  a.push_back('a');
+  a.push_back('b');
+  EXPECT_EQ('a', GetElementOr(a, 0, 'x'));
+  EXPECT_EQ('b', GetElementOr(a, 1, 'x'));
+  EXPECT_EQ('x', GetElementOr(a, -2, 'x'));
+  EXPECT_EQ('x', GetElementOr(a, 2, 'x'));
+}
+
+TEST(ContainerUtilityDeathTest, ShuffleRange) {
+  std::vector<int> a;
+  a.push_back(0);
+  a.push_back(1);
+  a.push_back(2);
+  testing::internal::Random random(1);
+
+  EXPECT_DEATH_IF_SUPPORTED(
+      ShuffleRange(&random, -1, 1, &a),
+      "Invalid shuffle range start -1: must be in range \\[0, 3\\]");
+  EXPECT_DEATH_IF_SUPPORTED(
+      ShuffleRange(&random, 4, 4, &a),
+      "Invalid shuffle range start 4: must be in range \\[0, 3\\]");
+  EXPECT_DEATH_IF_SUPPORTED(
+      ShuffleRange(&random, 3, 2, &a),
+      "Invalid shuffle range finish 2: must be in range \\[3, 3\\]");
+  EXPECT_DEATH_IF_SUPPORTED(
+      ShuffleRange(&random, 3, 4, &a),
+      "Invalid shuffle range finish 4: must be in range \\[3, 3\\]");
+}
+
+class VectorShuffleTest : public Test {
+ protected:
+  static const int kVectorSize = 20;
+
+  VectorShuffleTest() : random_(1) {
+    for (int i = 0; i < kVectorSize; i++) {
+      vector_.push_back(i);
+    }
+  }
+
+  static bool VectorIsCorrupt(const TestingVector& vector) {
+    if (kVectorSize != static_cast<int>(vector.size())) {
+      return true;
+    }
+
+    bool found_in_vector[kVectorSize] = { false };
+    for (size_t i = 0; i < vector.size(); i++) {
+      const int e = vector[i];
+      if (e < 0 || e >= kVectorSize || found_in_vector[e]) {
+        return true;
+      }
+      found_in_vector[e] = true;
+    }
+
+    // Vector size is correct, elements' range is correct, no
+    // duplicate elements.  Therefore no corruption has occurred.
+    return false;
+  }
+
+  static bool VectorIsNotCorrupt(const TestingVector& vector) {
+    return !VectorIsCorrupt(vector);
+  }
+
+  static bool RangeIsShuffled(const TestingVector& vector, int begin, int end) {
+    for (int i = begin; i < end; i++) {
+      if (i != vector[i]) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  static bool RangeIsUnshuffled(
+      const TestingVector& vector, int begin, int end) {
+    return !RangeIsShuffled(vector, begin, end);
+  }
+
+  static bool VectorIsShuffled(const TestingVector& vector) {
+    return RangeIsShuffled(vector, 0, static_cast<int>(vector.size()));
+  }
+
+  static bool VectorIsUnshuffled(const TestingVector& vector) {
+    return !VectorIsShuffled(vector);
+  }
+
+  testing::internal::Random random_;
+  TestingVector vector_;
+};  // class VectorShuffleTest
+
+const int VectorShuffleTest::kVectorSize;
+
+TEST_F(VectorShuffleTest, HandlesEmptyRange) {
+  // Tests an empty range at the beginning...
+  ShuffleRange(&random_, 0, 0, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+
+  // ...in the middle...
+  ShuffleRange(&random_, kVectorSize/2, kVectorSize/2, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+
+  // ...at the end...
+  ShuffleRange(&random_, kVectorSize - 1, kVectorSize - 1, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+
+  // ...and past the end.
+  ShuffleRange(&random_, kVectorSize, kVectorSize, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+}
+
+TEST_F(VectorShuffleTest, HandlesRangeOfSizeOne) {
+  // Tests a size one range at the beginning...
+  ShuffleRange(&random_, 0, 1, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+
+  // ...in the middle...
+  ShuffleRange(&random_, kVectorSize/2, kVectorSize/2 + 1, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+
+  // ...and at the end.
+  ShuffleRange(&random_, kVectorSize - 1, kVectorSize, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsUnshuffled, vector_);
+}
+
+// Because we use our own random number generator and a fixed seed,
+// we can guarantee that the following "random" tests will succeed.
+
+TEST_F(VectorShuffleTest, ShufflesEntireVector) {
+  Shuffle(&random_, &vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  EXPECT_FALSE(VectorIsUnshuffled(vector_)) << vector_;
+
+  // Tests the first and last elements in particular to ensure that
+  // there are no off-by-one problems in our shuffle algorithm.
+  EXPECT_NE(0, vector_[0]);
+  EXPECT_NE(kVectorSize - 1, vector_[kVectorSize - 1]);
+}
+
+TEST_F(VectorShuffleTest, ShufflesStartOfVector) {
+  const int kRangeSize = kVectorSize/2;
+
+  ShuffleRange(&random_, 0, kRangeSize, &vector_);
+
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  EXPECT_PRED3(RangeIsShuffled, vector_, 0, kRangeSize);
+  EXPECT_PRED3(RangeIsUnshuffled, vector_, kRangeSize, kVectorSize);
+}
+
+TEST_F(VectorShuffleTest, ShufflesEndOfVector) {
+  const int kRangeSize = kVectorSize / 2;
+  ShuffleRange(&random_, kRangeSize, kVectorSize, &vector_);
+
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  EXPECT_PRED3(RangeIsUnshuffled, vector_, 0, kRangeSize);
+  EXPECT_PRED3(RangeIsShuffled, vector_, kRangeSize, kVectorSize);
+}
+
+TEST_F(VectorShuffleTest, ShufflesMiddleOfVector) {
+  int kRangeSize = kVectorSize/3;
+  ShuffleRange(&random_, kRangeSize, 2*kRangeSize, &vector_);
+
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  EXPECT_PRED3(RangeIsUnshuffled, vector_, 0, kRangeSize);
+  EXPECT_PRED3(RangeIsShuffled, vector_, kRangeSize, 2*kRangeSize);
+  EXPECT_PRED3(RangeIsUnshuffled, vector_, 2*kRangeSize, kVectorSize);
+}
+
+TEST_F(VectorShuffleTest, ShufflesRepeatably) {
+  TestingVector vector2;
+  for (int i = 0; i < kVectorSize; i++) {
+    vector2.push_back(i);
+  }
+
+  random_.Reseed(1234);
+  Shuffle(&random_, &vector_);
+  random_.Reseed(1234);
+  Shuffle(&random_, &vector2);
+
+  ASSERT_PRED1(VectorIsNotCorrupt, vector_);
+  ASSERT_PRED1(VectorIsNotCorrupt, vector2);
+
+  for (int i = 0; i < kVectorSize; i++) {
+    EXPECT_EQ(vector_[i], vector2[i]) << " where i is " << i;
+  }
+}
+
+// Tests the size of the AssertHelper class.
+
+TEST(AssertHelperTest, AssertHelperIsSmall) {
+  // To avoid breaking clients that use lots of assertions in one
+  // function, we cannot grow the size of AssertHelper.
+  EXPECT_LE(sizeof(testing::internal::AssertHelper), sizeof(void*));
+}
+
+// Tests String::EndsWithCaseInsensitive().
+TEST(StringTest, EndsWithCaseInsensitive) {
+  EXPECT_TRUE(String::EndsWithCaseInsensitive("foobar", "BAR"));
+  EXPECT_TRUE(String::EndsWithCaseInsensitive("foobaR", "bar"));
+  EXPECT_TRUE(String::EndsWithCaseInsensitive("foobar", ""));
+  EXPECT_TRUE(String::EndsWithCaseInsensitive("", ""));
+
+  EXPECT_FALSE(String::EndsWithCaseInsensitive("Foobar", "foo"));
+  EXPECT_FALSE(String::EndsWithCaseInsensitive("foobar", "Foo"));
+  EXPECT_FALSE(String::EndsWithCaseInsensitive("", "foo"));
+}
+
+// C++Builder's preprocessor is buggy; it fails to expand macros that
+// appear in macro parameters after wide char literals.  Provide an alias
+// for NULL as a workaround.
+static const wchar_t* const kNull = NULL;
+
+// Tests String::CaseInsensitiveWideCStringEquals
+TEST(StringTest, CaseInsensitiveWideCStringEquals) {
+  EXPECT_TRUE(String::CaseInsensitiveWideCStringEquals(NULL, NULL));
+  EXPECT_FALSE(String::CaseInsensitiveWideCStringEquals(kNull, L""));
+  EXPECT_FALSE(String::CaseInsensitiveWideCStringEquals(L"", kNull));
+  EXPECT_FALSE(String::CaseInsensitiveWideCStringEquals(kNull, L"foobar"));
+  EXPECT_FALSE(String::CaseInsensitiveWideCStringEquals(L"foobar", kNull));
+  EXPECT_TRUE(String::CaseInsensitiveWideCStringEquals(L"foobar", L"foobar"));
+  EXPECT_TRUE(String::CaseInsensitiveWideCStringEquals(L"foobar", L"FOOBAR"));
+  EXPECT_TRUE(String::CaseInsensitiveWideCStringEquals(L"FOOBAR", L"foobar"));
+}
+
+#if GTEST_OS_WINDOWS
+
+// Tests String::ShowWideCString().
+TEST(StringTest, ShowWideCString) {
+  EXPECT_STREQ("(null)",
+               String::ShowWideCString(NULL).c_str());
+  EXPECT_STREQ("", String::ShowWideCString(L"").c_str());
+  EXPECT_STREQ("foo", String::ShowWideCString(L"foo").c_str());
+}
+
+# if GTEST_OS_WINDOWS_MOBILE
+TEST(StringTest, AnsiAndUtf16Null) {
+  EXPECT_EQ(NULL, String::AnsiToUtf16(NULL));
+  EXPECT_EQ(NULL, String::Utf16ToAnsi(NULL));
+}
+
+TEST(StringTest, AnsiAndUtf16ConvertBasic) {
+  const char* ansi = String::Utf16ToAnsi(L"str");
+  EXPECT_STREQ("str", ansi);
+  delete [] ansi;
+  const WCHAR* utf16 = String::AnsiToUtf16("str");
+  EXPECT_EQ(0, wcsncmp(L"str", utf16, 3));
+  delete [] utf16;
+}
+
+TEST(StringTest, AnsiAndUtf16ConvertPathChars) {
+  const char* ansi = String::Utf16ToAnsi(L".:\\ \"*?");
+  EXPECT_STREQ(".:\\ \"*?", ansi);
+  delete [] ansi;
+  const WCHAR* utf16 = String::AnsiToUtf16(".:\\ \"*?");
+  EXPECT_EQ(0, wcsncmp(L".:\\ \"*?", utf16, 3));
+  delete [] utf16;
+}
+# endif  // GTEST_OS_WINDOWS_MOBILE
+
+#endif  // GTEST_OS_WINDOWS
+
+// Tests TestProperty construction.
+TEST(TestPropertyTest, StringValue) {
+  TestProperty property("key", "1");
+  EXPECT_STREQ("key", property.key());
+  EXPECT_STREQ("1", property.value());
+}
+
+// Tests TestProperty replacing a value.
+TEST(TestPropertyTest, ReplaceStringValue) {
+  TestProperty property("key", "1");
+  EXPECT_STREQ("1", property.value());
+  property.SetValue("2");
+  EXPECT_STREQ("2", property.value());
+}
+
+// AddFatalFailure() and AddNonfatalFailure() must be stand-alone
+// functions (i.e. their definitions cannot be inlined at the call
+// sites), or C++Builder won't compile the code.
+static void AddFatalFailure() {
+  FAIL() << "Expected fatal failure.";
+}
+
+static void AddNonfatalFailure() {
+  ADD_FAILURE() << "Expected non-fatal failure.";
+}
+
+class ScopedFakeTestPartResultReporterTest : public Test {
+ public:  // Must be public and not protected due to a bug in g++ 3.4.2.
+  enum FailureMode {
+    FATAL_FAILURE,
+    NONFATAL_FAILURE
+  };
+  static void AddFailure(FailureMode failure) {
+    if (failure == FATAL_FAILURE) {
+      AddFatalFailure();
+    } else {
+      AddNonfatalFailure();
+    }
+  }
+};
+
+// Tests that ScopedFakeTestPartResultReporter intercepts test
+// failures.
+TEST_F(ScopedFakeTestPartResultReporterTest, InterceptsTestFailures) {
+  TestPartResultArray results;
+  {
+    ScopedFakeTestPartResultReporter reporter(
+        ScopedFakeTestPartResultReporter::INTERCEPT_ONLY_CURRENT_THREAD,
+        &results);
+    AddFailure(NONFATAL_FAILURE);
+    AddFailure(FATAL_FAILURE);
+  }
+
+  EXPECT_EQ(2, results.size());
+  EXPECT_TRUE(results.GetTestPartResult(0).nonfatally_failed());
+  EXPECT_TRUE(results.GetTestPartResult(1).fatally_failed());
+}
+
+TEST_F(ScopedFakeTestPartResultReporterTest, DeprecatedConstructor) {
+  TestPartResultArray results;
+  {
+    // Tests, that the deprecated constructor still works.
+    ScopedFakeTestPartResultReporter reporter(&results);
+    AddFailure(NONFATAL_FAILURE);
+  }
+  EXPECT_EQ(1, results.size());
+}
+
+#if GTEST_IS_THREADSAFE
+
+class ScopedFakeTestPartResultReporterWithThreadsTest
+  : public ScopedFakeTestPartResultReporterTest {
+ protected:
+  static void AddFailureInOtherThread(FailureMode failure) {
+    ThreadWithParam<FailureMode> thread(&AddFailure, failure, NULL);
+    thread.Join();
+  }
+};
+
+TEST_F(ScopedFakeTestPartResultReporterWithThreadsTest,
+       InterceptsTestFailuresInAllThreads) {
+  TestPartResultArray results;
+  {
+    ScopedFakeTestPartResultReporter reporter(
+        ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, &results);
+    AddFailure(NONFATAL_FAILURE);
+    AddFailure(FATAL_FAILURE);
+    AddFailureInOtherThread(NONFATAL_FAILURE);
+    AddFailureInOtherThread(FATAL_FAILURE);
+  }
+
+  EXPECT_EQ(4, results.size());
+  EXPECT_TRUE(results.GetTestPartResult(0).nonfatally_failed());
+  EXPECT_TRUE(results.GetTestPartResult(1).fatally_failed());
+  EXPECT_TRUE(results.GetTestPartResult(2).nonfatally_failed());
+  EXPECT_TRUE(results.GetTestPartResult(3).fatally_failed());
+}
+
+#endif  // GTEST_IS_THREADSAFE
+
+// Tests EXPECT_FATAL_FAILURE{,ON_ALL_THREADS}.  Makes sure that they
+// work even if the failure is generated in a called function rather than
+// the current context.
+
+typedef ScopedFakeTestPartResultReporterTest ExpectFatalFailureTest;
+
+TEST_F(ExpectFatalFailureTest, CatchesFatalFaliure) {
+  EXPECT_FATAL_FAILURE(AddFatalFailure(), "Expected fatal failure.");
+}
+
+#if GTEST_HAS_GLOBAL_STRING
+TEST_F(ExpectFatalFailureTest, AcceptsStringObject) {
+  EXPECT_FATAL_FAILURE(AddFatalFailure(), ::string("Expected fatal failure."));
+}
+#endif
+
+TEST_F(ExpectFatalFailureTest, AcceptsStdStringObject) {
+  EXPECT_FATAL_FAILURE(AddFatalFailure(),
+                       ::std::string("Expected fatal failure."));
+}
+
+TEST_F(ExpectFatalFailureTest, CatchesFatalFailureOnAllThreads) {
+  // We have another test below to verify that the macro catches fatal
+  // failures generated on another thread.
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(AddFatalFailure(),
+                                      "Expected fatal failure.");
+}
+
+#ifdef __BORLANDC__
+// Silences warnings: "Condition is always true"
+# pragma option push -w-ccc
+#endif
+
+// Tests that EXPECT_FATAL_FAILURE() can be used in a non-void
+// function even when the statement in it contains ASSERT_*.
+
+int NonVoidFunction() {
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(false), "");
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(FAIL(), "");
+  return 0;
+}
+
+TEST_F(ExpectFatalFailureTest, CanBeUsedInNonVoidFunction) {
+  NonVoidFunction();
+}
+
+// Tests that EXPECT_FATAL_FAILURE(statement, ...) doesn't abort the
+// current function even though 'statement' generates a fatal failure.
+
+void DoesNotAbortHelper(bool* aborted) {
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(false), "");
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(FAIL(), "");
+
+  *aborted = false;
+}
+
+#ifdef __BORLANDC__
+// Restores warnings after previous "#pragma option push" suppressed them.
+# pragma option pop
+#endif
+
+TEST_F(ExpectFatalFailureTest, DoesNotAbort) {
+  bool aborted = true;
+  DoesNotAbortHelper(&aborted);
+  EXPECT_FALSE(aborted);
+}
+
+// Tests that the EXPECT_FATAL_FAILURE{,_ON_ALL_THREADS} accepts a
+// statement that contains a macro which expands to code containing an
+// unprotected comma.
+
+static int global_var = 0;
+#define GTEST_USE_UNPROTECTED_COMMA_ global_var++, global_var++
+
+TEST_F(ExpectFatalFailureTest, AcceptsMacroThatExpandsToUnprotectedComma) {
+#ifndef __BORLANDC__
+  // ICE's in C++Builder.
+  EXPECT_FATAL_FAILURE({
+    GTEST_USE_UNPROTECTED_COMMA_;
+    AddFatalFailure();
+  }, "");
+#endif
+
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS({
+    GTEST_USE_UNPROTECTED_COMMA_;
+    AddFatalFailure();
+  }, "");
+}
+
+// Tests EXPECT_NONFATAL_FAILURE{,ON_ALL_THREADS}.
+
+typedef ScopedFakeTestPartResultReporterTest ExpectNonfatalFailureTest;
+
+TEST_F(ExpectNonfatalFailureTest, CatchesNonfatalFailure) {
+  EXPECT_NONFATAL_FAILURE(AddNonfatalFailure(),
+                          "Expected non-fatal failure.");
+}
+
+#if GTEST_HAS_GLOBAL_STRING
+TEST_F(ExpectNonfatalFailureTest, AcceptsStringObject) {
+  EXPECT_NONFATAL_FAILURE(AddNonfatalFailure(),
+                          ::string("Expected non-fatal failure."));
+}
+#endif
+
+TEST_F(ExpectNonfatalFailureTest, AcceptsStdStringObject) {
+  EXPECT_NONFATAL_FAILURE(AddNonfatalFailure(),
+                          ::std::string("Expected non-fatal failure."));
+}
+
+TEST_F(ExpectNonfatalFailureTest, CatchesNonfatalFailureOnAllThreads) {
+  // We have another test below to verify that the macro catches
+  // non-fatal failures generated on another thread.
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(AddNonfatalFailure(),
+                                         "Expected non-fatal failure.");
+}
+
+// Tests that the EXPECT_NONFATAL_FAILURE{,_ON_ALL_THREADS} accepts a
+// statement that contains a macro which expands to code containing an
+// unprotected comma.
+TEST_F(ExpectNonfatalFailureTest, AcceptsMacroThatExpandsToUnprotectedComma) {
+  EXPECT_NONFATAL_FAILURE({
+    GTEST_USE_UNPROTECTED_COMMA_;
+    AddNonfatalFailure();
+  }, "");
+
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS({
+    GTEST_USE_UNPROTECTED_COMMA_;
+    AddNonfatalFailure();
+  }, "");
+}
+
+#if GTEST_IS_THREADSAFE
+
+typedef ScopedFakeTestPartResultReporterWithThreadsTest
+    ExpectFailureWithThreadsTest;
+
+TEST_F(ExpectFailureWithThreadsTest, ExpectFatalFailureOnAllThreads) {
+  EXPECT_FATAL_FAILURE_ON_ALL_THREADS(AddFailureInOtherThread(FATAL_FAILURE),
+                                      "Expected fatal failure.");
+}
+
+TEST_F(ExpectFailureWithThreadsTest, ExpectNonFatalFailureOnAllThreads) {
+  EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(
+      AddFailureInOtherThread(NONFATAL_FAILURE), "Expected non-fatal failure.");
+}
+
+#endif  // GTEST_IS_THREADSAFE
+
+// Tests the TestProperty class.
+
+TEST(TestPropertyTest, ConstructorWorks) {
+  const TestProperty property("key", "value");
+  EXPECT_STREQ("key", property.key());
+  EXPECT_STREQ("value", property.value());
+}
+
+TEST(TestPropertyTest, SetValue) {
+  TestProperty property("key", "value_1");
+  EXPECT_STREQ("key", property.key());
+  property.SetValue("value_2");
+  EXPECT_STREQ("key", property.key());
+  EXPECT_STREQ("value_2", property.value());
+}
+
+// Tests the TestResult class
+
+// The test fixture for testing TestResult.
+class TestResultTest : public Test {
+ protected:
+  typedef std::vector<TestPartResult> TPRVector;
+
+  // We make use of 2 TestPartResult objects,
+  TestPartResult * pr1, * pr2;
+
+  // ... and 3 TestResult objects.
+  TestResult * r0, * r1, * r2;
+
+  virtual void SetUp() {
+    // pr1 is for success.
+    pr1 = new TestPartResult(TestPartResult::kSuccess,
+                             "foo/bar.cc",
+                             10,
+                             "Success!");
+
+    // pr2 is for fatal failure.
+    pr2 = new TestPartResult(TestPartResult::kFatalFailure,
+                             "foo/bar.cc",
+                             -1,  // This line number means "unknown"
+                             "Failure!");
+
+    // Creates the TestResult objects.
+    r0 = new TestResult();
+    r1 = new TestResult();
+    r2 = new TestResult();
+
+    // In order to test TestResult, we need to modify its internal
+    // state, in particular the TestPartResult vector it holds.
+    // test_part_results() returns a const reference to this vector.
+    // We cast it to a non-const object s.t. it can be modified (yes,
+    // this is a hack).
+    TPRVector* results1 = const_cast<TPRVector*>(
+        &TestResultAccessor::test_part_results(*r1));
+    TPRVector* results2 = const_cast<TPRVector*>(
+        &TestResultAccessor::test_part_results(*r2));
+
+    // r0 is an empty TestResult.
+
+    // r1 contains a single SUCCESS TestPartResult.
+    results1->push_back(*pr1);
+
+    // r2 contains a SUCCESS, and a FAILURE.
+    results2->push_back(*pr1);
+    results2->push_back(*pr2);
+  }
+
+  virtual void TearDown() {
+    delete pr1;
+    delete pr2;
+
+    delete r0;
+    delete r1;
+    delete r2;
+  }
+
+  // Helper that compares two two TestPartResults.
+  static void CompareTestPartResult(const TestPartResult& expected,
+                                    const TestPartResult& actual) {
+    EXPECT_EQ(expected.type(), actual.type());
+    EXPECT_STREQ(expected.file_name(), actual.file_name());
+    EXPECT_EQ(expected.line_number(), actual.line_number());
+    EXPECT_STREQ(expected.summary(), actual.summary());
+    EXPECT_STREQ(expected.message(), actual.message());
+    EXPECT_EQ(expected.passed(), actual.passed());
+    EXPECT_EQ(expected.failed(), actual.failed());
+    EXPECT_EQ(expected.nonfatally_failed(), actual.nonfatally_failed());
+    EXPECT_EQ(expected.fatally_failed(), actual.fatally_failed());
+  }
+};
+
+// Tests TestResult::total_part_count().
+TEST_F(TestResultTest, total_part_count) {
+  ASSERT_EQ(0, r0->total_part_count());
+  ASSERT_EQ(1, r1->total_part_count());
+  ASSERT_EQ(2, r2->total_part_count());
+}
+
+// Tests TestResult::Passed().
+TEST_F(TestResultTest, Passed) {
+  ASSERT_TRUE(r0->Passed());
+  ASSERT_TRUE(r1->Passed());
+  ASSERT_FALSE(r2->Passed());
+}
+
+// Tests TestResult::Failed().
+TEST_F(TestResultTest, Failed) {
+  ASSERT_FALSE(r0->Failed());
+  ASSERT_FALSE(r1->Failed());
+  ASSERT_TRUE(r2->Failed());
+}
+
+// Tests TestResult::GetTestPartResult().
+
+typedef TestResultTest TestResultDeathTest;
+
+TEST_F(TestResultDeathTest, GetTestPartResult) {
+  CompareTestPartResult(*pr1, r2->GetTestPartResult(0));
+  CompareTestPartResult(*pr2, r2->GetTestPartResult(1));
+  EXPECT_DEATH_IF_SUPPORTED(r2->GetTestPartResult(2), "");
+  EXPECT_DEATH_IF_SUPPORTED(r2->GetTestPartResult(-1), "");
+}
+
+// Tests TestResult has no properties when none are added.
+TEST(TestResultPropertyTest, NoPropertiesFoundWhenNoneAreAdded) {
+  TestResult test_result;
+  ASSERT_EQ(0, test_result.test_property_count());
+}
+
+// Tests TestResult has the expected property when added.
+TEST(TestResultPropertyTest, OnePropertyFoundWhenAdded) {
+  TestResult test_result;
+  TestProperty property("key_1", "1");
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property);
+  ASSERT_EQ(1, test_result.test_property_count());
+  const TestProperty& actual_property = test_result.GetTestProperty(0);
+  EXPECT_STREQ("key_1", actual_property.key());
+  EXPECT_STREQ("1", actual_property.value());
+}
+
+// Tests TestResult has multiple properties when added.
+TEST(TestResultPropertyTest, MultiplePropertiesFoundWhenAdded) {
+  TestResult test_result;
+  TestProperty property_1("key_1", "1");
+  TestProperty property_2("key_2", "2");
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_1);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_2);
+  ASSERT_EQ(2, test_result.test_property_count());
+  const TestProperty& actual_property_1 = test_result.GetTestProperty(0);
+  EXPECT_STREQ("key_1", actual_property_1.key());
+  EXPECT_STREQ("1", actual_property_1.value());
+
+  const TestProperty& actual_property_2 = test_result.GetTestProperty(1);
+  EXPECT_STREQ("key_2", actual_property_2.key());
+  EXPECT_STREQ("2", actual_property_2.value());
+}
+
+// Tests TestResult::RecordProperty() overrides values for duplicate keys.
+TEST(TestResultPropertyTest, OverridesValuesForDuplicateKeys) {
+  TestResult test_result;
+  TestProperty property_1_1("key_1", "1");
+  TestProperty property_2_1("key_2", "2");
+  TestProperty property_1_2("key_1", "12");
+  TestProperty property_2_2("key_2", "22");
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_1_1);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_2_1);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_1_2);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_2_2);
+
+  ASSERT_EQ(2, test_result.test_property_count());
+  const TestProperty& actual_property_1 = test_result.GetTestProperty(0);
+  EXPECT_STREQ("key_1", actual_property_1.key());
+  EXPECT_STREQ("12", actual_property_1.value());
+
+  const TestProperty& actual_property_2 = test_result.GetTestProperty(1);
+  EXPECT_STREQ("key_2", actual_property_2.key());
+  EXPECT_STREQ("22", actual_property_2.value());
+}
+
+// Tests TestResult::GetTestProperty().
+TEST(TestResultPropertyTest, GetTestProperty) {
+  TestResult test_result;
+  TestProperty property_1("key_1", "1");
+  TestProperty property_2("key_2", "2");
+  TestProperty property_3("key_3", "3");
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_1);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_2);
+  TestResultAccessor::RecordProperty(&test_result, "testcase", property_3);
+
+  const TestProperty& fetched_property_1 = test_result.GetTestProperty(0);
+  const TestProperty& fetched_property_2 = test_result.GetTestProperty(1);
+  const TestProperty& fetched_property_3 = test_result.GetTestProperty(2);
+
+  EXPECT_STREQ("key_1", fetched_property_1.key());
+  EXPECT_STREQ("1", fetched_property_1.value());
+
+  EXPECT_STREQ("key_2", fetched_property_2.key());
+  EXPECT_STREQ("2", fetched_property_2.value());
+
+  EXPECT_STREQ("key_3", fetched_property_3.key());
+  EXPECT_STREQ("3", fetched_property_3.value());
+
+  EXPECT_DEATH_IF_SUPPORTED(test_result.GetTestProperty(3), "");
+  EXPECT_DEATH_IF_SUPPORTED(test_result.GetTestProperty(-1), "");
+}
+
+// Tests the Test class.
+//
+// It's difficult to test every public method of this class (we are
+// already stretching the limit of Google Test by using it to test itself!).
+// Fortunately, we don't have to do that, as we are already testing
+// the functionalities of the Test class extensively by using Google Test
+// alone.
+//
+// Therefore, this section only contains one test.
+
+// Tests that GTestFlagSaver works on Windows and Mac.
+
+class GTestFlagSaverTest : public Test {
+ protected:
+  // Saves the Google Test flags such that we can restore them later, and
+  // then sets them to their default values.  This will be called
+  // before the first test in this test case is run.
+  static void SetUpTestCase() {
+    saver_ = new GTestFlagSaver;
+
+    GTEST_FLAG(also_run_disabled_tests) = false;
+    GTEST_FLAG(break_on_failure) = false;
+    GTEST_FLAG(catch_exceptions) = false;
+    GTEST_FLAG(death_test_use_fork) = false;
+    GTEST_FLAG(color) = "auto";
+    GTEST_FLAG(filter) = "";
+    GTEST_FLAG(list_tests) = false;
+    GTEST_FLAG(output) = "";
+    GTEST_FLAG(print_time) = true;
+    GTEST_FLAG(random_seed) = 0;
+    GTEST_FLAG(repeat) = 1;
+    GTEST_FLAG(shuffle) = false;
+    GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
+    GTEST_FLAG(stream_result_to) = "";
+    GTEST_FLAG(throw_on_failure) = false;
+  }
+
+  // Restores the Google Test flags that the tests have modified.  This will
+  // be called after the last test in this test case is run.
+  static void TearDownTestCase() {
+    delete saver_;
+    saver_ = NULL;
+  }
+
+  // Verifies that the Google Test flags have their default values, and then
+  // modifies each of them.
+  void VerifyAndModifyFlags() {
+    EXPECT_FALSE(GTEST_FLAG(also_run_disabled_tests));
+    EXPECT_FALSE(GTEST_FLAG(break_on_failure));
+    EXPECT_FALSE(GTEST_FLAG(catch_exceptions));
+    EXPECT_STREQ("auto", GTEST_FLAG(color).c_str());
+    EXPECT_FALSE(GTEST_FLAG(death_test_use_fork));
+    EXPECT_STREQ("", GTEST_FLAG(filter).c_str());
+    EXPECT_FALSE(GTEST_FLAG(list_tests));
+    EXPECT_STREQ("", GTEST_FLAG(output).c_str());
+    EXPECT_TRUE(GTEST_FLAG(print_time));
+    EXPECT_EQ(0, GTEST_FLAG(random_seed));
+    EXPECT_EQ(1, GTEST_FLAG(repeat));
+    EXPECT_FALSE(GTEST_FLAG(shuffle));
+    EXPECT_EQ(kMaxStackTraceDepth, GTEST_FLAG(stack_trace_depth));
+    EXPECT_STREQ("", GTEST_FLAG(stream_result_to).c_str());
+    EXPECT_FALSE(GTEST_FLAG(throw_on_failure));
+
+    GTEST_FLAG(also_run_disabled_tests) = true;
+    GTEST_FLAG(break_on_failure) = true;
+    GTEST_FLAG(catch_exceptions) = true;
+    GTEST_FLAG(color) = "no";
+    GTEST_FLAG(death_test_use_fork) = true;
+    GTEST_FLAG(filter) = "abc";
+    GTEST_FLAG(list_tests) = true;
+    GTEST_FLAG(output) = "xml:foo.xml";
+    GTEST_FLAG(print_time) = false;
+    GTEST_FLAG(random_seed) = 1;
+    GTEST_FLAG(repeat) = 100;
+    GTEST_FLAG(shuffle) = true;
+    GTEST_FLAG(stack_trace_depth) = 1;
+    GTEST_FLAG(stream_result_to) = "localhost:1234";
+    GTEST_FLAG(throw_on_failure) = true;
+  }
+
+ private:
+  // For saving Google Test flags during this test case.
+  static GTestFlagSaver* saver_;
+};
+
+GTestFlagSaver* GTestFlagSaverTest::saver_ = NULL;
+
+// Google Test doesn't guarantee the order of tests.  The following two
+// tests are designed to work regardless of their order.
+
+// Modifies the Google Test flags in the test body.
+TEST_F(GTestFlagSaverTest, ModifyGTestFlags) {
+  VerifyAndModifyFlags();
+}
+
+// Verifies that the Google Test flags in the body of the previous test were
+// restored to their original values.
+TEST_F(GTestFlagSaverTest, VerifyGTestFlags) {
+  VerifyAndModifyFlags();
+}
+
+// Sets an environment variable with the given name to the given
+// value.  If the value argument is "", unsets the environment
+// variable.  The caller must ensure that both arguments are not NULL.
+static void SetEnv(const char* name, const char* value) {
+#if GTEST_OS_WINDOWS_MOBILE
+  // Environment variables are not supported on Windows CE.
+  return;
+#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9)
+  // C++Builder's putenv only stores a pointer to its parameter; we have to
+  // ensure that the string remains valid as long as it might be needed.
+  // We use an std::map to do so.
+  static std::map<std::string, std::string*> added_env;
+
+  // Because putenv stores a pointer to the string buffer, we can't delete the
+  // previous string (if present) until after it's replaced.
+  std::string *prev_env = NULL;
+  if (added_env.find(name) != added_env.end()) {
+    prev_env = added_env[name];
+  }
+  added_env[name] = new std::string(
+      (Message() << name << "=" << value).GetString());
+
+  // The standard signature of putenv accepts a 'char*' argument. Other
+  // implementations, like C++Builder's, accept a 'const char*'.
+  // We cast away the 'const' since that would work for both variants.
+  putenv(const_cast<char*>(added_env[name]->c_str()));
+  delete prev_env;
+#elif GTEST_OS_WINDOWS  // If we are on Windows proper.
+  _putenv((Message() << name << "=" << value).GetString().c_str());
+#else
+  if (*value == '\0') {
+    unsetenv(name);
+  } else {
+    setenv(name, value, 1);
+  }
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+#if !GTEST_OS_WINDOWS_MOBILE
+// Environment variables are not supported on Windows CE.
+
+using testing::internal::Int32FromGTestEnv;
+
+// Tests Int32FromGTestEnv().
+
+// Tests that Int32FromGTestEnv() returns the default value when the
+// environment variable is not set.
+TEST(Int32FromGTestEnvTest, ReturnsDefaultWhenVariableIsNotSet) {
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "");
+  EXPECT_EQ(10, Int32FromGTestEnv("temp", 10));
+}
+
+# if !defined(GTEST_GET_INT32_FROM_ENV_)
+
+// Tests that Int32FromGTestEnv() returns the default value when the
+// environment variable overflows as an Int32.
+TEST(Int32FromGTestEnvTest, ReturnsDefaultWhenValueOverflows) {
+  printf("(expecting 2 warnings)\n");
+
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "12345678987654321");
+  EXPECT_EQ(20, Int32FromGTestEnv("temp", 20));
+
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "-12345678987654321");
+  EXPECT_EQ(30, Int32FromGTestEnv("temp", 30));
+}
+
+// Tests that Int32FromGTestEnv() returns the default value when the
+// environment variable does not represent a valid decimal integer.
+TEST(Int32FromGTestEnvTest, ReturnsDefaultWhenValueIsInvalid) {
+  printf("(expecting 2 warnings)\n");
+
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "A1");
+  EXPECT_EQ(40, Int32FromGTestEnv("temp", 40));
+
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "12X");
+  EXPECT_EQ(50, Int32FromGTestEnv("temp", 50));
+}
+
+# endif  // !defined(GTEST_GET_INT32_FROM_ENV_)
+
+// Tests that Int32FromGTestEnv() parses and returns the value of the
+// environment variable when it represents a valid decimal integer in
+// the range of an Int32.
+TEST(Int32FromGTestEnvTest, ParsesAndReturnsValidValue) {
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "123");
+  EXPECT_EQ(123, Int32FromGTestEnv("temp", 0));
+
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "TEMP", "-321");
+  EXPECT_EQ(-321, Int32FromGTestEnv("temp", 0));
+}
+#endif  // !GTEST_OS_WINDOWS_MOBILE
+
+// Tests ParseInt32Flag().
+
+// Tests that ParseInt32Flag() returns false and doesn't change the
+// output value when the flag has wrong format
+TEST(ParseInt32FlagTest, ReturnsFalseForInvalidFlag) {
+  Int32 value = 123;
+  EXPECT_FALSE(ParseInt32Flag("--a=100", "b", &value));
+  EXPECT_EQ(123, value);
+
+  EXPECT_FALSE(ParseInt32Flag("a=100", "a", &value));
+  EXPECT_EQ(123, value);
+}
+
+// Tests that ParseInt32Flag() returns false and doesn't change the
+// output value when the flag overflows as an Int32.
+TEST(ParseInt32FlagTest, ReturnsDefaultWhenValueOverflows) {
+  printf("(expecting 2 warnings)\n");
+
+  Int32 value = 123;
+  EXPECT_FALSE(ParseInt32Flag("--abc=12345678987654321", "abc", &value));
+  EXPECT_EQ(123, value);
+
+  EXPECT_FALSE(ParseInt32Flag("--abc=-12345678987654321", "abc", &value));
+  EXPECT_EQ(123, value);
+}
+
+// Tests that ParseInt32Flag() returns false and doesn't change the
+// output value when the flag does not represent a valid decimal
+// integer.
+TEST(ParseInt32FlagTest, ReturnsDefaultWhenValueIsInvalid) {
+  printf("(expecting 2 warnings)\n");
+
+  Int32 value = 123;
+  EXPECT_FALSE(ParseInt32Flag("--abc=A1", "abc", &value));
+  EXPECT_EQ(123, value);
+
+  EXPECT_FALSE(ParseInt32Flag("--abc=12X", "abc", &value));
+  EXPECT_EQ(123, value);
+}
+
+// Tests that ParseInt32Flag() parses the value of the flag and
+// returns true when the flag represents a valid decimal integer in
+// the range of an Int32.
+TEST(ParseInt32FlagTest, ParsesAndReturnsValidValue) {
+  Int32 value = 123;
+  EXPECT_TRUE(ParseInt32Flag("--" GTEST_FLAG_PREFIX_ "abc=456", "abc", &value));
+  EXPECT_EQ(456, value);
+
+  EXPECT_TRUE(ParseInt32Flag("--" GTEST_FLAG_PREFIX_ "abc=-789",
+                             "abc", &value));
+  EXPECT_EQ(-789, value);
+}
+
+// Tests that Int32FromEnvOrDie() parses the value of the var or
+// returns the correct default.
+// Environment variables are not supported on Windows CE.
+#if !GTEST_OS_WINDOWS_MOBILE
+TEST(Int32FromEnvOrDieTest, ParsesAndReturnsValidValue) {
+  EXPECT_EQ(333, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER_ "UnsetVar", 333));
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "UnsetVar", "123");
+  EXPECT_EQ(123, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER_ "UnsetVar", 333));
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "UnsetVar", "-123");
+  EXPECT_EQ(-123, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER_ "UnsetVar", 333));
+}
+#endif  // !GTEST_OS_WINDOWS_MOBILE
+
+// Tests that Int32FromEnvOrDie() aborts with an error message
+// if the variable is not an Int32.
+TEST(Int32FromEnvOrDieDeathTest, AbortsOnFailure) {
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "VAR", "xxx");
+  EXPECT_DEATH_IF_SUPPORTED(
+      Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER_ "VAR", 123),
+      ".*");
+}
+
+// Tests that Int32FromEnvOrDie() aborts with an error message
+// if the variable cannot be represnted by an Int32.
+TEST(Int32FromEnvOrDieDeathTest, AbortsOnInt32Overflow) {
+  SetEnv(GTEST_FLAG_PREFIX_UPPER_ "VAR", "1234567891234567891234");
+  EXPECT_DEATH_IF_SUPPORTED(
+      Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER_ "VAR", 123),
+      ".*");
+}
+
+// Tests that ShouldRunTestOnShard() selects all tests
+// where there is 1 shard.
+TEST(ShouldRunTestOnShardTest, IsPartitionWhenThereIsOneShard) {
+  EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 0));
+  EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 1));
+  EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 2));
+  EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 3));
+  EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 4));
+}
+
+class ShouldShardTest : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    index_var_ = GTEST_FLAG_PREFIX_UPPER_ "INDEX";
+    total_var_ = GTEST_FLAG_PREFIX_UPPER_ "TOTAL";
+  }
+
+  virtual void TearDown() {
+    SetEnv(index_var_, "");
+    SetEnv(total_var_, "");
+  }
+
+  const char* index_var_;
+  const char* total_var_;
+};
+
+// Tests that sharding is disabled if neither of the environment variables
+// are set.
+TEST_F(ShouldShardTest, ReturnsFalseWhenNeitherEnvVarIsSet) {
+  SetEnv(index_var_, "");
+  SetEnv(total_var_, "");
+
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, false));
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, true));
+}
+
+// Tests that sharding is not enabled if total_shards  == 1.
+TEST_F(ShouldShardTest, ReturnsFalseWhenTotalShardIsOne) {
+  SetEnv(index_var_, "0");
+  SetEnv(total_var_, "1");
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, false));
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, true));
+}
+
+// Tests that sharding is enabled if total_shards > 1 and
+// we are not in a death test subprocess.
+// Environment variables are not supported on Windows CE.
+#if !GTEST_OS_WINDOWS_MOBILE
+TEST_F(ShouldShardTest, WorksWhenShardEnvVarsAreValid) {
+  SetEnv(index_var_, "4");
+  SetEnv(total_var_, "22");
+  EXPECT_TRUE(ShouldShard(total_var_, index_var_, false));
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, true));
+
+  SetEnv(index_var_, "8");
+  SetEnv(total_var_, "9");
+  EXPECT_TRUE(ShouldShard(total_var_, index_var_, false));
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, true));
+
+  SetEnv(index_var_, "0");
+  SetEnv(total_var_, "9");
+  EXPECT_TRUE(ShouldShard(total_var_, index_var_, false));
+  EXPECT_FALSE(ShouldShard(total_var_, index_var_, true));
+}
+#endif  // !GTEST_OS_WINDOWS_MOBILE
+
+// Tests that we exit in error if the sharding values are not valid.
+
+typedef ShouldShardTest ShouldShardDeathTest;
+
+TEST_F(ShouldShardDeathTest, AbortsWhenShardingEnvVarsAreInvalid) {
+  SetEnv(index_var_, "4");
+  SetEnv(total_var_, "4");
+  EXPECT_DEATH_IF_SUPPORTED(ShouldShard(total_var_, index_var_, false), ".*");
+
+  SetEnv(index_var_, "4");
+  SetEnv(total_var_, "-2");
+  EXPECT_DEATH_IF_SUPPORTED(ShouldShard(total_var_, index_var_, false), ".*");
+
+  SetEnv(index_var_, "5");
+  SetEnv(total_var_, "");
+  EXPECT_DEATH_IF_SUPPORTED(ShouldShard(total_var_, index_var_, false), ".*");
+
+  SetEnv(index_var_, "");
+  SetEnv(total_var_, "5");
+  EXPECT_DEATH_IF_SUPPORTED(ShouldShard(total_var_, index_var_, false), ".*");
+}
+
+// Tests that ShouldRunTestOnShard is a partition when 5
+// shards are used.
+TEST(ShouldRunTestOnShardTest, IsPartitionWhenThereAreFiveShards) {
+  // Choose an arbitrary number of tests and shards.
+  const int num_tests = 17;
+  const int num_shards = 5;
+
+  // Check partitioning: each test should be on exactly 1 shard.
+  for (int test_id = 0; test_id < num_tests; test_id++) {
+    int prev_selected_shard_index = -1;
+    for (int shard_index = 0; shard_index < num_shards; shard_index++) {
+      if (ShouldRunTestOnShard(num_shards, shard_index, test_id)) {
+        if (prev_selected_shard_index < 0) {
+          prev_selected_shard_index = shard_index;
+        } else {
+          ADD_FAILURE() << "Shard " << prev_selected_shard_index << " and "
+            << shard_index << " are both selected to run test " << test_id;
+        }
+      }
+    }
+  }
+
+  // Check balance: This is not required by the sharding protocol, but is a
+  // desirable property for performance.
+  for (int shard_index = 0; shard_index < num_shards; shard_index++) {
+    int num_tests_on_shard = 0;
+    for (int test_id = 0; test_id < num_tests; test_id++) {
+      num_tests_on_shard +=
+        ShouldRunTestOnShard(num_shards, shard_index, test_id);
+    }
+    EXPECT_GE(num_tests_on_shard, num_tests / num_shards);
+  }
+}
+
+// For the same reason we are not explicitly testing everything in the
+// Test class, there are no separate tests for the following classes
+// (except for some trivial cases):
+//
+//   TestCase, UnitTest, UnitTestResultPrinter.
+//
+// Similarly, there are no separate tests for the following macros:
+//
+//   TEST, TEST_F, RUN_ALL_TESTS
+
+TEST(UnitTestTest, CanGetOriginalWorkingDir) {
+  ASSERT_TRUE(UnitTest::GetInstance()->original_working_dir() != NULL);
+  EXPECT_STRNE(UnitTest::GetInstance()->original_working_dir(), "");
+}
+
+TEST(UnitTestTest, ReturnsPlausibleTimestamp) {
+  EXPECT_LT(0, UnitTest::GetInstance()->start_timestamp());
+  EXPECT_LE(UnitTest::GetInstance()->start_timestamp(), GetTimeInMillis());
+}
+
+// When a property using a reserved key is supplied to this function, it
+// tests that a non-fatal failure is added, a fatal failure is not added,
+// and that the property is not recorded.
+void ExpectNonFatalFailureRecordingPropertyWithReservedKey(
+    const TestResult& test_result, const char* key) {
+  EXPECT_NONFATAL_FAILURE(Test::RecordProperty(key, "1"), "Reserved key");
+  ASSERT_EQ(0, test_result.test_property_count()) << "Property for key '" << key
+                                                  << "' recorded unexpectedly.";
+}
+
+void ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+    const char* key) {
+  const TestInfo* test_info = UnitTest::GetInstance()->current_test_info();
+  ASSERT_TRUE(test_info != NULL);
+  ExpectNonFatalFailureRecordingPropertyWithReservedKey(*test_info->result(),
+                                                        key);
+}
+
+void ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+    const char* key) {
+  const TestCase* test_case = UnitTest::GetInstance()->current_test_case();
+  ASSERT_TRUE(test_case != NULL);
+  ExpectNonFatalFailureRecordingPropertyWithReservedKey(
+      test_case->ad_hoc_test_result(), key);
+}
+
+void ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+    const char* key) {
+  ExpectNonFatalFailureRecordingPropertyWithReservedKey(
+      UnitTest::GetInstance()->ad_hoc_test_result(), key);
+}
+
+// Tests that property recording functions in UnitTest outside of tests
+// functions correcly.  Creating a separate instance of UnitTest ensures it
+// is in a state similar to the UnitTest's singleton's between tests.
+class UnitTestRecordPropertyTest :
+    public testing::internal::UnitTestRecordPropertyTestHelper {
+ public:
+  static void SetUpTestCase() {
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "disabled");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "errors");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "failures");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "name");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "tests");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTestCase(
+        "time");
+
+    Test::RecordProperty("test_case_key_1", "1");
+    const TestCase* test_case = UnitTest::GetInstance()->current_test_case();
+    ASSERT_TRUE(test_case != NULL);
+
+    ASSERT_EQ(1, test_case->ad_hoc_test_result().test_property_count());
+    EXPECT_STREQ("test_case_key_1",
+                 test_case->ad_hoc_test_result().GetTestProperty(0).key());
+    EXPECT_STREQ("1",
+                 test_case->ad_hoc_test_result().GetTestProperty(0).value());
+  }
+};
+
+// Tests TestResult has the expected property when added.
+TEST_F(UnitTestRecordPropertyTest, OnePropertyFoundWhenAdded) {
+  UnitTestRecordProperty("key_1", "1");
+
+  ASSERT_EQ(1, unit_test_.ad_hoc_test_result().test_property_count());
+
+  EXPECT_STREQ("key_1",
+               unit_test_.ad_hoc_test_result().GetTestProperty(0).key());
+  EXPECT_STREQ("1",
+               unit_test_.ad_hoc_test_result().GetTestProperty(0).value());
+}
+
+// Tests TestResult has multiple properties when added.
+TEST_F(UnitTestRecordPropertyTest, MultiplePropertiesFoundWhenAdded) {
+  UnitTestRecordProperty("key_1", "1");
+  UnitTestRecordProperty("key_2", "2");
+
+  ASSERT_EQ(2, unit_test_.ad_hoc_test_result().test_property_count());
+
+  EXPECT_STREQ("key_1",
+               unit_test_.ad_hoc_test_result().GetTestProperty(0).key());
+  EXPECT_STREQ("1", unit_test_.ad_hoc_test_result().GetTestProperty(0).value());
+
+  EXPECT_STREQ("key_2",
+               unit_test_.ad_hoc_test_result().GetTestProperty(1).key());
+  EXPECT_STREQ("2", unit_test_.ad_hoc_test_result().GetTestProperty(1).value());
+}
+
+// Tests TestResult::RecordProperty() overrides values for duplicate keys.
+TEST_F(UnitTestRecordPropertyTest, OverridesValuesForDuplicateKeys) {
+  UnitTestRecordProperty("key_1", "1");
+  UnitTestRecordProperty("key_2", "2");
+  UnitTestRecordProperty("key_1", "12");
+  UnitTestRecordProperty("key_2", "22");
+
+  ASSERT_EQ(2, unit_test_.ad_hoc_test_result().test_property_count());
+
+  EXPECT_STREQ("key_1",
+               unit_test_.ad_hoc_test_result().GetTestProperty(0).key());
+  EXPECT_STREQ("12",
+               unit_test_.ad_hoc_test_result().GetTestProperty(0).value());
+
+  EXPECT_STREQ("key_2",
+               unit_test_.ad_hoc_test_result().GetTestProperty(1).key());
+  EXPECT_STREQ("22",
+               unit_test_.ad_hoc_test_result().GetTestProperty(1).value());
+}
+
+TEST_F(UnitTestRecordPropertyTest,
+       AddFailureInsideTestsWhenUsingTestCaseReservedKeys) {
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "name");
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "value_param");
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "type_param");
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "status");
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "time");
+  ExpectNonFatalFailureRecordingPropertyWithReservedKeyForCurrentTest(
+      "classname");
+}
+
+TEST_F(UnitTestRecordPropertyTest,
+       AddRecordWithReservedKeysGeneratesCorrectPropertyList) {
+  EXPECT_NONFATAL_FAILURE(
+      Test::RecordProperty("name", "1"),
+      "'classname', 'name', 'status', 'time', 'type_param', and 'value_param'"
+      " are reserved");
+}
+
+class UnitTestRecordPropertyTestEnvironment : public Environment {
+ public:
+  virtual void TearDown() {
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "tests");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "failures");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "disabled");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "errors");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "name");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "timestamp");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "time");
+    ExpectNonFatalFailureRecordingPropertyWithReservedKeyOutsideOfTestCase(
+        "random_seed");
+  }
+};
+
+// This will test property recording outside of any test or test case.
+static Environment* record_property_env =
+    AddGlobalTestEnvironment(new UnitTestRecordPropertyTestEnvironment);
+
+// This group of tests is for predicate assertions (ASSERT_PRED*, etc)
+// of various arities.  They do not attempt to be exhaustive.  Rather,
+// view them as smoke tests that can be easily reviewed and verified.
+// A more complete set of tests for predicate assertions can be found
+// in gtest_pred_impl_unittest.cc.
+
+// First, some predicates and predicate-formatters needed by the tests.
+
+// Returns true iff the argument is an even number.
+bool IsEven(int n) {
+  return (n % 2) == 0;
+}
+
+// A functor that returns true iff the argument is an even number.
+struct IsEvenFunctor {
+  bool operator()(int n) { return IsEven(n); }
+};
+
+// A predicate-formatter function that asserts the argument is an even
+// number.
+AssertionResult AssertIsEven(const char* expr, int n) {
+  if (IsEven(n)) {
+    return AssertionSuccess();
+  }
+
+  Message msg;
+  msg << expr << " evaluates to " << n << ", which is not even.";
+  return AssertionFailure(msg);
+}
+
+// A predicate function that returns AssertionResult for use in
+// EXPECT/ASSERT_TRUE/FALSE.
+AssertionResult ResultIsEven(int n) {
+  if (IsEven(n))
+    return AssertionSuccess() << n << " is even";
+  else
+    return AssertionFailure() << n << " is odd";
+}
+
+// A predicate function that returns AssertionResult but gives no
+// explanation why it succeeds. Needed for testing that
+// EXPECT/ASSERT_FALSE handles such functions correctly.
+AssertionResult ResultIsEvenNoExplanation(int n) {
+  if (IsEven(n))
+    return AssertionSuccess();
+  else
+    return AssertionFailure() << n << " is odd";
+}
+
+// A predicate-formatter functor that asserts the argument is an even
+// number.
+struct AssertIsEvenFunctor {
+  AssertionResult operator()(const char* expr, int n) {
+    return AssertIsEven(expr, n);
+  }
+};
+
+// Returns true iff the sum of the arguments is an even number.
+bool SumIsEven2(int n1, int n2) {
+  return IsEven(n1 + n2);
+}
+
+// A functor that returns true iff the sum of the arguments is an even
+// number.
+struct SumIsEven3Functor {
+  bool operator()(int n1, int n2, int n3) {
+    return IsEven(n1 + n2 + n3);
+  }
+};
+
+// A predicate-formatter function that asserts the sum of the
+// arguments is an even number.
+AssertionResult AssertSumIsEven4(
+    const char* e1, const char* e2, const char* e3, const char* e4,
+    int n1, int n2, int n3, int n4) {
+  const int sum = n1 + n2 + n3 + n4;
+  if (IsEven(sum)) {
+    return AssertionSuccess();
+  }
+
+  Message msg;
+  msg << e1 << " + " << e2 << " + " << e3 << " + " << e4
+      << " (" << n1 << " + " << n2 << " + " << n3 << " + " << n4
+      << ") evaluates to " << sum << ", which is not even.";
+  return AssertionFailure(msg);
+}
+
+// A predicate-formatter functor that asserts the sum of the arguments
+// is an even number.
+struct AssertSumIsEven5Functor {
+  AssertionResult operator()(
+      const char* e1, const char* e2, const char* e3, const char* e4,
+      const char* e5, int n1, int n2, int n3, int n4, int n5) {
+    const int sum = n1 + n2 + n3 + n4 + n5;
+    if (IsEven(sum)) {
+      return AssertionSuccess();
+    }
+
+    Message msg;
+    msg << e1 << " + " << e2 << " + " << e3 << " + " << e4 << " + " << e5
+        << " ("
+        << n1 << " + " << n2 << " + " << n3 << " + " << n4 << " + " << n5
+        << ") evaluates to " << sum << ", which is not even.";
+    return AssertionFailure(msg);
+  }
+};
+
+
+// Tests unary predicate assertions.
+
+// Tests unary predicate assertions that don't use a custom formatter.
+TEST(Pred1Test, WithoutFormat) {
+  // Success cases.
+  EXPECT_PRED1(IsEvenFunctor(), 2) << "This failure is UNEXPECTED!";
+  ASSERT_PRED1(IsEven, 4);
+
+  // Failure cases.
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED1(IsEven, 5) << "This failure is expected.";
+  }, "This failure is expected.");
+  EXPECT_FATAL_FAILURE(ASSERT_PRED1(IsEvenFunctor(), 5),
+                       "evaluates to false");
+}
+
+// Tests unary predicate assertions that use a custom formatter.
+TEST(Pred1Test, WithFormat) {
+  // Success cases.
+  EXPECT_PRED_FORMAT1(AssertIsEven, 2);
+  ASSERT_PRED_FORMAT1(AssertIsEvenFunctor(), 4)
+    << "This failure is UNEXPECTED!";
+
+  // Failure cases.
+  const int n = 5;
+  EXPECT_NONFATAL_FAILURE(EXPECT_PRED_FORMAT1(AssertIsEvenFunctor(), n),
+                          "n evaluates to 5, which is not even.");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(AssertIsEven, 5) << "This failure is expected.";
+  }, "This failure is expected.");
+}
+
+// Tests that unary predicate assertions evaluates their arguments
+// exactly once.
+TEST(Pred1Test, SingleEvaluationOnFailure) {
+  // A success case.
+  static int n = 0;
+  EXPECT_PRED1(IsEven, n++);
+  EXPECT_EQ(1, n) << "The argument is not evaluated exactly once.";
+
+  // A failure case.
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT1(AssertIsEvenFunctor(), n++)
+        << "This failure is expected.";
+  }, "This failure is expected.");
+  EXPECT_EQ(2, n) << "The argument is not evaluated exactly once.";
+}
+
+
+// Tests predicate assertions whose arity is >= 2.
+
+// Tests predicate assertions that don't use a custom formatter.
+TEST(PredTest, WithoutFormat) {
+  // Success cases.
+  ASSERT_PRED2(SumIsEven2, 2, 4) << "This failure is UNEXPECTED!";
+  EXPECT_PRED3(SumIsEven3Functor(), 4, 6, 8);
+
+  // Failure cases.
+  const int n1 = 1;
+  const int n2 = 2;
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED2(SumIsEven2, n1, n2) << "This failure is expected.";
+  }, "This failure is expected.");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED3(SumIsEven3Functor(), 1, 2, 4);
+  }, "evaluates to false");
+}
+
+// Tests predicate assertions that use a custom formatter.
+TEST(PredTest, WithFormat) {
+  // Success cases.
+  ASSERT_PRED_FORMAT4(AssertSumIsEven4, 4, 6, 8, 10) <<
+    "This failure is UNEXPECTED!";
+  EXPECT_PRED_FORMAT5(AssertSumIsEven5Functor(), 2, 4, 6, 8, 10);
+
+  // Failure cases.
+  const int n1 = 1;
+  const int n2 = 2;
+  const int n3 = 4;
+  const int n4 = 6;
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(AssertSumIsEven4, n1, n2, n3, n4);
+  }, "evaluates to 13, which is not even.");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT5(AssertSumIsEven5Functor(), 1, 2, 4, 6, 8)
+        << "This failure is expected.";
+  }, "This failure is expected.");
+}
+
+// Tests that predicate assertions evaluates their arguments
+// exactly once.
+TEST(PredTest, SingleEvaluationOnFailure) {
+  // A success case.
+  int n1 = 0;
+  int n2 = 0;
+  EXPECT_PRED2(SumIsEven2, n1++, n2++);
+  EXPECT_EQ(1, n1) << "Argument 1 is not evaluated exactly once.";
+  EXPECT_EQ(1, n2) << "Argument 2 is not evaluated exactly once.";
+
+  // Another success case.
+  n1 = n2 = 0;
+  int n3 = 0;
+  int n4 = 0;
+  int n5 = 0;
+  ASSERT_PRED_FORMAT5(AssertSumIsEven5Functor(),
+                      n1++, n2++, n3++, n4++, n5++)
+                        << "This failure is UNEXPECTED!";
+  EXPECT_EQ(1, n1) << "Argument 1 is not evaluated exactly once.";
+  EXPECT_EQ(1, n2) << "Argument 2 is not evaluated exactly once.";
+  EXPECT_EQ(1, n3) << "Argument 3 is not evaluated exactly once.";
+  EXPECT_EQ(1, n4) << "Argument 4 is not evaluated exactly once.";
+  EXPECT_EQ(1, n5) << "Argument 5 is not evaluated exactly once.";
+
+  // A failure case.
+  n1 = n2 = n3 = 0;
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED3(SumIsEven3Functor(), ++n1, n2++, n3++)
+        << "This failure is expected.";
+  }, "This failure is expected.");
+  EXPECT_EQ(1, n1) << "Argument 1 is not evaluated exactly once.";
+  EXPECT_EQ(1, n2) << "Argument 2 is not evaluated exactly once.";
+  EXPECT_EQ(1, n3) << "Argument 3 is not evaluated exactly once.";
+
+  // Another failure case.
+  n1 = n2 = n3 = n4 = 0;
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT4(AssertSumIsEven4, ++n1, n2++, n3++, n4++);
+  }, "evaluates to 1, which is not even.");
+  EXPECT_EQ(1, n1) << "Argument 1 is not evaluated exactly once.";
+  EXPECT_EQ(1, n2) << "Argument 2 is not evaluated exactly once.";
+  EXPECT_EQ(1, n3) << "Argument 3 is not evaluated exactly once.";
+  EXPECT_EQ(1, n4) << "Argument 4 is not evaluated exactly once.";
+}
+
+
+// Some helper functions for testing using overloaded/template
+// functions with ASSERT_PREDn and EXPECT_PREDn.
+
+bool IsPositive(double x) {
+  return x > 0;
+}
+
+template <typename T>
+bool IsNegative(T x) {
+  return x < 0;
+}
+
+template <typename T1, typename T2>
+bool GreaterThan(T1 x1, T2 x2) {
+  return x1 > x2;
+}
+
+// Tests that overloaded functions can be used in *_PRED* as long as
+// their types are explicitly specified.
+TEST(PredicateAssertionTest, AcceptsOverloadedFunction) {
+  // C++Builder requires C-style casts rather than static_cast.
+  EXPECT_PRED1((bool (*)(int))(IsPositive), 5);  // NOLINT
+  ASSERT_PRED1((bool (*)(double))(IsPositive), 6.0);  // NOLINT
+}
+
+// Tests that template functions can be used in *_PRED* as long as
+// their types are explicitly specified.
+TEST(PredicateAssertionTest, AcceptsTemplateFunction) {
+  EXPECT_PRED1(IsNegative<int>, -5);
+  // Makes sure that we can handle templates with more than one
+  // parameter.
+  ASSERT_PRED2((GreaterThan<int, int>), 5, 0);
+}
+
+
+// Some helper functions for testing using overloaded/template
+// functions with ASSERT_PRED_FORMATn and EXPECT_PRED_FORMATn.
+
+AssertionResult IsPositiveFormat(const char* /* expr */, int n) {
+  return n > 0 ? AssertionSuccess() :
+      AssertionFailure(Message() << "Failure");
+}
+
+AssertionResult IsPositiveFormat(const char* /* expr */, double x) {
+  return x > 0 ? AssertionSuccess() :
+      AssertionFailure(Message() << "Failure");
+}
+
+template <typename T>
+AssertionResult IsNegativeFormat(const char* /* expr */, T x) {
+  return x < 0 ? AssertionSuccess() :
+      AssertionFailure(Message() << "Failure");
+}
+
+template <typename T1, typename T2>
+AssertionResult EqualsFormat(const char* /* expr1 */, const char* /* expr2 */,
+                             const T1& x1, const T2& x2) {
+  return x1 == x2 ? AssertionSuccess() :
+      AssertionFailure(Message() << "Failure");
+}
+
+// Tests that overloaded functions can be used in *_PRED_FORMAT*
+// without explicitly specifying their types.
+TEST(PredicateFormatAssertionTest, AcceptsOverloadedFunction) {
+  EXPECT_PRED_FORMAT1(IsPositiveFormat, 5);
+  ASSERT_PRED_FORMAT1(IsPositiveFormat, 6.0);
+}
+
+// Tests that template functions can be used in *_PRED_FORMAT* without
+// explicitly specifying their types.
+TEST(PredicateFormatAssertionTest, AcceptsTemplateFunction) {
+  EXPECT_PRED_FORMAT1(IsNegativeFormat, -5);
+  ASSERT_PRED_FORMAT2(EqualsFormat, 3, 3);
+}
+
+
+// Tests string assertions.
+
+// Tests ASSERT_STREQ with non-NULL arguments.
+TEST(StringAssertionTest, ASSERT_STREQ) {
+  const char * const p1 = "good";
+  ASSERT_STREQ(p1, p1);
+
+  // Let p2 have the same content as p1, but be at a different address.
+  const char p2[] = "good";
+  ASSERT_STREQ(p1, p2);
+
+  EXPECT_FATAL_FAILURE(ASSERT_STREQ("bad", "good"),
+                       "Expected: \"bad\"");
+}
+
+// Tests ASSERT_STREQ with NULL arguments.
+TEST(StringAssertionTest, ASSERT_STREQ_Null) {
+  ASSERT_STREQ(static_cast<const char *>(NULL), NULL);
+  EXPECT_FATAL_FAILURE(ASSERT_STREQ(NULL, "non-null"),
+                       "non-null");
+}
+
+// Tests ASSERT_STREQ with NULL arguments.
+TEST(StringAssertionTest, ASSERT_STREQ_Null2) {
+  EXPECT_FATAL_FAILURE(ASSERT_STREQ("non-null", NULL),
+                       "non-null");
+}
+
+// Tests ASSERT_STRNE.
+TEST(StringAssertionTest, ASSERT_STRNE) {
+  ASSERT_STRNE("hi", "Hi");
+  ASSERT_STRNE("Hi", NULL);
+  ASSERT_STRNE(NULL, "Hi");
+  ASSERT_STRNE("", NULL);
+  ASSERT_STRNE(NULL, "");
+  ASSERT_STRNE("", "Hi");
+  ASSERT_STRNE("Hi", "");
+  EXPECT_FATAL_FAILURE(ASSERT_STRNE("Hi", "Hi"),
+                       "\"Hi\" vs \"Hi\"");
+}
+
+// Tests ASSERT_STRCASEEQ.
+TEST(StringAssertionTest, ASSERT_STRCASEEQ) {
+  ASSERT_STRCASEEQ("hi", "Hi");
+  ASSERT_STRCASEEQ(static_cast<const char *>(NULL), NULL);
+
+  ASSERT_STRCASEEQ("", "");
+  EXPECT_FATAL_FAILURE(ASSERT_STRCASEEQ("Hi", "hi2"),
+                       "Ignoring case");
+}
+
+// Tests ASSERT_STRCASENE.
+TEST(StringAssertionTest, ASSERT_STRCASENE) {
+  ASSERT_STRCASENE("hi1", "Hi2");
+  ASSERT_STRCASENE("Hi", NULL);
+  ASSERT_STRCASENE(NULL, "Hi");
+  ASSERT_STRCASENE("", NULL);
+  ASSERT_STRCASENE(NULL, "");
+  ASSERT_STRCASENE("", "Hi");
+  ASSERT_STRCASENE("Hi", "");
+  EXPECT_FATAL_FAILURE(ASSERT_STRCASENE("Hi", "hi"),
+                       "(ignoring case)");
+}
+
+// Tests *_STREQ on wide strings.
+TEST(StringAssertionTest, STREQ_Wide) {
+  // NULL strings.
+  ASSERT_STREQ(static_cast<const wchar_t *>(NULL), NULL);
+
+  // Empty strings.
+  ASSERT_STREQ(L"", L"");
+
+  // Non-null vs NULL.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STREQ(L"non-null", NULL),
+                          "non-null");
+
+  // Equal strings.
+  EXPECT_STREQ(L"Hi", L"Hi");
+
+  // Unequal strings.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STREQ(L"abc", L"Abc"),
+                          "Abc");
+
+  // Strings containing wide characters.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STREQ(L"abc\x8119", L"abc\x8120"),
+                          "abc");
+
+  // The streaming variation.
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_STREQ(L"abc\x8119", L"abc\x8121") << "Expected failure";
+  }, "Expected failure");
+}
+
+// Tests *_STRNE on wide strings.
+TEST(StringAssertionTest, STRNE_Wide) {
+  // NULL strings.
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_STRNE(static_cast<const wchar_t *>(NULL), NULL);
+  }, "");
+
+  // Empty strings.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRNE(L"", L""),
+                          "L\"\"");
+
+  // Non-null vs NULL.
+  ASSERT_STRNE(L"non-null", NULL);
+
+  // Equal strings.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRNE(L"Hi", L"Hi"),
+                          "L\"Hi\"");
+
+  // Unequal strings.
+  EXPECT_STRNE(L"abc", L"Abc");
+
+  // Strings containing wide characters.
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRNE(L"abc\x8119", L"abc\x8119"),
+                          "abc");
+
+  // The streaming variation.
+  ASSERT_STRNE(L"abc\x8119", L"abc\x8120") << "This shouldn't happen";
+}
+
+// Tests for ::testing::IsSubstring().
+
+// Tests that IsSubstring() returns the correct result when the input
+// argument type is const char*.
+TEST(IsSubstringTest, ReturnsCorrectResultForCString) {
+  EXPECT_FALSE(IsSubstring("", "", NULL, "a"));
+  EXPECT_FALSE(IsSubstring("", "", "b", NULL));
+  EXPECT_FALSE(IsSubstring("", "", "needle", "haystack"));
+
+  EXPECT_TRUE(IsSubstring("", "", static_cast<const char*>(NULL), NULL));
+  EXPECT_TRUE(IsSubstring("", "", "needle", "two needles"));
+}
+
+// Tests that IsSubstring() returns the correct result when the input
+// argument type is const wchar_t*.
+TEST(IsSubstringTest, ReturnsCorrectResultForWideCString) {
+  EXPECT_FALSE(IsSubstring("", "", kNull, L"a"));
+  EXPECT_FALSE(IsSubstring("", "", L"b", kNull));
+  EXPECT_FALSE(IsSubstring("", "", L"needle", L"haystack"));
+
+  EXPECT_TRUE(IsSubstring("", "", static_cast<const wchar_t*>(NULL), NULL));
+  EXPECT_TRUE(IsSubstring("", "", L"needle", L"two needles"));
+}
+
+// Tests that IsSubstring() generates the correct message when the input
+// argument type is const char*.
+TEST(IsSubstringTest, GeneratesCorrectMessageForCString) {
+  EXPECT_STREQ("Value of: needle_expr\n"
+               "  Actual: \"needle\"\n"
+               "Expected: a substring of haystack_expr\n"
+               "Which is: \"haystack\"",
+               IsSubstring("needle_expr", "haystack_expr",
+                           "needle", "haystack").failure_message());
+}
+
+// Tests that IsSubstring returns the correct result when the input
+// argument type is ::std::string.
+TEST(IsSubstringTest, ReturnsCorrectResultsForStdString) {
+  EXPECT_TRUE(IsSubstring("", "", std::string("hello"), "ahellob"));
+  EXPECT_FALSE(IsSubstring("", "", "hello", std::string("world")));
+}
+
+#if GTEST_HAS_STD_WSTRING
+// Tests that IsSubstring returns the correct result when the input
+// argument type is ::std::wstring.
+TEST(IsSubstringTest, ReturnsCorrectResultForStdWstring) {
+  EXPECT_TRUE(IsSubstring("", "", ::std::wstring(L"needle"), L"two needles"));
+  EXPECT_FALSE(IsSubstring("", "", L"needle", ::std::wstring(L"haystack")));
+}
+
+// Tests that IsSubstring() generates the correct message when the input
+// argument type is ::std::wstring.
+TEST(IsSubstringTest, GeneratesCorrectMessageForWstring) {
+  EXPECT_STREQ("Value of: needle_expr\n"
+               "  Actual: L\"needle\"\n"
+               "Expected: a substring of haystack_expr\n"
+               "Which is: L\"haystack\"",
+               IsSubstring(
+                   "needle_expr", "haystack_expr",
+                   ::std::wstring(L"needle"), L"haystack").failure_message());
+}
+
+#endif  // GTEST_HAS_STD_WSTRING
+
+// Tests for ::testing::IsNotSubstring().
+
+// Tests that IsNotSubstring() returns the correct result when the input
+// argument type is const char*.
+TEST(IsNotSubstringTest, ReturnsCorrectResultForCString) {
+  EXPECT_TRUE(IsNotSubstring("", "", "needle", "haystack"));
+  EXPECT_FALSE(IsNotSubstring("", "", "needle", "two needles"));
+}
+
+// Tests that IsNotSubstring() returns the correct result when the input
+// argument type is const wchar_t*.
+TEST(IsNotSubstringTest, ReturnsCorrectResultForWideCString) {
+  EXPECT_TRUE(IsNotSubstring("", "", L"needle", L"haystack"));
+  EXPECT_FALSE(IsNotSubstring("", "", L"needle", L"two needles"));
+}
+
+// Tests that IsNotSubstring() generates the correct message when the input
+// argument type is const wchar_t*.
+TEST(IsNotSubstringTest, GeneratesCorrectMessageForWideCString) {
+  EXPECT_STREQ("Value of: needle_expr\n"
+               "  Actual: L\"needle\"\n"
+               "Expected: not a substring of haystack_expr\n"
+               "Which is: L\"two needles\"",
+               IsNotSubstring(
+                   "needle_expr", "haystack_expr",
+                   L"needle", L"two needles").failure_message());
+}
+
+// Tests that IsNotSubstring returns the correct result when the input
+// argument type is ::std::string.
+TEST(IsNotSubstringTest, ReturnsCorrectResultsForStdString) {
+  EXPECT_FALSE(IsNotSubstring("", "", std::string("hello"), "ahellob"));
+  EXPECT_TRUE(IsNotSubstring("", "", "hello", std::string("world")));
+}
+
+// Tests that IsNotSubstring() generates the correct message when the input
+// argument type is ::std::string.
+TEST(IsNotSubstringTest, GeneratesCorrectMessageForStdString) {
+  EXPECT_STREQ("Value of: needle_expr\n"
+               "  Actual: \"needle\"\n"
+               "Expected: not a substring of haystack_expr\n"
+               "Which is: \"two needles\"",
+               IsNotSubstring(
+                   "needle_expr", "haystack_expr",
+                   ::std::string("needle"), "two needles").failure_message());
+}
+
+#if GTEST_HAS_STD_WSTRING
+
+// Tests that IsNotSubstring returns the correct result when the input
+// argument type is ::std::wstring.
+TEST(IsNotSubstringTest, ReturnsCorrectResultForStdWstring) {
+  EXPECT_FALSE(
+      IsNotSubstring("", "", ::std::wstring(L"needle"), L"two needles"));
+  EXPECT_TRUE(IsNotSubstring("", "", L"needle", ::std::wstring(L"haystack")));
+}
+
+#endif  // GTEST_HAS_STD_WSTRING
+
+// Tests floating-point assertions.
+
+template <typename RawType>
+class FloatingPointTest : public Test {
+ protected:
+  // Pre-calculated numbers to be used by the tests.
+  struct TestValues {
+    RawType close_to_positive_zero;
+    RawType close_to_negative_zero;
+    RawType further_from_negative_zero;
+
+    RawType close_to_one;
+    RawType further_from_one;
+
+    RawType infinity;
+    RawType close_to_infinity;
+    RawType further_from_infinity;
+
+    RawType nan1;
+    RawType nan2;
+  };
+
+  typedef typename testing::internal::FloatingPoint<RawType> Floating;
+  typedef typename Floating::Bits Bits;
+
+  virtual void SetUp() {
+    const size_t max_ulps = Floating::kMaxUlps;
+
+    // The bits that represent 0.0.
+    const Bits zero_bits = Floating(0).bits();
+
+    // Makes some numbers close to 0.0.
+    values_.close_to_positive_zero = Floating::ReinterpretBits(
+        zero_bits + max_ulps/2);
+    values_.close_to_negative_zero = -Floating::ReinterpretBits(
+        zero_bits + max_ulps - max_ulps/2);
+    values_.further_from_negative_zero = -Floating::ReinterpretBits(
+        zero_bits + max_ulps + 1 - max_ulps/2);
+
+    // The bits that represent 1.0.
+    const Bits one_bits = Floating(1).bits();
+
+    // Makes some numbers close to 1.0.
+    values_.close_to_one = Floating::ReinterpretBits(one_bits + max_ulps);
+    values_.further_from_one = Floating::ReinterpretBits(
+        one_bits + max_ulps + 1);
+
+    // +infinity.
+    values_.infinity = Floating::Infinity();
+
+    // The bits that represent +infinity.
+    const Bits infinity_bits = Floating(values_.infinity).bits();
+
+    // Makes some numbers close to infinity.
+    values_.close_to_infinity = Floating::ReinterpretBits(
+        infinity_bits - max_ulps);
+    values_.further_from_infinity = Floating::ReinterpretBits(
+        infinity_bits - max_ulps - 1);
+
+    // Makes some NAN's.  Sets the most significant bit of the fraction so that
+    // our NaN's are quiet; trying to process a signaling NaN would raise an
+    // exception if our environment enables floating point exceptions.
+    values_.nan1 = Floating::ReinterpretBits(Floating::kExponentBitMask
+        | (static_cast<Bits>(1) << (Floating::kFractionBitCount - 1)) | 1);
+    values_.nan2 = Floating::ReinterpretBits(Floating::kExponentBitMask
+        | (static_cast<Bits>(1) << (Floating::kFractionBitCount - 1)) | 200);
+  }
+
+  void TestSize() {
+    EXPECT_EQ(sizeof(RawType), sizeof(Bits));
+  }
+
+  static TestValues values_;
+};
+
+template <typename RawType>
+typename FloatingPointTest<RawType>::TestValues
+    FloatingPointTest<RawType>::values_;
+
+// Instantiates FloatingPointTest for testing *_FLOAT_EQ.
+typedef FloatingPointTest<float> FloatTest;
+
+// Tests that the size of Float::Bits matches the size of float.
+TEST_F(FloatTest, Size) {
+  TestSize();
+}
+
+// Tests comparing with +0 and -0.
+TEST_F(FloatTest, Zeros) {
+  EXPECT_FLOAT_EQ(0.0, -0.0);
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(-0.0, 1.0),
+                          "1.0");
+  EXPECT_FATAL_FAILURE(ASSERT_FLOAT_EQ(0.0, 1.5),
+                       "1.5");
+}
+
+// Tests comparing numbers close to 0.
+//
+// This ensures that *_FLOAT_EQ handles the sign correctly and no
+// overflow occurs when comparing numbers whose absolute value is very
+// small.
+TEST_F(FloatTest, AlmostZeros) {
+  // In C++Builder, names within local classes (such as used by
+  // EXPECT_FATAL_FAILURE) cannot be resolved against static members of the
+  // scoping class.  Use a static local alias as a workaround.
+  // We use the assignment syntax since some compilers, like Sun Studio,
+  // don't allow initializing references using construction syntax
+  // (parentheses).
+  static const FloatTest::TestValues& v = this->values_;
+
+  EXPECT_FLOAT_EQ(0.0, v.close_to_positive_zero);
+  EXPECT_FLOAT_EQ(-0.0, v.close_to_negative_zero);
+  EXPECT_FLOAT_EQ(v.close_to_positive_zero, v.close_to_negative_zero);
+
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_FLOAT_EQ(v.close_to_positive_zero,
+                    v.further_from_negative_zero);
+  }, "v.further_from_negative_zero");
+}
+
+// Tests comparing numbers close to each other.
+TEST_F(FloatTest, SmallDiff) {
+  EXPECT_FLOAT_EQ(1.0, values_.close_to_one);
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(1.0, values_.further_from_one),
+                          "values_.further_from_one");
+}
+
+// Tests comparing numbers far apart.
+TEST_F(FloatTest, LargeDiff) {
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(2.5, 3.0),
+                          "3.0");
+}
+
+// Tests comparing with infinity.
+//
+// This ensures that no overflow occurs when comparing numbers whose
+// absolute value is very large.
+TEST_F(FloatTest, Infinity) {
+  EXPECT_FLOAT_EQ(values_.infinity, values_.close_to_infinity);
+  EXPECT_FLOAT_EQ(-values_.infinity, -values_.close_to_infinity);
+#if !GTEST_OS_SYMBIAN
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(values_.infinity, -values_.infinity),
+                          "-values_.infinity");
+
+  // This is interesting as the representations of infinity and nan1
+  // are only 1 DLP apart.
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(values_.infinity, values_.nan1),
+                          "values_.nan1");
+#endif  // !GTEST_OS_SYMBIAN
+}
+
+// Tests that comparing with NAN always returns false.
+TEST_F(FloatTest, NaN) {
+#if !GTEST_OS_SYMBIAN
+// Nokia's STLport crashes if we try to output infinity or NaN.
+
+  // In C++Builder, names within local classes (such as used by
+  // EXPECT_FATAL_FAILURE) cannot be resolved against static members of the
+  // scoping class.  Use a static local alias as a workaround.
+  // We use the assignment syntax since some compilers, like Sun Studio,
+  // don't allow initializing references using construction syntax
+  // (parentheses).
+  static const FloatTest::TestValues& v = this->values_;
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(v.nan1, v.nan1),
+                          "v.nan1");
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(v.nan1, v.nan2),
+                          "v.nan2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(1.0, v.nan1),
+                          "v.nan1");
+
+  EXPECT_FATAL_FAILURE(ASSERT_FLOAT_EQ(v.nan1, v.infinity),
+                       "v.infinity");
+#endif  // !GTEST_OS_SYMBIAN
+}
+
+// Tests that *_FLOAT_EQ are reflexive.
+TEST_F(FloatTest, Reflexive) {
+  EXPECT_FLOAT_EQ(0.0, 0.0);
+  EXPECT_FLOAT_EQ(1.0, 1.0);
+  ASSERT_FLOAT_EQ(values_.infinity, values_.infinity);
+}
+
+// Tests that *_FLOAT_EQ are commutative.
+TEST_F(FloatTest, Commutative) {
+  // We already tested EXPECT_FLOAT_EQ(1.0, values_.close_to_one).
+  EXPECT_FLOAT_EQ(values_.close_to_one, 1.0);
+
+  // We already tested EXPECT_FLOAT_EQ(1.0, values_.further_from_one).
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(values_.further_from_one, 1.0),
+                          "1.0");
+}
+
+// Tests EXPECT_NEAR.
+TEST_F(FloatTest, EXPECT_NEAR) {
+  EXPECT_NEAR(-1.0f, -1.1f, 0.2f);
+  EXPECT_NEAR(2.0f, 3.0f, 1.0f);
+  EXPECT_NONFATAL_FAILURE(EXPECT_NEAR(1.0f,1.5f, 0.25f),  // NOLINT
+                          "The difference between 1.0f and 1.5f is 0.5, "
+                          "which exceeds 0.25f");
+  // To work around a bug in gcc 2.95.0, there is intentionally no
+  // space after the first comma in the previous line.
+}
+
+// Tests ASSERT_NEAR.
+TEST_F(FloatTest, ASSERT_NEAR) {
+  ASSERT_NEAR(-1.0f, -1.1f, 0.2f);
+  ASSERT_NEAR(2.0f, 3.0f, 1.0f);
+  EXPECT_FATAL_FAILURE(ASSERT_NEAR(1.0f,1.5f, 0.25f),  // NOLINT
+                       "The difference between 1.0f and 1.5f is 0.5, "
+                       "which exceeds 0.25f");
+  // To work around a bug in gcc 2.95.0, there is intentionally no
+  // space after the first comma in the previous line.
+}
+
+// Tests the cases where FloatLE() should succeed.
+TEST_F(FloatTest, FloatLESucceeds) {
+  EXPECT_PRED_FORMAT2(FloatLE, 1.0f, 2.0f);  // When val1 < val2,
+  ASSERT_PRED_FORMAT2(FloatLE, 1.0f, 1.0f);  // val1 == val2,
+
+  // or when val1 is greater than, but almost equals to, val2.
+  EXPECT_PRED_FORMAT2(FloatLE, values_.close_to_positive_zero, 0.0f);
+}
+
+// Tests the cases where FloatLE() should fail.
+TEST_F(FloatTest, FloatLEFails) {
+  // When val1 is greater than val2 by a large margin,
+  EXPECT_NONFATAL_FAILURE(EXPECT_PRED_FORMAT2(FloatLE, 2.0f, 1.0f),
+                          "(2.0f) <= (1.0f)");
+
+  // or by a small yet non-negligible margin,
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(FloatLE, values_.further_from_one, 1.0f);
+  }, "(values_.further_from_one) <= (1.0f)");
+
+#if !GTEST_OS_SYMBIAN && !defined(__BORLANDC__)
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  // C++Builder gives bad results for ordered comparisons involving NaNs
+  // due to compiler bugs.
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(FloatLE, values_.nan1, values_.infinity);
+  }, "(values_.nan1) <= (values_.infinity)");
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(FloatLE, -values_.infinity, values_.nan1);
+  }, "(-values_.infinity) <= (values_.nan1)");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(FloatLE, values_.nan1, values_.nan1);
+  }, "(values_.nan1) <= (values_.nan1)");
+#endif  // !GTEST_OS_SYMBIAN && !defined(__BORLANDC__)
+}
+
+// Instantiates FloatingPointTest for testing *_DOUBLE_EQ.
+typedef FloatingPointTest<double> DoubleTest;
+
+// Tests that the size of Double::Bits matches the size of double.
+TEST_F(DoubleTest, Size) {
+  TestSize();
+}
+
+// Tests comparing with +0 and -0.
+TEST_F(DoubleTest, Zeros) {
+  EXPECT_DOUBLE_EQ(0.0, -0.0);
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(-0.0, 1.0),
+                          "1.0");
+  EXPECT_FATAL_FAILURE(ASSERT_DOUBLE_EQ(0.0, 1.0),
+                       "1.0");
+}
+
+// Tests comparing numbers close to 0.
+//
+// This ensures that *_DOUBLE_EQ handles the sign correctly and no
+// overflow occurs when comparing numbers whose absolute value is very
+// small.
+TEST_F(DoubleTest, AlmostZeros) {
+  // In C++Builder, names within local classes (such as used by
+  // EXPECT_FATAL_FAILURE) cannot be resolved against static members of the
+  // scoping class.  Use a static local alias as a workaround.
+  // We use the assignment syntax since some compilers, like Sun Studio,
+  // don't allow initializing references using construction syntax
+  // (parentheses).
+  static const DoubleTest::TestValues& v = this->values_;
+
+  EXPECT_DOUBLE_EQ(0.0, v.close_to_positive_zero);
+  EXPECT_DOUBLE_EQ(-0.0, v.close_to_negative_zero);
+  EXPECT_DOUBLE_EQ(v.close_to_positive_zero, v.close_to_negative_zero);
+
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_DOUBLE_EQ(v.close_to_positive_zero,
+                     v.further_from_negative_zero);
+  }, "v.further_from_negative_zero");
+}
+
+// Tests comparing numbers close to each other.
+TEST_F(DoubleTest, SmallDiff) {
+  EXPECT_DOUBLE_EQ(1.0, values_.close_to_one);
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(1.0, values_.further_from_one),
+                          "values_.further_from_one");
+}
+
+// Tests comparing numbers far apart.
+TEST_F(DoubleTest, LargeDiff) {
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(2.0, 3.0),
+                          "3.0");
+}
+
+// Tests comparing with infinity.
+//
+// This ensures that no overflow occurs when comparing numbers whose
+// absolute value is very large.
+TEST_F(DoubleTest, Infinity) {
+  EXPECT_DOUBLE_EQ(values_.infinity, values_.close_to_infinity);
+  EXPECT_DOUBLE_EQ(-values_.infinity, -values_.close_to_infinity);
+#if !GTEST_OS_SYMBIAN
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(values_.infinity, -values_.infinity),
+                          "-values_.infinity");
+
+  // This is interesting as the representations of infinity_ and nan1_
+  // are only 1 DLP apart.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(values_.infinity, values_.nan1),
+                          "values_.nan1");
+#endif  // !GTEST_OS_SYMBIAN
+}
+
+// Tests that comparing with NAN always returns false.
+TEST_F(DoubleTest, NaN) {
+#if !GTEST_OS_SYMBIAN
+  // In C++Builder, names within local classes (such as used by
+  // EXPECT_FATAL_FAILURE) cannot be resolved against static members of the
+  // scoping class.  Use a static local alias as a workaround.
+  // We use the assignment syntax since some compilers, like Sun Studio,
+  // don't allow initializing references using construction syntax
+  // (parentheses).
+  static const DoubleTest::TestValues& v = this->values_;
+
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(v.nan1, v.nan1),
+                          "v.nan1");
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(v.nan1, v.nan2), "v.nan2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(1.0, v.nan1), "v.nan1");
+  EXPECT_FATAL_FAILURE(ASSERT_DOUBLE_EQ(v.nan1, v.infinity),
+                       "v.infinity");
+#endif  // !GTEST_OS_SYMBIAN
+}
+
+// Tests that *_DOUBLE_EQ are reflexive.
+TEST_F(DoubleTest, Reflexive) {
+  EXPECT_DOUBLE_EQ(0.0, 0.0);
+  EXPECT_DOUBLE_EQ(1.0, 1.0);
+#if !GTEST_OS_SYMBIAN
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  ASSERT_DOUBLE_EQ(values_.infinity, values_.infinity);
+#endif  // !GTEST_OS_SYMBIAN
+}
+
+// Tests that *_DOUBLE_EQ are commutative.
+TEST_F(DoubleTest, Commutative) {
+  // We already tested EXPECT_DOUBLE_EQ(1.0, values_.close_to_one).
+  EXPECT_DOUBLE_EQ(values_.close_to_one, 1.0);
+
+  // We already tested EXPECT_DOUBLE_EQ(1.0, values_.further_from_one).
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(values_.further_from_one, 1.0),
+                          "1.0");
+}
+
+// Tests EXPECT_NEAR.
+TEST_F(DoubleTest, EXPECT_NEAR) {
+  EXPECT_NEAR(-1.0, -1.1, 0.2);
+  EXPECT_NEAR(2.0, 3.0, 1.0);
+  EXPECT_NONFATAL_FAILURE(EXPECT_NEAR(1.0, 1.5, 0.25),  // NOLINT
+                          "The difference between 1.0 and 1.5 is 0.5, "
+                          "which exceeds 0.25");
+  // To work around a bug in gcc 2.95.0, there is intentionally no
+  // space after the first comma in the previous statement.
+}
+
+// Tests ASSERT_NEAR.
+TEST_F(DoubleTest, ASSERT_NEAR) {
+  ASSERT_NEAR(-1.0, -1.1, 0.2);
+  ASSERT_NEAR(2.0, 3.0, 1.0);
+  EXPECT_FATAL_FAILURE(ASSERT_NEAR(1.0, 1.5, 0.25),  // NOLINT
+                       "The difference between 1.0 and 1.5 is 0.5, "
+                       "which exceeds 0.25");
+  // To work around a bug in gcc 2.95.0, there is intentionally no
+  // space after the first comma in the previous statement.
+}
+
+// Tests the cases where DoubleLE() should succeed.
+TEST_F(DoubleTest, DoubleLESucceeds) {
+  EXPECT_PRED_FORMAT2(DoubleLE, 1.0, 2.0);  // When val1 < val2,
+  ASSERT_PRED_FORMAT2(DoubleLE, 1.0, 1.0);  // val1 == val2,
+
+  // or when val1 is greater than, but almost equals to, val2.
+  EXPECT_PRED_FORMAT2(DoubleLE, values_.close_to_positive_zero, 0.0);
+}
+
+// Tests the cases where DoubleLE() should fail.
+TEST_F(DoubleTest, DoubleLEFails) {
+  // When val1 is greater than val2 by a large margin,
+  EXPECT_NONFATAL_FAILURE(EXPECT_PRED_FORMAT2(DoubleLE, 2.0, 1.0),
+                          "(2.0) <= (1.0)");
+
+  // or by a small yet non-negligible margin,
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(DoubleLE, values_.further_from_one, 1.0);
+  }, "(values_.further_from_one) <= (1.0)");
+
+#if !GTEST_OS_SYMBIAN && !defined(__BORLANDC__)
+  // Nokia's STLport crashes if we try to output infinity or NaN.
+  // C++Builder gives bad results for ordered comparisons involving NaNs
+  // due to compiler bugs.
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(DoubleLE, values_.nan1, values_.infinity);
+  }, "(values_.nan1) <= (values_.infinity)");
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_PRED_FORMAT2(DoubleLE, -values_.infinity, values_.nan1);
+  }, " (-values_.infinity) <= (values_.nan1)");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_PRED_FORMAT2(DoubleLE, values_.nan1, values_.nan1);
+  }, "(values_.nan1) <= (values_.nan1)");
+#endif  // !GTEST_OS_SYMBIAN && !defined(__BORLANDC__)
+}
+
+
+// Verifies that a test or test case whose name starts with DISABLED_ is
+// not run.
+
+// A test whose name starts with DISABLED_.
+// Should not run.
+TEST(DisabledTest, DISABLED_TestShouldNotRun) {
+  FAIL() << "Unexpected failure: Disabled test should not be run.";
+}
+
+// A test whose name does not start with DISABLED_.
+// Should run.
+TEST(DisabledTest, NotDISABLED_TestShouldRun) {
+  EXPECT_EQ(1, 1);
+}
+
+// A test case whose name starts with DISABLED_.
+// Should not run.
+TEST(DISABLED_TestCase, TestShouldNotRun) {
+  FAIL() << "Unexpected failure: Test in disabled test case should not be run.";
+}
+
+// A test case and test whose names start with DISABLED_.
+// Should not run.
+TEST(DISABLED_TestCase, DISABLED_TestShouldNotRun) {
+  FAIL() << "Unexpected failure: Test in disabled test case should not be run.";
+}
+
+// Check that when all tests in a test case are disabled, SetupTestCase() and
+// TearDownTestCase() are not called.
+class DisabledTestsTest : public Test {
+ protected:
+  static void SetUpTestCase() {
+    FAIL() << "Unexpected failure: All tests disabled in test case. "
+              "SetupTestCase() should not be called.";
+  }
+
+  static void TearDownTestCase() {
+    FAIL() << "Unexpected failure: All tests disabled in test case. "
+              "TearDownTestCase() should not be called.";
+  }
+};
+
+TEST_F(DisabledTestsTest, DISABLED_TestShouldNotRun_1) {
+  FAIL() << "Unexpected failure: Disabled test should not be run.";
+}
+
+TEST_F(DisabledTestsTest, DISABLED_TestShouldNotRun_2) {
+  FAIL() << "Unexpected failure: Disabled test should not be run.";
+}
+
+// Tests that disabled typed tests aren't run.
+
+#if GTEST_HAS_TYPED_TEST
+
+template <typename T>
+class TypedTest : public Test {
+};
+
+typedef testing::Types<int, double> NumericTypes;
+TYPED_TEST_CASE(TypedTest, NumericTypes);
+
+TYPED_TEST(TypedTest, DISABLED_ShouldNotRun) {
+  FAIL() << "Unexpected failure: Disabled typed test should not run.";
+}
+
+template <typename T>
+class DISABLED_TypedTest : public Test {
+};
+
+TYPED_TEST_CASE(DISABLED_TypedTest, NumericTypes);
+
+TYPED_TEST(DISABLED_TypedTest, ShouldNotRun) {
+  FAIL() << "Unexpected failure: Disabled typed test should not run.";
+}
+
+#endif  // GTEST_HAS_TYPED_TEST
+
+// Tests that disabled type-parameterized tests aren't run.
+
+#if GTEST_HAS_TYPED_TEST_P
+
+template <typename T>
+class TypedTestP : public Test {
+};
+
+TYPED_TEST_CASE_P(TypedTestP);
+
+TYPED_TEST_P(TypedTestP, DISABLED_ShouldNotRun) {
+  FAIL() << "Unexpected failure: "
+         << "Disabled type-parameterized test should not run.";
+}
+
+REGISTER_TYPED_TEST_CASE_P(TypedTestP, DISABLED_ShouldNotRun);
+
+INSTANTIATE_TYPED_TEST_CASE_P(My, TypedTestP, NumericTypes);
+
+template <typename T>
+class DISABLED_TypedTestP : public Test {
+};
+
+TYPED_TEST_CASE_P(DISABLED_TypedTestP);
+
+TYPED_TEST_P(DISABLED_TypedTestP, ShouldNotRun) {
+  FAIL() << "Unexpected failure: "
+         << "Disabled type-parameterized test should not run.";
+}
+
+REGISTER_TYPED_TEST_CASE_P(DISABLED_TypedTestP, ShouldNotRun);
+
+INSTANTIATE_TYPED_TEST_CASE_P(My, DISABLED_TypedTestP, NumericTypes);
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+
+// Tests that assertion macros evaluate their arguments exactly once.
+
+class SingleEvaluationTest : public Test {
+ public:  // Must be public and not protected due to a bug in g++ 3.4.2.
+  // This helper function is needed by the FailedASSERT_STREQ test
+  // below.  It's public to work around C++Builder's bug with scoping local
+  // classes.
+  static void CompareAndIncrementCharPtrs() {
+    ASSERT_STREQ(p1_++, p2_++);
+  }
+
+  // This helper function is needed by the FailedASSERT_NE test below.  It's
+  // public to work around C++Builder's bug with scoping local classes.
+  static void CompareAndIncrementInts() {
+    ASSERT_NE(a_++, b_++);
+  }
+
+ protected:
+  SingleEvaluationTest() {
+    p1_ = s1_;
+    p2_ = s2_;
+    a_ = 0;
+    b_ = 0;
+  }
+
+  static const char* const s1_;
+  static const char* const s2_;
+  static const char* p1_;
+  static const char* p2_;
+
+  static int a_;
+  static int b_;
+};
+
+const char* const SingleEvaluationTest::s1_ = "01234";
+const char* const SingleEvaluationTest::s2_ = "abcde";
+const char* SingleEvaluationTest::p1_;
+const char* SingleEvaluationTest::p2_;
+int SingleEvaluationTest::a_;
+int SingleEvaluationTest::b_;
+
+// Tests that when ASSERT_STREQ fails, it evaluates its arguments
+// exactly once.
+TEST_F(SingleEvaluationTest, FailedASSERT_STREQ) {
+  EXPECT_FATAL_FAILURE(SingleEvaluationTest::CompareAndIncrementCharPtrs(),
+                       "p2_++");
+  EXPECT_EQ(s1_ + 1, p1_);
+  EXPECT_EQ(s2_ + 1, p2_);
+}
+
+// Tests that string assertion arguments are evaluated exactly once.
+TEST_F(SingleEvaluationTest, ASSERT_STR) {
+  // successful EXPECT_STRNE
+  EXPECT_STRNE(p1_++, p2_++);
+  EXPECT_EQ(s1_ + 1, p1_);
+  EXPECT_EQ(s2_ + 1, p2_);
+
+  // failed EXPECT_STRCASEEQ
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRCASEEQ(p1_++, p2_++),
+                          "Ignoring case");
+  EXPECT_EQ(s1_ + 2, p1_);
+  EXPECT_EQ(s2_ + 2, p2_);
+}
+
+// Tests that when ASSERT_NE fails, it evaluates its arguments exactly
+// once.
+TEST_F(SingleEvaluationTest, FailedASSERT_NE) {
+  EXPECT_FATAL_FAILURE(SingleEvaluationTest::CompareAndIncrementInts(),
+                       "(a_++) != (b_++)");
+  EXPECT_EQ(1, a_);
+  EXPECT_EQ(1, b_);
+}
+
+// Tests that assertion arguments are evaluated exactly once.
+TEST_F(SingleEvaluationTest, OtherCases) {
+  // successful EXPECT_TRUE
+  EXPECT_TRUE(0 == a_++);  // NOLINT
+  EXPECT_EQ(1, a_);
+
+  // failed EXPECT_TRUE
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(-1 == a_++), "-1 == a_++");
+  EXPECT_EQ(2, a_);
+
+  // successful EXPECT_GT
+  EXPECT_GT(a_++, b_++);
+  EXPECT_EQ(3, a_);
+  EXPECT_EQ(1, b_);
+
+  // failed EXPECT_LT
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(a_++, b_++), "(a_++) < (b_++)");
+  EXPECT_EQ(4, a_);
+  EXPECT_EQ(2, b_);
+
+  // successful ASSERT_TRUE
+  ASSERT_TRUE(0 < a_++);  // NOLINT
+  EXPECT_EQ(5, a_);
+
+  // successful ASSERT_GT
+  ASSERT_GT(a_++, b_++);
+  EXPECT_EQ(6, a_);
+  EXPECT_EQ(3, b_);
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+void ThrowAnInteger() {
+  throw 1;
+}
+
+// Tests that assertion arguments are evaluated exactly once.
+TEST_F(SingleEvaluationTest, ExceptionTests) {
+  // successful EXPECT_THROW
+  EXPECT_THROW({  // NOLINT
+    a_++;
+    ThrowAnInteger();
+  }, int);
+  EXPECT_EQ(1, a_);
+
+  // failed EXPECT_THROW, throws different
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW({  // NOLINT
+    a_++;
+    ThrowAnInteger();
+  }, bool), "throws a different type");
+  EXPECT_EQ(2, a_);
+
+  // failed EXPECT_THROW, throws nothing
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW(a_++, bool), "throws nothing");
+  EXPECT_EQ(3, a_);
+
+  // successful EXPECT_NO_THROW
+  EXPECT_NO_THROW(a_++);
+  EXPECT_EQ(4, a_);
+
+  // failed EXPECT_NO_THROW
+  EXPECT_NONFATAL_FAILURE(EXPECT_NO_THROW({  // NOLINT
+    a_++;
+    ThrowAnInteger();
+  }), "it throws");
+  EXPECT_EQ(5, a_);
+
+  // successful EXPECT_ANY_THROW
+  EXPECT_ANY_THROW({  // NOLINT
+    a_++;
+    ThrowAnInteger();
+  });
+  EXPECT_EQ(6, a_);
+
+  // failed EXPECT_ANY_THROW
+  EXPECT_NONFATAL_FAILURE(EXPECT_ANY_THROW(a_++), "it doesn't");
+  EXPECT_EQ(7, a_);
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// Tests {ASSERT|EXPECT}_NO_FATAL_FAILURE.
+class NoFatalFailureTest : public Test {
+ protected:
+  void Succeeds() {}
+  void FailsNonFatal() {
+    ADD_FAILURE() << "some non-fatal failure";
+  }
+  void Fails() {
+    FAIL() << "some fatal failure";
+  }
+
+  void DoAssertNoFatalFailureOnFails() {
+    ASSERT_NO_FATAL_FAILURE(Fails());
+    ADD_FAILURE() << "shold not reach here.";
+  }
+
+  void DoExpectNoFatalFailureOnFails() {
+    EXPECT_NO_FATAL_FAILURE(Fails());
+    ADD_FAILURE() << "other failure";
+  }
+};
+
+TEST_F(NoFatalFailureTest, NoFailure) {
+  EXPECT_NO_FATAL_FAILURE(Succeeds());
+  ASSERT_NO_FATAL_FAILURE(Succeeds());
+}
+
+TEST_F(NoFatalFailureTest, NonFatalIsNoFailure) {
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_NO_FATAL_FAILURE(FailsNonFatal()),
+      "some non-fatal failure");
+  EXPECT_NONFATAL_FAILURE(
+      ASSERT_NO_FATAL_FAILURE(FailsNonFatal()),
+      "some non-fatal failure");
+}
+
+TEST_F(NoFatalFailureTest, AssertNoFatalFailureOnFatalFailure) {
+  TestPartResultArray gtest_failures;
+  {
+    ScopedFakeTestPartResultReporter gtest_reporter(&gtest_failures);
+    DoAssertNoFatalFailureOnFails();
+  }
+  ASSERT_EQ(2, gtest_failures.size());
+  EXPECT_EQ(TestPartResult::kFatalFailure,
+            gtest_failures.GetTestPartResult(0).type());
+  EXPECT_EQ(TestPartResult::kFatalFailure,
+            gtest_failures.GetTestPartResult(1).type());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "some fatal failure",
+                      gtest_failures.GetTestPartResult(0).message());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "it does",
+                      gtest_failures.GetTestPartResult(1).message());
+}
+
+TEST_F(NoFatalFailureTest, ExpectNoFatalFailureOnFatalFailure) {
+  TestPartResultArray gtest_failures;
+  {
+    ScopedFakeTestPartResultReporter gtest_reporter(&gtest_failures);
+    DoExpectNoFatalFailureOnFails();
+  }
+  ASSERT_EQ(3, gtest_failures.size());
+  EXPECT_EQ(TestPartResult::kFatalFailure,
+            gtest_failures.GetTestPartResult(0).type());
+  EXPECT_EQ(TestPartResult::kNonFatalFailure,
+            gtest_failures.GetTestPartResult(1).type());
+  EXPECT_EQ(TestPartResult::kNonFatalFailure,
+            gtest_failures.GetTestPartResult(2).type());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "some fatal failure",
+                      gtest_failures.GetTestPartResult(0).message());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "it does",
+                      gtest_failures.GetTestPartResult(1).message());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "other failure",
+                      gtest_failures.GetTestPartResult(2).message());
+}
+
+TEST_F(NoFatalFailureTest, MessageIsStreamable) {
+  TestPartResultArray gtest_failures;
+  {
+    ScopedFakeTestPartResultReporter gtest_reporter(&gtest_failures);
+    EXPECT_NO_FATAL_FAILURE(FAIL() << "foo") << "my message";
+  }
+  ASSERT_EQ(2, gtest_failures.size());
+  EXPECT_EQ(TestPartResult::kNonFatalFailure,
+            gtest_failures.GetTestPartResult(0).type());
+  EXPECT_EQ(TestPartResult::kNonFatalFailure,
+            gtest_failures.GetTestPartResult(1).type());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "foo",
+                      gtest_failures.GetTestPartResult(0).message());
+  EXPECT_PRED_FORMAT2(testing::IsSubstring, "my message",
+                      gtest_failures.GetTestPartResult(1).message());
+}
+
+// Tests non-string assertions.
+
+std::string EditsToString(const std::vector<EditType>& edits) {
+  std::string out;
+  for (size_t i = 0; i < edits.size(); ++i) {
+    static const char kEdits[] = " +-/";
+    out.append(1, kEdits[edits[i]]);
+  }
+  return out;
+}
+
+std::vector<size_t> CharsToIndices(const std::string& str) {
+  std::vector<size_t> out;
+  for (size_t i = 0; i < str.size(); ++i) {
+    out.push_back(str[i]);
+  }
+  return out;
+}
+
+std::vector<std::string> CharsToLines(const std::string& str) {
+  std::vector<std::string> out;
+  for (size_t i = 0; i < str.size(); ++i) {
+    out.push_back(str.substr(i, 1));
+  }
+  return out;
+}
+
+TEST(EditDistance, TestCases) {
+  struct Case {
+    int line;
+    const char* left;
+    const char* right;
+    const char* expected_edits;
+    const char* expected_diff;
+  };
+  static const Case kCases[] = {
+      // No change.
+      {__LINE__, "A", "A", " ", ""},
+      {__LINE__, "ABCDE", "ABCDE", "     ", ""},
+      // Simple adds.
+      {__LINE__, "X", "XA", " +", "@@ +1,2 @@\n X\n+A\n"},
+      {__LINE__, "X", "XABCD", " ++++", "@@ +1,5 @@\n X\n+A\n+B\n+C\n+D\n"},
+      // Simple removes.
+      {__LINE__, "XA", "X", " -", "@@ -1,2 @@\n X\n-A\n"},
+      {__LINE__, "XABCD", "X", " ----", "@@ -1,5 @@\n X\n-A\n-B\n-C\n-D\n"},
+      // Simple replaces.
+      {__LINE__, "A", "a", "/", "@@ -1,1 +1,1 @@\n-A\n+a\n"},
+      {__LINE__, "ABCD", "abcd", "////",
+       "@@ -1,4 +1,4 @@\n-A\n-B\n-C\n-D\n+a\n+b\n+c\n+d\n"},
+      // Path finding.
+      {__LINE__, "ABCDEFGH", "ABXEGH1", "  -/ -  +",
+       "@@ -1,8 +1,7 @@\n A\n B\n-C\n-D\n+X\n E\n-F\n G\n H\n+1\n"},
+      {__LINE__, "AAAABCCCC", "ABABCDCDC", "- /   + / ",
+       "@@ -1,9 +1,9 @@\n-A\n A\n-A\n+B\n A\n B\n C\n+D\n C\n-C\n+D\n C\n"},
+      {__LINE__, "ABCDE", "BCDCD", "-   +/",
+       "@@ -1,5 +1,5 @@\n-A\n B\n C\n D\n-E\n+C\n+D\n"},
+      {__LINE__, "ABCDEFGHIJKL", "BCDCDEFGJKLJK", "- ++     --   ++",
+       "@@ -1,4 +1,5 @@\n-A\n B\n+C\n+D\n C\n D\n"
+       "@@ -6,7 +7,7 @@\n F\n G\n-H\n-I\n J\n K\n L\n+J\n+K\n"},
+      {}};
+  for (const Case* c = kCases; c->left; ++c) {
+    EXPECT_TRUE(c->expected_edits ==
+                EditsToString(CalculateOptimalEdits(CharsToIndices(c->left),
+                                                    CharsToIndices(c->right))))
+        << "Left <" << c->left << "> Right <" << c->right << "> Edits <"
+        << EditsToString(CalculateOptimalEdits(
+               CharsToIndices(c->left), CharsToIndices(c->right))) << ">";
+    EXPECT_TRUE(c->expected_diff == CreateUnifiedDiff(CharsToLines(c->left),
+                                                      CharsToLines(c->right)))
+        << "Left <" << c->left << "> Right <" << c->right << "> Diff <"
+        << CreateUnifiedDiff(CharsToLines(c->left), CharsToLines(c->right))
+        << ">";
+  }
+}
+
+// Tests EqFailure(), used for implementing *EQ* assertions.
+TEST(AssertionTest, EqFailure) {
+  const std::string foo_val("5"), bar_val("6");
+  const std::string msg1(
+      EqFailure("foo", "bar", foo_val, bar_val, false)
+      .failure_message());
+  EXPECT_STREQ(
+      "      Expected: foo\n"
+      "      Which is: 5\n"
+      "To be equal to: bar\n"
+      "      Which is: 6",
+      msg1.c_str());
+
+  const std::string msg2(
+      EqFailure("foo", "6", foo_val, bar_val, false)
+      .failure_message());
+  EXPECT_STREQ(
+      "      Expected: foo\n"
+      "      Which is: 5\n"
+      "To be equal to: 6",
+      msg2.c_str());
+
+  const std::string msg3(
+      EqFailure("5", "bar", foo_val, bar_val, false)
+      .failure_message());
+  EXPECT_STREQ(
+      "      Expected: 5\n"
+      "To be equal to: bar\n"
+      "      Which is: 6",
+      msg3.c_str());
+
+  const std::string msg4(
+      EqFailure("5", "6", foo_val, bar_val, false).failure_message());
+  EXPECT_STREQ(
+      "      Expected: 5\n"
+      "To be equal to: 6",
+      msg4.c_str());
+
+  const std::string msg5(
+      EqFailure("foo", "bar",
+                std::string("\"x\""), std::string("\"y\""),
+                true).failure_message());
+  EXPECT_STREQ(
+      "      Expected: foo\n"
+      "      Which is: \"x\"\n"
+      "To be equal to: bar\n"
+      "      Which is: \"y\"\n"
+      "Ignoring case",
+      msg5.c_str());
+}
+
+TEST(AssertionTest, EqFailureWithDiff) {
+  const std::string left(
+      "1\\n2XXX\\n3\\n5\\n6\\n7\\n8\\n9\\n10\\n11\\n12XXX\\n13\\n14\\n15");
+  const std::string right(
+      "1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n11\\n12\\n13\\n14");
+  const std::string msg1(
+      EqFailure("left", "right", left, right, false).failure_message());
+  EXPECT_STREQ(
+      "      Expected: left\n"
+      "      Which is: "
+      "1\\n2XXX\\n3\\n5\\n6\\n7\\n8\\n9\\n10\\n11\\n12XXX\\n13\\n14\\n15\n"
+      "To be equal to: right\n"
+      "      Which is: 1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n11\\n12\\n13\\n14\n"
+      "With diff:\n@@ -1,5 +1,6 @@\n 1\n-2XXX\n+2\n 3\n+4\n 5\n 6\n"
+      "@@ -7,8 +8,6 @@\n 8\n 9\n-10\n 11\n-12XXX\n+12\n 13\n 14\n-15\n",
+      msg1.c_str());
+}
+
+// Tests AppendUserMessage(), used for implementing the *EQ* macros.
+TEST(AssertionTest, AppendUserMessage) {
+  const std::string foo("foo");
+
+  Message msg;
+  EXPECT_STREQ("foo",
+               AppendUserMessage(foo, msg).c_str());
+
+  msg << "bar";
+  EXPECT_STREQ("foo\nbar",
+               AppendUserMessage(foo, msg).c_str());
+}
+
+#ifdef __BORLANDC__
+// Silences warnings: "Condition is always true", "Unreachable code"
+# pragma option push -w-ccc -w-rch
+#endif
+
+// Tests ASSERT_TRUE.
+TEST(AssertionTest, ASSERT_TRUE) {
+  ASSERT_TRUE(2 > 1);  // NOLINT
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(2 < 1),
+                       "2 < 1");
+}
+
+// Tests ASSERT_TRUE(predicate) for predicates returning AssertionResult.
+TEST(AssertionTest, AssertTrueWithAssertionResult) {
+  ASSERT_TRUE(ResultIsEven(2));
+#ifndef __BORLANDC__
+  // ICE's in C++Builder.
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(ResultIsEven(3)),
+                       "Value of: ResultIsEven(3)\n"
+                       "  Actual: false (3 is odd)\n"
+                       "Expected: true");
+#endif
+  ASSERT_TRUE(ResultIsEvenNoExplanation(2));
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(ResultIsEvenNoExplanation(3)),
+                       "Value of: ResultIsEvenNoExplanation(3)\n"
+                       "  Actual: false (3 is odd)\n"
+                       "Expected: true");
+}
+
+// Tests ASSERT_FALSE.
+TEST(AssertionTest, ASSERT_FALSE) {
+  ASSERT_FALSE(2 < 1);  // NOLINT
+  EXPECT_FATAL_FAILURE(ASSERT_FALSE(2 > 1),
+                       "Value of: 2 > 1\n"
+                       "  Actual: true\n"
+                       "Expected: false");
+}
+
+// Tests ASSERT_FALSE(predicate) for predicates returning AssertionResult.
+TEST(AssertionTest, AssertFalseWithAssertionResult) {
+  ASSERT_FALSE(ResultIsEven(3));
+#ifndef __BORLANDC__
+  // ICE's in C++Builder.
+  EXPECT_FATAL_FAILURE(ASSERT_FALSE(ResultIsEven(2)),
+                       "Value of: ResultIsEven(2)\n"
+                       "  Actual: true (2 is even)\n"
+                       "Expected: false");
+#endif
+  ASSERT_FALSE(ResultIsEvenNoExplanation(3));
+  EXPECT_FATAL_FAILURE(ASSERT_FALSE(ResultIsEvenNoExplanation(2)),
+                       "Value of: ResultIsEvenNoExplanation(2)\n"
+                       "  Actual: true\n"
+                       "Expected: false");
+}
+
+#ifdef __BORLANDC__
+// Restores warnings after previous "#pragma option push" supressed them
+# pragma option pop
+#endif
+
+// Tests using ASSERT_EQ on double values.  The purpose is to make
+// sure that the specialization we did for integer and anonymous enums
+// isn't used for double arguments.
+TEST(ExpectTest, ASSERT_EQ_Double) {
+  // A success.
+  ASSERT_EQ(5.6, 5.6);
+
+  // A failure.
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(5.1, 5.2),
+                       "5.1");
+}
+
+// Tests ASSERT_EQ.
+TEST(AssertionTest, ASSERT_EQ) {
+  ASSERT_EQ(5, 2 + 3);
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(5, 2*3),
+                       "      Expected: 5\n"
+                       "To be equal to: 2*3\n"
+                       "      Which is: 6");
+}
+
+// Tests ASSERT_EQ(NULL, pointer).
+#if GTEST_CAN_COMPARE_NULL
+TEST(AssertionTest, ASSERT_EQ_NULL) {
+  // A success.
+  const char* p = NULL;
+  // Some older GCC versions may issue a spurious waring in this or the next
+  // assertion statement. This warning should not be suppressed with
+  // static_cast since the test verifies the ability to use bare NULL as the
+  // expected parameter to the macro.
+  ASSERT_EQ(NULL, p);
+
+  // A failure.
+  static int n = 0;
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(NULL, &n),
+                       "To be equal to: &n\n");
+}
+#endif  // GTEST_CAN_COMPARE_NULL
+
+// Tests ASSERT_EQ(0, non_pointer).  Since the literal 0 can be
+// treated as a null pointer by the compiler, we need to make sure
+// that ASSERT_EQ(0, non_pointer) isn't interpreted by Google Test as
+// ASSERT_EQ(static_cast<void*>(NULL), non_pointer).
+TEST(ExpectTest, ASSERT_EQ_0) {
+  int n = 0;
+
+  // A success.
+  ASSERT_EQ(0, n);
+
+  // A failure.
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(0, 5.6),
+                       "Expected: 0");
+}
+
+// Tests ASSERT_NE.
+TEST(AssertionTest, ASSERT_NE) {
+  ASSERT_NE(6, 7);
+  EXPECT_FATAL_FAILURE(ASSERT_NE('a', 'a'),
+                       "Expected: ('a') != ('a'), "
+                       "actual: 'a' (97, 0x61) vs 'a' (97, 0x61)");
+}
+
+// Tests ASSERT_LE.
+TEST(AssertionTest, ASSERT_LE) {
+  ASSERT_LE(2, 3);
+  ASSERT_LE(2, 2);
+  EXPECT_FATAL_FAILURE(ASSERT_LE(2, 0),
+                       "Expected: (2) <= (0), actual: 2 vs 0");
+}
+
+// Tests ASSERT_LT.
+TEST(AssertionTest, ASSERT_LT) {
+  ASSERT_LT(2, 3);
+  EXPECT_FATAL_FAILURE(ASSERT_LT(2, 2),
+                       "Expected: (2) < (2), actual: 2 vs 2");
+}
+
+// Tests ASSERT_GE.
+TEST(AssertionTest, ASSERT_GE) {
+  ASSERT_GE(2, 1);
+  ASSERT_GE(2, 2);
+  EXPECT_FATAL_FAILURE(ASSERT_GE(2, 3),
+                       "Expected: (2) >= (3), actual: 2 vs 3");
+}
+
+// Tests ASSERT_GT.
+TEST(AssertionTest, ASSERT_GT) {
+  ASSERT_GT(2, 1);
+  EXPECT_FATAL_FAILURE(ASSERT_GT(2, 2),
+                       "Expected: (2) > (2), actual: 2 vs 2");
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+void ThrowNothing() {}
+
+// Tests ASSERT_THROW.
+TEST(AssertionTest, ASSERT_THROW) {
+  ASSERT_THROW(ThrowAnInteger(), int);
+
+# ifndef __BORLANDC__
+
+  // ICE's in C++Builder 2007 and 2009.
+  EXPECT_FATAL_FAILURE(
+      ASSERT_THROW(ThrowAnInteger(), bool),
+      "Expected: ThrowAnInteger() throws an exception of type bool.\n"
+      "  Actual: it throws a different type.");
+# endif
+
+  EXPECT_FATAL_FAILURE(
+      ASSERT_THROW(ThrowNothing(), bool),
+      "Expected: ThrowNothing() throws an exception of type bool.\n"
+      "  Actual: it throws nothing.");
+}
+
+// Tests ASSERT_NO_THROW.
+TEST(AssertionTest, ASSERT_NO_THROW) {
+  ASSERT_NO_THROW(ThrowNothing());
+  EXPECT_FATAL_FAILURE(ASSERT_NO_THROW(ThrowAnInteger()),
+                       "Expected: ThrowAnInteger() doesn't throw an exception."
+                       "\n  Actual: it throws.");
+}
+
+// Tests ASSERT_ANY_THROW.
+TEST(AssertionTest, ASSERT_ANY_THROW) {
+  ASSERT_ANY_THROW(ThrowAnInteger());
+  EXPECT_FATAL_FAILURE(
+      ASSERT_ANY_THROW(ThrowNothing()),
+      "Expected: ThrowNothing() throws an exception.\n"
+      "  Actual: it doesn't.");
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// Makes sure we deal with the precedence of <<.  This test should
+// compile.
+TEST(AssertionTest, AssertPrecedence) {
+  ASSERT_EQ(1 < 2, true);
+  bool false_value = false;
+  ASSERT_EQ(true && false_value, false);
+}
+
+// A subroutine used by the following test.
+void TestEq1(int x) {
+  ASSERT_EQ(1, x);
+}
+
+// Tests calling a test subroutine that's not part of a fixture.
+TEST(AssertionTest, NonFixtureSubroutine) {
+  EXPECT_FATAL_FAILURE(TestEq1(2),
+                       "To be equal to: x");
+}
+
+// An uncopyable class.
+class Uncopyable {
+ public:
+  explicit Uncopyable(int a_value) : value_(a_value) {}
+
+  int value() const { return value_; }
+  bool operator==(const Uncopyable& rhs) const {
+    return value() == rhs.value();
+  }
+ private:
+  // This constructor deliberately has no implementation, as we don't
+  // want this class to be copyable.
+  Uncopyable(const Uncopyable&);  // NOLINT
+
+  int value_;
+};
+
+::std::ostream& operator<<(::std::ostream& os, const Uncopyable& value) {
+  return os << value.value();
+}
+
+
+bool IsPositiveUncopyable(const Uncopyable& x) {
+  return x.value() > 0;
+}
+
+// A subroutine used by the following test.
+void TestAssertNonPositive() {
+  Uncopyable y(-1);
+  ASSERT_PRED1(IsPositiveUncopyable, y);
+}
+// A subroutine used by the following test.
+void TestAssertEqualsUncopyable() {
+  Uncopyable x(5);
+  Uncopyable y(-1);
+  ASSERT_EQ(x, y);
+}
+
+// Tests that uncopyable objects can be used in assertions.
+TEST(AssertionTest, AssertWorksWithUncopyableObject) {
+  Uncopyable x(5);
+  ASSERT_PRED1(IsPositiveUncopyable, x);
+  ASSERT_EQ(x, x);
+  EXPECT_FATAL_FAILURE(TestAssertNonPositive(),
+    "IsPositiveUncopyable(y) evaluates to false, where\ny evaluates to -1");
+  EXPECT_FATAL_FAILURE(TestAssertEqualsUncopyable(),
+    "Expected: x\n      Which is: 5\nTo be equal to: y\n      Which is: -1");
+}
+
+// Tests that uncopyable objects can be used in expects.
+TEST(AssertionTest, ExpectWorksWithUncopyableObject) {
+  Uncopyable x(5);
+  EXPECT_PRED1(IsPositiveUncopyable, x);
+  Uncopyable y(-1);
+  EXPECT_NONFATAL_FAILURE(EXPECT_PRED1(IsPositiveUncopyable, y),
+    "IsPositiveUncopyable(y) evaluates to false, where\ny evaluates to -1");
+  EXPECT_EQ(x, x);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(x, y),
+    "Expected: x\n      Which is: 5\nTo be equal to: y\n      Which is: -1");
+}
+
+enum NamedEnum {
+  kE1 = 0,
+  kE2 = 1
+};
+
+TEST(AssertionTest, NamedEnum) {
+  EXPECT_EQ(kE1, kE1);
+  EXPECT_LT(kE1, kE2);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(kE1, kE2), "Which is: 0");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(kE1, kE2), "Which is: 1");
+}
+
+// The version of gcc used in XCode 2.2 has a bug and doesn't allow
+// anonymous enums in assertions.  Therefore the following test is not
+// done on Mac.
+// Sun Studio and HP aCC also reject this code.
+#if !GTEST_OS_MAC && !defined(__SUNPRO_CC) && !defined(__HP_aCC)
+
+// Tests using assertions with anonymous enums.
+enum {
+  kCaseA = -1,
+
+# if GTEST_OS_LINUX
+
+  // We want to test the case where the size of the anonymous enum is
+  // larger than sizeof(int), to make sure our implementation of the
+  // assertions doesn't truncate the enums.  However, MSVC
+  // (incorrectly) doesn't allow an enum value to exceed the range of
+  // an int, so this has to be conditionally compiled.
+  //
+  // On Linux, kCaseB and kCaseA have the same value when truncated to
+  // int size.  We want to test whether this will confuse the
+  // assertions.
+  kCaseB = testing::internal::kMaxBiggestInt,
+
+# else
+
+  kCaseB = INT_MAX,
+
+# endif  // GTEST_OS_LINUX
+
+  kCaseC = 42
+};
+
+TEST(AssertionTest, AnonymousEnum) {
+# if GTEST_OS_LINUX
+
+  EXPECT_EQ(static_cast<int>(kCaseA), static_cast<int>(kCaseB));
+
+# endif  // GTEST_OS_LINUX
+
+  EXPECT_EQ(kCaseA, kCaseA);
+  EXPECT_NE(kCaseA, kCaseB);
+  EXPECT_LT(kCaseA, kCaseB);
+  EXPECT_LE(kCaseA, kCaseB);
+  EXPECT_GT(kCaseB, kCaseA);
+  EXPECT_GE(kCaseA, kCaseA);
+  EXPECT_NONFATAL_FAILURE(EXPECT_GE(kCaseA, kCaseB),
+                          "(kCaseA) >= (kCaseB)");
+  EXPECT_NONFATAL_FAILURE(EXPECT_GE(kCaseA, kCaseC),
+                          "-1 vs 42");
+
+  ASSERT_EQ(kCaseA, kCaseA);
+  ASSERT_NE(kCaseA, kCaseB);
+  ASSERT_LT(kCaseA, kCaseB);
+  ASSERT_LE(kCaseA, kCaseB);
+  ASSERT_GT(kCaseB, kCaseA);
+  ASSERT_GE(kCaseA, kCaseA);
+
+# ifndef __BORLANDC__
+
+  // ICE's in C++Builder.
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(kCaseA, kCaseB),
+                       "To be equal to: kCaseB");
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(kCaseA, kCaseC),
+                       "Which is: 42");
+# endif
+
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(kCaseA, kCaseC),
+                       "Which is: -1");
+}
+
+#endif  // !GTEST_OS_MAC && !defined(__SUNPRO_CC)
+
+#if GTEST_OS_WINDOWS
+
+static HRESULT UnexpectedHRESULTFailure() {
+  return E_UNEXPECTED;
+}
+
+static HRESULT OkHRESULTSuccess() {
+  return S_OK;
+}
+
+static HRESULT FalseHRESULTSuccess() {
+  return S_FALSE;
+}
+
+// HRESULT assertion tests test both zero and non-zero
+// success codes as well as failure message for each.
+//
+// Windows CE doesn't support message texts.
+TEST(HRESULTAssertionTest, EXPECT_HRESULT_SUCCEEDED) {
+  EXPECT_HRESULT_SUCCEEDED(S_OK);
+  EXPECT_HRESULT_SUCCEEDED(S_FALSE);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_HRESULT_SUCCEEDED(UnexpectedHRESULTFailure()),
+    "Expected: (UnexpectedHRESULTFailure()) succeeds.\n"
+    "  Actual: 0x8000FFFF");
+}
+
+TEST(HRESULTAssertionTest, ASSERT_HRESULT_SUCCEEDED) {
+  ASSERT_HRESULT_SUCCEEDED(S_OK);
+  ASSERT_HRESULT_SUCCEEDED(S_FALSE);
+
+  EXPECT_FATAL_FAILURE(ASSERT_HRESULT_SUCCEEDED(UnexpectedHRESULTFailure()),
+    "Expected: (UnexpectedHRESULTFailure()) succeeds.\n"
+    "  Actual: 0x8000FFFF");
+}
+
+TEST(HRESULTAssertionTest, EXPECT_HRESULT_FAILED) {
+  EXPECT_HRESULT_FAILED(E_UNEXPECTED);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_HRESULT_FAILED(OkHRESULTSuccess()),
+    "Expected: (OkHRESULTSuccess()) fails.\n"
+    "  Actual: 0x0");
+  EXPECT_NONFATAL_FAILURE(EXPECT_HRESULT_FAILED(FalseHRESULTSuccess()),
+    "Expected: (FalseHRESULTSuccess()) fails.\n"
+    "  Actual: 0x1");
+}
+
+TEST(HRESULTAssertionTest, ASSERT_HRESULT_FAILED) {
+  ASSERT_HRESULT_FAILED(E_UNEXPECTED);
+
+# ifndef __BORLANDC__
+
+  // ICE's in C++Builder 2007 and 2009.
+  EXPECT_FATAL_FAILURE(ASSERT_HRESULT_FAILED(OkHRESULTSuccess()),
+    "Expected: (OkHRESULTSuccess()) fails.\n"
+    "  Actual: 0x0");
+# endif
+
+  EXPECT_FATAL_FAILURE(ASSERT_HRESULT_FAILED(FalseHRESULTSuccess()),
+    "Expected: (FalseHRESULTSuccess()) fails.\n"
+    "  Actual: 0x1");
+}
+
+// Tests that streaming to the HRESULT macros works.
+TEST(HRESULTAssertionTest, Streaming) {
+  EXPECT_HRESULT_SUCCEEDED(S_OK) << "unexpected failure";
+  ASSERT_HRESULT_SUCCEEDED(S_OK) << "unexpected failure";
+  EXPECT_HRESULT_FAILED(E_UNEXPECTED) << "unexpected failure";
+  ASSERT_HRESULT_FAILED(E_UNEXPECTED) << "unexpected failure";
+
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_HRESULT_SUCCEEDED(E_UNEXPECTED) << "expected failure",
+      "expected failure");
+
+# ifndef __BORLANDC__
+
+  // ICE's in C++Builder 2007 and 2009.
+  EXPECT_FATAL_FAILURE(
+      ASSERT_HRESULT_SUCCEEDED(E_UNEXPECTED) << "expected failure",
+      "expected failure");
+# endif
+
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_HRESULT_FAILED(S_OK) << "expected failure",
+      "expected failure");
+
+  EXPECT_FATAL_FAILURE(
+      ASSERT_HRESULT_FAILED(S_OK) << "expected failure",
+      "expected failure");
+}
+
+#endif  // GTEST_OS_WINDOWS
+
+#ifdef __BORLANDC__
+// Silences warnings: "Condition is always true", "Unreachable code"
+# pragma option push -w-ccc -w-rch
+#endif
+
+// Tests that the assertion macros behave like single statements.
+TEST(AssertionSyntaxTest, BasicAssertionsBehavesLikeSingleStatement) {
+  if (AlwaysFalse())
+    ASSERT_TRUE(false) << "This should never be executed; "
+                          "It's a compilation test only.";
+
+  if (AlwaysTrue())
+    EXPECT_FALSE(false);
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    ASSERT_LT(1, 3);
+
+  if (AlwaysFalse())
+    ;  // NOLINT
+  else
+    EXPECT_GT(3, 2) << "";
+}
+
+#if GTEST_HAS_EXCEPTIONS
+// Tests that the compiler will not complain about unreachable code in the
+// EXPECT_THROW/EXPECT_ANY_THROW/EXPECT_NO_THROW macros.
+TEST(ExpectThrowTest, DoesNotGenerateUnreachableCodeWarning) {
+  int n = 0;
+
+  EXPECT_THROW(throw 1, int);
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW(n++, int), "");
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW(throw 1, const char*), "");
+  EXPECT_NO_THROW(n++);
+  EXPECT_NONFATAL_FAILURE(EXPECT_NO_THROW(throw 1), "");
+  EXPECT_ANY_THROW(throw 1);
+  EXPECT_NONFATAL_FAILURE(EXPECT_ANY_THROW(n++), "");
+}
+
+TEST(AssertionSyntaxTest, ExceptionAssertionsBehavesLikeSingleStatement) {
+  if (AlwaysFalse())
+    EXPECT_THROW(ThrowNothing(), bool);
+
+  if (AlwaysTrue())
+    EXPECT_THROW(ThrowAnInteger(), int);
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    EXPECT_NO_THROW(ThrowAnInteger());
+
+  if (AlwaysTrue())
+    EXPECT_NO_THROW(ThrowNothing());
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    EXPECT_ANY_THROW(ThrowNothing());
+
+  if (AlwaysTrue())
+    EXPECT_ANY_THROW(ThrowAnInteger());
+  else
+    ;  // NOLINT
+}
+#endif  // GTEST_HAS_EXCEPTIONS
+
+TEST(AssertionSyntaxTest, NoFatalFailureAssertionsBehavesLikeSingleStatement) {
+  if (AlwaysFalse())
+    EXPECT_NO_FATAL_FAILURE(FAIL()) << "This should never be executed. "
+                                    << "It's a compilation test only.";
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    ASSERT_NO_FATAL_FAILURE(FAIL()) << "";
+  else
+    ;  // NOLINT
+
+  if (AlwaysTrue())
+    EXPECT_NO_FATAL_FAILURE(SUCCEED());
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    ;  // NOLINT
+  else
+    ASSERT_NO_FATAL_FAILURE(SUCCEED());
+}
+
+// Tests that the assertion macros work well with switch statements.
+TEST(AssertionSyntaxTest, WorksWithSwitch) {
+  switch (0) {
+    case 1:
+      break;
+    default:
+      ASSERT_TRUE(true);
+  }
+
+  switch (0)
+    case 0:
+      EXPECT_FALSE(false) << "EXPECT_FALSE failed in switch case";
+
+  // Binary assertions are implemented using a different code path
+  // than the Boolean assertions.  Hence we test them separately.
+  switch (0) {
+    case 1:
+    default:
+      ASSERT_EQ(1, 1) << "ASSERT_EQ failed in default switch handler";
+  }
+
+  switch (0)
+    case 0:
+      EXPECT_NE(1, 2);
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+void ThrowAString() {
+    throw "std::string";
+}
+
+// Test that the exception assertion macros compile and work with const
+// type qualifier.
+TEST(AssertionSyntaxTest, WorksWithConst) {
+    ASSERT_THROW(ThrowAString(), const char*);
+
+    EXPECT_THROW(ThrowAString(), const char*);
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+}  // namespace
+
+namespace testing {
+
+// Tests that Google Test tracks SUCCEED*.
+TEST(SuccessfulAssertionTest, SUCCEED) {
+  SUCCEED();
+  SUCCEED() << "OK";
+  EXPECT_EQ(2, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+// Tests that Google Test doesn't track successful EXPECT_*.
+TEST(SuccessfulAssertionTest, EXPECT) {
+  EXPECT_TRUE(true);
+  EXPECT_EQ(0, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+// Tests that Google Test doesn't track successful EXPECT_STR*.
+TEST(SuccessfulAssertionTest, EXPECT_STR) {
+  EXPECT_STREQ("", "");
+  EXPECT_EQ(0, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+// Tests that Google Test doesn't track successful ASSERT_*.
+TEST(SuccessfulAssertionTest, ASSERT) {
+  ASSERT_TRUE(true);
+  EXPECT_EQ(0, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+// Tests that Google Test doesn't track successful ASSERT_STR*.
+TEST(SuccessfulAssertionTest, ASSERT_STR) {
+  ASSERT_STREQ("", "");
+  EXPECT_EQ(0, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+}  // namespace testing
+
+namespace {
+
+// Tests the message streaming variation of assertions.
+
+TEST(AssertionWithMessageTest, EXPECT) {
+  EXPECT_EQ(1, 1) << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_NE(1, 1) << "Expected failure #1.",
+                          "Expected failure #1");
+  EXPECT_LE(1, 2) << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(1, 0) << "Expected failure #2.",
+                          "Expected failure #2.");
+  EXPECT_GE(1, 0) << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_GT(1, 2) << "Expected failure #3.",
+                          "Expected failure #3.");
+
+  EXPECT_STREQ("1", "1") << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRNE("1", "1") << "Expected failure #4.",
+                          "Expected failure #4.");
+  EXPECT_STRCASEEQ("a", "A") << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRCASENE("a", "A") << "Expected failure #5.",
+                          "Expected failure #5.");
+
+  EXPECT_FLOAT_EQ(1, 1) << "This should succeed.";
+  EXPECT_NONFATAL_FAILURE(EXPECT_DOUBLE_EQ(1, 1.2) << "Expected failure #6.",
+                          "Expected failure #6.");
+  EXPECT_NEAR(1, 1.1, 0.2) << "This should succeed.";
+}
+
+TEST(AssertionWithMessageTest, ASSERT) {
+  ASSERT_EQ(1, 1) << "This should succeed.";
+  ASSERT_NE(1, 2) << "This should succeed.";
+  ASSERT_LE(1, 2) << "This should succeed.";
+  ASSERT_LT(1, 2) << "This should succeed.";
+  ASSERT_GE(1, 0) << "This should succeed.";
+  EXPECT_FATAL_FAILURE(ASSERT_GT(1, 2) << "Expected failure.",
+                       "Expected failure.");
+}
+
+TEST(AssertionWithMessageTest, ASSERT_STR) {
+  ASSERT_STREQ("1", "1") << "This should succeed.";
+  ASSERT_STRNE("1", "2") << "This should succeed.";
+  ASSERT_STRCASEEQ("a", "A") << "This should succeed.";
+  EXPECT_FATAL_FAILURE(ASSERT_STRCASENE("a", "A") << "Expected failure.",
+                       "Expected failure.");
+}
+
+TEST(AssertionWithMessageTest, ASSERT_FLOATING) {
+  ASSERT_FLOAT_EQ(1, 1) << "This should succeed.";
+  ASSERT_DOUBLE_EQ(1, 1) << "This should succeed.";
+  EXPECT_FATAL_FAILURE(ASSERT_NEAR(1,1.2, 0.1) << "Expect failure.",  // NOLINT
+                       "Expect failure.");
+  // To work around a bug in gcc 2.95.0, there is intentionally no
+  // space after the first comma in the previous statement.
+}
+
+// Tests using ASSERT_FALSE with a streamed message.
+TEST(AssertionWithMessageTest, ASSERT_FALSE) {
+  ASSERT_FALSE(false) << "This shouldn't fail.";
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_FALSE(true) << "Expected failure: " << 2 << " > " << 1
+                       << " evaluates to " << true;
+  }, "Expected failure");
+}
+
+// Tests using FAIL with a streamed message.
+TEST(AssertionWithMessageTest, FAIL) {
+  EXPECT_FATAL_FAILURE(FAIL() << 0,
+                       "0");
+}
+
+// Tests using SUCCEED with a streamed message.
+TEST(AssertionWithMessageTest, SUCCEED) {
+  SUCCEED() << "Success == " << 1;
+}
+
+// Tests using ASSERT_TRUE with a streamed message.
+TEST(AssertionWithMessageTest, ASSERT_TRUE) {
+  ASSERT_TRUE(true) << "This should succeed.";
+  ASSERT_TRUE(true) << true;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_TRUE(false) << static_cast<const char *>(NULL)
+                       << static_cast<char *>(NULL);
+  }, "(null)(null)");
+}
+
+#if GTEST_OS_WINDOWS
+// Tests using wide strings in assertion messages.
+TEST(AssertionWithMessageTest, WideStringMessage) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_TRUE(false) << L"This failure is expected.\x8119";
+  }, "This failure is expected.");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_EQ(1, 2) << "This failure is "
+                    << L"expected too.\x8120";
+  }, "This failure is expected too.");
+}
+#endif  // GTEST_OS_WINDOWS
+
+// Tests EXPECT_TRUE.
+TEST(ExpectTest, EXPECT_TRUE) {
+  EXPECT_TRUE(true) << "Intentional success";
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(false) << "Intentional failure #1.",
+                          "Intentional failure #1.");
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(false) << "Intentional failure #2.",
+                          "Intentional failure #2.");
+  EXPECT_TRUE(2 > 1);  // NOLINT
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(2 < 1),
+                          "Value of: 2 < 1\n"
+                          "  Actual: false\n"
+                          "Expected: true");
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(2 > 3),
+                          "2 > 3");
+}
+
+// Tests EXPECT_TRUE(predicate) for predicates returning AssertionResult.
+TEST(ExpectTest, ExpectTrueWithAssertionResult) {
+  EXPECT_TRUE(ResultIsEven(2));
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(ResultIsEven(3)),
+                          "Value of: ResultIsEven(3)\n"
+                          "  Actual: false (3 is odd)\n"
+                          "Expected: true");
+  EXPECT_TRUE(ResultIsEvenNoExplanation(2));
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(ResultIsEvenNoExplanation(3)),
+                          "Value of: ResultIsEvenNoExplanation(3)\n"
+                          "  Actual: false (3 is odd)\n"
+                          "Expected: true");
+}
+
+// Tests EXPECT_FALSE with a streamed message.
+TEST(ExpectTest, EXPECT_FALSE) {
+  EXPECT_FALSE(2 < 1);  // NOLINT
+  EXPECT_FALSE(false) << "Intentional success";
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(true) << "Intentional failure #1.",
+                          "Intentional failure #1.");
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(true) << "Intentional failure #2.",
+                          "Intentional failure #2.");
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(2 > 1),
+                          "Value of: 2 > 1\n"
+                          "  Actual: true\n"
+                          "Expected: false");
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(2 < 3),
+                          "2 < 3");
+}
+
+// Tests EXPECT_FALSE(predicate) for predicates returning AssertionResult.
+TEST(ExpectTest, ExpectFalseWithAssertionResult) {
+  EXPECT_FALSE(ResultIsEven(3));
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(ResultIsEven(2)),
+                          "Value of: ResultIsEven(2)\n"
+                          "  Actual: true (2 is even)\n"
+                          "Expected: false");
+  EXPECT_FALSE(ResultIsEvenNoExplanation(3));
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(ResultIsEvenNoExplanation(2)),
+                          "Value of: ResultIsEvenNoExplanation(2)\n"
+                          "  Actual: true\n"
+                          "Expected: false");
+}
+
+#ifdef __BORLANDC__
+// Restores warnings after previous "#pragma option push" supressed them
+# pragma option pop
+#endif
+
+// Tests EXPECT_EQ.
+TEST(ExpectTest, EXPECT_EQ) {
+  EXPECT_EQ(5, 2 + 3);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(5, 2*3),
+                          "      Expected: 5\n"
+                          "To be equal to: 2*3\n"
+                          "      Which is: 6");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(5, 2 - 3),
+                          "2 - 3");
+}
+
+// Tests using EXPECT_EQ on double values.  The purpose is to make
+// sure that the specialization we did for integer and anonymous enums
+// isn't used for double arguments.
+TEST(ExpectTest, EXPECT_EQ_Double) {
+  // A success.
+  EXPECT_EQ(5.6, 5.6);
+
+  // A failure.
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(5.1, 5.2),
+                          "5.1");
+}
+
+#if GTEST_CAN_COMPARE_NULL
+// Tests EXPECT_EQ(NULL, pointer).
+TEST(ExpectTest, EXPECT_EQ_NULL) {
+  // A success.
+  const char* p = NULL;
+  // Some older GCC versions may issue a spurious warning in this or the next
+  // assertion statement. This warning should not be suppressed with
+  // static_cast since the test verifies the ability to use bare NULL as the
+  // expected parameter to the macro.
+  EXPECT_EQ(NULL, p);
+
+  // A failure.
+  int n = 0;
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(NULL, &n),
+                          "To be equal to: &n\n");
+}
+#endif  // GTEST_CAN_COMPARE_NULL
+
+// Tests EXPECT_EQ(0, non_pointer).  Since the literal 0 can be
+// treated as a null pointer by the compiler, we need to make sure
+// that EXPECT_EQ(0, non_pointer) isn't interpreted by Google Test as
+// EXPECT_EQ(static_cast<void*>(NULL), non_pointer).
+TEST(ExpectTest, EXPECT_EQ_0) {
+  int n = 0;
+
+  // A success.
+  EXPECT_EQ(0, n);
+
+  // A failure.
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(0, 5.6),
+                          "Expected: 0");
+}
+
+// Tests EXPECT_NE.
+TEST(ExpectTest, EXPECT_NE) {
+  EXPECT_NE(6, 7);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_NE('a', 'a'),
+                          "Expected: ('a') != ('a'), "
+                          "actual: 'a' (97, 0x61) vs 'a' (97, 0x61)");
+  EXPECT_NONFATAL_FAILURE(EXPECT_NE(2, 2),
+                          "2");
+  char* const p0 = NULL;
+  EXPECT_NONFATAL_FAILURE(EXPECT_NE(p0, p0),
+                          "p0");
+  // Only way to get the Nokia compiler to compile the cast
+  // is to have a separate void* variable first. Putting
+  // the two casts on the same line doesn't work, neither does
+  // a direct C-style to char*.
+  void* pv1 = (void*)0x1234;  // NOLINT
+  char* const p1 = reinterpret_cast<char*>(pv1);
+  EXPECT_NONFATAL_FAILURE(EXPECT_NE(p1, p1),
+                          "p1");
+}
+
+// Tests EXPECT_LE.
+TEST(ExpectTest, EXPECT_LE) {
+  EXPECT_LE(2, 3);
+  EXPECT_LE(2, 2);
+  EXPECT_NONFATAL_FAILURE(EXPECT_LE(2, 0),
+                          "Expected: (2) <= (0), actual: 2 vs 0");
+  EXPECT_NONFATAL_FAILURE(EXPECT_LE(1.1, 0.9),
+                          "(1.1) <= (0.9)");
+}
+
+// Tests EXPECT_LT.
+TEST(ExpectTest, EXPECT_LT) {
+  EXPECT_LT(2, 3);
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(2, 2),
+                          "Expected: (2) < (2), actual: 2 vs 2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(2, 1),
+                          "(2) < (1)");
+}
+
+// Tests EXPECT_GE.
+TEST(ExpectTest, EXPECT_GE) {
+  EXPECT_GE(2, 1);
+  EXPECT_GE(2, 2);
+  EXPECT_NONFATAL_FAILURE(EXPECT_GE(2, 3),
+                          "Expected: (2) >= (3), actual: 2 vs 3");
+  EXPECT_NONFATAL_FAILURE(EXPECT_GE(0.9, 1.1),
+                          "(0.9) >= (1.1)");
+}
+
+// Tests EXPECT_GT.
+TEST(ExpectTest, EXPECT_GT) {
+  EXPECT_GT(2, 1);
+  EXPECT_NONFATAL_FAILURE(EXPECT_GT(2, 2),
+                          "Expected: (2) > (2), actual: 2 vs 2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_GT(2, 3),
+                          "(2) > (3)");
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+// Tests EXPECT_THROW.
+TEST(ExpectTest, EXPECT_THROW) {
+  EXPECT_THROW(ThrowAnInteger(), int);
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW(ThrowAnInteger(), bool),
+                          "Expected: ThrowAnInteger() throws an exception of "
+                          "type bool.\n  Actual: it throws a different type.");
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_THROW(ThrowNothing(), bool),
+      "Expected: ThrowNothing() throws an exception of type bool.\n"
+      "  Actual: it throws nothing.");
+}
+
+// Tests EXPECT_NO_THROW.
+TEST(ExpectTest, EXPECT_NO_THROW) {
+  EXPECT_NO_THROW(ThrowNothing());
+  EXPECT_NONFATAL_FAILURE(EXPECT_NO_THROW(ThrowAnInteger()),
+                          "Expected: ThrowAnInteger() doesn't throw an "
+                          "exception.\n  Actual: it throws.");
+}
+
+// Tests EXPECT_ANY_THROW.
+TEST(ExpectTest, EXPECT_ANY_THROW) {
+  EXPECT_ANY_THROW(ThrowAnInteger());
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_ANY_THROW(ThrowNothing()),
+      "Expected: ThrowNothing() throws an exception.\n"
+      "  Actual: it doesn't.");
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// Make sure we deal with the precedence of <<.
+TEST(ExpectTest, ExpectPrecedence) {
+  EXPECT_EQ(1 < 2, true);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(true, true && false),
+                          "To be equal to: true && false");
+}
+
+
+// Tests the StreamableToString() function.
+
+// Tests using StreamableToString() on a scalar.
+TEST(StreamableToStringTest, Scalar) {
+  EXPECT_STREQ("5", StreamableToString(5).c_str());
+}
+
+// Tests using StreamableToString() on a non-char pointer.
+TEST(StreamableToStringTest, Pointer) {
+  int n = 0;
+  int* p = &n;
+  EXPECT_STRNE("(null)", StreamableToString(p).c_str());
+}
+
+// Tests using StreamableToString() on a NULL non-char pointer.
+TEST(StreamableToStringTest, NullPointer) {
+  int* p = NULL;
+  EXPECT_STREQ("(null)", StreamableToString(p).c_str());
+}
+
+// Tests using StreamableToString() on a C string.
+TEST(StreamableToStringTest, CString) {
+  EXPECT_STREQ("Foo", StreamableToString("Foo").c_str());
+}
+
+// Tests using StreamableToString() on a NULL C string.
+TEST(StreamableToStringTest, NullCString) {
+  char* p = NULL;
+  EXPECT_STREQ("(null)", StreamableToString(p).c_str());
+}
+
+// Tests using streamable values as assertion messages.
+
+// Tests using std::string as an assertion message.
+TEST(StreamableTest, string) {
+  static const std::string str(
+      "This failure message is a std::string, and is expected.");
+  EXPECT_FATAL_FAILURE(FAIL() << str,
+                       str.c_str());
+}
+
+// Tests that we can output strings containing embedded NULs.
+// Limited to Linux because we can only do this with std::string's.
+TEST(StreamableTest, stringWithEmbeddedNUL) {
+  static const char char_array_with_nul[] =
+      "Here's a NUL\0 and some more string";
+  static const std::string string_with_nul(char_array_with_nul,
+                                           sizeof(char_array_with_nul)
+                                           - 1);  // drops the trailing NUL
+  EXPECT_FATAL_FAILURE(FAIL() << string_with_nul,
+                       "Here's a NUL\\0 and some more string");
+}
+
+// Tests that we can output a NUL char.
+TEST(StreamableTest, NULChar) {
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    FAIL() << "A NUL" << '\0' << " and some more string";
+  }, "A NUL\\0 and some more string");
+}
+
+// Tests using int as an assertion message.
+TEST(StreamableTest, int) {
+  EXPECT_FATAL_FAILURE(FAIL() << 900913,
+                       "900913");
+}
+
+// Tests using NULL char pointer as an assertion message.
+//
+// In MSVC, streaming a NULL char * causes access violation.  Google Test
+// implemented a workaround (substituting "(null)" for NULL).  This
+// tests whether the workaround works.
+TEST(StreamableTest, NullCharPtr) {
+  EXPECT_FATAL_FAILURE(FAIL() << static_cast<const char*>(NULL),
+                       "(null)");
+}
+
+// Tests that basic IO manipulators (endl, ends, and flush) can be
+// streamed to testing::Message.
+TEST(StreamableTest, BasicIoManip) {
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    FAIL() << "Line 1." << std::endl
+           << "A NUL char " << std::ends << std::flush << " in line 2.";
+  }, "Line 1.\nA NUL char \\0 in line 2.");
+}
+
+// Tests the macros that haven't been covered so far.
+
+void AddFailureHelper(bool* aborted) {
+  *aborted = true;
+  ADD_FAILURE() << "Intentional failure.";
+  *aborted = false;
+}
+
+// Tests ADD_FAILURE.
+TEST(MacroTest, ADD_FAILURE) {
+  bool aborted = true;
+  EXPECT_NONFATAL_FAILURE(AddFailureHelper(&aborted),
+                          "Intentional failure.");
+  EXPECT_FALSE(aborted);
+}
+
+// Tests ADD_FAILURE_AT.
+TEST(MacroTest, ADD_FAILURE_AT) {
+  // Verifies that ADD_FAILURE_AT does generate a nonfatal failure and
+  // the failure message contains the user-streamed part.
+  EXPECT_NONFATAL_FAILURE(ADD_FAILURE_AT("foo.cc", 42) << "Wrong!", "Wrong!");
+
+  // Verifies that the user-streamed part is optional.
+  EXPECT_NONFATAL_FAILURE(ADD_FAILURE_AT("foo.cc", 42), "Failed");
+
+  // Unfortunately, we cannot verify that the failure message contains
+  // the right file path and line number the same way, as
+  // EXPECT_NONFATAL_FAILURE() doesn't get to see the file path and
+  // line number.  Instead, we do that in gtest_output_test_.cc.
+}
+
+// Tests FAIL.
+TEST(MacroTest, FAIL) {
+  EXPECT_FATAL_FAILURE(FAIL(),
+                       "Failed");
+  EXPECT_FATAL_FAILURE(FAIL() << "Intentional failure.",
+                       "Intentional failure.");
+}
+
+// Tests SUCCEED
+TEST(MacroTest, SUCCEED) {
+  SUCCEED();
+  SUCCEED() << "Explicit success.";
+}
+
+// Tests for EXPECT_EQ() and ASSERT_EQ().
+//
+// These tests fail *intentionally*, s.t. the failure messages can be
+// generated and tested.
+//
+// We have different tests for different argument types.
+
+// Tests using bool values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, Bool) {
+  EXPECT_EQ(true,  true);
+  EXPECT_FATAL_FAILURE({
+      bool false_value = false;
+      ASSERT_EQ(false_value, true);
+    }, "To be equal to: true");
+}
+
+// Tests using int values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, Int) {
+  ASSERT_EQ(32, 32);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(32, 33),
+                          "33");
+}
+
+// Tests using time_t values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, Time_T) {
+  EXPECT_EQ(static_cast<time_t>(0),
+            static_cast<time_t>(0));
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(static_cast<time_t>(0),
+                                 static_cast<time_t>(1234)),
+                       "1234");
+}
+
+// Tests using char values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, Char) {
+  ASSERT_EQ('z', 'z');
+  const char ch = 'b';
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ('\0', ch),
+                          "ch");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ('a', ch),
+                          "ch");
+}
+
+// Tests using wchar_t values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, WideChar) {
+  EXPECT_EQ(L'b', L'b');
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(L'\0', L'x'),
+                          "      Expected: L'\0'\n"
+                          "      Which is: L'\0' (0, 0x0)\n"
+                          "To be equal to: L'x'\n"
+                          "      Which is: L'x' (120, 0x78)");
+
+  static wchar_t wchar;
+  wchar = L'b';
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(L'a', wchar),
+                          "wchar");
+  wchar = 0x8119;
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(static_cast<wchar_t>(0x8120), wchar),
+                       "To be equal to: wchar");
+}
+
+// Tests using ::std::string values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, StdString) {
+  // Compares a const char* to an std::string that has identical
+  // content.
+  ASSERT_EQ("Test", ::std::string("Test"));
+
+  // Compares two identical std::strings.
+  static const ::std::string str1("A * in the middle");
+  static const ::std::string str2(str1);
+  EXPECT_EQ(str1, str2);
+
+  // Compares a const char* to an std::string that has different
+  // content
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ("Test", ::std::string("test")),
+                          "\"test\"");
+
+  // Compares an std::string to a char* that has different content.
+  char* const p1 = const_cast<char*>("foo");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(::std::string("bar"), p1),
+                          "p1");
+
+  // Compares two std::strings that have different contents, one of
+  // which having a NUL character in the middle.  This should fail.
+  static ::std::string str3(str1);
+  str3.at(2) = '\0';
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(str1, str3),
+                       "To be equal to: str3\n"
+                       "      Which is: \"A \\0 in the middle\"");
+}
+
+#if GTEST_HAS_STD_WSTRING
+
+// Tests using ::std::wstring values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, StdWideString) {
+  // Compares two identical std::wstrings.
+  const ::std::wstring wstr1(L"A * in the middle");
+  const ::std::wstring wstr2(wstr1);
+  ASSERT_EQ(wstr1, wstr2);
+
+  // Compares an std::wstring to a const wchar_t* that has identical
+  // content.
+  const wchar_t kTestX8119[] = { 'T', 'e', 's', 't', 0x8119, '\0' };
+  EXPECT_EQ(::std::wstring(kTestX8119), kTestX8119);
+
+  // Compares an std::wstring to a const wchar_t* that has different
+  // content.
+  const wchar_t kTestX8120[] = { 'T', 'e', 's', 't', 0x8120, '\0' };
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_EQ(::std::wstring(kTestX8119), kTestX8120);
+  }, "kTestX8120");
+
+  // Compares two std::wstrings that have different contents, one of
+  // which having a NUL character in the middle.
+  ::std::wstring wstr3(wstr1);
+  wstr3.at(2) = L'\0';
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(wstr1, wstr3),
+                          "wstr3");
+
+  // Compares a wchar_t* to an std::wstring that has different
+  // content.
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_EQ(const_cast<wchar_t*>(L"foo"), ::std::wstring(L"bar"));
+  }, "");
+}
+
+#endif  // GTEST_HAS_STD_WSTRING
+
+#if GTEST_HAS_GLOBAL_STRING
+// Tests using ::string values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, GlobalString) {
+  // Compares a const char* to a ::string that has identical content.
+  EXPECT_EQ("Test", ::string("Test"));
+
+  // Compares two identical ::strings.
+  const ::string str1("A * in the middle");
+  const ::string str2(str1);
+  ASSERT_EQ(str1, str2);
+
+  // Compares a ::string to a const char* that has different content.
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(::string("Test"), "test"),
+                          "test");
+
+  // Compares two ::strings that have different contents, one of which
+  // having a NUL character in the middle.
+  ::string str3(str1);
+  str3.at(2) = '\0';
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(str1, str3),
+                          "str3");
+
+  // Compares a ::string to a char* that has different content.
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_EQ(::string("bar"), const_cast<char*>("foo"));
+  }, "");
+}
+
+#endif  // GTEST_HAS_GLOBAL_STRING
+
+#if GTEST_HAS_GLOBAL_WSTRING
+
+// Tests using ::wstring values in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, GlobalWideString) {
+  // Compares two identical ::wstrings.
+  static const ::wstring wstr1(L"A * in the middle");
+  static const ::wstring wstr2(wstr1);
+  EXPECT_EQ(wstr1, wstr2);
+
+  // Compares a const wchar_t* to a ::wstring that has identical content.
+  const wchar_t kTestX8119[] = { 'T', 'e', 's', 't', 0x8119, '\0' };
+  ASSERT_EQ(kTestX8119, ::wstring(kTestX8119));
+
+  // Compares a const wchar_t* to a ::wstring that has different
+  // content.
+  const wchar_t kTestX8120[] = { 'T', 'e', 's', 't', 0x8120, '\0' };
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_EQ(kTestX8120, ::wstring(kTestX8119));
+  }, "Test\\x8119");
+
+  // Compares a wchar_t* to a ::wstring that has different content.
+  wchar_t* const p1 = const_cast<wchar_t*>(L"foo");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p1, ::wstring(L"bar")),
+                          "bar");
+
+  // Compares two ::wstrings that have different contents, one of which
+  // having a NUL character in the middle.
+  static ::wstring wstr3;
+  wstr3 = wstr1;
+  wstr3.at(2) = L'\0';
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(wstr1, wstr3),
+                       "wstr3");
+}
+
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+// Tests using char pointers in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, CharPointer) {
+  char* const p0 = NULL;
+  // Only way to get the Nokia compiler to compile the cast
+  // is to have a separate void* variable first. Putting
+  // the two casts on the same line doesn't work, neither does
+  // a direct C-style to char*.
+  void* pv1 = (void*)0x1234;  // NOLINT
+  void* pv2 = (void*)0xABC0;  // NOLINT
+  char* const p1 = reinterpret_cast<char*>(pv1);
+  char* const p2 = reinterpret_cast<char*>(pv2);
+  ASSERT_EQ(p1, p1);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p0, p2),
+                          "To be equal to: p2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p1, p2),
+                          "p2");
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(reinterpret_cast<char*>(0x1234),
+                                 reinterpret_cast<char*>(0xABC0)),
+                       "ABC0");
+}
+
+// Tests using wchar_t pointers in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, WideCharPointer) {
+  wchar_t* const p0 = NULL;
+  // Only way to get the Nokia compiler to compile the cast
+  // is to have a separate void* variable first. Putting
+  // the two casts on the same line doesn't work, neither does
+  // a direct C-style to char*.
+  void* pv1 = (void*)0x1234;  // NOLINT
+  void* pv2 = (void*)0xABC0;  // NOLINT
+  wchar_t* const p1 = reinterpret_cast<wchar_t*>(pv1);
+  wchar_t* const p2 = reinterpret_cast<wchar_t*>(pv2);
+  EXPECT_EQ(p0, p0);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p0, p2),
+                          "To be equal to: p2");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p1, p2),
+                          "p2");
+  void* pv3 = (void*)0x1234;  // NOLINT
+  void* pv4 = (void*)0xABC0;  // NOLINT
+  const wchar_t* p3 = reinterpret_cast<const wchar_t*>(pv3);
+  const wchar_t* p4 = reinterpret_cast<const wchar_t*>(pv4);
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(p3, p4),
+                          "p4");
+}
+
+// Tests using other types of pointers in {EXPECT|ASSERT}_EQ.
+TEST(EqAssertionTest, OtherPointer) {
+  ASSERT_EQ(static_cast<const int*>(NULL),
+            static_cast<const int*>(NULL));
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(static_cast<const int*>(NULL),
+                                 reinterpret_cast<const int*>(0x1234)),
+                       "0x1234");
+}
+
+// A class that supports binary comparison operators but not streaming.
+class UnprintableChar {
+ public:
+  explicit UnprintableChar(char ch) : char_(ch) {}
+
+  bool operator==(const UnprintableChar& rhs) const {
+    return char_ == rhs.char_;
+  }
+  bool operator!=(const UnprintableChar& rhs) const {
+    return char_ != rhs.char_;
+  }
+  bool operator<(const UnprintableChar& rhs) const {
+    return char_ < rhs.char_;
+  }
+  bool operator<=(const UnprintableChar& rhs) const {
+    return char_ <= rhs.char_;
+  }
+  bool operator>(const UnprintableChar& rhs) const {
+    return char_ > rhs.char_;
+  }
+  bool operator>=(const UnprintableChar& rhs) const {
+    return char_ >= rhs.char_;
+  }
+
+ private:
+  char char_;
+};
+
+// Tests that ASSERT_EQ() and friends don't require the arguments to
+// be printable.
+TEST(ComparisonAssertionTest, AcceptsUnprintableArgs) {
+  const UnprintableChar x('x'), y('y');
+  ASSERT_EQ(x, x);
+  EXPECT_NE(x, y);
+  ASSERT_LT(x, y);
+  EXPECT_LE(x, y);
+  ASSERT_GT(y, x);
+  EXPECT_GE(x, x);
+
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(x, y), "1-byte object <78>");
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(x, y), "1-byte object <79>");
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(y, y), "1-byte object <79>");
+  EXPECT_NONFATAL_FAILURE(EXPECT_GT(x, y), "1-byte object <78>");
+  EXPECT_NONFATAL_FAILURE(EXPECT_GT(x, y), "1-byte object <79>");
+
+  // Code tested by EXPECT_FATAL_FAILURE cannot reference local
+  // variables, so we have to write UnprintableChar('x') instead of x.
+#ifndef __BORLANDC__
+  // ICE's in C++Builder.
+  EXPECT_FATAL_FAILURE(ASSERT_NE(UnprintableChar('x'), UnprintableChar('x')),
+                       "1-byte object <78>");
+  EXPECT_FATAL_FAILURE(ASSERT_LE(UnprintableChar('y'), UnprintableChar('x')),
+                       "1-byte object <78>");
+#endif
+  EXPECT_FATAL_FAILURE(ASSERT_LE(UnprintableChar('y'), UnprintableChar('x')),
+                       "1-byte object <79>");
+  EXPECT_FATAL_FAILURE(ASSERT_GE(UnprintableChar('x'), UnprintableChar('y')),
+                       "1-byte object <78>");
+  EXPECT_FATAL_FAILURE(ASSERT_GE(UnprintableChar('x'), UnprintableChar('y')),
+                       "1-byte object <79>");
+}
+
+// Tests the FRIEND_TEST macro.
+
+// This class has a private member we want to test.  We will test it
+// both in a TEST and in a TEST_F.
+class Foo {
+ public:
+  Foo() {}
+
+ private:
+  int Bar() const { return 1; }
+
+  // Declares the friend tests that can access the private member
+  // Bar().
+  FRIEND_TEST(FRIEND_TEST_Test, TEST);
+  FRIEND_TEST(FRIEND_TEST_Test2, TEST_F);
+};
+
+// Tests that the FRIEND_TEST declaration allows a TEST to access a
+// class's private members.  This should compile.
+TEST(FRIEND_TEST_Test, TEST) {
+  ASSERT_EQ(1, Foo().Bar());
+}
+
+// The fixture needed to test using FRIEND_TEST with TEST_F.
+class FRIEND_TEST_Test2 : public Test {
+ protected:
+  Foo foo;
+};
+
+// Tests that the FRIEND_TEST declaration allows a TEST_F to access a
+// class's private members.  This should compile.
+TEST_F(FRIEND_TEST_Test2, TEST_F) {
+  ASSERT_EQ(1, foo.Bar());
+}
+
+// Tests the life cycle of Test objects.
+
+// The test fixture for testing the life cycle of Test objects.
+//
+// This class counts the number of live test objects that uses this
+// fixture.
+class TestLifeCycleTest : public Test {
+ protected:
+  // Constructor.  Increments the number of test objects that uses
+  // this fixture.
+  TestLifeCycleTest() { count_++; }
+
+  // Destructor.  Decrements the number of test objects that uses this
+  // fixture.
+  ~TestLifeCycleTest() { count_--; }
+
+  // Returns the number of live test objects that uses this fixture.
+  int count() const { return count_; }
+
+ private:
+  static int count_;
+};
+
+int TestLifeCycleTest::count_ = 0;
+
+// Tests the life cycle of test objects.
+TEST_F(TestLifeCycleTest, Test1) {
+  // There should be only one test object in this test case that's
+  // currently alive.
+  ASSERT_EQ(1, count());
+}
+
+// Tests the life cycle of test objects.
+TEST_F(TestLifeCycleTest, Test2) {
+  // After Test1 is done and Test2 is started, there should still be
+  // only one live test object, as the object for Test1 should've been
+  // deleted.
+  ASSERT_EQ(1, count());
+}
+
+}  // namespace
+
+// Tests that the copy constructor works when it is NOT optimized away by
+// the compiler.
+TEST(AssertionResultTest, CopyConstructorWorksWhenNotOptimied) {
+  // Checks that the copy constructor doesn't try to dereference NULL pointers
+  // in the source object.
+  AssertionResult r1 = AssertionSuccess();
+  AssertionResult r2 = r1;
+  // The following line is added to prevent the compiler from optimizing
+  // away the constructor call.
+  r1 << "abc";
+
+  AssertionResult r3 = r1;
+  EXPECT_EQ(static_cast<bool>(r3), static_cast<bool>(r1));
+  EXPECT_STREQ("abc", r1.message());
+}
+
+// Tests that AssertionSuccess and AssertionFailure construct
+// AssertionResult objects as expected.
+TEST(AssertionResultTest, ConstructionWorks) {
+  AssertionResult r1 = AssertionSuccess();
+  EXPECT_TRUE(r1);
+  EXPECT_STREQ("", r1.message());
+
+  AssertionResult r2 = AssertionSuccess() << "abc";
+  EXPECT_TRUE(r2);
+  EXPECT_STREQ("abc", r2.message());
+
+  AssertionResult r3 = AssertionFailure();
+  EXPECT_FALSE(r3);
+  EXPECT_STREQ("", r3.message());
+
+  AssertionResult r4 = AssertionFailure() << "def";
+  EXPECT_FALSE(r4);
+  EXPECT_STREQ("def", r4.message());
+
+  AssertionResult r5 = AssertionFailure(Message() << "ghi");
+  EXPECT_FALSE(r5);
+  EXPECT_STREQ("ghi", r5.message());
+}
+
+// Tests that the negation flips the predicate result but keeps the message.
+TEST(AssertionResultTest, NegationWorks) {
+  AssertionResult r1 = AssertionSuccess() << "abc";
+  EXPECT_FALSE(!r1);
+  EXPECT_STREQ("abc", (!r1).message());
+
+  AssertionResult r2 = AssertionFailure() << "def";
+  EXPECT_TRUE(!r2);
+  EXPECT_STREQ("def", (!r2).message());
+}
+
+TEST(AssertionResultTest, StreamingWorks) {
+  AssertionResult r = AssertionSuccess();
+  r << "abc" << 'd' << 0 << true;
+  EXPECT_STREQ("abcd0true", r.message());
+}
+
+TEST(AssertionResultTest, CanStreamOstreamManipulators) {
+  AssertionResult r = AssertionSuccess();
+  r << "Data" << std::endl << std::flush << std::ends << "Will be visible";
+  EXPECT_STREQ("Data\n\\0Will be visible", r.message());
+}
+
+// The next test uses explicit conversion operators -- a C++11 feature.
+#if GTEST_LANG_CXX11
+
+TEST(AssertionResultTest, ConstructibleFromContextuallyConvertibleToBool) {
+  struct ExplicitlyConvertibleToBool {
+    explicit operator bool() const { return value; }
+    bool value;
+  };
+  ExplicitlyConvertibleToBool v1 = {false};
+  ExplicitlyConvertibleToBool v2 = {true};
+  EXPECT_FALSE(v1);
+  EXPECT_TRUE(v2);
+}
+
+#endif  // GTEST_LANG_CXX11
+
+struct ConvertibleToAssertionResult {
+  operator AssertionResult() const { return AssertionResult(true); }
+};
+
+TEST(AssertionResultTest, ConstructibleFromImplicitlyConvertible) {
+  ConvertibleToAssertionResult obj;
+  EXPECT_TRUE(obj);
+}
+
+// Tests streaming a user type whose definition and operator << are
+// both in the global namespace.
+class Base {
+ public:
+  explicit Base(int an_x) : x_(an_x) {}
+  int x() const { return x_; }
+ private:
+  int x_;
+};
+std::ostream& operator<<(std::ostream& os,
+                         const Base& val) {
+  return os << val.x();
+}
+std::ostream& operator<<(std::ostream& os,
+                         const Base* pointer) {
+  return os << "(" << pointer->x() << ")";
+}
+
+TEST(MessageTest, CanStreamUserTypeInGlobalNameSpace) {
+  Message msg;
+  Base a(1);
+
+  msg << a << &a;  // Uses ::operator<<.
+  EXPECT_STREQ("1(1)", msg.GetString().c_str());
+}
+
+// Tests streaming a user type whose definition and operator<< are
+// both in an unnamed namespace.
+namespace {
+class MyTypeInUnnamedNameSpace : public Base {
+ public:
+  explicit MyTypeInUnnamedNameSpace(int an_x): Base(an_x) {}
+};
+std::ostream& operator<<(std::ostream& os,
+                         const MyTypeInUnnamedNameSpace& val) {
+  return os << val.x();
+}
+std::ostream& operator<<(std::ostream& os,
+                         const MyTypeInUnnamedNameSpace* pointer) {
+  return os << "(" << pointer->x() << ")";
+}
+}  // namespace
+
+TEST(MessageTest, CanStreamUserTypeInUnnamedNameSpace) {
+  Message msg;
+  MyTypeInUnnamedNameSpace a(1);
+
+  msg << a << &a;  // Uses <unnamed_namespace>::operator<<.
+  EXPECT_STREQ("1(1)", msg.GetString().c_str());
+}
+
+// Tests streaming a user type whose definition and operator<< are
+// both in a user namespace.
+namespace namespace1 {
+class MyTypeInNameSpace1 : public Base {
+ public:
+  explicit MyTypeInNameSpace1(int an_x): Base(an_x) {}
+};
+std::ostream& operator<<(std::ostream& os,
+                         const MyTypeInNameSpace1& val) {
+  return os << val.x();
+}
+std::ostream& operator<<(std::ostream& os,
+                         const MyTypeInNameSpace1* pointer) {
+  return os << "(" << pointer->x() << ")";
+}
+}  // namespace namespace1
+
+TEST(MessageTest, CanStreamUserTypeInUserNameSpace) {
+  Message msg;
+  namespace1::MyTypeInNameSpace1 a(1);
+
+  msg << a << &a;  // Uses namespace1::operator<<.
+  EXPECT_STREQ("1(1)", msg.GetString().c_str());
+}
+
+// Tests streaming a user type whose definition is in a user namespace
+// but whose operator<< is in the global namespace.
+namespace namespace2 {
+class MyTypeInNameSpace2 : public ::Base {
+ public:
+  explicit MyTypeInNameSpace2(int an_x): Base(an_x) {}
+};
+}  // namespace namespace2
+std::ostream& operator<<(std::ostream& os,
+                         const namespace2::MyTypeInNameSpace2& val) {
+  return os << val.x();
+}
+std::ostream& operator<<(std::ostream& os,
+                         const namespace2::MyTypeInNameSpace2* pointer) {
+  return os << "(" << pointer->x() << ")";
+}
+
+TEST(MessageTest, CanStreamUserTypeInUserNameSpaceWithStreamOperatorInGlobal) {
+  Message msg;
+  namespace2::MyTypeInNameSpace2 a(1);
+
+  msg << a << &a;  // Uses ::operator<<.
+  EXPECT_STREQ("1(1)", msg.GetString().c_str());
+}
+
+// Tests streaming NULL pointers to testing::Message.
+TEST(MessageTest, NullPointers) {
+  Message msg;
+  char* const p1 = NULL;
+  unsigned char* const p2 = NULL;
+  int* p3 = NULL;
+  double* p4 = NULL;
+  bool* p5 = NULL;
+  Message* p6 = NULL;
+
+  msg << p1 << p2 << p3 << p4 << p5 << p6;
+  ASSERT_STREQ("(null)(null)(null)(null)(null)(null)",
+               msg.GetString().c_str());
+}
+
+// Tests streaming wide strings to testing::Message.
+TEST(MessageTest, WideStrings) {
+  // Streams a NULL of type const wchar_t*.
+  const wchar_t* const_wstr = NULL;
+  EXPECT_STREQ("(null)",
+               (Message() << const_wstr).GetString().c_str());
+
+  // Streams a NULL of type wchar_t*.
+  wchar_t* wstr = NULL;
+  EXPECT_STREQ("(null)",
+               (Message() << wstr).GetString().c_str());
+
+  // Streams a non-NULL of type const wchar_t*.
+  const_wstr = L"abc\x8119";
+  EXPECT_STREQ("abc\xe8\x84\x99",
+               (Message() << const_wstr).GetString().c_str());
+
+  // Streams a non-NULL of type wchar_t*.
+  wstr = const_cast<wchar_t*>(const_wstr);
+  EXPECT_STREQ("abc\xe8\x84\x99",
+               (Message() << wstr).GetString().c_str());
+}
+
+
+// This line tests that we can define tests in the testing namespace.
+namespace testing {
+
+// Tests the TestInfo class.
+
+class TestInfoTest : public Test {
+ protected:
+  static const TestInfo* GetTestInfo(const char* test_name) {
+    const TestCase* const test_case = GetUnitTestImpl()->
+        GetTestCase("TestInfoTest", "", NULL, NULL);
+
+    for (int i = 0; i < test_case->total_test_count(); ++i) {
+      const TestInfo* const test_info = test_case->GetTestInfo(i);
+      if (strcmp(test_name, test_info->name()) == 0)
+        return test_info;
+    }
+    return NULL;
+  }
+
+  static const TestResult* GetTestResult(
+      const TestInfo* test_info) {
+    return test_info->result();
+  }
+};
+
+// Tests TestInfo::test_case_name() and TestInfo::name().
+TEST_F(TestInfoTest, Names) {
+  const TestInfo* const test_info = GetTestInfo("Names");
+
+  ASSERT_STREQ("TestInfoTest", test_info->test_case_name());
+  ASSERT_STREQ("Names", test_info->name());
+}
+
+// Tests TestInfo::result().
+TEST_F(TestInfoTest, result) {
+  const TestInfo* const test_info = GetTestInfo("result");
+
+  // Initially, there is no TestPartResult for this test.
+  ASSERT_EQ(0, GetTestResult(test_info)->total_part_count());
+
+  // After the previous assertion, there is still none.
+  ASSERT_EQ(0, GetTestResult(test_info)->total_part_count());
+}
+
+#define VERIFY_CODE_LOCATION \
+  const int expected_line = __LINE__ - 1; \
+  const TestInfo* const test_info = GetUnitTestImpl()->current_test_info(); \
+  ASSERT_TRUE(test_info); \
+  EXPECT_STREQ(__FILE__, test_info->file()); \
+  EXPECT_EQ(expected_line, test_info->line())
+
+TEST(CodeLocationForTEST, Verify) {
+  VERIFY_CODE_LOCATION;
+}
+
+class CodeLocationForTESTF : public Test {
+};
+
+TEST_F(CodeLocationForTESTF, Verify) {
+  VERIFY_CODE_LOCATION;
+}
+
+class CodeLocationForTESTP : public TestWithParam<int> {
+};
+
+TEST_P(CodeLocationForTESTP, Verify) {
+  VERIFY_CODE_LOCATION;
+}
+
+INSTANTIATE_TEST_CASE_P(, CodeLocationForTESTP, Values(0));
+
+template <typename T>
+class CodeLocationForTYPEDTEST : public Test {
+};
+
+TYPED_TEST_CASE(CodeLocationForTYPEDTEST, int);
+
+TYPED_TEST(CodeLocationForTYPEDTEST, Verify) {
+  VERIFY_CODE_LOCATION;
+}
+
+template <typename T>
+class CodeLocationForTYPEDTESTP : public Test {
+};
+
+TYPED_TEST_CASE_P(CodeLocationForTYPEDTESTP);
+
+TYPED_TEST_P(CodeLocationForTYPEDTESTP, Verify) {
+  VERIFY_CODE_LOCATION;
+}
+
+REGISTER_TYPED_TEST_CASE_P(CodeLocationForTYPEDTESTP, Verify);
+
+INSTANTIATE_TYPED_TEST_CASE_P(My, CodeLocationForTYPEDTESTP, int);
+
+#undef VERIFY_CODE_LOCATION
+
+// Tests setting up and tearing down a test case.
+
+class SetUpTestCaseTest : public Test {
+ protected:
+  // This will be called once before the first test in this test case
+  // is run.
+  static void SetUpTestCase() {
+    printf("Setting up the test case . . .\n");
+
+    // Initializes some shared resource.  In this simple example, we
+    // just create a C string.  More complex stuff can be done if
+    // desired.
+    shared_resource_ = "123";
+
+    // Increments the number of test cases that have been set up.
+    counter_++;
+
+    // SetUpTestCase() should be called only once.
+    EXPECT_EQ(1, counter_);
+  }
+
+  // This will be called once after the last test in this test case is
+  // run.
+  static void TearDownTestCase() {
+    printf("Tearing down the test case . . .\n");
+
+    // Decrements the number of test cases that have been set up.
+    counter_--;
+
+    // TearDownTestCase() should be called only once.
+    EXPECT_EQ(0, counter_);
+
+    // Cleans up the shared resource.
+    shared_resource_ = NULL;
+  }
+
+  // This will be called before each test in this test case.
+  virtual void SetUp() {
+    // SetUpTestCase() should be called only once, so counter_ should
+    // always be 1.
+    EXPECT_EQ(1, counter_);
+  }
+
+  // Number of test cases that have been set up.
+  static int counter_;
+
+  // Some resource to be shared by all tests in this test case.
+  static const char* shared_resource_;
+};
+
+int SetUpTestCaseTest::counter_ = 0;
+const char* SetUpTestCaseTest::shared_resource_ = NULL;
+
+// A test that uses the shared resource.
+TEST_F(SetUpTestCaseTest, Test1) {
+  EXPECT_STRNE(NULL, shared_resource_);
+}
+
+// Another test that uses the shared resource.
+TEST_F(SetUpTestCaseTest, Test2) {
+  EXPECT_STREQ("123", shared_resource_);
+}
+
+// The InitGoogleTestTest test case tests testing::InitGoogleTest().
+
+// The Flags struct stores a copy of all Google Test flags.
+struct Flags {
+  // Constructs a Flags struct where each flag has its default value.
+  Flags() : also_run_disabled_tests(false),
+            break_on_failure(false),
+            catch_exceptions(false),
+            death_test_use_fork(false),
+            filter(""),
+            list_tests(false),
+            output(""),
+            print_time(true),
+            random_seed(0),
+            repeat(1),
+            shuffle(false),
+            stack_trace_depth(kMaxStackTraceDepth),
+            stream_result_to(""),
+            throw_on_failure(false) {}
+
+  // Factory methods.
+
+  // Creates a Flags struct where the gtest_also_run_disabled_tests flag has
+  // the given value.
+  static Flags AlsoRunDisabledTests(bool also_run_disabled_tests) {
+    Flags flags;
+    flags.also_run_disabled_tests = also_run_disabled_tests;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_break_on_failure flag has
+  // the given value.
+  static Flags BreakOnFailure(bool break_on_failure) {
+    Flags flags;
+    flags.break_on_failure = break_on_failure;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_catch_exceptions flag has
+  // the given value.
+  static Flags CatchExceptions(bool catch_exceptions) {
+    Flags flags;
+    flags.catch_exceptions = catch_exceptions;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_death_test_use_fork flag has
+  // the given value.
+  static Flags DeathTestUseFork(bool death_test_use_fork) {
+    Flags flags;
+    flags.death_test_use_fork = death_test_use_fork;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_filter flag has the given
+  // value.
+  static Flags Filter(const char* filter) {
+    Flags flags;
+    flags.filter = filter;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_list_tests flag has the
+  // given value.
+  static Flags ListTests(bool list_tests) {
+    Flags flags;
+    flags.list_tests = list_tests;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_output flag has the given
+  // value.
+  static Flags Output(const char* output) {
+    Flags flags;
+    flags.output = output;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_print_time flag has the given
+  // value.
+  static Flags PrintTime(bool print_time) {
+    Flags flags;
+    flags.print_time = print_time;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_random_seed flag has
+  // the given value.
+  static Flags RandomSeed(Int32 random_seed) {
+    Flags flags;
+    flags.random_seed = random_seed;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_repeat flag has the given
+  // value.
+  static Flags Repeat(Int32 repeat) {
+    Flags flags;
+    flags.repeat = repeat;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_shuffle flag has
+  // the given value.
+  static Flags Shuffle(bool shuffle) {
+    Flags flags;
+    flags.shuffle = shuffle;
+    return flags;
+  }
+
+  // Creates a Flags struct where the GTEST_FLAG(stack_trace_depth) flag has
+  // the given value.
+  static Flags StackTraceDepth(Int32 stack_trace_depth) {
+    Flags flags;
+    flags.stack_trace_depth = stack_trace_depth;
+    return flags;
+  }
+
+  // Creates a Flags struct where the GTEST_FLAG(stream_result_to) flag has
+  // the given value.
+  static Flags StreamResultTo(const char* stream_result_to) {
+    Flags flags;
+    flags.stream_result_to = stream_result_to;
+    return flags;
+  }
+
+  // Creates a Flags struct where the gtest_throw_on_failure flag has
+  // the given value.
+  static Flags ThrowOnFailure(bool throw_on_failure) {
+    Flags flags;
+    flags.throw_on_failure = throw_on_failure;
+    return flags;
+  }
+
+  // These fields store the flag values.
+  bool also_run_disabled_tests;
+  bool break_on_failure;
+  bool catch_exceptions;
+  bool death_test_use_fork;
+  const char* filter;
+  bool list_tests;
+  const char* output;
+  bool print_time;
+  Int32 random_seed;
+  Int32 repeat;
+  bool shuffle;
+  Int32 stack_trace_depth;
+  const char* stream_result_to;
+  bool throw_on_failure;
+};
+
+// Fixture for testing InitGoogleTest().
+class InitGoogleTestTest : public Test {
+ protected:
+  // Clears the flags before each test.
+  virtual void SetUp() {
+    GTEST_FLAG(also_run_disabled_tests) = false;
+    GTEST_FLAG(break_on_failure) = false;
+    GTEST_FLAG(catch_exceptions) = false;
+    GTEST_FLAG(death_test_use_fork) = false;
+    GTEST_FLAG(filter) = "";
+    GTEST_FLAG(list_tests) = false;
+    GTEST_FLAG(output) = "";
+    GTEST_FLAG(print_time) = true;
+    GTEST_FLAG(random_seed) = 0;
+    GTEST_FLAG(repeat) = 1;
+    GTEST_FLAG(shuffle) = false;
+    GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
+    GTEST_FLAG(stream_result_to) = "";
+    GTEST_FLAG(throw_on_failure) = false;
+  }
+
+  // Asserts that two narrow or wide string arrays are equal.
+  template <typename CharType>
+  static void AssertStringArrayEq(size_t size1, CharType** array1,
+                                  size_t size2, CharType** array2) {
+    ASSERT_EQ(size1, size2) << " Array sizes different.";
+
+    for (size_t i = 0; i != size1; i++) {
+      ASSERT_STREQ(array1[i], array2[i]) << " where i == " << i;
+    }
+  }
+
+  // Verifies that the flag values match the expected values.
+  static void CheckFlags(const Flags& expected) {
+    EXPECT_EQ(expected.also_run_disabled_tests,
+              GTEST_FLAG(also_run_disabled_tests));
+    EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure));
+    EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions));
+    EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork));
+    EXPECT_STREQ(expected.filter, GTEST_FLAG(filter).c_str());
+    EXPECT_EQ(expected.list_tests, GTEST_FLAG(list_tests));
+    EXPECT_STREQ(expected.output, GTEST_FLAG(output).c_str());
+    EXPECT_EQ(expected.print_time, GTEST_FLAG(print_time));
+    EXPECT_EQ(expected.random_seed, GTEST_FLAG(random_seed));
+    EXPECT_EQ(expected.repeat, GTEST_FLAG(repeat));
+    EXPECT_EQ(expected.shuffle, GTEST_FLAG(shuffle));
+    EXPECT_EQ(expected.stack_trace_depth, GTEST_FLAG(stack_trace_depth));
+    EXPECT_STREQ(expected.stream_result_to,
+                 GTEST_FLAG(stream_result_to).c_str());
+    EXPECT_EQ(expected.throw_on_failure, GTEST_FLAG(throw_on_failure));
+  }
+
+  // Parses a command line (specified by argc1 and argv1), then
+  // verifies that the flag values are expected and that the
+  // recognized flags are removed from the command line.
+  template <typename CharType>
+  static void TestParsingFlags(int argc1, const CharType** argv1,
+                               int argc2, const CharType** argv2,
+                               const Flags& expected, bool should_print_help) {
+    const bool saved_help_flag = ::testing::internal::g_help_flag;
+    ::testing::internal::g_help_flag = false;
+
+#if GTEST_HAS_STREAM_REDIRECTION
+    CaptureStdout();
+#endif
+
+    // Parses the command line.
+    internal::ParseGoogleTestFlagsOnly(&argc1, const_cast<CharType**>(argv1));
+
+#if GTEST_HAS_STREAM_REDIRECTION
+    const std::string captured_stdout = GetCapturedStdout();
+#endif
+
+    // Verifies the flag values.
+    CheckFlags(expected);
+
+    // Verifies that the recognized flags are removed from the command
+    // line.
+    AssertStringArrayEq(argc1 + 1, argv1, argc2 + 1, argv2);
+
+    // ParseGoogleTestFlagsOnly should neither set g_help_flag nor print the
+    // help message for the flags it recognizes.
+    EXPECT_EQ(should_print_help, ::testing::internal::g_help_flag);
+
+#if GTEST_HAS_STREAM_REDIRECTION
+    const char* const expected_help_fragment =
+        "This program contains tests written using";
+    if (should_print_help) {
+      EXPECT_PRED_FORMAT2(IsSubstring, expected_help_fragment, captured_stdout);
+    } else {
+      EXPECT_PRED_FORMAT2(IsNotSubstring,
+                          expected_help_fragment, captured_stdout);
+    }
+#endif  // GTEST_HAS_STREAM_REDIRECTION
+
+    ::testing::internal::g_help_flag = saved_help_flag;
+  }
+
+  // This macro wraps TestParsingFlags s.t. the user doesn't need
+  // to specify the array sizes.
+
+#define GTEST_TEST_PARSING_FLAGS_(argv1, argv2, expected, should_print_help) \
+  TestParsingFlags(sizeof(argv1)/sizeof(*argv1) - 1, argv1, \
+                   sizeof(argv2)/sizeof(*argv2) - 1, argv2, \
+                   expected, should_print_help)
+};
+
+// Tests parsing an empty command line.
+TEST_F(InitGoogleTestTest, Empty) {
+  const char* argv[] = {
+    NULL
+  };
+
+  const char* argv2[] = {
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false);
+}
+
+// Tests parsing a command line that has no flag.
+TEST_F(InitGoogleTestTest, NoFlag) {
+  const char* argv[] = {
+    "foo.exe",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false);
+}
+
+// Tests parsing a bad --gtest_filter flag.
+TEST_F(InitGoogleTestTest, FilterBad) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_filter",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    "--gtest_filter",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Filter(""), true);
+}
+
+// Tests parsing an empty --gtest_filter flag.
+TEST_F(InitGoogleTestTest, FilterEmpty) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_filter=",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Filter(""), false);
+}
+
+// Tests parsing a non-empty --gtest_filter flag.
+TEST_F(InitGoogleTestTest, FilterNonEmpty) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_filter=abc",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Filter("abc"), false);
+}
+
+// Tests parsing --gtest_break_on_failure.
+TEST_F(InitGoogleTestTest, BreakOnFailureWithoutValue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure",
+    NULL
+};
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::BreakOnFailure(true), false);
+}
+
+// Tests parsing --gtest_break_on_failure=0.
+TEST_F(InitGoogleTestTest, BreakOnFailureFalse_0) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure=0",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::BreakOnFailure(false), false);
+}
+
+// Tests parsing --gtest_break_on_failure=f.
+TEST_F(InitGoogleTestTest, BreakOnFailureFalse_f) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure=f",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::BreakOnFailure(false), false);
+}
+
+// Tests parsing --gtest_break_on_failure=F.
+TEST_F(InitGoogleTestTest, BreakOnFailureFalse_F) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure=F",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::BreakOnFailure(false), false);
+}
+
+// Tests parsing a --gtest_break_on_failure flag that has a "true"
+// definition.
+TEST_F(InitGoogleTestTest, BreakOnFailureTrue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure=1",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::BreakOnFailure(true), false);
+}
+
+// Tests parsing --gtest_catch_exceptions.
+TEST_F(InitGoogleTestTest, CatchExceptions) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_catch_exceptions",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::CatchExceptions(true), false);
+}
+
+// Tests parsing --gtest_death_test_use_fork.
+TEST_F(InitGoogleTestTest, DeathTestUseFork) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_death_test_use_fork",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::DeathTestUseFork(true), false);
+}
+
+// Tests having the same flag twice with different values.  The
+// expected behavior is that the one coming last takes precedence.
+TEST_F(InitGoogleTestTest, DuplicatedFlags) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_filter=a",
+    "--gtest_filter=b",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Filter("b"), false);
+}
+
+// Tests having an unrecognized flag on the command line.
+TEST_F(InitGoogleTestTest, UnrecognizedFlag) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_break_on_failure",
+    "bar",  // Unrecognized by Google Test.
+    "--gtest_filter=b",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    "bar",
+    NULL
+  };
+
+  Flags flags;
+  flags.break_on_failure = true;
+  flags.filter = "b";
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, flags, false);
+}
+
+// Tests having a --gtest_list_tests flag
+TEST_F(InitGoogleTestTest, ListTestsFlag) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_list_tests",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ListTests(true), false);
+}
+
+// Tests having a --gtest_list_tests flag with a "true" value
+TEST_F(InitGoogleTestTest, ListTestsTrue) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_list_tests=1",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ListTests(true), false);
+}
+
+// Tests having a --gtest_list_tests flag with a "false" value
+TEST_F(InitGoogleTestTest, ListTestsFalse) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_list_tests=0",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ListTests(false), false);
+}
+
+// Tests parsing --gtest_list_tests=f.
+TEST_F(InitGoogleTestTest, ListTestsFalse_f) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_list_tests=f",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ListTests(false), false);
+}
+
+// Tests parsing --gtest_list_tests=F.
+TEST_F(InitGoogleTestTest, ListTestsFalse_F) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_list_tests=F",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ListTests(false), false);
+}
+
+// Tests parsing --gtest_output (invalid).
+TEST_F(InitGoogleTestTest, OutputEmpty) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_output",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    "--gtest_output",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), true);
+}
+
+// Tests parsing --gtest_output=xml
+TEST_F(InitGoogleTestTest, OutputXml) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_output=xml",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Output("xml"), false);
+}
+
+// Tests parsing --gtest_output=xml:file
+TEST_F(InitGoogleTestTest, OutputXmlFile) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_output=xml:file",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Output("xml:file"), false);
+}
+
+// Tests parsing --gtest_output=xml:directory/path/
+TEST_F(InitGoogleTestTest, OutputXmlDirectory) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_output=xml:directory/path/",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2,
+                            Flags::Output("xml:directory/path/"), false);
+}
+
+// Tests having a --gtest_print_time flag
+TEST_F(InitGoogleTestTest, PrintTimeFlag) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_print_time",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::PrintTime(true), false);
+}
+
+// Tests having a --gtest_print_time flag with a "true" value
+TEST_F(InitGoogleTestTest, PrintTimeTrue) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_print_time=1",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::PrintTime(true), false);
+}
+
+// Tests having a --gtest_print_time flag with a "false" value
+TEST_F(InitGoogleTestTest, PrintTimeFalse) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_print_time=0",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::PrintTime(false), false);
+}
+
+// Tests parsing --gtest_print_time=f.
+TEST_F(InitGoogleTestTest, PrintTimeFalse_f) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_print_time=f",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::PrintTime(false), false);
+}
+
+// Tests parsing --gtest_print_time=F.
+TEST_F(InitGoogleTestTest, PrintTimeFalse_F) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_print_time=F",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::PrintTime(false), false);
+}
+
+// Tests parsing --gtest_random_seed=number
+TEST_F(InitGoogleTestTest, RandomSeed) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_random_seed=1000",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::RandomSeed(1000), false);
+}
+
+// Tests parsing --gtest_repeat=number
+TEST_F(InitGoogleTestTest, Repeat) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_repeat=1000",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Repeat(1000), false);
+}
+
+// Tests having a --gtest_also_run_disabled_tests flag
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFlag) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2,
+                              Flags::AlsoRunDisabledTests(true), false);
+}
+
+// Tests having a --gtest_also_run_disabled_tests flag with a "true" value
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsTrue) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests=1",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2,
+                              Flags::AlsoRunDisabledTests(true), false);
+}
+
+// Tests having a --gtest_also_run_disabled_tests flag with a "false" value
+TEST_F(InitGoogleTestTest, AlsoRunDisabledTestsFalse) {
+    const char* argv[] = {
+      "foo.exe",
+      "--gtest_also_run_disabled_tests=0",
+      NULL
+    };
+
+    const char* argv2[] = {
+      "foo.exe",
+      NULL
+    };
+
+    GTEST_TEST_PARSING_FLAGS_(argv, argv2,
+                              Flags::AlsoRunDisabledTests(false), false);
+}
+
+// Tests parsing --gtest_shuffle.
+TEST_F(InitGoogleTestTest, ShuffleWithoutValue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_shuffle",
+    NULL
+};
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Shuffle(true), false);
+}
+
+// Tests parsing --gtest_shuffle=0.
+TEST_F(InitGoogleTestTest, ShuffleFalse_0) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_shuffle=0",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Shuffle(false), false);
+}
+
+// Tests parsing a --gtest_shuffle flag that has a "true"
+// definition.
+TEST_F(InitGoogleTestTest, ShuffleTrue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_shuffle=1",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Shuffle(true), false);
+}
+
+// Tests parsing --gtest_stack_trace_depth=number.
+TEST_F(InitGoogleTestTest, StackTraceDepth) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_stack_trace_depth=5",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::StackTraceDepth(5), false);
+}
+
+TEST_F(InitGoogleTestTest, StreamResultTo) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_stream_result_to=localhost:1234",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(
+      argv, argv2, Flags::StreamResultTo("localhost:1234"), false);
+}
+
+// Tests parsing --gtest_throw_on_failure.
+TEST_F(InitGoogleTestTest, ThrowOnFailureWithoutValue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_throw_on_failure",
+    NULL
+};
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ThrowOnFailure(true), false);
+}
+
+// Tests parsing --gtest_throw_on_failure=0.
+TEST_F(InitGoogleTestTest, ThrowOnFailureFalse_0) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_throw_on_failure=0",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ThrowOnFailure(false), false);
+}
+
+// Tests parsing a --gtest_throw_on_failure flag that has a "true"
+// definition.
+TEST_F(InitGoogleTestTest, ThrowOnFailureTrue) {
+  const char* argv[] = {
+    "foo.exe",
+    "--gtest_throw_on_failure=1",
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::ThrowOnFailure(true), false);
+}
+
+#if GTEST_OS_WINDOWS
+// Tests parsing wide strings.
+TEST_F(InitGoogleTestTest, WideStrings) {
+  const wchar_t* argv[] = {
+    L"foo.exe",
+    L"--gtest_filter=Foo*",
+    L"--gtest_list_tests=1",
+    L"--gtest_break_on_failure",
+    L"--non_gtest_flag",
+    NULL
+  };
+
+  const wchar_t* argv2[] = {
+    L"foo.exe",
+    L"--non_gtest_flag",
+    NULL
+  };
+
+  Flags expected_flags;
+  expected_flags.break_on_failure = true;
+  expected_flags.filter = "Foo*";
+  expected_flags.list_tests = true;
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, expected_flags, false);
+}
+# endif  // GTEST_OS_WINDOWS
+
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+class FlagfileTest : public InitGoogleTestTest {
+ public:
+  virtual void SetUp() {
+    InitGoogleTestTest::SetUp();
+
+    testdata_path_.Set(internal::FilePath(
+        internal::TempDir() + internal::GetCurrentExecutableName().string() +
+        "_flagfile_test"));
+    testing::internal::posix::RmDir(testdata_path_.c_str());
+    EXPECT_TRUE(testdata_path_.CreateFolder());
+  }
+
+  virtual void TearDown() {
+    testing::internal::posix::RmDir(testdata_path_.c_str());
+    InitGoogleTestTest::TearDown();
+  }
+
+  internal::FilePath CreateFlagfile(const char* contents) {
+    internal::FilePath file_path(internal::FilePath::GenerateUniqueFileName(
+        testdata_path_, internal::FilePath("unique"), "txt"));
+    FILE* f = testing::internal::posix::FOpen(file_path.c_str(), "w");
+    fprintf(f, "%s", contents);
+    fclose(f);
+    return file_path;
+  }
+
+ private:
+  internal::FilePath testdata_path_;
+};
+
+// Tests an empty flagfile.
+TEST_F(FlagfileTest, Empty) {
+  internal::FilePath flagfile_path(CreateFlagfile(""));
+  std::string flagfile_flag =
+      std::string("--" GTEST_FLAG_PREFIX_ "flagfile=") + flagfile_path.c_str();
+
+  const char* argv[] = {
+    "foo.exe",
+    flagfile_flag.c_str(),
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false);
+}
+
+// Tests passing a non-empty --gtest_filter flag via --gtest_flagfile.
+TEST_F(FlagfileTest, FilterNonEmpty) {
+  internal::FilePath flagfile_path(CreateFlagfile(
+      "--"  GTEST_FLAG_PREFIX_  "filter=abc"));
+  std::string flagfile_flag =
+      std::string("--" GTEST_FLAG_PREFIX_ "flagfile=") + flagfile_path.c_str();
+
+  const char* argv[] = {
+    "foo.exe",
+    flagfile_flag.c_str(),
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Filter("abc"), false);
+}
+
+// Tests passing several flags via --gtest_flagfile.
+TEST_F(FlagfileTest, SeveralFlags) {
+  internal::FilePath flagfile_path(CreateFlagfile(
+      "--"  GTEST_FLAG_PREFIX_  "filter=abc\n"
+      "--"  GTEST_FLAG_PREFIX_  "break_on_failure\n"
+      "--"  GTEST_FLAG_PREFIX_  "list_tests"));
+  std::string flagfile_flag =
+      std::string("--" GTEST_FLAG_PREFIX_ "flagfile=") + flagfile_path.c_str();
+
+  const char* argv[] = {
+    "foo.exe",
+    flagfile_flag.c_str(),
+    NULL
+  };
+
+  const char* argv2[] = {
+    "foo.exe",
+    NULL
+  };
+
+  Flags expected_flags;
+  expected_flags.break_on_failure = true;
+  expected_flags.filter = "abc";
+  expected_flags.list_tests = true;
+
+  GTEST_TEST_PARSING_FLAGS_(argv, argv2, expected_flags, false);
+}
+#endif  // GTEST_USE_OWN_FLAGFILE_FLAG_
+
+// Tests current_test_info() in UnitTest.
+class CurrentTestInfoTest : public Test {
+ protected:
+  // Tests that current_test_info() returns NULL before the first test in
+  // the test case is run.
+  static void SetUpTestCase() {
+    // There should be no tests running at this point.
+    const TestInfo* test_info =
+      UnitTest::GetInstance()->current_test_info();
+    EXPECT_TRUE(test_info == NULL)
+        << "There should be no tests running at this point.";
+  }
+
+  // Tests that current_test_info() returns NULL after the last test in
+  // the test case has run.
+  static void TearDownTestCase() {
+    const TestInfo* test_info =
+      UnitTest::GetInstance()->current_test_info();
+    EXPECT_TRUE(test_info == NULL)
+        << "There should be no tests running at this point.";
+  }
+};
+
+// Tests that current_test_info() returns TestInfo for currently running
+// test by checking the expected test name against the actual one.
+TEST_F(CurrentTestInfoTest, WorksForFirstTestInATestCase) {
+  const TestInfo* test_info =
+    UnitTest::GetInstance()->current_test_info();
+  ASSERT_TRUE(NULL != test_info)
+      << "There is a test running so we should have a valid TestInfo.";
+  EXPECT_STREQ("CurrentTestInfoTest", test_info->test_case_name())
+      << "Expected the name of the currently running test case.";
+  EXPECT_STREQ("WorksForFirstTestInATestCase", test_info->name())
+      << "Expected the name of the currently running test.";
+}
+
+// Tests that current_test_info() returns TestInfo for currently running
+// test by checking the expected test name against the actual one.  We
+// use this test to see that the TestInfo object actually changed from
+// the previous invocation.
+TEST_F(CurrentTestInfoTest, WorksForSecondTestInATestCase) {
+  const TestInfo* test_info =
+    UnitTest::GetInstance()->current_test_info();
+  ASSERT_TRUE(NULL != test_info)
+      << "There is a test running so we should have a valid TestInfo.";
+  EXPECT_STREQ("CurrentTestInfoTest", test_info->test_case_name())
+      << "Expected the name of the currently running test case.";
+  EXPECT_STREQ("WorksForSecondTestInATestCase", test_info->name())
+      << "Expected the name of the currently running test.";
+}
+
+}  // namespace testing
+
+// These two lines test that we can define tests in a namespace that
+// has the name "testing" and is nested in another namespace.
+namespace my_namespace {
+namespace testing {
+
+// Makes sure that TEST knows to use ::testing::Test instead of
+// ::my_namespace::testing::Test.
+class Test {};
+
+// Makes sure that an assertion knows to use ::testing::Message instead of
+// ::my_namespace::testing::Message.
+class Message {};
+
+// Makes sure that an assertion knows to use
+// ::testing::AssertionResult instead of
+// ::my_namespace::testing::AssertionResult.
+class AssertionResult {};
+
+// Tests that an assertion that should succeed works as expected.
+TEST(NestedTestingNamespaceTest, Success) {
+  EXPECT_EQ(1, 1) << "This shouldn't fail.";
+}
+
+// Tests that an assertion that should fail works as expected.
+TEST(NestedTestingNamespaceTest, Failure) {
+  EXPECT_FATAL_FAILURE(FAIL() << "This failure is expected.",
+                       "This failure is expected.");
+}
+
+}  // namespace testing
+}  // namespace my_namespace
+
+// Tests that one can call superclass SetUp and TearDown methods--
+// that is, that they are not private.
+// No tests are based on this fixture; the test "passes" if it compiles
+// successfully.
+class ProtectedFixtureMethodsTest : public Test {
+ protected:
+  virtual void SetUp() {
+    Test::SetUp();
+  }
+  virtual void TearDown() {
+    Test::TearDown();
+  }
+};
+
+// StreamingAssertionsTest tests the streaming versions of a representative
+// sample of assertions.
+TEST(StreamingAssertionsTest, Unconditional) {
+  SUCCEED() << "expected success";
+  EXPECT_NONFATAL_FAILURE(ADD_FAILURE() << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(FAIL() << "expected failure",
+                       "expected failure");
+}
+
+#ifdef __BORLANDC__
+// Silences warnings: "Condition is always true", "Unreachable code"
+# pragma option push -w-ccc -w-rch
+#endif
+
+TEST(StreamingAssertionsTest, Truth) {
+  EXPECT_TRUE(true) << "unexpected failure";
+  ASSERT_TRUE(true) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_TRUE(false) << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_TRUE(false) << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, Truth2) {
+  EXPECT_FALSE(false) << "unexpected failure";
+  ASSERT_FALSE(false) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_FALSE(true) << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_FALSE(true) << "expected failure",
+                       "expected failure");
+}
+
+#ifdef __BORLANDC__
+// Restores warnings after previous "#pragma option push" supressed them
+# pragma option pop
+#endif
+
+TEST(StreamingAssertionsTest, IntegerEquals) {
+  EXPECT_EQ(1, 1) << "unexpected failure";
+  ASSERT_EQ(1, 1) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(1, 2) << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_EQ(1, 2) << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, IntegerLessThan) {
+  EXPECT_LT(1, 2) << "unexpected failure";
+  ASSERT_LT(1, 2) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_LT(2, 1) << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_LT(2, 1) << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, StringsEqual) {
+  EXPECT_STREQ("foo", "foo") << "unexpected failure";
+  ASSERT_STREQ("foo", "foo") << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STREQ("foo", "bar") << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_STREQ("foo", "bar") << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, StringsNotEqual) {
+  EXPECT_STRNE("foo", "bar") << "unexpected failure";
+  ASSERT_STRNE("foo", "bar") << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRNE("foo", "foo") << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_STRNE("foo", "foo") << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, StringsEqualIgnoringCase) {
+  EXPECT_STRCASEEQ("foo", "FOO") << "unexpected failure";
+  ASSERT_STRCASEEQ("foo", "FOO") << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRCASEEQ("foo", "bar") << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_STRCASEEQ("foo", "bar") << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, StringNotEqualIgnoringCase) {
+  EXPECT_STRCASENE("foo", "bar") << "unexpected failure";
+  ASSERT_STRCASENE("foo", "bar") << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_STRCASENE("foo", "FOO") << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_STRCASENE("bar", "BAR") << "expected failure",
+                       "expected failure");
+}
+
+TEST(StreamingAssertionsTest, FloatingPointEquals) {
+  EXPECT_FLOAT_EQ(1.0, 1.0) << "unexpected failure";
+  ASSERT_FLOAT_EQ(1.0, 1.0) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_FLOAT_EQ(0.0, 1.0) << "expected failure",
+                          "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_FLOAT_EQ(0.0, 1.0) << "expected failure",
+                       "expected failure");
+}
+
+#if GTEST_HAS_EXCEPTIONS
+
+TEST(StreamingAssertionsTest, Throw) {
+  EXPECT_THROW(ThrowAnInteger(), int) << "unexpected failure";
+  ASSERT_THROW(ThrowAnInteger(), int) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_THROW(ThrowAnInteger(), bool) <<
+                          "expected failure", "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_THROW(ThrowAnInteger(), bool) <<
+                       "expected failure", "expected failure");
+}
+
+TEST(StreamingAssertionsTest, NoThrow) {
+  EXPECT_NO_THROW(ThrowNothing()) << "unexpected failure";
+  ASSERT_NO_THROW(ThrowNothing()) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_NO_THROW(ThrowAnInteger()) <<
+                          "expected failure", "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_NO_THROW(ThrowAnInteger()) <<
+                       "expected failure", "expected failure");
+}
+
+TEST(StreamingAssertionsTest, AnyThrow) {
+  EXPECT_ANY_THROW(ThrowAnInteger()) << "unexpected failure";
+  ASSERT_ANY_THROW(ThrowAnInteger()) << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE(EXPECT_ANY_THROW(ThrowNothing()) <<
+                          "expected failure", "expected failure");
+  EXPECT_FATAL_FAILURE(ASSERT_ANY_THROW(ThrowNothing()) <<
+                       "expected failure", "expected failure");
+}
+
+#endif  // GTEST_HAS_EXCEPTIONS
+
+// Tests that Google Test correctly decides whether to use colors in the output.
+
+TEST(ColoredOutputTest, UsesColorsWhenGTestColorFlagIsYes) {
+  GTEST_FLAG(color) = "yes";
+
+  SetEnv("TERM", "xterm");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+  EXPECT_TRUE(ShouldUseColor(false));  // Stdout is not a TTY.
+
+  SetEnv("TERM", "dumb");  // TERM doesn't support colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+  EXPECT_TRUE(ShouldUseColor(false));  // Stdout is not a TTY.
+}
+
+TEST(ColoredOutputTest, UsesColorsWhenGTestColorFlagIsAliasOfYes) {
+  SetEnv("TERM", "dumb");  // TERM doesn't support colors.
+
+  GTEST_FLAG(color) = "True";
+  EXPECT_TRUE(ShouldUseColor(false));  // Stdout is not a TTY.
+
+  GTEST_FLAG(color) = "t";
+  EXPECT_TRUE(ShouldUseColor(false));  // Stdout is not a TTY.
+
+  GTEST_FLAG(color) = "1";
+  EXPECT_TRUE(ShouldUseColor(false));  // Stdout is not a TTY.
+}
+
+TEST(ColoredOutputTest, UsesNoColorWhenGTestColorFlagIsNo) {
+  GTEST_FLAG(color) = "no";
+
+  SetEnv("TERM", "xterm");  // TERM supports colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+  EXPECT_FALSE(ShouldUseColor(false));  // Stdout is not a TTY.
+
+  SetEnv("TERM", "dumb");  // TERM doesn't support colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+  EXPECT_FALSE(ShouldUseColor(false));  // Stdout is not a TTY.
+}
+
+TEST(ColoredOutputTest, UsesNoColorWhenGTestColorFlagIsInvalid) {
+  SetEnv("TERM", "xterm");  // TERM supports colors.
+
+  GTEST_FLAG(color) = "F";
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  GTEST_FLAG(color) = "0";
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  GTEST_FLAG(color) = "unknown";
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+}
+
+TEST(ColoredOutputTest, UsesColorsWhenStdoutIsTty) {
+  GTEST_FLAG(color) = "auto";
+
+  SetEnv("TERM", "xterm");  // TERM supports colors.
+  EXPECT_FALSE(ShouldUseColor(false));  // Stdout is not a TTY.
+  EXPECT_TRUE(ShouldUseColor(true));    // Stdout is a TTY.
+}
+
+TEST(ColoredOutputTest, UsesColorsWhenTermSupportsColors) {
+  GTEST_FLAG(color) = "auto";
+
+#if GTEST_OS_WINDOWS
+  // On Windows, we ignore the TERM variable as it's usually not set.
+
+  SetEnv("TERM", "dumb");
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "");
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "xterm");
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+#else
+  // On non-Windows platforms, we rely on TERM to determine if the
+  // terminal supports colors.
+
+  SetEnv("TERM", "dumb");  // TERM doesn't support colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "emacs");  // TERM doesn't support colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "vt100");  // TERM doesn't support colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "xterm-mono");  // TERM doesn't support colors.
+  EXPECT_FALSE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "xterm");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "xterm-color");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "xterm-256color");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "screen");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "screen-256color");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "tmux");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "tmux-256color");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "rxvt-unicode");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "rxvt-unicode-256color");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "linux");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+
+  SetEnv("TERM", "cygwin");  // TERM supports colors.
+  EXPECT_TRUE(ShouldUseColor(true));  // Stdout is a TTY.
+#endif  // GTEST_OS_WINDOWS
+}
+
+// Verifies that StaticAssertTypeEq works in a namespace scope.
+
+static bool dummy1 GTEST_ATTRIBUTE_UNUSED_ = StaticAssertTypeEq<bool, bool>();
+static bool dummy2 GTEST_ATTRIBUTE_UNUSED_ =
+    StaticAssertTypeEq<const int, const int>();
+
+// Verifies that StaticAssertTypeEq works in a class.
+
+template <typename T>
+class StaticAssertTypeEqTestHelper {
+ public:
+  StaticAssertTypeEqTestHelper() { StaticAssertTypeEq<bool, T>(); }
+};
+
+TEST(StaticAssertTypeEqTest, WorksInClass) {
+  StaticAssertTypeEqTestHelper<bool>();
+}
+
+// Verifies that StaticAssertTypeEq works inside a function.
+
+typedef int IntAlias;
+
+TEST(StaticAssertTypeEqTest, CompilesForEqualTypes) {
+  StaticAssertTypeEq<int, IntAlias>();
+  StaticAssertTypeEq<int*, IntAlias*>();
+}
+
+TEST(GetCurrentOsStackTraceExceptTopTest, ReturnsTheStackTrace) {
+  testing::UnitTest* const unit_test = testing::UnitTest::GetInstance();
+
+  // We don't have a stack walker in Google Test yet.
+  EXPECT_STREQ("", GetCurrentOsStackTraceExceptTop(unit_test, 0).c_str());
+  EXPECT_STREQ("", GetCurrentOsStackTraceExceptTop(unit_test, 1).c_str());
+}
+
+TEST(HasNonfatalFailureTest, ReturnsFalseWhenThereIsNoFailure) {
+  EXPECT_FALSE(HasNonfatalFailure());
+}
+
+static void FailFatally() { FAIL(); }
+
+TEST(HasNonfatalFailureTest, ReturnsFalseWhenThereIsOnlyFatalFailure) {
+  FailFatally();
+  const bool has_nonfatal_failure = HasNonfatalFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_FALSE(has_nonfatal_failure);
+}
+
+TEST(HasNonfatalFailureTest, ReturnsTrueWhenThereIsNonfatalFailure) {
+  ADD_FAILURE();
+  const bool has_nonfatal_failure = HasNonfatalFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_nonfatal_failure);
+}
+
+TEST(HasNonfatalFailureTest, ReturnsTrueWhenThereAreFatalAndNonfatalFailures) {
+  FailFatally();
+  ADD_FAILURE();
+  const bool has_nonfatal_failure = HasNonfatalFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_nonfatal_failure);
+}
+
+// A wrapper for calling HasNonfatalFailure outside of a test body.
+static bool HasNonfatalFailureHelper() {
+  return testing::Test::HasNonfatalFailure();
+}
+
+TEST(HasNonfatalFailureTest, WorksOutsideOfTestBody) {
+  EXPECT_FALSE(HasNonfatalFailureHelper());
+}
+
+TEST(HasNonfatalFailureTest, WorksOutsideOfTestBody2) {
+  ADD_FAILURE();
+  const bool has_nonfatal_failure = HasNonfatalFailureHelper();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_nonfatal_failure);
+}
+
+TEST(HasFailureTest, ReturnsFalseWhenThereIsNoFailure) {
+  EXPECT_FALSE(HasFailure());
+}
+
+TEST(HasFailureTest, ReturnsTrueWhenThereIsFatalFailure) {
+  FailFatally();
+  const bool has_failure = HasFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_failure);
+}
+
+TEST(HasFailureTest, ReturnsTrueWhenThereIsNonfatalFailure) {
+  ADD_FAILURE();
+  const bool has_failure = HasFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_failure);
+}
+
+TEST(HasFailureTest, ReturnsTrueWhenThereAreFatalAndNonfatalFailures) {
+  FailFatally();
+  ADD_FAILURE();
+  const bool has_failure = HasFailure();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_failure);
+}
+
+// A wrapper for calling HasFailure outside of a test body.
+static bool HasFailureHelper() { return testing::Test::HasFailure(); }
+
+TEST(HasFailureTest, WorksOutsideOfTestBody) {
+  EXPECT_FALSE(HasFailureHelper());
+}
+
+TEST(HasFailureTest, WorksOutsideOfTestBody2) {
+  ADD_FAILURE();
+  const bool has_failure = HasFailureHelper();
+  ClearCurrentTestPartResults();
+  EXPECT_TRUE(has_failure);
+}
+
+class TestListener : public EmptyTestEventListener {
+ public:
+  TestListener() : on_start_counter_(NULL), is_destroyed_(NULL) {}
+  TestListener(int* on_start_counter, bool* is_destroyed)
+      : on_start_counter_(on_start_counter),
+        is_destroyed_(is_destroyed) {}
+
+  virtual ~TestListener() {
+    if (is_destroyed_)
+      *is_destroyed_ = true;
+  }
+
+ protected:
+  virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {
+    if (on_start_counter_ != NULL)
+      (*on_start_counter_)++;
+  }
+
+ private:
+  int* on_start_counter_;
+  bool* is_destroyed_;
+};
+
+// Tests the constructor.
+TEST(TestEventListenersTest, ConstructionWorks) {
+  TestEventListeners listeners;
+
+  EXPECT_TRUE(TestEventListenersAccessor::GetRepeater(&listeners) != NULL);
+  EXPECT_TRUE(listeners.default_result_printer() == NULL);
+  EXPECT_TRUE(listeners.default_xml_generator() == NULL);
+}
+
+// Tests that the TestEventListeners destructor deletes all the listeners it
+// owns.
+TEST(TestEventListenersTest, DestructionWorks) {
+  bool default_result_printer_is_destroyed = false;
+  bool default_xml_printer_is_destroyed = false;
+  bool extra_listener_is_destroyed = false;
+  TestListener* default_result_printer = new TestListener(
+      NULL, &default_result_printer_is_destroyed);
+  TestListener* default_xml_printer = new TestListener(
+      NULL, &default_xml_printer_is_destroyed);
+  TestListener* extra_listener = new TestListener(
+      NULL, &extra_listener_is_destroyed);
+
+  {
+    TestEventListeners listeners;
+    TestEventListenersAccessor::SetDefaultResultPrinter(&listeners,
+                                                        default_result_printer);
+    TestEventListenersAccessor::SetDefaultXmlGenerator(&listeners,
+                                                       default_xml_printer);
+    listeners.Append(extra_listener);
+  }
+  EXPECT_TRUE(default_result_printer_is_destroyed);
+  EXPECT_TRUE(default_xml_printer_is_destroyed);
+  EXPECT_TRUE(extra_listener_is_destroyed);
+}
+
+// Tests that a listener Append'ed to a TestEventListeners list starts
+// receiving events.
+TEST(TestEventListenersTest, Append) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+  {
+    TestEventListeners listeners;
+    listeners.Append(listener);
+    TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+        *UnitTest::GetInstance());
+    EXPECT_EQ(1, on_start_counter);
+  }
+  EXPECT_TRUE(is_destroyed);
+}
+
+// Tests that listeners receive events in the order they were appended to
+// the list, except for *End requests, which must be received in the reverse
+// order.
+class SequenceTestingListener : public EmptyTestEventListener {
+ public:
+  SequenceTestingListener(std::vector<std::string>* vector, const char* id)
+      : vector_(vector), id_(id) {}
+
+ protected:
+  virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {
+    vector_->push_back(GetEventDescription("OnTestProgramStart"));
+  }
+
+  virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {
+    vector_->push_back(GetEventDescription("OnTestProgramEnd"));
+  }
+
+  virtual void OnTestIterationStart(const UnitTest& /*unit_test*/,
+                                    int /*iteration*/) {
+    vector_->push_back(GetEventDescription("OnTestIterationStart"));
+  }
+
+  virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/,
+                                  int /*iteration*/) {
+    vector_->push_back(GetEventDescription("OnTestIterationEnd"));
+  }
+
+ private:
+  std::string GetEventDescription(const char* method) {
+    Message message;
+    message << id_ << "." << method;
+    return message.GetString();
+  }
+
+  std::vector<std::string>* vector_;
+  const char* const id_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(SequenceTestingListener);
+};
+
+TEST(EventListenerTest, AppendKeepsOrder) {
+  std::vector<std::string> vec;
+  TestEventListeners listeners;
+  listeners.Append(new SequenceTestingListener(&vec, "1st"));
+  listeners.Append(new SequenceTestingListener(&vec, "2nd"));
+  listeners.Append(new SequenceTestingListener(&vec, "3rd"));
+
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+  ASSERT_EQ(3U, vec.size());
+  EXPECT_STREQ("1st.OnTestProgramStart", vec[0].c_str());
+  EXPECT_STREQ("2nd.OnTestProgramStart", vec[1].c_str());
+  EXPECT_STREQ("3rd.OnTestProgramStart", vec[2].c_str());
+
+  vec.clear();
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramEnd(
+      *UnitTest::GetInstance());
+  ASSERT_EQ(3U, vec.size());
+  EXPECT_STREQ("3rd.OnTestProgramEnd", vec[0].c_str());
+  EXPECT_STREQ("2nd.OnTestProgramEnd", vec[1].c_str());
+  EXPECT_STREQ("1st.OnTestProgramEnd", vec[2].c_str());
+
+  vec.clear();
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestIterationStart(
+      *UnitTest::GetInstance(), 0);
+  ASSERT_EQ(3U, vec.size());
+  EXPECT_STREQ("1st.OnTestIterationStart", vec[0].c_str());
+  EXPECT_STREQ("2nd.OnTestIterationStart", vec[1].c_str());
+  EXPECT_STREQ("3rd.OnTestIterationStart", vec[2].c_str());
+
+  vec.clear();
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestIterationEnd(
+      *UnitTest::GetInstance(), 0);
+  ASSERT_EQ(3U, vec.size());
+  EXPECT_STREQ("3rd.OnTestIterationEnd", vec[0].c_str());
+  EXPECT_STREQ("2nd.OnTestIterationEnd", vec[1].c_str());
+  EXPECT_STREQ("1st.OnTestIterationEnd", vec[2].c_str());
+}
+
+// Tests that a listener removed from a TestEventListeners list stops receiving
+// events and is not deleted when the list is destroyed.
+TEST(TestEventListenersTest, Release) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  // Although Append passes the ownership of this object to the list,
+  // the following calls release it, and we need to delete it before the
+  // test ends.
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+  {
+    TestEventListeners listeners;
+    listeners.Append(listener);
+    EXPECT_EQ(listener, listeners.Release(listener));
+    TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+        *UnitTest::GetInstance());
+    EXPECT_TRUE(listeners.Release(listener) == NULL);
+  }
+  EXPECT_EQ(0, on_start_counter);
+  EXPECT_FALSE(is_destroyed);
+  delete listener;
+}
+
+// Tests that no events are forwarded when event forwarding is disabled.
+TEST(EventListenerTest, SuppressEventForwarding) {
+  int on_start_counter = 0;
+  TestListener* listener = new TestListener(&on_start_counter, NULL);
+
+  TestEventListeners listeners;
+  listeners.Append(listener);
+  ASSERT_TRUE(TestEventListenersAccessor::EventForwardingEnabled(listeners));
+  TestEventListenersAccessor::SuppressEventForwarding(&listeners);
+  ASSERT_FALSE(TestEventListenersAccessor::EventForwardingEnabled(listeners));
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+  EXPECT_EQ(0, on_start_counter);
+}
+
+// Tests that events generated by Google Test are not forwarded in
+// death test subprocesses.
+TEST(EventListenerDeathTest, EventsNotForwardedInDeathTestSubprecesses) {
+  EXPECT_DEATH_IF_SUPPORTED({
+      GTEST_CHECK_(TestEventListenersAccessor::EventForwardingEnabled(
+          *GetUnitTestImpl()->listeners())) << "expected failure";},
+      "expected failure");
+}
+
+// Tests that a listener installed via SetDefaultResultPrinter() starts
+// receiving events and is returned via default_result_printer() and that
+// the previous default_result_printer is removed from the list and deleted.
+TEST(EventListenerTest, default_result_printer) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+
+  TestEventListeners listeners;
+  TestEventListenersAccessor::SetDefaultResultPrinter(&listeners, listener);
+
+  EXPECT_EQ(listener, listeners.default_result_printer());
+
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+
+  EXPECT_EQ(1, on_start_counter);
+
+  // Replacing default_result_printer with something else should remove it
+  // from the list and destroy it.
+  TestEventListenersAccessor::SetDefaultResultPrinter(&listeners, NULL);
+
+  EXPECT_TRUE(listeners.default_result_printer() == NULL);
+  EXPECT_TRUE(is_destroyed);
+
+  // After broadcasting an event the counter is still the same, indicating
+  // the listener is not in the list anymore.
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+  EXPECT_EQ(1, on_start_counter);
+}
+
+// Tests that the default_result_printer listener stops receiving events
+// when removed via Release and that is not owned by the list anymore.
+TEST(EventListenerTest, RemovingDefaultResultPrinterWorks) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  // Although Append passes the ownership of this object to the list,
+  // the following calls release it, and we need to delete it before the
+  // test ends.
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+  {
+    TestEventListeners listeners;
+    TestEventListenersAccessor::SetDefaultResultPrinter(&listeners, listener);
+
+    EXPECT_EQ(listener, listeners.Release(listener));
+    EXPECT_TRUE(listeners.default_result_printer() == NULL);
+    EXPECT_FALSE(is_destroyed);
+
+    // Broadcasting events now should not affect default_result_printer.
+    TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+        *UnitTest::GetInstance());
+    EXPECT_EQ(0, on_start_counter);
+  }
+  // Destroying the list should not affect the listener now, too.
+  EXPECT_FALSE(is_destroyed);
+  delete listener;
+}
+
+// Tests that a listener installed via SetDefaultXmlGenerator() starts
+// receiving events and is returned via default_xml_generator() and that
+// the previous default_xml_generator is removed from the list and deleted.
+TEST(EventListenerTest, default_xml_generator) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+
+  TestEventListeners listeners;
+  TestEventListenersAccessor::SetDefaultXmlGenerator(&listeners, listener);
+
+  EXPECT_EQ(listener, listeners.default_xml_generator());
+
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+
+  EXPECT_EQ(1, on_start_counter);
+
+  // Replacing default_xml_generator with something else should remove it
+  // from the list and destroy it.
+  TestEventListenersAccessor::SetDefaultXmlGenerator(&listeners, NULL);
+
+  EXPECT_TRUE(listeners.default_xml_generator() == NULL);
+  EXPECT_TRUE(is_destroyed);
+
+  // After broadcasting an event the counter is still the same, indicating
+  // the listener is not in the list anymore.
+  TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+      *UnitTest::GetInstance());
+  EXPECT_EQ(1, on_start_counter);
+}
+
+// Tests that the default_xml_generator listener stops receiving events
+// when removed via Release and that is not owned by the list anymore.
+TEST(EventListenerTest, RemovingDefaultXmlGeneratorWorks) {
+  int on_start_counter = 0;
+  bool is_destroyed = false;
+  // Although Append passes the ownership of this object to the list,
+  // the following calls release it, and we need to delete it before the
+  // test ends.
+  TestListener* listener = new TestListener(&on_start_counter, &is_destroyed);
+  {
+    TestEventListeners listeners;
+    TestEventListenersAccessor::SetDefaultXmlGenerator(&listeners, listener);
+
+    EXPECT_EQ(listener, listeners.Release(listener));
+    EXPECT_TRUE(listeners.default_xml_generator() == NULL);
+    EXPECT_FALSE(is_destroyed);
+
+    // Broadcasting events now should not affect default_xml_generator.
+    TestEventListenersAccessor::GetRepeater(&listeners)->OnTestProgramStart(
+        *UnitTest::GetInstance());
+    EXPECT_EQ(0, on_start_counter);
+  }
+  // Destroying the list should not affect the listener now, too.
+  EXPECT_FALSE(is_destroyed);
+  delete listener;
+}
+
+// Sanity tests to ensure that the alternative, verbose spellings of
+// some of the macros work.  We don't test them thoroughly as that
+// would be quite involved.  Since their implementations are
+// straightforward, and they are rarely used, we'll just rely on the
+// users to tell us when they are broken.
+GTEST_TEST(AlternativeNameTest, Works) {  // GTEST_TEST is the same as TEST.
+  GTEST_SUCCEED() << "OK";  // GTEST_SUCCEED is the same as SUCCEED.
+
+  // GTEST_FAIL is the same as FAIL.
+  EXPECT_FATAL_FAILURE(GTEST_FAIL() << "An expected failure",
+                       "An expected failure");
+
+  // GTEST_ASSERT_XY is the same as ASSERT_XY.
+
+  GTEST_ASSERT_EQ(0, 0);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_EQ(0, 1) << "An expected failure",
+                       "An expected failure");
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_EQ(1, 0) << "An expected failure",
+                       "An expected failure");
+
+  GTEST_ASSERT_NE(0, 1);
+  GTEST_ASSERT_NE(1, 0);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_NE(0, 0) << "An expected failure",
+                       "An expected failure");
+
+  GTEST_ASSERT_LE(0, 0);
+  GTEST_ASSERT_LE(0, 1);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_LE(1, 0) << "An expected failure",
+                       "An expected failure");
+
+  GTEST_ASSERT_LT(0, 1);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_LT(0, 0) << "An expected failure",
+                       "An expected failure");
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_LT(1, 0) << "An expected failure",
+                       "An expected failure");
+
+  GTEST_ASSERT_GE(0, 0);
+  GTEST_ASSERT_GE(1, 0);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_GE(0, 1) << "An expected failure",
+                       "An expected failure");
+
+  GTEST_ASSERT_GT(1, 0);
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_GT(0, 1) << "An expected failure",
+                       "An expected failure");
+  EXPECT_FATAL_FAILURE(GTEST_ASSERT_GT(1, 1) << "An expected failure",
+                       "An expected failure");
+}
+
+// Tests for internal utilities necessary for implementation of the universal
+// printing.
+// TODO(vladl@google.com): Find a better home for them.
+
+class ConversionHelperBase {};
+class ConversionHelperDerived : public ConversionHelperBase {};
+
+// Tests that IsAProtocolMessage<T>::value is a compile-time constant.
+TEST(IsAProtocolMessageTest, ValueIsCompileTimeConstant) {
+  GTEST_COMPILE_ASSERT_(IsAProtocolMessage<ProtocolMessage>::value,
+                        const_true);
+  GTEST_COMPILE_ASSERT_(!IsAProtocolMessage<int>::value, const_false);
+}
+
+// Tests that IsAProtocolMessage<T>::value is true when T is
+// proto2::Message or a sub-class of it.
+TEST(IsAProtocolMessageTest, ValueIsTrueWhenTypeIsAProtocolMessage) {
+  EXPECT_TRUE(IsAProtocolMessage< ::proto2::Message>::value);
+  EXPECT_TRUE(IsAProtocolMessage<ProtocolMessage>::value);
+}
+
+// Tests that IsAProtocolMessage<T>::value is false when T is neither
+// ProtocolMessage nor a sub-class of it.
+TEST(IsAProtocolMessageTest, ValueIsFalseWhenTypeIsNotAProtocolMessage) {
+  EXPECT_FALSE(IsAProtocolMessage<int>::value);
+  EXPECT_FALSE(IsAProtocolMessage<const ConversionHelperBase>::value);
+}
+
+// Tests that CompileAssertTypesEqual compiles when the type arguments are
+// equal.
+TEST(CompileAssertTypesEqual, CompilesWhenTypesAreEqual) {
+  CompileAssertTypesEqual<void, void>();
+  CompileAssertTypesEqual<int*, int*>();
+}
+
+// Tests that RemoveReference does not affect non-reference types.
+TEST(RemoveReferenceTest, DoesNotAffectNonReferenceType) {
+  CompileAssertTypesEqual<int, RemoveReference<int>::type>();
+  CompileAssertTypesEqual<const char, RemoveReference<const char>::type>();
+}
+
+// Tests that RemoveReference removes reference from reference types.
+TEST(RemoveReferenceTest, RemovesReference) {
+  CompileAssertTypesEqual<int, RemoveReference<int&>::type>();
+  CompileAssertTypesEqual<const char, RemoveReference<const char&>::type>();
+}
+
+// Tests GTEST_REMOVE_REFERENCE_.
+
+template <typename T1, typename T2>
+void TestGTestRemoveReference() {
+  CompileAssertTypesEqual<T1, GTEST_REMOVE_REFERENCE_(T2)>();
+}
+
+TEST(RemoveReferenceTest, MacroVersion) {
+  TestGTestRemoveReference<int, int>();
+  TestGTestRemoveReference<const char, const char&>();
+}
+
+
+// Tests that RemoveConst does not affect non-const types.
+TEST(RemoveConstTest, DoesNotAffectNonConstType) {
+  CompileAssertTypesEqual<int, RemoveConst<int>::type>();
+  CompileAssertTypesEqual<char&, RemoveConst<char&>::type>();
+}
+
+// Tests that RemoveConst removes const from const types.
+TEST(RemoveConstTest, RemovesConst) {
+  CompileAssertTypesEqual<int, RemoveConst<const int>::type>();
+  CompileAssertTypesEqual<char[2], RemoveConst<const char[2]>::type>();
+  CompileAssertTypesEqual<char[2][3], RemoveConst<const char[2][3]>::type>();
+}
+
+// Tests GTEST_REMOVE_CONST_.
+
+template <typename T1, typename T2>
+void TestGTestRemoveConst() {
+  CompileAssertTypesEqual<T1, GTEST_REMOVE_CONST_(T2)>();
+}
+
+TEST(RemoveConstTest, MacroVersion) {
+  TestGTestRemoveConst<int, int>();
+  TestGTestRemoveConst<double&, double&>();
+  TestGTestRemoveConst<char, const char>();
+}
+
+// Tests GTEST_REMOVE_REFERENCE_AND_CONST_.
+
+template <typename T1, typename T2>
+void TestGTestRemoveReferenceAndConst() {
+  CompileAssertTypesEqual<T1, GTEST_REMOVE_REFERENCE_AND_CONST_(T2)>();
+}
+
+TEST(RemoveReferenceToConstTest, Works) {
+  TestGTestRemoveReferenceAndConst<int, int>();
+  TestGTestRemoveReferenceAndConst<double, double&>();
+  TestGTestRemoveReferenceAndConst<char, const char>();
+  TestGTestRemoveReferenceAndConst<char, const char&>();
+  TestGTestRemoveReferenceAndConst<const char*, const char*>();
+}
+
+// Tests that AddReference does not affect reference types.
+TEST(AddReferenceTest, DoesNotAffectReferenceType) {
+  CompileAssertTypesEqual<int&, AddReference<int&>::type>();
+  CompileAssertTypesEqual<const char&, AddReference<const char&>::type>();
+}
+
+// Tests that AddReference adds reference to non-reference types.
+TEST(AddReferenceTest, AddsReference) {
+  CompileAssertTypesEqual<int&, AddReference<int>::type>();
+  CompileAssertTypesEqual<const char&, AddReference<const char>::type>();
+}
+
+// Tests GTEST_ADD_REFERENCE_.
+
+template <typename T1, typename T2>
+void TestGTestAddReference() {
+  CompileAssertTypesEqual<T1, GTEST_ADD_REFERENCE_(T2)>();
+}
+
+TEST(AddReferenceTest, MacroVersion) {
+  TestGTestAddReference<int&, int>();
+  TestGTestAddReference<const char&, const char&>();
+}
+
+// Tests GTEST_REFERENCE_TO_CONST_.
+
+template <typename T1, typename T2>
+void TestGTestReferenceToConst() {
+  CompileAssertTypesEqual<T1, GTEST_REFERENCE_TO_CONST_(T2)>();
+}
+
+TEST(GTestReferenceToConstTest, Works) {
+  TestGTestReferenceToConst<const char&, char>();
+  TestGTestReferenceToConst<const int&, const int>();
+  TestGTestReferenceToConst<const double&, double>();
+  TestGTestReferenceToConst<const std::string&, const std::string&>();
+}
+
+// Tests that ImplicitlyConvertible<T1, T2>::value is a compile-time constant.
+TEST(ImplicitlyConvertibleTest, ValueIsCompileTimeConstant) {
+  GTEST_COMPILE_ASSERT_((ImplicitlyConvertible<int, int>::value), const_true);
+  GTEST_COMPILE_ASSERT_((!ImplicitlyConvertible<void*, int*>::value),
+                        const_false);
+}
+
+// Tests that ImplicitlyConvertible<T1, T2>::value is true when T1 can
+// be implicitly converted to T2.
+TEST(ImplicitlyConvertibleTest, ValueIsTrueWhenConvertible) {
+  EXPECT_TRUE((ImplicitlyConvertible<int, double>::value));
+  EXPECT_TRUE((ImplicitlyConvertible<double, int>::value));
+  EXPECT_TRUE((ImplicitlyConvertible<int*, void*>::value));
+  EXPECT_TRUE((ImplicitlyConvertible<int*, const int*>::value));
+  EXPECT_TRUE((ImplicitlyConvertible<ConversionHelperDerived&,
+                                     const ConversionHelperBase&>::value));
+  EXPECT_TRUE((ImplicitlyConvertible<const ConversionHelperBase,
+                                     ConversionHelperBase>::value));
+}
+
+// Tests that ImplicitlyConvertible<T1, T2>::value is false when T1
+// cannot be implicitly converted to T2.
+TEST(ImplicitlyConvertibleTest, ValueIsFalseWhenNotConvertible) {
+  EXPECT_FALSE((ImplicitlyConvertible<double, int*>::value));
+  EXPECT_FALSE((ImplicitlyConvertible<void*, int*>::value));
+  EXPECT_FALSE((ImplicitlyConvertible<const int*, int*>::value));
+  EXPECT_FALSE((ImplicitlyConvertible<ConversionHelperBase&,
+                                      ConversionHelperDerived&>::value));
+}
+
+// Tests IsContainerTest.
+
+class NonContainer {};
+
+TEST(IsContainerTestTest, WorksForNonContainer) {
+  EXPECT_EQ(sizeof(IsNotContainer), sizeof(IsContainerTest<int>(0)));
+  EXPECT_EQ(sizeof(IsNotContainer), sizeof(IsContainerTest<char[5]>(0)));
+  EXPECT_EQ(sizeof(IsNotContainer), sizeof(IsContainerTest<NonContainer>(0)));
+}
+
+TEST(IsContainerTestTest, WorksForContainer) {
+  EXPECT_EQ(sizeof(IsContainer),
+            sizeof(IsContainerTest<std::vector<bool> >(0)));
+  EXPECT_EQ(sizeof(IsContainer),
+            sizeof(IsContainerTest<std::map<int, double> >(0)));
+}
+
+// Tests ArrayEq().
+
+TEST(ArrayEqTest, WorksForDegeneratedArrays) {
+  EXPECT_TRUE(ArrayEq(5, 5L));
+  EXPECT_FALSE(ArrayEq('a', 0));
+}
+
+TEST(ArrayEqTest, WorksForOneDimensionalArrays) {
+  // Note that a and b are distinct but compatible types.
+  const int a[] = { 0, 1 };
+  long b[] = { 0, 1 };
+  EXPECT_TRUE(ArrayEq(a, b));
+  EXPECT_TRUE(ArrayEq(a, 2, b));
+
+  b[0] = 2;
+  EXPECT_FALSE(ArrayEq(a, b));
+  EXPECT_FALSE(ArrayEq(a, 1, b));
+}
+
+TEST(ArrayEqTest, WorksForTwoDimensionalArrays) {
+  const char a[][3] = { "hi", "lo" };
+  const char b[][3] = { "hi", "lo" };
+  const char c[][3] = { "hi", "li" };
+
+  EXPECT_TRUE(ArrayEq(a, b));
+  EXPECT_TRUE(ArrayEq(a, 2, b));
+
+  EXPECT_FALSE(ArrayEq(a, c));
+  EXPECT_FALSE(ArrayEq(a, 2, c));
+}
+
+// Tests ArrayAwareFind().
+
+TEST(ArrayAwareFindTest, WorksForOneDimensionalArray) {
+  const char a[] = "hello";
+  EXPECT_EQ(a + 4, ArrayAwareFind(a, a + 5, 'o'));
+  EXPECT_EQ(a + 5, ArrayAwareFind(a, a + 5, 'x'));
+}
+
+TEST(ArrayAwareFindTest, WorksForTwoDimensionalArray) {
+  int a[][2] = { { 0, 1 }, { 2, 3 }, { 4, 5 } };
+  const int b[2] = { 2, 3 };
+  EXPECT_EQ(a + 1, ArrayAwareFind(a, a + 3, b));
+
+  const int c[2] = { 6, 7 };
+  EXPECT_EQ(a + 3, ArrayAwareFind(a, a + 3, c));
+}
+
+// Tests CopyArray().
+
+TEST(CopyArrayTest, WorksForDegeneratedArrays) {
+  int n = 0;
+  CopyArray('a', &n);
+  EXPECT_EQ('a', n);
+}
+
+TEST(CopyArrayTest, WorksForOneDimensionalArrays) {
+  const char a[3] = "hi";
+  int b[3];
+#ifndef __BORLANDC__  // C++Builder cannot compile some array size deductions.
+  CopyArray(a, &b);
+  EXPECT_TRUE(ArrayEq(a, b));
+#endif
+
+  int c[3];
+  CopyArray(a, 3, c);
+  EXPECT_TRUE(ArrayEq(a, c));
+}
+
+TEST(CopyArrayTest, WorksForTwoDimensionalArrays) {
+  const int a[2][3] = { { 0, 1, 2 }, { 3, 4, 5 } };
+  int b[2][3];
+#ifndef __BORLANDC__  // C++Builder cannot compile some array size deductions.
+  CopyArray(a, &b);
+  EXPECT_TRUE(ArrayEq(a, b));
+#endif
+
+  int c[2][3];
+  CopyArray(a, 2, c);
+  EXPECT_TRUE(ArrayEq(a, c));
+}
+
+// Tests NativeArray.
+
+TEST(NativeArrayTest, ConstructorFromArrayWorks) {
+  const int a[3] = { 0, 1, 2 };
+  NativeArray<int> na(a, 3, RelationToSourceReference());
+  EXPECT_EQ(3U, na.size());
+  EXPECT_EQ(a, na.begin());
+}
+
+TEST(NativeArrayTest, CreatesAndDeletesCopyOfArrayWhenAskedTo) {
+  typedef int Array[2];
+  Array* a = new Array[1];
+  (*a)[0] = 0;
+  (*a)[1] = 1;
+  NativeArray<int> na(*a, 2, RelationToSourceCopy());
+  EXPECT_NE(*a, na.begin());
+  delete[] a;
+  EXPECT_EQ(0, na.begin()[0]);
+  EXPECT_EQ(1, na.begin()[1]);
+
+  // We rely on the heap checker to verify that na deletes the copy of
+  // array.
+}
+
+TEST(NativeArrayTest, TypeMembersAreCorrect) {
+  StaticAssertTypeEq<char, NativeArray<char>::value_type>();
+  StaticAssertTypeEq<int[2], NativeArray<int[2]>::value_type>();
+
+  StaticAssertTypeEq<const char*, NativeArray<char>::const_iterator>();
+  StaticAssertTypeEq<const bool(*)[2], NativeArray<bool[2]>::const_iterator>();
+}
+
+TEST(NativeArrayTest, MethodsWork) {
+  const int a[3] = { 0, 1, 2 };
+  NativeArray<int> na(a, 3, RelationToSourceCopy());
+  ASSERT_EQ(3U, na.size());
+  EXPECT_EQ(3, na.end() - na.begin());
+
+  NativeArray<int>::const_iterator it = na.begin();
+  EXPECT_EQ(0, *it);
+  ++it;
+  EXPECT_EQ(1, *it);
+  it++;
+  EXPECT_EQ(2, *it);
+  ++it;
+  EXPECT_EQ(na.end(), it);
+
+  EXPECT_TRUE(na == na);
+
+  NativeArray<int> na2(a, 3, RelationToSourceReference());
+  EXPECT_TRUE(na == na2);
+
+  const int b1[3] = { 0, 1, 1 };
+  const int b2[4] = { 0, 1, 2, 3 };
+  EXPECT_FALSE(na == NativeArray<int>(b1, 3, RelationToSourceReference()));
+  EXPECT_FALSE(na == NativeArray<int>(b2, 4, RelationToSourceCopy()));
+}
+
+TEST(NativeArrayTest, WorksForTwoDimensionalArray) {
+  const char a[2][3] = { "hi", "lo" };
+  NativeArray<char[3]> na(a, 2, RelationToSourceReference());
+  ASSERT_EQ(2U, na.size());
+  EXPECT_EQ(a, na.begin());
+}
+
+// Tests SkipPrefix().
+
+TEST(SkipPrefixTest, SkipsWhenPrefixMatches) {
+  const char* const str = "hello";
+
+  const char* p = str;
+  EXPECT_TRUE(SkipPrefix("", &p));
+  EXPECT_EQ(str, p);
+
+  p = str;
+  EXPECT_TRUE(SkipPrefix("hell", &p));
+  EXPECT_EQ(str + 4, p);
+}
+
+TEST(SkipPrefixTest, DoesNotSkipWhenPrefixDoesNotMatch) {
+  const char* const str = "world";
+
+  const char* p = str;
+  EXPECT_FALSE(SkipPrefix("W", &p));
+  EXPECT_EQ(str, p);
+
+  p = str;
+  EXPECT_FALSE(SkipPrefix("world!", &p));
+  EXPECT_EQ(str, p);
+}
+
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: keith.ray@gmail.com (Keith Ray)
+//
+// gtest_xml_outfile1_test_ writes some xml via TestProperty used by
+// gtest_xml_outfiles_test.py
+
+#include "gtest/gtest.h"
+
+class PropertyOne : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    RecordProperty("SetUpProp", 1);
+  }
+  virtual void TearDown() {
+    RecordProperty("TearDownProp", 1);
+  }
+};
+
+TEST_F(PropertyOne, TestSomeProperties) {
+  RecordProperty("TestSomeProperty", 1);
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: keith.ray@gmail.com (Keith Ray)
+//
+// gtest_xml_outfile2_test_ writes some xml via TestProperty used by
+// gtest_xml_outfiles_test.py
+
+#include "gtest/gtest.h"
+
+class PropertyTwo : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    RecordProperty("SetUpProp", 2);
+  }
+  virtual void TearDown() {
+    RecordProperty("TearDownProp", 2);
+  }
+};
+
+TEST_F(PropertyTwo, TestSomeProperties) {
+  RecordProperty("TestSomeProperty", 2);
+}
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// This is part of the unit test for include/gtest/gtest_prod.h.
+
+#include "production.h"
+
+PrivateCode::PrivateCode() : x_(0) {}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// Tests for death tests.
+
+#include "gtest/gtest-death-test.h"
+#include "gtest/gtest.h"
+#include "gtest/internal/gtest-filepath.h"
+
+using testing::internal::AlwaysFalse;
+using testing::internal::AlwaysTrue;
+
+#if GTEST_HAS_DEATH_TEST
+
+# if GTEST_OS_WINDOWS
+#  include <direct.h>          // For chdir().
+# else
+#  include <unistd.h>
+#  include <sys/wait.h>        // For waitpid.
+# endif  // GTEST_OS_WINDOWS
+
+# include <limits.h>
+# include <signal.h>
+# include <stdio.h>
+
+# if GTEST_OS_LINUX
+#  include <sys/time.h>
+# endif  // GTEST_OS_LINUX
+
+# include "gtest/gtest-spi.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+# define GTEST_IMPLEMENTATION_ 1
+# include "src/gtest-internal-inl.h"
+# undef GTEST_IMPLEMENTATION_
+
+namespace posix = ::testing::internal::posix;
+
+using testing::Message;
+using testing::internal::DeathTest;
+using testing::internal::DeathTestFactory;
+using testing::internal::FilePath;
+using testing::internal::GetLastErrnoDescription;
+using testing::internal::GetUnitTestImpl;
+using testing::internal::InDeathTestChild;
+using testing::internal::ParseNaturalNumber;
+
+namespace testing {
+namespace internal {
+
+// A helper class whose objects replace the death test factory for a
+// single UnitTest object during their lifetimes.
+class ReplaceDeathTestFactory {
+ public:
+  explicit ReplaceDeathTestFactory(DeathTestFactory* new_factory)
+      : unit_test_impl_(GetUnitTestImpl()) {
+    old_factory_ = unit_test_impl_->death_test_factory_.release();
+    unit_test_impl_->death_test_factory_.reset(new_factory);
+  }
+
+  ~ReplaceDeathTestFactory() {
+    unit_test_impl_->death_test_factory_.release();
+    unit_test_impl_->death_test_factory_.reset(old_factory_);
+  }
+ private:
+  // Prevents copying ReplaceDeathTestFactory objects.
+  ReplaceDeathTestFactory(const ReplaceDeathTestFactory&);
+  void operator=(const ReplaceDeathTestFactory&);
+
+  UnitTestImpl* unit_test_impl_;
+  DeathTestFactory* old_factory_;
+};
+
+}  // namespace internal
+}  // namespace testing
+
+void DieWithMessage(const ::std::string& message) {
+  fprintf(stderr, "%s", message.c_str());
+  fflush(stderr);  // Make sure the text is printed before the process exits.
+
+  // We call _exit() instead of exit(), as the former is a direct
+  // system call and thus safer in the presence of threads.  exit()
+  // will invoke user-defined exit-hooks, which may do dangerous
+  // things that conflict with death tests.
+  //
+  // Some compilers can recognize that _exit() never returns and issue the
+  // 'unreachable code' warning for code following this function, unless
+  // fooled by a fake condition.
+  if (AlwaysTrue())
+    _exit(1);
+}
+
+void DieInside(const ::std::string& function) {
+  DieWithMessage("death inside " + function + "().");
+}
+
+// Tests that death tests work.
+
+class TestForDeathTest : public testing::Test {
+ protected:
+  TestForDeathTest() : original_dir_(FilePath::GetCurrentDir()) {}
+
+  virtual ~TestForDeathTest() {
+    posix::ChDir(original_dir_.c_str());
+  }
+
+  // A static member function that's expected to die.
+  static void StaticMemberFunction() { DieInside("StaticMemberFunction"); }
+
+  // A method of the test fixture that may die.
+  void MemberFunction() {
+    if (should_die_)
+      DieInside("MemberFunction");
+  }
+
+  // True iff MemberFunction() should die.
+  bool should_die_;
+  const FilePath original_dir_;
+};
+
+// A class with a member function that may die.
+class MayDie {
+ public:
+  explicit MayDie(bool should_die) : should_die_(should_die) {}
+
+  // A member function that may die.
+  void MemberFunction() const {
+    if (should_die_)
+      DieInside("MayDie::MemberFunction");
+  }
+
+ private:
+  // True iff MemberFunction() should die.
+  bool should_die_;
+};
+
+// A global function that's expected to die.
+void GlobalFunction() { DieInside("GlobalFunction"); }
+
+// A non-void function that's expected to die.
+int NonVoidFunction() {
+  DieInside("NonVoidFunction");
+  return 1;
+}
+
+// A unary function that may die.
+void DieIf(bool should_die) {
+  if (should_die)
+    DieInside("DieIf");
+}
+
+// A binary function that may die.
+bool DieIfLessThan(int x, int y) {
+  if (x < y) {
+    DieInside("DieIfLessThan");
+  }
+  return true;
+}
+
+// Tests that ASSERT_DEATH can be used outside a TEST, TEST_F, or test fixture.
+void DeathTestSubroutine() {
+  EXPECT_DEATH(GlobalFunction(), "death.*GlobalFunction");
+  ASSERT_DEATH(GlobalFunction(), "death.*GlobalFunction");
+}
+
+// Death in dbg, not opt.
+int DieInDebugElse12(int* sideeffect) {
+  if (sideeffect) *sideeffect = 12;
+
+# ifndef NDEBUG
+
+  DieInside("DieInDebugElse12");
+
+# endif  // NDEBUG
+
+  return 12;
+}
+
+# if GTEST_OS_WINDOWS
+
+// Tests the ExitedWithCode predicate.
+TEST(ExitStatusPredicateTest, ExitedWithCode) {
+  // On Windows, the process's exit code is the same as its exit status,
+  // so the predicate just compares the its input with its parameter.
+  EXPECT_TRUE(testing::ExitedWithCode(0)(0));
+  EXPECT_TRUE(testing::ExitedWithCode(1)(1));
+  EXPECT_TRUE(testing::ExitedWithCode(42)(42));
+  EXPECT_FALSE(testing::ExitedWithCode(0)(1));
+  EXPECT_FALSE(testing::ExitedWithCode(1)(0));
+}
+
+# else
+
+// Returns the exit status of a process that calls _exit(2) with a
+// given exit code.  This is a helper function for the
+// ExitStatusPredicateTest test suite.
+static int NormalExitStatus(int exit_code) {
+  pid_t child_pid = fork();
+  if (child_pid == 0) {
+    _exit(exit_code);
+  }
+  int status;
+  waitpid(child_pid, &status, 0);
+  return status;
+}
+
+// Returns the exit status of a process that raises a given signal.
+// If the signal does not cause the process to die, then it returns
+// instead the exit status of a process that exits normally with exit
+// code 1.  This is a helper function for the ExitStatusPredicateTest
+// test suite.
+static int KilledExitStatus(int signum) {
+  pid_t child_pid = fork();
+  if (child_pid == 0) {
+    raise(signum);
+    _exit(1);
+  }
+  int status;
+  waitpid(child_pid, &status, 0);
+  return status;
+}
+
+// Tests the ExitedWithCode predicate.
+TEST(ExitStatusPredicateTest, ExitedWithCode) {
+  const int status0  = NormalExitStatus(0);
+  const int status1  = NormalExitStatus(1);
+  const int status42 = NormalExitStatus(42);
+  const testing::ExitedWithCode pred0(0);
+  const testing::ExitedWithCode pred1(1);
+  const testing::ExitedWithCode pred42(42);
+  EXPECT_PRED1(pred0,  status0);
+  EXPECT_PRED1(pred1,  status1);
+  EXPECT_PRED1(pred42, status42);
+  EXPECT_FALSE(pred0(status1));
+  EXPECT_FALSE(pred42(status0));
+  EXPECT_FALSE(pred1(status42));
+}
+
+// Tests the KilledBySignal predicate.
+TEST(ExitStatusPredicateTest, KilledBySignal) {
+  const int status_segv = KilledExitStatus(SIGSEGV);
+  const int status_kill = KilledExitStatus(SIGKILL);
+  const testing::KilledBySignal pred_segv(SIGSEGV);
+  const testing::KilledBySignal pred_kill(SIGKILL);
+  EXPECT_PRED1(pred_segv, status_segv);
+  EXPECT_PRED1(pred_kill, status_kill);
+  EXPECT_FALSE(pred_segv(status_kill));
+  EXPECT_FALSE(pred_kill(status_segv));
+}
+
+# endif  // GTEST_OS_WINDOWS
+
+// Tests that the death test macros expand to code which may or may not
+// be followed by operator<<, and that in either case the complete text
+// comprises only a single C++ statement.
+TEST_F(TestForDeathTest, SingleStatement) {
+  if (AlwaysFalse())
+    // This would fail if executed; this is a compilation test only
+    ASSERT_DEATH(return, "");
+
+  if (AlwaysTrue())
+    EXPECT_DEATH(_exit(1), "");
+  else
+    // This empty "else" branch is meant to ensure that EXPECT_DEATH
+    // doesn't expand into an "if" statement without an "else"
+    ;
+
+  if (AlwaysFalse())
+    ASSERT_DEATH(return, "") << "did not die";
+
+  if (AlwaysFalse())
+    ;
+  else
+    EXPECT_DEATH(_exit(1), "") << 1 << 2 << 3;
+}
+
+void DieWithEmbeddedNul() {
+  fprintf(stderr, "Hello%cmy null world.\n", '\0');
+  fflush(stderr);
+  _exit(1);
+}
+
+# if GTEST_USES_PCRE
+// Tests that EXPECT_DEATH and ASSERT_DEATH work when the error
+// message has a NUL character in it.
+TEST_F(TestForDeathTest, EmbeddedNulInMessage) {
+  // TODO(wan@google.com): <regex.h> doesn't support matching strings
+  // with embedded NUL characters - find a way to workaround it.
+  EXPECT_DEATH(DieWithEmbeddedNul(), "my null world");
+  ASSERT_DEATH(DieWithEmbeddedNul(), "my null world");
+}
+# endif  // GTEST_USES_PCRE
+
+// Tests that death test macros expand to code which interacts well with switch
+// statements.
+TEST_F(TestForDeathTest, SwitchStatement) {
+  // Microsoft compiler usually complains about switch statements without
+  // case labels. We suppress that warning for this test.
+  GTEST_DISABLE_MSC_WARNINGS_PUSH_(4065)
+
+  switch (0)
+    default:
+      ASSERT_DEATH(_exit(1), "") << "exit in default switch handler";
+
+  switch (0)
+    case 0:
+      EXPECT_DEATH(_exit(1), "") << "exit in switch case";
+
+  GTEST_DISABLE_MSC_WARNINGS_POP_()
+}
+
+// Tests that a static member function can be used in a "fast" style
+// death test.
+TEST_F(TestForDeathTest, StaticMemberFunctionFastStyle) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  ASSERT_DEATH(StaticMemberFunction(), "death.*StaticMember");
+}
+
+// Tests that a method of the test fixture can be used in a "fast"
+// style death test.
+TEST_F(TestForDeathTest, MemberFunctionFastStyle) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  should_die_ = true;
+  EXPECT_DEATH(MemberFunction(), "inside.*MemberFunction");
+}
+
+void ChangeToRootDir() { posix::ChDir(GTEST_PATH_SEP_); }
+
+// Tests that death tests work even if the current directory has been
+// changed.
+TEST_F(TestForDeathTest, FastDeathTestInChangedDir) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+
+  ChangeToRootDir();
+  EXPECT_EXIT(_exit(1), testing::ExitedWithCode(1), "");
+
+  ChangeToRootDir();
+  ASSERT_DEATH(_exit(1), "");
+}
+
+# if GTEST_OS_LINUX
+void SigprofAction(int, siginfo_t*, void*) { /* no op */ }
+
+// Sets SIGPROF action and ITIMER_PROF timer (interval: 1ms).
+void SetSigprofActionAndTimer() {
+  struct itimerval timer;
+  timer.it_interval.tv_sec = 0;
+  timer.it_interval.tv_usec = 1;
+  timer.it_value = timer.it_interval;
+  ASSERT_EQ(0, setitimer(ITIMER_PROF, &timer, NULL));
+  struct sigaction signal_action;
+  memset(&signal_action, 0, sizeof(signal_action));
+  sigemptyset(&signal_action.sa_mask);
+  signal_action.sa_sigaction = SigprofAction;
+  signal_action.sa_flags = SA_RESTART | SA_SIGINFO;
+  ASSERT_EQ(0, sigaction(SIGPROF, &signal_action, NULL));
+}
+
+// Disables ITIMER_PROF timer and ignores SIGPROF signal.
+void DisableSigprofActionAndTimer(struct sigaction* old_signal_action) {
+  struct itimerval timer;
+  timer.it_interval.tv_sec = 0;
+  timer.it_interval.tv_usec = 0;
+  timer.it_value = timer.it_interval;
+  ASSERT_EQ(0, setitimer(ITIMER_PROF, &timer, NULL));
+  struct sigaction signal_action;
+  memset(&signal_action, 0, sizeof(signal_action));
+  sigemptyset(&signal_action.sa_mask);
+  signal_action.sa_handler = SIG_IGN;
+  ASSERT_EQ(0, sigaction(SIGPROF, &signal_action, old_signal_action));
+}
+
+// Tests that death tests work when SIGPROF handler and timer are set.
+TEST_F(TestForDeathTest, FastSigprofActionSet) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  SetSigprofActionAndTimer();
+  EXPECT_DEATH(_exit(1), "");
+  struct sigaction old_signal_action;
+  DisableSigprofActionAndTimer(&old_signal_action);
+  EXPECT_TRUE(old_signal_action.sa_sigaction == SigprofAction);
+}
+
+TEST_F(TestForDeathTest, ThreadSafeSigprofActionSet) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+  SetSigprofActionAndTimer();
+  EXPECT_DEATH(_exit(1), "");
+  struct sigaction old_signal_action;
+  DisableSigprofActionAndTimer(&old_signal_action);
+  EXPECT_TRUE(old_signal_action.sa_sigaction == SigprofAction);
+}
+# endif  // GTEST_OS_LINUX
+
+// Repeats a representative sample of death tests in the "threadsafe" style:
+
+TEST_F(TestForDeathTest, StaticMemberFunctionThreadsafeStyle) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+  ASSERT_DEATH(StaticMemberFunction(), "death.*StaticMember");
+}
+
+TEST_F(TestForDeathTest, MemberFunctionThreadsafeStyle) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+  should_die_ = true;
+  EXPECT_DEATH(MemberFunction(), "inside.*MemberFunction");
+}
+
+TEST_F(TestForDeathTest, ThreadsafeDeathTestInLoop) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+
+  for (int i = 0; i < 3; ++i)
+    EXPECT_EXIT(_exit(i), testing::ExitedWithCode(i), "") << ": i = " << i;
+}
+
+TEST_F(TestForDeathTest, ThreadsafeDeathTestInChangedDir) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+
+  ChangeToRootDir();
+  EXPECT_EXIT(_exit(1), testing::ExitedWithCode(1), "");
+
+  ChangeToRootDir();
+  ASSERT_DEATH(_exit(1), "");
+}
+
+TEST_F(TestForDeathTest, MixedStyles) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+  EXPECT_DEATH(_exit(1), "");
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_DEATH(_exit(1), "");
+}
+
+# if GTEST_HAS_CLONE && GTEST_HAS_PTHREAD
+
+namespace {
+
+bool pthread_flag;
+
+void SetPthreadFlag() {
+  pthread_flag = true;
+}
+
+}  // namespace
+
+TEST_F(TestForDeathTest, DoesNotExecuteAtforkHooks) {
+  if (!testing::GTEST_FLAG(death_test_use_fork)) {
+    testing::GTEST_FLAG(death_test_style) = "threadsafe";
+    pthread_flag = false;
+    ASSERT_EQ(0, pthread_atfork(&SetPthreadFlag, NULL, NULL));
+    ASSERT_DEATH(_exit(1), "");
+    ASSERT_FALSE(pthread_flag);
+  }
+}
+
+# endif  // GTEST_HAS_CLONE && GTEST_HAS_PTHREAD
+
+// Tests that a method of another class can be used in a death test.
+TEST_F(TestForDeathTest, MethodOfAnotherClass) {
+  const MayDie x(true);
+  ASSERT_DEATH(x.MemberFunction(), "MayDie\\:\\:MemberFunction");
+}
+
+// Tests that a global function can be used in a death test.
+TEST_F(TestForDeathTest, GlobalFunction) {
+  EXPECT_DEATH(GlobalFunction(), "GlobalFunction");
+}
+
+// Tests that any value convertible to an RE works as a second
+// argument to EXPECT_DEATH.
+TEST_F(TestForDeathTest, AcceptsAnythingConvertibleToRE) {
+  static const char regex_c_str[] = "GlobalFunction";
+  EXPECT_DEATH(GlobalFunction(), regex_c_str);
+
+  const testing::internal::RE regex(regex_c_str);
+  EXPECT_DEATH(GlobalFunction(), regex);
+
+# if GTEST_HAS_GLOBAL_STRING
+
+  const string regex_str(regex_c_str);
+  EXPECT_DEATH(GlobalFunction(), regex_str);
+
+# endif  // GTEST_HAS_GLOBAL_STRING
+
+# if !GTEST_USES_PCRE
+
+  const ::std::string regex_std_str(regex_c_str);
+  EXPECT_DEATH(GlobalFunction(), regex_std_str);
+
+# endif  // !GTEST_USES_PCRE
+}
+
+// Tests that a non-void function can be used in a death test.
+TEST_F(TestForDeathTest, NonVoidFunction) {
+  ASSERT_DEATH(NonVoidFunction(), "NonVoidFunction");
+}
+
+// Tests that functions that take parameter(s) can be used in a death test.
+TEST_F(TestForDeathTest, FunctionWithParameter) {
+  EXPECT_DEATH(DieIf(true), "DieIf\\(\\)");
+  EXPECT_DEATH(DieIfLessThan(2, 3), "DieIfLessThan");
+}
+
+// Tests that ASSERT_DEATH can be used outside a TEST, TEST_F, or test fixture.
+TEST_F(TestForDeathTest, OutsideFixture) {
+  DeathTestSubroutine();
+}
+
+// Tests that death tests can be done inside a loop.
+TEST_F(TestForDeathTest, InsideLoop) {
+  for (int i = 0; i < 5; i++) {
+    EXPECT_DEATH(DieIfLessThan(-1, i), "DieIfLessThan") << "where i == " << i;
+  }
+}
+
+// Tests that a compound statement can be used in a death test.
+TEST_F(TestForDeathTest, CompoundStatement) {
+  EXPECT_DEATH({  // NOLINT
+    const int x = 2;
+    const int y = x + 1;
+    DieIfLessThan(x, y);
+  },
+  "DieIfLessThan");
+}
+
+// Tests that code that doesn't die causes a death test to fail.
+TEST_F(TestForDeathTest, DoesNotDie) {
+  EXPECT_NONFATAL_FAILURE(EXPECT_DEATH(DieIf(false), "DieIf"),
+                          "failed to die");
+}
+
+// Tests that a death test fails when the error message isn't expected.
+TEST_F(TestForDeathTest, ErrorMessageMismatch) {
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_DEATH(DieIf(true), "DieIfLessThan") << "End of death test message.";
+  }, "died but not with expected error");
+}
+
+// On exit, *aborted will be true iff the EXPECT_DEATH() statement
+// aborted the function.
+void ExpectDeathTestHelper(bool* aborted) {
+  *aborted = true;
+  EXPECT_DEATH(DieIf(false), "DieIf");  // This assertion should fail.
+  *aborted = false;
+}
+
+// Tests that EXPECT_DEATH doesn't abort the test on failure.
+TEST_F(TestForDeathTest, EXPECT_DEATH) {
+  bool aborted = true;
+  EXPECT_NONFATAL_FAILURE(ExpectDeathTestHelper(&aborted),
+                          "failed to die");
+  EXPECT_FALSE(aborted);
+}
+
+// Tests that ASSERT_DEATH does abort the test on failure.
+TEST_F(TestForDeathTest, ASSERT_DEATH) {
+  static bool aborted;
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    aborted = true;
+    ASSERT_DEATH(DieIf(false), "DieIf");  // This assertion should fail.
+    aborted = false;
+  }, "failed to die");
+  EXPECT_TRUE(aborted);
+}
+
+// Tests that EXPECT_DEATH evaluates the arguments exactly once.
+TEST_F(TestForDeathTest, SingleEvaluation) {
+  int x = 3;
+  EXPECT_DEATH(DieIf((++x) == 4), "DieIf");
+
+  const char* regex = "DieIf";
+  const char* regex_save = regex;
+  EXPECT_DEATH(DieIfLessThan(3, 4), regex++);
+  EXPECT_EQ(regex_save + 1, regex);
+}
+
+// Tests that run-away death tests are reported as failures.
+TEST_F(TestForDeathTest, RunawayIsFailure) {
+  EXPECT_NONFATAL_FAILURE(EXPECT_DEATH(static_cast<void>(0), "Foo"),
+                          "failed to die.");
+}
+
+// Tests that death tests report executing 'return' in the statement as
+// failure.
+TEST_F(TestForDeathTest, ReturnIsFailure) {
+  EXPECT_FATAL_FAILURE(ASSERT_DEATH(return, "Bar"),
+                       "illegal return in test statement.");
+}
+
+// Tests that EXPECT_DEBUG_DEATH works as expected, that is, you can stream a
+// message to it, and in debug mode it:
+// 1. Asserts on death.
+// 2. Has no side effect.
+//
+// And in opt mode, it:
+// 1.  Has side effects but does not assert.
+TEST_F(TestForDeathTest, TestExpectDebugDeath) {
+  int sideeffect = 0;
+
+  EXPECT_DEBUG_DEATH(DieInDebugElse12(&sideeffect), "death.*DieInDebugElse12")
+      << "Must accept a streamed message";
+
+# ifdef NDEBUG
+
+  // Checks that the assignment occurs in opt mode (sideeffect).
+  EXPECT_EQ(12, sideeffect);
+
+# else
+
+  // Checks that the assignment does not occur in dbg mode (no sideeffect).
+  EXPECT_EQ(0, sideeffect);
+
+# endif
+}
+
+// Tests that ASSERT_DEBUG_DEATH works as expected, that is, you can stream a
+// message to it, and in debug mode it:
+// 1. Asserts on death.
+// 2. Has no side effect.
+//
+// And in opt mode, it:
+// 1.  Has side effects but does not assert.
+TEST_F(TestForDeathTest, TestAssertDebugDeath) {
+  int sideeffect = 0;
+
+  ASSERT_DEBUG_DEATH(DieInDebugElse12(&sideeffect), "death.*DieInDebugElse12")
+      << "Must accept a streamed message";
+
+# ifdef NDEBUG
+
+  // Checks that the assignment occurs in opt mode (sideeffect).
+  EXPECT_EQ(12, sideeffect);
+
+# else
+
+  // Checks that the assignment does not occur in dbg mode (no sideeffect).
+  EXPECT_EQ(0, sideeffect);
+
+# endif
+}
+
+# ifndef NDEBUG
+
+void ExpectDebugDeathHelper(bool* aborted) {
+  *aborted = true;
+  EXPECT_DEBUG_DEATH(return, "") << "This is expected to fail.";
+  *aborted = false;
+}
+
+#  if GTEST_OS_WINDOWS
+TEST(PopUpDeathTest, DoesNotShowPopUpOnAbort) {
+  printf("This test should be considered failing if it shows "
+         "any pop-up dialogs.\n");
+  fflush(stdout);
+
+  EXPECT_DEATH({
+    testing::GTEST_FLAG(catch_exceptions) = false;
+    abort();
+  }, "");
+}
+#  endif  // GTEST_OS_WINDOWS
+
+// Tests that EXPECT_DEBUG_DEATH in debug mode does not abort
+// the function.
+TEST_F(TestForDeathTest, ExpectDebugDeathDoesNotAbort) {
+  bool aborted = true;
+  EXPECT_NONFATAL_FAILURE(ExpectDebugDeathHelper(&aborted), "");
+  EXPECT_FALSE(aborted);
+}
+
+void AssertDebugDeathHelper(bool* aborted) {
+  *aborted = true;
+  GTEST_LOG_(INFO) << "Before ASSERT_DEBUG_DEATH";
+  ASSERT_DEBUG_DEATH(GTEST_LOG_(INFO) << "In ASSERT_DEBUG_DEATH"; return, "")
+      << "This is expected to fail.";
+  GTEST_LOG_(INFO) << "After ASSERT_DEBUG_DEATH";
+  *aborted = false;
+}
+
+// Tests that ASSERT_DEBUG_DEATH in debug mode aborts the function on
+// failure.
+TEST_F(TestForDeathTest, AssertDebugDeathAborts) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts2) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts3) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts4) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts5) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts6) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts7) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts8) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts9) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+TEST_F(TestForDeathTest, AssertDebugDeathAborts10) {
+  static bool aborted;
+  aborted = false;
+  EXPECT_FATAL_FAILURE(AssertDebugDeathHelper(&aborted), "");
+  EXPECT_TRUE(aborted);
+}
+
+# endif  // _NDEBUG
+
+// Tests the *_EXIT family of macros, using a variety of predicates.
+static void TestExitMacros() {
+  EXPECT_EXIT(_exit(1),  testing::ExitedWithCode(1),  "");
+  ASSERT_EXIT(_exit(42), testing::ExitedWithCode(42), "");
+
+# if GTEST_OS_WINDOWS
+
+  // Of all signals effects on the process exit code, only those of SIGABRT
+  // are documented on Windows.
+  // See http://msdn.microsoft.com/en-us/library/dwwzkt4c(VS.71).aspx.
+  EXPECT_EXIT(raise(SIGABRT), testing::ExitedWithCode(3), "") << "b_ar";
+
+# else
+
+  EXPECT_EXIT(raise(SIGKILL), testing::KilledBySignal(SIGKILL), "") << "foo";
+  ASSERT_EXIT(raise(SIGUSR2), testing::KilledBySignal(SIGUSR2), "") << "bar";
+
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_EXIT(_exit(0), testing::KilledBySignal(SIGSEGV), "")
+      << "This failure is expected, too.";
+  }, "This failure is expected, too.");
+
+# endif  // GTEST_OS_WINDOWS
+
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_EXIT(raise(SIGSEGV), testing::ExitedWithCode(0), "")
+      << "This failure is expected.";
+  }, "This failure is expected.");
+}
+
+TEST_F(TestForDeathTest, ExitMacros) {
+  TestExitMacros();
+}
+
+TEST_F(TestForDeathTest, ExitMacrosUsingFork) {
+  testing::GTEST_FLAG(death_test_use_fork) = true;
+  TestExitMacros();
+}
+
+TEST_F(TestForDeathTest, InvalidStyle) {
+  testing::GTEST_FLAG(death_test_style) = "rococo";
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_DEATH(_exit(0), "") << "This failure is expected.";
+  }, "This failure is expected.");
+}
+
+TEST_F(TestForDeathTest, DeathTestFailedOutput) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_DEATH(DieWithMessage("death\n"),
+                   "expected message"),
+      "Actual msg:\n"
+      "[  DEATH   ] death\n");
+}
+
+TEST_F(TestForDeathTest, DeathTestUnexpectedReturnOutput) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_DEATH({
+          fprintf(stderr, "returning\n");
+          fflush(stderr);
+          return;
+        }, ""),
+      "    Result: illegal return in test statement.\n"
+      " Error msg:\n"
+      "[  DEATH   ] returning\n");
+}
+
+TEST_F(TestForDeathTest, DeathTestBadExitCodeOutput) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_EXIT(DieWithMessage("exiting with rc 1\n"),
+                  testing::ExitedWithCode(3),
+                  "expected message"),
+      "    Result: died but not with expected exit code:\n"
+      "            Exited with exit status 1\n"
+      "Actual msg:\n"
+      "[  DEATH   ] exiting with rc 1\n");
+}
+
+TEST_F(TestForDeathTest, DeathTestMultiLineMatchFail) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_NONFATAL_FAILURE(
+      EXPECT_DEATH(DieWithMessage("line 1\nline 2\nline 3\n"),
+                   "line 1\nxyz\nline 3\n"),
+      "Actual msg:\n"
+      "[  DEATH   ] line 1\n"
+      "[  DEATH   ] line 2\n"
+      "[  DEATH   ] line 3\n");
+}
+
+TEST_F(TestForDeathTest, DeathTestMultiLineMatchPass) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_DEATH(DieWithMessage("line 1\nline 2\nline 3\n"),
+               "line 1\nline 2\nline 3\n");
+}
+
+// A DeathTestFactory that returns MockDeathTests.
+class MockDeathTestFactory : public DeathTestFactory {
+ public:
+  MockDeathTestFactory();
+  virtual bool Create(const char* statement,
+                      const ::testing::internal::RE* regex,
+                      const char* file, int line, DeathTest** test);
+
+  // Sets the parameters for subsequent calls to Create.
+  void SetParameters(bool create, DeathTest::TestRole role,
+                     int status, bool passed);
+
+  // Accessors.
+  int AssumeRoleCalls() const { return assume_role_calls_; }
+  int WaitCalls() const { return wait_calls_; }
+  size_t PassedCalls() const { return passed_args_.size(); }
+  bool PassedArgument(int n) const { return passed_args_[n]; }
+  size_t AbortCalls() const { return abort_args_.size(); }
+  DeathTest::AbortReason AbortArgument(int n) const {
+    return abort_args_[n];
+  }
+  bool TestDeleted() const { return test_deleted_; }
+
+ private:
+  friend class MockDeathTest;
+  // If true, Create will return a MockDeathTest; otherwise it returns
+  // NULL.
+  bool create_;
+  // The value a MockDeathTest will return from its AssumeRole method.
+  DeathTest::TestRole role_;
+  // The value a MockDeathTest will return from its Wait method.
+  int status_;
+  // The value a MockDeathTest will return from its Passed method.
+  bool passed_;
+
+  // Number of times AssumeRole was called.
+  int assume_role_calls_;
+  // Number of times Wait was called.
+  int wait_calls_;
+  // The arguments to the calls to Passed since the last call to
+  // SetParameters.
+  std::vector<bool> passed_args_;
+  // The arguments to the calls to Abort since the last call to
+  // SetParameters.
+  std::vector<DeathTest::AbortReason> abort_args_;
+  // True if the last MockDeathTest returned by Create has been
+  // deleted.
+  bool test_deleted_;
+};
+
+
+// A DeathTest implementation useful in testing.  It returns values set
+// at its creation from its various inherited DeathTest methods, and
+// reports calls to those methods to its parent MockDeathTestFactory
+// object.
+class MockDeathTest : public DeathTest {
+ public:
+  MockDeathTest(MockDeathTestFactory *parent,
+                TestRole role, int status, bool passed) :
+      parent_(parent), role_(role), status_(status), passed_(passed) {
+  }
+  virtual ~MockDeathTest() {
+    parent_->test_deleted_ = true;
+  }
+  virtual TestRole AssumeRole() {
+    ++parent_->assume_role_calls_;
+    return role_;
+  }
+  virtual int Wait() {
+    ++parent_->wait_calls_;
+    return status_;
+  }
+  virtual bool Passed(bool exit_status_ok) {
+    parent_->passed_args_.push_back(exit_status_ok);
+    return passed_;
+  }
+  virtual void Abort(AbortReason reason) {
+    parent_->abort_args_.push_back(reason);
+  }
+
+ private:
+  MockDeathTestFactory* const parent_;
+  const TestRole role_;
+  const int status_;
+  const bool passed_;
+};
+
+
+// MockDeathTestFactory constructor.
+MockDeathTestFactory::MockDeathTestFactory()
+    : create_(true),
+      role_(DeathTest::OVERSEE_TEST),
+      status_(0),
+      passed_(true),
+      assume_role_calls_(0),
+      wait_calls_(0),
+      passed_args_(),
+      abort_args_() {
+}
+
+
+// Sets the parameters for subsequent calls to Create.
+void MockDeathTestFactory::SetParameters(bool create,
+                                         DeathTest::TestRole role,
+                                         int status, bool passed) {
+  create_ = create;
+  role_ = role;
+  status_ = status;
+  passed_ = passed;
+
+  assume_role_calls_ = 0;
+  wait_calls_ = 0;
+  passed_args_.clear();
+  abort_args_.clear();
+}
+
+
+// Sets test to NULL (if create_ is false) or to the address of a new
+// MockDeathTest object with parameters taken from the last call
+// to SetParameters (if create_ is true).  Always returns true.
+bool MockDeathTestFactory::Create(const char* /*statement*/,
+                                  const ::testing::internal::RE* /*regex*/,
+                                  const char* /*file*/,
+                                  int /*line*/,
+                                  DeathTest** test) {
+  test_deleted_ = false;
+  if (create_) {
+    *test = new MockDeathTest(this, role_, status_, passed_);
+  } else {
+    *test = NULL;
+  }
+  return true;
+}
+
+// A test fixture for testing the logic of the GTEST_DEATH_TEST_ macro.
+// It installs a MockDeathTestFactory that is used for the duration
+// of the test case.
+class MacroLogicDeathTest : public testing::Test {
+ protected:
+  static testing::internal::ReplaceDeathTestFactory* replacer_;
+  static MockDeathTestFactory* factory_;
+
+  static void SetUpTestCase() {
+    factory_ = new MockDeathTestFactory;
+    replacer_ = new testing::internal::ReplaceDeathTestFactory(factory_);
+  }
+
+  static void TearDownTestCase() {
+    delete replacer_;
+    replacer_ = NULL;
+    delete factory_;
+    factory_ = NULL;
+  }
+
+  // Runs a death test that breaks the rules by returning.  Such a death
+  // test cannot be run directly from a test routine that uses a
+  // MockDeathTest, or the remainder of the routine will not be executed.
+  static void RunReturningDeathTest(bool* flag) {
+    ASSERT_DEATH({  // NOLINT
+      *flag = true;
+      return;
+    }, "");
+  }
+};
+
+testing::internal::ReplaceDeathTestFactory* MacroLogicDeathTest::replacer_
+    = NULL;
+MockDeathTestFactory* MacroLogicDeathTest::factory_ = NULL;
+
+
+// Test that nothing happens when the factory doesn't return a DeathTest:
+TEST_F(MacroLogicDeathTest, NothingHappens) {
+  bool flag = false;
+  factory_->SetParameters(false, DeathTest::OVERSEE_TEST, 0, true);
+  EXPECT_DEATH(flag = true, "");
+  EXPECT_FALSE(flag);
+  EXPECT_EQ(0, factory_->AssumeRoleCalls());
+  EXPECT_EQ(0, factory_->WaitCalls());
+  EXPECT_EQ(0U, factory_->PassedCalls());
+  EXPECT_EQ(0U, factory_->AbortCalls());
+  EXPECT_FALSE(factory_->TestDeleted());
+}
+
+// Test that the parent process doesn't run the death test code,
+// and that the Passed method returns false when the (simulated)
+// child process exits with status 0:
+TEST_F(MacroLogicDeathTest, ChildExitsSuccessfully) {
+  bool flag = false;
+  factory_->SetParameters(true, DeathTest::OVERSEE_TEST, 0, true);
+  EXPECT_DEATH(flag = true, "");
+  EXPECT_FALSE(flag);
+  EXPECT_EQ(1, factory_->AssumeRoleCalls());
+  EXPECT_EQ(1, factory_->WaitCalls());
+  ASSERT_EQ(1U, factory_->PassedCalls());
+  EXPECT_FALSE(factory_->PassedArgument(0));
+  EXPECT_EQ(0U, factory_->AbortCalls());
+  EXPECT_TRUE(factory_->TestDeleted());
+}
+
+// Tests that the Passed method was given the argument "true" when
+// the (simulated) child process exits with status 1:
+TEST_F(MacroLogicDeathTest, ChildExitsUnsuccessfully) {
+  bool flag = false;
+  factory_->SetParameters(true, DeathTest::OVERSEE_TEST, 1, true);
+  EXPECT_DEATH(flag = true, "");
+  EXPECT_FALSE(flag);
+  EXPECT_EQ(1, factory_->AssumeRoleCalls());
+  EXPECT_EQ(1, factory_->WaitCalls());
+  ASSERT_EQ(1U, factory_->PassedCalls());
+  EXPECT_TRUE(factory_->PassedArgument(0));
+  EXPECT_EQ(0U, factory_->AbortCalls());
+  EXPECT_TRUE(factory_->TestDeleted());
+}
+
+// Tests that the (simulated) child process executes the death test
+// code, and is aborted with the correct AbortReason if it
+// executes a return statement.
+TEST_F(MacroLogicDeathTest, ChildPerformsReturn) {
+  bool flag = false;
+  factory_->SetParameters(true, DeathTest::EXECUTE_TEST, 0, true);
+  RunReturningDeathTest(&flag);
+  EXPECT_TRUE(flag);
+  EXPECT_EQ(1, factory_->AssumeRoleCalls());
+  EXPECT_EQ(0, factory_->WaitCalls());
+  EXPECT_EQ(0U, factory_->PassedCalls());
+  EXPECT_EQ(1U, factory_->AbortCalls());
+  EXPECT_EQ(DeathTest::TEST_ENCOUNTERED_RETURN_STATEMENT,
+            factory_->AbortArgument(0));
+  EXPECT_TRUE(factory_->TestDeleted());
+}
+
+// Tests that the (simulated) child process is aborted with the
+// correct AbortReason if it does not die.
+TEST_F(MacroLogicDeathTest, ChildDoesNotDie) {
+  bool flag = false;
+  factory_->SetParameters(true, DeathTest::EXECUTE_TEST, 0, true);
+  EXPECT_DEATH(flag = true, "");
+  EXPECT_TRUE(flag);
+  EXPECT_EQ(1, factory_->AssumeRoleCalls());
+  EXPECT_EQ(0, factory_->WaitCalls());
+  EXPECT_EQ(0U, factory_->PassedCalls());
+  // This time there are two calls to Abort: one since the test didn't
+  // die, and another from the ReturnSentinel when it's destroyed.  The
+  // sentinel normally isn't destroyed if a test doesn't die, since
+  // _exit(2) is called in that case by ForkingDeathTest, but not by
+  // our MockDeathTest.
+  ASSERT_EQ(2U, factory_->AbortCalls());
+  EXPECT_EQ(DeathTest::TEST_DID_NOT_DIE,
+            factory_->AbortArgument(0));
+  EXPECT_EQ(DeathTest::TEST_ENCOUNTERED_RETURN_STATEMENT,
+            factory_->AbortArgument(1));
+  EXPECT_TRUE(factory_->TestDeleted());
+}
+
+// Tests that a successful death test does not register a successful
+// test part.
+TEST(SuccessRegistrationDeathTest, NoSuccessPart) {
+  EXPECT_DEATH(_exit(1), "");
+  EXPECT_EQ(0, GetUnitTestImpl()->current_test_result()->total_part_count());
+}
+
+TEST(StreamingAssertionsDeathTest, DeathTest) {
+  EXPECT_DEATH(_exit(1), "") << "unexpected failure";
+  ASSERT_DEATH(_exit(1), "") << "unexpected failure";
+  EXPECT_NONFATAL_FAILURE({  // NOLINT
+    EXPECT_DEATH(_exit(0), "") << "expected failure";
+  }, "expected failure");
+  EXPECT_FATAL_FAILURE({  // NOLINT
+    ASSERT_DEATH(_exit(0), "") << "expected failure";
+  }, "expected failure");
+}
+
+// Tests that GetLastErrnoDescription returns an empty string when the
+// last error is 0 and non-empty string when it is non-zero.
+TEST(GetLastErrnoDescription, GetLastErrnoDescriptionWorks) {
+  errno = ENOENT;
+  EXPECT_STRNE("", GetLastErrnoDescription().c_str());
+  errno = 0;
+  EXPECT_STREQ("", GetLastErrnoDescription().c_str());
+}
+
+# if GTEST_OS_WINDOWS
+TEST(AutoHandleTest, AutoHandleWorks) {
+  HANDLE handle = ::CreateEvent(NULL, FALSE, FALSE, NULL);
+  ASSERT_NE(INVALID_HANDLE_VALUE, handle);
+
+  // Tests that the AutoHandle is correctly initialized with a handle.
+  testing::internal::AutoHandle auto_handle(handle);
+  EXPECT_EQ(handle, auto_handle.Get());
+
+  // Tests that Reset assigns INVALID_HANDLE_VALUE.
+  // Note that this cannot verify whether the original handle is closed.
+  auto_handle.Reset();
+  EXPECT_EQ(INVALID_HANDLE_VALUE, auto_handle.Get());
+
+  // Tests that Reset assigns the new handle.
+  // Note that this cannot verify whether the original handle is closed.
+  handle = ::CreateEvent(NULL, FALSE, FALSE, NULL);
+  ASSERT_NE(INVALID_HANDLE_VALUE, handle);
+  auto_handle.Reset(handle);
+  EXPECT_EQ(handle, auto_handle.Get());
+
+  // Tests that AutoHandle contains INVALID_HANDLE_VALUE by default.
+  testing::internal::AutoHandle auto_handle2;
+  EXPECT_EQ(INVALID_HANDLE_VALUE, auto_handle2.Get());
+}
+# endif  // GTEST_OS_WINDOWS
+
+# if GTEST_OS_WINDOWS
+typedef unsigned __int64 BiggestParsable;
+typedef signed __int64 BiggestSignedParsable;
+# else
+typedef unsigned long long BiggestParsable;
+typedef signed long long BiggestSignedParsable;
+# endif  // GTEST_OS_WINDOWS
+
+// We cannot use std::numeric_limits<T>::max() as it clashes with the
+// max() macro defined by <windows.h>.
+const BiggestParsable kBiggestParsableMax = ULLONG_MAX;
+const BiggestSignedParsable kBiggestSignedParsableMax = LLONG_MAX;
+
+TEST(ParseNaturalNumberTest, RejectsInvalidFormat) {
+  BiggestParsable result = 0;
+
+  // Rejects non-numbers.
+  EXPECT_FALSE(ParseNaturalNumber("non-number string", &result));
+
+  // Rejects numbers with whitespace prefix.
+  EXPECT_FALSE(ParseNaturalNumber(" 123", &result));
+
+  // Rejects negative numbers.
+  EXPECT_FALSE(ParseNaturalNumber("-123", &result));
+
+  // Rejects numbers starting with a plus sign.
+  EXPECT_FALSE(ParseNaturalNumber("+123", &result));
+  errno = 0;
+}
+
+TEST(ParseNaturalNumberTest, RejectsOverflownNumbers) {
+  BiggestParsable result = 0;
+
+  EXPECT_FALSE(ParseNaturalNumber("99999999999999999999999", &result));
+
+  signed char char_result = 0;
+  EXPECT_FALSE(ParseNaturalNumber("200", &char_result));
+  errno = 0;
+}
+
+TEST(ParseNaturalNumberTest, AcceptsValidNumbers) {
+  BiggestParsable result = 0;
+
+  result = 0;
+  ASSERT_TRUE(ParseNaturalNumber("123", &result));
+  EXPECT_EQ(123U, result);
+
+  // Check 0 as an edge case.
+  result = 1;
+  ASSERT_TRUE(ParseNaturalNumber("0", &result));
+  EXPECT_EQ(0U, result);
+
+  result = 1;
+  ASSERT_TRUE(ParseNaturalNumber("00000", &result));
+  EXPECT_EQ(0U, result);
+}
+
+TEST(ParseNaturalNumberTest, AcceptsTypeLimits) {
+  Message msg;
+  msg << kBiggestParsableMax;
+
+  BiggestParsable result = 0;
+  EXPECT_TRUE(ParseNaturalNumber(msg.GetString(), &result));
+  EXPECT_EQ(kBiggestParsableMax, result);
+
+  Message msg2;
+  msg2 << kBiggestSignedParsableMax;
+
+  BiggestSignedParsable signed_result = 0;
+  EXPECT_TRUE(ParseNaturalNumber(msg2.GetString(), &signed_result));
+  EXPECT_EQ(kBiggestSignedParsableMax, signed_result);
+
+  Message msg3;
+  msg3 << INT_MAX;
+
+  int int_result = 0;
+  EXPECT_TRUE(ParseNaturalNumber(msg3.GetString(), &int_result));
+  EXPECT_EQ(INT_MAX, int_result);
+
+  Message msg4;
+  msg4 << UINT_MAX;
+
+  unsigned int uint_result = 0;
+  EXPECT_TRUE(ParseNaturalNumber(msg4.GetString(), &uint_result));
+  EXPECT_EQ(UINT_MAX, uint_result);
+}
+
+TEST(ParseNaturalNumberTest, WorksForShorterIntegers) {
+  short short_result = 0;
+  ASSERT_TRUE(ParseNaturalNumber("123", &short_result));
+  EXPECT_EQ(123, short_result);
+
+  signed char char_result = 0;
+  ASSERT_TRUE(ParseNaturalNumber("123", &char_result));
+  EXPECT_EQ(123, char_result);
+}
+
+# if GTEST_OS_WINDOWS
+TEST(EnvironmentTest, HandleFitsIntoSizeT) {
+  // TODO(vladl@google.com): Remove this test after this condition is verified
+  // in a static assertion in gtest-death-test.cc in the function
+  // GetStatusFileDescriptor.
+  ASSERT_TRUE(sizeof(HANDLE) <= sizeof(size_t));
+}
+# endif  // GTEST_OS_WINDOWS
+
+// Tests that EXPECT_DEATH_IF_SUPPORTED/ASSERT_DEATH_IF_SUPPORTED trigger
+// failures when death tests are available on the system.
+TEST(ConditionalDeathMacrosDeathTest, ExpectsDeathWhenDeathTestsAvailable) {
+  EXPECT_DEATH_IF_SUPPORTED(DieInside("CondDeathTestExpectMacro"),
+                            "death inside CondDeathTestExpectMacro");
+  ASSERT_DEATH_IF_SUPPORTED(DieInside("CondDeathTestAssertMacro"),
+                            "death inside CondDeathTestAssertMacro");
+
+  // Empty statement will not crash, which must trigger a failure.
+  EXPECT_NONFATAL_FAILURE(EXPECT_DEATH_IF_SUPPORTED(;, ""), "");
+  EXPECT_FATAL_FAILURE(ASSERT_DEATH_IF_SUPPORTED(;, ""), "");
+}
+
+TEST(InDeathTestChildDeathTest, ReportsDeathTestCorrectlyInFastStyle) {
+  testing::GTEST_FLAG(death_test_style) = "fast";
+  EXPECT_FALSE(InDeathTestChild());
+  EXPECT_DEATH({
+    fprintf(stderr, InDeathTestChild() ? "Inside" : "Outside");
+    fflush(stderr);
+    _exit(1);
+  }, "Inside");
+}
+
+TEST(InDeathTestChildDeathTest, ReportsDeathTestCorrectlyInThreadSafeStyle) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+  EXPECT_FALSE(InDeathTestChild());
+  EXPECT_DEATH({
+    fprintf(stderr, InDeathTestChild() ? "Inside" : "Outside");
+    fflush(stderr);
+    _exit(1);
+  }, "Inside");
+}
+
+#else  // !GTEST_HAS_DEATH_TEST follows
+
+using testing::internal::CaptureStderr;
+using testing::internal::GetCapturedStderr;
+
+// Tests that EXPECT_DEATH_IF_SUPPORTED/ASSERT_DEATH_IF_SUPPORTED are still
+// defined but do not trigger failures when death tests are not available on
+// the system.
+TEST(ConditionalDeathMacrosTest, WarnsWhenDeathTestsNotAvailable) {
+  // Empty statement will not crash, but that should not trigger a failure
+  // when death tests are not supported.
+  CaptureStderr();
+  EXPECT_DEATH_IF_SUPPORTED(;, "");
+  std::string output = GetCapturedStderr();
+  ASSERT_TRUE(NULL != strstr(output.c_str(),
+                             "Death tests are not supported on this platform"));
+  ASSERT_TRUE(NULL != strstr(output.c_str(), ";"));
+
+  // The streamed message should not be printed as there is no test failure.
+  CaptureStderr();
+  EXPECT_DEATH_IF_SUPPORTED(;, "") << "streamed message";
+  output = GetCapturedStderr();
+  ASSERT_TRUE(NULL == strstr(output.c_str(), "streamed message"));
+
+  CaptureStderr();
+  ASSERT_DEATH_IF_SUPPORTED(;, "");  // NOLINT
+  output = GetCapturedStderr();
+  ASSERT_TRUE(NULL != strstr(output.c_str(),
+                             "Death tests are not supported on this platform"));
+  ASSERT_TRUE(NULL != strstr(output.c_str(), ";"));
+
+  CaptureStderr();
+  ASSERT_DEATH_IF_SUPPORTED(;, "") << "streamed message";  // NOLINT
+  output = GetCapturedStderr();
+  ASSERT_TRUE(NULL == strstr(output.c_str(), "streamed message"));
+}
+
+void FuncWithAssert(int* n) {
+  ASSERT_DEATH_IF_SUPPORTED(return;, "");
+  (*n)++;
+}
+
+// Tests that ASSERT_DEATH_IF_SUPPORTED does not return from the current
+// function (as ASSERT_DEATH does) if death tests are not supported.
+TEST(ConditionalDeathMacrosTest, AssertDeatDoesNotReturnhIfUnsupported) {
+  int n = 0;
+  FuncWithAssert(&n);
+  EXPECT_EQ(1, n);
+}
+
+#endif  // !GTEST_HAS_DEATH_TEST
+
+// Tests that the death test macros expand to code which may or may not
+// be followed by operator<<, and that in either case the complete text
+// comprises only a single C++ statement.
+//
+// The syntax should work whether death tests are available or not.
+TEST(ConditionalDeathMacrosSyntaxDeathTest, SingleStatement) {
+  if (AlwaysFalse())
+    // This would fail if executed; this is a compilation test only
+    ASSERT_DEATH_IF_SUPPORTED(return, "");
+
+  if (AlwaysTrue())
+    EXPECT_DEATH_IF_SUPPORTED(_exit(1), "");
+  else
+    // This empty "else" branch is meant to ensure that EXPECT_DEATH
+    // doesn't expand into an "if" statement without an "else"
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    ASSERT_DEATH_IF_SUPPORTED(return, "") << "did not die";
+
+  if (AlwaysFalse())
+    ;  // NOLINT
+  else
+    EXPECT_DEATH_IF_SUPPORTED(_exit(1), "") << 1 << 2 << 3;
+}
+
+// Tests that conditional death test macros expand to code which interacts
+// well with switch statements.
+TEST(ConditionalDeathMacrosSyntaxDeathTest, SwitchStatement) {
+  // Microsoft compiler usually complains about switch statements without
+  // case labels. We suppress that warning for this test.
+  GTEST_DISABLE_MSC_WARNINGS_PUSH_(4065)
+
+  switch (0)
+    default:
+      ASSERT_DEATH_IF_SUPPORTED(_exit(1), "")
+          << "exit in default switch handler";
+
+  switch (0)
+    case 0:
+      EXPECT_DEATH_IF_SUPPORTED(_exit(1), "") << "exit in switch case";
+
+  GTEST_DISABLE_MSC_WARNINGS_POP_()
+}
+
+// Tests that a test case whose name ends with "DeathTest" works fine
+// on Windows.
+TEST(NotADeathTest, Test) {
+  SUCCEED();
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: keith.ray@gmail.com (Keith Ray)
+//
+// Google Test filepath utilities
+//
+// This file tests classes and functions used internally by
+// Google Test.  They are subject to change without notice.
+//
+// This file is #included from gtest_unittest.cc, to avoid changing
+// build or make-files for some existing Google Test clients. Do not
+// #include this file anywhere else!
+
+#include "gtest/internal/gtest-filepath.h"
+#include "gtest/gtest.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+#if GTEST_OS_WINDOWS_MOBILE
+# include <windows.h>  // NOLINT
+#elif GTEST_OS_WINDOWS
+# include <direct.h>  // NOLINT
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+namespace testing {
+namespace internal {
+namespace {
+
+#if GTEST_OS_WINDOWS_MOBILE
+// TODO(wan@google.com): Move these to the POSIX adapter section in
+// gtest-port.h.
+
+// Windows CE doesn't have the remove C function.
+int remove(const char* path) {
+  LPCWSTR wpath = String::AnsiToUtf16(path);
+  int ret = DeleteFile(wpath) ? 0 : -1;
+  delete [] wpath;
+  return ret;
+}
+// Windows CE doesn't have the _rmdir C function.
+int _rmdir(const char* path) {
+  FilePath filepath(path);
+  LPCWSTR wpath = String::AnsiToUtf16(
+      filepath.RemoveTrailingPathSeparator().c_str());
+  int ret = RemoveDirectory(wpath) ? 0 : -1;
+  delete [] wpath;
+  return ret;
+}
+
+#else
+
+TEST(GetCurrentDirTest, ReturnsCurrentDir) {
+  const FilePath original_dir = FilePath::GetCurrentDir();
+  EXPECT_FALSE(original_dir.IsEmpty());
+
+  posix::ChDir(GTEST_PATH_SEP_);
+  const FilePath cwd = FilePath::GetCurrentDir();
+  posix::ChDir(original_dir.c_str());
+
+# if GTEST_OS_WINDOWS
+
+  // Skips the ":".
+  const char* const cwd_without_drive = strchr(cwd.c_str(), ':');
+  ASSERT_TRUE(cwd_without_drive != NULL);
+  EXPECT_STREQ(GTEST_PATH_SEP_, cwd_without_drive + 1);
+
+# else
+
+  EXPECT_EQ(GTEST_PATH_SEP_, cwd.string());
+
+# endif
+}
+
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+TEST(IsEmptyTest, ReturnsTrueForEmptyPath) {
+  EXPECT_TRUE(FilePath("").IsEmpty());
+}
+
+TEST(IsEmptyTest, ReturnsFalseForNonEmptyPath) {
+  EXPECT_FALSE(FilePath("a").IsEmpty());
+  EXPECT_FALSE(FilePath(".").IsEmpty());
+  EXPECT_FALSE(FilePath("a/b").IsEmpty());
+  EXPECT_FALSE(FilePath("a\\b\\").IsEmpty());
+}
+
+// RemoveDirectoryName "" -> ""
+TEST(RemoveDirectoryNameTest, WhenEmptyName) {
+  EXPECT_EQ("", FilePath("").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName "afile" -> "afile"
+TEST(RemoveDirectoryNameTest, ButNoDirectory) {
+  EXPECT_EQ("afile",
+      FilePath("afile").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName "/afile" -> "afile"
+TEST(RemoveDirectoryNameTest, RootFileShouldGiveFileName) {
+  EXPECT_EQ("afile",
+      FilePath(GTEST_PATH_SEP_ "afile").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName "adir/" -> ""
+TEST(RemoveDirectoryNameTest, WhereThereIsNoFileName) {
+  EXPECT_EQ("",
+      FilePath("adir" GTEST_PATH_SEP_).RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName "adir/afile" -> "afile"
+TEST(RemoveDirectoryNameTest, ShouldGiveFileName) {
+  EXPECT_EQ("afile",
+      FilePath("adir" GTEST_PATH_SEP_ "afile").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName "adir/subdir/afile" -> "afile"
+TEST(RemoveDirectoryNameTest, ShouldAlsoGiveFileName) {
+  EXPECT_EQ("afile",
+      FilePath("adir" GTEST_PATH_SEP_ "subdir" GTEST_PATH_SEP_ "afile")
+      .RemoveDirectoryName().string());
+}
+
+#if GTEST_HAS_ALT_PATH_SEP_
+
+// Tests that RemoveDirectoryName() works with the alternate separator
+// on Windows.
+
+// RemoveDirectoryName("/afile") -> "afile"
+TEST(RemoveDirectoryNameTest, RootFileShouldGiveFileNameForAlternateSeparator) {
+  EXPECT_EQ("afile", FilePath("/afile").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName("adir/") -> ""
+TEST(RemoveDirectoryNameTest, WhereThereIsNoFileNameForAlternateSeparator) {
+  EXPECT_EQ("", FilePath("adir/").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName("adir/afile") -> "afile"
+TEST(RemoveDirectoryNameTest, ShouldGiveFileNameForAlternateSeparator) {
+  EXPECT_EQ("afile", FilePath("adir/afile").RemoveDirectoryName().string());
+}
+
+// RemoveDirectoryName("adir/subdir/afile") -> "afile"
+TEST(RemoveDirectoryNameTest, ShouldAlsoGiveFileNameForAlternateSeparator) {
+  EXPECT_EQ("afile",
+            FilePath("adir/subdir/afile").RemoveDirectoryName().string());
+}
+
+#endif
+
+// RemoveFileName "" -> "./"
+TEST(RemoveFileNameTest, EmptyName) {
+#if GTEST_OS_WINDOWS_MOBILE
+  // On Windows CE, we use the root as the current directory.
+  EXPECT_EQ(GTEST_PATH_SEP_, FilePath("").RemoveFileName().string());
+#else
+  EXPECT_EQ("." GTEST_PATH_SEP_, FilePath("").RemoveFileName().string());
+#endif
+}
+
+// RemoveFileName "adir/" -> "adir/"
+TEST(RemoveFileNameTest, ButNoFile) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_,
+      FilePath("adir" GTEST_PATH_SEP_).RemoveFileName().string());
+}
+
+// RemoveFileName "adir/afile" -> "adir/"
+TEST(RemoveFileNameTest, GivesDirName) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_,
+            FilePath("adir" GTEST_PATH_SEP_ "afile").RemoveFileName().string());
+}
+
+// RemoveFileName "adir/subdir/afile" -> "adir/subdir/"
+TEST(RemoveFileNameTest, GivesDirAndSubDirName) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_ "subdir" GTEST_PATH_SEP_,
+      FilePath("adir" GTEST_PATH_SEP_ "subdir" GTEST_PATH_SEP_ "afile")
+      .RemoveFileName().string());
+}
+
+// RemoveFileName "/afile" -> "/"
+TEST(RemoveFileNameTest, GivesRootDir) {
+  EXPECT_EQ(GTEST_PATH_SEP_,
+      FilePath(GTEST_PATH_SEP_ "afile").RemoveFileName().string());
+}
+
+#if GTEST_HAS_ALT_PATH_SEP_
+
+// Tests that RemoveFileName() works with the alternate separator on
+// Windows.
+
+// RemoveFileName("adir/") -> "adir/"
+TEST(RemoveFileNameTest, ButNoFileForAlternateSeparator) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_,
+            FilePath("adir/").RemoveFileName().string());
+}
+
+// RemoveFileName("adir/afile") -> "adir/"
+TEST(RemoveFileNameTest, GivesDirNameForAlternateSeparator) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_,
+            FilePath("adir/afile").RemoveFileName().string());
+}
+
+// RemoveFileName("adir/subdir/afile") -> "adir/subdir/"
+TEST(RemoveFileNameTest, GivesDirAndSubDirNameForAlternateSeparator) {
+  EXPECT_EQ("adir" GTEST_PATH_SEP_ "subdir" GTEST_PATH_SEP_,
+            FilePath("adir/subdir/afile").RemoveFileName().string());
+}
+
+// RemoveFileName("/afile") -> "\"
+TEST(RemoveFileNameTest, GivesRootDirForAlternateSeparator) {
+  EXPECT_EQ(GTEST_PATH_SEP_, FilePath("/afile").RemoveFileName().string());
+}
+
+#endif
+
+TEST(MakeFileNameTest, GenerateWhenNumberIsZero) {
+  FilePath actual = FilePath::MakeFileName(FilePath("foo"), FilePath("bar"),
+      0, "xml");
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar.xml", actual.string());
+}
+
+TEST(MakeFileNameTest, GenerateFileNameNumberGtZero) {
+  FilePath actual = FilePath::MakeFileName(FilePath("foo"), FilePath("bar"),
+      12, "xml");
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar_12.xml", actual.string());
+}
+
+TEST(MakeFileNameTest, GenerateFileNameWithSlashNumberIsZero) {
+  FilePath actual = FilePath::MakeFileName(FilePath("foo" GTEST_PATH_SEP_),
+      FilePath("bar"), 0, "xml");
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar.xml", actual.string());
+}
+
+TEST(MakeFileNameTest, GenerateFileNameWithSlashNumberGtZero) {
+  FilePath actual = FilePath::MakeFileName(FilePath("foo" GTEST_PATH_SEP_),
+      FilePath("bar"), 12, "xml");
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar_12.xml", actual.string());
+}
+
+TEST(MakeFileNameTest, GenerateWhenNumberIsZeroAndDirIsEmpty) {
+  FilePath actual = FilePath::MakeFileName(FilePath(""), FilePath("bar"),
+      0, "xml");
+  EXPECT_EQ("bar.xml", actual.string());
+}
+
+TEST(MakeFileNameTest, GenerateWhenNumberIsNotZeroAndDirIsEmpty) {
+  FilePath actual = FilePath::MakeFileName(FilePath(""), FilePath("bar"),
+      14, "xml");
+  EXPECT_EQ("bar_14.xml", actual.string());
+}
+
+TEST(ConcatPathsTest, WorksWhenDirDoesNotEndWithPathSep) {
+  FilePath actual = FilePath::ConcatPaths(FilePath("foo"),
+                                          FilePath("bar.xml"));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar.xml", actual.string());
+}
+
+TEST(ConcatPathsTest, WorksWhenPath1EndsWithPathSep) {
+  FilePath actual = FilePath::ConcatPaths(FilePath("foo" GTEST_PATH_SEP_),
+                                          FilePath("bar.xml"));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar.xml", actual.string());
+}
+
+TEST(ConcatPathsTest, Path1BeingEmpty) {
+  FilePath actual = FilePath::ConcatPaths(FilePath(""),
+                                          FilePath("bar.xml"));
+  EXPECT_EQ("bar.xml", actual.string());
+}
+
+TEST(ConcatPathsTest, Path2BeingEmpty) {
+  FilePath actual = FilePath::ConcatPaths(FilePath("foo"), FilePath(""));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_, actual.string());
+}
+
+TEST(ConcatPathsTest, BothPathBeingEmpty) {
+  FilePath actual = FilePath::ConcatPaths(FilePath(""),
+                                          FilePath(""));
+  EXPECT_EQ("", actual.string());
+}
+
+TEST(ConcatPathsTest, Path1ContainsPathSep) {
+  FilePath actual = FilePath::ConcatPaths(FilePath("foo" GTEST_PATH_SEP_ "bar"),
+                                          FilePath("foobar.xml"));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar" GTEST_PATH_SEP_ "foobar.xml",
+            actual.string());
+}
+
+TEST(ConcatPathsTest, Path2ContainsPathSep) {
+  FilePath actual = FilePath::ConcatPaths(
+      FilePath("foo" GTEST_PATH_SEP_),
+      FilePath("bar" GTEST_PATH_SEP_ "bar.xml"));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar" GTEST_PATH_SEP_ "bar.xml",
+            actual.string());
+}
+
+TEST(ConcatPathsTest, Path2EndsWithPathSep) {
+  FilePath actual = FilePath::ConcatPaths(FilePath("foo"),
+                                          FilePath("bar" GTEST_PATH_SEP_));
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar" GTEST_PATH_SEP_, actual.string());
+}
+
+// RemoveTrailingPathSeparator "" -> ""
+TEST(RemoveTrailingPathSeparatorTest, EmptyString) {
+  EXPECT_EQ("", FilePath("").RemoveTrailingPathSeparator().string());
+}
+
+// RemoveTrailingPathSeparator "foo" -> "foo"
+TEST(RemoveTrailingPathSeparatorTest, FileNoSlashString) {
+  EXPECT_EQ("foo", FilePath("foo").RemoveTrailingPathSeparator().string());
+}
+
+// RemoveTrailingPathSeparator "foo/" -> "foo"
+TEST(RemoveTrailingPathSeparatorTest, ShouldRemoveTrailingSeparator) {
+  EXPECT_EQ("foo",
+      FilePath("foo" GTEST_PATH_SEP_).RemoveTrailingPathSeparator().string());
+#if GTEST_HAS_ALT_PATH_SEP_
+  EXPECT_EQ("foo", FilePath("foo/").RemoveTrailingPathSeparator().string());
+#endif
+}
+
+// RemoveTrailingPathSeparator "foo/bar/" -> "foo/bar/"
+TEST(RemoveTrailingPathSeparatorTest, ShouldRemoveLastSeparator) {
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar",
+            FilePath("foo" GTEST_PATH_SEP_ "bar" GTEST_PATH_SEP_)
+                .RemoveTrailingPathSeparator().string());
+}
+
+// RemoveTrailingPathSeparator "foo/bar" -> "foo/bar"
+TEST(RemoveTrailingPathSeparatorTest, ShouldReturnUnmodified) {
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar",
+            FilePath("foo" GTEST_PATH_SEP_ "bar")
+                .RemoveTrailingPathSeparator().string());
+}
+
+TEST(DirectoryTest, RootDirectoryExists) {
+#if GTEST_OS_WINDOWS  // We are on Windows.
+  char current_drive[_MAX_PATH];  // NOLINT
+  current_drive[0] = static_cast<char>(_getdrive() + 'A' - 1);
+  current_drive[1] = ':';
+  current_drive[2] = '\\';
+  current_drive[3] = '\0';
+  EXPECT_TRUE(FilePath(current_drive).DirectoryExists());
+#else
+  EXPECT_TRUE(FilePath("/").DirectoryExists());
+#endif  // GTEST_OS_WINDOWS
+}
+
+#if GTEST_OS_WINDOWS
+TEST(DirectoryTest, RootOfWrongDriveDoesNotExists) {
+  const int saved_drive_ = _getdrive();
+  // Find a drive that doesn't exist. Start with 'Z' to avoid common ones.
+  for (char drive = 'Z'; drive >= 'A'; drive--)
+    if (_chdrive(drive - 'A' + 1) == -1) {
+      char non_drive[_MAX_PATH];  // NOLINT
+      non_drive[0] = drive;
+      non_drive[1] = ':';
+      non_drive[2] = '\\';
+      non_drive[3] = '\0';
+      EXPECT_FALSE(FilePath(non_drive).DirectoryExists());
+      break;
+    }
+  _chdrive(saved_drive_);
+}
+#endif  // GTEST_OS_WINDOWS
+
+#if !GTEST_OS_WINDOWS_MOBILE
+// Windows CE _does_ consider an empty directory to exist.
+TEST(DirectoryTest, EmptyPathDirectoryDoesNotExist) {
+  EXPECT_FALSE(FilePath("").DirectoryExists());
+}
+#endif  // !GTEST_OS_WINDOWS_MOBILE
+
+TEST(DirectoryTest, CurrentDirectoryExists) {
+#if GTEST_OS_WINDOWS  // We are on Windows.
+# ifndef _WIN32_CE  // Windows CE doesn't have a current directory.
+
+  EXPECT_TRUE(FilePath(".").DirectoryExists());
+  EXPECT_TRUE(FilePath(".\\").DirectoryExists());
+
+# endif  // _WIN32_CE
+#else
+  EXPECT_TRUE(FilePath(".").DirectoryExists());
+  EXPECT_TRUE(FilePath("./").DirectoryExists());
+#endif  // GTEST_OS_WINDOWS
+}
+
+// "foo/bar" == foo//bar" == "foo///bar"
+TEST(NormalizeTest, MultipleConsecutiveSepaparatorsInMidstring) {
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar",
+            FilePath("foo" GTEST_PATH_SEP_ "bar").string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar",
+            FilePath("foo" GTEST_PATH_SEP_ GTEST_PATH_SEP_ "bar").string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_ "bar",
+            FilePath("foo" GTEST_PATH_SEP_ GTEST_PATH_SEP_
+                     GTEST_PATH_SEP_ "bar").string());
+}
+
+// "/bar" == //bar" == "///bar"
+TEST(NormalizeTest, MultipleConsecutiveSepaparatorsAtStringStart) {
+  EXPECT_EQ(GTEST_PATH_SEP_ "bar",
+    FilePath(GTEST_PATH_SEP_ "bar").string());
+  EXPECT_EQ(GTEST_PATH_SEP_ "bar",
+    FilePath(GTEST_PATH_SEP_ GTEST_PATH_SEP_ "bar").string());
+  EXPECT_EQ(GTEST_PATH_SEP_ "bar",
+    FilePath(GTEST_PATH_SEP_ GTEST_PATH_SEP_ GTEST_PATH_SEP_ "bar").string());
+}
+
+// "foo/" == foo//" == "foo///"
+TEST(NormalizeTest, MultipleConsecutiveSepaparatorsAtStringEnd) {
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+    FilePath("foo" GTEST_PATH_SEP_).string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+    FilePath("foo" GTEST_PATH_SEP_ GTEST_PATH_SEP_).string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+    FilePath("foo" GTEST_PATH_SEP_ GTEST_PATH_SEP_ GTEST_PATH_SEP_).string());
+}
+
+#if GTEST_HAS_ALT_PATH_SEP_
+
+// Tests that separators at the end of the string are normalized
+// regardless of their combination (e.g. "foo\" =="foo/\" ==
+// "foo\\/").
+TEST(NormalizeTest, MixAlternateSeparatorAtStringEnd) {
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+            FilePath("foo/").string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+            FilePath("foo" GTEST_PATH_SEP_ "/").string());
+  EXPECT_EQ("foo" GTEST_PATH_SEP_,
+            FilePath("foo//" GTEST_PATH_SEP_).string());
+}
+
+#endif
+
+TEST(AssignmentOperatorTest, DefaultAssignedToNonDefault) {
+  FilePath default_path;
+  FilePath non_default_path("path");
+  non_default_path = default_path;
+  EXPECT_EQ("", non_default_path.string());
+  EXPECT_EQ("", default_path.string());  // RHS var is unchanged.
+}
+
+TEST(AssignmentOperatorTest, NonDefaultAssignedToDefault) {
+  FilePath non_default_path("path");
+  FilePath default_path;
+  default_path = non_default_path;
+  EXPECT_EQ("path", default_path.string());
+  EXPECT_EQ("path", non_default_path.string());  // RHS var is unchanged.
+}
+
+TEST(AssignmentOperatorTest, ConstAssignedToNonConst) {
+  const FilePath const_default_path("const_path");
+  FilePath non_default_path("path");
+  non_default_path = const_default_path;
+  EXPECT_EQ("const_path", non_default_path.string());
+}
+
+class DirectoryCreationTest : public Test {
+ protected:
+  virtual void SetUp() {
+    testdata_path_.Set(FilePath(
+        TempDir() + GetCurrentExecutableName().string() +
+        "_directory_creation" GTEST_PATH_SEP_ "test" GTEST_PATH_SEP_));
+    testdata_file_.Set(testdata_path_.RemoveTrailingPathSeparator());
+
+    unique_file0_.Set(FilePath::MakeFileName(testdata_path_, FilePath("unique"),
+        0, "txt"));
+    unique_file1_.Set(FilePath::MakeFileName(testdata_path_, FilePath("unique"),
+        1, "txt"));
+
+    remove(testdata_file_.c_str());
+    remove(unique_file0_.c_str());
+    remove(unique_file1_.c_str());
+    posix::RmDir(testdata_path_.c_str());
+  }
+
+  virtual void TearDown() {
+    remove(testdata_file_.c_str());
+    remove(unique_file0_.c_str());
+    remove(unique_file1_.c_str());
+    posix::RmDir(testdata_path_.c_str());
+  }
+
+  void CreateTextFile(const char* filename) {
+    FILE* f = posix::FOpen(filename, "w");
+    fprintf(f, "text\n");
+    fclose(f);
+  }
+
+  // Strings representing a directory and a file, with identical paths
+  // except for the trailing separator character that distinquishes
+  // a directory named 'test' from a file named 'test'. Example names:
+  FilePath testdata_path_;  // "/tmp/directory_creation/test/"
+  FilePath testdata_file_;  // "/tmp/directory_creation/test"
+  FilePath unique_file0_;  // "/tmp/directory_creation/test/unique.txt"
+  FilePath unique_file1_;  // "/tmp/directory_creation/test/unique_1.txt"
+};
+
+TEST_F(DirectoryCreationTest, CreateDirectoriesRecursively) {
+  EXPECT_FALSE(testdata_path_.DirectoryExists()) << testdata_path_.string();
+  EXPECT_TRUE(testdata_path_.CreateDirectoriesRecursively());
+  EXPECT_TRUE(testdata_path_.DirectoryExists());
+}
+
+TEST_F(DirectoryCreationTest, CreateDirectoriesForAlreadyExistingPath) {
+  EXPECT_FALSE(testdata_path_.DirectoryExists()) << testdata_path_.string();
+  EXPECT_TRUE(testdata_path_.CreateDirectoriesRecursively());
+  // Call 'create' again... should still succeed.
+  EXPECT_TRUE(testdata_path_.CreateDirectoriesRecursively());
+}
+
+TEST_F(DirectoryCreationTest, CreateDirectoriesAndUniqueFilename) {
+  FilePath file_path(FilePath::GenerateUniqueFileName(testdata_path_,
+      FilePath("unique"), "txt"));
+  EXPECT_EQ(unique_file0_.string(), file_path.string());
+  EXPECT_FALSE(file_path.FileOrDirectoryExists());  // file not there
+
+  testdata_path_.CreateDirectoriesRecursively();
+  EXPECT_FALSE(file_path.FileOrDirectoryExists());  // file still not there
+  CreateTextFile(file_path.c_str());
+  EXPECT_TRUE(file_path.FileOrDirectoryExists());
+
+  FilePath file_path2(FilePath::GenerateUniqueFileName(testdata_path_,
+      FilePath("unique"), "txt"));
+  EXPECT_EQ(unique_file1_.string(), file_path2.string());
+  EXPECT_FALSE(file_path2.FileOrDirectoryExists());  // file not there
+  CreateTextFile(file_path2.c_str());
+  EXPECT_TRUE(file_path2.FileOrDirectoryExists());
+}
+
+TEST_F(DirectoryCreationTest, CreateDirectoriesFail) {
+  // force a failure by putting a file where we will try to create a directory.
+  CreateTextFile(testdata_file_.c_str());
+  EXPECT_TRUE(testdata_file_.FileOrDirectoryExists());
+  EXPECT_FALSE(testdata_file_.DirectoryExists());
+  EXPECT_FALSE(testdata_file_.CreateDirectoriesRecursively());
+}
+
+TEST(NoDirectoryCreationTest, CreateNoDirectoriesForDefaultXmlFile) {
+  const FilePath test_detail_xml("test_detail.xml");
+  EXPECT_FALSE(test_detail_xml.CreateDirectoriesRecursively());
+}
+
+TEST(FilePathTest, DefaultConstructor) {
+  FilePath fp;
+  EXPECT_EQ("", fp.string());
+}
+
+TEST(FilePathTest, CharAndCopyConstructors) {
+  const FilePath fp("spicy");
+  EXPECT_EQ("spicy", fp.string());
+
+  const FilePath fp_copy(fp);
+  EXPECT_EQ("spicy", fp_copy.string());
+}
+
+TEST(FilePathTest, StringConstructor) {
+  const FilePath fp(std::string("cider"));
+  EXPECT_EQ("cider", fp.string());
+}
+
+TEST(FilePathTest, Set) {
+  const FilePath apple("apple");
+  FilePath mac("mac");
+  mac.Set(apple);  // Implement Set() since overloading operator= is forbidden.
+  EXPECT_EQ("apple", mac.string());
+  EXPECT_EQ("apple", apple.string());
+}
+
+TEST(FilePathTest, ToString) {
+  const FilePath file("drink");
+  EXPECT_EQ("drink", file.string());
+}
+
+TEST(FilePathTest, RemoveExtension) {
+  EXPECT_EQ("app", FilePath("app.cc").RemoveExtension("cc").string());
+  EXPECT_EQ("app", FilePath("app.exe").RemoveExtension("exe").string());
+  EXPECT_EQ("APP", FilePath("APP.EXE").RemoveExtension("exe").string());
+}
+
+TEST(FilePathTest, RemoveExtensionWhenThereIsNoExtension) {
+  EXPECT_EQ("app", FilePath("app").RemoveExtension("exe").string());
+}
+
+TEST(FilePathTest, IsDirectory) {
+  EXPECT_FALSE(FilePath("cola").IsDirectory());
+  EXPECT_TRUE(FilePath("koala" GTEST_PATH_SEP_).IsDirectory());
+#if GTEST_HAS_ALT_PATH_SEP_
+  EXPECT_TRUE(FilePath("koala/").IsDirectory());
+#endif
+}
+
+TEST(FilePathTest, IsAbsolutePath) {
+  EXPECT_FALSE(FilePath("is" GTEST_PATH_SEP_ "relative").IsAbsolutePath());
+  EXPECT_FALSE(FilePath("").IsAbsolutePath());
+#if GTEST_OS_WINDOWS
+  EXPECT_TRUE(FilePath("c:\\" GTEST_PATH_SEP_ "is_not"
+                       GTEST_PATH_SEP_ "relative").IsAbsolutePath());
+  EXPECT_FALSE(FilePath("c:foo" GTEST_PATH_SEP_ "bar").IsAbsolutePath());
+  EXPECT_TRUE(FilePath("c:/" GTEST_PATH_SEP_ "is_not"
+                       GTEST_PATH_SEP_ "relative").IsAbsolutePath());
+#else
+  EXPECT_TRUE(FilePath(GTEST_PATH_SEP_ "is_not" GTEST_PATH_SEP_ "relative")
+              .IsAbsolutePath());
+#endif  // GTEST_OS_WINDOWS
+}
+
+TEST(FilePathTest, IsRootDirectory) {
+#if GTEST_OS_WINDOWS
+  EXPECT_TRUE(FilePath("a:\\").IsRootDirectory());
+  EXPECT_TRUE(FilePath("Z:/").IsRootDirectory());
+  EXPECT_TRUE(FilePath("e://").IsRootDirectory());
+  EXPECT_FALSE(FilePath("").IsRootDirectory());
+  EXPECT_FALSE(FilePath("b:").IsRootDirectory());
+  EXPECT_FALSE(FilePath("b:a").IsRootDirectory());
+  EXPECT_FALSE(FilePath("8:/").IsRootDirectory());
+  EXPECT_FALSE(FilePath("c|/").IsRootDirectory());
+#else
+  EXPECT_TRUE(FilePath("/").IsRootDirectory());
+  EXPECT_TRUE(FilePath("//").IsRootDirectory());
+  EXPECT_FALSE(FilePath("").IsRootDirectory());
+  EXPECT_FALSE(FilePath("\\").IsRootDirectory());
+  EXPECT_FALSE(FilePath("/x").IsRootDirectory());
+#endif
+}
+
+}  // namespace
+}  // namespace internal
+}  // namespace testing
+// Copyright 2009 Google Inc. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+//
+// The Google C++ Testing Framework (Google Test)
+//
+// This file verifies Google Test event listeners receive events at the
+// right times.
+
+#include "gtest/gtest.h"
+#include <vector>
+
+using ::testing::AddGlobalTestEnvironment;
+using ::testing::Environment;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListener;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+// Used by tests to register their events.
+std::vector<std::string>* g_events = NULL;
+
+namespace testing {
+namespace internal {
+
+class EventRecordingListener : public TestEventListener {
+ public:
+  explicit EventRecordingListener(const char* name) : name_(name) {}
+
+ protected:
+  virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnTestProgramStart"));
+  }
+
+  virtual void OnTestIterationStart(const UnitTest& /*unit_test*/,
+                                    int iteration) {
+    Message message;
+    message << GetFullMethodName("OnTestIterationStart")
+            << "(" << iteration << ")";
+    g_events->push_back(message.GetString());
+  }
+
+  virtual void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnEnvironmentsSetUpStart"));
+  }
+
+  virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnEnvironmentsSetUpEnd"));
+  }
+
+  virtual void OnTestCaseStart(const TestCase& /*test_case*/) {
+    g_events->push_back(GetFullMethodName("OnTestCaseStart"));
+  }
+
+  virtual void OnTestStart(const TestInfo& /*test_info*/) {
+    g_events->push_back(GetFullMethodName("OnTestStart"));
+  }
+
+  virtual void OnTestPartResult(const TestPartResult& /*test_part_result*/) {
+    g_events->push_back(GetFullMethodName("OnTestPartResult"));
+  }
+
+  virtual void OnTestEnd(const TestInfo& /*test_info*/) {
+    g_events->push_back(GetFullMethodName("OnTestEnd"));
+  }
+
+  virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {
+    g_events->push_back(GetFullMethodName("OnTestCaseEnd"));
+  }
+
+  virtual void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnEnvironmentsTearDownStart"));
+  }
+
+  virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnEnvironmentsTearDownEnd"));
+  }
+
+  virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/,
+                                  int iteration) {
+    Message message;
+    message << GetFullMethodName("OnTestIterationEnd")
+            << "("  << iteration << ")";
+    g_events->push_back(message.GetString());
+  }
+
+  virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {
+    g_events->push_back(GetFullMethodName("OnTestProgramEnd"));
+  }
+
+ private:
+  std::string GetFullMethodName(const char* name) {
+    return name_ + "." + name;
+  }
+
+  std::string name_;
+};
+
+class EnvironmentInvocationCatcher : public Environment {
+ protected:
+  virtual void SetUp() {
+    g_events->push_back("Environment::SetUp");
+  }
+
+  virtual void TearDown() {
+    g_events->push_back("Environment::TearDown");
+  }
+};
+
+class ListenerTest : public Test {
+ protected:
+  static void SetUpTestCase() {
+    g_events->push_back("ListenerTest::SetUpTestCase");
+  }
+
+  static void TearDownTestCase() {
+    g_events->push_back("ListenerTest::TearDownTestCase");
+  }
+
+  virtual void SetUp() {
+    g_events->push_back("ListenerTest::SetUp");
+  }
+
+  virtual void TearDown() {
+    g_events->push_back("ListenerTest::TearDown");
+  }
+};
+
+TEST_F(ListenerTest, DoesFoo) {
+  // Test execution order within a test case is not guaranteed so we are not
+  // recording the test name.
+  g_events->push_back("ListenerTest::* Test Body");
+  SUCCEED();  // Triggers OnTestPartResult.
+}
+
+TEST_F(ListenerTest, DoesBar) {
+  g_events->push_back("ListenerTest::* Test Body");
+  SUCCEED();  // Triggers OnTestPartResult.
+}
+
+}  // namespace internal
+
+}  // namespace testing
+
+using ::testing::internal::EnvironmentInvocationCatcher;
+using ::testing::internal::EventRecordingListener;
+
+void VerifyResults(const std::vector<std::string>& data,
+                   const char* const* expected_data,
+                   size_t expected_data_size) {
+  const size_t actual_size = data.size();
+  // If the following assertion fails, a new entry will be appended to
+  // data.  Hence we save data.size() first.
+  EXPECT_EQ(expected_data_size, actual_size);
+
+  // Compares the common prefix.
+  const size_t shorter_size = expected_data_size <= actual_size ?
+      expected_data_size : actual_size;
+  size_t i = 0;
+  for (; i < shorter_size; ++i) {
+    ASSERT_STREQ(expected_data[i], data[i].c_str())
+        << "at position " << i;
+  }
+
+  // Prints extra elements in the actual data.
+  for (; i < actual_size; ++i) {
+    printf("  Actual event #%lu: %s\n",
+        static_cast<unsigned long>(i), data[i].c_str());
+  }
+}
+
+int main(int argc, char **argv) {
+  std::vector<std::string> events;
+  g_events = &events;
+  InitGoogleTest(&argc, argv);
+
+  UnitTest::GetInstance()->listeners().Append(
+      new EventRecordingListener("1st"));
+  UnitTest::GetInstance()->listeners().Append(
+      new EventRecordingListener("2nd"));
+
+  AddGlobalTestEnvironment(new EnvironmentInvocationCatcher);
+
+  GTEST_CHECK_(events.size() == 0)
+      << "AddGlobalTestEnvironment should not generate any events itself.";
+
+  ::testing::GTEST_FLAG(repeat) = 2;
+  int ret_val = RUN_ALL_TESTS();
+
+  const char* const expected_events[] = {
+    "1st.OnTestProgramStart",
+    "2nd.OnTestProgramStart",
+    "1st.OnTestIterationStart(0)",
+    "2nd.OnTestIterationStart(0)",
+    "1st.OnEnvironmentsSetUpStart",
+    "2nd.OnEnvironmentsSetUpStart",
+    "Environment::SetUp",
+    "2nd.OnEnvironmentsSetUpEnd",
+    "1st.OnEnvironmentsSetUpEnd",
+    "1st.OnTestCaseStart",
+    "2nd.OnTestCaseStart",
+    "ListenerTest::SetUpTestCase",
+    "1st.OnTestStart",
+    "2nd.OnTestStart",
+    "ListenerTest::SetUp",
+    "ListenerTest::* Test Body",
+    "1st.OnTestPartResult",
+    "2nd.OnTestPartResult",
+    "ListenerTest::TearDown",
+    "2nd.OnTestEnd",
+    "1st.OnTestEnd",
+    "1st.OnTestStart",
+    "2nd.OnTestStart",
+    "ListenerTest::SetUp",
+    "ListenerTest::* Test Body",
+    "1st.OnTestPartResult",
+    "2nd.OnTestPartResult",
+    "ListenerTest::TearDown",
+    "2nd.OnTestEnd",
+    "1st.OnTestEnd",
+    "ListenerTest::TearDownTestCase",
+    "2nd.OnTestCaseEnd",
+    "1st.OnTestCaseEnd",
+    "1st.OnEnvironmentsTearDownStart",
+    "2nd.OnEnvironmentsTearDownStart",
+    "Environment::TearDown",
+    "2nd.OnEnvironmentsTearDownEnd",
+    "1st.OnEnvironmentsTearDownEnd",
+    "2nd.OnTestIterationEnd(0)",
+    "1st.OnTestIterationEnd(0)",
+    "1st.OnTestIterationStart(1)",
+    "2nd.OnTestIterationStart(1)",
+    "1st.OnEnvironmentsSetUpStart",
+    "2nd.OnEnvironmentsSetUpStart",
+    "Environment::SetUp",
+    "2nd.OnEnvironmentsSetUpEnd",
+    "1st.OnEnvironmentsSetUpEnd",
+    "1st.OnTestCaseStart",
+    "2nd.OnTestCaseStart",
+    "ListenerTest::SetUpTestCase",
+    "1st.OnTestStart",
+    "2nd.OnTestStart",
+    "ListenerTest::SetUp",
+    "ListenerTest::* Test Body",
+    "1st.OnTestPartResult",
+    "2nd.OnTestPartResult",
+    "ListenerTest::TearDown",
+    "2nd.OnTestEnd",
+    "1st.OnTestEnd",
+    "1st.OnTestStart",
+    "2nd.OnTestStart",
+    "ListenerTest::SetUp",
+    "ListenerTest::* Test Body",
+    "1st.OnTestPartResult",
+    "2nd.OnTestPartResult",
+    "ListenerTest::TearDown",
+    "2nd.OnTestEnd",
+    "1st.OnTestEnd",
+    "ListenerTest::TearDownTestCase",
+    "2nd.OnTestCaseEnd",
+    "1st.OnTestCaseEnd",
+    "1st.OnEnvironmentsTearDownStart",
+    "2nd.OnEnvironmentsTearDownStart",
+    "Environment::TearDown",
+    "2nd.OnEnvironmentsTearDownEnd",
+    "1st.OnEnvironmentsTearDownEnd",
+    "2nd.OnTestIterationEnd(1)",
+    "1st.OnTestIterationEnd(1)",
+    "2nd.OnTestProgramEnd",
+    "1st.OnTestProgramEnd"
+  };
+  VerifyResults(events,
+                expected_events,
+                sizeof(expected_events)/sizeof(expected_events[0]));
+
+  // We need to check manually for ad hoc test failures that happen after
+  // RUN_ALL_TESTS finishes.
+  if (UnitTest::GetInstance()->Failed())
+    ret_val = 1;
+
+  return ret_val;
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: keith.ray@gmail.com (Keith Ray)
+//
+// Google Test UnitTestOptions tests
+//
+// This file tests classes and functions used internally by
+// Google Test.  They are subject to change without notice.
+//
+// This file is #included from gtest.cc, to avoid changing build or
+// make-files on Windows and other platforms. Do not #include this file
+// anywhere else!
+
+#include "gtest/gtest.h"
+
+#if GTEST_OS_WINDOWS_MOBILE
+# include <windows.h>
+#elif GTEST_OS_WINDOWS
+# include <direct.h>
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+namespace internal {
+namespace {
+
+// Turns the given relative path into an absolute path.
+FilePath GetAbsolutePathOf(const FilePath& relative_path) {
+  return FilePath::ConcatPaths(FilePath::GetCurrentDir(), relative_path);
+}
+
+// Testing UnitTestOptions::GetOutputFormat/GetOutputFile.
+
+TEST(XmlOutputTest, GetOutputFormatDefault) {
+  GTEST_FLAG(output) = "";
+  EXPECT_STREQ("", UnitTestOptions::GetOutputFormat().c_str());
+}
+
+TEST(XmlOutputTest, GetOutputFormat) {
+  GTEST_FLAG(output) = "xml:filename";
+  EXPECT_STREQ("xml", UnitTestOptions::GetOutputFormat().c_str());
+}
+
+TEST(XmlOutputTest, GetOutputFileDefault) {
+  GTEST_FLAG(output) = "";
+  EXPECT_EQ(GetAbsolutePathOf(FilePath("test_detail.xml")).string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+}
+
+TEST(XmlOutputTest, GetOutputFileSingleFile) {
+  GTEST_FLAG(output) = "xml:filename.abc";
+  EXPECT_EQ(GetAbsolutePathOf(FilePath("filename.abc")).string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+}
+
+TEST(XmlOutputTest, GetOutputFileFromDirectoryPath) {
+  GTEST_FLAG(output) = "xml:path" GTEST_PATH_SEP_;
+  const std::string expected_output_file =
+      GetAbsolutePathOf(
+          FilePath(std::string("path") + GTEST_PATH_SEP_ +
+                   GetCurrentExecutableName().string() + ".xml")).string();
+  const std::string& output_file =
+      UnitTestOptions::GetAbsolutePathToOutputFile();
+#if GTEST_OS_WINDOWS
+  EXPECT_STRCASEEQ(expected_output_file.c_str(), output_file.c_str());
+#else
+  EXPECT_EQ(expected_output_file, output_file.c_str());
+#endif
+}
+
+TEST(OutputFileHelpersTest, GetCurrentExecutableName) {
+  const std::string exe_str = GetCurrentExecutableName().string();
+#if GTEST_OS_WINDOWS
+  const bool success =
+      _strcmpi("gtest-options_test", exe_str.c_str()) == 0 ||
+      _strcmpi("gtest-options-ex_test", exe_str.c_str()) == 0 ||
+      _strcmpi("gtest_all_test", exe_str.c_str()) == 0 ||
+      _strcmpi("gtest_dll_test", exe_str.c_str()) == 0;
+#else
+  // TODO(wan@google.com): remove the hard-coded "lt-" prefix when
+  //   Chandler Carruth's libtool replacement is ready.
+  const bool success =
+      exe_str == "gtest-options_test" ||
+      exe_str == "gtest_all_test" ||
+      exe_str == "lt-gtest_all_test" ||
+      exe_str == "gtest_dll_test";
+#endif  // GTEST_OS_WINDOWS
+  if (!success)
+    FAIL() << "GetCurrentExecutableName() returns " << exe_str;
+}
+
+class XmlOutputChangeDirTest : public Test {
+ protected:
+  virtual void SetUp() {
+    original_working_dir_ = FilePath::GetCurrentDir();
+    posix::ChDir("..");
+    // This will make the test fail if run from the root directory.
+    EXPECT_NE(original_working_dir_.string(),
+              FilePath::GetCurrentDir().string());
+  }
+
+  virtual void TearDown() {
+    posix::ChDir(original_working_dir_.string().c_str());
+  }
+
+  FilePath original_working_dir_;
+};
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithDefault) {
+  GTEST_FLAG(output) = "";
+  EXPECT_EQ(FilePath::ConcatPaths(original_working_dir_,
+                                  FilePath("test_detail.xml")).string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+}
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithDefaultXML) {
+  GTEST_FLAG(output) = "xml";
+  EXPECT_EQ(FilePath::ConcatPaths(original_working_dir_,
+                                  FilePath("test_detail.xml")).string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+}
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithRelativeFile) {
+  GTEST_FLAG(output) = "xml:filename.abc";
+  EXPECT_EQ(FilePath::ConcatPaths(original_working_dir_,
+                                  FilePath("filename.abc")).string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+}
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithRelativePath) {
+  GTEST_FLAG(output) = "xml:path" GTEST_PATH_SEP_;
+  const std::string expected_output_file =
+      FilePath::ConcatPaths(
+          original_working_dir_,
+          FilePath(std::string("path") + GTEST_PATH_SEP_ +
+                   GetCurrentExecutableName().string() + ".xml")).string();
+  const std::string& output_file =
+      UnitTestOptions::GetAbsolutePathToOutputFile();
+#if GTEST_OS_WINDOWS
+  EXPECT_STRCASEEQ(expected_output_file.c_str(), output_file.c_str());
+#else
+  EXPECT_EQ(expected_output_file, output_file.c_str());
+#endif
+}
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithAbsoluteFile) {
+#if GTEST_OS_WINDOWS
+  GTEST_FLAG(output) = "xml:c:\\tmp\\filename.abc";
+  EXPECT_EQ(FilePath("c:\\tmp\\filename.abc").string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+#else
+  GTEST_FLAG(output) ="xml:/tmp/filename.abc";
+  EXPECT_EQ(FilePath("/tmp/filename.abc").string(),
+            UnitTestOptions::GetAbsolutePathToOutputFile());
+#endif
+}
+
+TEST_F(XmlOutputChangeDirTest, PreserveOriginalWorkingDirWithAbsolutePath) {
+#if GTEST_OS_WINDOWS
+  const std::string path = "c:\\tmp\\";
+#else
+  const std::string path = "/tmp/";
+#endif
+
+  GTEST_FLAG(output) = "xml:" + path;
+  const std::string expected_output_file =
+      path + GetCurrentExecutableName().string() + ".xml";
+  const std::string& output_file =
+      UnitTestOptions::GetAbsolutePathToOutputFile();
+
+#if GTEST_OS_WINDOWS
+  EXPECT_STRCASEEQ(expected_output_file.c_str(), output_file.c_str());
+#else
+  EXPECT_EQ(expected_output_file, output_file.c_str());
+#endif
+}
+
+}  // namespace
+}  // namespace internal
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+//
+// Tests for Google Test itself.  This verifies that the basic constructs of
+// Google Test work.
+
+#include "gtest/gtest.h"
+
+#include "test/gtest-param-test_test.h"
+
+#if GTEST_HAS_PARAM_TEST
+
+using ::testing::Values;
+using ::testing::internal::ParamGenerator;
+
+// Tests that generators defined in a different translation unit
+// are functional. The test using extern_gen is defined
+// in gtest-param-test_test.cc.
+ParamGenerator<int> extern_gen = Values(33);
+
+// Tests that a parameterized test case can be defined in one translation unit
+// and instantiated in another. The test is defined in gtest-param-test_test.cc
+// and ExternalInstantiationTest fixture class is defined in
+// gtest-param-test_test.h.
+INSTANTIATE_TEST_CASE_P(MultiplesOf33,
+                        ExternalInstantiationTest,
+                        Values(33, 66));
+
+// Tests that a parameterized test case can be instantiated
+// in multiple translation units. Another instantiation is defined
+// in gtest-param-test_test.cc and InstantiationInMultipleTranslaionUnitsTest
+// fixture is defined in gtest-param-test_test.h
+INSTANTIATE_TEST_CASE_P(Sequence2,
+                        InstantiationInMultipleTranslaionUnitsTest,
+                        Values(42*3, 42*4, 42*5));
+
+#endif  // GTEST_HAS_PARAM_TEST
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+//
+// Tests for Google Test itself. This file verifies that the parameter
+// generators objects produce correct parameter sequences and that
+// Google Test runtime instantiates correct tests from those sequences.
+
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_PARAM_TEST
+
+# include <algorithm>
+# include <iostream>
+# include <list>
+# include <sstream>
+# include <string>
+# include <vector>
+
+// To include gtest-internal-inl.h.
+# define GTEST_IMPLEMENTATION_ 1
+# include "src/gtest-internal-inl.h"  // for UnitTestOptions
+# undef GTEST_IMPLEMENTATION_
+
+# include "test/gtest-param-test_test.h"
+
+using ::std::vector;
+using ::std::sort;
+
+using ::testing::AddGlobalTestEnvironment;
+using ::testing::Bool;
+using ::testing::Message;
+using ::testing::Range;
+using ::testing::TestWithParam;
+using ::testing::Values;
+using ::testing::ValuesIn;
+
+# if GTEST_HAS_COMBINE
+using ::testing::Combine;
+using ::testing::get;
+using ::testing::make_tuple;
+using ::testing::tuple;
+# endif  // GTEST_HAS_COMBINE
+
+using ::testing::internal::ParamGenerator;
+using ::testing::internal::UnitTestOptions;
+
+// Prints a value to a string.
+//
+// TODO(wan@google.com): remove PrintValue() when we move matchers and
+// EXPECT_THAT() from Google Mock to Google Test.  At that time, we
+// can write EXPECT_THAT(x, Eq(y)) to compare two tuples x and y, as
+// EXPECT_THAT() and the matchers know how to print tuples.
+template <typename T>
+::std::string PrintValue(const T& value) {
+  ::std::stringstream stream;
+  stream << value;
+  return stream.str();
+}
+
+# if GTEST_HAS_COMBINE
+
+// These overloads allow printing tuples in our tests.  We cannot
+// define an operator<< for tuples, as that definition needs to be in
+// the std namespace in order to be picked up by Google Test via
+// Argument-Dependent Lookup, yet defining anything in the std
+// namespace in non-STL code is undefined behavior.
+
+template <typename T1, typename T2>
+::std::string PrintValue(const tuple<T1, T2>& value) {
+  ::std::stringstream stream;
+  stream << "(" << get<0>(value) << ", " << get<1>(value) << ")";
+  return stream.str();
+}
+
+template <typename T1, typename T2, typename T3>
+::std::string PrintValue(const tuple<T1, T2, T3>& value) {
+  ::std::stringstream stream;
+  stream << "(" << get<0>(value) << ", " << get<1>(value)
+         << ", "<< get<2>(value) << ")";
+  return stream.str();
+}
+
+template <typename T1, typename T2, typename T3, typename T4, typename T5,
+          typename T6, typename T7, typename T8, typename T9, typename T10>
+::std::string PrintValue(
+    const tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>& value) {
+  ::std::stringstream stream;
+  stream << "(" << get<0>(value) << ", " << get<1>(value)
+         << ", "<< get<2>(value) << ", " << get<3>(value)
+         << ", "<< get<4>(value) << ", " << get<5>(value)
+         << ", "<< get<6>(value) << ", " << get<7>(value)
+         << ", "<< get<8>(value) << ", " << get<9>(value) << ")";
+  return stream.str();
+}
+
+# endif  // GTEST_HAS_COMBINE
+
+// Verifies that a sequence generated by the generator and accessed
+// via the iterator object matches the expected one using Google Test
+// assertions.
+template <typename T, size_t N>
+void VerifyGenerator(const ParamGenerator<T>& generator,
+                     const T (&expected_values)[N]) {
+  typename ParamGenerator<T>::iterator it = generator.begin();
+  for (size_t i = 0; i < N; ++i) {
+    ASSERT_FALSE(it == generator.end())
+        << "At element " << i << " when accessing via an iterator "
+        << "created with the copy constructor.\n";
+    // We cannot use EXPECT_EQ() here as the values may be tuples,
+    // which don't support <<.
+    EXPECT_TRUE(expected_values[i] == *it)
+        << "where i is " << i
+        << ", expected_values[i] is " << PrintValue(expected_values[i])
+        << ", *it is " << PrintValue(*it)
+        << ", and 'it' is an iterator created with the copy constructor.\n";
+    it++;
+  }
+  EXPECT_TRUE(it == generator.end())
+        << "At the presumed end of sequence when accessing via an iterator "
+        << "created with the copy constructor.\n";
+
+  // Test the iterator assignment. The following lines verify that
+  // the sequence accessed via an iterator initialized via the
+  // assignment operator (as opposed to a copy constructor) matches
+  // just the same.
+  it = generator.begin();
+  for (size_t i = 0; i < N; ++i) {
+    ASSERT_FALSE(it == generator.end())
+        << "At element " << i << " when accessing via an iterator "
+        << "created with the assignment operator.\n";
+    EXPECT_TRUE(expected_values[i] == *it)
+        << "where i is " << i
+        << ", expected_values[i] is " << PrintValue(expected_values[i])
+        << ", *it is " << PrintValue(*it)
+        << ", and 'it' is an iterator created with the copy constructor.\n";
+    it++;
+  }
+  EXPECT_TRUE(it == generator.end())
+        << "At the presumed end of sequence when accessing via an iterator "
+        << "created with the assignment operator.\n";
+}
+
+template <typename T>
+void VerifyGeneratorIsEmpty(const ParamGenerator<T>& generator) {
+  typename ParamGenerator<T>::iterator it = generator.begin();
+  EXPECT_TRUE(it == generator.end());
+
+  it = generator.begin();
+  EXPECT_TRUE(it == generator.end());
+}
+
+// Generator tests. They test that each of the provided generator functions
+// generates an expected sequence of values. The general test pattern
+// instantiates a generator using one of the generator functions,
+// checks the sequence produced by the generator using its iterator API,
+// and then resets the iterator back to the beginning of the sequence
+// and checks the sequence again.
+
+// Tests that iterators produced by generator functions conform to the
+// ForwardIterator concept.
+TEST(IteratorTest, ParamIteratorConformsToForwardIteratorConcept) {
+  const ParamGenerator<int> gen = Range(0, 10);
+  ParamGenerator<int>::iterator it = gen.begin();
+
+  // Verifies that iterator initialization works as expected.
+  ParamGenerator<int>::iterator it2 = it;
+  EXPECT_TRUE(*it == *it2) << "Initialized iterators must point to the "
+                           << "element same as its source points to";
+
+  // Verifies that iterator assignment works as expected.
+  it++;
+  EXPECT_FALSE(*it == *it2);
+  it2 = it;
+  EXPECT_TRUE(*it == *it2) << "Assigned iterators must point to the "
+                           << "element same as its source points to";
+
+  // Verifies that prefix operator++() returns *this.
+  EXPECT_EQ(&it, &(++it)) << "Result of the prefix operator++ must be "
+                          << "refer to the original object";
+
+  // Verifies that the result of the postfix operator++ points to the value
+  // pointed to by the original iterator.
+  int original_value = *it;  // Have to compute it outside of macro call to be
+                             // unaffected by the parameter evaluation order.
+  EXPECT_EQ(original_value, *(it++));
+
+  // Verifies that prefix and postfix operator++() advance an iterator
+  // all the same.
+  it2 = it;
+  it++;
+  ++it2;
+  EXPECT_TRUE(*it == *it2);
+}
+
+// Tests that Range() generates the expected sequence.
+TEST(RangeTest, IntRangeWithDefaultStep) {
+  const ParamGenerator<int> gen = Range(0, 3);
+  const int expected_values[] = {0, 1, 2};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Edge case. Tests that Range() generates the single element sequence
+// as expected when provided with range limits that are equal.
+TEST(RangeTest, IntRangeSingleValue) {
+  const ParamGenerator<int> gen = Range(0, 1);
+  const int expected_values[] = {0};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Edge case. Tests that Range() with generates empty sequence when
+// supplied with an empty range.
+TEST(RangeTest, IntRangeEmpty) {
+  const ParamGenerator<int> gen = Range(0, 0);
+  VerifyGeneratorIsEmpty(gen);
+}
+
+// Tests that Range() with custom step (greater then one) generates
+// the expected sequence.
+TEST(RangeTest, IntRangeWithCustomStep) {
+  const ParamGenerator<int> gen = Range(0, 9, 3);
+  const int expected_values[] = {0, 3, 6};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that Range() with custom step (greater then one) generates
+// the expected sequence when the last element does not fall on the
+// upper range limit. Sequences generated by Range() must not have
+// elements beyond the range limits.
+TEST(RangeTest, IntRangeWithCustomStepOverUpperBound) {
+  const ParamGenerator<int> gen = Range(0, 4, 3);
+  const int expected_values[] = {0, 3};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Verifies that Range works with user-defined types that define
+// copy constructor, operator=(), operator+(), and operator<().
+class DogAdder {
+ public:
+  explicit DogAdder(const char* a_value) : value_(a_value) {}
+  DogAdder(const DogAdder& other) : value_(other.value_.c_str()) {}
+
+  DogAdder operator=(const DogAdder& other) {
+    if (this != &other)
+      value_ = other.value_;
+    return *this;
+  }
+  DogAdder operator+(const DogAdder& other) const {
+    Message msg;
+    msg << value_.c_str() << other.value_.c_str();
+    return DogAdder(msg.GetString().c_str());
+  }
+  bool operator<(const DogAdder& other) const {
+    return value_ < other.value_;
+  }
+  const std::string& value() const { return value_; }
+
+ private:
+  std::string value_;
+};
+
+TEST(RangeTest, WorksWithACustomType) {
+  const ParamGenerator<DogAdder> gen =
+      Range(DogAdder("cat"), DogAdder("catdogdog"), DogAdder("dog"));
+  ParamGenerator<DogAdder>::iterator it = gen.begin();
+
+  ASSERT_FALSE(it == gen.end());
+  EXPECT_STREQ("cat", it->value().c_str());
+
+  ASSERT_FALSE(++it == gen.end());
+  EXPECT_STREQ("catdog", it->value().c_str());
+
+  EXPECT_TRUE(++it == gen.end());
+}
+
+class IntWrapper {
+ public:
+  explicit IntWrapper(int a_value) : value_(a_value) {}
+  IntWrapper(const IntWrapper& other) : value_(other.value_) {}
+
+  IntWrapper operator=(const IntWrapper& other) {
+    value_ = other.value_;
+    return *this;
+  }
+  // operator+() adds a different type.
+  IntWrapper operator+(int other) const { return IntWrapper(value_ + other); }
+  bool operator<(const IntWrapper& other) const {
+    return value_ < other.value_;
+  }
+  int value() const { return value_; }
+
+ private:
+  int value_;
+};
+
+TEST(RangeTest, WorksWithACustomTypeWithDifferentIncrementType) {
+  const ParamGenerator<IntWrapper> gen = Range(IntWrapper(0), IntWrapper(2));
+  ParamGenerator<IntWrapper>::iterator it = gen.begin();
+
+  ASSERT_FALSE(it == gen.end());
+  EXPECT_EQ(0, it->value());
+
+  ASSERT_FALSE(++it == gen.end());
+  EXPECT_EQ(1, it->value());
+
+  EXPECT_TRUE(++it == gen.end());
+}
+
+// Tests that ValuesIn() with an array parameter generates
+// the expected sequence.
+TEST(ValuesInTest, ValuesInArray) {
+  int array[] = {3, 5, 8};
+  const ParamGenerator<int> gen = ValuesIn(array);
+  VerifyGenerator(gen, array);
+}
+
+// Tests that ValuesIn() with a const array parameter generates
+// the expected sequence.
+TEST(ValuesInTest, ValuesInConstArray) {
+  const int array[] = {3, 5, 8};
+  const ParamGenerator<int> gen = ValuesIn(array);
+  VerifyGenerator(gen, array);
+}
+
+// Edge case. Tests that ValuesIn() with an array parameter containing a
+// single element generates the single element sequence.
+TEST(ValuesInTest, ValuesInSingleElementArray) {
+  int array[] = {42};
+  const ParamGenerator<int> gen = ValuesIn(array);
+  VerifyGenerator(gen, array);
+}
+
+// Tests that ValuesIn() generates the expected sequence for an STL
+// container (vector).
+TEST(ValuesInTest, ValuesInVector) {
+  typedef ::std::vector<int> ContainerType;
+  ContainerType values;
+  values.push_back(3);
+  values.push_back(5);
+  values.push_back(8);
+  const ParamGenerator<int> gen = ValuesIn(values);
+
+  const int expected_values[] = {3, 5, 8};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that ValuesIn() generates the expected sequence.
+TEST(ValuesInTest, ValuesInIteratorRange) {
+  typedef ::std::vector<int> ContainerType;
+  ContainerType values;
+  values.push_back(3);
+  values.push_back(5);
+  values.push_back(8);
+  const ParamGenerator<int> gen = ValuesIn(values.begin(), values.end());
+
+  const int expected_values[] = {3, 5, 8};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Edge case. Tests that ValuesIn() provided with an iterator range specifying a
+// single value generates a single-element sequence.
+TEST(ValuesInTest, ValuesInSingleElementIteratorRange) {
+  typedef ::std::vector<int> ContainerType;
+  ContainerType values;
+  values.push_back(42);
+  const ParamGenerator<int> gen = ValuesIn(values.begin(), values.end());
+
+  const int expected_values[] = {42};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Edge case. Tests that ValuesIn() provided with an empty iterator range
+// generates an empty sequence.
+TEST(ValuesInTest, ValuesInEmptyIteratorRange) {
+  typedef ::std::vector<int> ContainerType;
+  ContainerType values;
+  const ParamGenerator<int> gen = ValuesIn(values.begin(), values.end());
+
+  VerifyGeneratorIsEmpty(gen);
+}
+
+// Tests that the Values() generates the expected sequence.
+TEST(ValuesTest, ValuesWorks) {
+  const ParamGenerator<int> gen = Values(3, 5, 8);
+
+  const int expected_values[] = {3, 5, 8};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that Values() generates the expected sequences from elements of
+// different types convertible to ParamGenerator's parameter type.
+TEST(ValuesTest, ValuesWorksForValuesOfCompatibleTypes) {
+  const ParamGenerator<double> gen = Values(3, 5.0f, 8.0);
+
+  const double expected_values[] = {3.0, 5.0, 8.0};
+  VerifyGenerator(gen, expected_values);
+}
+
+TEST(ValuesTest, ValuesWorksForMaxLengthList) {
+  const ParamGenerator<int> gen = Values(
+      10, 20, 30, 40, 50, 60, 70, 80, 90, 100,
+      110, 120, 130, 140, 150, 160, 170, 180, 190, 200,
+      210, 220, 230, 240, 250, 260, 270, 280, 290, 300,
+      310, 320, 330, 340, 350, 360, 370, 380, 390, 400,
+      410, 420, 430, 440, 450, 460, 470, 480, 490, 500);
+
+  const int expected_values[] = {
+      10, 20, 30, 40, 50, 60, 70, 80, 90, 100,
+      110, 120, 130, 140, 150, 160, 170, 180, 190, 200,
+      210, 220, 230, 240, 250, 260, 270, 280, 290, 300,
+      310, 320, 330, 340, 350, 360, 370, 380, 390, 400,
+      410, 420, 430, 440, 450, 460, 470, 480, 490, 500};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Edge case test. Tests that single-parameter Values() generates the sequence
+// with the single value.
+TEST(ValuesTest, ValuesWithSingleParameter) {
+  const ParamGenerator<int> gen = Values(42);
+
+  const int expected_values[] = {42};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that Bool() generates sequence (false, true).
+TEST(BoolTest, BoolWorks) {
+  const ParamGenerator<bool> gen = Bool();
+
+  const bool expected_values[] = {false, true};
+  VerifyGenerator(gen, expected_values);
+}
+
+# if GTEST_HAS_COMBINE
+
+// Tests that Combine() with two parameters generates the expected sequence.
+TEST(CombineTest, CombineWithTwoParameters) {
+  const char* foo = "foo";
+  const char* bar = "bar";
+  const ParamGenerator<tuple<const char*, int> > gen =
+      Combine(Values(foo, bar), Values(3, 4));
+
+  tuple<const char*, int> expected_values[] = {
+    make_tuple(foo, 3), make_tuple(foo, 4),
+    make_tuple(bar, 3), make_tuple(bar, 4)};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that Combine() with three parameters generates the expected sequence.
+TEST(CombineTest, CombineWithThreeParameters) {
+  const ParamGenerator<tuple<int, int, int> > gen = Combine(Values(0, 1),
+                                                            Values(3, 4),
+                                                            Values(5, 6));
+  tuple<int, int, int> expected_values[] = {
+    make_tuple(0, 3, 5), make_tuple(0, 3, 6),
+    make_tuple(0, 4, 5), make_tuple(0, 4, 6),
+    make_tuple(1, 3, 5), make_tuple(1, 3, 6),
+    make_tuple(1, 4, 5), make_tuple(1, 4, 6)};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that the Combine() with the first parameter generating a single value
+// sequence generates a sequence with the number of elements equal to the
+// number of elements in the sequence generated by the second parameter.
+TEST(CombineTest, CombineWithFirstParameterSingleValue) {
+  const ParamGenerator<tuple<int, int> > gen = Combine(Values(42),
+                                                       Values(0, 1));
+
+  tuple<int, int> expected_values[] = {make_tuple(42, 0), make_tuple(42, 1)};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that the Combine() with the second parameter generating a single value
+// sequence generates a sequence with the number of elements equal to the
+// number of elements in the sequence generated by the first parameter.
+TEST(CombineTest, CombineWithSecondParameterSingleValue) {
+  const ParamGenerator<tuple<int, int> > gen = Combine(Values(0, 1),
+                                                       Values(42));
+
+  tuple<int, int> expected_values[] = {make_tuple(0, 42), make_tuple(1, 42)};
+  VerifyGenerator(gen, expected_values);
+}
+
+// Tests that when the first parameter produces an empty sequence,
+// Combine() produces an empty sequence, too.
+TEST(CombineTest, CombineWithFirstParameterEmptyRange) {
+  const ParamGenerator<tuple<int, int> > gen = Combine(Range(0, 0),
+                                                       Values(0, 1));
+  VerifyGeneratorIsEmpty(gen);
+}
+
+// Tests that when the second parameter produces an empty sequence,
+// Combine() produces an empty sequence, too.
+TEST(CombineTest, CombineWithSecondParameterEmptyRange) {
+  const ParamGenerator<tuple<int, int> > gen = Combine(Values(0, 1),
+                                                       Range(1, 1));
+  VerifyGeneratorIsEmpty(gen);
+}
+
+// Edge case. Tests that combine works with the maximum number
+// of parameters supported by Google Test (currently 10).
+TEST(CombineTest, CombineWithMaxNumberOfParameters) {
+  const char* foo = "foo";
+  const char* bar = "bar";
+  const ParamGenerator<tuple<const char*, int, int, int, int, int, int, int,
+                             int, int> > gen = Combine(Values(foo, bar),
+                                                       Values(1), Values(2),
+                                                       Values(3), Values(4),
+                                                       Values(5), Values(6),
+                                                       Values(7), Values(8),
+                                                       Values(9));
+
+  tuple<const char*, int, int, int, int, int, int, int, int, int>
+      expected_values[] = {make_tuple(foo, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+                           make_tuple(bar, 1, 2, 3, 4, 5, 6, 7, 8, 9)};
+  VerifyGenerator(gen, expected_values);
+}
+
+# endif  // GTEST_HAS_COMBINE
+
+// Tests that an generator produces correct sequence after being
+// assigned from another generator.
+TEST(ParamGeneratorTest, AssignmentWorks) {
+  ParamGenerator<int> gen = Values(1, 2);
+  const ParamGenerator<int> gen2 = Values(3, 4);
+  gen = gen2;
+
+  const int expected_values[] = {3, 4};
+  VerifyGenerator(gen, expected_values);
+}
+
+// This test verifies that the tests are expanded and run as specified:
+// one test per element from the sequence produced by the generator
+// specified in INSTANTIATE_TEST_CASE_P. It also verifies that the test's
+// fixture constructor, SetUp(), and TearDown() have run and have been
+// supplied with the correct parameters.
+
+// The use of environment object allows detection of the case where no test
+// case functionality is run at all. In this case TestCaseTearDown will not
+// be able to detect missing tests, naturally.
+template <int kExpectedCalls>
+class TestGenerationEnvironment : public ::testing::Environment {
+ public:
+  static TestGenerationEnvironment* Instance() {
+    static TestGenerationEnvironment* instance = new TestGenerationEnvironment;
+    return instance;
+  }
+
+  void FixtureConstructorExecuted() { fixture_constructor_count_++; }
+  void SetUpExecuted() { set_up_count_++; }
+  void TearDownExecuted() { tear_down_count_++; }
+  void TestBodyExecuted() { test_body_count_++; }
+
+  virtual void TearDown() {
+    // If all MultipleTestGenerationTest tests have been de-selected
+    // by the filter flag, the following checks make no sense.
+    bool perform_check = false;
+
+    for (int i = 0; i < kExpectedCalls; ++i) {
+      Message msg;
+      msg << "TestsExpandedAndRun/" << i;
+      if (UnitTestOptions::FilterMatchesTest(
+             "TestExpansionModule/MultipleTestGenerationTest",
+              msg.GetString().c_str())) {
+        perform_check = true;
+      }
+    }
+    if (perform_check) {
+      EXPECT_EQ(kExpectedCalls, fixture_constructor_count_)
+          << "Fixture constructor of ParamTestGenerationTest test case "
+          << "has not been run as expected.";
+      EXPECT_EQ(kExpectedCalls, set_up_count_)
+          << "Fixture SetUp method of ParamTestGenerationTest test case "
+          << "has not been run as expected.";
+      EXPECT_EQ(kExpectedCalls, tear_down_count_)
+          << "Fixture TearDown method of ParamTestGenerationTest test case "
+          << "has not been run as expected.";
+      EXPECT_EQ(kExpectedCalls, test_body_count_)
+          << "Test in ParamTestGenerationTest test case "
+          << "has not been run as expected.";
+    }
+  }
+
+ private:
+  TestGenerationEnvironment() : fixture_constructor_count_(0), set_up_count_(0),
+                                tear_down_count_(0), test_body_count_(0) {}
+
+  int fixture_constructor_count_;
+  int set_up_count_;
+  int tear_down_count_;
+  int test_body_count_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(TestGenerationEnvironment);
+};
+
+const int test_generation_params[] = {36, 42, 72};
+
+class TestGenerationTest : public TestWithParam<int> {
+ public:
+  enum {
+    PARAMETER_COUNT =
+        sizeof(test_generation_params)/sizeof(test_generation_params[0])
+  };
+
+  typedef TestGenerationEnvironment<PARAMETER_COUNT> Environment;
+
+  TestGenerationTest() {
+    Environment::Instance()->FixtureConstructorExecuted();
+    current_parameter_ = GetParam();
+  }
+  virtual void SetUp() {
+    Environment::Instance()->SetUpExecuted();
+    EXPECT_EQ(current_parameter_, GetParam());
+  }
+  virtual void TearDown() {
+    Environment::Instance()->TearDownExecuted();
+    EXPECT_EQ(current_parameter_, GetParam());
+  }
+
+  static void SetUpTestCase() {
+    bool all_tests_in_test_case_selected = true;
+
+    for (int i = 0; i < PARAMETER_COUNT; ++i) {
+      Message test_name;
+      test_name << "TestsExpandedAndRun/" << i;
+      if ( !UnitTestOptions::FilterMatchesTest(
+                "TestExpansionModule/MultipleTestGenerationTest",
+                test_name.GetString())) {
+        all_tests_in_test_case_selected = false;
+      }
+    }
+    EXPECT_TRUE(all_tests_in_test_case_selected)
+        << "When running the TestGenerationTest test case all of its tests\n"
+        << "must be selected by the filter flag for the test case to pass.\n"
+        << "If not all of them are enabled, we can't reliably conclude\n"
+        << "that the correct number of tests have been generated.";
+
+    collected_parameters_.clear();
+  }
+
+  static void TearDownTestCase() {
+    vector<int> expected_values(test_generation_params,
+                                test_generation_params + PARAMETER_COUNT);
+    // Test execution order is not guaranteed by Google Test,
+    // so the order of values in collected_parameters_ can be
+    // different and we have to sort to compare.
+    sort(expected_values.begin(), expected_values.end());
+    sort(collected_parameters_.begin(), collected_parameters_.end());
+
+    EXPECT_TRUE(collected_parameters_ == expected_values);
+  }
+
+ protected:
+  int current_parameter_;
+  static vector<int> collected_parameters_;
+
+ private:
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(TestGenerationTest);
+};
+vector<int> TestGenerationTest::collected_parameters_;
+
+TEST_P(TestGenerationTest, TestsExpandedAndRun) {
+  Environment::Instance()->TestBodyExecuted();
+  EXPECT_EQ(current_parameter_, GetParam());
+  collected_parameters_.push_back(GetParam());
+}
+INSTANTIATE_TEST_CASE_P(TestExpansionModule, TestGenerationTest,
+                        ValuesIn(test_generation_params));
+
+// This test verifies that the element sequence (third parameter of
+// INSTANTIATE_TEST_CASE_P) is evaluated in InitGoogleTest() and neither at
+// the call site of INSTANTIATE_TEST_CASE_P nor in RUN_ALL_TESTS().  For
+// that, we declare param_value_ to be a static member of
+// GeneratorEvaluationTest and initialize it to 0.  We set it to 1 in
+// main(), just before invocation of InitGoogleTest().  After calling
+// InitGoogleTest(), we set the value to 2.  If the sequence is evaluated
+// before or after InitGoogleTest, INSTANTIATE_TEST_CASE_P will create a
+// test with parameter other than 1, and the test body will fail the
+// assertion.
+class GeneratorEvaluationTest : public TestWithParam<int> {
+ public:
+  static int param_value() { return param_value_; }
+  static void set_param_value(int param_value) { param_value_ = param_value; }
+
+ private:
+  static int param_value_;
+};
+int GeneratorEvaluationTest::param_value_ = 0;
+
+TEST_P(GeneratorEvaluationTest, GeneratorsEvaluatedInMain) {
+  EXPECT_EQ(1, GetParam());
+}
+INSTANTIATE_TEST_CASE_P(GenEvalModule,
+                        GeneratorEvaluationTest,
+                        Values(GeneratorEvaluationTest::param_value()));
+
+// Tests that generators defined in a different translation unit are
+// functional. Generator extern_gen is defined in gtest-param-test_test2.cc.
+extern ParamGenerator<int> extern_gen;
+class ExternalGeneratorTest : public TestWithParam<int> {};
+TEST_P(ExternalGeneratorTest, ExternalGenerator) {
+  // Sequence produced by extern_gen contains only a single value
+  // which we verify here.
+  EXPECT_EQ(GetParam(), 33);
+}
+INSTANTIATE_TEST_CASE_P(ExternalGeneratorModule,
+                        ExternalGeneratorTest,
+                        extern_gen);
+
+// Tests that a parameterized test case can be defined in one translation
+// unit and instantiated in another. This test will be instantiated in
+// gtest-param-test_test2.cc. ExternalInstantiationTest fixture class is
+// defined in gtest-param-test_test.h.
+TEST_P(ExternalInstantiationTest, IsMultipleOf33) {
+  EXPECT_EQ(0, GetParam() % 33);
+}
+
+// Tests that a parameterized test case can be instantiated with multiple
+// generators.
+class MultipleInstantiationTest : public TestWithParam<int> {};
+TEST_P(MultipleInstantiationTest, AllowsMultipleInstances) {
+}
+INSTANTIATE_TEST_CASE_P(Sequence1, MultipleInstantiationTest, Values(1, 2));
+INSTANTIATE_TEST_CASE_P(Sequence2, MultipleInstantiationTest, Range(3, 5));
+
+// Tests that a parameterized test case can be instantiated
+// in multiple translation units. This test will be instantiated
+// here and in gtest-param-test_test2.cc.
+// InstantiationInMultipleTranslationUnitsTest fixture class
+// is defined in gtest-param-test_test.h.
+TEST_P(InstantiationInMultipleTranslaionUnitsTest, IsMultipleOf42) {
+  EXPECT_EQ(0, GetParam() % 42);
+}
+INSTANTIATE_TEST_CASE_P(Sequence1,
+                        InstantiationInMultipleTranslaionUnitsTest,
+                        Values(42, 42*2));
+
+// Tests that each iteration of parameterized test runs in a separate test
+// object.
+class SeparateInstanceTest : public TestWithParam<int> {
+ public:
+  SeparateInstanceTest() : count_(0) {}
+
+  static void TearDownTestCase() {
+    EXPECT_GE(global_count_, 2)
+        << "If some (but not all) SeparateInstanceTest tests have been "
+        << "filtered out this test will fail. Make sure that all "
+        << "GeneratorEvaluationTest are selected or de-selected together "
+        << "by the test filter.";
+  }
+
+ protected:
+  int count_;
+  static int global_count_;
+};
+int SeparateInstanceTest::global_count_ = 0;
+
+TEST_P(SeparateInstanceTest, TestsRunInSeparateInstances) {
+  EXPECT_EQ(0, count_++);
+  global_count_++;
+}
+INSTANTIATE_TEST_CASE_P(FourElemSequence, SeparateInstanceTest, Range(1, 4));
+
+// Tests that all instantiations of a test have named appropriately. Test
+// defined with TEST_P(TestCaseName, TestName) and instantiated with
+// INSTANTIATE_TEST_CASE_P(SequenceName, TestCaseName, generator) must be named
+// SequenceName/TestCaseName.TestName/i, where i is the 0-based index of the
+// sequence element used to instantiate the test.
+class NamingTest : public TestWithParam<int> {};
+
+TEST_P(NamingTest, TestsReportCorrectNamesAndParameters) {
+  const ::testing::TestInfo* const test_info =
+     ::testing::UnitTest::GetInstance()->current_test_info();
+
+  EXPECT_STREQ("ZeroToFiveSequence/NamingTest", test_info->test_case_name());
+
+  Message index_stream;
+  index_stream << "TestsReportCorrectNamesAndParameters/" << GetParam();
+  EXPECT_STREQ(index_stream.GetString().c_str(), test_info->name());
+
+  EXPECT_EQ(::testing::PrintToString(GetParam()), test_info->value_param());
+}
+
+INSTANTIATE_TEST_CASE_P(ZeroToFiveSequence, NamingTest, Range(0, 5));
+
+// Tests that user supplied custom parameter names are working correctly.
+// Runs the test with a builtin helper method which uses PrintToString,
+// as well as a custom function and custom functor to ensure all possible
+// uses work correctly.
+class CustomFunctorNamingTest : public TestWithParam<std::string> {};
+TEST_P(CustomFunctorNamingTest, CustomTestNames) {}
+
+struct CustomParamNameFunctor {
+  std::string operator()(const ::testing::TestParamInfo<std::string>& info) {
+    return info.param;
+  }
+};
+
+INSTANTIATE_TEST_CASE_P(CustomParamNameFunctor,
+                        CustomFunctorNamingTest,
+                        Values(std::string("FunctorName")),
+                        CustomParamNameFunctor());
+
+INSTANTIATE_TEST_CASE_P(AllAllowedCharacters,
+                        CustomFunctorNamingTest,
+                        Values("abcdefghijklmnopqrstuvwxyz",
+                               "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+                               "01234567890_"),
+                        CustomParamNameFunctor());
+
+inline std::string CustomParamNameFunction(
+    const ::testing::TestParamInfo<std::string>& info) {
+  return info.param;
+}
+
+class CustomFunctionNamingTest : public TestWithParam<std::string> {};
+TEST_P(CustomFunctionNamingTest, CustomTestNames) {}
+
+INSTANTIATE_TEST_CASE_P(CustomParamNameFunction,
+                        CustomFunctionNamingTest,
+                        Values(std::string("FunctionName")),
+                        CustomParamNameFunction);
+
+#if GTEST_LANG_CXX11
+
+// Test custom naming with a lambda
+
+class CustomLambdaNamingTest : public TestWithParam<std::string> {};
+TEST_P(CustomLambdaNamingTest, CustomTestNames) {}
+
+INSTANTIATE_TEST_CASE_P(CustomParamNameLambda,
+                        CustomLambdaNamingTest,
+                        Values(std::string("LambdaName")),
+                        [](const ::testing::TestParamInfo<std::string>& info) {
+                          return info.param;
+                        });
+
+#endif  // GTEST_LANG_CXX11
+
+TEST(CustomNamingTest, CheckNameRegistry) {
+  ::testing::UnitTest* unit_test = ::testing::UnitTest::GetInstance();
+  std::set<std::string> test_names;
+  for (int case_num = 0;
+       case_num < unit_test->total_test_case_count();
+       ++case_num) {
+    const ::testing::TestCase* test_case = unit_test->GetTestCase(case_num);
+    for (int test_num = 0;
+         test_num < test_case->total_test_count();
+         ++test_num) {
+      const ::testing::TestInfo* test_info = test_case->GetTestInfo(test_num);
+      test_names.insert(std::string(test_info->name()));
+    }
+  }
+  EXPECT_EQ(1u, test_names.count("CustomTestNames/FunctorName"));
+  EXPECT_EQ(1u, test_names.count("CustomTestNames/FunctionName"));
+#if GTEST_LANG_CXX11
+  EXPECT_EQ(1u, test_names.count("CustomTestNames/LambdaName"));
+#endif  // GTEST_LANG_CXX11
+}
+
+// Test a numeric name to ensure PrintToStringParamName works correctly.
+
+class CustomIntegerNamingTest : public TestWithParam<int> {};
+
+TEST_P(CustomIntegerNamingTest, TestsReportCorrectNames) {
+  const ::testing::TestInfo* const test_info =
+     ::testing::UnitTest::GetInstance()->current_test_info();
+  Message test_name_stream;
+  test_name_stream << "TestsReportCorrectNames/" << GetParam();
+  EXPECT_STREQ(test_name_stream.GetString().c_str(), test_info->name());
+}
+
+INSTANTIATE_TEST_CASE_P(PrintToString,
+                        CustomIntegerNamingTest,
+                        Range(0, 5),
+                        ::testing::PrintToStringParamName());
+
+// Test a custom struct with PrintToString.
+
+struct CustomStruct {
+  explicit CustomStruct(int value) : x(value) {}
+  int x;
+};
+
+std::ostream& operator<<(std::ostream& stream, const CustomStruct& val) {
+  stream << val.x;
+  return stream;
+}
+
+class CustomStructNamingTest : public TestWithParam<CustomStruct> {};
+
+TEST_P(CustomStructNamingTest, TestsReportCorrectNames) {
+  const ::testing::TestInfo* const test_info =
+     ::testing::UnitTest::GetInstance()->current_test_info();
+  Message test_name_stream;
+  test_name_stream << "TestsReportCorrectNames/" << GetParam();
+  EXPECT_STREQ(test_name_stream.GetString().c_str(), test_info->name());
+}
+
+INSTANTIATE_TEST_CASE_P(PrintToString,
+                        CustomStructNamingTest,
+                        Values(CustomStruct(0), CustomStruct(1)),
+                        ::testing::PrintToStringParamName());
+
+// Test that using a stateful parameter naming function works as expected.
+
+struct StatefulNamingFunctor {
+  StatefulNamingFunctor() : sum(0) {}
+  std::string operator()(const ::testing::TestParamInfo<int>& info) {
+    int value = info.param + sum;
+    sum += info.param;
+    return ::testing::PrintToString(value);
+  }
+  int sum;
+};
+
+class StatefulNamingTest : public ::testing::TestWithParam<int> {
+ protected:
+  StatefulNamingTest() : sum_(0) {}
+  int sum_;
+};
+
+TEST_P(StatefulNamingTest, TestsReportCorrectNames) {
+  const ::testing::TestInfo* const test_info =
+     ::testing::UnitTest::GetInstance()->current_test_info();
+  sum_ += GetParam();
+  Message test_name_stream;
+  test_name_stream << "TestsReportCorrectNames/" << sum_;
+  EXPECT_STREQ(test_name_stream.GetString().c_str(), test_info->name());
+}
+
+INSTANTIATE_TEST_CASE_P(StatefulNamingFunctor,
+                        StatefulNamingTest,
+                        Range(0, 5),
+                        StatefulNamingFunctor());
+
+// Class that cannot be streamed into an ostream.  It needs to be copyable
+// (and, in case of MSVC, also assignable) in order to be a test parameter
+// type.  Its default copy constructor and assignment operator do exactly
+// what we need.
+class Unstreamable {
+ public:
+  explicit Unstreamable(int value) : value_(value) {}
+
+ private:
+  int value_;
+};
+
+class CommentTest : public TestWithParam<Unstreamable> {};
+
+TEST_P(CommentTest, TestsCorrectlyReportUnstreamableParams) {
+  const ::testing::TestInfo* const test_info =
+     ::testing::UnitTest::GetInstance()->current_test_info();
+
+  EXPECT_EQ(::testing::PrintToString(GetParam()), test_info->value_param());
+}
+
+INSTANTIATE_TEST_CASE_P(InstantiationWithComments,
+                        CommentTest,
+                        Values(Unstreamable(1)));
+
+// Verify that we can create a hierarchy of test fixtures, where the base
+// class fixture is not parameterized and the derived class is. In this case
+// ParameterizedDerivedTest inherits from NonParameterizedBaseTest.  We
+// perform simple tests on both.
+class NonParameterizedBaseTest : public ::testing::Test {
+ public:
+  NonParameterizedBaseTest() : n_(17) { }
+ protected:
+  int n_;
+};
+
+class ParameterizedDerivedTest : public NonParameterizedBaseTest,
+                                 public ::testing::WithParamInterface<int> {
+ protected:
+  ParameterizedDerivedTest() : count_(0) { }
+  int count_;
+  static int global_count_;
+};
+
+int ParameterizedDerivedTest::global_count_ = 0;
+
+TEST_F(NonParameterizedBaseTest, FixtureIsInitialized) {
+  EXPECT_EQ(17, n_);
+}
+
+TEST_P(ParameterizedDerivedTest, SeesSequence) {
+  EXPECT_EQ(17, n_);
+  EXPECT_EQ(0, count_++);
+  EXPECT_EQ(GetParam(), global_count_++);
+}
+
+class ParameterizedDeathTest : public ::testing::TestWithParam<int> { };
+
+TEST_F(ParameterizedDeathTest, GetParamDiesFromTestF) {
+  EXPECT_DEATH_IF_SUPPORTED(GetParam(),
+                            ".* value-parameterized test .*");
+}
+
+INSTANTIATE_TEST_CASE_P(RangeZeroToFive, ParameterizedDerivedTest, Range(0, 5));
+
+#endif  // GTEST_HAS_PARAM_TEST
+
+TEST(CompileTest, CombineIsDefinedOnlyWhenGtestHasParamTestIsDefined) {
+#if GTEST_HAS_COMBINE && !GTEST_HAS_PARAM_TEST
+  FAIL() << "GTEST_HAS_COMBINE is defined while GTEST_HAS_PARAM_TEST is not\n"
+#endif
+}
+
+int main(int argc, char **argv) {
+#if GTEST_HAS_PARAM_TEST
+  // Used in TestGenerationTest test case.
+  AddGlobalTestEnvironment(TestGenerationTest::Environment::Instance());
+  // Used in GeneratorEvaluationTest test case. Tests that the updated value
+  // will be picked up for instantiating tests in GeneratorEvaluationTest.
+  GeneratorEvaluationTest::set_param_value(1);
+#endif  // GTEST_HAS_PARAM_TEST
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+#if GTEST_HAS_PARAM_TEST
+  // Used in GeneratorEvaluationTest test case. Tests that value updated
+  // here will NOT be used for instantiating tests in
+  // GeneratorEvaluationTest.
+  GeneratorEvaluationTest::set_param_value(2);
+#endif  // GTEST_HAS_PARAM_TEST
+
+  return RUN_ALL_TESTS();
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: vladl@google.com (Vlad Losev), wan@google.com (Zhanyong Wan)
+//
+// This file tests the internal cross-platform support utilities.
+
+#include "gtest/internal/gtest-port.h"
+
+#include <stdio.h>
+
+#if GTEST_OS_MAC
+# include <time.h>
+#endif  // GTEST_OS_MAC
+
+#include <list>
+#include <utility>  // For std::pair and std::make_pair.
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "gtest/gtest-spi.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+using std::make_pair;
+using std::pair;
+
+namespace testing {
+namespace internal {
+
+TEST(IsXDigitTest, WorksForNarrowAscii) {
+  EXPECT_TRUE(IsXDigit('0'));
+  EXPECT_TRUE(IsXDigit('9'));
+  EXPECT_TRUE(IsXDigit('A'));
+  EXPECT_TRUE(IsXDigit('F'));
+  EXPECT_TRUE(IsXDigit('a'));
+  EXPECT_TRUE(IsXDigit('f'));
+
+  EXPECT_FALSE(IsXDigit('-'));
+  EXPECT_FALSE(IsXDigit('g'));
+  EXPECT_FALSE(IsXDigit('G'));
+}
+
+TEST(IsXDigitTest, ReturnsFalseForNarrowNonAscii) {
+  EXPECT_FALSE(IsXDigit('\x80'));
+  EXPECT_FALSE(IsXDigit(static_cast<char>('0' | '\x80')));
+}
+
+TEST(IsXDigitTest, WorksForWideAscii) {
+  EXPECT_TRUE(IsXDigit(L'0'));
+  EXPECT_TRUE(IsXDigit(L'9'));
+  EXPECT_TRUE(IsXDigit(L'A'));
+  EXPECT_TRUE(IsXDigit(L'F'));
+  EXPECT_TRUE(IsXDigit(L'a'));
+  EXPECT_TRUE(IsXDigit(L'f'));
+
+  EXPECT_FALSE(IsXDigit(L'-'));
+  EXPECT_FALSE(IsXDigit(L'g'));
+  EXPECT_FALSE(IsXDigit(L'G'));
+}
+
+TEST(IsXDigitTest, ReturnsFalseForWideNonAscii) {
+  EXPECT_FALSE(IsXDigit(static_cast<wchar_t>(0x80)));
+  EXPECT_FALSE(IsXDigit(static_cast<wchar_t>(L'0' | 0x80)));
+  EXPECT_FALSE(IsXDigit(static_cast<wchar_t>(L'0' | 0x100)));
+}
+
+class Base {
+ public:
+  // Copy constructor and assignment operator do exactly what we need, so we
+  // use them.
+  Base() : member_(0) {}
+  explicit Base(int n) : member_(n) {}
+  virtual ~Base() {}
+  int member() { return member_; }
+
+ private:
+  int member_;
+};
+
+class Derived : public Base {
+ public:
+  explicit Derived(int n) : Base(n) {}
+};
+
+TEST(ImplicitCastTest, ConvertsPointers) {
+  Derived derived(0);
+  EXPECT_TRUE(&derived == ::testing::internal::ImplicitCast_<Base*>(&derived));
+}
+
+TEST(ImplicitCastTest, CanUseInheritance) {
+  Derived derived(1);
+  Base base = ::testing::internal::ImplicitCast_<Base>(derived);
+  EXPECT_EQ(derived.member(), base.member());
+}
+
+class Castable {
+ public:
+  explicit Castable(bool* converted) : converted_(converted) {}
+  operator Base() {
+    *converted_ = true;
+    return Base();
+  }
+
+ private:
+  bool* converted_;
+};
+
+TEST(ImplicitCastTest, CanUseNonConstCastOperator) {
+  bool converted = false;
+  Castable castable(&converted);
+  Base base = ::testing::internal::ImplicitCast_<Base>(castable);
+  EXPECT_TRUE(converted);
+}
+
+class ConstCastable {
+ public:
+  explicit ConstCastable(bool* converted) : converted_(converted) {}
+  operator Base() const {
+    *converted_ = true;
+    return Base();
+  }
+
+ private:
+  bool* converted_;
+};
+
+TEST(ImplicitCastTest, CanUseConstCastOperatorOnConstValues) {
+  bool converted = false;
+  const ConstCastable const_castable(&converted);
+  Base base = ::testing::internal::ImplicitCast_<Base>(const_castable);
+  EXPECT_TRUE(converted);
+}
+
+class ConstAndNonConstCastable {
+ public:
+  ConstAndNonConstCastable(bool* converted, bool* const_converted)
+      : converted_(converted), const_converted_(const_converted) {}
+  operator Base() {
+    *converted_ = true;
+    return Base();
+  }
+  operator Base() const {
+    *const_converted_ = true;
+    return Base();
+  }
+
+ private:
+  bool* converted_;
+  bool* const_converted_;
+};
+
+TEST(ImplicitCastTest, CanSelectBetweenConstAndNonConstCasrAppropriately) {
+  bool converted = false;
+  bool const_converted = false;
+  ConstAndNonConstCastable castable(&converted, &const_converted);
+  Base base = ::testing::internal::ImplicitCast_<Base>(castable);
+  EXPECT_TRUE(converted);
+  EXPECT_FALSE(const_converted);
+
+  converted = false;
+  const_converted = false;
+  const ConstAndNonConstCastable const_castable(&converted, &const_converted);
+  base = ::testing::internal::ImplicitCast_<Base>(const_castable);
+  EXPECT_FALSE(converted);
+  EXPECT_TRUE(const_converted);
+}
+
+class To {
+ public:
+  To(bool* converted) { *converted = true; }  // NOLINT
+};
+
+TEST(ImplicitCastTest, CanUseImplicitConstructor) {
+  bool converted = false;
+  To to = ::testing::internal::ImplicitCast_<To>(&converted);
+  (void)to;
+  EXPECT_TRUE(converted);
+}
+
+TEST(IteratorTraitsTest, WorksForSTLContainerIterators) {
+  StaticAssertTypeEq<int,
+      IteratorTraits< ::std::vector<int>::const_iterator>::value_type>();
+  StaticAssertTypeEq<bool,
+      IteratorTraits< ::std::list<bool>::iterator>::value_type>();
+}
+
+TEST(IteratorTraitsTest, WorksForPointerToNonConst) {
+  StaticAssertTypeEq<char, IteratorTraits<char*>::value_type>();
+  StaticAssertTypeEq<const void*, IteratorTraits<const void**>::value_type>();
+}
+
+TEST(IteratorTraitsTest, WorksForPointerToConst) {
+  StaticAssertTypeEq<char, IteratorTraits<const char*>::value_type>();
+  StaticAssertTypeEq<const void*,
+      IteratorTraits<const void* const*>::value_type>();
+}
+
+// Tests that the element_type typedef is available in scoped_ptr and refers
+// to the parameter type.
+TEST(ScopedPtrTest, DefinesElementType) {
+  StaticAssertTypeEq<int, ::testing::internal::scoped_ptr<int>::element_type>();
+}
+
+// TODO(vladl@google.com): Implement THE REST of scoped_ptr tests.
+
+TEST(GtestCheckSyntaxTest, BehavesLikeASingleStatement) {
+  if (AlwaysFalse())
+    GTEST_CHECK_(false) << "This should never be executed; "
+                           "It's a compilation test only.";
+
+  if (AlwaysTrue())
+    GTEST_CHECK_(true);
+  else
+    ;  // NOLINT
+
+  if (AlwaysFalse())
+    ;  // NOLINT
+  else
+    GTEST_CHECK_(true) << "";
+}
+
+TEST(GtestCheckSyntaxTest, WorksWithSwitch) {
+  switch (0) {
+    case 1:
+      break;
+    default:
+      GTEST_CHECK_(true);
+  }
+
+  switch (0)
+    case 0:
+      GTEST_CHECK_(true) << "Check failed in switch case";
+}
+
+// Verifies behavior of FormatFileLocation.
+TEST(FormatFileLocationTest, FormatsFileLocation) {
+  EXPECT_PRED_FORMAT2(IsSubstring, "foo.cc", FormatFileLocation("foo.cc", 42));
+  EXPECT_PRED_FORMAT2(IsSubstring, "42", FormatFileLocation("foo.cc", 42));
+}
+
+TEST(FormatFileLocationTest, FormatsUnknownFile) {
+  EXPECT_PRED_FORMAT2(
+      IsSubstring, "unknown file", FormatFileLocation(NULL, 42));
+  EXPECT_PRED_FORMAT2(IsSubstring, "42", FormatFileLocation(NULL, 42));
+}
+
+TEST(FormatFileLocationTest, FormatsUknownLine) {
+  EXPECT_EQ("foo.cc:", FormatFileLocation("foo.cc", -1));
+}
+
+TEST(FormatFileLocationTest, FormatsUknownFileAndLine) {
+  EXPECT_EQ("unknown file:", FormatFileLocation(NULL, -1));
+}
+
+// Verifies behavior of FormatCompilerIndependentFileLocation.
+TEST(FormatCompilerIndependentFileLocationTest, FormatsFileLocation) {
+  EXPECT_EQ("foo.cc:42", FormatCompilerIndependentFileLocation("foo.cc", 42));
+}
+
+TEST(FormatCompilerIndependentFileLocationTest, FormatsUknownFile) {
+  EXPECT_EQ("unknown file:42",
+            FormatCompilerIndependentFileLocation(NULL, 42));
+}
+
+TEST(FormatCompilerIndependentFileLocationTest, FormatsUknownLine) {
+  EXPECT_EQ("foo.cc", FormatCompilerIndependentFileLocation("foo.cc", -1));
+}
+
+TEST(FormatCompilerIndependentFileLocationTest, FormatsUknownFileAndLine) {
+  EXPECT_EQ("unknown file", FormatCompilerIndependentFileLocation(NULL, -1));
+}
+
+#if GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_QNX
+void* ThreadFunc(void* data) {
+  internal::Mutex* mutex = static_cast<internal::Mutex*>(data);
+  mutex->Lock();
+  mutex->Unlock();
+  return NULL;
+}
+
+TEST(GetThreadCountTest, ReturnsCorrectValue) {
+  const size_t starting_count = GetThreadCount();
+  pthread_t       thread_id;
+
+  internal::Mutex mutex;
+  {
+    internal::MutexLock lock(&mutex);
+    pthread_attr_t  attr;
+    ASSERT_EQ(0, pthread_attr_init(&attr));
+    ASSERT_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE));
+
+    const int status = pthread_create(&thread_id, &attr, &ThreadFunc, &mutex);
+    ASSERT_EQ(0, pthread_attr_destroy(&attr));
+    ASSERT_EQ(0, status);
+    EXPECT_EQ(starting_count + 1, GetThreadCount());
+  }
+
+  void* dummy;
+  ASSERT_EQ(0, pthread_join(thread_id, &dummy));
+
+  // The OS may not immediately report the updated thread count after
+  // joining a thread, causing flakiness in this test. To counter that, we
+  // wait for up to .5 seconds for the OS to report the correct value.
+  for (int i = 0; i < 5; ++i) {
+    if (GetThreadCount() == starting_count)
+      break;
+
+    SleepMilliseconds(100);
+  }
+
+  EXPECT_EQ(starting_count, GetThreadCount());
+}
+#else
+TEST(GetThreadCountTest, ReturnsZeroWhenUnableToCountThreads) {
+  EXPECT_EQ(0U, GetThreadCount());
+}
+#endif  // GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_QNX
+
+TEST(GtestCheckDeathTest, DiesWithCorrectOutputOnFailure) {
+  const bool a_false_condition = false;
+  const char regex[] =
+#ifdef _MSC_VER
+     "gtest-port_test\\.cc\\(\\d+\\):"
+#elif GTEST_USES_POSIX_RE
+     "gtest-port_test\\.cc:[0-9]+"
+#else
+     "gtest-port_test\\.cc:\\d+"
+#endif  // _MSC_VER
+     ".*a_false_condition.*Extra info.*";
+
+  EXPECT_DEATH_IF_SUPPORTED(GTEST_CHECK_(a_false_condition) << "Extra info",
+                            regex);
+}
+
+#if GTEST_HAS_DEATH_TEST
+
+TEST(GtestCheckDeathTest, LivesSilentlyOnSuccess) {
+  EXPECT_EXIT({
+      GTEST_CHECK_(true) << "Extra info";
+      ::std::cerr << "Success\n";
+      exit(0); },
+      ::testing::ExitedWithCode(0), "Success");
+}
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+// Verifies that Google Test choose regular expression engine appropriate to
+// the platform. The test will produce compiler errors in case of failure.
+// For simplicity, we only cover the most important platforms here.
+TEST(RegexEngineSelectionTest, SelectsCorrectRegexEngine) {
+#if !GTEST_USES_PCRE
+# if GTEST_HAS_POSIX_RE
+
+  EXPECT_TRUE(GTEST_USES_POSIX_RE);
+
+# else
+
+  EXPECT_TRUE(GTEST_USES_SIMPLE_RE);
+
+# endif
+#endif  // !GTEST_USES_PCRE
+}
+
+#if GTEST_USES_POSIX_RE
+
+# if GTEST_HAS_TYPED_TEST
+
+template <typename Str>
+class RETest : public ::testing::Test {};
+
+// Defines StringTypes as the list of all string types that class RE
+// supports.
+typedef testing::Types<
+    ::std::string,
+#  if GTEST_HAS_GLOBAL_STRING
+    ::string,
+#  endif  // GTEST_HAS_GLOBAL_STRING
+    const char*> StringTypes;
+
+TYPED_TEST_CASE(RETest, StringTypes);
+
+// Tests RE's implicit constructors.
+TYPED_TEST(RETest, ImplicitConstructorWorks) {
+  const RE empty(TypeParam(""));
+  EXPECT_STREQ("", empty.pattern());
+
+  const RE simple(TypeParam("hello"));
+  EXPECT_STREQ("hello", simple.pattern());
+
+  const RE normal(TypeParam(".*(\\w+)"));
+  EXPECT_STREQ(".*(\\w+)", normal.pattern());
+}
+
+// Tests that RE's constructors reject invalid regular expressions.
+TYPED_TEST(RETest, RejectsInvalidRegex) {
+  EXPECT_NONFATAL_FAILURE({
+    const RE invalid(TypeParam("?"));
+  }, "\"?\" is not a valid POSIX Extended regular expression.");
+}
+
+// Tests RE::FullMatch().
+TYPED_TEST(RETest, FullMatchWorks) {
+  const RE empty(TypeParam(""));
+  EXPECT_TRUE(RE::FullMatch(TypeParam(""), empty));
+  EXPECT_FALSE(RE::FullMatch(TypeParam("a"), empty));
+
+  const RE re(TypeParam("a.*z"));
+  EXPECT_TRUE(RE::FullMatch(TypeParam("az"), re));
+  EXPECT_TRUE(RE::FullMatch(TypeParam("axyz"), re));
+  EXPECT_FALSE(RE::FullMatch(TypeParam("baz"), re));
+  EXPECT_FALSE(RE::FullMatch(TypeParam("azy"), re));
+}
+
+// Tests RE::PartialMatch().
+TYPED_TEST(RETest, PartialMatchWorks) {
+  const RE empty(TypeParam(""));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam(""), empty));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam("a"), empty));
+
+  const RE re(TypeParam("a.*z"));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam("az"), re));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam("axyz"), re));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam("baz"), re));
+  EXPECT_TRUE(RE::PartialMatch(TypeParam("azy"), re));
+  EXPECT_FALSE(RE::PartialMatch(TypeParam("zza"), re));
+}
+
+# endif  // GTEST_HAS_TYPED_TEST
+
+#elif GTEST_USES_SIMPLE_RE
+
+TEST(IsInSetTest, NulCharIsNotInAnySet) {
+  EXPECT_FALSE(IsInSet('\0', ""));
+  EXPECT_FALSE(IsInSet('\0', "\0"));
+  EXPECT_FALSE(IsInSet('\0', "a"));
+}
+
+TEST(IsInSetTest, WorksForNonNulChars) {
+  EXPECT_FALSE(IsInSet('a', "Ab"));
+  EXPECT_FALSE(IsInSet('c', ""));
+
+  EXPECT_TRUE(IsInSet('b', "bcd"));
+  EXPECT_TRUE(IsInSet('b', "ab"));
+}
+
+TEST(IsAsciiDigitTest, IsFalseForNonDigit) {
+  EXPECT_FALSE(IsAsciiDigit('\0'));
+  EXPECT_FALSE(IsAsciiDigit(' '));
+  EXPECT_FALSE(IsAsciiDigit('+'));
+  EXPECT_FALSE(IsAsciiDigit('-'));
+  EXPECT_FALSE(IsAsciiDigit('.'));
+  EXPECT_FALSE(IsAsciiDigit('a'));
+}
+
+TEST(IsAsciiDigitTest, IsTrueForDigit) {
+  EXPECT_TRUE(IsAsciiDigit('0'));
+  EXPECT_TRUE(IsAsciiDigit('1'));
+  EXPECT_TRUE(IsAsciiDigit('5'));
+  EXPECT_TRUE(IsAsciiDigit('9'));
+}
+
+TEST(IsAsciiPunctTest, IsFalseForNonPunct) {
+  EXPECT_FALSE(IsAsciiPunct('\0'));
+  EXPECT_FALSE(IsAsciiPunct(' '));
+  EXPECT_FALSE(IsAsciiPunct('\n'));
+  EXPECT_FALSE(IsAsciiPunct('a'));
+  EXPECT_FALSE(IsAsciiPunct('0'));
+}
+
+TEST(IsAsciiPunctTest, IsTrueForPunct) {
+  for (const char* p = "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"; *p; p++) {
+    EXPECT_PRED1(IsAsciiPunct, *p);
+  }
+}
+
+TEST(IsRepeatTest, IsFalseForNonRepeatChar) {
+  EXPECT_FALSE(IsRepeat('\0'));
+  EXPECT_FALSE(IsRepeat(' '));
+  EXPECT_FALSE(IsRepeat('a'));
+  EXPECT_FALSE(IsRepeat('1'));
+  EXPECT_FALSE(IsRepeat('-'));
+}
+
+TEST(IsRepeatTest, IsTrueForRepeatChar) {
+  EXPECT_TRUE(IsRepeat('?'));
+  EXPECT_TRUE(IsRepeat('*'));
+  EXPECT_TRUE(IsRepeat('+'));
+}
+
+TEST(IsAsciiWhiteSpaceTest, IsFalseForNonWhiteSpace) {
+  EXPECT_FALSE(IsAsciiWhiteSpace('\0'));
+  EXPECT_FALSE(IsAsciiWhiteSpace('a'));
+  EXPECT_FALSE(IsAsciiWhiteSpace('1'));
+  EXPECT_FALSE(IsAsciiWhiteSpace('+'));
+  EXPECT_FALSE(IsAsciiWhiteSpace('_'));
+}
+
+TEST(IsAsciiWhiteSpaceTest, IsTrueForWhiteSpace) {
+  EXPECT_TRUE(IsAsciiWhiteSpace(' '));
+  EXPECT_TRUE(IsAsciiWhiteSpace('\n'));
+  EXPECT_TRUE(IsAsciiWhiteSpace('\r'));
+  EXPECT_TRUE(IsAsciiWhiteSpace('\t'));
+  EXPECT_TRUE(IsAsciiWhiteSpace('\v'));
+  EXPECT_TRUE(IsAsciiWhiteSpace('\f'));
+}
+
+TEST(IsAsciiWordCharTest, IsFalseForNonWordChar) {
+  EXPECT_FALSE(IsAsciiWordChar('\0'));
+  EXPECT_FALSE(IsAsciiWordChar('+'));
+  EXPECT_FALSE(IsAsciiWordChar('.'));
+  EXPECT_FALSE(IsAsciiWordChar(' '));
+  EXPECT_FALSE(IsAsciiWordChar('\n'));
+}
+
+TEST(IsAsciiWordCharTest, IsTrueForLetter) {
+  EXPECT_TRUE(IsAsciiWordChar('a'));
+  EXPECT_TRUE(IsAsciiWordChar('b'));
+  EXPECT_TRUE(IsAsciiWordChar('A'));
+  EXPECT_TRUE(IsAsciiWordChar('Z'));
+}
+
+TEST(IsAsciiWordCharTest, IsTrueForDigit) {
+  EXPECT_TRUE(IsAsciiWordChar('0'));
+  EXPECT_TRUE(IsAsciiWordChar('1'));
+  EXPECT_TRUE(IsAsciiWordChar('7'));
+  EXPECT_TRUE(IsAsciiWordChar('9'));
+}
+
+TEST(IsAsciiWordCharTest, IsTrueForUnderscore) {
+  EXPECT_TRUE(IsAsciiWordChar('_'));
+}
+
+TEST(IsValidEscapeTest, IsFalseForNonPrintable) {
+  EXPECT_FALSE(IsValidEscape('\0'));
+  EXPECT_FALSE(IsValidEscape('\007'));
+}
+
+TEST(IsValidEscapeTest, IsFalseForDigit) {
+  EXPECT_FALSE(IsValidEscape('0'));
+  EXPECT_FALSE(IsValidEscape('9'));
+}
+
+TEST(IsValidEscapeTest, IsFalseForWhiteSpace) {
+  EXPECT_FALSE(IsValidEscape(' '));
+  EXPECT_FALSE(IsValidEscape('\n'));
+}
+
+TEST(IsValidEscapeTest, IsFalseForSomeLetter) {
+  EXPECT_FALSE(IsValidEscape('a'));
+  EXPECT_FALSE(IsValidEscape('Z'));
+}
+
+TEST(IsValidEscapeTest, IsTrueForPunct) {
+  EXPECT_TRUE(IsValidEscape('.'));
+  EXPECT_TRUE(IsValidEscape('-'));
+  EXPECT_TRUE(IsValidEscape('^'));
+  EXPECT_TRUE(IsValidEscape('$'));
+  EXPECT_TRUE(IsValidEscape('('));
+  EXPECT_TRUE(IsValidEscape(']'));
+  EXPECT_TRUE(IsValidEscape('{'));
+  EXPECT_TRUE(IsValidEscape('|'));
+}
+
+TEST(IsValidEscapeTest, IsTrueForSomeLetter) {
+  EXPECT_TRUE(IsValidEscape('d'));
+  EXPECT_TRUE(IsValidEscape('D'));
+  EXPECT_TRUE(IsValidEscape('s'));
+  EXPECT_TRUE(IsValidEscape('S'));
+  EXPECT_TRUE(IsValidEscape('w'));
+  EXPECT_TRUE(IsValidEscape('W'));
+}
+
+TEST(AtomMatchesCharTest, EscapedPunct) {
+  EXPECT_FALSE(AtomMatchesChar(true, '\\', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, '\\', ' '));
+  EXPECT_FALSE(AtomMatchesChar(true, '_', '.'));
+  EXPECT_FALSE(AtomMatchesChar(true, '.', 'a'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, '\\', '\\'));
+  EXPECT_TRUE(AtomMatchesChar(true, '_', '_'));
+  EXPECT_TRUE(AtomMatchesChar(true, '+', '+'));
+  EXPECT_TRUE(AtomMatchesChar(true, '.', '.'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_d) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'd', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'd', 'a'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'd', '.'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'd', '0'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'd', '9'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_D) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'D', '0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'D', '9'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'D', '\0'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'D', 'a'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'D', '-'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_s) {
+  EXPECT_FALSE(AtomMatchesChar(true, 's', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 's', 'a'));
+  EXPECT_FALSE(AtomMatchesChar(true, 's', '.'));
+  EXPECT_FALSE(AtomMatchesChar(true, 's', '9'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 's', ' '));
+  EXPECT_TRUE(AtomMatchesChar(true, 's', '\n'));
+  EXPECT_TRUE(AtomMatchesChar(true, 's', '\t'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_S) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'S', ' '));
+  EXPECT_FALSE(AtomMatchesChar(true, 'S', '\r'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'S', '\0'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'S', 'a'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'S', '9'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_w) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'w', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'w', '+'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'w', ' '));
+  EXPECT_FALSE(AtomMatchesChar(true, 'w', '\n'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'w', '0'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'w', 'b'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'w', 'C'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'w', '_'));
+}
+
+TEST(AtomMatchesCharTest, Escaped_W) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'W', 'A'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'W', 'b'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'W', '9'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'W', '_'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'W', '\0'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'W', '*'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'W', '\n'));
+}
+
+TEST(AtomMatchesCharTest, EscapedWhiteSpace) {
+  EXPECT_FALSE(AtomMatchesChar(true, 'f', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'f', '\n'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'n', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'n', '\r'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'r', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'r', 'a'));
+  EXPECT_FALSE(AtomMatchesChar(true, 't', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 't', 't'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'v', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(true, 'v', '\f'));
+
+  EXPECT_TRUE(AtomMatchesChar(true, 'f', '\f'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'n', '\n'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'r', '\r'));
+  EXPECT_TRUE(AtomMatchesChar(true, 't', '\t'));
+  EXPECT_TRUE(AtomMatchesChar(true, 'v', '\v'));
+}
+
+TEST(AtomMatchesCharTest, UnescapedDot) {
+  EXPECT_FALSE(AtomMatchesChar(false, '.', '\n'));
+
+  EXPECT_TRUE(AtomMatchesChar(false, '.', '\0'));
+  EXPECT_TRUE(AtomMatchesChar(false, '.', '.'));
+  EXPECT_TRUE(AtomMatchesChar(false, '.', 'a'));
+  EXPECT_TRUE(AtomMatchesChar(false, '.', ' '));
+}
+
+TEST(AtomMatchesCharTest, UnescapedChar) {
+  EXPECT_FALSE(AtomMatchesChar(false, 'a', '\0'));
+  EXPECT_FALSE(AtomMatchesChar(false, 'a', 'b'));
+  EXPECT_FALSE(AtomMatchesChar(false, '$', 'a'));
+
+  EXPECT_TRUE(AtomMatchesChar(false, '$', '$'));
+  EXPECT_TRUE(AtomMatchesChar(false, '5', '5'));
+  EXPECT_TRUE(AtomMatchesChar(false, 'Z', 'Z'));
+}
+
+TEST(ValidateRegexTest, GeneratesFailureAndReturnsFalseForInvalid) {
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex(NULL)),
+                          "NULL is not a valid simple regular expression");
+  EXPECT_NONFATAL_FAILURE(
+      ASSERT_FALSE(ValidateRegex("a\\")),
+      "Syntax error at index 1 in simple regular expression \"a\\\": ");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("a\\")),
+                          "'\\' cannot appear at the end");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("\\n\\")),
+                          "'\\' cannot appear at the end");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("\\s\\hb")),
+                          "invalid escape sequence \"\\h\"");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("^^")),
+                          "'^' can only appear at the beginning");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex(".*^b")),
+                          "'^' can only appear at the beginning");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("$$")),
+                          "'$' can only appear at the end");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("^$a")),
+                          "'$' can only appear at the end");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("a(b")),
+                          "'(' is unsupported");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("ab)")),
+                          "')' is unsupported");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("[ab")),
+                          "'[' is unsupported");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("a{2")),
+                          "'{' is unsupported");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("?")),
+                          "'?' can only follow a repeatable token");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("^*")),
+                          "'*' can only follow a repeatable token");
+  EXPECT_NONFATAL_FAILURE(ASSERT_FALSE(ValidateRegex("5*+")),
+                          "'+' can only follow a repeatable token");
+}
+
+TEST(ValidateRegexTest, ReturnsTrueForValid) {
+  EXPECT_TRUE(ValidateRegex(""));
+  EXPECT_TRUE(ValidateRegex("a"));
+  EXPECT_TRUE(ValidateRegex(".*"));
+  EXPECT_TRUE(ValidateRegex("^a_+"));
+  EXPECT_TRUE(ValidateRegex("^a\\t\\&?"));
+  EXPECT_TRUE(ValidateRegex("09*$"));
+  EXPECT_TRUE(ValidateRegex("^Z$"));
+  EXPECT_TRUE(ValidateRegex("a\\^Z\\$\\(\\)\\|\\[\\]\\{\\}"));
+}
+
+TEST(MatchRepetitionAndRegexAtHeadTest, WorksForZeroOrOne) {
+  EXPECT_FALSE(MatchRepetitionAndRegexAtHead(false, 'a', '?', "a", "ba"));
+  // Repeating more than once.
+  EXPECT_FALSE(MatchRepetitionAndRegexAtHead(false, 'a', '?', "b", "aab"));
+
+  // Repeating zero times.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, 'a', '?', "b", "ba"));
+  // Repeating once.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, 'a', '?', "b", "ab"));
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, '#', '?', ".", "##"));
+}
+
+TEST(MatchRepetitionAndRegexAtHeadTest, WorksForZeroOrMany) {
+  EXPECT_FALSE(MatchRepetitionAndRegexAtHead(false, '.', '*', "a$", "baab"));
+
+  // Repeating zero times.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, '.', '*', "b", "bc"));
+  // Repeating once.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, '.', '*', "b", "abc"));
+  // Repeating more than once.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(true, 'w', '*', "-", "ab_1-g"));
+}
+
+TEST(MatchRepetitionAndRegexAtHeadTest, WorksForOneOrMany) {
+  EXPECT_FALSE(MatchRepetitionAndRegexAtHead(false, '.', '+', "a$", "baab"));
+  // Repeating zero times.
+  EXPECT_FALSE(MatchRepetitionAndRegexAtHead(false, '.', '+', "b", "bc"));
+
+  // Repeating once.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(false, '.', '+', "b", "abc"));
+  // Repeating more than once.
+  EXPECT_TRUE(MatchRepetitionAndRegexAtHead(true, 'w', '+', "-", "ab_1-g"));
+}
+
+TEST(MatchRegexAtHeadTest, ReturnsTrueForEmptyRegex) {
+  EXPECT_TRUE(MatchRegexAtHead("", ""));
+  EXPECT_TRUE(MatchRegexAtHead("", "ab"));
+}
+
+TEST(MatchRegexAtHeadTest, WorksWhenDollarIsInRegex) {
+  EXPECT_FALSE(MatchRegexAtHead("$", "a"));
+
+  EXPECT_TRUE(MatchRegexAtHead("$", ""));
+  EXPECT_TRUE(MatchRegexAtHead("a$", "a"));
+}
+
+TEST(MatchRegexAtHeadTest, WorksWhenRegexStartsWithEscapeSequence) {
+  EXPECT_FALSE(MatchRegexAtHead("\\w", "+"));
+  EXPECT_FALSE(MatchRegexAtHead("\\W", "ab"));
+
+  EXPECT_TRUE(MatchRegexAtHead("\\sa", "\nab"));
+  EXPECT_TRUE(MatchRegexAtHead("\\d", "1a"));
+}
+
+TEST(MatchRegexAtHeadTest, WorksWhenRegexStartsWithRepetition) {
+  EXPECT_FALSE(MatchRegexAtHead(".+a", "abc"));
+  EXPECT_FALSE(MatchRegexAtHead("a?b", "aab"));
+
+  EXPECT_TRUE(MatchRegexAtHead(".*a", "bc12-ab"));
+  EXPECT_TRUE(MatchRegexAtHead("a?b", "b"));
+  EXPECT_TRUE(MatchRegexAtHead("a?b", "ab"));
+}
+
+TEST(MatchRegexAtHeadTest,
+     WorksWhenRegexStartsWithRepetionOfEscapeSequence) {
+  EXPECT_FALSE(MatchRegexAtHead("\\.+a", "abc"));
+  EXPECT_FALSE(MatchRegexAtHead("\\s?b", "  b"));
+
+  EXPECT_TRUE(MatchRegexAtHead("\\(*a", "((((ab"));
+  EXPECT_TRUE(MatchRegexAtHead("\\^?b", "^b"));
+  EXPECT_TRUE(MatchRegexAtHead("\\\\?b", "b"));
+  EXPECT_TRUE(MatchRegexAtHead("\\\\?b", "\\b"));
+}
+
+TEST(MatchRegexAtHeadTest, MatchesSequentially) {
+  EXPECT_FALSE(MatchRegexAtHead("ab.*c", "acabc"));
+
+  EXPECT_TRUE(MatchRegexAtHead("ab.*c", "ab-fsc"));
+}
+
+TEST(MatchRegexAnywhereTest, ReturnsFalseWhenStringIsNull) {
+  EXPECT_FALSE(MatchRegexAnywhere("", NULL));
+}
+
+TEST(MatchRegexAnywhereTest, WorksWhenRegexStartsWithCaret) {
+  EXPECT_FALSE(MatchRegexAnywhere("^a", "ba"));
+  EXPECT_FALSE(MatchRegexAnywhere("^$", "a"));
+
+  EXPECT_TRUE(MatchRegexAnywhere("^a", "ab"));
+  EXPECT_TRUE(MatchRegexAnywhere("^", "ab"));
+  EXPECT_TRUE(MatchRegexAnywhere("^$", ""));
+}
+
+TEST(MatchRegexAnywhereTest, ReturnsFalseWhenNoMatch) {
+  EXPECT_FALSE(MatchRegexAnywhere("a", "bcde123"));
+  EXPECT_FALSE(MatchRegexAnywhere("a.+a", "--aa88888888"));
+}
+
+TEST(MatchRegexAnywhereTest, ReturnsTrueWhenMatchingPrefix) {
+  EXPECT_TRUE(MatchRegexAnywhere("\\w+", "ab1_ - 5"));
+  EXPECT_TRUE(MatchRegexAnywhere(".*=", "="));
+  EXPECT_TRUE(MatchRegexAnywhere("x.*ab?.*bc", "xaaabc"));
+}
+
+TEST(MatchRegexAnywhereTest, ReturnsTrueWhenMatchingNonPrefix) {
+  EXPECT_TRUE(MatchRegexAnywhere("\\w+", "$$$ ab1_ - 5"));
+  EXPECT_TRUE(MatchRegexAnywhere("\\.+=", "=  ...="));
+}
+
+// Tests RE's implicit constructors.
+TEST(RETest, ImplicitConstructorWorks) {
+  const RE empty("");
+  EXPECT_STREQ("", empty.pattern());
+
+  const RE simple("hello");
+  EXPECT_STREQ("hello", simple.pattern());
+}
+
+// Tests that RE's constructors reject invalid regular expressions.
+TEST(RETest, RejectsInvalidRegex) {
+  EXPECT_NONFATAL_FAILURE({
+    const RE normal(NULL);
+  }, "NULL is not a valid simple regular expression");
+
+  EXPECT_NONFATAL_FAILURE({
+    const RE normal(".*(\\w+");
+  }, "'(' is unsupported");
+
+  EXPECT_NONFATAL_FAILURE({
+    const RE invalid("^?");
+  }, "'?' can only follow a repeatable token");
+}
+
+// Tests RE::FullMatch().
+TEST(RETest, FullMatchWorks) {
+  const RE empty("");
+  EXPECT_TRUE(RE::FullMatch("", empty));
+  EXPECT_FALSE(RE::FullMatch("a", empty));
+
+  const RE re1("a");
+  EXPECT_TRUE(RE::FullMatch("a", re1));
+
+  const RE re("a.*z");
+  EXPECT_TRUE(RE::FullMatch("az", re));
+  EXPECT_TRUE(RE::FullMatch("axyz", re));
+  EXPECT_FALSE(RE::FullMatch("baz", re));
+  EXPECT_FALSE(RE::FullMatch("azy", re));
+}
+
+// Tests RE::PartialMatch().
+TEST(RETest, PartialMatchWorks) {
+  const RE empty("");
+  EXPECT_TRUE(RE::PartialMatch("", empty));
+  EXPECT_TRUE(RE::PartialMatch("a", empty));
+
+  const RE re("a.*z");
+  EXPECT_TRUE(RE::PartialMatch("az", re));
+  EXPECT_TRUE(RE::PartialMatch("axyz", re));
+  EXPECT_TRUE(RE::PartialMatch("baz", re));
+  EXPECT_TRUE(RE::PartialMatch("azy", re));
+  EXPECT_FALSE(RE::PartialMatch("zza", re));
+}
+
+#endif  // GTEST_USES_POSIX_RE
+
+#if !GTEST_OS_WINDOWS_MOBILE
+
+TEST(CaptureTest, CapturesStdout) {
+  CaptureStdout();
+  fprintf(stdout, "abc");
+  EXPECT_STREQ("abc", GetCapturedStdout().c_str());
+
+  CaptureStdout();
+  fprintf(stdout, "def%cghi", '\0');
+  EXPECT_EQ(::std::string("def\0ghi", 7), ::std::string(GetCapturedStdout()));
+}
+
+TEST(CaptureTest, CapturesStderr) {
+  CaptureStderr();
+  fprintf(stderr, "jkl");
+  EXPECT_STREQ("jkl", GetCapturedStderr().c_str());
+
+  CaptureStderr();
+  fprintf(stderr, "jkl%cmno", '\0');
+  EXPECT_EQ(::std::string("jkl\0mno", 7), ::std::string(GetCapturedStderr()));
+}
+
+// Tests that stdout and stderr capture don't interfere with each other.
+TEST(CaptureTest, CapturesStdoutAndStderr) {
+  CaptureStdout();
+  CaptureStderr();
+  fprintf(stdout, "pqr");
+  fprintf(stderr, "stu");
+  EXPECT_STREQ("pqr", GetCapturedStdout().c_str());
+  EXPECT_STREQ("stu", GetCapturedStderr().c_str());
+}
+
+TEST(CaptureDeathTest, CannotReenterStdoutCapture) {
+  CaptureStdout();
+  EXPECT_DEATH_IF_SUPPORTED(CaptureStdout(),
+                            "Only one stdout capturer can exist at a time");
+  GetCapturedStdout();
+
+  // We cannot test stderr capturing using death tests as they use it
+  // themselves.
+}
+
+#endif  // !GTEST_OS_WINDOWS_MOBILE
+
+TEST(ThreadLocalTest, DefaultConstructorInitializesToDefaultValues) {
+  ThreadLocal<int> t1;
+  EXPECT_EQ(0, t1.get());
+
+  ThreadLocal<void*> t2;
+  EXPECT_TRUE(t2.get() == NULL);
+}
+
+TEST(ThreadLocalTest, SingleParamConstructorInitializesToParam) {
+  ThreadLocal<int> t1(123);
+  EXPECT_EQ(123, t1.get());
+
+  int i = 0;
+  ThreadLocal<int*> t2(&i);
+  EXPECT_EQ(&i, t2.get());
+}
+
+class NoDefaultContructor {
+ public:
+  explicit NoDefaultContructor(const char*) {}
+  NoDefaultContructor(const NoDefaultContructor&) {}
+};
+
+TEST(ThreadLocalTest, ValueDefaultContructorIsNotRequiredForParamVersion) {
+  ThreadLocal<NoDefaultContructor> bar(NoDefaultContructor("foo"));
+  bar.pointer();
+}
+
+TEST(ThreadLocalTest, GetAndPointerReturnSameValue) {
+  ThreadLocal<std::string> thread_local_string;
+
+  EXPECT_EQ(thread_local_string.pointer(), &(thread_local_string.get()));
+
+  // Verifies the condition still holds after calling set.
+  thread_local_string.set("foo");
+  EXPECT_EQ(thread_local_string.pointer(), &(thread_local_string.get()));
+}
+
+TEST(ThreadLocalTest, PointerAndConstPointerReturnSameValue) {
+  ThreadLocal<std::string> thread_local_string;
+  const ThreadLocal<std::string>& const_thread_local_string =
+      thread_local_string;
+
+  EXPECT_EQ(thread_local_string.pointer(), const_thread_local_string.pointer());
+
+  thread_local_string.set("foo");
+  EXPECT_EQ(thread_local_string.pointer(), const_thread_local_string.pointer());
+}
+
+#if GTEST_IS_THREADSAFE
+
+void AddTwo(int* param) { *param += 2; }
+
+TEST(ThreadWithParamTest, ConstructorExecutesThreadFunc) {
+  int i = 40;
+  ThreadWithParam<int*> thread(&AddTwo, &i, NULL);
+  thread.Join();
+  EXPECT_EQ(42, i);
+}
+
+TEST(MutexDeathTest, AssertHeldShouldAssertWhenNotLocked) {
+  // AssertHeld() is flaky only in the presence of multiple threads accessing
+  // the lock. In this case, the test is robust.
+  EXPECT_DEATH_IF_SUPPORTED({
+    Mutex m;
+    { MutexLock lock(&m); }
+    m.AssertHeld();
+  },
+  "thread .*hold");
+}
+
+TEST(MutexTest, AssertHeldShouldNotAssertWhenLocked) {
+  Mutex m;
+  MutexLock lock(&m);
+  m.AssertHeld();
+}
+
+class AtomicCounterWithMutex {
+ public:
+  explicit AtomicCounterWithMutex(Mutex* mutex) :
+    value_(0), mutex_(mutex), random_(42) {}
+
+  void Increment() {
+    MutexLock lock(mutex_);
+    int temp = value_;
+    {
+      // We need to put up a memory barrier to prevent reads and writes to
+      // value_ rearranged with the call to SleepMilliseconds when observed
+      // from other threads.
+#if GTEST_HAS_PTHREAD
+      // On POSIX, locking a mutex puts up a memory barrier.  We cannot use
+      // Mutex and MutexLock here or rely on their memory barrier
+      // functionality as we are testing them here.
+      pthread_mutex_t memory_barrier_mutex;
+      GTEST_CHECK_POSIX_SUCCESS_(
+          pthread_mutex_init(&memory_barrier_mutex, NULL));
+      GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&memory_barrier_mutex));
+
+      SleepMilliseconds(random_.Generate(30));
+
+      GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&memory_barrier_mutex));
+      GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&memory_barrier_mutex));
+#elif GTEST_OS_WINDOWS
+      // On Windows, performing an interlocked access puts up a memory barrier.
+      volatile LONG dummy = 0;
+      ::InterlockedIncrement(&dummy);
+      SleepMilliseconds(random_.Generate(30));
+      ::InterlockedIncrement(&dummy);
+#else
+# error "Memory barrier not implemented on this platform."
+#endif  // GTEST_HAS_PTHREAD
+    }
+    value_ = temp + 1;
+  }
+  int value() const { return value_; }
+
+ private:
+  volatile int value_;
+  Mutex* const mutex_;  // Protects value_.
+  Random       random_;
+};
+
+void CountingThreadFunc(pair<AtomicCounterWithMutex*, int> param) {
+  for (int i = 0; i < param.second; ++i)
+      param.first->Increment();
+}
+
+// Tests that the mutex only lets one thread at a time to lock it.
+TEST(MutexTest, OnlyOneThreadCanLockAtATime) {
+  Mutex mutex;
+  AtomicCounterWithMutex locked_counter(&mutex);
+
+  typedef ThreadWithParam<pair<AtomicCounterWithMutex*, int> > ThreadType;
+  const int kCycleCount = 20;
+  const int kThreadCount = 7;
+  scoped_ptr<ThreadType> counting_threads[kThreadCount];
+  Notification threads_can_start;
+  // Creates and runs kThreadCount threads that increment locked_counter
+  // kCycleCount times each.
+  for (int i = 0; i < kThreadCount; ++i) {
+    counting_threads[i].reset(new ThreadType(&CountingThreadFunc,
+                                             make_pair(&locked_counter,
+                                                       kCycleCount),
+                                             &threads_can_start));
+  }
+  threads_can_start.Notify();
+  for (int i = 0; i < kThreadCount; ++i)
+    counting_threads[i]->Join();
+
+  // If the mutex lets more than one thread to increment the counter at a
+  // time, they are likely to encounter a race condition and have some
+  // increments overwritten, resulting in the lower then expected counter
+  // value.
+  EXPECT_EQ(kCycleCount * kThreadCount, locked_counter.value());
+}
+
+template <typename T>
+void RunFromThread(void (func)(T), T param) {
+  ThreadWithParam<T> thread(func, param, NULL);
+  thread.Join();
+}
+
+void RetrieveThreadLocalValue(
+    pair<ThreadLocal<std::string>*, std::string*> param) {
+  *param.second = param.first->get();
+}
+
+TEST(ThreadLocalTest, ParameterizedConstructorSetsDefault) {
+  ThreadLocal<std::string> thread_local_string("foo");
+  EXPECT_STREQ("foo", thread_local_string.get().c_str());
+
+  thread_local_string.set("bar");
+  EXPECT_STREQ("bar", thread_local_string.get().c_str());
+
+  std::string result;
+  RunFromThread(&RetrieveThreadLocalValue,
+                make_pair(&thread_local_string, &result));
+  EXPECT_STREQ("foo", result.c_str());
+}
+
+// Keeps track of whether of destructors being called on instances of
+// DestructorTracker.  On Windows, waits for the destructor call reports.
+class DestructorCall {
+ public:
+  DestructorCall() {
+    invoked_ = false;
+#if GTEST_OS_WINDOWS
+    wait_event_.Reset(::CreateEvent(NULL, TRUE, FALSE, NULL));
+    GTEST_CHECK_(wait_event_.Get() != NULL);
+#endif
+  }
+
+  bool CheckDestroyed() const {
+#if GTEST_OS_WINDOWS
+    if (::WaitForSingleObject(wait_event_.Get(), 1000) != WAIT_OBJECT_0)
+      return false;
+#endif
+    return invoked_;
+  }
+
+  void ReportDestroyed() {
+    invoked_ = true;
+#if GTEST_OS_WINDOWS
+    ::SetEvent(wait_event_.Get());
+#endif
+  }
+
+  static std::vector<DestructorCall*>& List() { return *list_; }
+
+  static void ResetList() {
+    for (size_t i = 0; i < list_->size(); ++i) {
+      delete list_->at(i);
+    }
+    list_->clear();
+  }
+
+ private:
+  bool invoked_;
+#if GTEST_OS_WINDOWS
+  AutoHandle wait_event_;
+#endif
+  static std::vector<DestructorCall*>* const list_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(DestructorCall);
+};
+
+std::vector<DestructorCall*>* const DestructorCall::list_ =
+    new std::vector<DestructorCall*>;
+
+// DestructorTracker keeps track of whether its instances have been
+// destroyed.
+class DestructorTracker {
+ public:
+  DestructorTracker() : index_(GetNewIndex()) {}
+  DestructorTracker(const DestructorTracker& /* rhs */)
+      : index_(GetNewIndex()) {}
+  ~DestructorTracker() {
+    // We never access DestructorCall::List() concurrently, so we don't need
+    // to protect this acccess with a mutex.
+    DestructorCall::List()[index_]->ReportDestroyed();
+  }
+
+ private:
+  static size_t GetNewIndex() {
+    DestructorCall::List().push_back(new DestructorCall);
+    return DestructorCall::List().size() - 1;
+  }
+  const size_t index_;
+
+  GTEST_DISALLOW_ASSIGN_(DestructorTracker);
+};
+
+typedef ThreadLocal<DestructorTracker>* ThreadParam;
+
+void CallThreadLocalGet(ThreadParam thread_local_param) {
+  thread_local_param->get();
+}
+
+// Tests that when a ThreadLocal object dies in a thread, it destroys
+// the managed object for that thread.
+TEST(ThreadLocalTest, DestroysManagedObjectForOwnThreadWhenDying) {
+  DestructorCall::ResetList();
+
+  {
+    ThreadLocal<DestructorTracker> thread_local_tracker;
+    ASSERT_EQ(0U, DestructorCall::List().size());
+
+    // This creates another DestructorTracker object for the main thread.
+    thread_local_tracker.get();
+    ASSERT_EQ(1U, DestructorCall::List().size());
+    ASSERT_FALSE(DestructorCall::List()[0]->CheckDestroyed());
+  }
+
+  // Now thread_local_tracker has died.
+  ASSERT_EQ(1U, DestructorCall::List().size());
+  EXPECT_TRUE(DestructorCall::List()[0]->CheckDestroyed());
+
+  DestructorCall::ResetList();
+}
+
+// Tests that when a thread exits, the thread-local object for that
+// thread is destroyed.
+TEST(ThreadLocalTest, DestroysManagedObjectAtThreadExit) {
+  DestructorCall::ResetList();
+
+  {
+    ThreadLocal<DestructorTracker> thread_local_tracker;
+    ASSERT_EQ(0U, DestructorCall::List().size());
+
+    // This creates another DestructorTracker object in the new thread.
+    ThreadWithParam<ThreadParam> thread(
+        &CallThreadLocalGet, &thread_local_tracker, NULL);
+    thread.Join();
+
+    // The thread has exited, and we should have a DestroyedTracker
+    // instance created for it. But it may not have been destroyed yet.
+    ASSERT_EQ(1U, DestructorCall::List().size());
+  }
+
+  // The thread has exited and thread_local_tracker has died.
+  ASSERT_EQ(1U, DestructorCall::List().size());
+  EXPECT_TRUE(DestructorCall::List()[0]->CheckDestroyed());
+
+  DestructorCall::ResetList();
+}
+
+TEST(ThreadLocalTest, ThreadLocalMutationsAffectOnlyCurrentThread) {
+  ThreadLocal<std::string> thread_local_string;
+  thread_local_string.set("Foo");
+  EXPECT_STREQ("Foo", thread_local_string.get().c_str());
+
+  std::string result;
+  RunFromThread(&RetrieveThreadLocalValue,
+                make_pair(&thread_local_string, &result));
+  EXPECT_TRUE(result.empty());
+}
+
+#endif  // GTEST_IS_THREADSAFE
+
+#if GTEST_OS_WINDOWS
+TEST(WindowsTypesTest, HANDLEIsVoidStar) {
+  StaticAssertTypeEq<HANDLE, void*>();
+}
+
+TEST(WindowsTypesTest, CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION) {
+  StaticAssertTypeEq<CRITICAL_SECTION, _RTL_CRITICAL_SECTION>();
+}
+#endif  // GTEST_OS_WINDOWS
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Google Test - The Google C++ Testing Framework
+//
+// This file tests the universal value printer.
+
+#include "gtest/gtest-printers.h"
+
+#include <ctype.h>
+#include <limits.h>
+#include <string.h>
+#include <algorithm>
+#include <deque>
+#include <list>
+#include <map>
+#include <set>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+// hash_map and hash_set are available under Visual C++, or on Linux.
+#if GTEST_HAS_HASH_MAP_
+# include <hash_map>            // NOLINT
+#endif  // GTEST_HAS_HASH_MAP_
+#if GTEST_HAS_HASH_SET_
+# include <hash_set>            // NOLINT
+#endif  // GTEST_HAS_HASH_SET_
+
+#if GTEST_HAS_STD_FORWARD_LIST_
+# include <forward_list> // NOLINT
+#endif  // GTEST_HAS_STD_FORWARD_LIST_
+
+// Some user-defined types for testing the universal value printer.
+
+// An anonymous enum type.
+enum AnonymousEnum {
+  kAE1 = -1,
+  kAE2 = 1
+};
+
+// An enum without a user-defined printer.
+enum EnumWithoutPrinter {
+  kEWP1 = -2,
+  kEWP2 = 42
+};
+
+// An enum with a << operator.
+enum EnumWithStreaming {
+  kEWS1 = 10
+};
+
+std::ostream& operator<<(std::ostream& os, EnumWithStreaming e) {
+  return os << (e == kEWS1 ? "kEWS1" : "invalid");
+}
+
+// An enum with a PrintTo() function.
+enum EnumWithPrintTo {
+  kEWPT1 = 1
+};
+
+void PrintTo(EnumWithPrintTo e, std::ostream* os) {
+  *os << (e == kEWPT1 ? "kEWPT1" : "invalid");
+}
+
+// A class implicitly convertible to BiggestInt.
+class BiggestIntConvertible {
+ public:
+  operator ::testing::internal::BiggestInt() const { return 42; }
+};
+
+// A user-defined unprintable class template in the global namespace.
+template <typename T>
+class UnprintableTemplateInGlobal {
+ public:
+  UnprintableTemplateInGlobal() : value_() {}
+ private:
+  T value_;
+};
+
+// A user-defined streamable type in the global namespace.
+class StreamableInGlobal {
+ public:
+  virtual ~StreamableInGlobal() {}
+};
+
+inline void operator<<(::std::ostream& os, const StreamableInGlobal& /* x */) {
+  os << "StreamableInGlobal";
+}
+
+void operator<<(::std::ostream& os, const StreamableInGlobal* /* x */) {
+  os << "StreamableInGlobal*";
+}
+
+namespace foo {
+
+// A user-defined unprintable type in a user namespace.
+class UnprintableInFoo {
+ public:
+  UnprintableInFoo() : z_(0) { memcpy(xy_, "\xEF\x12\x0\x0\x34\xAB\x0\x0", 8); }
+  double z() const { return z_; }
+ private:
+  char xy_[8];
+  double z_;
+};
+
+// A user-defined printable type in a user-chosen namespace.
+struct PrintableViaPrintTo {
+  PrintableViaPrintTo() : value() {}
+  int value;
+};
+
+void PrintTo(const PrintableViaPrintTo& x, ::std::ostream* os) {
+  *os << "PrintableViaPrintTo: " << x.value;
+}
+
+// A type with a user-defined << for printing its pointer.
+struct PointerPrintable {
+};
+
+::std::ostream& operator<<(::std::ostream& os,
+                           const PointerPrintable* /* x */) {
+  return os << "PointerPrintable*";
+}
+
+// A user-defined printable class template in a user-chosen namespace.
+template <typename T>
+class PrintableViaPrintToTemplate {
+ public:
+  explicit PrintableViaPrintToTemplate(const T& a_value) : value_(a_value) {}
+
+  const T& value() const { return value_; }
+ private:
+  T value_;
+};
+
+template <typename T>
+void PrintTo(const PrintableViaPrintToTemplate<T>& x, ::std::ostream* os) {
+  *os << "PrintableViaPrintToTemplate: " << x.value();
+}
+
+// A user-defined streamable class template in a user namespace.
+template <typename T>
+class StreamableTemplateInFoo {
+ public:
+  StreamableTemplateInFoo() : value_() {}
+
+  const T& value() const { return value_; }
+ private:
+  T value_;
+};
+
+template <typename T>
+inline ::std::ostream& operator<<(::std::ostream& os,
+                                  const StreamableTemplateInFoo<T>& x) {
+  return os << "StreamableTemplateInFoo: " << x.value();
+}
+
+}  // namespace foo
+
+namespace testing {
+namespace gtest_printers_test {
+
+using ::std::deque;
+using ::std::list;
+using ::std::make_pair;
+using ::std::map;
+using ::std::multimap;
+using ::std::multiset;
+using ::std::pair;
+using ::std::set;
+using ::std::vector;
+using ::testing::PrintToString;
+using ::testing::internal::FormatForComparisonFailureMessage;
+using ::testing::internal::ImplicitCast_;
+using ::testing::internal::NativeArray;
+using ::testing::internal::RE;
+using ::testing::internal::RelationToSourceReference;
+using ::testing::internal::Strings;
+using ::testing::internal::UniversalPrint;
+using ::testing::internal::UniversalPrinter;
+using ::testing::internal::UniversalTersePrint;
+using ::testing::internal::UniversalTersePrintTupleFieldsToStrings;
+using ::testing::internal::string;
+
+// The hash_* classes are not part of the C++ standard.  STLport
+// defines them in namespace std.  MSVC defines them in ::stdext.  GCC
+// defines them in ::.
+#ifdef _STLP_HASH_MAP  // We got <hash_map> from STLport.
+using ::std::hash_map;
+using ::std::hash_set;
+using ::std::hash_multimap;
+using ::std::hash_multiset;
+#elif _MSC_VER
+using ::stdext::hash_map;
+using ::stdext::hash_set;
+using ::stdext::hash_multimap;
+using ::stdext::hash_multiset;
+#endif
+
+// Prints a value to a string using the universal value printer.  This
+// is a helper for testing UniversalPrinter<T>::Print() for various types.
+template <typename T>
+string Print(const T& value) {
+  ::std::stringstream ss;
+  UniversalPrinter<T>::Print(value, &ss);
+  return ss.str();
+}
+
+// Prints a value passed by reference to a string, using the universal
+// value printer.  This is a helper for testing
+// UniversalPrinter<T&>::Print() for various types.
+template <typename T>
+string PrintByRef(const T& value) {
+  ::std::stringstream ss;
+  UniversalPrinter<T&>::Print(value, &ss);
+  return ss.str();
+}
+
+// Tests printing various enum types.
+
+TEST(PrintEnumTest, AnonymousEnum) {
+  EXPECT_EQ("-1", Print(kAE1));
+  EXPECT_EQ("1", Print(kAE2));
+}
+
+TEST(PrintEnumTest, EnumWithoutPrinter) {
+  EXPECT_EQ("-2", Print(kEWP1));
+  EXPECT_EQ("42", Print(kEWP2));
+}
+
+TEST(PrintEnumTest, EnumWithStreaming) {
+  EXPECT_EQ("kEWS1", Print(kEWS1));
+  EXPECT_EQ("invalid", Print(static_cast<EnumWithStreaming>(0)));
+}
+
+TEST(PrintEnumTest, EnumWithPrintTo) {
+  EXPECT_EQ("kEWPT1", Print(kEWPT1));
+  EXPECT_EQ("invalid", Print(static_cast<EnumWithPrintTo>(0)));
+}
+
+// Tests printing a class implicitly convertible to BiggestInt.
+
+TEST(PrintClassTest, BiggestIntConvertible) {
+  EXPECT_EQ("42", Print(BiggestIntConvertible()));
+}
+
+// Tests printing various char types.
+
+// char.
+TEST(PrintCharTest, PlainChar) {
+  EXPECT_EQ("'\\0'", Print('\0'));
+  EXPECT_EQ("'\\'' (39, 0x27)", Print('\''));
+  EXPECT_EQ("'\"' (34, 0x22)", Print('"'));
+  EXPECT_EQ("'?' (63, 0x3F)", Print('?'));
+  EXPECT_EQ("'\\\\' (92, 0x5C)", Print('\\'));
+  EXPECT_EQ("'\\a' (7)", Print('\a'));
+  EXPECT_EQ("'\\b' (8)", Print('\b'));
+  EXPECT_EQ("'\\f' (12, 0xC)", Print('\f'));
+  EXPECT_EQ("'\\n' (10, 0xA)", Print('\n'));
+  EXPECT_EQ("'\\r' (13, 0xD)", Print('\r'));
+  EXPECT_EQ("'\\t' (9)", Print('\t'));
+  EXPECT_EQ("'\\v' (11, 0xB)", Print('\v'));
+  EXPECT_EQ("'\\x7F' (127)", Print('\x7F'));
+  EXPECT_EQ("'\\xFF' (255)", Print('\xFF'));
+  EXPECT_EQ("' ' (32, 0x20)", Print(' '));
+  EXPECT_EQ("'a' (97, 0x61)", Print('a'));
+}
+
+// signed char.
+TEST(PrintCharTest, SignedChar) {
+  EXPECT_EQ("'\\0'", Print(static_cast<signed char>('\0')));
+  EXPECT_EQ("'\\xCE' (-50)",
+            Print(static_cast<signed char>(-50)));
+}
+
+// unsigned char.
+TEST(PrintCharTest, UnsignedChar) {
+  EXPECT_EQ("'\\0'", Print(static_cast<unsigned char>('\0')));
+  EXPECT_EQ("'b' (98, 0x62)",
+            Print(static_cast<unsigned char>('b')));
+}
+
+// Tests printing other simple, built-in types.
+
+// bool.
+TEST(PrintBuiltInTypeTest, Bool) {
+  EXPECT_EQ("false", Print(false));
+  EXPECT_EQ("true", Print(true));
+}
+
+// wchar_t.
+TEST(PrintBuiltInTypeTest, Wchar_t) {
+  EXPECT_EQ("L'\\0'", Print(L'\0'));
+  EXPECT_EQ("L'\\'' (39, 0x27)", Print(L'\''));
+  EXPECT_EQ("L'\"' (34, 0x22)", Print(L'"'));
+  EXPECT_EQ("L'?' (63, 0x3F)", Print(L'?'));
+  EXPECT_EQ("L'\\\\' (92, 0x5C)", Print(L'\\'));
+  EXPECT_EQ("L'\\a' (7)", Print(L'\a'));
+  EXPECT_EQ("L'\\b' (8)", Print(L'\b'));
+  EXPECT_EQ("L'\\f' (12, 0xC)", Print(L'\f'));
+  EXPECT_EQ("L'\\n' (10, 0xA)", Print(L'\n'));
+  EXPECT_EQ("L'\\r' (13, 0xD)", Print(L'\r'));
+  EXPECT_EQ("L'\\t' (9)", Print(L'\t'));
+  EXPECT_EQ("L'\\v' (11, 0xB)", Print(L'\v'));
+  EXPECT_EQ("L'\\x7F' (127)", Print(L'\x7F'));
+  EXPECT_EQ("L'\\xFF' (255)", Print(L'\xFF'));
+  EXPECT_EQ("L' ' (32, 0x20)", Print(L' '));
+  EXPECT_EQ("L'a' (97, 0x61)", Print(L'a'));
+  EXPECT_EQ("L'\\x576' (1398)", Print(static_cast<wchar_t>(0x576)));
+  EXPECT_EQ("L'\\xC74D' (51021)", Print(static_cast<wchar_t>(0xC74D)));
+}
+
+// Test that Int64 provides more storage than wchar_t.
+TEST(PrintTypeSizeTest, Wchar_t) {
+  EXPECT_LT(sizeof(wchar_t), sizeof(testing::internal::Int64));
+}
+
+// Various integer types.
+TEST(PrintBuiltInTypeTest, Integer) {
+  EXPECT_EQ("'\\xFF' (255)", Print(static_cast<unsigned char>(255)));  // uint8
+  EXPECT_EQ("'\\x80' (-128)", Print(static_cast<signed char>(-128)));  // int8
+  EXPECT_EQ("65535", Print(USHRT_MAX));  // uint16
+  EXPECT_EQ("-32768", Print(SHRT_MIN));  // int16
+  EXPECT_EQ("4294967295", Print(UINT_MAX));  // uint32
+  EXPECT_EQ("-2147483648", Print(INT_MIN));  // int32
+  EXPECT_EQ("18446744073709551615",
+            Print(static_cast<testing::internal::UInt64>(-1)));  // uint64
+  EXPECT_EQ("-9223372036854775808",
+            Print(static_cast<testing::internal::Int64>(1) << 63));  // int64
+}
+
+// Size types.
+TEST(PrintBuiltInTypeTest, Size_t) {
+  EXPECT_EQ("1", Print(sizeof('a')));  // size_t.
+#if !GTEST_OS_WINDOWS
+  // Windows has no ssize_t type.
+  EXPECT_EQ("-2", Print(static_cast<ssize_t>(-2)));  // ssize_t.
+#endif  // !GTEST_OS_WINDOWS
+}
+
+// Floating-points.
+TEST(PrintBuiltInTypeTest, FloatingPoints) {
+  EXPECT_EQ("1.5", Print(1.5f));   // float
+  EXPECT_EQ("-2.5", Print(-2.5));  // double
+}
+
+// Since ::std::stringstream::operator<<(const void *) formats the pointer
+// output differently with different compilers, we have to create the expected
+// output first and use it as our expectation.
+static string PrintPointer(const void *p) {
+  ::std::stringstream expected_result_stream;
+  expected_result_stream << p;
+  return expected_result_stream.str();
+}
+
+// Tests printing C strings.
+
+// const char*.
+TEST(PrintCStringTest, Const) {
+  const char* p = "World";
+  EXPECT_EQ(PrintPointer(p) + " pointing to \"World\"", Print(p));
+}
+
+// char*.
+TEST(PrintCStringTest, NonConst) {
+  char p[] = "Hi";
+  EXPECT_EQ(PrintPointer(p) + " pointing to \"Hi\"",
+            Print(static_cast<char*>(p)));
+}
+
+// NULL C string.
+TEST(PrintCStringTest, Null) {
+  const char* p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// Tests that C strings are escaped properly.
+TEST(PrintCStringTest, EscapesProperly) {
+  const char* p = "'\"?\\\a\b\f\n\r\t\v\x7F\xFF a";
+  EXPECT_EQ(PrintPointer(p) + " pointing to \"'\\\"?\\\\\\a\\b\\f"
+            "\\n\\r\\t\\v\\x7F\\xFF a\"",
+            Print(p));
+}
+
+// MSVC compiler can be configured to define whar_t as a typedef
+// of unsigned short. Defining an overload for const wchar_t* in that case
+// would cause pointers to unsigned shorts be printed as wide strings,
+// possibly accessing more memory than intended and causing invalid
+// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when
+// wchar_t is implemented as a native type.
+#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED)
+
+// const wchar_t*.
+TEST(PrintWideCStringTest, Const) {
+  const wchar_t* p = L"World";
+  EXPECT_EQ(PrintPointer(p) + " pointing to L\"World\"", Print(p));
+}
+
+// wchar_t*.
+TEST(PrintWideCStringTest, NonConst) {
+  wchar_t p[] = L"Hi";
+  EXPECT_EQ(PrintPointer(p) + " pointing to L\"Hi\"",
+            Print(static_cast<wchar_t*>(p)));
+}
+
+// NULL wide C string.
+TEST(PrintWideCStringTest, Null) {
+  const wchar_t* p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// Tests that wide C strings are escaped properly.
+TEST(PrintWideCStringTest, EscapesProperly) {
+  const wchar_t s[] = {'\'', '"', '?', '\\', '\a', '\b', '\f', '\n', '\r',
+                       '\t', '\v', 0xD3, 0x576, 0x8D3, 0xC74D, ' ', 'a', '\0'};
+  EXPECT_EQ(PrintPointer(s) + " pointing to L\"'\\\"?\\\\\\a\\b\\f"
+            "\\n\\r\\t\\v\\xD3\\x576\\x8D3\\xC74D a\"",
+            Print(static_cast<const wchar_t*>(s)));
+}
+#endif  // native wchar_t
+
+// Tests printing pointers to other char types.
+
+// signed char*.
+TEST(PrintCharPointerTest, SignedChar) {
+  signed char* p = reinterpret_cast<signed char*>(0x1234);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// const signed char*.
+TEST(PrintCharPointerTest, ConstSignedChar) {
+  signed char* p = reinterpret_cast<signed char*>(0x1234);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// unsigned char*.
+TEST(PrintCharPointerTest, UnsignedChar) {
+  unsigned char* p = reinterpret_cast<unsigned char*>(0x1234);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// const unsigned char*.
+TEST(PrintCharPointerTest, ConstUnsignedChar) {
+  const unsigned char* p = reinterpret_cast<const unsigned char*>(0x1234);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// Tests printing pointers to simple, built-in types.
+
+// bool*.
+TEST(PrintPointerToBuiltInTypeTest, Bool) {
+  bool* p = reinterpret_cast<bool*>(0xABCD);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// void*.
+TEST(PrintPointerToBuiltInTypeTest, Void) {
+  void* p = reinterpret_cast<void*>(0xABCD);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// const void*.
+TEST(PrintPointerToBuiltInTypeTest, ConstVoid) {
+  const void* p = reinterpret_cast<const void*>(0xABCD);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// Tests printing pointers to pointers.
+TEST(PrintPointerToPointerTest, IntPointerPointer) {
+  int** p = reinterpret_cast<int**>(0xABCD);
+  EXPECT_EQ(PrintPointer(p), Print(p));
+  p = NULL;
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// Tests printing (non-member) function pointers.
+
+void MyFunction(int /* n */) {}
+
+TEST(PrintPointerTest, NonMemberFunctionPointer) {
+  // We cannot directly cast &MyFunction to const void* because the
+  // standard disallows casting between pointers to functions and
+  // pointers to objects, and some compilers (e.g. GCC 3.4) enforce
+  // this limitation.
+  EXPECT_EQ(
+      PrintPointer(reinterpret_cast<const void*>(
+          reinterpret_cast<internal::BiggestInt>(&MyFunction))),
+      Print(&MyFunction));
+  int (*p)(bool) = NULL;  // NOLINT
+  EXPECT_EQ("NULL", Print(p));
+}
+
+// An assertion predicate determining whether a one string is a prefix for
+// another.
+template <typename StringType>
+AssertionResult HasPrefix(const StringType& str, const StringType& prefix) {
+  if (str.find(prefix, 0) == 0)
+    return AssertionSuccess();
+
+  const bool is_wide_string = sizeof(prefix[0]) > 1;
+  const char* const begin_string_quote = is_wide_string ? "L\"" : "\"";
+  return AssertionFailure()
+      << begin_string_quote << prefix << "\" is not a prefix of "
+      << begin_string_quote << str << "\"\n";
+}
+
+// Tests printing member variable pointers.  Although they are called
+// pointers, they don't point to a location in the address space.
+// Their representation is implementation-defined.  Thus they will be
+// printed as raw bytes.
+
+struct Foo {
+ public:
+  virtual ~Foo() {}
+  int MyMethod(char x) { return x + 1; }
+  virtual char MyVirtualMethod(int /* n */) { return 'a'; }
+
+  int value;
+};
+
+TEST(PrintPointerTest, MemberVariablePointer) {
+  EXPECT_TRUE(HasPrefix(Print(&Foo::value),
+                        Print(sizeof(&Foo::value)) + "-byte object "));
+  int (Foo::*p) = NULL;  // NOLINT
+  EXPECT_TRUE(HasPrefix(Print(p),
+                        Print(sizeof(p)) + "-byte object "));
+}
+
+// Tests printing member function pointers.  Although they are called
+// pointers, they don't point to a location in the address space.
+// Their representation is implementation-defined.  Thus they will be
+// printed as raw bytes.
+TEST(PrintPointerTest, MemberFunctionPointer) {
+  EXPECT_TRUE(HasPrefix(Print(&Foo::MyMethod),
+                        Print(sizeof(&Foo::MyMethod)) + "-byte object "));
+  EXPECT_TRUE(
+      HasPrefix(Print(&Foo::MyVirtualMethod),
+                Print(sizeof((&Foo::MyVirtualMethod))) + "-byte object "));
+  int (Foo::*p)(char) = NULL;  // NOLINT
+  EXPECT_TRUE(HasPrefix(Print(p),
+                        Print(sizeof(p)) + "-byte object "));
+}
+
+// Tests printing C arrays.
+
+// The difference between this and Print() is that it ensures that the
+// argument is a reference to an array.
+template <typename T, size_t N>
+string PrintArrayHelper(T (&a)[N]) {
+  return Print(a);
+}
+
+// One-dimensional array.
+TEST(PrintArrayTest, OneDimensionalArray) {
+  int a[5] = { 1, 2, 3, 4, 5 };
+  EXPECT_EQ("{ 1, 2, 3, 4, 5 }", PrintArrayHelper(a));
+}
+
+// Two-dimensional array.
+TEST(PrintArrayTest, TwoDimensionalArray) {
+  int a[2][5] = {
+    { 1, 2, 3, 4, 5 },
+    { 6, 7, 8, 9, 0 }
+  };
+  EXPECT_EQ("{ { 1, 2, 3, 4, 5 }, { 6, 7, 8, 9, 0 } }", PrintArrayHelper(a));
+}
+
+// Array of const elements.
+TEST(PrintArrayTest, ConstArray) {
+  const bool a[1] = { false };
+  EXPECT_EQ("{ false }", PrintArrayHelper(a));
+}
+
+// char array without terminating NUL.
+TEST(PrintArrayTest, CharArrayWithNoTerminatingNul) {
+  // Array a contains '\0' in the middle and doesn't end with '\0'.
+  char a[] = { 'H', '\0', 'i' };
+  EXPECT_EQ("\"H\\0i\" (no terminating NUL)", PrintArrayHelper(a));
+}
+
+// const char array with terminating NUL.
+TEST(PrintArrayTest, ConstCharArrayWithTerminatingNul) {
+  const char a[] = "\0Hi";
+  EXPECT_EQ("\"\\0Hi\"", PrintArrayHelper(a));
+}
+
+// const wchar_t array without terminating NUL.
+TEST(PrintArrayTest, WCharArrayWithNoTerminatingNul) {
+  // Array a contains '\0' in the middle and doesn't end with '\0'.
+  const wchar_t a[] = { L'H', L'\0', L'i' };
+  EXPECT_EQ("L\"H\\0i\" (no terminating NUL)", PrintArrayHelper(a));
+}
+
+// wchar_t array with terminating NUL.
+TEST(PrintArrayTest, WConstCharArrayWithTerminatingNul) {
+  const wchar_t a[] = L"\0Hi";
+  EXPECT_EQ("L\"\\0Hi\"", PrintArrayHelper(a));
+}
+
+// Array of objects.
+TEST(PrintArrayTest, ObjectArray) {
+  string a[3] = { "Hi", "Hello", "Ni hao" };
+  EXPECT_EQ("{ \"Hi\", \"Hello\", \"Ni hao\" }", PrintArrayHelper(a));
+}
+
+// Array with many elements.
+TEST(PrintArrayTest, BigArray) {
+  int a[100] = { 1, 2, 3 };
+  EXPECT_EQ("{ 1, 2, 3, 0, 0, 0, 0, 0, ..., 0, 0, 0, 0, 0, 0, 0, 0 }",
+            PrintArrayHelper(a));
+}
+
+// Tests printing ::string and ::std::string.
+
+#if GTEST_HAS_GLOBAL_STRING
+// ::string.
+TEST(PrintStringTest, StringInGlobalNamespace) {
+  const char s[] = "'\"?\\\a\b\f\n\0\r\t\v\x7F\xFF a";
+  const ::string str(s, sizeof(s));
+  EXPECT_EQ("\"'\\\"?\\\\\\a\\b\\f\\n\\0\\r\\t\\v\\x7F\\xFF a\\0\"",
+            Print(str));
+}
+#endif  // GTEST_HAS_GLOBAL_STRING
+
+// ::std::string.
+TEST(PrintStringTest, StringInStdNamespace) {
+  const char s[] = "'\"?\\\a\b\f\n\0\r\t\v\x7F\xFF a";
+  const ::std::string str(s, sizeof(s));
+  EXPECT_EQ("\"'\\\"?\\\\\\a\\b\\f\\n\\0\\r\\t\\v\\x7F\\xFF a\\0\"",
+            Print(str));
+}
+
+TEST(PrintStringTest, StringAmbiguousHex) {
+  // "\x6BANANA" is ambiguous, it can be interpreted as starting with either of:
+  // '\x6', '\x6B', or '\x6BA'.
+
+  // a hex escaping sequence following by a decimal digit
+  EXPECT_EQ("\"0\\x12\" \"3\"", Print(::std::string("0\x12" "3")));
+  // a hex escaping sequence following by a hex digit (lower-case)
+  EXPECT_EQ("\"mm\\x6\" \"bananas\"", Print(::std::string("mm\x6" "bananas")));
+  // a hex escaping sequence following by a hex digit (upper-case)
+  EXPECT_EQ("\"NOM\\x6\" \"BANANA\"", Print(::std::string("NOM\x6" "BANANA")));
+  // a hex escaping sequence following by a non-xdigit
+  EXPECT_EQ("\"!\\x5-!\"", Print(::std::string("!\x5-!")));
+}
+
+// Tests printing ::wstring and ::std::wstring.
+
+#if GTEST_HAS_GLOBAL_WSTRING
+// ::wstring.
+TEST(PrintWideStringTest, StringInGlobalNamespace) {
+  const wchar_t s[] = L"'\"?\\\a\b\f\n\0\r\t\v\xD3\x576\x8D3\xC74D a";
+  const ::wstring str(s, sizeof(s)/sizeof(wchar_t));
+  EXPECT_EQ("L\"'\\\"?\\\\\\a\\b\\f\\n\\0\\r\\t\\v"
+            "\\xD3\\x576\\x8D3\\xC74D a\\0\"",
+            Print(str));
+}
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+#if GTEST_HAS_STD_WSTRING
+// ::std::wstring.
+TEST(PrintWideStringTest, StringInStdNamespace) {
+  const wchar_t s[] = L"'\"?\\\a\b\f\n\0\r\t\v\xD3\x576\x8D3\xC74D a";
+  const ::std::wstring str(s, sizeof(s)/sizeof(wchar_t));
+  EXPECT_EQ("L\"'\\\"?\\\\\\a\\b\\f\\n\\0\\r\\t\\v"
+            "\\xD3\\x576\\x8D3\\xC74D a\\0\"",
+            Print(str));
+}
+
+TEST(PrintWideStringTest, StringAmbiguousHex) {
+  // same for wide strings.
+  EXPECT_EQ("L\"0\\x12\" L\"3\"", Print(::std::wstring(L"0\x12" L"3")));
+  EXPECT_EQ("L\"mm\\x6\" L\"bananas\"",
+            Print(::std::wstring(L"mm\x6" L"bananas")));
+  EXPECT_EQ("L\"NOM\\x6\" L\"BANANA\"",
+            Print(::std::wstring(L"NOM\x6" L"BANANA")));
+  EXPECT_EQ("L\"!\\x5-!\"", Print(::std::wstring(L"!\x5-!")));
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+// Tests printing types that support generic streaming (i.e. streaming
+// to std::basic_ostream<Char, CharTraits> for any valid Char and
+// CharTraits types).
+
+// Tests printing a non-template type that supports generic streaming.
+
+class AllowsGenericStreaming {};
+
+template <typename Char, typename CharTraits>
+std::basic_ostream<Char, CharTraits>& operator<<(
+    std::basic_ostream<Char, CharTraits>& os,
+    const AllowsGenericStreaming& /* a */) {
+  return os << "AllowsGenericStreaming";
+}
+
+TEST(PrintTypeWithGenericStreamingTest, NonTemplateType) {
+  AllowsGenericStreaming a;
+  EXPECT_EQ("AllowsGenericStreaming", Print(a));
+}
+
+// Tests printing a template type that supports generic streaming.
+
+template <typename T>
+class AllowsGenericStreamingTemplate {};
+
+template <typename Char, typename CharTraits, typename T>
+std::basic_ostream<Char, CharTraits>& operator<<(
+    std::basic_ostream<Char, CharTraits>& os,
+    const AllowsGenericStreamingTemplate<T>& /* a */) {
+  return os << "AllowsGenericStreamingTemplate";
+}
+
+TEST(PrintTypeWithGenericStreamingTest, TemplateType) {
+  AllowsGenericStreamingTemplate<int> a;
+  EXPECT_EQ("AllowsGenericStreamingTemplate", Print(a));
+}
+
+// Tests printing a type that supports generic streaming and can be
+// implicitly converted to another printable type.
+
+template <typename T>
+class AllowsGenericStreamingAndImplicitConversionTemplate {
+ public:
+  operator bool() const { return false; }
+};
+
+template <typename Char, typename CharTraits, typename T>
+std::basic_ostream<Char, CharTraits>& operator<<(
+    std::basic_ostream<Char, CharTraits>& os,
+    const AllowsGenericStreamingAndImplicitConversionTemplate<T>& /* a */) {
+  return os << "AllowsGenericStreamingAndImplicitConversionTemplate";
+}
+
+TEST(PrintTypeWithGenericStreamingTest, TypeImplicitlyConvertible) {
+  AllowsGenericStreamingAndImplicitConversionTemplate<int> a;
+  EXPECT_EQ("AllowsGenericStreamingAndImplicitConversionTemplate", Print(a));
+}
+
+#if GTEST_HAS_STRING_PIECE_
+
+// Tests printing StringPiece.
+
+TEST(PrintStringPieceTest, SimpleStringPiece) {
+  const StringPiece sp = "Hello";
+  EXPECT_EQ("\"Hello\"", Print(sp));
+}
+
+TEST(PrintStringPieceTest, UnprintableCharacters) {
+  const char str[] = "NUL (\0) and \r\t";
+  const StringPiece sp(str, sizeof(str) - 1);
+  EXPECT_EQ("\"NUL (\\0) and \\r\\t\"", Print(sp));
+}
+
+#endif  // GTEST_HAS_STRING_PIECE_
+
+// Tests printing STL containers.
+
+TEST(PrintStlContainerTest, EmptyDeque) {
+  deque<char> empty;
+  EXPECT_EQ("{}", Print(empty));
+}
+
+TEST(PrintStlContainerTest, NonEmptyDeque) {
+  deque<int> non_empty;
+  non_empty.push_back(1);
+  non_empty.push_back(3);
+  EXPECT_EQ("{ 1, 3 }", Print(non_empty));
+}
+
+#if GTEST_HAS_HASH_MAP_
+
+TEST(PrintStlContainerTest, OneElementHashMap) {
+  hash_map<int, char> map1;
+  map1[1] = 'a';
+  EXPECT_EQ("{ (1, 'a' (97, 0x61)) }", Print(map1));
+}
+
+TEST(PrintStlContainerTest, HashMultiMap) {
+  hash_multimap<int, bool> map1;
+  map1.insert(make_pair(5, true));
+  map1.insert(make_pair(5, false));
+
+  // Elements of hash_multimap can be printed in any order.
+  const string result = Print(map1);
+  EXPECT_TRUE(result == "{ (5, true), (5, false) }" ||
+              result == "{ (5, false), (5, true) }")
+                  << " where Print(map1) returns \"" << result << "\".";
+}
+
+#endif  // GTEST_HAS_HASH_MAP_
+
+#if GTEST_HAS_HASH_SET_
+
+TEST(PrintStlContainerTest, HashSet) {
+  hash_set<string
\ No newline at end of file
diff --git a/qa/workunits/rgw/jcksum/file-200b b/qa/workunits/rgw/jcksum/file-200b
new file mode 100644
index 000000000000..25460169dfd4
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/file-200b
@@ -0,0 +1,5 @@
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following condition
\ No newline at end of file
diff --git a/qa/workunits/rgw/jcksum/file-21983b b/qa/workunits/rgw/jcksum/file-21983b
new file mode 100644
index 000000000000..6f299ebdd6e7
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/file-21983b
@@ -0,0 +1,568 @@
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// C++Builder's IDE cannot build a static library from files with hyphens
+// in their name.  See http://qc.codegear.com/wc/qcmain.aspx?d=70977 .
+// This file serves as a workaround.
+
+#include "src/gtest-all.cc"
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// Links gtest.lib and gtest_main.lib into the current project in C++Builder.
+// This means that these libraries can't be renamed, but it's the only way to
+// ensure that Debug versus Release test builds are linked against the
+// appropriate Debug or Release build of the libraries.
+
+#pragma link "gtest.lib"
+#pragma link "gtest_main.lib"
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample2.h"
+
+#include <string.h>
+
+// Clones a 0-terminated C string, allocating memory using new.
+const char* MyString::CloneCString(const char* a_c_string) {
+  if (a_c_string == NULL) return NULL;
+
+  const size_t len = strlen(a_c_string);
+  char* const clone = new char[ len + 1 ];
+  memcpy(clone, a_c_string, len + 1);
+
+  return clone;
+}
+
+// Sets the 0-terminated C string this MyString object
+// represents.
+void MyString::Set(const char* a_c_string) {
+  // Makes sure this works when c_string == c_string_
+  const char* const temp = MyString::CloneCString(a_c_string);
+  delete[] c_string_;
+  c_string_ = temp;
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include <stdio.h>
+
+#include "sample4.h"
+
+// Returns the current counter value, and increments it.
+int Counter::Increment() {
+  return counter_++;
+}
+
+// Prints the current counter value to STDOUT.
+void Counter::Print() const {
+  printf("%d", counter_);
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample1.h"
+
+// Returns n! (the factorial of n).  For negative n, n! is defined to be 1.
+int Factorial(int n) {
+  int result = 1;
+  for (int i = 1; i <= n; i++) {
+    result *= i;
+  }
+
+  return result;
+}
+
+// Returns true iff n is a prime number.
+bool IsPrime(int n) {
+  // Trivial case 1: small numbers
+  if (n <= 1) return false;
+
+  // Trivial case 2: even numbers
+  if (n % 2 == 0) return n == 2;
+
+  // Now, we have that n is odd and n >= 3.
+
+  // Try to divide n by every odd number i, starting from 3
+  for (int i = 3; ; i += 2) {
+    // We only have to try i up to the squre root of n
+    if (i > n/i) break;
+
+    // Now, we have i <= n/i < n.
+    // If n is divisible by i, n is not prime.
+    if (n % i == 0) return false;
+  }
+
+  // n has no integer factor in the range (1, n), and thus is prime.
+  return true;
+}
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to use Google Test listener API to implement
+// a primitive leak checker.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+namespace {
+
+// We will track memory used by this class.
+class Water {
+ public:
+  // Normal Water declarations go here.
+
+  // operator new and operator delete help us control water allocation.
+  void* operator new(size_t allocation_size) {
+    allocated_++;
+    return malloc(allocation_size);
+  }
+
+  void operator delete(void* block, size_t /* allocation_size */) {
+    allocated_--;
+    free(block);
+  }
+
+  static int allocated() { return allocated_; }
+
+ private:
+  static int allocated_;
+};
+
+int Water::allocated_ = 0;
+
+// This event listener monitors how many Water objects are created and
+// destroyed by each test, and reports a failure if a test leaks some Water
+// objects. It does this by comparing the number of live Water objects at
+// the beginning of a test and at the end of a test.
+class LeakChecker : public EmptyTestEventListener {
+ private:
+  // Called before a test starts.
+  virtual void OnTestStart(const TestInfo& /* test_info */) {
+    initially_allocated_ = Water::allocated();
+  }
+
+  // Called after a test ends.
+  virtual void OnTestEnd(const TestInfo& /* test_info */) {
+    int difference = Water::allocated() - initially_allocated_;
+
+    // You can generate a failure in any event handler except
+    // OnTestPartResult. Just use an appropriate Google Test assertion to do
+    // it.
+    EXPECT_LE(difference, 0) << "Leaked " << difference << " unit(s) of Water!";
+  }
+
+  int initially_allocated_;
+};
+
+TEST(ListenersTest, DoesNotLeak) {
+  Water* water = new Water;
+  delete water;
+}
+
+// This should fail when the --check_for_leaks command line flag is
+// specified.
+TEST(ListenersTest, LeaksWater) {
+  Water* water = new Water;
+  EXPECT_TRUE(water != NULL);
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  bool check_for_leaks = false;
+  if (argc > 1 && strcmp(argv[1], "--check_for_leaks") == 0 )
+    check_for_leaks = true;
+  else
+    printf("%s\n", "Run this program with --check_for_leaks to enable "
+           "custom leak checking in the tests.");
+
+  // If we are given the --check_for_leaks command line flag, installs the
+  // leak checker.
+  if (check_for_leaks) {
+    TestEventListeners& listeners = UnitTest::GetInstance()->listeners();
+
+    // Adds the leak checker to the end of the test event listener list,
+    // after the default text output printer and the default XML report
+    // generator.
+    //
+    // The order is important - it ensures that failures generated in the
+    // leak checker's OnTestEnd() method are processed by the text and XML
+    // printers *before* their OnTestEnd() methods are called, such that
+    // they are attributed to the right test. Remember that a listener
+    // receives an OnXyzStart event *after* listeners preceding it in the
+    // list received that event, and receives an OnXyzEnd event *before*
+    // listeners preceding it.
+    //
+    // We don't need to worry about deleting the new listener later, as
+    // Google Test will do it.
+    listeners.Append(new LeakChecker);
+  }
+  return RUN_ALL_TESTS();
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// This sample shows how to write a simple unit test for a function,
+// using Google C++ testing framework.
+//
+// Writing a unit test using Google C++ testing framework is easy as 1-2-3:
+
+
+// Step 1. Include necessary header files such that the stuff your
+// test logic needs is declared.
+//
+// Don't forget gtest.h, which declares the testing framework.
+
+#include <limits.h>
+#include "sample1.h"
+#include "gtest/gtest.h"
+
+
+// Step 2. Use the TEST macro to define your tests.
+//
+// TEST has two parameters: the test case name and the test name.
+// After using the macro, you should define your test logic between a
+// pair of braces.  You can use a bunch of macros to indicate the
+// success or failure of a test.  EXPECT_TRUE and EXPECT_EQ are
+// examples of such macros.  For a complete list, see gtest.h.
+//
+// <TechnicalDetails>
+//
+// In Google Test, tests are grouped into test cases.  This is how we
+// keep test code organized.  You should put logically related tests
+// into the same test case.
+//
+// The test case name and the test name should both be valid C++
+// identifiers.  And you should not use underscore (_) in the names.
+//
+// Google Test guarantees that each test you define is run exactly
+// once, but it makes no guarantee on the order the tests are
+// executed.  Therefore, you should write your tests in such a way
+// that their results don't depend on their order.
+//
+// </TechnicalDetails>
+
+
+// Tests Factorial().
+
+// Tests factorial of negative numbers.
+TEST(FactorialTest, Negative) {
+  // This test is named "Negative", and belongs to the "FactorialTest"
+  // test case.
+  EXPECT_EQ(1, Factorial(-5));
+  EXPECT_EQ(1, Factorial(-1));
+  EXPECT_GT(Factorial(-10), 0);
+
+  // <TechnicalDetails>
+  //
+  // EXPECT_EQ(expected, actual) is the same as
+  //
+  //   EXPECT_TRUE((expected) == (actual))
+  //
+  // except that it will print both the expected value and the actual
+  // value when the assertion fails.  This is very helpful for
+  // debugging.  Therefore in this case EXPECT_EQ is preferred.
+  //
+  // On the other hand, EXPECT_TRUE accepts any Boolean expression,
+  // and is thus more general.
+  //
+  // </TechnicalDetails>
+}
+
+// Tests factorial of 0.
+TEST(FactorialTest, Zero) {
+  EXPECT_EQ(1, Factorial(0));
+}
+
+// Tests factorial of positive numbers.
+TEST(FactorialTest, Positive) {
+  EXPECT_EQ(1, Factorial(1));
+  EXPECT_EQ(2, Factorial(2));
+  EXPECT_EQ(6, Factorial(3));
+  EXPECT_EQ(40320, Factorial(8));
+}
+
+
+// Tests IsPrime()
+
+// Tests negative input.
+TEST(IsPrimeTest, Negative) {
+  // This test belongs to the IsPrimeTest test case.
+
+  EXPECT_FALSE(IsPrime(-1));
+  EXPECT_FALSE(IsPrime(-2));
+  EXPECT_FALSE(IsPrime(INT_MIN));
+}
+
+// Tests some trivial cases.
+TEST(IsPrimeTest, Trivial) {
+  EXPECT_FALSE(IsPrime(0));
+  EXPECT_FALSE(IsPrime(1));
+  EXPECT_TRUE(IsPrime(2));
+  EXPECT_TRUE(IsPrime(3));
+}
+
+// Tests positive input.
+TEST(IsPrimeTest, Positive) {
+  EXPECT_FALSE(IsPrime(4));
+  EXPECT_TRUE(IsPrime(5));
+  EXPECT_FALSE(IsPrime(6));
+  EXPECT_TRUE(IsPrime(23));
+}
+
+// Step 3. Call RUN_ALL_TESTS() in main().
+//
+// We do this by linking in src/gtest_main.cc file, which consists of
+// a main() function which calls RUN_ALL_TESTS() for us.
+//
+// This runs all the tests you've defined, prints the result, and
+// returns 0 if successful, or 1 otherwise.
+//
+// Did you notice that we didn't register the tests?  The
+// RUN_ALL_TESTS() macro magically knows about all the tests we
+// defined.  Isn't this convenient?
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, 
\ No newline at end of file
diff --git a/qa/workunits/rgw/jcksum/file-256k b/qa/workunits/rgw/jcksum/file-256k
new file mode 100644
index 000000000000..7b52e1829a17
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/file-256k
@@ -0,0 +1,7216 @@
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// C++Builder's IDE cannot build a static library from files with hyphens
+// in their name.  See http://qc.codegear.com/wc/qcmain.aspx?d=70977 .
+// This file serves as a workaround.
+
+#include "src/gtest-all.cc"
+// Copyright 2009, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: Josh Kelley (joshkel@gmail.com)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// Links gtest.lib and gtest_main.lib into the current project in C++Builder.
+// This means that these libraries can't be renamed, but it's the only way to
+// ensure that Debug versus Release test builds are linked against the
+// appropriate Debug or Release build of the libraries.
+
+#pragma link "gtest.lib"
+#pragma link "gtest_main.lib"
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample2.h"
+
+#include <string.h>
+
+// Clones a 0-terminated C string, allocating memory using new.
+const char* MyString::CloneCString(const char* a_c_string) {
+  if (a_c_string == NULL) return NULL;
+
+  const size_t len = strlen(a_c_string);
+  char* const clone = new char[ len + 1 ];
+  memcpy(clone, a_c_string, len + 1);
+
+  return clone;
+}
+
+// Sets the 0-terminated C string this MyString object
+// represents.
+void MyString::Set(const char* a_c_string) {
+  // Makes sure this works when c_string == c_string_
+  const char* const temp = MyString::CloneCString(a_c_string);
+  delete[] c_string_;
+  c_string_ = temp;
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include <stdio.h>
+
+#include "sample4.h"
+
+// Returns the current counter value, and increments it.
+int Counter::Increment() {
+  return counter_++;
+}
+
+// Prints the current counter value to STDOUT.
+void Counter::Print() const {
+  printf("%d", counter_);
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "sample1.h"
+
+// Returns n! (the factorial of n).  For negative n, n! is defined to be 1.
+int Factorial(int n) {
+  int result = 1;
+  for (int i = 1; i <= n; i++) {
+    result *= i;
+  }
+
+  return result;
+}
+
+// Returns true iff n is a prime number.
+bool IsPrime(int n) {
+  // Trivial case 1: small numbers
+  if (n <= 1) return false;
+
+  // Trivial case 2: even numbers
+  if (n % 2 == 0) return n == 2;
+
+  // Now, we have that n is odd and n >= 3.
+
+  // Try to divide n by every odd number i, starting from 3
+  for (int i = 3; ; i += 2) {
+    // We only have to try i up to the squre root of n
+    if (i > n/i) break;
+
+    // Now, we have i <= n/i < n.
+    // If n is divisible by i, n is not prime.
+    if (n % i == 0) return false;
+  }
+
+  // n has no integer factor in the range (1, n), and thus is prime.
+  return true;
+}
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to use Google Test listener API to implement
+// a primitive leak checker.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+namespace {
+
+// We will track memory used by this class.
+class Water {
+ public:
+  // Normal Water declarations go here.
+
+  // operator new and operator delete help us control water allocation.
+  void* operator new(size_t allocation_size) {
+    allocated_++;
+    return malloc(allocation_size);
+  }
+
+  void operator delete(void* block, size_t /* allocation_size */) {
+    allocated_--;
+    free(block);
+  }
+
+  static int allocated() { return allocated_; }
+
+ private:
+  static int allocated_;
+};
+
+int Water::allocated_ = 0;
+
+// This event listener monitors how many Water objects are created and
+// destroyed by each test, and reports a failure if a test leaks some Water
+// objects. It does this by comparing the number of live Water objects at
+// the beginning of a test and at the end of a test.
+class LeakChecker : public EmptyTestEventListener {
+ private:
+  // Called before a test starts.
+  virtual void OnTestStart(const TestInfo& /* test_info */) {
+    initially_allocated_ = Water::allocated();
+  }
+
+  // Called after a test ends.
+  virtual void OnTestEnd(const TestInfo& /* test_info */) {
+    int difference = Water::allocated() - initially_allocated_;
+
+    // You can generate a failure in any event handler except
+    // OnTestPartResult. Just use an appropriate Google Test assertion to do
+    // it.
+    EXPECT_LE(difference, 0) << "Leaked " << difference << " unit(s) of Water!";
+  }
+
+  int initially_allocated_;
+};
+
+TEST(ListenersTest, DoesNotLeak) {
+  Water* water = new Water;
+  delete water;
+}
+
+// This should fail when the --check_for_leaks command line flag is
+// specified.
+TEST(ListenersTest, LeaksWater) {
+  Water* water = new Water;
+  EXPECT_TRUE(water != NULL);
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  bool check_for_leaks = false;
+  if (argc > 1 && strcmp(argv[1], "--check_for_leaks") == 0 )
+    check_for_leaks = true;
+  else
+    printf("%s\n", "Run this program with --check_for_leaks to enable "
+           "custom leak checking in the tests.");
+
+  // If we are given the --check_for_leaks command line flag, installs the
+  // leak checker.
+  if (check_for_leaks) {
+    TestEventListeners& listeners = UnitTest::GetInstance()->listeners();
+
+    // Adds the leak checker to the end of the test event listener list,
+    // after the default text output printer and the default XML report
+    // generator.
+    //
+    // The order is important - it ensures that failures generated in the
+    // leak checker's OnTestEnd() method are processed by the text and XML
+    // printers *before* their OnTestEnd() methods are called, such that
+    // they are attributed to the right test. Remember that a listener
+    // receives an OnXyzStart event *after* listeners preceding it in the
+    // list received that event, and receives an OnXyzEnd event *before*
+    // listeners preceding it.
+    //
+    // We don't need to worry about deleting the new listener later, as
+    // Google Test will do it.
+    listeners.Append(new LeakChecker);
+  }
+  return RUN_ALL_TESTS();
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// This sample shows how to write a simple unit test for a function,
+// using Google C++ testing framework.
+//
+// Writing a unit test using Google C++ testing framework is easy as 1-2-3:
+
+
+// Step 1. Include necessary header files such that the stuff your
+// test logic needs is declared.
+//
+// Don't forget gtest.h, which declares the testing framework.
+
+#include <limits.h>
+#include "sample1.h"
+#include "gtest/gtest.h"
+
+
+// Step 2. Use the TEST macro to define your tests.
+//
+// TEST has two parameters: the test case name and the test name.
+// After using the macro, you should define your test logic between a
+// pair of braces.  You can use a bunch of macros to indicate the
+// success or failure of a test.  EXPECT_TRUE and EXPECT_EQ are
+// examples of such macros.  For a complete list, see gtest.h.
+//
+// <TechnicalDetails>
+//
+// In Google Test, tests are grouped into test cases.  This is how we
+// keep test code organized.  You should put logically related tests
+// into the same test case.
+//
+// The test case name and the test name should both be valid C++
+// identifiers.  And you should not use underscore (_) in the names.
+//
+// Google Test guarantees that each test you define is run exactly
+// once, but it makes no guarantee on the order the tests are
+// executed.  Therefore, you should write your tests in such a way
+// that their results don't depend on their order.
+//
+// </TechnicalDetails>
+
+
+// Tests Factorial().
+
+// Tests factorial of negative numbers.
+TEST(FactorialTest, Negative) {
+  // This test is named "Negative", and belongs to the "FactorialTest"
+  // test case.
+  EXPECT_EQ(1, Factorial(-5));
+  EXPECT_EQ(1, Factorial(-1));
+  EXPECT_GT(Factorial(-10), 0);
+
+  // <TechnicalDetails>
+  //
+  // EXPECT_EQ(expected, actual) is the same as
+  //
+  //   EXPECT_TRUE((expected) == (actual))
+  //
+  // except that it will print both the expected value and the actual
+  // value when the assertion fails.  This is very helpful for
+  // debugging.  Therefore in this case EXPECT_EQ is preferred.
+  //
+  // On the other hand, EXPECT_TRUE accepts any Boolean expression,
+  // and is thus more general.
+  //
+  // </TechnicalDetails>
+}
+
+// Tests factorial of 0.
+TEST(FactorialTest, Zero) {
+  EXPECT_EQ(1, Factorial(0));
+}
+
+// Tests factorial of positive numbers.
+TEST(FactorialTest, Positive) {
+  EXPECT_EQ(1, Factorial(1));
+  EXPECT_EQ(2, Factorial(2));
+  EXPECT_EQ(6, Factorial(3));
+  EXPECT_EQ(40320, Factorial(8));
+}
+
+
+// Tests IsPrime()
+
+// Tests negative input.
+TEST(IsPrimeTest, Negative) {
+  // This test belongs to the IsPrimeTest test case.
+
+  EXPECT_FALSE(IsPrime(-1));
+  EXPECT_FALSE(IsPrime(-2));
+  EXPECT_FALSE(IsPrime(INT_MIN));
+}
+
+// Tests some trivial cases.
+TEST(IsPrimeTest, Trivial) {
+  EXPECT_FALSE(IsPrime(0));
+  EXPECT_FALSE(IsPrime(1));
+  EXPECT_TRUE(IsPrime(2));
+  EXPECT_TRUE(IsPrime(3));
+}
+
+// Tests positive input.
+TEST(IsPrimeTest, Positive) {
+  EXPECT_FALSE(IsPrime(4));
+  EXPECT_TRUE(IsPrime(5));
+  EXPECT_FALSE(IsPrime(6));
+  EXPECT_TRUE(IsPrime(23));
+}
+
+// Step 3. Call RUN_ALL_TESTS() in main().
+//
+// We do this by linking in src/gtest_main.cc file, which consists of
+// a main() function which calls RUN_ALL_TESTS() for us.
+//
+// This runs all the tests you've defined, prints the result, and
+// returns 0 if successful, or 1 otherwise.
+//
+// Did you notice that we didn't register the tests?  The
+// RUN_ALL_TESTS() macro magically knows about all the tests we
+// defined.  Isn't this convenient?
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// This sample shows how to write a more complex unit test for a class
+// that has multiple member functions.
+//
+// Usually, it's a good idea to have one test for each method in your
+// class.  You don't have to do that exactly, but it helps to keep
+// your tests organized.  You may also throw in additional tests as
+// needed.
+
+#include "sample2.h"
+#include "gtest/gtest.h"
+
+// In this example, we test the MyString class (a simple string).
+
+// Tests the default c'tor.
+TEST(MyString, DefaultConstructor) {
+  const MyString s;
+
+  // Asserts that s.c_string() returns NULL.
+  //
+  // <TechnicalDetails>
+  //
+  // If we write NULL instead of
+  //
+  //   static_cast<const char *>(NULL)
+  //
+  // in this assertion, it will generate a warning on gcc 3.4.  The
+  // reason is that EXPECT_EQ needs to know the types of its
+  // arguments in order to print them when it fails.  Since NULL is
+  // #defined as 0, the compiler will use the formatter function for
+  // int to print it.  However, gcc thinks that NULL should be used as
+  // a pointer, not an int, and therefore complains.
+  //
+  // The root of the problem is C++'s lack of distinction between the
+  // integer number 0 and the null pointer constant.  Unfortunately,
+  // we have to live with this fact.
+  //
+  // </TechnicalDetails>
+  EXPECT_STREQ(NULL, s.c_string());
+
+  EXPECT_EQ(0u, s.Length());
+}
+
+const char kHelloString[] = "Hello, world!";
+
+// Tests the c'tor that accepts a C string.
+TEST(MyString, ConstructorFromCString) {
+  const MyString s(kHelloString);
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+  EXPECT_EQ(sizeof(kHelloString)/sizeof(kHelloString[0]) - 1,
+            s.Length());
+}
+
+// Tests the copy c'tor.
+TEST(MyString, CopyConstructor) {
+  const MyString s1(kHelloString);
+  const MyString s2 = s1;
+  EXPECT_EQ(0, strcmp(s2.c_string(), kHelloString));
+}
+
+// Tests the Set method.
+TEST(MyString, Set) {
+  MyString s;
+
+  s.Set(kHelloString);
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+
+  // Set should work when the input pointer is the same as the one
+  // already in the MyString object.
+  s.Set(s.c_string());
+  EXPECT_EQ(0, strcmp(s.c_string(), kHelloString));
+
+  // Can we set the MyString to NULL?
+  s.Set(NULL);
+  EXPECT_STREQ(NULL, s.c_string());
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// A sample program demonstrating using Google C++ testing framework.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+
+// In this example, we use a more advanced feature of Google Test called
+// test fixture.
+//
+// A test fixture is a place to hold objects and functions shared by
+// all tests in a test case.  Using a test fixture avoids duplicating
+// the test code necessary to initialize and cleanup those common
+// objects for each test.  It is also useful for defining sub-routines
+// that your tests need to invoke a lot.
+//
+// <TechnicalDetails>
+//
+// The tests share the test fixture in the sense of code sharing, not
+// data sharing.  Each test is given its own fresh copy of the
+// fixture.  You cannot expect the data modified by one test to be
+// passed on to another test, which is a bad idea.
+//
+// The reason for this design is that tests should be independent and
+// repeatable.  In particular, a test should not fail as the result of
+// another test's failure.  If one test depends on info produced by
+// another test, then the two tests should really be one big test.
+//
+// The macros for indicating the success/failure of a test
+// (EXPECT_TRUE, FAIL, etc) need to know what the current test is
+// (when Google Test prints the test result, it tells you which test
+// each failure belongs to).  Technically, these macros invoke a
+// member function of the Test class.  Therefore, you cannot use them
+// in a global function.  That's why you should put test sub-routines
+// in a test fixture.
+//
+// </TechnicalDetails>
+
+#include "sample3-inl.h"
+#include "gtest/gtest.h"
+
+// To use a test fixture, derive a class from testing::Test.
+class QueueTest : public testing::Test {
+ protected:  // You should make the members protected s.t. they can be
+             // accessed from sub-classes.
+
+  // virtual void SetUp() will be called before each test is run.  You
+  // should define it if you need to initialize the varaibles.
+  // Otherwise, this can be skipped.
+  virtual void SetUp() {
+    q1_.Enqueue(1);
+    q2_.Enqueue(2);
+    q2_.Enqueue(3);
+  }
+
+  // virtual void TearDown() will be called after each test is run.
+  // You should define it if there is cleanup work to do.  Otherwise,
+  // you don't have to provide it.
+  //
+  // virtual void TearDown() {
+  // }
+
+  // A helper function that some test uses.
+  static int Double(int n) {
+    return 2*n;
+  }
+
+  // A helper function for testing Queue::Map().
+  void MapTester(const Queue<int> * q) {
+    // Creates a new queue, where each element is twice as big as the
+    // corresponding one in q.
+    const Queue<int> * const new_q = q->Map(Double);
+
+    // Verifies that the new queue has the same size as q.
+    ASSERT_EQ(q->Size(), new_q->Size());
+
+    // Verifies the relationship between the elements of the two queues.
+    for ( const QueueNode<int> * n1 = q->Head(), * n2 = new_q->Head();
+          n1 != NULL; n1 = n1->next(), n2 = n2->next() ) {
+      EXPECT_EQ(2 * n1->element(), n2->element());
+    }
+
+    delete new_q;
+  }
+
+  // Declares the variables your tests want to use.
+  Queue<int> q0_;
+  Queue<int> q1_;
+  Queue<int> q2_;
+};
+
+// When you have a test fixture, you define a test using TEST_F
+// instead of TEST.
+
+// Tests the default c'tor.
+TEST_F(QueueTest, DefaultConstructor) {
+  // You can access data in the test fixture here.
+  EXPECT_EQ(0u, q0_.Size());
+}
+
+// Tests Dequeue().
+TEST_F(QueueTest, Dequeue) {
+  int * n = q0_.Dequeue();
+  EXPECT_TRUE(n == NULL);
+
+  n = q1_.Dequeue();
+  ASSERT_TRUE(n != NULL);
+  EXPECT_EQ(1, *n);
+  EXPECT_EQ(0u, q1_.Size());
+  delete n;
+
+  n = q2_.Dequeue();
+  ASSERT_TRUE(n != NULL);
+  EXPECT_EQ(2, *n);
+  EXPECT_EQ(1u, q2_.Size());
+  delete n;
+}
+
+// Tests the Queue::Map() function.
+TEST_F(QueueTest, Map) {
+  MapTester(&q0_);
+  MapTester(&q1_);
+  MapTester(&q2_);
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest.h"
+#include "sample4.h"
+
+// Tests the Increment() method.
+TEST(Counter, Increment) {
+  Counter c;
+
+  // EXPECT_EQ() evaluates its arguments exactly once, so they
+  // can have side effects.
+
+  EXPECT_EQ(0, c.Increment());
+  EXPECT_EQ(1, c.Increment());
+  EXPECT_EQ(2, c.Increment());
+}
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// This sample teaches how to reuse a test fixture in multiple test
+// cases by deriving sub-fixtures from it.
+//
+// When you define a test fixture, you specify the name of the test
+// case that will use this fixture.  Therefore, a test fixture can
+// be used by only one test case.
+//
+// Sometimes, more than one test cases may want to use the same or
+// slightly different test fixtures.  For example, you may want to
+// make sure that all tests for a GUI library don't leak important
+// system resources like fonts and brushes.  In Google Test, you do
+// this by putting the shared logic in a super (as in "super class")
+// test fixture, and then have each test case use a fixture derived
+// from this super fixture.
+
+#include <limits.h>
+#include <time.h>
+#include "sample3-inl.h"
+#include "gtest/gtest.h"
+#include "sample1.h"
+
+// In this sample, we want to ensure that every test finishes within
+// ~5 seconds.  If a test takes longer to run, we consider it a
+// failure.
+//
+// We put the code for timing a test in a test fixture called
+// "QuickTest".  QuickTest is intended to be the super fixture that
+// other fixtures derive from, therefore there is no test case with
+// the name "QuickTest".  This is OK.
+//
+// Later, we will derive multiple test fixtures from QuickTest.
+class QuickTest : public testing::Test {
+ protected:
+  // Remember that SetUp() is run immediately before a test starts.
+  // This is a good place to record the start time.
+  virtual void SetUp() {
+    start_time_ = time(NULL);
+  }
+
+  // TearDown() is invoked immediately after a test finishes.  Here we
+  // check if the test was too slow.
+  virtual void TearDown() {
+    // Gets the time when the test finishes
+    const time_t end_time = time(NULL);
+
+    // Asserts that the test took no more than ~5 seconds.  Did you
+    // know that you can use assertions in SetUp() and TearDown() as
+    // well?
+    EXPECT_TRUE(end_time - start_time_ <= 5) << "The test took too long.";
+  }
+
+  // The UTC time (in seconds) when the test starts
+  time_t start_time_;
+};
+
+
+// We derive a fixture named IntegerFunctionTest from the QuickTest
+// fixture.  All tests using this fixture will be automatically
+// required to be quick.
+class IntegerFunctionTest : public QuickTest {
+  // We don't need any more logic than already in the QuickTest fixture.
+  // Therefore the body is empty.
+};
+
+
+// Now we can write tests in the IntegerFunctionTest test case.
+
+// Tests Factorial()
+TEST_F(IntegerFunctionTest, Factorial) {
+  // Tests factorial of negative numbers.
+  EXPECT_EQ(1, Factorial(-5));
+  EXPECT_EQ(1, Factorial(-1));
+  EXPECT_GT(Factorial(-10), 0);
+
+  // Tests factorial of 0.
+  EXPECT_EQ(1, Factorial(0));
+
+  // Tests factorial of positive numbers.
+  EXPECT_EQ(1, Factorial(1));
+  EXPECT_EQ(2, Factorial(2));
+  EXPECT_EQ(6, Factorial(3));
+  EXPECT_EQ(40320, Factorial(8));
+}
+
+
+// Tests IsPrime()
+TEST_F(IntegerFunctionTest, IsPrime) {
+  // Tests negative input.
+  EXPECT_FALSE(IsPrime(-1));
+  EXPECT_FALSE(IsPrime(-2));
+  EXPECT_FALSE(IsPrime(INT_MIN));
+
+  // Tests some trivial cases.
+  EXPECT_FALSE(IsPrime(0));
+  EXPECT_FALSE(IsPrime(1));
+  EXPECT_TRUE(IsPrime(2));
+  EXPECT_TRUE(IsPrime(3));
+
+  // Tests positive input.
+  EXPECT_FALSE(IsPrime(4));
+  EXPECT_TRUE(IsPrime(5));
+  EXPECT_FALSE(IsPrime(6));
+  EXPECT_TRUE(IsPrime(23));
+}
+
+
+// The next test case (named "QueueTest") also needs to be quick, so
+// we derive another fixture from QuickTest.
+//
+// The QueueTest test fixture has some logic and shared objects in
+// addition to what's in QuickTest already.  We define the additional
+// stuff inside the body of the test fixture, as usual.
+class QueueTest : public QuickTest {
+ protected:
+  virtual void SetUp() {
+    // First, we need to set up the super fixture (QuickTest).
+    QuickTest::SetUp();
+
+    // Second, some additional setup for this fixture.
+    q1_.Enqueue(1);
+    q2_.Enqueue(2);
+    q2_.Enqueue(3);
+  }
+
+  // By default, TearDown() inherits the behavior of
+  // QuickTest::TearDown().  As we have no additional cleaning work
+  // for QueueTest, we omit it here.
+  //
+  // virtual void TearDown() {
+  //   QuickTest::TearDown();
+  // }
+
+  Queue<int> q0_;
+  Queue<int> q1_;
+  Queue<int> q2_;
+};
+
+
+// Now, let's write tests using the QueueTest fixture.
+
+// Tests the default constructor.
+TEST_F(QueueTest, DefaultConstructor) {
+  EXPECT_EQ(0u, q0_.Size());
+}
+
+// Tests Dequeue().
+TEST_F(QueueTest, Dequeue) {
+  int* n = q0_.Dequeue();
+  EXPECT_TRUE(n == NULL);
+
+  n = q1_.Dequeue();
+  EXPECT_TRUE(n != NULL);
+  EXPECT_EQ(1, *n);
+  EXPECT_EQ(0u, q1_.Size());
+  delete n;
+
+  n = q2_.Dequeue();
+  EXPECT_TRUE(n != NULL);
+  EXPECT_EQ(2, *n);
+  EXPECT_EQ(1u, q2_.Size());
+  delete n;
+}
+
+// If necessary, you can derive further test fixtures from a derived
+// fixture itself.  For example, you can derive another fixture from
+// QueueTest.  Google Test imposes no limit on how deep the hierarchy
+// can be.  In practice, however, you probably don't want it to be too
+// deep as to be confusing.
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// This sample shows how to test common properties of multiple
+// implementations of the same interface (aka interface tests).
+
+// The interface and its implementations are in this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+// First, we define some factory functions for creating instances of
+// the implementations.  You may be able to skip this step if all your
+// implementations can be constructed the same way.
+
+template <class T>
+PrimeTable* CreatePrimeTable();
+
+template <>
+PrimeTable* CreatePrimeTable<OnTheFlyPrimeTable>() {
+  return new OnTheFlyPrimeTable;
+}
+
+template <>
+PrimeTable* CreatePrimeTable<PreCalculatedPrimeTable>() {
+  return new PreCalculatedPrimeTable(10000);
+}
+
+// Then we define a test fixture class template.
+template <class T>
+class PrimeTableTest : public testing::Test {
+ protected:
+  // The ctor calls the factory function to create a prime table
+  // implemented by T.
+  PrimeTableTest() : table_(CreatePrimeTable<T>()) {}
+
+  virtual ~PrimeTableTest() { delete table_; }
+
+  // Note that we test an implementation via the base interface
+  // instead of the actual implementation class.  This is important
+  // for keeping the tests close to the real world scenario, where the
+  // implementation is invoked via the base interface.  It avoids
+  // got-yas where the implementation class has a method that shadows
+  // a method with the same name (but slightly different argument
+  // types) in the base interface, for example.
+  PrimeTable* const table_;
+};
+
+#if GTEST_HAS_TYPED_TEST
+
+using testing::Types;
+
+// Google Test offers two ways for reusing tests for different types.
+// The first is called "typed tests".  You should use it if you
+// already know *all* the types you are gonna exercise when you write
+// the tests.
+
+// To write a typed test case, first use
+//
+//   TYPED_TEST_CASE(TestCaseName, TypeList);
+//
+// to declare it and specify the type parameters.  As with TEST_F,
+// TestCaseName must match the test fixture name.
+
+// The list of types we want to test.
+typedef Types<OnTheFlyPrimeTable, PreCalculatedPrimeTable> Implementations;
+
+TYPED_TEST_CASE(PrimeTableTest, Implementations);
+
+// Then use TYPED_TEST(TestCaseName, TestName) to define a typed test,
+// similar to TEST_F.
+TYPED_TEST(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  // Inside the test body, you can refer to the type parameter by
+  // TypeParam, and refer to the fixture class by TestFixture.  We
+  // don't need them in this example.
+
+  // Since we are in the template world, C++ requires explicitly
+  // writing 'this->' when referring to members of the fixture class.
+  // This is something you have to learn to live with.
+  EXPECT_FALSE(this->table_->IsPrime(-5));
+  EXPECT_FALSE(this->table_->IsPrime(0));
+  EXPECT_FALSE(this->table_->IsPrime(1));
+  EXPECT_FALSE(this->table_->IsPrime(4));
+  EXPECT_FALSE(this->table_->IsPrime(6));
+  EXPECT_FALSE(this->table_->IsPrime(100));
+}
+
+TYPED_TEST(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(this->table_->IsPrime(2));
+  EXPECT_TRUE(this->table_->IsPrime(3));
+  EXPECT_TRUE(this->table_->IsPrime(5));
+  EXPECT_TRUE(this->table_->IsPrime(7));
+  EXPECT_TRUE(this->table_->IsPrime(11));
+  EXPECT_TRUE(this->table_->IsPrime(131));
+}
+
+TYPED_TEST(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, this->table_->GetNextPrime(0));
+  EXPECT_EQ(3, this->table_->GetNextPrime(2));
+  EXPECT_EQ(5, this->table_->GetNextPrime(3));
+  EXPECT_EQ(7, this->table_->GetNextPrime(5));
+  EXPECT_EQ(11, this->table_->GetNextPrime(7));
+  EXPECT_EQ(131, this->table_->GetNextPrime(128));
+}
+
+// That's it!  Google Test will repeat each TYPED_TEST for each type
+// in the type list specified in TYPED_TEST_CASE.  Sit back and be
+// happy that you don't have to define them multiple times.
+
+#endif  // GTEST_HAS_TYPED_TEST
+
+#if GTEST_HAS_TYPED_TEST_P
+
+using testing::Types;
+
+// Sometimes, however, you don't yet know all the types that you want
+// to test when you write the tests.  For example, if you are the
+// author of an interface and expect other people to implement it, you
+// might want to write a set of tests to make sure each implementation
+// conforms to some basic requirements, but you don't know what
+// implementations will be written in the future.
+//
+// How can you write the tests without committing to the type
+// parameters?  That's what "type-parameterized tests" can do for you.
+// It is a bit more involved than typed tests, but in return you get a
+// test pattern that can be reused in many contexts, which is a big
+// win.  Here's how you do it:
+
+// First, define a test fixture class template.  Here we just reuse
+// the PrimeTableTest fixture defined earlier:
+
+template <class T>
+class PrimeTableTest2 : public PrimeTableTest<T> {
+};
+
+// Then, declare the test case.  The argument is the name of the test
+// fixture, and also the name of the test case (as usual).  The _P
+// suffix is for "parameterized" or "pattern".
+TYPED_TEST_CASE_P(PrimeTableTest2);
+
+// Next, use TYPED_TEST_P(TestCaseName, TestName) to define a test,
+// similar to what you do with TEST_F.
+TYPED_TEST_P(PrimeTableTest2, ReturnsFalseForNonPrimes) {
+  EXPECT_FALSE(this->table_->IsPrime(-5));
+  EXPECT_FALSE(this->table_->IsPrime(0));
+  EXPECT_FALSE(this->table_->IsPrime(1));
+  EXPECT_FALSE(this->table_->IsPrime(4));
+  EXPECT_FALSE(this->table_->IsPrime(6));
+  EXPECT_FALSE(this->table_->IsPrime(100));
+}
+
+TYPED_TEST_P(PrimeTableTest2, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(this->table_->IsPrime(2));
+  EXPECT_TRUE(this->table_->IsPrime(3));
+  EXPECT_TRUE(this->table_->IsPrime(5));
+  EXPECT_TRUE(this->table_->IsPrime(7));
+  EXPECT_TRUE(this->table_->IsPrime(11));
+  EXPECT_TRUE(this->table_->IsPrime(131));
+}
+
+TYPED_TEST_P(PrimeTableTest2, CanGetNextPrime) {
+  EXPECT_EQ(2, this->table_->GetNextPrime(0));
+  EXPECT_EQ(3, this->table_->GetNextPrime(2));
+  EXPECT_EQ(5, this->table_->GetNextPrime(3));
+  EXPECT_EQ(7, this->table_->GetNextPrime(5));
+  EXPECT_EQ(11, this->table_->GetNextPrime(7));
+  EXPECT_EQ(131, this->table_->GetNextPrime(128));
+}
+
+// Type-parameterized tests involve one extra step: you have to
+// enumerate the tests you defined:
+REGISTER_TYPED_TEST_CASE_P(
+    PrimeTableTest2,  // The first argument is the test case name.
+    // The rest of the arguments are the test names.
+    ReturnsFalseForNonPrimes, ReturnsTrueForPrimes, CanGetNextPrime);
+
+// At this point the test pattern is done.  However, you don't have
+// any real test yet as you haven't said which types you want to run
+// the tests with.
+
+// To turn the abstract test pattern into real tests, you instantiate
+// it with a list of types.  Usually the test pattern will be defined
+// in a .h file, and anyone can #include and instantiate it.  You can
+// even instantiate it more than once in the same program.  To tell
+// different instances apart, you give each of them a name, which will
+// become part of the test case name and can be used in test filters.
+
+// The list of types we want to test.  Note that it doesn't have to be
+// defined at the time we write the TYPED_TEST_P()s.
+typedef Types<OnTheFlyPrimeTable, PreCalculatedPrimeTable>
+    PrimeTableImplementations;
+INSTANTIATE_TYPED_TEST_CASE_P(OnTheFlyAndPreCalculated,    // Instance name
+                              PrimeTableTest2,             // Test case name
+                              PrimeTableImplementations);  // Type list
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to test common properties of multiple
+// implementations of an interface (aka interface tests) using
+// value-parameterized tests. Each test in the test case has
+// a parameter that is an interface pointer to an implementation
+// tested.
+
+// The interface and its implementations are in this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_PARAM_TEST
+
+using ::testing::TestWithParam;
+using ::testing::Values;
+
+// As a general rule, to prevent a test from affecting the tests that come
+// after it, you should create and destroy the tested objects for each test
+// instead of reusing them.  In this sample we will define a simple factory
+// function for PrimeTable objects.  We will instantiate objects in test's
+// SetUp() method and delete them in TearDown() method.
+typedef PrimeTable* CreatePrimeTableFunc();
+
+PrimeTable* CreateOnTheFlyPrimeTable() {
+  return new OnTheFlyPrimeTable();
+}
+
+template <size_t max_precalculated>
+PrimeTable* CreatePreCalculatedPrimeTable() {
+  return new PreCalculatedPrimeTable(max_precalculated);
+}
+
+// Inside the test body, fixture constructor, SetUp(), and TearDown() you
+// can refer to the test parameter by GetParam().  In this case, the test
+// parameter is a factory function which we call in fixture's SetUp() to
+// create and store an instance of PrimeTable.
+class PrimeTableTest : public TestWithParam<CreatePrimeTableFunc*> {
+ public:
+  virtual ~PrimeTableTest() { delete table_; }
+  virtual void SetUp() { table_ = (*GetParam())(); }
+  virtual void TearDown() {
+    delete table_;
+    table_ = NULL;
+  }
+
+ protected:
+  PrimeTable* table_;
+};
+
+TEST_P(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  EXPECT_FALSE(table_->IsPrime(-5));
+  EXPECT_FALSE(table_->IsPrime(0));
+  EXPECT_FALSE(table_->IsPrime(1));
+  EXPECT_FALSE(table_->IsPrime(4));
+  EXPECT_FALSE(table_->IsPrime(6));
+  EXPECT_FALSE(table_->IsPrime(100));
+}
+
+TEST_P(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(table_->IsPrime(2));
+  EXPECT_TRUE(table_->IsPrime(3));
+  EXPECT_TRUE(table_->IsPrime(5));
+  EXPECT_TRUE(table_->IsPrime(7));
+  EXPECT_TRUE(table_->IsPrime(11));
+  EXPECT_TRUE(table_->IsPrime(131));
+}
+
+TEST_P(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, table_->GetNextPrime(0));
+  EXPECT_EQ(3, table_->GetNextPrime(2));
+  EXPECT_EQ(5, table_->GetNextPrime(3));
+  EXPECT_EQ(7, table_->GetNextPrime(5));
+  EXPECT_EQ(11, table_->GetNextPrime(7));
+  EXPECT_EQ(131, table_->GetNextPrime(128));
+}
+
+// In order to run value-parameterized tests, you need to instantiate them,
+// or bind them to a list of values which will be used as test parameters.
+// You can instantiate them in a different translation module, or even
+// instantiate them several times.
+//
+// Here, we instantiate our tests with a list of two PrimeTable object
+// factory functions:
+INSTANTIATE_TEST_CASE_P(
+    OnTheFlyAndPreCalculated,
+    PrimeTableTest,
+    Values(&CreateOnTheFlyPrimeTable, &CreatePreCalculatedPrimeTable<1000>));
+
+#else
+
+// Google Test may not support value-parameterized tests with some
+// compilers. If we use conditional compilation to compile out all
+// code referring to the gtest_main library, MSVC linker will not link
+// that library at all and consequently complain about missing entry
+// point defined in that library (fatal error LNK1561: entry point
+// must be defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
+
+#endif  // GTEST_HAS_PARAM_TEST
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to test code relying on some global flag variables.
+// Combine() helps with generating all possible combinations of such flags,
+// and each test is given one combination as a parameter.
+
+// Use class definitions to test from this header.
+#include "prime_tables.h"
+
+#include "gtest/gtest.h"
+
+#if GTEST_HAS_COMBINE
+
+// Suppose we want to introduce a new, improved implementation of PrimeTable
+// which combines speed of PrecalcPrimeTable and versatility of
+// OnTheFlyPrimeTable (see prime_tables.h). Inside it instantiates both
+// PrecalcPrimeTable and OnTheFlyPrimeTable and uses the one that is more
+// appropriate under the circumstances. But in low memory conditions, it can be
+// told to instantiate without PrecalcPrimeTable instance at all and use only
+// OnTheFlyPrimeTable.
+class HybridPrimeTable : public PrimeTable {
+ public:
+  HybridPrimeTable(bool force_on_the_fly, int max_precalculated)
+      : on_the_fly_impl_(new OnTheFlyPrimeTable),
+        precalc_impl_(force_on_the_fly ? NULL :
+                          new PreCalculatedPrimeTable(max_precalculated)),
+        max_precalculated_(max_precalculated) {}
+  virtual ~HybridPrimeTable() {
+    delete on_the_fly_impl_;
+    delete precalc_impl_;
+  }
+
+  virtual bool IsPrime(int n) const {
+    if (precalc_impl_ != NULL && n < max_precalculated_)
+      return precalc_impl_->IsPrime(n);
+    else
+      return on_the_fly_impl_->IsPrime(n);
+  }
+
+  virtual int GetNextPrime(int p) const {
+    int next_prime = -1;
+    if (precalc_impl_ != NULL && p < max_precalculated_)
+      next_prime = precalc_impl_->GetNextPrime(p);
+
+    return next_prime != -1 ? next_prime : on_the_fly_impl_->GetNextPrime(p);
+  }
+
+ private:
+  OnTheFlyPrimeTable* on_the_fly_impl_;
+  PreCalculatedPrimeTable* precalc_impl_;
+  int max_precalculated_;
+};
+
+using ::testing::TestWithParam;
+using ::testing::Bool;
+using ::testing::Values;
+using ::testing::Combine;
+
+// To test all code paths for HybridPrimeTable we must test it with numbers
+// both within and outside PreCalculatedPrimeTable's capacity and also with
+// PreCalculatedPrimeTable disabled. We do this by defining fixture which will
+// accept different combinations of parameters for instantiating a
+// HybridPrimeTable instance.
+class PrimeTableTest : public TestWithParam< ::testing::tuple<bool, int> > {
+ protected:
+  virtual void SetUp() {
+    // This can be written as
+    //
+    // bool force_on_the_fly;
+    // int max_precalculated;
+    // tie(force_on_the_fly, max_precalculated) = GetParam();
+    //
+    // once the Google C++ Style Guide allows use of ::std::tr1::tie.
+    //
+    bool force_on_the_fly = ::testing::get<0>(GetParam());
+    int max_precalculated = ::testing::get<1>(GetParam());
+    table_ = new HybridPrimeTable(force_on_the_fly, max_precalculated);
+  }
+  virtual void TearDown() {
+    delete table_;
+    table_ = NULL;
+  }
+  HybridPrimeTable* table_;
+};
+
+TEST_P(PrimeTableTest, ReturnsFalseForNonPrimes) {
+  // Inside the test body, you can refer to the test parameter by GetParam().
+  // In this case, the test parameter is a PrimeTable interface pointer which
+  // we can use directly.
+  // Please note that you can also save it in the fixture's SetUp() method
+  // or constructor and use saved copy in the tests.
+
+  EXPECT_FALSE(table_->IsPrime(-5));
+  EXPECT_FALSE(table_->IsPrime(0));
+  EXPECT_FALSE(table_->IsPrime(1));
+  EXPECT_FALSE(table_->IsPrime(4));
+  EXPECT_FALSE(table_->IsPrime(6));
+  EXPECT_FALSE(table_->IsPrime(100));
+}
+
+TEST_P(PrimeTableTest, ReturnsTrueForPrimes) {
+  EXPECT_TRUE(table_->IsPrime(2));
+  EXPECT_TRUE(table_->IsPrime(3));
+  EXPECT_TRUE(table_->IsPrime(5));
+  EXPECT_TRUE(table_->IsPrime(7));
+  EXPECT_TRUE(table_->IsPrime(11));
+  EXPECT_TRUE(table_->IsPrime(131));
+}
+
+TEST_P(PrimeTableTest, CanGetNextPrime) {
+  EXPECT_EQ(2, table_->GetNextPrime(0));
+  EXPECT_EQ(3, table_->GetNextPrime(2));
+  EXPECT_EQ(5, table_->GetNextPrime(3));
+  EXPECT_EQ(7, table_->GetNextPrime(5));
+  EXPECT_EQ(11, table_->GetNextPrime(7));
+  EXPECT_EQ(131, table_->GetNextPrime(128));
+}
+
+// In order to run value-parameterized tests, you need to instantiate them,
+// or bind them to a list of values which will be used as test parameters.
+// You can instantiate them in a different translation module, or even
+// instantiate them several times.
+//
+// Here, we instantiate our tests with a list of parameters. We must combine
+// all variations of the boolean flag suppressing PrecalcPrimeTable and some
+// meaningful values for tests. We choose a small value (1), and a value that
+// will put some of the tested numbers beyond the capability of the
+// PrecalcPrimeTable instance and some inside it (10). Combine will produce all
+// possible combinations.
+INSTANTIATE_TEST_CASE_P(MeaningfulTestParameters,
+                        PrimeTableTest,
+                        Combine(Bool(), Values(1, 10)));
+
+#else
+
+// Google Test may not support Combine() with some compilers. If we
+// use conditional compilation to compile out all code referring to
+// the gtest_main library, MSVC linker will not link that library at
+// all and consequently complain about missing entry point defined in
+// that library (fatal error LNK1561: entry point must be
+// defined). This dummy test keeps gtest_main linked in.
+TEST(DummyTest, CombineIsNotSupportedOnThisPlatform) {}
+
+#endif  // GTEST_HAS_COMBINE
+// Copyright 2009 Google Inc. All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: vladl@google.com (Vlad Losev)
+
+// This sample shows how to use Google Test listener API to implement
+// an alternative console output and how to use the UnitTest reflection API
+// to enumerate test cases and tests and to inspect their results.
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+using ::testing::EmptyTestEventListener;
+using ::testing::InitGoogleTest;
+using ::testing::Test;
+using ::testing::TestCase;
+using ::testing::TestEventListeners;
+using ::testing::TestInfo;
+using ::testing::TestPartResult;
+using ::testing::UnitTest;
+
+namespace {
+
+// Provides alternative output mode which produces minimal amount of
+// information about tests.
+class TersePrinter : public EmptyTestEventListener {
+ private:
+  // Called before any test activity starts.
+  virtual void OnTestProgramStart(const UnitTest& /* unit_test */) {}
+
+  // Called after all test activities have ended.
+  virtual void OnTestProgramEnd(const UnitTest& unit_test) {
+    fprintf(stdout, "TEST %s\n", unit_test.Passed() ? "PASSED" : "FAILED");
+    fflush(stdout);
+  }
+
+  // Called before a test starts.
+  virtual void OnTestStart(const TestInfo& test_info) {
+    fprintf(stdout,
+            "*** Test %s.%s starting.\n",
+            test_info.test_case_name(),
+            test_info.name());
+    fflush(stdout);
+  }
+
+  // Called after a failed assertion or a SUCCEED() invocation.
+  virtual void OnTestPartResult(const TestPartResult& test_part_result) {
+    fprintf(stdout,
+            "%s in %s:%d\n%s\n",
+            test_part_result.failed() ? "*** Failure" : "Success",
+            test_part_result.file_name(),
+            test_part_result.line_number(),
+            test_part_result.summary());
+    fflush(stdout);
+  }
+
+  // Called after a test ends.
+  virtual void OnTestEnd(const TestInfo& test_info) {
+    fprintf(stdout,
+            "*** Test %s.%s ending.\n",
+            test_info.test_case_name(),
+            test_info.name());
+    fflush(stdout);
+  }
+};  // class TersePrinter
+
+TEST(CustomOutputTest, PrintsMessage) {
+  printf("Printing something from the test body...\n");
+}
+
+TEST(CustomOutputTest, Succeeds) {
+  SUCCEED() << "SUCCEED() has been invoked from here";
+}
+
+TEST(CustomOutputTest, Fails) {
+  EXPECT_EQ(1, 2)
+      << "This test fails in order to demonstrate alternative failure messages";
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  InitGoogleTest(&argc, argv);
+
+  bool terse_output = false;
+  if (argc > 1 && strcmp(argv[1], "--terse_output") == 0 )
+    terse_output = true;
+  else
+    printf("%s\n", "Run this program with --terse_output to change the way "
+           "it prints its output.");
+
+  UnitTest& unit_test = *UnitTest::GetInstance();
+
+  // If we are given the --terse_output command line flag, suppresses the
+  // standard output and attaches own result printer.
+  if (terse_output) {
+    TestEventListeners& listeners = unit_test.listeners();
+
+    // Removes the default console output listener from the list so it will
+    // not receive events from Google Test and won't print any output. Since
+    // this operation transfers ownership of the listener to the caller we
+    // have to delete it as well.
+    delete listeners.Release(listeners.default_result_printer());
+
+    // Adds the custom output listener to the list. It will now receive
+    // events from Google Test and print the alternative output. We don't
+    // have to worry about deleting it since Google Test assumes ownership
+    // over it after adding it to the list.
+    listeners.Append(new TersePrinter);
+  }
+  int ret_val = RUN_ALL_TESTS();
+
+  // This is an example of using the UnitTest reflection API to inspect test
+  // results. Here we discount failures from the tests we expected to fail.
+  int unexpectedly_failed_tests = 0;
+  for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
+    const TestCase& test_case = *unit_test.GetTestCase(i);
+    for (int j = 0; j < test_case.total_test_count(); ++j) {
+      const TestInfo& test_info = *test_case.GetTestInfo(j);
+      // Counts failed tests that were not meant to fail (those without
+      // 'Fails' in the name).
+      if (test_info.result()->Failed() &&
+          strcmp(test_info.name(), "Fails") != 0) {
+        unexpectedly_failed_tests++;
+      }
+    }
+  }
+
+  // Test that were meant to fail should not affect the test program outcome.
+  if (unexpectedly_failed_tests == 0)
+    ret_val = 0;
+
+  return ret_val;
+}
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+// Google C++ Testing Framework (Google Test)
+//
+// Sometimes it's desirable to build Google Test by compiling a single file.
+// This file serves this purpose.
+
+// This line ensures that gtest.h can be compiled on its own, even
+// when it's fused.
+#include "gtest/gtest.h"
+
+// The following lines pull in the real gtest *.cc files.
+#include "src/gtest.cc"
+#include "src/gtest-death-test.cc"
+#include "src/gtest-filepath.cc"
+#include "src/gtest-port.cc"
+#include "src/gtest-printers.cc"
+#include "src/gtest-test-part.cc"
+#include "src/gtest-typed-test.cc"
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan), vladl@google.com (Vlad Losev)
+//
+// This file implements death tests.
+
+#include "gtest/gtest-death-test.h"
+#include "gtest/internal/gtest-port.h"
+#include "gtest/internal/custom/gtest.h"
+
+#if GTEST_HAS_DEATH_TEST
+
+# if GTEST_OS_MAC
+#  include <crt_externs.h>
+# endif  // GTEST_OS_MAC
+
+# include <errno.h>
+# include <fcntl.h>
+# include <limits.h>
+
+# if GTEST_OS_LINUX
+#  include <signal.h>
+# endif  // GTEST_OS_LINUX
+
+# include <stdarg.h>
+
+# if GTEST_OS_WINDOWS
+#  include <windows.h>
+# else
+#  include <sys/mman.h>
+#  include <sys/wait.h>
+# endif  // GTEST_OS_WINDOWS
+
+# if GTEST_OS_QNX
+#  include <spawn.h>
+# endif  // GTEST_OS_QNX
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-string.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+
+// Constants.
+
+// The default death test style.
+static const char kDefaultDeathTestStyle[] = "fast";
+
+GTEST_DEFINE_string_(
+    death_test_style,
+    internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle),
+    "Indicates how to run a death test in a forked child process: "
+    "\"threadsafe\" (child process re-executes the test binary "
+    "from the beginning, running only the specific death test) or "
+    "\"fast\" (child process runs the death test immediately "
+    "after forking).");
+
+GTEST_DEFINE_bool_(
+    death_test_use_fork,
+    internal::BoolFromGTestEnv("death_test_use_fork", false),
+    "Instructs to use fork()/_exit() instead of clone() in death tests. "
+    "Ignored and always uses fork() on POSIX systems where clone() is not "
+    "implemented. Useful when running under valgrind or similar tools if "
+    "those do not support clone(). Valgrind 3.3.1 will just fail if "
+    "it sees an unsupported combination of clone() flags. "
+    "It is not recommended to use this flag w/o valgrind though it will "
+    "work in 99% of the cases. Once valgrind is fixed, this flag will "
+    "most likely be removed.");
+
+namespace internal {
+GTEST_DEFINE_string_(
+    internal_run_death_test, "",
+    "Indicates the file, line number, temporal index of "
+    "the single death test to run, and a file descriptor to "
+    "which a success code may be sent, all separated by "
+    "the '|' characters.  This flag is specified if and only if the current "
+    "process is a sub-process launched for running a thread-safe "
+    "death test.  FOR INTERNAL USE ONLY.");
+}  // namespace internal
+
+#if GTEST_HAS_DEATH_TEST
+
+namespace internal {
+
+// Valid only for fast death tests. Indicates the code is running in the
+// child process of a fast style death test.
+# if !GTEST_OS_WINDOWS
+static bool g_in_fast_death_test_child = false;
+# endif
+
+// Returns a Boolean value indicating whether the caller is currently
+// executing in the context of the death test child process.  Tools such as
+// Valgrind heap checkers may need this to modify their behavior in death
+// tests.  IMPORTANT: This is an internal utility.  Using it may break the
+// implementation of death tests.  User code MUST NOT use it.
+bool InDeathTestChild() {
+# if GTEST_OS_WINDOWS
+
+  // On Windows, death tests are thread-safe regardless of the value of the
+  // death_test_style flag.
+  return !GTEST_FLAG(internal_run_death_test).empty();
+
+# else
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe")
+    return !GTEST_FLAG(internal_run_death_test).empty();
+  else
+    return g_in_fast_death_test_child;
+#endif
+}
+
+}  // namespace internal
+
+// ExitedWithCode constructor.
+ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) {
+}
+
+// ExitedWithCode function-call operator.
+bool ExitedWithCode::operator()(int exit_status) const {
+# if GTEST_OS_WINDOWS
+
+  return exit_status == exit_code_;
+
+# else
+
+  return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_;
+
+# endif  // GTEST_OS_WINDOWS
+}
+
+# if !GTEST_OS_WINDOWS
+// KilledBySignal constructor.
+KilledBySignal::KilledBySignal(int signum) : signum_(signum) {
+}
+
+// KilledBySignal function-call operator.
+bool KilledBySignal::operator()(int exit_status) const {
+#  if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
+  {
+    bool result;
+    if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) {
+      return result;
+    }
+  }
+#  endif  // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_)
+  return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_;
+}
+# endif  // !GTEST_OS_WINDOWS
+
+namespace internal {
+
+// Utilities needed for death tests.
+
+// Generates a textual description of a given exit code, in the format
+// specified by wait(2).
+static std::string ExitSummary(int exit_code) {
+  Message m;
+
+# if GTEST_OS_WINDOWS
+
+  m << "Exited with exit status " << exit_code;
+
+# else
+
+  if (WIFEXITED(exit_code)) {
+    m << "Exited with exit status " << WEXITSTATUS(exit_code);
+  } else if (WIFSIGNALED(exit_code)) {
+    m << "Terminated by signal " << WTERMSIG(exit_code);
+  }
+#  ifdef WCOREDUMP
+  if (WCOREDUMP(exit_code)) {
+    m << " (core dumped)";
+  }
+#  endif
+# endif  // GTEST_OS_WINDOWS
+
+  return m.GetString();
+}
+
+// Returns true if exit_status describes a process that was terminated
+// by a signal, or exited normally with a nonzero exit code.
+bool ExitedUnsuccessfully(int exit_status) {
+  return !ExitedWithCode(0)(exit_status);
+}
+
+# if !GTEST_OS_WINDOWS
+// Generates a textual failure message when a death test finds more than
+// one thread running, or cannot determine the number of threads, prior
+// to executing the given statement.  It is the responsibility of the
+// caller not to pass a thread_count of 1.
+static std::string DeathTestThreadWarning(size_t thread_count) {
+  Message msg;
+  msg << "Death tests use fork(), which is unsafe particularly"
+      << " in a threaded context. For this test, " << GTEST_NAME_ << " ";
+  if (thread_count == 0)
+    msg << "couldn't detect the number of threads.";
+  else
+    msg << "detected " << thread_count << " threads.";
+  return msg.GetString();
+}
+# endif  // !GTEST_OS_WINDOWS
+
+// Flag characters for reporting a death test that did not die.
+static const char kDeathTestLived = 'L';
+static const char kDeathTestReturned = 'R';
+static const char kDeathTestThrew = 'T';
+static const char kDeathTestInternalError = 'I';
+
+// An enumeration describing all of the possible ways that a death test can
+// conclude.  DIED means that the process died while executing the test
+// code; LIVED means that process lived beyond the end of the test code;
+// RETURNED means that the test statement attempted to execute a return
+// statement, which is not allowed; THREW means that the test statement
+// returned control by throwing an exception.  IN_PROGRESS means the test
+// has not yet concluded.
+// TODO(vladl@google.com): Unify names and possibly values for
+// AbortReason, DeathTestOutcome, and flag characters above.
+enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW };
+
+// Routine for aborting the program which is safe to call from an
+// exec-style death test child process, in which case the error
+// message is propagated back to the parent process.  Otherwise, the
+// message is simply printed to stderr.  In either case, the program
+// then exits with status 1.
+void DeathTestAbort(const std::string& message) {
+  // On a POSIX system, this function may be called from a threadsafe-style
+  // death test child process, which operates on a very small stack.  Use
+  // the heap for any additional non-minuscule memory requirements.
+  const InternalRunDeathTestFlag* const flag =
+      GetUnitTestImpl()->internal_run_death_test_flag();
+  if (flag != NULL) {
+    FILE* parent = posix::FDOpen(flag->write_fd(), "w");
+    fputc(kDeathTestInternalError, parent);
+    fprintf(parent, "%s", message.c_str());
+    fflush(parent);
+    _exit(1);
+  } else {
+    fprintf(stderr, "%s", message.c_str());
+    fflush(stderr);
+    posix::Abort();
+  }
+}
+
+// A replacement for CHECK that calls DeathTestAbort if the assertion
+// fails.
+# define GTEST_DEATH_TEST_CHECK_(expression) \
+  do { \
+    if (!::testing::internal::IsTrue(expression)) { \
+      DeathTestAbort( \
+          ::std::string("CHECK failed: File ") + __FILE__ +  ", line " \
+          + ::testing::internal::StreamableToString(__LINE__) + ": " \
+          + #expression); \
+    } \
+  } while (::testing::internal::AlwaysFalse())
+
+// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for
+// evaluating any system call that fulfills two conditions: it must return
+// -1 on failure, and set errno to EINTR when it is interrupted and
+// should be tried again.  The macro expands to a loop that repeatedly
+// evaluates the expression as long as it evaluates to -1 and sets
+// errno to EINTR.  If the expression evaluates to -1 but errno is
+// something other than EINTR, DeathTestAbort is called.
+# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \
+  do { \
+    int gtest_retval; \
+    do { \
+      gtest_retval = (expression); \
+    } while (gtest_retval == -1 && errno == EINTR); \
+    if (gtest_retval == -1) { \
+      DeathTestAbort( \
+          ::std::string("CHECK failed: File ") + __FILE__ + ", line " \
+          + ::testing::internal::StreamableToString(__LINE__) + ": " \
+          + #expression + " != -1"); \
+    } \
+  } while (::testing::internal::AlwaysFalse())
+
+// Returns the message describing the last system error in errno.
+std::string GetLastErrnoDescription() {
+    return errno == 0 ? "" : posix::StrError(errno);
+}
+
+// This is called from a death test parent process to read a failure
+// message from the death test child process and log it with the FATAL
+// severity. On Windows, the message is read from a pipe handle. On other
+// platforms, it is read from a file descriptor.
+static void FailFromInternalError(int fd) {
+  Message error;
+  char buffer[256];
+  int num_read;
+
+  do {
+    while ((num_read = posix::Read(fd, buffer, 255)) > 0) {
+      buffer[num_read] = '\0';
+      error << buffer;
+    }
+  } while (num_read == -1 && errno == EINTR);
+
+  if (num_read == 0) {
+    GTEST_LOG_(FATAL) << error.GetString();
+  } else {
+    const int last_error = errno;
+    GTEST_LOG_(FATAL) << "Error while reading death test internal: "
+                      << GetLastErrnoDescription() << " [" << last_error << "]";
+  }
+}
+
+// Death test constructor.  Increments the running death test count
+// for the current test.
+DeathTest::DeathTest() {
+  TestInfo* const info = GetUnitTestImpl()->current_test_info();
+  if (info == NULL) {
+    DeathTestAbort("Cannot run a death test outside of a TEST or "
+                   "TEST_F construct");
+  }
+}
+
+// Creates and returns a death test by dispatching to the current
+// death test factory.
+bool DeathTest::Create(const char* statement, const RE* regex,
+                       const char* file, int line, DeathTest** test) {
+  return GetUnitTestImpl()->death_test_factory()->Create(
+      statement, regex, file, line, test);
+}
+
+const char* DeathTest::LastMessage() {
+  return last_death_test_message_.c_str();
+}
+
+void DeathTest::set_last_death_test_message(const std::string& message) {
+  last_death_test_message_ = message;
+}
+
+std::string DeathTest::last_death_test_message_;
+
+// Provides cross platform implementation for some death functionality.
+class DeathTestImpl : public DeathTest {
+ protected:
+  DeathTestImpl(const char* a_statement, const RE* a_regex)
+      : statement_(a_statement),
+        regex_(a_regex),
+        spawned_(false),
+        status_(-1),
+        outcome_(IN_PROGRESS),
+        read_fd_(-1),
+        write_fd_(-1) {}
+
+  // read_fd_ is expected to be closed and cleared by a derived class.
+  ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); }
+
+  void Abort(AbortReason reason);
+  virtual bool Passed(bool status_ok);
+
+  const char* statement() const { return statement_; }
+  const RE* regex() const { return regex_; }
+  bool spawned() const { return spawned_; }
+  void set_spawned(bool is_spawned) { spawned_ = is_spawned; }
+  int status() const { return status_; }
+  void set_status(int a_status) { status_ = a_status; }
+  DeathTestOutcome outcome() const { return outcome_; }
+  void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; }
+  int read_fd() const { return read_fd_; }
+  void set_read_fd(int fd) { read_fd_ = fd; }
+  int write_fd() const { return write_fd_; }
+  void set_write_fd(int fd) { write_fd_ = fd; }
+
+  // Called in the parent process only. Reads the result code of the death
+  // test child process via a pipe, interprets it to set the outcome_
+  // member, and closes read_fd_.  Outputs diagnostics and terminates in
+  // case of unexpected codes.
+  void ReadAndInterpretStatusByte();
+
+ private:
+  // The textual content of the code this object is testing.  This class
+  // doesn't own this string and should not attempt to delete it.
+  const char* const statement_;
+  // The regular expression which test output must match.  DeathTestImpl
+  // doesn't own this object and should not attempt to delete it.
+  const RE* const regex_;
+  // True if the death test child process has been successfully spawned.
+  bool spawned_;
+  // The exit status of the child process.
+  int status_;
+  // How the death test concluded.
+  DeathTestOutcome outcome_;
+  // Descriptor to the read end of the pipe to the child process.  It is
+  // always -1 in the child process.  The child keeps its write end of the
+  // pipe in write_fd_.
+  int read_fd_;
+  // Descriptor to the child's write end of the pipe to the parent process.
+  // It is always -1 in the parent process.  The parent keeps its end of the
+  // pipe in read_fd_.
+  int write_fd_;
+};
+
+// Called in the parent process only. Reads the result code of the death
+// test child process via a pipe, interprets it to set the outcome_
+// member, and closes read_fd_.  Outputs diagnostics and terminates in
+// case of unexpected codes.
+void DeathTestImpl::ReadAndInterpretStatusByte() {
+  char flag;
+  int bytes_read;
+
+  // The read() here blocks until data is available (signifying the
+  // failure of the death test) or until the pipe is closed (signifying
+  // its success), so it's okay to call this in the parent before
+  // the child process has exited.
+  do {
+    bytes_read = posix::Read(read_fd(), &flag, 1);
+  } while (bytes_read == -1 && errno == EINTR);
+
+  if (bytes_read == 0) {
+    set_outcome(DIED);
+  } else if (bytes_read == 1) {
+    switch (flag) {
+      case kDeathTestReturned:
+        set_outcome(RETURNED);
+        break;
+      case kDeathTestThrew:
+        set_outcome(THREW);
+        break;
+      case kDeathTestLived:
+        set_outcome(LIVED);
+        break;
+      case kDeathTestInternalError:
+        FailFromInternalError(read_fd());  // Does not return.
+        break;
+      default:
+        GTEST_LOG_(FATAL) << "Death test child process reported "
+                          << "unexpected status byte ("
+                          << static_cast<unsigned int>(flag) << ")";
+    }
+  } else {
+    GTEST_LOG_(FATAL) << "Read from death test child process failed: "
+                      << GetLastErrnoDescription();
+  }
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd()));
+  set_read_fd(-1);
+}
+
+// Signals that the death test code which should have exited, didn't.
+// Should be called only in a death test child process.
+// Writes a status byte to the child's status file descriptor, then
+// calls _exit(1).
+void DeathTestImpl::Abort(AbortReason reason) {
+  // The parent process considers the death test to be a failure if
+  // it finds any data in our pipe.  So, here we write a single flag byte
+  // to the pipe, then exit.
+  const char status_ch =
+      reason == TEST_DID_NOT_DIE ? kDeathTestLived :
+      reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned;
+
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1));
+  // We are leaking the descriptor here because on some platforms (i.e.,
+  // when built as Windows DLL), destructors of global objects will still
+  // run after calling _exit(). On such systems, write_fd_ will be
+  // indirectly closed from the destructor of UnitTestImpl, causing double
+  // close if it is also closed here. On debug configurations, double close
+  // may assert. As there are no in-process buffers to flush here, we are
+  // relying on the OS to close the descriptor after the process terminates
+  // when the destructors are not run.
+  _exit(1);  // Exits w/o any normal exit hooks (we were supposed to crash)
+}
+
+// Returns an indented copy of stderr output for a death test.
+// This makes distinguishing death test output lines from regular log lines
+// much easier.
+static ::std::string FormatDeathTestOutput(const ::std::string& output) {
+  ::std::string ret;
+  for (size_t at = 0; ; ) {
+    const size_t line_end = output.find('\n', at);
+    ret += "[  DEATH   ] ";
+    if (line_end == ::std::string::npos) {
+      ret += output.substr(at);
+      break;
+    }
+    ret += output.substr(at, line_end + 1 - at);
+    at = line_end + 1;
+  }
+  return ret;
+}
+
+// Assesses the success or failure of a death test, using both private
+// members which have previously been set, and one argument:
+//
+// Private data members:
+//   outcome:  An enumeration describing how the death test
+//             concluded: DIED, LIVED, THREW, or RETURNED.  The death test
+//             fails in the latter three cases.
+//   status:   The exit status of the child process. On *nix, it is in the
+//             in the format specified by wait(2). On Windows, this is the
+//             value supplied to the ExitProcess() API or a numeric code
+//             of the exception that terminated the program.
+//   regex:    A regular expression object to be applied to
+//             the test's captured standard error output; the death test
+//             fails if it does not match.
+//
+// Argument:
+//   status_ok: true if exit_status is acceptable in the context of
+//              this particular death test, which fails if it is false
+//
+// Returns true iff all of the above conditions are met.  Otherwise, the
+// first failing condition, in the order given above, is the one that is
+// reported. Also sets the last death test message string.
+bool DeathTestImpl::Passed(bool status_ok) {
+  if (!spawned())
+    return false;
+
+  const std::string error_message = GetCapturedStderr();
+
+  bool success = false;
+  Message buffer;
+
+  buffer << "Death test: " << statement() << "\n";
+  switch (outcome()) {
+    case LIVED:
+      buffer << "    Result: failed to die.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case THREW:
+      buffer << "    Result: threw an exception.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case RETURNED:
+      buffer << "    Result: illegal return in test statement.\n"
+             << " Error msg:\n" << FormatDeathTestOutput(error_message);
+      break;
+    case DIED:
+      if (status_ok) {
+        const bool matched = RE::PartialMatch(error_message.c_str(), *regex());
+        if (matched) {
+          success = true;
+        } else {
+          buffer << "    Result: died but not with expected error.\n"
+                 << "  Expected: " << regex()->pattern() << "\n"
+                 << "Actual msg:\n" << FormatDeathTestOutput(error_message);
+        }
+      } else {
+        buffer << "    Result: died but not with expected exit code:\n"
+               << "            " << ExitSummary(status()) << "\n"
+               << "Actual msg:\n" << FormatDeathTestOutput(error_message);
+      }
+      break;
+    case IN_PROGRESS:
+    default:
+      GTEST_LOG_(FATAL)
+          << "DeathTest::Passed somehow called before conclusion of test";
+  }
+
+  DeathTest::set_last_death_test_message(buffer.GetString());
+  return success;
+}
+
+# if GTEST_OS_WINDOWS
+// WindowsDeathTest implements death tests on Windows. Due to the
+// specifics of starting new processes on Windows, death tests there are
+// always threadsafe, and Google Test considers the
+// --gtest_death_test_style=fast setting to be equivalent to
+// --gtest_death_test_style=threadsafe there.
+//
+// A few implementation notes:  Like the Linux version, the Windows
+// implementation uses pipes for child-to-parent communication. But due to
+// the specifics of pipes on Windows, some extra steps are required:
+//
+// 1. The parent creates a communication pipe and stores handles to both
+//    ends of it.
+// 2. The parent starts the child and provides it with the information
+//    necessary to acquire the handle to the write end of the pipe.
+// 3. The child acquires the write end of the pipe and signals the parent
+//    using a Windows event.
+// 4. Now the parent can release the write end of the pipe on its side. If
+//    this is done before step 3, the object's reference count goes down to
+//    0 and it is destroyed, preventing the child from acquiring it. The
+//    parent now has to release it, or read operations on the read end of
+//    the pipe will not return when the child terminates.
+// 5. The parent reads child's output through the pipe (outcome code and
+//    any possible error messages) from the pipe, and its stderr and then
+//    determines whether to fail the test.
+//
+// Note: to distinguish Win32 API calls from the local method and function
+// calls, the former are explicitly resolved in the global namespace.
+//
+class WindowsDeathTest : public DeathTestImpl {
+ public:
+  WindowsDeathTest(const char* a_statement,
+                   const RE* a_regex,
+                   const char* file,
+                   int line)
+      : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {}
+
+  // All of these virtual functions are inherited from DeathTest.
+  virtual int Wait();
+  virtual TestRole AssumeRole();
+
+ private:
+  // The name of the file in which the death test is located.
+  const char* const file_;
+  // The line number on which the death test is located.
+  const int line_;
+  // Handle to the write end of the pipe to the child process.
+  AutoHandle write_handle_;
+  // Child process handle.
+  AutoHandle child_handle_;
+  // Event the child process uses to signal the parent that it has
+  // acquired the handle to the write end of the pipe. After seeing this
+  // event the parent can release its own handles to make sure its
+  // ReadFile() calls return when the child terminates.
+  AutoHandle event_handle_;
+};
+
+// Waits for the child in a death test to exit, returning its exit
+// status, or 0 if no child process exists.  As a side effect, sets the
+// outcome data member.
+int WindowsDeathTest::Wait() {
+  if (!spawned())
+    return 0;
+
+  // Wait until the child either signals that it has acquired the write end
+  // of the pipe or it dies.
+  const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() };
+  switch (::WaitForMultipleObjects(2,
+                                   wait_handles,
+                                   FALSE,  // Waits for any of the handles.
+                                   INFINITE)) {
+    case WAIT_OBJECT_0:
+    case WAIT_OBJECT_0 + 1:
+      break;
+    default:
+      GTEST_DEATH_TEST_CHECK_(false);  // Should not get here.
+  }
+
+  // The child has acquired the write end of the pipe or exited.
+  // We release the handle on our side and continue.
+  write_handle_.Reset();
+  event_handle_.Reset();
+
+  ReadAndInterpretStatusByte();
+
+  // Waits for the child process to exit if it haven't already. This
+  // returns immediately if the child has already exited, regardless of
+  // whether previous calls to WaitForMultipleObjects synchronized on this
+  // handle or not.
+  GTEST_DEATH_TEST_CHECK_(
+      WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(),
+                                             INFINITE));
+  DWORD status_code;
+  GTEST_DEATH_TEST_CHECK_(
+      ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE);
+  child_handle_.Reset();
+  set_status(static_cast<int>(status_code));
+  return status();
+}
+
+// The AssumeRole process for a Windows death test.  It creates a child
+// process with the same executable as the current process to run the
+// death test.  The child process is given the --gtest_filter and
+// --gtest_internal_run_death_test flags such that it knows to run the
+// current death test only.
+DeathTest::TestRole WindowsDeathTest::AssumeRole() {
+  const UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const TestInfo* const info = impl->current_test_info();
+  const int death_test_index = info->result()->death_test_count();
+
+  if (flag != NULL) {
+    // ParseInternalRunDeathTestFlag() has performed all the necessary
+    // processing.
+    set_write_fd(flag->write_fd());
+    return EXECUTE_TEST;
+  }
+
+  // WindowsDeathTest uses an anonymous pipe to communicate results of
+  // a death test.
+  SECURITY_ATTRIBUTES handles_are_inheritable = {
+    sizeof(SECURITY_ATTRIBUTES), NULL, TRUE };
+  HANDLE read_handle, write_handle;
+  GTEST_DEATH_TEST_CHECK_(
+      ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable,
+                   0)  // Default buffer size.
+      != FALSE);
+  set_read_fd(::_open_osfhandle(reinterpret_cast<intptr_t>(read_handle),
+                                O_RDONLY));
+  write_handle_.Reset(write_handle);
+  event_handle_.Reset(::CreateEvent(
+      &handles_are_inheritable,
+      TRUE,    // The event will automatically reset to non-signaled state.
+      FALSE,   // The initial state is non-signalled.
+      NULL));  // The even is unnamed.
+  GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL);
+  const std::string filter_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" +
+      info->test_case_name() + "." + info->name();
+  const std::string internal_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag +
+      "=" + file_ + "|" + StreamableToString(line_) + "|" +
+      StreamableToString(death_test_index) + "|" +
+      StreamableToString(static_cast<unsigned int>(::GetCurrentProcessId())) +
+      // size_t has the same width as pointers on both 32-bit and 64-bit
+      // Windows platforms.
+      // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx.
+      "|" + StreamableToString(reinterpret_cast<size_t>(write_handle)) +
+      "|" + StreamableToString(reinterpret_cast<size_t>(event_handle_.Get()));
+
+  char executable_path[_MAX_PATH + 1];  // NOLINT
+  GTEST_DEATH_TEST_CHECK_(
+      _MAX_PATH + 1 != ::GetModuleFileNameA(NULL,
+                                            executable_path,
+                                            _MAX_PATH));
+
+  std::string command_line =
+      std::string(::GetCommandLineA()) + " " + filter_flag + " \"" +
+      internal_flag + "\"";
+
+  DeathTest::set_last_death_test_message("");
+
+  CaptureStderr();
+  // Flush the log buffers since the log streams are shared with the child.
+  FlushInfoLog();
+
+  // The child process will share the standard handles with the parent.
+  STARTUPINFOA startup_info;
+  memset(&startup_info, 0, sizeof(STARTUPINFO));
+  startup_info.dwFlags = STARTF_USESTDHANDLES;
+  startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE);
+  startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE);
+  startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE);
+
+  PROCESS_INFORMATION process_info;
+  GTEST_DEATH_TEST_CHECK_(::CreateProcessA(
+      executable_path,
+      const_cast<char*>(command_line.c_str()),
+      NULL,   // Retuned process handle is not inheritable.
+      NULL,   // Retuned thread handle is not inheritable.
+      TRUE,   // Child inherits all inheritable handles (for write_handle_).
+      0x0,    // Default creation flags.
+      NULL,   // Inherit the parent's environment.
+      UnitTest::GetInstance()->original_working_dir(),
+      &startup_info,
+      &process_info) != FALSE);
+  child_handle_.Reset(process_info.hProcess);
+  ::CloseHandle(process_info.hThread);
+  set_spawned(true);
+  return OVERSEE_TEST;
+}
+# else  // We are not on Windows.
+
+// ForkingDeathTest provides implementations for most of the abstract
+// methods of the DeathTest interface.  Only the AssumeRole method is
+// left undefined.
+class ForkingDeathTest : public DeathTestImpl {
+ public:
+  ForkingDeathTest(const char* statement, const RE* regex);
+
+  // All of these virtual functions are inherited from DeathTest.
+  virtual int Wait();
+
+ protected:
+  void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; }
+
+ private:
+  // PID of child process during death test; 0 in the child process itself.
+  pid_t child_pid_;
+};
+
+// Constructs a ForkingDeathTest.
+ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex)
+    : DeathTestImpl(a_statement, a_regex),
+      child_pid_(-1) {}
+
+// Waits for the child in a death test to exit, returning its exit
+// status, or 0 if no child process exists.  As a side effect, sets the
+// outcome data member.
+int ForkingDeathTest::Wait() {
+  if (!spawned())
+    return 0;
+
+  ReadAndInterpretStatusByte();
+
+  int status_value;
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0));
+  set_status(status_value);
+  return status_value;
+}
+
+// A concrete death test class that forks, then immediately runs the test
+// in the child process.
+class NoExecDeathTest : public ForkingDeathTest {
+ public:
+  NoExecDeathTest(const char* a_statement, const RE* a_regex) :
+      ForkingDeathTest(a_statement, a_regex) { }
+  virtual TestRole AssumeRole();
+};
+
+// The AssumeRole process for a fork-and-run death test.  It implements a
+// straightforward fork, with a simple pipe to transmit the status byte.
+DeathTest::TestRole NoExecDeathTest::AssumeRole() {
+  const size_t thread_count = GetThreadCount();
+  if (thread_count != 1) {
+    GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count);
+  }
+
+  int pipe_fd[2];
+  GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1);
+
+  DeathTest::set_last_death_test_message("");
+  CaptureStderr();
+  // When we fork the process below, the log file buffers are copied, but the
+  // file descriptors are shared.  We flush all log files here so that closing
+  // the file descriptors in the child process doesn't throw off the
+  // synchronization between descriptors and buffers in the parent process.
+  // This is as close to the fork as possible to avoid a race condition in case
+  // there are multiple threads running before the death test, and another
+  // thread writes to the log file.
+  FlushInfoLog();
+
+  const pid_t child_pid = fork();
+  GTEST_DEATH_TEST_CHECK_(child_pid != -1);
+  set_child_pid(child_pid);
+  if (child_pid == 0) {
+    GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0]));
+    set_write_fd(pipe_fd[1]);
+    // Redirects all logging to stderr in the child process to prevent
+    // concurrent writes to the log files.  We capture stderr in the parent
+    // process and append the child process' output to a log.
+    LogToStderr();
+    // Event forwarding to the listeners of event listener API mush be shut
+    // down in death test subprocesses.
+    GetUnitTestImpl()->listeners()->SuppressEventForwarding();
+    g_in_fast_death_test_child = true;
+    return EXECUTE_TEST;
+  } else {
+    GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1]));
+    set_read_fd(pipe_fd[0]);
+    set_spawned(true);
+    return OVERSEE_TEST;
+  }
+}
+
+// A concrete death test class that forks and re-executes the main
+// program from the beginning, with command-line flags set that cause
+// only this specific death test to be run.
+class ExecDeathTest : public ForkingDeathTest {
+ public:
+  ExecDeathTest(const char* a_statement, const RE* a_regex,
+                const char* file, int line) :
+      ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { }
+  virtual TestRole AssumeRole();
+ private:
+  static ::std::vector<testing::internal::string>
+  GetArgvsForDeathTestChildProcess() {
+    ::std::vector<testing::internal::string> args = GetInjectableArgvs();
+#  if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
+    ::std::vector<testing::internal::string> extra_args =
+        GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_();
+    args.insert(args.end(), extra_args.begin(), extra_args.end());
+#  endif  // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_)
+    return args;
+  }
+  // The name of the file in which the death test is located.
+  const char* const file_;
+  // The line number on which the death test is located.
+  const int line_;
+};
+
+// Utility class for accumulating command-line arguments.
+class Arguments {
+ public:
+  Arguments() {
+    args_.push_back(NULL);
+  }
+
+  ~Arguments() {
+    for (std::vector<char*>::iterator i = args_.begin(); i != args_.end();
+         ++i) {
+      free(*i);
+    }
+  }
+  void AddArgument(const char* argument) {
+    args_.insert(args_.end() - 1, posix::StrDup(argument));
+  }
+
+  template <typename Str>
+  void AddArguments(const ::std::vector<Str>& arguments) {
+    for (typename ::std::vector<Str>::const_iterator i = arguments.begin();
+         i != arguments.end();
+         ++i) {
+      args_.insert(args_.end() - 1, posix::StrDup(i->c_str()));
+    }
+  }
+  char* const* Argv() {
+    return &args_[0];
+  }
+
+ private:
+  std::vector<char*> args_;
+};
+
+// A struct that encompasses the arguments to the child process of a
+// threadsafe-style death test process.
+struct ExecDeathTestArgs {
+  char* const* argv;  // Command-line arguments for the child's call to exec
+  int close_fd;       // File descriptor to close; the read end of a pipe
+};
+
+#  if GTEST_OS_MAC
+inline char** GetEnviron() {
+  // When Google Test is built as a framework on MacOS X, the environ variable
+  // is unavailable. Apple's documentation (man environ) recommends using
+  // _NSGetEnviron() instead.
+  return *_NSGetEnviron();
+}
+#  else
+// Some POSIX platforms expect you to declare environ. extern "C" makes
+// it reside in the global namespace.
+extern "C" char** environ;
+inline char** GetEnviron() { return environ; }
+#  endif  // GTEST_OS_MAC
+
+#  if !GTEST_OS_QNX
+// The main function for a threadsafe-style death test child process.
+// This function is called in a clone()-ed process and thus must avoid
+// any potentially unsafe operations like malloc or libc functions.
+static int ExecDeathTestChildMain(void* child_arg) {
+  ExecDeathTestArgs* const args = static_cast<ExecDeathTestArgs*>(child_arg);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd));
+
+  // We need to execute the test program in the same environment where
+  // it was originally invoked.  Therefore we change to the original
+  // working directory first.
+  const char* const original_dir =
+      UnitTest::GetInstance()->original_working_dir();
+  // We can safely call chdir() as it's a direct system call.
+  if (chdir(original_dir) != 0) {
+    DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " +
+                   GetLastErrnoDescription());
+    return EXIT_FAILURE;
+  }
+
+  // We can safely call execve() as it's a direct system call.  We
+  // cannot use execvp() as it's a libc function and thus potentially
+  // unsafe.  Since execve() doesn't search the PATH, the user must
+  // invoke the test program via a valid path that contains at least
+  // one path separator.
+  execve(args->argv[0], args->argv, GetEnviron());
+  DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " +
+                 original_dir + " failed: " +
+                 GetLastErrnoDescription());
+  return EXIT_FAILURE;
+}
+#  endif  // !GTEST_OS_QNX
+
+// Two utility routines that together determine the direction the stack
+// grows.
+// This could be accomplished more elegantly by a single recursive
+// function, but we want to guard against the unlikely possibility of
+// a smart compiler optimizing the recursion away.
+//
+// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining
+// StackLowerThanAddress into StackGrowsDown, which then doesn't give
+// correct answer.
+void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_;
+void StackLowerThanAddress(const void* ptr, bool* result) {
+  int dummy;
+  *result = (&dummy < ptr);
+}
+
+// Make sure AddressSanitizer does not tamper with the stack here.
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+bool StackGrowsDown() {
+  int dummy;
+  bool result;
+  StackLowerThanAddress(&dummy, &result);
+  return result;
+}
+
+// Spawns a child process with the same executable as the current process in
+// a thread-safe manner and instructs it to run the death test.  The
+// implementation uses fork(2) + exec.  On systems where clone(2) is
+// available, it is used instead, being slightly more thread-safe.  On QNX,
+// fork supports only single-threaded environments, so this function uses
+// spawn(2) there instead.  The function dies with an error message if
+// anything goes wrong.
+static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) {
+  ExecDeathTestArgs args = { argv, close_fd };
+  pid_t child_pid = -1;
+
+#  if GTEST_OS_QNX
+  // Obtains the current directory and sets it to be closed in the child
+  // process.
+  const int cwd_fd = open(".", O_RDONLY);
+  GTEST_DEATH_TEST_CHECK_(cwd_fd != -1);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC));
+  // We need to execute the test program in the same environment where
+  // it was originally invoked.  Therefore we change to the original
+  // working directory first.
+  const char* const original_dir =
+      UnitTest::GetInstance()->original_working_dir();
+  // We can safely call chdir() as it's a direct system call.
+  if (chdir(original_dir) != 0) {
+    DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " +
+                   GetLastErrnoDescription());
+    return EXIT_FAILURE;
+  }
+
+  int fd_flags;
+  // Set close_fd to be closed after spawn.
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD));
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD,
+                                        fd_flags | FD_CLOEXEC));
+  struct inheritance inherit = {0};
+  // spawn is a system call.
+  child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron());
+  // Restores the current working directory.
+  GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd));
+
+#  else   // GTEST_OS_QNX
+#   if GTEST_OS_LINUX
+  // When a SIGPROF signal is received while fork() or clone() are executing,
+  // the process may hang. To avoid this, we ignore SIGPROF here and re-enable
+  // it after the call to fork()/clone() is complete.
+  struct sigaction saved_sigprof_action;
+  struct sigaction ignore_sigprof_action;
+  memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action));
+  sigemptyset(&ignore_sigprof_action.sa_mask);
+  ignore_sigprof_action.sa_handler = SIG_IGN;
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction(
+      SIGPROF, &ignore_sigprof_action, &saved_sigprof_action));
+#   endif  // GTEST_OS_LINUX
+
+#   if GTEST_HAS_CLONE
+  const bool use_fork = GTEST_FLAG(death_test_use_fork);
+
+  if (!use_fork) {
+    static const bool stack_grows_down = StackGrowsDown();
+    const size_t stack_size = getpagesize();
+    // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead.
+    void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE,
+                             MAP_ANON | MAP_PRIVATE, -1, 0);
+    GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED);
+
+    // Maximum stack alignment in bytes:  For a downward-growing stack, this
+    // amount is subtracted from size of the stack space to get an address
+    // that is within the stack space and is aligned on all systems we care
+    // about.  As far as I know there is no ABI with stack alignment greater
+    // than 64.  We assume stack and stack_size already have alignment of
+    // kMaxStackAlignment.
+    const size_t kMaxStackAlignment = 64;
+    void* const stack_top =
+        static_cast<char*>(stack) +
+            (stack_grows_down ? stack_size - kMaxStackAlignment : 0);
+    GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment &&
+        reinterpret_cast<intptr_t>(stack_top) % kMaxStackAlignment == 0);
+
+    child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args);
+
+    GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1);
+  }
+#   else
+  const bool use_fork = true;
+#   endif  // GTEST_HAS_CLONE
+
+  if (use_fork && (child_pid = fork()) == 0) {
+      ExecDeathTestChildMain(&args);
+      _exit(0);
+  }
+#  endif  // GTEST_OS_QNX
+#  if GTEST_OS_LINUX
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(
+      sigaction(SIGPROF, &saved_sigprof_action, NULL));
+#  endif  // GTEST_OS_LINUX
+
+  GTEST_DEATH_TEST_CHECK_(child_pid != -1);
+  return child_pid;
+}
+
+// The AssumeRole process for a fork-and-exec death test.  It re-executes the
+// main program from the beginning, setting the --gtest_filter
+// and --gtest_internal_run_death_test flags to cause only the current
+// death test to be re-run.
+DeathTest::TestRole ExecDeathTest::AssumeRole() {
+  const UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const TestInfo* const info = impl->current_test_info();
+  const int death_test_index = info->result()->death_test_count();
+
+  if (flag != NULL) {
+    set_write_fd(flag->write_fd());
+    return EXECUTE_TEST;
+  }
+
+  int pipe_fd[2];
+  GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1);
+  // Clear the close-on-exec flag on the write end of the pipe, lest
+  // it be closed when the child process does an exec:
+  GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1);
+
+  const std::string filter_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "="
+      + info->test_case_name() + "." + info->name();
+  const std::string internal_flag =
+      std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "="
+      + file_ + "|" + StreamableToString(line_) + "|"
+      + StreamableToString(death_test_index) + "|"
+      + StreamableToString(pipe_fd[1]);
+  Arguments args;
+  args.AddArguments(GetArgvsForDeathTestChildProcess());
+  args.AddArgument(filter_flag.c_str());
+  args.AddArgument(internal_flag.c_str());
+
+  DeathTest::set_last_death_test_message("");
+
+  CaptureStderr();
+  // See the comment in NoExecDeathTest::AssumeRole for why the next line
+  // is necessary.
+  FlushInfoLog();
+
+  const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]);
+  GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1]));
+  set_child_pid(child_pid);
+  set_read_fd(pipe_fd[0]);
+  set_spawned(true);
+  return OVERSEE_TEST;
+}
+
+# endif  // !GTEST_OS_WINDOWS
+
+// Creates a concrete DeathTest-derived class that depends on the
+// --gtest_death_test_style flag, and sets the pointer pointed to
+// by the "test" argument to its address.  If the test should be
+// skipped, sets that pointer to NULL.  Returns true, unless the
+// flag is set to an invalid value.
+bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex,
+                                     const char* file, int line,
+                                     DeathTest** test) {
+  UnitTestImpl* const impl = GetUnitTestImpl();
+  const InternalRunDeathTestFlag* const flag =
+      impl->internal_run_death_test_flag();
+  const int death_test_index = impl->current_test_info()
+      ->increment_death_test_count();
+
+  if (flag != NULL) {
+    if (death_test_index > flag->index()) {
+      DeathTest::set_last_death_test_message(
+          "Death test count (" + StreamableToString(death_test_index)
+          + ") somehow exceeded expected maximum ("
+          + StreamableToString(flag->index()) + ")");
+      return false;
+    }
+
+    if (!(flag->file() == file && flag->line() == line &&
+          flag->index() == death_test_index)) {
+      *test = NULL;
+      return true;
+    }
+  }
+
+# if GTEST_OS_WINDOWS
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe" ||
+      GTEST_FLAG(death_test_style) == "fast") {
+    *test = new WindowsDeathTest(statement, regex, file, line);
+  }
+
+# else
+
+  if (GTEST_FLAG(death_test_style) == "threadsafe") {
+    *test = new ExecDeathTest(statement, regex, file, line);
+  } else if (GTEST_FLAG(death_test_style) == "fast") {
+    *test = new NoExecDeathTest(statement, regex);
+  }
+
+# endif  // GTEST_OS_WINDOWS
+
+  else {  // NOLINT - this is more readable than unbalanced brackets inside #if.
+    DeathTest::set_last_death_test_message(
+        "Unknown death test style \"" + GTEST_FLAG(death_test_style)
+        + "\" encountered");
+    return false;
+  }
+
+  return true;
+}
+
+# if GTEST_OS_WINDOWS
+// Recreates the pipe and event handles from the provided parameters,
+// signals the event, and returns a file descriptor wrapped around the pipe
+// handle. This function is called in the child process only.
+int GetStatusFileDescriptor(unsigned int parent_process_id,
+                            size_t write_handle_as_size_t,
+                            size_t event_handle_as_size_t) {
+  AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE,
+                                                   FALSE,  // Non-inheritable.
+                                                   parent_process_id));
+  if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) {
+    DeathTestAbort("Unable to open parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  // TODO(vladl@google.com): Replace the following check with a
+  // compile-time assertion when available.
+  GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t));
+
+  const HANDLE write_handle =
+      reinterpret_cast<HANDLE>(write_handle_as_size_t);
+  HANDLE dup_write_handle;
+
+  // The newly initialized handle is accessible only in in the parent
+  // process. To obtain one accessible within the child, we need to use
+  // DuplicateHandle.
+  if (!::DuplicateHandle(parent_process_handle.Get(), write_handle,
+                         ::GetCurrentProcess(), &dup_write_handle,
+                         0x0,    // Requested privileges ignored since
+                                 // DUPLICATE_SAME_ACCESS is used.
+                         FALSE,  // Request non-inheritable handler.
+                         DUPLICATE_SAME_ACCESS)) {
+    DeathTestAbort("Unable to duplicate the pipe handle " +
+                   StreamableToString(write_handle_as_size_t) +
+                   " from the parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  const HANDLE event_handle = reinterpret_cast<HANDLE>(event_handle_as_size_t);
+  HANDLE dup_event_handle;
+
+  if (!::DuplicateHandle(parent_process_handle.Get(), event_handle,
+                         ::GetCurrentProcess(), &dup_event_handle,
+                         0x0,
+                         FALSE,
+                         DUPLICATE_SAME_ACCESS)) {
+    DeathTestAbort("Unable to duplicate the event handle " +
+                   StreamableToString(event_handle_as_size_t) +
+                   " from the parent process " +
+                   StreamableToString(parent_process_id));
+  }
+
+  const int write_fd =
+      ::_open_osfhandle(reinterpret_cast<intptr_t>(dup_write_handle), O_APPEND);
+  if (write_fd == -1) {
+    DeathTestAbort("Unable to convert pipe handle " +
+                   StreamableToString(write_handle_as_size_t) +
+                   " to a file descriptor");
+  }
+
+  // Signals the parent that the write end of the pipe has been acquired
+  // so the parent can release its own write end.
+  ::SetEvent(dup_event_handle);
+
+  return write_fd;
+}
+# endif  // GTEST_OS_WINDOWS
+
+// Returns a newly created InternalRunDeathTestFlag object with fields
+// initialized from the GTEST_FLAG(internal_run_death_test) flag if
+// the flag is specified; otherwise returns NULL.
+InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() {
+  if (GTEST_FLAG(internal_run_death_test) == "") return NULL;
+
+  // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we
+  // can use it here.
+  int line = -1;
+  int index = -1;
+  ::std::vector< ::std::string> fields;
+  SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields);
+  int write_fd = -1;
+
+# if GTEST_OS_WINDOWS
+
+  unsigned int parent_process_id = 0;
+  size_t write_handle_as_size_t = 0;
+  size_t event_handle_as_size_t = 0;
+
+  if (fields.size() != 6
+      || !ParseNaturalNumber(fields[1], &line)
+      || !ParseNaturalNumber(fields[2], &index)
+      || !ParseNaturalNumber(fields[3], &parent_process_id)
+      || !ParseNaturalNumber(fields[4], &write_handle_as_size_t)
+      || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) {
+    DeathTestAbort("Bad --gtest_internal_run_death_test flag: " +
+                   GTEST_FLAG(internal_run_death_test));
+  }
+  write_fd = GetStatusFileDescriptor(parent_process_id,
+                                     write_handle_as_size_t,
+                                     event_handle_as_size_t);
+# else
+
+  if (fields.size() != 4
+      || !ParseNaturalNumber(fields[1], &line)
+      || !ParseNaturalNumber(fields[2], &index)
+      || !ParseNaturalNumber(fields[3], &write_fd)) {
+    DeathTestAbort("Bad --gtest_internal_run_death_test flag: "
+        + GTEST_FLAG(internal_run_death_test));
+  }
+
+# endif  // GTEST_OS_WINDOWS
+
+  return new InternalRunDeathTestFlag(fields[0], line, index, write_fd);
+}
+
+}  // namespace internal
+
+#endif  // GTEST_HAS_DEATH_TEST
+
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Authors: keith.ray@gmail.com (Keith Ray)
+
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-filepath.h"
+#include "gtest/internal/gtest-port.h"
+
+#include <stdlib.h>
+
+#if GTEST_OS_WINDOWS_MOBILE
+# include <windows.h>
+#elif GTEST_OS_WINDOWS
+# include <direct.h>
+# include <io.h>
+#elif GTEST_OS_SYMBIAN
+// Symbian OpenC has PATH_MAX in sys/syslimits.h
+# include <sys/syslimits.h>
+#else
+# include <limits.h>
+# include <climits>  // Some Linux distributions define PATH_MAX here.
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+#if GTEST_OS_WINDOWS
+# define GTEST_PATH_MAX_ _MAX_PATH
+#elif defined(PATH_MAX)
+# define GTEST_PATH_MAX_ PATH_MAX
+#elif defined(_XOPEN_PATH_MAX)
+# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX
+#else
+# define GTEST_PATH_MAX_ _POSIX_PATH_MAX
+#endif  // GTEST_OS_WINDOWS
+
+#include "gtest/internal/gtest-string.h"
+
+namespace testing {
+namespace internal {
+
+#if GTEST_OS_WINDOWS
+// On Windows, '\\' is the standard path separator, but many tools and the
+// Windows API also accept '/' as an alternate path separator. Unless otherwise
+// noted, a file path can contain either kind of path separators, or a mixture
+// of them.
+const char kPathSeparator = '\\';
+const char kAlternatePathSeparator = '/';
+const char kAlternatePathSeparatorString[] = "/";
+# if GTEST_OS_WINDOWS_MOBILE
+// Windows CE doesn't have a current directory. You should not use
+// the current directory in tests on Windows CE, but this at least
+// provides a reasonable fallback.
+const char kCurrentDirectoryString[] = "\\";
+// Windows CE doesn't define INVALID_FILE_ATTRIBUTES
+const DWORD kInvalidFileAttributes = 0xffffffff;
+# else
+const char kCurrentDirectoryString[] = ".\\";
+# endif  // GTEST_OS_WINDOWS_MOBILE
+#else
+const char kPathSeparator = '/';
+const char kCurrentDirectoryString[] = "./";
+#endif  // GTEST_OS_WINDOWS
+
+// Returns whether the given character is a valid path separator.
+static bool IsPathSeparator(char c) {
+#if GTEST_HAS_ALT_PATH_SEP_
+  return (c == kPathSeparator) || (c == kAlternatePathSeparator);
+#else
+  return c == kPathSeparator;
+#endif
+}
+
+// Returns the current working directory, or "" if unsuccessful.
+FilePath FilePath::GetCurrentDir() {
+#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT
+  // Windows CE doesn't have a current directory, so we just return
+  // something reasonable.
+  return FilePath(kCurrentDirectoryString);
+#elif GTEST_OS_WINDOWS
+  char cwd[GTEST_PATH_MAX_ + 1] = { '\0' };
+  return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd);
+#else
+  char cwd[GTEST_PATH_MAX_ + 1] = { '\0' };
+  char* result = getcwd(cwd, sizeof(cwd));
+# if GTEST_OS_NACL
+  // getcwd will likely fail in NaCl due to the sandbox, so return something
+  // reasonable. The user may have provided a shim implementation for getcwd,
+  // however, so fallback only when failure is detected.
+  return FilePath(result == NULL ? kCurrentDirectoryString : cwd);
+# endif  // GTEST_OS_NACL
+  return FilePath(result == NULL ? "" : cwd);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+// Returns a copy of the FilePath with the case-insensitive extension removed.
+// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns
+// FilePath("dir/file"). If a case-insensitive extension is not
+// found, returns a copy of the original FilePath.
+FilePath FilePath::RemoveExtension(const char* extension) const {
+  const std::string dot_extension = std::string(".") + extension;
+  if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) {
+    return FilePath(pathname_.substr(
+        0, pathname_.length() - dot_extension.length()));
+  }
+  return *this;
+}
+
+// Returns a pointer to the last occurence of a valid path separator in
+// the FilePath. On Windows, for example, both '/' and '\' are valid path
+// separators. Returns NULL if no path separator was found.
+const char* FilePath::FindLastPathSeparator() const {
+  const char* const last_sep = strrchr(c_str(), kPathSeparator);
+#if GTEST_HAS_ALT_PATH_SEP_
+  const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator);
+  // Comparing two pointers of which only one is NULL is undefined.
+  if (last_alt_sep != NULL &&
+      (last_sep == NULL || last_alt_sep > last_sep)) {
+    return last_alt_sep;
+  }
+#endif
+  return last_sep;
+}
+
+// Returns a copy of the FilePath with the directory part removed.
+// Example: FilePath("path/to/file").RemoveDirectoryName() returns
+// FilePath("file"). If there is no directory part ("just_a_file"), it returns
+// the FilePath unmodified. If there is no file part ("just_a_dir/") it
+// returns an empty FilePath ("").
+// On Windows platform, '\' is the path separator, otherwise it is '/'.
+FilePath FilePath::RemoveDirectoryName() const {
+  const char* const last_sep = FindLastPathSeparator();
+  return last_sep ? FilePath(last_sep + 1) : *this;
+}
+
+// RemoveFileName returns the directory path with the filename removed.
+// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/".
+// If the FilePath is "a_file" or "/a_file", RemoveFileName returns
+// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does
+// not have a file, like "just/a/dir/", it returns the FilePath unmodified.
+// On Windows platform, '\' is the path separator, otherwise it is '/'.
+FilePath FilePath::RemoveFileName() const {
+  const char* const last_sep = FindLastPathSeparator();
+  std::string dir;
+  if (last_sep) {
+    dir = std::string(c_str(), last_sep + 1 - c_str());
+  } else {
+    dir = kCurrentDirectoryString;
+  }
+  return FilePath(dir);
+}
+
+// Helper functions for naming files in a directory for xml output.
+
+// Given directory = "dir", base_name = "test", number = 0,
+// extension = "xml", returns "dir/test.xml". If number is greater
+// than zero (e.g., 12), returns "dir/test_12.xml".
+// On Windows platform, uses \ as the separator rather than /.
+FilePath FilePath::MakeFileName(const FilePath& directory,
+                                const FilePath& base_name,
+                                int number,
+                                const char* extension) {
+  std::string file;
+  if (number == 0) {
+    file = base_name.string() + "." + extension;
+  } else {
+    file = base_name.string() + "_" + StreamableToString(number)
+        + "." + extension;
+  }
+  return ConcatPaths(directory, FilePath(file));
+}
+
+// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml".
+// On Windows, uses \ as the separator rather than /.
+FilePath FilePath::ConcatPaths(const FilePath& directory,
+                               const FilePath& relative_path) {
+  if (directory.IsEmpty())
+    return relative_path;
+  const FilePath dir(directory.RemoveTrailingPathSeparator());
+  return FilePath(dir.string() + kPathSeparator + relative_path.string());
+}
+
+// Returns true if pathname describes something findable in the file-system,
+// either a file, directory, or whatever.
+bool FilePath::FileOrDirectoryExists() const {
+#if GTEST_OS_WINDOWS_MOBILE
+  LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str());
+  const DWORD attributes = GetFileAttributes(unicode);
+  delete [] unicode;
+  return attributes != kInvalidFileAttributes;
+#else
+  posix::StatStruct file_stat;
+  return posix::Stat(pathname_.c_str(), &file_stat) == 0;
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+// Returns true if pathname describes a directory in the file-system
+// that exists.
+bool FilePath::DirectoryExists() const {
+  bool result = false;
+#if GTEST_OS_WINDOWS
+  // Don't strip off trailing separator if path is a root directory on
+  // Windows (like "C:\\").
+  const FilePath& path(IsRootDirectory() ? *this :
+                                           RemoveTrailingPathSeparator());
+#else
+  const FilePath& path(*this);
+#endif
+
+#if GTEST_OS_WINDOWS_MOBILE
+  LPCWSTR unicode = String::AnsiToUtf16(path.c_str());
+  const DWORD attributes = GetFileAttributes(unicode);
+  delete [] unicode;
+  if ((attributes != kInvalidFileAttributes) &&
+      (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+    result = true;
+  }
+#else
+  posix::StatStruct file_stat;
+  result = posix::Stat(path.c_str(), &file_stat) == 0 &&
+      posix::IsDir(file_stat);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+  return result;
+}
+
+// Returns true if pathname describes a root directory. (Windows has one
+// root directory per disk drive.)
+bool FilePath::IsRootDirectory() const {
+#if GTEST_OS_WINDOWS
+  // TODO(wan@google.com): on Windows a network share like
+  // \\server\share can be a root directory, although it cannot be the
+  // current directory.  Handle this properly.
+  return pathname_.length() == 3 && IsAbsolutePath();
+#else
+  return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]);
+#endif
+}
+
+// Returns true if pathname describes an absolute path.
+bool FilePath::IsAbsolutePath() const {
+  const char* const name = pathname_.c_str();
+#if GTEST_OS_WINDOWS
+  return pathname_.length() >= 3 &&
+     ((name[0] >= 'a' && name[0] <= 'z') ||
+      (name[0] >= 'A' && name[0] <= 'Z')) &&
+     name[1] == ':' &&
+     IsPathSeparator(name[2]);
+#else
+  return IsPathSeparator(name[0]);
+#endif
+}
+
+// Returns a pathname for a file that does not currently exist. The pathname
+// will be directory/base_name.extension or
+// directory/base_name_<number>.extension if directory/base_name.extension
+// already exists. The number will be incremented until a pathname is found
+// that does not already exist.
+// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'.
+// There could be a race condition if two or more processes are calling this
+// function at the same time -- they could both pick the same filename.
+FilePath FilePath::GenerateUniqueFileName(const FilePath& directory,
+                                          const FilePath& base_name,
+                                          const char* extension) {
+  FilePath full_pathname;
+  int number = 0;
+  do {
+    full_pathname.Set(MakeFileName(directory, base_name, number++, extension));
+  } while (full_pathname.FileOrDirectoryExists());
+  return full_pathname;
+}
+
+// Returns true if FilePath ends with a path separator, which indicates that
+// it is intended to represent a directory. Returns false otherwise.
+// This does NOT check that a directory (or file) actually exists.
+bool FilePath::IsDirectory() const {
+  return !pathname_.empty() &&
+         IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]);
+}
+
+// Create directories so that path exists. Returns true if successful or if
+// the directories already exist; returns false if unable to create directories
+// for any reason.
+bool FilePath::CreateDirectoriesRecursively() const {
+  if (!this->IsDirectory()) {
+    return false;
+  }
+
+  if (pathname_.length() == 0 || this->DirectoryExists()) {
+    return true;
+  }
+
+  const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName());
+  return parent.CreateDirectoriesRecursively() && this->CreateFolder();
+}
+
+// Create the directory so that path exists. Returns true if successful or
+// if the directory already exists; returns false if unable to create the
+// directory for any reason, including if the parent directory does not
+// exist. Not named "CreateDirectory" because that's a macro on Windows.
+bool FilePath::CreateFolder() const {
+#if GTEST_OS_WINDOWS_MOBILE
+  FilePath removed_sep(this->RemoveTrailingPathSeparator());
+  LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str());
+  int result = CreateDirectory(unicode, NULL) ? 0 : -1;
+  delete [] unicode;
+#elif GTEST_OS_WINDOWS
+  int result = _mkdir(pathname_.c_str());
+#else
+  int result = mkdir(pathname_.c_str(), 0777);
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+  if (result == -1) {
+    return this->DirectoryExists();  // An error is OK if the directory exists.
+  }
+  return true;  // No error.
+}
+
+// If input name has a trailing separator character, remove it and return the
+// name, otherwise return the name string unmodified.
+// On Windows platform, uses \ as the separator, other platforms use /.
+FilePath FilePath::RemoveTrailingPathSeparator() const {
+  return IsDirectory()
+      ? FilePath(pathname_.substr(0, pathname_.length() - 1))
+      : *this;
+}
+
+// Removes any redundant separators that might be in the pathname.
+// For example, "bar///foo" becomes "bar/foo". Does not eliminate other
+// redundancies that might be in a pathname involving "." or "..".
+// TODO(wan@google.com): handle Windows network shares (e.g. \\server\share).
+void FilePath::Normalize() {
+  if (pathname_.c_str() == NULL) {
+    pathname_ = "";
+    return;
+  }
+  const char* src = pathname_.c_str();
+  char* const dest = new char[pathname_.length() + 1];
+  char* dest_ptr = dest;
+  memset(dest_ptr, 0, pathname_.length() + 1);
+
+  while (*src != '\0') {
+    *dest_ptr = *src;
+    if (!IsPathSeparator(*src)) {
+      src++;
+    } else {
+#if GTEST_HAS_ALT_PATH_SEP_
+      if (*dest_ptr == kAlternatePathSeparator) {
+        *dest_ptr = kPathSeparator;
+      }
+#endif
+      while (IsPathSeparator(*src))
+        src++;
+    }
+    dest_ptr++;
+  }
+  *dest_ptr = '\0';
+  pathname_ = dest;
+  delete[] dest;
+}
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/internal/gtest-port.h"
+
+#include <limits.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fstream>
+
+#if GTEST_OS_WINDOWS
+# include <windows.h>
+# include <io.h>
+# include <sys/stat.h>
+# include <map>  // Used in ThreadLocal.
+#else
+# include <unistd.h>
+#endif  // GTEST_OS_WINDOWS
+
+#if GTEST_OS_MAC
+# include <mach/mach_init.h>
+# include <mach/task.h>
+# include <mach/vm_map.h>
+#endif  // GTEST_OS_MAC
+
+#if GTEST_OS_QNX
+# include <devctl.h>
+# include <fcntl.h>
+# include <sys/procfs.h>
+#endif  // GTEST_OS_QNX
+
+#if GTEST_OS_AIX
+# include <procinfo.h>
+# include <sys/types.h>
+#endif  // GTEST_OS_AIX
+
+#include "gtest/gtest-spi.h"
+#include "gtest/gtest-message.h"
+#include "gtest/internal/gtest-internal.h"
+#include "gtest/internal/gtest-string.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+namespace internal {
+
+#if defined(_MSC_VER) || defined(__BORLANDC__)
+// MSVC and C++Builder do not provide a definition of STDERR_FILENO.
+const int kStdOutFileno = 1;
+const int kStdErrFileno = 2;
+#else
+const int kStdOutFileno = STDOUT_FILENO;
+const int kStdErrFileno = STDERR_FILENO;
+#endif  // _MSC_VER
+
+#if GTEST_OS_LINUX
+
+namespace {
+template <typename T>
+T ReadProcFileField(const string& filename, int field) {
+  std::string dummy;
+  std::ifstream file(filename.c_str());
+  while (field-- > 0) {
+    file >> dummy;
+  }
+  T output = 0;
+  file >> output;
+  return output;
+}
+}  // namespace
+
+// Returns the number of active threads, or 0 when there is an error.
+size_t GetThreadCount() {
+  const string filename =
+      (Message() << "/proc/" << getpid() << "/stat").GetString();
+  return ReadProcFileField<int>(filename, 19);
+}
+
+#elif GTEST_OS_MAC
+
+size_t GetThreadCount() {
+  const task_t task = mach_task_self();
+  mach_msg_type_number_t thread_count;
+  thread_act_array_t thread_list;
+  const kern_return_t status = task_threads(task, &thread_list, &thread_count);
+  if (status == KERN_SUCCESS) {
+    // task_threads allocates resources in thread_list and we need to free them
+    // to avoid leaks.
+    vm_deallocate(task,
+                  reinterpret_cast<vm_address_t>(thread_list),
+                  sizeof(thread_t) * thread_count);
+    return static_cast<size_t>(thread_count);
+  } else {
+    return 0;
+  }
+}
+
+#elif GTEST_OS_QNX
+
+// Returns the number of threads running in the process, or 0 to indicate that
+// we cannot detect it.
+size_t GetThreadCount() {
+  const int fd = open("/proc/self/as", O_RDONLY);
+  if (fd < 0) {
+    return 0;
+  }
+  procfs_info process_info;
+  const int status =
+      devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL);
+  close(fd);
+  if (status == EOK) {
+    return static_cast<size_t>(process_info.num_threads);
+  } else {
+    return 0;
+  }
+}
+
+#elif GTEST_OS_AIX
+
+size_t GetThreadCount() {
+  struct procentry64 entry;
+  pid_t pid = getpid();
+  int status = getprocs64(&entry, sizeof(entry), NULL, 0, &pid, 1);
+  if (status == 1) {
+    return entry.pi_thcount;
+  } else {
+    return 0;
+  }
+}
+
+#else
+
+size_t GetThreadCount() {
+  // There's no portable way to detect the number of threads, so we just
+  // return 0 to indicate that we cannot detect it.
+  return 0;
+}
+
+#endif  // GTEST_OS_LINUX
+
+#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
+
+void SleepMilliseconds(int n) {
+  ::Sleep(n);
+}
+
+AutoHandle::AutoHandle()
+    : handle_(INVALID_HANDLE_VALUE) {}
+
+AutoHandle::AutoHandle(Handle handle)
+    : handle_(handle) {}
+
+AutoHandle::~AutoHandle() {
+  Reset();
+}
+
+AutoHandle::Handle AutoHandle::Get() const {
+  return handle_;
+}
+
+void AutoHandle::Reset() {
+  Reset(INVALID_HANDLE_VALUE);
+}
+
+void AutoHandle::Reset(HANDLE handle) {
+  // Resetting with the same handle we already own is invalid.
+  if (handle_ != handle) {
+    if (IsCloseable()) {
+      ::CloseHandle(handle_);
+    }
+    handle_ = handle;
+  } else {
+    GTEST_CHECK_(!IsCloseable())
+        << "Resetting a valid handle to itself is likely a programmer error "
+            "and thus not allowed.";
+  }
+}
+
+bool AutoHandle::IsCloseable() const {
+  // Different Windows APIs may use either of these values to represent an
+  // invalid handle.
+  return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE;
+}
+
+Notification::Notification()
+    : event_(::CreateEvent(NULL,   // Default security attributes.
+                           TRUE,   // Do not reset automatically.
+                           FALSE,  // Initially unset.
+                           NULL)) {  // Anonymous event.
+  GTEST_CHECK_(event_.Get() != NULL);
+}
+
+void Notification::Notify() {
+  GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE);
+}
+
+void Notification::WaitForNotification() {
+  GTEST_CHECK_(
+      ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0);
+}
+
+Mutex::Mutex()
+    : owner_thread_id_(0),
+      type_(kDynamic),
+      critical_section_init_phase_(0),
+      critical_section_(new CRITICAL_SECTION) {
+  ::InitializeCriticalSection(critical_section_);
+}
+
+Mutex::~Mutex() {
+  // Static mutexes are leaked intentionally. It is not thread-safe to try
+  // to clean them up.
+  // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires
+  // nothing to clean it up but is available only on Vista and later.
+  // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx
+  if (type_ == kDynamic) {
+    ::DeleteCriticalSection(critical_section_);
+    delete critical_section_;
+    critical_section_ = NULL;
+  }
+}
+
+void Mutex::Lock() {
+  ThreadSafeLazyInit();
+  ::EnterCriticalSection(critical_section_);
+  owner_thread_id_ = ::GetCurrentThreadId();
+}
+
+void Mutex::Unlock() {
+  ThreadSafeLazyInit();
+  // We don't protect writing to owner_thread_id_ here, as it's the
+  // caller's responsibility to ensure that the current thread holds the
+  // mutex when this is called.
+  owner_thread_id_ = 0;
+  ::LeaveCriticalSection(critical_section_);
+}
+
+// Does nothing if the current thread holds the mutex. Otherwise, crashes
+// with high probability.
+void Mutex::AssertHeld() {
+  ThreadSafeLazyInit();
+  GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId())
+      << "The current thread is not holding the mutex @" << this;
+}
+
+// Initializes owner_thread_id_ and critical_section_ in static mutexes.
+void Mutex::ThreadSafeLazyInit() {
+  // Dynamic mutexes are initialized in the constructor.
+  if (type_ == kStatic) {
+    switch (
+        ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) {
+      case 0:
+        // If critical_section_init_phase_ was 0 before the exchange, we
+        // are the first to test it and need to perform the initialization.
+        owner_thread_id_ = 0;
+        critical_section_ = new CRITICAL_SECTION;
+        ::InitializeCriticalSection(critical_section_);
+        // Updates the critical_section_init_phase_ to 2 to signal
+        // initialization complete.
+        GTEST_CHECK_(::InterlockedCompareExchange(
+                          &critical_section_init_phase_, 2L, 1L) ==
+                      1L);
+        break;
+      case 1:
+        // Somebody else is already initializing the mutex; spin until they
+        // are done.
+        while (::InterlockedCompareExchange(&critical_section_init_phase_,
+                                            2L,
+                                            2L) != 2L) {
+          // Possibly yields the rest of the thread's time slice to other
+          // threads.
+          ::Sleep(0);
+        }
+        break;
+
+      case 2:
+        break;  // The mutex is already initialized and ready for use.
+
+      default:
+        GTEST_CHECK_(false)
+            << "Unexpected value of critical_section_init_phase_ "
+            << "while initializing a static mutex.";
+    }
+  }
+}
+
+namespace {
+
+class ThreadWithParamSupport : public ThreadWithParamBase {
+ public:
+  static HANDLE CreateThread(Runnable* runnable,
+                             Notification* thread_can_start) {
+    ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start);
+    DWORD thread_id;
+    // TODO(yukawa): Consider to use _beginthreadex instead.
+    HANDLE thread_handle = ::CreateThread(
+        NULL,    // Default security.
+        0,       // Default stack size.
+        &ThreadWithParamSupport::ThreadMain,
+        param,   // Parameter to ThreadMainStatic
+        0x0,     // Default creation flags.
+        &thread_id);  // Need a valid pointer for the call to work under Win98.
+    GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error "
+                                        << ::GetLastError() << ".";
+    if (thread_handle == NULL) {
+      delete param;
+    }
+    return thread_handle;
+  }
+
+ private:
+  struct ThreadMainParam {
+    ThreadMainParam(Runnable* runnable, Notification* thread_can_start)
+        : runnable_(runnable),
+          thread_can_start_(thread_can_start) {
+    }
+    scoped_ptr<Runnable> runnable_;
+    // Does not own.
+    Notification* thread_can_start_;
+  };
+
+  static DWORD WINAPI ThreadMain(void* ptr) {
+    // Transfers ownership.
+    scoped_ptr<ThreadMainParam> param(static_cast<ThreadMainParam*>(ptr));
+    if (param->thread_can_start_ != NULL)
+      param->thread_can_start_->WaitForNotification();
+    param->runnable_->Run();
+    return 0;
+  }
+
+  // Prohibit instantiation.
+  ThreadWithParamSupport();
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport);
+};
+
+}  // namespace
+
+ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable,
+                                         Notification* thread_can_start)
+      : thread_(ThreadWithParamSupport::CreateThread(runnable,
+                                                     thread_can_start)) {
+}
+
+ThreadWithParamBase::~ThreadWithParamBase() {
+  Join();
+}
+
+void ThreadWithParamBase::Join() {
+  GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0)
+      << "Failed to join the thread with error " << ::GetLastError() << ".";
+}
+
+// Maps a thread to a set of ThreadIdToThreadLocals that have values
+// instantiated on that thread and notifies them when the thread exits.  A
+// ThreadLocal instance is expected to persist until all threads it has
+// values on have terminated.
+class ThreadLocalRegistryImpl {
+ public:
+  // Registers thread_local_instance as having value on the current thread.
+  // Returns a value that can be used to identify the thread from other threads.
+  static ThreadLocalValueHolderBase* GetValueOnCurrentThread(
+      const ThreadLocalBase* thread_local_instance) {
+    DWORD current_thread = ::GetCurrentThreadId();
+    MutexLock lock(&mutex_);
+    ThreadIdToThreadLocals* const thread_to_thread_locals =
+        GetThreadLocalsMapLocked();
+    ThreadIdToThreadLocals::iterator thread_local_pos =
+        thread_to_thread_locals->find(current_thread);
+    if (thread_local_pos == thread_to_thread_locals->end()) {
+      thread_local_pos = thread_to_thread_locals->insert(
+          std::make_pair(current_thread, ThreadLocalValues())).first;
+      StartWatcherThreadFor(current_thread);
+    }
+    ThreadLocalValues& thread_local_values = thread_local_pos->second;
+    ThreadLocalValues::iterator value_pos =
+        thread_local_values.find(thread_local_instance);
+    if (value_pos == thread_local_values.end()) {
+      value_pos =
+          thread_local_values
+              .insert(std::make_pair(
+                  thread_local_instance,
+                  linked_ptr<ThreadLocalValueHolderBase>(
+                      thread_local_instance->NewValueForCurrentThread())))
+              .first;
+    }
+    return value_pos->second.get();
+  }
+
+  static void OnThreadLocalDestroyed(
+      const ThreadLocalBase* thread_local_instance) {
+    std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+    // Clean up the ThreadLocalValues data structure while holding the lock, but
+    // defer the destruction of the ThreadLocalValueHolderBases.
+    {
+      MutexLock lock(&mutex_);
+      ThreadIdToThreadLocals* const thread_to_thread_locals =
+          GetThreadLocalsMapLocked();
+      for (ThreadIdToThreadLocals::iterator it =
+          thread_to_thread_locals->begin();
+          it != thread_to_thread_locals->end();
+          ++it) {
+        ThreadLocalValues& thread_local_values = it->second;
+        ThreadLocalValues::iterator value_pos =
+            thread_local_values.find(thread_local_instance);
+        if (value_pos != thread_local_values.end()) {
+          value_holders.push_back(value_pos->second);
+          thread_local_values.erase(value_pos);
+          // This 'if' can only be successful at most once, so theoretically we
+          // could break out of the loop here, but we don't bother doing so.
+        }
+      }
+    }
+    // Outside the lock, let the destructor for 'value_holders' deallocate the
+    // ThreadLocalValueHolderBases.
+  }
+
+  static void OnThreadExit(DWORD thread_id) {
+    GTEST_CHECK_(thread_id != 0) << ::GetLastError();
+    std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders;
+    // Clean up the ThreadIdToThreadLocals data structure while holding the
+    // lock, but defer the destruction of the ThreadLocalValueHolderBases.
+    {
+      MutexLock lock(&mutex_);
+      ThreadIdToThreadLocals* const thread_to_thread_locals =
+          GetThreadLocalsMapLocked();
+      ThreadIdToThreadLocals::iterator thread_local_pos =
+          thread_to_thread_locals->find(thread_id);
+      if (thread_local_pos != thread_to_thread_locals->end()) {
+        ThreadLocalValues& thread_local_values = thread_local_pos->second;
+        for (ThreadLocalValues::iterator value_pos =
+            thread_local_values.begin();
+            value_pos != thread_local_values.end();
+            ++value_pos) {
+          value_holders.push_back(value_pos->second);
+        }
+        thread_to_thread_locals->erase(thread_local_pos);
+      }
+    }
+    // Outside the lock, let the destructor for 'value_holders' deallocate the
+    // ThreadLocalValueHolderBases.
+  }
+
+ private:
+  // In a particular thread, maps a ThreadLocal object to its value.
+  typedef std::map<const ThreadLocalBase*,
+                   linked_ptr<ThreadLocalValueHolderBase> > ThreadLocalValues;
+  // Stores all ThreadIdToThreadLocals having values in a thread, indexed by
+  // thread's ID.
+  typedef std::map<DWORD, ThreadLocalValues> ThreadIdToThreadLocals;
+
+  // Holds the thread id and thread handle that we pass from
+  // StartWatcherThreadFor to WatcherThreadFunc.
+  typedef std::pair<DWORD, HANDLE> ThreadIdAndHandle;
+
+  static void StartWatcherThreadFor(DWORD thread_id) {
+    // The returned handle will be kept in thread_map and closed by
+    // watcher_thread in WatcherThreadFunc.
+    HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION,
+                                 FALSE,
+                                 thread_id);
+    GTEST_CHECK_(thread != NULL);
+    // We need to to pass a valid thread ID pointer into CreateThread for it
+    // to work correctly under Win98.
+    DWORD watcher_thread_id;
+    HANDLE watcher_thread = ::CreateThread(
+        NULL,   // Default security.
+        0,      // Default stack size
+        &ThreadLocalRegistryImpl::WatcherThreadFunc,
+        reinterpret_cast<LPVOID>(new ThreadIdAndHandle(thread_id, thread)),
+        CREATE_SUSPENDED,
+        &watcher_thread_id);
+    GTEST_CHECK_(watcher_thread != NULL);
+    // Give the watcher thread the same priority as ours to avoid being
+    // blocked by it.
+    ::SetThreadPriority(watcher_thread,
+                        ::GetThreadPriority(::GetCurrentThread()));
+    ::ResumeThread(watcher_thread);
+    ::CloseHandle(watcher_thread);
+  }
+
+  // Monitors exit from a given thread and notifies those
+  // ThreadIdToThreadLocals about thread termination.
+  static DWORD WINAPI WatcherThreadFunc(LPVOID param) {
+    const ThreadIdAndHandle* tah =
+        reinterpret_cast<const ThreadIdAndHandle*>(param);
+    GTEST_CHECK_(
+        ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0);
+    OnThreadExit(tah->first);
+    ::CloseHandle(tah->second);
+    delete tah;
+    return 0;
+  }
+
+  // Returns map of thread local instances.
+  static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() {
+    mutex_.AssertHeld();
+    static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals;
+    return map;
+  }
+
+  // Protects access to GetThreadLocalsMapLocked() and its return value.
+  static Mutex mutex_;
+  // Protects access to GetThreadMapLocked() and its return value.
+  static Mutex thread_map_mutex_;
+};
+
+Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex);
+Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex);
+
+ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread(
+      const ThreadLocalBase* thread_local_instance) {
+  return ThreadLocalRegistryImpl::GetValueOnCurrentThread(
+      thread_local_instance);
+}
+
+void ThreadLocalRegistry::OnThreadLocalDestroyed(
+      const ThreadLocalBase* thread_local_instance) {
+  ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance);
+}
+
+#endif  // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS
+
+#if GTEST_USES_POSIX_RE
+
+// Implements RE.  Currently only needed for death tests.
+
+RE::~RE() {
+  if (is_valid_) {
+    // regfree'ing an invalid regex might crash because the content
+    // of the regex is undefined. Since the regex's are essentially
+    // the same, one cannot be valid (or invalid) without the other
+    // being so too.
+    regfree(&partial_regex_);
+    regfree(&full_regex_);
+  }
+  free(const_cast<char*>(pattern_));
+}
+
+// Returns true iff regular expression re matches the entire str.
+bool RE::FullMatch(const char* str, const RE& re) {
+  if (!re.is_valid_) return false;
+
+  regmatch_t match;
+  return regexec(&re.full_regex_, str, 1, &match, 0) == 0;
+}
+
+// Returns true iff regular expression re matches a substring of str
+// (including str itself).
+bool RE::PartialMatch(const char* str, const RE& re) {
+  if (!re.is_valid_) return false;
+
+  regmatch_t match;
+  return regexec(&re.partial_regex_, str, 1, &match, 0) == 0;
+}
+
+// Initializes an RE from its string representation.
+void RE::Init(const char* regex) {
+  pattern_ = posix::StrDup(regex);
+
+  // Reserves enough bytes to hold the regular expression used for a
+  // full match.
+  const size_t full_regex_len = strlen(regex) + 10;
+  char* const full_pattern = new char[full_regex_len];
+
+  snprintf(full_pattern, full_regex_len, "^(%s)$", regex);
+  is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0;
+  // We want to call regcomp(&partial_regex_, ...) even if the
+  // previous expression returns false.  Otherwise partial_regex_ may
+  // not be properly initialized can may cause trouble when it's
+  // freed.
+  //
+  // Some implementation of POSIX regex (e.g. on at least some
+  // versions of Cygwin) doesn't accept the empty string as a valid
+  // regex.  We change it to an equivalent form "()" to be safe.
+  if (is_valid_) {
+    const char* const partial_regex = (*regex == '\0') ? "()" : regex;
+    is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0;
+  }
+  EXPECT_TRUE(is_valid_)
+      << "Regular expression \"" << regex
+      << "\" is not a valid POSIX Extended regular expression.";
+
+  delete[] full_pattern;
+}
+
+#elif GTEST_USES_SIMPLE_RE
+
+// Returns true iff ch appears anywhere in str (excluding the
+// terminating '\0' character).
+bool IsInSet(char ch, const char* str) {
+  return ch != '\0' && strchr(str, ch) != NULL;
+}
+
+// Returns true iff ch belongs to the given classification.  Unlike
+// similar functions in <ctype.h>, these aren't affected by the
+// current locale.
+bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; }
+bool IsAsciiPunct(char ch) {
+  return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~");
+}
+bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); }
+bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); }
+bool IsAsciiWordChar(char ch) {
+  return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') ||
+      ('0' <= ch && ch <= '9') || ch == '_';
+}
+
+// Returns true iff "\\c" is a supported escape sequence.
+bool IsValidEscape(char c) {
+  return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW"));
+}
+
+// Returns true iff the given atom (specified by escaped and pattern)
+// matches ch.  The result is undefined if the atom is invalid.
+bool AtomMatchesChar(bool escaped, char pattern_char, char ch) {
+  if (escaped) {  // "\\p" where p is pattern_char.
+    switch (pattern_char) {
+      case 'd': return IsAsciiDigit(ch);
+      case 'D': return !IsAsciiDigit(ch);
+      case 'f': return ch == '\f';
+      case 'n': return ch == '\n';
+      case 'r': return ch == '\r';
+      case 's': return IsAsciiWhiteSpace(ch);
+      case 'S': return !IsAsciiWhiteSpace(ch);
+      case 't': return ch == '\t';
+      case 'v': return ch == '\v';
+      case 'w': return IsAsciiWordChar(ch);
+      case 'W': return !IsAsciiWordChar(ch);
+    }
+    return IsAsciiPunct(pattern_char) && pattern_char == ch;
+  }
+
+  return (pattern_char == '.' && ch != '\n') || pattern_char == ch;
+}
+
+// Helper function used by ValidateRegex() to format error messages.
+std::string FormatRegexSyntaxError(const char* regex, int index) {
+  return (Message() << "Syntax error at index " << index
+          << " in simple regular expression \"" << regex << "\": ").GetString();
+}
+
+// Generates non-fatal failures and returns false if regex is invalid;
+// otherwise returns true.
+bool ValidateRegex(const char* regex) {
+  if (regex == NULL) {
+    // TODO(wan@google.com): fix the source file location in the
+    // assertion failures to match where the regex is used in user
+    // code.
+    ADD_FAILURE() << "NULL is not a valid simple regular expression.";
+    return false;
+  }
+
+  bool is_valid = true;
+
+  // True iff ?, *, or + can follow the previous atom.
+  bool prev_repeatable = false;
+  for (int i = 0; regex[i]; i++) {
+    if (regex[i] == '\\') {  // An escape sequence
+      i++;
+      if (regex[i] == '\0') {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1)
+                      << "'\\' cannot appear at the end.";
+        return false;
+      }
+
+      if (!IsValidEscape(regex[i])) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1)
+                      << "invalid escape sequence \"\\" << regex[i] << "\".";
+        is_valid = false;
+      }
+      prev_repeatable = true;
+    } else {  // Not an escape sequence.
+      const char ch = regex[i];
+
+      if (ch == '^' && i > 0) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'^' can only appear at the beginning.";
+        is_valid = false;
+      } else if (ch == '$' && regex[i + 1] != '\0') {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'$' can only appear at the end.";
+        is_valid = false;
+      } else if (IsInSet(ch, "()[]{}|")) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'" << ch << "' is unsupported.";
+        is_valid = false;
+      } else if (IsRepeat(ch) && !prev_repeatable) {
+        ADD_FAILURE() << FormatRegexSyntaxError(regex, i)
+                      << "'" << ch << "' can only follow a repeatable token.";
+        is_valid = false;
+      }
+
+      prev_repeatable = !IsInSet(ch, "^$?*+");
+    }
+  }
+
+  return is_valid;
+}
+
+// Matches a repeated regex atom followed by a valid simple regular
+// expression.  The regex atom is defined as c if escaped is false,
+// or \c otherwise.  repeat is the repetition meta character (?, *,
+// or +).  The behavior is undefined if str contains too many
+// characters to be indexable by size_t, in which case the test will
+// probably time out anyway.  We are fine with this limitation as
+// std::string has it too.
+bool MatchRepetitionAndRegexAtHead(
+    bool escaped, char c, char repeat, const char* regex,
+    const char* str) {
+  const size_t min_count = (repeat == '+') ? 1 : 0;
+  const size_t max_count = (repeat == '?') ? 1 :
+      static_cast<size_t>(-1) - 1;
+  // We cannot call numeric_limits::max() as it conflicts with the
+  // max() macro on Windows.
+
+  for (size_t i = 0; i <= max_count; ++i) {
+    // We know that the atom matches each of the first i characters in str.
+    if (i >= min_count && MatchRegexAtHead(regex, str + i)) {
+      // We have enough matches at the head, and the tail matches too.
+      // Since we only care about *whether* the pattern matches str
+      // (as opposed to *how* it matches), there is no need to find a
+      // greedy match.
+      return true;
+    }
+    if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i]))
+      return false;
+  }
+  return false;
+}
+
+// Returns true iff regex matches a prefix of str.  regex must be a
+// valid simple regular expression and not start with "^", or the
+// result is undefined.
+bool MatchRegexAtHead(const char* regex, const char* str) {
+  if (*regex == '\0')  // An empty regex matches a prefix of anything.
+    return true;
+
+  // "$" only matches the end of a string.  Note that regex being
+  // valid guarantees that there's nothing after "$" in it.
+  if (*regex == '$')
+    return *str == '\0';
+
+  // Is the first thing in regex an escape sequence?
+  const bool escaped = *regex == '\\';
+  if (escaped)
+    ++regex;
+  if (IsRepeat(regex[1])) {
+    // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so
+    // here's an indirect recursion.  It terminates as the regex gets
+    // shorter in each recursion.
+    return MatchRepetitionAndRegexAtHead(
+        escaped, regex[0], regex[1], regex + 2, str);
+  } else {
+    // regex isn't empty, isn't "$", and doesn't start with a
+    // repetition.  We match the first atom of regex with the first
+    // character of str and recurse.
+    return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) &&
+        MatchRegexAtHead(regex + 1, str + 1);
+  }
+}
+
+// Returns true iff regex matches any substring of str.  regex must be
+// a valid simple regular expression, or the result is undefined.
+//
+// The algorithm is recursive, but the recursion depth doesn't exceed
+// the regex length, so we won't need to worry about running out of
+// stack space normally.  In rare cases the time complexity can be
+// exponential with respect to the regex length + the string length,
+// but usually it's must faster (often close to linear).
+bool MatchRegexAnywhere(const char* regex, const char* str) {
+  if (regex == NULL || str == NULL)
+    return false;
+
+  if (*regex == '^')
+    return MatchRegexAtHead(regex + 1, str);
+
+  // A successful match can be anywhere in str.
+  do {
+    if (MatchRegexAtHead(regex, str))
+      return true;
+  } while (*str++ != '\0');
+  return false;
+}
+
+// Implements the RE class.
+
+RE::~RE() {
+  free(const_cast<char*>(pattern_));
+  free(const_cast<char*>(full_pattern_));
+}
+
+// Returns true iff regular expression re matches the entire str.
+bool RE::FullMatch(const char* str, const RE& re) {
+  return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str);
+}
+
+// Returns true iff regular expression re matches a substring of str
+// (including str itself).
+bool RE::PartialMatch(const char* str, const RE& re) {
+  return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str);
+}
+
+// Initializes an RE from its string representation.
+void RE::Init(const char* regex) {
+  pattern_ = full_pattern_ = NULL;
+  if (regex != NULL) {
+    pattern_ = posix::StrDup(regex);
+  }
+
+  is_valid_ = ValidateRegex(regex);
+  if (!is_valid_) {
+    // No need to calculate the full pattern when the regex is invalid.
+    return;
+  }
+
+  const size_t len = strlen(regex);
+  // Reserves enough bytes to hold the regular expression used for a
+  // full match: we need space to prepend a '^', append a '$', and
+  // terminate the string with '\0'.
+  char* buffer = static_cast<char*>(malloc(len + 3));
+  full_pattern_ = buffer;
+
+  if (*regex != '^')
+    *buffer++ = '^';  // Makes sure full_pattern_ starts with '^'.
+
+  // We don't use snprintf or strncpy, as they trigger a warning when
+  // compiled with VC++ 8.0.
+  memcpy(buffer, regex, len);
+  buffer += len;
+
+  if (len == 0 || regex[len - 1] != '$')
+    *buffer++ = '$';  // Makes sure full_pattern_ ends with '$'.
+
+  *buffer = '\0';
+}
+
+#endif  // GTEST_USES_POSIX_RE
+
+const char kUnknownFile[] = "unknown file";
+
+// Formats a source file path and a line number as they would appear
+// in an error message from the compiler used to compile this code.
+GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) {
+  const std::string file_name(file == NULL ? kUnknownFile : file);
+
+  if (line < 0) {
+    return file_name + ":";
+  }
+#ifdef _MSC_VER
+  return file_name + "(" + StreamableToString(line) + "):";
+#else
+  return file_name + ":" + StreamableToString(line) + ":";
+#endif  // _MSC_VER
+}
+
+// Formats a file location for compiler-independent XML output.
+// Although this function is not platform dependent, we put it next to
+// FormatFileLocation in order to contrast the two functions.
+// Note that FormatCompilerIndependentFileLocation() does NOT append colon
+// to the file location it produces, unlike FormatFileLocation().
+GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(
+    const char* file, int line) {
+  const std::string file_name(file == NULL ? kUnknownFile : file);
+
+  if (line < 0)
+    return file_name;
+  else
+    return file_name + ":" + StreamableToString(line);
+}
+
+GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line)
+    : severity_(severity) {
+  const char* const marker =
+      severity == GTEST_INFO ?    "[  INFO ]" :
+      severity == GTEST_WARNING ? "[WARNING]" :
+      severity == GTEST_ERROR ?   "[ ERROR ]" : "[ FATAL ]";
+  GetStream() << ::std::endl << marker << " "
+              << FormatFileLocation(file, line).c_str() << ": ";
+}
+
+// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program.
+GTestLog::~GTestLog() {
+  GetStream() << ::std::endl;
+  if (severity_ == GTEST_FATAL) {
+    fflush(stderr);
+    posix::Abort();
+  }
+}
+// Disable Microsoft deprecation warnings for POSIX functions called from
+// this class (creat, dup, dup2, and close)
+GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
+
+#if GTEST_HAS_STREAM_REDIRECTION
+
+// Object that captures an output stream (stdout/stderr).
+class CapturedStream {
+ public:
+  // The ctor redirects the stream to a temporary file.
+  explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) {
+# if GTEST_OS_WINDOWS
+    char temp_dir_path[MAX_PATH + 1] = { '\0' };  // NOLINT
+    char temp_file_path[MAX_PATH + 1] = { '\0' };  // NOLINT
+
+    ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path);
+    const UINT success = ::GetTempFileNameA(temp_dir_path,
+                                            "gtest_redir",
+                                            0,  // Generate unique file name.
+                                            temp_file_path);
+    GTEST_CHECK_(success != 0)
+        << "Unable to create a temporary file in " << temp_dir_path;
+    const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE);
+    GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file "
+                                    << temp_file_path;
+    filename_ = temp_file_path;
+# else
+    // There's no guarantee that a test has write access to the current
+    // directory, so we create the temporary file in the /tmp directory
+    // instead. We use /tmp on most systems, and /sdcard on Android.
+    // That's because Android doesn't have /tmp.
+#  if GTEST_OS_LINUX_ANDROID
+    // Note: Android applications are expected to call the framework's
+    // Context.getExternalStorageDirectory() method through JNI to get
+    // the location of the world-writable SD Card directory. However,
+    // this requires a Context handle, which cannot be retrieved
+    // globally from native code. Doing so also precludes running the
+    // code as part of a regular standalone executable, which doesn't
+    // run in a Dalvik process (e.g. when running it through 'adb shell').
+    //
+    // The location /sdcard is directly accessible from native code
+    // and is the only location (unofficially) supported by the Android
+    // team. It's generally a symlink to the real SD Card mount point
+    // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or
+    // other OEM-customized locations. Never rely on these, and always
+    // use /sdcard.
+    char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX";
+#  else
+    char name_template[] = "/tmp/captured_stream.XXXXXX";
+#  endif  // GTEST_OS_LINUX_ANDROID
+    const int captured_fd = mkstemp(name_template);
+    filename_ = name_template;
+# endif  // GTEST_OS_WINDOWS
+    fflush(NULL);
+    dup2(captured_fd, fd_);
+    close(captured_fd);
+  }
+
+  ~CapturedStream() {
+    remove(filename_.c_str());
+  }
+
+  std::string GetCapturedString() {
+    if (uncaptured_fd_ != -1) {
+      // Restores the original stream.
+      fflush(NULL);
+      dup2(uncaptured_fd_, fd_);
+      close(uncaptured_fd_);
+      uncaptured_fd_ = -1;
+    }
+
+    FILE* const file = posix::FOpen(filename_.c_str(), "r");
+    const std::string content = ReadEntireFile(file);
+    posix::FClose(file);
+    return content;
+  }
+
+ private:
+  const int fd_;  // A stream to capture.
+  int uncaptured_fd_;
+  // Name of the temporary file holding the stderr output.
+  ::std::string filename_;
+
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream);
+};
+
+GTEST_DISABLE_MSC_WARNINGS_POP_()
+
+static CapturedStream* g_captured_stderr = NULL;
+static CapturedStream* g_captured_stdout = NULL;
+
+// Starts capturing an output stream (stdout/stderr).
+void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) {
+  if (*stream != NULL) {
+    GTEST_LOG_(FATAL) << "Only one " << stream_name
+                      << " capturer can exist at a time.";
+  }
+  *stream = new CapturedStream(fd);
+}
+
+// Stops capturing the output stream and returns the captured string.
+std::string GetCapturedStream(CapturedStream** captured_stream) {
+  const std::string content = (*captured_stream)->GetCapturedString();
+
+  delete *captured_stream;
+  *captured_stream = NULL;
+
+  return content;
+}
+
+// Starts capturing stdout.
+void CaptureStdout() {
+  CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout);
+}
+
+// Starts capturing stderr.
+void CaptureStderr() {
+  CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr);
+}
+
+// Stops capturing stdout and returns the captured string.
+std::string GetCapturedStdout() {
+  return GetCapturedStream(&g_captured_stdout);
+}
+
+// Stops capturing stderr and returns the captured string.
+std::string GetCapturedStderr() {
+  return GetCapturedStream(&g_captured_stderr);
+}
+
+#endif  // GTEST_HAS_STREAM_REDIRECTION
+
+std::string TempDir() {
+#if GTEST_OS_WINDOWS_MOBILE
+  return "\\temp\\";
+#elif GTEST_OS_WINDOWS
+  const char* temp_dir = posix::GetEnv("TEMP");
+  if (temp_dir == NULL || temp_dir[0] == '\0')
+    return "\\temp\\";
+  else if (temp_dir[strlen(temp_dir) - 1] == '\\')
+    return temp_dir;
+  else
+    return std::string(temp_dir) + "\\";
+#elif GTEST_OS_LINUX_ANDROID
+  return "/sdcard/";
+#else
+  return "/tmp/";
+#endif  // GTEST_OS_WINDOWS_MOBILE
+}
+
+size_t GetFileSize(FILE* file) {
+  fseek(file, 0, SEEK_END);
+  return static_cast<size_t>(ftell(file));
+}
+
+std::string ReadEntireFile(FILE* file) {
+  const size_t file_size = GetFileSize(file);
+  char* const buffer = new char[file_size];
+
+  size_t bytes_last_read = 0;  // # of bytes read in the last fread()
+  size_t bytes_read = 0;       // # of bytes read so far
+
+  fseek(file, 0, SEEK_SET);
+
+  // Keeps reading the file until we cannot read further or the
+  // pre-determined file size is reached.
+  do {
+    bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file);
+    bytes_read += bytes_last_read;
+  } while (bytes_last_read > 0 && bytes_read < file_size);
+
+  const std::string content(buffer, bytes_read);
+  delete[] buffer;
+
+  return content;
+}
+
+#if GTEST_HAS_DEATH_TEST
+
+static const ::std::vector<testing::internal::string>* g_injected_test_argvs =
+                                        NULL;  // Owned.
+
+void SetInjectableArgvs(const ::std::vector<testing::internal::string>* argvs) {
+  if (g_injected_test_argvs != argvs)
+    delete g_injected_test_argvs;
+  g_injected_test_argvs = argvs;
+}
+
+const ::std::vector<testing::internal::string>& GetInjectableArgvs() {
+  if (g_injected_test_argvs != NULL) {
+    return *g_injected_test_argvs;
+  }
+  return GetArgvs();
+}
+#endif  // GTEST_HAS_DEATH_TEST
+
+#if GTEST_OS_WINDOWS_MOBILE
+namespace posix {
+void Abort() {
+  DebugBreak();
+  TerminateProcess(GetCurrentProcess(), 1);
+}
+}  // namespace posix
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+// Returns the name of the environment variable corresponding to the
+// given flag.  For example, FlagToEnvVar("foo") will return
+// "GTEST_FOO" in the open-source version.
+static std::string FlagToEnvVar(const char* flag) {
+  const std::string full_flag =
+      (Message() << GTEST_FLAG_PREFIX_ << flag).GetString();
+
+  Message env_var;
+  for (size_t i = 0; i != full_flag.length(); i++) {
+    env_var << ToUpper(full_flag.c_str()[i]);
+  }
+
+  return env_var.GetString();
+}
+
+// Parses 'str' for a 32-bit signed integer.  If successful, writes
+// the result to *value and returns true; otherwise leaves *value
+// unchanged and returns false.
+bool ParseInt32(const Message& src_text, const char* str, Int32* value) {
+  // Parses the environment variable as a decimal integer.
+  char* end = NULL;
+  const long long_value = strtol(str, &end, 10);  // NOLINT
+
+  // Has strtol() consumed all characters in the string?
+  if (*end != '\0') {
+    // No - an invalid character was encountered.
+    Message msg;
+    msg << "WARNING: " << src_text
+        << " is expected to be a 32-bit integer, but actually"
+        << " has value \"" << str << "\".\n";
+    printf("%s", msg.GetString().c_str());
+    fflush(stdout);
+    return false;
+  }
+
+  // Is the parsed value in the range of an Int32?
+  const Int32 result = static_cast<Int32>(long_value);
+  if (long_value == LONG_MAX || long_value == LONG_MIN ||
+      // The parsed value overflows as a long.  (strtol() returns
+      // LONG_MAX or LONG_MIN when the input overflows.)
+      result != long_value
+      // The parsed value overflows as an Int32.
+      ) {
+    Message msg;
+    msg << "WARNING: " << src_text
+        << " is expected to be a 32-bit integer, but actually"
+        << " has value " << str << ", which overflows.\n";
+    printf("%s", msg.GetString().c_str());
+    fflush(stdout);
+    return false;
+  }
+
+  *value = result;
+  return true;
+}
+
+// Reads and returns the Boolean environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+//
+// The value is considered true iff it's not "0".
+bool BoolFromGTestEnv(const char* flag, bool default_value) {
+#if defined(GTEST_GET_BOOL_FROM_ENV_)
+  return GTEST_GET_BOOL_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_BOOL_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* const string_value = posix::GetEnv(env_var.c_str());
+  return string_value == NULL ?
+      default_value : strcmp(string_value, "0") != 0;
+}
+
+// Reads and returns a 32-bit integer stored in the environment
+// variable corresponding to the given flag; if it isn't set or
+// doesn't represent a valid 32-bit integer, returns default_value.
+Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) {
+#if defined(GTEST_GET_INT32_FROM_ENV_)
+  return GTEST_GET_INT32_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_INT32_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* const string_value = posix::GetEnv(env_var.c_str());
+  if (string_value == NULL) {
+    // The environment variable is not set.
+    return default_value;
+  }
+
+  Int32 result = default_value;
+  if (!ParseInt32(Message() << "Environment variable " << env_var,
+                  string_value, &result)) {
+    printf("The default value %s is used.\n",
+           (Message() << default_value).GetString().c_str());
+    fflush(stdout);
+    return default_value;
+  }
+
+  return result;
+}
+
+// Reads and returns the string environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+std::string StringFromGTestEnv(const char* flag, const char* default_value) {
+#if defined(GTEST_GET_STRING_FROM_ENV_)
+  return GTEST_GET_STRING_FROM_ENV_(flag, default_value);
+#endif  // defined(GTEST_GET_STRING_FROM_ENV_)
+  const std::string env_var = FlagToEnvVar(flag);
+  const char* value = posix::GetEnv(env_var.c_str());
+  if (value != NULL) {
+    return value;
+  }
+
+  // As a special case for the 'output' flag, if GTEST_OUTPUT is not
+  // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build
+  // system.  The value of XML_OUTPUT_FILE is a filename without the
+  // "xml:" prefix of GTEST_OUTPUT.
+  //
+  // The net priority order after flag processing is thus:
+  //   --gtest_output command line flag
+  //   GTEST_OUTPUT environment variable
+  //   XML_OUTPUT_FILE environment variable
+  //   'default_value'
+  if (strcmp(flag, "output") == 0) {
+    value = posix::GetEnv("XML_OUTPUT_FILE");
+    if (value != NULL) {
+      return std::string("xml:") + value;
+    }
+  }
+  return default_value;
+}
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+// Google Test - The Google C++ Testing Framework
+//
+// This file implements a universal value printer that can print a
+// value of any type T:
+//
+//   void ::testing::internal::UniversalPrinter<T>::Print(value, ostream_ptr);
+//
+// It uses the << operator when possible, and prints the bytes in the
+// object otherwise.  A user can override its behavior for a class
+// type Foo by defining either operator<<(::std::ostream&, const Foo&)
+// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that
+// defines Foo.
+
+#include "gtest/gtest-printers.h"
+#include <ctype.h>
+#include <stdio.h>
+#include <cwchar>
+#include <ostream>  // NOLINT
+#include <string>
+#include "gtest/internal/gtest-port.h"
+
+namespace testing {
+
+namespace {
+
+using ::std::ostream;
+
+// Prints a segment of bytes in the given object.
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start,
+                                size_t count, ostream* os) {
+  char text[5] = "";
+  for (size_t i = 0; i != count; i++) {
+    const size_t j = start + i;
+    if (i != 0) {
+      // Organizes the bytes into groups of 2 for easy parsing by
+      // human.
+      if ((j % 2) == 0)
+        *os << ' ';
+      else
+        *os << '-';
+    }
+    GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]);
+    *os << text;
+  }
+}
+
+// Prints the bytes in the given value to the given ostream.
+void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count,
+                              ostream* os) {
+  // Tells the user how big the object is.
+  *os << count << "-byte object <";
+
+  const size_t kThreshold = 132;
+  const size_t kChunkSize = 64;
+  // If the object size is bigger than kThreshold, we'll have to omit
+  // some details by printing only the first and the last kChunkSize
+  // bytes.
+  // TODO(wan): let the user control the threshold using a flag.
+  if (count < kThreshold) {
+    PrintByteSegmentInObjectTo(obj_bytes, 0, count, os);
+  } else {
+    PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os);
+    *os << " ... ";
+    // Rounds up to 2-byte boundary.
+    const size_t resume_pos = (count - kChunkSize + 1)/2*2;
+    PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os);
+  }
+  *os << ">";
+}
+
+}  // namespace
+
+namespace internal2 {
+
+// Delegates to PrintBytesInObjectToImpl() to print the bytes in the
+// given object.  The delegation simplifies the implementation, which
+// uses the << operator and thus is easier done outside of the
+// ::testing::internal namespace, which contains a << operator that
+// sometimes conflicts with the one in STL.
+void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count,
+                          ostream* os) {
+  PrintBytesInObjectToImpl(obj_bytes, count, os);
+}
+
+}  // namespace internal2
+
+namespace internal {
+
+// Depending on the value of a char (or wchar_t), we print it in one
+// of three formats:
+//   - as is if it's a printable ASCII (e.g. 'a', '2', ' '),
+//   - as a hexidecimal escape sequence (e.g. '\x7F'), or
+//   - as a special escape sequence (e.g. '\r', '\n').
+enum CharFormat {
+  kAsIs,
+  kHexEscape,
+  kSpecialEscape
+};
+
+// Returns true if c is a printable ASCII character.  We test the
+// value of c directly instead of calling isprint(), which is buggy on
+// Windows Mobile.
+inline bool IsPrintableAscii(wchar_t c) {
+  return 0x20 <= c && c <= 0x7E;
+}
+
+// Prints a wide or narrow char c as a character literal without the
+// quotes, escaping it when necessary; returns how c was formatted.
+// The template argument UnsignedChar is the unsigned version of Char,
+// which is the type of c.
+template <typename UnsignedChar, typename Char>
+static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) {
+  switch (static_cast<wchar_t>(c)) {
+    case L'\0':
+      *os << "\\0";
+      break;
+    case L'\'':
+      *os << "\\'";
+      break;
+    case L'\\':
+      *os << "\\\\";
+      break;
+    case L'\a':
+      *os << "\\a";
+      break;
+    case L'\b':
+      *os << "\\b";
+      break;
+    case L'\f':
+      *os << "\\f";
+      break;
+    case L'\n':
+      *os << "\\n";
+      break;
+    case L'\r':
+      *os << "\\r";
+      break;
+    case L'\t':
+      *os << "\\t";
+      break;
+    case L'\v':
+      *os << "\\v";
+      break;
+    default:
+      if (IsPrintableAscii(c)) {
+        *os << static_cast<char>(c);
+        return kAsIs;
+      } else {
+        *os << "\\x" + String::FormatHexInt(static_cast<UnsignedChar>(c));
+        return kHexEscape;
+      }
+  }
+  return kSpecialEscape;
+}
+
+// Prints a wchar_t c as if it's part of a string literal, escaping it when
+// necessary; returns how c was formatted.
+static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) {
+  switch (c) {
+    case L'\'':
+      *os << "'";
+      return kAsIs;
+    case L'"':
+      *os << "\\\"";
+      return kSpecialEscape;
+    default:
+      return PrintAsCharLiteralTo<wchar_t>(c, os);
+  }
+}
+
+// Prints a char c as if it's part of a string literal, escaping it when
+// necessary; returns how c was formatted.
+static CharFormat PrintAsStringLiteralTo(char c, ostream* os) {
+  return PrintAsStringLiteralTo(
+      static_cast<wchar_t>(static_cast<unsigned char>(c)), os);
+}
+
+// Prints a wide or narrow character c and its code.  '\0' is printed
+// as "'\\0'", other unprintable characters are also properly escaped
+// using the standard C++ escape sequence.  The template argument
+// UnsignedChar is the unsigned version of Char, which is the type of c.
+template <typename UnsignedChar, typename Char>
+void PrintCharAndCodeTo(Char c, ostream* os) {
+  // First, print c as a literal in the most readable form we can find.
+  *os << ((sizeof(c) > 1) ? "L'" : "'");
+  const CharFormat format = PrintAsCharLiteralTo<UnsignedChar>(c, os);
+  *os << "'";
+
+  // To aid user debugging, we also print c's code in decimal, unless
+  // it's 0 (in which case c was printed as '\\0', making the code
+  // obvious).
+  if (c == 0)
+    return;
+  *os << " (" << static_cast<int>(c);
+
+  // For more convenience, we print c's code again in hexidecimal,
+  // unless c was already printed in the form '\x##' or the code is in
+  // [1, 9].
+  if (format == kHexEscape || (1 <= c && c <= 9)) {
+    // Do nothing.
+  } else {
+    *os << ", 0x" << String::FormatHexInt(static_cast<UnsignedChar>(c));
+  }
+  *os << ")";
+}
+
+void PrintTo(unsigned char c, ::std::ostream* os) {
+  PrintCharAndCodeTo<unsigned char>(c, os);
+}
+void PrintTo(signed char c, ::std::ostream* os) {
+  PrintCharAndCodeTo<unsigned char>(c, os);
+}
+
+// Prints a wchar_t as a symbol if it is printable or as its internal
+// code otherwise and also as its code.  L'\0' is printed as "L'\\0'".
+void PrintTo(wchar_t wc, ostream* os) {
+  PrintCharAndCodeTo<wchar_t>(wc, os);
+}
+
+// Prints the given array of characters to the ostream.  CharType must be either
+// char or wchar_t.
+// The array starts at begin, the length is len, it may include '\0' characters
+// and may not be NUL-terminated.
+template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+static void PrintCharsAsStringTo(
+    const CharType* begin, size_t len, ostream* os) {
+  const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\"";
+  *os << kQuoteBegin;
+  bool is_previous_hex = false;
+  for (size_t index = 0; index < len; ++index) {
+    const CharType cur = begin[index];
+    if (is_previous_hex && IsXDigit(cur)) {
+      // Previous character is of '\x..' form and this character can be
+      // interpreted as another hexadecimal digit in its number. Break string to
+      // disambiguate.
+      *os << "\" " << kQuoteBegin;
+    }
+    is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape;
+  }
+  *os << "\"";
+}
+
+// Prints a (const) char/wchar_t array of 'len' elements, starting at address
+// 'begin'.  CharType must be either char or wchar_t.
+template <typename CharType>
+GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_
+GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_
+static void UniversalPrintCharArray(
+    const CharType* begin, size_t len, ostream* os) {
+  // The code
+  //   const char kFoo[] = "foo";
+  // generates an array of 4, not 3, elements, with the last one being '\0'.
+  //
+  // Therefore when printing a char array, we don't print the last element if
+  // it's '\0', such that the output matches the string literal as it's
+  // written in the source code.
+  if (len > 0 && begin[len - 1] == '\0') {
+    PrintCharsAsStringTo(begin, len - 1, os);
+    return;
+  }
+
+  // If, however, the last element in the array is not '\0', e.g.
+  //    const char kFoo[] = { 'f', 'o', 'o' };
+  // we must print the entire array.  We also print a message to indicate
+  // that the array is not NUL-terminated.
+  PrintCharsAsStringTo(begin, len, os);
+  *os << " (no terminating NUL)";
+}
+
+// Prints a (const) char array of 'len' elements, starting at address 'begin'.
+void UniversalPrintArray(const char* begin, size_t len, ostream* os) {
+  UniversalPrintCharArray(begin, len, os);
+}
+
+// Prints a (const) wchar_t array of 'len' elements, starting at address
+// 'begin'.
+void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) {
+  UniversalPrintCharArray(begin, len, os);
+}
+
+// Prints the given C string to the ostream.
+void PrintTo(const char* s, ostream* os) {
+  if (s == NULL) {
+    *os << "NULL";
+  } else {
+    *os << ImplicitCast_<const void*>(s) << " pointing to ";
+    PrintCharsAsStringTo(s, strlen(s), os);
+  }
+}
+
+// MSVC compiler can be configured to define whar_t as a typedef
+// of unsigned short. Defining an overload for const wchar_t* in that case
+// would cause pointers to unsigned shorts be printed as wide strings,
+// possibly accessing more memory than intended and causing invalid
+// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when
+// wchar_t is implemented as a native type.
+#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED)
+// Prints the given wide C string to the ostream.
+void PrintTo(const wchar_t* s, ostream* os) {
+  if (s == NULL) {
+    *os << "NULL";
+  } else {
+    *os << ImplicitCast_<const void*>(s) << " pointing to ";
+    PrintCharsAsStringTo(s, std::wcslen(s), os);
+  }
+}
+#endif  // wchar_t is native
+
+// Prints a ::string object.
+#if GTEST_HAS_GLOBAL_STRING
+void PrintStringTo(const ::string& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_GLOBAL_STRING
+
+void PrintStringTo(const ::std::string& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+
+// Prints a ::wstring object.
+#if GTEST_HAS_GLOBAL_WSTRING
+void PrintWideStringTo(const ::wstring& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+#if GTEST_HAS_STD_WSTRING
+void PrintWideStringTo(const ::std::wstring& s, ostream* os) {
+  PrintCharsAsStringTo(s.data(), s.size(), os);
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+}  // namespace internal
+
+}  // namespace testing
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: mheule@google.com (Markus Heule)
+//
+// The Google C++ Testing Framework (Google Test)
+
+#include "gtest/gtest-test-part.h"
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick exists to
+// prevent the accidental inclusion of gtest-internal-inl.h in the
+// user's code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+namespace testing {
+
+using internal::GetUnitTestImpl;
+
+// Gets the summary of the failure message by omitting the stack trace
+// in it.
+std::string TestPartResult::ExtractSummary(const char* message) {
+  const char* const stack_trace = strstr(message, internal::kStackTraceMarker);
+  return stack_trace == NULL ? message :
+      std::string(message, stack_trace);
+}
+
+// Prints a TestPartResult object.
+std::ostream& operator<<(std::ostream& os, const TestPartResult& result) {
+  return os
+      << result.file_name() << ":" << result.line_number() << ": "
+      << (result.type() == TestPartResult::kSuccess ? "Success" :
+          result.type() == TestPartResult::kFatalFailure ? "Fatal failure" :
+          "Non-fatal failure") << ":\n"
+      << result.message() << std::endl;
+}
+
+// Appends a TestPartResult to the array.
+void TestPartResultArray::Append(const TestPartResult& result) {
+  array_.push_back(result);
+}
+
+// Returns the TestPartResult at the given index (0-based).
+const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const {
+  if (index < 0 || index >= size()) {
+    printf("\nInvalid index (%d) into TestPartResultArray.\n", index);
+    internal::posix::Abort();
+  }
+
+  return array_[index];
+}
+
+// Returns the number of TestPartResult objects in the array.
+int TestPartResultArray::size() const {
+  return static_cast<int>(array_.size());
+}
+
+namespace internal {
+
+HasNewFatalFailureHelper::HasNewFatalFailureHelper()
+    : has_new_fatal_failure_(false),
+      original_reporter_(GetUnitTestImpl()->
+                         GetTestPartResultReporterForCurrentThread()) {
+  GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this);
+}
+
+HasNewFatalFailureHelper::~HasNewFatalFailureHelper() {
+  GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(
+      original_reporter_);
+}
+
+void HasNewFatalFailureHelper::ReportTestPartResult(
+    const TestPartResult& result) {
+  if (result.fatally_failed())
+    has_new_fatal_failure_ = true;
+  original_reporter_->ReportTestPartResult(result);
+}
+
+}  // namespace internal
+
+}  // namespace testing
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+
+#include "gtest/gtest-typed-test.h"
+#include "gtest/gtest.h"
+
+namespace testing {
+namespace internal {
+
+#if GTEST_HAS_TYPED_TEST_P
+
+// Skips to the first non-space char in str. Returns an empty string if str
+// contains only whitespace characters.
+static const char* SkipSpaces(const char* str) {
+  while (IsSpace(*str))
+    str++;
+  return str;
+}
+
+static std::vector<std::string> SplitIntoTestNames(const char* src) {
+  std::vector<std::string> name_vec;
+  src = SkipSpaces(src);
+  for (; src != NULL; src = SkipComma(src)) {
+    name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src)));
+  }
+  return name_vec;
+}
+
+// Verifies that registered_tests match the test names in
+// registered_tests_; returns registered_tests if successful, or
+// aborts the program otherwise.
+const char* TypedTestCasePState::VerifyRegisteredTestNames(
+    const char* file, int line, const char* registered_tests) {
+  typedef RegisteredTestsMap::const_iterator RegisteredTestIter;
+  registered_ = true;
+
+  std::vector<std::string> name_vec = SplitIntoTestNames(registered_tests);
+
+  Message errors;
+
+  std::set<std::string> tests;
+  for (std::vector<std::string>::const_iterator name_it = name_vec.begin();
+       name_it != name_vec.end(); ++name_it) {
+    const std::string& name = *name_it;
+    if (tests.count(name) != 0) {
+      errors << "Test " << name << " is listed more than once.\n";
+      continue;
+    }
+
+    bool found = false;
+    for (RegisteredTestIter it = registered_tests_.begin();
+         it != registered_tests_.end();
+         ++it) {
+      if (name == it->first) {
+        found = true;
+        break;
+      }
+    }
+
+    if (found) {
+      tests.insert(name);
+    } else {
+      errors << "No test named " << name
+             << " can be found in this test case.\n";
+    }
+  }
+
+  for (RegisteredTestIter it = registered_tests_.begin();
+       it != registered_tests_.end();
+       ++it) {
+    if (tests.count(it->first) == 0) {
+      errors << "You forgot to list test " << it->first << ".\n";
+    }
+  }
+
+  const std::string& errors_str = errors.GetString();
+  if (errors_str != "") {
+    fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(),
+            errors_str.c_str());
+    fflush(stderr);
+    posix::Abort();
+  }
+
+  return registered_tests;
+}
+
+#endif  // GTEST_HAS_TYPED_TEST_P
+
+}  // namespace internal
+}  // namespace testing
+// Copyright 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Author: wan@google.com (Zhanyong Wan)
+//
+// The Google C++ Testing Framework (Google Test)
+
+#include "gtest/gtest.h"
+#include "gtest/internal/custom/gtest.h"
+#include "gtest/gtest-spi.h"
+
+#include <ctype.h>
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <wchar.h>
+#include <wctype.h>
+
+#include <algorithm>
+#include <iomanip>
+#include <limits>
+#include <list>
+#include <map>
+#include <ostream>  // NOLINT
+#include <sstream>
+#include <vector>
+
+#if GTEST_OS_LINUX
+
+// TODO(kenton@google.com): Use autoconf to detect availability of
+// gettimeofday().
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+
+# include <fcntl.h>  // NOLINT
+# include <limits.h>  // NOLINT
+# include <sched.h>  // NOLINT
+// Declares vsnprintf().  This header is not available on Windows.
+# include <strings.h>  // NOLINT
+# include <sys/mman.h>  // NOLINT
+# include <sys/time.h>  // NOLINT
+# include <unistd.h>  // NOLINT
+# include <string>
+
+#elif GTEST_OS_SYMBIAN
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+# include <sys/time.h>  // NOLINT
+
+#elif GTEST_OS_ZOS
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+# include <sys/time.h>  // NOLINT
+
+// On z/OS we additionally need strings.h for strcasecmp.
+# include <strings.h>  // NOLINT
+
+#elif GTEST_OS_WINDOWS_MOBILE  // We are on Windows CE.
+
+# include <windows.h>  // NOLINT
+# undef min
+
+#elif GTEST_OS_WINDOWS  // We are on Windows proper.
+
+# include <io.h>  // NOLINT
+# include <sys/timeb.h>  // NOLINT
+# include <sys/types.h>  // NOLINT
+# include <sys/stat.h>  // NOLINT
+
+# if GTEST_OS_WINDOWS_MINGW
+// MinGW has gettimeofday() but not _ftime64().
+// TODO(kenton@google.com): Use autoconf to detect availability of
+//   gettimeofday().
+// TODO(kenton@google.com): There are other ways to get the time on
+//   Windows, like GetTickCount() or GetSystemTimeAsFileTime().  MinGW
+//   supports these.  consider using them instead.
+#  define GTEST_HAS_GETTIMEOFDAY_ 1
+#  include <sys/time.h>  // NOLINT
+# endif  // GTEST_OS_WINDOWS_MINGW
+
+// cpplint thinks that the header is already included, so we want to
+// silence it.
+# include <windows.h>  // NOLINT
+# undef min
+
+#else
+
+// Assume other platforms have gettimeofday().
+// TODO(kenton@google.com): Use autoconf to detect availability of
+//   gettimeofday().
+# define GTEST_HAS_GETTIMEOFDAY_ 1
+
+// cpplint thinks that the header is already included, so we want to
+// silence it.
+# include <sys/time.h>  // NOLINT
+# include <unistd.h>  // NOLINT
+
+#endif  // GTEST_OS_LINUX
+
+#if GTEST_HAS_EXCEPTIONS
+# include <stdexcept>
+#endif
+
+#if GTEST_CAN_STREAM_RESULTS_
+# include <arpa/inet.h>  // NOLINT
+# include <netdb.h>  // NOLINT
+# include <sys/socket.h>  // NOLINT
+# include <sys/types.h>  // NOLINT
+#endif
+
+// Indicates that this translation unit is part of Google Test's
+// implementation.  It must come before gtest-internal-inl.h is
+// included, or there will be a compiler error.  This trick is to
+// prevent a user from accidentally including gtest-internal-inl.h in
+// his code.
+#define GTEST_IMPLEMENTATION_ 1
+#include "src/gtest-internal-inl.h"
+#undef GTEST_IMPLEMENTATION_
+
+#if GTEST_OS_WINDOWS
+# define vsnprintf _vsnprintf
+#endif  // GTEST_OS_WINDOWS
+
+namespace testing {
+
+using internal::CountIf;
+using internal::ForEach;
+using internal::GetElementOr;
+using internal::Shuffle;
+
+// Constants.
+
+// A test whose test case name or test name matches this filter is
+// disabled and not run.
+static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*";
+
+// A test case whose name matches this filter is considered a death
+// test case and will be run before test cases whose name doesn't
+// match this filter.
+static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*";
+
+// A test filter that matches everything.
+static const char kUniversalFilter[] = "*";
+
+// The default output file for XML output.
+static const char kDefaultOutputFile[] = "test_detail.xml";
+
+// The environment variable name for the test shard index.
+static const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
+// The environment variable name for the total number of test shards.
+static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
+// The environment variable name for the test shard status file.
+static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE";
+
+namespace internal {
+
+// The text used in failure messages to indicate the start of the
+// stack trace.
+const char kStackTraceMarker[] = "\nStack trace:\n";
+
+// g_help_flag is true iff the --help flag or an equivalent form is
+// specified on the command line.
+bool g_help_flag = false;
+
+}  // namespace internal
+
+static const char* GetDefaultFilter() {
+#ifdef GTEST_TEST_FILTER_ENV_VAR_
+  const char* const testbridge_test_only = getenv(GTEST_TEST_FILTER_ENV_VAR_);
+  if (testbridge_test_only != NULL) {
+    return testbridge_test_only;
+  }
+#endif  // GTEST_TEST_FILTER_ENV_VAR_
+  return kUniversalFilter;
+}
+
+GTEST_DEFINE_bool_(
+    also_run_disabled_tests,
+    internal::BoolFromGTestEnv("also_run_disabled_tests", false),
+    "Run disabled tests too, in addition to the tests normally being run.");
+
+GTEST_DEFINE_bool_(
+    break_on_failure,
+    internal::BoolFromGTestEnv("break_on_failure", false),
+    "True iff a failed assertion should be a debugger break-point.");
+
+GTEST_DEFINE_bool_(
+    catch_exceptions,
+    internal::BoolFromGTestEnv("catch_exceptions", true),
+    "True iff " GTEST_NAME_
+    " should catch exceptions and treat them as test failures.");
+
+GTEST_DEFINE_string_(
+    color,
+    internal::StringFromGTestEnv("color", "auto"),
+    "Whether to use colors in the output.  Valid values: yes, no, "
+    "and auto.  'auto' means to use colors if the output is "
+    "being sent to a terminal and the TERM environment variable "
+    "is set to a terminal type that supports colors.");
+
+GTEST_DEFINE_string_(
+    filter,
+    internal::StringFromGTestEnv("filter", GetDefaultFilter()),
+    "A colon-separated list of glob (not regex) patterns "
+    "for filtering the tests to run, optionally followed by a "
+    "'-' and a : separated list of negative patterns (tests to "
+    "exclude).  A test is run if it matches one of the positive "
+    "patterns and does not match any of the negative patterns.");
+
+GTEST_DEFINE_bool_(list_tests, false,
+                   "List all tests without running them.");
+
+GTEST_DEFINE_string_(
+    output,
+    internal::StringFromGTestEnv("output", ""),
+    "A format (currently must be \"xml\"), optionally followed "
+    "by a colon and an output file name or directory. A directory "
+    "is indicated by a trailing pathname separator. "
+    "Examples: \"xml:filename.xml\", \"xml::directoryname/\". "
+    "If a directory is specified, output files will be created "
+    "within that directory, with file-names based on the test "
+    "executable's name and, if necessary, made unique by adding "
+    "digits.");
+
+GTEST_DEFINE_bool_(
+    print_time,
+    internal::BoolFromGTestEnv("print_time", true),
+    "True iff " GTEST_NAME_
+    " should display elapsed time in text output.");
+
+GTEST_DEFINE_int32_(
+    random_seed,
+    internal::Int32FromGTestEnv("random_seed", 0),
+    "Random number seed to use when shuffling test orders.  Must be in range "
+    "[1, 99999], or 0 to use a seed based on the current time.");
+
+GTEST_DEFINE_int32_(
+    repeat,
+    internal::Int32FromGTestEnv("repeat", 1),
+    "How many times to repeat each test.  Specify a negative number "
+    "for repeating forever.  Useful for shaking out flaky tests.");
+
+GTEST_DEFINE_bool_(
+    show_internal_stack_frames, false,
+    "True iff " GTEST_NAME_ " should include internal stack frames when "
+    "printing test failure stack traces.");
+
+GTEST_DEFINE_bool_(
+    shuffle,
+    internal::BoolFromGTestEnv("shuffle", false),
+    "True iff " GTEST_NAME_
+    " should randomize tests' order on every run.");
+
+GTEST_DEFINE_int32_(
+    stack_trace_depth,
+    internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth),
+    "The maximum number of stack frames to print when an "
+    "assertion fails.  The valid range is 0 through 100, inclusive.");
+
+GTEST_DEFINE_string_(
+    stream_result_to,
+    internal::StringFromGTestEnv("stream_result_to", ""),
+    "This flag specifies the host name and the port number on which to stream "
+    "test results. Example: \"localhost:555\". The flag is effective only on "
+    "Linux.");
+
+GTEST_DEFINE_bool_(
+    throw_on_failure,
+    internal::BoolFromGTestEnv("throw_on_failure", false),
+    "When this flag is specified, a failed assertion will throw an exception "
+    "if exceptions are enabled or exit the program with a non-zero code "
+    "otherwise.");
+
+#if GTEST_USE_OWN_FLAGFILE_FLAG_
+GTEST_DEFINE_string_(
+    flagfile,
+    internal::StringFromGTestEnv("flagfile", ""),
+    "This flag specifies the flagfile to read command-line flags from.");
+#endif  // GTEST_USE_OWN_FLAGFILE_FLAG_
+
+namespace internal {
+
+// Generates a random number from [0, range), using a Linear
+// Congruential Generator (LCG).  Crashes if 'range' is 0 or greater
+// than kMaxRange.
+UInt32 Random::Generate(UInt32 range) {
+  // These constants are the same as are used in glibc's rand(3).
+  state_ = (1103515245U*state_ + 12345U) % kMaxRange;
+
+  GTEST_CHECK_(range > 0)
+      << "Cannot generate a number in the range [0, 0).";
+  GTEST_CHECK_(range <= kMaxRange)
+      << "Generation of a number in [0, " << range << ") was requested, "
+      << "but this can only generate numbers in [0, " << kMaxRange << ").";
+
+  // Converting via modulus introduces a bit of downward bias, but
+  // it's simple, and a linear congruential generator isn't too good
+  // to begin with.
+  return state_ % range;
+}
+
+// GTestIsInitialized() returns true iff the user has initialized
+// Google Test.  Useful for catching the user mistake of not initializing
+// Google Test before calling RUN_ALL_TESTS().
+static bool GTestIsInitialized() { return GetArgvs().size() > 0; }
+
+// Iterates over a vector of TestCases, keeping a running sum of the
+// results of calling a given int-returning method on each.
+// Returns the sum.
+static int SumOverTestCaseList(const std::vector<TestCase*>& case_list,
+                               int (TestCase::*method)() const) {
+  int sum = 0;
+  for (size_t i = 0; i < case_list.size(); i++) {
+    sum += (case_list[i]->*method)();
+  }
+  return sum;
+}
+
+// Returns true iff the test case passed.
+static bool TestCasePassed(const TestCase* test_case) {
+  return test_case->should_run() && test_case->Passed();
+}
+
+// Returns true iff the test case failed.
+static bool TestCaseFailed(const TestCase* test_case) {
+  return test_case->should_run() && test_case->Failed();
+}
+
+// Returns true iff test_case contains at least one test that should
+// run.
+static bool ShouldRunTestCase(const TestCase* test_case) {
+  return test_case->should_run();
+}
+
+// AssertHelper constructor.
+AssertHelper::AssertHelper(TestPartResult::Type type,
+                           const char* file,
+                           int line,
+                           const char* message)
+    : data_(new AssertHelperData(type, file, line, message)) {
+}
+
+AssertHelper::~AssertHelper() {
+  delete data_;
+}
+
+// Message assignment, for assertion streaming support.
+void AssertHelper::operator=(const Message& message) const {
+  UnitTest::GetInstance()->
+    AddTestPartResult(data_->type, data_->file, data_->line,
+                      AppendUserMessage(data_->message, message),
+                      UnitTest::GetInstance()->impl()
+                      ->CurrentOsStackTraceExceptTop(1)
+                      // Skips the stack frame for this function itself.
+                      );  // NOLINT
+}
+
+// Mutex for linked pointers.
+GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex);
+
+// A copy of all command line arguments.  Set by InitGoogleTest().
+::std::vector<testing::internal::string> g_argvs;
+
+const ::std::vector<testing::internal::string>& GetArgvs() {
+#if defined(GTEST_CUSTOM_GET_ARGVS_)
+  return GTEST_CUSTOM_GET_ARGVS_();
+#else  // defined(GTEST_CUSTOM_GET_ARGVS_)
+  return g_argvs;
+#endif  // defined(GTEST_CUSTOM_GET_ARGVS_)
+}
+
+// Returns the current application's name, removing directory path if that
+// is present.
+FilePath GetCurrentExecutableName() {
+  FilePath result;
+
+#if GTEST_OS_WINDOWS
+  result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe"));
+#else
+  result.Set(FilePath(GetArgvs()[0]));
+#endif  // GTEST_OS_WINDOWS
+
+  return result.RemoveDirectoryName();
+}
+
+// Functions for processing the gtest_output flag.
+
+// Returns the output format, or "" for normal printed output.
+std::string UnitTestOptions::GetOutputFormat() {
+  const char* const gtest_output_flag = GTEST_FLAG(output).c_str();
+  if (gtest_output_flag == NULL) return std::string("");
+
+  const char* const colon = strchr(gtest_output_flag, ':');
+  return (colon == NULL) ?
+      std::string(gtest_output_flag) :
+      std::string(gtest_output_flag, colon - gtest_output_flag);
+}
+
+// Returns the name of the requested output file, or the default if none
+// was explicitly specified.
+std::string UnitTestOptions::GetAbsolutePathToOutputFile() {
+  const char* const gtest_output_flag = GTEST_FLAG(output).c_str();
+  if (gtest_output_flag == NULL)
+    return "";
+
+  const char* const colon = strchr(gtest_output_flag, ':');
+  if (colon == NULL)
+    return internal::FilePath::ConcatPaths(
+        internal::FilePath(
+            UnitTest::GetInstance()->original_working_dir()),
+        internal::FilePath(kDefaultOutputFile)).string();
+
+  internal::FilePath output_name(colon + 1);
+  if (!output_name.IsAbsolutePath())
+    // TODO(wan@google.com): on Windows \some\path is not an absolute
+    // path (as its meaning depends on the current drive), yet the
+    // following logic for turning it into an absolute path is wrong.
+    // Fix it.
+    output_name = internal::FilePath::ConcatPaths(
+        internal::FilePath(UnitTest::GetInstance()->original_working_dir()),
+        internal::FilePath(colon + 1));
+
+  if (!output_name.IsDirectory())
+    return output_name.string();
+
+  internal::FilePath result(internal::FilePath::GenerateUniqueFileName(
+      output_name, internal::GetCurrentExecutableName(),
+      GetOutputFormat().c_str()));
+  return result.string();
+}
+
+// Returns true iff the wildcard pattern matches the string.  The
+// first ':' or '\0' character in pattern marks the end of it.
+//
+// This recursive algorithm isn't very efficient, but is clear and
+// works well enough for matching test names, which are short.
+bool UnitTestOptions::PatternMatchesString(const char *pattern,
+                                           const char *str) {
+  switch (*pattern) {
+    case '\0':
+    case ':':  // Either ':' or '\0' marks the end of the pattern.
+      return *str == '\0';
+    case '?':  // Matches any single character.
+      return *str != '\0' && PatternMatchesString(pattern + 1, str + 1);
+    case '*':  // Matches any string (possibly empty) of characters.
+      return (*str != '\0' && PatternMatchesString(pattern, str + 1)) ||
+          PatternMatchesString(pattern + 1, str);
+    default:  // Non-special character.  Matches itself.
+      return *pattern == *str &&
+          PatternMatchesString(pattern + 1, str + 1);
+  }
+}
+
+bool UnitTestOptions::MatchesFilter(
+    const std::string& name, const char* filter) {
+  const char *cur_pattern = filter;
+  for (;;) {
+    if (PatternMatchesString(cur_pattern, name.c_str())) {
+      return true;
+    }
+
+    // Finds the next pattern in the filter.
+    cur_pattern = strchr(cur_pattern, ':');
+
+    // Returns if no more pattern can be found.
+    if (cur_pattern == NULL) {
+      return false;
+    }
+
+    // Skips the pattern separater (the ':' character).
+    cur_pattern++;
+  }
+}
+
+// Returns true iff the user-specified filter matches the test case
+// name and the test name.
+bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name,
+                                        const std::string &test_name) {
+  const std::string& full_name = test_case_name + "." + test_name.c_str();
+
+  // Split --gtest_filter at '-', if there is one, to separate into
+  // positive filter and negative filter portions
+  const char* const p = GTEST_FLAG(filter).c_str();
+  const char* const dash = strchr(p, '-');
+  std::string positive;
+  std::string negative;
+  if (dash == NULL) {
+    positive = GTEST_FLAG(filter).c_str();  // Whole string is a positive filter
+    negative = "";
+  } else {
+    positive = std::string(p, dash);   // Everything up to the dash
+    negative = std::string(dash + 1);  // Everything after the dash
+    if (positive.empty()) {
+      // Treat '-test1' as the same as '*-test1'
+      positive = kUniversalFilter;
+    }
+  }
+
+  // A filter is a colon-separated list of patterns.  It matches a
+  // test if any pattern in it matches the test.
+  return (MatchesFilter(full_name, positive.c_str()) &&
+          !MatchesFilter(full_name, negative.c_str()));
+}
+
+#if GTEST_HAS_SEH
+// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the
+// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise.
+// This function is useful as an __except condition.
+int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) {
+  // Google Test should handle a SEH exception if:
+  //   1. the user wants it to, AND
+  //   2. this is not a breakpoint exception, AND
+  //   3. this is not a C++ exception (VC++ implements them via SEH,
+  //      apparently).
+  //
+  // SEH exception code for C++ exceptions.
+  // (see http://support.microsoft.com/kb/185294 for more information).
+  const DWORD kCxxExceptionCode = 0xe06d7363;
+
+  bool should_handle = true;
+
+  if (!GTEST_FLAG(catch_exceptions))
+    should_handle = false;
+  else if (exception_code == EXCEPTION_BREAKPOINT)
+    should_handle = false;
+  else if (exception_code == kCxxExceptionCode)
+    should_handle = false;
+
+  return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH;
+}
+#endif  // GTEST_HAS_SEH
+
+}  // namespace internal
+
+// The c'tor sets this object as the test part result reporter used by
+// Google Test.  The 'result' parameter specifies where to report the
+// results. Intercepts only failures from the current thread.
+ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter(
+    TestPartResultArray* result)
+    : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD),
+      result_(result) {
+  Init();
+}
+
+// The c'tor sets this object as the test part result reporter used by
+// Google Test.  The 'result' parameter specifies where to report the
+// results.
+ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter(
+    InterceptMode intercept_mode, TestPartResultArray* result)
+    : intercept_mode_(intercept_mode),
+      result_(result) {
+  Init();
+}
+
+void ScopedFakeTestPartResultReporter::Init() {
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  if (intercept_mode_ == INTERCEPT_ALL_THREADS) {
+    old_reporter_ = impl->GetGlobalTestPartResultReporter();
+    impl->SetGlobalTestPartResultReporter(this);
+  } else {
+    old_reporter_ = impl->GetTestPartResultReporterForCurrentThread();
+    impl->SetTestPartResultReporterForCurrentThread(this);
+  }
+}
+
+// The d'tor restores the test part result reporter used by Google Test
+// before.
+ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() {
+  internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
+  if (intercept_mode_ == INTERCEPT_ALL_THREADS) {
+    impl->SetGlobalTestPartResultReporter(old_reporter_);
+  } else {
+    impl->SetTestPartResultReporterForCurrentThread(old_reporter_);
+  }
+}
+
+// Increments the test part result count and remembers the result.
+// This method is from the TestPartResultReporterInterface interface.
+void ScopedFakeTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  result_->Append(result);
+}
+
+namespace internal {
+
+// Returns the type ID of ::testing::Test.  We should always call this
+// instead of GetTypeId< ::testing::Test>() to get the type ID of
+// testing::Test.  This is to work around a suspected linker bug when
+// using Google Test as a framework on Mac OS X.  The bug causes
+// GetTypeId< ::testing::Test>() to return different values depending
+// on whether the call is from the Google Test framework itself or
+// from user test code.  GetTestTypeId() is guaranteed to always
+// return the same value, as it always calls GetTypeId<>() from the
+// gtest.cc, which is within the Google Test framework.
+TypeId GetTestTypeId() {
+  return GetTypeId<Test>();
+}
+
+// The value of GetTestTypeId() as seen from within the Google Test
+// library.  This is solely for testing GetTestTypeId().
+extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId();
+
+// This predicate-formatter checks that 'results' contains a test part
+// failure of the given type and that the failure message contains the
+// given substring.
+AssertionResult HasOneFailure(const char* /* results_expr */,
+                              const char* /* type_expr */,
+                              const char* /* substr_expr */,
+                              const TestPartResultArray& results,
+                              TestPartResult::Type type,
+                              const string& substr) {
+  const std::string expected(type == TestPartResult::kFatalFailure ?
+                        "1 fatal failure" :
+                        "1 non-fatal failure");
+  Message msg;
+  if (results.size() != 1) {
+    msg << "Expected: " << expected << "\n"
+        << "  Actual: " << results.size() << " failures";
+    for (int i = 0; i < results.size(); i++) {
+      msg << "\n" << results.GetTestPartResult(i);
+    }
+    return AssertionFailure() << msg;
+  }
+
+  const TestPartResult& r = results.GetTestPartResult(0);
+  if (r.type() != type) {
+    return AssertionFailure() << "Expected: " << expected << "\n"
+                              << "  Actual:\n"
+                              << r;
+  }
+
+  if (strstr(r.message(), substr.c_str()) == NULL) {
+    return AssertionFailure() << "Expected: " << expected << " containing \""
+                              << substr << "\"\n"
+                              << "  Actual:\n"
+                              << r;
+  }
+
+  return AssertionSuccess();
+}
+
+// The constructor of SingleFailureChecker remembers where to look up
+// test part results, what type of failure we expect, and what
+// substring the failure message should contain.
+SingleFailureChecker:: SingleFailureChecker(
+    const TestPartResultArray* results,
+    TestPartResult::Type type,
+    const string& substr)
+    : results_(results),
+      type_(type),
+      substr_(substr) {}
+
+// The destructor of SingleFailureChecker verifies that the given
+// TestPartResultArray contains exactly one failure that has the given
+// type and contains the given substring.  If that's not the case, a
+// non-fatal failure will be generated.
+SingleFailureChecker::~SingleFailureChecker() {
+  EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_);
+}
+
+DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter(
+    UnitTestImpl* unit_test) : unit_test_(unit_test) {}
+
+void DefaultGlobalTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  unit_test_->current_test_result()->AddTestPartResult(result);
+  unit_test_->listeners()->repeater()->OnTestPartResult(result);
+}
+
+DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter(
+    UnitTestImpl* unit_test) : unit_test_(unit_test) {}
+
+void DefaultPerThreadTestPartResultReporter::ReportTestPartResult(
+    const TestPartResult& result) {
+  unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result);
+}
+
+// Returns the global test part result reporter.
+TestPartResultReporterInterface*
+UnitTestImpl::GetGlobalTestPartResultReporter() {
+  internal::MutexLock lock(&global_test_part_result_reporter_mutex_);
+  return global_test_part_result_repoter_;
+}
+
+// Sets the global test part result reporter.
+void UnitTestImpl::SetGlobalTestPartResultReporter(
+    TestPartResultReporterInterface* reporter) {
+  internal::MutexLock lock(&global_test_part_result_reporter_mutex_);
+  global_test_part_result_repoter_ = reporter;
+}
+
+// Returns the test part result reporter for the current thread.
+TestPartResultReporterInterface*
+UnitTestImpl::GetTestPartResultReporterForCurrentThread() {
+  return per_thread_test_part_result_reporter_.get();
+}
+
+// Sets the test part result reporter for the current thread.
+void UnitTestImpl::SetTestPartResultReporterForCurrentThread(
+    TestPartResultReporterInterface* reporter) {
+  per_thread_test_part_result_reporter_.set(reporter);
+}
+
+// Gets the number of successful test cases.
+int UnitTestImpl::successful_test_case_count() const {
+  return CountIf(test_cases_, TestCasePassed);
+}
+
+// Gets the number of failed test cases.
+int UnitTestImpl::failed_test_case_count() const {
+  return CountIf(test_cases_, TestCaseFailed);
+}
+
+// Gets the number of all test cases.
+int UnitTestImpl::total_test_case_count() const {
+  return static_cast<int>(test_cases_.size());
+}
+
+// Gets the number of all test cases that contain at least one test
+// that should run.
+int UnitTestImpl::test_case_to_run_count() const {
+  return CountIf(test_cases_, ShouldRunTestCase);
+}
+
+// Gets the number of successful tests.
+int UnitTestImpl::successful_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count);
+}
+
+// Gets the number of failed tests.
+int UnitTestImpl::failed_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count);
+}
+
+// Gets the number of disabled tests that will be reported in the XML report.
+int UnitTestImpl::reportable_disabled_test_count() const {
+  return SumOverTestCaseList(test_cases_,
+                             &TestCase::reportable_disabled_test_count);
+}
+
+// Gets the number of disabled tests.
+int UnitTestImpl::disabled_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count);
+}
+
+// Gets the number of tests to be printed in the XML report.
+int UnitTestImpl::reportable_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count);
+}
+
+// Gets the number of all tests.
+int UnitTestImpl::total_test_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::total_test_count);
+}
+
+// Gets the number of tests that should run.
+int UnitTestImpl::test_to_run_count() const {
+  return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count);
+}
+
+// Returns the current OS stack trace as an std::string.
+//
+// The maximum number of stack frames to be included is specified by
+// the gtest_stack_trace_depth flag.  The skip_count parameter
+// specifies the number of top frames to be skipped, which doesn't
+// count against the number of frames to be included.
+//
+// For example, if Foo() calls Bar(), which in turn calls
+// CurrentOsStackTraceExceptTop(1), Foo() will be included in the
+// trace but Bar() and CurrentOsStackTraceExceptTop() won't.
+std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) {
+  return os_stack_trace_getter()->CurrentStackTrace(
+      static_cast<int>(GTEST_FLAG(stack_trace_depth)),
+      skip_count + 1
+      // Skips the user-specified number of frames plus this function
+      // itself.
+      );  // NOLINT
+}
+
+// Returns the current time in milliseconds.
+TimeInMillis GetTimeInMillis() {
+#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__)
+  // Difference between 1970-01-01 and 1601-01-01 in milliseconds.
+  // http://analogous.blogspot.com/2005/04/epoch.html
+  const TimeInMillis kJavaEpochToWinFileTimeDelta =
+    static_cast<TimeInMillis>(116444736UL) * 100000UL;
+  const DWORD kTenthMicrosInMilliSecond = 10000;
+
+  SYSTEMTIME now_systime;
+  FILETIME now_filetime;
+  ULARGE_INTEGER now_int64;
+  // TODO(kenton@google.com): Shouldn't this just use
+  //   GetSystemTimeAsFileTime()?
+  GetSystemTime(&now_systime);
+  if (SystemTimeToFileTime(&now_systime, &now_filetime)) {
+    now_int64.LowPart = now_filetime.dwLowDateTime;
+    now_int64.HighPart = now_filetime.dwHighDateTime;
+    now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) -
+      kJavaEpochToWinFileTimeDelta;
+    return now_int64.QuadPart;
+  }
+  return 0;
+#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_
+  __timeb64 now;
+
+  // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996
+  // (deprecated function) there.
+  // TODO(kenton@google.com): Use GetTickCount()?  Or use
+  //   SystemTimeToFileTime()
+  GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996)
+  _ftime64(&now);
+  GTEST_DISABLE_MSC_WARNINGS_POP_()
+
+  return static_cast<TimeInMillis>(now.time) * 1000 + now.millitm;
+#elif GTEST_HAS_GETTIMEOFDAY_
+  struct timeval now;
+  gettimeofday(&now, NULL);
+  return static_cast<TimeInMillis>(now.tv_sec) * 1000 + now.tv_usec / 1000;
+#else
+# error "Don't know how to get the current time on your system."
+#endif
+}
+
+// Utilities
+
+// class String.
+
+#if GTEST_OS_WINDOWS_MOBILE
+// Creates a UTF-16 wide string from the given ANSI string, allocating
+// memory using new. The caller is responsible for deleting the return
+// value using delete[]. Returns the wide string, or NULL if the
+// input is NULL.
+LPCWSTR String::AnsiToUtf16(const char* ansi) {
+  if (!ansi) return NULL;
+  const int length = strlen(ansi);
+  const int unicode_length =
+      MultiByteToWideChar(CP_ACP, 0, ansi, length,
+                          NULL, 0);
+  WCHAR* unicode = new WCHAR[unicode_length + 1];
+  MultiByteToWideChar(CP_ACP, 0, ansi, length,
+                      unicode, unicode_length);
+  unicode[unicode_length] = 0;
+  return unicode;
+}
+
+// Creates an ANSI string from the given wide string, allocating
+// memory using new. The caller is responsible for deleting the return
+// value using delete[]. Returns the ANSI string, or NULL if the
+// input is NULL.
+const char* String::Utf16ToAnsi(LPCWSTR utf16_str)  {
+  if (!utf16_str) return NULL;
+  const int ansi_length =
+      WideCharToMultiByte(CP_ACP, 0, utf16_str, -1,
+                          NULL, 0, NULL, NULL);
+  char* ansi = new char[ansi_length + 1];
+  WideCharToMultiByte(CP_ACP, 0, utf16_str, -1,
+                      ansi, ansi_length, NULL, NULL);
+  ansi[ansi_length] = 0;
+  return ansi;
+}
+
+#endif  // GTEST_OS_WINDOWS_MOBILE
+
+// Compares two C strings.  Returns true iff they have the same content.
+//
+// Unlike strcmp(), this function can handle NULL argument(s).  A NULL
+// C string is considered different to any non-NULL C string,
+// including the empty string.
+bool String::CStringEquals(const char * lhs, const char * rhs) {
+  if ( lhs == NULL ) return rhs == NULL;
+
+  if ( rhs == NULL ) return false;
+
+  return strcmp(lhs, rhs) == 0;
+}
+
+#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING
+
+// Converts an array of wide chars to a narrow string using the UTF-8
+// encoding, and streams the result to the given Message object.
+static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length,
+                                     Message* msg) {
+  for (size_t i = 0; i != length; ) {  // NOLINT
+    if (wstr[i] != L'\0') {
+      *msg << WideStringToUtf8(wstr + i, static_cast<int>(length - i));
+      while (i != length && wstr[i] != L'\0')
+        i++;
+    } else {
+      *msg << '\0';
+      i++;
+    }
+  }
+}
+
+#endif  // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING
+
+void SplitString(const ::std::string& str, char delimiter,
+                 ::std::vector< ::std::string>* dest) {
+  ::std::vector< ::std::string> parsed;
+  ::std::string::size_type pos = 0;
+  while (::testing::internal::AlwaysTrue()) {
+    const ::std::string::size_type colon = str.find(delimiter, pos);
+    if (colon == ::std::string::npos) {
+      parsed.push_back(str.substr(pos));
+      break;
+    } else {
+      parsed.push_back(str.substr(pos, colon - pos));
+      pos = colon + 1;
+    }
+  }
+  dest->swap(parsed);
+}
+
+}  // namespace internal
+
+// Constructs an empty Message.
+// We allocate the stringstream separately because otherwise each use of
+// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's
+// stack frame leading to huge stack frames in some cases; gcc does not reuse
+// the stack space.
+Message::Message() : ss_(new ::std::stringstream) {
+  // By default, we want there to be enough precision when printing
+  // a double to a Message.
+  *ss_ << std::setprecision(std::numeric_limits<double>::digits10 + 2);
+}
+
+// These two overloads allow streaming a wide C string to a Message
+// using the UTF-8 encoding.
+Message& Message::operator <<(const wchar_t* wide_c_str) {
+  return *this << internal::String::ShowWideCString(wide_c_str);
+}
+Message& Message::operator <<(wchar_t* wide_c_str) {
+  return *this << internal::String::ShowWideCString(wide_c_str);
+}
+
+#if GTEST_HAS_STD_WSTRING
+// Converts the given wide string to a narrow string using the UTF-8
+// encoding, and streams the result to this Message object.
+Message& Message::operator <<(const ::std::wstring& wstr) {
+  internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this);
+  return *this;
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+#if GTEST_HAS_GLOBAL_WSTRING
+// Converts the given wide string to a narrow string using the UTF-8
+// encoding, and streams the result to this Message object.
+Message& Message::operator <<(const ::wstring& wstr) {
+  internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this);
+  return *this;
+}
+#endif  // GTEST_HAS_GLOBAL_WSTRING
+
+// Gets the text streamed to this object so far as an std::string.
+// Each '\0' character in the buffer is replaced with "\\0".
+std::string Message::GetString() const {
+  return internal::StringStreamToString(ss_.get());
+}
+
+// AssertionResult constructors.
+// Used in EXPECT_TRUE/FALSE(assertion_result).
+AssertionResult::AssertionResult(const AssertionResult& other)
+    : success_(other.success_),
+      message_(other.message_.get() != NULL ?
+               new ::std::string(*other.message_) :
+               static_cast< ::std::string*>(NULL)) {
+}
+
+// Swaps two AssertionResults.
+void AssertionResult::swap(AssertionResult& other) {
+  using std::swap;
+  swap(success_, other.success_);
+  swap(message_, other.message_);
+}
+
+// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE.
+AssertionResult AssertionResult::operator!() const {
+  AssertionResult negation(!success_);
+  if (message_.get() != NULL)
+    negation << *message_;
+  return negation;
+}
+
+// Makes a successful assertion result.
+AssertionResult AssertionSuccess() {
+  return AssertionResult(true);
+}
+
+// Makes a failed assertion result.
+AssertionResult AssertionFailure() {
+  return AssertionResult(false);
+}
+
+// Makes a failed assertion result with the given failure message.
+// Deprecated; use AssertionFailure() << message.
+AssertionResult AssertionFailure(const Message& message) {
+  return AssertionFailure() << message;
+}
+
+namespace internal {
+
+namespace edit_distance {
+std::vector<EditType> CalculateOptimalEdits(const std::vector<size_t>& left,
+                                            const std::vector<size_t>& right) {
+  std::vector<std::vector<double> > costs(
+      left.size() + 1, std::vector<double>(right.size() + 1));
+  std::vector<std::vector<EditType> > best_move(
+      left.size() + 1, std::vector<EditType>(right.size() + 1));
+
+  // Populate for empty right.
+  for (size_t l_i = 0; l_i < costs.size(); ++l_i) {
+    costs[l_i][0] = static_cast<double>(l_i);
+    best_move[l_i][0] = kRemove;
+  }
+  // Populate for empty left.
+  for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) {
+    costs[0][r_i] = static_cast<double>(r_i);
+    best_move[0][r_i] = kAdd;
+  }
+
+  for (size_t l_i = 0; l_i < left.size(); ++l_i) {
+    for (size_t r_i = 0; r_i < right.size(); ++r_i) {
+      if (left[l_i] == right[r_i]) {
+        // Found a match. Consume it.
+        costs[l_i + 1][r_i + 1] = costs[l_i][r_i];
+        best_move[l_i + 1][r_i + 1] = kMatch;
+        continue;
+      }
+
+      const double add = costs[l_i + 1][r_i];
+      const double remove = costs[l_i][r_i + 1];
+      const double replace = costs[l_i][r_i];
+      if (add < remove && add < replace) {
+        costs[l_i + 1][r_i + 1] = add + 1;
+        best_move[l_i + 1][r_i + 1] = kAdd;
+      } else if (remove < add && remove < replace) {
+        costs[l_i + 1][r_i + 1] = remove + 1;
+        best_move[l_i + 1][r_i + 1] = kRemove;
+      } else {
+        // We make replace a little more expensive than add/remove to lower
+        // their priority.
+        costs[l_i + 1][r_i + 1] = replace + 1.00001;
+        best_move[l_i + 1][r_i + 1] = kReplace;
+      }
+    }
+  }
+
+  // Reconstruct the best path. We do it in reverse order.
+  std::vector<EditType> best_path;
+  for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) {
+    EditType move = best_move[l_i][r_i];
+    best_path.push_back(move);
+    l_i -= move != kAdd;
+    r_i -= move != kRemove;
+  }
+  std::reverse(best_path.begin(), best_path.end());
+  return best_path;
+}
+
+namespace {
+
+// Helper class to convert string into ids with deduplication.
+class InternalStrings {
+ public:
+  size_t GetId(const std::string& str) {
+    IdMap::iterator it = ids_.find(str);
+    if (it != ids_.end()) return it->second;
+    size_t id = ids_.size();
+    return ids_[str] = id;
+  }
+
+ private:
+  typedef std::map<std::string, size_t> IdMap;
+  IdMap ids_;
+};
+
+}  // namespace
+
+std::vector<EditType> CalculateOptimalEdits(
+    const std::vector<std::string>& left,
+    const std::vector<std::string>& right) {
+  std::vector<size_t> left_ids, right_ids;
+  {
+    InternalStrings intern_table;
+    for (size_t i = 0; i < left.size(); ++i) {
+      left_ids.push_back(intern_table.GetId(left[i]));
+    }
+    for (size_t i = 0; i < right.size(); ++i) {
+      right_ids.push_back(intern_table.GetId(right[i]));
+    }
+  }
+  return CalculateOptimalEdits(left_ids, right_ids);
+}
+
+namespace {
+
+// Helper class that holds the state for one hunk and prints it out to the
+// stream.
+// It reorders adds/removes when possible to group all removes before all
+// adds. It also adds the hunk header before printint into the stream.
+class Hunk {
+ public:
+  Hunk(size_t left_start, size_t right_start)
+      : left_start_(left_start),
+        right_start_(right_start),
+        adds_(),
+        removes_(),
+        common_() {}
+
+  void PushLine(char edit, const char* line) {
+    switch (edit) {
+      case ' ':
+        ++common_;
+        FlushEdits();
+        hunk_.push_back(std::make_pair(' ', line));
+        break;
+      case '-':
+        ++removes_;
+        hunk_removes_.push_back(std::make_pair('-', line));
+        break;
+      case '+':
+        ++adds_;
+        hunk_adds_.push_back(std::make_pair('+', line));
+        break;
+    }
+  }
+
+  void PrintTo(std::ostream* os) {
+    PrintHeader(os);
+    FlushEdits();
+    for (std::list<std::pair<char, const char*> >::const_iterator it =
+             hunk_.begin();
+         it != hunk_.end(); ++it) {
+      *os << it->first << it->second << "\n";
+    }
+  }
+
+  bool has_edits() const { return adds_ || removes_; }
+
+ private:
+  void FlushEdits() {
+    hunk_.splice(hunk_.end(), hunk_removes_);
+    hunk_.splice(hunk_.end(), hunk_adds_);
+  }
+
+  // Print a unified diff header for one hunk.
+  // The format is
+  //   "@@ -<left_start>,<left_length> +<right_start>,<right_length> @@"
+  // where the left/right parts are ommitted if unnecessary.
+  void PrintHeader(std::ostream* ss) const {
+    *ss << "@@ ";
+    if (removes_) {
+      *ss << "-" << left_start_ << "," << (removes_ + common_);
+    }
+    if (removes_ && adds_) {
+      *ss << " ";
+    }
+    if (adds_) {
+      *ss << "+" << right_start_ << "," << (adds_ + common_);
+    }
+    *ss << " @@\n";
+  }
+
+  size_t left_start_, right_start_;
+  size_t adds_, removes_, common_;
+  std::list<std::pair<char, const char*> > hunk_, hunk_adds_, hunk_removes_;
+};
+
+}  // namespace
+
+// Create a list of diff hunks in Unified diff format.
+// Each hunk has a header generated by PrintHeader above plus a body with
+// lines prefixed with ' ' for no change, '-' for deletion and '+' for
+// addition.
+// 'context' represents the desired unchanged prefix/suffix around the diff.
+// If two hunks are close enough that their contexts overlap, then they are
+// joined into one hunk.
+std::string CreateUnifiedDiff(const std::vector<std::string>& left,
+                              const std::vector<std::string>& right,
+                              size_t context) {
+  const std::vector<EditType> edits = CalculateOptimalEdits(left, right);
+
+  size_t l_i = 0, r_i = 0, edit_i = 0;
+  std::stringstream ss;
+  while (edit_i < edits.size()) {
+    // Find first edit.
+    while (edit_i < edits.size() && edits[edit_i] == kMatch) {
+      ++l_i;
+      ++r_i;
+      ++edit_i;
+    }
+
+    // Find the first line to include in the hunk.
+    const size_t prefix_context = std::min(l_i, context);
+    Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1);
+    for (size_t i = prefix_context; i > 0; --i) {
+      hunk.PushLine(' ', left[l_i - i].c_str());
+    }
+
+    // Iterate the edits until we found enough suffix for the hunk or the input
+    // is over.
+    size_t n_suffix = 0;
+    for (; edit_i < edits.size(); ++edit_i) {
+      if (n_suffix >= context) {
+        // Continue only if the next hunk is very close.
+        std::vector<EditType>::const_iterator it = edits.begin() + edit_i;
+        while (it != edits.end() && *it == kMatch) ++it;
+        if (it == edits.end() || (it - edits.begin()) - edit_i >= context) {
+          // There is no next edit or it is too far away.
+          break;
+        }
+      }
+
+      EditType edit = edits[edit_i];
+      // Reset count when a non match is found.
+      n_suffix = edit == kMatch ? n_suffix + 1 : 0;
+
+      if (edit == kMatch || edit == kRemove || edit == kReplace) {
+        hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str());
+      }
+      if (edit == kAdd || edit == kReplace) {
+        hunk.PushLine('+', right[r_i].c_str());
+      }
+
+      // Advance indices, depending on edit type.
+      l_i += edit != kAdd;
+      r_i += edit != kRemove;
+    }
+
+    if (!hunk.has_edits()) {
+      // We are done. We don't want this hunk.
+      break;
+    }
+
+    hunk.PrintTo(&ss);
+  }
+  return ss.str();
+}
+
+}  // namespace edit_distance
+
+namespace {
+
+// The string representation of the values received in EqFailure() are already
+// escaped. Split them on escaped '\n' boundaries. Leave all other escaped
+// characters the same.
+std::vector<std::string> SplitEscapedString(const std::string& str) {
+  std::vector<std::string> lines;
+  size_t start = 0, end = str.size();
+  if (end > 2 && str[0] == '"' && str[end - 1] == '"') {
+    ++start;
+    --end;
+  }
+  bool escaped = false;
+  for (size_t i = start; i + 1 < end; ++i) {
+    if (escaped) {
+      escaped = false;
+      if (str[i] == 'n') {
+        lines.push_back(str.substr(start, i - start - 1));
+        start = i + 1;
+      }
+    } else {
+      escaped = str[i] == '\\';
+    }
+  }
+  lines.push_back(str.substr(start, end - start));
+  return lines;
+}
+
+}  // namespace
+
+// Constructs and returns the message for an equality assertion
+// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure.
+//
+// The first four parameters are the expressions used in the assertion
+// and their values, as strings.  For example, for ASSERT_EQ(foo, bar)
+// where foo is 5 and bar is 6, we have:
+//
+//   lhs_expression: "foo"
+//   rhs_expression: "bar"
+//   lhs_value:      "5"
+//   rhs_value:      "6"
+//
+// The ignoring_case parameter is true iff the assertion is a
+// *_STRCASEEQ*.  When it's true, the string "Ignoring case" will
+// be inserted into the message.
+AssertionResult EqFailure(const char* lhs_expression,
+                          const char* rhs_expression,
+                          const std::string& lhs_value,
+                          const std::string& rhs_value,
+                          bool ignoring_case) {
+  Message msg;
+  msg << "      Expected: " << lhs_expression;
+  if (lhs_value != lhs_expression) {
+    msg << "\n      Which is: " << lhs_value;
+  }
+  msg << "\nTo be equal to: " << rhs_expression;
+  if (rhs_value != rhs_expression) {
+    msg << "\n      Which is: " << rhs_value;
+  }
+
+  if (ignoring_case) {
+    msg << "\nIgnoring case";
+  }
+
+  if (!lhs_value.empty() && !rhs_value.empty()) {
+    const std::vector<std::string> lhs_lines =
+        SplitEscapedString(lhs_value);
+    const std::vector<std::string> rhs_lines =
+        SplitEscapedString(rhs_value);
+    if (lhs_lines.size() > 1 || rhs_lines.size() > 1) {
+      msg << "\nWith diff:\n"
+          << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines);
+    }
+  }
+
+  return AssertionFailure() << msg;
+}
+
+// Constructs a failure message for Boolean assertions such as EXPECT_TRUE.
+std::string GetBoolAssertionFailureMessage(
+    const AssertionResult& assertion_result,
+    const char* expression_text,
+    const char* actual_predicate_value,
+    const char* expected_predicate_value) {
+  const char* actual_message = assertion_result.message();
+  Message msg;
+  msg << "Value of: " << expression_text
+      << "\n  Actual: " << actual_predicate_value;
+  if (actual_message[0] != '\0')
+    msg << " (" << actual_message << ")";
+  msg << "\nExpected: " << expected_predicate_value;
+  return msg.GetString();
+}
+
+// Helper function for implementing ASSERT_NEAR.
+AssertionResult DoubleNearPredFormat(const char* expr1,
+                                     const char* expr2,
+                                     const char* abs_error_expr,
+                                     double val1,
+                                     double val2,
+                                     double abs_error) {
+  const double diff = fabs(val1 - val2);
+  if (diff <= abs_error) return AssertionSuccess();
+
+  // TODO(wan): do not print the value of an expression if it's
+  // already a literal.
+  return AssertionFailure()
+      << "The difference between " << expr1 << " and " << expr2
+      << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n"
+      << expr1 << " evaluates to " << val1 << ",\n"
+      << expr2 << " evaluates to " << val2 << ", and\n"
+      << abs_error_expr << " evaluates to " << abs_error << ".";
+}
+
+
+// Helper template for implementing FloatLE() and DoubleLE().
+template <typename RawType>
+AssertionResult FloatingPointLE(const char* expr1,
+                                const char* expr2,
+                                RawType val1,
+                                RawType val2) {
+  // Returns success if val1 is less than val2,
+  if (val1 < val2) {
+    return AssertionSuccess();
+  }
+
+  // or if val1 is almost equal to val2.
+  const FloatingPoint<RawType> lhs(val1), rhs(val2);
+  if (lhs.AlmostEquals(rhs)) {
+    return AssertionSuccess();
+  }
+
+  // Note that the above two checks will both fail if either val1 or
+  // val2 is NaN, as the IEEE floating-point standard requires that
+  // any predicate involving a NaN must return false.
+
+  ::std::stringstream val1_ss;
+  val1_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
+          << val1;
+
+  ::std::stringstream val2_ss;
+  val2_ss << std::setprecision(std::numeric_limits<RawType>::digits10 + 2)
+          << val2;
+
+  return AssertionFailure()
+      << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n"
+      << "  Actual: " << StringStreamToString(&val1_ss) << " vs "
+      << StringStreamToString(&val2_ss);
+}
+
+}  // namespace internal
+
+// Asserts that val1 is less than, or almost equal to, val2.  Fails
+// otherwise.  In particular, it fails if either val1 or val2 is NaN.
+AssertionResult FloatLE(const char* expr1, const char* expr2,
+                        float val1, float val2) {
+  return internal::FloatingPointLE<float>(expr1, expr2, val1, val2);
+}
+
+// Asserts that val1 is less than, or almost equal to, val2.  Fails
+// otherwise.  In particular, it fails if either val1 or val2 is NaN.
+AssertionResult DoubleLE(const char* expr1, const char* expr2,
+                         double val1, double val2) {
+  return internal::FloatingPointLE<double>(expr1, expr2, val1, val2);
+}
+
+namespace internal {
+
+// The helper function for {ASSERT|EXPECT}_EQ with int or enum
+// arguments.
+AssertionResult CmpHelperEQ(const char* lhs_expression,
+                            const char* rhs_expression,
+                            BiggestInt lhs,
+                            BiggestInt rhs) {
+  if (lhs == rhs) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   FormatForComparisonFailureMessage(lhs, rhs),
+                   FormatForComparisonFailureMessage(rhs, lhs),
+                   false);
+}
+
+// A macro for implementing the helper functions needed to implement
+// ASSERT_?? and EXPECT_?? with integer or enum arguments.  It is here
+// just to avoid copy-and-paste of similar code.
+#define GTEST_IMPL_CMP_HELPER_(op_name, op)\
+AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \
+                                   BiggestInt val1, BiggestInt val2) {\
+  if (val1 op val2) {\
+    return AssertionSuccess();\
+  } else {\
+    return AssertionFailure() \
+        << "Expected: (" << expr1 << ") " #op " (" << expr2\
+        << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\
+        << " vs " << FormatForComparisonFailureMessage(val2, val1);\
+  }\
+}
+
+// Implements the helper function for {ASSERT|EXPECT}_NE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(NE, !=)
+// Implements the helper function for {ASSERT|EXPECT}_LE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(LE, <=)
+// Implements the helper function for {ASSERT|EXPECT}_LT with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(LT, < )
+// Implements the helper function for {ASSERT|EXPECT}_GE with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(GE, >=)
+// Implements the helper function for {ASSERT|EXPECT}_GT with int or
+// enum arguments.
+GTEST_IMPL_CMP_HELPER_(GT, > )
+
+#undef GTEST_IMPL_CMP_HELPER_
+
+// The helper function for {ASSERT|EXPECT}_STREQ.
+AssertionResult CmpHelperSTREQ(const char* lhs_expression,
+                               const char* rhs_expression,
+                               const char* lhs,
+                               const char* rhs) {
+  if (String::CStringEquals(lhs, rhs)) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   PrintToString(lhs),
+                   PrintToString(rhs),
+                   false);
+}
+
+// The helper function for {ASSERT|EXPECT}_STRCASEEQ.
+AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression,
+                                   const char* rhs_expression,
+                                   const char* lhs,
+                                   const char* rhs) {
+  if (String::CaseInsensitiveCStringEquals(lhs, rhs)) {
+    return AssertionSuccess();
+  }
+
+  return EqFailure(lhs_expression,
+                   rhs_expression,
+                   PrintToString(lhs),
+                   PrintToString(rhs),
+                   true);
+}
+
+// The helper function for {ASSERT|EXPECT}_STRNE.
+AssertionResult CmpHelperSTRNE(const char* s1_expression,
+                               const char* s2_expression,
+                               const char* s1,
+                               const char* s2) {
+  if (!String::CStringEquals(s1, s2)) {
+    return AssertionSuccess();
+  } else {
+    return AssertionFailure() << "Expected: (" << s1_expression << ") != ("
+                              << s2_expression << "), actual: \""
+                              << s1 << "\" vs \"" << s2 << "\"";
+  }
+}
+
+// The helper function for {ASSERT|EXPECT}_STRCASENE.
+AssertionResult CmpHelperSTRCASENE(const char* s1_expression,
+                                   const char* s2_expression,
+                                   const char* s1,
+                                   const char* s2) {
+  if (!String::CaseInsensitiveCStringEquals(s1, s2)) {
+    return AssertionSuccess();
+  } else {
+    return AssertionFailure()
+        << "Expected: (" << s1_expression << ") != ("
+        << s2_expression << ") (ignoring case), actual: \""
+        << s1 << "\" vs \"" << s2 << "\"";
+  }
+}
+
+}  // namespace internal
+
+namespace {
+
+// Helper functions for implementing IsSubString() and IsNotSubstring().
+
+// This group of overloaded functions return true iff needle is a
+// substring of haystack.  NULL is considered a substring of itself
+// only.
+
+bool IsSubstringPred(const char* needle, const char* haystack) {
+  if (needle == NULL || haystack == NULL)
+    return needle == haystack;
+
+  return strstr(haystack, needle) != NULL;
+}
+
+bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) {
+  if (needle == NULL || haystack == NULL)
+    return needle == haystack;
+
+  return wcsstr(haystack, needle) != NULL;
+}
+
+// StringType here can be either ::std::string or ::std::wstring.
+template <typename StringType>
+bool IsSubstringPred(const StringType& needle,
+                     const StringType& haystack) {
+  return haystack.find(needle) != StringType::npos;
+}
+
+// This function implements either IsSubstring() or IsNotSubstring(),
+// depending on the value of the expected_to_be_substring parameter.
+// StringType here can be const char*, const wchar_t*, ::std::string,
+// or ::std::wstring.
+template <typename StringType>
+AssertionResult IsSubstringImpl(
+    bool expected_to_be_substring,
+    const char* needle_expr, const char* haystack_expr,
+    const StringType& needle, const StringType& haystack) {
+  if (IsSubstringPred(needle, haystack) == expected_to_be_substring)
+    return AssertionSuccess();
+
+  const bool is_wide_string = sizeof(needle[0]) > 1;
+  const char* const begin_string_quote = is_wide_string ? "L\"" : "\"";
+  return AssertionFailure()
+      << "Value of: " << needle_expr << "\n"
+      << "  Actual: " << begin_string_quote << needle << "\"\n"
+      << "Expected: " << (expected_to_be_substring ? "" : "not ")
+      << "a substring of " << haystack_expr << "\n"
+      << "Which is: " << begin_string_quote << haystack << "\"";
+}
+
+}  // namespace
+
+// IsSubstring() and IsNotSubstring() check whether needle is a
+// substring of haystack (NULL is considered a substring of itself
+// only), and return an appropriate error message when they fail.
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const char* needle, const char* haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const wchar_t* needle, const wchar_t* haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const char* needle, const char* haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const wchar_t* needle, const wchar_t* haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::string& needle, const ::std::string& haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::string& needle, const ::std::string& haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+
+#if GTEST_HAS_STD_WSTRING
+AssertionResult IsSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::wstring& needle, const ::std::wstring& haystack) {
+  return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack);
+}
+
+AssertionResult IsNotSubstring(
+    const char* needle_expr, const char* haystack_expr,
+    const ::std::wstring& needle, const ::std::wstring& haystack) {
+  return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack);
+}
+#endif  // GTEST_HAS_STD_WSTRING
+
+namespace internal {
+
+#if GTEST_OS_WINDOWS
+
+namespace {
+
+// Helper function for IsHRESULT{SuccessFailure} predicates
+AssertionResult HRESULTFailureHelper(const char* expr,
+                                     const char* expected,
+                                     long hr) {  // NOLINT
+# if GTEST_OS_WINDOWS_MOBILE
+
+  // Windows CE doesn't support FormatMessage.
+  const char error_text[] = "";
+
+# else
+
+  // Looks up the human-readable system message for the HRESULT code
+  // and since we're not passing any params to FormatMessage, we don't
+  // want inserts expanded.
+  const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS;
+  const DWORD kBufSize = 4096;
+  // Gets the system's human readable message string for this HRESULT.
+  char error_text[kBufSize] = { '\0' };
+  DWORD message_length = ::FormatMessageA(kFlags,
+                                          0,  // no source, we're asking system
+                                          hr,  // the error
+                                          0,  // no line width restrictions
+                                          error_text,  // output buffer
+                                          kBufSize,  // buf size
+                                          NULL);  // no arguments for inserts
+  // Trims tailing white space (FormatMessage leaves a trailing CR-LF)
+  for (; message_length && IsSpace(error_text[message_length - 1]);
+          --message_length) {
+    error_text[message_length - 1] = '\0';
+  }
+
+# endif  // GTEST_OS_WINDOWS_MOBILE
+
+  const std::string error_hex("0x" + String::FormatHexInt(hr));
+  return ::testing::AssertionFailure()
+      << "Expected: " << expr << " " << expected << ".\n"
+      << "  Actual: " << error_hex << " " << error_text << "\n";
+}
+
+}  // namespace
+
+AssertionResult IsHRESULTSuccess(const char* expr, long hr) {  // NOLINT
+  if (SUCCEEDED(hr)) {
+    return AssertionSuccess();
+  }
+  return HRESULTFailureHelper(expr, "succeeds", hr);
+}
+
+AssertionResult IsHRESULTFailure(const char* expr, long hr) {  // NOLINT
+  if (FAILED(hr)) {
+    return AssertionSuccess();
+  }
+  return HRESULTFailureHelper(expr, "fails", hr);
+}
+
+#endif  // GTEST_OS_WINDOWS
+
+// Utility functions for encoding Unicode text (wide strings) in
+// UTF-8.
+
+// A Unicode code-point can have upto 21 bits, and is encoded in UTF-8
+// like this:
+//
+// Code-point length   Encoding
+//   0 -  7 bits       0xxxxxxx
+//   8 - 11 bits       110xxxxx 10xxxxxx
+//  12 - 16 bits       1110xxxx 10xxxxxx 10xxxxxx
+//  17 - 21 bits       11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+
+// The maximum code-point a one-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint1 = (static_cast<UInt32>(1) <<  7) - 1;
+
+// The maximum code-point a two-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint2 = (static_cast<UInt32>(1) << (5 + 6)) - 1;
+
+// The maximum code-point a three-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint3 = (static_cast<UInt32>(1) << (4 + 2*6)) - 1;
+
+// The maximum code-point a four-byte UTF-8 sequence can represent.
+const UInt32 kMaxCodePoint4 = (static_cast<UInt32>(1) << (3 + 3*6)) - 1;
+
+// Chops off the n lowest bits from a bit pattern.  Returns the n
+// lowest bits.  As a side effect, the original bit pattern will be
+// shifted to the right by n bits.
+inline UInt32 ChopLowBits(UInt32* bits, int n) {
+  const UInt32 low_bits = *bits & ((static_cast<UInt32>(1) << n) - 1);
+  *bits >>= n;
+  return low_bits;
+}
+
+// Converts a Unicode code point to a narrow string in UTF-8 encoding.
+// code_point parameter is of type UInt32 because wchar_t may not be
+// wide enough to contain a code point.
+// If the code_point is not a valid Unicode code point
+// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted
+// to "(Invalid Unicode 0xXXXXXXXX)".
+std::string CodePointToUtf8(UInt32 code_point) {
+  if (code_point > kMaxCodePoint4) {
+    return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")";
+  }
+
+  char str[5];  // Big enough for the largest valid code point.
+  if (code_point <= kMaxCodePoint1) {
+    str[1] = '\0';
+    str[0] = static_cast<char>(code_point);                          // 0xxxxxxx
+  } else if (code_point <= kMaxCodePoint2) {
+    str[2] = '\0';
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xC0 | code_point);                   // 110xxxxx
+  } else if (code_point <= kMaxCodePoint3) {
+    str[3] = '\0';
+    str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xE0 | code_point);                   // 1110xxxx
+  } else {  // code_point <= kMaxCodePoint4
+    str[4] = '\0';
+    str[3] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6));  // 10xxxxxx
+    str[0] = static_cast<char>(0xF0 | code_point);                   // 11110xxx
+  }
+  return str;
+}
+
+// The following two functions only make sense if the the system
+// uses UTF-16 for wide string encoding. All supported systems
+// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16.
+
+// Determines if the arguments constitute UTF-16 surrogate pair
+// and thus should be combined into a single Unicode code point
+// using CreateCodePointFromUtf16SurrogatePair.
+inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) {
+  return sizeof(wchar_t) == 2 &&
+      (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00;
+}
+
+// Creates a Unicode code point from UTF16 surrogate pair.
+inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first,
+                                                    wchar_t second) {
+  const UInt32 mask = (1 << 10) - 1;
+  return (sizeof(wchar_t) == 2) ?
+      (((first & mask) << 10) | (second & mask)) + 0x10000 :
+      // This function should not be called when the condition is
+      // false, but we provide a sensible default in case it is.
+      static_cast<UInt32>(first);
+}
+
+// Converts a wide string to a narrow string in UTF-8 encoding.
+// The wide string is assumed to have the following encoding:
+//   UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS)
+//   UTF-32 if sizeof(wchar_t) == 4 (on Linux)
+// Parameter str points to a null-terminated wide string.
+// Parameter num_chars may additionally limit the number
+// of wchar_t characters processed. -1 is used when the entire string
+// should be processed.
+// If the string contains code points that are not valid Unicode code points
+// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output
+// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding
+// and contains invalid UTF-16 surrogate pairs, values in those pairs
+// will be encoded as individual Unicode characters from Basic Normal Plane.
+std::string WideStringToUtf8(const wchar_t* str, int num_chars) {
+  if (num_chars == -1)
+    num_chars = static_cast<int>(wcslen(str));
+
+  ::std::stringstream stream;
+  for (int i = 0; i < num_chars; ++i) {
+    UInt32 unicode_code_point;
+
+    if (str[i] == L'\0') {
+      break;
+    } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) {
+      unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i],
+                                                                 str[i + 1]);
+      i++;
+    } else {
+      unicode_code_point = static_cast<UInt32>(str[i]);
+    }
+
+    stream << CodePointToUtf8(unicode_code_point);
+  }
+  return St
\ No newline at end of file
diff --git a/qa/workunits/rgw/jcksum/file-8b b/qa/workunits/rgw/jcksum/file-8b
new file mode 100644
index 000000000000..540606b9d0e9
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/file-8b
@@ -0,0 +1 @@
+// Copy
diff --git a/qa/workunits/rgw/jcksum/mvnw b/qa/workunits/rgw/jcksum/mvnw
new file mode 100755
index 000000000000..8d937f4c14f1
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/mvnw
@@ -0,0 +1,308 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.2.0
+#
+# Required ENV vars:
+# ------------------
+#   JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+#   MAVEN_OPTS - parameters passed to the Java VM when running Maven
+#     e.g. to debug Maven itself, use
+#       set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+#   MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+  if [ -f /usr/local/etc/mavenrc ] ; then
+    . /usr/local/etc/mavenrc
+  fi
+
+  if [ -f /etc/mavenrc ] ; then
+    . /etc/mavenrc
+  fi
+
+  if [ -f "$HOME/.mavenrc" ] ; then
+    . "$HOME/.mavenrc"
+  fi
+
+fi
+
+# OS specific support.  $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "$(uname)" in
+  CYGWIN*) cygwin=true ;;
+  MINGW*) mingw=true;;
+  Darwin*) darwin=true
+    # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+    # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+    if [ -z "$JAVA_HOME" ]; then
+      if [ -x "/usr/libexec/java_home" ]; then
+        JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME
+      else
+        JAVA_HOME="/Library/Java/Home"; export JAVA_HOME
+      fi
+    fi
+    ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+  if [ -r /etc/gentoo-release ] ; then
+    JAVA_HOME=$(java-config --jre-home)
+  fi
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+  [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] &&
+    JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  javaExecutable="$(which javac)"
+  if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then
+    # readlink(1) is not available as standard on Solaris 10.
+    readLink=$(which readlink)
+    if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
+      if $darwin ; then
+        javaHome="$(dirname "\"$javaExecutable\"")"
+        javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac"
+      else
+        javaExecutable="$(readlink -f "\"$javaExecutable\"")"
+      fi
+      javaHome="$(dirname "\"$javaExecutable\"")"
+      javaHome=$(expr "$javaHome" : '\(.*\)/bin')
+      JAVA_HOME="$javaHome"
+      export JAVA_HOME
+    fi
+  fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+  if [ -n "$JAVA_HOME"  ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+    fi
+  else
+    JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)"
+  fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+  echo "Error: JAVA_HOME is not defined correctly." >&2
+  echo "  We cannot execute $JAVACMD" >&2
+  exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+  echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+  if [ -z "$1" ]
+  then
+    echo "Path not specified to find_maven_basedir"
+    return 1
+  fi
+
+  basedir="$1"
+  wdir="$1"
+  while [ "$wdir" != '/' ] ; do
+    if [ -d "$wdir"/.mvn ] ; then
+      basedir=$wdir
+      break
+    fi
+    # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+    if [ -d "${wdir}" ]; then
+      wdir=$(cd "$wdir/.." || exit 1; pwd)
+    fi
+    # end of workaround
+  done
+  printf '%s' "$(cd "$basedir" || exit 1; pwd)"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+  if [ -f "$1" ]; then
+    # Remove \r in case we run on Windows within Git Bash
+    # and check out the repository with auto CRLF management
+    # enabled. Otherwise, we may read lines that are delimited with
+    # \r\n and produce $'-Xarg\r' rather than -Xarg due to word
+    # splitting rules.
+    tr -s '\r\n' ' ' < "$1"
+  fi
+}
+
+log() {
+  if [ "$MVNW_VERBOSE" = true ]; then
+    printf '%s\n' "$1"
+  fi
+}
+
+BASE_DIR=$(find_maven_basedir "$(dirname "$0")")
+if [ -z "$BASE_DIR" ]; then
+  exit 1;
+fi
+
+MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR
+log "$MAVEN_PROJECTBASEDIR"
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar"
+if [ -r "$wrapperJarPath" ]; then
+    log "Found $wrapperJarPath"
+else
+    log "Couldn't find $wrapperJarPath, downloading it ..."
+
+    if [ -n "$MVNW_REPOURL" ]; then
+      wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+    else
+      wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+    fi
+    while IFS="=" read -r key value; do
+      # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' )
+      safeValue=$(echo "$value" | tr -d '\r')
+      case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;;
+      esac
+    done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+    log "Downloading from: $wrapperUrl"
+
+    if $cygwin; then
+      wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
+    fi
+
+    if command -v wget > /dev/null; then
+        log "Found wget ... using wget"
+        [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet"
+        if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+            wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+        else
+            wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+        fi
+    elif command -v curl > /dev/null; then
+        log "Found curl ... using curl"
+        [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent"
+        if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+            curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+        else
+            curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+        fi
+    else
+        log "Falling back to using Java to download"
+        javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java"
+        javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class"
+        # For Cygwin, switch paths to Windows format before running javac
+        if $cygwin; then
+          javaSource=$(cygpath --path --windows "$javaSource")
+          javaClass=$(cygpath --path --windows "$javaClass")
+        fi
+        if [ -e "$javaSource" ]; then
+            if [ ! -e "$javaClass" ]; then
+                log " - Compiling MavenWrapperDownloader.java ..."
+                ("$JAVA_HOME/bin/javac" "$javaSource")
+            fi
+            if [ -e "$javaClass" ]; then
+                log " - Running MavenWrapperDownloader.java ..."
+                ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath"
+            fi
+        fi
+    fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+# If specified, validate the SHA-256 sum of the Maven wrapper jar file
+wrapperSha256Sum=""
+while IFS="=" read -r key value; do
+  case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;;
+  esac
+done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+if [ -n "$wrapperSha256Sum" ]; then
+  wrapperSha256Result=false
+  if command -v sha256sum > /dev/null; then
+    if echo "$wrapperSha256Sum  $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then
+      wrapperSha256Result=true
+    fi
+  elif command -v shasum > /dev/null; then
+    if echo "$wrapperSha256Sum  $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then
+      wrapperSha256Result=true
+    fi
+  else
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available."
+    echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties."
+    exit 1
+  fi
+  if [ $wrapperSha256Result = false ]; then
+    echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2
+    echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2
+    echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2
+    exit 1
+  fi
+fi
+
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+  [ -n "$JAVA_HOME" ] &&
+    JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
+  [ -n "$CLASSPATH" ] &&
+    CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+  [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+    MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+# shellcheck disable=SC2086 # safe args
+exec "$JAVACMD" \
+  $MAVEN_OPTS \
+  $MAVEN_DEBUG_OPTS \
+  -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+  "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+  ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/qa/workunits/rgw/jcksum/mvnw.cmd b/qa/workunits/rgw/jcksum/mvnw.cmd
new file mode 100644
index 000000000000..c4586b564e6f
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/mvnw.cmd
@@ -0,0 +1,205 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.2.0
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM     e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on"  echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
+if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+    IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Found %WRAPPER_JAR%
+    )
+) else (
+    if not "%MVNW_REPOURL%" == "" (
+        SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+    )
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Couldn't find %WRAPPER_JAR%, downloading it ...
+        echo Downloading from: %WRAPPER_URL%
+    )
+
+    powershell -Command "&{"^
+		"$webclient = new-object System.Net.WebClient;"^
+		"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+		"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+		"}"^
+		"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
+		"}"
+    if "%MVNW_VERBOSE%" == "true" (
+        echo Finished downloading %WRAPPER_JAR%
+    )
+)
+@REM End of extension
+
+@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file
+SET WRAPPER_SHA_256_SUM=""
+FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+    IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B
+)
+IF NOT %WRAPPER_SHA_256_SUM%=="" (
+    powershell -Command "&{"^
+       "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^
+       "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^
+       "  Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^
+       "  Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^
+       "  Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^
+       "  exit 1;"^
+       "}"^
+       "}"
+    if ERRORLEVEL 1 goto error
+)
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% ^
+  %JVM_CONFIG_MAVEN_PROPS% ^
+  %MAVEN_OPTS% ^
+  %MAVEN_DEBUG_OPTS% ^
+  -classpath %WRAPPER_JAR% ^
+  "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
+  %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
+if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%"=="on" pause
+
+if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
+
+cmd /C exit /B %ERROR_CODE%
diff --git a/qa/workunits/rgw/jcksum/pom-SNAPSHOT.xml b/qa/workunits/rgw/jcksum/pom-SNAPSHOT.xml
new file mode 100644
index 000000000000..637f36f7499b
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/pom-SNAPSHOT.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>com.example</groupId>
+	<artifactId>junit5-jupiter-starter-maven-snapshot</artifactId>
+	<version>1.0-SNAPSHOT</version>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>${maven.compiler.source}</maven.compiler.target>
+	</properties>
+
+	<dependencyManagement>
+		<dependencies>
+			<dependency>
+				<groupId>org.junit</groupId>
+				<artifactId>junit-bom</artifactId>
+				<version>5.10.1-SNAPSHOT</version>
+				<type>pom</type>
+				<scope>import</scope>
+			</dependency>
+		</dependencies>
+	</dependencyManagement>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<repositories>
+		<repository>
+			<id>oss-sonatype</id>
+			<name>oss-sonatype</name>
+			<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+		</repository>
+	</repositories>
+
+	<build>
+		<plugins>
+			<plugin>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<version>3.0.0</version>
+			</plugin>
+		</plugins>
+	</build>
+
+</project>
diff --git a/qa/workunits/rgw/jcksum/pom.xml b/qa/workunits/rgw/jcksum/pom.xml
new file mode 100644
index 000000000000..ab874e57638e
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/pom.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>io.ceph</groupId>
+  <artifactId>jcksum</artifactId>
+  <version>1.0</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>1.9</maven.compiler.source>
+    <maven.compiler.target>${maven.compiler.source}</maven.compiler.target>
+    <aws.java.sdk.version>2.20.43</aws.java.sdk.version>
+    <slf4j.version>1.7.28</slf4j.version>
+    <junit5.version>5.10.1</junit5.version>
+    <junit.platform.version>1.10.1</junit.platform.version>
+  </properties>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+	<groupId>org.junit</groupId>
+	<artifactId>junit-bom</artifactId>
+	<version>${junit5.version}</version>
+	<type>pom</type>
+	<scope>import</scope>
+      </dependency>
+
+      <dependency>
+	<groupId>software.amazon.awssdk</groupId>
+	<artifactId>bom</artifactId>
+	<version>${aws.java.sdk.version}</version>
+	<type>pom</type>
+	<scope>import</scope>
+      </dependency>
+      
+    </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>software.amazon.awssdk</groupId>
+          <artifactId>netty-nio-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>software.amazon.awssdk</groupId>
+          <artifactId>apache-client</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>apache-client</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.15.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.15</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${junit5.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>${junit5.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <version>${junit5.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+
+    <!-- Needed to adapt Apache Commons Logging used by Apache HTTP Client to Slf4j to avoid
+         ClassNotFoundException: org.apache.commons.logging.impl.LogFactoryImpl during runtime -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jcl-over-slf4j</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-suite</artifactId>
+      <version>${junit.platform.version}</version>
+      <scope>test</scope>
+    </dependency>
+    
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+	<artifactId>maven-surefire-plugin</artifactId>
+	<version>3.2.3</version>
+      </plugin>
+      <plugin>
+	<artifactId>maven-failsafe-plugin</artifactId>
+	<version>3.2.3</version>
+      </plugin>
+      
+    </plugins>
+  </build>
+</project>
diff --git a/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java b/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
new file mode 100644
index 000000000000..cc1632f25631
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
@@ -0,0 +1,370 @@
+package io.ceph.jcksum;
+
+import java.io.*;
+import java.util.*;
+import java.net.*; // HTTP, URI, ...
+import java.util.stream.*;
+
+import software.amazon.awssdk.auth.credentials.*;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.*;
+import software.amazon.awssdk.services.s3.model.*;
+import software.amazon.awssdk.services.s3.waiters.*;
+import software.amazon.awssdk.utils.*; // AttributeMap
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
+import software.amazon.awssdk.core.sync.*; // RequestBody
+import software.amazon.awssdk.core.checksums.*;
+import software.amazon.awssdk.core.checksums.Algorithm;
+import software.amazon.awssdk.core.waiters.*;
+
+/* MD5Sum */
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.apache.commons.codec.digest.DigestUtils;
+
+public class jcksum {
+
+	static Region region = Region.US_EAST_1;
+	static S3Client client, ssl_client;
+	
+	static String bucket_name = "sheik";
+	static String object_name = "jerbuti";
+	static String access_key = "0555b35654ad1656d804";
+	static String secret_key = "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==";
+	
+	static String http_endpoint = "http://192.168.111.1:8000";
+	static String ssl_endpoint = "https://192.168.111.1:8443";
+	
+	static int mpu_size = 5 * 1024 * 1024;
+
+	/* files containing test data of the corresponding names/sizes */
+	public static Stream<String> inputFileNames() {
+	    return Stream.of(
+	    		"file-8b",
+	    		"file-200b",
+	    		"file-21983b",
+	    		"file-5519b",
+	    		"file-204329b",
+	    		"file-256k",
+	    		"file-1m",
+	    		"file-1038757b"
+	    		);
+	} /* inputFileNames */
+
+	public static Stream<String> mpuFileNames() {
+	    return Stream.of(
+	    		"file-5m",
+	    		"file-10m",
+	    		"file-100m"
+	    		);
+	} /* mpuFileNames */
+
+  public static void createBucket(S3Client s3Client, String bucket_name) {
+    try {
+      S3Waiter s3Waiter = s3Client.waiter();
+      CreateBucketRequest bucketRequest = CreateBucketRequest.builder()
+        .bucket(bucket_name)
+        .build();
+
+      s3Client.createBucket(bucketRequest);
+      HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder()
+        .bucket(bucket_name)
+        .build();
+
+      // Wait until the bucket is created and print out the response.
+      WaiterResponse<HeadBucketResponse> waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait);
+      waiterResponse.matched().response().ifPresent(System.out::println);
+      System.out.println(bucket_name +" is ready");
+
+    } catch (S3Exception e) {
+      System.err.println(e.awsErrorDetails().errorMessage());
+      System.exit(1);
+    }
+  } /* createBucket */
+
+	public static void listBucket(S3Client s3) {
+        try {
+            ListObjectsRequest listObjects = ListObjectsRequest.builder()
+                .bucket(bucket_name)
+                .build();
+
+            ListObjectsResponse res = s3.listObjects(listObjects);
+            List<S3Object> objects = res.contents();
+            for (S3Object obj: objects) {
+            	System.out.println(
+            		String.format("obj key: %s owner: %s size: %d", obj.key(), obj.owner(), obj.size()));
+            }
+
+        } catch (S3Exception e) {
+            System.err.println(e.awsErrorDetails().errorMessage());
+            System.exit(1);
+        }
+	}
+
+	public static GetObjectResponse GetObject(S3Client s3, String in_key_name, String out_file_name) {
+			GetObjectResponse resp = null;
+			
+			GetObjectRequest get_req =
+					GetObjectRequest.builder()
+						.bucket(bucket_name)
+						.key(in_key_name)
+						.build();
+			try {
+				File f = new File(out_file_name);
+				if (f.exists()) {
+					f.delete();
+				}
+				resp = s3.getObject(get_req, ResponseTransformer.toFile(f));
+			}  catch (S3Exception e) {
+	            System.err.println(e.awsErrorDetails().errorMessage());
+	            System.exit(1);
+	        } catch (Exception e) {
+	        	e.printStackTrace();
+	        }
+			
+			return resp;
+	}
+	
+	public static CompleteMultipartUploadResponse mpuObjectFromFile(S3Client s3, String in_file_path, String out_key_name) {
+		File f = new File(in_file_path);
+		CompleteMultipartUploadResponse completedUploadResponse = null;
+		CreateMultipartUploadRequest create_req =
+				CreateMultipartUploadRequest.builder()
+					.bucket(bucket_name)
+					.key(out_key_name)
+					.checksumAlgorithm(ChecksumAlgorithm.SHA256)
+					.build();
+		
+		CreateMultipartUploadResponse createdUpload = s3.createMultipartUpload(create_req);
+		
+		/* the file streaming method shown in aws-doc-sdk-examples/.../CheckObjectIntegrity.java
+		 * creates a FileInputStream from a file, but then copies each chunk into a ByteBuffer by
+		 * hand before uploading--which per code comments, forces RequestBody to copy the buffer
+		 * again before sending it--let's see if we can use RequestBody.fromInputStream() instead,
+		 * it seems to be designed for this purpose (I'm not clear why you would share the InputStream,
+		 * and the only apparent reason to prefer the buffer even with an async client seems to be
+		 * avoid a deferred close on it) */
+		
+		try {
+			InputStream in = new FileInputStream(f);
+			List<CompletedPart> completedParts = new ArrayList<CompletedPart>();
+			int partNumber = 1;
+			
+			for (long resid = f.length(); resid > 0;) {
+				long bytes = Math.min(mpu_size, resid);
+                UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
+                        .partNumber(partNumber)
+                        .uploadId(createdUpload.uploadId())
+                        .bucket(bucket_name)
+                        .key(out_key_name)
+                        .checksumAlgorithm(ChecksumAlgorithm.SHA256)
+                        .build();
+                UploadPartResponse uploadedPart = s3.uploadPart(uploadPartRequest,
+                	RequestBody.fromInputStream(in, bytes));
+                CompletedPart part = CompletedPart.builder().
+                        partNumber(partNumber)
+                        .checksumSHA256(uploadedPart.checksumSHA256())
+                        .eTag(uploadedPart.eTag()).build();
+                completedParts.add(part);
+                partNumber++;
+                resid -= bytes;
+			} /* for all chunks * bytes */
+			
+            CompletedMultipartUpload completedMultipartUpload =
+            		CompletedMultipartUpload.builder().parts(completedParts).build();
+            completedUploadResponse = s3.completeMultipartUpload(
+                    CompleteMultipartUploadRequest.builder()
+                            .bucket(bucket_name)
+                            .key(out_key_name)
+                            .uploadId(createdUpload.uploadId())
+                            .multipartUpload(completedMultipartUpload).build());
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		return completedUploadResponse;
+	} /* mpuObjectFromFile */
+	
+	public static CompleteMultipartUploadResponse mpuObjectFromFileNoCksum(S3Client s3, String in_file_path, String out_key_name) {
+		File f = new File(in_file_path);
+		CompleteMultipartUploadResponse completedUploadResponse = null;
+		CreateMultipartUploadRequest create_req =
+				CreateMultipartUploadRequest.builder()
+					.bucket(bucket_name)
+					.key(out_key_name)
+					/* .checksumAlgorithm(ChecksumAlgorithm.SHA256) */
+					.build();
+		
+		CreateMultipartUploadResponse createdUpload = s3.createMultipartUpload(create_req);
+		
+		/* the file streaming method shown in aws-doc-sdk-examples/.../CheckObjectIntegrity.java
+		 * creates a FileInputStream from a file, but then copies each chunk into a ByteBuffer by
+		 * hand before uploading--which per code comments, forces RequestBody to copy the buffer
+		 * again before sending it--let's see if we can use RequestBody.fromInputStream() instead,
+		 * it seems to be designed for this purpose (I'm not clear why you would share the InputStream,
+		 * and the only apparent reason to prefer the buffer even with an async client seems to be
+		 * avoid a deferred close on it) */
+		
+		try {
+			InputStream in = new FileInputStream(f);
+			List<CompletedPart> completedParts = new ArrayList<CompletedPart>();
+			int partNumber = 1;
+			
+			for (long resid = f.length(); resid > 0;) {
+				long bytes = Math.min(mpu_size, resid);
+                UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
+                        .partNumber(partNumber)
+                        .uploadId(createdUpload.uploadId())
+                        .bucket(bucket_name)
+                        .key(out_key_name)
+                        /* .checksumAlgorithm(ChecksumAlgorithm.SHA256) */
+                        .build();
+                UploadPartResponse uploadedPart = s3.uploadPart(uploadPartRequest,
+                	RequestBody.fromInputStream(in, bytes));
+                CompletedPart part = CompletedPart.builder().
+                        partNumber(partNumber)
+                        .checksumSHA256(uploadedPart.checksumSHA256())
+                        .eTag(uploadedPart.eTag()).build();
+                completedParts.add(part);
+                partNumber++;
+                resid -= bytes;
+			} /* for all chunks * bytes */
+			
+            CompletedMultipartUpload completedMultipartUpload =
+            		CompletedMultipartUpload.builder().parts(completedParts).build();
+            completedUploadResponse = s3.completeMultipartUpload(
+                    CompleteMultipartUploadRequest.builder()
+                            .bucket(bucket_name)
+                            .key(out_key_name)
+                            .uploadId(createdUpload.uploadId())
+                            .multipartUpload(completedMultipartUpload).build());
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		return completedUploadResponse;
+	} /* mpuObjectFromFileNoCksum */
+	
+	/* without mpu and without explicit checksum request, chunked encoding is
+	 * not (automatically?) sent;  with a checksum specified, it is */
+	public static PutObjectResponse putObjectFromFileNoCksum(S3Client s3, String in_file_path, String out_key_name) {
+       	PutObjectResponse resp = null;
+		try {
+            Map<String, String> metadata = new HashMap<>();
+            metadata.put("x-amz-meta-wax", "ahatchee");
+            PutObjectRequest putOb = PutObjectRequest.builder()
+                .bucket(bucket_name)
+                .key(out_key_name)
+                .metadata(metadata)
+                .build();
+
+            resp = s3.putObject(putOb, RequestBody.fromFile(new File(in_file_path))); // "using the full contents of the specified file"
+
+        } catch (S3Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
+        return resp;
+    } /* putObjectFromFileNoCksum */
+	
+	/* without mpu and without explicit checksum request, chunked encoding is
+	 * not (automatically?) sent;  with a checksum specified, it is */
+	public static PutObjectResponse putObjectFromFile(S3Client s3, String in_file_path, String out_key_name) {
+		PutObjectResponse resp = null;
+        try {
+            Map<String, String> metadata = new HashMap<>();
+            metadata.put("x-amz-meta-wax", "ahatchee");
+            PutObjectRequest putOb = PutObjectRequest.builder()
+                .bucket(bucket_name)
+                .key(out_key_name)
+                .metadata(metadata)
+                .checksumAlgorithm(ChecksumAlgorithm.SHA256)
+                .build();
+
+            RequestBody rbody = RequestBody.fromFile(new File(in_file_path));
+            resp = s3.putObject(putOb, rbody); // "using the full contents of the specified file"
+            System.out.println("PutObjectResponse");
+        } catch (S3Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+        }
+        return resp;
+    } /* putObjectFromFile */
+	
+	
+    public static String getMD5Sum(String filePath) throws IOException {
+        try (InputStream is = Files.newInputStream(Paths.get(filePath))) {
+            return DigestUtils.md5Hex(is);
+        }
+    }
+    
+    public static String getSHA512Sum(String filePath) throws IOException {
+        try (InputStream is = Files.newInputStream(Paths.get(filePath))) {
+            return DigestUtils.sha512Hex(is);
+        }
+    }
+	
+	public static void main(String[] args) throws URISyntaxException {
+		
+		AwsCredentials creds = AwsBasicCredentials.create(access_key, secret_key);
+		URI http_uri = new URI(http_endpoint);
+		
+		/* ah, yeah.  so many options.
+		 * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ApacheHttpClient.Builder.html
+		 */		
+		SdkHttpClient apacheHttpClient = ApacheHttpClient.builder()
+	            .buildWithDefaults(AttributeMap.builder().put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true).build());
+		
+		/* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */
+        client = S3Client.builder()
+        		.endpointOverride(http_uri)
+                .credentialsProvider(StaticCredentialsProvider.create(creds))
+                .region(region)
+                .build();
+
+		URI ssl_uri = new URI(ssl_endpoint);
+        ssl_client = S3Client.builder()
+        		.httpClient(apacheHttpClient)
+        		.endpointOverride(ssl_uri)
+                .credentialsProvider(StaticCredentialsProvider.create(creds))
+                .region(region)
+                .build();
+		
+        //listBucket(client);
+        //listBucket(ssl_client);
+        
+        String out_name = "object_out";
+        
+        // if !ssl, we see x-amz-trailer-signature (in the trailer)
+        //putObjectFromFile(client, "file-8b", out_name); // minimal STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER case
+        putObjectFromFile(client, "file-200b", out_name); // STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (multi) (200) (checksum?)
+        //putObjectFromFile(client, "file-21983b", out_name); // STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (multi) (200) (checksum?)
+        //putObjectFromFile(client, "file-256k", out_name); // x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (multi) (200) (checksum?)
+        //putObjectFromFile(client, "file-1M", out_name); // x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (multi) (200) (checksum?)
+       
+        /* ok to here! */
+        
+        // XXXX minimal streaming unsigned checksum trailer case
+        //putObjectFromFile(ssl_client, "file-8b", out_name);
+        //putObjectFromFile(ssl_client, "file-200b", out_name); // STREAMING-UNSIGNED-PAYLOAD-TRAILER (400)
+        //putObjectFromFile(ssl_client, "file-21983b", out_name); // x-amz-content-sha256:STREAMING-UNSIGNED-PAYLOAD-TRAILER (400)
+        //putObjectFromFile(ssl_client, "file-256k", out_name); //x-amz-content-sha256:STREAMING-UNSIGNED-PAYLOAD-TRAILER (multi) (400)
+        
+        // minimal, traditional awssigv4 streaming hmac sha256 case (works)
+        //putObjectFromFileNoCksum(client, "file-8b", out_name);
+        
+        //putObjectFromFileNoCksum(client, "file-200b", object_name); // STREAMING-AWS4-HMAC-SHA256-PAYLOAD (multi) 200
+        //putObjectFromFileNoCksum(ssl_client, "file-200b", out_name); // UNSIGNED-PAYLOAD (no completer) 200
+
+        //mpuObjectFromFile(client, "file-200b", out_name); // STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER(multi) 400 (no completer?)
+        //mpuObjectFromFile(client, "file-256k", out_name);
+        //mpuObjectFromFile(ssl_client, "file-200b", out_name); // STREAMING-UNSIGNED-PAYLOAD-TRAILER (no completer) 400
+        //mpuObjectFromFile(ssl_client, "file-256k", out_name);
+        
+        //mpuObjectFromFileNoCksum(client, "file-200b", out_name); // AWS4-HMAC-SHA256-PAYLOAD (no completer?) 200
+        //mpuObjectFromFileNoCksum(client, "file-256k", out_name);
+        //mpuObjectFromFileNoCksum(ssl_client, "file-200b", out_name); //x-amz-content-sha256:UNSIGNED-PAYLOAD (no completer) 200
+        //mpuObjectFromFileNoCksum(ssl_client, "file-256k", out_name);
+		System.out.println("all that way...");
+	} /* main */
+} /* jcksum */
diff --git a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
new file mode 100644
index 000000000000..afdcdb90ddc9
--- /dev/null
+++ b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
@@ -0,0 +1,308 @@
+/**
+ * 
+ */
+package io.ceph.jcksum;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.stream.*;
+import java.nio.*; // ByteBuffer
+import java.nio.file.Files.*; //newByteChannel
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.nio.channels.*;
+import java.lang.Math.*;
+
+import io.ceph.jcksum.*;
+import software.amazon.awssdk.auth.credentials.*;
+import software.amazon.awssdk.http.*;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+
+import software.amazon.awssdk.services.s3.*;
+import software.amazon.awssdk.services.s3.model.*;
+import software.amazon.awssdk.utils.*; // AttributeMap
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
+import software.amazon.awssdk.core.sync.*; // RequestBody
+import software.amazon.awssdk.core.checksums.*;
+import software.amazon.awssdk.core.checksums.Algorithm;
+
+import org.junit.jupiter.api.*; /* BeforeAll, Test, &c */
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestInstance.*;
+
+import org.junit.jupiter.params.*;
+import org.junit.jupiter.params.provider.*;
+
+/**
+ * 
+ */
+@TestInstance(Lifecycle.PER_CLASS)
+class PutObjects {
+
+	public AwsCredentials creds;
+	public URI http_uri, ssl_uri;
+	static S3Client client, ssl_client;
+	
+	void generateFile(String in_file_path, String out_file_path, long length) {
+		try {
+			Path ifp = Paths.get(in_file_path);
+			File f = ifp.toFile();
+
+			long if_size = f.length();
+			if (if_size < (1024 * 1024)) {
+				throw new IOException("in_file_path is supposed to be file-1m (i.e., a 1Mb file");
+			}
+
+			File of = new File(out_file_path);
+			if (of.exists()) {
+				of.delete();
+			}
+			
+			FileOutputStream fout = new FileOutputStream(of);
+			FileChannel wch = fout.getChannel();
+
+			long resid = length;
+			long r_offset = 0;
+			long f_resid = 0;
+			
+			FileInputStream fin = new FileInputStream(f);
+			FileChannel rch = fin.getChannel();
+			
+			while (resid > 0) {
+				long to_write = Long.min(resid, f_resid);
+				while (to_write > 0) {
+					long written = rch.transferTo(r_offset, to_write, wch);
+					r_offset += written;
+					to_write -= written;
+					resid -= written;
+					f_resid -= written;
+				}
+				if (f_resid < 0) {
+					throw new IOException("read overrun (logic error)");
+				}
+				if (f_resid == 0) {
+					rch.position(0);
+					f_resid = 1024 * 1024;
+					r_offset = 0;
+					
+				}
+			}
+			if (rch != null) {
+				rch.close();
+			}
+			if (wch != null) {
+				wch.close();
+			}
+		} catch (IOException e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+	} /* generateFile */
+
+  void readEnvironmentVars() {
+    jcksum.access_key = System.getProperty("AWS_ACCESS_KEY_ID", "0555b35654ad1656d804");
+    jcksum.secret_key = System.getProperty("AWS_SECRET_ACCESS_KEY", "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==");
+
+    jcksum.http_endpoint = System.getProperty("RGW_HTTP_ENDPOINT_URL", "http://192.168.111.1:8000");
+    jcksum.http_endpoint = System.getProperty("RGW_HTTPS_ENDPOINT_URL", "https://192.168.111.1:8443");
+  } /* readEnvironmentVArs */
+
+	void generateBigFiles() {
+		generateFile("file-1m", "file-5m", 5 * 1024 * 1024);
+		generateFile("file-1m", "file-10m", 10 * 1024 * 1024);
+		generateFile("file-1m", "file-100m", 100 * 1024 * 1024);
+		/* the next lengths happen to be prime */
+		generateFile("file-1m", "file-5519b", 5519);
+		generateFile("file-1m", "file-204329b", 204329);
+		generateFile("file-1m", "file-1038757b", 1038757);
+	}
+
+	@BeforeAll
+	void setup() throws URISyntaxException {
+
+		creds = AwsBasicCredentials.create(jcksum.access_key, jcksum.secret_key);
+		http_uri = new URI(jcksum.http_endpoint);
+
+		SdkHttpClient apacheHttpClient = ApacheHttpClient.builder()
+	            .buildWithDefaults(AttributeMap.builder().put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true).build());
+		
+		/* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */
+        client = S3Client.builder()
+        		.endpointOverride(http_uri)
+                .credentialsProvider(StaticCredentialsProvider.create(creds))
+                .region(jcksum.region)
+                .build();
+
+		ssl_uri = new URI(jcksum.ssl_endpoint);
+        ssl_client = S3Client.builder()
+        		.httpClient(apacheHttpClient)
+        		.endpointOverride(ssl_uri)
+                .credentialsProvider(StaticCredentialsProvider.create(creds))
+                .region(jcksum.region)
+                .build();
+		
+    generateBigFiles();
+    readEnvironmentVars();
+
+    /* create test bucket if it doesn't exist yet */
+		try {
+      jcksum.createBucket(client, jcksum.bucket_name);
+		} catch (Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+  } /* setup */
+
+	/* TODO: zap */
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
+	void testWithExplicitLocalMethodSource(String argument) {
+	    assertNotNull(argument);
+	    System.out.println("arg: " + argument);
+	}
+
+	boolean compareFileDigests(String lhp, String rhp) throws IOException {
+		String lh5 = jcksum.getSHA512Sum(lhp);
+		String rh5 = jcksum.getSHA512Sum(rhp);
+		return lh5.equals(rh5);
+	}
+	
+	boolean putAndVerifyCksum(S3Client s3, String in_file_path) {
+		boolean md5_check = false;
+		try {
+			String out_key_name = "out_key_name"; // name we'll give the object in S3
+			PutObjectResponse put_rsp = jcksum.putObjectFromFile(s3, in_file_path, out_key_name);
+			String out_file_path = "out_file_name"; // name of the temp object when we download it back
+			GetObjectResponse get_rsp = jcksum.GetObject(s3, out_key_name, out_file_path);
+			md5_check = compareFileDigests(in_file_path, out_file_path);
+		} catch (Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+		return md5_check;
+	}
+
+	boolean putAndVerifyNoCksum(S3Client s3, String in_file_path) {
+		boolean md5_check = false;
+		try {
+			String out_key_name = "out_key_name"; // name we'll give the object in S3
+			PutObjectResponse put_rsp = jcksum.putObjectFromFileNoCksum(s3, in_file_path, out_key_name);
+			String out_file_path = "out_file_name"; // name of the temp object when we download it back
+			GetObjectResponse get_rsp = jcksum.GetObject(s3, out_key_name, out_file_path);
+			md5_check = compareFileDigests(in_file_path, out_file_path);
+		} catch (Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+		return md5_check;
+	}
+
+	boolean mpuAndVerifyCksum(S3Client s3, String in_file_path) {
+		boolean md5_check = false;
+		try {
+			String out_key_name = "out_key_name"; // name we'll give the object in S3
+			CompleteMultipartUploadResponse put_rsp = jcksum.mpuObjectFromFile(s3, in_file_path, out_key_name);
+			String out_file_path = "out_file_name"; // name of the temp object when we download it back
+			GetObjectResponse get_rsp = jcksum.GetObject(s3, out_key_name, out_file_path);
+			md5_check = compareFileDigests(in_file_path, out_file_path);
+		} catch (Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+		return md5_check;
+	}
+
+	boolean mpuAndVerifyNoCksum(S3Client s3, String in_file_path) {
+		boolean md5_check = false;
+		try {
+			String out_key_name = "out_key_name"; // name we'll give the object in S3
+			CompleteMultipartUploadResponse put_rsp = jcksum.mpuObjectFromFileNoCksum(s3, in_file_path, out_key_name);
+			String out_file_path = "out_file_name"; // name of the temp object when we download it back
+			GetObjectResponse get_rsp = jcksum.GetObject(s3, out_key_name, out_file_path);
+			md5_check = compareFileDigests(in_file_path, out_file_path);
+		} catch (Exception e) {
+            System.err.println(e.getMessage());
+            System.exit(1);
+		}
+		return md5_check;
+	}
+	
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
+	void putObjectFromFileCksum(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("putObjectFromFileCksum called with " + in_file_path);
+		rslt = putAndVerifyCksum(client, in_file_path);
+		assertTrue(rslt);
+	}
+	
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
+	void putObjectFromFileNoCksum(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("putObjectFromFileNoCksum called with " + in_file_path);
+		rslt = putAndVerifyNoCksum(client, in_file_path);
+		assertTrue(rslt);
+	}
+
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
+	void mpuObjectFromFileCksum(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("mpuObjectFromFileCksum called with " + in_file_path);
+		rslt = mpuAndVerifyCksum(client, in_file_path);
+		assertTrue(rslt);
+	}
+
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
+	void mpuObjectFromFileNoCksum(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("mpuObjectFromFileNoCksum called with " + in_file_path);
+		rslt = mpuAndVerifyNoCksum(client, in_file_path);
+		assertTrue(rslt);
+	}
+	
+	/* SSL */
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
+	void putObjectFromFileCksumSSL(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("putObjectFromFileCksumSSL called with " + in_file_path);
+		rslt = putAndVerifyCksum(ssl_client, in_file_path);
+		assertTrue(rslt);
+	}
+	
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
+	void putObjectFromFileNoCksumSSL(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("putObjectFromFileNoCksumSSL called with " + in_file_path);
+		rslt = putAndVerifyNoCksum(ssl_client, in_file_path);
+		assertTrue(rslt);
+	}
+
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
+	void mpuObjectFromFileCksumSSL(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("mpuObjectFromFileCksumSSL called with " + in_file_path);
+		rslt = mpuAndVerifyCksum(ssl_client, in_file_path);
+		assertTrue(rslt);
+	}
+
+	@ParameterizedTest
+	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
+	void mpuObjectFromFileNoCksumSSL(String in_file_path) {
+		boolean rslt = false;
+		System.out.println("mpuObjectFromFileNoCksumSSL called with " + in_file_path);
+		rslt = mpuAndVerifyNoCksum(ssl_client, in_file_path);
+		assertTrue(rslt);
+	}
+
+} /* class PutObjects */
diff --git a/qa/workunits/rgw/test_awssdkv4_sig.sh b/qa/workunits/rgw/test_awssdkv4_sig.sh
new file mode 100755
index 000000000000..fa06d875cfcc
--- /dev/null
+++ b/qa/workunits/rgw/test_awssdkv4_sig.sh
@@ -0,0 +1,42 @@
+#!/bin/sh -e
+#
+# To run this test script with a cluster created via vstart.sh:
+# $PATH needs to be set for radosgw-admin and ceph_test_librgw executables.
+# $KEYRING need to be set as the path for a vstart clusters Ceph keyring.
+#
+# Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw:
+# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_librgw_file.sh
+
+if [ -z ${AWS_ACCESS_KEY_ID} ]
+then
+    export AWS_ACCESS_KEY_ID=`openssl rand -base64 20`
+    export AWS_SECRET_ACCESS_KEY=`openssl rand -base64 40`
+
+    radosgw-admin user create --uid ceph-test-librgw-file \
+       --access-key $AWS_ACCESS_KEY_ID \
+       --secret $AWS_SECRET_ACCESS_KEY \
+       --display-name "maven test user" \
+       --email librgw@example.com || echo "maven user exists"
+
+    # keyring override for teuthology env
+    if [ -z ${KEYRING} ]
+    then
+      KEYRING="/etc/ceph/ceph.keyring"
+    fi
+    K="-k ${KEYRING}"
+fi
+
+# the required S3 access_key and secret_key are already exported above, but
+# we need to hook up the S3 endpoints
+
+# the following are taken from 
+export RGW_HTTP_ENDPOINT_URL="http://localhost:80"
+export RGW_HTTPS_ENDPOINT_URL="https://localhost:443"
+
+# rgw/test_awssdkv4_sig.sh
+pushd jcksum
+
+./mvnw clean package
+./mvnw test -Dtest=PutObjects
+
+exit 0

From dc3063786c8fa4ddc0fad2fbef3b692937e3738f Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Thu, 11 Jan 2024 18:12:22 -0500
Subject: [PATCH 1889/2492] awssigv4: fix signing of 0-byte object

1. correctly match signature of 0-length chunk
2. initialize lf_bytes

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 qa/workunits/rgw/jcksum/file-0b                    |  0
 .../src/main/java/io/ceph/jcksum/jcksum.java       |  1 +
 qa/workunits/rgw/test_awssdkv4_sig.sh              |  2 +-
 src/rgw/rgw_auth_s3.cc                             | 14 +++++++++++---
 src/rgw/rgw_auth_s3.h                              |  1 +
 5 files changed, 14 insertions(+), 4 deletions(-)
 create mode 100644 qa/workunits/rgw/jcksum/file-0b

diff --git a/qa/workunits/rgw/jcksum/file-0b b/qa/workunits/rgw/jcksum/file-0b
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java b/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
index cc1632f25631..8ac68a27e3c3 100644
--- a/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
+++ b/qa/workunits/rgw/jcksum/src/main/java/io/ceph/jcksum/jcksum.java
@@ -42,6 +42,7 @@ public class jcksum {
 	/* files containing test data of the corresponding names/sizes */
 	public static Stream<String> inputFileNames() {
 	    return Stream.of(
+          "file-0b",
 	    		"file-8b",
 	    		"file-200b",
 	    		"file-21983b",
diff --git a/qa/workunits/rgw/test_awssdkv4_sig.sh b/qa/workunits/rgw/test_awssdkv4_sig.sh
index fa06d875cfcc..0f1cfeb2c075 100755
--- a/qa/workunits/rgw/test_awssdkv4_sig.sh
+++ b/qa/workunits/rgw/test_awssdkv4_sig.sh
@@ -39,4 +39,4 @@ pushd jcksum
 ./mvnw clean package
 ./mvnw test -Dtest=PutObjects
 
-exit 0
+exit
diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 765d19bfa689..86ea87388dbb 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -1187,10 +1187,18 @@ bool AWSv4ComplMulti::is_signature_mismatched()
 		     << calc_signature << dendl;
     ldout(cct(), 16) << "AWSv4ComplMulti: prev_chunk_signature="
 		     << prev_chunk_signature << dendl;
-
   }
 
-  if (chunk_meta.get_signature() != calc_signature) {
+  auto match_signatures = [&]() -> bool {
+    /* sentinel case: 0-length chunk, likely chunk 0 */
+    if (chunk_meta.get_offset() == 0) [[unlikely]] {
+      return chunk_meta.get_signature() == prev_chunk_signature;
+    }
+    /* all other cases */
+    return chunk_meta.get_signature() == calc_signature;
+  };
+
+  if (! match_signatures()) [[unlikely]] {
     ldout(cct(), 16) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
                    << dendl;
     return true;
@@ -1198,7 +1206,7 @@ bool AWSv4ComplMulti::is_signature_mismatched()
     prev_chunk_signature = chunk_meta.get_signature();
     return false;
   }
-}
+} /* AWSv4ComplMulti::is_signature_mismatched */
 
 AWSv4ComplMulti::ReceiveChunkResult AWSv4ComplMulti::recv_chunk(
   char* const buf, const size_t buf_max, uint32_t cnt, bool& eof)
diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h
index f68768200565..b2b1238f3130 100644
--- a/src/rgw/rgw_auth_s3.h
+++ b/src/rgw/rgw_auth_s3.h
@@ -410,6 +410,7 @@ class AWSv4ComplMulti : public rgw::auth::Completer,
       /* The evolving state. */
       chunk_meta(ChunkMeta::create_first(
 		   seed_signature, flags, 0 /* first call in cycle */)),
+      lf_bytes(0),
       stream_pos(0),
       sha256_hash(calc_hash_sha256_open_stream()),
       prev_chunk_signature(std::move(seed_signature))

From d374a21beb4faac63c0bd920572651079af25209 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Sat, 13 Jan 2024 13:57:36 -0500
Subject: [PATCH 1890/2492] awssigv4: fix recognition of trailer boundary when
 no trailing signature

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_auth_s3.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 86ea87388dbb..82b2524bd502 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -1552,9 +1552,15 @@ bool AWSv4ComplMulti::complete()
     std::string_view expected_trailer_signature;
     std::string calculated_trailer_signature;
 
-    if (tbuf_pos > sarrlen("\r\n0;")) {
-      const std::string_view sv_trailer(trailer_vec.data() + sarrlen("\r\n0;"),
-                                        tbuf_pos - sarrlen("\r\n0;"));
+    /* the trailer boundary is just "\r\n0" when we have no trailer
+     * signature */
+    if (tbuf_pos > sarrlen("\r\n0")) {
+      auto trailer_off = sarrlen("\r\n0");
+      if (*(trailer_vec.data() + trailer_off) == ';') {
+	++trailer_off;
+      }
+      const std::string_view sv_trailer(
+        trailer_vec.data() + trailer_off, tbuf_pos - trailer_off);
 
       if (cct()->_conf->subsys.should_gather(ceph_subsys_rgw, 10)) [[unlikely]] {
         ldout(cct(), 10) << "trailer_section: " << sv_trailer << dendl;

From 80c9433847dd0af0a7b6d7ff0decad1d6c892940 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Fri, 2 Feb 2024 14:59:20 -0500
Subject: [PATCH 1891/2492] rgw_sigv4: fixes to bootstrap maven/junit5 suite

The junit5 suite in fact chooses selects transport security (SSL)
strictly from the endpoint URL.  The test_awssdkv4_sig.sh (or its
caller?) only needs to export RGW_HTTP_ENDPOINT_URL appropriately
to get one or the other.

Fix several mistakes in refactoring caught by Ali Maredia.
Print AccessKey, SecretKey and EndpointURL on startup

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 .../test/java/io/ceph/jcksum/PutObjects.java  | 83 ++++++-------------
 qa/workunits/rgw/test_awssdkv4_sig.sh         | 15 ++--
 2 files changed, 36 insertions(+), 62 deletions(-)

diff --git a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
index afdcdb90ddc9..685914abee6d 100644
--- a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
+++ b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
@@ -45,8 +45,8 @@
 class PutObjects {
 
 	public AwsCredentials creds;
-	public URI http_uri, ssl_uri;
-	static S3Client client, ssl_client;
+	public URI http_uri;
+	static S3Client client;
 	
 	void generateFile(String in_file_path, String out_file_path, long length) {
 		try {
@@ -104,12 +104,19 @@ void generateFile(String in_file_path, String out_file_path, long length) {
 		}
 	} /* generateFile */
 
-  void readEnvironmentVars() {
-    jcksum.access_key = System.getProperty("AWS_ACCESS_KEY_ID", "0555b35654ad1656d804");
-    jcksum.secret_key = System.getProperty("AWS_SECRET_ACCESS_KEY", "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==");
 
-    jcksum.http_endpoint = System.getProperty("RGW_HTTP_ENDPOINT_URL", "http://192.168.111.1:8000");
-    jcksum.http_endpoint = System.getProperty("RGW_HTTPS_ENDPOINT_URL", "https://192.168.111.1:8443");
+  String get_envvar(String key, String defstr) {
+    String var = System.getenv(key);
+    if (var == null) {
+      return defstr;
+    }
+    return var;
+  }
+
+  void readEnvironmentVars() {
+    jcksum.access_key = get_envvar("AWS_ACCESS_KEY_ID", "0555b35654ad1656d804");
+    jcksum.secret_key = get_envvar("AWS_SECRET_ACCESS_KEY", "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==");
+    jcksum.http_endpoint = get_envvar("RGW_HTTP_ENDPOINT_URL", "");
   } /* readEnvironmentVArs */
 
 	void generateBigFiles() {
@@ -125,6 +132,13 @@ void generateBigFiles() {
 	@BeforeAll
 	void setup() throws URISyntaxException {
 
+    readEnvironmentVars();
+
+    System.out.println("PutObjects.java: starting test run:");
+    System.out.println("\tAccessKey=" + jcksum.access_key);
+    System.out.println("\tSecretKey=" + jcksum.secret_key);
+    System.out.println("\tEndpointUrl=" + jcksum.http_endpoint);
+
 		creds = AwsBasicCredentials.create(jcksum.access_key, jcksum.secret_key);
 		http_uri = new URI(jcksum.http_endpoint);
 
@@ -133,21 +147,13 @@ void setup() throws URISyntaxException {
 		
 		/* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */
         client = S3Client.builder()
-        		.endpointOverride(http_uri)
-                .credentialsProvider(StaticCredentialsProvider.create(creds))
-                .region(jcksum.region)
-                .build();
+          .endpointOverride(http_uri)
+          .credentialsProvider(StaticCredentialsProvider.create(creds))
+          .region(jcksum.region)
+          .forcePathStyle(true)
+          .build();
 
-		ssl_uri = new URI(jcksum.ssl_endpoint);
-        ssl_client = S3Client.builder()
-        		.httpClient(apacheHttpClient)
-        		.endpointOverride(ssl_uri)
-                .credentialsProvider(StaticCredentialsProvider.create(creds))
-                .region(jcksum.region)
-                .build();
-		
     generateBigFiles();
-    readEnvironmentVars();
 
     /* create test bucket if it doesn't exist yet */
 		try {
@@ -267,42 +273,5 @@ void mpuObjectFromFileNoCksum(String in_file_path) {
 		rslt = mpuAndVerifyNoCksum(client, in_file_path);
 		assertTrue(rslt);
 	}
-	
-	/* SSL */
-	@ParameterizedTest
-	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
-	void putObjectFromFileCksumSSL(String in_file_path) {
-		boolean rslt = false;
-		System.out.println("putObjectFromFileCksumSSL called with " + in_file_path);
-		rslt = putAndVerifyCksum(ssl_client, in_file_path);
-		assertTrue(rslt);
-	}
-	
-	@ParameterizedTest
-	@MethodSource("io.ceph.jcksum.jcksum#inputFileNames")
-	void putObjectFromFileNoCksumSSL(String in_file_path) {
-		boolean rslt = false;
-		System.out.println("putObjectFromFileNoCksumSSL called with " + in_file_path);
-		rslt = putAndVerifyNoCksum(ssl_client, in_file_path);
-		assertTrue(rslt);
-	}
-
-	@ParameterizedTest
-	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
-	void mpuObjectFromFileCksumSSL(String in_file_path) {
-		boolean rslt = false;
-		System.out.println("mpuObjectFromFileCksumSSL called with " + in_file_path);
-		rslt = mpuAndVerifyCksum(ssl_client, in_file_path);
-		assertTrue(rslt);
-	}
-
-	@ParameterizedTest
-	@MethodSource("io.ceph.jcksum.jcksum#mpuFileNames")
-	void mpuObjectFromFileNoCksumSSL(String in_file_path) {
-		boolean rslt = false;
-		System.out.println("mpuObjectFromFileNoCksumSSL called with " + in_file_path);
-		rslt = mpuAndVerifyNoCksum(ssl_client, in_file_path);
-		assertTrue(rslt);
-	}
 
 } /* class PutObjects */
diff --git a/qa/workunits/rgw/test_awssdkv4_sig.sh b/qa/workunits/rgw/test_awssdkv4_sig.sh
index 0f1cfeb2c075..5c56a46285f6 100755
--- a/qa/workunits/rgw/test_awssdkv4_sig.sh
+++ b/qa/workunits/rgw/test_awssdkv4_sig.sh
@@ -5,18 +5,18 @@
 # $KEYRING need to be set as the path for a vstart clusters Ceph keyring.
 #
 # Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw:
-# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_librgw_file.sh
+# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_awssdkv4_sig.sh
 
 if [ -z ${AWS_ACCESS_KEY_ID} ]
 then
     export AWS_ACCESS_KEY_ID=`openssl rand -base64 20`
     export AWS_SECRET_ACCESS_KEY=`openssl rand -base64 40`
 
-    radosgw-admin user create --uid ceph-test-librgw-file \
+    radosgw-admin user create --uid ceph-test-maven \
        --access-key $AWS_ACCESS_KEY_ID \
        --secret $AWS_SECRET_ACCESS_KEY \
        --display-name "maven test user" \
-       --email librgw@example.com || echo "maven user exists"
+       --email sigv4@example.com || echo "sigv4 maven user exists"
 
     # keyring override for teuthology env
     if [ -z ${KEYRING} ]
@@ -29,9 +29,14 @@ fi
 # the required S3 access_key and secret_key are already exported above, but
 # we need to hook up the S3 endpoints
 
-# the following are taken from 
+# the following are taken from
+
+# XXXX ok, so I think there should be only RGW_HTTP_ENDPOINT_URL and that
+# it may already be set before this script runs, though if we can figure it
+# out, it's ok if we set it...
+
 export RGW_HTTP_ENDPOINT_URL="http://localhost:80"
-export RGW_HTTPS_ENDPOINT_URL="https://localhost:443"
+#export RGW_HTTPS_ENDPOINT_URL="https://localhost:443"
 
 # rgw/test_awssdkv4_sig.sh
 pushd jcksum

From 613575719191ce55bc8997577965f3a03e22453c Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 14 Feb 2024 00:38:46 +0000
Subject: [PATCH 1892/2492] osd: always send returnvec-on-errors for client's
 retry

Currently there is a discrepancy in terms of the returnvec's
presence between MOSDOpReplys sent for original requests and
those on dups. The former always contain the returnvec if
an error happened, even if `allows_returnvec()` is `false`.

This commit extends the behavior on dups.

For RCA please see: https://tracker.ceph.com/issues/64192#note-9

Fixes: https://tracker.ceph.com/issues/64192
Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 src/osd/PrimaryLogPG.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index ca83a563e4f8..3a49b0ffafd1 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -4288,8 +4288,11 @@ void PrimaryLogPG::execute_ctx(OpContext *ctx)
     }
     reply->add_flags(CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK);
     // append to pg log for dup detection - don't save buffers for now
-    record_write_error(op, soid, reply, result,
-		       ctx->op->allows_returnvec() ? ctx : nullptr);
+    // store op's returnvec unconditionally-on-errors to ensure coherency
+    // with the original request handling (see `ignore_out_data` above).
+    record_write_error(
+      op, soid, reply, result,
+      (ctx->op->allows_returnvec() || result < 0) ? ctx : nullptr);
     close_op_ctx(ctx);
     return;
   }

From 3f067b480dcb9f783907ab938bf0f83b5fba5af2 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Mon, 12 Feb 2024 11:41:32 +0530
Subject: [PATCH 1893/2492] mgr/dashboard: update nvmeof protoc to 1.0.0

Fixes: https://tracker.ceph.com/issues/64384
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/nvmeof.py       |   54 +-
 src/pybind/mgr/dashboard/openapi.yaml         |  108 +-
 .../mgr/dashboard/services/nvmeof_client.py   |   55 +-
 .../dashboard/services/proto/gateway.proto    |  137 ++-
 .../dashboard/services/proto/gateway_pb2.py   | 1078 +++++++++++------
 .../services/proto/gateway_pb2_grpc.py        |  997 ++++++++-------
 6 files changed, 1420 insertions(+), 1009 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
index 125b93dc3cc7..6184f6a4166d 100644
--- a/src/pybind/mgr/dashboard/controllers/nvmeof.py
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -27,8 +27,7 @@ def list(self, subsystem_nqn: str):
         @CreatePermission
         def create(self, rbd_pool: str, rbd_image: str, subsystem_nqn: str,
                    create_image: Optional[bool] = True, image_size: Optional[int] = 1024,
-                   block_size: int = 512, nsid: Optional[int] = 1,
-                   uuid: Optional[str] = None, anagrpid: Optional[int] = 1):
+                   block_size: int = 512):
             """
             Create a new NVMeoF namespace
             :param rbd_pool: RBD pool name
@@ -37,24 +36,20 @@ def create(self, rbd_pool: str, rbd_image: str, subsystem_nqn: str,
             :param create_image: Create RBD image
             :param image_size: RBD image size
             :param block_size: NVMeoF namespace block size
-            :param nsid: NVMeoF namespace ID
-            :param uuid: NVMeoF namespace UUID
-            :param anagrpid: NVMeoF namespace ANA group ID
             """
             response = NVMeoFClient().create_namespace(rbd_pool, rbd_image,
                                                        subsystem_nqn, block_size,
-                                                       nsid, uuid, anagrpid,
                                                        create_image, image_size)
             return json.loads(MessageToJson(response))
 
         @Endpoint('DELETE', path='{subsystem_nqn}')
-        def delete(self, subsystem_nqn: str, nsid: int):
+        @DeletePermission
+        def delete(self, subsystem_nqn: str):
             """
             Delete an existing NVMeoF namespace
             :param subsystem_nqn: NVMeoF subsystem NQN
-            :param nsid: NVMeoF namespace ID
             """
-            response = NVMeoFClient().delete_namespace(subsystem_nqn, nsid)
+            response = NVMeoFClient().delete_namespace(subsystem_nqn)
             return json.loads(MessageToJson(response))
 
     @APIRouter('/nvmeof/subsystem', Scope.NVME_OF)
@@ -64,39 +59,36 @@ class NvmeofSubsystem(RESTController):
         @EndpointDoc("List all NVMeoF gateways",
                      parameters={
                          'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'serial_number': (str, 'NVMeoF subsystem serial number')
                      })
-        def list(self, subsystem_nqn: Optional[str] = None, serial_number: Optional[str] = None):
+        @ReadPermission
+        def list(self, subsystem_nqn: Optional[str] = None):
             response = MessageToJson(NVMeoFClient().list_subsystems(
-                subsystem_nqn=subsystem_nqn, serial_number=serial_number))
+                subsystem_nqn=subsystem_nqn))
 
             return json.loads(response)
 
         @CreatePermission
         def create(self, subsystem_nqn: str, serial_number: Optional[str] = None,
-                   max_namespaces: Optional[int] = 256, ana_reporting: Optional[bool] = False,
-                   enable_ha: Optional[bool] = False):
+                   max_namespaces: Optional[int] = 256):
             """
             Create a new NVMeoF subsystem
 
             :param subsystem_nqn: NVMeoF subsystem NQN
             :param serial_number: NVMeoF subsystem serial number
             :param max_namespaces: NVMeoF subsystem maximum namespaces
-            :param ana_reporting: NVMeoF subsystem ANA reporting
-            :param enable_ha: NVMeoF subsystem enable HA
             """
-            response = NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces,
-                                                       ana_reporting, enable_ha)
+            response = NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces)
             return json.loads(MessageToJson(response))
 
         @DeletePermission
         @Endpoint('DELETE', path='{subsystem_nqn}')
-        def delete(self, subsystem_nqn: str):
+        def delete(self, subsystem_nqn: str, force: Optional[bool] = False):
             """
             Delete an existing NVMeoF subsystem
             :param subsystem_nqn: NVMeoF subsystem NQN
+            :param force: Force delete
             """
-            response = NVMeoFClient().delete_subsystem(subsystem_nqn)
+            response = NVMeoFClient().delete_subsystem(subsystem_nqn, force)
             return json.loads(MessageToJson(response))
 
     @APIRouter('/nvmeof/hosts', Scope.NVME_OF)
@@ -144,39 +136,25 @@ def list(self, subsystem_nqn: str):
             return json.loads(response)
 
         @CreatePermission
-        def create(self, nqn: str, gateway: str, traddr: Optional[str] = None,
-                   trtype: Optional[str] = 'TCP', adrfam: Optional[str] = 'IPV4',
-                   trsvcid: Optional[int] = 4420,
-                   auto_ha_state: Optional[str] = 'AUTO_HA_UNSET'):
+        def create(self, nqn: str, gateway: str, traddr: Optional[str] = None):
             """
             Create a new NVMeoF listener
             :param nqn: NVMeoF subsystem NQN
             :param gateway: NVMeoF gateway
             :param traddr: NVMeoF transport address
-            :param trtype: NVMeoF transport type
-            :param adrfam: NVMeoF address family
-            :param trsvcid: NVMeoF transport service ID
-            :param auto_ha_state: NVMeoF auto HA state
             """
-            response = NVMeoFClient().create_listener(nqn, gateway, traddr,
-                                                      trtype, adrfam, trsvcid, auto_ha_state)
+            response = NVMeoFClient().create_listener(nqn, gateway, traddr)
             return json.loads(MessageToJson(response))
 
         @DeletePermission
-        def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None,
-                   transport_type: Optional[str] = 'TCP', addr_family: Optional[str] = 'IPV4',
-                   transport_svc_id: Optional[int] = 4420):
+        def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None):
             """
             Delete an existing NVMeoF listener
             :param nqn: NVMeoF subsystem NQN
             :param gateway: NVMeoF gateway
             :param traddr: NVMeoF transport address
-            :param transport_type: NVMeoF transport type
-            :param addr_family: NVMeoF address family
-            :param transport_svc_id: NVMeoF transport service ID
             """
-            response = NVMeoFClient().delete_listener(nqn, gateway, traddr, transport_type,
-                                                      addr_family, transport_svc_id)
+            response = NVMeoFClient().delete_listener(nqn, gateway, traddr)
             return json.loads(MessageToJson(response))
 
     @APIRouter('/nvmeof/gateway', Scope.NVME_OF)
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 3f6e22c57c5e..9b121ca079af 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -7806,34 +7806,19 @@ paths:
     post:
       description: "\n            Create a new NVMeoF listener\n            :param\
         \ nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
-        \            :param traddr: NVMeoF transport address\n            :param trtype:\
-        \ NVMeoF transport type\n            :param adrfam: NVMeoF address family\n\
-        \            :param trsvcid: NVMeoF transport service ID\n            :param\
-        \ auto_ha_state: NVMeoF auto HA state\n            "
+        \            :param traddr: NVMeoF transport address\n            "
       parameters: []
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                adrfam:
-                  default: IPV4
-                  type: string
-                auto_ha_state:
-                  default: AUTO_HA_UNSET
-                  type: string
                 gateway:
                   type: string
                 nqn:
                   type: string
                 traddr:
                   type: string
-                trsvcid:
-                  default: 4420
-                  type: integer
-                trtype:
-                  default: TCP
-                  type: string
               required:
               - nqn
               - gateway
@@ -7866,10 +7851,7 @@ paths:
     delete:
       description: "\n            Delete an existing NVMeoF listener\n           \
         \ :param nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
-        \            :param traddr: NVMeoF transport address\n            :param transport_type:\
-        \ NVMeoF transport type\n            :param addr_family: NVMeoF address family\n\
-        \            :param transport_svc_id: NVMeoF transport service ID\n      \
-        \      "
+        \            :param traddr: NVMeoF transport address\n            "
       parameters:
       - in: path
         name: nqn
@@ -7886,21 +7868,6 @@ paths:
         name: traddr
         schema:
           type: string
-      - default: TCP
-        in: query
-        name: transport_type
-        schema:
-          type: string
-      - default: IPV4
-        in: query
-        name: addr_family
-        schema:
-          type: string
-      - default: 4420
-        in: query
-        name: transport_svc_id
-        schema:
-          type: integer
       responses:
         '202':
           content:
@@ -7932,18 +7899,13 @@ paths:
         \            :param subsystem_nqn: NVMeoF subsystem NQN\n            :param\
         \ create_image: Create RBD image\n            :param image_size: RBD image\
         \ size\n            :param block_size: NVMeoF namespace block size\n     \
-        \       :param nsid: NVMeoF namespace ID\n            :param uuid: NVMeoF\
-        \ namespace UUID\n            :param anagrpid: NVMeoF namespace ANA group\
-        \ ID\n            "
+        \       "
       parameters: []
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                anagrpid:
-                  default: 1
-                  type: integer
                 block_size:
                   default: 512
                   type: integer
@@ -7953,17 +7915,12 @@ paths:
                 image_size:
                   default: 1024
                   type: integer
-                nsid:
-                  default: 1
-                  type: integer
                 rbd_image:
                   type: string
                 rbd_pool:
                   type: string
                 subsystem_nqn:
                   type: string
-                uuid:
-                  type: string
               required:
               - rbd_pool
               - rbd_image
@@ -7994,8 +7951,9 @@ paths:
       tags:
       - NVMe-oF
   /api/nvmeof/namespace/{subsystem_nqn}:
-    get:
-      description: "\n            List all NVMeoF namespaces\n            "
+    delete:
+      description: "\n            Delete an existing NVMeoF namespace\n          \
+        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            "
       parameters:
       - in: path
         name: subsystem_nqn
@@ -8003,11 +7961,16 @@ paths:
         schema:
           type: string
       responses:
-        '200':
+        '202':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: OK
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
         '400':
           description: Operation exception. Please check the response body for details.
         '401':
@@ -8021,33 +7984,20 @@ paths:
       - jwt: []
       tags:
       - NVMe-oF
-  /api/nvmeof/namespace/{subsystem_nqn}/{nsid}:
-    delete:
-      description: "\n            Delete an existing NVMeoF namespace\n          \
-        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            :param nsid: NVMeoF\
-        \ namespace ID\n            "
+    get:
+      description: "\n            List all NVMeoF namespaces\n            "
       parameters:
       - in: path
         name: subsystem_nqn
         required: true
         schema:
           type: string
-      - in: path
-        name: nsid
-        required: true
-        schema:
-          type: string
       responses:
-        '202':
-          content:
-            application/vnd.ceph.api.v1.0+json:
-              type: object
-          description: Operation is still executing. Please check the task queue.
-        '204':
+        '200':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: Resource deleted.
+          description: OK
         '400':
           description: Operation exception. Please check the response body for details.
         '401':
@@ -8070,12 +8020,6 @@ paths:
         name: subsystem_nqn
         schema:
           type: string
-      - allowEmptyValue: true
-        description: NVMeoF subsystem serial number
-        in: query
-        name: serial_number
-        schema:
-          type: string
       responses:
         '200':
           content:
@@ -8100,21 +8044,13 @@ paths:
       description: "\n            Create a new NVMeoF subsystem\n\n            :param\
         \ subsystem_nqn: NVMeoF subsystem NQN\n            :param serial_number: NVMeoF\
         \ subsystem serial number\n            :param max_namespaces: NVMeoF subsystem\
-        \ maximum namespaces\n            :param ana_reporting: NVMeoF subsystem ANA\
-        \ reporting\n            :param enable_ha: NVMeoF subsystem enable HA\n  \
-        \          "
+        \ maximum namespaces\n            "
       parameters: []
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                ana_reporting:
-                  default: false
-                  type: boolean
-                enable_ha:
-                  default: false
-                  type: boolean
                 max_namespaces:
                   default: 256
                   type: integer
@@ -8152,13 +8088,19 @@ paths:
   /api/nvmeof/subsystem/{subsystem_nqn}:
     delete:
       description: "\n            Delete an existing NVMeoF subsystem\n          \
-        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            "
+        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            :param force: Force\
+        \ delete\n            "
       parameters:
       - in: path
         name: subsystem_nqn
         required: true
         schema:
           type: string
+      - default: false
+        in: query
+        name: force
+        schema:
+          type: boolean
       responses:
         '202':
           content:
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index bebaef7d9ecb..2dc0e473c613 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -1,7 +1,6 @@
 import logging
 from typing import Optional
 
-from ..tools import str_to_bool
 from .nvmeof_conf import NvmeofGatewaysConfig
 
 logger = logging.getLogger('nvmeof_client')
@@ -33,50 +32,40 @@ def list_subsystems(self, subsystem_nqn: Optional[str] = None,
                 serial_number=serial_number
             ))
 
-        def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int,
-                             ana_reporting: bool, enable_ha: bool):
+        def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int):
             return self.stub.create_subsystem(pb2.create_subsystem_req(
                 subsystem_nqn=subsystem_nqn,
                 serial_number=serial_number,
-                max_namespaces=int(max_namespaces),
-                ana_reporting=str_to_bool(ana_reporting),
-                enable_ha=str_to_bool(enable_ha)
+                max_namespaces=int(max_namespaces)
             ))
 
-        def delete_subsystem(self, subsystem_nqn: str):
+        def delete_subsystem(self, subsystem_nqn: str, force: Optional[bool] = False):
             return self.stub.delete_subsystem(pb2.delete_subsystem_req(
-                subsystem_nqn=subsystem_nqn
+                subsystem_nqn=subsystem_nqn,
+                force=force
             ))
 
-        def list_namespaces(self, subsystem_nqn: str, nsid: Optional[int] = 1,
-                            uuid: Optional[str] = None):
+        def list_namespaces(self, subsystem_nqn: str):
             return self.stub.list_namespaces(pb2.list_namespaces_req(
-                subsystem=subsystem_nqn,
-                nsid=int(nsid),
-                uuid=uuid
+                subsystem=subsystem_nqn
             ))
 
         def create_namespace(self, rbd_pool_name: str, rbd_image_name: str,
                              subsystem_nqn: str, block_size: int = 512,
-                             nsid: Optional[int] = 1, uuid: Optional[str] = None,
-                             anagrpid: Optional[int] = 1, create_image: Optional[bool] = True,
+                             create_image: Optional[bool] = True,
                              size: Optional[int] = 1024):
             return self.stub.namespace_add(pb2.namespace_add_req(
                 rbd_pool_name=rbd_pool_name,
                 rbd_image_name=rbd_image_name,
                 subsystem_nqn=subsystem_nqn,
-                nsid=int(nsid),
                 block_size=block_size,
-                uuid=uuid,
-                anagrpid=anagrpid,
                 create_image=create_image,
                 size=size
             ))
 
-        def delete_namespace(self, subsystem_nqn: str, nsid: int):
-            return self.stub.remove_namespace(pb2.remove_namespace_req(
-                subsystem_nqn=subsystem_nqn,
-                nsid=nsid
+        def delete_namespace(self, subsystem_nqn: str):
+            return self.stub.namespace_delete(pb2.namespace_delete_req(
+                subsystem_nqn=subsystem_nqn
             ))
 
         def list_hosts(self, subsystem_nqn: str):
@@ -101,11 +90,7 @@ def list_listeners(self, subsystem_nqn: str):
                 subsystem=subsystem_nqn
             ))
 
-        def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
-                            transport_type: Optional[str] = 'TCP',
-                            addr_family: Optional[str] = 'IPV4',
-                            transport_svc_id: Optional[int] = 4420,
-                            auto_ha_state: Optional[str] = 'AUTO_HA_UNSET'):
+        def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
             traddr = None
             if traddr is None:
                 addr = self.gateway_addr
@@ -115,18 +100,11 @@ def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
             req = pb2.create_listener_req(
                 nqn=nqn,
                 gateway_name=gateway,
-                traddr=traddr,
-                trtype=pb2.TransportType.Value(transport_type.upper()),
-                adrfam=pb2.AddressFamily.Value(addr_family.lower()),
-                trsvcid=transport_svc_id,
-                auto_ha_state=pb2.AutoHAState.Value(auto_ha_state.upper())
+                traddr=traddr
             )
             return self.stub.create_listener(req)
 
-        def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
-                            transport_type: Optional[str] = 'TCP',
-                            addr_family: Optional[str] = 'IPV4',
-                            transport_svc_id: Optional[int] = 4420):
+        def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
             traddr = None
             if traddr is None:
                 addr = self.gateway_addr
@@ -136,10 +114,7 @@ def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None,
             return self.stub.delete_listener(pb2.delete_listener_req(
                 nqn=nqn,
                 gateway_name=gateway,
-                traddr=traddr,
-                trtype=pb2.TransportType.Value(transport_type.upper()),
-                adrfam=pb2.AddressFamily.Value(addr_family.lower()),
-                trsvcid=int(transport_svc_id)
+                traddr=traddr
             ))
 
         def gateway_info(self):
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway.proto b/src/pybind/mgr/dashboard/services/proto/gateway.proto
index 787aefb1ce9f..81e2b7205ceb 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway.proto
+++ b/src/pybind/mgr/dashboard/services/proto/gateway.proto
@@ -10,37 +10,17 @@
 
 syntax = "proto3";
 
-enum TransportType {
-    INVALID = 0;
-    RDMA = 1;
-    FC = 2;
-    TCP = 3;
-    PCIE = 256;
-    VFIOUSER = 1024;
-    CUSTOM = 4096;
-}
-
 enum AddressFamily {
-    invalid = 0;
-    ipv4 = 1;
-    ipv6 = 2;
-    ib = 3;
-    fc = 4;
+    ipv4 = 0;
+    ipv6 = 1;
 }
 
 enum LogLevel {
-    DISABLED = 0;
-    ERROR = 1;
-    WARNING = 2;
-    NOTICE = 3;
-    INFO = 4;
-    DEBUG = 5;
-}
-
-enum AutoHAState {
-    AUTO_HA_UNSET = 0;
-    AUTO_HA_OFF = 1;
-    AUTO_HA_ON = 2;
+    ERROR = 0;
+    WARNING = 1;
+    NOTICE = 2;
+    INFO = 3;
+    DEBUG = 4;
 }
 
 service Gateway {
@@ -93,7 +73,13 @@ service Gateway {
 	rpc list_listeners(list_listeners_req) returns(listeners_info) {}
 
 	// List subsystems
-	rpc list_subsystems(list_subsystems_req) returns(subsystems_info) {}
+	rpc list_subsystems(list_subsystems_req) returns(subsystems_info_cli) {}
+
+        // Gets subsystems
+        rpc get_subsystems(get_subsystems_req) returns(subsystems_info) {}
+
+	// Set gateway ANA states
+	rpc set_ana_state(ana_info) returns(req_status) {}
 
 	// Gets spdk nvmf log flags and level
 	rpc get_spdk_nvmf_log_flags_and_level(get_spdk_nvmf_log_flags_and_level_req) returns(spdk_nvmf_log_flags_and_level_info) {}
@@ -120,6 +106,7 @@ message namespace_add_req {
 	optional int32 anagrpid = 7;
 	optional bool create_image = 8;
 	optional uint32 size = 9;
+	optional bool force = 10;
 }
 
 message namespace_resize_req {
@@ -162,8 +149,7 @@ message create_subsystem_req {
 	string subsystem_nqn = 1;
 	string serial_number = 2;
 	optional uint32 max_namespaces = 3;
-	bool ana_reporting = 4;
-	bool enable_ha = 5;
+	bool enable_ha = 4;
 }
 
 message delete_subsystem_req {
@@ -199,17 +185,14 @@ message create_listener_req {
 	string nqn = 1;
 	string gateway_name = 2;
 	string traddr = 3;
-	optional TransportType trtype = 4;
 	optional AddressFamily adrfam = 5;
 	optional uint32 trsvcid = 6;
-	optional AutoHAState auto_ha_state = 7;
 }
 
 message delete_listener_req {
 	string nqn = 1;
 	string gateway_name = 2;
 	string traddr = 3;
-	optional TransportType trtype = 4;
 	optional AddressFamily adrfam = 5;
 	optional uint32 trsvcid = 6;
 }
@@ -223,6 +206,9 @@ message list_subsystems_req {
 	optional string serial_number = 2;
 }
 
+message get_subsystems_req {
+}
+
 message get_spdk_nvmf_log_flags_and_level_req {
 }
 
@@ -238,14 +224,39 @@ message get_gateway_info_req {
 	optional string cli_version = 1;
 }
 
-// Return messages 
+// From https://nvmexpress.org/wp-content/uploads/NVM-Express-1_4-2019.06.10-Ratified.pdf page 138
+// Asymmetric Namespace Access state for all namespaces in this ANA
+// Group when accessed through this controller.
+// Value Description Reference
+// 01h ANA Optimized state 8.20.3.1
+// 02h ANA Non-Optimized state 8.20.3.2
+// 03h ANA Inaccessible state 8.20.3.3
+// 04h ANA Persistent Loss state 8.20.3.4
+// 0Fh ANA Change state 8.20.3.5
+// All others Reserved
+enum ana_state {
+  UNSET         = 0;
+  OPTIMIZED     = 1;
+  NON_OPTIMIZED = 2;
+  INACCESSIBLE  = 3;
+}
 
-message bdev_status {
-	int32 status = 1;
-	string error_message = 2;
-	string bdev_name = 3;
+message ana_group_state {
+	uint32     grp_id = 1;  // groupd id
+	ana_state  state  = 2;  // ANA state
+}
+
+message nqn_ana_states {
+	string     nqn                     = 1; // subsystem nqn
+	repeated   ana_group_state  states = 2; // list of group states
 }
 
+message ana_info {
+	repeated nqn_ana_states states = 1; // list of nqn states
+}
+
+// Return messages 
+
 message req_status {
 	int32 status = 1;
 	string error_message = 2;
@@ -258,12 +269,48 @@ message nsid_status {
 }
 
 message subsystems_info {
+        repeated subsystem subsystems = 1;
+}
+
+message subsystem {
+        string nqn = 1;
+        string subtype = 2;
+        repeated listen_address listen_addresses = 3;
+        repeated host hosts = 4;
+        bool allow_any_host = 5;
+        optional string serial_number = 6;
+        optional string model_number = 7;
+        optional uint32 max_namespaces = 8;
+        optional uint32 min_cntlid = 9;
+        optional uint32 max_cntlid = 10;
+        repeated namespace namespaces = 11;
+}
+
+message listen_address {
+        string trtype = 1;
+        string adrfam = 2;
+        string traddr = 3;
+        string trsvcid = 4;
+        optional string transport = 5;
+}
+
+message namespace {
+    uint32 nsid = 1;
+    string name = 2;
+    optional string bdev_name = 3;
+    optional string nguid = 4;
+    optional string uuid = 5;
+    optional uint32 anagrpid = 6;
+    optional string nonce = 7;
+}
+
+message subsystems_info_cli {
 	int32 status = 1;
 	string error_message = 2;
-	repeated subsystem subsystems = 3;
+	repeated subsystem_cli subsystems = 3;
 }
 
-message subsystem {
+message subsystem_cli {
 	string nqn = 1;
 	bool enable_ha = 2;
 	string serial_number = 3;
@@ -284,7 +331,7 @@ message gateway_info {
 	bool bool_status = 7;
 	int32 status = 8;
 	string error_message = 9;
-	string spdk_version = 10;
+	optional string spdk_version = 10;
 }
 
 message cli_version {
@@ -301,7 +348,7 @@ message gw_version {
 
 message listener_info {
 	string gateway_name = 1;
-	TransportType trtype = 2;
+	string trtype = 2;
 	AddressFamily adrfam = 3;
 	string traddr = 4;
 	uint32 trsvcid = 5;
@@ -329,7 +376,7 @@ message connection {
 	string nqn = 1;
 	string traddr = 2;
 	uint32 trsvcid = 3;
-	TransportType trtype = 4;
+        string trtype = 4;
 	AddressFamily adrfam = 5;
 	bool connected = 6;
 	int32 qpairs_count = 7;
@@ -343,7 +390,7 @@ message connections_info {
 	repeated connection connections = 4;
 }
 
-message namespace {
+message namespace_cli {
 	uint32 nsid = 1;
 	string bdev_name = 2;
 	string rbd_image_name = 3;
@@ -362,7 +409,7 @@ message namespaces_info {
 	int32 status = 1;
 	string error_message = 2;
 	string subsystem_nqn = 3;
-	repeated namespace namespaces = 4;
+	repeated namespace_cli namespaces = 4;
 }
 
 message namespace_io_stats_info {
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
index 00349b5256cd..6bd40aaee473 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
@@ -19,60 +19,9 @@
     syntax='proto3',
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
-    serialized_pb=b'\n\rgateway.proto\"\x91\x02\n\x11namespace_add_req\x12\x15\n\rrbd_pool_name\x18\x01 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x11\n\x04nsid\x18\x04 \x01(\rH\x00\x88\x01\x01\x12\x12\n\nblock_size\x18\x05 \x01(\r\x12\x11\n\x04uuid\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x07 \x01(\x05H\x02\x88\x01\x01\x12\x19\n\x0c\x63reate_image\x18\x08 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04size\x18\t \x01(\rH\x04\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x0f\n\r_create_imageB\x07\n\x05_size\"w\n\x14namespace_resize_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08new_size\x18\x04 \x01(\rB\x07\n\x05_nsidB\x07\n\x05_uuid\"k\n\x1anamespace_get_io_stats_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\xcc\x02\n\x15namespace_set_qos_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x1e\n\x11rw_ios_per_second\x18\x04 \x01(\x04H\x02\x88\x01\x01\x12!\n\x14rw_mbytes_per_second\x18\x05 \x01(\x04H\x03\x88\x01\x01\x12 \n\x13r_mbytes_per_second\x18\x06 \x01(\x04H\x04\x88\x01\x01\x12 \n\x13w_mbytes_per_second\x18\x07 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x14\n\x12_rw_ios_per_secondB\x17\n\x15_rw_mbytes_per_secondB\x16\n\x14_r_mbytes_per_secondB\x16\n\x14_w_mbytes_per_second\"\x8c\x01\n)namespace_change_load_balancing_group_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08\x61nagrpid\x18\x04 \x01(\x05\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"e\n\x14namespace_delete_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\x9e\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x1b\n\x0emax_namespaces\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x15\n\rana_reporting\x18\x04 \x01(\x08\x12\x11\n\tenable_ha\x18\x05 \x01(\x08\x42\x11\n\x0f_max_namespaces\"K\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x12\n\x05\x66orce\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x08\n\x06_force\"`\n\x13list_namespaces_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"#\n\x0elist_hosts_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\")\n\x14list_connections_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"\x86\x02\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportTypeH\x00\x88\x01\x01\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x01\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x02\x88\x01\x01\x12(\n\rauto_ha_state\x18\x07 \x01(\x0e\x32\x0c.AutoHAStateH\x03\x88\x01\x01\x42\t\n\x07_trtypeB\t\n\x07_adrfamB\n\n\x08_trsvcidB\x10\n\x0e_auto_ha_state\"\xca\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportTypeH\x00\x88\x01\x01\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x01\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x02\x88\x01\x01\x42\t\n\x07_trtypeB\t\n\x07_adrfamB\n\n\x08_trsvcid\"\'\n\x12list_listeners_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"q\n\x13list_subsystems_req\x12\x1a\n\rsubsystem_nqn\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rserial_number\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x10\n\x0e_subsystem_nqnB\x10\n\x0e_serial_number\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"@\n\x14get_gateway_info_req\x12\x18\n\x0b\x63li_version\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_cli_version\"G\n\x0b\x62\x64\x65v_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x11\n\tbdev_name\x18\x03 \x01(\t\"3\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\"B\n\x0bnsid_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0c\n\x04nsid\x18\x03 \x01(\r\"X\n\x0fsubsystems_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x1e\n\nsubsystems\x18\x03 \x03(\x0b\x32\n.subsystem\"\xaa\x01\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x11\n\tenable_ha\x18\x02 \x01(\x08\x12\x15\n\rserial_number\x18\x03 \x01(\t\x12\x14\n\x0cmodel_number\x18\x04 \x01(\t\x12\x12\n\nmin_cntlid\x18\x05 \x01(\r\x12\x12\n\nmax_cntlid\x18\x06 \x01(\r\x12\x17\n\x0fnamespace_count\x18\x07 \x01(\r\x12\x0f\n\x07subtype\x18\x08 \x01(\t\"\xbf\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05group\x18\x04 \x01(\t\x12\x0c\n\x04\x61\x64\x64r\x18\x05 \x01(\t\x12\x0c\n\x04port\x18\x06 \x01(\t\x12\x13\n\x0b\x62ool_status\x18\x07 \x01(\x08\x12\x0e\n\x06status\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\x12\x14\n\x0cspdk_version\x18\n \x01(\t\"E\n\x0b\x63li_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"D\n\ngw_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"\x86\x01\n\rlistener_info\x12\x14\n\x0cgateway_name\x18\x01 \x01(\t\x12\x1e\n\x06trtype\x18\x02 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\r\"Z\n\x0elisteners_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12!\n\tlisteners\x18\x03 \x03(\x0b\x32\x0e.listener_info\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"x\n\nhosts_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x16\n\x0e\x61llow_any_host\x18\x03 \x01(\x08\x12\x15\n\rsubsystem_nqn\x18\x04 \x01(\t\x12\x14\n\x05hosts\x18\x05 \x03(\x0b\x32\x05.host\"\xba\x01\n\nconnection\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0e\n\x06traddr\x18\x02 \x01(\t\x12\x0f\n\x07trsvcid\x18\x03 \x01(\r\x12\x1e\n\x06trtype\x18\x04 \x01(\x0e\x32\x0e.TransportType\x12\x1e\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamily\x12\x11\n\tconnected\x18\x06 \x01(\x08\x12\x14\n\x0cqpairs_count\x18\x07 \x01(\x05\x12\x15\n\rcontroller_id\x18\x08 \x01(\x05\"r\n\x10\x63onnections_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12 \n\x0b\x63onnections\x18\x04 \x03(\x0b\x32\x0b.connection\"\xa6\x02\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x04 \x01(\t\x12\x1c\n\x14load_balancing_group\x18\x05 \x01(\r\x12\x12\n\nblock_size\x18\x06 \x01(\r\x12\x16\n\x0erbd_image_size\x18\x07 \x01(\x04\x12\x0c\n\x04uuid\x18\x08 \x01(\t\x12\x19\n\x11rw_ios_per_second\x18\t \x01(\x04\x12\x1c\n\x14rw_mbytes_per_second\x18\n \x01(\x04\x12\x1b\n\x13r_mbytes_per_second\x18\x0b \x01(\x04\x12\x1b\n\x13w_mbytes_per_second\x18\x0c \x01(\x04\"o\n\x0fnamespaces_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x1e\n\nnamespaces\x18\x04 \x03(\x0b\x32\n.namespace\"\xb7\x05\n\x17namespace_io_stats_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x0c\n\x04nsid\x18\x04 \x01(\r\x12\x0c\n\x04uuid\x18\x05 \x01(\t\x12\x11\n\tbdev_name\x18\x06 \x01(\t\x12\x11\n\ttick_rate\x18\x07 \x01(\x04\x12\r\n\x05ticks\x18\x08 \x01(\x04\x12\x12\n\nbytes_read\x18\t \x01(\x04\x12\x14\n\x0cnum_read_ops\x18\n \x01(\x04\x12\x15\n\rbytes_written\x18\x0b \x01(\x04\x12\x15\n\rnum_write_ops\x18\x0c \x01(\x04\x12\x16\n\x0e\x62ytes_unmapped\x18\r \x01(\x04\x12\x15\n\rnum_unmap_ops\x18\x0e \x01(\x04\x12\x1a\n\x12read_latency_ticks\x18\x0f \x01(\x04\x12\x1e\n\x16max_read_latency_ticks\x18\x10 \x01(\x04\x12\x1e\n\x16min_read_latency_ticks\x18\x11 \x01(\x04\x12\x1b\n\x13write_latency_ticks\x18\x12 \x01(\x04\x12\x1f\n\x17max_write_latency_ticks\x18\x13 \x01(\x04\x12\x1f\n\x17min_write_latency_ticks\x18\x14 \x01(\x04\x12\x1b\n\x13unmap_latency_ticks\x18\x15 \x01(\x04\x12\x1f\n\x17max_unmap_latency_ticks\x18\x16 \x01(\x04\x12\x1f\n\x17min_unmap_latency_ticks\x18\x17 \x01(\x04\x12\x1a\n\x12\x63opy_latency_ticks\x18\x18 \x01(\x04\x12\x1e\n\x16max_copy_latency_ticks\x18\x19 \x01(\x04\x12\x1e\n\x16min_copy_latency_ticks\x18\x1a \x01(\x04\x12\x10\n\x08io_error\x18\x1b \x03(\r\"3\n\x12spdk_log_flag_info\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x02 \x01(\x08\"\xba\x01\n\"spdk_nvmf_log_flags_and_level_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12+\n\x0envmf_log_flags\x18\x03 \x03(\x0b\x32\x13.spdk_log_flag_info\x12\x1c\n\tlog_level\x18\x04 \x01(\x0e\x32\t.LogLevel\x12\"\n\x0flog_print_level\x18\x05 \x01(\x0e\x32\t.LogLevel*^\n\rTransportType\x12\x0b\n\x07INVALID\x10\x00\x12\x08\n\x04RDMA\x10\x01\x12\x06\n\x02\x46\x43\x10\x02\x12\x07\n\x03TCP\x10\x03\x12\t\n\x04PCIE\x10\x80\x02\x12\r\n\x08VFIOUSER\x10\x80\x08\x12\x0b\n\x06\x43USTOM\x10\x80 *@\n\rAddressFamily\x12\x0b\n\x07invalid\x10\x00\x12\x08\n\x04ipv4\x10\x01\x12\x08\n\x04ipv6\x10\x02\x12\x06\n\x02ib\x10\x03\x12\x06\n\x02\x66\x63\x10\x04*Q\n\x08LogLevel\x12\x0c\n\x08\x44ISABLED\x10\x00\x12\t\n\x05\x45RROR\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\n\n\x06NOTICE\x10\x03\x12\x08\n\x04INFO\x10\x04\x12\t\n\x05\x44\x45\x42UG\x10\x05*A\n\x0b\x41utoHAState\x12\x11\n\rAUTO_HA_UNSET\x10\x00\x12\x0f\n\x0b\x41UTO_HA_OFF\x10\x01\x12\x0e\n\nAUTO_HA_ON\x10\x02\x32\xc0\n\n\x07Gateway\x12\x33\n\rnamespace_add\x12\x12.namespace_add_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12;\n\x0flist_namespaces\x12\x14.list_namespaces_req\x1a\x10.namespaces_info\"\x00\x12\x38\n\x10namespace_resize\x12\x15.namespace_resize_req\x1a\x0b.req_status\"\x00\x12Q\n\x16namespace_get_io_stats\x12\x1b.namespace_get_io_stats_req\x1a\x18.namespace_io_stats_info\"\x00\x12\x41\n\x18namespace_set_qos_limits\x12\x16.namespace_set_qos_req\x1a\x0b.req_status\"\x00\x12\x62\n%namespace_change_load_balancing_group\x12*.namespace_change_load_balancing_group_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10namespace_delete\x12\x15.namespace_delete_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12,\n\nlist_hosts\x12\x0f.list_hosts_req\x1a\x0b.hosts_info\"\x00\x12>\n\x10list_connections\x12\x15.list_connections_req\x1a\x11.connections_info\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x38\n\x0elist_listeners\x12\x13.list_listeners_req\x1a\x0f.listeners_info\"\x00\x12;\n\x0flist_subsystems\x12\x14.list_subsystems_req\x1a\x10.subsystems_info\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3'
+    serialized_pb=b'\n\rgateway.proto\"\x91\x02\n\x11namespace_add_req\x12\x15\n\rrbd_pool_name\x18\x01 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x11\n\x04nsid\x18\x04 \x01(\rH\x00\x88\x01\x01\x12\x12\n\nblock_size\x18\x05 \x01(\r\x12\x11\n\x04uuid\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x07 \x01(\x05H\x02\x88\x01\x01\x12\x19\n\x0c\x63reate_image\x18\x08 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04size\x18\t \x01(\rH\x04\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x0f\n\r_create_imageB\x07\n\x05_size\"w\n\x14namespace_resize_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08new_size\x18\x04 \x01(\rB\x07\n\x05_nsidB\x07\n\x05_uuid\"k\n\x1anamespace_get_io_stats_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\xcc\x02\n\x15namespace_set_qos_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x1e\n\x11rw_ios_per_second\x18\x04 \x01(\x04H\x02\x88\x01\x01\x12!\n\x14rw_mbytes_per_second\x18\x05 \x01(\x04H\x03\x88\x01\x01\x12 \n\x13r_mbytes_per_second\x18\x06 \x01(\x04H\x04\x88\x01\x01\x12 \n\x13w_mbytes_per_second\x18\x07 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x14\n\x12_rw_ios_per_secondB\x17\n\x15_rw_mbytes_per_secondB\x16\n\x14_r_mbytes_per_secondB\x16\n\x14_w_mbytes_per_second\"\x8c\x01\n)namespace_change_load_balancing_group_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08\x61nagrpid\x18\x04 \x01(\x05\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"e\n\x14namespace_delete_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\x87\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x1b\n\x0emax_namespaces\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x11\n\tenable_ha\x18\x04 \x01(\x08\x42\x11\n\x0f_max_namespaces\"K\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x12\n\x05\x66orce\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x08\n\x06_force\"`\n\x13list_namespaces_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"#\n\x0elist_hosts_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\")\n\x14list_connections_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"\x9a\x01\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\x9a\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\'\n\x12list_listeners_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"q\n\x13list_subsystems_req\x12\x1a\n\rsubsystem_nqn\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rserial_number\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x10\n\x0e_subsystem_nqnB\x10\n\x0e_serial_number\"\x14\n\x12get_subsystems_req\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"@\n\x14get_gateway_info_req\x12\x18\n\x0b\x63li_version\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_cli_version\"<\n\x0f\x61na_group_state\x12\x0e\n\x06grp_id\x18\x01 \x01(\r\x12\x19\n\x05state\x18\x02 \x01(\x0e\x32\n.ana_state\"?\n\x0enqn_ana_states\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12 \n\x06states\x18\x02 \x03(\x0b\x32\x10.ana_group_state\"+\n\x08\x61na_info\x12\x1f\n\x06states\x18\x01 \x03(\x0b\x32\x0f.nqn_ana_states\"3\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\"B\n\x0bnsid_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0c\n\x04nsid\x18\x03 \x01(\r\"1\n\x0fsubsystems_info\x12\x1e\n\nsubsystems\x18\x01 \x03(\x0b\x32\n.subsystem\"\xfc\x02\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0f\n\x07subtype\x18\x02 \x01(\t\x12)\n\x10listen_addresses\x18\x03 \x03(\x0b\x32\x0f.listen_address\x12\x14\n\x05hosts\x18\x04 \x03(\x0b\x32\x05.host\x12\x16\n\x0e\x61llow_any_host\x18\x05 \x01(\x08\x12\x1a\n\rserial_number\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0cmodel_number\x18\x07 \x01(\tH\x01\x88\x01\x01\x12\x1b\n\x0emax_namespaces\x18\x08 \x01(\rH\x02\x88\x01\x01\x12\x17\n\nmin_cntlid\x18\t \x01(\rH\x03\x88\x01\x01\x12\x17\n\nmax_cntlid\x18\n \x01(\rH\x04\x88\x01\x01\x12\x1e\n\nnamespaces\x18\x0b \x03(\x0b\x32\n.namespaceB\x10\n\x0e_serial_numberB\x0f\n\r_model_numberB\x11\n\x0f_max_namespacesB\r\n\x0b_min_cntlidB\r\n\x0b_max_cntlid\"w\n\x0elisten_address\x12\x0e\n\x06trtype\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x64rfam\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12\x0f\n\x07trsvcid\x18\x04 \x01(\t\x12\x16\n\ttransport\x18\x05 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_transport\"\xc9\x01\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\tbdev_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05nguid\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x12\n\x05nonce\x18\x07 \x01(\tH\x04\x88\x01\x01\x42\x0c\n\n_bdev_nameB\x08\n\x06_nguidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x08\n\x06_nonce\"`\n\x13subsystems_info_cli\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\"\n\nsubsystems\x18\x03 \x03(\x0b\x32\x0e.subsystem_cli\"\xae\x01\n\rsubsystem_cli\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x11\n\tenable_ha\x18\x02 \x01(\x08\x12\x15\n\rserial_number\x18\x03 \x01(\t\x12\x14\n\x0cmodel_number\x18\x04 \x01(\t\x12\x12\n\nmin_cntlid\x18\x05 \x01(\r\x12\x12\n\nmax_cntlid\x18\x06 \x01(\r\x12\x17\n\x0fnamespace_count\x18\x07 \x01(\r\x12\x0f\n\x07subtype\x18\x08 \x01(\t\"\xd5\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05group\x18\x04 \x01(\t\x12\x0c\n\x04\x61\x64\x64r\x18\x05 \x01(\t\x12\x0c\n\x04port\x18\x06 \x01(\t\x12\x13\n\x0b\x62ool_status\x18\x07 \x01(\x08\x12\x0e\n\x06status\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\x12\x19\n\x0cspdk_version\x18\n \x01(\tH\x00\x88\x01\x01\x42\x0f\n\r_spdk_version\"E\n\x0b\x63li_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"D\n\ngw_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"v\n\rlistener_info\x12\x14\n\x0cgateway_name\x18\x01 \x01(\t\x12\x0e\n\x06trtype\x18\x02 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\r\"Z\n\x0elisteners_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12!\n\tlisteners\x18\x03 \x03(\x0b\x32\x0e.listener_info\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"x\n\nhosts_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x16\n\x0e\x61llow_any_host\x18\x03 \x01(\x08\x12\x15\n\rsubsystem_nqn\x18\x04 \x01(\t\x12\x14\n\x05hosts\x18\x05 \x03(\x0b\x32\x05.host\"\xaa\x01\n\nconnection\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0e\n\x06traddr\x18\x02 \x01(\t\x12\x0f\n\x07trsvcid\x18\x03 \x01(\r\x12\x0e\n\x06trtype\x18\x04 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamily\x12\x11\n\tconnected\x18\x06 \x01(\x08\x12\x14\n\x0cqpairs_count\x18\x07 \x01(\x05\x12\x15\n\rcontroller_id\x18\x08 \x01(\x05\"r\n\x10\x63onnections_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12 \n\x0b\x63onnections\x18\x04 \x03(\x0b\x32\x0b.connection\"\xaa\x02\n\rnamespace_cli\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x04 \x01(\t\x12\x1c\n\x14load_balancing_group\x18\x05 \x01(\r\x12\x12\n\nblock_size\x18\x06 \x01(\r\x12\x16\n\x0erbd_image_size\x18\x07 \x01(\x04\x12\x0c\n\x04uuid\x18\x08 \x01(\t\x12\x19\n\x11rw_ios_per_second\x18\t \x01(\x04\x12\x1c\n\x14rw_mbytes_per_second\x18\n \x01(\x04\x12\x1b\n\x13r_mbytes_per_second\x18\x0b \x01(\x04\x12\x1b\n\x13w_mbytes_per_second\x18\x0c \x01(\x04\"s\n\x0fnamespaces_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\"\n\nnamespaces\x18\x04 \x03(\x0b\x32\x0e.namespace_cli\"\xb7\x05\n\x17namespace_io_stats_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x0c\n\x04nsid\x18\x04 \x01(\r\x12\x0c\n\x04uuid\x18\x05 \x01(\t\x12\x11\n\tbdev_name\x18\x06 \x01(\t\x12\x11\n\ttick_rate\x18\x07 \x01(\x04\x12\r\n\x05ticks\x18\x08 \x01(\x04\x12\x12\n\nbytes_read\x18\t \x01(\x04\x12\x14\n\x0cnum_read_ops\x18\n \x01(\x04\x12\x15\n\rbytes_written\x18\x0b \x01(\x04\x12\x15\n\rnum_write_ops\x18\x0c \x01(\x04\x12\x16\n\x0e\x62ytes_unmapped\x18\r \x01(\x04\x12\x15\n\rnum_unmap_ops\x18\x0e \x01(\x04\x12\x1a\n\x12read_latency_ticks\x18\x0f \x01(\x04\x12\x1e\n\x16max_read_latency_ticks\x18\x10 \x01(\x04\x12\x1e\n\x16min_read_latency_ticks\x18\x11 \x01(\x04\x12\x1b\n\x13write_latency_ticks\x18\x12 \x01(\x04\x12\x1f\n\x17max_write_latency_ticks\x18\x13 \x01(\x04\x12\x1f\n\x17min_write_latency_ticks\x18\x14 \x01(\x04\x12\x1b\n\x13unmap_latency_ticks\x18\x15 \x01(\x04\x12\x1f\n\x17max_unmap_latency_ticks\x18\x16 \x01(\x04\x12\x1f\n\x17min_unmap_latency_ticks\x18\x17 \x01(\x04\x12\x1a\n\x12\x63opy_latency_ticks\x18\x18 \x01(\x04\x12\x1e\n\x16max_copy_latency_ticks\x18\x19 \x01(\x04\x12\x1e\n\x16min_copy_latency_ticks\x18\x1a \x01(\x04\x12\x10\n\x08io_error\x18\x1b \x03(\r\"3\n\x12spdk_log_flag_info\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x02 \x01(\x08\"\xba\x01\n\"spdk_nvmf_log_flags_and_level_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12+\n\x0envmf_log_flags\x18\x03 \x03(\x0b\x32\x13.spdk_log_flag_info\x12\x1c\n\tlog_level\x18\x04 \x01(\x0e\x32\t.LogLevel\x12\"\n\x0flog_print_level\x18\x05 \x01(\x0e\x32\t.LogLevel*#\n\rAddressFamily\x12\x08\n\x04ipv4\x10\x00\x12\x08\n\x04ipv6\x10\x01*C\n\x08LogLevel\x12\t\n\x05\x45RROR\x10\x00\x12\x0b\n\x07WARNING\x10\x01\x12\n\n\x06NOTICE\x10\x02\x12\x08\n\x04INFO\x10\x03\x12\t\n\x05\x44\x45\x42UG\x10\x04*J\n\tana_state\x12\t\n\x05UNSET\x10\x00\x12\r\n\tOPTIMIZED\x10\x01\x12\x11\n\rNON_OPTIMIZED\x10\x02\x12\x10\n\x0cINACCESSIBLE\x10\x03\x32\xaa\x0b\n\x07Gateway\x12\x33\n\rnamespace_add\x12\x12.namespace_add_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12;\n\x0flist_namespaces\x12\x14.list_namespaces_req\x1a\x10.namespaces_info\"\x00\x12\x38\n\x10namespace_resize\x12\x15.namespace_resize_req\x1a\x0b.req_status\"\x00\x12Q\n\x16namespace_get_io_stats\x12\x1b.namespace_get_io_stats_req\x1a\x18.namespace_io_stats_info\"\x00\x12\x41\n\x18namespace_set_qos_limits\x12\x16.namespace_set_qos_req\x1a\x0b.req_status\"\x00\x12\x62\n%namespace_change_load_balancing_group\x12*.namespace_change_load_balancing_group_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10namespace_delete\x12\x15.namespace_delete_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12,\n\nlist_hosts\x12\x0f.list_hosts_req\x1a\x0b.hosts_info\"\x00\x12>\n\x10list_connections\x12\x15.list_connections_req\x1a\x11.connections_info\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x38\n\x0elist_listeners\x12\x13.list_listeners_req\x1a\x0f.listeners_info\"\x00\x12?\n\x0flist_subsystems\x12\x14.list_subsystems_req\x1a\x14.subsystems_info_cli\"\x00\x12\x39\n\x0eget_subsystems\x12\x13.get_subsystems_req\x1a\x10.subsystems_info\"\x00\x12)\n\rset_ana_state\x12\t.ana_info\x1a\x0b.req_status\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3'
 )
 
-_TRANSPORTTYPE = _descriptor.EnumDescriptor(
-    name='TransportType',
-    full_name='TransportType',
-    filename=None,
-    file=DESCRIPTOR,
-    create_key=_descriptor._internal_create_key,
-    values=[
-        _descriptor.EnumValueDescriptor(
-            name='INVALID', index=0, number=0,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='RDMA', index=1, number=1,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='FC', index=2, number=2,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='TCP', index=3, number=3,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='PCIE', index=4, number=256,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='VFIOUSER', index=5, number=1024,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='CUSTOM', index=6, number=4096,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-    ],
-    containing_type=None,
-    serialized_options=None,
-    serialized_start=5347,
-    serialized_end=5441,
-)
-_sym_db.RegisterEnumDescriptor(_TRANSPORTTYPE)
-
-TransportType = enum_type_wrapper.EnumTypeWrapper(_TRANSPORTTYPE)
 _ADDRESSFAMILY = _descriptor.EnumDescriptor(
     name='AddressFamily',
     full_name='AddressFamily',
@@ -81,35 +30,20 @@
     create_key=_descriptor._internal_create_key,
     values=[
         _descriptor.EnumValueDescriptor(
-            name='invalid', index=0, number=0,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='ipv4', index=1, number=1,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='ipv6', index=2, number=2,
+            name='ipv4', index=0, number=0,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='ib', index=3, number=3,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='fc', index=4, number=4,
+            name='ipv6', index=1, number=1,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=5443,
-    serialized_end=5507,
+    serialized_start=6057,
+    serialized_end=6092,
 )
 _sym_db.RegisterEnumDescriptor(_ADDRESSFAMILY)
 
@@ -122,96 +56,86 @@
     create_key=_descriptor._internal_create_key,
     values=[
         _descriptor.EnumValueDescriptor(
-            name='DISABLED', index=0, number=0,
-            serialized_options=None,
-            type=None,
-            create_key=_descriptor._internal_create_key),
-        _descriptor.EnumValueDescriptor(
-            name='ERROR', index=1, number=1,
+            name='ERROR', index=0, number=0,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='WARNING', index=2, number=2,
+            name='WARNING', index=1, number=1,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='NOTICE', index=3, number=3,
+            name='NOTICE', index=2, number=2,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='INFO', index=4, number=4,
+            name='INFO', index=3, number=3,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='DEBUG', index=5, number=5,
+            name='DEBUG', index=4, number=4,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=5509,
-    serialized_end=5590,
+    serialized_start=6094,
+    serialized_end=6161,
 )
 _sym_db.RegisterEnumDescriptor(_LOGLEVEL)
 
 LogLevel = enum_type_wrapper.EnumTypeWrapper(_LOGLEVEL)
-_AUTOHASTATE = _descriptor.EnumDescriptor(
-    name='AutoHAState',
-    full_name='AutoHAState',
+_ANA_STATE = _descriptor.EnumDescriptor(
+    name='ana_state',
+    full_name='ana_state',
     filename=None,
     file=DESCRIPTOR,
     create_key=_descriptor._internal_create_key,
     values=[
         _descriptor.EnumValueDescriptor(
-            name='AUTO_HA_UNSET', index=0, number=0,
+            name='UNSET', index=0, number=0,
+            serialized_options=None,
+            type=None,
+            create_key=_descriptor._internal_create_key),
+        _descriptor.EnumValueDescriptor(
+            name='OPTIMIZED', index=1, number=1,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='AUTO_HA_OFF', index=1, number=1,
+            name='NON_OPTIMIZED', index=2, number=2,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
         _descriptor.EnumValueDescriptor(
-            name='AUTO_HA_ON', index=2, number=2,
+            name='INACCESSIBLE', index=3, number=3,
             serialized_options=None,
             type=None,
             create_key=_descriptor._internal_create_key),
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=5592,
-    serialized_end=5657,
+    serialized_start=6163,
+    serialized_end=6237,
 )
-_sym_db.RegisterEnumDescriptor(_AUTOHASTATE)
-
-AutoHAState = enum_type_wrapper.EnumTypeWrapper(_AUTOHASTATE)
-INVALID = 0
-RDMA = 1
-FC = 2
-TCP = 3
-PCIE = 256
-VFIOUSER = 1024
-CUSTOM = 4096
-invalid = 0
-ipv4 = 1
-ipv6 = 2
-ib = 3
-fc = 4
-DISABLED = 0
-ERROR = 1
-WARNING = 2
-NOTICE = 3
-INFO = 4
-DEBUG = 5
-AUTO_HA_UNSET = 0
-AUTO_HA_OFF = 1
-AUTO_HA_ON = 2
+_sym_db.RegisterEnumDescriptor(_ANA_STATE)
+
+ana_state = enum_type_wrapper.EnumTypeWrapper(_ANA_STATE)
+ipv4 = 0
+ipv6 = 1
+ERROR = 0
+WARNING = 1
+NOTICE = 2
+INFO = 3
+DEBUG = 4
+UNSET = 0
+OPTIMIZED = 1
+NON_OPTIMIZED = 2
+INACCESSIBLE = 3
 
 
 _NAMESPACE_ADD_REQ = _descriptor.Descriptor(
@@ -699,19 +623,12 @@
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='ana_reporting', full_name='create_subsystem_req.ana_reporting', index=3,
+            name='enable_ha', full_name='create_subsystem_req.enable_ha', index=3,
             number=4, type=8, cpp_type=7, label=1,
             has_default_value=False, default_value=False,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-        _descriptor.FieldDescriptor(
-            name='enable_ha', full_name='create_subsystem_req.enable_ha', index=4,
-            number=5, type=8, cpp_type=7, label=1,
-            has_default_value=False, default_value=False,
-            message_type=None, enum_type=None, containing_type=None,
-            is_extension=False, extension_scope=None,
-            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -730,7 +647,7 @@
             fields=[]),
     ],
     serialized_start=1105,
-    serialized_end=1263,
+    serialized_end=1240,
 )
 
 
@@ -773,8 +690,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1265,
-    serialized_end=1340,
+    serialized_start=1242,
+    serialized_end=1317,
 )
 
 
@@ -829,8 +746,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1342,
-    serialized_end=1438,
+    serialized_start=1319,
+    serialized_end=1415,
 )
 
 
@@ -868,8 +785,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1440,
-    serialized_end=1495,
+    serialized_start=1417,
+    serialized_end=1472,
 )
 
 
@@ -907,8 +824,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1497,
-    serialized_end=1555,
+    serialized_start=1474,
+    serialized_end=1532,
 )
 
 
@@ -939,8 +856,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1557,
-    serialized_end=1592,
+    serialized_start=1534,
+    serialized_end=1569,
 )
 
 
@@ -971,8 +888,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1594,
-    serialized_end=1635,
+    serialized_start=1571,
+    serialized_end=1612,
 )
 
 
@@ -1006,33 +923,19 @@
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='trtype', full_name='create_listener_req.trtype', index=3,
-            number=4, type=14, cpp_type=8, label=1,
-            has_default_value=False, default_value=0,
-            message_type=None, enum_type=None, containing_type=None,
-            is_extension=False, extension_scope=None,
-            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-        _descriptor.FieldDescriptor(
-            name='adrfam', full_name='create_listener_req.adrfam', index=4,
+            name='adrfam', full_name='create_listener_req.adrfam', index=3,
             number=5, type=14, cpp_type=8, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='trsvcid', full_name='create_listener_req.trsvcid', index=5,
+            name='trsvcid', full_name='create_listener_req.trsvcid', index=4,
             number=6, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-        _descriptor.FieldDescriptor(
-            name='auto_ha_state', full_name='create_listener_req.auto_ha_state', index=6,
-            number=7, type=14, cpp_type=8, label=1,
-            has_default_value=False, default_value=0,
-            message_type=None, enum_type=None, containing_type=None,
-            is_extension=False, extension_scope=None,
-            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -1044,29 +947,19 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
-        _descriptor.OneofDescriptor(
-            name='_trtype', full_name='create_listener_req._trtype',
-            index=0, containing_type=None,
-            create_key=_descriptor._internal_create_key,
-            fields=[]),
         _descriptor.OneofDescriptor(
             name='_adrfam', full_name='create_listener_req._adrfam',
-            index=1, containing_type=None,
+            index=0, containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[]),
         _descriptor.OneofDescriptor(
             name='_trsvcid', full_name='create_listener_req._trsvcid',
-            index=2, containing_type=None,
-            create_key=_descriptor._internal_create_key,
-            fields=[]),
-        _descriptor.OneofDescriptor(
-            name='_auto_ha_state', full_name='create_listener_req._auto_ha_state',
-            index=3, containing_type=None,
+            index=1, containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1638,
-    serialized_end=1900,
+    serialized_start=1615,
+    serialized_end=1769,
 )
 
 
@@ -1100,21 +993,14 @@
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='trtype', full_name='delete_listener_req.trtype', index=3,
-            number=4, type=14, cpp_type=8, label=1,
-            has_default_value=False, default_value=0,
-            message_type=None, enum_type=None, containing_type=None,
-            is_extension=False, extension_scope=None,
-            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-        _descriptor.FieldDescriptor(
-            name='adrfam', full_name='delete_listener_req.adrfam', index=4,
+            name='adrfam', full_name='delete_listener_req.adrfam', index=3,
             number=5, type=14, cpp_type=8, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='trsvcid', full_name='delete_listener_req.trsvcid', index=5,
+            name='trsvcid', full_name='delete_listener_req.trsvcid', index=4,
             number=6, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
@@ -1131,24 +1017,19 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
-        _descriptor.OneofDescriptor(
-            name='_trtype', full_name='delete_listener_req._trtype',
-            index=0, containing_type=None,
-            create_key=_descriptor._internal_create_key,
-            fields=[]),
         _descriptor.OneofDescriptor(
             name='_adrfam', full_name='delete_listener_req._adrfam',
-            index=1, containing_type=None,
+            index=0, containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[]),
         _descriptor.OneofDescriptor(
             name='_trsvcid', full_name='delete_listener_req._trsvcid',
-            index=2, containing_type=None,
+            index=1, containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1903,
-    serialized_end=2105,
+    serialized_start=1772,
+    serialized_end=1926,
 )
 
 
@@ -1179,8 +1060,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2107,
-    serialized_end=2146,
+    serialized_start=1928,
+    serialized_end=1967,
 )
 
 
@@ -1228,8 +1109,33 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=2148,
-    serialized_end=2261,
+    serialized_start=1969,
+    serialized_end=2082,
+)
+
+
+_GET_SUBSYSTEMS_REQ = _descriptor.Descriptor(
+    name='get_subsystems_req',
+    full_name='get_subsystems_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2084,
+    serialized_end=2104,
 )
 
 
@@ -1253,8 +1159,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2263,
-    serialized_end=2302,
+    serialized_start=2106,
+    serialized_end=2145,
 )
 
 
@@ -1278,8 +1184,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2304,
-    serialized_end=2332,
+    serialized_start=2147,
+    serialized_end=2175,
 )
 
 
@@ -1327,23 +1233,255 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=2334,
-    serialized_end=2460,
+    serialized_start=2177,
+    serialized_end=2303,
+)
+
+
+_GET_GATEWAY_INFO_REQ = _descriptor.Descriptor(
+    name='get_gateway_info_req',
+    full_name='get_gateway_info_req',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='cli_version', full_name='get_gateway_info_req.cli_version', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_cli_version', full_name='get_gateway_info_req._cli_version',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+    ],
+    serialized_start=2305,
+    serialized_end=2369,
+)
+
+
+_ANA_GROUP_STATE = _descriptor.Descriptor(
+    name='ana_group_state',
+    full_name='ana_group_state',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='grp_id', full_name='ana_group_state.grp_id', index=0,
+            number=1, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='state', full_name='ana_group_state.state', index=1,
+            number=2, type=14, cpp_type=8, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2371,
+    serialized_end=2431,
+)
+
+
+_NQN_ANA_STATES = _descriptor.Descriptor(
+    name='nqn_ana_states',
+    full_name='nqn_ana_states',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='nqn', full_name='nqn_ana_states.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='states', full_name='nqn_ana_states.states', index=1,
+            number=2, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2433,
+    serialized_end=2496,
+)
+
+
+_ANA_INFO = _descriptor.Descriptor(
+    name='ana_info',
+    full_name='ana_info',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='states', full_name='ana_info.states', index=0,
+            number=1, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2498,
+    serialized_end=2541,
+)
+
+
+_REQ_STATUS = _descriptor.Descriptor(
+    name='req_status',
+    full_name='req_status',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='req_status.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='req_status.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2543,
+    serialized_end=2594,
+)
+
+
+_NSID_STATUS = _descriptor.Descriptor(
+    name='nsid_status',
+    full_name='nsid_status',
+    filename=None,
+    file=DESCRIPTOR,
+    containing_type=None,
+    create_key=_descriptor._internal_create_key,
+    fields=[
+        _descriptor.FieldDescriptor(
+            name='status', full_name='nsid_status.status', index=0,
+            number=1, type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='error_message', full_name='nsid_status.error_message', index=1,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nsid', full_name='nsid_status.nsid', index=2,
+            number=3, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    ],
+    extensions=[
+    ],
+    nested_types=[],
+    enum_types=[
+    ],
+    serialized_options=None,
+    is_extendable=False,
+    syntax='proto3',
+    extension_ranges=[],
+    oneofs=[
+    ],
+    serialized_start=2596,
+    serialized_end=2662,
 )
 
 
-_GET_GATEWAY_INFO_REQ = _descriptor.Descriptor(
-    name='get_gateway_info_req',
-    full_name='get_gateway_info_req',
+_SUBSYSTEMS_INFO = _descriptor.Descriptor(
+    name='subsystems_info',
+    full_name='subsystems_info',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='cli_version', full_name='get_gateway_info_req.cli_version', index=0,
-            number=1, type=9, cpp_type=9, label=1,
-            has_default_value=False, default_value=b"".decode('utf-8'),
+            name='subsystems', full_name='subsystems_info.subsystems', index=0,
+            number=1, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
@@ -1358,46 +1496,97 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
-        _descriptor.OneofDescriptor(
-            name='_cli_version', full_name='get_gateway_info_req._cli_version',
-            index=0, containing_type=None,
-            create_key=_descriptor._internal_create_key,
-            fields=[]),
     ],
-    serialized_start=2462,
-    serialized_end=2526,
+    serialized_start=2664,
+    serialized_end=2713,
 )
 
 
-_BDEV_STATUS = _descriptor.Descriptor(
-    name='bdev_status',
-    full_name='bdev_status',
+_SUBSYSTEM = _descriptor.Descriptor(
+    name='subsystem',
+    full_name='subsystem',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='status', full_name='bdev_status.status', index=0,
-            number=1, type=5, cpp_type=1, label=1,
-            has_default_value=False, default_value=0,
+            name='nqn', full_name='subsystem.nqn', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='error_message', full_name='bdev_status.error_message', index=1,
+            name='subtype', full_name='subsystem.subtype', index=1,
             number=2, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='bdev_name', full_name='bdev_status.bdev_name', index=2,
-            number=3, type=9, cpp_type=9, label=1,
+            name='listen_addresses', full_name='subsystem.listen_addresses', index=2,
+            number=3, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='hosts', full_name='subsystem.hosts', index=3,
+            number=4, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='allow_any_host', full_name='subsystem.allow_any_host', index=4,
+            number=5, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='serial_number', full_name='subsystem.serial_number', index=5,
+            number=6, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='model_number', full_name='subsystem.model_number', index=6,
+            number=7, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_namespaces', full_name='subsystem.max_namespaces', index=7,
+            number=8, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='min_cntlid', full_name='subsystem.min_cntlid', index=8,
+            number=9, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_cntlid', full_name='subsystem.max_cntlid', index=9,
+            number=10, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='namespaces', full_name='subsystem.namespaces', index=10,
+            number=11, type=11, cpp_type=10, label=3,
+            has_default_value=False, default_value=[],
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -1409,34 +1598,80 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_serial_number', full_name='subsystem._serial_number',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_model_number', full_name='subsystem._model_number',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_max_namespaces', full_name='subsystem._max_namespaces',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_min_cntlid', full_name='subsystem._min_cntlid',
+            index=3, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_max_cntlid', full_name='subsystem._max_cntlid',
+            index=4, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
     ],
-    serialized_start=2528,
-    serialized_end=2599,
+    serialized_start=2716,
+    serialized_end=3096,
 )
 
 
-_REQ_STATUS = _descriptor.Descriptor(
-    name='req_status',
-    full_name='req_status',
+_LISTEN_ADDRESS = _descriptor.Descriptor(
+    name='listen_address',
+    full_name='listen_address',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='status', full_name='req_status.status', index=0,
-            number=1, type=5, cpp_type=1, label=1,
-            has_default_value=False, default_value=0,
+            name='trtype', full_name='listen_address.trtype', index=0,
+            number=1, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='error_message', full_name='req_status.error_message', index=1,
+            name='adrfam', full_name='listen_address.adrfam', index=1,
             number=2, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='traddr', full_name='listen_address.traddr', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='trsvcid', full_name='listen_address.trsvcid', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='transport', full_name='listen_address.transport', index=4,
+            number=5, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -1448,41 +1683,74 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_transport', full_name='listen_address._transport',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
     ],
-    serialized_start=2601,
-    serialized_end=2652,
+    serialized_start=3098,
+    serialized_end=3217,
 )
 
 
-_NSID_STATUS = _descriptor.Descriptor(
-    name='nsid_status',
-    full_name='nsid_status',
+_NAMESPACE = _descriptor.Descriptor(
+    name='namespace',
+    full_name='namespace',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='status', full_name='nsid_status.status', index=0,
-            number=1, type=5, cpp_type=1, label=1,
+            name='nsid', full_name='namespace.nsid', index=0,
+            number=1, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='error_message', full_name='nsid_status.error_message', index=1,
+            name='name', full_name='namespace.name', index=1,
             number=2, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='nsid', full_name='nsid_status.nsid', index=2,
-            number=3, type=13, cpp_type=3, label=1,
+            name='bdev_name', full_name='namespace.bdev_name', index=2,
+            number=3, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nguid', full_name='namespace.nguid', index=3,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='uuid', full_name='namespace.uuid', index=4,
+            number=5, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='anagrpid', full_name='namespace.anagrpid', index=5,
+            number=6, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='nonce', full_name='namespace.nonce', index=6,
+            number=7, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -1494,36 +1762,61 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_bdev_name', full_name='namespace._bdev_name',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_nguid', full_name='namespace._nguid',
+            index=1, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_uuid', full_name='namespace._uuid',
+            index=2, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_anagrpid', full_name='namespace._anagrpid',
+            index=3, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_nonce', full_name='namespace._nonce',
+            index=4, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
     ],
-    serialized_start=2654,
-    serialized_end=2720,
+    serialized_start=3220,
+    serialized_end=3421,
 )
 
 
-_SUBSYSTEMS_INFO = _descriptor.Descriptor(
-    name='subsystems_info',
-    full_name='subsystems_info',
+_SUBSYSTEMS_INFO_CLI = _descriptor.Descriptor(
+    name='subsystems_info_cli',
+    full_name='subsystems_info_cli',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='status', full_name='subsystems_info.status', index=0,
+            name='status', full_name='subsystems_info_cli.status', index=0,
             number=1, type=5, cpp_type=1, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='error_message', full_name='subsystems_info.error_message', index=1,
+            name='error_message', full_name='subsystems_info_cli.error_message', index=1,
             number=2, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='subsystems', full_name='subsystems_info.subsystems', index=2,
+            name='subsystems', full_name='subsystems_info_cli.subsystems', index=2,
             number=3, type=11, cpp_type=10, label=3,
             has_default_value=False, default_value=[],
             message_type=None, enum_type=None, containing_type=None,
@@ -1541,70 +1834,70 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2722,
-    serialized_end=2810,
+    serialized_start=3423,
+    serialized_end=3519,
 )
 
 
-_SUBSYSTEM = _descriptor.Descriptor(
-    name='subsystem',
-    full_name='subsystem',
+_SUBSYSTEM_CLI = _descriptor.Descriptor(
+    name='subsystem_cli',
+    full_name='subsystem_cli',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='nqn', full_name='subsystem.nqn', index=0,
+            name='nqn', full_name='subsystem_cli.nqn', index=0,
             number=1, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='enable_ha', full_name='subsystem.enable_ha', index=1,
+            name='enable_ha', full_name='subsystem_cli.enable_ha', index=1,
             number=2, type=8, cpp_type=7, label=1,
             has_default_value=False, default_value=False,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='serial_number', full_name='subsystem.serial_number', index=2,
+            name='serial_number', full_name='subsystem_cli.serial_number', index=2,
             number=3, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='model_number', full_name='subsystem.model_number', index=3,
+            name='model_number', full_name='subsystem_cli.model_number', index=3,
             number=4, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='min_cntlid', full_name='subsystem.min_cntlid', index=4,
+            name='min_cntlid', full_name='subsystem_cli.min_cntlid', index=4,
             number=5, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='max_cntlid', full_name='subsystem.max_cntlid', index=5,
+            name='max_cntlid', full_name='subsystem_cli.max_cntlid', index=5,
             number=6, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='namespace_count', full_name='subsystem.namespace_count', index=6,
+            name='namespace_count', full_name='subsystem_cli.namespace_count', index=6,
             number=7, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='subtype', full_name='subsystem.subtype', index=7,
+            name='subtype', full_name='subsystem_cli.subtype', index=7,
             number=8, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
@@ -1622,8 +1915,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2813,
-    serialized_end=2983,
+    serialized_start=3522,
+    serialized_end=3696,
 )
 
 
@@ -1716,9 +2009,14 @@
     syntax='proto3',
     extension_ranges=[],
     oneofs=[
+        _descriptor.OneofDescriptor(
+            name='_spdk_version', full_name='gateway_info._spdk_version',
+            index=0, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
     ],
-    serialized_start=2986,
-    serialized_end=3177,
+    serialized_start=3699,
+    serialized_end=3912,
 )
 
 
@@ -1763,8 +2061,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3179,
-    serialized_end=3248,
+    serialized_start=3914,
+    serialized_end=3983,
 )
 
 
@@ -1809,8 +2107,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3250,
-    serialized_end=3318,
+    serialized_start=3985,
+    serialized_end=4053,
 )
 
 
@@ -1831,8 +2129,8 @@
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
             name='trtype', full_name='listener_info.trtype', index=1,
-            number=2, type=14, cpp_type=8, label=1,
-            has_default_value=False, default_value=0,
+            number=2, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
@@ -1869,8 +2167,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3321,
-    serialized_end=3455,
+    serialized_start=4055,
+    serialized_end=4173,
 )
 
 
@@ -1915,8 +2213,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3457,
-    serialized_end=3547,
+    serialized_start=4175,
+    serialized_end=4265,
 )
 
 
@@ -1947,8 +2245,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3549,
-    serialized_end=3568,
+    serialized_start=4267,
+    serialized_end=4286,
 )
 
 
@@ -2007,8 +2305,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3570,
-    serialized_end=3690,
+    serialized_start=4288,
+    serialized_end=4408,
 )
 
 
@@ -2043,8 +2341,8 @@
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
             name='trtype', full_name='connection.trtype', index=3,
-            number=4, type=14, cpp_type=8, label=1,
-            has_default_value=False, default_value=0,
+            number=4, type=9, cpp_type=9, label=1,
+            has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
@@ -2088,8 +2386,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3693,
-    serialized_end=3879,
+    serialized_start=4411,
+    serialized_end=4581,
 )
 
 
@@ -2141,98 +2439,98 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3881,
-    serialized_end=3995,
+    serialized_start=4583,
+    serialized_end=4697,
 )
 
 
-_NAMESPACE = _descriptor.Descriptor(
-    name='namespace',
-    full_name='namespace',
+_NAMESPACE_CLI = _descriptor.Descriptor(
+    name='namespace_cli',
+    full_name='namespace_cli',
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
-            name='nsid', full_name='namespace.nsid', index=0,
+            name='nsid', full_name='namespace_cli.nsid', index=0,
             number=1, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='bdev_name', full_name='namespace.bdev_name', index=1,
+            name='bdev_name', full_name='namespace_cli.bdev_name', index=1,
             number=2, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='rbd_image_name', full_name='namespace.rbd_image_name', index=2,
+            name='rbd_image_name', full_name='namespace_cli.rbd_image_name', index=2,
             number=3, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='rbd_pool_name', full_name='namespace.rbd_pool_name', index=3,
+            name='rbd_pool_name', full_name='namespace_cli.rbd_pool_name', index=3,
             number=4, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='load_balancing_group', full_name='namespace.load_balancing_group', index=4,
+            name='load_balancing_group', full_name='namespace_cli.load_balancing_group', index=4,
             number=5, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='block_size', full_name='namespace.block_size', index=5,
+            name='block_size', full_name='namespace_cli.block_size', index=5,
             number=6, type=13, cpp_type=3, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='rbd_image_size', full_name='namespace.rbd_image_size', index=6,
+            name='rbd_image_size', full_name='namespace_cli.rbd_image_size', index=6,
             number=7, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='uuid', full_name='namespace.uuid', index=7,
+            name='uuid', full_name='namespace_cli.uuid', index=7,
             number=8, type=9, cpp_type=9, label=1,
             has_default_value=False, default_value=b"".decode('utf-8'),
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='rw_ios_per_second', full_name='namespace.rw_ios_per_second', index=8,
+            name='rw_ios_per_second', full_name='namespace_cli.rw_ios_per_second', index=8,
             number=9, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='rw_mbytes_per_second', full_name='namespace.rw_mbytes_per_second', index=9,
+            name='rw_mbytes_per_second', full_name='namespace_cli.rw_mbytes_per_second', index=9,
             number=10, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='r_mbytes_per_second', full_name='namespace.r_mbytes_per_second', index=10,
+            name='r_mbytes_per_second', full_name='namespace_cli.r_mbytes_per_second', index=10,
             number=11, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
-            name='w_mbytes_per_second', full_name='namespace.w_mbytes_per_second', index=11,
+            name='w_mbytes_per_second', full_name='namespace_cli.w_mbytes_per_second', index=11,
             number=12, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
@@ -2250,8 +2548,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3998,
-    serialized_end=4292,
+    serialized_start=4700,
+    serialized_end=4998,
 )
 
 
@@ -2303,8 +2601,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4294,
-    serialized_end=4405,
+    serialized_start=5000,
+    serialized_end=5115,
 )
 
 
@@ -2517,8 +2815,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4408,
-    serialized_end=5103,
+    serialized_start=5118,
+    serialized_end=5813,
 )
 
 
@@ -2556,8 +2854,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5105,
-    serialized_end=5156,
+    serialized_start=5815,
+    serialized_end=5866,
 )
 
 
@@ -2616,8 +2914,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5159,
-    serialized_end=5345,
+    serialized_start=5869,
+    serialized_end=6055,
 )
 
 _NAMESPACE_ADD_REQ.oneofs_by_name['_nsid'].fields.append(
@@ -2691,26 +2989,14 @@
 _LIST_NAMESPACES_REQ.oneofs_by_name['_uuid'].fields.append(
     _LIST_NAMESPACES_REQ.fields_by_name['uuid'])
 _LIST_NAMESPACES_REQ.fields_by_name['uuid'].containing_oneof = _LIST_NAMESPACES_REQ.oneofs_by_name['_uuid']
-_CREATE_LISTENER_REQ.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
 _CREATE_LISTENER_REQ.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
-_CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'].enum_type = _AUTOHASTATE
-_CREATE_LISTENER_REQ.oneofs_by_name['_trtype'].fields.append(
-    _CREATE_LISTENER_REQ.fields_by_name['trtype'])
-_CREATE_LISTENER_REQ.fields_by_name['trtype'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_trtype']
 _CREATE_LISTENER_REQ.oneofs_by_name['_adrfam'].fields.append(
     _CREATE_LISTENER_REQ.fields_by_name['adrfam'])
 _CREATE_LISTENER_REQ.fields_by_name['adrfam'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_adrfam']
 _CREATE_LISTENER_REQ.oneofs_by_name['_trsvcid'].fields.append(
     _CREATE_LISTENER_REQ.fields_by_name['trsvcid'])
 _CREATE_LISTENER_REQ.fields_by_name['trsvcid'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_trsvcid']
-_CREATE_LISTENER_REQ.oneofs_by_name['_auto_ha_state'].fields.append(
-    _CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'])
-_CREATE_LISTENER_REQ.fields_by_name['auto_ha_state'].containing_oneof = _CREATE_LISTENER_REQ.oneofs_by_name['_auto_ha_state']
-_DELETE_LISTENER_REQ.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
 _DELETE_LISTENER_REQ.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
-_DELETE_LISTENER_REQ.oneofs_by_name['_trtype'].fields.append(
-    _DELETE_LISTENER_REQ.fields_by_name['trtype'])
-_DELETE_LISTENER_REQ.fields_by_name['trtype'].containing_oneof = _DELETE_LISTENER_REQ.oneofs_by_name['_trtype']
 _DELETE_LISTENER_REQ.oneofs_by_name['_adrfam'].fields.append(
     _DELETE_LISTENER_REQ.fields_by_name['adrfam'])
 _DELETE_LISTENER_REQ.fields_by_name['adrfam'].containing_oneof = _DELETE_LISTENER_REQ.oneofs_by_name['_adrfam']
@@ -2734,15 +3020,56 @@
 _GET_GATEWAY_INFO_REQ.oneofs_by_name['_cli_version'].fields.append(
     _GET_GATEWAY_INFO_REQ.fields_by_name['cli_version'])
 _GET_GATEWAY_INFO_REQ.fields_by_name['cli_version'].containing_oneof = _GET_GATEWAY_INFO_REQ.oneofs_by_name['_cli_version']
+_ANA_GROUP_STATE.fields_by_name['state'].enum_type = _ANA_STATE
+_NQN_ANA_STATES.fields_by_name['states'].message_type = _ANA_GROUP_STATE
+_ANA_INFO.fields_by_name['states'].message_type = _NQN_ANA_STATES
 _SUBSYSTEMS_INFO.fields_by_name['subsystems'].message_type = _SUBSYSTEM
-_LISTENER_INFO.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
+_SUBSYSTEM.fields_by_name['listen_addresses'].message_type = _LISTEN_ADDRESS
+_SUBSYSTEM.fields_by_name['hosts'].message_type = _HOST
+_SUBSYSTEM.fields_by_name['namespaces'].message_type = _NAMESPACE
+_SUBSYSTEM.oneofs_by_name['_serial_number'].fields.append(
+    _SUBSYSTEM.fields_by_name['serial_number'])
+_SUBSYSTEM.fields_by_name['serial_number'].containing_oneof = _SUBSYSTEM.oneofs_by_name['_serial_number']
+_SUBSYSTEM.oneofs_by_name['_model_number'].fields.append(
+    _SUBSYSTEM.fields_by_name['model_number'])
+_SUBSYSTEM.fields_by_name['model_number'].containing_oneof = _SUBSYSTEM.oneofs_by_name['_model_number']
+_SUBSYSTEM.oneofs_by_name['_max_namespaces'].fields.append(
+    _SUBSYSTEM.fields_by_name['max_namespaces'])
+_SUBSYSTEM.fields_by_name['max_namespaces'].containing_oneof = _SUBSYSTEM.oneofs_by_name['_max_namespaces']
+_SUBSYSTEM.oneofs_by_name['_min_cntlid'].fields.append(
+    _SUBSYSTEM.fields_by_name['min_cntlid'])
+_SUBSYSTEM.fields_by_name['min_cntlid'].containing_oneof = _SUBSYSTEM.oneofs_by_name['_min_cntlid']
+_SUBSYSTEM.oneofs_by_name['_max_cntlid'].fields.append(
+    _SUBSYSTEM.fields_by_name['max_cntlid'])
+_SUBSYSTEM.fields_by_name['max_cntlid'].containing_oneof = _SUBSYSTEM.oneofs_by_name['_max_cntlid']
+_LISTEN_ADDRESS.oneofs_by_name['_transport'].fields.append(
+    _LISTEN_ADDRESS.fields_by_name['transport'])
+_LISTEN_ADDRESS.fields_by_name['transport'].containing_oneof = _LISTEN_ADDRESS.oneofs_by_name['_transport']
+_NAMESPACE.oneofs_by_name['_bdev_name'].fields.append(
+    _NAMESPACE.fields_by_name['bdev_name'])
+_NAMESPACE.fields_by_name['bdev_name'].containing_oneof = _NAMESPACE.oneofs_by_name['_bdev_name']
+_NAMESPACE.oneofs_by_name['_nguid'].fields.append(
+    _NAMESPACE.fields_by_name['nguid'])
+_NAMESPACE.fields_by_name['nguid'].containing_oneof = _NAMESPACE.oneofs_by_name['_nguid']
+_NAMESPACE.oneofs_by_name['_uuid'].fields.append(
+    _NAMESPACE.fields_by_name['uuid'])
+_NAMESPACE.fields_by_name['uuid'].containing_oneof = _NAMESPACE.oneofs_by_name['_uuid']
+_NAMESPACE.oneofs_by_name['_anagrpid'].fields.append(
+    _NAMESPACE.fields_by_name['anagrpid'])
+_NAMESPACE.fields_by_name['anagrpid'].containing_oneof = _NAMESPACE.oneofs_by_name['_anagrpid']
+_NAMESPACE.oneofs_by_name['_nonce'].fields.append(
+    _NAMESPACE.fields_by_name['nonce'])
+_NAMESPACE.fields_by_name['nonce'].containing_oneof = _NAMESPACE.oneofs_by_name['_nonce']
+_SUBSYSTEMS_INFO_CLI.fields_by_name['subsystems'].message_type = _SUBSYSTEM_CLI
+_GATEWAY_INFO.oneofs_by_name['_spdk_version'].fields.append(
+    _GATEWAY_INFO.fields_by_name['spdk_version'])
+_GATEWAY_INFO.fields_by_name['spdk_version'].containing_oneof = _GATEWAY_INFO.oneofs_by_name['_spdk_version']
 _LISTENER_INFO.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
 _LISTENERS_INFO.fields_by_name['listeners'].message_type = _LISTENER_INFO
 _HOSTS_INFO.fields_by_name['hosts'].message_type = _HOST
-_CONNECTION.fields_by_name['trtype'].enum_type = _TRANSPORTTYPE
 _CONNECTION.fields_by_name['adrfam'].enum_type = _ADDRESSFAMILY
 _CONNECTIONS_INFO.fields_by_name['connections'].message_type = _CONNECTION
-_NAMESPACES_INFO.fields_by_name['namespaces'].message_type = _NAMESPACE
+_NAMESPACES_INFO.fields_by_name['namespaces'].message_type = _NAMESPACE_CLI
 _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['nvmf_log_flags'].message_type = _SPDK_LOG_FLAG_INFO
 _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['log_level'].enum_type = _LOGLEVEL
 _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO.fields_by_name['log_print_level'].enum_type = _LOGLEVEL
@@ -2763,15 +3090,22 @@
 DESCRIPTOR.message_types_by_name['delete_listener_req'] = _DELETE_LISTENER_REQ
 DESCRIPTOR.message_types_by_name['list_listeners_req'] = _LIST_LISTENERS_REQ
 DESCRIPTOR.message_types_by_name['list_subsystems_req'] = _LIST_SUBSYSTEMS_REQ
+DESCRIPTOR.message_types_by_name['get_subsystems_req'] = _GET_SUBSYSTEMS_REQ
 DESCRIPTOR.message_types_by_name['get_spdk_nvmf_log_flags_and_level_req'] = _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ
 DESCRIPTOR.message_types_by_name['disable_spdk_nvmf_logs_req'] = _DISABLE_SPDK_NVMF_LOGS_REQ
 DESCRIPTOR.message_types_by_name['set_spdk_nvmf_logs_req'] = _SET_SPDK_NVMF_LOGS_REQ
 DESCRIPTOR.message_types_by_name['get_gateway_info_req'] = _GET_GATEWAY_INFO_REQ
-DESCRIPTOR.message_types_by_name['bdev_status'] = _BDEV_STATUS
+DESCRIPTOR.message_types_by_name['ana_group_state'] = _ANA_GROUP_STATE
+DESCRIPTOR.message_types_by_name['nqn_ana_states'] = _NQN_ANA_STATES
+DESCRIPTOR.message_types_by_name['ana_info'] = _ANA_INFO
 DESCRIPTOR.message_types_by_name['req_status'] = _REQ_STATUS
 DESCRIPTOR.message_types_by_name['nsid_status'] = _NSID_STATUS
 DESCRIPTOR.message_types_by_name['subsystems_info'] = _SUBSYSTEMS_INFO
 DESCRIPTOR.message_types_by_name['subsystem'] = _SUBSYSTEM
+DESCRIPTOR.message_types_by_name['listen_address'] = _LISTEN_ADDRESS
+DESCRIPTOR.message_types_by_name['namespace'] = _NAMESPACE
+DESCRIPTOR.message_types_by_name['subsystems_info_cli'] = _SUBSYSTEMS_INFO_CLI
+DESCRIPTOR.message_types_by_name['subsystem_cli'] = _SUBSYSTEM_CLI
 DESCRIPTOR.message_types_by_name['gateway_info'] = _GATEWAY_INFO
 DESCRIPTOR.message_types_by_name['cli_version'] = _CLI_VERSION
 DESCRIPTOR.message_types_by_name['gw_version'] = _GW_VERSION
@@ -2781,15 +3115,14 @@
 DESCRIPTOR.message_types_by_name['hosts_info'] = _HOSTS_INFO
 DESCRIPTOR.message_types_by_name['connection'] = _CONNECTION
 DESCRIPTOR.message_types_by_name['connections_info'] = _CONNECTIONS_INFO
-DESCRIPTOR.message_types_by_name['namespace'] = _NAMESPACE
+DESCRIPTOR.message_types_by_name['namespace_cli'] = _NAMESPACE_CLI
 DESCRIPTOR.message_types_by_name['namespaces_info'] = _NAMESPACES_INFO
 DESCRIPTOR.message_types_by_name['namespace_io_stats_info'] = _NAMESPACE_IO_STATS_INFO
 DESCRIPTOR.message_types_by_name['spdk_log_flag_info'] = _SPDK_LOG_FLAG_INFO
 DESCRIPTOR.message_types_by_name['spdk_nvmf_log_flags_and_level_info'] = _SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO
-DESCRIPTOR.enum_types_by_name['TransportType'] = _TRANSPORTTYPE
 DESCRIPTOR.enum_types_by_name['AddressFamily'] = _ADDRESSFAMILY
 DESCRIPTOR.enum_types_by_name['LogLevel'] = _LOGLEVEL
-DESCRIPTOR.enum_types_by_name['AutoHAState'] = _AUTOHASTATE
+DESCRIPTOR.enum_types_by_name['ana_state'] = _ANA_STATE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 namespace_add_req = _reflection.GeneratedProtocolMessageType('namespace_add_req', (_message.Message,), {
@@ -2911,6 +3244,13 @@
 })
 _sym_db.RegisterMessage(list_subsystems_req)
 
+get_subsystems_req = _reflection.GeneratedProtocolMessageType('get_subsystems_req', (_message.Message,), {
+    'DESCRIPTOR': _GET_SUBSYSTEMS_REQ,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:get_subsystems_req)
+})
+_sym_db.RegisterMessage(get_subsystems_req)
+
 get_spdk_nvmf_log_flags_and_level_req = _reflection.GeneratedProtocolMessageType('get_spdk_nvmf_log_flags_and_level_req', (_message.Message,), {
     'DESCRIPTOR': _GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
     '__module__': 'gateway_pb2'
@@ -2939,12 +3279,26 @@
 })
 _sym_db.RegisterMessage(get_gateway_info_req)
 
-bdev_status = _reflection.GeneratedProtocolMessageType('bdev_status', (_message.Message,), {
-    'DESCRIPTOR': _BDEV_STATUS,
+ana_group_state = _reflection.GeneratedProtocolMessageType('ana_group_state', (_message.Message,), {
+    'DESCRIPTOR': _ANA_GROUP_STATE,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:ana_group_state)
+})
+_sym_db.RegisterMessage(ana_group_state)
+
+nqn_ana_states = _reflection.GeneratedProtocolMessageType('nqn_ana_states', (_message.Message,), {
+    'DESCRIPTOR': _NQN_ANA_STATES,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:nqn_ana_states)
+})
+_sym_db.RegisterMessage(nqn_ana_states)
+
+ana_info = _reflection.GeneratedProtocolMessageType('ana_info', (_message.Message,), {
+    'DESCRIPTOR': _ANA_INFO,
     '__module__': 'gateway_pb2'
-    # @@protoc_insertion_point(class_scope:bdev_status)
+    # @@protoc_insertion_point(class_scope:ana_info)
 })
-_sym_db.RegisterMessage(bdev_status)
+_sym_db.RegisterMessage(ana_info)
 
 req_status = _reflection.GeneratedProtocolMessageType('req_status', (_message.Message,), {
     'DESCRIPTOR': _REQ_STATUS,
@@ -2974,6 +3328,34 @@
 })
 _sym_db.RegisterMessage(subsystem)
 
+listen_address = _reflection.GeneratedProtocolMessageType('listen_address', (_message.Message,), {
+    'DESCRIPTOR': _LISTEN_ADDRESS,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:listen_address)
+})
+_sym_db.RegisterMessage(listen_address)
+
+namespace = _reflection.GeneratedProtocolMessageType('namespace', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:namespace)
+})
+_sym_db.RegisterMessage(namespace)
+
+subsystems_info_cli = _reflection.GeneratedProtocolMessageType('subsystems_info_cli', (_message.Message,), {
+    'DESCRIPTOR': _SUBSYSTEMS_INFO_CLI,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:subsystems_info_cli)
+})
+_sym_db.RegisterMessage(subsystems_info_cli)
+
+subsystem_cli = _reflection.GeneratedProtocolMessageType('subsystem_cli', (_message.Message,), {
+    'DESCRIPTOR': _SUBSYSTEM_CLI,
+    '__module__': 'gateway_pb2'
+    # @@protoc_insertion_point(class_scope:subsystem_cli)
+})
+_sym_db.RegisterMessage(subsystem_cli)
+
 gateway_info = _reflection.GeneratedProtocolMessageType('gateway_info', (_message.Message,), {
     'DESCRIPTOR': _GATEWAY_INFO,
     '__module__': 'gateway_pb2'
@@ -3037,12 +3419,12 @@
 })
 _sym_db.RegisterMessage(connections_info)
 
-namespace = _reflection.GeneratedProtocolMessageType('namespace', (_message.Message,), {
-    'DESCRIPTOR': _NAMESPACE,
+namespace_cli = _reflection.GeneratedProtocolMessageType('namespace_cli', (_message.Message,), {
+    'DESCRIPTOR': _NAMESPACE_CLI,
     '__module__': 'gateway_pb2'
-    # @@protoc_insertion_point(class_scope:namespace)
+    # @@protoc_insertion_point(class_scope:namespace_cli)
 })
-_sym_db.RegisterMessage(namespace)
+_sym_db.RegisterMessage(namespace_cli)
 
 namespaces_info = _reflection.GeneratedProtocolMessageType('namespaces_info', (_message.Message,), {
     'DESCRIPTOR': _NAMESPACES_INFO,
@@ -3080,8 +3462,8 @@
     index=0,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
-    serialized_start=5660,
-    serialized_end=7004,
+    serialized_start=6240,
+    serialized_end=7690,
     methods=[
         _descriptor.MethodDescriptor(
             name='namespace_add',
@@ -3249,14 +3631,34 @@
             index=16,
             containing_service=None,
             input_type=_LIST_SUBSYSTEMS_REQ,
+            output_type=_SUBSYSTEMS_INFO_CLI,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
+        _descriptor.MethodDescriptor(
+            name='get_subsystems',
+            full_name='Gateway.get_subsystems',
+            index=17,
+            containing_service=None,
+            input_type=_GET_SUBSYSTEMS_REQ,
             output_type=_SUBSYSTEMS_INFO,
             serialized_options=None,
             create_key=_descriptor._internal_create_key,
         ),
+        _descriptor.MethodDescriptor(
+            name='set_ana_state',
+            full_name='Gateway.set_ana_state',
+            index=18,
+            containing_service=None,
+            input_type=_ANA_INFO,
+            output_type=_REQ_STATUS,
+            serialized_options=None,
+            create_key=_descriptor._internal_create_key,
+        ),
         _descriptor.MethodDescriptor(
             name='get_spdk_nvmf_log_flags_and_level',
             full_name='Gateway.get_spdk_nvmf_log_flags_and_level',
-            index=17,
+            index=19,
             containing_service=None,
             input_type=_GET_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_REQ,
             output_type=_SPDK_NVMF_LOG_FLAGS_AND_LEVEL_INFO,
@@ -3266,7 +3668,7 @@
         _descriptor.MethodDescriptor(
             name='disable_spdk_nvmf_logs',
             full_name='Gateway.disable_spdk_nvmf_logs',
-            index=18,
+            index=20,
             containing_service=None,
             input_type=_DISABLE_SPDK_NVMF_LOGS_REQ,
             output_type=_REQ_STATUS,
@@ -3276,7 +3678,7 @@
         _descriptor.MethodDescriptor(
             name='set_spdk_nvmf_logs',
             full_name='Gateway.set_spdk_nvmf_logs',
-            index=19,
+            index=21,
             containing_service=None,
             input_type=_SET_SPDK_NVMF_LOGS_REQ,
             output_type=_REQ_STATUS,
@@ -3286,7 +3688,7 @@
         _descriptor.MethodDescriptor(
             name='get_gateway_info',
             full_name='Gateway.get_gateway_info',
-            index=20,
+            index=22,
             containing_service=None,
             input_type=_GET_GATEWAY_INFO_REQ,
             output_type=_GATEWAY_INFO,
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
index 9993ca4cf6b9..cc97bb26c53b 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
@@ -15,110 +15,120 @@ def __init__(self, channel):
             channel: A grpc.Channel.
         """
         self.namespace_add = channel.unary_unary(
-            '/Gateway/namespace_add',
-            request_serializer=gateway__pb2.namespace_add_req.SerializeToString,
-            response_deserializer=gateway__pb2.nsid_status.FromString,
-        )
+                '/Gateway/namespace_add',
+                request_serializer=gateway__pb2.namespace_add_req.SerializeToString,
+                response_deserializer=gateway__pb2.nsid_status.FromString,
+                )
         self.create_subsystem = channel.unary_unary(
-            '/Gateway/create_subsystem',
-            request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/create_subsystem',
+                request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.delete_subsystem = channel.unary_unary(
-            '/Gateway/delete_subsystem',
-            request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/delete_subsystem',
+                request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.list_namespaces = channel.unary_unary(
-            '/Gateway/list_namespaces',
-            request_serializer=gateway__pb2.list_namespaces_req.SerializeToString,
-            response_deserializer=gateway__pb2.namespaces_info.FromString,
-        )
+                '/Gateway/list_namespaces',
+                request_serializer=gateway__pb2.list_namespaces_req.SerializeToString,
+                response_deserializer=gateway__pb2.namespaces_info.FromString,
+                )
         self.namespace_resize = channel.unary_unary(
-            '/Gateway/namespace_resize',
-            request_serializer=gateway__pb2.namespace_resize_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/namespace_resize',
+                request_serializer=gateway__pb2.namespace_resize_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.namespace_get_io_stats = channel.unary_unary(
-            '/Gateway/namespace_get_io_stats',
-            request_serializer=gateway__pb2.namespace_get_io_stats_req.SerializeToString,
-            response_deserializer=gateway__pb2.namespace_io_stats_info.FromString,
-        )
+                '/Gateway/namespace_get_io_stats',
+                request_serializer=gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+                response_deserializer=gateway__pb2.namespace_io_stats_info.FromString,
+                )
         self.namespace_set_qos_limits = channel.unary_unary(
-            '/Gateway/namespace_set_qos_limits',
-            request_serializer=gateway__pb2.namespace_set_qos_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/namespace_set_qos_limits',
+                request_serializer=gateway__pb2.namespace_set_qos_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.namespace_change_load_balancing_group = channel.unary_unary(
-            '/Gateway/namespace_change_load_balancing_group',
-            request_serializer=gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/namespace_change_load_balancing_group',
+                request_serializer=gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.namespace_delete = channel.unary_unary(
-            '/Gateway/namespace_delete',
-            request_serializer=gateway__pb2.namespace_delete_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/namespace_delete',
+                request_serializer=gateway__pb2.namespace_delete_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.add_host = channel.unary_unary(
-            '/Gateway/add_host',
-            request_serializer=gateway__pb2.add_host_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/add_host',
+                request_serializer=gateway__pb2.add_host_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.remove_host = channel.unary_unary(
-            '/Gateway/remove_host',
-            request_serializer=gateway__pb2.remove_host_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/remove_host',
+                request_serializer=gateway__pb2.remove_host_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.list_hosts = channel.unary_unary(
-            '/Gateway/list_hosts',
-            request_serializer=gateway__pb2.list_hosts_req.SerializeToString,
-            response_deserializer=gateway__pb2.hosts_info.FromString,
-        )
+                '/Gateway/list_hosts',
+                request_serializer=gateway__pb2.list_hosts_req.SerializeToString,
+                response_deserializer=gateway__pb2.hosts_info.FromString,
+                )
         self.list_connections = channel.unary_unary(
-            '/Gateway/list_connections',
-            request_serializer=gateway__pb2.list_connections_req.SerializeToString,
-            response_deserializer=gateway__pb2.connections_info.FromString,
-        )
+                '/Gateway/list_connections',
+                request_serializer=gateway__pb2.list_connections_req.SerializeToString,
+                response_deserializer=gateway__pb2.connections_info.FromString,
+                )
         self.create_listener = channel.unary_unary(
-            '/Gateway/create_listener',
-            request_serializer=gateway__pb2.create_listener_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/create_listener',
+                request_serializer=gateway__pb2.create_listener_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.delete_listener = channel.unary_unary(
-            '/Gateway/delete_listener',
-            request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/delete_listener',
+                request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.list_listeners = channel.unary_unary(
-            '/Gateway/list_listeners',
-            request_serializer=gateway__pb2.list_listeners_req.SerializeToString,
-            response_deserializer=gateway__pb2.listeners_info.FromString,
-        )
+                '/Gateway/list_listeners',
+                request_serializer=gateway__pb2.list_listeners_req.SerializeToString,
+                response_deserializer=gateway__pb2.listeners_info.FromString,
+                )
         self.list_subsystems = channel.unary_unary(
-            '/Gateway/list_subsystems',
-            request_serializer=gateway__pb2.list_subsystems_req.SerializeToString,
-            response_deserializer=gateway__pb2.subsystems_info.FromString,
-        )
+                '/Gateway/list_subsystems',
+                request_serializer=gateway__pb2.list_subsystems_req.SerializeToString,
+                response_deserializer=gateway__pb2.subsystems_info_cli.FromString,
+                )
+        self.get_subsystems = channel.unary_unary(
+                '/Gateway/get_subsystems',
+                request_serializer=gateway__pb2.get_subsystems_req.SerializeToString,
+                response_deserializer=gateway__pb2.subsystems_info.FromString,
+                )
+        self.set_ana_state = channel.unary_unary(
+                '/Gateway/set_ana_state',
+                request_serializer=gateway__pb2.ana_info.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.get_spdk_nvmf_log_flags_and_level = channel.unary_unary(
-            '/Gateway/get_spdk_nvmf_log_flags_and_level',
-            request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-            response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-        )
+                '/Gateway/get_spdk_nvmf_log_flags_and_level',
+                request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+                response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+                )
         self.disable_spdk_nvmf_logs = channel.unary_unary(
-            '/Gateway/disable_spdk_nvmf_logs',
-            request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/disable_spdk_nvmf_logs',
+                request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.set_spdk_nvmf_logs = channel.unary_unary(
-            '/Gateway/set_spdk_nvmf_logs',
-            request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-            response_deserializer=gateway__pb2.req_status.FromString,
-        )
+                '/Gateway/set_spdk_nvmf_logs',
+                request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+                response_deserializer=gateway__pb2.req_status.FromString,
+                )
         self.get_gateway_info = channel.unary_unary(
-            '/Gateway/get_gateway_info',
-            request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
-            response_deserializer=gateway__pb2.gateway_info.FromString,
-        )
+                '/Gateway/get_gateway_info',
+                request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
+                response_deserializer=gateway__pb2.gateway_info.FromString,
+                )
 
 
 class GatewayServicer(object):
@@ -243,6 +253,20 @@ def list_subsystems(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def get_subsystems(self, request, context):
+        """Gets subsystems
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def set_ana_state(self, request, context):
+        """Set gateway ANA states
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
     def get_spdk_nvmf_log_flags_and_level(self, request, context):
         """Gets spdk nvmf log flags and level
         """
@@ -274,475 +298,518 @@ def get_gateway_info(self, request, context):
 
 def add_GatewayServicer_to_server(servicer, server):
     rpc_method_handlers = {
-        'namespace_add': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_add,
-            request_deserializer=gateway__pb2.namespace_add_req.FromString,
-            response_serializer=gateway__pb2.nsid_status.SerializeToString,
-        ),
-        'create_subsystem': grpc.unary_unary_rpc_method_handler(
-            servicer.create_subsystem,
-            request_deserializer=gateway__pb2.create_subsystem_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'delete_subsystem': grpc.unary_unary_rpc_method_handler(
-            servicer.delete_subsystem,
-            request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'list_namespaces': grpc.unary_unary_rpc_method_handler(
-            servicer.list_namespaces,
-            request_deserializer=gateway__pb2.list_namespaces_req.FromString,
-            response_serializer=gateway__pb2.namespaces_info.SerializeToString,
-        ),
-        'namespace_resize': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_resize,
-            request_deserializer=gateway__pb2.namespace_resize_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'namespace_get_io_stats': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_get_io_stats,
-            request_deserializer=gateway__pb2.namespace_get_io_stats_req.FromString,
-            response_serializer=gateway__pb2.namespace_io_stats_info.SerializeToString,
-        ),
-        'namespace_set_qos_limits': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_set_qos_limits,
-            request_deserializer=gateway__pb2.namespace_set_qos_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'namespace_change_load_balancing_group': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_change_load_balancing_group,
-            request_deserializer=gateway__pb2.namespace_change_load_balancing_group_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'namespace_delete': grpc.unary_unary_rpc_method_handler(
-            servicer.namespace_delete,
-            request_deserializer=gateway__pb2.namespace_delete_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'add_host': grpc.unary_unary_rpc_method_handler(
-            servicer.add_host,
-            request_deserializer=gateway__pb2.add_host_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'remove_host': grpc.unary_unary_rpc_method_handler(
-            servicer.remove_host,
-            request_deserializer=gateway__pb2.remove_host_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'list_hosts': grpc.unary_unary_rpc_method_handler(
-            servicer.list_hosts,
-            request_deserializer=gateway__pb2.list_hosts_req.FromString,
-            response_serializer=gateway__pb2.hosts_info.SerializeToString,
-        ),
-        'list_connections': grpc.unary_unary_rpc_method_handler(
-            servicer.list_connections,
-            request_deserializer=gateway__pb2.list_connections_req.FromString,
-            response_serializer=gateway__pb2.connections_info.SerializeToString,
-        ),
-        'create_listener': grpc.unary_unary_rpc_method_handler(
-            servicer.create_listener,
-            request_deserializer=gateway__pb2.create_listener_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'delete_listener': grpc.unary_unary_rpc_method_handler(
-            servicer.delete_listener,
-            request_deserializer=gateway__pb2.delete_listener_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'list_listeners': grpc.unary_unary_rpc_method_handler(
-            servicer.list_listeners,
-            request_deserializer=gateway__pb2.list_listeners_req.FromString,
-            response_serializer=gateway__pb2.listeners_info.SerializeToString,
-        ),
-        'list_subsystems': grpc.unary_unary_rpc_method_handler(
-            servicer.list_subsystems,
-            request_deserializer=gateway__pb2.list_subsystems_req.FromString,
-            response_serializer=gateway__pb2.subsystems_info.SerializeToString,
-        ),
-        'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
-            servicer.get_spdk_nvmf_log_flags_and_level,
-            request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
-            response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
-        ),
-        'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-            servicer.disable_spdk_nvmf_logs,
-            request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-            servicer.set_spdk_nvmf_logs,
-            request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
-            response_serializer=gateway__pb2.req_status.SerializeToString,
-        ),
-        'get_gateway_info': grpc.unary_unary_rpc_method_handler(
-            servicer.get_gateway_info,
-            request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
-            response_serializer=gateway__pb2.gateway_info.SerializeToString,
-        ),
+            'namespace_add': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_add,
+                    request_deserializer=gateway__pb2.namespace_add_req.FromString,
+                    response_serializer=gateway__pb2.nsid_status.SerializeToString,
+            ),
+            'create_subsystem': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_subsystem,
+                    request_deserializer=gateway__pb2.create_subsystem_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'delete_subsystem': grpc.unary_unary_rpc_method_handler(
+                    servicer.delete_subsystem,
+                    request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'list_namespaces': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_namespaces,
+                    request_deserializer=gateway__pb2.list_namespaces_req.FromString,
+                    response_serializer=gateway__pb2.namespaces_info.SerializeToString,
+            ),
+            'namespace_resize': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_resize,
+                    request_deserializer=gateway__pb2.namespace_resize_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'namespace_get_io_stats': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_get_io_stats,
+                    request_deserializer=gateway__pb2.namespace_get_io_stats_req.FromString,
+                    response_serializer=gateway__pb2.namespace_io_stats_info.SerializeToString,
+            ),
+            'namespace_set_qos_limits': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_set_qos_limits,
+                    request_deserializer=gateway__pb2.namespace_set_qos_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'namespace_change_load_balancing_group': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_change_load_balancing_group,
+                    request_deserializer=gateway__pb2.namespace_change_load_balancing_group_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'namespace_delete': grpc.unary_unary_rpc_method_handler(
+                    servicer.namespace_delete,
+                    request_deserializer=gateway__pb2.namespace_delete_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'add_host': grpc.unary_unary_rpc_method_handler(
+                    servicer.add_host,
+                    request_deserializer=gateway__pb2.add_host_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'remove_host': grpc.unary_unary_rpc_method_handler(
+                    servicer.remove_host,
+                    request_deserializer=gateway__pb2.remove_host_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'list_hosts': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_hosts,
+                    request_deserializer=gateway__pb2.list_hosts_req.FromString,
+                    response_serializer=gateway__pb2.hosts_info.SerializeToString,
+            ),
+            'list_connections': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_connections,
+                    request_deserializer=gateway__pb2.list_connections_req.FromString,
+                    response_serializer=gateway__pb2.connections_info.SerializeToString,
+            ),
+            'create_listener': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_listener,
+                    request_deserializer=gateway__pb2.create_listener_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'delete_listener': grpc.unary_unary_rpc_method_handler(
+                    servicer.delete_listener,
+                    request_deserializer=gateway__pb2.delete_listener_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'list_listeners': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_listeners,
+                    request_deserializer=gateway__pb2.list_listeners_req.FromString,
+                    response_serializer=gateway__pb2.listeners_info.SerializeToString,
+            ),
+            'list_subsystems': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_subsystems,
+                    request_deserializer=gateway__pb2.list_subsystems_req.FromString,
+                    response_serializer=gateway__pb2.subsystems_info_cli.SerializeToString,
+            ),
+            'get_subsystems': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_subsystems,
+                    request_deserializer=gateway__pb2.get_subsystems_req.FromString,
+                    response_serializer=gateway__pb2.subsystems_info.SerializeToString,
+            ),
+            'set_ana_state': grpc.unary_unary_rpc_method_handler(
+                    servicer.set_ana_state,
+                    request_deserializer=gateway__pb2.ana_info.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_spdk_nvmf_log_flags_and_level,
+                    request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
+                    response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
+            ),
+            'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+                    servicer.disable_spdk_nvmf_logs,
+                    request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+                    servicer.set_spdk_nvmf_logs,
+                    request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
+                    response_serializer=gateway__pb2.req_status.SerializeToString,
+            ),
+            'get_gateway_info': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_gateway_info,
+                    request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
+                    response_serializer=gateway__pb2.gateway_info.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-        'Gateway', rpc_method_handlers)
+            'Gateway', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
- # This class is part of an EXPERIMENTAL API.
-
 
+ # This class is part of an EXPERIMENTAL API.
 class Gateway(object):
     """Missing associated documentation comment in .proto file."""
 
     @staticmethod
     def namespace_add(request,
-                      target,
-                      options=(),
-                      channel_credentials=None,
-                      call_credentials=None,
-                      insecure=False,
-                      compression=None,
-                      wait_for_ready=None,
-                      timeout=None,
-                      metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_add',
-                                             gateway__pb2.namespace_add_req.SerializeToString,
-                                             gateway__pb2.nsid_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_add_req.SerializeToString,
+            gateway__pb2.nsid_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_subsystem(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_subsystem',
-                                             gateway__pb2.create_subsystem_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.create_subsystem_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_subsystem(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_subsystem',
-                                             gateway__pb2.delete_subsystem_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.delete_subsystem_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_namespaces(request,
-                        target,
-                        options=(),
-                        channel_credentials=None,
-                        call_credentials=None,
-                        insecure=False,
-                        compression=None,
-                        wait_for_ready=None,
-                        timeout=None,
-                        metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_namespaces',
-                                             gateway__pb2.list_namespaces_req.SerializeToString,
-                                             gateway__pb2.namespaces_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.list_namespaces_req.SerializeToString,
+            gateway__pb2.namespaces_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_resize(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_resize',
-                                             gateway__pb2.namespace_resize_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_resize_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_get_io_stats(request,
-                               target,
-                               options=(),
-                               channel_credentials=None,
-                               call_credentials=None,
-                               insecure=False,
-                               compression=None,
-                               wait_for_ready=None,
-                               timeout=None,
-                               metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_get_io_stats',
-                                             gateway__pb2.namespace_get_io_stats_req.SerializeToString,
-                                             gateway__pb2.namespace_io_stats_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+            gateway__pb2.namespace_io_stats_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_set_qos_limits(request,
-                                 target,
-                                 options=(),
-                                 channel_credentials=None,
-                                 call_credentials=None,
-                                 insecure=False,
-                                 compression=None,
-                                 wait_for_ready=None,
-                                 timeout=None,
-                                 metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_set_qos_limits',
-                                             gateway__pb2.namespace_set_qos_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_set_qos_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_change_load_balancing_group(request,
-                                              target,
-                                              options=(),
-                                              channel_credentials=None,
-                                              call_credentials=None,
-                                              insecure=False,
-                                              compression=None,
-                                              wait_for_ready=None,
-                                              timeout=None,
-                                              metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_change_load_balancing_group',
-                                             gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_delete(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_delete',
-                                             gateway__pb2.namespace_delete_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.namespace_delete_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def add_host(request,
-                 target,
-                 options=(),
-                 channel_credentials=None,
-                 call_credentials=None,
-                 insecure=False,
-                 compression=None,
-                 wait_for_ready=None,
-                 timeout=None,
-                 metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/add_host',
-                                             gateway__pb2.add_host_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.add_host_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def remove_host(request,
-                    target,
-                    options=(),
-                    channel_credentials=None,
-                    call_credentials=None,
-                    insecure=False,
-                    compression=None,
-                    wait_for_ready=None,
-                    timeout=None,
-                    metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/remove_host',
-                                             gateway__pb2.remove_host_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.remove_host_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_hosts(request,
-                   target,
-                   options=(),
-                   channel_credentials=None,
-                   call_credentials=None,
-                   insecure=False,
-                   compression=None,
-                   wait_for_ready=None,
-                   timeout=None,
-                   metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_hosts',
-                                             gateway__pb2.list_hosts_req.SerializeToString,
-                                             gateway__pb2.hosts_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.list_hosts_req.SerializeToString,
+            gateway__pb2.hosts_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_connections(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_connections',
-                                             gateway__pb2.list_connections_req.SerializeToString,
-                                             gateway__pb2.connections_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.list_connections_req.SerializeToString,
+            gateway__pb2.connections_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_listener(request,
-                        target,
-                        options=(),
-                        channel_credentials=None,
-                        call_credentials=None,
-                        insecure=False,
-                        compression=None,
-                        wait_for_ready=None,
-                        timeout=None,
-                        metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_listener',
-                                             gateway__pb2.create_listener_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.create_listener_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_listener(request,
-                        target,
-                        options=(),
-                        channel_credentials=None,
-                        call_credentials=None,
-                        insecure=False,
-                        compression=None,
-                        wait_for_ready=None,
-                        timeout=None,
-                        metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_listener',
-                                             gateway__pb2.delete_listener_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.delete_listener_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_listeners(request,
-                       target,
-                       options=(),
-                       channel_credentials=None,
-                       call_credentials=None,
-                       insecure=False,
-                       compression=None,
-                       wait_for_ready=None,
-                       timeout=None,
-                       metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_listeners',
-                                             gateway__pb2.list_listeners_req.SerializeToString,
-                                             gateway__pb2.listeners_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.list_listeners_req.SerializeToString,
+            gateway__pb2.listeners_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_subsystems(request,
-                        target,
-                        options=(),
-                        channel_credentials=None,
-                        call_credentials=None,
-                        insecure=False,
-                        compression=None,
-                        wait_for_ready=None,
-                        timeout=None,
-                        metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_subsystems',
-                                             gateway__pb2.list_subsystems_req.SerializeToString,
-                                             gateway__pb2.subsystems_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.list_subsystems_req.SerializeToString,
+            gateway__pb2.subsystems_info_cli.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def get_subsystems(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/get_subsystems',
+            gateway__pb2.get_subsystems_req.SerializeToString,
+            gateway__pb2.subsystems_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def set_ana_state(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/Gateway/set_ana_state',
+            gateway__pb2.ana_info.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_spdk_nvmf_log_flags_and_level(request,
-                                          target,
-                                          options=(),
-                                          channel_credentials=None,
-                                          call_credentials=None,
-                                          insecure=False,
-                                          compression=None,
-                                          wait_for_ready=None,
-                                          timeout=None,
-                                          metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_spdk_nvmf_log_flags_and_level',
-                                             gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-                                             gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+            gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def disable_spdk_nvmf_logs(request,
-                               target,
-                               options=(),
-                               channel_credentials=None,
-                               call_credentials=None,
-                               insecure=False,
-                               compression=None,
-                               wait_for_ready=None,
-                               timeout=None,
-                               metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/disable_spdk_nvmf_logs',
-                                             gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def set_spdk_nvmf_logs(request,
-                           target,
-                           options=(),
-                           channel_credentials=None,
-                           call_credentials=None,
-                           insecure=False,
-                           compression=None,
-                           wait_for_ready=None,
-                           timeout=None,
-                           metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/set_spdk_nvmf_logs',
-                                             gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-                                             gateway__pb2.req_status.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+            gateway__pb2.req_status.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_gateway_info(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_gateway_info',
-                                             gateway__pb2.get_gateway_info_req.SerializeToString,
-                                             gateway__pb2.gateway_info.FromString,
-                                             options, channel_credentials,
-                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+            gateway__pb2.get_gateway_info_req.SerializeToString,
+            gateway__pb2.gateway_info.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

From b3f21b83724dfc1b312789d4f6cde28636c4d774 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 12 Feb 2024 12:22:41 +0000
Subject: [PATCH 1894/2492] node-proxy: improve http error handling in
 fetch_oob_details

This makes the daemon handle the error nicer in fetch_oob_details()

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/main.py | 30 +++++++++++++--------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/main.py b/src/ceph-node-proxy/ceph_node_proxy/main.py
index 2a6479c4238e..9a449ecf8845 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/main.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/main.py
@@ -2,6 +2,7 @@
 from ceph_node_proxy.api import NodeProxyApi
 from ceph_node_proxy.reporter import Reporter
 from ceph_node_proxy.util import Config, get_logger, http_req, write_tmp_file, CONFIG
+from urllib.error import HTTPError
 from typing import Dict, Any, Optional
 
 import argparse
@@ -33,6 +34,8 @@ def __init__(self, **kw: Any) -> None:
         self.cephx = {'cephx': {'name': self.cephx_name,
                                 'secret': self.cephx_secret}}
         self.config = Config('/etc/ceph/node-proxy.yml', config=CONFIG)
+        self.username: str = ''
+        self.password: str = ''
 
     def run(self) -> None:
         self.init()
@@ -44,15 +47,16 @@ def init(self) -> None:
         self.init_api()
 
     def fetch_oob_details(self) -> Dict[str, str]:
-        headers, result, status = http_req(hostname=self.mgr_host,
-                                           port=self.mgr_agent_port,
-                                           data=json.dumps(self.cephx),
-                                           endpoint='/node-proxy/oob',
-                                           ssl_ctx=self.ssl_ctx)
-        if status != 200:
-            msg = f'No out of band tool details could be loaded: {status}, {result}'
+        try:
+            headers, result, status = http_req(hostname=self.mgr_host,
+                                               port=self.mgr_agent_port,
+                                               data=json.dumps(self.cephx),
+                                               endpoint='/node-proxy/oob',
+                                               ssl_ctx=self.ssl_ctx)
+        except HTTPError as e:
+            msg = f'No out of band tool details could be loaded: {e.code}, {e.reason}'
             self.log.debug(msg)
-            raise RuntimeError(msg)
+            raise
 
         result_json = json.loads(result)
         oob_details: Dict[str, str] = {
@@ -64,9 +68,13 @@ def fetch_oob_details(self) -> Dict[str, str]:
         return oob_details
 
     def init_system(self) -> None:
-        oob_details = self.fetch_oob_details()
-        self.username: str = oob_details['username']
-        self.password: str = oob_details['password']
+        try:
+            oob_details = self.fetch_oob_details()
+            self.username = oob_details['username']
+            self.password = oob_details['password']
+        except HTTPError:
+            self.log.warning('No oob details could be loaded, exiting...')
+            raise SystemExit(1)
         try:
             self.system = RedfishDellSystem(host=oob_details['host'],
                                             port=oob_details['port'],

From 42e3d217ffcd30f9ea7fd06684468fb87739cf97 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 12 Feb 2024 14:25:24 +0000
Subject: [PATCH 1895/2492] node-proxy: call time.sleep() after each reporter
 iteration

This makes sure time.sleep() is always call in the reporter loop.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/reporter.py b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
index 4e9c4e2798b3..20d43b59d332 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/reporter.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/reporter.py
@@ -63,7 +63,7 @@ def main(self) -> None:
                                 self.system.previous_data = self.system.get_system()
                         else:
                             self.log.debug('no diff, not sending data to the mgr.')
-                    time.sleep(5)
             self.log.debug('lock released in reporter loop.')
+            time.sleep(5)
         self.log.debug('exiting reporter loop.')
         raise SystemExit(0)

From 1a4a128d67774c59c073d2b98d53844c055a67e4 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Mon, 12 Feb 2024 16:20:42 +0000
Subject: [PATCH 1896/2492] mgr/cephadm: conditionally deploy node-proxy

Only attempt to deploy the node-proxy agent when oob details
are provided for a given host.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/serve.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index abb9310fa047..27775087d05d 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -888,6 +888,13 @@ def update_progress() -> None:
                         hosts_altered.add(d.hostname)
                         break
 
+                # do not attempt to deploy node-proxy agent when oob details are not provided.
+                if slot.daemon_type == 'node-proxy' and slot.hostname not in self.mgr.node_proxy_cache.oob.keys():
+                    self.log.debug(
+                        f'Not deploying node-proxy agent on {slot.hostname} as oob details are not present.'
+                    )
+                    continue
+
                 # deploy new daemon
                 daemon_id = slot.name
 

From 056d4f7a3d4ca8564b374880551a574858e68395 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 14 Feb 2024 09:02:50 +0000
Subject: [PATCH 1897/2492] mgr/cephadm: add fullreport in ceph orch CLI
 (node-proxy)

This adds the `fullreport` category to the `ceph orch hardware status` CLI.

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/pybind/mgr/cephadm/module.py          |  4 ++++
 src/pybind/mgr/orchestrator/_interface.py |  8 ++++++++
 src/pybind/mgr/orchestrator/module.py     | 10 ++++++++++
 3 files changed, 22 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 1451943ab28a..87f7024bb255 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1716,6 +1716,10 @@ def hardware_powercycle(self, hostname: str, yes_i_really_mean_it: bool = False)
             raise OrchestratorValidationError(f"Can't perform powercycle on node {hostname}: {e}")
         return f'Powercycle scheduled on {hostname}'
 
+    @handle_orch_error
+    def node_proxy_fullreport(self, hostname: Optional[str] = None) -> Dict[str, Any]:
+        return self.node_proxy_cache.fullreport(hostname=hostname)
+
     @handle_orch_error
     def node_proxy_summary(self, hostname: Optional[str] = None) -> Dict[str, Any]:
         return self.node_proxy_cache.summary(hostname=hostname)
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index ed8d47c8cac4..bc1721ab2511 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -403,6 +403,14 @@ def node_proxy_summary(self, hostname: Optional[str] = None) -> OrchResult[Dict[
         """
         raise NotImplementedError()
 
+    def node_proxy_fullreport(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
+        """
+        Return node-proxy full report
+
+        :param hostname: hostname
+        """
+        raise NotImplementedError()
+
     def node_proxy_firmwares(self, hostname: Optional[str] = None) -> OrchResult[Dict[str, Any]]:
         """
         Return node-proxy firmwares report
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index c8606a6286e4..c36c85e7d99e 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -497,6 +497,7 @@ def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int
         """
         table_heading_mapping = {
             'summary': ['HOST', 'STORAGE', 'CPU', 'NET', 'MEMORY', 'POWER', 'FANS'],
+            'fullreport': [],
             'firmwares': ['HOST', 'COMPONENT', 'NAME', 'DATE', 'VERSION', 'STATUS'],
             'criticals': ['HOST', 'COMPONENT', 'NAME', 'STATUS', 'STATE'],
             'memory': ['HOST', 'NAME', 'STATUS', 'STATE'],
@@ -525,6 +526,15 @@ def _hardware_status(self, hostname: Optional[str] = None, _end_positional_: int
                     row.extend([v['status'][key] for key in ['storage', 'processors', 'network', 'memory', 'power', 'fans']])
                     table.add_row(row)
                 output = table.get_string()
+        elif category == 'fullreport':
+            if hostname is None:
+                output = "Missing host name"
+            elif format != Format.json:
+                output = "fullreport only supports json output"
+            else:
+                completion = self.node_proxy_fullreport(hostname=hostname)
+                fullreport: Dict[str, Any] = raise_if_exception(completion)
+                output = json.dumps(fullreport)
         elif category == 'firmwares':
             output = "Missing host name" if hostname is None else self._firmwares_table(hostname, table, format)
         elif category == 'criticals':

From e2f8d35a9d0cebf47a4e235064a479d5e24348ee Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 12 Feb 2024 10:08:09 +0000
Subject: [PATCH 1898/2492] mgr/dashboard: added cephfs mount details

Fixes: https://tracker.ceph.com/issues/64405

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-list/cephfs-list.component.ts      | 39 ++++++++++++++++++-
 .../cephfs-mount-details.component.html       | 38 ++++++++++++++++++
 .../cephfs-mount-details.component.scss       |  0
 .../cephfs-mount-details.component.spec.ts    | 30 ++++++++++++++
 .../cephfs-mount-details.component.ts         | 37 ++++++++++++++++++
 ...ephfs-snapshotschedule-list.component.html | 16 ++++----
 .../cephfs-snapshotschedule-list.component.ts |  9 ++++-
 .../cephfs-subvolume-list.component.ts        | 29 +++++++++++++-
 .../src/app/ceph/cephfs/cephfs.module.ts      |  4 +-
 .../src/app/shared/api/cephfs.service.ts      |  4 ++
 .../code-block/code-block.component.html      |  8 ++--
 .../code-block/code-block.component.scss      |  4 +-
 .../code-block/code-block.component.ts        |  6 +++
 .../copy2clipboard-button.component.ts        |  2 +-
 .../src/app/shared/constants/app.constants.ts |  3 ++
 15 files changed, 208 insertions(+), 21 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
index 26e79727c12d..0943ed82574e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
@@ -21,6 +21,10 @@ import { ModalService } from '~/app/shared/services/modal.service';
 import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 import { FinishedTask } from '~/app/shared/models/finished-task';
 import { NotificationService } from '~/app/shared/services/notification.service';
+import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
+import { CephfsMountDetailsComponent } from '../cephfs-mount-details/cephfs-mount-details.component';
+import { map, switchMap } from 'rxjs/operators';
+import { HealthService } from '~/app/shared/api/health.service';
 
 const BASE_URL = 'cephfs';
 
@@ -38,6 +42,7 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
   permissions: Permissions;
   icons = Icons;
   monAllowPoolDelete = false;
+  modalRef!: NgbModalRef;
 
   constructor(
     private authStorageService: AuthStorageService,
@@ -48,7 +53,8 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
     private configurationService: ConfigurationService,
     private modalService: ModalService,
     private taskWrapper: TaskWrapperService,
-    public notificationService: NotificationService
+    public notificationService: NotificationService,
+    private healthService: HealthService
   ) {
     super();
     this.permissions = this.authStorageService.getPermissions();
@@ -89,6 +95,13 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
         click: () =>
           this.router.navigate([this.urlBuilder.getEdit(String(this.selection.first().id))])
       },
+      {
+        name: this.actionLabels.ATTACH,
+        permission: 'read',
+        icon: Icons.bars,
+        disable: () => !this.selection?.hasSelection,
+        click: () => this.showAttachInfo()
+      },
       {
         permission: 'delete',
         icon: Icons.destroy,
@@ -125,6 +138,30 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
     this.selection = selection;
   }
 
+  showAttachInfo() {
+    const selectedFileSystem = this.selection?.selected?.[0];
+
+    this.cephfsService
+      .getFsRootDirectory(selectedFileSystem.id)
+      .pipe(
+        switchMap((fsData) =>
+          this.healthService.getClusterFsid().pipe(map((data) => ({ clusterId: data, fs: fsData })))
+        )
+      )
+      .subscribe({
+        next: (val) => {
+          this.modalRef = this.modalService.show(CephfsMountDetailsComponent, {
+            onSubmit: () => this.modalRef.close(),
+            mountData: {
+              fsId: val.clusterId,
+              fsName: selectedFileSystem?.mdsmap?.fs_name,
+              rootPath: val.fs['path']
+            }
+          });
+        }
+      });
+  }
+
   removeVolumeModal() {
     const volName = this.selection.first().mdsmap['fs_name'];
     this.modalService.show(CriticalConfirmationModalComponent, {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.html
new file mode 100644
index 000000000000..a8c30afb1eba
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.html
@@ -0,0 +1,38 @@
+<cd-modal (hide)="cancel()">
+  <ng-container class="modal-title">
+    <span i18n>Attach commands</span>
+  </ng-container>
+  <ng-container class="modal-content">
+    <div class="modal-body">
+      <h5 class="fw-bold"
+          i18n>
+        Using Mount command
+      </h5>
+      <cd-code-block textWrap="true"
+                     [codes]="[mount]"></cd-code-block>
+
+      <h5 class="fw-bold"
+          i18n>
+        Using FUSE command
+      </h5>
+      <cd-code-block textWrap="true"
+                     [codes]="[fuse]"></cd-code-block>
+
+      <h5 class="fw-bold"
+          i18n>
+          Using NFS Command
+      </h5>
+      <cd-code-block textWrap="true"
+                     [codes]="[nfs]"></cd-code-block>
+    </div>
+    <div class="modal-footer">
+      <cd-submit-button (submitAction)="cancel()"
+                        i18n>
+        Close
+      </cd-submit-button>
+    </div>
+  </ng-container>
+</cd-modal>
+
+
+
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.spec.ts
new file mode 100644
index 000000000000..141ae428bdd7
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.spec.ts
@@ -0,0 +1,30 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsMountDetailsComponent } from './cephfs-mount-details.component';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { SharedModule } from '~/app/shared/shared.module';
+import { ToastrModule } from 'ngx-toastr';
+import { RouterTestingModule } from '@angular/router/testing';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { configureTestBed } from '~/testing/unit-test-helper';
+
+describe('CephfsSnapshotscheduleListComponent', () => {
+  let component: CephfsMountDetailsComponent;
+  let fixture: ComponentFixture<CephfsMountDetailsComponent>;
+
+  configureTestBed({
+    declarations: [CephfsMountDetailsComponent],
+    imports: [HttpClientTestingModule, SharedModule, ToastrModule.forRoot(), RouterTestingModule],
+    providers: [NgbActiveModal]
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(CephfsMountDetailsComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
new file mode 100644
index 000000000000..77a3f4afadc6
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
@@ -0,0 +1,37 @@
+import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+
+@Component({
+  selector: 'cd-cephfs-mount-details',
+  templateUrl: './cephfs-mount-details.component.html',
+  styleUrls: ['./cephfs-mount-details.component.scss']
+})
+export class CephfsMountDetailsComponent implements OnInit, OnDestroy {
+  @ViewChild('mountDetailsTpl', { static: true })
+  mountDetailsTpl: any;
+  onCancel?: Function;
+  private canceled = false;
+  private MOUNT_DIRECTORY = '<MOUNT_DIRECTORY>';
+  mountData!: Record<string, any>;
+  constructor(public activeModal: NgbActiveModal) {}
+  mount!: string;
+  fuse!: string;
+  nfs!: string;
+
+  ngOnInit(): void {
+    this.mount = `sudo <CLIENT_USER>@${this.mountData?.fsId}.${this.mountData?.fsName}=${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
+    this.fuse = `sudo ceph-fuse  ${this.MOUNT_DIRECTORY} -r ${this.mountData?.rootPath} --client_mds_namespace=${this.mountData?.fsName}`;
+    this.nfs = `sudo mount -t nfs -o port=<PORT> <IP of active_mds daemon>:${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
+  }
+
+  ngOnDestroy(): void {
+    if (this.onCancel && this.canceled) {
+      this.onCancel();
+    }
+  }
+
+  cancel() {
+    this.canceled = true;
+    this.activeModal.close();
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
index f26f63e755a6..4142724f7cfe 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.html
@@ -21,14 +21,14 @@
     class="fw-bold"
     [ngbTooltip]="fullpathTpl"
     triggers="click:blur">
-    {{ row.path?.split?.("@")?.[0] | path }}
+    {{ row.pathForSelection?.split?.("@")?.[0] | path }}
   </span>
 
   <span
   *ngIf="row.active; else inactiveStatusTpl">
     <i
       [ngClass]="[icons.success, icons.large]"
-      ngbTooltip="{{ row.path?.split?.('@')?.[0] }} is active"
+      ngbTooltip="{{ row.pathForSelection?.split?.('@')?.[0] }} is active"
       class="text-success"
     ></i>
   </span>
@@ -37,19 +37,19 @@
     <i
       [ngClass]="[icons.warning, icons.large]"
       class="text-warning"
-      ngbTooltip="{{ row.path?.split?.('@')?.[0] }} has been deactivated"
+      ngbTooltip="{{ row.pathForSelection?.split?.('@')?.[0] }} has been deactivated"
     ></i>
   </ng-template>
 
-  <ng-template #fullpathTpl>
+  <ng-template #fullpathForSelectionTpl>
     <span
       data-toggle="tooltip"
-      [title]="row.path"
+      [title]="row.pathForSelection"
       class="font-monospace"
-      >{{ row.path?.split?.("@")?.[0] }}
+      >{{ row.pathForSelection?.split?.("@")?.[0] }}
       <cd-copy-2-clipboard-button
-        *ngIf="row.path"
-        [source]="row.path?.split?.('@')?.[0]"
+        *ngIf="row.pathForSelection"
+        [source]="row.pathForSelection?.split?.('@')?.[0]"
         [byId]="false"
         [showIconOnly]="true"
       >
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index 581ee6e2fa3a..ecc2036d5dd3 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -130,7 +130,11 @@ export class CephfsSnapshotscheduleListComponent
             }
             return this.snapshotScheduleService
               .getSnapshotScheduleList('/', this.fsName)
-              .pipe(map((list) => list.map((l) => ({ ...l, path: `${l.path}@${l.schedule}` }))));
+              .pipe(
+                map((list) =>
+                  list.map((l) => ({ ...l, pathForSelection: `${l.path}@${l.schedule}` }))
+                )
+              );
           }),
           shareReplay(1)
         )
@@ -138,7 +142,8 @@ export class CephfsSnapshotscheduleListComponent
     );
 
     this.columns = [
-      { prop: 'path', name: $localize`Path`, flexGrow: 3, cellTemplate: this.pathTpl },
+      { prop: 'pathForSelection', name: $localize`Path`, flexGrow: 3, cellTemplate: this.pathTpl },
+      { prop: 'path', isHidden: true },
       { prop: 'subvol', name: $localize`Subvolume`, cellTemplate: this.subvolTpl },
       { prop: 'scheduleCopy', name: $localize`Repeat interval` },
       { prop: 'schedule', isHidden: true },
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
index 92c139f8e5dd..58d849c901ef 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolume-list/cephfs-subvolume-list.component.ts
@@ -31,6 +31,8 @@ import { CdForm } from '~/app/shared/forms/cd-form';
 import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
 import { CephfsSubvolumeGroupService } from '~/app/shared/api/cephfs-subvolume-group.service';
 import { CephfsSubvolumeGroup } from '~/app/shared/models/cephfs-subvolume-group.model';
+import { CephfsMountDetailsComponent } from '../cephfs-mount-details/cephfs-mount-details.component';
+import { HealthService } from '~/app/shared/api/health.service';
 
 @Component({
   selector: 'cd-cephfs-subvolume-list',
@@ -86,7 +88,8 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
     private modalService: ModalService,
     private authStorageService: AuthStorageService,
     private taskWrapper: TaskWrapperService,
-    private cephfsSubvolumeGroupService: CephfsSubvolumeGroupService
+    private cephfsSubvolumeGroupService: CephfsSubvolumeGroupService,
+    private healthService: HealthService
   ) {
     super();
     this.permissions = this.authStorageService.getPermissions();
@@ -149,6 +152,13 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
         icon: Icons.edit,
         click: () => this.openModal(true)
       },
+      {
+        name: this.actionLabels.ATTACH,
+        permission: 'read',
+        icon: Icons.bars,
+        disable: () => !this.selection?.hasSelection,
+        click: () => this.showAttachInfo()
+      },
       {
         name: this.actionLabels.REMOVE,
         permission: 'delete',
@@ -188,6 +198,23 @@ export class CephfsSubvolumeListComponent extends CdForm implements OnInit, OnCh
     this.selection = selection;
   }
 
+  showAttachInfo() {
+    const selectedSubVolume = this.selection?.selected?.[0];
+
+    this.healthService.getClusterFsid().subscribe({
+      next: (clusterId: string) => {
+        this.modalRef = this.modalService.show(CephfsMountDetailsComponent, {
+          onSubmit: () => this.modalRef.close(),
+          mountData: {
+            fsId: clusterId,
+            fsName: this.fsName,
+            rootPath: selectedSubVolume.info.path
+          }
+        });
+      }
+    });
+  }
+
   openModal(edit = false) {
     this.modalService.show(
       CephfsSubvolumeFormComponent,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index 687dd0b93ee9..14481d838224 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -30,6 +30,7 @@ import { CephfsSnapshotscheduleListComponent } from './cephfs-snapshotschedule-l
 import { DataTableModule } from '../../shared/datatable/datatable.module';
 import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component';
 import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component';
+import { CephfsMountDetailsComponent } from './cephfs-mount-details/cephfs-mount-details.component';
 
 @NgModule({
   imports: [
@@ -64,7 +65,8 @@ import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-f
     CephfsSubvolumeSnapshotsListComponent,
     CephfsSnapshotscheduleListComponent,
     CephfsSnapshotscheduleFormComponent,
-    CephfsSubvolumeSnapshotsFormComponent
+    CephfsSubvolumeSnapshotsFormComponent,
+    CephfsMountDetailsComponent
   ]
 })
 export class CephfsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
index d2dfbc0e2a76..ab43343f9edd 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
@@ -50,6 +50,10 @@ export class CephfsService {
     return this.http.get(`${this.baseURL}/${id}/mds_counters`);
   }
 
+  getFsRootDirectory(id: string) {
+    return this.http.get(`${this.baseURL}/${id}/get_root_directory`);
+  }
+
   mkSnapshot(id: number, path: string, name?: string) {
     let params = new HttpParams();
     params = params.append('path', path);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
index 7cf78b8d1fbe..47eac6364e48 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.html
@@ -1,8 +1,8 @@
 <ng-container *ngIf="codes.length > 1; else singleCodeBlock">
   <pre id="bigCodeBlock">
     <span *ngFor="let code of codes"
-          class="d-flex p-2 align-items-center justify-content-between text-dark">
-      <span>{{code}}</span>
+          class="d-flex px-2 py-3 align-items-center justify-content-between text-dark">
+      <span [ngClass]="{'text-wrap': textWrap}">{{code}}</span>
       <cd-copy-2-clipboard-button
           [source]="code"
           [byId]="false"></cd-copy-2-clipboard-button>
@@ -11,9 +11,9 @@
 </ng-container>
 
 <ng-template #singleCodeBlock>
-  <pre class="d-flex p-2 align-items-center justify-content-between text-dark"
+  <pre class="d-flex px-2 py-3 align-items-center justify-content-between text-dark"
        id="singleCodeBlock">
-    <span>{{codes}}</span>
+    <span [ngClass]="{'text-wrap': textWrap}">{{codes}}</span>
     <cd-copy-2-clipboard-button
       [source]="codes"
       [byId]="false"></cd-copy-2-clipboard-button>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
index f601dfe6609f..d22855f75198 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.scss
@@ -1,6 +1,4 @@
-@use './src/styles/vendor/variables' as vv;
-
 pre {
-  background-color: vv.$code-block-bg;
+  background-color: var(--gray-200);
   border-radius: 0.5rem;
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts
index 91d2d991f378..1021b8c97578 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/code-block/code-block.component.ts
@@ -8,4 +8,10 @@ import { Component, Input } from '@angular/core';
 export class CodeBlockComponent {
   @Input()
   codes: string[];
+
+  @Input()
+  textWrap: boolean = false;
+
+  @Input()
+  grayBg: boolean = false;
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/copy2clipboard-button/copy2clipboard-button.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/copy2clipboard-button/copy2clipboard-button.component.ts
index 80c7acbf28ae..b6b8ca77e8ad 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/copy2clipboard-button/copy2clipboard-button.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/copy2clipboard-button/copy2clipboard-button.component.ts
@@ -26,7 +26,7 @@ export class Copy2ClipboardButtonComponent {
 
   private getText(): string {
     const element = document.getElementById(this.source) as HTMLInputElement;
-    return element.value;
+    return element?.value || element?.textContent;
   }
 
   @HostListener('click')
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index 2cf3f1047bab..fc32efd1d38a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -143,6 +143,7 @@ export class ActionLabelsI18n {
   START_UPGRADE: string;
   ACTIVATE: string;
   DEACTIVATE: string;
+  ATTACH: string;
 
   constructor() {
     /* Create a new item */
@@ -223,6 +224,8 @@ export class ActionLabelsI18n {
 
     this.ACTIVATE = $localize`Activate`;
     this.DEACTIVATE = $localize`Deactivate`;
+
+    this.ATTACH = $localize`Attach`;
   }
 }
 

From a8b27af84910d324023a37e616f42d6d3faa4ecf Mon Sep 17 00:00:00 2001
From: Piotr Parczewski <piotr@stackhpc.com>
Date: Wed, 14 Feb 2024 11:15:09 +0100
Subject: [PATCH 1899/2492] doc/start: fix wording & syntax

Signed-off-by: Piotr Parczewski <piotr@stackhpc.com>
---
 doc/start/hardware-recommendations.rst | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
index 76dce78013ce..196114b5445c 100644
--- a/doc/start/hardware-recommendations.rst
+++ b/doc/start/hardware-recommendations.rst
@@ -38,7 +38,7 @@ clusters and up to about fourteen cores on single OSDs in isolation. So cores
 per OSD are no longer as pressing a concern as they were. When selecting
 hardware, select for IOPS per core.
 
-.. tip:: When we speak of CPU _cores_, we mean _threads_ when hyperthreading
+.. tip:: When we speak of CPU *cores*, we mean *threads* when hyperthreading
 	 is enabled.  Hyperthreading is usually beneficial for Ceph servers.
 
 Monitor nodes and Manager nodes do not have heavy CPU demands and require only
@@ -447,10 +447,11 @@ Additional Considerations
 Ceph operators typically provision  multiple OSDs per host, but you should
 ensure that the aggregate throughput of your OSD drives doesn't exceed the
 network bandwidth required to service a client's read and write operations.
-You should also each host's percentage of the cluster's overall capacity. If
-the percentage located on a particular host is large and the host fails, it
-can lead to problems such as recovery causing OSDs to exceed the ``full ratio``,
-which in turn causes Ceph to halt operations to prevent data loss.
+You should also consider each host's percentage of the cluster's overall
+capacity. If the percentage located on a particular host is large and the host
+fails, it can lead to problems such as recovery causing OSDs to exceed the
+``full ratio``, which in turn causes Ceph to halt operations to prevent data
+loss.
 
 When you run multiple OSDs per host, you also need to ensure that the kernel
 is up to date. See `OS Recommendations`_ for notes on ``glibc`` and
@@ -539,7 +540,7 @@ Minimum Hardware Recommendations
 
 Ceph can run on inexpensive commodity hardware. Small production clusters
 and development clusters can run successfully with modest hardware.  As
-we noted above: when we speak of CPU _cores_, we mean _threads_ when
+we noted above: when we speak of CPU *cores*, we mean *threads* when
 hyperthreading (HT) is enabled.  Each modern physical x64 CPU core typically
 provides two logical CPU threads; other CPU architectures may vary.
 

From f9619805cb357c34e96b06a52dfd75ee8815da22 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 11 Feb 2024 19:53:44 +0200
Subject: [PATCH 1900/2492] doc/osd: fix 'scrub a PG' commands in the
 documentation

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 doc/rados/operations/placement-groups.rst | 44 ++++++++++++++++++-----
 1 file changed, 35 insertions(+), 9 deletions(-)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index fb21863bb098..45830613b17b 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -747,22 +747,48 @@ To see statistics for a particular PG, run a command of the following form:
 
    ceph pg {pg-id} query
 
-
 Scrub a PG
 ==========
 
-To scrub a PG, run a command of the following form:
+To force an immediate scrub of a PG, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph tell {pg-id} scrub
+
+or
+
+.. prompt:: bash #
+
+   ceph tell {pg-id} deep-scrub
+
+Ceph checks the primary and replica OSDs and generates a catalog of all objects in
+the PG. For each object, Ceph compares all instances of the object (in the primary
+and replica OSDs) to ensure that they are consistent. For shallow scrubs (initiated
+by the first command format), only object metadata is compared. Deep scrubs
+(initiated by the second command format) compare the contents of the objects as well.
+If the replicas all match, a final semantic sweep takes place to ensure that all
+snapshot-related object metadata is consistent.  Errors are reported in logs.
+
+Scrubs initiated using the command format above are deemed high priority, and
+are performed immediately. Such scrubs are not subject to any day-of-week or
+time-of-day restrictions that are in effect for regular, periodic, scrubs.
+They are not limited by 'osd_max_scrubs', and are not required to wait for
+their replicas' scrub resources.
+
+A second command format exists for initiating a scrub as-if it were a regular
+scrub. This command format is as follows:
+
+.. prompt:: bash #
+
+   ceph tell {pg-id} schedule-scrub
+
+or
 
 .. prompt:: bash #
 
-   ceph pg scrub {pg-id}
+   ceph tell {pg-id} schedule-deep-scrub
 
-Ceph checks the primary and replica OSDs, generates a catalog of all objects in
-the PG, and compares the objects against each other in order to ensure that no
-objects are missing or mismatched and that their contents are consistent. If
-the replicas all match, then a final semantic sweep takes place to ensure that
-all snapshot-related object metadata is consistent.  Errors are reported in
-logs.
 
 To scrub all PGs from a specific pool, run a command of the following form:
 

From 746cb28449903ceec2fe3ffcfa5e925eb78eb7b2 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Mon, 12 Feb 2024 11:00:45 +0100
Subject: [PATCH 1901/2492] librbd: fix split() for SparseExtent and
 SparseBufferlistExtent

SparseExtents and SparseBufferlist are typedefs for interval_map.  In
both cases, split() handler is broken: for the former the extent isn't
actually split and for the latter incorrect bufferlist is attached to
the split extent.

Fortunately, both SnapshotDelta as produced by ObjectListSnapsRequest
and SparseBufferlist used in a couple of places seem to be collections
where only disjoint intervals are inserted and splitting doesn't occur
(at least in the common case).  But still, this is a landmine waiting
for someone to step on it.

Fixes: https://tracker.ceph.com/issues/64423
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/io/Types.h                         |  9 +-
 src/test/librbd/io/test_mock_ObjectRequest.cc | 91 +++++++++++++++++++
 2 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/src/librbd/io/Types.h b/src/librbd/io/Types.h
index ad032a853345..c720a91e98da 100644
--- a/src/librbd/io/Types.h
+++ b/src/librbd/io/Types.h
@@ -179,8 +179,9 @@ struct SparseExtent {
 std::ostream& operator<<(std::ostream& os, const SparseExtent& state);
 
 struct SparseExtentSplitMerge {
-  SparseExtent split(uint64_t offset, uint64_t length, SparseExtent &se) const {
-    return SparseExtent(se.state, se.length);
+  SparseExtent split(uint64_t offset, uint64_t length,
+                     const SparseExtent& se) const {
+    return SparseExtent(se.state, length);
   }
 
   bool can_merge(const SparseExtent& left, const SparseExtent& right) const {
@@ -231,10 +232,10 @@ struct SparseBufferlistExtent : public SparseExtent {
 
 struct SparseBufferlistExtentSplitMerge {
   SparseBufferlistExtent split(uint64_t offset, uint64_t length,
-                               SparseBufferlistExtent& sbe) const {
+                               const SparseBufferlistExtent& sbe) const {
     ceph::bufferlist bl;
     if (sbe.state == SPARSE_EXTENT_STATE_DATA) {
-      bl.substr_of(bl, offset, length);
+      bl.substr_of(sbe.bl, offset, length);
     }
     return SparseBufferlistExtent(sbe.state, length, std::move(bl));
   }
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index d7f9156d7eda..5ea3aed923a8 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -2070,6 +2070,97 @@ TEST_F(TestMockIoObjectRequest, ListSnapsNoSnapsInSnapSet) {
   EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
 }
 
+TEST(SparseExtents, Split) {
+  SparseExtents extents;
+  extents.insert(50, 100, {SPARSE_EXTENT_STATE_DATA, 100});
+  extents.erase(80, 30);
+  extents.insert(45, 10, {SPARSE_EXTENT_STATE_ZEROED, 10});
+  extents.insert(140, 20, {SPARSE_EXTENT_STATE_DNE, 20});
+  extents.insert(125, 5, {SPARSE_EXTENT_STATE_ZEROED, 5});
+
+  SparseExtents expected_extents = {
+    {45, {10, {SPARSE_EXTENT_STATE_ZEROED, 10}}},
+    {55, {25, {SPARSE_EXTENT_STATE_DATA, 25}}},
+    {110, {15, {SPARSE_EXTENT_STATE_DATA, 15}}},
+    {125, {5, {SPARSE_EXTENT_STATE_ZEROED, 5}}},
+    {130, {10, {SPARSE_EXTENT_STATE_DATA, 10}}},
+    {140, {20, {SPARSE_EXTENT_STATE_DNE, 20}}}
+  };
+  EXPECT_EQ(expected_extents, extents);
+}
+
+TEST(SparseBufferlist, Split) {
+  bufferlist bl;
+  bl.append(std::string(5, '1'));
+  bl.append(std::string(25, '2'));
+  bl.append(std::string(30, '3'));
+  bl.append(std::string(15, '4'));
+  bl.append(std::string(5, '5'));
+  bl.append(std::string(10, '6'));
+  bl.append(std::string(10, '7'));
+  bufferlist expected_bl1;
+  expected_bl1.append(std::string(25, '2'));
+  bufferlist expected_bl2;
+  expected_bl2.append(std::string(15, '4'));
+  bufferlist expected_bl3;
+  expected_bl3.append(std::string(10, '6'));
+
+  SparseBufferlist extents;
+  extents.insert(50, 100, {SPARSE_EXTENT_STATE_DATA, 100, std::move(bl)});
+  extents.erase(80, 30);
+  extents.insert(45, 10, {SPARSE_EXTENT_STATE_ZEROED, 10});
+  extents.insert(140, 20, {SPARSE_EXTENT_STATE_DNE, 20});
+  extents.insert(125, 5, {SPARSE_EXTENT_STATE_ZEROED, 5});
+
+  SparseBufferlist expected_extents = {
+    {45, {10, {SPARSE_EXTENT_STATE_ZEROED, 10}}},
+    {55, {25, {SPARSE_EXTENT_STATE_DATA, 25, std::move(expected_bl1)}}},
+    {110, {15, {SPARSE_EXTENT_STATE_DATA, 15, std::move(expected_bl2)}}},
+    {125, {5, {SPARSE_EXTENT_STATE_ZEROED, 5}}},
+    {130, {10, {SPARSE_EXTENT_STATE_DATA, 10, std::move(expected_bl3)}}},
+    {140, {20, {SPARSE_EXTENT_STATE_DNE, 20}}}
+  };
+  EXPECT_EQ(expected_extents, extents);
+}
+
+TEST(SparseBufferlist, SplitData) {
+  bufferlist bl1;
+  bl1.append(std::string(100, '1'));
+  bufferlist bl2;
+  bl2.append(std::string(15, '2'));
+  bufferlist bl3;
+  bl3.append(std::string(40, '3'));
+  bufferlist bl4;
+  bl4.append(std::string(10, '4'));
+  bufferlist expected_bl1 = bl2;
+  bufferlist expected_bl2;
+  expected_bl2.append(std::string(35, '1'));
+  bufferlist expected_bl3 = bl4;
+  bufferlist expected_bl4;
+  expected_bl4.append(std::string(30, '1'));
+  bufferlist expected_bl5;
+  expected_bl5.append(std::string(5, '3'));
+  bufferlist expected_bl6;
+  expected_bl6.append(std::string(15, '3'));
+
+  SparseBufferlist extents;
+  extents.insert(50, 100, {SPARSE_EXTENT_STATE_DATA, 100, std::move(bl1)});
+  extents.insert(40, 15, {SPARSE_EXTENT_STATE_DATA, 15, std::move(bl2)});
+  extents.insert(130, 40, {SPARSE_EXTENT_STATE_DATA, 40, std::move(bl3)});
+  extents.erase(135, 20);
+  extents.insert(90, 10, {SPARSE_EXTENT_STATE_DATA, 10, std::move(bl4)});
+
+  SparseBufferlist expected_extents = {
+    {40, {15, {SPARSE_EXTENT_STATE_DATA, 15, std::move(expected_bl1)}}},
+    {55, {35, {SPARSE_EXTENT_STATE_DATA, 35, std::move(expected_bl2)}}},
+    {90, {10, {SPARSE_EXTENT_STATE_DATA, 10, std::move(expected_bl3)}}},
+    {100, {30, {SPARSE_EXTENT_STATE_DATA, 30, std::move(expected_bl4)}}},
+    {130, {5, {SPARSE_EXTENT_STATE_DATA, 5, std::move(expected_bl5)}}},
+    {155, {15, {SPARSE_EXTENT_STATE_DATA, 15, std::move(expected_bl6)}}}
+  };
+  EXPECT_EQ(expected_extents, extents);
+}
+
 } // namespace io
 } // namespace librbd
 

From 69c18cfd291e63fbcd261d3cf20be8dfa0c7488d Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Mon, 12 Feb 2024 13:07:22 +0100
Subject: [PATCH 1902/2492] librbd: refactor merge() for SparseBufferlistExtent

- pass left.length + right.length instead of bl.length()
  for consistency and to avoid circumventing the assert in
  SparseBufferlistExtent constructor
- claim_append() takes an lvalue reference, no need to move
- follow the pattern used in split()

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/io/Types.h                         | 11 ++---
 src/test/librbd/io/test_mock_ObjectRequest.cc | 49 +++++++++++++++++++
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/src/librbd/io/Types.h b/src/librbd/io/Types.h
index c720a91e98da..0f9afd07e82f 100644
--- a/src/librbd/io/Types.h
+++ b/src/librbd/io/Types.h
@@ -247,14 +247,13 @@ struct SparseBufferlistExtentSplitMerge {
 
   SparseBufferlistExtent merge(SparseBufferlistExtent&& left,
                                SparseBufferlistExtent&& right) const {
+    ceph::bufferlist bl;
     if (left.state == SPARSE_EXTENT_STATE_DATA) {
-      ceph::bufferlist bl{std::move(left.bl)};
-      bl.claim_append(std::move(right.bl));
-      return SparseBufferlistExtent(SPARSE_EXTENT_STATE_DATA,
-                                    bl.length(), std::move(bl));
-    } else {
-      return SparseBufferlistExtent(left.state, left.length + right.length, {});
+      bl.claim_append(left.bl);
+      bl.claim_append(right.bl);
     }
+    return SparseBufferlistExtent(left.state, left.length + right.length,
+                                  std::move(bl));
   }
 
   uint64_t length(const SparseBufferlistExtent& sbe) const {
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index 5ea3aed923a8..50ee5b718fad 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -2089,6 +2089,25 @@ TEST(SparseExtents, Split) {
   EXPECT_EQ(expected_extents, extents);
 }
 
+TEST(SparseExtents, Merge) {
+  SparseExtents extents;
+  extents.insert(50, 100, {SPARSE_EXTENT_STATE_DATA, 100});
+  extents.insert(30, 15, {SPARSE_EXTENT_STATE_ZEROED, 15});
+  extents.insert(45, 10, {SPARSE_EXTENT_STATE_DATA, 10});
+  extents.insert(200, 40, {SPARSE_EXTENT_STATE_DNE, 40});
+  extents.insert(160, 25, {SPARSE_EXTENT_STATE_DNE, 25});
+  extents.insert(140, 20, {SPARSE_EXTENT_STATE_DATA, 20});
+  extents.insert(25, 5, {SPARSE_EXTENT_STATE_ZEROED, 5});
+  extents.insert(185, 15, {SPARSE_EXTENT_STATE_DNE, 15});
+
+  SparseExtents expected_extents = {
+    {25, {20, {SPARSE_EXTENT_STATE_ZEROED, 20}}},
+    {45, {115, {SPARSE_EXTENT_STATE_DATA, 115}}},
+    {160, {80, {SPARSE_EXTENT_STATE_DNE, 80}}}
+  };
+  EXPECT_EQ(expected_extents, extents);
+}
+
 TEST(SparseBufferlist, Split) {
   bufferlist bl;
   bl.append(std::string(5, '1'));
@@ -2161,6 +2180,36 @@ TEST(SparseBufferlist, SplitData) {
   EXPECT_EQ(expected_extents, extents);
 }
 
+TEST(SparseBufferlist, Merge) {
+  bufferlist bl1;
+  bl1.append(std::string(100, '1'));
+  bufferlist bl2;
+  bl2.append(std::string(10, '2'));
+  bufferlist bl3;
+  bl3.append(std::string(20, '3'));
+  bufferlist expected_bl;
+  expected_bl.append(std::string(10, '2'));
+  expected_bl.append(std::string(85, '1'));
+  expected_bl.append(std::string(20, '3'));
+
+  SparseBufferlist extents;
+  extents.insert(50, 100, {SPARSE_EXTENT_STATE_DATA, 100, std::move(bl1)});
+  extents.insert(30, 15, {SPARSE_EXTENT_STATE_ZEROED, 15});
+  extents.insert(45, 10, {SPARSE_EXTENT_STATE_DATA, 10, std::move(bl2)});
+  extents.insert(200, 40, {SPARSE_EXTENT_STATE_DNE, 40});
+  extents.insert(160, 25, {SPARSE_EXTENT_STATE_DNE, 25});
+  extents.insert(140, 20, {SPARSE_EXTENT_STATE_DATA, 20, std::move(bl3)});
+  extents.insert(25, 5, {SPARSE_EXTENT_STATE_ZEROED, 5});
+  extents.insert(185, 15, {SPARSE_EXTENT_STATE_DNE, 15});
+
+  SparseBufferlist expected_extents = {
+    {25, {20, {SPARSE_EXTENT_STATE_ZEROED, 20}}},
+    {45, {115, {SPARSE_EXTENT_STATE_DATA, 115, std::move(expected_bl)}}},
+    {160, {80, {SPARSE_EXTENT_STATE_DNE, 80}}}
+  };
+  EXPECT_EQ(expected_extents, extents);
+}
+
 } // namespace io
 } // namespace librbd
 

From bbbf5c54bdce3061ae9831c49fe065311a2e156d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 14 Feb 2024 09:43:14 -0500
Subject: [PATCH 1903/2492] rgw/putobj: RadosWriter uses part head object for
 multipart parts

the cleanup logic in the RadosWrite destructor was using the wrong
`head_obj` to avoid races between cleanup and part re-uploads. it
pointed at the final location of the multipart upload, rather than the
head object of the current part

Fixes: https://tracker.ceph.com/issues/63642

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_putobj_processor.cc | 8 ++++++++
 src/rgw/driver/rados/rgw_putobj_processor.h  | 5 ++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index d7462587c87c..846e2323caf7 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -124,6 +124,11 @@ void RadosWriter::add_write_hint(librados::ObjectWriteOperation& op) {
   op.set_alloc_hint2(0, 0, alloc_hint_flags);
 }
 
+void RadosWriter::set_head_obj(const rgw_obj& head)
+{
+  head_obj = head;
+}
+
 int RadosWriter::set_stripe_obj(const rgw_raw_obj& raw_obj)
 {
   return rgw_get_rados_ref(dpp, store->get_rados_handle(), raw_obj,
@@ -458,6 +463,9 @@ int MultipartObjectProcessor::prepare_head()
   RGWSI_Tier_RADOS::raw_obj_to_obj(head_obj.bucket, stripe_obj, &head_obj);
   head_obj.index_hash_source = target_obj.key.name;
 
+  // point part uploads at the part head instead of the final multipart head
+  writer.set_head_obj(head_obj);
+
   r = writer.set_stripe_obj(stripe_obj);
   if (r < 0) {
     return r;
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index 53b14c41ee08..17388f3fbaf3 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -68,7 +68,7 @@ class RadosWriter : public rgw::sal::DataProcessor {
   RGWRados *const store;
   const RGWBucketInfo& bucket_info;
   RGWObjectCtx& obj_ctx;
-  const rgw_obj head_obj;
+  rgw_obj head_obj;
   rgw_rados_ref stripe_obj; // current stripe object
   RawObjSet written; // set of written objects for deletion
   const DoutPrefixProvider *dpp;
@@ -87,6 +87,9 @@ class RadosWriter : public rgw::sal::DataProcessor {
   // add alloc hint to osd
   void add_write_hint(librados::ObjectWriteOperation& op);
 
+  // change the head object
+  void set_head_obj(const rgw_obj& head);
+
   // change the current stripe object
   int set_stripe_obj(const rgw_raw_obj& obj);
 

From 9344893e0857be8c6ae134f6d4d5e9057e2cb99e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 15 Feb 2024 01:32:40 +1000
Subject: [PATCH 1904/2492] doc/rados: add link to pg blog post

Add a link in the "Placement Groups" section that links to Sage Weil's
April 5, 2019 blog post "New in Nautilus: PG merging and autotuning".

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/placement-groups.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index fb21863bb098..2e2e385a5161 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -4,6 +4,11 @@
  Placement Groups
 ==================
 
+See Sage Weil's blog post `New in Nautilus: PG merging and autotuning
+<https://ceph.io/en/news/blog/2019/new-in-nautilus-pg-merging-and-autotuning/>`_
+for information about the relationship of placement groups to pools and to
+objects.
+
 .. _pg-autoscaler:
 
 Autoscaling placement groups

From 46bddfa6c4bb3be0ce12b243961f76ddb002cc23 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 14 Feb 2024 10:37:24 -0500
Subject: [PATCH 1905/2492] script: add squid to backports

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/script/backport-create-issue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/script/backport-create-issue b/src/script/backport-create-issue
index 9dfec4d7d1d6..9fb627c6fac7 100755
--- a/src/script/backport-create-issue
+++ b/src/script/backport-create-issue
@@ -137,7 +137,7 @@ def releases():
     return ('argonaut', 'bobtail', 'cuttlefish', 'dumpling', 'emperor',
             'firefly', 'giant', 'hammer', 'infernalis', 'jewel', 'kraken',
             'luminous', 'mimic', 'nautilus', 'octopus', 'pacific', 'quincy',
-            'reef')
+            'reef', 'squid')
 
 def populate_status_dict(r):
     for status in r.issue_status.all():

From 88ab55e60be8ad5cd122e7dfdf1b321de2794a48 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Thu, 8 Feb 2024 14:17:39 -0500
Subject: [PATCH 1906/2492] rgw_sigv4: catch exceptions on client builder

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 .../test/java/io/ceph/jcksum/PutObjects.java  | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
index 685914abee6d..5a3b788b924d 100644
--- a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
+++ b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
@@ -100,7 +100,6 @@ void generateFile(String in_file_path, String out_file_path, long length) {
 			}
 		} catch (IOException e) {
             System.err.println(e.getMessage());
-            System.exit(1);
 		}
 	} /* generateFile */
 
@@ -146,12 +145,18 @@ void setup() throws URISyntaxException {
 	            .buildWithDefaults(AttributeMap.builder().put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true).build());
 		
 		/* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */
-        client = S3Client.builder()
-          .endpointOverride(http_uri)
-          .credentialsProvider(StaticCredentialsProvider.create(creds))
-          .region(jcksum.region)
-          .forcePathStyle(true)
-          .build();
+
+    try {
+      client = S3Client.builder()
+        .endpointOverride(http_uri)
+        .credentialsProvider(StaticCredentialsProvider.create(creds))
+        .region(jcksum.region)
+        .forcePathStyle(true) /* XXX change in future */
+        .build();
+		} catch (Exception e) {
+      System.err.println(e.getMessage());
+      System.exit(1);
+		}
 
     generateBigFiles();
 

From 639ea51a716b9c37c3dd95d8c1264a95584e9924 Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Mon, 5 Feb 2024 11:52:38 -0500
Subject: [PATCH 1907/2492] qa: enable test_awssdkv4_sig workunit to run in
 teuthology

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 qa/suites/rgw/verify/tasks/cls.yaml           |  2 +-
 qa/tasks/rgw.py                               | 11 ++++
 .../test/java/io/ceph/jcksum/PutObjects.java  |  7 +++
 qa/workunits/rgw/test_awssdkv4_sig.sh         | 54 ++++++++++---------
 4 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/qa/suites/rgw/verify/tasks/cls.yaml b/qa/suites/rgw/verify/tasks/cls.yaml
index 71cca69d603e..568395a0dd0e 100644
--- a/qa/suites/rgw/verify/tasks/cls.yaml
+++ b/qa/suites/rgw/verify/tasks/cls.yaml
@@ -13,4 +13,4 @@ tasks:
         - rgw/test_rgw_gc_log.sh
         - rgw/test_rgw_obj.sh
         - rgw/test_librgw_file.sh
-	- rgw/test_awssdkv4_sig.sh
+        - rgw/test_awssdkv4_sig.sh
diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py
index 1adf7f7ee14c..e733b73c61e4 100644
--- a/qa/tasks/rgw.py
+++ b/qa/tasks/rgw.py
@@ -61,6 +61,16 @@ def start_rgw(ctx, config, clients):
         log.info("Using %s as radosgw frontend", ctx.rgw.frontend)
 
         endpoint = ctx.rgw.role_endpoints[client]
+
+        # create a file with rgw endpoint in it for test_awssdkv4 workunit
+        url = endpoint.url()
+        # remove trailing slash from the url
+        if url[-1] == '/':
+            url = url[:-1]
+        url_file = '{tdir}/url_file'.format(tdir=testdir)
+        ctx.cluster.only(client).run(args=['sudo', 'echo', '-n', '{url}'.format(url=url), run.Raw('|'), 'sudo', 'tee', url_file])
+        ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', url_file])
+
         frontends = ctx.rgw.frontend
         frontend_prefix = client_config.get('frontend_prefix', None)
         if frontend_prefix:
@@ -239,6 +249,7 @@ def start_rgw(ctx, config, clients):
                     ],
                 )
             ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', token_path])
+            ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', url_file])
             rgwadmin(ctx, client, cmd=['gc', 'process', '--include-all'], check_status=True)
 
 def assign_endpoints(ctx, config, default_cert):
diff --git a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
index 5a3b788b924d..9f9d3475c04f 100644
--- a/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
+++ b/qa/workunits/rgw/jcksum/src/test/java/io/ceph/jcksum/PutObjects.java
@@ -50,6 +50,7 @@ class PutObjects {
 	
 	void generateFile(String in_file_path, String out_file_path, long length) {
 		try {
+                        System.out.println("DEBUG: Generating File");
 			Path ifp = Paths.get(in_file_path);
 			File f = ifp.toFile();
 
@@ -98,6 +99,7 @@ void generateFile(String in_file_path, String out_file_path, long length) {
 			if (wch != null) {
 				wch.close();
 			}
+                        System.out.println("DEBUG: File Generated");
 		} catch (IOException e) {
             System.err.println(e.getMessage());
 		}
@@ -146,6 +148,7 @@ void setup() throws URISyntaxException {
 		
 		/* https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html */
 
+    System.out.println("DEBUG: Environment Variables Read");
     try {
       client = S3Client.builder()
         .endpointOverride(http_uri)
@@ -158,7 +161,9 @@ void setup() throws URISyntaxException {
       System.exit(1);
 		}
 
+    System.out.println("DEBUG: S3 Client Initialized");
     generateBigFiles();
+    System.out.println("DEBUG: Generated Big Files");
 
     /* create test bucket if it doesn't exist yet */
 		try {
@@ -167,6 +172,8 @@ void setup() throws URISyntaxException {
             System.err.println(e.getMessage());
             System.exit(1);
 		}
+
+    System.out.println("DEBUG: Test Bucket Created");
   } /* setup */
 
 	/* TODO: zap */
diff --git a/qa/workunits/rgw/test_awssdkv4_sig.sh b/qa/workunits/rgw/test_awssdkv4_sig.sh
index 5c56a46285f6..0f4782260c49 100755
--- a/qa/workunits/rgw/test_awssdkv4_sig.sh
+++ b/qa/workunits/rgw/test_awssdkv4_sig.sh
@@ -1,45 +1,47 @@
-#!/bin/sh -e
+#!/bin/sh
 #
 # To run this test script with a cluster created via vstart.sh:
-# $PATH needs to be set for radosgw-admin and ceph_test_librgw executables.
-# $KEYRING need to be set as the path for a vstart clusters Ceph keyring.
+# $PATH needs to be set for radosgw-admin executables.
+# $CEPH_ROOT needs to be set to the path of the Ceph source code
+# $RGW_HTTP_ENDPOINT_URL needs to be set to the endpoint of the RGW
 #
 # Example when ceph source is cloned into $HOME and a vstart cluster is already running with a radosgw:
-# $ PATH=~/ceph/build/bin/:$PATH KEYRING=~/ceph/build/keyring ~/ceph/qa/workunits/rgw/test_awssdkv4_sig.sh
+# $ PATH=~/ceph/build/bin/:$PATH CEPH_ROOT=~/ceph RGW_HTTP_ENDPOINT=http://localhost:8000 ~/ceph/qa/workunits/rgw/test_awssdkv4_sig.sh
+#
+
+set -x
 
 if [ -z ${AWS_ACCESS_KEY_ID} ]
 then
-    export AWS_ACCESS_KEY_ID=`openssl rand -base64 20`
-    export AWS_SECRET_ACCESS_KEY=`openssl rand -base64 40`
+    export AWS_ACCESS_KEY_ID="lNCnR47C2g+ZidCWBAUuwfSAA7Q="
+    export AWS_SECRET_ACCESS_KEY="tYuA2Y+Uu1ow2l9Xe59tWKVml3gMuVfyhUjjJwfwEI0vFFONIcqf4g=="
 
     radosgw-admin user create --uid ceph-test-maven \
        --access-key $AWS_ACCESS_KEY_ID \
        --secret $AWS_SECRET_ACCESS_KEY \
        --display-name "maven test user" \
        --email sigv4@example.com || echo "sigv4 maven user exists"
-
-    # keyring override for teuthology env
-    if [ -z ${KEYRING} ]
-    then
-      KEYRING="/etc/ceph/ceph.keyring"
-    fi
-    K="-k ${KEYRING}"
 fi
 
-# the required S3 access_key and secret_key are already exported above, but
-# we need to hook up the S3 endpoints
-
-# the following are taken from
-
-# XXXX ok, so I think there should be only RGW_HTTP_ENDPOINT_URL and that
-# it may already be set before this script runs, though if we can figure it
-# out, it's ok if we set it...
-
-export RGW_HTTP_ENDPOINT_URL="http://localhost:80"
-#export RGW_HTTPS_ENDPOINT_URL="https://localhost:443"
+if [ -z ${RGW_HTTP_ENDPOINT_URL} ]
+then
+  # TESTDIR and this block are meant for when this script is run in a teuthology environment
+  if [ -z ${TESTDIR} ]
+  then
+    echo "TESTDIR is not defined, cannot set RGW_HTTP_ENDPOINT_URL in teuthology"
+    exit
+  else
+    export RGW_HTTP_ENDPOINT_URL=$(cat ${TESTDIR}/url_file)
+  fi
+fi
 
-# rgw/test_awssdkv4_sig.sh
-pushd jcksum
+if [ -z ${CEPH_ROOT} ]
+then
+  echo "CEPH_ROOT is not defined"
+  exit
+else
+  cd $CEPH_ROOT/qa/workunits/rgw/jcksum
+fi
 
 ./mvnw clean package
 ./mvnw test -Dtest=PutObjects

From ccbb1f5aa9c970ca1d2a15690543f8039566d2ae Mon Sep 17 00:00:00 2001
From: Ali Maredia <amaredia@redhat.com>
Date: Mon, 12 Feb 2024 18:25:17 -0500
Subject: [PATCH 1908/2492] qa: move cert import into rgw task

Signed-off-by: Ali Maredia <amaredia@redhat.com>
---
 qa/tasks/rgw.py          | 16 ++++++++++++++++
 qa/tasks/s3tests_java.py | 17 -----------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py
index e733b73c61e4..fef38e505f30 100644
--- a/qa/tasks/rgw.py
+++ b/qa/tasks/rgw.py
@@ -1,6 +1,7 @@
 """
 rgw routines
 """
+from io import BytesIO
 import argparse
 import contextlib
 import logging
@@ -80,6 +81,21 @@ def start_rgw(ctx, config, clients):
             # add the ssl certificate path
             frontends += ' ssl_certificate={}'.format(endpoint.cert.certificate)
             frontends += ' ssl_port={}'.format(endpoint.port)
+            path = 'lib/security/cacerts'
+            ctx.cluster.only(client).run(
+                args=['sudo',
+                      'keytool',
+                      '-import', '-alias', '{alias}'.format(
+                          alias=endpoint.hostname),
+                      '-keystore',
+                      run.Raw(
+                          '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path=path)),
+                      '-file', endpoint.cert.certificate,
+                      '-storepass', 'changeit',
+                      ],
+                stdout=BytesIO()
+            )
+
         else:
             frontends += ' port={}'.format(endpoint.port)
 
diff --git a/qa/tasks/s3tests_java.py b/qa/tasks/s3tests_java.py
index 78a1edb6f613..3e20e10d06ca 100644
--- a/qa/tasks/s3tests_java.py
+++ b/qa/tasks/s3tests_java.py
@@ -159,23 +159,6 @@ def install_required_packages(self, client):
             stdout=BytesIO()
         )
 
-        endpoint = self.ctx.rgw.role_endpoints[client]
-        if endpoint.cert:
-            path = 'lib/security/cacerts'
-            self.ctx.cluster.only(client).run(
-                args=['sudo',
-                      'keytool',
-                      '-import', '-alias', '{alias}'.format(
-                          alias=endpoint.hostname),
-                      '-keystore',
-                      run.Raw(
-                          '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path=path)),
-                      '-file', endpoint.cert.certificate,
-                      '-storepass', 'changeit',
-                      ],
-                stdout=BytesIO()
-            )
-
     def create_users(self):
         """
         Create a main and an alternative s3 user.

From d203a97e1bf1e06433365ea38e3ab2b6430cefff Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 14 Feb 2024 12:02:09 -0500
Subject: [PATCH 1909/2492] cephadm: rm podman-auth.json if removing last
 cluster

We have points in rm-cluster where we check that
there are no other clusters on the host. If that
is the case, we can also clear /etc/ceph/podman-auth.json
which gets written out when we log in to a registry
while using podman

Fixes: https://tracker.ceph.com/issues/64433

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5a7ed9e76656..22704537e409 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4373,6 +4373,8 @@ def _rm_cluster(ctx: CephadmContext, keep_logs: bool, zap_osds: bool) -> None:
             for fname in glob(f'{ctx.log_dir}/cephadm.log*'):
                 os.remove(fname)
 
+        unlink_file(Path('/etc/ceph/podman-auth.json'), missing_ok=True, ignore_errors=True)
+
     # rm sysctl settings
     sysctl_dirs: List[Path] = [Path(ctx.sysctl_dir), Path('/usr/lib/sysctl.d')]
 

From 626179f7a265a9ffc2bcac187709a04fbbc24e17 Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Tue, 13 Feb 2024 11:03:04 -0500
Subject: [PATCH 1910/2492] rgw/multisite: handle object stat output of attrs
 "user.rgw.replicated-at"

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 PendingReleaseNotes  | 3 +++
 src/rgw/rgw_admin.cc | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 0ecec83ec81f..ecc52f2b67e6 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,5 +1,8 @@
 >=19.0.0
 
+* RGW: GetObject and HeadObject requests now return a x-rgw-replicated-at
+  header for replicated objects. This timestamp can be compared against the
+  Last-Modified header to determine how long the object took to replicate.
 * The cephfs-shell utility is now packaged for RHEL 9 / CentOS 9 as required
   python dependencies are now available in EPEL9.
 * RGW: S3 multipart uploads using Server-Side Encryption now replicate correctly in
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8265852973f9..5d0f9eb84379 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -8355,7 +8355,12 @@ int main(int argc, const char **argv)
 
     formatter->open_object_section("attrs");
     for (iter = other_attrs.begin(); iter != other_attrs.end(); ++iter) {
-      dump_string(iter->first.c_str(), iter->second, formatter.get());
+      bufferlist& bl = iter->second;
+      if (iter->first == RGW_ATTR_OBJ_REPLICATION_TIMESTAMP) {
+        decode_dump<ceph::real_time>("user.rgw.replicated-at", bl, formatter.get());
+      } else {
+        dump_string(iter->first.c_str(), iter->second, formatter.get());
+      }
     }
     formatter->close_section();
     formatter->close_section();

From dec3d3730712bbc0994f5cbd1989761d64f93019 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 12 Feb 2024 23:08:27 +1000
Subject: [PATCH 1911/2492] doc/glossary: improve OSD definitions

Remove expletive construction from three OSD-id-related terms.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index f4c1bb7b1666..15dddadc2296 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -298,18 +298,19 @@
                 mid-2010s to insist that "OSD" should refer to "Object Storage
                 Device", so it is important to know which meaning is intended. 
 
-	OSD fsid
-                This is a unique identifier used to identify an OSD. It is
-                found in the OSD path in a file called ``osd_fsid``. The
-                term ``fsid`` is used interchangeably with ``uuid``
-
-	OSD id
-                The integer that defines an OSD. It is generated by the
-                monitors during the creation of each OSD.
-
-	OSD uuid
-                This is the unique identifier of an OSD. This term is used
-                interchangeably with ``fsid``
+	OSD FSID 
+                The OSD fsid is a unique identifier that is used to identify an
+                OSD. It is found in the OSD path in a file called ``osd_fsid``.
+                The term ``FSID`` is used interchangeably with ``UUID``.
+
+	OSD ID 
+                The OSD id an integer unique to each OSD (each OSD has a unique
+                OSD ID). Each OSD id is generated by the monitors during the
+                creation of its associated OSD.
+
+	OSD UUID 
+                The OSD UUID is the unique identifier of an OSD. This term is
+                used interchangeably with ``FSID``.
 
         Period
                 In the context of :term:`RGW`, a period is the configuration

From ac4a48c4be1791a32cfd2cf3cf5af51eee581d21 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 15 Feb 2024 13:34:32 +0300
Subject: [PATCH 1912/2492] test/store_test: fix DeferredWrite test when
 prefer_deferred_size=0

Fixes: https://tracker.ceph.com/issues/64443

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc      | 4 +++-
 src/test/objectstore/store_test.cc | 9 +++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 29aff834fa46..876f766f7642 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -16252,7 +16252,9 @@ int BlueStore::_do_alloc_write(
     if (!g_conf()->bluestore_debug_omit_block_device_write) {
       if (data_size < prefer_deferred_size_snapshot) {
 	dout(20) << __func__ << " deferring 0x" << std::hex
-		 << l->length() << std::dec << " write via deferred" << dendl;
+		 << l->length()  << " write via deferred, pds=0x"
+                 << prefer_deferred_size_snapshot
+                 << std::dec<< dendl;
 	bluestore_deferred_op_t *op = _get_deferred_op(txc, l->length());
 	op->op = bluestore_deferred_op_t::OP_WRITE;
 	int r = wi.b->get_blob().map(
diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index 61ac86a8feeb..14c6454d5cf9 100644
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -7030,6 +7030,15 @@ TEST_P(DeferredWriteTest, NewData) {
   SetVal(g_conf(), "bluestore_min_alloc_size", stringify(t.min_alloc_size).c_str());
   SetVal(g_conf(), "bluestore_max_blob_size", stringify(t.max_blob_size).c_str());
   SetVal(g_conf(), "bluestore_prefer_deferred_size", stringify(t.prefer_deferred_size).c_str());
+  // bluestore_prefer_deferred_size set to 0 is a special case
+  // when hdd-/ssd-specific settings applied.
+  // Need to adjust them as well if we want to have no deferred ops at all
+  // Fixes: https://tracker.ceph.com/issues/64443
+  //
+  if (0 == t.prefer_deferred_size) {
+    SetVal(g_conf(), "bluestore_prefer_deferred_size_hdd", "0");
+    SetVal(g_conf(), "bluestore_prefer_deferred_size_ssd", "0");
+  }
   g_conf().apply_changes(nullptr);
   DeferredSetup();
 

From 4e13f936171cfb3d6cbb2264effbc1de369fccdc Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 15 Feb 2024 15:00:42 +0300
Subject: [PATCH 1913/2492] os/bluestore: eliminate build warnings

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.cc | 2 +-
 src/os/bluestore/BlueStore.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 29aff834fa46..ed4f4a93a851 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2865,7 +2865,7 @@ uint32_t BlueStore::Blob::merge_blob(CephContext* cct, Blob* blob_to_dissolve)
     auto buf = src->bc.buffer_map.extract(src->bc.buffer_map.cbegin());
     buf.mapped().space = &dst->bc;
     if (dst->bc.buffer_map.count(buf.key()) == 0) {
-      dst->bc.buffer_map.insert({buf.key(), std::move(buf.mapped())});
+      dst->bc.buffer_map.emplace(buf.key(), std::move(buf.mapped()));
     }
   }
   // move BufferSpace writing
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index f9dba48220c4..80537058c8de 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1049,7 +1049,7 @@ class BlueStore : public ObjectStore,
 
     void bound_encode_spanning_blobs(size_t& p);
     void encode_spanning_blobs(ceph::buffer::list::contiguous_appender& p);
-    BlobRef get_spanning_blob(int id) {
+    BlobRef& get_spanning_blob(int id) {
       auto p = spanning_blob_map.find(id);
       ceph_assert(p != spanning_blob_map.end());
       return p->second;

From 7953fe1b3920c92c086c981bf4e3d2c41ea7e450 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Thu, 15 Feb 2024 09:24:23 -0500
Subject: [PATCH 1914/2492] qa/cephadm: don't test certain workunits with agent

There are a handful of workunits that don't work
with or don't make sense with the agent.
The test for the cephadm timeout only works if
the mgr directly runs ceph-volume inventory which
it won't do with the agent present. The adoption
test is just running direct cephadm commands that
are irrelevant to the agent. The test_orch_cli tests
rely on refresh timings that are different with
the agent running, causing spurious failures.

Signed-off-by: Adam King <adking@redhat.com>
---
 qa/suites/orch/cephadm/no-agent-workunits/%                      | 0
 qa/suites/orch/cephadm/no-agent-workunits/.qa                    | 1 +
 qa/suites/orch/cephadm/no-agent-workunits/0-distro               | 1 +
 qa/suites/orch/cephadm/no-agent-workunits/mon_election           | 1 +
 .../{workunits => no-agent-workunits}/task/test_adoption.yaml    | 0
 .../task/test_cephadm_timeout.yaml                               | 0
 .../{workunits => no-agent-workunits}/task/test_orch_cli.yaml    | 0
 .../task/test_orch_cli_mon.yaml                                  | 0
 8 files changed, 3 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/no-agent-workunits/%
 create mode 120000 qa/suites/orch/cephadm/no-agent-workunits/.qa
 create mode 120000 qa/suites/orch/cephadm/no-agent-workunits/0-distro
 create mode 120000 qa/suites/orch/cephadm/no-agent-workunits/mon_election
 rename qa/suites/orch/cephadm/{workunits => no-agent-workunits}/task/test_adoption.yaml (100%)
 rename qa/suites/orch/cephadm/{workunits => no-agent-workunits}/task/test_cephadm_timeout.yaml (100%)
 rename qa/suites/orch/cephadm/{workunits => no-agent-workunits}/task/test_orch_cli.yaml (100%)
 rename qa/suites/orch/cephadm/{workunits => no-agent-workunits}/task/test_orch_cli_mon.yaml (100%)

diff --git a/qa/suites/orch/cephadm/no-agent-workunits/% b/qa/suites/orch/cephadm/no-agent-workunits/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/.qa b/qa/suites/orch/cephadm/no-agent-workunits/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/orch/cephadm/no-agent-workunits/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/0-distro b/qa/suites/orch/cephadm/no-agent-workunits/0-distro
new file mode 120000
index 000000000000..66187855738e
--- /dev/null
+++ b/qa/suites/orch/cephadm/no-agent-workunits/0-distro
@@ -0,0 +1 @@
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/no-agent-workunits/mon_election b/qa/suites/orch/cephadm/no-agent-workunits/mon_election
new file mode 120000
index 000000000000..3f331e6214c4
--- /dev/null
+++ b/qa/suites/orch/cephadm/no-agent-workunits/mon_election
@@ -0,0 +1 @@
+.qa/mon_election
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_adoption.yaml
similarity index 100%
rename from qa/suites/orch/cephadm/workunits/task/test_adoption.yaml
rename to qa/suites/orch/cephadm/no-agent-workunits/task/test_adoption.yaml
diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_cephadm_timeout.yaml
similarity index 100%
rename from qa/suites/orch/cephadm/workunits/task/test_cephadm_timeout.yaml
rename to qa/suites/orch/cephadm/no-agent-workunits/task/test_cephadm_timeout.yaml
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli.yaml
similarity index 100%
rename from qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml
rename to qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli.yaml
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml b/qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli_mon.yaml
similarity index 100%
rename from qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
rename to qa/suites/orch/cephadm/no-agent-workunits/task/test_orch_cli_mon.yaml

From 7e153bafec50a4299a40999e05e924692b59407f Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Thu, 15 Feb 2024 17:42:20 +0300
Subject: [PATCH 1915/2492] os/bluestore: remove redundant virtual declarations
 in CacheShard class

Fixes: https://tracker.ceph.com/issues/64445

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/BlueStore.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 80537058c8de..dce2731ba12a 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1455,19 +1455,19 @@ class BlueStore : public ObjectStore,
       _trim_to(0);
     }
 
-    virtual void shift_bins() {
+    void shift_bins() {
       std::lock_guard l(lock);
       age_bins.push_front(std::make_shared<int64_t>(0));
     }
-    virtual uint32_t get_bin_count() {
+    uint32_t get_bin_count() {
       std::lock_guard l(lock);
       return age_bins.capacity();
     }
-    virtual void set_bin_count(uint32_t count) {
+    void set_bin_count(uint32_t count) {
       std::lock_guard l(lock);
       age_bins.set_capacity(count);
     }
-    virtual uint64_t sum_bins(uint32_t start, uint32_t end) {
+    uint64_t sum_bins(uint32_t start, uint32_t end) {
       std::lock_guard l(lock);
       auto size = age_bins.size();
       if (size < start) {

From 671e1265a6c0e9d135bf87f59d1ac3ef96635794 Mon Sep 17 00:00:00 2001
From: Matt Vandermeulen <matt@reenigne.net>
Date: Thu, 15 Feb 2024 13:05:02 -0400
Subject: [PATCH 1916/2492] common: remove lingering bdev_async_discard option

Signed-off-by: Matt Vandermeulen <matt@reenigne.net>
---
 src/common/options/global.yaml.in | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 6f255f9d5768..ae1931cf0437 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4008,31 +4008,18 @@ options:
   flags:
   - runtime
   see_also:
-  - bdev_async_discard
-  - bdev_async_discard_threads
-- name: bdev_async_discard
-  desc: send discards to the block device in one or more threads
-  type: uint
-  level: advanced
-  default: false
-  with_legacy: false
-  flags:
-  - runtime
-  see_also:
-  - bdev_enable_discard
   - bdev_async_discard_threads
 - name: bdev_async_discard_threads
   desc: number of discard threads used to issue discards to the device
   type: uint
   level: advanced
-  default: 1
-  min: 1
+  default: 0
+  min: 0
   with_legacy: false
   flags:
   - runtime
   see_also:
   - bdev_enable_discard
-  - bdev_async_discard
 - name: bdev_flock_retry_interval
   type: float
   level: advanced

From 8a1d50e9980a837c80d11a95383a156e90c5e06b Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Thu, 15 Feb 2024 09:14:12 -0800
Subject: [PATCH 1917/2492] qa/tests: added squid option

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 src/script/build-integration-branch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/script/build-integration-branch b/src/script/build-integration-branch
index c4e1d604f0c8..f1d12a901c10 100755
--- a/src/script/build-integration-branch
+++ b/src/script/build-integration-branch
@@ -39,7 +39,7 @@ postfix = "-" + time.strftime(TIME_FORMAT, time.localtime())
 
 current_branch = check_output('git rev-parse --abbrev-ref HEAD',
                               shell=True).strip().decode()
-if current_branch in 'mimic nautilus octopus pacific quincy reef'.split():
+if current_branch in 'mimic nautilus octopus pacific quincy reef squid'.split():
     postfix += '-' + current_branch
     print(f"Adding current branch name '-{current_branch}' as a postfix")
 

From 0a031e5f4e90bc96d91bc5d57cd03e2f51aae688 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 15 Feb 2024 12:11:22 -0600
Subject: [PATCH 1918/2492] osd: clean compiler warnings

... for unused variables, for code that has no effect, and
for trying to std::move out of a const lvalue.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PrimaryLogPG.cc | 3 +--
 src/test/crush/crush.cc | 5 -----
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index ca83a563e4f8..6e6e081444e5 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -11531,7 +11531,7 @@ void PrimaryLogPG::submit_log_entries(
   }
 
   pgbackend->call_write_ordered(
-    [this, entries, repop, on_complete]() {
+    [this, entries, repop, on_complete]() mutable {
       ObjectStore::Transaction t;
       eversion_t old_last_update = info.last_update;
       recovery_state.merge_new_log_entries(
@@ -12109,7 +12109,6 @@ int PrimaryLogPG::find_object_context(const hobject_t& oid,
   dout(20) << __func__ << " " << soid
 	   << " snapset " << obc->ssc->snapset
 	   << dendl;
-  snapid_t first, last;
   auto p = obc->ssc->snapset.clone_snaps.find(soid.snap);
   ceph_assert(p != obc->ssc->snapset.clone_snaps.end());
   if (p->second.empty()) {
diff --git a/src/test/crush/crush.cc b/src/test/crush/crush.cc
index 2be7d5540d8d..c9f670a9e322 100644
--- a/src/test/crush/crush.cc
+++ b/src/test/crush/crush.cc
@@ -337,11 +337,6 @@ TEST_P(IndepTest, out_progressive) {
       if (i > 0) cout << "marked out " << i - 1 << " ";
       cout << x << " -> " << out << std::endl;
 
-      int num_none = 0;
-      for (unsigned k=0; k<out.size(); ++k) {
-	if (out[k] == CRUSH_ITEM_NONE)
-	  num_none++;
-      }
       ASSERT_EQ(0, get_num_dups(out));
 
       // make sure nothing moved

From 1e0ef76adbb34a8f1b8d42d3409c7205d2fe95f7 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 15 Feb 2024 12:17:57 -0600
Subject: [PATCH 1919/2492] osd: avoid copying chunks in a loop

to clear a static analyzer warning

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/PrimaryLogPG.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 6e6e081444e5..54462ab89a86 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -10503,7 +10503,7 @@ int PrimaryLogPG::start_dedup(OpRequestRef op, ObjectContextRef obc)
     obc_g ? &(obc_g->obs.oi.manifest) : nullptr,
     refs);
 
-  for (auto p : chunks) {
+  for (const auto& p : chunks) {
     hobject_t target = mop->new_manifest.chunk_map[p.first].oid;
     if (refs.find(target) == refs.end()) {
       continue;

From 66969c07dc0cd5e0f01685ec19de26dae09279f5 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 16 Feb 2024 00:04:05 +0000
Subject: [PATCH 1920/2492] unittest-seastar-socket: debug to error on
 unexpected return from dispatch_rw_bounded

Related: https://tracker.ceph.com/issues/64457
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crimson/test_socket.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc
index 2b61196ead8d..9a025f3c6767 100644
--- a/src/test/crimson/test_socket.cc
+++ b/src/test/crimson/test_socket.cc
@@ -449,7 +449,7 @@ future<> test_unexpected_down(bool is_fixed_cpu) {
     [](auto cs) {
       return Connection::dispatch_rw_bounded(cs, 128, true
         ).handle_exception_type([](const std::system_error& e) {
-        logger().debug("test_unexpected_down(): client get error {}", e);
+        logger().error("test_unexpected_down(): client get error {}", e);
         ceph_assert(e.code() == error::read_eof);
       });
     },

From a04a9c64b86fc253997081c4d769c887d13166a1 Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Thu, 15 Feb 2024 18:04:23 -0600
Subject: [PATCH 1921/2492] qa/workunits: opt in to new collection after
 upgrade from reef to main

We added the "basic_pool_flags" collection to main (squid), so
we need to re-opt-in after the upgrade to opt in to this collection.

Otherwise, a "TELEMETRY_CHANGED" warning comes up in the log and fails
the test.

Fixes: https://tracker.ceph.com/issues/64458
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/workunits/test_telemetry_reef_x.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/workunits/test_telemetry_reef_x.sh b/qa/workunits/test_telemetry_reef_x.sh
index 4e1a905669b0..8558d35cc423 100755
--- a/qa/workunits/test_telemetry_reef_x.sh
+++ b/qa/workunits/test_telemetry_reef_x.sh
@@ -12,7 +12,7 @@ fi
 ceph -s
 
 COLLECTIONS=$(ceph telemetry collection ls)
-NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics")
+NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_flags")
 for col in ${NEW_COLLECTIONS[@]}; do
     if ! [[ $COLLECTIONS == *$col* ]];
     then
@@ -26,8 +26,8 @@ ceph telemetry preview
 ceph telemetry preview-device
 ceph telemetry preview-all
 
-# Opt in to new collections
-# Currently, no new collections between latest reef and squid (dev)
+# Opt in to new collections (basic_pool_flags)
+ceph telemetry on --license sharing-1-0
 
 # Run show commands
 ceph telemetry show

From ace4bacdf4c4f0a17b40f04b5818bdf9c8862524 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 5 Feb 2024 06:02:14 -0600
Subject: [PATCH 1922/2492] osd/scrub: log parameter entering scrub_requested()

... and a minor formatting fix in the same function.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/pg_scrubber.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc
index 1c87c3b88a25..5ba720b2c503 100644
--- a/src/osd/scrubber/pg_scrubber.cc
+++ b/src/osd/scrubber/pg_scrubber.cc
@@ -606,11 +606,12 @@ scrub_level_t PgScrubber::scrub_requested(
   const bool deep_requested = (scrub_level == scrub_level_t::deep) ||
 			      (scrub_type == scrub_type_t::do_repair);
   dout(10) << fmt::format(
-		  "{}: {} {} scrub requested. Prev stamp: {}. Registered? {}",
+		  "{}: {}{} scrub requested. "
+		  "@entry:{},last-stamp:{:s},Registered?{}",
 		  __func__,
-		  (scrub_type == scrub_type_t::do_repair ? " repair + "
-							 : " not-repair + "),
-		  (deep_requested ? "deep" : "shallow"),
+		  (scrub_type == scrub_type_t::do_repair ? "repair + "
+							 : "not-repair + "),
+		  (deep_requested ? "deep" : "shallow"), req_flags,
 		  m_scrub_job->get_sched_time(), registration_state())
 	   << dendl;
 
@@ -620,7 +621,6 @@ scrub_level_t PgScrubber::scrub_requested(
   // User might intervene, so clear this
   req_flags.need_auto = false;
   req_flags.req_scrub = true;
-
   dout(20) << fmt::format("{}: planned scrub:{}", __func__, req_flags) << dendl;
 
   update_scrub_job(req_flags);

From a1cf175ee1874824b8b2152c242dee5530eebd6b Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 12 Feb 2024 08:50:22 -0600
Subject: [PATCH 1923/2492] test/osd: fix test_scrub_sched following scrubber
 changes

Replacing PgScrubber::determine_scrub_time() with a local copy,
as a stop-gap measure to keep the test running.
The scrub scheduling refactoring will remove the need for
this function, and the test will be updated accordingly.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/osd/test_scrub_sched.cc | 46 +++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/test/osd/test_scrub_sched.cc b/src/test/osd/test_scrub_sched.cc
index b6c069c4b5f4..0fec55bf0622 100644
--- a/src/test/osd/test_scrub_sched.cc
+++ b/src/test/osd/test_scrub_sched.cc
@@ -20,8 +20,9 @@
 #include "osd/PG.h"
 #include "osd/osd_types.h"
 #include "osd/osd_types_fmt.h"
-#include "osd/scrubber/osd_scrub_sched.h"
 #include "osd/scrubber_common.h"
+#include "osd/scrubber/pg_scrubber.h"
+#include "osd/scrubber/osd_scrub_sched.h"
 
 int main(int argc, char** argv)
 {
@@ -47,6 +48,9 @@ using ScrubJobRef = Scrub::ScrubJobRef;
 using qu_state_t = Scrub::qu_state_t;
 using scrub_schedule_t = Scrub::scrub_schedule_t;
 using ScrubQContainer = Scrub::ScrubQContainer;
+using sched_params_t = Scrub::sched_params_t;
+
+
 
 /// enabling access into ScrubQueue internals
 class ScrubSchedTestWrapper : public ScrubQueue {
@@ -85,6 +89,46 @@ class ScrubSchedTestWrapper : public ScrubQueue {
     return m_time_for_testing.value_or(ceph_clock_now());
   }
 
+  /**
+  * a temporary implementation of PgScrubber::determine_scrub_time(). That
+  * function is to be removed in the near future, and modifying the
+  * test to use the actual PgScrubber::determine_scrub_time() would create
+  * an unneeded coupling between objects that are due for separation.
+  */
+  sched_params_t determine_scrub_time(
+      const requested_scrub_t& request_flags,
+      const pg_info_t& pg_info,
+      const pool_opts_t& pool_conf)
+  {
+    sched_params_t res;
+
+    if (request_flags.must_scrub || request_flags.need_auto) {
+
+      // Set the smallest time that isn't utime_t()
+      res.proposed_time = PgScrubber::scrub_must_stamp();
+      res.is_must = Scrub::must_scrub_t::mandatory;
+      // we do not need the interval data in this case
+
+    } else if (pg_info.stats.stats_invalid && conf()->osd_scrub_invalid_stats) {
+      res.proposed_time = time_now();
+      res.is_must = Scrub::must_scrub_t::mandatory;
+
+    } else {
+      res.proposed_time = pg_info.history.last_scrub_stamp;
+      res.min_interval =
+	  pool_conf.value_or(pool_opts_t::SCRUB_MIN_INTERVAL, 0.0);
+      res.max_interval =
+	  pool_conf.value_or(pool_opts_t::SCRUB_MAX_INTERVAL, 0.0);
+    }
+
+    std::cout << fmt::format(
+	"suggested: {:s} hist: {:s} v:{}/{} must:{} pool-min:{} {}\n",
+	res.proposed_time, pg_info.history.last_scrub_stamp,
+	(bool)pg_info.stats.stats_invalid, conf()->osd_scrub_invalid_stats,
+	(res.is_must == Scrub::must_scrub_t::mandatory ? "y" : "n"),
+	res.min_interval, request_flags);
+    return res;
+  }
   ~ScrubSchedTestWrapper() override = default;
 };
 

From bede01863606e4a2c52147483e64d5de461b29b5 Mon Sep 17 00:00:00 2001
From: Ponnuvel Palaniyappan <pponnuvel@gmail.com>
Date: Thu, 15 Sep 2022 15:55:06 +0100
Subject: [PATCH 1924/2492] mon, mgr: do not output network ping stats

When doing PG dump using 'ceph pg dump --format json-pretty'
the output is extremely big that the command hangs and also
the ceph-mgr hangs and eventuall fails over.

The exact size depends on the number of OSDs in the cluster
and the number of peers for each OSD.

In tests, it's been identified that the network ping times
is the largest component in terms of size which is removed
from the output now so as to limit the overall size.

Fixes https://tracker.ceph.com/issues/57460

Signed-off-by: Ponnuvel Palaniyappan <pponnuvel@gmail.com>
---
 PendingReleaseNotes | 4 ++++
 src/mon/PGMap.h     | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 2e41462a1eb6..e28c35efce53 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -135,6 +135,10 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config
   permissions unless explicitly granted by topic policy.
 * RBD: The option ``--image-id`` has been added to `rbd children` CLI command,
   so it can be run for images in the trash.
+* PG dump: The default output of `ceph pg dump --format json` has changed. The
+  default json format produces a rather massive output in large clusters and
+  isn't scalable. So we have removed the 'network_ping_times' section from
+  the output. Details in the tracker: https://tracker.ceph.com/issues/57460
 
 >=18.0.0
 
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index 2a127b802a86..d1135f2a1a18 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -452,12 +452,12 @@ class PGMap : public PGMapDigest {
   int64_t get_rule_avail(const OSDMap& osdmap, int ruleno) const;
   void get_rules_avail(const OSDMap& osdmap,
 		       std::map<int,int64_t> *avail_map) const;
-  void dump(ceph::Formatter *f, bool with_net = true) const;
+  void dump(ceph::Formatter *f, bool with_net = false) const;
   void dump_basic(ceph::Formatter *f) const;
   void dump_pg_stats(ceph::Formatter *f, bool brief) const;
   void dump_pg_progress(ceph::Formatter *f) const;
   void dump_pool_stats(ceph::Formatter *f) const;
-  void dump_osd_stats(ceph::Formatter *f, bool with_net = true) const;
+  void dump_osd_stats(ceph::Formatter *f, bool with_net = false) const;
   void dump_osd_ping_times(ceph::Formatter *f) const;
   void dump_delta(ceph::Formatter *f) const;
   void dump_filtered_pg_stats(ceph::Formatter *f, std::set<pg_t>& pgs) const;

From 5c4a2341bf724b0c69aea8a4d45358e7944acfac Mon Sep 17 00:00:00 2001
From: Matt Vandermeulen <matt@reenigne.net>
Date: Fri, 16 Feb 2024 13:06:14 -0400
Subject: [PATCH 1925/2492] blk: support bdev_async_discard_threads == 0

Signed-off-by: Matt Vandermeulen <matt@reenigne.net>
---
 src/blk/kernel/KernelDevice.cc | 40 +++++++++++++++++++---------------
 src/blk/kernel/KernelDevice.h  |  2 +-
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc
index 6ce5b3a92933..6337292f5dec 100644
--- a/src/blk/kernel/KernelDevice.cc
+++ b/src/blk/kernel/KernelDevice.cc
@@ -778,30 +778,28 @@ void KernelDevice::_discard_thread(uint64_t tid)
   dout(10) << __func__ << " thread " << tid << " finish" << dendl;
 }
 
-int KernelDevice::_queue_discard(interval_set<uint64_t> &to_release)
+// this is private and is expected that the caller checks that discard
+// threads are running via _discard_started()
+void KernelDevice::_queue_discard(interval_set<uint64_t> &to_release)
 {
-  // if bdev_async_discard enabled on the fly, discard_thread is not started here, fallback to sync discard
-  if (!_discard_started())
-    return -1;
-
   if (to_release.empty())
-    return 0;
+    return;
 
   std::lock_guard l(discard_lock);
   discard_queued.insert(to_release);
   discard_cond.notify_one();
-  return 0;
 }
 
-// return true only if _queue_discard succeeded, so caller won't have to do alloc->release
-// otherwise false
+// return true only if discard was queued, so caller won't have to do
+// alloc->release, otherwise return false
 bool KernelDevice::try_discard(interval_set<uint64_t> &to_release, bool async)
 {
   if (!support_discard || !cct->_conf->bdev_enable_discard)
     return false;
 
-  if (async) {
-    return 0 == _queue_discard(to_release);
+  if (async && _discard_started()) {
+    _queue_discard(to_release);
+    return true;
   } else {
     for (auto p = to_release.begin(); p != to_release.end(); ++p) {
       _discard(p.get_start(), p.get_len());
@@ -1528,13 +1526,19 @@ void KernelDevice::handle_conf_change(const ConfigProxy& conf,
       // Decrease? Signal threads after telling them to stop
       dout(10) << __func__ << " stopping " << (oldval - newval) << " existing discard threads" << dendl;
 
-      // Signal the last threads to quit, and stop tracking them
-      for(uint64_t i = oldval - 1; i >= newval; i--)
-      {
-        // Also detach the thread so we no longer need to join
-        discard_threads[i]->stop = true;
-        discard_threads[i]->detach();
-        discard_threads.erase(discard_threads.begin() + i);
+      // Decreasing to zero is exactly the same as disabling async discard.
+      // Signal all threads to stop
+      if(newval == 0) {
+        _discard_stop();
+      } else {
+        // Signal the last threads to quit, and stop tracking them
+        for(uint64_t i = oldval - 1; i >= newval; i--)
+        {
+          // Also detach the thread so we no longer need to join
+          discard_threads[i]->stop = true;
+          discard_threads[i]->detach();
+          discard_threads.erase(discard_threads.begin() + i);
+        }
       }
 
       discard_cond.notify_all();
diff --git a/src/blk/kernel/KernelDevice.h b/src/blk/kernel/KernelDevice.h
index 326a94339915..914f05e64c43 100644
--- a/src/blk/kernel/KernelDevice.h
+++ b/src/blk/kernel/KernelDevice.h
@@ -87,7 +87,7 @@ class KernelDevice : public BlockDevice,
 
   void _aio_thread();
   void _discard_thread(uint64_t tid);
-  int _queue_discard(interval_set<uint64_t> &to_release);
+  void _queue_discard(interval_set<uint64_t> &to_release);
   bool try_discard(interval_set<uint64_t> &to_release, bool async = true) override;
 
   int _aio_start();

From e84518e63806ca5beef57d995923dcd3698276e9 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 16 Feb 2024 21:40:17 +0000
Subject: [PATCH 1926/2492] unittest-seastar-socket: tolerate connection_reset
 in test_unexpected_down

Fixes: https://tracker.ceph.com/issues/64457
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crimson/test_socket.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc
index 9a025f3c6767..aad91d46a82c 100644
--- a/src/test/crimson/test_socket.cc
+++ b/src/test/crimson/test_socket.cc
@@ -450,7 +450,8 @@ future<> test_unexpected_down(bool is_fixed_cpu) {
       return Connection::dispatch_rw_bounded(cs, 128, true
         ).handle_exception_type([](const std::system_error& e) {
         logger().error("test_unexpected_down(): client get error {}", e);
-        ceph_assert(e.code() == error::read_eof);
+        ceph_assert(e.code() == error::read_eof ||
+		    e.code() == std::errc::connection_reset);
       });
     },
     [](auto ss) { return Connection::dispatch_rw_unbounded(ss); }

From 4c14f143b59c96ecb2c23e5142829d47e2c03e22 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 14 Feb 2024 07:27:39 -0500
Subject: [PATCH 1927/2492] cephfs-mirror: typo ending bracket

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/tools/cephfs_mirror/Mirror.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/cephfs_mirror/Mirror.cc b/src/tools/cephfs_mirror/Mirror.cc
index 784e975fb14f..cc811f0a1c7f 100644
--- a/src/tools/cephfs_mirror/Mirror.cc
+++ b/src/tools/cephfs_mirror/Mirror.cc
@@ -520,7 +520,7 @@ void Mirror::update_fs_mirrors() {
       if (!mirror_action.action_in_progress && !_is_restarting(filesystem)) {
 	if (failed_restart || blocklisted_restart) {
 	  dout(5) << ": filesystem=" << filesystem << " failed mirroring (failed: "
-		  << failed_restart << ", blocklisted: " << blocklisted_restart << dendl;
+		  << failed_restart << ", blocklisted: " << blocklisted_restart << ")" << dendl;
 	  _set_restarting(filesystem);
 	  auto peers = mirror_action.fs_mirror->get_peers();
 	  auto ctx =  new C_RestartMirroring(this, filesystem, mirror_action.pool_id, peers);

From 658ee6c40116e07f45085ad0df7ef33d3c8ed78e Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 14 Feb 2024 07:28:58 -0500
Subject: [PATCH 1928/2492] cephfs_mirror: add labeled replication performance
 metrics

Fixes: http://tracker.ceph.com/issues/63945
Signed-off-by: Jos Collin <jcollin@redhat.com>
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/common/options/cephfs-mirror.yaml.in | 13 ++++-
 src/pybind/mgr/mgr_module.py             |  2 +-
 src/tools/cephfs_mirror/FSMirror.cc      | 40 +++++++++++++++
 src/tools/cephfs_mirror/FSMirror.h       |  2 +
 src/tools/cephfs_mirror/Mirror.cc        | 44 ++++++++++++++++
 src/tools/cephfs_mirror/Mirror.h         |  2 +
 src/tools/cephfs_mirror/PeerReplayer.cc  | 65 ++++++++++++++++++++++++
 src/tools/cephfs_mirror/PeerReplayer.h   |  2 +
 8 files changed, 168 insertions(+), 2 deletions(-)

diff --git a/src/common/options/cephfs-mirror.yaml.in b/src/common/options/cephfs-mirror.yaml.in
index 78f86dfb1a76..f826161872b8 100644
--- a/src/common/options/cephfs-mirror.yaml.in
+++ b/src/common/options/cephfs-mirror.yaml.in
@@ -91,4 +91,15 @@ options:
   default: 10
   services:
   - cephfs-mirror
-  min: 0
\ No newline at end of file
+  min: 0
+- name: cephfs_mirror_perf_stats_prio
+  type: int
+  level: advanced
+  desc: Priority level for mirror daemon replication perf counters
+  long_desc: The daemon will send perf counter data to the manager daemon if the priority
+    is not lower than mgr_stats_threshold.
+  default: 5
+  services:
+  - cephfs-mirror
+  min: 0
+  max: 11
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 28fd69738eb9..13cf9386bcdb 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -2074,7 +2074,7 @@ def get_latest_avg(self, daemon_type: str, daemon_name: str, counter: str) -> Tu
     @profile_method()
     def get_unlabeled_perf_counters(self, prio_limit: int = PRIO_USEFUL,
                               services: Sequence[str] = ("mds", "mon", "osd",
-                                                         "rbd-mirror", "rgw",
+                                                         "rbd-mirror", "cephfs-mirror", "rgw",
                                                          "tcmu-runner")) -> Dict[str, dict]:
         """
         Return the perf counters currently known to this ceph-mgr
diff --git a/src/tools/cephfs_mirror/FSMirror.cc b/src/tools/cephfs_mirror/FSMirror.cc
index 7ea798e6bec2..3d5bf2d1c724 100644
--- a/src/tools/cephfs_mirror/FSMirror.cc
+++ b/src/tools/cephfs_mirror/FSMirror.cc
@@ -8,6 +8,8 @@
 #include "common/debug.h"
 #include "common/errno.h"
 #include "common/WorkQueue.h"
+#include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
 #include "include/stringify.h"
 #include "msg/Messenger.h"
 #include "FSMirror.h"
@@ -25,6 +27,14 @@
 
 using namespace std;
 
+// Performance Counters
+enum {
+  l_cephfs_mirror_fs_mirror_first = 5000,
+  l_cephfs_mirror_fs_mirror_peers,
+  l_cephfs_mirror_fs_mirror_dir_count,
+  l_cephfs_mirror_fs_mirror_last,
+};
+
 namespace cephfs {
 namespace mirror {
 
@@ -107,6 +117,18 @@ FSMirror::FSMirror(CephContext *cct, const Filesystem &filesystem, uint64_t pool
     m_asok_hook(new MirrorAdminSocketHook(cct, filesystem, this)) {
   m_service_daemon->add_or_update_fs_attribute(m_filesystem.fscid, SERVICE_DAEMON_DIR_COUNT_KEY,
                                                (uint64_t)0);
+
+  std::string labels = ceph::perf_counters::key_create("cephfs_mirror_mirrored_filesystems",
+						       {{"filesystem", m_filesystem.fs_name}});
+  PerfCountersBuilder plb(m_cct, labels, l_cephfs_mirror_fs_mirror_first,
+			  l_cephfs_mirror_fs_mirror_last);
+  auto prio = m_cct->_conf.get_val<int64_t>("cephfs_mirror_perf_stats_prio");
+  plb.add_u64(l_cephfs_mirror_fs_mirror_peers,
+	      "mirroring_peers", "Mirroring Peers", "mpee", prio);
+  plb.add_u64(l_cephfs_mirror_fs_mirror_dir_count,
+	      "directory_count", "Directory Count", "dirc", prio);
+  m_perf_counters = plb.create_perf_counters();
+  m_cct->get_perfcounters_collection()->add(m_perf_counters);
 }
 
 FSMirror::~FSMirror() {
@@ -120,6 +142,12 @@ FSMirror::~FSMirror() {
   // outside the lock so that in-progress commands can acquire
   // lock and finish executing.
   delete m_asok_hook;
+  PerfCounters *perf_counters = nullptr;
+  std::swap(perf_counters, m_perf_counters);
+  if (perf_counters != nullptr) {
+    m_cct->get_perfcounters_collection()->remove(perf_counters);
+    delete perf_counters;
+  }
 }
 
 int FSMirror::init_replayer(PeerReplayer *peer_replayer) {
@@ -355,6 +383,9 @@ void FSMirror::handle_acquire_directory(string_view dir_path) {
       peer_replayer->add_directory(dir_path);
     }
   }
+  if (m_perf_counters) {
+    m_perf_counters->set(l_cephfs_mirror_fs_mirror_dir_count, m_directories.size());
+  }
 }
 
 void FSMirror::handle_release_directory(string_view dir_path) {
@@ -372,6 +403,9 @@ void FSMirror::handle_release_directory(string_view dir_path) {
         peer_replayer->remove_directory(dir_path);
       }
     }
+    if (m_perf_counters) {
+      m_perf_counters->set(l_cephfs_mirror_fs_mirror_dir_count, m_directories.size());
+    }
   }
 }
 
@@ -395,6 +429,9 @@ void FSMirror::add_peer(const Peer &peer) {
   }
   m_peer_replayers.emplace(peer, std::move(replayer));
   ceph_assert(m_peer_replayers.size() == 1); // support only a single peer
+  if (m_perf_counters) {
+    m_perf_counters->inc(l_cephfs_mirror_fs_mirror_peers);
+  }
 }
 
 void FSMirror::remove_peer(const Peer &peer) {
@@ -415,6 +452,9 @@ void FSMirror::remove_peer(const Peer &peer) {
     dout(5) << ": shutting down replayers for peer=" << peer << dendl;
     shutdown_replayer(replayer.get());
   }
+  if (m_perf_counters) {
+    m_perf_counters->dec(l_cephfs_mirror_fs_mirror_peers);
+  }
 }
 
 void FSMirror::mirror_status(Formatter *f) {
diff --git a/src/tools/cephfs_mirror/FSMirror.h b/src/tools/cephfs_mirror/FSMirror.h
index a9c1fab1025d..75fca758520d 100644
--- a/src/tools/cephfs_mirror/FSMirror.h
+++ b/src/tools/cephfs_mirror/FSMirror.h
@@ -154,6 +154,8 @@ class FSMirror {
 
   MountRef m_mount;
 
+  PerfCounters *m_perf_counters;
+
   int init_replayer(PeerReplayer *peer_replayer);
   void shutdown_replayer(PeerReplayer *peer_replayer);
   void cleanup();
diff --git a/src/tools/cephfs_mirror/Mirror.cc b/src/tools/cephfs_mirror/Mirror.cc
index cc811f0a1c7f..0ad9f101e371 100644
--- a/src/tools/cephfs_mirror/Mirror.cc
+++ b/src/tools/cephfs_mirror/Mirror.cc
@@ -9,6 +9,8 @@
 #include "common/errno.h"
 #include "common/Timer.h"
 #include "common/WorkQueue.h"
+#include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
 #include "include/types.h"
 #include "mon/MonClient.h"
 #include "msg/Messenger.h"
@@ -20,6 +22,14 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "cephfs::mirror::Mirror " << __func__
 
+// Performance Counters
+enum {
+  l_cephfs_mirror_first = 4000,
+  l_cephfs_mirror_file_systems_mirrorred,
+  l_cephfs_mirror_file_systems_mirror_enable_failures,
+  l_cephfs_mirror_last,
+};
+
 namespace cephfs {
 namespace mirror {
 
@@ -277,6 +287,17 @@ int Mirror::init(std::string &reason) {
     return r;
   }
 
+  std::string labels = ceph::perf_counters::key_create("cephfs_mirror");
+  PerfCountersBuilder plb(m_cct, labels, l_cephfs_mirror_first, l_cephfs_mirror_last);
+
+  auto prio = m_cct->_conf.get_val<int64_t>("cephfs_mirror_perf_stats_prio");
+  plb.add_u64(l_cephfs_mirror_file_systems_mirrorred,
+	      "mirrored_filesystems", "Filesystems mirrored", "mir", prio);
+  plb.add_u64_counter(l_cephfs_mirror_file_systems_mirror_enable_failures,
+		      "mirror_enable_failures", "Mirroring enable failures", "mirf", prio);
+  m_perf_counters = plb.create_perf_counters();
+  m_cct->get_perfcounters_collection()->add(m_perf_counters);
+
   return 0;
 }
 
@@ -285,6 +306,13 @@ void Mirror::shutdown() {
   m_stopping = true;
   m_cluster_watcher->shutdown();
   m_cond.notify_all();
+
+  PerfCounters *perf_counters = nullptr;
+  std::swap(perf_counters, m_perf_counters);
+  if (perf_counters != nullptr) {
+    m_cct->get_perfcounters_collection()->remove(perf_counters);
+    delete perf_counters;
+  }
 }
 
 void Mirror::reopen_logs() {
@@ -328,6 +356,9 @@ void Mirror::handle_enable_mirroring(const Filesystem &filesystem,
     m_service_daemon->add_or_update_fs_attribute(filesystem.fscid,
                                                  SERVICE_DAEMON_MIRROR_ENABLE_FAILED_KEY,
                                                  true);
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_file_systems_mirror_enable_failures);
+    }
     return;
   }
 
@@ -341,6 +372,9 @@ void Mirror::handle_enable_mirroring(const Filesystem &filesystem,
   }
 
   dout(10) << ": Initialized FSMirror for filesystem=" << filesystem << dendl;
+  if (m_perf_counters) {
+    m_perf_counters->inc(l_cephfs_mirror_file_systems_mirrorred);
+  }
 }
 
 void Mirror::handle_enable_mirroring(const Filesystem &filesystem, int r) {
@@ -358,6 +392,9 @@ void Mirror::handle_enable_mirroring(const Filesystem &filesystem, int r) {
     m_service_daemon->add_or_update_fs_attribute(filesystem.fscid,
                                                  SERVICE_DAEMON_MIRROR_ENABLE_FAILED_KEY,
                                                  true);
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_file_systems_mirror_enable_failures);
+    }
     return;
   }
 
@@ -367,6 +404,9 @@ void Mirror::handle_enable_mirroring(const Filesystem &filesystem, int r) {
   m_cond.notify_all();
 
   dout(10) << ": Initialized FSMirror for filesystem=" << filesystem << dendl;
+  if (m_perf_counters) {
+    m_perf_counters->inc(l_cephfs_mirror_file_systems_mirrorred);
+  }
 }
 
 void Mirror::enable_mirroring(const Filesystem &filesystem, uint64_t local_pool_id,
@@ -422,6 +462,10 @@ void Mirror::handle_disable_mirroring(const Filesystem &filesystem, int r) {
       m_mirror_actions.erase(filesystem);
     }
   }
+
+  if (m_perf_counters) {
+    m_perf_counters->dec(l_cephfs_mirror_file_systems_mirrorred);
+  }
 }
 
 void Mirror::disable_mirroring(const Filesystem &filesystem, Context *on_finish) {
diff --git a/src/tools/cephfs_mirror/Mirror.h b/src/tools/cephfs_mirror/Mirror.h
index 2081b5b53050..5e37b5df4580 100644
--- a/src/tools/cephfs_mirror/Mirror.h
+++ b/src/tools/cephfs_mirror/Mirror.h
@@ -104,6 +104,8 @@ class Mirror {
   RadosRef m_local;
   std::unique_ptr<ServiceDaemon> m_service_daemon;
 
+  PerfCounters *m_perf_counters;
+
   int init_mon_client();
 
   // called via listener
diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index bd47046bb121..6f3fb724a659 100644
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -12,6 +12,8 @@
 #include "common/ceph_context.h"
 #include "common/debug.h"
 #include "common/errno.h"
+#include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
 #include "FSMirror.h"
 #include "PeerReplayer.h"
 #include "Utils.h"
@@ -26,6 +28,18 @@
 
 using namespace std;
 
+// Performance Counters
+enum {
+  l_cephfs_mirror_peer_replayer_first = 6000,
+  l_cephfs_mirror_peer_replayer_snaps_synced,
+  l_cephfs_mirror_peer_replayer_snaps_deleted,
+  l_cephfs_mirror_peer_replayer_snaps_renamed,
+  l_cephfs_mirror_peer_replayer_snap_sync_failures,
+  l_cephfs_mirror_peer_replayer_avg_sync_time,
+  l_cephfs_mirror_peer_replayer_sync_bytes,
+  l_cephfs_mirror_peer_replayer_last,
+};
+
 namespace cephfs {
 namespace mirror {
 
@@ -161,10 +175,39 @@ PeerReplayer::PeerReplayer(CephContext *cct, FSMirror *fs_mirror,
                                                  SERVICE_DAEMON_FAILED_DIR_COUNT_KEY, (uint64_t)0);
   m_service_daemon->add_or_update_peer_attribute(m_filesystem.fscid, m_peer,
                                                  SERVICE_DAEMON_RECOVERED_DIR_COUNT_KEY, (uint64_t)0);
+
+  std::string labels = ceph::perf_counters::key_create("cephfs_mirror_peers",
+						       {{"source_fscid", stringify(m_filesystem.fscid)},
+							{"source_filesystem", m_filesystem.fs_name},
+							{"peer_cluster_name", m_peer.remote.cluster_name},
+							{"peer_cluster_filesystem", m_peer.remote.fs_name}});
+  PerfCountersBuilder plb(m_cct, labels, l_cephfs_mirror_peer_replayer_first,
+			  l_cephfs_mirror_peer_replayer_last);
+  auto prio = m_cct->_conf.get_val<int64_t>("cephfs_mirror_perf_stats_prio");
+  plb.add_u64_counter(l_cephfs_mirror_peer_replayer_snaps_synced,
+		      "snaps_synced", "Snapshots Synchronized", "sync", prio);
+  plb.add_u64_counter(l_cephfs_mirror_peer_replayer_snaps_deleted,
+		      "snaps_deleted", "Snapshots Deleted", "del", prio);
+  plb.add_u64_counter(l_cephfs_mirror_peer_replayer_snaps_renamed,
+		      "snaps_renamed", "Snapshots Renamed", "ren", prio);
+  plb.add_u64_counter(l_cephfs_mirror_peer_replayer_snap_sync_failures,
+		      "sync_failures", "Snapshot Sync Failures", "fail", prio);
+  plb.add_time_avg(l_cephfs_mirror_peer_replayer_avg_sync_time,
+		   "avg_sync_time", "Average Sync Time", "asyn", prio);
+  plb.add_u64_counter(l_cephfs_mirror_peer_replayer_sync_bytes,
+		      "sync_bytes", "Sync Bytes", "sbye", prio);
+  m_perf_counters = plb.create_perf_counters();
+  m_cct->get_perfcounters_collection()->add(m_perf_counters);
 }
 
 PeerReplayer::~PeerReplayer() {
   delete m_asok_hook;
+  PerfCounters *perf_counters = nullptr;
+  std::swap(perf_counters, m_perf_counters);
+  if (perf_counters != nullptr) {
+    m_cct->get_perfcounters_collection()->remove(perf_counters);
+    delete perf_counters;
+  }
 }
 
 int PeerReplayer::init() {
@@ -516,6 +559,9 @@ int PeerReplayer::propagate_snap_deletes(const std::string &dir_root,
       return r;
     }
     inc_deleted_snap(dir_root);
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_peer_replayer_snaps_deleted);
+    }
   }
 
   return 0;
@@ -539,6 +585,9 @@ int PeerReplayer::propagate_snap_renames(
       return r;
     }
     inc_renamed_snap(dir_root);
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_peer_replayer_snaps_renamed);
+    }
   }
 
   return 0;
@@ -694,6 +743,9 @@ int PeerReplayer::remote_file_op(const std::string &dir_root, const std::string
         derr << ": failed to copy path=" << epath << ": " << cpp_strerror(r) << dendl;
         return r;
       }
+      if (m_perf_counters) {
+	m_perf_counters->inc(l_cephfs_mirror_peer_replayer_sync_bytes, stx.stx_size);
+      }
     } else if (S_ISLNK(stx.stx_mode)) {
       // free the remote link before relinking
       r = ceph_unlinkat(m_remote_mount, fh.r_fd_dir_root, epath.c_str(), 0);
@@ -1457,7 +1509,17 @@ int PeerReplayer::do_sync_snaps(const std::string &dir_root) {
       clear_current_syncing_snap(dir_root);
       return r;
     }
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_peer_replayer_snaps_synced);
+    }
     std::chrono::duration<double> duration = clock::now() - start;
+
+    utime_t d;
+    d.set_from_double(duration.count());
+    if (m_perf_counters) {
+      m_perf_counters->tinc(l_cephfs_mirror_peer_replayer_avg_sync_time, d);
+    }
+
     set_last_synced_stat(dir_root, it->first, it->second, duration.count());
     if (--snaps_per_cycle == 0) {
       break;
@@ -1481,6 +1543,9 @@ void PeerReplayer::sync_snaps(const std::string &dir_root,
   locker.lock();
   if (r < 0) {
     _inc_failed_count(dir_root);
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_cephfs_mirror_peer_replayer_snap_sync_failures);
+    }
   } else {
     _reset_failed_count(dir_root);
   }
diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 0511d154a759..63e1bd9e8a77 100644
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -269,6 +269,8 @@ class PeerReplayer {
 
   ServiceDaemonStats m_service_daemon_stats;
 
+  PerfCounters *m_perf_counters;
+
   void run(SnapshotReplayerThread *replayer);
 
   boost::optional<std::string> pick_directory();

From 164c547edcff86ed7dbb0d398b6db54f71e3f9b8 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 12 Feb 2024 02:21:08 -0500
Subject: [PATCH 1929/2492] mds: export per-client metrics as labelled perf
 counters

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDSRank.h           |   4 +
 src/mds/MetricAggregator.cc | 169 ++++++++++++++++++++++++++++++++++++
 src/mds/MetricAggregator.h  |   5 ++
 3 files changed, 178 insertions(+)

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index fd680863ca97..adebb9b4d08d 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -254,6 +254,10 @@ class MDSRank {
       progress_thread.signal();
     }
 
+    uint64_t get_global_id() const {
+      return monc->get_global_id();
+    }
+
     // Daemon lifetime functions: these guys break the abstraction
     // and call up into the parent MDSDaemon instance.  It's kind
     // of unavoidable: if we want any depth into our calls 
diff --git a/src/mds/MetricAggregator.cc b/src/mds/MetricAggregator.cc
index 6487084fb639..9765d4d5b517 100644
--- a/src/mds/MetricAggregator.cc
+++ b/src/mds/MetricAggregator.cc
@@ -4,6 +4,9 @@
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>
 
+#include "common/ceph_context.h"
+#include "common/perf_counters_key.h"
+
 #include "MDSRank.h"
 #include "MetricAggregator.h"
 #include "mgr/MgrClient.h"
@@ -13,8 +16,36 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "mds.metric.aggregator" << " " << __func__
 
+// Performance Counters
+ enum {
+  l_mds_client_metrics_start = 10000,
+  l_mds_client_metrics_num_clients,
+  l_mds_client_metrics_last
+ };
+
+enum {
+  l_mds_per_client_metrics_start = 20000,
+  l_mds_per_client_metrics_cap_hits,
+  l_mds_per_client_metrics_cap_misses,
+  l_mds_per_client_metrics_avg_read_latency,
+  l_mds_per_client_metrics_avg_write_latency,
+  l_mds_per_client_metrics_avg_metadata_latency,
+  l_mds_per_client_metrics_dentry_lease_hits,
+  l_mds_per_client_metrics_dentry_lease_misses,
+  l_mds_per_client_metrics_opened_files,
+  l_mds_per_client_metrics_opened_inodes,
+  l_mds_per_client_metrics_pinned_icaps,
+  l_mds_per_client_metrics_total_inodes,
+  l_mds_per_client_metrics_total_read_ops,
+  l_mds_per_client_metrics_total_read_size,
+  l_mds_per_client_metrics_total_write_ops,
+  l_mds_per_client_metrics_total_write_size,
+  l_mds_per_client_metrics_last
+ };
+
 MetricAggregator::MetricAggregator(CephContext *cct, MDSRank *mds, MgrClient *mgrc)
   : Dispatcher(cct),
+    m_cct(cct),
     mds(mds),
     mgrc(mgrc),
     mds_pinger(mds) {
@@ -32,6 +63,15 @@ void MetricAggregator::ping_all_active_ranks() {
 int MetricAggregator::init() {
   dout(10) << dendl;
 
+  std::string labels = ceph::perf_counters::key_create("mds_client_metrics",
+						       {{"fs_name", mds->mdsmap->get_fs_name()},
+							{"id", stringify(mds->get_global_id())}});
+  PerfCountersBuilder plb(m_cct, labels, l_mds_client_metrics_start, l_mds_client_metrics_last);
+  plb.add_u64(l_mds_client_metrics_num_clients,
+	      "num_clients", "Numer of client sessions", "mcli", PerfCountersBuilder::PRIO_CRITICAL);
+  m_perf_counters = plb.create_perf_counters();
+  m_cct->get_perfcounters_collection()->add(m_perf_counters);
+
   pinger = std::thread([this]() {
       std::unique_lock locker(lock);
       while (!stopping) {
@@ -61,6 +101,24 @@ void MetricAggregator::shutdown() {
     std::scoped_lock locker(lock);
     ceph_assert(!stopping);
     stopping = true;
+
+    // dealloc per-client perf counter
+    for (auto [crpair, pc] : client_perf_counters) {
+      PerfCounters *perf_counters = nullptr;
+      std::swap(perf_counters, pc);
+      if (perf_counters != nullptr) {
+	m_cct->get_perfcounters_collection()->remove(perf_counters);
+	delete perf_counters;
+      }
+    }
+    client_perf_counters.clear();
+
+    PerfCounters *perf_counters = nullptr;
+    std::swap(perf_counters, m_perf_counters);
+    if (perf_counters != nullptr) {
+      m_cct->get_perfcounters_collection()->remove(perf_counters);
+      delete perf_counters;
+    }
   }
 
   if (pinger.joinable()) {
@@ -97,10 +155,110 @@ void MetricAggregator::refresh_metrics_for_rank(const entity_inst_t &client,
            << metrics << dendl;
 
   auto &p = clients_by_rank.at(rank);
+  auto crpair = std::make_pair(client, rank);
   bool ins = p.insert(client).second;
   if (ins) {
     dout(20) << ": rank=" << rank << " has " << p.size() << " connected"
              << " client(s)" << dendl;
+    if (m_perf_counters) {
+      m_perf_counters->inc(l_mds_client_metrics_num_clients);
+    }
+
+    std::string labels = ceph::perf_counters::key_create("mds_client_metrics-" + std::string(mds->mdsmap->get_fs_name()),
+							 {{"client", stringify(client.name)},
+							  {"rank", stringify(rank)}});
+    PerfCountersBuilder plb(m_cct, labels, l_mds_per_client_metrics_start, l_mds_per_client_metrics_last);
+    plb.add_u64(l_mds_per_client_metrics_cap_hits,
+		"cap_hits", "Capability hits", "hcap", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_cap_misses,
+		"cap_miss", "Capability misses", "mcap", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_time(l_mds_per_client_metrics_avg_read_latency,
+		 "avg_read_latency", "Average Read Latency", "arlt", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_time(l_mds_per_client_metrics_avg_write_latency,
+		 "avg_write_latency", "Average Write Latency", "awlt", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_time(l_mds_per_client_metrics_avg_metadata_latency,
+		 "avg_metadata_latency", "Average Metadata Latency", "amlt", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_dentry_lease_hits,
+		"dentry_lease_hits", "Dentry Lease Hits", "hden", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_dentry_lease_misses,
+		"dentry_lease_miss", "Dentry Lease Misses", "mden", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_opened_files,
+		"opened_files", "Open Files", "ofil", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_opened_inodes,
+		"opened_inodes", "Open Inodes", "oino", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_pinned_icaps,
+		"pinned_icaps", "Pinned Inode Caps", "pino", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_total_inodes,
+		"total_inodes", "Total Inodes", "tino", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_total_read_ops,
+		"total_read_ops", "Total Read Operations", "rops", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_total_read_size,
+		"total_read_size", "Total Read Size", "rsiz", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_total_write_ops,
+		"total_write_ops", "Total Write Operations", "wops", PerfCountersBuilder::PRIO_CRITICAL);
+    plb.add_u64(l_mds_per_client_metrics_total_write_size,
+		"total_write_size", "Total Write Size", "wsiz", PerfCountersBuilder::PRIO_CRITICAL);
+    client_perf_counters[crpair] = plb.create_perf_counters();
+    m_cct->get_perfcounters_collection()->add(client_perf_counters[crpair]);
+  }
+
+  // update perf counters
+  PerfCounters *perf_counter_ptr = nullptr;
+  if (client_perf_counters.contains(crpair)) {
+    perf_counter_ptr = client_perf_counters[crpair];
+  }
+
+  if (perf_counter_ptr) {
+    // client capability hit ratio
+    perf_counter_ptr->set(l_mds_per_client_metrics_cap_hits, metrics.cap_hit_metric.hits);
+    perf_counter_ptr->set(l_mds_per_client_metrics_cap_misses, metrics.cap_hit_metric.misses);
+
+    // some averages
+    if (metrics.read_latency_metric.updated) {
+      utime_t ravg(metrics.read_latency_metric.mean.tv.tv_sec * 100,
+		   metrics.read_latency_metric.mean.tv.tv_nsec / 1000000);
+      perf_counter_ptr->tset(l_mds_per_client_metrics_avg_read_latency, ravg);
+    }
+    if (metrics.write_latency_metric.updated) {
+      utime_t wavg(metrics.write_latency_metric.mean.tv.tv_sec * 100,
+		   metrics.write_latency_metric.mean.tv.tv_nsec / 1000000);
+      perf_counter_ptr->set(l_mds_per_client_metrics_avg_write_latency, wavg);
+    }
+    if (metrics.metadata_latency_metric.updated) {
+      utime_t mavg(metrics.metadata_latency_metric.mean.tv.tv_sec * 100,
+		   metrics.metadata_latency_metric.mean.tv.tv_nsec / 1000000);
+      perf_counter_ptr->set(l_mds_per_client_metrics_avg_metadata_latency, mavg);
+    }
+
+    // dentry leases
+    if (metrics.dentry_lease_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_dentry_lease_hits, metrics.dentry_lease_metric.hits);
+      perf_counter_ptr->set(l_mds_per_client_metrics_dentry_lease_misses, metrics.dentry_lease_metric.misses);
+    }
+
+    // file+inode opens, pinned inode caps
+    if (metrics.opened_files_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_opened_files, metrics.opened_files_metric.opened_files);
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_inodes, metrics.opened_files_metric.total_inodes);
+    }
+    if (metrics.opened_inodes_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_opened_inodes, metrics.opened_inodes_metric.total_inodes);
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_inodes, metrics.opened_inodes_metric.total_inodes);
+    }
+    if (metrics.pinned_icaps_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_pinned_icaps, metrics.pinned_icaps_metric.pinned_icaps);
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_inodes, metrics.pinned_icaps_metric.total_inodes);
+    }
+
+    // read+write io metrics
+    if (metrics.read_io_sizes_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_read_ops, metrics.read_io_sizes_metric.total_ops);
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_read_size, metrics.read_io_sizes_metric.total_size);
+    }
+    if (metrics.write_io_sizes_metric.updated) {
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_write_ops, metrics.write_io_sizes_metric.total_ops);
+      perf_counter_ptr->set(l_mds_per_client_metrics_total_write_size, metrics.write_io_sizes_metric.total_size);
+    }
   }
 
   auto update_counter_func = [&metrics](const MDSPerformanceCounterDescriptor &d,
@@ -260,6 +418,13 @@ void MetricAggregator::remove_metrics_for_rank(const entity_inst_t &client,
     ceph_assert(rm);
     dout(20) << ": rank=" << rank << " has " << p.size() << " connected"
              << " client(s)" << dendl;
+    auto crpair = std::make_pair(client, rank);
+    m_cct->get_perfcounters_collection()->remove(client_perf_counters[crpair]);
+    delete client_perf_counters[crpair];
+    client_perf_counters.erase(crpair);
+  }
+  if (m_perf_counters) {
+    m_perf_counters->dec(l_mds_client_metrics_num_clients);
   }
 
   auto sub_key_func = [client, rank](const MDSPerfMetricSubKeyDescriptor &d,
@@ -315,6 +480,10 @@ void MetricAggregator::handle_mds_metrics(const cref_t<MMDSMetrics> &m) {
            << rank << " with sequence number " << seq << dendl;
 
   std::scoped_lock locker(lock);
+  if (stopping) {
+    dout(10) << ": stopping" << dendl;
+    return;
+  }
   if (!mds_pinger.pong_received(rank, seq)) {
     return;
   }
diff --git a/src/mds/MetricAggregator.h b/src/mds/MetricAggregator.h
index fe9aef2e3bc4..6d48756f76dc 100644
--- a/src/mds/MetricAggregator.h
+++ b/src/mds/MetricAggregator.h
@@ -11,6 +11,7 @@
 #include "msg/msg_types.h"
 #include "msg/Dispatcher.h"
 #include "common/ceph_mutex.h"
+#include "common/perf_counters.h"
 #include "include/common_fwd.h"
 #include "messages/MMDSMetrics.h"
 
@@ -55,6 +56,7 @@ class MetricAggregator : public Dispatcher {
   // drop this lock when calling ->send_message_mds() else mds might
   // deadlock
   ceph::mutex lock = ceph::make_mutex("MetricAggregator::lock");
+  CephContext *m_cct;
   MDSRank *mds;
   MgrClient *mgrc;
 
@@ -72,6 +74,9 @@ class MetricAggregator : public Dispatcher {
 
   bool stopping = false;
 
+  PerfCounters *m_perf_counters;
+  std::map<std::pair<entity_inst_t, mds_rank_t>, PerfCounters*> client_perf_counters;
+
   void handle_mds_metrics(const cref_t<MMDSMetrics> &m);
 
   void refresh_metrics_for_rank(const entity_inst_t &client, mds_rank_t rank,

From 36e24585d562d0023398196e715522cc8cf74fb3 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 13 Feb 2024 11:13:28 -0500
Subject: [PATCH 1930/2492] qa: test per-client labelled perf counters

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_admin.py | 53 +++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index 6edea8d8bb15..ee2c72f900ef 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -7,6 +7,7 @@
 from time import sleep
 
 from teuthology.exceptions import CommandFailedError
+from teuthology.contextutil import safe_while
 
 from tasks.cephfs.cephfs_test_case import CephFSTestCase, classhook
 from tasks.cephfs.filesystem import FileLayout, FSMissing
@@ -16,6 +17,58 @@
 
 log = logging.getLogger(__name__)
 
+class TestLabeledPerfCounters(CephFSTestCase):
+    CLIENTS_REQUIRED = 2
+    MDSS_REQUIRED = 1
+
+    def test_per_client_labeled_perf_counters(self):
+        """
+        That the per-client labelled perf counters depict the clients
+        performaing IO.
+        """
+        def get_counters_for(filesystem, client_id):
+            dump = self.fs.rank_tell(["counter", "dump"])
+            per_client_metrics_key = f'mds_client_metrics-{filesystem}'
+            counters = [c["counters"] for \
+                        c in dump[per_client_metrics_key] if c["labels"]["client"] == client_id]
+            return counters[0]
+
+        # sleep a bit so that we get updated clients...
+        sleep(10)
+
+        # lookout for clients...
+        dump = self.fs.rank_tell(["counter", "dump"])
+
+        fs_suffix = dump["mds_client_metrics"][0]["labels"]["fs_name"]
+        self.assertGreaterEqual(dump["mds_client_metrics"][0]["counters"]["num_clients"], 2)
+
+        per_client_metrics_key = f'mds_client_metrics-{fs_suffix}'
+        mount_a_id = f'client.{self.mount_a.get_global_id()}'
+        mount_b_id = f'client.{self.mount_b.get_global_id()}'
+
+        clients = [c["labels"]["client"] for c in dump[per_client_metrics_key]]
+        self.assertIn(mount_a_id, clients)
+        self.assertIn(mount_b_id, clients)
+
+        # write workload
+        self.mount_a.create_n_files("test_dir/test_file", 1000, sync=True)
+        with safe_while(sleep=1, tries=30, action=f'wait for counters - {mount_a_id}') as proceed:
+            counters_dump_a = get_counters_for(fs_suffix, mount_a_id)
+            while proceed():
+                if counters_dump_a["total_write_ops"] > 0 and counters_dump_a["total_write_size"] > 0:
+                    return True
+
+        # read from the other client
+        for i in range(100):
+            self.mount_b.open_background(basename=f'test_dir/test_file_{i}', write=False)
+        with safe_while(sleep=1, tries=30, action=f'wait for counters - {mount_b_id}') as proceed:
+            counters_dump_b = get_counters_for(fs_suffix, mount_b_id)
+            while proceed():
+                if counters_dump_b["total_read_ops"] > 0 and counters_dump_b["total_read_size"] > 0:
+                    return True
+
+        self.fs.teardown()
+
 class TestAdminCommands(CephFSTestCase):
     """
     Tests for administration command.

From f29dd57cd0a1f609f54d62f97025ec2d67979472 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 14 Feb 2024 13:18:23 -0500
Subject: [PATCH 1931/2492] qa: verify labelled replication perf metrics

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_mirroring.py | 83 ++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 12 deletions(-)

diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py
index afd49f6b2ffc..bad2d2daffdc 100644
--- a/qa/tasks/cephfs/test_mirroring.py
+++ b/qa/tasks/cephfs/test_mirroring.py
@@ -21,6 +21,10 @@ class TestMirroring(CephFSTestCase):
 
     MODULE_NAME = "mirroring"
 
+    PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR = "cephfs_mirror"
+    PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS = "cephfs_mirror_mirrored_filesystems"
+    PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER = "cephfs_mirror_peers"
+
     def setUp(self):
         super(TestMirroring, self).setUp()
         self.primary_fs_name = self.fs.name
@@ -40,6 +44,9 @@ def disable_mirroring_module(self):
         self.run_ceph_cmd("mgr", "module", "disable", TestMirroring.MODULE_NAME)
 
     def enable_mirroring(self, fs_name, fs_id):
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0]
+
         self.run_ceph_cmd("fs", "snapshot", "mirror", "enable", fs_name)
         time.sleep(10)
         # verify via asok
@@ -48,7 +55,19 @@ def enable_mirroring(self, fs_name, fs_id):
         self.assertTrue(res['peers'] == {})
         self.assertTrue(res['snap_dirs']['dir_count'] == 0)
 
+        # verify labelled perf counter
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        self.assertEqual(res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]["labels"]["filesystem"],
+                         fs_name)
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0]
+
+        self.assertGreater(vafter["counters"]["mirrored_filesystems"],
+                           vbefore["counters"]["mirrored_filesystems"])
+
     def disable_mirroring(self, fs_name, fs_id):
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0]
+
         self.run_ceph_cmd("fs", "snapshot", "mirror", "disable", fs_name)
         time.sleep(10)
         # verify via asok
@@ -60,6 +79,13 @@ def disable_mirroring(self, fs_name, fs_id):
         else:
             raise RuntimeError('expected admin socket to be unavailable')
 
+        # verify labelled perf counter
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR][0]
+
+        self.assertLess(vafter["counters"]["mirrored_filesystems"],
+                        vbefore["counters"]["mirrored_filesystems"])
+
     def verify_peer_added(self, fs_name, fs_id, peer_spec, remote_fs_name=None):
         # verify via asok
         res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}',
@@ -74,7 +100,11 @@ def verify_peer_added(self, fs_name, fs_id, peer_spec, remote_fs_name=None):
         else:
             self.assertTrue(self.fs_name == res['peers'][peer_uuid]['remote']['fs_name'])
 
-    def peer_add(self, fs_name, fs_id, peer_spec, remote_fs_name=None):
+    def peer_add(self, fs_name, fs_id, peer_spec, remote_fs_name=None, check_perf_counter=True):
+        if check_perf_counter:
+            res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+            vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+
         if remote_fs_name:
             self.run_ceph_cmd("fs", "snapshot", "mirror", "peer_add", fs_name, peer_spec, remote_fs_name)
         else:
@@ -82,7 +112,15 @@ def peer_add(self, fs_name, fs_id, peer_spec, remote_fs_name=None):
         time.sleep(10)
         self.verify_peer_added(fs_name, fs_id, peer_spec, remote_fs_name)
 
+        if check_perf_counter:
+            res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+            vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+            self.assertGreater(vafter["counters"]["mirroring_peers"], vbefore["counters"]["mirroring_peers"])
+
     def peer_remove(self, fs_name, fs_id, peer_spec):
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+
         peer_uuid = self.get_peer_uuid(peer_spec)
         self.run_ceph_cmd("fs", "snapshot", "mirror", "peer_remove", fs_name, peer_uuid)
         time.sleep(10)
@@ -91,6 +129,11 @@ def peer_remove(self, fs_name, fs_id, peer_spec):
                                          'fs', 'mirror', 'status', f'{fs_name}@{fs_id}')
         self.assertTrue(res['peers'] == {} and res['snap_dirs']['dir_count'] == 0)
 
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+
+        self.assertLess(vafter["counters"]["mirroring_peers"], vbefore["counters"]["mirroring_peers"])
+
     def bootstrap_peer(self, fs_name, client_name, site_name):
         outj = json.loads(self.get_ceph_cmd_stdout(
             "fs", "snapshot", "mirror", "peer_bootstrap", "create", fs_name,
@@ -101,7 +144,11 @@ def import_peer(self, fs_name, token):
         self.run_ceph_cmd("fs", "snapshot", "mirror", "peer_bootstrap",
                           "import", fs_name, token)
 
-    def add_directory(self, fs_name, fs_id, dir_name):
+    def add_directory(self, fs_name, fs_id, dir_name, check_perf_counter=True):
+        if check_perf_counter:
+            res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+            vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+
         # get initial dir count
         res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}',
                                          'fs', 'mirror', 'status', f'{fs_name}@{fs_id}')
@@ -118,7 +165,14 @@ def add_directory(self, fs_name, fs_id, dir_name):
         log.debug(f'new dir_count={new_dir_count}')
         self.assertTrue(new_dir_count > dir_count)
 
+        if check_perf_counter:
+            res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+            vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+            self.assertGreater(vafter["counters"]["directory_count"], vbefore["counters"]["directory_count"])
+
     def remove_directory(self, fs_name, fs_id, dir_name):
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
         # get initial dir count
         res = self.mirror_daemon_command(f'mirror status for fs: {fs_name}',
                                          'fs', 'mirror', 'status', f'{fs_name}@{fs_id}')
@@ -135,6 +189,11 @@ def remove_directory(self, fs_name, fs_id, dir_name):
         log.debug(f'new dir_count={new_dir_count}')
         self.assertTrue(new_dir_count < dir_count)
 
+        res = self.mirror_daemon_command(f'counter dump for fs: {fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_FS][0]
+
+        self.assertLess(vafter["counters"]["directory_count"], vbefore["counters"]["directory_count"])
+
     def check_peer_status(self, fs_name, fs_id, peer_spec, dir_name, expected_snap_name,
                           expected_snap_count):
         peer_uuid = self.get_peer_uuid(peer_spec)
@@ -268,7 +327,7 @@ def test_matching_peer(self):
         self.enable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
         try:
-            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph")
+            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError('invalid errno when adding a matching remote peer')
@@ -282,7 +341,7 @@ def test_matching_peer(self):
 
         # and explicitly specifying the spec (via filesystem name) should fail too
         try:
-            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name)
+            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name, check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError('invalid errno when adding a matching remote peer')
@@ -303,7 +362,7 @@ def test_mirror_peer_add_existing(self):
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
         # adding the same peer should be idempotent
-        self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
+        self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name, check_perf_counter=False)
 
         # remove peer
         self.peer_remove(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph")
@@ -313,7 +372,7 @@ def test_mirror_peer_add_existing(self):
     def test_peer_commands_with_mirroring_disabled(self):
         # try adding peer when mirroring is not enabled
         try:
-            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
+            self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name, check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a peer')
@@ -332,7 +391,7 @@ def test_peer_commands_with_mirroring_disabled(self):
     def test_add_directory_with_mirroring_disabled(self):
         # try adding a directory when mirroring is not enabled
         try:
-            self.add_directory(self.primary_fs_name, self.primary_fs_id, "/d1")
+            self.add_directory(self.primary_fs_name, self.primary_fs_id, "/d1", check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a directory')
@@ -344,7 +403,7 @@ def test_directory_commands(self):
         self.enable_mirroring(self.primary_fs_name, self.primary_fs_id)
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1')
         try:
-            self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1')
+            self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1', check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EEXIST:
                 raise RuntimeError(-errno.EINVAL, 'incorrect error code when re-adding a directory')
@@ -364,7 +423,7 @@ def test_directory_commands(self):
     def test_add_relative_directory_path(self):
         self.enable_mirroring(self.primary_fs_name, self.primary_fs_id)
         try:
-            self.add_directory(self.primary_fs_name, self.primary_fs_id, './d1')
+            self.add_directory(self.primary_fs_name, self.primary_fs_id, './d1', check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a relative path dir')
@@ -378,7 +437,7 @@ def test_add_directory_path_normalization(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1/d2/d3')
         def check_add_command_failure(dir_path):
             try:
-                self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path)
+                self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path, check_perf_counter=False)
             except CommandFailedError as ce:
                 if ce.exitstatus != errno.EEXIST:
                     raise RuntimeError(-errno.EINVAL, 'incorrect error code when re-adding a directory')
@@ -402,7 +461,7 @@ def test_add_ancestor_and_child_directory(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1/d2/')
         def check_add_command_failure(dir_path):
             try:
-                self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path)
+                self.add_directory(self.primary_fs_name, self.primary_fs_id, dir_path, check_perf_counter=False)
             except CommandFailedError as ce:
                 if ce.exitstatus != errno.EINVAL:
                     raise RuntimeError(-errno.EINVAL, 'incorrect error code when adding a directory')
@@ -1158,7 +1217,7 @@ def test_cephfs_mirror_peer_add_primary(self):
         # try adding the primary file system as a peer to secondary file
         # system
         try:
-            self.peer_add(self.secondary_fs_name, self.secondary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name)
+            self.peer_add(self.secondary_fs_name, self.secondary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name, check_perf_counter=False)
         except CommandFailedError as ce:
             if ce.exitstatus != errno.EINVAL:
                 raise RuntimeError('invalid errno when adding a primary file system')

From 441dbb8d80ce5afe11c2e14e087c648d63348820 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 5 Feb 2024 19:08:10 +0530
Subject: [PATCH 1932/2492] src/test: test async I/O with invalid/closed file
 handle

Fixes: https://tracker.ceph.com/issues/64313
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/test/client/nonblocking.cc | 86 +++++++++++++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc
index 427bf0723ab7..9b47291cff69 100644
--- a/src/test/client/nonblocking.cc
+++ b/src/test/client/nonblocking.cc
@@ -526,4 +526,88 @@ TEST_F(TestClient, LlreadvLlwritevZeroBytes) {
 
   client->ll_release(fh);
   ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
-}
\ No newline at end of file
+}
+
+TEST_F(TestClient, LlreadvLlwritevInvalidFileHandle) {
+  /* Test provding null or invalid file handle returns an error
+  as expected*/
+
+  Fh *fh_null = NULL;
+
+  char out_buf_0[] = "hello ";
+  char out_buf_1[] = "world\n";
+  struct iovec iov_out[2] = {
+    {out_buf_0, sizeof(out_buf_0)},
+    {out_buf_1, sizeof(out_buf_1)},
+  };
+
+  char in_buf_0[sizeof(out_buf_0)];
+  char in_buf_1[sizeof(out_buf_1)];
+  struct iovec iov_in[2] = {
+    {in_buf_0, sizeof(in_buf_0)},
+    {in_buf_1, sizeof(in_buf_1)},
+  };
+
+  std::unique_ptr<C_SaferCond> writefinish = nullptr;
+  std::unique_ptr<C_SaferCond> readfinish = nullptr;
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-null-fh"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-null-fh"));
+
+  int64_t rc;
+  bufferlist bl;
+  ssize_t bytes_written = 0, bytes_read = 0;
+
+  rc = client->ll_preadv_pwritev(fh_null, iov_out, 2, 0, true,
+                                 writefinish.get(), nullptr);
+  ASSERT_EQ(rc, 0);
+  bytes_written = writefinish->wait();
+  ASSERT_EQ(bytes_written, -CEPHFS_EBADF);
+
+  rc = client->ll_preadv_pwritev(fh_null, iov_in, 2, 0, false,
+                                 readfinish.get(), &bl);
+  ASSERT_EQ(rc, 0);
+  bytes_read = readfinish->wait();
+  ASSERT_EQ(bytes_read, -CEPHFS_EBADF);
+  ASSERT_EQ(bl.length(), 0);
+
+  // test after closing the file handle
+  int mypid = getpid();
+  char filename[256];
+
+  client->unmount();
+  TearDown();
+  SetUp();
+
+  sprintf(filename, "test_llreadvllwritevinvalidfhfile%u", mypid);
+
+  Inode *root, *file;
+  root = client->get_root();
+  ASSERT_NE(root, (Inode *)NULL);
+
+  Fh *fh;
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, client->ll_createx(root, filename, 0666,
+				  O_RDWR | O_CREAT | O_TRUNC,
+				  &file, &fh, &stx, 0, 0, myperm));
+
+  client->ll_release(fh);
+  ASSERT_EQ(0, client->ll_unlink(root, filename, myperm));
+
+  writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-invalid-fh"));
+  readfinish.reset(new C_SaferCond("test-nonblocking-readfinish-invalid-fh"));
+
+  rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, writefinish.get(),
+                                 nullptr);
+  ASSERT_EQ(rc, 0);
+  bytes_written = writefinish->wait();
+  ASSERT_EQ(bytes_written, -CEPHFS_EBADF);
+
+  rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, readfinish.get(),
+                                 &bl);
+  ASSERT_EQ(rc, 0);
+  bytes_read = readfinish->wait();
+  ASSERT_EQ(bytes_read, -CEPHFS_EBADF);
+  ASSERT_EQ(bl.length(), 0);
+}

From e1452649b8e765e5ab1efdd71fe1d15461b6ef14 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Wed, 12 Apr 2023 09:39:52 +0000
Subject: [PATCH 1933/2492] common/tracer: fix decoding when jaeger tracing is
 disabled

We aren't currently using jaeger tracing on Windows. The issue is
that Windows hosts (or any other host that doesn't use jaeger)
are experiencing message decoding failures after a recent change [1].

This change updates the tracer encoding so that messages from
non-jaeger hosts may be decoded by services that use jaeger.

[1] https://github.com/ceph/ceph/pull/47457

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>

This commit rebrings 3701ffa6733b001d4278a0b68395c5efe2382f25 which
got reverted due to an implicit dependency with other revert. Please
see https://github.com/ceph/ceph/pull/52114#issuecomment-1950288188.

Conflicts:
	src/common/tracer.h
	  formatting conflict with 7179ac0037320480f089e1a9e291cdc37282d250
---
 src/common/tracer.h | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/common/tracer.h b/src/common/tracer.h
index b039d304a79a..ee0b74407545 100644
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -152,9 +152,20 @@ struct Tracer {
   jspan_ptr add_span(std::string_view span_name, const jspan_context& parent_ctx) { return {}; }
 };
 
-inline void encode(const jspan_context& span, bufferlist& bl, uint64_t f=0) {}
-inline void decode(jspan_context& span_ctx, ceph::buffer::list::const_iterator& bl) {}
+inline void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f = 0) {
+  ENCODE_START(1, 1, bl);
+  // jaeger is missing, set "is_valid" to false.
+  bool is_valid = false;
+  encode(is_valid, bl);
+  ENCODE_FINISH(bl);
+}
 
+inline void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
+  DECODE_START(254, bl);
+  // jaeger is missing, consume the buffer but do not decode it.
+  DECODE_FINISH(bl);
 }
 
+} // namespace tracing
+
 #endif // !HAVE_JAEGER

From b4b1868a0047890ada2c8b39c3a932ae51255714 Mon Sep 17 00:00:00 2001
From: Oguzhan Ozmen <oozmen@bloomberg.net>
Date: Tue, 13 Feb 2024 11:40:38 -0500
Subject: [PATCH 1934/2492] rgw/http/client-side: disable curl path
 normalization

test_multi.py:test_object_sync is updated to reproduce the issue.
Without the fix, objects "." and ".." are not replicated and the test
fails (times out).

Fixes: https://tracker.ceph.com/issues/64366
Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
---
 src/rgw/rgw_http_client.cc      | 2 ++
 src/test/rgw/rgw_multi/tests.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_http_client.cc b/src/rgw/rgw_http_client.cc
index 587b33d04780..a1873586b013 100644
--- a/src/rgw/rgw_http_client.cc
+++ b/src/rgw/rgw_http_client.cc
@@ -591,6 +591,8 @@ int RGWHTTPClient::init_request(rgw_http_req_data *_req_data)
   curl_easy_setopt(easy_handle, CURLOPT_READFUNCTION, send_http_data);
   curl_easy_setopt(easy_handle, CURLOPT_READDATA, (void *)req_data);
   curl_easy_setopt(easy_handle, CURLOPT_BUFFERSIZE, cct->_conf->rgw_curl_buffersize);
+  curl_easy_setopt(easy_handle, CURLOPT_PATH_AS_IS, 1L);
+
   if (send_data_hint || is_upload_request(method)) {
     curl_easy_setopt(easy_handle, CURLOPT_UPLOAD, 1L);
   }
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 3f5ae0fea5d3..eed721ad59ba 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -612,7 +612,8 @@ def test_object_sync():
     zonegroup_conns = ZonegroupConns(zonegroup)
     buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
 
-    objnames = [ 'myobj', '_myobj', ':', '&' ]
+    objnames = [ 'myobj', '_myobj', ':', '&', '.', '..', '...',  '.o', '.o.']
+
     content = 'asdasd'
 
     # don't wait for meta sync just yet

From 39c809b33fe27064277292b721639cbb144fac6a Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 17 Feb 2024 15:28:24 +1000
Subject: [PATCH 1935/2492] doc/rados: add PG definition

Add a definition of Placement Groups to
doc/rados/operations/placement-groups.rst.

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/placement-groups.rst | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index d32feaf95235..f9b87528b528 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -4,9 +4,19 @@
  Placement Groups
 ==================
 
+Placement groups (PGs) are subsets of each logical Ceph pool. Placement groups
+perform the function of placing objects (as a group) into OSDs. Ceph manages
+data internally at placement-group granularity: this scales better than would
+managing individual RADOS objects. A cluster that has a larger number of
+placement groups (for example, 150 per OSD) is better balanced than an
+otherwise identical cluster with a smaller number of placement groups.
+
+Ceph’s internal RADOS objects are each mapped to a specific placement group,
+and each placement group belongs to exactly one Ceph pool.
+
 See Sage Weil's blog post `New in Nautilus: PG merging and autotuning
 <https://ceph.io/en/news/blog/2019/new-in-nautilus-pg-merging-and-autotuning/>`_
-for information about the relationship of placement groups to pools and to
+for more information about the relationship of placement groups to pools and to
 objects.
 
 .. _pg-autoscaler:

From 44ec668d43951c2710271fb3ca1799c4c4ded940 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 18 Feb 2024 16:01:46 +1000
Subject: [PATCH 1936/2492] doc/man: edit "manipulating the omap key"

Edit the section "Manipulating the Object Map Key" in
doc/man/8/ceph-objectstore-tool.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/man/8/ceph-objectstore-tool.rst | 41 ++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/doc/man/8/ceph-objectstore-tool.rst b/doc/man/8/ceph-objectstore-tool.rst
index 88b0da6ca5bb..9ac294f0c8b5 100644
--- a/doc/man/8/ceph-objectstore-tool.rst
+++ b/doc/man/8/ceph-objectstore-tool.rst
@@ -246,45 +246,56 @@ Procedure
 Manipulating the Object Map Key
 -------------------------------
 
-Use the **ceph-objectstore-tool** utility to change the object map (OMAP) key. You need to provide the data path, the placement group identifier (PG ID), the object, and the key in the OMAP.
-Note
+Use the **ceph-objectstore-tool** utility to change the object map (OMAP) key.
+Provide the data path, the placement group identifier (PG ID), the object, and
+the key in the OMAP.
 
 Prerequisites
+^^^^^^^^^^^^^
 
     * Having root access to the Ceph OSD node.
     * Stopping the ceph-osd daemon. 
 
-Procedure
+Commands
+^^^^^^^^
+
+Run the commands in this section as ``root`` on an OSD node.
+
+* **Getting the object map key**
 
-    Get the object map key:
+   Syntax:
 
-    Syntax::
+   .. code-block:: ini 
      
-       ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omap $KEY > $OBJECT_MAP_FILE_NAME
+      ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omap $KEY > $OBJECT_MAP_FILE_NAME
 
    Example::
 
-    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omap "" > zone_info.default.omap.txt
+    ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omap "" > zone_info.default.omap.txt
 
-   Set the object map key:
+* **Setting the object map key**
 
-   Syntax::
+   Syntax:
+
+   .. code-block:: ini 
 
-    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-omap $KEY < $OBJECT_MAP_FILE_NAME
+      ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-omap $KEY < $OBJECT_MAP_FILE_NAME
 
    Example::
 
-    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omap "" < zone_info.default.omap.txt
+    ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omap "" < zone_info.default.omap.txt
 
-   Remove the object map key:
+* **Removing the object map key**
 
-   Syntax::
+   Syntax:
+
+   .. code-block:: ini 
 
-    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-omap $KEY
+      ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-omap $KEY
 
    Example::
 
-    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-omap ""
+    ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-omap ""
 
 
 Listing an Object's Attributes

From 0eebbc3f02ad153d41d23de227bd989da50e54cf Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 18 Feb 2024 10:29:13 -0500
Subject: [PATCH 1937/2492] rgw: RGWSI_SysObj_Cache::remove() invalidates after
 successful delete

invalidating the cache before the librados delete means that a racing call
to `RGWSI_SysObj_Cache::read()` may succeed and repopulate the cache. in
that case, subsequent reads will continue to return cached data even after
the librados delete succeeds

Fixes: https://tracker.ceph.com/issues/64480

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/services/svc_sys_obj_cache.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc
index cd4e2c405abf..0fa926d0f4b5 100644
--- a/src/rgw/services/svc_sys_obj_cache.cc
+++ b/src/rgw/services/svc_sys_obj_cache.cc
@@ -89,6 +89,11 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
                                optional_yield y)
 
 {
+  int r = RGWSI_SysObj_Core::remove(dpp, objv_tracker, obj, y);
+  if (r < 0) {
+    return r;
+  }
+
   rgw_pool pool;
   string oid;
   normalize_pool_and_obj(obj.pool, obj.oid, pool, oid);
@@ -97,12 +102,12 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
   cache.invalidate_remove(dpp, name);
 
   ObjectCacheInfo info;
-  int r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
+  r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to distribute cache: r=" << r << dendl;
-  }
+  } // not fatal
 
-  return RGWSI_SysObj_Core::remove(dpp, objv_tracker, obj, y);
+  return 0;
 }
 
 int RGWSI_SysObj_Cache::read(const DoutPrefixProvider *dpp,

From fe5c13d2e207473d48aa818484584d67dc3d23b5 Mon Sep 17 00:00:00 2001
From: Xavi Hernandez <xhernandez@gmail.com>
Date: Fri, 16 Feb 2024 19:14:07 +0100
Subject: [PATCH 1938/2492] client: fix leak of file handles

Based on posix specification, the fd passed to fdopendir() will be
closed by closedir(). However CephFS client wasn't doing that. If the
user opened a directory using ceph_openat(), for example, and then
passed the returned fd to ceph_fdopendir(), the created Fh associated
with the new open was never destroyed.

This patch records the fd used in ceph_fdopendir() so that it can be
closed when ceph_closedir() is called.

Fixes: https://tracker.ceph.com/issues/64479
Signed-off-by: Xavi Hernandez <xhernandez@gmail.com>
---
 src/client/Client.cc | 18 +++++++++++++-----
 src/client/Client.h  |  6 ++++--
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 6ee5e742b9b2..5290351778cd 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -257,10 +257,10 @@ int Client::get_fd_inode(int fd, InodeRef *in) {
   return r;
 }
 
-dir_result_t::dir_result_t(Inode *in, const UserPerm& perms)
+dir_result_t::dir_result_t(Inode *in, const UserPerm& perms, int fd)
   : inode(in), offset(0), next_offset(2),
     release_count(0), ordered_count(0), cache_index(0), start_shared_gen(0),
-    perms(perms)
+    perms(perms), fd(fd)
   { }
 
 void Client::_reset_faked_inos()
@@ -9122,7 +9122,9 @@ int Client::fdopendir(int dirfd, dir_result_t **dirpp, const UserPerm &perms) {
       return r;
     }
   }
-  r = _opendir(dirinode.get(), dirpp, perms);
+  // Posix says that closedir will also close the file descriptor passed to fdopendir, so we associate
+  // dirfd to the new dir_result_t so that it can be closed later.
+  r = _opendir(dirinode.get(), dirpp, perms, dirfd);
   /* if ENOTDIR, dirpp will be an uninitialized point and it's very dangerous to access its value */
   if (r != -CEPHFS_ENOTDIR) {
       tout(cct) << (uintptr_t)*dirpp << std::endl;
@@ -9130,11 +9132,11 @@ int Client::fdopendir(int dirfd, dir_result_t **dirpp, const UserPerm &perms) {
   return r;
 }
 
-int Client::_opendir(Inode *in, dir_result_t **dirpp, const UserPerm& perms)
+int Client::_opendir(Inode *in, dir_result_t **dirpp, const UserPerm& perms, int fd)
 {
   if (!in->is_dir())
     return -CEPHFS_ENOTDIR;
-  *dirpp = new dir_result_t(in, perms);
+  *dirpp = new dir_result_t(in, perms, fd);
   opened_dirs.insert(*dirpp);
   ldout(cct, 8) << __func__ << "(" << in->ino << ") = " << 0 << " (" << *dirpp << ")" << dendl;
   return 0;
@@ -9162,6 +9164,12 @@ void Client::_closedir(dir_result_t *dirp)
   }
   _readdir_drop_dirp_buffer(dirp);
   opened_dirs.erase(dirp);
+
+  /* Close the associated fd if this dir_result_t comes from an fdopendir request. */
+  if (dirp->fd >= 0) {
+    _close(dirp->fd);
+  }
+
   delete dirp;
 }
 
diff --git a/src/client/Client.h b/src/client/Client.h
index 712583aa0d3c..61098481b06e 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -163,7 +163,7 @@ struct dir_result_t {
   };
 
 
-  explicit dir_result_t(Inode *in, const UserPerm& perms);
+  explicit dir_result_t(Inode *in, const UserPerm& perms, int fd);
 
 
   static uint64_t make_fpos(unsigned h, unsigned l, bool hash) {
@@ -240,6 +240,8 @@ struct dir_result_t {
 
   std::vector<dentry> buffer;
   struct dirent de;
+
+  int fd;                // fd attached using fdopendir (-1 if none)
 };
 
 class Client : public Dispatcher, public md_config_obs_t {
@@ -1563,7 +1565,7 @@ class Client : public Dispatcher, public md_config_obs_t {
 
   void fill_dirent(struct dirent *de, const char *name, int type, uint64_t ino, loff_t next_off);
 
-  int _opendir(Inode *in, dir_result_t **dirpp, const UserPerm& perms);
+  int _opendir(Inode *in, dir_result_t **dirpp, const UserPerm& perms, int fd = -1);
   void _readdir_drop_dirp_buffer(dir_result_t *dirp);
   bool _readdir_have_frag(dir_result_t *dirp);
   void _readdir_next_frag(dir_result_t *dirp);

From 450f2692351c43c80f3246d2d22b76249e1ba30a Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Mon, 19 Feb 2024 16:52:07 +0800
Subject: [PATCH 1939/2492] debian/control: add new dependencies protobuf for
 crismon

As PR https://github.com/ceph/ceph/pull/55444 update seastar version,
the new seastar imports new dependencies protobuf.

Fixes: https://tracker.ceph.com/issues/64420

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 debian/control | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/control b/debian/control
index b4967b398f4d..bd5c88aaf710 100644
--- a/debian/control
+++ b/debian/control
@@ -83,11 +83,13 @@ Build-Depends: automake,
                libndctl-dev (>= 63) <pkg.ceph.pmdk>,
                libpmem-dev <pkg.ceph.pmdk>,
                libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
+	       libprotobuf-dev <pkg.ceph.crimson>,
                ninja-build,
                nlohmann-json3-dev,
                patch,
                pkg-config,
                prometheus <pkg.ceph.check>,
+	       protobuf-compiler <pkg.ceph.crimson>,
                python3-all-dev,
                python3-cherrypy3,
                python3-natsort,

From 68400ff5450db1fd2c582ef016c9859829dbdf89 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 19 Feb 2024 12:22:16 +0000
Subject: [PATCH 1940/2492] crimson/osd/main: enable multicore client msgr

Taken from: f78e99c05943ea336a3fe2cc1568fea81e6a9855

Co-authored-by: Yingxin Cheng <yingxin.cheng@intel.com>
Co-authored-by: Chunmei Liu <chunmei.liu@intel.com>
Co-authored-by: Xinyu Huang <xinyu.huang@intel.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/main.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc
index 1e817415d03b..6a569702deba 100644
--- a/src/crimson/osd/main.cc
+++ b/src/crimson/osd/main.cc
@@ -186,8 +186,13 @@ int main(int argc, const char* argv[])
           const auto nonce = crimson::osd::get_nonce();
           crimson::net::MessengerRef cluster_msgr, client_msgr;
           crimson::net::MessengerRef hb_front_msgr, hb_back_msgr;
+          for (auto [msgr, name] : {make_pair(std::ref(client_msgr), "client"s)}) {
+            msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),
+                                                   name,
+                                                   nonce,
+                                                   false);
+          }
           for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s),
-                                    make_pair(std::ref(client_msgr), "client"s),
                                     make_pair(std::ref(hb_front_msgr), "hb_front"s),
                                     make_pair(std::ref(hb_back_msgr), "hb_back"s)}) {
             msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),

From 8aaa0bca47fa49073e797ee427bcec282ef1adbe Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 19 Feb 2024 12:42:34 +0000
Subject: [PATCH 1941/2492] mgr/dashboard: fix subvolume group edit

Fixes: https://tracker.ceph.com/issues/64487
Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-subvolumegroup-form.component.html         |  3 +--
 .../cephfs-subvolumegroup-form.component.ts           | 11 ++++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.html
index 58bb86021bdb..bd91014a0e6a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.html
@@ -132,8 +132,7 @@
                                    inputField="mode"
                                    [isTableForOctalMode]="true"
                                    [initialValue]="initialMode"
-                                   [scopes]="scopes"
-                                   [isDisabled]="isEdit"></cd-checked-table-form>
+                                   [scopes]="scopes"></cd-checked-table-form>
           </div>
         </div>
       </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
index 8ecf1eafa8c9..accbb61b7eee 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
@@ -123,9 +123,6 @@ export class CephfsSubvolumegroupFormComponent extends CdForm implements OnInit
       .subscribe((resp: any) => {
         // Disabled these fields since its not editable
         this.subvolumegroupForm.get('subvolumegroupName').disable();
-        this.subvolumegroupForm.get('pool').disable();
-        this.subvolumegroupForm.get('uid').disable();
-        this.subvolumegroupForm.get('gid').disable();
 
         this.subvolumegroupForm.get('subvolumegroupName').setValue(this.subvolumegroupName);
         if (resp.bytes_quota !== 'infinite') {
@@ -155,10 +152,14 @@ export class CephfsSubvolumegroupFormComponent extends CdForm implements OnInit
           task: new FinishedTask('cephfs/subvolume/group/' + URLVerbs.EDIT, {
             subvolumegroupName: subvolumegroupName
           }),
-          call: this.cephfsSubvolumeGroupService.update(
+          call: this.cephfsSubvolumeGroupService.create(
             this.fsName,
             subvolumegroupName,
-            String(editSize)
+            pool,
+            String(editSize),
+            uid,
+            gid,
+            mode
           )
         })
         .subscribe({

From 39ad6264aa1c97b04974e04046033887325ed2b2 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 19 Feb 2024 18:41:45 +1000
Subject: [PATCH 1942/2492] doc/cephfs: edit add-remove-mds

Disambiguate a note in doc/cephfs/add-remove-mds.rst to help readers
distinguish between cases in which they might want to use an automated
tool such as cephadm to deploy MDSes and cases in which they might want
to manually deploy MDSes.

See: https://github.com/ceph/ceph/pull/45639

Tracker: https://tracker.ceph.com/issues/54551

Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/cephfs/add-remove-mds.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/doc/cephfs/add-remove-mds.rst b/doc/cephfs/add-remove-mds.rst
index 4f5ee06aa8b7..2fec7873d228 100644
--- a/doc/cephfs/add-remove-mds.rst
+++ b/doc/cephfs/add-remove-mds.rst
@@ -1,11 +1,13 @@
 .. _cephfs_add_remote_mds:
 
-.. note::
-   It is highly recommended to use :doc:`/cephadm/index` or another Ceph
-   orchestrator for setting up the ceph cluster. Use this approach only if you
-   are setting up the ceph cluster manually. If one still intends to use the
-   manual way for deploying MDS daemons, :doc:`/cephadm/services/mds/` can
-   also be used.
+.. warning:: The material on this page is to be used only for manually setting
+   up a Ceph cluster. If you intend to use an automated tool such as
+   :doc:`/cephadm/index` to set up a Ceph cluster, do not use the
+   instructions on this page.
+
+.. note:: If you are certain that you know what you are doing and you intend to
+   manually deploy MDS daemons, see :doc:`/cephadm/services/mds/` before
+   proceeding.
 
 ============================
  Deploying Metadata Servers

From bbaadacd48fd9ba679ef1495c1094dfb7401447a Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 19 Feb 2024 20:44:46 -0500
Subject: [PATCH 1943/2492] client: log debug message when requesting unmount

Importantly: do this before any locks are to be acquired.

Fixes: https://tracker.ceph.com/issues/64503
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/client/Client.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 6ee5e742b9b2..d52aa6a5bc7a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -6981,11 +6981,13 @@ void Client::_unmount(bool abort)
 
 void Client::unmount()
 {
+  ldout(cct, 2) << __func__ << dendl;
   _unmount(false);
 }
 
 void Client::abort_conn()
 {
+  ldout(cct, 2) << __func__ << dendl;
   _unmount(true);
 }
 

From 78abfeaff27fee343fb664db633de5b221699a73 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 15 Feb 2024 10:28:32 -0500
Subject: [PATCH 1944/2492] mds: reverse MDSMap encoding of
 max_xattr_size/bal_rank_mask

Commit e134c890 adds the bal_rank_mask with encoded (ev) version 17.  This was
merged into main Oct 2022 and made it into the reef release normally.

Commit 7b8def5c adds the max_xattr_size also with encoded (ev) version 17 but
places it before bal_rank_mask. This is problematic as there were no plans to
backport e134c890 to quincy or pacific so piggybacking on the ev 17 bump would
not work and otherwise would require the backports to be done as a set to
ensure consistency (including with the kernel client).

However, the real issue is that 7b8def5c was not merged until after reef was
already cut. This required 7b8def5c to be backported separately in [1] which
was not merged until after v18.2.1 (current reef HEAD as of this commit).
Ultimately, this means that there are reef versions (v18.2.[01]) in the wild
which expect bal_rank_mask to be encoded at ev17 and not (max_xattr_size,
bal_rank_mask). Adding to the complications, the kernel client has already
merged code [2] expecting max_xattr_size for ev17.

It was decided in a github discussion [3] to move bal_rank_mask to ev18 to
avoid updating the kernel client which was done in the main branch via 36ee8e7e
and update the reef max_xattr_size backport with the same change (d8cebd67).

Unfortunately, this breaks upgrades v18.2.[01] to newer reef versions or to
main.  The reason is that monitors will encode v17 with bal_rank_mask
(max_xattr_size is not merged yet) and send that to upgraded mgrs (which are
upgraded first). The mgr will attempt to decode bal_rank_mask as a uint64_t
(max_xattr_size) but fail because an empty (by default) bal_rank_mask is simply
encoded as a signed 32-bit integer. Consequently, the mgr will fail decoding
with:

    failed to decode message of type 45 v1: End of buffer [buffer:2]

Of course the problem does not stop there, even if the mgr were able to handle
this, the monitors/mds/clients would fail in similar fashion.

So the only choice left is to fix max_xattr_size to be encoded at ev18.
Fortunately, v18.2.2 has not been released nor has any max_xattr_size backport
to quincy/pacific been merged. The main downside will be that kernels will
wrongly decode ev17 (which is already true for ceph clusters running
v18.2.[01]). A follow-up kernel fix will be required.

[1] https://tracker.ceph.com/issues/59405
[2] linux.git d93231a6bc8a452323d5fef16cca7107ce483a27
[3] https://github.com/ceph/ceph/pull/53340#discussion_r1399255031

Fixes: https://tracker.ceph.com/issues/64440
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSMap.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 978f7b7dee15..51f514eb953c 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -800,8 +800,8 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
     encode(min_compat_client, bl);
   }
   encode(required_client_features, bl);
-  encode(max_xattr_size, bl);
   encode(bal_rank_mask, bl);
+  encode(max_xattr_size, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -949,11 +949,11 @@ void MDSMap::decode(bufferlist::const_iterator& p)
   }
 
   if (ev >= 17) {
-    decode(max_xattr_size, p);
+    decode(bal_rank_mask, p);
   }
 
   if (ev >= 18) {
-    decode(bal_rank_mask, p);
+    decode(max_xattr_size, p);
   }
 
   /* All MDS since at least v14.0.0 understand INLINE */

From 52c09aa1e50e3d9f36909354c43da89dec8f9bb5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 14 Feb 2024 21:49:17 -0500
Subject: [PATCH 1945/2492] qa: do upgrades from quincy and older reef minor
 releases

Fixes: https://tracker.ceph.com/issues/64441
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../mds_upgrade_sequence/tasks/0-from/reef/$  |  0
 .../tasks/0-from/reef/reef.yaml               | 31 +++++++++++++++++++
 .../tasks/0-from/reef/v18.2.0.yaml            | 31 +++++++++++++++++++
 .../tasks/0-from/reef/v18.2.1.yaml            | 31 +++++++++++++++++++
 4 files changed, 93 insertions(+)
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/$
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/reef.yaml
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.0.yaml
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.1.yaml

diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/$ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/$
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/reef.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/reef.yaml
new file mode 100644
index 000000000000..c53e8b55d1b6
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/reef.yaml
@@ -0,0 +1,31 @@
+meta:
+- desc: |
+   setup ceph/reef
+
+tasks:
+- install:
+    branch: reef
+    exclude_packages:
+      - ceph-volume
+- print: "**** done install task..."
+- cephadm:
+    image: quay.ceph.io/ceph-ci/ceph:reef
+    roleless: true
+    compiled_cephadm_branch: reef
+    conf:
+      osd:
+        #set config option for which cls modules are allowed to be loaded / used
+        osd_class_load_list: "*"
+        osd_class_default_list: "*"
+- print: "**** done end installing reef cephadm ..."
+- cephadm.shell:
+    host.a:
+      - ceph config set mgr mgr/cephadm/use_repo_digest true --force
+- print: "**** done cephadm.shell ceph config set mgr..."
+- cephadm.shell:
+    host.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.0.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.0.yaml
new file mode 100644
index 000000000000..98bb210d1807
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.0.yaml
@@ -0,0 +1,31 @@
+meta:
+- desc: |
+   setup ceph/v18.2.0
+
+tasks:
+- install:
+    tag: v18.2.0
+    exclude_packages:
+      - ceph-volume
+- print: "**** done install task..."
+- cephadm:
+    image: quay.io/ceph/ceph:v18.2.0
+    roleless: true
+    compiled_cephadm_branch: reef
+    conf:
+      osd:
+        #set config option for which cls modules are allowed to be loaded / used
+        osd_class_load_list: "*"
+        osd_class_default_list: "*"
+- print: "**** done end installing v18.2.0 cephadm ..."
+- cephadm.shell:
+    host.a:
+      - ceph config set mgr mgr/cephadm/use_repo_digest true --force
+- print: "**** done cephadm.shell ceph config set mgr..."
+- cephadm.shell:
+    host.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.1.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.1.yaml
new file mode 100644
index 000000000000..ce45d9ea9bec
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-from/reef/v18.2.1.yaml
@@ -0,0 +1,31 @@
+meta:
+- desc: |
+   setup ceph/v18.2.1
+
+tasks:
+- install:
+    tag: v18.2.1
+    exclude_packages:
+      - ceph-volume
+- print: "**** done install task..."
+- cephadm:
+    image: quay.io/ceph/ceph:v18.2.1
+    roleless: true
+    compiled_cephadm_branch: reef
+    conf:
+      osd:
+        #set config option for which cls modules are allowed to be loaded / used
+        osd_class_load_list: "*"
+        osd_class_default_list: "*"
+- print: "**** done end installing v18.2.1 cephadm ..."
+- cephadm.shell:
+    host.a:
+      - ceph config set mgr mgr/cephadm/use_repo_digest true --force
+- print: "**** done cephadm.shell ceph config set mgr..."
+- cephadm.shell:
+    host.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls

From 427ad7c0f93d374db45b86073f2c7787d2e8cb9a Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 15 Feb 2024 14:31:47 -0500
Subject: [PATCH 1946/2492] mds: update comment on kclient decoding of MDSMap

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSMap.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 51f514eb953c..4096b23ab9d8 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -852,7 +852,8 @@ void MDSMap::decode(bufferlist::const_iterator& p)
     decode(cas_pool, p);
   }
 
-  // kclient ignores everything from here
+  // kclient skips most of what's below
+  // see fs/ceph/mdsmap.c for current decoding
   __u16 ev = 1;
   if (struct_v >= 2)
     decode(ev, p);

From 0e5e847f087cde893cc49ed3434c78006055d0b5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 15 Feb 2024 20:16:20 -0500
Subject: [PATCH 1947/2492] qa: remove ignorelist error parenthesis

Some messages are duplicated to the cluster log lookign like:

    2024-02-15T22:54:31.244 INFO:teuthology.orchestra.run.smithi033.stdout:2024-02-15T22:50:00.000263+0000 mon.smithi033 (mon.0) 558 : cluster 4 [ERR] MDS_ALL_DOWN: 1 filesystem is offline

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/ignorelist_health.yaml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
index d8b8192882e7..ad7a8e03b765 100644
--- a/qa/cephfs/overrides/ignorelist_health.yaml
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -1,13 +1,13 @@
 overrides:
   ceph:
     log-ignorelist:
+      - FS_DEGRADED
+      - FS_INLINE_DATA_DEPRECATED
+      - FS_WITH_FAILED_MDS
+      - MDS_ALL_DOWN
+      - MDS_DAMAGE
+      - MDS_DEGRADED
+      - MDS_FAILED
+      - MDS_UP_LESS_THAN_MAX
+      - POOL_APP_NOT_ENABLED
       - overall HEALTH_
-      - \(FS_DEGRADED\)
-      - \(MDS_FAILED\)
-      - \(MDS_DEGRADED\)
-      - \(FS_WITH_FAILED_MDS\)
-      - \(MDS_DAMAGE\)
-      - \(MDS_ALL_DOWN\)
-      - \(MDS_UP_LESS_THAN_MAX\)
-      - \(FS_INLINE_DATA_DEPRECATED\)
-      - \(POOL_APP_NOT_ENABLED\)

From 560300f1c5ea44a3c81d87f3bf9d10727622901c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 15 Feb 2024 20:23:32 -0500
Subject: [PATCH 1948/2492] qa: ignore MDS_INSUFFICIENT_STANDBY

This is expected when bringing a volume and its mds up initially.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/ignorelist_health.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
index ad7a8e03b765..9fd5924703c1 100644
--- a/qa/cephfs/overrides/ignorelist_health.yaml
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -8,6 +8,7 @@ overrides:
       - MDS_DAMAGE
       - MDS_DEGRADED
       - MDS_FAILED
+      - MDS_INSUFFICIENT_STANDBY
       - MDS_UP_LESS_THAN_MAX
       - POOL_APP_NOT_ENABLED
       - overall HEALTH_

From 75d76f97b0d7c842bd52cb609b00ffe3508c01b5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 16 Feb 2024 09:46:18 -0500
Subject: [PATCH 1949/2492] qa: ignore warning "Replacing daemon"

This is expected for cephadm deployments where join_fs is configured, causing
affinity replacements.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/ignorelist_health.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
index 9fd5924703c1..90811d6f233b 100644
--- a/qa/cephfs/overrides/ignorelist_health.yaml
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -12,3 +12,4 @@ overrides:
       - MDS_UP_LESS_THAN_MAX
       - POOL_APP_NOT_ENABLED
       - overall HEALTH_
+      - Replacing daemon

From 4bcaaa45eb45f12df2d1835831c01fe0eb6ca595 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 16 Feb 2024 10:09:53 -0500
Subject: [PATCH 1950/2492] qa: ignore OSD_DOWN during cephadm upgrades

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../mds_upgrade_sequence/overrides/ignorelist_upgrade.yaml    | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/ignorelist_upgrade.yaml

diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/ignorelist_upgrade.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/ignorelist_upgrade.yaml
new file mode 100644
index 000000000000..713adb9628ab
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/ignorelist_upgrade.yaml
@@ -0,0 +1,4 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - OSD_DOWN

From 182f4c0f54385ab15ba13e1f06c1146ba16fd64b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 16 Feb 2024 13:25:52 -0500
Subject: [PATCH 1951/2492] qa: test fuse/kclient for mds upgrade seq

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../fs/upgrade/mds_upgrade_sequence/tasks/2-client/fuse.yaml   | 3 +++
 .../tasks/{2-client.yaml => 2-client/kclient.yaml}             | 0
 2 files changed, 3 insertions(+)
 create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/fuse.yaml
 rename qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/{2-client.yaml => 2-client/kclient.yaml} (100%)

diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/fuse.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/fuse.yaml
new file mode 100644
index 000000000000..5318fd1a936e
--- /dev/null
+++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/fuse.yaml
@@ -0,0 +1,3 @@
+tasks:
+- ceph-fuse:
+- print: "**** done client"
diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/kclient.yaml
similarity index 100%
rename from qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml
rename to qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client/kclient.yaml

From 0f660886ddae734bba9ac0071edeeef964a87684 Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Sat, 27 Jan 2024 14:59:11 +0800
Subject: [PATCH 1952/2492] CMakeLists: Modify CEPH_TEST_TIMEOUT from 3600s to
 7200s

There are some older Arm server running pretty slow, the make
check jobs like `check-generated.sh` are killed as the job timeout.
Make CEPH_TEST_TIMEOUT more longer.

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 CMakeLists.txt                  | 2 +-
 cmake/modules/AddCephTest.cmake | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f2df55a75e12..5e80c260b403 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -586,7 +586,7 @@ option(PG_DEBUG_REFS "PG Ref debugging is enabled" OFF)
 
 option(WITH_TESTS "enable the build of ceph-test package scripts/binaries" ON)
 set(UNIT_TESTS_BUILT ${WITH_TESTS})
-set(CEPH_TEST_TIMEOUT 3600 CACHE STRING 
+set(CEPH_TEST_TIMEOUT 7200 CACHE STRING
   "Maximum time before a CTest gets killed" )
 
 # fio
diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index 2784567c6871..591552f834ea 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -19,7 +19,6 @@ function(add_ceph_test test_name test_path)
     PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
     PYTHONPATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cython_modules/lib.3:${CMAKE_SOURCE_DIR}/src/pybind
     CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV})
-  # none of the tests should take more than 1 hour to complete
   set_property(TEST ${test_name}
     PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
 endfunction()

From 8af910f877d1c7b873c2a13cec5d4ea1a4de4a79 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Tue, 20 Feb 2024 22:06:11 +1000
Subject: [PATCH 1953/2492] doc/radosgw: edit multisite.rst

Edit "Varieties of Multi-Site Configuration" in
doc/radosgw/multisite.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/radosgw/multisite.rst | 59 +++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
index cbd664971e71..7c03a4d14066 100644
--- a/doc/radosgw/multisite.rst
+++ b/doc/radosgw/multisite.rst
@@ -24,49 +24,48 @@ Varieties of Multi-site Configuration
 
 .. versionadded:: Jewel
 
-Beginning with the Kraken release, Ceph supports several multi-site
-configurations for the Ceph Object Gateway:
+Since the Kraken release, Ceph has supported several multi-site configurations
+for the Ceph Object Gateway:
 
-- **Multi-zone:** A more advanced topology, the "multi-zone" configuration, is
-  possible. A multi-zone configuration consists of one zonegroup and
-  multiple zones, with each zone consisting of one or more `ceph-radosgw`
-  instances. **Each zone is backed by its own Ceph Storage Cluster.**
+- **Multi-zone:** The "multi-zone" configuration has a complex topology. A
+  multi-zone configuration consists of one zonegroup and multiple zones. Each
+  zone consists of one or more `ceph-radosgw` instances. **Each zone is backed
+  by its own Ceph Storage Cluster.**
   
   The presence of multiple zones in a given zonegroup provides disaster
   recovery for that zonegroup in the event that one of the zones experiences a
-  significant failure. Beginning with the Kraken release, each zone is active
-  and can receive write operations. A multi-zone configuration that contains
-  multiple active zones enhances disaster recovery and can also be used as a
-  foundation for content delivery networks. 
+  significant failure. Each zone is active and can receive write operations. A
+  multi-zone configuration that contains multiple active zones enhances
+  disaster recovery and can be used as a foundation for content-delivery
+  networks. 
 
 - **Multi-zonegroups:** Ceph Object Gateway supports multiple zonegroups (which
   were formerly called "regions"). Each zonegroup contains one or more zones.
-  If two zones are in the same zonegroup, and if that zonegroup is in the same
-  realm as a second zonegroup, then the objects stored in the two zones share
-  a global object namespace. This global object namespace ensures unique
-  object IDs across zonegroups and zones.
+  If two zones are in the same zonegroup and that zonegroup is in the same
+  realm as a second zonegroup, then the objects stored in the two zones share a
+  global object namespace. This global object namespace ensures unique object
+  IDs across zonegroups and zones.
 
   Each bucket is owned by the zonegroup where it was created (except where
   overridden by the :ref:`LocationConstraint<s3_bucket_placement>` on
-  bucket creation), and its object data will only replicate to other zones in
-  that zonegroup. Any request for data in that bucket that are sent to other
+  bucket creation), and its object data will replicate only to other zones in
+  that zonegroup. Any request for data in that bucket that is sent to other
   zonegroups will redirect to the zonegroup where the bucket resides.
 
   It can be useful to create multiple zonegroups when you want to share a
-  namespace of users and buckets across many zones, but isolate the object data
-  to a subset of those zones. It might be that you have several connected sites
-  that share storage, but only require a single backup for purposes of disaster
-  recovery. In such a case, it could make sense to create several zonegroups
-  with only two zones each to avoid replicating all objects to all zones.
-
-  In other cases, it might make more sense to isolate things in separate
-  realms, with each realm having a single zonegroup. Zonegroups provide
-  flexibility by making it possible to control the isolation of data and
-  metadata separately.
-
-- **Multiple Realms:** Beginning with the Kraken release, the Ceph Object
-  Gateway supports "realms", which are containers for zonegroups. Realms make
-  it possible to set policies that apply to multiple zonegroups. Realms have a
+  namespace of users and buckets across many zones and isolate the object data
+  to a subset of those zones. Maybe you have several connected sites that share
+  storage but require only a single backup for purposes of disaster recovery.
+  In such a case, you could create several zonegroups with only two zones each
+  to avoid replicating all objects to all zones.
+
+  In other cases, you might isolate data in separate realms, with each realm
+  having a single zonegroup. Zonegroups provide flexibility by making it
+  possible to control the isolation of data and metadata separately.
+
+- **Multiple Realms:** Since the Kraken release, the Ceph Object Gateway
+  supports "realms", which are containers for zonegroups. Realms make it
+  possible to set policies that apply to multiple zonegroups. Realms have a
   globally unique namespace and can contain either a single zonegroup or
   multiple zonegroups. If you choose to make use of multiple realms, you can
   define multiple namespaces and multiple configurations (this means that each

From 5d1255c81b0513fc9e70a4c136b3c3a61a85aee1 Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Mon, 5 Feb 2024 19:00:31 +0530
Subject: [PATCH 1954/2492] client: add function to check if file handle exists

It is named _ll_fh_exists meaning it is a helper func
only to be used with low level APIs.

Fixes: https://tracker.ceph.com/issues/64313
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/client/Client.h b/src/client/Client.h
index 712583aa0d3c..84142f8e44e8 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -1026,6 +1026,9 @@ class Client : public Dispatcher, public md_config_obs_t {
     return it->second;
   }
   int get_fd_inode(int fd, InodeRef *in);
+  bool _ll_fh_exists(Fh *f) {
+    return ll_unclosed_fh_set.count(f);
+  }
 
   // helpers
   void wake_up_session_caps(MetaSession *s, bool reconnect);

From 5099c4ef5841c0f80a9ca1844d254ecce25d269b Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 20 Feb 2024 17:32:45 +0530
Subject: [PATCH 1955/2492] client: check for bad file handle in
 ll_preadv_pwritev

Along with some minor adjustments to the code to make use of
same int for all the ops.

Fixes: https://tracker.ceph.com/issues/64313
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 6ee5e742b9b2..b999726e479a 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -15942,23 +15942,34 @@ int64_t Client::ll_preadv_pwritev(struct Fh *fh, const struct iovec *iov,
                                   Context *onfinish, bufferlist *bl,
                                   bool do_fsync, bool syncdataonly)
 {
+    int64_t retval = -1;
+
     RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
     if (!mref_reader.is_state_satisfied()) {
-      int64_t rc = -CEPHFS_ENOTCONN;
+      retval = -CEPHFS_ENOTCONN;
       if (onfinish != nullptr) {
-        onfinish->complete(rc);
+        onfinish->complete(retval);
         /* async call should always return zero to caller and allow the
         caller to wait on callback for the actual errno. */
-        rc = 0;
+        retval = 0;
+      }
+      return retval;
+    }
+
+    if(fh == NULL || !_ll_fh_exists(fh)) {
+      ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
+      retval = -CEPHFS_EBADF;
+      if (onfinish != nullptr) {
+        onfinish->complete(retval);
+        retval = 0;
       }
-      return rc;
+      return retval;
     }
 
     std::scoped_lock cl(client_lock);
 
-    int64_t retval = _preadv_pwritev_locked(fh, iov, iovcnt, offset, write,
-                                            true, onfinish, bl, do_fsync,
-                                            syncdataonly);
+    retval = _preadv_pwritev_locked(fh, iov, iovcnt, offset, write, true,
+                                    onfinish, bl, do_fsync, syncdataonly);
     /* There are two scenarios with each having two cases to handle here
     1) async io
       1.a) r == 0:

From 6614933808a01322911e1d1c90402f0fd9f099ed Mon Sep 17 00:00:00 2001
From: Dhairya Parmar <dparmar@redhat.com>
Date: Tue, 20 Feb 2024 17:45:06 +0530
Subject: [PATCH 1956/2492] client: check for bad file handle in low level I/O
 APIs

and guard the `if (!mref_reader.is_state_satisfied())` stmt with braces.

Fixes: https://tracker.ceph.com/issues/64313
Signed-off-by: Dhairya Parmar <dparmar@redhat.com>
---
 src/client/Client.cc | 38 +++++++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index b999726e479a..3cdb7850aca1 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -15758,8 +15758,14 @@ loff_t Client::ll_lseek(Fh *fh, loff_t offset, int whence)
 int Client::ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl)
 {
   RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
-  if (!mref_reader.is_state_satisfied())
+  if (!mref_reader.is_state_satisfied()) {
     return -CEPHFS_ENOTCONN;
+  }
+
+  if (fh == NULL || !_ll_fh_exists(fh)) {
+    ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
+    return -CEPHFS_EBADF;
+  }
 
   ldout(cct, 3) << "ll_read " << fh << " " << fh->inode->ino << " " << " " << off << "~" << len << dendl;
   tout(cct) << "ll_read" << std::endl;
@@ -15896,6 +15902,16 @@ int Client::ll_commit_blocks(Inode *in,
 
 int Client::ll_write(Fh *fh, loff_t off, loff_t len, const char *data)
 {
+  RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
+  if (!mref_reader.is_state_satisfied()) {
+    return -CEPHFS_ENOTCONN;
+  }
+
+  if (fh == NULL || !_ll_fh_exists(fh)) {
+    ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
+    return -CEPHFS_EBADF;
+  }
+
   ldout(cct, 3) << "ll_write " << fh << " " << fh->inode->ino << " " << off <<
     "~" << len << dendl;
   tout(cct) << "ll_write" << std::endl;
@@ -15903,10 +15919,6 @@ int Client::ll_write(Fh *fh, loff_t off, loff_t len, const char *data)
   tout(cct) << off << std::endl;
   tout(cct) << len << std::endl;
 
-  RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
-  if (!mref_reader.is_state_satisfied())
-    return -CEPHFS_ENOTCONN;
-
   /* We can't return bytes written larger than INT_MAX, clamp len to that */
   len = std::min(len, (loff_t)INT_MAX);
   std::scoped_lock lock(client_lock);
@@ -15920,8 +15932,14 @@ int Client::ll_write(Fh *fh, loff_t off, loff_t len, const char *data)
 int64_t Client::ll_writev(struct Fh *fh, const struct iovec *iov, int iovcnt, int64_t off)
 {
   RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
-  if (!mref_reader.is_state_satisfied())
+  if (!mref_reader.is_state_satisfied()) {
     return -CEPHFS_ENOTCONN;
+  }
+
+  if (fh == NULL || !_ll_fh_exists(fh)) {
+    ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
+    return -CEPHFS_EBADF;
+  }
 
   std::scoped_lock cl(client_lock);
   return _preadv_pwritev_locked(fh, iov, iovcnt, off, true, false);
@@ -15930,8 +15948,14 @@ int64_t Client::ll_writev(struct Fh *fh, const struct iovec *iov, int iovcnt, in
 int64_t Client::ll_readv(struct Fh *fh, const struct iovec *iov, int iovcnt, int64_t off)
 {
   RWRef_t mref_reader(mount_state, CLIENT_MOUNTING);
-  if (!mref_reader.is_state_satisfied())
+  if (!mref_reader.is_state_satisfied()) {
     return -CEPHFS_ENOTCONN;
+  }
+
+  if (fh == NULL || !_ll_fh_exists(fh)) {
+    ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
+    return -CEPHFS_EBADF;
+  }
 
   std::scoped_lock cl(client_lock);
   return _preadv_pwritev_locked(fh, iov, iovcnt, off, false, false);

From 512154fa8d8fb58aefe5ddcedad865ad37a1634a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 24 Nov 2023 14:45:01 -0500
Subject: [PATCH 1957/2492] cephadm: allow passing pathlib.Path objects to
 file_utils.makedirs

Update the type annotations to allow passing pathlib.Path objects to the
makedirs function. All the calls makedirs uses already can accept Path
objects. This causes mypy to accept calling makedirs with a Path
argument and Paths are nice.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
(cherry picked from commit 21a16007e4e578e04907775aa4e96febf02d862e)
---
 src/cephadm/cephadmlib/file_utils.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadmlib/file_utils.py b/src/cephadm/cephadmlib/file_utils.py
index 9c1ca50b9255..399729f2dccb 100644
--- a/src/cephadm/cephadmlib/file_utils.py
+++ b/src/cephadm/cephadmlib/file_utils.py
@@ -81,14 +81,13 @@ def write_tmp(s, uid, gid):
     return tmp_f
 
 
-def makedirs(dir, uid, gid, mode):
-    # type: (str, int, int, int) -> None
-    if not os.path.exists(dir):
-        os.makedirs(dir, mode=mode)
+def makedirs(dest: Union[Path, str], uid: int, gid: int, mode: int) -> None:
+    if not os.path.exists(dest):
+        os.makedirs(dest, mode=mode)
     else:
-        os.chmod(dir, mode)
-    os.chown(dir, uid, gid)
-    os.chmod(dir, mode)  # the above is masked by umask...
+        os.chmod(dest, mode)
+    os.chown(dest, uid, gid)
+    os.chmod(dest, mode)  # the above is masked by umask...
 
 
 def recursive_chown(path: str, uid: int, gid: int) -> None:

From f73c0665f2be46b6aef535a988e5550a92669cc1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 24 Nov 2023 14:45:15 -0500
Subject: [PATCH 1958/2492] cephadm: add a prepare_data_dir method to container
 daemon form

The prepare_data_dir method is a general way for classes to prepare the
data dir (e.g. `/var/lib/ceph/$FSID/$DAEMON_TYPE.$DAEMON_ID`) before
containers will use it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
(cherry picked from commit 044e4d64790d3c06cbf67a6c783c19edcfcf2d51)
---
 src/cephadm/cephadmlib/container_daemon_form.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index 1b65194aae7a..cab57871b8de 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -118,6 +118,9 @@ def default_entrypoint(self) -> str:
         """
         return ''
 
+    def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None:
+        pass
+
 
 def daemon_to_container(
     ctx: CephadmContext,

From bf7e40b4d1fee439a4a7f48cf033d4a0885890d0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 24 Nov 2023 14:45:34 -0500
Subject: [PATCH 1959/2492] cephadm: call container daemon form
 prepare_data_dir

Instead of always climbing through an "if ladder" based on daemon type
variables we will have the option of using the common method provided
by container daemon form classes. This will initially be used by the
smb daemon. I don't have the energy to refactor all the existing stuff
at the moment.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
(cherry picked from commit 0f6a18a09cba35515e100f457e224537ad1e5608)
---
 src/cephadm/cephadm.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5a7ed9e76656..4cdd52972faf 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -805,6 +805,11 @@ def create_daemon_dirs(
         node_proxy = NodeProxy.init(ctx, fsid, ident.daemon_id)
         node_proxy.create_daemon_dirs(data_dir, uid, gid)
 
+    else:
+        daemon = daemon_form_create(ctx, ident)
+        if isinstance(daemon, ContainerDaemonForm):
+            daemon.prepare_data_dir(data_dir, uid, gid)
+
     _write_custom_conf_files(ctx, ident, uid, gid)
 
 

From db74c10e008caeb7092eb4c1268cbd6f044afa19 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Mon, 19 Feb 2024 18:17:52 +0100
Subject: [PATCH 1960/2492] rgw: fix root element for GetPublicAccessBlock

From the AWS doc (https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetPublicAccessBlock.html) it needs to be PublicAccessBlockConfiguration.

Fixes: https://tracker.ceph.com/issues/64492
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 src/rgw/rgw_public_access.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_public_access.cc b/src/rgw/rgw_public_access.cc
index d388a59a7bb9..6d86ad3516e5 100644
--- a/src/rgw/rgw_public_access.cc
+++ b/src/rgw/rgw_public_access.cc
@@ -9,7 +9,7 @@ void PublicAccessBlockConfiguration::decode_xml(XMLObj *obj) {
 }
 
 void PublicAccessBlockConfiguration::dump_xml(Formatter *f) const {
-  Formatter::ObjectSection os(*f, "BlockPublicAccessBlockConfiguration");
+  Formatter::ObjectSection os(*f, "PublicAccessBlockConfiguration");
   // Note: AWS spec mentions the values to be ALL CAPs, but clients seem to
   // require all small letters, and S3 itself doesn't seem to follow the API
   // spec here

From 54bc5eba002cff04e5c4e25168aeb2408ba100b2 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Mon, 19 Feb 2024 18:20:44 +0100
Subject: [PATCH 1961/2492] rgw: fix the op in GetBucketPublicAccessBlock
 verify_permission

s3GetBucketPublicAccessBlock needs to be checked for RGWGetBucketPublicAccessBlock operation.

Fixes: https://tracker.ceph.com/issues/64492
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 src/rgw/rgw_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index f6011dbe5c9d..445119dd4aee 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -8983,7 +8983,7 @@ int RGWGetBucketPublicAccessBlock::verify_permission(optional_yield y)
   if (has_s3_resource_tag)
     rgw_iam_add_buckettags(this, s);
 
-  if (!verify_bucket_permission(this, s, rgw::IAM::s3GetBucketPolicy)) {
+  if (!verify_bucket_permission(this, s, rgw::IAM::s3GetBucketPublicAccessBlock)) {
     return -EACCES;
   }
 

From 146d4d98698d48dcbc0f036de0793e0a5f9bb8a6 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Mon, 19 Feb 2024 18:23:07 +0100
Subject: [PATCH 1962/2492] rgw: return 404 on no bucket public access block

Return 404 when there is no public access block on the bucket.

Fixes: https://tracker.ceph.com/issues/64492
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 src/rgw/rgw_common.cc | 1 +
 src/rgw/rgw_common.h  | 1 +
 src/rgw/rgw_op.cc     | 5 ++++-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 3ee98fa18ca1..45d3dd066184 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -131,6 +131,7 @@ rgw_http_errors rgw_http_s3_errors({
     { ERR_ZERO_IN_URL, {400, "InvalidRequest" }},
     { ERR_NO_SUCH_TAG_SET, {404, "NoSuchTagSet"}},
     { ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION, {404, "ServerSideEncryptionConfigurationNotFoundError"}},
+    { ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION, {404, "NoSuchPublicAccessBlockConfiguration"}},
 });
 
 rgw_http_errors rgw_http_swift_errors({
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index aecb5721e7a1..c2066a756f8a 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -282,6 +282,7 @@ static inline const char* to_mime_type(const RGWFormat f)
 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION  2046
 #define ERR_INVALID_RETENTION_PERIOD 2047
 #define ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION 2048
+#define ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION 2049
 #define ERR_USER_SUSPENDED       2100
 #define ERR_INTERNAL_ERROR       2200
 #define ERR_NOT_IMPLEMENTED      2201
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 445119dd4aee..ab19a15dbc58 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -8997,7 +8997,10 @@ void RGWGetBucketPublicAccessBlock::execute(optional_yield y)
       aiter == attrs.end()) {
     ldpp_dout(this, 0) << "can't find bucket IAM POLICY attr bucket_name = "
 		       << s->bucket_name << dendl;
-    // return the default;
+
+    op_ret = -ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION;
+    s->err.message = "The public access block configuration was not found";
+
     return;
   } else {
     bufferlist::const_iterator iter{&aiter->second};

From 48a454971c9979dfd9a95cc9619ca864aeda26a5 Mon Sep 17 00:00:00 2001
From: Seena Fallah <seenafallah@gmail.com>
Date: Tue, 20 Feb 2024 15:54:06 +0100
Subject: [PATCH 1963/2492] rgw: return 204 on bucket public access block
 deletion

According to AWS doc (https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeletePublicAccessBlock.html#API_DeletePublicAccessBlock_ResponseSyntax) 204 shall be returned on delete request.

Fixes: https://tracker.ceph.com/issues/64492
Signed-off-by: Seena Fallah <seenafallah@gmail.com>
---
 src/rgw/rgw_op.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index ab19a15dbc58..9e32c4982809 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -9017,9 +9017,12 @@ void RGWGetBucketPublicAccessBlock::execute(optional_yield y)
 
 void RGWDeleteBucketPublicAccessBlock::send_response()
 {
-  if (op_ret) {
-    set_req_state_err(s, op_ret);
+  if (!op_ret) {
+    /* A successful Delete request should return a 204 */
+    op_ret = STATUS_NO_CONTENT;
   }
+
+  set_req_state_err(s, op_ret);
   dump_errno(s);
   end_header(s);
 }

From 99029020f3f704499df528c15d9b7a71f7e66092 Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Tue, 20 Feb 2024 14:37:17 +0100
Subject: [PATCH 1964/2492] mgr/rook: raise an exception when user calls OSD
 related methods

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index 91099710c4b3..afd789a8ba23 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -615,6 +615,19 @@ def add_host_label(self, host: str, label: str) -> OrchResult[str]:
     def remove_host_label(self, host: str, label: str, force: bool = False) -> OrchResult[str]:
         return self.rook_cluster.remove_host_label(host, label)
 
+    @handle_orch_error
+    def create_osds(self, drive_group: DriveGroupSpec) -> str:
+        raise orchestrator.OrchestratorError('Creating OSDs is not supported by rook orchestrator. Please, use Rook operator.')
+
+    @handle_orch_error
+    def remove_osds(self,
+                    osd_ids: List[str],
+                    replace: bool = False,
+                    force: bool = False,
+                    zap: bool = False,
+                    no_destroy: bool = False) -> str:
+        raise orchestrator.OrchestratorError('Removing OSDs is not supported by rook orchestrator. Please, use Rook operator.')
+
     @handle_orch_error
     def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
         return self.rook_cluster.blink_light(ident_fault, on, locs)

From e5099d67a2478e199bba6a435300964756005525 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 19 Feb 2024 01:36:32 -0500
Subject: [PATCH 1965/2492] qa: disable snapshot_clone_no_wait for some tests

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/tasks/cephfs/test_volumes.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py
index 650ebcf26c8f..037b046304eb 100644
--- a/qa/tasks/cephfs/test_volumes.py
+++ b/qa/tasks/cephfs/test_volumes.py
@@ -5760,6 +5760,9 @@ def test_subvolume_snapshot_info_if_no_clone_pending(self):
         snapshot = self._gen_subvol_snap_name()
         clone_list =  [f'clone_{i}' for i in range(3)]
 
+        # disable "capped" clones
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', False)
+
         # create subvolume.
         self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777")
 
@@ -5810,6 +5813,9 @@ def test_subvolume_snapshot_info_if_clone_pending_for_no_group(self):
         # insert delay at the beginning of snapshot clone
         self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 5)
 
+        # disable "capped" clones
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', False)
+
         # schedule a clones
         for clone in clone_list:
             self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone)
@@ -5913,6 +5919,9 @@ def test_subvolume_snapshot_info_if_orphan_clone(self):
         # insert delay at the beginning of snapshot clone
         self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 15)
 
+        # disable "capped" clones
+        self.config_set('mgr', 'mgr/volumes/snapshot_clone_no_wait', False)
+
         # schedule a clones
         for clone in clone_list:
             self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone)

From 2e2519367099461415248353d7c7725c0567511d Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 21 Feb 2024 04:07:07 +1000
Subject: [PATCH 1966/2492] doc/rados: fix broken links

Fix broken links in doc/rados/operations/add-or-rm-mons.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/add-or-rm-mons.rst | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/doc/rados/operations/add-or-rm-mons.rst b/doc/rados/operations/add-or-rm-mons.rst
index 3688bb7986ee..4d197ea93bd2 100644
--- a/doc/rados/operations/add-or-rm-mons.rst
+++ b/doc/rados/operations/add-or-rm-mons.rst
@@ -344,12 +344,13 @@ addresses, repeat this process.
 Changing a Monitor's IP address (Advanced Method)
 -------------------------------------------------
 
-There are cases in which the method outlined in :ref"`<Changing a Monitor's IP
-Address (Preferred Method)> operations_add_or_rm_mons_changing_mon_ip` cannot
-be used. For example, it might be necessary to move the cluster's monitors to a
-different network, to a different part of the datacenter, or to a different
-datacenter altogether. It is still possible to change the monitors' IP
-addresses, but a different method must be used.
+There are cases in which the method outlined in
+:ref:`operations_add_or_rm_mons_changing_mon_ip` cannot be used. For example,
+it might be necessary to move the cluster's monitors to a different network, to
+a different part of the datacenter, or to a different datacenter altogether. It
+is still possible to change the monitors' IP addresses, but a different method
+must be used.
+
 
 For such cases, a new monitor map with updated IP addresses for every monitor
 in the cluster must be generated and injected on each monitor. Although this
@@ -357,11 +358,11 @@ method is not particularly easy, such a major migration is unlikely to be a
 routine task. As stated at the beginning of this section, existing monitors are
 not supposed to change their IP addresses.
 
-Continue with the monitor configuration in the example from :ref"`<Changing a
-Monitor's IP Address (Preferred Method)>
-operations_add_or_rm_mons_changing_mon_ip` . Suppose that all of the monitors
-are to be moved from the ``10.0.0.x`` range to the ``10.1.0.x`` range, and that
-these networks are unable to communicate. Carry out the following procedure:
+Continue with the monitor configuration in the example from
+:ref:`operations_add_or_rm_mons_changing_mon_ip`. Suppose that all of the
+monitors are to be moved from the ``10.0.0.x`` range to the ``10.1.0.x`` range,
+and that these networks are unable to communicate. Carry out the following
+procedure:
 
 #. Retrieve the monitor map (``{tmp}`` is the path to the retrieved monitor
    map, and ``{filename}`` is the name of the file that contains the retrieved

From d25b6b9dd2cee81d33f2ad2f0718c6ccdeadbe2b Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Tue, 20 Feb 2024 22:54:01 +0100
Subject: [PATCH 1967/2492] doc/rados/configuration/osd-config-ref: fix typo

Signed-off-by: Pierre Riteau <pierre@stackhpc.com>
---
 doc/rados/configuration/osd-config-ref.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
index 634690b88c06..5064a95851ca 100644
--- a/doc/rados/configuration/osd-config-ref.rst
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -151,7 +151,7 @@ generates a catalog of all objects in each placement group and compares each
 primary object to its replicas, ensuring that no objects are missing or
 mismatched. Light scrubbing checks the object size and attributes, and is
 usually done daily. Deep scrubbing reads the data and uses checksums to ensure
-data integrity, and is usually done weekly. The freqeuncies of both light
+data integrity, and is usually done weekly. The frequencies of both light
 scrubbing and deep scrubbing are determined by the cluster's configuration,
 which is fully under your control and subject to the settings explained below
 in this section.

From 6cc1c2d4e0c8fd4a29ee63b2eee954f909eec2bc Mon Sep 17 00:00:00 2001
From: Shilpa Jagannath <smanjara@redhat.com>
Date: Sat, 17 Feb 2024 00:09:07 -0500
Subject: [PATCH 1968/2492] rgw: set log_data_change to true during link olh
 only in multisite config

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 7925cddf86fc..8837dd56b920 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -3263,7 +3263,8 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   state = NULL;
 
   if (versioned_op && meta.olh_epoch) {
-    r = store->set_olh(rctx.dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, rctx.y, meta.zones_trace, log_op);
+    bool add_log = log_op && store->svc.zone->need_to_log_data();
+    r = store->set_olh(rctx.dpp, target->get_ctx(), target->get_bucket_info(), obj, false, NULL, *meta.olh_epoch, real_time(), false, rctx.y, meta.zones_trace, add_log);
     if (r < 0) {
       return r;
     }

From f719449280d93e96cd03923def9719e0d479e4c1 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 20 Feb 2024 23:56:26 +0000
Subject: [PATCH 1969/2492] cmake/.../FindSanitizers: add check for
 Sanitizers_FIBER_SUPPPORT

With newer clang and gcc versions (observed on clang-17.0.6 as
well as gcc 12/13), asan is throwing stack-use-after-return
during OSD startup related to usage of seastar::async, which
relies on swapcontext internally.

seastar/src/core/thread.cc supports asan's hooks, but only if
SEASTAR_HAVE_ASAN_FIBER_SUPPORT is set.  seastar's CMakeList.txt
sets it based on Sanitizers_FIBER_SUPPORT, which probably should
be set by the module at src/seastar/cmake/FindSanitizers.cmake,
but that module doesn't seem to be actually invoked anywhere.

Ceph's version of that module (cmake/modules/FindSanitizers.cmake)
does not set Sanitizers_FIBER_SUPPORT.

This commit adds that check as well as the related code snippet.

Fixes: https://tracker.ceph.com/issues/64512
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 cmake/modules/FindSanitizers.cmake                |  3 +++
 cmake/modules/code_tests/Sanitizers_fiber_test.cc | 11 +++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 cmake/modules/code_tests/Sanitizers_fiber_test.cc

diff --git a/cmake/modules/FindSanitizers.cmake b/cmake/modules/FindSanitizers.cmake
index adafc5ebe3f7..bfb99821a9bd 100644
--- a/cmake/modules/FindSanitizers.cmake
+++ b/cmake/modules/FindSanitizers.cmake
@@ -57,6 +57,9 @@ string (REPLACE ";" " " CMAKE_REQUIRED_FLAGS "${Sanitizers_COMPILE_OPTIONS}")
 set(CMAKE_REQUIRED_LIBRARIES ${Sanitizers_COMPILE_OPTIONS})
 check_cxx_source_compiles("int main() {}"
   Sanitizers_ARE_SUPPORTED)
+
+file (READ ${CMAKE_CURRENT_LIST_DIR}/code_tests/Sanitizers_fiber_test.cc _sanitizers_fiber_test_code)
+check_cxx_source_compiles ("${_sanitizers_fiber_test_code}" Sanitizers_FIBER_SUPPORT)
 cmake_pop_check_state()
 
 include(FindPackageHandleStandardArgs)
diff --git a/cmake/modules/code_tests/Sanitizers_fiber_test.cc b/cmake/modules/code_tests/Sanitizers_fiber_test.cc
new file mode 100644
index 000000000000..9df531f2675f
--- /dev/null
+++ b/cmake/modules/code_tests/Sanitizers_fiber_test.cc
@@ -0,0 +1,11 @@
+#include <cstddef>
+
+extern "C" {
+    void __sanitizer_start_switch_fiber(void**, const void*, size_t);
+    void __sanitizer_finish_switch_fiber(void*, const void**, size_t*);
+}
+
+int main() {
+    __sanitizer_start_switch_fiber(nullptr, nullptr, 0);
+    __sanitizer_finish_switch_fiber(nullptr, nullptr, nullptr);
+}

From 4d430f673a64bbfab0682911de8b22a44fc052bb Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 20 Feb 2024 16:41:26 -0800
Subject: [PATCH 1970/2492] crimson/.../shard_services.cc: trivial formatting
 fix

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/shard_services.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 604c045eb9b8..e20e310dc2cc 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -809,7 +809,7 @@ seastar::future<MURef<MOSDMap>> OSDSingletonState::build_incremental_map_msg(
                             monc.get_fsid(),
                             osdmap->get_encoding_features()),
                           [this, &first, FNAME, last](unsigned int map_message_max,
-                                                      auto& m) {
+                                                      auto &m) {
     m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
     m->newest_map = superblock.get_newest_map();
     auto maybe_handle_mapgap = seastar::now();

From e03495dca75be80f642453782ab7793c3c72712e Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 20 Feb 2024 16:43:16 -0800
Subject: [PATCH 1971/2492] crimson/.../shard_services.cc: pass by reference
 from do_with

Capturing the value passed by do_with by value causes later captures
by reference to be invalid past the lifetime of the lambda frame.

Fixes: https://tracker.ceph.com/issues/64513
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/shard_services.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index e20e310dc2cc..8140ca3f91d5 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -808,7 +808,7 @@ seastar::future<MURef<MOSDMap>> OSDSingletonState::build_incremental_map_msg(
                           crimson::make_message<MOSDMap>(
                             monc.get_fsid(),
                             osdmap->get_encoding_features()),
-                          [this, &first, FNAME, last](unsigned int map_message_max,
+                          [this, &first, FNAME, last](auto &map_message_max,
                                                       auto &m) {
     m->cluster_osdmap_trim_lower_bound = superblock.cluster_osdmap_trim_lower_bound;
     m->newest_map = superblock.get_newest_map();

From a1f9b59c1ad4106126f816a3ef4fdfae5df555bb Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Wed, 7 Feb 2024 11:21:50 +0800
Subject: [PATCH 1972/2492] cmake/AddCephTest: Specify resoureces to crimson
 unittest

When running crimson unittest, the seastar framework always
use and only use cpu0, and with many parallel crimson unittest
jobs, all the jobs are running on cpu0, the other cpu cores
can't used, make the make check run very slow, even timeout
happens. Use set_property RESOURCE_GROUPS to specify cpu resources
to crimson unittest, and accelerate make check running.

Fixes: https://tracker.ceph.com/issues/64117

Co-authored-by: Kefu Chai <tchaikov@gmail.com>
Signed-off-by: luo rixin <luorixin@huawei.com>
---
 cmake/modules/AddCephTest.cmake | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index 591552f834ea..ccd3f8dee0b5 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -21,6 +21,25 @@ function(add_ceph_test test_name test_path)
     CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV})
   set_property(TEST ${test_name}
     PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
+  # Crimson seastar unittest always run with --smp N to start N threads. By default, crimson seastar unittest
+  # will take cpu cores[0, N), starting one thread per core. When running many crimson seastar unittests
+  # parallely, the front N cpu cores are shared, and the left cpu cores are idle. Lots of cpu cores are wasted.
+  # Using CTest resource allocation feature(https://cmake.org/cmake/help/latest/manual/ctest.1.html#resource-allocation),
+  # ctest can specify cpu cores resources to crimson seastar unittests.
+  # 3 steps to enable CTest resource allocation feature:
+  #  Step 1: Generate a resource specification file to describe available resource, $(nproc) CPUs with id 0 to $(nproc) - 1
+  #  Step 2: Set RESOURCE_GROUPS property to a test with value "${smp_count},cpus:1"
+  #  Step 3: Read a series of environment variables CTEST_RESOURCE_GROUP_* and set seastar smp_opts while running a test
+  list(FIND ARGV "--smp" smp_pos)
+  if(smp_pos GREATER -1)
+    if(smp_pos EQUAL ARGC)
+      message(FATAL_ERROR "${test_name} --smp requires an argument")
+    endif()
+    math(EXPR i "${smp_pos} + 1")
+    list(GET ARGV ${i} smp_count)
+    set_property(TEST ${test_name}
+      PROPERTY RESOURCE_GROUPS "${smp_count},cpus:1")
+  endif()
 endfunction()
 
 option(WITH_GTEST_PARALLEL "Enable running gtest based tests in parallel" OFF)

From 5aa832c5c60e0469127647570bb102ff64a3fe32 Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Wed, 7 Feb 2024 11:33:44 +0800
Subject: [PATCH 1973/2492] run-make-check: Enable ctest resource allocation

Co-authored-by: Kefu Chai <tchaikov@gmail.com>
Signed-off-by: luo rixin <luorixin@huawei.com>
---
 run-make-check.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/run-make-check.sh b/run-make-check.sh
index 0ebe2b5de6bd..c4190a6426c5 100755
--- a/run-make-check.sh
+++ b/run-make-check.sh
@@ -22,6 +22,14 @@ source src/script/run-make.sh
 
 set -e
 
+function gen_ctest_resource_file() {
+    local file_name=$(mktemp /tmp/ctest-resource-XXXXXX)
+    local max_cpuid=$(($(nproc) - 1))
+    jq -n '$ARGS.positional | map({id:., slots:1}) | {cpus:.} | {version: {major:1, minor:0}, local:[.]}' \
+        --args $(seq 0 $max_cpuid) > $file_name
+    echo "$file_name"
+}
+
 function run() {
     # to prevent OSD EMFILE death on tests, make sure ulimit >= 1024
     $DRY_RUN ulimit -n $(ulimit -Hn)
@@ -43,14 +51,16 @@ function run() {
     fi
 
     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
+    CTEST_RESOURCE_FILE=$(gen_ctest_resource_file)
+    CHECK_MAKEOPTS+=" --resource-spec-file ${CTEST_RESOURCE_FILE}"
     if in_jenkins; then
         if ! ctest $CHECK_MAKEOPTS --no-compress-output --output-on-failure --test-output-size-failed 1024000 -T Test; then
             # do not return failure, as the jenkins publisher will take care of this
-            rm -fr ${TMPDIR:-/tmp}/ceph-asok.*
+            rm -fr ${TMPDIR:-/tmp}/ceph-asok.* ${CTEST_RESOURCE_FILE}
         fi
     else
         if ! $DRY_RUN ctest $CHECK_MAKEOPTS --output-on-failure; then
-            rm -fr ${TMPDIR:-/tmp}/ceph-asok.*
+            rm -fr ${TMPDIR:-/tmp}/ceph-asok.* ${CTEST_RESOURCE_FILE}
             return 1
         fi
     fi

From 7fe2323d24b5f08d8f478c2d3da677867fb1a8fa Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Wed, 7 Feb 2024 11:48:30 +0800
Subject: [PATCH 1974/2492] test/crimson: construct app_template with smp_opts
 imported from ctest

Co-authored-by: Kefu Chai <tchaikov@gmail.com>
Signed-off-by: luo rixin <luorixin@huawei.com>
---
 src/test/crimson/ctest_utils.h                | 78 +++++++++++++++++++
 src/test/crimson/seastar_runner.h             |  4 +-
 src/test/crimson/test_alien_echo.cc           |  3 +-
 .../crimson/test_alienstore_thread_pool.cc    |  3 +-
 src/test/crimson/test_buffer.cc               |  3 +-
 src/test/crimson/test_config.cc               |  3 +-
 src/test/crimson/test_messenger.cc            |  3 +-
 src/test/crimson/test_messenger_thrash.cc     |  3 +-
 src/test/crimson/test_monc.cc                 |  3 +-
 src/test/crimson/test_perfcounters.cc         |  3 +-
 src/test/crimson/test_socket.cc               |  3 +-
 11 files changed, 99 insertions(+), 10 deletions(-)
 create mode 100644 src/test/crimson/ctest_utils.h

diff --git a/src/test/crimson/ctest_utils.h b/src/test/crimson/ctest_utils.h
new file mode 100644
index 000000000000..9bbd081dc186
--- /dev/null
+++ b/src/test/crimson/ctest_utils.h
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <cstdlib>
+#include <iostream>
+#include <optional>
+#include <regex>
+#include <string>
+#include <vector>
+
+#include <boost/algorithm/string.hpp>
+#include <fmt/format.h>
+#include <seastar/core/resource.hh>
+#include <seastar/core/app-template.hh>
+
+struct ctest_resource {
+  int id;
+  int slots;
+  ctest_resource(int id, int slots) : id(id), slots(slots) {}
+};
+
+static std::vector<ctest_resource> parse_ctest_resources(const std::string& resource_spec) {
+  std::vector<std::string> resources;
+  boost::split(resources, resource_spec, boost::is_any_of(";"));
+  std::regex res_regex("id:([0-9]+),slots:([0-9]+)");
+  std::vector<ctest_resource> ctest_resources;
+  for (auto& resource : resources) {
+    std::smatch matched;
+    if (std::regex_match(resource, matched, res_regex)) {
+      int id = std::stoi(matched[1].str());
+      int slots = std::stoi(matched[2].str());
+      ctest_resources.emplace_back(id, slots);
+    }
+  }
+  return ctest_resources;
+}
+
+static std::optional<seastar::resource::cpuset> get_cpuset_from_ctest_resource_group() {
+  int nr_groups = 0;
+  auto group_count = std::getenv("CTEST_RESOURCE_GROUP_COUNT");
+  if (group_count != nullptr) {
+    nr_groups = std::stoi(group_count);
+  } else {
+    return {};
+  }
+
+  seastar::resource::cpuset cpuset;
+  for (int num = 0; num < nr_groups; num++) {
+    std::string resource_type_name;
+    fmt::format_to(std::back_inserter(resource_type_name), "CTEST_RESOURCE_GROUP_{}", num);
+    // only a single resource type is supported for now
+    std::string resource_type = std::getenv(resource_type_name.data());
+    if (resource_type == "cpus") {
+      std::transform(resource_type.begin(), resource_type.end(), resource_type.begin(), ::toupper);
+      std::string resource_group;
+      fmt::format_to(std::back_inserter(resource_group), "CTEST_RESOURCE_GROUP_{}_{}", num, resource_type);
+      std::string resource_spec = std::getenv(resource_group.data());
+      for (auto& resource : parse_ctest_resources(resource_spec)) {
+        // each id has a single cpu slot
+        cpuset.insert(resource.id);
+      }
+    } else {
+      fmt::print(std::cerr, "unsupported resource type: {}", resource_type);
+    }
+  }
+  return cpuset;
+}
+
+static seastar::app_template::seastar_options get_smp_opts_from_ctest() {
+  seastar::app_template::seastar_options opts;
+  auto cpuset = get_cpuset_from_ctest_resource_group();
+  if (cpuset) {
+    opts.smp_opts.cpuset.set_value(*cpuset);
+  }
+  return opts;
+}
diff --git a/src/test/crimson/seastar_runner.h b/src/test/crimson/seastar_runner.h
index 58d3f8119e66..63cc50d9f059 100644
--- a/src/test/crimson/seastar_runner.h
+++ b/src/test/crimson/seastar_runner.h
@@ -13,6 +13,8 @@
 #include <seastar/core/alien.hh>
 #include <seastar/core/thread.hh>
 
+#include "test/crimson/ctest_utils.h"
+
 struct SeastarRunner {
   static constexpr eventfd_t APP_RUNNING = 1;
   static constexpr eventfd_t APP_NOT_RUN = 2;
@@ -26,7 +28,7 @@ struct SeastarRunner {
   bool begin_signaled = false;
 
   SeastarRunner() :
-    begin_fd{seastar::file_desc::eventfd(0, 0)} {}
+    app{get_smp_opts_from_ctest()}, begin_fd{seastar::file_desc::eventfd(0, 0)} {}
 
   ~SeastarRunner() {}
 
diff --git a/src/test/crimson/test_alien_echo.cc b/src/test/crimson/test_alien_echo.cc
index 8bef5e6517b0..f6b23305ce5c 100644
--- a/src/test/crimson/test_alien_echo.cc
+++ b/src/test/crimson/test_alien_echo.cc
@@ -8,6 +8,7 @@
 #include "crimson/net/Connection.h"
 #include "crimson/net/Dispatcher.h"
 #include "crimson/net/Messenger.h"
+#include "test/crimson/ctest_utils.h"
 
 #include <seastar/core/alien.hh>
 #include <seastar/core/app-template.hh>
@@ -266,7 +267,7 @@ int main(int argc, char** argv)
   }
 
   auto count = vm["count"].as<unsigned>();
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   SeastarContext sc;
   auto job = sc.with_seastar([&] {
     auto fut = seastar::alien::submit_to(app.alien(), 0, [addr, role, count] {
diff --git a/src/test/crimson/test_alienstore_thread_pool.cc b/src/test/crimson/test_alienstore_thread_pool.cc
index dbeed26cd7db..5f8e2d1b707c 100644
--- a/src/test/crimson/test_alienstore_thread_pool.cc
+++ b/src/test/crimson/test_alienstore_thread_pool.cc
@@ -6,6 +6,7 @@
 #include "crimson/common/config_proxy.h"
 #include "crimson/os/alienstore/thread_pool.h"
 #include "include/msgr.h"
+#include "test/crimson/ctest_utils.h"
 
 using namespace std::chrono_literals;
 using ThreadPool = crimson::os::ThreadPool;
@@ -37,7 +38,7 @@ seastar::future<> test_void_return(ThreadPool& tp) {
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [] {
     std::vector<const char*> args;
     std::string cluster;
diff --git a/src/test/crimson/test_buffer.cc b/src/test/crimson/test_buffer.cc
index 64a815bd2764..61b2785dd607 100644
--- a/src/test/crimson/test_buffer.cc
+++ b/src/test/crimson/test_buffer.cc
@@ -3,6 +3,7 @@
 #include <seastar/core/future-util.hh>
 #include <seastar/core/reactor.hh>
 #include "include/buffer.h"
+#include "test/crimson/ctest_utils.h"
 
 // allocate a foreign buffer on each cpu, collect them all into a bufferlist,
 // and destruct it on this cpu
@@ -36,7 +37,7 @@ seastar::future<> test_foreign_bufferlist()
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [] {
     return seastar::now().then(
       &test_foreign_bufferlist
diff --git a/src/test/crimson/test_config.cc b/src/test/crimson/test_config.cc
index 7541c0931254..7b1dddb715ff 100644
--- a/src/test/crimson/test_config.cc
+++ b/src/test/crimson/test_config.cc
@@ -6,6 +6,7 @@
 #include "common/ceph_argparse.h"
 #include "common/config_obs.h"
 #include "crimson/common/config_proxy.h"
+#include "test/crimson/ctest_utils.h"
 
 using namespace std::literals;
 using Config = crimson::common::ConfigProxy;
@@ -88,7 +89,7 @@ static seastar::future<> test_config()
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [&] {
     return test_config().then([] {
       std::cout << "All tests succeeded" << std::endl;
diff --git a/src/test/crimson/test_messenger.cc b/src/test/crimson/test_messenger.cc
index a4257224658d..75bbcd7a6476 100644
--- a/src/test/crimson/test_messenger.cc
+++ b/src/test/crimson/test_messenger.cc
@@ -29,6 +29,7 @@
 #include <seastar/core/with_timeout.hh>
 
 #include "test_messenger.h"
+#include "test/crimson/ctest_utils.h"
 
 using namespace std::chrono_literals;
 namespace bpo = boost::program_options;
@@ -3845,7 +3846,7 @@ seastar::future<int> do_test(seastar::app_template& app)
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   app.add_options()
     ("verbose,v", bpo::value<bool>()->default_value(false),
      "chatty if true")
diff --git a/src/test/crimson/test_messenger_thrash.cc b/src/test/crimson/test_messenger_thrash.cc
index f2b1828f1c5a..28efb60fd5cc 100644
--- a/src/test/crimson/test_messenger_thrash.cc
+++ b/src/test/crimson/test_messenger_thrash.cc
@@ -20,6 +20,7 @@
 #include "crimson/net/Connection.h"
 #include "crimson/net/Dispatcher.h"
 #include "crimson/net/Messenger.h"
+#include "test/crimson/ctest_utils.h"
 
 using namespace std::chrono_literals;
 namespace bpo = boost::program_options;
@@ -662,7 +663,7 @@ seastar::future<int> do_test(seastar::app_template& app)
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   app.add_options()
     ("verbose,v", bpo::value<bool>()->default_value(false),
      "chatty if true");
diff --git a/src/test/crimson/test_monc.cc b/src/test/crimson/test_monc.cc
index e60df4525768..c30098fe87f9 100644
--- a/src/test/crimson/test_monc.cc
+++ b/src/test/crimson/test_monc.cc
@@ -5,6 +5,7 @@
 #include "crimson/mon/MonClient.h"
 #include "crimson/net/Connection.h"
 #include "crimson/net/Messenger.h"
+#include "test/crimson/ctest_utils.h"
 
 using Config = crimson::common::ConfigProxy;
 using MonClient = crimson::mon::Client;
@@ -63,7 +64,7 @@ static seastar::future<> test_monc()
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [&] {
     return test_monc().then([] {
       std::cout << "All tests succeeded" << std::endl;
diff --git a/src/test/crimson/test_perfcounters.cc b/src/test/crimson/test_perfcounters.cc
index 8aecbf911cc9..2b7020a39ae6 100644
--- a/src/test/crimson/test_perfcounters.cc
+++ b/src/test/crimson/test_perfcounters.cc
@@ -6,6 +6,7 @@
 #include "common/Formatter.h"
 #include "common/perf_counters.h"
 #include "crimson/common/perf_counters_collection.h"
+#include "test/crimson/ctest_utils.h"
 
 #include <seastar/core/app-template.hh>
 #include <seastar/core/sharded.hh>
@@ -47,7 +48,7 @@ static seastar::future<> test_perfcounters(){
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [&] {
     return test_perfcounters().then([] {
       std::cout << "All tests succeeded" << std::endl;
diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc
index 2b61196ead8d..ca0677e1e76d 100644
--- a/src/test/crimson/test_socket.cc
+++ b/src/test/crimson/test_socket.cc
@@ -14,6 +14,7 @@
 #include "crimson/net/Errors.h"
 #include "crimson/net/Fwd.h"
 #include "crimson/net/Socket.h"
+#include "test/crimson/ctest_utils.h"
 
 using crimson::common::local_conf;
 
@@ -551,7 +552,7 @@ seastar::future<int> do_test(seastar::app_template& app)
 
 int main(int argc, char** argv)
 {
-  seastar::app_template app;
+  seastar::app_template app{get_smp_opts_from_ctest()};
   return app.run(argc, argv, [&app] {
     return do_test(app);
   });

From 6f0803a86af78965eb954d98e6095d96f0c32f1f Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 16 Jan 2024 16:58:17 +0800
Subject: [PATCH 1975/2492] crimson/os: add FuturizedStore::Shard::exists()
 interface

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/alienstore/alien_store.cc | 13 +++++++++++++
 src/crimson/os/alienstore/alien_store.h  |  3 +++
 src/crimson/os/cyanstore/cyan_store.cc   | 16 ++++++++++++++++
 src/crimson/os/cyanstore/cyan_store.h    |  6 +++++-
 src/crimson/os/futurized_store.h         |  5 +++++
 src/crimson/os/seastore/seastore.cc      | 23 +++++++++++++++++++++++
 src/crimson/os/seastore/seastore.h       |  4 ++++
 7 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 32696775bb09..bfa8dbf42e79 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -136,6 +136,19 @@ seastar::future<> AlienStore::stop()
   });
 }
 
+AlienStore::base_errorator::future<bool>
+AlienStore::exists(
+  CollectionRef ch,
+  const ghobject_t& oid)
+{
+  return seastar::with_gate(op_gate, [=, this] {
+    return tp->submit(ch->get_cid().hash_to_shard(tp->size()), [=, this] {
+      auto c = static_cast<AlienCollection*>(ch.get());
+      return store->exists(c->collection, oid);
+    });
+  });
+}
+
 AlienStore::mount_ertr::future<> AlienStore::mount()
 {
   logger().debug("{}", __func__);
diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h
index 7138da955aa1..910bb153dcb4 100644
--- a/src/crimson/os/alienstore/alien_store.h
+++ b/src/crimson/os/alienstore/alien_store.h
@@ -33,6 +33,9 @@ class AlienStore final : public FuturizedStore,
   mount_ertr::future<> mount() final;
   seastar::future<> umount() final;
 
+  base_errorator::future<bool> exists(
+    CollectionRef c,
+    const ghobject_t& oid) final;
   mkfs_ertr::future<> mkfs(uuid_d new_osd_fsid) final;
   read_errorator::future<ceph::bufferlist> read(CollectionRef c,
                                    const ghobject_t& oid,
diff --git a/src/crimson/os/cyanstore/cyan_store.cc b/src/crimson/os/cyanstore/cyan_store.cc
index 36230893d36b..df68e8bd20f9 100644
--- a/src/crimson/os/cyanstore/cyan_store.cc
+++ b/src/crimson/os/cyanstore/cyan_store.cc
@@ -242,6 +242,22 @@ CyanStore::Shard::list_collections()
   return seastar::make_ready_future<std::vector<coll_core_t>>(std::move(collections));
 }
 
+CyanStore::Shard::base_errorator::future<bool>
+CyanStore::Shard::exists(
+  CollectionRef ch,
+  const ghobject_t &oid)
+{
+  auto c = static_cast<Collection*>(ch.get());
+  if (!c->exists) {
+    return base_errorator::make_ready_future<bool>(false);
+  }
+  auto o = c->get_object(oid);
+  if (!o) {
+    return base_errorator::make_ready_future<bool>(false);
+  }
+  return base_errorator::make_ready_future<bool>(true);
+}
+
 CyanStore::Shard::read_errorator::future<ceph::bufferlist>
 CyanStore::Shard::read(
   CollectionRef ch,
diff --git a/src/crimson/os/cyanstore/cyan_store.h b/src/crimson/os/cyanstore/cyan_store.h
index 518222d82f65..04df5c707a56 100644
--- a/src/crimson/os/cyanstore/cyan_store.h
+++ b/src/crimson/os/cyanstore/cyan_store.h
@@ -26,6 +26,7 @@ class Transaction;
 
 namespace crimson::os {
 class CyanStore final : public FuturizedStore {
+public:
   class Shard : public FuturizedStore::Shard {
   public:
     Shard(std::string path)
@@ -35,6 +36,10 @@ class CyanStore final : public FuturizedStore {
       CollectionRef c,
       const ghobject_t& oid) final;
 
+    base_errorator::future<bool> exists(
+      CollectionRef ch,
+      const ghobject_t& oid) final;
+
     read_errorator::future<ceph::bufferlist> read(
       CollectionRef c,
       const ghobject_t& oid,
@@ -158,7 +163,6 @@ class CyanStore final : public FuturizedStore {
     std::map<coll_t, boost::intrusive_ptr<Collection>> new_coll_map;
   };
 
-public:
   CyanStore(const std::string& path);
   ~CyanStore() final;
 
diff --git a/src/crimson/os/futurized_store.h b/src/crimson/os/futurized_store.h
index 783cd74859e3..8398a5289b2d 100644
--- a/src/crimson/os/futurized_store.h
+++ b/src/crimson/os/futurized_store.h
@@ -36,6 +36,7 @@ class FuturizedStore {
     const Shard& operator=(const Shard& o) = delete;
 
     using CollectionRef = boost::intrusive_ptr<FuturizedCollection>;
+    using base_errorator = crimson::errorator<crimson::ct_error::input_output_error>;
     using read_errorator = crimson::errorator<crimson::ct_error::enoent,
 					      crimson::ct_error::input_output_error>;
     virtual read_errorator::future<ceph::bufferlist> read(
@@ -51,6 +52,10 @@ class FuturizedStore {
       interval_set<uint64_t>& m,
       uint32_t op_flags = 0) = 0;
 
+    virtual base_errorator::future<bool> exists(
+      CollectionRef c,
+      const ghobject_t& oid) = 0;
+
     using get_attr_errorator = crimson::errorator<
       crimson::ct_error::enoent,
       crimson::ct_error::enodata>;
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index eba19eabebeb..86eace1dd9b4 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -852,6 +852,29 @@ SeaStore::Shard::read(
     });
 }
 
+SeaStore::Shard::base_errorator::future<bool>
+SeaStore::Shard::exists(
+  CollectionRef c,
+  const ghobject_t& oid)
+{
+  LOG_PREFIX(SeaStore::exists);
+  DEBUG("oid {}", oid);
+  return repeat_with_onode<bool>(
+    c,
+    oid,
+    Transaction::src_t::READ,
+    "oid_exists",
+    op_type_t::READ,
+    [](auto&, auto&) {
+    return seastar::make_ready_future<bool>(true);
+  }).handle_error(
+    crimson::ct_error::enoent::handle([] {
+      return seastar::make_ready_future<bool>(false);
+    }),
+    crimson::ct_error::assert_all{"unexpected error"}
+  );
+}
+
 SeaStore::Shard::read_errorator::future<ceph::bufferlist>
 SeaStore::Shard::readv(
   CollectionRef ch,
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index bff0092f4533..8baa4cd658b0 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -116,6 +116,10 @@ class SeaStore final : public FuturizedStore {
       interval_set<uint64_t>& m,
       uint32_t op_flags = 0) final;
 
+    base_errorator::future<bool> exists(
+      CollectionRef c,
+      const ghobject_t& oid) final;
+
     get_attr_errorator::future<ceph::bufferlist> get_attr(
       CollectionRef c,
       const ghobject_t& oid,

From 3928995ca962cb187fb2729b5a1ae8cf34600c3f Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Wed, 21 Feb 2024 08:27:53 +0100
Subject: [PATCH 1976/2492] mgr/rook: adding empty calls to upgrade_ls and
 upgrade_status

added empty calls to upgrade_ls and upgrade_status to avoid
dashboard errors when entering the view Cluster > Upgrade. Empty
calls are used because we don't support the upgrade functionality
in rook as we do for normal Ceph deployments. In case of rook user
has to follow a different process to upgrade Ceph.

Fixes: https://tracker.ceph.com/issues/64516

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 src/pybind/mgr/rook/module.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
index ca2d168db636..16ad3b458284 100644
--- a/src/pybind/mgr/rook/module.py
+++ b/src/pybind/mgr/rook/module.py
@@ -766,3 +766,11 @@ def has_osds(matching_hosts: List[str]) -> bool:
     @handle_orch_error
     def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
         return self.rook_cluster.blink_light(ident_fault, on, locs)
+
+    @handle_orch_error
+    def upgrade_status(self) -> orchestrator.UpgradeStatusSpec:
+        return orchestrator.UpgradeStatusSpec()
+
+    @handle_orch_error
+    def upgrade_ls(self, image: Optional[str], tags: bool, show_all_versions: Optional[bool]) -> Dict[Any, Any]:
+        return {}

From c714c3c4f92994413e34a5900492c8987c85669e Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 19 Feb 2024 12:24:52 +0000
Subject: [PATCH 1977/2492] common/buffer_seastar: fix alien threads memory

The underlying raw_seastar_foreign_ptr::ptr is allocated from seastar.
This ptr is wrapped with seastar::foreign_ptr:
```
/// \c foreign_ptr<> wraps smart pointers -- \ref seastar::shared_ptr<>,
/// or similar, and remembers on what core this happened.
/// When the \c foreign_ptr<> object is destroyed, it sends a message to
/// the original core so that the wrapped object can be safely destroyed.
```

The issue is that once the pointer is de-allocated from an alien thread
it is unable to send a message to the original core.
Fix this issue by making use of seastar::alien integration with non-seastar applications.
In case ~raw_seastar_foreign_ptr() will be called from an alien thread, we will submit *and wait*
for the memory to be released from the origin core.

Fixes: https://tracker.ceph.com/issues/64086

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/common/buffer_seastar.cc | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/common/buffer_seastar.cc b/src/common/buffer_seastar.cc
index bc529c937ab2..fa040a4661c2 100644
--- a/src/common/buffer_seastar.cc
+++ b/src/common/buffer_seastar.cc
@@ -14,6 +14,8 @@
 
 #include <seastar/core/sharded.hh>
 #include <seastar/net/packet.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/core/alien.hh>
 
 #include "include/buffer_raw.h"
 #include "buffer_seastar.h"
@@ -24,9 +26,21 @@ namespace ceph::buffer {
 
 class raw_seastar_foreign_ptr : public raw {
   seastar::foreign_ptr<temporary_buffer> ptr;
+  seastar::alien::instance& alien;
  public:
   raw_seastar_foreign_ptr(temporary_buffer&& buf)
-    : raw(buf.get_write(), buf.size()), ptr(std::move(buf)) {}
+    : raw(buf.get_write(), buf.size()), ptr(std::move(buf)),
+      alien(seastar::engine().alien()) {}
+
+  ~raw_seastar_foreign_ptr() {
+    if (!seastar::engine_is_ready()) {
+      // we should let a seastar reactor destroy this memory, we are alien.
+      seastar::alien::run_on(alien, ptr.get_owner_shard(),
+      [_ptr = std::move(ptr)]() mutable noexcept {
+        _ptr.reset();
+      });
+    }
+  }
 };
 
 class raw_seastar_local_ptr : public raw {

From c8d8f739f8c3d0b260547e9e788ae2e191b3f446 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 19 Feb 2024 16:11:53 +0000
Subject: [PATCH 1978/2492] common/buffer_seastar: move to
 crimson/crimson/buffer_seastar

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/CMakeLists.txt                 | 2 +-
 src/crimson/auth/KeyRing.cc                | 2 +-
 src/{ => crimson}/common/buffer_seastar.cc | 0
 src/{ => crimson}/common/buffer_seastar.h  | 0
 src/test/crimson/test_denc.cc              | 2 +-
 5 files changed, 3 insertions(+), 3 deletions(-)
 rename src/{ => crimson}/common/buffer_seastar.cc (100%)
 rename src/{ => crimson}/common/buffer_seastar.h (100%)

diff --git a/src/crimson/CMakeLists.txt b/src/crimson/CMakeLists.txt
index 510ffbd9df99..6bbd7b49ec75 100644
--- a/src/crimson/CMakeLists.txt
+++ b/src/crimson/CMakeLists.txt
@@ -24,6 +24,7 @@ set(crimson_common_srcs
   common/throttle.cc
   common/tmap_helpers.cc
   common/tri_mutex.cc
+  common/buffer_seastar.cc
   crush/CrushLocation.cc)
 
 # the specialized version of ceph-common, where
@@ -35,7 +36,6 @@ add_library(crimson-common STATIC
   ${PROJECT_SOURCE_DIR}/src/common/bit_str.cc
   ${PROJECT_SOURCE_DIR}/src/common/bloom_filter.cc
   ${PROJECT_SOURCE_DIR}/src/common/buffer.cc
-  ${PROJECT_SOURCE_DIR}/src/common/buffer_seastar.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_argparse.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_context.cc
   ${PROJECT_SOURCE_DIR}/src/common/ceph_crypto.cc
diff --git a/src/crimson/auth/KeyRing.cc b/src/crimson/auth/KeyRing.cc
index 436e29c1bdd0..b64d2d0f78a8 100644
--- a/src/crimson/auth/KeyRing.cc
+++ b/src/crimson/auth/KeyRing.cc
@@ -10,7 +10,7 @@
 #include <seastar/core/future-util.hh>
 #include <seastar/core/reactor.hh>
 
-#include "common/buffer_seastar.h"
+#include "crimson/common/buffer_seastar.h"
 #include "auth/KeyRing.h"
 #include "include/denc.h"
 #include "crimson/common/buffer_io.h"
diff --git a/src/common/buffer_seastar.cc b/src/crimson/common/buffer_seastar.cc
similarity index 100%
rename from src/common/buffer_seastar.cc
rename to src/crimson/common/buffer_seastar.cc
diff --git a/src/common/buffer_seastar.h b/src/crimson/common/buffer_seastar.h
similarity index 100%
rename from src/common/buffer_seastar.h
rename to src/crimson/common/buffer_seastar.h
diff --git a/src/test/crimson/test_denc.cc b/src/test/crimson/test_denc.cc
index 10ebd6dce575..568f81cdf7f6 100644
--- a/src/test/crimson/test_denc.cc
+++ b/src/test/crimson/test_denc.cc
@@ -2,7 +2,7 @@
 #include <seastar/core/temporary_buffer.hh>
 #include <gtest/gtest.h>
 #include "include/denc.h"
-#include "common/buffer_seastar.h"
+#include "crimson/common/buffer_seastar.h"
 
 using temporary_buffer = seastar::temporary_buffer<char>;
 using buffer_iterator = seastar_buffer_iterator;

From e56e90857d6849c6045cc3edef896baffe70b6af Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 30 Jun 2023 12:33:35 +0800
Subject: [PATCH 1979/2492] crimson/osd: put snapmapper's key-value pairs into
 dedicated objs

Otherwise, PG::read_log_and_missing() will meet those key-values and
won't know what to do with them. This is modeling what the classic
osd is doing

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/pg.cc             | 14 ++++++++++++--
 src/crimson/osd/pg.h              | 16 ++++++++++++++--
 src/crimson/osd/shard_services.cc | 21 +++++++++++----------
 3 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 54c8101cd115..f5541da96194 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -136,7 +136,7 @@ PG::PG(
     osdriver(
       &shard_services.get_store(),
       coll_ref,
-      pgid.make_pgmeta_oid()),
+      make_snapmapper_oid()),
     snap_mapper(
       this->shard_services.get_cct(),
       &osdriver,
@@ -596,7 +596,7 @@ void PG::schedule_renew_lease(epoch_t last_peering_reset, ceph::timespan delay)
 }
 
 
-void PG::init(
+seastar::future<> PG::init(
   int role,
   const vector<int>& newup, int new_up_primary,
   const vector<int>& newacting, int new_acting_primary,
@@ -607,6 +607,16 @@ void PG::init(
   peering_state.init(
     role, newup, new_up_primary, newacting,
     new_acting_primary, history, pi, t);
+  assert(coll_ref);
+  return shard_services.get_store().exists(
+    get_collection_ref(), make_snapmapper_oid()
+  ).safe_then([&t, this](bool existed) {
+      if (!existed) {
+        t.touch(coll_ref->get_cid(), make_snapmapper_oid());
+      }
+    },
+    ::crimson::ct_error::assert_all{"unexpected eio"}
+  );
 }
 
 seastar::future<> PG::read_state(crimson::os::FuturizedStore::Shard* store)
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index e103d3bb5eec..0552d49f2ad4 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -41,6 +41,8 @@
 #include "crimson/osd/object_context_loader.h"
 #include "crimson/osd/scrub/pg_scrubber.h"
 
+#define SNAPMAPPER_OID "snapmapper"
+
 class MQuery;
 class OSDMap;
 class PGBackend;
@@ -471,7 +473,7 @@ class PG : public boost::intrusive_ref_counter<
   }
 
   /// initialize created PG
-  void init(
+  seastar::future<> init(
     int role,
     const std::vector<int>& up,
     int up_primary,
@@ -647,12 +649,22 @@ class PG : public boost::intrusive_ref_counter<
 private:
   OSDriver osdriver;
   SnapMapper snap_mapper;
-
+  ghobject_t make_snapmapper_oid() const {
+    return ghobject_t(hobject_t(
+      sobject_t(
+       object_t(SNAPMAPPER_OID),
+       0),
+      std::string(),
+      pgid.ps(),
+      pgid.pool(),
+      std::string()));
+  }
 public:
   // PeeringListener
   void publish_stats_to_osd() final;
   void clear_publish_stats() final;
   pg_stat_t get_stats() const;
+
 private:
   std::optional<pg_stat_t> pg_stats;
 
diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index 604c045eb9b8..d34ee1977c60 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -671,17 +671,17 @@ seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
 	    pg_shard_t(local_state.whoami, info->pgid.shard),
 	    acting);
 
-	  PeeringCtx rctx;
+	  std::unique_ptr<PeeringCtx> rctx = std::make_unique<PeeringCtx>();
 	  create_pg_collection(
-	    rctx.transaction,
+	    rctx->transaction,
 	    info->pgid,
 	    info->pgid.get_split_bits(pp->get_pg_num()));
 	  init_pg_ondisk(
-	    rctx.transaction,
+	    rctx->transaction,
 	    info->pgid,
 	    pp);
 
-	  pg->init(
+	  return pg->init(
 	    role,
 	    up,
 	    up_primary,
@@ -689,12 +689,13 @@ seastar::future<Ref<PG>> ShardServices::handle_pg_create_info(
 	    acting_primary,
 	    info->history,
 	    info->past_intervals,
-	    rctx.transaction);
-
-	  return start_operation<PGAdvanceMap>(
-	    pg, *this, get_map()->get_epoch(), std::move(rctx), true
-	  ).second.then([pg=pg] {
-	    return seastar::make_ready_future<Ref<PG>>(pg);
+	    rctx->transaction
+	  ).then([this, pg=pg, rctx=std::move(rctx)] {
+	    return start_operation<PGAdvanceMap>(
+	      pg, *this, get_map()->get_epoch(), std::move(*rctx), true
+	    ).second.then([pg=pg] {
+	      return seastar::make_ready_future<Ref<PG>>(pg);
+	    });
 	  });
 	});
     });

From 8f7921e8fb8a2035fa3f6c2cf6227b8176d6a654 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 21 Feb 2024 14:37:13 +0800
Subject: [PATCH 1980/2492] crimson/osd/pg: filter out snapmapper objects when
 doing pgls/pgnls

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/osd/ops_executer.cc | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index f03827b03d6b..7d6f11145316 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -1166,6 +1166,10 @@ static PG::interruptible_future<hobject_t> pgls_filter(
   }
 }
 
+static inline bool is_snapmapper_oid(const hobject_t &obj) {
+  return obj.oid.name == SNAPMAPPER_OID;
+}
+
 static PG::interruptible_future<ceph::bufferlist> do_pgnls_common(
   const hobject_t& pg_start,
   const hobject_t& pg_end,
@@ -1186,6 +1190,13 @@ static PG::interruptible_future<ceph::bufferlist> do_pgnls_common(
     [&backend, filter, nspace](auto&& ret)
     -> PG::interruptible_future<std::tuple<std::vector<hobject_t>, hobject_t>> {
       auto& [objects, next] = ret;
+      auto is_snapmapper = [](const hobject_t &obj) {
+	if (is_snapmapper_oid(obj)) {
+	  return false;
+	} else {
+	  return true;
+	}
+      };
       auto in_my_namespace = [&nspace](const hobject_t& obj) {
         using crimson::common::local_conf;
         if (obj.get_namespace() == local_conf()->osd_hit_set_namespace) {
@@ -1213,7 +1224,8 @@ static PG::interruptible_future<ceph::bufferlist> do_pgnls_common(
         }
       };
 
-      auto range = objects | boost::adaptors::filtered(in_my_namespace)
+      auto range = objects | boost::adaptors::filtered(is_snapmapper)
+			   | boost::adaptors::filtered(in_my_namespace)
                            | boost::adaptors::transformed(to_pglsed);
       logger().debug("do_pgnls_common: finishing the 1st stage of pgls");
       return seastar::when_all_succeed(std::begin(range),
@@ -1346,6 +1358,9 @@ static PG::interruptible_future<ceph::bufferlist> do_pgls_common(
         PG::interruptor::map_reduce(std::move(objects),
           [&backend, filter, nspace](const hobject_t& obj)
 	  -> PG::interruptible_future<hobject_t>{
+	    if (is_snapmapper_oid(obj)) {
+	      return seastar::make_ready_future<hobject_t>();
+	    }
             if (obj.get_namespace() == nspace) {
               if (filter) {
                 return pgls_filter(*filter, backend, obj);

From 518bff9c7ed6f6756aff8aa8013c48a5bfdd7b32 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Tue, 13 Feb 2024 15:56:09 +0530
Subject: [PATCH 1981/2492] mgr/dashboard: Handle errors for /api/osd/settings

Fixes https://tracker.ceph.com/issues/62089

issue:
=====
/api/osd/settings returns "TypeError: string indices must be
integers" sometimes.
The result is coming from `osd dump` command which instead of returning
an object returns an error message which then displays error on
dashboard.

fix:
====
Added a try-catch block to handle error and updated frontend code to
handle those

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 src/pybind/mgr/dashboard/controllers/osd.py   | 15 ++++--
 .../dashboard-pie/dashboard-pie.component.ts  | 49 ++++++++++---------
 .../ceph/dashboard/health/health.component.ts |  7 ++-
 .../usage-bar/usage-bar.component.html        |  2 +-
 4 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/osd.py b/src/pybind/mgr/dashboard/controllers/osd.py
index f6f8ce1f58a8..c9d141772000 100644
--- a/src/pybind/mgr/dashboard/controllers/osd.py
+++ b/src/pybind/mgr/dashboard/controllers/osd.py
@@ -168,11 +168,18 @@ def gauge_stats(osd, osd_spec):
     @RESTController.Collection('GET', version=APIVersion.EXPERIMENTAL)
     @ReadPermission
     def settings(self):
-        result = CephService.send_command('mon', 'osd dump')
-        return {
-            'nearfull_ratio': result['nearfull_ratio'],
-            'full_ratio': result['full_ratio']
+        data = {
+            'nearfull_ratio': -1,
+            'full_ratio': -1
         }
+        try:
+            result = CephService.send_command('mon', 'osd dump')
+            data['nearfull_ratio'] = result['nearfull_ratio']
+            data['full_ratio'] = result['full_ratio']
+        except TypeError:
+            logger.error(
+                'Error setting nearfull_ratio and full_ratio:', exc_info=True)
+        return data
 
     def _get_operational_status(self, osd_id: int, removing_osd_ids: Optional[List[int]]):
         if removing_osd_ids is None:
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
index 4680fb850a14..b0c253c33e9d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
@@ -59,7 +59,7 @@ export class DashboardPieComponent implements OnChanges, OnInit {
   constructor(private cssHelper: CssHelper, private dimlessBinary: DimlessBinaryPipe) {
     this.chartConfig = {
       chartType: 'doughnut',
-      labels: ['', '', ''],
+      labels: [],
       dataset: [
         {
           label: null,
@@ -97,19 +97,20 @@ export class DashboardPieComponent implements OnChanges, OnInit {
                   fillStyle: chart.data.datasets[1].backgroundColor[0],
                   strokeStyle: chart.data.datasets[1].backgroundColor[0]
                 };
-                labels[1] = {
-                  text: $localize`Warning: ${chart.data.datasets[0].data[0]}%`,
-                  fillStyle: chart.data.datasets[0].backgroundColor[1],
-                  strokeStyle: chart.data.datasets[0].backgroundColor[1]
-                };
-                labels[2] = {
-                  text: $localize`Danger: ${
-                    chart.data.datasets[0].data[0] + chart.data.datasets[0].data[1]
-                  }%`,
-                  fillStyle: chart.data.datasets[0].backgroundColor[2],
-                  strokeStyle: chart.data.datasets[0].backgroundColor[2]
-                };
-
+                if (chart.data.datasets[0].data?.length) {
+                  labels[1] = {
+                    text: $localize`Warning: ${chart.data.datasets[0].data[0]}%`,
+                    fillStyle: chart.data.datasets[0].backgroundColor[1],
+                    strokeStyle: chart.data.datasets[0].backgroundColor[1]
+                  };
+                  labels[2] = {
+                    text: $localize`Danger: ${
+                      chart.data.datasets[0].data[0] + chart.data.datasets[0].data[1]
+                    }%`,
+                    fillStyle: chart.data.datasets[0].backgroundColor[2],
+                    strokeStyle: chart.data.datasets[0].backgroundColor[2]
+                  };
+                }
                 return labels;
               }
             }
@@ -158,19 +159,24 @@ export class DashboardPieComponent implements OnChanges, OnInit {
     const fullRatioPercent = this.highThreshold * 100;
     const percentAvailable = this.calcPercentage(data.max - data.current, data.max);
     const percentUsed = this.calcPercentage(data.current, data.max);
-    if (percentUsed >= fullRatioPercent) {
+
+    if (fullRatioPercent >= 0 && percentUsed >= fullRatioPercent) {
       this.color = 'chart-color-red';
-    } else if (percentUsed >= nearFullRatioPercent) {
+    } else if (nearFullRatioPercent >= 0 && percentUsed >= nearFullRatioPercent) {
       this.color = 'chart-color-yellow';
     } else {
       this.color = 'chart-color-blue';
     }
 
-    chart.dataset[0].data = [
-      Math.round(nearFullRatioPercent),
-      Math.round(Math.abs(nearFullRatioPercent - fullRatioPercent)),
-      Math.round(100 - fullRatioPercent)
-    ];
+    if (fullRatioPercent >= 0 && nearFullRatioPercent >= 0) {
+      chart.dataset[0].data = [
+        Math.round(nearFullRatioPercent),
+        Math.round(Math.abs(nearFullRatioPercent - fullRatioPercent)),
+        Math.round(100 - fullRatioPercent)
+      ];
+    } else {
+      chart.dataset[1].backgroundColor[1] = this.cssHelper.propertyValue('chart-color-light-gray');
+    }
 
     chart.dataset[1].data = [
       percentUsed,
@@ -178,7 +184,6 @@ export class DashboardPieComponent implements OnChanges, OnInit {
       this.dimlessBinary.transform(data.current)
     ];
     chart.dataset[1].backgroundColor[0] = this.cssHelper.propertyValue(this.color);
-
     chart.dataset[0].label = [`${percentUsed}%\nof ${this.dimlessBinary.transform(data.max)}`];
   }
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
index 8210a4c81037..722886d44775 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts
@@ -164,9 +164,12 @@ export class HealthComponent implements OnInit, OnDestroy {
       data.df.stats.total_bytes
     );
 
-    if (percentUsed / 100 >= this.osdSettings.nearfull_ratio) {
+    const nearfullRatio = this.osdSettings.nearfull_ratio;
+    const fullRatio = this.osdSettings.nearfull_ratio;
+
+    if (nearfullRatio >= 0 && percentUsed / 100 >= nearfullRatio) {
       this.color = 'chart-color-red';
-    } else if (percentUsed / 100 >= this.osdSettings.full_ratio) {
+    } else if (fullRatio >= 0 && percentUsed / 100 >= fullRatio) {
       this.color = 'chart-color-yellow';
     } else {
       this.color = 'chart-color-blue';
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
index e7d7b17f0791..9a0a3398a3c0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -29,7 +29,7 @@
      data-placement="left"
      [ngbTooltip]="usageTooltipTpl">
   <div class="progress-bar bg-info"
-       [ngClass]="{'bg-warning': usedPercentage/100 >= warningThreshold, 'bg-danger': usedPercentage/100 >= errorThreshold}"
+       [ngClass]="{'bg-warning': (warningThreshold >= 0) && (usedPercentage/100 >= warningThreshold), 'bg-danger': (errorThreshold >= 0) && (usedPercentage/100 >= errorThreshold)}"
        role="progressbar"
        [attr.aria-label]="{ title }"
        i18n-aria-label="The title of this usage bar is { title }"

From 862fca945f5bf48144b6a589f1d3cd971444daf7 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 19 Feb 2024 11:14:11 -0500
Subject: [PATCH 1982/2492] cephadm: create ceph-exporter sock dir if it's not
 present

Since this is usually /var/run/ceph/ which ends up getting
created by other daemons as well, it was common to see
ceph-exporter fail to deploy and then deploy fine after
once other daemons were down on the host. I don't see any
reason we can't just try to make the directory here instead
of bailing out.

Fixes: https://tracker.ceph.com/issues/64491wq

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadmlib/daemons/ceph.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/ceph.py b/src/cephadm/cephadmlib/daemons/ceph.py
index 0afb8f734af5..45562fe4fdd4 100644
--- a/src/cephadm/cephadmlib/daemons/ceph.py
+++ b/src/cephadm/cephadmlib/daemons/ceph.py
@@ -299,8 +299,6 @@ def __init__(
         self.prio_limit = config_json.get('prio-limit', 5)
         self.stats_period = config_json.get('stats-period', 5)
 
-        self.validate()
-
     @classmethod
     def init(
         cls, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str]
@@ -329,7 +327,7 @@ def get_daemon_args(self) -> List[str]:
 
     def validate(self) -> None:
         if not os.path.isdir(self.sock_dir):
-            raise Error(f'Directory does not exist. Got: {self.sock_dir}')
+            raise Error(f'Desired sock dir for ceph-exporter is not directory: {self.sock_dir}')
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = daemon_to_container(ctx, self)
@@ -369,6 +367,13 @@ def customize_container_envs(
     def default_entrypoint(self) -> str:
         return self.entrypoint
 
+    def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None:
+        if not os.path.exists(self.sock_dir):
+            os.mkdir(self.sock_dir)
+        # part of validation is for the sock dir, so we postpone
+        # it until now
+        self.validate()
+
 
 def get_ceph_mounts_for_type(
     ctx: CephadmContext, fsid: str, daemon_type: str

From e462f76aedbb89e8db57dc2324d4f5e9fe54cf9e Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 12 Feb 2024 10:23:15 -0600
Subject: [PATCH 1983/2492] osd: improve hobject_t::to_str() performance

The new version of the code takes only ~70% of the time of the old one.
See https://github.com/ronen-fr/hobjtostr/tree/rf-1 for the code used
to benchmark the performance of this and various other implementations.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/fmt_common.h  |  1 +
 src/common/hobject.cc    | 50 ++++++++++++++++++----------------------
 src/common/hobject.h     |  6 ++---
 src/include/object_fmt.h |  3 ++-
 src/include/types.h      |  2 +-
 5 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/src/common/fmt_common.h b/src/common/fmt_common.h
index 1a2a6eac9357..53ce8a10ddb7 100644
--- a/src/common/fmt_common.h
+++ b/src/common/fmt_common.h
@@ -7,6 +7,7 @@
 /**
  * \file default fmtlib formatters for specifically-tagged types
  */
+#include <fmt/compile.h>
 #include <fmt/format.h>
 
 /**
diff --git a/src/common/hobject.cc b/src/common/hobject.cc
index 1aee4cc42546..9f8e67f10b8b 100644
--- a/src/common/hobject.cc
+++ b/src/common/hobject.cc
@@ -2,6 +2,8 @@
 // vim: ts=8 sw=2 smarttab
 
 #include <charconv>
+#include <fmt/compile.h>
+#include <fmt/core.h>
 
 #include "hobject.h"
 #include "common/Formatter.h"
@@ -14,23 +16,25 @@ using std::string;
 using ceph::bufferlist;
 using ceph::Formatter;
 
-static void append_escaped(const string &in, string *out)
+namespace {
+void escape_special_chars(const string& in, string* out)
 {
-  for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
-    if (*i == '%') {
+  for (auto c : in) {
+    if (c == '%') {
       out->push_back('%');
       out->push_back('p');
-    } else if (*i == '.') {
+    } else if (c == '.') {
       out->push_back('%');
       out->push_back('e');
-    } else if (*i == '_') {
+    } else if (c == '_') {
       out->push_back('%');
       out->push_back('u');
     } else {
-      out->push_back(*i);
+      out->push_back(c);
     }
   }
 }
+}  // namespace
 
 set<string> hobject_t::get_prefixes(
   uint32_t bits,
@@ -80,33 +84,25 @@ set<string> hobject_t::get_prefixes(
 
 string hobject_t::to_str() const
 {
-  string out;
-
-  char snap_with_hash[1000];
-  char *t = snap_with_hash;
-  const char *end = t + sizeof(snap_with_hash);
-
   uint64_t poolid(pool);
-  t += snprintf(t, end - t, "%.*llX", 16, (long long unsigned)poolid);
-
   uint32_t revhash(get_nibblewise_key_u32());
-  t += snprintf(t, end - t, ".%.*X", 8, revhash);
 
-  if (snap == CEPH_NOSNAP)
-    t += snprintf(t, end - t, ".head");
-  else if (snap == CEPH_SNAPDIR)
-    t += snprintf(t, end - t, ".snapdir");
-  else
-    t += snprintf(t, end - t, ".%llx", (long long unsigned)snap);
-
-  out.append(snap_with_hash, t);
+  string out;
+  if (snap == CEPH_NOSNAP) {
+    out = fmt::format(FMT_COMPILE("{:016X}.{:08X}.head."), poolid, revhash);
+  } else if (snap == CEPH_SNAPDIR) {
+    out = fmt::format(FMT_COMPILE("{:016X}.{:08X}.snapdir."), poolid, revhash);
+  } else {
+    out = fmt::format(
+	FMT_COMPILE("{:016X}.{:08X}.{:X}."), poolid, revhash,
+	(unsigned long long)snap);
+  }
 
+  escape_special_chars(oid.name, &out);
   out.push_back('.');
-  append_escaped(oid.name, &out);
-  out.push_back('.');
-  append_escaped(get_key(), &out);
+  escape_special_chars(get_key(), &out);
   out.push_back('.');
-  append_escaped(nspace, &out);
+  escape_special_chars(nspace, &out);
 
   return out;
 }
diff --git a/src/common/hobject.h b/src/common/hobject.h
index e35e2b0732f6..fe7f9a705c98 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -318,7 +318,7 @@ struct hobject_t {
   void dump(ceph::Formatter *f) const;
   static void generate_test_instances(std::list<hobject_t*>& o);
   friend int cmp(const hobject_t& l, const hobject_t& r);
-  auto operator<=>(const hobject_t &rhs) const noexcept {
+  constexpr auto operator<=>(const hobject_t &rhs) const noexcept {
     auto cmp = max <=> rhs.max;
     if (cmp != 0) return cmp;
     cmp = pool <=> rhs.pool;
@@ -335,7 +335,7 @@ struct hobject_t {
     if (cmp != 0) return cmp;
     return snap <=> rhs.snap;
   }
-  bool operator==(const hobject_t& rhs) const noexcept {
+  constexpr bool operator==(const hobject_t& rhs) const noexcept {
     return operator<=>(rhs) == 0;
   }
   friend struct ghobject_t;
@@ -495,7 +495,7 @@ struct ghobject_t {
   void dump(ceph::Formatter *f) const;
   static void generate_test_instances(std::list<ghobject_t*>& o);
   friend int cmp(const ghobject_t& l, const ghobject_t& r);
-  auto operator<=>(const ghobject_t&) const = default;
+  constexpr auto operator<=>(const ghobject_t&) const = default;
   bool operator==(const ghobject_t&) const = default;
 };
 WRITE_CLASS_ENCODER(ghobject_t)
diff --git a/src/include/object_fmt.h b/src/include/object_fmt.h
index 33df5e3fbd99..25fbc6714c8e 100644
--- a/src/include/object_fmt.h
+++ b/src/include/object_fmt.h
@@ -5,6 +5,7 @@
 /**
  * \file fmtlib formatters for some object.h structs
  */
+#include <fmt/compile.h>
 #include <fmt/format.h>
 
 #include "object.h"
@@ -24,6 +25,6 @@ struct fmt::formatter<snapid_t> {
     if (snp == CEPH_SNAPDIR) {
       return fmt::format_to(ctx.out(), "snapdir");
     }
-    return fmt::format_to(ctx.out(), "{:x}", snp.val);
+    return fmt::format_to(ctx.out(), FMT_COMPILE("{:x}"), snp.val);
   }
 };
diff --git a/src/include/types.h b/src/include/types.h
index a50a506eb0bc..b3777d9de4fd 100644
--- a/src/include/types.h
+++ b/src/include/types.h
@@ -513,7 +513,7 @@ struct shard_id_t {
   shard_id_t() : id(0) {}
   constexpr explicit shard_id_t(int8_t _id) : id(_id) {}
 
-  operator int8_t() const { return id; }
+  constexpr operator int8_t() const { return id; }
 
   const static shard_id_t NO_SHARD;
 

From 941d3f2aac87160f9d09de3e2f1ddea1e51e7275 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 15 Feb 2024 12:28:11 -0600
Subject: [PATCH 1984/2492] common: fold object_fmt.h into object.h

as fmtlib is now an accepted dependency in all of Ceph.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/hobject_fmt.h           |  1 -
 src/include/object.h               | 26 ++++++++++++++++++++++++--
 src/include/object_fmt.h           | 30 ------------------------------
 src/test/osd/scrubber_generators.h |  2 +-
 4 files changed, 25 insertions(+), 34 deletions(-)
 delete mode 100644 src/include/object_fmt.h

diff --git a/src/common/hobject_fmt.h b/src/common/hobject_fmt.h
index 622611121ae6..a2a730e1c5fc 100644
--- a/src/common/hobject_fmt.h
+++ b/src/common/hobject_fmt.h
@@ -9,7 +9,6 @@
 #include <fmt/ranges.h>
 
 #include "common/hobject.h"
-#include "include/object_fmt.h"
 #include "msg/msg_fmt.h"
 
 // \todo reimplement
diff --git a/src/include/object.h b/src/include/object.h
index 2e5fb471c139..4e6001d09e60 100644
--- a/src/include/object.h
+++ b/src/include/object.h
@@ -23,6 +23,9 @@
 #include <string>
 #include <string_view>
 
+#include <fmt/compile.h>
+#include <fmt/format.h>
+
 #include "include/rados.h"
 #include "include/unordered_map.h"
 #include "common/Formatter.h"
@@ -112,10 +115,10 @@ struct file_object_t {
 struct snapid_t {
   uint64_t val;
   // cppcheck-suppress noExplicitConstructor
-  snapid_t(uint64_t v=0) : val(v) {}
+  constexpr snapid_t(uint64_t v=0) : val(v) {}
   snapid_t operator+=(snapid_t o) { val += o.val; return *this; }
   snapid_t operator++() { ++val; return *this; }
-  operator uint64_t() const { return val; }
+  constexpr operator uint64_t() const { return val; }
 };
 
 inline void encode(snapid_t i, ceph::buffer::list &bl) {
@@ -153,6 +156,25 @@ inline std::ostream& operator<<(std::ostream& out, const snapid_t& s) {
     return out << std::hex << s.val << std::dec;
 }
 
+namespace fmt {
+template <>
+struct formatter<snapid_t> {
+
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const snapid_t& snp, FormatContext& ctx) const
+  {
+    if (snp == CEPH_NOSNAP) {
+      return fmt::format_to(ctx.out(), "head");
+    }
+    if (snp == CEPH_SNAPDIR) {
+      return fmt::format_to(ctx.out(), "snapdir");
+    }
+    return fmt::format_to(ctx.out(), FMT_COMPILE("{:x}"), snp.val);
+  }
+};
+} // namespace fmt
 
 struct sobject_t {
   object_t oid;
diff --git a/src/include/object_fmt.h b/src/include/object_fmt.h
deleted file mode 100644
index 25fbc6714c8e..000000000000
--- a/src/include/object_fmt.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-#pragma once
-
-/**
- * \file fmtlib formatters for some object.h structs
- */
-#include <fmt/compile.h>
-#include <fmt/format.h>
-
-#include "object.h"
-
-
-template <>
-struct fmt::formatter<snapid_t> {
-
-  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
-
-  template <typename FormatContext>
-  auto format(const snapid_t& snp, FormatContext& ctx) const
-  {
-    if (snp == CEPH_NOSNAP) {
-      return fmt::format_to(ctx.out(), "head");
-    }
-    if (snp == CEPH_SNAPDIR) {
-      return fmt::format_to(ctx.out(), "snapdir");
-    }
-    return fmt::format_to(ctx.out(), FMT_COMPILE("{:x}"), snp.val);
-  }
-};
diff --git a/src/test/osd/scrubber_generators.h b/src/test/osd/scrubber_generators.h
index d0cbb22c4c80..43b123250cc1 100644
--- a/src/test/osd/scrubber_generators.h
+++ b/src/test/osd/scrubber_generators.h
@@ -13,7 +13,7 @@
 
 #include "include/buffer.h"
 #include "include/buffer_raw.h"
-#include "include/object_fmt.h"
+#include "include/object.h"
 #include "osd/osd_types_fmt.h"
 #include "osd/scrubber/pg_scrubber.h"
 

From ec1cd79b37153dd23391c742536d64df1ba8c52d Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 18 Feb 2024 00:32:10 -0600
Subject: [PATCH 1985/2492] common: improve hobject_t fmtlib formatter
 performance

The new version of the code only takes about 38% of the time of the old
one.  See https://github.com/ronen-fr/hobjtostr/tree/rf-2 for the code
used to benchmark the new version.

hobject_fmt.h is folded into hobject.h, as fmtlib is now an accepted
dependency in all of Ceph.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/hobject.h                  | 51 ++++++++++++++++++++++++++
 src/common/hobject_fmt.h              | 52 ---------------------------
 src/crimson/osd/backfill_state.cc     |  2 +-
 src/crimson/osd/pg.cc                 |  2 +-
 src/crimson/osd/scrub/scrub_machine.h |  1 -
 src/msg/msg_types.h                   |  9 +++++
 src/osd/osd_types_fmt.h               |  3 +-
 7 files changed, 64 insertions(+), 56 deletions(-)
 delete mode 100644 src/common/hobject_fmt.h

diff --git a/src/common/hobject.h b/src/common/hobject.h
index fe7f9a705c98..d8754550a0da 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -15,6 +15,9 @@
 #ifndef __CEPH_OS_HOBJECT_H
 #define __CEPH_OS_HOBJECT_H
 
+#include <fmt/compile.h>
+#include <fmt/format.h>
+
 #if FMT_VERSION >= 90000
 #include <fmt/ostream.h>
 #endif
@@ -351,6 +354,54 @@ template<> struct hash<hobject_t> {
 };
 } // namespace std
 
+namespace fmt {
+template <>
+struct formatter<hobject_t> {
+
+  template <typename FormatContext>
+  static inline auto
+  append_sanitized(FormatContext& ctx, const std::string& in, int sep = 0)
+  {
+    for (const auto i : in) {
+      if (i == '%' || i == ':' || i == '/' || i < 32 || i >= 127) {
+	fmt::format_to(
+	    ctx.out(), FMT_COMPILE("%{:02x}"), static_cast<unsigned char>(i));
+      } else {
+	fmt::format_to(ctx.out(), FMT_COMPILE("{:c}"), i);
+      }
+    }
+    if (sep) {
+      fmt::format_to(
+	  ctx.out(), FMT_COMPILE("{:c}"), sep);
+    }
+    return ctx.out();
+  }
+
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const hobject_t& ho, FormatContext& ctx)
+  {
+    if (ho == hobject_t{}) {
+      return fmt::format_to(ctx.out(), "MIN");
+    }
+
+    if (ho.is_max()) {
+      return fmt::format_to(ctx.out(), "MAX");
+    }
+
+    fmt::format_to(
+	ctx.out(), FMT_COMPILE("{}:{:08x}:"), static_cast<uint64_t>(ho.pool),
+	ho.get_bitwise_key_u32());
+    append_sanitized(ctx, ho.nspace, ':');
+    append_sanitized(ctx, ho.get_key(), ':');
+    append_sanitized(ctx, ho.oid.name);
+    return fmt::format_to(ctx.out(), FMT_COMPILE(":{}"), ho.snap);
+  }
+};
+}  // namespace fmt
+
+
 std::ostream& operator<<(std::ostream& out, const hobject_t& o);
 
 template <typename T>
diff --git a/src/common/hobject_fmt.h b/src/common/hobject_fmt.h
deleted file mode 100644
index a2a730e1c5fc..000000000000
--- a/src/common/hobject_fmt.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-#pragma once
-
-/**
- * \file fmtlib formatters for some hobject.h classes
- */
-#include <fmt/format.h>
-#include <fmt/ranges.h>
-
-#include "common/hobject.h"
-#include "msg/msg_fmt.h"
-
-// \todo reimplement
-static inline void append_out_escaped(const std::string& in, std::string* out)
-{
-  for (auto i = in.cbegin(); i != in.cend(); ++i) {
-    if (*i == '%' || *i == ':' || *i == '/' || *i < 32 || *i >= 127) {
-      char buf[4];
-      snprintf(buf, sizeof(buf), "%%%02x", (int)(unsigned char)*i);
-      out->append(buf);
-    } else {
-      out->push_back(*i);
-    }
-  }
-}
-
-template <> struct fmt::formatter<hobject_t> {
-
-  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
-
-  template <typename FormatContext> auto format(const hobject_t& ho, FormatContext& ctx)
-  {
-    if (ho == hobject_t{}) {
-      return fmt::format_to(ctx.out(), "MIN");
-    }
-
-    if (ho.is_max()) {
-      return fmt::format_to(ctx.out(), "MAX");
-    }
-
-    std::string v;
-    append_out_escaped(ho.nspace, &v);
-    v.push_back(':');
-    append_out_escaped(ho.get_key(), &v);
-    v.push_back(':');
-    append_out_escaped(ho.oid.name, &v);
-
-    return fmt::format_to(ctx.out(), "{}:{:08x}:{}:{}", static_cast<uint64_t>(ho.pool),
-			  ho.get_bitwise_key_u32(), v, ho.snap);
-  }
-};
diff --git a/src/crimson/osd/backfill_state.cc b/src/crimson/osd/backfill_state.cc
index 46a270ffe54d..92b3ea8714cd 100644
--- a/src/crimson/osd/backfill_state.cc
+++ b/src/crimson/osd/backfill_state.cc
@@ -4,7 +4,7 @@
 #include <algorithm>
 #include <boost/type_index.hpp>
 #include <fmt/ranges.h>
-#include "common/hobject_fmt.h"
+#include "common/hobject.h"
 #include "crimson/osd/backfill_state.h"
 #include "osd/osd_types_fmt.h"
 
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 54c8101cd115..697f4e2225ca 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -14,7 +14,7 @@
 #include <fmt/format.h>
 #include <fmt/ostream.h>
 
-#include "common/hobject_fmt.h"
+#include "common/hobject.h"
 
 #include "messages/MOSDOp.h"
 #include "messages/MOSDOpReply.h"
diff --git a/src/crimson/osd/scrub/scrub_machine.h b/src/crimson/osd/scrub/scrub_machine.h
index d2d127adc0d9..f6cec5cba71a 100644
--- a/src/crimson/osd/scrub/scrub_machine.h
+++ b/src/crimson/osd/scrub/scrub_machine.h
@@ -18,7 +18,6 @@
 
 #include "common/fmt_common.h"
 #include "common/hobject.h"
-#include "common/hobject_fmt.h"
 #include "crimson/common/log.h"
 #include "osd/osd_types_fmt.h"
 #include "scrub_validator.h"
diff --git a/src/msg/msg_types.h b/src/msg/msg_types.h
index b39120cc08d4..e1e3ecca749f 100644
--- a/src/msg/msg_types.h
+++ b/src/msg/msg_types.h
@@ -98,6 +98,15 @@ class entity_name_t {
   }
   void dump(ceph::Formatter *f) const;
 
+  template <typename FormatContext>
+  auto fmt_print_ctx(FormatContext& ctx) const {
+    if (is_new() || _num < 0) {
+      return fmt::format_to(ctx.out(), "{}.?", type_str());
+    } else {
+      return fmt::format_to(ctx.out(), "{}.{}",type_str(), _num);
+    }
+  }
+
   static void generate_test_instances(std::list<entity_name_t*>& o);
 };
 WRITE_CLASS_DENC(entity_name_t)
diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h
index e467d5d23044..a88bf08c5ec0 100644
--- a/src/osd/osd_types_fmt.h
+++ b/src/osd/osd_types_fmt.h
@@ -5,9 +5,10 @@
  * \file fmtlib formatters for some types.h classes
  */
 
-#include "common/hobject_fmt.h"
+#include "common/hobject.h"
 #include "osd/osd_types.h"
 #include <fmt/chrono.h>
+#include <fmt/ranges.h>
 #if FMT_VERSION >= 90000
 #include <fmt/ostream.h>
 #endif

From 5e2f4e853679b23f17b3ba725549776e4d5861a5 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 18 Feb 2024 05:53:50 -0600
Subject: [PATCH 1986/2492] osd: use empty() instead of length()

for std::string

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_types.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 8f08e298ee53..b95379ba8758 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -315,9 +315,9 @@ inline bool operator!=(const object_locator_t& l, const object_locator_t& r) {
 inline std::ostream& operator<<(std::ostream& out, const object_locator_t& loc)
 {
   out << "@" << loc.pool;
-  if (loc.nspace.length())
+  if (!loc.nspace.empty())
     out << ";" << loc.nspace;
-  if (loc.key.length())
+  if (!loc.key.empty())
     out << ":" << loc.key;
   return out;
 }

From cbf78503390e35395c1aaede46b88de66e5f15d6 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 18 Feb 2024 09:42:07 -0600
Subject: [PATCH 1987/2492] test/scrub: fixes to aggregate initialization in
 test_crimson_scrub.cc

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/crimson/test_crimson_scrub.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/crimson/test_crimson_scrub.cc b/src/test/crimson/test_crimson_scrub.cc
index 65b1f3152539..b72c8cb979b0 100644
--- a/src/test/crimson/test_crimson_scrub.cc
+++ b/src/test/crimson/test_crimson_scrub.cc
@@ -854,7 +854,7 @@ TEST_P(TestSingleError, SingleError) {
   bool found_selected_oi = false;
   for (const auto &shard : shards) {
     auto siter = obj_error.shards.find(
-      librados::osd_shard_t(shard.osd, shard.shard)
+      librados::osd_shard_t{shard.osd, shard.shard}
     );
     if (siter == obj_error.shards.end()) {
       EXPECT_NE(siter, obj_error.shards.end());
@@ -1125,7 +1125,7 @@ TEST_P(TestSnapSetCloneError, CloneError) {
       auto to_insert = make_clone(name, should_exist[i]);
       if (GetParam().should_inject_size(i)) {
 	expected_error.set_size_mismatch();
-	to_insert.second = so_builder_t(to_insert.second).set_size(
+	to_insert.second = so_builder_t{to_insert.second}.set_size(
 	  so_get_oi(to_insert.second)->size + 1).get();
       }
       map.objects.insert(to_insert);

From fe26408d214f1d8e949aff6379b58de56b43cd4c Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 19 Feb 2024 07:32:34 -0600
Subject: [PATCH 1988/2492] test: fixing CI compilation of test_transaction.cc

Fixing a compilation warning in test_transaction.cc, as the CI
builds it with '-Werror'.

See https://jenkins.ceph.com/job/ceph-pull-requests/129517/

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/test/objectstore/test_transaction.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/test/objectstore/test_transaction.cc b/src/test/objectstore/test_transaction.cc
index 381b9df7d642..a2113addeb95 100644
--- a/src/test/objectstore/test_transaction.cc
+++ b/src/test/objectstore/test_transaction.cc
@@ -92,7 +92,6 @@ ObjectStore::Transaction generate_transaction()
 
   coll_t acid;
   object_t aobj("another_test_name");
-  snapid_t asnap(0);
   hobject_t ahoid(obj, "another_key", snap, 0, 0, "another_nspace");
   ghobject_t aoid(hoid);
   std::set<string> keys;
@@ -153,7 +152,6 @@ TEST(Transaction, GetNumBytes)
 
   coll_t acid;
   object_t aobj("another_test_name");
-  snapid_t asnap(0);
   hobject_t ahoid(obj, "another_key", snap, 0, 0, "another_nspace");
   ghobject_t aoid(hoid);
   std::set<string> keys;

From 94f1cadda240292086eae7d1e484503005a6ceb1 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 21 Feb 2024 01:49:35 -0600
Subject: [PATCH 1989/2492] test: verify that hobject_t formatting is stable

using 'Squid' as baseline.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/hobject.h            |   1 +
 src/test/common/test_hobject.cc | 307 ++++++++++++++++++++++++++++++++
 2 files changed, 308 insertions(+)

diff --git a/src/common/hobject.h b/src/common/hobject.h
index d8754550a0da..8e0af9da778e 100644
--- a/src/common/hobject.h
+++ b/src/common/hobject.h
@@ -342,6 +342,7 @@ struct hobject_t {
     return operator<=>(rhs) == 0;
   }
   friend struct ghobject_t;
+  friend struct test_hobject_fmt_t;
 };
 WRITE_CLASS_ENCODER(hobject_t)
 
diff --git a/src/test/common/test_hobject.cc b/src/test/common/test_hobject.cc
index 0bb4aef9ee18..19945754004e 100644
--- a/src/test/common/test_hobject.cc
+++ b/src/test/common/test_hobject.cc
@@ -1,6 +1,13 @@
+#include <string>
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
 #include "common/hobject.h"
 #include "gtest/gtest.h"
 
+using namespace std::string_literals;
+using std::string;
+
 TEST(HObject, cmp)
 {
   hobject_t c{object_t{"fooc"}, "food", CEPH_NOSNAP, 42, 0, "nspace"};
@@ -9,3 +16,303 @@ TEST(HObject, cmp)
   ASSERT_EQ(-1, cmp(c, d));
   ASSERT_EQ(-1, cmp(d, e));
 }
+
+// ---- test methods that 'stringify' the object while escaping special characters ----
+
+
+/*
+ * Two methods are used here: first - using a preset list of objects & hobjects,
+ * comparing the output to the expected string; and a second method: comparing the
+ * output for a "random"(*) object to the results when using the code from 'Squid'.
+ *
+ * (*) the object is not random, but it's not part of the preset list.
+ */
+
+
+struct obj_n_expected_t {
+  hobject_t obj;
+  std::string expected_to_str;
+  std::string expected_fmt;
+};
+
+static std::vector<obj_n_expected_t> known_examples = {
+
+    // the first entry will be modified (by setting the max flag)
+    {hobject_t{}, "MAX", "MAX"},
+
+    {hobject_t{object_t("o%:/name2"), "aaaa"s, CEPH_NOSNAP, 67, 0, "n1"s},
+     "0000000000000000.34000000.head.o%p:/name2.aaaa.n1",
+     "0:c2000000:n1:aaaa:o%25%3a%2fname2:head"},
+
+    {hobject_t{object_t("okey"), "okey"s, CEPH_NOSNAP, 1, 0, "n12"s},
+     "0000000000000000.10000000.head.okey..n12", "0:80000000:n12::okey:head"},
+
+    {hobject_t{}, "8000000000000000.00000000.0...", "MIN"},
+
+/// \todo not sure whether the '-1' or the 'FFF..' is correct:
+#if 0
+   {hobject_t{object_t("oname"), std::string{}, 1, 234, -1, ""s},
+      "FFFFFFFFFFFFFFFF.AE000000.1.oxxname..",
+     "18446744073709551615:57000000:::oname:1"},
+#endif
+    {hobject_t{object_t{"oname3"}, "oname3"s, CEPH_SNAPDIR, 910, 1, "n2"s},
+     "0000000000000001.E8300000.snapdir.oname3..n2",
+     "1:71c00000:n2::oname3:snapdir"},
+
+    {hobject_t{
+	 object_t("nonprint\030%_%.%"), "c"s, 0x12345678, 0xe0e0f0f0, 0x2727,
+	 "n5"s},
+     "0000000000002727.0F0F0E0E.12345678.nonprint\x18%p%u%p%e%p.c.n5",
+     "10023:0f0f0707:n5:c:nonprint%18%25_%25.%25:12345678"},
+
+    {hobject_t{object_t("o//////"), string("ZZ"), 0xaaaa, 65, 1, "zzzzz"},
+     "0000000000000001.14000000.aaaa.o//////.ZZ.zzzzz",
+     "1:82000000:zzzzz:ZZ:o%2f%2f%2f%2f%2f%2f:aaaa"}};
+
+// original Ceph code as it was in version Squid
+
+struct test_hobject_fmt_t : public hobject_t {
+
+  template <typename... ARGS>
+  test_hobject_fmt_t(ARGS&&... args) : hobject_t{std::forward<ARGS>(args)...}
+  {}
+
+  test_hobject_fmt_t(const test_hobject_fmt_t& rhs) = default;
+  test_hobject_fmt_t(test_hobject_fmt_t&& rhs) = default;
+  test_hobject_fmt_t& operator=(const test_hobject_fmt_t& rhs) = default;
+  test_hobject_fmt_t& operator=(test_hobject_fmt_t&& rhs) = default;
+  test_hobject_fmt_t(hobject_t_max&& singleton) : test_hobject_fmt_t()
+  {
+    max = true;
+  }
+  test_hobject_fmt_t& operator=(hobject_t_max&& singleton)
+  {
+    *this = hobject_t();
+    max = true;
+    return *this;
+  }
+  bool is_max() const { return max; }
+  bool is_min() const
+  {
+    // this needs to match how it's constructed
+    return snap == 0 && hash == 0 && !max && pool == INT64_MIN;
+  }
+
+  constexpr auto operator<=>(const test_hobject_fmt_t& rhs) const noexcept
+  {
+    auto cmp = is_max() <=> rhs.is_max();
+    if (cmp != 0)
+      return cmp;
+    cmp = pool <=> rhs.pool;
+    if (cmp != 0)
+      return cmp;
+    cmp = get_bitwise_key() <=> rhs.get_bitwise_key();
+    if (cmp != 0)
+      return cmp;
+    cmp = nspace <=> rhs.nspace;
+    if (cmp != 0)
+      return cmp;
+    if (!(get_key().empty() && rhs.get_key().empty())) {
+      cmp = get_effective_key() <=> rhs.get_effective_key();
+      if (cmp != 0)
+	return cmp;
+    }
+    cmp = oid <=> rhs.oid;
+    if (cmp != 0)
+      return cmp;
+    return snap <=> rhs.snap;
+  }
+  constexpr bool operator==(const hobject_t& rhs) const noexcept
+  {
+    return operator<=>(rhs) == 0;
+  }
+};
+
+static inline void append_out_escaped(const std::string& in, std::string* out)
+{
+  for (auto i = in.cbegin(); i != in.cend(); ++i) {
+    if (*i == '%' || *i == ':' || *i == '/' || *i < 32 || *i >= 127) {
+      char buf[4];
+      snprintf(buf, sizeof(buf), "%%%02x", (int)(unsigned char)*i);
+      out->append(buf);
+    } else {
+      out->push_back(*i);
+    }
+  }
+}
+
+// why don't we escape non-printable characters?
+static void append_escaped(const string& in, string* out)
+{
+  for (string::const_iterator i = in.begin(); i != in.end(); ++i) {
+    if (*i == '%') {
+      out->push_back('%');
+      out->push_back('p');
+    } else if (*i == '.') {
+      out->push_back('%');
+      out->push_back('e');
+    } else if (*i == '_') {
+      out->push_back('%');
+      out->push_back('u');
+    } else {
+      out->push_back(*i);
+    }
+  }
+}
+
+// original Ceph code as it was in version Squid
+string hobject_t::to_str() const
+{
+  string out;
+
+  char snap_with_hash[1000];
+  char* t = snap_with_hash;
+  const char* end = t + sizeof(snap_with_hash);
+
+  uint64_t poolid(pool);
+  t += snprintf(t, end - t, "%.*llX", 16, (long long unsigned)poolid);
+
+  uint32_t revhash(get_nibblewise_key_u32());
+  t += snprintf(t, end - t, ".%.*X", 8, revhash);
+
+  if (snap == CEPH_NOSNAP)
+    t += snprintf(t, end - t, ".head");
+  else if (snap == CEPH_SNAPDIR)
+    t += snprintf(t, end - t, ".snapdir");
+  else
+    t += snprintf(t, end - t, ".%llx", (long long unsigned)snap);
+
+  out.append(snap_with_hash, t);
+
+  out.push_back('.');
+  append_escaped(oid.name, &out);
+  out.push_back('.');
+  append_escaped(get_key(), &out);
+  out.push_back('.');
+  append_escaped(nspace, &out);
+
+  return out;
+}
+
+
+namespace fmt {
+// original Ceph code as it was in version Squid
+// (modified to use test_hobject_fmt_t)
+template <>
+struct formatter<test_hobject_fmt_t> {
+
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const test_hobject_fmt_t& ho, FormatContext& ctx)
+  {
+    if (ho == hobject_t{}) {
+      return fmt::format_to(ctx.out(), "MIN");
+    }
+
+    if (ho.is_max()) {
+      return fmt::format_to(ctx.out(), "MAX");
+    }
+
+    std::string v;
+    append_out_escaped(ho.nspace, &v);
+    v.push_back(':');
+    append_out_escaped(ho.get_key(), &v);
+    v.push_back(':');
+    append_out_escaped(ho.oid.name, &v);
+
+    return fmt::format_to(
+	ctx.out(), "{}:{:08x}:{}:{}", static_cast<uint64_t>(ho.pool),
+	ho.get_bitwise_key_u32(), v, ho.snap);
+  }
+};
+}  // namespace fmt
+
+
+TEST(HObject, to_str)
+{
+  const auto dbg = false;  // turns on debug output
+  known_examples[0].obj = hobject_t::get_max();
+
+  for (const auto& [obj, expected_to_str, expected_fmt] : known_examples) {
+    if (obj.is_max()) {
+      // no 'max' for to_str()
+      continue;
+    }
+    test_hobject_fmt_t legacy_obj{obj};
+    if (dbg) {
+      std::cout << "to_str(): legacy: " << legacy_obj.to_str()
+		<< " . Now: " << obj.to_str() << std::endl;
+    }
+    EXPECT_EQ(legacy_obj.to_str(), obj.to_str());
+    EXPECT_EQ(expected_to_str, obj.to_str());
+  }
+}
+
+// test the fmt::formatter for hobject_t vs legacy & the stream operator
+TEST(HObject, fmt)
+{
+  const auto dbg = false;  // turns on debug output
+  known_examples[0].obj = hobject_t::get_max();
+
+  for (const auto& [obj, expected_to_str, expected_fmt] : known_examples) {
+
+    test_hobject_fmt_t legacy_obj{obj};
+    if (dbg) {
+      std::cout << fmt::format("fmt: legacy: {} now: {}", legacy_obj, obj)
+		<< std::endl;
+    }
+    EXPECT_EQ(fmt::format("{}", legacy_obj), fmt::format("{}", obj));
+    EXPECT_EQ(expected_fmt, fmt::format("{}", obj));
+
+    if (dbg) {
+      std::cout << "ostream: legacy: " << legacy_obj << " . Now: " << obj
+		<< std::endl;
+    }
+    std::ostringstream oss;
+    oss << obj;
+    std::ostringstream oss_legacy;
+    oss_legacy << legacy_obj;
+    EXPECT_EQ(oss_legacy.str(), oss.str());
+    EXPECT_EQ(oss.str(), fmt::format("{}", obj));
+  }
+}
+
+TEST(HObject, fmt_random)
+{
+  const auto dbg = false;  // turns on debug output
+  for (uint32_t i = 0; i < 10; i++) {
+
+    auto name_length = (i * 17) % 51;
+    std::string name;
+    for (int j = 0; j < name_length; j++) {
+      name.push_back((i * name_length + j) % 256);
+    }
+
+    std::string key =
+	(i % 3) ? fmt::format("key_{}::", static_cast<unsigned char>(i)) : name;
+
+    snapid_t snap = (i % 7) ? i : ((i % 2) ? CEPH_SNAPDIR : CEPH_NOSNAP);
+
+    hobject_t obj{object_t{name}, key, snap, i, i % 10, "n:_%.space"s};
+
+    test_hobject_fmt_t legacy_obj{obj};
+
+    if (dbg) {
+      std::cout << fmt::format("fmt: legacy: {} now: {}", legacy_obj, obj)
+		<< std::endl;
+    }
+    EXPECT_EQ(fmt::format("{}", legacy_obj), fmt::format("{}", obj));
+
+    if (dbg) {
+      std::cout << "ostream: legacy: " << legacy_obj << " . Now: " << obj
+		<< std::endl;
+    }
+    std::ostringstream oss;
+    oss << obj;
+    std::ostringstream oss_legacy;
+    oss_legacy << legacy_obj;
+    EXPECT_EQ(oss_legacy.str(), oss.str());
+    EXPECT_EQ(oss.str(), fmt::format("{}", obj));
+  }
+}

From 8d766c57173cf6bdea0d8fcfa8fa3c1b61fd10ef Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Thu, 22 Feb 2024 10:41:43 +0800
Subject: [PATCH 1990/2492] doc/crimson: cleanup duplicate seastore description

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 doc/dev/crimson/crimson.rst | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/dev/crimson/crimson.rst b/doc/dev/crimson/crimson.rst
index cbc20b773185..421c2102ab44 100644
--- a/doc/dev/crimson/crimson.rst
+++ b/doc/dev/crimson/crimson.rst
@@ -195,9 +195,6 @@ The following options aree handy when using ``vstart.sh``,
     Valid types include ``HDD``, ``SSD``(default), ``ZNS``, and ``RANDOM_BLOCK_SSD``
     Note secondary devices should not be faster than the main device.
 
-``--seastore``
-    Use SeaStore as the object store backend.
-
 To start a cluster with a single Crimson node, run::
 
   $  MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x \

From 3f21557b63f34ede803cfd6d9af2c52bfac43f02 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 9 Jan 2024 10:45:36 +0530
Subject: [PATCH 1991/2492] mgr/dashboard: multi-cluster management in ceph
 dashboard

Fixes: https://tracker.ceph.com/issues/64530
Signed-off-by: Nizamudeen A <nia@redhat.com>
Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/auth.py  |  54 +++++-
 .../dashboard/controllers/multi_cluster.py    | 181 ++++++++++++++++++
 .../frontend/src/app/app-routing.module.ts    |   5 +
 .../src/app/ceph/cluster/cluster.module.ts    |   6 +-
 .../multi-cluster-form.component.html         | 160 ++++++++++++++++
 .../multi-cluster-form.component.scss         |   0
 .../multi-cluster-form.component.spec.ts      |  39 ++++
 .../multi-cluster-form.component.ts           | 152 +++++++++++++++
 .../multi-cluster.component.html              |  61 ++++++
 .../multi-cluster.component.scss              |   7 +
 .../multi-cluster.component.spec.ts           |  25 +++
 .../multi-cluster/multi-cluster.component.ts  |  57 ++++++
 .../app/core/auth/login/login.component.ts    |   1 +
 .../workbench-layout.component.html           |   2 +-
 .../workbench-layout.component.ts             |   3 +
 .../navigation/navigation.component.html      |  47 ++++-
 .../navigation/navigation.component.scss      |  11 ++
 .../navigation/navigation.component.ts        |  76 ++++++++
 .../shared/api/multi-cluster.service.spec.ts  |  19 ++
 .../app/shared/api/multi-cluster.service.ts   |  69 +++++++
 .../components/modal/modal.component.scss     |   4 +
 .../src/app/shared/constants/app.constants.ts |   2 +
 .../services/api-interceptor.service.ts       |  86 ++++++++-
 src/pybind/mgr/dashboard/module.py            |  68 +------
 src/pybind/mgr/dashboard/openapi.yaml         | 153 +++++++++++++++
 src/pybind/mgr/dashboard/settings.py          |   4 +
 src/pybind/mgr/dashboard/tests/test_auth.py   |   5 +
 src/pybind/mgr/dashboard/tools.py             |  69 +++++++
 28 files changed, 1291 insertions(+), 75 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/controllers/multi_cluster.py
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts

diff --git a/src/pybind/mgr/dashboard/controllers/auth.py b/src/pybind/mgr/dashboard/controllers/auth.py
index 196f027b293e..e8bb4bbef8e1 100644
--- a/src/pybind/mgr/dashboard/controllers/auth.py
+++ b/src/pybind/mgr/dashboard/controllers/auth.py
@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 
 import http.cookies
+import json
 import logging
 import sys
 
+import cherrypy
+
 from .. import mgr
 from ..exceptions import InvalidCredentialsError, UserDoesNotExist
 from ..services.auth import AuthManager, JwtManager
@@ -34,17 +37,66 @@ class Auth(RESTController, ControllerAuthMixin):
     """
     Provide authenticates and returns JWT token.
     """
-
+    # pylint: disable=R0912
     def create(self, username, password):
         user_data = AuthManager.authenticate(username, password)
         user_perms, pwd_expiration_date, pwd_update_required = None, None, None
         max_attempt = Settings.ACCOUNT_LOCKOUT_ATTEMPTS
+        origin = cherrypy.request.headers.get('Origin', None)
+        try:
+            fsid = mgr.get('config')['fsid']
+        except KeyError:
+            fsid = ''
         if max_attempt == 0 or mgr.ACCESS_CTRL_DB.get_attempt(username) < max_attempt:
             if user_data:
                 user_perms = user_data.get('permissions')
                 pwd_expiration_date = user_data.get('pwdExpirationDate', None)
                 pwd_update_required = user_data.get('pwdUpdateRequired', False)
 
+            if isinstance(Settings.MULTICLUSTER_CONFIG, str):
+                try:
+                    item_to_dict = json.loads(Settings.MULTICLUSTER_CONFIG)
+                except json.JSONDecodeError:
+                    item_to_dict = {}
+                multicluster_config = item_to_dict.copy()
+            else:
+                multicluster_config = Settings.MULTICLUSTER_CONFIG.copy()
+            try:
+                if fsid in multicluster_config['config']:
+                    existing_entries = multicluster_config['config'][fsid]
+                    if not any(entry['user'] == username for entry in existing_entries):
+                        existing_entries.append({
+                            "name": fsid,
+                            "url": origin,
+                            "cluster_alias": "local-cluster",
+                            "user": username
+                        })
+                else:
+                    multicluster_config['config'][fsid] = [{
+                        "name": fsid,
+                        "url": origin,
+                        "cluster_alias": "local-cluster",
+                        "user": username
+                    }]
+
+            except KeyError:
+                multicluster_config = {
+                    'current_url': origin,
+                    'current_user': username,
+                    'hub_url': origin,
+                    'config': {
+                        fsid: [
+                            {
+                                "name": fsid,
+                                "url": origin,
+                                "cluster_alias": "local-cluster",
+                                "user": username
+                            }
+                        ]
+                    }
+                }
+            Settings.MULTICLUSTER_CONFIG = multicluster_config
+
             if user_perms is not None:
                 url_prefix = 'https' if mgr.get_localized_module_option('ssl') else 'http'
 
diff --git a/src/pybind/mgr/dashboard/controllers/multi_cluster.py b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
new file mode 100644
index 000000000000..d7acec22bebb
--- /dev/null
+++ b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+
+import json
+
+import requests
+
+from ..exceptions import DashboardException
+from ..security import Scope
+from ..settings import Settings
+from ..tools import configure_cors
+from . import APIDoc, APIRouter, CreatePermission, Endpoint, EndpointDoc, \
+    ReadPermission, RESTController, UIRouter, UpdatePermission
+
+
+@APIRouter('/multi-cluster', Scope.CONFIG_OPT)
+@APIDoc('Multi-cluster Management API', 'Multi-cluster')
+class MultiCluster(RESTController):
+    def _proxy(self, method, base_url, path, params=None, payload=None, verify=False,
+               token=None):
+        try:
+            if token:
+                headers = {
+                    'Accept': 'application/vnd.ceph.api.v1.0+json',
+                    'Authorization': 'Bearer ' + token,
+                }
+            else:
+                headers = {
+                    'Accept': 'application/vnd.ceph.api.v1.0+json',
+                    'Content-Type': 'application/json',
+                }
+            response = requests.request(method, base_url + path, params=params,
+                                        json=payload, verify=verify, headers=headers)
+        except Exception as e:
+            raise DashboardException(
+                "Could not reach {}, {}".format(base_url+path, e),
+                http_status_code=404,
+                component='dashboard')
+
+        try:
+            content = json.loads(response.content, strict=False)
+        except json.JSONDecodeError as e:
+            raise DashboardException(
+                "Error parsing Dashboard API response: {}".format(e.msg),
+                component='dashboard')
+        return content
+
+    @Endpoint('POST')
+    @CreatePermission
+    @EndpointDoc("Authenticate to a remote cluster")
+    def auth(self, url: str, cluster_alias: str, username=None,
+             password=None, token=None, hub_url=None):
+
+        multi_cluster_config = self.load_multi_cluster_config()
+
+        if not url.endswith('/'):
+            url = url + '/'
+
+        if username and password:
+            payload = {
+                'username': username,
+                'password': password
+            }
+            content = self._proxy('POST', url, 'api/auth', payload=payload)
+            if 'token' not in content:
+                raise DashboardException(
+                    "Could not authenticate to remote cluster",
+                    http_status_code=400,
+                    component='dashboard')
+
+            token = content['token']
+
+        if token:
+            self._proxy('PUT', url, 'ui-api/multi-cluster/set_cors_endpoint',
+                        payload={'url': hub_url}, token=token)
+            fsid = self._proxy('GET', url, 'api/health/get_cluster_fsid', token=token)
+            content = self._proxy('POST', url, 'api/auth/check', payload={'token': token},
+                                  token=token)
+            if 'username' in content:
+                username = content['username']
+
+            if 'config' not in multi_cluster_config:
+                multi_cluster_config['config'] = {}
+
+            if fsid in multi_cluster_config['config']:
+                existing_entries = multi_cluster_config['config'][fsid]
+                if not any(entry['user'] == username for entry in existing_entries):
+                    existing_entries.append({
+                        "name": fsid,
+                        "url": url,
+                        "cluster_alias": cluster_alias,
+                        "user": username,
+                        "token": token,
+                    })
+            else:
+                multi_cluster_config['current_user'] = username
+                multi_cluster_config['config'][fsid] = [{
+                    "name": fsid,
+                    "url": url,
+                    "cluster_alias": cluster_alias,
+                    "user": username,
+                    "token": token,
+                }]
+
+            Settings.MULTICLUSTER_CONFIG = multi_cluster_config
+
+    def load_multi_cluster_config(self):
+        if isinstance(Settings.MULTICLUSTER_CONFIG, str):
+            try:
+                itemw_to_dict = json.loads(Settings.MULTICLUSTER_CONFIG)
+            except json.JSONDecodeError:
+                itemw_to_dict = {}
+            multi_cluster_config = itemw_to_dict.copy()
+        else:
+            multi_cluster_config = Settings.MULTICLUSTER_CONFIG.copy()
+
+        return multi_cluster_config
+
+    @Endpoint('PUT')
+    @UpdatePermission
+    def set_config(self, config: object):
+        multicluster_config = self.load_multi_cluster_config()
+        multicluster_config.update({'current_url': config['url']})
+        multicluster_config.update({'current_user': config['user']})
+        Settings.MULTICLUSTER_CONFIG = multicluster_config
+        return Settings.MULTICLUSTER_CONFIG
+
+    @Endpoint('POST')
+    @CreatePermission
+    # pylint: disable=R0911
+    def verify_connection(self, url: str, username=None, password=None, token=None):
+        if not url.endswith('/'):
+            url = url + '/'
+
+        if token:
+            try:
+                payload = {
+                    'token': token
+                }
+                content = self._proxy('POST', url, 'api/auth/check', payload=payload)
+                if 'permissions' not in content:
+                    return content['detail']
+                user_content = self._proxy('GET', url, f'api/user/{username}',
+                                           token=content['token'])
+                if 'status' in user_content and user_content['status'] == '403 Forbidden':
+                    return 'User is not an administrator'
+            except Exception as e:  # pylint: disable=broad-except
+                if '[Errno 111] Connection refused' in str(e):
+                    return 'Connection refused'
+                return 'Connection failed'
+
+        if username and password:
+            try:
+                payload = {
+                    'username': username,
+                    'password': password
+                }
+                content = self._proxy('POST', url, 'api/auth', payload=payload)
+                if 'token' not in content:
+                    return content['detail']
+                user_content = self._proxy('GET', url, f'api/user/{username}',
+                                           token=content['token'])
+                if 'status' in user_content and user_content['status'] == '403 Forbidden':
+                    return 'User is not an administrator'
+            except Exception as e:  # pylint: disable=broad-except
+                if '[Errno 111] Connection refused' in str(e):
+                    return 'Connection refused'
+                return 'Connection failed'
+        return 'Connection successful'
+
+    @Endpoint()
+    @ReadPermission
+    def get_config(self):
+        return Settings.MULTICLUSTER_CONFIG
+
+
+@UIRouter('/multi-cluster', Scope.CONFIG_OPT)
+class MultiClusterUi(RESTController):
+    @Endpoint('PUT')
+    @UpdatePermission
+    def set_cors_endpoint(self, url: str):
+        configure_cors(url)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
index 2ba634fa25d0..48224c844d47 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
@@ -48,6 +48,7 @@ import { NoSsoGuardService } from './shared/services/no-sso-guard.service';
 import { UpgradeComponent } from './ceph/cluster/upgrade/upgrade.component';
 import { CephfsVolumeFormComponent } from './ceph/cephfs/cephfs-form/cephfs-form.component';
 import { UpgradeProgressComponent } from './ceph/cluster/upgrade/upgrade-progress/upgrade-progress.component';
+import { MultiClusterComponent } from './ceph/cluster/multi-cluster/multi-cluster.component';
 
 @Injectable()
 export class PerformanceCounterBreadcrumbsResolver extends BreadcrumbsResolver {
@@ -184,6 +185,10 @@ const routes: Routes = [
           }
         ]
       },
+      {
+        path: 'multi-cluster',
+        component: MultiClusterComponent
+      },
       {
         path: 'inventory',
         canActivate: [ModuleStatusGuardService],
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
index 74657ec4010f..b1eb9275a462 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
@@ -61,6 +61,8 @@ import { TelemetryComponent } from './telemetry/telemetry.component';
 import { UpgradeComponent } from './upgrade/upgrade.component';
 import { UpgradeStartModalComponent } from './upgrade/upgrade-form/upgrade-start-modal.component';
 import { UpgradeProgressComponent } from './upgrade/upgrade-progress/upgrade-progress.component';
+import { MultiClusterComponent } from './multi-cluster/multi-cluster.component';
+import { MultiClusterFormComponent } from './multi-cluster/multi-cluster-form/multi-cluster-form.component';
 
 @NgModule({
   imports: [
@@ -124,7 +126,9 @@ import { UpgradeProgressComponent } from './upgrade/upgrade-progress/upgrade-pro
     CreateClusterReviewComponent,
     UpgradeComponent,
     UpgradeStartModalComponent,
-    UpgradeProgressComponent
+    UpgradeProgressComponent,
+    MultiClusterComponent,
+    MultiClusterFormComponent
   ],
   providers: [NgbActiveModal]
 })
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
new file mode 100644
index 000000000000..cc9ed7453fc4
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
@@ -0,0 +1,160 @@
+<cd-modal [modalRef]="activeModal">
+  <ng-container i18n="form title"
+                class="modal-title">Connect Cluster
+  </ng-container>
+  <ng-container class="modal-content">
+    <form name="remoteClusterForm"
+          #frm="ngForm"
+          [formGroup]="remoteClusterForm">
+      <div class="modal-body">
+        <cd-alert-panel *ngIf="connectionVerified !== undefined && !connectionVerified && connectionMessage !== 'Connection refused'"
+                        type="error"
+                        spacingClass="mb-3"
+                        i18n>{{ connectionMessage }}
+        </cd-alert-panel>
+        <cd-alert-panel *ngIf="connectionVerified !== undefined && connectionVerified"
+                        type="success"
+                        spacingClass="mb-3"
+                        i18n>{{ connectionMessage }}
+        </cd-alert-panel>
+        <cd-alert-panel type="info"
+                        spacingClass="mb-3"
+                        i18n
+                        *ngIf="connectionVerified !== undefined && !connectionVerified && connectionMessage === 'Connection refused'">
+        <p>You need to set this cluster's url as the cross origin url in the remote cluster you are trying to connect.
+          You can do it by running this CLI command in your remote cluster and proceed with authentication via token.</p>
+          <cd-code-block [codes]="[crossOriginCmd]"></cd-code-block>
+        </cd-alert-panel>
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="remoteClusterUrl"
+                 i18n>Cluster API URL
+            <cd-helper>Enter the Dashboard API URL. You can retrieve it from the CLI with: <b>ceph mgr services</b></cd-helper>
+          </label>
+          <div class="cd-col-form-input">
+            <input class="form-control"
+                   type="text"
+                   placeholder="https://localhost:4202"
+                   id="remoteClusterUrl"
+                   name="remoteClusterUrl"
+                   formControlName="remoteClusterUrl">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('remoteClusterUrl', frm, 'required')"
+                  i18n>This field is required.
+            </span>
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('remoteClusterUrl', frm, 'endpoint')"
+                  i18n>Please enter a valid URL.
+            </span>
+          </div>
+        </div>
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="clusterAlias"
+                 i18n>Alias Name
+          </label>
+          <div class="cd-col-form-input">
+            <input id="clusterAlias"
+                   name="clusterAlias"
+                   class="form-control"
+                   type="text"
+                   placeholder="Name/Text to uniquely identify cluster"
+                   formControlName="clusterAlias">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('clusterAlias', frm, 'required')"
+                  i18n>This field is required.
+            </span>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError">
+          <label class="cd-col-form-label required"
+                 for="apiToken"
+                 i18n>Username
+          </label>
+          <div class="cd-col-form-input">
+            <input id="username"
+                   name="username"
+                   class="form-control"
+                   type="text"
+                   formControlName="username">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('username', frm, 'required')"
+                  i18n>This field is required.
+            </span>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError">
+          <label class="cd-col-form-label required"
+                 for="password"
+                 i18n>Password
+          </label>
+          <div class="cd-col-form-input">
+            <input id="password"
+                   name="password"
+                   class="form-control"
+                   type="password"
+                   formControlName="password">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('password', frm, 'required')"
+                  i18n>This field is required.
+            </span>
+          </div>
+        </div>
+        <div class="form-group row">
+          <div class="cd-col-form-offset">
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="showToken"
+                     type="checkbox"
+                     (click)="showToken = !showToken"
+                     formControlName="showToken"
+                     [readonly]="true">
+              <label class="custom-control-label"
+                     for="showToken"
+                     i18n>Auth with token</label>
+            </div>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="remoteClusterForm.getValue('showToken')">
+          <label class="cd-col-form-label required"
+                 for="apiToken"
+                 i18n>Token
+          </label>
+          <div class="cd-col-form-input">
+            <input id="apiToken"
+                   name="apiToken"
+                   class="form-control"
+                   type="text"
+                   formControlName="apiToken">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('apiToken', frm, 'required')"
+                  i18n>This field is required.</span>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="!showCrossOriginError">
+          <div class="cd-col-form-offset">
+            <div class="custom-control">
+              <button class="btn btn-primary"
+                      type="button"
+                      [disabled]="(remoteClusterForm.getValue('showToken') && remoteClusterForm.getValue('apiToken') === '') || (!remoteClusterForm.getValue('showToken') && (remoteClusterForm.getValue('username') === '' || remoteClusterForm.getValue('password') === ''))"
+                      (click)="verifyConnection()">
+                Verify Connection
+              </button>
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="modal-footer">
+        <cd-form-button-panel (submitActionEvent)="onSubmit()"
+                              [submitText]="actionLabels.CONNECT"
+                              [disabled]="!connectionVerified && !showCrossOriginError"
+                              [form]="remoteClusterForm">
+        </cd-form-button-panel>
+      </div>
+    </form>
+  </ng-container>
+</cd-modal>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.spec.ts
new file mode 100644
index 000000000000..71521de56f11
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.spec.ts
@@ -0,0 +1,39 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { MultiClusterFormComponent } from './multi-cluster-form.component';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { ToastrModule } from 'ngx-toastr';
+import { NotificationService } from '~/app/shared/services/notification.service';
+import { CdDatePipe } from '~/app/shared/pipes/cd-date.pipe';
+import { DatePipe } from '@angular/common';
+import { ReactiveFormsModule } from '@angular/forms';
+import { RouterTestingModule } from '@angular/router/testing';
+import { SharedModule } from '~/app/shared/shared.module';
+
+describe('MultiClusterFormComponent', () => {
+  let component: MultiClusterFormComponent;
+  let fixture: ComponentFixture<MultiClusterFormComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [
+        SharedModule,
+        ReactiveFormsModule,
+        RouterTestingModule,
+        HttpClientTestingModule,
+        ToastrModule.forRoot()
+      ],
+      declarations: [MultiClusterFormComponent],
+      providers: [NgbActiveModal, NotificationService, CdDatePipe, DatePipe]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(MultiClusterFormComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
new file mode 100644
index 000000000000..473a49dab7f7
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
@@ -0,0 +1,152 @@
+import { Component, OnDestroy, OnInit } from '@angular/core';
+import { FormControl, Validators } from '@angular/forms';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import _ from 'lodash';
+import { Subscription } from 'rxjs';
+import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { NotificationType } from '~/app/shared/enum/notification-type.enum';
+import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
+import { CdValidators } from '~/app/shared/forms/cd-validators';
+import { NotificationService } from '~/app/shared/services/notification.service';
+
+@Component({
+  selector: 'cd-multi-cluster-form',
+  templateUrl: './multi-cluster-form.component.html',
+  styleUrls: ['./multi-cluster-form.component.scss']
+})
+export class MultiClusterFormComponent implements OnInit, OnDestroy {
+  readonly endpoints = /^((https?:\/\/)|(www.))(?:([a-zA-Z]+)|(\d+\.\d+.\d+.\d+)):\d{2,5}\/?$/;
+  readonly ipv4Rgx = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/i;
+  readonly ipv6Rgx = /^(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4}$/i;
+  remoteClusterForm: CdFormGroup;
+  showToken = false;
+  connectionVerified: boolean;
+  connectionMessage = '';
+  private subs = new Subscription();
+  showCrossOriginError = false;
+  crossOriginCmd: string;
+
+  constructor(
+    public activeModal: NgbActiveModal,
+    public actionLabels: ActionLabelsI18n,
+    public notificationService: NotificationService,
+    private multiClusterService: MultiClusterService
+  ) {
+    this.createForm();
+  }
+  ngOnInit(): void {}
+
+  createForm() {
+    this.remoteClusterForm = new CdFormGroup({
+      showToken: new FormControl(false),
+      username: new FormControl('', [
+        CdValidators.requiredIf({
+          showToken: false
+        })
+      ]),
+      password: new FormControl('', [
+        CdValidators.requiredIf({
+          showToken: false
+        })
+      ]),
+      remoteClusterUrl: new FormControl(null, {
+        validators: [
+          CdValidators.custom('endpoint', (value: string) => {
+            if (_.isEmpty(value)) {
+              return false;
+            } else {
+              return (
+                !this.endpoints.test(value) &&
+                !this.ipv4Rgx.test(value) &&
+                !this.ipv6Rgx.test(value)
+              );
+            }
+          }),
+          Validators.required
+        ]
+      }),
+      apiToken: new FormControl('', [
+        CdValidators.requiredIf({
+          showToken: true
+        })
+      ]),
+      clusterAlias: new FormControl('', {
+        validators: [Validators.required]
+      })
+    });
+  }
+
+  ngOnDestroy() {
+    this.subs.unsubscribe();
+  }
+
+  onSubmit() {
+    const url = this.remoteClusterForm.getValue('remoteClusterUrl');
+    const clusterAlias = this.remoteClusterForm.getValue('clusterAlias');
+    const username = this.remoteClusterForm.getValue('username');
+    const password = this.remoteClusterForm.getValue('password');
+    const token = this.remoteClusterForm.getValue('apiToken');
+
+    this.subs.add(
+      this.multiClusterService
+        .addCluster(url, clusterAlias, username, password, token, window.location.origin)
+        .subscribe({
+          error: () => {
+            this.remoteClusterForm.setErrors({ cdSubmitButton: true });
+          },
+          complete: () => {
+            this.notificationService.show(
+              NotificationType.success,
+              $localize`Cluster added successfully`
+            );
+            this.activeModal.close();
+          }
+        })
+    );
+  }
+
+  verifyConnection() {
+    const url = this.remoteClusterForm.getValue('remoteClusterUrl');
+    const username = this.remoteClusterForm.getValue('username');
+    const password = this.remoteClusterForm.getValue('password');
+    const token = this.remoteClusterForm.getValue('apiToken');
+
+    this.subs.add(
+      this.multiClusterService
+        .verifyConnection(url, username, password, token)
+        .subscribe((resp: string) => {
+          switch (resp) {
+            case 'Connection successful':
+              this.connectionVerified = true;
+              this.connectionMessage = 'Connection Verified Successfully';
+              this.notificationService.show(
+                NotificationType.success,
+                $localize`Connection Verified Successfully`
+              );
+              break;
+
+            case 'Connection refused':
+              this.connectionVerified = false;
+              this.showCrossOriginError = true;
+              this.connectionMessage = resp;
+              this.crossOriginCmd = `ceph config set mgr mgr/dashboard/cross_origin_url ${window.location.origin} `;
+              this.notificationService.show(
+                NotificationType.error,
+                $localize`Connection to the cluster failed`
+              );
+              break;
+
+            default:
+              this.connectionVerified = false;
+              this.connectionMessage = resp;
+              this.notificationService.show(
+                NotificationType.error,
+                $localize`Connection to the cluster failed`
+              );
+              break;
+          }
+        })
+    );
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
new file mode 100644
index 000000000000..5009909ea3fa
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
@@ -0,0 +1,61 @@
+<ng-template #emptyCluster>
+  <div class="container h-75">
+    <div class="row h-100 justify-content-center align-items-center">
+      <div class="blank-page">
+        <i class="mx-auto d-block"
+           [ngClass]="icons.wrench">
+        </i>
+      <div class="mt-4 text-center">
+        <h3><b>Connect Cluster </b></h3>
+        <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortlessly.
+            Click on the "Connect Cluster" button to begin the process.</h4>
+      </div>
+      <div class="mt-4">
+        <div class="text-center">
+          <button class="btn btn-primary"
+                  (click)="openRemoteClusterInfoModal()">
+            <i class="mx-auto"
+               [ngClass]="icons.add">
+            </i> Connect Cluster
+          </button>
+        </div>
+      </div>
+      </div>
+    </div>
+  </div>
+</ng-template>
+
+<ng-template #nametpl>
+  <div class="datatable-body-cell-label">
+    <span title="{{cluster}}">
+      <a href="#">
+        {{cluster}}
+      </a>
+    </span>
+  </div>
+</ng-template>
+
+<div class="container-fluid h-100 p-4">
+  <div *ngIf="dashboardClustersMap?.size === 1">
+    <ng-container *ngTemplateOutlet="emptyCluster"></ng-container>
+  </div>
+
+  <span *ngIf="loading"
+        class="d-flex justify-content-center">
+    <i [ngClass]="[icons.large3x, icons.spinner, icons.spin]"></i>
+  </span>
+  <div *ngIf="dashboardClustersMap?.size > 1">
+    <div *ngIf="!loading">
+      <div class="mt-4">
+        <div class="text-center">
+          <button class="btn btn-primary"
+                  (click)="openRemoteClusterInfoModal()">
+            <i class="mx-auto"
+               [ngClass]="icons.add">
+            </i> Connect Cluster
+          </button>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.scss
new file mode 100644
index 000000000000..2931ef94fba2
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.scss
@@ -0,0 +1,7 @@
+@use '../../../../styles/vendor/variables' as vv;
+
+.fa-wrench {
+  color: vv.$info;
+  font-size: 6em;
+  margin-top: 200px;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
new file mode 100644
index 000000000000..8db81cd790fc
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { MultiClusterComponent } from './multi-cluster.component';
+
+describe('MultiClusterComponent', () => {
+  let component: MultiClusterComponent;
+  let fixture: ComponentFixture<MultiClusterComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [HttpClientTestingModule],
+      declarations: [MultiClusterComponent],
+      providers: [NgbActiveModal]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(MultiClusterComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
new file mode 100644
index 000000000000..2630c839a424
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
@@ -0,0 +1,57 @@
+import { Component, OnInit, ViewChild } from '@angular/core';
+import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
+import { Subscription } from 'rxjs';
+import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { ModalService } from '~/app/shared/services/modal.service';
+import { MultiClusterFormComponent } from './multi-cluster-form/multi-cluster-form.component';
+
+@Component({
+  selector: 'cd-multi-cluster',
+  templateUrl: './multi-cluster.component.html',
+  styleUrls: ['./multi-cluster.component.scss']
+})
+export class MultiClusterComponent implements OnInit {
+  @ViewChild('nameTpl', { static: true })
+  nameTpl: any;
+
+  private subs = new Subscription();
+  dashboardClustersMap: Map<string, string> = new Map<string, string>();
+  icons = Icons;
+  loading = true;
+  bsModalRef: NgbModalRef;
+
+  constructor(
+    private multiClusterService: MultiClusterService,
+    private modalService: ModalService
+  ) {}
+
+  ngOnInit(): void {
+    this.subs.add(
+      this.multiClusterService.subscribe((resp: any) => {
+        const clustersConfig = resp['config'];
+        if (clustersConfig) {
+          Object.keys(clustersConfig).forEach((clusterKey: string) => {
+            const clusterDetailsList = clustersConfig[clusterKey];
+
+            clusterDetailsList.forEach((clusterDetails: any) => {
+              const clusterUrl = clusterDetails['url'];
+              const clusterName = clusterDetails['name'];
+              this.dashboardClustersMap.set(clusterUrl, clusterName);
+            });
+          });
+
+          if (this.dashboardClustersMap.size >= 1) {
+            this.loading = false;
+          }
+        }
+      })
+    );
+  }
+
+  openRemoteClusterInfoModal() {
+    this.bsModalRef = this.modalService.show(MultiClusterFormComponent, {
+      size: 'xl'
+    });
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts
index a98548f94c76..57039c0f6d0c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts
@@ -64,6 +64,7 @@ export class LoginComponent implements OnInit {
   }
 
   login() {
+    localStorage.setItem('cluster_api_url', window.location.origin);
     this.authService.login(this.model).subscribe(() => {
       const urlPath = this.postInstalled ? '/' : '/expand-cluster';
       let url = _.get(this.route.snapshot.queryParams, 'returnUrl', urlPath);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
index fe3bfc6acf9e..1c1846dae15e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
@@ -1,7 +1,7 @@
 <block-ui>
   <cd-navigation>
     <div class="container-fluid h-100"
-         [ngClass]="{'dashboard': (router.url == '/dashboard' || router.url == '/dashboard_3'), 'rgw-dashboard': (router.url == '/rgw/overview')}">
+         [ngClass]="{'dashboard': (router.url == '/dashboard' || router.url == '/dashboard_3' || router.url == '/multi-cluster'), 'rgw-dashboard': (router.url == '/rgw/overview')}">
     <cd-context></cd-context>
       <cd-breadcrumbs></cd-breadcrumbs>
       <router-outlet></router-outlet>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
index afc7a83bb277..1d7c4bb751cb 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
@@ -2,6 +2,7 @@ import { Component, OnDestroy, OnInit } from '@angular/core';
 import { Router } from '@angular/router';
 
 import { Subscription } from 'rxjs';
+import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
 
 import { FaviconService } from '~/app/shared/services/favicon.service';
 import { SummaryService } from '~/app/shared/services/summary.service';
@@ -20,10 +21,12 @@ export class WorkbenchLayoutComponent implements OnInit, OnDestroy {
     public router: Router,
     private summaryService: SummaryService,
     private taskManagerService: TaskManagerService,
+    private multiClusterService: MultiClusterService,
     private faviconService: FaviconService
   ) {}
 
   ngOnInit() {
+    this.subs.add(this.multiClusterService.startPolling());
     this.subs.add(this.summaryService.startPolling());
     this.subs.add(this.taskManagerService.init(this.summaryService));
     this.faviconService.init();
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
index abd2e1ae6506..6af3799b4ef8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
@@ -30,6 +30,29 @@
 
       <div class="collapse navbar-collapse"
            [ngClass]="{'show': rightSidebarOpen}">
+        <ng-container *ngIf="clustersMap?.size > 1">
+          <div ngbDropdown
+               placement="bottom-left"
+               class="d-inline-block ms-5">
+            <button ngbDropdownToggle
+                    class="btn btn-outline-light cd-context-bar"
+                    i18n-title
+                    title="Selected Cluster:">
+              <span class="dropdown-text"> {{ selectedCluster?.name }} </span>
+              <span>- {{ selectedCluster?.cluster_alias }} - {{ selectedCluster?.user }}</span>
+            </button>
+            <div ngbDropdownMenu>
+              <ng-container *ngFor="let cluster of clustersMap | keyvalue">
+                <button ngbDropdownItem
+                        (click)="onClusterSelection(cluster.value)">
+                  <div class="dropdown-text">{{ cluster.value.name }}</div>
+                  <div *ngIf="cluster.value.cluster_alias"
+                       class="text-secondary">{{ cluster.value.cluster_alias }} - {{ cluster.value.user }}</div>
+                </button>
+              </ng-container>
+            </div>
+          </div>
+        </ng-container>
         <ul class="nav navbar-nav cd-navbar-utility my-2 my-md-0">
           <ng-container *ngTemplateOutlet="cd_utilities"> </ng-container>
         </ul>
@@ -91,7 +114,29 @@
           </i>
         </a>
       </li>
-
+      <!-- Multi-cluster Dashboard -->
+      <li routerLinkActive="active"
+          class="nav-item tc_menuitem_multi_cluster">
+        <a (click)="toggleSubMenu('multiCluster')"
+           class="nav-link dropdown-toggle"
+           [attr.aria-expanded]="displayedSubMenu.multiCluster"
+           aria-controls="multi-cluster-nav"
+           role="button">
+          <ng-container i18n>
+            <i [ngClass]="[icons.sitemap]"></i>
+              Multi-Cluster
+          </ng-container>
+        </a>
+        <ul class="list-unstyled"
+            id="multi-cluster-nav"
+            [ngbCollapse]="!displayedSubMenu.multiCluster">
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_multiCluster_overview">
+            <a i18n
+               routerLink="/multi-cluster">Overview</a>
+          </li>
+        </ul>
+      </li>
       <!-- Cluster -->
       <li routerLinkActive="active"
           class="nav-item tc_menuitem_cluster"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
index 8e0dfa364cd9..4d2c829e93da 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.scss
@@ -143,6 +143,17 @@
       background-color: vv.$primary;
     }
   }
+
+  .cd-context-bar {
+    background-color: vv.$white;
+    color: vv.$secondary;
+    cursor: pointer;
+  }
+
+  .dropdown-text {
+    font-size: small;
+    font-weight: 600;
+  }
 }
 
 /* ---------------------------------------------------
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index e5615012e370..10963042d25f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -1,7 +1,9 @@
 import { Component, HostBinding, OnDestroy, OnInit } from '@angular/core';
+import { Router } from '@angular/router';
 
 import * as _ from 'lodash';
 import { Subscription } from 'rxjs';
+import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
 
 import { Icons } from '~/app/shared/enum/icons.enum';
 import { Permissions } from '~/app/shared/models/permissions';
@@ -22,6 +24,7 @@ import { TelemetryNotificationService } from '~/app/shared/services/telemetry-no
 })
 export class NavigationComponent implements OnInit, OnDestroy {
   notifications: string[] = [];
+  clusterDetails: any[] = [];
   @HostBinding('class') get class(): string {
     return 'top-notification-' + this.notifications.length;
   }
@@ -40,8 +43,13 @@ export class NavigationComponent implements OnInit, OnDestroy {
   displayedSubMenu = {};
   private subs = new Subscription();
 
+  clustersMap: Map<string, any> = new Map<string, any>();
+  selectedCluster: object;
+
   constructor(
     private authStorageService: AuthStorageService,
+    private multiClusterService: MultiClusterService,
+    private router: Router,
     private summaryService: SummaryService,
     private featureToggles: FeatureTogglesService,
     private telemetryNotificationService: TelemetryNotificationService,
@@ -53,6 +61,31 @@ export class NavigationComponent implements OnInit, OnDestroy {
   }
 
   ngOnInit() {
+    this.subs.add(
+      this.multiClusterService.subscribe((resp: any) => {
+        const clustersConfig = resp['config'];
+        if (clustersConfig) {
+          Object.keys(clustersConfig).forEach((clusterKey: string) => {
+            const clusterDetailsList = clustersConfig[clusterKey];
+            clusterDetailsList.forEach((clusterDetails: any) => {
+              const clusterName = clusterDetails['name'];
+              const clusterUser = clusterDetails['user'];
+              const clusterUrl = clusterDetails['url'];
+              const clusterUniqueKey = `${clusterUrl}-${clusterUser}`;
+              this.clustersMap.set(clusterUniqueKey, {
+                name: clusterName,
+                cluster_alias: clusterDetails['cluster_alias'],
+                user: clusterDetails['user'],
+                url: clusterUrl
+              });
+            });
+          });
+          this.selectedCluster =
+            this.clustersMap.get(`${resp['current_url']}-${resp['current_user']}`) || {};
+        }
+      })
+    );
+
     this.subs.add(
       this.summaryService.subscribe((summary) => {
         this.summaryData = summary;
@@ -116,4 +149,47 @@ export class NavigationComponent implements OnInit, OnDestroy {
       }
     }
   }
+
+  onClusterSelection(value: object) {
+    this.multiClusterService.setCluster(value).subscribe(
+      (resp: any) => {
+        localStorage.setItem('cluster_api_url', value['url']);
+        this.selectedCluster = this.clustersMap.get(`${value['url']}-${value['user']}`) || {};
+        const clustersConfig = resp['config'];
+        if (clustersConfig && typeof clustersConfig === 'object') {
+          Object.keys(clustersConfig).forEach((clusterKey: string) => {
+            const clusterDetailsList = clustersConfig[clusterKey];
+
+            clusterDetailsList.forEach((clusterDetails: any) => {
+              const clusterName = clusterDetails['name'];
+              const clusterToken = clusterDetails['token'];
+              const clusterUser = clusterDetails['user'];
+
+              if (
+                clusterName === this.selectedCluster['name'] &&
+                clusterUser === this.selectedCluster['user']
+              ) {
+                localStorage.setItem('token_of_selected_cluster', clusterToken);
+              }
+            });
+          });
+        }
+      },
+      () => {},
+      () => {
+        this.multiClusterService.refresh();
+        this.summaryService.refresh();
+        const currentRoute = this.router.url.split('?')[0];
+        if (currentRoute.includes('dashboard')) {
+          this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
+            this.router.navigate([currentRoute]);
+          });
+        } else {
+          this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
+            this.router.navigate([currentRoute]);
+          });
+        }
+      }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.spec.ts
new file mode 100644
index 000000000000..88140fe769ea
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.spec.ts
@@ -0,0 +1,19 @@
+import { TestBed } from '@angular/core/testing';
+
+import { MultiClusterService } from './multi-cluster.service';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+
+describe('MultiClusterService', () => {
+  let service: MultiClusterService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({
+      imports: [HttpClientTestingModule]
+    });
+    service = TestBed.inject(MultiClusterService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
new file mode 100644
index 000000000000..5a17645092df
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
@@ -0,0 +1,69 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { BehaviorSubject, Subscription } from 'rxjs';
+import { TimerService } from '../services/timer.service';
+import { filter } from 'rxjs/operators';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class MultiClusterService {
+  private msSource = new BehaviorSubject<any>(null);
+  msData$ = this.msSource.asObservable();
+  constructor(private http: HttpClient, private timerService: TimerService) {}
+
+  startPolling(): Subscription {
+    return this.timerService
+      .get(() => this.getCluster(), 5000)
+      .subscribe(this.getClusterObserver());
+  }
+
+  refresh(): Subscription {
+    return this.getCluster().subscribe(this.getClusterObserver());
+  }
+
+  subscribe(next: (data: any) => void, error?: (error: any) => void) {
+    return this.msData$.pipe(filter((value) => !!value)).subscribe(next, error);
+  }
+
+  setCluster(cluster: object) {
+    return this.http.put('api/multi-cluster/set_config', { config: cluster });
+  }
+
+  getCluster() {
+    return this.http.get('api/multi-cluster/get_config');
+  }
+
+  addCluster(
+    url: any,
+    clusterAlias: string,
+    username: string,
+    password: string,
+    token = '',
+    hub_url = ''
+  ) {
+    return this.http.post('api/multi-cluster/auth', {
+      url,
+      cluster_alias: clusterAlias,
+      username,
+      password,
+      token,
+      hub_url
+    });
+  }
+
+  verifyConnection(url: string, username: string, password: string, token = '') {
+    return this.http.post('api/multi-cluster/verify_connection', {
+      url,
+      username,
+      password,
+      token
+    });
+  }
+
+  private getClusterObserver() {
+    return (data: any) => {
+      this.msSource.next(data);
+    };
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.scss
index ceeb614273b9..1681472cf859 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.scss
@@ -16,6 +16,10 @@
     overflow-x: hidden;
     overflow-y: auto;
   }
+
+  .modal-content {
+    display: table;
+  }
 }
 
 button.close {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index fc32efd1d38a..df007c93d8a6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -144,6 +144,7 @@ export class ActionLabelsI18n {
   ACTIVATE: string;
   DEACTIVATE: string;
   ATTACH: string;
+  CONNECT: string;
 
   constructor() {
     /* Create a new item */
@@ -226,6 +227,7 @@ export class ActionLabelsI18n {
     this.DEACTIVATE = $localize`Deactivate`;
 
     this.ATTACH = $localize`Attach`;
+    this.CONNECT = $localize`Connect`;
   }
 }
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
index fb7a9f73395f..e4e30d6a3682 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
@@ -16,8 +16,10 @@ import { CdHelperClass } from '~/app/shared/classes/cd-helper.class';
 import { NotificationType } from '../enum/notification-type.enum';
 import { CdNotificationConfig } from '../models/cd-notification';
 import { FinishedTask } from '../models/finished-task';
-import { AuthStorageService } from './auth-storage.service';
 import { NotificationService } from './notification.service';
+import { MultiClusterService } from '../api/multi-cluster.service';
+import { SummaryService } from './summary.service';
+import { AuthStorageService } from './auth-storage.service';
 
 export class CdHttpErrorResponse extends HttpErrorResponse {
   preventDefault: Function;
@@ -28,15 +30,42 @@ export class CdHttpErrorResponse extends HttpErrorResponse {
   providedIn: 'root'
 })
 export class ApiInterceptorService implements HttpInterceptor {
+  localClusterDetails: object;
+  dashboardClustersMap: Map<string, string> = new Map<string, string>();
   constructor(
     private router: Router,
+    public notificationService: NotificationService,
+    private summaryService: SummaryService,
     private authStorageService: AuthStorageService,
-    public notificationService: NotificationService
-  ) {}
+    private multiClusterService: MultiClusterService
+  ) {
+    this.multiClusterService.subscribe((resp: any) => {
+      const clustersConfig = resp['config'];
+      const hub_url = resp['hub_url'];
+      if (clustersConfig) {
+        Object.keys(clustersConfig).forEach((clusterKey: string) => {
+          const clusterDetailsList = clustersConfig[clusterKey];
+
+          clusterDetailsList.forEach((clusterDetails: any) => {
+            const clusterUrl = clusterDetails['url'];
+            const clusterName = clusterDetails['name'];
+
+            this.dashboardClustersMap.set(clusterUrl, clusterName);
+
+            if (clusterDetails['url'] === hub_url) {
+              this.localClusterDetails = clusterDetails;
+            }
+          });
+        });
+      }
+    });
+  }
 
   intercept(request: HttpRequest<any>, next: HttpHandler): Observable<HttpEvent<any>> {
     const acceptHeader = request.headers.get('Accept');
     let reqWithVersion: HttpRequest<any>;
+
+    const origin = window.location.origin;
     if (acceptHeader && acceptHeader.startsWith('application/vnd.ceph.api.v')) {
       reqWithVersion = request.clone();
     } else {
@@ -46,6 +75,35 @@ export class ApiInterceptorService implements HttpInterceptor {
         }
       });
     }
+
+    const apiUrl = localStorage.getItem('cluster_api_url');
+    const currentRoute = this.router.url.split('?')[0];
+
+    const ALWAYS_TO_HUB_APIs = [
+      'api/auth/login',
+      'api/auth/logout',
+      'api/multi-cluster/get_config',
+      'api/multi-cluster/set_config',
+      'api/multi-cluster/auth'
+    ];
+
+    const token = localStorage.getItem('token_of_selected_cluster');
+
+    if (
+      !currentRoute.includes('login') &&
+      !ALWAYS_TO_HUB_APIs.includes(request.url) &&
+      apiUrl &&
+      !apiUrl.includes(origin)
+    ) {
+      reqWithVersion = reqWithVersion.clone({
+        url: `${apiUrl}${reqWithVersion.url}`,
+        setHeaders: {
+          'Access-Control-Allow-Origin': origin,
+          Authorization: `Bearer ${token}`
+        }
+      });
+    }
+
     return next.handle(reqWithVersion).pipe(
       catchError((resp: CdHttpErrorResponse) => {
         if (resp instanceof HttpErrorResponse) {
@@ -69,8 +127,26 @@ export class ApiInterceptorService implements HttpInterceptor {
               timeoutId = this.notificationService.notifyTask(finishedTask);
               break;
             case 401:
-              this.authStorageService.remove();
-              this.router.navigate(['/login']);
+              if (this.dashboardClustersMap.size > 1) {
+                this.multiClusterService.setCluster(this.localClusterDetails).subscribe(() => {
+                  localStorage.setItem('cluster_api_url', this.localClusterDetails['url']);
+                });
+                this.multiClusterService.refresh();
+                this.summaryService.refresh();
+                const currentRoute = this.router.url.split('?')[0];
+                if (currentRoute.includes('dashboard')) {
+                  this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
+                    this.router.navigate([currentRoute]);
+                  });
+                } else {
+                  this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
+                    this.router.navigate([currentRoute]);
+                  });
+                }
+              } else {
+                this.authStorageService.remove();
+                this.router.navigate(['/login']);
+              }
               break;
             case 403:
               this.router.navigate(['error'], {
diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index 41160b698aae..837841976564 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -36,7 +36,7 @@
 from .services.sso import SSO_COMMANDS, handle_sso_command
 from .settings import handle_option_command, options_command_list, options_schema_list
 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
-    prepare_url_prefix, str_to_bool
+    configure_cors, prepare_url_prefix, str_to_bool
 
 try:
     import cherrypy
@@ -120,7 +120,7 @@ def _configure(self):
 
         # Initialize custom handlers.
         cherrypy.tools.authenticate = AuthManagerTool()
-        self.configure_cors()
+        configure_cors()
         cherrypy.tools.plugin_hooks_filter_request = cherrypy.Tool(
             'before_handler',
             lambda: PLUGIN_MANAGER.hook.filter_request_before_handler(request=cherrypy.request),
@@ -223,70 +223,6 @@ def await_configuration(self):
                 self.log.info("Configured CherryPy, starting engine...")  # type: ignore
                 return uri
 
-    def configure_cors(self):
-        """
-        Allow CORS requests if the cross_origin_url option is set.
-        """
-        cross_origin_url = mgr.get_localized_module_option('cross_origin_url', '')
-        if cross_origin_url:
-            cherrypy.tools.CORS = cherrypy.Tool('before_handler', self.cors_tool)
-            config = {
-                'tools.CORS.on': True,
-            }
-            self.update_cherrypy_config(config)
-
-    def cors_tool(self):
-        '''
-        Handle both simple and complex CORS requests
-
-        Add CORS headers to each response. If the request is a CORS preflight
-        request swap out the default handler with a simple, single-purpose handler
-        that verifies the request and provides a valid CORS response.
-        '''
-        req_head = cherrypy.request.headers
-        resp_head = cherrypy.response.headers
-
-        # Always set response headers necessary for 'simple' CORS.
-        req_header_cross_origin_url = req_head.get('Access-Control-Allow-Origin')
-        cross_origin_urls = mgr.get_localized_module_option('cross_origin_url', '')
-        cross_origin_url_list = [url.strip() for url in cross_origin_urls.split(',')]
-        if req_header_cross_origin_url in cross_origin_url_list:
-            resp_head['Access-Control-Allow-Origin'] = req_header_cross_origin_url
-        resp_head['Access-Control-Expose-Headers'] = 'GET, POST'
-        resp_head['Access-Control-Allow-Credentials'] = 'true'
-
-        # Non-simple CORS preflight request; short-circuit the normal handler.
-        if cherrypy.request.method == 'OPTIONS':
-            req_header_origin_url = req_head.get('Origin')
-            if req_header_origin_url in cross_origin_url_list:
-                resp_head['Access-Control-Allow-Origin'] = req_header_origin_url
-            ac_method = req_head.get('Access-Control-Request-Method', None)
-
-            allowed_methods = ['GET', 'POST', 'PUT']
-            allowed_headers = [
-                'Content-Type',
-                'Authorization',
-                'Accept',
-                'Access-Control-Allow-Origin'
-            ]
-
-            if ac_method and ac_method in allowed_methods:
-                resp_head['Access-Control-Allow-Methods'] = ', '.join(allowed_methods)
-                resp_head['Access-Control-Allow-Headers'] = ', '.join(allowed_headers)
-
-                resp_head['Connection'] = 'keep-alive'
-                resp_head['Access-Control-Max-Age'] = '3600'
-
-            # CORS requests should short-circuit the other tools.
-            cherrypy.response.body = ''.encode('utf8')
-            cherrypy.response.status = 200
-            cherrypy.serving.request.handler = None
-
-            # Needed to avoid the auth_tool check.
-            if cherrypy.request.config.get('tools.sessions.on', False):
-                cherrypy.session['token'] = True
-            return True
-
 
 if TYPE_CHECKING:
     SslConfigKey = Literal['crt', 'key']
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index ec3ae76cb654..912718d9ee77 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -6958,6 +6958,157 @@ paths:
       summary: Get Monitor Details
       tags:
       - Monitor
+  /api/multi-cluster/auth:
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                cluster_alias:
+                  type: string
+                hub_url:
+                  type: string
+                password:
+                  type: string
+                token:
+                  type: string
+                url:
+                  type: string
+                username:
+                  type: string
+              required:
+              - url
+              - cluster_alias
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Authenticate to a remote cluster
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/get_config:
+    get:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/set_config:
+    put:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                config:
+                  type: string
+              required:
+              - config
+              type: object
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource updated.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/verify_connection:
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                password:
+                  type: string
+                token:
+                  type: string
+                url:
+                  type: string
+                username:
+                  type: string
+              required:
+              - url
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
   /api/nfs-ganesha/cluster:
     get:
       parameters: []
@@ -13991,6 +14142,8 @@ tags:
   name: MonPerfCounter
 - description: Get Monitor Details
   name: Monitor
+- description: Multi-cluster Management API
+  name: Multi-cluster
 - description: NFS-Ganesha Cluster Management API
   name: NFS-Ganesha
 - description: NVMe-oF Gateway Management API
diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py
index 6018f0d7f9c7..acff17e94e5f 100644
--- a/src/pybind/mgr/dashboard/settings.py
+++ b/src/pybind/mgr/dashboard/settings.py
@@ -119,6 +119,10 @@ class Options(object):
                                                   'gateway', 'logs', 'crush', 'maps']),
                                         [str])
 
+    MULTICLUSTER_CONFIG = Setting({}, [dict, str])
+
+    UNSAFE_TLS_v1_2 = Setting(False, [bool])
+
     @staticmethod
     def has_default_value(name):
         return getattr(Settings, name, None) is None or \
diff --git a/src/pybind/mgr/dashboard/tests/test_auth.py b/src/pybind/mgr/dashboard/tests/test_auth.py
index d9755de98e45..70e841a667be 100644
--- a/src/pybind/mgr/dashboard/tests/test_auth.py
+++ b/src/pybind/mgr/dashboard/tests/test_auth.py
@@ -40,6 +40,11 @@ def test_request_not_authorized(self):
         self.assertStatus(401)
 
     @patch('dashboard.controllers.auth.JwtManager.gen_token', Mock(return_value='my-token'))
+    @patch('dashboard.mgr.get', Mock(return_value={
+        'config': {
+            'fsid': '943949f0-ce37-47ca-a33c-3413d46ee9ec'
+        }
+    }))
     @patch('dashboard.controllers.auth.AuthManager.authenticate', Mock(return_value={
         'permissions': {'rgw': ['read']},
         'pwdExpirationDate': 1000000,
diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py
index 4e4837d9323e..5619b5bb90d5 100644
--- a/src/pybind/mgr/dashboard/tools.py
+++ b/src/pybind/mgr/dashboard/tools.py
@@ -838,3 +838,72 @@ def merge_list_of_dicts_by_key(target_list: list, source_list: list, key: str):
                 target_list[sdict[key]].update(sdict)
     target_list = [value for value in target_list.values()]
     return target_list
+
+
+def configure_cors(url: str = ''):
+    """
+    Allow CORS requests if the cross_origin_url option is set.
+    """
+    if url:
+        cross_origin_url = url
+        mgr.set_module_option('cross_origin_url', cross_origin_url)
+    else:
+        cross_origin_url = mgr.get_localized_module_option('cross_origin_url', '')
+    if cross_origin_url:
+        cherrypy.tools.CORS = cherrypy.Tool('before_handler', cors_tool)
+        config = {
+            'tools.CORS.on': True,
+        }
+        cherrypy.config.update(config)
+
+
+def cors_tool():
+    '''
+    Handle both simple and complex CORS requests
+    Add CORS headers to each response. If the request is a CORS preflight
+    request swap out the default handler with a simple, single-purpose handler
+    that verifies the request and provides a valid CORS response.
+    '''
+    req_head = cherrypy.request.headers
+    resp_head = cherrypy.response.headers
+
+    # Always set response headers necessary for 'simple' CORS.
+    req_header_cross_origin_url = req_head.get('Access-Control-Allow-Origin')
+    cross_origin_urls = mgr.get_localized_module_option('cross_origin_url', '')
+    cross_origin_url_list = [url.strip() for url in cross_origin_urls.split(',')]
+    if req_header_cross_origin_url in cross_origin_url_list:
+        resp_head['Access-Control-Allow-Origin'] = req_header_cross_origin_url
+    resp_head['Access-Control-Expose-Headers'] = 'GET, POST'
+    resp_head['Access-Control-Allow-Credentials'] = 'true'
+
+    # Non-simple CORS preflight request; short-circuit the normal handler.
+    if cherrypy.request.method == 'OPTIONS':
+        req_header_origin_url = req_head.get('Origin')
+        if req_header_origin_url in cross_origin_url_list:
+            resp_head['Access-Control-Allow-Origin'] = req_header_origin_url
+        ac_method = req_head.get('Access-Control-Request-Method', None)
+
+        allowed_methods = ['GET', 'POST', 'PUT']
+        allowed_headers = [
+            'Content-Type',
+            'Authorization',
+            'Accept',
+            'Access-Control-Allow-Origin'
+        ]
+
+        if ac_method and ac_method in allowed_methods:
+            resp_head['Access-Control-Allow-Methods'] = ', '.join(allowed_methods)
+            resp_head['Access-Control-Allow-Headers'] = ', '.join(allowed_headers)
+
+            resp_head['Connection'] = 'keep-alive'
+            resp_head['Access-Control-Max-Age'] = '3600'
+
+        # CORS requests should short-circuit the other tools.
+        cherrypy.response.body = ''.encode('utf8')
+        cherrypy.response.status = 200
+        cherrypy.serving.request.handler = None
+
+        # Needed to avoid the auth_tool check.
+        if cherrypy.request.config.get('tools.sessions.on', False):
+            cherrypy.session['token'] = True
+        return True

From 495f669faf39cacbd1cbca562e97340338ef5cd1 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Thu, 25 Jan 2024 12:44:01 +0530
Subject: [PATCH 1992/2492] mgr/dashboard: Add a manage clusters page to the
 multi-cluster nav to list/connect/disconnect/edit clusters in multi-cluster
 setup

Fixes: https://tracker.ceph.com/issues/64530
Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 monitoring/ceph-mixin/dashboards.libsonnet    |    1 +
 .../dashboards/multi-cluster.libsonnet        |  964 ++++++++
 .../ceph-mixin/dashboards/utils.libsonnet     |    6 +-
 .../multi-cluster-overview.json               | 2123 +++++++++++++++++
 .../dashboard/controllers/multi_cluster.py    |  169 +-
 .../frontend/src/app/app-routing.module.ts    |   18 +-
 .../src/app/ceph/cluster/cluster.module.ts    |    4 +-
 .../multi-cluster-form.component.html         |   72 +-
 .../multi-cluster-form.component.ts           |  138 +-
 .../multi-cluster-list.component.html         |   37 +
 .../multi-cluster-list.component.scss         |    0
 .../multi-cluster-list.component.spec.ts      |   30 +
 .../multi-cluster-list.component.ts           |  213 ++
 .../multi-cluster.component.html              |   10 -
 .../multi-cluster/multi-cluster.component.ts  |    5 +-
 .../workbench-layout.component.ts             |    1 +
 .../navigation/navigation.component.html      |   10 +-
 .../navigation/navigation.component.ts        |   43 +-
 .../app/shared/api/multi-cluster.service.ts   |   94 +-
 .../src/app/shared/constants/app.constants.ts |    4 +
 .../src/app/shared/models/multi-cluster.ts    |    8 +
 .../services/api-interceptor.service.ts       |    6 +-
 src/pybind/mgr/dashboard/openapi.yaml         |  169 +-
 23 files changed, 4005 insertions(+), 120 deletions(-)
 create mode 100644 monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
 create mode 100644 monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts

diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet
index 8cb0bcceaf79..82e1888e036d 100644
--- a/monitoring/ceph-mixin/dashboards.libsonnet
+++ b/monitoring/ceph-mixin/dashboards.libsonnet
@@ -8,5 +8,6 @@
     (import 'dashboards/rgw.libsonnet') +
     (import 'dashboards/ceph-cluster.libsonnet') +
     (import 'dashboards/rgw-s3-analytics.libsonnet') +
+    (import 'dashboards/multi-cluster.libsonnet') +
     { _config:: $._config },
 }
diff --git a/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet b/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
new file mode 100644
index 000000000000..ec725f4dfa28
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
@@ -0,0 +1,964 @@
+local g = import 'grafonnet/grafana.libsonnet';
+
+(import 'utils.libsonnet') {
+  'multi-cluster-overview.json':
+    $.dashboardSchema(
+      'Ceph - Multi-cluster',
+      '',
+      'BnxelG7Sz',
+      'now-1h',
+      '30s',
+      22,
+      $._config.dashboardTags,
+      ''
+    )
+    .addAnnotation(
+      $.addAnnotationSchema(
+        1,
+        '-- Grafana --',
+        true,
+        true,
+        'rgba(0, 211, 255, 1)',
+        'Annotations & Alerts',
+        'dashboard'
+      )
+    )
+    .addTemplate(
+      g.template.datasource('DS_PROMETHEUS', 'prometheus', 'default', label='Data Source')
+    )
+
+    .addTemplate(
+      $.addTemplateSchema('Cluster',
+                          '$DS_PROMETHEUS',
+                          'label_values(ceph_health_status, cluster)',
+                          2,
+                          true,
+                          0,
+                          null,
+                          '',
+                          current='All')
+    )
+    .addPanels([
+      $.addRowSchema(false, true, 'Clusters') + { gridPos: { x: 0, y: 1, w: 24, h: 1 } },
+      $.addStatPanel(
+        title='Status',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 0, y: 2, w: 5, h: 7 },
+        graphMode='none',
+        colorMode='value',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'text', value: null },
+      ])
+      .addOverrides(
+        [
+          {
+            matcher: { id: 'byName', options: 'Warning' },
+            properties: [
+              {
+                id: 'thresholds',
+                value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'semi-dark-yellow', value: 1 }] },
+              },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Error' },
+            properties: [
+              {
+                id: 'thresholds',
+                value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'semi-dark-red', value: 1 }] },
+              },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Healthy' },
+            properties: [
+              {
+                id: 'thresholds',
+                value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'semi-dark-green', value: 1 }] },
+              },
+            ],
+          },
+        ]
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(ceph_health_status==0) or vector(0)',
+          datasource='${DS_PROMETHEUS}',
+          legendFormat='Healthy',
+        ),
+        $.addTargetSchema(
+          expr='count(ceph_health_status==1)',
+          datasource='${DS_PROMETHEUS}',
+          legendFormat='Warning'
+        ),
+        $.addTargetSchema(
+          expr='count(ceph_health_status==2)',
+          datasource='${DS_PROMETHEUS}',
+          legendFormat='Error'
+        ),
+      ]),
+
+      $.addTableExtended(
+        datasource='${DS_PROMETHEUS}',
+        title='Details',
+        gridPosition={ h: 7, w: 19, x: 5, y: 2 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+        },
+        custom={ align: 'left', cellOptions: { type: 'color-text' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'text' },
+          ],
+        },
+        overrides=[
+          {
+            matcher: { id: 'byName', options: 'Value #A' },
+            properties: [
+              { id: 'mappings', value: [{ options: { '0': { color: 'semi-dark-green', index: 2, text: 'Healthy' }, '1': { color: 'semi-dark-yellow', index: 0, text: 'Warning' }, '2': { color: 'semi-dark-red', index: 1, text: 'Error' } }, type: 'value' }] },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'IOPS' },
+            properties: [
+              { id: 'unit', value: 'ops' },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Value #E' },
+            properties: [
+              { id: 'unit', value: 'bytes' },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Capacity Used' },
+            properties: [
+              { id: 'unit', value: 'bytes' },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Cluster' },
+            properties: [
+              { id: 'links', value: [{ title: '', url: '/d/GQ3MHvnIz/ceph-cluster-new?var-cluster=${__data.fields.Cluster}&${DS_PROMETHEUS:queryparam}' }] },
+            ],
+          },
+          {
+            matcher: { id: 'byName', options: 'Alerts' },
+            properties: [
+              { id: 'mappings', value: [{ options: { match: null, result: { index: 0, text: '0' } }, type: 'special' }] },
+            ],
+          },
+        ],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'joinByField',
+          options: { byField: 'cluster', mode: 'outer' },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Value #B': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              ceph_daemon: true,
+              device_class: true,
+              hostname: true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'replica 1': true,
+              'replica 2': true,
+              'replica 3': true,
+            },
+            indexByName: {
+              'Time 1': 8,
+              'Time 2': 13,
+              'Time 3': 21,
+              'Time 4': 7,
+              'Time 5': 22,
+              'Time 6': 23,
+              'Value #A': 1,
+              'Value #B': 20,
+              'Value #C': 3,
+              'Value #D': 4,
+              'Value #E': 5,
+              'Value #F': 6,
+              '__name__ 1': 9,
+              '__name__ 2': 14,
+              '__name__ 3': 24,
+              ceph_daemon: 15,
+              ceph_version: 2,
+              cluster: 0,
+              device_class: 25,
+              hostname: 16,
+              'instance 1': 10,
+              'instance 2': 17,
+              'instance 3': 26,
+              'job 1': 11,
+              'job 2': 18,
+              'job 3': 27,
+              'replica 1': 12,
+              'replica 2': 19,
+              'replica 3': 28,
+            },
+            renameByName: {
+              'Value #A': 'Status',
+              'Value #C': 'Alerts',
+              'Value #D': 'IOPS',
+              'Value #E': 'Throughput',
+              'Value #F': 'Capacity Used',
+              ceph_version: 'Version',
+              cluster: 'Cluster',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_health_status',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_mgr_metadata',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='count(ALERTS{alertstate="firing", cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (cluster) (irate(ceph_pool_wr[$__interval]))  \n+ sum by (cluster) (irate(ceph_pool_rd[$__interval])) ',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (cluster) (irate(ceph_pool_rd_bytes[$__interval]))\n+ sum by (cluster) (irate(ceph_pool_wr_bytes[$__interval])) ',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_cluster_by_class_total_used_bytes',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+      ]),
+
+
+      $.addRowSchema(false, true, 'Overview') + { gridPos: { x: 0, y: 9, w: 24, h: 1 } },
+      $.addStatPanel(
+        title='Cluster Count',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 0, y: 10, w: 3, h: 4 },
+        graphMode='none',
+        colorMode='value',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'text', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(ceph_health_status{cluster=~"$Cluster"}) or vector(0)',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+      ]),
+
+      $.addGaugePanel(
+        title='Capacity Used',
+        gridPosition={ h: 8, w: 4, x: 3, y: 10 },
+        unit='percentunit',
+        max=1,
+        min=0,
+        interval='1m',
+        pluginVersion='9.4.7'
+      )
+      .addThresholds([
+        { color: 'green', value: null },
+        { color: 'semi-dark-yellow', value: 0.75 },
+        { color: 'red', value: 0.85 },
+      ])
+      .addTarget($.addTargetSchema(
+        expr='sum(ceph_cluster_total_used_bytes{cluster=~"$Cluster"}) / sum(ceph_cluster_total_bytes{cluster=~"$Cluster"})',
+        instant=true,
+        legendFormat='Used',
+        datasource='${DS_PROMETHEUS}',
+      )),
+
+      $.addStatPanel(
+        title='Total Capacity',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 7, y: 10, w: 3, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        unit='bytes',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_cluster_total_bytes{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=false,
+          interval='',
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='OSDs',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 10, y: 10, w: 3, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        unit='none',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(ceph_osd_metadata{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=false,
+          interval='',
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Hosts',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 13, y: 10, w: 3, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        unit='none',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(sum by (hostname) (ceph_osd_metadata{cluster=~"$Cluster"}))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=false,
+          interval='',
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Client IOPS',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 16, y: 10, w: 4, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        unit='ops',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(irate(ceph_pool_wr{cluster=~"$Cluster"}[$__interval]))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          instant=false,
+          legendFormat='Write',
+          range=true,
+        ),
+        $.addTargetSchema(
+          expr='sum(irate(ceph_pool_rd{cluster=~"$Cluster"}[$__interval]))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          legendFormat='Read',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='OSD Latencies',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 20, y: 10, w: 4, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        unit='ms',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='avg(ceph_osd_apply_latency_ms{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          instant=false,
+          legendFormat='Apply',
+          range=true,
+        ),
+        $.addTargetSchema(
+          expr='avg(ceph_osd_commit_latency_ms{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          legendFormat='Commit',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Alert Count',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 0, y: 14, w: 3, h: 4 },
+        graphMode='none',
+        colorMode='value',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'text', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(ALERTS{alertstate="firing", cluster=~"$Cluster"}) or vector(0)',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Total Used',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 7, y: 14, w: 3, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        unit='bytes',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_cluster_total_used_bytes{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=false,
+          interval='',
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Capacity Prediction',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 10, y: 14, w: 3, h: 4 },
+        graphMode='none',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        unit='s',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='predict_linear(avg(increase(ceph_cluster_total_used_bytes{cluster=~"${Cluster}"}[1d]))[7d:1h],120)',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Pools',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 13, y: 14, w: 3, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        unit='none',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='count(ceph_pool_metadata{cluster=~"$Cluster"})',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=false,
+          interval='',
+          legendFormat='__auto',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Client Bandwidth',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 16, y: 14, w: 4, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        unit='binBps',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(irate(ceph_pool_rd_bytes{cluster=~"$Cluster"}[$__interval]))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          instant=false,
+          legendFormat='Write',
+          range=true,
+        ),
+        $.addTargetSchema(
+          expr='sum(irate(ceph_pool_wr_bytes{cluster=~"$Cluster"}[$__interval]))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          legendFormat='Read',
+          range=true,
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Recovery Rate',
+        datasource='${DS_PROMETHEUS}',
+        gridPosition={ x: 20, y: 14, w: 4, h: 4 },
+        graphMode='area',
+        colorMode='none',
+        orientation='auto',
+        justifyMode='center',
+        thresholdsMode='absolute',
+        unit='binBps',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(irate(ceph_osd_recovery_ops{cluster=~"$Cluster"}[$__interval]))',
+          datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+          hide=false,
+          exemplar=false,
+          instant=false,
+          legendFormat='Write',
+          range=true,
+        ),
+      ]),
+
+
+      $.addRowSchema(false, true, 'Alerts', collapsed=true)
+      .addPanels([
+        $.addStatPanel(
+          title='Status',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ x: 0, y: 19, w: 5, h: 7 },
+          graphMode='area',
+          colorMode='value',
+          orientation='auto',
+          justifyMode='center',
+          thresholdsMode='absolute',
+          pluginVersion='9.4.7',
+        ).addThresholds([
+          { color: 'text', value: null },
+        ])
+        .addOverrides(
+          [
+            {
+              matcher: { id: 'byName', options: 'Critical' },
+              properties: [
+                {
+                  id: 'thresholds',
+                  value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'semi-dark-red', value: 1 }] },
+                },
+              ],
+            },
+            {
+              matcher: { id: 'byName', options: 'Warning' },
+              properties: [
+                {
+                  id: 'thresholds',
+                  value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'semi-dark-yellow', value: 1 }] },
+                },
+              ],
+            },
+          ]
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='count(ALERTS{alertstate="firing",severity="critical", cluster=~"$Cluster"}) OR vector(0)',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='Critical',
+            instant=true,
+            range=false
+          ),
+          $.addTargetSchema(
+            expr='count(ALERTS{alertstate="firing",severity="warning", cluster=~"$Cluster"}) OR vector(0)',
+            datasource='${DS_PROMETHEUS}',
+            legendFormat='Warning',
+            instant=true,
+            range=false
+          ),
+        ]),
+
+
+        $.addTableExtended(
+          datasource='${DS_PROMETHEUS}',
+          title='Alerts',
+          gridPosition={ h: 7, w: 19, x: 5, y: 19 },
+          options={
+            footer: {
+              fields: '',
+              reducer: ['sum'],
+              countRows: false,
+              enablePagination: false,
+              show: false,
+            },
+            frameIndex: 1,
+            showHeader: true,
+            sortBy: [{ desc: false, displayName: 'Severity' }],
+          },
+          custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: true, inspect: false },
+          thresholds={
+            mode: 'absolute',
+            steps: [
+              { color: 'green' },
+              { color: 'red', value: 80 },
+            ],
+          },
+          pluginVersion='9.4.7'
+        )
+        .addTransformations([
+          {
+            id: 'joinByField',
+            options: { byField: 'cluster', mode: 'outer' },
+          },
+          {
+            id: 'organize',
+            options: {
+              excludeByName: {
+                Time: true,
+                Value: true,
+                __name__: true,
+                instance: true,
+                job: true,
+                oid: true,
+                replica: true,
+                type: true,
+              },
+              indexByName: {
+                Time: 0,
+                Value: 9,
+                __name__: 1,
+                alertname: 2,
+                alertstate: 4,
+                cluster: 3,
+                instance: 6,
+                job: 7,
+                severity: 5,
+                type: 8,
+              },
+              renameByName: {
+                alertname: 'Name',
+                alertstate: 'State',
+                cluster: 'Cluster',
+                severity: 'Severity',
+              },
+            },
+          },
+        ]).addTargets([
+          $.addTargetSchema(
+            expr='ALERTS{alertstate="firing", cluster=~"$Cluster"}',
+            datasource={ type: 'prometheus', uid: '${DS_PROMETHEUS}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+        ]),
+
+        $.addAlertListPanel(
+          title='Alerts(Grouped)',
+          datasource={
+            type: 'datasource',
+            uid: 'grafana',
+          },
+          gridPosition={ h: 8, w: 24, x: 0, y: 26 },
+          alertName='',
+          dashboardAlerts=false,
+          groupBy=[],
+          groupMode='default',
+          maxItems=20,
+          sortOrder=1,
+          stateFilter={
+            'error': true,
+            firing: true,
+            noData: false,
+            normal: false,
+            pending: true,
+          },
+        ),
+      ]) + { gridPos: { x: 0, y: 18, w: 24, h: 1 } },
+
+      $.addRowSchema(false, true, 'Cluster Stats', collapsed=true)
+      .addPanels([
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Top 5 - Capacity Utilization(%)',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 7, w: 8, x: 0, y: 30 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='percentunit',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'multi', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=false,
+          decimals=2,
+          thresholdsMode='percentage',
+          sortBy='Last',
+          sortDesc=true
+        )
+        .addCalcs(['last'])
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='topk(5, ceph_cluster_total_used_bytes/ceph_cluster_total_bytes)',
+              datasource='${DS_PROMETHEUS}',
+              instant=false,
+              legendFormat='{{cluster}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ),
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Top 5 - Cluster IOPS',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 7, w: 8, x: 8, y: 30 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='ops',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'multi', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=false,
+          decimals=2,
+          thresholdsMode='percentage',
+          sortBy='Last',
+          sortDesc=true
+        )
+        .addCalcs(['last'])
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='topk(10, sum by (cluster) (irate(ceph_osd_op_w[$__interval]))  \n+ sum by (cluster) (irate(ceph_osd_op_r[$__interval])) )',
+              datasource='${DS_PROMETHEUS}',
+              instant=false,
+              legendFormat='{{cluster}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ),
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Top 10 - Capacity Utilization(%) by Pool',
+          datasource='${DS_PROMETHEUS}',
+          gridPosition={ h: 7, w: 8, x: 16, y: 30 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='percentunit',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'multi', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=false,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last',
+          sortDesc=true
+        )
+        .addCalcs(['last'])
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='topk(10, ceph_pool_bytes_used{cluster=~"$Cluster"}/ceph_pool_max_avail{cluster=~"$Cluster"} * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata{cluster=~"$Cluster"})',
+              datasource='${DS_PROMETHEUS}',
+              instant=false,
+              legendFormat='{{cluster}} - {{name}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ),
+      ]) + { gridPos: { x: 0, y: 29, w: 24, h: 1 } },
+    ]),
+}
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index 397c00fe53ad..a1c2fe942142 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -117,7 +117,8 @@ local timeSeries = import 'timeseries_panel.libsonnet';
                     regex,
                     hide='',
                     multi=false,
-                    allValues=null)::
+                    allValues=null,
+                    current=null)::
     g.template.new(name=name,
                    datasource=datasource,
                    query=query,
@@ -128,7 +129,8 @@ local timeSeries = import 'timeseries_panel.libsonnet';
                    regex=regex,
                    hide=hide,
                    multi=multi,
-                   allValues=allValues),
+                   allValues=allValues,
+                   current=current),
 
   addAnnotationSchema(builtIn,
                       datasource,
diff --git a/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json b/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
new file mode 100644
index 000000000000..91b2934f0654
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
@@ -0,0 +1,2123 @@
+{
+   "__inputs": [ ],
+   "__requires": [ ],
+   "annotations": {
+      "list": [
+         {
+            "builtIn": 1,
+            "datasource": "-- Grafana --",
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "showIn": 0,
+            "tags": [ ],
+            "type": "dashboard"
+         }
+      ]
+   },
+   "description": "",
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 1
+         },
+         "id": 2,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Clusters",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "text",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "none"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Warning"
+                  },
+                  "properties": [
+                     {
+                        "id": "thresholds",
+                        "value": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "text",
+                                 "value": null
+                              },
+                              {
+                                 "color": "semi-dark-yellow",
+                                 "value": 1
+                              }
+                           ]
+                        }
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Error"
+                  },
+                  "properties": [
+                     {
+                        "id": "thresholds",
+                        "value": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "text",
+                                 "value": null
+                              },
+                              {
+                                 "color": "semi-dark-red",
+                                 "value": 1
+                              }
+                           ]
+                        }
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Healthy"
+                  },
+                  "properties": [
+                     {
+                        "id": "thresholds",
+                        "value": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "text",
+                                 "value": null
+                              },
+                              {
+                                 "color": "semi-dark-green",
+                                 "value": 1
+                              }
+                           ]
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 5,
+            "x": 0,
+            "y": 2
+         },
+         "id": 3,
+         "links": [ ],
+         "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "count(ceph_health_status==0) or vector(0)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Healthy",
+               "refId": "A"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "count(ceph_health_status==1)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Warning",
+               "refId": "B"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "count(ceph_health_status==2)",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Error",
+               "refId": "C"
+            }
+         ],
+         "title": "Status",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${DS_PROMETHEUS}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "left",
+                  "cellOptions": {
+                     "type": "color-text"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "text"
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Value #A"
+                  },
+                  "properties": [
+                     {
+                        "id": "mappings",
+                        "value": [
+                           {
+                              "options": {
+                                 "0": {
+                                    "color": "semi-dark-green",
+                                    "index": 2,
+                                    "text": "Healthy"
+                                 },
+                                 "1": {
+                                    "color": "semi-dark-yellow",
+                                    "index": 0,
+                                    "text": "Warning"
+                                 },
+                                 "2": {
+                                    "color": "semi-dark-red",
+                                    "index": 1,
+                                    "text": "Error"
+                                 }
+                              },
+                              "type": "value"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "IOPS"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ops"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Value #E"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "bytes"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Capacity Used"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "bytes"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Cluster"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "",
+                              "url": "/d/GQ3MHvnIz/ceph-cluster-new?var-cluster=${__data.fields.Cluster}&${DS_PROMETHEUS:queryparam}"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Alerts"
+                  },
+                  "properties": [
+                     {
+                        "id": "mappings",
+                        "value": [
+                           {
+                              "options": {
+                                 "match": null,
+                                 "result": {
+                                    "index": 0,
+                                    "text": "0"
+                                 }
+                              },
+                              "type": "special"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 19,
+            "x": 5,
+            "y": 2
+         },
+         "id": 4,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "ceph_health_status",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "ceph_mgr_metadata",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(ALERTS{alertstate=\"firing\", cluster=~\"$Cluster\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum by (cluster) (irate(ceph_pool_wr[$__interval]))  \n+ sum by (cluster) (irate(ceph_pool_rd[$__interval])) ",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum by (cluster) (irate(ceph_pool_rd_bytes[$__interval]))\n+ sum by (cluster) (irate(ceph_pool_wr_bytes[$__interval])) ",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "E"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "ceph_cluster_by_class_total_used_bytes",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "F"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Details",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "cluster",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Value #B": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "ceph_daemon": true,
+                     "device_class": true,
+                     "hostname": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "replica 1": true,
+                     "replica 2": true,
+                     "replica 3": true
+                  },
+                  "indexByName": {
+                     "Time 1": 8,
+                     "Time 2": 13,
+                     "Time 3": 21,
+                     "Time 4": 7,
+                     "Time 5": 22,
+                     "Time 6": 23,
+                     "Value #A": 1,
+                     "Value #B": 20,
+                     "Value #C": 3,
+                     "Value #D": 4,
+                     "Value #E": 5,
+                     "Value #F": 6,
+                     "__name__ 1": 9,
+                     "__name__ 2": 14,
+                     "__name__ 3": 24,
+                     "ceph_daemon": 15,
+                     "ceph_version": 2,
+                     "cluster": 0,
+                     "device_class": 25,
+                     "hostname": 16,
+                     "instance 1": 10,
+                     "instance 2": 17,
+                     "instance 3": 26,
+                     "job 1": 11,
+                     "job 2": 18,
+                     "job 3": 27,
+                     "replica 1": 12,
+                     "replica 2": 19,
+                     "replica 3": 28
+                  },
+                  "renameByName": {
+                     "Value #A": "Status",
+                     "Value #C": "Alerts",
+                     "Value #D": "IOPS",
+                     "Value #E": "Throughput",
+                     "Value #F": "Capacity Used",
+                     "ceph_version": "Version",
+                     "cluster": "Cluster"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 9
+         },
+         "id": 5,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Overview",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "text",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 0,
+            "y": 10
+         },
+         "id": 6,
+         "links": [ ],
+         "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(ceph_health_status{cluster=~\"$Cluster\"}) or vector(0)",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "A"
+            }
+         ],
+         "title": "Cluster Count",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "links": [ ],
+               "mappings": [ ],
+               "max": 1,
+               "min": 0,
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "semi-dark-yellow",
+                        "value": 0.75
+                     },
+                     {
+                        "color": "red",
+                        "value": 0.84999999999999998
+                     }
+                  ]
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 4,
+            "x": 3,
+            "y": 10
+         },
+         "id": 7,
+         "interval": "1m",
+         "links": [ ],
+         "maxDataPoints": 100,
+         "options": {
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "showThresholdLabels": false,
+            "showThresholdMarkers": true
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "expr": "sum(ceph_cluster_total_used_bytes{cluster=~\"$Cluster\"}) / sum(ceph_cluster_total_bytes{cluster=~\"$Cluster\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Used",
+               "refId": "A"
+            }
+         ],
+         "title": "Capacity Used",
+         "transparent": false,
+         "type": "gauge"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 7,
+            "y": 10
+         },
+         "id": 8,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(ceph_cluster_total_bytes{cluster=~\"$Cluster\"})",
+               "format": "table",
+               "hide": false,
+               "instant": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total Capacity",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 10,
+            "y": 10
+         },
+         "id": 9,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(ceph_osd_metadata{cluster=~\"$Cluster\"})",
+               "format": "table",
+               "hide": false,
+               "instant": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "OSDs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 13,
+            "y": 10
+         },
+         "id": 10,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(sum by (hostname) (ceph_osd_metadata{cluster=~\"$Cluster\"}))",
+               "format": "table",
+               "hide": false,
+               "instant": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Hosts",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "ops"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 16,
+            "y": 10
+         },
+         "id": 11,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(irate(ceph_pool_wr{cluster=~\"$Cluster\"}[$__interval]))",
+               "format": "time_series",
+               "hide": false,
+               "instant": false,
+               "intervalFactor": 1,
+               "legendFormat": "Write",
+               "range": true,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(irate(ceph_pool_rd{cluster=~\"$Cluster\"}[$__interval]))",
+               "format": "time_series",
+               "hide": false,
+               "intervalFactor": 1,
+               "legendFormat": "Read",
+               "range": true,
+               "refId": "B"
+            }
+         ],
+         "title": "Client IOPS",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "ms"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 20,
+            "y": 10
+         },
+         "id": 12,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "avg(ceph_osd_apply_latency_ms{cluster=~\"$Cluster\"})",
+               "format": "time_series",
+               "hide": false,
+               "instant": false,
+               "intervalFactor": 1,
+               "legendFormat": "Apply",
+               "range": true,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "avg(ceph_osd_commit_latency_ms{cluster=~\"$Cluster\"})",
+               "format": "time_series",
+               "hide": false,
+               "intervalFactor": 1,
+               "legendFormat": "Commit",
+               "range": true,
+               "refId": "B"
+            }
+         ],
+         "title": "OSD Latencies",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "text",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 0,
+            "y": 14
+         },
+         "id": 13,
+         "links": [ ],
+         "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(ALERTS{alertstate=\"firing\", cluster=~\"$Cluster\"}) or vector(0)",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "A"
+            }
+         ],
+         "title": "Alert Count",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "bytes"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 7,
+            "y": 14
+         },
+         "id": 14,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(ceph_cluster_total_used_bytes{cluster=~\"$Cluster\"})",
+               "format": "table",
+               "hide": false,
+               "instant": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total Used",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "s"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 10,
+            "y": 14
+         },
+         "id": 15,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "predict_linear(avg(increase(ceph_cluster_total_used_bytes{cluster=~\"${Cluster}\"}[1d]))[7d:1h],120)",
+               "format": "time_series",
+               "hide": false,
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Capacity Prediction",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 3,
+            "x": 13,
+            "y": 14
+         },
+         "id": 16,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "count(ceph_pool_metadata{cluster=~\"$Cluster\"})",
+               "format": "table",
+               "hide": false,
+               "instant": false,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Pools",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "binBps"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 16,
+            "y": 14
+         },
+         "id": 17,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(irate(ceph_pool_rd_bytes{cluster=~\"$Cluster\"}[$__interval]))",
+               "format": "time_series",
+               "hide": false,
+               "instant": false,
+               "intervalFactor": 1,
+               "legendFormat": "Write",
+               "range": true,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(irate(ceph_pool_wr_bytes{cluster=~\"$Cluster\"}[$__interval]))",
+               "format": "time_series",
+               "hide": false,
+               "intervalFactor": 1,
+               "legendFormat": "Read",
+               "range": true,
+               "refId": "B"
+            }
+         ],
+         "title": "Client Bandwidth",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${DS_PROMETHEUS}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               },
+               "unit": "binBps"
+            }
+         },
+         "gridPos": {
+            "h": 4,
+            "w": 4,
+            "x": 20,
+            "y": 14
+         },
+         "id": 18,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${DS_PROMETHEUS}"
+               },
+               "exemplar": false,
+               "expr": "sum(irate(ceph_osd_recovery_ops{cluster=~\"$Cluster\"}[$__interval]))",
+               "format": "time_series",
+               "hide": false,
+               "instant": false,
+               "intervalFactor": 1,
+               "legendFormat": "Write",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Recovery Rate",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 18
+         },
+         "id": 19,
+         "panels": [
+            {
+               "colors": null,
+               "datasource": "${DS_PROMETHEUS}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "decimals": 0,
+                     "links": [ ],
+                     "mappings": [ ],
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "text",
+                              "value": null
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byName",
+                           "options": "Critical"
+                        },
+                        "properties": [
+                           {
+                              "id": "thresholds",
+                              "value": {
+                                 "mode": "absolute",
+                                 "steps": [
+                                    {
+                                       "color": "text",
+                                       "value": null
+                                    },
+                                    {
+                                       "color": "semi-dark-red",
+                                       "value": 1
+                                    }
+                                 ]
+                              }
+                           }
+                        ]
+                     },
+                     {
+                        "matcher": {
+                           "id": "byName",
+                           "options": "Warning"
+                        },
+                        "properties": [
+                           {
+                              "id": "thresholds",
+                              "value": {
+                                 "mode": "absolute",
+                                 "steps": [
+                                    {
+                                       "color": "text",
+                                       "value": null
+                                    },
+                                    {
+                                       "color": "semi-dark-yellow",
+                                       "value": 1
+                                    }
+                                 ]
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 5,
+                  "x": 0,
+                  "y": 19
+               },
+               "id": 20,
+               "links": [ ],
+               "options": {
+                  "colorMode": "value",
+                  "graphMode": "area",
+                  "justifyMode": "center",
+                  "orientation": "auto",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "fields": "",
+                     "values": false
+                  },
+                  "textMode": "auto"
+               },
+               "pluginVersion": "9.4.7",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "count(ALERTS{alertstate=\"firing\",severity=\"critical\", cluster=~\"$Cluster\"}) OR vector(0)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "Critical",
+                     "range": false,
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "count(ALERTS{alertstate=\"firing\",severity=\"warning\", cluster=~\"$Cluster\"}) OR vector(0)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "Warning",
+                     "range": false,
+                     "refId": "B"
+                  }
+               ],
+               "title": "Status",
+               "transparent": false,
+               "type": "stat"
+            },
+            {
+               "columns": [ ],
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "align": "auto",
+                        "cellOptions": {
+                           "type": "auto"
+                        },
+                        "filterable": true,
+                        "inspect": false
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           },
+                           {
+                              "color": "red",
+                              "value": 80
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 19,
+                  "x": 5,
+                  "y": 19
+               },
+               "id": 21,
+               "links": [ ],
+               "options": {
+                  "footer": {
+                     "countRows": false,
+                     "enablePagination": false,
+                     "fields": "",
+                     "reducer": [
+                        "sum"
+                     ],
+                     "show": false
+                  },
+                  "frameIndex": 1,
+                  "showHeader": true,
+                  "sortBy": [
+                     {
+                        "desc": false,
+                        "displayName": "Severity"
+                     }
+                  ]
+               },
+               "pluginVersion": "9.4.7",
+               "styles": "",
+               "targets": [
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                     },
+                     "exemplar": false,
+                     "expr": "ALERTS{alertstate=\"firing\", cluster=~\"$Cluster\"}",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Alerts",
+               "transformations": [
+                  {
+                     "id": "joinByField",
+                     "options": {
+                        "byField": "cluster",
+                        "mode": "outer"
+                     }
+                  },
+                  {
+                     "id": "organize",
+                     "options": {
+                        "excludeByName": {
+                           "Time": true,
+                           "Value": true,
+                           "__name__": true,
+                           "instance": true,
+                           "job": true,
+                           "oid": true,
+                           "replica": true,
+                           "type": true
+                        },
+                        "indexByName": {
+                           "Time": 0,
+                           "Value": 9,
+                           "__name__": 1,
+                           "alertname": 2,
+                           "alertstate": 4,
+                           "cluster": 3,
+                           "instance": 6,
+                           "job": 7,
+                           "severity": 5,
+                           "type": 8
+                        },
+                        "renameByName": {
+                           "alertname": "Name",
+                           "alertstate": "State",
+                           "cluster": "Cluster",
+                           "severity": "Severity"
+                        }
+                     }
+                  }
+               ],
+               "type": "table"
+            },
+            {
+               "datasource": {
+                  "type": "datasource",
+                  "uid": "grafana"
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 24,
+                  "x": 0,
+                  "y": 26
+               },
+               "id": 22,
+               "limit": 10,
+               "onlyAlertsOnDashboard": true,
+               "options": {
+                  "alertName": "",
+                  "dashboardAlerts": false,
+                  "groupBy": [ ],
+                  "groupMode": "default",
+                  "maxItems": 20,
+                  "sortOrder": 1,
+                  "stateFilter": {
+                     "error": true,
+                     "firing": true,
+                     "noData": false,
+                     "normal": false,
+                     "pending": true
+                  },
+                  "viewMode": "list"
+               },
+               "show": "current",
+               "sortOrder": 1,
+               "stateFilter": [ ],
+               "title": "Alerts(Grouped)",
+               "type": "alertlist"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Alerts",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 29
+         },
+         "id": 23,
+         "panels": [
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "percentage",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "percentunit"
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 8,
+                  "x": 0,
+                  "y": 30
+               },
+               "id": 24,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "last"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "multi",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "topk(5, ceph_cluster_total_used_bytes/ceph_cluster_total_bytes)",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{cluster}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Top 5 - Capacity Utilization(%)",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "percentage",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "ops"
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 8,
+                  "x": 8,
+                  "y": 30
+               },
+               "id": 25,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "last"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "multi",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "topk(10, sum by (cluster) (irate(ceph_osd_op_w[$__interval]))  \n+ sum by (cluster) (irate(ceph_osd_op_r[$__interval])) )",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{cluster}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Top 5 - Cluster IOPS",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${DS_PROMETHEUS}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": false,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "percentunit"
+                  },
+                  "overrides": [ ]
+               },
+               "gridPos": {
+                  "h": 7,
+                  "w": 8,
+                  "x": 16,
+                  "y": 30
+               },
+               "id": 26,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "last"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "multi",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${DS_PROMETHEUS}",
+                     "expr": "topk(10, ceph_pool_bytes_used{cluster=~\"$Cluster\"}/ceph_pool_max_avail{cluster=~\"$Cluster\"} * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata{cluster=~\"$Cluster\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{cluster}} - {{name}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Top 10 - Capacity Utilization(%) by Pool",
+               "type": "timeseries"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Cluster Stats",
+         "titleSize": "h6",
+         "type": "row"
+      }
+   ],
+   "refresh": "30s",
+   "rows": [ ],
+   "schemaVersion": 22,
+   "style": "dark",
+   "tags": [
+      "ceph-mixin"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data Source",
+            "name": "DS_PROMETHEUS",
+            "options": [ ],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "allValue": null,
+            "current": {
+               "text": "All",
+               "value": "All"
+            },
+            "datasource": "$DS_PROMETHEUS",
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": false,
+            "name": "Cluster",
+            "options": [ ],
+            "query": "label_values(ceph_health_status, cluster)",
+            "refresh": 2,
+            "regex": "",
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "",
+   "title": "Ceph - Multi-cluster",
+   "uid": "BnxelG7Sz",
+   "version": 0
+}
diff --git a/src/pybind/mgr/dashboard/controllers/multi_cluster.py b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
index d7acec22bebb..c918c2ec3c24 100644
--- a/src/pybind/mgr/dashboard/controllers/multi_cluster.py
+++ b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
+import base64
 import json
+import time
 
 import requests
 
@@ -8,8 +10,8 @@
 from ..security import Scope
 from ..settings import Settings
 from ..tools import configure_cors
-from . import APIDoc, APIRouter, CreatePermission, Endpoint, EndpointDoc, \
-    ReadPermission, RESTController, UIRouter, UpdatePermission
+from . import APIDoc, APIRouter, CreatePermission, DeletePermission, Endpoint, \
+    EndpointDoc, ReadPermission, RESTController, UIRouter, UpdatePermission
 
 
 @APIRouter('/multi-cluster', Scope.CONFIG_OPT)
@@ -17,6 +19,8 @@
 class MultiCluster(RESTController):
     def _proxy(self, method, base_url, path, params=None, payload=None, verify=False,
                token=None):
+        if not base_url.endswith('/'):
+            base_url = base_url + '/'
         try:
             if token:
                 headers = {
@@ -48,12 +52,7 @@ def _proxy(self, method, base_url, path, params=None, payload=None, verify=False
     @CreatePermission
     @EndpointDoc("Authenticate to a remote cluster")
     def auth(self, url: str, cluster_alias: str, username=None,
-             password=None, token=None, hub_url=None):
-
-        multi_cluster_config = self.load_multi_cluster_config()
-
-        if not url.endswith('/'):
-            url = url + '/'
+             password=None, token=None, hub_url=None, cluster_fsid=None):
 
         if username and password:
             payload = {
@@ -67,41 +66,40 @@ def auth(self, url: str, cluster_alias: str, username=None,
                     http_status_code=400,
                     component='dashboard')
 
-            token = content['token']
+            cluster_token = content['token']
 
-        if token:
             self._proxy('PUT', url, 'ui-api/multi-cluster/set_cors_endpoint',
-                        payload={'url': hub_url}, token=token)
-            fsid = self._proxy('GET', url, 'api/health/get_cluster_fsid', token=token)
-            content = self._proxy('POST', url, 'api/auth/check', payload={'token': token},
-                                  token=token)
-            if 'username' in content:
-                username = content['username']
-
-            if 'config' not in multi_cluster_config:
-                multi_cluster_config['config'] = {}
-
-            if fsid in multi_cluster_config['config']:
-                existing_entries = multi_cluster_config['config'][fsid]
-                if not any(entry['user'] == username for entry in existing_entries):
-                    existing_entries.append({
-                        "name": fsid,
-                        "url": url,
-                        "cluster_alias": cluster_alias,
-                        "user": username,
-                        "token": token,
-                    })
-            else:
-                multi_cluster_config['current_user'] = username
-                multi_cluster_config['config'][fsid] = [{
+                        payload={'url': hub_url}, token=cluster_token)
+
+            fsid = self._proxy('GET', url, 'api/health/get_cluster_fsid', token=cluster_token)
+
+            self.set_multi_cluster_config(fsid, username, url, cluster_alias, cluster_token)
+
+        if token and cluster_fsid and username:
+            self.set_multi_cluster_config(cluster_fsid, username, url, cluster_alias, token)
+
+    def set_multi_cluster_config(self, fsid, username, url, cluster_alias, token):
+        multi_cluster_config = self.load_multi_cluster_config()
+        if fsid in multi_cluster_config['config']:
+            existing_entries = multi_cluster_config['config'][fsid]
+            if not any(entry['user'] == username for entry in existing_entries):
+                existing_entries.append({
                     "name": fsid,
                     "url": url,
                     "cluster_alias": cluster_alias,
                     "user": username,
                     "token": token,
-                }]
-
-            Settings.MULTICLUSTER_CONFIG = multi_cluster_config
+                })
+        else:
+            multi_cluster_config['current_user'] = username
+            multi_cluster_config['config'][fsid] = [{
+                "name": fsid,
+                "url": url,
+                "cluster_alias": cluster_alias,
+                "user": username,
+                "token": token,
+            }]
+        Settings.MULTICLUSTER_CONFIG = multi_cluster_config
 
     def load_multi_cluster_config(self):
         if isinstance(Settings.MULTICLUSTER_CONFIG, str):
@@ -124,13 +122,71 @@ def set_config(self, config: object):
         Settings.MULTICLUSTER_CONFIG = multicluster_config
         return Settings.MULTICLUSTER_CONFIG
 
-    @Endpoint('POST')
+    @Endpoint('PUT')
     @CreatePermission
-    # pylint: disable=R0911
-    def verify_connection(self, url: str, username=None, password=None, token=None):
-        if not url.endswith('/'):
-            url = url + '/'
+    # pylint: disable=unused-variable
+    def reconnect_cluster(self, url: str, username=None, password=None, token=None):
+        multicluster_config = self.load_multi_cluster_config()
+        if username and password:
+            payload = {
+                'username': username,
+                'password': password
+            }
+            content = self._proxy('POST', url, 'api/auth', payload=payload)
+            if 'token' not in content:
+                raise DashboardException(
+                    "Could not authenticate to remote cluster",
+                    http_status_code=400,
+                    component='dashboard')
 
+            token = content['token']
+
+        if username and token:
+            if "config" in multicluster_config:
+                for key, cluster_details in multicluster_config["config"].items():
+                    for cluster in cluster_details:
+                        if cluster["url"] == url and cluster["user"] == username:
+                            cluster['token'] = token
+            Settings.MULTICLUSTER_CONFIG = multicluster_config
+        return Settings.MULTICLUSTER_CONFIG
+
+    @Endpoint('PUT')
+    @UpdatePermission
+    # pylint: disable=unused-variable
+    def edit_cluster(self, url, cluster_alias, username):
+        multicluster_config = self.load_multi_cluster_config()
+        if "config" in multicluster_config:
+            for key, cluster_details in multicluster_config["config"].items():
+                for cluster in cluster_details:
+                    if cluster["url"] == url and cluster["user"] == username:
+                        cluster['cluster_alias'] = cluster_alias
+        Settings.MULTICLUSTER_CONFIG = multicluster_config
+        return Settings.MULTICLUSTER_CONFIG
+
+    @Endpoint(method='DELETE')
+    @DeletePermission
+    def delete_cluster(self, cluster_name, cluster_user):
+        multicluster_config = self.load_multi_cluster_config()
+        if "config" in multicluster_config:
+            keys_to_remove = []
+            for key, cluster_details in multicluster_config["config"].items():
+                cluster_details_copy = list(cluster_details)
+                for cluster in cluster_details_copy:
+                    if cluster["name"] == cluster_name and cluster["user"] == cluster_user:
+                        cluster_details.remove(cluster)
+                        if not cluster_details:
+                            keys_to_remove.append(key)
+
+            for key in keys_to_remove:
+                del multicluster_config["config"][key]
+
+        Settings.MULTICLUSTER_CONFIG = multicluster_config
+        return Settings.MULTICLUSTER_CONFIG
+
+    @Endpoint()
+    @ReadPermission
+    # pylint: disable=R0911
+    def verify_connection(self, url=None, username=None, password=None, token=None):
         if token:
             try:
                 payload = {
@@ -172,6 +228,37 @@ def verify_connection(self, url: str, username=None, password=None, token=None):
     def get_config(self):
         return Settings.MULTICLUSTER_CONFIG
 
+    def is_token_expired(self, jwt_token):
+        split_message = jwt_token.split(".")
+        base64_message = split_message[1]
+        decoded_token = json.loads(base64.urlsafe_b64decode(base64_message + "===="))
+        expiration_time = decoded_token['exp']
+        current_time = time.time()
+        return expiration_time < current_time
+
+    def check_token_status_expiration(self, token):
+        if self.is_token_expired(token):
+            return 1
+        return 0
+
+    def check_token_status_array(self, clusters_token_array):
+        token_status_map = {}
+
+        for item in clusters_token_array:
+            cluster_name = item['name']
+            token = item['token']
+            user = item['user']
+            status = self.check_token_status_expiration(token)
+            token_status_map[cluster_name] = {'status': status, 'user': user}
+
+        return token_status_map
+
+    @Endpoint()
+    @ReadPermission
+    def check_token_status(self, clustersTokenMap=None):
+        clusters_token_map = json.loads(clustersTokenMap)
+        return self.check_token_status_array(clusters_token_map)
+
 
 @UIRouter('/multi-cluster', Scope.CONFIG_OPT)
 class MultiClusterUi(RESTController):
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
index 48224c844d47..c54681b065f2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
@@ -49,6 +49,7 @@ import { UpgradeComponent } from './ceph/cluster/upgrade/upgrade.component';
 import { CephfsVolumeFormComponent } from './ceph/cephfs/cephfs-form/cephfs-form.component';
 import { UpgradeProgressComponent } from './ceph/cluster/upgrade/upgrade-progress/upgrade-progress.component';
 import { MultiClusterComponent } from './ceph/cluster/multi-cluster/multi-cluster.component';
+import { MultiClusterListComponent } from './ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component';
 
 @Injectable()
 export class PerformanceCounterBreadcrumbsResolver extends BreadcrumbsResolver {
@@ -187,7 +188,22 @@ const routes: Routes = [
       },
       {
         path: 'multi-cluster',
-        component: MultiClusterComponent
+        children: [
+          {
+            path: 'overview',
+            component: MultiClusterComponent,
+            data: {
+              breadcrumbs: 'Multi-Cluster/Overview'
+            }
+          },
+          {
+            path: 'manage-clusters',
+            component: MultiClusterListComponent,
+            data: {
+              breadcrumbs: 'Multi-Cluster/Manage Clusters'
+            }
+          }
+        ]
       },
       {
         path: 'inventory',
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
index b1eb9275a462..2f0734885d85 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
@@ -63,6 +63,7 @@ import { UpgradeStartModalComponent } from './upgrade/upgrade-form/upgrade-start
 import { UpgradeProgressComponent } from './upgrade/upgrade-progress/upgrade-progress.component';
 import { MultiClusterComponent } from './multi-cluster/multi-cluster.component';
 import { MultiClusterFormComponent } from './multi-cluster/multi-cluster-form/multi-cluster-form.component';
+import { MultiClusterListComponent } from './multi-cluster/multi-cluster-list/multi-cluster-list.component';
 
 @NgModule({
   imports: [
@@ -128,7 +129,8 @@ import { MultiClusterFormComponent } from './multi-cluster/multi-cluster-form/mu
     UpgradeStartModalComponent,
     UpgradeProgressComponent,
     MultiClusterComponent,
-    MultiClusterFormComponent
+    MultiClusterFormComponent,
+    MultiClusterListComponent
   ],
   providers: [NgbActiveModal]
 })
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
index cc9ed7453fc4..c875557306a8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
@@ -1,6 +1,6 @@
 <cd-modal [modalRef]="activeModal">
   <ng-container i18n="form title"
-                class="modal-title">Connect Cluster
+                class="modal-title">{{ action | titlecase }} Cluster
   </ng-container>
   <ng-container class="modal-content">
     <form name="remoteClusterForm"
@@ -64,12 +64,16 @@
                   *ngIf="remoteClusterForm.showError('clusterAlias', frm, 'required')"
                   i18n>This field is required.
             </span>
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('clusterAlias', frm, 'uniqueName')"
+                  i18n>The chosen alias name is already in use.
+            </span>
           </div>
         </div>
         <div class="form-group row"
-             *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError">
+             *ngIf="action !== 'edit'">
           <label class="cd-col-form-label required"
-                 for="apiToken"
+                 for="username"
                  i18n>Username
           </label>
           <div class="cd-col-form-input">
@@ -82,10 +86,32 @@
                   *ngIf="remoteClusterForm.showError('username', frm, 'required')"
                   i18n>This field is required.
             </span>
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('username', frm, 'uniqueUrlandUser')"
+                  i18n>A cluster with the chosen user is already connected.
+            </span>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="remoteClusterForm.getValue('showToken') && action !== 'edit'">
+          <label class="cd-col-form-label required"
+                 for="clusterFsid"
+                 i18n>Cluster FSID
+          </label>
+          <div class="cd-col-form-input">
+            <input id="clusterFsid"
+                   name="clusterFsid"
+                   class="form-control"
+                   type="text"
+                   formControlName="clusterFsid">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('clusterFsid', frm, 'required')"
+                  i18n>This field is required.
+            </span>
           </div>
         </div>
         <div class="form-group row"
-             *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError">
+             *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError && action !== 'edit'">
           <label class="cd-col-form-label required"
                  for="password"
                  i18n>Password
@@ -102,23 +128,8 @@
             </span>
           </div>
         </div>
-        <div class="form-group row">
-          <div class="cd-col-form-offset">
-            <div class="custom-control custom-checkbox">
-              <input class="custom-control-input"
-                     id="showToken"
-                     type="checkbox"
-                     (click)="showToken = !showToken"
-                     formControlName="showToken"
-                     [readonly]="true">
-              <label class="custom-control-label"
-                     for="showToken"
-                     i18n>Auth with token</label>
-            </div>
-          </div>
-        </div>
         <div class="form-group row"
-             *ngIf="remoteClusterForm.getValue('showToken')">
+             *ngIf="remoteClusterForm.getValue('showToken') && action !== 'edit'">
           <label class="cd-col-form-label required"
                  for="apiToken"
                  i18n>Token
@@ -135,7 +146,23 @@
           </div>
         </div>
         <div class="form-group row"
-             *ngIf="!showCrossOriginError">
+             *ngIf="action !== 'edit'">
+          <div class="cd-col-form-offset">
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="showToken"
+                     type="checkbox"
+                     [checked]="showToken"
+                     (change)="toggleToken()"
+                     formControlName="showToken">
+              <label class="custom-control-label"
+                     for="showToken"
+                     i18n>Auth with token</label>
+            </div>
+          </div>
+        </div>
+        <div class="form-group row"
+             *ngIf="!showCrossOriginError && action !== 'edit' && !remoteClusterForm.getValue('showToken')">
           <div class="cd-col-form-offset">
             <div class="custom-control">
               <button class="btn btn-primary"
@@ -150,8 +177,7 @@
       </div>
       <div class="modal-footer">
         <cd-form-button-panel (submitActionEvent)="onSubmit()"
-                              [submitText]="actionLabels.CONNECT"
-                              [disabled]="!connectionVerified && !showCrossOriginError"
+                              [submitText]="(action | titlecase) + ' ' + 'Cluster'"
                               [form]="remoteClusterForm">
         </cd-form-button-panel>
       </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
index 473a49dab7f7..83eb9fb5d51e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
@@ -1,4 +1,4 @@
-import { Component, OnDestroy, OnInit } from '@angular/core';
+import { Component, EventEmitter, OnDestroy, OnInit, Output } from '@angular/core';
 import { FormControl, Validators } from '@angular/forms';
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
 import _ from 'lodash';
@@ -8,6 +8,7 @@ import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
 import { NotificationType } from '~/app/shared/enum/notification-type.enum';
 import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
 import { CdValidators } from '~/app/shared/forms/cd-validators';
+import { MultiCluster } from '~/app/shared/models/multi-cluster';
 import { NotificationService } from '~/app/shared/services/notification.service';
 
 @Component({
@@ -16,6 +17,8 @@ import { NotificationService } from '~/app/shared/services/notification.service'
   styleUrls: ['./multi-cluster-form.component.scss']
 })
 export class MultiClusterFormComponent implements OnInit, OnDestroy {
+  @Output()
+  submitAction = new EventEmitter();
   readonly endpoints = /^((https?:\/\/)|(www.))(?:([a-zA-Z]+)|(\d+\.\d+.\d+.\d+)):\d{2,5}\/?$/;
   readonly ipv4Rgx = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/i;
   readonly ipv6Rgx = /^(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4}$/i;
@@ -26,6 +29,13 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
   private subs = new Subscription();
   showCrossOriginError = false;
   crossOriginCmd: string;
+  action: string;
+  cluster: MultiCluster;
+  clustersData: MultiCluster[];
+  clusterAliasNames: string[];
+  clusterUrls: string[];
+  clusterUsers: string[];
+  clusterUrlUserMap: Map<string, string>;
 
   constructor(
     public activeModal: NgbActiveModal,
@@ -35,21 +45,57 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
   ) {
     this.createForm();
   }
-  ngOnInit(): void {}
+  ngOnInit(): void {
+    if (this.action === 'edit') {
+      this.remoteClusterForm.get('remoteClusterUrl').setValue(this.cluster.url);
+      this.remoteClusterForm.get('remoteClusterUrl').disable();
+      this.remoteClusterForm.get('clusterAlias').setValue(this.cluster.cluster_alias);
+    }
+    if (this.action === 'reconnect') {
+      this.remoteClusterForm.get('remoteClusterUrl').setValue(this.cluster.url);
+      this.remoteClusterForm.get('remoteClusterUrl').disable();
+      this.remoteClusterForm.get('clusterAlias').setValue(this.cluster.cluster_alias);
+      this.remoteClusterForm.get('clusterAlias').disable();
+      this.remoteClusterForm.get('username').setValue(this.cluster.user);
+      this.remoteClusterForm.get('username').disable();
+      this.remoteClusterForm.get('clusterFsid').setValue(this.cluster.name);
+      this.remoteClusterForm.get('clusterFsid').disable();
+    }
+    [this.clusterAliasNames, this.clusterUrls, this.clusterUsers] = [
+      'cluster_alias',
+      'url',
+      'user'
+    ].map((prop) => this.clustersData?.map((cluster) => cluster[prop]));
+  }
 
   createForm() {
     this.remoteClusterForm = new CdFormGroup({
       showToken: new FormControl(false),
       username: new FormControl('', [
-        CdValidators.requiredIf({
-          showToken: false
+        CdValidators.custom('uniqueUrlandUser', (username: string) => {
+          let remoteClusterUrl = '';
+          if (
+            this.remoteClusterForm &&
+            this.remoteClusterForm.getValue('remoteClusterUrl') &&
+            this.remoteClusterForm.getValue('remoteClusterUrl').endsWith('/')
+          ) {
+            remoteClusterUrl = this.remoteClusterForm.getValue('remoteClusterUrl').slice(0, -1);
+          } else if (this.remoteClusterForm) {
+            remoteClusterUrl = this.remoteClusterForm.getValue('remoteClusterUrl');
+          }
+          return (
+            this.remoteClusterForm &&
+            this.clusterUrls?.includes(remoteClusterUrl) &&
+            this.clusterUsers?.includes(username)
+          );
         })
       ]),
-      password: new FormControl('', [
+      clusterFsid: new FormControl('', [
         CdValidators.requiredIf({
-          showToken: false
+          showToken: true
         })
       ]),
+      password: new FormControl('', []),
       remoteClusterUrl: new FormControl(null, {
         validators: [
           CdValidators.custom('endpoint', (value: string) => {
@@ -71,8 +117,17 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
           showToken: true
         })
       ]),
-      clusterAlias: new FormControl('', {
-        validators: [Validators.required]
+      clusterAlias: new FormControl(null, {
+        validators: [
+          Validators.required,
+          CdValidators.custom('uniqueName', (clusterAlias: string) => {
+            return (
+              (this.action === 'connect' || this.action === 'edit') &&
+              this.clusterAliasNames &&
+              this.clusterAliasNames.indexOf(clusterAlias) !== -1
+            );
+          })
+        ]
       })
     });
   }
@@ -83,27 +138,78 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
 
   onSubmit() {
     const url = this.remoteClusterForm.getValue('remoteClusterUrl');
+    const updatedUrl = url.endsWith('/') ? url.slice(0, -1) : url;
     const clusterAlias = this.remoteClusterForm.getValue('clusterAlias');
     const username = this.remoteClusterForm.getValue('username');
     const password = this.remoteClusterForm.getValue('password');
     const token = this.remoteClusterForm.getValue('apiToken');
+    const clusterFsid = this.remoteClusterForm.getValue('clusterFsid');
 
-    this.subs.add(
-      this.multiClusterService
-        .addCluster(url, clusterAlias, username, password, token, window.location.origin)
-        .subscribe({
+    if (this.action === 'edit') {
+      this.subs.add(
+        this.multiClusterService
+          .editCluster(this.cluster.url, clusterAlias, this.cluster.user)
+          .subscribe({
+            error: () => {
+              this.remoteClusterForm.setErrors({ cdSubmitButton: true });
+            },
+            complete: () => {
+              this.notificationService.show(
+                NotificationType.success,
+                $localize`Cluster updated successfully`
+              );
+              this.submitAction.emit();
+              this.activeModal.close();
+            }
+          })
+      );
+    }
+
+    if (this.action === 'reconnect') {
+      this.subs.add(
+        this.multiClusterService.reConnectCluster(updatedUrl, username, password, token).subscribe({
           error: () => {
             this.remoteClusterForm.setErrors({ cdSubmitButton: true });
           },
           complete: () => {
             this.notificationService.show(
               NotificationType.success,
-              $localize`Cluster added successfully`
+              $localize`Cluster reconnected successfully`
             );
+            this.submitAction.emit();
             this.activeModal.close();
           }
         })
-    );
+      );
+    }
+
+    if (this.action === 'connect') {
+      this.subs.add(
+        this.multiClusterService
+          .addCluster(
+            updatedUrl,
+            clusterAlias,
+            username,
+            password,
+            token,
+            window.location.origin,
+            clusterFsid
+          )
+          .subscribe({
+            error: () => {
+              this.remoteClusterForm.setErrors({ cdSubmitButton: true });
+            },
+            complete: () => {
+              this.notificationService.show(
+                NotificationType.success,
+                $localize`Cluster connected successfully`
+              );
+              this.submitAction.emit();
+              this.activeModal.close();
+            }
+          })
+      );
+    }
   }
 
   verifyConnection() {
@@ -149,4 +255,8 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
         })
     );
   }
+
+  toggleToken() {
+    this.showToken = !this.showToken;
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
new file mode 100644
index 000000000000..7aea2f470761
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
@@ -0,0 +1,37 @@
+<nav ngbNav
+     #nav="ngbNav"
+     class="nav-tabs">
+  <ng-container ngbNavItem>
+    <a ngbNavLink
+       i18n>Clusters List</a>
+    <ng-template ngbNavContent>
+      <cd-table #table
+                [data]="data"
+                [columns]="columns"
+                columnMode="flex"
+                selectionType="single"
+                [maxLimit]="25"
+                (updateSelection)="updateSelection($event)">
+        <div class="table-actions btn-toolbar">
+          <cd-table-actions [permission]="permissions.user"
+                            [selection]="selection"
+                            class="btn-group"
+                            id="cluster-actions"
+                            [tableActions]="tableActions">
+          </cd-table-actions>
+        </div>
+      </cd-table>
+    </ng-template>
+  </ng-container>
+</nav>
+
+<ng-template #urlTpl
+             let-row="row">
+  <a target="_blank"
+     [href]="row.url">
+      {{ row.url.endsWith('/') ? row.url.slice(0, -1) : row.url }}
+    <i class="fa fa-external-link"></i>
+  </a>
+</ng-template>
+
+<div [ngbNavOutlet]="nav"></div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.spec.ts
new file mode 100644
index 000000000000..d69b3a4bb8ca
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.spec.ts
@@ -0,0 +1,30 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { ToastrModule } from 'ngx-toastr';
+import { NgbNavModule } from '@ng-bootstrap/ng-bootstrap';
+
+import { MultiClusterListComponent } from './multi-cluster-list.component';
+import { CdDatePipe } from '~/app/shared/pipes/cd-date.pipe';
+import { TableActionsComponent } from '~/app/shared/datatable/table-actions/table-actions.component';
+import { SharedModule } from '~/app/shared/shared.module';
+
+describe('MultiClusterListComponent', () => {
+  let component: MultiClusterListComponent;
+  let fixture: ComponentFixture<MultiClusterListComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [HttpClientTestingModule, ToastrModule.forRoot(), NgbNavModule, SharedModule],
+      declarations: [MultiClusterListComponent],
+      providers: [CdDatePipe, TableActionsComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(MultiClusterListComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
new file mode 100644
index 000000000000..4496b3a34ceb
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
@@ -0,0 +1,213 @@
+import { Component, TemplateRef, ViewChild } from '@angular/core';
+import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
+import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { CdTableAction } from '~/app/shared/models/cd-table-action';
+import { CdTableColumn } from '~/app/shared/models/cd-table-column';
+import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
+import { ModalService } from '~/app/shared/services/modal.service';
+import { MultiClusterFormComponent } from '../multi-cluster-form/multi-cluster-form.component';
+import { TableComponent } from '~/app/shared/datatable/table/table.component';
+import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
+import { Permissions } from '~/app/shared/models/permissions';
+import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component';
+import { NotificationService } from '~/app/shared/services/notification.service';
+import { NotificationType } from '~/app/shared/enum/notification-type.enum';
+import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+import { MultiCluster } from '~/app/shared/models/multi-cluster';
+import { SummaryService } from '~/app/shared/services/summary.service';
+import { Router } from '@angular/router';
+
+@Component({
+  selector: 'cd-multi-cluster-list',
+  templateUrl: './multi-cluster-list.component.html',
+  styleUrls: ['./multi-cluster-list.component.scss']
+})
+export class MultiClusterListComponent {
+  @ViewChild(TableComponent)
+  table: TableComponent;
+  @ViewChild('urlTpl', { static: true })
+  public urlTpl: TemplateRef<any>;
+
+  permissions: Permissions;
+  tableActions: CdTableAction[];
+  clusterTokenStatus: object = {};
+  columns: Array<CdTableColumn> = [];
+  data: any;
+  selection = new CdTableSelection();
+  bsModalRef: NgbModalRef;
+  clustersTokenMap: Map<string, string> = new Map<string, string>();
+  newData: any;
+  modalRef: NgbModalRef;
+
+  constructor(
+    private multiClusterService: MultiClusterService,
+    private router: Router,
+    private summaryService: SummaryService,
+    public actionLabels: ActionLabelsI18n,
+    private notificationService: NotificationService,
+    private authStorageService: AuthStorageService,
+    private modalService: ModalService
+  ) {
+    this.tableActions = [
+      {
+        permission: 'create',
+        icon: Icons.add,
+        name: this.actionLabels.CONNECT,
+        click: () => this.openRemoteClusterInfoModal('connect')
+      },
+      {
+        permission: 'update',
+        icon: Icons.edit,
+        name: this.actionLabels.EDIT,
+        disable: (selection: CdTableSelection) => this.getDisable('edit', selection),
+        click: () => this.openRemoteClusterInfoModal('edit')
+      },
+      {
+        permission: 'update',
+        icon: Icons.refresh,
+        name: this.actionLabels.RECONNECT,
+        disable: (selection: CdTableSelection) => this.getDisable('reconnect', selection),
+        click: () => this.openRemoteClusterInfoModal('reconnect')
+      },
+      {
+        permission: 'delete',
+        icon: Icons.destroy,
+        name: this.actionLabels.DISCONNECT,
+        disable: (selection: CdTableSelection) => this.getDisable('disconnect', selection),
+        click: () => this.openDeleteClusterModal()
+      }
+    ];
+    this.permissions = this.authStorageService.getPermissions();
+  }
+
+  ngOnInit(): void {
+    this.multiClusterService.subscribe((resp: object) => {
+      if (resp && resp['config']) {
+        const clusterDetailsArray = Object.values(resp['config']).flat();
+        this.data = clusterDetailsArray;
+        this.checkClusterConnectionStatus();
+      }
+    });
+
+    this.columns = [
+      {
+        prop: 'cluster_alias',
+        name: $localize`Alias`,
+        flexGrow: 2
+      },
+      {
+        prop: 'cluster_connection_status',
+        name: $localize`Connection`,
+        flexGrow: 2,
+        cellTransformation: CellTemplate.badge,
+        customTemplateConfig: {
+          map: {
+            1: { value: 'DISCONNECTED', class: 'badge-danger' },
+            0: { value: 'CONNECTED', class: 'badge-success' },
+            2: { value: 'CHECKING..', class: 'badge-info' }
+          }
+        }
+      },
+      {
+        prop: 'name',
+        name: $localize`FSID`,
+        flexGrow: 2
+      },
+      {
+        prop: 'url',
+        name: $localize`URL`,
+        flexGrow: 2,
+        cellTemplate: this.urlTpl
+      },
+      {
+        prop: 'user',
+        name: $localize`User`,
+        flexGrow: 2
+      }
+    ];
+
+    this.multiClusterService.subscribeClusterTokenStatus((resp: object) => {
+      this.clusterTokenStatus = resp;
+      this.checkClusterConnectionStatus();
+    });
+  }
+
+  checkClusterConnectionStatus() {
+    if (this.clusterTokenStatus && this.data) {
+      this.data.forEach((cluster: MultiCluster) => {
+        const clusterStatus = this.clusterTokenStatus[cluster.name];
+
+        if (clusterStatus !== undefined) {
+          cluster.cluster_connection_status = clusterStatus.status;
+        } else {
+          cluster.cluster_connection_status = 2;
+        }
+
+        if (cluster.cluster_alias === 'local-cluster') {
+          cluster.cluster_connection_status = 0;
+        }
+      });
+    }
+  }
+
+  openRemoteClusterInfoModal(action: string) {
+    const initialState = {
+      clustersData: this.data,
+      action: action,
+      cluster: this.selection.first()
+    };
+    this.bsModalRef = this.modalService.show(MultiClusterFormComponent, initialState, {
+      size: 'xl'
+    });
+    this.bsModalRef.componentInstance.submitAction.subscribe(() => {
+      this.multiClusterService.refresh();
+      this.summaryService.refresh();
+      const currentRoute = this.router.url.split('?')[0];
+      if (currentRoute.includes('dashboard')) {
+        this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
+          this.router.navigate([currentRoute]);
+        });
+      } else {
+        this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
+          this.router.navigate([currentRoute]);
+        });
+      }
+    });
+  }
+
+  updateSelection(selection: CdTableSelection) {
+    this.selection = selection;
+  }
+
+  openDeleteClusterModal() {
+    const cluster = this.selection.first();
+    this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      actionDescription: $localize`Disconnect`,
+      itemDescription: $localize`Cluster`,
+      itemNames: [cluster['cluster_alias'] + ' - ' + cluster['user']],
+      submitAction: () =>
+        this.multiClusterService.deleteCluster(cluster['name'], cluster['user']).subscribe(() => {
+          this.modalRef.close();
+          this.notificationService.show(
+            NotificationType.success,
+            $localize`Disconnected cluster '${cluster['cluster_alias']}'`
+          );
+        })
+    });
+  }
+
+  getDisable(action: string, selection: CdTableSelection): string | boolean {
+    if (!selection.hasSelection) {
+      return $localize`Please select one or more clusters to ${action}`;
+    }
+    if (selection.hasSingleSelection) {
+      const cluster = selection.first();
+      if (cluster['cluster_alias'] === 'local-cluster') {
+        return $localize`Cannot ${action} local cluster`;
+      }
+    }
+    return false;
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
index 5009909ea3fa..c826f155c40b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
@@ -46,16 +46,6 @@ <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortles
   </span>
   <div *ngIf="dashboardClustersMap?.size > 1">
     <div *ngIf="!loading">
-      <div class="mt-4">
-        <div class="text-center">
-          <button class="btn btn-primary"
-                  (click)="openRemoteClusterInfoModal()">
-            <i class="mx-auto"
-               [ngClass]="icons.add">
-            </i> Connect Cluster
-          </button>
-        </div>
-      </div>
     </div>
   </div>
 </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
index 2630c839a424..dbbf10e74848 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
@@ -50,7 +50,10 @@ export class MultiClusterComponent implements OnInit {
   }
 
   openRemoteClusterInfoModal() {
-    this.bsModalRef = this.modalService.show(MultiClusterFormComponent, {
+    const initialState = {
+      action: 'connect'
+    };
+    this.bsModalRef = this.modalService.show(MultiClusterFormComponent, initialState, {
       size: 'xl'
     });
   }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
index 1d7c4bb751cb..8ddbddf2fe81 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.ts
@@ -27,6 +27,7 @@ export class WorkbenchLayoutComponent implements OnInit, OnDestroy {
 
   ngOnInit() {
     this.subs.add(this.multiClusterService.startPolling());
+    this.subs.add(this.multiClusterService.startClusterTokenStatusPolling());
     this.subs.add(this.summaryService.startPolling());
     this.subs.add(this.taskManagerService.init(this.summaryService));
     this.faviconService.init();
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
index 6af3799b4ef8..8f2633ed0a15 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html
@@ -44,7 +44,8 @@
             <div ngbDropdownMenu>
               <ng-container *ngFor="let cluster of clustersMap | keyvalue">
                 <button ngbDropdownItem
-                        (click)="onClusterSelection(cluster.value)">
+                        (click)="onClusterSelection(cluster.value)"
+                        [disabled]="cluster.value.cluster_connection_status === 1">
                   <div class="dropdown-text">{{ cluster.value.name }}</div>
                   <div *ngIf="cluster.value.cluster_alias"
                        class="text-secondary">{{ cluster.value.cluster_alias }} - {{ cluster.value.user }}</div>
@@ -133,7 +134,12 @@
           <li routerLinkActive="active"
               class="tc_submenuitem tc_submenuitem_multiCluster_overview">
             <a i18n
-               routerLink="/multi-cluster">Overview</a>
+               routerLink="/multi-cluster/overview">Overview</a>
+          </li>
+          <li routerLinkActive="active"
+              class="tc_submenuitem tc_submenuitem_multiCluster_manage_clusters">
+            <a i18n
+               routerLink="/multi-cluster/manage-clusters">Manage Clusters</a>
           </li>
         </ul>
       </li>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index 10963042d25f..4ae8d1897e27 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -6,6 +6,7 @@ import { Subscription } from 'rxjs';
 import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
 
 import { Icons } from '~/app/shared/enum/icons.enum';
+import { MultiCluster } from '~/app/shared/models/multi-cluster';
 import { Permissions } from '~/app/shared/models/permissions';
 import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
 import {
@@ -31,6 +32,7 @@ export class NavigationComponent implements OnInit, OnDestroy {
 
   permissions: Permissions;
   enabledFeature$: FeatureTogglesMap$;
+  clusterTokenStatus: object = {};
   summaryData: any;
   icons = Icons;
 
@@ -62,22 +64,17 @@ export class NavigationComponent implements OnInit, OnDestroy {
 
   ngOnInit() {
     this.subs.add(
-      this.multiClusterService.subscribe((resp: any) => {
+      this.multiClusterService.subscribe((resp: object) => {
         const clustersConfig = resp['config'];
         if (clustersConfig) {
           Object.keys(clustersConfig).forEach((clusterKey: string) => {
             const clusterDetailsList = clustersConfig[clusterKey];
-            clusterDetailsList.forEach((clusterDetails: any) => {
-              const clusterName = clusterDetails['name'];
+            clusterDetailsList.forEach((clusterDetails: MultiCluster) => {
               const clusterUser = clusterDetails['user'];
               const clusterUrl = clusterDetails['url'];
               const clusterUniqueKey = `${clusterUrl}-${clusterUser}`;
-              this.clustersMap.set(clusterUniqueKey, {
-                name: clusterName,
-                cluster_alias: clusterDetails['cluster_alias'],
-                user: clusterDetails['user'],
-                url: clusterUrl
-              });
+              this.clustersMap.set(clusterUniqueKey, clusterDetails);
+              this.checkClusterConnectionStatus();
             });
           });
           this.selectedCluster =
@@ -111,12 +108,40 @@ export class NavigationComponent implements OnInit, OnDestroy {
         this.showTopNotification('motdNotificationEnabled', _.isPlainObject(motd));
       })
     );
+    this.subs.add(
+      this.multiClusterService.subscribeClusterTokenStatus((resp: object) => {
+        this.clusterTokenStatus = resp;
+        this.checkClusterConnectionStatus();
+      })
+    );
   }
 
   ngOnDestroy(): void {
     this.subs.unsubscribe();
   }
 
+  checkClusterConnectionStatus() {
+    this.clustersMap.forEach((clusterDetails, clusterName) => {
+      const clusterTokenStatus = this.clusterTokenStatus[clusterDetails.name];
+      const connectionStatus = clusterTokenStatus ? clusterTokenStatus.status : 0;
+      const user = clusterTokenStatus ? clusterTokenStatus.user : clusterDetails.user;
+
+      this.clustersMap.set(clusterName, {
+        ...clusterDetails,
+        cluster_connection_status: connectionStatus,
+        user: user
+      });
+
+      if (clusterDetails.cluster_alias === 'local-cluster') {
+        this.clustersMap.set(clusterName, {
+          ...clusterDetails,
+          cluster_connection_status: 0,
+          user: user
+        });
+      }
+    });
+  }
+
   blockHealthColor() {
     if (this.summaryData && this.summaryData.rbd_mirroring) {
       if (this.summaryData.rbd_mirroring.errors > 0) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
index 5a17645092df..7252e969e60a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
@@ -1,6 +1,6 @@
-import { HttpClient } from '@angular/common/http';
+import { HttpClient, HttpParams } from '@angular/common/http';
 import { Injectable } from '@angular/core';
-import { BehaviorSubject, Subscription } from 'rxjs';
+import { BehaviorSubject, Observable, Subscription } from 'rxjs';
 import { TimerService } from '../services/timer.service';
 import { filter } from 'rxjs/operators';
 
@@ -8,8 +8,11 @@ import { filter } from 'rxjs/operators';
   providedIn: 'root'
 })
 export class MultiClusterService {
+  TOKEN_CHECK_INTERVAL = 600000; // 10m interval
   private msSource = new BehaviorSubject<any>(null);
   msData$ = this.msSource.asObservable();
+  private tokenStatusSource = new BehaviorSubject<any>(null);
+  tokenStatusSource$ = this.tokenStatusSource.asObservable();
   constructor(private http: HttpClient, private timerService: TimerService) {}
 
   startPolling(): Subscription {
@@ -18,6 +21,44 @@ export class MultiClusterService {
       .subscribe(this.getClusterObserver());
   }
 
+  startClusterTokenStatusPolling() {
+    let clustersTokenMap = new Map<string, { token: string; user: string }>();
+    const dataSubscription = this.subscribe((resp: any) => {
+      const clustersConfig = resp['config'];
+      const tempMap = new Map<string, { token: string; user: string }>();
+      if (clustersConfig) {
+        Object.keys(clustersConfig).forEach((clusterKey: string) => {
+          const clusterDetailsList = clustersConfig[clusterKey];
+          clusterDetailsList.forEach((clusterDetails: any) => {
+            if (clusterDetails['token'] && clusterDetails['name'] && clusterDetails['user']) {
+              tempMap.set(clusterDetails['name'], {
+                token: clusterDetails['token'],
+                user: clusterDetails['user']
+              });
+            }
+          });
+        });
+
+        if (tempMap.size > 0) {
+          clustersTokenMap = tempMap;
+          dataSubscription.unsubscribe();
+          this.checkAndStartTimer(clustersTokenMap);
+        }
+      }
+    });
+  }
+
+  private checkAndStartTimer(clustersTokenMap: Map<string, { token: string; user: string }>) {
+    this.checkTokenStatus(clustersTokenMap).subscribe(this.getClusterTokenStatusObserver());
+    this.timerService
+      .get(() => this.checkTokenStatus(clustersTokenMap), this.TOKEN_CHECK_INTERVAL)
+      .subscribe(this.getClusterTokenStatusObserver());
+  }
+
+  subscribeClusterTokenStatus(next: (data: any) => void, error?: (error: any) => void) {
+    return this.tokenStatusSource$.pipe(filter((value) => !!value)).subscribe(next, error);
+  }
+
   refresh(): Subscription {
     return this.getCluster().subscribe(this.getClusterObserver());
   }
@@ -34,13 +75,26 @@ export class MultiClusterService {
     return this.http.get('api/multi-cluster/get_config');
   }
 
+  deleteCluster(clusterName: string, clusterUser: string): Observable<any> {
+    return this.http.delete(`api/multi-cluster/delete_cluster/${clusterName}/${clusterUser}`);
+  }
+
+  editCluster(url: any, clusterAlias: string, username: string) {
+    return this.http.put('api/multi-cluster/edit_cluster', {
+      url,
+      cluster_alias: clusterAlias,
+      username
+    });
+  }
+
   addCluster(
     url: any,
     clusterAlias: string,
     username: string,
     password: string,
     token = '',
-    hub_url = ''
+    hub_url = '',
+    clusterFsid = ''
   ) {
     return this.http.post('api/multi-cluster/auth', {
       url,
@@ -48,12 +102,13 @@ export class MultiClusterService {
       username,
       password,
       token,
-      hub_url
+      hub_url,
+      cluster_fsid: clusterFsid
     });
   }
 
-  verifyConnection(url: string, username: string, password: string, token = '') {
-    return this.http.post('api/multi-cluster/verify_connection', {
+  reConnectCluster(url: any, username: string, password: string, token = '') {
+    return this.http.put('api/multi-cluster/reconnect_cluster', {
       url,
       username,
       password,
@@ -61,9 +116,36 @@ export class MultiClusterService {
     });
   }
 
+  verifyConnection(url: string, username: string, password: string, token = ''): Observable<any> {
+    let params = new HttpParams()
+      .set('url', url)
+      .set('username', username)
+      .set('password', password)
+      .set('token', token);
+
+    return this.http.get('api/multi-cluster/verify_connection', { params });
+  }
+
   private getClusterObserver() {
     return (data: any) => {
       this.msSource.next(data);
     };
   }
+
+  private getClusterTokenStatusObserver() {
+    return (data: any) => {
+      this.tokenStatusSource.next(data);
+    };
+  }
+
+  checkTokenStatus(
+    clustersTokenMap: Map<string, { token: string; user: string }>
+  ): Observable<object> {
+    let data = [...clustersTokenMap].map(([key, { token, user }]) => ({ name: key, token, user }));
+
+    let params = new HttpParams();
+    params = params.set('clustersTokenMap', JSON.stringify(data));
+
+    return this.http.get<object>('api/multi-cluster/check_token_status', { params });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index df007c93d8a6..876e22bbc1b6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -145,6 +145,8 @@ export class ActionLabelsI18n {
   DEACTIVATE: string;
   ATTACH: string;
   CONNECT: string;
+  DISCONNECT: string;
+  RECONNECT: string;
 
   constructor() {
     /* Create a new item */
@@ -228,6 +230,8 @@ export class ActionLabelsI18n {
 
     this.ATTACH = $localize`Attach`;
     this.CONNECT = $localize`Connect`;
+    this.DISCONNECT = $localize`Disconnect`;
+    this.RECONNECT = $localize`Reconnect`;
   }
 }
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts
new file mode 100644
index 000000000000..ce4e02603f83
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts
@@ -0,0 +1,8 @@
+export interface MultiCluster {
+  name: string;
+  url: string;
+  user: string;
+  token: string;
+  cluster_alias: string;
+  cluster_connection_status: number;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
index e4e30d6a3682..5cbd99911fb2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
@@ -76,7 +76,11 @@ export class ApiInterceptorService implements HttpInterceptor {
       });
     }
 
-    const apiUrl = localStorage.getItem('cluster_api_url');
+    let apiUrl = localStorage.getItem('cluster_api_url');
+
+    if (apiUrl && !apiUrl.endsWith('/')) {
+      apiUrl += '/';
+    }
     const currentRoute = this.router.url.split('?')[0];
 
     const ALWAYS_TO_HUB_APIs = [
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 912718d9ee77..4e58517a9636 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -6968,6 +6968,8 @@ paths:
               properties:
                 cluster_alias:
                   type: string
+                cluster_fsid:
+                  type: string
                 hub_url:
                   type: string
                 password:
@@ -7007,9 +7009,14 @@ paths:
       summary: Authenticate to a remote cluster
       tags:
       - Multi-cluster
-  /api/multi-cluster/get_config:
+  /api/multi-cluster/check_token_status:
     get:
-      parameters: []
+      parameters:
+      - allowEmptyValue: true
+        in: query
+        name: clustersTokenMap
+        schema:
+          type: string
       responses:
         '200':
           content:
@@ -7029,7 +7036,44 @@ paths:
       - jwt: []
       tags:
       - Multi-cluster
-  /api/multi-cluster/set_config:
+  /api/multi-cluster/delete_cluster/{cluster_name}/{cluster_user}:
+    delete:
+      parameters:
+      - in: path
+        name: cluster_name
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: cluster_user
+        required: true
+        schema:
+          type: string
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/edit_cluster:
     put:
       parameters: []
       requestBody:
@@ -7037,10 +7081,16 @@ paths:
           application/json:
             schema:
               properties:
-                config:
+                cluster_alias:
+                  type: string
+                url:
+                  type: string
+                username:
                   type: string
               required:
-              - config
+              - url
+              - cluster_alias
+              - username
               type: object
       responses:
         '200':
@@ -7066,8 +7116,30 @@ paths:
       - jwt: []
       tags:
       - Multi-cluster
-  /api/multi-cluster/verify_connection:
-    post:
+  /api/multi-cluster/get_config:
+    get:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/reconnect_cluster:
+    put:
       parameters: []
       requestBody:
         content:
@@ -7086,11 +7158,48 @@ paths:
               - url
               type: object
       responses:
-        '201':
+        '200':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: Resource created.
+          description: Resource updated.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
+  /api/multi-cluster/set_config:
+    put:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                config:
+                  type: string
+              required:
+              - config
+              type: object
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource updated.
         '202':
           content:
             application/vnd.ceph.api.v1.0+json:
@@ -7109,6 +7218,48 @@ paths:
       - jwt: []
       tags:
       - Multi-cluster
+  /api/multi-cluster/verify_connection:
+    get:
+      parameters:
+      - allowEmptyValue: true
+        in: query
+        name: url
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: username
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: password
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: token
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Multi-cluster
   /api/nfs-ganesha/cluster:
     get:
       parameters: []

From b469cbb507675108cecfcfd9da3bd5144deabcf9 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 22 Feb 2024 17:03:43 +0800
Subject: [PATCH 1993/2492] crimson/os/seastore: fix wrong-positioned asserts

Fixes: https://tracker.ceph.com/issues/64535
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index eba19eabebeb..85a680b2ee2f 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -725,12 +725,12 @@ SeaStore::Shard::list_objects(CollectionRef ch,
 		  LOG_PREFIX(SeaStore::list_objects);
 		  DEBUGT("got {} objects, left limit {}",
 		    t, next_objects.size(), limit);
-		  if (last && std::get<1>(ret) == pend) {
-		    std::get<1>(ret) = end;
-		  }
 		  assert(limit == 0 ||
 			 std::get<1>(ret) == pend ||
 			 std::get<1>(ret) == ghobject_t::get_max());
+		  if (last && std::get<1>(ret) == pend) {
+		    std::get<1>(ret) = end;
+		  }
 		  return list_iertr::make_ready_future<
 		    seastar::stop_iteration
 		    >(seastar::stop_iteration::no);

From ab239ad6cc5eb1cb4ca71d5b409bc4001124342f Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 22 Feb 2024 09:35:14 +0000
Subject: [PATCH 1994/2492] crimson/osd/main: Enable multicore cluster
 messenger

Taken from: f78e99c

Co-authored-by: Yingxin Cheng <yingxin.cheng@intel.com>
Co-authored-by: Chunmei Liu <chunmei.liu@intel.com>
Co-authored-by: Xinyu Huang <xinyu.huang@intel.com>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/main.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc
index 6a569702deba..fa387804dcda 100644
--- a/src/crimson/osd/main.cc
+++ b/src/crimson/osd/main.cc
@@ -186,14 +186,14 @@ int main(int argc, const char* argv[])
           const auto nonce = crimson::osd::get_nonce();
           crimson::net::MessengerRef cluster_msgr, client_msgr;
           crimson::net::MessengerRef hb_front_msgr, hb_back_msgr;
-          for (auto [msgr, name] : {make_pair(std::ref(client_msgr), "client"s)}) {
+          for (auto [msgr, name] : {make_pair(std::ref(client_msgr), "client"s),
+                                    make_pair(std::ref(cluster_msgr), "cluster"s)}) {
             msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),
                                                    name,
                                                    nonce,
                                                    false);
           }
-          for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s),
-                                    make_pair(std::ref(hb_front_msgr), "hb_front"s),
+          for (auto [msgr, name] : {make_pair(std::ref(hb_front_msgr), "hb_front"s),
                                     make_pair(std::ref(hb_back_msgr), "hb_back"s)}) {
             msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),
                                                    name,

From 6f28930602a2dfdcd402537c1a274df1836109ce Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 22 Feb 2024 09:58:00 +0000
Subject: [PATCH 1995/2492] test/rgw/notifications: split tests between basic,
 kafka and amqp

see comment: https://tracker.ceph.com/issues/64184#note-3

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 qa/suites/rgw/notifications/centos_latest.yaml      |  1 -
 qa/suites/rgw/notifications/tasks/{ => amqp}/+      |  0
 .../rgw/notifications/tasks/amqp/0-install.yaml     | 13 +++++++++++++
 .../rgw/notifications/tasks/amqp/centos_latest.yaml |  1 +
 .../notifications/tasks/{ => amqp}/test_amqp.yaml   |  0
 qa/suites/rgw/notifications/tasks/kafka/+           |  0
 .../notifications/tasks/{ => kafka}/0-install.yaml  |  0
 .../rgw/notifications/tasks/kafka/supported-distros |  1 +
 .../notifications/tasks/{ => kafka}/test_kafka.yaml |  0
 qa/suites/rgw/notifications/tasks/others/+          |  0
 .../rgw/notifications/tasks/others/0-install.yaml   | 13 +++++++++++++
 .../notifications/tasks/others/supported-distros    |  1 +
 .../tasks/{ => others}/test_others.yaml             |  0
 13 files changed, 29 insertions(+), 1 deletion(-)
 delete mode 120000 qa/suites/rgw/notifications/centos_latest.yaml
 rename qa/suites/rgw/notifications/tasks/{ => amqp}/+ (100%)
 create mode 100644 qa/suites/rgw/notifications/tasks/amqp/0-install.yaml
 create mode 120000 qa/suites/rgw/notifications/tasks/amqp/centos_latest.yaml
 rename qa/suites/rgw/notifications/tasks/{ => amqp}/test_amqp.yaml (100%)
 create mode 100644 qa/suites/rgw/notifications/tasks/kafka/+
 rename qa/suites/rgw/notifications/tasks/{ => kafka}/0-install.yaml (100%)
 create mode 120000 qa/suites/rgw/notifications/tasks/kafka/supported-distros
 rename qa/suites/rgw/notifications/tasks/{ => kafka}/test_kafka.yaml (100%)
 create mode 100644 qa/suites/rgw/notifications/tasks/others/+
 create mode 100644 qa/suites/rgw/notifications/tasks/others/0-install.yaml
 create mode 120000 qa/suites/rgw/notifications/tasks/others/supported-distros
 rename qa/suites/rgw/notifications/tasks/{ => others}/test_others.yaml (100%)

diff --git a/qa/suites/rgw/notifications/centos_latest.yaml b/qa/suites/rgw/notifications/centos_latest.yaml
deleted file mode 120000
index bd9854e70298..000000000000
--- a/qa/suites/rgw/notifications/centos_latest.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/distros/supported/centos_latest.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/notifications/tasks/+ b/qa/suites/rgw/notifications/tasks/amqp/+
similarity index 100%
rename from qa/suites/rgw/notifications/tasks/+
rename to qa/suites/rgw/notifications/tasks/amqp/+
diff --git a/qa/suites/rgw/notifications/tasks/amqp/0-install.yaml b/qa/suites/rgw/notifications/tasks/amqp/0-install.yaml
new file mode 100644
index 000000000000..9e7852d4d25f
--- /dev/null
+++ b/qa/suites/rgw/notifications/tasks/amqp/0-install.yaml
@@ -0,0 +1,13 @@
+tasks:
+- install:
+- ceph:
+- openssl_keys:
+- rgw:
+    client.0:
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
diff --git a/qa/suites/rgw/notifications/tasks/amqp/centos_latest.yaml b/qa/suites/rgw/notifications/tasks/amqp/centos_latest.yaml
new file mode 120000
index 000000000000..a7df972290a4
--- /dev/null
+++ b/qa/suites/rgw/notifications/tasks/amqp/centos_latest.yaml
@@ -0,0 +1 @@
+../../.qa/distros/supported/centos_latest.yaml
\ No newline at end of file
diff --git a/qa/suites/rgw/notifications/tasks/test_amqp.yaml b/qa/suites/rgw/notifications/tasks/amqp/test_amqp.yaml
similarity index 100%
rename from qa/suites/rgw/notifications/tasks/test_amqp.yaml
rename to qa/suites/rgw/notifications/tasks/amqp/test_amqp.yaml
diff --git a/qa/suites/rgw/notifications/tasks/kafka/+ b/qa/suites/rgw/notifications/tasks/kafka/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/notifications/tasks/0-install.yaml b/qa/suites/rgw/notifications/tasks/kafka/0-install.yaml
similarity index 100%
rename from qa/suites/rgw/notifications/tasks/0-install.yaml
rename to qa/suites/rgw/notifications/tasks/kafka/0-install.yaml
diff --git a/qa/suites/rgw/notifications/tasks/kafka/supported-distros b/qa/suites/rgw/notifications/tasks/kafka/supported-distros
new file mode 120000
index 000000000000..46280a42a96d
--- /dev/null
+++ b/qa/suites/rgw/notifications/tasks/kafka/supported-distros
@@ -0,0 +1 @@
+../../.qa/distros/supported-random-distro$/
\ No newline at end of file
diff --git a/qa/suites/rgw/notifications/tasks/test_kafka.yaml b/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml
similarity index 100%
rename from qa/suites/rgw/notifications/tasks/test_kafka.yaml
rename to qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml
diff --git a/qa/suites/rgw/notifications/tasks/others/+ b/qa/suites/rgw/notifications/tasks/others/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rgw/notifications/tasks/others/0-install.yaml b/qa/suites/rgw/notifications/tasks/others/0-install.yaml
new file mode 100644
index 000000000000..9e7852d4d25f
--- /dev/null
+++ b/qa/suites/rgw/notifications/tasks/others/0-install.yaml
@@ -0,0 +1,13 @@
+tasks:
+- install:
+- ceph:
+- openssl_keys:
+- rgw:
+    client.0:
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
diff --git a/qa/suites/rgw/notifications/tasks/others/supported-distros b/qa/suites/rgw/notifications/tasks/others/supported-distros
new file mode 120000
index 000000000000..46280a42a96d
--- /dev/null
+++ b/qa/suites/rgw/notifications/tasks/others/supported-distros
@@ -0,0 +1 @@
+../../.qa/distros/supported-random-distro$/
\ No newline at end of file
diff --git a/qa/suites/rgw/notifications/tasks/test_others.yaml b/qa/suites/rgw/notifications/tasks/others/test_others.yaml
similarity index 100%
rename from qa/suites/rgw/notifications/tasks/test_others.yaml
rename to qa/suites/rgw/notifications/tasks/others/test_others.yaml

From d06f34c5e772bb834bd17a593801ad97e4596338 Mon Sep 17 00:00:00 2001
From: wanglinke <wanglinke@cmss.chinamobile.com>
Date: Thu, 22 Feb 2024 10:19:56 +0800
Subject: [PATCH 1996/2492] os/bluestore: fix bluefs perf counters about
 l_bluefs_log_compactions

In BlueFS::_compact_log_sync_LNF_LD,l_bluefs_log_compactions
is being counted two times.

Fixes: https://tracker.ceph.com/issues/64533
co-author: Jrchyang Yu <yuzhiqiang_yewu@cmss.chinamobile.com>
Signed-off-by: Wang Linke <wanglinke_yewu@cmss.chinamobile.com>
---
 src/os/bluestore/BlueFS.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index bb3fd8a99746..a8b1fb25ee85 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -2763,7 +2763,6 @@ void BlueFS::_rewrite_log_and_layout_sync_LNF_LD(bool permit_dev_fallback,
 
   // we're mostly done
   dout(10) << __func__ << " log extents " << log_file->fnode.extents << dendl;
-  logger->inc(l_bluefs_log_compactions);
 
   // Part 4
   // Finalization. Release old space.

From a0194583ad9bb290138163af30823aba588dfc34 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 13 Dec 2023 19:05:45 +0300
Subject: [PATCH 1997/2492] os/bluestore: introduce
 bluestore_debug_enforce_min_alloc_size config parameter.

This allows to override persistent min_alloc_size if needed.
This might be helpful to troubleshoot and work around issues like
https://tracker.ceph.com/issues/63618

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/common/options/global.yaml.in         | 15 +++++++++
 src/os/bluestore/BitmapFreelistManager.cc | 16 +++++++++
 src/os/bluestore/BitmapFreelistManager.h  |  2 ++
 src/os/bluestore/BlueStore.cc             | 40 +++++++++++++++--------
 src/os/bluestore/FreelistManager.h        |  2 ++
 5 files changed, 62 insertions(+), 13 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index ead0d0caedd3..8d506a4f6a3f 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4406,6 +4406,21 @@ options:
   flags:
   - create
   with_legacy: true
+- name: bluestore_debug_enforce_min_alloc_size
+  type: uint
+  level: dev
+  desc: Enforces specific min_alloc size usages
+  long_desc: This overrides actual min_alloc_size value persisted on mkfs
+    (and originally obtained from bluestore_min_alloc_size) and permits to
+    use arbitrary value for this value. Intended primarily for dev/debug
+    purposes and should be used with care and deep understanding of potential
+    consequences, e.g. data corruption.
+  default: 0
+  see_also:
+  - bluestore_min_alloc_size
+  flags:
+  - startup
+  with_legacy: true
 - name: bluestore_use_optimal_io_size_for_min_alloc_size 
   type: bool
   level: advanced
diff --git a/src/os/bluestore/BitmapFreelistManager.cc b/src/os/bluestore/BitmapFreelistManager.cc
index f1f4831d5671..cb2ecd689e17 100644
--- a/src/os/bluestore/BitmapFreelistManager.cc
+++ b/src/os/bluestore/BitmapFreelistManager.cc
@@ -610,3 +610,19 @@ void BitmapFreelistManager::get_meta(
   res->emplace_back("bfm_bytes_per_block", stringify(bytes_per_block));
   res->emplace_back("bfm_blocks_per_key", stringify(blocks_per_key));
 }
+
+bool BitmapFreelistManager::validate(uint64_t min_alloc_size) const
+{
+  bool ret = true;
+  auto my_alloc_size = get_alloc_size();
+  ceph_assert(my_alloc_size);
+  ceph_assert(min_alloc_size);
+  if (!is_null_manager() &&
+       ((min_alloc_size < my_alloc_size) || (min_alloc_size % my_alloc_size))) {
+    derr << __func__ << " inconsistent alloc units:" << std::hex
+         << "0x" << get_alloc_size() << " vs. 0x" << min_alloc_size
+         << std::dec << dendl;
+    ret = false;
+  }
+  return ret;
+}
diff --git a/src/os/bluestore/BitmapFreelistManager.h b/src/os/bluestore/BitmapFreelistManager.h
index 5b04e8fd28cc..30389056aa75 100644
--- a/src/os/bluestore/BitmapFreelistManager.h
+++ b/src/os/bluestore/BitmapFreelistManager.h
@@ -94,6 +94,8 @@ class BitmapFreelistManager : public FreelistManager {
   }
   void get_meta(uint64_t target_size,
     std::vector<std::pair<std::string, std::string>>*) const override;
+
+  bool validate(uint64_t min_alloc_size) const override;
 };
 
 #endif
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 7d02b1551e0f..afd28dc35ec4 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -6791,6 +6791,14 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t,
       return r;
     }
   }
+  dout(1) << __func__ << " effective freelist_type = " << freelist_type << std::hex
+          << ", freelist_alloc_size = 0x" << fm->get_alloc_size()
+          << ", min_alloc_size = 0x" << min_alloc_size
+          << std::dec << dendl;
+  if (!fm->validate(min_alloc_size)) {
+    derr << __func__ << " freelist validation failed, unable to proceed." << dendl;
+    ceph_assert(false);
+  }
   // if space size tracked by free list manager is that higher than actual
   // dev size one can hit out-of-space allocation which will result
   // in data loss and/or assertions
@@ -13116,20 +13124,26 @@ int BlueStore::_open_super_meta()
   }
 
   {
-    bufferlist bl;
-    db->get(PREFIX_SUPER, "min_alloc_size", &bl);
-    auto p = bl.cbegin();
-    try {
-      uint64_t val;
-      decode(val, p);
-      min_alloc_size = val;
-      min_alloc_size_order = std::countr_zero(val);
-      min_alloc_size_mask  = min_alloc_size - 1;
+    if(cct->_conf->bluestore_debug_enforce_min_alloc_size == 0) {
+      bufferlist bl;
+      db->get(PREFIX_SUPER, "min_alloc_size", &bl);
+      auto p = bl.cbegin();
+      try {
+        uint64_t val;
+        decode(val, p);
+        min_alloc_size = val;
+        min_alloc_size_order = std::countr_zero(val);
+        min_alloc_size_mask  = min_alloc_size - 1;
 
-      ceph_assert(min_alloc_size == 1u << min_alloc_size_order);
-    } catch (ceph::buffer::error& e) {
-      derr << __func__ << " unable to read min_alloc_size" << dendl;
-      return -EIO;
+        ceph_assert(min_alloc_size == 1u << min_alloc_size_order);
+      } catch (ceph::buffer::error& e) {
+        derr << __func__ << " unable to read min_alloc_size" << dendl;
+        return -EIO;
+      }
+    } else {
+      min_alloc_size = cct->_conf->bluestore_debug_enforce_min_alloc_size;
+      min_alloc_size_order = std::countr_zero(min_alloc_size);
+      min_alloc_size_mask  = min_alloc_size - 1;
     }
     dout(1) << __func__ << " min_alloc_size 0x" << std::hex << min_alloc_size
 	     << std::dec << dendl;
diff --git a/src/os/bluestore/FreelistManager.h b/src/os/bluestore/FreelistManager.h
index b647e55c2c16..7827a47ddd13 100644
--- a/src/os/bluestore/FreelistManager.h
+++ b/src/os/bluestore/FreelistManager.h
@@ -52,6 +52,8 @@ class FreelistManager {
   virtual void get_meta(uint64_t target_size,
   std::vector<std::pair<std::string, std::string>>*) const = 0;
 
+  virtual bool validate(uint64_t min_alloc_size) const = 0;
+
   void set_null_manager() {
     null_manager = true;
   }

From 080e386098eca47fb53789090efda21bc7748b85 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Wed, 21 Feb 2024 00:07:00 +0800
Subject: [PATCH 1998/2492] debian/control: replace <tab> with spaces

to be consistent with the rest of this file

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 debian/control | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/control b/debian/control
index f9cc22d377ff..318a822386ef 100644
--- a/debian/control
+++ b/debian/control
@@ -83,13 +83,13 @@ Build-Depends: automake,
                libndctl-dev (>= 63) <pkg.ceph.pmdk>,
                libpmem-dev <pkg.ceph.pmdk>,
                libpmemobj-dev (>= 1.8) <pkg.ceph.pmdk>,
-	       libprotobuf-dev <pkg.ceph.crimson>,
+               libprotobuf-dev <pkg.ceph.crimson>,
                ninja-build,
                nlohmann-json3-dev,
                patch,
                pkg-config,
                prometheus <pkg.ceph.check>,
-	       protobuf-compiler <pkg.ceph.crimson>,
+               protobuf-compiler <pkg.ceph.crimson>,
                python3-all-dev,
                python3-cherrypy3,
                python3-natsort,

From ea3a567f7f035aa439b6d5541d69f3781f47bd50 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 8 Feb 2024 19:32:37 -0500
Subject: [PATCH 1999/2492] qa/workunits: make wait_for_status_in_pool_dir()
 reentrant

In rbd_mirror_helpers.sh, the `wait_for_status_in_pool_dir()` helper
stored `mirror image status` and `mirror pool status` command outputs
in files that could be shared over successive calls or calls from
multiple threads. Instead store the command outputs in local variables
to make `wait_for_status_in_pool_dir()` reentrant.

Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 qa/workunits/rbd/rbd_mirror_helpers.sh | 31 +++++++++++++-------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index f7a3e5de0d99..d7b4abc4d8f2 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -823,23 +823,23 @@ test_status_in_pool_dir()
     local description_pattern="$5"
     local service_pattern="$6"
 
-    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status)
-    CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} |
-        tee ${status_log} >&2
-    grep "^  state: .*${state_pattern}" ${status_log} || return 1
-    grep "^  description: .*${description_pattern}" ${status_log} || return 1
+    local status
+    status=$(CEPH_ARGS='' rbd --cluster ${cluster} mirror image status \
+                 ${pool}/${image})
+    grep "^  state: .*${state_pattern}" <<< "$status" || return 1
+    grep "^  description: .*${description_pattern}" <<< "$status" || return 1
 
     if [ -n "${service_pattern}" ]; then
-        grep "service: *${service_pattern}" ${status_log} || return 1
+        grep "service: *${service_pattern}" <<< "$status" || return 1
     elif echo ${state_pattern} | grep '^up+'; then
-        grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1
+        grep "service: *${MIRROR_USER_ID_PREFIX}.* on " <<< "$status" || return 1
     else
-        grep "service: " ${status_log} && return 1
+        grep "service: " <<< "$status" && return 1
     fi
 
     # recheck using `mirror pool status` command to stress test it.
-
-    local last_update="$(sed -nEe 's/^  last_update: *(.*) *$/\1/p' ${status_log})"
+    local last_update
+    last_update="$(sed -nEe 's/^  last_update: *(.*) *$/\1/p' <<< "$status")"
     test_mirror_pool_status_verbose \
         ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" &&
     return 0
@@ -856,16 +856,15 @@ test_mirror_pool_status_verbose()
     local state_pattern="$4"
     local prev_last_update="$5"
 
-    local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}.mirror_status)
-
-    rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \
-        > ${status_log}
+    local status
+    status=$(CEPH_ARGS='' rbd --cluster ${cluster} mirror pool status ${pool} \
+                 --verbose --format xml)
 
     local last_update state
     last_update=$($XMLSTARLET sel -t -v \
-        "//images/image[name='${image}']/last_update" < ${status_log})
+        "//images/image[name='${image}']/last_update" <<< "$status")
     state=$($XMLSTARLET sel -t -v \
-        "//images/image[name='${image}']/state" < ${status_log})
+        "//images/image[name='${image}']/state" <<< "$status")
 
     echo "${state}" | grep "${state_pattern}" ||
     test "${last_update}" '>' "${prev_last_update}"

From b7aae5c3c5a1dd24c4cb7ceb499292af00bae680 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 25 May 2023 16:48:12 +0000
Subject: [PATCH 2000/2492] qa: Add tests to validate syncing of images using
 rbd-mirror

Introduce functional tests to validate that the images under
workloads are correctly mirrored between two clusters using snapshot
based mirroring.

Run workload on a primary image using a krbd or nbd client. Take
mirror snapshots of the image under workload. Unmount the mapped image
and calculate its MD5 checksum before demoting it. After demotion,
wait for the mirror status of the image to be 'up+unknown' in both
the clusters. This is to make sure that the non-primary image in the
other cluster is ready to be promoted. Now promote the non-primary
image in the other cluster. Map the promoted image and calculate its
MD5 checksum. Verify that the checksums of the demoted and promoted
images in the two clusters are the same.

The above test is run as part of two different workunits:
 - a workunit that validates the syncing of multiple mirrored images
   with workloads running on them
 - another workunit that validates the syncing of a single mirrored
   image with workload running on it and the image is set as primary
   alternatively between the two clusters, as it happens during
   failover and failback scenarios.

Fixes: https://tracker.ceph.com/issues/61617
Signed-off-by: Ramana Raja <rraja@redhat.com>
Co-authored-by: Ilya Dryomov <idryomov@redhat.com>
Co-authored-by: Christopher Hoffman <choffman@redhat.com>
---
 ...e-mirror-image-alternate-primary-krbd.yaml |  13 ++
 ...re-mirror-image-alternate-primary-nbd.yaml |  15 ++
 .../workloads/compare-mirror-images-krbd.yaml |  13 ++
 .../workloads/compare-mirror-images-nbd.yaml  |  15 ++
 .../compare_mirror_image_alternate_primary.sh | 106 +++++++++++
 qa/workunits/rbd/compare_mirror_images.sh     | 170 ++++++++++++++++++
 6 files changed, 332 insertions(+)
 create mode 100644 qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-krbd.yaml
 create mode 100644 qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-nbd.yaml
 create mode 100644 qa/suites/rbd/mirror/workloads/compare-mirror-images-krbd.yaml
 create mode 100644 qa/suites/rbd/mirror/workloads/compare-mirror-images-nbd.yaml
 create mode 100755 qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
 create mode 100755 qa/workunits/rbd/compare_mirror_images.sh

diff --git a/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-krbd.yaml b/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-krbd.yaml
new file mode 100644
index 000000000000..771400d01eed
--- /dev/null
+++ b/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-krbd.yaml
@@ -0,0 +1,13 @@
+overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_image_alternate_primary.sh
+    env:
+      RBD_DEVICE_TYPE: 'krbd'
+    timeout: 3h
diff --git a/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-nbd.yaml b/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-nbd.yaml
new file mode 100644
index 000000000000..e87d0e8cecc6
--- /dev/null
+++ b/qa/suites/rbd/mirror/workloads/compare-mirror-image-alternate-primary-nbd.yaml
@@ -0,0 +1,15 @@
+overrides:
+  install:
+    ceph:
+      extra_packages:
+        - rbd-nbd
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_image_alternate_primary.sh
+    env:
+      RBD_DEVICE_TYPE: 'nbd'
+    timeout: 3h
diff --git a/qa/suites/rbd/mirror/workloads/compare-mirror-images-krbd.yaml b/qa/suites/rbd/mirror/workloads/compare-mirror-images-krbd.yaml
new file mode 100644
index 000000000000..fc161987f7bd
--- /dev/null
+++ b/qa/suites/rbd/mirror/workloads/compare-mirror-images-krbd.yaml
@@ -0,0 +1,13 @@
+overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_images.sh
+    env:
+      RBD_DEVICE_TYPE: 'krbd'
+    timeout: 3h
diff --git a/qa/suites/rbd/mirror/workloads/compare-mirror-images-nbd.yaml b/qa/suites/rbd/mirror/workloads/compare-mirror-images-nbd.yaml
new file mode 100644
index 000000000000..ed02ed25702f
--- /dev/null
+++ b/qa/suites/rbd/mirror/workloads/compare-mirror-images-nbd.yaml
@@ -0,0 +1,15 @@
+overrides:
+  install:
+    ceph:
+      extra_packages:
+        - rbd-nbd
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_images.sh
+    env:
+      RBD_DEVICE_TYPE: 'nbd'
+    timeout: 3h
diff --git a/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh b/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
new file mode 100755
index 000000000000..338f43f1e530
--- /dev/null
+++ b/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE=image-alternate-primary
+MIRROR_IMAGE_MODE=snapshot
+MIRROR_POOL_MODE=image
+MOUNT=test-alternate-primary
+RBD_IMAGE_FEATURES='layering,exclusive-lock,object-map,fast-diff'
+RBD_MIRROR_INSTANCES=1
+RBD_MIRROR_MODE=snapshot
+RBD_MIRROR_USE_EXISTING_CLUSTER=1
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+take_mirror_snapshots() {
+  local cluster=$1
+  local pool=$2
+  local image=$3
+
+  for i in {1..30}; do
+    mirror_image_snapshot $cluster $pool $image
+    sleep 3
+  done
+}
+
+slow_untar_workload() {
+  local mountpt=$1
+
+  cp linux-5.4.tar.gz $mountpt
+  # run workload that updates the data and metadata of multiple files on disk.
+  # rate limit the workload such that the mirror snapshots can be taken as the
+  # contents of the image are progressively changed by the workload.
+  local ret=0
+  timeout 5m bash -c "zcat $mountpt/linux-5.4.tar.gz \
+    | pv -L 256K | tar xf - -C $mountpt" || ret=$?
+  if ((ret != 124)); then
+    echo "Workload completed prematurely"
+    return 1
+  fi
+}
+
+setup
+
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
+
+# initial setup
+create_image_and_enable_mirror ${CLUSTER1} ${POOL} ${IMAGE} \
+  ${RBD_MIRROR_MODE} 10G
+
+if [[ $RBD_DEVICE_TYPE == "nbd" ]]; then
+  DEV=$(sudo rbd --cluster ${CLUSTER1} device map -t nbd \
+           -o try-netlink ${POOL}/${IMAGE})
+elif [[ $RBD_DEVICE_TYPE == "krbd" ]]; then
+  DEV=$(sudo rbd --cluster ${CLUSTER1} device map -t krbd \
+           ${POOL}/${IMAGE})
+else
+  echo "Unknown RBD_DEVICE_TYPE: ${RBD_DEVICE_TYPE}"
+  exit 1
+fi
+sudo mkfs.ext4 ${DEV}
+mkdir ${MOUNT}
+
+wget https://download.ceph.com/qa/linux-5.4.tar.gz
+
+for i in {1..25}; do
+  # create mirror snapshots every few seconds under I/O
+  sudo mount ${DEV} ${MOUNT}
+  sudo chown $(whoami) ${MOUNT}
+  rm -rf ${MOUNT}/*
+  take_mirror_snapshots ${CLUSTER1} ${POOL} ${IMAGE} &
+  SNAP_PID=$!
+  slow_untar_workload ${MOUNT}
+  wait $SNAP_PID
+  sudo umount ${MOUNT}
+
+  # calculate hash before demotion of primary image
+  DEMOTE_MD5=$(sudo md5sum ${DEV} | awk '{print $1}')
+  sudo rbd --cluster ${CLUSTER1} device unmap -t ${RBD_DEVICE_TYPE} ${DEV}
+
+  demote_image ${CLUSTER1} ${POOL} ${IMAGE}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${IMAGE} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${IMAGE} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${IMAGE}
+
+  # calculate hash after promotion of secondary image
+  if [[ $RBD_DEVICE_TYPE == "nbd" ]]; then
+    DEV=$(sudo rbd --cluster ${CLUSTER2} device map -t nbd \
+             -o try-netlink ${POOL}/${IMAGE})
+  elif [[ $RBD_DEVICE_TYPE == "krbd" ]]; then
+    DEV=$(sudo rbd --cluster ${CLUSTER2} device map -t krbd ${POOL}/${IMAGE})
+  fi
+  PROMOTE_MD5=$(sudo md5sum ${DEV} | awk '{print $1}')
+
+  if [[ "${DEMOTE_MD5}" != "${PROMOTE_MD5}" ]]; then
+    echo "Mismatch at iteration ${i}: ${DEMOTE_MD5} != ${PROMOTE_MD5}"
+    exit 1
+  fi
+
+  TEMP=${CLUSTER1}
+  CLUSTER1=${CLUSTER2}
+  CLUSTER2=${TEMP}
+done
+
+echo OK
diff --git a/qa/workunits/rbd/compare_mirror_images.sh b/qa/workunits/rbd/compare_mirror_images.sh
new file mode 100755
index 000000000000..1b3cc300de98
--- /dev/null
+++ b/qa/workunits/rbd/compare_mirror_images.sh
@@ -0,0 +1,170 @@
+#!/bin/bash
+
+set -ex
+
+IMG_PREFIX=image-primary
+MIRROR_IMAGE_MODE=snapshot
+MIRROR_POOL_MODE=image
+MNTPT_PREFIX=test-primary
+RBD_IMAGE_FEATURES='layering,exclusive-lock,object-map,fast-diff'
+RBD_MIRROR_INSTANCES=1
+RBD_MIRROR_MODE=snapshot
+RBD_MIRROR_USE_EXISTING_CLUSTER=1
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+take_mirror_snapshots() {
+  local cluster=$1
+  local pool=$2
+  local image=$3
+
+  for i in {1..30}; do
+    mirror_image_snapshot $cluster $pool $image
+    sleep 3
+  done
+}
+
+slow_untar_workload() {
+  local mountpt=$1
+
+  cp linux-5.4.tar.gz $mountpt
+  # run workload that updates the data and metadata of multiple files on disk.
+  # rate limit the workload such that the mirror snapshots can be taken as the
+  # contents of the image are progressively changed by the workload.
+  local ret=0
+  timeout 5m bash -c "zcat $mountpt/linux-5.4.tar.gz \
+    | pv -L 256K | tar xf - -C $mountpt" || ret=$?
+  if ((ret != 124)); then
+    echo "Workload completed prematurely"
+    return 1
+  fi
+}
+
+wait_for_image_removal() {
+  local cluster=$1
+  local pool=$2
+  local image=$3
+
+  for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+    if ! rbd --cluster $cluster ls $pool | grep -wq $image; then
+      return 0
+    fi
+    sleep $s
+  done
+
+  echo "image ${pool}/${image} not removed from cluster ${cluster}"
+  return 1
+}
+
+compare_demoted_promoted_image() {
+  local dev=${DEVS[$1-1]}
+  local img=${IMG_PREFIX}$1
+  local mntpt=${MNTPT_PREFIX}$1
+  local demote_md5 promote_md5
+
+  sudo umount ${mntpt}
+
+  # calculate hash before demotion of primary image
+  demote_md5=$(sudo md5sum ${dev} | awk '{print $1}')
+  sudo rbd --cluster ${CLUSTER1} device unmap -t ${RBD_DEVICE_TYPE} \
+      ${POOL}/${img}
+
+  demote_image ${CLUSTER1} ${POOL} ${img}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${img} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${img} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${img}
+
+  # calculate hash after promotion of secondary image
+  if [[ $RBD_DEVICE_TYPE == "nbd" ]]; then
+    dev=$(sudo rbd --cluster ${CLUSTER2} device map -t nbd \
+             -o try-netlink ${POOL}/${img})
+  elif [[ $RBD_DEVICE_TYPE == "krbd" ]]; then
+    dev=$(sudo rbd --cluster ${CLUSTER2} device map -t krbd ${POOL}/${img})
+  fi
+  promote_md5=$(sudo md5sum ${dev} | awk '{print $1}')
+  sudo rbd --cluster ${CLUSTER2} device unmap -t ${RBD_DEVICE_TYPE} ${dev}
+
+  if [[ "${demote_md5}" != "${promote_md5}" ]]; then
+    echo "Mismatch for image ${POOL}/${img}: ${demote_md5} != ${promote_md5}"
+    return 1
+  fi
+}
+
+setup
+
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
+
+wget https://download.ceph.com/qa/linux-5.4.tar.gz
+
+for i in {1..10}; do
+  DEVS=()
+  SNAP_PIDS=()
+  COMPARE_PIDS=()
+  WORKLOAD_PIDS=()
+  RET=0
+  for j in {1..10}; do
+    IMG=${IMG_PREFIX}${j}
+    MNTPT=${MNTPT_PREFIX}${j}
+    create_image_and_enable_mirror ${CLUSTER1} ${POOL} ${IMG} \
+      ${RBD_MIRROR_MODE} 10G
+    if [[ $RBD_DEVICE_TYPE == "nbd" ]]; then
+      DEV=$(sudo rbd --cluster ${CLUSTER1} device map -t nbd \
+	      -o try-netlink ${POOL}/${IMG})
+    elif [[ $RBD_DEVICE_TYPE == "krbd" ]]; then
+      DEV=$(sudo rbd --cluster ${CLUSTER1} device map -t krbd \
+	      ${POOL}/${IMG})
+    else
+      echo "Unknown RBD_DEVICE_TYPE: ${RBD_DEVICE_TYPE}"
+      exit 1
+    fi
+    DEVS+=($DEV)
+    sudo mkfs.ext4 ${DEV}
+    mkdir ${MNTPT}
+    sudo mount ${DEV} ${MNTPT}
+    sudo chown $(whoami) ${MNTPT}
+    # create mirror snapshots under I/O every few seconds
+    take_mirror_snapshots ${CLUSTER1} ${POOL} ${IMG} &
+    SNAP_PIDS+=($!)
+    slow_untar_workload ${MNTPT} &
+    WORKLOAD_PIDS+=($!)
+  done
+  for pid in ${SNAP_PIDS[@]}; do
+    wait $pid || RET=$?
+  done
+  if ((RET != 0)); then
+    echo "take_mirror_snapshots failed"
+    exit 1
+  fi
+  for pid in ${WORKLOAD_PIDS[@]}; do
+    wait $pid || RET=$?
+  done
+  if ((RET != 0)); then
+    echo "slow_untar_workload failed"
+    exit 1
+  fi
+
+  for j in {1..10}; do
+    compare_demoted_promoted_image $j &
+    COMPARE_PIDS+=($!)
+  done
+  for pid in ${COMPARE_PIDS[@]}; do
+    wait $pid || RET=$?
+  done
+  if ((RET != 0)); then
+    echo "compare_demoted_promoted_image failed"
+    exit 1
+  fi
+
+  for j in {1..10}; do
+    IMG=${IMG_PREFIX}${j}
+    # Allow for removal of non-primary image by checking that mirroring
+    # image status is "up+replaying"
+    wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${IMG}
+    remove_image ${CLUSTER2} ${POOL} ${IMG}
+    wait_for_image_removal ${CLUSTER1} ${POOL} ${IMG}
+    rm -rf ${MNTPT_PREFIX}${j}
+  done
+done
+
+echo OK

From 9f86c35a0d308c6ff24d3a033f5314ec86bf896b Mon Sep 17 00:00:00 2001
From: Orit Wasserman <owasserm@ibm.com>
Date: Mon, 12 Feb 2024 14:39:38 +0200
Subject: [PATCH 2001/2492] doc: Add NVMe-oF gateway documentation

- Add nvmeof-initiator-esx.rst
- Add nvmeof-initiator-linux.rst
- Add nvmeof-initiators.rst
- Add nvmeof-overview.rst
- Add nvmeof-requirements.rst
- Add nvmeof-target-configure.rst
- Add links to rbd-integrations.rst

Co-authored-by: Ilya Dryomov <idryomov@redhat.com>
Co-authored-by: Zac Dover <zac.dover@proton.me>
Signed-off-by: Orit Wasserman <owasserm@ibm.com>
---
 doc/rbd/nvmeof-initiator-esx.rst    |  70 ++++++++++++++++
 doc/rbd/nvmeof-initiator-linux.rst  |  83 +++++++++++++++++++
 doc/rbd/nvmeof-initiators.rst       |  16 ++++
 doc/rbd/nvmeof-overview.rst         |  48 +++++++++++
 doc/rbd/nvmeof-requirements.rst     |  14 ++++
 doc/rbd/nvmeof-target-configure.rst | 122 ++++++++++++++++++++++++++++
 doc/rbd/rbd-integrations.rst        |   1 +
 7 files changed, 354 insertions(+)
 create mode 100644 doc/rbd/nvmeof-initiator-esx.rst
 create mode 100644 doc/rbd/nvmeof-initiator-linux.rst
 create mode 100644 doc/rbd/nvmeof-initiators.rst
 create mode 100644 doc/rbd/nvmeof-overview.rst
 create mode 100644 doc/rbd/nvmeof-requirements.rst
 create mode 100644 doc/rbd/nvmeof-target-configure.rst

diff --git a/doc/rbd/nvmeof-initiator-esx.rst b/doc/rbd/nvmeof-initiator-esx.rst
new file mode 100644
index 000000000000..6afa29f1e9f9
--- /dev/null
+++ b/doc/rbd/nvmeof-initiator-esx.rst
@@ -0,0 +1,70 @@
+---------------------------------
+NVMe/TCP Initiator for VMware ESX
+---------------------------------
+
+Prerequisites
+=============
+
+- A VMware ESXi host running VMware vSphere Hypervisor (ESXi) 7.0U3 version or later.
+- Deployed Ceph NVMe-oF gateway.
+- Ceph cluster with NVMe-oF configuration.
+- Subsystem defined in the gateway.
+
+Configuration
+=============
+
+The following instructions will use the default vSphere web client and esxcli.
+
+1. Enable NVMe/TCP on a NIC:
+
+   .. prompt:: bash #
+    
+      esxcli nvme fabric enable --protocol TCP --device vmnicN
+
+   Replace ``N`` with the number of the NIC.
+
+2. Tag a VMKernel NIC to permit NVMe/TCP traffic:
+
+   .. prompt:: bash #
+    
+      esxcli network uip interface tag add --interface-nme vmkN --tagname NVMeTCP
+
+   Replace ``N`` with the ID of the VMkernel.
+
+3. Configure the VMware ESXi host for NVMe/TCP:
+
+    #. List the NVMe-oF adapter:
+    
+       .. prompt:: bash #
+        
+          esxcli nvme adapter list
+
+    #. Discover NVMe-oF subsystems:
+    
+       .. prompt:: bash #
+        
+          esxcli nvme fabric discover -a NVME_TCP_ADAPTER -i GATEWAY_IP -p 4420
+    
+    #. Connect to NVME-oF gateway subsystem:
+    
+       .. prompt:: bash #
+        
+          esxcli nvme connect -a NVME_TCP_ADAPTER -i GATEWAY_IP -p 4420 -s SUBSYSTEM_NQN
+
+    #. List the NVMe/TCP controllers:
+    
+       .. prompt:: bash #
+        
+          esxcli nvme controller list
+
+    #. List the NVMe-oF namespaces in the subsystem:
+    
+       .. prompt:: bash #
+        
+          esxcli nvme namespace list
+
+4. Verify that the initiator has been set up correctly:
+
+    #. From the vSphere client go to the ESXi host.
+    #. On the Storage page go to the Devices tab.
+    #. Verify that the NVME/TCP disks are listed in the table.
diff --git a/doc/rbd/nvmeof-initiator-linux.rst b/doc/rbd/nvmeof-initiator-linux.rst
new file mode 100644
index 000000000000..4889e4132c1e
--- /dev/null
+++ b/doc/rbd/nvmeof-initiator-linux.rst
@@ -0,0 +1,83 @@
+==============================
+ NVMe/TCP Initiator for Linux
+==============================
+
+Prerequisites
+=============
+
+- Kernel 5.0 or later
+- RHEL 9.2 or later
+- Ubuntu 24.04 or later
+- SLES 15 SP3 or later
+
+Installation
+============
+
+1. Install the nvme-cli:
+
+   .. prompt:: bash #
+   
+      yum install nvme-cli
+
+2. Load the NVMe-oF module:
+
+   .. prompt:: bash # 
+   
+      modprobe nvme-fabrics
+
+3. Verify the NVMe/TCP target is reachable:
+
+   .. prompt::  bash #
+   
+      nvme discover -t tcp -a GATEWAY_IP -s 4420
+
+4. Connect to the NVMe/TCP target:
+
+   .. prompt:: bash #
+   
+      nvme connect -t tcp -a GATEWAY_IP -n SUBSYSTEM_NQN
+
+Next steps
+==========
+
+Verify that the initiator is set up correctly:
+
+1. List the NVMe block devices:
+
+   .. prompt:: bash #
+   
+      nvme list
+
+2. Create a filesystem on the desired device:
+
+   .. prompt:: bash #
+   
+      mkfs.ext4 NVME_NODE_PATH
+
+3. Mount the filesystem:
+
+   .. prompt:: bash #
+   
+      mkdir /mnt/nvmeof
+
+   .. prompt:: bash #
+   
+      mount NVME_NODE_PATH /mnt/nvmeof
+
+4. List the NVME-oF files:
+
+   .. prompt:: bash #
+   
+      ls /mnt/nvmeof
+
+5. Create a text file in the ``/mnt/nvmeof`` directory:
+
+   .. prompt:: bash #
+   
+      echo "Hello NVME-oF" > /mnt/nvmeof/hello.text
+
+6. Verify that the file can be accessed:
+
+   .. prompt:: bash #
+   
+      cat /mnt/nvmeof/hello.text
diff --git a/doc/rbd/nvmeof-initiators.rst b/doc/rbd/nvmeof-initiators.rst
new file mode 100644
index 000000000000..8fa4a5b9d89c
--- /dev/null
+++ b/doc/rbd/nvmeof-initiators.rst
@@ -0,0 +1,16 @@
+.. _configuring-the-nvmeof-initiators:
+
+====================================
+ Configuring the NVMe-oF Initiators 
+====================================
+
+- `NVMe/TCP Initiator for Linux <../nvmeof-initiator-linux>`_
+
+- `NVMe/TCP Initiator for VMware ESX <../nvmeof-initiator-esx>`_
+
+.. toctree::
+  :maxdepth: 1
+  :hidden:
+
+  Linux <nvmeof-initiator-linux>
+  VMware ESX <nvmeof-initiator-esx>
diff --git a/doc/rbd/nvmeof-overview.rst b/doc/rbd/nvmeof-overview.rst
new file mode 100644
index 000000000000..070024a3abfa
--- /dev/null
+++ b/doc/rbd/nvmeof-overview.rst
@@ -0,0 +1,48 @@
+.. _ceph-nvmeof:
+
+======================
+ Ceph NVMe-oF Gateway
+======================
+
+The NVMe-oF Gateway presents an NVMe-oF target that exports
+RADOS Block Device (RBD) images as NVMe namespaces. The NVMe-oF protocol allows
+clients (initiators) to send NVMe commands to storage devices (targets) over a
+TCP/IP network, enabling clients without native Ceph client support to access
+Ceph block storage.  
+
+Each NVMe-oF gateway consists of an `SPDK <https://spdk.io/>`_ NVMe-oF target
+with ``bdev_rbd`` and a control daemon. Ceph’s NVMe-oF gateway can be used to
+provision a fully integrated block-storage infrastructure with all the features
+and benefits of a conventional Storage Area Network (SAN).
+
+.. ditaa::
+                  Cluster Network (optional)
+                 +-------------------------------------------+
+                 |             |               |             |
+             +-------+     +-------+       +-------+     +-------+
+             |       |     |       |       |       |     |       |
+             | OSD 1 |     | OSD 2 |       | OSD 3 |     | OSD N |
+             |    {s}|     |    {s}|       |    {s}|     |    {s}|
+             +-------+     +-------+       +-------+     +-------+
+                 |             |               |             |
+      +--------->|             |  +---------+  |             |<----------+
+      :          |             |  |   RBD   |  |             |           :
+      |          +----------------|  Image  |----------------+           |
+      |           Public Network  |    {d}  |                            |
+      |                           +---------+                            |
+      |                                                                  |
+      |                      +--------------------+                      |
+      |   +--------------+   | NVMeoF Initiators  |   +--------------+   |
+      |   |  NVMe‐oF GW  |   |    +-----------+   |   | NVMe‐oF GW   |   |
+      +-->|  RBD Module  |<--+    | Various   |   +-->|  RBD Module  |<--+
+          |              |   |    | Operating |   |   |              |
+          +--------------+   |    | Systems   |   |   +--------------+
+                             |    +-----------+   |
+                             +--------------------+
+
+.. toctree::
+  :maxdepth: 1
+
+  Requirements <nvmeof-requirements>
+  Configuring the NVME-oF Target <nvmeof-target-configure>
+  Configuring the NVMe-oF Initiators <nvmeof-initiators>
diff --git a/doc/rbd/nvmeof-requirements.rst b/doc/rbd/nvmeof-requirements.rst
new file mode 100644
index 000000000000..a53d1c2d76ce
--- /dev/null
+++ b/doc/rbd/nvmeof-requirements.rst
@@ -0,0 +1,14 @@
+============================
+NVME-oF Gateway Requirements
+============================
+
+We recommend that you provision at least two NVMe/TCP gateways on different
+nodes to implement a highly-available Ceph NVMe/TCP solution.
+
+We recommend at a minimum a single 10Gb Ethernet link in the Ceph public
+network for the gateway. For hardware recommendations, see
+:ref:`hardware-recommendations` .
+
+.. note:: On the NVMe-oF gateway, the memory footprint is a function of the
+   number of mapped RBD images and can grow to be large. Plan memory
+   requirements accordingly based on the number RBD images to be mapped.
diff --git a/doc/rbd/nvmeof-target-configure.rst b/doc/rbd/nvmeof-target-configure.rst
new file mode 100644
index 000000000000..13c22397d9c0
--- /dev/null
+++ b/doc/rbd/nvmeof-target-configure.rst
@@ -0,0 +1,122 @@
+==========================================
+Installing and Configuring NVMe-oF Targets
+==========================================
+
+Traditionally, block-level access to a Ceph storage cluster has been limited to
+(1) QEMU and ``librbd`` (which is a key enabler for adoption within OpenStack
+environments), and (2) the Linux kernel client. Starting with the Ceph Reef
+release, block-level access has been expanded to offer standard NVMe/TCP
+support, allowing wider platform usage and potentially opening new use cases.
+
+Prerequisites
+=============
+
+-  Red Hat Enterprise Linux/CentOS 8.0 (or newer); Linux kernel v4.16 (or newer)
+
+-  A working Ceph Reef or later storage cluster, deployed with ``cephadm``
+
+-  NVMe-oF gateways, which can either be colocated with OSD nodes or on dedicated nodes
+
+-  Separate network subnets for NVME-oF front-end traffic and Ceph back-end traffic
+
+Explanation
+===========
+
+The Ceph NVMe-oF gateway is both an NVMe-oF target and a Ceph client. Think of
+it as a "translator" between Ceph's RBD interface and the NVME-oF protocol. The
+Ceph NVMe-oF gateway can run on a standalone node or be colocated with other
+daemons, for example on a Ceph Object Store Disk (OSD) node. When colocating
+the Ceph NVMe-oF gateway with other daemons, ensure that sufficient CPU and
+memory are available. The steps below explain how to install and configure the
+Ceph NVMe/TCP gateway for basic operation.
+
+
+Installation
+============
+
+Complete the following steps to install the Ceph NVME-oF gateway:
+
+#. Create a pool in which the gateways configuration can be managed:
+
+   .. prompt:: bash #
+
+      ceph osd pool create NVME-OF_POOL_NAME
+
+#. Enable RBD on the NVMe-oF pool:
+
+   .. prompt:: bash #
+   
+      rbd pool init NVME-OF_POOL_NAME
+
+#. Deploy the NVMe-oF gateway daemons on a specific set of nodes:
+
+   .. prompt:: bash #
+   
+      ceph orch apply nvmeof NVME-OF_POOL_NAME --placment="host01, host02"
+
+Configuration
+=============
+
+Download the ``nvmeof-cli`` container before first use.
+To download it use the following command:
+
+.. prompt:: bash #
+   
+   podman pull quay.io/ceph/nvmeof-cli:latest
+
+#. Create an NVMe subsystem:
+
+.. prompt:: bash #
+   
+   podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 subsystem add --subsystem SUSYSTEM_NQN
+
+   The subsystem NQN is a user defined string, for example ``nqn.2016-06.io.spdk:cnode1``.
+
+#. Define the IP port on the gateway that will process the NVME/TCP commands and I/O:
+
+    a. On the install node, get the NVME-oF Gateway name:
+
+       .. prompt:: bash #
+       
+          ceph orch ps | grep nvme
+
+    b. Define the IP port for the gateway:
+
+       .. prompt:: bash #
+    
+          podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 listener add --subsystem SUBSYSTEM_NQN --gateway-name GATEWAY_NAME --traddr GATEWAY_IP --trsvcid 4420
+
+#. Get the host NQN (NVME Qualified Name) for each host:
+
+   .. prompt:: bash #
+
+      cat /etc/nvme/hostnqn
+
+   .. prompt:: bash #
+    
+      esxcli nvme info get
+
+#. Allow the initiator host to connect to the newly-created NVMe subsystem:
+
+   .. prompt:: bash #
+    
+      podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 host add --subsystem SUBSYSTEM_NQN --host "HOST_NQN1, HOST_NQN2"
+
+#. List all subsystems configured in the gateway:
+
+   .. prompt:: bash #
+    
+      podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 subsystem list
+
+#. Create a new NVMe namespace:
+
+   .. prompt:: bash #
+    
+      podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 namespace add --subsystem SUBSYSTEM_NQN --rbd-pool POOL_NAME --rbd-image IMAGE_NAME
+
+#. List all namespaces in the subsystem:
+
+   .. prompt:: bash #
+    
+      podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 namespace list --subsystem SUBSYSTEM_NQN
+
diff --git a/doc/rbd/rbd-integrations.rst b/doc/rbd/rbd-integrations.rst
index f55604a6fcf7..3c4afe38f3d4 100644
--- a/doc/rbd/rbd-integrations.rst
+++ b/doc/rbd/rbd-integrations.rst
@@ -14,3 +14,4 @@
    CloudStack <rbd-cloudstack>
    LIO iSCSI Gateway <iscsi-overview>
    Windows <rbd-windows>
+   NVMe-oF Gateway <nvmeof-overview>

From f1351ea3d463a6cb669ee66cf24d0b3c2debef3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Zarzy=C5=84ski?= <rzarzyns@redhat.com>
Date: Thu, 22 Feb 2024 20:48:55 +0100
Subject: [PATCH 2002/2492] ec: rename object_size -> stripe_width in
 get_chunk_size()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`object_size` was misleading in my perception; it suggested to me
that the chunking happens on objects, not their stripes.

`stripe_width` corresponds with the naming in the ctor of `ECBackend`.

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>
---
 src/erasure-code/ErasureCodeInterface.h          | 10 +++++-----
 src/erasure-code/clay/ErasureCodeClay.cc         |  4 ++--
 src/erasure-code/clay/ErasureCodeClay.h          |  2 +-
 src/erasure-code/isa/ErasureCodeIsa.cc           |  4 ++--
 src/erasure-code/isa/ErasureCodeIsa.h            |  2 +-
 src/erasure-code/jerasure/ErasureCodeJerasure.cc | 10 +++++-----
 src/erasure-code/jerasure/ErasureCodeJerasure.h  |  2 +-
 src/erasure-code/lrc/ErasureCodeLrc.cc           |  4 ++--
 src/erasure-code/lrc/ErasureCodeLrc.h            |  2 +-
 src/erasure-code/shec/ErasureCodeShec.cc         |  6 +++---
 src/erasure-code/shec/ErasureCodeShec.h          |  2 +-
 11 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/erasure-code/ErasureCodeInterface.h b/src/erasure-code/ErasureCodeInterface.h
index b0c24e1e42a7..7107f978dd4f 100644
--- a/src/erasure-code/ErasureCodeInterface.h
+++ b/src/erasure-code/ErasureCodeInterface.h
@@ -261,21 +261,21 @@ namespace ceph {
     /**
      * Return the size (in bytes) of a single chunk created by a call
      * to the **decode** method. The returned size multiplied by
-     * **get_chunk_count()** is greater or equal to **object_size**.
+     * **get_chunk_count()** is greater or equal to **stripe_width**.
      *
      * If the object size is properly aligned, the chunk size is
-     * **object_size / get_chunk_count()**. However, if
-     * **object_size** is not a multiple of **get_chunk_count** or if
+     * **stripe_width / get_chunk_count()**. However, if
+     * **stripe_width** is not a multiple of **get_chunk_count** or if
      * the implementation imposes additional alignment constraints,
      * the chunk size may be larger.
      *
      * The byte found at offset **B** of the original object is mapped
      * to chunk **B / get_chunk_size()** at offset **B % get_chunk_size()**.
      *
-     * @param [in] object_size the number of bytes of the object to **encode()**
+     * @param [in] stripe_width the number of bytes of the object to **encode()**
      * @return the size (in bytes) of a single chunk created by **encode()**
      */
-    virtual unsigned int get_chunk_size(unsigned int object_size) const = 0;
+    virtual unsigned int get_chunk_size(unsigned int stripe_width) const = 0;
 
     /**
      * Compute the smallest subset of **available** chunks that needs
diff --git a/src/erasure-code/clay/ErasureCodeClay.cc b/src/erasure-code/clay/ErasureCodeClay.cc
index 7162cecbd9f9..ba37b8c721c7 100644
--- a/src/erasure-code/clay/ErasureCodeClay.cc
+++ b/src/erasure-code/clay/ErasureCodeClay.cc
@@ -87,12 +87,12 @@ int ErasureCodeClay::init(ErasureCodeProfile &profile,
 
 }
 
-unsigned int ErasureCodeClay::get_chunk_size(unsigned int object_size) const
+unsigned int ErasureCodeClay::get_chunk_size(unsigned int stripe_width) const
 {
   unsigned int alignment_scalar_code = pft.erasure_code->get_chunk_size(1);
   unsigned int alignment = sub_chunk_no * k * alignment_scalar_code;
   
-  return round_up_to(object_size, alignment) / k;
+  return round_up_to(stripe_width, alignment) / k;
 }
 
 int ErasureCodeClay::minimum_to_decode(const set<int> &want_to_read,
diff --git a/src/erasure-code/clay/ErasureCodeClay.h b/src/erasure-code/clay/ErasureCodeClay.h
index 9b23b9907c3b..3697df721ef3 100644
--- a/src/erasure-code/clay/ErasureCodeClay.h
+++ b/src/erasure-code/clay/ErasureCodeClay.h
@@ -58,7 +58,7 @@ class ErasureCodeClay final : public ceph::ErasureCode {
     return sub_chunk_no;
   }
 
-  unsigned int get_chunk_size(unsigned int object_size) const override;
+  unsigned int get_chunk_size(unsigned int stripe_width) const override;
 
   int minimum_to_decode(const std::set<int> &want_to_read,
 			const std::set<int> &available,
diff --git a/src/erasure-code/isa/ErasureCodeIsa.cc b/src/erasure-code/isa/ErasureCodeIsa.cc
index 58aff4b0e0dd..1c2eadfdd0fe 100644
--- a/src/erasure-code/isa/ErasureCodeIsa.cc
+++ b/src/erasure-code/isa/ErasureCodeIsa.cc
@@ -63,10 +63,10 @@ ErasureCodeIsa::init(ErasureCodeProfile &profile, ostream *ss)
 // -----------------------------------------------------------------------------
 
 unsigned int
-ErasureCodeIsa::get_chunk_size(unsigned int object_size) const
+ErasureCodeIsa::get_chunk_size(unsigned int stripe_width) const
 {
   unsigned alignment = get_alignment();
-  unsigned chunk_size = ( object_size + k - 1 ) / k;
+  unsigned chunk_size = (stripe_width + k - 1) / k;
   dout(20) << "get_chunk_size: chunk_size " << chunk_size
            << " must be modulo " << alignment << dendl;
   unsigned modulo = chunk_size % alignment;
diff --git a/src/erasure-code/isa/ErasureCodeIsa.h b/src/erasure-code/isa/ErasureCodeIsa.h
index 705a1723aa65..06c51bbc907b 100644
--- a/src/erasure-code/isa/ErasureCodeIsa.h
+++ b/src/erasure-code/isa/ErasureCodeIsa.h
@@ -71,7 +71,7 @@ class ErasureCodeIsa : public ceph::ErasureCode {
     return k;
   }
 
-  unsigned int get_chunk_size(unsigned int object_size) const override;
+  unsigned int get_chunk_size(unsigned int stripe_width) const override;
 
   int encode_chunks(const std::set<int> &want_to_encode,
                     std::map<int, ceph::buffer::list> *encoded) override;
diff --git a/src/erasure-code/jerasure/ErasureCodeJerasure.cc b/src/erasure-code/jerasure/ErasureCodeJerasure.cc
index 3a780de811cb..7c4cfa4f8a1f 100644
--- a/src/erasure-code/jerasure/ErasureCodeJerasure.cc
+++ b/src/erasure-code/jerasure/ErasureCodeJerasure.cc
@@ -77,12 +77,12 @@ int ErasureCodeJerasure::parse(ErasureCodeProfile &profile,
   return err;
 }
 
-unsigned int ErasureCodeJerasure::get_chunk_size(unsigned int object_size) const
+unsigned int ErasureCodeJerasure::get_chunk_size(unsigned int stripe_width) const
 {
   unsigned alignment = get_alignment();
   if (per_chunk_alignment) {
-    unsigned chunk_size = object_size / k;
-    if (object_size % k)
+    unsigned chunk_size = stripe_width / k;
+    if (stripe_width % k)
       chunk_size++;
     dout(20) << "get_chunk_size: chunk_size " << chunk_size
 	     << " must be modulo " << alignment << dendl; 
@@ -95,8 +95,8 @@ unsigned int ErasureCodeJerasure::get_chunk_size(unsigned int object_size) const
     }
     return chunk_size;
   } else {
-    unsigned tail = object_size % alignment;
-    unsigned padded_length = object_size + ( tail ?  ( alignment - tail ) : 0 );
+    unsigned tail = stripe_width % alignment;
+    unsigned padded_length = stripe_width + (tail ? (alignment - tail) : 0);
     ceph_assert(padded_length % k == 0);
     return padded_length / k;
   }
diff --git a/src/erasure-code/jerasure/ErasureCodeJerasure.h b/src/erasure-code/jerasure/ErasureCodeJerasure.h
index 2272e0aad637..75d5c3c1a56c 100644
--- a/src/erasure-code/jerasure/ErasureCodeJerasure.h
+++ b/src/erasure-code/jerasure/ErasureCodeJerasure.h
@@ -54,7 +54,7 @@ class ErasureCodeJerasure : public ceph::ErasureCode {
     return k;
   }
 
-  unsigned int get_chunk_size(unsigned int object_size) const override;
+  unsigned int get_chunk_size(unsigned int stripe_width) const override;
 
   int encode_chunks(const std::set<int> &want_to_encode,
 		    std::map<int, ceph::buffer::list> *encoded) override;
diff --git a/src/erasure-code/lrc/ErasureCodeLrc.cc b/src/erasure-code/lrc/ErasureCodeLrc.cc
index bea861f1adee..eb8ebd810456 100644
--- a/src/erasure-code/lrc/ErasureCodeLrc.cc
+++ b/src/erasure-code/lrc/ErasureCodeLrc.cc
@@ -555,9 +555,9 @@ set<int> ErasureCodeLrc::get_erasures(const set<int> &want,
   return result;
 }
 
-unsigned int ErasureCodeLrc::get_chunk_size(unsigned int object_size) const
+unsigned int ErasureCodeLrc::get_chunk_size(unsigned int stripe_width) const
 {
-  return layers.front().erasure_code->get_chunk_size(object_size);
+  return layers.front().erasure_code->get_chunk_size(stripe_width);
 }
 
 void p(const set<int> &s) { cerr << s; } // for gdb
diff --git a/src/erasure-code/lrc/ErasureCodeLrc.h b/src/erasure-code/lrc/ErasureCodeLrc.h
index e5b0915ba86d..d5e3a07e847a 100644
--- a/src/erasure-code/lrc/ErasureCodeLrc.h
+++ b/src/erasure-code/lrc/ErasureCodeLrc.h
@@ -103,7 +103,7 @@ class ErasureCodeLrc final : public ceph::ErasureCode {
     return data_chunk_count;
   }
 
-  unsigned int get_chunk_size(unsigned int object_size) const override;
+  unsigned int get_chunk_size(unsigned int stripe_width) const override;
 
   int encode_chunks(const std::set<int> &want_to_encode,
 		    std::map<int, ceph::buffer::list> *encoded) override;
diff --git a/src/erasure-code/shec/ErasureCodeShec.cc b/src/erasure-code/shec/ErasureCodeShec.cc
index 3634be2fb3b9..70b1372bda40 100644
--- a/src/erasure-code/shec/ErasureCodeShec.cc
+++ b/src/erasure-code/shec/ErasureCodeShec.cc
@@ -58,11 +58,11 @@ int ErasureCodeShec::init(ErasureCodeProfile &profile,
   return ErasureCode::init(profile, ss);
 }
 
-unsigned int ErasureCodeShec::get_chunk_size(unsigned int object_size) const
+unsigned int ErasureCodeShec::get_chunk_size(unsigned int stripe_width) const
 {
   unsigned alignment = get_alignment();
-  unsigned tail = object_size % alignment;
-  unsigned padded_length = object_size + ( tail ?  ( alignment - tail ) : 0 );
+  unsigned tail = stripe_width % alignment;
+  unsigned padded_length = stripe_width + (tail ? (alignment - tail) : 0);
 
   ceph_assert(padded_length % k == 0);
   return padded_length / k;
diff --git a/src/erasure-code/shec/ErasureCodeShec.h b/src/erasure-code/shec/ErasureCodeShec.h
index 44a0778854fe..51e20359a418 100644
--- a/src/erasure-code/shec/ErasureCodeShec.h
+++ b/src/erasure-code/shec/ErasureCodeShec.h
@@ -69,7 +69,7 @@ class ErasureCodeShec : public ceph::ErasureCode {
     return k;
   }
 
-  unsigned int get_chunk_size(unsigned int object_size) const override;
+  unsigned int get_chunk_size(unsigned int stripe_width) const override;
 
   int _minimum_to_decode(const std::set<int> &want_to_read,
 			 const std::set<int> &available_chunks,

From 0e223fdf43561811dc4ad4fa29979e9f0722fa7f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 22 Feb 2024 16:54:54 -0500
Subject: [PATCH 2003/2492] rgw/aio: avoid infinite recursion in aio_abstract()

a recent regression from 320a2179a3c6c1981a0fd2494938515997c1bfad causes
aio_abstract() to recurse when given an empty optional_yield. this is
exposed by the librgw_file tests

Fixes: https://tracker.ceph.com/issues/64543

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_aio.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index 1bc50c89229c..293dea13217c 100644
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -59,7 +59,7 @@ Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, jspan_context* trace_ctx
         (void)trace_ctx; // suppress unused trace_ctx warning. until we will support the read op trace
         r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data);
       } else {
-        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, trace_ctx);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, 0, trace_ctx);
       }
       if (r.result < 0) {
         // cb() won't be called, so release everything here
@@ -124,7 +124,7 @@ Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y, jspan_c
     return aio_abstract(std::move(ctx), std::forward<Op>(op),
                         y.get_io_context(), y.get_yield_context(), trace_ctx);
   }
-  return aio_abstract(std::move(ctx), std::forward<Op>(op), null_yield, trace_ctx);
+  return aio_abstract(std::move(ctx), std::forward<Op>(op), trace_ctx);
 }
 
 } // anonymous namespace

From f953c419d6992c8124f3e123ae8030d6a45ffd1a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 23 Feb 2024 09:02:29 -0500
Subject: [PATCH 2004/2492] rgw/auth: do_aws4_auth_completion() catches
 exceptions

AWSv4ComplMulti::complete() throws exceptions on errors, but nothing
catches these

Fixes: https://tracker.ceph.com/issues/64549

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index b7e50e3421f7..64f6f3aaa335 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1407,16 +1407,20 @@ int RGWOp::do_aws4_auth_completion()
 {
   ldpp_dout(this, 5) << "NOTICE: call to do_aws4_auth_completion"  << dendl;
   if (s->auth.completer) {
-    if (!s->auth.completer->complete()) {
-      return -ERR_AMZ_CONTENT_SHA256_MISMATCH;
-    } else {
-      ldpp_dout(this, 10) << "v4 auth ok -- do_aws4_auth_completion" << dendl;
-    }
-
     /* TODO(rzarzynski): yes, we're really called twice on PUTs. Only first
      * call passes, so we disable second one. This is old behaviour, sorry!
      * Plan for tomorrow: seek and destroy. */
-    s->auth.completer = nullptr;
+    auto completer = std::move(s->auth.completer);
+
+    try {
+      if (!completer->complete()) {
+        return -ERR_AMZ_CONTENT_SHA256_MISMATCH;
+      }
+    } catch (const rgw::io::Exception& e) {
+      return -e.code().value();
+    }
+
+    ldpp_dout(this, 10) << "v4 auth ok -- do_aws4_auth_completion" << dendl;
   }
 
   return 0;

From 6df12da2cd0946092fa49f8dbb7661e4487c3c22 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 24 Feb 2024 02:05:42 +1000
Subject: [PATCH 2005/2492] doc/rbd: repair ordered list

Fix the numbering in an ordered list. The numbering was thrown off
because a ".. prompt" directive was improperly indented (it wasn't
indented at all).

See https://github.com/ceph/ceph/pull/55540#discussion_r1500051264

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rbd/nvmeof-target-configure.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/rbd/nvmeof-target-configure.rst b/doc/rbd/nvmeof-target-configure.rst
index 13c22397d9c0..4aa7d6ab73f4 100644
--- a/doc/rbd/nvmeof-target-configure.rst
+++ b/doc/rbd/nvmeof-target-configure.rst
@@ -66,9 +66,9 @@ To download it use the following command:
 
 #. Create an NVMe subsystem:
 
-.. prompt:: bash #
+   .. prompt:: bash #
    
-   podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 subsystem add --subsystem SUSYSTEM_NQN
+      podman run -it quay.io/ceph/nvmeof-cli:latest --server-address GATEWAY_IP --server-port GATEWAY_PORT 5500 subsystem add --subsystem SUSYSTEM_NQN
 
    The subsystem NQN is a user defined string, for example ``nqn.2016-06.io.spdk:cnode1``.
 

From dfdb5e124a19e0c5ce1e7ba47e574e0504e7091f Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 24 Feb 2024 02:21:54 +1000
Subject: [PATCH 2006/2492] doc/dev: fix spelling in crimson.rst

Repair a confusing sentence in doc/dev/crimson.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/dev/crimson/crimson.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/dev/crimson/crimson.rst b/doc/dev/crimson/crimson.rst
index 421c2102ab44..ea00ceebf56f 100644
--- a/doc/dev/crimson/crimson.rst
+++ b/doc/dev/crimson/crimson.rst
@@ -148,7 +148,7 @@ options. By default, ``log-to-stdout`` is enabled, and ``--log-to-syslog`` is di
 vstart.sh
 ---------
 
-The following options aree handy when using ``vstart.sh``,
+The following options can be used with ``vstart.sh``.
 
 ``--crimson``
     Start ``crimson-osd`` instead of ``ceph-osd``.

From 93ec6284fb3002b4778c4e54972ff1d864060922 Mon Sep 17 00:00:00 2001
From: Avan Thakkar <athakkar@redhat.com>
Date: Thu, 22 Feb 2024 16:30:06 +0530
Subject: [PATCH 2007/2492] cephadm/nvmeof: scrape nvmeof prometheus endpoint

Fixes: https://tracker.ceph.com/issues/64536
Signed-off-by: Avan Thakkar <athakkar@redhat.com>
---
 src/cephadm/cephadmlib/constants.py           |  2 +-
 src/pybind/mgr/cephadm/module.py              |  2 +-
 src/pybind/mgr/cephadm/service_discovery.py   | 17 ++++++++++++++++
 src/pybind/mgr/cephadm/services/monitoring.py |  4 +++-
 src/pybind/mgr/cephadm/services/nvmeof.py     |  1 +
 .../services/nvmeof/ceph-nvmeof.conf.j2       |  3 +++
 .../services/prometheus/prometheus.yml.j2     | 20 +++++++++++++++++++
 .../cephadm/tests/test_service_discovery.py   | 17 ++++++++++++++++
 src/pybind/mgr/cephadm/tests/test_services.py | 20 +++++++++++++++++++
 9 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index dfa660f48986..119f43b459b6 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -12,7 +12,7 @@
 DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.12'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:1.0.0'
 DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'
 DEFAULT_ELASTICSEARCH_IMAGE = 'quay.io/omrizeneva/elasticsearch:6.8.23'
 DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29'
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 87f7024bb255..c43152856a60 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -117,7 +117,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_IMAGE = 'quay.io/ceph/ceph'
 DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.43.0'
 DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.5.0'
-DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
+DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:1.0.0'
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'
diff --git a/src/pybind/mgr/cephadm/service_discovery.py b/src/pybind/mgr/cephadm/service_discovery.py
index b681cc8e7ff7..2b82f87493ff 100644
--- a/src/pybind/mgr/cephadm/service_discovery.py
+++ b/src/pybind/mgr/cephadm/service_discovery.py
@@ -19,6 +19,7 @@ class Server:  # type: ignore
 from cephadm.services.ingress import IngressSpec
 from cephadm.ssl_cert_utils import SSLCerts
 from cephadm.services.cephadmservice import CephExporterService
+from cephadm.services.nvmeof import NvmeofService
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -145,6 +146,7 @@ def index(self) -> str:
 <p><a href='prometheus/sd-config?service=node-exporter'>Node exporter http sd-config</a></p>
 <p><a href='prometheus/sd-config?service=haproxy'>HAProxy http sd-config</a></p>
 <p><a href='prometheus/sd-config?service=ceph-exporter'>Ceph exporter http sd-config</a></p>
+<p><a href='prometheus/sd-config?service=nvmeof'>NVMeoF http sd-config</a></p>
 <p><a href='prometheus/rules'>Prometheus rules</a></p>
 </body>
 </html>'''
@@ -163,6 +165,8 @@ def get_sd_config(self, service: str) -> List[Dict[str, Collection[str]]]:
             return self.haproxy_sd_config()
         elif service == 'ceph-exporter':
             return self.ceph_exporter_sd_config()
+        elif service == 'nvmeof':
+            return self.nvmeof_sd_config()
         else:
             return []
 
@@ -231,6 +235,19 @@ def ceph_exporter_sd_config(self) -> List[Dict[str, Collection[str]]]:
             })
         return srv_entries
 
+    def nvmeof_sd_config(self) -> List[Dict[str, Collection[str]]]:
+        """Return <http_sd_config> compatible prometheus config for nvmeof service."""
+        srv_entries = []
+        for dd in self.mgr.cache.get_daemons_by_type('nvmeof'):
+            assert dd.hostname is not None
+            addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname)
+            port = NvmeofService.PROMETHEUS_PORT
+            srv_entries.append({
+                'targets': [build_url(host=addr, port=port).lstrip('/')],
+                'labels': {'instance': dd.hostname}
+            })
+        return srv_entries
+
     @cherrypy.expose(alias='prometheus/rules')
     def get_prometheus_rules(self) -> str:
         """Return currently configured prometheus rules as Yaml."""
diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index 114c848608a3..d3439c04d04f 100644
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -402,6 +402,7 @@ def generate_config(
         haproxy_sd_url = f'{srv_end_point}service=haproxy' if haproxy_cnt > 0 else None
         mgr_prometheus_sd_url = f'{srv_end_point}service=mgr-prometheus'  # always included
         ceph_exporter_sd_url = f'{srv_end_point}service=ceph-exporter'  # always included
+        nvmeof_sd_url = f'{srv_end_point}service=nvmeof'  # always included
 
         alertmanager_user, alertmanager_password = self.mgr._get_alertmanager_credentials()
         prometheus_user, prometheus_password = self.mgr._get_prometheus_credentials()
@@ -417,7 +418,8 @@ def generate_config(
             'node_exporter_sd_url': node_exporter_sd_url,
             'alertmanager_sd_url': alertmanager_sd_url,
             'haproxy_sd_url': haproxy_sd_url,
-            'ceph_exporter_sd_url': ceph_exporter_sd_url
+            'ceph_exporter_sd_url': ceph_exporter_sd_url,
+            'nvmeof_sd_url': nvmeof_sd_url,
         }
 
         web_context = {
diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index 7d2de75f67f9..6cd1f4604d31 100644
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -16,6 +16,7 @@
 
 class NvmeofService(CephService):
     TYPE = 'nvmeof'
+    PROMETHEUS_PORT = 10008
 
     def config(self, spec: NvmeofServiceSpec) -> None:  # type: ignore
         assert self.TYPE == spec.service_type
diff --git a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
index 4aa0b9093577..711af0ee724a 100644
--- a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
@@ -10,6 +10,9 @@ state_update_interval_sec = 5
 min_controller_id = {{ spec.min_controller_id }}
 max_controller_id = {{ spec.max_controller_id }}
 enable_spdk_discovery_controller = {{ spec.enable_spdk_discovery_controller }}
+enable_prometheus_exporter = True
+prometheus_exporter_ssl = False
+prometheus_port = 10008
 
 [ceph]
 pool = {{ spec.pool }}
diff --git a/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
index b56843994555..57d2f8a3f4b4 100644
--- a/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
+++ b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
@@ -107,3 +107,23 @@ scrape_configs:
     - url: {{ ceph_exporter_sd_url }}
 {% endif %}
 {% endif %}
+
+{% if nvmeof_sd_url %}
+  - job_name: 'nvmeof'
+{% if secure_monitoring_stack %}
+    honor_labels: true
+    scheme: https
+    tls_config:
+      ca_file: root_cert.pem
+    http_sd_configs:
+    - url: {{ nvmeof_sd_url }}
+      basic_auth:
+        username: {{ service_discovery_username }}
+        password: {{ service_discovery_password }}
+      tls_config:
+        ca_file: root_cert.pem
+{% else %}
+    http_sd_configs:
+    - url: {{ nvmeof_sd_url }}
+{% endif %}
+{% endif %}
diff --git a/src/pybind/mgr/cephadm/tests/test_service_discovery.py b/src/pybind/mgr/cephadm/tests/test_service_discovery.py
index ff98a13885f8..687b64553eaa 100644
--- a/src/pybind/mgr/cephadm/tests/test_service_discovery.py
+++ b/src/pybind/mgr/cephadm/tests/test_service_discovery.py
@@ -19,6 +19,9 @@ def get_daemons_by_service(self, service_type):
         if service_type == 'ceph-exporter':
             return [FakeDaemonDescription('1.2.3.4', [9926], 'node0'),
                     FakeDaemonDescription('1.2.3.5', [9926], 'node1')]
+        if service_type == 'nvmeof':
+            return [FakeDaemonDescription('1.2.3.4', [10008], 'node0'),
+                    FakeDaemonDescription('1.2.3.5', [10008], 'node1')]
 
         return [FakeDaemonDescription('1.2.3.4', [9100], 'node0'),
                 FakeDaemonDescription('1.2.3.5', [9200], 'node1')]
@@ -171,6 +174,20 @@ def test_get_sd_config_ceph_exporter(self):
         # check content
         assert cfg[0]['targets'] == ['1.2.3.4:9926']
 
+    def test_get_sd_config_nvmeof(self):
+        mgr = FakeMgr()
+        root = Root(mgr, 5000, '0.0.0.0')
+        cfg = root.get_sd_config('nvmeof')
+
+        # check response structure
+        assert cfg
+        for entry in cfg:
+            assert 'labels' in entry
+            assert 'targets' in entry
+
+        # check content
+        assert cfg[0]['targets'] == ['1.2.3.4:10008']
+
     def test_get_sd_config_invalid_service(self):
         mgr = FakeMgr()
         root = Root(mgr, 5000, '0.0.0.0')
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index dbab022058d4..6c157ea43363 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -393,6 +393,9 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
 min_controller_id = 1
 max_controller_id = 65519
 enable_spdk_discovery_controller = False
+enable_prometheus_exporter = True
+prometheus_exporter_ssl = False
+prometheus_port = 10008
 
 [ceph]
 pool = {pool}
@@ -716,6 +719,10 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                     honor_labels: true
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=ceph-exporter
+
+                  - job_name: 'nvmeof'
+                    http_sd_configs:
+                    - url: http://[::1]:8765/sd/prometheus/sd-config?service=nvmeof
                 """).lstrip()
 
                 _run_cephadm.assert_called_with(
@@ -872,6 +879,19 @@ def gen_cert(host, addr):
                         password: sd_password
                       tls_config:
                         ca_file: root_cert.pem
+
+                  - job_name: 'nvmeof'
+                    honor_labels: true
+                    scheme: https
+                    tls_config:
+                      ca_file: root_cert.pem
+                    http_sd_configs:
+                    - url: https://[::1]:8765/sd/prometheus/sd-config?service=nvmeof
+                      basic_auth:
+                        username: sd_user
+                        password: sd_password
+                      tls_config:
+                        ca_file: root_cert.pem
                 """).lstrip()
 
                 _run_cephadm.assert_called_with(

From e31c7249b3984d40fd0c2abfef1d2402112ed655 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Tue, 20 Feb 2024 14:04:19 -0700
Subject: [PATCH 2008/2492] kv/RocksDBStore: Configure compact-on-deletion for
 all CFs

update_column_family_options() is called only for non-default CFs,
whereas load_rocksdb_options() sets options for all CFs, including
default. This isn't really a problem for new installs, where the default
CF isn't used for very much, but for upgrades where resharding has not
yet happened, the bulk of rocksdb data is still in the default CF and so
it's important that it also gets compact-on-deletion configured, if
desired.

Fixes: https://tracker.ceph.com/issues/64511
Signed-off-by: Joshua Baergen <jbaergen@digitalocean.com>
---
 src/kv/RocksDBStore.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/kv/RocksDBStore.cc b/src/kv/RocksDBStore.cc
index fd2b26f3803e..28217118609e 100644
--- a/src/kv/RocksDBStore.cc
+++ b/src/kv/RocksDBStore.cc
@@ -585,6 +585,14 @@ int RocksDBStore::load_rocksdb_options(bool create_if_missing, rocksdb::Options&
   if (cct->_conf.get_val<Option::size_t>("rocksdb_metadata_block_size") > 0)
     bbt_opts.metadata_block_size = cct->_conf.get_val<Option::size_t>("rocksdb_metadata_block_size");
 
+  // Set Compact on Deletion Factory
+  if (cct->_conf->rocksdb_cf_compact_on_deletion) {
+    size_t sliding_window = cct->_conf->rocksdb_cf_compact_on_deletion_sliding_window;
+    size_t trigger = cct->_conf->rocksdb_cf_compact_on_deletion_trigger;
+    opt.table_properties_collector_factories.emplace_back(
+        rocksdb::NewCompactOnDeletionCollectorFactory(sliding_window, trigger));
+  }
+
   opt.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbt_opts));
   dout(10) << __func__ << " block size " << cct->_conf->rocksdb_block_size
            << ", block_cache size " << byte_u_t(block_cache_size)
@@ -934,14 +942,6 @@ int RocksDBStore::update_column_family_options(const std::string& base_name,
       return r;
     }
   }
-
-  // Set Compact on Deletion Factory
-  if (cct->_conf->rocksdb_cf_compact_on_deletion) {
-    size_t sliding_window = cct->_conf->rocksdb_cf_compact_on_deletion_sliding_window;
-    size_t trigger = cct->_conf->rocksdb_cf_compact_on_deletion_trigger;
-    cf_opt->table_properties_collector_factories.emplace_back(
-        rocksdb::NewCompactOnDeletionCollectorFactory(sliding_window, trigger));
-  }
   return 0;
 }
 

From 1d8baa1d4c3583f4a8dc0f46d81f198a3d027e4d Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 26 Feb 2024 07:14:25 +1000
Subject: [PATCH 2009/2492] doc/glossary: Add "OMAP" to glossary

Add an entry for "OMAP" to the glossary. This commit represents only the
first improvement to the documentation of OMAP in the Ceph docs suite.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 15dddadc2296..8f1c5659756d 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -287,6 +287,19 @@
 	Object Storage Device
                 See :term:`OSD`.
 
+        OMAP
+                "object map". A key-value store (a database) that is used to
+                reduce the time it takes to read data from and to write to the
+                Ceph cluster. RGW bucket indexes are stored as OMAPs.
+                Erasure-coded pools cannot store RADOS OMAP data structures.
+               
+                Run the command ``ceph osd df`` to see your OMAPs.
+
+                See Eleanor Cawthon's 2012 paper `A Distributed Key-Value Store
+                using Ceph
+                <https://ceph.io/assets/pdfs/CawthonKeyValueStore.pdf>`_ (17
+                pages).
+
 	OSD
                 Probably :term:`Ceph OSD`, but not necessarily. Sometimes
                 (especially in older correspondence, and especially in

From feb1e69034026583a6770512a90b0172b6b5c76a Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Fri, 9 Feb 2024 16:56:58 +1300
Subject: [PATCH 2010/2492] ceph-mixins: Add vars to support nvmeof alerts

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 monitoring/ceph-mixin/config.libsonnet | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/monitoring/ceph-mixin/config.libsonnet b/monitoring/ceph-mixin/config.libsonnet
index e14bce4a424e..41916efdfdfc 100644
--- a/monitoring/ceph-mixin/config.libsonnet
+++ b/monitoring/ceph-mixin/config.libsonnet
@@ -9,5 +9,15 @@
     CephNodeNetworkPacketDropsPerSec: 10,
     CephRBDMirrorImageTransferBandwidthThreshold: 0.8,
     CephRBDMirrorImagesPerDaemonThreshold: 100,
+    NVMeoFMaxGatewaysPerGroup: 2,
+    NVMeoFMaxGatewaysPerCluster: 4,
+    NVMeoFHighGatewayCPU: 80,
+    NVMeoFMaxSubsystemsPerGateway: 16,
+    NVMeoFHighClientCount: 32,
+    NVMeoFHighHostCPU: 80,
+    //
+    // Read/Write latency is defined in ms
+    NVMeoFHighClientReadLatency: 10,
+    NVMeoFHighClientWriteLatency: 20,
   },
 }

From d5639cc670d248da7c84e6f2eaf73ce40bd93db4 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Wed, 22 Nov 2023 16:13:08 +0800
Subject: [PATCH 2011/2492] doc/man/8/mount.ceph.rst: add more mount options

Fixes: https://tracker.ceph.com/issues/62837
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 doc/man/8/mount.ceph.rst | 44 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/doc/man/8/mount.ceph.rst b/doc/man/8/mount.ceph.rst
index fbe8790ddbdc..e45030cd096c 100644
--- a/doc/man/8/mount.ceph.rst
+++ b/doc/man/8/mount.ceph.rst
@@ -199,6 +199,50 @@ Advanced
     option is enabled, a namespace operation may complete before the MDS
     replies, if it has sufficient capabilities to do so.
 
+:command:`crush_location=x`
+    Specify the location of the client in terms of CRUSH hierarchy (since 5.8).
+    This is a set of key-value pairs separated from each other by '|', with
+    keys separated from values by ':'.  Note that '|' may need to be quoted
+    or escaped to avoid it being interpreted as a pipe by the shell. The key
+    is the bucket type name (e.g. rack, datacenter or region with default
+    bucket types) and the value is the bucket name. For example, to indicate
+    that the client is local to rack "myrack", data center "mydc" and region
+    "myregion"::
+
+      crush_location=rack:myrack|datacenter:mydc|region:myregion
+
+    Each key-value pair stands on its own: "myrack" doesn't need to reside in
+    "mydc", which in turn doesn't need to reside in "myregion".  The location
+    is not a path to the root of the hierarchy but rather a set of nodes that
+    are matched independently.  "Multipath" locations are supported, so it is
+    possible to indicate locality for multiple parallel hierarchies::
+
+      crush_location=rack:myrack1|rack:myrack2|datacenter:mydc
+
+
+:command:`read_from_replica=<no|balance|localize>`
+    - ``no``: Disable replica reads, always pick the primary OSD (since 5.8, default).
+
+    - ``balance``: When a replicated pool receives a read request, pick a random
+      OSD from the PG's acting set to serve it (since 5.8).
+
+      This mode is safe for general use only since Octopus (i.e. after "ceph osd
+      require-osd-release octopus"). Otherwise it should be limited to read-only
+      workloads such as snapshots.
+
+    - ``localize``: When a replicated pool receives a read request, pick the most
+      local OSD to serve it (since 5.8). The locality metric is calculated against
+      the location of the client given with crush_location; a match with the
+      lowest-valued bucket type wins.  For example, an OSD in a matching rack
+      is closer than an OSD in a matching data center, which in turn is closer
+      than an OSD in a matching region.
+
+      This mode is safe for general use only since Octopus (i.e. after "ceph osd
+      require-osd-release octopus").  Otherwise it should be limited to read-only
+      workloads such as snapshots.
+
+
+
 Examples
 ========
 

From 5b03e4d3d73466c34e57f094e5ce2e51af56bd78 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Mon, 26 Feb 2024 14:17:48 +0800
Subject: [PATCH 2012/2492] test/crimson/test_socket: complete write-side
 errors in test_unexpected_down()

Related: https://tracker.ceph.com/issues/64457
Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/test/crimson/test_socket.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc
index 8a13fb77ec7f..99c8aee84f77 100644
--- a/src/test/crimson/test_socket.cc
+++ b/src/test/crimson/test_socket.cc
@@ -451,8 +451,11 @@ future<> test_unexpected_down(bool is_fixed_cpu) {
       return Connection::dispatch_rw_bounded(cs, 128, true
         ).handle_exception_type([](const std::system_error& e) {
         logger().error("test_unexpected_down(): client get error {}", e);
+        // union of errors from both read and write
+        // also see dispatch_write_unbounded() and dispatch_read_unbounded()
         ceph_assert(e.code() == error::read_eof ||
-		    e.code() == std::errc::connection_reset);
+		    e.code() == std::errc::connection_reset ||
+                    e.code() == std::errc::broken_pipe);
       });
     },
     [](auto ss) { return Connection::dispatch_rw_unbounded(ss); }

From 4313f9a59c676067ca23242808c8e87068d0d953 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 22 Feb 2024 11:06:21 +0100
Subject: [PATCH 2013/2492] mgr/dashboard: fix volume creation with multiple
 hosts

Fixes: https://tracker.ceph.com/issues/64559
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/cephfs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index ad2cafe0ba8e..6849f34e9219 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -56,7 +56,7 @@ def create(self, name: str, service_spec: Dict[str, Any]):
             service_spec_str = service_spec_str[:-1]
         if 'hosts' in service_spec['placement']:
             for host in service_spec['placement']['hosts']:
-                service_spec_str += f'{host},'
+                service_spec_str += f'{host} '
             service_spec_str = service_spec_str[:-1]
 
         error_code, _, err = mgr.remote('volumes', '_cmd_fs_volume_create', None,

From 878463ef3a26f49b9ccad9696ad130fb5aee47e7 Mon Sep 17 00:00:00 2001
From: "yite.gu" <yitegu0@gmail.com>
Date: Mon, 29 Jan 2024 18:26:20 +0800
Subject: [PATCH 2014/2492] client: check mds down status bofore getting
 mds_gid_t from mdsmap

Get mds_gid_t from up of MDSMap, will cause to core dump if target
mds_rank_t does not exist up map:

from: src/mds/MDSMap.h
  const auto& get_info(mds_rank_t m) const {
    return mds_info.at(up.at(m));
  }

reproduct:
1. ceph fs fail <fs_name>
2. curl -X GET "example.com:8080/api/cephfs/1/clients" ...
up.at(m) will cause to core dump.

Fixes: https://tracker.ceph.com/issues/61844
Signed-off-by: Yite Gu <yitegu0@gmail.com>
---
 qa/tasks/mgr/dashboard/test_cephfs.py          | 14 ++++++++++++++
 src/client/Client.cc                           |  5 +++++
 src/pybind/mgr/dashboard/controllers/cephfs.py |  3 ++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/mgr/dashboard/test_cephfs.py b/qa/tasks/mgr/dashboard/test_cephfs.py
index d8e00fe60d9d..7a6b912a3fb8 100644
--- a/qa/tasks/mgr/dashboard/test_cephfs.py
+++ b/qa/tasks/mgr/dashboard/test_cephfs.py
@@ -347,3 +347,17 @@ def test_statfs(self):
         self.assertEqual(stats['subdirs'], 1)
 
         self.rm_dir('/animal')
+
+    def test_cephfs_clients_get_after_mds_down(self):
+        fs_id = self.get_fs_id()
+        self._get(f"/api/cephfs/{fs_id}/clients")
+        self.assertStatus(200)
+
+        self.fs.fail()
+        params = {'suppress_client_ls_errors': 'False'}
+        self._get(f"/api/cephfs/{fs_id}/clients", params=params)
+        self.assertStatus(500)
+
+        self.fs.set_joinable()
+        self._get(f"/api/cephfs/{fs_id}/clients")
+        self.assertStatus(200)
diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8966bf60adb0..6af4d24cae53 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -6273,6 +6273,11 @@ int Client::resolve_mds(
   if (role_r == 0) {
     // We got a role, resolve it to a GID
     const auto& mdsmap = fsmap->get_filesystem(role.fscid).get_mds_map();
+    if (mdsmap.is_down(role.rank)) {
+      lderr(cct) << __func__ << ": targets rank: " << role.rank
+                 << " is down" << dendl;
+      return -CEPHFS_EAGAIN;
+    }
     auto& info = mdsmap.get_info(role.rank);
     ldout(cct, 10) << __func__ << ": resolved " << mds_spec << " to role '"
       << role << "' aka " << info.human_name() << dendl;
diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index fca5ddc95ee1..84af1b9e2f2e 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=too-many-lines
 import errno
 import json
 import logging
@@ -110,7 +111,7 @@ def clients(self, fs_id, **kwargs):
         if flag not in ('True', 'False'):
             raise DashboardException(msg='suppress_client_ls_errors value '
                                          'needs to be either True or False '
-                                         f'but provided {flag}',
+                                         f'but provided "{flag}"',
                                      component='cephfs')
 
         fs_id = self.fs_id_to_int(fs_id)

From b4f3ddfe4385c12baa0f44fff9d2708c445625d3 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 26 Feb 2024 09:38:52 -0500
Subject: [PATCH 2015/2492] test/rgw: increase timeouts in
 unittest_rgw_dmclock_scheduler

1ms sleeps are generally below the timer's resolution. increase run_for()
durations to 50ms to make the tests far less sensitive to timing. in
practice, none of the sleeps actually wait the full 50ms

Fixes: https://tracker.ceph.com/issues/64568

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/test/rgw/test_rgw_dmclock_scheduler.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc
index 36652d22f038..c9b4a853fd4d 100644
--- a/src/test/rgw/test_rgw_dmclock_scheduler.cc
+++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc
@@ -105,7 +105,7 @@ TEST(Queue, RateLimit)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -163,7 +163,7 @@ TEST(Queue, AsyncRequest)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -217,7 +217,7 @@ TEST(Queue, Cancel)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -265,7 +265,7 @@ TEST(Queue, CancelClient)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -315,7 +315,7 @@ TEST(Queue, CancelOnDestructor)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -369,20 +369,20 @@ TEST(Queue, CrossExecutorRequest)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  callback_context.run_for(std::chrono::milliseconds(1));
+  callback_context.poll();
   // maintains work on callback executor while in queue
   EXPECT_FALSE(callback_context.stopped());
 
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  queue_context.run_for(std::chrono::milliseconds(1));
+  queue_context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(queue_context.stopped());
 
   EXPECT_FALSE(ec1); // no callbacks until callback executor runs
   EXPECT_FALSE(ec2);
 
-  callback_context.run_for(std::chrono::milliseconds(1));
+  callback_context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(callback_context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -421,7 +421,7 @@ TEST(Queue, SpawnAsyncRequest)
     EXPECT_EQ(PhaseType::priority, p2);
   });
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 }
 

From 9677ad6dddf34302e5ad1cde6e72342d06e73642 Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Fri, 9 Feb 2024 14:41:19 +0530
Subject: [PATCH 2016/2492] qa: Add benign cluster warning from
 ec-inconsistent-hinfo test to ignorelist

The changes introduced in PR: https://github.com/ceph/ceph/pull/53524
made the randomized values of osd_op_queue and osd_op_queue_cut_off
consistent across all OSD shards.

Due to the above, ec-inconsistent-hinfo test could fail with the following
cluster warning (benign) depending on the randomly selected scheduler type.

"cluster [WRN] Error(s) ignored for 2:ad551702:::test:head enough copies
available"

In summary, the warning is generated due to the difference in the PG
deletion rates between WPQ and mClock schedulers. Therefore, the warning
shows up in cases where the mClock scheduler is the op queue scheduler
chosen randomly for the test. The PG deletion rate with mClock scheduler
is quicker compared to the WPQ scheduler since it doesn't use sleeps
between each delete transaction and relies on the cost of the deletion
which in turn is proportional to the average size of the objects in the PG.

For a more detailed analysis, see the associated tracker.

Fixes: https://tracker.ceph.com/issues/64573
Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml b/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml
index 31724f9e81b1..84abb702cf46 100644
--- a/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml
+++ b/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml
@@ -30,6 +30,7 @@ tasks:
       - slow request
       - unfound
       - \(POOL_APP_NOT_ENABLED\)
+      - enough copies available
     conf:
       osd:
         osd min pg log entries: 5

From 9c34973932bf3a0ec50c1c63bcba5e35bfe407e5 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 16 Feb 2024 11:24:32 -0500
Subject: [PATCH 2017/2492] mgr/cephadm: catch CancelledError in asyncio
 timeout handler

Specifically, concurrent.futures.CancelledError. At least on
python 3.9, this error can be raised when certain commands
being run asynchronously fail. Not catching this results in
the whole cephadm module crashing with something like

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/utils.py", line 94, in do_work
    return f(*arg)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 267, in refresh
    r = self._refresh_facts(host)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 370, in _refresh_facts
    val = self.mgr.wait_async(self._run_cephadm_json(
  File "/usr/share/ceph/mgr/cephadm/module.py", line 671, in wait_async
    return self.event_loop.get_result(coro, timeout)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 64, in get_result
    return future.result(timeout)
  File "/lib64/python3.9/concurrent/futures/_base.py", line 444, in result
    raise CancelledError()
concurrent.futures._base.CancelledError

Fixes: https://tracker.ceph.com/issues/64473

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 87f7024bb255..7d9aa02bade4 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -761,6 +761,16 @@ def async_timeout_handler(self, host: Optional[str] = '',
             else:
                 err_str += (f'(default {self.default_cephadm_command_timeout} second timeout)')
             raise OrchestratorError(err_str)
+        except concurrent.futures.CancelledError as e:
+            err_str = ''
+            if cmd:
+                err_str = f'Command "{cmd}" failed '
+            else:
+                err_str = 'Command failed '
+            if host:
+                err_str += f'on host {host} '
+            err_str += f' - {str(e)}'
+            raise OrchestratorError(err_str)
 
     def set_container_image(self, entity: str, image: str) -> None:
         self.check_mon_command({

From f1573b76f3485aed148a8aeb69a40ef69df58912 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Fri, 9 Feb 2024 17:00:28 +1300
Subject: [PATCH 2018/2492] ceph-mixins: Add nvmeof alerts

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 .../ceph-mixin/prometheus_alerts.libsonnet    | 145 ++++++++++++++++++
 1 file changed, 145 insertions(+)

diff --git a/monitoring/ceph-mixin/prometheus_alerts.libsonnet b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
index b5fed4fe7cf7..fa2899b22c18 100644
--- a/monitoring/ceph-mixin/prometheus_alerts.libsonnet
+++ b/monitoring/ceph-mixin/prometheus_alerts.libsonnet
@@ -846,5 +846,150 @@
         },
       ],
     },
+    {
+      name: 'nvmeof',
+      rules: [
+        {
+          alert: 'NVMeoFSubsystemNamespaceLimit',
+          'for': '1m',
+          expr: '(count by(nqn) (ceph_nvmeof_subsystem_namespace_metadata)) >= ceph_nvmeof_subsystem_namespace_limit',
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: '{{ $labels.nqn }} subsystem has reached its maximum number of namespaces %(cluster)s' % $.MultiClusterSummary(),
+            description: 'Subsystems have a max namespace limit defined at creation time. This alert means that no more namespaces can be added to {{ $labels.nqn }}',
+          },
+        },
+        {
+          alert: 'NVMeoFTooManyGateways',
+          'for': '1m',
+          expr: 'count(ceph_nvmeof_gateway_info) > %.2f' % [$._config.NVMeoFMaxGatewaysPerCluster],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'Max supported gateways exceeded %(cluster)s' % $.MultiClusterSummary(),
+            description: 'You may create many gateways, but %(NVMeoFMaxGatewaysPerCluster)d is the tested limit' % $._config,
+          },
+        },
+        {
+          alert: 'NVMeoFMaxGatewayGroupSize',
+          'for': '1m',
+          expr: 'count by(group) (ceph_nvmeof_gateway_info) > %.2f' % [$._config.NVMeoFMaxGatewaysPerGroup],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'Max gateways within a gateway group ({{ $labels.group }}) exceeded %(cluster)s' % $.MultiClusterSummary(),
+            description: 'You may create many gateways in a gateway group, but %(NVMeoFMaxGatewaysPerGroup)d is the tested limit' % $._config,
+          },
+        },
+        {
+          alert: 'NVMeoFSingleGatewayGroup',
+          'for': '5m',
+          expr: 'count by(group) (ceph_nvmeof_gateway_info) == 1',
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The gateway group {{ $labels.group }} consists of a single gateway - HA is not possible %(cluster)s' % $.MultiClusterSummary(),
+            description: 'Although a single member gateway group is valid, it should only be used for test purposes',
+          },
+        },
+        {
+          alert: 'NVMeoFHighGatewayCPU',
+          'for': '10m',
+          expr: 'label_replace(avg by(instance) (rate(ceph_nvmeof_reactor_seconds_total{mode="busy"}[1m])),"instance","$1","instance","(.*):.*") > %.2f' % [$._config.NVMeoFHighGatewayCPU],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'CPU used by {{ $labels.instance }} NVMe-oF Gateway is high %(cluster)s' % $.MultiClusterSummary(),
+            description: 'Typically, high CPU may indicate degraded performance. Consider increasing the number of reactor cores',
+          },
+        },
+        {
+          alert: 'NVMeoFGatewayOpenSecurity',
+          'for': '5m',
+          expr: 'ceph_nvmeof_subsystem_metadata{allow_any_host="yes"}',
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'Subsystem {{ $labels.nqn }} has been defined without host level security %(cluster)s' % $.MultiClusterSummary(),
+            description: 'It is good practice to ensure subsystems use host security to reduce the risk of unexpected data loss',
+          },
+        },
+        {
+          alert: 'NVMeoFTooManySubsystems',
+          'for': '1m',
+          expr: 'count by(gateway_host) (label_replace(ceph_nvmeof_subsystem_metadata,"gateway_host","$1","instance","(.*):.*")) > %.2f' % [$._config.NVMeoFMaxSubsystemsPerGateway],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The number of subsystems defined to the gateway exceeds supported values %(cluster)s' % $.MultiClusterSummary(),
+            description: 'Although you may continue to create subsystems in {{ $labels.gateway_host }}, the configuration may not be supported',
+          },
+        },
+        {
+          alert: 'NVMeoFVersionMismatch',
+          'for': '1h',
+          expr: 'count(count by(version) (ceph_nvmeof_gateway_info)) > 1',
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The cluster has different NVMe-oF gateway releases active %(cluster)s' % $.MultiClusterSummary(),
+            description: 'This may indicate an issue with deployment. Check cephadm logs',
+          },
+        },
+        {
+          alert: 'NVMeoFHighClientCount',
+          'for': '1m',
+          expr: 'ceph_nvmeof_subsystem_host_count > %.2f' % [$._config.NVMeoFHighClientCount],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The number of clients connected to {{ $labels.nqn }} is too high %(cluster)s' % $.MultiClusterSummary(),
+            description: 'The supported limit for clients connecting to a subsystem is %(NVMeoFHighClientCount)d' % $._config,
+          },
+        },
+        {
+          alert: 'NVMeoFHighHostCPU',
+          'for': '10m',
+          expr: '100-((100*(avg by(host) (label_replace(rate(node_cpu_seconds_total{mode="idle"}[5m]),"host","$1","instance","(.*):.*")) * on(host) group_right label_replace(ceph_nvmeof_gateway_info,"host","$1","instance","(.*):.*")))) >= %.2f' % [$._config.NVMeoFHighHostCPU],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The CPU is high ({{ $value }}%%) on NVMeoF Gateway host ({{ $labels.host }}) %(cluster)s' % $.MultiClusterSummary(),
+            description: 'High CPU on a gateway host can lead to CPU contention and performance degradation',
+          },
+        },
+        {
+          alert: 'NVMeoFInterfaceDown',
+          'for': '30s',
+          expr: 'ceph_nvmeof_subsystem_listener_iface_info{operstate="down"}',
+          labels: { severity: 'warning', type: 'ceph_default', oid: '1.3.6.1.4.1.50495.1.2.1.14.1' },
+          annotations: {
+            summary: 'Network interface {{ $labels.device }} is down %(cluster)s' % $.MultiClusterSummary(),
+            description: 'A NIC used by one or more subsystems is in a down state',
+          },
+        },
+        {
+          alert: 'NVMeoFInterfaceDuplex',
+          'for': '30s',
+          expr: 'ceph_nvmeof_subsystem_listener_iface_info{duplex!="full"}',
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'Network interface {{ $labels.device }} is not running in full duplex mode %(cluster)s' % $.MultiClusterSummary(),
+            description: 'Until this is resolved, performance from the gateway will be degraded',
+          },
+        },
+        {
+          alert: 'NVMeoFHighReadLatency',
+          'for': '5m',
+          expr: 'label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_read_seconds_total[1m]) / rate(ceph_nvmeof_bdev_reads_completed_total[1m])))),"gateway","$1","instance","(.*):.*") > %.2f' % [$._config.NVMeoFHighClientReadLatency / 1000],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The average read latency over the last 5 mins has reached %(NVMeoFHighClientReadLatency)d ms or more on {{ $labels.gateway }}' % $._config,
+            description: 'High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate',
+          },
+        },
+        {
+          alert: 'NVMeoFHighWriteLatency',
+          'for': '5m',
+          expr: 'label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_write_seconds_total[5m]) / rate(ceph_nvmeof_bdev_writes_completed_total[5m])))),"gateway","$1","instance","(.*):.*") > %.2f' % [$._config.NVMeoFHighClientWriteLatency / 1000],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'The average write latency over the last 5 mins has reached %(NVMeoFHighClientWriteLatency)d ms or more on {{ $labels.gateway }}' % $._config,
+            description: 'High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate',
+          },
+        },
+      ],
+    },
   ],
 }

From e7d25482d1b418bf6b1c75dac60c64fd24ab6f01 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Tue, 13 Feb 2024 13:54:48 +1300
Subject: [PATCH 2019/2492] ceph-mixins: nvmeof alerts added

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 monitoring/ceph-mixin/prometheus_alerts.yml | 129 ++++++++++++++++++++
 1 file changed, 129 insertions(+)

diff --git a/monitoring/ceph-mixin/prometheus_alerts.yml b/monitoring/ceph-mixin/prometheus_alerts.yml
index 5b011aa3568a..84452e5845a9 100644
--- a/monitoring/ceph-mixin/prometheus_alerts.yml
+++ b/monitoring/ceph-mixin/prometheus_alerts.yml
@@ -756,3 +756,132 @@ groups:
           oid: "1.3.6.1.4.1.50495.1.2.1.10.5"
           severity: "warning"
           type: "ceph_default"
+  - name: "nvmeof"
+    rules:
+      - alert: "NVMeoFSubsystemNamespaceLimit"
+        annotations:
+          description: "Subsystems have a max namespace limit defined at creation time. This alert means that no more namespaces can be added to {{ $labels.nqn }}"
+          summary: "{{ $labels.nqn }} subsystem has reached its maximum number of namespaces "
+        expr: "(count by(nqn) (ceph_nvmeof_subsystem_namespace_metadata)) >= ceph_nvmeof_subsystem_namespace_limit"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFTooManyGateways"
+        annotations:
+          description: "You may create many gateways, but 4 is the tested limit"
+          summary: "Max supported gateways exceeded "
+        expr: "count(ceph_nvmeof_gateway_info) > 4.00"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFMaxGatewayGroupSize"
+        annotations:
+          description: "You may create many gateways in a gateway group, but 2 is the tested limit"
+          summary: "Max gateways within a gateway group ({{ $labels.group }}) exceeded "
+        expr: "count by(group) (ceph_nvmeof_gateway_info) > 2.00"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFSingleGatewayGroup"
+        annotations:
+          description: "Although a single member gateway group is valid, it should only be used for test purposes"
+          summary: "The gateway group {{ $labels.group }} consists of a single gateway - HA is not possible "
+        expr: "count by(group) (ceph_nvmeof_gateway_info) == 1"
+        for: "5m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFHighGatewayCPU"
+        annotations:
+          description: "Typically, high CPU may indicate degraded performance. Consider increasing the number of reactor cores"
+          summary: "CPU used by {{ $labels.instance }} NVMe-oF Gateway is high "
+        expr: "label_replace(avg by(instance) (rate(ceph_nvmeof_reactor_seconds_total{mode=\"busy\"}[1m])),\"instance\",\"$1\",\"instance\",\"(.*):.*\") > 80.00"
+        for: "10m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFGatewayOpenSecurity"
+        annotations:
+          description: "It is good practice to ensure subsystems use host security to reduce the risk of unexpected data loss"
+          summary: "Subsystem {{ $labels.nqn }} has been defined without host level security "
+        expr: "ceph_nvmeof_subsystem_metadata{allow_any_host=\"yes\"}"
+        for: "5m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFTooManySubsystems"
+        annotations:
+          description: "Although you may continue to create subsystems in {{ $labels.gateway_host }}, the configuration may not be supported"
+          summary: "The number of subsystems defined to the gateway exceeds supported values "
+        expr: "count by(gateway_host) (label_replace(ceph_nvmeof_subsystem_metadata,\"gateway_host\",\"$1\",\"instance\",\"(.*):.*\")) > 16.00"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFVersionMismatch"
+        annotations:
+          description: "This may indicate an issue with deployment. Check cephadm logs"
+          summary: "The cluster has different NVMe-oF gateway releases active "
+        expr: "count(count by(version) (ceph_nvmeof_gateway_info)) > 1"
+        for: "1h"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFHighClientCount"
+        annotations:
+          description: "The supported limit for clients connecting to a subsystem is 32"
+          summary: "The number of clients connected to {{ $labels.nqn }} is too high "
+        expr: "ceph_nvmeof_subsystem_host_count > 32.00"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFHighHostCPU"
+        annotations:
+          description: "High CPU on a gateway host can lead to CPU contention and performance degradation"
+          summary: "The CPU is high ({{ $value }}%) on NVMeoF Gateway host ({{ $labels.host }}) "
+        expr: "100-((100*(avg by(host) (label_replace(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]),\"host\",\"$1\",\"instance\",\"(.*):.*\")) * on(host) group_right label_replace(ceph_nvmeof_gateway_info,\"host\",\"$1\",\"instance\",\"(.*):.*\")))) >= 80.00"
+        for: "10m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFInterfaceDown"
+        annotations:
+          description: "A NIC used by one or more subsystems is in a down state"
+          summary: "Network interface {{ $labels.device }} is down "
+        expr: "ceph_nvmeof_subsystem_listener_iface_info{operstate=\"down\"}"
+        for: "30s"
+        labels:
+          oid: "1.3.6.1.4.1.50495.1.2.1.14.1"
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFInterfaceDuplex"
+        annotations:
+          description: "Until this is resolved, performance from the gateway will be degraded"
+          summary: "Network interface {{ $labels.device }} is not running in full duplex mode "
+        expr: "ceph_nvmeof_subsystem_listener_iface_info{duplex!=\"full\"}"
+        for: "30s"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFHighReadLatency"
+        annotations:
+          description: "High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate"
+          summary: "The average read latency over the last 5 mins has reached 10 ms or more on {{ $labels.gateway }}"
+        expr: "label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_read_seconds_total[1m]) / rate(ceph_nvmeof_bdev_reads_completed_total[1m])))),\"gateway\",\"$1\",\"instance\",\"(.*):.*\") > 0.01"
+        for: "5m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
+      - alert: "NVMeoFHighWriteLatency"
+        annotations:
+          description: "High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate"
+          summary: "The average write latency over the last 5 mins has reached 20 ms or more on {{ $labels.gateway }}"
+        expr: "label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_write_seconds_total[5m]) / rate(ceph_nvmeof_bdev_writes_completed_total[5m])))),\"gateway\",\"$1\",\"instance\",\"(.*):.*\") > 0.02"
+        for: "5m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"

From c2534a6dbacf0f2ff5c649f7a0e04b5a94dbfdc0 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Tue, 13 Feb 2024 13:55:24 +1300
Subject: [PATCH 2020/2492] ceph-mixins: Add test cases for nvmeof alerts

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 .../ceph-mixin/tests_alerts/test_alerts.yml   | 423 ++++++++++++++++++
 1 file changed, 423 insertions(+)

diff --git a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
index 9cb688e9ca0e..40d6f4d0983a 100644
--- a/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
+++ b/monitoring/ceph-mixin/tests_alerts/test_alerts.yml
@@ -2219,3 +2219,426 @@ tests:
         exp_annotations:
           summary: Fan error(s) detected
           description: "Fan error(s) detected. Check `ceph health detail`."
+
+# nvmeof Tests
+ # NVMeoFSubsystemNamespaceLimit
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_namespace_limit{nqn="wah"}'
+      values: '5x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk1"}'
+      values: '1x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk2"}'
+      values: '1x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk3"}'
+      values: '1x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk4"}'
+      values: '1x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk5"}'
+      values: '1x10'
+    - series: 'ceph_nvmeof_subsystem_namespace_metadata{nqn="wah", bdev_name="disk6"}'
+      values: '1x10'      
+   promql_expr_test:
+     - expr: (count by(nqn) (ceph_nvmeof_subsystem_namespace_metadata)) >= ceph_nvmeof_subsystem_namespace_limit
+       eval_time: 1m
+       exp_samples:
+         - labels: '{nqn="wah"}'
+           value: 6
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFSubsystemNamespaceLimit
+      exp_alerts:
+      - exp_labels:
+          nqn: wah
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "wah subsystem has reached its maximum number of namespaces "
+          description: "Subsystems have a max namespace limit defined at creation time. This alert means that no more namespaces can be added to wah"
+
+ # NVMeoFTooManyGateways
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_gateway_info{addr="1.1.1.1"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{addr="1.1.1.2"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{addr="1.1.1.3"}'
+      values: '1+0x20'      
+    - series: 'ceph_nvmeof_gateway_info{addr="1.1.1.4"}'
+      values: '1+0x20' 
+    - series: 'ceph_nvmeof_gateway_info{addr="1.1.1.5"}'
+      values: '1+0x20'             
+   promql_expr_test:
+     - expr: count(ceph_nvmeof_gateway_info) > 4.00
+       eval_time: 1m
+       exp_samples:
+         - labels: '{}'
+           value: 5
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFTooManyGateways
+      exp_alerts:
+      - exp_labels:
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "Max supported gateways exceeded "
+          description: "You may create many gateways, but 4 is the tested limit"
+
+ # NVMeoFMaxGatewayGroupSize
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_gateway_info{group="group-1",addr="1.1.1.1"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-1",addr="1.1.1.2"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-1",addr="1.1.1.3"}'
+      values: '1+0x20'      
+    - series: 'ceph_nvmeof_gateway_info{group="group-2",addr="1.1.1.4"}'
+      values: '1+0x20' 
+    - series: 'ceph_nvmeof_gateway_info{group="group-2",addr="1.1.1.5"}'
+      values: '1+0x20'             
+   promql_expr_test:
+     - expr: count by(group) (ceph_nvmeof_gateway_info) > 2.00
+       eval_time: 1m
+       exp_samples:
+         - labels: '{group="group-1"}'
+           value: 3
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFMaxGatewayGroupSize
+      exp_alerts:
+      - exp_labels:
+          group: group-1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "Max gateways within a gateway group (group-1) exceeded "
+          description: "You may create many gateways in a gateway group, but 2 is the tested limit"
+
+ # NVMeoFSingleGatewayGroup
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_gateway_info{group="group-1",addr="1.1.1.2"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-2",addr="1.1.1.4"}'
+      values: '1+0x20' 
+    - series: 'ceph_nvmeof_gateway_info{group="group-2",addr="1.1.1.5"}'
+      values: '1+0x20'             
+   promql_expr_test:
+     - expr: count by(group) (ceph_nvmeof_gateway_info) == 1
+       eval_time: 1m
+       exp_samples:
+         - labels: '{group="group-1"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFSingleGatewayGroup
+      exp_alerts:
+      - exp_labels:
+          group: group-1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The gateway group group-1 consists of a single gateway - HA is not possible "
+          description: "Although a single member gateway group is valid, it should only be used for test purposes" 
+
+ # NVMeoFHighGatewayCPU
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_reactor_seconds_total{mode="busy",name="nvmf_tgt_poll_group_0",instance="node-1:10008"}'
+      values: '880+5080x20'
+   promql_expr_test:
+     - expr: label_replace(avg by(instance) (rate(ceph_nvmeof_reactor_seconds_total{mode="busy"}[1m])),"instance","$1","instance","(.*):.*") > 80
+       eval_time: 5m
+       exp_samples:
+         - labels: '{instance="node-1"}'
+           value: 8.466666666666667E+01
+   alert_rule_test:
+    - eval_time: 15m
+      alertname: NVMeoFHighGatewayCPU
+      exp_alerts:
+      - exp_labels:
+          instance: node-1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "CPU used by node-1 NVMe-oF Gateway is high "
+          description: "Typically, high CPU may indicate degraded performance. Consider increasing the number of reactor cores" 
+ 
+ # NVMeoFGatewayOpenSecurity
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_metadata{nqn="nqn.good", allow_any_host="no"}'
+      values: '1+0x10'
+    - series: 'ceph_nvmeof_subsystem_metadata{nqn="nqn.bad", allow_any_host="yes"}'
+      values: '1+0x10'
+   promql_expr_test:
+     - expr: ceph_nvmeof_subsystem_metadata{allow_any_host="yes"}
+       eval_time: 1m
+       exp_samples:
+         - labels: '{__name__="ceph_nvmeof_subsystem_metadata",nqn="nqn.bad",allow_any_host="yes"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFGatewayOpenSecurity
+      exp_alerts:
+      - exp_labels:
+          allow_any_host: yes
+          nqn: nqn.bad
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "Subsystem nqn.bad has been defined without host level security "
+          description: "It is good practice to ensure subsystems use host security to reduce the risk of unexpected data loss" 
+
+ # NVMeoFTooManySubsystems
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn1"}'
+      values: '1+0x10'
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn2"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn3"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn4"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn5"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn6"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn7"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn8"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn9"}'
+      values: '1+0x10'                                             
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn10"}'
+      values: '1+0x10'
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn11"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn12"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn13"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn14"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn15"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn16"}'
+      values: '1+0x10'  
+    - series: 'ceph_nvmeof_subsystem_metadata{instance="node-1:10008",nqn="nqn17"}'
+      values: '1+0x10'  
+   promql_expr_test:
+     - expr: count by(gateway_host) (label_replace(ceph_nvmeof_subsystem_metadata,"gateway_host","$1","instance","(.*):.*")) > 16
+       eval_time: 1m
+       exp_samples:
+         - labels: '{gateway_host="node-1"}'
+           value: 17
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFTooManySubsystems
+      exp_alerts:
+      - exp_labels:
+          gateway_host: node-1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The number of subsystems defined to the gateway exceeds supported values "
+          description: "Although you may continue to create subsystems in node-1, the configuration may not be supported" 
+
+ # NVMeoFVersionMismatch
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_gateway_info{version="0.0.7"}'
+      values: '1+0x80'
+    - series: 'ceph_nvmeof_gateway_info{version="1.0.0"}'
+      values: '1+0x80'
+   promql_expr_test:
+     - expr: count(count by(version) (ceph_nvmeof_gateway_info)) > 1
+       eval_time: 1m
+       exp_samples:
+         - labels: '{}'
+           value: 2
+   alert_rule_test:
+    - eval_time: 1h
+      alertname: NVMeoFVersionMismatch
+      exp_alerts:
+      - exp_labels:
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The cluster has different NVMe-oF gateway releases active "
+          description: "This may indicate an issue with deployment. Check cephadm logs"  
+
+ # NVMeoFHighClientCount
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_host_count{nqn="nqn1"}'
+      values: '2 2 2 4 4 8 8 8 10 10 20 20 32 34 34 38 38 40 44 44'
+    - series: 'ceph_nvmeof_subsystem_host_count{nqn="nqn2"}'
+      values: '2 2 2 8 8 8 16 16 16 16 16 16 16 16 16 16 16 16 16 16'
+   promql_expr_test:
+     - expr: ceph_nvmeof_subsystem_host_count > 32.00
+       eval_time: 15m
+       exp_samples:
+         - labels: '{__name__="ceph_nvmeof_subsystem_host_count",nqn="nqn1"}'
+           value: 38
+   alert_rule_test:
+    - eval_time: 20m
+      alertname: NVMeoFHighClientCount
+      exp_alerts:
+      - exp_labels:
+          nqn: nqn1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The number of clients connected to nqn1 is too high "
+          description: "The supported limit for clients connecting to a subsystem is 32" 
+ 
+ # NVMeoFHighHostCPU
+ - interval: 1m
+   input_series:
+    - series: 'node_cpu_seconds_total{mode="idle",instance="node-1:9100",cpu="0"}'
+      values: '0+18x10 180+9x20'
+    - series: 'node_cpu_seconds_total{mode="idle",instance="node-1:9100",cpu="1"}'
+      values: '0+18x10 180+9x20'
+    - series: 'ceph_nvmeof_gateway_info{instance="node-1:10008"}'
+      values: '1.00+0x20'
+   promql_expr_test:
+     - expr: 100-((100*(avg by(host) (label_replace(rate(node_cpu_seconds_total{mode="idle"}[5m]),"host","$1","instance","(.*):.*")) * on(host) group_right label_replace(ceph_nvmeof_gateway_info,"host","$1","instance","(.*):.*")))) >= 80
+       eval_time: 16m
+       exp_samples:
+         - labels: '{host="node-1",instance="node-1:10008"}'
+           value: 85
+   alert_rule_test:
+    # negative match at 15m
+    - eval_time: 15m
+      alertname: NVMeoFHighHostCPU
+    # positive match at 25m      
+    - eval_time: 25m
+      alertname: NVMeoFHighHostCPU
+      exp_alerts:
+      - exp_labels:
+          instance: node-1:10008
+          host: node-1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The CPU is high (85%) on NVMeoF Gateway host (node-1) "
+          description: "High CPU on a gateway host can lead to CPU contention and performance degradation"  
+
+ # NVMeoFInterfaceDown - triggered on eth0 only
+ - interval: 30s
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_listener_iface_info{operstate="down", device="eth0"}'
+      values: '1+0x30'
+    - series: 'ceph_nvmeof_subsystem_listener_iface_info{operstate="up", device="eth1"}'
+      values: '1+0x30'      
+   promql_expr_test:
+     - expr: ceph_nvmeof_subsystem_listener_iface_info{operstate="down"}
+       eval_time: 1m
+       exp_samples:
+         - labels: '{__name__="ceph_nvmeof_subsystem_listener_iface_info", device="eth0", operstate="down"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFInterfaceDown
+      exp_alerts:
+      - exp_labels:
+          oid: 1.3.6.1.4.1.50495.1.2.1.14.1
+          operstate: down
+          device: eth0
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "Network interface eth0 is down "
+          description: "A NIC used by one or more subsystems is in a down state" 
+
+ # NVMeoFInterfaceDuplex - triggered on eth1 only
+ - interval: 30s
+   input_series:
+    - series: 'ceph_nvmeof_subsystem_listener_iface_info{duplex="full", device="eth0"}'
+      values: '1+0x30'   
+    - series: 'ceph_nvmeof_subsystem_listener_iface_info{duplex="half", device="eth1"}'
+      values: '1+0x30'
+   promql_expr_test:
+     - expr: ceph_nvmeof_subsystem_listener_iface_info{duplex!="full"}
+       eval_time: 30s
+       exp_samples:
+         - labels: '{__name__="ceph_nvmeof_subsystem_listener_iface_info", device="eth1", duplex="half"}'
+           value: 1
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFInterfaceDuplex
+      exp_alerts:
+      - exp_labels:
+          duplex: half
+          device: eth1
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "Network interface eth1 is not running in full duplex mode "
+          description: "Until this is resolved, performance from the gateway will be degraded" 
+
+ # NVMeoFHighReadLatency 
+ - interval: 30s
+   input_series:
+    - series: 'ceph_nvmeof_bdev_read_seconds_total{instance="node-1:10008",bdev_name="disk1"}'
+      values: '0+1680x10 19800+3000x20'   
+    - series: 'ceph_nvmeof_bdev_reads_completed_total{instance="node-1:10008",bdev_name="disk1"}'
+      values: '0+286000x10 2980000+120000x20'
+   promql_expr_test:
+     - expr: label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_read_seconds_total[1m]) / rate(ceph_nvmeof_bdev_reads_completed_total[1m])))),"gateway","$1","instance","(.*):.*") > 0.02
+       eval_time: 10m
+       exp_samples:
+         - labels: '{gateway="node-1",instance="node-1:10008"}'
+           value: 0.025
+   alert_rule_test:
+    # negative test - latency is lower than 0.02s
+    - eval_time: 4m
+      alertname: NVMeoFHighReadLatency
+    # positive test - latency is higher than 0.02s
+    - eval_time: 15m
+      alertname: NVMeoFHighReadLatency
+      exp_alerts:
+      - exp_labels:
+          gateway: node-1
+          instance: node-1:10008
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The average read latency over the last 5 mins has reached 10 ms or more on node-1"
+          description: "High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate" 
+
+ # NVMeoFHighWriteLatency 
+ - interval: 30s
+   input_series:
+    - series: 'ceph_nvmeof_bdev_write_seconds_total{instance="node-1:10008",bdev_name="disk1"}'
+      values: '0+1680x10 19800+3000x20'   
+    - series: 'ceph_nvmeof_bdev_writes_completed_total{instance="node-1:10008",bdev_name="disk1"}'
+      values: '0+286000x10 2980000+120000x20'
+   promql_expr_test:
+     - expr: label_replace((avg by(instance) ((rate(ceph_nvmeof_bdev_write_seconds_total[1m]) / rate(ceph_nvmeof_bdev_writes_completed_total[1m])))),"gateway","$1","instance","(.*):.*") > 0.02
+       eval_time: 10m
+       exp_samples:
+         - labels: '{gateway="node-1",instance="node-1:10008"}'
+           value: 0.025
+   alert_rule_test:
+    # negative test - latency is lower than 0.02s
+    - eval_time: 4m
+      alertname: NVMeoFHighWriteLatency
+    # positive test - latency is higher than 0.02s
+    - eval_time: 15m
+      alertname: NVMeoFHighWriteLatency
+      exp_alerts:
+      - exp_labels:
+          gateway: node-1
+          instance: node-1:10008
+          severity: warning
+          type: ceph_default
+        exp_annotations:
+          summary: "The average write latency over the last 5 mins has reached 20 ms or more on node-1"
+          description: "High latencies may indicate a constraint within the cluster e.g. CPU, network. Please investigate"
+ 
\ No newline at end of file

From 19ce7abd11d0fc6cc6a30115d31242f45f15ed05 Mon Sep 17 00:00:00 2001
From: Paul Cuzner <pcuzner@ibm.com>
Date: Fri, 16 Feb 2024 11:56:03 +1300
Subject: [PATCH 2021/2492] ceph-mixins: Update MIB to include nvmeof
 notification

Signed-off-by: Paul Cuzner <pcuzner@ibm.com>
---
 monitoring/snmp/CEPH-MIB.txt | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/monitoring/snmp/CEPH-MIB.txt b/monitoring/snmp/CEPH-MIB.txt
index 5f0e5b2cbcda..aa13d08b18e2 100644
--- a/monitoring/snmp/CEPH-MIB.txt
+++ b/monitoring/snmp/CEPH-MIB.txt
@@ -74,6 +74,7 @@ promPool          OBJECT IDENTIFIER ::= { prometheus 9 }
 promRados         OBJECT IDENTIFIER ::= { prometheus 10 }
 promCephadm       OBJECT IDENTIFIER ::= { prometheus 11 }
 promPrometheus    OBJECT IDENTIFIER ::= { prometheus 12 }
+promNVMeGateway   OBJECT IDENTIFIER ::= { prometheus 14 }
 
 promGenericNotification NOTIFICATION-TYPE
     STATUS      current
@@ -279,6 +280,11 @@ promPrometheusJobMissing NOTIFICATION-TYPE
     STATUS      current
     DESCRIPTION "The prometheus scrape job is not defined."
 ::= { promPrometheus 1 }
+
+promNVMeGatewayNicDown NOTIFICATION-TYPE
+    STATUS      current
+    DESCRIPTION "A NIC used for NVMe gateway client traffic is down."
+::= { promNVMeGateway 1 }
 -- ---------------------------------------------------------- --
 -- IEEE 802.1D MIB - Conformance Information
 -- ---------------------------------------------------------- --
@@ -336,7 +342,8 @@ cephNotificationGroup NOTIFICATION-GROUP
         promRadosRBDMirrorHighBandwidth,
         promCephadmDaemonDown,
         promCephadmUpgradeFailure,
-        promPrometheusJobMissing
+        promPrometheusJobMissing,
+        promNVMeGatewayNicDown
     }
     STATUS current
     DESCRIPTION

From c841876cb1ec8cf576083cf2e6c076924e9daf4d Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 27 Feb 2024 16:06:02 +0800
Subject: [PATCH 2022/2492] crimson/os/seastore: attr related debug outputs

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../seastore/omap_manager/btree/omap_btree_node_impl.cc   | 1 +
 src/crimson/os/seastore/seastore.cc                       | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
index 96a1e3bdefe7..307f82c05e34 100644
--- a/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
+++ b/src/crimson/os/seastore/omap_manager/btree/omap_btree_node_impl.cc
@@ -653,6 +653,7 @@ OMapLeafNode::list(
 
   for (; iter != liter && result.size() < config.max_result_size; iter++) {
     result.emplace(std::make_pair(iter->get_key(), iter->get_val()));
+    DEBUGT("found key {}", oc.t, iter->get_key());
   }
 
   complete = (iter == liter);
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 85a680b2ee2f..bf16ce831836 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -938,17 +938,19 @@ SeaStore::Shard::get_attrs(
       auto& layout = onode.get_layout();
       return omap_list(onode, layout.xattr_root, t, std::nullopt,
         OMapManager::omap_list_config_t().with_inclusive(false, false)
-      ).si_then([&layout](auto p) {
+      ).si_then([&layout, &t, FNAME](auto p) {
         auto& attrs = std::get<1>(p);
         ceph::bufferlist bl;
         if (layout.oi_size) {
           bl.append(ceph::bufferptr(&layout.oi[0], layout.oi_size));
           attrs.emplace(OI_ATTR, std::move(bl));
+         DEBUGT("set oi from onode layout", t);
         }
         if (layout.ss_size) {
           bl.clear();
           bl.append(ceph::bufferptr(&layout.ss[0], layout.ss_size));
           attrs.emplace(SS_ATTR, std::move(bl));
+         DEBUGT("set ss from onode layout", t);
         }
         return seastar::make_ready_future<omap_values_t>(std::move(attrs));
       });
@@ -1841,6 +1843,7 @@ SeaStore::Shard::_setattrs(
       }
       onode->update_object_info(*ctx.transaction, val);
       aset.erase(it);
+      DEBUGT("set oi in onode layout", *ctx.transaction);
     } else {
       onode->clear_object_info(*ctx.transaction);
     }
@@ -1855,15 +1858,18 @@ SeaStore::Shard::_setattrs(
       }
       onode->update_snapset(*ctx.transaction, val);
       aset.erase(it);
+      DEBUGT("set ss in onode layout", *ctx.transaction);
     } else {
       onode->clear_snapset(*ctx.transaction);
     }
   }
 
   if (aset.empty()) {
+    DEBUGT("all attrs set in onode layout", *ctx.transaction);
     return fut;
   }
 
+  DEBUGT("set attrs in omap", *ctx.transaction);
   return fut.si_then(
     [this, onode, &ctx, aset=std::move(aset)]() mutable {
     return _omap_set_kvs(

From abdae14dd017dc2fce6e6c2ed014d7c024864dfc Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 27 Feb 2024 16:18:57 +0800
Subject: [PATCH 2023/2492] crimson/os/seastore: get_attrs should return all
 attrs

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index bf16ce831836..8363b1480333 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -937,7 +937,9 @@ SeaStore::Shard::get_attrs(
     [=, this](auto &t, auto& onode) {
       auto& layout = onode.get_layout();
       return omap_list(onode, layout.xattr_root, t, std::nullopt,
-        OMapManager::omap_list_config_t().with_inclusive(false, false)
+        OMapManager::omap_list_config_t()
+	  .with_inclusive(false, false)
+	  .without_max()
       ).si_then([&layout, &t, FNAME](auto p) {
         auto& attrs = std::get<1>(p);
         ceph::bufferlist bl;

From 6e5efb626f4b3503a12371a8f609fdb5175a3ebe Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 26 Feb 2024 15:02:19 +0530
Subject: [PATCH 2024/2492] mgr/dashboard: replace piechart plugin charts with
 native pie chart panel

Fixes: https://tracker.ceph.com/issues/64579

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../ceph-mixin/dashboards/osd.libsonnet       |  33 ++---
 .../dashboards/piechart_panel.libsonnet       |  70 +++++++++
 .../ceph-mixin/dashboards/utils.libsonnet     |  30 ++++
 .../dashboards_out/osds-overview.json         | 137 ++++++++++++++----
 4 files changed, 220 insertions(+), 50 deletions(-)
 create mode 100644 monitoring/ceph-mixin/dashboards/piechart_panel.libsonnet

diff --git a/monitoring/ceph-mixin/dashboards/osd.libsonnet b/monitoring/ceph-mixin/dashboards/osd.libsonnet
index 259bb00d76f5..9ac55613084d 100644
--- a/monitoring/ceph-mixin/dashboards/osd.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/osd.libsonnet
@@ -1,5 +1,6 @@
 local g = import 'grafonnet/grafana.libsonnet';
 
+
 (import 'utils.libsonnet') {
   'osds-overview.json':
     $.dashboardSchema(
@@ -195,28 +196,18 @@ local g = import 'grafonnet/grafana.libsonnet';
           true
         )
       ) + { gridPos: { x: 20, y: 0, w: 4, h: 8 } },
-      $.simplePieChart(
-        {}, '', 'OSD Types Summary'
-      )
+      $.pieChartPanel('OSD Types Summary', '', '$datasource', { x: 0, y: 8, w: 4, h: 8 }, 'table', 'bottom', true, ['percent'], { mode: 'single', sort: 'none' }, 'pie', ['percent', 'value'], 'palette-classic')
       .addTarget(
         $.addTargetSchema('count by (device_class) (ceph_osd_metadata{%(matchers)s})' % $.matchers(), '{{device_class}}')
-      ) + { gridPos: { x: 0, y: 8, w: 4, h: 8 } },
-      $.simplePieChart(
-        { 'Non-Encrypted': '#E5AC0E' }, '', 'OSD Objectstore Types'
-      )
-      .addTarget(
-        $.addTargetSchema(
-          'count(ceph_bluefs_wal_total_bytes{%(matchers)s})' % $.matchers(), 'bluestore', 'time_series', 2
-        )
-      )
-      .addTarget(
-        $.addTargetSchema(
-          'absent(ceph_bluefs_wal_total_bytes{%(matchers)s}) * count(ceph_osd_metadata{%(matchers)s})' % $.matchers(), 'filestore', 'time_series', 2
-        )
-      ) + { gridPos: { x: 4, y: 8, w: 4, h: 8 } },
-      $.simplePieChart(
-        {}, 'The pie chart shows the various OSD sizes used within the cluster', 'OSD Size Summary'
-      )
+      ),
+      $.pieChartPanel('OSD Objectstore Types', '', '$datasource', { x: 4, y: 8, w: 4, h: 8 }, 'table', 'bottom', true, ['percent'], { mode: 'single', sort: 'none' }, 'pie', ['percent', 'value'], 'palette-classic')
+      .addTarget($.addTargetSchema(
+        'count(ceph_bluefs_wal_total_bytes{%(matchers)s})' % $.matchers(), 'bluestore', 'time_series', 2
+      ))
+      .addTarget($.addTargetSchema(
+        'absent(ceph_bluefs_wal_total_bytes{job=~"$job"}) * count(ceph_osd_metadata{job=~"$job"})' % $.matchers(), 'filestore', 'time_series', 2
+      )),
+      $.pieChartPanel('OSD Size Summary', 'The pie chart shows the various OSD sizes used within the cluster', '$datasource', { x: 8, y: 8, w: 4, h: 8 }, 'table', 'bottom', true, ['percent'], { mode: 'single', sort: 'none' }, 'pie', ['percent', 'value'], 'palette-classic')
       .addTarget($.addTargetSchema(
         'count(ceph_osd_stat_bytes{%(matchers)s} < 1099511627776)' % $.matchers(), '<1TB', 'time_series', 2
       ))
@@ -243,7 +234,7 @@ local g = import 'grafonnet/grafana.libsonnet';
       ))
       .addTarget($.addTargetSchema(
         'count(ceph_osd_stat_bytes{%(matchers)s} >= 13194139533312)' % $.matchers(), '<12TB+', 'time_series', 2
-      )) + { gridPos: { x: 8, y: 8, w: 4, h: 8 } },
+      )),
       g.graphPanel.new(bars=true,
                        datasource='$datasource',
                        title='Distribution of PGs per OSD',
diff --git a/monitoring/ceph-mixin/dashboards/piechart_panel.libsonnet b/monitoring/ceph-mixin/dashboards/piechart_panel.libsonnet
new file mode 100644
index 000000000000..abcf914d7a76
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/piechart_panel.libsonnet
@@ -0,0 +1,70 @@
+{
+  /**
+   * Creates a pie chart panel.
+   *
+   * @name pieChartPanel.new
+   *
+   * @param title The title of the pie chart panel.
+   * @param description (default `''`) Description of the panel
+   * @param datasource (optional) Datasource
+   * @param pieType (default `'pie'`) Type of pie chart (one of pie or donut)
+   *
+   * @method addTarget(target) Adds a target object.
+   */
+  new(
+    title,
+    description='',
+    datasource=null,
+    gridPos={},
+    displayMode='table',
+    placement='bottom',
+    showLegend=true,
+    displayLabels=[],
+    tooltip={},
+    pieType='pie',
+    values=[],
+    colorMode='auto'
+  ):: {
+    type: 'piechart',
+    [if description != null then 'description']: description,
+    title: title,
+    gridPos: gridPos,
+    datasource: datasource,
+    options: {
+      legend: {
+        calcs: [],
+        values: values,
+        displayMode: displayMode,
+        placement: placement,
+        showLegend: showLegend,
+      },
+      pieType: pieType,
+      tooltip: tooltip,
+      displayLabels: displayLabels,
+    },
+    fieldConfig: {
+      defaults: {
+        color: { mode: colorMode },
+        mappings: [],
+        custom: {
+          hideFrom: {
+            legend: false,
+            tooltip: false,
+            viz: false,
+          },
+        },
+      },
+      overrides: [],
+    },
+    targets: [
+    ],
+    _nextTarget:: 0,
+    addTarget(target):: self {
+      // automatically ref id in added targets.
+      local nextTarget = super._nextTarget,
+      _nextTarget: nextTarget + 1,
+      targets+: [target { refId: std.char(std.codepoint('A') + nextTarget) }],
+    },
+    addTargets(targets):: std.foldl(function(p, t) p.addTarget(t), targets, self),
+  },
+}
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index 397c00fe53ad..57d971b0abe7 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -1,4 +1,5 @@
 local g = import 'grafonnet/grafana.libsonnet';
+local pieChartPanel = import 'piechart_panel.libsonnet';
 local timeSeries = import 'timeseries_panel.libsonnet';
 
 {
@@ -646,6 +647,35 @@ local timeSeries = import 'timeseries_panel.libsonnet';
       [if interval != null then 'interval']: interval,
     },
 
+  pieChartPanel(
+    title,
+    description='',
+    datasource=null,
+    gridPos={},
+    displayMode='table',
+    placement='bottom',
+    showLegend=true,
+    displayLabels=[],
+    tooltip={},
+    pieType='pie',
+    values=[],
+    colorMode='auto'
+  )::
+    pieChartPanel.new(
+      title,
+      description=description,
+      datasource=datasource,
+      gridPos=gridPos,
+      displayMode=displayMode,
+      placement=placement,
+      showLegend=showLegend,
+      displayLabels=displayLabels,
+      tooltip=tooltip,
+      pieType=pieType,
+      values=values,
+      colorMode=colorMode
+    ),
+
   heatMapPanel(
     title='',
     datasource=null,
diff --git a/monitoring/ceph-mixin/dashboards_out/osds-overview.json b/monitoring/ceph-mixin/dashboards_out/osds-overview.json
index 79210f3cf2eb..ecd05f827c4a 100644
--- a/monitoring/ceph-mixin/dashboards_out/osds-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/osds-overview.json
@@ -442,9 +442,24 @@
          "type": "table"
       },
       {
-         "aliasColors": { },
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  }
+               },
+               "mappings": [ ]
+            },
+            "overrides": [ ]
+         },
          "gridPos": {
             "h": 8,
             "w": 4,
@@ -452,13 +467,26 @@
             "y": 8
          },
          "id": 6,
-         "legend": {
-            "percentage": true,
-            "show": true,
-            "values": true
+         "options": {
+            "displayLabels": [
+               "percent"
+            ],
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true,
+               "values": [
+                  "percent",
+                  "value"
+               ]
+            },
+            "pieType": "pie",
+            "tooltip": {
+               "mode": "single",
+               "sort": "none"
+            }
          },
-         "legendType": "Under graph",
-         "pieType": "pie",
          "targets": [
             {
                "expr": "count by (device_class) (ceph_osd_metadata{job=~\"$job\"})",
@@ -469,15 +497,27 @@
             }
          ],
          "title": "OSD Types Summary",
-         "type": "grafana-piechart-panel",
-         "valueName": "current"
+         "type": "piechart"
       },
       {
-         "aliasColors": {
-            "Non-Encrypted": "#E5AC0E"
-         },
          "datasource": "$datasource",
          "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  }
+               },
+               "mappings": [ ]
+            },
+            "overrides": [ ]
+         },
          "gridPos": {
             "h": 8,
             "w": 4,
@@ -485,13 +525,26 @@
             "y": 8
          },
          "id": 7,
-         "legend": {
-            "percentage": true,
-            "show": true,
-            "values": true
+         "options": {
+            "displayLabels": [
+               "percent"
+            ],
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true,
+               "values": [
+                  "percent",
+                  "value"
+               ]
+            },
+            "pieType": "pie",
+            "tooltip": {
+               "mode": "single",
+               "sort": "none"
+            }
          },
-         "legendType": "Under graph",
-         "pieType": "pie",
          "targets": [
             {
                "expr": "count(ceph_bluefs_wal_total_bytes{job=~\"$job\"})",
@@ -509,13 +562,27 @@
             }
          ],
          "title": "OSD Objectstore Types",
-         "type": "grafana-piechart-panel",
-         "valueName": "current"
+         "type": "piechart"
       },
       {
-         "aliasColors": { },
          "datasource": "$datasource",
          "description": "The pie chart shows the various OSD sizes used within the cluster",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "palette-classic"
+               },
+               "custom": {
+                  "hideFrom": {
+                     "legend": false,
+                     "tooltip": false,
+                     "viz": false
+                  }
+               },
+               "mappings": [ ]
+            },
+            "overrides": [ ]
+         },
          "gridPos": {
             "h": 8,
             "w": 4,
@@ -523,13 +590,26 @@
             "y": 8
          },
          "id": 8,
-         "legend": {
-            "percentage": true,
-            "show": true,
-            "values": true
+         "options": {
+            "displayLabels": [
+               "percent"
+            ],
+            "legend": {
+               "calcs": [ ],
+               "displayMode": "table",
+               "placement": "bottom",
+               "showLegend": true,
+               "values": [
+                  "percent",
+                  "value"
+               ]
+            },
+            "pieType": "pie",
+            "tooltip": {
+               "mode": "single",
+               "sort": "none"
+            }
          },
-         "legendType": "Under graph",
-         "pieType": "pie",
          "targets": [
             {
                "expr": "count(ceph_osd_stat_bytes{job=~\"$job\"} < 1099511627776)",
@@ -596,8 +676,7 @@
             }
          ],
          "title": "OSD Size Summary",
-         "type": "grafana-piechart-panel",
-         "valueName": "current"
+         "type": "piechart"
       },
       {
          "aliasColors": { },

From 8cc25049a897ebafccd2b9bded6d09300575ef2c Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 27 Feb 2024 16:27:58 +0800
Subject: [PATCH 2025/2492] crimson/os/seastore: also remove xattrs when
 removing onodes

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 43 +++++++++++++++++++++--------
 src/crimson/os/seastore/seastore.h  |  4 +++
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 8363b1480333..a2ccd3bd1770 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1480,28 +1480,49 @@ SeaStore::Shard::_do_transaction_step(
 }
 
 SeaStore::Shard::tm_ret
-SeaStore::Shard::_remove(
+SeaStore::Shard::_remove_omaps(
   internal_context_t &ctx,
-  OnodeRef &onode)
+  OnodeRef &onode,
+  omap_root_t &&omap_root)
 {
-  LOG_PREFIX(SeaStore::_remove);
-  DEBUGT("onode={}", *ctx.transaction, *onode);
-  auto fut = BtreeOMapManager::omap_clear_iertr::now();
-  auto omap_root = onode->get_layout().omap_root.get(
-    onode->get_metadata_hint(device->get_block_size()));
   if (omap_root.get_location() != L_ADDR_NULL) {
-    fut = seastar::do_with(
+    return seastar::do_with(
       BtreeOMapManager(*transaction_manager),
-      onode->get_layout().omap_root.get(
-	onode->get_metadata_hint(device->get_block_size())),
+      std::move(omap_root),
       [&ctx, onode](auto &omap_manager, auto &omap_root) {
       return omap_manager.omap_clear(
 	omap_root,
 	*ctx.transaction
+      ).handle_error_interruptible(
+	crimson::ct_error::input_output_error::pass_further(),
+	crimson::ct_error::assert_all{
+	  "Invalid error in SeaStore::_remove"
+	}
       );
     });
   }
-  return fut.si_then([this, &ctx, onode] {
+  return tm_iertr::now();
+}
+
+SeaStore::Shard::tm_ret
+SeaStore::Shard::_remove(
+  internal_context_t &ctx,
+  OnodeRef &onode)
+{
+  LOG_PREFIX(SeaStore::_remove);
+  DEBUGT("onode={}", *ctx.transaction, *onode);
+  return _remove_omaps(
+    ctx,
+    onode,
+    onode->get_layout().omap_root.get(
+      onode->get_metadata_hint(device->get_block_size()))
+  ).si_then([this, &ctx, onode]() mutable {
+    return _remove_omaps(
+      ctx,
+      onode,
+      onode->get_layout().xattr_root.get(
+	onode->get_metadata_hint(device->get_block_size())));
+  }).si_then([this, &ctx, onode] {
     return seastar::do_with(
       ObjectDataHandler(max_object_size),
       [=, this, &ctx](auto &objhandler) {
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index bff0092f4533..79f0bb82e12c 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -343,6 +343,10 @@ class SeaStore final : public FuturizedStore {
       std::vector<OnodeRef> &d_onodes,
       ceph::os::Transaction::iterator &i);
 
+    tm_ret _remove_omaps(
+      internal_context_t &ctx,
+      OnodeRef &onode,
+      omap_root_t &&omap_root);
     tm_ret _remove(
       internal_context_t &ctx,
       OnodeRef &onode);

From 0b6739a4aa2dd931d97b8af9b3a352969f8767c2 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 26 Feb 2024 20:03:48 +1000
Subject: [PATCH 2026/2492] doc/rados: add "change public network" procedure

Add a procedure to /doc/rados/operations/add-or-rm-mons.rst that
explains how to change the public_network in a Ceph cluster deployed
with cephadm. This procedure was developed by Eugen Block, and can be
seen in its original form here:
https://heiterbiswolkig.blogs.nde.ag/2024/02/22/cephadm-change-public-network/

Co-authored-by: Eugen Block <eblock@nde.ag>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/add-or-rm-mons.rst | 128 ++++++++++++++++++++++++
 1 file changed, 128 insertions(+)

diff --git a/doc/rados/operations/add-or-rm-mons.rst b/doc/rados/operations/add-or-rm-mons.rst
index 4d197ea93bd2..e97c0b94dde2 100644
--- a/doc/rados/operations/add-or-rm-mons.rst
+++ b/doc/rados/operations/add-or-rm-mons.rst
@@ -449,7 +449,135 @@ and inject the modified monitor map into each new monitor.
 Migration to the new location is now complete. The monitors should operate
 successfully.
 
+Using cephadm to change the public network
+==========================================
 
+Overview
+--------
+
+The procedure in this overview section provides only the broad outlines of
+using ``cephadm`` to change the public network.
+
+#. Create backups of all keyrings, configuration files, and the current monmap.
+
+#. Stop the cluster and disable ``ceph.target`` to prevent the daemons from
+   starting.
+
+#. Move the servers and power them on.
+
+#. Change the network setup as desired.
+
+
+Example Procedure 
+-----------------
+
+.. note:: In this procedure, the "old network" has addresses of the form
+   ``10.10.10.0/24`` and the "new network" has addresses of the form
+   ``192.168.160.0/24``.
+
+#. Enter the shell of the first monitor:
+
+   .. prompt:: bash #
+
+      cephadm shell --name mon.reef1
+
+#. Extract the current monmap from ``mon.reef1``:
+
+   .. prompt:: bash #
+      
+      ceph-mon -i reef1 --extract-monmap monmap
+
+#. Print the content of the monmap:
+
+   .. prompt:: bash #
+
+      monmaptool --print monmap
+
+   ::
+
+      monmaptool: monmap file monmap
+      epoch 5
+      fsid 2851404a-d09a-11ee-9aaa-fa163e2de51a
+      last_changed 2024-02-21T09:32:18.292040+0000
+      created 2024-02-21T09:18:27.136371+0000
+      min_mon_release 18 (reef)
+      election_strategy: 1
+      0: [v2:10.10.10.11:3300/0,v1:10.10.10.11:6789/0] mon.reef1
+      1: [v2:10.10.10.12:3300/0,v1:10.10.10.12:6789/0] mon.reef2
+      2: [v2:10.10.10.13:3300/0,v1:10.10.10.13:6789/0] mon.reef3
+
+#. Remove monitors with old addresses:
+
+   .. prompt:: bash #
+
+      monmaptool --rm reef1 --rm reef2 --rm reef3 monmap
+
+#. Add monitors with new addresses:
+
+   .. prompt:: bash #
+
+      monmaptool --addv reef1 [v2:192.168.160.11:3300/0,v1:192.168.160.11:6789/0] --addv reef2 [v2:192.168.160.12:3300/0,v1:192.168.160.12:6789/0] --addv reef3 [v2:192.168.160.13:3300/0,v1:192.168.160.13:6789/0] monmap
+  
+#. Verify that the changes to the monmap have been made successfully:
+
+   .. prompt:: bash #
+
+      monmaptool --print monmap 
+
+   ::
+
+      monmaptool: monmap file monmap
+      epoch 4
+      fsid 2851404a-d09a-11ee-9aaa-fa163e2de51a
+      last_changed 2024-02-21T09:32:18.292040+0000
+      created 2024-02-21T09:18:27.136371+0000
+      min_mon_release 18 (reef)
+      election_strategy: 1
+      0: [v2:192.168.160.11:3300/0,v1:192.168.160.11:6789/0] mon.reef1
+      1: [v2:192.168.160.12:3300/0,v1:192.168.160.12:6789/0] mon.reef2
+      2: [v2:192.168.160.13:3300/0,v1:192.168.160.13:6789/0] mon.reef3
+
+#. Inject the new monmap into the Ceph cluster:
+
+   .. prompt:: bash #
+
+      ceph-mon -i reef1 --inject-monmap monmap
+
+#. Repeat the steps above for all other monitors in the cluster.
+
+#. Update ``/var/lib/ceph/{FSID}/mon.{MON}/config``.
+
+#. Start the monitors.
+
+#. Update the ceph ``public_network``:
+
+   .. prompt:: bash #
+
+      ceph config set mon public_network 192.168.160.0/24
+
+#. Update the configuration files of the managers
+   (``/var/lib/ceph/{FSID}/mgr.{mgr}/config``) and start them. Orchestrator
+   will now be available, but it will attempt to connect to the old network
+   because the host list contains the old addresses.
+
+#. Update the host addresses by running commands of the following form:
+
+   .. prompt:: bash #
+
+      ceph orch host set-addr reef1 192.168.160.11
+      ceph orch host set-addr reef2 192.168.160.12
+      ceph orch host set-addr reef3 192.168.160.13
+
+#. Wait a few minutes for the orchestrator to connect to each host.
+
+#. Reconfigure the OSDs so that their config files are automatically updated:
+   
+   .. prompt:: bash #
+    
+      ceph orch reconfig osd
+
+*The above procedure was developed by Eugen Block and was successfully tested
+in February 2024 on Ceph version 18.2.1 (Reef).*
 
 .. _Manual Deployment: ../../../install/manual-deployment
 .. _Monitor Bootstrap: ../../../dev/mon-bootstrap

From e913e5c6ce51753724e00491fab4de64502952f3 Mon Sep 17 00:00:00 2001
From: Yantao xue <xueyantao2114@163.com>
Date: Tue, 30 Jan 2024 17:14:43 +0800
Subject: [PATCH 2027/2492] mds: check file layout in mknod Fixes:
 https://tracker.ceph.com/issues/64061 Signed-off-by: Xue Yantao
 <xueyantao2114@163.com>

---
 src/mds/Server.cc | 57 ++++++++++++++++++++++-------------------------
 src/mds/Server.h  |  3 +++
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index f3a9287cd053..ed42ac439d82 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -4627,6 +4627,20 @@ class C_MDS_openc_finish : public ServerLogContext {
   }
 };
 
+bool Server::is_valid_layout(file_layout_t *layout)
+{
+  if (!layout->is_valid()) {
+    dout(10) << " invalid initial file layout" << dendl;
+    return false;
+  }
+  if (!mds->mdsmap->is_data_pool(layout->pool_id)) {
+    dout(10) << " invalid data pool " << layout->pool_id << dendl;
+    return false;
+  }
+
+  return true;
+}
+
 /* This function takes responsibility for the passed mdr*/
 void Server::handle_client_openc(const MDRequestRef& mdr)
 {
@@ -4701,13 +4715,7 @@ void Server::handle_client_openc(const MDRequestRef& mdr)
     access |= MAY_SET_VXATTR;
   }
 
-  if (!layout.is_valid()) {
-    dout(10) << " invalid initial file layout" << dendl;
-    respond_to_request(mdr, -CEPHFS_EINVAL);
-    return;
-  }
-  if (!mds->mdsmap->is_data_pool(layout.pool_id)) {
-    dout(10) << " invalid data pool " << layout.pool_id << dendl;
+  if (!is_valid_layout(&layout)) {
     respond_to_request(mdr, -CEPHFS_EINVAL);
     return;
   }
@@ -5593,13 +5601,7 @@ void Server::handle_client_setlayout(const MDRequestRef& mdr)
     access |= MAY_SET_VXATTR;
   }
 
-  if (!layout.is_valid()) {
-    dout(10) << "bad layout" << dendl;
-    respond_to_request(mdr, -CEPHFS_EINVAL);
-    return;
-  }
-  if (!mds->mdsmap->is_data_pool(layout.pool_id)) {
-    dout(10) << " invalid data pool " << layout.pool_id << dendl;
+  if (!is_valid_layout(&layout)) {
     respond_to_request(mdr, -CEPHFS_EINVAL);
     return;
   }
@@ -5725,14 +5727,8 @@ void Server::handle_client_setdirlayout(const MDRequestRef& mdr)
   if (layout != old_layout) {
     access |= MAY_SET_VXATTR;
   }
-
-  if (!layout.is_valid()) {
-    dout(10) << "bad layout" << dendl;
-    respond_to_request(mdr, -CEPHFS_EINVAL);
-    return;
-  }
-  if (!mds->mdsmap->is_data_pool(layout.pool_id)) {
-    dout(10) << " invalid data pool " << layout.pool_id << dendl;
+  
+  if (!is_valid_layout(&layout)) {
     respond_to_request(mdr, -CEPHFS_EINVAL);
     return;
   }
@@ -5909,15 +5905,11 @@ int Server::parse_layout_vxattr(string name, string value, const OSDMap& osdmap,
   if (r < 0) {
     return r;
   }
-
-  if (validate && !layout->is_valid()) {
-    dout(10) << __func__ << ": bad layout" << dendl;
-    return -CEPHFS_EINVAL;
-  }
-  if (!mds->mdsmap->is_data_pool(layout->pool_id)) {
-    dout(10) << __func__ << ": invalid data pool " << layout->pool_id << dendl;
-    return -CEPHFS_EINVAL;
+  
+  if (!is_valid_layout(layout)) {
+     return -CEPHFS_EINVAL;
   }
+  
   return 0;
 }
 
@@ -7002,6 +6994,11 @@ void Server::handle_client_mknod(const MDRequestRef& mdr)
   else
     layout = mdcache->default_file_layout;
 
+  if (!is_valid_layout(&layout)) {
+    respond_to_request(mdr, -CEPHFS_EINVAL);
+    return;
+  }
+
   CInode *newi = prepare_new_inode(mdr, dn->get_dir(), inodeno_t(req->head.ino), mode, &layout);
   ceph_assert(newi);
 
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 89cda03155dc..63d05ea32abb 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -237,6 +237,9 @@ class Server {
   void handle_client_removexattr(const MDRequestRef& mdr);
 
   void handle_client_fsync(const MDRequestRef& mdr);
+  
+  // check layout
+  bool is_valid_layout(file_layout_t *layout);
 
   // open
   void handle_client_open(const MDRequestRef& mdr);

From 4aba45e2374168e4485f31fea4fa1c441c35443e Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Tue, 27 Feb 2024 19:21:57 +0000
Subject: [PATCH 2028/2492] src/scripts/ceph-backport.sh: Add PR checklist

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/script/ceph-backport.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/script/ceph-backport.sh b/src/script/ceph-backport.sh
index 2de5f91f0393..626d9805146e 100755
--- a/src/script/ceph-backport.sh
+++ b/src/script/ceph-backport.sh
@@ -45,6 +45,7 @@ redmine_login=""
 redmine_user_id=""
 setup_ok=""
 this_script=$(basename "$full_path")
+gh_pr_template="../../../ceph/.github/pull_request_template.md"
 
 if [[ $* == *--debug* ]]; then
     set -x
@@ -1699,6 +1700,11 @@ if [ "$PR_PHASE" ] ; then
         [ "$original_issue" ] && desc="${desc}\nparent tracker: $(number_to_url "redmine" "${original_issue}")"
     fi
     desc="${desc}\n\nthis backport was staged using ceph-backport.sh version ${SCRIPT_VERSION}\nfind the latest version at ${github_endpoint}/blob/main/src/script/ceph-backport.sh"
+    desc="$desc\n\n"
+
+    while read line; do
+        desc="$desc$line"
+    done < ${gh_pr_template}
     
     debug "Generating backport PR title"
     if [ "$original_pr" ] ; then

From caa34c1f6102f88e2c07f58af72bd886cbcc33b3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 1 Feb 2024 09:15:37 -0500
Subject: [PATCH 2029/2492] mds: ensure snapclient is synced before corruption
 check

During resolve, orphan dirfrags are purged which may result in dentry commits.
If the snapclient is not sync'd (which normally happens as the last step of
up:resolve), then the snapclient indicates the last sequence is 1 which is
always less than a dentry's legitimate "first" value.

Fixes: https://tracker.ceph.com/issues/64058
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CDentry.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/CDentry.cc b/src/mds/CDentry.cc
index b6d169b9e0f8..6722f0f2a367 100644
--- a/src/mds/CDentry.cc
+++ b/src/mds/CDentry.cc
@@ -702,7 +702,7 @@ bool CDentry::check_corruption(bool load)
 {
   auto&& snapclient = dir->mdcache->mds->snapclient;
   auto next_snap = snapclient->get_last_seq()+1;
-  if (first > last || (snapclient->is_server_ready() && first > next_snap)) {
+  if (first > last || (snapclient->is_synced() && first > next_snap)) {
     if (load) {
       dout(1) << "loaded already corrupt dentry: " << *this << dendl;
       corrupt_first_loaded = true;

From 5f3f31921c9f3ed909a94f287bfd4cc3f0987649 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 27 Feb 2024 12:48:56 +0530
Subject: [PATCH 2030/2492] mgr/dashboard: fix roles page for roles without
 policies

If there is no permission policies attached to a role, it fails the
UI with an exception.

Fixes: https://tracker.ceph.com/issues/64588
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../app/shared/datatable/crud-table/crud-table.component.ts    | 2 +-
 src/pybind/mgr/dashboard/services/rgw_client.py                | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
index 098a454b1d7b..340ddf7005bc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/crud-table/crud-table.component.ts
@@ -147,7 +147,7 @@ export class CRUDTableComponent implements OnInit {
   setExpandedRow(event: any) {
     for (let i = 0; i < this.meta.detail_columns.length; i++) {
       let column = this.meta.detail_columns[i];
-      let columnDetail = event[column];
+      let columnDetail = event?.[column];
       this.expandedRow[column] = this.formatColumnDetails(columnDetail);
     }
   }
diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py
index ea79c86c3fe8..aed70260362e 100644
--- a/src/pybind/mgr/dashboard/services/rgw_client.py
+++ b/src/pybind/mgr/dashboard/services/rgw_client.py
@@ -843,6 +843,9 @@ def list_roles(self) -> List[Dict[str, Any]]:
             logger.warning('Error listing roles with code %d: %s', code, err)
             return []
 
+        for role in roles:
+            if 'PermissionPolicies' not in role:
+                role['PermissionPolicies'] = []
         return roles
 
     def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: str) -> None:

From c884db34e049716577f7868b76685df182f82678 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xxhdx1985126@gmail.com>
Date: Tue, 27 Feb 2024 17:06:07 +0800
Subject: [PATCH 2031/2492] crimson/os/alienstore: use homebrew
 counting_semaphore

std::counting_semaphore has the bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104928,
go back to our homebrew version

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
---
 src/crimson/os/alienstore/thread_pool.h | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/crimson/os/alienstore/thread_pool.h b/src/crimson/os/alienstore/thread_pool.h
index 78e18219a88a..7a21eb34b8a9 100644
--- a/src/crimson/os/alienstore/thread_pool.h
+++ b/src/crimson/os/alienstore/thread_pool.h
@@ -15,14 +15,10 @@
 #include <seastar/core/semaphore.hh>
 #include <seastar/core/sharded.hh>
 
-#if __cplusplus > 201703L
-#include <semaphore>
-namespace crimson {
-  using std::counting_semaphore;
-}
-#else
+// std::counting_semaphore is buggy in libstdc++-11
+// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104928),
+// so we switch back to the homebrew version for now.
 #include "semaphore.h"
-#endif
 
 namespace crimson::os {
 

From 226117a5ac99486fda8208e6a49db097c6667264 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 28 Feb 2024 11:51:22 +0530
Subject: [PATCH 2032/2492] mgr/dashboard: listener addr is always none

Even if the traddr is passed from the API, it gets overriden by the None
hard-coded in the code.

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/services/nvmeof_client.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index 2dc0e473c613..131bfff2aeb9 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -91,7 +91,6 @@ def list_listeners(self, subsystem_nqn: str):
             ))
 
         def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            traddr = None
             if traddr is None:
                 addr = self.gateway_addr
                 ip_address, _ = addr.split(':')
@@ -105,7 +104,6 @@ def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
             return self.stub.create_listener(req)
 
         def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            traddr = None
             if traddr is None:
                 addr = self.gateway_addr
                 ip_address, _ = addr.split(':')

From 0c0d90d3ff7736c1c08b9d65d903768680ade6ec Mon Sep 17 00:00:00 2001
From: Nitzan Mordechai <nmordech@redhat.com>
Date: Tue, 8 Aug 2023 07:44:34 +0000
Subject: [PATCH 2033/2492] erasure-code: add crush rule profile to suites
 Adding new yaml entry to handle create of crush profile before creating new
 pool, will be skipped if no crush profile name was set.

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 qa/tasks/ceph_manager.py | 27 +++++++++++++++++++++++----
 qa/tasks/rados.py        |  9 +++++++++
 qa/tasks/util/rados.py   | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 95e5dac51aba..4690896b7682 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -20,7 +20,7 @@
 from subprocess import DEVNULL
 from teuthology import misc as teuthology
 from tasks.scrub import Scrubber
-from tasks.util.rados import cmd_erasure_code_profile
+from tasks.util.rados import cmd_erasure_code_profile, cmd_ec_crush_profile
 from tasks.util import get_remote
 
 from teuthology.contextutil import safe_while
@@ -2114,8 +2114,18 @@ def create_erasure_code_profile(self, profile_name, profile):
             args = cmd_erasure_code_profile(profile_name, profile)
             self.raw_cluster_cmd(*args)
 
+    def create_erasure_code_crush_rule(self, rule_name, profile):
+        """
+        Create an erasure code crush rule that can be used as a parameter
+        when creating an erasure coded pool.
+        """
+        with self.lock:
+            args = cmd_ec_crush_profile(rule_name, profile)
+            self.raw_cluster_cmd(*args)
+
     def create_pool_with_unique_name(self, pg_num=16,
                                      erasure_code_profile_name=None,
+                                     erasure_code_crush_rule_name=None,
                                      min_size=None,
                                      erasure_code_use_overwrites=False):
         """
@@ -2129,6 +2139,7 @@ def create_pool_with_unique_name(self, pg_num=16,
                 name,
                 pg_num,
                 erasure_code_profile_name=erasure_code_profile_name,
+                erasure_code_crush_rule_name=erasure_code_crush_rule_name,
                 min_size=min_size,
                 erasure_code_use_overwrites=erasure_code_use_overwrites)
         return name
@@ -2141,6 +2152,7 @@ def pool(self, pool_name, pg_num=16, erasure_code_profile_name=None):
 
     def create_pool(self, pool_name, pg_num=16,
                     erasure_code_profile_name=None,
+                    erasure_code_crush_rule_name=None,
                     min_size=None,
                     erasure_code_use_overwrites=False):
         """
@@ -2149,6 +2161,8 @@ def create_pool(self, pool_name, pg_num=16,
         :param pg_num: initial number of pgs.
         :param erasure_code_profile_name: if set and !None create an
                                           erasure coded pool using the profile
+        :param erasure_code_crush_rule_name: if set and !None create an
+                                             erasure coded pool using the crush rule
         :param erasure_code_use_overwrites: if true, allow overwrites
         """
         with self.lock:
@@ -2157,9 +2171,14 @@ def create_pool(self, pool_name, pg_num=16,
             assert pool_name not in self.pools
             self.log("creating pool_name %s" % (pool_name,))
             if erasure_code_profile_name:
-                self.raw_cluster_cmd('osd', 'pool', 'create',
-                                     pool_name, str(pg_num), str(pg_num),
-                                     'erasure', erasure_code_profile_name)
+                cmd_args = ['osd', 'pool', 'create', 
+                            pool_name, str(pg_num), 
+                            str(pg_num), 'erasure', 
+                            erasure_code_profile_name]
+
+                if erasure_code_crush_rule_name:
+                    cmd_args.extend([erasure_code_crush_rule_name])
+                self.raw_cluster_cmd(*cmd_args)
             else:
                 self.raw_cluster_cmd('osd', 'pool', 'create',
                                      pool_name, str(pg_num))
diff --git a/qa/tasks/rados.py b/qa/tasks/rados.py
index 7e786fb68e7e..44dfd8d4215c 100644
--- a/qa/tasks/rados.py
+++ b/qa/tasks/rados.py
@@ -237,8 +237,16 @@ def thread():
             profile = config.get('erasure_code_profile', {})
             profile_name = profile.get('name', 'teuthologyprofile')
             manager.create_erasure_code_profile(profile_name, profile)
+            crush_prof = config.get('erasure_code_crush', {})
+            crush_name = None
+            if crush_prof:
+                crush_name = crush_prof.get('name', 'teuthologycrush')
+                manager.create_erasure_code_crush_rule(crush_name, crush_prof)
+
         else:
             profile_name = None
+            crush_name = None
+
         for i in range(int(config.get('runs', '1'))):
             log.info("starting run %s out of %s", str(i), config.get('runs', '1'))
             tests = {}
@@ -256,6 +264,7 @@ def thread():
                 else:
                     pool = manager.create_pool_with_unique_name(
                         erasure_code_profile_name=profile_name,
+                        erasure_code_crush_rule_name=crush_name,
                         erasure_code_use_overwrites=
                           config.get('erasure_code_use_overwrites', False)
                     )
diff --git a/qa/tasks/util/rados.py b/qa/tasks/util/rados.py
index a0c54ce4eadc..f3936216477f 100644
--- a/qa/tasks/util/rados.py
+++ b/qa/tasks/util/rados.py
@@ -1,6 +1,7 @@
 import logging
 
 from teuthology import misc as teuthology
+from teuthology.orchestra.run import Raw
 
 log = logging.getLogger(__name__)
 
@@ -85,3 +86,35 @@ def cmd_erasure_code_profile(profile_name, profile):
         'osd', 'erasure-code-profile', 'set',
         profile_name
         ] + [ str(key) + '=' + str(value) for key, value in profile.items() ]
+
+def cmd_ec_crush_profile(crush_name, profile):
+    """
+    Return the shell command to run to create the erasure code crush rule
+    described by the profile parameter.
+    
+    :param crush_name: a string matching [A-Za-z0-9-_.]+
+    :param profile: a map whose semantic depends on the crush rule syntax
+    :returns: a shell command as an array suitable for Remote.run
+
+    If profile is {}, return an empty list.
+    """
+
+    if profile == {}:
+        return []
+
+    id_value = profile.get('id', 1)
+    ec_type = profile.get('type', 'erasure')
+    min_size = profile.get('min_size', 1)
+    max_size = profile.get('max_size', 10)
+    steps = profile.get('steps', [])
+
+    crush_content = f"rule {crush_name}-{id_value} {{"
+    crush_content += f"    id {id_value}"
+    crush_content += f"    type {ec_type}"
+    crush_content += f"    min_size {min_size}"
+    crush_content += f"    max_size {max_size}"
+    for step in steps:
+        crush_content += f"    step {step}"
+    crush_content += "}"
+
+    return ['osd', 'crush', 'rule', 'create-erasure', crush_name, '-i', '-', Raw("<<<"), crush_content]

From 5bc1f46527091a9a8b60e7bcc5a94c439ab498cc Mon Sep 17 00:00:00 2001
From: Nitzan Mordechai <nmordech@redhat.com>
Date: Tue, 8 Aug 2023 07:45:22 +0000
Subject: [PATCH 2034/2492] suites: add crush rules to thrash erasure code

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 qa/clusters/fixed-4.yaml                      |  4 ++
 ...c-rados-plugin=jerasure-k=2-m=2-crush.yaml | 52 +++++++++++++++++++
 ...c-rados-plugin=jerasure-k=8-m=6-crush.yaml | 48 +++++++++++++++++
 .../rados/thrash-erasure-code-crush-4-nodes/% |  0
 .../thrash-erasure-code-crush-4-nodes/.qa     |  1 +
 .../arch/.qa                                  |  1 +
 .../arch/x86_64.yaml                          |  1 +
 .../ceph.yaml                                 |  1 +
 .../mon_election                              |  1 +
 .../msgr-failures                             |  1 +
 .../objectstore                               |  1 +
 .../rados.yaml                                |  1 +
 .../recovery-overrides                        |  1 +
 .../supported-random-distro$                  |  1 +
 .../thrashers                                 |  1 +
 .../thrashosds-health.yaml                    |  1 +
 .../workloads/.qa                             |  1 +
 ...c-rados-plugin=jerasure-k=2-m=2-crush.yaml |  1 +
 ...c-rados-plugin=jerasure-k=8-m=6-crush.yaml |  1 +
 ...c-rados-plugin=jerasure-k=8-m=6-crush.yaml |  1 +
 qa/suites/rados/thrash/clusters/fixed-2.yaml  |  1 -
 qa/suites/rados/thrash/clusters/fixed-4.yaml  |  1 +
 qa/suites/rados/thrash/thrashers/careful.yaml |  2 +
 qa/suites/rados/thrash/thrashers/default.yaml |  2 +
 qa/suites/rados/thrash/thrashers/mapgap.yaml  |  1 +
 .../rados/thrash/thrashers/morepggrow.yaml    |  1 +
 qa/suites/rados/thrash/thrashers/pggrow.yaml  |  2 +
 27 files changed, 129 insertions(+), 1 deletion(-)
 create mode 100644 qa/erasure-code/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
 create mode 100644 qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
 create mode 100644 qa/suites/rados/thrash-erasure-code-crush-4-nodes/%
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/.qa
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/.qa
 create mode 100644 qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/x86_64.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/ceph.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/mon_election
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/msgr-failures
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/objectstore
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/rados.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/recovery-overrides
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/supported-random-distro$
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashers
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashosds-health.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/.qa
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
 create mode 120000 qa/suites/rados/thrash-erasure-code/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
 delete mode 120000 qa/suites/rados/thrash/clusters/fixed-2.yaml
 create mode 120000 qa/suites/rados/thrash/clusters/fixed-4.yaml

diff --git a/qa/clusters/fixed-4.yaml b/qa/clusters/fixed-4.yaml
index df767f35710e..b73163751ab6 100644
--- a/qa/clusters/fixed-4.yaml
+++ b/qa/clusters/fixed-4.yaml
@@ -3,6 +3,10 @@ roles:
 - [mon.b, osd.1, osd.5, osd.9, osd.13] 
 - [mon.c, osd.2, osd.6, osd.10, osd.14] 
 - [mgr.x, osd.3, osd.7, osd.11, osd.15, client.0]
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 10 # GB
 overrides:
   ceph:
     conf:
diff --git a/qa/erasure-code/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml b/qa/erasure-code/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
new file mode 100644
index 000000000000..3cfbcd4e85bc
--- /dev/null
+++ b/qa/erasure-code/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
@@ -0,0 +1,52 @@
+roles: 
+- [mon.a, mgr.y, osd.0]
+- [mon.b, osd.1] 
+- [mon.c, osd.2] 
+- [mgr.x, osd.3, client.0]
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
+tasks:
+- rados:
+    clients: [client.0]
+    ops: 400000
+    max_seconds: 600
+    max_in_flight: 64
+    objects: 1024
+    size: 16384
+    ec_pool: true
+    write_append_excl: false
+    erasure_code_crush:
+      name: jerasure22crush
+      id: 22
+      type: erasure
+      min_size: 3
+      max_size: 6
+      steps: [
+        'set_chooseleaf_tries 5',
+        'set_choose_tries 100',
+        'take default class hdd',
+        'step chooseleaf indep 4 type host',
+        'choose indep 1 type osd',
+        'emit'
+      ]
+    erasure_code_profile:
+      name: jerasure22profile
+      plugin: jerasure
+      k: 2
+      m: 2
+      technique: reed_sol_van
+      crush-failure-domain: osd
+    op_weights:
+      read: 100
+      write: 0
+      append: 100
+      delete: 50
+      snap_create: 50
+      snap_remove: 50
+      rollback: 50
+      copy_from: 50
+      setattr: 25
+      rmattr: 25
diff --git a/qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml b/qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
new file mode 100644
index 000000000000..5003c24ff14c
--- /dev/null
+++ b/qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
@@ -0,0 +1,48 @@
+overrides:
+  roles:
+    - [mon.a, mgr.y, osd.0, osd.4, osd.8, osd.12]
+    - [mon.b, osd.1, osd.5, osd.9, osd.13]
+    - [mon.c, osd.2, osd.6, osd.10, osd.14]
+    - [mgr.x, osd.3, osd.7, osd.11, osd.15, client.0]
+tasks:
+- rados:
+    clients: [client.0]
+    ops: 400000
+    max_seconds: 600
+    max_in_flight: 64
+    objects: 1024
+    size: 16384
+    ec_pool: true
+    write_append_excl: false
+    erasure_code_crush:
+      name: jerasure86crush
+      id: 86
+      type: erasure
+      min_size: 3
+      max_size: 6
+      steps: [
+        'set_chooseleaf_tries 5',
+        'set_choose_tries 100',
+        'take default class hdd',
+        'choose indep 4 type host',
+        'choose indep 4 type osd',
+        'emit'
+      ]
+    erasure_code_profile:
+      name: jerasure86profile
+      plugin: jerasure
+      k: 8
+      m: 6
+      technique: reed_sol_van
+      crush-failure-domain: osd
+    op_weights:
+      read: 100
+      write: 0
+      append: 100
+      delete: 50
+      snap_create: 50
+      snap_remove: 50
+      rollback: 50
+      copy_from: 50
+      setattr: 25
+      rmattr: 25
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/% b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/.qa b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/.qa b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/x86_64.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/x86_64.yaml
new file mode 100644
index 000000000000..c2409f5d0dc4
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/arch/x86_64.yaml
@@ -0,0 +1 @@
+arch: x86_64
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/ceph.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/ceph.yaml
new file mode 120000
index 000000000000..a2fd139cbffb
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/ceph.yaml
@@ -0,0 +1 @@
+../thrash/ceph.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/mon_election b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/mon_election
new file mode 120000
index 000000000000..3f331e6214c4
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/mon_election
@@ -0,0 +1 @@
+.qa/mon_election
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/msgr-failures b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/msgr-failures
new file mode 120000
index 000000000000..03689aa44a3e
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/msgr-failures
@@ -0,0 +1 @@
+../thrash/msgr-failures
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/objectstore b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/objectstore
new file mode 120000
index 000000000000..848c65f9e131
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/objectstore
@@ -0,0 +1 @@
+.qa/objectstore_debug
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/rados.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/rados.yaml
new file mode 120000
index 000000000000..d256979c0299
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/rados.yaml
@@ -0,0 +1 @@
+.qa/config/rados.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/recovery-overrides b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/recovery-overrides
new file mode 120000
index 000000000000..1957f2c427e3
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/recovery-overrides
@@ -0,0 +1 @@
+../thrash/2-recovery-overrides
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/supported-random-distro$ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/supported-random-distro$
new file mode 120000
index 000000000000..7cef21eeffd2
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/supported-random-distro$
@@ -0,0 +1 @@
+../basic/supported-random-distro$
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashers b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashers
new file mode 120000
index 000000000000..f461dadc3f21
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashers
@@ -0,0 +1 @@
+../thrash/thrashers
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashosds-health.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashosds-health.yaml
new file mode 120000
index 000000000000..9124eb1aa29a
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/thrashosds-health.yaml
@@ -0,0 +1 @@
+.qa/tasks/thrashosds-health.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/.qa b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
new file mode 120000
index 000000000000..cb75cd191047
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
@@ -0,0 +1 @@
+.qa/erasure-code/ec-rados-plugin=jerasure-k=2-m=2-crush.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
new file mode 120000
index 000000000000..bab649c2405a
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code-crush-4-nodes/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
@@ -0,0 +1 @@
+.qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash-erasure-code/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml b/qa/suites/rados/thrash-erasure-code/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
new file mode 120000
index 000000000000..bab649c2405a
--- /dev/null
+++ b/qa/suites/rados/thrash-erasure-code/workloads/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
@@ -0,0 +1 @@
+.qa/erasure-code/ec-rados-plugin=jerasure-k=8-m=6-crush.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash/clusters/fixed-2.yaml b/qa/suites/rados/thrash/clusters/fixed-2.yaml
deleted file mode 120000
index 230ff0fdab41..000000000000
--- a/qa/suites/rados/thrash/clusters/fixed-2.yaml
+++ /dev/null
@@ -1 +0,0 @@
-.qa/clusters/fixed-2.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash/clusters/fixed-4.yaml b/qa/suites/rados/thrash/clusters/fixed-4.yaml
new file mode 120000
index 000000000000..aa88300715ac
--- /dev/null
+++ b/qa/suites/rados/thrash/clusters/fixed-4.yaml
@@ -0,0 +1 @@
+.qa/clusters/fixed-4.yaml
\ No newline at end of file
diff --git a/qa/suites/rados/thrash/thrashers/careful.yaml b/qa/suites/rados/thrash/thrashers/careful.yaml
index 8190657f3942..7a866fc009d0 100644
--- a/qa/suites/rados/thrash/thrashers/careful.yaml
+++ b/qa/suites/rados/thrash/thrashers/careful.yaml
@@ -20,6 +20,8 @@ overrides:
 tasks:
 - thrashosds:
     timeout: 1200
+    min_in: 2
+    thrash_hosts: true
     chance_pgnum_grow: 1
     chance_pgnum_shrink: 1
     chance_pgpnum_fix: 1
diff --git a/qa/suites/rados/thrash/thrashers/default.yaml b/qa/suites/rados/thrash/thrashers/default.yaml
index 5a300a9ff701..6a0cf59a2afd 100644
--- a/qa/suites/rados/thrash/thrashers/default.yaml
+++ b/qa/suites/rados/thrash/thrashers/default.yaml
@@ -21,6 +21,8 @@ overrides:
 tasks:
 - thrashosds:
     timeout: 1200
+    min_in: 2
+    thrash_hosts: true
     chance_pgnum_grow: 1
     chance_pgnum_shrink: 1
     chance_pgpnum_fix: 1
diff --git a/qa/suites/rados/thrash/thrashers/mapgap.yaml b/qa/suites/rados/thrash/thrashers/mapgap.yaml
index ea7a0c911b5c..bc000db3b9f5 100644
--- a/qa/suites/rados/thrash/thrashers/mapgap.yaml
+++ b/qa/suites/rados/thrash/thrashers/mapgap.yaml
@@ -22,6 +22,7 @@ overrides:
 tasks:
 - thrashosds:
     timeout: 1800
+    min_in: 2
     chance_pgnum_grow: 0.25
     chance_pgnum_shrink: 0.25
     chance_pgpnum_fix: 0.25
diff --git a/qa/suites/rados/thrash/thrashers/morepggrow.yaml b/qa/suites/rados/thrash/thrashers/morepggrow.yaml
index f18a88711a00..d07f051db4fa 100644
--- a/qa/suites/rados/thrash/thrashers/morepggrow.yaml
+++ b/qa/suites/rados/thrash/thrashers/morepggrow.yaml
@@ -15,6 +15,7 @@ overrides:
 tasks:
 - thrashosds:
     timeout: 1200
+    min_in: 2
     chance_pgnum_grow: 3
     chance_pgpnum_fix: 1
 openstack:
diff --git a/qa/suites/rados/thrash/thrashers/pggrow.yaml b/qa/suites/rados/thrash/thrashers/pggrow.yaml
index 54498d0cfef9..d3c5a63d0d22 100644
--- a/qa/suites/rados/thrash/thrashers/pggrow.yaml
+++ b/qa/suites/rados/thrash/thrashers/pggrow.yaml
@@ -20,5 +20,7 @@ overrides:
 tasks:
 - thrashosds:
     timeout: 1200
+    min_in: 2
+    thrash_hosts: true
     chance_pgnum_grow: 2
     chance_pgpnum_fix: 1

From b8ae42d68e0f8d91de0f5b0a0d08711654ae26b3 Mon Sep 17 00:00:00 2001
From: Nitzan Mordechai <nmordech@redhat.com>
Date: Tue, 15 Aug 2023 10:14:19 +0000
Subject: [PATCH 2035/2492] suites: remove extra filestore_debug.. config that
 left

Extra un needed checks and sets for filestore in suites setup need to
be removed.

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 qa/suites/rados/singleton/all/random-eio.yaml | 2 --
 qa/tasks/ceph_manager.py                      | 7 -------
 src/common/options/global.yaml.in             | 6 ------
 3 files changed, 15 deletions(-)

diff --git a/qa/suites/rados/singleton/all/random-eio.yaml b/qa/suites/rados/singleton/all/random-eio.yaml
index 258ae90edbf8..3f238e631e06 100644
--- a/qa/suites/rados/singleton/all/random-eio.yaml
+++ b/qa/suites/rados/singleton/all/random-eio.yaml
@@ -28,7 +28,6 @@ tasks:
 - full_sequential:
   - exec:
       client.0:
-        - sudo ceph tell osd.1 injectargs -- --filestore_debug_random_read_err=0.33
         - sudo ceph tell osd.1 injectargs -- --bluestore_debug_random_read_err=0.33
         - sudo ceph osd pool create test 16 16
         - sudo ceph osd pool set test size 3
@@ -42,5 +41,4 @@ tasks:
       create_pool: false
   - exec:
       client.0:
-        - sudo ceph tell osd.1 injectargs -- --filestore_debug_random_read_err=0.0
         - sudo ceph tell osd.1 injectargs -- --bluestore_debug_random_read_err=0.0
diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index 4690896b7682..e35040b056f2 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -564,8 +564,6 @@ def revive_osd(self, osd=None, skip_admin_check=False):
         self.dead_osds.remove(osd)
         self.live_osds.append(osd)
         if self.random_eio > 0 and osd == self.rerrosd:
-            self.ceph_manager.set_config(self.rerrosd,
-                                         filestore_debug_random_read_err = self.random_eio)
             self.ceph_manager.set_config(self.rerrosd,
                                          bluestore_debug_random_read_err = self.random_eio)
 
@@ -1406,9 +1404,6 @@ def _do_thrash(self):
         delay = self.config.get("op_delay", 5)
         self.rerrosd = self.live_osds[0]
         if self.random_eio > 0:
-            self.ceph_manager.inject_args('osd', self.rerrosd,
-                                          'filestore_debug_random_read_err',
-                                          self.random_eio)
             self.ceph_manager.inject_args('osd', self.rerrosd,
                                           'bluestore_debug_random_read_err',
                                           self.random_eio)
@@ -1442,8 +1437,6 @@ def _do_thrash(self):
             time.sleep(delay)
         self.all_up()
         if self.random_eio > 0:
-            self.ceph_manager.inject_args('osd', self.rerrosd,
-                                          'filestore_debug_random_read_err', '0.0')
             self.ceph_manager.inject_args('osd', self.rerrosd,
                                           'bluestore_debug_random_read_err', '0.0')
         for pool in list(self.pools_to_fix_pgp_num):
diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 882fbd4379a6..9d4e1d251b20 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -5694,12 +5694,6 @@ options:
   level: dev
   default: false
   with_legacy: true
-- name: filestore_debug_random_read_err
-  type: float
-  level: dev
-  default: 0
-  with_legacy: true
-# Expensive debugging check on sync
 - name: filestore_debug_omap_check
   type: bool
   level: dev

From 190a7617e6f34e1fc7cc0f51acc1cc1adfa20f85 Mon Sep 17 00:00:00 2001
From: Nitzan Mordechai <nmordech@redhat.com>
Date: Mon, 11 Sep 2023 04:54:59 +0000
Subject: [PATCH 2036/2492] suites: add host thrasher

To simulate real world maintanace, we will usualy shut down hosts and not just osd
the following commit will add host thrasher option to Thraser, when thrash_hosts
is True we won't thrash osds one by one, we will choose entire host and thrash all
the osds under that host.

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
---
 qa/tasks/ceph_manager.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index e35040b056f2..b81fb9bb213d 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -567,6 +567,25 @@ def revive_osd(self, osd=None, skip_admin_check=False):
             self.ceph_manager.set_config(self.rerrosd,
                                          bluestore_debug_random_read_err = self.random_eio)
 
+    def out_host(self, host=None):
+        """
+        Make all osds on a host out
+        :param host: Host to be marked.
+        """
+        # check that all osd remotes have a valid console
+        osds = self.ceph_manager.ctx.cluster.only(teuthology.is_type('osd', self.ceph_manager.cluster))
+        if host is None:
+            host = random.choice(list(osds.remotes.keys()))
+        self.log("Removing all osds in host %s" % (host,))
+
+        for role in osds.remotes[host]:
+            if not role.startswith("osd."):
+                continue
+            osdid = int(role.split('.')[1])
+            if self.in_osds.count(osdid) == 0:
+                continue
+            self.out_osd(osdid)
+
 
     def out_osd(self, osd=None):
         """
@@ -1227,13 +1246,19 @@ def choose_action(self):
         minout = int(self.config.get("min_out", 0))
         minlive = int(self.config.get("min_live", 2))
         mindead = int(self.config.get("min_dead", 0))
+        thrash_hosts = self.config.get("thrash_hosts", False)
 
         self.log('choose_action: min_in %d min_out '
                  '%d min_live %d min_dead %d '
                  'chance_down %.2f' %
                  (minin, minout, minlive, mindead, chance_down))
         actions = []
-        if len(self.in_osds) > minin:
+        if thrash_hosts:
+            self.log("check thrash_hosts")
+            if len(self.in_osds) > minin:
+                self.log("check thrash_hosts: in_osds > minin")
+                actions.append((self.out_host, 1.0,))
+        elif len(self.in_osds) > minin:
             actions.append((self.out_osd, 1.0,))
         if len(self.live_osds) > minlive and chance_down > 0:
             actions.append((self.kill_osd, chance_down,))

From eff9755f80443c33304dcee79750b8da1df9a420 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 28 Feb 2024 13:42:04 +0800
Subject: [PATCH 2037/2492] crimson/os/seastore: copy attrs and omaps when
 cloning objects

At present, we just copy attrs and omaps one by one, which is not
efficient but very important in terms of functionality especially for
the teuthology tests

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 69 ++++++++++++++++++++++++++++-
 src/crimson/os/seastore/seastore.h  | 10 +++++
 2 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index a2ccd3bd1770..d3a84fc49758 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1584,6 +1584,69 @@ SeaStore::Shard::_write(
     });
 }
 
+SeaStore::Shard::tm_ret
+SeaStore::Shard::_clone_omaps(
+  internal_context_t &ctx,
+  OnodeRef &onode,
+  OnodeRef &d_onode,
+  const omap_type_t otype)
+{
+  return trans_intr::repeat([&ctx, onode, d_onode, this, otype] {
+    return seastar::do_with(
+      std::optional<std::string>(std::nullopt),
+      [&ctx, onode, d_onode, this, otype](auto &start) {
+      auto& layout = onode->get_layout();
+      return omap_list(
+	*onode,
+	otype == omap_type_t::XATTR
+	  ? layout.xattr_root
+	  : layout.omap_root,
+	*ctx.transaction,
+	start,
+	OMapManager::omap_list_config_t().with_inclusive(false, false)
+      ).si_then([&ctx, onode, d_onode, this, otype, &start](auto p) mutable {
+	auto complete = std::get<0>(p);
+	auto &attrs = std::get<1>(p);
+	if (attrs.empty()) {
+	  assert(complete);
+	  return tm_iertr::make_ready_future<
+	    seastar::stop_iteration>(
+	      seastar::stop_iteration::yes);
+	}
+	std::string nstart = attrs.rbegin()->first;
+	return _omap_set_kvs(
+	  d_onode,
+	  otype == omap_type_t::XATTR
+	    ? d_onode->get_layout().xattr_root
+	    : d_onode->get_layout().omap_root,
+	  *ctx.transaction,
+	  std::map<std::string, ceph::bufferlist>(attrs.begin(), attrs.end())
+	).si_then([complete, nstart=std::move(nstart),
+		  &start, &ctx, d_onode, otype](auto root) mutable {
+	  if (root.must_update()) {
+	    if (otype == omap_type_t::XATTR) {
+	      d_onode->update_xattr_root(*ctx.transaction, root);
+	    } else {
+	      assert(otype == omap_type_t::OMAP);
+	      d_onode->update_omap_root(*ctx.transaction, root);
+	    }
+	  }
+	  if (complete) {
+	    return seastar::make_ready_future<
+	      seastar::stop_iteration>(
+		seastar::stop_iteration::yes);
+	  } else {
+	    start = std::move(nstart);
+	    return seastar::make_ready_future<
+	      seastar::stop_iteration>(
+		seastar::stop_iteration::no);
+	  }
+	});
+      });
+    });
+  });
+}
+
 SeaStore::Shard::tm_ret
 SeaStore::Shard::_clone(
   internal_context_t &ctx,
@@ -1595,8 +1658,6 @@ SeaStore::Shard::_clone(
   return seastar::do_with(
     ObjectDataHandler(max_object_size),
     [this, &ctx, &onode, &d_onode](auto &objHandler) {
-    //TODO: currently, we only care about object data, leaving cloning
-    //      of xattr/omap for future work
     auto &object_size = onode->get_layout().size;
     d_onode->update_onode_size(*ctx.transaction, object_size);
     return objHandler.clone(
@@ -1605,6 +1666,10 @@ SeaStore::Shard::_clone(
 	*ctx.transaction,
 	*onode,
 	d_onode.get()});
+  }).si_then([&ctx, &onode, &d_onode, this] {
+    return _clone_omaps(ctx, onode, d_onode, omap_type_t::XATTR);
+  }).si_then([&ctx, &onode, &d_onode, this] {
+    return _clone_omaps(ctx, onode, d_onode, omap_type_t::OMAP);
   });
 }
 
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index 79f0bb82e12c..3f9ec175c905 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -359,6 +359,16 @@ class SeaStore final : public FuturizedStore {
       uint64_t offset, size_t len,
       ceph::bufferlist &&bl,
       uint32_t fadvise_flags);
+    enum class omap_type_t : uint8_t {
+      XATTR = 0,
+      OMAP,
+      NUM_TYPES
+    };
+    tm_ret _clone_omaps(
+      internal_context_t &ctx,
+      OnodeRef &onode,
+      OnodeRef &d_onode,
+      const omap_type_t otype);
     tm_ret _clone(
       internal_context_t &ctx,
       OnodeRef &onode,

From 11fdcf800b14b5edbd2c02f2a6cfac72d174b3e9 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Wed, 28 Feb 2024 09:53:57 +0000
Subject: [PATCH 2038/2492] mgr/dashboard: fix snap schedule date format

Fixes: https://tracker.ceph.com/issues/64613

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-snapshotschedule-form.component.ts          | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index d14d7debcce9..076c99014a6b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -1,7 +1,7 @@
 import { ChangeDetectorRef, Component, OnInit } from '@angular/core';
 import { AbstractControl, FormArray, FormControl, FormGroup, Validators } from '@angular/forms';
 import { NgbActiveModal, NgbDateStruct, NgbTimeStruct } from '@ng-bootstrap/ng-bootstrap';
-import { uniq } from 'lodash';
+import { padStart, uniq } from 'lodash';
 import { Observable, OperatorFunction, of, timer } from 'rxjs';
 import { catchError, debounceTime, distinctUntilChanged, map, switchMap } from 'rxjs/operators';
 import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
@@ -224,9 +224,11 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
 
   parseDatetime(date: NgbDateStruct, time?: NgbTimeStruct): string {
     if (!date || !time) return null;
-    return `${date.year}-${date.month}-${date.day}T${time.hour || '00'}:${time.minute || '00'}:${
-      time.second || '00'
-    }`;
+    return `${date.year}-${padStart(date.month.toString(), 2, '0')}-${padStart(
+      date.day.toString(),
+      2,
+      '0'
+    )}T${time.hour || '00'}:${time.minute || '00'}:${time.second || '00'}`;
   }
   parseSchedule(interval: number, frequency: string): string {
     return `${interval}${frequency}`;

From be3bb09cca7ff8d10a36a59702bb16fd10aba84e Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 26 Feb 2024 11:56:18 +0000
Subject: [PATCH 2039/2492] mgr/dashboard: add snap schedule M, Y frequencies

Added M and Y (month and year) repeat frequencies to snap schedule
create form.

Fixes: https://tracker.ceph.com/issues/64614

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../src/app/shared/enum/repeat-frequency.enum.ts     | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
index 06fb1c3afc0c..48257504defa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/repeat-frequency.enum.ts
@@ -1,17 +1,23 @@
 export enum RepeatFrequency {
   Hourly = 'h',
   Daily = 'd',
-  Weekly = 'w'
+  Weekly = 'w',
+  Monthly = 'M',
+  Yearly = 'Y'
 }
 
 export enum RepeaFrequencySingular {
   h = 'hour',
   d = 'day',
-  w = 'week'
+  w = 'week',
+  M = 'month',
+  Y = 'year'
 }
 
 export enum RepeaFrequencyPlural {
   h = 'hours',
   d = 'days',
-  w = 'weeks'
+  w = 'weeks',
+  M = 'months',
+  Y = 'years'
 }

From 350401ea0ad129c52f1e2b0adb4747d84cb65dcf Mon Sep 17 00:00:00 2001
From: Redouane Kachach <rkachach@redhat.com>
Date: Tue, 27 Feb 2024 15:52:25 +0100
Subject: [PATCH 2040/2492] doc: adding documentation for secure monitoring
 stack configuration Fixes: https://tracker.ceph.com/issues/64596

Signed-off-by: Redouane Kachach <rkachach@redhat.com>
---
 doc/cephadm/services/monitoring.rst | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst
index 33bffdc0157e..985661469ff6 100644
--- a/doc/cephadm/services/monitoring.rst
+++ b/doc/cephadm/services/monitoring.rst
@@ -83,6 +83,33 @@ steps below:
 
      ceph orch apply grafana
 
+Enabling security for the monitoring stack
+----------------------------------------------
+
+By default, in a cephadm managed cluster, the monitoring components are set up and configured without incorporating any security measures.
+While this setup might suffice for certain deployments, other users with stricter security needs may find it necessary to protect their
+monitoring stack against unauthorized access to metrics and data. In such cases, cephadm relies on a specific configuration parameter,
+`mgr/cephadm/secure_monitoring_stack`, which toggles the security settings for all monitoring components. To activate security
+measures, users must set this variable to true, as following:
+
+   .. prompt:: bash #
+
+     ceph config set mgr mgr/cephadm/secure_monitoring_stack true
+
+This configuration change will trigger a sequence of reconfigurations across all monitoring daemons, typically requiring
+few minutes until all components are fully operational. The updated secure configuration includes the following modifications:
+
+#. Prometheus: basic authentication is requiered to access the web portal and TLS is enabled for secure communication.
+#. Alertmanager: basic authentication is requiered to access the web portal and TLS is enabled for secure communication.
+#. Node Exporter: TLS is enabled for secure communication.
+#. Grafana: TLS is enabled and authentication is requiered to access the datasource information.
+
+In this secure setup, users will need to setup authentication (username/password) for both Prometheus and Alertmanager. By default user/password are
+set to admin/admin. The user can change these value through the commands `orch prometheus set-credentials` and `orch alertmanager set-credentials`
+respectively. These commands offer the flexibility to input the username/password either as parameters or via a JSON file, which enhances security. Additionally,
+Cephadm provides commands such as `orch prometheus get-credentials` and `orch alertmanager get-credentials` to retrieve the currently configured credentials such
+as default values.
+
 .. _cephadm-monitoring-centralized-logs:
 
 Centralized Logging in Ceph

From 715ea617ba61aaf51486249d5e57c9076b847c6c Mon Sep 17 00:00:00 2001
From: rkhudov <rhudov@gmail.com>
Date: Wed, 28 Feb 2024 17:27:39 +0000
Subject: [PATCH 2041/2492] src/test/common/test_hobject: remove constexpr

Signed-off-by: rkhudov <rhudov@gmail.com>
---
 src/test/common/test_hobject.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/common/test_hobject.cc b/src/test/common/test_hobject.cc
index 19945754004e..d9600c83e1a7 100644
--- a/src/test/common/test_hobject.cc
+++ b/src/test/common/test_hobject.cc
@@ -98,7 +98,7 @@ struct test_hobject_fmt_t : public hobject_t {
     return snap == 0 && hash == 0 && !max && pool == INT64_MIN;
   }
 
-  constexpr auto operator<=>(const test_hobject_fmt_t& rhs) const noexcept
+  auto operator<=>(const test_hobject_fmt_t& rhs) const noexcept
   {
     auto cmp = is_max() <=> rhs.is_max();
     if (cmp != 0)
@@ -122,7 +122,7 @@ struct test_hobject_fmt_t : public hobject_t {
       return cmp;
     return snap <=> rhs.snap;
   }
-  constexpr bool operator==(const hobject_t& rhs) const noexcept
+  bool operator==(const hobject_t& rhs) const noexcept
   {
     return operator<=>(rhs) == 0;
   }
@@ -201,7 +201,7 @@ namespace fmt {
 template <>
 struct formatter<test_hobject_fmt_t> {
 
-  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+  auto parse(format_parse_context& ctx) { return ctx.begin(); }
 
   template <typename FormatContext>
   auto format(const test_hobject_fmt_t& ho, FormatContext& ctx)

From 416425c5fde39a9b42c5d70bb6bda3bd576456c3 Mon Sep 17 00:00:00 2001
From: Teoman ONAY <tonay@ibm.com>
Date: Tue, 28 Mar 2023 14:18:14 +0200
Subject: [PATCH 2042/2492] mgr/cephadm: add idmap.conf customization for  NFS
 server

Allows idmapd.conf to be customized to control the NFSv4.x server configuration

Fixes: https://tracker.ceph.com/issues/64577

Signed-off-by: Teoman ONAY <tonay@ibm.com>
---
 src/cephadm/cephadmlib/daemons/nfs.py         |  2 +-
 src/cephadm/tests/test_deploy.py              |  1 +
 src/cephadm/tests/test_nfs.py                 |  1 +
 src/pybind/mgr/cephadm/services/nfs.py        | 20 +++++++++++++++++++
 .../templates/services/nfs/ganesha.conf.j2    |  3 +++
 src/pybind/mgr/cephadm/tests/test_services.py |  3 ++-
 .../ceph/deployment/service_spec.py           |  2 ++
 .../ceph/tests/test_service_spec.py           |  6 ++++++
 8 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/nfs.py b/src/cephadm/cephadmlib/daemons/nfs.py
index 6e2f2a945cae..f09374d5f464 100644
--- a/src/cephadm/cephadmlib/daemons/nfs.py
+++ b/src/cephadm/cephadmlib/daemons/nfs.py
@@ -31,7 +31,7 @@ class NFSGanesha(ContainerDaemonForm):
     entrypoint = '/usr/bin/ganesha.nfsd'
     daemon_args = ['-F', '-L', 'STDERR']
 
-    required_files = ['ganesha.conf']
+    required_files = ['ganesha.conf', 'idmap.conf']
 
     port_map = {
         'nfs': 2049,
diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index cfde3fbce0aa..fc2bbe4b3e3d 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -49,6 +49,7 @@ def test_deploy_nfs_container(cephadm_fs, funkypatch):
             'pool': 'foo',
             'files': {
                 'ganesha.conf': 'FAKE',
+                'idmap.conf': 'FAKE',
             },
             'config': 'BALONEY',
             'keyring': 'BUNKUS',
diff --git a/src/cephadm/tests/test_nfs.py b/src/cephadm/tests/test_nfs.py
index aae8113382dc..1b468516e67b 100644
--- a/src/cephadm/tests/test_nfs.py
+++ b/src/cephadm/tests/test_nfs.py
@@ -25,6 +25,7 @@ def nfs_json(**kwargs):
     if kwargs.get("files"):
         result["files"] = {
             "ganesha.conf": "",
+            "idmap.conf": "",
         }
     if kwargs.get("rgw_content"):
         result["rgw"] = dict(kwargs["rgw_content"])
diff --git a/src/pybind/mgr/cephadm/services/nfs.py b/src/pybind/mgr/cephadm/services/nfs.py
index f94a00f5bdf9..e0c61b117e7e 100644
--- a/src/pybind/mgr/cephadm/services/nfs.py
+++ b/src/pybind/mgr/cephadm/services/nfs.py
@@ -5,6 +5,8 @@
 import subprocess
 import tempfile
 from typing import Dict, Tuple, Any, List, cast, Optional
+from configparser import ConfigParser
+from io import StringIO
 
 from mgr_module import HandleCommandResult
 from mgr_module import NFS_POOL_NAME as POOL_NAME
@@ -79,6 +81,8 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
 
         nodeid = f'{daemon_spec.service_name}.{daemon_spec.rank}'
 
+        nfs_idmap_conf = '/etc/ganesha/idmap.conf'
+
         # create the RADOS recovery pool keyring
         rados_user = f'{daemon_type}.{daemon_id}'
         rados_keyring = self.create_keyring(daemon_spec)
@@ -115,12 +119,27 @@ def get_ganesha_conf() -> str:
                 "port": daemon_spec.ports[0] if daemon_spec.ports else 2049,
                 "bind_addr": bind_addr,
                 "haproxy_hosts": [],
+                "nfs_idmap_conf": nfs_idmap_conf,
             }
             if spec.enable_haproxy_protocol:
                 context["haproxy_hosts"] = self._haproxy_hosts()
                 logger.debug("selected haproxy_hosts: %r", context["haproxy_hosts"])
             return self.mgr.template.render('services/nfs/ganesha.conf.j2', context)
 
+        # generate the idmap config
+        def get_idmap_conf() -> str:
+            idmap_conf = spec.idmap_conf
+            output = ''
+            if idmap_conf is not None:
+                cp = ConfigParser()
+                out = StringIO()
+                cp.read_dict(idmap_conf)
+                cp.write(out)
+                out.seek(0)
+                output = out.read()
+                out.close()
+            return output
+
         # generate the cephadm config json
         def get_cephadm_config() -> Dict[str, Any]:
             config: Dict[str, Any] = {}
@@ -130,6 +149,7 @@ def get_cephadm_config() -> Dict[str, Any]:
             config['extra_args'] = ['-N', 'NIV_EVENT']
             config['files'] = {
                 'ganesha.conf': get_ganesha_conf(),
+                'idmap.conf': get_idmap_conf()
             }
             config.update(
                 self.get_config_and_keyring(
diff --git a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2
index ab8df71923b4..7bc0278d7ed1 100644
--- a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2
@@ -16,6 +16,9 @@ NFSv4 {
         Delegations = false;
         RecoveryBackend = 'rados_cluster';
         Minor_Versions = 1, 2;
+{% if nfs_idmap_conf %}
+        IdmapConf = "{{ nfs_idmap_conf }}";
+{% endif %}
 }
 
 RADOS_KV {
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index dbab022058d4..93c426f61a92 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -2448,6 +2448,7 @@ def fake_keys():
             '        Delegations = false;\n'
             "        RecoveryBackend = 'rados_cluster';\n"
             '        Minor_Versions = 1, 2;\n'
+            '        IdmapConf = "/etc/ganesha/idmap.conf";\n'
             '}\n'
             '\n'
             'RADOS_KV {\n'
@@ -2471,7 +2472,7 @@ def fake_keys():
             "%url    rados://.nfs/foo/conf-nfs.foo"
         )
         nfs_expected_conf = {
-            'files': {'ganesha.conf': nfs_ganesha_txt},
+            'files': {'ganesha.conf': nfs_ganesha_txt, 'idmap.conf': ''},
             'config': '',
             'extra_args': ['-N', 'NIV_EVENT'],
             'keyring': (
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index f6d290f07188..23ac1486715f 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1093,6 +1093,7 @@ def __init__(self,
                  enable_haproxy_protocol: bool = False,
                  extra_container_args: Optional[GeneralArgList] = None,
                  extra_entrypoint_args: Optional[GeneralArgList] = None,
+                 idmap_conf: Optional[Dict[str, Dict[str, str]]] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
                  ):
         assert service_type == 'nfs'
@@ -1105,6 +1106,7 @@ def __init__(self,
         self.port = port
         self.virtual_ip = virtual_ip
         self.enable_haproxy_protocol = enable_haproxy_protocol
+        self.idmap_conf = idmap_conf
 
     def get_port_start(self) -> List[int]:
         if self.port:
diff --git a/src/python-common/ceph/tests/test_service_spec.py b/src/python-common/ceph/tests/test_service_spec.py
index 9a55b0a81359..538ee97edfae 100644
--- a/src/python-common/ceph/tests/test_service_spec.py
+++ b/src/python-common/ceph/tests/test_service_spec.py
@@ -396,6 +396,12 @@ def test_osd_unmanaged():
 service_id: mynfs
 service_name: nfs.mynfs
 spec:
+  idmap_conf:
+    general:
+      local-realms: domain.org
+    mapping:
+      nobody-group: nfsnobody
+      nobody-user: nfsnobody
   port: 1234
 ---
 service_type: iscsi

From 83fc4eba3e62e1afc49ead645fb3ed007953fd72 Mon Sep 17 00:00:00 2001
From: Dan Mick <dmick@redhat.com>
Date: Tue, 27 Feb 2024 18:59:46 -0800
Subject: [PATCH 2043/2492] doc/dev/release-process.rst: modify for prerelease
 mechanism

Signed-off-by: Dan Mick <dmick@redhat.com>
---
 doc/dev/release-process.rst | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/doc/dev/release-process.rst b/doc/dev/release-process.rst
index 269983de6eb5..53f6e9470845 100644
--- a/doc/dev/release-process.rst
+++ b/doc/dev/release-process.rst
@@ -33,8 +33,11 @@ Summarized build process
 4. Packages are pushed to chacra.ceph.com.
 5. Packages are pulled from chacra.ceph.com to the Signer VM.
 6. Packages are signed.
-7. Packages are pushed to download.ceph.com.
-8. Release containers are built and pushed to quay.io.
+7. Packages are pushed to a prerelease area on download.ceph.com.
+8. Prerelease containers are built and pushed to quay.ceph.io.
+9. Final test and validation are done on prerelease packages and containers.
+10. Prerelease packages and containers are promoted to official releases on
+    download.ceph.com and quay.io.
 
 Hotfix Release Process Deviation
 --------------------------------
@@ -205,14 +208,28 @@ See `the Ceph Tracker wiki page that explains how to write the release notes <ht
 
       sync-push octopus
 
+This leaves the packages in a password-protected prerelease area
+at https://download.ceph.com/prerelease.  Verify them from there.  
+When done and ready for release, mv the directories to the parent
+directory (that is, "mv <whatever you're promoting> ..".  
+
+
 5. Build Containers
 ===================
 
-Start the following two jobs:
+Prerelease containers (x86_64 only) are built by 
+https://2.jenkins.ceph.com/job/ceph-container-prerelease-build; run it 
+with appropriate parameters.  Test container images will appear on
+quay.ceph.io in the ceph/prerelease repo, built from the prerelease area
+on download.ceph.com.  When satisfied with them, and after you have promoted
+the prerelease packages to released status as above, start the following two jobs:
 
 #. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs/
 #. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs-arm64/
 
+which will rebuild and publish both architectures using the released packages
+on download.ceph.com (into a multiarchitecture container image).
+
 6. Announce the Release
 =======================
 

From 9f94e080cb2a1d54f73ec70880fe2982608a113d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 12:44:12 -0500
Subject: [PATCH 2044/2492] qa/crontab: update smoke tests for releases

Always schedule a series on Sunday at midnight + 1 hour per release (for
ordering).

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index bc2a10fcef57..f638e80e7c2b 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -31,10 +31,11 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 @daily SUITE_NAME=~/src/ceph-qa-suite_main/suites/ceph-ansible; crontab=$(teuthology-describe-tests --show-facet no $SUITE_NAME | perl -p -e 's/</&lt;/g; s/>/&gt;/g; s/&/&amp;/g') ; header=$(echo h4. $SUITE_NAME ; echo " "; echo " ") ; curl --verbose -X PUT --header 'Content-type: application/xml' --data-binary '<?xml version="1.0"?><wiki_page><text>'"$header"'&lt;pre&gt;'"$crontab"'&lt;/pre&gt;</text></wiki_page>' http://tracker.ceph.com/projects/ceph-qa-suite/wiki/ceph-ansible.xml?key=$(cat /etc/redmine-key)
 
 
-## ********** smoke tests on main, quincy, reef branches
-0 5  * * 0,2,4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-0 8  * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-7 8  * * 6 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+## ********** smoke tests on main and release branches
+0 1  * * 0,2,4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+0 2  * * 0     CEPH_BRANCH=squid; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+0 3  * * 0     CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+0 4  * * 0     CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
 
 
 ## ********** windows tests on main branch - weekly

From 8ec9e79661ca8ab69e0061fe727a156dad4d1877 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 12:49:53 -0500
Subject: [PATCH 2045/2492] qa/crontab: add spacing

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index f638e80e7c2b..f8f29fe969cd 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -41,9 +41,11 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## ********** windows tests on main branch - weekly
 # 00 03 * * 1 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s windows -k distro -e $CEPH_QA_EMAIL
 
+
 ## ********** crimson tests on main branch - weekly
 # 01 01 * * 0 CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=crimson-rados; KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
 
+
 ## ********** teuthology/nop on main branch - daily
 0 0 * * * CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 10 -m $MACHINE_NAME -s teuthology/nop -k distro -e $CEPH_QA_EMAIL -p 50
 

From 3dbb32ce1510304d1f5009e2a3a1e21aaba4a084 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 12:51:21 -0500
Subject: [PATCH 2046/2492] qa/crontab: remove EOL branches

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 55 ----------------------------------
 1 file changed, 55 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index f8f29fe969cd..101b4792926e 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -67,61 +67,6 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 ##########################
 
-#********** nautilus branch START - weekly
-
-# 25 13 * * 5  CEPH_BRANCH=nautilus; MACHINE_NAME=smithi; SUITE_NAME=kcephfs;  KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 2999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 15 05 * * 0  CEPH_BRANCH=nautilus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL
-
-
-#********** nautilus branch END
-
-#********** octopus branch START - weekly
-
-# 30 03 * * 3  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; SUITE_NAME=rados;      KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 00 06 * * 4  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; SUITE_NAME=rbd;        KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 10 04 * * 5  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; SUITE_NAME=fs;         KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 15 13 * * 6  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; SUITE_NAME=multimds;   KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 15 12 * * 0  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; SUITE_NAME=powercycle; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 05 05 * * 1  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL
-# 15 05 * * 2  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL
-
-## upgrades suites for on octopus
-# 30 02 * * 4  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -m $MACHINE_NAME -s upgrade/mimic-x -e $CEPH_QA_EMAIL
-# 23 14 * * 5  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 100 -m $MACHINE_NAME -s upgrade/nautilus-x -e $CEPH_QA_EMAIL
-# 25 01 * * 6  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade/octopus-p2p -k distro -e $CEPH_QA_EMAIL
-
-
-## !!!! three suites below MUST use --suite-branch luminous, mimic, nautilus (see https://tracker.ceph.com/issues/24021)
-## The suites below run without filters
-
-# 47 01 * * 5  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade/client-upgrade-luminous-octopus -k distro -e $CEPH_QA_EMAIL --suite-branch luminous -t py2
-# 50 01 * * 5  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade/client-upgrade-mimic-octopus -k distro -e $CEPH_QA_EMAIL --suite-branch mimic -t py2
-# 50 01 * * 5  CEPH_BRANCH=octopus; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-nautilus-octopus -k distro -e $CEPH_QA_EMAIL --suite-branch nautilus
-
-#********** octopus branch END
-
-
-#********** pacific branch START - frequency 4(2) times a week
-
-# 31 03 * * 0   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; SUITE_NAME=rados;      KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 99999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 500 --force-priority
-# 07 06 * * 1   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; SUITE_NAME=rbd;        KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 99999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 500 --force-priority
-# 17 04 * * 2   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; SUITE_NAME=fs;         KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 500 --force-priority
-# 17 12 * * 3   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; SUITE_NAME=powercycle; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh  9999 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 500 --force-priority
-# 07 05 * * 4   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s rgw -k distro -e $CEPH_QA_EMAIL -p 500
-# 17 05 * * 5   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s krbd -k testing -e $CEPH_QA_EMAIL -p 500
-# 23 14 * * 6   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -k distro -n 100 -m $MACHINE_NAME -s upgrade/nautilus-x -e $CEPH_QA_EMAIL -p 500
-# 20 01 * * 6   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-pacific -k distro -e $CEPH_QA_EMAIL --suite-branch octopus -p 500
-
-# 20 07 * * 6   CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-nautilus-pacific -k distro -e $CEPH_QA_EMAIL --suite-branch nautilus -p 500
-
-
-# 22 14 * * 6    CEPH_BRANCH=pacific; MACHINE_NAME=smithi; SUITE_NAME=upgrade:octopus-x; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 10 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 500 --force-priority
-
-# 25 01 * * 7  CEPH_BRANCH=pacific; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH  -n 100 -m $MACHINE_NAME -s upgrade/pacific-p2p -k distro -e $CEPH_QA_EMAIL
-
-
-#********** pacific branch END
-
 
 ### upgrade runs for quincy release
 ###### on smithi

From 3a222ee2bae908da64873a3e6b73a1875923b658 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 13:11:39 -0500
Subject: [PATCH 2047/2492] qa/crontab: update quincy upgrade tests

- use lower priority
- use schedule_subset
- schedule all tests around the same time

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index 101b4792926e..aab47b74595b 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -73,16 +73,11 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 ## !!!! the client suites below MUST use --suite-branch octopus, pacific (see https://tracker.ceph.com/issues/24021)
 
-20 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
-
-25 01 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
-
-22 14 * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:octopus-x --subset 111/120000 -e $CEPH_QA_EMAIL -k distro -p 100 --force-priority
-
-23 14 * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x --subset 111/120000 -e $CEPH_QA_EMAIL -k distro -p 100 --force-priority
-
-35 01 * * 7  CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL -p 75 --force-priority
-
+20 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 175
+25 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 175
+30 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:octopus-x $CEPH_QA_EMAIL distro -p 175
+35 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:pacific-x $CEPH_QA_EMAIL distro -p 175
+40 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL -p 175
 
 ### upgrade runs for reef release
 ###### on smithi

From 176f8c52c9c3f470a2118c58944855dd54d50c56 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 13:14:21 -0500
Subject: [PATCH 2048/2492] qa/crontab: update reef upgrade tests

- use lower priority (higher than quincy)
- schedule around the same time weekly

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index aab47b74595b..11d46755eee3 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -82,16 +82,8 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ### upgrade runs for reef release
 ###### on smithi
 
-
-23 14 * * 6 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x -k distro -e $CEPH_QA_EMAIL -p 100 --force-priority
-
-23 14 * * 6 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL -p 100 --force-priority
-
-20 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-reef -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
-
-25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-reef -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
-
-25 01 * * 4 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-quincy-reef -k distro -e $CEPH_QA_EMAIL --suite-branch quincy --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
+20 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x -k distro -e $CEPH_QA_EMAIL -p 170
+25 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL -p 170
 
 ### upgrade runs for squid release
 ###### on smithi

From 529377b3d7e6409e97e2ecd19cf980988287a94e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 13:45:10 -0500
Subject: [PATCH 2049/2492] qa/crontab: update upgrade runs for squid/main

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index 11d46755eee3..ec70baa6c356 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -88,5 +88,9 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ### upgrade runs for squid release
 ###### on smithi
 
-25 02 * * 4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:reef-x -k distro -e $CEPH_QA_EMAIL --suite-branch main --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 50 --force-priority
+00 00 * * 5   CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=upgrade;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 165
 
+### upgrade runs for main
+###### on smithi
+
+00 00 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=upgrade;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 160

From e16ab1df72c3640db907e377da7ca97ce3eaa26e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 29 Feb 2024 17:40:57 +1000
Subject: [PATCH 2050/2492] doc/dev: edit internals.rst

Edit doc/dev/internals.rst. Make a few sentences pithier.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/dev/internals.rst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/dev/internals.rst b/doc/dev/internals.rst
index a894394c9704..e72d2738b6c0 100644
--- a/doc/dev/internals.rst
+++ b/doc/dev/internals.rst
@@ -2,10 +2,14 @@
  Ceph Internals
 ================
 
-.. note:: If you're looking for how to use Ceph as a library from your
-   own software, please see :doc:`/api/index`.
+.. note:: For information on how to use Ceph as a library (from your own
+   software), see :doc:`/api/index`.
 
-You can start a development mode Ceph cluster, after compiling the source, with::
+Starting a Development-mode Ceph Cluster
+----------------------------------------
+
+Compile the source and then run the following commands to start a
+development-mode Ceph cluster::
 
 	cd build
 	OSD=3 MON=3 MGR=3 ../src/vstart.sh -n -x

From f19ad233025957964559410e6697aa8a2a024bc1 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 29 Feb 2024 14:46:31 +0000
Subject: [PATCH 2051/2492] mon/OSDMonitor: fix rmsnap command

```
There are 2 ways to remove pool snaps, rados tool or mon command (ceph osd pool rmsnap).
It seems that the monitor command is not reporting the actual removal via new_removed_snaps which is later proceed in OSDMap::apply_incremental.
This will result in a clone object leakage since the snap id won't be marked as purged (and won't be trimmed).
```

Fixes: https://tracker.ceph.com/issues/64646

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 6d6771c366cb..3f1254cd7188 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -13087,6 +13087,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
     if (sn) {
       pp->remove_snap(sn);
       pp->set_snap_epoch(pending_inc.epoch);
+      pending_inc.new_removed_snaps[pool].insert(sn);
       ss << "removed pool " << poolstr << " snap " << snapname;
     } else {
       ss << "already removed pool " << poolstr << " snap " << snapname;

From 52a924e7fd4b84801b8606436f287ff15c46f40a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 19 Feb 2024 19:14:52 -0500
Subject: [PATCH 2052/2492] qa/tasks: change map_vips to raise exceptions
 instead of returning None

None of the callers of map_vips ever checks for a None return. So
instead of handling any error conditions it would always just blow
up with a semi-obscure TypeError. Convert the function to always
raise an exception (one that tries to breifly explain the condition)
when something goes wrong. I also take the opportunity to make
more clearer logging and reduce an indentation level.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/vip.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/qa/tasks/vip.py b/qa/tasks/vip.py
index 52114b104228..8e5f44efd6d5 100644
--- a/qa/tasks/vip.py
+++ b/qa/tasks/vip.py
@@ -5,6 +5,7 @@
 
 from teuthology import misc as teuthology
 from teuthology.config import config as teuth_config
+from teuthology.exceptions import ConfigError
 
 log = logging.getLogger(__name__)
 
@@ -69,23 +70,32 @@ def exec(ctx, config):
 
 
 def map_vips(mip, count):
-    for mapping in teuth_config.get('vip', []):
+    vip_entries = teuth_config.get('vip', [])
+    if not vip_entries:
+        raise ConfigError(
+            'at least one item must be configured for "vip" config key'
+            ' to use the vip task'
+        )
+    for mapping in vip_entries:
         mnet = ipaddress.ip_network(mapping['machine_subnet'])
         vnet = ipaddress.ip_network(mapping['virtual_subnet'])
         if vnet.prefixlen >= mnet.prefixlen:
             log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix")
-            return None
-        if mip in mnet:
-            pos = list(mnet.hosts()).index(mip)
-            log.info(f"{mip} in {mnet}, pos {pos}")
-            r = []
-            for sub in vnet.subnets(new_prefix=mnet.prefixlen):
-                r += [list(sub.hosts())[pos]]
-                count -= 1
-                if count == 0:
-                    break
-            return vnet, r
-    return None
+            raise ConfigError('virtual subnet too small')
+        if mip not in mnet:
+            # not our machine subnet
+            log.info(f"machine ip {mip} not in machine subnet {mnet}")
+            continue
+        pos = list(mnet.hosts()).index(mip)
+        log.info(f"{mip} in {mnet}, pos {pos}")
+        r = []
+        for sub in vnet.subnets(new_prefix=mnet.prefixlen):
+            r += [list(sub.hosts())[pos]]
+            count -= 1
+            if count == 0:
+                break
+        return vnet, r
+    raise ConfigError(f"no matching machine subnet found for {mip}")
 
 
 @contextlib.contextmanager

From addc970c1991d6d1e9f0cbc736c279fcde003775 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 19 Feb 2024 19:18:08 -0500
Subject: [PATCH 2053/2492] qa/tasks: make _map_vips private

Nothing outside of vip.py called map_vips, so let us make sure this
is considered a private function and prefix it with the underscore.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/vip.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/vip.py b/qa/tasks/vip.py
index 8e5f44efd6d5..6554f1287538 100644
--- a/qa/tasks/vip.py
+++ b/qa/tasks/vip.py
@@ -69,7 +69,7 @@ def exec(ctx, config):
                 )
 
 
-def map_vips(mip, count):
+def _map_vips(mip, count):
     vip_entries = teuth_config.get('vip', [])
     if not vip_entries:
         raise ConfigError(
@@ -146,7 +146,7 @@ def task(ctx, config):
         ip = remote.ssh.get_transport().getpeername()[0]
         log.info(f'peername {ip}')
         mip = ipaddress.ip_address(ip)
-        vnet, vips = map_vips(mip, count + 1)
+        vnet, vips = _map_vips(mip, count + 1)
         static = vips.pop(0)
         log.info(f"{remote.hostname} static {static}, vnet {vnet}")
 

From 19e664fcdfeb62134db5d40210c40c63f7406a05 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 20 Feb 2024 10:09:50 -0500
Subject: [PATCH 2054/2492] qa/tasks: fix VIPs log line

While testing my previous patches were correct I noticed that the string
here was logged exactly as written, and was thus pretty useless.  This
was probably meant to be an f-string. So make it one. Also get rid of
the unnecessary map call, the list and IP address type can repr
themselves just fine IMO.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/vip.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/tasks/vip.py b/qa/tasks/vip.py
index 6554f1287538..d907fbbd2348 100644
--- a/qa/tasks/vip.py
+++ b/qa/tasks/vip.py
@@ -153,7 +153,7 @@ def task(ctx, config):
         if not ctx.vip:
             # do this only once (use the first remote we see), since we only need 1
             # set of virtual IPs, regardless of how many remotes we have.
-            log.info("VIPs are {map(str, vips)}")
+            log.info(f"VIPs are {vips!r}")
             ctx.vip = {
                 'vnet': vnet,
                 'vips': vips,

From 7bd85b52fd735d427048c38c322ba899681ab1e4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 22 Feb 2024 13:49:10 -0500
Subject: [PATCH 2055/2492] qa/tasks: add templating functions to cephadm
 module

Add functions to cephadm.py that will be later used to template
strings within the yaml files in the cephadm suites. This will be used
to replace the specific subst_vip call with generic calls that let
tests access "any" variables stored on the test ctx.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 61 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 60 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index f60aabc84359..8ee6d6e57524 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -4,13 +4,16 @@
 import argparse
 import configobj
 import contextlib
+import functools
+import json
 import logging
 import os
-import json
 import re
 import uuid
 import yaml
 
+import jinja2
+
 from copy import deepcopy
 from io import BytesIO, StringIO
 from tarfile import ReadError
@@ -34,6 +37,62 @@
 log = logging.getLogger(__name__)
 
 
+def _convert_strs_in(o, conv):
+    """A function to walk the contents of a dict/list and recurisvely apply
+    a conversion function (`conv`) to the strings within.
+    """
+    if isinstance(o, str):
+        return conv(o)
+    if isinstance(o, dict):
+        for k in o:
+            o[k] = _convert_strs_in(o[k], conv)
+    if isinstance(o, list):
+        o[:] = [_convert_strs_in(v, conv) for v in o]
+    return o
+
+
+def _apply_template(jinja_env, rctx, template):
+    """Apply jinja2 templating to the template string `template` via the jinja
+    environment `jinja_env`, passing a dictionary containing top-level context
+    to render into the template.
+    """
+    if '{{' in template or '{%' in template:
+        return jinja_env.from_string(template).render(**rctx)
+    return template
+
+
+def _template_transform(ctx, config, target):
+    """Apply jinja2 based templates to strings within the target object,
+    returning a transformed target. Target objects may be a list or dict or
+    str.
+
+    Note that only string values in the list or dict objects are modified.
+    Therefore one can read & parse yaml or json that contain templates in
+    string values without the risk of changing the structure of the yaml/json.
+    """
+    jenv = getattr(ctx, '_jinja_env', None)
+    if jenv is None:
+        loader = jinja2.BaseLoader()
+        jenv = jinja2.Environment(loader=loader)
+        setattr(ctx, '_jinja_env', jenv)
+    rctx = dict(ctx=ctx, config=config, cluster_name=config.get('cluster', ''))
+    _vip_vars(rctx)
+    conv = functools.partial(_apply_template, jenv, rctx)
+    return _convert_strs_in(target, conv)
+
+
+def _vip_vars(rctx):
+    """For backwards compat with the previous subst_vip function."""
+    ctx = rctx['ctx']
+    if 'vnet' in getattr(ctx, 'vip', {}):
+        rctx['VIPPREFIXLEN'] = str(ctx.vip["vnet"].prefixlen)
+        rctx['VIPSUBNET'] = str(ctx.vip["vnet"].network_address)
+    if 'vips' in getattr(ctx, 'vip', {}):
+        vips = ctx.vip['vips']
+        for idx, vip in enumerate(vips):
+            rctx[f'VIP{idx}'] = str(vip)
+
+
 def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs):
     teuthology.get_testdir(ctx)
     return remote.run(

From 4f1f09531ad7dd94c1489eb4890244247f77d4c7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 22 Feb 2024 13:51:16 -0500
Subject: [PATCH 2056/2492] qa/tasks: replace uses of subst_vip with new
 templating function

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 8ee6d6e57524..248ce68e12f1 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -30,7 +30,6 @@
 
 # these items we use from ceph.py should probably eventually move elsewhere
 from tasks.ceph import get_mons, healthy
-from tasks.vip import subst_vip
 
 CEPH_ROLE_TYPES = ['mon', 'mgr', 'osd', 'mds', 'rgw', 'prometheus']
 
@@ -1415,18 +1414,19 @@ def shell(ctx, config):
         roles = teuthology.all_roles(ctx.cluster)
         config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
 
+    config = _template_transform(ctx, config, config)
     for role, cmd in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()
         log.info('Running commands on role %s host %s', role, remote.name)
         if isinstance(cmd, list):
             for c in cmd:
                 _shell(ctx, cluster_name, remote,
-                       ['bash', '-c', subst_vip(ctx, c)],
+                       ['bash', '-c', c],
                        extra_cephadm_args=args)
         else:
             assert isinstance(cmd, str)
             _shell(ctx, cluster_name, remote,
-                   ['bash', '-ex', '-c', subst_vip(ctx, cmd)],
+                   ['bash', '-ex', '-c', cmd],
                    extra_cephadm_args=args)
 
 
@@ -1452,7 +1452,8 @@ def apply(ctx, config):
     cluster_name = config.get('cluster', 'ceph')
 
     specs = config.get('specs', [])
-    y = subst_vip(ctx, yaml.dump_all(specs))
+    specs = _template_transform(ctx, config, specs)
+    y = yaml.dump_all(specs)
 
     log.info(f'Applying spec(s):\n{y}')
     _shell(

From 3a65a0ad14fb3c4d836441a417d594826fb6e711 Mon Sep 17 00:00:00 2001
From: daijufang <daijufang_yewu@cmss.chinamobile.com>
Date: Mon, 4 Dec 2023 07:46:29 +0000
Subject: [PATCH 2057/2492] src/rgw: fix for the multipart interface in the
 WORM function

1. Save the WORM configuration information in the initialization chunk information for use when merging chunks.
2. Support x-amz-bypass-governance-retention when merging chunks.

Fixes: https://tracker.ceph.com/issues/63724

Signed-off-by: daijufang <daijufang_yewu@cmss.chinamobile.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 24 +++++++++++++++++
 src/rgw/driver/rados/rgw_sal_rados.h  |  1 +
 src/rgw/rgw_common.h                  | 22 ++++++++++++++--
 src/rgw/rgw_op.cc                     | 26 +++++++++++++++++--
 src/rgw/rgw_op.h                      |  6 +++++
 src/rgw/rgw_rest.cc                   |  6 +++++
 src/rgw/rgw_rest_s3.cc                | 37 ++++++++++++++++++++++++++-
 src/rgw/rgw_sal.h                     |  4 +++
 8 files changed, 121 insertions(+), 5 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 24aba37d430c..f46b1eaa97da 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2300,6 +2300,15 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
 
     multipart_upload_info upload_info;
     upload_info.dest_placement = dest_placement;
+    
+    if (obj_legal_hold) {
+      upload_info.obj_legal_hold_exist = true;
+      upload_info.obj_legal_hold = (*obj_legal_hold);
+    }
+    if (obj_retention) {
+      upload_info.obj_retention_exist = true;
+      upload_info.obj_retention = (*obj_retention);
+    }
 
     bufferlist bl;
     encode(upload_info, bl);
@@ -2563,6 +2572,20 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
 
   attrs[RGW_ATTR_ETAG] = etag_bl;
 
+  rgw_placement_rule* ru;
+  ru = &placement;
+  ret = RadosMultipartUpload::get_info(dpp, y, &ru, &attrs);
+  if (upload_information.obj_retention_exist) {
+    bufferlist obj_retention_bl;
+    upload_information.obj_retention.encode(obj_retention_bl);
+    attrs[RGW_ATTR_OBJECT_RETENTION] = std::move(obj_retention_bl);
+  }
+  if (upload_information.obj_legal_hold_exist) {
+    bufferlist obj_legal_hold_bl;
+    upload_information.obj_legal_hold.encode(obj_legal_hold_bl);
+    attrs[RGW_ATTR_OBJECT_LEGAL_HOLD] = std::move(obj_legal_hold_bl);
+  }
+
   if (compressed) {
     // write compression attribute to full object
     bufferlist tmp;
@@ -2667,6 +2690,7 @@ int RadosMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield
     return -EIO;
   }
   placement = upload_info.dest_placement;
+  upload_information = upload_info;
   *rule = &placement;
 
   return 0;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index a4f4809a2f05..39a554f15c95 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -602,6 +602,7 @@ class RadosMultipartUpload : public StoreMultipartUpload {
   ceph::real_time mtime;
   rgw_placement_rule placement;
   RGWObjManifest manifest;
+  multipart_upload_info upload_information;
 
 public:
   RadosMultipartUpload(RadosStore* _store, Bucket* _bucket, const std::string& oid,
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index aecb5721e7a1..5bc5c6de2b5b 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1360,16 +1360,34 @@ inline std::ostream& operator<<(std::ostream& out, const rgw_obj &o) {
 struct multipart_upload_info
 {
   rgw_placement_rule dest_placement;
+  //object lock
+  bool obj_retention_exist{false};
+  bool obj_legal_hold_exist{false};
+  RGWObjectRetention obj_retention;
+  RGWObjectLegalHold obj_legal_hold;
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     encode(dest_placement, bl);
+    encode(obj_retention_exist, bl);
+    encode(obj_legal_hold_exist, bl);
+    encode(obj_retention, bl);
+    encode(obj_legal_hold, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     decode(dest_placement, bl);
+    if (struct_v >= 2) {
+      decode(obj_retention_exist, bl);
+      decode(obj_legal_hold_exist, bl);
+      decode(obj_retention, bl);
+      decode(obj_legal_hold, bl);
+    } else {
+      obj_retention_exist = false;
+      obj_legal_hold_exist = false;
+    }
     DECODE_FINISH(bl);
   }
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index b7e50e3421f7..73adf0a9c5e7 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -5267,7 +5267,6 @@ void RGWDeleteObj::execute(optional_yield y)
     {
       RGWObjState* astate = nullptr;
       bool check_obj_lock = s->object->have_instance() && s->bucket->get_info().obj_lock_enabled();
-
       op_ret = s->object->get_obj_state(this, &astate, s->yield, true);
       if (op_ret < 0) {
         if (need_object_expiration() || multipart_delete) {
@@ -5290,7 +5289,6 @@ void RGWDeleteObj::execute(optional_yield y)
 
       // ignore return value from get_obj_attrs in all other cases
       op_ret = 0;
-
       if (check_obj_lock) {
         ceph_assert(astate);
         int object_lock_response = verify_object_lock(this, astate->attrset, bypass_perm, bypass_governance_mode);
@@ -6527,6 +6525,8 @@ void RGWInitMultipart::execute(optional_yield y)
   std::unique_ptr<rgw::sal::MultipartUpload> upload;
   upload = s->bucket->get_multipart_upload(s->object->get_name(),
 				       upload_id);
+  upload->obj_legal_hold = obj_legal_hold;
+  upload->obj_retention = obj_retention;
   op_ret = upload->init(this, s->yield, s->owner, s->dest_placement, attrs);
 
   if (op_ret == 0) {
@@ -6547,6 +6547,28 @@ int RGWCompleteMultipart::verify_permission(optional_yield y)
   rgw_iam_add_crypt_attrs(s->env, s->info.crypt_attribute_map);
 
   if (s->iam_policy || ! s->iam_user_policies.empty() || ! s->session_policies.empty()) {
+    if (s->bucket->get_info().obj_lock_enabled() && bypass_governance_mode) {
+      auto r = eval_identity_or_session_policies(this, s->iam_user_policies, s->env,
+                                  rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name()));
+      if (r == Effect::Deny) {
+        bypass_perm = false;
+      } else if (r == Effect::Pass && s->iam_policy) {
+        ARN arn(s->bucket->get_key(), s->object->get_name());
+        r = s->iam_policy->eval(s->env, *s->auth.identity, rgw::IAM::s3BypassGovernanceRetention, arn);    
+        if (r == Effect::Deny) {
+          bypass_perm = false;
+        }
+      } else if (r == Effect::Pass && !s->session_policies.empty()) {
+        r = eval_identity_or_session_policies(this, s->session_policies, s->env,
+                               rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name()));
+        if (r == Effect::Deny) {
+          bypass_perm = false;
+        }
+      } else if (r == Effect::Pass) {
+        bypass_perm = false;
+      }
+      bypass_governance_mode &= bypass_perm;
+    }
     auto identity_policy_res = eval_identity_or_session_policies(this, s->iam_user_policies, s->env,
                                               rgw::IAM::s3PutObject,
                                               s->object->get_obj());
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index fcfb24786e8f..6e962de4f473 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1811,6 +1811,9 @@ class RGWInitMultipart : public RGWOp {
   RGWAccessControlPolicy policy;
   ceph::real_time mtime;
   jspan_ptr multipart_trace;
+  //object lock
+  std::optional<RGWObjectRetention> obj_retention = std::nullopt;
+  std::optional<RGWObjectLegalHold> obj_legal_hold = std::nullopt;
 
 public:
   RGWInitMultipart() {}
@@ -1840,6 +1843,9 @@ class RGWCompleteMultipart : public RGWOp {
   std::unique_ptr<rgw::sal::Notification> res;
   std::unique_ptr<rgw::sal::Object> meta_obj;
   off_t ofs = 0;
+  //object lock
+  bool bypass_perm = true;
+  bool bypass_governance_mode = false;
 
 public:
   RGWCompleteMultipart() {}
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index a7c35f3773cc..d830fd40caad 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1572,6 +1572,12 @@ int RGWCompleteMultipart_ObjStore::get_params(optional_yield y)
   std::tie(op_ret, data) = read_all_input(s, max_size);
   if (op_ret < 0)
     return op_ret;
+  
+  const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
+  if (bypass_gov_header) {
+    std::string bypass_gov_decoded = url_decode(bypass_gov_header);
+    bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true");
+  }
 
   return 0;
 }
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 0da1d9919dc6..bb43029f7766 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3935,7 +3935,42 @@ int RGWInitMultipart_ObjStore_S3::get_params(optional_yield y)
     return ret;
   }
 
-  return create_s3_policy(s, driver, policy, s->owner);
+  ret = create_s3_policy(s, driver, policy, s->owner);
+  if (ret < 0)
+    return ret;
+
+  //handle object lock
+  auto obj_lock_mode_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_MODE");
+  auto obj_lock_date_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE");
+  auto obj_legal_hold_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD");
+  if (obj_lock_mode_str && obj_lock_date_str) {
+    boost::optional<ceph::real_time> date = ceph::from_iso_8601(obj_lock_date_str);
+    if (boost::none == date || ceph::real_clock::to_time_t(*date) <= ceph_clock_now()) {
+      ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl;
+      return -EINVAL;;
+    }
+    if (strcmp(obj_lock_mode_str, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str, "COMPLIANCE") != 0) {
+      ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl;
+      return -EINVAL;
+    }
+    obj_retention = RGWObjectRetention(obj_lock_mode_str, *date);
+  } else if ((obj_lock_mode_str && !obj_lock_date_str) || (!obj_lock_mode_str && obj_lock_date_str)) {
+    ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl;
+    return -EINVAL;
+  }
+  if (obj_legal_hold_str) {
+    if (strcmp(obj_legal_hold_str, "ON") != 0 && strcmp(obj_legal_hold_str, "OFF") != 0) {
+      ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl;
+      return -EINVAL;
+    }
+    obj_legal_hold = RGWObjectLegalHold(obj_legal_hold_str);
+  }
+  if (!s->bucket->get_info().obj_lock_enabled() && (obj_retention || obj_legal_hold)) {
+    ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl;
+    return -ERR_INVALID_REQUEST;
+  }
+
+  return 0;
 }
 
 void RGWInitMultipart_ObjStore_S3::send_response()
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 278ed1965bb4..eb4d0c348fad 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1120,6 +1120,10 @@ class MultipartPart {
  */
 class MultipartUpload {
 public:
+  //object lock
+  std::optional<RGWObjectRetention> obj_retention = std::nullopt;
+  std::optional<RGWObjectLegalHold> obj_legal_hold = std::nullopt;
+
   MultipartUpload() = default;
   virtual ~MultipartUpload() = default;
 

From 97445bb6ea24275de5eb050b55207cad237c81f8 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 13 Dec 2023 13:53:46 -0500
Subject: [PATCH 2058/2492] rgw: don't overwrite target attrs checking mpu info

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index f46b1eaa97da..e04e25842e2b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2574,7 +2574,9 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
 
   rgw_placement_rule* ru;
   ru = &placement;
-  ret = RadosMultipartUpload::get_info(dpp, y, &ru, &attrs);
+  rgw::sal::Attrs mpu_attrs; // don't overwrite the target object attrs we are updating
+  ret = RadosMultipartUpload::get_info(dpp, y, &ru, &mpu_attrs);
+
   if (upload_information.obj_retention_exist) {
     bufferlist obj_retention_bl;
     upload_information.obj_retention.encode(obj_retention_bl);

From af43f6162464a20625319a2783a104f7032ba21e Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 29 Feb 2024 11:55:08 -0500
Subject: [PATCH 2059/2492] qa/suites/rbd: rename nbd folder to device folder

Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 qa/suites/rbd/{nbd => device}/%                                   | 0
 qa/suites/rbd/{nbd => device}/.qa                                 | 0
 qa/suites/rbd/{nbd => device}/base                                | 0
 qa/suites/rbd/{nbd => device}/cluster/+                           | 0
 qa/suites/rbd/{nbd => device}/cluster/.qa                         | 0
 qa/suites/rbd/{nbd => device}/cluster/fixed-3.yaml                | 0
 qa/suites/rbd/{nbd => device}/cluster/openstack.yaml              | 0
 qa/suites/rbd/{nbd => device}/conf                                | 0
 qa/suites/rbd/{nbd => device}/msgr-failures                       | 0
 qa/suites/rbd/{nbd => device}/objectstore                         | 0
 qa/suites/rbd/{nbd => device}/supported-random-distro$            | 0
 qa/suites/rbd/{nbd => device}/thrashers                           | 0
 qa/suites/rbd/{nbd => device}/thrashosds-health.yaml              | 0
 qa/suites/rbd/{nbd => device}/workloads/.qa                       | 0
 qa/suites/rbd/{nbd => device}/workloads/rbd_fsx_nbd.yaml          | 0
 qa/suites/rbd/{nbd => device}/workloads/rbd_nbd.yaml              | 0
 .../rbd/{nbd => device}/workloads/rbd_nbd_diff_continuous.yaml    | 0
 17 files changed, 0 insertions(+), 0 deletions(-)
 rename qa/suites/rbd/{nbd => device}/% (100%)
 rename qa/suites/rbd/{nbd => device}/.qa (100%)
 rename qa/suites/rbd/{nbd => device}/base (100%)
 rename qa/suites/rbd/{nbd => device}/cluster/+ (100%)
 rename qa/suites/rbd/{nbd => device}/cluster/.qa (100%)
 rename qa/suites/rbd/{nbd => device}/cluster/fixed-3.yaml (100%)
 rename qa/suites/rbd/{nbd => device}/cluster/openstack.yaml (100%)
 rename qa/suites/rbd/{nbd => device}/conf (100%)
 rename qa/suites/rbd/{nbd => device}/msgr-failures (100%)
 rename qa/suites/rbd/{nbd => device}/objectstore (100%)
 rename qa/suites/rbd/{nbd => device}/supported-random-distro$ (100%)
 rename qa/suites/rbd/{nbd => device}/thrashers (100%)
 rename qa/suites/rbd/{nbd => device}/thrashosds-health.yaml (100%)
 rename qa/suites/rbd/{nbd => device}/workloads/.qa (100%)
 rename qa/suites/rbd/{nbd => device}/workloads/rbd_fsx_nbd.yaml (100%)
 rename qa/suites/rbd/{nbd => device}/workloads/rbd_nbd.yaml (100%)
 rename qa/suites/rbd/{nbd => device}/workloads/rbd_nbd_diff_continuous.yaml (100%)

diff --git a/qa/suites/rbd/nbd/% b/qa/suites/rbd/device/%
similarity index 100%
rename from qa/suites/rbd/nbd/%
rename to qa/suites/rbd/device/%
diff --git a/qa/suites/rbd/nbd/.qa b/qa/suites/rbd/device/.qa
similarity index 100%
rename from qa/suites/rbd/nbd/.qa
rename to qa/suites/rbd/device/.qa
diff --git a/qa/suites/rbd/nbd/base b/qa/suites/rbd/device/base
similarity index 100%
rename from qa/suites/rbd/nbd/base
rename to qa/suites/rbd/device/base
diff --git a/qa/suites/rbd/nbd/cluster/+ b/qa/suites/rbd/device/cluster/+
similarity index 100%
rename from qa/suites/rbd/nbd/cluster/+
rename to qa/suites/rbd/device/cluster/+
diff --git a/qa/suites/rbd/nbd/cluster/.qa b/qa/suites/rbd/device/cluster/.qa
similarity index 100%
rename from qa/suites/rbd/nbd/cluster/.qa
rename to qa/suites/rbd/device/cluster/.qa
diff --git a/qa/suites/rbd/nbd/cluster/fixed-3.yaml b/qa/suites/rbd/device/cluster/fixed-3.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/cluster/fixed-3.yaml
rename to qa/suites/rbd/device/cluster/fixed-3.yaml
diff --git a/qa/suites/rbd/nbd/cluster/openstack.yaml b/qa/suites/rbd/device/cluster/openstack.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/cluster/openstack.yaml
rename to qa/suites/rbd/device/cluster/openstack.yaml
diff --git a/qa/suites/rbd/nbd/conf b/qa/suites/rbd/device/conf
similarity index 100%
rename from qa/suites/rbd/nbd/conf
rename to qa/suites/rbd/device/conf
diff --git a/qa/suites/rbd/nbd/msgr-failures b/qa/suites/rbd/device/msgr-failures
similarity index 100%
rename from qa/suites/rbd/nbd/msgr-failures
rename to qa/suites/rbd/device/msgr-failures
diff --git a/qa/suites/rbd/nbd/objectstore b/qa/suites/rbd/device/objectstore
similarity index 100%
rename from qa/suites/rbd/nbd/objectstore
rename to qa/suites/rbd/device/objectstore
diff --git a/qa/suites/rbd/nbd/supported-random-distro$ b/qa/suites/rbd/device/supported-random-distro$
similarity index 100%
rename from qa/suites/rbd/nbd/supported-random-distro$
rename to qa/suites/rbd/device/supported-random-distro$
diff --git a/qa/suites/rbd/nbd/thrashers b/qa/suites/rbd/device/thrashers
similarity index 100%
rename from qa/suites/rbd/nbd/thrashers
rename to qa/suites/rbd/device/thrashers
diff --git a/qa/suites/rbd/nbd/thrashosds-health.yaml b/qa/suites/rbd/device/thrashosds-health.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/thrashosds-health.yaml
rename to qa/suites/rbd/device/thrashosds-health.yaml
diff --git a/qa/suites/rbd/nbd/workloads/.qa b/qa/suites/rbd/device/workloads/.qa
similarity index 100%
rename from qa/suites/rbd/nbd/workloads/.qa
rename to qa/suites/rbd/device/workloads/.qa
diff --git a/qa/suites/rbd/nbd/workloads/rbd_fsx_nbd.yaml b/qa/suites/rbd/device/workloads/rbd_fsx_nbd.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/workloads/rbd_fsx_nbd.yaml
rename to qa/suites/rbd/device/workloads/rbd_fsx_nbd.yaml
diff --git a/qa/suites/rbd/nbd/workloads/rbd_nbd.yaml b/qa/suites/rbd/device/workloads/rbd_nbd.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/workloads/rbd_nbd.yaml
rename to qa/suites/rbd/device/workloads/rbd_nbd.yaml
diff --git a/qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml b/qa/suites/rbd/device/workloads/rbd_nbd_diff_continuous.yaml
similarity index 100%
rename from qa/suites/rbd/nbd/workloads/rbd_nbd_diff_continuous.yaml
rename to qa/suites/rbd/device/workloads/rbd_nbd_diff_continuous.yaml

From 81ff12566e94103c2e6412b96e592e6c083ca113 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 28 Feb 2024 13:37:55 -0500
Subject: [PATCH 2060/2492] qa/crontab: bring back nightlies of release
 branches

Including squid.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 47 ++++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index ec70baa6c356..80b5b14fcabf 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -49,17 +49,48 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## ********** teuthology/nop on main branch - daily
 0 0 * * * CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 10 -m $MACHINE_NAME -s teuthology/nop -k distro -e $CEPH_QA_EMAIL -p 50
 
-## quincy branch runs - weekly
+
+## main branch runs - weekly
+## suites rados and rbd use --subset arg and must be call with schedule_subset.sh
+## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
+
+# rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
+00 05 * * 0   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 201
+10 05 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+20 05 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+30 05 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+40 05 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+50 05 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+60 05 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+
+
+## squid branch runs - twice weekly
 ## suites rados and rbd use --subset arg and must be call with schedule_subset.sh
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
-# 01 07 * * 0 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 07 07 * * 0 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 01 02 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 15 03 * * 2 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 15 11 * * 3 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
-# 05 03 * * 4 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s $SUITE_NAME -k $KERNEL -e $CEPH_QA_EMAIL
-# 20 03 * * 5 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s $SUITE_NAME -k $KERNEL -e $CEPH_QA_EMAIL
+# rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
+00 06 * * 0,4 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 101
+10 06 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+20 06 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+30 06 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+40 06 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+50 06 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+60 06 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+
+
+## reef branch runs - weekly
+## suites rados and rbd use --subset arg and must be call with schedule_subset.sh
+## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
+
+# rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
+00 07 * * 0   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 181
+10 07 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+20 07 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+30 07 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+40 07 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+50 07 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+60 07 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+
 
 ###  The suite below must run on bare-metal because it's performance suite and run 3 times to produce more data points
 # 57 03 * * 6 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s perf-basic -k distro -e $CEPH_QA_EMAIL -N 3

From 92b254138dd43755490392ffcf5ebca04872e116 Mon Sep 17 00:00:00 2001
From: Ramana Raja <rraja@redhat.com>
Date: Thu, 29 Feb 2024 12:12:19 -0500
Subject: [PATCH 2061/2492] qa/suites: add diff-continuous and
 compare-mirror-image tests

... to rbd and krbd suites respectively.

This allows the compare-mirror-image tests introduced in ea3a567
to be run against various kernel branches, e.g., testing branch.
And allows diff_continuous test in rbd_suite to run against distro
kernel.

Fixes: https://tracker.ceph.com/issues/64574
Signed-off-by: Ramana Raja <rraja@redhat.com>
---
 qa/suites/krbd/mirror/%                         |  0
 qa/suites/krbd/mirror/.qa                       |  1 +
 qa/suites/krbd/mirror/bluestore-bitmap.yaml     |  1 +
 qa/suites/krbd/mirror/clusters/.qa              |  1 +
 qa/suites/krbd/mirror/clusters/2-node.yaml      | 17 +++++++++++++++++
 qa/suites/krbd/mirror/conf.yaml                 |  6 ++++++
 qa/suites/krbd/mirror/install/.qa               |  1 +
 qa/suites/krbd/mirror/install/ceph.yaml         | 14 ++++++++++++++
 qa/suites/krbd/mirror/ms_mode$/.qa              |  1 +
 .../krbd/mirror/ms_mode$/crc-rxbounce.yaml      |  5 +++++
 qa/suites/krbd/mirror/ms_mode$/crc.yaml         |  5 +++++
 .../krbd/mirror/ms_mode$/legacy-rxbounce.yaml   |  5 +++++
 qa/suites/krbd/mirror/ms_mode$/legacy.yaml      |  5 +++++
 qa/suites/krbd/mirror/ms_mode$/secure.yaml      |  5 +++++
 qa/suites/krbd/mirror/tasks/.qa                 |  1 +
 .../compare-mirror-image-alternate-primary.yaml | 14 ++++++++++++++
 .../mirror/tasks/compare-mirror-images.yaml     | 14 ++++++++++++++
 .../device/workloads/diff-continuous-krbd.yaml  | 12 ++++++++++++
 ...continuous.yaml => diff-continuous-nbd.yaml} |  0
 19 files changed, 108 insertions(+)
 create mode 100644 qa/suites/krbd/mirror/%
 create mode 120000 qa/suites/krbd/mirror/.qa
 create mode 120000 qa/suites/krbd/mirror/bluestore-bitmap.yaml
 create mode 120000 qa/suites/krbd/mirror/clusters/.qa
 create mode 100644 qa/suites/krbd/mirror/clusters/2-node.yaml
 create mode 100644 qa/suites/krbd/mirror/conf.yaml
 create mode 120000 qa/suites/krbd/mirror/install/.qa
 create mode 100644 qa/suites/krbd/mirror/install/ceph.yaml
 create mode 120000 qa/suites/krbd/mirror/ms_mode$/.qa
 create mode 100644 qa/suites/krbd/mirror/ms_mode$/crc-rxbounce.yaml
 create mode 100644 qa/suites/krbd/mirror/ms_mode$/crc.yaml
 create mode 100644 qa/suites/krbd/mirror/ms_mode$/legacy-rxbounce.yaml
 create mode 100644 qa/suites/krbd/mirror/ms_mode$/legacy.yaml
 create mode 100644 qa/suites/krbd/mirror/ms_mode$/secure.yaml
 create mode 120000 qa/suites/krbd/mirror/tasks/.qa
 create mode 100644 qa/suites/krbd/mirror/tasks/compare-mirror-image-alternate-primary.yaml
 create mode 100644 qa/suites/krbd/mirror/tasks/compare-mirror-images.yaml
 create mode 100644 qa/suites/rbd/device/workloads/diff-continuous-krbd.yaml
 rename qa/suites/rbd/device/workloads/{rbd_nbd_diff_continuous.yaml => diff-continuous-nbd.yaml} (100%)

diff --git a/qa/suites/krbd/mirror/% b/qa/suites/krbd/mirror/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/krbd/mirror/.qa b/qa/suites/krbd/mirror/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/krbd/mirror/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/bluestore-bitmap.yaml b/qa/suites/krbd/mirror/bluestore-bitmap.yaml
new file mode 120000
index 000000000000..a59cf5175069
--- /dev/null
+++ b/qa/suites/krbd/mirror/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+.qa/objectstore/bluestore-bitmap.yaml
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/clusters/.qa b/qa/suites/krbd/mirror/clusters/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/krbd/mirror/clusters/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/clusters/2-node.yaml b/qa/suites/krbd/mirror/clusters/2-node.yaml
new file mode 100644
index 000000000000..e5036ea72494
--- /dev/null
+++ b/qa/suites/krbd/mirror/clusters/2-node.yaml
@@ -0,0 +1,17 @@
+meta:
+- desc: 2 ceph clusters with 1 mon, 1 mgr and 3 osd each
+roles:
+- - cluster1.mon.a
+  - cluster1.mgr.x
+  - cluster1.osd.0
+  - cluster1.osd.1
+  - cluster1.osd.2
+- - cluster2.mon.a
+  - cluster2.mgr.x
+  - cluster2.osd.0
+  - cluster2.osd.1
+  - cluster2.osd.2
+  - cluster1.client.mirror
+  - cluster1.client.mirror.0
+  - cluster2.client.mirror
+  - cluster2.client.mirror.0
diff --git a/qa/suites/krbd/mirror/conf.yaml b/qa/suites/krbd/mirror/conf.yaml
new file mode 100644
index 000000000000..eb6d72a801af
--- /dev/null
+++ b/qa/suites/krbd/mirror/conf.yaml
@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon warn on pool no app: false
+        ms die on skipped message: false
diff --git a/qa/suites/krbd/mirror/install/.qa b/qa/suites/krbd/mirror/install/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/krbd/mirror/install/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/install/ceph.yaml b/qa/suites/krbd/mirror/install/ceph.yaml
new file mode 100644
index 000000000000..08bb1faa6b0d
--- /dev/null
+++ b/qa/suites/krbd/mirror/install/ceph.yaml
@@ -0,0 +1,14 @@
+tasks:
+- install:
+    extra_packages:
+    - rbd-mirror
+- ceph:
+    cluster: cluster1
+- ceph:
+    cluster: cluster2
+- rbd-mirror:
+    client: cluster1.client.mirror.0
+    thrash: False
+- rbd-mirror:
+    client: cluster2.client.mirror.0
+    thrash: False
diff --git a/qa/suites/krbd/mirror/ms_mode$/.qa b/qa/suites/krbd/mirror/ms_mode$/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/ms_mode$/crc-rxbounce.yaml b/qa/suites/krbd/mirror/ms_mode$/crc-rxbounce.yaml
new file mode 100644
index 000000000000..4d27d01133cd
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/crc-rxbounce.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd default map options: ms_mode=crc,rxbounce
diff --git a/qa/suites/krbd/mirror/ms_mode$/crc.yaml b/qa/suites/krbd/mirror/ms_mode$/crc.yaml
new file mode 100644
index 000000000000..3b072578f1fd
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/crc.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd default map options: ms_mode=crc
diff --git a/qa/suites/krbd/mirror/ms_mode$/legacy-rxbounce.yaml b/qa/suites/krbd/mirror/ms_mode$/legacy-rxbounce.yaml
new file mode 100644
index 000000000000..244e45cbc764
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/legacy-rxbounce.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd default map options: ms_mode=legacy,rxbounce
diff --git a/qa/suites/krbd/mirror/ms_mode$/legacy.yaml b/qa/suites/krbd/mirror/ms_mode$/legacy.yaml
new file mode 100644
index 000000000000..0048dcb0cec2
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/legacy.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd default map options: ms_mode=legacy
diff --git a/qa/suites/krbd/mirror/ms_mode$/secure.yaml b/qa/suites/krbd/mirror/ms_mode$/secure.yaml
new file mode 100644
index 000000000000..a735db18d2c3
--- /dev/null
+++ b/qa/suites/krbd/mirror/ms_mode$/secure.yaml
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd default map options: ms_mode=secure
diff --git a/qa/suites/krbd/mirror/tasks/.qa b/qa/suites/krbd/mirror/tasks/.qa
new file mode 120000
index 000000000000..a602a0353e75
--- /dev/null
+++ b/qa/suites/krbd/mirror/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/krbd/mirror/tasks/compare-mirror-image-alternate-primary.yaml b/qa/suites/krbd/mirror/tasks/compare-mirror-image-alternate-primary.yaml
new file mode 100644
index 000000000000..42ee5a274b2a
--- /dev/null
+++ b/qa/suites/krbd/mirror/tasks/compare-mirror-image-alternate-primary.yaml
@@ -0,0 +1,14 @@
+overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_image_alternate_primary.sh
+    env:
+      RBD_DEVICE_TYPE: 'krbd'
+      RBD_MIRROR_USE_RBD_MIRROR: '1'
+    timeout: 3h
diff --git a/qa/suites/krbd/mirror/tasks/compare-mirror-images.yaml b/qa/suites/krbd/mirror/tasks/compare-mirror-images.yaml
new file mode 100644
index 000000000000..30d147de912f
--- /dev/null
+++ b/qa/suites/krbd/mirror/tasks/compare-mirror-images.yaml
@@ -0,0 +1,14 @@
+overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      cluster1.client.mirror:
+        - rbd/compare_mirror_images.sh
+    env:
+      RBD_DEVICE_TYPE: 'krbd'
+      RBD_MIRROR_USE_RBD_MIRROR: '1'
+    timeout: 3h
diff --git a/qa/suites/rbd/device/workloads/diff-continuous-krbd.yaml b/qa/suites/rbd/device/workloads/diff-continuous-krbd.yaml
new file mode 100644
index 000000000000..5907718d508f
--- /dev/null
+++ b/qa/suites/rbd/device/workloads/diff-continuous-krbd.yaml
@@ -0,0 +1,12 @@
+overrides:
+  install:
+    ceph:
+      extra_system_packages:
+        - pv
+tasks:
+- workunit:
+    clients:
+      all:
+        - rbd/diff_continuous.sh
+    env:
+      RBD_DEVICE_TYPE: "krbd"
diff --git a/qa/suites/rbd/device/workloads/rbd_nbd_diff_continuous.yaml b/qa/suites/rbd/device/workloads/diff-continuous-nbd.yaml
similarity index 100%
rename from qa/suites/rbd/device/workloads/rbd_nbd_diff_continuous.yaml
rename to qa/suites/rbd/device/workloads/diff-continuous-nbd.yaml

From 2c6983d8b4e841f2bfbc4e4f82bb0e41e10aec7e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 29 Feb 2024 18:08:10 +1000
Subject: [PATCH 2062/2492] doc/glossary: improve "MDS" entry

Improve the entry for "MDS" in doc/glossary.rst by linking to the
"ceph-mds" man page and mentioning the relationship between clients and
MDS (or MDSes).

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst       | 11 +++++++++--
 doc/man/8/ceph-mds.rst |  2 ++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 8f1c5659756d..262b89598c50 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -263,11 +263,18 @@
                 Ceph-specific information about devices and its relationship
                 with OSDs.
 
-	:ref:`MDS<cephfs_add_remote_mds>`
+	MDS
                 The Ceph **M**\eta\ **D**\ata **S**\erver daemon. Also referred
                 to as "ceph-mds". The Ceph metadata server daemon must be
                 running in any Ceph cluster that runs the CephFS file system.
-                The MDS stores all filesystem metadata. 
+                The MDS stores all filesystem metadata. :term:`Client`\s work
+                together with either a single MDS or a group of MDSes to
+                maintain a distributed metadata cache that is required by
+                CephFS.
+
+                See :ref:`Deploying Metadata Servers<cephfs_add_remote_mds>`.
+
+                See the :ref:`ceph-mds man page<ceph_mds_man>`.
 
 	MGR
                 The Ceph manager software, which collects all the state from
diff --git a/doc/man/8/ceph-mds.rst b/doc/man/8/ceph-mds.rst
index fd8fc800eca3..c68d3d110afa 100644
--- a/doc/man/8/ceph-mds.rst
+++ b/doc/man/8/ceph-mds.rst
@@ -1,5 +1,7 @@
 :orphan:
 
+.. _ceph_mds_man:
+
 =========================================
  ceph-mds -- ceph metadata server daemon
 =========================================

From 979c1916507092591469f8ea3b0a0440f2b88066 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 6 Feb 2024 17:19:43 -0800
Subject: [PATCH 2063/2492] crimson/.../errorator: simplify
 maybe_handle_error_t::handle()

Using FuturatorT::invoke covers both of the last two cases and also
correctly deals with handlers that return interrupible futures.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/errorator.h | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h
index ebc5a95bb3e6..692498b75aed 100644
--- a/src/crimson/common/errorator.h
+++ b/src/crimson/common/errorator.h
@@ -354,17 +354,10 @@ class maybe_handle_error_t {
         // to throwing an exception by the handler.
         std::invoke(std::forward<ErrorVisitorT>(errfunc),
                     ErrorT::error_t::from_exception_ptr(std::move(ep)));
-      } else if constexpr (seastar::Future<decltype(result)>) {
-        // result is seastar::future but return_t is e.g. int. If so,
-        // the else clause cannot be used as seastar::future lacks
-        // errorator_type member.
-        result = seastar::make_ready_future<return_t>(
-          std::invoke(std::forward<ErrorVisitorT>(errfunc),
-                      ErrorT::error_t::from_exception_ptr(std::move(ep))));
       } else {
-        result = FuturatorT::type::errorator_type::template make_ready_future<return_t>(
-          std::invoke(std::forward<ErrorVisitorT>(errfunc),
-                      ErrorT::error_t::from_exception_ptr(std::move(ep))));
+        result = FuturatorT::invoke(
+	  std::forward<ErrorVisitorT>(errfunc),
+	  ErrorT::error_t::from_exception_ptr(std::move(ep)));
       }
     }
   }

From e72f1779f4505616253cba7c41c10aa2b772adaa Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 5 Feb 2024 20:45:04 -0800
Subject: [PATCH 2064/2492] crimson/.../ops_executor: no need to warn in normal
 pg op execution path

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/ops_executer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index f03827b03d6b..91db2a58b975 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -1454,7 +1454,7 @@ static PG::interruptible_future<> do_pgls_filtered(
 PgOpsExecuter::interruptible_future<>
 PgOpsExecuter::execute_op(OSDOp& osd_op)
 {
-  logger().warn("handling op {}", ceph_osd_op_name(osd_op.op.op));
+  logger().debug("handling op {}", ceph_osd_op_name(osd_op.op.op));
   switch (const ceph_osd_op& op = osd_op.op; op.op) {
   case CEPH_OSD_OP_PGLS:
     return do_pgls(pg, nspace, osd_op);

From 3f377f013c690d9c17685967f1ebc4dc2adbf838 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 16 Mar 2023 20:22:22 +0000
Subject: [PATCH 2065/2492] test/crimson/test_errorator: move tests into TEST_F
 declarations

Slightly easier to read this way.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/test/crimson/test_errorator.cc | 98 +++++++++++++-----------------
 1 file changed, 43 insertions(+), 55 deletions(-)

diff --git a/src/test/crimson/test_errorator.cc b/src/test/crimson/test_errorator.cc
index 939c6cde81ae..4eccb1a9dcb7 100644
--- a/src/test/crimson/test_errorator.cc
+++ b/src/test/crimson/test_errorator.cc
@@ -13,33 +13,7 @@
 
 struct errorator_test_t : public seastar_test_suite_t {
   using ertr = crimson::errorator<crimson::ct_error::invarg>;
-  ertr::future<> test_do_until() {
-    return crimson::repeat([i=0]() mutable {
-      if (i < 5) {
-        ++i;
-        return ertr::make_ready_future<seastar::stop_iteration>(
-          seastar::stop_iteration::no);
-      } else {
-        return ertr::make_ready_future<seastar::stop_iteration>(
-          seastar::stop_iteration::yes);
-      }
-    });
-  }
-  static constexpr int SIZE = 42;
-  ertr::future<> test_parallel_for_each() {
-    auto sum = std::make_unique<int>(0);
-    return ertr::parallel_for_each(
-      boost::make_counting_iterator(0),
-      boost::make_counting_iterator(SIZE),
-      [sum=sum.get()](int i) {
-	*sum += i;
-    }).safe_then([sum=std::move(sum)] {
-      int expected = std::accumulate(boost::make_counting_iterator(0),
-				     boost::make_counting_iterator(SIZE),
-				     0);
-      ASSERT_EQ(*sum, expected);
-    });
-  }
+
   struct noncopyable_t {
     constexpr noncopyable_t() = default;
     ~noncopyable_t() = default;
@@ -48,52 +22,66 @@ struct errorator_test_t : public seastar_test_suite_t {
     noncopyable_t(const noncopyable_t&) = delete;
     noncopyable_t& operator=(const noncopyable_t&) = delete;
   };
-  ertr::future<> test_non_copy_then() {
-    return create_noncopyable().safe_then([](auto t) {
-      return ertr::now();
-    });
-  }
-  ertr::future<int> test_futurization() {
-    // we don't want to be enforced to always do `make_ready_future(...)`.
-    // as in seastar::future, the futurization should take care about
-    // turning non-future types (e.g. int) into futurized ones (e.g.
-    // ertr::future<int>).
-    return ertr::now().safe_then([] {
-      return 42;
-    }).safe_then([](int life) {
-      return ertr::make_ready_future<int>(life);
-    });
-  }
-private:
-  ertr::future<noncopyable_t> create_noncopyable() {
-    return ertr::make_ready_future<noncopyable_t>();
-  }
 };
 
 TEST_F(errorator_test_t, basic)
 {
-  run_async([this] {
-    test_do_until().unsafe_get0();
+  run_async([] {
+    return crimson::repeat([i=0]() mutable {
+      if (i < 5) {
+        ++i;
+        return ertr::make_ready_future<seastar::stop_iteration>(
+          seastar::stop_iteration::no);
+      } else {
+        return ertr::make_ready_future<seastar::stop_iteration>(
+          seastar::stop_iteration::yes);
+      }
+    }).unsafe_get0();
   });
 }
 
 TEST_F(errorator_test_t, parallel_for_each)
 {
-  run_async([this] {
-    test_parallel_for_each().unsafe_get0();
+  run_async([] {
+    static constexpr int SIZE = 42;
+    auto sum = std::make_unique<int>(0);
+    return ertr::parallel_for_each(
+      boost::make_counting_iterator(0),
+      boost::make_counting_iterator(SIZE),
+      [sum=sum.get()](int i) {
+	*sum += i;
+      }).safe_then([sum=std::move(sum)] {
+	int expected = std::accumulate(boost::make_counting_iterator(0),
+				       boost::make_counting_iterator(SIZE),
+				       0);
+	ASSERT_EQ(*sum, expected);
+      }).unsafe_get0();
   });
 }
 
 TEST_F(errorator_test_t, non_copy_then)
 {
-  run_async([this] {
-    test_non_copy_then().unsafe_get0();
+  run_async([] {
+    auto create_noncopyable = [] {
+      return ertr::make_ready_future<noncopyable_t>();
+    };
+    return create_noncopyable().safe_then([](auto) {
+      return ertr::now();
+    }).unsafe_get0();
   });
 }
 
 TEST_F(errorator_test_t, test_futurization)
 {
-  run_async([this] {
-    test_futurization().unsafe_get0();
+  run_async([] {
+    // we don't want to be enforced to always do `make_ready_future(...)`.
+    // as in seastar::future, the futurization should take care about
+    // turning non-future types (e.g. int) into futurized ones (e.g.
+    // ertr::future<int>).
+    return ertr::now().safe_then([] {
+      return 42;
+    }).safe_then([](int life) {
+      return ertr::make_ready_future<int>(life);
+    }).unsafe_get0();
   });
 }

From 53c4acde1d6b00ed703a71c3137bee92df4be46a Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 31 Jan 2024 15:32:30 -0800
Subject: [PATCH 2066/2492] crimson/common: expose set_coroutine for crimson
 future specializations/wrappers

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/errorator.h            | 4 ++++
 src/crimson/common/interruptible_future.h | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h
index 692498b75aed..7e7d2c64a2d6 100644
--- a/src/crimson/common/errorator.h
+++ b/src/crimson/common/errorator.h
@@ -607,6 +607,10 @@ struct errorator {
                     "ErrorT is not enlisted in errorator");
     }
 
+    void set_coroutine(seastar::task& coroutine) noexcept {
+      base_t::set_coroutine(coroutine);
+    }
+
     template <class ValueFuncT, class ErrorVisitorT>
     auto safe_then(ValueFuncT&& valfunc, ErrorVisitorT&& errfunc) {
       static_assert((... && std::is_invocable_v<ErrorVisitorT,
diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 5f8323a66d14..ec49eac0b0fc 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -420,6 +420,10 @@ class [[nodiscard]] interruptible_future_detail<InterruptCond, seastar::future<T
     : core_type(std::move(base))
   {}
 
+  void set_coroutine(seastar::task& coroutine) noexcept {
+    core_type::set_coroutine(coroutine);
+  }
+
   using value_type = typename seastar::future<T>::value_type;
   using tuple_type = typename seastar::future<T>::tuple_type;
 
@@ -773,6 +777,10 @@ class [[nodiscard]] interruptible_future_detail<
     : core_type(::seastar::futurize<core_type>::make_exception_future(std::move(ep))) {
   }
 
+  void set_coroutine(seastar::task& coroutine) noexcept {
+    core_type::set_coroutine(coroutine);
+  }
+
   template<bool interruptible = true, typename ValueInterruptCondT, typename ErrorVisitorT,
 	   std::enable_if_t<!interruptible, int> = 0>
   [[gnu::always_inline]]

From 1f152bf9861dd04ace70a62ba8a82c33136b5fb0 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 31 Jan 2024 15:33:33 -0800
Subject: [PATCH 2067/2492] crimson/common/interruptible_future: add
 interruptor::future convenience declaration

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index ec49eac0b0fc..ff80d71257ed 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -1096,6 +1096,9 @@ struct interruptor
 public:
   using condition = InterruptCond;
 
+  template <typename T>
+  using future = interruptible_future<InterruptCond, T>;
+
   static const void *get_interrupt_cond() {
     return (const void*)interrupt_cond<InterruptCond>.interrupt_cond.get();
   }

From 0afd9fa4326d4e7fe5966347a37623379072fd98 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 7 Feb 2024 17:25:38 -0800
Subject: [PATCH 2068/2492] crimson/.../errorator: expose get_exception

Useful for intentionally ignoring an exceptional future.  Otherwise,
seastar warns.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/errorator.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h
index 7e7d2c64a2d6..a9d743fcb84f 100644
--- a/src/crimson/common/errorator.h
+++ b/src/crimson/common/errorator.h
@@ -522,9 +522,9 @@ struct errorator {
     }
 
   protected:
-    using base_t::get_exception;
     friend class ::transaction_manager_test_t;
   public:
+    using base_t::get_exception;
     using errorator_type = ::crimson::errorator<AllowedErrors...>;
     using promise_type = seastar::promise<ValueT>;
 

From 52800dcee08164f9d17d9d293e01cce173670f63 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 3 Feb 2024 22:46:34 -0800
Subject: [PATCH 2069/2492] crimson/common: add
 interruptible_errorator::make_interruptible

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index ff80d71257ed..290de440949c 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -694,6 +694,12 @@ struct interruptible_errorator {
 	Errorator::template make_ready_future<ValueT>(
 	  std::forward<A>(value)...));
   }
+
+  template <template <typename> typename FutureType, typename ValueT>
+  static future<ValueT> make_interruptible(FutureType<ValueT> &&fut) {
+    return std::move(fut);
+  }
+
   static interruptible_future_detail<
     InterruptCond,
     typename Errorator::template future<>> now() {

From 11698a6da9d518329003a90777b0ef31454d2bfd Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Sat, 3 Feb 2024 22:47:04 -0800
Subject: [PATCH 2070/2492] crimson/common: expose unsafe_get0 for
 interruptible errorated futures

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 290de440949c..d7e5934fdd7a 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -721,6 +721,7 @@ class [[nodiscard]] interruptible_future_detail<
 {
 public:
   using core_type = ErroratedFuture<crimson::errorated_future_marker<T>>;
+  using core_type::unsafe_get0;
   using errorator_type = typename core_type::errorator_type;
   using interrupt_errorator_type =
     interruptible_errorator<InterruptCond, errorator_type>;

From 9852f4d97e7f4ff6a83b90bf7fa5a15b59e4342a Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 1 Mar 2024 10:32:32 +0800
Subject: [PATCH 2071/2492] crimson/os/seastore: adjust
 SeaStore::_omap_set_kvs() params

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 2 +-
 src/crimson/os/seastore/seastore.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index d3a84fc49758..801667c9c30a 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1705,7 +1705,7 @@ SeaStore::Shard::_zero(
 
 SeaStore::Shard::omap_set_kvs_ret
 SeaStore::Shard::_omap_set_kvs(
-  OnodeRef &onode,
+  const OnodeRef &onode,
   const omap_root_le_t& omap_root,
   Transaction& t,
   std::map<std::string, ceph::bufferlist>&& kvs)
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index 3f9ec175c905..e0e4d7eea929 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -426,7 +426,7 @@ class SeaStore final : public FuturizedStore {
       const coll_t& cid);
     using omap_set_kvs_ret = tm_iertr::future<omap_root_t>;
     omap_set_kvs_ret _omap_set_kvs(
-      OnodeRef &onode,
+      const OnodeRef &onode,
       const omap_root_le_t& omap_root,
       Transaction& t,
       std::map<std::string, ceph::bufferlist>&& kvs);

From 1fda8ed2d4a97b5d8e882ada2dd4b582c4da622b Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Thu, 29 Feb 2024 13:05:42 +0530
Subject: [PATCH 2072/2492] qa: Add multifs root_squash testcase

Fixes: https://tracker.ceph.com/issues/64641
Signed-off-by: Kotresh HR <khiremat@redhat.com>
---
 qa/tasks/cephfs/test_admin.py | 42 +++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py
index ee2c72f900ef..3de6b32f3578 100644
--- a/qa/tasks/cephfs/test_admin.py
+++ b/qa/tasks/cephfs/test_admin.py
@@ -1367,6 +1367,8 @@ def test_mirroring_with_filesystem_reset(self):
 class TestFsAuthorize(CephFSTestCase):
     client_id = 'testuser'
     client_name = 'client.' + client_id
+    CLIENTS_REQUIRED = 2
+    MDSS_REQUIRED = 3
 
     def test_single_path_r(self):
         PERM = 'r'
@@ -1406,6 +1408,46 @@ def test_single_path_rootsquash(self):
         self.captester.conduct_neg_test_for_chown_caps()
         self.captester.conduct_neg_test_for_truncate_caps()
 
+    def test_multifs_single_path_rootsquash(self):
+        """
+        Test root_squash with multi fs
+        """
+        self.fs1 = self.fs
+        self.fs2 = self.mds_cluster.newfs('testcephfs2')
+        self.mount_b.remount(cephfs_name=self.fs2.name)
+        self.captesters = (CapTester(self.mount_a), CapTester(self.mount_b))
+
+        if not isinstance(self.mount_a, FuseMount):
+            self.skipTest("only FUSE client has CEPHFS_FEATURE_MDS_AUTH_CAPS "
+                          "needed to enforce root_squash MDS caps")
+
+        # Authorize client to fs1
+        PERM = 'rw'
+        FS_AUTH_CAPS = (('/', PERM, 'root_squash'),)
+        self.captester = CapTester(self.mount_a, '/')
+        self.fs1.authorize(self.client_id, FS_AUTH_CAPS)
+
+        # Authorize client to fs2
+        self.fs2.authorize(self.client_id, FS_AUTH_CAPS)
+        keyring = self.fs.mon_manager.get_keyring(self.client_id)
+
+        self._remount(self.mount_a, self.fs1.name, keyring)
+        self._remount(self.mount_b, self.fs2.name, keyring)
+        # testing MDS caps...
+        # Since root_squash is set in client caps, client can read but not
+        # write even though access level is set to "rw" on both fses
+        self.captester[0].conduct_pos_test_for_read_caps()
+        self.captester[0].conduct_pos_test_for_open_caps()
+        self.captester[0].conduct_neg_test_for_write_caps(sudo_write=True)
+        self.captester[0].conduct_neg_test_for_chown_caps()
+        self.captester[0].conduct_neg_test_for_truncate_caps()
+
+        self.captester[1].conduct_pos_test_for_read_caps()
+        self.captester[1].conduct_pos_test_for_open_caps()
+        self.captester[1].conduct_neg_test_for_write_caps(sudo_write=True)
+        self.captester[1].conduct_neg_test_for_chown_caps()
+        self.captester[1].conduct_neg_test_for_truncate_caps()
+
     def test_single_path_rootsquash_issue_56067(self):
         """
         That a FS client using root squash MDS caps allows non-root user to write data

From f50385ef71a3d0ac175018cb070dbf2e767e6ed3 Mon Sep 17 00:00:00 2001
From: Gal Salomon <gal.salomon@gmail.com>
Date: Tue, 13 Feb 2024 17:47:26 +0200
Subject: [PATCH 2073/2492] adding recent update: json-output and more

Signed-off-by: Gal Salomon <gal.salomon@gmail.com>
---
 src/rgw/rgw_s3select.cc | 3 ++-
 src/s3select            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_s3select.cc b/src/rgw/rgw_s3select.cc
index 7a407b530469..bb2c3686792a 100644
--- a/src/rgw/rgw_s3select.cc
+++ b/src/rgw/rgw_s3select.cc
@@ -529,7 +529,8 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_json(const char* query, const char
   }
     
   //initializing json processor
-  m_s3_json_object.set_json_query(&s3select_syntax);
+  json_object::csv_definitions output_definition;
+  m_s3_json_object.set_json_query(&s3select_syntax,output_definition);
 
   if (input == nullptr) {
     input = "";
diff --git a/src/s3select b/src/s3select
index 91d843873ff0..1ae458f1b8df 160000
--- a/src/s3select
+++ b/src/s3select
@@ -1 +1 @@
-Subproject commit 91d843873ff0411ec08445ab71ec85e62f5e2ea7
+Subproject commit 1ae458f1b8df4447860cb666031e3e416e2c931f

From 94803e907abd52f6fdc272098fcfcbc6c44d7a39 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xxhdx1985126@gmail.com>
Date: Thu, 29 Feb 2024 21:20:56 +0800
Subject: [PATCH 2074/2492] crimson/os/seastore/omap_manager: expand omap tree
 nodes

Teuthology tests can issue large omap kvs

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
---
 src/crimson/os/seastore/omap_manager.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/omap_manager.h b/src/crimson/os/seastore/omap_manager.h
index fc4e03e2b9d2..f558f5edc3e6 100644
--- a/src/crimson/os/seastore/omap_manager.h
+++ b/src/crimson/os/seastore/omap_manager.h
@@ -14,8 +14,10 @@
 #include "crimson/os/seastore/seastore_types.h"
 #include "crimson/os/seastore/transaction_manager.h"
 
-#define OMAP_INNER_BLOCK_SIZE 4096
-#define OMAP_LEAF_BLOCK_SIZE 8192
+//TODO: calculate the max key and value sizes the current layout supports,
+//	and return errors during insert if the max is exceeded.
+#define OMAP_INNER_BLOCK_SIZE 8192
+#define OMAP_LEAF_BLOCK_SIZE 65536
 
 namespace crimson::os::seastore {
 

From 9d8f5d7132741c996a3f66d62a8ec9e9a34fa0c0 Mon Sep 17 00:00:00 2001
From: Gal Salomon <gal.salomon@gmail.com>
Date: Fri, 1 Mar 2024 14:48:13 +0200
Subject: [PATCH 2075/2492] add a fix related to RGW crash upon compression
 (assert is replaced by exception)

Signed-off-by: Gal Salomon <gal.salomon@gmail.com>
---
 src/s3select | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/s3select b/src/s3select
index 1ae458f1b8df..071146cf108d 160000
--- a/src/s3select
+++ b/src/s3select
@@ -1 +1 @@
-Subproject commit 1ae458f1b8df4447860cb666031e3e416e2c931f
+Subproject commit 071146cf108d84119e4cd825834fd00f14b83497

From 4acd7beed7b3f5b457e8641313f9384c36ae6140 Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Fri, 1 Mar 2024 14:01:08 +0000
Subject: [PATCH 2076/2492] os/bluestore: Mute warnings

+ simplify op_bl size estimation
+ fix seq size estimation
+ skip useless calculation of crc32c

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
---
 src/os/bluestore/bluefs_types.cc | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/os/bluestore/bluefs_types.cc b/src/os/bluestore/bluefs_types.cc
index ff9c96ab660f..5b2281a9ffd5 100644
--- a/src/os/bluestore/bluefs_types.cc
+++ b/src/os/bluestore/bluefs_types.cc
@@ -228,21 +228,15 @@ std::ostream& operator<<(std::ostream& out, const bluefs_fnode_delta_t& delta)
 
 // bluefs_transaction_t
 
-DENC_HELPERS
 void bluefs_transaction_t::bound_encode(size_t &s) const {
-  uint32_t crc = op_bl.crc32c(-1);
-  DENC_START(1, 1, s);
+  uint32_t crc = -1;
+  s += 1; // version
+  s += 1; // compat
+  s += 4; // size
   denc(uuid, s);
-  denc_varint(seq, s);
-  // not using bufferlist encode method, as it merely copies the bufferptr and not
-  // contents, meaning we're left with fragmented target bl
-  __u32 len = op_bl.length();
-  denc(len, s);
-  for (auto& it : op_bl.buffers()) {
-    s += it.length();
-  }
+  denc(seq, s);
+  denc(op_bl, s);
   denc(crc, s);
-  DENC_FINISH(s);
 }
 
 void bluefs_transaction_t::encode(bufferlist& bl) const

From 1576c1f5b98ecc838b47a916199e3cae8c925acb Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 1 Mar 2024 10:15:27 -0500
Subject: [PATCH 2077/2492] rgw: remove unused object lock stuff from
 CompleteMultipart

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_op.cc   | 22 ----------------------
 src/rgw/rgw_op.h    |  3 ---
 src/rgw/rgw_rest.cc |  6 ------
 3 files changed, 31 deletions(-)

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 73adf0a9c5e7..1d8db038d0f4 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6547,28 +6547,6 @@ int RGWCompleteMultipart::verify_permission(optional_yield y)
   rgw_iam_add_crypt_attrs(s->env, s->info.crypt_attribute_map);
 
   if (s->iam_policy || ! s->iam_user_policies.empty() || ! s->session_policies.empty()) {
-    if (s->bucket->get_info().obj_lock_enabled() && bypass_governance_mode) {
-      auto r = eval_identity_or_session_policies(this, s->iam_user_policies, s->env,
-                                  rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name()));
-      if (r == Effect::Deny) {
-        bypass_perm = false;
-      } else if (r == Effect::Pass && s->iam_policy) {
-        ARN arn(s->bucket->get_key(), s->object->get_name());
-        r = s->iam_policy->eval(s->env, *s->auth.identity, rgw::IAM::s3BypassGovernanceRetention, arn);    
-        if (r == Effect::Deny) {
-          bypass_perm = false;
-        }
-      } else if (r == Effect::Pass && !s->session_policies.empty()) {
-        r = eval_identity_or_session_policies(this, s->session_policies, s->env,
-                               rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name()));
-        if (r == Effect::Deny) {
-          bypass_perm = false;
-        }
-      } else if (r == Effect::Pass) {
-        bypass_perm = false;
-      }
-      bypass_governance_mode &= bypass_perm;
-    }
     auto identity_policy_res = eval_identity_or_session_policies(this, s->iam_user_policies, s->env,
                                               rgw::IAM::s3PutObject,
                                               s->object->get_obj());
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 6e962de4f473..fa61aefc5632 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1843,9 +1843,6 @@ class RGWCompleteMultipart : public RGWOp {
   std::unique_ptr<rgw::sal::Notification> res;
   std::unique_ptr<rgw::sal::Object> meta_obj;
   off_t ofs = 0;
-  //object lock
-  bool bypass_perm = true;
-  bool bypass_governance_mode = false;
 
 public:
   RGWCompleteMultipart() {}
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index d830fd40caad..a7c35f3773cc 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -1572,12 +1572,6 @@ int RGWCompleteMultipart_ObjStore::get_params(optional_yield y)
   std::tie(op_ret, data) = read_all_input(s, max_size);
   if (op_ret < 0)
     return op_ret;
-  
-  const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
-  if (bypass_gov_header) {
-    std::string bypass_gov_decoded = url_decode(bypass_gov_header);
-    bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true");
-  }
 
   return 0;
 }

From da91b786eee3429e085b20bdb79fb3faef140606 Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Wed, 21 Feb 2024 08:23:16 -0800
Subject: [PATCH 2078/2492] doc: 16.2.15 Release Notes

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
Signed-off-by: Laura Flores <lflores@ibm.com>
Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 doc/releases/index.rst    |   1 +
 doc/releases/pacific.rst  | 234 ++++++++++++++++++++++++++++++++++++++
 doc/releases/releases.yml |   2 +
 3 files changed, 237 insertions(+)

diff --git a/doc/releases/index.rst b/doc/releases/index.rst
index 6b0d16f384f9..41445033c3ec 100644
--- a/doc/releases/index.rst
+++ b/doc/releases/index.rst
@@ -77,6 +77,7 @@ Release timeline
 .. _17.2.7: quincy#v17-2-7-quincy
 
 .. _Pacific: pacific
+.. _16.2.15: pacific#v16-2-15-pacific
 .. _16.2.14: pacific#v16-2-14-pacific
 .. _16.2.13: pacific#v16-2-13-pacific
 .. _16.2.12: pacific#v16-2-12-pacific
diff --git a/doc/releases/pacific.rst b/doc/releases/pacific.rst
index c25d4ea55583..36e6483fae6f 100644
--- a/doc/releases/pacific.rst
+++ b/doc/releases/pacific.rst
@@ -2,6 +2,240 @@
 Pacific
 =======
 
+v16.2.15 Pacific
+================
+
+This is the fifteenth, and expected to be last, backport release in the Pacific series.
+
+Notable Changes
+---------------
+
+* `ceph config dump --format <json|xml>` output will display the localized
+  option names instead of their normalized version. For example,
+  "mgr/prometheus/x/server_port" will be displayed instead of
+  "mgr/prometheus/server_port". This matches the output of the non pretty-print
+  formatted version of the command.
+
+* CephFS: MDS evicts clients who are not advancing their request tids, which causes
+  a large buildup of session metadata, resulting in the MDS going read-only due to
+  the RADOS operation exceeding the size threshold. The `mds_session_metadata_threshold`
+  config controls the maximum size that an (encoded) session metadata can grow.
+
+* RADOS: The `get_pool_is_selfmanaged_snaps_mode` C++ API has been deprecated
+  due to its susceptibility to false negative results.  Its safer replacement is
+  `pool_is_in_selfmanaged_snaps_mode`.
+
+* RBD: When diffing against the beginning of time (`fromsnapname == NULL`) in
+  fast-diff mode (`whole_object == true` with `fast-diff` image feature enabled
+  and valid), diff-iterate is now guaranteed to execute locally if exclusive
+  lock is available.  This brings a dramatic performance improvement for QEMU
+  live disk synchronization and backup use cases.
+
+Changelog
+---------
+
+* [CVE-2023-43040] rgw: Fix bucket validation against POST policies (`pr#53758 <https://github.com/ceph/ceph/pull/53758>`_, Joshua Baergen)
+* admin/doc-requirements: bump Sphinx to 5.0.2 (`pr#55258 <https://github.com/ceph/ceph/pull/55258>`_, Nizamudeen A)
+* blk/kernel: Add O_EXCL for block devices (`pr#53567 <https://github.com/ceph/ceph/pull/53567>`_, Adam Kupczyk)
+* Bluestore: fix bluestore collection_list latency perf counter (`pr#52949 <https://github.com/ceph/ceph/pull/52949>`_, Wangwenjuan)
+* bluestore: Fix problem with volume selector (`pr#53587 <https://github.com/ceph/ceph/pull/53587>`_, Adam Kupczyk)
+* ceph-volume,python-common: Data allocate fraction (`pr#53581 <https://github.com/ceph/ceph/pull/53581>`_, Jonas Pfefferle)
+* ceph-volume: add --osd-id option to raw prepare (`pr#52928 <https://github.com/ceph/ceph/pull/52928>`_, Guillaume Abrioux)
+* ceph-volume: fix a bug in _check_generic_reject_reasons (`pr#54707 <https://github.com/ceph/ceph/pull/54707>`_, Kim Minjong, Guillaume Abrioux, Michael English)
+* ceph-volume: fix raw list for lvm devices (`pr#52981 <https://github.com/ceph/ceph/pull/52981>`_, Guillaume Abrioux)
+* ceph-volume: fix zap_partitions() in devices.lvm.zap (`pr#55658 <https://github.com/ceph/ceph/pull/55658>`_, Guillaume Abrioux)
+* ceph-volume: fix zap_partitions() in devices.lvm.zap (`pr#55481 <https://github.com/ceph/ceph/pull/55481>`_, Guillaume Abrioux)
+* ceph-volume: fixes fallback to stat in is_device and is_partition (`pr#54709 <https://github.com/ceph/ceph/pull/54709>`_, Guillaume Abrioux, Teoman ONAY)
+* ceph: allow xlock state to be LOCK_PREXLOCK when putting it (`pr#53662 <https://github.com/ceph/ceph/pull/53662>`_, Xiubo Li)
+* cephadm: add tcmu-runner to logrotate config (`pr#53975 <https://github.com/ceph/ceph/pull/53975>`_, Adam King)
+* cephadm: Adding support to configure public_network cfg section (`pr#52411 <https://github.com/ceph/ceph/pull/52411>`_, Redouane Kachach)
+* cephadm: allow ports to be opened in firewall during adoption, reconfig, redeploy (`pr#52083 <https://github.com/ceph/ceph/pull/52083>`_, Adam King)
+* cephadm: make custom_configs work for tcmu-runner container (`pr#53469 <https://github.com/ceph/ceph/pull/53469>`_, Adam King)
+* cephadm: run tcmu-runner through script to do restart on failure (`pr#53977 <https://github.com/ceph/ceph/pull/53977>`_, Adam King, Raimund Sacherer)
+* cephfs-journal-tool: disambiguate usage of all keyword (in tool help) (`pr#53645 <https://github.com/ceph/ceph/pull/53645>`_, Manish M Yathnalli)
+* cephfs-mirror: do not run concurrent C_RestartMirroring context (`issue#62072 <http://tracker.ceph.com/issues/62072>`_, `pr#53640 <https://github.com/ceph/ceph/pull/53640>`_, Venky Shankar)
+* cephfs-top: include the missing fields in --dump output (`pr#53453 <https://github.com/ceph/ceph/pull/53453>`_, Jos Collin)
+* cephfs: upgrade cephfs-shell's path wherever necessary (`pr#54144 <https://github.com/ceph/ceph/pull/54144>`_, Rishabh Dave)
+* cephfs_mirror: correctly set top level dir permissions (`pr#53270 <https://github.com/ceph/ceph/pull/53270>`_, Milind Changire)
+* client: always refresh mds feature bits on session open (`issue#63188 <http://tracker.ceph.com/issues/63188>`_, `pr#54245 <https://github.com/ceph/ceph/pull/54245>`_, Venky Shankar)
+* client: fix sync fs to force flush mdlog for all sessions (`pr#53981 <https://github.com/ceph/ceph/pull/53981>`_, Xiubo Li)
+* client: issue a cap release immediately if no cap exists (`pr#52852 <https://github.com/ceph/ceph/pull/52852>`_, Xiubo Li)
+* client: queue a delay cap flushing if there are ditry caps/snapcaps (`pr#54472 <https://github.com/ceph/ceph/pull/54472>`_, Xiubo Li)
+* cmake/modules/BuildRocksDB.cmake: inherit parent's CMAKE_CXX_FLAGS (`pr#55500 <https://github.com/ceph/ceph/pull/55500>`_, Kefu Chai)
+* common/weighted_shuffle: don't feed std::discrete_distribution with all-zero weights (`pr#55155 <https://github.com/ceph/ceph/pull/55155>`_, Radosław Zarzyński)
+* common:  intrusive_lru destructor add (`pr#54558 <https://github.com/ceph/ceph/pull/54558>`_, Ali Maredia)
+* doc/cephfs: note regarding start time time zone (`pr#53576 <https://github.com/ceph/ceph/pull/53576>`_, Milind Changire)
+* doc/cephfs: write cephfs commands fully in docs (`pr#53403 <https://github.com/ceph/ceph/pull/53403>`_, Rishabh Dave)
+* doc/rados/configuration/bluestore-config-ref: Fix lowcase typo (`pr#54696 <https://github.com/ceph/ceph/pull/54696>`_, Adam Kupczyk)
+* doc/rados: update config for autoscaler (`pr#55440 <https://github.com/ceph/ceph/pull/55440>`_, Zac Dover)
+* doc: clarify use of `rados rm` command (`pr#51260 <https://github.com/ceph/ceph/pull/51260>`_, J. Eric Ivancich)
+* doc: discuss the standard multi-tenant CephFS security model (`pr#53560 <https://github.com/ceph/ceph/pull/53560>`_, Greg Farnum)
+* Fixing example of BlueStore resharding (`pr#54474 <https://github.com/ceph/ceph/pull/54474>`_, Adam Kupczyk)
+* isa-l: incorporate fix for aarch64 text relocation (`pr#51314 <https://github.com/ceph/ceph/pull/51314>`_, luo rixin)
+* libcephsqlite: fill 0s in unread portion of buffer (`pr#53103 <https://github.com/ceph/ceph/pull/53103>`_, Patrick Donnelly)
+* librados: make querying pools for selfmanaged snaps reliable (`pr#55024 <https://github.com/ceph/ceph/pull/55024>`_, Ilya Dryomov)
+* librbd: Append one journal event per image request (`pr#54820 <https://github.com/ceph/ceph/pull/54820>`_, Joshua Baergen)
+* librbd: don't report HOLE_UPDATED when diffing against a hole (`pr#54949 <https://github.com/ceph/ceph/pull/54949>`_, Ilya Dryomov)
+* librbd: fix regressions in ObjectListSnapsRequest (`pr#54860 <https://github.com/ceph/ceph/pull/54860>`_, Ilya Dryomov)
+* librbd: improve rbd_diff_iterate2() performance in fast-diff mode (`pr#55256 <https://github.com/ceph/ceph/pull/55256>`_, Ilya Dryomov)
+* librbd: kick ExclusiveLock state machine on client being blocklisted when waiting for lock (`pr#53295 <https://github.com/ceph/ceph/pull/53295>`_, Ramana Raja)
+* librbd: make CreatePrimaryRequest remove any unlinked mirror snapshots (`pr#53274 <https://github.com/ceph/ceph/pull/53274>`_, Ilya Dryomov)
+* log: fix the formatting when dumping thread IDs (`pr#53465 <https://github.com/ceph/ceph/pull/53465>`_, Radoslaw Zarzynski)
+* log: Make log_max_recent have an effect again (`pr#48311 <https://github.com/ceph/ceph/pull/48311>`_, Joshua Baergen)
+* make-dist: don't use --continue option for wget (`pr#55090 <https://github.com/ceph/ceph/pull/55090>`_, Casey Bodley)
+* make-dist: download liburing from kernel.io instead of github (`pr#53197 <https://github.com/ceph/ceph/pull/53197>`_, Laura Flores)
+* MClientRequest: properly handle ceph_mds_request_head_legacy for ext_num_retry, ext_num_fwd, owner_uid, owner_gid (`pr#54410 <https://github.com/ceph/ceph/pull/54410>`_, Alexander Mikhalitsyn)
+* mds,qa: some balancer debug messages (<=5) not printed when debug_mds is >=5 (`pr#53552 <https://github.com/ceph/ceph/pull/53552>`_, Patrick Donnelly)
+* mds/Server: mark a cap acquisition throttle event in the request (`pr#53169 <https://github.com/ceph/ceph/pull/53169>`_, Leonid Usov)
+* mds: acquire inode snaplock in open (`pr#53185 <https://github.com/ceph/ceph/pull/53185>`_, Patrick Donnelly)
+* mds: add event for batching getattr/lookup (`pr#53556 <https://github.com/ceph/ceph/pull/53556>`_, Patrick Donnelly)
+* mds: adjust pre_segments_size for MDLog when trimming segments for st… (`issue#59833 <http://tracker.ceph.com/issues/59833>`_, `pr#54033 <https://github.com/ceph/ceph/pull/54033>`_, Venky Shankar)
+* mds: blocklist clients with "bloated" session metadata (`issue#61947 <http://tracker.ceph.com/issues/61947>`_, `issue#62873 <http://tracker.ceph.com/issues/62873>`_, `pr#53634 <https://github.com/ceph/ceph/pull/53634>`_, Venky Shankar)
+* mds: drop locks and retry when lock set changes (`pr#53243 <https://github.com/ceph/ceph/pull/53243>`_, Patrick Donnelly)
+* mds: ensure next replay is queued on req drop (`pr#54314 <https://github.com/ceph/ceph/pull/54314>`_, Patrick Donnelly)
+* mds: fix deadlock between unlinking and linkmerge (`pr#53495 <https://github.com/ceph/ceph/pull/53495>`_, Xiubo Li)
+* mds: fix issuing redundant reintegrate/migrate_stray requests (`pr#54517 <https://github.com/ceph/ceph/pull/54517>`_, Xiubo Li)
+* mds: log message when exiting due to asok command (`pr#53550 <https://github.com/ceph/ceph/pull/53550>`_, Patrick Donnelly)
+* mds: replacing bootstrap session only if handle client session message (`pr#53362 <https://github.com/ceph/ceph/pull/53362>`_, Mer Xuanyi)
+* mds: report clients laggy due laggy OSDs only after checking any OSD is laggy (`pr#54120 <https://github.com/ceph/ceph/pull/54120>`_, Dhairya Parmar)
+* mds: set the loner to true for LOCK_EXCL_XSYN (`pr#54912 <https://github.com/ceph/ceph/pull/54912>`_, Xiubo Li)
+* mds: use variable g_ceph_context directly in MDSAuthCaps (`pr#52821 <https://github.com/ceph/ceph/pull/52821>`_, Rishabh Dave)
+* mgr/BaseMgrModule: Optimize CPython Call in Finish Function (`pr#55109 <https://github.com/ceph/ceph/pull/55109>`_, Nitzan Mordechai)
+* mgr/cephadm: Add "networks" parameter to orch apply rgw (`pr#53974 <https://github.com/ceph/ceph/pull/53974>`_, Teoman ONAY)
+* mgr/cephadm: ceph orch add fails when ipv6 address is surrounded by square brackets (`pr#53978 <https://github.com/ceph/ceph/pull/53978>`_, Teoman ONAY)
+* mgr/dashboard: add 'omit_usage' query param to dashboard api 'get rbd' endpoint (`pr#54192 <https://github.com/ceph/ceph/pull/54192>`_, Cory Snyder)
+* mgr/dashboard: allow tls 1.2 with a config option (`pr#53781 <https://github.com/ceph/ceph/pull/53781>`_, Nizamudeen A)
+* mgr/dashboard: Consider null values as zero in grafana panels (`pr#54542 <https://github.com/ceph/ceph/pull/54542>`_, Aashish Sharma)
+* mgr/dashboard: fix CephPGImbalance alert (`pr#49478 <https://github.com/ceph/ceph/pull/49478>`_, Aashish Sharma)
+* mgr/dashboard: Fix CephPoolGrowthWarning alert (`pr#49477 <https://github.com/ceph/ceph/pull/49477>`_, Aashish Sharma)
+* mgr/dashboard: fix constraints.txt (`pr#54652 <https://github.com/ceph/ceph/pull/54652>`_, Ernesto Puerta)
+* mgr/dashboard: fix rgw page issues when hostname not resolvable (`pr#53215 <https://github.com/ceph/ceph/pull/53215>`_, Nizamudeen A)
+* mgr/dashboard: set CORS header for unauthorized access (`pr#53202 <https://github.com/ceph/ceph/pull/53202>`_, Nizamudeen A)
+* mgr/prometheus: avoid duplicates and deleted entries for rbd_stats_pools (`pr#48524 <https://github.com/ceph/ceph/pull/48524>`_, Avan Thakkar)
+* mgr/prometheus: change pg_repaired_objects name to pool_repaired_objects (`pr#48439 <https://github.com/ceph/ceph/pull/48439>`_, Pere Diaz Bou)
+* mgr/prometheus: fix pool_objects_repaired and daemon_health_metrics format (`pr#51692 <https://github.com/ceph/ceph/pull/51692>`_, banuchka)
+* mgr/rbd_support: fix recursive locking on CreateSnapshotRequests lock (`pr#54293 <https://github.com/ceph/ceph/pull/54293>`_, Ramana Raja)
+* mgr/snap-schedule: use the right way to check the result returned by… (`pr#53355 <https://github.com/ceph/ceph/pull/53355>`_, Mer Xuanyi)
+* mgr/snap_schedule: allow retention spec 'n' to be user defined (`pr#52750 <https://github.com/ceph/ceph/pull/52750>`_, Milind Changire, Jakob Haufe)
+* mgr/volumes: Fix pending_subvolume_deletions in volume info (`pr#53574 <https://github.com/ceph/ceph/pull/53574>`_, Kotresh HR)
+* mgr: Add one finisher thread per module (`pr#51045 <https://github.com/ceph/ceph/pull/51045>`_, Kotresh HR, Patrick Donnelly)
+* mgr: add throttle policy for DaemonServer (`pr#54013 <https://github.com/ceph/ceph/pull/54013>`_, ericqzhao)
+* mgr: don't dump global config holding gil (`pr#50194 <https://github.com/ceph/ceph/pull/50194>`_, Mykola Golub)
+* mgr: fix a race condition in DaemonServer::handle_report() (`pr#52993 <https://github.com/ceph/ceph/pull/52993>`_, Radoslaw Zarzynski)
+* mgr: register OSDs in ms_handle_accept (`pr#53189 <https://github.com/ceph/ceph/pull/53189>`_, Patrick Donnelly)
+* mgr: remove out&down osd from mgr daemons (`pr#54553 <https://github.com/ceph/ceph/pull/54553>`_, shimin)
+* mon/ConfigMonitor: Show localized name in "config dump --format json" output (`pr#53984 <https://github.com/ceph/ceph/pull/53984>`_, Sridhar Seshasayee)
+* mon/MonClient: resurrect original client_mount_timeout handling (`pr#52533 <https://github.com/ceph/ceph/pull/52533>`_, Ilya Dryomov)
+* mon/Monitor.cc: exit function if !osdmon()->is_writeable() && mon/OSDMonitor: Added extra check before mon.go_recovery_stretch_mode() (`pr#51414 <https://github.com/ceph/ceph/pull/51414>`_, Kamoltat)
+* mon/Monitor: during shutdown don't accept new authentication and crea… (`pr#55113 <https://github.com/ceph/ceph/pull/55113>`_, Nitzan Mordechai)
+* mon: add exception handling to ceph health mute (`pr#55118 <https://github.com/ceph/ceph/pull/55118>`_, Daniel Radjenovic)
+* mon: add proxy to cache tier options (`pr#50552 <https://github.com/ceph/ceph/pull/50552>`_, tan changzhi)
+* mon: fix health store size growing infinitely (`pr#55472 <https://github.com/ceph/ceph/pull/55472>`_, Wei Wang)
+* mon: fix iterator mishandling in PGMap::apply_incremental (`pr#52555 <https://github.com/ceph/ceph/pull/52555>`_, Oliver Schmidt)
+* mon: fix mds metadata lost in one case (`pr#54318 <https://github.com/ceph/ceph/pull/54318>`_, shimin)
+* msg/async: initialize worker in RDMAStack::create_worker() and drop Stack::num_workers (`pr#55443 <https://github.com/ceph/ceph/pull/55443>`_, Kefu Chai)
+* msg/AsyncMessenger: re-evaluate the stop condition when woken up in 'wait()' (`pr#53716 <https://github.com/ceph/ceph/pull/53716>`_, Leonid Usov)
+* nofail option in fstab not supported (`pr#52987 <https://github.com/ceph/ceph/pull/52987>`_, Leonid Usov)
+* os/bluestore: don't require bluestore_db_block_size when attaching new (`pr#52948 <https://github.com/ceph/ceph/pull/52948>`_, Igor Fedotov)
+* os/bluestore: get rid off resulting lba alignment in allocators (`pr#54434 <https://github.com/ceph/ceph/pull/54434>`_, Igor Fedotov)
+* osd,bluestore: gracefully handle a failure during meta collection load (`pr#53135 <https://github.com/ceph/ceph/pull/53135>`_, Igor Fedotov)
+* osd/OpRequest: Add detail description for delayed op in osd log file (`pr#53693 <https://github.com/ceph/ceph/pull/53693>`_, Yite Gu)
+* osd/OSD: introduce reset_purged_snaps_last (`pr#53970 <https://github.com/ceph/ceph/pull/53970>`_, Matan Breizman)
+* osd/OSDMap: Check for uneven weights & != 2 buckets post stretch mode (`pr#52459 <https://github.com/ceph/ceph/pull/52459>`_, Kamoltat)
+* osd/scrub: Fix scrub starts messages spamming the cluster log (`pr#53430 <https://github.com/ceph/ceph/pull/53430>`_, Prashant D)
+* osd: don't require RWEXCL lock for stat+write ops (`pr#54593 <https://github.com/ceph/ceph/pull/54593>`_, Alice Zhao)
+* osd: ensure async recovery does not drop a pg below min_size (`pr#54548 <https://github.com/ceph/ceph/pull/54548>`_, Samuel Just)
+* osd: fix shard-threads cannot wakeup bug (`pr#51262 <https://github.com/ceph/ceph/pull/51262>`_, Jianwei Zhang)
+* osd: fix use-after-move in build_incremental_map_msg() (`pr#54268 <https://github.com/ceph/ceph/pull/54268>`_, Ronen Friedman)
+* osd: log the number of extents for sparse read (`pr#54604 <https://github.com/ceph/ceph/pull/54604>`_, Xiubo Li)
+* pacifc: Revert "mgr/dashboard: unselect rows in datatables" (`pr#55415 <https://github.com/ceph/ceph/pull/55415>`_, Nizamudeen A)
+* pybind/mgr/autoscaler: Donot show NEW PG_NUM value if autoscaler is not on (`pr#53464 <https://github.com/ceph/ceph/pull/53464>`_, Prashant D)
+* pybind/mgr/mgr_util: fix to_pretty_timedelta() (`pr#51243 <https://github.com/ceph/ceph/pull/51243>`_, Sage Weil)
+* pybind/mgr/volumes: log mutex locks to help debug deadlocks (`pr#53916 <https://github.com/ceph/ceph/pull/53916>`_, Kotresh HR)
+* pybind/mgr: ceph osd status crash with ZeroDivisionError (`pr#46696 <https://github.com/ceph/ceph/pull/46696>`_, Nitzan Mordechai, Kefu Chai)
+* pybind/rados: don't close watch in dealloc if already closed (`pr#51259 <https://github.com/ceph/ceph/pull/51259>`_, Tim Serong)
+* pybind/rados: fix missed changes for PEP484 style type annotations (`pr#54361 <https://github.com/ceph/ceph/pull/54361>`_, Igor Fedotov)
+* pybind/rbd: don't produce info on errors in aio_mirror_image_get_info() (`pr#54053 <https://github.com/ceph/ceph/pull/54053>`_, Ilya Dryomov)
+* python-common/drive_group: handle fields outside of 'spec' even when 'spec' is provided (`pr#52413 <https://github.com/ceph/ceph/pull/52413>`_, Adam King)
+* python-common/drive_selection: lower log level of limit policy message (`pr#52412 <https://github.com/ceph/ceph/pull/52412>`_, Adam King)
+* qa/distros: backport update from rhel 8.4 -> 8.6 (`pr#54901 <https://github.com/ceph/ceph/pull/54901>`_, Casey Bodley, David Galloway)
+* qa/suites/krbd: stress test for recovering from watch errors (`pr#53784 <https://github.com/ceph/ceph/pull/53784>`_, Ilya Dryomov)
+* qa/suites/orch: whitelist warnings that are expected in test environments (`pr#55523 <https://github.com/ceph/ceph/pull/55523>`_, Laura Flores)
+* qa/suites/rbd: add test to check rbd_support module recovery (`pr#54294 <https://github.com/ceph/ceph/pull/54294>`_, Ramana Raja)
+* qa/suites/upgrade/pacific-p2p: run librbd python API tests from pacific tip (`pr#55418 <https://github.com/ceph/ceph/pull/55418>`_, Yuri Weinstein)
+* qa/suites/upgrade/pacific-p2p: skip TestClsRbd.mirror_snapshot test (`pr#53204 <https://github.com/ceph/ceph/pull/53204>`_, Ilya Dryomov)
+* qa/suites: added more whitelisting + fix typo (`pr#55717 <https://github.com/ceph/ceph/pull/55717>`_, Kamoltat)
+* qa/tasks/cephadm: enable mon_cluster_log_to_file (`pr#55429 <https://github.com/ceph/ceph/pull/55429>`_, Dan van der Ster)
+* qa/upgrade: disable a failing ceph_test_cls_cmpomap test case (`pr#55519 <https://github.com/ceph/ceph/pull/55519>`_, Casey Bodley)
+* qa/upgrade: use ragweed branch for starting ceph release (`pr#55382 <https://github.com/ceph/ceph/pull/55382>`_, Casey Bodley)
+* qa/workunits/rbd/cli_generic.sh: narrow race window when checking that rbd_support module command fails after blocklisting the module's client (`pr#54771 <https://github.com/ceph/ceph/pull/54771>`_, Ramana Raja)
+* qa: assign file system affinity for replaced MDS (`issue#61764 <http://tracker.ceph.com/issues/61764>`_, `pr#54039 <https://github.com/ceph/ceph/pull/54039>`_, Venky Shankar)
+* qa: ignore expected cluster warning from damage tests (`pr#53486 <https://github.com/ceph/ceph/pull/53486>`_, Patrick Donnelly)
+* qa: lengthen shutdown timeout for thrashed MDS (`pr#53555 <https://github.com/ceph/ceph/pull/53555>`_, Patrick Donnelly)
+* qa: pass arg as list to fix test case failure (`pr#52763 <https://github.com/ceph/ceph/pull/52763>`_, Dhairya Parmar)
+* qa: remove duplicate import (`pr#53447 <https://github.com/ceph/ceph/pull/53447>`_, Patrick Donnelly)
+* qa: run kernel_untar_build with newer tarball (`pr#54713 <https://github.com/ceph/ceph/pull/54713>`_, Milind Changire)
+* qa: wait for file to have correct size (`pr#52744 <https://github.com/ceph/ceph/pull/52744>`_, Patrick Donnelly)
+* rados: build minimally when "WITH_MGR" is off (`pr#51250 <https://github.com/ceph/ceph/pull/51250>`_, J. Eric Ivancich)
+* rados: increase osd_max_write_op_reply_len default to 64 bytes (`pr#53470 <https://github.com/ceph/ceph/pull/53470>`_, Matt Benjamin)
+* RadosGW API: incorrect bucket quota in response to HEAD /{bucket}/?usage (`pr#53439 <https://github.com/ceph/ceph/pull/53439>`_, shreyanshjain7174)
+* radosgw-admin: allow 'bi purge' to delete index if entrypoint doesn't exist (`pr#54010 <https://github.com/ceph/ceph/pull/54010>`_, Casey Bodley)
+* radosgw-admin: don't crash on --placement-id without --storage-class (`pr#53474 <https://github.com/ceph/ceph/pull/53474>`_, Casey Bodley)
+* radosgw-admin: fix segfault on pipe modify without source/dest zone specified (`pr#51256 <https://github.com/ceph/ceph/pull/51256>`_, caisan)
+* rbd-nbd: fix stuck with disable request (`pr#54256 <https://github.com/ceph/ceph/pull/54256>`_, Prasanna Kumar Kalever)
+* rgw - Fix NoSuchTagSet error (`pr#50533 <https://github.com/ceph/ceph/pull/50533>`_, Daniel Gryniewicz)
+* rgw/auth: ignoring signatures for HTTP OPTIONS calls (`pr#55550 <https://github.com/ceph/ceph/pull/55550>`_, Tobias Urdin)
+* rgw/beast: add max_header_size option with 16k default, up from 4k (`pr#52113 <https://github.com/ceph/ceph/pull/52113>`_, Casey Bodley)
+* rgw/keystone: EC2Engine uses reject() for ERR_SIGNATURE_NO_MATCH (`pr#53764 <https://github.com/ceph/ceph/pull/53764>`_, Casey Bodley)
+* rgw/notification: remove non x-amz-meta-\* attributes from bucket notifications (`pr#53376 <https://github.com/ceph/ceph/pull/53376>`_, Juan Zhu)
+* rgw/putobj: RadosWriter uses part head object for multipart parts (`pr#55586 <https://github.com/ceph/ceph/pull/55586>`_, Casey Bodley)
+* rgw/s3: ListObjectsV2 returns correct object owners (`pr#54160 <https://github.com/ceph/ceph/pull/54160>`_, Casey Bodley)
+* rgw/sts: AssumeRole no longer writes to user metadata (`pr#52051 <https://github.com/ceph/ceph/pull/52051>`_, Casey Bodley)
+* rgw/sts: code for returning an error when an IAM policy (`pr#44462 <https://github.com/ceph/ceph/pull/44462>`_, Pritha Srivastava)
+* rgw/sts: code to fetch certs using .well-known/openid-configuration URL (`pr#44464 <https://github.com/ceph/ceph/pull/44464>`_, Pritha Srivastava)
+* rgw/sts: createbucket op should take session_policies into account (`pr#44476 <https://github.com/ceph/ceph/pull/44476>`_, Pritha Srivastava)
+* rgw/sts: fix read_obj_policy permission evaluation (`pr#44471 <https://github.com/ceph/ceph/pull/44471>`_, Pritha Srivastava)
+* rgw/sts: fixes getsessiontoken authenticated with LDAP (`pr#44463 <https://github.com/ceph/ceph/pull/44463>`_, Pritha Srivastava)
+* rgw/swift: check position of first slash in slo manifest files (`pr#51600 <https://github.com/ceph/ceph/pull/51600>`_, Marcio Roberto Starke)
+* rgw/sync-policy: Correct "sync status" & "sync group" commands (`pr#53410 <https://github.com/ceph/ceph/pull/53410>`_, Soumya Koduri)
+* rgw: 'bucket check' deletes index of multipart meta when its pending_map is nonempty (`pr#54016 <https://github.com/ceph/ceph/pull/54016>`_, Huber-ming)
+* rgw: add radosgw-admin bucket check olh/unlinked commands (`pr#53808 <https://github.com/ceph/ceph/pull/53808>`_, Cory Snyder)
+* rgw: Avoid segfault when OPA authz is enabled (`pr#46106 <https://github.com/ceph/ceph/pull/46106>`_, Benoît Knecht)
+* rgw: beast frontend checks for local_endpoint() errors (`pr#54167 <https://github.com/ceph/ceph/pull/54167>`_, Casey Bodley)
+* rgw: Drain async_processor request queue during shutdown (`pr#53472 <https://github.com/ceph/ceph/pull/53472>`_, Soumya Koduri)
+* rgw: fix 2 null versionID after convert_plain_entry_to_versioned (`pr#53400 <https://github.com/ceph/ceph/pull/53400>`_, rui ma, zhuo li)
+* rgw: Fix Browser POST content-length-range min value (`pr#52936 <https://github.com/ceph/ceph/pull/52936>`_, Robin H. Johnson)
+* rgw: fix FP error when calculating enteries per bi shard (`pr#53593 <https://github.com/ceph/ceph/pull/53593>`_, J. Eric Ivancich)
+* rgw: fix rgw cache invalidation after unregister_watch() error (`pr#54014 <https://github.com/ceph/ceph/pull/54014>`_, lichaochao)
+* rgw: fix SignatureDoesNotMatch when extra headers start with 'x-amz' (`pr#53772 <https://github.com/ceph/ceph/pull/53772>`_, rui ma)
+* rgw: Fix truncated ListBuckets response (`pr#49526 <https://github.com/ceph/ceph/pull/49526>`_, Joshua Baergen)
+* rgw: fix unwatch crash at radosgw startup (`pr#53759 <https://github.com/ceph/ceph/pull/53759>`_, lichaochao)
+* rgw: fix UploadPartCopy error code when src object not exist and src bucket not exist (`pr#53356 <https://github.com/ceph/ceph/pull/53356>`_, yuliyang)
+* rgw: handle http options CORS with v4 auth (`pr#53416 <https://github.com/ceph/ceph/pull/53416>`_, Tobias Urdin)
+* rgw: improve buffer list utilization in the chunkupload scenario (`pr#53775 <https://github.com/ceph/ceph/pull/53775>`_, liubingrun)
+* rgw: multisite data log flag not used (`pr#52055 <https://github.com/ceph/ceph/pull/52055>`_, J. Eric Ivancich)
+* rgw: pick http_date in case of http_x_amz_date absence (`pr#53443 <https://github.com/ceph/ceph/pull/53443>`_, Seena Fallah, Mohamed Awnallah)
+* rgw: prevent spurious/lost notifications in the index completion thread (`pr#49093 <https://github.com/ceph/ceph/pull/49093>`_, Casey Bodley, Yuval Lifshitz)
+* rgw: retry metadata cache notifications with INVALIDATE_OBJ (`pr#52797 <https://github.com/ceph/ceph/pull/52797>`_, Casey Bodley)
+* rgw: s3 object lock avoids overflow in retention date (`pr#52605 <https://github.com/ceph/ceph/pull/52605>`_, Casey Bodley)
+* rgw: s3website doesn't prefetch for web_dir() check (`pr#53769 <https://github.com/ceph/ceph/pull/53769>`_, Casey Bodley)
+* rgw: set keys from from master zone on admin api user create (`pr#51602 <https://github.com/ceph/ceph/pull/51602>`_, Ali Maredia)
+* rgw: Solving the issue of not populating etag in Multipart upload result (`pr#51445 <https://github.com/ceph/ceph/pull/51445>`_, Ali Masarwa)
+* rgw: swift : check for valid key in POST forms (`pr#52729 <https://github.com/ceph/ceph/pull/52729>`_, Abhishek Lekshmanan)
+* rgw: Update "CEPH_RGW_DIR_SUGGEST_LOG_OP" for remove entries (`pr#50540 <https://github.com/ceph/ceph/pull/50540>`_, Soumya Koduri)
+* rgw: use unique_ptr for flat_map emplace in BucketTrimWatche (`pr#52996 <https://github.com/ceph/ceph/pull/52996>`_, Vedansh Bhartia)
+* rgwlc: prevent lc for one bucket from exceeding time budget (`pr#53562 <https://github.com/ceph/ceph/pull/53562>`_, Matt Benjamin)
+* test/lazy-omap-stats: Various enhancements (`pr#50518 <https://github.com/ceph/ceph/pull/50518>`_, Brad Hubbard)
+* test/librbd: avoid config-related crashes in DiscardWithPruneWriteOverlap (`pr#54859 <https://github.com/ceph/ceph/pull/54859>`_, Ilya Dryomov)
+* test/store_test: adjust physical extents to inject error against (`pr#54782 <https://github.com/ceph/ceph/pull/54782>`_, Igor Fedotov)
+* tools/ceph_objectstore_tool: action_on_all_objects_in_pg to skip pgmeta (`pr#54691 <https://github.com/ceph/ceph/pull/54691>`_, Matan Breizman)
+* tools/ceph_objectstore_tool: Support get/set/superblock (`pr#55013 <https://github.com/ceph/ceph/pull/55013>`_, Matan Breizman)
+* tools/osdmaptool: fix possible segfaults when there are down osds (`pr#52203 <https://github.com/ceph/ceph/pull/52203>`_, Mykola Golub)
+* Tools/rados: Improve Error Messaging for Object Name Resolution (`pr#55111 <https://github.com/ceph/ceph/pull/55111>`_, Nitzan Mordechai)
+* vstart_runner: maintain log level when --debug is passed (`pr#52977 <https://github.com/ceph/ceph/pull/52977>`_, Rishabh Dave)
+* vstart_runner: use FileNotFoundError when os.stat() fails (`pr#52978 <https://github.com/ceph/ceph/pull/52978>`_, Rishabh Dave)
+* win32_deps_build.sh: change Boost URL (`pr#55086 <https://github.com/ceph/ceph/pull/55086>`_, Lucian Petrut)
+
 v16.2.14 Pacific
 ================
 
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
index 2793ae351006..4b769684c816 100644
--- a/doc/releases/releases.yml
+++ b/doc/releases/releases.yml
@@ -42,6 +42,8 @@ releases:
   pacific:
     target_eol: 2023-10-01
     releases:
+      - version: 16.2.15
+        released: 2024-03-04
       - version: 16.2.14
         released: 2023-08-30
       - version: 16.2.13

From 6ee5fff0c8dc71c4789496a587e2599ff2e0bc13 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 18 Oct 2023 16:22:18 -0400
Subject: [PATCH 2079/2492] mgr/cephadm: add option for grafana to bind on
 specific network

For example, with a spec like

```
[root@vm-00 ~]# cat grafana.yaml
service_type: grafana
service_name: grafana
placement:
  count: 1
networks:
  - 10.2.1.0/24
spec:
  anonymous_access: true
  protocol: https
  only_bind_port_on_networks: true
```

where the networks is set and the
"only_bind_port_on_networks" option is
set to true, the grafana daemon will bind
to its port (3000 in this case since it's
the default and I didn't set a port) only
on an IP from that network. I tested this
by holding port 3000 on an IP from a different
network on the host and then deploying
grafana. Without this patch it would have
failed with a port conflict error.

```
[root@vm-00 ~]# netstat -tulpn | grep 3000
tcp        0      0 10.2.1.61:3000          0.0.0.0:*               LISTEN      34178/grafana
tcp        0      0 192.168.122.251:3000    0.0.0.0:*               LISTEN      33274/nc
```

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/module.py                  | 11 +++++++++++
 src/pybind/mgr/cephadm/services/monitoring.py     | 15 +++++++++++++--
 src/python-common/ceph/deployment/service_spec.py |  7 +++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 7c124d787339..5cc9b894e505 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -997,6 +997,17 @@ def update_failed_daemon_health_check(self) -> None:
             self.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len(
                 failed_daemons), failed_daemons)
 
+    def get_first_matching_network_ip(self, host: str, sspec: ServiceSpec) -> Optional[str]:
+        sspec_networks = sspec.networks
+        for subnet, ifaces in self.cache.networks.get(host, {}).items():
+            host_network = ipaddress.ip_network(subnet)
+            for spec_network_str in sspec_networks:
+                spec_network = ipaddress.ip_network(spec_network_str)
+                if host_network.overlaps(spec_network):
+                    return list(ifaces.values())[0][0]
+                logger.error(f'{spec_network} from {sspec.service_name()} spec does not overlap with {host_network} on {host}')
+        return None
+
     @staticmethod
     def can_run() -> Tuple[bool, str]:
         if asyncssh is not None:
diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index d3439c04d04f..bdcf214721c6 100644
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -67,13 +67,24 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
 
         spec: GrafanaSpec = cast(
             GrafanaSpec, self.mgr.spec_store.active_specs[daemon_spec.service_name])
+
+        grafana_port = daemon_spec.ports[0] if daemon_spec.ports else self.DEFAULT_SERVICE_PORT
+        grafana_ip = daemon_spec.ip if daemon_spec.ip else ''
+
+        if spec.only_bind_port_on_networks and spec.networks:
+            assert daemon_spec.host is not None
+            ip_to_bind_to = self.mgr.get_first_matching_network_ip(daemon_spec.host, spec)
+            if ip_to_bind_to:
+                daemon_spec.port_ips = {str(grafana_port): ip_to_bind_to}
+                grafana_ip = ip_to_bind_to
+
         grafana_ini = self.mgr.template.render(
             'services/grafana/grafana.ini.j2', {
                 'anonymous_access': spec.anonymous_access,
                 'initial_admin_password': spec.initial_admin_password,
-                'http_port': daemon_spec.ports[0] if daemon_spec.ports else self.DEFAULT_SERVICE_PORT,
+                'http_port': grafana_port,
                 'protocol': spec.protocol,
-                'http_addr': daemon_spec.ip if daemon_spec.ip else ''
+                'http_addr': grafana_ip
             })
 
         if 'dashboard' in self.mgr.get('mgr_map')['modules'] and spec.initial_admin_password:
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index f6d290f07188..6808567500f8 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1833,6 +1833,7 @@ def __init__(self,
                  preview_only: bool = False,
                  config: Optional[Dict[str, str]] = None,
                  networks: Optional[List[str]] = None,
+                 only_bind_port_on_networks: bool = False,
                  port: Optional[int] = None,
                  protocol: Optional[str] = 'https',
                  initial_admin_password: Optional[str] = None,
@@ -1853,6 +1854,12 @@ def __init__(self,
         self.anonymous_access = anonymous_access
         self.protocol = protocol
 
+        # whether ports daemons for this service bind to should
+        # bind to only hte networks listed in networks param, or
+        # to all networks. Defaults to false which is saying to bind
+        # on all networks.
+        self.only_bind_port_on_networks = only_bind_port_on_networks
+
     def validate(self) -> None:
         super(GrafanaSpec, self).validate()
         if self.protocol not in ['http', 'https']:

From e6943dd61374ee8ce337751e9ef9ba55f10a83ed Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Thu, 29 Feb 2024 13:04:43 -0500
Subject: [PATCH 2080/2492] mgr/cephadm: allow having prometheus bind on
 specific network

To avoid port conflicts in certain setups where
something else is using the same port on a different
network on the host.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadmlib/daemons/monitoring.py      |  5 ++++-
 src/cephadm/tests/test_deploy.py                  |  3 ++-
 src/pybind/mgr/cephadm/services/monitoring.py     | 11 ++++++++++-
 src/pybind/mgr/cephadm/tests/test_services.py     |  9 +++++++--
 src/python-common/ceph/deployment/service_spec.py |  2 ++
 5 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/monitoring.py b/src/cephadm/cephadmlib/daemons/monitoring.py
index e0667853dd7b..e6ad2412a7cb 100644
--- a/src/cephadm/cephadmlib/daemons/monitoring.py
+++ b/src/cephadm/cephadmlib/daemons/monitoring.py
@@ -251,9 +251,11 @@ def get_daemon_args(self) -> List[str]:
                     ip = meta['ip']
                 if 'ports' in meta and meta['ports']:
                     port = meta['ports'][0]
-            r += [f'--web.listen-address={ip}:{port}']
             if daemon_type == 'prometheus':
                 config = fetch_configs(ctx)
+                ip_to_bind_to = config.get('ip_to_bind_to', '')
+                if ip_to_bind_to:
+                    ip = ip_to_bind_to
                 retention_time = config.get('retention_time', '15d')
                 retention_size = config.get(
                     'retention_size', '0'
@@ -270,6 +272,7 @@ def get_daemon_args(self) -> List[str]:
                     addr = next(iter(ipv4_addrs or ipv6_addrs), None)
                     host = wrap_ipv6(addr) if addr else host
                 r += [f'--web.external-url={scheme}://{host}:{port}']
+            r += [f'--web.listen-address={ip}:{port}']
         if daemon_type == 'alertmanager':
             config = fetch_configs(ctx)
             peers = config.get('peers', list())  # type: ignore
diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index cfde3fbce0aa..9a6da3aad4b9 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -305,6 +305,7 @@ def test_deploy_a_monitoring_container(cephadm_fs, funkypatch):
             'files': {
                 'prometheus.yml': 'bettercallherc',
             },
+            'ip_to_bind_to': '1.2.3.4'
         }
         _cephadm._common_deploy(ctx)
 
@@ -314,7 +315,7 @@ def test_deploy_a_monitoring_container(cephadm_fs, funkypatch):
         runfile_lines = f.read().splitlines()
     assert 'podman' in runfile_lines[-1]
     assert runfile_lines[-1].endswith(
-        'quay.io/titans/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.listen-address=:9095 --storage.tsdb.retention.time=15d --storage.tsdb.retention.size=0 --web.external-url=http://10.10.10.10:9095'
+        'quay.io/titans/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=15d --storage.tsdb.retention.size=0 --web.external-url=http://10.10.10.10:9095 --web.listen-address=1.2.3.4:9095'
     )
     assert '--user 8765' in runfile_lines[-1]
     assert f'-v /var/lib/ceph/{fsid}/prometheus.fire/etc/prometheus:/etc/prometheus:Z' in runfile_lines[-1]
diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index bdcf214721c6..10ddcbbd02f0 100644
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -433,6 +433,13 @@ def generate_config(
             'nvmeof_sd_url': nvmeof_sd_url,
         }
 
+        ip_to_bind_to = ''
+        if spec.only_bind_port_on_networks and spec.networks:
+            assert daemon_spec.host is not None
+            ip_to_bind_to = self.mgr.get_first_matching_network_ip(daemon_spec.host, spec) or ''
+            if ip_to_bind_to:
+                daemon_spec.port_ips = {str(port): ip_to_bind_to}
+
         web_context = {
             'prometheus_web_user': prometheus_user,
             'prometheus_web_password': password_hash(prometheus_password),
@@ -459,6 +466,7 @@ def generate_config(
                     },
                     'retention_time': retention_time,
                     'retention_size': retention_size,
+                    'ip_to_bind_to': ip_to_bind_to,
                     'web_config': '/etc/prometheus/web.yml'
                 }
         else:
@@ -467,7 +475,8 @@ def generate_config(
                     'prometheus.yml': self.mgr.template.render('services/prometheus/prometheus.yml.j2', context)
                 },
                 'retention_time': retention_time,
-                'retention_size': retention_size
+                'retention_size': retention_size,
+                'ip_to_bind_to': ip_to_bind_to
             }
 
         # include alerts, if present in the container
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index f0b5360e6e30..5631152ba555 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -685,7 +685,9 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                                                              keepalived_password='12345',
                                                              virtual_ip="1.2.3.4/32",
                                                              backend_service='rgw.foo')) as _, \
-                    with_service(cephadm_module, PrometheusSpec('prometheus')) as _:
+                    with_service(cephadm_module, PrometheusSpec('prometheus',
+                                                                networks=['1.2.3.0/24'],
+                                                                only_bind_port_on_networks=True)) as _:
 
                 y = dedent("""
                 # This file is generated by cephadm.
@@ -737,11 +739,12 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                         "deploy_arguments": [],
                         "params": {
                             'tcp_ports': [9095],
+                            'port_ips': {'8765': '1.2.3.1'}
                         },
                         "meta": {
                             'service_name': 'prometheus',
                             'ports': [9095],
-                            'ip': None,
+                            'ip': '1.2.3.1',
                             'deployed_by': [],
                             'rank': None,
                             'rank_generation': None,
@@ -755,6 +758,7 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                             },
                             'retention_time': '15d',
                             'retention_size': '0',
+                            'ip_to_bind_to': '1.2.3.1',
                         },
                     }),
                 )
@@ -929,6 +933,7 @@ def gen_cert(host, addr):
                             },
                             'retention_time': '15d',
                             'retention_size': '0',
+                            'ip_to_bind_to': '',
                             'web_config': '/etc/prometheus/web.yml',
                         },
                     }),
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 6808567500f8..d5a59c9b1215 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1885,6 +1885,7 @@ def __init__(self,
                  preview_only: bool = False,
                  config: Optional[Dict[str, str]] = None,
                  networks: Optional[List[str]] = None,
+                 only_bind_port_on_networks: bool = False,
                  port: Optional[int] = None,
                  retention_time: Optional[str] = None,
                  retention_size: Optional[str] = None,
@@ -1902,6 +1903,7 @@ def __init__(self,
 
         self.retention_time = retention_time.strip() if retention_time else None
         self.retention_size = retention_size.strip() if retention_size else None
+        self.only_bind_port_on_networks = only_bind_port_on_networks
 
     def validate(self) -> None:
         super(PrometheusSpec, self).validate()

From 106f34ba31c82dd87f4c3f9ad82d8ace81e6c689 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 14 Feb 2024 11:28:11 -0500
Subject: [PATCH 2081/2492] mgr/cephadm: fix placement with label and host
 pattern

Previously, when both the label and host pattern were
provided, only the label was actually used for the placement

Fixes: https://tracker.ceph.com/issues/64428

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/schedule.py                |  2 ++
 src/pybind/mgr/cephadm/tests/test_scheduling.py   | 11 +++++++++++
 src/python-common/ceph/deployment/service_spec.py |  5 +++--
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py
index 6666d761ebcf..98d2fe99897e 100644
--- a/src/pybind/mgr/cephadm/schedule.py
+++ b/src/pybind/mgr/cephadm/schedule.py
@@ -413,6 +413,8 @@ def get_candidates(self) -> List[DaemonPlacement]:
                                 hostname=x.hostname, ports=self.ports_start)
                 for x in self.hosts_by_label(self.spec.placement.label)
             ]
+            if self.spec.placement.host_pattern:
+                ls = [h for h in ls if h.hostname in self.spec.placement.filter_matching_hostspecs(self.hosts)]
         elif self.spec.placement.host_pattern:
             ls = [
                 DaemonPlacement(daemon_type=self.primary_daemon_type,
diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py
index b307cd9d34dd..f445ed6f0933 100644
--- a/src/pybind/mgr/cephadm/tests/test_scheduling.py
+++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py
@@ -637,6 +637,17 @@ class NodeAssignmentTest(NamedTuple):
              'rgw:host2(*:81)', 'rgw:host3(*:81)'],
             ['rgw.c']
         ),
+        # label + host pattern
+        # Note all hosts will get the "foo" label, we are checking
+        # that it also filters on the host pattern when label is provided
+        NodeAssignmentTest(
+            'mgr',
+            PlacementSpec(label='foo', host_pattern='mgr*'),
+            'mgr1 mgr2 osd1'.split(),
+            [],
+            None, None,
+            ['mgr:mgr1', 'mgr:mgr2'], ['mgr:mgr1', 'mgr:mgr2'], []
+        ),
         # cephadm.py teuth case
         NodeAssignmentTest(
             'mgr',
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index f6d290f07188..4da4f46c1fad 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -306,8 +306,9 @@ def filter_matching_hostspecs(self, hostspecs: Iterable[HostSpec]) -> List[str]:
             all_hosts = [hs.hostname for hs in hostspecs]
             return [h.hostname for h in self.hosts if h.hostname in all_hosts]
         if self.label:
-            return [hs.hostname for hs in hostspecs if self.label in hs.labels]
-        all_hosts = [hs.hostname for hs in hostspecs]
+            all_hosts = [hs.hostname for hs in hostspecs if self.label in hs.labels]
+        else:
+            all_hosts = [hs.hostname for hs in hostspecs]
         if self.host_pattern:
             return self.host_pattern.filter_hosts(all_hosts)
         return all_hosts

From 96e8850ff3b6661c86eb361051282c8d66ca2031 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Mon, 12 Jun 2023 08:00:40 +0000
Subject: [PATCH 2082/2492] rbd-wnbd: introduce RbdMapping class

We're moving most of the WNBD mapping handling to a separate
class called RbdMapping. This simplifies cleanup and makes it
easier to reuse.

The WnbdHandler class covers WNBD specific operations and IO
callbacks while the RbdMapping wrapper will take care of RBD
operations.

A subsequent change will make use of it while switching from
one process per mapping to a single process per host.

While at it, we're also moving the rbd-wnbd config helpers
to separate files.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/tools/rbd_wnbd/CMakeLists.txt        |   1 +
 src/tools/rbd_wnbd/rbd_mapping.cc        | 180 ++++++++++++++
 src/tools/rbd_wnbd/rbd_mapping.h         |  88 +++++++
 src/tools/rbd_wnbd/rbd_mapping_config.cc | 119 +++++++++
 src/tools/rbd_wnbd/rbd_mapping_config.h  |  77 ++++++
 src/tools/rbd_wnbd/rbd_wnbd.cc           | 294 +----------------------
 src/tools/rbd_wnbd/rbd_wnbd.h            |  61 +----
 src/tools/rbd_wnbd/wnbd_handler.h        |   2 -
 8 files changed, 479 insertions(+), 343 deletions(-)
 create mode 100644 src/tools/rbd_wnbd/rbd_mapping.cc
 create mode 100644 src/tools/rbd_wnbd/rbd_mapping.h
 create mode 100644 src/tools/rbd_wnbd/rbd_mapping_config.cc
 create mode 100644 src/tools/rbd_wnbd/rbd_mapping_config.h

diff --git a/src/tools/rbd_wnbd/CMakeLists.txt b/src/tools/rbd_wnbd/CMakeLists.txt
index ff09cd80a152..e757c71601eb 100644
--- a/src/tools/rbd_wnbd/CMakeLists.txt
+++ b/src/tools/rbd_wnbd/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_executable(
     rbd-wnbd
+    rbd_mapping.cc rbd_mapping_config.cc
     rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc
     ../../common/win32/code_page.rc)
 set_target_properties(
diff --git a/src/tools/rbd_wnbd/rbd_mapping.cc b/src/tools/rbd_wnbd/rbd_mapping.cc
new file mode 100644
index 000000000000..db136dbf6b92
--- /dev/null
+++ b/src/tools/rbd_wnbd/rbd_mapping.cc
@@ -0,0 +1,180 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "rbd_mapping.h"
+
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+
+#include "global/global_init.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd-wnbd: "
+
+
+int RbdMapping::init()
+{
+  librbd::image_info_t info;
+
+  int r = rados.ioctx_create(cfg.poolname.c_str(), io_ctx);
+  if (r < 0) {
+    derr << "rbd-wnbd: couldn't create IO context: " << cpp_strerror(r)
+         << dendl;
+    return r;
+  }
+
+  io_ctx.set_namespace(cfg.nsname);
+
+  r = rbd.open(io_ctx, image, cfg.imgname.c_str());
+  if (r < 0) {
+    derr << "rbd-wnbd: couldn't open rbd image: " << cpp_strerror(r)
+         << dendl;
+    return r;
+  }
+
+  if (cfg.exclusive) {
+    r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE);
+    if (r < 0) {
+      derr << "rbd-wnbd: failed to acquire exclusive lock: " << cpp_strerror(r)
+           << dendl;
+      return r;
+    }
+  }
+
+  if (!cfg.snapname.empty()) {
+    r = image.snap_set(cfg.snapname.c_str());
+    if (r < 0) {
+      derr << "rbd-wnbd: couldn't use snapshot: " << cpp_strerror(r)
+         << dendl;
+      return r;
+    }
+  }
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  initial_image_size = info.size;
+
+  // We're storing mapping details in the registry even for non-persistent
+  // mappings. This allows us to easily retrieve mapping details such
+  // as the rbd pool or admin socket path.
+  // We're cleaning up the registry entry when the non-persistent mapping
+  // gets disconnected or when the ceph service restarts.
+  r = save_config_to_registry(&cfg, command_line);
+  if (r < 0)
+    return r;
+
+  handler = new WnbdHandler(image, cfg.devpath,
+                            info.size / RBD_WNBD_BLKSIZE,
+                            RBD_WNBD_BLKSIZE,
+                            !cfg.snapname.empty() || cfg.readonly,
+                            g_conf().get_val<bool>("rbd_cache"),
+                            cfg.io_req_workers,
+                            cfg.io_reply_workers);
+  return 0;
+}
+
+void RbdMapping::shutdown()
+{
+  std::unique_lock l{shutdown_lock};
+
+  int r = 0;
+  if (!cfg.persistent) {
+    dout(5) << __func__ << ": cleaning up non-persistent mapping: "
+            << cfg.devpath << dendl;
+    r = remove_config_from_registry(&cfg);
+    if (r) {
+      derr << __func__ << ": could not clean up non-persistent mapping: "
+           << cfg.devpath << ". Error: " << cpp_strerror(r) << dendl;
+    }
+  }
+
+  if (watch_ctx) {
+    r = image.update_unwatch(watch_handle);
+    if (r < 0) {
+      derr << __func__ << ": update_unwatch failed with error: "
+           << cpp_strerror(r) << dendl;
+    }
+    delete watch_ctx;
+    watch_ctx = nullptr;
+  }
+
+  if (handler) {
+    handler->shutdown();
+    delete handler;
+    handler = nullptr;
+  }
+
+  image.close();
+  io_ctx.close();
+  rados.shutdown();
+}
+
+int RbdMapping::start()
+{
+  int r = init();
+  if (r < 0) {
+    return r;
+  }
+
+  r = handler->start();
+  if (r) {
+    return r == ERROR_ALREADY_EXISTS ? -EEXIST : -EINVAL;
+  }
+
+  watch_ctx = new WNBDWatchCtx(io_ctx, handler, image, initial_image_size);
+  r = image.update_watch(watch_ctx, &watch_handle);
+  if (r < 0) {
+    derr << __func__ << ": update_watch failed with error: "
+         << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  // We're informing the parent processes that the initialization
+  // was successful.
+  int err = 0;
+  if (!cfg.parent_pipe.empty()) {
+    HANDLE parent_pipe_handle = CreateFile(
+      cfg.parent_pipe.c_str(), GENERIC_WRITE, 0, NULL,
+      OPEN_EXISTING, 0, NULL);
+    if (parent_pipe_handle == INVALID_HANDLE_VALUE) {
+      err = GetLastError();
+      derr << "Could not open parent pipe: " << win32_strerror(err) << dendl;
+    } else if (!WriteFile(parent_pipe_handle, "a", 1, NULL, NULL)) {
+      // TODO: consider exiting in this case. The parent didn't wait for us,
+      // maybe it was killed after a timeout.
+      err = GetLastError();
+      derr << "Failed to communicate with the parent: "
+           << win32_strerror(err) << dendl;
+    } else {
+      dout(5) << __func__ << ": submitted parent notification." << dendl;
+    }
+
+    if (parent_pipe_handle != INVALID_HANDLE_VALUE)
+      CloseHandle(parent_pipe_handle);
+
+    global_init_postfork_finish(g_ceph_context);
+  }
+
+  return 0;
+}
+
+int RbdMapping::wait() {
+  if (handler) {
+    return handler->wait();
+  }
+  return 0;
+}
diff --git a/src/tools/rbd_wnbd/rbd_mapping.h b/src/tools/rbd_wnbd/rbd_mapping.h
new file mode 100644
index 000000000000..ce96beeec9e3
--- /dev/null
+++ b/src/tools/rbd_wnbd/rbd_mapping.h
@@ -0,0 +1,88 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include "rbd_mapping_config.h"
+#include "wnbd_handler.h"
+
+class WNBDWatchCtx : public librbd::UpdateWatchCtx
+{
+private:
+  librados::IoCtx &io_ctx;
+  WnbdHandler* handler;
+  librbd::Image &image;
+  uint64_t size;
+public:
+  WNBDWatchCtx(librados::IoCtx& io_ctx, WnbdHandler* handler,
+               librbd::Image& image, uint64_t size)
+    : io_ctx(io_ctx)
+    , handler(handler)
+    , image(image)
+    , size(size)
+  {
+  }
+
+  ~WNBDWatchCtx() override {}
+
+  void handle_notify() override
+  {
+    uint64_t new_size;
+
+    if (image.size(&new_size) == 0 && new_size != size &&
+        handler->resize(new_size) == 0) {
+      size = new_size;
+    }
+  }
+};
+
+
+class RbdMapping
+{
+private:
+  Config cfg;
+  // We're sharing the rados object across mappings in order to
+  // reuse the OSD connections.
+  librados::Rados &rados;
+  std::string command_line;
+
+  librbd::RBD rbd;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  uint64_t initial_image_size;
+
+  WnbdHandler* handler = nullptr;
+  uint64_t watch_handle;
+  WNBDWatchCtx* watch_ctx = nullptr;
+
+  ceph::mutex shutdown_lock = ceph::make_mutex("RbdMapping::ShutdownLock");
+
+  int init();
+  void shutdown();
+
+public:
+  RbdMapping(Config& _cfg, librados::Rados& _rados,
+             std::string _command_line)
+    : cfg(_cfg)
+    , rados(_rados)
+    , command_line(_command_line)
+  {
+  }
+
+  ~RbdMapping()
+  {
+      shutdown();
+  }
+
+  int start();
+  int wait();
+};
diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.cc b/src/tools/rbd_wnbd/rbd_mapping_config.cc
new file mode 100644
index 000000000000..c78920ea8d32
--- /dev/null
+++ b/src/tools/rbd_wnbd/rbd_mapping_config.cc
@@ -0,0 +1,119 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 SUSE LINUX GmbH
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "rbd_mapping_config.h"
+
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/win32/registry.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd-wnbd: "
+
+int construct_devpath_if_missing(Config* cfg)
+{
+  // Windows doesn't allow us to request specific disk paths when mapping an
+  // image. This will just be used by rbd-wnbd and wnbd as an identifier.
+  if (cfg->devpath.empty()) {
+    if (cfg->imgname.empty()) {
+      derr << "Missing image name." << dendl;
+      return -EINVAL;
+    }
+
+    if (!cfg->poolname.empty()) {
+      cfg->devpath += cfg->poolname;
+      cfg->devpath += '/';
+    }
+    if (!cfg->nsname.empty()) {
+      cfg->devpath += cfg->nsname;
+      cfg->devpath += '/';
+    }
+
+    cfg->devpath += cfg->imgname;
+
+    if (!cfg->snapname.empty()) {
+      cfg->devpath += '@';
+      cfg->devpath += cfg->snapname;
+    }
+  }
+
+  return 0;
+}
+
+int save_config_to_registry(Config* cfg, std::string command_line)
+{
+  std::string strKey{ SERVICE_REG_KEY };
+  strKey.append("\\");
+  strKey.append(cfg->devpath);
+  auto reg_key = RegistryKey(
+    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), true);
+  if (!reg_key.hKey) {
+      return -EINVAL;
+  }
+
+  int ret_val = 0;
+  // Registry writes are immediately available to other processes.
+  // Still, we'll do a flush to ensure that the mapping can be
+  // recreated after a system crash.
+  if (reg_key.set("pid", getpid()) ||
+      reg_key.set("devpath", cfg->devpath) ||
+      reg_key.set("poolname", cfg->poolname) ||
+      reg_key.set("nsname", cfg->nsname) ||
+      reg_key.set("imgname", cfg->imgname) ||
+      reg_key.set("snapname", cfg->snapname) ||
+      reg_key.set("command_line", command_line) ||
+      reg_key.set("persistent", cfg->persistent) ||
+      reg_key.set("admin_sock_path", g_conf()->admin_socket) ||
+      reg_key.flush()) {
+    ret_val = -EINVAL;
+  }
+
+  return ret_val;
+}
+
+int remove_config_from_registry(Config* cfg)
+{
+  std::string strKey{ SERVICE_REG_KEY };
+  strKey.append("\\");
+  strKey.append(cfg->devpath);
+  return RegistryKey::remove(
+    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str());
+}
+
+int load_mapping_config_from_registry(std::string devpath, Config* cfg)
+{
+  std::string strKey{ SERVICE_REG_KEY };
+  strKey.append("\\");
+  strKey.append(devpath);
+  auto reg_key = RegistryKey(
+    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), false);
+  if (!reg_key.hKey) {
+    if (reg_key.missingKey)
+      return -ENOENT;
+    else
+      return -EINVAL;
+  }
+
+  reg_key.get("devpath", cfg->devpath);
+  reg_key.get("poolname", cfg->poolname);
+  reg_key.get("nsname", cfg->nsname);
+  reg_key.get("imgname", cfg->imgname);
+  reg_key.get("snapname", cfg->snapname);
+  reg_key.get("command_line", cfg->command_line);
+  reg_key.get("persistent", cfg->persistent);
+  reg_key.get("admin_sock_path", cfg->admin_sock_path);
+
+  return 0;
+}
diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.h b/src/tools/rbd_wnbd/rbd_mapping_config.h
new file mode 100644
index 000000000000..60e3fa20a45e
--- /dev/null
+++ b/src/tools/rbd_wnbd/rbd_mapping_config.h
@@ -0,0 +1,77 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 SUSE LINUX GmbH
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <string>
+
+#include <wnbd.h>
+
+#define SERVICE_REG_KEY "SYSTEM\\CurrentControlSet\\Services\\rbd-wnbd"
+
+#define DEFAULT_SERVICE_START_TIMEOUT 120
+#define DEFAULT_IMAGE_MAP_TIMEOUT 20
+#define DEFAULT_SERVICE_THREAD_COUNT 8
+#define DEFAULT_SOFT_REMOVE_TIMEOUT 15
+#define DEFAULT_IO_WORKER_COUNT 4
+
+#define RBD_WNBD_BLKSIZE 512UL
+
+struct Config {
+  bool exclusive = false;
+  bool readonly = false;
+
+  std::string parent_pipe;
+
+  std::string poolname;
+  std::string nsname;
+  std::string imgname;
+  std::string snapname;
+  std::string devpath;
+
+  std::string format;
+  bool pretty_format = false;
+
+  bool hard_disconnect = false;
+  int soft_disconnect_timeout = DEFAULT_SOFT_REMOVE_TIMEOUT;
+  bool hard_disconnect_fallback = true;
+
+  int service_start_timeout = DEFAULT_SERVICE_START_TIMEOUT;
+  int image_map_timeout = DEFAULT_IMAGE_MAP_TIMEOUT;
+  bool remap_failure_fatal = false;
+  bool adapter_monitoring_enabled = false;
+
+  // TODO: consider moving those fields to a separate structure. Those
+  // provide connection information without actually being configurable.
+  // The disk number is provided by Windows.
+  int disk_number = -1;
+  int pid = 0;
+  std::string serial_number;
+  bool active = false;
+  bool wnbd_mapped = false;
+  std::string command_line;
+  std::string admin_sock_path;
+
+  WnbdLogLevel wnbd_log_level = WnbdLogLevelInfo;
+  int io_req_workers = DEFAULT_IO_WORKER_COUNT;
+  int io_reply_workers = DEFAULT_IO_WORKER_COUNT;
+  int service_thread_count = DEFAULT_SERVICE_THREAD_COUNT;
+
+  // register the mapping, recreating it when the Ceph service starts.
+  bool persistent = true;
+};
+
+int construct_devpath_if_missing(Config* cfg);
+int save_config_to_registry(Config* cfg, std::string command_line);
+int remove_config_from_registry(Config* cfg);
+int load_mapping_config_from_registry(std::string devpath, Config* cfg);
diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc
index 1946e83ff967..08cb7ca9359d 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.cc
+++ b/src/tools/rbd_wnbd/rbd_wnbd.cc
@@ -307,7 +307,7 @@ int send_map_request(std::string arguments) {
     &reply,
     sizeof(reply),
     &bytes_read,
-    DEFAULT_MAP_TIMEOUT_MS);
+    DEFAULT_IMAGE_MAP_TIMEOUT * 1000);
   if (!success) {
     DWORD err = GetLastError();
     derr << "Could not send device map request. "
@@ -519,79 +519,11 @@ BOOL WINAPI console_handler_routine(DWORD dwCtrlType)
   dout(0) << "Received control signal: " << dwCtrlType
           << ". Exiting." << dendl;
 
-  std::unique_lock l{shutdown_lock};
-  if (handler)
-    handler->shutdown();
+  // TODO: shutdown all mappings
 
   return true;
 }
 
-int save_config_to_registry(Config* cfg)
-{
-  std::string strKey{ SERVICE_REG_KEY };
-  strKey.append("\\");
-  strKey.append(cfg->devpath);
-  auto reg_key = RegistryKey(
-    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), true);
-  if (!reg_key.hKey) {
-      return -EINVAL;
-  }
-
-  int ret_val = 0;
-  // Registry writes are immediately available to other processes.
-  // Still, we'll do a flush to ensure that the mapping can be
-  // recreated after a system crash.
-  if (reg_key.set("pid", getpid()) ||
-      reg_key.set("devpath", cfg->devpath) ||
-      reg_key.set("poolname", cfg->poolname) ||
-      reg_key.set("nsname", cfg->nsname) ||
-      reg_key.set("imgname", cfg->imgname) ||
-      reg_key.set("snapname", cfg->snapname) ||
-      reg_key.set("command_line", get_cli_args()) ||
-      reg_key.set("persistent", cfg->persistent) ||
-      reg_key.set("admin_sock_path", g_conf()->admin_socket) ||
-      reg_key.flush()) {
-    ret_val = -EINVAL;
-  }
-
-  return ret_val;
-}
-
-int remove_config_from_registry(Config* cfg)
-{
-  std::string strKey{ SERVICE_REG_KEY };
-  strKey.append("\\");
-  strKey.append(cfg->devpath);
-  return RegistryKey::remove(
-    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str());
-}
-
-int load_mapping_config_from_registry(string devpath, Config* cfg)
-{
-  std::string strKey{ SERVICE_REG_KEY };
-  strKey.append("\\");
-  strKey.append(devpath);
-  auto reg_key = RegistryKey(
-    g_ceph_context, HKEY_LOCAL_MACHINE, strKey.c_str(), false);
-  if (!reg_key.hKey) {
-    if (reg_key.missingKey)
-      return -ENOENT;
-    else
-      return -EINVAL;
-  }
-
-  reg_key.get("devpath", cfg->devpath);
-  reg_key.get("poolname", cfg->poolname);
-  reg_key.get("nsname", cfg->nsname);
-  reg_key.get("imgname", cfg->imgname);
-  reg_key.get("snapname", cfg->snapname);
-  reg_key.get("command_line", cfg->command_line);
-  reg_key.get("persistent", cfg->persistent);
-  reg_key.get("admin_sock_path", cfg->admin_sock_path);
-
-  return 0;
-}
-
 int restart_registered_mappings(
   int worker_count,
   int total_timeout,
@@ -793,7 +725,8 @@ class RBDService : public ServiceBase {
           // TODO: use the configured service map timeout.
           // TODO: add ceph.conf options.
           return map_device_using_suprocess(
-            (char*)request->arguments, DEFAULT_MAP_TIMEOUT_MS);
+            (char*)request->arguments,
+            DEFAULT_IMAGE_MAP_TIMEOUT * 1000);
         default:
           dout(1) << "Received unsupported command: "
                   << request->command << dendl;
@@ -1042,35 +975,6 @@ class RBDService : public ServiceBase {
     }
 };
 
-class WNBDWatchCtx : public librbd::UpdateWatchCtx
-{
-private:
-  librados::IoCtx &io_ctx;
-  WnbdHandler* handler;
-  librbd::Image &image;
-  uint64_t size;
-public:
-  WNBDWatchCtx(librados::IoCtx& io_ctx, WnbdHandler* handler,
-               librbd::Image& image, uint64_t size)
-    : io_ctx(io_ctx)
-    , handler(handler)
-    , image(image)
-    , size(size)
-  { }
-
-  ~WNBDWatchCtx() override {}
-
-  void handle_notify() override
-  {
-    uint64_t new_size;
-
-    if (image.size(&new_size) == 0 && new_size != size &&
-        handler->resize(new_size) == 0) {
-      size = new_size;
-    }
-  }
-};
-
 static void usage()
 {
   const char* usage_str =R"(
@@ -1138,36 +1042,6 @@ Common options:
 
 static Command cmd = None;
 
-int construct_devpath_if_missing(Config* cfg)
-{
-  // Windows doesn't allow us to request specific disk paths when mapping an
-  // image. This will just be used by rbd-wnbd and wnbd as an identifier.
-  if (cfg->devpath.empty()) {
-    if (cfg->imgname.empty()) {
-      derr << "Missing image name." << dendl;
-      return -EINVAL;
-    }
-
-    if (!cfg->poolname.empty()) {
-      cfg->devpath += cfg->poolname;
-      cfg->devpath += '/';
-    }
-    if (!cfg->nsname.empty()) {
-      cfg->devpath += cfg->nsname;
-      cfg->devpath += '/';
-    }
-
-    cfg->devpath += cfg->imgname;
-
-    if (!cfg->snapname.empty()) {
-      cfg->devpath += '@';
-      cfg->devpath += cfg->snapname;
-    }
-  }
-
-  return 0;
-}
-
 boost::intrusive_ptr<CephContext> do_global_init(
   int argc, const char *argv[], Config *cfg)
 {
@@ -1233,16 +1107,6 @@ static int wait_mapped_disk(Config *cfg)
 
 static int do_map(Config *cfg)
 {
-  int r;
-
-  librados::Rados rados;
-  librbd::RBD rbd;
-  librados::IoCtx io_ctx;
-  librbd::Image image;
-  librbd::image_info_t info;
-  HANDLE parent_pipe_handle = INVALID_HANDLE_VALUE;
-  int err = 0;
-
   if (g_conf()->daemonize && cfg->parent_pipe.empty()) {
     r = send_map_request(get_cli_args());
     if (r < 0) {
@@ -1254,85 +1118,18 @@ static int do_map(Config *cfg)
 
   dout(0) << "Mapping RBD image: " << cfg->devpath << dendl;
 
-  r = rados.init_with_context(g_ceph_context);
+  librados::Rados rados;
+  int r = rados.init_with_context(g_ceph_context);
   if (r < 0) {
     derr << "rbd-wnbd: couldn't initialize rados: " << cpp_strerror(r)
          << dendl;
-    goto close_ret;
-  }
-
-  r = rados.connect();
-  if (r < 0) {
-    derr << "rbd-wnbd: couldn't connect to rados: " << cpp_strerror(r)
-         << dendl;
-    goto close_ret;
-  }
-
-  r = rados.ioctx_create(cfg->poolname.c_str(), io_ctx);
-  if (r < 0) {
-    derr << "rbd-wnbd: couldn't create IO context: " << cpp_strerror(r)
-         << dendl;
-    goto close_ret;
-  }
-
-  io_ctx.set_namespace(cfg->nsname);
-
-  r = rbd.open(io_ctx, image, cfg->imgname.c_str());
-  if (r < 0) {
-    derr << "rbd-wnbd: couldn't open rbd image: " << cpp_strerror(r)
-         << dendl;
-    goto close_ret;
-  }
-
-  if (cfg->exclusive) {
-    r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE);
-    if (r < 0) {
-      derr << "rbd-wnbd: failed to acquire exclusive lock: " << cpp_strerror(r)
-           << dendl;
-      goto close_ret;
-    }
-  }
-
-  if (!cfg->snapname.empty()) {
-    r = image.snap_set(cfg->snapname.c_str());
-    if (r < 0) {
-      derr << "rbd-wnbd: couldn't use snapshot: " << cpp_strerror(r)
-         << dendl;
-      goto close_ret;
-    }
-  }
-
-  r = image.stat(info, sizeof(info));
-  if (r < 0)
-    goto close_ret;
-
-  if (info.size > _UI64_MAX) {
-    r = -EFBIG;
-    derr << "rbd-wnbd: image is too large (" << byte_u_t(info.size)
-         << ", max is " << byte_u_t(_UI64_MAX) << ")" << dendl;
-    goto close_ret;
+    return r;
   }
 
-  // We're storing mapping details in the registry even for non-persistent
-  // mappings. This allows us to easily retrieve mapping details such
-  // as the rbd pool or admin socket path.
-  // We're cleaning up the registry entry when the non-persistent mapping
-  // gets disconnected or when the ceph service restarts.
-  r = save_config_to_registry(cfg);
-  if (r < 0)
-    goto close_ret;
-
-  handler = new WnbdHandler(image, cfg->devpath,
-                            info.size / RBD_WNBD_BLKSIZE,
-                            RBD_WNBD_BLKSIZE,
-                            !cfg->snapname.empty() || cfg->readonly,
-                            g_conf().get_val<bool>("rbd_cache"),
-                            cfg->io_req_workers,
-                            cfg->io_reply_workers);
-  r = handler->start();
+  RbdMapping rbd_mapping(*cfg, rados, get_cli_args());
+  r = rbd_mapping.start();
   if (r) {
-    r = r == ERROR_ALREADY_EXISTS ? -EEXIST : -EINVAL;
-    goto close_ret;
+    return r;
   }
 
   // TODO: consider substracting the time it took to perform the
@@ -1342,75 +1139,8 @@ static int do_map(Config *cfg)
     goto close_ret;
   }
 
-  // We're informing the parent processes that the initialization
-  // was successful.
-  if (!cfg->parent_pipe.empty()) {
-    parent_pipe_handle = CreateFile(
-      cfg->parent_pipe.c_str(), GENERIC_WRITE, 0, NULL,
-      OPEN_EXISTING, 0, NULL);
-    if (parent_pipe_handle == INVALID_HANDLE_VALUE) {
-      derr << "Could not open parent pipe: " << win32_strerror(err) << dendl;
-    } else if (!WriteFile(parent_pipe_handle, "a", 1, NULL, NULL)) {
-      // TODO: consider exiting in this case. The parent didn't wait for us,
-      // maybe it was killed after a timeout.
-      err = GetLastError();
-      derr << "Failed to communicate with the parent: "
-           << win32_strerror(err) << dendl;
-    } else {
-      dout(5) << __func__ << ": submitted parent notification." << dendl;
-    }
-
-    if (parent_pipe_handle != INVALID_HANDLE_VALUE)
-      CloseHandle(parent_pipe_handle);
-
-    global_init_postfork_finish(g_ceph_context);
-  }
-
-  {
-    uint64_t watch_handle;
-    WNBDWatchCtx watch_ctx(io_ctx, handler, image, info.size);
-    r = image.update_watch(&watch_ctx, &watch_handle);
-    if (r < 0) {
-      derr << __func__ << ": update_watch failed with error: "
-           << cpp_strerror(r) << dendl;
-
-      handler->shutdown();
-      goto close_ret;
-    }
-
-    handler->wait();
-
-    r = image.update_unwatch(watch_handle);
-    if (r < 0)
-      derr << __func__ << ": update_unwatch failed with error: "
-           << cpp_strerror(r) << dendl;
-
-    handler->shutdown();
-  }
-
-close_ret:
-  // The registry record shouldn't be removed for (already) running mappings.
-  if (!cfg->persistent) {
-    dout(5) << __func__ << ": cleaning up non-persistent mapping: "
-            << cfg->devpath << dendl;
-    r = remove_config_from_registry(cfg);
-    if (r) {
-      derr << __func__ << ": could not clean up non-persistent mapping: "
-           << cfg->devpath << dendl;
-    }
-  }
-
-  std::unique_lock l{shutdown_lock};
-
-  image.close();
-  io_ctx.close();
-  rados.shutdown();
-  if (handler) {
-    delete handler;
-    handler = nullptr;
-  }
-
-  return r;
+  dout(0) << "Successfully mapped RBD image: " << cfg->devpath << dendl;
+  return rbd_mapping.wait();
 }
 
 static int do_unmap(Config *cfg, bool unregister)
diff --git a/src/tools/rbd_wnbd/rbd_wnbd.h b/src/tools/rbd_wnbd/rbd_wnbd.h
index ac24e9de4aaa..ba6280031a36 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.h
+++ b/src/tools/rbd_wnbd/rbd_wnbd.h
@@ -20,19 +20,13 @@
 #include "include/compat.h"
 #include "common/win32/registry.h"
 
-#include "wnbd_handler.h"
+#include "rbd_mapping_config.h"
+#include "rbd_mapping.h"
 
-#define SERVICE_REG_KEY "SYSTEM\\CurrentControlSet\\Services\\rbd-wnbd"
 #define SERVICE_PIPE_NAME "\\\\.\\pipe\\rbd-wnbd"
 #define SERVICE_PIPE_TIMEOUT_MS 5000
 #define SERVICE_PIPE_BUFFSZ 4096
 
-#define DEFAULT_MAP_TIMEOUT_MS 30000
-
-#define RBD_WNBD_BLKSIZE 512UL
-
-#define DEFAULT_SERVICE_START_TIMEOUT 120
-#define DEFAULT_IMAGE_MAP_TIMEOUT 20
 #define DISK_STATUS_POLLING_INTERVAL_MS 500
 
 #define HELP_INFO 1
@@ -41,52 +35,6 @@
 #define WNBD_STATUS_ACTIVE "active"
 #define WNBD_STATUS_INACTIVE "inactive"
 
-#define DEFAULT_SERVICE_THREAD_COUNT 8
-
-struct Config {
-  bool exclusive = false;
-  bool readonly = false;
-
-  std::string parent_pipe;
-
-  std::string poolname;
-  std::string nsname;
-  std::string imgname;
-  std::string snapname;
-  std::string devpath;
-
-  std::string format;
-  bool pretty_format = false;
-
-  bool hard_disconnect = false;
-  int soft_disconnect_timeout = DEFAULT_SOFT_REMOVE_TIMEOUT;
-  bool hard_disconnect_fallback = true;
-
-  int service_start_timeout = DEFAULT_SERVICE_START_TIMEOUT;
-  int image_map_timeout = DEFAULT_IMAGE_MAP_TIMEOUT;
-  bool remap_failure_fatal = false;
-  bool adapter_monitoring_enabled = false;
-
-  // TODO: consider moving those fields to a separate structure. Those
-  // provide connection information without actually being configurable.
-  // The disk number is provided by Windows.
-  int disk_number = -1;
-  int pid = 0;
-  std::string serial_number;
-  bool active = false;
-  bool wnbd_mapped = false;
-  std::string command_line;
-  std::string admin_sock_path;
-
-  WnbdLogLevel wnbd_log_level = WnbdLogLevelInfo;
-  int io_req_workers = DEFAULT_IO_WORKER_COUNT;
-  int io_reply_workers = DEFAULT_IO_WORKER_COUNT;
-  int service_thread_count = DEFAULT_SERVICE_THREAD_COUNT;
-
-  // register the mapping, recreating it when the Ceph service starts.
-  bool persistent = true;
-};
-
 enum Command {
   None,
   Connect,
@@ -118,11 +66,6 @@ int restart_registered_mappings(
   int worker_count, int total_timeout, int image_map_timeout);
 int map_device_using_suprocess(std::string command_line);
 
-int construct_devpath_if_missing(Config* cfg);
-int save_config_to_registry(Config* cfg);
-int remove_config_from_registry(Config* cfg);
-int load_mapping_config_from_registry(std::string devpath, Config* cfg);
-
 BOOL WINAPI console_handler_routine(DWORD dwCtrlType);
 
 static int parse_args(std::vector<const char*>& args,
diff --git a/src/tools/rbd_wnbd/wnbd_handler.h b/src/tools/rbd_wnbd/wnbd_handler.h
index c1ab5676bb85..f3b3e4341b01 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.h
+++ b/src/tools/rbd_wnbd/wnbd_handler.h
@@ -27,8 +27,6 @@
 // TODO: make this configurable.
 #define RBD_WNBD_MAX_TRANSFER 2 * 1024 * 1024
 #define SOFT_REMOVE_RETRY_INTERVAL 2
-#define DEFAULT_SOFT_REMOVE_TIMEOUT 15
-#define DEFAULT_IO_WORKER_COUNT 4
 
 // Not defined by mingw.
 #ifndef SCSI_ADSENSE_UNRECOVERED_ERROR

From 661c55002db8bc91037d8150278dacb9a1cfe46b Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Mon, 12 Jun 2023 13:16:39 +0000
Subject: [PATCH 2083/2492] rbd-wnbd: use one daemon process per host

We're currently using one rbd-wnbd process per image mapping.
Since OSD connections aren't shared across those processes,
we end up with an excessive amount of TCP sessions, potentially
exceeding Windows limits:
https://ask.cloudbase.it/question/3598/ceph-for-windows-tcp-session-count/

In order to improve rbd-wnbd's scalability, we're going to use
a single process per host (unless "-f" is passed when mapping the
image, in which case the daemon will run as part of the same
process). This allows OSD sessions to be shared across image
mappings.

Another advantage is that the "ceph-rbd" service starts faster,
especially when having a large number of image mappings.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/tools/rbd_wnbd/CMakeLists.txt        |   1 +
 src/tools/rbd_wnbd/rados_client_cache.cc |  91 +++++++
 src/tools/rbd_wnbd/rados_client_cache.h  |  39 +++
 src/tools/rbd_wnbd/rbd_mapping.cc        | 151 +++++++++--
 src/tools/rbd_wnbd/rbd_mapping.h         |  56 +++-
 src/tools/rbd_wnbd/rbd_mapping_config.cc |   4 +-
 src/tools/rbd_wnbd/rbd_mapping_config.h  |   5 +-
 src/tools/rbd_wnbd/rbd_wnbd.cc           | 309 ++++++-----------------
 src/tools/rbd_wnbd/rbd_wnbd.h            |   6 +-
 src/tools/rbd_wnbd/wnbd_handler.cc       |   9 +-
 10 files changed, 388 insertions(+), 283 deletions(-)
 create mode 100644 src/tools/rbd_wnbd/rados_client_cache.cc
 create mode 100644 src/tools/rbd_wnbd/rados_client_cache.h

diff --git a/src/tools/rbd_wnbd/CMakeLists.txt b/src/tools/rbd_wnbd/CMakeLists.txt
index e757c71601eb..12a54986fba7 100644
--- a/src/tools/rbd_wnbd/CMakeLists.txt
+++ b/src/tools/rbd_wnbd/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_executable(
     rbd-wnbd
+    rados_client_cache.cc
     rbd_mapping.cc rbd_mapping_config.cc
     rbd_wnbd.cc wnbd_handler.cc wnbd_wmi.cc
     ../../common/win32/code_page.rc)
diff --git a/src/tools/rbd_wnbd/rados_client_cache.cc b/src/tools/rbd_wnbd/rados_client_cache.cc
new file mode 100644
index 000000000000..4c1acabde7bc
--- /dev/null
+++ b/src/tools/rbd_wnbd/rados_client_cache.cc
@@ -0,0 +1,91 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "rados_client_cache.h"
+
+#include "common/errno.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd-wnbd: "
+
+std::shared_ptr<librados::Rados> RadosClientCache::init_client(
+  std::string& entity_name, std::string& cluster_name)
+{
+  auto rados = std::make_shared<librados::Rados>();
+
+  int r = rados->init2(entity_name.c_str(), cluster_name.c_str(), 0);
+  if (r < 0) {
+    derr << "couldn't initialize rados: " << cpp_strerror(r)
+         << dendl;
+    return std::shared_ptr<librados::Rados>();
+  }
+
+  r = rados->conf_read_file(nullptr);
+  if (r < 0) {
+    derr << "couldn't read conf file: " << cpp_strerror(r)
+         << dendl;
+    return std::shared_ptr<librados::Rados>();
+  }
+
+  r = rados->connect();
+  if (r < 0) {
+    derr << "couldn't establish rados connection: "
+         << cpp_strerror(r) << dendl;
+    return std::shared_ptr<librados::Rados>();
+  } else {
+    dout(1) << "successfully initialized rados connection" << dendl;
+  }
+
+  return rados;
+}
+
+std::shared_ptr<librados::Rados> RadosClientCache::get_client(
+  std::string& entity_name, std::string& cluster_name)
+{
+  std::unique_lock l{cache_lock};
+
+  remove_expired();
+
+  std::string key = entity_name + "@" + cluster_name;
+  auto cached_client_weak = cache.find(key);
+  if (cached_client_weak != cache.end()) {
+    if (auto cached_client = cached_client_weak->second.lock()) {
+      dout(1) << "reusing cached rados client: " << key << dendl;
+      return cached_client;
+    } else {
+      dout(5) << "cleaning up expired rados ref: "
+              << cached_client_weak->first << dendl;
+      cache.erase(cached_client_weak);
+    }
+  }
+
+  dout(1) << "creating new rados client: " << key << dendl;
+  auto client = init_client(entity_name, cluster_name);
+  cache.insert(std::pair{key, client});
+  return client;
+}
+
+void RadosClientCache::remove_expired()
+{
+  auto i = cache.begin();
+  while (i != cache.end()) {
+    if (i->second.expired()) {
+      dout(5) << "removing expired rados ref: "
+              << i->first << dendl;
+      i = cache.erase(i);
+      continue;
+    }
+    i++;
+  }
+}
diff --git a/src/tools/rbd_wnbd/rados_client_cache.h b/src/tools/rbd_wnbd/rados_client_cache.h
new file mode 100644
index 000000000000..15841b0d3629
--- /dev/null
+++ b/src/tools/rbd_wnbd/rados_client_cache.h
@@ -0,0 +1,39 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Cloudbase Solutions
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include "common/debug.h"
+#include "common/dout.h"
+
+#include "global/global_init.h"
+
+#include "include/rados/librados.hpp"
+
+// In order to re-use OSD connections, we're caching one rados client
+// per cluster.
+class RadosClientCache
+{
+private:
+  std::map<std::string, std::weak_ptr<librados::Rados>> cache;
+  ceph::mutex cache_lock = ceph::make_mutex("RadosClientCache::MapLock");
+
+  // Remove deleted objects from the map.
+  void remove_expired();
+
+  std::shared_ptr<librados::Rados> init_client(
+    std::string& entity_name, std::string& cluster_name);
+
+public:
+  std::shared_ptr<librados::Rados> get_client(
+    std::string& entity_name, std::string& cluster_name);
+};
diff --git a/src/tools/rbd_wnbd/rbd_mapping.cc b/src/tools/rbd_wnbd/rbd_mapping.cc
index db136dbf6b92..b2d7cff93bc0 100644
--- a/src/tools/rbd_wnbd/rbd_mapping.cc
+++ b/src/tools/rbd_wnbd/rbd_mapping.cc
@@ -23,14 +23,22 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "rbd-wnbd: "
 
+#define DISK_STATUS_POLLING_INTERVAL_MS 500
+
 
 int RbdMapping::init()
 {
   librbd::image_info_t info;
 
-  int r = rados.ioctx_create(cfg.poolname.c_str(), io_ctx);
+  rados = client_cache.get_client(cfg.entity_name, cfg.cluster_name);
+  if (!rados) {
+    return -EINVAL;
+  }
+
+  int r = rados->ioctx_create(cfg.poolname.c_str(), io_ctx);
   if (r < 0) {
     derr << "rbd-wnbd: couldn't create IO context: " << cpp_strerror(r)
+         << ". Pool name: " << cfg.poolname
          << dendl;
     return r;
   }
@@ -73,7 +81,7 @@ int RbdMapping::init()
   // as the rbd pool or admin socket path.
   // We're cleaning up the registry entry when the non-persistent mapping
   // gets disconnected or when the ceph service restarts.
-  r = save_config_to_registry(&cfg, command_line);
+  r = save_config_to_registry(&cfg);
   if (r < 0)
     return r;
 
@@ -91,6 +99,8 @@ void RbdMapping::shutdown()
 {
   std::unique_lock l{shutdown_lock};
 
+  dout(5) << __func__ << ": removing RBD mapping: " << cfg.devpath << dendl;
+
   int r = 0;
   if (!cfg.persistent) {
     dout(5) << __func__ << ": cleaning up non-persistent mapping: "
@@ -120,21 +130,23 @@ void RbdMapping::shutdown()
 
   image.close();
   io_ctx.close();
-  rados.shutdown();
 }
 
 int RbdMapping::start()
 {
+  dout(10) << "initializing mapping" << dendl;
   int r = init();
   if (r < 0) {
     return r;
   }
 
+  dout(10) << "starting wnbd handler" << dendl;
   r = handler->start();
   if (r) {
     return r == ERROR_ALREADY_EXISTS ? -EEXIST : -EINVAL;
   }
 
+  dout(10) << "setting up watcher" << dendl;
   watch_ctx = new WNBDWatchCtx(io_ctx, handler, image, initial_image_size);
   r = image.update_watch(watch_ctx, &watch_handle);
   if (r < 0) {
@@ -143,38 +155,123 @@ int RbdMapping::start()
     return r;
   }
 
-  // We're informing the parent processes that the initialization
-  // was successful.
-  int err = 0;
-  if (!cfg.parent_pipe.empty()) {
-    HANDLE parent_pipe_handle = CreateFile(
-      cfg.parent_pipe.c_str(), GENERIC_WRITE, 0, NULL,
-      OPEN_EXISTING, 0, NULL);
-    if (parent_pipe_handle == INVALID_HANDLE_VALUE) {
-      err = GetLastError();
-      derr << "Could not open parent pipe: " << win32_strerror(err) << dendl;
-    } else if (!WriteFile(parent_pipe_handle, "a", 1, NULL, NULL)) {
-      // TODO: consider exiting in this case. The parent didn't wait for us,
-      // maybe it was killed after a timeout.
-      err = GetLastError();
-      derr << "Failed to communicate with the parent: "
-           << win32_strerror(err) << dendl;
-    } else {
-      dout(5) << __func__ << ": submitted parent notification." << dendl;
-    }
-
-    if (parent_pipe_handle != INVALID_HANDLE_VALUE)
-      CloseHandle(parent_pipe_handle);
+  // Wait for the mapped disk to become available.
+  r = wait_mapped_disk(cfg);
+  if (r < 0) {
+    return r;
+  }
 
-    global_init_postfork_finish(g_ceph_context);
+  if (disconnect_cbk) {
+    monitor_thread = std::thread([this]{
+      int ret = this->wait();
+      // Allow "this" to be destroyed by the disconnect callback.
+      this->monitor_thread.detach();
+      dout(5) << "finished waiting for: " << this->cfg.devpath
+              << ", ret: " << ret << dendl;
+      disconnect_cbk(this->cfg.devpath, ret);
+    });
   }
 
   return 0;
 }
 
-int RbdMapping::wait() {
+// Wait until the image gets disconnected.
+int RbdMapping::wait()
+{
   if (handler) {
     return handler->wait();
   }
   return 0;
 }
+
+RbdMapping::~RbdMapping()
+{
+  dout(10) << __func__ << ": cleaning up rbd mapping: "
+           << cfg.devpath << dendl;
+  shutdown();
+}
+
+// Wait for the mapped disk to become available.
+int wait_mapped_disk(Config& cfg)
+{
+  DWORD status = WnbdPollDiskNumber(
+    cfg.devpath.c_str(),
+    TRUE, // ExpectMapped
+    TRUE, // TryOpen
+    cfg.image_map_timeout * 1000,
+    DISK_STATUS_POLLING_INTERVAL_MS,
+    (PDWORD) &cfg.disk_number);
+  if (status) {
+    derr << "WNBD disk unavailable, error: "
+         << win32_strerror(status) << dendl;
+    return -EINVAL;
+  }
+  dout(0) << "Successfully mapped image: " << cfg.devpath
+          << ". Windows disk path: "
+          << "\\\\.\\PhysicalDrive" + std::to_string(cfg.disk_number)
+          << dendl;
+  return 0;
+}
+
+int RbdMappingDispatcher::create(Config& cfg)
+{
+  if (cfg.devpath.empty()) {
+    derr << "missing device identifier" << dendl;
+    return -EINVAL;
+  }
+
+  if (get_mapping(cfg.devpath)) {
+    derr << "already mapped: " << cfg.devpath << dendl;
+    return -EEXIST;
+  }
+
+  auto rbd_mapping = std::make_shared<RbdMapping>(
+    cfg, client_cache,
+    std::bind(
+      &RbdMappingDispatcher::disconnect_cbk,
+      this,
+      std::placeholders::_1,
+      std::placeholders::_2));
+
+  int r = rbd_mapping.get()->start();
+  if (!r) {
+    std::unique_lock l{map_mutex};
+    mappings.insert(std::make_pair(cfg.devpath, rbd_mapping));
+  }
+  return r;
+}
+
+std::shared_ptr<RbdMapping> RbdMappingDispatcher::get_mapping(
+  std::string& devpath)
+{
+  std::unique_lock l{map_mutex};
+
+  auto mapping_it = mappings.find(devpath);
+  if (mapping_it == mappings.end()) {
+    // not found
+    return std::shared_ptr<RbdMapping>();
+  } else {
+    return mapping_it->second;
+  }
+}
+
+void RbdMappingDispatcher::disconnect_cbk(std::string devpath, int ret)
+{
+  dout(10) << "RbdMappingDispatcher: cleaning up stopped mapping" << dendl;
+  if (ret) {
+    derr << "rbd mapping wait error: " << ret
+         << ", allowing cleanup to proceed"
+         << dendl;
+  }
+
+  auto mapping = get_mapping(devpath);
+  if (mapping) {
+    // This step can be fairly time consuming, especially when
+    // cumulated. For this reason, we'll ensure that multiple mappings
+    // can be cleaned up simultaneously.
+    mapping->shutdown();
+
+    std::unique_lock l{map_mutex};
+    mappings.erase(devpath);
+  }
+}
diff --git a/src/tools/rbd_wnbd/rbd_mapping.h b/src/tools/rbd_wnbd/rbd_mapping.h
index ce96beeec9e3..52404ed503f2 100644
--- a/src/tools/rbd_wnbd/rbd_mapping.h
+++ b/src/tools/rbd_wnbd/rbd_mapping.h
@@ -12,6 +12,7 @@
 
 #pragma once
 
+#include "rados_client_cache.h"
 #include "rbd_mapping_config.h"
 #include "wnbd_handler.h"
 
@@ -45,6 +46,7 @@ class WNBDWatchCtx : public librbd::UpdateWatchCtx
   }
 };
 
+typedef std::function<void(std::string devpath, int ret)> disconnect_cbk_t;
 
 class RbdMapping
 {
@@ -52,8 +54,8 @@ class RbdMapping
   Config cfg;
   // We're sharing the rados object across mappings in order to
   // reuse the OSD connections.
-  librados::Rados &rados;
-  std::string command_line;
+  RadosClientCache& client_cache;
+  std::shared_ptr<librados::Rados> rados;
 
   librbd::RBD rbd;
   librados::IoCtx io_ctx;
@@ -63,26 +65,54 @@ class RbdMapping
   WnbdHandler* handler = nullptr;
   uint64_t watch_handle;
   WNBDWatchCtx* watch_ctx = nullptr;
+  disconnect_cbk_t disconnect_cbk;
 
   ceph::mutex shutdown_lock = ceph::make_mutex("RbdMapping::ShutdownLock");
+  std::thread monitor_thread;
 
   int init();
-  void shutdown();
 
 public:
-  RbdMapping(Config& _cfg, librados::Rados& _rados,
-             std::string _command_line)
+  RbdMapping(Config& _cfg,
+             RadosClientCache& _client_cache)
     : cfg(_cfg)
-    , rados(_rados)
-    , command_line(_command_line)
-  {
-  }
+    , client_cache(_client_cache)
+  {}
 
-  ~RbdMapping()
-  {
-      shutdown();
-  }
+  RbdMapping(Config& _cfg,
+             RadosClientCache& _client_cache,
+             disconnect_cbk_t _disconnect_cbk)
+    : cfg(_cfg)
+    , client_cache(_client_cache)
+    , disconnect_cbk(_disconnect_cbk)
+  {}
+
+  ~RbdMapping();
 
   int start();
+  // Wait until the image gets disconnected.
   int wait();
+  void shutdown();
+};
+
+// Wait for the mapped disk to become available.
+int wait_mapped_disk(Config& cfg);
+
+class RbdMappingDispatcher
+{
+private:
+  RadosClientCache& client_cache;
+
+  std::map<std::string, std::shared_ptr<RbdMapping>> mappings;
+  ceph::mutex map_mutex = ceph::make_mutex("RbdMappingDispatcher::MapMutex");
+
+  void disconnect_cbk(std::string devpath, int ret);
+
+public:
+  RbdMappingDispatcher(RadosClientCache& _client_cache)
+    : client_cache(_client_cache)
+  {}
+
+  int create(Config& cfg);
+  std::shared_ptr<RbdMapping> get_mapping(std::string& devpath);
 };
diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.cc b/src/tools/rbd_wnbd/rbd_mapping_config.cc
index c78920ea8d32..3cdd7f6dda51 100644
--- a/src/tools/rbd_wnbd/rbd_mapping_config.cc
+++ b/src/tools/rbd_wnbd/rbd_mapping_config.cc
@@ -52,7 +52,7 @@ int construct_devpath_if_missing(Config* cfg)
   return 0;
 }
 
-int save_config_to_registry(Config* cfg, std::string command_line)
+int save_config_to_registry(Config* cfg)
 {
   std::string strKey{ SERVICE_REG_KEY };
   strKey.append("\\");
@@ -73,7 +73,7 @@ int save_config_to_registry(Config* cfg, std::string command_line)
       reg_key.set("nsname", cfg->nsname) ||
       reg_key.set("imgname", cfg->imgname) ||
       reg_key.set("snapname", cfg->snapname) ||
-      reg_key.set("command_line", command_line) ||
+      reg_key.set("command_line", cfg->command_line) ||
       reg_key.set("persistent", cfg->persistent) ||
       reg_key.set("admin_sock_path", g_conf()->admin_socket) ||
       reg_key.flush()) {
diff --git a/src/tools/rbd_wnbd/rbd_mapping_config.h b/src/tools/rbd_wnbd/rbd_mapping_config.h
index 60e3fa20a45e..55262f6d02bb 100644
--- a/src/tools/rbd_wnbd/rbd_mapping_config.h
+++ b/src/tools/rbd_wnbd/rbd_mapping_config.h
@@ -31,7 +31,8 @@ struct Config {
   bool exclusive = false;
   bool readonly = false;
 
-  std::string parent_pipe;
+  std::string cluster_name;
+  std::string entity_name;
 
   std::string poolname;
   std::string nsname;
@@ -72,6 +73,6 @@ struct Config {
 };
 
 int construct_devpath_if_missing(Config* cfg);
-int save_config_to_registry(Config* cfg, std::string command_line);
+int save_config_to_registry(Config* cfg);
 int remove_config_from_registry(Config* cfg);
 int load_mapping_config_from_registry(std::string devpath, Config* cfg);
diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc
index 08cb7ca9359d..b3a42285b2c1 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.cc
+++ b/src/tools/rbd_wnbd/rbd_wnbd.cc
@@ -29,6 +29,7 @@
 #include "wnbd_handler.h"
 #include "wnbd_wmi.h"
 #include "rbd_wnbd.h"
+#include "rados_client_cache.h"
 
 #include <fstream>
 #include <memory>
@@ -72,9 +73,12 @@ using namespace std;
 // Wait for wmi events up to two seconds
 #define WMI_EVENT_TIMEOUT 2
 
-static WnbdHandler* handler = nullptr;
 static ceph::mutex shutdown_lock = ceph::make_mutex("RbdWnbd::ShutdownLock");
 
+static RadosClientCache client_cache;
+static RbdMappingDispatcher mapping_dispatcher(client_cache);
+static RbdMapping* daemon_mapping = nullptr;
+
 bool is_process_running(DWORD pid)
 {
   HANDLE process = OpenProcess(SYNCHRONIZE, FALSE, pid);
@@ -326,192 +330,55 @@ int send_map_request(std::string arguments) {
   return reply.status;
 }
 
-// Spawn a subprocess using the specified "rbd-wnbd" command
-// arguments. A pipe is passed to the child process,
-// which will allow it to communicate the mapping status
-int map_device_using_suprocess(std::string arguments, int timeout_ms)
+int map_device_using_same_process(std::string command_line)
 {
-  STARTUPINFOW si;
-  PROCESS_INFORMATION pi;
-  char ch;
-  DWORD err = 0, status = 0;
-  int exit_code = 0;
-  std::ostringstream command_line;
-  std::string exe_path;
-  // Windows async IO context
-  OVERLAPPED connect_o, read_o;
-  HANDLE connect_event = NULL, read_event = NULL;
-  // Used for waiting on multiple events that are going to be initialized later.
-  HANDLE wait_events[2] = { INVALID_HANDLE_VALUE, INVALID_HANDLE_VALUE};
-  DWORD bytes_read = 0;
-  // We may get a command line containing an old pipe handle when
-  // recreating mappings, so we'll have to replace it.
-  std::regex pipe_pattern("([\'\"]?--pipe-name[\'\"]? +[\'\"]?[^ ]+[\'\"]?)");
-
-  uuid_d uuid;
-  uuid.generate_random();
-  std::ostringstream pipe_name;
-  pipe_name << "\\\\.\\pipe\\rbd-wnbd-" << uuid;
-
-  // Create an unique named pipe to communicate with the child. */
-  HANDLE pipe_handle = CreateNamedPipe(
-    pipe_name.str().c_str(),
-    PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE |
-      FILE_FLAG_OVERLAPPED,
-    PIPE_WAIT,
-    1, // Only accept one instance
-    SERVICE_PIPE_BUFFSZ,
-    SERVICE_PIPE_BUFFSZ,
-    SERVICE_PIPE_TIMEOUT_MS,
-    NULL);
-  if (pipe_handle == INVALID_HANDLE_VALUE) {
-    err = GetLastError();
-    derr << "CreateNamedPipe failed: " << win32_strerror(err) << dendl;
-    exit_code = -ECHILD;
-    goto finally;
-  }
-  connect_event = CreateEvent(0, TRUE, FALSE, NULL);
-  read_event = CreateEvent(0, TRUE, FALSE, NULL);
-  if (!connect_event || !read_event) {
-    err = GetLastError();
-    derr << "CreateEvent failed: " << win32_strerror(err) << dendl;
-    exit_code = -ECHILD;
-    goto finally;
-  }
-  connect_o.hEvent = connect_event;
-  read_o.hEvent = read_event;
-
-  status = ConnectNamedPipe(pipe_handle, &connect_o);
-  err = GetLastError();
-  if (status || err != ERROR_IO_PENDING) {
-    if (status)
-      err = status;
-    derr << "ConnectNamedPipe failed: " << win32_strerror(err) << dendl;
-    exit_code = -ECHILD;
-    goto finally;
+  dout(5) << "Creating mapping using the same process. Command line: "
+          << command_line << dendl;
+
+  int argc;
+  // CommandLineToArgvW only has an UTF-16 variant.
+  LPWSTR* argv_w = CommandLineToArgvW(
+    to_wstring(command_line).c_str(), &argc);
+  if (!argv_w) {
+    DWORD err = GetLastError();
+    derr << "Couldn't parse args, error: "
+         << win32_strerror(err) << dendl;
+    return -EINVAL;
   }
-  err = 0;
 
-  dout(5) << __func__ << ": command arguments: " << arguments << dendl;
-
-  // We'll avoid running arbitrary commands, instead using the executable
-  // path of this process (expected to be the full rbd-wnbd.exe path).
-  err = get_exe_path(exe_path);
-  if (err) {
-    exit_code = -EINVAL;
-    goto finally;
-  }
-  command_line << std::quoted(exe_path)
-               << " " << std::regex_replace(arguments, pipe_pattern, "")
-               << " --pipe-name " << pipe_name.str();
-
-  dout(5) << __func__ << ": command line: " << command_line.str() << dendl;
-
-  GetStartupInfoW(&si);
-  // Create a detached child
-  if (!CreateProcessW(
-      NULL, const_cast<wchar_t*>(to_wstring(command_line.str()).c_str()),
-      NULL, NULL, FALSE, DETACHED_PROCESS,
-      NULL, NULL, &si, &pi)) {
-    err = GetLastError();
-    derr << "CreateProcess failed: " << win32_strerror(err) << dendl;
-    exit_code = -ECHILD;
-    goto finally;
+  std::vector<const char*> args;
+  std::vector<std::string> argv_sv;
+  // We're reserving the vector size in order to avoid resizes,
+  // which would invalidate our char* pointers.
+  argv_sv.reserve(argc);
+  args.reserve(argc);
+  for (int i = 0; i < argc; i++) {
+    argv_sv.push_back(to_string(argv_w[i]));
+    args.push_back(argv_sv[i].c_str());
   }
+  LocalFree(argv_w);
 
-  wait_events[0] = connect_event;
-  wait_events[1] = pi.hProcess;
-  status = WaitForMultipleObjects(2, wait_events, FALSE, timeout_ms);
-  switch(status) {
-    case WAIT_OBJECT_0:
-      if (!GetOverlappedResult(pipe_handle, &connect_o, &bytes_read, TRUE)) {
-        err = GetLastError();
-        derr << "Couldn't establish a connection with the child process. "
-             << "Error: " << win32_strerror(err) << dendl;
-        exit_code = -ECHILD;
-        goto clean_process;
-      }
-      // We have an incoming connection.
-      break;
-    case WAIT_OBJECT_0 + 1:
-      // The process has exited prematurely.
-      goto clean_process;
-    case WAIT_TIMEOUT:
-      derr << "Timed out waiting for child process connection." << dendl;
-      goto clean_process;
-    default:
-      derr << "Failed waiting for child process. Status: " << status << dendl;
-      goto clean_process;
-  }
-  // Block and wait for child to say it is ready.
-  dout(5) << __func__ << ": waiting for child notification." << dendl;
-  if (!ReadFile(pipe_handle, &ch, 1, NULL, &read_o)) {
-    err = GetLastError();
-    if (err != ERROR_IO_PENDING) {
-      derr << "Receiving child process reply failed with: "
-           << win32_strerror(err) << dendl;
-      exit_code = -ECHILD;
-      goto clean_process;
-    }
+  Config cfg;
+  cfg.command_line = command_line;
+  Command parsed_cmd = None;
+  std::ostringstream err_msg;
+  int r = parse_args(args, &err_msg, &parsed_cmd, &cfg);
+  if (r) {
+    derr << "Couldn't parse args, error: " << r
+         << ". Error message: " << err_msg.str() << dendl;
+    return -EINVAL;
   }
-  wait_events[0] = read_event;
-  wait_events[1] = pi.hProcess;
-  // The RBD daemon is expected to write back right after opening the
-  // pipe. We'll use the same timeout value for now.
-  status = WaitForMultipleObjects(2, wait_events, FALSE, timeout_ms);
-  switch(status) {
-    case WAIT_OBJECT_0:
-      if (!GetOverlappedResult(pipe_handle, &read_o, &bytes_read, TRUE)) {
-        err = GetLastError();
-        derr << "Receiving child process reply failed with: "
-             << win32_strerror(err) << dendl;
-        exit_code = -ECHILD;
-        goto clean_process;
-      }
-      break;
-    case WAIT_OBJECT_0 + 1:
-      // The process has exited prematurely.
-      goto clean_process;
-    case WAIT_TIMEOUT:
-      derr << "Timed out waiting for child process message." << dendl;
-      goto clean_process;
-    default:
-      derr << "Failed waiting for child process. Status: " << status << dendl;
-      goto clean_process;
+  if (parsed_cmd != Connect) {
+    derr << "Unexpected map command: " << parsed_cmd
+         << ", expecting: " << Connect << dendl;
+    return -EINVAL;
   }
 
-  dout(5) << __func__ << ": received child notification." << dendl;
-  goto finally;
-
-  clean_process:
-    if (!is_process_running(pi.dwProcessId)) {
-      GetExitCodeProcess(pi.hProcess, (PDWORD)&exit_code);
-      if (!exit_code) {
-        // Child terminated unexpectedly.
-        exit_code = -ECHILD;
-      } else if (exit_code > 0) {
-        // Make sure to return a negative error code.
-        exit_code = -exit_code;
-      }
-      derr << "Daemon failed with: " << cpp_strerror(exit_code) << dendl;
-    } else {
-      // The process closed the pipe without notifying us or exiting.
-      // This is quite unlikely, but we'll terminate the process.
-      dout(0) << "Terminating unresponsive process." << dendl;
-      TerminateProcess(pi.hProcess, 1);
-      exit_code = -EINVAL;
-    }
+  if (construct_devpath_if_missing(&cfg)) {
+    return -EINVAL;
+  }
 
-  finally:
-    if (exit_code)
-      derr << "Could not start RBD daemon." << dendl;
-    if (pipe_handle)
-      CloseHandle(pipe_handle);
-    if (connect_event)
-      CloseHandle(connect_event);
-    if (read_event)
-      CloseHandle(read_event);
-  return exit_code;
+  return mapping_dispatcher.create(cfg);
 }
 
 BOOL WINAPI console_handler_routine(DWORD dwCtrlType)
@@ -519,7 +386,10 @@ BOOL WINAPI console_handler_routine(DWORD dwCtrlType)
   dout(0) << "Received control signal: " << dwCtrlType
           << ". Exiting." << dendl;
 
-  // TODO: shutdown all mappings
+  std::unique_lock l{shutdown_lock};
+  if (daemon_mapping) {
+    daemon_mapping->shutdown();
+  }
 
   return true;
 }
@@ -591,7 +461,7 @@ int restart_registered_mappings(
 
         // We'll try to map all devices and return a non-zero value
         // if any of them fails.
-        int r = map_device_using_suprocess(cfg.command_line, time_left_ms);
+        int r = map_device_using_same_process(cfg.command_line);
         if (r) {
           err = r;
           derr << "Could not create mapping: "
@@ -724,9 +594,8 @@ class RBDService : public ServiceBase {
                   << (char*)request->arguments << dendl;
           // TODO: use the configured service map timeout.
           // TODO: add ceph.conf options.
-          return map_device_using_suprocess(
-            (char*)request->arguments,
-            DEFAULT_IMAGE_MAP_TIMEOUT * 1000);
+          return map_device_using_same_process(
+            std::string((char*) request->arguments));
         default:
           dout(1) << "Received unsupported command: "
                   << request->command << dendl;
@@ -939,6 +808,8 @@ class RBDService : public ServiceBase {
         } else {
           dout(0) << "Ignoring image remap failure." << dendl;
         }
+      } else {
+        dout(0) << "successfully restarted mappings" << dendl;
       }
 
       if (adapter_monitoring_enabled) {
@@ -1070,7 +941,7 @@ boost::intrusive_ptr<CephContext> do_global_init(
   global_pre_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, code_env, flags);
   // Avoid cluttering the console when spawning a mapping that will run
   // in the background.
-  if (g_conf()->daemonize && cfg->parent_pipe.empty()) {
+  if (g_conf()->daemonize) {
     flags |= CINIT_FLAG_NO_DAEMON_ACTIONS;
   }
   auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
@@ -1083,61 +954,17 @@ boost::intrusive_ptr<CephContext> do_global_init(
   return cct;
 }
 
-// Wait for the mapped disk to become available.
-static int wait_mapped_disk(Config *cfg)
-{
-  DWORD status = WnbdPollDiskNumber(
-    cfg->devpath.c_str(),
-    TRUE, // ExpectMapped
-    TRUE, // TryOpen
-    cfg->image_map_timeout,
-    DISK_STATUS_POLLING_INTERVAL_MS,
-    (PDWORD) &cfg->disk_number);
-  if (status) {
-    derr << "WNBD disk unavailable, error: "
-         << win32_strerror(status) << dendl;
-    return -EINVAL;
-  }
-  dout(0) << "Successfully mapped image: " << cfg->devpath
-          << ". Windows disk path: "
-          << "\\\\.\\PhysicalDrive" + std::to_string(cfg->disk_number)
-          << dendl;
-  return 0;
-}
-
 static int do_map(Config *cfg)
 {
-  if (g_conf()->daemonize && cfg->parent_pipe.empty()) {
-    r = send_map_request(get_cli_args());
-    if (r < 0) {
-      return r;
-    }
-
-    return wait_mapped_disk(cfg);
-  }
-
   dout(0) << "Mapping RBD image: " << cfg->devpath << dendl;
 
-  librados::Rados rados;
-  int r = rados.init_with_context(g_ceph_context);
-  if (r < 0) {
-    derr << "rbd-wnbd: couldn't initialize rados: " << cpp_strerror(r)
-         << dendl;
-    return r;
-  }
-
-  RbdMapping rbd_mapping(*cfg, rados, get_cli_args());
-  r = rbd_mapping.start();
+  RbdMapping rbd_mapping(*cfg, client_cache);
+  int r = rbd_mapping.start();
   if (r) {
     return r;
   }
 
-  // TODO: consider substracting the time it took to perform the
-  // above operations from cfg->image_map_timeout in wait_mapped_disk().
-  r = wait_mapped_disk(cfg);
-  if (r < 0) {
-    goto close_ret;
-  }
+  daemon_mapping = &rbd_mapping;
 
   dout(0) << "Successfully mapped RBD image: " << cfg->devpath << dendl;
   return rbd_mapping.wait();
@@ -1371,10 +1198,17 @@ static int parse_args(std::vector<const char*>& args,
   }
   config.parse_env(CEPH_ENTITY_TYPE_CLIENT);
   config.parse_argv(args);
+
+  cfg->cluster_name = string(config->cluster);
+  cfg->entity_name = config->name.to_str();
   cfg->poolname = config.get_val<std::string>("rbd_default_pool");
 
   std::vector<const char*>::iterator i;
   std::ostringstream err;
+  // The parent pipe parameter has been deprecated since we're no longer
+  // using separate processes per mapping (unless "-f" is passed).
+  // TODO: remove this parameter eventually.
+  std::string parent_pipe;
 
   // TODO: consider using boost::program_options like Device.cc does.
   // This should simplify argument parsing. Also, some arguments must be tied
@@ -1400,12 +1234,14 @@ static int parse_args(std::vector<const char*>& args,
       cfg->remap_failure_fatal = true;
     } else if (ceph_argparse_flag(args, i, "--adapter-monitoring-enabled", (char *)NULL)) {
       cfg->adapter_monitoring_enabled = true;
-    } else if (ceph_argparse_witharg(args, i, &cfg->parent_pipe, err,
+    } else if (ceph_argparse_witharg(args, i, &parent_pipe, err,
                                      "--pipe-name", (char *)NULL)) {
       if (!err.str().empty()) {
         *err_msg << "rbd-wnbd: " << err.str();
         return -EINVAL;
       }
+      std::cerr << "WARNING: '--pipe-name' has been deprecated and is currently ignored."
+                << std::endl;
     } else if (ceph_argparse_witharg(args, i, (int*)&cfg->wnbd_log_level,
                                      err, "--wnbd-log-level", (char *)NULL)) {
       if (!err.str().empty()) {
@@ -1547,6 +1383,7 @@ static int parse_args(std::vector<const char*>& args,
 static int rbd_wnbd(int argc, const char *argv[])
 {
   Config cfg;
+  cfg.command_line = get_cli_args();
   auto args = argv_to_vec(argc, argv);
 
   // Avoid using dout before calling "do_global_init"
@@ -1578,6 +1415,14 @@ static int rbd_wnbd(int argc, const char *argv[])
       if (construct_devpath_if_missing(&cfg)) {
         return -EINVAL;
       }
+      if (g_conf()->daemonize) {
+        r = send_map_request(cfg.command_line);
+        if (r < 0) {
+          return r;
+        }
+        return wait_mapped_disk(cfg);
+      }
+
       r = do_map(&cfg);
       if (r < 0)
         return r;
diff --git a/src/tools/rbd_wnbd/rbd_wnbd.h b/src/tools/rbd_wnbd/rbd_wnbd.h
index ba6280031a36..6ec4851e8a7c 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.h
+++ b/src/tools/rbd_wnbd/rbd_wnbd.h
@@ -27,8 +27,6 @@
 #define SERVICE_PIPE_TIMEOUT_MS 5000
 #define SERVICE_PIPE_BUFFSZ 4096
 
-#define DISK_STATUS_POLLING_INTERVAL_MS 500
-
 #define HELP_INFO 1
 #define VERSION_INFO 2
 
@@ -64,16 +62,16 @@ int disconnect_all_mappings(
   int worker_count);
 int restart_registered_mappings(
   int worker_count, int total_timeout, int image_map_timeout);
-int map_device_using_suprocess(std::string command_line);
+int map_device_using_same_process(std::string command_line);
 
 BOOL WINAPI console_handler_routine(DWORD dwCtrlType);
 
 static int parse_args(std::vector<const char*>& args,
                       std::ostream *err_msg,
                       Command *command, Config *cfg);
+static int do_map(Config *cfg);
 static int do_unmap(Config *cfg, bool unregister);
 
-
 class BaseIterator {
   public:
     virtual ~BaseIterator() {};
diff --git a/src/tools/rbd_wnbd/wnbd_handler.cc b/src/tools/rbd_wnbd/wnbd_handler.cc
index ba53b872697c..d5f4356ad177 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.cc
+++ b/src/tools/rbd_wnbd/wnbd_handler.cc
@@ -50,12 +50,15 @@ int WnbdHandler::wait()
 {
   int err = 0;
   if (started && wnbd_disk) {
-    dout(10) << __func__ << ": waiting" << dendl;
+    dout(10) << "waiting for WNBD mapping: " << instance_name << dendl;
 
     err = WnbdWaitDispatcher(wnbd_disk);
     if (err) {
-      derr << __func__ << " failed waiting for dispatcher to stop: "
-           << err << dendl;
+      derr << __func__ << ": failed waiting for dispatcher to stop: "
+           << instance_name
+           << ". Error: " << err << dendl;
+    } else {
+      dout(10) << "WNBD mapping disconnected: " << instance_name << dendl;
     }
   }
 

From 5f9c69d53c522f3b939c91c974cedb9fdd085a60 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Wed, 19 Jul 2023 12:32:40 +0000
Subject: [PATCH 2084/2492] rbd-wnbd: update registry settings handling

This commit will store the mapping config in the Windows registry
only after initializing the mapping. This ensures that we aren't
replacing the registry settings for already mapped images.

We'll also check if the registry setting was added by us before
cleaning it up.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/tools/rbd_wnbd/rbd_mapping.cc | 23 +++++++++++++----------
 src/tools/rbd_wnbd/rbd_mapping.h  |  1 +
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/tools/rbd_wnbd/rbd_mapping.cc b/src/tools/rbd_wnbd/rbd_mapping.cc
index b2d7cff93bc0..eaa79180929c 100644
--- a/src/tools/rbd_wnbd/rbd_mapping.cc
+++ b/src/tools/rbd_wnbd/rbd_mapping.cc
@@ -76,15 +76,6 @@ int RbdMapping::init()
 
   initial_image_size = info.size;
 
-  // We're storing mapping details in the registry even for non-persistent
-  // mappings. This allows us to easily retrieve mapping details such
-  // as the rbd pool or admin socket path.
-  // We're cleaning up the registry entry when the non-persistent mapping
-  // gets disconnected or when the ceph service restarts.
-  r = save_config_to_registry(&cfg);
-  if (r < 0)
-    return r;
-
   handler = new WnbdHandler(image, cfg.devpath,
                             info.size / RBD_WNBD_BLKSIZE,
                             RBD_WNBD_BLKSIZE,
@@ -102,7 +93,7 @@ void RbdMapping::shutdown()
   dout(5) << __func__ << ": removing RBD mapping: " << cfg.devpath << dendl;
 
   int r = 0;
-  if (!cfg.persistent) {
+  if (!cfg.persistent && saved_cfg_to_registry) {
     dout(5) << __func__ << ": cleaning up non-persistent mapping: "
             << cfg.devpath << dendl;
     r = remove_config_from_registry(&cfg);
@@ -161,6 +152,18 @@ int RbdMapping::start()
     return r;
   }
 
+  // We're storing mapping details in the registry even for non-persistent
+  // mappings. This allows us to easily retrieve mapping details such
+  // as the rbd pool or admin socket path.
+  // We're cleaning up the registry entry when the non-persistent mapping
+  // gets disconnected or when the ceph service restarts.
+  r = save_config_to_registry(&cfg);
+  if (r < 0) {
+    return r;
+  } else {
+    saved_cfg_to_registry = true;
+  }
+
   if (disconnect_cbk) {
     monitor_thread = std::thread([this]{
       int ret = this->wait();
diff --git a/src/tools/rbd_wnbd/rbd_mapping.h b/src/tools/rbd_wnbd/rbd_mapping.h
index 52404ed503f2..1255880cb793 100644
--- a/src/tools/rbd_wnbd/rbd_mapping.h
+++ b/src/tools/rbd_wnbd/rbd_mapping.h
@@ -65,6 +65,7 @@ class RbdMapping
   WnbdHandler* handler = nullptr;
   uint64_t watch_handle;
   WNBDWatchCtx* watch_ctx = nullptr;
+  bool saved_cfg_to_registry = false;
   disconnect_cbk_t disconnect_cbk;
 
   ceph::mutex shutdown_lock = ceph::make_mutex("RbdMapping::ShutdownLock");

From 0d73d31b6f92d45c602d704b39d11d11536b0a93 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Wed, 2 Aug 2023 09:50:37 +0000
Subject: [PATCH 2085/2492] qa: update rbd-wnbd test, retrying image rm
 operations

The "rbd-wnbd unmap" command is currently telling the WNBD driver
to remove the mapping without contacting the rbd-wnbd daemon
and waiting for it to perform its cleanup.

For this reason, attempting to delete the image immediately after
unmapping it can fail due to existing watchers.

As a temporary solution, we'll retry the image remove operation.
At a later time, we'll update the "rbd-wnbd unmap" command to go
through the rbd-wnbd daemon, ensuring that all the necessary
cleanup is performed before returning.

While at it, we're dropping a redundant LOG.error call so that we
won't print expected exceptions.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 qa/workunits/windows/test_rbd_wnbd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/workunits/windows/test_rbd_wnbd.py b/qa/workunits/windows/test_rbd_wnbd.py
index db14234a2b5f..0eb7653b0e98 100644
--- a/qa/workunits/windows/test_rbd_wnbd.py
+++ b/qa/workunits/windows/test_rbd_wnbd.py
@@ -162,7 +162,6 @@ def execute(*args, **kwargs):
         exc = CommandFailed(
             command=args, returncode=result.returncode,
             stdout=result.stdout, stderr=result.stderr)
-        LOG.error(exc)
         raise exc
     return result
 
@@ -367,6 +366,7 @@ def unmap(self):
             self.mapped = False
 
     @Tracer.trace
+    @retry_decorator()
     def remove(self):
         if not self.removed:
             LOG.info("Removing image: %s", self.name)

From 83d58ab307063b4a37f3a297af2b0e6ba27e84dc Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Tue, 26 Sep 2023 08:00:19 +0000
Subject: [PATCH 2086/2492] rbd-wnbd: adjust admin socket hook to accept image
 path

For each rbd-wnbd mapping we set an admin socket hook that can
be used to retrieve IO stats.

Now that the same daemon is reused for multiple mappings, we need
to distinguish the images when receiving a "stats" request.

For this reason, we'll add the image identifier to "wnbd stats"
admin socket commands.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/tools/rbd_wnbd/rbd_wnbd.cc     | 4 ++--
 src/tools/rbd_wnbd/wnbd_handler.cc | 9 ++++++++-
 src/tools/rbd_wnbd/wnbd_handler.h  | 7 ++-----
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/tools/rbd_wnbd/rbd_wnbd.cc b/src/tools/rbd_wnbd/rbd_wnbd.cc
index b3a42285b2c1..57691bb72360 100644
--- a/src/tools/rbd_wnbd/rbd_wnbd.cc
+++ b/src/tools/rbd_wnbd/rbd_wnbd.cc
@@ -1159,8 +1159,8 @@ static int do_stats(std::string search_devpath)
 
     AdminSocketClient client = AdminSocketClient(cfg.admin_sock_path);
     std::string output;
-    std::string result = client.do_request("{\"prefix\":\"wnbd stats\"}",
-                                           &output);
+    std::string cmd = "{\"prefix\":\"wnbd stats " + cfg.devpath + "\"}";
+    std::string result = client.do_request(cmd, &output);
     if (!result.empty()) {
       std::cerr << "Admin socket error: " << result << std::endl;
       return -EINVAL;
diff --git a/src/tools/rbd_wnbd/wnbd_handler.cc b/src/tools/rbd_wnbd/wnbd_handler.cc
index d5f4356ad177..d6597e32c449 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.cc
+++ b/src/tools/rbd_wnbd/wnbd_handler.cc
@@ -65,6 +65,13 @@ int WnbdHandler::wait()
   return err;
 }
 
+WnbdAdminHook::WnbdAdminHook(WnbdHandler *handler) : m_handler(handler)
+{
+  g_ceph_context->get_admin_socket()->register_command(
+    std::string("wnbd stats ") + m_handler->instance_name,
+    this, "get WNBD stats");
+}
+
 int WnbdAdminHook::call (
   std::string_view command, const cmdmap_t& cmdmap,
   const bufferlist&,
@@ -72,7 +79,7 @@ int WnbdAdminHook::call (
   std::ostream& errss,
   bufferlist& out)
 {
-  if (command == "wnbd stats") {
+  if (command == "wnbd stats " + m_handler->instance_name) {
     return m_handler->dump_stats(f);
   }
   return -ENOSYS;
diff --git a/src/tools/rbd_wnbd/wnbd_handler.h b/src/tools/rbd_wnbd/wnbd_handler.h
index f3b3e4341b01..6b9886a6e38d 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.h
+++ b/src/tools/rbd_wnbd/wnbd_handler.h
@@ -44,11 +44,7 @@ class WnbdAdminHook : public AdminSocketHook {
   WnbdHandler *m_handler;
 
 public:
-  explicit WnbdAdminHook(WnbdHandler *handler) :
-        m_handler(handler) {
-    g_ceph_context->get_admin_socket()->register_command(
-      "wnbd stats", this, "get WNBD stats");
-  }
+  explicit WnbdAdminHook(WnbdHandler *handler);
   ~WnbdAdminHook() override {
     g_ceph_context->get_admin_socket()->unregister_commands(this);
   }
@@ -140,6 +136,7 @@ class WnbdHandler
     void set_sense(uint8_t sense_key, uint8_t asc);
   };
 
+  friend WnbdAdminHook;
   friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);
 
   void send_io_response(IOContext *ctx);

From a14003c492ad26ed4eef13e3d73e42212dd2bd93 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <lpetrut@cloudbasesolutions.com>
Date: Tue, 26 Sep 2023 11:52:31 +0000
Subject: [PATCH 2087/2492] rbd-wnbd: use the right AdminSocket instance

The rbd-wnbd daemon currently caches one rados context per cluster.
However, it's registering hooks against the global context
admin socket, which won't be available. For this reason,
the "rbd-wnbd stats" command no longer works.

To address this issue, we'll ensure that rbd-wnbd sets command hooks
against the right admin socket instance, leveraging the image
context.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>
---
 src/tools/rbd_wnbd/rbd_mapping.cc  |  6 +++++-
 src/tools/rbd_wnbd/wnbd_handler.cc | 15 +++++++++++----
 src/tools/rbd_wnbd/wnbd_handler.h  | 12 ++++++++----
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/src/tools/rbd_wnbd/rbd_mapping.cc b/src/tools/rbd_wnbd/rbd_mapping.cc
index eaa79180929c..35e3b7718c88 100644
--- a/src/tools/rbd_wnbd/rbd_mapping.cc
+++ b/src/tools/rbd_wnbd/rbd_mapping.cc
@@ -76,13 +76,17 @@ int RbdMapping::init()
 
   initial_image_size = info.size;
 
+  CephContext* cct = reinterpret_cast<CephContext*>(io_ctx.cct());
+  ceph_assert(cct != nullptr);
+
   handler = new WnbdHandler(image, cfg.devpath,
                             info.size / RBD_WNBD_BLKSIZE,
                             RBD_WNBD_BLKSIZE,
                             !cfg.snapname.empty() || cfg.readonly,
                             g_conf().get_val<bool>("rbd_cache"),
                             cfg.io_req_workers,
-                            cfg.io_reply_workers);
+                            cfg.io_reply_workers,
+                            cct->get_admin_socket());
   return 0;
 }
 
diff --git a/src/tools/rbd_wnbd/wnbd_handler.cc b/src/tools/rbd_wnbd/wnbd_handler.cc
index d6597e32c449..82b01c906f26 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.cc
+++ b/src/tools/rbd_wnbd/wnbd_handler.cc
@@ -65,11 +65,18 @@ int WnbdHandler::wait()
   return err;
 }
 
-WnbdAdminHook::WnbdAdminHook(WnbdHandler *handler) : m_handler(handler)
+WnbdAdminHook::WnbdAdminHook(WnbdHandler *handler, AdminSocket* admin_socket)
+  : m_handler(handler)
+  , m_admin_socket(admin_socket)
 {
-  g_ceph_context->get_admin_socket()->register_command(
-    std::string("wnbd stats ") + m_handler->instance_name,
-    this, "get WNBD stats");
+  if (m_admin_socket) {
+    m_admin_socket->register_command(
+      std::string("wnbd stats ") + m_handler->instance_name,
+      this, "get WNBD stats");
+  } else {
+    dout(0) << "no admin socket provided, skipped registering wnbd hooks"
+            << dendl;
+  }
 }
 
 int WnbdAdminHook::call (
diff --git a/src/tools/rbd_wnbd/wnbd_handler.h b/src/tools/rbd_wnbd/wnbd_handler.h
index 6b9886a6e38d..139621483242 100644
--- a/src/tools/rbd_wnbd/wnbd_handler.h
+++ b/src/tools/rbd_wnbd/wnbd_handler.h
@@ -42,11 +42,14 @@ class WnbdHandler;
 
 class WnbdAdminHook : public AdminSocketHook {
   WnbdHandler *m_handler;
+  AdminSocket *m_admin_socket;
 
 public:
-  explicit WnbdAdminHook(WnbdHandler *handler);
+  explicit WnbdAdminHook(WnbdHandler *handler, AdminSocket* admin_socket);
   ~WnbdAdminHook() override {
-    g_ceph_context->get_admin_socket()->unregister_commands(this);
+    if (m_admin_socket) {
+      m_admin_socket->unregister_commands(this);
+    }
   }
 
   int call(std::string_view command, const cmdmap_t& cmdmap,
@@ -74,7 +77,8 @@ class WnbdHandler
               uint64_t _block_count, uint32_t _block_size,
               bool _readonly, bool _rbd_cache_enabled,
               uint32_t _io_req_workers,
-              uint32_t _io_reply_workers)
+              uint32_t _io_reply_workers,
+              AdminSocket* admin_socket)
     : image(_image)
     , instance_name(_instance_name)
     , block_count(_block_count)
@@ -84,7 +88,7 @@ class WnbdHandler
     , io_req_workers(_io_req_workers)
     , io_reply_workers(_io_reply_workers)
   {
-    admin_hook = new WnbdAdminHook(this);
+    admin_hook = new WnbdAdminHook(this, admin_socket);
     // Instead of relying on librbd's own thread pool, we're going to use a
     // separate one. This allows us to make assumptions on the threads that
     // are going to send the IO replies and thus be able to cache Windows

From 565bc9503838906995fa48f59debcd2843775b18 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 1 Mar 2024 22:11:14 +1000
Subject: [PATCH 2088/2492] doc/install: add manual RADOSGW install procedure

Add a manual RADOSGW installation procedure to
doc/install/manual-deployment.rst. This procedure was developed by Janne
Johansson and reported to the ceph-users mailing list on 29 Jan 2024
here: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/message/LB3YRIKAPOHXYCW7MKLVUJPYWYRQVARU/

Co-authored-by: Janne Johansson <icepic.dz@gmail.com>
Co-authored-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/install/manual-deployment.rst | 46 +++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/doc/install/manual-deployment.rst b/doc/install/manual-deployment.rst
index 6716ecb5beb5..0eb759de701c 100644
--- a/doc/install/manual-deployment.rst
+++ b/doc/install/manual-deployment.rst
@@ -461,6 +461,52 @@ In the below instructions, ``{id}`` is an arbitrary name, such as the hostname o
 
 #. Now you are ready to `create a Ceph file system`_.
 
+Manually Installing RADOSGW
+===========================
+
+For a more involved discussion of the procedure presented here, see `this
+thread on the ceph-users mailing list
+<https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/message/LB3YRIKAPOHXYCW7MKLVUJPYWYRQVARU/>`_.
+
+#. Install ``radosgw`` packages on the nodes that will be the RGW nodes.
+
+#. From a monitor or from a node with admin privileges, run a command of the
+   following form:
+
+   .. prompt:: bash #
+      
+      ceph auth get-or-create client.short-hostname-of-rgw mon 'allow rw' osd 'allow rwx'
+
+#. On one of the RGW nodes, do the following:
+
+   a. Create a ``ceph-user``-owned directory. For example: 
+
+      .. prompt:: bash #
+
+         install -d -o ceph -g ceph /var/lib/ceph/radosgw/ceph-$(hostname -s)
+
+   b. Enter the directory just created and create a ``keyring`` file: 
+
+      .. prompt:: bash #
+
+         touch /var/lib/ceph/radosgw/ceph-$(hostname -s)/keyring
+
+      Use a command similar to this one to put the key from the earlier ``ceph
+      auth get-or-create`` step in the ``keyring`` file. Use your preferred
+      editor:
+
+      .. prompt:: bash #
+
+         $EDITOR /var/lib/ceph/radosgw/ceph-$(hostname -s)/keyring
+
+   c. Repeat these steps on every RGW node.
+
+#. Start the RADOSGW service by running the following command:
+
+   .. prompt:: bash #
+
+      systemctl start ceph-radosgw@$(hostname -s).service
+
 
 Summary
 =======

From 4a1d6122eb00f5af851ad2e7d58d32a982a31fe3 Mon Sep 17 00:00:00 2001
From: Dan Mick <dmick@redhat.com>
Date: Thu, 29 Feb 2024 11:36:51 -0800
Subject: [PATCH 2089/2492] .github/workflows/create-backport-trackers.yml:
 update versions of actions

Getting warning about node16 being deprecated.  The workflow doesn't use node
directly, but through the external actions.  Moving to node20 requires
changing setup-python version; Bhacaz/checkout-files is deprecated and
recommends actions/checkout.

Signed-off-by: Dan Mick <dmick@redhat.com>
---
 .github/workflows/create-backport-trackers.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/create-backport-trackers.yml b/.github/workflows/create-backport-trackers.yml
index fe05d09b0ec1..1f358ab5e27b 100644
--- a/.github/workflows/create-backport-trackers.yml
+++ b/.github/workflows/create-backport-trackers.yml
@@ -36,10 +36,13 @@ jobs:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main'
     steps:
-      - uses: Bhacaz/checkout-files@e3e34e7daef91a5f237485bb88a260aee4be29dd
+      - uses: actions/checkout@v4
         with:
-          files: src/script/backport-create-issue src/script/requirements.backport-create-issue.txt
-      - uses: actions/setup-python@v4
+          sparse-checkout: | 
+              src/script/backport-create-issue
+              src/script/requirements.backport-create-issue.txt
+          sparse-checkout-cone-mode: false
+      - uses: actions/setup-python@v5
         with:
           python-version: '>=3.6 <3.12'
           cache: 'pip'

From 5a6eb2c4017b126895326a3559c966c10f0c24c3 Mon Sep 17 00:00:00 2001
From: Kamoltat <ksirivad@redhat.com>
Date: Fri, 1 Mar 2024 23:13:21 +0000
Subject: [PATCH 2090/2492] Revert "src/scripts/ceph-backport.sh: Add PR
 checklist"

This reverts commit 4aba45e2374168e4485f31fea4fa1c441c35443e

commit causes backport script to fail
reverting and recommiting is the best option

Fixes: https://tracker.ceph.com/issues/64674

Signed-off-by: Kamoltat <ksirivad@redhat.com>
---
 src/script/ceph-backport.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/script/ceph-backport.sh b/src/script/ceph-backport.sh
index 626d9805146e..2de5f91f0393 100755
--- a/src/script/ceph-backport.sh
+++ b/src/script/ceph-backport.sh
@@ -45,7 +45,6 @@ redmine_login=""
 redmine_user_id=""
 setup_ok=""
 this_script=$(basename "$full_path")
-gh_pr_template="../../../ceph/.github/pull_request_template.md"
 
 if [[ $* == *--debug* ]]; then
     set -x
@@ -1700,11 +1699,6 @@ if [ "$PR_PHASE" ] ; then
         [ "$original_issue" ] && desc="${desc}\nparent tracker: $(number_to_url "redmine" "${original_issue}")"
     fi
     desc="${desc}\n\nthis backport was staged using ceph-backport.sh version ${SCRIPT_VERSION}\nfind the latest version at ${github_endpoint}/blob/main/src/script/ceph-backport.sh"
-    desc="$desc\n\n"
-
-    while read line; do
-        desc="$desc$line"
-    done < ${gh_pr_template}
     
     debug "Generating backport PR title"
     if [ "$original_pr" ] ; then

From 7f42b1d0b7295c2b62d204d14ebe38b6654ba0ac Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 1 Mar 2024 19:10:00 +0000
Subject: [PATCH 2091/2492] src/seastar: update seastar submodule to fix
 prometheus build failure

Fixes: https://tracker.ceph.com/issues/64589
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/seastar | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/seastar b/src/seastar
index f3a1484c83b8..1a2cd05b7bf5 160000
--- a/src/seastar
+++ b/src/seastar
@@ -1 +1 @@
-Subproject commit f3a1484c83b8b8ed774582f85fe33bf957390400
+Subproject commit 1a2cd05b7bf515f68841fa06ba9e3bdaecd20716

From 2806cdb15109c391c828b3419e9d141991d4003d Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 1 Mar 2024 20:00:32 +0000
Subject: [PATCH 2092/2492] crimson/.../interruptible_future: remove
 SEASTAR_CONCEPT guard

Seastar commit 8dc3398a removed this macro, no longer necessary.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/interruptible_future.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h
index 5f8323a66d14..a44c99a1867b 100644
--- a/src/crimson/common/interruptible_future.h
+++ b/src/crimson/common/interruptible_future.h
@@ -69,7 +69,6 @@ namespace seastar::internal {
   {};
 }
 
-SEASTAR_CONCEPT(
 namespace crimson::interruptible {
   template<typename InterruptCond, typename FutureType>
   class interruptible_future_detail;
@@ -79,7 +78,6 @@ namespace seastar::impl {
   struct is_tuple_of_futures<std::tuple<crimson::interruptible::interruptible_future_detail<InterruptCond, FutureType>, Rest...>>
     : is_tuple_of_futures<std::tuple<Rest...>> {};
 }
-)
 
 namespace crimson::interruptible {
 

From 95e03f88094918005a14c2ac719b991c40e656ab Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 27 Feb 2024 22:25:32 +0800
Subject: [PATCH 2093/2492] cmake: bump liburing from 0.7 to 2.5

this allows us to use newer liburing features. Seastar is using
some of them which are not provided by liburing 0.7.

in this change, `--use-libc` is passed to configure. otherwise
it does not link against libc, and the symbles like memset()
won't be available when compiling liburing.so with -fPIC using
clang, which does not pull libc in that case.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/Builduring.cmake | 4 ++--
 make-dist                      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/modules/Builduring.cmake b/cmake/modules/Builduring.cmake
index 4b27a989ac4b..4e4107fb5ac7 100644
--- a/cmake/modules/Builduring.cmake
+++ b/cmake/modules/Builduring.cmake
@@ -9,7 +9,7 @@ function(build_uring)
     set(source_dir_args
       SOURCE_DIR ${CMAKE_BINARY_DIR}/src/liburing
       GIT_REPOSITORY https://github.com/axboe/liburing.git
-      GIT_TAG "liburing-0.7"
+      GIT_TAG "liburing-2.5"
       GIT_SHALLOW TRUE
       GIT_CONFIG advice.detachedHead=false)
   endif()
@@ -17,7 +17,7 @@ function(build_uring)
   include(ExternalProject)
   ExternalProject_Add(liburing_ext
     ${source_dir_args}
-    CONFIGURE_COMMAND env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} <SOURCE_DIR>/configure
+    CONFIGURE_COMMAND env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} <SOURCE_DIR>/configure --use-libc
     BUILD_COMMAND ${make_cmd} "CFLAGS=${CMAKE_C_FLAGS} -fPIC" -C src -s
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS "<SOURCE_DIR>/src/liburing.a"
diff --git a/make-dist b/make-dist
index 41cee50a6d39..d8031040dd74 100755
--- a/make-dist
+++ b/make-dist
@@ -195,7 +195,7 @@ boost_version=1.82.0
 download_boost $boost_version a6e1ab9b0860e6a2881dd7b21fe9f737a095e5f33a3a874afc6a345228597ee6 \
                https://download.ceph.com/qa \
                https://archives.boost.io/release/$boost_version/source
-download_liburing 0.7 8e2842cfe947f3a443af301bdd6d034455536c38a455c7a700d0c1ad165a7543 \
+download_liburing 2.5 456f5f882165630f0dc7b75e8fd53bd01a955d5d4720729b4323097e6e9f2a98 \
                   https://github.com/axboe/liburing/archive \
                   https://git.kernel.dk/cgit/liburing/snapshot
 pmdk_version=1.10

From e277fbf63cff08538d1e2d3dc99c0c7d69ac5e13 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xxhdx1985126@gmail.com>
Date: Sat, 2 Mar 2024 16:44:22 +0800
Subject: [PATCH 2094/2492] crimson/os/seastore: omap_get_values with range
 [start, "infinite") should return all results from "start"

Currently, this range retrieval has a default max result size, which is
wrong

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
---
 src/crimson/os/seastore/seastore.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 801667c9c30a..9c5199ed9ab8 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1134,7 +1134,9 @@ SeaStore::Shard::omap_get_values(
 	onode.get_layout().omap_root,
 	t,
 	start,
-	OMapManager::omap_list_config_t().with_inclusive(false, false));
+	OMapManager::omap_list_config_t()
+	  .with_inclusive(false, false)
+	  .without_max());
   });
 }
 

From af6bb91505669bfb4f7bde790ad31368313b2817 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sat, 2 Mar 2024 18:02:05 +0800
Subject: [PATCH 2095/2492] cmake/modules/BuildRocksDB.cmake: inherit
 CMAKE_C_COMPILER from parent
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

if we set the CFLAGS globally, and the CFLAGS contains options
only acceptable by a certain C compiler, RocksDB could fail to
configure. for instance, if we set CXXFLAGS so it contains
`--config /usr/lib/rpm/redhat/redhat-hardened-clang.cfg` and use clang++
as the CMAKE_CXX_COMPILER, while keep CMAKE_C_COMPILER unchanged.
RocksDB would fail to configure like:

```-- Check for working C compiler: /usr/bin/cc - broken
CMake Error at /usr/share/cmake/Modules/CMakeTestCCompiler.cmake:67 (message):
  The C compiler

    "/usr/bin/cc"

  is not able to compile a simple test program.

  It fails with the following output:

    Change Dir: /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-1717-g0f726187/rpm/el9/BUILD/ceph-19.0.0-1717-g0f726187/redhat-linux-build/src/rocksdb/CMakeFiles/CMakeScratch/TryCompile-RU5UFV

    Run Build Command(s):/usr/bin/cmake -E env VERBOSE=1 /usr/bin/gmake -f Makefile cmTC_65e36/fast && gmake[3]: Entering directory '/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-1717-g0f726187/rpm/el9/BUILD/ceph-19.0.0-1717-g0f726187/redhat-linux-build/src/rocksdb/CMakeFiles/CMakeScratch/TryCompile-RU5UFV'
    /usr/bin/gmake  -f CMakeFiles/cmTC_65e36.dir/build.make CMakeFiles/cmTC_65e36.dir/build
    gmake[4]: Entering directory '/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-1717-g0f726187/rpm/el9/BUILD/ceph-19.0.0-1717-g0f726187/redhat-linux-build/src/rocksdb/CMakeFiles/CMakeScratch/TryCompile-RU5UFV'
    Building C object CMakeFiles/cmTC_65e36.dir/testCCompiler.c.o
    /usr/bin/cc   -O2 -flto=thin -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security  -Wp,-D_GLIBCXX_ASSERTIONS --config /usr/lib/rpm/redhat/redhat-hardened-clang.cfg -fstack-protector-strong -m64 -march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection  -fPIE -o CMakeFiles/cmTC_65e36.dir/testCCompiler.c.o -c /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-1717-g0f726187/rpm/el9/BUILD/ceph-19.0.0-1717-g0f726187/redhat-linux-build/src/rocksdb/CMakeFiles/CMakeScratch/TryCompile-RU5UFV/testCCompiler.c
    cc: error: unrecognized command-line option ‘--config’; did you mean ‘-mpconfig’?
    gmake[4]: *** [CMakeFiles/cmTC_65e36.dir/build.make:78: CMakeFiles/cmTC_65e36.dir/testCCompiler.c.o] Error 1
```

where RocksDB tries to check C compiler -- /usr/bin/cc along with
the said CFLAGS, and fails to compile the test C program, because
GCC does not support this option.

so, in this change, let's pass the CMAKE_C_COMPILER as well.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/BuildRocksDB.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/modules/BuildRocksDB.cmake b/cmake/modules/BuildRocksDB.cmake
index e0208f6545b7..c1f4823963f2 100644
--- a/cmake/modules/BuildRocksDB.cmake
+++ b/cmake/modules/BuildRocksDB.cmake
@@ -24,6 +24,7 @@ function(build_rocksdb)
   endif()
 
   list(APPEND rocksdb_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
+  list(APPEND rocksdb_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
 
   list(APPEND rocksdb_CMAKE_ARGS -DWITH_SNAPPY=${SNAPPY_FOUND})
   if(SNAPPY_FOUND)

From ef1111a17084f6672fd40e366f92f8168a09d0a0 Mon Sep 17 00:00:00 2001
From: matt benjamin <mbenjamin@redhat.com>
Date: Sat, 2 Mar 2024 13:49:52 -0500
Subject: [PATCH 2096/2492] rgw_sigv4: handle '0;' as trailer-chunk boundary

https://tracker.ceph.com/issues/64676

Signed-off-by: matt benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_auth_s3.cc | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc
index 82b2524bd502..fd7fd07a54e1 100644
--- a/src/rgw/rgw_auth_s3.cc
+++ b/src/rgw/rgw_auth_s3.cc
@@ -1552,11 +1552,21 @@ bool AWSv4ComplMulti::complete()
     std::string_view expected_trailer_signature;
     std::string calculated_trailer_signature;
 
-    /* the trailer boundary is just "\r\n0" when we have no trailer
-     * signature */
+    /* I have seen variations in the 0-byte case, with and without
+     * ssl transport. I have observed "\r\n0;" but also "0;" in the
+     * trailer-signature case.  I have observed only "\r\n0" in the
+     * no-trailer-signature case--but assume "0" might be possible.
+     * The logic below handles all 4 cases. */
     if (tbuf_pos > sarrlen("\r\n0")) {
-      auto trailer_off = sarrlen("\r\n0");
-      if (*(trailer_vec.data() + trailer_off) == ';') {
+      const char* tv_data = trailer_vec.data();
+      auto trailer_off = 0;
+      if (*(tv_data + trailer_off) == '\r') {
+	trailer_off += 2;
+      }
+      if (*(tv_data + trailer_off) == '0') {
+	++trailer_off;
+      }
+      if (*(tv_data + trailer_off) == ';') {
 	++trailer_off;
       }
       const std::string_view sv_trailer(

From ccb851d2a431647124754e1252df66ed11d0e900 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 3 Mar 2024 20:28:00 +1000
Subject: [PATCH 2097/2492] doc/rados: remove PGcalc from docs

Remove mention of the "PG calc" tool from the documentation. I have
removed all mention of this in one fell swoop to help posterity restore
mention of this tool if we decide we need to do so.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/placement-groups.rst |  4 ----
 doc/rados/operations/pools.rst            | 19 ++++++++-----------
 doc/radosgw/pools.rst                     | 17 +++++++----------
 3 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index f9b87528b528..84dad884e22b 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -641,9 +641,6 @@ pools, each with 512 PGs on 10 OSDs, the OSDs will have to handle ~50,000 PGs
 each. This cluster will require significantly more resources and significantly
 more time for peering.
 
-For determining the optimal number of PGs per OSD, we recommend the `PGCalc`_
-tool.
-
 
 .. _setting the number of placement groups:
 
@@ -935,4 +932,3 @@ about it entirely (if it is too new to have a previous version). To mark the
 
 .. _Create a Pool: ../pools#createpool
 .. _Mapping PGs to OSDs: ../../../architecture#mapping-pgs-to-osds
-.. _pgcalc: https://old.ceph.com/pgcalc/
diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index afe9926589a4..e1770c8087a4 100644
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -18,15 +18,14 @@ Pools provide:
   <../erasure-code>`_, resilience is defined as the number of coding chunks
   (for example, ``m = 2`` in the default **erasure code profile**).
 
-- **Placement Groups**: You can set the number of placement groups (PGs) for
-  the pool. In a typical configuration, the target number of PGs is
-  approximately one hundred PGs per OSD. This provides reasonable balancing
-  without consuming excessive computing resources.  When setting up multiple
-  pools, be careful to set an appropriate number of PGs for each pool and for
-  the cluster as a whole. Each PG belongs to a specific pool: when multiple
-  pools use the same OSDs, make sure that the **sum** of PG replicas per OSD is
-  in the desired PG-per-OSD target range. To calculate an appropriate number of
-  PGs for your pools, use the `pgcalc`_ tool.
+- **Placement Groups**: The :ref:`autoscaler <pg-autoscaler>` sets the number
+  of placement groups (PGs) for the pool. In a typical configuration, the
+  target number of PGs is approximately one-hundred and fifty PGs per OSD. This
+  provides reasonable balancing without consuming excessive computing
+  resources. When setting up multiple pools, set an appropriate number of PGs
+  for each pool and for the cluster as a whole. Each PG belongs to a specific
+  pool: when multiple pools use the same OSDs, make sure that the **sum** of PG
+  replicas per OSD is in the desired PG-per-OSD target range. 
 
 - **CRUSH Rules**: When data is stored in a pool, the placement of the object
   and its replicas (or chunks, in the case of erasure-coded pools) in your
@@ -735,8 +734,6 @@ Managing pools that are flagged with ``--bulk``
 ===============================================
 See :ref:`managing_bulk_flagged_pools`.
 
-
-.. _pgcalc: https://old.ceph.com/pgcalc/
 .. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
 .. _Bloom Filter: https://en.wikipedia.org/wiki/Bloom_filter
 .. _setting the number of placement groups: ../placement-groups#set-the-number-of-placement-groups
diff --git a/doc/radosgw/pools.rst b/doc/radosgw/pools.rst
index bb1246c1fcac..acd6a6321145 100644
--- a/doc/radosgw/pools.rst
+++ b/doc/radosgw/pools.rst
@@ -11,16 +11,13 @@ multiple zones.
 Tuning
 ======
 
-When ``radosgw`` first tries to operate on a zone pool that does not
-exist, it will create that pool with the default values from
-``osd pool default pg num`` and ``osd pool default pgp num``. These defaults
-are sufficient for some pools, but others (especially those listed in
-``placement_pools`` for the bucket index and data) will require additional
-tuning. We recommend using the `Ceph Placement Group’s per Pool
-Calculator <https://old.ceph.com/pgcalc/>`__ to calculate a suitable number of
-placement groups for these pools. See
-`Pools <http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__
-for details on pool creation.
+When ``radosgw`` first tries to operate on a zone pool that does not exist, it
+will create that pool with the default values from ``osd pool default pg num``
+and ``osd pool default pgp num``. These defaults are sufficient for some pools,
+but others (especially those listed in ``placement_pools`` for the bucket index
+and data) will require additional tuning. See `Pools
+<http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__ for details
+on pool creation.
 
 .. _radosgw-pool-namespaces:
 

From 8fedb30c1bd6cd393becf88f8d2fe07ea9c6040c Mon Sep 17 00:00:00 2001
From: Gal Salomon <gal.salomon@gmail.com>
Date: Sun, 3 Mar 2024 14:44:49 +0200
Subject: [PATCH 2098/2492] using the it.length() for segment processing.
 s3select submodule

Signed-off-by: Gal Salomon <gal.salomon@gmail.com>
---
 src/rgw/rgw_s3select.cc | 11 +++++++----
 src/s3select            |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/rgw/rgw_s3select.cc b/src/rgw/rgw_s3select.cc
index bb2c3686792a..c8e887d59b01 100644
--- a/src/rgw/rgw_s3select.cc
+++ b/src/rgw/rgw_s3select.cc
@@ -410,7 +410,7 @@ int RGWSelectObj_ObjStore_S3::run_s3select_on_csv(const char* query, const char*
   } else if(m_header_info.compare("USE")==0) {
     csv.use_header_info=true;
   }
-  //m_s3_csv_object.set_external_debug_system(fp_debug_mesg);
+  m_s3_csv_object.set_external_debug_system(fp_debug_mesg);
   m_s3_csv_object.set_result_formatters(fp_s3select_result_format,fp_result_header_format);
   m_s3_csv_object.set_csv_query(&s3select_syntax, csv);
   if (s3select_syntax.get_error_description().empty() == false) {
@@ -865,20 +865,23 @@ int RGWSelectObj_ObjStore_S3::csv_processing(bufferlist& bl, off_t ofs, off_t le
         continue;
       }
 
-      if((ofs + len) > it.length()){
+
+      if(ofs > it.length()){
+      //safety check
 	ldpp_dout(this, 10) << "offset and length may cause invalid read: ofs = " << ofs << " len = " << len << " it.length() = " << it.length() << dendl;
 	ofs = 0;
-	len = it.length();
       }
 
     if(m_is_trino_request){
+      //TODO replace len with it.length() ? ; test Trino flow with compressed objects.
+      //is it possible to send get-by-ranges? in parallel?
       shape_chunk_per_trino_requests(&(it)[0], ofs, len); 
     }
 
     ldpp_dout(this, 10) << "s3select: chunk:  ofs = " << ofs << " len = " << len << " it.length() = " << it.length() << " m_object_size_for_processing = " << m_object_size_for_processing << dendl;
     
     m_aws_response_handler.update_processed_size(it.length());//NOTE : to run analysis to validate len is aligned with m_processed_bytes
-    status = run_s3select_on_csv(m_sql_query.c_str(), &(it)[0] + ofs, len);
+    status = run_s3select_on_csv(m_sql_query.c_str(), &(it)[0] + ofs, it.length());
     if (status<0) {
 	  return -EINVAL;
     }
diff --git a/src/s3select b/src/s3select
index 071146cf108d..232ac7061ec3 160000
--- a/src/s3select
+++ b/src/s3select
@@ -1 +1 @@
-Subproject commit 071146cf108d84119e4cd825834fd00f14b83497
+Subproject commit 232ac7061ec3ab52fb4a76264a11e3c23af2972d

From 945b18195465583334cb88ba69c12bbf42aedd6d Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 3 Mar 2024 09:45:56 +0000
Subject: [PATCH 2099/2492] src/seastar: update seastar submodule to fix FTBFS

See: https://github.com/ceph/seastar/commit/d382f247629a7349afc0cea495a1cd0502dc0b22

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/seastar | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/seastar b/src/seastar
index 1a2cd05b7bf5..09a44e0d403a 160000
--- a/src/seastar
+++ b/src/seastar
@@ -1 +1 @@
-Subproject commit 1a2cd05b7bf515f68841fa06ba9e3bdaecd20716
+Subproject commit 09a44e0d403a97db696837e75b4b61f592baf35b

From 70c1d79d1d0f52cd65541f59cedb00e39cba3f76 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Sun, 3 Mar 2024 13:43:10 +0000
Subject: [PATCH 2100/2492] mon/OSDMonitor: unify remove_pool_snap callers

No changes in behavior.

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/mon/OSDMonitor.cc | 21 +++++++++++++--------
 src/mon/OSDMonitor.h  |  4 ++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 3f1254cd7188..fe106a71d8e9 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -4570,6 +4570,17 @@ void OSDMonitor::send_incremental(epoch_t first,
   }
 }
 
+bool OSDMonitor::remove_pool_snap(std::string_view snapname,
+                                  pg_pool_t &pp, int64_t pool) {
+  snapid_t snapid = pp.snap_exists(snapname);
+  if (snapid) {
+    pp.remove_snap(snapid);
+    pending_inc.new_removed_snaps[pool].insert(snapid);
+    return true;
+  }
+  return false;
+};
+
 int OSDMonitor::get_version(version_t ver, bufferlist& bl)
 {
   return get_version(ver, mon.get_quorum_con_features(), bl);
@@ -13083,11 +13094,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       pp = &pending_inc.new_pools[pool];
       *pp = *p;
     }
-    snapid_t sn = pp->snap_exists(snapname.c_str());
-    if (sn) {
-      pp->remove_snap(sn);
+    if (remove_pool_snap(snapname, *pp, pool)) {
       pp->set_snap_epoch(pending_inc.epoch);
-      pending_inc.new_removed_snaps[pool].insert(sn);
       ss << "removed pool " << poolstr << " snap " << snapname;
     } else {
       ss << "already removed pool " << poolstr << " snap " << snapname;
@@ -14307,10 +14315,7 @@ bool OSDMonitor::prepare_pool_op(MonOpRequestRef op)
 
   case POOL_OP_DELETE_SNAP:
     {
-      snapid_t s = pp.snap_exists(m->name.c_str());
-      if (s) {
-	pp.remove_snap(s);
-	pending_inc.new_removed_snaps[m->pool].insert(s);
+      if (remove_pool_snap(m->name, pp, m->pool)) {
 	changed = true;
       }
     }
diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 08aa4f61783b..398daf9243e1 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -404,6 +404,10 @@ class OSDMonitor : public PaxosService,
   MOSDMap *build_incremental(epoch_t first, epoch_t last, uint64_t features);
   void send_full(MonOpRequestRef op);
   void send_incremental(MonOpRequestRef op, epoch_t first);
+
+  bool remove_pool_snap(std::string_view snapname,
+                        pg_pool_t &pp, int64_t pool);
+
 public:
   /**
    * Make sure the existing (up) OSDs support the given features

From 913b52d6eefc1c1c763c03f2e6e54cceae5601ff Mon Sep 17 00:00:00 2001
From: TomNewChao <chaotomzhu@gmail.com>
Date: Fri, 26 Jan 2024 11:49:05 +0800
Subject: [PATCH 2101/2492] fix review about mgr/dashboard/frontend:Ceph
 dashboard supports multiple languages

1.Set the default value of DASHBOARD_FRONTEND_LANGS to 'ALL'.
2.Fix bug about receive the `"ALL"` or `'ALL'`from process.env.DASHBOARD_FRONTEND_LANGS that is not equal 'ALL' in cd.js.
3.When we use the make-dist script to build the front-end UI to release the ceph, we should add DASHBOARD_FRONTEND_LANGS="ALL" to support multiple languages.

Signed-off-by: TomNewChao <chaotomzhu@gmail.com>
---
 CMakeLists.txt                                         |  2 +-
 ceph.spec.in                                           | 10 ----------
 make-dist                                              |  2 +-
 src/pybind/mgr/dashboard/frontend/cd.js                |  1 +
 .../frontend/cypress/e2e/ui/language.e2e-spec.ts       |  2 +-
 5 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78881cb0188d..11c8bee6a544 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -696,7 +696,7 @@ if(WITH_SYSTEM_NPM)
     message(FATAL_ERROR "Can't find npm.")
   endif()
 endif()
-set(DASHBOARD_FRONTEND_LANGS "" CACHE STRING
+set(DASHBOARD_FRONTEND_LANGS "ALL" CACHE STRING
   "List of comma separated ceph-dashboard frontend languages to build. \
   Use value `ALL` to build all languages")
 CMAKE_DEPENDENT_OPTION(WITH_MGR_ROOK_CLIENT "Enable the mgr's Rook support" ON
diff --git a/ceph.spec.in b/ceph.spec.in
index 4d9df44f6265..1fc998d014db 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -24,7 +24,6 @@
 %bcond_with zbd
 %bcond_with cmake_verbose_logging
 %bcond_without ceph_test_package
-%bcond_without mgr_dashboard_frontend_support_multi_language
 %ifarch s390
 %bcond_with tcmalloc
 %else
@@ -469,9 +468,6 @@ BuildRequires:  libnuma-devel
 %if 0%{?rhel} >= 8
 BuildRequires:  /usr/bin/pathfix.py
 %endif
-%if 0%{with mgr_dashboard_frontend_support_multi_language}
-BuildRequires:  npm
-%endif
 
 %description
 Ceph is a massively scalable, open-source, distributed storage system that runs
@@ -1353,13 +1349,7 @@ cmake .. \
     -DSYSTEMD_SYSTEM_UNIT_DIR:PATH=%{_unitdir} \
     -DWITH_MANPAGE:BOOL=ON \
     -DWITH_PYTHON3:STRING=%{python3_version} \
-%if 0%{with mgr_dashboard_frontend_support_multi_language}
-    -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=ON \
-    -DDASHBOARD_FRONTEND_LANGS:STRING="cs,de,es,fr,id,it,ja,ko,pl,zh-Hans,zh-Hant,pt" \
-    -DWITH_SYSTEM_NPM:BOOL=ON \
-%else
     -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF \
-%endif
 %if 0%{?suse_version}
     -DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF \
 %endif
diff --git a/make-dist b/make-dist
index 0ed52e573644..6fb8609bd8bb 100755
--- a/make-dist
+++ b/make-dist
@@ -140,7 +140,7 @@ build_dashboard_frontend() {
   echo "Building ceph-dashboard frontend with build:localize script";
   # we need to use "--" because so that "--configuration production"
   # survives accross all scripts redirections inside package.json
-  npm run build:localize -- --configuration production
+  DASHBOARD_FRONTEND_LANGS="ALL" npm run build:localize -- --configuration production
   deactivate
   cd $CURR_DIR
   rm -rf $TEMP_DIR
diff --git a/src/pybind/mgr/dashboard/frontend/cd.js b/src/pybind/mgr/dashboard/frontend/cd.js
index 34d0ce29f2d3..db92308d585f 100755
--- a/src/pybind/mgr/dashboard/frontend/cd.js
+++ b/src/pybind/mgr/dashboard/frontend/cd.js
@@ -46,6 +46,7 @@ function prepareLocales() {
   }
 
   let langs = process.env.DASHBOARD_FRONTEND_LANGS || '';
+  langs = langs.replace(/\"\'/g, '')
   if (langs == 'ALL') {
     logger(`Preparing build of all languages.`);
     return;
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/language.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/language.e2e-spec.ts
index fa20f0be5427..86b170d4320c 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/language.e2e-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/language.e2e-spec.ts
@@ -14,6 +14,6 @@ describe('Shared pages', () => {
 
   it('should check all available languages', () => {
     language.getLanguageBtn().click();
-    language.getAllLanguages().should('have.length', 1).should('contain.text', 'English');
+    language.getAllLanguages().should('have.length', 13).should('contain.text', 'English');
   });
 });

From 2d37276071e978f6bfebb1dd67ed5802db581371 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Tue, 6 Feb 2024 12:40:11 +0100
Subject: [PATCH 2102/2492] mgr/dashboard: add hardware status summary

On the landing page of the Dashboard, add the hardware status summary

Fixes:https://tracker.ceph.com/issues/64329
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../mgr/dashboard/controllers/hardware.py     | 21 ++++++
 .../dashboard/dashboard-v3.component.html     | 16 +++-
 .../dashboard/dashboard-v3.component.ts       | 35 ++++++++-
 .../app/shared/api/hardware.service.spec.ts   | 23 ++++++
 .../src/app/shared/api/hardware.service.ts    | 18 +++++
 .../card-row/card-row.component.html          | 64 +++++++++++++++-
 .../card-row/card-row.component.scss          | 15 ++++
 .../components/card-row/card-row.component.ts | 17 +++++
 .../src/app/shared/enum/hardware.enum.ts      |  8 ++
 .../src/app/shared/pipes/pipes.module.ts      |  7 +-
 .../app/shared/pipes/pluralize.pipe.spec.ts   |  8 ++
 .../src/app/shared/pipes/pluralize.pipe.ts    | 14 ++++
 src/pybind/mgr/dashboard/openapi.yaml         | 39 ++++++++++
 src/pybind/mgr/dashboard/services/hardware.py | 75 +++++++++++++++++++
 .../mgr/dashboard/services/orchestrator.py    |  8 ++
 15 files changed, 358 insertions(+), 10 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/controllers/hardware.py
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/enum/hardware.enum.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.ts
 create mode 100644 src/pybind/mgr/dashboard/services/hardware.py

diff --git a/src/pybind/mgr/dashboard/controllers/hardware.py b/src/pybind/mgr/dashboard/controllers/hardware.py
new file mode 100644
index 000000000000..72550ed195f2
--- /dev/null
+++ b/src/pybind/mgr/dashboard/controllers/hardware.py
@@ -0,0 +1,21 @@
+
+from typing import List, Optional
+
+from ..services.hardware import HardwareService
+from . import APIDoc, APIRouter, EndpointDoc, RESTController
+from ._version import APIVersion
+
+
+@APIRouter('/hardware')
+@APIDoc("Hardware management API", "Hardware")
+class Hardware(RESTController):
+
+    @RESTController.Collection('GET', version=APIVersion.EXPERIMENTAL)
+    @EndpointDoc("Retrieve a summary of the hardware health status")
+    def summary(self, categories: Optional[List[str]] = None, hostname: Optional[List[str]] = None):
+        """
+        Get the health status of as many hardware categories, or all of them if none is given
+        :param categories: The hardware type, all of them by default
+        :param hostname: The host to retrieve from, all of them by default
+        """
+        return HardwareService.get_summary(categories, hostname)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
index 4c290746b45b..bad69c50122b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.html
@@ -56,7 +56,9 @@
                      link="/hosts"
                      title="Host"
                      summaryType="simplified"
-                     *ngIf="healthData.hosts != null"></cd-card-row>
+                     *ngIf="healthData.hosts != null"
+                     [dropdownData]="(isHardwareEnabled$ | async) && (hardwareSummary$ | async)">
+        </cd-card-row>
         <!-- Monitors -->
         <cd-card-row [data]="healthData.mon_status.monmap.mons.length"
                      link="/monitor"
@@ -141,7 +143,7 @@
                 </ul>
               </ng-template>
 
-              <div class="d-flex flex-row">
+              <div class="d-flex flex-row col-md-3 ms-4">
                 <i *ngIf="healthData.health?.status"
                    [ngClass]="[healthData.health.status | healthIcon, icons.large2x]"
                    [ngStyle]="healthData.health.status | healthColor"
@@ -159,6 +161,16 @@
                       i18n>Cluster</span>
               </div>
             </div>
+
+            <div class="d-flex flex-column col-md-3">
+              <div *ngIf="hasHardwareError"
+                   class="d-flex flex-row">
+                <i class="text-danger"
+                   [ngClass]="[icons.danger, icons.large2x]"></i>
+                <span class="ms-2 mt-n1 lead"
+                      i18n>Hardware</span>
+              </div>
+            </div>
             <section class="footer alerts"
                      *ngIf="isAlertmanagerConfigured && prometheusAlertService.alerts.length">
               <div class="d-flex flex-wrap ms-4 me-4 mb-3 mt-3">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
index 3c44bd36a890..853eed2d695f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard/dashboard-v3.component.ts
@@ -1,8 +1,8 @@
 import { Component, OnDestroy, OnInit } from '@angular/core';
 
 import _ from 'lodash';
-import { Observable, Subscription } from 'rxjs';
-import { take } from 'rxjs/operators';
+import { BehaviorSubject, Observable, Subscription, of } from 'rxjs';
+import { switchMap, take } from 'rxjs/operators';
 
 import { HealthService } from '~/app/shared/api/health.service';
 import { OsdService } from '~/app/shared/api/osd.service';
@@ -24,6 +24,7 @@ import { PrometheusAlertService } from '~/app/shared/services/prometheus-alert.s
 import { OrchestratorService } from '~/app/shared/api/orchestrator.service';
 import { MgrModuleService } from '~/app/shared/api/mgr-module.service';
 import { AlertClass } from '~/app/shared/enum/health-icon.enum';
+import { HardwareService } from '~/app/shared/api/hardware.service';
 
 @Component({
   selector: 'cd-dashboard-v3',
@@ -69,6 +70,12 @@ export class DashboardV3Component extends PrometheusListHelper implements OnInit
   telemetryEnabled: boolean;
   telemetryURL = 'https://telemetry-public.ceph.com/';
   origin = window.location.origin;
+  hardwareHealth: any;
+  hardwareEnabled: boolean = false;
+  hasHardwareError: boolean = false;
+  isHardwareEnabled$: Observable<boolean>;
+  hardwareSummary$: Observable<any>;
+  hardwareSubject = new BehaviorSubject<any>([]);
 
   constructor(
     private summaryService: SummaryService,
@@ -80,7 +87,8 @@ export class DashboardV3Component extends PrometheusListHelper implements OnInit
     public prometheusService: PrometheusService,
     private mgrModuleService: MgrModuleService,
     private refreshIntervalService: RefreshIntervalService,
-    public prometheusAlertService: PrometheusAlertService
+    public prometheusAlertService: PrometheusAlertService,
+    private hardwareService: HardwareService
   ) {
     super(prometheusService);
     this.permissions = this.authStorageService.getPermissions();
@@ -89,9 +97,21 @@ export class DashboardV3Component extends PrometheusListHelper implements OnInit
 
   ngOnInit() {
     super.ngOnInit();
+    this.isHardwareEnabled$ = this.getHardwareConfig();
+    this.hardwareSummary$ = this.hardwareSubject.pipe(
+      switchMap(() =>
+        this.hardwareService.getSummary().pipe(
+          switchMap((data: any) => {
+            this.hasHardwareError = data.host.flawed;
+            return of(data);
+          })
+        )
+      )
+    );
     this.interval = this.refreshIntervalService.intervalData$.subscribe(() => {
       this.getHealth();
       this.getCapacityCardData();
+      if (this.hardwareEnabled) this.hardwareSubject.next([]);
     });
     this.getPrometheusData(this.prometheusService.lastHourDateObject);
     this.getDetailsCardData();
@@ -163,4 +183,13 @@ export class DashboardV3Component extends PrometheusListHelper implements OnInit
   trackByFn(index: any) {
     return index;
   }
+
+  getHardwareConfig(): Observable<any> {
+    return this.mgrModuleService.getConfig('cephadm').pipe(
+      switchMap((resp: any) => {
+        this.hardwareEnabled = resp?.hw_monitoring;
+        return of(resp?.hw_monitoring);
+      })
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.spec.ts
new file mode 100644
index 000000000000..b9deac3fc8ba
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.spec.ts
@@ -0,0 +1,23 @@
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { TestBed } from '@angular/core/testing';
+
+import { configureTestBed } from '~/testing/unit-test-helper';
+import { HardwareService } from './hardware.service';
+
+describe('HardwareService', () => {
+  let service: HardwareService;
+
+  configureTestBed({
+    providers: [HardwareService],
+    imports: [HttpClientTestingModule]
+  });
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(HardwareService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.ts
new file mode 100644
index 000000000000..3238493ebe5e
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/hardware.service.ts
@@ -0,0 +1,18 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class HardwareService {
+  baseURL = 'api/hardware';
+
+  constructor(private http: HttpClient) {}
+
+  getSummary(category: string[] = []): any {
+    return this.http.get<any>(`${this.baseURL}/summary`, {
+      params: { categories: category },
+      headers: { Accept: 'application/vnd.ceph.api.v0.1+json' }
+    });
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.html
index 4e193717252c..a1e3e6b0b371 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.html
@@ -1,6 +1,6 @@
 <hr>
 <li class="list-group-item">
-  <div class="d-flex pl-1 pb-2 pt-2">
+  <div class="d-flex pl-1 pb-2 pt-2 position-relative">
     <div class="ms-4 me-auto">
       <a [routerLink]="link"
          *ngIf="link && total > 0; else noLinkTitle"
@@ -12,7 +12,7 @@
         <ng-template ngPluralCase="other">{{ title }}s</ng-template>
       </a>
     </div>
-    <span class="me-3">
+    <span class="me-4">
       <ng-container [ngSwitch]="summaryType">
         <ng-container *ngSwitchCase="'iscsi'">
           <ng-container *ngTemplateOutlet="iscsiSummary"></ng-container>
@@ -28,9 +28,22 @@
         </ng-container>
       </ng-container>
     </span>
+    <span *ngIf="dropdownData && dropdownData.total.total.total > 0"
+          class="position-absolute end-0 me-2">
+      <a (click)="toggleDropdown()"
+         class="dropdown-toggle"
+         [attr.aria-expanded]="dropdownToggled"
+         aria-controls="row-dropdwon"
+         role="button"></a>
+    </span>
   </div>
 </li>
 
+<div *ngIf="dropdownToggled">
+  <hr>
+  <ng-container *ngTemplateOutlet="dropdownTemplate"></ng-container>
+</div>
+
 <ng-template #defaultSummary>
   <span *ngIf="data.success || data.categoryPgAmount?.clean || (data.success === 0 && data.total === 0)">
     <span *ngIf="data.success || (data.success === 0 && data.total === 0)">
@@ -153,11 +166,23 @@
 </ng-template>
 
 <ng-template #simplifiedSummary>
-  <span>
+  <span *ngIf="!dropdownTotalError else showErrorNum">
     {{ data }}
     <i class="text-success"
        [ngClass]="[icons.success]"></i>
   </span>
+  <ng-template #showErrorNum>
+    <span *ngIf="data - dropdownTotalError  > 0">
+      {{ data - dropdownTotalError  }}
+    <i class="text-success"
+       [ngClass]="[icons.success]"></i>
+    </span>
+    <span>
+      {{ dropdownTotalError  }}
+      <i class="text-danger"
+         [ngClass]="[icons.danger]"></i>
+    </span>
+  </ng-template>
 </ng-template>
 
 <ng-template #noLinkTitle>
@@ -169,3 +194,36 @@
     <ng-template ngPluralCase="other">{{ title }}s</ng-template>
   </span>
 </ng-template>
+
+<ng-template #dropdownTemplate>
+  <ng-container *ngFor="let data of dropdownData?.total.category | keyvalue">
+    <li class="list-group-item">
+      <div class="d-flex pb-2 pt-2">
+        <div class="ms-5 me-auto">
+          <span *ngIf="data.value.total"
+                [ngPlural]="data.value.total"
+                i18n>
+              {{ data.value.total }}
+            <ng-template ngPluralCase="=0">{{ hwNames[data.key] }}</ng-template>
+            <ng-template ngPluralCase="=1">{{ hwNames[data.key] }}</ng-template>
+            <ng-template ngPluralCase="other">{{ hwNames[data.key] | pluralize }}</ng-template>
+          </span>
+        </div>
+        <span [ngClass]="data.value.error ? 'me-2' : 'me-4'">
+          {{ data.value.ok }}
+          <i class="text-success"
+             *ngIf="data.value.ok"
+             [ngClass]="[icons.success]">
+          </i>
+        </span>
+        <span *ngIf="data.value.error"
+              class="me-4 ms-2">
+              {{ data.value.error }}
+          <i class="text-danger"
+             [ngClass]="[icons.danger]">
+          </i>
+        </span>
+      </div>
+    </li>
+  </ng-container>
+</ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.scss
index 29901b832d3a..f93d6313aac3 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.scss
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.scss
@@ -2,3 +2,18 @@
   border: 0;
   font-size: 14px;
 }
+
+a.dropdown-toggle {
+  &::after {
+    border: 0;
+    content: '\f054';
+    font-family: 'ForkAwesome';
+    font-size: 1rem;
+    margin-top: 0.15rem;
+    transition: transform 0.3s ease-in-out;
+  }
+
+  &[aria-expanded='true']::after {
+    transform: rotate(90deg);
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.ts
index 90c939160eb9..d977e905f531 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-row/card-row.component.ts
@@ -1,5 +1,6 @@
 import { Component, Input, OnChanges } from '@angular/core';
 import { Icons } from '~/app/shared/enum/icons.enum';
+import { HardwareNameMapping } from '~/app/shared/enum/hardware.enum';
 
 @Component({
   selector: 'cd-card-row',
@@ -19,8 +20,14 @@ export class CardRowComponent implements OnChanges {
   @Input()
   summaryType = 'default';
 
+  @Input()
+  dropdownData: any;
+
+  hwNames = HardwareNameMapping;
   icons = Icons;
   total: number;
+  dropdownTotalError: number = 0;
+  dropdownToggled: boolean = false;
 
   ngOnChanges(): void {
     if (this.data.total || this.data.total === 0) {
@@ -30,5 +37,15 @@ export class CardRowComponent implements OnChanges {
     } else {
       this.total = this.data;
     }
+
+    if (this.dropdownData) {
+      if (this.title == 'Host') {
+        this.dropdownTotalError = this.dropdownData.host.flawed;
+      }
+    }
+  }
+
+  toggleDropdown(): void {
+    this.dropdownToggled = !this.dropdownToggled;
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/hardware.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/hardware.enum.ts
new file mode 100644
index 000000000000..7956dfa5d7c0
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/hardware.enum.ts
@@ -0,0 +1,8 @@
+export enum HardwareNameMapping {
+  memory = 'Memory',
+  storage = 'Drive',
+  processors = 'CPU',
+  network = 'Network',
+  power = 'Power supply',
+  fans = 'Fan module'
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pipes.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pipes.module.ts
index b5267aa71216..53f8f9f309f4 100755
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pipes.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pipes.module.ts
@@ -37,6 +37,7 @@ import { TruncatePipe } from './truncate.pipe';
 import { UpperFirstPipe } from './upper-first.pipe';
 import { OctalToHumanReadablePipe } from './octal-to-human-readable.pipe';
 import { PathPipe } from './path.pipe';
+import { PluralizePipe } from './pluralize.pipe';
 
 @NgModule({
   imports: [CommonModule],
@@ -76,7 +77,8 @@ import { PathPipe } from './path.pipe';
     MdsSummaryPipe,
     OsdSummaryPipe,
     OctalToHumanReadablePipe,
-    PathPipe
+    PathPipe,
+    PluralizePipe
   ],
   exports: [
     ArrayPipe,
@@ -114,7 +116,8 @@ import { PathPipe } from './path.pipe';
     MdsSummaryPipe,
     OsdSummaryPipe,
     OctalToHumanReadablePipe,
-    PathPipe
+    PathPipe,
+    PluralizePipe
   ],
   providers: [
     ArrayPipe,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.spec.ts
new file mode 100644
index 000000000000..72ba02025a93
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.spec.ts
@@ -0,0 +1,8 @@
+import { PluralizePipe } from './pluralize.pipe';
+
+describe('PluralizePipe', () => {
+  it('create an instance', () => {
+    const pipe = new PluralizePipe();
+    expect(pipe).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.ts
new file mode 100644
index 000000000000..c4035ad28499
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/pluralize.pipe.ts
@@ -0,0 +1,14 @@
+import { Pipe, PipeTransform } from '@angular/core';
+
+@Pipe({
+  name: 'pluralize'
+})
+export class PluralizePipe implements PipeTransform {
+  transform(value: string): string {
+    if (value.endsWith('y')) {
+      return value.slice(0, -1) + 'ies';
+    } else {
+      return value + 's';
+    }
+  }
+}
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index ad4d379fffbb..23d8a66013b6 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -4715,6 +4715,43 @@ paths:
       - jwt: []
       tags:
       - Grafana
+  /api/hardware/summary:
+    get:
+      description: "\n        Get the health status of as many hardware categories,\
+        \ or all of them if none is given\n        :param categories: The hardware\
+        \ type, all of them by default\n        :param hostname: The host to retrieve\
+        \ from, all of them by default\n        "
+      parameters:
+      - allowEmptyValue: true
+        in: query
+        name: categories
+        schema:
+          type: string
+      - allowEmptyValue: true
+        in: query
+        name: hostname
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v0.1+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Retrieve a summary of the hardware health status
+      tags:
+      - Hardware
   /api/health/full:
     get:
       parameters: []
@@ -14029,6 +14066,8 @@ tags:
   name: FeatureTogglesEndpoint
 - description: Grafana Management API
   name: Grafana
+- description: Hardware management API
+  name: Hardware
 - description: Display Detailed Cluster health Status
   name: Health
 - description: Get Host Details
diff --git a/src/pybind/mgr/dashboard/services/hardware.py b/src/pybind/mgr/dashboard/services/hardware.py
new file mode 100644
index 000000000000..df2266443388
--- /dev/null
+++ b/src/pybind/mgr/dashboard/services/hardware.py
@@ -0,0 +1,75 @@
+
+
+from typing import Any, Dict, List, Optional
+
+from ..exceptions import DashboardException
+from ..services.orchestrator import OrchClient
+
+
+class HardwareService(object):
+
+    @staticmethod
+    def get_summary(categories: Optional[List[str]] = None,
+                    hostname: Optional[List[str]] = None):
+        total_count = {'total': 0, 'ok': 0, 'error': 0}
+
+        output: Dict[str, Any] = {
+            'total': {
+                'category': {},
+                'total': {}
+            },
+            'host': {
+                'flawed': 0
+            }
+        }
+
+        categories = HardwareService.validate_categories(categories)
+
+        orch_hardware_instance = OrchClient.instance().hardware
+        for category in categories:
+            data = orch_hardware_instance.common(category, hostname)
+            category_total = {
+                'total': sum(len(items) for items in data.values()),
+                'ok': sum(item['status']['health'] == 'OK' for items in data.values()
+                          for item in items.values()),
+                'error': 0
+            }
+
+            for host, items in data.items():
+                output['host'].setdefault(host, {'flawed': False})
+                if not output['host'][host]['flawed']:
+                    output['host'][host]['flawed'] = any(
+                        item['status']['health'] != 'OK' for item in items.values())
+
+            category_total['error'] = category_total['total'] - category_total['ok']
+            output['total']['category'].setdefault(category, {})
+            output['total']['category'][category] = category_total
+
+            total_count['total'] += category_total['total']
+            total_count['ok'] += category_total['ok']
+            total_count['error'] += category_total['error']
+
+        output['total']['total'] = total_count
+
+        output['host']['flawed'] = sum(1 for host in output['host']
+                                       if host != 'flawed' and output['host'][host]['flawed'])
+
+        return output
+
+    @staticmethod
+    def validate_categories(categories: Optional[List[str]]) -> List[str]:
+        categories_list = ['memory', 'storage', 'processors',
+                           'network', 'power', 'fans']
+
+        if isinstance(categories, str):
+            categories = [categories]
+        elif categories is None:
+            categories = categories_list
+        elif not isinstance(categories, list):
+            raise DashboardException(msg=f'{categories} is not a list',
+                                     component='Hardware')
+        if not all(item in categories_list for item in categories):
+            raise DashboardException(msg=f'Invalid category, there is no {categories}',
+                                     component='Hardware')
+
+        return categories
diff --git a/src/pybind/mgr/dashboard/services/orchestrator.py b/src/pybind/mgr/dashboard/services/orchestrator.py
index e49ab80bfc5d..97776dec335c 100644
--- a/src/pybind/mgr/dashboard/services/orchestrator.py
+++ b/src/pybind/mgr/dashboard/services/orchestrator.py
@@ -200,6 +200,13 @@ def stop(self) -> str:
         return self.api.upgrade_stop()
 
 
+class HardwareManager(ResourceManager):
+
+    @wait_api_result
+    def common(self, category: str, hostname: Optional[List[str]] = None) -> str:
+        return self.api.node_proxy_common(category, hostname=hostname)
+
+
 class OrchClient(object):
 
     _instance = None
@@ -220,6 +227,7 @@ def __init__(self):
         self.osds = OsdManager(self.api)
         self.daemons = DaemonManager(self.api)
         self.upgrades = UpgradeManager(self.api)
+        self.hardware = HardwareManager(self.api)
 
     def available(self, features: Optional[List[str]] = None) -> bool:
         available = self.status()['available']

From 90393de9690b3e2234f7bd0ed0fb91c39e15e818 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Wed, 14 Feb 2024 00:37:51 -0500
Subject: [PATCH 2103/2492] mds: relax certain asserts in mdlog replay thread

The calls to journaler->is_readable() and journaler->get_error()
in MDLog::_replay_thread() will drop Journaler::lock between
invocations, so, theoretically, its possible that the initial check:

  // loop
  int r = 0;
  while (1) {
    // wait for read?
    while (!journaler->is_readable() &&
       journaler->get_read_pos() < journaler->get_write_pos() &&
       !journaler->get_error()) {
      C_SaferCond readable_waiter;
      journaler->wait_for_readable(&readable_waiter);
      r = readable_waiter.wait();
    }
    if (journaler->get_error()) {
      r = journaler->get_error();
      dout(0) << "_replay journaler got error " << r << ", aborting" << dendl;

journaler->is_readable() returned true, thereby breaking out of
the (inner) while loop and by passing the journaler->get_error()
check, and by the time this hits the next set of checks:

    if (!journaler->is_readable() &&
    journaler->get_read_pos() == journaler->get_write_pos())
      break;

    ceph_assert(journaler->is_readable() || mds->is_daemon_stopping());

It's possible that the journal is unreadable due to some error that
happened during prefetch. In short, these checks are racy.

So, remove these racy assert check along with journaler->is_readable()
check when validating the journal end and rely on the next iteration
of reading the journal for error handling.

Fixes: http://tracker.ceph.com/issues/57048
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/mds/MDLog.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/mds/MDLog.cc b/src/mds/MDLog.cc
index 5b1146ccd9d5..cd274f8edc4e 100644
--- a/src/mds/MDLog.cc
+++ b/src/mds/MDLog.cc
@@ -1403,11 +1403,10 @@ void MDLog::_replay_thread()
       break;
     }
 
-    if (!journaler->is_readable() &&
-	journaler->get_read_pos() == journaler->get_write_pos())
+    if (journaler->get_read_pos() == journaler->get_write_pos()) {
+      dout(10) << "_replay: read_pos == write_pos" << dendl;
       break;
-    
-    ceph_assert(journaler->is_readable() || mds->is_daemon_stopping());
+    }
     
     // read it
     uint64_t pos = journaler->get_read_pos();

From 4c62ae3dccabfd96a50472b05911b4f9cd4743ab Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Mon, 4 Mar 2024 11:15:52 +0800
Subject: [PATCH 2104/2492] osd/SnapManager: fix std::string constructed from 0

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/osd/SnapMapper.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index e79bc5d450a2..fa31cd76a368 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -147,7 +147,7 @@ int OSDriver::get_next_or_current(
     ch, hoid, FuturizedStore::Shard::omap_keys_t{key}
   ).safe_then([&key, next_or_current] (FuturizedStore::Shard::omap_values_t&& vals) {
     assert(vals.size() == 1);
-    *next_or_current = std::make_pair(key, std::move(vals[0]));
+    *next_or_current = std::make_pair(key, std::move(vals.begin()->second));
     return 0;
   }, FuturizedStore::Shard::read_errorator::all_same_way(
     [next_or_current, &key, this] {

From 61c7d6b5483851e90e60d3a82b61740e7df559e6 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Sat, 2 Mar 2024 17:36:33 +0800
Subject: [PATCH 2105/2492] crimson/os/seastore/async_cleaner: add comments to
 clarify what kind of extents would be considered dead by the cleaner

This could prevent future developers from rendering the cleaner abnormal
unexpectedly

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/async_cleaner.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc
index fd043f4ea35e..7fe6237eea36 100644
--- a/src/crimson/os/seastore/async_cleaner.cc
+++ b/src/crimson/os/seastore/async_cleaner.cc
@@ -1057,6 +1057,15 @@ SegmentCleaner::do_reclaim_space(
     std::size_t &reclaimed,
     std::size_t &runs)
 {
+  // Extents satisfying any of the following requirements
+  // are considered DEAD:
+  // 1. can't find the corresponding mapping in both the
+  // 	backref tree and the backref cache;
+  // 2. the extent is logical, but its lba mapping doesn't
+  // 	exist in the lba tree or the lba mapping in the lba
+  // 	tree doesn't match the extent's paddr
+  // 3. the extent is physical and doesn't exist in the
+  // 	lba tree, backref tree or backref cache;
   return repeat_eagain([this, &backref_extents,
                         &pin_list, &reclaimed, &runs] {
     reclaimed = 0;

From 9da81b87ca161370c869b49f5d4e6b50edd54a0e Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 4 Mar 2024 20:41:16 +1000
Subject: [PATCH 2106/2492] doc/rados: link to pg setting commands

Link to the instructions for manually setting the number of PGs per
pool, from the mention of placement groups. These instructions are
included here in response to a request from Ronen Friedman on the
occasion of the removal of links to the PGcalc (see
https://github.com/ceph/ceph/pull/55899#pullrequestreview-1912940118).

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/pools.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index e1770c8087a4..c3fe3b7d82ee 100644
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -25,7 +25,10 @@ Pools provide:
   resources. When setting up multiple pools, set an appropriate number of PGs
   for each pool and for the cluster as a whole. Each PG belongs to a specific
   pool: when multiple pools use the same OSDs, make sure that the **sum** of PG
-  replicas per OSD is in the desired PG-per-OSD target range. 
+  replicas per OSD is in the desired PG-per-OSD target range. See :ref:`Setting
+  the Number of Placement Groups <setting the number of placement groups>` for
+  instructions on how to manually set the number of placement groups per pool
+  (this procedure works only when the autoscaler is not used).
 
 - **CRUSH Rules**: When data is stored in a pool, the placement of the object
   and its replicas (or chunks, in the case of erasure-coded pools) in your

From eff0a1d2aeab269e57b2bcf9e9bef48ffa6b5d07 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Tue, 24 Oct 2023 16:13:43 +0300
Subject: [PATCH 2107/2492] qa/tasks/cephfs: give the tests more time to run
 heavy fs workloads

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 qa/tasks/cephfs/mount.py          | 13 +++++++------
 qa/tasks/cephfs/test_snapshots.py |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index be1ef6ea428e..5263c5bfac47 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -761,17 +761,18 @@ def create_destroy(self):
             'rm', '-f', os.path.join(self.hostfs_mntpt, filename)
         ])
 
-    def _run_python(self, pyscript, py_version='python3', sudo=False):
+    def _run_python(self, pyscript, py_version='python3', sudo=False, timeout=None):
         args, omit_sudo = [], True
         if sudo:
             args.append('sudo')
             omit_sudo = False
-        args += ['stdin-killer', '--', py_version, '-c', pyscript]
+        timeout_args = ['--timeout', "%d" % timeout] if timeout is not None else []
+        args += ['stdin-killer', *timeout_args, '--', py_version, '-c', pyscript]
         return self.client_remote.run(args=args, wait=False, stdin=run.PIPE,
                                       stdout=StringIO(), omit_sudo=omit_sudo)
 
-    def run_python(self, pyscript, py_version='python3', sudo=False):
-        p = self._run_python(pyscript, py_version, sudo=sudo)
+    def run_python(self, pyscript, py_version='python3', sudo=False, timeout=None):
+        p = self._run_python(pyscript, py_version, sudo=sudo, timeout=timeout)
         p.wait()
         return p.stdout.getvalue().strip()
 
@@ -1243,7 +1244,7 @@ def write_test_pattern(self, filename, size):
             size=size
         )))
 
-    def validate_test_pattern(self, filename, size):
+    def validate_test_pattern(self, filename, size, timeout=None):
         log.info("Validating {0} bytes from {1}".format(size, filename))
         # Use sudo because cephfs-data-scan may recreate the file with owner==root
         return self.run_python(dedent("""
@@ -1262,7 +1263,7 @@ def validate_test_pattern(self, filename, size):
         """.format(
             path=os.path.join(self.hostfs_mntpt, filename),
             size=size
-        )), sudo=True)
+        )), sudo=True, timeout=timeout)
 
     def open_n_background(self, fs_path, count):
         """
diff --git a/qa/tasks/cephfs/test_snapshots.py b/qa/tasks/cephfs/test_snapshots.py
index a9639a7ebbca..4cca133c3182 100644
--- a/qa/tasks/cephfs/test_snapshots.py
+++ b/qa/tasks/cephfs/test_snapshots.py
@@ -376,7 +376,7 @@ def test_multimds_mksnap(self):
         self.mount_a.write_test_pattern("d0/d1/file_a", 8 * 1024 * 1024)
         self.mount_a.run_shell(["mkdir", "d0/.snap/s1"])
         self.mount_a.run_shell(["rm", "-f", "d0/d1/file_a"])
-        self.mount_a.validate_test_pattern("d0/.snap/s1/d1/file_a", 8 * 1024 * 1024)
+        self.mount_a.validate_test_pattern("d0/.snap/s1/d1/file_a", 8 * 1024 * 1024, timeout=20)
 
         self.mount_a.run_shell(["rmdir", "d0/.snap/s1"])
         self.mount_a.run_shell(["rm", "-rf", "d0"])

From 71f12805055996d01599023f1caa09a2c95cedc8 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Sun, 3 Dec 2023 10:49:19 +0200
Subject: [PATCH 2108/2492] common/Cond: make C_SaferCond private members
 protected to facilitate inheritance

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/common/Cond.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/common/Cond.h b/src/common/Cond.h
index f41d0bf40702..2ea4b9e2de95 100644
--- a/src/common/Cond.h
+++ b/src/common/Cond.h
@@ -74,6 +74,7 @@ class C_SafeCond : public Context {
  * until wait() returns.
  */
 class C_SaferCond : public Context {
+protected:
   ceph::mutex lock;  ///< Mutex to take
   ceph::condition_variable cond;     ///< Cond to signal
   bool done = false; ///< true after finish() has been called

From 641279c4d0feb1fd18fe0544e40c86c4b0ada38a Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 8 Feb 2024 13:39:31 +0200
Subject: [PATCH 2109/2492] encoding: add emplace variants for map dencoders

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/include/encoding.h | 59 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/include/encoding.h b/src/include/encoding.h
index 08c67c33eecf..575580f41a8f 100644
--- a/src/include/encoding.h
+++ b/src/include/encoding.h
@@ -1095,6 +1095,22 @@ inline std::enable_if_t<!t_traits::supported || !u_traits::supported>
     decode(m[k], p);
   }
 }
+template <std::move_constructible T, std::move_constructible U, class Comp, class Alloc,
+    typename t_traits, typename u_traits>
+inline std::enable_if_t<!t_traits::supported || !u_traits::supported>
+decode(std::map<T, U, Comp, Alloc>& m, bufferlist::const_iterator& p)
+{
+  __u32 n;
+  decode(n, p);
+  m.clear();
+  while (n--) {
+    T k;
+    U v;
+    decode(k, p);
+    decode(v, p);
+    m.emplace(std::move(k), std::move(v));
+  }
+}
 template<class T, class U, class Comp, class Alloc>
 inline void decode_noclear(std::map<T,U,Comp,Alloc>& m, bufferlist::const_iterator& p)
 {
@@ -1106,6 +1122,19 @@ inline void decode_noclear(std::map<T,U,Comp,Alloc>& m, bufferlist::const_iterat
     decode(m[k], p);
   }
 }
+template<std::move_constructible T, std::move_constructible U, class Comp, class Alloc>
+inline void decode_noclear(std::map<T,U,Comp,Alloc>& m, bufferlist::const_iterator& p)
+{
+  __u32 n;
+  decode(n, p);
+  while (n--) {
+    T k;
+    U v;
+    decode(k, p);
+    decode(v, p);
+    m.emplace(std::move(k), std::move(v));
+  }
+}
 template<class T, class U, class Comp, class Alloc,
 	 typename t_traits, typename u_traits>
 inline std::enable_if_t<!t_traits::supported || !u_traits::supported>
@@ -1139,6 +1168,21 @@ inline std::enable_if_t<!t_traits::supported || !u_traits::supported>
   }
 }
 
+template <std::move_constructible T, std::move_constructible U, class Comp, class Alloc,
+    typename t_traits, typename u_traits>
+inline std::enable_if_t<!t_traits::supported || !u_traits::supported>
+decode_nohead(int n, std::map<T, U, Comp, Alloc>& m, bufferlist::const_iterator& p)
+{
+  m.clear();
+  while (n--) {
+    T k;
+    U v;
+    decode(k, p);
+    decode(v, p);
+    m.emplace(std::move(k), std::move(v));
+  }
+}
+
 // boost::container::flat-map
 template<class T, class U, class Comp, class Alloc,
 	 typename t_traits, typename u_traits>
@@ -1290,6 +1334,21 @@ inline void decode(unordered_map<T,U,Hash,Pred,Alloc>& m, bufferlist::const_iter
   }
 }
 
+template <std::move_constructible T, std::move_constructible U, class Hash, class Pred, class Alloc>
+inline void decode(unordered_map<T, U, Hash, Pred, Alloc>& m, bufferlist::const_iterator& p)
+{
+  __u32 n;
+  decode(n, p);
+  m.clear();
+  while (n--) {
+    T k;
+    U v;
+    decode(k, p);
+    decode(v, p);
+    m.emplace(std::move(k), std::move(v));
+  }
+}
+
 // ceph::unordered_set
 template<class T, class Hash, class Pred, class Alloc>
 inline void encode(const ceph::unordered_set<T,Hash,Pred,Alloc>& m, bufferlist& bl)

From 60cd6d117163ffc635ebe9577bf4748d757efcee Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 8 Feb 2024 14:06:30 +0200
Subject: [PATCH 2110/2492] mds: MDSRank.cc: return status from
 `send_message_mds`

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/MDSRank.cc | 12 ++++++------
 src/mds/MDSRank.h  |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 6f8439470a65..9cedeea1b136 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1450,15 +1450,15 @@ class C_MDS_RetrySendMessageMDS : public MDSInternalContext {
 };
 
 
-void MDSRank::send_message_mds(const ref_t<Message>& m, mds_rank_t mds)
+int MDSRank::send_message_mds(const ref_t<Message>& m, mds_rank_t mds)
 {
   if (!mdsmap->is_up(mds)) {
     dout(10) << "send_message_mds mds." << mds << " not up, dropping " << *m << dendl;
-    return;
+    return ENOENT;
   } else if (mdsmap->is_bootstrapping(mds)) {
     dout(5) << __func__ << "mds." << mds << " is bootstrapping, deferring " << *m << dendl;
     wait_for_bootstrapped_peer(mds, new C_MDS_RetrySendMessageMDS(this, mds, m));
-    return;
+    return 0;
   }
 
   // send mdsmap first?
@@ -1470,12 +1470,12 @@ void MDSRank::send_message_mds(const ref_t<Message>& m, mds_rank_t mds)
   }
 
   // send message
-  send_message_mds(m, addrs);
+  return send_message_mds(m, addrs);
 }
 
-void MDSRank::send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &addr)
+int MDSRank::send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &addr)
 {
-  messenger->send_to_mds(ref_t<Message>(m).detach(), addr);
+  return messenger->send_to_mds(ref_t<Message>(m).detach(), addr);
 }
 
 void MDSRank::forward_message_mds(const MDRequestRef& mdr, mds_rank_t mds)
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index adebb9b4d08d..f8e186b56c73 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -309,8 +309,8 @@ class MDSRank {
 
     double get_dispatch_queue_max_age(utime_t now) const;
 
-    void send_message_mds(const ref_t<Message>& m, mds_rank_t mds);
-    void send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &addr);
+    int send_message_mds(const ref_t<Message>& m, mds_rank_t mds);
+    int send_message_mds(const ref_t<Message>& m, const entity_addrvec_t &addr);
     void forward_message_mds(const MDRequestRef& mdr, mds_rank_t mds);
     void send_message_client_counted(const ref_t<Message>& m, client_t client);
     void send_message_client_counted(const ref_t<Message>& m, Session* session);

From c1c884212f45b387ba54215b6f12545572066ce2 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 22 Jan 2024 14:37:44 +0200
Subject: [PATCH 2111/2492] common/Timer.cc: improve debug messages from the
 timer_thread

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/common/Timer.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/common/Timer.cc b/src/common/Timer.cc
index 2a3277a27659..a7c35fb8ef52 100644
--- a/src/common/Timer.cc
+++ b/src/common/Timer.cc
@@ -102,6 +102,7 @@ void CommonSafeTimer<Mutex>::timer_thread()
       }
       #endif
 
+      ldout(cct, 20) << "timer_thread going to execute and remove the top of a schedule sized " << schedule.size() << dendl;
       Context *callback = p->second;
       events.erase(callback);
       schedule.erase(p);
@@ -120,10 +121,11 @@ void CommonSafeTimer<Mutex>::timer_thread()
     if (!safe_callbacks && stopping)
       break;
 
-    ldout(cct,20) << "timer_thread going to sleep" << dendl;
     if (schedule.empty()) {
+      ldout(cct, 20) << "timer_thread going to sleep with an empty schedule" << dendl;
       cond.wait(l);
     } else {
+      ldout(cct, 20) << "timer_thread going to sleep with a schedule size " << schedule.size() << dendl;
       auto when = schedule.begin()->first;
       cond.wait_until(l, when);
     }

From 0e61c44238ef5ec1d2b0406600a4ce981b07f87a Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Tue, 31 Oct 2023 14:46:55 +0200
Subject: [PATCH 2112/2492] mds/quiesce: QuiesceDb.h and QuiesceDbManager with
 tests

Quiesce DB is one of the components of the "Consistent Snapshots" epic.
The solution is discussed in a slide deck available for viewing to @redhat users:
https://docs.google.com/presentation/d/1wE3-e9AAme7Q3qmeshUSthJoQGw7-fKTrtS9PsdAIVo/edit?usp=sharing

This commit is focusing on the replicated quiesce database maintained by the MDS rank cluster.
One of the major goals was to design the component in a way that can be easily tested
outside of the MDS infrastructure, which is why the communication layer
has been asbtracted out by introducing just two communication callbacks
that will need to be implemented by the infrastructure.

The most of the component code is delivered in a single coherent commit, along with the uint tests.
Other commits will be dedicated to integration with the MDS infrastructure and other changes
that can't be attributed to the core quiesce db code or its tests.

The quiesce db component is composed of the following major parts/actors:

* QuiesceDbManager is the main actor, implementing both the leader and the replica roles.
  Normally, there will be an instance of the manager per MDS rank, although given the
  decoupling of the infrastructure and the manager, one can run any number of instances
  on a single node, which is how test are working.
* The manager interfaces to the infrastructure via two main APIs with the infrastructure
  that provides communication and cluster configuration (actor 2) and the quiesce db
  client that is responsible for the quiescing of the roots (actor 3)
** ClusterMembership is how manager is configured to be part of a (virtual) cluster.
   This structure will deliver information about other peers, the leader and provide
   two communication APIs: send_listing_to for db replication from the leader to replicas
   and send_ack for reporting quiesce success from the agents.
** Client Interface consists of a QuisceMap notify callback and a dedicated manager
   method to submit asynchronous acks following the agent (rank) quiesce progress.

The API of the quiesce db is described in the slide deck mentioned above. The full scope
of capabilities are encapsulated in a single QuiesceDbRequest structure. This should
simplify the implementation of other components that will have to propagate the functionality
to the administrator user of the volumes plugin.

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/common/subsys.h           |    1 +
 src/mds/BoostUrlImpl.cc       |    1 +
 src/mds/QuiesceAgent.h        |   19 +
 src/mds/QuiesceDb.h           |  711 +++++++++++++++
 src/mds/QuiesceDbManager.cc   | 1119 +++++++++++++++++++++++
 src/mds/QuiesceDbManager.h    |  303 +++++++
 src/test/mds/CMakeLists.txt   |   10 +
 src/test/mds/TestQuiesceDb.cc | 1613 +++++++++++++++++++++++++++++++++
 8 files changed, 3777 insertions(+)
 create mode 100644 src/mds/BoostUrlImpl.cc
 create mode 100644 src/mds/QuiesceAgent.h
 create mode 100644 src/mds/QuiesceDb.h
 create mode 100644 src/mds/QuiesceDbManager.cc
 create mode 100644 src/mds/QuiesceDbManager.h
 create mode 100644 src/test/mds/TestQuiesceDb.cc

diff --git a/src/common/subsys.h b/src/common/subsys.h
index d52018c880dd..0bc92346530f 100644
--- a/src/common/subsys.h
+++ b/src/common/subsys.h
@@ -28,6 +28,7 @@ SUBSYS(mds_locker, 1, 5)
 SUBSYS(mds_log, 1, 5)
 SUBSYS(mds_log_expire, 1, 5)
 SUBSYS(mds_migrator, 1, 5)
+SUBSYS(mds_quiesce, 3, 5)
 SUBSYS(buffer, 0, 1)
 SUBSYS(timer, 0, 1)
 SUBSYS(filer, 0, 1)
diff --git a/src/mds/BoostUrlImpl.cc b/src/mds/BoostUrlImpl.cc
new file mode 100644
index 000000000000..c2e992a06e90
--- /dev/null
+++ b/src/mds/BoostUrlImpl.cc
@@ -0,0 +1 @@
+#include <boost/url/src.hpp>
diff --git a/src/mds/QuiesceAgent.h b/src/mds/QuiesceAgent.h
new file mode 100644
index 000000000000..6a059dec3bc5
--- /dev/null
+++ b/src/mds/QuiesceAgent.h
@@ -0,0 +1,19 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#pragma once
+#include "QuiesceDb.h"
+
+class QuiesceAgent {
+  // subscribe to the QM map
+  // keeps runtime version of the map
+  // notifies the QM when runtime version is different from the last know requested version
+};
\ No newline at end of file
diff --git a/src/mds/QuiesceDb.h b/src/mds/QuiesceDb.h
new file mode 100644
index 000000000000..5fedcfe5a5c1
--- /dev/null
+++ b/src/mds/QuiesceDb.h
@@ -0,0 +1,711 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#pragma once
+#include <string>
+#include <map>
+#include <unordered_map>
+#include <optional>
+#include <vector>
+#include <ranges>
+
+#include "mds/mdstypes.h"
+#include "common/ceph_time.h"
+
+// NB! The order of the states in the enum is important!
+// There are places in the code that aggregate multiple states
+// via min or max, depending on the task.
+// The order of states represents the natural lifecycle
+// of a set and its members, this is specifically important
+// for the active states.
+enum QuiesceState: uint8_t {
+  QS__INVALID,
+
+  // these states are considered "active"
+  QS_QUIESCING, QS__ACTIVE = QS_QUIESCING,
+  QS_QUIESCED,
+  QS_RELEASING,
+
+  // the below states are all terminal, or "inactive"
+  QS_RELEASED, QS__TERMINAL = QS_RELEASED,
+  // the below states are all about types of failure
+  QS_EXPIRED, QS__FAILURE = QS_EXPIRED,
+
+  QS_FAILED,
+
+  // the below states aren't allowed for roots, only for sets
+  QS_CANCELED, QS__SET_ONLY = QS_CANCELED,
+  QS_TIMEDOUT,
+
+  QS__MAX,
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceState& qs)
+{
+  switch (qs) {
+  case QS__INVALID:
+    return os << "QS__INVALID (" << (int)qs << ")";
+  case QS_QUIESCING:
+    return os << "QS_QUIESCING (" << (int)qs << ")";
+  case QS_QUIESCED:
+    return os << "QS_QUIESCED (" << (int)qs << ")";
+  case QS_RELEASING:
+    return os << "QS_RELEASING (" << (int)qs << ")";
+  case QS_RELEASED:
+    return os << "QS_RELEASED (" << (int)qs << ")";
+  case QS_FAILED:
+    return os << "QS_FAILED (" << (int)qs << ")";
+  case QS_CANCELED:
+    return os << "QS_CANCELED (" << (int)qs << ")";
+  case QS_TIMEDOUT:
+    return os << "QS_TIMEDOUT (" << (int)qs << ")";
+  case QS_EXPIRED:
+    return os << "QS_EXPIRED (" << (int)qs << ")";
+  default:
+    return os << "!Unknown quiesce state! (" << (int)qs << ")";
+  }
+};
+
+inline const char * quiesce_state_name(QuiesceState state) {
+  switch (state) {
+  case QS__INVALID:
+    return "<invalid>";
+  case QS_QUIESCING:
+    return "QUIESCING";
+  case QS_QUIESCED:
+    return "QUIESCED";
+  case QS_RELEASING:
+    return "RELEASING";
+  case QS_RELEASED:
+    return "RELEASED";
+  case QS_FAILED:
+    return "FAILED";
+  case QS_CANCELED:
+    return "CANCELED";
+  case QS_TIMEDOUT:
+    return "TIMEDOUT";
+  case QS_EXPIRED:
+    return "EXPIRED";
+  default:
+    return "<unknown>";
+  }
+}
+
+// Since MDS clock is not syncrhonized, and the quiesce db has to be replicated,
+// we measure all events in the quiesce database relative to the database age.
+// The age of the database is just large enough to have earliest events carry
+// a non-negative age stamp.
+// This is sufficient because we only care to honor the timeouts that are relative
+// to the other recorded database events.
+// This approach also relieves us from storing or transfering absolute time stamps:
+// every client can deduce the lower boundary of event's absolute time given the 
+// message roundrip timing - if they bother enough. Otherwise, they can just subtract
+// the received database age from now() and get their own absolute time reference.
+
+using QuiesceClock = ceph::coarse_real_clock;
+using QuiesceTimePoint = QuiesceClock::time_point;
+using QuiesceTimeInterval = QuiesceClock::duration;
+using QuiesceSetId = std::string;
+using QuiesceRoot = std::string;
+using QuiesceSetVersion = uint64_t;
+
+struct QuiesceDbVersion {
+  epoch_t epoch;
+  QuiesceSetVersion set_version;
+  auto operator<=>(QuiesceDbVersion const& other) const = default;
+  QuiesceDbVersion& operator+(unsigned int delta) {
+    set_version += delta;
+    return *this;
+  }
+};
+
+inline auto operator==(int const& set_version, QuiesceDbVersion const& db_version)
+{
+  return db_version.set_version == (QuiesceSetVersion)set_version;
+}
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbVersion& dbv)
+{
+  return os << "(" << dbv.epoch << ":" << dbv.set_version << ")";
+}
+
+struct QuiesceTimeIntervalSec {
+  const QuiesceTimeInterval interval;
+  QuiesceTimeIntervalSec(const QuiesceTimeInterval &interval) : interval(interval) {}
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceTimeIntervalSec& sec)
+{
+  using std::chrono::duration_cast;
+  using dd = std::chrono::duration<double>;
+  const auto precision = os.precision();
+  const auto flags = os.flags();
+
+  os
+    << std::fixed
+    << std::setprecision(1)
+    << duration_cast<dd>(sec.interval).count()
+    << std::setprecision(precision);
+
+  os.flags(flags);
+  return os;
+}
+
+struct RecordedQuiesceState {
+  QuiesceState state;
+  QuiesceTimeInterval at_age;
+
+  operator QuiesceState() {
+    return state;
+  }
+
+  bool update(const QuiesceState &state, const QuiesceTimeInterval &at_age) {
+    if (state != this->state) {
+      this->state = state;
+      this->at_age = at_age;
+      return true;
+    }
+    return false;
+  }
+
+  RecordedQuiesceState(QuiesceState state, QuiesceTimeInterval at_age) : state(state), at_age(at_age) {}
+  RecordedQuiesceState() : RecordedQuiesceState (QS__INVALID, QuiesceTimeInterval::zero()) {}
+  RecordedQuiesceState(RecordedQuiesceState const&) = default;
+  RecordedQuiesceState(RecordedQuiesceState &&) = default;
+  RecordedQuiesceState& operator=(RecordedQuiesceState const&) = default;
+  RecordedQuiesceState& operator=(RecordedQuiesceState &&) = default;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const RecordedQuiesceState& rstate)
+{
+  return os << rstate.state;
+}
+
+/// @brief  `QuiesceSet` is the only record type in the quiesce database
+///         It encodes sufficient information to have the db taken over by
+///         a new manager and correctly decide on the next state transition.
+struct QuiesceSet {
+  /// @brief  A member of a set represents a single root this set wants quiesced
+  ///         It carries the information about the current known state of this root
+  ///         and whether it got excluded from this set.
+  ///         It's possible that holding on to excluded members is an overkill.
+  struct MemberInfo {
+    RecordedQuiesceState rstate;
+    bool excluded = false;
+    MemberInfo(QuiesceState state, QuiesceTimeInterval at_age)
+        : rstate(state, at_age)
+        , excluded(false)
+    {
+    }
+    MemberInfo(QuiesceTimeInterval at_age)
+        : MemberInfo(QS_QUIESCING, at_age)
+    {
+    }
+    MemberInfo() = default;
+    MemberInfo(MemberInfo const& o) = default;
+    MemberInfo(MemberInfo &&) = default;
+    MemberInfo& operator=(MemberInfo const& o) = default;
+    MemberInfo& operator=(MemberInfo &&) = default;
+
+    bool is_quiescing() const { return rstate.state < QS_QUIESCED; }
+    bool is_failed() const { return rstate.state >= QS__FAILURE; }
+  };
+  
+  /// @brief  The db version when this set got modified last
+  QuiesceSetVersion version = 0;
+  /// @brief  The last recorded state change of this set
+  RecordedQuiesceState rstate;
+  /// @brief  How much time to give every new member to reach the quiesced state
+  ///         By default the value is zero. It means that new sets which don't
+  ///         have this field explicitly updated will immediately reach QS_TIMEDOUT.
+  QuiesceTimeInterval timeout = QuiesceTimeInterval::zero();
+  /// @brief  How much time to allow the set to spend in QS_QUIESCED or QS_RELEASING states
+  ///         The expiration timer is reset every time a successful await is executed
+  ///         on a QS_QUIESCED set.
+  QuiesceTimeInterval expiration = QuiesceTimeInterval::zero();
+  using Members = std::unordered_map<QuiesceRoot, MemberInfo>;
+  Members members;
+
+  /// @brief The effective state of a member is just a max of its parent set state and its own state
+  ///        The exception is when the set is releasing: we want to consider any ack from peers
+  ///        that confirms quiesced state of the member to be treated as RELEASED.
+  /// @param member_state the reported state of the member
+  /// @return the effective member state
+  QuiesceState get_effective_member_state(QuiesceState reported_state) const
+  {
+    if (is_releasing()) {
+      if (reported_state >= QS_QUIESCED && reported_state <= QS_RELEASED) {
+        return QS_RELEASED;
+      }
+    }
+    if (is_quiesced() && reported_state < QS_QUIESCED) {
+      // we need to change back to quiescing
+      return QS_QUIESCING;
+    }
+    return std::max(reported_state, rstate.state);
+  }
+
+  /// @brief The requested state of a member is what we send to the agents for 
+  ///        executing the quiesce protocol. This state is deliberately reduced
+  ///        to provoke clients to ack back and thus confirm their current state
+  /// @param set_state the state of the set this member is from
+  /// @return the effective member state
+  QuiesceState get_requested_member_state() const
+  {
+    if (rstate.state >= QS__TERMINAL) {
+      return rstate.state;
+    }
+    if (rstate.state <= QS_QUIESCED) {
+      // request quiescing even if we are already quiesced
+      return QS_QUIESCING;
+    }
+    // we can't have anything else unless the state enum was changed
+    // which will have to be addressed here.
+    ceph_assert(rstate.state == QS_RELEASING);
+    return QS_RELEASING;
+  }
+
+  bool is_active() const {
+    return
+      rstate.state > QS__INVALID
+      && rstate.state < QS__TERMINAL;
+  }
+
+  QuiesceState next_state(QuiesceState min_member_state) const;
+
+  bool is_quiescing() const { return rstate.state < QS_QUIESCED; }
+  bool is_quiesced() const { return rstate.state == QS_QUIESCED; }
+  bool is_releasing() const { return rstate.state == QS_RELEASING; }
+  bool is_released() const { return rstate.state == QS_RELEASED; }
+
+  QuiesceSet() = default;
+  QuiesceSet(QuiesceSet const &) = default;
+  QuiesceSet(QuiesceSet &&) = default;
+  QuiesceSet& operator=(QuiesceSet const &) = default;
+  QuiesceSet& operator=(QuiesceSet &&) = default;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceSet::MemberInfo& member)
+{
+  return os << (member.excluded ? "(excluded)" : "") << member.rstate;
+}
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceSet& set)
+{
+  size_t active = 0, inactive = 0;
+
+  for (auto && [_, m]: set.members) {
+    if (m.excluded) {
+      ++inactive;
+    } else {
+      ++active;
+    }
+  }
+
+  return os << "q-set[" << set.rstate << " v:" << set.version << ", m:" << active << "/" << inactive
+    << ", t:" << QuiesceTimeIntervalSec(set.timeout) << ", e:" << QuiesceTimeIntervalSec(set.expiration) << "]";
+}
+
+/// @brief QuiesceDbRequest is the only client interface to the database.
+///        This structure alone should be capable of encoding the full variety
+///        of different requests that can be submitted by the client.
+struct QuiesceDbRequest {
+  /// @brief `RootsOp` should be considered together with the `roots` set below
+  ///        to know the operation. Each name in the enum has two verbs: first
+  ///        verb is for the case when `roots` is not empty, and the second is
+  ///        for when `roots` is empty
+  enum RootsOp: uint8_t {
+    INCLUDE_OR_QUERY,
+    EXCLUDE_OR_RELEASE,
+    RESET_OR_CANCEL,
+    __INVALID
+  };
+
+  enum Flags: uint8_t {
+    NONE = 0,
+    VERBOSE = 1,
+    EXCLUSIVE = 2,
+  };
+
+  struct Control {
+    union {
+      struct {
+        RootsOp roots_op;
+        Flags flags;
+      };
+      uint64_t raw;
+    };
+    Control() : raw(0) {}
+    Control(RootsOp op) : raw(0) {
+      roots_op = op;
+    }
+    bool operator==(const Control& other) const {
+      return other.raw == raw;
+    }
+  };
+
+  Control control;
+
+  /// @brief `set_id` is optional to allow for the following operations:
+  ///         * including roots without providing a set id will generate a new set with a unique id
+  ///           ** NB! the new set id will stored in this field by the db manager
+  ///         * querying without a set id will return the full db
+  ///         * cancelling without a set id will cancel all active sets
+  std::optional<std::string> set_id;
+  /// @brief When `if_version` provided, the request will only be executed
+  ///        if the named set has exactly the version, otherwise ESTALE is returned
+  ///        and no set modification is performed.
+  ///        Requires a set_id.
+  std::optional<QuiesceSetVersion> if_version;
+  /// @brief Updates the quiesce timeout of an active set.
+  ///        Requires a set_id. Attempt to update an inactive set will result in EPERM
+  std::optional<QuiesceTimeInterval> timeout;
+  /// @brief Updates the quiesce expiration of an active set.
+  ///        Requires a set_id. Attempt to update an inactive set will result in EPERM
+  std::optional<QuiesceTimeInterval> expiration;
+  /// @brief When `await` is non-null, then after performing other encoded operations
+  ///        this request is put on the await queue of the given set.
+  ///        The value of this member defines the await timeout.
+  ///        Requires a set id. The result code is one of the following:
+  ///         EPERM     - the set is not in one of the awaitable states:
+  ///                     [QS_QUIESCING, QS_QUIESCED, QS_RELEASING, QS_RELEASED]
+  ///         SUCCESS   - the set is currently QS_QUIESCED or QS_RELEASED.
+  ///                     When an await is completed successfully for a QS_QUIESCED set,
+  ///                     this set's quiesce expiration timer is reset.
+  ///         EINTR     - the set had a change in members or in state
+  ///         ECANCELED - the set was canceled
+  ///         ETIMEDOUT - at least one of the set members failed to quiesce 
+  ///                     within the configured quiesce timeout.
+  ///                     OR the set is RELEASING and it couldn't reach RELEASED before it expired
+  ///                     NB: the quiesce timeout is measured for every member separately
+  ///                     from the moment that member is included.
+  ///         EINPROGRESS - the time limit configured for this await call has elapsed 
+  ///                     before the set changed state.
+  std::optional<QuiesceTimeInterval> await;
+  using Roots = std::unordered_set<QuiesceRoot>;
+  /// @brief `roots` help identify the wanted operation as well as providing
+  ///        the actual roots to mutate the members of the set.
+  Roots roots;
+
+  bool operator==(const QuiesceDbRequest&) const = default;
+
+  bool is_valid() const {
+    return control.roots_op < __INVALID && (
+        // Everything goes if a set id is provided
+        set_id
+        // or it's a new set creation, in which case the request should be including roots
+        || includes_roots()
+        // Otherwise, the allowed wildcard operations are: query and cancel all.
+        // Also, one can't await a wildcard
+        || ((is_cancel_all() || is_query()) && !await && !timeout && !expiration && !if_version)
+    );
+  }
+
+  bool is_mutating() const { return (control.roots_op != INCLUDE_OR_QUERY) || !roots.empty() || timeout || expiration; }
+  bool is_cancel_all() const { return !set_id && is_cancel(); }
+  bool excludes_roots() const { return control.roots_op == RESET_OR_CANCEL || (control.roots_op == EXCLUDE_OR_RELEASE && !roots.empty()); }
+  bool includes_roots() const { return (control.roots_op == RESET_OR_CANCEL || control.roots_op == INCLUDE_OR_QUERY) && !roots.empty(); }
+
+  bool is_include() const { return control.roots_op == INCLUDE_OR_QUERY && !roots.empty(); }
+  bool is_query() const { return control.roots_op == INCLUDE_OR_QUERY && roots.empty(); }
+  bool is_exclude() const { return control.roots_op == EXCLUDE_OR_RELEASE && !roots.empty(); }
+  bool is_release() const { return control.roots_op == EXCLUDE_OR_RELEASE && roots.empty(); }
+  bool is_reset() const { return control.roots_op == RESET_OR_CANCEL && !roots.empty(); }
+  bool is_cancel() const { return control.roots_op == RESET_OR_CANCEL && roots.empty(); }
+
+  bool is_verbose() const { return control.flags & Flags::VERBOSE; }
+  bool is_exclusive() const { return control.flags & Flags::EXCLUSIVE; }
+
+  bool should_exclude(QuiesceRoot root) const {
+    switch (control.roots_op) {
+    case INCLUDE_OR_QUERY:
+      return false;
+    case EXCLUDE_OR_RELEASE:
+      return roots.contains(root);
+    case RESET_OR_CANCEL:
+      return !roots.contains(root);
+      default: ceph_abort("unknown roots_op"); return false;
+    }
+  }
+
+  void reset(std::invocable<QuiesceDbRequest&> auto const &config)
+  {
+    set_id.reset();
+    if_version.reset();
+    timeout.reset();
+    expiration.reset();
+    await.reset();
+    roots.clear();
+    control.raw = 0; // implies roots_op == INCLUDE_OR_QUERY;
+
+    config(*this);
+  }
+  void reset() {
+    reset([](auto&r){});
+  }
+
+  template<typename R = Roots>
+  requires requires ( R&& roots) {
+    Roots(std::forward<R>(roots));
+  }
+  void set_roots(RootsOp op, R&& roots) {
+    control.roots_op = op;
+    this->roots = Roots(std::forward<R>(roots));
+  }
+
+  template <std::ranges::range R>
+  void set_roots(RootsOp op, const R& roots_range)
+  {
+    control.roots_op = op;
+    this->roots = Roots(roots_range.begin(), roots_range.end());
+  }
+
+  template <typename R = Roots>
+  void include_roots(R&& roots)
+  {
+    set_roots(INCLUDE_OR_QUERY, std::forward<R>(roots));
+  }
+
+  template <typename R = Roots>
+  void exclude_roots(R&& roots)
+  {
+    set_roots(EXCLUDE_OR_RELEASE, std::forward<R>(roots));
+  }
+
+  void release_roots() {
+    set_roots(EXCLUDE_OR_RELEASE, {});
+  }
+
+  template <typename R = Roots>
+  void reset_roots(R&& roots)
+  {
+    set_roots(RESET_OR_CANCEL, std::forward<R>(roots));
+  }
+
+  void cancel_roots()
+  {
+    set_roots(RESET_OR_CANCEL, {});
+  }
+
+  template <typename S = std::string>
+  void query(S&& set_id) {
+    reset([set_id](auto &r){
+      r.set_id = std::forward<S>(set_id);
+    });
+  }
+
+  const char * op_string() const {
+    switch (control.roots_op) {
+    case INCLUDE_OR_QUERY:
+      return roots.empty() ? "query" : "include";
+    case EXCLUDE_OR_RELEASE:
+      return roots.empty() ? "release" : "exclude";
+    case RESET_OR_CANCEL:
+      return roots.empty() ? "cancel" : "reset";
+    default:
+      return "<unknown>";
+    }
+  }
+
+
+  QuiesceDbRequest() {}
+  QuiesceDbRequest(const QuiesceDbRequest &) = default;
+  QuiesceDbRequest(QuiesceDbRequest &&) = default;
+  QuiesceDbRequest(std::invocable<QuiesceDbRequest&> auto const &config) {
+    reset(config);
+  }
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbRequest& req)
+{
+  os << "q-req[" << req.op_string();
+
+  if (req.set_id) {
+    os << " \"" << req.set_id << "\"";
+  }
+
+  if (req.if_version) {
+    os << " ?v:" << req.if_version;
+  }
+
+  if (req.await) {
+    os << " a:" << QuiesceTimeIntervalSec(*req.await);
+  }
+
+  return os << " roots:" << req.roots.size() << "]";
+}
+
+/// @brief  A `QuiesceDbListing` represents a subset of the database, up to
+///         a full database. The contents of the listing is decided by the leader
+///         based on the acks it got from every given replica: the update will
+///         contain all sets that have their version > than the last acked by the peer.
+struct QuiesceDbListing {
+  QuiesceDbVersion db_version = {0, 0};
+  /// @brief  Crusially, the precise `db_age` must be included in every db listing
+  ///         This data is used by all replicas to update their calculated DB TIME ZERO.
+  ///         All events in the database are measured relative to the DB TIME ZERO
+  QuiesceTimeInterval db_age = QuiesceTimeInterval::zero();
+  std::unordered_map<QuiesceSetId, QuiesceSet> sets;
+
+  void clear() {
+    db_version = {0, 0};
+    db_age = QuiesceTimeInterval::zero();
+    sets.clear();
+  }
+
+  QuiesceDbListing(epoch_t epoch) : db_version {epoch, 0} {}
+  QuiesceDbListing() = default;
+  QuiesceDbListing(QuiesceDbListing const&) = default;
+  QuiesceDbListing(QuiesceDbListing &&) = default;
+  QuiesceDbListing& operator=(QuiesceDbListing const&) = default;
+  QuiesceDbListing& operator=(QuiesceDbListing &&) = default;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbListing& dbl)
+{
+  size_t active = 0, inactive = 0;
+
+  for (auto&& [_, s] : dbl.sets) {
+    if (s.is_active()) {
+      ++active;
+    } else {
+      ++inactive;
+    }
+  }
+
+  return os << "q-db[v:" << dbl.db_version << " sets:" << active << "/" << inactive << "]";
+}
+
+/// @brief  `QuiesceMap` is a root-centric representation of the quiesce database
+///         It lists roots with their effective states as of particular version.
+///         Additionally, the same structure is used by the peers when reporting
+///         actual roots states which are different from what the DB version encodes
+struct QuiesceMap {
+  QuiesceDbVersion db_version;
+  struct RootInfo {
+    QuiesceState state;
+    QuiesceTimeInterval ttl = QuiesceTimeInterval::zero();
+    bool is_valid() const { return state > QS__INVALID && state < QS__SET_ONLY; }
+    RootInfo() : RootInfo(QS__INVALID) {}
+    RootInfo(QuiesceState state) : RootInfo(state,QuiesceTimeInterval::zero()) {}
+    RootInfo(QuiesceState state, QuiesceTimeInterval ttl)
+        : state(state)
+        , ttl(ttl)
+    {
+    }
+    inline bool operator==(const RootInfo& other) const {
+      return state == other.state && ttl == other.ttl;
+    }
+
+    RootInfo(RootInfo const&) = default;
+    RootInfo(RootInfo &&) = default;
+    RootInfo& operator=(RootInfo const&) = default;
+    RootInfo& operator=(RootInfo &&) = default;
+  };
+  using Roots = std::unordered_map<QuiesceRoot, RootInfo>;
+  Roots roots;
+  void reset() {
+    db_version = {0, 0};
+    roots.clear();
+  }
+
+  QuiesceMap() : db_version({0, 0}), roots() { }
+  QuiesceMap(QuiesceDbVersion db_version) : db_version(db_version), roots() { }
+  QuiesceMap(QuiesceDbVersion db_version, Roots &&roots) : db_version(db_version), roots(roots) { }
+  QuiesceMap(QuiesceDbVersion db_version, Roots const& roots) : db_version(db_version), roots(roots) { }
+
+  QuiesceMap(QuiesceMap const&) = default;
+  QuiesceMap(QuiesceMap &&) = default;
+  QuiesceMap& operator=(QuiesceMap const&) = default;
+  QuiesceMap& operator=(QuiesceMap &&) = default;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceMap& map)
+{
+
+  size_t active = 0, inactive = 0;
+
+  for (auto&& [_, r] : map.roots) {
+    if (r.state < QS__TERMINAL) {
+      ++active;
+    } else {
+      ++inactive;
+    }
+  }
+
+  return os << "q-map[v:" << map.db_version << " roots:" << active << "/" << inactive << "]";
+}
+
+inline QuiesceTimeInterval interval_saturate_add(QuiesceTimeInterval lhs, QuiesceTimeInterval rhs)
+{
+  // assuming an unsigned time interval.
+  // TODO: make this function generic and also saturate add signed values
+  assert(std::is_unsigned_v<QuiesceTimeInterval::rep>);
+
+  QuiesceTimeInterval result = lhs + rhs;
+
+  // the sum can't be smaller than either part
+  // since we're working with an unsigned value
+  if (result < lhs || result < rhs) {
+    // this must have been an overflow
+    return QuiesceTimeInterval::max();
+  }
+
+  return result;
+};
+
+inline QuiesceTimePoint interval_saturate_add_now(QuiesceTimeInterval interval) {
+  return QuiesceTimePoint(interval_saturate_add(QuiesceClock::now().time_since_epoch(), interval));
+};
+
+namespace QuiesceInterface {
+  using PeerId = mds_gid_t;
+  /// @brief  A callback from the manager to the agent with an up-to-date root list
+  ///         The map is mutable and will be used as synchronous agent ack if the return value is true
+  using AgentNotify = std::function<bool(QuiesceMap&)>;
+  /// @brief  Used to send asyncrhonous acks from agents about changes to the root states
+  ///         The transport layer should include sufficient information to know the sender of the ack
+  using AgentAck = std::function<int(QuiesceMap&&)>;
+  /// @brief  Used by the leader to replicate the DB changes to its peers
+  using DbPeerUpdate = std::function<int(PeerId, QuiesceDbListing&&)>;
+
+  using RequestHandle = metareqid_t;
+  /// @brief  Used by the agent to initiate an ongoing quiesce request for the given quiesce root
+  ///         The context will be completed when the quiescing is achieved by this rank. The IO pause
+  ///         should continue until the request is canceled.
+  ///         Repeated requests for the same root should succeed, returning a _new_ request id;
+  ///         the old context should be completed with an error EINTR, and the old request id should be invalidated.
+  ///         If the root has already reached quiescence by the time the repeated request is submitted
+  ///         then the new context should be immediately (syncrhonously) completed with success and then discarded.
+  ///         Syncrhonous errors should be reported by completing the supplied context, and the return value
+  ///         should be std::nullopt in such cases
+  using RequestSubmit = std::function<std::optional<RequestHandle>(QuiesceRoot, Context*)>;
+  /// @brief  Cancels the quiesce request. May be called at any time after the request got submitted
+  using RequestCancel = std::function<int(const RequestHandle&)>;
+};
diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
new file mode 100644
index 000000000000..3eb9009f86b0
--- /dev/null
+++ b/src/mds/QuiesceDbManager.cc
@@ -0,0 +1,1119 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "mds/QuiesceDbManager.h"
+#include "common/debug.h"
+#include "fmt/format.h"
+#include "include/ceph_assert.h"
+#include <algorithm>
+#include <random>
+#include <ranges>
+#include <type_traits>
+#include "boost/url.hpp"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mds_quiesce
+#undef dout_prefix
+#define dout_prefix *_dout << "quiesce.mgr <" << __func__ << "> "
+
+#undef dout
+#define dout(lvl)                                                        \
+  do {                                                                   \
+    auto subsys = ceph_subsys_mds;                                       \
+    if ((dout_context)->_conf->subsys.should_gather(dout_subsys, lvl)) { \
+      subsys = dout_subsys;                                              \
+    }                                                                    \
+  dout_impl(dout_context, ceph::dout::need_dynamic(subsys), lvl) dout_prefix
+
+#undef dendl
+#define dendl \
+  dendl_impl; \
+  }           \
+  while (0)
+
+#define dset(suffix) "[" << set_id << "@" << set.version << "] " << suffix
+#define dsetroot(suffix) "[" << set_id << "@" << set.version << "," << root << "] " << suffix
+
+const QuiesceInterface::PeerId QuiesceClusterMembership::INVALID_MEMBER = MDS_GID_NONE;
+
+static QuiesceTimeInterval time_distance(QuiesceTimePoint lhs, QuiesceTimePoint rhs) {
+  if (lhs > rhs) {
+    return lhs - rhs;
+  } else {
+    return rhs - lhs;
+  }
+}
+
+bool QuiesceDbManager::db_thread_has_work() const
+{
+  return false
+      || pending_acks.size() > 0
+      || pending_requests.size() > 0
+      || pending_db_updates.size() > 0
+      || (agent_callback.has_value() && agent_callback->if_newer < db_version())
+      || (!cluster_membership.has_value() || cluster_membership->epoch != membership.epoch);
+}
+
+void* QuiesceDbManager::quiesce_db_thread_main()
+{
+  db_thread_enter();
+
+  std::unique_lock ls(submit_mutex);
+  QuiesceTimeInterval next_event_at_age = QuiesceTimeInterval::max();
+  QuiesceDbVersion last_acked = {0, 0};
+
+  while (true) {
+
+    auto db_age = db.get_age();
+
+    if (!db_thread_has_work() && next_event_at_age > db_age) {
+      submit_condition.wait_for(ls, next_event_at_age - db_age);
+    }
+
+    if (!membership_upkeep()) {
+      break;
+    }
+
+    {
+      decltype(pending_acks) acks(std::move(pending_acks));
+      decltype(pending_requests) requests(std::move(pending_requests));
+      decltype(pending_db_updates) db_updates(std::move(pending_db_updates));
+
+      ls.unlock();
+
+      if (membership.is_leader()) {
+        if (leader_bootstrap(std::move(db_updates), next_event_at_age)) {
+          // we're good to process things
+          next_event_at_age = leader_upkeep(std::move(acks), std::move(requests));
+        } else {
+          // not yet there. Put the requests back onto the queue
+          ls.lock();
+          while (!requests.empty()) {
+            pending_requests.emplace_front(std::move(requests.back()));
+            requests.pop_back();
+          }
+          continue;
+        }
+      } else {
+        next_event_at_age = replica_upkeep(std::move(db_updates));
+      }
+    }
+  
+    complete_requests();
+
+    // by default, only send ack if the version has changed
+    bool send_ack = last_acked != db_version();
+    QuiesceMap quiesce_map(db_version());
+    {
+      std::lock_guard lc(agent_mutex);
+      if (agent_callback) {
+        if (agent_callback->if_newer < db_version()) {
+          dout(20) << "notifying agent with db version " << db_version() << dendl;
+          calculate_quiesce_map(quiesce_map);
+          send_ack = agent_callback->notify(quiesce_map);
+          agent_callback->if_newer = db_version();
+        } else {
+          send_ack = false;
+        }
+      } else {
+        // by default, ack the db version and agree to whatever was sent
+        // This means that a quiesce cluster member with an empty agent callback 
+        // will cause roots to stay quiescing indefinitely
+        dout(5) << "no agent callback registered, responding with an empty ack" << dendl;
+      }
+    }
+
+    if (send_ack) {
+      auto db_version = quiesce_map.db_version;
+      dout(20) << "synchronous agent ack: " << quiesce_map << dendl;
+      auto rc = membership.send_ack(std::move(quiesce_map));
+      if (rc != 0) {
+        dout(1) << "ERROR ("<< rc <<") when sending synchronous agent ack " 
+        << quiesce_map << dendl;
+      } else {
+        last_acked = db_version;
+      }
+    }
+
+    ls.lock();
+  }
+
+  ls.unlock();
+
+  db_thread_exit();
+
+  return 0;
+}
+
+void QuiesceDbManager::update_membership(const QuiesceClusterMembership& new_membership, RequestContext* inject_request)
+{
+  std::unique_lock lock(submit_mutex);
+
+  bool will_participate = new_membership.members.contains(new_membership.me);
+  dout(20) << "will participate: " << std::boolalpha << will_participate << std::noboolalpha << dendl;
+
+  if (cluster_membership && !will_participate) {
+    // stop the thread
+    cluster_membership.reset();
+    submit_condition.notify_all();
+    lock.unlock();
+    ceph_assert(quiesce_db_thread.is_started());
+    dout(5) << "stopping the db mgr thread at epoch: " << new_membership.epoch << dendl;
+    quiesce_db_thread.join();
+  } else if (will_participate) {
+    if (!cluster_membership) {
+      // start the thread
+      dout(5) << "starting the db mgr thread at epoch: " << new_membership.epoch << dendl;
+      quiesce_db_thread.create("quiesce_db_mgr");
+    } else {
+      submit_condition.notify_all();
+    }
+    if (inject_request) {
+      pending_requests.push_front(inject_request);
+    }
+    cluster_membership = new_membership;
+    
+    std::lock_guard lc(agent_mutex);
+    if (agent_callback) {
+        agent_callback->if_newer = {0, 0};
+    }
+  }
+
+  if (!will_participate && inject_request) {
+    inject_request->complete(-EPERM);
+  }
+}
+
+bool QuiesceDbManager::membership_upkeep()
+{
+  if (cluster_membership && cluster_membership->epoch == membership.epoch) {
+    // no changes
+    return true;
+  }
+
+  bool was_leader = membership.epoch > 0 && membership.leader == membership.me;
+  bool is_leader = cluster_membership && cluster_membership->leader == cluster_membership->me;
+  if (cluster_membership) {
+    dout(10) << "epoch: " << cluster_membership->epoch << " is_leader: " << is_leader << " was_leader: " << was_leader << dendl;
+  } else {
+    dout(10) << "shutdown! was_leader: " << was_leader << dendl;
+  }
+
+  if (is_leader) {
+    // remove peers that aren't present anymore
+    for (auto peer_it = peers.begin(); peer_it != peers.end();) {
+      if (cluster_membership->members.contains(peer_it->first)) {
+        peer_it++;
+      } else {
+        peer_it = peers.erase(peer_it);
+      }
+    }
+    // create empty info for new peers
+    for (auto peer : cluster_membership->members) {
+      peers.try_emplace(peer);
+    }
+
+    if (db.set_version == 0) {
+      db.time_zero = QuiesceClock::now();
+      db.sets.clear();
+    }
+
+  } else {
+    peers.clear();
+    // abort awaits with EINPROGRESS
+    // the reason is that we don't really have a new version
+    // of any of the sets, we just aren't authoritative anymore
+    // hence, EINPROGRESS is a more appropriate response than, say, EINTR
+    for (auto & [_, await_ctx]: awaits) {
+      done_requests[await_ctx.req_ctx] = EINPROGRESS;
+    }
+    awaits.clear();
+    // reject pending requests
+    while (!pending_requests.empty()) {
+      done_requests[pending_requests.front()] = EPERM;
+      pending_requests.pop_front();
+    }
+  }
+
+  if (cluster_membership) {
+    membership = *cluster_membership;
+  }
+
+  return cluster_membership.has_value();
+}
+
+QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates)&& db_updates)
+{
+  // as a replica, we only care about the latest update
+  while (db_updates.size() > 1) {
+    dout(10) << "skipping an older update from " << db_updates.front().first << " version " << db_updates.front().second.db_version << dendl;
+    db_updates.pop();
+  }
+
+  if (db_updates.empty()) {
+    // no db updates, wait forever
+    return QuiesceTimeInterval::max();
+  }
+
+  QuiesceDbListing &update = db_updates.back().second;
+
+  if (update.db_version.epoch != membership.epoch) {
+    dout(10) << "ignoring db update from another epoch: " << update.db_version << " != " << db_version() << dendl;
+    return QuiesceTimeInterval::max();
+  }
+
+  if (update.db_version.set_version == 0) {
+    // this is a call from a leader
+    // to upload our local db version
+    update.sets = db.sets;
+    update.db_version.set_version = db.set_version;
+    update.db_age = db.get_age();
+    membership.send_listing_to(membership.leader, std::move(update));
+    return QuiesceTimeInterval::max();
+  }
+
+  auto time_zero = QuiesceClock::now() - update.db_age;
+  if (time_distance(time_zero, db.time_zero) > std::chrono::seconds(1)) {
+    dout(10) << "significant db_time_zero change to " << time_zero << " from " << db.time_zero << dendl;
+  }
+  db.time_zero = time_zero;
+
+  if (db.set_version > update.db_version.set_version) {
+    dout(3) << "got an older version of DB from the leader: " << db.set_version << " > " << update.db_version.set_version << dendl;
+    dout(3) << "discarding the DB" << dendl;
+    db.reset();
+  } else {
+    for (auto& [qs_id, qs] : update.sets) {
+      db.sets.insert_or_assign(qs_id, std::move(qs));
+    }
+    db.set_version = update.db_version.set_version;
+  }
+
+  // wait forever
+  return QuiesceTimeInterval::max();
+}
+
+bool QuiesceDbManager::leader_bootstrap(decltype(pending_db_updates)&& db_updates, QuiesceTimeInterval &next_event_at_age)
+{
+  // check that we've heard from all peers in this epoch
+  std::unordered_set<QuiesceInterface::PeerId> unknown_peers;
+  for (auto&& [peer, info] : peers) {
+    if (info.diff_map.db_version.epoch < membership.epoch && info.diff_map.db_version.set_version == 0) {
+      if (peer != membership.me) {
+        unknown_peers.insert(peer);
+      }
+    }
+  }
+
+  // only consider db submissions from unknown peers
+  while (!unknown_peers.empty() && !db_updates.empty()) {
+    auto &[from, update] = db_updates.front();
+    if (update.db_version.epoch == membership.epoch && unknown_peers.erase(from) > 0) {
+      // see if this peer's version is newer than mine
+      if (db.set_version < update.db_version.set_version) {
+        dout(3) << "preferring version from peer " 
+          << from << " (" << update.db_version 
+          << ") over mine (" << db_version() << ")" 
+          << " and incrementing it to collect acks" << dendl;
+        db.time_zero = QuiesceClock::now() - update.db_age;
+        db.set_version = update.db_version.set_version + 1;
+        db.sets = update.sets;
+      }
+      // record that we've seen this peer;
+      // set the epoch correctly but use set version 0 because it's not an ack yet.
+      peers[from] =  PeerInfo {QuiesceMap({membership.epoch, 0}), QuiesceClock::now()};
+    }
+    db_updates.pop();
+  }
+
+  for (auto & peer: unknown_peers) {
+    PeerInfo & info = peers[peer];
+
+    QuiesceTimePoint next_discovery = info.last_seen + std::chrono::seconds(1);
+    if (info.last_seen == QuiesceClock::zero() || next_discovery < QuiesceClock::now()) {
+      // send a discovery request to unknown peers
+      dout(5) << " sending a discovery request to " << peer << dendl;
+      membership.send_listing_to(peer, QuiesceDbListing(membership.epoch));
+      info.last_seen = QuiesceClock::now();
+      next_discovery = info.last_seen + std::chrono::seconds(1);
+    }
+    QuiesceTimeInterval next_discovery_at_age = next_discovery - db.time_zero;
+
+    next_event_at_age = std::min(next_event_at_age, next_discovery_at_age);
+  }
+
+  // true if all peers are known
+  return unknown_peers.empty();
+}
+
+QuiesceTimeInterval QuiesceDbManager::leader_upkeep(decltype(pending_acks)&& acks, decltype(pending_requests)&& requests)
+{
+  // record peer acks
+  while (!acks.empty()) {
+    auto& [from, diff_map] = acks.front();
+    leader_record_ack(from, std::move(diff_map));
+    acks.pop();
+  }
+
+  // process requests
+  while (!requests.empty()) {
+    auto req_ctx = requests.front();
+    int result = leader_process_request(req_ctx);
+    if (result != EBUSY) {
+      done_requests[req_ctx] = result;
+    }
+    requests.pop_front();
+  }
+
+  QuiesceTimeInterval next_db_event_at_age = leader_upkeep_db();
+  QuiesceTimeInterval next_await_event_at_age = leader_upkeep_awaits();
+
+  return std::min(next_db_event_at_age, next_await_event_at_age);
+}
+
+void QuiesceDbManager::complete_requests() {
+  for (auto [req, res]: done_requests) {
+    auto & r = req->response;
+    r.clear();
+    if (membership.leader == membership.me) {
+      r.db_age = db.get_age();
+      r.db_version = db_version();
+
+      if (req->request.set_id) {
+        Db::Sets::const_iterator it = db.sets.find(*req->request.set_id);
+        if (it != db.sets.end()) {
+          r.sets.emplace(*it);
+        }
+      } else if (req->request.is_query()) {
+        for (auto && it : std::as_const(db.sets)) {
+          r.sets.emplace(it);
+        }
+      }
+    }
+    // non-zero result codes are all errors
+    req->complete(-res);
+  }
+  done_requests.clear();
+}
+
+void QuiesceDbManager::leader_record_ack(QuiesceInterface::PeerId from, QuiesceMap&& diff_map)
+{
+  auto it = peers.find(from);
+
+  if (it == peers.end()) {
+    // ignore updates from unknown peers
+    return;
+  }
+
+  auto & info = it->second;
+
+  if (diff_map.db_version > db_version()) {
+    dout(3) << "ignoring unknown version ack by rank " << from << " (" << diff_map.db_version << " > " << db_version() << ")" << dendl;
+    dout(5) << "will send the peer a full DB" << dendl;
+    info.diff_map.reset();
+  } else {
+    info.diff_map = std::move(diff_map);
+    info.last_seen = QuiesceClock::now();
+  }
+}
+
+static std::string random_hex_string() {
+  std::mt19937 gen(std::random_device {} ());
+  return fmt::format("{:x}", gen());
+}
+
+bool QuiesceDbManager::sanitize_roots(QuiesceDbRequest::Roots& roots)
+{
+  static const std::string file_scheme = "file";
+  static const std::string inode_scheme = "inode";
+  static const std::unordered_set<std::string> supported_schemes { file_scheme, inode_scheme };
+  QuiesceDbRequest::Roots result;
+  for (auto &root : roots) {
+    auto parsed_uri = boost::urls::parse_uri_reference(root);
+    if (!parsed_uri) {
+      dout(2) << "Couldn't parse root '" << root << "' as URI (error: " << parsed_uri.error() << ")" << dendl;
+      return false;
+    }
+
+    boost::url root_url = parsed_uri.value();
+    root_url.normalize();
+
+    if (!root_url.has_scheme()) {
+      root_url.set_scheme(file_scheme);
+    } else if (!supported_schemes.contains(root_url.scheme())) {
+      dout(2) << "Unsupported root URL scheme '" << root_url.scheme() << "'" << dendl;
+      return false;
+    }
+
+    if (root_url.has_authority()) {
+      auto auth_str = root_url.authority().buffer();
+      bool ok_remove = false;
+      if (auth_str == membership.fs_name) {
+        ok_remove = true;
+      } else {
+        try {
+          ok_remove = std::stoll(auth_str) == membership.fs_id;
+        } catch (...) { }
+      }
+      if (ok_remove) {
+        // OK, but remove the authority for now
+        // we may want to enforce it if we decide to keep a single database for all file systems
+        dout(10) << "Removing the fs name or id '" << auth_str << "' from the root url authority section" << dendl;
+        root_url.remove_authority();
+      } else {
+        dout(2) << "The root url '" << root_url.buffer() 
+          << "' includes an authority section '" << auth_str 
+          << "' which doesn't match the fs id (" << membership.fs_id 
+          << ") or name ('" << membership.fs_name << "')" << dendl;
+        return false;
+      }
+    }
+
+    std::string sanitized_path;
+    sanitized_path.reserve(root_url.path().size());
+    // deal with the file path
+    //  * make it absolute (start with a slash)
+    //  * remove repeated slashes
+    //  * remove the trailing slash
+    bool skip_slash = true;
+    for (auto&& c : root_url.path()) {
+      if (c != '/' || !skip_slash) {
+        sanitized_path.push_back(c);
+      }
+      skip_slash = c == '/';
+    }
+
+    if (sanitized_path.size() > 0 && sanitized_path.back() == '/') {
+      sanitized_path.pop_back();
+    }
+
+    if (root_url.scheme() == file_scheme) {
+      sanitized_path.insert(sanitized_path.begin(), '/');
+    } else if (root_url.scheme() == inode_scheme) {
+      uint64_t inodeno = 0;
+      try {
+        inodeno = std::stoull(sanitized_path);
+      } catch (...) { }
+
+      if (!inodeno || fmt::format("{}", inodeno) != sanitized_path) {
+        dout(2) << "Root '" << root << "' does not encode a vaild inode number" << dendl;
+        return false;
+      }
+    }
+
+    root_url.set_path(sanitized_path);
+
+    if (root_url.buffer() != root) {
+      dout(10) << "Normalized root '" << root << "' to '" << root_url.buffer() << "'" << dendl;
+    }
+    result.insert(root_url.buffer());
+  }
+  roots.swap(result);
+  return true;
+}
+
+int QuiesceDbManager::leader_process_request(RequestContext* req_ctx)
+{
+  QuiesceDbRequest &request = req_ctx->request;
+
+  if (!request.is_valid()) {
+    dout(2) << "rejecting an invalid request" << dendl;
+    return EINVAL;
+  }
+
+  if (!sanitize_roots(request.roots)) {
+    dout(2) << "failed to sanitize roots for a request" << dendl;
+    return EINVAL;
+  }
+
+  const auto db_age = db.get_age();
+
+  if (request.is_cancel_all()) {
+    dout(3) << "WARNING: got a cancel all request" << dendl;
+    // special case - reset all
+    // this will only work on active sets
+    for (auto &[set_id, set]: db.sets) {
+      if (set.is_active()) {
+        bool did_update = false;
+        for (auto&& [_, member]: set.members) {
+          did_update |= !member.excluded;
+          member.excluded = true;
+        }
+
+        ceph_assert(did_update);
+        ceph_assert(set.rstate.update(QS_CANCELED, db_age));
+        set.version = db.set_version+1;
+      }
+    }
+    return 0;
+  }
+
+  // figure out the set to update
+  auto set_it = db.sets.end();
+
+  if (request.set_id) {
+    set_it = db.sets.find(*request.set_id);
+  } else if (request.if_version > 0) {
+    dout(2) << "can't expect a non-zero version (" << *request.if_version << ") for a new set" << dendl;
+    return EINVAL;
+  }
+
+  if (set_it == db.sets.end()) {
+    if (request.includes_roots() && request.if_version <= 0) {
+      // such requests may introduce a new set
+      if (!request.set_id) {
+        // we should generate a unique set id
+        QuiesceSetId new_set_id;
+        do {
+          new_set_id = random_hex_string();
+        } while (db.sets.contains(new_set_id));
+        // update the set_id in the request so that we can
+        // later know which set got created
+        request.set_id.emplace(std::move(new_set_id));
+      }
+      set_it = db.sets.emplace(*request.set_id, QuiesceSet()).first;
+    } else if (request.is_mutating() || request.await) {
+      ceph_assert(request.set_id.has_value());
+      dout(2) << "coudn't find set with id '" << *request.set_id <<  "'" << dendl;
+      return ENOENT;
+    }
+  }
+
+  if (set_it != db.sets.end()) {
+    auto& [set_id, set] = *set_it;
+
+    int result = leader_update_set(*set_it, request);
+    if (result != 0) {
+      return result;
+    }
+
+    if (request.await) {
+      // this check may have a false negative for a quiesced set
+      // that will be released in another request in the same batch
+      // in that case, this await will be enqueued but then found and completed
+      // with the same error in `leader_upkeep_awaits`
+      if ((set.is_releasing() || set.is_released()) && !request.is_release()) {
+        dout(2) << dset("can't quiesce-await a set that was released (") << set.rstate.state << ")" << dendl;
+        return EPERM;
+      }
+
+      auto expire_at_age = interval_saturate_add(db_age, *request.await);
+      awaits.emplace(std::piecewise_construct,
+          std::forward_as_tuple(set_id),
+          std::forward_as_tuple(expire_at_age, req_ctx));
+      // let the caller know that the request isn't done yet
+      return EBUSY;
+    }
+  }
+
+  // if we got here it must be a success
+  return 0;
+}
+
+int QuiesceDbManager::leader_update_set(Db::Sets::value_type& set_it, const QuiesceDbRequest& request)
+{
+  auto & [set_id, set] = set_it;
+  if (request.if_version && set.version != *request.if_version) {
+    dout(10) << dset("is newer than requested (") << *request.if_version << ") " << dendl;
+    return ESTALE;
+  }
+
+  if (!request.is_mutating()) {
+    return 0;
+  }
+
+  bool did_update = false;
+  bool did_update_roots = false;
+
+  if (request.is_release()) {
+    // the release command is allowed in states
+    // quiesced, releasing, released
+    switch (set.rstate.state) {
+      case QS_QUIESCED:
+        // we only update the state to RELEASING,
+        // and not the age. This is to keep counting
+        // towards the quiesce expiration.
+        // TODO: this could be reconsidered, but would
+        // then probably require an additional timestamp
+        set.rstate.state = QS_RELEASING;
+        did_update = true;
+        dout(15) << dset("") << "updating state to: " << set.rstate.state << dendl;
+      case QS_RELEASING:
+      case QS_RELEASED:
+        break;
+      default:
+        dout(2) << dset("can't release in the state: ") << set.rstate.state << dendl;
+        return EPERM;
+    }
+  } else {
+    const auto db_age = db.get_age();
+    bool reset = false;
+
+    if (!request.is_reset()) {
+      // only active or new sets can be modified
+      if (!set.is_active() && set.version > 0) {
+        dout(2) << dset("rejecting modification in the terminal state: ") << set.rstate.state << dendl;
+        return EPERM;
+      } else if (request.includes_roots() && set.is_releasing()) {
+        dout(2) << dset("rejecting new roots in the QS_RELEASING state") << dendl;
+        return EPERM;
+      }
+    } else {
+      // a reset request can be used to resurrect a set from whichever state it's in now
+      if (set.rstate.state > QS_QUIESCED) {
+        dout(5) << dset("reset back to a QUIESCING state") << dendl;
+        did_update = set.rstate.update(QS_QUIESCING, db_age);
+        ceph_assert(did_update);
+        reset = true;
+      }
+    }
+
+    if (request.timeout) {
+      set.timeout = *request.timeout;
+      did_update = true;
+    }
+
+    if (request.expiration) {
+      set.expiration = *request.expiration;
+      did_update = true;
+    }
+
+    size_t included_count = 0;
+    QuiesceState min_member_state = QS__MAX;
+
+    for (auto& [root, info] : set.members) {
+      if (request.should_exclude(root)) {
+        did_update_roots |= !info.excluded;
+        info.excluded = true;
+      } else if (!info.excluded) {
+        included_count ++;
+
+        QuiesceState effective_member_state;
+
+        if (reset) {
+          dout(5) << dsetroot("reset back to a QUIESCING state") << dendl;
+          info.rstate.state = QS_QUIESCING;
+          info.rstate.at_age = db_age;
+          did_update_roots = true;
+          effective_member_state = info.rstate.state;
+        } else {
+          QuiesceState min_reported_state;
+          QuiesceState max_reported_state;
+          size_t reporting_peers = check_peer_reports(set_id, set, root, info, min_reported_state, max_reported_state);
+
+          if (reporting_peers == peers.size() && max_reported_state < QS__FAILURE) {
+            effective_member_state = set.get_effective_member_state(min_reported_state);
+          } else {
+            effective_member_state = info.rstate.state;
+          }
+        }
+
+        min_member_state = std::min(min_member_state, effective_member_state);
+      }
+    }
+
+    if (request.includes_roots()) {
+      for (auto const& root : request.roots) {
+        auto const& [member_it, emplaced] = set.members.try_emplace(root, db_age);
+        auto& [_, info] = *member_it;
+        if (emplaced || info.excluded) {
+          info.excluded = false;
+          did_update_roots = true;
+          included_count++;
+          info.rstate = { QS_QUIESCING, db_age };
+          min_member_state = std::min(min_member_state, QS_QUIESCING);
+        }
+      }
+    }
+
+    did_update |= did_update_roots;
+
+    if (included_count == 0) {
+      dout(20) << dset("cancelled due to 0 included members") << dendl;
+      did_update = set.rstate.update(QS_CANCELED, db_age);
+      ceph_assert(did_update);
+    } else if (min_member_state < QS__MAX) {
+      auto next_state = set.next_state(min_member_state);
+      if (did_update |= set.rstate.update(next_state, db_age)) {
+        dout(15) << dset("updated to match the min state of the remaining (") << included_count << ") members: " << set.rstate.state << dendl;
+      }
+    }
+  }
+
+  if (did_update) {
+    dout(20) << dset("updating version from ") << set.version << " to " << db.set_version + 1 << dendl;
+    set.version = db.set_version + 1;
+    if (did_update_roots) {
+      // any awaits pending on this set must be interrupted
+      // NB! even though the set may be QUIESCED now, it could only
+      // get there due to exclusion of quiescing roots, which is
+      // not a vaild way to successfully await a set, hence EINTR
+      // However, if the set had all roots removed then we
+      // should respond in ECANCELED to notify that no more await
+      // attempts will be permitted
+      auto range = awaits.equal_range(set_id);
+      int rc = EINTR;
+      if (!set.is_active()) {
+        ceph_assert(set.rstate.state == QS_CANCELED);
+        rc = ECANCELED;
+      }
+      for (auto it = range.first; it != range.second; it++) {
+        done_requests[it->second.req_ctx] = rc;
+      }
+      if (range.first != range.second) {
+        dout(10) << dset("interrupting awaits with rc = ") << rc << " due to a change in members" << dendl;
+      }
+      awaits.erase(range.first, range.second);
+    }
+  }
+
+  return 0;
+}
+
+QuiesceTimeInterval QuiesceDbManager::leader_upkeep_db()
+{
+  std::map<QuiesceInterface::PeerId, std::deque<std::reference_wrapper<Db::Sets::value_type>>> peer_updates;
+
+  QuiesceTimeInterval next_event_at_age = QuiesceTimeInterval::max();
+  QuiesceSetVersion max_set_version = db.set_version;
+
+  for(auto & set_it: db.sets) {
+    auto & [set_id, set] = set_it;
+    auto next_set_event_at_age = leader_upkeep_set(set_it);
+
+    max_set_version = std::max(max_set_version, set.version);
+    next_event_at_age = std::min(next_event_at_age, next_set_event_at_age);
+
+    for(auto const & [peer, info]: peers) {
+      // update remote peers if their version is lower than this set's
+      // don't update myself
+      if (peer == membership.me) {
+        continue;
+      }
+      if (info.diff_map.db_version.set_version < set.version) {
+        peer_updates[peer].emplace_back(set_it);
+      }
+    }
+  }
+
+  db.set_version = max_set_version;
+
+  // update the peers
+  for (auto &[peer, sets]: peer_updates) {
+    QuiesceDbListing update;
+    update.db_age = db.get_age();
+    update.db_version = db_version();
+    std::ranges::copy(sets, std::inserter(update.sets, update.sets.end()));
+
+    dout(20) << "updating peer " << peer << " with " << sets.size() 
+      << " sets modified in db version range (" 
+      << peers[peer].diff_map.db_version << ".." << db.set_version << "]" << dendl;
+
+    auto rc = membership.send_listing_to(peer, std::move(update));
+    if (rc != 0) {
+      dout(1) << "ERROR (" << rc << ") trying to replicate db version " 
+        << db.set_version << " with " << sets.size() 
+        << " sets to the peer " << peer << dendl;
+    }
+  }
+
+  return next_event_at_age;
+}
+
+QuiesceState QuiesceSet::next_state(QuiesceState min_member_state) const {
+  ceph_assert(min_member_state > QS__INVALID);
+  ceph_assert(rstate.state < QS__TERMINAL);
+
+  if (is_releasing() && min_member_state == QS_QUIESCED) {
+    // keep releasing
+    return QS_RELEASING;
+  }
+
+  // otherwise, follow the member state
+  return min_member_state;
+}
+
+size_t QuiesceDbManager::check_peer_reports(const QuiesceSetId& set_id, const QuiesceSet& set, const QuiesceRoot& root, const QuiesceSet::MemberInfo& member, QuiesceState& min_reported_state, QuiesceState& max_reported_state) {
+  min_reported_state = QS__MAX;
+  max_reported_state = QS__INVALID;
+
+  size_t up_to_date_peers = 0;
+
+  for (auto& [peer, info] : peers) {
+    // we consider the last bit of information we had from a given peer
+    // however, we want to skip peers which haven't been bootstrapped yet
+    if (info.diff_map.db_version.set_version == 0) {
+      continue;
+    }
+    auto dit = info.diff_map.roots.find(root);
+    QuiesceState reported_state = set.get_requested_member_state();
+
+    if (dit != info.diff_map.roots.end()) {
+      // the peer has something to say about this root
+      auto const& pr_state = dit->second;
+      if (!pr_state.is_valid()) {
+        dout(5) << dsetroot("ignoring an invalid peer state ") << pr_state.state << dendl;
+        continue;
+      }
+      reported_state = pr_state.state;
+    }
+
+    // but we only consider the peer up to date given the version
+    if (info.diff_map.db_version >= QuiesceDbVersion { membership.epoch, set.version }) {
+      up_to_date_peers++;
+    }
+
+    min_reported_state = std::min(min_reported_state, reported_state);
+    max_reported_state = std::max(max_reported_state, reported_state);
+  }
+
+  if (min_reported_state == QS__MAX) {
+    min_reported_state = set.get_requested_member_state();
+    max_reported_state = set.get_requested_member_state();
+  }
+
+  return up_to_date_peers;
+}
+
+QuiesceTimeInterval QuiesceDbManager::leader_upkeep_set(Db::Sets::value_type& set_it)
+{
+  auto& [set_id, set] = set_it;
+
+  if (!set.is_active()) {
+    return QuiesceTimeInterval::max();
+  }
+
+  QuiesceTimeInterval end_of_life = QuiesceTimeInterval::max();
+
+  const auto db_age = db.get_age();
+  // no quiescing could have started before the current db_age
+
+  QuiesceState min_member_state = QS__MAX;
+  size_t included_members = 0;
+  // for each included member, apply recorded acks and check quiesce timeouts
+  for (auto& [root, member] : set.members) {
+    if (member.excluded) {
+      continue;
+    }
+    included_members++;
+
+    QuiesceState min_reported_state;
+    QuiesceState max_reported_state;
+
+    size_t reporting_peers = check_peer_reports(set_id, set, root, member, min_reported_state, max_reported_state);
+    auto effective_state = set.get_effective_member_state(min_reported_state);
+
+    if (max_reported_state >= QS__FAILURE) {
+      // if at least one peer is reporting a failure state then move to it
+      dout(5) << dsetroot("reported by at least one peer as: ") << max_reported_state << dendl;
+      if (member.rstate.update(max_reported_state, db_age)) {
+        dout(15) << dsetroot("updating member state to ") << member.rstate.state << dendl;
+        set.version = db.set_version + 1;
+      }
+    } else if (effective_state < member.rstate.state) {
+      // someone has reported a rollback state for the root
+      dout(15) << dsetroot("reported by at least one peer as ") << min_reported_state << " vs. the expected " << member.rstate.state << dendl;
+      if (member.rstate.update(effective_state, db_age)) {
+        dout(15) << dsetroot("updating member state to ") << member.rstate.state << dendl;
+        set.version = db.set_version + 1;
+      }
+    } else if (reporting_peers == peers.size()) {
+      dout(20) << dsetroot("min reported state for all (") << reporting_peers << ") peers: " << min_reported_state 
+          << ". Effective state: " << effective_state << dendl;
+      if (member.rstate.update(effective_state, db_age)) {
+        dout(15) << dsetroot("updating member state to ") << member.rstate.state << dendl;
+        set.version = db.set_version + 1;
+      }
+    }
+
+    if (member.is_quiescing()) {
+      // the quiesce timeout applies in this case
+      auto timeout_at_age = interval_saturate_add(member.rstate.at_age, set.timeout);
+      if (timeout_at_age <= db_age) {
+        // NB: deliberately not changing the member state
+        dout(10) << dsetroot("detected a member quiesce timeout") << dendl;
+        ceph_assert(set.rstate.update(QS_TIMEDOUT, db_age));
+        set.version = db.set_version + 1;
+        break;
+      }
+      end_of_life = std::min(end_of_life, timeout_at_age);
+    } else if (member.is_failed()) {
+      // if at least one member is in a failure state
+      // then the set must receive it as well
+      dout(5) << dsetroot("propagating the terminal member state to the set level: ") << member.rstate.state << dendl;
+      ceph_assert(set.rstate.update(member.rstate.state, db_age));
+      set.version = db.set_version + 1;
+      break;
+    }
+
+    min_member_state = std::min(min_member_state, member.rstate.state);
+  }
+
+  if (!set.is_active()) {
+    return QuiesceTimeInterval::max();
+  }
+
+  // we should have at least one included members to be active
+  ceph_assert(included_members > 0);
+  auto next_state = set.next_state(min_member_state);
+
+  if (set.rstate.update(next_state, db_age)) {
+    set.version = db.set_version + 1;
+    dout(15) << dset("updated set state to match member reports: ") << set.rstate.state << dendl;
+  }
+
+  if (set.is_quiesced() || set.is_released()) {
+    // any awaits pending on this set should be completed now,
+    // before the set may enter a QS_EXPIRED state
+    // due to a zero expiration timeout.
+    // this could be used for barriers.
+    auto range = awaits.equal_range(set_id);
+    for (auto it = range.first; it != range.second; it++) {
+      done_requests[it->second.req_ctx] = 0;
+      if (set.is_quiesced()) {
+        // since we've just completed a _quiesce_ await
+        // we should also reset the recorded age of the QUIESCED state
+        // to postpone the expiration time checked below
+        set.rstate.at_age = db_age;
+        set.version = db.set_version + 1;
+        dout(20) << dset("reset quiesced state age upon successful await") << dendl;
+      }
+    }
+    awaits.erase(range.first, range.second);
+  }
+
+  // check timeouts:
+  if (set.is_quiescing()) {
+    // sanity check that we haven't missed this before
+    ceph_assert(end_of_life > db_age);
+  } else if (set.is_active()) {
+    auto expire_at_age = interval_saturate_add(set.rstate.at_age, set.expiration);
+    if (expire_at_age <= db_age) {
+      // we have expired
+      ceph_assert(set.rstate.update(QS_EXPIRED, db_age));
+      set.version = db.set_version + 1;
+    } else {
+      end_of_life = std::min(end_of_life, expire_at_age);
+    }
+  }
+
+  return end_of_life;
+}
+
+QuiesceTimeInterval QuiesceDbManager::leader_upkeep_awaits()
+{
+  QuiesceTimeInterval next_event_at_age = QuiesceTimeInterval::max();
+  for (auto it = awaits.begin(); it != awaits.end();) {
+    auto & [set_id, actx] = *it;
+    Db::Sets::const_iterator set_it = db.sets.find(set_id);
+
+    int rc = db.get_age() >= actx.expire_at_age ? EINPROGRESS : EBUSY;
+
+    if (set_it == db.sets.cend()) {
+      rc = ENOENT;
+    } else {
+      auto const & set = set_it->second;
+
+      switch(set.rstate.state) {
+        case QS_CANCELED:
+          rc = ECANCELED;
+          break;
+        case QS_EXPIRED:
+        case QS_TIMEDOUT:
+          rc = ETIMEDOUT; 
+          break;
+        case QS_QUIESCED:
+          rc = 0; // fallthrough
+        case QS_QUIESCING:
+          ceph_assert(!actx.req_ctx->request.is_release());
+          break;
+        case QS_RELEASED:
+          rc = 0; // fallthrough
+        case QS_RELEASING:
+          if (!actx.req_ctx->request.is_release()) {
+            // technically possible for a quiesce await
+            // to get here if a concurrent release request
+            // was submitted in the same batch;
+            // see the corresponding check in
+            // `leader_process_request`
+            rc = EPERM;
+          }
+          break;
+        case QS_FAILED:
+          rc = EBADF;
+          break;
+        default: ceph_abort("unexpected quiesce set state");
+      }
+    }
+
+    if (rc != EBUSY) {
+      dout(10) << "completing an await for the set '" << set_id << "' with rc: " << rc << dendl;
+      done_requests[actx.req_ctx] = rc;
+      it = awaits.erase(it);
+    } else {
+      next_event_at_age = std::min(next_event_at_age, actx.expire_at_age);
+      ++it;
+    }
+  }
+  return next_event_at_age;
+}
+
+static QuiesceTimeInterval get_root_ttl(const QuiesceSet & set, const QuiesceSet::MemberInfo &member, QuiesceTimeInterval db_age) {
+
+  QuiesceTimeInterval end_of_life = db_age;
+
+  if (set.is_quiesced() || set.is_releasing()) {
+    end_of_life = set.rstate.at_age + set.expiration;
+  } else if (set.is_active()) {
+    auto age = db_age; // taking the upper bound here
+    if (member.is_quiescing()) {
+      // we know that this member is on a timer
+      age = member.rstate.at_age;
+    }
+    end_of_life = age + set.timeout; 
+  }
+
+  if (end_of_life > db_age) {
+    return end_of_life - db_age;
+  } else {
+    return QuiesceTimeInterval::zero();
+  }
+}
+
+void QuiesceDbManager::calculate_quiesce_map(QuiesceMap &map)
+{
+  map.roots.clear();
+  map.db_version = db_version();
+  auto db_age = db.get_age();
+
+  for(auto & [set_id, set]: db.sets) {
+    if (set.is_active()) {
+      // we only report active sets;
+      for(auto & [root, member]: set.members) {
+        if (member.excluded) {
+          continue;
+        }
+
+        // for a quiesce map, we want to report active roots as either QUIESCING or RELEASING
+        // this is to make sure that clients always have a reason to report back and confirm
+        // the quiesced state.
+        auto requested = set.get_requested_member_state();
+        auto ttl = get_root_ttl(set, member, db_age);
+        auto root_it = map.roots.try_emplace(root, QuiesceMap::RootInfo { requested, ttl }).first;
+
+        // the min below resolves conditions when members representing the same root have different state/ttl
+        // e.g. if at least one member is QUIESCING then the root should be QUIESCING
+        root_it->second.state = std::min(root_it->second.state, requested);
+        root_it->second.ttl = std::min(root_it->second.ttl, ttl);
+      }
+    }
+  }
+}
diff --git a/src/mds/QuiesceDbManager.h b/src/mds/QuiesceDbManager.h
new file mode 100644
index 000000000000..a20b0cfc70ac
--- /dev/null
+++ b/src/mds/QuiesceDbManager.h
@@ -0,0 +1,303 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#pragma once
+#include "mds/QuiesceDb.h"
+#include "include/Context.h"
+#include <memory>
+#include <functional>
+#include <mutex>
+#include <condition_variable>
+#include <set>
+#include <queue>
+
+template <>
+struct std::hash<mds_gid_t> {
+  size_t operator()(const mds_gid_t& gid) const
+  {
+    return hash<uint64_t> {}(gid);
+  }
+};
+
+struct QuiesceClusterMembership {
+  static const QuiesceInterface::PeerId INVALID_MEMBER;
+
+  epoch_t epoch = 0;
+  fs_cluster_id_t fs_id = FS_CLUSTER_ID_NONE;
+  std::string fs_name;
+
+  QuiesceInterface::PeerId me = INVALID_MEMBER;
+  QuiesceInterface::PeerId leader = INVALID_MEMBER;
+  std::set<QuiesceInterface::PeerId> members;
+
+  // A courier interface to decouple from the messaging layer
+  // Failures can be ignored, manager will call this repeatedly if needed
+  QuiesceInterface::DbPeerUpdate send_listing_to;
+  QuiesceInterface::AgentAck send_ack;
+
+  bool is_leader() const { return leader == me && me != INVALID_MEMBER; }
+};
+
+class QuiesceDbManager {
+  public:
+
+    struct RequestContext : public Context {
+      QuiesceDbRequest request;
+      QuiesceDbListing response;
+    };
+
+    QuiesceDbManager() : quiesce_db_thread(this) {};
+    virtual ~QuiesceDbManager()
+    {
+      update_membership({});
+    }
+
+    // This will reset the manager state
+    // according to the new cluster config
+    void update_membership(const QuiesceClusterMembership& new_membership) {
+      update_membership(new_membership, nullptr);
+    }
+    void update_membership(const QuiesceClusterMembership& new_membership, RequestContext* inject_request);
+
+    // ============================
+    // quiesce db leader interface: 
+    //    -> EPERM unless this is the leader
+    
+    // client interface to the DB
+    int submit_request(RequestContext* ctx) {
+      std::lock_guard l(submit_mutex);
+
+      if (!cluster_membership || !cluster_membership->is_leader()) {
+        return -ENOTTY;
+      }
+
+      pending_requests.push_back(ctx);
+      submit_condition.notify_all();
+      return 0;
+    }
+    // acks the messaging system
+    int submit_ack_from(QuiesceInterface::PeerId sender, const QuiesceMap& diff_map) {
+      std::lock_guard l(submit_mutex);
+
+      if (!cluster_membership || !cluster_membership->is_leader()) {
+        return -EPERM;
+      }
+
+      if (!cluster_membership->members.contains(sender)) {
+        return -ESTALE;
+      }
+
+      pending_acks.push({ sender, diff_map });
+      submit_condition.notify_all();
+      return 0;
+    }
+    
+    // =============================
+    // quiesce db replica interface:
+    //    -> EPERM if this is the leader
+
+    // process an incoming listing from a leader
+    int submit_listing_from(QuiesceInterface::PeerId sender, QuiesceDbListing&& listing) {
+      std::lock_guard l(submit_mutex);
+
+      if (!cluster_membership) {
+        return -EPERM;
+      }
+
+      if (cluster_membership->epoch != listing.db_version.epoch) {
+        return -ESTALE;
+      }
+
+      pending_db_updates.push({sender, std::move(listing)});
+      submit_condition.notify_all();
+      return 0;
+    }
+
+    // =============================
+    // Quiesce Agent interface:
+
+    int submit_agent_ack(QuiesceMap&& diff_map)
+    {
+      std::unique_lock l(submit_mutex);
+      if (!cluster_membership) {
+        return -EPERM;
+      }
+
+      if (cluster_membership->leader == cluster_membership->me) {
+        // local delivery
+        pending_acks.push({ cluster_membership->me, std::move(diff_map) });
+        submit_condition.notify_all();
+      } else {
+        // send to the leader outside of the lock
+        auto send_ack = cluster_membership->send_ack;
+        l.unlock();
+        send_ack(std::move(diff_map));
+      }
+      return 0;
+    }
+
+    struct AgentCallback {
+      using Notify = std::function<bool(QuiesceMap&)>;
+      Notify notify;
+      QuiesceDbVersion if_newer = {0, 0};
+
+      AgentCallback(const Notify &notify, QuiesceDbVersion if_newer = {0, 0})
+          : notify(notify)
+          , if_newer(if_newer)
+      {
+      }
+    };
+
+    std::optional<AgentCallback> reset_agent_callback(AgentCallback::Notify notify, QuiesceDbVersion if_newer = {0, 0}) {
+      return reset_agent_callback(AgentCallback(notify, if_newer));
+    }
+
+    std::optional<AgentCallback> reset_agent_callback(std::optional<AgentCallback> callback_if_newer = std::nullopt)
+    {
+      std::lock_guard ls(submit_mutex);
+      std::lock_guard lc(agent_mutex);
+      agent_callback.swap(callback_if_newer);
+      if (agent_callback) {
+        submit_condition.notify_all();
+      }
+      return callback_if_newer;
+    }
+
+    std::optional<AgentCallback> reset_agent_callback(QuiesceDbVersion if_newer)
+    {
+      std::lock_guard ls(submit_mutex);
+      std::lock_guard lc(agent_mutex);
+      if (agent_callback) {
+        agent_callback->if_newer = if_newer;
+        submit_condition.notify_all();
+      }
+      return agent_callback;
+    }
+
+    std::optional<AgentCallback> get_agent_callback() const
+    {
+      std::lock_guard lc(agent_mutex);
+      return agent_callback;
+    }
+
+  protected:
+    mutable std::mutex submit_mutex;
+    mutable std::mutex agent_mutex;
+    std::condition_variable submit_condition;
+
+    std::optional<AgentCallback> agent_callback;
+    std::optional<QuiesceClusterMembership> cluster_membership;
+    std::queue<std::pair<QuiesceInterface::PeerId, QuiesceDbListing>> pending_db_updates;
+    std::queue<std::pair<QuiesceInterface::PeerId, QuiesceMap>> pending_acks;
+    std::deque<RequestContext*> pending_requests;
+
+    class QuiesceDbThread : public Thread {
+      public:
+        explicit QuiesceDbThread(QuiesceDbManager* qm)
+            : qm(qm)
+        {
+        }
+        void* entry() override
+        {
+          return qm->quiesce_db_thread_main();
+        }
+
+      private:
+        QuiesceDbManager* qm;
+    } quiesce_db_thread;
+
+    // =============================================
+    // The below is managed by the quiesce db thread
+
+    // the database.
+    struct Db {
+      QuiesceTimePoint time_zero;
+      QuiesceSetVersion set_version = 0;
+      using Sets = std::unordered_map<QuiesceSetId, QuiesceSet>;
+      Sets sets;
+
+      QuiesceTimeInterval get_age() const {
+        return QuiesceClock::now() - time_zero;
+      }
+      void reset() { 
+        set_version = 0; 
+        sets.clear();
+        time_zero = QuiesceClock::now();
+      }
+    } db;
+
+    QuiesceDbVersion db_version() const { return {membership.epoch, db.set_version}; }
+
+    QuiesceClusterMembership membership;
+
+    struct PeerInfo {
+        QuiesceMap diff_map;
+        QuiesceTimePoint last_seen;
+        PeerInfo(QuiesceMap&& diff_map, QuiesceTimePoint last_seen)
+            : diff_map(diff_map)
+            , last_seen(last_seen)
+        {
+        }
+        PeerInfo() { }
+    };
+    std::unordered_map<QuiesceInterface::PeerId, PeerInfo> peers;
+
+    struct AwaitContext {
+      QuiesceTimeInterval expire_at_age = QuiesceTimeInterval::zero();
+      RequestContext* req_ctx = nullptr;
+      AwaitContext(QuiesceTimeInterval exp, RequestContext* r)
+          : expire_at_age(exp)
+          , req_ctx(r)
+      {
+      }
+    };
+    // multiple awaits may be active per set
+    std::unordered_multimap<QuiesceSetId, AwaitContext> awaits;
+    std::unordered_map<RequestContext*, int> done_requests;
+
+    void* quiesce_db_thread_main();
+
+    void db_thread_enter() {
+      // this will invalidate the membership, see membership_upkeep()
+      membership.epoch = 0;
+      peers.clear();
+      awaits.clear();
+      done_requests.clear();
+      db.reset();
+    }
+
+    void db_thread_exit() {
+      complete_requests();
+    }
+
+    bool db_thread_has_work() const;
+
+    bool membership_upkeep();
+
+    QuiesceTimeInterval replica_upkeep(decltype(pending_db_updates)&& db_updates);
+    bool leader_bootstrap(decltype(pending_db_updates)&& db_updates, QuiesceTimeInterval &next_event_at_age);
+    QuiesceTimeInterval leader_upkeep(decltype(pending_acks)&& acks, decltype(pending_requests)&& requests);
+    
+
+    void leader_record_ack(QuiesceInterface::PeerId from, QuiesceMap&& diff_map);
+    int leader_process_request(RequestContext* req_ctx);
+    bool sanitize_roots(QuiesceDbRequest::Roots &roots);
+    int leader_update_set(Db::Sets::value_type& set_it, const QuiesceDbRequest& req);
+    QuiesceTimeInterval leader_upkeep_set(Db::Sets::value_type& set_it);
+    QuiesceTimeInterval leader_upkeep_db();
+    QuiesceTimeInterval leader_upkeep_awaits();
+
+    size_t check_peer_reports(const QuiesceSetId& set_id, const QuiesceSet& set, const QuiesceRoot& root, const QuiesceSet::MemberInfo& member, QuiesceState& min_reported_state, QuiesceState& max_reported_state);
+
+    void calculate_quiesce_map(QuiesceMap &map);
+
+    void complete_requests();
+};
\ No newline at end of file
diff --git a/src/test/mds/CMakeLists.txt b/src/test/mds/CMakeLists.txt
index 857b205e1966..39b47bb2da4d 100644
--- a/src/test/mds/CMakeLists.txt
+++ b/src/test/mds/CMakeLists.txt
@@ -14,3 +14,13 @@ add_executable(unittest_mds_sessionfilter
 add_ceph_unittest(unittest_mds_sessionfilter)
 target_link_libraries(unittest_mds_sessionfilter mds osdc ceph-common global ${BLKID_LIBRARIES})
 
+# unittest_mds_quiesce_db
+add_executable(unittest_mds_quiesce_db
+  TestQuiesceDb.cc
+  ../../../src/mds/QuiesceDbManager.cc
+  ../../../src/mds/BoostUrlImpl.cc
+  $<TARGET_OBJECTS:unit-main>
+)
+add_ceph_unittest(unittest_mds_quiesce_db)
+target_link_libraries(unittest_mds_quiesce_db ceph-common global)
+
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
new file mode 100644
index 000000000000..e820db404483
--- /dev/null
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -0,0 +1,1613 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, RedHat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "mds/QuiesceDbManager.h"
+#include "gtest/gtest.h"
+#include "common/Cond.h"
+#include <ranges>
+#include <system_error>
+#include <thread>
+#include <queue>
+#include <functional>
+#include <algorithm>
+#include <iostream>
+#include <future>
+#include <list>
+#include <array>
+#include <utility>
+#include <cstdlib>
+#include "fmt/format.h"
+#include "common/debug.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mds_quiesce
+#undef dout_prefix
+#define dout_prefix *_dout << "== test == "
+
+struct GenericVerboseErrorCode {
+  int error_code;
+  GenericVerboseErrorCode(int error_code) : error_code(std::abs(error_code)) {}
+  auto operator<=>(const GenericVerboseErrorCode&) const = default;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const GenericVerboseErrorCode& ec)
+{
+  if (0 == ec.error_code) {
+    return os << "Success(0)";
+  } else {
+    return os << std::generic_category().message(ec.error_code) << "(" << ec.error_code << ")";
+  }
+};
+
+class QuiesceDbTest: public testing::Test {
+  protected:
+    template <class _Rep = std::chrono::seconds::rep, class _Period = std::chrono::seconds::period, typename D = std::chrono::duration<_Rep, _Period>, class Function, class... Args>
+    static bool timed_run(D timeout, Function&& f, Args&&... args)
+    {
+      std::promise<void> done;
+      auto future = done.get_future();
+
+      auto job = std::bind(f, args...);
+
+      auto tt = std::thread([job = std::move(job)](std::promise<void> done) {
+        job();
+        done.set_value();
+      },
+          std::move(done));
+
+      tt.detach();
+
+      return future.wait_for(timeout) != std::future_status::timeout;
+    }
+    struct TestQuiesceDbManager: public QuiesceDbManager
+    {
+      using QuiesceDbManager::QuiesceDbManager;
+      using QuiesceDbManager::Db;
+      Db& internal_db() {
+        return db;
+      }
+      QuiesceClusterMembership& internal_membership() {
+        return membership;
+      }
+      decltype(pending_requests)& internal_pending_requests() {
+        return pending_requests;
+      }
+      decltype(awaits)& internal_awaits() {
+        return awaits;
+      }
+      decltype(peers)& internal_peers() {
+        return peers;
+      }
+    };
+
+    epoch_t epoch = 0;
+    std::map<QuiesceInterface::PeerId, std::unique_ptr<TestQuiesceDbManager>> managers;
+
+    std::mutex comms_mutex;
+    std::condition_variable comms_cond;
+
+    fs_cluster_id_t fs_id = 1;
+    std::string fs_name = "a";
+
+    std::unordered_map<QuiesceInterface::PeerId, QuiesceMap> latest_acks;
+    using AckHook = std::function<bool(QuiesceInterface::PeerId, QuiesceMap&)>;
+    std::list<std::pair<AckHook, std::promise<void>>> ack_hooks;
+
+    std::future<void> add_ack_hook(AckHook&& predicate)
+    {
+      std::lock_guard l(comms_mutex);
+      auto &&[_, promise] = ack_hooks.emplace_back(predicate, std::promise<void> {});
+      return promise.get_future();
+    }
+
+    void SetUp() override {
+      for (QuiesceInterface::PeerId r = mds_gid_t(1); r < mds_gid_t(11); r++) {
+        managers[r].reset(new TestQuiesceDbManager());
+      }
+    }
+
+    void TearDown() override
+    {
+      dout(6) << "\n tearing down the cluster" << dendl;
+      // We want to cause the managers to destruct
+      // before we have the last_request destructed.
+      // We should remove entries from `managers` under the comms lock
+      // to avoid race with attempts of messaging between the managers.
+      // Then we actually clear the map, destructing the managers,
+      // outside the lock: the destruction will join the db threads
+      // which in turn migh attempt to send a message
+      std::unique_lock l(comms_mutex);
+      auto mgrs = std::move(managers);
+      l.unlock();
+      mgrs.clear();
+    }
+
+    void configure_cluster(std::vector<QuiesceInterface::PeerId> leader_and_replicas = { mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) })
+    {
+      ++epoch;
+      ASSERT_GE(leader_and_replicas.size(), 1);
+      std::set<QuiesceInterface::PeerId> members(leader_and_replicas.begin(), leader_and_replicas.end());
+      auto leader = leader_and_replicas[0];
+      for (const auto &[this_peer, mgr] : managers) {
+        QuiesceClusterMembership mem = {
+          epoch,
+          fs_id,
+          fs_name,
+          this_peer,
+          leader,
+          members,
+          [epoch = this->epoch, this, leader, me = this_peer](auto recipient, auto listing) {
+            std::unique_lock l(comms_mutex);
+            if (epoch == this->epoch) {
+              if (this->managers.contains(recipient)) {
+                dout(10) << "listing from " << me << " (leader=" << leader << ") to " << recipient << " for version " << listing.db_version << " with " << listing.sets.size() << " sets" << dendl;
+                this->managers[recipient]->submit_listing_from(me, std::move(listing));
+                comms_cond.notify_all();
+                return 0;
+              }
+            }
+            return -1;
+          },
+          [epoch = this->epoch, this, leader, me = this_peer](auto diff_map) {
+            std::unique_lock l(comms_mutex);
+            if (epoch == this->epoch) {
+              if (this->managers.contains(leader)) {
+                std::queue<std::promise<void>> done_hooks;
+                dout(10) << "ack from " << me << " to the leader (" << leader << ") for version " << diff_map.db_version << " with " << diff_map.roots.size() << " roots" << dendl;
+                auto [it, inserted] = latest_acks.insert({me, diff_map});
+                if (!inserted) {
+                  if (it->second.db_version == diff_map.db_version) {
+                    if (it->second.roots == diff_map.roots) {
+                      dout(1) << "WARNING: detected a potentialy redundant ack" << dendl;
+                    }
+                  }
+                  it->second = diff_map;
+                }
+                for (auto it = ack_hooks.begin(); it != ack_hooks.end();) {
+                  if (it->first(me, diff_map)) {
+                    done_hooks.emplace(std::move(it->second));
+                    it = ack_hooks.erase(it);
+                  } else {
+                    it++;
+                  }
+                }
+                this->managers[leader]->submit_ack_from(me, std::move(diff_map));
+                comms_cond.notify_all();
+                l.unlock();
+                while(!done_hooks.empty()) {
+                  done_hooks.front().set_value();
+                  done_hooks.pop();
+                }
+                return 0;
+              }
+            }
+            return -1;
+          }
+        };
+        mgr->update_membership(mem);
+      }
+      dout(6) << "\n === configured cluster with the following members, starting with the leader: " << leader_and_replicas << dendl;
+    }
+
+    struct TestRequestContext: public QuiesceDbManager::RequestContext, public C_SaferCond {
+      void finish(int r) override { C_SaferCond::finish(r); }
+      void complete(int r) override { C_SaferCond::complete(r); }
+
+      const QuiesceDbTest& parent;
+      TestRequestContext(const QuiesceDbTest& parent) : parent(parent) {}
+      ~TestRequestContext() {
+        wait();
+      }
+
+      bool start(std::invocable<QuiesceDbRequest&> auto const & c)
+      {
+        done = false;
+        response.clear();
+        request.reset(c);
+
+        int rr = -1;
+
+        for (auto& [rank, mgr] : parent.managers) {
+          if (!(rr = mgr->submit_request(this))) {
+            break;
+          }
+        }
+
+        if (rr == EPERM) {
+          // change the error to something never returned for a request
+          // EPIPE seems reasonable as we couldn't find the leader to send the command to
+          complete(EPIPE);
+          return false;
+        }
+
+        return true;
+      }
+
+      GenericVerboseErrorCode check_result() {
+        std::unique_lock l{lock};
+        if (done) {
+          return ERR(rval);
+        }
+        // this error is never returned by the manager
+        return NA();
+      }
+
+      GenericVerboseErrorCode wait_result() {
+        return ERR(wait());
+      }
+
+      GenericVerboseErrorCode wait_result_for(double seconds)
+      {
+        return ERR(wait_for(seconds));
+      }
+    };
+
+    std::deque<std::unique_ptr<TestRequestContext>> requests;
+    std::unique_ptr<TestRequestContext> last_request;
+
+    const QuiesceDbManager::AgentCallback::Notify QUIESCING_AGENT_CB = [](QuiesceMap& quiesce_map) {
+      dout(15) << "QUIESCING_AGENT_CB: notified with " << quiesce_map.roots.size() << " roots for version " << quiesce_map.db_version << dendl;
+      for (auto it = quiesce_map.roots.begin(); it != quiesce_map.roots.end();) {
+        switch (it->second.state) {
+        case QS_QUIESCING:
+          it->second.state = QS_QUIESCED;
+          dout(10) << "QUIESCING_AGENT_CB: reporting '" << it->first << "' as " << it->second.state << dendl;
+          it++;
+          break;
+        default:
+          it = quiesce_map.roots.erase(it);
+          break;
+        }
+      }
+      return true;
+    };
+
+    const QuiesceDbManager::AgentCallback::Notify FAILING_AGENT_CB = [](QuiesceMap& quiesce_map) {
+      dout(15) << "FAILING_AGENT_CB: notified with " << quiesce_map.roots.size() << " roots for version " << quiesce_map.db_version << dendl;
+      for (auto it = quiesce_map.roots.begin(); it != quiesce_map.roots.end();) {
+        switch (it->second.state) {
+        case QS_QUIESCING:
+          it->second.state = QS_FAILED;
+          dout(10) << "FAILING_AGENT_CB: reporting '" << it->first << "' as " << it->second.state << dendl;
+          it++;
+          break;
+        default:
+          it = quiesce_map.roots.erase(it);
+          break;
+        }
+      }
+      return true;
+    };
+
+    const QuiesceDbManager::AgentCallback::Notify SILENT_AGENT_CB = [](QuiesceMap& quiesce_map) {
+      dout(15) << "SILENT_AGENT_CB: nacking quiesce map version " << quiesce_map.db_version << " with " << quiesce_map.roots.size() << " roots" << dendl;
+      return false;
+    };
+
+    GenericVerboseErrorCode
+    run_request(std::invocable<QuiesceDbRequest&> auto const& c)
+    {
+      last_request.reset(new TestRequestContext(*this));
+      last_request->start(c);
+      return ERR(last_request->wait());
+    }
+
+    GenericVerboseErrorCode
+    run_request_for(double seconds, std::invocable<QuiesceDbRequest&> auto const& c)
+    {
+      last_request.reset(new TestRequestContext(*this));
+      last_request->start(c);
+      return ERR(last_request->wait_for(seconds));
+    }
+
+    TestRequestContext& start_request(std::invocable<QuiesceDbRequest&> auto const& c)
+    {
+      auto &ptr = requests.emplace_back(new TestRequestContext(*this));
+      ptr->start(c);
+      return *ptr;
+    }
+
+    TestQuiesceDbManager::Db& db(QuiesceInterface::PeerId peer) {
+      return managers[peer]->internal_db();
+    }
+
+    static GenericVerboseErrorCode ERR(int val) {
+      return GenericVerboseErrorCode(val);
+    }
+    static GenericVerboseErrorCode OK()
+    {
+      return ERR(0);
+    }
+    static GenericVerboseErrorCode NA() {
+      return ERR(EBUSY);
+    }
+
+    static QuiesceTimeInterval sec(double val) {
+      return std::chrono::duration_cast<QuiesceTimeInterval>(std::chrono::duration<double>(val));
+    }
+};
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, ManagerStartup) {
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+  ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(2) }));
+  ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2) }));
+  ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, SetCreation) {
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  // create a named set by resetting roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set0";
+    r.reset_roots({"root1"});
+  }));
+
+  // the set must have timed out immediately since we haven't configured
+  // the expiration timeout.
+  ASSERT_TRUE(last_request->response.sets.contains("set0"));
+  EXPECT_EQ(QS_TIMEDOUT, last_request->response.sets.at("set0").rstate.state);
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.contains(*last_request->request.set_id));
+
+  // create a named set by including roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.include_roots({"root1"});
+  }));
+
+  // the set must have timed out immediately since we haven't configured
+  // the expiration timeout. 
+  ASSERT_TRUE(last_request->response.sets.contains("set1"));
+  EXPECT_EQ(QS_TIMEDOUT, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.contains(*last_request->request.set_id));
+
+  // create a new unique set by including roots
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.include_roots({"root2"});
+  }));
+
+  // the manager must have filled the set id with a unique value
+  ASSERT_TRUE(last_request->request.set_id.has_value());
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.contains(*last_request->request.set_id));
+
+  // create a new unique set by resetting roots
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.reset_roots({"root2"});
+  }));
+
+  // the manager must have filled the set id with a unique value
+  ASSERT_TRUE(last_request->request.set_id.has_value());
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.contains(*last_request->request.set_id));
+
+  // prevent modification of a named set when a new set is desired
+  EXPECT_EQ(ERR(ESTALE), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.if_version = 0;
+    r.roots.emplace("root3");
+  }));
+  EXPECT_EQ(1, last_request->response.sets.size());
+  EXPECT_TRUE(last_request->response.sets.contains("set1"));
+
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.if_version = 0;
+    r.roots.emplace("root4");
+  }));
+
+  EXPECT_EQ(1, last_request->response.sets.size());
+  EXPECT_TRUE(last_request->response.sets.contains("set2"));
+  EXPECT_EQ(QS_TIMEDOUT, last_request->response.sets.at("set2").rstate.state);
+
+  // let's try to create a new named but expect it to have non-zero version
+  EXPECT_EQ(ERR(ENOENT), run_request([](auto& r) {
+    r.set_id = "set3";
+    r.if_version = 1;
+    r.roots.emplace("root4");
+  }));
+
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  // let's try to create a new anonymous but expect it to have non-zero version
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.if_version = 2;
+    r.roots.emplace("root4");
+  }));
+
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  // an empty string is a valid set id.
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "";
+    r.roots.emplace("root1");
+  }));
+}
+
+template<class T>
+constexpr
+std::array<std::optional<T>, 2> nullopt_and_default() {
+  return {std::nullopt, T{}};
+}
+
+template<class F, class... V, size_t... S>
+  requires std::invocable<F, V...>
+void cartesian_apply(F func, std::array<V, S> const & ... array_args) {
+  // inspired by https://stackoverflow.com/a/31169617/5171225
+
+  // the iteration count is a product of all array sizes
+  const long long N = (S * ...);
+
+  for (long long n = 0; n < N; ++n) {
+    std::lldiv_t q { n, 0 };
+
+    // we use parameter pack expansion as part of the brace initializer
+    // to perform sequential calculation of the 
+    auto apply_tuple = std::tuple<V const &...> { 
+      (q = div(q.quot, array_args.size()), array_args.at(q.rem)) 
+      ... 
+    };
+
+    if (!std::apply(func, apply_tuple)) {
+      return;
+    }
+  }
+}
+
+template<class... Args>
+void coutall(Args&&... args) {
+  int dummy[sizeof...(args)] = { (std::cout << args, std::cout << " ", 0)... };
+  std::cout << std::endl;
+}
+
+TEST_F(QuiesceDbTest, QuiesceRequestValidation)
+{
+
+  auto checkRequest = [](
+    decltype(std::declval<QuiesceDbRequest>().control.roots_op) const& op,
+    decltype(std::declval<QuiesceDbRequest>().set_id)           const& set_id,
+    decltype(std::declval<QuiesceDbRequest>().if_version)       const& if_version,
+    decltype(std::declval<QuiesceDbRequest>().timeout)          const& timeout,
+    decltype(std::declval<QuiesceDbRequest>().expiration)       const& expiration,
+    decltype(std::declval<QuiesceDbRequest>().await)            const& await,
+    decltype(std::declval<QuiesceDbRequest>().roots)            const& roots) {
+      QuiesceDbRequest r;
+      r.control.roots_op = op;
+      r.set_id = set_id;
+      r.if_version = if_version;
+      r.timeout = timeout;
+      r.expiration = expiration;
+      r.await = await;
+      r.roots = roots;
+
+      if (op >= QuiesceDbRequest::RootsOp::__INVALID) {
+        EXPECT_FALSE(r.is_valid())
+          << "op: " << r.op_string() << ", set_id: " << bool(set_id) 
+          << ", if_version: " << bool(if_version) 
+          << ", timeout: " << bool(timeout) << ", expiration: " 
+          << bool(expiration) << ", await: " 
+          << bool(await) << ", roots.size(): " << roots.size();
+      } else {
+        // if set id is provided, all goes
+        if (set_id) {
+          EXPECT_TRUE(r.is_valid())
+            << "op: " << r.op_string() << ", set_id: " << bool(set_id) 
+            << ", if_version: " << bool(if_version) 
+            << ", timeout: " << bool(timeout) << ", expiration: " 
+            << bool(expiration) << ", await: " 
+            << bool(await) << ", roots.size(): " << roots.size();
+        } else {
+          // without the set id we can create a new set
+          // or perform operations on all sets
+          if (roots.size() > 0) {
+            // if roots are provided, we assume creation
+            // all combinations are valid unless it's an exclude,
+            // which doesn't make sense without a set id
+            EXPECT_NE(r.is_exclude(), r.is_valid())
+              << "op: " << r.op_string() << ", set_id: " << bool(set_id) 
+              << ", if_version: " << bool(if_version) 
+              << ", timeout: " << bool(timeout) << ", expiration: " 
+              << bool(expiration) << ", await: " 
+              << bool(await) << ", roots.size(): " << roots.size();
+          } else {
+            // means it's a query or a "cancel all"
+            // no other parameters should be set
+            if (if_version || timeout || expiration || await) {
+              EXPECT_FALSE(r.is_valid())
+                << "op: " << r.op_string() << ", set_id: " << bool(set_id) 
+                << ", if_version: " << bool(if_version) 
+                << ", timeout: " << bool(timeout) << ", expiration: " 
+                << bool(expiration) << ", await: " 
+                << bool(await) << ", roots.size(): " << roots.size();
+            } else {
+              EXPECT_NE(r.is_release(), r.is_valid())
+                << "op: " << r.op_string() << ", set_id: " << bool(set_id) 
+                << ", if_version: " << bool(if_version) 
+                << ", timeout: " << bool(timeout) << ", expiration: " 
+                << bool(expiration) << ", await: " 
+                << bool(await) << ", roots.size(): " << roots.size();
+            }
+          }
+        }
+      }
+
+      return !testing::Test::HasFailure();
+  };
+
+  const auto ops = std::array { QuiesceDbRequest::RootsOp::INCLUDE_OR_QUERY, QuiesceDbRequest::RootsOp::EXCLUDE_OR_RELEASE, QuiesceDbRequest::RootsOp::RESET_OR_CANCEL, QuiesceDbRequest::RootsOp::__INVALID };
+  const auto strings = nullopt_and_default<std::string>();
+  const auto versions = nullopt_and_default<QuiesceSetVersion>();
+  const auto intervals = nullopt_and_default<QuiesceTimeInterval>();
+  const auto roots = std::array { QuiesceDbRequest::Roots {}, QuiesceDbRequest::Roots { "root1" } };
+
+  cartesian_apply(checkRequest,
+      ops, strings, versions, intervals, intervals, intervals, roots);
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, RootSanitization)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+  // a positive test with all kinds of expected fixes
+  ASSERT_EQ(OK(), run_request([this](auto& r) {
+    r.set_id = "set1";
+    r.include_roots({
+      "file:root1",
+      fmt::format("file://{}/root2", fs_id),
+      fmt::format("//{}/root3", fs_name),
+      fmt::format("inode://{}/4", fs_id),
+      fmt::format("inode://{}/5", fs_name),
+      "inode:18446744073709551615",
+      "inode:/18446744073709551614",
+      "inode:/18446744073709551613/",
+      "root6/.///./..////root6//"
+    });
+  }));
+
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root1"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root2"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root3"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("inode:4"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("inode:5"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("inode:18446744073709551615"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("inode:18446744073709551614"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("inode:18446744073709551613"));
+  EXPECT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root6/root6"));
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.include_roots({
+      "//10/root1"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "//badfsname/root1"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode://badfsname/1"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:-4"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:18446744073709551616" // too big to fit a uint64_t
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:1/2/3/4"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:abcd"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:123-456"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:"
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.set_id = "badset";
+    r.include_roots({
+      "inode:0" // zero is an invalid inodeno
+    });
+  }));
+  EXPECT_EQ(0, last_request->response.sets.size());
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, SetModification)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  // create a named set by including roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(60);
+    r.expiration = sec(60);
+    r.include_roots({"root1"});
+  }));
+
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.contains("set1"));
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root1"));
+
+  // include more roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.include_roots({"root2", "root3"});
+  }));
+
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root2"));
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root3"));
+  ASSERT_EQ(db(mds_gid_t(1)).sets.at("set1").members.size(), 3);
+
+  auto latest_v = last_request->response.sets.at("set1").version;
+
+  // including present roots shouldn't bump the version
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.include_roots({ "root2", "root3" });
+  }));
+
+  ASSERT_EQ(latest_v, last_request->response.sets.at("set1").version);
+  ASSERT_EQ(latest_v, db(mds_gid_t(1)).sets.at("set1").version);
+
+  // resetting to the same roots shouldn't bump the version
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.reset_roots({ "root1","root2", "root3" });
+  }));
+
+  ASSERT_EQ(latest_v, last_request->response.sets.at("set1").version);
+  ASSERT_EQ(latest_v, db(mds_gid_t(1)).sets.at("set1").version);
+
+  // exclude roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.exclude_roots({ "root1", "root4" }); // root4 wasn't included, noop
+  }));
+
+  // the db doesn't delete set memebers, only marks them as excluded
+  ASSERT_EQ(db(mds_gid_t(1)).sets.at("set1").members.size(), 3);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root1").excluded);
+  ASSERT_FALSE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root2").excluded);
+  ASSERT_FALSE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root3").excluded);
+  ASSERT_FALSE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root4"));
+
+  // reset roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.reset_roots({"root4"});
+  }));
+
+  ASSERT_EQ(db(mds_gid_t(1)).sets.at("set1").members.size(), 4);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root1").excluded);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root2").excluded);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root3").excluded);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.contains("file:/root4"));
+  ASSERT_FALSE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root4").excluded);
+
+  // reset is an including op, should allow creating a set with it
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.timeout = sec(60);
+    r.expiration = sec(60);
+    r.reset_roots({"root5"});
+  }));
+
+  ASSERT_FALSE(db(mds_gid_t(1)).sets.at("set2").members.at("file:/root5").excluded);
+
+  // cancel with no set_id should cancel all active sets
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.control.roots_op = QuiesceDbRequest::RootsOp::RESET_OR_CANCEL;
+  }));
+
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set1").members.at("file:/root4").excluded);
+  ASSERT_TRUE(db(mds_gid_t(1)).sets.at("set2").members.at("file:/root5").excluded);
+
+  ASSERT_EQ(QuiesceState::QS_CANCELED, db(mds_gid_t(1)).sets.at("set1").rstate.state);
+  ASSERT_EQ(QuiesceState::QS_CANCELED, db(mds_gid_t(1)).sets.at("set2").rstate.state);
+
+  // reset can be used to resurrect a set from a terminal state
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(60);
+    r.expiration = sec(60);
+    r.reset_roots({ "root5" });
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, db(mds_gid_t(1)).sets.at("set1").rstate.state);
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, Timeouts) {
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  // install the agent callback to reach the QUIESCED state
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(0.1);
+    r.expiration = sec(0.1);
+    r.include_roots({"root1"});
+    r.await = sec(1);
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+
+  std::this_thread::sleep_for(sec(0.15));
+
+  ASSERT_EQ(QuiesceState::QS_EXPIRED, db(mds_gid_t(1)).sets.at("set1").rstate.state);
+
+  // reset can be used to resurrect a set from a terminal state
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.reset_roots({ "root5" });
+  }));
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, last_request->response.sets.at("set1").rstate.state);
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.timeout = sec(0.1);
+    r.expiration = sec(0.1);
+    r.include_roots({ "root1" });
+    r.await = sec(1);
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+
+  // prevent the db agent from acking the roots
+  managers.at(mds_gid_t(1))->reset_agent_callback(SILENT_AGENT_CB);
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.release_roots();
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_RELEASING, last_request->response.sets.at("set2").rstate.state);
+
+  std::this_thread::sleep_for(sec(0.15));
+
+  ASSERT_EQ(QuiesceState::QS_EXPIRED, db(mds_gid_t(1)).sets.at("set2").rstate.state);
+
+  // reset can be used to resurrect a set from a terminal state
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.reset_roots({ "root1" });
+  }));
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, last_request->response.sets.at("set2").rstate.state);
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set3";
+    r.timeout = sec(0.1);
+    r.include_roots({ "root1" });
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, db(mds_gid_t(1)).sets.at("set3").rstate.state);
+
+  std::this_thread::sleep_for(sec(0.15));
+
+  ASSERT_EQ(QuiesceState::QS_TIMEDOUT, db(mds_gid_t(1)).sets.at("set3").rstate.state);  // reset can be used to resurrect a set from a terminal state
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set3";
+    r.reset_roots({ "root1" });
+  }));
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, last_request->response.sets.at("set3").rstate.state);
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, Failures) {
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(0.1);
+    r.expiration = sec(0.1);
+    r.include_roots({"root1"});
+  }));
+
+  EXPECT_EQ(QuiesceState::QS_QUIESCING, last_request->response.sets.at("set1").rstate.state);
+
+  {
+    // wait for the agent to ack root1 as failed
+    auto did_ack = add_ack_hook([](auto rank, auto const& ack) {
+      return ack.roots.contains("file:/root1") && ack.roots.at("file:/root1").state == QS_FAILED;
+    });
+
+    // allow acks
+    managers.at(mds_gid_t(1))->reset_agent_callback(FAILING_AGENT_CB);
+
+    EXPECT_EQ(std::future_status::ready, did_ack.wait_for(std::chrono::milliseconds(100)));
+  }
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+  }));
+
+  EXPECT_EQ(QuiesceState::QS_FAILED, db(mds_gid_t(1)).sets.at("set1").rstate.state);
+  EXPECT_EQ(QuiesceState::QS_FAILED, last_request->response.sets.at("set1").rstate.state);
+
+  ASSERT_EQ(ERR(EBADF), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.timeout = sec(0.1);
+    r.expiration = sec(0.1);
+    r.include_roots({ "root1" });
+    r.await = sec(1);
+  }));
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, InterruptedQuiesceAwait)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  auto then = QuiesceClock::now();
+
+  // await timeout should result in a EINPROGRESS given that the set
+  // isn't modified in the meantime
+  ASSERT_EQ(ERR(EINPROGRESS), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(100);
+    r.roots.emplace("root1");
+    r.await = sec(0.1);
+  }));
+
+  ASSERT_EQ(QuiesceState::QS_QUIESCING, db(mds_gid_t(1)).sets.at("set1").rstate.state);
+  ASSERT_GE(QuiesceClock::now() - then, *last_request->request.await);
+
+  // start an asyncrhonous await request
+  auto & await = start_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  });
+
+  // flush the pending requests by running a simple query
+  EXPECT_EQ(OK(), run_request([](auto& r) { r.query("set1"); }));
+
+  // still running
+  EXPECT_EQ(NA(), await.check_result());
+
+  // modify the set but don't change roots 
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.expiration = sec(100);
+    r.timeout = sec(10);
+    r.roots.emplace("root1");
+  }));
+
+  // should still be running
+  EXPECT_EQ(NA(), await.check_result());
+
+  // add another set
+  then = QuiesceClock::now();
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.timeout = sec(0.1);
+    r.roots.emplace("root1");
+  }));
+
+  // should still be running
+  EXPECT_EQ(NA(), await.check_result());
+
+  // modify roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.roots.emplace("root2");
+  }));
+
+  EXPECT_EQ(ERR(EINTR), await.wait_result());
+
+  // start async await on set2
+  auto & await2 = start_request([](auto& r) {
+    r.set_id = "set2";
+    r.await = sec(100);
+  });
+
+  // should be running
+  EXPECT_EQ(NA(), await2.check_result());
+
+  // and another one, this time wait for it to finish
+  ASSERT_EQ(ERR(ETIMEDOUT), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.await = sec(100);
+  }));
+
+  // the other await on the same set must have finished with the same result
+  EXPECT_EQ(ERR(ETIMEDOUT), await2.wait_result());
+
+  // shouldn't have taken much longer than the timeout configured on the set
+  auto epsilon = sec(0.01);
+  ASSERT_LE(QuiesceClock::now() - then - epsilon, last_request->response.sets.at("set2").timeout);
+
+  // let's cancel set 1 while awaiting it a few times
+
+  // start async await on set1
+  auto& await3 = start_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  });
+
+  auto& await4 = start_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  });
+
+  // should be running
+  EXPECT_EQ(NA(), await3.check_result());
+  EXPECT_EQ(NA(), await4.check_result());
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.reset_roots({});
+  }));
+
+  EXPECT_EQ(ERR(ECANCELED), await3.wait_result());
+  EXPECT_EQ(ERR(ECANCELED), await4.wait_result());
+
+  // awaiting a set in a terminal state should immediately
+  // complete with the corresponding error
+  ASSERT_EQ(ERR(ECANCELED), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  }));
+  ASSERT_EQ(ERR(ETIMEDOUT), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.await = sec(100);
+  }));
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, RepeatedQuiesceAwait) {
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  // let us reach quiescing
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  // pick an expiration timeout
+  auto expiration = sec(0.1);
+
+  // create a set and let it quiesce
+  ASSERT_EQ(OK(), run_request([=](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(0.1);
+    r.expiration = expiration;
+    r.roots.emplace("root1");
+    r.await = QuiesceTimeInterval::max();
+  }));
+
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+
+  // sleep for half the expiration interval multiple times
+  // each time sending another await request
+  // the expectation is that every time we call await
+  // the expiration timer is reset, hence we should be able to
+  // sustain the loop for arbitrarily long
+  for (int i = 0; i < 10; i++) {
+    std::this_thread::sleep_for(expiration/2);
+    ASSERT_EQ(OK(), run_request([i](auto& r) {
+      r.set_id = "set1";
+      if (i % 2) {
+        // this shouldn't affect anything
+        r.reset_roots({"root1"});
+      }
+      r.await = sec(0);
+    }));
+  }
+
+  // Prevent the set from reaching the RELEASED state
+  managers.at(mds_gid_t(1))->reset_agent_callback(SILENT_AGENT_CB);
+
+  // start releasing and observe that the timer isn't reset in this case,
+  // so after a few EINPROGRESS we eventually reach timeout due to expiration
+  for (int i = 0; i < 2; i++) {
+    ASSERT_EQ(ERR(EINPROGRESS), run_request([=](auto& r) {
+      r.set_id = "set1";
+      r.release_roots();
+      r.await = (expiration*2)/5;
+    }));
+  }
+
+  // NB: the ETIMEDOUT is the await result, while the set itself should be EXPIRED
+  EXPECT_EQ(ERR(ETIMEDOUT), run_request([=](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+    r.await = expiration;
+  }));
+
+  EXPECT_EQ(QS_EXPIRED, last_request->response.sets.at("set1").rstate.state);
+
+  EXPECT_EQ(ERR(ETIMEDOUT), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(0.1);
+  }));
+
+  EXPECT_EQ(ERR(EPERM), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+  }));
+
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, ReleaseAwait)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  // create some sets
+  for (auto&& set_id : { "set1", "set2", "set3" }) {
+    ASSERT_EQ(OK(), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.timeout = sec(100);
+      r.expiration = sec(100);
+      r.include_roots({ "root1", "root2"});
+    })) << "creating " << set_id;
+    EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at(set_id).rstate.state);
+  }
+
+  // we shouldn't be able to release-await a quiescing set
+  for (auto&& set_id : { "set1", "set2" }) {
+    ASSERT_EQ(ERR(EPERM), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.release_roots();
+      r.await = sec(1);
+    })) << "bad release-await " << set_id;
+  }
+
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  for (auto&& set_id : { "set1", "set2", "set3" }) {
+    ASSERT_EQ(OK(), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.await = sec(0.1);
+    })) << "quiesce-await " << set_id;
+    EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at(set_id).rstate.state);
+  }
+
+  managers.at(mds_gid_t(1))->reset_agent_callback(SILENT_AGENT_CB);
+
+  auto & release_await1 = start_request([](auto &r) {
+    r.set_id = "set1";
+    r.release_roots();
+    r.await = sec(100);
+  });
+
+  auto& release_await2 = start_request([](auto& r) {
+    r.set_id = "set2";
+    r.release_roots();
+    r.await = sec(100);
+  });
+
+  EXPECT_EQ(OK(), run_request([](auto &r){}));
+  // releasing should be in progress
+  EXPECT_EQ(NA(), release_await1.check_result());
+  EXPECT_EQ(NA(), release_await2.check_result());
+  EXPECT_EQ(QS_RELEASING, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_RELEASING, last_request->response.sets.at("set2").rstate.state);
+  auto releasing_v1 = last_request->response.sets.at("set1").version;
+
+  // we can request release again without any version bump
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+  }));
+
+  EXPECT_EQ(releasing_v1, last_request->response.sets.at("set1").version );
+
+  // we can release-await with a short await timeout
+  EXPECT_EQ(ERR(EINPROGRESS), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+    r.await = sec(0.1);
+  }));
+
+  // we can't quiesce-await a set that's releasing
+  EXPECT_EQ(ERR(EPERM), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(0.1);
+  }));
+
+  // shouldn't be able to add roots to a releasing set
+  EXPECT_EQ(ERR(EPERM), run_request([](auto &r) {
+    r.set_id = "set1";
+    r.include_roots({"root3"});
+  }));
+
+  // still on the same set version
+  EXPECT_EQ(releasing_v1, last_request->response.sets.at("set1").version );
+
+  // it should be allowed to exclude roots from a releasing set
+  EXPECT_EQ(OK(), run_request([](auto &r) {
+    r.set_id = "set2";
+    r.exclude_roots({"root2"});
+  }));
+
+  // the corresponding await must have been interrupted due to the change to the members
+  EXPECT_EQ(ERR(EINTR), release_await2.wait_result_for(0.1));
+
+  // still releasing
+  EXPECT_EQ(QS_RELEASING, last_request->response.sets.at("set2").rstate.state);
+
+  // await again
+  auto& release_await22 = start_request([](auto& r) {
+    r.set_id = "set2";
+    r.release_roots();
+    r.await = sec(100);
+  });
+
+  EXPECT_EQ(NA(), release_await22.check_result());
+
+  // excluding the last root should cancel the set
+  EXPECT_EQ(OK(), run_request([](auto &r) {
+    r.set_id = "set2";
+    r.exclude_roots({"root1"});
+  }));
+
+  EXPECT_EQ(ERR(ECANCELED), release_await22.wait_result_for(0.1));
+
+  std::atomic<QuiesceState> root1_state(QS__INVALID);
+  managers.at(mds_gid_t(1))->reset_agent_callback([&](auto &map){
+    if (map.roots.contains("file:/root1")) {
+      root1_state = map.roots.at("file:/root1").state;
+      root1_state.notify_all();
+    }
+    return false;
+  });
+
+  // validate that root1 is still reported to the agents as QUIESCING
+  // even though we are already releasing set1
+  // this is because there is another set with this root which is not releasing
+  EXPECT_TRUE(timed_run(sec(0.1), [&](){root1_state.wait(QS__INVALID);}));
+  EXPECT_EQ(QS_QUIESCING, root1_state.load());
+
+  // allow acks
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+  EXPECT_EQ(OK(), release_await1.wait_result_for(0.1));
+
+  EXPECT_EQ(QS_RELEASED, release_await1.response.sets.at("set1").rstate.state);
+
+  // it should be OK to request release or release-await on a RELEASED set
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+  }));
+
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+    r.await = sec(0.1);
+  }));
+
+  // it's invalid to send a release without a set id
+  EXPECT_EQ(ERR(EINVAL), run_request([](auto& r) {
+    r.release_roots();
+  }));
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, LeaderShutdown)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1) }));
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.timeout = sec(60);
+    r.expiration = sec(60);
+    r.include_roots({ "root1" });
+  }));
+
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.timeout = sec(60);
+    r.expiration = sec(60);
+    r.include_roots({ "root2", "root3"});
+  }));
+
+  std::queue<TestRequestContext*> outstanding_awaits;
+  std::queue<TestRequestContext*> pending_requests;
+
+  // let's have several awaits pending
+  for(auto&& set_id: {"set1", "set2"}) {
+    for (int i=0; i<2; i++) {
+      outstanding_awaits.emplace(&start_request([set_id](auto&r) {
+        r.set_id = set_id;
+        r.await = sec(100);
+      }));
+      EXPECT_EQ(NA(), outstanding_awaits.front()->check_result());
+    }
+  }
+
+  // flush the pending requests by running a simple query
+  EXPECT_EQ(OK(), run_request([](auto& r) { r.query("set1"); }));
+
+  ASSERT_EQ(outstanding_awaits.size(), managers.at(mds_gid_t(1))->internal_awaits().size());
+
+  std::mutex agent_mutex;
+  std::condition_variable agent_cond;
+  bool callback_reached = false;
+
+  // block the db thread with a malicious agent callback
+  managers.at(mds_gid_t(1))->reset_agent_callback([&](auto& map) {
+    std::unique_lock l(agent_mutex);
+    callback_reached = true;
+    agent_cond.notify_all();
+    l.unlock();
+    std::this_thread::sleep_for(sec(0.1));
+    return false;
+  });
+
+  {
+    std::unique_lock l(agent_mutex);
+    agent_cond.wait(l, [&]{return callback_reached;});
+  }
+
+  // now that the db thread is sleeping we can pile up some pending requests
+  pending_requests.emplace(&start_request([](auto& r) {
+    r.set_id = "set3";
+    r.include_roots({"root4"});
+  }));
+  EXPECT_EQ(NA(), pending_requests.front()->check_result());
+
+  pending_requests.emplace(&start_request([](auto& r) {
+    r.set_id = "set4";
+    r.include_roots({"root5"});
+  }));
+  EXPECT_EQ(NA(), pending_requests.front()->check_result());
+
+  pending_requests.emplace(&start_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  }));
+  EXPECT_EQ(NA(), pending_requests.front()->check_result());
+
+  ASSERT_EQ(managers.at(mds_gid_t(1))->internal_pending_requests().size(), pending_requests.size());
+
+  // reset the membership of the manager
+  // this will block until the db thread exits
+  managers.at(mds_gid_t(1))->update_membership({});
+
+  // as of now all requests must have finished
+  while(!outstanding_awaits.empty()) {
+    auto& r = *outstanding_awaits.front();
+    EXPECT_EQ(ERR(EINPROGRESS), r.check_result());
+    outstanding_awaits.pop();
+  }
+
+  while (!pending_requests.empty()) {
+    auto& r = *pending_requests.front();
+    EXPECT_EQ(ERR(EPERM), r.check_result());
+    pending_requests.pop();
+  }
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, MultiRankQuiesce)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({  mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
+
+  std::vector<TestRequestContext*> awaits;
+
+  // create and await several sets
+  // we deliberately avoid setting the expiration timeout in this test
+  for (auto&& set_id: {"set1", "set2", "set3"}) {
+    awaits.emplace_back(&start_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.timeout = sec(100);
+      r.include_roots({"root1"});
+      r.await = sec(100);
+    }));
+  }
+
+  // flush the pending requests by running a simple query
+  ASSERT_EQ(OK(), run_request([](auto&r){r.query("set1");}));
+
+  ASSERT_EQ(awaits.size(), managers.at(mds_gid_t(1))->internal_awaits().size());
+
+  for (auto&& await: awaits) {
+    EXPECT_EQ(NA(), await->check_result()) << await->request.set_id.value();
+  }
+
+  {
+    std::unordered_set<QuiesceInterface::PeerId> peers_quiesced;
+    auto did_ack = add_ack_hook([&](auto p, auto const &m) {
+      if (m.roots.contains("file:/root1") && (m.roots.at("file:/root1").state == QS_QUIESCED)) {
+        peers_quiesced.insert(p);
+      }
+      return peers_quiesced.size() >= 2;
+    });
+
+    // let two of the three peers ack quiescing of the root
+    managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+    managers.at(mds_gid_t(2))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+    ASSERT_EQ(std::future_status::ready, did_ack.wait_for(std::chrono::milliseconds(100)));
+  }
+
+  // kick the db queue with a simple query
+  ASSERT_EQ(OK(), run_request([](auto& r) { r.query("set1"); }));
+
+  // should still be waiting for the last agent
+  EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at("set1").rstate.state);
+  for (auto&& await: awaits) {
+    EXPECT_EQ(NA(), await->check_result()) << await->request.set_id.value();
+  }
+
+  {
+    // wait for the late peer to ack root1 as released
+    auto did_ack = add_ack_hook([](auto gid, auto const& ack) {
+      return gid == mds_gid_t(3) && ack.roots.contains("file:/root1") && ack.roots.at("file:/root1").state == QS_QUIESCED;
+    });
+
+    // allow acks
+    managers.at(mds_gid_t(3))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+    EXPECT_EQ(std::future_status::ready, did_ack.wait_for(std::chrono::milliseconds(100)));
+  }
+
+  // kick the db queue with a simple query
+  ASSERT_EQ(OK(), run_request([](auto& r) {}));
+
+  // first three sets must be expired because they had 0 expiration
+  EXPECT_EQ(QS_EXPIRED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_EXPIRED, last_request->response.sets.at("set2").rstate.state);
+  EXPECT_EQ(QS_EXPIRED, last_request->response.sets.at("set3").rstate.state);
+
+  // pending quiesce requests must have all completed successfully
+  // even though some of the sets got expired immediately
+  for (auto&& await : awaits) {
+    EXPECT_EQ(OK(), await->check_result()) << await->request.set_id.value();
+  }
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, MultiRankRelease)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+  managers.at(mds_gid_t(2))->reset_agent_callback(QUIESCING_AGENT_CB);
+  managers.at(mds_gid_t(3))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  // quiesce two sets
+  for (auto&& set_id : { "set1", "set2" }) {
+    ASSERT_EQ(OK(), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.timeout = sec(60);
+      r.expiration = sec(60);
+      r.await = sec(100);
+      r.include_roots({ "root1" });
+    }));
+    EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at(set_id).rstate.state);
+  }
+
+  auto quiesced_v = db(mds_gid_t(1)).sets.at("set1").version;
+
+  // prevent one of the acks
+  managers.at(mds_gid_t(2))->reset_agent_callback(SILENT_AGENT_CB);
+
+  // release roots
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+  }));
+
+  EXPECT_EQ(QS_RELEASING, last_request->response.sets.at("set1").rstate.state);
+  auto releasing_v = last_request->response.sets.at("set1").version;
+  ASSERT_NE(quiesced_v, releasing_v);
+
+  auto &async_release = start_request([](auto& r) {
+    r.set_id = "set2";
+    r.await = sec(100);
+    r.release_roots();
+  });
+
+  EXPECT_EQ(NA(), async_release.check_result());
+
+  // shouldn't hurt to run release twice for set 1
+  ASSERT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.release_roots();
+  }));
+
+  EXPECT_EQ(releasing_v, last_request->response.sets.at("set1").version);
+
+  // we shouldn't be able to quiesce-await a releasing set
+  ASSERT_EQ(ERR(EPERM), run_request_for(1, [](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(100);
+  }));
+
+  auto latest_v = db(mds_gid_t(1)).set_version;
+
+  // wait for all peers to sync version 
+  {
+    std::unique_lock l(comms_mutex);
+    auto result = comms_cond.wait_for(l, std::chrono::milliseconds(100), [&] {
+      auto min_v = std::min({ db(mds_gid_t(1)).set_version, db(mds_gid_t(2)).set_version, db(mds_gid_t(3)).set_version });
+      return min_v >= latest_v;
+    });
+    ASSERT_TRUE(result);
+  }
+
+  // all replicas must agree
+  for (auto&& gid : {mds_gid_t(1), mds_gid_t(2), mds_gid_t(3)}) {
+    EXPECT_EQ(QS_RELEASING, db(gid).sets.at("set1").rstate.state) << "db of gid " << gid;
+    EXPECT_EQ(QS_RELEASING, db(gid).sets.at("set2").rstate.state) << "db of gid " << gid;
+  }
+
+  // wait for the late peer to ack back
+  auto did_ack = add_ack_hook([](auto gid, auto const &ack){
+    return gid == mds_gid_t(2);
+  });
+
+  // allow acks
+  managers.at(mds_gid_t(2))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  EXPECT_EQ(std::future_status::ready, did_ack.wait_for(std::chrono::milliseconds(100)));
+
+  ASSERT_EQ(OK(), run_request([](auto& r) { }));
+
+  EXPECT_EQ(QS_RELEASED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_RELEASED, last_request->response.sets.at("set2").rstate.state);
+  EXPECT_EQ(OK(), async_release.check_result());
+
+  // validate that we can release-await RELEASED sets
+  // but can't quiesce-await the same
+  for (auto&& set_id : { "set1", "set2" }) {
+    ASSERT_EQ(OK(), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.await = sec(100);
+      r.release_roots();
+    }));
+    ASSERT_EQ(ERR(EPERM), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.await = sec(100);
+    }));
+  }
+}
+
+/* ================================================================ */
+TEST_F(QuiesceDbTest, MultiRankRecovery)
+{
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
+  managers.at(mds_gid_t(1))->reset_agent_callback(QUIESCING_AGENT_CB);
+  managers.at(mds_gid_t(2))->reset_agent_callback(QUIESCING_AGENT_CB);
+  managers.at(mds_gid_t(3))->reset_agent_callback(QUIESCING_AGENT_CB);
+
+  // quiesce two sets
+  for (auto&& set_id : { "set1", "set2" }) {
+    ASSERT_EQ(OK(), run_request([set_id](auto& r) {
+      r.set_id = set_id;
+      r.timeout = sec(60);
+      r.expiration = sec(60);
+      r.await = sec(100);
+      r.include_roots({ "root1" });
+    }));
+    EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at(set_id).rstate.state);
+  }
+
+
+  auto did_ack41 = add_ack_hook([](auto gid, auto const &ack){
+    return gid == mds_gid_t(4) && ack.db_version.set_version > 0;
+  });
+
+  // reconfigure the cluster so that a new member is assigned leader
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(4), mds_gid_t(2), mds_gid_t(3) }));
+
+  EXPECT_EQ(std::future_status::ready, did_ack41.wait_for(std::chrono::milliseconds(2000)));
+
+  // we expect the db to be populated since the new leader must have discovered newer versions
+  // we expect the sets to become quiescing since there's at least one member that's not acking (the new one)
+  EXPECT_EQ(OK(), run_request([](auto& r) {}));
+  ASSERT_EQ(2, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at("set2").rstate.state);
+
+  // reconfigure the cluster back to quiescing members
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
+
+  // we expect the db to be populated since the new leader must have discovered newer versions
+  // we expect the sets to become quiescing since there's at least one member that's not acking (the new one)
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set1";
+    r.await = sec(1);
+  }));
+  ASSERT_EQ(1, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(OK(), run_request([](auto& r) {
+    r.set_id = "set2";
+    r.await = sec(1);
+  }));
+  ASSERT_EQ(1, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+
+  // lose a non-leader node
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2) }));
+
+  EXPECT_EQ(OK(), run_request([](auto& r) {}));
+  ASSERT_EQ(2, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+
+  auto did_ack3 = add_ack_hook([](auto gid, auto const &ack){
+    return gid == mds_gid_t(3) && ack.db_version.set_version > 0;
+  });
+
+  // add back a quiescing peer
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3)}));
+
+  EXPECT_EQ(OK(), run_request([](auto& r) {}));
+  ASSERT_EQ(2, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+
+  EXPECT_EQ(std::future_status::ready, did_ack3.wait_for(std::chrono::milliseconds(2000)));
+
+  EXPECT_EQ(OK(), run_request([](auto& r) {}));
+  ASSERT_EQ(2, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+
+  auto did_ack42 = add_ack_hook([](auto gid, auto const &ack){
+    return gid == mds_gid_t(4) && ack.db_version.set_version > 0;
+  });
+
+  // add a non-quiescing peer
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3), mds_gid_t(4) }));
+
+  EXPECT_EQ(std::future_status::ready, did_ack42.wait_for(std::chrono::milliseconds(2000)));
+  EXPECT_EQ(OK(), run_request([](auto& r) {}));
+  ASSERT_EQ(2, last_request->response.sets.size());
+  EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at("set1").rstate.state);
+  EXPECT_EQ(QS_QUIESCING, last_request->response.sets.at("set2").rstate.state);
+}
\ No newline at end of file

From 3de0882ad36d4f08f2f171b1ffd263da5a78f00f Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Sun, 26 Nov 2023 13:29:11 +0200
Subject: [PATCH 2113/2492] mds/quiesce: QuiesceAgent implementation and unit
 tests

QuiesceAgent is the layer that converts updates from the QuiesceDb
into calls to the QuiesceProtocol APIs, and then sends async acks
back to the db manager following the quiesce protocol events.

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/QuiesceAgent.cc          | 255 +++++++++++++++
 src/mds/QuiesceAgent.h           | 242 +++++++++++++-
 src/test/mds/CMakeLists.txt      |   8 +
 src/test/mds/TestQuiesceAgent.cc | 545 +++++++++++++++++++++++++++++++
 4 files changed, 1044 insertions(+), 6 deletions(-)
 create mode 100644 src/mds/QuiesceAgent.cc
 create mode 100644 src/test/mds/TestQuiesceAgent.cc

diff --git a/src/mds/QuiesceAgent.cc b/src/mds/QuiesceAgent.cc
new file mode 100644
index 000000000000..c1550549ba67
--- /dev/null
+++ b/src/mds/QuiesceAgent.cc
@@ -0,0 +1,255 @@
+#include "mds/QuiesceAgent.h"
+#include "common/debug.h"
+#include "include/ceph_assert.h"
+#include <future>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mds_quiesce
+#undef dout_prefix
+#define dout_prefix *_dout << "quiesce.agt <" << __func__ << "> "
+
+#undef dout
+#define dout(lvl)                                                        \
+  do {                                                                   \
+    auto subsys = ceph_subsys_mds;                                       \
+    if ((dout_context)->_conf->subsys.should_gather(dout_subsys, lvl)) { \
+      subsys = dout_subsys;                                              \
+    }                                                                    \
+  dout_impl(dout_context, ceph::dout::need_dynamic(subsys), lvl) dout_prefix
+
+#undef dendl
+#define dendl \
+  dendl_impl; \
+  }           \
+  while (0)
+
+template <class CharT, class Traits>
+std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceAgent::TrackedRootsVersion& tr)
+{
+  return os << "tracked_roots[armed: " << tr.armed << ", v: " << tr.db_version << " r: " << tr.roots.size() << "]";
+}
+
+bool QuiesceAgent::db_update(QuiesceMap& map)
+{
+  // copy of the current roots
+  TrackedRoots current_roots = tracked_roots();
+  TrackedRoots new_roots;
+
+  dout(20) << "got a db update version " << map.db_version << " with " << map.roots.size() << " roots" << dendl;
+
+  for (auto their_it = map.roots.begin(); their_it != map.roots.end();) {
+    auto &[root, info] = *their_it;
+    TrackedRootRef tracked_root_ref;
+
+    ceph_assert(info.state > QS__INVALID);
+
+    if (info.state >= QS__FAILURE) {
+      // we don't care about roots in failed states
+      dout(5) << "ignoring a root in a failed state: '" << root << "', " << info.state << dendl;
+      their_it = map.roots.erase(their_it);
+      continue;
+    }
+
+    if (const auto& my_it = current_roots.find(root); my_it != current_roots.end()) {
+      // keep the existing root
+      new_roots.insert(*my_it);
+      tracked_root_ref = my_it->second;
+    } else {
+      // introduce a new root
+      tracked_root_ref = std::make_shared<TrackedRoot>(info.state, info.ttl);
+      new_roots[root] = tracked_root_ref;
+    }
+
+    std::lock_guard l(*tracked_root_ref);
+    tracked_root_ref->update_committed(info);
+
+    auto actual_state = tracked_root_ref->get_actual_state();
+    
+    if (actual_state != info.state) {
+      // we have an update for the state
+      info.state = actual_state;
+      info.ttl = tracked_root_ref->get_ttl();
+    } else {
+      // our tracked root has the same state as the db
+      // we can just drop it from the response
+      their_it = map.roots.erase(their_it);
+      continue;
+    }
+    ++their_it;
+  }
+
+  // ack with the known state stored in `map`
+  set_pending_roots(map.db_version, std::move(new_roots));
+
+  // always send a synchronous ack
+  return true;
+}
+
+void* QuiesceAgent::agent_thread_main() {
+  working.clear();
+  std::unique_lock lock(agent_mutex);
+
+  while(!stop_agent_thread) {
+    if (pending.armed) {
+      working.roots.swap(pending.roots);
+      working.db_version = pending.db_version;
+    } else {
+      // copy current roots
+      working.roots = current.roots;
+      working.db_version = current.db_version;
+    }
+
+    dout(20)
+        << "current = " << current.db_version
+        << ", working = " << working.db_version
+        << ", pending = " << pending.db_version << dendl;
+
+    current.armed = false;
+    working.armed = true;
+
+    // it's safe to clear the pending roots under lock because it shouldn't
+    // ever hold a last shared ptr to quiesced tracked roots, causing their destructors to run cancel.
+    pending.clear();
+    lock.unlock();
+
+    QuiesceMap ack(working.db_version);
+  
+    // upkeep what we believe is the current state.
+    for (auto& [root, info] : working.roots) {
+
+      info->lock();
+      bool should_quiesce = info->should_quiesce();
+      bool issue_quiesce = should_quiesce && !info->quiesce_request && !info->quiesce_result;
+      std::optional<QuiesceInterface::RequestHandle> cancel_handle;
+      if (!should_quiesce && !info->cancel_result) {
+        cancel_handle = info->quiesce_request;
+      }
+      auto actual_state = info->get_actual_state();
+      if (info->committed_state != actual_state) {
+        ack.roots[root] = { actual_state, info->get_ttl() };
+      }
+      info->unlock();
+
+      if (issue_quiesce) {
+        std::weak_ptr<TrackedRoot> weak_info = info;
+        auto request_handle = quiesce_control.submit_request(root, new LambdaContext([weak_info, submitted_root = root, this](int rc) {
+          if (auto info = weak_info.lock()) {
+            dout(20) << "completing request (rc=" << rc << ") for '" << submitted_root << "'" << dendl;
+            info->lock();
+            info->quiesce_result = rc;
+            info->unlock();
+
+            // TODO: capturing QuiesceAgent& `this` is potentially dangerous
+            //       the assumption is that since the root pointer is weak
+            //       it will have been deleted by the QuiesceAgent shutdown sequence
+            set_upkeep_needed();
+          }
+          dout(20) << "done with submit callback for '" << submitted_root << "'" << dendl;
+        }));
+
+        dout(10) << "got request handle <" << request_handle << "> for '" << root << "'" << dendl;
+        info->lock();
+        info->quiesce_request = request_handle;
+        info->cancel = quiesce_control.cancel_request;
+        info->unlock();
+      } else if (cancel_handle) {
+        dout(10) << "Calling `cancel` on `" << root << "` with handle <" << *cancel_handle << ">" << dendl;
+        int rc = quiesce_control.cancel_request(*cancel_handle);
+        if (rc != 0) {
+          dout(1) << "ERROR (" << rc << ") when trying to cancel quiesce request id: " << *cancel_handle << dendl;
+        }
+        info->lock();
+        info->cancel_result = rc;
+        info->unlock();
+      }
+    }
+
+    lock.lock();
+
+    bool new_version = current.db_version < working.db_version;
+    current.roots.swap(working.roots);
+    current.db_version = working.db_version;
+
+    lock.unlock();
+
+    // clear the old roots and send the ack outside of the lock
+    working.roots.clear();
+    if (new_version || !ack.roots.empty()) {
+      dout(20) << "asyncrhonous ack for " << (new_version ? "a new" : "the current") << " version: " << ack << dendl;
+      int rc = quiesce_control.agent_ack(std::move(ack));
+      if (rc != 0) {
+        dout(3) << "got error: " << rc << " trying to send " << ack << dendl;
+      }
+    }
+    ack.reset();
+
+    lock.lock();
+
+    // notify that we're done working on this version and all acks (if any) were sent
+    working.clear();
+
+    // a new pending version could be set while we weren't locked
+    // if that's the case just go for another pass
+    // otherwise, wait for updates
+    if (!pending.armed && !current.armed && !stop_agent_thread) {
+      // for somebody waiting for the thread to idle
+      agent_cond.notify_all();
+      agent_cond.wait(lock);
+    }
+  }
+  agent_cond.notify_all();
+  return nullptr;
+}
+
+void QuiesceAgent::set_pending_roots(QuiesceDbVersion version, TrackedRoots&& new_roots)
+{
+  std::unique_lock l(agent_mutex);
+
+  auto actual_version = std::max(current.db_version, working.db_version);
+  bool rollback = actual_version > version;
+  
+  if (rollback) {
+    dout(5) << "version rollback to " << version 
+      << ". current = " << current.db_version 
+      << ", working = " << working.db_version 
+      << ", pending = " << pending.db_version << dendl;
+  }
+
+  // set the pending version unconditionally
+  pending.db_version = version;
+  pending.roots = std::move(new_roots);
+  pending.armed = true;
+
+  agent_cond.notify_all();
+}
+
+void QuiesceAgent::set_upkeep_needed()
+{
+  std::unique_lock l(agent_mutex);
+
+  dout(20)
+      << "current = " << current.db_version
+      << ", working = " << working.db_version
+      << ", pending = " << pending.db_version << dendl;
+
+  current.armed = true;
+  agent_cond.notify_all();
+}
+
+QuiesceAgent::TrackedRoot::~TrackedRoot()
+{
+  std::optional<QuiesceInterface::RequestHandle> request_handle;
+
+  lock();
+  request_handle.swap(quiesce_request);
+  bool should_cancel = !cancel_result.has_value();
+  unlock();
+
+  if (should_cancel && request_handle && cancel) {
+    dout(10) << "Calling `cancel` on an abandoned root with handle <" << request_handle << ">" << dendl;
+    cancel(*request_handle);
+  }
+
+  dout(20) << "done with request handle <" << request_handle << ">" << dendl;
+}
diff --git a/src/mds/QuiesceAgent.h b/src/mds/QuiesceAgent.h
index 6a059dec3bc5..f5be435f2a20 100644
--- a/src/mds/QuiesceAgent.h
+++ b/src/mds/QuiesceAgent.h
@@ -5,15 +5,245 @@
  *
  * This is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
+ * License db_version 2.1, as published by the Free Software
  * Foundation.  See file COPYING.
  *
  */
 #pragma once
-#include "QuiesceDb.h"
+#include "mds/QuiesceDb.h"
+#include <functional>
+#include <optional>
+#include <map>
+#include <mutex>
+#include <thread>
 
 class QuiesceAgent {
-  // subscribe to the QM map
-  // keeps runtime version of the map
-  // notifies the QM when runtime version is different from the last know requested version
-};
\ No newline at end of file
+  public:
+    struct ControlInterface {
+      QuiesceInterface::RequestSubmit submit_request;
+      QuiesceInterface::RequestCancel cancel_request;
+      QuiesceInterface::AgentAck agent_ack;
+      // TODO: do we need a "cancel all"?
+    };
+
+    QuiesceAgent(const ControlInterface& quiesce_control)
+        : quiesce_control(quiesce_control)
+        , stop_agent_thread(false)
+        , agent_thread(this) {
+      agent_thread.create("quiesce.agt");
+    };
+
+    ~QuiesceAgent() {
+      agent_thread.kill(SIGTERM);
+    }
+
+    /// @brief  WARNING: will reset syncrhonously
+    ///         this may call cancel on active roots
+    ///         which may lead to a deadlock if the MDS
+    ///         lock is being held when calling this.
+    ///         Consider `reset_async` if you're holding
+    ///         the MDS lock.
+    void reset() {
+      std::unique_lock l(agent_mutex);
+
+      // prevent any pending change
+      pending.clear();
+
+      // let the system settle
+      await_idle_locked(l);
+
+      // we are idle, hence the current holds
+      // our only tracked set
+      TrackedRoots current_roots = current.clear();
+
+      l.unlock();
+
+      // do this outside of the lock
+      current_roots.clear();
+    }
+
+    void reset_async() {
+      set_pending_roots({0, 0}, {});
+    }
+
+    void shutdown()
+    {
+      std::unique_lock l(agent_mutex);
+      stop_agent_thread = true;
+      agent_cond.notify_all();
+      l.unlock();
+      agent_thread.join();
+
+      current.clear();
+      pending.clear();
+    }
+
+    bool db_update(QuiesceMap& map);
+
+    struct TrackedRoot {
+        std::optional<QuiesceInterface::RequestHandle> quiesce_request;
+        // we could have hidden the request handle
+        // inside the cancel functor, but then we'd lose
+        // the ability to identify individual requests
+        // when looking at the tracked root.
+        QuiesceInterface::RequestCancel cancel; 
+        std::optional<int> quiesce_result;
+        std::optional<int> cancel_result;
+
+        QuiesceState committed_state;
+        QuiesceTimePoint expires_at;
+
+        TrackedRoot(QuiesceState state, QuiesceTimeInterval ttl)
+            : committed_state(state)
+            , expires_at(interval_saturate_add_now(ttl))
+            , busy_lock(false)
+        {
+        }
+
+        TrackedRoot() : TrackedRoot(QS__INVALID, QuiesceTimeInterval::zero()) {}
+
+        bool should_quiesce() const
+        {
+          return committed_state == QS_QUIESCING || committed_state == QS_QUIESCED;
+        }
+
+        bool should_release() const {
+          return committed_state == QS_RELEASING || committed_state == QS_RELEASED;
+        }
+
+        ~TrackedRoot();
+
+        void update_committed(QuiesceMap::RootInfo const & info) {
+          committed_state = info.state;
+          expires_at = interval_saturate_add_now(info.ttl);
+        }
+
+        QuiesceTimeInterval get_ttl() const
+        {
+          auto now = QuiesceClock::now();
+          if (expires_at.time_since_epoch() == QuiesceTimeInterval::max()) {
+            return QuiesceTimeInterval::max();
+          }
+          if (expires_at > now) {
+            return expires_at - now;
+          } else {
+            return QuiesceTimeInterval::zero();
+          }
+        }
+
+        QuiesceState get_actual_state() const {
+          QuiesceState result = QS_QUIESCING;
+          bool did_quiesce = quiesce_result == 0;
+          bool did_cancel = cancel_result == 0;
+          if (did_quiesce) {
+            if (cancel_result.has_value()) {
+              result = did_cancel ? QS_RELEASED : QS_EXPIRED;
+            } else {
+              result = QS_QUIESCED;
+            }
+          } else {
+            if (quiesce_result.has_value()) {
+              result = QS_FAILED;
+            } else if (should_release()) {
+              // we must have lost track of this root,
+              // probably, due to expiration. But even if due to an error,
+              // this is our best guess for the situation
+              result = QS_EXPIRED;
+            }
+          }
+          return result;
+        }
+
+        void lock() const {
+          while (busy_lock.test_and_set(std::memory_order_acquire))
+            ; // spin
+        }
+
+        void unlock() const {
+          busy_lock.clear(std::memory_order_release);
+        }
+      private:
+        mutable std::atomic_flag busy_lock;
+    };
+
+    using TrackedRootRef = std::shared_ptr<TrackedRoot>;
+
+    using TrackedRoots = std::unordered_map<QuiesceRoot, TrackedRootRef>;
+
+    TrackedRoots tracked_roots() {
+      std::lock_guard l(agent_mutex);
+      return current.roots;
+    }
+
+    TrackedRootRef get_tracked_root(QuiesceRoot root) {
+      std::lock_guard l(agent_mutex);
+      if (auto it = current.roots.find(root); it != current.roots.end()) {
+        return it->second;
+      } else {
+        return nullptr;
+      }
+    }
+
+    QuiesceDbVersion get_current_version() {
+      std::lock_guard l(agent_mutex);
+      return current.db_version;
+    }
+
+  protected:
+    ControlInterface quiesce_control;
+
+    struct TrackedRootsVersion {
+      TrackedRoots roots;
+      QuiesceDbVersion db_version = {0, 0};
+      bool armed = false;
+      TrackedRoots clear() {
+        armed = false;
+        db_version = {0, 0};
+        TrackedRoots old = std::move(roots);
+        roots.clear();
+        return old;
+      }
+    };
+
+    template <class CharT, class Traits>
+    friend std::basic_ostream<CharT, Traits>&
+    operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceAgent::TrackedRootsVersion& tr);
+
+    TrackedRootsVersion current;
+    TrackedRootsVersion working;
+    TrackedRootsVersion pending;
+
+    std::mutex agent_mutex;
+    std::condition_variable agent_cond;
+    bool stop_agent_thread;
+  
+    template<class L>
+    QuiesceDbVersion await_idle_locked(L &lock) {
+      agent_cond.wait(lock, [this] {
+        return !(current.armed || working.armed || pending.armed);
+      });
+
+      return current.db_version;
+    }
+
+    void set_pending_roots(QuiesceDbVersion db_version, TrackedRoots&& new_roots);
+
+    void set_upkeep_needed();
+
+    class AgentThread : public Thread {
+      public:
+          explicit AgentThread(QuiesceAgent* qa)
+              : qa(qa)
+          {
+          }
+          void* entry() override
+          {
+            return qa->agent_thread_main();
+          }
+
+      private:
+          QuiesceAgent* qa;
+    } agent_thread;
+
+    void* agent_thread_main();
+};
diff --git a/src/test/mds/CMakeLists.txt b/src/test/mds/CMakeLists.txt
index 39b47bb2da4d..f80abe75083f 100644
--- a/src/test/mds/CMakeLists.txt
+++ b/src/test/mds/CMakeLists.txt
@@ -24,3 +24,11 @@ add_executable(unittest_mds_quiesce_db
 add_ceph_unittest(unittest_mds_quiesce_db)
 target_link_libraries(unittest_mds_quiesce_db ceph-common global)
 
+# unittest_mds_quiesce_agent
+add_executable(unittest_mds_quiesce_agent
+  TestQuiesceAgent.cc
+  ../../../src/mds/QuiesceAgent.cc
+  $<TARGET_OBJECTS:unit-main>
+)
+add_ceph_unittest(unittest_mds_quiesce_agent)
+target_link_libraries(unittest_mds_quiesce_agent ceph-common global)
diff --git a/src/test/mds/TestQuiesceAgent.cc b/src/test/mds/TestQuiesceAgent.cc
new file mode 100644
index 000000000000..d7d526ae9a6c
--- /dev/null
+++ b/src/test/mds/TestQuiesceAgent.cc
@@ -0,0 +1,545 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, RedHat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "common/Cond.h"
+#include "mds/QuiesceAgent.h"
+#include "gtest/gtest.h"
+#include <algorithm>
+#include <functional>
+#include <queue>
+#include <ranges>
+#include <system_error>
+#include <thread>
+#include <future>
+
+class QuiesceAgentTest : public testing::Test {
+  using RequestHandle = QuiesceInterface::RequestHandle;
+  using QuiescingRoot = std::pair<RequestHandle, Context*>;
+  protected:
+    template< class _Rep = std::chrono::seconds::rep, class _Period = std::chrono::seconds::period, typename D = std::chrono::duration<_Rep, _Period>, class Function, class... Args >
+    static bool timed_run(D timeout, Function&& f, Args&&... args ) {
+      std::promise<void> done;
+      auto future = done.get_future();
+
+      auto job = std::bind(f, args...);
+
+      auto tt = std::thread([job=std::move(job)](std::promise<void> done) {
+        job();
+        done.set_value();
+      }, std::move(done));
+
+      tt.detach();
+
+      return future.wait_for(timeout) != std::future_status::timeout;
+    }
+
+    struct TestQuiesceAgent : public QuiesceAgent {
+      using QuiesceAgent::QuiesceAgent;
+      AgentThread& get_agent_thread() {
+        return agent_thread;
+      }
+
+      QuiesceDbVersion get_latest_version()
+      {
+        std::lock_guard l(agent_mutex);
+        return std::max({current.db_version, working.db_version, pending.db_version});
+      }
+      TrackedRoots& mutable_tracked_roots() {
+        return current.roots;
+      }
+
+      QuiesceDbVersion await_idle() {
+        std::unique_lock l(agent_mutex);
+        return await_idle_locked(l);
+      }
+    };
+    QuiesceMap latest_ack;
+    std::unordered_map<QuiesceRoot, QuiescingRoot> quiesce_requests;
+    ceph_tid_t last_tid;
+    std::mutex mutex;
+
+    std::unique_ptr<TestQuiesceAgent> agent;
+
+    bool complete_quiesce(QuiesceRoot root, int rc = 0) {
+      std::lock_guard l(mutex);
+      if (auto it = quiesce_requests.find(root); it != quiesce_requests.end()) {
+        if (it->second.second) {
+          it->second.second->complete(rc);
+          it->second.second = nullptr;
+          if (rc != 0) {
+            // there was an error, no need to keep this request anymore
+            quiesce_requests.erase(it);
+          }
+          return true;
+        }
+      }
+      return false;
+    }
+
+    void SetUp() override {
+      
+      QuiesceAgent::ControlInterface ci;
+      quiesce_requests.clear();
+
+      ci.submit_request = [this](QuiesceRoot r, Context* c) {
+        std::lock_guard l(mutex);
+
+        // always create a new request id
+        auto req_id = metareqid_t(entity_name_t::MDS(0), ++last_tid);
+
+        auto [it, inserted] = quiesce_requests.try_emplace(r, req_id, c);
+
+        if (!inserted) {
+          // we must update the request id so that old one can't cancel this request.
+          it->second.first = req_id;
+          if (it->second.second) {
+            it->second.second->complete(-EINTR);
+            it->second.second = c;
+          } else {
+            // if we have no context, it means we've completed it
+            // since we weren't inserted, we must have successfully quiesced
+            c->complete(0);
+          }
+        }
+
+        return it->second.first;
+      };
+      
+      ci.cancel_request = [this](RequestHandle h) {
+        std::lock_guard l(mutex);
+        
+        for (auto it = quiesce_requests.cbegin(); it != quiesce_requests.cend(); it++) {
+          if (it->second.first == h) {
+            if (it->second.second) {
+              it->second.second->complete(-ECANCELED);
+            }
+            quiesce_requests.erase(it);
+            return 0;
+          }
+        }
+
+        return ENOENT;
+      };
+
+      ci.agent_ack = [this](QuiesceMap const& update) {
+        std::lock_guard l(mutex);
+        latest_ack = update;
+        return 0;
+      };
+
+      agent = std::make_unique<TestQuiesceAgent>(ci);
+    }
+
+    void TearDown() override {
+      if (agent) {
+        agent->shutdown();
+        agent.reset();
+      }
+    }
+
+
+    using R = QuiesceMap::Roots::value_type;
+    using RootInitList = std::initializer_list<R>;
+
+    std::optional<QuiesceMap> update(QuiesceDbVersion v, RootInitList roots)
+    {
+      QuiesceMap map(v, QuiesceMap::Roots { roots });
+
+      if (agent->db_update(map)) {
+        return map;
+      }
+
+      return std::nullopt;
+    }
+
+    std::optional<QuiesceMap> update(QuiesceSetVersion v, RootInitList roots)
+    {
+      return update(QuiesceDbVersion { 1, v }, roots);
+    }
+
+    std::optional<QuiesceMap> update(RootInitList roots)
+    {
+      return update(agent->get_latest_version() + 1, roots);
+    }
+
+    template <class _Rep = std::chrono::seconds::rep, class _Period = std::chrono::seconds::period, typename D = std::chrono::duration<_Rep, _Period>>
+    bool await_idle_v(QuiesceDbVersion version, D timeout = std::chrono::duration_cast<D>(std::chrono::seconds(10)))
+    {
+      return timed_run(timeout, [this, version] {
+        while (version > agent->await_idle()) { };
+      });
+    }
+
+    template <class _Rep = std::chrono::seconds::rep, class _Period = std::chrono::seconds::period, typename D = std::chrono::duration<_Rep, _Period>>
+    bool await_idle(D timeout = std::chrono::duration_cast<D>(std::chrono::seconds(10)))
+    {
+      return timed_run(timeout, [this] {
+        agent->await_idle();
+      });
+    }
+};
+
+TEST_F(QuiesceAgentTest, ThreadManagement) {
+  EXPECT_TRUE(agent->get_agent_thread().is_started());
+
+  EXPECT_TRUE(await_idle());
+
+  EXPECT_TRUE(update({ { "root1", QS_QUIESCING } }).has_value());
+
+  EXPECT_TRUE(await_idle());
+
+  EXPECT_TRUE(update({ { "root2", QS_QUIESCING } }).has_value());
+
+  agent->reset();
+
+  EXPECT_TRUE(await_idle());
+
+  EXPECT_TRUE(update({ { "root3", QS_QUIESCING } }).has_value());
+
+  // make sure that the agent thread completes in a timely fashion
+  EXPECT_TRUE(timed_run(std::chrono::seconds(1), [this] { agent->shutdown(); agent.reset(); }));
+}
+
+TEST_F(QuiesceAgentTest, DbUpdates) {
+  {
+    auto ack = update(1, { 
+      { "root0", QS_QUIESCING }, // this shouldn't be reported because its state isn't different from QUIESCING
+      { "root1", QS_QUIESCING }, // ditto
+      { "root2", QS_QUIESCED }, // this should be reported back as quiescing
+      { "root3", QS_RELEASING }, // this should be reported back as expired
+      { "root4", QS_RELEASED }, // this should be reported back as expired
+      { "root5", QS_EXPIRED }, // this should be ignored
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(1, ack->db_version);
+    EXPECT_EQ(3, ack->roots.size());
+    EXPECT_EQ(QS_QUIESCING, ack->roots.at("root2").state);
+    EXPECT_EQ(QS_EXPIRED, ack->roots.at("root3").state);
+    EXPECT_EQ(QS_EXPIRED, ack->roots.at("root4").state);
+    EXPECT_TRUE(await_idle());
+  }
+
+  EXPECT_EQ(1, agent->get_current_version());
+
+  {
+    auto roots = agent->tracked_roots();
+    EXPECT_EQ(5, roots.size());
+    EXPECT_EQ(QS_QUIESCING, roots.at("root0")->committed_state);
+    EXPECT_EQ(QS_QUIESCING, roots.at("root1")->committed_state);
+    EXPECT_EQ(QS_QUIESCED, roots.at("root2")->committed_state);
+    EXPECT_EQ(QS_RELEASING, roots.at("root3")->committed_state);
+    EXPECT_EQ(QS_RELEASED, roots.at("root4")->committed_state);
+  
+    // manipulate root0 and root1 as if they were quiesced and root2 as if it was released
+    auto& root0 = *roots.at("root0");
+    root0.quiesce_result = 0;
+    EXPECT_EQ(QS_QUIESCED, root0.get_actual_state());
+
+    auto& root1 = *roots.at("root1");
+    root1.quiesce_result = 0;
+    EXPECT_EQ(QS_QUIESCED, root1.get_actual_state());
+
+    auto& root2 = *roots.at("root2");
+    root2.quiesce_result = 0;
+    root2.cancel_result = 0;
+    EXPECT_EQ(QS_RELEASED, root2.get_actual_state());
+  }
+
+  {
+    auto ack = update(2, { 
+      { "root0", QS_RELEASING }, // this should be reported back as quiesced
+      { "root1", QS_QUIESCING }, // this should be reported back as quiesced
+      { "root2", QS_RELEASING }, // this should be reported back as released
+    });
+
+    EXPECT_EQ(2, ack->db_version);
+    EXPECT_EQ(3, ack->roots.size());
+    EXPECT_EQ(QS_QUIESCED, ack->roots.at("root0").state);
+    EXPECT_EQ(QS_QUIESCED, ack->roots.at("root1").state);
+    EXPECT_EQ(QS_RELEASED, ack->roots.at("root2").state);
+  }
+
+  EXPECT_TRUE(await_idle());
+  {
+    auto roots = agent->tracked_roots();
+    EXPECT_EQ(3, roots.size());
+    EXPECT_EQ(QS_RELEASING, roots.at("root0")->committed_state);
+    EXPECT_EQ(QS_QUIESCING, roots.at("root1")->committed_state);
+    EXPECT_EQ(QS_RELEASING, roots.at("root2")->committed_state);
+  }
+
+  {
+    // we should be able to set pending version to anything
+    // and the agent should follow, including rolling back to 0
+    auto ack = update({200, 0}, {});
+
+    EXPECT_TRUE(await_idle());
+    EXPECT_EQ(0, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+    EXPECT_EQ((QuiesceDbVersion {200, 0}), agent->get_current_version());
+  }
+}
+
+TEST_F(QuiesceAgentTest, QuiesceProtocol) {
+
+  {
+    auto ack = update(1, { 
+      { "root1", QS_QUIESCING },
+      { "root2", QS_QUIESCING },
+      { "root3", QS_QUIESCING },
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(1, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  {
+    auto tracked = agent->tracked_roots();
+    EXPECT_EQ(3, tracked.size());
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root1")->committed_state);
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root2")->committed_state);
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root3")->committed_state);
+
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root1")->get_actual_state());
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root2")->get_actual_state());
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root3")->get_actual_state());
+
+    // we should have seen the quiesce requests for all roots
+    EXPECT_EQ(tracked.at("root1")->quiesce_request.value(), quiesce_requests.at("root1").first);
+    EXPECT_EQ(tracked.at("root2")->quiesce_request.value(), quiesce_requests.at("root2").first);
+    EXPECT_EQ(tracked.at("root3")->quiesce_request.value(), quiesce_requests.at("root3").first);
+  }
+
+  EXPECT_EQ(3, quiesce_requests.size());
+
+  // complete one root with success
+  EXPECT_TRUE(complete_quiesce("root1"));
+
+  EXPECT_TRUE(await_idle());
+  // we should have seen an ack sent
+  EXPECT_EQ(1, latest_ack.db_version);
+  EXPECT_EQ(1, latest_ack.roots.size());
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
+
+  latest_ack.reset();
+
+  // complete the other root with failure
+  EXPECT_TRUE(complete_quiesce("root2", -1));
+
+  EXPECT_TRUE(await_idle());
+  EXPECT_EQ(1, latest_ack.db_version);
+  ASSERT_EQ(2, latest_ack.roots.size());
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
+  EXPECT_EQ(QS_FAILED, latest_ack.roots.at("root2").state);
+
+  latest_ack.reset();
+
+  // complete the third root with success
+  // complete one root with success
+  EXPECT_TRUE(complete_quiesce("root3"));
+
+  EXPECT_TRUE(await_idle());
+
+  // we should see the two quiesced roots in the ack,
+  EXPECT_EQ(1, latest_ack.db_version);
+  ASSERT_EQ(3, latest_ack.roots.size());
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
+  EXPECT_EQ(QS_FAILED, latest_ack.roots.at("root2").state);
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root3").state);
+
+  {
+    auto ack = update(2, {
+      { "root2", QS_QUIESCING },
+      { "root3", QS_RELEASING },
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(2, ack->db_version);
+    // this update doesn't have root1, so it should be untracked and cancelled
+    // root2 is still quiescing, no updates for it
+    // root3 is released asyncrhonously so for now it should be QUIESCED
+    ASSERT_EQ(2, ack->roots.size());
+    EXPECT_EQ(QS_FAILED, latest_ack.roots.at("root2").state);
+    EXPECT_EQ(QS_QUIESCED, ack->roots.at("root3").state);
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  {
+    // make sure that root1 isn't tracked
+    auto tracked = agent->tracked_roots();
+    EXPECT_EQ(2, agent->get_current_version());
+    ASSERT_EQ(2, tracked.size());
+    EXPECT_EQ(QS_QUIESCING, tracked.at("root2")->committed_state);
+    EXPECT_EQ(QS_RELEASING, tracked.at("root3")->committed_state);
+  }
+
+  // we should have also seen cancelations for root1 and root3.
+  // We observe this by missing them from the quiesce_requests
+  // NB: root2 shouldn't be part of requests either since it was completed with failure
+  EXPECT_EQ(0, quiesce_requests.size());
+}
+
+TEST_F(QuiesceAgentTest, DuplicateQuiesceRequest) {
+  {
+    auto ack = update(1, { 
+      { "root1", QS_QUIESCING },
+      { "root2", QS_QUIESCING },
+      { "root3", QS_QUIESCING },
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(1, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  QuiesceAgent::TrackedRootRef pinned1, pinned2;
+
+  {
+    auto tracked = agent->tracked_roots();
+    ASSERT_EQ(3, tracked.size());
+    EXPECT_EQ(tracked.at("root1")->quiesce_request.value(), quiesce_requests.at("root1").first);
+    EXPECT_EQ(tracked.at("root2")->quiesce_request.value(), quiesce_requests.at("root2").first);
+    EXPECT_EQ(tracked.at("root3")->quiesce_request.value(), quiesce_requests.at("root3").first);
+
+    // copying the shared ref will keep the object alive
+    pinned1 = tracked.at("root1");
+    pinned2 = tracked.at("root2");
+  }
+
+  // root 1 should be quiesced now
+  EXPECT_TRUE(complete_quiesce("root1"));
+
+  EXPECT_EQ(QS_QUIESCED, pinned1->get_actual_state());
+  EXPECT_EQ(QS_QUIESCING, pinned2->get_actual_state());
+
+  // imagine that we lost our root records for a moment
+  {
+    auto ack = update(2, {
+      { "root3", QS_QUIESCING },
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(2, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  {
+    auto tracked = agent->tracked_roots();
+    EXPECT_EQ(1, tracked.size());
+    EXPECT_EQ(tracked.at("root3")->quiesce_request.value(), quiesce_requests.at("root3").first);
+  }
+
+  // since we have those pinned, they should still be live
+
+  EXPECT_TRUE(pinned1.unique());
+  EXPECT_TRUE(pinned2.unique());
+
+  EXPECT_EQ(QS_QUIESCED, pinned1->get_actual_state());
+  EXPECT_EQ(QS_QUIESCING, pinned2->get_actual_state());
+
+  EXPECT_TRUE(quiesce_requests.contains("root1"));
+  EXPECT_TRUE(quiesce_requests.contains("root2"));
+
+  latest_ack.reset();
+  // now, bring the roots back
+  {
+    auto ack = update(3, { 
+      { "root1", QS_QUIESCING },
+      { "root2", QS_QUIESCING },
+      { "root3", QS_QUIESCING },
+    });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(3, ack->db_version);
+    // even though root1 is already quiesced,
+    // we should not know about it synchronously
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  // now we should have seen the ack with root2 quiesced
+  EXPECT_EQ(3, latest_ack.db_version);
+  EXPECT_EQ(1, latest_ack.roots.size());
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
+
+  // the actual state of the pinned objects shouldn't have changed
+  EXPECT_EQ(QS_QUIESCED, pinned1->get_actual_state());
+  EXPECT_EQ(QS_FAILED, pinned2->get_actual_state());
+
+  EXPECT_EQ(0, *pinned1->quiesce_result);
+  EXPECT_EQ(-EINTR, *pinned2->quiesce_result);
+
+  // releasing the pinned objects will attempt to cancel, but that shouldn't interfere with the current state
+  pinned1.reset();
+  pinned2.reset();
+
+  EXPECT_TRUE(quiesce_requests.contains("root1"));
+  EXPECT_TRUE(quiesce_requests.contains("root2"));
+
+  EXPECT_TRUE(complete_quiesce("root2"));
+
+  EXPECT_TRUE(await_idle());
+  EXPECT_EQ(3, latest_ack.db_version);
+  EXPECT_EQ(2, latest_ack.roots.size());
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
+  EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root2").state);
+}
+
+TEST_F(QuiesceAgentTest, TimeoutBeforeComplete)
+{
+  {
+    auto ack = update(1, {
+                             { "root1", QS_QUIESCING },
+                         });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(1, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  // QuiesceAgent::TrackedRootRef pinned1, pinned2;
+
+  {
+    auto tracked = agent->tracked_roots();
+    EXPECT_EQ(1, tracked.size());
+    EXPECT_EQ(tracked.at("root1")->quiesce_request.value(), quiesce_requests.at("root1").first);
+  }
+
+  // with a new update we got our root 1 timedout (this is the same as not listing it at all)
+  {
+    auto ack = update(2, {
+                             { "root1", QS_TIMEDOUT },
+                         });
+
+    ASSERT_TRUE(ack.has_value());
+    EXPECT_EQ(2, ack->db_version);
+    EXPECT_EQ(0, ack->roots.size());
+  }
+
+  EXPECT_TRUE(await_idle());
+
+  {
+    auto tracked = agent->tracked_roots();
+    EXPECT_EQ(0, tracked.size());
+  }
+}

From 9907efd0132197feed4b7cc4e7c17a7ec9104a47 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Sun, 21 Jan 2024 19:37:52 +0200
Subject: [PATCH 2114/2492] pybind/mgr: add a `one-shot` parameter to
 send_command

with the parameter set, the message won't be held on to when the remote end resets
or fails to reconnect.

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/client/Client.cc             | 69 +++++++++++++++++++++++++-------
 src/client/Client.h              | 24 ++++++++++-
 src/mgr/BaseMgrModule.cc         | 25 +++++++++---
 src/pybind/mgr/ceph_module.pyi   |  4 +-
 src/pybind/mgr/mgr_module.py     | 12 ++++--
 src/pybind/mgr/tests/__init__.py |  2 +-
 6 files changed, 109 insertions(+), 27 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 473b6cb229fe..bbbc3b60e05c 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -3028,15 +3028,10 @@ void Client::handle_fs_map_user(const MConstRef<MFSMapUser>& m)
 // Cancel all the commands for missing or laggy GIDs
 void Client::cancel_commands(const MDSMap& newmap)
 {
-  std::vector<ceph_tid_t> cancel_ops;
-
-  std::scoped_lock cmd_lock(command_lock);
-  auto &commands = command_table.get_commands();
-  for (const auto &[tid, op] : commands) {
+  cancel_commands_if([=, this](MDSCommandOp const& op) {
     const mds_gid_t op_mds_gid = op.mds_gid;
     if (newmap.is_dne_gid(op_mds_gid) || newmap.is_laggy_gid(op_mds_gid)) {
-      ldout(cct, 1) << __func__ << ": cancelling command op " << tid << dendl;
-      cancel_ops.push_back(tid);
+      ldout(cct, 1) << "cancel_commands: cancelling command op " << op.tid << dendl;
       if (op.outs) {
         std::ostringstream ss;
         ss << "MDS " << op_mds_gid << " went away";
@@ -3048,13 +3043,10 @@ void Client::cancel_commands(const MDSMap& newmap)
        * has its own lock.
        */
       op.con->mark_down();
-      if (op.on_finish)
-        op.on_finish->complete(-CEPHFS_ETIMEDOUT);
+      return -CEPHFS_ETIMEDOUT;
     }
-  }
-
-  for (const auto &tid : cancel_ops)
-    command_table.erase(tid);
+    return 0;
+  });
 }
 
 void Client::handle_mds_map(const MConstRef<MMDSMap>& m)
@@ -6416,7 +6408,8 @@ int Client::mds_command(
     const bufferlist& inbl,
     bufferlist *outbl,
     string *outs,
-    Context *onfinish)
+    Context *onfinish,
+    bool one_shot)
 {
   RWRef_t iref_reader(initialize_state, CLIENT_INITIALIZED);
   if (!iref_reader.is_state_satisfied())
@@ -6475,6 +6468,9 @@ int Client::mds_command(
 
     // Open a connection to the target MDS
     ConnectionRef conn = messenger->connect_to_mds(info.get_addrs());
+    if (one_shot) {
+      conn->send_keepalive();
+    }
 
     cl.unlock();
     {
@@ -6489,6 +6485,7 @@ int Client::mds_command(
       op.inbl = inbl;
       op.mds_gid = target_gid;
       op.con = conn;
+      op.one_shot = one_shot;
 
       ldout(cct, 4) << __func__ << ": new command op to " << target_gid
         << " tid=" << op.tid << " multi_id=" << op.multi_target_id << " "<< cmd << dendl;
@@ -16639,13 +16636,41 @@ void Client::ms_handle_connect(Connection *con)
 bool Client::ms_handle_reset(Connection *con)
 {
   ldout(cct, 0) << __func__ << " on " << con->get_peer_addr() << dendl;
+
+  cancel_commands_if([=, this](MDSCommandOp const& op) {
+    if (op.one_shot && op.con.get() == con) {
+      ldout(cct, 1) << "ms_handle_reset: aborting one-shot command op " << op.tid << dendl;
+      if (op.outs) {
+        std::ostringstream ss;
+        ss << "MDS connection reset";
+        *(op.outs) = ss.str();
+      }
+      return -EPIPE;
+    }
+    return 0;
+  });
+
   return false;
 }
 
 void Client::ms_handle_remote_reset(Connection *con)
 {
-  std::scoped_lock lock(client_lock);
   ldout(cct, 0) << __func__ << " on " << con->get_peer_addr() << dendl;
+
+  cancel_commands_if([=, this](MDSCommandOp const& op) {
+    if (op.one_shot && op.con.get() == con) {
+      ldout(cct, 1) << "ms_handle_remote_reset: aborting one-shot command op " << op.tid << dendl;
+      if (op.outs) {
+        std::ostringstream ss;
+        ss << "MDS remote session reset";
+        *(op.outs) = ss.str();
+      }
+      return -EPIPE;
+    }
+    return 0;
+  });
+
+  std::scoped_lock lock(client_lock);
   switch (con->get_peer_type()) {
   case CEPH_ENTITY_TYPE_MDS:
     {
@@ -16704,6 +16729,20 @@ void Client::ms_handle_remote_reset(Connection *con)
 bool Client::ms_handle_refused(Connection *con)
 {
   ldout(cct, 1) << __func__ << " on " << con->get_peer_addr() << dendl;
+
+  cancel_commands_if([=, this](MDSCommandOp const& op) {
+    if (op.one_shot && op.con.get() == con) {
+      ldout(cct, 1) << "ms_handle_refused: aborting one-shot command op " << op.tid << dendl;
+      if (op.outs) {
+        std::ostringstream ss;
+        ss << "MDS connection refused";
+        *(op.outs) = ss.str();
+      }
+      return -EPIPE;
+    }
+    return 0;
+  });
+
   return false;
 }
 
diff --git a/src/client/Client.h b/src/client/Client.h
index 712583aa0d3c..e611b6693dbb 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -96,6 +96,7 @@ class MDSCommandOp : public CommandOp
 {
   public:
   mds_gid_t     mds_gid;
+  bool          one_shot = false;
 
   explicit MDSCommandOp(ceph_tid_t t) : CommandOp(t) {}
   explicit MDSCommandOp(ceph_tid_t t, ceph_tid_t multi_id) : CommandOp(t, multi_id) {}
@@ -333,7 +334,7 @@ class Client : public Dispatcher, public md_config_obs_t {
     const std::string &mds_spec,
     const std::vector<std::string>& cmd,
     const bufferlist& inbl,
-    bufferlist *poutbl, std::string *prs, Context *onfinish);
+    bufferlist *poutbl, std::string *prs, Context *onfinish, bool one_shot = false);
 
   // these should (more or less) mirror the actual system calls.
   int statfs(const char *path, struct statvfs *stbuf, const UserPerm& perms);
@@ -712,6 +713,27 @@ class Client : public Dispatcher, public md_config_obs_t {
   virtual void shutdown();
 
   // messaging
+  int cancel_commands_if(std::regular_invocable<MDSCommandOp const&> auto && error_for_op)
+  {
+    std::vector<ceph_tid_t> cancel_ops;
+
+    std::scoped_lock cmd_lock(command_lock);
+    auto& commands = command_table.get_commands();
+    for (const auto &[tid, op]: commands) {
+      int rc = static_cast<int>(error_for_op(op));
+      if (rc) {
+        cancel_ops.push_back(tid);
+        if (op.on_finish)
+          op.on_finish->complete(rc);
+      }
+    }
+
+    for (const auto& tid : cancel_ops)
+      command_table.erase(tid);
+
+    return cancel_ops.size();
+  }
+
   void cancel_commands(const MDSMap& newmap);
   void handle_mds_map(const MConstRef<MMDSMap>& m);
   void handle_fs_map(const MConstRef<MFSMap>& m);
diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc
index 6cb3a6bce245..67d9986ef8e9 100644
--- a/src/mgr/BaseMgrModule.cc
+++ b/src/mgr/BaseMgrModule.cc
@@ -108,7 +108,7 @@ class MonCommandCompletion : public Context
 
 
 static PyObject*
-ceph_send_command(BaseMgrModule *self, PyObject *args)
+ceph_send_command(BaseMgrModule *self, PyObject *args, PyObject *kwargs)
 {
   // Like mon, osd, mds
   char *type = nullptr;
@@ -122,9 +122,23 @@ ceph_send_command(BaseMgrModule *self, PyObject *args)
   Py_ssize_t inbuf_len = 0;
   bufferlist inbuf = {};
 
+  static const char * keywords[] {
+    "result",
+    "svc_type",
+    "svc_id",
+    "command",
+    "tag",
+    "inbuf",
+    // --- kwargs star here
+    "one_shot",   // whether to keep the command while we reestablish connection
+    nullptr       // must be the last element
+  };
+
+  int one_shot = false;
+
   PyObject *completion = nullptr;
-  if (!PyArg_ParseTuple(args, "Ossssz#:ceph_send_command",
-        &completion, &type, &name, &cmd_json, &tag, &inbuf_ptr, &inbuf_len)) {
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ossssz#|$p:ceph_send_command", const_cast<char**>(keywords),
+        &completion, &type, &name, &cmd_json, &tag, &inbuf_ptr, &inbuf_len, &one_shot)) {
     return nullptr;
   }
 
@@ -199,7 +213,8 @@ ceph_send_command(BaseMgrModule *self, PyObject *args)
         inbuf,
         &command_c->outbl,
         &command_c->outs,
-        new C_OnFinisher(command_c, &self->py_modules->cmd_finisher));
+        new C_OnFinisher(command_c, &self->py_modules->cmd_finisher),
+        one_shot);
     if (r != 0) {
       string msg("failed to send command to mds: ");
       msg.append(cpp_strerror(r));
@@ -1426,7 +1441,7 @@ PyMethodDef BaseMgrModule_methods[] = {
   {"_ceph_get_daemon_status", (PyCFunction)get_daemon_status, METH_VARARGS,
    "Get a service's status"},
 
-  {"_ceph_send_command", (PyCFunction)ceph_send_command, METH_VARARGS,
+  {"_ceph_send_command", (PyCFunction)ceph_send_command, METH_VARARGS | METH_KEYWORDS,
    "Send a mon command"},
 
   {"_ceph_set_health_checks", (PyCFunction)ceph_set_health_checks, METH_VARARGS,
diff --git a/src/pybind/mgr/ceph_module.pyi b/src/pybind/mgr/ceph_module.pyi
index df4a3782a0c9..3777c469a1fd 100644
--- a/src/pybind/mgr/ceph_module.pyi
+++ b/src/pybind/mgr/ceph_module.pyi
@@ -83,7 +83,9 @@ class BaseMgrModule(object):
                            svc_id: str,
                            command: str,
                            tag: str,
-                           inbuf: Optional[str]) -> None: ...
+                           inbuf: Optional[str],
+                           *,
+                           one_shot: bool) -> None: ...
     def _ceph_set_health_checks(self, checks: Mapping[str, HealthCheckT]) -> None: ...
     def _ceph_get_mgr_id(self) -> str: ...
     def _ceph_get_ceph_conf_path(self) -> str: ...
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 13cf9386bcdb..51ed4271821e 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1707,7 +1707,7 @@ def osd_command(self, cmd_dict: dict, inbuf: Optional[str] = None) -> Tuple[int,
 
         return r
 
-    def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf: Optional[str] = None) -> Tuple[int, str, str]:
+    def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf: Optional[str] = None, one_shot: bool = False) -> Tuple[int, str, str]:
         """
         Helper for `ceph tell` command execution.
 
@@ -1722,7 +1722,7 @@ def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf:
         """
         t1 = time.time()
         result = CommandResult()
-        self.send_command(result, daemon_type, daemon_id, json.dumps(cmd_dict), "", inbuf)
+        self.send_command(result, daemon_type, daemon_id, json.dumps(cmd_dict), "", inbuf, one_shot=one_shot)
         r = result.wait()
         t2 = time.time()
 
@@ -1739,7 +1739,9 @@ def send_command(
             svc_id: str,
             command: str,
             tag: str,
-            inbuf: Optional[str] = None) -> None:
+            inbuf: Optional[str] = None,
+            *, # kw-only args go below
+            one_shot: bool = False) -> None:
         """
         Called by the plugin to send a command to the mon
         cluster.
@@ -1760,8 +1762,10 @@ def send_command(
             triggered, with notify_type set to "command", and notify_id set to
             the tag of the command.
         :param str inbuf: input buffer for sending additional data.
+        :param bool one_shot: a keyword-only param to make the command abort
+            with EPIPE when the target resets or refuses to reconnect
         """
-        self._ceph_send_command(result, svc_type, svc_id, command, tag, inbuf)
+        self._ceph_send_command(result, svc_type, svc_id, command, tag, inbuf, one_shot=one_shot)
 
     def tool_exec(
         self,
diff --git a/src/pybind/mgr/tests/__init__.py b/src/pybind/mgr/tests/__init__.py
index 633959084ccb..8ae6ea54b462 100644
--- a/src/pybind/mgr/tests/__init__.py
+++ b/src/pybind/mgr/tests/__init__.py
@@ -100,7 +100,7 @@ def _ceph_set_module_option(self, module, key, val):
         def _ceph_get(self, data_name):
             return self.mock_store_get('_ceph_get', data_name, mock.MagicMock())
 
-        def _ceph_send_command(self, res, svc_type, svc_id, command, tag, inbuf):
+        def _ceph_send_command(self, res, svc_type, svc_id, command, tag, inbuf, *, one_shot=False):
 
             cmd = json.loads(command)
             getattr(self, '_mon_commands_sent', []).append(cmd)

From 78afc6136118e14e2cf6a5651bfe11777c9881a4 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 4 Jan 2024 19:51:32 +0200
Subject: [PATCH 2115/2492] mgr/volumes: use `volume_exception_to_retval` as a
 decorator

When used as a decorator, it saves one indented try-catch block inside the decorated method.
This can be applied to most of the methods in the file, subject to a separate refactoring commit

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/pybind/mgr/volumes/fs/volume.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 0c4a07598054..8e89fed007a1 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -2,7 +2,9 @@
 import errno
 import logging
 import mgr_util
-from typing import TYPE_CHECKING
+import inspect
+import functools
+from typing import TYPE_CHECKING, Any, Callable, Optional
 
 import cephfs
 
@@ -86,11 +88,28 @@ def cluster_log(self, msg, lvl=None):
             lvl = self.mgr.ClusterLogPrio.WARN
         self.mgr.cluster_log("cluster", lvl, msg)
 
-    def volume_exception_to_retval(self, ve):
+    def volume_exception_to_retval(self_or_method: Any, ve: Optional[VolumeException] = None):
         """
         return a tuple representation from a volume exception
+        OR wrap the decorated method into a try:catch:
+        that will convert VolumeException to the tuple
         """
-        return ve.to_tuple()
+        if ve is None and callable(self_or_method):
+            # used as a decorator
+            method: Callable = self_or_method
+            @functools.wraps(method)
+            def wrapper(self, *args, **kwargs):
+                try:
+                    return method(self, *args, **kwargs)
+                except VolumeException as ve:
+                    return self.volume_exception_to_retval(ve)
+            return wrapper
+        elif ve is not None:
+            # used as a method on self with a VolumeException argument
+            return ve.to_tuple()
+        else:
+            # shouldn't get here, bad call
+            assert(ve is not None)
 
     ### volume operations -- create, rm, ls
 

From d151876d5bf34bbd7aff2cd8ce6dd2e1dffbc414 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 4 Jan 2024 19:52:44 +0200
Subject: [PATCH 2116/2492] mgr/volumes: support for `fs subvolume quiesce`

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/pybind/mgr/mgr_module.py        | 56 +++++++++++++++++++++++++++++
 src/pybind/mgr/volumes/fs/volume.py | 45 +++++++++++++++++++++++
 src/pybind/mgr/volumes/module.py    | 27 ++++++++++++++
 3 files changed, 128 insertions(+)

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 51ed4271821e..e75a57659025 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1731,6 +1731,62 @@ def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf:
         ))
 
         return r
+    
+    MDS_STATE_ORD = {
+        "down:dne":              0, # CEPH_MDS_STATE_DNE,
+        "down:stopped":         -1, # CEPH_MDS_STATE_STOPPED,
+        "down:damaged":         15, # CEPH_MDS_STATE_DAMAGED,
+        "up:boot":              -4, # CEPH_MDS_STATE_BOOT,
+        "up:standby":           -5, # CEPH_MDS_STATE_STANDBY,
+        "up:standby-replay":    -8, # CEPH_MDS_STATE_STANDBY_REPLAY,
+        "up:oneshot-replay":    -9, # CEPH_MDS_STATE_REPLAYONCE,
+        "up:creating":          -6, # CEPH_MDS_STATE_CREATING,
+        "up:starting":          -7, # CEPH_MDS_STATE_STARTING,
+        "up:replay":             8, # CEPH_MDS_STATE_REPLAY,
+        "up:resolve":            9, # CEPH_MDS_STATE_RESOLVE,
+        "up:reconnect":         10, # CEPH_MDS_STATE_RECONNECT,
+        "up:rejoin":            11, # CEPH_MDS_STATE_REJOIN,
+        "up:clientreplay":      12, # CEPH_MDS_STATE_CLIENTREPLAY,
+        "up:active":            13, # CEPH_MDS_STATE_ACTIVE,
+        "up:stopping":          14, # CEPH_MDS_STATE_STOPPING,
+    }
+    MDS_STATE_ACTIVE_ORD = MDS_STATE_ORD["up:active"]
+
+    def get_quiesce_leader_info(self, fscid: str) -> dict:
+        leader_info = None
+
+        for fs in self.get("fs_map")['filesystems']:
+            if fscid != fs["id"]:
+                continue
+            
+            # quiesce leader is the lowest rank
+            # with the highest state
+            mdsmap = fs["mdsmap"]
+            for info in mdsmap['info'].values():
+                if info['rank'] == -1:
+                    continue
+                if leader_info is None:
+                    leader_info = info
+                else:
+                    if info['rank'] < leader_info['rank']:
+                        leader_info = info
+                    elif info['rank'] == leader_info['rank']:
+                        state_ord = self.MDS_STATE_ORD.get(info['state'])
+                        leader_state_ord = self.MDS_STATE_ORD.get(leader_info['state'])
+
+                        if state_ord <= self.MDS_STATE_ACTIVE_ORD and state_ord > leader_state_ord:
+                            leader_info = info
+            break
+
+        return leader_info
+
+    def tell_quiesce_leader(self, fscid: str, cmd_dict: dict, inbuf: Optional[str] = None) -> Tuple[int, str, str]:
+        qleader: dict = self.get_quiesce_leader_info(fscid)
+        if qleader is None:
+            self.log.warn("Couldn't resolve the quiesce leader for fscid %s" % fscid)
+            return (-errno.ENOENT, "", "Couldn't resolve the quiesce leader for fscid %s" % fscid)
+        self.log.debug("resolved quiesce leader for fscid {fscid} at daemon '{name}' gid {gid} rank {rank} ({state})".format(fscid=fscid, **qleader))
+        return self.tell_command('mds', str(qleader['gid']), cmd_dict, one_shot=True)
 
     def send_command(
             self,
diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 8e89fed007a1..ef3b171af762 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -5,6 +5,7 @@
 import inspect
 import functools
 from typing import TYPE_CHECKING, Any, Callable, Optional
+from urllib.parse import urlsplit, urlunsplit
 
 import cephfs
 
@@ -450,6 +451,50 @@ def subvolume_info(self, **kwargs):
             ret = self.volume_exception_to_retval(ve)
         return ret
 
+    @volume_exception_to_retval
+    def quiesce(self, cmd):
+        volname    = cmd['vol_name']
+        default_group_name  = cmd.get('group_name', None)
+        roots = []
+        fscid = None
+
+        with open_volume(self, volname) as fs_handle:
+            fscid = fs_handle.get_fscid()
+
+            if cmd.get('leader', False):
+                leader_info = self.mgr.get_quiesce_leader_info(fscid)
+                if leader_info is None:
+                    return -errno.ENOENT, "", "Couldn't resolve the quiesce leader for volume %s (%s)" % (volname, fscid)
+                return (
+                    0,
+                    "mds.%d" % leader_info['gid'],
+                    "Resolved the quiesce leader for volume '{volname}' as daemon '{name}' ({gid}) {state} rank {rank}".format(volname=volname, **leader_info)
+                )
+
+
+            for member in cmd.get('members', []):
+                try:
+                    member_parts = urlsplit(member)
+                except ValueError as ve:
+                    return -errno.EINVAL, "", str(ve)
+                group_name = default_group_name
+
+                *maybe_group_name, subvol_name = member_parts.path.strip('/').split('/')
+                if len(maybe_group_name) > 1:
+                    return -errno.EINVAL, "", "The `<group>/<subvol>` member syntax is accepted with no more than one group"
+                elif len(maybe_group_name) == 1:
+                    group_name = maybe_group_name[0]
+
+                with open_group(fs_handle, self.volspec, group_name) as group:
+                    with open_subvol(self.mgr, fs_handle, self.volspec, group, subvol_name, SubvolumeOpType.GETPATH) as subvol:
+                        member_parts = member_parts._replace(path=subvol.path.decode('utf-8'))
+                        roots.append(urlunsplit(member_parts))
+        
+        cmd['roots'] = roots
+        cmd['prefix'] = 'quiesce db'
+
+        return self.mgr.tell_quiesce_leader(fscid, cmd)
+
     def set_user_metadata(self, **kwargs):
         ret        = 0, "", ""
         volname    = kwargs['vol_name']
diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index 4a28fdc869ea..ee36005d406d 100644
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -272,6 +272,29 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                     "and optionally, in a specific subvolume group",
             'perm': 'rw'
         },
+        {
+            'cmd': 'fs quiesce '
+                   'name=vol_name,type=CephString '
+                   'name=members,type=CephString,n=N,req=false '
+                   '-- '
+                   'name=set_id,type=CephString,req=false '
+                   'name=timeout,type=CephFloat,range=0,req=false '
+                   'name=expiration,type=CephFloat,range=0,req=false '
+                   'name=await_for,type=CephFloat,range=0,req=false '
+                   'name=await,type=CephBool,req=false '
+                   'name=if_version,type=CephInt,range=0,req=false '
+                   'name=include,type=CephBool,req=false '
+                   'name=exclude,type=CephBool,req=false '
+                   'name=reset,type=CephBool,req=false '
+                   'name=release,type=CephBool,req=false '
+                   'name=query,type=CephBool,req=false '
+                   'name=all,type=CephBool,req=false '
+                   'name=cancel,type=CephBool,req=false '
+                   'name=group_name,type=CephString,req=false '
+                   'name=leader,type=CephBool,req=false ',
+            'desc': "Manage quiesce sets of subvolumes",
+            'perm': 'rw'
+        },
         {
             'cmd': 'fs subvolumegroup pin'
                    ' name=vol_name,type=CephString'
@@ -738,6 +761,10 @@ def _cmd_fs_subvolume_metadata_rm(self, inbuf, cmd):
                                       key_name=cmd['key_name'],
                                       group_name=cmd.get('group_name', None),
                                       force=cmd.get('force', False))
+    
+    @mgr_cmd_wrap
+    def _cmd_fs_quiesce(self, inbuf, cmd):
+        return self.vc.quiesce(cmd)
 
     @mgr_cmd_wrap
     def _cmd_fs_subvolumegroup_pin(self, inbuf, cmd):

From 88fb668938f051b723c518a7faa6aa759bc39829 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 8 Jan 2024 17:37:20 +0200
Subject: [PATCH 2117/2492] doc: fixes for local dev builds

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 doc/.gitignore            |  1 +
 doc/_ext/ceph_commands.py |  7 ++++---
 doc/conf.py               | 10 +++++++---
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/.gitignore b/doc/.gitignore
index 0c7c74746ae9..9ee3c337d0ab 100644
--- a/doc/.gitignore
+++ b/doc/.gitignore
@@ -1,2 +1,3 @@
 /overview.png
 /object_store.png
+_build/
diff --git a/doc/_ext/ceph_commands.py b/doc/_ext/ceph_commands.py
index 861a013ea5a4..3822a8792938 100644
--- a/doc/_ext/ceph_commands.py
+++ b/doc/_ext/ceph_commands.py
@@ -177,7 +177,7 @@ def parse_cmd(cmd):
 
     @staticmethod
     def parse_args(args):
-        return [Sig._parse_arg_desc(arg) for arg in args.split()]
+        return [Sig._parse_arg_desc(arg) for arg in args]
 
 
 TEMPLATE = '''
@@ -358,8 +358,9 @@ def run(self):
         cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
         self._render_cmds(cmds)
 
-        orig_rgw_mod = sys.modules['pybind_rgw_mod']
-        sys.modules['rgw'] = orig_rgw_mod
+        if 'pybind_rgw_mod' in sys.modules:
+            orig_rgw_mod = sys.modules['pybind_rgw_mod']
+            sys.modules['rgw'] = orig_rgw_mod
 
         return []
 
diff --git a/doc/conf.py b/doc/conf.py
index 2a083fae0e84..4fdc9a53b757 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -13,9 +13,13 @@
         os.path.dirname(
             os.path.abspath(__file__)))
 
-pybind_rgw_mod = __import__('rgw', globals(), locals(), [], 0)
-sys.modules['pybind_rgw_mod'] = pybind_rgw_mod
-
+# it could be that ceph was built without RGW support
+# e.g. in a local development environment
+try:
+    pybind_rgw_mod = __import__('rgw', globals(), locals(), [], 0)
+    sys.modules['pybind_rgw_mod'] = pybind_rgw_mod
+except Exception:
+    pass
 
 def parse_ceph_release():
     with open(os.path.join(top_level, 'src/ceph_release')) as f:

From 09e08ac6a43005b09598d936b1faccd15be85067 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 8 Jan 2024 15:08:26 +0200
Subject: [PATCH 2118/2492] doc/cephfs/fs-volumes: Add info about the quiesce
 command

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 doc/cephfs/fs-volumes.rst         | 373 +++++++++++++++++++++++++++++-
 doc/cephfs/quiesce-set-states.svg | 142 ++++++++++++
 2 files changed, 514 insertions(+), 1 deletion(-)
 create mode 100644 doc/cephfs/quiesce-set-states.svg

diff --git a/doc/cephfs/fs-volumes.rst b/doc/cephfs/fs-volumes.rst
index dc9be8379aa3..476e211c1587 100644
--- a/doc/cephfs/fs-volumes.rst
+++ b/doc/cephfs/fs-volumes.rst
@@ -306,7 +306,7 @@ Resize a subvolume using:
    ceph fs subvolume resize <vol_name> <subvol_name> <new_size> [--group_name <subvol_group_name>] [--no_shrink]
 
 The command resizes the subvolume quota using the size specified by ``new_size``.
-The `--no_shrink`` flag prevents the subvolume from shrinking below the current  used size of the subvolume.
+The ``--no_shrink`` flag prevents the subvolume from shrinking below the current  used size of the subvolume.
 
 The subvolume can be resized to an unlimited (but sparse) logical size by passing ``inf`` or ``infinite`` as `` new_size``.
 
@@ -785,6 +785,377 @@ Will enable distributed subtree partitioning policy for the "csi" subvolume
 group.  This will cause every subvolume within the group to be automatically
 pinned to one of the available ranks on the file system.
 
+Subvolume quiesce
+-----------------
+
+It may be needed to pause IO to a set of subvolumes of a given volume (file system).
+A good example of such case is a consistent snapshot spanning multiple subvolumes.
+Such a task arises often in an environment such as k8s, where a single deployed application
+can work with many mounted subvolumes across several hosts. When a snapshot of such a system is needed,
+the application may not find the result consistent unless the snapshots were taken
+under an active write pause.
+
+The volumes plugin provides a tool to initiate and await such a pause across a set of subvolumes:
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce --set-id myset1 <vol_name> <[group_name/]sub_name...> --await
+  # perform actions while the IO pause is active, like taking snapshots
+  $ ceph fs quiesce --set-id myset1 --release --await
+  # if successful, all members of the set were confirmed as still in pause and released from such
+
+The ``quiesce`` functionality is itself based on top of a lower level QuiesceDb service maintained by the MDS
+daemons. Volumes plugin merely maps the subvolume names to their corresponding paths on the given file system
+and then issues the appropriate quiesce command to the MDS. You can learn more about the feature in the developer guides.
+
+Operations
+~~~~~~~~~~
+
+The IO pause (referred to as `quiesce`) is requested for a group of one or more subvolumes (i.e. paths in a filesystem).
+The group is referred to as "quiesce set", and every quiesce set must have a unique string id to interact with.
+A quiesce set can be manipulated in the following ways:
+
+* **include** one or more subvolumes - quiesce set members
+* **exclude** one or more members
+* **cancel** the set, asynchronously aborting the pause on all its current members
+* **release** the set, requesting the end of the pause from all members and expecting an ack from all clients
+* **query** the current state of a set by id or all active sets or all known sets
+* **cancel all** active sets in case an immediate resume of IO is required.
+
+The operations listed above are non-blocking: they perform the intended modification if it's applicable
+and get back with an up to date version of the target set, whether the operation was successful or not. 
+The set may change states as a result of the modification, and the version that's returned in the response 
+is guaranteed to be in a state consistent with this and potentialy other successful operations from 
+the same control loop batch.
+
+Some set states are `awaitable`. We will discuss those below, but for now it's important to mention that
+any of the commands above can be amended with an **await** modifier, which will cause them to block
+on the set after applying their intended modification, as long as the resulting set state is `awaitable`.
+Such a command will block until the set reaches the awaited state, gets modified by another command,
+or transitions into another state. The reason for the unblock will be clear by the result code, while
+the contents of the response will always be the most recent set state.
+
+.. image:: quiesce-set-states.svg
+
+`Awaitable` states on the diagram are marked with ``(a)`` or ``(A)``. Blocking versions of the operations
+will pend while the set is in an ``(a)`` state and will complete with success if it reaches an ``(A)`` state.
+If the set is already at an ``(A)`` state, the operation completes immediately with a success. 
+
+Most of the operations require a set-id. The exceptions are:
+
+* creation of a new set without specifying a set id,
+* query of active or all known sets, and
+* the cancel all
+
+Creating a new set is achieved by including member(s) via the `include` or `reset` commands.
+It's possible to specify a set id, and if it's a new id then the set will be created
+with the specified member(s) in the `QUIESCING` state. When no set id is specified while including
+or resetting members, then a new set with a unique set id is created. The set id will be known
+to the caller by inspecting the output
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a sub1 --set-id=unique-id
+  {
+      "epoch": 3,
+      "db_version": 1,
+      "sets": {
+          "unique-id": {
+              "db_version": 1,
+              "age_ref": 0.0,
+              "state": {
+                  "name": "TIMEDOUT",
+                  "age": 0.0
+              },
+              "timeout": 0.0,
+              "expiration": 0.0,
+              "members": {
+                  "file:/volumes/_nogroup/sub1/b1fcce76-3418-42dd-aa76-f9076d047dd3": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 0.0
+                      }
+                  }
+              }
+          }
+      }
+  }
+
+The output contains the set we just created successfully, however it's already `TIMEDOUT`. 
+This is expected, since we have not specified the timeout for this quiesce,
+and we can see in the output that it was initialized to 0 by default, along with the expiration.
+
+Timeouts
+~~~~~~~~
+
+The two timeout parameters, `timeout` and `expiration`, are the main guards against 
+accidentally causing a DOS condition for our application. Any command to an active set
+may carry the ``--timeout`` or ``--expiration`` arguments to update these values for the set.
+If present, the values will be applied before the action this command requests.
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a --set-id=unique-id --timeout=10 > /dev/null
+  Error EPERM:  
+
+It's too late for our ``unique-id`` set, as it's in a terminal state. No changes are allowed
+to sets that are in their terminal states, i.e. inactive. Let's create a new set
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a sub1 --timeout 60
+  {
+      "epoch": 3,
+      "db_version": 2,
+      "sets": {
+          "8988b419": {
+              "db_version": 2,
+              "age_ref": 0.0,
+              "state": {
+                  "name": "QUIESCING",
+                  "age": 0.0
+              },
+              "timeout": 60.0,
+              "expiration": 0.0,
+              "members": {
+                  "file:/volumes/_nogroup/sub1/b1fcce76-3418-42dd-aa76-f9076d047dd3": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 0.0
+                      }
+                  }
+              }
+          }
+      }
+  }
+
+This time, we haven't specified a set id, so the system created a new one. We see its id
+in the output, it's ``8988b419``. The command was a success and we see that 
+this time the set is `QUIESCING`. At this point, we can add more members to the set
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a --set-id 8988b419 --include sub2 sub3
+  {
+      "epoch": 3,
+      "db_version": 3,
+      "sets": {
+          "8988b419": {
+              "db_version": 3,
+              "age_ref": 0.0,
+              "state": {
+                  "name": "QUIESCING",
+                  "age": 30.7
+              },
+              "timeout": 60.0,
+              "expiration": 0.0,
+              "members": {
+                  "file:/volumes/_nogroup/sub1/b1fcce76-3418-42dd-aa76-f9076d047dd3": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 30.7
+                      }
+                  },
+                  "file:/volumes/_nogroup/sub2/bc8f770e-7a43-48f3-aa26-d6d76ef98d3e": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 0.0
+                      }
+                  },
+                  "file:/volumes/_nogroup/sub3/24c4b57b-e249-4b89-b4fa-7a810edcd35b": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 0.0
+                      }
+                  }
+              }
+          }
+      }
+  }
+
+The ``--include`` bit is optional, as if no operation is given while members are provided, 
+then "include" is assumed.
+
+As we have seen, the timeout argument specifies how much time we are ready to give the system
+to reach the `QUIESCED` state on the set. However, since new members can be added to an
+active set at any time, it wouldn't be fair to measure the timeout from the set creation time.
+Hence, the timeout is tracked per member: every member has `timeout` seconds to quiesce,
+and if any one takes longer than that, the whole set is marked as `TIMEDOUT` and the pause is released.
+
+Once the set is in the `QUIESCED` state, it will begin its expiration timer. This timer is tracked
+per set as a whole, not per members. Once the `expiration` seconds elapse, the set will transition
+into an `EXPIRED` state, unless it was actively released or canceled by a dedicated operation.
+
+It's possible to add new members to a `QUIESCED` set. In this case, it will transition back to `QUIESCING`,
+and the new member(s) will have their own timeout to quiesce. If they succeed, then the set will
+again be `QUIESCED` and the expiration timer will restart. 
+
+.. warning:: 
+  * The `expiration timer` doesn't apply when a set is `QUIESCING`; it is reset to the
+    value of the `expiration` property when the **set** becomes `QUIESCED`
+  * The `timeout` doesn't apply to **members** that are `QUIESCED`
+
+Awaiting
+~~~~~~~~
+
+Note that the commands above are all non-blocking. If we want to wait for the quiesce set
+to reach the `QUIESCED` state, we should await it at some point. ``--await`` can be given
+along with other arguments to let the system know our intention.
+
+Technically, there are two types of await: `quiesce await` and `release await`. The former
+is the default, and the latter can only be achieved with ``--release`` present in the argument list.
+To avoid confision, it is not permitted to issue a `quiesce await` when the set is already `RELEASING`
+or `RELEASED`. Trying to ``--release`` a set that is not `QUIESCED` is an ``EPERM`` error as well, regardless
+of whether await is requested alongside. However, it's not an error to `release await` 
+an already released set, or to `quiesce await` a `QUIESCED` one.
+
+When awaiting, one may also specify a maximum duration that they would like this await request to block for,
+not affecting the two intrinsic timeouts discussed above. If the target awaited state isn't reached
+within the specified duration, then ``EINPROGRESS`` is returned. For that, one should use the argument
+``--await-for=<seconds>``. One could think of ``--await`` as equivalent to ``--await-for=Infinity``.
+While it doesn't make sense to specify both arguments, it is not considered an error. If
+both ``--await`` and ``--await-for`` are present, then the former is ignored, and the time limit
+from ``--await-for`` is honored.
+
+.. prompt:: bash $ auto
+
+  $ time ceph fs quiesce a sub1 --timeout=10 --await-for=2
+  {
+      "epoch": 6,
+      "db_version": 3,
+      "sets": {
+          "c3c1d8de": {
+              "db_version": 3,
+              "age_ref": 0.0,
+              "state": {
+                  "name": "QUIESCING",
+                  "age": 2.0
+              },
+              "timeout": 10.0,
+              "expiration": 0.0,
+              "members": {
+                  "file:/volumes/_nogroup/sub1/b1fcce76-3418-42dd-aa76-f9076d047dd3": {
+                      "excluded": false,
+                      "state": {
+                          "name": "QUIESCING",
+                          "age": 2.0
+                      }
+                  }
+              }
+          }
+      }
+  }
+  Error EINPROGRESS: 
+  ceph fs quiesce a sub1 --timeout=10 --await-for=2  0.41s user 0.04s system 17% cpu 2.563 total
+
+(there is a ~0.5 sec overhead that the ceph client adds, at least in a local debug setup)
+
+Quiesce-Await and Expiration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Quiesce await has a side effect: it resets the internal expiration timer. This allows for a watchdog
+approach to a long running multistep process under the IO pause by repeatedly ``--await``\ ing an already
+`QUIESCED` set. Consider the following example script:
+
+.. prompt:: bash $ auto
+
+  $ set -e   # (1)
+  $ ceph fs quiesce a sub1 sub2 sub3 --timeout=30 --expiration=10 --set-id="snapshots" --await # (2)
+  $ ceph fs subvolume snapshot create a sub1 snap1-sub1  # (3)
+  $ ceph fs quiesce a --set-id="snapshots" --await  # (4)
+  $ ceph fs subvolume snapshot create a sub2 snap1-sub2  # (3)
+  $ ceph fs quiesce a --set-id="snapshots" --await  # (4)
+  $ ceph fs subvolume snapshot create a sub3 snap1-sub3  # (3)
+  $ ceph fs quiesce a --set-id="snapshots" --release --await  # (5)
+
+.. warning:: This example uses arbitrary timeouts to convey the concept. In real life, the values must be carefully
+  chosen in accordance with the actual system requirements and specifications.
+
+The goal of the script is to take consistent snapshots of 3 subvolumes. 
+We begin by setting the bash ``-e`` option `(1)` to exit this script if any or the following commands 
+returns with a non-zero status.
+
+We go on requesting an IO pause for the three subvolumes `(2)`. We set our timeouts allowing 
+the system to spend up to 30 seconds reaching the quiesced state across all members
+and stay quiesced for up to 10 seconds before the quiesce expires and the IO
+is resumed. We also specify ``--await`` to only proceed once the quiesce is reached.
+
+We then proceed with a set of command pairs that take the next snapshot and call ``--await`` on our set
+to extend the expiration timeout for 10 more seconds `(3,4)`. This approach gives us up to 10 seconds
+for every snapshot, but also allows taking as many snapshots as we need without losing the IO pause,
+and with it - consistency. If we wanted, we could update the `expiration` every time we called for await.
+
+If any of the snapshots gets stuck and takes longer than 10 seconds to complete, then the next call
+to ``--await`` will return an error since the set will be `EXPIRED` which is not an awaitable state.
+This limits the impact on the applications in the bad case scenarios.
+
+We could have set the `expiration` timeout to 30 at the beginning `(2)`, but that would mean that
+a single stuck snapshot would keep the applications pending for all this time.
+
+If Version
+~~~~~~~~~~
+
+Sometimes, it's not enough to just observe the successful quiesce or release. The reason could be
+a concurrent change of the set by another client. Consider this example:
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a sub1 sub2 sub3 --timeout=30 --expiration=60 --set-id="snapshots" --await  # (1)
+  $ ceph fs subvolume snapshot create a sub1 snap1-sub1  # (2)
+  $ ceph fs subvolume snapshot create a sub2 snap1-sub2  # (3)
+  $ ceph fs subvolume snapshot create a sub3 snap1-sub3  # (4)
+  $ ceph fs quiesce a --set-id="snapshots" --release --await  # (5)
+
+The sequence looks good, and the release `(5)` completes successfully. However, it could be that
+before snap for sub3 `(4)` is taken, another session excludes sub3 from the set, resuming its IOs
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a --set-id="snapshots" --exclude sub3
+
+Since removing a member from a set doesn't affect its `QUIESCED` state, the release command `(5)`
+has no reason to fail. It will ack the two unexcluded members sub1 and sub2 and report success.
+
+In order to address this or similar problems, the quiesce command supports an optimistic concurrency
+mode. To activate it, one needs to pass an ``--if-version=<db_version>`` that will be compared
+to the set's db version and the operation will only proceed if the values match. Otherwise, the command
+will not be executed and the return status will be ``ESTALE``.
+
+It's easy to know which version to expect of a set, since every command that modifies a set will return
+this set on the stdout, regarldess of the exit status. In the examples above one can notice that every
+set carries a ``"db_version"`` property which is the last db version where this set got modified.
+
+In the example at the beginning of this subsection, the initial quiesce command `(1)` would have returned
+the newly created set with id ``"snapshots"`` and some version, let's say ``13``. Since we don't expect any other
+changes to the set while we are making snapshots with the commands `(2,3,4)`, the release command `(5)`
+could have looked like
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a --set-id="snapshots" --release --await --if-version=13 # (5)
+
+This way, the result of the release command would have been ``ESTALE`` instead of 0, and we would
+know that something wasn't right with the quiesce set and our snapshots might not be consistent.
+
+.. tip:: When ``--if-version`` and the command returns ``ESTALE``, the requested action is **not** executed.
+  It means that the script may want to execute some unconditional command on the set to adjust its state
+  according to the requirements
+
+There is another use of the ``--if-version`` argument which could come handy for automation software.
+As we have discussed earlier, it is possible to create a new quiesce set with a given set id. Drivers like
+the CSI for k8s could use their internal request id to eliminate the need to keep an additional mapping
+to the quiesce set id. However, to guarantee uniqueness, the driver may want to verify that the set is
+indeed new. For that, ``if-version=0`` may be used, and it will only create the new set if no other
+set with this id was present in the database
+
+.. prompt:: bash $ auto
+
+  $ ceph fs quiesce a sub1 sub2 sub3 --set-id="external-id" --if-version=0
 
 .. _manila: https://github.com/openstack/manila
 .. _CSI: https://github.com/ceph/ceph-csi
diff --git a/doc/cephfs/quiesce-set-states.svg b/doc/cephfs/quiesce-set-states.svg
new file mode 100644
index 000000000000..82f53886c7f3
--- /dev/null
+++ b/doc/cephfs/quiesce-set-states.svg
@@ -0,0 +1,142 @@
+<svg version="1.1" viewBox="0.0 0.0 960.0 540.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10"
+  xmlns:xlink="http://www.w3.org/1999/xlink"
+  xmlns="http://www.w3.org/2000/svg">
+  <clipPath id="g2b2bd9b3a18_0_19.0">
+    <path d="m0 0l960.0 0l0 540.0l-960.0 0l0 -540.0z" clip-rule="nonzero"/>
+  </clipPath>
+  <g clip-path="url(#g2b2bd9b3a18_0_19.0)">
+    <path fill="#0b2e35" d="m0 0l960.0 0l0 540.0l-960.0 0z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m798.38306 486.98688l135.83734 0l0 37.1286l-135.83734 0z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m32.72441 46.721786l894.55115 0l0 60.125984l-894.55115 0z" fill-rule="evenodd"/>
+    <path fill="#ffffff" d="m61.31816 94.40553l-2.0625 -3.046875q-0.90625 0.296875 -1.890625 0.46875q-0.96875 0.1875 -2.125 0.1875q-3.140625 0 -5.328125 -1.1875q-2.171875 -1.1875 -3.515625 -3.15625q-1.34375 -1.96875 -1.96875 -4.359375q-0.609375 -2.390625 -0.609375 -4.8125q0 -2.421875 0.609375 -4.8125q0.625 -2.390625 1.96875 -4.359375q1.34375 -1.96875 3.515625 -3.15625q2.1875 -1.1875 5.328125 -1.1875q3.125 0 5.296875 1.1875q2.1875 1.1875 3.53125 3.15625q1.34375 1.96875 1.9375 4.359375q0.609375 2.390625 0.609375 4.8125q0 1.953125 -0.390625 3.890625q-0.375 1.9375 -1.203125 3.671875q-0.828125 1.71875 -2.171875 3.0625l2.015625 2.921875l-3.546875 2.359375zm-6.09375 -7.046875q0.34375 0 0.640625 -0.0625q0.3125 -0.0625 0.5625 -0.171875l-2.015625 -2.984375l3.609375 -2.359375l1.953125 2.96875q0.875 -1.25 1.25 -2.96875q0.375 -1.734375 0.375 -3.28125q0 -1.5625 -0.328125 -3.140625q-0.328125 -1.578125 -1.0625 -2.875q-0.734375 -1.296875 -1.96875 -2.09375q-1.21875 -0.796875 -3.0 -0.796875q-1.8125 0 -3.046875 0.828125q-1.234375 0.8125 -1.96875 2.15625q-0.734375 1.328125 -1.0625 2.890625q-0.328125 1.546875 -0.328125 3.03125q0 1.546875 0.328125 3.125q0.328125 1.5625 1.078125 2.875q0.765625 1.296875 1.984375 2.078125q1.21875 0.78125 3.0 0.78125zm26.769577 4.65625q-3.28125 0 -5.609375 -1.21875q-2.3125 -1.234375 -3.546875 -3.5625q-1.21875 -2.328125 -1.21875 -5.640625l0 -16.15625l4.84375 0l0 16.15625q0 2.96875 1.3125 4.375q1.328125 1.390625 4.21875 1.390625q1.84375 0 3.0625 -0.578125q1.234375 -0.578125 1.84375 -1.84375q0.625 -1.28125 0.625 -3.34375l0 -16.15625l4.859375 0l0 16.15625q0 2.765625 -0.8125 4.75q-0.8125 1.96875 -2.25 3.234375q-1.421875 1.25 -3.3125 1.84375q-1.875 0.59375 -4.015625 0.59375zm16.390213 -0.453125l0 -26.125l4.84375 0l0 26.125l-4.84375 0zm10.955841 0l0 -26.125l17.65625 0l0 4.59375l-12.8125 0l0 5.734375l7.59375 0l0 4.59375l-7.59375 0l0 6.578125l13.59375 0l0 4.625l-18.4375 0zm31.43512 0.453125q-2.4375 0 -4.4375 -0.859375q-2.0 -0.859375 -3.4375 -2.4375q-1.421875 -1.578125 -2.125 -3.71875l4.375 -1.625q0.84375 1.78125 2.34375 2.9375q1.515625 1.140625 3.390625 1.140625q2.0 0 3.1875 -0.828125q1.1875 -0.84375 1.1875 -2.453125q0 -1.09375 -0.703125 -1.828125q-0.703125 -0.75 -1.890625 -1.28125q-1.1875 -0.53125 -2.671875 -1.03125q-1.4375 -0.484375 -2.90625 -1.078125q-1.453125 -0.609375 -2.671875 -1.46875q-1.203125 -0.875 -1.9375 -2.1875q-0.734375 -1.328125 -0.734375 -3.25q0 -1.921875 1.015625 -3.53125q1.03125 -1.609375 2.984375 -2.5625q1.953125 -0.96875 4.734375 -0.96875q2.28125 0 4.03125 0.78125q1.765625 0.765625 2.921875 2.09375q1.15625 1.328125 1.609375 3.0l-4.328125 1.4375q-0.515625 -1.171875 -1.59375 -1.96875q-1.078125 -0.796875 -2.90625 -0.796875q-1.71875 0 -2.671875 0.671875q-0.953125 0.671875 -0.953125 1.90625q0 0.84375 0.640625 1.4375q0.65625 0.578125 1.75 1.046875q1.09375 0.46875 2.46875 0.953125q1.546875 0.53125 3.078125 1.1875q1.546875 0.65625 2.796875 1.609375q1.265625 0.953125 2.03125 2.375q0.765625 1.40625 0.765625 3.4375q0 2.5 -1.234375 4.25q-1.234375 1.75 -3.34375 2.6875q-2.109375 0.921875 -4.765625 0.921875zm24.338379 -0.046875q-3.046875 0 -5.15625 -1.171875q-2.09375 -1.171875 -3.375 -3.125q-1.265625 -1.953125 -1.84375 -4.34375q-0.578125 -2.390625 -0.578125 -4.828125q0 -2.3125 0.578125 -4.6875q0.59375 -2.390625 1.890625 -4.390625q1.296875 -2.015625 3.390625 -3.21875q2.09375 -1.21875 5.09375 -1.21875q2.484375 0 4.3125 0.828125q1.828125 0.828125 3.03125 2.15625q1.203125 1.3125 1.84375 2.75l-4.328125 2.015625q-0.640625 -0.96875 -1.3125 -1.671875q-0.671875 -0.703125 -1.53125 -1.078125q-0.84375 -0.390625 -2.015625 -0.390625q-1.6875 0 -2.828125 0.8125q-1.125 0.8125 -1.828125 2.140625q-0.6875 1.328125 -0.984375 2.890625q-0.296875 1.5625 -0.296875 3.0625q0 1.578125 0.328125 3.15625q0.34375 1.5625 1.0625 2.859375q0.734375 1.28125 1.859375 2.0625q1.125 0.765625 2.6875 0.765625q1.671875 0 2.8125 -0.9375q1.15625 -0.953125 2.0625 -2.421875l4.421875 1.65625q-0.84375 1.8125 -2.109375 3.25q-1.265625 1.4375 -3.046875 2.265625q-1.765625 0.8125 -4.140625 0.8125zm13.544952 -0.40625l0 -26.125l17.65625 0l0 4.59375l-12.8125 0l0 5.734375l7.59375 0l0 4.59375l-7.59375 0l0 6.578125l13.59375 0l0 4.625l-18.4375 0zm41.14052 0.453125q-2.4375 0 -4.4375 -0.859375q-2.0 -0.859375 -3.4375 -2.4375q-1.421875 -1.578125 -2.125 -3.71875l4.375 -1.625q0.84375 1.78125 2.34375 2.9375q1.515625 1.140625 3.390625 1.140625q2.0 0 3.1875 -0.828125q1.1875 -0.84375 1.1875 -2.453125q0 -1.09375 -0.703125 -1.828125q-0.703125 -0.75 -1.890625 -1.28125q-1.1875 -0.53125 -2.671875 -1.03125q-1.4375 -0.484375 -2.90625 -1.078125q-1.453125 -0.609375 -2.671875 -1.46875q-1.203125 -0.875 -1.9375 -2.1875q-0.734375 -1.328125 -0.734375 -3.25q0 -1.921875 1.015625 -3.53125q1.03125 -1.609375 2.984375 -2.5625q1.953125 -0.96875 4.734375 -0.96875q2.28125 0 4.03125 0.78125q1.765625 0.765625 2.921875 2.09375q1.15625 1.328125 1.609375 3.0l-4.328125 1.4375q-0.515625 -1.171875 -1.59375 -1.96875q-1.078125 -0.796875 -2.90625 -0.796875q-1.71875 0 -2.671875 0.671875q-0.953125 0.671875 -0.953125 1.90625q0 0.84375 0.640625 1.4375q0.65625 0.578125 1.75 1.046875q1.09375 0.46875 2.46875 0.953125q1.546875 0.53125 3.078125 1.1875q1.546875 0.65625 2.796875 1.609375q1.265625 0.953125 2.03125 2.375q0.765625 1.40625 0.765625 3.4375q0 2.5 -1.234375 4.25q-1.234375 1.75 -3.34375 2.6875q-2.109375 0.921875 -4.765625 0.921875zm14.385254 -0.453125l0 -26.125l17.65625 0l0 4.59375l-12.8125 0l0 5.734375l7.59375 0l0 4.59375l-7.59375 0l0 6.578125l13.59375 0l0 4.625l-18.4375 0zm29.067566 0l0 -21.484375l-7.4062347 0l0 -4.640625l19.687485 0l0 4.640625l-7.453125 0l0 21.484375l-4.828125 0z" fill-rule="nonzero"/>
+    <path fill="#cccccc" fill-opacity="0.7" d="m337.3832 126.591866l0 0c0 -26.092194 21.151886 -47.244095 47.24408 -47.244095l0 0c12.529907 0 24.54663 4.9774857 33.406616 13.837471c8.859985 8.859985 13.837494 20.876709 13.837494 33.406624l0 0c0 26.092186 -21.151917 47.244087 -47.24411 47.244087l0 0c-26.092194 0 -47.24408 -21.151901 -47.24408 -47.244087z" fill-rule="evenodd"/>
+    <path stroke="#9e9e9e" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m337.3832 126.591866l0 0c0 -26.092194 21.151886 -47.244095 47.24408 -47.244095l0 0c12.529907 0 24.54663 4.9774857 33.406616 13.837471c8.859985 8.859985 13.837494 20.876709 13.837494 33.406624l0 0c0 26.092186 -21.151917 47.244087 -47.24411 47.244087l0 0c-26.092194 0 -47.24408 -21.151901 -47.24408 -47.244087z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m355.56253 131.84373q-1.03125 0 -1.796875 -0.46875q-0.75 -0.484375 -1.15625 -1.359375q-0.40625 -0.875 -0.40625 -2.078125q0 -1.21875 0.40625 -2.09375q0.421875 -0.875 1.171875 -1.359375q0.765625 -0.484375 1.8125 -0.484375q1.15625 0 1.921875 0.546875q0.78125 0.53125 1.125 1.59375l-1.734375 0.578125q-0.171875 -0.46875 -0.515625 -0.71875q-0.328125 -0.25 -0.78125 -0.25q-0.703125 0 -1.125 0.59375q-0.421875 0.578125 -0.421875 1.59375q0 1.0 0.40625 1.59375q0.40625 0.578125 1.109375 0.578125q0.484375 0 0.828125 -0.28125q0.359375 -0.28125 0.53125 -0.8125l1.71875 0.515625q-0.328125 1.15625 -1.109375 1.734375q-0.78125 0.578125 -1.984375 0.578125zm7.112915 0q-1.234375 0 -1.953125 -0.65625q-0.703125 -0.671875 -0.703125 -1.796875q0 -0.734375 0.359375 -1.28125q0.375 -0.5625 1.03125 -0.859375q0.671875 -0.3125 1.5625 -0.3125q0.484375 0 0.9375 0.09375q0.46875 0.09375 0.8125 0.265625l0 -0.46875q0 -0.640625 -0.359375 -0.953125q-0.34375 -0.328125 -1.03125 -0.328125q-0.5625 0 -1.140625 0.203125q-0.578125 0.203125 -1.171875 0.59375l-0.546875 -1.453125q0.640625 -0.421875 1.4375 -0.65625q0.796875 -0.234375 1.59375 -0.234375q1.515625 0 2.265625 0.71875q0.75 0.71875 0.75 2.125l0 4.828125l-1.796875 0l0 -0.59375q-0.375 0.375 -0.90625 0.578125q-0.515625 0.1875 -1.140625 0.1875zm0.421875 -1.546875q0.453125 0 0.875 -0.171875q0.421875 -0.1875 0.75 -0.515625l0 -0.890625q-0.3125 -0.140625 -0.703125 -0.21875q-0.375 -0.09375 -0.765625 -0.09375q-0.671875 0 -1.0625 0.265625q-0.390625 0.25 -0.390625 0.71875q0 0.421875 0.34375 0.671875q0.34375 0.234375 0.953125 0.234375zm5.6108093 1.375l0 -7.484375l1.828125 0l0 0.71875q0.359375 -0.453125 0.859375 -0.671875q0.515625 -0.234375 1.15625 -0.234375q1.25 0 1.921875 0.78125q0.6875 0.765625 0.6875 2.21875l0 4.671875l-1.8125 0l0 -4.109375q0 -0.9375 -0.328125 -1.375q-0.328125 -0.4375 -1.015625 -0.4375q-0.75 0 -1.109375 0.4375q-0.359375 0.4375 -0.359375 1.34375l0 4.140625l-1.828125 0zm11.718475 0.171875q-1.03125 0 -1.796875 -0.46875q-0.75 -0.484375 -1.15625 -1.359375q-0.40625 -0.875 -0.40625 -2.078125q0 -1.21875 0.40625 -2.09375q0.421875 -0.875 1.171875 -1.359375q0.765625 -0.484375 1.8125 -0.484375q1.15625 0 1.921875 0.546875q0.78125 0.53125 1.125 1.59375l-1.734375 0.578125q-0.171875 -0.46875 -0.515625 -0.71875q-0.328125 -0.25 -0.78125 -0.25q-0.703125 0 -1.125 0.59375q-0.421875 0.578125 -0.421875 1.59375q0 1.0 0.40625 1.59375q0.40625 0.578125 1.109375 0.578125q0.484375 0 0.828125 -0.28125q0.359375 -0.28125 0.53125 -0.8125l1.71875 0.515625q-0.328125 1.15625 -1.109375 1.734375q-0.78125 0.578125 -1.984375 0.578125zm8.031952 0q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm6.8196716 4.515625l0 -9.75l1.828125 -0.828125l0 10.578125l-1.828125 0zm4.264923 0l0 -9.75l1.828125 -0.828125l0 10.578125l-1.828125 0zm7.280548 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25z" fill-rule="nonzero"/>
+    <path fill="#595959" d="m809.48364 140.20383l74.71503 0l0 0c9.787842 0 17.722473 11.156784 17.722473 24.919373c0 13.7625885 -7.9346313 24.919357 -17.722473 24.919357l-74.71503 0l0 0c-9.787842 0 -17.722473 -11.156769 -17.722473 -24.919357c0 -13.7625885 7.9346313 -24.919373 17.722473 -24.919373z" fill-rule="evenodd"/>
+    <path stroke="#ffffff" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m809.48364 140.20383l74.71503 0l0 0c9.787842 0 17.722473 11.156784 17.722473 24.919373c0 13.7625885 -7.9346313 24.919357 -17.722473 24.919357l-74.71503 0l0 0c-9.787842 0 -17.722473 -11.156769 -17.722473 -24.919357c0 -13.7625885 7.9346313 -24.919373 17.722473 -24.919373z" fill-rule="evenodd"/>
+    <path fill="#efefef" d="m836.03644 172.26195q-1.359375 0 -2.359375 -0.609375q-0.984375 -0.609375 -1.53125 -1.734375q-0.53125 -1.125 -0.53125 -2.65625q0 -1.53125 0.546875 -2.640625q0.546875 -1.125 1.53125 -1.71875q1.0 -0.609375 2.34375 -0.609375q1.15625 0 2.09375 0.484375q0.953125 0.484375 1.515625 1.484375q0.5625 1.0 0.5625 2.546875q0 0.25 -0.015625 0.59375q-0.015625 0.34375 -0.046875 0.703125l-6.234375 0q0.046875 0.65625 0.3125 1.15625q0.265625 0.484375 0.71875 0.75q0.46875 0.265625 1.109375 0.265625q0.6875 0 1.1875 -0.21875q0.5 -0.234375 0.953125 -0.734375l1.34375 1.390625q-0.625 0.6875 -1.5 1.125q-0.859375 0.421875 -2.0 0.421875zm-2.109375 -5.96875l4.015625 0q-0.046875 -0.625 -0.28125 -1.078125q-0.234375 -0.453125 -0.65625 -0.6875q-0.40625 -0.25 -1.015625 -0.25q-0.8125 0 -1.390625 0.5q-0.578125 0.5 -0.671875 1.515625zm8.644409 5.75l0 -9.53125l2.328125 0l0 0.90625q0.46875 -0.5625 1.09375 -0.84375q0.640625 -0.28125 1.46875 -0.28125q1.59375 0 2.453125 0.984375q0.875 0.984375 0.875 2.828125l0 5.9375l-2.328125 0l0 -5.21875q0 -1.203125 -0.421875 -1.75q-0.40625 -0.5625 -1.28125 -0.5625q-0.953125 0 -1.40625 0.5625q-0.453125 0.546875 -0.453125 1.703125l0 5.265625l-2.328125 0zm14.559692 0.21875q-1.828125 0 -2.875 -1.3125q-1.046875 -1.328125 -1.046875 -3.6875q0 -1.484375 0.484375 -2.59375q0.5 -1.125 1.390625 -1.75q0.890625 -0.625 2.078125 -0.625q1.265625 0 2.078125 0.921875l0 -3.59375l2.328125 -1.046875l0 13.46875l-2.328125 0l0 -0.78125q-0.6875 1.0 -2.109375 1.0zm0.390625 -2.21875q0.59375 0 1.015625 -0.203125q0.421875 -0.21875 0.703125 -0.640625l0 -3.859375q-0.28125 -0.390625 -0.75 -0.609375q-0.453125 -0.21875 -0.96875 -0.21875q-0.953125 0 -1.46875 0.71875q-0.5 0.703125 -0.5 2.03125q0 0.875 0.234375 1.5q0.234375 0.625 0.671875 0.953125q0.4375 0.328125 1.0625 0.328125z" fill-rule="nonzero"/>
+    <path fill="#ffffff" d="m51.07563 429.20023l75.1072 0l0 0c9.839249 0 17.815536 11.253235 17.815536 25.134796c0 13.881561 -7.976288 25.134796 -17.815536 25.134796l-75.1072 0l0 0c-9.839249 0 -17.815533 -11.253235 -17.815533 -25.134796c0 -13.881561 7.976284 -25.134796 17.815533 -25.134796z" fill-rule="evenodd"/>
+    <path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m51.07563 429.20023l75.1072 0l0 0c9.839249 0 17.815536 11.253235 17.815536 25.134796c0 13.881561 -7.976288 25.134796 -17.815536 25.134796l-75.1072 0l0 0c-9.839249 0 -17.815533 -11.253235 -17.815533 -25.134796c0 -13.881561 7.976284 -25.134796 17.815533 -25.134796z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m56.739033 461.255l0 -9.53125l2.328125 0l0 0.90625q0.46875 -0.5625 1.09375 -0.84375q0.640625 -0.28125 1.46875 -0.28125q1.59375 0 2.4531288 0.984375q0.875 0.984375 0.875 2.828125l0 5.9375l-2.3281288 0l0 -5.21875q0 -1.203125 -0.421875 -1.75q-0.40625 -0.5625 -1.28125 -0.5625q-0.953125 0 -1.40625 0.5625q-0.453125 0.546875 -0.453125 1.703125l0 5.265625l-2.328125 0zm15.059727 0.21875q-1.359375 0 -2.359375 -0.609375q-0.984375 -0.609375 -1.53125 -1.734375q-0.53125 -1.125 -0.53125 -2.65625q0 -1.53125 0.546875 -2.640625q0.546875 -1.125 1.53125 -1.71875q1.0 -0.609375 2.34375 -0.609375q1.15625 0 2.09375 0.484375q0.953125 0.484375 1.515625 1.484375q0.5625 1.0 0.5625 2.546875q0 0.25 -0.015625 0.59375q-0.015625 0.34375 -0.046875 0.703125l-6.234375 0q0.046875 0.65625 0.3125 1.15625q0.265625 0.484375 0.71875 0.75q0.46875 0.265625 1.109375 0.265625q0.6875 0 1.1875 -0.21875q0.5 -0.234375 0.953125 -0.734375l1.34375 1.390625q-0.625 0.6875 -1.5 1.125q-0.859375 0.421875 -2.0 0.421875zm-2.109375 -5.96875l4.015625 0q-0.046875 -0.625 -0.28125 -1.078125q-0.234375 -0.453125 -0.65625 -0.6875q-0.40625 -0.25 -1.015625 -0.25q-0.8125 0 -1.390625 0.5q-0.578125 0.5 -0.671875 1.515625zm10.3294525 5.75l-2.625 -9.53125l2.296875 0l1.15625 4.765625q0.0625 0.25 0.125 0.609375q0.078125 0.359375 0.125 0.671875q0.0625 -0.296875 0.140625 -0.65625q0.09375 -0.375 0.15625 -0.625l1.28125 -4.765625l2.03125 0l1.265625 4.765625q0.0625 0.265625 0.15625 0.625q0.09375 0.359375 0.140625 0.65625q0.0625 -0.296875 0.125 -0.65625q0.0625 -0.359375 0.125 -0.625l1.1875 -4.765625l2.296875 0l-2.65625 9.53125l-2.0625 0l-1.25 -4.765625q-0.0625 -0.265625 -0.171875 -0.6875q-0.09375 -0.4375 -0.171875 -0.765625q-0.0625 0.328125 -0.171875 0.765625q-0.09375 0.421875 -0.171875 0.6875l-1.25 4.765625l-2.078125 0zm19.900192 0.21875q-1.125 0 -2.15625 -0.46875q-1.015625 -0.484375 -1.609375 -1.28125l1.53125 -1.28125q0.46875 0.5 1.09375 0.796875q0.625 0.296875 1.203125 0.296875q0.703125 0 1.078125 -0.234375q0.390625 -0.25 0.390625 -0.671875q0 -0.28125 -0.1875 -0.5q-0.171875 -0.21875 -0.640625 -0.453125q-0.453125 -0.234375 -1.28125 -0.53125q-1.609375 -0.578125 -2.28125 -1.265625q-0.671875 -0.6875 -0.671875 -1.75q0 -1.171875 0.953125 -1.890625q0.953125 -0.734375 2.484375 -0.734375q1.046875 0 1.921875 0.390625q0.890625 0.390625 1.515625 1.140625l-1.515625 1.265625q-0.84375 -0.859375 -1.953125 -0.859375q-0.578125 0 -0.921875 0.21875q-0.34375 0.203125 -0.34375 0.5625q0 0.34375 0.40625 0.625q0.40625 0.265625 1.515625 0.625q1.109375 0.34375 1.78125 0.78125q0.6875 0.4375 1.015625 1.015625q0.328125 0.578125 0.328125 1.328125q0 1.34375 -0.984375 2.109375q-0.96875 0.765625 -2.671875 0.765625zm9.807739 0q-1.359375 0 -2.359375 -0.609375q-0.984375 -0.609375 -1.53125 -1.734375q-0.53125 -1.125 -0.53125 -2.65625q0 -1.53125 0.546875 -2.640625q0.546875 -1.125 1.53125 -1.71875q1.0 -0.609375 2.34375 -0.609375q1.15625 0 2.09375 0.484375q0.953125 0.484375 1.515625 1.484375q0.5625 1.0 0.5625 2.546875q0 0.25 -0.015625 0.59375q-0.015625 0.34375 -0.046875 0.703125l-6.234375 0q0.046875 0.65625 0.3125 1.15625q0.265625 0.484375 0.71875 0.75q0.46875 0.265625 1.109375 0.265625q0.6875 0 1.1875 -0.21875q0.5 -0.234375 0.953125 -0.734375l1.34375 1.390625q-0.625 0.6875 -1.5 1.125q-0.859375 0.421875 -2.0 0.421875zm-2.109375 -5.96875l4.015625 0q-0.046875 -0.625 -0.28125 -1.078125q-0.234375 -0.453125 -0.65625 -0.6875q-0.40625 -0.25 -1.015625 -0.25q-0.8125 0 -1.390625 0.5q-0.578125 0.5 -0.671875 1.515625zm11.2669525 5.96875q-1.140625 0 -1.71875 -0.625q-0.5625 -0.625 -0.5625 -1.859375l0 -5.21875l-1.390625 0l0 -2.046875l1.390625 0l0 -2.828125l2.3125 -1.109375l0 3.9375l2.171875 0l0 2.046875l-2.171875 0l0 4.796875q0 0.453125 0.1875 0.671875q0.1875 0.203125 0.578125 0.203125q0.6875 0 1.5 -0.421875l-0.265625 2.03125q-0.421875 0.21875 -0.9375 0.3125q-0.5 0.109375 -1.09375 0.109375z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m229.23839 322.5333c0 -12.5000305 -8.35434 -26.728394 -16.708664 -25.000061c-8.35434 1.7283325 -16.708664 19.413422 -16.708664 38.826813" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m229.23837 322.5333c0 -12.5000305 -8.354324 -26.728394 -16.708664 -25.000061c-4.1771545 0.86416626 -8.354324 5.7175293 -11.487198 12.781525c-1.5664368 3.5319824 -2.871811 7.616638 -3.785553 12.031616c-0.45687866 2.207489 -0.8158417 4.497589 -1.0606232 6.8424683c-0.030593872 0.29309082 -0.059402466 0.58706665 -0.086380005 0.88183594l-0.025146484 0.29516602" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m194.43468 330.29324l1.4506378 4.6063232l1.8496246 -4.46109z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m229.23839 417.02148c0 12.5 8.354324 26.728394 16.708649 25.00003c8.35434 -1.7283325 16.708664 -19.413391 16.708664 -38.826813" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m229.23837 417.02148c0 12.5 8.354324 26.728394 16.708664 25.000061c4.17717 -0.8641968 8.354324 -5.7175293 11.487198 -12.781555c1.5664368 -3.5319824 2.8717957 -7.616638 3.785553 -12.031616c0.45687866 -2.2074585 0.81585693 -4.497589 1.0606079 -6.8424377c0.03060913 -0.29309082 0.059417725 -0.58706665 0.08639526 -0.88183594l0.025146484 -0.29516602" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m264.04208 409.26154l-1.4506531 -4.6063232l-1.8496399 4.46109z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m262.645 336.37076c0 -24.740173 37.370087 -49.480316 74.74017 -49.480316" fill-rule="evenodd"/>
+    <path stroke="#0097a7" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="12.0,9.0" d="m262.645 336.37076c0 -12.370087 9.342529 -24.740173 23.356293 -34.01773c7.006897 -4.6387634 15.18161 -8.504425 23.940216 -11.210358c2.1896667 -0.6765137 4.415802 -1.2805176 6.6693115 -1.8059998c0.563385 -0.13137817 1.128479 -0.2578125 1.6951294 -0.37930298c0.2833557 -0.06072998 0.5670471 -0.12020874 0.8511658 -0.17840576l0.32073975 -0.0642395" fill-rule="evenodd"/>
+    <path fill="#0097a7" stroke="#0097a7" stroke-width="3.0" stroke-linecap="butt" d="m319.98004 293.6444l13.041992 -6.3094788l-14.046417 -3.5498962z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m351.22067 253.47395c0 0.993927 -30.496063 -6.645996 -60.992126 1.9878693c-30.496063 8.63385 -60.992126 33.541504 -60.992126 67.08301" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m351.22067 253.47395c0 0.99394226 -30.496063 -6.645996 -60.992126 1.9878693c-15.248016 4.316925 -30.496063 12.702301 -41.932083 24.201141c-5.7180023 5.7493896 -10.483017 12.277191 -13.818527 19.463959c-1.6677551 3.5933838 -2.9781342 7.3515015 -3.871582 11.259491c-0.44670105 1.953949 -0.7891998 3.945404 -1.0200043 5.9724426l-0.020812988 0.19500732" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m227.91629 316.46326l1.4003296 4.6218567l1.8981628 -4.4406433z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m384.6273 334.12466c0 12.5000305 8.35434 26.728394 16.70868 25.000061c8.354309 -1.728363 16.708649 -19.413422 16.708649 -38.826813" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m384.6273 334.12466c0 12.500061 8.35434 26.728394 16.70868 25.000061c4.1771545 -0.8641968 8.354309 -5.7175293 11.487183 -12.781494c1.5664368 -3.532013 2.8718262 -7.6166687 3.785553 -12.031647c0.45690918 -2.207489 0.81588745 -4.497589 1.0606384 -6.8424683c0.030578613 -0.29309082 0.059387207 -0.58706665 0.08639526 -0.88183594l0.025115967 -0.29519653" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m419.431 326.36472l-1.4506531 -4.6063232l-1.8496094 4.46109z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m261.0958 233.7454l80.50394 0l0 16.157486l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m280.8013 246.54541l0 -6.8125l1.15625 0l0 6.8125l-1.15625 0zm0.5625 -8.046875q-0.296875 0 -0.515625 -0.203125q-0.203125 -0.21875 -0.203125 -0.515625q0 -0.3125 0.203125 -0.53125q0.21875 -0.21875 0.515625 -0.21875q0.3125 0 0.515625 0.21875q0.21875 0.21875 0.21875 0.53125q0 0.296875 -0.21875 0.515625q-0.203125 0.203125 -0.515625 0.203125zm2.80719 8.046875l0 -6.8125l1.15625 0l0 0.6875q0.3125 -0.421875 0.75 -0.625q0.4375 -0.21875 1.015625 -0.21875q1.234375 0 1.828125 0.75q0.609375 0.75 0.609375 2.296875l0 3.921875l-1.171875 0l0 -4.125q0 -0.859375 -0.390625 -1.3125q-0.375 -0.46875 -1.0625 -0.46875q-0.78125 0 -1.1875 0.453125q-0.390625 0.4375 -0.390625 1.3125l0 4.140625l-1.15625 0zm10.12558 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.835724 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.5528564 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m48.37664 371.37927l80.503944 0l0 16.15747l-80.503944 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m68.08214 384.17926l0 -6.8125l1.15625 0l0 6.8125l-1.15625 0zm0.5625 -8.046875q-0.296875 0 -0.515625 -0.203125q-0.203125 -0.21875 -0.203125 -0.515625q0 -0.3125 0.203125 -0.53125q0.21875 -0.21875 0.515625 -0.21875q0.3125 0 0.515625 0.21875q0.21875 0.21875 0.21875 0.53125q0 0.296875 -0.21875 0.515625q-0.203125 0.203125 -0.515625 0.203125zm2.8072052 8.046875l0 -6.8125l1.15625 0l0 0.6875q0.3125 -0.421875 0.75 -0.625q0.4375 -0.21875 1.015625 -0.21875q1.234375 0 1.828125 0.75q0.609375 0.75 0.609375 2.296875l0 3.921875l-1.171875 0l0 -4.125q0 -0.859375 -0.390625 -1.3125q-0.375 -0.46875 -1.0625 -0.46875q-0.78125 0 -1.1875 0.453125q-0.390625 0.4375 -0.390625 1.3125l0 4.140625l-1.15625 0zm10.12558 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8357086 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.5528717 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m163.06955 426.0118l80.50394 0l0 16.15747l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m184.29839 438.96805q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125zm5.395508 3.9375l2.28125 -3.546875l-2.09375 -3.265625l1.28125 0l1.078125 1.65625q0.125 0.171875 0.21875 0.34375q0.09375 0.15625 0.171875 0.3125q0.09375 -0.171875 0.1875 -0.328125q0.109375 -0.15625 0.21875 -0.328125l1.0625 -1.65625l1.296875 0l-2.109375 3.265625l2.296875 3.546875l-1.3125 0l-1.25 -1.9375q-0.109375 -0.171875 -0.203125 -0.328125q-0.09375 -0.15625 -0.1875 -0.3125q-0.078125 0.15625 -0.171875 0.3125q-0.09375 0.15625 -0.21875 0.328125l-1.25 1.9375l-1.296875 0zm9.521866 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8357086 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.5528717 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m262.645 403.184c0 -1.1741943 30.496063 6.2854614 60.992126 -2.3483887c30.496063 -8.633881 60.992126 -33.361237 60.992126 -66.72247" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m262.645 403.18402c0 -1.1742249 30.496063 6.285431 60.992126 -2.3484192c15.248047 -4.3169556 30.496094 -12.657257 41.9321 -24.08847c5.7180176 -5.715637 10.483032 -12.203949 13.8185425 -19.34848c1.6677246 -3.572235 2.9781494 -7.308563 3.8715515 -11.194336c0.4467163 -1.9428711 0.7892151 -3.9231567 1.0200195 -5.938965l0.017333984 -0.16143799" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m385.94586 340.19547l-1.397644 -4.6226807l-1.9007568 4.4395447z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m284.96588 367.12598l80.50394 0l0 16.157501l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m306.1947 380.08224q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125zm5.395508 3.9375l2.28125 -3.546875l-2.09375 -3.265625l1.28125 0l1.078125 1.65625q0.125 0.171875 0.21875 0.34375q0.09375 0.15625 0.171875 0.3125q0.09375 -0.171875 0.1875 -0.328125q0.109375 -0.15625 0.21875 -0.328125l1.0625 -1.65625l1.296875 0l-2.109375 3.265625l2.296875 3.546875l-1.3125 0l-1.25 -1.9375q-0.109375 -0.171875 -0.203125 -0.328125q-0.09375 -0.15625 -0.1875 -0.3125q-0.078125 0.15625 -0.171875 0.3125q-0.09375 0.15625 -0.21875 0.328125l-1.25 1.9375l-1.296875 0zm9.521881 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8356934 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.552887 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m262.645 403.184c0 5.548828 34.504547 12.434479 58.275787 11.097687c23.77124 -1.336792 36.809143 -10.896027 58.275787 -21.123657c21.466644 -10.2276 51.362 -21.123627 102.72403 -21.123627" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m262.645 403.18402c0 5.5487976 34.504547 12.434448 58.275787 11.097626c23.77124 -1.3367615 36.809143 -10.895996 58.275787 -21.123627c10.733307 -5.1138 23.573822 -10.394745 40.151245 -14.397156c8.288727 -2.0012207 17.511658 -3.6828613 27.872528 -4.8641663c5.1804504 -0.5906372 10.645386 -1.0562439 16.420288 -1.3741455c2.8874207 -0.15896606 5.852356 -0.28094482 8.89798 -0.36328125c0.76138306 -0.020568848 1.527832 -0.03866577 2.2993774 -0.054229736l1.0828247 -0.019989014" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m475.93475 373.73672l4.523987 -1.6899719l-4.5518494 -1.6133728z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m404.05643 331.7047l80.50394 0l0 16.157501l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m425.28525 344.66098q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125zm5.395508 3.9375l2.28125 -3.546875l-2.09375 -3.265625l1.28125 0l1.078125 1.65625q0.125 0.171875 0.21875 0.34375q0.09375 0.15625 0.171875 0.3125q0.09375 -0.171875 0.1875 -0.328125q0.109375 -0.15625 0.21875 -0.328125l1.0625 -1.65625l1.296875 0l-2.109375 3.265625l2.296875 3.546875l-1.3125 0l-1.25 -1.9375q-0.109375 -0.171875 -0.203125 -0.328125q-0.09375 -0.15625 -0.1875 -0.3125q-0.078125 0.15625 -0.171875 0.3125q-0.09375 0.15625 -0.21875 0.328125l-1.25 1.9375l-1.296875 0zm9.521881 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8356934 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.552887 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m229.23839 322.5333c0 -97.96852 54.07872 -195.93701 108.157486 -195.93701" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m229.23837 322.5333c0 -48.984253 13.519684 -97.96852 33.799225 -134.70671c10.13974 -18.369095 21.969482 -33.676666 34.644196 -44.391968c6.3373413 -5.357666 12.885925 -9.5672455 19.540161 -12.437408c3.3270874 -1.4351044 6.6806335 -2.535324 10.047333 -3.2767868c0.8416748 -0.18536377 1.6841736 -0.34830475 2.5273132 -0.48845673c0.42156982 -0.07006836 0.84329224 -0.13443756 1.2651062 -0.19306946l0.34725952 -0.04588318" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m331.5182 128.64113l4.418976 -1.9481888l-4.63739 -1.3480453z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m229.23885 126.59449l80.50392 0l0 16.157478l-80.50392 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m250.46767 139.55074q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125zm5.395508 3.9375l2.2812653 -3.546875l-2.0937653 -3.265625l1.28125 0l1.0781403 1.65625q0.125 0.171875 0.21875 0.34375q0.09375 0.15625 0.171875 0.3125q0.09375 -0.171875 0.1875 -0.328125q0.109375 -0.15625 0.21875 -0.328125l1.0625 -1.65625l1.296875 0l-2.109375 3.265625l2.296875 3.546875l-1.3125 0l-1.25 -1.9375q-0.109375 -0.171875 -0.203125 -0.328125q-0.09375 -0.15625 -0.1875 -0.3125q-0.078125 0.15625 -0.171875 0.3125q-0.09375 0.15625 -0.21875 0.328125l-1.2500153 1.9375l-1.296875 0zm9.521866 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.835724 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.5528564 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m410.13123 388.84515l80.50394 0l0 32.31494l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m428.63525 403.37952q-0.203125 -0.375 -0.390625 -0.953125q-0.171875 -0.578125 -0.28125 -1.28125q-0.109375 -0.71875 -0.109375 -1.515625q0 -1.40625 0.25 -2.546875q0.265625 -1.15625 0.6875 -2.078125q0.421875 -0.921875 0.9375 -1.625q0.515625 -0.703125 1.015625 -1.21875l1.078125 0q-0.203125 0.21875 -0.5625 0.65625q-0.359375 0.4375 -0.75 1.09375q-0.390625 0.65625 -0.75 1.5q-0.359375 0.84375 -0.59375 1.875q-0.21875 1.015625 -0.21875 2.203125q0 0.9375 0.140625 1.75q0.140625 0.796875 0.3125 1.34375q0.1875 0.5625 0.296875 0.796875l-1.0625 0zm6.770508 0.9375l0.59375 -3.3125q-0.4375 0.40625 -0.875 0.609375q-0.4375 0.1875 -0.9375 0.1875q-0.6875 0 -1.1875 -0.328125q-0.484375 -0.328125 -0.75 -0.9375q-0.265625 -0.625 -0.265625 -1.46875q0 -1.28125 0.4375 -2.265625q0.4375 -1.0 1.1875 -1.5625q0.765625 -0.5625 1.75 -0.5625q0.53125 0 0.953125 0.203125q0.4375 0.203125 0.671875 0.5625l0.09375 -0.609375l1.125 0l-1.578125 9.03125l-1.21875 0.453125zm-0.90625 -3.5625q0.421875 0 0.859375 -0.234375q0.4375 -0.25 0.84375 -0.71875l0.546875 -3.15625q-0.140625 -0.4375 -0.5 -0.6875q-0.359375 -0.25 -0.859375 -0.25q-0.625 0 -1.140625 0.4375q-0.515625 0.421875 -0.8125 1.171875q-0.296875 0.734375 -0.296875 1.65625q0 0.859375 0.34375 1.328125q0.359375 0.453125 1.015625 0.453125zm6.94693 1.046875q-0.765625 0 -1.25 -0.34375q-0.46875 -0.34375 -0.640625 -1.015625q-0.15625 -0.671875 0.015625 -1.640625l0.6875 -3.96875l1.125 0l-0.71875 4.046875q-0.15625 0.921875 0.109375 1.40625q0.28125 0.46875 0.96875 0.46875q0.765625 0 1.234375 -0.453125q0.484375 -0.453125 0.640625 -1.328125l0.71875 -4.140625l1.125 0l-1.1875 6.8125l-1.140625 0l0.125 -0.65625q-0.328125 0.40625 -0.78125 0.609375q-0.453125 0.203125 -1.03125 0.203125zm4.9536133 -0.15625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm3.8880615 8.21875q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm7.4560547 4.09375q-0.5 0 -0.953125 -0.140625q-0.4375 -0.140625 -0.796875 -0.40625q-0.359375 -0.28125 -0.59375 -0.65625l0.84375 -0.65625q0.28125 0.40625 0.6875 0.640625q0.421875 0.234375 0.859375 0.234375q0.609375 0 0.96875 -0.28125q0.359375 -0.28125 0.359375 -0.75q0 -0.328125 -0.28125 -0.59375q-0.265625 -0.265625 -0.984375 -0.59375q-0.9375 -0.4375 -1.328125 -0.890625q-0.390625 -0.46875 -0.390625 -1.09375q0 -0.578125 0.296875 -1.015625q0.3125 -0.4375 0.84375 -0.671875q0.53125 -0.25 1.234375 -0.25q0.71875 0 1.296875 0.28125q0.578125 0.265625 0.890625 0.765625l-0.8125 0.625q-0.578125 -0.703125 -1.453125 -0.703125q-0.53125 0 -0.875 0.265625q-0.34375 0.25 -0.34375 0.640625q0 0.296875 0.25 0.546875q0.25 0.234375 0.921875 0.515625q1.015625 0.453125 1.4375 0.953125q0.421875 0.5 0.421875 1.203125q0 0.921875 -0.6875 1.484375q-0.671875 0.546875 -1.8125 0.546875zm6.1588745 0q-1.046875 0 -1.6875 -0.765625q-0.625 -0.765625 -0.625 -2.0625q0 -1.28125 0.421875 -2.25q0.421875 -0.96875 1.171875 -1.5q0.75 -0.546875 1.75 -0.546875q0.8125 0 1.390625 0.421875q0.578125 0.40625 0.75 1.15625l-1.078125 0.375q-0.09375 -0.4375 -0.40625 -0.671875q-0.296875 -0.25 -0.78125 -0.25q-0.609375 0 -1.078125 0.421875q-0.46875 0.40625 -0.734375 1.140625q-0.265625 0.71875 -0.265625 1.671875q0 0.859375 0.328125 1.34375q0.328125 0.46875 0.9375 0.46875q0.515625 0 0.90625 -0.25q0.40625 -0.265625 0.640625 -0.78125l0.953125 0.375q-0.359375 0.828125 -1.046875 1.265625q-0.671875 0.4375 -1.546875 0.4375zm6.544983 0q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375z" fill-rule="nonzero"/>
+    <path fill="#d9d9d9" d="m428.54086 417.8014q-0.6875 0 -0.984375 -0.46875q-0.28125 -0.46875 -0.125 -1.359375l0.71875 -4.125l-1.109375 0l0.1875 -1.015625l1.09375 0l0.421875 -2.375l1.203125 -0.4375l-0.5 2.8125l1.609375 0l-0.171875 1.015625l-1.609375 0l-0.6875 3.875q-0.109375 0.609375 -0.015625 0.828125q0.109375 0.21875 0.5 0.21875q0.234375 0 0.46875 -0.078125q0.25 -0.078125 0.5625 -0.265625l-0.296875 1.09375q-0.515625 0.28125 -1.265625 0.28125zm2.748352 -0.15625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm1.2943115 8.0625l1.1875 -6.8125l1.109375 0l-0.109375 0.65625q0.359375 -0.375 0.828125 -0.59375q0.484375 -0.21875 0.984375 -0.21875q0.5625 0 0.96875 0.265625q0.421875 0.265625 0.609375 0.765625q0.4375 -0.515625 0.96875 -0.765625q0.53125 -0.265625 1.171875 -0.265625q0.671875 0 1.140625 0.328125q0.46875 0.3125 0.65625 0.875q0.203125 0.5625 0.078125 1.296875l-0.796875 4.46875l-1.140625 0l0.734375 -4.203125q0.140625 -0.8125 -0.125 -1.265625q-0.25 -0.46875 -0.859375 -0.46875q-0.609375 0 -1.09375 0.484375q-0.46875 0.484375 -0.59375 1.234375l-0.75 4.21875l-1.140625 0l0.734375 -4.203125q0.15625 -0.828125 -0.109375 -1.28125q-0.25 -0.453125 -0.890625 -0.453125q-0.609375 0 -1.09375 0.484375q-0.46875 0.46875 -0.59375 1.234375l-0.75 4.21875l-1.125 0zm13.556122 0.15625q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm8.173553 4.09375q-1.171875 0 -1.84375 -0.75q-0.65625 -0.75 -0.65625 -2.078125q0 -1.234375 0.453125 -2.203125q0.46875 -0.96875 1.25 -1.53125q0.796875 -0.5625 1.796875 -0.5625q1.125 0 1.796875 0.765625q0.671875 0.75 0.671875 2.03125q0 1.25 -0.453125 2.234375q-0.453125 0.96875 -1.25 1.53125q-0.78125 0.5625 -1.765625 0.5625zm0.03125 -1.046875q0.65625 0 1.171875 -0.421875q0.53125 -0.421875 0.828125 -1.140625q0.296875 -0.71875 0.296875 -1.640625q0 -0.859375 -0.375 -1.34375q-0.375 -0.5 -1.015625 -0.5q-0.65625 0 -1.1875 0.421875q-0.515625 0.421875 -0.8125 1.15625q-0.296875 0.71875 -0.296875 1.625q0 0.859375 0.359375 1.359375q0.375 0.484375 1.03125 0.484375zm6.7669983 1.046875q-0.765625 0 -1.25 -0.34375q-0.46875 -0.34375 -0.640625 -1.015625q-0.15625 -0.671875 0.015625 -1.640625l0.6875 -3.96875l1.125 0l-0.71875 4.046875q-0.15625 0.921875 0.109375 1.40625q0.28125 0.46875 0.96875 0.46875q0.765625 0 1.234375 -0.453125q0.484375 -0.453125 0.640625 -1.328125l0.71875 -4.140625l1.125 0l-1.1875 6.8125l-1.140625 0l0.125 -0.65625q-0.328125 0.40625 -0.78125 0.609375q-0.453125 0.203125 -1.03125 0.203125zm6.656708 0q-0.6875 0 -0.984375 -0.46875q-0.28125 -0.46875 -0.125 -1.359375l0.71875 -4.125l-1.109375 0l0.1875 -1.015625l1.09375 0l0.421875 -2.375l1.203125 -0.4375l-0.5 2.8125l1.609375 0l-0.171875 1.015625l-1.609375 0l-0.6875 3.875q-0.109375 0.609375 -0.015625 0.828125q0.109375 0.21875 0.5 0.21875q0.234375 0 0.46875 -0.078125q0.25 -0.078125 0.5625 -0.265625l-0.296875 1.09375q-0.515625 0.28125 -1.265625 0.28125zm2.4212646 1.578125q0.1875 -0.234375 0.515625 -0.65625q0.328125 -0.421875 0.703125 -1.0625q0.375 -0.625 0.71875 -1.4375q0.34375 -0.828125 0.5625 -1.828125q0.21875 -1.0 0.21875 -2.1875q0 -0.953125 -0.15625 -1.796875q-0.140625 -0.84375 -0.34375 -1.4375q-0.1875 -0.59375 -0.3125 -0.8125l1.046875 0q0.234375 0.375 0.421875 0.984375q0.203125 0.59375 0.328125 1.34375q0.125 0.734375 0.125 1.5625q0 1.375 -0.25 2.5q-0.234375 1.125 -0.640625 2.015625q-0.390625 0.890625 -0.890625 1.578125q-0.484375 0.703125 -0.984375 1.234375l-1.0625 0z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m384.6273 239.63649c0 -16.450134 0.01574707 -24.675873 0.03149414 -32.900284c0.01574707 -8.224396 0.03149414 -16.447495 0.03149414 -32.895004" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m384.6273 239.63649c0 -16.450134 0.01574707 -24.675858 0.03149414 -32.90027c0.007873535 -4.112213 0.01574707 -8.224075 0.02166748 -13.363846c0.002960205 -2.5698853 0.0054016113 -5.3967285 0.0071411133 -8.609085l0.0012207031 -4.9220886" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m386.34055 179.84161l-1.6506042 -4.538498l-1.6528625 4.537689z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m313.07217 178.51575l80.50394 0l0 16.157486l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m334.301 191.472q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125zm5.395508 3.9375l2.28125 -3.546875l-2.09375 -3.265625l1.28125 0l1.078125 1.65625q0.125 0.171875 0.21875 0.34375q0.09375 0.15625 0.171875 0.3125q0.09375 -0.171875 0.1875 -0.328125q0.109375 -0.15625 0.21875 -0.328125l1.0625 -1.65625l1.296875 0l-2.109375 3.265625l2.296875 3.546875l-1.3125 0l-1.25 -1.9375q-0.109375 -0.171875 -0.203125 -0.328125q-0.09375 -0.15625 -0.1875 -0.3125q-0.078125 0.15625 -0.171875 0.3125q-0.09375 0.15625 -0.21875 0.328125l-1.25 1.9375l-1.296875 0zm9.521881 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8356934 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.552887 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m418.0339 93.18524c0 -19.41864 107.20474 -44.71523 214.40945 -38.837276c107.20471 5.8779526 214.40948 42.930447 214.40948 85.8609" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m418.0339 93.18524c0 -19.41864 107.20474 -44.71523 214.40945 -38.837276c53.602417 2.9389763 107.20477 13.671589 147.40656 29.035763c20.100891 7.682083 36.851624 16.522064 48.577087 26.124672c5.862793 4.8013 10.469238 9.793274 13.609985 14.926483c1.5703735 2.5666046 2.774353 5.1685257 3.5856934 7.7995834c0.20288086 0.6577606 0.38116455 1.3173523 0.5345459 1.978653l0.007873535 0.035308838" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m844.52423 134.43776l2.1610718 4.3188477l1.1206055 -4.697525z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m576.3989 372.0315c135.2284 0 270.45673 -90.99213 270.45673 -181.98425" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m576.3989 372.0315c67.6142 0 135.2284 -22.748016 185.93903 -56.870087c25.355286 -17.061005 46.48474 -36.965546 61.27533 -58.29181c7.3952637 -10.663147 13.205872 -21.681732 17.167664 -32.87802c1.980896 -5.59816 3.4995728 -11.240738 4.5230103 -16.905518c0.51171875 -2.8324127 0.89959717 -5.670349 1.1596069 -8.511078c0.06506348 -0.7101898 0.12207031 -1.4205322 0.17095947 -2.1310272l0.025268555 -0.39990234" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m848.31067 196.09796l-1.5027466 -4.5895996l-1.7989502 4.481781z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m88.629234 429.20023c0 -29.716553 46.677162 -59.433075 93.35433 -59.433075" fill-rule="evenodd"/>
+    <path stroke="#0097a7" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m88.629234 429.20023c0 -14.858276 11.669289 -29.716553 29.173225 -40.86026c8.751968 -5.5718384 18.9626 -10.215057 29.902557 -13.4653015c2.7350006 -0.81256104 5.515564 -1.5380554 8.330322 -2.1692505c1.4073792 -0.31558228 2.8233032 -0.607605 4.2463684 -0.8751221c0.71151733 -0.13375854 1.42482 -0.26138306 2.1397247 -0.3828125c0.3574524 -0.060699463 0.7153015 -0.11984253 1.0735474 -0.1774292l0.5445099 -0.08502197" fill-rule="evenodd"/>
+    <path fill="#0097a7" stroke="#0097a7" stroke-width="3.0" stroke-linecap="butt" d="m164.42981 376.12482l13.181671 -6.012207l-13.962326 -3.8673706z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m268.0092 311.11417l80.50394 0l0 16.157501l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m279.61774 325.64856q-0.203125 -0.375 -0.390625 -1.0q-0.171875 -0.609375 -0.28125 -1.359375q-0.109375 -0.75 -0.109375 -1.53125q0 -1.328125 0.25 -2.453125q0.25 -1.125 0.65625 -2.03125q0.421875 -0.921875 0.921875 -1.625q0.5 -0.703125 0.984375 -1.21875l1.609375 0q-0.234375 0.25 -0.59375 0.703125q-0.359375 0.453125 -0.765625 1.109375q-0.390625 0.640625 -0.734375 1.484375q-0.34375 0.828125 -0.5625 1.828125q-0.21875 0.984375 -0.21875 2.140625q0 0.875 0.125 1.6875q0.140625 0.796875 0.3125 1.40625q0.171875 0.609375 0.296875 0.859375l-1.5 0zm5.4656067 -1.578125q-0.984375 0 -1.53125 -0.53125q-0.546875 -0.546875 -0.546875 -1.4375q0 -0.75 0.359375 -1.3125q0.375 -0.5625 1.03125 -0.875q0.671875 -0.3125 1.5625 -0.3125q0.421875 0 0.84375 0.109375q0.4375 0.09375 0.703125 0.265625l0.0625 -0.375q0.125 -0.609375 -0.125 -0.921875q-0.25 -0.328125 -0.84375 -0.328125q-0.5 0 -1.0625 0.1875q-0.5625 0.171875 -1.09375 0.5l-0.234375 -1.296875q0.625 -0.40625 1.28125 -0.59375q0.671875 -0.203125 1.34375 -0.203125q0.828125 0 1.390625 0.328125q0.5625 0.328125 0.8125 0.921875q0.25 0.59375 0.109375 1.375l-0.765625 4.34375l-1.578125 0l0.09375 -0.546875q-0.296875 0.3125 -0.796875 0.515625q-0.5 0.1875 -1.015625 0.1875zm0.421875 -1.421875q0.421875 0 0.84375 -0.15625q0.4375 -0.15625 0.796875 -0.453125l0.125 -0.796875q-0.25 -0.140625 -0.578125 -0.21875q-0.328125 -0.09375 -0.703125 -0.09375q-0.4375 0 -0.765625 0.125q-0.3125 0.125 -0.484375 0.34375q-0.171875 0.21875 -0.171875 0.515625q0 0.359375 0.25 0.546875q0.265625 0.1875 0.6875 0.1875zm4.599945 1.265625l1.578125 -9.078125l1.703125 -0.546875l-1.703125 9.625l-1.578125 0zm3.545288 0l1.578125 -9.078125l1.703125 -0.546875l-1.703125 9.625l-1.578125 0zm8.726074 0.15625q-0.984375 0 -1.53125 -0.53125q-0.546875 -0.546875 -0.546875 -1.4375q0 -0.75 0.359375 -1.3125q0.375 -0.5625 1.03125 -0.875q0.671875 -0.3125 1.5625 -0.3125q0.421875 0 0.84375 0.109375q0.4375 0.09375 0.703125 0.265625l0.0625 -0.375q0.125 -0.609375 -0.125 -0.921875q-0.25 -0.328125 -0.84375 -0.328125q-0.5 0 -1.0625 0.1875q-0.5625 0.171875 -1.09375 0.5l-0.234375 -1.296875q0.625 -0.40625 1.28125 -0.59375q0.671875 -0.203125 1.34375 -0.203125q0.828125 0 1.390625 0.328125q0.5625 0.328125 0.8125 0.921875q0.25 0.59375 0.109375 1.375l-0.765625 4.34375l-1.578125 0l0.09375 -0.546875q-0.296875 0.3125 -0.796875 0.515625q-0.5 0.1875 -1.015625 0.1875zm0.421875 -1.421875q0.421875 0 0.84375 -0.15625q0.4375 -0.15625 0.796875 -0.453125l0.125 -0.796875q-0.25 -0.140625 -0.578125 -0.21875q-0.328125 -0.09375 -0.703125 -0.09375q-0.4375 0 -0.765625 0.125q-0.3125 0.125 -0.484375 0.34375q-0.171875 0.21875 -0.171875 0.515625q0 0.359375 0.25 0.546875q0.265625 0.1875 0.6875 0.1875zm7.40329 1.421875q-1.203125 0 -1.90625 -0.796875q-0.703125 -0.8125 -0.703125 -2.1875q0 -1.203125 0.453125 -2.140625q0.453125 -0.9375 1.25 -1.46875q0.8125 -0.53125 1.828125 -0.53125q0.9375 0 1.59375 0.515625q0.65625 0.515625 0.84375 1.390625l-1.5625 0.515625q-0.09375 -0.4375 -0.375 -0.65625q-0.265625 -0.234375 -0.6875 -0.234375q-0.5 0 -0.890625 0.328125q-0.375 0.328125 -0.609375 0.890625q-0.21875 0.5625 -0.21875 1.265625q0 0.734375 0.28125 1.15625q0.296875 0.40625 0.796875 0.40625q0.453125 0 0.796875 -0.25q0.34375 -0.265625 0.546875 -0.75l1.4375 0.484375q-0.359375 0.96875 -1.125 1.515625q-0.765625 0.546875 -1.75 0.546875zm4.0050354 -0.15625l1.59375 -9.078125l1.6875 -0.546875l-0.8125 4.671875l2.078125 -1.859375l2.046875 0l-2.359375 2.0625l1.53125 4.75l-1.6875 0l-1.140625 -3.625l-0.84375 0.75l-0.515625 2.875l-1.578125 0zm9.610138 0.15625q-0.84375 0 -1.46875 -0.328125q-0.609375 -0.34375 -0.953125 -0.984375q-0.34375 -0.65625 -0.34375 -1.578125q0 -0.8125 0.234375 -1.5625q0.25 -0.765625 0.71875 -1.359375q0.484375 -0.609375 1.15625 -0.953125q0.6875 -0.359375 1.546875 -0.359375q0.828125 0 1.390625 0.34375q0.578125 0.328125 0.875 0.9375q0.3125 0.609375 0.3125 1.421875q0 0.375 -0.0625 0.75q-0.0625 0.375 -0.1875 0.734375l-4.375 0q0 0.484375 0.15625 0.828125q0.171875 0.328125 0.46875 0.515625q0.3125 0.171875 0.71875 0.171875q0.40625 0 0.8125 -0.1875q0.421875 -0.1875 0.734375 -0.5l0.8125 1.0625q-0.5 0.484375 -1.109375 0.765625q-0.609375 0.28125 -1.4375 0.28125zm-0.90625 -4.3125l2.828125 0q0.015625 -0.421875 -0.109375 -0.734375q-0.125 -0.3125 -0.40625 -0.484375q-0.265625 -0.1875 -0.671875 -0.1875q-0.40625 0 -0.75 0.171875q-0.328125 0.171875 -0.5625 0.5q-0.234375 0.3125 -0.328125 0.734375zm7.558655 4.3125q-1.0 0 -1.59375 -0.765625q-0.59375 -0.765625 -0.59375 -2.0625q0 -1.234375 0.4375 -2.203125q0.4375 -0.984375 1.203125 -1.53125q0.765625 -0.5625 1.734375 -0.5625q0.46875 0 0.828125 0.171875q0.359375 0.15625 0.5625 0.484375l0.484375 -2.765625l1.703125 -0.546875l-1.6875 9.625l-1.578125 0l0.109375 -0.609375q-0.28125 0.359375 -0.703125 0.5625q-0.40625 0.203125 -0.90625 0.203125zm0.53125 -1.546875q0.375 0 0.734375 -0.171875q0.359375 -0.171875 0.578125 -0.453125l0.46875 -2.78125q-0.125 -0.296875 -0.40625 -0.46875q-0.28125 -0.171875 -0.640625 -0.171875q-0.5625 0 -0.984375 0.328125q-0.40625 0.3125 -0.640625 0.90625q-0.234375 0.578125 -0.234375 1.34375q0 0.6875 0.296875 1.078125q0.296875 0.390625 0.828125 0.390625zm3.641571 3.125q0.234375 -0.25 0.59375 -0.71875q0.359375 -0.453125 0.75 -1.109375q0.390625 -0.65625 0.734375 -1.484375q0.34375 -0.828125 0.546875 -1.8125q0.21875 -0.984375 0.21875 -2.125q0 -0.875 -0.140625 -1.6875q-0.125 -0.828125 -0.296875 -1.421875q-0.171875 -0.609375 -0.3125 -0.859375l1.5 0q0.21875 0.375 0.40625 1.0q0.1875 0.609375 0.296875 1.375q0.125 0.75 0.125 1.53125q0 1.328125 -0.265625 2.453125q-0.25 1.109375 -0.65625 2.015625q-0.40625 0.90625 -0.90625 1.609375q-0.5 0.71875 -0.984375 1.234375l-1.609375 0z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m431.8714 286.88058c52.098175 0 78.1452 -0.5590515 104.19635 -1.118103c26.051208 -0.55908203 52.106567 -1.1181335 104.21307 -1.1181335" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m431.8714 286.88058c52.098175 0 78.1452 -0.5590515 104.19641 -1.118103c13.025574 -0.27954102 26.052185 -0.55908203 42.335754 -0.7687073c8.141724 -0.10482788 17.097778 -0.19216919 27.275024 -0.25332642c5.088562 -0.030578613 10.482483 -0.054595947 16.232605 -0.07098389c2.875 -0.008178711 5.8391113 -0.014465332 8.898621 -0.018676758l3.4710083 -0.002380371" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m634.282 286.30014l4.5369263 -1.6547852l-4.5391846 -1.6486816z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m734.7533 284.64304c56.04724 0 112.09448 -47.307083 112.09448 -94.61417" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m734.7533 284.64304c28.02362 0 56.04724 -11.826752 77.06494 -29.566925c10.50885 -8.870071 19.266235 -19.218506 25.396423 -30.306091c3.0650024 -5.543808 5.4733276 -11.2724 7.1154175 -17.093384c0.82092285 -2.910492 1.4503784 -5.8440857 1.8745728 -8.78923c0.10601807 -0.73628235 0.19927979 -1.4732971 0.27941895 -2.2108612c0.020019531 -0.18437195 0.03930664 -0.36880493 0.057678223 -0.5532532l0.009765625 -0.101745605" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m848.2012 196.10312l-1.4255981 -4.6141357l-1.8738403 4.451004z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m474.14697 292.20734l159.4961 0l0 16.157501l-159.4961 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m500.1101 306.74173q-0.203125 -0.375 -0.390625 -0.953125q-0.171875 -0.578125 -0.28125 -1.28125q-0.109375 -0.71875 -0.109375 -1.515625q0 -1.40625 0.25 -2.546875q0.265625 -1.15625 0.6875 -2.078125q0.421875 -0.921875 0.9375 -1.625q0.515625 -0.703125 1.015625 -1.21875l1.078125 0q-0.203125 0.21875 -0.5625 0.65625q-0.359375 0.4375 -0.75 1.09375q-0.390625 0.65625 -0.75 1.5q-0.359375 0.84375 -0.59375 1.875q-0.21875 1.015625 -0.21875 2.203125q0 0.9375 0.140625 1.75q0.140625 0.796875 0.3125 1.34375q0.1875 0.5625 0.296875 0.796875l-1.0625 0zm6.770508 0.9375l0.59375 -3.3125q-0.4375 0.40625 -0.875 0.609375q-0.4375 0.1875 -0.9375 0.1875q-0.6875 0 -1.1875 -0.328125q-0.484375 -0.328125 -0.75 -0.9375q-0.265625 -0.625 -0.265625 -1.46875q0 -1.28125 0.4375 -2.265625q0.4375 -1.0 1.1875 -1.5625q0.765625 -0.5625 1.75 -0.5625q0.53125 0 0.953125 0.203125q0.4375 0.203125 0.671875 0.5625l0.09375 -0.609375l1.125 0l-1.578125 9.03125l-1.21875 0.453125zm-0.90625 -3.5625q0.421875 0 0.859375 -0.234375q0.4375 -0.25 0.84375 -0.71875l0.546875 -3.15625q-0.140625 -0.4375 -0.5 -0.6875q-0.359375 -0.25 -0.859375 -0.25q-0.625 0 -1.140625 0.4375q-0.515625 0.421875 -0.8125 1.171875q-0.296875 0.734375 -0.296875 1.65625q0 0.859375 0.34375 1.328125q0.359375 0.453125 1.015625 0.453125zm6.9468994 1.046875q-0.765625 0 -1.2499695 -0.34375q-0.46875 -0.34375 -0.640625 -1.015625q-0.15625 -0.671875 0.015625 -1.640625l0.6875 -3.96875l1.1249695 0l-0.71875 4.046875q-0.15621948 0.921875 0.109375 1.40625q0.28125 0.46875 0.96875 0.46875q0.765625 0 1.234375 -0.453125q0.484375 -0.453125 0.640625 -1.328125l0.71875 -4.140625l1.125 0l-1.1875 6.8125l-1.140625 0l0.125 -0.65625q-0.328125 0.40625 -0.78125 0.609375q-0.453125 0.203125 -1.03125 0.203125zm4.9536133 -0.15625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm3.8880615 8.21875q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm7.4560547 4.09375q-0.5 0 -0.953125 -0.140625q-0.4375 -0.140625 -0.796875 -0.40625q-0.359375 -0.28125 -0.59375 -0.65625l0.84375 -0.65625q0.28125 0.40625 0.6875 0.640625q0.421875 0.234375 0.859375 0.234375q0.609375 0 0.96875 -0.28125q0.359375 -0.28125 0.359375 -0.75q0 -0.328125 -0.28125 -0.59375q-0.265625 -0.265625 -0.984375 -0.59375q-0.9375 -0.4375 -1.328125 -0.890625q-0.390625 -0.46875 -0.390625 -1.09375q0 -0.578125 0.296875 -1.015625q0.3125 -0.4375 0.84375 -0.671875q0.53125 -0.25 1.234375 -0.25q0.71875 0 1.296875 0.28125q0.578125 0.265625 0.890625 0.765625l-0.8125 0.625q-0.578125 -0.703125 -1.453125 -0.703125q-0.53125 0 -0.875 0.265625q-0.34375 0.25 -0.34375 0.640625q0 0.296875 0.25 0.546875q0.25 0.234375 0.921875 0.515625q1.015625 0.453125 1.4375 0.953125q0.421875 0.5 0.421875 1.203125q0 0.921875 -0.6875 1.484375q-0.671875 0.546875 -1.8125 0.546875zm6.1588745 0q-1.046875 0 -1.6875 -0.765625q-0.625 -0.765625 -0.625 -2.0625q0 -1.28125 0.421875 -2.25q0.421875 -0.96875 1.171875 -1.5q0.75 -0.546875 1.75 -0.546875q0.8125 0 1.390625 0.421875q0.578125 0.40625 0.75 1.15625l-1.078125 0.375q-0.09375 -0.4375 -0.40625 -0.671875q-0.296875 -0.25 -0.78125 -0.25q-0.609375 0 -1.078125 0.421875q-0.46875 0.40625 -0.734375 1.140625q-0.265625 0.71875 -0.265625 1.671875q0 0.859375 0.328125 1.34375q0.328125 0.46875 0.9375 0.46875q0.515625 0 0.90625 -0.25q0.40625 -0.265625 0.640625 -0.78125l0.953125 0.375q-0.359375 0.828125 -1.046875 1.265625q-0.671875 0.4375 -1.546875 0.4375zm6.544983 0q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm11.096924 4.09375q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm4.470642 3.9375l2.84375 -3.53125l-1.484375 -3.28125l1.21875 0l0.828125 1.828125q0.046875 0.109375 0.109375 0.234375q0.0625 0.125 0.125 0.28125q0.09375 -0.140625 0.1875 -0.25q0.09375 -0.125 0.1875 -0.25l1.5 -1.84375l1.234375 0l-2.65625 3.296875l1.609375 3.515625l-1.234375 0l-0.953125 -2.09375q-0.046875 -0.109375 -0.109375 -0.234375q-0.046875 -0.140625 -0.109375 -0.25q-0.078125 0.125 -0.171875 0.25q-0.09375 0.109375 -0.203125 0.234375l-1.6875 2.09375l-1.234375 0zm6.706726 2.703125l1.671875 -9.515625l1.140625 0l-0.109375 0.609375q0.765625 -0.765625 1.84375 -0.765625q0.65625 0 1.140625 0.359375q0.484375 0.34375 0.75 0.984375q0.28125 0.625 0.28125 1.484375q0 1.234375 -0.421875 2.203125q-0.421875 0.96875 -1.15625 1.53125q-0.734375 0.5625 -1.65625 0.5625q-0.546875 0 -1.0 -0.203125q-0.453125 -0.21875 -0.765625 -0.625l-0.5 2.921875l-1.21875 0.453125zm3.421875 -3.59375q0.640625 0 1.125 -0.390625q0.484375 -0.40625 0.75 -1.109375q0.28125 -0.703125 0.28125 -1.640625q0 -0.90625 -0.359375 -1.40625q-0.359375 -0.5 -1.0 -0.5q-0.515625 0 -0.984375 0.265625q-0.453125 0.25 -0.75 0.6875l-0.5625 3.171875q0.25 0.453125 0.625 0.6875q0.390625 0.234375 0.875 0.234375zm4.3395996 0.890625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm1.2943726 8.0625l1.1875 -6.8125l1.140625 0l-0.15625 0.875q0.359375 -0.484375 0.828125 -0.75q0.484375 -0.28125 0.96875 -0.28125q0.5625 0 0.890625 0.328125l-0.34375 1.078125q-0.203125 -0.15625 -0.421875 -0.21875q-0.21875 -0.0625 -0.453125 -0.0625q-0.421875 0 -0.8125 0.25q-0.375 0.25 -0.65625 0.703125q-0.265625 0.453125 -0.359375 1.046875l-0.6875 3.84375l-1.125 0zm6.728821 0.15625q-0.90625 0 -1.453125 -0.53125q-0.546875 -0.53125 -0.546875 -1.4375q0 -0.765625 0.34375 -1.34375q0.359375 -0.59375 1.0 -0.90625q0.640625 -0.328125 1.5 -0.328125q0.46875 0 0.890625 0.125q0.421875 0.125 0.765625 0.359375l0.125 -0.65625q0.125 -0.671875 -0.15625 -1.046875q-0.265625 -0.375 -0.890625 -0.375q-0.46875 0 -1.0625 0.203125q-0.578125 0.1875 -1.09375 0.515625l-0.140625 -0.9375q0.609375 -0.375 1.234375 -0.5625q0.625 -0.203125 1.1875 -0.203125q0.71875 0 1.21875 0.296875q0.5 0.296875 0.703125 0.84375q0.203125 0.546875 0.078125 1.28125l-0.796875 4.546875l-1.078125 0l0.09375 -0.5625q-0.375 0.34375 -0.890625 0.53125q-0.5 0.1875 -1.03125 0.1875zm0.234375 -0.984375q0.46875 0 0.984375 -0.234375q0.515625 -0.25 0.90625 -0.65625l0.203125 -1.1875q-0.34375 -0.234375 -0.734375 -0.359375q-0.375 -0.140625 -0.75 -0.140625q-0.546875 0 -0.9375 0.1875q-0.390625 0.171875 -0.609375 0.515625q-0.21875 0.328125 -0.21875 0.78125q0 0.5 0.3125 0.796875q0.3125 0.296875 0.84375 0.296875zm6.208496 0.984375q-0.6875 0 -0.984375 -0.46875q-0.28125 -0.46875 -0.125 -1.359375l0.71875 -4.125l-1.109375 0l0.1875 -1.015625l1.09375 0l0.421875 -2.375l1.203125 -0.4375l-0.5 2.8125l1.609375 0l-0.171875 1.015625l-1.609375 0l-0.6875 3.875q-0.109375 0.609375 -0.015625 0.828125q0.109375 0.21875 0.5 0.21875q0.234375 0 0.46875 -0.078125q0.25 -0.078125 0.5625 -0.265625l-0.296875 1.09375q-0.515625 0.28125 -1.265625 0.28125zm2.748352 -0.15625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm3.8568726 8.21875q-1.171875 0 -1.84375 -0.75q-0.65625 -0.75 -0.65625 -2.078125q0 -1.234375 0.453125 -2.203125q0.46875 -0.96875 1.25 -1.53125q0.796875 -0.5625 1.796875 -0.5625q1.125 0 1.796875 0.765625q0.671875 0.75 0.671875 2.03125q0 1.25 -0.453125 2.234375q-0.453125 0.96875 -1.25 1.53125q-0.78125 0.5625 -1.765625 0.5625zm0.03125 -1.046875q0.65625 0 1.171875 -0.421875q0.53125 -0.421875 0.828125 -1.140625q0.296875 -0.71875 0.296875 -1.640625q0 -0.859375 -0.375 -1.34375q-0.375 -0.5 -1.015625 -0.5q-0.65625 0 -1.1875 0.421875q-0.515625 0.421875 -0.8125 1.15625q-0.296875 0.71875 -0.296875 1.625q0 0.859375 0.359375 1.359375q0.375 0.484375 1.03125 0.484375zm4.3634033 0.890625l1.1875 -6.8125l1.140625 0l-0.140625 0.734375q0.359375 -0.4375 0.828125 -0.65625q0.484375 -0.234375 1.0 -0.234375q0.765625 0 1.234375 0.375q0.46875 0.359375 0.609375 1.0625q0.15625 0.6875 -0.015625 1.671875l-0.671875 3.859375l-1.140625 0l0.71875 -4.09375q0.15625 -0.875 -0.109375 -1.359375q-0.265625 -0.484375 -0.9375 -0.484375q-0.6875 0 -1.21875 0.515625q-0.515625 0.515625 -0.65625 1.34375l-0.703125 4.078125l-1.125 0zm6.1567383 1.734375q0.1875 -0.234375 0.515625 -0.65625q0.328125 -0.421875 0.703125 -1.0625q0.375 -0.625 0.71875 -1.4375q0.34375 -0.828125 0.5625 -1.828125q0.21875 -1.0 0.21875 -2.1875q0 -0.953125 -0.15625 -1.796875q-0.140625 -0.84375 -0.34375 -1.4375q-0.1875 -0.59375 -0.3125 -0.8125l1.046875 0q0.234375 0.375 0.421875 0.984375q0.203125 0.59375 0.328125 1.34375q0.125 0.734375 0.125 1.5625q0 1.375 -0.25 2.5q-0.234375 1.125 -0.640625 2.015625q-0.390625 0.890625 -0.890625 1.578125q-0.484375 0.703125 -0.984375 1.234375l-1.0625 0z" fill-rule="nonzero"/>
+    <path fill="#ffd966" d="m181.9943 369.77737l0 0c0 -26.092194 21.151901 -47.24408 47.244095 -47.24408l0 0c12.529907 0 24.54663 4.977478 33.4066 13.837463c8.859985 8.859985 13.837494 20.876709 13.837494 33.406616l0 0c0 26.092194 -21.151917 47.24411 -47.244095 47.24411l0 0c-26.092194 0 -47.244095 -21.151917 -47.244095 -47.24411z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m201.8942 377.79486l0 -3.5q-0.3125 0.359375 -0.71875 0.546875q-0.40625 0.1875 -0.921875 0.1875q-0.9375 0 -1.640625 -0.484375q-0.703125 -0.484375 -1.09375 -1.359375q-0.375 -0.875 -0.375 -2.0625q0 -1.21875 0.390625 -2.09375q0.390625 -0.890625 1.09375 -1.359375q0.71875 -0.484375 1.6875 -0.484375q0.515625 0 0.890625 0.1875q0.375 0.171875 0.6875 0.546875l0 -0.546875l1.828125 0l0 9.59375l-1.828125 0.828125zm-1.375 -4.515625q0.4375 0 0.796875 -0.171875q0.375 -0.171875 0.578125 -0.5l0 -3.015625q-0.265625 -0.328125 -0.609375 -0.484375q-0.328125 -0.171875 -0.765625 -0.171875q-0.71875 0 -1.125 0.578125q-0.40625 0.578125 -0.40625 1.609375q0 1.03125 0.390625 1.59375q0.40625 0.5625 1.140625 0.5625zm8.042007 1.75q-1.25 0 -1.953125 -0.78125q-0.6875 -0.78125 -0.6875 -2.203125l0 -4.671875l1.84375 0l0 4.09375q0 0.953125 0.3125 1.390625q0.328125 0.421875 1.03125 0.421875q0.734375 0 1.09375 -0.4375q0.359375 -0.4375 0.359375 -1.34375l0 -4.125l1.828125 0l0 7.484375l-1.828125 0l0 -0.703125q-0.34375 0.4375 -0.84375 0.65625q-0.5 0.21875 -1.15625 0.21875zm6.1295166 -0.171875l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm6.3039246 8.59375q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.055359 4.6875q-0.875 0 -1.6875 -0.375q-0.796875 -0.375 -1.265625 -1.0l1.203125 -1.015625q0.375 0.40625 0.859375 0.640625q0.484375 0.21875 0.9375 0.21875q0.5625 0 0.859375 -0.1875q0.296875 -0.1875 0.296875 -0.515625q0 -0.21875 -0.140625 -0.390625q-0.140625 -0.171875 -0.5 -0.34375q-0.359375 -0.1875 -1.015625 -0.421875q-1.25 -0.46875 -1.78125 -1.0q-0.53125 -0.546875 -0.53125 -1.375q0 -0.921875 0.75 -1.5q0.75 -0.578125 1.953125 -0.578125q0.8125 0 1.5 0.3125q0.703125 0.3125 1.203125 0.90625l-1.203125 0.984375q-0.65625 -0.671875 -1.53125 -0.671875q-0.453125 0 -0.734375 0.171875q-0.265625 0.171875 -0.265625 0.4375q0 0.28125 0.3125 0.5q0.328125 0.203125 1.203125 0.484375q0.859375 0.28125 1.390625 0.625q0.546875 0.34375 0.796875 0.796875q0.265625 0.453125 0.265625 1.046875q0 1.046875 -0.765625 1.65625q-0.765625 0.59375 -2.109375 0.59375zm7.594406 0q-1.03125 0 -1.796875 -0.46875q-0.75 -0.484375 -1.15625 -1.359375q-0.40625 -0.875 -0.40625 -2.078125q0 -1.21875 0.40625 -2.09375q0.421875 -0.875 1.171875 -1.359375q0.765625 -0.484375 1.8125 -0.484375q1.15625 0 1.921875 0.546875q0.78125 0.53125 1.125 1.59375l-1.734375 0.578125q-0.171875 -0.46875 -0.515625 -0.71875q-0.328125 -0.25 -0.78125 -0.25q-0.703125 0 -1.125 0.59375q-0.421875 0.578125 -0.421875 1.59375q0 1.0 0.40625 1.59375q0.40625 0.578125 1.109375 0.578125q0.484375 0 0.828125 -0.28125q0.359375 -0.28125 0.53125 -0.8125l1.71875 0.515625q-0.328125 1.15625 -1.109375 1.734375q-0.78125 0.578125 -1.984375 0.578125zm4.983139 -0.171875l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm3.2101746 8.421875l0 -7.484375l1.828125 0l0 0.71875q0.359375 -0.453125 0.859375 -0.671875q0.515625 -0.234375 1.15625 -0.234375q1.25 0 1.921875 0.78125q0.6875 0.765625 0.6875 2.21875l0 4.671875l-1.8125 0l0 -4.109375q0 -0.9375 -0.328125 -1.375q-0.328125 -0.4375 -1.015625 -0.4375q-0.75 0 -1.109375 0.4375q-0.359375 0.4375 -0.359375 1.34375l0 4.140625l-1.828125 0zm11.390335 3.125l-1.0 -1.53125q1.0625 -0.078125 1.65625 -0.25q0.59375 -0.15625 0.828125 -0.453125q0.234375 -0.296875 0.234375 -0.78125l0 -0.546875q-0.578125 0.609375 -1.609375 0.609375q-0.984375 0 -1.6874847 -0.46875q-0.703125 -0.46875 -1.078125 -1.34375q-0.375 -0.890625 -0.375 -2.109375q0 -1.1875 0.390625 -2.0625q0.390625 -0.890625 1.09375 -1.375q0.71873474 -0.484375 1.6718597 -0.484375q0.484375 0 0.890625 0.15625q0.421875 0.15625 0.703125 0.453125l0 -0.421875l1.828125 0l0 6.8125q0 1.0 -0.1875 1.6875q-0.1875 0.6875 -0.609375 1.125q-0.40625 0.4375 -1.078125 0.671875q-0.671875 0.234375 -1.671875 0.3125zm0.40625 -4.703125q0.390625 0 0.71875 -0.171875q0.34375 -0.1875 0.59375 -0.5l0 -3.015625q-0.203125 -0.28125 -0.5625 -0.46875q-0.359375 -0.1875 -0.734375 -0.1875q-0.78125 0 -1.203125 0.5625q-0.40625 0.546875 -0.40625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.546875 0.75q0.359375 0.25 0.875 0.25z" fill-rule="nonzero"/>
+    <path fill="#ffd966" d="m211.18192 383.053l36.125977 0l0 15.685059l-36.125977 0z" fill-rule="evenodd"/>
+    <path fill="#1155cc" d="m223.13303 399.85178q-1.78125 -1.109375 -2.71875 -2.78125q-0.9375 -1.671875 -0.9375 -3.71875q0 -2.03125 0.9375 -3.703125q0.953125 -1.6875 2.734375 -2.796875l1.3125 1.234375q-1.34375 0.9375 -2.015625 2.328125q-0.65625 1.375 -0.65625 2.9375q0 1.5625 0.65625 2.953125q0.671875 1.375 2.015625 2.3125l-1.328125 1.234375zm5.005539 -2.59375q-0.96875 0 -1.75 -0.515625q-0.765625 -0.515625 -1.21875 -1.421875q-0.453125 -0.921875 -0.453125 -2.09375q0 -1.171875 0.453125 -2.078125q0.453125 -0.921875 1.234375 -1.4375q0.796875 -0.515625 1.8125 -0.515625q0.5625 0 1.015625 0.15625q0.46875 0.15625 0.8125 0.453125q0.359375 0.28125 0.609375 0.65625q0.25 0.375 0.359375 0.796875l-0.484375 -0.0625l0 -1.84375l2.328125 0l0 7.78125l-2.359375 0l0 -1.875l0.515625 -0.015625q-0.109375 0.40625 -0.375 0.78125q-0.265625 0.359375 -0.65625 0.640625q-0.375 0.28125 -0.84375 0.4375q-0.46875 0.15625 -1.0 0.15625zm0.65625 -1.96875q0.53125 0 0.921875 -0.25q0.40625 -0.25 0.625 -0.703125q0.21875 -0.46875 0.21875 -1.109375q0 -0.625 -0.21875 -1.078125q-0.21875 -0.46875 -0.625 -0.71875q-0.390625 -0.265625 -0.921875 -0.265625q-0.53125 0 -0.921875 0.265625q-0.390625 0.25 -0.609375 0.71875q-0.21875 0.453125 -0.21875 1.078125q0 0.640625 0.21875 1.109375q0.21875 0.453125 0.609375 0.703125q0.390625 0.25 0.921875 0.25zm6.5491333 4.5625l-1.328125 -1.234375q1.359375 -0.9375 2.015625 -2.3125q0.65625 -1.390625 0.65625 -2.953125q0 -1.5625 -0.65625 -2.9375q-0.65625 -1.390625 -2.015625 -2.328125l1.34375 -1.234375q1.78125 1.109375 2.71875 2.78125q0.9375 1.671875 0.9375 3.71875q0 2.03125 -0.953125 3.71875q-0.9375 1.671875 -2.71875 2.78125z" fill-rule="nonzero"/>
+    <path fill="#f1c232" d="m337.3832 286.88058l0 0c0 -26.092194 21.151886 -47.244095 47.24408 -47.244095l0 0c12.529907 0 24.54663 4.977478 33.406616 13.837463c8.859985 8.860001 13.837494 20.876724 13.837494 33.40663l0 0c0 26.092194 -21.151917 47.24408 -47.24411 47.24408l0 0c-26.092194 0 -47.24408 -21.151886 -47.24408 -47.24408z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m359.5364 294.89807l0 -3.5q-0.3125 0.359375 -0.71875 0.546875q-0.40625 0.1875 -0.921875 0.1875q-0.9375 0 -1.640625 -0.484375q-0.703125 -0.484375 -1.09375 -1.359375q-0.375 -0.875 -0.375 -2.0625q0 -1.21875 0.390625 -2.09375q0.390625 -0.890625 1.09375 -1.359375q0.71875 -0.484375 1.6875 -0.484375q0.515625 0 0.890625 0.1875q0.375 0.171875 0.6875 0.546875l0 -0.546875l1.828125 0l0 9.59375l-1.828125 0.828125zm-1.375 -4.515625q0.4375 0 0.796875 -0.171875q0.375 -0.171875 0.578125 -0.5l0 -3.015625q-0.265625 -0.328125 -0.609375 -0.484375q-0.328125 -0.171875 -0.765625 -0.171875q-0.71875 0 -1.125 0.578125q-0.40625 0.578125 -0.40625 1.609375q0 1.03125 0.390625 1.59375q0.40625 0.5625 1.140625 0.5625zm8.042023 1.75q-1.25 0 -1.953125 -0.78125q-0.6875 -0.78125 -0.6875 -2.203125l0 -4.671875l1.84375 0l0 4.09375q0 0.953125 0.3125 1.390625q0.328125 0.421875 1.03125 0.421875q0.734375 0 1.09375 -0.4375q0.359375 -0.4375 0.359375 -1.34375l0 -4.125l1.828125 0l0 7.484375l-1.828125 0l0 -0.703125q-0.34375 0.4375 -0.84375 0.65625q-0.5 0.21875 -1.15625 0.21875zm6.1295166 -0.171875l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm6.3039246 8.59375q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.055359 4.6875q-0.875 0 -1.6875 -0.375q-0.796875 -0.375 -1.265625 -1.0l1.203125 -1.015625q0.375 0.40625 0.859375 0.640625q0.484375 0.21875 0.9375 0.21875q0.5625 0 0.859375 -0.1875q0.296875 -0.1875 0.296875 -0.515625q0 -0.21875 -0.140625 -0.390625q-0.140625 -0.171875 -0.5 -0.34375q-0.359375 -0.1875 -1.015625 -0.421875q-1.25 -0.46875 -1.78125 -1.0q-0.53125 -0.546875 -0.53125 -1.375q0 -0.921875 0.75 -1.5q0.75 -0.578125 1.953125 -0.578125q0.8125 0 1.5 0.3125q0.703125 0.3125 1.203125 0.90625l-1.203125 0.984375q-0.65625 -0.671875 -1.53125 -0.671875q-0.453125 0 -0.734375 0.171875q-0.265625 0.171875 -0.265625 0.4375q0 0.28125 0.3125 0.5q0.328125 0.203125 1.203125 0.484375q0.859375 0.28125 1.390625 0.625q0.546875 0.34375 0.796875 0.796875q0.265625 0.453125 0.265625 1.046875q0 1.046875 -0.765625 1.65625q-0.765625 0.59375 -2.109375 0.59375zm7.594391 0q-1.03125 0 -1.796875 -0.46875q-0.75 -0.484375 -1.15625 -1.359375q-0.40625 -0.875 -0.40625 -2.078125q0 -1.21875 0.40625 -2.09375q0.421875 -0.875 1.171875 -1.359375q0.765625 -0.484375 1.8125 -0.484375q1.15625 0 1.921875 0.546875q0.78125 0.53125 1.125 1.59375l-1.734375 0.578125q-0.171875 -0.46875 -0.515625 -0.71875q-0.328125 -0.25 -0.78125 -0.25q-0.703125 0 -1.125 0.59375q-0.421875 0.578125 -0.421875 1.59375q0 1.0 0.40625 1.59375q0.40625 0.578125 1.109375 0.578125q0.484375 0 0.828125 -0.28125q0.359375 -0.28125 0.53125 -0.8125l1.71875 0.515625q-0.328125 1.15625 -1.109375 1.734375q-0.78125 0.578125 -1.984375 0.578125zm8.031952 0q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25z" fill-rule="nonzero"/>
+    <path fill="#f1c232" d="m364.3755 299.07678l40.503937 0l0 17.763794l-40.503937 0z" fill-rule="evenodd"/>
+    <path fill="#1155cc" d="m378.07574 315.87555q-1.78125 -1.109375 -2.71875 -2.78125q-0.9375 -1.671875 -0.9375 -3.71875q0 -2.03125 0.9375 -3.703125q0.953125 -1.6875 2.734375 -2.796875l1.3125 1.234375q-1.34375 0.9375 -2.015625 2.328125q-0.65625 1.375 -0.65625 2.9375q0 1.5625 0.65625 2.953125q0.671875 1.375 2.015625 2.3125l-1.328125 1.234375zm1.4503479 -2.71875l4.0625 -10.265625l2.109375 0l4.03125 10.265625l-2.515625 0l-1.9375 -5.203125q-0.125 -0.3125 -0.25 -0.65625q-0.109375 -0.359375 -0.234375 -0.734375q-0.109375 -0.375 -0.21875 -0.734375q-0.109375 -0.359375 -0.1875 -0.65625l0.453125 -0.015625q-0.09375 0.359375 -0.203125 0.71875q-0.109375 0.34375 -0.21875 0.6875q-0.109375 0.34375 -0.25 0.6875q-0.125 0.34375 -0.25 0.71875l-1.921875 5.1875l-2.46875 0zm1.9375 -1.96875l0.78125 -1.875l4.71875 0l0.75 1.875l-6.25 0zm9.702728 4.6875l-1.328125 -1.234375q1.359375 -0.9375 2.015625 -2.3125q0.65625 -1.390625 0.65625 -2.953125q0 -1.5625 -0.65625 -2.9375q-0.65625 -1.390625 -2.015625 -2.328125l1.34375 -1.234375q1.78125 1.109375 2.71875 2.78125q0.9375 1.671875 0.9375 3.71875q0 2.03125 -0.953125 3.71875q-0.9375 1.671875 -2.71875 2.78125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m418.0339 253.47395c0 -20.97638 32.299225 -41.95276 64.59845 -41.95276" fill-rule="evenodd"/>
+    <path stroke="#0097a7" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m418.0339 253.47395c0 -10.48819 8.074799 -20.97638 20.187012 -28.842514c6.056122 -3.933075 13.1215515 -7.2106323 20.69168 -9.504929c1.8925171 -0.5735626 3.8166199 -1.0856781 5.764374 -1.5312195l0.07336426 -0.016448975" fill-rule="evenodd"/>
+    <path fill="#0097a7" stroke="#0097a7" stroke-width="3.0" stroke-linecap="butt" d="m465.31677 218.50154l12.958618 -6.4789886l-14.091492 -3.3664398z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m563.2767 178.10257c0 -25.385834 38.48822 -50.77166 76.97638 -50.77166" fill-rule="evenodd"/>
+    <path stroke="#0097a7" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="12.0,9.0" d="m563.2767 178.10257c0 -12.692917 9.62207 -25.385834 24.055115 -34.905518c7.2165527 -4.759842 15.635864 -8.726379 24.656555 -11.502945c2.255127 -0.69415283 4.5479126 -1.3139191 6.8688354 -1.8531189c0.58026123 -0.13479614 1.1622314 -0.26457214 1.7458496 -0.38919067c0.29180908 -0.062316895 0.5840454 -0.12335205 0.8765869 -0.18309021l0.86035156 -0.17160034" fill-rule="evenodd"/>
+    <path fill="#0097a7" stroke="#0097a7" stroke-width="3.0" stroke-linecap="butt" d="m622.8262 134.02838l13.062378 -6.2671356l-14.034851 -3.5954285z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m577.1142 211.50919c38.48816 0 76.97638 19.858261 76.97638 39.716537" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m577.1142 211.50919c19.24408 0 38.48816 4.964569 52.921265 12.411423c7.2164917 3.7234192 13.230286 8.067413 17.439941 12.721695c2.1047974 2.3271484 3.758606 4.7318573 4.8861694 7.1753693c0.14099121 0.30541992 0.27368164 0.61146545 0.3980713 0.9180298c0.062194824 0.15330505 0.12225342 0.30671692 0.18023682 0.4602661l0.047912598 0.13195801" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m651.36426 245.63148l2.4576416 4.157242l0.7895508 -4.764374z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m734.7533 127.33071c14.251953 0 21.37793 9.448814 28.503906 18.897636c7.1259766 9.448822 14.252014 18.897644 28.503967 18.897644" fill-rule="evenodd"/>
+    <path stroke="#0097a7" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m734.7533 127.33071c14.251953 0 21.37793 9.448814 28.503906 18.897636c1.7814941 2.3621979 3.5629883 4.724411 5.4558716 6.93898c0.9463501 1.1072845 1.9205933 2.177658 2.9367065 3.1926575c0.5079346 0.5075073 1.0264282 1.0011749 1.5570068 1.4786835c0.26531982 0.23875427 0.5336914 0.47348022 0.80529785 0.7038727c0.13580322 0.11520386 0.27246094 0.22932434 0.4099121 0.34231567l0.3529663 0.28549194" fill-rule="evenodd"/>
+    <path fill="#0097a7" stroke="#0097a7" stroke-width="3.0" stroke-linecap="butt" d="m773.13544 163.84645l14.4869995 -0.17155457l-11.207947 -9.180649z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m577.5958 174.37796l80.50391 0l0 16.15747l-80.50391 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m589.20435 188.91232q-0.203125 -0.375 -0.390625 -1.0q-0.171875 -0.609375 -0.28125 -1.359375q-0.109375 -0.75 -0.109375 -1.53125q0 -1.328125 0.25 -2.453125q0.25 -1.125 0.65625 -2.03125q0.421875 -0.921875 0.921875 -1.625q0.5 -0.703125 0.984375 -1.21875l1.609375 0q-0.234375 0.25 -0.59375 0.703125q-0.359375 0.453125 -0.765625 1.109375q-0.390625 0.640625 -0.734375 1.484375q-0.34375 0.828125 -0.5625 1.828125q-0.21875 0.984375 -0.21875 2.140625q0 0.875 0.125 1.6875q0.140625 0.796875 0.3125 1.40625q0.171875 0.609375 0.296875 0.859375l-1.5 0zm5.465637 -1.578125q-0.984375 0 -1.53125 -0.53125q-0.546875 -0.546875 -0.546875 -1.4375q0 -0.75 0.359375 -1.3125q0.375 -0.5625 1.03125 -0.875q0.671875 -0.3125 1.5625 -0.3125q0.421875 0 0.84375 0.109375q0.4375 0.09375 0.703125 0.265625l0.0625 -0.375q0.125 -0.609375 -0.125 -0.921875q-0.25 -0.328125 -0.84375 -0.328125q-0.5 0 -1.0625 0.1875q-0.5625 0.171875 -1.09375 0.5l-0.234375 -1.296875q0.625 -0.40625 1.28125 -0.59375q0.671875 -0.203125 1.34375 -0.203125q0.828125 0 1.390625 0.328125q0.5625 0.328125 0.8125 0.921875q0.25 0.59375 0.109375 1.375l-0.765625 4.34375l-1.578125 0l0.09375 -0.546875q-0.296875 0.3125 -0.796875 0.515625q-0.5 0.1875 -1.015625 0.1875zm0.421875 -1.421875q0.421875 0 0.84375 -0.15625q0.4375 -0.15625 0.796875 -0.453125l0.125 -0.796875q-0.25 -0.140625 -0.578125 -0.21875q-0.328125 -0.09375 -0.703125 -0.09375q-0.4375 0 -0.765625 0.125q-0.3125 0.125 -0.484375 0.34375q-0.171875 0.21875 -0.171875 0.515625q0 0.359375 0.25 0.546875q0.265625 0.1875 0.6875 0.1875zm4.5999146 1.265625l1.578125 -9.078125l1.703125 -0.546875l-1.703125 9.625l-1.578125 0zm3.545288 0l1.578125 -9.078125l1.703125 -0.546875l-1.703125 9.625l-1.578125 0zm8.726074 0.15625q-0.984375 0 -1.53125 -0.53125q-0.546875 -0.546875 -0.546875 -1.4375q0 -0.75 0.359375 -1.3125q0.375 -0.5625 1.03125 -0.875q0.671875 -0.3125 1.5625 -0.3125q0.421875 0 0.84375 0.109375q0.4375 0.09375 0.703125 0.265625l0.0625 -0.375q0.125 -0.609375 -0.125 -0.921875q-0.25 -0.328125 -0.84375 -0.328125q-0.5 0 -1.0625 0.1875q-0.5625 0.171875 -1.09375 0.5l-0.234375 -1.296875q0.625 -0.40625 1.28125 -0.59375q0.671875 -0.203125 1.34375 -0.203125q0.828125 0 1.390625 0.328125q0.5625 0.328125 0.8125 0.921875q0.25 0.59375 0.109375 1.375l-0.765625 4.34375l-1.578125 0l0.09375 -0.546875q-0.296875 0.3125 -0.796875 0.515625q-0.5 0.1875 -1.015625 0.1875zm0.421875 -1.421875q0.421875 0 0.84375 -0.15625q0.4375 -0.15625 0.796875 -0.453125l0.125 -0.796875q-0.25 -0.140625 -0.578125 -0.21875q-0.328125 -0.09375 -0.703125 -0.09375q-0.4375 0 -0.765625 0.125q-0.3125 0.125 -0.484375 0.34375q-0.171875 0.21875 -0.171875 0.515625q0 0.359375 0.25 0.546875q0.265625 0.1875 0.6875 0.1875zm7.4032593 1.421875q-1.203125 0 -1.90625 -0.796875q-0.703125 -0.8125 -0.703125 -2.1875q0 -1.203125 0.453125 -2.140625q0.453125 -0.9375 1.25 -1.46875q0.8125 -0.53125 1.828125 -0.53125q0.9375 0 1.59375 0.515625q0.65625 0.515625 0.84375 1.390625l-1.5625 0.515625q-0.09375 -0.4375 -0.375 -0.65625q-0.265625 -0.234375 -0.6875 -0.234375q-0.5 0 -0.890625 0.328125q-0.375 0.328125 -0.609375 0.890625q-0.21875 0.5625 -0.21875 1.265625q0 0.734375 0.28125 1.15625q0.296875 0.40625 0.796875 0.40625q0.453125 0 0.796875 -0.25q0.34375 -0.265625 0.546875 -0.75l1.4375 0.484375q-0.359375 0.96875 -1.125 1.515625q-0.765625 0.546875 -1.75 0.546875zm4.005066 -0.15625l1.59375 -9.078125l1.6875 -0.546875l-0.8125 4.671875l2.078125 -1.859375l2.046875 0l-2.359375 2.0625l1.53125 4.75l-1.6875 0l-1.140625 -3.625l-0.84375 0.75l-0.515625 2.875l-1.578125 0zm9.610107 0.15625q-0.84375 0 -1.46875 -0.328125q-0.609375 -0.34375 -0.953125 -0.984375q-0.34375 -0.65625 -0.34375 -1.578125q0 -0.8125 0.234375 -1.5625q0.25 -0.765625 0.71875 -1.359375q0.484375 -0.609375 1.15625 -0.953125q0.6875 -0.359375 1.546875 -0.359375q0.828125 0 1.390625 0.34375q0.578125 0.328125 0.875 0.9375q0.3125 0.609375 0.3125 1.421875q0 0.375 -0.0625 0.75q-0.0625 0.375 -0.1875 0.734375l-4.375 0q0 0.484375 0.15625 0.828125q0.171875 0.328125 0.46875 0.515625q0.3125 0.171875 0.71875 0.171875q0.40625 0 0.8125 -0.1875q0.421875 -0.1875 0.734375 -0.5l0.8125 1.0625q-0.5 0.484375 -1.109375 0.765625q-0.609375 0.28125 -1.4375 0.28125zm-0.90625 -4.3125l2.828125 0q0.015625 -0.421875 -0.109375 -0.734375q-0.125 -0.3125 -0.40625 -0.484375q-0.265625 -0.1875 -0.671875 -0.1875q-0.40625 0 -0.75 0.171875q-0.328125 0.171875 -0.5625 0.5q-0.234375 0.3125 -0.328125 0.734375zm7.558716 4.3125q-1.0 0 -1.59375 -0.765625q-0.59375 -0.765625 -0.59375 -2.0625q0 -1.234375 0.4375 -2.203125q0.4375 -0.984375 1.203125 -1.53125q0.765625 -0.5625 1.734375 -0.5625q0.46875 0 0.828125 0.171875q0.359375 0.15625 0.5625 0.484375l0.484375 -2.765625l1.703125 -0.546875l-1.6875 9.625l-1.578125 0l0.109375 -0.609375q-0.28125 0.359375 -0.703125 0.5625q-0.40625 0.203125 -0.90625 0.203125zm0.53125 -1.546875q0.375 0 0.734375 -0.171875q0.359375 -0.171875 0.578125 -0.453125l0.46875 -2.78125q-0.125 -0.296875 -0.40625 -0.46875q-0.28125 -0.171875 -0.640625 -0.171875q-0.5625 0 -0.984375 0.328125q-0.40625 0.3125 -0.640625 0.90625q-0.234375 0.578125 -0.234375 1.34375q0 0.6875 0.296875 1.078125q0.296875 0.390625 0.828125 0.390625zm3.6415405 3.125q0.234375 -0.25 0.59375 -0.71875q0.359375 -0.453125 0.75 -1.109375q0.390625 -0.65625 0.734375 -1.484375q0.34375 -0.828125 0.546875 -1.8125q0.21875 -0.984375 0.21875 -2.125q0 -0.875 -0.140625 -1.6875q-0.125 -0.828125 -0.296875 -1.421875q-0.171875 -0.609375 -0.3125 -0.859375l1.5 0q0.21875 0.375 0.40625 1.0q0.1875 0.609375 0.296875 1.375q0.125 0.75 0.125 1.53125q0 1.328125 -0.265625 2.453125q-0.25 1.109375 -0.65625 2.015625q-0.40625 0.90625 -0.90625 1.609375q-0.5 0.71875 -0.984375 1.234375l-1.609375 0z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m419.47507 236.91208l80.50394 0l0 16.15747l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m438.19626 249.71207l0 -6.8125l1.65625 0l0 0.734375q0.1875 -0.390625 0.578125 -0.640625q0.40625 -0.25 0.84375 -0.25q0.734375 0 1.234375 0.5l-0.234375 1.53125q-0.28125 -0.234375 -0.546875 -0.328125q-0.25 -0.09375 -0.5625 -0.09375q-0.40625 0 -0.703125 0.203125q-0.296875 0.1875 -0.453125 0.5625q-0.15625 0.359375 -0.15625 0.875l0 3.71875l-1.65625 0zm8.008026 0.15625q-0.984375 0 -1.6875 -0.4375q-0.703125 -0.4375 -1.09375 -1.234375q-0.375 -0.8125 -0.375 -1.890625q0 -1.109375 0.375 -1.90625q0.390625 -0.796875 1.09375 -1.21875q0.71875 -0.4375 1.6875 -0.4375q0.828125 0 1.5 0.34375q0.671875 0.34375 1.0625 1.0625q0.40625 0.71875 0.40625 1.828125q0 0.171875 -0.015625 0.421875q0 0.25 -0.03125 0.5l-4.4375 0q0.03125 0.46875 0.21875 0.828125q0.1875 0.34375 0.515625 0.53125q0.34375 0.1875 0.796875 0.1875q0.484375 0 0.84375 -0.15625q0.359375 -0.171875 0.671875 -0.53125l0.96875 1.0q-0.453125 0.5 -1.078125 0.8125q-0.609375 0.296875 -1.421875 0.296875zm-1.515625 -4.265625l2.875 0q-0.03125 -0.453125 -0.203125 -0.765625q-0.15625 -0.328125 -0.46875 -0.5q-0.296875 -0.171875 -0.71875 -0.171875q-0.59375 0 -1.0 0.359375q-0.40625 0.359375 -0.484375 1.078125zm6.2049866 4.109375l0 -8.875l1.65625 -0.75l0 9.625l-1.65625 0zm6.6128235 0.15625q-0.984375 0 -1.6875 -0.4375q-0.703125 -0.4375 -1.09375 -1.234375q-0.375 -0.8125 -0.375 -1.890625q0 -1.109375 0.375 -1.90625q0.390625 -0.796875 1.09375 -1.21875q0.71875 -0.4375 1.6875 -0.4375q0.828125 0 1.5 0.34375q0.671875 0.34375 1.0625 1.0625q0.40625 0.71875 0.40625 1.828125q0 0.171875 -0.015625 0.421875q0 0.25 -0.03125 0.5l-4.4375 0q0.03125 0.46875 0.21875 0.828125q0.1875 0.34375 0.515625 0.53125q0.34375 0.1875 0.796875 0.1875q0.484375 0 0.84375 -0.15625q0.359375 -0.171875 0.671875 -0.53125l0.96875 1.0q-0.453125 0.5 -1.078125 0.8125q-0.609375 0.296875 -1.421875 0.296875zm-1.515625 -4.265625l2.875 0q-0.03125 -0.453125 -0.203125 -0.765625q-0.15625 -0.328125 -0.46875 -0.5q-0.296875 -0.171875 -0.71875 -0.171875q-0.59375 0 -1.0 0.359375q-0.40625 0.359375 -0.484375 1.078125zm8.075836 4.265625q-1.109375 0 -1.765625 -0.59375q-0.640625 -0.609375 -0.640625 -1.640625q0 -0.671875 0.328125 -1.171875q0.34375 -0.5 0.9375 -0.78125q0.609375 -0.28125 1.421875 -0.28125q0.4375 0 0.859375 0.09375q0.421875 0.078125 0.734375 0.234375l0 -0.421875q0 -0.578125 -0.328125 -0.859375q-0.328125 -0.296875 -0.953125 -0.296875q-0.5 0 -1.03125 0.1875q-0.515625 0.171875 -1.0625 0.53125l-0.5 -1.328125q0.59375 -0.375 1.3125 -0.578125q0.71875 -0.21875 1.453125 -0.21875q1.375 0 2.046875 0.65625q0.6875 0.640625 0.6875 1.921875l0 4.390625l-1.625 0l0 -0.546875q-0.34375 0.34375 -0.828125 0.53125q-0.46875 0.171875 -1.046875 0.171875zm0.390625 -1.40625q0.421875 0 0.796875 -0.15625q0.390625 -0.171875 0.6875 -0.46875l0 -0.796875q-0.296875 -0.140625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.609375 0 -0.96875 0.234375q-0.359375 0.234375 -0.359375 0.65625q0 0.390625 0.3125 0.609375q0.3125 0.21875 0.875 0.21875zm7.2518616 1.40625q-0.796875 0 -1.53125 -0.328125q-0.734375 -0.34375 -1.15625 -0.921875l1.09375 -0.921875q0.34375 0.359375 0.78125 0.578125q0.453125 0.203125 0.859375 0.203125q0.5 0 0.765625 -0.171875q0.28125 -0.171875 0.28125 -0.46875q0 -0.203125 -0.125 -0.359375q-0.125 -0.15625 -0.453125 -0.3125q-0.328125 -0.171875 -0.921875 -0.390625q-1.140625 -0.421875 -1.625 -0.90625q-0.484375 -0.484375 -0.484375 -1.234375q0 -0.84375 0.671875 -1.359375q0.6875 -0.53125 1.78125 -0.53125q0.75 0 1.375 0.28125q0.640625 0.28125 1.09375 0.8125l-1.09375 0.90625q-0.59375 -0.609375 -1.40625 -0.609375q-0.40625 0 -0.65625 0.15625q-0.234375 0.140625 -0.234375 0.390625q0 0.25 0.28125 0.453125q0.296875 0.1875 1.09375 0.453125q0.78125 0.234375 1.265625 0.546875q0.5 0.3125 0.71875 0.734375q0.234375 0.40625 0.234375 0.953125q0 0.953125 -0.703125 1.5q-0.6875 0.546875 -1.90625 0.546875zm7.0155334 0q-0.984375 0 -1.6875 -0.4375q-0.703125 -0.4375 -1.09375 -1.234375q-0.375 -0.8125 -0.375 -1.890625q0 -1.109375 0.375 -1.90625q0.390625 -0.796875 1.09375 -1.21875q0.71875 -0.4375 1.6875 -0.4375q0.828125 0 1.5 0.34375q0.671875 0.34375 1.0625 1.0625q0.40625 0.71875 0.40625 1.828125q0 0.171875 -0.015625 0.421875q0 0.25 -0.03125 0.5l-4.4375 0q0.03125 0.46875 0.21875 0.828125q0.1875 0.34375 0.515625 0.53125q0.34375 0.1875 0.796875 0.1875q0.484375 0 0.84375 -0.15625q0.359375 -0.171875 0.671875 -0.53125l0.96875 1.0q-0.453125 0.5 -1.078125 0.8125q-0.609375 0.296875 -1.421875 0.296875zm-1.515625 -4.265625l2.875 0q-0.03125 -0.453125 -0.203125 -0.765625q-0.15625 -0.328125 -0.46875 -0.5q-0.296875 -0.171875 -0.71875 -0.171875q-0.59375 0 -1.0 0.359375q-0.40625 0.359375 -0.484375 1.078125z" fill-rule="nonzero"/>
+    <path fill="#93c47d" d="m482.62598 211.50919l0 0c0 -26.092194 21.151917 -47.244095 47.24408 -47.244095l0 0c12.529907 0 24.54663 4.9774933 33.406616 13.837479c8.859985 8.859985 13.837524 20.876709 13.837524 33.406616l0 0c0 26.092194 -21.151917 47.24411 -47.24414 47.24411l0 0c-26.092163 0 -47.24408 -21.151917 -47.24408 -47.24411z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m499.51492 216.58919l0 -7.484375l1.828125 0l0 0.796875q0.21875 -0.421875 0.65625 -0.703125q0.4375 -0.28125 0.921875 -0.28125q0.796875 0 1.34375 0.5625l-0.25 1.671875q-0.3125 -0.25 -0.59375 -0.34375q-0.28125 -0.109375 -0.625 -0.109375q-0.453125 0 -0.78125 0.21875q-0.328125 0.203125 -0.5 0.609375q-0.171875 0.40625 -0.171875 0.96875l0 4.09375l-1.828125 0zm8.824005 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm6.819702 4.515625l0 -9.75l1.828125 -0.828125l0 10.578125l-1.828125 0zm7.2805176 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm8.887024 4.6875q-1.234375 0 -1.953125 -0.65625q-0.703125 -0.671875 -0.703125 -1.796875q0 -0.734375 0.359375 -1.28125q0.375 -0.5625 1.03125 -0.859375q0.671875 -0.3125 1.5625 -0.3125q0.484375 0 0.9375 0.09375q0.46875 0.09375 0.8125 0.265625l0 -0.46875q0 -0.640625 -0.359375 -0.953125q-0.34375 -0.328125 -1.03125 -0.328125q-0.5625 0 -1.140625 0.203125q-0.578125 0.203125 -1.171875 0.59375l-0.546875 -1.453125q0.640625 -0.421875 1.4375 -0.65625q0.796875 -0.234375 1.59375 -0.234375q1.515625 0 2.265625 0.71875q0.75 0.71875 0.75 2.125l0 4.828125l-1.796875 0l0 -0.59375q-0.375 0.375 -0.90625 0.578125q-0.515625 0.1875 -1.140625 0.1875zm0.421875 -1.546875q0.453125 0 0.875 -0.171875q0.421875 -0.1875 0.75 -0.515625l0 -0.890625q-0.3125 -0.140625 -0.703125 -0.21875q-0.375 -0.09375 -0.765625 -0.09375q-0.671875 0 -1.0625 0.265625q-0.390625 0.25 -0.390625 0.71875q0 0.421875 0.34375 0.671875q0.34375 0.234375 0.953125 0.234375zm7.975891 1.546875q-0.875 0 -1.6875 -0.375q-0.796875 -0.375 -1.265625 -1.0l1.203125 -1.015625q0.375 0.40625 0.859375 0.640625q0.484375 0.21875 0.9375 0.21875q0.5625 0 0.859375 -0.1875q0.296875 -0.1875 0.296875 -0.515625q0 -0.21875 -0.140625 -0.390625q-0.140625 -0.171875 -0.5 -0.34375q-0.359375 -0.1875 -1.015625 -0.421875q-1.25 -0.46875 -1.78125 -1.0q-0.53125 -0.546875 -0.53125 -1.375q0 -0.921875 0.75 -1.5q0.75 -0.578125 1.953125 -0.578125q0.8125 0 1.5 0.3125q0.703125 0.3125 1.203125 0.90625l-1.203125 0.984375q-0.65625 -0.671875 -1.53125 -0.671875q-0.453125 0 -0.734375 0.171875q-0.265625 0.171875 -0.265625 0.4375q0 0.28125 0.3125 0.5q0.328125 0.203125 1.203125 0.484375q0.859375 0.28125 1.390625 0.625q0.546875 0.34375 0.796875 0.796875q0.265625 0.453125 0.265625 1.046875q0 1.046875 -0.765625 1.65625q-0.765625 0.59375 -2.109375 0.59375zm4.7806396 -0.171875l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm3.210144 8.421875l0 -7.484375l1.828125 0l0 0.71875q0.359375 -0.453125 0.859375 -0.671875q0.515625 -0.234375 1.15625 -0.234375q1.25 0 1.921875 0.78125q0.6875 0.765625 0.6875 2.21875l0 4.671875l-1.8125 0l0 -4.109375q0 -0.9375 -0.328125 -1.375q-0.328125 -0.4375 -1.015625 -0.4375q-0.75 0 -1.109375 0.4375q-0.359375 0.4375 -0.359375 1.34375l0 4.140625l-1.828125 0zm11.390381 3.125l-1.0 -1.53125q1.0625 -0.078125 1.65625 -0.25q0.59375 -0.15625 0.828125 -0.453125q0.234375 -0.296875 0.234375 -0.78125l0 -0.546875q-0.578125 0.609375 -1.609375 0.609375q-0.984375 0 -1.6875 -0.46875q-0.703125 -0.46875 -1.078125 -1.34375q-0.375 -0.890625 -0.375 -2.109375q0 -1.1875 0.390625 -2.0625q0.390625 -0.890625 1.09375 -1.375q0.71875 -0.484375 1.671875 -0.484375q0.484375 0 0.890625 0.15625q0.421875 0.15625 0.703125 0.453125l0 -0.421875l1.828125 0l0 6.8125q0 1.0 -0.1875 1.6875q-0.1875 0.6875 -0.609375 1.125q-0.40625 0.4375 -1.078125 0.671875q-0.671875 0.234375 -1.671875 0.3125zm0.40625 -4.703125q0.390625 0 0.71875 -0.171875q0.34375 -0.1875 0.59375 -0.5l0 -3.015625q-0.203125 -0.28125 -0.5625 -0.46875q-0.359375 -0.1875 -0.734375 -0.1875q-0.78125 0 -1.203125 0.5625q-0.40625 0.546875 -0.40625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.546875 0.75q0.359375 0.25 0.875 0.25z" fill-rule="nonzero"/>
+    <path fill="#93c47d" d="m509.6117 226.56891l40.503967 0l0 17.763779l-40.503967 0z" fill-rule="evenodd"/>
+    <path fill="#1155cc" d="m523.7518 243.36766q-1.78125 -1.109375 -2.71875 -2.78125q-0.9375 -1.671875 -0.9375 -3.71875q0 -2.03125 0.9375 -3.703125q0.953125 -1.6875 2.734375 -2.796875l1.3125 1.234375q-1.34375 0.9375 -2.015625 2.328125q-0.65625 1.375 -0.65625 2.9375q0 1.5625 0.65625 2.953125q0.671875 1.375 2.015625 2.3125l-1.328125 1.234375zm5.005554 -2.59375q-0.96875 0 -1.75 -0.515625q-0.765625 -0.515625 -1.21875 -1.421875q-0.453125 -0.921875 -0.453125 -2.09375q0 -1.171875 0.453125 -2.078125q0.453125 -0.921875 1.234375 -1.4375q0.796875 -0.515625 1.8125 -0.515625q0.5625 0 1.015625 0.15625q0.46875 0.15625 0.8125 0.453125q0.359375 0.28125 0.609375 0.65625q0.25 0.375 0.359375 0.796875l-0.484375 -0.0625l0 -1.84375l2.328125 0l0 7.78125l-2.359375 0l0 -1.875l0.515625 -0.015625q-0.109375 0.40625 -0.375 0.78125q-0.265625 0.359375 -0.65625 0.640625q-0.375 0.28125 -0.84375 0.4375q-0.46875 0.15625 -1.0 0.15625zm0.65625 -1.96875q0.53125 0 0.921875 -0.25q0.40625 -0.25 0.625 -0.703125q0.21875 -0.46875 0.21875 -1.109375q0 -0.625 -0.21875 -1.078125q-0.21875 -0.46875 -0.625 -0.71875q-0.390625 -0.265625 -0.921875 -0.265625q-0.53125 0 -0.921875 0.265625q-0.390625 0.25 -0.609375 0.71875q-0.21875 0.453125 -0.21875 1.078125q0 0.640625 0.21875 1.109375q0.21875 0.453125 0.609375 0.703125q0.390625 0.25 0.921875 0.25zm6.5491333 4.5625l-1.328125 -1.234375q1.359375 -0.9375 2.015625 -2.3125q0.65625 -1.390625 0.65625 -2.953125q0 -1.5625 -0.65625 -2.9375q-0.65625 -1.390625 -2.015625 -2.328125l1.34375 -1.234375q1.78125 1.109375 2.71875 2.78125q0.9375 1.671875 0.9375 3.71875q0 2.03125 -0.953125 3.71875q-0.9375 1.671875 -2.71875 2.78125z" fill-rule="nonzero"/>
+    <path fill="#6aa84f" d="m640.2651 127.33071l0 0c0 -26.092194 21.151917 -47.244095 47.24414 -47.244095l0 0c12.529907 0 24.54663 4.9774857 33.406616 13.837471c8.859985 8.859985 13.837463 20.876709 13.837463 33.406624l0 0c0 26.092186 -21.151917 47.244087 -47.24408 47.244087l0 0c-26.092224 0 -47.24414 -21.151901 -47.24414 -47.244087z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m659.46234 132.4107l0 -7.484375l1.828125 0l0 0.796875q0.21875 -0.421875 0.65625 -0.703125q0.4375 -0.28125 0.921875 -0.28125q0.796875 0 1.34375 0.5625l-0.25 1.671875q-0.3125 -0.25 -0.59375 -0.34375q-0.28125 -0.109375 -0.625 -0.109375q-0.453125 0 -0.78125 0.21875q-0.328125 0.203125 -0.5 0.609375q-0.171875 0.40625 -0.171875 0.96875l0 4.09375l-1.828125 0zm8.824036 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm6.819641 4.515625l0 -9.75l1.828125 -0.828125l0 10.578125l-1.828125 0zm7.2805786 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm8.886963 4.6875q-1.234375 0 -1.953125 -0.65625q-0.703125 -0.671875 -0.703125 -1.796875q0 -0.734375 0.359375 -1.28125q0.375 -0.5625 1.03125 -0.859375q0.671875 -0.3125 1.5625 -0.3125q0.484375 0 0.9375 0.09375q0.46875 0.09375 0.8125 0.265625l0 -0.46875q0 -0.640625 -0.359375 -0.953125q-0.34375 -0.328125 -1.03125 -0.328125q-0.5625 0 -1.140625 0.203125q-0.578125 0.203125 -1.171875 0.59375l-0.546875 -1.453125q0.640625 -0.421875 1.4375 -0.65625q0.796875 -0.234375 1.59375 -0.234375q1.515625 0 2.265625 0.71875q0.75 0.71875 0.75 2.125l0 4.828125l-1.796875 0l0 -0.59375q-0.375 0.375 -0.90625 0.578125q-0.515625 0.1875 -1.140625 0.1875zm0.421875 -1.546875q0.453125 0 0.875 -0.171875q0.421875 -0.1875 0.75 -0.515625l0 -0.890625q-0.3125 -0.140625 -0.703125 -0.21875q-0.375 -0.09375 -0.765625 -0.09375q-0.671875 0 -1.0625 0.265625q-0.390625 0.25 -0.390625 0.71875q0 0.421875 0.34375 0.671875q0.34375 0.234375 0.953125 0.234375zm7.975952 1.546875q-0.875 0 -1.6875 -0.375q-0.796875 -0.375 -1.265625 -1.0l1.203125 -1.015625q0.375 0.40625 0.859375 0.640625q0.484375 0.21875 0.9375 0.21875q0.5625 0 0.859375 -0.1875q0.296875 -0.1875 0.296875 -0.515625q0 -0.21875 -0.140625 -0.390625q-0.140625 -0.171875 -0.5 -0.34375q-0.359375 -0.1875 -1.015625 -0.421875q-1.25 -0.46875 -1.78125 -1.0q-0.53125 -0.546875 -0.53125 -1.375q0 -0.921875 0.75 -1.5q0.75 -0.578125 1.953125 -0.578125q0.8125 0 1.5 0.3125q0.703125 0.3125 1.203125 0.90625l-1.203125 0.984375q-0.65625 -0.671875 -1.53125 -0.671875q-0.453125 0 -0.734375 0.171875q-0.265625 0.171875 -0.265625 0.4375q0 0.28125 0.3125 0.5q0.328125 0.203125 1.203125 0.484375q0.859375 0.28125 1.390625 0.625q0.546875 0.34375 0.796875 0.796875q0.265625 0.453125 0.265625 1.046875q0 1.046875 -0.765625 1.65625q-0.765625 0.59375 -2.109375 0.59375zm7.7194214 0q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m667.25214 141.41537l40.503906 0l0 17.763779l-40.503906 0z" fill-rule="evenodd"/>
+    <path fill="#1155cc" d="m680.9524 158.21413q-1.78125 -1.109375 -2.71875 -2.78125q-0.9375 -1.671875 -0.9375 -3.71875q0 -2.03125 0.9375 -3.703125q0.953125 -1.6875 2.734375 -2.796875l1.3125 1.234375q-1.34375 0.9375 -2.015625 2.328125q-0.65625 1.375 -0.65625 2.9375q0 1.5625 0.65625 2.953125q0.671875 1.375 2.015625 2.3125l-1.328125 1.234375zm1.4503174 -2.71875l4.0625 -10.265625l2.109375 0l4.03125 10.265625l-2.515625 0l-1.9375 -5.203125q-0.125 -0.3125 -0.25 -0.65625q-0.109375 -0.359375 -0.234375 -0.734375q-0.109375 -0.375 -0.21875 -0.734375q-0.109375 -0.359375 -0.1875 -0.65625l0.453125 -0.015625q-0.09375 0.359375 -0.203125 0.71875q-0.109375 0.34375 -0.21875 0.6875q-0.109375 0.34375 -0.25 0.6875q-0.125 0.34375 -0.25 0.71875l-1.921875 5.1875l-2.46875 0zm1.9375 -1.96875l0.78125 -1.875l4.71875 0l0.75 1.875l-6.25 0zm9.702759 4.6875l-1.328125 -1.234375q1.359375 -0.9375 2.015625 -2.3125q0.65625 -1.390625 0.65625 -2.953125q0 -1.5625 -0.65625 -2.9375q-0.65625 -1.390625 -2.015625 -2.328125l1.34375 -1.234375q1.78125 1.109375 2.71875 2.78125q0.9375 1.671875 0.9375 3.71875q0 2.03125 -0.953125 3.71875q-0.9375 1.671875 -2.71875 2.78125z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m134.64568 293.8596l80.50394 0l0 16.15747l-80.50394 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m154.35117 306.65958l0 -6.8125l1.15625 0l0 6.8125l-1.15625 0zm0.5625 -8.046875q-0.296875 0 -0.515625 -0.203125q-0.203125 -0.21875 -0.203125 -0.515625q0 -0.3125 0.203125 -0.53125q0.21875 -0.21875 0.515625 -0.21875q0.3125 0 0.515625 0.21875q0.21875 0.21875 0.21875 0.53125q0 0.296875 -0.21875 0.515625q-0.203125 0.203125 -0.515625 0.203125zm2.8072052 8.046875l0 -6.8125l1.15625 0l0 0.6875q0.3125 -0.421875 0.75 -0.625q0.4375 -0.21875 1.015625 -0.21875q1.234375 0 1.828125 0.75q0.609375 0.75 0.609375 2.296875l0 3.921875l-1.171875 0l0 -4.125q0 -0.859375 -0.390625 -1.3125q-0.375 -0.46875 -1.0625 -0.46875q-0.78125 0 -1.1875 0.453125q-0.390625 0.4375 -0.390625 1.3125l0 4.140625l-1.15625 0zm10.12558 0.15625q-0.875 0 -1.515625 -0.421875q-0.625 -0.4375 -0.96875 -1.234375q-0.34375 -0.796875 -0.34375 -1.90625q0 -1.109375 0.34375 -1.90625q0.359375 -0.796875 1.0 -1.21875q0.640625 -0.4375 1.53125 -0.4375q0.890625 0 1.5 0.40625q0.609375 0.40625 0.890625 1.171875l-1.09375 0.40625q-0.15625 -0.4375 -0.5 -0.671875q-0.328125 -0.234375 -0.78125 -0.234375q-0.828125 0 -1.265625 0.640625q-0.4375 0.640625 -0.4375 1.84375q0 1.1875 0.4375 1.828125q0.4375 0.640625 1.234375 0.640625q0.515625 0 0.875 -0.265625q0.359375 -0.265625 0.5 -0.75l1.09375 0.359375q-0.25 0.84375 -0.90625 1.296875q-0.65625 0.453125 -1.59375 0.453125zm4.304657 -0.15625l0 -9.078125l1.15625 -0.546875l0 9.625l-1.15625 0zm5.8357086 0.15625q-1.25 0 -1.859375 -0.734375q-0.609375 -0.75 -0.609375 -2.296875l0 -3.9375l1.15625 0l0 4.109375q0 0.859375 0.375 1.328125q0.375 0.453125 1.09375 0.453125q0.765625 0 1.15625 -0.4375q0.40625 -0.453125 0.40625 -1.3125l0 -4.140625l1.15625 0l0 6.8125l-1.15625 0l0 -0.65625q-0.328125 0.421875 -0.734375 0.625q-0.40625 0.1875 -0.984375 0.1875zm7.437042 0q-1.296875 0 -2.0625 -0.953125q-0.75 -0.96875 -0.75 -2.609375q0 -1.09375 0.34375 -1.890625q0.34375 -0.8125 0.96875 -1.234375q0.640625 -0.4375 1.5 -0.4375q1.0 0 1.71875 0.640625l0 -2.75l1.15625 -0.546875l0 9.625l-1.15625 0l0 -0.53125q-0.671875 0.6875 -1.71875 0.6875zm0.15625 -1.078125q0.484375 0 0.890625 -0.234375q0.421875 -0.25 0.671875 -0.6875l0 -3.1875q-0.25 -0.40625 -0.671875 -0.640625q-0.40625 -0.234375 -0.890625 -0.234375q-0.859375 0 -1.328125 0.65625q-0.46875 0.640625 -0.46875 1.84375q0 0.734375 0.21875 1.296875q0.234375 0.5625 0.640625 0.875q0.40625 0.3125 0.9375 0.3125zm7.5528717 1.078125q-0.9375 0 -1.625 -0.453125q-0.671875 -0.46875 -1.046875 -1.265625q-0.359375 -0.8125 -0.359375 -1.84375q0 -1.125 0.375 -1.921875q0.390625 -0.796875 1.0625 -1.21875q0.6875 -0.421875 1.59375 -0.421875q0.765625 0 1.40625 0.34375q0.640625 0.34375 1.015625 1.0625q0.390625 0.71875 0.390625 1.828125q0 0.15625 -0.015625 0.375q0 0.203125 -0.03125 0.421875l-4.609375 0q0.0625 0.625 0.296875 1.09375q0.25 0.46875 0.640625 0.734375q0.390625 0.25 0.9375 0.25q0.421875 0 0.796875 -0.140625q0.375 -0.15625 0.6875 -0.484375l0.6875 0.765625q-0.390625 0.375 -0.953125 0.625q-0.5625 0.25 -1.25 0.25zm-1.859375 -4.09375l3.5 0q0 -0.59375 -0.203125 -1.046875q-0.1875 -0.453125 -0.546875 -0.703125q-0.359375 -0.265625 -0.890625 -0.265625q-0.75 0 -1.28125 0.484375q-0.515625 0.46875 -0.578125 1.53125z" fill-rule="nonzero"/>
+    <path fill="#e6b8af" fill-opacity="0.7" d="m640.2651 454.91733l0 0c0 -26.092194 21.151917 -47.24411 47.24414 -47.24411l0 0c12.529907 0 24.54663 4.9775085 33.406616 13.837494c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151917 47.24408 -47.24408 47.24408l0 0c-26.092224 0 -47.24414 -21.151886 -47.24414 -47.24408z" fill-rule="evenodd"/>
+    <path stroke="#9e9e9e" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m640.2651 454.91733l0 0c0 -26.092194 21.151917 -47.24411 47.24414 -47.24411l0 0c12.529907 0 24.54663 4.9775085 33.406616 13.837494c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151917 47.24408 -47.24408 47.24408l0 0c-26.092224 0 -47.24414 -21.151886 -47.24414 -47.24408z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m669.6953 459.9973l0 -5.890625l-1.046875 0l0 -1.59375l1.046875 0l0 -1.15625q0 -0.890625 0.46875 -1.34375q0.46875 -0.453125 1.359375 -0.453125q0.5 0 0.890625 0.09375q0.390625 0.09375 0.75 0.28125l0 1.484375q-0.296875 -0.125 -0.578125 -0.1875q-0.265625 -0.078125 -0.5 -0.078125q-0.28125 0 -0.421875 0.140625q-0.125 0.140625 -0.125 0.421875l0 0.796875l1.625 0l0 1.59375l-1.625 0l0 5.890625l-1.84375 0zm7.0739746 0.171875q-1.234375 0 -1.953125 -0.65625q-0.703125 -0.671875 -0.703125 -1.796875q0 -0.734375 0.359375 -1.28125q0.375 -0.5625 1.03125 -0.859375q0.671875 -0.3125 1.5625 -0.3125q0.484375 0 0.9375 0.09375q0.46875 0.09375 0.8125 0.265625l0 -0.46875q0 -0.640625 -0.359375 -0.953125q-0.34375 -0.328125 -1.03125 -0.328125q-0.5625 0 -1.140625 0.203125q-0.578125 0.203125 -1.171875 0.59375l-0.546875 -1.453125q0.640625 -0.421875 1.4375 -0.65625q0.796875 -0.234375 1.59375 -0.234375q1.515625 0 2.265625 0.71875q0.75 0.71875 0.75 2.125l0 4.828125l-1.796875 0l0 -0.59375q-0.375 0.375 -0.90625 0.578125q-0.515625 0.1875 -1.140625 0.1875zm0.421875 -1.546875q0.453125 0 0.875 -0.171875q0.421875 -0.1875 0.75 -0.515625l0 -0.890625q-0.3125 -0.140625 -0.703125 -0.21875q-0.375 -0.09375 -0.765625 -0.09375q-0.671875 0 -1.0625 0.265625q-0.390625 0.25 -0.390625 0.71875q0 0.421875 0.34375 0.671875q0.34375 0.234375 0.953125 0.234375zm5.538452 1.375l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm3.288269 8.421875l0 -9.75l1.828125 -0.828125l0 10.578125l-1.828125 0zm7.2805786 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m262.645 403.184c0 25.874023 188.80316 51.748047 377.6063 51.748047" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m262.645 403.18402c0 12.937012 47.200806 25.874023 118.00198 35.57675c35.400604 4.8513794 76.701294 8.894196 120.95203 11.724182c22.125366 1.414978 44.98822 2.526764 68.21991 3.2847595c11.615784 0.37902832 23.323792 0.6696167 35.07788 0.865448c5.877075 0.09790039 11.765625 0.17211914 17.659973 0.2218628c2.9472046 0.024871826 5.895813 0.043640137 8.845154 0.056152344l2.8494263 0.009094238" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m634.24866 456.57404l4.5407715 -1.6443787l-4.5354004 -1.6590881z" fill-rule="evenodd"/>
+    <path fill="#000000" fill-opacity="0.0" d="m734.7533 454.91733c56.04724 0 112.09448 -132.44095 112.09448 -264.8819" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m734.7533 454.91733c28.02362 0 56.04724 -33.11023 77.06494 -82.775604c10.50885 -24.832672 19.266235 -53.80414 25.396423 -84.84497c3.0650024 -15.520416 5.4733276 -31.558197 7.1154175 -47.854645c0.82092285 -8.148224 1.4503784 -16.361115 1.8745728 -24.606339c0.21209717 -4.1226044 0.37286377 -8.253296 0.48065186 -12.388031c0.053894043 -2.0673828 0.09448242 -4.1357574 0.12164307 -6.204651l0.002380371 -0.20777893" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m848.461 196.0459l-1.6226196 -4.5485992l-1.6807861 4.52742z" fill-rule="evenodd"/>
+    <path fill="#a2c4c9" fill-opacity="0.7" d="m640.2651 284.64304l0 0c0 -26.092194 21.151917 -47.24408 47.24414 -47.24408l0 0c12.529907 0 24.54663 4.977478 33.406616 13.837463c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151917 47.24411 -47.24408 47.24411l0 0c-26.092224 0 -47.24414 -21.151917 -47.24414 -47.24411z" fill-rule="evenodd"/>
+    <path stroke="#9e9e9e" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m640.2651 284.64304l0 0c0 -26.092194 21.151917 -47.24408 47.24414 -47.24408l0 0c12.529907 0 24.54663 4.977478 33.406616 13.837463c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151917 47.24411 -47.24408 47.24411l0 0c-26.092224 0 -47.24414 -21.151917 -47.24414 -47.24411z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m665.982 289.89493q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm5.6864624 4.515625l2.734375 -3.890625l-2.546875 -3.59375l2.109375 0l1.03125 1.46875q0.125 0.1875 0.25 0.375q0.125 0.1875 0.203125 0.34375q0.09375 -0.15625 0.203125 -0.34375q0.125 -0.203125 0.25 -0.375l1.046875 -1.46875l2.109375 0l-2.546875 3.59375l2.734375 3.890625l-2.109375 0l-1.25 -1.75q-0.109375 -0.15625 -0.234375 -0.34375q-0.109375 -0.203125 -0.203125 -0.359375q-0.078125 0.15625 -0.203125 0.359375q-0.125 0.1875 -0.25 0.34375l-1.21875 1.75l-2.109375 0zm9.181641 2.96875l0 -10.453125l1.828125 0l0 0.546875q0.625 -0.734375 1.65625 -0.734375q0.953125 0 1.640625 0.484375q0.703125 0.46875 1.078125 1.34375q0.375 0.875 0.375 2.09375q0 1.1875 -0.390625 2.078125q-0.375 0.875 -1.078125 1.359375q-0.703125 0.484375 -1.640625 0.484375q-0.515625 0 -0.9375 -0.1875q-0.40625 -0.1875 -0.703125 -0.546875l0 2.703125l-1.828125 0.828125zm3.1875 -4.546875q0.75 0 1.15625 -0.5625q0.40625 -0.578125 0.40625 -1.609375q0 -1.03125 -0.421875 -1.59375q-0.40625 -0.578125 -1.140625 -0.578125q-0.4375 0 -0.78125 0.171875q-0.328125 0.15625 -0.578125 0.484375l0 3.015625q0.203125 0.328125 0.5625 0.5q0.375 0.171875 0.796875 0.171875zm5.2993774 1.578125l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm3.210144 8.421875l0 -7.484375l1.828125 0l0 0.796875q0.21875 -0.421875 0.65625 -0.703125q0.4375 -0.28125 0.921875 -0.28125q0.796875 0 1.34375 0.5625l-0.25 1.671875q-0.3125 -0.25 -0.59375 -0.34375q-0.28125 -0.109375 -0.625 -0.109375q-0.453125 0 -0.78125 0.21875q-0.328125 0.203125 -0.5 0.609375q-0.171875 0.40625 -0.171875 0.96875l0 4.09375l-1.828125 0zm8.824036 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m605.23755 211.41602l80.50391 0l0 16.15747l-80.50391 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m629.9058 225.95038q-0.203125 -0.375 -0.390625 -0.953125q-0.171875 -0.578125 -0.28125 -1.28125q-0.109375 -0.71875 -0.109375 -1.515625q0 -1.40625 0.25 -2.546875q0.265625 -1.15625 0.6875 -2.078125q0.421875 -0.921875 0.9375 -1.625q0.515625 -0.703125 1.015625 -1.21875l1.078125 0q-0.203125 0.21875 -0.5625 0.65625q-0.359375 0.4375 -0.75 1.09375q-0.390625 0.65625 -0.75 1.5q-0.359375 0.84375 -0.59375 1.875q-0.21875 1.015625 -0.21875 2.203125q0 0.9375 0.140625 1.75q0.140625 0.796875 0.3125 1.34375q0.1875 0.5625 0.296875 0.796875l-1.0625 0zm2.7771606 -1.734375l1.1875 -6.8125l1.140625 0l-0.140625 0.734375q0.359375 -0.4375 0.828125 -0.65625q0.484375 -0.234375 1.0 -0.234375q0.765625 0 1.234375 0.375q0.46875 0.359375 0.609375 1.0625q0.15625 0.6875 -0.015625 1.671875l-0.671875 3.859375l-1.140625 0l0.71875 -4.09375q0.15625 -0.875 -0.109375 -1.359375q-0.265625 -0.484375 -0.9375 -0.484375q-0.6875 0 -1.21875 0.515625q-0.515625 0.515625 -0.65625 1.34375l-0.703125 4.078125l-1.125 0zm9.194641 0.15625q-0.90625 0 -1.453125 -0.53125q-0.546875 -0.53125 -0.546875 -1.4375q0 -0.765625 0.34375 -1.34375q0.359375 -0.59375 1.0 -0.90625q0.640625 -0.328125 1.5 -0.328125q0.46875 0 0.890625 0.125q0.421875 0.125 0.765625 0.359375l0.125 -0.65625q0.125 -0.671875 -0.15625 -1.046875q-0.265625 -0.375 -0.890625 -0.375q-0.46875 0 -1.0625 0.203125q-0.578125 0.1875 -1.09375 0.515625l-0.140625 -0.9375q0.609375 -0.375 1.234375 -0.5625q0.625 -0.203125 1.1875 -0.203125q0.71875 0 1.21875 0.296875q0.5 0.296875 0.703125 0.84375q0.203125 0.546875 0.078125 1.28125l-0.796875 4.546875l-1.078125 0l0.09375 -0.5625q-0.375 0.34375 -0.890625 0.53125q-0.5 0.1875 -1.03125 0.1875zm0.234375 -0.984375q0.46875 0 0.984375 -0.234375q0.515625 -0.25 0.90625 -0.65625l0.203125 -1.1875q-0.34375 -0.234375 -0.734375 -0.359375q-0.375 -0.140625 -0.75 -0.140625q-0.546875 0 -0.9375 0.1875q-0.390625 0.171875 -0.609375 0.515625q-0.21875 0.328125 -0.21875 0.78125q0 0.5 0.3125 0.796875q0.3125 0.296875 0.84375 0.296875zm7.1783447 0.984375q-1.046875 0 -1.6875 -0.765625q-0.625 -0.765625 -0.625 -2.0625q0 -1.28125 0.421875 -2.25q0.421875 -0.96875 1.171875 -1.5q0.75 -0.546875 1.75 -0.546875q0.8125 0 1.390625 0.421875q0.578125 0.40625 0.75 1.15625l-1.078125 0.375q-0.09375 -0.4375 -0.40625 -0.671875q-0.296875 -0.25 -0.78125 -0.25q-0.609375 0 -1.078125 0.421875q-0.46875 0.40625 -0.734375 1.140625q-0.265625 0.71875 -0.265625 1.671875q0 0.859375 0.328125 1.34375q0.328125 0.46875 0.9375 0.46875q0.515625 0 0.90625 -0.25q0.40625 -0.265625 0.640625 -0.78125l0.953125 0.375q-0.359375 0.828125 -1.046875 1.265625q-0.671875 0.4375 -1.546875 0.4375zm3.9645386 -0.15625l1.609375 -9.1875l1.203125 -0.4375l-0.921875 5.203125l2.609375 -2.390625l1.359375 0l-2.140625 1.953125l1.59375 4.859375l-1.234375 0l-1.234375 -4.0625l-1.203125 1.109375l-0.515625 2.953125l-1.125 0zm6.1102905 1.734375q0.1875 -0.234375 0.515625 -0.65625q0.328125 -0.421875 0.703125 -1.0625q0.375 -0.625 0.71875 -1.4375q0.34375 -0.828125 0.5625 -1.828125q0.21875 -1.0 0.21875 -2.1875q0 -0.953125 -0.15625 -1.796875q-0.140625 -0.84375 -0.34375 -1.4375q-0.1875 -0.59375 -0.3125 -0.8125l1.046875 0q0.234375 0.375 0.421875 0.984375q0.203125 0.59375 0.328125 1.34375q0.125 0.734375 0.125 1.5625q0 1.375 -0.25 2.5q-0.234375 1.125 -0.640625 2.015625q-0.390625 0.890625 -0.890625 1.578125q-0.484375 0.703125 -0.984375 1.234375l-1.0625 0z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m549.6588 469.0433l80.50391 0l0 32.314972l-80.50391 0z" fill-rule="evenodd"/>
+    <path fill="#d9d9d9" d="m568.16284 483.57767q-0.203125 -0.375 -0.390625 -0.953125q-0.171875 -0.578125 -0.28125 -1.28125q-0.109375 -0.71875 -0.109375 -1.515625q0 -1.40625 0.25 -2.546875q0.265625 -1.15625 0.6875 -2.078125q0.421875 -0.921875 0.9375 -1.625q0.515625 -0.703125 1.015625 -1.21875l1.078125 0q-0.203125 0.21875 -0.5625 0.65625q-0.359375 0.4375 -0.75 1.09375q-0.390625 0.65625 -0.75 1.5q-0.359375 0.84375 -0.59375 1.875q-0.21875 1.015625 -0.21875 2.203125q0 0.9375 0.140625 1.75q0.140625 0.796875 0.3125 1.34375q0.1875 0.5625 0.296875 0.796875l-1.0625 0zm6.770508 0.9375l0.59375 -3.3125q-0.4375 0.40625 -0.875 0.609375q-0.4375 0.1875 -0.9375 0.1875q-0.6875 0 -1.1875 -0.328125q-0.484375 -0.328125 -0.75 -0.9375q-0.265625 -0.625 -0.265625 -1.46875q0 -1.28125 0.4375 -2.265625q0.4375 -1.0 1.1875 -1.5625q0.765625 -0.5625 1.75 -0.5625q0.53125 0 0.953125 0.203125q0.4375 0.203125 0.671875 0.5625l0.09375 -0.609375l1.125 0l-1.578125 9.03125l-1.21875 0.453125zm-0.90625 -3.5625q0.421875 0 0.859375 -0.234375q0.4375 -0.25 0.84375 -0.71875l0.546875 -3.15625q-0.140625 -0.4375 -0.5 -0.6875q-0.359375 -0.25 -0.859375 -0.25q-0.625 0 -1.140625 0.4375q-0.515625 0.421875 -0.8125 1.171875q-0.296875 0.734375 -0.296875 1.65625q0 0.859375 0.34375 1.328125q0.359375 0.453125 1.015625 0.453125zm6.9468994 1.046875q-0.765625 0 -1.25 -0.34375q-0.46875 -0.34375 -0.640625 -1.015625q-0.15625 -0.671875 0.015625 -1.640625l0.6875 -3.96875l1.125 0l-0.71875 4.046875q-0.15625 0.921875 0.109375 1.40625q0.28125 0.46875 0.96875 0.46875q0.765625 0 1.234375 -0.453125q0.484375 -0.453125 0.640625 -1.328125l0.71875 -4.140625l1.125 0l-1.1875 6.8125l-1.140625 0l0.125 -0.65625q-0.328125 0.40625 -0.78125 0.609375q-0.453125 0.203125 -1.03125 0.203125zm4.9536133 -0.15625l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm3.8880615 8.21875q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm7.4560547 4.09375q-0.5 0 -0.953125 -0.140625q-0.4375 -0.140625 -0.796875 -0.40625q-0.359375 -0.28125 -0.59375 -0.65625l0.84375 -0.65625q0.28125 0.40625 0.6875 0.640625q0.421875 0.234375 0.859375 0.234375q0.609375 0 0.96875 -0.28125q0.359375 -0.28125 0.359375 -0.75q0 -0.328125 -0.28125 -0.59375q-0.265625 -0.265625 -0.984375 -0.59375q-0.9375 -0.4375 -1.328125 -0.890625q-0.390625 -0.46875 -0.390625 -1.09375q0 -0.578125 0.296875 -1.015625q0.3125 -0.4375 0.84375 -0.671875q0.53125 -0.25 1.234375 -0.25q0.71875 0 1.296875 0.28125q0.578125 0.265625 0.890625 0.765625l-0.8125 0.625q-0.578125 -0.703125 -1.453125 -0.703125q-0.53125 0 -0.875 0.265625q-0.34375 0.25 -0.34375 0.640625q0 0.296875 0.25 0.546875q0.25 0.234375 0.921875 0.515625q1.015625 0.453125 1.4375 0.953125q0.421875 0.5 0.421875 1.203125q0 0.921875 -0.6875 1.484375q-0.671875 0.546875 -1.8125 0.546875zm6.1588745 0q-1.046875 0 -1.6875 -0.765625q-0.625 -0.765625 -0.625 -2.0625q0 -1.28125 0.421875 -2.25q0.421875 -0.96875 1.171875 -1.5q0.75 -0.546875 1.75 -0.546875q0.8125 0 1.390625 0.421875q0.578125 0.40625 0.75 1.15625l-1.078125 0.375q-0.09375 -0.4375 -0.40625 -0.671875q-0.296875 -0.25 -0.78125 -0.25q-0.609375 0 -1.078125 0.421875q-0.46875 0.40625 -0.734375 1.140625q-0.265625 0.71875 -0.265625 1.671875q0 0.859375 0.328125 1.34375q0.328125 0.46875 0.9375 0.46875q0.515625 0 0.90625 -0.25q0.40625 -0.265625 0.640625 -0.78125l0.953125 0.375q-0.359375 0.828125 -1.046875 1.265625q-0.671875 0.4375 -1.546875 0.4375zm6.544983 0q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375z" fill-rule="nonzero"/>
+    <path fill="#d9d9d9" d="m570.6471 497.8433l1.03125 -5.796875l-1.0625 0l0.171875 -1.015625l1.0625 0l0.21875 -1.1875q0.125 -0.734375 0.5625 -1.109375q0.4375 -0.390625 1.140625 -0.390625q0.296875 0 0.5625 0.078125q0.28125 0.078125 0.578125 0.25l-0.171875 0.96875q-0.25 -0.140625 -0.46875 -0.203125q-0.203125 -0.0625 -0.421875 -0.0625q-0.28125 0 -0.453125 0.1875q-0.171875 0.171875 -0.234375 0.53125l-0.171875 0.9375l1.484375 0l-0.171875 1.015625l-1.5 0l-1.015625 5.796875l-1.140625 0zm5.89917 0.15625q-0.90625 0 -1.453125 -0.53125q-0.546875 -0.53125 -0.546875 -1.4375q0 -0.765625 0.34375 -1.34375q0.359375 -0.59375 1.0 -0.90625q0.640625 -0.328125 1.5 -0.328125q0.46875 0 0.890625 0.125q0.421875 0.125 0.765625 0.359375l0.125 -0.65625q0.125 -0.671875 -0.15625 -1.046875q-0.265625 -0.375 -0.890625 -0.375q-0.46875 0 -1.0625 0.203125q-0.578125 0.1875 -1.09375 0.515625l-0.140625 -0.9375q0.609375 -0.375 1.234375 -0.5625q0.625 -0.203125 1.1875 -0.203125q0.71875 0 1.21875 0.296875q0.5 0.296875 0.703125 0.84375q0.203125 0.546875 0.078125 1.28125l-0.796875 4.546875l-1.078125 0l0.09375 -0.5625q-0.375 0.34375 -0.890625 0.53125q-0.5 0.1875 -1.03125 0.1875zm0.234375 -0.984375q0.46875 0 0.984375 -0.234375q0.515625 -0.25 0.90625 -0.65625l0.203125 -1.1875q-0.34375 -0.234375 -0.734375 -0.359375q-0.375 -0.140625 -0.75 -0.140625q-0.546875 0 -0.9375 0.1875q-0.390625 0.171875 -0.609375 0.515625q-0.21875 0.328125 -0.21875 0.78125q0 0.5 0.3125 0.796875q0.3125 0.296875 0.84375 0.296875zm4.7053833 0.828125l1.203125 -6.8125l1.125 0l-1.203125 6.8125l-1.125 0zm1.984375 -8.0625q-0.25 0 -0.421875 -0.171875q-0.171875 -0.1875 -0.171875 -0.421875q0 -0.34375 0.234375 -0.59375q0.25 -0.25 0.5625 -0.25q0.265625 0 0.4375 0.171875q0.171875 0.171875 0.171875 0.421875q0 0.34375 -0.25 0.59375q-0.234375 0.25 -0.5625 0.25zm1.3411865 8.0625l1.625 -9.1875l1.1875 -0.4375l-1.6875 9.625l-1.125 0zm5.6486816 0.15625q-0.765625 0 -1.25 -0.34375q-0.46875 -0.34375 -0.640625 -1.015625q-0.15625 -0.671875 0.015625 -1.640625l0.6875 -3.96875l1.125 0l-0.71875 4.046875q-0.15625 0.921875 0.109375 1.40625q0.28125 0.46875 0.96875 0.46875q0.765625 0 1.234375 -0.453125q0.484375 -0.453125 0.640625 -1.328125l0.71875 -4.140625l1.125 0l-1.1875 6.8125l-1.140625 0l0.125 -0.65625q-0.328125 0.40625 -0.78125 0.609375q-0.453125 0.203125 -1.03125 0.203125zm5.0202637 -0.15625l1.1875 -6.8125l1.140625 0l-0.15625 0.875q0.359375 -0.484375 0.828125 -0.75q0.484375 -0.28125 0.96875 -0.28125q0.5625 0 0.890625 0.328125l-0.34375 1.078125q-0.203125 -0.15625 -0.421875 -0.21875q-0.21875 -0.0625 -0.453125 -0.0625q-0.421875 0 -0.8125 0.25q-0.375 0.25 -0.65625 0.703125q-0.265625 0.453125 -0.359375 1.046875l-0.6875 3.84375l-1.125 0zm7.3184204 0.15625q-0.78125 0 -1.34375 -0.359375q-0.5625 -0.375 -0.859375 -1.015625q-0.296875 -0.65625 -0.296875 -1.484375q0 -0.890625 0.25 -1.65625q0.25 -0.78125 0.703125 -1.359375q0.46875 -0.59375 1.109375 -0.921875q0.640625 -0.328125 1.40625 -0.328125q0.71875 0 1.25 0.28125q0.53125 0.265625 0.828125 0.84375q0.296875 0.5625 0.296875 1.453125q0 0.40625 -0.078125 0.765625q-0.078125 0.359375 -0.171875 0.671875l-4.4375 0q0 0.640625 0.140625 1.109375q0.15625 0.46875 0.484375 0.734375q0.328125 0.25 0.859375 0.25q0.359375 0 0.765625 -0.140625q0.40625 -0.15625 0.78125 -0.453125l0.53125 0.796875q-0.453125 0.375 -1.015625 0.59375q-0.546875 0.21875 -1.203125 0.21875zm-1.1875 -4.09375l3.375 0q0.109375 -0.59375 0 -1.046875q-0.109375 -0.453125 -0.421875 -0.703125q-0.3125 -0.265625 -0.859375 -0.265625q-0.59375 0 -1.015625 0.28125q-0.421875 0.28125 -0.6875 0.75q-0.265625 0.453125 -0.390625 0.984375zm4.8706055 5.671875q0.1875 -0.234375 0.515625 -0.65625q0.328125 -0.421875 0.703125 -1.0625q0.375 -0.625 0.71875 -1.4375q0.34375 -0.828125 0.5625 -1.828125q0.21875 -1.0 0.21875 -2.1875q0 -0.953125 -0.15625 -1.796875q-0.140625 -0.84375 -0.34375 -1.4375q-0.1875 -0.59375 -0.3125 -0.8125l1.046875 0q0.234375 0.375 0.421875 0.984375q0.203125 0.59375 0.328125 1.34375q0.125 0.734375 0.125 1.5625q0 1.375 -0.25 2.5q-0.234375 1.125 -0.640625 2.015625q-0.390625 0.890625 -0.890625 1.578125q-0.484375 0.703125 -0.984375 1.234375l-1.0625 0z" fill-rule="nonzero"/>
+    <path fill="#000000" fill-opacity="0.0" d="m529.87006 258.7533c0 12.944855 55.1969 25.88974 110.39374 25.88974" fill-rule="evenodd"/>
+    <path stroke="#eeeeee" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m529.87006 258.7533c0 6.472412 13.799255 12.944855 34.498047 17.799194c10.349426 2.4271545 22.423706 4.4497986 35.360474 5.8656616c6.468384 0.70791626 13.152405 1.2641296 19.944214 1.6433716c3.395874 0.18963623 6.8187256 0.33502197 10.255066 0.4329834c0.8590698 0.024505615 1.7189941 0.046020508 2.5795898 0.06454468c0.43023682 0.009246826 0.86065674 0.01776123 1.2912598 0.025512695l0.465271 0.007537842" fill-rule="evenodd"/>
+    <path fill="#eeeeee" stroke="#eeeeee" stroke-width="1.0" stroke-linecap="butt" d="m634.25 286.24377l4.551941 -1.6131287l-4.523926 -1.6902161z" fill-rule="evenodd"/>
+    <path fill="#e6b8af" fill-opacity="0.7" d="m481.91077 372.0315l0 0c0 -26.092194 21.151886 -47.24408 47.24408 -47.24408l0 0c12.529907 0 24.54663 4.977478 33.406616 13.837463c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151855 47.24411 -47.24408 47.24411l0 0c-26.092194 0 -47.24408 -21.151917 -47.24408 -47.24411z" fill-rule="evenodd"/>
+    <path stroke="#9e9e9e" stroke-width="3.0" stroke-linejoin="round" stroke-linecap="butt" d="m481.91077 372.0315l0 0c0 -26.092194 21.151886 -47.24408 47.24408 -47.24408l0 0c12.529907 0 24.54663 4.977478 33.406616 13.837463c8.859985 8.859985 13.837463 20.876709 13.837463 33.406616l0 0c0 26.092194 -21.151855 47.24411 -47.24408 47.24411l0 0c-26.092194 0 -47.24408 -21.151917 -47.24408 -47.24411z" fill-rule="evenodd"/>
+    <path fill="#000000" d="m501.41577 377.28336q-0.890625 0 -1.34375 -0.484375q-0.453125 -0.5 -0.453125 -1.46875l0 -4.109375l-1.078125 0l0 -1.59375l1.078125 0l0 -2.21875l1.828125 -0.875l0 3.09375l1.703125 0l0 1.59375l-1.703125 0l0 3.78125q0 0.34375 0.140625 0.515625q0.140625 0.171875 0.453125 0.171875q0.53125 0 1.1875 -0.34375l-0.21875 1.609375q-0.328125 0.15625 -0.734375 0.234375q-0.390625 0.09375 -0.859375 0.09375zm3.2872925 -0.171875l0 -7.484375l1.828125 0l0 7.484375l-1.828125 0zm0.921875 -8.421875q-0.4375 0 -0.765625 -0.3125q-0.3125 -0.328125 -0.3125 -0.765625q0 -0.4375 0.3125 -0.75q0.3125 -0.3125 0.765625 -0.3125q0.453125 0 0.765625 0.3125q0.3125 0.296875 0.3125 0.75q0 0.4375 -0.3125 0.765625q-0.3125 0.3125 -0.765625 0.3125zm3.2101746 8.421875l0 -7.484375l1.828125 0l0 0.6875q0.359375 -0.453125 0.8125 -0.65625q0.453125 -0.21875 1.0312195 -0.21875q0.65625 0 1.125 0.28125q0.484375 0.265625 0.75 0.78125q0.375 -0.5 0.953125 -0.78125q0.59375 -0.28125 1.34375 -0.28125q1.375 0 2.078125 0.75q0.703125 0.734375 0.703125 2.1875l0 4.734375l-1.828125 0l0 -4.09375q0 -0.9375 -0.3125 -1.375q-0.296875 -0.453125 -0.953125 -0.453125q-0.4375 0 -0.734375 0.203125q-0.28125 0.1875 -0.4375 0.59375q-0.140625 0.390625 -0.140625 1.0l0 4.125l-1.8125 0l0 -4.09375q0 -0.953125 -0.3125 -1.390625q-0.296875 -0.4375 -0.9530945 -0.4375q-0.65625 0 -0.984375 0.4375q-0.328125 0.421875 -0.328125 1.34375l0 4.140625l-1.828125 0zm16.005768 0.171875q-1.078125 0 -1.859375 -0.484375q-0.78125 -0.484375 -1.203125 -1.359375q-0.421875 -0.890625 -0.421875 -2.078125q0 -1.21875 0.421875 -2.09375q0.4375 -0.875 1.21875 -1.34375q0.78125 -0.484375 1.84375 -0.484375q0.90625 0 1.640625 0.390625q0.75 0.375 1.1875 1.171875q0.4375 0.78125 0.4375 2.0q0 0.1875 -0.015625 0.46875q-0.015625 0.265625 -0.046875 0.546875l-4.875 0q0.03125 0.515625 0.234375 0.90625q0.203125 0.375 0.5625 0.59375q0.375 0.203125 0.875 0.203125q0.53125 0 0.921875 -0.171875q0.40625 -0.1875 0.75 -0.578125l1.0625 1.09375q-0.484375 0.546875 -1.171875 0.890625q-0.671875 0.328125 -1.5625 0.328125zm-1.671875 -4.6875l3.15625 0q-0.03125 -0.5 -0.21875 -0.84375q-0.1875 -0.359375 -0.515625 -0.546875q-0.328125 -0.203125 -0.796875 -0.203125q-0.640625 0 -1.09375 0.40625q-0.453125 0.390625 -0.53125 1.1875zm9.693054 4.6875q-1.4375 0 -2.265625 -1.03125q-0.8125 -1.046875 -0.8125 -2.890625q0 -1.171875 0.390625 -2.046875q0.390625 -0.890625 1.078125 -1.375q0.703125 -0.5 1.640625 -0.5q0.984375 0 1.625 0.734375l0 -2.8125l1.828125 -0.828125l0 10.578125l-1.828125 0l0 -0.609375q-0.53125 0.78125 -1.65625 0.78125zm0.3125 -1.75q0.46875 0 0.796875 -0.15625q0.328125 -0.171875 0.546875 -0.5l0 -3.03125q-0.21875 -0.296875 -0.59375 -0.46875q-0.359375 -0.1875 -0.75 -0.1875q-0.75 0 -1.15625 0.5625q-0.390625 0.5625 -0.390625 1.609375q0 0.6875 0.171875 1.171875q0.1875 0.484375 0.53125 0.75q0.34375 0.25 0.84375 0.25zm8.472229 1.75q-1.578125 0 -2.5 -1.046875q-0.90625 -1.046875 -0.90625 -2.859375q0 -1.203125 0.40625 -2.078125q0.421875 -0.890625 1.1875 -1.375q0.765625 -0.484375 1.8125 -0.484375q1.046875 0 1.8125 0.484375q0.765625 0.46875 1.171875 1.359375q0.40625 0.875 0.40625 2.078125q0 1.21875 -0.40625 2.109375q-0.40625 0.875 -1.171875 1.34375q-0.765625 0.46875 -1.8125 0.46875zm0 -1.75q0.75 0 1.15625 -0.5625q0.40625 -0.578125 0.40625 -1.609375q0 -1.015625 -0.421875 -1.59375q-0.40625 -0.578125 -1.140625 -0.578125q-0.734375 0 -1.15625 0.578125q-0.40625 0.578125 -0.40625 1.609375q0 1.015625 0.40625 1.59375q0.421875 0.5625 1.15625 0.5625zm7.8494873 1.75q-1.25 0 -1.953125 -0.78125q-0.6875 -0.78125 -0.6875 -2.203125l0 -4.671875l1.84375 0l0 4.09375q0 0.953125 0.3125 1.390625q0.328125 0.421875 1.03125 0.421875q0.734375 0 1.09375 -0.4375q0.359375 -0.4375 0.359375 -1.34375l0 -4.125l1.828125 0l0 7.484375l-1.828125 0l0 -0.703125q-0.34375 0.4375 -0.84375 0.65625q-0.5 0.21875 -1.15625 0.21875zm8.301392 0q-0.890625 0 -1.34375 -0.484375q-0.453125 -0.5 -0.453125 -1.46875l0 -4.109375l-1.078125 0l0 -1.59375l1.078125 0l0 -2.21875l1.828125 -0.875l0 3.09375l1.703125 0l0 1.59375l-1.703125 0l0 3.78125q0 0.34375 0.140625 0.515625q0.140625 0.171875 0.453125 0.171875q0.53125 0 1.1875 -0.34375l-0.21875 1.609375q-0.328125 0.15625 -0.734375 0.234375q-0.390625 0.09375 -0.859375 0.09375z" fill-rule="nonzero"/>
+  </g>
+</svg>
\ No newline at end of file

From edf4bce948477ebd57eaeb9eecdd4beae6a2c6a7 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 30 Nov 2023 16:42:22 +0200
Subject: [PATCH 2119/2492] mds/quiesce: MDSRankQuiesce - integration of the
 quiesce db manager

* create an instance of the QuiesceDbManager in the rank
* update membership with a new mdsmap
* add an admin socket command for sending requests to the manager

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/CMakeLists.txt    |   4 +
 src/mds/MDCache.h         |  73 ++++++++
 src/mds/MDSDaemon.cc      |  19 ++
 src/mds/MDSRank.cc        |  25 ++-
 src/mds/MDSRank.h         |   9 +
 src/mds/MDSRankQuiesce.cc | 385 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 514 insertions(+), 1 deletion(-)
 create mode 100644 src/mds/MDSRankQuiesce.cc

diff --git a/src/mds/CMakeLists.txt b/src/mds/CMakeLists.txt
index 88c8a1db0854..0c6c31a3c51a 100644
--- a/src/mds/CMakeLists.txt
+++ b/src/mds/CMakeLists.txt
@@ -42,6 +42,10 @@ set(mds_srcs
   MDSPinger.cc
   MetricAggregator.cc
   MetricsHandler.cc
+  QuiesceDbManager.cc
+  QuiesceAgent.cc
+  MDSRankQuiesce.cc
+  BoostUrlImpl.cc
   ${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
   ${CMAKE_SOURCE_DIR}/src/common/MemoryModel.cc
   ${CMAKE_SOURCE_DIR}/src/osdc/Journaler.cc
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 94347b8249cc..30e004b97a6e 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -15,6 +15,7 @@
 #define CEPH_MDCACHE_H
 
 #include <atomic>
+#include <chrono>
 #include <string_view>
 #include <thread>
 
@@ -137,6 +138,8 @@ static const int PREDIRTY_PRIMARY = 1; // primary dn, adjust nested accounting
 static const int PREDIRTY_DIR = 2;     // update parent dir mtime/size
 static const int PREDIRTY_SHALLOW = 4; // only go to immediate parent (for easier rollback)
 
+using namespace std::literals::chrono_literals;
+
 class MDCache {
  public:
   typedef std::map<mds_rank_t, ref_t<MCacheExpire>> expiremap;
@@ -525,6 +528,73 @@ class MDCache {
 			       std::map<dirfrag_t,std::vector<dirfrag_t> >& subtrees);
   ESubtreeMap *create_subtree_map();
 
+  class QuiesceStatistics {
+public:
+    void inc_inodes() {
+      inodes++;
+    }
+    void inc_inodes_quiesced() {
+      inodes_quiesced++;
+    }
+    uint64_t get_inodes() const {
+      return inodes;
+    }
+    uint64_t get_inodes_quiesced() const {
+      return inodes_quiesced;
+    }
+    void add_failed(const MDRequestRef& mdr, int rc) {
+      failed[mdr] = rc;
+    }
+    int get_failed(const MDRequestRef& mdr) const {
+      auto it = failed.find(mdr);
+      return it == failed.end() ? 0 : it->second;
+    }
+    const auto& get_failed() const {
+      return failed;
+    }
+    void dump(Formatter* f) const {
+      f->dump_unsigned("inodes", inodes);
+      f->dump_unsigned("inodes_quiesced", inodes_quiesced);
+      f->open_array_section("failed");
+      for (auto& [mdr, rc] : failed) {
+        f->open_object_section("failure");
+        f->dump_object("request", *mdr);
+        f->dump_int("result", rc);
+        f->close_section();
+      }
+      f->close_section();
+    }
+private:
+    uint64_t inodes = 0;
+    uint64_t inodes_quiesced = 0;
+    std::map<MDRequestRef, int> failed;
+  };
+  class C_MDS_QuiescePath : public MDSInternalContext {
+  public:
+    C_MDS_QuiescePath(MDCache *c, Context* _finisher=nullptr) :
+      MDSInternalContext(c->mds), cache(c), finisher(_finisher) {}
+    ~C_MDS_QuiescePath() {
+      if (finisher) {
+        finisher->complete(-ECANCELED);
+        finisher = nullptr;
+      }
+    }
+    void set_req(const MDRequestRef& _mdr) {
+      mdr = _mdr;
+    }
+    void finish(int r) override {
+      if (finisher) {
+        finisher->complete(r);
+        finisher = nullptr;
+      }
+    }
+    QuiesceStatistics qs;
+    MDCache *cache;
+    MDRequestRef mdr;
+    Context* finisher = nullptr;
+  };
+  MDRequestRef quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *f = nullptr, std::chrono::milliseconds delay = 0ms) { c->complete(-ENOTSUP); return nullptr; }
+
   void clean_open_file_lists();
   void dump_openfiles(Formatter *f);
   bool dump_inode(Formatter *f, uint64_t number);
@@ -1366,6 +1436,9 @@ class MDCache {
   void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op);
   void rollback_uncommitted_fragment(dirfrag_t basedirfrag, frag_vec_t&& old_frags);
 
+  void dispatch_quiesce_path(const MDRequestRef& mdr) { }
+  void dispatch_quiesce_inode(const MDRequestRef& mdr) { }
+
   void upkeep_main(void);
 
   uint64_t cache_memory_limit;
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index dc9ea99e6c73..e69bacf8f49d 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -458,6 +458,25 @@ void MDSDaemon::set_up_admin_socket()
 				     asok_hook,
 				     "Respawn this MDS");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("quiesce db "
+                                     "name=roots,type=CephString,n=N,req=false "
+                                     "-- "
+                                     "name=set_id,type=CephString,req=false "
+                                     "name=timeout,type=CephFloat,range=0,req=false "
+                                     "name=expiration,type=CephFloat,range=0,req=false "
+                                     "name=await_for,type=CephFloat,range=0,req=false "
+                                     "name=await,type=CephBool,req=false "
+                                     "name=if_version,type=CephInt,range=0,req=false "
+                                     "name=include,type=CephBool,req=false "
+                                     "name=exclude,type=CephBool,req=false "
+                                     "name=reset,type=CephBool,req=false "
+                                     "name=release,type=CephBool,req=false "
+                                     "name=query,type=CephBool,req=false "
+                                     "name=all,type=CephBool,req=false "
+                                     "name=cancel,type=CephBool,req=false",
+      asok_hook,
+      "submit queries to the local QuiesceDbManager");
+  ceph_assert(r == 0);
   r = admin_socket->register_command(
     "heap " \
     "name=heapcmd,type=CephChoices,strings="				\
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 9cedeea1b136..7d3f9cda9a22 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -18,6 +18,7 @@
 #include "common/errno.h"
 #include "common/likely.h"
 #include "common/async/blocked_completion.h"
+#include "common/cmdparse.h"
 
 #include "messages/MClientRequestForward.h"
 #include "messages/MMDSLoadTargets.h"
@@ -42,9 +43,13 @@
 #include "events/ELid.h"
 #include "Mutation.h"
 
-
 #include "MDSRank.h"
 
+#include "QuiesceDbManager.h"
+#include "QuiesceAgent.h"
+
+#include <cmath>
+
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds
 #undef dout_prefix
@@ -548,6 +553,8 @@ MDSRank::MDSRank(
   server = new Server(this, &metrics_handler);
   locker = new Locker(this, mdcache);
 
+  quiesce_db_manager.reset(new QuiesceDbManager());
+
   _heartbeat_reset_grace = g_conf().get_val<uint64_t>("mds_heartbeat_reset_grace");
   heartbeat_grace = g_conf().get_val<double>("mds_heartbeat_grace");
   op_tracker.set_complaint_and_threshold(cct->_conf->mds_op_complaint_time,
@@ -859,12 +866,21 @@ void MDSRankDispatcher::shutdown()
 
   progress_thread.shutdown();
 
+  if (quiesce_db_manager) {
+    // shutdown the manager
+    quiesce_db_manager->update_membership({});
+  }
+
   // release mds_lock for finisher/messenger threads (e.g.
   // MDSDaemon::ms_handle_reset called from Messenger).
   mds_lock.unlock();
 
   // shut down messenger
   messenger->shutdown();
+  if (quiesce_agent) {
+    // reset any tracked roots
+    quiesce_agent->shutdown();
+  }
 
   mds_lock.lock();
 
@@ -2131,6 +2147,8 @@ void MDSRank::active_start()
   mdcache->reissue_all_caps();
 
   finish_contexts(g_ceph_context, waiting_for_active);  // kick waiters
+
+  quiesce_agent_setup();
 }
 
 void MDSRank::recovery_done(int oldstate)
@@ -2588,6 +2606,8 @@ void MDSRankDispatcher::handle_mds_map(
     metric_aggregator->notify_mdsmap(*mdsmap);
   }
   metrics_handler.notify_mdsmap(*mdsmap);
+
+  quiesce_cluster_update();
 }
 
 void MDSRank::handle_mds_recovery(mds_rank_t who)
@@ -2935,6 +2955,9 @@ void MDSRankDispatcher::handle_asok_command(
       goto out;
     }
     damage_table.erase(id);
+  } else if (command == "quiesce db") {
+    command_quiesce_db(cmdmap, on_finish);
+    return;
   } else {
     r = -CEPHFS_ENOSYS;
   }
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index f8e186b56c73..ab321964c2cd 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -151,6 +151,8 @@ class MgrClient;
 class Finisher;
 class ScrubStack;
 class C_ExecAndReply;
+class QuiesceDbManager;
+class QuiesceAgent;
 
 /**
  * The public part of this class's interface is what's exposed to all
@@ -437,6 +439,9 @@ class MDSRank {
 
     bool cluster_degraded = false;
 
+    std::shared_ptr<QuiesceDbManager> quiesce_db_manager;
+    std::shared_ptr<QuiesceAgent> quiesce_agent;
+
     Finisher *finisher;
   protected:
     typedef enum {
@@ -528,6 +533,7 @@ class MDSRank {
     void command_dump_inode(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss);
     void command_dump_dir(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss);
     void command_cache_drop(uint64_t timeout, Formatter *f, Context *on_finish);
+    void command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int, const std::string&, bufferlist&)> on_finish);
 
     // FIXME the state machine logic should be separable from the dispatch
     // logic that calls it.
@@ -566,6 +572,9 @@ class MDSRank {
     void handle_mds_recovery(mds_rank_t who);
     void handle_mds_failure(mds_rank_t who);
 
+    void quiesce_cluster_update();
+    void quiesce_agent_setup();
+
     /* Update MDSMap export_targets for this rank. Called on ::tick(). */
     void update_targets();
 
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
new file mode 100644
index 000000000000..49e6f3952253
--- /dev/null
+++ b/src/mds/MDSRankQuiesce.cc
@@ -0,0 +1,385 @@
+#include "MDSRank.h"
+#include "MDCache.h"
+
+#include "QuiesceDbManager.h"
+#include "QuiesceAgent.h"
+#include <boost/url.hpp>
+#include <chrono>
+#include <ranges>
+#include <algorithm>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mds_quiesce
+#undef dout_prefix
+#define dout_prefix *_dout << "quiesce.mds." << whoami << " <" << __func__ << "> "
+
+#undef dout
+#define dout(lvl)                                                        \
+  do {                                                                   \
+    auto subsys = ceph_subsys_mds;                                       \
+    if ((dout_context)->_conf->subsys.should_gather(dout_subsys, lvl)) { \
+      subsys = dout_subsys;                                              \
+    }                                                                    \
+  dout_impl(dout_context, ceph::dout::need_dynamic(subsys), lvl) dout_prefix
+
+#undef dendl
+#define dendl \
+  dendl_impl; \
+  }           \
+  while (0)
+
+void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int, const std::string&, bufferlist&)> on_finish)
+{
+  // validate the command:
+  using ceph::common::cmd_getval;
+  using ceph::common::cmd_getval_or;
+  using std::chrono::duration_cast;
+  using dd = std::chrono::duration<double>;
+
+  bool op_include = cmd_getval_or<bool>(cmdmap, "include", false);
+  bool op_query = cmd_getval_or<bool>(cmdmap, "query", false);
+  bool op_exclude = cmd_getval_or<bool>(cmdmap, "exclude", false);
+  bool op_reset = cmd_getval_or<bool>(cmdmap, "reset", false);
+  bool op_release = cmd_getval_or<bool>(cmdmap, "release", false);
+  bool op_cancel = cmd_getval_or<bool>(cmdmap, "cancel", false);
+  bool all = cmd_getval_or<bool>(cmdmap, "all", false);
+  std::optional<std::string> set_id = cmd_getval<std::string>(cmdmap, "set_id");
+
+  auto roots = cmd_getval_or<std::vector<std::string>>(cmdmap, "roots", std::vector<std::string> {});
+
+  int all_ops = op_include + op_exclude + op_reset + op_release + op_cancel + op_query;
+
+  if (all_ops > 1) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operations [include, exclude, reset, release, cancel, query] are mutually exclusive", bl);
+    return;
+  } else if (all_ops == 0) {
+    op_include = true;
+  }
+
+  if ((op_release || op_cancel) && roots.size() > 0) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operations [release, cancel] don't take roots", bl);
+    return;
+  }
+
+  if (op_cancel && !set_id && !all) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operation `cancel` requires a `--set-id` or `--all` to cancel all active sets", bl);
+    return;
+  }
+
+  if (op_reset && !set_id) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operation `reset` requires a `--set-id`", bl);
+    return;
+  }
+
+  if (op_reset && roots.empty()) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operation `reset` expects at least one root", bl);
+    return;
+  }
+
+  if (op_query && roots.size() > 0) {
+    bufferlist bl;
+    on_finish(-EINVAL, "Operation `query` doesn't take any roots", bl);
+    return;
+  }
+
+  if (!quiesce_db_manager) {
+    bufferlist bl;
+    on_finish(-EFAULT, "No quiesce db manager instance", bl);
+    return;
+  }
+
+  struct Ctx : public QuiesceDbManager::RequestContext {
+    std::function<void(int, const std::string&, bufferlist&)> on_finish;
+    bool all = false;
+
+    double sec(QuiesceTimeInterval duration) {
+      return duration_cast<dd>(duration).count();
+    }
+
+    double age(QuiesceTimeInterval of, QuiesceTimeInterval ref) {
+      return sec(ref - of);
+    }
+
+    double age(QuiesceTimeInterval of = QuiesceTimeInterval::zero()) {
+      return age(of, response.db_age);
+    }
+
+    void finish(int rc)
+    {
+      auto f = Formatter::create_unique("json-pretty");
+      CachedStackStringStream css;
+      bufferlist outbl;
+
+      auto dump_seconds = [&f](const std::string_view& name, double seconds) {
+        f->dump_format_unquoted(name, "%0.1f", seconds);
+      };
+
+      f->open_object_section("response"); {
+        f->dump_int("epoch", response.db_version.epoch);
+        f->dump_int("set_version", response.db_version.set_version);
+        f->open_object_section("sets"); {
+          for (auto&& [set_id, set] : response.sets) {
+            if (!all && !set.is_active() && set_id != request.set_id) {
+              continue;
+            }
+            f->open_object_section(set_id); {
+              f->dump_int("version", set.version);
+              QuiesceTimeInterval ref = response.db_age;
+              if (!set.is_active()) {
+                ref = set.rstate.at_age;
+              }
+              dump_seconds("age_ref", age(ref));
+              f->open_object_section("state"); {
+                f->dump_string("name", quiesce_state_name(set.rstate.state));
+                dump_seconds("age", age(set.rstate.at_age, ref));
+              } f->close_section();
+              dump_seconds("timeout", sec(set.timeout));
+              dump_seconds("expiration", sec(set.expiration));
+              f->open_object_section("members"); {
+                for (auto&& [root, info] : set.members) {
+                  f->open_object_section(root); {
+                    f->dump_bool("excluded", info.excluded);
+                    f->open_object_section("state"); {
+                      f->dump_string("name", quiesce_state_name(info.rstate.state));
+                      dump_seconds("age", age(info.rstate.at_age, ref));
+                    } f->close_section();
+                  } f->close_section();
+                }
+              } f->close_section();
+            } f->close_section();
+          }
+        } f->close_section();
+      } f->close_section();
+
+      f->flush(outbl);
+      on_finish(rc, css->str(), outbl);
+    }
+  };
+
+  auto* ctx = new Ctx();
+
+  ctx->on_finish = std::move(on_finish);
+  ctx->all = all;
+
+  ctx->request.reset([&](auto& r) {
+    r.set_id = set_id;
+
+    if (op_include) {
+      r.include_roots(roots);
+    } else if (op_exclude) {
+      r.exclude_roots(roots);
+    } else if (op_reset) {
+      r.reset_roots(roots);
+    } else if (op_release) {
+      r.release_roots();
+    } else if (op_cancel) {
+      r.cancel_roots();
+    }
+
+    double timeout;
+
+    if (cmd_getval(cmdmap, "await_for", timeout)) {
+      r.await = duration_cast<QuiesceTimeInterval>(dd(timeout));
+    } else if (cmd_getval_or<bool>(cmdmap, "await", false)) {
+      r.await = QuiesceTimeInterval::max();
+    }
+
+    if (cmd_getval(cmdmap, "expiration", timeout)) {
+      r.expiration = duration_cast<QuiesceTimeInterval>(dd(timeout));
+    }
+
+    if (cmd_getval(cmdmap, "timeout", timeout)) {
+      r.timeout = duration_cast<QuiesceTimeInterval>(dd(timeout));
+    }
+
+    int64_t ifv;
+    if (cmd_getval(cmdmap, "if_version", ifv)) {
+      r.if_version = QuiesceSetVersion(ifv);
+    }
+  });
+
+  dout(20) << "Submitting a quiesce db request " << (set_id ? "for" : "without a") << " setid " << set_id.value_or("") << ", operation: " << ctx->request.op_string() << dendl;
+  int rc = quiesce_db_manager->submit_request(ctx);
+  if (rc != 0) {
+    bufferlist bl;
+    delete ctx;
+    on_finish(rc, "Error submitting the command to the local db manager", bl);
+  }
+}
+
+void MDSRank::quiesce_cluster_update() {
+  QuiesceClusterMembership membership;
+
+  mds_rank_t leader = 0; // MAYBE LATER: initialize this from the map
+
+  membership.epoch = mdsmap->get_epoch();
+  membership.leader = leader;
+  membership.me = whoami;
+  membership.fs_id = mdsmap->get_info(whoami).join_fscid;
+  membership.fs_name = mdsmap->get_fs_name();
+  mdsmap->get_mds_set(membership.members);
+
+  dout(5) << "epoch:" << membership.epoch << " leader:" << membership.leader << " members:" << membership.members << dendl;
+
+  membership.send_ack = [=,this](QuiesceMap&& ack) {
+    if (whoami == leader) {
+      // loopback
+      quiesce_db_manager->submit_ack_from(whoami, std::move(ack));
+      return 0;
+    } else {
+      // TODO: implement messaging
+      return -ENOTSUP;
+    }
+  };
+
+  membership.send_listing_to = [=](mds_rank_t to, QuiesceDbListing&& db) {
+    // TODO: implement messaging
+    return -ENOTSUP;
+  };
+
+  quiesce_db_manager->update_membership(membership);
+}
+
+void MDSRank::quiesce_agent_setup() {
+  // TODO: replace this with a non-debug implementation
+  //       Potentially, allow the debug interface under some runtime configuration
+
+  ceph_assert(quiesce_db_manager);
+
+  using RequestHandle = QuiesceInterface::RequestHandle;
+  using QuiescingRoot = std::pair<RequestHandle, Context*>;
+  auto quiesce_requests = std::make_shared<std::unordered_map<QuiesceRoot, QuiescingRoot>>();
+
+  QuiesceAgent::ControlInterface ci;
+
+  ci.submit_request = [this, quiesce_requests](QuiesceRoot root, Context* c)
+      -> std::optional<RequestHandle> {
+    auto uri = boost::urls::parse_uri_reference(root);
+    if (!uri) {
+      dout(5) << "error parsing the quiesce root as an URI: " << uri.error() << dendl;
+      c->complete(uri.error());
+      return std::nullopt;
+    } else {
+      dout(20) << "parsed root '" << root <<"' as : " << uri->path() << " " << uri->query() << dendl;
+    }
+
+    std::chrono::milliseconds quiesce_delay_ms = 0ms;
+    if (auto pit = uri->params().find("delayms"); pit != uri->params().end()) {
+      try {
+        quiesce_delay_ms = std::chrono::milliseconds((*pit).has_value ? std::stoul((*pit).value) : 1000);
+      } catch (...) {
+        dout(5) << "error parsing the time to quiesce for query: " << uri->query() << dendl;
+        c->complete(-EINVAL);
+        return std::nullopt;
+      }
+    }
+    std::optional<double> dummy_quiesce_after;
+    if (auto pit = uri->params().find("q"); pit != uri->params().end()) {
+      try {
+        dummy_quiesce_after = (*pit).has_value ? std::stod((*pit).value) : 1 /*second*/;
+      } catch (...) {
+        dout(5) << "error parsing the time for debug quiesce for query: " << uri->query() << dendl;
+        c->complete(-EINVAL);
+        return std::nullopt;
+      }
+    }
+
+    auto path = uri->path();
+    dout(20) << "got request to quiesce '" << path << "'" << dendl;
+
+    std::lock_guard l(mds_lock);
+
+    if (!dummy_quiesce_after) {
+      // the real deal!
+      auto qc = new MDCache::C_MDS_QuiescePath(mdcache, c);
+      auto mdr = mdcache->quiesce_path(filepath(path), qc, nullptr, quiesce_delay_ms);
+      return mdr ? mdr->reqid : std::optional<RequestHandle>();
+    } else {
+      /* dummy quiesce */
+      // always create a new request id
+      auto req_id = metareqid_t(entity_name_t::MDS(whoami), issue_tid());
+      auto [it, inserted] = quiesce_requests->try_emplace(path, req_id, c);
+
+      if (!inserted) {
+        dout(3) << "duplicate quiesce request for root '" << it->first << "'" << dendl;
+        // we must update the request id so that old one can't cancel this request.
+        it->second.first = req_id;
+        if (it->second.second) {
+          it->second.second->complete(-EINTR);
+          it->second.second = c;
+        } else {
+          // if we have no context, it means we've completed it
+          // since we weren't inserted, we must have successfully quiesced
+          c->complete(0);
+        }
+      } else {
+        // do quiesce if needed
+
+        auto quiesce_task = new LambdaContext([quiesce_requests, req_id, this](int) {
+          // the mds lock should be held by the timer
+          dout(20) << "quiesce_task: callback by the timer" << dendl;
+          auto it = std::ranges::find(*quiesce_requests, req_id, [](auto x) { return x.second.first; });
+          if (it != quiesce_requests->end() && it->second.second != nullptr) {
+            dout(20) << "quiesce_task: completing the root '" << it->first << "'" << dendl;
+            it->second.second->complete(0);
+            it->second.second = nullptr;
+          }
+          dout(20) << "quiesce_task: done" << dendl;
+        });
+
+        dout(20) << "scheduling a quiesce_task (" << quiesce_task
+                 << ") to fire after " << *dummy_quiesce_after
+                 << " seconds on timer " << &timer << dendl;
+        timer.add_event_after(*dummy_quiesce_after, quiesce_task);
+      }
+      return it->second.first;
+    }
+  };
+
+  ci.cancel_request = [this, quiesce_requests](RequestHandle h) {
+    std::lock_guard l(mds_lock);
+
+    if (mdcache->have_request(h)) {
+      auto qimdr = mdcache->request_get(h);
+      mdcache->request_kill(qimdr);
+      return 0;
+    }
+
+    auto it = std::ranges::find(*quiesce_requests, h, [](auto x) { return x.second.first; });
+    if (it != quiesce_requests->end()) {
+      if (auto ctx = it->second.second; ctx) {
+        dout(20) << "canceling request with id '" << h << "' for root '" << it->first << "'" << dendl;
+        ctx->complete(-ECANCELED);
+      }
+      quiesce_requests->erase(it);
+      return 0;
+    }
+
+    return ENOENT;
+  };
+
+  std::weak_ptr<QuiesceDbManager> weak_db_manager = quiesce_db_manager;
+  ci.agent_ack = [weak_db_manager](QuiesceMap && update) {
+    if (auto manager = weak_db_manager.lock()) {
+      return manager->submit_agent_ack( std::move(update));
+    } else {
+      return ENOENT;
+    }
+  };
+
+  quiesce_agent.reset(new QuiesceAgent(ci));
+
+  std::weak_ptr<QuiesceAgent> weak_agent = quiesce_agent;
+  quiesce_db_manager->reset_agent_callback([weak_agent](QuiesceMap& update) {
+    if (auto agent = weak_agent.lock()) {
+      return agent->db_update(update);
+    } else {
+      return false;
+    }
+  });
+};

From 82f3dbc3ef7b577219f8e1cfaeae4009a809029e Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 8 Feb 2024 14:07:28 +0200
Subject: [PATCH 2120/2492] mds,messages: quiesce db inter-rank messaging

Fixes: https://tracker.ceph.com/issues/63708
Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/MDSRank.cc                  |  12 ++
 src/mds/MDSRank.h                   |   1 +
 src/mds/MDSRankQuiesce.cc           | 286 ++++++++++++++++++++++++----
 src/mds/QuiesceDbEncoding.h         | 157 +++++++++++++++
 src/mds/QuiesceDbManager.cc         |   4 +-
 src/messages/MMDSQuiesceDbAck.h     |  63 ++++++
 src/messages/MMDSQuiesceDbListing.h |  63 ++++++
 src/msg/Message.cc                  |  10 +
 src/msg/Message.h                   |   2 +
 9 files changed, 557 insertions(+), 41 deletions(-)
 create mode 100644 src/mds/QuiesceDbEncoding.h
 create mode 100644 src/messages/MMDSQuiesceDbAck.h
 create mode 100644 src/messages/MMDSQuiesceDbListing.h

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 7d3f9cda9a22..12f6865b2bc1 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1065,6 +1065,10 @@ bool MDSRankDispatcher::ms_dispatch(const cref_t<Message> &m)
 
 bool MDSRank::_dispatch(const cref_t<Message> &m, bool new_msg)
 {
+  if (quiesce_dispatch(m)) {
+    return true;
+  }
+
   if (is_stale_message(m)) {
     return true;
   }
@@ -1216,6 +1220,7 @@ bool MDSRank::is_valid_message(const cref_t<Message> &m) {
     return true;
   }
 
+  dout(10) << "invalid message type: " << std::hex << type << std::dec << dendl;
   return false;
 }
 
@@ -1283,6 +1288,13 @@ void MDSRank::handle_message(const cref_t<Message> &m)
       }
       break;
 
+    case MSG_MDS_QUIESCE_DB_LISTING:
+    case MSG_MDS_QUIESCE_DB_ACK:
+      ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
+      quiesce_dispatch(m);
+      break;
+
+
     case MSG_MDS_LOCK:
     case MSG_MDS_INODEFILECAPS:
       ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS);
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index ab321964c2cd..83fc3109fdb4 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -574,6 +574,7 @@ class MDSRank {
 
     void quiesce_cluster_update();
     void quiesce_agent_setup();
+    bool quiesce_dispatch(const cref_t<Message> &m);
 
     /* Update MDSMap export_targets for this rank. Called on ::tick(). */
     void update_targets();
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 49e6f3952253..27425497b14c 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -3,10 +3,15 @@
 
 #include "QuiesceDbManager.h"
 #include "QuiesceAgent.h"
+
+#include "messages/MMDSQuiesceDbListing.h"
+#include "messages/MMDSQuiesceDbAck.h"
+
 #include <boost/url.hpp>
 #include <chrono>
 #include <ranges>
 #include <algorithm>
+#include <queue>
 
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds_quiesce
@@ -203,46 +208,203 @@ void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int,
     }
   });
 
-  dout(20) << "Submitting a quiesce db request " << (set_id ? "for" : "without a") << " setid " << set_id.value_or("") << ", operation: " << ctx->request.op_string() << dendl;
+  dout(20) << "Submitting " << ctx->request << dendl;
   int rc = quiesce_db_manager->submit_request(ctx);
   if (rc != 0) {
     bufferlist bl;
+    // on_finish was moved there, so should only call via the ctx.
+    ctx->on_finish(rc, "Error submitting the command to the local db manager", bl);
     delete ctx;
-    on_finish(rc, "Error submitting the command to the local db manager", bl);
   }
 }
 
+static void rebind_agent_callback(std::shared_ptr<QuiesceAgent> agt, std::shared_ptr<QuiesceDbManager> mgr) {
+  if (!agt || !mgr) {
+    return;
+  }
+  std::weak_ptr<QuiesceAgent> weak_agent = agt;
+  mgr->reset_agent_callback([weak_agent](QuiesceMap& update) {
+    if (auto agent = weak_agent.lock()) {
+      return agent->db_update(update);
+    } else {
+      return false;
+    }
+  });
+}
+
 void MDSRank::quiesce_cluster_update() {
+  // the quiesce leader is the lowest rank with the highest state up to ACTIVE
+  auto less_leader = [](MDSMap::mds_info_t const* l, MDSMap::mds_info_t const* r) {
+    ceph_assert(l->rank != MDS_RANK_NONE);
+    ceph_assert(r->rank != MDS_RANK_NONE);
+    ceph_assert(l->state <= MDSMap::STATE_ACTIVE);
+    ceph_assert(r->state <= MDSMap::STATE_ACTIVE);
+    if (l->rank == r->rank) {
+      return l->state < r->state;
+    } else {
+      return l->rank > r->rank;
+    }
+  };
+
+  std::priority_queue<MDSMap::mds_info_t const*, std::vector<MDSMap::mds_info_t const*>, decltype(less_leader)> member_info(less_leader);
   QuiesceClusterMembership membership;
 
-  mds_rank_t leader = 0; // MAYBE LATER: initialize this from the map
+  QuiesceInterface::PeerId me = mds_gid_t(monc->get_global_id());
+
+  for (auto&& [gid, info] : mdsmap->get_mds_info()) {
+    // if it has a rank and state <= ACTIVE, it's good enough
+    if (info.rank != MDS_RANK_NONE && info.state <= MDSMap::STATE_ACTIVE) {
+      member_info.push(&info);
+      membership.members.insert(info.global_id);
+    }
+  }
+
+  QuiesceInterface::PeerId leader = 
+    member_info.empty() 
+    ? QuiesceClusterMembership::INVALID_MEMBER 
+    : member_info.top()->global_id;
 
   membership.epoch = mdsmap->get_epoch();
   membership.leader = leader;
-  membership.me = whoami;
-  membership.fs_id = mdsmap->get_info(whoami).join_fscid;
+  membership.me = me;
   membership.fs_name = mdsmap->get_fs_name();
-  mdsmap->get_mds_set(membership.members);
 
-  dout(5) << "epoch:" << membership.epoch << " leader:" << membership.leader << " members:" << membership.members << dendl;
+  dout(5) << "epoch:" << membership.epoch << " me:" << me << " leader:" << leader << " members:" << membership.members 
+    << (mdsmap->is_degraded() ? " (degraded)" : "") << dendl;
 
-  membership.send_ack = [=,this](QuiesceMap&& ack) {
-    if (whoami == leader) {
-      // loopback
-      quiesce_db_manager->submit_ack_from(whoami, std::move(ack));
-      return 0;
-    } else {
-      // TODO: implement messaging
-      return -ENOTSUP;
+  if (leader != QuiesceClusterMembership::INVALID_MEMBER) {
+    membership.send_ack = [=, this](QuiesceMap&& ack) {
+      if (me == leader) {
+        // loopback
+        quiesce_db_manager->submit_ack_from(me, std::move(ack));
+        return 0;
+      } else {
+        std::lock_guard guard(mds_lock);
+
+        if (mdsmap->get_state_gid(leader) == MDSMap::STATE_NULL) {
+          dout(5) << "couldn't find the leader " << leader << " in the map" << dendl;
+          return -ENOENT;
+        }
+        auto addrs = mdsmap->get_info_gid(leader).addrs;
+
+        auto ack_msg = make_message<MMDSQuiesceDbAck>(me);
+        dout(10) << "sending ack " << ack << " to the leader " << leader << dendl;
+        ack_msg->diff_map = std::move(ack);
+        return send_message_mds(ack_msg, addrs);
+      }
+    };
+
+    membership.send_listing_to = [=, this](QuiesceInterface::PeerId to, QuiesceDbListing&& db) {
+      std::lock_guard guard(mds_lock);
+      if (mdsmap->get_state_gid(to) == MDSMap::STATE_NULL) {
+        dout(5) << "couldn't find the peer " << to << " in the map" << dendl;
+        return -ENOENT;
+      }
+      auto addrs = mdsmap->get_info_gid(to).addrs;
+      auto listing_msg = make_message<MMDSQuiesceDbListing>(me);
+      dout(10) << "sending listing " << db << " to the peer " << to << dendl;
+      listing_msg->db_listing = std::move(db);
+      return send_message_mds(listing_msg, addrs);
+    };
+  }
+
+  QuiesceDbManager::RequestContext* inject_request = nullptr;
+
+  bool degraded = mdsmap->is_degraded();
+
+  if (degraded && membership.is_leader()) {
+    dout(5) << "WARNING: injecting a cancel all request"
+      << " members: " << membership.members
+      << " in: " << mdsmap->get_num_in_mds() 
+      << " up: " << mdsmap->get_num_up_mds() 
+      << " sr: " << mdsmap->get_num_standby_replay_mds()
+      << dendl;
+    
+    struct CancelAll: public QuiesceDbManager::RequestContext {
+      mds_rank_t whoami;
+      CancelAll(mds_rank_t whoami) : whoami(whoami) {
+        request.cancel_roots();
+      }
+      void finish(int rc) override {
+        dout(rc == 0 ? 15 : 3) << "injected cancel all completed with rc: " << rc << dendl;
+      }
+    };
+
+    inject_request = new CancelAll(whoami);
+  }
+
+  if (!is_active()) {
+    quiesce_db_manager->reset_agent_callback([whoami = whoami, degraded, is_sr = is_standby_replay()](QuiesceMap& quiesce_map) {
+      for (auto it = quiesce_map.roots.begin(); it != quiesce_map.roots.end();) {
+        switch (it->second.state) {
+        case QS_QUIESCING:
+          if (degraded) {
+            it->second.state = QS_FAILED;
+            dout(3) << "DEGRADED RESPONDER: reporting '" << it->first << "' as " << it->second.state << dendl;
+            ++it;
+          } else if (is_sr) {
+            it->second.state = QS_QUIESCED;
+            dout(15) << "STANDBY REPLAY RESPONDER: reporting '" << it->first << "' as " << it->second.state << dendl;
+            ++it;
+          } else {
+            // just ack.
+            dout(20) << "INTACTIVE RESPONDER: reporting '" << it->first << "' as " << it->second.state << dendl;
+            it = quiesce_map.roots.erase(it);
+          }
+          break;
+        default:
+          it = quiesce_map.roots.erase(it);
+          break;
+        }
+      }
+      return true;
+    });
+
+    if (quiesce_agent) {
+      // reset the agent if it's present
+      // because it won't receive any more callbacks
+      quiesce_agent->reset_async();
     }
-  };
+  } else {
+    rebind_agent_callback(quiesce_agent, quiesce_db_manager);
+  }
 
-  membership.send_listing_to = [=](mds_rank_t to, QuiesceDbListing&& db) {
-    // TODO: implement messaging
-    return -ENOTSUP;
-  };
+  quiesce_db_manager->update_membership(membership, inject_request);
+}
 
-  quiesce_db_manager->update_membership(membership);
+bool MDSRank::quiesce_dispatch(const cref_t<Message> &m) {
+  switch(m->get_type()) {
+    case MSG_MDS_QUIESCE_DB_LISTING:
+    {
+      const auto& req = ref_cast<MMDSQuiesceDbListing>(m);
+      if (quiesce_db_manager) {
+        dout(10) << "got " << req->db_listing << " from peer " << req->gid << dendl;
+        int result = quiesce_db_manager->submit_listing_from(req->gid, std::move(req->db_listing));
+        if (result != 0) {
+          dout(3) << "error (" << result << ") submitting " << req->db_listing << " from peer " << req->gid << dendl;
+        }
+      } else {
+        dout(5) << "no db manager to process " << req->db_listing << dendl;
+      }
+      return true;
+    }
+    case MSG_MDS_QUIESCE_DB_ACK:
+    {
+      const auto& req = ref_cast<MMDSQuiesceDbAck>(m);
+      if (quiesce_db_manager) {
+        dout(10) << "got ack " << req->diff_map << " from peer " << req->gid << dendl;
+        int result = quiesce_db_manager->submit_ack_from(req->gid, std::move(req->diff_map));
+        if (result != 0) {
+          dout(3) << "error (" << result << ") submitting an ack from peer " << req->gid << dendl;
+        }
+      } else {
+        dout(5) << "no db manager to process an ack: " << req->diff_map << dendl;
+      }
+      return true;
+    }
+    default:
+      return false;
+  }
 }
 
 void MDSRank::quiesce_agent_setup() {
@@ -278,29 +440,62 @@ void MDSRank::quiesce_agent_setup() {
         return std::nullopt;
       }
     }
-    std::optional<double> dummy_quiesce_after;
+    std::optional<double> debug_quiesce_after;
     if (auto pit = uri->params().find("q"); pit != uri->params().end()) {
       try {
-        dummy_quiesce_after = (*pit).has_value ? std::stod((*pit).value) : 1 /*second*/;
+        debug_quiesce_after = (*pit).has_value ? std::stod((*pit).value) : 1 /*second*/;
       } catch (...) {
         dout(5) << "error parsing the time for debug quiesce for query: " << uri->query() << dendl;
         c->complete(-EINVAL);
         return std::nullopt;
       }
     }
+    std::optional<double> debug_fail_after;
+    if (auto pit = uri->params().find("f"); pit != uri->params().end()) {
+      try {
+        debug_fail_after = (*pit).has_value ? std::stod((*pit).value) : 1 /*second*/;
+      } catch (...) {
+        dout(5) << "error parsing the time for debug fail for query: " << uri->query() << dendl;
+        c->complete(-EINVAL);
+        return std::nullopt;
+      }
+    }
+    std::optional<mds_rank_t> debug_rank;
+    if (auto pit = uri->params().find("r"); pit != uri->params().end()) {
+      try {
+        if ((*pit).has_value) {
+          debug_rank = (mds_rank_t)std::stoul((*pit).value);
+        }
+      } catch (...) {
+        dout(5) << "error parsing the rank for debug pin for query: " << uri->query() << dendl;
+        c->complete(-EINVAL);
+        return std::nullopt;
+      }
+    }
+
+    if (debug_rank && (debug_rank >= mdsmap->get_max_mds())) {
+        dout(5) << "invalid rank: " << uri->query() << dendl;
+        c->complete(-EINVAL);
+        return std::nullopt;
+    }
 
     auto path = uri->path();
     dout(20) << "got request to quiesce '" << path << "'" << dendl;
 
     std::lock_guard l(mds_lock);
 
-    if (!dummy_quiesce_after) {
+    if (!debug_quiesce_after && !debug_fail_after && !debug_rank) {
       // the real deal!
+      if (mdsmap->is_degraded()) {
+        dout(3) << "DEGRADED: refusing to quiesce" << dendl;
+        c->complete(EPERM);
+        return std::nullopt;
+      }
       auto qc = new MDCache::C_MDS_QuiescePath(mdcache, c);
       auto mdr = mdcache->quiesce_path(filepath(path), qc, nullptr, quiesce_delay_ms);
       return mdr ? mdr->reqid : std::optional<RequestHandle>();
     } else {
-      /* dummy quiesce */
+      /* dummy quiesce/fail */
       // always create a new request id
       auto req_id = metareqid_t(entity_name_t::MDS(whoami), issue_tid());
       auto [it, inserted] = quiesce_requests->try_emplace(path, req_id, c);
@@ -317,25 +512,44 @@ void MDSRank::quiesce_agent_setup() {
           // since we weren't inserted, we must have successfully quiesced
           c->complete(0);
         }
+      } else if (debug_rank && (debug_rank != whoami)) {
+        // the root was pinned to a different rank
+        // we should acknowledge the quiesce regardless of the other flags
+        it->second.second->complete(0);
+        it->second.second = nullptr;
       } else {
-        // do quiesce if needed
+        // do quiesce or fail
+
+        bool do_fail = false;
+        double delay;
+        if (debug_quiesce_after.has_value() && debug_fail_after.has_value()) {
+          do_fail = debug_fail_after < debug_quiesce_after;
+        } else {
+          do_fail = debug_fail_after.has_value();
+        }
+
+        if (do_fail) {
+          delay = debug_fail_after.value();
+        } else {
+          delay = debug_quiesce_after.value();
+        }
 
-        auto quiesce_task = new LambdaContext([quiesce_requests, req_id, this](int) {
+        auto quiesce_task = new LambdaContext([quiesce_requests, req_id, do_fail, this](int) {
           // the mds lock should be held by the timer
           dout(20) << "quiesce_task: callback by the timer" << dendl;
           auto it = std::ranges::find(*quiesce_requests, req_id, [](auto x) { return x.second.first; });
           if (it != quiesce_requests->end() && it->second.second != nullptr) {
-            dout(20) << "quiesce_task: completing the root '" << it->first << "'" << dendl;
-            it->second.second->complete(0);
+            dout(20) << "quiesce_task: completing the root '" << it->first << "' as failed: " << do_fail << dendl;
+            it->second.second->complete(do_fail ? -EBADF : 0);
             it->second.second = nullptr;
           }
           dout(20) << "quiesce_task: done" << dendl;
         });
 
         dout(20) << "scheduling a quiesce_task (" << quiesce_task
-                 << ") to fire after " << *dummy_quiesce_after
+                 << ") to fire after " << delay
                  << " seconds on timer " << &timer << dendl;
-        timer.add_event_after(*dummy_quiesce_after, quiesce_task);
+        timer.add_event_after(delay, quiesce_task);
       }
       return it->second.first;
     }
@@ -373,13 +587,5 @@ void MDSRank::quiesce_agent_setup() {
   };
 
   quiesce_agent.reset(new QuiesceAgent(ci));
-
-  std::weak_ptr<QuiesceAgent> weak_agent = quiesce_agent;
-  quiesce_db_manager->reset_agent_callback([weak_agent](QuiesceMap& update) {
-    if (auto agent = weak_agent.lock()) {
-      return agent->db_update(update);
-    } else {
-      return false;
-    }
-  });
+  rebind_agent_callback(quiesce_agent, quiesce_db_manager);
 };
diff --git a/src/mds/QuiesceDbEncoding.h b/src/mds/QuiesceDbEncoding.h
new file mode 100644
index 000000000000..756e63cf9e3e
--- /dev/null
+++ b/src/mds/QuiesceDbEncoding.h
@@ -0,0 +1,157 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+#include "QuiesceDb.h"
+#include "include/encoding.h"
+#include <stdint.h>
+
+void encode(QuiesceDbVersion const& v, bufferlist& bl, uint64_t features = 0)
+{
+  encode(v.epoch, bl, features);
+  encode(v.set_version, bl, features);
+}
+
+void decode(QuiesceDbVersion& v, bufferlist::const_iterator& p)
+{
+  decode(v.epoch, p);
+  decode(v.set_version, p);
+}
+
+void encode(QuiesceState const & state, bufferlist& bl, uint64_t features=0)
+{
+  static_assert(QuiesceState::QS__MAX <= UINT8_MAX);
+  uint8_t v = (uint8_t)state;
+  encode(v, bl, features);
+}
+
+void decode(QuiesceState & state, bufferlist::const_iterator& p)
+{
+  uint8_t v = 0;
+  decode(v, p);
+  state = (QuiesceState)v;
+}
+
+void encode(QuiesceTimeInterval const & interval, bufferlist& bl, uint64_t features=0)
+{
+  encode(interval.count(), bl, features);
+}
+
+void decode(QuiesceTimeInterval & interval, bufferlist::const_iterator& p)
+{
+  QuiesceClock::rep count;
+  decode(count, p);
+  interval = QuiesceTimeInterval { count };
+}
+
+void encode(RecordedQuiesceState const& rstate, bufferlist& bl, uint64_t features = 0)
+{
+  encode(rstate.state, bl, features);
+  encode(rstate.at_age.count(), bl, features);
+}
+
+void decode(RecordedQuiesceState& rstate, bufferlist::const_iterator& p)
+{
+  decode(rstate.state, p);
+  decode(rstate.at_age, p);
+}
+
+void encode(QuiesceSet::MemberInfo const& member, bufferlist& bl, uint64_t features = 0)
+{
+  encode(member.rstate, bl, features);
+  encode(member.excluded, bl, features);
+}
+
+void decode(QuiesceSet::MemberInfo& member, bufferlist::const_iterator& p)
+{
+  decode(member.rstate, p);
+  decode(member.excluded, p);
+}
+
+void encode(QuiesceSet const& set, bufferlist& bl, uint64_t features = 0)
+{
+  encode(set.version, bl, features);
+  encode(set.rstate, bl, features);
+  encode(set.timeout, bl, features);
+  encode(set.expiration, bl, features);
+  encode(set.members, bl, features);
+}
+
+void decode(QuiesceSet& set, bufferlist::const_iterator& p)
+{
+  decode(set.version, p);
+  decode(set.rstate, p);
+  decode(set.timeout, p);
+  decode(set.expiration, p);
+  decode(set.members, p);
+}
+
+void encode(QuiesceDbRequest const& req, bufferlist& bl, uint64_t features = 0)
+{
+  encode(req.control.raw, bl, features);
+  encode(req.set_id, bl);
+  encode(req.if_version, bl);
+  encode(req.timeout, bl);
+  encode(req.expiration, bl);
+  encode(req.await, bl);
+  encode(req.roots, bl);
+}
+
+void decode(QuiesceDbRequest& req, bufferlist::const_iterator& p)
+{
+  decode(req.control.raw, p);
+  decode(req.set_id, p);
+  decode(req.if_version, p);
+  decode(req.timeout, p);
+  decode(req.expiration, p);
+  decode(req.await, p);
+  decode(req.roots, p);
+}
+
+void encode(QuiesceDbListing const& listing, bufferlist& bl, uint64_t features = 0)
+{
+  encode(listing.db_version, bl, features);
+  encode(listing.db_age, bl, features);
+  encode(listing.sets, bl, features);
+}
+
+void decode(QuiesceDbListing& listing, bufferlist::const_iterator& p)
+{
+  decode(listing.db_version, p);
+  decode(listing.db_age, p);
+  decode(listing.sets, p);
+}
+
+void encode(QuiesceMap::RootInfo const& root, bufferlist& bl, uint64_t features = 0)
+{
+  encode(root.state, bl, features);
+  encode(root.ttl, bl, features);
+}
+
+void decode(QuiesceMap::RootInfo& root, bufferlist::const_iterator& p)
+{
+  decode(root.state, p);
+  decode(root.ttl, p);
+}
+
+void encode(QuiesceMap const& map, bufferlist& bl, uint64_t features = 0)
+{
+  encode(map.db_version, bl, features);
+  encode(map.roots, bl, features);
+}
+
+void decode(QuiesceMap& map, bufferlist::const_iterator& p)
+{
+  decode(map.db_version, p);
+  decode(map.roots, p);
+}
+
diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
index 3eb9009f86b0..1ff998f60686 100644
--- a/src/mds/QuiesceDbManager.cc
+++ b/src/mds/QuiesceDbManager.cc
@@ -202,7 +202,9 @@ bool QuiesceDbManager::membership_upkeep()
   bool was_leader = membership.epoch > 0 && membership.leader == membership.me;
   bool is_leader = cluster_membership && cluster_membership->leader == cluster_membership->me;
   if (cluster_membership) {
-    dout(10) << "epoch: " << cluster_membership->epoch << " is_leader: " << is_leader << " was_leader: " << was_leader << dendl;
+    dout(10) << "epoch:" << cluster_membership->epoch << " leader:" 
+      << std::boolalpha << was_leader << "->" << is_leader << std::noboolalpha
+      << " members:" << cluster_membership->members << dendl;
   } else {
     dout(10) << "shutdown! was_leader: " << was_leader << dendl;
   }
diff --git a/src/messages/MMDSQuiesceDbAck.h b/src/messages/MMDSQuiesceDbAck.h
new file mode 100644
index 000000000000..907db239bd18
--- /dev/null
+++ b/src/messages/MMDSQuiesceDbAck.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#pragma once
+
+#include "messages/MMDSOp.h"
+#include "mds/QuiesceDbEncoding.h"
+
+class MMDSQuiesceDbAck final : public MMDSOp {
+public:
+  mds_gid_t gid;
+  mutable QuiesceMap diff_map;
+
+protected:
+  MMDSQuiesceDbAck(mds_gid_t gid) : MMDSOp{MSG_MDS_QUIESCE_DB_ACK}, gid(gid) {}
+  MMDSQuiesceDbAck() : MMDSQuiesceDbAck(MDS_GID_NONE) {}
+  ~MMDSQuiesceDbAck() final {}
+
+public:
+  std::string_view get_type_name() const override { return "mds_quiesce_db_ack"; }
+  void print(std::ostream& o) const override {
+
+  }
+
+  void encode_payload(uint64_t features) override
+  {
+    using ceph::encode;
+
+    ceph_assert(gid != MDS_GID_NONE);
+
+    ENCODE_START(1, 1, payload);
+    encode(gid, payload);
+    encode(diff_map, payload);
+    ENCODE_FINISH(payload);
+  }
+
+  void decode_payload() override {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    DECODE_START(1, p);
+    decode(gid, p);
+    decode(diff_map, p);
+    DECODE_FINISH(p);
+  }
+
+private:
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+  template<class T, typename... Args>
+  friend MURef<T> crimson::make_message(Args&&... args);
+};
diff --git a/src/messages/MMDSQuiesceDbListing.h b/src/messages/MMDSQuiesceDbListing.h
new file mode 100644
index 000000000000..9d3ce20fdab4
--- /dev/null
+++ b/src/messages/MMDSQuiesceDbListing.h
@@ -0,0 +1,63 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#pragma once
+
+#include "messages/MMDSOp.h"
+#include "mds/QuiesceDbEncoding.h"
+
+class MMDSQuiesceDbListing final : public MMDSOp {
+public:
+  mds_gid_t gid;
+  mutable QuiesceDbListing db_listing;
+
+protected:
+  MMDSQuiesceDbListing(mds_gid_t gid) : MMDSOp{MSG_MDS_QUIESCE_DB_LISTING}, gid(gid) {}
+  MMDSQuiesceDbListing() : MMDSQuiesceDbListing(MDS_GID_NONE) {}
+  ~MMDSQuiesceDbListing() final {}
+
+public:
+  std::string_view get_type_name() const override { return "mds_quiesce_db_listing"; }
+  void print(std::ostream& o) const override {
+
+  }
+
+  void encode_payload(uint64_t features) override
+  {
+    using ceph::encode;
+
+    ceph_assert(gid != MDS_GID_NONE);
+
+    ENCODE_START(1, 1, payload);
+    encode(gid, payload);
+    encode(db_listing, payload);
+    ENCODE_FINISH(payload);
+  }
+
+  void decode_payload() override {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    DECODE_START(1, p);
+    decode(gid, p);
+    decode(db_listing, p);
+    DECODE_FINISH(p);
+  }
+
+private:
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+  template<class T, typename... Args>
+  friend MURef<T> crimson::make_message(Args&&... args);
+};
diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 1faadb22a1cd..22208d2d1f42 100644
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -131,6 +131,8 @@
 #include "messages/MClientMetrics.h"
 
 #include "messages/MMDSPeerRequest.h"
+#include "messages/MMDSQuiesceDbListing.h"
+#include "messages/MMDSQuiesceDbAck.h"
 
 #include "messages/MMDSMap.h"
 #include "messages/MFSMap.h"
@@ -848,6 +850,14 @@ Message *decode_message(CephContext *cct,
 
   case MSG_MDS_TABLE_REQUEST:
     m = make_message<MMDSTableRequest>();
+    break;
+
+  case MSG_MDS_QUIESCE_DB_LISTING:
+    m = make_message<MMDSQuiesceDbListing>();
+    break;
+
+  case MSG_MDS_QUIESCE_DB_ACK:
+    m = make_message<MMDSQuiesceDbAck>();
     break;
 
 	/*  case MSG_MDS_INODEUPDATE:
diff --git a/src/msg/Message.h b/src/msg/Message.h
index e56e4bb1b6d0..15eb3feadced 100644
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -199,6 +199,8 @@
 #define MSG_MDS_METRICS            0x501  // for mds metric aggregator
 #define MSG_MDS_PING               0x502  // for mds pinger
 #define MSG_MDS_SCRUB_STATS        0x503  // for mds scrub stack
+#define MSG_MDS_QUIESCE_DB_LISTING 0x505  // quiesce db replication
+#define MSG_MDS_QUIESCE_DB_ACK     0x506  // quiesce agent ack back to the db
 
 // *** generic ***
 #define MSG_TIMECHECK             0x600

From 7e42824478ea24f588dabfaa8e317a99a0cee8cc Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 15 Feb 2024 15:15:09 +0200
Subject: [PATCH 2121/2492] mds/quiesce: only use ACTIVE daemons for the
 quiesce cluster

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/MDSRankQuiesce.cc    | 3 ++-
 src/pybind/mgr/mgr_module.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 27425497b14c..0b753865698c 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -253,7 +253,8 @@ void MDSRank::quiesce_cluster_update() {
 
   for (auto&& [gid, info] : mdsmap->get_mds_info()) {
     // if it has a rank and state <= ACTIVE, it's good enough
-    if (info.rank != MDS_RANK_NONE && info.state <= MDSMap::STATE_ACTIVE) {
+    // if (info.rank != MDS_RANK_NONE && info.state <= MDSMap::STATE_ACTIVE) {
+    if (info.rank != MDS_RANK_NONE && info.state == MDSMap::STATE_ACTIVE) {
       member_info.push(&info);
       membership.members.insert(info.global_id);
     }
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index e75a57659025..16cad3641824 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1774,7 +1774,7 @@ def get_quiesce_leader_info(self, fscid: str) -> dict:
                         state_ord = self.MDS_STATE_ORD.get(info['state'])
                         leader_state_ord = self.MDS_STATE_ORD.get(leader_info['state'])
 
-                        if state_ord <= self.MDS_STATE_ACTIVE_ORD and state_ord > leader_state_ord:
+                        if state_ord == self.MDS_STATE_ACTIVE_ORD and state_ord > leader_state_ord:
                             leader_info = info
             break
 

From 12d687bd4447687f34a43f637f16af72131189da Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 15 Feb 2024 19:16:53 +0200
Subject: [PATCH 2122/2492] pybind/mgr: correct type hints for
 `get_quiesce_leader_info`

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/pybind/mgr/mgr_module.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 16cad3641824..219afbf270fe 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1752,8 +1752,8 @@ def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf:
     }
     MDS_STATE_ACTIVE_ORD = MDS_STATE_ORD["up:active"]
 
-    def get_quiesce_leader_info(self, fscid: str) -> dict:
-        leader_info = None
+    def get_quiesce_leader_info(self, fscid: str) -> Optional[dict]:
+        leader_info: Optional[dict] = None
 
         for fs in self.get("fs_map")['filesystems']:
             if fscid != fs["id"]:
@@ -1772,16 +1772,16 @@ def get_quiesce_leader_info(self, fscid: str) -> dict:
                         leader_info = info
                     elif info['rank'] == leader_info['rank']:
                         state_ord = self.MDS_STATE_ORD.get(info['state'])
-                        leader_state_ord = self.MDS_STATE_ORD.get(leader_info['state'])
-
-                        if state_ord == self.MDS_STATE_ACTIVE_ORD and state_ord > leader_state_ord:
+                        # if there are more than one daemons with the same rank
+                        # only one of them can be active
+                        if state_ord == self.MDS_STATE_ACTIVE_ORD:
                             leader_info = info
             break
 
         return leader_info
 
-    def tell_quiesce_leader(self, fscid: str, cmd_dict: dict, inbuf: Optional[str] = None) -> Tuple[int, str, str]:
-        qleader: dict = self.get_quiesce_leader_info(fscid)
+    def tell_quiesce_leader(self, fscid: str, cmd_dict: dict) -> Tuple[int, str, str]:
+        qleader = self.get_quiesce_leader_info(fscid)
         if qleader is None:
             self.log.warn("Couldn't resolve the quiesce leader for fscid %s" % fscid)
             return (-errno.ENOENT, "", "Couldn't resolve the quiesce leader for fscid %s" % fscid)

From 629ffe1d317558a85d59d09e59f0ff117d71338e Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 26 Feb 2024 13:33:18 +0200
Subject: [PATCH 2123/2492] doc/cephfs/fs-volumes: doc fixes and updates

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 doc/cephfs/fs-volumes.rst | 82 ++++++++++++++++++++-------------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/doc/cephfs/fs-volumes.rst b/doc/cephfs/fs-volumes.rst
index 476e211c1587..4c14fd055ad8 100644
--- a/doc/cephfs/fs-volumes.rst
+++ b/doc/cephfs/fs-volumes.rst
@@ -306,9 +306,9 @@ Resize a subvolume using:
    ceph fs subvolume resize <vol_name> <subvol_name> <new_size> [--group_name <subvol_group_name>] [--no_shrink]
 
 The command resizes the subvolume quota using the size specified by ``new_size``.
-The ``--no_shrink`` flag prevents the subvolume from shrinking below the current  used size of the subvolume.
+The ``--no_shrink`` flag prevents the subvolume from shrinking below the current used size of the subvolume.
 
-The subvolume can be resized to an unlimited (but sparse) logical size by passing ``inf`` or ``infinite`` as `` new_size``.
+The subvolume can be resized to an unlimited (but sparse) logical size by passing ``inf`` or ``infinite`` as ``new_size``.
 
 Authorize cephx auth IDs, the read/read-write access to fs subvolumes:
 
@@ -790,22 +790,23 @@ Subvolume quiesce
 
 It may be needed to pause IO to a set of subvolumes of a given volume (file system).
 A good example of such case is a consistent snapshot spanning multiple subvolumes.
-Such a task arises often in an environment such as k8s, where a single deployed application
+The task arises often in an orchestrated environment such as Kubernetes, where a single deployed application
 can work with many mounted subvolumes across several hosts. When a snapshot of such a system is needed,
 the application may not find the result consistent unless the snapshots were taken
-under an active write pause.
+during an active write pause.
 
-The volumes plugin provides a tool to initiate and await such a pause across a set of subvolumes:
+The `volumes` plugin provides a tool to initiate and await such a pause across a set of subvolumes:
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce --set-id myset1 <vol_name> <[group_name/]sub_name...> --await
+  $ ceph fs quiesce <vol_name> --set-id myset1 <[group_name/]sub_name...> --await
   # perform actions while the IO pause is active, like taking snapshots
-  $ ceph fs quiesce --set-id myset1 --release --await
+  $ ceph fs quiesce <vol_name> --set-id myset1 --release --await
   # if successful, all members of the set were confirmed as still in pause and released from such
 
-The ``quiesce`` functionality is itself based on top of a lower level QuiesceDb service maintained by the MDS
-daemons. Volumes plugin merely maps the subvolume names to their corresponding paths on the given file system
+The ``quiesce`` functionality is itself based on a lower level QuiesceDb service provided by the MDS
+daemons, which operates at a file system path granularity. 
+The `volumes` plugin merely maps the subvolume names to their corresponding paths on the given file system
 and then issues the appropriate quiesce command to the MDS. You can learn more about the feature in the developer guides.
 
 Operations
@@ -822,8 +823,8 @@ A quiesce set can be manipulated in the following ways:
 * **query** the current state of a set by id or all active sets or all known sets
 * **cancel all** active sets in case an immediate resume of IO is required.
 
-The operations listed above are non-blocking: they perform the intended modification if it's applicable
-and get back with an up to date version of the target set, whether the operation was successful or not. 
+The operations listed above are non-blocking: they attempt the intended modification 
+and return with an up to date version of the target set, whether the operation was successful or not. 
 The set may change states as a result of the modification, and the version that's returned in the response 
 is guaranteed to be in a state consistent with this and potentialy other successful operations from 
 the same control loop batch.
@@ -832,8 +833,8 @@ Some set states are `awaitable`. We will discuss those below, but for now it's i
 any of the commands above can be amended with an **await** modifier, which will cause them to block
 on the set after applying their intended modification, as long as the resulting set state is `awaitable`.
 Such a command will block until the set reaches the awaited state, gets modified by another command,
-or transitions into another state. The reason for the unblock will be clear by the result code, while
-the contents of the response will always be the most recent set state.
+or transitions into another state. The return code will unambiguously identify the exit condition, and
+the contents of the response will always carry the latest known set state.
 
 .. image:: quiesce-set-states.svg
 
@@ -855,13 +856,13 @@ to the caller by inspecting the output
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a sub1 --set-id=unique-id
+  $ ceph fs quiesce fs1 sub1 --set-id=unique-id
   {
       "epoch": 3,
-      "db_version": 1,
+      "set_version": 1,
       "sets": {
           "unique-id": {
-              "db_version": 1,
+              "version": 1,
               "age_ref": 0.0,
               "state": {
                   "name": "TIMEDOUT",
@@ -896,21 +897,21 @@ If present, the values will be applied before the action this command requests.
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a --set-id=unique-id --timeout=10 > /dev/null
+  $ ceph fs quiesce fs1 --set-id=unique-id --timeout=10 > /dev/null
   Error EPERM:  
 
 It's too late for our ``unique-id`` set, as it's in a terminal state. No changes are allowed
-to sets that are in their terminal states, i.e. inactive. Let's create a new set
+to sets that are in their terminal states, i.e. inactive. Let's create a new set:
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a sub1 --timeout 60
+  $ ceph fs quiesce fs1 sub1 --timeout 60
   {
       "epoch": 3,
-      "db_version": 2,
+      "set_version": 2,
       "sets": {
           "8988b419": {
-              "db_version": 2,
+              "version": 2,
               "age_ref": 0.0,
               "state": {
                   "name": "QUIESCING",
@@ -937,13 +938,13 @@ this time the set is `QUIESCING`. At this point, we can add more members to the
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a --set-id 8988b419 --include sub2 sub3
+  $ ceph fs quiesce fs1 --set-id 8988b419 --include sub2 sub3
   {
       "epoch": 3,
-      "db_version": 3,
+      "set_version": 3,
       "sets": {
           "8988b419": {
-              "db_version": 3,
+              "version": 3,
               "age_ref": 0.0,
               "state": {
                   "name": "QUIESCING",
@@ -1024,13 +1025,13 @@ from ``--await-for`` is honored.
 
 .. prompt:: bash $ auto
 
-  $ time ceph fs quiesce a sub1 --timeout=10 --await-for=2
+  $ time ceph fs quiesce fs1 sub1 --timeout=10 --await-for=2
   {
       "epoch": 6,
-      "db_version": 3,
+      "set_version": 3,
       "sets": {
           "c3c1d8de": {
-              "db_version": 3,
+              "version": 3,
               "age_ref": 0.0,
               "state": {
                   "name": "QUIESCING",
@@ -1051,7 +1052,7 @@ from ``--await-for`` is honored.
       }
   }
   Error EINPROGRESS: 
-  ceph fs quiesce a sub1 --timeout=10 --await-for=2  0.41s user 0.04s system 17% cpu 2.563 total
+  ceph fs quiesce fs1 sub1 --timeout=10 --await-for=2  0.41s user 0.04s system 17% cpu 2.563 total
 
 (there is a ~0.5 sec overhead that the ceph client adds, at least in a local debug setup)
 
@@ -1065,13 +1066,13 @@ approach to a long running multistep process under the IO pause by repeatedly ``
 .. prompt:: bash $ auto
 
   $ set -e   # (1)
-  $ ceph fs quiesce a sub1 sub2 sub3 --timeout=30 --expiration=10 --set-id="snapshots" --await # (2)
+  $ ceph fs quiesce fs1 sub1 sub2 sub3 --timeout=30 --expiration=10 --set-id="snapshots" --await # (2)
   $ ceph fs subvolume snapshot create a sub1 snap1-sub1  # (3)
-  $ ceph fs quiesce a --set-id="snapshots" --await  # (4)
+  $ ceph fs quiesce fs1 --set-id="snapshots" --await  # (4)
   $ ceph fs subvolume snapshot create a sub2 snap1-sub2  # (3)
-  $ ceph fs quiesce a --set-id="snapshots" --await  # (4)
+  $ ceph fs quiesce fs1 --set-id="snapshots" --await  # (4)
   $ ceph fs subvolume snapshot create a sub3 snap1-sub3  # (3)
-  $ ceph fs quiesce a --set-id="snapshots" --release --await  # (5)
+  $ ceph fs quiesce fs1 --set-id="snapshots" --release --await  # (5)
 
 .. warning:: This example uses arbitrary timeouts to convey the concept. In real life, the values must be carefully
   chosen in accordance with the actual system requirements and specifications.
@@ -1105,30 +1106,31 @@ a concurrent change of the set by another client. Consider this example:
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a sub1 sub2 sub3 --timeout=30 --expiration=60 --set-id="snapshots" --await  # (1)
+  $ ceph fs quiesce fs1 sub1 sub2 sub3 --timeout=30 --expiration=60 --set-id="snapshots" --await  # (1)
   $ ceph fs subvolume snapshot create a sub1 snap1-sub1  # (2)
   $ ceph fs subvolume snapshot create a sub2 snap1-sub2  # (3)
   $ ceph fs subvolume snapshot create a sub3 snap1-sub3  # (4)
-  $ ceph fs quiesce a --set-id="snapshots" --release --await  # (5)
+  $ ceph fs quiesce fs1 --set-id="snapshots" --release --await  # (5)
 
 The sequence looks good, and the release `(5)` completes successfully. However, it could be that
 before snap for sub3 `(4)` is taken, another session excludes sub3 from the set, resuming its IOs
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a --set-id="snapshots" --exclude sub3
+  $ ceph fs quiesce fs1 --set-id="snapshots" --exclude sub3
 
 Since removing a member from a set doesn't affect its `QUIESCED` state, the release command `(5)`
 has no reason to fail. It will ack the two unexcluded members sub1 and sub2 and report success.
 
 In order to address this or similar problems, the quiesce command supports an optimistic concurrency
-mode. To activate it, one needs to pass an ``--if-version=<db_version>`` that will be compared
+mode. To activate it, one needs to pass an ``--if-version=<version>`` that will be compared
 to the set's db version and the operation will only proceed if the values match. Otherwise, the command
 will not be executed and the return status will be ``ESTALE``.
 
 It's easy to know which version to expect of a set, since every command that modifies a set will return
 this set on the stdout, regarldess of the exit status. In the examples above one can notice that every
-set carries a ``"db_version"`` property which is the last db version where this set got modified.
+set carries a ``"version"`` property which gets updated whenever this set is modified, explicitly
+by the user or implicitly during 
 
 In the example at the beginning of this subsection, the initial quiesce command `(1)` would have returned
 the newly created set with id ``"snapshots"`` and some version, let's say ``13``. Since we don't expect any other
@@ -1137,7 +1139,7 @@ could have looked like
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a --set-id="snapshots" --release --await --if-version=13 # (5)
+  $ ceph fs quiesce fs1 --set-id="snapshots" --release --await --if-version=13 # (5)
 
 This way, the result of the release command would have been ``ESTALE`` instead of 0, and we would
 know that something wasn't right with the quiesce set and our snapshots might not be consistent.
@@ -1148,14 +1150,14 @@ know that something wasn't right with the quiesce set and our snapshots might no
 
 There is another use of the ``--if-version`` argument which could come handy for automation software.
 As we have discussed earlier, it is possible to create a new quiesce set with a given set id. Drivers like
-the CSI for k8s could use their internal request id to eliminate the need to keep an additional mapping
+the CSI for Kubernetes could use their internal request id to eliminate the need to keep an additional mapping
 to the quiesce set id. However, to guarantee uniqueness, the driver may want to verify that the set is
 indeed new. For that, ``if-version=0`` may be used, and it will only create the new set if no other
 set with this id was present in the database
 
 .. prompt:: bash $ auto
 
-  $ ceph fs quiesce a sub1 sub2 sub3 --set-id="external-id" --if-version=0
+  $ ceph fs quiesce fs1 sub1 sub2 sub3 --set-id="external-id" --if-version=0
 
 .. _manila: https://github.com/openstack/manila
 .. _CSI: https://github.com/ceph/ceph-csi

From 9846d35a2ca0fc68c0464657616d259b19273b79 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Tue, 27 Feb 2024 13:36:16 +0200
Subject: [PATCH 2124/2492] mds/quiesce-db: incorporate review comments

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/BoostUrlImpl.cc      |  7 +++++++
 src/mds/MDSRank.cc           |  2 --
 src/mds/MDSRankQuiesce.cc    | 35 ++++++++++++++++++++---------------
 src/mds/QuiesceAgent.h       |  7 +++++--
 src/pybind/mgr/mgr_module.py | 14 ++++++++++++++
 5 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/src/mds/BoostUrlImpl.cc b/src/mds/BoostUrlImpl.cc
index c2e992a06e90..479f4c6d75d0 100644
--- a/src/mds/BoostUrlImpl.cc
+++ b/src/mds/BoostUrlImpl.cc
@@ -1 +1,8 @@
+/*
+ * https://www.boost.org/doc/libs/1_82_0/libs/url/doc/html/url/overview.html#url.overview.requirements
+ *
+ * To use the library as header-only; that is, to eliminate the requirement 
+ * to link a program to a static or dynamic Boost.URL library, 
+ * simply place the following line in exactly one source file in your project.
+ */
 #include <boost/url/src.hpp>
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 12f6865b2bc1..c14ca9e1cb1e 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -48,8 +48,6 @@
 #include "QuiesceDbManager.h"
 #include "QuiesceAgent.h"
 
-#include <cmath>
-
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds
 #undef dout_prefix
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 0b753865698c..820cd33fcffa 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -349,7 +349,7 @@ void MDSRank::quiesce_cluster_update() {
             ++it;
           } else {
             // just ack.
-            dout(20) << "INTACTIVE RESPONDER: reporting '" << it->first << "' as " << it->second.state << dendl;
+            dout(20) << "INACTIVE RESPONDER: reporting '" << it->first << "' as " << it->second.state << dendl;
             it = quiesce_map.roots.erase(it);
           }
           break;
@@ -416,21 +416,21 @@ void MDSRank::quiesce_agent_setup() {
 
   using RequestHandle = QuiesceInterface::RequestHandle;
   using QuiescingRoot = std::pair<RequestHandle, Context*>;
-  auto quiesce_requests = std::make_shared<std::unordered_map<QuiesceRoot, QuiescingRoot>>();
+  auto dummy_requests = std::make_shared<std::unordered_map<QuiesceRoot, QuiescingRoot>>();
 
   QuiesceAgent::ControlInterface ci;
 
-  ci.submit_request = [this, quiesce_requests](QuiesceRoot root, Context* c)
+  ci.submit_request = [this, dummy_requests](QuiesceRoot root, Context* c)
       -> std::optional<RequestHandle> {
     auto uri = boost::urls::parse_uri_reference(root);
     if (!uri) {
       dout(5) << "error parsing the quiesce root as an URI: " << uri.error() << dendl;
       c->complete(uri.error());
       return std::nullopt;
-    } else {
-      dout(20) << "parsed root '" << root <<"' as : " << uri->path() << " " << uri->query() << dendl;
     }
 
+    dout(10) << "submit_request: " << uri << dendl;
+
     std::chrono::milliseconds quiesce_delay_ms = 0ms;
     if (auto pit = uri->params().find("delayms"); pit != uri->params().end()) {
       try {
@@ -481,7 +481,6 @@ void MDSRank::quiesce_agent_setup() {
     }
 
     auto path = uri->path();
-    dout(20) << "got request to quiesce '" << path << "'" << dendl;
 
     std::lock_guard l(mds_lock);
 
@@ -496,10 +495,10 @@ void MDSRank::quiesce_agent_setup() {
       auto mdr = mdcache->quiesce_path(filepath(path), qc, nullptr, quiesce_delay_ms);
       return mdr ? mdr->reqid : std::optional<RequestHandle>();
     } else {
-      /* dummy quiesce/fail */
+      /* we use this branch to allow for quiesce emulation for testing purposes */
       // always create a new request id
       auto req_id = metareqid_t(entity_name_t::MDS(whoami), issue_tid());
-      auto [it, inserted] = quiesce_requests->try_emplace(path, req_id, c);
+      auto [it, inserted] = dummy_requests->try_emplace(path, req_id, c);
 
       if (!inserted) {
         dout(3) << "duplicate quiesce request for root '" << it->first << "'" << dendl;
@@ -535,11 +534,12 @@ void MDSRank::quiesce_agent_setup() {
           delay = debug_quiesce_after.value();
         }
 
-        auto quiesce_task = new LambdaContext([quiesce_requests, req_id, do_fail, this](int) {
+        auto quiesce_task = new LambdaContext([dummy_requests, req_id, do_fail, this](int) {
           // the mds lock should be held by the timer
+          ceph_assert(ceph_mutex_is_locked_by_me(mds_lock));
           dout(20) << "quiesce_task: callback by the timer" << dendl;
-          auto it = std::ranges::find(*quiesce_requests, req_id, [](auto x) { return x.second.first; });
-          if (it != quiesce_requests->end() && it->second.second != nullptr) {
+          auto it = std::ranges::find(*dummy_requests, req_id, [](auto x) { return x.second.first; });
+          if (it != dummy_requests->end() && it->second.second != nullptr) {
             dout(20) << "quiesce_task: completing the root '" << it->first << "' as failed: " << do_fail << dendl;
             it->second.second->complete(do_fail ? -EBADF : 0);
             it->second.second = nullptr;
@@ -556,25 +556,30 @@ void MDSRank::quiesce_agent_setup() {
     }
   };
 
-  ci.cancel_request = [this, quiesce_requests](RequestHandle h) {
+  ci.cancel_request = [this, dummy_requests](RequestHandle h) {
     std::lock_guard l(mds_lock);
 
     if (mdcache->have_request(h)) {
       auto qimdr = mdcache->request_get(h);
       mdcache->request_kill(qimdr);
+      // no reason to waste time checking for dummy requests
       return 0;
     }
 
-    auto it = std::ranges::find(*quiesce_requests, h, [](auto x) { return x.second.first; });
-    if (it != quiesce_requests->end()) {
+    // if we get here then it could be a test (dummy) quiesce
+    auto it = std::ranges::find(*dummy_requests, h, [](auto x) { return x.second.first; });
+    if (it != dummy_requests->end()) {
       if (auto ctx = it->second.second; ctx) {
         dout(20) << "canceling request with id '" << h << "' for root '" << it->first << "'" << dendl;
         ctx->complete(-ECANCELED);
       }
-      quiesce_requests->erase(it);
+      dummy_requests->erase(it);
       return 0;
     }
 
+    // we must indicate that the handle wasn't found
+    // so that the agent can properly report a missing
+    // outstanding quiesce, preventing a RELEASED transition 
     return ENOENT;
   };
 
diff --git a/src/mds/QuiesceAgent.h b/src/mds/QuiesceAgent.h
index f5be435f2a20..4b1ef84b4a54 100644
--- a/src/mds/QuiesceAgent.h
+++ b/src/mds/QuiesceAgent.h
@@ -34,7 +34,7 @@ class QuiesceAgent {
     };
 
     ~QuiesceAgent() {
-      agent_thread.kill(SIGTERM);
+      shutdown();
     }
 
     /// @brief  WARNING: will reset syncrhonously
@@ -72,7 +72,10 @@ class QuiesceAgent {
       stop_agent_thread = true;
       agent_cond.notify_all();
       l.unlock();
-      agent_thread.join();
+
+      if (agent_thread.is_started()) {
+        agent_thread.join();
+      }
 
       current.clear();
       pending.clear();
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 219afbf270fe..2e25f994b6b7 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1753,6 +1753,18 @@ def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf:
     MDS_STATE_ACTIVE_ORD = MDS_STATE_ORD["up:active"]
 
     def get_quiesce_leader_info(self, fscid: str) -> Optional[dict]:
+        """
+        Helper for `tell_quiesce_leader` to chose the mds to send the command to.
+
+        Quiesce DB is managed by a leader which is selected based on the current MDSMap
+        The logic is currently implemented both here and on the MDS side,
+        see MDSRank::quiesce_cluster_update().
+
+        Ideally, this logic should be part of the MDSMonitor and the result should
+        be exposed via a dedicated field in the map, but until that is implemented
+        this function will have to be kept in sync with the corresponding logic
+        on the MDS side
+        """
         leader_info: Optional[dict] = None
 
         for fs in self.get("fs_map")['filesystems']:
@@ -1786,6 +1798,8 @@ def tell_quiesce_leader(self, fscid: str, cmd_dict: dict) -> Tuple[int, str, str
             self.log.warn("Couldn't resolve the quiesce leader for fscid %s" % fscid)
             return (-errno.ENOENT, "", "Couldn't resolve the quiesce leader for fscid %s" % fscid)
         self.log.debug("resolved quiesce leader for fscid {fscid} at daemon '{name}' gid {gid} rank {rank} ({state})".format(fscid=fscid, **qleader))
+        # We use the one_shot here to cover for cases when the mds crashes
+        # without this parameter the client may get stuck awaiting response from a dead MDS
         return self.tell_command('mds', str(qleader['gid']), cmd_dict, one_shot=True)
 
     def send_command(

From 2fbe40e72e8352a3bf47190d7bc8c80bb60eb7bd Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Tue, 27 Feb 2024 23:25:20 +0200
Subject: [PATCH 2125/2492] messages: avoid using mutable members in
 MMDSQuiesce*

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/MDSRankQuiesce.cc           | 81 ++++++++++++++++++-----------
 src/messages/MMDSQuiesceDbAck.h     | 17 +++---
 src/messages/MMDSQuiesceDbListing.h | 18 ++++---
 3 files changed, 73 insertions(+), 43 deletions(-)

diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 820cd33fcffa..5d92c0bd818a 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -288,9 +288,9 @@ void MDSRank::quiesce_cluster_update() {
         }
         auto addrs = mdsmap->get_info_gid(leader).addrs;
 
-        auto ack_msg = make_message<MMDSQuiesceDbAck>(me);
+        auto ack_msg = make_message<MMDSQuiesceDbAck>();
         dout(10) << "sending ack " << ack << " to the leader " << leader << dendl;
-        ack_msg->diff_map = std::move(ack);
+        ack_msg->encode_payload_from(me, ack);
         return send_message_mds(ack_msg, addrs);
       }
     };
@@ -302,9 +302,9 @@ void MDSRank::quiesce_cluster_update() {
         return -ENOENT;
       }
       auto addrs = mdsmap->get_info_gid(to).addrs;
-      auto listing_msg = make_message<MMDSQuiesceDbListing>(me);
+      auto listing_msg = make_message<MMDSQuiesceDbListing>();
       dout(10) << "sending listing " << db << " to the peer " << to << dendl;
-      listing_msg->db_listing = std::move(db);
+      listing_msg->encode_payload_from(me, db);
       return send_message_mds(listing_msg, addrs);
     };
   }
@@ -374,38 +374,59 @@ void MDSRank::quiesce_cluster_update() {
 }
 
 bool MDSRank::quiesce_dispatch(const cref_t<Message> &m) {
-  switch(m->get_type()) {
-    case MSG_MDS_QUIESCE_DB_LISTING:
-    {
-      const auto& req = ref_cast<MMDSQuiesceDbListing>(m);
-      if (quiesce_db_manager) {
-        dout(10) << "got " << req->db_listing << " from peer " << req->gid << dendl;
-        int result = quiesce_db_manager->submit_listing_from(req->gid, std::move(req->db_listing));
-        if (result != 0) {
-          dout(3) << "error (" << result << ") submitting " << req->db_listing << " from peer " << req->gid << dendl;
+  try {
+    switch(m->get_type()) {
+      case MSG_MDS_QUIESCE_DB_LISTING:
+      {
+        const auto& req = ref_cast<MMDSQuiesceDbListing>(m);
+        mds_gid_t gid;
+        QuiesceDbListing db_listing;
+        req->decode_payload_into(gid, db_listing);
+        if (quiesce_db_manager) {
+          dout(10) << "got " << db_listing << " from peer " << gid << dendl;
+          int result = quiesce_db_manager->submit_listing_from(gid, std::move(db_listing));
+          if (result != 0) {
+            dout(3) << "error (" << result << ") submitting " << db_listing << " from peer " << gid << dendl;
+          }
+        } else {
+          dout(5) << "no db manager to process " << db_listing << dendl;
         }
-      } else {
-        dout(5) << "no db manager to process " << req->db_listing << dendl;
+        return true;
       }
-      return true;
-    }
-    case MSG_MDS_QUIESCE_DB_ACK:
-    {
-      const auto& req = ref_cast<MMDSQuiesceDbAck>(m);
-      if (quiesce_db_manager) {
-        dout(10) << "got ack " << req->diff_map << " from peer " << req->gid << dendl;
-        int result = quiesce_db_manager->submit_ack_from(req->gid, std::move(req->diff_map));
-        if (result != 0) {
-          dout(3) << "error (" << result << ") submitting an ack from peer " << req->gid << dendl;
+      case MSG_MDS_QUIESCE_DB_ACK:
+      {
+        const auto& req = ref_cast<MMDSQuiesceDbAck>(m);
+        mds_gid_t gid;
+        QuiesceMap diff_map;
+        req->decode_payload_into(gid, diff_map);
+        if (quiesce_db_manager) {
+          dout(10) << "got ack " << diff_map << " from peer " << gid << dendl;
+          int result = quiesce_db_manager->submit_ack_from(gid, std::move(diff_map));
+          if (result != 0) {
+            dout(3) << "error (" << result << ") submitting an ack from peer " << gid << dendl;
+          }
+        } else {
+          dout(5) << "no db manager to process an ack: " << diff_map << dendl;
         }
-      } else {
-        dout(5) << "no db manager to process an ack: " << req->diff_map << dendl;
+        return true;
+      }
+      default: break;
+    }
+  }
+  catch (const ceph::buffer::error &e) {
+    if (cct) {
+      dout(-1) << "failed to decode message of type " << m->get_type()
+                 << " v" << m->get_header().version
+                 << ": " << e.what() << dendl;
+      dout(10) << "dump: \n";
+      m->get_payload().hexdump(*_dout);
+      *_dout << dendl;
+      if (cct->_conf->ms_die_on_bad_msg) {
+        ceph_abort();
       }
-      return true;
     }
-    default:
-      return false;
   }
+  return false;
 }
 
 void MDSRank::quiesce_agent_setup() {
diff --git a/src/messages/MMDSQuiesceDbAck.h b/src/messages/MMDSQuiesceDbAck.h
index 907db239bd18..1d56451e89bc 100644
--- a/src/messages/MMDSQuiesceDbAck.h
+++ b/src/messages/MMDSQuiesceDbAck.h
@@ -19,13 +19,8 @@
 #include "mds/QuiesceDbEncoding.h"
 
 class MMDSQuiesceDbAck final : public MMDSOp {
-public:
-  mds_gid_t gid;
-  mutable QuiesceMap diff_map;
-
 protected:
-  MMDSQuiesceDbAck(mds_gid_t gid) : MMDSOp{MSG_MDS_QUIESCE_DB_ACK}, gid(gid) {}
-  MMDSQuiesceDbAck() : MMDSQuiesceDbAck(MDS_GID_NONE) {}
+  MMDSQuiesceDbAck() : MMDSOp{MSG_MDS_QUIESCE_DB_ACK} {}
   ~MMDSQuiesceDbAck() final {}
 
 public:
@@ -35,6 +30,11 @@ class MMDSQuiesceDbAck final : public MMDSOp {
   }
 
   void encode_payload(uint64_t features) override
+  {
+    // noop to prevent unnecessary overheads
+  }
+
+  void encode_payload_from(mds_gid_t const&gid, QuiesceMap const&diff_map)
   {
     using ceph::encode;
 
@@ -47,6 +47,11 @@ class MMDSQuiesceDbAck final : public MMDSOp {
   }
 
   void decode_payload() override {
+    // noop to prevent unnecessary overheads
+  }
+
+  void decode_payload_into(mds_gid_t &gid, QuiesceMap &diff_map) const
+  {
     using ceph::decode;
     auto p = payload.cbegin();
     DECODE_START(1, p);
diff --git a/src/messages/MMDSQuiesceDbListing.h b/src/messages/MMDSQuiesceDbListing.h
index 9d3ce20fdab4..f57de50e22fb 100644
--- a/src/messages/MMDSQuiesceDbListing.h
+++ b/src/messages/MMDSQuiesceDbListing.h
@@ -19,13 +19,8 @@
 #include "mds/QuiesceDbEncoding.h"
 
 class MMDSQuiesceDbListing final : public MMDSOp {
-public:
-  mds_gid_t gid;
-  mutable QuiesceDbListing db_listing;
-
 protected:
-  MMDSQuiesceDbListing(mds_gid_t gid) : MMDSOp{MSG_MDS_QUIESCE_DB_LISTING}, gid(gid) {}
-  MMDSQuiesceDbListing() : MMDSQuiesceDbListing(MDS_GID_NONE) {}
+  MMDSQuiesceDbListing() : MMDSOp{MSG_MDS_QUIESCE_DB_LISTING} {}
   ~MMDSQuiesceDbListing() final {}
 
 public:
@@ -34,7 +29,11 @@ class MMDSQuiesceDbListing final : public MMDSOp {
 
   }
 
-  void encode_payload(uint64_t features) override
+  void encode_payload(uint64_t features) override { 
+    // noop to prevent unnecessary overheads
+  }
+
+  void encode_payload_from(mds_gid_t const& gid, QuiesceDbListing const& db_listing)
   {
     using ceph::encode;
 
@@ -47,6 +46,11 @@ class MMDSQuiesceDbListing final : public MMDSOp {
   }
 
   void decode_payload() override {
+    // noop to prevent unnecessary overheads
+  }
+
+  void decode_payload_into(mds_gid_t &gid, QuiesceDbListing &db_listing) const
+  {
     using ceph::decode;
     auto p = payload.cbegin();
     DECODE_START(1, p);

From 42a5fb3b7562dac09dbf4575558820297317965e Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 28 Feb 2024 14:41:13 +0200
Subject: [PATCH 2126/2492] include/types: add an I/O helper for
 std::unordered_map

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/include/types.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/include/types.h b/src/include/types.h
index b3777d9de4fd..c3969f88a34c 100644
--- a/src/include/types.h
+++ b/src/include/types.h
@@ -104,6 +104,8 @@ inline std::ostream& operator<<(std::ostream& out, const std::list<A,Alloc>& ili
 template<class A, class Comp, class Alloc>
 inline std::ostream& operator<<(std::ostream& out, const std::set<A, Comp, Alloc>& iset);
 template<class A, class Comp, class Alloc>
+inline std::ostream& operator<<(std::ostream& out, const std::unordered_set<A, Comp, Alloc>& iset);
+template<class A, class Comp, class Alloc>
 inline std::ostream& operator<<(std::ostream& out, const std::multiset<A,Comp,Alloc>& iset);
 template<class A, class B, class Comp, class Alloc>
 inline std::ostream& operator<<(std::ostream& out, const std::map<A,B,Comp,Alloc>& m);
@@ -209,6 +211,17 @@ inline std::ostream& operator<<(std::ostream& out, const std::set<A, Comp, Alloc
   return out;
 }
 
+template<class A, class Comp, class Alloc>
+inline std::ostream& operator<<(std::ostream& out, const std::unordered_set<A, Comp, Alloc>& iset) {
+  for (auto it = iset.begin();
+       it != iset.end();
+       ++it) {
+    if (it != iset.begin()) out << ",";
+    out << *it;
+  }
+  return out;
+}
+
 template<class A, class Comp, class Alloc>
 inline std::ostream& operator<<(std::ostream& out, const std::multiset<A,Comp,Alloc>& iset) {
   for (auto it = iset.begin();

From b4c1d552aa4066efdda3493192927cb780693469 Mon Sep 17 00:00:00 2001
From: Rishabh Dave <ridave@redhat.com>
Date: Mon, 4 Mar 2024 17:11:21 +0530
Subject: [PATCH 2127/2492] mgr/vol: don't import IndexException since it's
 unused

Introduced-by: 079f722c37ef6cc8bd3cc26c49ae119dd83431f9
Signed-off-by: Rishabh Dave <ridave@redhat.com>
---
 src/pybind/mgr/volumes/fs/volume.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 0c4a07598054..2e96f830636b 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -20,7 +20,7 @@
 
 from .vol_spec import VolSpec
 from .exception import VolumeException, ClusterError, ClusterTimeout, \
-    EvictionError, IndexException
+    EvictionError
 from .async_cloner import Cloner
 from .purge_queue import ThreadPoolPurgeQueueMixin
 from .operations.template import SubvolumeOpType

From 6a1613ac0b089b2bb2ef4fb8b6c954d0dbe38866 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 4 Mar 2024 23:09:20 +1000
Subject: [PATCH 2128/2492] doc/architecture: fix spelling and syntax

Fix the spelling of the word "algorithm" (which was "algoritm") and make
a sentence a little more natural.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index fc9e06da8cd8..5d82ebf08650 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -51,10 +51,10 @@ A Ceph Metadata Server (MDS) manages file metadata when CephFS is used to
 provide file services.
 
 Storage cluster clients and :term:`Ceph OSD Daemon`\s use the CRUSH algorithm
-to compute information about the location of data. Use of the CRUSH algoritm
-means that clients and OSDs are not bottlenecked by a central lookup table.
+to compute information about the location of data.  By using the CRUSH
+algorithm, clients and OSDs avoid being bottlenecked by a central lookup table.
 Ceph's high-level features include a native interface to the Ceph Storage
-Cluster via ``librados``, and a number of service interfaces built on top of
+Cluster via ``librados`` and a number of service interfaces built on top of
 ``librados``.
 
 Storing Data

From 759925748e8be355a743f6edd106052d5a026dfd Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 28 Feb 2024 15:11:17 +0200
Subject: [PATCH 2129/2492] mds/quiesce: resolve the quiesce cluster at the mds
 monitor

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 qa/tasks/mgr/dashboard/test_health.py |  4 +-
 src/include/cephfs/types.h            |  8 +++
 src/mds/FSMap.h                       | 16 ++++-
 src/mds/MDSMap.cc                     | 16 ++++-
 src/mds/MDSMap.h                      | 26 +++++++++
 src/mds/MDSRankQuiesce.cc             | 53 ++++++-----------
 src/mds/QuiesceDbManager.h            | 12 +---
 src/mon/MDSMonitor.cc                 | 52 +++++++++++++++++
 src/mon/MDSMonitor.h                  |  2 +
 src/pybind/mgr/mgr_module.py          | 84 ++++++++-------------------
 src/pybind/mgr/volumes/fs/volume.py   | 17 +++---
 src/pybind/mgr/volumes/module.py      |  3 +-
 src/test/mds/TestQuiesceDb.cc         |  2 +-
 13 files changed, 175 insertions(+), 120 deletions(-)

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index 0b7b7a3b449c..3ebea97e3d29 100644
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -63,7 +63,9 @@ class HealthTest(DashboardTestCase):
             'balance_automate': bool,
         }),
         'ever_allowed_features': int,
-        'root': int
+        'root': int,
+        'qdb_leader': int,
+        'qdb_cluster': JList(int)
     })
 
     def test_minimal_health(self):
diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 068c9ef81995..23f6e44a9c2b 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -48,6 +48,14 @@
 BOOST_STRONG_TYPEDEF(uint64_t, mds_gid_t)
 extern const mds_gid_t MDS_GID_NONE;
 
+template <>
+struct std::hash<mds_gid_t> {
+  size_t operator()(const mds_gid_t& gid) const
+  {
+    return hash<uint64_t> {}(gid);
+  }
+};
+
 typedef int32_t fs_cluster_id_t;
 constexpr fs_cluster_id_t FS_CLUSTER_ID_NONE = -1;
 
diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h
index a6aa92f218be..e2046fb7af86 100644
--- a/src/mds/FSMap.h
+++ b/src/mds/FSMap.h
@@ -21,6 +21,7 @@
 #include <set>
 #include <string>
 #include <string_view>
+#include <type_traits>
 
 #include <errno.h>
 
@@ -479,9 +480,18 @@ class FSMap {
   void modify_filesystem(fs_cluster_id_t fscid, T&& fn)
   {
     auto& fs = filesystems.at(fscid);
-    fn(fs);
-    fs.mds_map.epoch = epoch;
-    fs.mds_map.modified = ceph_clock_now();
+    bool did_update = true;
+
+    if constexpr (std::is_convertible_v<std::invoke_result_t<T, Filesystem&>, bool>) {
+      did_update = fn(fs);
+    } else {
+      fn(fs);
+    }
+    
+    if (did_update) {
+      fs.mds_map.epoch = epoch;
+      fs.mds_map.modified = ceph_clock_now();
+    }
   }
 
   /* This is method is written for the option of "ceph fs swap" commmand
diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index 4096b23ab9d8..bedeed165ab9 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -228,6 +228,12 @@ void MDSMap::dump(Formatter *f) const
   f->dump_string("balancer", balancer);
   f->dump_string("bal_rank_mask", bal_rank_mask);
   f->dump_int("standby_count_wanted", std::max(0, standby_count_wanted));
+  f->dump_unsigned("qdb_leader", qdb_cluster_leader);
+  f->open_array_section("qdb_cluster");
+  for (auto m: qdb_cluster_members) {
+    f->dump_int("member", m);
+  }
+  f->close_section();
 }
 
 void MDSMap::dump_flags_state(Formatter *f) const
@@ -290,6 +296,7 @@ void MDSMap::print(ostream& out) const
   out << "balancer\t" << balancer << "\n";
   out << "bal_rank_mask\t" << bal_rank_mask << "\n";
   out << "standby_count_wanted\t" << std::max(0, standby_count_wanted) << "\n";
+  out << "qdb_cluster\tleader: " << qdb_cluster_leader << " members: " << qdb_cluster_members << std::endl;
 
   multimap< pair<mds_rank_t, unsigned>, mds_gid_t > foo;
   for (const auto &p : mds_info) {
@@ -773,7 +780,7 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
   encode(data_pools, bl);
   encode(cas_pool, bl);
 
-  __u16 ev = 18;
+  __u16 ev = 19;
   encode(ev, bl);
   encode(compat, bl);
   encode(metadata_pool, bl);
@@ -802,6 +809,8 @@ void MDSMap::encode(bufferlist& bl, uint64_t features) const
   encode(required_client_features, bl);
   encode(bal_rank_mask, bl);
   encode(max_xattr_size, bl);
+  encode(qdb_cluster_leader, bl);
+  encode(qdb_cluster_members, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -957,6 +966,11 @@ void MDSMap::decode(bufferlist::const_iterator& p)
     decode(max_xattr_size, p);
   }
 
+  if (ev >= 19) {
+    decode(qdb_cluster_leader, p);
+    decode(qdb_cluster_members, p);
+  }
+
   /* All MDS since at least v14.0.0 understand INLINE */
   /* TODO: remove after R is released */
   compat.incompat.insert(MDS_FEATURE_INCOMPAT_INLINE);
diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 746ae8597151..2dd8fba83423 100644
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -19,6 +19,7 @@
 #include <map>
 #include <set>
 #include <string>
+#include <ranges>
 #include <string_view>
 
 #include <errno.h>
@@ -312,6 +313,29 @@ class MDSMap {
   mds_rank_t get_tableserver() const { return tableserver; }
   mds_rank_t get_root() const { return root; }
 
+  void get_quiesce_db_cluster(mds_gid_t &leader, std::unordered_set<mds_gid_t> &members) const {
+    leader = qdb_cluster_leader;
+    members = qdb_cluster_members; 
+  }
+
+  mds_gid_t get_quiesce_db_cluster_leader() {
+    return qdb_cluster_leader;
+  }
+
+  bool update_quiesce_db_cluster(mds_gid_t const& leader, std::same_as<std::unordered_set<mds_gid_t>> auto && members) {
+    if (leader == qdb_cluster_leader && members == qdb_cluster_members) {
+      return false;
+    }
+
+    ceph_assert(leader == MDS_GID_NONE || mds_info.contains(leader));
+    ceph_assert(std::ranges::all_of(members, [this](auto &m) {return mds_info.contains(m);}));
+
+    qdb_cluster_leader = leader;
+    qdb_cluster_members = members;
+
+    return true;
+  }
+
   const std::vector<int64_t> &get_data_pools() const { return data_pools; }
   int64_t get_first_data_pool() const { return *data_pools.begin(); }
   int64_t get_metadata_pool() const { return metadata_pool; }
@@ -634,6 +658,8 @@ class MDSMap {
 
   mds_rank_t tableserver = 0;   // which MDS has snaptable
   mds_rank_t root = 0;          // which MDS has root directory
+  std::unordered_set<mds_gid_t> qdb_cluster_members;
+  mds_gid_t qdb_cluster_leader = MDS_GID_NONE;
 
   __u32 session_timeout = 60;
   __u32 session_autoclose = 300;
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 5d92c0bd818a..bcc8e149ed7f 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -101,6 +101,7 @@ void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int,
   struct Ctx : public QuiesceDbManager::RequestContext {
     std::function<void(int, const std::string&, bufferlist&)> on_finish;
     bool all = false;
+    mds_gid_t me;
 
     double sec(QuiesceTimeInterval duration) {
       return duration_cast<dd>(duration).count();
@@ -126,6 +127,7 @@ void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int,
 
       f->open_object_section("response"); {
         f->dump_int("epoch", response.db_version.epoch);
+        f->dump_int("leader", me);
         f->dump_int("set_version", response.db_version.set_version);
         f->open_object_section("sets"); {
           for (auto&& [set_id, set] : response.sets) {
@@ -168,8 +170,10 @@ void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int,
 
   auto* ctx = new Ctx();
 
+  QuiesceInterface::PeerId me = mds_gid_t(monc->get_global_id());
   ctx->on_finish = std::move(on_finish);
   ctx->all = all;
+  ctx->me = me;
 
   ctx->request.reset([&](auto& r) {
     r.set_id = set_id;
@@ -212,6 +216,12 @@ void MDSRank::command_quiesce_db(const cmdmap_t& cmdmap, std::function<void(int,
   int rc = quiesce_db_manager->submit_request(ctx);
   if (rc != 0) {
     bufferlist bl;
+    auto f = Formatter::create_unique("json-pretty");
+    f->open_object_section("response");
+    f->dump_int("epoch", mdsmap->get_epoch());
+    f->dump_int("leader", mdsmap->get_quiesce_db_cluster_leader());
+    f->close_section();
+    f->flush(bl);
     // on_finish was moved there, so should only call via the ctx.
     ctx->on_finish(rc, "Error submitting the command to the local db manager", bl);
     delete ctx;
@@ -234,62 +244,35 @@ static void rebind_agent_callback(std::shared_ptr<QuiesceAgent> agt, std::shared
 
 void MDSRank::quiesce_cluster_update() {
   // the quiesce leader is the lowest rank with the highest state up to ACTIVE
-  auto less_leader = [](MDSMap::mds_info_t const* l, MDSMap::mds_info_t const* r) {
-    ceph_assert(l->rank != MDS_RANK_NONE);
-    ceph_assert(r->rank != MDS_RANK_NONE);
-    ceph_assert(l->state <= MDSMap::STATE_ACTIVE);
-    ceph_assert(r->state <= MDSMap::STATE_ACTIVE);
-    if (l->rank == r->rank) {
-      return l->state < r->state;
-    } else {
-      return l->rank > r->rank;
-    }
-  };
-
-  std::priority_queue<MDSMap::mds_info_t const*, std::vector<MDSMap::mds_info_t const*>, decltype(less_leader)> member_info(less_leader);
   QuiesceClusterMembership membership;
-
   QuiesceInterface::PeerId me = mds_gid_t(monc->get_global_id());
 
-  for (auto&& [gid, info] : mdsmap->get_mds_info()) {
-    // if it has a rank and state <= ACTIVE, it's good enough
-    // if (info.rank != MDS_RANK_NONE && info.state <= MDSMap::STATE_ACTIVE) {
-    if (info.rank != MDS_RANK_NONE && info.state == MDSMap::STATE_ACTIVE) {
-      member_info.push(&info);
-      membership.members.insert(info.global_id);
-    }
-  }
-
-  QuiesceInterface::PeerId leader = 
-    member_info.empty() 
-    ? QuiesceClusterMembership::INVALID_MEMBER 
-    : member_info.top()->global_id;
+  mdsmap->get_quiesce_db_cluster(membership.leader, membership.members);
 
   membership.epoch = mdsmap->get_epoch();
-  membership.leader = leader;
   membership.me = me;
   membership.fs_name = mdsmap->get_fs_name();
 
-  dout(5) << "epoch:" << membership.epoch << " me:" << me << " leader:" << leader << " members:" << membership.members 
+  dout(5) << "epoch:" << membership.epoch << " me:" << me << " leader:" << membership.leader << " members:" << membership.members 
     << (mdsmap->is_degraded() ? " (degraded)" : "") << dendl;
 
-  if (leader != QuiesceClusterMembership::INVALID_MEMBER) {
+  if (membership.leader != QuiesceClusterMembership::INVALID_MEMBER) {
     membership.send_ack = [=, this](QuiesceMap&& ack) {
-      if (me == leader) {
+      if (me == membership.leader) {
         // loopback
         quiesce_db_manager->submit_ack_from(me, std::move(ack));
         return 0;
       } else {
         std::lock_guard guard(mds_lock);
 
-        if (mdsmap->get_state_gid(leader) == MDSMap::STATE_NULL) {
-          dout(5) << "couldn't find the leader " << leader << " in the map" << dendl;
+        if (mdsmap->get_state_gid(membership.leader) == MDSMap::STATE_NULL) {
+          dout(5) << "couldn't find the leader " << membership.leader << " in the map" << dendl;
           return -ENOENT;
         }
-        auto addrs = mdsmap->get_info_gid(leader).addrs;
+        auto addrs = mdsmap->get_info_gid(membership.leader).addrs;
 
         auto ack_msg = make_message<MMDSQuiesceDbAck>();
-        dout(10) << "sending ack " << ack << " to the leader " << leader << dendl;
+        dout(10) << "sending ack " << ack << " to the leader " << membership.leader << dendl;
         ack_msg->encode_payload_from(me, ack);
         return send_message_mds(ack_msg, addrs);
       }
diff --git a/src/mds/QuiesceDbManager.h b/src/mds/QuiesceDbManager.h
index a20b0cfc70ac..7b7e77ff2f67 100644
--- a/src/mds/QuiesceDbManager.h
+++ b/src/mds/QuiesceDbManager.h
@@ -19,14 +19,6 @@
 #include <set>
 #include <queue>
 
-template <>
-struct std::hash<mds_gid_t> {
-  size_t operator()(const mds_gid_t& gid) const
-  {
-    return hash<uint64_t> {}(gid);
-  }
-};
-
 struct QuiesceClusterMembership {
   static const QuiesceInterface::PeerId INVALID_MEMBER;
 
@@ -36,7 +28,7 @@ struct QuiesceClusterMembership {
 
   QuiesceInterface::PeerId me = INVALID_MEMBER;
   QuiesceInterface::PeerId leader = INVALID_MEMBER;
-  std::set<QuiesceInterface::PeerId> members;
+  std::unordered_set<QuiesceInterface::PeerId> members;
 
   // A courier interface to decouple from the messaging layer
   // Failures can be ignored, manager will call this repeatedly if needed
@@ -69,7 +61,7 @@ class QuiesceDbManager {
 
     // ============================
     // quiesce db leader interface: 
-    //    -> EPERM unless this is the leader
+    //    -> ENOTTY unless this is the leader
     
     // client interface to the DB
     int submit_request(RequestContext* ctx) {
diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index aeefe5c9592d..c319715a6a85 100644
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -14,6 +14,8 @@
 
 #include <regex>
 #include <sstream>
+#include <queue>
+#include <ranges>
 #include <boost/utility.hpp>
 
 #include "MDSMonitor.h"
@@ -174,11 +176,61 @@ void MDSMonitor::create_pending()
   dout(10) << "create_pending e" << fsmap.get_epoch() << dendl;
 }
 
+void MDSMonitor::assign_quiesce_db_leader(FSMap &fsmap) {
+
+  // the quiesce leader is the lowest rank with the highest state up to ACTIVE
+  auto less_leader = [](MDSMap::mds_info_t const* l, MDSMap::mds_info_t const* r) {
+    ceph_assert(l->rank != MDS_RANK_NONE);
+    ceph_assert(r->rank != MDS_RANK_NONE);
+    ceph_assert(l->state <= MDSMap::STATE_ACTIVE);
+    ceph_assert(r->state <= MDSMap::STATE_ACTIVE);
+    if (l->rank == r->rank) {
+      return l->state < r->state;
+    } else {
+      return l->rank > r->rank;
+    }
+  };
+
+  for (const auto& [fscid, fs] : std::as_const(fsmap)) {
+    auto &&mdsmap = fs.get_mds_map();
+
+    if (mdsmap.get_epoch() < fsmap.get_epoch()) {
+      // no changes in this fs, we can skip the calculation below
+      // NB! be careful with this clause when updating the leader selection logic.
+      // When the input from outside of this fsmap will affect the decision
+      // this clause will have to be updated, too.
+      continue;
+    }
+
+    std::priority_queue<MDSMap::mds_info_t const*, std::vector<MDSMap::mds_info_t const*>, decltype(less_leader)> 
+      member_info(less_leader);
+    
+    std::unordered_set<mds_gid_t> members;
+
+    for (auto&& [gid, info] : mdsmap.get_mds_info()) {
+      // if it has a rank and state <= ACTIVE, it's good enough
+      // if (info.rank != MDS_RANK_NONE && info.state <= MDSMap::STATE_ACTIVE) {
+      if (info.rank != MDS_RANK_NONE && info.state == MDSMap::STATE_ACTIVE) {
+        member_info.push(&info);
+        members.insert(info.global_id);
+      }
+    }
+
+    auto leader = member_info.empty() ? MDS_GID_NONE : member_info.top()->global_id;
+
+    fsmap.modify_filesystem(fscid, [&leader, &members](auto &writable_fs) -> bool {
+      return writable_fs.get_mds_map().update_quiesce_db_cluster(leader, std::move(members));
+    });
+  }
+}
+
 void MDSMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 {
   auto &pending = get_pending_fsmap_writeable();
   auto epoch = pending.get_epoch();
 
+  assign_quiesce_db_leader(pending);
+
   dout(10) << "encode_pending e" << epoch << dendl;
 
   // print map iff 'debug mon = 30' or higher
diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h
index 36d53fe4e483..25f27535c77a 100644
--- a/src/mon/MDSMonitor.h
+++ b/src/mon/MDSMonitor.h
@@ -129,6 +129,8 @@ class MDSMonitor : public PaxosService, public PaxosFSMap, protected CommandHand
   int load_metadata(std::map<mds_gid_t, Metadata>& m);
   void count_metadata(const std::string& field, ceph::Formatter *f);
 
+  void assign_quiesce_db_leader(FSMap &fsmap);
+
 public:
   void print_fs_summary(std::ostream& out) {
     get_fsmap().print_fs_summary(out);
diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 2e25f994b6b7..41e54fd8790e 100644
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1731,42 +1731,9 @@ def tell_command(self, daemon_type: str, daemon_id: str, cmd_dict: dict, inbuf:
         ))
 
         return r
-    
-    MDS_STATE_ORD = {
-        "down:dne":              0, # CEPH_MDS_STATE_DNE,
-        "down:stopped":         -1, # CEPH_MDS_STATE_STOPPED,
-        "down:damaged":         15, # CEPH_MDS_STATE_DAMAGED,
-        "up:boot":              -4, # CEPH_MDS_STATE_BOOT,
-        "up:standby":           -5, # CEPH_MDS_STATE_STANDBY,
-        "up:standby-replay":    -8, # CEPH_MDS_STATE_STANDBY_REPLAY,
-        "up:oneshot-replay":    -9, # CEPH_MDS_STATE_REPLAYONCE,
-        "up:creating":          -6, # CEPH_MDS_STATE_CREATING,
-        "up:starting":          -7, # CEPH_MDS_STATE_STARTING,
-        "up:replay":             8, # CEPH_MDS_STATE_REPLAY,
-        "up:resolve":            9, # CEPH_MDS_STATE_RESOLVE,
-        "up:reconnect":         10, # CEPH_MDS_STATE_RECONNECT,
-        "up:rejoin":            11, # CEPH_MDS_STATE_REJOIN,
-        "up:clientreplay":      12, # CEPH_MDS_STATE_CLIENTREPLAY,
-        "up:active":            13, # CEPH_MDS_STATE_ACTIVE,
-        "up:stopping":          14, # CEPH_MDS_STATE_STOPPING,
-    }
-    MDS_STATE_ACTIVE_ORD = MDS_STATE_ORD["up:active"]
-
-    def get_quiesce_leader_info(self, fscid: str) -> Optional[dict]:
-        """
-        Helper for `tell_quiesce_leader` to chose the mds to send the command to.
-
-        Quiesce DB is managed by a leader which is selected based on the current MDSMap
-        The logic is currently implemented both here and on the MDS side,
-        see MDSRank::quiesce_cluster_update().
-
-        Ideally, this logic should be part of the MDSMonitor and the result should
-        be exposed via a dedicated field in the map, but until that is implemented
-        this function will have to be kept in sync with the corresponding logic
-        on the MDS side
-        """
-        leader_info: Optional[dict] = None
 
+    def get_quiesce_leader_gid(self, fscid: str) -> Optional[int]:
+        leader_gid : Optional[int] = None
         for fs in self.get("fs_map")['filesystems']:
             if fscid != fs["id"]:
                 continue
@@ -1774,33 +1741,30 @@ def get_quiesce_leader_info(self, fscid: str) -> Optional[dict]:
             # quiesce leader is the lowest rank
             # with the highest state
             mdsmap = fs["mdsmap"]
-            for info in mdsmap['info'].values():
-                if info['rank'] == -1:
-                    continue
-                if leader_info is None:
-                    leader_info = info
-                else:
-                    if info['rank'] < leader_info['rank']:
-                        leader_info = info
-                    elif info['rank'] == leader_info['rank']:
-                        state_ord = self.MDS_STATE_ORD.get(info['state'])
-                        # if there are more than one daemons with the same rank
-                        # only one of them can be active
-                        if state_ord == self.MDS_STATE_ACTIVE_ORD:
-                            leader_info = info
+            leader_gid = mdsmap.get("qdb_leader", None)
             break
 
-        return leader_info
-
-    def tell_quiesce_leader(self, fscid: str, cmd_dict: dict) -> Tuple[int, str, str]:
-        qleader = self.get_quiesce_leader_info(fscid)
-        if qleader is None:
-            self.log.warn("Couldn't resolve the quiesce leader for fscid %s" % fscid)
-            return (-errno.ENOENT, "", "Couldn't resolve the quiesce leader for fscid %s" % fscid)
-        self.log.debug("resolved quiesce leader for fscid {fscid} at daemon '{name}' gid {gid} rank {rank} ({state})".format(fscid=fscid, **qleader))
-        # We use the one_shot here to cover for cases when the mds crashes
-        # without this parameter the client may get stuck awaiting response from a dead MDS
-        return self.tell_command('mds', str(qleader['gid']), cmd_dict, one_shot=True)
+        return leader_gid
+
+    def tell_quiesce_leader(self, leader: int, cmd_dict: dict) -> Tuple[int, str, str]:
+        max_retries = 5
+        for _ in range(max_retries):
+            # We use "one_shot" here to cover for cases when the mds crashes
+            # without this parameter the client may get stuck awaiting response from a dead MDS
+            # (which is particularly bad for the volumes plugin finisher thread)
+            rc, stdout, stderr = self.tell_command('mds', str(leader), cmd_dict, one_shot=True)
+            if rc == -errno.ENOTTY:
+                try:
+                    resp = json.loads(stdout)
+                    leader = int(resp['leader'])
+                    self.log.info("Retrying a quiesce db command with leader %d" % leader)
+                except Exception as e:
+                    self.log.error("Couldn't parse ENOTTY response from an mds with error: %s\n%s" % (str(e), stdout))
+                    break
+            else:
+                break
+
+        return (rc, stdout, stderr)
 
     def send_command(
             self,
diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index ef3b171af762..8fb7defed711 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -456,19 +456,20 @@ def quiesce(self, cmd):
         volname    = cmd['vol_name']
         default_group_name  = cmd.get('group_name', None)
         roots = []
-        fscid = None
+        leader_gid = cmd.get('with_leader', None)
 
         with open_volume(self, volname) as fs_handle:
-            fscid = fs_handle.get_fscid()
+            if leader_gid is None:
+                fscid = fs_handle.get_fscid()
+                leader_gid = self.mgr.get_quiesce_leader_gid(fscid)
+                if leader_gid is None:
+                    return -errno.ENOENT, "", "Couldn't resolve the quiesce leader for volume %s (%s)" % (volname, fscid)
 
             if cmd.get('leader', False):
-                leader_info = self.mgr.get_quiesce_leader_info(fscid)
-                if leader_info is None:
-                    return -errno.ENOENT, "", "Couldn't resolve the quiesce leader for volume %s (%s)" % (volname, fscid)
                 return (
                     0,
-                    "mds.%d" % leader_info['gid'],
-                    "Resolved the quiesce leader for volume '{volname}' as daemon '{name}' ({gid}) {state} rank {rank}".format(volname=volname, **leader_info)
+                    "mds.%d" % leader_gid,
+                    "Resolved the quiesce leader for volume '{volname}' as gid {gid}".format(volname=volname, gid=leader_gid)
                 )
 
 
@@ -493,7 +494,7 @@ def quiesce(self, cmd):
         cmd['roots'] = roots
         cmd['prefix'] = 'quiesce db'
 
-        return self.mgr.tell_quiesce_leader(fscid, cmd)
+        return self.mgr.tell_quiesce_leader(leader_gid, cmd)
 
     def set_user_metadata(self, **kwargs):
         ret        = 0, "", ""
diff --git a/src/pybind/mgr/volumes/module.py b/src/pybind/mgr/volumes/module.py
index ee36005d406d..e059648261e9 100644
--- a/src/pybind/mgr/volumes/module.py
+++ b/src/pybind/mgr/volumes/module.py
@@ -291,7 +291,8 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                    'name=all,type=CephBool,req=false '
                    'name=cancel,type=CephBool,req=false '
                    'name=group_name,type=CephString,req=false '
-                   'name=leader,type=CephBool,req=false ',
+                   'name=leader,type=CephBool,req=false '
+                   'name=with_leader,type=CephInt,range=0,req=false ',
             'desc': "Manage quiesce sets of subvolumes",
             'perm': 'rw'
         },
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
index e820db404483..3c48474fa32b 100644
--- a/src/test/mds/TestQuiesceDb.cc
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -136,7 +136,7 @@ class QuiesceDbTest: public testing::Test {
     {
       ++epoch;
       ASSERT_GE(leader_and_replicas.size(), 1);
-      std::set<QuiesceInterface::PeerId> members(leader_and_replicas.begin(), leader_and_replicas.end());
+      std::unordered_set<QuiesceInterface::PeerId> members(leader_and_replicas.begin(), leader_and_replicas.end());
       auto leader = leader_and_replicas[0];
       for (const auto &[this_peer, mgr] : managers) {
         QuiesceClusterMembership mem = {

From 205fd3388ef0ed3011bb21384e79b34b6a6611ec Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Thu, 29 Feb 2024 14:08:18 +0200
Subject: [PATCH 2130/2492] mds/quiesce: declare QuiesceDbPeerListing and
 QuiesceDbPeerAck

With these dedicated structs we can fully defer to QuiesceDbEncoding
when encoding/decoding quiesce db messages

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/include/cephfs/types.h          | 11 ++++++++++
 src/mds/MDSRankQuiesce.cc           | 30 ++++++++++++++--------------
 src/mds/QuiesceDb.h                 | 31 +++++++++++++++++++++++++++--
 src/mds/QuiesceDbEncoding.h         | 28 ++++++++++++++++++++++++++
 src/mds/QuiesceDbManager.cc         |  7 ++++---
 src/mds/QuiesceDbManager.h          | 17 ++++++++--------
 src/messages/MMDSQuiesceDbAck.h     | 19 ++++++------------
 src/messages/MMDSQuiesceDbListing.h | 19 ++++++------------
 src/test/mds/TestQuiesceDb.cc       |  4 ++--
 9 files changed, 110 insertions(+), 56 deletions(-)

diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 23f6e44a9c2b..1b66929240b6 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -56,6 +56,17 @@ struct std::hash<mds_gid_t> {
   }
 };
 
+inline void encode(const mds_gid_t &v, bufferlist& bl, uint64_t features = 0) {
+  uint64_t vv = v;
+  encode_raw(vv, bl);
+}
+
+inline void decode(mds_gid_t &v, bufferlist::const_iterator& p) {
+  uint64_t vv;
+  decode_raw(vv, p);
+  v = vv;
+}
+
 typedef int32_t fs_cluster_id_t;
 constexpr fs_cluster_id_t FS_CLUSTER_ID_NONE = -1;
 
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index bcc8e149ed7f..53d82cb13648 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -260,7 +260,7 @@ void MDSRank::quiesce_cluster_update() {
     membership.send_ack = [=, this](QuiesceMap&& ack) {
       if (me == membership.leader) {
         // loopback
-        quiesce_db_manager->submit_ack_from(me, std::move(ack));
+        quiesce_db_manager->submit_peer_ack({me, std::move(ack)});
         return 0;
       } else {
         std::lock_guard guard(mds_lock);
@@ -273,7 +273,7 @@ void MDSRank::quiesce_cluster_update() {
 
         auto ack_msg = make_message<MMDSQuiesceDbAck>();
         dout(10) << "sending ack " << ack << " to the leader " << membership.leader << dendl;
-        ack_msg->encode_payload_from(me, ack);
+        ack_msg->encode_payload_from({me, std::move(ack)});
         return send_message_mds(ack_msg, addrs);
       }
     };
@@ -287,7 +287,7 @@ void MDSRank::quiesce_cluster_update() {
       auto addrs = mdsmap->get_info_gid(to).addrs;
       auto listing_msg = make_message<MMDSQuiesceDbListing>();
       dout(10) << "sending listing " << db << " to the peer " << to << dendl;
-      listing_msg->encode_payload_from(me, db);
+      listing_msg->encode_payload_from({me, std::move(db)});
       return send_message_mds(listing_msg, addrs);
     };
   }
@@ -363,16 +363,16 @@ bool MDSRank::quiesce_dispatch(const cref_t<Message> &m) {
       {
         const auto& req = ref_cast<MMDSQuiesceDbListing>(m);
         mds_gid_t gid;
-        QuiesceDbListing db_listing;
-        req->decode_payload_into(gid, db_listing);
+        QuiesceDbPeerListing peer_listing;
+        req->decode_payload_into(peer_listing);
         if (quiesce_db_manager) {
-          dout(10) << "got " << db_listing << " from peer " << gid << dendl;
-          int result = quiesce_db_manager->submit_listing_from(gid, std::move(db_listing));
+          dout(10) << "got " << peer_listing << dendl;
+          int result = quiesce_db_manager->submit_peer_listing(std::move(peer_listing));
           if (result != 0) {
-            dout(3) << "error (" << result << ") submitting " << db_listing << " from peer " << gid << dendl;
+            dout(3) << "error (" << result << ") submitting " << peer_listing << dendl;
           }
         } else {
-          dout(5) << "no db manager to process " << db_listing << dendl;
+          dout(5) << "no db manager to process " << peer_listing << dendl;
         }
         return true;
       }
@@ -380,16 +380,16 @@ bool MDSRank::quiesce_dispatch(const cref_t<Message> &m) {
       {
         const auto& req = ref_cast<MMDSQuiesceDbAck>(m);
         mds_gid_t gid;
-        QuiesceMap diff_map;
-        req->decode_payload_into(gid, diff_map);
+        QuiesceDbPeerAck peer_ack;
+        req->decode_payload_into(peer_ack);
         if (quiesce_db_manager) {
-          dout(10) << "got ack " << diff_map << " from peer " << gid << dendl;
-          int result = quiesce_db_manager->submit_ack_from(gid, std::move(diff_map));
+          dout(10) << "got " << peer_ack << dendl;
+          int result = quiesce_db_manager->submit_peer_ack(std::move(peer_ack));
           if (result != 0) {
-            dout(3) << "error (" << result << ") submitting an ack from peer " << gid << dendl;
+            dout(3) << "error (" << result << ") submitting and ack from " << peer_ack.origin << dendl;
           }
         } else {
-          dout(5) << "no db manager to process an ack: " << diff_map << dendl;
+          dout(5) << "no db manager to process " << peer_ack << dendl;
         }
         return true;
       }
diff --git a/src/mds/QuiesceDb.h b/src/mds/QuiesceDb.h
index 5fedcfe5a5c1..8fccc43d448e 100644
--- a/src/mds/QuiesceDb.h
+++ b/src/mds/QuiesceDb.h
@@ -119,6 +119,10 @@ using QuiesceSetId = std::string;
 using QuiesceRoot = std::string;
 using QuiesceSetVersion = uint64_t;
 
+namespace QuiesceInterface {
+  using PeerId = mds_gid_t;
+}
+
 struct QuiesceDbVersion {
   epoch_t epoch;
   QuiesceSetVersion set_version;
@@ -563,7 +567,7 @@ operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbRequest& req)
 ///         contain all sets that have their version > than the last acked by the peer.
 struct QuiesceDbListing {
   QuiesceDbVersion db_version = {0, 0};
-  /// @brief  Crusially, the precise `db_age` must be included in every db listing
+  /// @brief  Crucially, the precise `db_age` must be included in every db listing
   ///         This data is used by all replicas to update their calculated DB TIME ZERO.
   ///         All events in the database are measured relative to the DB TIME ZERO
   QuiesceTimeInterval db_age = QuiesceTimeInterval::zero();
@@ -600,6 +604,18 @@ operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbListing& dbl)
   return os << "q-db[v:" << dbl.db_version << " sets:" << active << "/" << inactive << "]";
 }
 
+struct QuiesceDbPeerListing {
+  QuiesceInterface::PeerId origin;
+  QuiesceDbListing db;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbPeerListing& dbl)
+{
+  return os << dbl.db << " from " << dbl.origin;
+}
+
 /// @brief  `QuiesceMap` is a root-centric representation of the quiesce database
 ///         It lists roots with their effective states as of particular version.
 ///         Additionally, the same structure is used by the peers when reporting
@@ -662,6 +678,18 @@ operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceMap& map)
   return os << "q-map[v:" << map.db_version << " roots:" << active << "/" << inactive << "]";
 }
 
+struct QuiesceDbPeerAck {
+  QuiesceInterface::PeerId origin;
+  QuiesceMap diff_map;
+};
+
+template <class CharT, class Traits>
+static std::basic_ostream<CharT, Traits>&
+operator<<(std::basic_ostream<CharT, Traits>& os, const QuiesceDbPeerAck& ack)
+{
+  return os << "ack " << ack.diff_map << " from " << ack.origin;
+}
+
 inline QuiesceTimeInterval interval_saturate_add(QuiesceTimeInterval lhs, QuiesceTimeInterval rhs)
 {
   // assuming an unsigned time interval.
@@ -685,7 +713,6 @@ inline QuiesceTimePoint interval_saturate_add_now(QuiesceTimeInterval interval)
 };
 
 namespace QuiesceInterface {
-  using PeerId = mds_gid_t;
   /// @brief  A callback from the manager to the agent with an up-to-date root list
   ///         The map is mutable and will be used as synchronous agent ack if the return value is true
   using AgentNotify = std::function<bool(QuiesceMap&)>;
diff --git a/src/mds/QuiesceDbEncoding.h b/src/mds/QuiesceDbEncoding.h
index 756e63cf9e3e..ff2167c5d405 100644
--- a/src/mds/QuiesceDbEncoding.h
+++ b/src/mds/QuiesceDbEncoding.h
@@ -15,6 +15,11 @@
 #include "include/encoding.h"
 #include <stdint.h>
 
+struct QuiesceDbEncoding {
+  static constexpr int version = 1;
+  static constexpr int compat = 1;
+};
+
 void encode(QuiesceDbVersion const& v, bufferlist& bl, uint64_t features = 0)
 {
   encode(v.epoch, bl, features);
@@ -131,6 +136,18 @@ void decode(QuiesceDbListing& listing, bufferlist::const_iterator& p)
   decode(listing.sets, p);
 }
 
+void encode(QuiesceDbPeerListing const& listing, bufferlist& bl, uint64_t features = 0)
+{
+  encode(listing.origin, bl, features);
+  encode(listing.db, bl, features);
+}
+
+void decode(QuiesceDbPeerListing& listing, bufferlist::const_iterator& p)
+{
+  decode(listing.origin, p);
+  decode(listing.db, p);
+}
+
 void encode(QuiesceMap::RootInfo const& root, bufferlist& bl, uint64_t features = 0)
 {
   encode(root.state, bl, features);
@@ -155,3 +172,14 @@ void decode(QuiesceMap& map, bufferlist::const_iterator& p)
   decode(map.roots, p);
 }
 
+void encode(QuiesceDbPeerAck const& ack, bufferlist& bl, uint64_t features = 0)
+{
+  encode(ack.origin, bl, features);
+  encode(ack.diff_map, bl, features);
+}
+
+void decode(QuiesceDbPeerAck& ack, bufferlist::const_iterator& p)
+{
+  decode(ack.origin, p);
+  decode(ack.diff_map, p);
+}
diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
index 1ff998f60686..ff497aafe452 100644
--- a/src/mds/QuiesceDbManager.cc
+++ b/src/mds/QuiesceDbManager.cc
@@ -256,7 +256,7 @@ QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates
 {
   // as a replica, we only care about the latest update
   while (db_updates.size() > 1) {
-    dout(10) << "skipping an older update from " << db_updates.front().first << " version " << db_updates.front().second.db_version << dendl;
+    dout(10) << "skipping an older update from " << db_updates.front().origin << " version " << db_updates.front().db.db_version << dendl;
     db_updates.pop();
   }
 
@@ -265,7 +265,7 @@ QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates
     return QuiesceTimeInterval::max();
   }
 
-  QuiesceDbListing &update = db_updates.back().second;
+  QuiesceDbListing &update = db_updates.back().db;
 
   if (update.db_version.epoch != membership.epoch) {
     dout(10) << "ignoring db update from another epoch: " << update.db_version << " != " << db_version() << dendl;
@@ -317,7 +317,8 @@ bool QuiesceDbManager::leader_bootstrap(decltype(pending_db_updates)&& db_update
 
   // only consider db submissions from unknown peers
   while (!unknown_peers.empty() && !db_updates.empty()) {
-    auto &[from, update] = db_updates.front();
+    auto &from = db_updates.front().origin;
+    auto &update = db_updates.front().db;
     if (update.db_version.epoch == membership.epoch && unknown_peers.erase(from) > 0) {
       // see if this peer's version is newer than mine
       if (db.set_version < update.db_version.set_version) {
diff --git a/src/mds/QuiesceDbManager.h b/src/mds/QuiesceDbManager.h
index 7b7e77ff2f67..08c8392d9815 100644
--- a/src/mds/QuiesceDbManager.h
+++ b/src/mds/QuiesceDbManager.h
@@ -75,19 +75,20 @@ class QuiesceDbManager {
       submit_condition.notify_all();
       return 0;
     }
+  
     // acks the messaging system
-    int submit_ack_from(QuiesceInterface::PeerId sender, const QuiesceMap& diff_map) {
+    int submit_peer_ack(QuiesceDbPeerAck&& ack) {
       std::lock_guard l(submit_mutex);
 
       if (!cluster_membership || !cluster_membership->is_leader()) {
         return -EPERM;
       }
 
-      if (!cluster_membership->members.contains(sender)) {
+      if (!cluster_membership->members.contains(ack.origin)) {
         return -ESTALE;
       }
 
-      pending_acks.push({ sender, diff_map });
+      pending_acks.push(std::move(ack));
       submit_condition.notify_all();
       return 0;
     }
@@ -97,18 +98,18 @@ class QuiesceDbManager {
     //    -> EPERM if this is the leader
 
     // process an incoming listing from a leader
-    int submit_listing_from(QuiesceInterface::PeerId sender, QuiesceDbListing&& listing) {
+    int submit_peer_listing(QuiesceDbPeerListing&& listing) {
       std::lock_guard l(submit_mutex);
 
       if (!cluster_membership) {
         return -EPERM;
       }
 
-      if (cluster_membership->epoch != listing.db_version.epoch) {
+      if (cluster_membership->epoch != listing.db.db_version.epoch) {
         return -ESTALE;
       }
 
-      pending_db_updates.push({sender, std::move(listing)});
+      pending_db_updates.push(std::move(listing));
       submit_condition.notify_all();
       return 0;
     }
@@ -187,8 +188,8 @@ class QuiesceDbManager {
 
     std::optional<AgentCallback> agent_callback;
     std::optional<QuiesceClusterMembership> cluster_membership;
-    std::queue<std::pair<QuiesceInterface::PeerId, QuiesceDbListing>> pending_db_updates;
-    std::queue<std::pair<QuiesceInterface::PeerId, QuiesceMap>> pending_acks;
+    std::queue<QuiesceDbPeerListing> pending_db_updates;
+    std::queue<QuiesceDbPeerAck> pending_acks;
     std::deque<RequestContext*> pending_requests;
 
     class QuiesceDbThread : public Thread {
diff --git a/src/messages/MMDSQuiesceDbAck.h b/src/messages/MMDSQuiesceDbAck.h
index 1d56451e89bc..b68445dfb091 100644
--- a/src/messages/MMDSQuiesceDbAck.h
+++ b/src/messages/MMDSQuiesceDbAck.h
@@ -34,15 +34,10 @@ class MMDSQuiesceDbAck final : public MMDSOp {
     // noop to prevent unnecessary overheads
   }
 
-  void encode_payload_from(mds_gid_t const&gid, QuiesceMap const&diff_map)
+  void encode_payload_from(QuiesceDbPeerAck const& ack)
   {
-    using ceph::encode;
-
-    ceph_assert(gid != MDS_GID_NONE);
-
-    ENCODE_START(1, 1, payload);
-    encode(gid, payload);
-    encode(diff_map, payload);
+    ENCODE_START(QuiesceDbEncoding::version, QuiesceDbEncoding::compat, payload);
+    encode(ack, payload);
     ENCODE_FINISH(payload);
   }
 
@@ -50,13 +45,11 @@ class MMDSQuiesceDbAck final : public MMDSOp {
     // noop to prevent unnecessary overheads
   }
 
-  void decode_payload_into(mds_gid_t &gid, QuiesceMap &diff_map) const
+  void decode_payload_into(QuiesceDbPeerAck &ack) const
   {
-    using ceph::decode;
     auto p = payload.cbegin();
-    DECODE_START(1, p);
-    decode(gid, p);
-    decode(diff_map, p);
+    DECODE_START(QuiesceDbEncoding::version, p);
+    decode(ack, p);
     DECODE_FINISH(p);
   }
 
diff --git a/src/messages/MMDSQuiesceDbListing.h b/src/messages/MMDSQuiesceDbListing.h
index f57de50e22fb..5fd068adb560 100644
--- a/src/messages/MMDSQuiesceDbListing.h
+++ b/src/messages/MMDSQuiesceDbListing.h
@@ -33,15 +33,10 @@ class MMDSQuiesceDbListing final : public MMDSOp {
     // noop to prevent unnecessary overheads
   }
 
-  void encode_payload_from(mds_gid_t const& gid, QuiesceDbListing const& db_listing)
+  void encode_payload_from(QuiesceDbPeerListing const& peer_listing)
   {
-    using ceph::encode;
-
-    ceph_assert(gid != MDS_GID_NONE);
-
-    ENCODE_START(1, 1, payload);
-    encode(gid, payload);
-    encode(db_listing, payload);
+    ENCODE_START(QuiesceDbEncoding::version, QuiesceDbEncoding::compat, payload);
+    encode(peer_listing, payload);
     ENCODE_FINISH(payload);
   }
 
@@ -49,13 +44,11 @@ class MMDSQuiesceDbListing final : public MMDSOp {
     // noop to prevent unnecessary overheads
   }
 
-  void decode_payload_into(mds_gid_t &gid, QuiesceDbListing &db_listing) const
+  void decode_payload_into(QuiesceDbPeerListing &peer_listing) const
   {
-    using ceph::decode;
     auto p = payload.cbegin();
-    DECODE_START(1, p);
-    decode(gid, p);
-    decode(db_listing, p);
+    DECODE_START(QuiesceDbEncoding::version, p);
+    decode(peer_listing, p);
     DECODE_FINISH(p);
   }
 
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
index 3c48474fa32b..8cd168424e56 100644
--- a/src/test/mds/TestQuiesceDb.cc
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -151,7 +151,7 @@ class QuiesceDbTest: public testing::Test {
             if (epoch == this->epoch) {
               if (this->managers.contains(recipient)) {
                 dout(10) << "listing from " << me << " (leader=" << leader << ") to " << recipient << " for version " << listing.db_version << " with " << listing.sets.size() << " sets" << dendl;
-                this->managers[recipient]->submit_listing_from(me, std::move(listing));
+                this->managers[recipient]->submit_peer_listing({me, std::move(listing)});
                 comms_cond.notify_all();
                 return 0;
               }
@@ -181,7 +181,7 @@ class QuiesceDbTest: public testing::Test {
                     it++;
                   }
                 }
-                this->managers[leader]->submit_ack_from(me, std::move(diff_map));
+                this->managers[leader]->submit_peer_ack({me, std::move(diff_map)});
                 comms_cond.notify_all();
                 l.unlock();
                 while(!done_hooks.empty()) {

From 3e012f7ba5b8871d1bcf74d402c58553e18835dd Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Mon, 4 Mar 2024 00:45:07 +0200
Subject: [PATCH 2131/2492] mds/quiesce-db: incorporate review comments

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/FSMap.cc                    |  7 ++++
 src/mds/MDSMap.h                    |  7 +++-
 src/mds/MDSRankQuiesce.cc           | 12 ++++++
 src/mds/QuiesceAgent.cc             |  2 +-
 src/mds/QuiesceDb.h                 |  4 +-
 src/mds/QuiesceDbEncoding.h         | 61 +++++++++++++++++++++++------
 src/mds/QuiesceDbManager.cc         |  2 +-
 src/messages/MMDSQuiesceDbAck.h     | 12 ++----
 src/messages/MMDSQuiesceDbListing.h | 10 ++---
 src/test/mds/TestQuiesceAgent.cc    |  6 +--
 src/test/mds/TestQuiesceDb.cc       | 15 +++++++
 11 files changed, 102 insertions(+), 36 deletions(-)

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index 2f14a780b84e..a266ad253afb 100644
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -13,6 +13,8 @@
  */
 
 #include <ostream>
+#include <algorithm>
+#include <ranges>
 
 #include "FSMap.h"
 #include "common/debug.h"
@@ -868,6 +870,11 @@ void FSMap::sanity(bool pending) const
       ceph_assert(info.compat.writeable(fs.mds_map.compat));
     }
 
+    auto const& leader = fs.mds_map.get_quiesce_db_cluster_leader();
+    auto const& members = fs.mds_map.get_quiesce_db_cluster_members();
+    ceph_assert(leader == MDS_GID_NONE || members.contains(leader));
+    ceph_assert(std::ranges::all_of(members, [&infos = fs.mds_map.mds_info](auto m){return infos.contains(m);}));
+
     for (const auto &j : fs.mds_map.up) {
       mds_rank_t rank = j.first;
       ceph_assert(fs.mds_map.in.count(rank) == 1);
diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 2dd8fba83423..c2cb9b009ff5 100644
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -318,10 +318,15 @@ class MDSMap {
     members = qdb_cluster_members; 
   }
 
-  mds_gid_t get_quiesce_db_cluster_leader() {
+  mds_gid_t get_quiesce_db_cluster_leader() const {
     return qdb_cluster_leader;
   }
 
+  std::unordered_set<mds_gid_t> const& get_quiesce_db_cluster_members() const
+  {
+    return qdb_cluster_members;
+  }
+
   bool update_quiesce_db_cluster(mds_gid_t const& leader, std::same_as<std::unordered_set<mds_gid_t>> auto && members) {
     if (leader == qdb_cluster_leader && members == qdb_cluster_members) {
       return false;
diff --git a/src/mds/MDSRankQuiesce.cc b/src/mds/MDSRankQuiesce.cc
index 53d82cb13648..004e15364831 100644
--- a/src/mds/MDSRankQuiesce.cc
+++ b/src/mds/MDSRankQuiesce.cc
@@ -1,3 +1,15 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2024 IBM, Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
 #include "MDSRank.h"
 #include "MDCache.h"
 
diff --git a/src/mds/QuiesceAgent.cc b/src/mds/QuiesceAgent.cc
index c1550549ba67..1b5dfe44460e 100644
--- a/src/mds/QuiesceAgent.cc
+++ b/src/mds/QuiesceAgent.cc
@@ -182,7 +182,7 @@ void* QuiesceAgent::agent_thread_main() {
         dout(3) << "got error: " << rc << " trying to send " << ack << dendl;
       }
     }
-    ack.reset();
+    ack.clear();
 
     lock.lock();
 
diff --git a/src/mds/QuiesceDb.h b/src/mds/QuiesceDb.h
index 8fccc43d448e..e95bfcf59e33 100644
--- a/src/mds/QuiesceDb.h
+++ b/src/mds/QuiesceDb.h
@@ -464,7 +464,7 @@ struct QuiesceDbRequest {
 
     config(*this);
   }
-  void reset() {
+  void clear() {
     reset([](auto&r){});
   }
 
@@ -644,7 +644,7 @@ struct QuiesceMap {
   };
   using Roots = std::unordered_map<QuiesceRoot, RootInfo>;
   Roots roots;
-  void reset() {
+  void clear() {
     db_version = {0, 0};
     roots.clear();
   }
diff --git a/src/mds/QuiesceDbEncoding.h b/src/mds/QuiesceDbEncoding.h
index ff2167c5d405..c76ed2d0c523 100644
--- a/src/mds/QuiesceDbEncoding.h
+++ b/src/mds/QuiesceDbEncoding.h
@@ -15,21 +15,20 @@
 #include "include/encoding.h"
 #include <stdint.h>
 
-struct QuiesceDbEncoding {
-  static constexpr int version = 1;
-  static constexpr int compat = 1;
-};
-
 void encode(QuiesceDbVersion const& v, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(v.epoch, bl, features);
   encode(v.set_version, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceDbVersion& v, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(v.epoch, p);
   decode(v.set_version, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceState const & state, bufferlist& bl, uint64_t features=0)
@@ -60,48 +59,61 @@ void decode(QuiesceTimeInterval & interval, bufferlist::const_iterator& p)
 
 void encode(RecordedQuiesceState const& rstate, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(rstate.state, bl, features);
   encode(rstate.at_age.count(), bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(RecordedQuiesceState& rstate, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(rstate.state, p);
   decode(rstate.at_age, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceSet::MemberInfo const& member, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(member.rstate, bl, features);
   encode(member.excluded, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceSet::MemberInfo& member, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(member.rstate, p);
   decode(member.excluded, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceSet const& set, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(set.version, bl, features);
   encode(set.rstate, bl, features);
-  encode(set.timeout, bl, features);
-  encode(set.expiration, bl, features);
+  ::encode(set.timeout, bl, features);
+  ::encode(set.expiration, bl, features);
   encode(set.members, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceSet& set, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(set.version, p);
   decode(set.rstate, p);
-  decode(set.timeout, p);
-  decode(set.expiration, p);
+  ::decode(set.timeout, p);
+  ::decode(set.expiration, p);
   decode(set.members, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceDbRequest const& req, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(req.control.raw, bl, features);
   encode(req.set_id, bl);
   encode(req.if_version, bl);
@@ -109,10 +121,12 @@ void encode(QuiesceDbRequest const& req, bufferlist& bl, uint64_t features = 0)
   encode(req.expiration, bl);
   encode(req.await, bl);
   encode(req.roots, bl);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceDbRequest& req, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(req.control.raw, p);
   decode(req.set_id, p);
   decode(req.if_version, p);
@@ -120,66 +134,87 @@ void decode(QuiesceDbRequest& req, bufferlist::const_iterator& p)
   decode(req.expiration, p);
   decode(req.await, p);
   decode(req.roots, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceDbListing const& listing, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(listing.db_version, bl, features);
-  encode(listing.db_age, bl, features);
+  ::encode(listing.db_age, bl, features);
   encode(listing.sets, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceDbListing& listing, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(listing.db_version, p);
-  decode(listing.db_age, p);
+  ::decode(listing.db_age, p);
   decode(listing.sets, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceDbPeerListing const& listing, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(listing.origin, bl, features);
   encode(listing.db, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceDbPeerListing& listing, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(listing.origin, p);
   decode(listing.db, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceMap::RootInfo const& root, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(root.state, bl, features);
-  encode(root.ttl, bl, features);
+  ::encode(root.ttl, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceMap::RootInfo& root, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(root.state, p);
-  decode(root.ttl, p);
+  ::decode(root.ttl, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceMap const& map, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(map.db_version, bl, features);
   encode(map.roots, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceMap& map, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(map.db_version, p);
   decode(map.roots, p);
+  DECODE_FINISH(p);
 }
 
 void encode(QuiesceDbPeerAck const& ack, bufferlist& bl, uint64_t features = 0)
 {
+  ENCODE_START(1, 1, bl);
   encode(ack.origin, bl, features);
   encode(ack.diff_map, bl, features);
+  ENCODE_FINISH(bl);
 }
 
 void decode(QuiesceDbPeerAck& ack, bufferlist::const_iterator& p)
 {
+  DECODE_START(1, p);
   decode(ack.origin, p);
   decode(ack.diff_map, p);
+  DECODE_FINISH(p);
 }
diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
index ff497aafe452..ca69fef73fda 100644
--- a/src/mds/QuiesceDbManager.cc
+++ b/src/mds/QuiesceDbManager.cc
@@ -421,7 +421,7 @@ void QuiesceDbManager::leader_record_ack(QuiesceInterface::PeerId from, QuiesceM
   if (diff_map.db_version > db_version()) {
     dout(3) << "ignoring unknown version ack by rank " << from << " (" << diff_map.db_version << " > " << db_version() << ")" << dendl;
     dout(5) << "will send the peer a full DB" << dendl;
-    info.diff_map.reset();
+    info.diff_map.clear();
   } else {
     info.diff_map = std::move(diff_map);
     info.last_seen = QuiesceClock::now();
diff --git a/src/messages/MMDSQuiesceDbAck.h b/src/messages/MMDSQuiesceDbAck.h
index b68445dfb091..48bc5e37a86a 100644
--- a/src/messages/MMDSQuiesceDbAck.h
+++ b/src/messages/MMDSQuiesceDbAck.h
@@ -3,12 +3,12 @@
 /*
  * Ceph - scalable distributed file system
  *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ * Copyright (C) 2024 IBM, Red Hat
  *
  * This is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License version 2.1, as published by the Free Software 
- * Foundation.  See file COPYING.
+ * Foundation. See file COPYING.
  * 
  */
 
@@ -36,9 +36,7 @@ class MMDSQuiesceDbAck final : public MMDSOp {
 
   void encode_payload_from(QuiesceDbPeerAck const& ack)
   {
-    ENCODE_START(QuiesceDbEncoding::version, QuiesceDbEncoding::compat, payload);
-    encode(ack, payload);
-    ENCODE_FINISH(payload);
+    ::encode(ack, payload);
   }
 
   void decode_payload() override {
@@ -48,9 +46,7 @@ class MMDSQuiesceDbAck final : public MMDSOp {
   void decode_payload_into(QuiesceDbPeerAck &ack) const
   {
     auto p = payload.cbegin();
-    DECODE_START(QuiesceDbEncoding::version, p);
-    decode(ack, p);
-    DECODE_FINISH(p);
+    ::decode(ack, p);
   }
 
 private:
diff --git a/src/messages/MMDSQuiesceDbListing.h b/src/messages/MMDSQuiesceDbListing.h
index 5fd068adb560..39d72fb8eb4b 100644
--- a/src/messages/MMDSQuiesceDbListing.h
+++ b/src/messages/MMDSQuiesceDbListing.h
@@ -3,7 +3,7 @@
 /*
  * Ceph - scalable distributed file system
  *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ * Copyright (C) 2024 IBM, Red Hat
  *
  * This is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -35,9 +35,7 @@ class MMDSQuiesceDbListing final : public MMDSOp {
 
   void encode_payload_from(QuiesceDbPeerListing const& peer_listing)
   {
-    ENCODE_START(QuiesceDbEncoding::version, QuiesceDbEncoding::compat, payload);
-    encode(peer_listing, payload);
-    ENCODE_FINISH(payload);
+    ::encode(peer_listing, payload);
   }
 
   void decode_payload() override {
@@ -47,9 +45,7 @@ class MMDSQuiesceDbListing final : public MMDSOp {
   void decode_payload_into(QuiesceDbPeerListing &peer_listing) const
   {
     auto p = payload.cbegin();
-    DECODE_START(QuiesceDbEncoding::version, p);
-    decode(peer_listing, p);
-    DECODE_FINISH(p);
+    ::decode(peer_listing, p);
   }
 
 private:
diff --git a/src/test/mds/TestQuiesceAgent.cc b/src/test/mds/TestQuiesceAgent.cc
index d7d526ae9a6c..ae9511589545 100644
--- a/src/test/mds/TestQuiesceAgent.cc
+++ b/src/test/mds/TestQuiesceAgent.cc
@@ -333,7 +333,7 @@ TEST_F(QuiesceAgentTest, QuiesceProtocol) {
   EXPECT_EQ(1, latest_ack.roots.size());
   EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
 
-  latest_ack.reset();
+  latest_ack.clear();
 
   // complete the other root with failure
   EXPECT_TRUE(complete_quiesce("root2", -1));
@@ -344,7 +344,7 @@ TEST_F(QuiesceAgentTest, QuiesceProtocol) {
   EXPECT_EQ(QS_QUIESCED, latest_ack.roots.at("root1").state);
   EXPECT_EQ(QS_FAILED, latest_ack.roots.at("root2").state);
 
-  latest_ack.reset();
+  latest_ack.clear();
 
   // complete the third root with success
   // complete one root with success
@@ -457,7 +457,7 @@ TEST_F(QuiesceAgentTest, DuplicateQuiesceRequest) {
   EXPECT_TRUE(quiesce_requests.contains("root1"));
   EXPECT_TRUE(quiesce_requests.contains("root2"));
 
-  latest_ack.reset();
+  latest_ack.clear();
   // now, bring the roots back
   {
     auto ack = update(3, { 
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
index 8cd168424e56..f930f6c042a6 100644
--- a/src/test/mds/TestQuiesceDb.cc
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -10,6 +10,7 @@
  *
  */
 #include "mds/QuiesceDbManager.h"
+#include "mds/QuiesceDbEncoding.h"
 #include "gtest/gtest.h"
 #include "common/Cond.h"
 #include <ranges>
@@ -151,6 +152,13 @@ class QuiesceDbTest: public testing::Test {
             if (epoch == this->epoch) {
               if (this->managers.contains(recipient)) {
                 dout(10) << "listing from " << me << " (leader=" << leader << ") to " << recipient << " for version " << listing.db_version << " with " << listing.sets.size() << " sets" << dendl;
+
+                ceph::bufferlist bl;
+                encode(listing, bl);
+                listing.clear();
+                auto p = bl.cbegin();
+                decode(listing, p);
+
                 this->managers[recipient]->submit_peer_listing({me, std::move(listing)});
                 comms_cond.notify_all();
                 return 0;
@@ -181,6 +189,13 @@ class QuiesceDbTest: public testing::Test {
                     it++;
                   }
                 }
+
+                ceph::bufferlist bl;
+                encode(diff_map, bl);
+                diff_map.clear();
+                auto p = bl.cbegin();
+                decode(diff_map, p);
+
                 this->managers[leader]->submit_peer_ack({me, std::move(diff_map)});
                 comms_cond.notify_all();
                 l.unlock();

From 109de8bdab86e1adaad580d9e7322c18fa01bc09 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 4 Mar 2024 18:53:53 +0530
Subject: [PATCH 2132/2492] mds: disable `defer_client_eviction_on_laggy_osds'
 by default

This config can result in a single client holding up mds to service
other clients since once a client is deferred from eviction due to
laggy OSD(s), a new clients cap acquire request can be possibly
blocked until the other laggy client resumes operation, i.e., when
the laggy OSD is considered non-laggy anymore.

Disable the config by default till the issue is fixed.

Fixes: http://tracker.ceph.com/issues/64685
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/common/options/mds.yaml.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 77d5fe253d55..7bd958f5951f 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -1582,7 +1582,7 @@ options:
   long_desc: Laggy OSD(s) can make clients laggy or unresponsive, this can
     lead to their eviction, this option once enabled can help defer client
     eviction.
-  default: true
+  default: false
   services:
   - mds
   flags:

From 7791509741d8c21e273a46374a087dcca53a2816 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 4 Mar 2024 23:41:20 +1000
Subject: [PATCH 2133/2492] doc/architecture: remove pleonasm

Remove the word "detailed" in a sentence immediately following a
sentence containing the word "detail".

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index fc9e06da8cd8..12d2fbdf4b61 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -133,8 +133,8 @@ massive scale by distributing the work to all the OSD daemons in the cluster
 and all the clients that communicate with them. CRUSH uses intelligent data
 replication to ensure resiliency, which is better suited to hyper-scale
 storage. The following sections provide additional details on how CRUSH works.
-For a detailed discussion of CRUSH, see `CRUSH - Controlled, Scalable,
-Decentralized Placement of Replicated Data`_.
+For an in-depth, academic discussion of CRUSH, see `CRUSH - Controlled,
+Scalable, Decentralized Placement of Replicated Data`_.
 
 .. index:: architecture; cluster map
 

From b854bfcc70ff372cd087c068e0302d36ad86391f Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 3 Mar 2024 16:30:59 +0200
Subject: [PATCH 2134/2492] common: fix string creation from '0' in LogEntry

C++23 disallows conversion from 'int' to 'string'.
That includes returning '0' from a function that returns a string.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/LogEntry.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/common/LogEntry.cc b/src/common/LogEntry.cc
index d7b44a2110bd..7bb49432268b 100644
--- a/src/common/LogEntry.cc
+++ b/src/common/LogEntry.cc
@@ -183,7 +183,6 @@ string clog_type_to_string(clog_type t)
       return "crit";
     default:
       ceph_abort();
-      return 0;
   }
 }
 

From 958c60d8a74e3c38abe043e7f2cfbe4224cfb411 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Mon, 4 Mar 2024 18:22:48 +0530
Subject: [PATCH 2135/2492] mgr/dashboard: debugging make check failure

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/requirements.txt b/src/pybind/mgr/dashboard/requirements.txt
index 5643b55f9647..f7fc52bf92d8 100644
--- a/src/pybind/mgr/dashboard/requirements.txt
+++ b/src/pybind/mgr/dashboard/requirements.txt
@@ -6,7 +6,7 @@ requests
 Routes
 -e ../../../python-common
 prettytable
-pytest
+pytest==8.0.2
 pyyaml
 natsort
 setuptools

From 90b8ba9fc09ace4e4114152b194239061c7acb31 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Fri, 1 Mar 2024 13:22:44 -0500
Subject: [PATCH 2136/2492] cephadm: improve cephadm pull usage message

Generally, it's uncommon for users to run this
directly, but in case they need to for debugging
purposes, we should include how to pass the
image to be pulled in the usage message.

Additionally, include that this is only to be used
for pulling ceph images in the help message, as
that isn't necessarily clear. Pulling anything
else will result in a traceback as it tries
to run `ceph --version` inside the container.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 21523cd09ea0..b13ca406f015 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4857,7 +4857,9 @@ def _get_parser():
     )
 
     parser_pull = subparsers.add_parser(
-        'pull', help='pull the default container image')
+        'pull',
+        help='pull a ceph container image (will pull the default image if --image not provided)',
+        usage='cephadm pull (for default image) | cephadm --image <image-name> pull (for custom ceph image)')
     parser_pull.set_defaults(func=command_pull)
     parser_pull.add_argument(
         '--insecure',

From a8cf37b6e8e33c53d9be4cc1998d3b2f80700fbd Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Fri, 1 Mar 2024 18:54:12 -0500
Subject: [PATCH 2137/2492] rgw: make rgw-restore-bucket-index more robust

This experimental tool write a series of temporary files, the combined
size of which is roughly proportional to some combination of the
number of objects in the bucket and their respective sizes. Previously
the script stored all temporary files in /tmp, which may not be
sufficiently large in many cases.

There are now two enhancements. First, using the "-t" command-line
argument, the directory for temporary files can be specified. Second,
if the script consumes all available space on the file system
containing the temporary directory, an error message to that effect is
displayed, so the user understands what corrective action to take.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 doc/man/8/rgw-restore-bucket-index.rst |   7 ++
 src/rgw/rgw-restore-bucket-index       | 128 ++++++++++++++++---------
 2 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/doc/man/8/rgw-restore-bucket-index.rst b/doc/man/8/rgw-restore-bucket-index.rst
index d721dd9702f7..b25cd23d0435 100644
--- a/doc/man/8/rgw-restore-bucket-index.rst
+++ b/doc/man/8/rgw-restore-bucket-index.rst
@@ -79,6 +79,13 @@ Command-Line Arguments
    multiple buckets, it could be more efficient to re-use the same
    listing.
 
+.. option:: -t <temporary-directory>
+
+   Optional, specify a directory in which to store temporary files.
+   The size of the temporary files is highly dependent on the number
+   of bucket entries involved, so the partition on which the temporary
+   directory exists should be of suitable size.
+
 .. option:: -y
 
    Optional, proceed without further prompting. Without this option
diff --git a/src/rgw/rgw-restore-bucket-index b/src/rgw/rgw-restore-bucket-index
index 243a4f49068e..3b00ac9767a1 100755
--- a/src/rgw/rgw-restore-bucket-index
+++ b/src/rgw/rgw-restore-bucket-index
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# version 2023-10-30
+# version 2024-03-04
 
 # rgw-restore-bucket-index is an EXPERIMENTAL tool to use in case
 # bucket index entries for objects in the bucket are somehow lost. It
@@ -13,7 +13,7 @@
 # Because this script must process json objects, the `jq` tool must be
 # installed on the system.
 #
-# Usage: $0 [--proceed] <bucket-name> [data-pool-name]
+# Usage: see the usage() function below for details
 #
 # This tool is designed to be interactive, allowing the user to
 # examine the list of objects to be reindexed before
@@ -27,31 +27,64 @@ export TOP_PID=$$
 # relies on this ordering
 export LC_ALL=C
 
-# temporary files
-export bkt_entry=/tmp/rgwrbi-bkt-entry.$$
-export bkt_inst=/tmp/rgwrbi-bkt-inst.$$
-export marker_ls=/tmp/rgwrbi-marker-ls.$$
-export obj_list=/tmp/rgwrbi-object-list.$$
-export obj_list_ver=/tmp/rgwrbi-object-list-ver.$$
-export obj_reindex_script=/tmp/rgwrbi-object-list-script.$$
-export zone_info=/tmp/rgwrbi-zone-info.$$
-export olh_info_enc=/tmp/rgwrbi-olh-info-enc.$$
-export olh_info_json=/tmp/rgwrbi-olh-info-json.$$
-export debug_log=/tmp/rgwrbi-debug-log.$$
-
+# whether or not the temporary files are cleaned on completion
 export clean_temps=1
 
 # make explicit tabs easier to see in code
 export TAB="	"
 
 
+#
+# helper functions
+#
+
+super_exit() {
+    kill -s TERM -${TOP_PID}
+}
+
+usage() {
+  >&2 cat << EOF
+
+Usage: $0 -b <bucket-name> [-l <rados-ls-file>] [-p <pool>] [-y]
+
+where:
+  -b <bucket-name>     Required - name of the bucket to operate on
+  -l <rados-ls-file>   Optional - file containing the output of 'rados ls -p <pool>'
+  -r <realm-name>      Optional - specify the realm if not applying to the default realm"
+  -g <zonegroup-name>  Optional - specify the zonegroup if not applying to the default zonegroup"
+  -z <zone-name>       Optional - specify the zone if not applying to the default zone"
+  -p <pool>            Optional - data pool; if not provided will be inferred from bucket and zone information
+  -t <tmp-dir>         Optional - specify a directory for temporary files other than the default of /tmp
+  -y                   Optional - proceed with restoring without confirming with the user
+                       USE WITH CAUTION.
+  -d                   Optional - run with debugging output
+EOF
+  super_exit
+}
+
+# cleans all temporary files
 clean() {
   if [ "$clean_temps" == 1 ] ;then
-    rm -f $bkt_entry $bkt_inst $marker_ls $obj_list $obj_list_ver \
-       $obj_reindex_script $zone_info $olh_info_enc $olh_info_json
+    rm -f $bkt_entry $temp_file_list
+       $zone_info $olh_info_enc $olh_info_json
   fi
 }
 
+test_temp_space() {
+    # use df to determine percentage of data and inodes used; strip
+    # out spaces and percent signs from the output, so we just have a
+    # number from 0 to 100
+    pcent=$(df -k $temp_dir --output=pcent | tail -1 | sed 's/[ %]//g')
+    ipcent=$(df -k $temp_dir --output=ipcent | tail -1 | sed 's/[ %]//g')
+    if [ "$pcent" -eq 100 -o "$ipcent" -eq 100 ] ;then
+	>&2 echo "ERROR: the temporary directory's partition is full, preventing continuation."
+	>&2 echo "    NOTE: the temporary directory is \"${temp_dir}\"."
+	>&2 df -k $temp_dir -h --output="target,used,avail,pcent,iused,iavail,ipcent"
+	>&2 echo "    NOTE: cleaning temporary files before exiting...."
+	super_exit
+    fi
+}
+
 # number of seconds for a bucket index pending op to be completed via
 # dir_suggest mechanism
 export pending_op_secs=120
@@ -88,33 +121,6 @@ if [ "$exit_code" -ne 0 ] ;then
     exit $exit_code
 fi
 
-#
-# helper functions
-#
-
-super_exit() {
-   kill -s TERM $TOP_PID
-}
-
-usage() {
-  >&2 cat << EOF
-
-Usage: $0 -b <bucket-name> [-l <rados-ls-file>] [-p <pool>] [-y]
-
-where:
-  -b <bucket-name>     Required - name of the bucket to operate on
-  -l <rados-ls-file>   Optional - file containing the output of 'rados ls -p <pool>'
-  -r <realm-name>      Optional - specify the realm if not applying to the default realm"
-  -g <zonegroup-name>  Optional - specify the zonegroup if not applying to the default zonegroup"
-  -z <zone-name>       Optional - specify the zone if not applying to the default zone"
-  -p <pool>            Optional - data pool; if not provided will be inferred from bucket and zone information
-  -y                   Optional - proceed with restoring without confirming with the user
-                       USE WITH CAUTION.
-  -d                   Optional - run with debugging output
-EOF
-  super_exit
-}
-
 # Determines the name of the data pool. Expects the optional
 # command-line argument to appear as $1 if there is one. The
 # command-line has the highest priority, then the "explicit_placement"
@@ -137,6 +143,7 @@ get_pool() {
   fi
 
   radosgw-admin zone get $multisite_spec >$zone_info 2>/dev/null
+  test_temp_space
   pool=$(jq -r ".placement_pools [] | select(.key | contains(\"${plmt_pool}\")) .val .storage_classes.${plmt_class}.data_pool" $zone_info)
 
   if [ -z "$pool" ] ;then
@@ -147,12 +154,13 @@ get_pool() {
 }
 
 export bucket=""
+export temp_dir=/tmp
 pool=""
 multisite_spec=""
 lsoutput=""
 debug=0
 
-while getopts "b:l:p:r:g:z:yd" o; do
+while getopts "b:l:p:r:g:z:ydt:" o; do
     case "${o}" in
 	b)
 	    bucket="${OPTARG}"
@@ -187,6 +195,9 @@ while getopts "b:l:p:r:g:z:yd" o; do
 	    debug=1
 	    clean_temps=0
 	    ;;
+	t)
+	    temp_dir="${OPTARG}"
+	    ;;
 	*)
 	    echo
 	    usage
@@ -202,6 +213,24 @@ else
     export debugging_rgwadmin=" 2>/dev/null "
 fi
 
+if [ ! -d "$temp_dir" ] ;then
+    echo "ERROR: temporary directory $temp_dir is not a directory"
+    exit 1
+fi
+
+# temporary files
+export bkt_entry=${temp_dir}/rgwrbi-bkt-entry.$$
+export bkt_inst=${temp_dir}/rgwrbi-bkt-inst.$$
+export marker_ls=${temp_dir}/rgwrbi-marker-ls.$$
+export obj_list=${temp_dir}/rgwrbi-object-list.$$
+export obj_list_ver=${temp_dir}/rgwrbi-object-list-ver.$$
+export zone_info=${temp_dir}/rgwrbi-zone-info.$$
+export olh_info_enc=${temp_dir}/rgwrbi-olh-info-enc.$$
+export olh_info_json=${temp_dir}/rgwrbi-olh-info-json.$$
+export debug_log=${temp_dir}/rgwrbi-debug-log.$$
+
+export temp_file_list="$bkt_entry $bkt_inst $marker_ls $obj_list $obj_list_ver $zone_info $olh_info_enc $olh_info_json"
+
 # special code path for versioned buckets
 handle_versioned() {
     while read o ;do
@@ -213,7 +242,9 @@ handle_versioned() {
 
 	# process OLH object; determine final instance or delete-marker
 	rados -p $pool getxattr $olh_obj user.rgw.olh.info --object-locator "$olh_loc" >$olh_info_enc
+	test_temp_space
 	ceph-dencoder import $olh_info_enc type RGWOLHInfo decode dump_json >$olh_info_json
+	test_temp_space
 	last_instance=$(jq -r ".target.key.instance" $olh_info_json)
 	if [ -z "$last_instance" ] ;then
 	    # filters out entry without an instance
@@ -228,6 +259,7 @@ handle_versioned() {
 	    echo "last instance is $last_instance"
 	    echo "filter_out_last_instance is $filter_out_last_instance"
 	fi >>$debug_log
+	test_temp_space
 
 	# we currently don't need the delete marker, but we can have access to it
 	# delete_marker=$(jq -r ".removed" $olh_info_json) # true or false
@@ -237,6 +269,7 @@ handle_versioned() {
 		if [ "$debug" == 1 ] ;then
 		    echo "obj=$obj ; loc=$loc" >>$debug_log
 		fi
+		test_temp_space
 		rados -p $pool stat2 $obj --object-locator "$loc"
 	    done | # output of stat2, which includes mtime
 	    sort -k 3 | # stat2 but sorted by mtime earlier to later
@@ -255,6 +288,8 @@ handle_versioned() {
 		-e "s/^/${o}\t/"
 	echo "${o}${TAB}$last_instance" # now add the final instance; could be delete marker
     done <$obj_list 2>/dev/null | sed 's/\t$//' >$obj_list_ver
+    test_temp_space
+
 } # handle_versioned
 
 if [ -z "$bucket" ]; then
@@ -265,6 +300,7 @@ fi
 
 # read bucket entry metadata
 eval "radosgw-admin metadata get bucket:$bucket $debugging_rgwadmin $multisite_spec >$bkt_entry"
+test_temp_space
 export marker=$(jq -r ".data.bucket.marker" $bkt_entry)
 export bucket_id=$(jq -r ".data.bucket.bucket_id" $bkt_entry)
 if [ -z "$marker" -o -z "$bucket_id" ] ;then
@@ -282,6 +318,7 @@ echo bucket_id is $bucket_id
 
 # read bucket instance metadata
 eval "radosgw-admin metadata get bucket.instance:${bucket}:$bucket_id $multisite_spec $debugging_rgwadmin >$bkt_inst"
+test_temp_space
 
 # examine number of bucket index shards
 num_shards=$(jq ".data.bucket_info.num_shards" $bkt_inst)
@@ -309,11 +346,14 @@ fi
 # single.
 if [ -z "$lsoutput" ]; then
   ( rados -p $pool ls | grep "^${marker}_" >$marker_ls ) 2>/dev/null
+  test_temp_space
 else
   ( grep "^${marker}_" "${lsoutput}" >$marker_ls ) 2>/dev/null
+  test_temp_space
 fi
 
 ( sed -E 's/\t.*//' $marker_ls | grep -v -E "^${marker}__[^_]+_" | sed -E "s/^${marker}_(.*)/\1/" | sed 's/^__/_/' >$obj_list ) 2>/dev/null
+test_temp_space
 
 # mask bit indicating it's a versioned bucket
 export is_versioned=$(( $bkt_flags & 2))

From 1126ec3cf6285abbf6a875ab27daff397f09220b Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Wed, 31 Jan 2024 15:34:34 -0800
Subject: [PATCH 2138/2492] crimson/common: add coroutine integration for
 crimson futures

Adds coroutine machinery for crimson errorated and interruptible futures.

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/common/coroutine.h             | 310 +++++++++++++++++++
 src/test/crimson/CMakeLists.txt            |   8 +
 src/test/crimson/gtest_seastar.h           |  27 ++
 src/test/crimson/test_crimson_coroutine.cc | 327 +++++++++++++++++++++
 4 files changed, 672 insertions(+)
 create mode 100644 src/crimson/common/coroutine.h
 create mode 100644 src/test/crimson/test_crimson_coroutine.cc

diff --git a/src/crimson/common/coroutine.h b/src/crimson/common/coroutine.h
new file mode 100644
index 000000000000..c4dfca58c581
--- /dev/null
+++ b/src/crimson/common/coroutine.h
@@ -0,0 +1,310 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 smarttab expandtab
+
+#pragma once
+
+#include <seastar/core/coroutine.hh>
+
+#include "crimson/common/errorator.h"
+#include "crimson/common/interruptible_future.h"
+
+
+namespace crimson {
+namespace internal {
+
+template <typename Interruptor, typename Errorator>
+struct to_future {
+  template <typename T>
+  using future = crimson::interruptible::interruptible_future_detail<
+    typename Interruptor::condition,
+    typename Errorator::template future<T>>;
+};
+
+template <typename Errorator>
+struct to_future<void, Errorator> {
+  template <typename T>
+  using future = typename Errorator::template future<T>;
+};
+
+
+template <typename Interruptor>
+struct to_future<Interruptor, void> {
+  template <typename T>
+  using future = ::crimson::interruptible::interruptible_future<
+    typename Interruptor::condition, T>;
+};
+
+template <>
+struct to_future<void, void> {
+  template <typename T>
+  using future = seastar::future<T>;
+};
+
+
+template <typename Future>
+struct cond_checker {
+  using ref = std::unique_ptr<cond_checker>;
+  virtual std::optional<Future> may_interrupt() = 0;
+  virtual ~cond_checker() = default;
+};
+
+template <typename Interruptor>
+struct interrupt_cond_capture {
+  using InterruptCond = typename Interruptor::condition;
+  interruptible::InterruptCondRef<InterruptCond> cond;
+
+  template <typename Future>
+  struct type_erased_cond_checker final : cond_checker<Future> {
+    interruptible::InterruptCondRef<InterruptCond> cond;
+
+    template <typename T>
+    type_erased_cond_checker(T &&t) : cond(std::forward<T>(t)) {}
+
+    std::optional<Future> may_interrupt() final {
+      return cond->template may_interrupt<Future>();
+    }
+  };
+
+  template <typename Future>
+  typename cond_checker<Future>::ref capture_and_get_checker() {
+    ceph_assert(interruptible::interrupt_cond<InterruptCond>.interrupt_cond);
+    cond = interruptible::interrupt_cond<InterruptCond>.interrupt_cond;
+    return typename cond_checker<Future>::ref{
+      new type_erased_cond_checker<Future>{cond}
+    };
+  }
+
+  void restore() {
+    ceph_assert(cond);
+    interruptible::interrupt_cond<InterruptCond>.set(cond);
+  }
+
+  void reset() {
+    interruptible::interrupt_cond<InterruptCond>.reset();
+  }
+};
+
+template <>
+struct interrupt_cond_capture<void> {
+  template <typename Future>
+  typename cond_checker<Future>::ref capture_and_get_checker() {
+    return nullptr;
+  }
+};
+
+template <typename Interruptor>
+struct seastar_task_ancestor : protected seastar::task {};
+
+template <>
+struct seastar_task_ancestor<void> : public seastar::task {};
+
+template <typename Interruptor, typename Errorator, typename T>
+class promise_base : public seastar_task_ancestor<Interruptor> {
+protected:
+  seastar::promise<T> _promise;
+
+public:
+  interrupt_cond_capture<Interruptor> cond;
+
+  using errorator_type = Errorator;
+  using interruptor = Interruptor;
+  static constexpr bool is_errorated = !std::is_void<Errorator>::value;
+  static constexpr bool is_interruptible = !std::is_void<Interruptor>::value;
+
+  using _to_future =  to_future<Interruptor, Errorator>;
+
+  template <typename U=void>
+  using future = typename _to_future::template future<U>;
+
+  promise_base() = default;
+  promise_base(promise_base&&) = delete;
+  promise_base(const promise_base&) = delete;
+
+  void set_exception(std::exception_ptr&& eptr) noexcept {
+    _promise.set_exception(std::move(eptr));
+  }
+
+  void unhandled_exception() noexcept {
+    _promise.set_exception(std::current_exception());
+  }
+
+  future<T> get_return_object() noexcept {
+    return _promise.get_future();
+  }
+
+  std::suspend_never initial_suspend() noexcept { return { }; }
+  std::suspend_never final_suspend() noexcept { return { }; }
+
+  void run_and_dispose() noexcept final {
+    if constexpr (is_interruptible) {
+      cond.restore();
+    }
+    auto handle = std::coroutine_handle<promise_base>::from_promise(*this);
+    handle.resume();
+    if constexpr (is_interruptible) {
+      cond.reset();
+    }
+  }
+
+  seastar::task *waiting_task() noexcept override {
+    return _promise.waiting_task();
+  }
+  seastar::task *get_seastar_task() { return this; }
+};
+
+template <typename Interruptor, typename Errorator, typename T=void>
+class coroutine_traits {
+public:
+  class promise_type final : public promise_base<Interruptor, Errorator, T> {
+    using base = promise_base<Interruptor, Errorator, T>;
+  public:
+    template <typename... U>
+    void return_value(U&&... value) {
+      base::_promise.set_value(std::forward<U>(value)...);
+    }
+  };
+};
+
+
+template <typename Interruptor, typename Errorator>
+class coroutine_traits<Interruptor, Errorator> {
+public:
+  class promise_type final : public promise_base<Interruptor, Errorator, void> {
+    using base = promise_base<Interruptor, Errorator, void>;
+  public:
+    void return_void() noexcept {
+      base::_promise.set_value();
+    }
+  };
+};
+
+template <typename Interruptor, typename Errorator,
+	  bool CheckPreempt, typename T=void>
+struct awaiter {
+  static constexpr bool is_errorated = !std::is_void<Errorator>::value;
+  static constexpr bool is_interruptible = !std::is_void<Interruptor>::value;
+
+  template <typename U=void>
+  using future = typename to_future<Interruptor, Errorator>::template future<U>;
+
+  future<T> _future;
+
+  typename cond_checker<future<T>>::ref checker;
+public:
+  explicit awaiter(future<T>&& f) noexcept : _future(std::move(f)) { }
+
+  awaiter(const awaiter&) = delete;
+  awaiter(awaiter&&) = delete;
+
+  bool await_ready() const noexcept {
+    return _future.available() && (!CheckPreempt || !seastar::need_preempt());
+  }
+
+  template <typename U>
+  void await_suspend(std::coroutine_handle<U> hndl) noexcept {
+    if constexpr (is_errorated) {
+      using dest_errorator_t  = typename U::errorator_type;
+      static_assert(dest_errorator_t::template contains_once_v<Errorator>,
+		    "conversion is possible to more-or-eq errorated future!");
+    }
+
+    checker =
+      hndl.promise().cond.template capture_and_get_checker<future<T>>();
+    if (!CheckPreempt || !_future.available()) {
+      _future.set_coroutine(*hndl.promise().get_seastar_task());
+    } else {
+      ::seastar::schedule(hndl.promise().get_seastar_task());
+    }
+  }
+
+  T await_resume() {
+    if (auto maybe_fut = checker ? checker->may_interrupt() : std::nullopt) {
+      // silence warning that we are discarding an exceptional future
+      if (_future.failed()) _future.get_exception();
+      if constexpr (is_errorated) {
+	return maybe_fut->unsafe_get0();
+      } else {
+	return maybe_fut->get0();
+      }
+    } else {
+      if constexpr (is_errorated) {
+	return _future.unsafe_get0();
+      } else {
+	return _future.get0();
+      }
+    }
+  }
+};
+
+}
+}
+
+template <template <typename> typename Container, typename T>
+auto operator co_await(
+  Container<crimson::errorated_future_marker<T>> f) noexcept {
+  using Errorator = typename seastar::futurize<decltype(f)>::errorator_type;
+  return crimson::internal::awaiter<void, Errorator, true, T>(std::move(f));
+}
+
+template <typename InterruptCond, typename T>
+auto operator co_await(
+  crimson::interruptible::interruptible_future_detail<
+    InterruptCond, seastar::future<T>
+  > f) noexcept {
+  return crimson::internal::awaiter<
+    crimson::interruptible::interruptor<InterruptCond>, void, true, T>(
+  std::move(f));
+}
+
+template <template <typename> typename Container,
+	  typename InterruptCond, typename T>
+auto operator co_await(
+  crimson::interruptible::interruptible_future_detail<
+    InterruptCond, Container<crimson::errorated_future_marker<T>>
+  > f) noexcept {
+  using Errorator = typename seastar::futurize<decltype(f)>::errorator_type;
+  return crimson::internal::awaiter<
+    crimson::interruptible::interruptor<InterruptCond>,
+    typename Errorator::base_ertr, true, T>(
+  std::move(f));
+}
+
+namespace std {
+
+template <template <typename> typename Container,
+	  typename T, typename... Args>
+class coroutine_traits<Container<crimson::errorated_future_marker<T>>, Args...> :
+    public crimson::internal::coroutine_traits<
+      void,
+      typename seastar::futurize<
+	Container<crimson::errorated_future_marker<T>>
+	>::errorator_type,
+  T> {};
+
+template <typename InterruptCond,
+	  typename T, typename... Args>
+class coroutine_traits<
+  crimson::interruptible::interruptible_future_detail<
+    InterruptCond, seastar::future<T>
+    >, Args...> : public crimson::internal::coroutine_traits<
+  crimson::interruptible::interruptor<InterruptCond>,
+  void,
+  T> {};
+
+template <template <typename> typename Container,
+	  typename InterruptCond,
+	  typename T, typename... Args>
+class coroutine_traits<
+  crimson::interruptible::interruptible_future_detail<
+    InterruptCond, Container<crimson::errorated_future_marker<T>>
+    >, Args...> :
+    public crimson::internal::coroutine_traits<
+      crimson::interruptible::interruptor<InterruptCond>,
+      typename seastar::futurize<
+        crimson::interruptible::interruptible_future_detail<
+	  InterruptCond,
+          Container<crimson::errorated_future_marker<T>>
+	  >
+      >::errorator_type::base_ertr,
+      T> {};
+}
diff --git a/src/test/crimson/CMakeLists.txt b/src/test/crimson/CMakeLists.txt
index e1a5dfe73dfd..c8c5c84e65c6 100644
--- a/src/test/crimson/CMakeLists.txt
+++ b/src/test/crimson/CMakeLists.txt
@@ -111,6 +111,14 @@ target_link_libraries(
 add_ceph_unittest(unittest-seastar-errorator
   --memory 256M --smp 1)
 
+add_executable(unittest-crimson-coroutine
+  test_crimson_coroutine.cc)
+target_link_libraries(
+  unittest-crimson-coroutine
+  crimson::gtest)
+add_ceph_unittest(unittest-crimson-coroutine
+  --memory 256M --smp 1)
+
 add_executable(unittest-crimson-scrub
   test_crimson_scrub.cc
   ${PROJECT_SOURCE_DIR}/src/crimson/osd/scrub/scrub_machine.cc
diff --git a/src/test/crimson/gtest_seastar.h b/src/test/crimson/gtest_seastar.h
index 20709a3eec48..34967c253151 100644
--- a/src/test/crimson/gtest_seastar.h
+++ b/src/test/crimson/gtest_seastar.h
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include "crimson/common/errorator.h"
 #include "gtest/gtest.h"
 
 #include "seastar_runner.h"
@@ -15,6 +16,15 @@ struct seastar_test_suite_t : public ::testing::Test {
     return seastar_env.run(std::forward<Func>(func));
   }
 
+  template <typename Func>
+  void run_ertr(Func &&func) {
+    return run(
+      [func=std::forward<Func>(func)]() mutable {
+	return std::invoke(std::move(func)).handle_error(
+	  crimson::ct_error::assert_all("error"));
+      });
+  }
+
   template <typename Func>
   void run_async(Func &&func) {
     run(
@@ -23,6 +33,23 @@ struct seastar_test_suite_t : public ::testing::Test {
       });
   }
 
+  template <typename F>
+  auto scl(F &&f) {
+    return [fptr = std::make_unique<F>(std::forward<F>(f))]() mutable {
+      return std::invoke(*fptr).finally([fptr=std::move(fptr)] {});
+    };
+  }
+
+  auto run_scl(auto &&f) {
+    return run([this, f=std::forward<decltype(f)>(f)]() mutable {
+      return std::invoke(scl(std::move(f)));
+    });
+  }
+
+  auto run_ertr_scl(auto &&f) {
+    return run_ertr(scl(std::forward<decltype(f)>(f)));
+  }
+
   virtual seastar::future<> set_up_fut() { return seastar::now(); }
   void SetUp() final {
     return run([this] { return set_up_fut(); });
diff --git a/src/test/crimson/test_crimson_coroutine.cc b/src/test/crimson/test_crimson_coroutine.cc
new file mode 100644
index 000000000000..2b19ca07d8e0
--- /dev/null
+++ b/src/test/crimson/test_crimson_coroutine.cc
@@ -0,0 +1,327 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <boost/iterator/counting_iterator.hpp>
+#include <numeric>
+
+#include "seastar/core/sleep.hh"
+
+#include "crimson/common/coroutine.h"
+#include "crimson/common/errorator.h"
+#include "crimson/common/interruptible_future.h"
+#include "crimson/common/log.h"
+
+#include "test/crimson/gtest_seastar.h"
+
+struct coroutine_test_t : public seastar_test_suite_t {
+  struct interruption_state_t {
+    bool interrupted = false;
+  } interruption_state;
+
+  class test_interruption : public std::exception
+  {};
+
+  class test_interrupt_cond {
+    interruption_state_t *int_state = nullptr;
+  public:
+    test_interrupt_cond() = delete;
+    test_interrupt_cond(interruption_state_t *int_state)
+      : int_state(int_state) {}
+
+    template <typename T>
+    std::optional<T> may_interrupt() {
+      ceph_assert(int_state);
+      if (int_state->interrupted) {
+	return seastar::futurize<T>::make_exception_future(
+	  test_interruption()
+	);
+      } else {
+	return std::nullopt;
+      }
+    }
+
+    template <typename T>
+    static constexpr bool is_interruption_v = std::is_same_v<
+      T, test_interruption>;
+
+    static bool is_interruption(std::exception_ptr& eptr) {
+      if (*eptr.__cxa_exception_type() == typeid(test_interruption))
+	return true;
+      return false;
+    }
+  };
+  using interruptor = crimson::interruptible::interruptor<test_interrupt_cond>;
+
+  using ertr = crimson::errorator<crimson::ct_error::invarg>;
+  using iertr = crimson::interruptible::interruptible_errorator<
+    test_interrupt_cond,
+    ertr>;
+
+  using ertr2 = ertr::extend<
+    crimson::ct_error::eagain>;
+  using iertr2 = crimson::interruptible::interruptible_errorator<
+    test_interrupt_cond,
+    ertr2>;
+
+  using ertr3 = ertr::extend<
+    crimson::ct_error::enoent>;
+  using iertr3 = crimson::interruptible::interruptible_errorator<
+    test_interrupt_cond,
+    ertr3>;
+
+  void interrupt() {
+    interruption_state.interrupted = true;
+  }
+
+  seastar::future<> set_up_fut() final {
+    interruption_state.interrupted = false;
+    return seastar::now();
+  }
+
+
+  template <typename E, typename F>
+  auto cwi(E &&errf, F &&f) {
+    return interruptor::with_interruption(
+      scl(std::forward<F>(f)),
+      std::forward<E>(errf),
+      &interruption_state);
+  }
+};
+
+namespace crimson::interruptible {
+template
+thread_local interrupt_cond_t<coroutine_test_t::test_interrupt_cond>
+interrupt_cond<coroutine_test_t::test_interrupt_cond>;
+}
+
+TEST_F(coroutine_test_t, test_coroutine)
+{
+  run_scl([]() -> seastar::future<> {
+    constexpr int CHECK = 20;
+    auto unwrapped = co_await seastar::make_ready_future<int>(CHECK);
+    EXPECT_EQ(unwrapped, CHECK);
+  });
+}
+
+TEST_F(coroutine_test_t, test_ertr_coroutine_basic)
+{
+  run_ertr_scl([]() -> ertr::future<> {
+    constexpr int CHECK = 20;
+    auto unwrapped = co_await ertr::make_ready_future<int>(CHECK);
+    EXPECT_EQ(unwrapped, CHECK);
+  });
+}
+
+TEST_F(coroutine_test_t, test_ertr_coroutine_vanilla_future)
+{
+  run_ertr_scl([]() -> ertr::future<> {
+    constexpr int CHECK = 20;
+    auto unwrapped = co_await seastar::make_ready_future<int>(CHECK);
+    EXPECT_EQ(unwrapped, CHECK);
+  });
+}
+
+TEST_F(coroutine_test_t, test_ertr_coroutine_error)
+{
+  run_scl([this]() -> seastar::future<> {
+    auto fut = scl([]() -> ertr::future<int> {
+      std::ignore = co_await ertr::future<int>(
+	crimson::ct_error::invarg::make()
+      );
+      EXPECT_EQ("above co_await should throw", nullptr);
+      co_return 10;
+    })();
+    auto ret = co_await std::move(fut).handle_error(
+      [](const crimson::ct_error::invarg &e) {
+	return 20;
+      }
+    );
+    EXPECT_EQ(ret, 20);
+  });
+}
+
+#if 0
+// This one is left in, but commented out, as a test which *should fail to
+// build* due to trying to co_await a more errorated future.
+TEST_F(coroutine_test_t, test_basic_ertr_coroutine_error_should_not_build)
+{
+  run_ertr_scl([]() -> ertr::future<int> {
+    constexpr int CHECK = 20;
+    auto unwrapped = co_await ertr2::make_ready_future<int>(CHECK);
+    EXPECT_EQ(unwrapped, CHECK);
+    co_return 10;
+  });
+}
+#endif
+
+TEST_F(coroutine_test_t, interruptible_coroutine_basic)
+{
+  run_scl([this]() -> seastar::future<> {
+    seastar::promise<int> p;
+    auto ret = cwi(
+      [](auto) { return 2; },
+      [f=p.get_future()]() mutable -> interruptor::future<int> {
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	co_return x;
+      });
+    p.set_value(0);
+    auto awaited = co_await std::move(ret);
+    EXPECT_EQ(awaited, 0);
+  });
+}
+
+TEST_F(coroutine_test_t, interruptible_coroutine_interrupted)
+{
+  run_scl([this]() -> seastar::future<> {
+    seastar::promise<int> p;
+    auto ret = cwi(
+      [](auto) { return 2; },
+      [f=p.get_future()]() mutable -> interruptor::future<int> {
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	co_return x;
+      });
+    interrupt();
+    p.set_value(0);
+    auto awaited = co_await std::move(ret);
+    EXPECT_EQ(awaited, 2);
+  });
+}
+
+TEST_F(coroutine_test_t, dual_interruptible_coroutine)
+{
+  run_scl([this]() -> seastar::future<> {
+    seastar::promise<int> p, p2;
+    auto fut1 = cwi(
+      [](auto) { return 2; },
+      [&p, f=p2.get_future()]() mutable -> interruptor::future<int> {
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	p.set_value(1);
+	co_return x;
+      });
+    auto fut2 = cwi(
+      [](auto) { return 2; },
+      [&p2, f=p.get_future()]() mutable -> interruptor::future<int> {
+	p2.set_value(0);
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	co_return x;
+      });
+
+    auto ret1 = co_await std::move(fut1);
+    auto ret2 = co_await std::move(fut2);
+    EXPECT_EQ(ret1, 0);
+    EXPECT_EQ(ret2, 1);
+  });
+}
+
+TEST_F(coroutine_test_t, dual_interruptible_coroutine_interrupted)
+{
+  run_scl([this]() -> seastar::future<> {
+    seastar::promise<int> p, p2;
+    auto fut1 = cwi(
+      [](auto) { return 2; },
+      [this, &p, f=p2.get_future()]() mutable -> interruptor::future<int> {
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	interrupt();
+	p.set_value(1);
+	co_return x;
+      });
+    auto fut2 = cwi(
+      [](auto) { return 2; },
+      [&p2, f=p.get_future()]() mutable -> interruptor::future<int> {
+	p2.set_value(0);
+	auto x = co_await interruptor::make_interruptible(std::move(f));
+	co_return x;
+      });
+
+    auto ret1 = co_await std::move(fut1);
+    auto ret2 = co_await std::move(fut2);
+    EXPECT_EQ(ret1, 0);
+    EXPECT_EQ(ret2, 2);
+  });
+}
+
+TEST_F(coroutine_test_t, test_iertr_coroutine_basic)
+{
+  run_ertr_scl([this]() -> ertr2::future<> {
+    auto ret = co_await cwi(
+      [](auto) { return 10; },
+      []() -> iertr::future<int> {
+	co_return 20;
+      });
+    EXPECT_EQ(ret, 20);
+  });
+}
+
+TEST_F(coroutine_test_t, test_iertr_coroutine_interruption_as_error)
+{
+  run_ertr_scl([this]() -> ertr2::future<> {
+    auto ret = co_await cwi(
+      [](auto) {
+	return ertr2::future<int>(crimson::ct_error::eagain::make());
+      },
+      []() -> iertr::future<int> {
+	co_return 20;
+      });
+    EXPECT_EQ(ret, 20);
+  });
+}
+
+TEST_F(coroutine_test_t, test_iertr_coroutine_interruption_as_error_interrupted)
+{
+  run_ertr_scl([this]() -> ertr::future<> {
+    seastar::promise<> p;
+    auto f = cwi(
+      [](auto) {
+	return ertr2::future<int>(crimson::ct_error::eagain::make());
+      },
+      [&p]() -> iertr::future<int> {
+        co_await iertr::make_interruptible(p.get_future());
+	co_return 20;
+      });
+    interrupt();
+    p.set_value();
+    auto ret = co_await f.handle_error(
+      crimson::ct_error::eagain::handle([](const auto &) {
+	return 30;
+      }),
+      crimson::ct_error::pass_further_all{}
+    );
+    EXPECT_EQ(ret, 30);
+  });
+}
+
+#if 0
+// the cwi invocation below would yield an ertr2 due to the interruption handler
+TEST_F(coroutine_test_t, test_iertr_coroutine_interruption_should_not_compile)
+{
+  run_ertr_scl([this]() -> ertr::future<> {
+    auto ret = co_await cwi(
+      [](auto) {
+	ertr2::future<int>(crimson::ct_error::eagain::make());
+      },
+      []() -> iertr::future<int> {
+	co_return 20;
+      });
+    EXPECT_EQ(ret, 20);
+  });
+}
+#endif
+
+#if 0
+// can't co_await a vanilla future from an interruptible coroutine
+TEST_F(coroutine_test_t, test_iertr_coroutine_interruption_should_not_compile2)
+{
+  run_ertr_scl([this]() -> ertr2::future<> {
+    auto ret = co_await cwi(
+      [](auto) {
+	return ertr2::future<int>(crimson::ct_error::eagain::make());
+      },
+      []() -> iertr::future<int> {
+	co_await seastar::now();
+	co_return 20;
+      });
+    EXPECT_EQ(ret, 20);
+  });
+}
+#endif
+

From 82b50b4eac819a71542d766b573f65819046f403 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Tue, 12 Dec 2023 14:08:26 +0530
Subject: [PATCH 2139/2492] mgr/dashboard: add prometheus federation config for
 mulkti-cluster monitoring

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/cephadm/module.py              | 34 +++++++++++++++++
 src/pybind/mgr/cephadm/services/monitoring.py | 10 ++++-
 .../services/prometheus/prometheus.yml.j2     | 37 +++++++++++++++++++
 src/pybind/mgr/cephadm/tests/test_services.py | 34 +++++++++++++++++
 src/pybind/mgr/orchestrator/_interface.py     |  8 ++++
 src/pybind/mgr/orchestrator/module.py         | 12 ++++++
 .../ceph/deployment/service_spec.py           |  8 +++-
 7 files changed, 140 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 7c124d787339..467950afea33 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -16,6 +16,8 @@
 
 from cephadm.service_discovery import ServiceDiscovery
 
+from ceph.deployment.service_spec import PrometheusSpec
+
 import string
 from typing import List, Dict, Optional, Callable, Tuple, TypeVar, \
     Any, Set, TYPE_CHECKING, cast, NamedTuple, Sequence, Type, \
@@ -2980,6 +2982,38 @@ def set_prometheus_access_info(self, user: str, password: str) -> str:
         self.set_store(PrometheusService.PASS_CFG_KEY, password)
         return 'prometheus credentials updated correctly'
 
+    @handle_orch_error
+    def set_prometheus_target(self, url: str) -> str:
+        prometheus_spec = cast(PrometheusSpec, self.spec_store['prometheus'].spec)
+        if url not in prometheus_spec.targets:
+            prometheus_spec.targets.append(url)
+        else:
+            return f"Target '{url}' already exists.\n"
+        if not prometheus_spec:
+            return "Service prometheus not found\n"
+        daemons: List[orchestrator.DaemonDescription] = self.cache.get_daemons_by_type('prometheus')
+        spec = ServiceSpec.from_json(prometheus_spec.to_json())
+        self.apply([spec], no_overwrite=False)
+        for daemon in daemons:
+            self.daemon_action(action='redeploy', daemon_name=daemon.daemon_name)
+        return 'prometheus multi-cluster targets updated'
+
+    @handle_orch_error
+    def remove_prometheus_target(self, url: str) -> str:
+        prometheus_spec = cast(PrometheusSpec, self.spec_store['prometheus'].spec)
+        if url in prometheus_spec.targets:
+            prometheus_spec.targets.remove(url)
+        else:
+            return f"Target '{url}' does not exist.\n"
+        if not prometheus_spec:
+            return "Service prometheus not found\n"
+        daemons: List[orchestrator.DaemonDescription] = self.cache.get_daemons_by_type('prometheus')
+        spec = ServiceSpec.from_json(prometheus_spec.to_json())
+        self.apply([spec], no_overwrite=False)
+        for daemon in daemons:
+            self.daemon_action(action='redeploy', daemon_name=daemon.daemon_name)
+        return 'prometheus multi-cluster targets updated'
+
     @handle_orch_error
     def set_alertmanager_access_info(self, user: str, password: str) -> str:
         self.set_store(AlertmanagerService.USER_CFG_KEY, user)
diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index d3439c04d04f..a8aef7377e63 100644
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -376,12 +376,17 @@ def generate_config(
 
         assert self.TYPE == daemon_spec.daemon_type
         spec = cast(PrometheusSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
-
         try:
             retention_time = spec.retention_time if spec.retention_time else '15d'
         except AttributeError:
             retention_time = '15d'
 
+        try:
+            targets = spec.targets
+        except AttributeError:
+            logger.warning('Prometheus targets not found in the spec. Using empty list.')
+            targets = []
+
         try:
             retention_size = spec.retention_size if spec.retention_size else '0'
         except AttributeError:
@@ -406,6 +411,7 @@ def generate_config(
 
         alertmanager_user, alertmanager_password = self.mgr._get_alertmanager_credentials()
         prometheus_user, prometheus_password = self.mgr._get_prometheus_credentials()
+        FSID = self.mgr._cluster_fsid
 
         # generate the prometheus configuration
         context = {
@@ -420,6 +426,8 @@ def generate_config(
             'haproxy_sd_url': haproxy_sd_url,
             'ceph_exporter_sd_url': ceph_exporter_sd_url,
             'nvmeof_sd_url': nvmeof_sd_url,
+            'external_prometheus_targets': targets,
+            'cluster_fsid': FSID
         }
 
         web_context = {
diff --git a/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
index 57d2f8a3f4b4..931913668ae8 100644
--- a/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
+++ b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2
@@ -2,6 +2,11 @@
 global:
   scrape_interval: 10s
   evaluation_interval: 10s
+{% if not secure_monitoring_stack %}
+  external_labels:
+    cluster: {{ cluster_fsid }}
+{% endif %}
+
 rule_files:
   - /etc/prometheus/alerting/*
 
@@ -45,6 +50,10 @@ scrape_configs:
         ca_file: root_cert.pem
 {% else %}
     honor_labels: true
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: {{ cluster_fsid }}
     http_sd_configs:
     - url: {{ mgr_prometheus_sd_url }}
 {% endif %}
@@ -65,6 +74,10 @@ scrape_configs:
 {% else %}
     http_sd_configs:
     - url: {{ node_exporter_sd_url }}
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: {{ cluster_fsid }}
 {% endif %}
 {% endif %}
 
@@ -84,6 +97,10 @@ scrape_configs:
 {% else %}
     http_sd_configs:
     - url: {{ haproxy_sd_url }}
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: {{ cluster_fsid }}
 {% endif %}
 {% endif %}
 
@@ -103,6 +120,10 @@ scrape_configs:
         ca_file: root_cert.pem
 {% else %}
     honor_labels: true
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: {{ cluster_fsid }}
     http_sd_configs:
     - url: {{ ceph_exporter_sd_url }}
 {% endif %}
@@ -127,3 +148,19 @@ scrape_configs:
     - url: {{ nvmeof_sd_url }}
 {% endif %}
 {% endif %}
+
+{% if not secure_monitoring_stack %}
+  - job_name: 'federate'
+    scrape_interval: 15s
+    honor_labels: true
+    metrics_path: '/federate'
+    params:
+      'match[]':
+        - '{job="ceph"}'
+        - '{job="node"}'
+        - '{job="haproxy"}'
+        - '{job="ceph-exporter"}'
+    static_configs:
+    - targets: {{ external_prometheus_targets }}
+{% endif %}
+
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index f0b5360e6e30..8301c3b8b7c0 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -692,6 +692,9 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                 global:
                   scrape_interval: 10s
                   evaluation_interval: 10s
+                  external_labels:
+                    cluster: fsid
+
                 rule_files:
                   - /etc/prometheus/alerting/*
 
@@ -704,25 +707,54 @@ def test_prometheus_config_security_disabled(self, _run_cephadm, cephadm_module:
                 scrape_configs:
                   - job_name: 'ceph'
                     honor_labels: true
+                    relabel_configs:
+                    - source_labels: [__address__]
+                      target_label: cluster
+                      replacement: fsid
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=mgr-prometheus
 
                   - job_name: 'node'
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=node-exporter
+                    relabel_configs:
+                    - source_labels: [__address__]
+                      target_label: cluster
+                      replacement: fsid
 
                   - job_name: 'haproxy'
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=haproxy
+                    relabel_configs:
+                    - source_labels: [__address__]
+                      target_label: cluster
+                      replacement: fsid
 
                   - job_name: 'ceph-exporter'
                     honor_labels: true
+                    relabel_configs:
+                    - source_labels: [__address__]
+                      target_label: cluster
+                      replacement: fsid
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=ceph-exporter
 
                   - job_name: 'nvmeof'
                     http_sd_configs:
                     - url: http://[::1]:8765/sd/prometheus/sd-config?service=nvmeof
+
+                  - job_name: 'federate'
+                    scrape_interval: 15s
+                    honor_labels: true
+                    metrics_path: '/federate'
+                    params:
+                      'match[]':
+                        - '{job="ceph"}'
+                        - '{job="node"}'
+                        - '{job="haproxy"}'
+                        - '{job="ceph-exporter"}'
+                    static_configs:
+                    - targets: []
                 """).lstrip()
 
                 _run_cephadm.assert_called_with(
@@ -810,6 +842,7 @@ def gen_cert(host, addr):
                 global:
                   scrape_interval: 10s
                   evaluation_interval: 10s
+
                 rule_files:
                   - /etc/prometheus/alerting/*
 
@@ -892,6 +925,7 @@ def gen_cert(host, addr):
                         password: sd_password
                       tls_config:
                         ca_file: root_cert.pem
+
                 """).lstrip()
 
                 _run_cephadm.assert_called_with(
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index bc1721ab2511..f0fb2c429069 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -775,6 +775,14 @@ def set_prometheus_access_info(self, user: str, password: str) -> OrchResult[str
         """set prometheus access information"""
         raise NotImplementedError()
 
+    def set_prometheus_target(self, url: str) -> OrchResult[str]:
+        """set prometheus target for multi-cluster"""
+        raise NotImplementedError()
+
+    def remove_prometheus_target(self, url: str) -> OrchResult[str]:
+        """remove prometheus target for multi-cluster"""
+        raise NotImplementedError()
+
     def get_alertmanager_access_info(self) -> OrchResult[Dict[str, str]]:
         """get alertmanager access information"""
         raise NotImplementedError()
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index c36c85e7d99e..96a2d9104012 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -1143,6 +1143,18 @@ def _set_prometheus_access_info(self, username: Optional[str] = None, password:
         except ArgumentError as e:
             return HandleCommandResult(-errno.EINVAL, "", (str(e)))
 
+    @_cli_write_command('orch prometheus set-target')
+    def _set_prometheus_target(self, url: str) -> HandleCommandResult:
+        completion = self.set_prometheus_target(url)
+        result = raise_if_exception(completion)
+        return HandleCommandResult(stdout=json.dumps(result))
+
+    @_cli_write_command('orch prometheus remove-target')
+    def _remove_prometheus_target(self, url: str) -> HandleCommandResult:
+        completion = self.remove_prometheus_target(url)
+        result = raise_if_exception(completion)
+        return HandleCommandResult(stdout=json.dumps(result))
+
     @_cli_write_command('orch alertmanager set-credentials')
     def _set_alertmanager_access_info(self, username: Optional[str] = None, password: Optional[str] = None, inbuf: Optional[str] = None) -> HandleCommandResult:
         try:
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index f6d290f07188..1de5b6b18209 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -818,6 +818,7 @@ def __init__(self,
                  unmanaged: bool = False,
                  preview_only: bool = False,
                  networks: Optional[List[str]] = None,
+                 targets: Optional[List[str]] = None,
                  extra_container_args: Optional[GeneralArgList] = None,
                  extra_entrypoint_args: Optional[GeneralArgList] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
@@ -854,6 +855,7 @@ def __init__(self,
         #: :ref:`cephadm-monitoring-networks-ports`,
         #: :ref:`cephadm-rgw-networks` and :ref:`cephadm-mgr-networks`.
         self.networks: List[str] = networks or []
+        self.targets: List[str] = targets or []
 
         self.config: Optional[Dict[str, str]] = None
         if config:
@@ -1733,6 +1735,7 @@ def __init__(self,
                  unmanaged: bool = False,
                  preview_only: bool = False,
                  port: Optional[int] = None,
+                 targets: Optional[List[str]] = None,
                  extra_container_args: Optional[GeneralArgList] = None,
                  extra_entrypoint_args: Optional[GeneralArgList] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
@@ -1746,7 +1749,7 @@ def __init__(self,
             preview_only=preview_only, config=config,
             networks=networks, extra_container_args=extra_container_args,
             extra_entrypoint_args=extra_entrypoint_args,
-            custom_configs=custom_configs)
+            custom_configs=custom_configs, targets=targets)
 
         self.service_type = service_type
         self.port = port
@@ -1881,6 +1884,7 @@ def __init__(self,
                  port: Optional[int] = None,
                  retention_time: Optional[str] = None,
                  retention_size: Optional[str] = None,
+                 targets: Optional[List[str]] = None,
                  extra_container_args: Optional[GeneralArgList] = None,
                  extra_entrypoint_args: Optional[GeneralArgList] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
@@ -1889,7 +1893,7 @@ def __init__(self,
         super(PrometheusSpec, self).__init__(
             'prometheus', service_id=service_id,
             placement=placement, unmanaged=unmanaged,
-            preview_only=preview_only, config=config, networks=networks, port=port,
+            preview_only=preview_only, config=config, networks=networks, port=port, targets=targets,
             extra_container_args=extra_container_args, extra_entrypoint_args=extra_entrypoint_args,
             custom_configs=custom_configs)
 

From 339a635503933e24cdd6e723b862a78052aed4fa Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Tue, 5 Mar 2024 06:53:52 +0000
Subject: [PATCH 2140/2492] crimson/os/seastore: make less verbose log when
 testing performance on RBM

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/async_cleaner.cc                     | 4 ++--
 src/crimson/os/seastore/random_block_manager/avlallocator.cc | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc
index fd043f4ea35e..53785fb5fc2b 100644
--- a/src/crimson/os/seastore/async_cleaner.cc
+++ b/src/crimson/os/seastore/async_cleaner.cc
@@ -1634,7 +1634,7 @@ void RBMCleaner::mark_space_used(
   for (auto rbm : rbms) {
     if (addr.get_device_id() == rbm->get_device_id()) {
       if (rbm->get_start() <= addr) {
-	INFO("allocate addr: {} len: {}", addr, len);
+	DEBUG("allocate addr: {} len: {}", addr, len);
 	stats.used_bytes += len;
 	rbm->mark_space_used(addr, len);
       }
@@ -1653,7 +1653,7 @@ void RBMCleaner::mark_space_free(
   for (auto rbm : rbms) {
     if (addr.get_device_id() == rbm->get_device_id()) {
       if (rbm->get_start() <= addr) {
-	INFO("free addr: {} len: {}", addr, len);
+	DEBUG("free addr: {} len: {}", addr, len);
 	ceph_assert(stats.used_bytes >= len);
 	stats.used_bytes -= len;
 	rbm->mark_space_free(addr, len);
diff --git a/src/crimson/os/seastore/random_block_manager/avlallocator.cc b/src/crimson/os/seastore/random_block_manager/avlallocator.cc
index 51624b1c8b29..ab1f0c30e561 100644
--- a/src/crimson/os/seastore/random_block_manager/avlallocator.cc
+++ b/src/crimson/os/seastore/random_block_manager/avlallocator.cc
@@ -192,7 +192,7 @@ std::optional<interval_set<rbm_abs_addr>> AvlAllocator::alloc_extent(
   assert(!result.empty());
   assert(result.num_intervals() == 1);
   for (auto p : result) {
-    INFO("result start: {}, end: {}", p.first, p.first + p.second);
+    DEBUG("result start: {}, end: {}", p.first, p.first + p.second);
     if (detailed) {
       assert(!reserved_extent_tracker.contains(p.first, p.second));
       reserved_extent_tracker.insert(p.first, p.second);
@@ -236,7 +236,7 @@ std::optional<interval_set<rbm_abs_addr>> AvlAllocator::alloc_extents(
 
   assert(!result.empty());
   for (auto p : result) {
-    INFO("result start: {}, end: {}", p.first, p.first + p.second);
+    DEBUG("result start: {}, end: {}", p.first, p.first + p.second);
     if (detailed) {
       assert(!reserved_extent_tracker.contains(p.first, p.second));
       reserved_extent_tracker.insert(p.first, p.second);

From da0d4048b8249b3038596818545eaf16ca3ee082 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 3 Nov 2023 17:10:55 +0800
Subject: [PATCH 2141/2492] crimson/os/seastore: use LBAMapping::is_stable()
 wherever appropriate

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 .../os/seastore/object_data_handler.cc        | 35 +++++++------------
 src/crimson/os/seastore/transaction_manager.h |  3 +-
 2 files changed, 14 insertions(+), 24 deletions(-)

diff --git a/src/crimson/os/seastore/object_data_handler.cc b/src/crimson/os/seastore/object_data_handler.cc
index 96963ea96f7a..6b1fb45b1ae8 100644
--- a/src/crimson/os/seastore/object_data_handler.cc
+++ b/src/crimson/os/seastore/object_data_handler.cc
@@ -641,6 +641,8 @@ struct overwrite_plan_t {
 
   // helper member
   extent_len_t block_size;
+  bool is_left_stable;
+  bool is_right_stable;
 
 public:
   extent_len_t get_left_size() const {
@@ -690,14 +692,15 @@ struct overwrite_plan_t {
 	       << ", left_operation=" << overwrite_plan.left_operation
 	       << ", right_operation=" << overwrite_plan.right_operation
 	       << ", block_size=" << overwrite_plan.block_size
+	       << ", is_left_stable=" << overwrite_plan.is_left_stable
+	       << ", is_right_stable=" << overwrite_plan.is_right_stable
 	       << ")";
   }
 
   overwrite_plan_t(laddr_t offset,
 		   extent_len_t len,
 		   const lba_pin_list_t& pins,
-		   extent_len_t block_size,
-		   Transaction& t) :
+		   extent_len_t block_size) :
       pin_begin(pins.front()->get_key()),
       pin_end(pins.back()->get_key() + pins.back()->get_length()),
       left_paddr(pins.front()->get_val()),
@@ -708,9 +711,11 @@ struct overwrite_plan_t {
       aligned_data_end(p2roundup((uint64_t)data_end, (uint64_t)block_size)),
       left_operation(overwrite_operation_t::UNKNOWN),
       right_operation(overwrite_operation_t::UNKNOWN),
-      block_size(block_size) {
+      block_size(block_size),
+      is_left_stable(pins.front()->is_stable()),
+      is_right_stable(pins.back()->is_stable()) {
     validate();
-    evaluate_operations(t);
+    evaluate_operations();
     assert(left_operation != overwrite_operation_t::UNKNOWN);
     assert(right_operation != overwrite_operation_t::UNKNOWN);
   }
@@ -738,31 +743,17 @@ struct overwrite_plan_t {
    * original extent into at most three parts: origin-left, part-to-be-modified
    * and origin-right.
    */
-  void evaluate_operations(Transaction& t) {
+  void evaluate_operations() {
     auto actual_write_size = get_pins_size();
     auto aligned_data_size = get_aligned_data_size();
     auto left_ext_size = get_left_extent_size();
     auto right_ext_size = get_right_extent_size();
 
-    auto can_merge = [](Transaction& t, paddr_t paddr) {
-      CachedExtentRef ext;
-      if (paddr.is_relative() || paddr.is_delayed()) {
-	  return true;
-      } else if (t.get_extent(paddr, &ext) ==
-	Transaction::get_extent_ret::PRESENT) {
-	// FIXME: there is no need to lookup the cache if the pin can 
-	// be associated with the extent state
-	if (ext->is_mutable()) {
-	  return true;
-	}
-      }
-      return false;
-    };
     if (left_paddr.is_zero()) {
       actual_write_size -= left_ext_size;
       left_ext_size = 0;
       left_operation = overwrite_operation_t::OVERWRITE_ZERO;
-    } else if (can_merge(t, left_paddr)) {
+    } else if (!is_left_stable) {
       aligned_data_size += left_ext_size;
       left_ext_size = 0;
       left_operation = overwrite_operation_t::MERGE_EXISTING;
@@ -772,7 +763,7 @@ struct overwrite_plan_t {
       actual_write_size -= right_ext_size;
       right_ext_size = 0;
       right_operation = overwrite_operation_t::OVERWRITE_ZERO;
-    } else if (can_merge(t, right_paddr)) {
+    } else if (!is_right_stable) {
       aligned_data_size += right_ext_size;
       right_ext_size = 0;
       right_operation = overwrite_operation_t::MERGE_EXISTING;
@@ -1282,7 +1273,7 @@ ObjectDataHandler::write_ret ObjectDataHandler::overwrite(
   if (bl.has_value()) {
     assert(bl->length() == len);
   }
-  overwrite_plan_t overwrite_plan(offset, len, _pins, ctx.tm.get_block_size(), ctx.t);
+  overwrite_plan_t overwrite_plan(offset, len, _pins, ctx.tm.get_block_size());
   return seastar::do_with(
     std::move(_pins),
     extent_to_write_list_t(),
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 640b98f79426..9f2edcdb5622 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -433,8 +433,7 @@ class TransactionManager : public ExtentCallbackInterface {
     ceph_assert(total_remap_len < original_len);
 #endif
 
-    // FIXME: paddr can be absolute and pending
-    ceph_assert(pin->get_val().is_absolute());
+    // The according extent might be stable or pending.
     return cache->get_extent_if_cached(
       t, pin->get_val(), T::TYPE
     ).si_then([this, &t, remaps,

From 3c7ef1bbbef45b0901e000ae8b793f5a2279675d Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Sat, 2 Mar 2024 08:58:19 +0000
Subject: [PATCH 2142/2492] test/crimson/seastore: fixes UT to use
 LBAMapping::is_stable()

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/test/crimson/seastore/test_object_data_handler.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/test/crimson/seastore/test_object_data_handler.cc b/src/test/crimson/seastore/test_object_data_handler.cc
index d054fc18f480..0404fbdd7531 100644
--- a/src/test/crimson/seastore/test_object_data_handler.cc
+++ b/src/test/crimson/seastore/test_object_data_handler.cc
@@ -657,10 +657,9 @@ TEST_P(object_data_handler_test_t, multiple_remap) {
   run_async([this] {
     multiple_write();
     auto pins = get_mappings(0, 128<<10);
-    EXPECT_EQ(pins.size(), 10);
+    EXPECT_EQ(pins.size(), 3);
 
-    size_t res[10] = {0, 4<<10, 12<<10, 20<<10, 32<<10,
-		      36<<10, 60<<10, 96<<10, 120<<10, 124<<10};
+    size_t res[3] = {0, 120<<10, 124<<10};
     auto base = pins.front()->get_key();
     int i = 0;
     for (auto &pin : pins) {

From eeb09b19581d353fb68f86bb4c2871f026a35c29 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Tue, 5 Mar 2024 12:37:21 +0000
Subject: [PATCH 2143/2492] mgr/dashboard: fixed cephfs mount command

Fixes: https://tracker.ceph.com/issues/64716

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-mount-details/cephfs-mount-details.component.ts      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
index 77a3f4afadc6..af0b93a96af6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
@@ -19,7 +19,7 @@ export class CephfsMountDetailsComponent implements OnInit, OnDestroy {
   nfs!: string;
 
   ngOnInit(): void {
-    this.mount = `sudo <CLIENT_USER>@${this.mountData?.fsId}.${this.mountData?.fsName}=${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
+    this.mount = `sudo mount -t ceph <CLIENT_USER>@${this.mountData?.fsId}.${this.mountData?.fsName}=${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
     this.fuse = `sudo ceph-fuse  ${this.MOUNT_DIRECTORY} -r ${this.mountData?.rootPath} --client_mds_namespace=${this.mountData?.fsName}`;
     this.nfs = `sudo mount -t nfs -o port=<PORT> <IP of active_mds daemon>:${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
   }

From b8811c844fa13b533be18015adb25e90dac2bc58 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 6 Feb 2024 18:09:41 +0530
Subject: [PATCH 2144/2492] mgr/dashboard: introduce multi-cluster overview
 page

https://tracker.ceph.com/issues/64530
Signed-off-by: Nizamudeen A <nia@redhat.com>
Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../dashboards/multi-cluster.libsonnet        |   2 +-
 .../multi-cluster-overview.json               |   2 +-
 .../dashboard/controllers/multi_cluster.py    | 104 ++++--
 .../mgr/dashboard/controllers/prometheus.py   |   5 +
 .../frontend/cypress/e2e/ui/navigation.po.ts  |  19 +-
 .../frontend/src/app/app-routing.module.ts    |   5 +-
 .../src/app/ceph/cluster/cluster.module.ts    |   4 +-
 .../multi-cluster-form.component.html         |  67 +++-
 .../multi-cluster-form.component.ts           |  65 +++-
 .../multi-cluster-list.component.html         |   2 +-
 .../multi-cluster.component.html              | 197 +++++++++++-
 .../multi-cluster.component.spec.ts           |   6 +-
 .../multi-cluster/multi-cluster.component.ts  | 304 +++++++++++++++++-
 .../dashboard-area-chart.component.html       |   8 +-
 .../dashboard-area-chart.component.ts         |   6 +-
 .../ceph/dashboard-v3/dashboard-v3.module.ts  |   7 +-
 .../workbench-layout.component.html           |   2 +-
 .../app/shared/api/multi-cluster.service.ts   |  47 ++-
 .../src/app/shared/api/prometheus.service.ts  |  97 +++++-
 .../card-group/card-group.component.html      |  10 +
 .../card-group/card-group.component.scss      |   4 +
 .../card-group/card-group.component.spec.ts   |  22 ++
 .../card-group/card-group.component.ts        |  11 +
 .../shared/components/components.module.ts    |   7 +-
 .../app/shared/enum/dashboard-promqls.enum.ts |  28 ++
 .../src/app/shared/models/multi-cluster.ts    |   2 +
 src/pybind/mgr/dashboard/openapi.yaml         |  87 +++--
 27 files changed, 976 insertions(+), 144 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.ts

diff --git a/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet b/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
index ec725f4dfa28..a77056540083 100644
--- a/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/multi-cluster.libsonnet
@@ -5,7 +5,7 @@ local g = import 'grafonnet/grafana.libsonnet';
     $.dashboardSchema(
       'Ceph - Multi-cluster',
       '',
-      'BnxelG7Sz',
+      'BnxelG7Sx',
       'now-1h',
       '30s',
       22,
diff --git a/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json b/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
index 91b2934f0654..ff8bcdd02544 100644
--- a/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
+++ b/monitoring/ceph-mixin/dashboards_out/multi-cluster-overview.json
@@ -2118,6 +2118,6 @@
    },
    "timezone": "",
    "title": "Ceph - Multi-cluster",
-   "uid": "BnxelG7Sz",
+   "uid": "BnxelG7Sx",
    "version": 0
 }
diff --git a/src/pybind/mgr/dashboard/controllers/multi_cluster.py b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
index c918c2ec3c24..d69a7da26094 100644
--- a/src/pybind/mgr/dashboard/controllers/multi_cluster.py
+++ b/src/pybind/mgr/dashboard/controllers/multi_cluster.py
@@ -6,6 +6,7 @@
 
 import requests
 
+from .. import mgr
 from ..exceptions import DashboardException
 from ..security import Scope
 from ..settings import Settings
@@ -18,9 +19,10 @@
 @APIDoc('Multi-cluster Management API', 'Multi-cluster')
 class MultiCluster(RESTController):
     def _proxy(self, method, base_url, path, params=None, payload=None, verify=False,
-               token=None):
+               token=None, cert=None):
         if not base_url.endswith('/'):
             base_url = base_url + '/'
+
         try:
             if token:
                 headers = {
@@ -33,7 +35,7 @@ def _proxy(self, method, base_url, path, params=None, payload=None, verify=False
                     'Content-Type': 'application/json',
                 }
             response = requests.request(method, base_url + path, params=params,
-                                        json=payload, verify=verify, headers=headers)
+                                        json=payload, verify=verify, cert=cert, headers=headers)
         except Exception as e:
             raise DashboardException(
                 "Could not reach {}, {}".format(base_url+path, e),
@@ -51,10 +53,10 @@ def _proxy(self, method, base_url, path, params=None, payload=None, verify=False
     @Endpoint('POST')
     @CreatePermission
     @EndpointDoc("Authenticate to a remote cluster")
-    def auth(self, url: str, cluster_alias: str, username=None,
-             password=None, token=None, hub_url=None, cluster_fsid=None):
-
-        if username and password:
+    def auth(self, url: str, cluster_alias: str, username: str,
+             password=None, token=None, hub_url=None, cluster_fsid=None,
+             prometheus_api_url=None, ssl_verify=False, ssl_certificate=None):
+        if password:
             payload = {
                 'username': username,
                 'password': password
@@ -69,16 +71,28 @@ def auth(self, url: str, cluster_alias: str, username=None,
             cluster_token = content['token']
 
             self._proxy('PUT', url, 'ui-api/multi-cluster/set_cors_endpoint',
-                        payload={'url': hub_url}, token=cluster_token)
-
+                        payload={'url': hub_url}, token=cluster_token, verify=ssl_verify,
+                        cert=ssl_certificate)
             fsid = self._proxy('GET', url, 'api/health/get_cluster_fsid', token=cluster_token)
 
-            self.set_multi_cluster_config(fsid, username, url, cluster_alias, cluster_token)
+            # add prometheus targets
+            prometheus_url = self._proxy('GET', url, 'api/settings/PROMETHEUS_API_HOST',
+                                         token=cluster_token)
+            _set_prometheus_targets(prometheus_url['value'])
 
-        if token and cluster_fsid and username:
-            self.set_multi_cluster_config(cluster_fsid, username, url, cluster_alias, token)
+            self.set_multi_cluster_config(fsid, username, url, cluster_alias,
+                                          cluster_token, prometheus_url['value'],
+                                          ssl_verify, ssl_certificate)
+            return
 
-    def set_multi_cluster_config(self, fsid, username, url, cluster_alias, token):
+        if token and cluster_fsid and prometheus_api_url:
+            _set_prometheus_targets(prometheus_api_url)
+            self.set_multi_cluster_config(cluster_fsid, username, url,
+                                          cluster_alias, token, prometheus_api_url,
+                                          ssl_verify, ssl_certificate)
+
+    def set_multi_cluster_config(self, fsid, username, url, cluster_alias, token,
+                                 prometheus_url=None, ssl_verify=False, ssl_certificate=None):
         multi_cluster_config = self.load_multi_cluster_config()
         if fsid in multi_cluster_config['config']:
             existing_entries = multi_cluster_config['config'][fsid]
@@ -89,6 +103,9 @@ def set_multi_cluster_config(self, fsid, username, url, cluster_alias, token):
                     "cluster_alias": cluster_alias,
                     "user": username,
                     "token": token,
+                    "prometheus_url": prometheus_url if prometheus_url else '',
+                    "ssl_verify": ssl_verify,
+                    "ssl_certificate": ssl_certificate if ssl_certificate else ''
                 })
         else:
             multi_cluster_config['current_user'] = username
@@ -98,6 +115,9 @@ def set_multi_cluster_config(self, fsid, username, url, cluster_alias, token):
                 "cluster_alias": cluster_alias,
                 "user": username,
                 "token": token,
+                "prometheus_url": prometheus_url if prometheus_url else '',
+                "ssl_verify": ssl_verify,
+                "ssl_certificate": ssl_certificate if ssl_certificate else ''
             }]
         Settings.MULTICLUSTER_CONFIG = multi_cluster_config
 
@@ -123,16 +143,18 @@ def set_config(self, config: object):
         return Settings.MULTICLUSTER_CONFIG
 
     @Endpoint('PUT')
-    @CreatePermission
+    @UpdatePermission
     # pylint: disable=unused-variable
-    def reconnect_cluster(self, url: str, username=None, password=None, token=None):
+    def reconnect_cluster(self, url: str, username=None, password=None, token=None,
+                          ssl_verify=False, ssl_certificate=None):
         multicluster_config = self.load_multi_cluster_config()
         if username and password:
             payload = {
                 'username': username,
                 'password': password
             }
-            content = self._proxy('POST', url, 'api/auth', payload=payload)
+            content = self._proxy('POST', url, 'api/auth', payload=payload,
+                                  verify=ssl_verify, cert=ssl_certificate)
             if 'token' not in content:
                 raise DashboardException(
                     "Could not authenticate to remote cluster",
@@ -143,7 +165,7 @@ def reconnect_cluster(self, url: str, username=None, password=None, token=None):
 
         if username and token:
             if "config" in multicluster_config:
-                for key, cluster_details in multicluster_config["config"].items():
+                for _, cluster_details in multicluster_config["config"].items():
                     for cluster in cluster_details:
                         if cluster["url"] == url and cluster["user"] == username:
                             cluster['token'] = token
@@ -168,31 +190,38 @@ def edit_cluster(self, url, cluster_alias, username):
     def delete_cluster(self, cluster_name, cluster_user):
         multicluster_config = self.load_multi_cluster_config()
         if "config" in multicluster_config:
-            keys_to_remove = []
-            for key, cluster_details in multicluster_config["config"].items():
-                cluster_details_copy = list(cluster_details)
-                for cluster in cluster_details_copy:
-                    if cluster["name"] == cluster_name and cluster["user"] == cluster_user:
-                        cluster_details.remove(cluster)
-                        if not cluster_details:
-                            keys_to_remove.append(key)
-
-            for key in keys_to_remove:
-                del multicluster_config["config"][key]
+            for key, value in list(multicluster_config['config'].items()):
+                if value[0]['name'] == cluster_name and value[0]['user'] == cluster_user:
+
+                    orch_backend = mgr.get_module_option_ex('orchestrator', 'orchestrator')
+                    try:
+                        if orch_backend == 'cephadm':
+                            cmd = {
+                                'prefix': 'orch prometheus remove-target',
+                                'url': value[0]['prometheus_url'].replace('http://', '').replace('https://', '')  # noqa E501 #pylint: disable=line-too-long
+                            }
+                            mgr.mon_command(cmd)
+                    except KeyError:
+                        pass
+
+                    del multicluster_config['config'][key]
+                    break
 
         Settings.MULTICLUSTER_CONFIG = multicluster_config
         return Settings.MULTICLUSTER_CONFIG
 
-    @Endpoint()
-    @ReadPermission
+    @Endpoint('POST')
+    @CreatePermission
     # pylint: disable=R0911
-    def verify_connection(self, url=None, username=None, password=None, token=None):
+    def verify_connection(self, url=None, username=None, password=None, token=None,
+                          ssl_verify=False, ssl_certificate=None):
         if token:
             try:
                 payload = {
                     'token': token
                 }
-                content = self._proxy('POST', url, 'api/auth/check', payload=payload)
+                content = self._proxy('POST', url, 'api/auth/check', payload=payload,
+                                      verify=ssl_verify, cert=ssl_certificate)
                 if 'permissions' not in content:
                     return content['detail']
                 user_content = self._proxy('GET', url, f'api/user/{username}',
@@ -210,7 +239,8 @@ def verify_connection(self, url=None, username=None, password=None, token=None):
                     'username': username,
                     'password': password
                 }
-                content = self._proxy('POST', url, 'api/auth', payload=payload)
+                content = self._proxy('POST', url, 'api/auth', payload=payload,
+                                      verify=ssl_verify, cert=ssl_certificate)
                 if 'token' not in content:
                     return content['detail']
                 user_content = self._proxy('GET', url, f'api/user/{username}',
@@ -266,3 +296,13 @@ class MultiClusterUi(RESTController):
     @UpdatePermission
     def set_cors_endpoint(self, url: str):
         configure_cors(url)
+
+
+def _set_prometheus_targets(prometheus_url: str):
+    orch_backend = mgr.get_module_option_ex('orchestrator', 'orchestrator')
+    if orch_backend == 'cephadm':
+        cmd = {
+            'prefix': 'orch prometheus set-target',
+            'url': prometheus_url.replace('http://', '').replace('https://', '')
+        }
+        mgr.mon_command(cmd)
diff --git a/src/pybind/mgr/dashboard/controllers/prometheus.py b/src/pybind/mgr/dashboard/controllers/prometheus.py
index b639d8826273..7f5f193f9ab4 100644
--- a/src/pybind/mgr/dashboard/controllers/prometheus.py
+++ b/src/pybind/mgr/dashboard/controllers/prometheus.py
@@ -146,6 +146,11 @@ def create_silence(self, **params):
     def delete_silence(self, s_id):
         return self.alert_proxy('DELETE', '/silence/' + s_id) if s_id else None
 
+    @RESTController.Collection(method='GET', path='/prometheus_query_data')
+    def get_prometeus_query_data(self, **params):
+        params['query'] = params.pop('params')
+        return self.prometheus_proxy('GET', '/query', params)
+
 
 @APIRouter('/prometheus/notifications', Scope.PROMETHEUS)
 @APIDoc("Prometheus Notifications Management API", "PrometheusNotifications")
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
index 7d84939b8807..f2eefd826d8a 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/ui/navigation.po.ts
@@ -7,6 +7,13 @@ export class NavigationPageHelper extends PageHelper {
 
   navigations = [
     { menu: 'Dashboard', component: 'cd-dashboard' },
+    {
+      menu: 'Multi-Cluster',
+      submenus: [
+        { menu: 'Overview', component: 'cd-multi-cluster' },
+        { menu: 'Manage Clusters', component: 'cd-multi-cluster-list' }
+      ]
+    },
     {
       menu: 'Cluster',
       submenus: [
@@ -78,7 +85,11 @@ export class NavigationPageHelper extends PageHelper {
     cy.intercept('/ui-api/block/rbd/status', { fixture: 'block-rbd-status.json' });
 
     navs.forEach((nav: any) => {
-      cy.contains('.simplebar-content li.nav-item a', nav.menu).click();
+      cy.get('.simplebar-content li.nav-item a').each(($link) => {
+        if ($link.text().trim() === nav.menu.trim()) {
+          cy.wrap($link).click();
+        }
+      });
       if (nav.submenus) {
         this.checkNavSubMenu(nav.menu, nav.submenus);
       } else {
@@ -89,8 +100,10 @@ export class NavigationPageHelper extends PageHelper {
 
   checkNavSubMenu(menu: any, submenu: any) {
     submenu.forEach((nav: any) => {
-      cy.contains('.simplebar-content li.nav-item', menu).within(() => {
-        cy.contains(`ul.list-unstyled li a`, nav.menu).click();
+      cy.get('.simplebar-content li.nav-item a').each(($link) => {
+        if ($link.text().trim() === menu.trim()) {
+          cy.contains(`ul.list-unstyled li a`, nav.menu).click();
+        }
       });
     });
   }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
index c54681b065f2..6744e9cf23b3 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts
@@ -191,10 +191,7 @@ const routes: Routes = [
         children: [
           {
             path: 'overview',
-            component: MultiClusterComponent,
-            data: {
-              breadcrumbs: 'Multi-Cluster/Overview'
-            }
+            component: MultiClusterComponent
           },
           {
             path: 'manage-clusters',
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
index 2f0734885d85..b76189612b8b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts
@@ -64,6 +64,7 @@ import { UpgradeProgressComponent } from './upgrade/upgrade-progress/upgrade-pro
 import { MultiClusterComponent } from './multi-cluster/multi-cluster.component';
 import { MultiClusterFormComponent } from './multi-cluster/multi-cluster-form/multi-cluster-form.component';
 import { MultiClusterListComponent } from './multi-cluster/multi-cluster-list/multi-cluster-list.component';
+import { DashboardV3Module } from '../dashboard-v3/dashboard-v3.module';
 
 @NgModule({
   imports: [
@@ -84,7 +85,8 @@ import { MultiClusterListComponent } from './multi-cluster/multi-cluster-list/mu
     NgbPopoverModule,
     NgbDropdownModule,
     NgxPipeFunctionModule,
-    NgbProgressbarModule
+    NgbProgressbarModule,
+    DashboardV3Module
   ],
   declarations: [
     HostsComponent,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
index c875557306a8..a2d36e4232aa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
@@ -117,15 +117,25 @@
                  i18n>Password
           </label>
           <div class="cd-col-form-input">
-            <input id="password"
-                   name="password"
-                   class="form-control"
-                   type="password"
-                   formControlName="password">
-            <span class="invalid-feedback"
-                  *ngIf="remoteClusterForm.showError('password', frm, 'required')"
-                  i18n>This field is required.
-            </span>
+            <div class="input-group">
+              <input id="password"
+                     name="password"
+                     class="form-control"
+                     type="password"
+                     formControlName="password">
+              <span class="input-group-button">
+                <button type="button"
+                        class="btn btn-light"
+                        cdPasswordButton="password">
+                </button>
+                <cd-copy-2-clipboard-button source="password">
+                </cd-copy-2-clipboard-button>
+              </span>
+              <span class="invalid-feedback"
+                    *ngIf="remoteClusterForm.showError('password', frm, 'required')"
+                    i18n>This field is required.
+              </span>
+            </div>
           </div>
         </div>
         <div class="form-group row"
@@ -161,6 +171,45 @@
             </div>
           </div>
         </div>
+        <!-- ssl -->
+        <div class="form-group row">
+          <div class="cd-col-form-offset">
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="ssl"
+                     type="checkbox"
+                     formControlName="ssl">
+              <label class="custom-control-label"
+                     for="ssl"
+                     i18n>SSL</label>
+            </div>
+          </div>
+        </div>
+
+        <!-- ssl_cert -->
+          <div *ngIf="remoteClusterForm.controls.ssl.value"
+               class="form-group row">
+            <label class="cd-col-form-label"
+                   for="ssl_cert">
+              <span i18n>Certificate</span>
+              <cd-helper i18n>The SSL certificate in PEM format.</cd-helper>
+            </label>
+            <div class="cd-col-form-input">
+              <textarea id="ssl_cert"
+                        class="form-control resize-vertical text-monospace text-pre"
+                        formControlName="ssl_cert"
+                        rows="5">
+              </textarea>
+              <input type="file"
+                     (change)="fileUpload($event.target.files, 'ssl_cert')">
+              <span class="invalid-feedback"
+                    *ngIf="remoteClusterForm.showError('ssl_cert', frm, 'required')"
+                    i18n>This field is required.</span>
+              <span class="invalid-feedback"
+                    *ngIf="remoteClusterForm.showError('ssl_cert', frm, 'pattern')"
+                    i18n>Invalid SSL certificate.</span>
+            </div>
+          </div>
         <div class="form-group row"
              *ngIf="!showCrossOriginError && action !== 'edit' && !remoteClusterForm.getValue('showToken')">
           <div class="cd-col-form-offset">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
index 83eb9fb5d51e..ee39a51d4700 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
@@ -1,5 +1,5 @@
 import { Component, EventEmitter, OnDestroy, OnInit, Output } from '@angular/core';
-import { FormControl, Validators } from '@angular/forms';
+import { AbstractControl, FormControl, Validators } from '@angular/forms';
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
 import _ from 'lodash';
 import { Subscription } from 'rxjs';
@@ -50,6 +50,8 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
       this.remoteClusterForm.get('remoteClusterUrl').setValue(this.cluster.url);
       this.remoteClusterForm.get('remoteClusterUrl').disable();
       this.remoteClusterForm.get('clusterAlias').setValue(this.cluster.cluster_alias);
+      this.remoteClusterForm.get('ssl').setValue(this.cluster.ssl_verify);
+      this.remoteClusterForm.get('ssl_cert').setValue(this.cluster.ssl_certificate);
     }
     if (this.action === 'reconnect') {
       this.remoteClusterForm.get('remoteClusterUrl').setValue(this.cluster.url);
@@ -60,6 +62,8 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
       this.remoteClusterForm.get('username').disable();
       this.remoteClusterForm.get('clusterFsid').setValue(this.cluster.name);
       this.remoteClusterForm.get('clusterFsid').disable();
+      this.remoteClusterForm.get('ssl').setValue(this.cluster.ssl_verify);
+      this.remoteClusterForm.get('ssl_cert').setValue(this.cluster.ssl_certificate);
     }
     [this.clusterAliasNames, this.clusterUrls, this.clusterUsers] = [
       'cluster_alias',
@@ -128,6 +132,14 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
             );
           })
         ]
+      }),
+      ssl: new FormControl(false),
+      ssl_cert: new FormControl('', {
+        validators: [
+          CdValidators.requiredIf({
+            ssl: true
+          })
+        ]
       })
     });
   }
@@ -144,6 +156,8 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
     const password = this.remoteClusterForm.getValue('password');
     const token = this.remoteClusterForm.getValue('apiToken');
     const clusterFsid = this.remoteClusterForm.getValue('clusterFsid');
+    const ssl = this.remoteClusterForm.getValue('ssl');
+    const ssl_certificate = this.remoteClusterForm.getValue('ssl_cert')?.trim();
 
     if (this.action === 'edit') {
       this.subs.add(
@@ -167,19 +181,21 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
 
     if (this.action === 'reconnect') {
       this.subs.add(
-        this.multiClusterService.reConnectCluster(updatedUrl, username, password, token).subscribe({
-          error: () => {
-            this.remoteClusterForm.setErrors({ cdSubmitButton: true });
-          },
-          complete: () => {
-            this.notificationService.show(
-              NotificationType.success,
-              $localize`Cluster reconnected successfully`
-            );
-            this.submitAction.emit();
-            this.activeModal.close();
-          }
-        })
+        this.multiClusterService
+          .reConnectCluster(updatedUrl, username, password, token, ssl, ssl_certificate)
+          .subscribe({
+            error: () => {
+              this.remoteClusterForm.setErrors({ cdSubmitButton: true });
+            },
+            complete: () => {
+              this.notificationService.show(
+                NotificationType.success,
+                $localize`Cluster reconnected successfully`
+              );
+              this.submitAction.emit();
+              this.activeModal.close();
+            }
+          })
       );
     }
 
@@ -193,7 +209,9 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
             password,
             token,
             window.location.origin,
-            clusterFsid
+            clusterFsid,
+            ssl,
+            ssl_certificate
           )
           .subscribe({
             error: () => {
@@ -217,10 +235,12 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
     const username = this.remoteClusterForm.getValue('username');
     const password = this.remoteClusterForm.getValue('password');
     const token = this.remoteClusterForm.getValue('apiToken');
+    const ssl = this.remoteClusterForm.getValue('ssl');
+    const ssl_certificate = this.remoteClusterForm.getValue('ssl_cert')?.trim();
 
     this.subs.add(
       this.multiClusterService
-        .verifyConnection(url, username, password, token)
+        .verifyConnection(url, username, password, token, ssl, ssl_certificate)
         .subscribe((resp: string) => {
           switch (resp) {
             case 'Connection successful':
@@ -259,4 +279,17 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
   toggleToken() {
     this.showToken = !this.showToken;
   }
+
+  fileUpload(files: FileList, controlName: string) {
+    const file: File = files[0];
+    const reader = new FileReader();
+    reader.addEventListener('load', (event: ProgressEvent<FileReader>) => {
+      const control: AbstractControl = this.remoteClusterForm.get(controlName);
+      control.setValue(event.target.result);
+      control.markAsDirty();
+      control.markAsTouched();
+      control.updateValueAndValidity();
+    });
+    reader.readAsText(file, 'utf8');
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
index 7aea2f470761..74cfc78ab8af 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.html
@@ -29,7 +29,7 @@
              let-row="row">
   <a target="_blank"
      [href]="row.url">
-      {{ row.url.endsWith('/') ? row.url.slice(0, -1) : row.url }}
+      {{ row?.url?.endsWith('/') ? row?.url?.slice(0, -1) : row.url }}
     <i class="fa fa-external-link"></i>
   </a>
 </ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
index c826f155c40b..0542b1868bae 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
@@ -6,7 +6,7 @@
            [ngClass]="icons.wrench">
         </i>
       <div class="mt-4 text-center">
-        <h3><b>Connect Cluster </b></h3>
+        <h3 class="fw-bold">Connect Cluster</h3>
         <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortlessly.
             Click on the "Connect Cluster" button to begin the process.</h4>
       </div>
@@ -35,17 +35,190 @@ <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortles
   </div>
 </ng-template>
 
-<div class="container-fluid h-100 p-4">
-  <div *ngIf="dashboardClustersMap?.size === 1">
-    <ng-container *ngTemplateOutlet="emptyCluster"></ng-container>
-  </div>
-
-  <span *ngIf="loading"
-        class="d-flex justify-content-center">
-    <i [ngClass]="[icons.large3x, icons.spinner, icons.spin]"></i>
-  </span>
-  <div *ngIf="dashboardClustersMap?.size > 1">
-    <div *ngIf="!loading">
+<ng-template #loadingTpl>
+  <div class="container h-75">
+    <div class="row h-100 justify-content-center align-items-center">
+      <div class="blank-page">
+        <i class="mx-auto d-block"
+           [ngClass]="[icons.large3x, icons.spinner, icons.spin]">
+        </i>
+      </div>
     </div>
   </div>
+</ng-template>
+
+<div class="container-fluid h-100 p-4"
+     *ngIf="isMultiCluster; else emptyCluster">
+  <ng-container *ngIf="!loading; else loadingTpl">
+    <cd-card-group>
+      <div class="col-lg-4">
+        <div class="row">
+          <cd-card cardTitle="Clusters"
+                   i18n-title
+                   class="col-sm-6 m-0 p-0 ps-4 pe-2"
+                   aria-label="Clusters"
+                   [fullHeight]="true"
+                   *ngIf="queriesResults.CLUSTER_COUNT && queriesResults.CLUSTER_COUNT[0]">
+            <span class="text-center">
+              <h3 *ngIf="queriesResults['HEALTH_ERROR_COUNT'][0][1] === '0' && queriesResults['HEALTH_WARNING_COUNT'][0][1] === '0'">{{ queriesResults.CLUSTER_COUNT[0][1] }}</h3>
+              <h3 class="text-danger"
+                  *ngIf="queriesResults.HEALTH_ERROR_COUNT[0][1] !== '0'">
+                <i [ngClass]="icons.danger"></i>
+                {{ queriesResults.HEALTH_ERROR_COUNT[0][1] }}
+              </h3>
+              <h3 class="text-warning"
+                  *ngIf="queriesResults.HEALTH_WARNING_COUNT[0][1] !== '0'">
+                <i [ngClass]="icons.warning"></i>
+                  {{ queriesResults.HEALTH_WARNING_COUNT[0][1] }}
+              </h3>
+            </span>
+          </cd-card>
+          <cd-card cardTitle="Alerts"
+                   i18n-title
+                   class="col-sm-6 m-0 p-0 ps-2 pe-2"
+                   aria-label="Alerts"
+                   *ngIf="queriesResults['ALERTS_COUNT'] && queriesResults['ALERTS_COUNT'][0]">
+            <span class="text-center">
+              <h3 *ngIf="queriesResults['CRITICAL_ALERTS_COUNT'][0][1] === '0' && queriesResults['WARNING_ALERTS_COUNT'][0][1] === '0'">
+                  {{ queriesResults['ALERTS_COUNT'][0][1] }}
+              </h3>
+              <h3 class="text-danger"
+                  *ngIf="queriesResults['CRITICAL_ALERTS_COUNT'][0][1] !== '0'">
+                <i [ngClass]="icons.danger"></i>
+                {{ queriesResults['CRITICAL_ALERTS_COUNT'][0][1] }}
+              </h3>
+              <h3 class="text-warning"
+                  *ngIf="queriesResults['WARNING_ALERTS_COUNT'][0][1] !== '0'">
+                <i [ngClass]="icons.warning"></i>
+                  {{ queriesResults['WARNING_ALERTS_COUNT'][0][1] }}
+              </h3>
+            </span>
+          </cd-card>
+        </div>
+        <div class="row pt-3">
+          <cd-card cardTitle="Connection Errors"
+                   i18n-title
+                   class="col-sm-6 m-0 p-0 ps-4 pe-2"
+                   aria-label="Connection Errors">
+            <span class="text-center">
+              <h3 [ngClass]="{'text-danger': connectionErrorsCount > 0}">
+                <i [ngClass]="icons.danger"
+                   *ngIf="connectionErrorsCount > 0"></i>
+                {{ connectionErrorsCount }}
+              </h3>
+            </span>
+          </cd-card>
+          <cd-card cardTitle="Hosts"
+                   i18n-title
+                   class="col-sm-6 m-0 p-0 ps-2 pe-2"
+                   aria-label="Total number of hosts"
+                   *ngIf="queriesResults['TOTAL_HOSTS'][0][1] !== '0'">
+            <span class="text-center">
+              <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
+            </span>
+          </cd-card>
+        </div>
+
+        <div class="row pt-3">
+          <cd-card cardTitle="Capacity"
+                   i18n-title
+                   class="col-sm-12 m-0 p-0 ps-4 pe-2"
+                   aria-label="Capacity card"
+                   *ngIf="queriesResults['TOTAL_CLUSTERS_CAPACITY'] && queriesResults['TOTAL_CLUSTERS_CAPACITY'][0] && queriesResults['TOTAL_USED_CAPACITY'] && queriesResults['TOTAL_USED_CAPACITY'][0]">
+            <ng-container class="ms-4 me-4">
+              <cd-dashboard-pie [data]="{max: queriesResults['TOTAL_CLUSTERS_CAPACITY'][0][1], current: queriesResults['TOTAL_USED_CAPACITY'][0][1]}"
+                                lowThreshold=".95"
+                                highThreshold=".99">
+              </cd-dashboard-pie>
+            </ng-container>
+          </cd-card>
+        </div>
+      </div>
+
+      <div class="col-sm-8 ps-2">
+        <cd-card [cardTitle]="'Top ' + COUNT_OF_UTILIZATION_CHARTS + ' Cluster Utilization'"
+                 i18n-title
+                 [fullHeight]="true"
+                 aria-label="Cluster Utilization card"
+                 *ngIf="clusters">
+          <div class="ms-4 me-4 mt-0">
+            <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
+            </cd-dashboard-time-selector>
+            <cd-dashboard-area-chart  chartTitle="Capacity"
+                                      [labelsArray]="capacityLabels"
+                                      dataUnits="B"
+                                      [dataArray]="capacityValues"
+                                      [truncateLabel]="true"
+                                      *ngIf="capacityLabels && capacityValues">
+            </cd-dashboard-area-chart>
+            <cd-dashboard-area-chart chartTitle="IOPS"
+                                     [labelsArray]="iopsLabels"
+                                     dataUnits=""
+                                     decimals="0"
+                                     [dataArray]="iopsValues"
+                                     [truncateLabel]="true"
+                                     *ngIf="iopsLabels && iopsValues">
+            </cd-dashboard-area-chart>
+            <cd-dashboard-area-chart chartTitle="Throughput"
+                                     [labelsArray]="throughputLabels"
+                                     dataUnits="B/s"
+                                     decimals="2"
+                                     [dataArray]="throughputValues"
+                                     [truncateLabel]="true"
+                                     *ngIf="throughputLabels && throughputLabels">
+            </cd-dashboard-area-chart>
+          </div>
+        </cd-card>
+      </div>
+    </cd-card-group>
+
+    <cd-card-group>
+      <div class="col-lg-12 mt-3 m-0 p-0 ps-4 pe-4">
+        <cd-table [data]="clusters"
+                  [columns]="columns"
+                  [limit]="5"
+                  *ngIf="clusters">
+        </cd-table>
+      </div>
+    </cd-card-group>
+
+    <cd-card-group>
+      <div class="col-lg-12 mb-4 m-0 p-0 ps-4 pe-4">
+        <div class="row">
+          <cd-card [cardTitle]="'Top ' + COUNT_OF_UTILIZATION_CHARTS + ' Pools Utilization'"
+                   i18n-title
+                   aria-label="Pools Utilization card"
+                   *ngIf="clusters">
+            <div class="ms-4 me-4 mt-0">
+              <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
+              </cd-dashboard-time-selector>
+              <cd-dashboard-area-chart  chartTitle="Capacity"
+                                        [labelsArray]="poolCapacityLabels"
+                                        dataUnits="B"
+                                        [dataArray]="poolCapacityValues"
+                                        *ngIf="poolCapacityLabels && poolCapacityValues"
+                                        [truncateLabel]="true">
+              </cd-dashboard-area-chart>
+              <cd-dashboard-area-chart chartTitle="IOPS"
+                                       [labelsArray]="poolIOPSLabels"
+                                       dataUnits=""
+                                       decimals="0"
+                                       [dataArray]="poolIOPSValues"
+                                       *ngIf="poolIOPSLabels && poolIOPSValues"
+                                       [truncateLabel]="true">
+              </cd-dashboard-area-chart>
+              <cd-dashboard-area-chart chartTitle="Client Throughput"
+                                       [labelsArray]="poolThroughputLabels"
+                                       dataUnits="B/s"
+                                       decimals="2"
+                                       [dataArray]="poolThroughputValues"
+                                       *ngIf="poolThroughputLabels && poolThroughputValues"
+                                       [truncateLabel]="true">
+              </cd-dashboard-area-chart>
+            </div>
+          </cd-card>
+        </div>
+      </div>
+    </cd-card-group>
+  </ng-container>
 </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
index 8db81cd790fc..ad210968aa5b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.spec.ts
@@ -2,6 +2,8 @@ import { ComponentFixture, TestBed } from '@angular/core/testing';
 import { HttpClientTestingModule } from '@angular/common/http/testing';
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
 import { MultiClusterComponent } from './multi-cluster.component';
+import { SharedModule } from '~/app/shared/shared.module';
+import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 
 describe('MultiClusterComponent', () => {
   let component: MultiClusterComponent;
@@ -9,9 +11,9 @@ describe('MultiClusterComponent', () => {
 
   beforeEach(async () => {
     await TestBed.configureTestingModule({
-      imports: [HttpClientTestingModule],
+      imports: [HttpClientTestingModule, SharedModule],
       declarations: [MultiClusterComponent],
-      providers: [NgbActiveModal]
+      providers: [NgbActiveModal, DimlessBinaryPipe]
     }).compileComponents();
 
     fixture = TestBed.createComponent(MultiClusterComponent);
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
index dbbf10e74848..ab8b413e7362 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
@@ -5,6 +5,11 @@ import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
 import { Icons } from '~/app/shared/enum/icons.enum';
 import { ModalService } from '~/app/shared/services/modal.service';
 import { MultiClusterFormComponent } from './multi-cluster-form/multi-cluster-form.component';
+import { PrometheusService } from '~/app/shared/api/prometheus.service';
+import { MultiClusterPromqls as queries } from '~/app/shared/enum/dashboard-promqls.enum';
+import { CdTableColumn } from '~/app/shared/models/cd-table-column';
+import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
+import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 
 @Component({
   selector: 'cd-multi-cluster',
@@ -12,41 +17,150 @@ import { MultiClusterFormComponent } from './multi-cluster-form/multi-cluster-fo
   styleUrls: ['./multi-cluster.component.scss']
 })
 export class MultiClusterComponent implements OnInit {
+  COUNT_OF_UTILIZATION_CHARTS = 5;
+
   @ViewChild('nameTpl', { static: true })
   nameTpl: any;
 
+  columns: Array<CdTableColumn> = [];
+
+  queriesResults: any = {
+    ALERTS_COUNT: 0,
+    CLUSTER_COUNT: 0,
+    HEALTH_OK_COUNT: 0,
+    HEALTH_WARNING_COUNT: 0,
+    HEALTH_ERROR_COUNT: 0,
+    TOTAL_CLUSTERS_CAPACITY: 0,
+    TOTAL_USED_CAPACITY: 0,
+    CLUSTER_CAPACITY_UTILIZATION: 0,
+    CLUSTER_IOPS_UTILIZATION: 0,
+    CLUSTER_THROUGHPUT_UTILIZATION: 0,
+    POOL_CAPACITY_UTILIZATION: 0,
+    POOL_IOPS_UTILIZATION: 0,
+    POOL_THROUGHPUT_UTILIZATION: 0,
+    TOTAL_CAPACITY: 0,
+    USED_CAPACITY: 0,
+    HOSTS: 0,
+    POOLS: 0,
+    OSDS: 0,
+    CLUSTER_ALERTS: 0,
+    version: ''
+  };
+  alerts: any;
+
   private subs = new Subscription();
   dashboardClustersMap: Map<string, string> = new Map<string, string>();
   icons = Icons;
   loading = true;
   bsModalRef: NgbModalRef;
+  isMultiCluster = true;
+  clusterTokenStatus: object = {};
+  localClusterName: string;
+  clusters: any;
+  connectionErrorsCount = 0;
+
+  capacityLabels: string[] = [];
+  iopsLabels: string[] = [];
+  throughputLabels: string[] = [];
+  poolIOPSLabels: string[] = [];
+  poolCapacityLabels: string[] = [];
+  poolThroughputLabels: string[] = [];
+
+  capacityValues: string[] = [];
+  iopsValues: string[] = [];
+  throughputValues: string[] = [];
+  poolIOPSValues: string[] = [];
+  poolCapacityValues: string[] = [];
+  poolThroughputValues: string[] = [];
 
   constructor(
     private multiClusterService: MultiClusterService,
-    private modalService: ModalService
+    private modalService: ModalService,
+    private prometheusService: PrometheusService,
+    private dimlessBinaryPipe: DimlessBinaryPipe
   ) {}
 
   ngOnInit(): void {
+    this.columns = [
+      {
+        prop: 'cluster',
+        name: $localize`Cluster Name`,
+        flexGrow: 2,
+        cellTemplate: this.nameTpl
+      },
+      {
+        prop: 'cluster_connection_status',
+        name: $localize`Connection`,
+        flexGrow: 2,
+        cellTransformation: CellTemplate.badge,
+        customTemplateConfig: {
+          map: {
+            1: { value: 'DISCONNECTED', class: 'badge-danger' },
+            0: { value: 'CONNECTED', class: 'badge-success' },
+            2: { value: 'CHECKING..', class: 'badge-info' }
+          }
+        }
+      },
+      {
+        prop: 'status',
+        name: $localize`Status`,
+        flexGrow: 1,
+        cellTransformation: CellTemplate.badge,
+        customTemplateConfig: {
+          map: {
+            1: { value: 'WARN', class: 'badge-warning' },
+            0: { value: 'OK', class: 'badge-success' },
+            2: { value: 'ERROR', class: 'badge-danger' }
+          }
+        }
+      },
+      { prop: 'alert', name: $localize`Alerts`, flexGrow: 1 },
+      { prop: 'version', name: $localize`Version`, flexGrow: 2 },
+      {
+        prop: 'total_capacity',
+        name: $localize`Total Capacity`,
+        pipe: this.dimlessBinaryPipe,
+        flexGrow: 1
+      },
+      {
+        prop: 'used_capacity',
+        name: $localize`Used Capacity`,
+        pipe: this.dimlessBinaryPipe,
+        flexGrow: 1
+      },
+      {
+        prop: 'available_capacity',
+        name: $localize`Available Capacity`,
+        pipe: this.dimlessBinaryPipe,
+        flexGrow: 1
+      },
+      { prop: 'pools', name: $localize`Pools`, flexGrow: 1 },
+      { prop: 'hosts', name: $localize`Hosts`, flexGrow: 1 },
+      { prop: 'osds', name: $localize`OSDs`, flexGrow: 1 }
+    ];
+
     this.subs.add(
       this.multiClusterService.subscribe((resp: any) => {
-        const clustersConfig = resp['config'];
-        if (clustersConfig) {
-          Object.keys(clustersConfig).forEach((clusterKey: string) => {
-            const clusterDetailsList = clustersConfig[clusterKey];
-
-            clusterDetailsList.forEach((clusterDetails: any) => {
-              const clusterUrl = clusterDetails['url'];
-              const clusterName = clusterDetails['name'];
-              this.dashboardClustersMap.set(clusterUrl, clusterName);
-            });
-          });
-
-          if (this.dashboardClustersMap.size >= 1) {
-            this.loading = false;
+        this.isMultiCluster = Object.keys(resp['config']).length > 1;
+        const hubUrl = resp['hub_url'];
+        for (const key in resp['config']) {
+          if (resp['config'].hasOwnProperty(key)) {
+            const cluster = resp['config'][key][0];
+            if (hubUrl === cluster.url) {
+              this.localClusterName = cluster.name;
+              break;
+            }
           }
         }
       })
     );
+
+    this.subs.add(
+      this.multiClusterService.subscribeClusterTokenStatus((resp: object) => {
+        this.clusterTokenStatus = resp;
+      })
+    );
+    this.getPrometheusData(this.prometheusService.lastHourDateObject);
   }
 
   openRemoteClusterInfoModal() {
@@ -54,7 +168,165 @@ export class MultiClusterComponent implements OnInit {
       action: 'connect'
     };
     this.bsModalRef = this.modalService.show(MultiClusterFormComponent, initialState, {
-      size: 'xl'
+      size: 'lg'
+    });
+  }
+
+  getPrometheusData(selectedTime: any) {
+    this.prometheusService
+      .getMultiClusterQueriesData(selectedTime, queries, this.queriesResults)
+      .subscribe((data: any) => {
+        this.queriesResults = data;
+        this.loading = false;
+        this.alerts = this.queriesResults.ALERTS;
+        this.getAlertsInfo();
+        this.getClustersInfo();
+      });
+  }
+
+  getAlertsInfo() {
+    interface Alert {
+      alertName: string;
+      alertState: string;
+      severity: string;
+      cluster: string;
+    }
+
+    const alerts: Alert[] = [];
+
+    this.alerts?.forEach((item: any) => {
+      const metric = item.metric;
+      const alert: Alert = {
+        alertName: metric.alertname,
+        cluster: metric.cluster,
+        alertState: metric.alertstate,
+        severity: metric.severity
+      };
+      alerts.push(alert);
+    });
+
+    this.alerts = alerts;
+  }
+
+  getClustersInfo() {
+    interface ClusterInfo {
+      cluster: string;
+      status: number;
+      alert: number;
+      total_capacity: number;
+      used_capacity: number;
+      available_capacity: number;
+      pools: number;
+      osds: number;
+      hosts: number;
+      version: string;
+      cluster_connection_status: number;
+    }
+
+    const clusters: ClusterInfo[] = [];
+
+    this.queriesResults.TOTAL_CAPACITY?.forEach((totalCapacityMetric: any) => {
+      const clusterName = totalCapacityMetric.metric.cluster;
+      const totalCapacity = parseInt(totalCapacityMetric.value[1]);
+      const getMgrMetadata = this.findCluster(this.queriesResults?.MGR_METADATA, clusterName);
+      const version = this.getVersion(getMgrMetadata.metric.ceph_version);
+
+      const usedCapacity = this.findClusterData(this.queriesResults?.USED_CAPACITY, clusterName);
+      const pools = this.findClusterData(this.queriesResults?.POOLS, clusterName);
+      const hosts = this.findClusterData(this.queriesResults?.HOSTS, clusterName);
+      const alert = this.findClusterData(this.queriesResults?.CLUSTER_ALERTS, clusterName);
+      const osds = this.findClusterData(this.queriesResults?.OSDS, clusterName);
+      const status = this.findClusterData(this.queriesResults?.HEALTH_STATUS, clusterName);
+      const available_capacity = totalCapacity - usedCapacity;
+
+      clusters.push({
+        cluster: clusterName,
+        status,
+        alert,
+        total_capacity: totalCapacity,
+        used_capacity: usedCapacity,
+        available_capacity: available_capacity,
+        pools,
+        osds,
+        hosts,
+        version,
+        cluster_connection_status: 2
+      });
     });
+
+    if (this.clusterTokenStatus) {
+      clusters.forEach((cluster: any) => {
+        cluster.cluster_connection_status = this.clusterTokenStatus[cluster.cluster]?.status;
+        if (cluster.cluster === this.localClusterName) {
+          cluster.cluster_connection_status = 0;
+        }
+      });
+      this.connectionErrorsCount = clusters.filter(
+        (cluster) => cluster.cluster_connection_status === 1
+      ).length;
+    }
+
+    this.clusters = clusters;
+
+    // Generate labels and metrics for utilization charts
+    this.capacityLabels = this.generateQueryLabel(this.queriesResults.CLUSTER_CAPACITY_UTILIZATION);
+    this.iopsLabels = this.generateQueryLabel(this.queriesResults.CLUSTER_IOPS_UTILIZATION);
+    this.throughputLabels = this.generateQueryLabel(
+      this.queriesResults.CLUSTER_THROUGHPUT_UTILIZATION
+    );
+    this.poolCapacityLabels = this.generateQueryLabel(
+      this.queriesResults.POOL_CAPACITY_UTILIZATION,
+      true
+    );
+    this.poolIOPSLabels = this.generateQueryLabel(this.queriesResults.POOL_IOPS_UTILIZATION, true);
+    this.poolThroughputLabels = this.generateQueryLabel(
+      this.queriesResults.POOL_THROUGHPUT_UTILIZATION,
+      true
+    );
+
+    this.capacityValues = this.getQueryValues(this.queriesResults.CLUSTER_CAPACITY_UTILIZATION);
+    this.iopsValues = this.getQueryValues(this.queriesResults.CLUSTER_IOPS_UTILIZATION);
+    this.throughputValues = this.getQueryValues(this.queriesResults.CLUSTER_THROUGHPUT_UTILIZATION);
+    this.poolCapacityValues = this.getQueryValues(this.queriesResults.POOL_CAPACITY_UTILIZATION);
+    this.poolIOPSValues = this.getQueryValues(this.queriesResults.POOL_IOPS_UTILIZATION);
+    this.poolThroughputValues = this.getQueryValues(
+      this.queriesResults.POOL_THROUGHPUT_UTILIZATION
+    );
+  }
+
+  findClusterData(metrics: any, clusterName: string) {
+    const clusterMetrics = this.findCluster(metrics, clusterName);
+    return parseInt(clusterMetrics?.value[1] || 0);
+  }
+
+  findCluster(metrics: any, clusterName: string) {
+    return metrics.find((metric: any) => metric?.metric?.cluster === clusterName);
+  }
+
+  getVersion(fullVersion: string) {
+    const version = fullVersion.replace('ceph version ', '').split(' ');
+    return version[0] + ' ' + version.slice(2, version.length).join(' ');
+  }
+
+  generateQueryLabel(query: any, name = false, count = this.COUNT_OF_UTILIZATION_CHARTS) {
+    let labels = [];
+    for (let i = 0; i < count; i++) {
+      let label = '';
+      if (query[i]) {
+        label = query[i]?.metric?.cluster;
+        if (name) label = query[i]?.metric?.name + ' - ' + label;
+      }
+      labels.push(label);
+    }
+    // console.log(labels)
+    return labels;
+  }
+
+  getQueryValues(query: any, count = this.COUNT_OF_UTILIZATION_CHARTS) {
+    let values = [];
+    for (let i = 0; i < count; i++) {
+      if (query[i]) values.push(query[i]?.values);
+    }
+    return values;
   }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
index 6151843e4e01..2b4878e995d2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
@@ -10,8 +10,12 @@
         <div class="box"
              [style.background-color]="data.pointBackgroundColor">
         </div>
-        <div *ngIf="!chartTitle.includes(data.label)">{{ data.label }}:</div>
-          {{ data?.currentData || 'N/A' }} {{ data?.currentDataUnits }}
+        <ng-container *ngIf="!chartTitle.includes(data.label)">
+          <span [ngClass]="{'d-inline-block text-truncate': truncateLabel}"
+                [ngStyle]="{'width': truncateLabel ? '10rem' : 'auto'}"
+                [title]="data.label">{{ data.label }}</span>:
+        </ng-container>
+        <span>{{ data?.currentData || 'N/A' }} {{ data?.currentDataUnits }}</span>
         <div *ngIf="maxValue && data.currentData">
           used of {{ maxConvertedValue }} {{ maxConvertedValueUnits }}
         </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index ac0b9ac2ff43..607a3b7d51ad 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -29,6 +29,8 @@ export class DashboardAreaChartComponent implements OnChanges {
   labelsArray?: string[] = []; // Array of chart labels
   @Input()
   decimals?: number = 1;
+  @Input()
+  truncateLabel = false;
 
   currentDataUnits: string;
   currentData: number;
@@ -201,8 +203,8 @@ export class DashboardAreaChartComponent implements OnChanges {
       this.currentChartData = this.chartData;
       for (let index = 0; index < this.dataArray.length; index++) {
         this.chartData.dataset[index].data = this.formatData(this.dataArray[index]);
-        let currentDataValue = this.dataArray[index][this.dataArray[index].length - 1]
-          ? this.dataArray[index][this.dataArray[index].length - 1][1]
+        let currentDataValue = this.dataArray?.[index]?.[this.dataArray[index]?.length - 1]
+          ? this.dataArray[index][this.dataArray[index]?.length - 1][1]
           : 0;
         if (currentDataValue) {
           [
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
index 73b4f9fa840f..82843289b383 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-v3.module.ts
@@ -38,6 +38,11 @@ import { PgSummaryPipe } from './pg-summary.pipe';
     DashboardTimeSelectorComponent
   ],
 
-  exports: [DashboardV3Component, DashboardAreaChartComponent, DashboardTimeSelectorComponent]
+  exports: [
+    DashboardV3Component,
+    DashboardAreaChartComponent,
+    DashboardTimeSelectorComponent,
+    DashboardPieComponent
+  ]
 })
 export class DashboardV3Module {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
index 1c1846dae15e..2b3c82bfe20b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/layouts/workbench-layout/workbench-layout.component.html
@@ -1,7 +1,7 @@
 <block-ui>
   <cd-navigation>
     <div class="container-fluid h-100"
-         [ngClass]="{'dashboard': (router.url == '/dashboard' || router.url == '/dashboard_3' || router.url == '/multi-cluster'), 'rgw-dashboard': (router.url == '/rgw/overview')}">
+         [ngClass]="{'dashboard': (router.url == '/dashboard' || router.url == '/dashboard_3' || router.url == '/multi-cluster/overview'), 'rgw-dashboard': (router.url == '/rgw/overview')}">
     <cd-context></cd-context>
       <cd-breadcrumbs></cd-breadcrumbs>
       <router-outlet></router-outlet>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
index 7252e969e60a..ffb312de4d9d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
@@ -94,7 +94,9 @@ export class MultiClusterService {
     password: string,
     token = '',
     hub_url = '',
-    clusterFsid = ''
+    clusterFsid = '',
+    ssl = false,
+    cert = ''
   ) {
     return this.http.post('api/multi-cluster/auth', {
       url,
@@ -103,27 +105,46 @@ export class MultiClusterService {
       password,
       token,
       hub_url,
-      cluster_fsid: clusterFsid
+      cluster_fsid: clusterFsid,
+      ssl_verify: ssl,
+      ssl_certificate: cert
     });
   }
 
-  reConnectCluster(url: any, username: string, password: string, token = '') {
-    return this.http.put('api/multi-cluster/reconnect_cluster', {
+  reConnectCluster(
+    url: any,
+    username: string,
+    password: string,
+    token = '',
+    ssl = false,
+    cert = ''
+  ) {
+    return this.http.post('api/multi-cluster/reconnect_cluster', {
       url,
       username,
       password,
-      token
+      token,
+      ssl_verify: ssl,
+      ssl_certificate: cert
     });
   }
 
-  verifyConnection(url: string, username: string, password: string, token = ''): Observable<any> {
-    let params = new HttpParams()
-      .set('url', url)
-      .set('username', username)
-      .set('password', password)
-      .set('token', token);
-
-    return this.http.get('api/multi-cluster/verify_connection', { params });
+  verifyConnection(
+    url: string,
+    username: string,
+    password: string,
+    token = '',
+    ssl = false,
+    cert = ''
+  ): Observable<any> {
+    return this.http.post('api/multi-cluster/verify_connection', {
+      url: url,
+      username: username,
+      password: password,
+      token: token,
+      ssl_verify: ssl,
+      ssl_certificate: cert
+    });
   }
 
   private getClusterObserver() {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
index e1aa7a07cafc..b7db0bc2f3cc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
@@ -1,7 +1,7 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 
-import { Observable, Subscription, timer } from 'rxjs';
+import { Observable, Subscription, forkJoin, timer } from 'rxjs';
 import { map } from 'rxjs/operators';
 
 import { AlertmanagerSilence } from '../models/alertmanager-silence';
@@ -191,4 +191,99 @@ export class PrometheusService {
     };
     return formattedDate;
   }
+
+  getMultiClusterData(params: any): any {
+    return this.http.get<any>(`${this.baseURL}/prometheus_query_data`, { params });
+  }
+
+  getMultiClusterQueryRangeData(params: any): any {
+    return this.http.get<any>(`${this.baseURL}/data`, { params });
+  }
+
+  getMultiClusterQueriesData(selectedTime: any, queries: any, queriesResults: any) {
+    return new Observable((observer) => {
+      this.ifPrometheusConfigured(() => {
+        if (this.timerGetPrometheusDataSub) {
+          this.timerGetPrometheusDataSub.unsubscribe();
+        }
+
+        this.timerGetPrometheusDataSub = timer(0, this.timerTime).subscribe(() => {
+          selectedTime = this.updateTimeStamp(selectedTime);
+
+          const requests = [];
+          for (const queryName in queries) {
+            if (queries.hasOwnProperty(queryName)) {
+              const validRangeQueries1 = [
+                'CLUSTER_CAPACITY_UTILIZATION',
+                'CLUSTER_IOPS_UTILIZATION',
+                'CLUSTER_THROUGHPUT_UTILIZATION',
+                'POOL_CAPACITY_UTILIZATION',
+                'POOL_IOPS_UTILIZATION',
+                'POOL_THROUGHPUT_UTILIZATION'
+              ];
+              if (validRangeQueries1.includes(queryName)) {
+                const query = queries[queryName];
+                const request = this.getMultiClusterQueryRangeData({
+                  params: encodeURIComponent(query),
+                  start: selectedTime['start'],
+                  end: selectedTime['end'],
+                  step: selectedTime['step']
+                });
+                requests.push(request);
+              } else {
+                const query = queries[queryName];
+                const request = this.getMultiClusterData({
+                  params: encodeURIComponent(query),
+                  start: selectedTime['start'],
+                  end: selectedTime['end'],
+                  step: selectedTime['step']
+                });
+                requests.push(request);
+              }
+            }
+          }
+
+          forkJoin(requests).subscribe(
+            (responses: any[]) => {
+              for (let i = 0; i < responses.length; i++) {
+                const data = responses[i];
+                const queryName = Object.keys(queries)[i];
+                const validQueries = [
+                  'ALERTS',
+                  'MGR_METADATA',
+                  'HEALTH_STATUS',
+                  'TOTAL_CAPACITY',
+                  'USED_CAPACITY',
+                  'POOLS',
+                  'OSDS',
+                  'CLUSTER_CAPACITY_UTILIZATION',
+                  'CLUSTER_IOPS_UTILIZATION',
+                  'CLUSTER_THROUGHPUT_UTILIZATION',
+                  'POOL_CAPACITY_UTILIZATION',
+                  'POOL_IOPS_UTILIZATION',
+                  'POOL_THROUGHPUT_UTILIZATION',
+                  'HOSTS',
+                  'CLUSTER_ALERTS'
+                ];
+                if (data.result.length) {
+                  if (validQueries.includes(queryName)) {
+                    queriesResults[queryName] = data.result;
+                  } else {
+                    queriesResults[queryName] = data.result.map(
+                      (result: { value: any }) => result.value
+                    );
+                  }
+                }
+              }
+              observer.next(queriesResults);
+              observer.complete();
+            },
+            (error: Error) => {
+              observer.error(error);
+            }
+          );
+        });
+      });
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.html
new file mode 100644
index 000000000000..04ecabfc8130
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.html
@@ -0,0 +1,10 @@
+<div class="row"
+     *ngIf="groupTitle">
+  <div class="info-group-title">
+    <span i18n>{{ groupTitle }}</span>
+  </div>
+</div>
+
+<div class="row">
+  <ng-content></ng-content>
+</div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.scss
new file mode 100644
index 000000000000..b30e1a71c47b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.scss
@@ -0,0 +1,4 @@
+.info-group-title {
+  font-size: 1.75rem;
+  margin: 0 0 0.5vw;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.spec.ts
new file mode 100644
index 000000000000..35c7955d4940
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.spec.ts
@@ -0,0 +1,22 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CardGroupComponent } from './card-group.component';
+
+describe('CardGroupComponent', () => {
+  let component: CardGroupComponent;
+  let fixture: ComponentFixture<CardGroupComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [CardGroupComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(CardGroupComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.ts
new file mode 100644
index 000000000000..c7de8caa5da8
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/card-group/card-group.component.ts
@@ -0,0 +1,11 @@
+import { Component, Input } from '@angular/core';
+
+@Component({
+  selector: 'cd-card-group',
+  templateUrl: './card-group.component.html',
+  styleUrls: ['./card-group.component.scss']
+})
+export class CardGroupComponent {
+  @Input()
+  groupTitle = '';
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index 5b533f1cddb3..867ef3b16d87 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -53,6 +53,7 @@ import { CardComponent } from './card/card.component';
 import { CardRowComponent } from './card-row/card-row.component';
 import { CodeBlockComponent } from './code-block/code-block.component';
 import { VerticalNavigationComponent } from './vertical-navigation/vertical-navigation.component';
+import { CardGroupComponent } from './card-group/card-group.component';
 
 @NgModule({
   imports: [
@@ -109,7 +110,8 @@ import { VerticalNavigationComponent } from './vertical-navigation/vertical-navi
     CardComponent,
     CardRowComponent,
     CodeBlockComponent,
-    VerticalNavigationComponent
+    VerticalNavigationComponent,
+    CardGroupComponent
   ],
   providers: [],
   exports: [
@@ -143,7 +145,8 @@ import { VerticalNavigationComponent } from './vertical-navigation/vertical-navi
     CardComponent,
     CardRowComponent,
     CodeBlockComponent,
-    VerticalNavigationComponent
+    VerticalNavigationComponent,
+    CardGroupComponent
   ]
 })
 export class ComponentsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
index 2d8aa22819dc..9a85d108a6b5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
@@ -16,3 +16,31 @@ export enum RgwPromqls {
   GET_BANDWIDTH = 'sum(rate(ceph_rgw_op_get_obj_bytes[1m]))',
   PUT_BANDWIDTH = 'sum(rate(ceph_rgw_op_put_obj_bytes[1m]))'
 }
+
+export enum MultiClusterPromqls {
+  ALERTS_COUNT = 'count(ALERTS{alertstate="firing"}) or vector(0)',
+  CLUSTER_COUNT = 'count(ceph_health_status) or vector(0)',
+  HEALTH_OK_COUNT = 'count(ceph_health_status==0) or vector(0)',
+  HEALTH_WARNING_COUNT = 'count(ceph_health_status==1) or vector(0)',
+  HEALTH_ERROR_COUNT = 'count(ceph_health_status==2) or vector(0)',
+  TOTAL_CLUSTERS_CAPACITY = 'sum(ceph_cluster_total_bytes) or vector(0)',
+  TOTAL_USED_CAPACITY = 'sum(ceph_cluster_by_class_total_used_bytes) or vector(0)',
+  HEALTH_STATUS = 'ceph_health_status',
+  MGR_METADATA = 'ceph_mgr_metadata',
+  TOTAL_CAPACITY = 'ceph_cluster_total_bytes',
+  USED_CAPACITY = 'ceph_cluster_total_used_bytes',
+  POOLS = 'count by (cluster) (ceph_pool_metadata) or vector(0)',
+  OSDS = 'count by (cluster) (ceph_osd_metadata) or vector(0)',
+  CRITICAL_ALERTS_COUNT = 'count(ALERTS{alertstate="firing",severity="critical"}) or vector(0)',
+  WARNING_ALERTS_COUNT = 'count(ALERTS{alertstate="firing",severity="warning"}) or vector(0)',
+  ALERTS = 'ALERTS{alertstate="firing"}',
+  HOSTS = 'sum by (hostname, cluster) (group by (hostname, cluster) (ceph_osd_metadata)) or vector(0)',
+  TOTAL_HOSTS = 'count by (cluster) (ceph_osd_metadata) or vector(0)',
+  CLUSTER_ALERTS = 'count by (cluster) (ALERTS{alertstate="firing"}) or vector(0)',
+  CLUSTER_CAPACITY_UTILIZATION = 'topk(2, ceph_cluster_total_used_bytes)',
+  CLUSTER_IOPS_UTILIZATION = 'topk(2, sum by (cluster) (rate(ceph_pool_wr[1m])) + sum by (cluster) (rate(ceph_pool_rd[1m])) )',
+  CLUSTER_THROUGHPUT_UTILIZATION = 'topk(2, sum by (cluster) (rate(ceph_pool_wr_bytes[1m])) + sum by (cluster) (rate(ceph_pool_rd_bytes[1m])) )',
+  POOL_CAPACITY_UTILIZATION = 'topk(2, ceph_pool_bytes_used/ceph_pool_max_avail * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata)',
+  POOL_IOPS_UTILIZATION = 'topk(2, (rate(ceph_pool_rd[1m]) + rate(ceph_pool_wr[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )',
+  POOL_THROUGHPUT_UTILIZATION = 'topk(2, (irate(ceph_pool_rd_bytes[1m]) + irate(ceph_pool_wr_bytes[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )'
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts
index ce4e02603f83..329aefb592eb 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/multi-cluster.ts
@@ -5,4 +5,6 @@ export interface MultiCluster {
   token: string;
   cluster_alias: string;
   cluster_connection_status: number;
+  ssl_verify: boolean;
+  ssl_certificate: string;
 }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 4e58517a9636..21f5ef86000a 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -6974,6 +6974,13 @@ paths:
                   type: string
                 password:
                   type: string
+                prometheus_api_url:
+                  type: string
+                ssl_certificate:
+                  type: string
+                ssl_verify:
+                  default: false
+                  type: boolean
                 token:
                   type: string
                 url:
@@ -6983,6 +6990,7 @@ paths:
               required:
               - url
               - cluster_alias
+              - username
               type: object
       responses:
         '201':
@@ -7148,6 +7156,11 @@ paths:
               properties:
                 password:
                   type: string
+                ssl_certificate:
+                  type: string
+                ssl_verify:
+                  default: false
+                  type: boolean
                 token:
                   type: string
                 url:
@@ -7219,34 +7232,38 @@ paths:
       tags:
       - Multi-cluster
   /api/multi-cluster/verify_connection:
-    get:
-      parameters:
-      - allowEmptyValue: true
-        in: query
-        name: url
-        schema:
-          type: string
-      - allowEmptyValue: true
-        in: query
-        name: username
-        schema:
-          type: string
-      - allowEmptyValue: true
-        in: query
-        name: password
-        schema:
-          type: string
-      - allowEmptyValue: true
-        in: query
-        name: token
-        schema:
-          type: string
+    post:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                password:
+                  type: string
+                ssl_certificate:
+                  type: string
+                ssl_verify:
+                  default: false
+                  type: boolean
+                token:
+                  type: string
+                url:
+                  type: string
+                username:
+                  type: string
+              type: object
       responses:
-        '200':
+        '201':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: OK
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
         '400':
           description: Operation exception. Please check the response body for details.
         '401':
@@ -10017,6 +10034,28 @@ paths:
       - jwt: []
       tags:
       - PrometheusNotifications
+  /api/prometheus/prometheus_query_data:
+    get:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      tags:
+      - Prometheus
   /api/prometheus/rules:
     get:
       parameters: []

From 6fb4c5d0c0694c9cfdbf0ed729d9f95886a0673f Mon Sep 17 00:00:00 2001
From: Mark Kogan <mkogan@redhat.com>
Date: Tue, 5 Mar 2024 16:48:56 +0200
Subject: [PATCH 2145/2492] rgw/beast: enablment of SSL session-id reuse
 speedup mechanism

Enable the OpenSSL session-id reuse acceleration mechanism that is described in:

https://www.openssl.org/docs/man1.0.2/man3/SSL_CTX_set_session_id_context.html
SSL_CTX_set_session_id_context, SSL_set_session_id_context - set context
within which session can be reused (server side only)

Fixes: https://tracker.ceph.com/issues/64719

Signed-off-by: Mark Kogan <mkogan@redhat.com>
---
 src/rgw/rgw_asio_frontend.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc
index 42ad12061488..49b69614b0ab 100644
--- a/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@ -1042,10 +1042,10 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec)
         handle_connection(context, env, stream, timeout, header_limit,
                           conn->buffer, true, pause_mutex, scheduler.get(),
                           uri_prefix, ec, yield);
-        if (!ec) {
-          // ssl shutdown (ignoring errors)
-          stream.async_shutdown(yield[ec]);
-        }
+
+        // ssl shutdown (ignoring errors)
+        stream.async_shutdown(yield[ec]);
+        
         conn->socket.shutdown(tcp::socket::shutdown_both, ec);
       }, make_stack_allocator());
   } else {

From c4a147d079cb9f268340e5f79ec5350a94296658 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Mon, 27 Nov 2023 15:51:02 -0500
Subject: [PATCH 2146/2492] rgw/multisite-notification: Add support to
 replicate topic creation  in multisite config

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/rgw/CMakeLists.txt                 |   1 +
 src/rgw/driver/daos/rgw_sal_daos.h     |   3 +
 src/rgw/driver/motr/rgw_sal_motr.h     |   3 +
 src/rgw/driver/rados/rgw_notify.cc     |   1 +
 src/rgw/driver/rados/rgw_rest_pubsub.h |   1 +
 src/rgw/driver/rados/rgw_sal_rados.cc  |  61 ++++++++
 src/rgw/driver/rados/rgw_sal_rados.h   |  18 +++
 src/rgw/driver/rados/rgw_service.cc    |  22 ++-
 src/rgw/driver/rados/rgw_service.h     |   5 +
 src/rgw/driver/rados/rgw_sync.cc       |   1 +
 src/rgw/driver/rados/rgw_zone.h        |  11 +-
 src/rgw/rgw_admin.cc                   |  28 +++-
 src/rgw/rgw_pubsub.cc                  | 140 ++++++++++++++++-
 src/rgw/rgw_pubsub.h                   |  25 ++-
 src/rgw/rgw_rest_pubsub.cc             |  88 +++++++++--
 src/rgw/rgw_rest_s3.cc                 |  12 +-
 src/rgw/rgw_sal.h                      |  22 ++-
 src/rgw/rgw_sal_dbstore.h              |   3 +
 src/rgw/rgw_sal_filter.h               |  24 +++
 src/rgw/rgw_sal_store.h                |  21 +++
 src/rgw/rgw_zone.cc                    |  11 +-
 src/rgw/rgw_zone_features.h            |   9 +-
 src/rgw/services/svc_topic_rados.cc    | 207 +++++++++++++++++++++++++
 src/rgw/services/svc_topic_rados.h     |  96 ++++++++++++
 24 files changed, 773 insertions(+), 40 deletions(-)
 create mode 100644 src/rgw/services/svc_topic_rados.cc
 create mode 100644 src/rgw/services/svc_topic_rados.h

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 3c2f1423808b..2987b70b3826 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -53,6 +53,7 @@ set(librgw_common_srcs
   services/svc_sys_obj_cache.cc
   services/svc_sys_obj_core.cc
   services/svc_tier_rados.cc
+  services/svc_topic_rados.cc
   services/svc_user.cc
   services/svc_user_rados.cc
   services/svc_zone.cc
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index c5cfefc222d1..479f5b84a5e3 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -430,6 +430,9 @@ class DaosZoneGroup : public StoreZoneGroup {
     return std::make_unique<DaosZoneGroup>(store, group);
   }
   const RGWZoneGroup& get_group() { return group; }
+  virtual bool supports_feature(std::string_view feature) const override {
+    return group.supports(feature);
+  }
 };
 
 class DaosZone : public StoreZone {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 3cc3b37fa9ad..9ff14a58307d 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -474,6 +474,9 @@ class MotrZoneGroup : public StoreZoneGroup {
   virtual std::unique_ptr<ZoneGroup> clone() override {
     return std::make_unique<MotrZoneGroup>(store, group);
   }
+  virtual bool supports_feature(std::string_view feature) const override {
+    return group.supports(feature);
+  }
   friend class MotrZone;
 };
 
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 9baed182ed94..4a2cf8271651 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -14,6 +14,7 @@
 #include "rgw_sal_rados.h"
 #include "rgw_pubsub.h"
 #include "rgw_pubsub_push.h"
+#include "rgw_zone_features.h"
 #include "rgw_perf_counters.h"
 #include "common/dout.h"
 #include <chrono>
diff --git a/src/rgw/driver/rados/rgw_rest_pubsub.h b/src/rgw/driver/rados/rgw_rest_pubsub.h
index 27bde7a95d5a..8b37992b9188 100644
--- a/src/rgw/driver/rados/rgw_rest_pubsub.h
+++ b/src/rgw/driver/rados/rgw_rest_pubsub.h
@@ -34,5 +34,6 @@ class RGWHandler_REST_PSTopic_AWS : public RGWHandler_REST {
   int postauth_init(optional_yield) override { return 0; }
   int authorize(const DoutPrefixProvider* dpp, optional_yield y) override;
   static bool action_exists(const req_state* s);
+  static bool action_exists(const req_info& info);
 };
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index e04e25842e2b..608800c7e573 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -62,6 +62,7 @@
 #include "services/svc_role_rados.h"
 #include "services/svc_user.h"
 #include "services/svc_sys_obj_cache.h"
+#include "services/svc_topic_rados.h"
 #include "cls/rgw/cls_rgw_client.h"
 
 #include "rgw_pubsub.h"
@@ -1111,6 +1112,66 @@ int RadosStore::remove_topics(const std::string& tenant, RGWObjVersionTracker* o
       objv_tracker, y);
 }
 
+int RadosStore::read_topic_v2(const std::string& topic_name,
+                              const std::string& tenant,
+                              rgw_pubsub_topic& topic,
+                              RGWObjVersionTracker* objv_tracker,
+                              optional_yield y,
+                              const DoutPrefixProvider* dpp) {
+  bufferlist bl;
+  auto mtime = ceph::real_clock::zero();
+  RGWSI_MBSObj_GetParams params(&bl, nullptr, &mtime);
+  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
+      svc()->topic->svc.meta_be->alloc_ctx());
+  ctx->init(svc()->topic->get_be_handler());
+  const int ret = svc()->topic->svc.meta_be->get(
+      ctx.get(), get_topic_key(topic_name, tenant), params, objv_tracker, y,
+      dpp);
+  if (ret < 0) {
+    return ret;
+  }
+
+  auto iter = bl.cbegin();
+  try {
+    decode(topic, iter);
+  } catch (buffer::error& err) {
+    ldpp_dout(dpp, 20) << " failed to decode topic: " << topic_name
+                       << ". error: " << err.what() << dendl;
+    return -EIO;
+  }
+  return 0;
+}
+
+int RadosStore::write_topic_v2(const rgw_pubsub_topic& topic,
+                               RGWObjVersionTracker* objv_tracker,
+                               optional_yield y,
+                               const DoutPrefixProvider* dpp) {
+  bufferlist bl;
+  encode(topic, bl);
+  RGWSI_MBSObj_PutParams params(bl, nullptr, ceph::real_clock::zero(),
+                                /*exclusive*/ false);
+  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
+      svc()->topic->svc.meta_be->alloc_ctx());
+  ctx->init(svc()->topic->get_be_handler());
+  return svc()->topic->svc.meta_be->put(
+      ctx.get(), get_topic_key(topic.name, topic.user.tenant), params,
+      objv_tracker, y, dpp);
+}
+
+int RadosStore::remove_topic_v2(const std::string& topic_name,
+                                const std::string& tenant,
+                                RGWObjVersionTracker* objv_tracker,
+                                optional_yield y,
+                                const DoutPrefixProvider* dpp) {
+  RGWSI_MBSObj_RemoveParams params;
+  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
+      svc()->topic->svc.meta_be->alloc_ctx());
+  ctx->init(svc()->topic->get_be_handler());
+  return svc()->topic->svc.meta_be->remove(ctx.get(),
+                                           get_topic_key(topic_name, tenant),
+                                           params, objv_tracker, y, dpp);
+}
+
 int RadosStore::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y)
 {
   return rados->delete_raw_obj(dpp, obj, y);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 39a554f15c95..33db603d785a 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -85,6 +85,9 @@ class RadosZoneGroup : public StoreZoneGroup {
   virtual std::unique_ptr<ZoneGroup> clone() override {
     return std::make_unique<RadosZoneGroup>(store, group);
   }
+  virtual bool supports_feature(std::string_view feature) const override {
+    return group.supports(feature);
+  }
   const RGWZoneGroup& get_group() const { return group; }
 };
 
@@ -166,6 +169,21 @@ class RadosStore : public StoreDriver {
 	optional_yield y, const DoutPrefixProvider *dpp) override;
     int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) override;
+    int read_topic_v2(const std::string& topic_name,
+                      const std::string& tenant,
+                      rgw_pubsub_topic& topic,
+                      RGWObjVersionTracker* objv_tracker,
+                      optional_yield y,
+                      const DoutPrefixProvider* dpp) override;
+    int write_topic_v2(const rgw_pubsub_topic& topic,
+                       RGWObjVersionTracker* objv_tracker,
+                       optional_yield y,
+                       const DoutPrefixProvider* dpp) override;
+    int remove_topic_v2(const std::string& topic_name,
+                        const std::string& tenant,
+                        RGWObjVersionTracker* objv_tracker,
+                        optional_yield y,
+                        const DoutPrefixProvider* dpp) override;
     virtual RGWLC* get_rgwlc(void) override { return rados->get_lc(); }
     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return rados->get_cr_registry(); }
 
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 4be0738bae2b..1f05495fb3d4 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -26,6 +26,7 @@
 #include "services/svc_sys_obj_core.h"
 #include "services/svc_user_rados.h"
 #include "services/svc_role_rados.h"
+#include "services/svc_topic_rados.h"
 
 #include "common/errno.h"
 
@@ -37,6 +38,7 @@
 #include "rgw_sal_rados.h"
 #include "rgw_user.h"
 #include "rgw_role.h"
+#include "rgw_pubsub.h"
 
 #define dout_subsys ceph_subsys_rgw
 
@@ -80,6 +82,7 @@ int RGWServices_Def::init(CephContext *cct,
   role_rados = std::make_unique<RGWSI_Role_RADOS>(cct);
   async_processor = std::make_unique<RGWAsyncRadosProcessor>(
     cct, cct->_conf->rgw_num_async_rados_threads);
+  topic_rados = std::make_unique<RGWSI_Topic_RADOS>(cct);
 
   if (have_cache) {
     sysobj_cache = std::make_unique<RGWSI_SysObj_Cache>(dpp, cct);
@@ -124,7 +127,7 @@ int RGWServices_Def::init(CephContext *cct,
   user_rados->init(driver->getRados()->get_rados_handle(), zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
   role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
-
+  topic_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
   can_shutdown = true;
 
   int r = finisher->start(y, dpp);
@@ -255,7 +258,12 @@ int RGWServices_Def::init(CephContext *cct,
       ldout(cct, 0) << "ERROR: failed to start role_rados service (" << cpp_strerror(-r) << dendl;
       return r;
     }
-
+    r = topic_rados->start(y, dpp);
+    if (r < 0) {
+      ldout(cct, 0) << "ERROR: failed to start topic_rados service ("
+                    << cpp_strerror(-r) << dendl;
+      return r;
+    }
   }
 
   /* cache or core services will be started by sysobj */
@@ -273,6 +281,7 @@ void RGWServices_Def::shutdown()
     return;
   }
 
+  topic_rados->shutdown();
   role_rados->shutdown();
   datalog_rados.reset();
   user_rados->shutdown();
@@ -342,6 +351,7 @@ int RGWServices::do_init(CephContext *_cct, rgw::sal::RadosStore* driver, bool h
   user = _svc.user_rados.get();
   role = _svc.role_rados.get();
   async_processor = _svc.async_processor.get();
+  topic = _svc.topic_rados.get();
 
   return 0;
 }
@@ -390,6 +400,7 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, const DoutPrefix
 
   meta.otp.reset(RGWOTPMetaHandlerAllocator::alloc());
   meta.role = std::make_unique<rgw::sal::RGWRoleMetadataHandler>(driver, svc.role);
+  meta.topic = std::make_unique<RGWTopicMetadataHandler>(driver, svc.topic);
 
   user.reset(new RGWUserCtl(svc.zone, svc.user, (RGWUserMetadataHandler *)meta.user.get()));
   bucket.reset(new RGWBucketCtl(svc.zone,
@@ -436,6 +447,7 @@ int RGWCtl::init(RGWServices *_svc, rgw::sal::Driver* driver, const DoutPrefixPr
   meta.bucket_instance = _ctl.meta.bucket_instance.get();
   meta.otp = _ctl.meta.otp.get();
   meta.role = _ctl.meta.role.get();
+  meta.topic = _ctl.meta.topic.get();
 
   user = _ctl.user.get();
   bucket = _ctl.bucket.get();
@@ -470,6 +482,12 @@ int RGWCtl::init(RGWServices *_svc, rgw::sal::Driver* driver, const DoutPrefixPr
     ldout(cct, 0) << "ERROR: failed to start init otp ctl (" << cpp_strerror(-r) << dendl;
     return r;
   }
+  r = meta.topic->attach(meta.mgr);
+  if (r < 0) {
+    ldout(cct, 0) << "ERROR: failed to start init topic ctl ("
+                  << cpp_strerror(-r) << dendl;
+    return r;
+  }
   return 0;
 }
 
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 9996b42e2514..08873e6058e5 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -78,6 +78,7 @@ class RGWSI_User_RADOS;
 class RGWDataChangesLog;
 class RGWSI_Role_RADOS;
 class RGWAsyncRadosProcessor;
+class RGWSI_Topic_RADOS;
 
 struct RGWServices_Def
 {
@@ -109,6 +110,7 @@ struct RGWServices_Def
   std::unique_ptr<RGWSI_Role_RADOS> role_rados;
   std::unique_ptr<RGWAsyncRadosProcessor> async_processor;
 
+  std::unique_ptr<RGWSI_Topic_RADOS> topic_rados;
   RGWServices_Def();
   ~RGWServices_Def();
 
@@ -153,6 +155,7 @@ struct RGWServices
   RGWSI_User *user{nullptr};
   RGWSI_Role_RADOS *role{nullptr};
   RGWAsyncRadosProcessor* async_processor;
+  RGWSI_Topic_RADOS* topic{nullptr};
 
   int do_init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
 	      bool raw_storage, bool run_sync, optional_yield y,
@@ -187,6 +190,7 @@ struct RGWCtlDef {
     std::unique_ptr<RGWMetadataHandler> user;
     std::unique_ptr<RGWMetadataHandler> otp;
     std::unique_ptr<RGWMetadataHandler> role;
+    std::unique_ptr<RGWMetadataHandler> topic;
 
     _meta();
     ~_meta();
@@ -216,6 +220,7 @@ struct RGWCtl {
     RGWMetadataHandler *user{nullptr};
     RGWMetadataHandler *otp{nullptr};
     RGWMetadataHandler *role{nullptr};
+    RGWMetadataHandler* topic{nullptr};
   } meta;
 
   RGWUserCtl *user{nullptr};
diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc
index c5ea9f99ec5f..a9ea2ecf5491 100644
--- a/src/rgw/driver/rados/rgw_sync.cc
+++ b/src/rgw/driver/rados/rgw_sync.cc
@@ -886,6 +886,7 @@ class RGWFetchAllMetaCR : public RGWCoroutine {
     append_section_from_set(all_sections, "bucket.instance");
     append_section_from_set(all_sections, "bucket");
     append_section_from_set(all_sections, "roles");
+    append_section_from_set(all_sections, "topic");
 
     std::move(all_sections.begin(), all_sections.end(),
               std::back_inserter(sections));
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index f0dccdbc4e94..2eb3e725253f 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -114,6 +114,7 @@ struct RGWZoneParams : RGWSystemMetaObj {
   rgw_pool otp_pool;
   rgw_pool oidc_pool;
   rgw_pool notif_pool;
+  rgw_pool topics_pool;
 
   RGWAccessKey system_key;
 
@@ -150,7 +151,7 @@ struct RGWZoneParams : RGWSystemMetaObj {
   const std::string& get_compression_type(const rgw_placement_rule& placement_rule) const;
   
   void encode(bufferlist& bl) const override {
-    ENCODE_START(14, 1, bl);
+    ENCODE_START(15, 1, bl);
     encode(domain_root, bl);
     encode(control_pool, bl);
     encode(gc_pool, bl);
@@ -176,11 +177,12 @@ struct RGWZoneParams : RGWSystemMetaObj {
     encode(tier_config, bl);
     encode(oidc_pool, bl);
     encode(notif_pool, bl);
+    encode(topics_pool, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) override {
-    DECODE_START(14, bl);
+    DECODE_START(15, bl);
     decode(domain_root, bl);
     decode(control_pool, bl);
     decode(gc_pool, bl);
@@ -249,6 +251,11 @@ struct RGWZoneParams : RGWSystemMetaObj {
     } else {
       notif_pool = log_pool.name + ":notif";
     }
+    if (struct_v >= 15) {
+      decode(topics_pool, bl);
+    } else {
+      topics_pool = name + ".rgw.meta:topics";
+    }
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 0be8dacf862b..ced7b15120d6 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -10634,10 +10634,15 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::PUBSUB_TOPIC_LIST) {
-    RGWPubSub ps(driver, tenant);
-
+    auto site = std::make_unique<rgw::SiteConfig>();
+    ret = site->load(dpp(), null_yield, cfgstore.get());
+    if (ret < 0) {
+      std::cerr << "Unable to initialize site config." << std::endl;
+      exit(1);
+    }
+    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
     rgw_pubsub_topics result;
-    int ret = ps.get_topics(dpp(), result, null_yield);
+    ret = ps.get_topics(dpp(), result, null_yield);
     if (ret < 0 && ret != -ENOENT) {
       cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -10661,8 +10666,13 @@ int main(int argc, const char **argv)
       cerr << "ERROR: topic name was not provided (via --topic)" << std::endl;
       return EINVAL;
     }
-
-    RGWPubSub ps(driver, tenant);
+    auto site = std::make_unique<rgw::SiteConfig>();
+    ret = site->load(dpp(), null_yield, cfgstore.get());
+    if (ret < 0) {
+      std::cerr << "Unable to initialize site config." << std::endl;
+      exit(1);
+    }
+    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
 
     rgw_pubsub_topic topic;
     ret = ps.get_topic(dpp(), topic_name, topic, null_yield);
@@ -10723,7 +10733,13 @@ int main(int argc, const char **argv)
       return -ret;
     }
 
-    RGWPubSub ps(driver, tenant);
+    auto site = std::make_unique<rgw::SiteConfig>();
+    ret = site->load(dpp(), null_yield, cfgstore.get());
+    if (ret < 0) {
+      std::cerr << "Unable to initialize site config." << std::endl;
+      exit(1);
+    }
+    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
 
     ret = ps.remove_topic(dpp(), topic_name, null_yield);
     if (ret < 0) {
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 7031c2363f0b..c563b959863a 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -9,6 +9,7 @@
 #include "rgw_xml.h"
 #include "rgw_arn.h"
 #include "rgw_pubsub_push.h"
+#include "common/errno.h"
 #include <regex>
 #include <algorithm>
 
@@ -371,6 +372,15 @@ void rgw_pubsub_topic::dump_xml_as_attributes(Formatter *f) const
   f->close_section(); // Attributes
 }
 
+void rgw_pubsub_topic::decode_json(JSONObj* f) {
+  JSONDecoder::decode_json("user", user, f);
+  JSONDecoder::decode_json("name", name, f);
+  JSONDecoder::decode_json("dest", dest, f);
+  JSONDecoder::decode_json("arn", arn, f);
+  JSONDecoder::decode_json("opaqueData", opaque_data, f);
+  JSONDecoder::decode_json("policy", policy_text, f);
+}
+
 void encode_json(const char *name, const rgw::notify::EventTypeList& l, Formatter *f)
 {
   f->open_array_section(name);
@@ -462,13 +472,74 @@ std::string rgw_pubsub_dest::to_json_str() const
   return ss.str();
 }
 
+void rgw_pubsub_dest::decode_json(JSONObj* f) {
+  using rgw::notify::DEFAULT_CONFIG;
+  using rgw::notify::DEFAULT_GLOBAL_VALUE;
+  JSONDecoder::decode_json("push_endpoint", push_endpoint, f);
+  JSONDecoder::decode_json("push_endpoint_args", push_endpoint_args, f);
+  JSONDecoder::decode_json("push_endpoint_topic", arn_topic, f);
+  JSONDecoder::decode_json("stored_secret", stored_secret, f);
+  JSONDecoder::decode_json("persistent", persistent, f);
+  std::string ttl;
+  JSONDecoder::decode_json("time_to_live", ttl, f);
+  time_to_live = ttl == DEFAULT_CONFIG ? DEFAULT_GLOBAL_VALUE : std::stoul(ttl);
+
+  std::string max_retry;
+  JSONDecoder::decode_json("max_retries", max_retry, f);
+  max_retries = max_retry == DEFAULT_CONFIG ? DEFAULT_GLOBAL_VALUE
+                                            : std::stoul(max_retry);
+
+  std::string sleep_dur;
+  JSONDecoder::decode_json("retry_sleep_duration", sleep_dur, f);
+  retry_sleep_duration = sleep_dur == DEFAULT_CONFIG ? DEFAULT_GLOBAL_VALUE
+                                                     : std::stoul(sleep_dur);
+}
+
 RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver, const std::string& _tenant)
   : driver(_driver), tenant(_tenant)
 {}
 
-int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result, 
+RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver,
+                     const std::string& _tenant,
+                     const std::map<std::string, RGWZoneGroup>* _zonegroups)
+    : driver(_driver), tenant(_tenant), zonegroups(_zonegroups) {
+  use_notification_v2 = do_all_zonegroups_support_notification_v2(*zonegroups);
+}
+
+int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result,
     RGWObjVersionTracker *objv_tracker, optional_yield y) const
 {
+  if (use_notification_v2) {
+    void* handle = NULL;
+    auto ret =
+        driver->meta_list_keys_init(dpp, "topic", std::string(), &handle);
+    if (ret < 0) {
+      return ret;
+    }
+    bool truncated;
+    int max = 1000;
+    do {
+      std::list<std::string> topics;
+      ret = driver->meta_list_keys_next(dpp, handle, max, topics, &truncated);
+      if (ret < 0) {
+            ldpp_dout(dpp, 1)
+                << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl;
+            break;
+      }
+      for (auto& topic_name : topics) {
+            rgw_pubsub_topic topic;
+            int ret = get_topic(dpp, topic_name, topic, y);
+            if (ret < 0) {
+              ldpp_dout(dpp, 1) << "ERROR: failed to read topic '" << topic_name
+                                << "' info: ret=" << ret << dendl;
+              continue;
+            }
+            result.topics[topic_name] = std::move(topic);
+      }
+    } while (truncated);
+    driver->meta_list_keys_complete(handle);
+    return ret;
+  }
   const int ret = driver->read_topics(tenant, result, objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 10) << "WARNING: failed to read topics info: ret=" << ret << dendl;
@@ -514,6 +585,14 @@ int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pub
 
 int RGWPubSub::get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic& result, optional_yield y) const
 {
+  if (use_notification_v2) {
+    const int ret = driver->read_topic(name, tenant, result, nullptr, y, dpp);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1) << "failed to read topic info for name: " << name
+                        << " tenant: " << tenant << ", ret=" << ret << dendl;
+    }
+    return ret;
+  }
   rgw_pubsub_topics topics;
   const int ret = read_topics(dpp, topics, nullptr, y);
   if (ret < 0) {
@@ -563,6 +642,15 @@ int RGWPubSub::Bucket::get_notification_by_id(const DoutPrefixProvider *dpp, con
 int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
     const rgw::notify::EventTypeList& events, optional_yield y) const {
   return create_notification(dpp, topic_name, events, std::nullopt, "", y);
+    }
+bool do_all_zonegroups_support_notification_v2(
+    std::map<std::string, RGWZoneGroup> zonegroups) {
+  for (const auto& [_, zonegroup] : zonegroups) {
+    if (!zonegroup.supports(rgw::zone_features::notification_v2)) {
+      return false;
+    }
+  }
+  return true;
 }
 
 int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
@@ -694,6 +782,19 @@ int RGWPubSub::Bucket::remove_notifications(const DoutPrefixProvider *dpp, optio
   return 0;
 }
 
+int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
+                            const rgw_pubsub_topic& topic,
+                            optional_yield y) const {
+  RGWObjVersionTracker objv_tracker;
+  const auto ret = driver->write_topic(topic, &objv_tracker, y, dpp);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret
+                      << dendl;
+  }
+
+  return ret;
+}
+
 int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const std::string& name,
                             const rgw_pubsub_dest& dest, const std::string& arn,
@@ -701,6 +802,16 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const rgw_user& user,
                             const std::string& policy_text,
                             optional_yield y) const {
+  if (use_notification_v2) {
+    rgw_pubsub_topic new_topic;
+    new_topic.user = user;
+    new_topic.name = name;
+    new_topic.dest = dest;
+    new_topic.arn = arn;
+    new_topic.opaque_data = opaque_data;
+    new_topic.policy_text = policy_text;
+    return create_topic(dpp, new_topic, y);
+  }
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topics topics;
 
@@ -728,8 +839,34 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
   return 0;
 }
 
+int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
+                               const std::string& name,
+                               optional_yield y) const {
+  RGWObjVersionTracker objv_tracker;
+  rgw_pubsub_topic topic;
+  int ret = get_topic(dpp, name, topic, y);
+  if (ret < 0 && ret != -ENOENT) {
+    return ret;
+  } else if (ret == -ENOENT) {
+    // its not an error if no topics exist, just a no-op
+    ldpp_dout(dpp, 10) << "WARNING: topic name:" << name
+                       << " does not exist, deletion is a no-op: ret=" << ret
+                       << dendl;
+    return 0;
+  }
+  ret = driver->remove_topic(name, tenant, &objv_tracker, y, dpp);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to remove topic info: ret=" << ret
+                      << dendl;
+  }
+  return ret;
+}
+
 int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const
 {
+  if (use_notification_v2) {
+    return remove_topic_v2(dpp, name, y);
+  }
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topics topics;
 
@@ -753,4 +890,3 @@ int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& na
 
   return 0;
 }
-
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index ddc72f99b076..4afc101c63cc 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -389,6 +389,7 @@ struct rgw_pubsub_dest {
   void dump(Formatter *f) const;
   void dump_xml(Formatter *f) const;
   std::string to_json_str() const;
+  void decode_json(JSONObj* obj);
 };
 WRITE_CLASS_ENCODER(rgw_pubsub_dest)
 
@@ -435,6 +436,7 @@ struct rgw_pubsub_topic {
   void dump(Formatter *f) const;
   void dump_xml(Formatter *f) const;
   void dump_xml_as_attributes(Formatter *f) const;
+  void decode_json(JSONObj* obj);
 
   bool operator<(const rgw_pubsub_topic& t) const {
     return to_str().compare(t.to_str());
@@ -558,6 +560,8 @@ class RGWPubSub
 
   rgw::sal::Driver* const driver;
   const std::string tenant;
+  const std::map<std::string, RGWZoneGroup>* zonegroups = nullptr;
+  bool use_notification_v2 = false;
 
   int read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result, 
       RGWObjVersionTracker* objv_tracker, optional_yield y) const;
@@ -567,6 +571,10 @@ class RGWPubSub
 public:
   RGWPubSub(rgw::sal::Driver* _driver, const std::string& tenant);
 
+ RGWPubSub(rgw::sal::Driver* _driver,
+           const std::string& _tenant,
+           const std::map<std::string, RGWZoneGroup>* zonegroups);
+
   class Bucket {
     friend class RGWPubSub;
     const RGWPubSub& ps;
@@ -639,6 +647,18 @@ class RGWPubSub
   // if the topic does not exists it is a no-op (considered success)
   // return 0 on success, error code otherwise
   int remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const;
+  // remove a topic according to its name
+  // if the topic does not exists it is a no-op (considered success)
+  // return 0 on success, error code otherwise
+  int remove_topic_v2(const DoutPrefixProvider* dpp,
+                      const std::string& name,
+                      optional_yield y) const;
+  // create a topic with a name only
+  // if the topic already exists it is a no-op (considered success)
+  // return 0 on success, error code otherwise
+  int create_topic(const DoutPrefixProvider* dpp,
+                   const rgw_pubsub_topic& topic,
+                   optional_yield y) const;
 };
 
 namespace rgw::notify {
@@ -648,4 +668,7 @@ namespace rgw::notify {
   constexpr uint32_t DEFAULT_GLOBAL_VALUE = UINT32_MAX;
   // Used in case the topic is using the default global value for dumping in a formatter
   constexpr static const std::string_view DEFAULT_CONFIG{"None"};
-}
\ No newline at end of file
+}
+
+bool do_all_zonegroups_support_notification_v2(
+    std::map<std::string, RGWZoneGroup> zonegroups);
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 191f535d82bd..7396fcfddd41 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -17,6 +17,7 @@
 #include "services/svc_zone.h"
 #include "common/dout.h"
 #include "rgw_url.h"
+#include "rgw_process_env.h"
 
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_rgw
@@ -138,21 +139,13 @@ class RGWPSCreateTopicOp : public RGWOp {
       return -EINVAL;
     }
 
-    // Remove the args that are parsed, so the push_endpoint_args only contains
-    // necessary one's.
     opaque_data = s->info.args.get("OpaqueData");
-    s->info.args.remove("OpaqueData");
 
     dest.push_endpoint = s->info.args.get("push-endpoint");
-    s->info.args.remove("push-endpoint");
     s->info.args.get_bool("persistent", &dest.persistent, false);
-    s->info.args.remove("persistent");
     s->info.args.get_int("time_to_live", reinterpret_cast<int *>(&dest.time_to_live), rgw::notify::DEFAULT_GLOBAL_VALUE);
-    s->info.args.remove("time_to_live");
     s->info.args.get_int("max_retries", reinterpret_cast<int *>(&dest.max_retries), rgw::notify::DEFAULT_GLOBAL_VALUE);
-    s->info.args.remove("max_retries");
     s->info.args.get_int("retry_sleep_duration", reinterpret_cast<int *>(&dest.retry_sleep_duration), rgw::notify::DEFAULT_GLOBAL_VALUE);
-    s->info.args.remove("retry_sleep_duration");
 
     if (!validate_and_update_endpoint_secret(dest, s->cct, *(s->info.env))) {
       return -EINVAL;
@@ -162,8 +155,19 @@ class RGWPSCreateTopicOp : public RGWOp {
     if (!policy_text.empty() && !get_policy_from_text(s, policy_text)) {
       return -ERR_MALFORMED_DOC;
     }
-    s->info.args.remove("Policy");
 
+    // Remove the args that are parsed, so the push_endpoint_args only contains
+    // necessary one's which is parsed after this if. but only if master zone,
+    // else we do not remove as request is forwarded to master.
+    if (driver->is_meta_master()) {
+      s->info.args.remove("OpaqueData");
+      s->info.args.remove("push-endpoint");
+      s->info.args.remove("persistent");
+      s->info.args.remove("time_to_live");
+      s->info.args.remove("max_retries");
+      s->info.args.remove("retry_sleep_duration");
+      s->info.args.remove("Policy");
+    }
     for (const auto& param : s->info.args.get_params()) {
       if (param.first == "Action" || param.first == "Name" || param.first == "PayloadHash") {
         continue;
@@ -193,7 +197,8 @@ class RGWPSCreateTopicOp : public RGWOp {
       return ret;
     }
 
-    const RGWPubSub ps(driver, s->owner.id.tenant);
+    const RGWPubSub ps(driver, s->owner.id.tenant,
+                       &s->penv.site->get_period()->get_map().zonegroups);
     rgw_pubsub_topic result;
     ret = ps.get_topic(this, topic_name, result, y);
     if (ret == -ENOENT) {
@@ -252,6 +257,18 @@ class RGWPSCreateTopicOp : public RGWOp {
 };
 
 void RGWPSCreateTopicOp::execute(optional_yield y) {
+  // master request will replicate the topic creation.
+  bufferlist indata;
+  if (!driver->is_meta_master()) {
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1)
+          << "CreateTopic forward_request_to_master returned ret = " << op_ret
+          << dendl;
+      return;
+    }
+  }
   if (!dest.push_endpoint.empty() && dest.persistent) {
     op_ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
     if (op_ret < 0) {
@@ -261,7 +278,8 @@ void RGWPSCreateTopicOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
                            s->owner.id, policy_text, y);
   if (op_ret < 0) {
@@ -316,7 +334,8 @@ class RGWPSListTopicsOp : public RGWOp {
 };
 
 void RGWPSListTopicsOp::execute(optional_yield y) {
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
   op_ret = ps.get_topics(this, result, y);
   // if there are no topics it is not considered an error
   op_ret = op_ret == -ENOENT ? 0 : op_ret;
@@ -403,7 +422,8 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -487,7 +507,8 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -615,7 +636,8 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
       return ret;
     }
     rgw_pubsub_topic result;
-    const RGWPubSub ps(driver, s->owner.id.tenant);
+    const RGWPubSub ps(driver, s->owner.id.tenant,
+                       &s->penv.site->get_period()->get_map().zonegroups);
     ret = ps.get_topic(this, topic_name, result, y);
     if (ret < 0) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name
@@ -664,6 +686,17 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
 };
 
 void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
+  if (!driver->is_meta_master()) {
+    bufferlist indata;
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1)
+          << "SetTopicAttributes forward_request_to_master returned ret = "
+          << op_ret << dendl;
+      return;
+    }
+  }
   if (!dest.push_endpoint.empty() && dest.persistent) {
     op_ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
     if (op_ret < 0) {
@@ -682,7 +715,8 @@ void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
                            topic_owner, policy_text, y);
   if (op_ret < 0) {
@@ -752,7 +786,20 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant);
+  if (!driver->is_meta_master()) {
+    bufferlist indata;
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1)
+          << "DeleteTopic forward_request_to_master returned ret = " << op_ret
+          << dendl;
+      return;
+    }
+  }
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
+
   rgw_pubsub_topic result;
   op_ret = ps.get_topic(this, topic_name, result, y);
   if (op_ret == 0) {
@@ -809,6 +856,13 @@ bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s)
   }
   return false;
 }
+bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_info& info) {
+  if (info.args.exists("Action")) {
+    const std::string action_name = info.args.get("Action");
+    return op_generators.contains(action_name);
+  }
+  return false;
+}
 
 RGWOp *RGWHandler_REST_PSTopic_AWS::op_post()
 {
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index bb43029f7766..186fb109c2fc 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5206,8 +5206,12 @@ void parse_post_action(const std::string& post_body, req_state* s)
       }
     }
   }
-  const auto payload_hash = rgw::auth::s3::calc_v4_payload_hash(post_body);
-  s->info.args.append("PayloadHash", payload_hash);
+  // PayloadHash is present if request is fwd from secondary site in multisite
+  // environment, so then do not calculate and append.
+  if (!s->info.args.exists("PayloadHash")) {
+    const auto payload_hash = rgw::auth::s3::calc_v4_payload_hash(post_body);
+    s->info.args.append("PayloadHash", payload_hash);
+  }
 }
 
 RGWHandler_REST* RGWRESTMgr_S3::get_handler(rgw::sal::Driver* driver,
@@ -5623,7 +5627,9 @@ AWSSignerV4::prepare(const DoutPrefixProvider *dpp,
 
   const char* exp_payload_hash = nullptr;
   string payload_hash;
-  if (is_non_s3_op) {
+  // if the request is related to topics (bucket notification), they are part of
+  // sns service and hence it's a no_s3_op,
+  if (is_non_s3_op || RGWHandler_REST_PSTopic_AWS::action_exists(info)) {
     //For non s3 ops, we need to calculate the payload hash
     payload_hash = info.args.get("PayloadHash");
     exp_payload_hash = payload_hash.c_str();
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index eb4d0c348fad..89ac23341d23 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -44,7 +44,7 @@ class RGWCompressionInfo;
 struct rgw_pubsub_topics;
 struct rgw_pubsub_bucket_topics;
 class RGWZonePlacementInfo;
-
+struct rgw_pubsub_topic;
 
 using RGWBucketListNameFilter = std::function<bool (const std::string&)>;
 
@@ -314,6 +314,24 @@ class Driver {
     /** Remove the topic config, optionally a specific version */
     virtual int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
         optional_yield y,const DoutPrefixProvider *dpp) = 0;
+    /** Read the topic config entry into data and (optionally) objv_tracker */
+    virtual int read_topic_v2(const std::string& topic_name,
+                              const std::string& tenant,
+                              rgw_pubsub_topic& topic,
+                              RGWObjVersionTracker* objv_tracker,
+                              optional_yield y,
+                              const DoutPrefixProvider* dpp) = 0;
+    /** Write topic info and (optionally) @a objv_tracker into the config */
+    virtual int write_topic_v2(const rgw_pubsub_topic& topic,
+                               RGWObjVersionTracker* objv_tracker,
+                               optional_yield y,
+                               const DoutPrefixProvider* dpp) = 0;
+    /** Remove the topic config, optionally a specific version */
+    virtual int remove_topic_v2(const std::string& topic_name,
+                                const std::string& tenant,
+                                RGWObjVersionTracker* objv_tracker,
+                                optional_yield y,
+                                const DoutPrefixProvider* dpp) = 0;
     /** Get access to the lifecycle management thread */
     virtual RGWLC* get_rgwlc(void) = 0;
     /** Get access to the coroutine registry.  Used to create new coroutine managers */
@@ -1449,6 +1467,8 @@ class ZoneGroup {
   virtual int list_zones(std::list<std::string>& zone_ids) = 0;
   /** Clone a copy of this zonegroup. */
   virtual std::unique_ptr<ZoneGroup> clone() = 0;
+  /** Determine if zonegroup |feature| is supported.*/
+  virtual bool supports_feature(std::string_view feature) const = 0;
 };
 
 /**
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 3c0c7c765198..140b28396626 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -259,6 +259,9 @@ class DBNotification : public StoreNotification {
       std::unique_ptr<RGWZoneGroup>zg = std::make_unique<RGWZoneGroup>(*group.get());
       return std::make_unique<DBZoneGroup>(store, std::move(zg));
     }
+    virtual bool supports_feature(std::string_view feature) const override {
+      return group->supports(feature);
+    }
   };
 
   class DBZone : public StoreZone {
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index b5c4c4dfc681..71991378e321 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -75,6 +75,9 @@ class FilterZoneGroup : public ZoneGroup {
     std::unique_ptr<ZoneGroup> nzg = next->clone();
     return std::make_unique<FilterZoneGroup>(std::move(nzg));
   }
+  virtual bool supports_feature(std::string_view feature) const override {
+    return next->supports_feature(feature);
+  }
 };
 
 class FilterZone : public Zone {
@@ -194,6 +197,27 @@ class FilterDriver : public Driver {
       optional_yield y, const DoutPrefixProvider *dpp) override {
     return next->remove_topics(tenant, objv_tracker, y, dpp);
   }
+  int read_topic_v2(const std::string& topic_name,
+                    const std::string& tenant,
+                    rgw_pubsub_topic& topic,
+                    RGWObjVersionTracker* objv_tracker,
+                    optional_yield y,
+                    const DoutPrefixProvider* dpp) override {
+    return next->read_topic_v2(topic_name, tenant, topic, objv_tracker, y, dpp);
+  }
+  int write_topic_v2(const rgw_pubsub_topic& topic,
+                     RGWObjVersionTracker* objv_tracker,
+                     optional_yield y,
+                     const DoutPrefixProvider* dpp) override {
+    return next->write_topic_v2(topic, objv_tracker, y, dpp);
+  }
+  int remove_topic_v2(const std::string& topic_name,
+                      const std::string& tenant,
+                      RGWObjVersionTracker* objv_tracker,
+                      optional_yield y,
+                      const DoutPrefixProvider* dpp) override {
+    return next->remove_topic_v2(topic_name, tenant, objv_tracker, y, dpp);
+  }
 
   virtual RGWLC* get_rgwlc(void) override;
   virtual RGWCoroutinesManagerRegistry* get_cr_registry() override;
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index 7c35258dd5dc..eda0f08ede6f 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -34,6 +34,27 @@ class StoreDriver : public Driver {
 	optional_yield y, const DoutPrefixProvider *dpp) override {return -ENOENT;}
     int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) override {return -ENOENT;}
+    int read_topic_v2(const std::string& topic_name,
+                      const std::string& tenant,
+                      rgw_pubsub_topic& topic,
+                      RGWObjVersionTracker* objv_tracker,
+                      optional_yield y,
+                      const DoutPrefixProvider* dpp) override {
+      return -EOPNOTSUPP;
+    }
+    int write_topic_v2(const rgw_pubsub_topic& topic,
+                       RGWObjVersionTracker* objv_tracker,
+                       optional_yield y,
+                       const DoutPrefixProvider* dpp) override {
+      return -ENOENT;
+    }
+    int remove_topic_v2(const std::string& topic_name,
+                        const std::string& tenant,
+                        RGWObjVersionTracker* objv_tracker,
+                        optional_yield y,
+                        const DoutPrefixProvider* dpp) override {
+      return -ENOENT;
+    }
 };
 
 class StoreUser : public User {
diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc
index aeb58e2f48fe..ed438dead930 100644
--- a/src/rgw/rgw_zone.cc
+++ b/src/rgw/rgw_zone.cc
@@ -296,12 +296,12 @@ void RGWZoneParams::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("user_swift_pool", user_swift_pool, obj);
   JSONDecoder::decode_json("user_uid_pool", user_uid_pool, obj);
   JSONDecoder::decode_json("otp_pool", otp_pool, obj);
+  JSONDecoder::decode_json("notif_pool", notif_pool, obj);
+  JSONDecoder::decode_json("topics_pool", topics_pool, obj);
   JSONDecoder::decode_json("system_key", system_key, obj);
   JSONDecoder::decode_json("placement_pools", placement_pools, obj);
   JSONDecoder::decode_json("tier_config", tier_config, obj);
   JSONDecoder::decode_json("realm_id", realm_id, obj);
-  JSONDecoder::decode_json("notif_pool", notif_pool, obj);
-
 }
 
 void RGWZoneParams::dump(Formatter *f) const
@@ -321,11 +321,12 @@ void RGWZoneParams::dump(Formatter *f) const
   encode_json("user_swift_pool", user_swift_pool, f);
   encode_json("user_uid_pool", user_uid_pool, f);
   encode_json("otp_pool", otp_pool, f);
+  encode_json("notif_pool", notif_pool, f);
+  encode_json("topics_pool", topics_pool, f);
   encode_json_plain("system_key", system_key, f);
   encode_json("placement_pools", placement_pools, f);
   encode_json("tier_config", tier_config, f);
   encode_json("realm_id", realm_id, f);
-  encode_json("notif_pool", notif_pool, f);
 }
 
 int RGWZoneParams::init(const DoutPrefixProvider *dpp, 
@@ -480,6 +481,7 @@ void add_zone_pools(const RGWZoneParams& info,
   pools.insert(info.reshard_pool);
   pools.insert(info.oidc_pool);
   pools.insert(info.notif_pool);
+  pools.insert(info.topics_pool);
 
   for (const auto& [pname, placement] : info.placement_pools) {
     pools.insert(placement.index_pool);
@@ -584,6 +586,7 @@ int RGWZoneParams::fix_pool_names(const DoutPrefixProvider *dpp, optional_yield
   otp_pool = fix_zone_pool_dup(pools, name, ".rgw.otp", otp_pool);
   oidc_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:oidc", oidc_pool);
   notif_pool = fix_zone_pool_dup(pools, name ,".rgw.log:notif", notif_pool);
+  topics_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:topics", topics_pool);
 
   for(auto& iter : placement_pools) {
     iter.second.index_pool = fix_zone_pool_dup(pools, name, "." + default_bucket_index_pool_suffix,
@@ -1245,6 +1248,8 @@ int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y,
   info.otp_pool = fix_zone_pool_dup(pools, info.name, ".rgw.otp", info.otp_pool);
   info.oidc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:oidc", info.oidc_pool);
   info.notif_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:notif", info.notif_pool);
+  info.topics_pool =
+      fix_zone_pool_dup(pools, info.name, ".rgw.meta:topics", info.topics_pool);
 
   for (auto& [pname, placement] : info.placement_pools) {
     placement.index_pool = fix_zone_pool_dup(pools, info.name, "." + default_bucket_index_pool_suffix, placement.index_pool);
diff --git a/src/rgw/rgw_zone_features.h b/src/rgw/rgw_zone_features.h
index 5e1a435d488e..600460735a84 100644
--- a/src/rgw/rgw_zone_features.h
+++ b/src/rgw/rgw_zone_features.h
@@ -15,11 +15,13 @@ namespace rgw::zone_features {
 // zone feature names
 inline constexpr std::string_view resharding = "resharding";
 inline constexpr std::string_view compress_encrypted = "compress-encrypted";
+inline constexpr std::string_view notification_v2 = "notification_v2";
 
 // static list of features supported by this release
 inline constexpr std::initializer_list<std::string_view> supported = {
-  resharding,
-  compress_encrypted,
+    resharding,
+    compress_encrypted,
+    notification_v2,
 };
 
 inline constexpr bool supports(std::string_view feature) {
@@ -33,7 +35,8 @@ inline constexpr bool supports(std::string_view feature) {
 
 // static list of features enabled by default on new zonegroups
 inline constexpr std::initializer_list<std::string_view> enabled = {
-  resharding,
+    resharding,
+    notification_v2,
 };
 
 
diff --git a/src/rgw/services/svc_topic_rados.cc b/src/rgw/services/svc_topic_rados.cc
new file mode 100644
index 000000000000..64c9106776cc
--- /dev/null
+++ b/src/rgw/services/svc_topic_rados.cc
@@ -0,0 +1,207 @@
+#include "svc_topic_rados.h"
+#include "rgw_notify.h"
+#include "rgw_tools.h"
+#include "rgw_zone.h"
+#include "svc_meta.h"
+#include "svc_meta_be_sobj.h"
+#include "svc_zone.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+static std::string topic_oid_prefix = "topic.";
+static constexpr char topic_tenant_delim[] = ":";
+
+std::string get_topic_key(const std::string& topic_name,
+                          const std::string& tenant) {
+  if (tenant.empty()) {
+    return topic_name;
+  }
+  return tenant + topic_tenant_delim + topic_name;
+}
+
+void parse_topic_entry(const std::string& topic_entry,
+                       std::string* tenant_name,
+                       std::string* topic_name) {
+  // expected format: [tenant_name:]topic_name*
+  auto pos = topic_entry.find(topic_tenant_delim);
+  if (pos != std::string::npos) {
+    *tenant_name = topic_entry.substr(0, pos);
+    *topic_name = topic_entry.substr(pos + 1);
+  } else {
+    tenant_name->clear();
+    *topic_name = topic_entry;
+  }
+}
+class RGWSI_Topic_Module : public RGWSI_MBSObj_Handler_Module {
+  RGWSI_Topic_RADOS::Svc& svc;
+  const std::string prefix;
+
+ public:
+  RGWSI_Topic_Module(RGWSI_Topic_RADOS::Svc& _svc)
+      : RGWSI_MBSObj_Handler_Module("topic"),
+        svc(_svc),
+        prefix(topic_oid_prefix) {}
+
+  void get_pool_and_oid(const std::string& key,
+                        rgw_pool* pool,
+                        std::string* oid) override {
+    if (pool) {
+      *pool = svc.zone->get_zone_params().topics_pool;
+    }
+
+    if (oid) {
+      *oid = key_to_oid(key);
+    }
+  }
+
+  bool is_valid_oid(const std::string& oid) override {
+    return boost::algorithm::starts_with(oid, prefix);
+  }
+
+  std::string key_to_oid(const std::string& key) override {
+    return prefix + key;
+  }
+
+  // This is called after `is_valid_oid` and is assumed to be a valid oid
+  std::string oid_to_key(const std::string& oid) override {
+    return oid.substr(prefix.size());
+  }
+
+  const std::string& get_oid_prefix() { return prefix; }
+};
+
+RGWSI_MetaBackend_Handler* RGWSI_Topic_RADOS::get_be_handler() {
+  return be_handler;
+}
+
+void RGWSI_Topic_RADOS::init(RGWSI_Zone* _zone_svc,
+                             RGWSI_Meta* _meta_svc,
+                             RGWSI_MetaBackend* _meta_be_svc,
+                             RGWSI_SysObj* _sysobj_svc) {
+  svc.zone = _zone_svc;
+  svc.meta = _meta_svc;
+  svc.meta_be = _meta_be_svc;
+  svc.sysobj = _sysobj_svc;
+}
+
+int RGWSI_Topic_RADOS::do_start(optional_yield y,
+                                const DoutPrefixProvider* dpp) {
+  int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ,
+                                      &be_handler);
+  if (r < 0) {
+    ldout(ctx(), 0) << "ERROR: failed to create be_handler for Topics: r=" << r
+                    << dendl;
+    return r;
+  }
+
+  auto module = new RGWSI_Topic_Module(svc);
+  RGWSI_MetaBackend_Handler_SObj* bh =
+      static_cast<RGWSI_MetaBackend_Handler_SObj*>(be_handler);
+  be_module.reset(module);
+  bh->set_module(module);
+  return 0;
+}
+
+RGWTopicMetadataHandler::RGWTopicMetadataHandler(rgw::sal::Driver* driver,
+                                                 RGWSI_Topic_RADOS* topic_svc) {
+  this->driver = driver;
+  this->topic_svc = topic_svc;
+  base_init(topic_svc->ctx(), topic_svc->get_be_handler());
+}
+
+RGWMetadataObject* RGWTopicMetadataHandler::get_meta_obj(
+    JSONObj* jo, const obj_version& objv, const ceph::real_time& mtime) {
+  rgw_pubsub_topic topic;
+  try {
+    topic.decode_json(jo);
+  } catch (JSONDecoder::err& e) {
+    return nullptr;
+  }
+
+  return new RGWTopicMetadataObject(topic, objv, mtime, driver);
+}
+
+int RGWTopicMetadataHandler::do_get(RGWSI_MetaBackend_Handler::Op* op,
+                                    std::string& entry, RGWMetadataObject** obj,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp) {
+  rgw_pubsub_topic result;
+  std::string topic_name;
+  std::string tenant;
+  parse_topic_entry(entry, &tenant, &topic_name);
+  RGWPubSub ps(driver, tenant,
+               &topic_svc->svc.zone->get_current_period().get_map().zonegroups);
+  int ret = ps.get_topic(dpp, topic_name, result, y);
+  if (ret < 0) {
+    return ret;
+  }
+  ceph::real_time mtime;
+  obj_version ver;
+  RGWTopicMetadataObject* rdo =
+      new RGWTopicMetadataObject(result, ver, mtime, driver);
+  *obj = rdo;
+  return 0;
+}
+
+int RGWTopicMetadataHandler::do_remove(RGWSI_MetaBackend_Handler::Op* op,
+                                       std::string& entry,
+                                       RGWObjVersionTracker& objv_tracker,
+                                       optional_yield y,
+                                       const DoutPrefixProvider* dpp) {
+  auto ret = rgw::notify::remove_persistent_topic(entry, y);
+  if (ret != -ENOENT && ret < 0) {
+    return ret;
+  }
+  std::string topic_name;
+  std::string tenant;
+  parse_topic_entry(entry, &tenant, &topic_name);
+  RGWPubSub ps(driver, tenant,
+               &topic_svc->svc.zone->get_current_period().get_map().zonegroups);
+  return ps.remove_topic(dpp, topic_name, y);
+}
+
+class RGWMetadataHandlerPut_Topic : public RGWMetadataHandlerPut_SObj {
+  RGWTopicMetadataHandler* rhandler;
+  RGWTopicMetadataObject* mdo;
+
+ public:
+  RGWMetadataHandlerPut_Topic(RGWTopicMetadataHandler* handler,
+                              RGWSI_MetaBackend_Handler::Op* op,
+                              std::string& entry, RGWMetadataObject* obj,
+                              RGWObjVersionTracker& objv_tracker,
+                              optional_yield y, RGWMDLogSyncType type,
+                              bool from_remote_zone)
+      : RGWMetadataHandlerPut_SObj(handler, op, entry, obj, objv_tracker, y,
+                                   type, from_remote_zone),
+        rhandler(handler) {
+    mdo = static_cast<RGWTopicMetadataObject*>(obj);
+  }
+
+  int put_checked(const DoutPrefixProvider* dpp) override {
+    auto& topic = mdo->get_topic_info();
+    auto* driver = mdo->get_driver();
+    auto ret = rgw::notify::add_persistent_topic(entry, y);
+    if (ret < 0) {
+      return ret;
+    }
+    RGWObjVersionTracker objv_tracker;
+    ret = driver->write_topic_v2(topic, &objv_tracker, y, dpp);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret
+                        << dendl;
+    }
+    return ret;
+  }
+};
+
+int RGWTopicMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op* op,
+                                    std::string& entry, RGWMetadataObject* obj,
+                                    RGWObjVersionTracker& objv_tracker,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp,
+                                    RGWMDLogSyncType type,
+                                    bool from_remote_zone) {
+  RGWMetadataHandlerPut_Topic put_op(this, op, entry, obj, objv_tracker, y,
+                                     type, from_remote_zone);
+  return do_put_operate(&put_op, dpp);
+}
diff --git a/src/rgw/services/svc_topic_rados.h b/src/rgw/services/svc_topic_rados.h
new file mode 100644
index 000000000000..e630a610e974
--- /dev/null
+++ b/src/rgw/services/svc_topic_rados.h
@@ -0,0 +1,96 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include "rgw_pubsub.h"
+#include "rgw_service.h"
+#include "svc_meta_be.h"
+
+class RGWSI_Topic_RADOS : public RGWServiceInstance {
+ public:
+  struct Svc {
+    RGWSI_Zone* zone{nullptr};
+    RGWSI_Meta* meta{nullptr};
+    RGWSI_MetaBackend* meta_be{nullptr};
+    RGWSI_SysObj* sysobj{nullptr};
+  } svc;
+
+  RGWSI_Topic_RADOS(CephContext* cct) : RGWServiceInstance(cct) {}
+  ~RGWSI_Topic_RADOS() {}
+
+  void init(RGWSI_Zone* _zone_svc,
+            RGWSI_Meta* _meta_svc,
+            RGWSI_MetaBackend* _meta_be_svc,
+            RGWSI_SysObj* _sysobj_svc);
+
+  RGWSI_MetaBackend_Handler* get_be_handler();
+  int do_start(optional_yield y, const DoutPrefixProvider* dpp) override;
+
+ private:
+  RGWSI_MetaBackend_Handler* be_handler;
+  std::unique_ptr<RGWSI_MetaBackend::Module> be_module;
+};
+
+class RGWTopicMetadataObject : public RGWMetadataObject {
+  rgw_pubsub_topic topic;
+  rgw::sal::Driver* driver;
+
+ public:
+  RGWTopicMetadataObject() = default;
+  RGWTopicMetadataObject(rgw_pubsub_topic& topic, const obj_version& v,
+                         real_time m, rgw::sal::Driver* driver)
+      : RGWMetadataObject(v, m), topic(topic), driver(driver) {}
+
+  void dump(Formatter* f) const override { topic.dump(f); }
+
+  rgw_pubsub_topic& get_topic_info() { return topic; }
+
+  rgw::sal::Driver* get_driver() { return driver; }
+};
+class RGWTopicMetadataHandler : public RGWMetadataHandler_GenericMetaBE {
+ public:
+  RGWTopicMetadataHandler(rgw::sal::Driver* driver,
+                          RGWSI_Topic_RADOS* role_svc);
+
+  std::string get_type() final { return "topic"; }
+
+  RGWMetadataObject* get_meta_obj(JSONObj* jo, const obj_version& objv,
+                                  const ceph::real_time& mtime);
+
+  int do_get(RGWSI_MetaBackend_Handler::Op* op, std::string& entry,
+             RGWMetadataObject** obj, optional_yield y,
+             const DoutPrefixProvider* dpp) final;
+
+  int do_remove(RGWSI_MetaBackend_Handler::Op* op, std::string& entry,
+                RGWObjVersionTracker& objv_tracker, optional_yield y,
+                const DoutPrefixProvider* dpp) final;
+
+  int do_put(RGWSI_MetaBackend_Handler::Op* op, std::string& entr,
+             RGWMetadataObject* obj, RGWObjVersionTracker& objv_tracker,
+             optional_yield y, const DoutPrefixProvider* dpp,
+             RGWMDLogSyncType type, bool from_remote_zone) override;
+
+ private:
+  rgw::sal::Driver* driver;
+  RGWSI_Topic_RADOS* topic_svc;
+};
+
+std::string get_topic_key(const std::string& topic_name,
+                          const std::string& tenant);
+
+void parse_topic_entry(const std::string& topic_entry,
+                       std::string* tenant_name,
+                       std::string* topic_name);

From ab74e2cfe4e3738b85d38a132f835d1dca113258 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Tue, 5 Dec 2023 12:26:48 -0500
Subject: [PATCH 2147/2492] rgw/multisite-notification: Add support to
 replicate bucket notifications in multisite config.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/rgw/driver/rados/rgw_bucket.cc |  23 ++-
 src/rgw/driver/rados/rgw_notify.cc |  61 +++++++-
 src/rgw/driver/rados/rgw_notify.h  |   2 +-
 src/rgw/rgw_admin.cc               | 102 ++++++++-----
 src/rgw/rgw_common.h               |   2 +
 src/rgw/rgw_pubsub.cc              | 134 +++++++++++-----
 src/rgw/rgw_pubsub.h               |  20 +++
 src/rgw/rgw_rest_pubsub.cc         | 235 +++++++++++++++++++++++++----
 8 files changed, 460 insertions(+), 119 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index d0eacf81e8c1..9f5565531829 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1337,13 +1337,24 @@ static int bucket_stats(rgw::sal::Driver* driver,
   }
 
   // bucket notifications
-  RGWPubSub ps(driver, tenant_name);
   rgw_pubsub_bucket_topics result;
-  const RGWPubSub::Bucket b(ps, bucket.get());
-  ret = b.get_topics(dpp, result, y);
-  if (ret < 0 && ret != -ENOENT) {
-    cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
-    return -ret;
+  if (driver->get_zone()->get_zonegroup().supports_feature(
+          rgw::zone_features::notification_v2)) {
+    ret = get_bucket_notifications(dpp, bucket.get(), result);
+    if (ret < 0) {
+      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
+           << std::endl;
+      return -ret;
+    }
+  } else {
+    RGWPubSub ps(driver, tenant_name);
+    const RGWPubSub::Bucket b(ps, bucket.get());
+    ret = b.get_topics(dpp, result, y);
+    if (ret < 0 && ret != -ENOENT) {
+      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
+           << std::endl;
+      return -ret;
+    }
   }
   result.dump(formatter);
 
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 4a2cf8271651..e5d5dd602f59 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -16,6 +16,7 @@
 #include "rgw_pubsub_push.h"
 #include "rgw_zone_features.h"
 #include "rgw_perf_counters.h"
+#include "services/svc_zone.h"
 #include "common/dout.h"
 #include <chrono>
 
@@ -980,13 +981,35 @@ static inline bool notification_match(reservation_t& res,
 		      reservation_t& res,
 		      const RGWObjTags* req_tags)
 {
-  const RGWPubSub ps(res.store, res.user_tenant);
-  const RGWPubSub::Bucket ps_bucket(ps, res.bucket);
   rgw_pubsub_bucket_topics bucket_topics;
-  auto rc = ps_bucket.get_topics(res.dpp, bucket_topics, res.yield);
-  if (rc < 0) {
-    // failed to fetch bucket topics
-    return rc;
+  if (do_all_zonegroups_support_notification_v2(
+          res.store->svc()->zone->get_current_period().get_map().zonegroups)) {
+    auto ret = 0;
+    if (!res.s) {
+      //  for non S3-request caller (e.g., lifecycle, ObjectSync), bucket attrs
+      //  are not loaded, so force to reload the bucket, that reloads the attr.
+      // for non S3-request caller, res.s is nullptr
+      ret = res.bucket->load_bucket(dpp, res.yield);
+      if (ret < 0) {
+        ldpp_dout(dpp, 1)
+            << "ERROR: failed to reload bucket: '" << res.bucket->get_name()
+            << "' to get bucket notification attrs with error ret= " << ret
+            << dendl;
+        return ret;
+      }
+    }
+    ret = get_bucket_notifications(dpp, res.bucket, bucket_topics);
+    if (ret < 0) {
+      return ret;
+    }
+  } else {
+    const RGWPubSub ps(res.store, res.user_tenant);
+    const RGWPubSub::Bucket ps_bucket(ps, res.bucket);
+    auto rc = ps_bucket.get_topics(res.dpp, bucket_topics, res.yield);
+    if (rc < 0) {
+      // failed to fetch bucket topics
+      return rc;
+    }
   }
   for (const auto& bucket_topic : bucket_topics.topics) {
     const rgw_pubsub_topic_filter& topic_filter = bucket_topic.second;
@@ -1027,7 +1050,31 @@ static inline bool notification_match(reservation_t& res,
         return ret;
       }
     }
-    res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id);
+    // load the topic,if there is change in topic config while it's stored in
+    // notification.
+    rgw_pubsub_topic result;
+    const RGWPubSub ps(
+        res.store, res.user_tenant,
+        &res.store->svc()->zone->get_current_period().get_map().zonegroups);
+    auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield);
+    if (ret < 0) {
+      ldpp_dout(res.dpp, 1)
+          << "INFO: failed to load topic: " << topic_cfg.name
+          << ". error: " << ret
+          << " while storing the persistent notification event" << dendl;
+      if (ret == -ENOENT) {
+        // either the topic is deleted but the corresponding notification still
+        // exist or in v2 mode the notification could have synced first but
+        // topic is not synced yet.
+        return 0;
+      }
+      ldpp_dout(res.dpp, 1)
+          << "WARN: Using the stored topic from bucket notification struct."
+          << dendl;
+      res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id);
+    } else {
+      res.topics.emplace_back(topic_filter.s3_id, result, res_id);
+    }
   }
   return 0;
 }
diff --git a/src/rgw/driver/rados/rgw_notify.h b/src/rgw/driver/rados/rgw_notify.h
index 20f0c17e5bb5..5117f9eecd8f 100644
--- a/src/rgw/driver/rados/rgw_notify.h
+++ b/src/rgw/driver/rados/rgw_notify.h
@@ -63,7 +63,7 @@ struct reservation_t {
   size_t size;
   rgw::sal::Object* const object;
   rgw::sal::Object* const src_object; // may differ from object
-  rgw::sal::Bucket* const bucket;
+  rgw::sal::Bucket* bucket;
   const std::string* const object_name;
   boost::optional<const RGWObjTags&> tagset;
   meta_map_t x_meta_map; // metadata cached by value
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index ced7b15120d6..afb161c90aeb 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -10614,20 +10614,28 @@ int main(int argc, const char **argv)
       return EINVAL;
     }
 
-    RGWPubSub ps(driver, tenant);
-
     rgw_pubsub_bucket_topics result;
     int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-
-    const RGWPubSub::Bucket b(ps, bucket.get());
-    ret = b.get_topics(dpp(), result, null_yield);
-    if (ret < 0 && ret != -ENOENT) {
-      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
+    if (driver->get_zone()->get_zonegroup().supports_feature(
+            rgw::zone_features::notification_v2)) {
+      ret = get_bucket_notifications(dpp(), bucket.get(), result);
+      if (ret < 0) {
+        cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
+             << std::endl;
+        return -ret;
+      }
+    } else {
+      RGWPubSub ps(driver, tenant);
+      const RGWPubSub::Bucket b(ps, bucket.get());
+      ret = b.get_topics(dpp(), result, null_yield);
+      if (ret < 0 && ret != -ENOENT) {
+        cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
     }
     encode_json("result", result, formatter.get());
     formatter->flush(cout);
@@ -10699,24 +10707,30 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-
-    RGWPubSub ps(driver, tenant);
-
     rgw_pubsub_bucket_topics bucket_topics;
-    const RGWPubSub::Bucket b(ps, bucket.get());
-    ret = b.get_topics(dpp(), bucket_topics, null_yield);
-    if (ret < 0 && ret != -ENOENT) {
-      cerr << "ERROR: could not get bucket notifications: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
+    if (driver->get_zone()->get_zonegroup().supports_feature(
+            rgw::zone_features::notification_v2)) {
+      ret = get_bucket_notifications(dpp(), bucket.get(), bucket_topics);
+      if (ret < 0) {
+        cerr << "ERROR: could not get bucket notifications: "
+             << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+    } else {
+      RGWPubSub ps(driver, tenant);
+      const RGWPubSub::Bucket b(ps, bucket.get());
+      ret = b.get_topics(dpp(), bucket_topics, null_yield);
+      if (ret < 0 && ret != -ENOENT) {
+        cerr << "ERROR: could not get bucket notifications: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
     }
-
-    rgw_pubsub_topic_filter bucket_topic;
-    ret = b.get_notification_by_id(dpp(), notification_id, bucket_topic, null_yield);
-    if (ret < 0) {
-      cerr << "ERROR: could not get notification: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
+    auto iter = find_unique_topic(bucket_topics, notification_id);
+    if (!iter) {
+      cerr << "ERROR: notification was not found" << std::endl;
+      return -ENOENT;
     }
-    encode_json("notification", bucket_topic, formatter.get());
+    encode_json("notification", *iter, formatter.get());
     formatter->flush(cout);
   }
 
@@ -10725,7 +10739,10 @@ int main(int argc, const char **argv)
       cerr << "ERROR: topic name was not provided (via --topic)" << std::endl;
       return EINVAL;
     }
-
+    if (!driver->is_meta_master()) {
+      cerr << "ERROR: Run 'topic rm' from master zone " << std::endl;
+      return -EINVAL;
+    }
     ret = rgw::notify::remove_persistent_topic(
         dpp(), static_cast<rgw::sal::RadosStore*>(driver)->getRados()->get_notif_pool_ctx(), topic_name, null_yield);
     if (ret < 0) {
@@ -10753,28 +10770,37 @@ int main(int argc, const char **argv)
       cerr << "ERROR: bucket name was not provided (via --bucket)" << std::endl;
       return EINVAL;
     }
-
+    if (!driver->is_meta_master()) {
+      cerr << "ERROR: Run 'notification rm' from master zone " << std::endl;
+      return -EINVAL;
+    }
     int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
 
-    RGWPubSub ps(driver, tenant);
+    if (driver->get_zone()->get_zonegroup().supports_feature(
+            rgw::zone_features::notification_v2)) {
+      ret = remove_notification_v2(dpp(), driver, bucket.get(), notification_id,
+                                   null_yield);
+    } else {
+      RGWPubSub ps(driver, tenant);
 
-    rgw_pubsub_bucket_topics bucket_topics;
-    const RGWPubSub::Bucket b(ps, bucket.get());
-    ret = b.get_topics(dpp(), bucket_topics, null_yield);
-    if (ret < 0 && ret != -ENOENT) {
-      cerr << "ERROR: could not get bucket notifications: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
-    }
+      rgw_pubsub_bucket_topics bucket_topics;
+      const RGWPubSub::Bucket b(ps, bucket.get());
+      ret = b.get_topics(dpp(), bucket_topics, null_yield);
+      if (ret < 0 && ret != -ENOENT) {
+        cerr << "ERROR: could not get bucket notifications: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
 
-    rgw_pubsub_topic_filter bucket_topic;
-    if(notification_id.empty()) {
-      ret = b.remove_notifications(dpp(), null_yield);
-    } else {
-      ret = b.remove_notification_by_id(dpp(), notification_id, null_yield);
+      rgw_pubsub_topic_filter bucket_topic;
+      if(notification_id.empty()) {
+        ret = b.remove_notifications(dpp(), null_yield);
+      } else {
+        ret = b.remove_notification_by_id(dpp(), notification_id, null_yield);
+      }
     }
   }
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 5bc5c6de2b5b..9b4dbfda9de4 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -171,6 +171,8 @@ using ceph::crypto::MD5;
 
 #define RGW_ATTR_TRACE RGW_ATTR_PREFIX "trace"
 
+#define RGW_ATTR_BUCKET_NOTIFICATION RGW_ATTR_PREFIX "bucket-notification"
+
 enum class RGWFormat : int8_t {
   BAD_FORMAT = -1,
   PLAIN = 0,
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index c563b959863a..628f57901afe 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -586,7 +586,7 @@ int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pub
 int RGWPubSub::get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic& result, optional_yield y) const
 {
   if (use_notification_v2) {
-    const int ret = driver->read_topic(name, tenant, result, nullptr, y, dpp);
+    const int ret = driver->read_topic_v2(name, tenant, result, nullptr, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, 1) << "failed to read topic info for name: " << name
                         << " tenant: " << tenant << ", ret=" << ret << dendl;
@@ -610,39 +610,26 @@ int RGWPubSub::get_topic(const DoutPrefixProvider *dpp, const std::string& name,
   return 0;
 }
 
-// from list of bucket topics, find the one that was auto-generated by a notification
-auto find_unique_topic(const rgw_pubsub_bucket_topics &bucket_topics, const std::string &notification_id) {
-  auto it = std::find_if(bucket_topics.topics.begin(), bucket_topics.topics.end(),
-                         [&](const auto& val) { return notification_id == val.second.s3_id; });
-  return it != bucket_topics.topics.end() ?
-         std::optional<std::reference_wrapper<const rgw_pubsub_topic_filter>>(it->second):
-         std::nullopt;
-}
-
-int RGWPubSub::Bucket::get_notification_by_id(const DoutPrefixProvider *dpp, const std::string& notification_id,
-                                              rgw_pubsub_topic_filter& result, optional_yield y) const {
-  rgw_pubsub_bucket_topics bucket_topics;
-  const int ret = read_topics(dpp, bucket_topics, nullptr, y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 1) << "ERROR: failed to read bucket_topics info: ret=" << ret << dendl;
-    return ret;
+int get_bucket_notifications(const DoutPrefixProvider* dpp,
+                             rgw::sal::Bucket* bucket,
+                             rgw_pubsub_bucket_topics& bucket_topics) {
+  const rgw::sal::Attrs& attrs = bucket->get_attrs();
+  auto iter = attrs.find(RGW_ATTR_BUCKET_NOTIFICATION);
+  if (iter == attrs.end()) {
+    return 0;
   }
-
-  auto iter = find_unique_topic(bucket_topics, notification_id);
-  if (!iter) {
-    ldpp_dout(dpp, 1) << "ERROR: notification was not found" << dendl;
-    return -ENOENT;
+  try {
+    const auto& bl = iter->second;
+    auto biter = bl.cbegin();
+    bucket_topics.decode(biter);
+  } catch (buffer::error& err) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to decode bucket topics for bucket: "
+                      << bucket->get_name() << dendl;
+    return -EIO;
   }
-
-  result = iter->get();
   return 0;
 }
 
-
-int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
-    const rgw::notify::EventTypeList& events, optional_yield y) const {
-  return create_notification(dpp, topic_name, events, std::nullopt, "", y);
-    }
 bool do_all_zonegroups_support_notification_v2(
     std::map<std::string, RGWZoneGroup> zonegroups) {
   for (const auto& [_, zonegroup] : zonegroups) {
@@ -653,7 +640,85 @@ bool do_all_zonegroups_support_notification_v2(
   return true;
 }
 
-int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
+std::string topic_to_unique(const std::string& topic,
+                            const std::string& notification) {
+  return notification + "_" + topic;
+}
+
+// from list of bucket topics, find the one that was auto-generated by a notification
+std::optional<rgw_pubsub_topic_filter> find_unique_topic(
+    const rgw_pubsub_bucket_topics& bucket_topics,
+    const std::string& notification_id) {
+  auto it = std::find_if(bucket_topics.topics.begin(), bucket_topics.topics.end(),
+                         [&](const auto& val) { return notification_id == val.second.s3_id; });
+  if (it != bucket_topics.topics.end())
+    return it->second;
+  return std::nullopt;
+}
+
+int delete_all_notifications(const DoutPrefixProvider* dpp,
+                             const rgw_pubsub_bucket_topics& bucket_topics,
+                             std::map<std::string, bufferlist>& attrs,
+                             rgw::sal::Bucket* bucket,
+                             rgw::sal::Driver* driver,
+                             optional_yield y) {
+  auto iter = attrs.find(RGW_ATTR_BUCKET_NOTIFICATION);
+  if (iter == attrs.end()) {
+    return 0;
+  }
+  // delete all notifications of on a bucket
+  attrs.erase(iter);
+  const auto ret = bucket->merge_and_store_attrs(dpp, attrs, y);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1)
+        << "Failed to remove RGW_ATTR_BUCKET_NOTIFICATION attr on bucket="
+        << bucket->get_name() << " ret= " << ret << dendl;
+  }
+  return ret;
+}
+
+int remove_notification_v2(const DoutPrefixProvider* dpp,
+                           rgw::sal::Driver* driver,
+                           rgw::sal::Bucket* bucket,
+                           const std::string& notification_id,
+                           optional_yield y) {
+  rgw_pubsub_bucket_topics bucket_topics;
+  auto ret = get_bucket_notifications(dpp, bucket, bucket_topics);
+  if (ret < 0) {
+    return -ret;
+  }
+  // no notifications on the bucket.
+  if (bucket_topics.topics.empty()) {
+    return 0;
+  }
+  rgw::sal::Attrs& attrs = bucket->get_attrs();
+  if (notification_id.empty()) {
+    return delete_all_notifications(dpp, bucket_topics, attrs, bucket, driver,
+                                    y);
+  }
+  // delete a specific notification
+  const auto unique_topic = find_unique_topic(bucket_topics, notification_id);
+  if (!unique_topic) {
+    // notification to be removed is not found - considered success
+    ldpp_dout(dpp, 20) << "notification '" << notification_id
+                       << "' already removed" << dendl;
+    return 0;
+  }
+  const auto& topic_name = unique_topic->topic.name;
+  bucket_topics.topics.erase(topic_to_unique(topic_name, notification_id));
+  bufferlist bl;
+  bucket_topics.encode(bl);
+  attrs[RGW_ATTR_BUCKET_NOTIFICATION] = std::move(bl);
+  ret = bucket->merge_and_store_attrs(dpp, attrs, y);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1)
+        << "Failed to store RGW_ATTR_BUCKET_NOTIFICATION on bucket="
+        << bucket->get_name() << " returned err= " << ret << dendl;
+  }
+  return ret;
+}
+
+int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
     const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const {
   rgw_pubsub_topic topic_info;
 
@@ -720,7 +785,7 @@ int RGWPubSub::Bucket::remove_notification_inner(const DoutPrefixProvider *dpp,
       ldpp_dout(dpp, 1) << "ERROR: notification was not found" << dendl;
       return -ENOENT;
     }
-    topic_name = std::make_unique<std::string>(iter->get().topic.name);
+    topic_name = std::make_unique<std::string>(iter->topic.name);
   }
 
   if (bucket_topics.topics.erase(*topic_name) == 0) {
@@ -786,12 +851,11 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const rgw_pubsub_topic& topic,
                             optional_yield y) const {
   RGWObjVersionTracker objv_tracker;
-  const auto ret = driver->write_topic(topic, &objv_tracker, y, dpp);
+  auto ret = driver->write_topic_v2(topic, &objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret
                       << dendl;
   }
-
   return ret;
 }
 
@@ -848,13 +912,13 @@ int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
   if (ret < 0 && ret != -ENOENT) {
     return ret;
   } else if (ret == -ENOENT) {
-    // its not an error if no topics exist, just a no-op
+    // it's not an error if no topics exist, just a no-op
     ldpp_dout(dpp, 10) << "WARNING: topic name:" << name
                        << " does not exist, deletion is a no-op: ret=" << ret
                        << dendl;
     return 0;
   }
-  ret = driver->remove_topic(name, tenant, &objv_tracker, y, dpp);
+  ret = driver->remove_topic_v2(name, tenant, &objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to remove topic info: ret=" << ret
                       << dendl;
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index 4afc101c63cc..46da8e045a9f 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -672,3 +672,23 @@ namespace rgw::notify {
 
 bool do_all_zonegroups_support_notification_v2(
     std::map<std::string, RGWZoneGroup> zonegroups);
+
+std::string topic_to_unique(const std::string& topic,
+                            const std::string& notification);
+
+std::optional<rgw_pubsub_topic_filter> find_unique_topic(
+    const rgw_pubsub_bucket_topics& bucket_topics,
+    const std::string& notif_name);
+
+// Delete the bucket notification if |notification_id| is passed, else delete
+// all the bucket notifications for the given |bucket| and update the topic
+// bucket mapping.
+int remove_notification_v2(const DoutPrefixProvider* dpp,
+                           rgw::sal::Driver* driver,
+                           rgw::sal::Bucket* bucket,
+                           const std::string& notification_id,
+                           optional_yield y);
+
+int get_bucket_notifications(const DoutPrefixProvider* dpp,
+                             rgw::sal::Bucket* bucket,
+                             rgw_pubsub_bucket_topics& bucket_topics);
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 7396fcfddd41..f2e5439208fe 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -894,29 +894,6 @@ int RGWHandler_REST_PSTopic_AWS::authorize(const DoutPrefixProvider* dpp, option
   return 0;
 }
 
-namespace {
-// return a unique topic by prefexing with the notification name: <notification>_<topic>
-std::string topic_to_unique(const std::string& topic, const std::string& notification) {
-  return notification + "_" + topic;
-}
-
-// extract the topic from a unique topic of the form: <notification>_<topic>
-[[maybe_unused]] std::string unique_to_topic(const std::string& unique_topic, const std::string& notification) {
-  if (unique_topic.find(notification + "_") == std::string::npos) {
-    return "";
-  }
-  return unique_topic.substr(notification.length() + 1);
-}
-
-// from list of bucket topics, find the one that was auto-generated by a notification
-auto find_unique_topic(const rgw_pubsub_bucket_topics& bucket_topics, const std::string& notif_name) {
-    auto it = std::find_if(bucket_topics.topics.begin(), bucket_topics.topics.end(), [&](const auto& val) { return notif_name == val.second.s3_id; });
-    return it != bucket_topics.topics.end() ?
-        std::optional<std::reference_wrapper<const rgw_pubsub_topic_filter>>(it->second):
-        std::nullopt;
-}
-}
-
 int remove_notification_by_topic(const DoutPrefixProvider *dpp, const std::string& topic_name, const RGWPubSub::Bucket& b, optional_yield y, const RGWPubSub& ps) {
   int op_ret = b.remove_notification(dpp, topic_name, y);
   if (op_ret < 0) {
@@ -944,6 +921,7 @@ int delete_all_notifications(const DoutPrefixProvider *dpp, const rgw_pubsub_buc
 // a "notification" and a subscription will be auto-generated
 // actual configuration is XML encoded in the body of the message
 class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
+  bufferlist data;
   int verify_params() override {
     bool exists;
     const auto no_value = s->info.args.get("notification", &exists);
@@ -965,7 +943,6 @@ class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
   int get_params_from_body(rgw_pubsub_s3_notifications& configurations) {
     const auto max_size = s->cct->_conf->rgw_max_put_param_size;
     int r;
-    bufferlist data;
     std::tie(r, data) = read_all_input(s, max_size, false);
 
     if (r < 0) {
@@ -1010,9 +987,14 @@ class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
 
 
   void execute(optional_yield) override;
+  void execute_v2(optional_yield);
 };
 
 void RGWPSCreateNotifOp::execute(optional_yield y) {
+  if (do_all_zonegroups_support_notification_v2(
+          s->penv.site->get_period()->get_map().zonegroups)) {
+    return execute_v2(y);
+  }
   op_ret = verify_params();
   if (op_ret < 0) {
     return;
@@ -1023,6 +1005,16 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
+  if (!driver->is_meta_master()) {
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &data, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "CreateBucketNotification "
+                            "forward_request_to_master returned ret = "
+                         << op_ret << dendl;
+      return;
+    }
+  }
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
   op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
@@ -1134,6 +1126,132 @@ int RGWPSCreateNotifOp::verify_permission(optional_yield y) {
   return 0;
 }
 
+void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
+  op_ret = verify_params();
+  if (op_ret < 0) {
+    return;
+  }
+
+  rgw_pubsub_s3_notifications configurations;
+  op_ret = get_params_from_body(configurations);
+  if (op_ret < 0) {
+    return;
+  }
+  if (!driver->is_meta_master()) {
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &data, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "CreateBucketNotification "
+                            "forward_request_to_master returned ret = "
+                         << op_ret << dendl;
+      return;
+    }
+  }
+
+  std::unique_ptr<rgw::sal::Bucket> bucket;
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &bucket, y);
+  if (op_ret < 0) {
+    ldpp_dout(this, 1) << "failed to get bucket '"
+                       << (s->bucket_tenant.empty()
+                               ? s->bucket_name
+                               : s->bucket_tenant + ":" + s->bucket_name)
+                       << "' info, ret = " << op_ret << dendl;
+    return;
+  }
+  if (configurations.list.empty()) {
+    op_ret = remove_notification_v2(this, driver, bucket.get(),
+                                    /*delete all notif=true*/ "", y);
+    return;
+  }
+  rgw_pubsub_bucket_topics bucket_topics;
+  op_ret = get_bucket_notifications(this, bucket.get(), bucket_topics);
+  if (op_ret < 0) {
+    ldpp_dout(this, 1)
+        << "failed to load existing bucket notification on bucket: "
+        << (s->bucket_tenant.empty() ? s->bucket_name
+                                     : s->bucket_tenant + ":" + s->bucket_name)
+        << "' , ret = " << op_ret << dendl;
+    return;
+  }
+  const RGWPubSub ps(driver, s->owner.id.tenant,
+                     &s->penv.site->get_period()->get_map().zonegroups);
+  std::unordered_map<std::string, rgw_pubsub_topic> topics;
+  const auto rgwbucket = rgw_bucket(s->bucket_tenant, s->bucket_name, "");
+  for (const auto& c : configurations.list) {
+    const auto& notif_name = c.id;
+    if (notif_name.empty()) {
+      ldpp_dout(this, 1) << "missing notification id" << dendl;
+      op_ret = -EINVAL;
+      return;
+    }
+    if (c.topic_arn.empty()) {
+      ldpp_dout(this, 1) << "missing topic ARN in notification: '" << notif_name
+                         << "'" << dendl;
+      op_ret = -EINVAL;
+      return;
+    }
+
+    const auto arn = rgw::ARN::parse(c.topic_arn);
+    if (!arn || arn->resource.empty()) {
+      ldpp_dout(this, 1) << "topic ARN has invalid format: '" << c.topic_arn
+                         << "' in notification: '" << notif_name << "'"
+                         << dendl;
+      op_ret = -EINVAL;
+      return;
+    }
+
+    if (std::find(c.events.begin(), c.events.end(),
+                  rgw::notify::UnknownEvent) != c.events.end()) {
+      ldpp_dout(this, 1) << "unknown event type in notification: '"
+                         << notif_name << "'" << dendl;
+      op_ret = -EINVAL;
+      return;
+    }
+    const auto& topic_name = arn->resource;
+    if (!topics.contains(topic_name)) {
+      // get topic information. destination information is stored in the topic
+      rgw_pubsub_topic topic_info;
+      op_ret = ps.get_topic(this, topic_name, topic_info, y);
+      if (op_ret < 0) {
+        ldpp_dout(this, 1) << "failed to get topic '" << topic_name
+                           << "', ret=" << op_ret << dendl;
+        return;
+      }
+      op_ret = verify_topic_owner_or_policy(
+          s, topic_info, driver->get_zone()->get_zonegroup().get_name(),
+          rgw::IAM::snsPublish);
+      if (op_ret != 0) {
+        ldpp_dout(this, 1) << "failed to create notification for topic '"
+                           << topic_name << "' topic owned by other user"
+                           << dendl;
+        return;
+      }
+      topics[topic_name] = std::move(topic_info);
+    }
+    auto& topic_filter =
+        bucket_topics.topics[topic_to_unique(topic_name, notif_name)];
+    topic_filter.topic = topics[topic_name];
+    topic_filter.events = c.events;
+    topic_filter.s3_id = notif_name;
+    topic_filter.s3_filter = c.filter;
+  }
+  // finally store all the bucket notifications as attr.
+  bufferlist bl;
+  bucket_topics.encode(bl);
+  rgw::sal::Attrs& attrs = bucket->get_attrs();
+  attrs[RGW_ATTR_BUCKET_NOTIFICATION] = std::move(bl);
+  op_ret = bucket->merge_and_store_attrs(this, attrs, y);
+  if (op_ret < 0) {
+    ldpp_dout(this, 1)
+        << "Failed to store RGW_ATTR_BUCKET_NOTIFICATION on bucket="
+        << bucket->get_name() << " returned err= " << op_ret << dendl;
+    return;
+  }
+  ldpp_dout(this, 20) << "successfully created bucket notification for bucket: "
+                      << bucket->get_name() << dendl;
+}
+
 // command (extension to S3): DELETE /bucket?notification[=<notification-id>]
 class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
   int get_params(std::string& notif_name) const {
@@ -1149,8 +1267,9 @@ class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
     }
     return 0;
   }
+  void execute_v2(optional_yield y);
 
-public:
+ public:
   int verify_permission(optional_yield y) override;
 
   void pre_exec() override {
@@ -1165,11 +1284,26 @@ class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
 };
 
 void RGWPSDeleteNotifOp::execute(optional_yield y) {
+  if (do_all_zonegroups_support_notification_v2(
+          s->penv.site->get_period()->get_map().zonegroups)) {
+    return execute_v2(y);
+  }
   std::string notif_name;
   op_ret = get_params(notif_name);
   if (op_ret < 0) {
     return;
   }
+  if (!driver->is_meta_master()) {
+    bufferlist indata;
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "DeleteBucketNotification "
+                            "forward_request_to_master returned error ret= "
+                         << op_ret << dendl;
+      return;
+    }
+  }
 
   std::unique_ptr<rgw::sal::Bucket> bucket;
   op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
@@ -1196,7 +1330,7 @@ void RGWPSDeleteNotifOp::execute(optional_yield y) {
     // delete a specific notification
     const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
     if (unique_topic) {
-      const auto unique_topic_name = unique_topic->get().topic.name;
+      const auto unique_topic_name = unique_topic->topic.name;
       op_ret = remove_notification_by_topic(this, unique_topic_name, b, y, ps);
       return;
     }
@@ -1216,6 +1350,38 @@ int RGWPSDeleteNotifOp::verify_permission(optional_yield y) {
   return 0;
 }
 
+void RGWPSDeleteNotifOp::execute_v2(optional_yield y) {
+  std::string notif_name;
+  op_ret = get_params(notif_name);
+  if (op_ret < 0) {
+    return;
+  }
+  if (!driver->is_meta_master()) {
+    bufferlist indata;
+    op_ret = rgw_forward_request_to_master(
+        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "DeleteBucketNotification "
+                            "forward_request_to_master returned error ret= "
+                         << op_ret << dendl;
+      return;
+    }
+  }
+
+  std::unique_ptr<rgw::sal::Bucket> bucket;
+  op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
+                               &bucket, y);
+  if (op_ret < 0) {
+    ldpp_dout(this, 1) << "failed to get bucket '"
+                       << (s->bucket_tenant.empty()
+                               ? s->bucket_name
+                               : s->bucket_tenant + ":" + s->bucket_name)
+                       << "' info, ret = " << op_ret << dendl;
+    return;
+  }
+  op_ret = remove_notification_v2(this, driver, bucket.get(), notif_name, y);
+}
+
 // command (S3 compliant): GET /bucket?notification[=<notification-id>]
 class RGWPSListNotifsOp : public RGWOp {
   rgw_pubsub_s3_notifications notifications;
@@ -1278,21 +1444,26 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
     return;
   }
 
-  const RGWPubSub ps(driver, s->owner.id.tenant);
-  const RGWPubSub::Bucket b(ps, bucket.get());
-  
   // get all topics on a bucket
   rgw_pubsub_bucket_topics bucket_topics;
-  op_ret = b.get_topics(this, bucket_topics, y);
+  if (do_all_zonegroups_support_notification_v2(
+          s->penv.site->get_period()->get_map().zonegroups)) {
+    op_ret = get_bucket_notifications(this, bucket.get(), bucket_topics);
+  } else {
+    const RGWPubSub ps(driver, s->owner.id.tenant);
+    const RGWPubSub::Bucket b(ps, bucket.get());
+    op_ret = b.get_topics(this, bucket_topics, y);
+  }
   if (op_ret < 0) {
-    ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << s->bucket_name << "', ret=" << op_ret << dendl;
+    ldpp_dout(this, 1) << "failed to get list of topics from bucket '"
+                       << s->bucket_name << "', ret=" << op_ret << dendl;
     return;
   }
   if (!notif_name.empty()) {
     // get info of a specific notification
     const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
     if (unique_topic) {
-      notifications.list.emplace_back(unique_topic->get());
+      notifications.list.emplace_back(*unique_topic);
       return;
     }
     op_ret = -ENOENT;

From df668e9f65cec50227f671eb2e624c0ee02f2c37 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Thu, 21 Dec 2023 17:23:05 -0500
Subject: [PATCH 2148/2492] rgw/multisite-notification: Add omap object to
 store the mapping between bucket and topics.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/rgw/driver/rados/rgw_bucket.cc    | 132 +++++++++++++++++++++-----
 src/rgw/driver/rados/rgw_notify.cc    |   5 +-
 src/rgw/driver/rados/rgw_rados.cc     |  10 ++
 src/rgw/driver/rados/rgw_rados.h      |   4 +
 src/rgw/driver/rados/rgw_sal_rados.cc | 132 ++++++++++++++++++++++++++
 src/rgw/driver/rados/rgw_sal_rados.h  |  17 ++++
 src/rgw/rgw_admin.cc                  |  39 +++++++-
 src/rgw/rgw_pubsub.cc                 | 132 +++++++++++++++++++-------
 src/rgw/rgw_pubsub.h                  |  20 ++--
 src/rgw/rgw_rest_pubsub.cc            |  29 ++++--
 src/rgw/rgw_sal.h                     |  28 ++++++
 src/rgw/rgw_sal_filter.h              |  28 +++++-
 src/rgw/rgw_sal_store.h               |  29 +++++-
 src/rgw/services/svc_topic_rados.cc   |   8 +-
 src/rgw/services/svc_topic_rados.h    |   2 +
 15 files changed, 530 insertions(+), 85 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index 9f5565531829..ce869f399d6e 100644
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1336,28 +1336,6 @@ static int bucket_stats(rgw::sal::Driver* driver,
     }
   }
 
-  // bucket notifications
-  rgw_pubsub_bucket_topics result;
-  if (driver->get_zone()->get_zonegroup().supports_feature(
-          rgw::zone_features::notification_v2)) {
-    ret = get_bucket_notifications(dpp, bucket.get(), result);
-    if (ret < 0) {
-      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
-           << std::endl;
-      return -ret;
-    }
-  } else {
-    RGWPubSub ps(driver, tenant_name);
-    const RGWPubSub::Bucket b(ps, bucket.get());
-    ret = b.get_topics(dpp, result, y);
-    if (ret < 0 && ret != -ENOENT) {
-      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
-           << std::endl;
-      return -ret;
-    }
-  }
-  result.dump(formatter);
-
   // TODO: bucket CORS
   // TODO: bucket LC
   formatter->close_section();
@@ -2132,6 +2110,92 @@ int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider *dpp)
   return ret;
 }
 
+int update_bucket_topic_mappings(const DoutPrefixProvider* dpp,
+                                 RGWBucketCompleteInfo* orig_bci,
+                                 RGWBucketCompleteInfo* current_bci,
+                                 rgw::sal::Driver* driver) {
+  const auto decode_attrs = [](const rgw::sal::Attrs& attrs,
+                               rgw_pubsub_bucket_topics& bucket_topics) -> int {
+    auto iter = attrs.find(RGW_ATTR_BUCKET_NOTIFICATION);
+    if (iter == attrs.end()) {
+      return 0;
+    }
+    try {
+      const auto& bl = iter->second;
+      auto biter = bl.cbegin();
+      bucket_topics.decode(biter);
+    } catch (buffer::error& err) {
+      return -EIO;
+    }
+    return 0;
+  };
+  std::string bucket_name;
+  std::string bucket_tenant;
+  rgw_pubsub_bucket_topics old_bucket_topics;
+  if (orig_bci) {
+    auto ret = decode_attrs(orig_bci->attrs, old_bucket_topics);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1)
+          << "ERROR: failed to decode OLD bucket topics for bucket: "
+          << orig_bci->info.bucket.name << dendl;
+      return ret;
+    }
+    bucket_name = orig_bci->info.bucket.name;
+    bucket_tenant = orig_bci->info.bucket.tenant;
+  }
+  rgw_pubsub_bucket_topics current_bucket_topics;
+  if (current_bci) {
+    auto ret = decode_attrs(current_bci->attrs, current_bucket_topics);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1)
+          << "ERROR: failed to decode current bucket topics for bucket: "
+          << current_bci->info.bucket.name << dendl;
+      return ret;
+    }
+    bucket_name = current_bci->info.bucket.name;
+    bucket_tenant = current_bci->info.bucket.tenant;
+  }
+  // fetch the list of subscribed topics stored inside old_bucket attrs.
+  std::unordered_map<std::string, rgw_pubsub_topic> old_topics;
+  for (const auto& [_, topic_filter] : old_bucket_topics.topics) {
+    old_topics[topic_filter.topic.name] = topic_filter.topic;
+  }
+  // fetch the list of subscribed topics stored inside current_bucket attrs.
+  std::unordered_map<std::string, rgw_pubsub_topic> current_topics;
+  for (const auto& [_, topic_filter] : current_bucket_topics.topics) {
+    current_topics[topic_filter.topic.name] = topic_filter.topic;
+  }
+  // traverse thru old topics and check if they are not in current, then delete
+  // the mapping, if present in both current and old then delete from current
+  // set as we do not need to update those mapping.
+  int ret = 0;
+  for (const auto& [topic_name, topic] : old_topics) {
+    auto it = current_topics.find(topic_name);
+    if (it == current_topics.end()) {
+      const auto op_ret = driver->update_bucket_topic_mapping(
+          topic, rgw_make_bucket_entry_name(bucket_tenant, bucket_name),
+          /*add_mapping=*/false, null_yield, dpp);
+      if (op_ret < 0) {
+        ret = op_ret;
+      }
+    } else {
+      // already that attr is present, so do not update the mapping.
+      current_topics.erase(it);
+    }
+  }
+  // traverse thru current topics and check if they are any present, then add
+  // the mapping.
+  for (const auto& [topic_name, topic] : current_topics) {
+    const auto op_ret = driver->update_bucket_topic_mapping(
+        topic, rgw_make_bucket_entry_name(bucket_tenant, bucket_name),
+        /*add_mapping=*/true, null_yield, dpp);
+    if (op_ret < 0) {
+      ret = op_ret;
+    }
+  }
+  return ret;
+}
+
 static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) {
 
    char md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
@@ -2443,7 +2507,14 @@ class RGWBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandler
     if (ret < 0 && ret != -ENOENT)
       return ret;
 
-    return svc.bucket->remove_bucket_instance_info(ctx, entry, bci.info, &bci.info.objv_tracker, y, dpp);
+    ret = svc.bucket->remove_bucket_instance_info(
+        ctx, entry, bci.info, &bci.info.objv_tracker, y, dpp);
+    if (ret < 0)
+      return ret;
+    ret = update_bucket_topic_mappings(dpp, &bci, /*current_bci=*/nullptr,
+                                       driver);
+    // update_bucket_topic_mapping error is swallowed.
+    return 0;
   }
 
   int call(std::function<int(RGWSI_Bucket_BI_Ctx& ctx)> f) {
@@ -2648,6 +2719,21 @@ int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp
     }
   } /* update lc */
 
+  /* update bucket topic mapping */
+  {
+    auto* orig_obj = static_cast<RGWBucketInstanceMetadataObject*>(old_obj);
+    auto* orig_bci = (orig_obj ? &orig_obj->get_bci() : nullptr);
+    ret = update_bucket_topic_mappings(dpp, orig_bci, &bci, bihandler->driver);
+    if (ret < 0) {
+      ldpp_dout(dpp, 0) << __func__
+                        << " failed to apply bucket topic mapping for "
+                        << bci.info.bucket.name << dendl;
+      return ret;
+    }
+    ldpp_dout(dpp, 20) << __func__
+                       << " successfully applied bucket topic mapping for "
+                       << bci.info.bucket.name << dendl;
+  }
   return STATUS_APPLIED;
 }
 
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index e5d5dd602f59..e3ee61ba9df0 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -494,7 +494,8 @@ class Manager : public DoutPrefixProvider {
           RGWPubSub ps(&rados_store, tenant_name);
 
           rgw_pubsub_topic topic;
-          auto ret_of_get_topic = ps.get_topic(this, queue_name, topic, optional_yield(io_context, yield));
+          auto ret_of_get_topic = ps.get_topic(this, queue_name, topic,
+                           optional_yield(io_context, yield), nullptr);
           if (ret_of_get_topic < 0) {
             // we can't migrate entries without topic info
             ldpp_dout(this, 1) << "ERROR: failed to fetch topic: " << queue_name << " error: "
@@ -1056,7 +1057,7 @@ static inline bool notification_match(reservation_t& res,
     const RGWPubSub ps(
         res.store, res.user_tenant,
         &res.store->svc()->zone->get_current_period().get_map().zonegroups);
-    auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield);
+    auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr);
     if (ret < 0) {
       ldpp_dout(res.dpp, 1)
           << "INFO: failed to load topic: " << topic_cfg.name
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 5e111103c1c2..b8fcfa2f368f 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1211,6 +1211,10 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
   if (ret < 0)
     return ret;
 
+  ret = open_topics_pool_ctx(dpp);
+  if (ret < 0)
+    return ret;
+
   pools_initialized = true;
 
   if (use_gc) {
@@ -1446,6 +1450,12 @@ int RGWRados::open_notif_pool_ctx(const DoutPrefixProvider *dpp)
   return rgw_init_ioctx(dpp, get_rados_handle(), svc.zone->get_zone_params().notif_pool, notif_pool_ctx, true, true);
 }
 
+int RGWRados::open_topics_pool_ctx(const DoutPrefixProvider* dpp) {
+  return rgw_init_ioctx(dpp, get_rados_handle(),
+                        svc.zone->get_zone_params().topics_pool,
+                        topics_pool_ctx, true, true);
+}
+
 int RGWRados::open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx,
 			    bool mostly_omap, bool bulk)
 {
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 264f5eb4f33a..7e7a58480a65 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -358,6 +358,7 @@ class RGWRados
   int open_objexp_pool_ctx(const DoutPrefixProvider *dpp);
   int open_reshard_pool_ctx(const DoutPrefixProvider *dpp);
   int open_notif_pool_ctx(const DoutPrefixProvider *dpp);
+  int open_topics_pool_ctx(const DoutPrefixProvider* dpp);
 
   int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx&  io_ctx,
 		    bool mostly_omap, bool bulk);
@@ -447,6 +448,7 @@ class RGWRados
   librados::IoCtx objexp_pool_ctx;
   librados::IoCtx reshard_pool_ctx;
   librados::IoCtx notif_pool_ctx;     // .rgw.notif
+  librados::IoCtx topics_pool_ctx;  // .rgw.meta:topics
 
   bool pools_initialized{false};
 
@@ -533,6 +535,8 @@ class RGWRados
     return notif_pool_ctx;
   }
 
+  librados::IoCtx& get_topics_pool_ctx() { return topics_pool_ctx; }
+  
   void set_context(CephContext *_cct) {
     cct = _cct;
   }
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 608800c7e573..2bde7d192e5a 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -325,6 +325,30 @@ int RadosBucket::remove(const DoutPrefixProvider* dpp,
       this, get_attrs(), merge_attrs);
   }
 
+  // remove bucket-topic mapping
+  auto iter = get_attrs().find(RGW_ATTR_BUCKET_NOTIFICATION);
+  if (iter != get_attrs().end()) {
+    rgw_pubsub_bucket_topics bucket_topics;
+    try {
+      const auto& bl = iter->second;
+      auto biter = bl.cbegin();
+      bucket_topics.decode(biter);
+    } catch (buffer::error& err) {
+      ldpp_dout(dpp, 1) << "ERROR: failed to decode bucket topics for bucket: "
+                        << get_name() << dendl;
+    }
+    if (!bucket_topics.topics.empty()) {
+      ret = store->remove_bucket_mapping_from_topics(
+          bucket_topics, rgw_make_bucket_entry_name(get_tenant(), get_name()),
+          y, dpp);
+      if (ret < 0) {
+        ldpp_dout(dpp, 1)
+            << "ERROR: unable to remove notifications from bucket "
+            << get_name() << ". ret=" << ret << dendl;
+      }
+    }
+  }
+
   ret = store->ctl()->bucket->sync_user_stats(dpp, info.owner, info, y, nullptr);
   if (ret < 0) {
      ldout(store->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" <<  ret << dendl;
@@ -1172,6 +1196,114 @@ int RadosStore::remove_topic_v2(const std::string& topic_name,
                                            params, objv_tracker, y, dpp);
 }
 
+int RadosStore::remove_bucket_mapping_from_topics(
+    const rgw_pubsub_bucket_topics& bucket_topics,
+    const std::string& bucket_key,
+    optional_yield y,
+    const DoutPrefixProvider* dpp) {
+  // remove the bucket name from  the topic-bucket omap for each topic
+  // subscribed.
+  std::unordered_set<std::string> topics_mapping_to_remove;
+  int ret = 0;
+  for (const auto& [_, topic_filter] : bucket_topics.topics) {
+    if (!topics_mapping_to_remove.insert(topic_filter.topic.name).second) {
+      continue;  // already removed.
+    }
+    int op_ret = update_bucket_topic_mapping(topic_filter.topic, bucket_key,
+                                             /*add_mapping=*/false, y, dpp);
+    if (op_ret < 0) {
+      ret = op_ret;
+    }
+  }
+  return ret;
+}
+
+int RadosStore::update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                            const std::string& bucket_key,
+                                            bool add_mapping,
+                                            optional_yield y,
+                                            const DoutPrefixProvider* dpp) {
+  bufferlist empty_bl;
+  librados::ObjectWriteOperation op;
+  int ret = 0;
+  if (add_mapping) {
+    std::map<std::string, bufferlist> mapping{{bucket_key, empty_bl}};
+    op.omap_set(mapping);
+  } else {
+    std::set<std::string> to_rm{{bucket_key}};
+    op.omap_rm_keys(to_rm);
+  }
+  ret = rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
+                          get_bucket_topic_mapping_oid(topic), &op, y);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to " << (add_mapping ? "add" : "remove")
+                      << " topic bucket mapping for bucket: " << bucket_key
+                      << " and topic: " << topic.name << " with ret:" << ret << dendl;
+    return ret;
+  }
+  ldpp_dout(dpp, 20) << "Successfully " << (add_mapping ? "added" : "removed")
+                     << " topic bucket mapping for bucket: " << bucket_key
+                     << " and topic: " << topic.name << dendl;
+  return ret;
+}
+
+int RadosStore::get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                         std::set<std::string>& bucket_keys,
+                                         optional_yield y,
+                                         const DoutPrefixProvider* dpp) {
+  constexpr auto max_chunk = 1024U;
+  std::string start_after;
+  bool more = true;
+  int rval;
+  while (more) {
+    librados::ObjectReadOperation op;
+    std::set<std::string> curr_keys;
+    op.omap_get_keys2(start_after, max_chunk, &curr_keys, &more, &rval);
+    const auto ret =
+        rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
+                          get_bucket_topic_mapping_oid(topic), &op, nullptr, y);
+    if (ret == -ENOENT) {
+      // mapping object was not created - nothing to do
+      return 0;
+    }
+    if (ret < 0) {
+      // TODO: do we need to check on rval as well as ret?
+      ldpp_dout(dpp, 1)
+          << "ERROR: failed to read bucket topic mapping object for topic: "
+          << topic.name << ", ret= " << ret << dendl;
+      return ret;
+    }
+    if (more) {
+      if (curr_keys.empty()) {
+        return -EINVAL;  // something wrong.
+      }
+      start_after = *curr_keys.rbegin();
+    }
+    bucket_keys.merge(curr_keys);
+  }
+  return 0;
+}
+
+int RadosStore::delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                            optional_yield y,
+                                            const DoutPrefixProvider* dpp) {
+  librados::ObjectWriteOperation op;
+  op.remove();
+  const int ret =
+      rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
+                        get_bucket_topic_mapping_oid(topic), &op, y);
+  if (ret < 0 && ret != -ENOENT) {
+    ldpp_dout(dpp, 1)
+        << "ERROR: failed removing bucket topic mapping omap for topic: "
+        << topic.name << ", ret=" << ret << dendl;
+    return ret;
+  }
+  ldpp_dout(dpp, 20)
+      << "Successfully deleted topic bucket mapping omap for topic: "
+      << topic.name << dendl;
+  return 0;
+}
+
 int RadosStore::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, optional_yield y)
 {
   return rados->delete_raw_obj(dpp, obj, y);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 33db603d785a..85612eec1a9b 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -184,6 +184,23 @@ class RadosStore : public StoreDriver {
                         RGWObjVersionTracker* objv_tracker,
                         optional_yield y,
                         const DoutPrefixProvider* dpp) override;
+    int update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                    const std::string& bucket_key,
+                                    bool add_mapping,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp) override;
+    int remove_bucket_mapping_from_topics(
+        const rgw_pubsub_bucket_topics& bucket_topics,
+        const std::string& bucket_key,
+        optional_yield y,
+        const DoutPrefixProvider* dpp) override;
+    int get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                 std::set<std::string>& bucket_keys,
+                                 optional_yield y,
+                                 const DoutPrefixProvider* dpp) override;
+    int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp) override;
     virtual RGWLC* get_rgwlc(void) override { return rados->get_lc(); }
     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return rados->get_cr_registry(); }
 
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index afb161c90aeb..bbd9231893fb 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1168,6 +1168,15 @@ static void show_reshard_status(
   formatter->flush(cout);
 }
 
+static void show_topics_info_v2(const rgw_pubsub_topic& topic,
+                                std::set<std::string> subscribed_buckets,
+                                Formatter* formatter) {
+  formatter->open_object_section("topic");
+  topic.dump(formatter);
+  encode_json("subscribed_buckets", subscribed_buckets, formatter);
+  formatter->close_section();
+}
+
 class StoreDestructor {
   rgw::sal::Driver* driver;
 public:
@@ -10665,7 +10674,24 @@ int main(int argc, const char **argv)
         }
       }
     }
-    encode_json("result", result, formatter.get());
+    if (driver->get_zone()->get_zonegroup().supports_feature(
+            rgw::zone_features::notification_v2)) {
+      Formatter::ObjectSection top_section(*formatter, "result");
+      Formatter::ArraySection s(*formatter, "topics");
+      for (const auto& [_, topic] : result.topics) {
+        std::set<std::string> subscribed_buckets;
+        ret = driver->get_bucket_topic_mapping(topic, subscribed_buckets,
+                                               null_yield, dpp());
+        if (ret < 0) {
+          cerr << "failed to fetch bucket topic mapping info for topic: "
+               << topic.name << ", ret=" << ret << std::endl;
+        } else {
+          show_topics_info_v2(topic, subscribed_buckets, formatter.get());
+        }
+      }
+    } else {
+      encode_json("result", result, formatter.get());
+    }
     formatter->flush(cout);
   }
 
@@ -10683,12 +10709,19 @@ int main(int argc, const char **argv)
     RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
 
     rgw_pubsub_topic topic;
-    ret = ps.get_topic(dpp(), topic_name, topic, null_yield);
+    std::set<std::string> subscribed_buckets;
+    ret =
+        ps.get_topic(dpp(), topic_name, topic, null_yield, &subscribed_buckets);
     if (ret < 0) {
       cerr << "ERROR: could not get topic: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    encode_json("topic", topic, formatter.get());
+    if (driver->get_zone()->get_zonegroup().supports_feature(
+            rgw::zone_features::notification_v2)) {
+      show_topics_info_v2(topic, subscribed_buckets, formatter.get());
+    } else {
+      encode_json("topic", topic, formatter.get());
+    }
     formatter->flush(cout);
   }
 
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 628f57901afe..dec38ee87e87 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -9,7 +9,9 @@
 #include "rgw_xml.h"
 #include "rgw_arn.h"
 #include "rgw_pubsub_push.h"
+#include "rgw_bucket.h"
 #include "common/errno.h"
+#include "svc_topic_rados.h"
 #include <regex>
 #include <algorithm>
 
@@ -369,6 +371,7 @@ void rgw_pubsub_topic::dump_xml_as_attributes(Formatter *f) const
   encode_xml_key_value_entry("TopicArn", arn, f);
   encode_xml_key_value_entry("OpaqueData", opaque_data, f);
   encode_xml_key_value_entry("Policy", policy_text, f);
+  std::ostringstream stream;
   f->close_section(); // Attributes
 }
 
@@ -526,15 +529,20 @@ int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& res
                 << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl;
             break;
       }
-      for (auto& topic_name : topics) {
-            rgw_pubsub_topic topic;
-            int ret = get_topic(dpp, topic_name, topic, y);
-            if (ret < 0) {
-              ldpp_dout(dpp, 1) << "ERROR: failed to read topic '" << topic_name
-                                << "' info: ret=" << ret << dendl;
-              continue;
-            }
-            result.topics[topic_name] = std::move(topic);
+      for (auto& topic_entry : topics) {
+        std::string topic_name;
+        std::string topic_tenant;
+        parse_topic_entry(topic_entry, &topic_tenant, &topic_name);
+        if (tenant != topic_tenant) {
+          continue;
+        }
+        rgw_pubsub_topic topic;
+        const auto op_ret = get_topic(dpp, topic_name, topic, y, nullptr);
+        if (op_ret < 0) {
+          ret = op_ret;
+          continue;
+        }
+        result.topics[topic_name] = std::move(topic);
       }
     } while (truncated);
     driver->meta_list_keys_complete(handle);
@@ -583,13 +591,26 @@ int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pub
   return 0;
 }
 
-int RGWPubSub::get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic& result, optional_yield y) const
-{
+int RGWPubSub::get_topic(const DoutPrefixProvider* dpp,
+                         const std::string& name,
+                         rgw_pubsub_topic& result,
+                         optional_yield y,
+                         std::set<std::string>* subscribed_buckets) const {
   if (use_notification_v2) {
-    const int ret = driver->read_topic_v2(name, tenant, result, nullptr, y, dpp);
+    int ret = driver->read_topic_v2(name, tenant, result, nullptr, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, 1) << "failed to read topic info for name: " << name
                         << " tenant: " << tenant << ", ret=" << ret << dendl;
+      return ret;
+    }
+    if (subscribed_buckets) {
+      ret =
+          driver->get_bucket_topic_mapping(result, *subscribed_buckets, y, dpp);
+      if (ret < 0) {
+        ldpp_dout(dpp, 1)
+            << "failed to fetch bucket topic mapping info for topic: " << name
+            << " tenant: " << tenant << ", ret=" << ret << dendl;
+      }
     }
     return ret;
   }
@@ -656,19 +677,56 @@ std::optional<rgw_pubsub_topic_filter> find_unique_topic(
   return std::nullopt;
 }
 
-int delete_all_notifications(const DoutPrefixProvider* dpp,
-                             const rgw_pubsub_bucket_topics& bucket_topics,
-                             std::map<std::string, bufferlist>& attrs,
-                             rgw::sal::Bucket* bucket,
-                             rgw::sal::Driver* driver,
-                             optional_yield y) {
+int store_bucket_attrs_and_update_mapping(
+    const DoutPrefixProvider* dpp,
+    rgw::sal::Driver* driver,
+    rgw::sal::Bucket* bucket,
+    rgw_pubsub_bucket_topics& bucket_topics,
+    const rgw_pubsub_topic& topic,
+    optional_yield y) {
+  rgw::sal::Attrs& attrs = bucket->get_attrs();
+  if (!bucket_topics.topics.empty()) {
+    bufferlist bl;
+    bucket_topics.encode(bl);
+    attrs[RGW_ATTR_BUCKET_NOTIFICATION] = std::move(bl);
+  } else {
+    auto it = attrs.find(RGW_ATTR_BUCKET_NOTIFICATION);
+    if (it != attrs.end()) {
+      attrs.erase(it);
+    }
+  }
+  auto ret = bucket->merge_and_store_attrs(dpp, attrs, y);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1)
+        << "Failed to store RGW_ATTR_BUCKET_NOTIFICATION on bucket="
+        << bucket->get_name() << " returned err= " << ret << dendl;
+    return ret;
+  }
+  if (bucket_topics.topics.empty()) {
+    // remove the bucket name from  the topic-bucket omap
+    auto op_ret = driver->update_bucket_topic_mapping(
+        topic,
+        rgw_make_bucket_entry_name(bucket->get_tenant(), bucket->get_name()),
+        /*add_mapping=*/false, y, dpp);
+    if (op_ret < 0) {
+      // TODO: should the error be reported, as attrs are already deleted.
+      // ret = op_ret;
+    }
+  }
+  return ret;
+}
+
+int delete_notification_attrs(const DoutPrefixProvider* dpp,
+                              rgw::sal::Bucket* bucket,
+                              optional_yield y) {
+  auto& attrs = bucket->get_attrs();
   auto iter = attrs.find(RGW_ATTR_BUCKET_NOTIFICATION);
   if (iter == attrs.end()) {
     return 0;
   }
   // delete all notifications of on a bucket
   attrs.erase(iter);
-  const auto ret = bucket->merge_and_store_attrs(dpp, attrs, y);
+  auto ret = bucket->merge_and_store_attrs(dpp, attrs, y);
   if (ret < 0) {
     ldpp_dout(dpp, 1)
         << "Failed to remove RGW_ATTR_BUCKET_NOTIFICATION attr on bucket="
@@ -691,11 +749,23 @@ int remove_notification_v2(const DoutPrefixProvider* dpp,
   if (bucket_topics.topics.empty()) {
     return 0;
   }
-  rgw::sal::Attrs& attrs = bucket->get_attrs();
+  // delete all notifications
   if (notification_id.empty()) {
-    return delete_all_notifications(dpp, bucket_topics, attrs, bucket, driver,
-                                    y);
+    ret = delete_notification_attrs(dpp, bucket, y);
+    if (ret < 0) {
+      return ret;
+    }
+    int op_ret = driver->remove_bucket_mapping_from_topics(
+        bucket_topics,
+        rgw_make_bucket_entry_name(bucket->get_tenant(), bucket->get_name()), y,
+        dpp);
+    if (op_ret < 0) {
+      // TODO: should the error be reported, as attrs are already deleted.
+      // ret = op_ret;
+    }
+    return ret;
   }
+
   // delete a specific notification
   const auto unique_topic = find_unique_topic(bucket_topics, notification_id);
   if (!unique_topic) {
@@ -706,23 +776,15 @@ int remove_notification_v2(const DoutPrefixProvider* dpp,
   }
   const auto& topic_name = unique_topic->topic.name;
   bucket_topics.topics.erase(topic_to_unique(topic_name, notification_id));
-  bufferlist bl;
-  bucket_topics.encode(bl);
-  attrs[RGW_ATTR_BUCKET_NOTIFICATION] = std::move(bl);
-  ret = bucket->merge_and_store_attrs(dpp, attrs, y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 1)
-        << "Failed to store RGW_ATTR_BUCKET_NOTIFICATION on bucket="
-        << bucket->get_name() << " returned err= " << ret << dendl;
-  }
-  return ret;
+  return store_bucket_attrs_and_update_mapping(
+      dpp, driver, bucket, bucket_topics, unique_topic->topic, y);
 }
 
 int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
     const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const {
   rgw_pubsub_topic topic_info;
 
-  int ret = ps.get_topic(dpp, topic_name, topic_info, y);
+  int ret = ps.get_topic(dpp, topic_name, topic_info, y, nullptr);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to read topic '" << topic_name << "' info: ret=" << ret << dendl;
     return ret;
@@ -908,7 +970,7 @@ int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
                                optional_yield y) const {
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topic topic;
-  int ret = get_topic(dpp, name, topic, y);
+  int ret = get_topic(dpp, name, topic, y, nullptr);
   if (ret < 0 && ret != -ENOENT) {
     return ret;
   } else if (ret == -ENOENT) {
@@ -922,7 +984,9 @@ int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to remove topic info: ret=" << ret
                       << dendl;
+    return ret;
   }
+  ret = driver->delete_bucket_topic_mapping(topic, y, dpp);
   return ret;
 }
 
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index 46da8e045a9f..8509d86e2255 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -602,19 +602,13 @@ class RGWPubSub
     int get_topics(const DoutPrefixProvider *dpp, rgw_pubsub_bucket_topics& result, optional_yield y) const {
       return read_topics(dpp, result, nullptr, y);
     }
-    // get a bucket_topic with by its name and populate it into "result"
-    // return -ENOENT if the topic does not exists
-    // return 0 on success, error code otherwise
-    int get_notification_by_id(const DoutPrefixProvider *dpp, const std::string& notification_id, rgw_pubsub_topic_filter& result, optional_yield y) const;
     // adds a topic + filter (event list, and possibly name metadata or tags filters) to a bucket
     // assigning a notification name is optional (needed for S3 compatible notifications)
     // if the topic already exist on the bucket, the filter event list may be updated
     // for S3 compliant notifications the version with: s3_filter and notif_name should be used
     // return -ENOENT if the topic does not exists
     // return 0 on success, error code otherwise
-    int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
-        const rgw::notify::EventTypeList& events, optional_yield y) const;
-    int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, 
+    int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
         const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const;
     // remove a topic and filter from bucket
     // if the topic does not exists on the bucket it is a no-op (considered success)
@@ -633,9 +627,15 @@ class RGWPubSub
     return read_topics(dpp, result, nullptr, y);
   }
   // get a topic with by its name and populate it into "result"
-  // return -ENOENT if the topic does not exists 
-  // return 0 on success, error code otherwise
-  int get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic& result, optional_yield y) const;
+  // return -ENOENT if the topic does not exists
+  // return 0 on success, error code otherwise.
+  // if |subscribed_buckets| valid, then for notification_v2 read the bucket
+  // topic mapping object.
+  int get_topic(const DoutPrefixProvider* dpp,
+                const std::string& name,
+                rgw_pubsub_topic& result,
+                optional_yield y,
+                std::set<std::string>* subscribed_buckets) const;
   // create a topic with a name only
   // if the topic already exists it is a no-op (considered success)
   // return 0 on success, error code otherwise
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index f2e5439208fe..38943736e451 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -200,7 +200,7 @@ class RGWPSCreateTopicOp : public RGWOp {
     const RGWPubSub ps(driver, s->owner.id.tenant,
                        &s->penv.site->get_period()->get_map().zonegroups);
     rgw_pubsub_topic result;
-    ret = ps.get_topic(this, topic_name, result, y);
+    ret = ps.get_topic(this, topic_name, result, y, nullptr);
     if (ret == -ENOENT) {
       // topic not present
       return 0;
@@ -424,7 +424,7 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
   }
   const RGWPubSub ps(driver, s->owner.id.tenant,
                      &s->penv.site->get_period()->get_map().zonegroups);
-  op_ret = ps.get_topic(this, topic_name, result, y);
+  op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
     return;
@@ -509,7 +509,7 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
   }
   const RGWPubSub ps(driver, s->owner.id.tenant,
                      &s->penv.site->get_period()->get_map().zonegroups);
-  op_ret = ps.get_topic(this, topic_name, result, y);
+  op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
     return;
@@ -638,7 +638,7 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
     rgw_pubsub_topic result;
     const RGWPubSub ps(driver, s->owner.id.tenant,
                        &s->penv.site->get_period()->get_map().zonegroups);
-    ret = ps.get_topic(this, topic_name, result, y);
+    ret = ps.get_topic(this, topic_name, result, y, nullptr);
     if (ret < 0) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name
                          << "', ret=" << ret << dendl;
@@ -801,7 +801,7 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
                      &s->penv.site->get_period()->get_map().zonegroups);
 
   rgw_pubsub_topic result;
-  op_ret = ps.get_topic(this, topic_name, result, y);
+  op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
   if (op_ret == 0) {
     op_ret = verify_topic_owner_or_policy(
         s, result, driver->get_zone()->get_zonegroup().get_name(),
@@ -1071,8 +1071,8 @@ void RGWPSCreateNotifOp::execute(optional_yield y) {
     const auto topic_name = arn->resource;
 
     // get topic information. destination information is stored in the topic
-    rgw_pubsub_topic topic_info;  
-    op_ret = ps.get_topic(this, topic_name, topic_info, y);
+    rgw_pubsub_topic topic_info;
+    op_ret = ps.get_topic(this, topic_name, topic_info, y, nullptr);
     if (op_ret < 0) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
       return;
@@ -1177,7 +1177,6 @@ void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
   const RGWPubSub ps(driver, s->owner.id.tenant,
                      &s->penv.site->get_period()->get_map().zonegroups);
   std::unordered_map<std::string, rgw_pubsub_topic> topics;
-  const auto rgwbucket = rgw_bucket(s->bucket_tenant, s->bucket_name, "");
   for (const auto& c : configurations.list) {
     const auto& notif_name = c.id;
     if (notif_name.empty()) {
@@ -1212,7 +1211,7 @@ void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
     if (!topics.contains(topic_name)) {
       // get topic information. destination information is stored in the topic
       rgw_pubsub_topic topic_info;
-      op_ret = ps.get_topic(this, topic_name, topic_info, y);
+      op_ret = ps.get_topic(this, topic_name, topic_info, y,nullptr);
       if (op_ret < 0) {
         ldpp_dout(this, 1) << "failed to get topic '" << topic_name
                            << "', ret=" << op_ret << dendl;
@@ -1248,6 +1247,18 @@ void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
         << bucket->get_name() << " returned err= " << op_ret << dendl;
     return;
   }
+  for (const auto& [_, topic] : topics) {
+    const auto ret = driver->update_bucket_topic_mapping(
+        topic,
+        rgw_make_bucket_entry_name(bucket->get_tenant(), bucket->get_name()),
+        /*add_mapping=*/true, y, this);
+    if (ret < 0) {
+      ldpp_dout(this, 1) << "Failed to remove topic mapping on bucket="
+                         << bucket->get_name() << " ret= " << ret << dendl;
+      // error should be reported ??
+      // op_ret = ret;
+    }
+  }
   ldpp_dout(this, 20) << "successfully created bucket notification for bucket: "
                       << bucket->get_name() << dendl;
 }
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 89ac23341d23..4dc3c9dc8365 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -332,6 +332,34 @@ class Driver {
                                 RGWObjVersionTracker* objv_tracker,
                                 optional_yield y,
                                 const DoutPrefixProvider* dpp) = 0;
+    /** Update the bucket-topic mapping in the store, if |add_mapping|=true then
+     * adding the |bucket_key| |topic| mapping to store, else delete the
+     * |bucket_key| |topic| mapping from the store.  The |bucket_key| is
+     * in the format |tenant_name + "/" + bucket_name| if tenant is not empty
+     * else |bucket_name|*/
+    virtual int update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                            const std::string& bucket_key,
+                                            bool add_mapping,
+                                            optional_yield y,
+                                            const DoutPrefixProvider* dpp) = 0;
+    /** Remove the |bucket_key| from bucket-topic mapping in the store, for all
+    the topics under |bucket_topics|*/
+    virtual int remove_bucket_mapping_from_topics(
+        const rgw_pubsub_bucket_topics& bucket_topics,
+        const std::string& bucket_key,
+        optional_yield y,
+        const DoutPrefixProvider* dpp) = 0;
+    /** Get the bucket-topic mapping from the backend store. The |bucket_keys|
+     * are in the format |tenant_name + "/" + bucket_name| if tenant is not
+     * empty else |bucket_name|*/
+    virtual int get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                         std::set<std::string>& bucket_keys,
+                                         optional_yield y,
+                                         const DoutPrefixProvider* dpp) = 0;
+    /** Remove the bucket-topic mapping from the backend store. */
+    virtual int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                            optional_yield y,
+                                            const DoutPrefixProvider* dpp) = 0;
     /** Get access to the lifecycle management thread */
     virtual RGWLC* get_rgwlc(void) = 0;
     /** Get access to the coroutine registry.  Used to create new coroutine managers */
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 71991378e321..d5bf9afe248c 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -218,7 +218,33 @@ class FilterDriver : public Driver {
                       const DoutPrefixProvider* dpp) override {
     return next->remove_topic_v2(topic_name, tenant, objv_tracker, y, dpp);
   }
-
+  int update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                  const std::string& bucket_key,
+                                  bool add_mapping,
+                                  optional_yield y,
+                                  const DoutPrefixProvider* dpp) override {
+    return next->update_bucket_topic_mapping(topic, bucket_key, add_mapping, y,
+                                             dpp);
+  }
+  int remove_bucket_mapping_from_topics(
+      const rgw_pubsub_bucket_topics& bucket_topics,
+      const std::string& bucket_key,
+      optional_yield y,
+      const DoutPrefixProvider* dpp) override {
+    return next->remove_bucket_mapping_from_topics(bucket_topics, bucket_key, y,
+                                                   dpp);
+  }
+  int get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                               std::set<std::string>& bucket_keys,
+                               optional_yield y,
+                               const DoutPrefixProvider* dpp) override {
+    return next->get_bucket_topic_mapping(topic, bucket_keys, y, dpp);
+  }
+  int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                  optional_yield y,
+                                  const DoutPrefixProvider* dpp) override {
+    return next->delete_bucket_topic_mapping(topic, y, dpp);
+  }
   virtual RGWLC* get_rgwlc(void) override;
   virtual RGWCoroutinesManagerRegistry* get_cr_registry() override;
 
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index eda0f08ede6f..b34276a9daaf 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -46,14 +46,39 @@ class StoreDriver : public Driver {
                        RGWObjVersionTracker* objv_tracker,
                        optional_yield y,
                        const DoutPrefixProvider* dpp) override {
-      return -ENOENT;
+      return -EOPNOTSUPP;
     }
     int remove_topic_v2(const std::string& topic_name,
                         const std::string& tenant,
                         RGWObjVersionTracker* objv_tracker,
                         optional_yield y,
                         const DoutPrefixProvider* dpp) override {
-      return -ENOENT;
+      return -EOPNOTSUPP;
+    }
+    int update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                    const std::string& bucket_key,
+                                    bool add_mapping,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp) override {
+      return -EOPNOTSUPP;
+    }
+    int remove_bucket_mapping_from_topics(
+        const rgw_pubsub_bucket_topics& bucket_topics,
+        const std::string& bucket_key,
+        optional_yield y,
+        const DoutPrefixProvider* dpp) override {
+      return -EOPNOTSUPP;
+    }
+    int get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                 std::set<std::string>& bucket_keys,
+                                 optional_yield y,
+                                 const DoutPrefixProvider* dpp) override {
+      return -EOPNOTSUPP;
+    }
+    int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
+                                    optional_yield y,
+                                    const DoutPrefixProvider* dpp) override {
+      return -EOPNOTSUPP;
     }
 };
 
diff --git a/src/rgw/services/svc_topic_rados.cc b/src/rgw/services/svc_topic_rados.cc
index 64c9106776cc..9e0b75d46647 100644
--- a/src/rgw/services/svc_topic_rados.cc
+++ b/src/rgw/services/svc_topic_rados.cc
@@ -10,6 +10,7 @@
 
 static std::string topic_oid_prefix = "topic.";
 static constexpr char topic_tenant_delim[] = ":";
+static std::string bucket_topic_oid_prefix = "buckets.";
 
 std::string get_topic_key(const std::string& topic_name,
                           const std::string& tenant) {
@@ -32,6 +33,11 @@ void parse_topic_entry(const std::string& topic_entry,
     *topic_name = topic_entry;
   }
 }
+
+std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic) {
+  return bucket_topic_oid_prefix + get_topic_key(topic.name, topic.user.tenant);
+}
+
 class RGWSI_Topic_Module : public RGWSI_MBSObj_Handler_Module {
   RGWSI_Topic_RADOS::Svc& svc;
   const std::string prefix;
@@ -131,7 +137,7 @@ int RGWTopicMetadataHandler::do_get(RGWSI_MetaBackend_Handler::Op* op,
   parse_topic_entry(entry, &tenant, &topic_name);
   RGWPubSub ps(driver, tenant,
                &topic_svc->svc.zone->get_current_period().get_map().zonegroups);
-  int ret = ps.get_topic(dpp, topic_name, result, y);
+  int ret = ps.get_topic(dpp, topic_name, result, y, nullptr);
   if (ret < 0) {
     return ret;
   }
diff --git a/src/rgw/services/svc_topic_rados.h b/src/rgw/services/svc_topic_rados.h
index e630a610e974..bc4e35373459 100644
--- a/src/rgw/services/svc_topic_rados.h
+++ b/src/rgw/services/svc_topic_rados.h
@@ -94,3 +94,5 @@ std::string get_topic_key(const std::string& topic_name,
 void parse_topic_entry(const std::string& topic_entry,
                        std::string* tenant_name,
                        std::string* topic_name);
+
+std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic);
\ No newline at end of file

From 1ed8df24ae0f279a6e7d294231b5e6c1e45fb663 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Mon, 11 Dec 2023 15:47:30 -0500
Subject: [PATCH 2149/2492] rgw/multisite-notification: Add integration test
 for bucket notifications in multisite config.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/test/rgw/rgw_multi/conn.py       |  23 +++++
 src/test/rgw/rgw_multi/multisite.py  |   7 +-
 src/test/rgw/rgw_multi/tests.py      | 140 ++++++++++++++++++++++++++-
 src/test/rgw/rgw_multi/zone_cloud.py |  21 ++++
 src/test/rgw/rgw_multi/zone_es.py    |  21 ++++
 src/test/rgw/rgw_multi/zone_rados.py |  28 ++++++
 6 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/src/test/rgw/rgw_multi/conn.py b/src/test/rgw/rgw_multi/conn.py
index 59bc2fdd3d2f..0ef66b0c4ca9 100644
--- a/src/test/rgw/rgw_multi/conn.py
+++ b/src/test/rgw/rgw_multi/conn.py
@@ -1,6 +1,7 @@
 import boto
 import boto.s3.connection
 import boto.iam.connection
+import boto3
 
 def get_gateway_connection(gateway, credentials):
     """ connect to the given gateway """
@@ -39,3 +40,25 @@ def get_gateway_iam_connection(gateway, credentials):
                 port = gateway.port,
                 is_secure = False)
     return gateway.iam_connection
+
+
+def get_gateway_s3_client(gateway, credentials, region):
+  """ connect to boto3 s3 client api of the given gateway """
+  if gateway.s3_client is None:
+      gateway.s3_client = boto3.client('s3',
+                                        endpoint_url='http://' + gateway.host + ':' + str(gateway.port),
+                                        aws_access_key_id=credentials.access_key,
+                                        aws_secret_access_key=credentials.secret,
+                                        region_name=region)
+  return gateway.s3_client
+
+
+def get_gateway_sns_client(gateway, credentials, region):
+  """ connect to boto3 s3 client api of the given gateway """
+  if gateway.sns_client is None:
+      gateway.sns_client = boto3.client('sns',
+                                        endpoint_url='http://' + gateway.host + ':' + str(gateway.port),
+                                        aws_access_key_id=credentials.access_key,
+                                        aws_secret_access_key=credentials.secret,
+                                        region_name=region)
+  return gateway.sns_client
diff --git a/src/test/rgw/rgw_multi/multisite.py b/src/test/rgw/rgw_multi/multisite.py
index 5d4dcd1aa7ae..8642ea3a57ce 100644
--- a/src/test/rgw/rgw_multi/multisite.py
+++ b/src/test/rgw/rgw_multi/multisite.py
@@ -3,7 +3,7 @@
 
 import json
 
-from .conn import get_gateway_connection, get_gateway_iam_connection, get_gateway_secure_connection
+from .conn import get_gateway_connection, get_gateway_iam_connection, get_gateway_secure_connection, get_gateway_s3_client, get_gateway_sns_client
 
 class Cluster:
     """ interface to run commands against a distinct ceph cluster """
@@ -27,6 +27,8 @@ def __init__(self, host = None, port = None, cluster = None, zone = None, ssl_po
         self.secure_connection = None
         self.ssl_port = ssl_port
         self.iam_connection = None
+        self.s3_client = None
+        self.sns_client = None
 
     @abstractmethod
     def start(self, args = []):
@@ -190,6 +192,9 @@ def __init__(self, zone, credentials):
             self.secure_conn = get_gateway_secure_connection(self.zone.gateways[0], self.credentials)
 
             self.iam_conn = get_gateway_iam_connection(self.zone.gateways[0], self.credentials)
+            region = "" if self.zone.zonegroup is None else self.zone.zonegroup.name
+            self.s3_client = get_gateway_s3_client(self.zone.gateways[0], self.credentials, region)
+            self.sns_client = get_gateway_sns_client(self.zone.gateways[0], self.credentials,region)
 
             # create connections for the rest of the gateways (if exist)
             for gw in list(self.zone.gateways):
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 2d0e10916513..70e003c050c0 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -17,7 +17,7 @@
 from boto.s3.cors import CORSConfiguration
 
 from nose.tools import eq_ as eq
-from nose.tools import assert_not_equal, assert_equal
+from nose.tools import assert_not_equal, assert_equal, assert_true, assert_false
 from nose.plugins.attrib import attr
 from nose.plugins.skip import SkipTest
 
@@ -66,6 +66,7 @@ def get_realm():
 run_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
 
 num_roles = 0
+num_topic = 0
 
 def get_zone_connection(zone, credentials):
     """ connect to the zone's first gateway """
@@ -455,6 +456,13 @@ def gen_role_name():
     num_roles += 1
     return "roles" + '-' + run_prefix + '-' + str(num_roles)
 
+
+def gen_topic_name():
+    global num_topic
+
+    num_topic += 1
+    return "topic" + '-' + run_prefix + '-' + str(num_topic)
+
 class ZonegroupConns:
     def __init__(self, zonegroup):
         self.zonegroup = zonegroup
@@ -502,6 +510,34 @@ def check_all_buckets_dont_exist(zone_conn, buckets):
 
     return True
 
+
+def get_topics(zone):
+    """
+    Get list of topics in cluster.
+    """
+    cmd = ['topic', 'list'] + zone.zone_args()
+    topics_json, _ = zone.cluster.admin(cmd, read_only=True)
+    topics = json.loads(topics_json)
+    return topics['topics']
+
+
+def create_topic_per_zone(zonegroup_conns, topics_per_zone=1):
+    topics = []
+    zone_topic = []
+    for zone in zonegroup_conns.rw_zones:
+        for _ in range(topics_per_zone):
+            topic_name = gen_topic_name()
+            log.info('create topic zone=%s name=%s', zone.name, topic_name)
+            attributes = {
+                "push-endpoint": "http://kaboom:9999",
+                "persistent": "true",
+            }
+            topic_arn = zone.create_topic(topic_name, attributes)
+            topics.append(topic_arn)
+            zone_topic.append((zone, topic_arn))
+
+    return topics, zone_topic
+
 def create_role_per_zone(zonegroup_conns, roles_per_zone = 1):
     roles = []
     zone_role = []
@@ -3110,3 +3146,105 @@ def test_sync_flow_symmetrical_zonegroup_all_rgw_down():
         test_sync_flow_symmetrical_zonegroup_all()
     finally:
         start_2nd_rgw(zonegroup)
+
+def test_topic_notification_sync():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_meta_checkpoint(zonegroup)
+    # let wait for users and other settings to sync across all zones.
+    time.sleep(config.checkpoint_delay)
+    # create topics in each zone.
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns)
+    log.debug("topic_arns: %s", topic_arns)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # verify topics exists in all zones
+    for conn in zonegroup_conns.zones:
+        topic_list = conn.list_topics()
+        log.debug("topics for zone=%s = %s", conn.name, topic_list)
+        assert_equal(len(topic_list), len(topic_arns))
+        for topic_arn_map in topic_list:
+            assert_true(topic_arn_map['TopicArn'] in topic_arns)
+
+    # create a bucket
+    bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name())
+    log.debug('created bucket=%s', bucket.name)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # create bucket_notification in each zone.
+    notification_ids = []
+    num = 1
+    for zone_conn, topic_arn in zone_topic:
+        noti_id = "bn" + '-' + run_prefix + '-' + str(num)
+        notification_ids.append(noti_id)
+        topic_conf = {'Id': noti_id,
+                      'TopicArn': topic_arn,
+                      'Events': ['s3:ObjectCreated:*']
+                     }
+        num += 1
+        log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id)
+        zone_conn.create_notification(bucket.name, [topic_conf])
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # verify notifications exists in all zones
+    for conn in zonegroup_conns.zones:
+        notification_list = conn.list_notifications(bucket.name)
+        log.debug("notifications for zone=%s = %s", conn.name, notification_list)
+        assert_equal(len(notification_list), len(topic_arns))
+        for notification in notification_list:
+            assert_true(notification['Id'] in notification_ids)
+
+    # verify bucket_topic mapping
+    # create a new bucket and subcribe it to first topic.
+    bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name())
+    notif_id = "bn-2" + '-' + run_prefix
+    topic_conf = {'Id': notif_id,
+                  'TopicArn': topic_arns[0],
+                  'Events': ['s3:ObjectCreated:*']
+                  }
+    zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf])
+    zonegroup_meta_checkpoint(zonegroup)
+    for conn in zonegroup_conns.zones:
+        topics = get_topics(conn.zone)
+        for topic in topics:
+            if topic['arn'] == topic_arns[0]:
+                assert_equal(len(topic['subscribed_buckets']), 2)
+                assert_true(bucket_2.name in topic['subscribed_buckets'])
+            else:
+                assert_equal(len(topic['subscribed_buckets']), 1)
+            assert_true(bucket.name in topic['subscribed_buckets'])
+
+    # delete the 2nd bucket and verify the mapping is removed.
+    zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name)
+    zonegroup_meta_checkpoint(zonegroup)
+    for conn in zonegroup_conns.zones:
+        topics = get_topics(conn.zone)
+        for topic in topics:
+            assert_equal(len(topic['subscribed_buckets']), 1)
+        '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802
+           is fixed, as the secondary site bucket instance info is currently not
+           getting deleted coz of the bug hence the bucket-topic mapping
+           deletion is not invoked on secondary sites.)'''
+        break
+
+    # delete notifications
+    zonegroup_conns.rw_zones[0].delete_notifications(bucket.name)
+    log.debug('Deleting all notifications for  bucket=%s', bucket.name)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # verify notification deleted in all zones
+    for conn in zonegroup_conns.zones:
+        notification_list = conn.list_notifications(bucket.name)
+        assert_equal(len(notification_list), 0)
+
+    # delete topics
+    for zone_conn, topic_arn in zone_topic:
+        log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn)
+        zone_conn.delete_topic(topic_arn)
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # verify topics deleted in all zones
+    for conn in zonegroup_conns.zones:
+        topic_list = conn.list_topics()
+        assert_equal(len(topic_list), 0)
diff --git a/src/test/rgw/rgw_multi/zone_cloud.py b/src/test/rgw/rgw_multi/zone_cloud.py
index 7c94aaa8a60f..fdec751ff522 100644
--- a/src/test/rgw/rgw_multi/zone_cloud.py
+++ b/src/test/rgw/rgw_multi/zone_cloud.py
@@ -310,6 +310,27 @@ def delete_role(self, role_name):
         def has_role(self, role_name):
             assert False
 
+        def create_topic(self, topicname, attributes):
+            assert False
+
+        def delete_topic(self, topic_arn):
+            assert False
+
+        def get_topic(self, topic_arn):
+            assert False
+
+        def list_topics(self):
+            assert False
+
+        def create_notification(self, bucket_name, config):
+            assert False
+
+        def delete_notifications(self, bucket_name):
+            assert False
+
+        def list_notifications(self, bucket_name):
+            assert False
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 
diff --git a/src/test/rgw/rgw_multi/zone_es.py b/src/test/rgw/rgw_multi/zone_es.py
index 84628b775d1c..2ccdcf042332 100644
--- a/src/test/rgw/rgw_multi/zone_es.py
+++ b/src/test/rgw/rgw_multi/zone_es.py
@@ -252,6 +252,27 @@ def delete_role(self, role_name):
         def has_role(self, role_name):
             assert False
 
+        def create_topic(self, topicname, attributes):
+            assert False
+
+        def delete_topic(self, topic_arn):
+            assert False
+
+        def list_topics(self):
+            assert False
+
+        def get_topic(self, topic_arn):
+            assert False
+
+        def create_notification(self, bucket_name, config):
+            assert False
+
+        def delete_notification(self, bucket_name):
+            assert False
+
+        def list_notifications(self, bucket_name):
+            assert False
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 
diff --git a/src/test/rgw/rgw_multi/zone_rados.py b/src/test/rgw/rgw_multi/zone_rados.py
index 7b7fe5228cbf..3761676a3d78 100644
--- a/src/test/rgw/rgw_multi/zone_rados.py
+++ b/src/test/rgw/rgw_multi/zone_rados.py
@@ -142,6 +142,34 @@ def has_role(self, role_name):
                 return False
             return True
 
+        def create_topic(self, topicname, attributes):
+            result = self.sns_client.create_topic(Name=topicname, Attributes=attributes)
+            self.topic_arn = result['TopicArn']
+            return self.topic_arn
+
+        def delete_topic(self, topic_arn):
+            return self.sns_client.delete_topic(TopicArn=topic_arn)
+
+        def get_topic(self, topic_arn):
+            return self.sns_client.get_topic_attributes(TopicArn=topic_arn)
+
+        def list_topics(self):
+            return self.sns_client.list_topics()['Topics']
+
+        def create_notification(self, bucket_name, topic_conf_list):
+            return self.s3_client.put_bucket_notification_configuration(
+                Bucket=bucket_name, NotificationConfiguration={'TopicConfigurations': topic_conf_list})
+
+        def delete_notifications(self, bucket_name):
+            return self.s3_client.put_bucket_notification_configuration(Bucket=bucket_name,
+                                                                        NotificationConfiguration={})
+
+        def list_notifications(self, bucket_name):
+            out = self.s3_client.get_bucket_notification_configuration(Bucket=bucket_name)
+            if 'TopicConfigurations' in out:
+              return out['TopicConfigurations']
+            return []
+
     def get_conn(self, credentials):
         return self.Conn(self, credentials)
 

From 7502215d70a423e642e5147a0d444f627a7979f2 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 21 Dec 2023 18:18:45 +0000
Subject: [PATCH 2150/2492] test/rgw/notifications: support running tests in
 multisite environment

both locally and in teuthology

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 qa/tasks/notification_tests.py                |   2 +
 src/test/rgw/bucket_notification/README.rst   |  14 ++
 src/test/rgw/bucket_notification/__init__.py  |  16 +-
 src/test/rgw/bucket_notification/api.py       |   4 +-
 .../bucket_notification/bntests.conf.SAMPLE   |   2 +
 src/test/rgw/bucket_notification/test_bn.py   | 153 +++++++++---------
 6 files changed, 110 insertions(+), 81 deletions(-)

diff --git a/qa/tasks/notification_tests.py b/qa/tasks/notification_tests.py
index f7b91e10b903..86e95dfb5f29 100644
--- a/qa/tasks/notification_tests.py
+++ b/qa/tasks/notification_tests.py
@@ -298,6 +298,8 @@ def task(ctx,config):
                     {
                     'port':endpoint.port,
                     'host':endpoint.dns_name,
+                    'zonegroup':'default',
+                    'cluster':'noname'
                     },
                 's3 main':{}
             }
diff --git a/src/test/rgw/bucket_notification/README.rst b/src/test/rgw/bucket_notification/README.rst
index 20eee8463cf1..db34545e1d53 100644
--- a/src/test/rgw/bucket_notification/README.rst
+++ b/src/test/rgw/bucket_notification/README.rst
@@ -9,6 +9,20 @@ with the `vstart.sh` script.
 For the tests covering Kafka and RabbitMQ security, the RGW will need to accept use/password without TLS connection between the client and the RGW.
 So, the cluster will have to be started with the following ``rgw_allow_notification_secrets_in_cleartext`` parameter set to ``true``.
 
+The test suite can be run against a multisite setup, in the configuration file we will have to decide which RGW and which cluster will be used for the test.
+For example, if the ``test-rgw-multisite.sh`` script is used to setup multisite, and we want to run the test agianst the first RGW in the first cluster, 
+we would need the following configuration file::
+
+				[DEFAULT]
+				port = 8101
+				host = localhost
+				zonegroup = zg1
+				cluster = c1
+
+				[s3 main]
+				access_key = 1234567890
+				secret_key = pencil
+
 
 ===========
 Kafka Tests
diff --git a/src/test/rgw/bucket_notification/__init__.py b/src/test/rgw/bucket_notification/__init__.py
index 6785fce92634..5d8fac8e215c 100644
--- a/src/test/rgw/bucket_notification/__init__.py
+++ b/src/test/rgw/bucket_notification/__init__.py
@@ -25,9 +25,15 @@ def setup():
     global default_port
     default_port = int(defaults.get("port"))
 
+    global default_zonegroup
+    default_zonegroup = defaults.get("zonegroup")
+
+    global default_cluster
+    default_cluster = defaults.get("cluster")
+
     global main_access_key
     main_access_key = cfg.get('s3 main',"access_key")
-    
+
     global main_secret_key
     main_secret_key = cfg.get('s3 main',"secret_key")
 
@@ -39,6 +45,14 @@ def get_config_port():
     global default_port
     return default_port
 
+def get_config_zonegroup():
+    global default_zonegroup
+    return default_zonegroup
+
+def get_config_cluster():
+    global default_cluster
+    return default_cluster
+
 def get_access_key():
     global main_access_key
     return main_access_key
diff --git a/src/test/rgw/bucket_notification/api.py b/src/test/rgw/bucket_notification/api.py
index 0b53d32c7365..c23b7bb910ab 100644
--- a/src/test/rgw/bucket_notification/api.py
+++ b/src/test/rgw/bucket_notification/api.py
@@ -236,8 +236,8 @@ def bash(cmd, **kwargs):
     s = process.communicate()[0].decode('utf-8')
     return (s, process.returncode)
 
-def admin(args, **kwargs):
+def admin(args, cluster='noname', **kwargs):
     """ radosgw-admin command """
-    cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', 'noname'] + args
+    cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', cluster] + args
     return bash(cmd, **kwargs)
 
diff --git a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
index eb3291dafa8e..998fcd1ef842 100644
--- a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
+++ b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
@@ -1,6 +1,8 @@
 [DEFAULT]
 port = 8000
 host = localhost
+zonegroup = default
+cluster = noname
 
 [s3 main]
 access_key = 0555b35654ad1656d804
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 30cbfdfe7865..8e9ebd126c63 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -9,6 +9,7 @@
 import os
 import io
 import string
+# XXX this should be converted to use boto3
 import boto
 from botocore.exceptions import ClientError
 from http import server as http_server
@@ -17,16 +18,16 @@
 # XXX this should be converted to use pytest
 from nose.plugins.attrib import attr
 import boto3
+from boto.s3.connection import S3Connection
 import datetime
 from cloudevents.http import from_http
 from dateutil import parser
 
-# XXX this should be converted to use boto3
-from boto.s3.connection import S3Connection
-
 from . import(
     get_config_host,
     get_config_port,
+    get_config_zonegroup,
+    get_config_cluster,
     get_access_key,
     get_secret_key
     )
@@ -48,7 +49,6 @@
 NOTIFICATION_SUFFIX = "_notif"
 UID_PREFIX = "superman"
 
-
 num_buckets = 0
 run_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
 
@@ -538,9 +538,9 @@ def another_user(tenant=None):
     secret_key = str(time.time())
     uid = UID_PREFIX + str(time.time())
     if tenant:
-        _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+        _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'], get_config_cluster())
     else:
-        _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+        _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'], get_config_cluster())
 
     assert_equal(result, 0)
     conn = S3Connection(aws_access_key_id=access_key,
@@ -560,15 +560,15 @@ def test_ps_s3_topic_on_master():
     
     access_key = str(time.time())
     secret_key = str(time.time())
-    uid = 'superman' + str(time.time())
+    uid = UID_PREFIX + str(time.time())
     tenant = 'kaboom'
-    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'], get_config_cluster())
     assert_equal(result, 0)
     conn = S3Connection(aws_access_key_id=access_key,
                   aws_secret_access_key=secret_key,
                       is_secure=False, port=get_config_port(), host=get_config_host(), 
                       calling_format='boto.s3.connection.OrdinaryCallingFormat')
-    zonegroup = 'default' 
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
 
@@ -641,15 +641,15 @@ def test_ps_s3_topic_admin_on_master():
     
     access_key = str(time.time())
     secret_key = str(time.time())
-    uid = 'superman' + str(time.time())
+    uid = UID_PREFIX + str(time.time())
     tenant = 'kaboom'
-    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])  
+    _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'], get_config_cluster())
     assert_equal(result, 0)
     conn = S3Connection(aws_access_key_id=access_key,
                   aws_secret_access_key=secret_key,
                       is_secure=False, port=get_config_port(), host=get_config_host(), 
                       calling_format='boto.s3.connection.OrdinaryCallingFormat')
-    zonegroup = 'default' 
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
 
@@ -686,34 +686,34 @@ def test_ps_s3_topic_admin_on_master():
                  'arn:aws:sns:' + zonegroup + ':' + tenant + ':' + topic_name + '_3')
 
     # get topic 3 via commandline
-    result = admin(['topic', 'get', '--topic', topic_name+'_3', '--tenant', tenant])  
+    result = admin(['topic', 'get', '--topic', topic_name+'_3', '--tenant', tenant], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['arn'], topic_arn3)
     matches = [tenant, UID_PREFIX]
     assert_true( all([x in parsed_result['user'] for x in matches]))
 
     # delete topic 3
-    _, result = admin(['topic', 'rm', '--topic', topic_name+'_3', '--tenant', tenant])  
+    _, result = admin(['topic', 'rm', '--topic', topic_name+'_3', '--tenant', tenant], get_config_cluster())
     assert_equal(result, 0)
 
     # try to get a deleted topic
-    _, result = admin(['topic', 'get', '--topic', topic_name+'_3', '--tenant', tenant])  
+    _, result = admin(['topic', 'get', '--topic', topic_name+'_3', '--tenant', tenant], get_config_cluster())
     print('"topic not found" error is expected')
     assert_equal(result, 2)
 
     # get the remaining 2 topics
-    result = admin(['topic', 'list', '--tenant', tenant])  
+    result = admin(['topic', 'list', '--tenant', tenant], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(len(parsed_result['topics']), 2)
 
     # delete topics
-    _, result = admin(['topic', 'rm', '--topic', topic_name+'_1', '--tenant', tenant])  
+    _, result = admin(['topic', 'rm', '--topic', topic_name+'_1', '--tenant', tenant], get_config_cluster())
     assert_equal(result, 0)
-    _, result = admin(['topic', 'rm', '--topic', topic_name+'_2', '--tenant', tenant])  
+    _, result = admin(['topic', 'rm', '--topic', topic_name+'_2', '--tenant', tenant], get_config_cluster())
     assert_equal(result, 0)
 
     # get topic list, make sure it is empty
-    result = admin(['topic', 'list', '--tenant', tenant])  
+    result = admin(['topic', 'list', '--tenant', tenant], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(len(parsed_result['topics']), 0)
 
@@ -722,7 +722,7 @@ def test_ps_s3_topic_admin_on_master():
 def test_ps_s3_notification_configuration_admin_on_master():
     """ test s3 notification list/get/delete on master """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     bucket = conn.create_bucket(bucket_name)
     topic_name = bucket_name + TOPIC_SUFFIX
@@ -764,33 +764,33 @@ def test_ps_s3_notification_configuration_admin_on_master():
     assert_equal(status/100, 2)
 
     # list notification
-    result = admin(['notification', 'list', '--bucket', bucket_name])
+    result = admin(['notification', 'list', '--bucket', bucket_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(len(parsed_result['notifications']), 3)
     assert_equal(result[1], 0)
 
     # get notification 1
-    result = admin(['notification', 'get', '--bucket', bucket_name, '--notification-id', notification_name+'_1'])
+    result = admin(['notification', 'get', '--bucket', bucket_name, '--notification-id', notification_name+'_1'], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Id'], notification_name+'_1')
     assert_equal(result[1], 0)
 
     # remove notification 3
-    _, result = admin(['notification', 'rm', '--bucket', bucket_name, '--notification-id', notification_name+'_3'])
+    _, result = admin(['notification', 'rm', '--bucket', bucket_name, '--notification-id', notification_name+'_3'], get_config_cluster())
     assert_equal(result, 0)
 
     # list notification
-    result = admin(['notification', 'list', '--bucket', bucket_name])
+    result = admin(['notification', 'list', '--bucket', bucket_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(len(parsed_result['notifications']), 2)
     assert_equal(result[1], 0)
 
     # delete notifications
-    _, result = admin(['notification', 'rm', '--bucket', bucket_name])
+    _, result = admin(['notification', 'rm', '--bucket', bucket_name], get_config_cluster())
     assert_equal(result, 0)
 
     # list notification, make sure it is empty
-    result = admin(['notification', 'list', '--bucket', bucket_name])
+    result = admin(['notification', 'list', '--bucket', bucket_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(len(parsed_result['notifications']), 0)
     assert_equal(result[1], 0)
@@ -805,7 +805,7 @@ def test_ps_s3_topic_with_secret_on_master():
     if conn.secure_conn is None:
         return SkipTest('secure connection is needed to test topic with secrets')
 
-    zonegroup = 'default' 
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
 
@@ -851,7 +851,7 @@ def test_ps_s3_topic_with_secret_on_master():
 def test_ps_s3_notification_on_master():
     """ test s3 notification set/get/delete on master """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     # create bucket
     bucket = conn.create_bucket(bucket_name)
@@ -915,7 +915,7 @@ def test_ps_s3_notification_on_master_empty_config():
 
     conn = connection()
 
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -974,7 +974,7 @@ def test_ps_s3_notification_filter_on_master():
     conn = connection()
     ps_zone = conn
 
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -1146,7 +1146,7 @@ def test_ps_s3_notification_filter_on_master():
 def test_ps_s3_notification_errors_on_master():
     """ test s3 notification set/get/delete on master """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     # create bucket
     bucket = conn.create_bucket(bucket_name)
@@ -1245,7 +1245,7 @@ def test_ps_s3_notification_push_amqp_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -1349,7 +1349,7 @@ def test_ps_s3_notification_push_amqp_idleness_check():
     return SkipTest("only used in manual testing")
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -1478,7 +1478,7 @@ def test_ps_s3_notification_push_amqp_idleness_check():
 def test_ps_s3_notification_push_kafka_on_master():
     """ test pushing kafka s3 notification on master """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -1578,7 +1578,7 @@ def test_ps_s3_notification_multi_delete_on_master():
     """ test deletion of multiple keys on master """
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -1646,7 +1646,7 @@ def test_ps_s3_notification_push_http_on_master():
     """ test pushing http s3 notification on master """
     hostname = get_ip_http()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -1730,7 +1730,7 @@ def test_ps_s3_notification_push_cloudevents_on_master():
     """ test pushing cloudevents notification on master """
     hostname = get_ip_http()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -1814,7 +1814,7 @@ def test_ps_s3_opaque_data_on_master():
     """ test that opaque id set in topic, is sent in notification on master """
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -1883,7 +1883,7 @@ def test_ps_s3_lifecycle_on_master():
     """ test that when object is deleted due to lifecycle policy, notification is sent on master """
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -1947,7 +1947,7 @@ def test_ps_s3_lifecycle_on_master():
     )
 
     # start lifecycle processing
-    admin(['lc', 'process'])
+    admin(['lc', 'process'], get_config_cluster())
     print('wait for 5sec for the messages...')
     time.sleep(5)
 
@@ -1994,7 +1994,7 @@ def test_ps_s3_lifecycle_abort_mpu_on_master():
     """ test that when a multipart upload is aborted by lifecycle policy, notification is sent on master """
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -2056,7 +2056,7 @@ def test_ps_s3_lifecycle_abort_mpu_on_master():
     )
 
     # start lifecycle processing
-    admin(['lc', 'process'])
+    admin(['lc', 'process'], get_config_cluster())
     print('wait for 20s (2 days) for the messages...')
     time.sleep(20)
 
@@ -2093,7 +2093,7 @@ def ps_s3_creation_triggers_on_master(external_endpoint_address=None, ca_locatio
     """ test object creation s3 notifications in using put/copy/post on master"""
     
     if not external_endpoint_address:
-        hostname = 'localhost'
+        hostname = get_ip()
         proc = init_rabbitmq()
         if proc is  None:
             return SkipTest('end2end amqp tests require rabbitmq-server installed')
@@ -2101,8 +2101,7 @@ def ps_s3_creation_triggers_on_master(external_endpoint_address=None, ca_locatio
         proc = None
 
     conn = connection()
-    hostname = 'localhost'
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2335,7 +2334,7 @@ def test_http_post_object_upload():
     import requests
 
     hostname = get_ip()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     conn = connection()
 
     endpoint = "http://%s:%d" % (get_config_host(), get_config_port())
@@ -2467,7 +2466,7 @@ def test_ps_s3_multipart_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2556,7 +2555,7 @@ def test_ps_s3_metadata_filter_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2660,7 +2659,7 @@ def test_ps_s3_metadata_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2754,7 +2753,7 @@ def test_ps_s3_tags_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2865,7 +2864,7 @@ def test_ps_s3_versioning_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -2940,7 +2939,7 @@ def test_ps_s3_versioned_deletion_on_master():
 
     hostname = get_ip()
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -3039,7 +3038,7 @@ def test_ps_s3_persistent_cleanup():
     """ test reservation cleanup after gateway crash """
     return SkipTest("only used in manual testing")
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -3142,7 +3141,7 @@ def test_ps_s3_persistent_cleanup():
 def test_ps_s3_persistent_topic_stats():
     """ test persistent topic stats """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -3176,7 +3175,7 @@ def test_ps_s3_persistent_topic_stats():
     http_server.close()
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 0)
     assert_equal(result[1], 0)
@@ -3196,7 +3195,7 @@ def test_ps_s3_persistent_topic_stats():
     print('average time for creation + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], number_of_objects)
     assert_equal(result[1], 0)
@@ -3218,7 +3217,7 @@ def test_ps_s3_persistent_topic_stats():
             start_time = time.time()
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 2*number_of_objects)
     assert_equal(result[1], 0)
@@ -3230,7 +3229,7 @@ def test_ps_s3_persistent_topic_stats():
     time.sleep(delay)
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 0)
     assert_equal(result[1], 0)
@@ -3245,7 +3244,7 @@ def test_ps_s3_persistent_topic_stats():
 
 def ps_s3_persistent_topic_configs(persistency_time, config_dict):
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -3278,7 +3277,7 @@ def ps_s3_persistent_topic_configs(persistency_time, config_dict):
     time.sleep(delay)
     http_server.close()
     # topic get
-    result = admin(['topic', 'get', '--topic', topic_name])
+    result = admin(['topic', 'get', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     parsed_result_dest = parsed_result["dest"]
     for key, value in config_dict.items():
@@ -3286,7 +3285,7 @@ def ps_s3_persistent_topic_configs(persistency_time, config_dict):
     assert_equal(result[1], 0)
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 0)
     assert_equal(result[1], 0)
@@ -3306,14 +3305,14 @@ def ps_s3_persistent_topic_configs(persistency_time, config_dict):
     print('average time for creation + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], number_of_objects)
     assert_equal(result[1], 0)
 
     # wait as much as ttl and check if the persistent topics have expired
     time.sleep(persistency_time)
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 0)
     assert_equal(result[1], 0)
@@ -3335,14 +3334,14 @@ def ps_s3_persistent_topic_configs(persistency_time, config_dict):
             start_time = time.time()
 
     # topic stats
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], number_of_objects)
     assert_equal(result[1], 0)
 
     # wait as much as ttl and check if the persistent topics have expired
     time.sleep(persistency_time)
-    result = admin(['topic', 'stats', '--topic', topic_name])
+    result = admin(['topic', 'stats', '--topic', topic_name], get_config_cluster())
     parsed_result = json.loads(result[0])
     assert_equal(parsed_result['Topic Stats']['Entries'], 0)
     assert_equal(result[1], 0)
@@ -3385,7 +3384,7 @@ def test_ps_s3_persistent_notification_pushback():
     """ test pushing persistent notification pushback """
     return SkipTest("only used in manual testing")
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -3465,10 +3464,8 @@ def test_ps_s3_persistent_notification_pushback():
 @attr('kafka_test')
 def test_ps_s3_notification_kafka_idle_behaviour():
     """ test pushing kafka s3 notification idle behaviour check """
-    # TODO convert this test to actual running test by changing
-    # os.system call to verify the process idleness
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -3602,7 +3599,7 @@ def test_ps_s3_persistent_gateways_recovery():
     return SkipTest('This test requires two gateways.')
 
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     # create random port for the http server
     host = get_ip()
     port = random.randint(10000, 20000)
@@ -3689,7 +3686,7 @@ def test_ps_s3_persistent_multiple_gateways():
     return SkipTest('This test requires two gateways.')
 
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     # create random port for the http server
     host = get_ip()
     port = random.randint(10000, 20000)
@@ -3798,7 +3795,7 @@ def test_ps_s3_persistent_multiple_gateways():
 def test_ps_s3_persistent_multiple_endpoints():
     """ test pushing persistent notification when one of the endpoints has error """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create random port for the http server
     host = get_ip()
@@ -3881,7 +3878,7 @@ def test_ps_s3_persistent_multiple_endpoints():
 def persistent_notification(endpoint_type):
     """ test pushing persistent notification """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -4014,7 +4011,7 @@ def test_ps_s3_persistent_notification_large():
     """ test pushing persistent notification of large notifications """
 
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
 
     # create bucket
     bucket_name = gen_bucket_name()
@@ -4215,7 +4212,7 @@ def test_ps_s3_notification_update():
     bucket_name = gen_bucket_name()
     topic_name1 = bucket_name+'amqp'+TOPIC_SUFFIX
     topic_name2 = bucket_name+'http'+TOPIC_SUFFIX
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     # create topics
     # start amqp receiver in a separate thread
     exchange = 'ex1'
@@ -4299,7 +4296,7 @@ def test_ps_s3_multiple_topics_notification():
     return SkipTest('This test is yet to be modified.')
 
     hostname = get_ip()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     conn = connection()
     ps_zone = None
     bucket_name = gen_bucket_name()
@@ -4401,7 +4398,7 @@ def test_ps_s3_topic_permissions():
     """ test s3 topic set/get/delete permissions """
     conn1 = connection()
     conn2 = another_user()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     bucket_name = gen_bucket_name()
     topic_name = bucket_name + TOPIC_SUFFIX
     topic_policy = json.dumps({
@@ -4582,7 +4579,7 @@ def test_ps_s3_topic_no_permissions():
 def kafka_security(security_type, mechanism='PLAIN'):
     """ test pushing kafka s3 notification securly to master """
     conn = connection()
-    zonegroup = 'default'
+    zonegroup = get_config_zonegroup()
     # create bucket
     bucket_name = gen_bucket_name()
     bucket = conn.create_bucket(bucket_name)

From d2db9df224659cf711aee80479b5d462ff3ca716 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Thu, 28 Dec 2023 12:05:09 +0000
Subject: [PATCH 2151/2492] rgw/multisite-notification: allow enabling v1/v2 in
 tests

v1 could be enabled only in local tests. teuthology tests would run with v2

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 qa/tasks/notification_tests.py                       |  3 ++-
 src/test/rgw/bucket_notification/README.rst          |  6 ++++--
 src/test/rgw/bucket_notification/__init__.py         | 12 ++++++++++++
 src/test/rgw/bucket_notification/bntests.conf.SAMPLE |  1 +
 4 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/qa/tasks/notification_tests.py b/qa/tasks/notification_tests.py
index 86e95dfb5f29..129937cee456 100644
--- a/qa/tasks/notification_tests.py
+++ b/qa/tasks/notification_tests.py
@@ -299,7 +299,8 @@ def task(ctx,config):
                     'port':endpoint.port,
                     'host':endpoint.dns_name,
                     'zonegroup':'default',
-                    'cluster':'noname'
+                    'cluster':'noname',
+                    'version':'v2'
                     },
                 's3 main':{}
             }
diff --git a/src/test/rgw/bucket_notification/README.rst b/src/test/rgw/bucket_notification/README.rst
index db34545e1d53..050a2e380c29 100644
--- a/src/test/rgw/bucket_notification/README.rst
+++ b/src/test/rgw/bucket_notification/README.rst
@@ -9,8 +9,9 @@ with the `vstart.sh` script.
 For the tests covering Kafka and RabbitMQ security, the RGW will need to accept use/password without TLS connection between the client and the RGW.
 So, the cluster will have to be started with the following ``rgw_allow_notification_secrets_in_cleartext`` parameter set to ``true``.
 
-The test suite can be run against a multisite setup, in the configuration file we will have to decide which RGW and which cluster will be used for the test.
-For example, if the ``test-rgw-multisite.sh`` script is used to setup multisite, and we want to run the test agianst the first RGW in the first cluster, 
+The test suite can be run against a multisite setup, in the configuration file we will have to decide which RGW and which cluster will be used for the test,
+and using which version (v1 or v2) of topics/notifications we should use. By default, a new cluster would use v2, which means that if a realm was never defined, v1 cannot be used.
+For example, if the ``test-rgw-multisite.sh`` script is used to setup multisite, and we want to test v1 against the first RGW in the first cluster, 
 we would need the following configuration file::
 
 				[DEFAULT]
@@ -18,6 +19,7 @@ we would need the following configuration file::
 				host = localhost
 				zonegroup = zg1
 				cluster = c1
+				version = v1
 
 				[s3 main]
 				access_key = 1234567890
diff --git a/src/test/rgw/bucket_notification/__init__.py b/src/test/rgw/bucket_notification/__init__.py
index 5d8fac8e215c..222af7489f30 100644
--- a/src/test/rgw/bucket_notification/__init__.py
+++ b/src/test/rgw/bucket_notification/__init__.py
@@ -1,5 +1,6 @@
 import configparser
 import os
+from .api import admin
 
 def setup():
     cfg = configparser.RawConfigParser()
@@ -30,6 +31,17 @@ def setup():
 
     global default_cluster
     default_cluster = defaults.get("cluster")
+    
+    version = defaults.get("version")
+    if version == "v1":
+        _, result = admin(['zonegroup', 'modify', '--disable-feature=notification_v2'], default_cluster)
+        if result != 0:
+            raise RuntimeError('Failed to disable v2 notifications feature. error: '+str(result))
+        _, result = admin(['period', 'update', '--commit'], default_cluster)
+        if result != 0:
+            raise RuntimeError('Failed to commit changes to period. error: '+str(result))
+    elif version != "v2":
+        raise RuntimeError('Invalid notification version: '+version)
 
     global main_access_key
     main_access_key = cfg.get('s3 main',"access_key")
diff --git a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
index 998fcd1ef842..2ab614c5a3e7 100644
--- a/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
+++ b/src/test/rgw/bucket_notification/bntests.conf.SAMPLE
@@ -3,6 +3,7 @@ port = 8000
 host = localhost
 zonegroup = default
 cluster = noname
+version = v2
 
 [s3 main]
 access_key = 0555b35654ad1656d804

From 0f333f1cc5a265015d4d436e1275ffadc2d0d8d4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 4 Feb 2024 11:37:57 -0500
Subject: [PATCH 2152/2492] radosgw-admin: make SiteConfig available to
 commands

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_admin.cc | 21 +--------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index bbd9231893fb..be91c66e9ff8 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -4093,6 +4093,7 @@ int main(int argc, const char **argv)
   common_init_finish(g_ceph_context);
 
   std::unique_ptr<rgw::sal::ConfigStore> cfgstore;
+  std::unique_ptr<rgw::SiteConfig> site;
 
   if (args.empty()) {
     usage();
@@ -4279,8 +4280,6 @@ int main(int argc, const char **argv)
       return EIO;
     }
 
-    std::unique_ptr<rgw::SiteConfig> site;
-
     if (raw_storage_op) {
       site = rgw::SiteConfig::make_fake();
       driver = DriverManager::get_raw_storage(dpp(), g_ceph_context,
@@ -10651,12 +10650,6 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::PUBSUB_TOPIC_LIST) {
-    auto site = std::make_unique<rgw::SiteConfig>();
-    ret = site->load(dpp(), null_yield, cfgstore.get());
-    if (ret < 0) {
-      std::cerr << "Unable to initialize site config." << std::endl;
-      exit(1);
-    }
     RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
     rgw_pubsub_topics result;
     ret = ps.get_topics(dpp(), result, null_yield);
@@ -10700,12 +10693,6 @@ int main(int argc, const char **argv)
       cerr << "ERROR: topic name was not provided (via --topic)" << std::endl;
       return EINVAL;
     }
-    auto site = std::make_unique<rgw::SiteConfig>();
-    ret = site->load(dpp(), null_yield, cfgstore.get());
-    if (ret < 0) {
-      std::cerr << "Unable to initialize site config." << std::endl;
-      exit(1);
-    }
     RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
 
     rgw_pubsub_topic topic;
@@ -10783,12 +10770,6 @@ int main(int argc, const char **argv)
       return -ret;
     }
 
-    auto site = std::make_unique<rgw::SiteConfig>();
-    ret = site->load(dpp(), null_yield, cfgstore.get());
-    if (ret < 0) {
-      std::cerr << "Unable to initialize site config." << std::endl;
-      exit(1);
-    }
     RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
 
     ret = ps.remove_topic(dpp(), topic_name, null_yield);

From 0057bb7d0f02974f3a8cb1d34e73ead8d92c3062 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 4 Feb 2024 13:43:10 -0500
Subject: [PATCH 2153/2492] rgw/rados: store SiteConfig with RGWServices

make the SiteConfig available to all of RGWRados via svc.site instead
of storing it in sal::RadosStore

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rados.cc     | 24 ++++++++++--------------
 src/rgw/driver/rados/rgw_rados.h      | 11 ++++-------
 src/rgw/driver/rados/rgw_sal_rados.cc |  5 ++---
 src/rgw/driver/rados/rgw_sal_rados.h  |  7 ++-----
 src/rgw/driver/rados/rgw_service.cc   |  3 ++-
 src/rgw/driver/rados/rgw_service.h    | 14 +++++++++-----
 src/rgw/rgw_sal.cc                    | 15 +++++++--------
 src/test/rgw/test_rgw_iam_policy.cc   |  3 +--
 src/test/rgw/test_rgw_lua.cc          | 15 ++++-----------
 9 files changed, 41 insertions(+), 56 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b8fcfa2f368f..18ff24f21813 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1366,29 +1366,25 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
   return ret;
 }
 
-int RGWRados::init_svc(bool raw, const DoutPrefixProvider *dpp)
+int RGWRados::init_svc(bool raw, const DoutPrefixProvider *dpp,
+                       const rgw::SiteConfig& site)
 {
   if (raw) {
-    return svc.init_raw(cct, driver, use_cache, null_yield, dpp);
+    return svc.init_raw(cct, driver, use_cache, null_yield, dpp, site);
   }
 
-  return svc.init(cct, driver, use_cache, run_sync_thread, null_yield, dpp);
-}
-
-int RGWRados::init_ctl(const DoutPrefixProvider *dpp)
-{
-  return ctl.init(&svc, driver, dpp);
+  return svc.init(cct, driver, use_cache, run_sync_thread, null_yield, dpp, site);
 }
 
 /** 
  * Initialize the RADOS instance and prepare to do other ops
  * Returns 0 on success, -ERR# on failure.
  */
-int RGWRados::init_begin(const DoutPrefixProvider *dpp)
+int RGWRados::init_begin(CephContext* _cct, const DoutPrefixProvider *dpp,
+                         const rgw::SiteConfig& site)
 {
-  int ret;
-
-  ret = driver->init_neorados(dpp);
+  set_context(_cct);
+  int ret = driver->init_neorados(dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to initialize neorados (ret=" << cpp_strerror(-ret) << ")" << dendl;
     return ret;
@@ -1399,13 +1395,13 @@ int RGWRados::init_begin(const DoutPrefixProvider *dpp)
     return ret;
   }
 
-  ret = init_svc(false, dpp);
+  ret = init_svc(false, dpp, site);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl;
     return ret;
   }
 
-  ret = init_ctl(dpp);
+  ret = ctl.init(&svc, driver, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to init ctls (ret=" << cpp_strerror(-ret) << ")" << dendl;
     return ret;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 7e7a58480a65..f43b1b4e5314 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -58,6 +58,7 @@ struct RGWZoneGroup;
 struct RGWZoneParams;
 class RGWReshard;
 class RGWReshardWait;
+namespace rgw { class SiteConfig; }
 
 struct get_obj_data;
 
@@ -593,15 +594,11 @@ class RGWRados
 
   CephContext *ctx() { return cct; }
   /** do all necessary setup of the storage device */
-  int init_begin(CephContext *_cct, const DoutPrefixProvider *dpp) {
-    set_context(_cct);
-    return init_begin(dpp);
-  }
+  int init_begin(CephContext *_cct, const DoutPrefixProvider *dpp,
+                         const rgw::SiteConfig& site);
   /** Initialize the RADOS instance and prepare to do other ops */
-  int init_svc(bool raw, const DoutPrefixProvider *dpp);
-  int init_ctl(const DoutPrefixProvider *dpp);
+  int init_svc(bool raw, const DoutPrefixProvider *dpp, const rgw::SiteConfig& site);
   virtual int init_rados();
-  int init_begin(const DoutPrefixProvider *dpp);
   int init_complete(const DoutPrefixProvider *dpp, optional_yield y);
   void finalize();
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 2bde7d192e5a..6e4d346120e2 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -3953,11 +3953,10 @@ int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y)
 
 extern "C" {
 
-void* newRadosStore(void* io_context, const void* site_config)
+void* newRadosStore(void* io_context)
 {
   rgw::sal::RadosStore* store = new rgw::sal::RadosStore(
-    *static_cast<boost::asio::io_context*>(io_context),
-    *static_cast<const rgw::SiteConfig*>(site_config));
+    *static_cast<boost::asio::io_context*>(io_context));
   if (store) {
     RGWRados* rados = new RGWRados();
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 85612eec1a9b..e85b3fde1d4f 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -120,7 +120,6 @@ class RadosZone : public StoreZone {
 class RadosStore : public StoreDriver {
   private:
     boost::asio::io_context& io_context;
-    const rgw::SiteConfig& site_config;
     RGWRados* rados;
     RGWUserCtl* user_ctl;
     std::unique_ptr<RadosZone> zone;
@@ -128,9 +127,8 @@ class RadosStore : public StoreDriver {
     std::string topics_oid(const std::string& tenant) const;
 
   public:
-    RadosStore(boost::asio::io_context& io_context,
-	       const rgw::SiteConfig& site_config)
-      : io_context(io_context), site_config(site_config), rados(nullptr) {
+    RadosStore(boost::asio::io_context& io_context)
+      : io_context(io_context), rados(nullptr) {
       }
     ~RadosStore() {
       delete rados;
@@ -285,7 +283,6 @@ class RadosStore : public StoreDriver {
     void setRados(RGWRados * st) { rados = st; }
     RGWRados* getRados(void) { return rados; }
     boost::asio::io_context& get_io_context() { return io_context; }
-    const rgw::SiteConfig& get_siteconfig() { return site_config; }
     neorados::RADOS& get_neorados() { return *neorados; }
 
     RGWServices* svc() { return &rados->svc; }
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 1f05495fb3d4..0c0e2bbea659 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -314,9 +314,10 @@ void RGWServices_Def::shutdown()
   has_shutdown = true;
 }
 
-int RGWServices::do_init(CephContext *_cct, rgw::sal::RadosStore* driver, bool have_cache, bool raw, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp)
+int RGWServices::do_init(CephContext *_cct, rgw::sal::RadosStore* driver, bool have_cache, bool raw, bool run_sync, optional_yield y, const DoutPrefixProvider *dpp, const rgw::SiteConfig& _site)
 {
   cct = _cct;
+  site = &_site;
 
   int r = _svc.init(cct, driver, have_cache, raw, run_sync, y, dpp);
   if (r < 0) {
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 08873e6058e5..03b37f1ad5a2 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -120,12 +120,14 @@ struct RGWServices_Def
   void shutdown();
 };
 
+namespace rgw { class SiteConfig; }
 
 struct RGWServices
 {
   RGWServices_Def _svc;
 
   CephContext *cct;
+  const rgw::SiteConfig* site{nullptr};
 
   RGWSI_Finisher *finisher{nullptr};
   RGWSI_Bucket *bucket{nullptr};
@@ -159,17 +161,19 @@ struct RGWServices
 
   int do_init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
 	      bool raw_storage, bool run_sync, optional_yield y,
-	      const DoutPrefixProvider *dpp);
+	      const DoutPrefixProvider *dpp, const rgw::SiteConfig& site);
 
   int init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
-	   bool run_sync, optional_yield y, const DoutPrefixProvider *dpp) {
-    return do_init(cct, store, have_cache, false, run_sync, y, dpp);
+	   bool run_sync, optional_yield y, const DoutPrefixProvider *dpp,
+	   const rgw::SiteConfig& site) {
+    return do_init(cct, store, have_cache, false, run_sync, y, dpp, site);
   }
 
   int init_raw(CephContext *cct, rgw::sal::RadosStore* store,
 	       bool have_cache, optional_yield y,
-	       const DoutPrefixProvider *dpp) {
-    return do_init(cct, store, have_cache, true, false, y, dpp);
+	       const DoutPrefixProvider *dpp,
+	       const rgw::SiteConfig& site) {
+    return do_init(cct, store, have_cache, true, false, y, dpp, site);
   }
   void shutdown() {
     _svc.shutdown();
diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc
index 5e4603b81162..d9dd1bde6033 100644
--- a/src/rgw/rgw_sal.cc
+++ b/src/rgw/rgw_sal.cc
@@ -46,8 +46,7 @@
 #define dout_subsys ceph_subsys_rgw
 
 extern "C" {
-extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context,
-				       const rgw::SiteConfig* site_config);
+extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context);
 #ifdef WITH_RADOSGW_DBSTORE
 extern rgw::sal::Driver* newDBStore(CephContext *cct);
 #endif
@@ -118,7 +117,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
   rgw::sal::Driver* driver{nullptr};
 
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore(&io_context, &site_config);
+    driver = newRadosStore(&io_context);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     if ((*rados).set_use_cache(use_cache)
@@ -130,7 +129,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
                 .set_run_sync_thread(run_sync_thread)
                 .set_run_reshard_thread(run_reshard_thread)
                 .set_run_notification_thread(run_notification_thread)
-                .init_begin(cct, dpp) < 0) {
+                .init_begin(cct, dpp, site_config) < 0) {
       delete driver;
       return nullptr;
     }
@@ -144,7 +143,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
     }
   }
   else if (cfg.store_name.compare("d3n") == 0) {
-    driver = new rgw::sal::RadosStore(io_context, site_config);
+    driver = new rgw::sal::RadosStore(io_context);
     RGWRados* rados = new D3nRGWDataCache<RGWRados>;
     dynamic_cast<rgw::sal::RadosStore*>(driver)->setRados(rados);
     rados->set_store(static_cast<rgw::sal::RadosStore* >(driver));
@@ -157,7 +156,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider*
                 .set_run_sync_thread(run_sync_thread)
                 .set_run_reshard_thread(run_reshard_thread)
                 .set_run_notification_thread(run_notification_thread)
-                .init_begin(cct, dpp) < 0) {
+                .init_begin(cct, dpp, site_config) < 0) {
       delete driver;
       return nullptr;
     }
@@ -270,7 +269,7 @@ rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvi
 {
   rgw::sal::Driver* driver = nullptr;
   if (cfg.store_name.compare("rados") == 0) {
-    driver = newRadosStore(&io_context, &site_config);
+    driver = newRadosStore(&io_context);
     RGWRados* rados = static_cast<rgw::sal::RadosStore* >(driver)->getRados();
 
     rados->set_context(cct);
@@ -280,7 +279,7 @@ rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvi
       return nullptr;
     }
 
-    int ret = rados->init_svc(true, dpp);
+    int ret = rados->init_svc(true, dpp, site_config);
     if (ret < 0) {
       ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl;
       delete driver;
diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index e69910395a8f..f36dfe5ab440 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -913,8 +913,7 @@ TEST_F(IPPolicyTest, IPEnvironment) {
   // Unfortunately RGWCivetWeb is too tightly tied to civetweb to test RGWCivetWeb::init_env.
   RGWEnv rgw_env;
   ceph::async::io_context_pool context_pool(cct->_conf->rgw_thread_pool_size); \
-  auto site = rgw::SiteConfig::make_fake();
-  rgw::sal::RadosStore store(context_pool, *site);
+  rgw::sal::RadosStore store(context_pool);
   std::unique_ptr<rgw::sal::User> user = store.get_user(rgw_user());
   rgw_env.set("REMOTE_ADDR", "192.168.1.1");
   rgw_env.set("HTTP_HOST", "1.2.3.4");
diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index 8aed91bb6f5a..e6014513ba3a 100644
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -8,7 +8,6 @@
 #include "rgw_lua_request.h"
 #include "rgw_lua_background.h"
 #include "rgw_lua_data_filter.h"
-#include "driver/rados/rgw_zone.h"
 #include "rgw_sal_config.h"
 
 using namespace std;
@@ -165,23 +164,17 @@ tracing::Tracer tracer;
 inline std::unique_ptr<sal::RadosStore> make_store() {
   auto context_pool = std::make_unique<ceph::async::io_context_pool>(
     g_cct->_conf->rgw_thread_pool_size);
-  std::unique_ptr<rgw::SiteConfig> site = rgw::SiteConfig::make_fake();
-
 
   struct StoreBundle : public sal::RadosStore {
     std::unique_ptr<ceph::async::io_context_pool> context_pool;
-    std::unique_ptr<rgw::SiteConfig> site;
-    StoreBundle(std::unique_ptr<ceph::async::io_context_pool> context_pool_,
-                std::unique_ptr<rgw::SiteConfig> site_)
-      : sal::RadosStore(*context_pool_.get(), *site_),
-        context_pool(std::move(context_pool_)),
-        site(std::move(site_)) {
+    StoreBundle(std::unique_ptr<ceph::async::io_context_pool> context_pool_)
+      : sal::RadosStore(*context_pool_.get()),
+        context_pool(std::move(context_pool_)) {
       setRados(new RGWRados);
     }
     virtual ~StoreBundle() = default;
   };
-  return std::make_unique<StoreBundle>(std::move(context_pool),
-                                       std::move(site));
+  return std::make_unique<StoreBundle>(std::move(context_pool));
 };
 
 #define DEFINE_REQ_STATE RGWProcessEnv pe; \

From 0b8b62bed0e79dd54d1baac240695b2a398d407f Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 4 Feb 2024 14:04:11 -0500
Subject: [PATCH 2154/2492] rgw/rados: RGWTopicMetadataHandler doesn't depend
 on RGWPubSub

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/services/svc_topic_rados.cc | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/rgw/services/svc_topic_rados.cc b/src/rgw/services/svc_topic_rados.cc
index 9e0b75d46647..c08d688b2719 100644
--- a/src/rgw/services/svc_topic_rados.cc
+++ b/src/rgw/services/svc_topic_rados.cc
@@ -135,16 +135,16 @@ int RGWTopicMetadataHandler::do_get(RGWSI_MetaBackend_Handler::Op* op,
   std::string topic_name;
   std::string tenant;
   parse_topic_entry(entry, &tenant, &topic_name);
-  RGWPubSub ps(driver, tenant,
-               &topic_svc->svc.zone->get_current_period().get_map().zonegroups);
-  int ret = ps.get_topic(dpp, topic_name, result, y, nullptr);
+  RGWObjVersionTracker objv;
+  // TODO: read metadata directly from rados, without calling through
+  // sal::Driver::read_topic_v2()
+  int ret = driver->read_topic_v2(topic_name, tenant, result, &objv, y, dpp);
   if (ret < 0) {
     return ret;
   }
   ceph::real_time mtime;
-  obj_version ver;
   RGWTopicMetadataObject* rdo =
-      new RGWTopicMetadataObject(result, ver, mtime, driver);
+      new RGWTopicMetadataObject(result, objv.read_version, mtime, driver);
   *obj = rdo;
   return 0;
 }
@@ -161,9 +161,9 @@ int RGWTopicMetadataHandler::do_remove(RGWSI_MetaBackend_Handler::Op* op,
   std::string topic_name;
   std::string tenant;
   parse_topic_entry(entry, &tenant, &topic_name);
-  RGWPubSub ps(driver, tenant,
-               &topic_svc->svc.zone->get_current_period().get_map().zonegroups);
-  return ps.remove_topic(dpp, topic_name, y);
+  // TODO: remove metadata directly from rados, without calling through
+  // sal::Driver::remove_topic_v2()
+  return driver->remove_topic_v2(topic_name, tenant, &objv_tracker, y, dpp);
 }
 
 class RGWMetadataHandlerPut_Topic : public RGWMetadataHandlerPut_SObj {
@@ -191,6 +191,8 @@ class RGWMetadataHandlerPut_Topic : public RGWMetadataHandlerPut_SObj {
       return ret;
     }
     RGWObjVersionTracker objv_tracker;
+    // TODO: write metadata directly from rados, without calling through
+    // sal::Driver::write_topic_v2()
     ret = driver->write_topic_v2(topic, &objv_tracker, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret

From 1fccdad21d5bcdebbdd7167b7ed18391a1e8d1e2 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 4 Feb 2024 15:17:47 -0500
Subject: [PATCH 2155/2492] rgw: RGWPubSub requires SiteConfig

RGWPubSub constructor takes SiteConfig instead of zonegroup map

replace do_all_zonegroups_support_notification_v2() with a generic
function rgw::all_zonegroups_support() that handles non-realm
configurations too

remove unused sal::ZoneGroup::supports_feature()

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/daos/rgw_sal_daos.h    |  3 ---
 src/rgw/driver/motr/rgw_sal_motr.h    |  3 ---
 src/rgw/driver/rados/rgw_cr_rados.cc  |  2 +-
 src/rgw/driver/rados/rgw_notify.cc    | 22 ++++++++--------
 src/rgw/driver/rados/rgw_notify.h     |  5 +++-
 src/rgw/driver/rados/rgw_rados.cc     |  2 +-
 src/rgw/driver/rados/rgw_sal_rados.cc |  4 +--
 src/rgw/driver/rados/rgw_sal_rados.h  |  3 ---
 src/rgw/driver/rados/rgw_zone.cc      | 13 ++++++++++
 src/rgw/driver/rados/rgw_zone.h       |  3 +++
 src/rgw/rgw_admin.cc                  | 27 ++++++++------------
 src/rgw/rgw_pubsub.cc                 | 17 +++----------
 src/rgw/rgw_pubsub.h                  |  6 +----
 src/rgw/rgw_rest_pubsub.cc            | 36 +++++++++------------------
 src/rgw/rgw_sal.h                     |  2 --
 src/rgw/rgw_sal_dbstore.h             |  3 ---
 src/rgw/rgw_sal_filter.h              |  3 ---
 17 files changed, 64 insertions(+), 90 deletions(-)

diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index 479f5b84a5e3..c5cfefc222d1 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -430,9 +430,6 @@ class DaosZoneGroup : public StoreZoneGroup {
     return std::make_unique<DaosZoneGroup>(store, group);
   }
   const RGWZoneGroup& get_group() { return group; }
-  virtual bool supports_feature(std::string_view feature) const override {
-    return group.supports(feature);
-  }
 };
 
 class DaosZone : public StoreZone {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 9ff14a58307d..3cc3b37fa9ad 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -474,9 +474,6 @@ class MotrZoneGroup : public StoreZoneGroup {
   virtual std::unique_ptr<ZoneGroup> clone() override {
     return std::make_unique<MotrZoneGroup>(store, group);
   }
-  virtual bool supports_feature(std::string_view feature) const override {
-    return group.supports(feature);
-  }
   friend class MotrZone;
 };
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index b7d362423e53..fb5722528c39 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -809,7 +809,7 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
                   req_id, null_yield);
 
         auto notify_res = static_cast<rgw::sal::RadosNotification*>(notify.get())->get_reservation();
-        int ret = rgw::notify::publish_reserve(dpp, rgw::notify::ObjectSyncedCreate, notify_res, &obj_tags);
+        int ret = rgw::notify::publish_reserve(dpp, *store->svc()->site, rgw::notify::ObjectSyncedCreate, notify_res, &obj_tags);
         if (ret < 0) {
           ldpp_dout(dpp, 1) << "ERROR: reserving notification failed, with error: " << ret << dendl;
           // no need to return, the sync already happened
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index e3ee61ba9df0..a1290ec9b20a 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -121,6 +121,7 @@ class Manager : public DoutPrefixProvider {
   const uint32_t stale_reservations_period_s;
   const uint32_t reservations_cleanup_period_s;
   queues_persistency_tracker topics_persistency_tracker;
+  const SiteConfig& site;
 public:
   rgw::sal::RadosStore& rados_store;
 
@@ -491,7 +492,7 @@ class Manager : public DoutPrefixProvider {
           std::string tenant_name;
           // TODO: extract tenant name from queue_name once it is fixed
           uint64_t size_to_migrate = 0;
-          RGWPubSub ps(&rados_store, tenant_name);
+          RGWPubSub ps(&rados_store, tenant_name, site);
 
           rgw_pubsub_topic topic;
           auto ret_of_get_topic = ps.get_topic(this, queue_name, topic,
@@ -669,7 +670,8 @@ class Manager : public DoutPrefixProvider {
   Manager(CephContext* _cct, uint32_t _max_queue_size, uint32_t _queues_update_period_ms, 
           uint32_t _queues_update_retry_ms, uint32_t _queue_idle_sleep_us, u_int32_t failover_time_ms, 
           uint32_t _stale_reservations_period_s, uint32_t _reservations_cleanup_period_s,
-          uint32_t _worker_count, rgw::sal::RadosStore* store) :
+          uint32_t _worker_count, rgw::sal::RadosStore* store,
+          const SiteConfig& site) :
     max_queue_size(_max_queue_size),
     queues_update_period_ms(_queues_update_period_ms),
     queues_update_retry_ms(_queues_update_retry_ms),
@@ -681,6 +683,7 @@ class Manager : public DoutPrefixProvider {
     worker_count(_worker_count),
     stale_reservations_period_s(_stale_reservations_period_s),
     reservations_cleanup_period_s(_reservations_cleanup_period_s),
+    site(site),
     rados_store(*store)
     {
       spawn::spawn(io_context, [this](spawn::yield_context yield) {
@@ -753,7 +756,8 @@ constexpr uint32_t WORKER_COUNT = 1;                 // 1 worker thread
 constexpr uint32_t STALE_RESERVATIONS_PERIOD_S = 120;   // cleanup reservations that are more than 2 minutes old
 constexpr uint32_t RESERVATIONS_CLEANUP_PERIOD_S = 30; // reservation cleanup every 30 seconds
 
-bool init(CephContext* cct, rgw::sal::RadosStore* store, const DoutPrefixProvider *dpp) {
+bool init(CephContext* cct, rgw::sal::RadosStore* store,
+          const SiteConfig& site, const DoutPrefixProvider *dpp) {
   if (s_manager) {
     return false;
   }
@@ -763,7 +767,7 @@ bool init(CephContext* cct, rgw::sal::RadosStore* store, const DoutPrefixProvide
       IDLE_TIMEOUT_USEC, FAILOVER_TIME_MSEC, 
       STALE_RESERVATIONS_PERIOD_S, RESERVATIONS_CLEANUP_PERIOD_S,
       WORKER_COUNT,
-      store);
+      store, site);
   return true;
 }
 
@@ -978,13 +982,13 @@ static inline bool notification_match(reservation_t& res,
 }
 
   int publish_reserve(const DoutPrefixProvider* dpp,
+		      const SiteConfig& site,
 		      EventType event_type,
 		      reservation_t& res,
 		      const RGWObjTags* req_tags)
 {
   rgw_pubsub_bucket_topics bucket_topics;
-  if (do_all_zonegroups_support_notification_v2(
-          res.store->svc()->zone->get_current_period().get_map().zonegroups)) {
+  if (all_zonegroups_support(site, zone_features::notification_v2)) {
     auto ret = 0;
     if (!res.s) {
       //  for non S3-request caller (e.g., lifecycle, ObjectSync), bucket attrs
@@ -1004,7 +1008,7 @@ static inline bool notification_match(reservation_t& res,
       return ret;
     }
   } else {
-    const RGWPubSub ps(res.store, res.user_tenant);
+    const RGWPubSub ps(res.store, res.user_tenant, site);
     const RGWPubSub::Bucket ps_bucket(ps, res.bucket);
     auto rc = ps_bucket.get_topics(res.dpp, bucket_topics, res.yield);
     if (rc < 0) {
@@ -1054,9 +1058,7 @@ static inline bool notification_match(reservation_t& res,
     // load the topic,if there is change in topic config while it's stored in
     // notification.
     rgw_pubsub_topic result;
-    const RGWPubSub ps(
-        res.store, res.user_tenant,
-        &res.store->svc()->zone->get_current_period().get_map().zonegroups);
+    const RGWPubSub ps(res.store, res.user_tenant, site);
     auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr);
     if (ret < 0) {
       ldpp_dout(res.dpp, 1)
diff --git a/src/rgw/driver/rados/rgw_notify.h b/src/rgw/driver/rados/rgw_notify.h
index 5117f9eecd8f..4f2f57341ecb 100644
--- a/src/rgw/driver/rados/rgw_notify.h
+++ b/src/rgw/driver/rados/rgw_notify.h
@@ -12,6 +12,7 @@
 #include "rgw_pubsub.h"
 
 // forward declarations
+namespace rgw { class SiteConfig; }
 namespace rgw::sal {
     class RadosStore;
     class RGWObject;
@@ -25,7 +26,8 @@ namespace rgw::notify {
 // initialize the notification manager
 // notification manager is dequeuing the 2-phase-commit queues
 // and send the notifications to the endpoints
-bool init(CephContext* cct, rgw::sal::RadosStore* store, const DoutPrefixProvider *dpp);
+bool init(CephContext* cct, rgw::sal::RadosStore* store,
+          const rgw::SiteConfig& site, const DoutPrefixProvider *dpp);
 
 // shutdown the notification manager
 void shutdown();
@@ -110,6 +112,7 @@ struct rgw_topic_stats {
 
 // create a reservation on the 2-phase-commit queue
 int publish_reserve(const DoutPrefixProvider *dpp,
+		      const SiteConfig& site,
 		      EventType event_type,
 		      reservation_t& reservation,
 		      const RGWObjTags* req_tags);
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 18ff24f21813..6084c21ef8ea 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1357,7 +1357,7 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
   index_completion_manager = new RGWIndexCompletionManager(this);
 
   if (run_notification_thread) {
-    ret = rgw::notify::init(cct, driver, dpp);
+    ret = rgw::notify::init(cct, driver, *svc.site, dpp);
     if (ret < 0 ) {
       ldpp_dout(dpp, 1) << "ERROR: failed to initialize notification manager" << dendl;
     }
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 6e4d346120e2..6aba08828b72 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -367,7 +367,7 @@ int RadosBucket::remove(const DoutPrefixProvider* dpp,
 
   // if bucket has notification definitions associated with it
   // they should be removed (note that any pending notifications on the bucket are still going to be sent)
-  const RGWPubSub ps(store, info.owner.tenant);
+  const RGWPubSub ps(store, info.owner.tenant, *store->svc()->site);
   const RGWPubSub::Bucket ps_bucket(ps, this);
   const auto ps_ret = ps_bucket.remove_notifications(dpp, y);
   if (ps_ret < 0 && ps_ret != -ENOENT) {
@@ -3061,7 +3061,7 @@ std::unique_ptr<LCSerializer> RadosLifecycle::get_serializer(const std::string&
 
 int RadosNotification::publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags)
 {
-  return rgw::notify::publish_reserve(dpp, event_type, res, obj_tags);
+  return rgw::notify::publish_reserve(dpp, *store->svc()->site, event_type, res, obj_tags);
 }
 
 int RadosNotification::publish_commit(const DoutPrefixProvider* dpp, uint64_t size,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index e85b3fde1d4f..e15c754d1b49 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -85,9 +85,6 @@ class RadosZoneGroup : public StoreZoneGroup {
   virtual std::unique_ptr<ZoneGroup> clone() override {
     return std::make_unique<RadosZoneGroup>(store, group);
   }
-  virtual bool supports_feature(std::string_view feature) const override {
-    return group.supports(feature);
-  }
   const RGWZoneGroup& get_group() const { return group; }
 };
 
diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
index b93f319ac176..2c504712381c 100644
--- a/src/rgw/driver/rados/rgw_zone.cc
+++ b/src/rgw/driver/rados/rgw_zone.cc
@@ -1117,6 +1117,19 @@ auto find_zone_placement(const DoutPrefixProvider* dpp,
   return &i->second;
 }
 
+bool all_zonegroups_support(const SiteConfig& site, std::string_view feature)
+{
+  const auto& period = site.get_period();
+  if (!period) {
+    // if we're not in a realm, just check the local zonegroup
+    return site.get_zonegroup().supports(feature);
+  }
+  const auto& zgs = period->period_map.zonegroups;
+  return std::all_of(zgs.begin(), zgs.end(), [feature] (const auto& pair) {
+      return pair.second.supports(feature);
+    });
+}
+
 static int read_or_create_default_zone(const DoutPrefixProvider* dpp,
                                        optional_yield y,
                                        sal::ConfigStore* cfgstore,
diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h
index 2eb3e725253f..345cbc6a2171 100644
--- a/src/rgw/driver/rados/rgw_zone.h
+++ b/src/rgw/driver/rados/rgw_zone.h
@@ -1009,4 +1009,7 @@ class SiteConfig {
 };
 
 
+/// Test whether all zonegroups in the realm support the given zone feature.
+bool all_zonegroups_support(const SiteConfig& site, std::string_view feature);
+
 } // namespace rgw
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index be91c66e9ff8..395e05ce517e 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -10628,8 +10628,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    if (driver->get_zone()->get_zonegroup().supports_feature(
-            rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
       ret = get_bucket_notifications(dpp(), bucket.get(), result);
       if (ret < 0) {
         cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
@@ -10637,7 +10636,7 @@ int main(int argc, const char **argv)
         return -ret;
       }
     } else {
-      RGWPubSub ps(driver, tenant);
+      RGWPubSub ps(driver, tenant, *site);
       const RGWPubSub::Bucket b(ps, bucket.get());
       ret = b.get_topics(dpp(), result, null_yield);
       if (ret < 0 && ret != -ENOENT) {
@@ -10650,7 +10649,7 @@ int main(int argc, const char **argv)
   }
 
   if (opt_cmd == OPT::PUBSUB_TOPIC_LIST) {
-    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
+    RGWPubSub ps(driver, tenant, *site);
     rgw_pubsub_topics result;
     ret = ps.get_topics(dpp(), result, null_yield);
     if (ret < 0 && ret != -ENOENT) {
@@ -10667,8 +10666,7 @@ int main(int argc, const char **argv)
         }
       }
     }
-    if (driver->get_zone()->get_zonegroup().supports_feature(
-            rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
       Formatter::ObjectSection top_section(*formatter, "result");
       Formatter::ArraySection s(*formatter, "topics");
       for (const auto& [_, topic] : result.topics) {
@@ -10693,7 +10691,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: topic name was not provided (via --topic)" << std::endl;
       return EINVAL;
     }
-    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
+    RGWPubSub ps(driver, tenant, *site);
 
     rgw_pubsub_topic topic;
     std::set<std::string> subscribed_buckets;
@@ -10703,8 +10701,7 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not get topic: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    if (driver->get_zone()->get_zonegroup().supports_feature(
-            rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
       show_topics_info_v2(topic, subscribed_buckets, formatter.get());
     } else {
       encode_json("topic", topic, formatter.get());
@@ -10728,8 +10725,7 @@ int main(int argc, const char **argv)
       return -ret;
     }
     rgw_pubsub_bucket_topics bucket_topics;
-    if (driver->get_zone()->get_zonegroup().supports_feature(
-            rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
       ret = get_bucket_notifications(dpp(), bucket.get(), bucket_topics);
       if (ret < 0) {
         cerr << "ERROR: could not get bucket notifications: "
@@ -10737,7 +10733,7 @@ int main(int argc, const char **argv)
         return -ret;
       }
     } else {
-      RGWPubSub ps(driver, tenant);
+      RGWPubSub ps(driver, tenant, *site);
       const RGWPubSub::Bucket b(ps, bucket.get());
       ret = b.get_topics(dpp(), bucket_topics, null_yield);
       if (ret < 0 && ret != -ENOENT) {
@@ -10770,7 +10766,7 @@ int main(int argc, const char **argv)
       return -ret;
     }
 
-    RGWPubSub ps(driver, tenant, &site->get_period()->get_map().zonegroups);
+    RGWPubSub ps(driver, tenant, *site);
 
     ret = ps.remove_topic(dpp(), topic_name, null_yield);
     if (ret < 0) {
@@ -10794,12 +10790,11 @@ int main(int argc, const char **argv)
       return -ret;
     }
 
-    if (driver->get_zone()->get_zonegroup().supports_feature(
-            rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
       ret = remove_notification_v2(dpp(), driver, bucket.get(), notification_id,
                                    null_yield);
     } else {
-      RGWPubSub ps(driver, tenant);
+      RGWPubSub ps(driver, tenant, *site);
 
       rgw_pubsub_bucket_topics bucket_topics;
       const RGWPubSub::Bucket b(ps, bucket.get());
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index dec38ee87e87..f4ddb118cd31 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -504,9 +504,10 @@ RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver, const std::string& _tenant)
 
 RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver,
                      const std::string& _tenant,
-                     const std::map<std::string, RGWZoneGroup>* _zonegroups)
-    : driver(_driver), tenant(_tenant), zonegroups(_zonegroups) {
-  use_notification_v2 = do_all_zonegroups_support_notification_v2(*zonegroups);
+                     const rgw::SiteConfig& site)
+    : driver(_driver), tenant(_tenant),
+      use_notification_v2(rgw::all_zonegroups_support(site, rgw::zone_features::notification_v2))
+{
 }
 
 int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result,
@@ -651,16 +652,6 @@ int get_bucket_notifications(const DoutPrefixProvider* dpp,
   return 0;
 }
 
-bool do_all_zonegroups_support_notification_v2(
-    std::map<std::string, RGWZoneGroup> zonegroups) {
-  for (const auto& [_, zonegroup] : zonegroups) {
-    if (!zonegroup.supports(rgw::zone_features::notification_v2)) {
-      return false;
-    }
-  }
-  return true;
-}
-
 std::string topic_to_unique(const std::string& topic,
                             const std::string& notification) {
   return notification + "_" + topic;
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index 8509d86e2255..ed7856721756 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -560,7 +560,6 @@ class RGWPubSub
 
   rgw::sal::Driver* const driver;
   const std::string tenant;
-  const std::map<std::string, RGWZoneGroup>* zonegroups = nullptr;
   bool use_notification_v2 = false;
 
   int read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result, 
@@ -573,7 +572,7 @@ class RGWPubSub
 
  RGWPubSub(rgw::sal::Driver* _driver,
            const std::string& _tenant,
-           const std::map<std::string, RGWZoneGroup>* zonegroups);
+           const rgw::SiteConfig& site);
 
   class Bucket {
     friend class RGWPubSub;
@@ -670,9 +669,6 @@ namespace rgw::notify {
   constexpr static const std::string_view DEFAULT_CONFIG{"None"};
 }
 
-bool do_all_zonegroups_support_notification_v2(
-    std::map<std::string, RGWZoneGroup> zonegroups);
-
 std::string topic_to_unique(const std::string& topic,
                             const std::string& notification);
 
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 38943736e451..c64b57853372 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -197,8 +197,7 @@ class RGWPSCreateTopicOp : public RGWOp {
       return ret;
     }
 
-    const RGWPubSub ps(driver, s->owner.id.tenant,
-                       &s->penv.site->get_period()->get_map().zonegroups);
+    const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
     rgw_pubsub_topic result;
     ret = ps.get_topic(this, topic_name, result, y, nullptr);
     if (ret == -ENOENT) {
@@ -278,8 +277,7 @@ void RGWPSCreateTopicOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
                            s->owner.id, policy_text, y);
   if (op_ret < 0) {
@@ -334,8 +332,7 @@ class RGWPSListTopicsOp : public RGWOp {
 };
 
 void RGWPSListTopicsOp::execute(optional_yield y) {
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   op_ret = ps.get_topics(this, result, y);
   // if there are no topics it is not considered an error
   op_ret = op_ret == -ENOENT ? 0 : op_ret;
@@ -422,8 +419,7 @@ void RGWPSGetTopicOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -507,8 +503,7 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
   if (op_ret < 0) {
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
   if (op_ret < 0) {
     ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
@@ -636,8 +631,7 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
       return ret;
     }
     rgw_pubsub_topic result;
-    const RGWPubSub ps(driver, s->owner.id.tenant,
-                       &s->penv.site->get_period()->get_map().zonegroups);
+    const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
     ret = ps.get_topic(this, topic_name, result, y, nullptr);
     if (ret < 0) {
       ldpp_dout(this, 1) << "failed to get topic '" << topic_name
@@ -715,8 +709,7 @@ void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data,
                            topic_owner, policy_text, y);
   if (op_ret < 0) {
@@ -797,8 +790,7 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
       return;
     }
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
 
   rgw_pubsub_topic result;
   op_ret = ps.get_topic(this, topic_name, result, y, nullptr);
@@ -991,8 +983,7 @@ class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
 };
 
 void RGWPSCreateNotifOp::execute(optional_yield y) {
-  if (do_all_zonegroups_support_notification_v2(
-          s->penv.site->get_period()->get_map().zonegroups)) {
+  if (rgw::all_zonegroups_support(*s->penv.site, rgw::zone_features::notification_v2)) {
     return execute_v2(y);
   }
   op_ret = verify_params();
@@ -1174,8 +1165,7 @@ void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
         << "' , ret = " << op_ret << dendl;
     return;
   }
-  const RGWPubSub ps(driver, s->owner.id.tenant,
-                     &s->penv.site->get_period()->get_map().zonegroups);
+  const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
   std::unordered_map<std::string, rgw_pubsub_topic> topics;
   for (const auto& c : configurations.list) {
     const auto& notif_name = c.id;
@@ -1295,8 +1285,7 @@ class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
 };
 
 void RGWPSDeleteNotifOp::execute(optional_yield y) {
-  if (do_all_zonegroups_support_notification_v2(
-          s->penv.site->get_period()->get_map().zonegroups)) {
+  if (rgw::all_zonegroups_support(*s->penv.site, rgw::zone_features::notification_v2)) {
     return execute_v2(y);
   }
   std::string notif_name;
@@ -1457,8 +1446,7 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
 
   // get all topics on a bucket
   rgw_pubsub_bucket_topics bucket_topics;
-  if (do_all_zonegroups_support_notification_v2(
-          s->penv.site->get_period()->get_map().zonegroups)) {
+  if (rgw::all_zonegroups_support(*s->penv.site, rgw::zone_features::notification_v2)) {
     op_ret = get_bucket_notifications(this, bucket.get(), bucket_topics);
   } else {
     const RGWPubSub ps(driver, s->owner.id.tenant);
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 4dc3c9dc8365..f1bc455835f6 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1495,8 +1495,6 @@ class ZoneGroup {
   virtual int list_zones(std::list<std::string>& zone_ids) = 0;
   /** Clone a copy of this zonegroup. */
   virtual std::unique_ptr<ZoneGroup> clone() = 0;
-  /** Determine if zonegroup |feature| is supported.*/
-  virtual bool supports_feature(std::string_view feature) const = 0;
 };
 
 /**
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 140b28396626..3c0c7c765198 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -259,9 +259,6 @@ class DBNotification : public StoreNotification {
       std::unique_ptr<RGWZoneGroup>zg = std::make_unique<RGWZoneGroup>(*group.get());
       return std::make_unique<DBZoneGroup>(store, std::move(zg));
     }
-    virtual bool supports_feature(std::string_view feature) const override {
-      return group->supports(feature);
-    }
   };
 
   class DBZone : public StoreZone {
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index d5bf9afe248c..8bb704ce17c1 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -75,9 +75,6 @@ class FilterZoneGroup : public ZoneGroup {
     std::unique_ptr<ZoneGroup> nzg = next->clone();
     return std::make_unique<FilterZoneGroup>(std::move(nzg));
   }
-  virtual bool supports_feature(std::string_view feature) const override {
-    return next->supports_feature(feature);
-  }
 };
 
 class FilterZone : public Zone {

From 6000732706890fa5d276d22972d1d83be32bb5c8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Sun, 3 Dec 2023 14:10:30 -0500
Subject: [PATCH 2156/2492] rgw/metadata: add RGWMetadataLister abstraction

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_metadata_lister.h | 73 ++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 src/rgw/driver/rados/rgw_metadata_lister.h

diff --git a/src/rgw/driver/rados/rgw_metadata_lister.h b/src/rgw/driver/rados/rgw_metadata_lister.h
new file mode 100644
index 000000000000..9fbb21ac3b40
--- /dev/null
+++ b/src/rgw/driver/rados/rgw_metadata_lister.h
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <iterator>
+#include <list>
+#include <string>
+#include <vector>
+#include "services/svc_sys_obj.h"
+
+class DoutPrefixProvider;
+
+class RGWMetadataLister {
+  RGWSI_SysObj::Pool pool;
+  RGWSI_SysObj::Pool::Op listing;
+
+  virtual void filter_transform(std::vector<std::string>& oids,
+                                std::list<std::string>& keys) {
+    // use all oids as keys
+    std::move(oids.begin(), oids.end(), std::back_inserter(keys));
+  }
+
+ public:
+  explicit RGWMetadataLister(RGWSI_SysObj::Pool pool)
+    : pool(pool), listing(this->pool) {}
+  virtual ~RGWMetadataLister() {}
+
+  int init(const DoutPrefixProvider* dpp,
+           const std::string& marker,
+           const std::string& prefix)
+  {
+    return listing.init(dpp, marker, prefix);
+  }
+
+  int get_next(const DoutPrefixProvider* dpp, int max,
+               std::list<std::string>& keys, bool* truncated)
+  {
+    std::vector<std::string> oids;
+    int r = listing.get_next(dpp, max, &oids, truncated);
+    if (r == -ENOENT) {
+      if (truncated) {
+        *truncated = false;
+      }
+      return 0;
+    }
+    if (r < 0) {
+      return r;
+    }
+    filter_transform(oids, keys);
+    return 0;
+  }
+
+  std::string get_marker()
+  {
+    std::string marker;
+    listing.get_marker(&marker);
+    return marker;
+  }
+};

From 7827b2527f6b656331ae337ebc196c17b2716e48 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Fri, 5 Jan 2024 10:26:00 -0500
Subject: [PATCH 2157/2492] rgw/mdlog: add complete_entry() for most common
 usage

metadata sync ignores entries other than MDLOG_STATUS_COMPLETE, so we
don't need to write separate prepare/complete entries. metadata
mutations can just call complete_entry() on success

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/services/svc_mdlog.cc | 18 ++++++++++++++++++
 src/rgw/services/svc_mdlog.h  |  5 +++++
 2 files changed, 23 insertions(+)

diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc
index 46a158ceb77b..03a967d12673 100644
--- a/src/rgw/services/svc_mdlog.cc
+++ b/src/rgw/services/svc_mdlog.cc
@@ -544,6 +544,24 @@ int RGWSI_MDLog::add_entry(const DoutPrefixProvider *dpp, const string& hash_key
   return current_log->add_entry(dpp, hash_key, section, key, bl, y);
 }
 
+int RGWSI_MDLog::complete_entry(const DoutPrefixProvider* dpp, optional_yield y,
+                                const std::string& section, const std::string& key,
+                                const RGWObjVersionTracker* objv)
+{
+  RGWMetadataLogData entry;
+  if (objv) {
+    entry.read_version = objv->read_version;
+    entry.write_version = objv->write_version;
+  }
+  entry.status = MDLOG_STATUS_COMPLETE;
+
+  bufferlist bl;
+  encode(entry, bl);
+
+  const std::string hash_key = fmt::format("{}:{}", section, key);
+  return add_entry(dpp, hash_key, section, key, bl, y);
+}
+
 int RGWSI_MDLog::get_shard_id(const string& hash_key, int *shard_id)
 {
   ceph_assert(current_log); // must have called init()
diff --git a/src/rgw/services/svc_mdlog.h b/src/rgw/services/svc_mdlog.h
index f169ee88db9b..c8d40b138742 100644
--- a/src/rgw/services/svc_mdlog.h
+++ b/src/rgw/services/svc_mdlog.h
@@ -106,6 +106,11 @@ class RGWSI_MDLog : public RGWServiceInstance
 
   int add_entry(const DoutPrefixProvider *dpp, const std::string& hash_key, const std::string& section, const std::string& key, bufferlist& bl, optional_yield y);
 
+  // encode a RGWMetadataLogData with MDLOG_STATUS_COMPLETE and add it
+  int complete_entry(const DoutPrefixProvider* dpp, optional_yield y,
+                     const std::string& section, const std::string& key,
+                     const RGWObjVersionTracker* objv);
+
   int get_shard_id(const std::string& hash_key, int *shard_id);
 
   RGWPeriodHistory *get_period_history() {

From db6c73a0cdcf60a920c91b6d4506df36d98b7308 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 9 Jan 2024 18:55:40 -0500
Subject: [PATCH 2158/2492] rgw: paginate ListTopics

rename read_topics()/write_topics() to 'v1' and only call them from
internal v1 call paths

public get_topics() now calls read_topics_v1() for the v1 case, and does
the paginated listing with driver->meta_list_keys_next() for v2

RGWPSListTopicsOp now uses the NextToken request/response params with
the paginated get_topics(), limiting responses to 100 entries like AWS

'radosgw-admin topic list' also paginates the listing according to
--max-entries to avoid reading everything into memory at once

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/rgw_admin.cc       |  62 ++++++++++++----------
 src/rgw/rgw_pubsub.cc      | 102 +++++++++++++++++++++----------------
 src/rgw/rgw_pubsub.h       |  20 ++++----
 src/rgw/rgw_rest_pubsub.cc |  11 +++-
 4 files changed, 114 insertions(+), 81 deletions(-)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 395e05ce517e..a5933604b7e1 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1169,7 +1169,7 @@ static void show_reshard_status(
 }
 
 static void show_topics_info_v2(const rgw_pubsub_topic& topic,
-                                std::set<std::string> subscribed_buckets,
+                                const std::set<std::string>& subscribed_buckets,
                                 Formatter* formatter) {
   formatter->open_object_section("topic");
   topic.dump(formatter);
@@ -10650,39 +10650,47 @@ int main(int argc, const char **argv)
 
   if (opt_cmd == OPT::PUBSUB_TOPIC_LIST) {
     RGWPubSub ps(driver, tenant, *site);
-    rgw_pubsub_topics result;
-    ret = ps.get_topics(dpp(), result, null_yield);
-    if (ret < 0 && ret != -ENOENT) {
-      cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
-    }
-    if (!rgw::sal::User::empty(user)) {
-      for (auto it = result.topics.cbegin(); it != result.topics.cend();) {
-        const auto& topic = it->second;
-        if (user->get_id() != topic.user) {
-          result.topics.erase(it++);
-        } else {
-          ++it;
-        }
+    std::string next_token = marker;
+
+    formatter->open_object_section("result");
+    formatter->open_array_section("topics");
+    do {
+      rgw_pubsub_topics result;
+      int ret = ps.get_topics(dpp(), next_token, max_entries,
+                              result, next_token, null_yield);
+      if (ret < 0 && ret != -ENOENT) {
+        cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
       }
-    }
-    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
-      Formatter::ObjectSection top_section(*formatter, "result");
-      Formatter::ArraySection s(*formatter, "topics");
       for (const auto& [_, topic] : result.topics) {
+        if (!rgw::sal::User::empty(user) && user->get_id() != topic.user) {
+          continue;
+        }
         std::set<std::string> subscribed_buckets;
-        ret = driver->get_bucket_topic_mapping(topic, subscribed_buckets,
-                                               null_yield, dpp());
-        if (ret < 0) {
-          cerr << "failed to fetch bucket topic mapping info for topic: "
-               << topic.name << ", ret=" << ret << std::endl;
-        } else {
+        if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+          ret = driver->get_bucket_topic_mapping(topic, subscribed_buckets,
+                                                 null_yield, dpp());
+          if (ret < 0) {
+            cerr << "failed to fetch bucket topic mapping info for topic: "
+                 << topic.name << ", ret=" << ret << std::endl;
+          }
           show_topics_info_v2(topic, subscribed_buckets, formatter.get());
+        } else {
+          encode_json("result", result, formatter.get());
+        }
+        if (max_entries_specified) {
+          --max_entries;
         }
       }
-    } else {
-      encode_json("result", result, formatter.get());
+    } while (!next_token.empty() && max_entries > 0);
+    formatter->close_section(); // topics
+    if (max_entries_specified) {
+      encode_json("truncated", !next_token.empty(), formatter.get());
+      if (!next_token.empty()) {
+        encode_json("marker", next_token, formatter.get());
+      }
     }
+    formatter->close_section(); // result
     formatter->flush(cout);
   }
 
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index f4ddb118cd31..bec78b687c83 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -510,45 +510,61 @@ RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver,
 {
 }
 
-int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result,
-    RGWObjVersionTracker *objv_tracker, optional_yield y) const
+int RGWPubSub::get_topics(const DoutPrefixProvider* dpp,
+                          const std::string& start_marker, int max_items,
+                          rgw_pubsub_topics& result, std::string& next_marker,
+                          optional_yield y) const
 {
-  if (use_notification_v2) {
-    void* handle = NULL;
-    auto ret =
-        driver->meta_list_keys_init(dpp, "topic", std::string(), &handle);
-    if (ret < 0) {
-      return ret;
-    }
-    bool truncated;
-    int max = 1000;
-    do {
-      std::list<std::string> topics;
-      ret = driver->meta_list_keys_next(dpp, handle, max, topics, &truncated);
-      if (ret < 0) {
-            ldpp_dout(dpp, 1)
-                << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl;
-            break;
-      }
-      for (auto& topic_entry : topics) {
-        std::string topic_name;
-        std::string topic_tenant;
-        parse_topic_entry(topic_entry, &topic_tenant, &topic_name);
-        if (tenant != topic_tenant) {
-          continue;
-        }
-        rgw_pubsub_topic topic;
-        const auto op_ret = get_topic(dpp, topic_name, topic, y, nullptr);
-        if (op_ret < 0) {
-          ret = op_ret;
-          continue;
-        }
-        result.topics[topic_name] = std::move(topic);
-      }
-    } while (truncated);
-    driver->meta_list_keys_complete(handle);
+  if (!use_notification_v2) {
+    // v1 returns all topics, ignoring marker/max_items
+    return read_topics_v1(dpp, result, nullptr, y);
+  }
+
+  // TODO: prefix filter on 'tenant:'
+  void* handle = NULL;
+  int ret = driver->meta_list_keys_init(dpp, "topic", start_marker, &handle);
+  if (ret < 0) {
+    return ret;
+  }
+  auto g = make_scope_guard(
+      [this, handle] { driver->meta_list_keys_complete(handle); });
+
+  if (max_items > 1000) {
+    max_items = 1000;
+  }
+  std::list<std::string> topics;
+  bool truncated = false;
+  ret = driver->meta_list_keys_next(dpp, handle, max_items, topics, &truncated);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1)
+        << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl;
     return ret;
   }
+  for (auto& topic_entry : topics) {
+    std::string topic_name;
+    std::string topic_tenant;
+    parse_topic_entry(topic_entry, &topic_tenant, &topic_name);
+    if (tenant != topic_tenant) {
+      continue;
+    }
+    rgw_pubsub_topic topic;
+    int r = get_topic(dpp, topic_name, topic, y, nullptr);
+    if (r < 0) {
+      continue;
+    }
+    result.topics[topic_name] = std::move(topic);
+  }
+  if (truncated) {
+    next_marker = driver->meta_get_marker(handle);
+  } else {
+    next_marker.clear();
+  }
+  return ret;
+}
+
+int RGWPubSub::read_topics_v1(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result,
+                              RGWObjVersionTracker *objv_tracker, optional_yield y) const
+{
   const int ret = driver->read_topics(tenant, result, objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 10) << "WARNING: failed to read topics info: ret=" << ret << dendl;
@@ -557,8 +573,8 @@ int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& res
   return 0;
 }
 
-int RGWPubSub::write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
-				     RGWObjVersionTracker *objv_tracker, optional_yield y) const
+int RGWPubSub::write_topics_v1(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
+                               RGWObjVersionTracker *objv_tracker, optional_yield y) const
 {
   const int ret = driver->write_topics(tenant, topics, objv_tracker, y, dpp);
   if (ret < 0 && ret != -ENOENT) {
@@ -616,7 +632,7 @@ int RGWPubSub::get_topic(const DoutPrefixProvider* dpp,
     return ret;
   }
   rgw_pubsub_topics topics;
-  const int ret = read_topics(dpp, topics, nullptr, y);
+  const int ret = read_topics_v1(dpp, topics, nullptr, y);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
     return ret;
@@ -932,7 +948,7 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topics topics;
 
-  int ret = read_topics(dpp, topics, &objv_tracker, y);
+  int ret = read_topics_v1(dpp, topics, &objv_tracker, y);
   if (ret < 0 && ret != -ENOENT) {
     // its not an error if not topics exist, we create one
     ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
@@ -947,7 +963,7 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
   new_topic.opaque_data = opaque_data;
   new_topic.policy_text = policy_text;
 
-  ret = write_topics(dpp, topics, &objv_tracker, y);
+  ret = write_topics_v1(dpp, topics, &objv_tracker, y);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
     return ret;
@@ -989,7 +1005,7 @@ int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& na
   RGWObjVersionTracker objv_tracker;
   rgw_pubsub_topics topics;
 
-  int ret = read_topics(dpp, topics, &objv_tracker, y);
+  int ret = read_topics_v1(dpp, topics, &objv_tracker, y);
   if (ret < 0 && ret != -ENOENT) {
     ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
     return ret;
@@ -1001,7 +1017,7 @@ int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& na
 
   topics.topics.erase(name);
 
-  ret = write_topics(dpp, topics, &objv_tracker, y);
+  ret = write_topics_v1(dpp, topics, &objv_tracker, y);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to remove topics info: ret=" << ret << dendl;
     return ret;
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index ed7856721756..519c1053ab31 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -562,10 +562,10 @@ class RGWPubSub
   const std::string tenant;
   bool use_notification_v2 = false;
 
-  int read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result, 
-      RGWObjVersionTracker* objv_tracker, optional_yield y) const;
-  int write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
-			RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+  int read_topics_v1(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result,
+                     RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+  int write_topics_v1(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
+                      RGWObjVersionTracker* objv_tracker, optional_yield y) const;
 
 public:
   RGWPubSub(rgw::sal::Driver* _driver, const std::string& tenant);
@@ -620,11 +620,13 @@ class RGWPubSub
     int remove_notifications(const DoutPrefixProvider *dpp, optional_yield y) const;
   };
 
-  // get the list of topics
-  // return 0 on success or if no topic was associated with the bucket, error code otherwise
-  int get_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics& result, optional_yield y) const {
-    return read_topics(dpp, result, nullptr, y);
-  }
+  // get a paginated list of topics
+  // return 0 on success, error code otherwise
+  int get_topics(const DoutPrefixProvider* dpp,
+                 const std::string& start_marker, int max_items,
+                 rgw_pubsub_topics& result, std::string& next_marker,
+                 optional_yield y) const;
+
   // get a topic with by its name and populate it into "result"
   // return -ENOENT if the topic does not exists
   // return 0 on success, error code otherwise.
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index c64b57853372..66574cc82724 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -293,6 +293,7 @@ void RGWPSCreateTopicOp::execute(optional_yield y) {
 class RGWPSListTopicsOp : public RGWOp {
 private:
   rgw_pubsub_topics result;
+  std::string next_token;
 
 public:
   int verify_permission(optional_yield) override {
@@ -325,15 +326,21 @@ class RGWPSListTopicsOp : public RGWOp {
     f->close_section(); // ListTopicsResult
     f->open_object_section("ResponseMetadata");
     encode_xml("RequestId", s->req_id, f); 
-    f->close_section(); // ResponseMetadat
+    f->close_section(); // ResponseMetadata
+    if (!next_token.empty()) {
+      encode_xml("NextToken", next_token, f);
+    }
     f->close_section(); // ListTopicsResponse
     rgw_flush_formatter_and_reset(s, f);
   }
 };
 
 void RGWPSListTopicsOp::execute(optional_yield y) {
+  const std::string start_token = s->info.args.get("NextToken");
+
   const RGWPubSub ps(driver, s->owner.id.tenant, *s->penv.site);
-  op_ret = ps.get_topics(this, result, y);
+  constexpr int max_items = 100;
+  op_ret = ps.get_topics(this, start_token, max_items, result, next_token, y);
   // if there are no topics it is not considered an error
   op_ret = op_ret == -ENOENT ? 0 : op_ret;
   if (op_ret < 0) {

From 1caf954dab136e7d5cff97f76ce362c1bfb8d3f4 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 10 Jan 2024 17:07:27 -0500
Subject: [PATCH 2159/2492] rgw/topic: metadata key format in rgw_pubsub.h

the format of topic metadata keys is agnostic to the backend, so the
parsing/formatting functions should be in rgw_pubsub.h

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_sal_rados.cc | 10 ++++-----
 src/rgw/rgw_pubsub.cc                 | 29 +++++++++++++++++++++++++--
 src/rgw/rgw_pubsub.h                  |  7 +++++++
 src/rgw/services/svc_topic_rados.h    |  9 +--------
 4 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 6aba08828b72..0e6cbf96aea8 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1149,8 +1149,8 @@ int RadosStore::read_topic_v2(const std::string& topic_name,
       svc()->topic->svc.meta_be->alloc_ctx());
   ctx->init(svc()->topic->get_be_handler());
   const int ret = svc()->topic->svc.meta_be->get(
-      ctx.get(), get_topic_key(topic_name, tenant), params, objv_tracker, y,
-      dpp);
+      ctx.get(), get_topic_metadata_key(tenant, topic_name),
+      params, objv_tracker, y, dpp);
   if (ret < 0) {
     return ret;
   }
@@ -1178,8 +1178,8 @@ int RadosStore::write_topic_v2(const rgw_pubsub_topic& topic,
       svc()->topic->svc.meta_be->alloc_ctx());
   ctx->init(svc()->topic->get_be_handler());
   return svc()->topic->svc.meta_be->put(
-      ctx.get(), get_topic_key(topic.name, topic.user.tenant), params,
-      objv_tracker, y, dpp);
+      ctx.get(), get_topic_metadata_key(topic.user.tenant, topic.name),
+      params, objv_tracker, y, dpp);
 }
 
 int RadosStore::remove_topic_v2(const std::string& topic_name,
@@ -1192,7 +1192,7 @@ int RadosStore::remove_topic_v2(const std::string& topic_name,
       svc()->topic->svc.meta_be->alloc_ctx());
   ctx->init(svc()->topic->get_be_handler());
   return svc()->topic->svc.meta_be->remove(ctx.get(),
-                                           get_topic_key(topic_name, tenant),
+                                           get_topic_metadata_key(tenant, topic_name),
                                            params, objv_tracker, y, dpp);
 }
 
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index bec78b687c83..474a7c231634 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -5,6 +5,7 @@
 #include "rgw_b64.h"
 #include "rgw_sal.h"
 #include "rgw_pubsub.h"
+#include "rgw_string.h"
 #include "rgw_tools.h"
 #include "rgw_xml.h"
 #include "rgw_arn.h"
@@ -17,6 +18,30 @@
 
 #define dout_subsys ceph_subsys_rgw
 
+static constexpr std::string_view topic_tenant_delim = ":";
+
+// format and parse topic metadata keys as tenant:name
+std::string get_topic_metadata_key(std::string_view tenant,
+                                   std::string_view topic_name)
+{
+  return string_cat_reserve(tenant, topic_tenant_delim, topic_name);
+}
+
+void parse_topic_metadata_key(const std::string& key,
+                              std::string& tenant,
+                              std::string& name)
+{
+  // expected format: tenant_name:topic_name*
+  auto pos = key.find(topic_tenant_delim);
+  if (pos != std::string::npos) {
+    tenant = key.substr(0, pos);
+    name = key.substr(pos + 1);
+  } else {
+    tenant.clear();
+    name = key;
+  }
+}
+
 void set_event_id(std::string& id, const std::string& hash, const utime_t& ts) {
   char buf[64];
   const auto len = snprintf(buf, sizeof(buf), "%010ld.%06ld.%s", (long)ts.sec(), (long)ts.usec(), hash.c_str());
@@ -540,10 +565,10 @@ int RGWPubSub::get_topics(const DoutPrefixProvider* dpp,
         << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl;
     return ret;
   }
-  for (auto& topic_entry : topics) {
+  for (const auto& key : topics) {
     std::string topic_name;
     std::string topic_tenant;
-    parse_topic_entry(topic_entry, &topic_tenant, &topic_name);
+    parse_topic_metadata_key(key, topic_tenant, topic_name);
     if (tenant != topic_tenant) {
       continue;
     }
diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h
index 519c1053ab31..f03d7542b738 100644
--- a/src/rgw/rgw_pubsub.h
+++ b/src/rgw/rgw_pubsub.h
@@ -690,3 +690,10 @@ int remove_notification_v2(const DoutPrefixProvider* dpp,
 int get_bucket_notifications(const DoutPrefixProvider* dpp,
                              rgw::sal::Bucket* bucket,
                              rgw_pubsub_bucket_topics& bucket_topics);
+
+// format and parse topic metadata keys as tenant:name
+std::string get_topic_metadata_key(std::string_view topic_name,
+                                   std::string_view tenant);
+void parse_topic_metadata_key(const std::string& key,
+                              std::string& tenant_name,
+                              std::string& topic_name);
diff --git a/src/rgw/services/svc_topic_rados.h b/src/rgw/services/svc_topic_rados.h
index bc4e35373459..4d7f37cc02d4 100644
--- a/src/rgw/services/svc_topic_rados.h
+++ b/src/rgw/services/svc_topic_rados.h
@@ -88,11 +88,4 @@ class RGWTopicMetadataHandler : public RGWMetadataHandler_GenericMetaBE {
   RGWSI_Topic_RADOS* topic_svc;
 };
 
-std::string get_topic_key(const std::string& topic_name,
-                          const std::string& tenant);
-
-void parse_topic_entry(const std::string& topic_entry,
-                       std::string* tenant_name,
-                       std::string* topic_name);
-
-std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic);
\ No newline at end of file
+std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic);

From 6d1d036afb2d1624674fef43f2e70ef3b3ae2859 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 10 Jan 2024 15:33:25 -0500
Subject: [PATCH 2160/2492] rgw/topic: add rgwrados::topic interface for topic
 metadata

add a new interface for topic metadata that doesn't depend on metadata
backends. this low-level interface is used by both RadosStore and the
topic metadata handler

remove Driver::delete_bucket_topic_mapping() from sal because the omap
object is deleted internally by rgwrados::topic::remove()

remove the RGWRados::topics_pool_ctx member

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/CMakeLists.txt                |   3 +-
 src/rgw/driver/rados/rgw_rados.cc     |  10 -
 src/rgw/driver/rados/rgw_rados.h      |   4 -
 src/rgw/driver/rados/rgw_sal_rados.cc | 136 +++------
 src/rgw/driver/rados/rgw_sal_rados.h  |   9 +-
 src/rgw/driver/rados/rgw_service.cc   |   9 +
 src/rgw/driver/rados/rgw_service.h    |   9 +
 src/rgw/driver/rados/topic.cc         | 383 ++++++++++++++++++++++++++
 src/rgw/driver/rados/topic.h          |  94 +++++++
 src/rgw/rgw_pubsub.cc                 |   7 +-
 src/rgw/rgw_sal.h                     |  12 +-
 src/rgw/rgw_sal_filter.h              |  13 +-
 src/rgw/rgw_sal_store.h               |  11 +-
 13 files changed, 554 insertions(+), 146 deletions(-)
 create mode 100644 src/rgw/driver/rados/topic.cc
 create mode 100644 src/rgw/driver/rados/topic.h

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 2987b70b3826..031fc47bfc29 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -194,7 +194,8 @@ set(librgw_common_srcs
   driver/rados/rgw_trim_mdlog.cc
   driver/rados/rgw_user.cc
   driver/rados/rgw_zone.cc
-  driver/rados/sync_fairness.cc)
+  driver/rados/sync_fairness.cc
+  driver/rados/topic.cc)
 
 list(APPEND librgw_common_srcs
   driver/immutable_config/store.cc
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 6084c21ef8ea..1d8b13b92978 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -1211,10 +1211,6 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
   if (ret < 0)
     return ret;
 
-  ret = open_topics_pool_ctx(dpp);
-  if (ret < 0)
-    return ret;
-
   pools_initialized = true;
 
   if (use_gc) {
@@ -1446,12 +1442,6 @@ int RGWRados::open_notif_pool_ctx(const DoutPrefixProvider *dpp)
   return rgw_init_ioctx(dpp, get_rados_handle(), svc.zone->get_zone_params().notif_pool, notif_pool_ctx, true, true);
 }
 
-int RGWRados::open_topics_pool_ctx(const DoutPrefixProvider* dpp) {
-  return rgw_init_ioctx(dpp, get_rados_handle(),
-                        svc.zone->get_zone_params().topics_pool,
-                        topics_pool_ctx, true, true);
-}
-
 int RGWRados::open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx,
 			    bool mostly_omap, bool bulk)
 {
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index f43b1b4e5314..3d7776b0fa0d 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -359,7 +359,6 @@ class RGWRados
   int open_objexp_pool_ctx(const DoutPrefixProvider *dpp);
   int open_reshard_pool_ctx(const DoutPrefixProvider *dpp);
   int open_notif_pool_ctx(const DoutPrefixProvider *dpp);
-  int open_topics_pool_ctx(const DoutPrefixProvider* dpp);
 
   int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx&  io_ctx,
 		    bool mostly_omap, bool bulk);
@@ -449,7 +448,6 @@ class RGWRados
   librados::IoCtx objexp_pool_ctx;
   librados::IoCtx reshard_pool_ctx;
   librados::IoCtx notif_pool_ctx;     // .rgw.notif
-  librados::IoCtx topics_pool_ctx;  // .rgw.meta:topics
 
   bool pools_initialized{false};
 
@@ -535,8 +533,6 @@ class RGWRados
   librados::IoCtx& get_notif_pool_ctx() {
     return notif_pool_ctx;
   }
-
-  librados::IoCtx& get_topics_pool_ctx() { return topics_pool_ctx; }
   
   void set_context(CephContext *_cct) {
     cct = _cct;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 0e6cbf96aea8..dcec859cc498 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -66,6 +66,7 @@
 #include "cls/rgw/cls_rgw_client.h"
 
 #include "rgw_pubsub.h"
+#include "topic.h"
 
 #define dout_subsys ceph_subsys_rgw
 
@@ -1141,59 +1142,35 @@ int RadosStore::read_topic_v2(const std::string& topic_name,
                               rgw_pubsub_topic& topic,
                               RGWObjVersionTracker* objv_tracker,
                               optional_yield y,
-                              const DoutPrefixProvider* dpp) {
-  bufferlist bl;
-  auto mtime = ceph::real_clock::zero();
-  RGWSI_MBSObj_GetParams params(&bl, nullptr, &mtime);
-  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
-      svc()->topic->svc.meta_be->alloc_ctx());
-  ctx->init(svc()->topic->get_be_handler());
-  const int ret = svc()->topic->svc.meta_be->get(
-      ctx.get(), get_topic_metadata_key(tenant, topic_name),
-      params, objv_tracker, y, dpp);
-  if (ret < 0) {
-    return ret;
-  }
-
-  auto iter = bl.cbegin();
-  try {
-    decode(topic, iter);
-  } catch (buffer::error& err) {
-    ldpp_dout(dpp, 20) << " failed to decode topic: " << topic_name
-                       << ". error: " << err.what() << dendl;
-    return -EIO;
-  }
-  return 0;
+                              const DoutPrefixProvider* dpp)
+{
+  const RGWZoneParams& zone = svc()->zone->get_zone_params();
+  const std::string key = get_topic_metadata_key(tenant, topic_name);
+  return rgwrados::topic::read(dpp, y, *svc()->sysobj, svc()->cache,
+                               zone, key, topic, *ctl()->meta.topic_cache,
+                               nullptr, objv_tracker);
 }
 
-int RadosStore::write_topic_v2(const rgw_pubsub_topic& topic,
-                               RGWObjVersionTracker* objv_tracker,
+int RadosStore::write_topic_v2(const rgw_pubsub_topic& topic, bool exclusive,
+                               RGWObjVersionTracker& objv_tracker,
                                optional_yield y,
-                               const DoutPrefixProvider* dpp) {
-  bufferlist bl;
-  encode(topic, bl);
-  RGWSI_MBSObj_PutParams params(bl, nullptr, ceph::real_clock::zero(),
-                                /*exclusive*/ false);
-  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
-      svc()->topic->svc.meta_be->alloc_ctx());
-  ctx->init(svc()->topic->get_be_handler());
-  return svc()->topic->svc.meta_be->put(
-      ctx.get(), get_topic_metadata_key(topic.user.tenant, topic.name),
-      params, objv_tracker, y, dpp);
+                               const DoutPrefixProvider* dpp)
+{
+  const RGWZoneParams& zone = svc()->zone->get_zone_params();
+  return rgwrados::topic::write(dpp, y, *svc()->sysobj, svc()->mdlog, zone,
+                                topic, objv_tracker, {}, exclusive);
 }
 
 int RadosStore::remove_topic_v2(const std::string& topic_name,
                                 const std::string& tenant,
-                                RGWObjVersionTracker* objv_tracker,
+                                RGWObjVersionTracker& objv_tracker,
                                 optional_yield y,
-                                const DoutPrefixProvider* dpp) {
-  RGWSI_MBSObj_RemoveParams params;
-  std::unique_ptr<RGWSI_MetaBackend::Context> ctx(
-      svc()->topic->svc.meta_be->alloc_ctx());
-  ctx->init(svc()->topic->get_be_handler());
-  return svc()->topic->svc.meta_be->remove(ctx.get(),
-                                           get_topic_metadata_key(tenant, topic_name),
-                                           params, objv_tracker, y, dpp);
+                                const DoutPrefixProvider* dpp)
+{
+  const RGWZoneParams& zone = svc()->zone->get_zone_params();
+  const std::string key = get_topic_metadata_key(tenant, topic_name);
+  return rgwrados::topic::remove(dpp, y, *svc()->sysobj, svc()->mdlog,
+                                 zone, key, objv_tracker);
 }
 
 int RadosStore::remove_bucket_mapping_from_topics(
@@ -1223,18 +1200,15 @@ int RadosStore::update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
                                             bool add_mapping,
                                             optional_yield y,
                                             const DoutPrefixProvider* dpp) {
-  bufferlist empty_bl;
-  librados::ObjectWriteOperation op;
+  librados::Rados& rados = *getRados()->get_rados_handle();
+  const RGWZoneParams& zone = svc()->zone->get_zone_params();
+  const std::string key = get_topic_metadata_key(topic.user.tenant, topic.name);
   int ret = 0;
   if (add_mapping) {
-    std::map<std::string, bufferlist> mapping{{bucket_key, empty_bl}};
-    op.omap_set(mapping);
+    ret = rgwrados::topic::link_bucket(dpp, y, rados, zone, key, bucket_key);
   } else {
-    std::set<std::string> to_rm{{bucket_key}};
-    op.omap_rm_keys(to_rm);
+    ret = rgwrados::topic::unlink_bucket(dpp, y, rados, zone, key, bucket_key);
   }
-  ret = rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
-                          get_bucket_topic_mapping_oid(topic), &op, y);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to " << (add_mapping ? "add" : "remove")
                       << " topic bucket mapping for bucket: " << bucket_key
@@ -1250,57 +1224,25 @@ int RadosStore::update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
 int RadosStore::get_bucket_topic_mapping(const rgw_pubsub_topic& topic,
                                          std::set<std::string>& bucket_keys,
                                          optional_yield y,
-                                         const DoutPrefixProvider* dpp) {
-  constexpr auto max_chunk = 1024U;
-  std::string start_after;
-  bool more = true;
-  int rval;
-  while (more) {
-    librados::ObjectReadOperation op;
-    std::set<std::string> curr_keys;
-    op.omap_get_keys2(start_after, max_chunk, &curr_keys, &more, &rval);
-    const auto ret =
-        rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
-                          get_bucket_topic_mapping_oid(topic), &op, nullptr, y);
-    if (ret == -ENOENT) {
-      // mapping object was not created - nothing to do
-      return 0;
-    }
+                                         const DoutPrefixProvider* dpp)
+{
+  librados::Rados& rados = *getRados()->get_rados_handle();
+  const RGWZoneParams& zone = svc()->zone->get_zone_params();
+  const std::string key = get_topic_metadata_key(topic.user.tenant, topic.name);
+  constexpr int max_chunk = 1024;
+  std::string marker;
+
+  do {
+    int ret = rgwrados::topic::list_buckets(dpp, y, rados, zone, key, marker,
+                                            max_chunk, bucket_keys, marker);
     if (ret < 0) {
-      // TODO: do we need to check on rval as well as ret?
       ldpp_dout(dpp, 1)
           << "ERROR: failed to read bucket topic mapping object for topic: "
           << topic.name << ", ret= " << ret << dendl;
       return ret;
     }
-    if (more) {
-      if (curr_keys.empty()) {
-        return -EINVAL;  // something wrong.
-      }
-      start_after = *curr_keys.rbegin();
-    }
-    bucket_keys.merge(curr_keys);
-  }
-  return 0;
-}
+  } while (!marker.empty());
 
-int RadosStore::delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
-                                            optional_yield y,
-                                            const DoutPrefixProvider* dpp) {
-  librados::ObjectWriteOperation op;
-  op.remove();
-  const int ret =
-      rgw_rados_operate(dpp, rados->get_topics_pool_ctx(),
-                        get_bucket_topic_mapping_oid(topic), &op, y);
-  if (ret < 0 && ret != -ENOENT) {
-    ldpp_dout(dpp, 1)
-        << "ERROR: failed removing bucket topic mapping omap for topic: "
-        << topic.name << ", ret=" << ret << dendl;
-    return ret;
-  }
-  ldpp_dout(dpp, 20)
-      << "Successfully deleted topic bucket mapping omap for topic: "
-      << topic.name << dendl;
   return 0;
 }
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index e15c754d1b49..1eccb89dad33 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -170,13 +170,13 @@ class RadosStore : public StoreDriver {
                       RGWObjVersionTracker* objv_tracker,
                       optional_yield y,
                       const DoutPrefixProvider* dpp) override;
-    int write_topic_v2(const rgw_pubsub_topic& topic,
-                       RGWObjVersionTracker* objv_tracker,
+    int write_topic_v2(const rgw_pubsub_topic& topic, bool exclusive,
+                       RGWObjVersionTracker& objv_tracker,
                        optional_yield y,
                        const DoutPrefixProvider* dpp) override;
     int remove_topic_v2(const std::string& topic_name,
                         const std::string& tenant,
-                        RGWObjVersionTracker* objv_tracker,
+                        RGWObjVersionTracker& objv_tracker,
                         optional_yield y,
                         const DoutPrefixProvider* dpp) override;
     int update_bucket_topic_mapping(const rgw_pubsub_topic& topic,
@@ -193,9 +193,6 @@ class RadosStore : public StoreDriver {
                                  std::set<std::string>& bucket_keys,
                                  optional_yield y,
                                  const DoutPrefixProvider* dpp) override;
-    int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
-                                    optional_yield y,
-                                    const DoutPrefixProvider* dpp) override;
     virtual RGWLC* get_rgwlc(void) override { return rados->get_lc(); }
     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return rados->get_cr_registry(); }
 
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index 0c0e2bbea659..b264a7c2e776 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -39,6 +39,7 @@
 #include "rgw_user.h"
 #include "rgw_role.h"
 #include "rgw_pubsub.h"
+#include "topic.h"
 
 #define dout_subsys ceph_subsys_rgw
 
@@ -416,6 +417,13 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, const DoutPrefix
   bucket_meta_handler->init(svc.bucket, bucket.get());
   bi_meta_handler->init(svc.zone, svc.bucket, svc.bi);
 
+  meta.topic_cache = std::make_unique<RGWChainedCacheImpl<rgwrados::topic::cache_entry>>();
+  meta.topic_cache->init(svc.cache);
+
+  meta.topic = rgwrados::topic::create_metadata_handler(
+      *svc.sysobj, svc.cache, *svc.mdlog, svc.zone->get_zone_params(),
+      *meta.topic_cache);
+
   RGWOTPMetadataHandlerBase *otp_handler = static_cast<RGWOTPMetadataHandlerBase *>(meta.otp.get());
   otp_handler->init(svc.zone, svc.meta_be_otp, svc.otp);
 
@@ -449,6 +457,7 @@ int RGWCtl::init(RGWServices *_svc, rgw::sal::Driver* driver, const DoutPrefixPr
   meta.otp = _ctl.meta.otp.get();
   meta.role = _ctl.meta.role.get();
   meta.topic = _ctl.meta.topic.get();
+  meta.topic_cache = _ctl.meta.topic_cache.get();
 
   user = _ctl.user.get();
   bucket = _ctl.bucket.get();
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 03b37f1ad5a2..617516b8a028 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -17,6 +17,8 @@ class RadosStore;
 
 struct RGWServices_Def;
 
+namespace rgwrados::topic { struct cache_entry; }
+
 class RGWServiceInstance
 {
   friend struct RGWServices_Def;
@@ -186,6 +188,9 @@ class RGWUserCtl;
 class RGWBucketCtl;
 class RGWOTPCtl;
 
+template <class T>
+class RGWChainedCacheImpl;
+
 struct RGWCtlDef {
   struct _meta {
     std::unique_ptr<RGWMetadataManager> mgr;
@@ -196,6 +201,8 @@ struct RGWCtlDef {
     std::unique_ptr<RGWMetadataHandler> role;
     std::unique_ptr<RGWMetadataHandler> topic;
 
+    std::unique_ptr<RGWChainedCacheImpl<rgwrados::topic::cache_entry>> topic_cache;
+
     _meta();
     ~_meta();
   } meta;
@@ -225,6 +232,8 @@ struct RGWCtl {
     RGWMetadataHandler *otp{nullptr};
     RGWMetadataHandler *role{nullptr};
     RGWMetadataHandler* topic{nullptr};
+
+    RGWChainedCacheImpl<rgwrados::topic::cache_entry>* topic_cache{nullptr};
   } meta;
 
   RGWUserCtl *user{nullptr};
diff --git a/src/rgw/driver/rados/topic.cc b/src/rgw/driver/rados/topic.cc
new file mode 100644
index 000000000000..3e409e0b07b3
--- /dev/null
+++ b/src/rgw/driver/rados/topic.cc
@@ -0,0 +1,383 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "topic.h"
+#include "common/errno.h"
+#include "rgw_common.h"
+#include "rgw_metadata.h"
+#include "rgw_metadata_lister.h"
+#include "rgw_pubsub.h"
+#include "rgw_rados.h"
+#include "rgw_string.h"
+#include "rgw_tools.h"
+#include "rgw_zone.h"
+#include "svc_mdlog.h"
+#include "svc_sys_obj_cache.h"
+
+namespace rgwrados::topic {
+
+static const std::string oid_prefix = "topic.";
+static constexpr std::string_view buckets_oid_prefix = "buckets.";
+
+static rgw_raw_obj get_topic_obj(const RGWZoneParams& zone,
+                                 std::string_view metadata_key)
+{
+  std::string oid = string_cat_reserve(oid_prefix, metadata_key);
+  return {zone.topics_pool, std::move(oid)};
+}
+
+static rgw_raw_obj get_buckets_obj(const RGWZoneParams& zone,
+                                   std::string_view metadata_key)
+{
+  std::string oid = string_cat_reserve(buckets_oid_prefix, metadata_key);
+  return {zone.topics_pool, std::move(oid)};
+}
+
+
+int read(const DoutPrefixProvider* dpp, optional_yield y,
+         RGWSI_SysObj& sysobj, RGWSI_SysObj_Cache* cache_svc,
+         const RGWZoneParams& zone, const std::string& topic_key,
+         rgw_pubsub_topic& info, RGWChainedCacheImpl<cache_entry>& cache,
+         ceph::real_time* pmtime, RGWObjVersionTracker* pobjv)
+{
+  if (auto e = cache.find(topic_key)) {
+    if (pmtime) {
+      *pmtime = e->mtime;
+    }
+    if (pobjv) {
+      *pobjv = std::move(e->objv);
+    }
+    info = std::move(e->info);
+    return 0;
+  }
+
+  const rgw_raw_obj obj = get_topic_obj(zone, topic_key);
+
+  bufferlist bl;
+  cache_entry entry;
+  rgw_cache_entry_info cache_info;
+  int r = rgw_get_system_obj(&sysobj, obj.pool, obj.oid, bl, &entry.objv,
+                             &entry.mtime, y, dpp, nullptr, &cache_info);
+  if (r < 0) {
+    return r;
+  }
+
+  try {
+    auto p = bl.cbegin();
+    decode(entry.info, p);
+  } catch (const buffer::error&) {
+    return -EIO;
+  }
+
+  cache.put(dpp, cache_svc, topic_key, &entry, {&cache_info});
+
+  if (pmtime) {
+    *pmtime = entry.mtime;
+  }
+  if (pobjv) {
+    *pobjv = std::move(entry.objv);
+  }
+  info = std::move(entry.info);
+  return 0;
+}
+
+int write(const DoutPrefixProvider* dpp, optional_yield y,
+          RGWSI_SysObj& sysobj, RGWSI_MDLog* mdlog, const RGWZoneParams& zone,
+          const rgw_pubsub_topic& info, RGWObjVersionTracker& objv,
+          ceph::real_time mtime, bool exclusive)
+{
+  const std::string topic_key = get_topic_metadata_key(info.user.tenant, info.name);
+  const rgw_raw_obj obj = get_topic_obj(zone, topic_key);
+
+  bufferlist bl;
+  encode(info, bl);
+
+  int r = rgw_put_system_obj(dpp, &sysobj, obj.pool, obj.oid,
+                             bl, exclusive, &objv, mtime, y);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to write topic obj " << obj.oid
+        << " with: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  // record in the mdlog on success
+  if (mdlog) {
+    return mdlog->complete_entry(dpp, y, "topic", topic_key, &objv);
+  }
+  return 0;
+}
+
+int remove(const DoutPrefixProvider* dpp, optional_yield y,
+           RGWSI_SysObj& sysobj, RGWSI_MDLog* mdlog, const RGWZoneParams& zone,
+           const std::string& topic_key, RGWObjVersionTracker& objv)
+{
+  // delete topic info
+  const rgw_raw_obj topic = get_topic_obj(zone, topic_key);
+  int r = rgw_delete_system_obj(dpp, &sysobj, topic.pool, topic.oid, &objv, y);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to remove topic obj "
+        << topic.oid << " with: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  // delete the buckets object
+  const rgw_raw_obj buckets = get_buckets_obj(zone, topic_key);
+  r = rgw_delete_system_obj(dpp, &sysobj, buckets.pool,
+                            buckets.oid, nullptr, y);
+  if (r < 0) {
+    ldpp_dout(dpp, 20) << "WARNING: failed to remove topic buckets obj "
+        << buckets.oid << " with: " << cpp_strerror(r) << dendl;
+  } // not fatal
+
+  // record in the mdlog on success
+  if (mdlog) {
+    return mdlog->complete_entry(dpp, y, "topic", topic_key, &objv);
+  }
+  return 0;
+}
+
+
+int link_bucket(const DoutPrefixProvider* dpp, optional_yield y,
+                librados::Rados& rados, const RGWZoneParams& zone,
+                const std::string& topic_key,
+                const std::string& bucket_key)
+{
+  const rgw_raw_obj obj = get_buckets_obj(zone, topic_key);
+
+  rgw_rados_ref ref;
+  int r = rgw_get_rados_ref(dpp, &rados, obj, &ref);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation op;
+  op.omap_set({{bucket_key, bufferlist{}}});
+
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
+}
+
+int unlink_bucket(const DoutPrefixProvider* dpp, optional_yield y,
+                  librados::Rados& rados, const RGWZoneParams& zone,
+                  const std::string& topic_key,
+                  const std::string& bucket_key)
+{
+  const rgw_raw_obj obj = get_buckets_obj(zone, topic_key);
+
+  rgw_rados_ref ref;
+  int r = rgw_get_rados_ref(dpp, &rados, obj, &ref);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectWriteOperation op;
+  op.omap_rm_keys({{bucket_key}});
+
+  return rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, y);
+}
+
+int list_buckets(const DoutPrefixProvider* dpp, optional_yield y,
+                 librados::Rados& rados, const RGWZoneParams& zone,
+                 const std::string& topic_key,
+                 const std::string& marker, int max_items,
+                 std::set<std::string>& bucket_keys,
+                 std::string& next_marker)
+{
+  const rgw_raw_obj obj = get_buckets_obj(zone, topic_key);
+
+  rgw_rados_ref ref;
+  int r = rgw_get_rados_ref(dpp, &rados, obj, &ref);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::ObjectReadOperation op;
+  std::set<std::string> keys;
+  bool more = false;
+  int rval = 0;
+  op.omap_get_keys2(marker, max_items, &keys, &more, &rval);
+  r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, nullptr, y);
+  if (r == -ENOENT) {
+    return 0;
+  }
+  if (r < 0) {
+    return r;
+  }
+  if (rval < 0) {
+    return rval;
+  }
+
+  if (more && !keys.empty()) {
+    next_marker = *keys.rbegin();
+  } else {
+    next_marker.clear();
+  }
+  bucket_keys.merge(std::move(keys));
+
+  return 0;
+}
+
+
+class MetadataObject : public RGWMetadataObject {
+  rgw_pubsub_topic info;
+public:
+  MetadataObject(const rgw_pubsub_topic& info, const obj_version& v, real_time m)
+    : RGWMetadataObject(v, m), info(info) {}
+
+  void dump(Formatter *f) const override {
+    info.dump(f);
+  }
+
+  rgw_pubsub_topic& get_topic_info() {
+    return info;
+  }
+};
+
+class MetadataLister : public RGWMetadataLister {
+ public:
+  using RGWMetadataLister::RGWMetadataLister;
+
+  virtual void filter_transform(std::vector<std::string>& oids,
+                                std::list<std::string>& keys) {
+    // remove the oid prefix from keys
+    constexpr auto trim = [] (const std::string& oid) {
+      return oid.substr(oid_prefix.size());
+    };
+    std::transform(oids.begin(), oids.end(),
+                   std::back_inserter(keys),
+                   trim);
+  }
+};
+
+class MetadataHandler : public RGWMetadataHandler {
+  RGWSI_SysObj& sysobj;
+  RGWSI_SysObj_Cache* cache_svc;
+  RGWSI_MDLog& mdlog;
+  const RGWZoneParams& zone;
+  RGWChainedCacheImpl<cache_entry>& cache;
+ public:
+  MetadataHandler(RGWSI_SysObj& sysobj, RGWSI_SysObj_Cache* cache_svc,
+                  RGWSI_MDLog& mdlog, const RGWZoneParams& zone,
+                  RGWChainedCacheImpl<cache_entry>& cache)
+    : sysobj(sysobj), cache_svc(cache_svc), mdlog(mdlog),
+      zone(zone), cache(cache)
+  {}
+
+  std::string get_type() final { return "topic";  }
+
+  RGWMetadataObject* get_meta_obj(JSONObj *jo,
+                                  const obj_version& objv,
+                                  const ceph::real_time& mtime) override
+  {
+    rgw_pubsub_topic info;
+
+    try {
+      info.decode_json(jo);
+    } catch (JSONDecoder:: err& e) {
+      return nullptr;
+    }
+
+    return new MetadataObject(info, objv, mtime);
+  }
+
+  int get(std::string& entry, RGWMetadataObject** obj,
+          optional_yield y, const DoutPrefixProvider* dpp) override
+  {
+    cache_entry e;
+    int ret = read(dpp, y, sysobj, cache_svc, zone, entry,
+                   e.info, cache, &e.mtime, &e.objv);
+    if (ret < 0) {
+      return ret;
+    }
+
+    *obj = new MetadataObject(e.info, e.objv.read_version, e.mtime);
+    return 0;
+  }
+
+  int put(std::string& entry, RGWMetadataObject* obj,
+          RGWObjVersionTracker& objv_tracker,
+          optional_yield y, const DoutPrefixProvider* dpp,
+          RGWMDLogSyncType type, bool from_remote_zone) override
+  {
+    auto robj = static_cast<MetadataObject*>(obj);
+    auto& info = robj->get_topic_info();
+    auto mtime = robj->get_mtime();
+
+    constexpr bool exclusive = false;
+    int ret = write(dpp, y, sysobj, &mdlog, zone, info,
+                    objv_tracker, mtime, exclusive);
+    return ret < 0 ? ret : STATUS_APPLIED;
+  }
+
+  int remove(std::string& entry, RGWObjVersionTracker& objv_tracker,
+             optional_yield y, const DoutPrefixProvider *dpp) override
+  {
+    return topic::remove(dpp, y, sysobj, &mdlog, zone, entry, objv_tracker);
+  }
+
+  int mutate(const std::string& entry, const ceph::real_time& mtime,
+             RGWObjVersionTracker* objv_tracker, optional_yield y,
+             const DoutPrefixProvider* dpp, RGWMDLogStatus op_type,
+             std::function<int()> f) override
+  {
+    return -ENOTSUP; // unused
+  }
+
+  int list_keys_init(const DoutPrefixProvider* dpp,
+                     const std::string& marker,
+                     void** phandle) override
+  {
+    const auto& pool = zone.topics_pool;
+    auto lister = std::make_unique<MetadataLister>(sysobj.get_pool(pool));
+    int ret = lister->init(dpp, marker, oid_prefix);
+    if (ret < 0) {
+      return ret;
+    }
+    *phandle = lister.release(); // release ownership
+    return 0;
+  }
+
+  int list_keys_next(const DoutPrefixProvider* dpp,
+                     void* handle, int max,
+                     std::list<std::string>& keys,
+                     bool* truncated) override
+  {
+    auto lister = static_cast<RGWMetadataLister*>(handle);
+    return lister->get_next(dpp, max, keys, truncated);
+  }
+
+  void list_keys_complete(void *handle) override
+  {
+    delete static_cast<RGWMetadataLister*>(handle);
+  }
+
+  std::string get_marker(void *handle) override
+  {
+    auto lister = static_cast<RGWMetadataLister*>(handle);
+    return lister->get_marker();
+  }
+};
+
+
+auto create_metadata_handler(RGWSI_SysObj& sysobj,
+                             RGWSI_SysObj_Cache* cache_svc,
+                             RGWSI_MDLog& mdlog, const RGWZoneParams& zone,
+                             RGWChainedCacheImpl<cache_entry>& cache)
+    -> std::unique_ptr<RGWMetadataHandler>
+{
+  return std::make_unique<MetadataHandler>(sysobj, cache_svc, mdlog,
+                                           zone, cache);
+}
+
+} // rgwrados::topic
diff --git a/src/rgw/driver/rados/topic.h b/src/rgw/driver/rados/topic.h
new file mode 100644
index 000000000000..3799d001ec7a
--- /dev/null
+++ b/src/rgw/driver/rados/topic.h
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include "include/rados/librados_fwd.hpp"
+#include "common/ceph_time.h"
+#include "rgw_pubsub.h"
+
+class DoutPrefixProvider;
+class optional_yield;
+class RGWMetadataHandler;
+class RGWObjVersionTracker;
+class RGWSI_MDLog;
+class RGWSI_SysObj;
+class RGWSI_SysObj_Cache;
+class RGWZoneParams;
+
+template <typename T> class RGWChainedCacheImpl;
+
+// Rados interface for v2 topic metadata
+namespace rgwrados::topic {
+
+struct cache_entry {
+  rgw_pubsub_topic info;
+  RGWObjVersionTracker objv;
+  ceph::real_time mtime;
+};
+
+/// Read topic info by metadata key.
+int read(const DoutPrefixProvider* dpp, optional_yield y,
+         RGWSI_SysObj& sysobj, RGWSI_SysObj_Cache* cache_svc,
+         const RGWZoneParams& zone, const std::string& topic_key,
+         rgw_pubsub_topic& info, RGWChainedCacheImpl<cache_entry>& cache,
+         ceph::real_time* pmtime = nullptr,
+         RGWObjVersionTracker* pobjv = nullptr);
+
+/// Write or overwrite topic info.
+int write(const DoutPrefixProvider* dpp, optional_yield y,
+          RGWSI_SysObj& sysobj, RGWSI_MDLog* mdlog, const RGWZoneParams& zone,
+          const rgw_pubsub_topic& info, RGWObjVersionTracker& objv,
+          ceph::real_time mtime, bool exclusive);
+
+/// Remove a topic by metadata key.
+int remove(const DoutPrefixProvider* dpp, optional_yield y,
+           RGWSI_SysObj& sysobj, RGWSI_MDLog* mdlog,
+           const RGWZoneParams& zone, const std::string& topic_key,
+           RGWObjVersionTracker& objv);
+
+
+/// Add a bucket key to the topic's list of buckets.
+int link_bucket(const DoutPrefixProvider* dpp, optional_yield y,
+                librados::Rados& rados, const RGWZoneParams& zone,
+                const std::string& topic_key,
+                const std::string& bucket_key);
+
+/// Remove a bucket key from the topic's list of buckets.
+int unlink_bucket(const DoutPrefixProvider* dpp, optional_yield y,
+                  librados::Rados& rados, const RGWZoneParams& zone,
+                  const std::string& topic_key,
+                  const std::string& bucket_key);
+
+/// List the bucket keys associated with a given topic.
+int list_buckets(const DoutPrefixProvider* dpp, optional_yield y,
+                 librados::Rados& rados, const RGWZoneParams& zone,
+                 const std::string& topic_key,
+                 const std::string& marker, int max_items,
+                 std::set<std::string>& bucket_keys,
+                 std::string& next_marker);
+
+
+/// Topic metadata handler factory.
+auto create_metadata_handler(RGWSI_SysObj& sysobj,
+                             RGWSI_SysObj_Cache* cache_svc,
+                             RGWSI_MDLog& mdlog, const RGWZoneParams& zone,
+                             RGWChainedCacheImpl<cache_entry>& cache)
+    -> std::unique_ptr<RGWMetadataHandler>;
+
+} // rgwrados::topic
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 474a7c231634..18c604978e82 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -945,7 +945,9 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const rgw_pubsub_topic& topic,
                             optional_yield y) const {
   RGWObjVersionTracker objv_tracker;
-  auto ret = driver->write_topic_v2(topic, &objv_tracker, y, dpp);
+  objv_tracker.generate_new_write_ver(dpp->get_cct());
+  constexpr bool exclusive = false;
+  auto ret = driver->write_topic_v2(topic, exclusive, objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret
                       << dendl;
@@ -1012,13 +1014,12 @@ int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
                        << dendl;
     return 0;
   }
-  ret = driver->remove_topic_v2(name, tenant, &objv_tracker, y, dpp);
+  ret = driver->remove_topic_v2(name, tenant, objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to remove topic info: ret=" << ret
                       << dendl;
     return ret;
   }
-  ret = driver->delete_bucket_topic_mapping(topic, y, dpp);
   return ret;
 }
 
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index f1bc455835f6..060cfc1e3513 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -321,15 +321,15 @@ class Driver {
                               RGWObjVersionTracker* objv_tracker,
                               optional_yield y,
                               const DoutPrefixProvider* dpp) = 0;
-    /** Write topic info and (optionally) @a objv_tracker into the config */
-    virtual int write_topic_v2(const rgw_pubsub_topic& topic,
-                               RGWObjVersionTracker* objv_tracker,
+    /** Write topic info and @a objv_tracker into the config */
+    virtual int write_topic_v2(const rgw_pubsub_topic& topic, bool exclusive,
+                               RGWObjVersionTracker& objv_tracker,
                                optional_yield y,
                                const DoutPrefixProvider* dpp) = 0;
     /** Remove the topic config, optionally a specific version */
     virtual int remove_topic_v2(const std::string& topic_name,
                                 const std::string& tenant,
-                                RGWObjVersionTracker* objv_tracker,
+                                RGWObjVersionTracker& objv_tracker,
                                 optional_yield y,
                                 const DoutPrefixProvider* dpp) = 0;
     /** Update the bucket-topic mapping in the store, if |add_mapping|=true then
@@ -356,10 +356,6 @@ class Driver {
                                          std::set<std::string>& bucket_keys,
                                          optional_yield y,
                                          const DoutPrefixProvider* dpp) = 0;
-    /** Remove the bucket-topic mapping from the backend store. */
-    virtual int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
-                                            optional_yield y,
-                                            const DoutPrefixProvider* dpp) = 0;
     /** Get access to the lifecycle management thread */
     virtual RGWLC* get_rgwlc(void) = 0;
     /** Get access to the coroutine registry.  Used to create new coroutine managers */
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 8bb704ce17c1..83832922f90d 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -202,15 +202,15 @@ class FilterDriver : public Driver {
                     const DoutPrefixProvider* dpp) override {
     return next->read_topic_v2(topic_name, tenant, topic, objv_tracker, y, dpp);
   }
-  int write_topic_v2(const rgw_pubsub_topic& topic,
-                     RGWObjVersionTracker* objv_tracker,
+  int write_topic_v2(const rgw_pubsub_topic& topic, bool exclusive,
+                     RGWObjVersionTracker& objv_tracker,
                      optional_yield y,
                      const DoutPrefixProvider* dpp) override {
-    return next->write_topic_v2(topic, objv_tracker, y, dpp);
+    return next->write_topic_v2(topic, exclusive, objv_tracker, y, dpp);
   }
   int remove_topic_v2(const std::string& topic_name,
                       const std::string& tenant,
-                      RGWObjVersionTracker* objv_tracker,
+                      RGWObjVersionTracker& objv_tracker,
                       optional_yield y,
                       const DoutPrefixProvider* dpp) override {
     return next->remove_topic_v2(topic_name, tenant, objv_tracker, y, dpp);
@@ -237,11 +237,6 @@ class FilterDriver : public Driver {
                                const DoutPrefixProvider* dpp) override {
     return next->get_bucket_topic_mapping(topic, bucket_keys, y, dpp);
   }
-  int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
-                                  optional_yield y,
-                                  const DoutPrefixProvider* dpp) override {
-    return next->delete_bucket_topic_mapping(topic, y, dpp);
-  }
   virtual RGWLC* get_rgwlc(void) override;
   virtual RGWCoroutinesManagerRegistry* get_cr_registry() override;
 
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index b34276a9daaf..f0ac762554e6 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -42,15 +42,15 @@ class StoreDriver : public Driver {
                       const DoutPrefixProvider* dpp) override {
       return -EOPNOTSUPP;
     }
-    int write_topic_v2(const rgw_pubsub_topic& topic,
-                       RGWObjVersionTracker* objv_tracker,
+    int write_topic_v2(const rgw_pubsub_topic& topic, bool exclusive,
+                       RGWObjVersionTracker& objv_tracker,
                        optional_yield y,
                        const DoutPrefixProvider* dpp) override {
       return -EOPNOTSUPP;
     }
     int remove_topic_v2(const std::string& topic_name,
                         const std::string& tenant,
-                        RGWObjVersionTracker* objv_tracker,
+                        RGWObjVersionTracker& objv_tracker,
                         optional_yield y,
                         const DoutPrefixProvider* dpp) override {
       return -EOPNOTSUPP;
@@ -75,11 +75,6 @@ class StoreDriver : public Driver {
                                  const DoutPrefixProvider* dpp) override {
       return -EOPNOTSUPP;
     }
-    int delete_bucket_topic_mapping(const rgw_pubsub_topic& topic,
-                                    optional_yield y,
-                                    const DoutPrefixProvider* dpp) override {
-      return -EOPNOTSUPP;
-    }
 };
 
 class StoreUser : public User {

From be9cc0c7bc163f4e578bd5459866a2c1e5c8a2dd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 10 Jan 2024 17:40:11 -0500
Subject: [PATCH 2161/2492] rgw/topic: remove unused RGWSI_Topic_RADOS

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/CMakeLists.txt                |   1 -
 src/rgw/driver/rados/rgw_sal_rados.cc |   1 -
 src/rgw/driver/rados/rgw_service.cc   |  12 --
 src/rgw/driver/rados/rgw_service.h    |   3 -
 src/rgw/rgw_pubsub.cc                 |   1 -
 src/rgw/services/svc_topic_rados.cc   | 215 --------------------------
 src/rgw/services/svc_topic_rados.h    |  91 -----------
 7 files changed, 324 deletions(-)
 delete mode 100644 src/rgw/services/svc_topic_rados.cc
 delete mode 100644 src/rgw/services/svc_topic_rados.h

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 031fc47bfc29..1bf433cb3951 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -53,7 +53,6 @@ set(librgw_common_srcs
   services/svc_sys_obj_cache.cc
   services/svc_sys_obj_core.cc
   services/svc_tier_rados.cc
-  services/svc_topic_rados.cc
   services/svc_user.cc
   services/svc_user_rados.cc
   services/svc_zone.cc
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index dcec859cc498..0a6a6b7b19f8 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -62,7 +62,6 @@
 #include "services/svc_role_rados.h"
 #include "services/svc_user.h"
 #include "services/svc_sys_obj_cache.h"
-#include "services/svc_topic_rados.h"
 #include "cls/rgw/cls_rgw_client.h"
 
 #include "rgw_pubsub.h"
diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc
index b264a7c2e776..e0d885b86a6e 100644
--- a/src/rgw/driver/rados/rgw_service.cc
+++ b/src/rgw/driver/rados/rgw_service.cc
@@ -26,7 +26,6 @@
 #include "services/svc_sys_obj_core.h"
 #include "services/svc_user_rados.h"
 #include "services/svc_role_rados.h"
-#include "services/svc_topic_rados.h"
 
 #include "common/errno.h"
 
@@ -83,7 +82,6 @@ int RGWServices_Def::init(CephContext *cct,
   role_rados = std::make_unique<RGWSI_Role_RADOS>(cct);
   async_processor = std::make_unique<RGWAsyncRadosProcessor>(
     cct, cct->_conf->rgw_num_async_rados_threads);
-  topic_rados = std::make_unique<RGWSI_Topic_RADOS>(cct);
 
   if (have_cache) {
     sysobj_cache = std::make_unique<RGWSI_SysObj_Cache>(dpp, cct);
@@ -128,7 +126,6 @@ int RGWServices_Def::init(CephContext *cct,
   user_rados->init(driver->getRados()->get_rados_handle(), zone.get(), sysobj.get(), sysobj_cache.get(),
                    meta.get(), meta_be_sobj.get(), sync_modules.get());
   role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
-  topic_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get());
   can_shutdown = true;
 
   int r = finisher->start(y, dpp);
@@ -259,12 +256,6 @@ int RGWServices_Def::init(CephContext *cct,
       ldout(cct, 0) << "ERROR: failed to start role_rados service (" << cpp_strerror(-r) << dendl;
       return r;
     }
-    r = topic_rados->start(y, dpp);
-    if (r < 0) {
-      ldout(cct, 0) << "ERROR: failed to start topic_rados service ("
-                    << cpp_strerror(-r) << dendl;
-      return r;
-    }
   }
 
   /* cache or core services will be started by sysobj */
@@ -282,7 +273,6 @@ void RGWServices_Def::shutdown()
     return;
   }
 
-  topic_rados->shutdown();
   role_rados->shutdown();
   datalog_rados.reset();
   user_rados->shutdown();
@@ -353,7 +343,6 @@ int RGWServices::do_init(CephContext *_cct, rgw::sal::RadosStore* driver, bool h
   user = _svc.user_rados.get();
   role = _svc.role_rados.get();
   async_processor = _svc.async_processor.get();
-  topic = _svc.topic_rados.get();
 
   return 0;
 }
@@ -402,7 +391,6 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, const DoutPrefix
 
   meta.otp.reset(RGWOTPMetaHandlerAllocator::alloc());
   meta.role = std::make_unique<rgw::sal::RGWRoleMetadataHandler>(driver, svc.role);
-  meta.topic = std::make_unique<RGWTopicMetadataHandler>(driver, svc.topic);
 
   user.reset(new RGWUserCtl(svc.zone, svc.user, (RGWUserMetadataHandler *)meta.user.get()));
   bucket.reset(new RGWBucketCtl(svc.zone,
diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h
index 617516b8a028..ebab083f27ec 100644
--- a/src/rgw/driver/rados/rgw_service.h
+++ b/src/rgw/driver/rados/rgw_service.h
@@ -80,7 +80,6 @@ class RGWSI_User_RADOS;
 class RGWDataChangesLog;
 class RGWSI_Role_RADOS;
 class RGWAsyncRadosProcessor;
-class RGWSI_Topic_RADOS;
 
 struct RGWServices_Def
 {
@@ -112,7 +111,6 @@ struct RGWServices_Def
   std::unique_ptr<RGWSI_Role_RADOS> role_rados;
   std::unique_ptr<RGWAsyncRadosProcessor> async_processor;
 
-  std::unique_ptr<RGWSI_Topic_RADOS> topic_rados;
   RGWServices_Def();
   ~RGWServices_Def();
 
@@ -159,7 +157,6 @@ struct RGWServices
   RGWSI_User *user{nullptr};
   RGWSI_Role_RADOS *role{nullptr};
   RGWAsyncRadosProcessor* async_processor;
-  RGWSI_Topic_RADOS* topic{nullptr};
 
   int do_init(CephContext *cct, rgw::sal::RadosStore* store, bool have_cache,
 	      bool raw_storage, bool run_sync, optional_yield y,
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 18c604978e82..b1b71efdafa5 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -12,7 +12,6 @@
 #include "rgw_pubsub_push.h"
 #include "rgw_bucket.h"
 #include "common/errno.h"
-#include "svc_topic_rados.h"
 #include <regex>
 #include <algorithm>
 
diff --git a/src/rgw/services/svc_topic_rados.cc b/src/rgw/services/svc_topic_rados.cc
deleted file mode 100644
index c08d688b2719..000000000000
--- a/src/rgw/services/svc_topic_rados.cc
+++ /dev/null
@@ -1,215 +0,0 @@
-#include "svc_topic_rados.h"
-#include "rgw_notify.h"
-#include "rgw_tools.h"
-#include "rgw_zone.h"
-#include "svc_meta.h"
-#include "svc_meta_be_sobj.h"
-#include "svc_zone.h"
-
-#define dout_subsys ceph_subsys_rgw
-
-static std::string topic_oid_prefix = "topic.";
-static constexpr char topic_tenant_delim[] = ":";
-static std::string bucket_topic_oid_prefix = "buckets.";
-
-std::string get_topic_key(const std::string& topic_name,
-                          const std::string& tenant) {
-  if (tenant.empty()) {
-    return topic_name;
-  }
-  return tenant + topic_tenant_delim + topic_name;
-}
-
-void parse_topic_entry(const std::string& topic_entry,
-                       std::string* tenant_name,
-                       std::string* topic_name) {
-  // expected format: [tenant_name:]topic_name*
-  auto pos = topic_entry.find(topic_tenant_delim);
-  if (pos != std::string::npos) {
-    *tenant_name = topic_entry.substr(0, pos);
-    *topic_name = topic_entry.substr(pos + 1);
-  } else {
-    tenant_name->clear();
-    *topic_name = topic_entry;
-  }
-}
-
-std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic) {
-  return bucket_topic_oid_prefix + get_topic_key(topic.name, topic.user.tenant);
-}
-
-class RGWSI_Topic_Module : public RGWSI_MBSObj_Handler_Module {
-  RGWSI_Topic_RADOS::Svc& svc;
-  const std::string prefix;
-
- public:
-  RGWSI_Topic_Module(RGWSI_Topic_RADOS::Svc& _svc)
-      : RGWSI_MBSObj_Handler_Module("topic"),
-        svc(_svc),
-        prefix(topic_oid_prefix) {}
-
-  void get_pool_and_oid(const std::string& key,
-                        rgw_pool* pool,
-                        std::string* oid) override {
-    if (pool) {
-      *pool = svc.zone->get_zone_params().topics_pool;
-    }
-
-    if (oid) {
-      *oid = key_to_oid(key);
-    }
-  }
-
-  bool is_valid_oid(const std::string& oid) override {
-    return boost::algorithm::starts_with(oid, prefix);
-  }
-
-  std::string key_to_oid(const std::string& key) override {
-    return prefix + key;
-  }
-
-  // This is called after `is_valid_oid` and is assumed to be a valid oid
-  std::string oid_to_key(const std::string& oid) override {
-    return oid.substr(prefix.size());
-  }
-
-  const std::string& get_oid_prefix() { return prefix; }
-};
-
-RGWSI_MetaBackend_Handler* RGWSI_Topic_RADOS::get_be_handler() {
-  return be_handler;
-}
-
-void RGWSI_Topic_RADOS::init(RGWSI_Zone* _zone_svc,
-                             RGWSI_Meta* _meta_svc,
-                             RGWSI_MetaBackend* _meta_be_svc,
-                             RGWSI_SysObj* _sysobj_svc) {
-  svc.zone = _zone_svc;
-  svc.meta = _meta_svc;
-  svc.meta_be = _meta_be_svc;
-  svc.sysobj = _sysobj_svc;
-}
-
-int RGWSI_Topic_RADOS::do_start(optional_yield y,
-                                const DoutPrefixProvider* dpp) {
-  int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ,
-                                      &be_handler);
-  if (r < 0) {
-    ldout(ctx(), 0) << "ERROR: failed to create be_handler for Topics: r=" << r
-                    << dendl;
-    return r;
-  }
-
-  auto module = new RGWSI_Topic_Module(svc);
-  RGWSI_MetaBackend_Handler_SObj* bh =
-      static_cast<RGWSI_MetaBackend_Handler_SObj*>(be_handler);
-  be_module.reset(module);
-  bh->set_module(module);
-  return 0;
-}
-
-RGWTopicMetadataHandler::RGWTopicMetadataHandler(rgw::sal::Driver* driver,
-                                                 RGWSI_Topic_RADOS* topic_svc) {
-  this->driver = driver;
-  this->topic_svc = topic_svc;
-  base_init(topic_svc->ctx(), topic_svc->get_be_handler());
-}
-
-RGWMetadataObject* RGWTopicMetadataHandler::get_meta_obj(
-    JSONObj* jo, const obj_version& objv, const ceph::real_time& mtime) {
-  rgw_pubsub_topic topic;
-  try {
-    topic.decode_json(jo);
-  } catch (JSONDecoder::err& e) {
-    return nullptr;
-  }
-
-  return new RGWTopicMetadataObject(topic, objv, mtime, driver);
-}
-
-int RGWTopicMetadataHandler::do_get(RGWSI_MetaBackend_Handler::Op* op,
-                                    std::string& entry, RGWMetadataObject** obj,
-                                    optional_yield y,
-                                    const DoutPrefixProvider* dpp) {
-  rgw_pubsub_topic result;
-  std::string topic_name;
-  std::string tenant;
-  parse_topic_entry(entry, &tenant, &topic_name);
-  RGWObjVersionTracker objv;
-  // TODO: read metadata directly from rados, without calling through
-  // sal::Driver::read_topic_v2()
-  int ret = driver->read_topic_v2(topic_name, tenant, result, &objv, y, dpp);
-  if (ret < 0) {
-    return ret;
-  }
-  ceph::real_time mtime;
-  RGWTopicMetadataObject* rdo =
-      new RGWTopicMetadataObject(result, objv.read_version, mtime, driver);
-  *obj = rdo;
-  return 0;
-}
-
-int RGWTopicMetadataHandler::do_remove(RGWSI_MetaBackend_Handler::Op* op,
-                                       std::string& entry,
-                                       RGWObjVersionTracker& objv_tracker,
-                                       optional_yield y,
-                                       const DoutPrefixProvider* dpp) {
-  auto ret = rgw::notify::remove_persistent_topic(entry, y);
-  if (ret != -ENOENT && ret < 0) {
-    return ret;
-  }
-  std::string topic_name;
-  std::string tenant;
-  parse_topic_entry(entry, &tenant, &topic_name);
-  // TODO: remove metadata directly from rados, without calling through
-  // sal::Driver::remove_topic_v2()
-  return driver->remove_topic_v2(topic_name, tenant, &objv_tracker, y, dpp);
-}
-
-class RGWMetadataHandlerPut_Topic : public RGWMetadataHandlerPut_SObj {
-  RGWTopicMetadataHandler* rhandler;
-  RGWTopicMetadataObject* mdo;
-
- public:
-  RGWMetadataHandlerPut_Topic(RGWTopicMetadataHandler* handler,
-                              RGWSI_MetaBackend_Handler::Op* op,
-                              std::string& entry, RGWMetadataObject* obj,
-                              RGWObjVersionTracker& objv_tracker,
-                              optional_yield y, RGWMDLogSyncType type,
-                              bool from_remote_zone)
-      : RGWMetadataHandlerPut_SObj(handler, op, entry, obj, objv_tracker, y,
-                                   type, from_remote_zone),
-        rhandler(handler) {
-    mdo = static_cast<RGWTopicMetadataObject*>(obj);
-  }
-
-  int put_checked(const DoutPrefixProvider* dpp) override {
-    auto& topic = mdo->get_topic_info();
-    auto* driver = mdo->get_driver();
-    auto ret = rgw::notify::add_persistent_topic(entry, y);
-    if (ret < 0) {
-      return ret;
-    }
-    RGWObjVersionTracker objv_tracker;
-    // TODO: write metadata directly from rados, without calling through
-    // sal::Driver::write_topic_v2()
-    ret = driver->write_topic_v2(topic, &objv_tracker, y, dpp);
-    if (ret < 0) {
-      ldpp_dout(dpp, 1) << "ERROR: failed to write topic info: ret=" << ret
-                        << dendl;
-    }
-    return ret;
-  }
-};
-
-int RGWTopicMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op* op,
-                                    std::string& entry, RGWMetadataObject* obj,
-                                    RGWObjVersionTracker& objv_tracker,
-                                    optional_yield y,
-                                    const DoutPrefixProvider* dpp,
-                                    RGWMDLogSyncType type,
-                                    bool from_remote_zone) {
-  RGWMetadataHandlerPut_Topic put_op(this, op, entry, obj, objv_tracker, y,
-                                     type, from_remote_zone);
-  return do_put_operate(&put_op, dpp);
-}
diff --git a/src/rgw/services/svc_topic_rados.h b/src/rgw/services/svc_topic_rados.h
deleted file mode 100644
index 4d7f37cc02d4..000000000000
--- a/src/rgw/services/svc_topic_rados.h
+++ /dev/null
@@ -1,91 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
-
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2023
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-
-#pragma once
-
-#include "rgw_pubsub.h"
-#include "rgw_service.h"
-#include "svc_meta_be.h"
-
-class RGWSI_Topic_RADOS : public RGWServiceInstance {
- public:
-  struct Svc {
-    RGWSI_Zone* zone{nullptr};
-    RGWSI_Meta* meta{nullptr};
-    RGWSI_MetaBackend* meta_be{nullptr};
-    RGWSI_SysObj* sysobj{nullptr};
-  } svc;
-
-  RGWSI_Topic_RADOS(CephContext* cct) : RGWServiceInstance(cct) {}
-  ~RGWSI_Topic_RADOS() {}
-
-  void init(RGWSI_Zone* _zone_svc,
-            RGWSI_Meta* _meta_svc,
-            RGWSI_MetaBackend* _meta_be_svc,
-            RGWSI_SysObj* _sysobj_svc);
-
-  RGWSI_MetaBackend_Handler* get_be_handler();
-  int do_start(optional_yield y, const DoutPrefixProvider* dpp) override;
-
- private:
-  RGWSI_MetaBackend_Handler* be_handler;
-  std::unique_ptr<RGWSI_MetaBackend::Module> be_module;
-};
-
-class RGWTopicMetadataObject : public RGWMetadataObject {
-  rgw_pubsub_topic topic;
-  rgw::sal::Driver* driver;
-
- public:
-  RGWTopicMetadataObject() = default;
-  RGWTopicMetadataObject(rgw_pubsub_topic& topic, const obj_version& v,
-                         real_time m, rgw::sal::Driver* driver)
-      : RGWMetadataObject(v, m), topic(topic), driver(driver) {}
-
-  void dump(Formatter* f) const override { topic.dump(f); }
-
-  rgw_pubsub_topic& get_topic_info() { return topic; }
-
-  rgw::sal::Driver* get_driver() { return driver; }
-};
-class RGWTopicMetadataHandler : public RGWMetadataHandler_GenericMetaBE {
- public:
-  RGWTopicMetadataHandler(rgw::sal::Driver* driver,
-                          RGWSI_Topic_RADOS* role_svc);
-
-  std::string get_type() final { return "topic"; }
-
-  RGWMetadataObject* get_meta_obj(JSONObj* jo, const obj_version& objv,
-                                  const ceph::real_time& mtime);
-
-  int do_get(RGWSI_MetaBackend_Handler::Op* op, std::string& entry,
-             RGWMetadataObject** obj, optional_yield y,
-             const DoutPrefixProvider* dpp) final;
-
-  int do_remove(RGWSI_MetaBackend_Handler::Op* op, std::string& entry,
-                RGWObjVersionTracker& objv_tracker, optional_yield y,
-                const DoutPrefixProvider* dpp) final;
-
-  int do_put(RGWSI_MetaBackend_Handler::Op* op, std::string& entr,
-             RGWMetadataObject* obj, RGWObjVersionTracker& objv_tracker,
-             optional_yield y, const DoutPrefixProvider* dpp,
-             RGWMDLogSyncType type, bool from_remote_zone) override;
-
- private:
-  rgw::sal::Driver* driver;
-  RGWSI_Topic_RADOS* topic_svc;
-};
-
-std::string get_bucket_topic_mapping_oid(const rgw_pubsub_topic& topic);

From 633a5bef314b5fbc3087271d684f22f9beafb21a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 18 Jan 2024 15:34:16 -0500
Subject: [PATCH 2162/2492] rgw/topic: metadata handler adds/removes persistent
 queues

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/topic.cc | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/rgw/driver/rados/topic.cc b/src/rgw/driver/rados/topic.cc
index 3e409e0b07b3..86ce6bb819b4 100644
--- a/src/rgw/driver/rados/topic.cc
+++ b/src/rgw/driver/rados/topic.cc
@@ -18,6 +18,7 @@
 #include "rgw_common.h"
 #include "rgw_metadata.h"
 #include "rgw_metadata_lister.h"
+#include "rgw_notify.h"
 #include "rgw_pubsub.h"
 #include "rgw_rados.h"
 #include "rgw_string.h"
@@ -315,15 +316,39 @@ class MetadataHandler : public RGWMetadataHandler {
     auto mtime = robj->get_mtime();
 
     constexpr bool exclusive = false;
-    int ret = write(dpp, y, sysobj, &mdlog, zone, info,
-                    objv_tracker, mtime, exclusive);
-    return ret < 0 ? ret : STATUS_APPLIED;
+    int r = write(dpp, y, sysobj, &mdlog, zone, info,
+                  objv_tracker, mtime, exclusive);
+    if (r < 0) {
+      return r;
+    }
+    if (!info.dest.push_endpoint.empty() && info.dest.persistent) {
+      r = rgw::notify::add_persistent_topic(info.name, y);
+      if (r < 0) {
+        ldpp_dout(dpp, 1) << "ERROR: failed to create queue for persistent topic "
+            << info.name << " with: " << cpp_strerror(r) << dendl;
+        return r;
+      }
+    }
+    return STATUS_APPLIED;
   }
 
   int remove(std::string& entry, RGWObjVersionTracker& objv_tracker,
              optional_yield y, const DoutPrefixProvider *dpp) override
   {
-    return topic::remove(dpp, y, sysobj, &mdlog, zone, entry, objv_tracker);
+    int r = topic::remove(dpp, y, sysobj, &mdlog, zone, entry, objv_tracker);
+    if (r < 0) {
+      return r;
+    }
+    // delete persistent topic queue. expect ENOENT for non-persistent topics
+    std::string name;
+    std::string tenant;
+    parse_topic_metadata_key(entry, tenant, name);
+    r = rgw::notify::remove_persistent_topic(name, y);
+    if (r < 0 && r != -ENOENT) {
+      ldpp_dout(dpp, 1) << "Failed to delete queue for persistent topic: "
+                        << name << " with error: " << r << dendl;
+    } // not fatal
+    return 0;
   }
 
   int mutate(const std::string& entry, const ceph::real_time& mtime,

From 9d46c21bc6e522b1d85056e1e545125743aace65 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 12 Feb 2024 16:44:00 -0500
Subject: [PATCH 2163/2492] rgw/pubsub: topic apis forward post body

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/driver/rados/rgw_rest_pubsub.h |  6 ++-
 src/rgw/rgw_rest_pubsub.cc             | 56 ++++++++++++++------------
 src/rgw/rgw_rest_s3.cc                 |  2 +-
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_rest_pubsub.h b/src/rgw/driver/rados/rgw_rest_pubsub.h
index 8b37992b9188..91c39ac1008a 100644
--- a/src/rgw/driver/rados/rgw_rest_pubsub.h
+++ b/src/rgw/driver/rados/rgw_rest_pubsub.h
@@ -25,11 +25,13 @@ class RGWHandler_REST_PSNotifs_S3 : public RGWHandler_REST_S3 {
 // AWS compliant topics handler factory
 class RGWHandler_REST_PSTopic_AWS : public RGWHandler_REST {
   const rgw::auth::StrategyRegistry& auth_registry;
+  bufferlist bl_post_body;
 protected:
   RGWOp* op_post() override;
 public:
-  RGWHandler_REST_PSTopic_AWS(const rgw::auth::StrategyRegistry& _auth_registry) : 
-      auth_registry(_auth_registry) {}
+  RGWHandler_REST_PSTopic_AWS(const rgw::auth::StrategyRegistry& _auth_registry,
+                              bufferlist bl_post_body)
+    :  auth_registry(_auth_registry), bl_post_body(std::move(bl_post_body)) {}
   virtual ~RGWHandler_REST_PSTopic_AWS() = default;
   int postauth_init(optional_yield) override { return 0; }
   int authorize(const DoutPrefixProvider* dpp, optional_yield y) override;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 66574cc82724..1ba234f9d41d 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -126,6 +126,7 @@ int verify_topic_owner_or_policy(req_state* const s,
 // Action=CreateTopic&Name=<topic-name>[&OpaqueData=data][&push-endpoint=<endpoint>[&persistent][&<arg1>=<value1>]]
 class RGWPSCreateTopicOp : public RGWOp {
   private:
+  bufferlist bl_post_body;
   std::string topic_name;
   rgw_pubsub_dest dest;
   std::string topic_arn;
@@ -190,8 +191,11 @@ class RGWPSCreateTopicOp : public RGWOp {
     return 0;
   }
 
-  public:
-   int verify_permission(optional_yield y) override {
+ public:
+  explicit RGWPSCreateTopicOp(bufferlist bl_post_body)
+    : bl_post_body(std::move(bl_post_body)) {}
+
+  int verify_permission(optional_yield y) override {
     auto ret = get_params();
     if (ret < 0) {
       return ret;
@@ -257,10 +261,9 @@ class RGWPSCreateTopicOp : public RGWOp {
 
 void RGWPSCreateTopicOp::execute(optional_yield y) {
   // master request will replicate the topic creation.
-  bufferlist indata;
   if (!driver->is_meta_master()) {
     op_ret = rgw_forward_request_to_master(
-        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+        this, *s->penv.site, s->user->get_id(), &bl_post_body, nullptr, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 1)
           << "CreateTopic forward_request_to_master returned ret = " << op_ret
@@ -537,6 +540,7 @@ void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
 // Action=SetTopicAttributes&TopicArn=<topic-arn>&AttributeName=<attribute-name>&AttributeValue=<attribute-value>
 class RGWPSSetTopicAttributesOp : public RGWOp {
  private:
+  bufferlist bl_post_body;
   std::string topic_name;
   std::string topic_arn;
   std::string opaque_data;
@@ -632,6 +636,9 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
   }
 
  public:
+  explicit RGWPSSetTopicAttributesOp(bufferlist bl_post_body)
+    : bl_post_body(std::move(bl_post_body)) {}
+
   int verify_permission(optional_yield y) override {
     auto ret = get_params();
     if (ret < 0) {
@@ -688,9 +695,8 @@ class RGWPSSetTopicAttributesOp : public RGWOp {
 
 void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
   if (!driver->is_meta_master()) {
-    bufferlist indata;
     op_ret = rgw_forward_request_to_master(
-        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+        this, *s->penv.site, s->user->get_id(), &bl_post_body, nullptr, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 1)
           << "SetTopicAttributes forward_request_to_master returned ret = "
@@ -733,6 +739,7 @@ void RGWPSSetTopicAttributesOp::execute(optional_yield y) {
 // Action=DeleteTopic&TopicArn=<topic-arn>
 class RGWPSDeleteTopicOp : public RGWOp {
   private:
+  bufferlist bl_post_body;
   std::string topic_name;
   
   int get_params() {
@@ -747,7 +754,10 @@ class RGWPSDeleteTopicOp : public RGWOp {
     return 0;
   }
 
-  public:
+ public:
+  explicit RGWPSDeleteTopicOp(bufferlist bl_post_body)
+    : bl_post_body(std::move(bl_post_body)) {}
+
   int verify_permission(optional_yield) override {
     return 0;
   }
@@ -787,9 +797,8 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
     return;
   }
   if (!driver->is_meta_master()) {
-    bufferlist indata;
     op_ret = rgw_forward_request_to_master(
-        this, *s->penv.site, s->user->get_id(), &indata, nullptr, s->info, y);
+        this, *s->penv.site, s->user->get_id(), &bl_post_body, nullptr, s->info, y);
     if (op_ret < 0) {
       ldpp_dout(this, 1)
           << "DeleteTopic forward_request_to_master returned ret = " << op_ret
@@ -836,32 +845,29 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
   ldpp_dout(this, 1) << "successfully removed topic '" << topic_name << "'" << dendl;
 }
 
-using op_generator = RGWOp*(*)();
+using op_generator = RGWOp*(*)(bufferlist);
 static const std::unordered_map<std::string, op_generator> op_generators = {
-    {"CreateTopic", []() -> RGWOp* { return new RGWPSCreateTopicOp; }},
-    {"DeleteTopic", []() -> RGWOp* { return new RGWPSDeleteTopicOp; }},
-    {"ListTopics", []() -> RGWOp* { return new RGWPSListTopicsOp; }},
-    {"GetTopic", []() -> RGWOp* { return new RGWPSGetTopicOp; }},
+    {"CreateTopic", [](bufferlist bl) -> RGWOp* { return new RGWPSCreateTopicOp(std::move(bl)); }},
+    {"DeleteTopic", [](bufferlist bl) -> RGWOp* { return new RGWPSDeleteTopicOp(std::move(bl)); }},
+    {"ListTopics", [](bufferlist bl) -> RGWOp* { return new RGWPSListTopicsOp; }},
+    {"GetTopic", [](bufferlist bl) -> RGWOp* { return new RGWPSGetTopicOp; }},
     {"GetTopicAttributes",
-     []() -> RGWOp* { return new RGWPSGetTopicAttributesOp; }},
+     [](bufferlist bl) -> RGWOp* { return new RGWPSGetTopicAttributesOp; }},
     {"SetTopicAttributes",
-     []() -> RGWOp* { return new RGWPSSetTopicAttributesOp; }}};
+     [](bufferlist bl) -> RGWOp* { return new RGWPSSetTopicAttributesOp(std::move(bl)); }}};
 
-bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s) 
+bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_info& info)
 {
-  if (s->info.args.exists("Action")) {
-    const std::string action_name = s->info.args.get("Action");
-    return op_generators.contains(action_name);
-  }
-  return false;
-}
-bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_info& info) {
   if (info.args.exists("Action")) {
     const std::string action_name = info.args.get("Action");
     return op_generators.contains(action_name);
   }
   return false;
 }
+bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s)
+{
+  return action_exists(s->info);
+}
 
 RGWOp *RGWHandler_REST_PSTopic_AWS::op_post()
 {
@@ -872,7 +878,7 @@ RGWOp *RGWHandler_REST_PSTopic_AWS::op_post()
     const std::string action_name = s->info.args.get("Action");
     const auto action_it = op_generators.find(action_name);
     if (action_it != op_generators.end()) {
-      return action_it->second();
+      return action_it->second(std::move(bl_post_body));
     }
     ldpp_dout(s, 10) << "unknown action '" << action_name << "' for Topic handler" << dendl;
   } else {
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 186fb109c2fc..d27131c83d6e 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5257,7 +5257,7 @@ RGWHandler_REST* RGWRESTMgr_S3::get_handler(rgw::sal::Driver* driver,
         return new RGWHandler_REST_IAM(auth_registry, data);
       }
       if (enable_pubsub && RGWHandler_REST_PSTopic_AWS::action_exists(s)) {
-        return new RGWHandler_REST_PSTopic_AWS(auth_registry); 
+        return new RGWHandler_REST_PSTopic_AWS(auth_registry, std::move(data));
       }
       return nullptr;
     }

From 5b2ae726a95c9922a5c5ba984d6e170dd115128b Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 13 Feb 2024 10:57:08 -0500
Subject: [PATCH 2164/2492] rgw: move rgw_rest_pubsub.h out of driver/rados/

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 src/rgw/{driver/rados => }/rgw_rest_pubsub.h | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/rgw/{driver/rados => }/rgw_rest_pubsub.h (100%)

diff --git a/src/rgw/driver/rados/rgw_rest_pubsub.h b/src/rgw/rgw_rest_pubsub.h
similarity index 100%
rename from src/rgw/driver/rados/rgw_rest_pubsub.h
rename to src/rgw/rgw_rest_pubsub.h

From bcd79d23df63e72839c652cbc20ebb67a024ba85 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Sat, 10 Feb 2024 16:38:30 +0000
Subject: [PATCH 2165/2492] rgw/notifications: handle migration state between
 v1 and v2

test instructions:
https://gist.github.com/yuvalif/21449e301732b719cd1ed97c3eeeabb2

* during migration all topic and notification operations must fail with HTTP error code 503
* read operations should return the values of the v1 topics and notifications
* sending notifications should continue based on v1 values

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/driver/rados/rgw_notify.cc    |  5 +++--
 src/rgw/driver/rados/rgw_sal_rados.cc |  5 +++++
 src/rgw/driver/rados/rgw_sal_rados.h  |  1 +
 src/rgw/rgw_admin.cc                  | 17 +++++++++++++----
 src/rgw/rgw_common.cc                 |  1 +
 src/rgw/rgw_pubsub.cc                 | 26 +++++++++++++++++++++++---
 src/rgw/rgw_rest_pubsub.cc            | 17 ++++++++++++++++-
 src/rgw/rgw_sal.h                     |  2 ++
 src/rgw/rgw_sal_filter.h              |  3 +++
 src/rgw/rgw_sal_store.h               |  1 +
 10 files changed, 68 insertions(+), 10 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index a1290ec9b20a..19d4c0cbb6e3 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -988,7 +988,8 @@ static inline bool notification_match(reservation_t& res,
 		      const RGWObjTags* req_tags)
 {
   rgw_pubsub_bucket_topics bucket_topics;
-  if (all_zonegroups_support(site, zone_features::notification_v2)) {
+  if (all_zonegroups_support(site, zone_features::notification_v2) &&
+      res.store->stat_topics_v1(res.user_tenant, res.yield, res.dpp) == -ENOENT) {
     auto ret = 0;
     if (!res.s) {
       //  for non S3-request caller (e.g., lifecycle, ObjectSync), bucket attrs
@@ -1064,7 +1065,7 @@ static inline bool notification_match(reservation_t& res,
       ldpp_dout(res.dpp, 1)
           << "INFO: failed to load topic: " << topic_cfg.name
           << ". error: " << ret
-          << " while storing the persistent notification event" << dendl;
+          << " while resrving persistent notification event" << dendl;
       if (ret == -ENOENT) {
         // either the topic is deleted but the corresponding notification still
         // exist or in v2 mode the notification could have synced first but
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 0a6a6b7b19f8..193dfb007e80 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -35,6 +35,7 @@
 #include "rgw_acl_s3.h"
 #include "rgw_aio.h"
 #include "rgw_aio_throttle.h"
+#include "rgw_tools.h"
 #include "rgw_tracer.h"
 
 #include "rgw_zone.h"
@@ -1117,6 +1118,10 @@ int RadosStore::read_topics(const std::string& tenant, rgw_pubsub_topics& topics
   return 0;
 }
 
+int RadosStore::stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) {
+  return rgw_stat_system_obj(dpp, svc()->sysobj, svc()->zone->get_zone_params().log_pool, topics_oid(tenant), nullptr, nullptr, y, nullptr);
+}
+
 int RadosStore::write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
 	optional_yield y, const DoutPrefixProvider *dpp) {
   bufferlist bl;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 1eccb89dad33..c97d5e1832d4 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -160,6 +160,7 @@ class RadosStore : public StoreDriver {
     std::string& _req_id, optional_yield y) override;
     int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) override;
+    int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) override;
     int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
 	optional_yield y, const DoutPrefixProvider *dpp) override;
     int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index a5933604b7e1..71f7abda8058 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -10628,7 +10628,8 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) &&
+        driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) {
       ret = get_bucket_notifications(dpp(), bucket.get(), result);
       if (ret < 0) {
         cerr << "ERROR: could not get topics: " << cpp_strerror(-ret)
@@ -10667,7 +10668,8 @@ int main(int argc, const char **argv)
           continue;
         }
         std::set<std::string> subscribed_buckets;
-        if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+        if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) &&
+            driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) {
           ret = driver->get_bucket_topic_mapping(topic, subscribed_buckets,
                                                  null_yield, dpp());
           if (ret < 0) {
@@ -10709,7 +10711,8 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not get topic: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) &&
+        driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) {
       show_topics_info_v2(topic, subscribed_buckets, formatter.get());
     } else {
       encode_json("topic", topic, formatter.get());
@@ -10733,7 +10736,8 @@ int main(int argc, const char **argv)
       return -ret;
     }
     rgw_pubsub_bucket_topics bucket_topics;
-    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+    if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2) &&
+        driver->stat_topics_v1(tenant, null_yield, dpp()) == -ENOENT) {
       ret = get_bucket_notifications(dpp(), bucket.get(), bucket_topics);
       if (ret < 0) {
         cerr << "ERROR: could not get bucket notifications: "
@@ -10799,6 +10803,11 @@ int main(int argc, const char **argv)
     }
 
     if (rgw::all_zonegroups_support(*site, rgw::zone_features::notification_v2)) {
+      if (ret = driver->stat_topics_v1(tenant, null_yield, dpp()); ret != -ENOENT) {
+        cerr << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+          << ". please try again later" << std::endl;
+        return -ret;
+      }
       ret = remove_notification_v2(dpp(), driver, bucket.get(), notification_id,
                                    null_yield);
     } else {
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 3ee98fa18ca1..bfefde9fd729 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -127,6 +127,7 @@ rgw_http_errors rgw_http_s3_errors({
     { ERR_INTERNAL_ERROR, {500, "InternalError" }},
     { ERR_NOT_IMPLEMENTED, {501, "NotImplemented" }},
     { ERR_SERVICE_UNAVAILABLE, {503, "ServiceUnavailable"}},
+    { EBUSY, {503, "ServiceUnavailable"}},
     { ERR_RATE_LIMITED, {503, "SlowDown"}},
     { ERR_ZERO_IN_URL, {400, "InvalidRequest" }},
     { ERR_NO_SUCH_TAG_SET, {404, "NoSuchTagSet"}},
diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index b1b71efdafa5..cf207db632c6 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -539,11 +539,12 @@ int RGWPubSub::get_topics(const DoutPrefixProvider* dpp,
                           rgw_pubsub_topics& result, std::string& next_marker,
                           optional_yield y) const
 {
-  if (!use_notification_v2) {
+  if (!use_notification_v2 || driver->stat_topics_v1(tenant, y, dpp) != -ENOENT) {
+    // in case of v1 or during migration we use v1 topics
     // v1 returns all topics, ignoring marker/max_items
     return read_topics_v1(dpp, result, nullptr, y);
   }
-
+ 
   // TODO: prefix filter on 'tenant:'
   void* handle = NULL;
   int ret = driver->meta_list_keys_init(dpp, "topic", start_marker, &handle);
@@ -623,6 +624,13 @@ int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pub
 					RGWObjVersionTracker *objv_tracker,
 					optional_yield y) const
 {
+  if (ps.use_notification_v2) { 
+    if (const auto ret = ps.driver->stat_topics_v1(bucket->get_tenant(), y, dpp); ret != -ENOENT) {
+      ldpp_dout(dpp, 1) << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+        << ". please try again later" << dendl; 
+      return -ERR_SERVICE_UNAVAILABLE;
+    }
+  }
   const int ret = bucket->write_topics(topics, objv_tracker, y, dpp);
   if (ret < 0) {
     ldpp_dout(dpp, 1) << "ERROR: failed to write bucket topics info: ret=" << ret << dendl;
@@ -637,7 +645,8 @@ int RGWPubSub::get_topic(const DoutPrefixProvider* dpp,
                          rgw_pubsub_topic& result,
                          optional_yield y,
                          std::set<std::string>* subscribed_buckets) const {
-  if (use_notification_v2) {
+  if (use_notification_v2 && driver->stat_topics_v1(tenant, y, dpp) == -ENOENT) {
+    // in case of v1 or during migration we use v1 topics
     int ret = driver->read_topic_v2(name, tenant, result, nullptr, y, dpp);
     if (ret < 0) {
       ldpp_dout(dpp, 1) << "failed to read topic info for name: " << name
@@ -962,6 +971,11 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
                             const std::string& policy_text,
                             optional_yield y) const {
   if (use_notification_v2) {
+    if (const auto ret = driver->stat_topics_v1(tenant, y, dpp); ret != -ENOENT) {
+      ldpp_dout(dpp, 1) << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+        << ". please try again later" << dendl; 
+      return -ERR_SERVICE_UNAVAILABLE;
+    }
     rgw_pubsub_topic new_topic;
     new_topic.user = user;
     new_topic.name = name;
@@ -994,6 +1008,7 @@ int RGWPubSub::create_topic(const DoutPrefixProvider* dpp,
     ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
     return ret;
   }
+  ldpp_dout(dpp, 1) << "INFO: successfully created v1 topic" << dendl;
 
   return 0;
 }
@@ -1025,6 +1040,11 @@ int RGWPubSub::remove_topic_v2(const DoutPrefixProvider* dpp,
 int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const
 {
   if (use_notification_v2) {
+    if (const auto ret = driver->stat_topics_v1(tenant, y, dpp); ret != -ENOENT) {
+      ldpp_dout(dpp, 1) << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+        << ". please try again later" << dendl; 
+      return -ERR_SERVICE_UNAVAILABLE;
+    }
     return remove_topic_v2(dpp, name, y);
   }
   RGWObjVersionTracker objv_tracker;
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 1ba234f9d41d..138150b00275 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -1152,6 +1152,13 @@ void RGWPSCreateNotifOp::execute_v2(optional_yield y) {
     }
   }
 
+  if (const auto ret = driver->stat_topics_v1(s->bucket_tenant, y, this); ret != -ENOENT) {
+    ldpp_dout(this, 1) << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+      << ". please try again later" << dendl; 
+    op_ret = -ERR_SERVICE_UNAVAILABLE;
+    return;
+  }
+
   std::unique_ptr<rgw::sal::Bucket> bucket;
   op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &bucket, y);
@@ -1381,6 +1388,13 @@ void RGWPSDeleteNotifOp::execute_v2(optional_yield y) {
     }
   }
 
+  if (const auto ret = driver->stat_topics_v1(s->bucket_tenant, y, this); ret != -ENOENT) {
+    ldpp_dout(this, 1) << "WARNING: " << (ret == 0 ? "topic migration in process" : "cannot determine topic migration status. ret = " + std::to_string(ret))
+      << ". please try again later" << dendl; 
+    op_ret = -ERR_SERVICE_UNAVAILABLE;
+    return;
+  }
+
   std::unique_ptr<rgw::sal::Bucket> bucket;
   op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
                                &bucket, y);
@@ -1459,7 +1473,8 @@ void RGWPSListNotifsOp::execute(optional_yield y) {
 
   // get all topics on a bucket
   rgw_pubsub_bucket_topics bucket_topics;
-  if (rgw::all_zonegroups_support(*s->penv.site, rgw::zone_features::notification_v2)) {
+  if (rgw::all_zonegroups_support(*s->penv.site, rgw::zone_features::notification_v2) &&
+      driver->stat_topics_v1(s->bucket_tenant, y, this) == -ENOENT) {
     op_ret = get_bucket_notifications(this, bucket.get(), bucket_topics);
   } else {
     const RGWPubSub ps(driver, s->owner.id.tenant);
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 060cfc1e3513..7202d9c90dca 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -308,6 +308,8 @@ class Driver {
     /** Read the topic config entry into @a data and (optionally) @a objv_tracker */
     virtual int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) = 0;
+    /** check if the v1 topics object exists */
+    virtual int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) = 0;
     /** Write @a info and (optionally) @a objv_tracker into the config */
     virtual int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) = 0;
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 83832922f90d..5095f675f162 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -186,6 +186,9 @@ class FilterDriver : public Driver {
       optional_yield y, const DoutPrefixProvider *dpp) override {
     return next->read_topics(tenant, topics, objv_tracker, y, dpp);
   }
+  int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) override {
+    return next->stat_topics_v1(tenant, y, dpp);
+  }
   int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
       optional_yield y, const DoutPrefixProvider *dpp) override {
     return next->write_topics(tenant, topics, objv_tracker, y, dpp);
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index f0ac762554e6..23fc3eb76856 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -30,6 +30,7 @@ class StoreDriver : public Driver {
 
     int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) override {return -EOPNOTSUPP;}
+    int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) override {return -EOPNOTSUPP;}
     int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
 	optional_yield y, const DoutPrefixProvider *dpp) override {return -ENOENT;}
     int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,

From 666e79f1fb78fe8128791e9e23159571f76cfe70 Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 13 Feb 2024 16:36:51 +0000
Subject: [PATCH 2166/2492] rgw/notifications: delete persistent queue only if
 topic is deleted

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/rgw/driver/rados/rgw_notify.cc |  2 +-
 src/rgw/rgw_admin.cc               | 13 ++++++------
 src/rgw/rgw_rest_pubsub.cc         | 32 +++++++++++++++---------------
 3 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 19d4c0cbb6e3..3fe441eec14e 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -1065,7 +1065,7 @@ static inline bool notification_match(reservation_t& res,
       ldpp_dout(res.dpp, 1)
           << "INFO: failed to load topic: " << topic_cfg.name
           << ". error: " << ret
-          << " while resrving persistent notification event" << dendl;
+          << " while reserving persistent notification event" << dendl;
       if (ret == -ENOENT) {
         // either the topic is deleted but the corresponding notification still
         // exist or in v2 mode the notification could have synced first but
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 71f7abda8058..0125451d9a23 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -10771,12 +10771,6 @@ int main(int argc, const char **argv)
       cerr << "ERROR: Run 'topic rm' from master zone " << std::endl;
       return -EINVAL;
     }
-    ret = rgw::notify::remove_persistent_topic(
-        dpp(), static_cast<rgw::sal::RadosStore*>(driver)->getRados()->get_notif_pool_ctx(), topic_name, null_yield);
-    if (ret < 0) {
-      cerr << "ERROR: could not remove persistent topic: " << cpp_strerror(-ret) << std::endl;
-      return -ret;
-    }
 
     RGWPubSub ps(driver, tenant, *site);
 
@@ -10785,6 +10779,13 @@ int main(int argc, const char **argv)
       cerr << "ERROR: could not remove topic: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
+    
+    ret = rgw::notify::remove_persistent_topic(
+        dpp(), static_cast<rgw::sal::RadosStore*>(driver)->getRados()->get_notif_pool_ctx(), topic_name, null_yield);
+    if (ret < 0 && ret != -ENOENT) {
+      cerr << "ERROR: could not remove persistent topic: " << cpp_strerror(-ret) << std::endl;
+      return -ret;
+    }
   }
 
   if (opt_cmd == OPT::PUBSUB_NOTIFICATION_RM) {
diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc
index 138150b00275..585eb68caf9b 100644
--- a/src/rgw/rgw_rest_pubsub.cc
+++ b/src/rgw/rgw_rest_pubsub.cc
@@ -814,35 +814,35 @@ void RGWPSDeleteTopicOp::execute(optional_yield y) {
     op_ret = verify_topic_owner_or_policy(
         s, result, driver->get_zone()->get_zonegroup().get_name(),
         rgw::IAM::snsDeleteTopic);
-    if (op_ret != 0) {
+    if (op_ret < 0) {
       ldpp_dout(this, 1) << "no permission to remove topic '" << topic_name
                          << "'" << dendl;
       return;
     }
-  } else {
+    op_ret = ps.remove_topic(this, topic_name, y);
+    if (op_ret < 0 && op_ret != -ENOENT) {
+      ldpp_dout(this, 1) << "failed to remove topic '" << topic_name << ", ret=" << op_ret << dendl;
+      return;
+    }
+    ldpp_dout(this, 1) << "successfully removed topic '" << topic_name << "'" << dendl;
+  } else if (op_ret != -ENOENT) {
     ldpp_dout(this, 1) << "failed to fetch topic '" << topic_name
                        << "' with error: " << op_ret << dendl;
-    if (op_ret == -ENOENT) {
-      // its not an error if no topics exist, just a no-op
-      op_ret = 0;
-    }
     return;
   }
+  if (op_ret == -ENOENT) {
+    // its not an error if no topics exist, just a no-op
+    op_ret = 0;
+  }
   // upon deletion it is not known if topic is persistent or not
   // will try to delete the persistent topic anyway
-  op_ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
-  if (op_ret != -ENOENT && op_ret < 0) {
+  // doing this regardless of the topic being previously deleted
+  // to allow for cleanup if only the queue deletion failed
+  if (const auto ret = rgw::notify::remove_persistent_topic(topic_name, s->yield); ret < 0 && ret != -ENOENT) {
     ldpp_dout(this, 1) << "DeleteTopic Action failed to remove queue for "
                           "persistent topics. error:"
-                       << op_ret << dendl;
-    return;
-  }
-  op_ret = ps.remove_topic(this, topic_name, y);
-  if (op_ret < 0) {
-    ldpp_dout(this, 1) << "failed to remove topic '" << topic_name << ", ret=" << op_ret << dendl;
-    return;
+                       << ret << dendl;
   }
-  ldpp_dout(this, 1) << "successfully removed topic '" << topic_name << "'" << dendl;
 }
 
 using op_generator = RGWOp*(*)(bufferlist);

From 181a671f9d4b86cabb444d8b8455f301fddc5194 Mon Sep 17 00:00:00 2001
From: Yuri Weinstein <yweinste@redhat.com>
Date: Tue, 5 Mar 2024 08:47:45 -0800
Subject: [PATCH 2167/2492] doc: reef 18.2.2 release notes

Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
---
 doc/releases/index.rst    |  1 +
 doc/releases/reef.rst     | 18 ++++++++++++++++++
 doc/releases/releases.yml |  2 ++
 3 files changed, 21 insertions(+)

diff --git a/doc/releases/index.rst b/doc/releases/index.rst
index 6b0d16f384f9..f44150d9dc04 100644
--- a/doc/releases/index.rst
+++ b/doc/releases/index.rst
@@ -65,6 +65,7 @@ Release timeline
 .. _Reef: reef
 .. _18.2.0: reef#v18-2-0-reef
 .. _18.2.1: reef#v18-2-1-reef
+.. _18.2.2: reef#v18-2-2-reef
 
 .. _Quincy: quincy
 .. _17.2.0: quincy#v17-2-0-quincy
diff --git a/doc/releases/reef.rst b/doc/releases/reef.rst
index 4eff836fd3eb..5149cd20ad06 100644
--- a/doc/releases/reef.rst
+++ b/doc/releases/reef.rst
@@ -5,6 +5,24 @@ Reef
 Reef is the 18th stable release of Ceph. It is named after the reef squid
 (Sepioteuthis).
 
+
+v18.2.2 Reef
+============
+
+This is a hotfix release that resolves several flaws including Prometheus crashes and an encoder fix.
+
+Notable Changes
+---------------
+
+* mgr/Prometheus: refine the orchestrator availability check to prevent against crashes in the prometheus module during startup. Introduce additional checks to handle daemon_ids generated within the Rook environment, thus preventing potential issues during RGW metrics metadata generation.
+
+Changelog
+---------
+
+* mgr/prometheus: fix orch check to prevent Prometheus crash (`pr#55491 <https://github.com/ceph/ceph/pull/55491>`_, Redouane Kachach)
+* debian/\*.postinst: add adduser as a dependency and specify --home when adduser (`pr#55709 <https://github.com/ceph/ceph/pull/55709>`_, Kefu Chai)
+* src/osd/OSDMap.cc: Fix encoder to produce same bytestream (`pr#55712 <https://github.com/ceph/ceph/pull/55712>`_, Kamoltat)
+
 v18.2.1 Reef
 ============
 
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
index 2793ae351006..2c6d36a5df4d 100644
--- a/doc/releases/releases.yml
+++ b/doc/releases/releases.yml
@@ -19,6 +19,8 @@ releases:
         released: 2023-08-07
       - version: 18.2.1
         released: 2023-12-18
+      - version: 18.2.2
+        released: 2024-03-11
   quincy:
     target_eol: 2024-06-01
     releases:

From 728e564f6172d45310646b690c7c259a32520b49 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Wed, 6 Mar 2024 00:46:37 +0000
Subject: [PATCH 2168/2492] crimson/os/seastore: allow to remap the dirty
 extent

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/transaction_manager.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 640b98f79426..28524592eaef 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -459,8 +459,6 @@ class TransactionManager : public ExtentCallbackInterface {
 	(intermediate_base == L_ADDR_NULL)
 	  == (intermediate_key == L_ADDR_NULL));
       if (ext) {
-        // FIXME: cannot and will not remap a dirty extent for now.
-        ceph_assert(!ext->is_dirty());
         ceph_assert(!ext->is_mutable());
         ceph_assert(ext->get_length() >= original_len);
 	ceph_assert(ext->get_paddr() == original_paddr);

From e87a9f15af52925d8988805b65a9b12c2a88b164 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 5 Mar 2024 16:09:25 +0530
Subject: [PATCH 2169/2492] mgr/dashboard: fix nvmeof api documentation

From Aviv:
POST /api/nvmeof/hosts - the description of the command it wrong IMO. It is not about creating a host. It is about allowing a host X to access subsystem Y.
GET /api/nvmeof/hosts/{subsystem_nqn} - also the description is not accurate. The command lists all hosts that are allowed to access this subsystem.
DELETE /api/nvmeof/hosts/{subsystem_nqn}/{host_nqn} - again the description should be changed as above.
POST /api/nvmeof/namespace - bad formatting of the description
GET /api/nvmeof/subsystem - the description is wrong, should say - "List all NVMeoF subsystems". And it shouldn't get any param.
POST /api/nvmeof/subsystem - few issues here. The serial_number, and max_namespaces are optional (we need to mention that). Also it is missing the --enable-ha argument that is also optional.
Some commands are missing: log_level, connection.i

Fixes: https://tracker.ceph.com/issues/64714
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/controllers/nvmeof.py       |  127 +-
 src/pybind/mgr/dashboard/openapi.yaml         |   93 +-
 .../mgr/dashboard/services/nvmeof_client.py   |   12 +-
 .../services/proto/gateway_pb2_grpc.py        | 1017 +++++++++--------
 4 files changed, 620 insertions(+), 629 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
index 6184f6a4166d..b485a5aa17b2 100644
--- a/src/pybind/mgr/dashboard/controllers/nvmeof.py
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -17,38 +17,39 @@
     @APIDoc('NVMe-oF Namespace Management API', 'NVMe-oF')
     class NvmeofNamespace(RESTController):
         @ReadPermission
+        @EndpointDoc('List all NVMeoF namespaces',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
+                     })
         def list(self, subsystem_nqn: str):
-            """
-            List all NVMeoF namespaces
-            """
             response = MessageToJson(NVMeoFClient().list_namespaces(subsystem_nqn))
             return json.loads(response)
 
         @CreatePermission
+        @EndpointDoc('Create a new NVMeoF namespace',
+                     parameters={
+                         'rbd_pool': (str, 'RBD pool name'),
+                         'rbd_image': (str, 'RBD image name'),
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'create_image': (bool, 'Create RBD image'),
+                         'image_size': (int, 'RBD image size'),
+                         'block_size': (int, 'NVMeoF namespace block size')
+                     })
         def create(self, rbd_pool: str, rbd_image: str, subsystem_nqn: str,
                    create_image: Optional[bool] = True, image_size: Optional[int] = 1024,
                    block_size: int = 512):
-            """
-            Create a new NVMeoF namespace
-            :param rbd_pool: RBD pool name
-            :param rbd_image: RBD image name
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            :param create_image: Create RBD image
-            :param image_size: RBD image size
-            :param block_size: NVMeoF namespace block size
-            """
             response = NVMeoFClient().create_namespace(rbd_pool, rbd_image,
                                                        subsystem_nqn, block_size,
                                                        create_image, image_size)
             return json.loads(MessageToJson(response))
 
         @Endpoint('DELETE', path='{subsystem_nqn}')
+        @EndpointDoc('Delete an existing NVMeoF namespace',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
+                     })
         @DeletePermission
         def delete(self, subsystem_nqn: str):
-            """
-            Delete an existing NVMeoF namespace
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            """
             response = NVMeoFClient().delete_namespace(subsystem_nqn)
             return json.loads(MessageToJson(response))
 
@@ -56,7 +57,7 @@ def delete(self, subsystem_nqn: str):
     @APIDoc('NVMe-oF Subsystem Management API', 'NVMe-oF')
     class NvmeofSubsystem(RESTController):
         @ReadPermission
-        @EndpointDoc("List all NVMeoF gateways",
+        @EndpointDoc("List all NVMeoF subsystems",
                      parameters={
                          'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
                      })
@@ -68,26 +69,24 @@ def list(self, subsystem_nqn: Optional[str] = None):
             return json.loads(response)
 
         @CreatePermission
-        def create(self, subsystem_nqn: str, serial_number: Optional[str] = None,
-                   max_namespaces: Optional[int] = 256):
-            """
-            Create a new NVMeoF subsystem
-
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            :param serial_number: NVMeoF subsystem serial number
-            :param max_namespaces: NVMeoF subsystem maximum namespaces
-            """
-            response = NVMeoFClient().create_subsystem(subsystem_nqn, serial_number, max_namespaces)
+        @EndpointDoc('Create a new NVMeoF subsystem',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'serial_number': (str, 'NVMeoF subsystem serial number'),
+                         'max_namespaces': (int, 'Maximum number of namespaces')
+                     })
+        def create(self, subsystem_nqn: str):
+            response = NVMeoFClient().create_subsystem(subsystem_nqn)
             return json.loads(MessageToJson(response))
 
         @DeletePermission
         @Endpoint('DELETE', path='{subsystem_nqn}')
+        @EndpointDoc('Delete an existing NVMeoF subsystem',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'force': (bool, 'Force delete')
+                     })
         def delete(self, subsystem_nqn: str, force: Optional[bool] = False):
-            """
-            Delete an existing NVMeoF subsystem
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            :param force: Force delete
-            """
             response = NVMeoFClient().delete_subsystem(subsystem_nqn, force)
             return json.loads(MessageToJson(response))
 
@@ -95,31 +94,31 @@ def delete(self, subsystem_nqn: str, force: Optional[bool] = False):
     @APIDoc('NVMe-oF Host Management API', 'NVMe-oF')
     class NvmeofHost(RESTController):
         @ReadPermission
+        @EndpointDoc('List all allowed hosts for an NVMeoF subsystem',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
+                     })
         def list(self, subsystem_nqn: str):
-            """
-            List all NVMeoF hosts
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            """
             response = MessageToJson(NVMeoFClient().list_hosts(subsystem_nqn))
             return json.loads(response)
 
         @CreatePermission
+        @EndpointDoc('Allow hosts to access an NVMeoF subsystem',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'host_nqn': (str, 'NVMeoF host NQN')
+                     })
         def create(self, subsystem_nqn: str, host_nqn: str):
-            """
-            Create a new NVMeoF host
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            :param host_nqn: NVMeoF host NQN
-            """
             response = NVMeoFClient().add_host(subsystem_nqn, host_nqn)
             return json.loads(MessageToJson(response))
 
         @DeletePermission
+        @EndpointDoc('Disallow hosts from accessing an NVMeoF subsystem',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
+                         'host_nqn': (str, 'NVMeoF host NQN')
+                     })
         def delete(self, subsystem_nqn: str, host_nqn: str):
-            """
-            Delete an existing NVMeoF host
-            :param subsystem_nqn: NVMeoF subsystem NQN
-            :param host_nqn: NVMeoF host NQN
-            """
             response = NVMeoFClient().remove_host(subsystem_nqn, host_nqn)
             return json.loads(MessageToJson(response))
 
@@ -127,33 +126,33 @@ def delete(self, subsystem_nqn: str, host_nqn: str):
     @APIDoc('NVMe-oF Listener Management API', 'NVMe-oF')
     class NvmeofListener(RESTController):
         @ReadPermission
+        @EndpointDoc('List all NVMeoF listeners',
+                     parameters={
+                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
+                     })
         def list(self, subsystem_nqn: str):
-            """
-            List all NVMeoF listeners
-            :param nqn: NVMeoF subsystem NQN
-            """
             response = MessageToJson(NVMeoFClient().list_listeners(subsystem_nqn))
             return json.loads(response)
 
         @CreatePermission
+        @EndpointDoc('Create a new NVMeoF listener',
+                     parameters={
+                         'nqn': (str, 'NVMeoF subsystem NQN'),
+                         'gateway': (str, 'NVMeoF gateway'),
+                         'traddr': (str, 'NVMeoF transport address')
+                     })
         def create(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            """
-            Create a new NVMeoF listener
-            :param nqn: NVMeoF subsystem NQN
-            :param gateway: NVMeoF gateway
-            :param traddr: NVMeoF transport address
-            """
             response = NVMeoFClient().create_listener(nqn, gateway, traddr)
             return json.loads(MessageToJson(response))
 
         @DeletePermission
+        @EndpointDoc('Delete an existing NVMeoF listener',
+                     parameters={
+                         'nqn': (str, 'NVMeoF subsystem NQN'),
+                         'gateway': (str, 'NVMeoF gateway'),
+                         'traddr': (str, 'NVMeoF transport address')
+                     })
         def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            """
-            Delete an existing NVMeoF listener
-            :param nqn: NVMeoF subsystem NQN
-            :param gateway: NVMeoF gateway
-            :param traddr: NVMeoF transport address
-            """
             response = NVMeoFClient().delete_listener(nqn, gateway, traddr)
             return json.loads(MessageToJson(response))
 
@@ -162,9 +161,7 @@ def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None):
     class NvmeofGateway(RESTController):
         @ReadPermission
         @Endpoint()
+        @EndpointDoc('List all NVMeoF gateways')
         def info(self):
-            """
-            Get NVMeoF gateway information
-            """
             response = MessageToJson(NVMeoFClient().gateway_info())
             return json.loads(response)
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index ec3ae76cb654..3429a117b99a 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -7641,7 +7641,6 @@ paths:
       - NFS-Ganesha
   /api/nvmeof/gateway/info:
     get:
-      description: "\n            Get NVMeoF gateway information\n            "
       parameters: []
       responses:
         '200':
@@ -7660,13 +7659,11 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: List all NVMeoF gateways
       tags:
       - NVMe-oF
   /api/nvmeof/hosts:
     post:
-      description: "\n            Create a new NVMeoF host\n            :param subsystem_nqn:\
-        \ NVMeoF subsystem NQN\n            :param host_nqn: NVMeoF host NQN\n   \
-        \         "
       parameters: []
       requestBody:
         content:
@@ -7674,8 +7671,10 @@ paths:
             schema:
               properties:
                 host_nqn:
+                  description: NVMeoF host NQN
                   type: string
                 subsystem_nqn:
+                  description: NVMeoF subsystem NQN
                   type: string
               required:
               - subsystem_nqn
@@ -7703,14 +7702,14 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Allow hosts to access an NVMeoF subsystem
       tags:
       - NVMe-oF
   /api/nvmeof/hosts/{subsystem_nqn}:
     get:
-      description: "\n            List all NVMeoF hosts\n            :param subsystem_nqn:\
-        \ NVMeoF subsystem NQN\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: subsystem_nqn
         required: true
         schema:
@@ -7732,20 +7731,20 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: List all allowed hosts for an NVMeoF subsystem
       tags:
       - NVMe-oF
   /api/nvmeof/hosts/{subsystem_nqn}/{host_nqn}:
     delete:
-      description: "\n            Delete an existing NVMeoF host\n            :param\
-        \ subsystem_nqn: NVMeoF subsystem NQN\n            :param host_nqn: NVMeoF\
-        \ host NQN\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: subsystem_nqn
         required: true
         schema:
           type: string
-      - in: path
+      - description: NVMeoF host NQN
+        in: path
         name: host_nqn
         required: true
         schema:
@@ -7772,14 +7771,14 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Disallow hosts from accessing an NVMeoF subsystem
       tags:
       - NVMe-oF
   /api/nvmeof/listener:
     get:
-      description: "\n            List all NVMeoF listeners\n            :param nqn:\
-        \ NVMeoF subsystem NQN\n            "
       parameters:
-      - in: query
+      - description: NVMeoF subsystem NQN
+        in: query
         name: subsystem_nqn
         required: true
         schema:
@@ -7801,12 +7800,10 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: List all NVMeoF listeners
       tags:
       - NVMe-oF
     post:
-      description: "\n            Create a new NVMeoF listener\n            :param\
-        \ nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
-        \            :param traddr: NVMeoF transport address\n            "
       parameters: []
       requestBody:
         content:
@@ -7814,10 +7811,13 @@ paths:
             schema:
               properties:
                 gateway:
+                  description: NVMeoF gateway
                   type: string
                 nqn:
+                  description: NVMeoF subsystem NQN
                   type: string
                 traddr:
+                  description: NVMeoF transport address
                   type: string
               required:
               - nqn
@@ -7845,25 +7845,26 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Create a new NVMeoF listener
       tags:
       - NVMe-oF
   /api/nvmeof/listener/{nqn}/{gateway}:
     delete:
-      description: "\n            Delete an existing NVMeoF listener\n           \
-        \ :param nqn: NVMeoF subsystem NQN\n            :param gateway: NVMeoF gateway\n\
-        \            :param traddr: NVMeoF transport address\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: nqn
         required: true
         schema:
           type: string
-      - in: path
+      - description: NVMeoF gateway
+        in: path
         name: gateway
         required: true
         schema:
           type: string
       - allowEmptyValue: true
+        description: NVMeoF transport address
         in: query
         name: traddr
         schema:
@@ -7890,16 +7891,11 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Delete an existing NVMeoF listener
       tags:
       - NVMe-oF
   /api/nvmeof/namespace:
     post:
-      description: "\n            Create a new NVMeoF namespace\n            :param\
-        \ rbd_pool: RBD pool name\n            :param rbd_image: RBD image name\n\
-        \            :param subsystem_nqn: NVMeoF subsystem NQN\n            :param\
-        \ create_image: Create RBD image\n            :param image_size: RBD image\
-        \ size\n            :param block_size: NVMeoF namespace block size\n     \
-        \       "
       parameters: []
       requestBody:
         content:
@@ -7908,18 +7904,24 @@ paths:
               properties:
                 block_size:
                   default: 512
+                  description: NVMeoF namespace block size
                   type: integer
                 create_image:
                   default: true
+                  description: Create RBD image
                   type: boolean
                 image_size:
                   default: 1024
+                  description: RBD image size
                   type: integer
                 rbd_image:
+                  description: RBD image name
                   type: string
                 rbd_pool:
+                  description: RBD pool name
                   type: string
                 subsystem_nqn:
+                  description: NVMeoF subsystem NQN
                   type: string
               required:
               - rbd_pool
@@ -7948,14 +7950,14 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Create a new NVMeoF namespace
       tags:
       - NVMe-oF
   /api/nvmeof/namespace/{subsystem_nqn}:
     delete:
-      description: "\n            Delete an existing NVMeoF namespace\n          \
-        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: subsystem_nqn
         required: true
         schema:
@@ -7982,12 +7984,13 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Delete an existing NVMeoF namespace
       tags:
       - NVMe-oF
     get:
-      description: "\n            List all NVMeoF namespaces\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: subsystem_nqn
         required: true
         schema:
@@ -8009,6 +8012,7 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: List all NVMeoF namespaces
       tags:
       - NVMe-oF
   /api/nvmeof/subsystem:
@@ -8037,26 +8041,18 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: List all NVMeoF gateways
+      summary: List all NVMeoF subsystems
       tags:
       - NVMe-oF
     post:
-      description: "\n            Create a new NVMeoF subsystem\n\n            :param\
-        \ subsystem_nqn: NVMeoF subsystem NQN\n            :param serial_number: NVMeoF\
-        \ subsystem serial number\n            :param max_namespaces: NVMeoF subsystem\
-        \ maximum namespaces\n            "
       parameters: []
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                max_namespaces:
-                  default: 256
-                  type: integer
-                serial_number:
-                  type: integer
                 subsystem_nqn:
+                  description: NVMeoF subsystem NQN
                   type: string
               required:
               - subsystem_nqn
@@ -8083,20 +8079,20 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Create a new NVMeoF subsystem
       tags:
       - NVMe-oF
   /api/nvmeof/subsystem/{subsystem_nqn}:
     delete:
-      description: "\n            Delete an existing NVMeoF subsystem\n          \
-        \  :param subsystem_nqn: NVMeoF subsystem NQN\n            :param force: Force\
-        \ delete\n            "
       parameters:
-      - in: path
+      - description: NVMeoF subsystem NQN
+        in: path
         name: subsystem_nqn
         required: true
         schema:
           type: string
       - default: false
+        description: Force delete
         in: query
         name: force
         schema:
@@ -8123,6 +8119,7 @@ paths:
             trace.
       security:
       - jwt: []
+      summary: Delete an existing NVMeoF subsystem
       tags:
       - NVMe-oF
   /api/osd:
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index 131bfff2aeb9..c51d7e80edb7 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -25,18 +25,14 @@ def __init__(self):
             logger.info('Found nvmeof gateway at %s', self.gateway_addr)
             self.stub = pb2_grpc.GatewayStub(self.channel)
 
-        def list_subsystems(self, subsystem_nqn: Optional[str] = None,
-                            serial_number: Optional[str] = None):
+        def list_subsystems(self, subsystem_nqn: Optional[str] = None):
             return self.stub.list_subsystems(pb2.list_subsystems_req(
-                subsystem_nqn=subsystem_nqn,
-                serial_number=serial_number
+                subsystem_nqn=subsystem_nqn
             ))
 
-        def create_subsystem(self, subsystem_nqn: str, serial_number: str, max_namespaces: int):
+        def create_subsystem(self, subsystem_nqn: str):
             return self.stub.create_subsystem(pb2.create_subsystem_req(
-                subsystem_nqn=subsystem_nqn,
-                serial_number=serial_number,
-                max_namespaces=int(max_namespaces)
+                subsystem_nqn=subsystem_nqn
             ))
 
         def delete_subsystem(self, subsystem_nqn: str, force: Optional[bool] = False):
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
index cc97bb26c53b..599142a16668 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2_grpc.py
@@ -15,120 +15,120 @@ def __init__(self, channel):
             channel: A grpc.Channel.
         """
         self.namespace_add = channel.unary_unary(
-                '/Gateway/namespace_add',
-                request_serializer=gateway__pb2.namespace_add_req.SerializeToString,
-                response_deserializer=gateway__pb2.nsid_status.FromString,
-                )
+            '/Gateway/namespace_add',
+            request_serializer=gateway__pb2.namespace_add_req.SerializeToString,
+            response_deserializer=gateway__pb2.nsid_status.FromString,
+        )
         self.create_subsystem = channel.unary_unary(
-                '/Gateway/create_subsystem',
-                request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/create_subsystem',
+            request_serializer=gateway__pb2.create_subsystem_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.delete_subsystem = channel.unary_unary(
-                '/Gateway/delete_subsystem',
-                request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/delete_subsystem',
+            request_serializer=gateway__pb2.delete_subsystem_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.list_namespaces = channel.unary_unary(
-                '/Gateway/list_namespaces',
-                request_serializer=gateway__pb2.list_namespaces_req.SerializeToString,
-                response_deserializer=gateway__pb2.namespaces_info.FromString,
-                )
+            '/Gateway/list_namespaces',
+            request_serializer=gateway__pb2.list_namespaces_req.SerializeToString,
+            response_deserializer=gateway__pb2.namespaces_info.FromString,
+        )
         self.namespace_resize = channel.unary_unary(
-                '/Gateway/namespace_resize',
-                request_serializer=gateway__pb2.namespace_resize_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/namespace_resize',
+            request_serializer=gateway__pb2.namespace_resize_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.namespace_get_io_stats = channel.unary_unary(
-                '/Gateway/namespace_get_io_stats',
-                request_serializer=gateway__pb2.namespace_get_io_stats_req.SerializeToString,
-                response_deserializer=gateway__pb2.namespace_io_stats_info.FromString,
-                )
+            '/Gateway/namespace_get_io_stats',
+            request_serializer=gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+            response_deserializer=gateway__pb2.namespace_io_stats_info.FromString,
+        )
         self.namespace_set_qos_limits = channel.unary_unary(
-                '/Gateway/namespace_set_qos_limits',
-                request_serializer=gateway__pb2.namespace_set_qos_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/namespace_set_qos_limits',
+            request_serializer=gateway__pb2.namespace_set_qos_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.namespace_change_load_balancing_group = channel.unary_unary(
-                '/Gateway/namespace_change_load_balancing_group',
-                request_serializer=gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/namespace_change_load_balancing_group',
+            request_serializer=gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.namespace_delete = channel.unary_unary(
-                '/Gateway/namespace_delete',
-                request_serializer=gateway__pb2.namespace_delete_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/namespace_delete',
+            request_serializer=gateway__pb2.namespace_delete_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.add_host = channel.unary_unary(
-                '/Gateway/add_host',
-                request_serializer=gateway__pb2.add_host_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/add_host',
+            request_serializer=gateway__pb2.add_host_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.remove_host = channel.unary_unary(
-                '/Gateway/remove_host',
-                request_serializer=gateway__pb2.remove_host_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/remove_host',
+            request_serializer=gateway__pb2.remove_host_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.list_hosts = channel.unary_unary(
-                '/Gateway/list_hosts',
-                request_serializer=gateway__pb2.list_hosts_req.SerializeToString,
-                response_deserializer=gateway__pb2.hosts_info.FromString,
-                )
+            '/Gateway/list_hosts',
+            request_serializer=gateway__pb2.list_hosts_req.SerializeToString,
+            response_deserializer=gateway__pb2.hosts_info.FromString,
+        )
         self.list_connections = channel.unary_unary(
-                '/Gateway/list_connections',
-                request_serializer=gateway__pb2.list_connections_req.SerializeToString,
-                response_deserializer=gateway__pb2.connections_info.FromString,
-                )
+            '/Gateway/list_connections',
+            request_serializer=gateway__pb2.list_connections_req.SerializeToString,
+            response_deserializer=gateway__pb2.connections_info.FromString,
+        )
         self.create_listener = channel.unary_unary(
-                '/Gateway/create_listener',
-                request_serializer=gateway__pb2.create_listener_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/create_listener',
+            request_serializer=gateway__pb2.create_listener_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.delete_listener = channel.unary_unary(
-                '/Gateway/delete_listener',
-                request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/delete_listener',
+            request_serializer=gateway__pb2.delete_listener_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.list_listeners = channel.unary_unary(
-                '/Gateway/list_listeners',
-                request_serializer=gateway__pb2.list_listeners_req.SerializeToString,
-                response_deserializer=gateway__pb2.listeners_info.FromString,
-                )
+            '/Gateway/list_listeners',
+            request_serializer=gateway__pb2.list_listeners_req.SerializeToString,
+            response_deserializer=gateway__pb2.listeners_info.FromString,
+        )
         self.list_subsystems = channel.unary_unary(
-                '/Gateway/list_subsystems',
-                request_serializer=gateway__pb2.list_subsystems_req.SerializeToString,
-                response_deserializer=gateway__pb2.subsystems_info_cli.FromString,
-                )
+            '/Gateway/list_subsystems',
+            request_serializer=gateway__pb2.list_subsystems_req.SerializeToString,
+            response_deserializer=gateway__pb2.subsystems_info_cli.FromString,
+        )
         self.get_subsystems = channel.unary_unary(
-                '/Gateway/get_subsystems',
-                request_serializer=gateway__pb2.get_subsystems_req.SerializeToString,
-                response_deserializer=gateway__pb2.subsystems_info.FromString,
-                )
+            '/Gateway/get_subsystems',
+            request_serializer=gateway__pb2.get_subsystems_req.SerializeToString,
+            response_deserializer=gateway__pb2.subsystems_info.FromString,
+        )
         self.set_ana_state = channel.unary_unary(
-                '/Gateway/set_ana_state',
-                request_serializer=gateway__pb2.ana_info.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/set_ana_state',
+            request_serializer=gateway__pb2.ana_info.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.get_spdk_nvmf_log_flags_and_level = channel.unary_unary(
-                '/Gateway/get_spdk_nvmf_log_flags_and_level',
-                request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-                response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-                )
+            '/Gateway/get_spdk_nvmf_log_flags_and_level',
+            request_serializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+            response_deserializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+        )
         self.disable_spdk_nvmf_logs = channel.unary_unary(
-                '/Gateway/disable_spdk_nvmf_logs',
-                request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/disable_spdk_nvmf_logs',
+            request_serializer=gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.set_spdk_nvmf_logs = channel.unary_unary(
-                '/Gateway/set_spdk_nvmf_logs',
-                request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-                response_deserializer=gateway__pb2.req_status.FromString,
-                )
+            '/Gateway/set_spdk_nvmf_logs',
+            request_serializer=gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+            response_deserializer=gateway__pb2.req_status.FromString,
+        )
         self.get_gateway_info = channel.unary_unary(
-                '/Gateway/get_gateway_info',
-                request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
-                response_deserializer=gateway__pb2.gateway_info.FromString,
-                )
+            '/Gateway/get_gateway_info',
+            request_serializer=gateway__pb2.get_gateway_info_req.SerializeToString,
+            response_deserializer=gateway__pb2.gateway_info.FromString,
+        )
 
 
 class GatewayServicer(object):
@@ -298,518 +298,519 @@ def get_gateway_info(self, request, context):
 
 def add_GatewayServicer_to_server(servicer, server):
     rpc_method_handlers = {
-            'namespace_add': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_add,
-                    request_deserializer=gateway__pb2.namespace_add_req.FromString,
-                    response_serializer=gateway__pb2.nsid_status.SerializeToString,
-            ),
-            'create_subsystem': grpc.unary_unary_rpc_method_handler(
-                    servicer.create_subsystem,
-                    request_deserializer=gateway__pb2.create_subsystem_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'delete_subsystem': grpc.unary_unary_rpc_method_handler(
-                    servicer.delete_subsystem,
-                    request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'list_namespaces': grpc.unary_unary_rpc_method_handler(
-                    servicer.list_namespaces,
-                    request_deserializer=gateway__pb2.list_namespaces_req.FromString,
-                    response_serializer=gateway__pb2.namespaces_info.SerializeToString,
-            ),
-            'namespace_resize': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_resize,
-                    request_deserializer=gateway__pb2.namespace_resize_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'namespace_get_io_stats': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_get_io_stats,
-                    request_deserializer=gateway__pb2.namespace_get_io_stats_req.FromString,
-                    response_serializer=gateway__pb2.namespace_io_stats_info.SerializeToString,
-            ),
-            'namespace_set_qos_limits': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_set_qos_limits,
-                    request_deserializer=gateway__pb2.namespace_set_qos_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'namespace_change_load_balancing_group': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_change_load_balancing_group,
-                    request_deserializer=gateway__pb2.namespace_change_load_balancing_group_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'namespace_delete': grpc.unary_unary_rpc_method_handler(
-                    servicer.namespace_delete,
-                    request_deserializer=gateway__pb2.namespace_delete_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'add_host': grpc.unary_unary_rpc_method_handler(
-                    servicer.add_host,
-                    request_deserializer=gateway__pb2.add_host_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'remove_host': grpc.unary_unary_rpc_method_handler(
-                    servicer.remove_host,
-                    request_deserializer=gateway__pb2.remove_host_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'list_hosts': grpc.unary_unary_rpc_method_handler(
-                    servicer.list_hosts,
-                    request_deserializer=gateway__pb2.list_hosts_req.FromString,
-                    response_serializer=gateway__pb2.hosts_info.SerializeToString,
-            ),
-            'list_connections': grpc.unary_unary_rpc_method_handler(
-                    servicer.list_connections,
-                    request_deserializer=gateway__pb2.list_connections_req.FromString,
-                    response_serializer=gateway__pb2.connections_info.SerializeToString,
-            ),
-            'create_listener': grpc.unary_unary_rpc_method_handler(
-                    servicer.create_listener,
-                    request_deserializer=gateway__pb2.create_listener_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'delete_listener': grpc.unary_unary_rpc_method_handler(
-                    servicer.delete_listener,
-                    request_deserializer=gateway__pb2.delete_listener_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'list_listeners': grpc.unary_unary_rpc_method_handler(
-                    servicer.list_listeners,
-                    request_deserializer=gateway__pb2.list_listeners_req.FromString,
-                    response_serializer=gateway__pb2.listeners_info.SerializeToString,
-            ),
-            'list_subsystems': grpc.unary_unary_rpc_method_handler(
-                    servicer.list_subsystems,
-                    request_deserializer=gateway__pb2.list_subsystems_req.FromString,
-                    response_serializer=gateway__pb2.subsystems_info_cli.SerializeToString,
-            ),
-            'get_subsystems': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_subsystems,
-                    request_deserializer=gateway__pb2.get_subsystems_req.FromString,
-                    response_serializer=gateway__pb2.subsystems_info.SerializeToString,
-            ),
-            'set_ana_state': grpc.unary_unary_rpc_method_handler(
-                    servicer.set_ana_state,
-                    request_deserializer=gateway__pb2.ana_info.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_spdk_nvmf_log_flags_and_level,
-                    request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
-                    response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
-            ),
-            'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-                    servicer.disable_spdk_nvmf_logs,
-                    request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
-                    servicer.set_spdk_nvmf_logs,
-                    request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
-                    response_serializer=gateway__pb2.req_status.SerializeToString,
-            ),
-            'get_gateway_info': grpc.unary_unary_rpc_method_handler(
-                    servicer.get_gateway_info,
-                    request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
-                    response_serializer=gateway__pb2.gateway_info.SerializeToString,
-            ),
+        'namespace_add': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_add,
+            request_deserializer=gateway__pb2.namespace_add_req.FromString,
+            response_serializer=gateway__pb2.nsid_status.SerializeToString,
+        ),
+        'create_subsystem': grpc.unary_unary_rpc_method_handler(
+            servicer.create_subsystem,
+            request_deserializer=gateway__pb2.create_subsystem_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'delete_subsystem': grpc.unary_unary_rpc_method_handler(
+            servicer.delete_subsystem,
+            request_deserializer=gateway__pb2.delete_subsystem_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_namespaces': grpc.unary_unary_rpc_method_handler(
+            servicer.list_namespaces,
+            request_deserializer=gateway__pb2.list_namespaces_req.FromString,
+            response_serializer=gateway__pb2.namespaces_info.SerializeToString,
+        ),
+        'namespace_resize': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_resize,
+            request_deserializer=gateway__pb2.namespace_resize_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_get_io_stats': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_get_io_stats,
+            request_deserializer=gateway__pb2.namespace_get_io_stats_req.FromString,
+            response_serializer=gateway__pb2.namespace_io_stats_info.SerializeToString,
+        ),
+        'namespace_set_qos_limits': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_set_qos_limits,
+            request_deserializer=gateway__pb2.namespace_set_qos_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_change_load_balancing_group': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_change_load_balancing_group,
+            request_deserializer=gateway__pb2.namespace_change_load_balancing_group_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'namespace_delete': grpc.unary_unary_rpc_method_handler(
+            servicer.namespace_delete,
+            request_deserializer=gateway__pb2.namespace_delete_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'add_host': grpc.unary_unary_rpc_method_handler(
+            servicer.add_host,
+            request_deserializer=gateway__pb2.add_host_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'remove_host': grpc.unary_unary_rpc_method_handler(
+            servicer.remove_host,
+            request_deserializer=gateway__pb2.remove_host_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_hosts': grpc.unary_unary_rpc_method_handler(
+            servicer.list_hosts,
+            request_deserializer=gateway__pb2.list_hosts_req.FromString,
+            response_serializer=gateway__pb2.hosts_info.SerializeToString,
+        ),
+        'list_connections': grpc.unary_unary_rpc_method_handler(
+            servicer.list_connections,
+            request_deserializer=gateway__pb2.list_connections_req.FromString,
+            response_serializer=gateway__pb2.connections_info.SerializeToString,
+        ),
+        'create_listener': grpc.unary_unary_rpc_method_handler(
+            servicer.create_listener,
+            request_deserializer=gateway__pb2.create_listener_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'delete_listener': grpc.unary_unary_rpc_method_handler(
+            servicer.delete_listener,
+            request_deserializer=gateway__pb2.delete_listener_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'list_listeners': grpc.unary_unary_rpc_method_handler(
+            servicer.list_listeners,
+            request_deserializer=gateway__pb2.list_listeners_req.FromString,
+            response_serializer=gateway__pb2.listeners_info.SerializeToString,
+        ),
+        'list_subsystems': grpc.unary_unary_rpc_method_handler(
+            servicer.list_subsystems,
+            request_deserializer=gateway__pb2.list_subsystems_req.FromString,
+            response_serializer=gateway__pb2.subsystems_info_cli.SerializeToString,
+        ),
+        'get_subsystems': grpc.unary_unary_rpc_method_handler(
+            servicer.get_subsystems,
+            request_deserializer=gateway__pb2.get_subsystems_req.FromString,
+            response_serializer=gateway__pb2.subsystems_info.SerializeToString,
+        ),
+        'set_ana_state': grpc.unary_unary_rpc_method_handler(
+            servicer.set_ana_state,
+            request_deserializer=gateway__pb2.ana_info.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'get_spdk_nvmf_log_flags_and_level': grpc.unary_unary_rpc_method_handler(
+            servicer.get_spdk_nvmf_log_flags_and_level,
+            request_deserializer=gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.FromString,
+            response_serializer=gateway__pb2.spdk_nvmf_log_flags_and_level_info.SerializeToString,
+        ),
+        'disable_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+            servicer.disable_spdk_nvmf_logs,
+            request_deserializer=gateway__pb2.disable_spdk_nvmf_logs_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'set_spdk_nvmf_logs': grpc.unary_unary_rpc_method_handler(
+            servicer.set_spdk_nvmf_logs,
+            request_deserializer=gateway__pb2.set_spdk_nvmf_logs_req.FromString,
+            response_serializer=gateway__pb2.req_status.SerializeToString,
+        ),
+        'get_gateway_info': grpc.unary_unary_rpc_method_handler(
+            servicer.get_gateway_info,
+            request_deserializer=gateway__pb2.get_gateway_info_req.FromString,
+            response_serializer=gateway__pb2.gateway_info.SerializeToString,
+        ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-            'Gateway', rpc_method_handlers)
+        'Gateway', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
-
  # This class is part of an EXPERIMENTAL API.
+
+
 class Gateway(object):
     """Missing associated documentation comment in .proto file."""
 
     @staticmethod
     def namespace_add(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                      target,
+                      options=(),
+                      channel_credentials=None,
+                      call_credentials=None,
+                      insecure=False,
+                      compression=None,
+                      wait_for_ready=None,
+                      timeout=None,
+                      metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_add',
-            gateway__pb2.namespace_add_req.SerializeToString,
-            gateway__pb2.nsid_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_add_req.SerializeToString,
+                                             gateway__pb2.nsid_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_subsystem(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_subsystem',
-            gateway__pb2.create_subsystem_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.create_subsystem_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_subsystem(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_subsystem',
-            gateway__pb2.delete_subsystem_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.delete_subsystem_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_namespaces(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_namespaces',
-            gateway__pb2.list_namespaces_req.SerializeToString,
-            gateway__pb2.namespaces_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.list_namespaces_req.SerializeToString,
+                                             gateway__pb2.namespaces_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_resize(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_resize',
-            gateway__pb2.namespace_resize_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_resize_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_get_io_stats(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                               target,
+                               options=(),
+                               channel_credentials=None,
+                               call_credentials=None,
+                               insecure=False,
+                               compression=None,
+                               wait_for_ready=None,
+                               timeout=None,
+                               metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_get_io_stats',
-            gateway__pb2.namespace_get_io_stats_req.SerializeToString,
-            gateway__pb2.namespace_io_stats_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_get_io_stats_req.SerializeToString,
+                                             gateway__pb2.namespace_io_stats_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_set_qos_limits(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                                 target,
+                                 options=(),
+                                 channel_credentials=None,
+                                 call_credentials=None,
+                                 insecure=False,
+                                 compression=None,
+                                 wait_for_ready=None,
+                                 timeout=None,
+                                 metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_set_qos_limits',
-            gateway__pb2.namespace_set_qos_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_set_qos_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_change_load_balancing_group(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                                              target,
+                                              options=(),
+                                              channel_credentials=None,
+                                              call_credentials=None,
+                                              insecure=False,
+                                              compression=None,
+                                              wait_for_ready=None,
+                                              timeout=None,
+                                              metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_change_load_balancing_group',
-            gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_change_load_balancing_group_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def namespace_delete(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/namespace_delete',
-            gateway__pb2.namespace_delete_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.namespace_delete_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def add_host(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                 target,
+                 options=(),
+                 channel_credentials=None,
+                 call_credentials=None,
+                 insecure=False,
+                 compression=None,
+                 wait_for_ready=None,
+                 timeout=None,
+                 metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/add_host',
-            gateway__pb2.add_host_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.add_host_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def remove_host(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                    target,
+                    options=(),
+                    channel_credentials=None,
+                    call_credentials=None,
+                    insecure=False,
+                    compression=None,
+                    wait_for_ready=None,
+                    timeout=None,
+                    metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/remove_host',
-            gateway__pb2.remove_host_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.remove_host_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_hosts(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                   target,
+                   options=(),
+                   channel_credentials=None,
+                   call_credentials=None,
+                   insecure=False,
+                   compression=None,
+                   wait_for_ready=None,
+                   timeout=None,
+                   metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_hosts',
-            gateway__pb2.list_hosts_req.SerializeToString,
-            gateway__pb2.hosts_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.list_hosts_req.SerializeToString,
+                                             gateway__pb2.hosts_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_connections(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_connections',
-            gateway__pb2.list_connections_req.SerializeToString,
-            gateway__pb2.connections_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.list_connections_req.SerializeToString,
+                                             gateway__pb2.connections_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def create_listener(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/create_listener',
-            gateway__pb2.create_listener_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.create_listener_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def delete_listener(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/delete_listener',
-            gateway__pb2.delete_listener_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.delete_listener_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_listeners(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                       target,
+                       options=(),
+                       channel_credentials=None,
+                       call_credentials=None,
+                       insecure=False,
+                       compression=None,
+                       wait_for_ready=None,
+                       timeout=None,
+                       metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_listeners',
-            gateway__pb2.list_listeners_req.SerializeToString,
-            gateway__pb2.listeners_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.list_listeners_req.SerializeToString,
+                                             gateway__pb2.listeners_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def list_subsystems(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                        target,
+                        options=(),
+                        channel_credentials=None,
+                        call_credentials=None,
+                        insecure=False,
+                        compression=None,
+                        wait_for_ready=None,
+                        timeout=None,
+                        metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/list_subsystems',
-            gateway__pb2.list_subsystems_req.SerializeToString,
-            gateway__pb2.subsystems_info_cli.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.list_subsystems_req.SerializeToString,
+                                             gateway__pb2.subsystems_info_cli.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_subsystems(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                       target,
+                       options=(),
+                       channel_credentials=None,
+                       call_credentials=None,
+                       insecure=False,
+                       compression=None,
+                       wait_for_ready=None,
+                       timeout=None,
+                       metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_subsystems',
-            gateway__pb2.get_subsystems_req.SerializeToString,
-            gateway__pb2.subsystems_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.get_subsystems_req.SerializeToString,
+                                             gateway__pb2.subsystems_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def set_ana_state(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                      target,
+                      options=(),
+                      channel_credentials=None,
+                      call_credentials=None,
+                      insecure=False,
+                      compression=None,
+                      wait_for_ready=None,
+                      timeout=None,
+                      metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/set_ana_state',
-            gateway__pb2.ana_info.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.ana_info.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_spdk_nvmf_log_flags_and_level(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                                          target,
+                                          options=(),
+                                          channel_credentials=None,
+                                          call_credentials=None,
+                                          insecure=False,
+                                          compression=None,
+                                          wait_for_ready=None,
+                                          timeout=None,
+                                          metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_spdk_nvmf_log_flags_and_level',
-            gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
-            gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.get_spdk_nvmf_log_flags_and_level_req.SerializeToString,
+                                             gateway__pb2.spdk_nvmf_log_flags_and_level_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def disable_spdk_nvmf_logs(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                               target,
+                               options=(),
+                               channel_credentials=None,
+                               call_credentials=None,
+                               insecure=False,
+                               compression=None,
+                               wait_for_ready=None,
+                               timeout=None,
+                               metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/disable_spdk_nvmf_logs',
-            gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.disable_spdk_nvmf_logs_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def set_spdk_nvmf_logs(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                           target,
+                           options=(),
+                           channel_credentials=None,
+                           call_credentials=None,
+                           insecure=False,
+                           compression=None,
+                           wait_for_ready=None,
+                           timeout=None,
+                           metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/set_spdk_nvmf_logs',
-            gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
-            gateway__pb2.req_status.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.set_spdk_nvmf_logs_req.SerializeToString,
+                                             gateway__pb2.req_status.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
     def get_gateway_info(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                         target,
+                         options=(),
+                         channel_credentials=None,
+                         call_credentials=None,
+                         insecure=False,
+                         compression=None,
+                         wait_for_ready=None,
+                         timeout=None,
+                         metadata=None):
         return grpc.experimental.unary_unary(request, target, '/Gateway/get_gateway_info',
-            gateway__pb2.get_gateway_info_req.SerializeToString,
-            gateway__pb2.gateway_info.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             gateway__pb2.get_gateway_info_req.SerializeToString,
+                                             gateway__pb2.gateway_info.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

From 51796e80f35da8ca13b72f31eeafefd4c1e40773 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 6 Mar 2024 17:15:20 +1000
Subject: [PATCH 2170/2492] doc/dev: edit teuthology workflow

Improve a couple of sentences in
doc/dev/developer_guide_testing_integration_tests/tests-integration-testing-teutology-workflow.rst.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../tests-integration-testing-teuthology-workflow.rst    | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
index 6b8877886a78..427c84bd038c 100644
--- a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst
@@ -209,13 +209,14 @@ For example: for the above test ID, the path is::
 
 This method can be used to view the log more quickly than would be possible through a browser.
 
-In addition to teuthology.log, there are other files included for debugging purposes:
+In addition to ``teuthology.log``, some other files are included for debugging
+purposes:
 
-* unit_test_summary.yaml: It provides a summary of all unit test failures and 
-  is optionally generated when the ``unit_test_scan`` configuration option is 
+* ``unit_test_summary.yaml``: Provides a summary of all unit test failures.
+  Generated (optionally) when the ``unit_test_scan`` configuration option is
   used in the job's YAML file.
 
-* valgrind.yaml: It summarizes any Valgrind errors that may occur.
+* ``valgrind.yaml``: Summarizes any Valgrind errors that may occur.
 
 .. note:: To access archives more conveniently, ``/a/`` has been symbolically
    linked to ``/ceph/teuthology-archive/``. For instance, to access the previous

From 32cbe079c60031f67ee91e2f74f39244a027eba6 Mon Sep 17 00:00:00 2001
From: Shinya Hayashi <shinya-hayashi@cybozu.co.jp>
Date: Mon, 5 Feb 2024 04:16:26 +0000
Subject: [PATCH 2171/2492] systemd: add systemd unit file for ceph-exporter

Signed-off-by: Shinya Hayashi <shinya-hayashi@cybozu.co.jp>
---
 ceph.spec.in                     |  1 +
 debian/ceph-exporter.install     |  1 +
 systemd/CMakeLists.txt           |  1 +
 systemd/ceph-exporter.service.in | 29 +++++++++++++++++++++++++++++
 4 files changed, 32 insertions(+)
 create mode 100644 systemd/ceph-exporter.service.in

diff --git a/ceph.spec.in b/ceph.spec.in
index 58ccf067997a..ff2bb741c6d2 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -2052,6 +2052,7 @@ fi
 
 %files -n ceph-exporter
 %{_bindir}/ceph-exporter
+%{_unitdir}/ceph-exporter.service
 
 %files -n rbd-fuse
 %{_bindir}/rbd-fuse
diff --git a/debian/ceph-exporter.install b/debian/ceph-exporter.install
index 618cf9eadcc1..1ac0edcd2a18 100644
--- a/debian/ceph-exporter.install
+++ b/debian/ceph-exporter.install
@@ -1 +1,2 @@
+lib/systemd/system/ceph-exporter*
 usr/bin/ceph-exporter
diff --git a/systemd/CMakeLists.txt b/systemd/CMakeLists.txt
index ad75ce61cd8d..366bab4195df 100644
--- a/systemd/CMakeLists.txt
+++ b/systemd/CMakeLists.txt
@@ -14,6 +14,7 @@ set(CEPH_SYSTEMD_ENV_DIR "/etc/sysconfig"
 set(SYSTEMD_ENV_FILE "${CEPH_SYSTEMD_ENV_DIR}/ceph")
 foreach(service
     ceph-crash
+    ceph-exporter
     ceph-fuse@
     ceph-mds@
     ceph-mgr@
diff --git a/systemd/ceph-exporter.service.in b/systemd/ceph-exporter.service.in
new file mode 100644
index 000000000000..f4f6d05c4b4d
--- /dev/null
+++ b/systemd/ceph-exporter.service.in
@@ -0,0 +1,29 @@
+[Unit]
+Description=Ceph cluster exporter daemon
+PartOf=ceph.target
+After=network-online.target local-fs.target
+Before=ceph.target
+Wants=network-online.target local-fs.target ceph.target ceph-mon.target
+
+[Service]
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStart=/usr/bin/ceph-exporter -f --id %i --setuser ceph --setgroup ceph
+LockPersonality=true
+NoNewPrivileges=true
+PrivateDevices=yes
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=full
+Restart=on-failure
+RestartSec=10
+RestrictSUIDSGID=true
+StartLimitBurst=3
+StartLimitInterval=30min
+
+[Install]
+WantedBy=multi-user.target ceph.target

From d96a1805f3975e7eb098cb32a9a1da0e10321fb7 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Thu, 18 Jan 2024 12:14:42 +0530
Subject: [PATCH 2172/2492] mgr/dashboard: Allow the user to add the
 access/secret key on zone edit and not on zone creation

Fixes: https://tracker.ceph.com/issues/64080

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 .../rgw-multisite-zone-form.component.html    | 42 ++++++++++++-------
 .../rgw-multisite-zone-form.component.ts      |  4 +-
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.html
index 3856c42f0ea7..98741bed3990 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.html
@@ -110,7 +110,7 @@
         </div>
       </div>
       <div class="form-group row">
-        <label class="cd-col-form-label required"
+        <label class="cd-col-form-label"
                for="access_key"
                i18n>S3 access key
           <cd-helper>
@@ -118,29 +118,41 @@
           </cd-helper>
         </label>
         <div class="cd-col-form-input">
-          <input class="form-control"
-                 type="text"
-                 placeholder="DiPt4V7WWvy2njL1z6aC"
-                 id="access_key"
-                 name="access_key"
-                 formControlName="access_key">
+          <div class="input-group">
+            <input class="form-control"
+                   type="password"
+                   placeholder="DiPt4V7WWvy2njL1z6aC"
+                   id="access_key"
+                   name="access_key"
+                   formControlName="access_key">
+            <button type="button"
+                    class="btn btn-light"
+                    cdPasswordButton="access_key">
+            </button>
+          </div>
         </div>
       </div>
       <div class="form-group row">
-        <label class="cd-col-form-label required"
-               for="access_key"
+        <label class="cd-col-form-label"
+               for="secret_key"
                i18n>S3 secret key
           <cd-helper>
             <span>To see or copy your S3 access key, go to <b>Object Gateway > Users</b> and click on your user name. In <b>Keys</b>, click <b>Show</b>. View the secret key by clicking Show and copy the key by clicking <b>Copy to Clipboard</b>.</span>
           </cd-helper>
         </label>
         <div class="cd-col-form-input">
-          <input class="form-control"
-                 type="text"
-                 placeholder="xSZUdYky0bTctAdCEEW8ikhfBVKsBV5LFYL82vvh"
-                 id="secret_key"
-                 name="secret_key"
-                 formControlName="secret_key">
+          <div class="input-group">
+            <input class="form-control"
+                   type="password"
+                   placeholder="xSZUdYky0bTctAdCEEW8ikhfBVKsBV5LFYL82vvh"
+                   id="secret_key"
+                   name="secret_key"
+                   formControlName="secret_key">
+            <button type="button"
+                    class="btn btn-light"
+                    cdPasswordButton="secret_key">
+            </button>
+          </div>
         </div>
       </div>
       <div class="form-group row"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.ts
index 76e2970dde7f..21106c53eb3b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-multisite-zone-form/rgw-multisite-zone-form.component.ts
@@ -112,8 +112,8 @@ export class RgwMultisiteZoneFormComponent implements OnInit {
           Validators.required
         ]
       }),
-      access_key: new UntypedFormControl(null, Validators.required),
-      secret_key: new UntypedFormControl(null, Validators.required),
+      access_key: new UntypedFormControl('', {}),
+      secret_key: new UntypedFormControl('', {}),
       placementTarget: new UntypedFormControl(null),
       placementDataPool: new UntypedFormControl(''),
       placementIndexPool: new UntypedFormControl(null),

From cc279a3dd23d9219e508b80512f363a4ae577392 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Tue, 5 Mar 2024 10:05:18 +0000
Subject: [PATCH 2173/2492] mgr/node-proxy: handle 'None' statuses returned by
 RedFish

Looks like RedFish might return 'None' values for some attributes.

for instance:

```
[root@ceph-node-01 ~]# curl -s -k -X GET https://169.254.1.1/redfish/v1/Systems/System.Embedded.1/Storage/AHCI.SL.6-1/Drives/Disk.Direct.0-0:AHCI.SL.6-1 -H "X-Auth-Token: 3264251c28191fa5e7c9ebec49ef90fc"  | jq .Status
{
  "Health": "OK",
  "HealthRollup": "OK",
  "State": "Enabled"
}
[root@ceph-node-01 ~]# curl -s -k -X GET https://169.254.1.1/redfish/v1/Systems/System.Embedded.1/Storage/NonRAID.Slot.2-1/Drives/Disk.Bay.0:Enclosure.Internal.0-1:NonRAID.Slot.2-1 -H "X-Auth-Token: 3264251c28191fa5e7c9ebec49ef90fc" | jq .Status
{
  "Health": null,
  "HealthRollup": null,
  "State": "Enabled"
}
[root@ceph-node-01 ~]#
```

Although this seems to be a bug from RedFish, we need to handle
the case when it happens otherwise it makes the mgr orchestrator module
throw an error.

The idea here is to create a new status "unknown" when we can't fetch the
real status of a component.

Fixes: https://tracker.ceph.com/issues/64712

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/util.py |  2 ++
 src/pybind/mgr/cephadm/inventory.py         | 24 ++++++++++++++-------
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/util.py b/src/ceph-node-proxy/ceph_node_proxy/util.py
index f6ed0fb483d6..677161c63fd3 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/util.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/util.py
@@ -126,6 +126,8 @@ def normalize_dict(test_dict: Dict) -> Dict:
         if isinstance(test_dict[key], dict):
             res[key.lower()] = normalize_dict(test_dict[key])
         else:
+            if test_dict[key] is None:
+                test_dict[key] = 'unknown'
             res[key.lower()] = test_dict[key]
     return res
 
diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py
index 235737ef10e7..966ffc0461c8 100644
--- a/src/pybind/mgr/cephadm/inventory.py
+++ b/src/pybind/mgr/cephadm/inventory.py
@@ -8,7 +8,7 @@
 import math
 import socket
 from typing import TYPE_CHECKING, Dict, List, Iterator, Optional, Any, Tuple, Set, Mapping, cast, \
-    NamedTuple, Type
+    NamedTuple, Type, ValuesView
 
 import orchestrator
 from ceph.deployment import inventory
@@ -1485,10 +1485,12 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
         """
         hostname = kw.get('hostname')
         hosts = [hostname] if hostname else self.data.keys()
-        mapper: Dict[bool, str] = {
-            True: 'error',
-            False: 'ok'
-        }
+
+        def is_unknown(statuses: ValuesView) -> bool:
+            return any([status['status']['health'].lower() == 'unknown' for status in statuses]) and not is_error(statuses)
+
+        def is_error(statuses: ValuesView) -> bool:
+            return any([status['status']['health'].lower() == 'error' for status in statuses])
 
         _result: Dict[str, Any] = {}
 
@@ -1496,9 +1498,15 @@ def summary(self, **kw: Any) -> Dict[str, Any]:
             _result[host] = {}
             _result[host]['status'] = {}
             data = self.data[host]
-            for component, details in data['status'].items():
-                res = any([member['status']['health'].lower() != 'ok' for member in data['status'][component].values()])
-                _result[host]['status'][component] = mapper[res]
+            for component in data['status'].keys():
+                values = data['status'][component].values()
+                if is_error(values):
+                    state = 'error'
+                elif is_unknown(values):
+                    state = 'unknown'
+                else:
+                    state = 'ok'
+                _result[host]['status'][component] = state
             _result[host]['sn'] = data['sn']
             _result[host]['host'] = data['host']
             _result[host]['firmwares'] = data['firmwares']

From 71c9fe55d6a4ba3dbff166c155cc74d485f75330 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 4 Mar 2024 06:39:06 +0530
Subject: [PATCH 2174/2492] qa: use allowlist instead of whitelist for selinux
 denials

https://github.com/ceph/teuthology/pull/1757 introduced this change,
however, qa/distros continued to use `whitelist' keyword.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/distros/all/centos_7.6.yaml                                  | 2 +-
 qa/distros/all/centos_8.0.yaml                                  | 2 +-
 qa/distros/all/centos_8.1.yaml                                  | 2 +-
 qa/distros/all/centos_8.2.yaml                                  | 2 +-
 qa/distros/all/centos_8.3.yaml                                  | 2 +-
 qa/distros/all/centos_8.stream.yaml                             | 2 +-
 qa/distros/all/rhel_7.6.yaml                                    | 2 +-
 qa/distros/all/rhel_7.7.yaml                                    | 2 +-
 qa/distros/all/rhel_8.0.yaml                                    | 2 +-
 qa/distros/all/rhel_8.1.yaml                                    | 2 +-
 qa/distros/all/rhel_8.3.yaml                                    | 2 +-
 qa/distros/all/rhel_8.4.yaml                                    | 2 +-
 qa/distros/all/rhel_8.5.yaml                                    | 2 +-
 qa/distros/all/rhel_8.6.yaml                                    | 2 +-
 qa/distros/container-hosts/centos_8.stream_container_tools.yaml | 2 +-
 .../container-hosts/centos_8.stream_container_tools_crun.yaml   | 2 +-
 qa/distros/container-hosts/centos_9.stream.yaml                 | 2 +-
 qa/distros/container-hosts/centos_9.stream_runc.yaml            | 2 +-
 qa/distros/podman/centos_8.stream_container_tools.yaml          | 2 +-
 qa/distros/podman/centos_9.stream.yaml                          | 2 +-
 20 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/qa/distros/all/centos_7.6.yaml b/qa/distros/all/centos_7.6.yaml
index 81014e102f17..1eddf680c110 100644
--- a/qa/distros/all/centos_7.6.yaml
+++ b/qa/distros/all/centos_7.6.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "7.6"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/centos_8.0.yaml b/qa/distros/all/centos_8.0.yaml
index 1679bf0d5580..787bbfeb01ee 100644
--- a/qa/distros/all/centos_8.0.yaml
+++ b/qa/distros/all/centos_8.0.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "8.0"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/centos_8.1.yaml b/qa/distros/all/centos_8.1.yaml
index f764e507992b..986e74abf9ce 100644
--- a/qa/distros/all/centos_8.1.yaml
+++ b/qa/distros/all/centos_8.1.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "8.1"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/centos_8.2.yaml b/qa/distros/all/centos_8.2.yaml
index 1ccbd8abdba1..d9f6178c6e66 100644
--- a/qa/distros/all/centos_8.2.yaml
+++ b/qa/distros/all/centos_8.2.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "8.2"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/centos_8.3.yaml b/qa/distros/all/centos_8.3.yaml
index b9a7c2579a3a..54e1c3b1161b 100644
--- a/qa/distros/all/centos_8.3.yaml
+++ b/qa/distros/all/centos_8.3.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "8.3"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/centos_8.stream.yaml b/qa/distros/all/centos_8.stream.yaml
index 5ae75c6bee3f..f9607d2c6803 100644
--- a/qa/distros/all/centos_8.stream.yaml
+++ b/qa/distros/all/centos_8.stream.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "8.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_7.6.yaml b/qa/distros/all/rhel_7.6.yaml
index 37bc0fb4a814..5a075301497e 100644
--- a/qa/distros/all/rhel_7.6.yaml
+++ b/qa/distros/all/rhel_7.6.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "7.6"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_7.7.yaml b/qa/distros/all/rhel_7.7.yaml
index ac44fe18e99d..654c73177d37 100644
--- a/qa/distros/all/rhel_7.7.yaml
+++ b/qa/distros/all/rhel_7.7.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "7.7"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.0.yaml b/qa/distros/all/rhel_8.0.yaml
index da6a33ed0619..7605739901f1 100644
--- a/qa/distros/all/rhel_8.0.yaml
+++ b/qa/distros/all/rhel_8.0.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.0"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.1.yaml b/qa/distros/all/rhel_8.1.yaml
index c73893149dee..4b0439c0f070 100644
--- a/qa/distros/all/rhel_8.1.yaml
+++ b/qa/distros/all/rhel_8.1.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.1"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.3.yaml b/qa/distros/all/rhel_8.3.yaml
index 4e44bbff59c1..48414fe75d18 100644
--- a/qa/distros/all/rhel_8.3.yaml
+++ b/qa/distros/all/rhel_8.3.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.3"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.4.yaml b/qa/distros/all/rhel_8.4.yaml
index 5a299ffc5396..2d4088392fe7 100644
--- a/qa/distros/all/rhel_8.4.yaml
+++ b/qa/distros/all/rhel_8.4.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.4"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.5.yaml b/qa/distros/all/rhel_8.5.yaml
index 3e02bb1965bd..591b22661d02 100644
--- a/qa/distros/all/rhel_8.5.yaml
+++ b/qa/distros/all/rhel_8.5.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.5"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/all/rhel_8.6.yaml b/qa/distros/all/rhel_8.6.yaml
index 1f9a6b73fda5..f70647ed823d 100644
--- a/qa/distros/all/rhel_8.6.yaml
+++ b/qa/distros/all/rhel_8.6.yaml
@@ -2,6 +2,6 @@ os_type: rhel
 os_version: "8.6"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 ktype: distro
diff --git a/qa/distros/container-hosts/centos_8.stream_container_tools.yaml b/qa/distros/container-hosts/centos_8.stream_container_tools.yaml
index 4a76306f19d5..3dad3b845d46 100644
--- a/qa/distros/container-hosts/centos_8.stream_container_tools.yaml
+++ b/qa/distros/container-hosts/centos_8.stream_container_tools.yaml
@@ -2,7 +2,7 @@ os_type: centos
 os_version: "8.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 
 tasks:
diff --git a/qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml b/qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml
index b06e1c87d7c2..d1d395b93246 100644
--- a/qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml
+++ b/qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml
@@ -2,7 +2,7 @@ os_type: centos
 os_version: "8.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 
 tasks:
diff --git a/qa/distros/container-hosts/centos_9.stream.yaml b/qa/distros/container-hosts/centos_9.stream.yaml
index d3bc430e63b9..de05533cea80 100644
--- a/qa/distros/container-hosts/centos_9.stream.yaml
+++ b/qa/distros/container-hosts/centos_9.stream.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "9.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 
diff --git a/qa/distros/container-hosts/centos_9.stream_runc.yaml b/qa/distros/container-hosts/centos_9.stream_runc.yaml
index 2e4f9fe3a61e..10d2137a5b32 100644
--- a/qa/distros/container-hosts/centos_9.stream_runc.yaml
+++ b/qa/distros/container-hosts/centos_9.stream_runc.yaml
@@ -2,7 +2,7 @@ os_type: centos
 os_version: "9.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 
 tasks:
diff --git a/qa/distros/podman/centos_8.stream_container_tools.yaml b/qa/distros/podman/centos_8.stream_container_tools.yaml
index 4a76306f19d5..3dad3b845d46 100644
--- a/qa/distros/podman/centos_8.stream_container_tools.yaml
+++ b/qa/distros/podman/centos_8.stream_container_tools.yaml
@@ -2,7 +2,7 @@ os_type: centos
 os_version: "8.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 
 tasks:
diff --git a/qa/distros/podman/centos_9.stream.yaml b/qa/distros/podman/centos_9.stream.yaml
index d3bc430e63b9..de05533cea80 100644
--- a/qa/distros/podman/centos_9.stream.yaml
+++ b/qa/distros/podman/centos_9.stream.yaml
@@ -2,6 +2,6 @@ os_type: centos
 os_version: "9.stream"
 overrides:
   selinux:
-    whitelist:
+    allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
 

From 7a6389272affd04aab9ea88352ac0791dc472480 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 4 Mar 2024 06:45:11 +0530
Subject: [PATCH 2175/2492] qa: ignore container checkpoint/restore related
 selinux denials for centos9

Fixes: http://tracker.ceph.com/issues/64616
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/distros/all/centos_9.stream.yaml    | 4 ++++
 qa/distros/podman/centos_9.stream.yaml | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/qa/distros/all/centos_9.stream.yaml b/qa/distros/all/centos_9.stream.yaml
index 52d553bd6397..7fe93afcfa95 100644
--- a/qa/distros/all/centos_9.stream.yaml
+++ b/qa/distros/all/centos_9.stream.yaml
@@ -1,2 +1,6 @@
 os_type: centos
 os_version: "9.stream"
+overrides:
+  selinux:
+    allowlist:
+      - scontext=system_u:system_r:getty_t:s0
diff --git a/qa/distros/podman/centos_9.stream.yaml b/qa/distros/podman/centos_9.stream.yaml
index de05533cea80..c45ae3f1ae3f 100644
--- a/qa/distros/podman/centos_9.stream.yaml
+++ b/qa/distros/podman/centos_9.stream.yaml
@@ -4,4 +4,4 @@ overrides:
   selinux:
     allowlist:
       - scontext=system_u:system_r:logrotate_t:s0
-
+      - scontext=system_u:system_r:getty_t:s0

From f5a23d5811829d0f43831740ec3dced6a3b725f3 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 26 Feb 2024 11:12:07 +0530
Subject: [PATCH 2176/2492] qa/cephfs: add `MON_DOWN' and `deprecated feature
 inline_data' to ignorelist

Fixes: http://tracker.ceph.com/issues/64746
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 qa/cephfs/overrides/ignorelist_health.yaml              | 1 +
 qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
index 90811d6f233b..aef498f92c60 100644
--- a/qa/cephfs/overrides/ignorelist_health.yaml
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -13,3 +13,4 @@ overrides:
       - POOL_APP_NOT_ENABLED
       - overall HEALTH_
       - Replacing daemon
+      - deprecated feature inline_data
diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
index 41ba84f04db5..c9c46cc48f41 100644
--- a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
+++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
@@ -7,3 +7,4 @@ overrides:
       - but it is still running
 # MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a'
       - is not responding
+      - MON_DOWN

From 59dbbb218cf06abc135f041308deace6b859c8bd Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Wed, 6 Mar 2024 16:33:45 +0530
Subject: [PATCH 2177/2492] cephfs_mirror, qa: fix typo in current_syncing_snap
 status

Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 qa/tasks/cephfs/test_mirroring.py       | 2 +-
 src/tools/cephfs_mirror/PeerReplayer.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py
index bad2d2daffdc..d38148a79648 100644
--- a/qa/tasks/cephfs/test_mirroring.py
+++ b/qa/tasks/cephfs/test_mirroring.py
@@ -229,7 +229,7 @@ def check_peer_snap_in_progress(self, fs_name, fs_id,
                                          'fs', 'mirror', 'peer', 'status',
                                          f'{fs_name}@{fs_id}', peer_uuid)
         self.assertTrue('syncing' == res[dir_name]['state'])
-        self.assertTrue(res[dir_name]['current_sycning_snap']['name'] == snap_name)
+        self.assertTrue(res[dir_name]['current_syncing_snap']['name'] == snap_name)
 
     def verify_snapshot(self, dir_name, snap_name):
         snap_list = self.mount_b.ls(path=f'{dir_name}/.snap')
diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index 6f3fb724a659..ca4116ca3704 100644
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -1611,7 +1611,7 @@ void PeerReplayer::peer_status(Formatter *f) {
       f->dump_string("state", "idle");
     } else {
       f->dump_string("state", "syncing");
-      f->open_object_section("current_sycning_snap");
+      f->open_object_section("current_syncing_snap");
       f->dump_unsigned("id", (*sync_stat.current_syncing_snap).first);
       f->dump_string("name", (*sync_stat.current_syncing_snap).second);
       f->close_section();

From ae08855cf870173dce2a47a28f3bbb22e7ae0ca2 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 6 Mar 2024 21:40:10 +1000
Subject: [PATCH 2178/2492] doc/architecture: correct typo

s/client/clients/ where necessary, and add a link to the glossary.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/architecture.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/architecture.rst b/doc/architecture.rst
index b87abc2e057c..3f0f9519f5e2 100644
--- a/doc/architecture.rst
+++ b/doc/architecture.rst
@@ -21,8 +21,8 @@ The Ceph Storage Cluster
 Ceph provides an infinitely scalable :term:`Ceph Storage Cluster` based upon
 :abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, a reliable,
 distributed storage service that uses the intelligence in each of its nodes to
-secure the data it stores and to provide that data to client. See Sage Weil's
-"`The RADOS Object Store
+secure the data it stores and to provide that data to :term:`client`\s. See
+Sage Weil's "`The RADOS Object Store
 <https://ceph.io/en/news/blog/2009/the-rados-distributed-object-store/>`_" blog
 post for a brief explanation of RADOS and see `RADOS - A Scalable, Reliable
 Storage Service for Petabyte-scale Storage Clusters`_ for an exhaustive

From e0a25e59d0d819e9556058b65c577bad19abb411 Mon Sep 17 00:00:00 2001
From: Zhipeng Li <qiuxinyidian@gmail.com>
Date: Wed, 31 Jan 2024 16:59:58 +0800
Subject: [PATCH 2179/2492] rgw: add help info for radosgw-admin list object in
 bucket Fixes:https://tracker.ceph.com/issues/64264

Signed-off-by: Zhipeng Li <qiuxinyidian@gmail.com>
---
 src/rgw/rgw_admin.cc              | 3 +++
 src/test/cli/radosgw-admin/help.t | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 3d817c6039bf..48741272496f 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -489,6 +489,9 @@ void usage()
   cout << "\nradoslist options:\n";
   cout << "   --rgw-obj-fs                  the field separator that will separate the rados object name from the rgw object name;\n";
   cout << "                                 additionally rados objects for incomplete multipart uploads will not be output\n";
+  cout << "\nBucket list objects options:\n";
+  cout << "   --max-entries                 max number of entries listed (default 1000)\n";
+  cout << "   --marker                      the marker used to specify on which entry the listing begins, default none (i.e., very first entry)\n";
   cout << "\n";
   generic_client_usage();
 }
diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 8f6fc36190cc..295a599dca1d 100644
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -369,6 +369,10 @@
      --rgw-obj-fs                  the field separator that will separate the rados object name from the rgw object name;
                                    additionally rados objects for incomplete multipart uploads will not be output
   
+  Bucket list objects options:
+     --max-entries                 max number of entries listed (default 1000)
+     --marker                      the marker used to specify on which entry the listing begins, default none (i.e., very first entry)
+  
     --conf/-c FILE    read configuration from the given configuration file
     --id ID           set ID portion of my name
     --name/-n TYPE.ID set name

From 25ce4845f57d1771742672754ba3633622006615 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 6 Mar 2024 09:25:18 -0500
Subject: [PATCH 2180/2492] rpm: disable system_qat for non-x86_64 arch

Fixes: https://tracker.ceph.com/issues/64678

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 ceph.spec.in | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index ed3674655277..38dd579868cb 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -111,9 +111,14 @@
 # this is tracked in https://bugzilla.redhat.com/2152265
 %bcond_with system_arrow
 %endif
+# qat only supported for intel devices
+%ifarch x86_64
 %if 0%{?fedora} || 0%{?rhel} >= 9
 %bcond_without system_qat
-%else
+%else # not fedora/rhel
+%bcond_with system_qat
+%endif
+%else # not x86_64
 %bcond_with system_qat
 %endif
 %if 0%{?fedora} || 0%{?suse_version} || 0%{?rhel} >= 8 || 0%{?openEuler}

From 2d4a3ca5c94908a6865dce0192677d8346e1f848 Mon Sep 17 00:00:00 2001
From: Daniel Gryniewicz <dang@redhat.com>
Date: Thu, 16 Jun 2022 11:46:24 -0400
Subject: [PATCH 2181/2492] RGW - Swift retarget needs bucket set on object

When a bucket list is retargeted to an object get, then the object may
not yet have it's bucket set.  Ensure the bucket is set on the object.

Fixes: https://tracker.ceph.com/issues/56029
Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
---
 src/rgw/rgw_rest_swift.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index cc75b8146524..dd8d5a47321a 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -2597,6 +2597,7 @@ RGWOp* RGWSwiftWebsiteHandler::get_ws_index_op()
   } else {
     s->object->set_name(s->bucket->get_info().website_conf.get_index_doc());
   }
+  s->object->set_bucket(s->bucket.get());
 
   auto getop = new RGWGetObj_ObjStore_SWIFT;
   getop->set_get_data(boost::algorithm::equals("GET", s->info.method));

From 8b896a9e145796119f0451201dc4d53ddac97db0 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Wed, 6 Mar 2024 18:06:50 +0200
Subject: [PATCH 2182/2492] mds/quiesce-db: keep the db thread alive until
 shutdown

With the change we can now avoid having to join it during the membership update, preventing potential deadlocks

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/MDSRank.cc            |  8 ++--
 src/mds/QuiesceDbManager.cc   | 90 ++++++++++++++++++-----------------
 src/mds/QuiesceDbManager.h    | 33 ++++++-------
 src/test/mds/TestQuiesceDb.cc | 15 +++---
 4 files changed, 73 insertions(+), 73 deletions(-)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index c14ca9e1cb1e..a2bab2802a82 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -864,17 +864,15 @@ void MDSRankDispatcher::shutdown()
 
   progress_thread.shutdown();
 
-  if (quiesce_db_manager) {
-    // shutdown the manager
-    quiesce_db_manager->update_membership({});
-  }
-
   // release mds_lock for finisher/messenger threads (e.g.
   // MDSDaemon::ms_handle_reset called from Messenger).
   mds_lock.unlock();
 
   // shut down messenger
   messenger->shutdown();
+
+  // the quiesce db membership is
+  // managed by the mds map update, no need to address that here
   if (quiesce_agent) {
     // reset any tracked roots
     quiesce_agent->shutdown();
diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
index ca69fef73fda..6fccaacf10c4 100644
--- a/src/mds/QuiesceDbManager.cc
+++ b/src/mds/QuiesceDbManager.cc
@@ -22,7 +22,7 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds_quiesce
 #undef dout_prefix
-#define dout_prefix *_dout << "quiesce.mgr <" << __func__ << "> "
+#define dout_prefix *_dout << "quiesce.mgr." << membership.me << " <" << __func__ << "> "
 
 #undef dout
 #define dout(lvl)                                                        \
@@ -54,23 +54,23 @@ static QuiesceTimeInterval time_distance(QuiesceTimePoint lhs, QuiesceTimePoint
 
 bool QuiesceDbManager::db_thread_has_work() const
 {
-  return false
+  return db_thread_should_exit
       || pending_acks.size() > 0
       || pending_requests.size() > 0
       || pending_db_updates.size() > 0
       || (agent_callback.has_value() && agent_callback->if_newer < db_version())
-      || (!cluster_membership.has_value() || cluster_membership->epoch != membership.epoch);
+      || (cluster_membership.has_value() && cluster_membership->epoch != membership.epoch);
 }
 
 void* QuiesceDbManager::quiesce_db_thread_main()
 {
-  db_thread_enter();
-
   std::unique_lock ls(submit_mutex);
   QuiesceTimeInterval next_event_at_age = QuiesceTimeInterval::max();
   QuiesceDbVersion last_acked = {0, 0};
 
-  while (true) {
+  dout(5) << "Entering the main thread" << dendl;
+  bool keep_working = true;
+  while (keep_working) {
 
     auto db_age = db.get_age();
 
@@ -78,11 +78,10 @@ void* QuiesceDbManager::quiesce_db_thread_main()
       submit_condition.wait_for(ls, next_event_at_age - db_age);
     }
 
-    if (!membership_upkeep()) {
-      break;
-    }
+    auto [is_member, should_exit] = membership_upkeep();
+    keep_working = !should_exit;
 
-    {
+    if (is_member) {
       decltype(pending_acks) acks(std::move(pending_acks));
       decltype(pending_requests) requests(std::move(pending_requests));
       decltype(pending_db_updates) db_updates(std::move(pending_db_updates));
@@ -105,6 +104,10 @@ void* QuiesceDbManager::quiesce_db_thread_main()
       } else {
         next_event_at_age = replica_upkeep(std::move(db_updates));
       }
+    } else {
+      ls.unlock();
+      dout(15) << "not a cluster member, keeping idle " << dendl;
+      next_event_at_age = QuiesceTimeInterval::max();
     }
   
     complete_requests();
@@ -131,7 +134,7 @@ void* QuiesceDbManager::quiesce_db_thread_main()
       }
     }
 
-    if (send_ack) {
+    if (is_member && send_ack) {
       auto db_version = quiesce_map.db_version;
       dout(20) << "synchronous agent ack: " << quiesce_map << dendl;
       auto rc = membership.send_ack(std::move(quiesce_map));
@@ -148,7 +151,7 @@ void* QuiesceDbManager::quiesce_db_thread_main()
 
   ls.unlock();
 
-  db_thread_exit();
+  dout(5) << "Exiting the main thread" << dendl;
 
   return 0;
 }
@@ -160,43 +163,36 @@ void QuiesceDbManager::update_membership(const QuiesceClusterMembership& new_mem
   bool will_participate = new_membership.members.contains(new_membership.me);
   dout(20) << "will participate: " << std::boolalpha << will_participate << std::noboolalpha << dendl;
 
-  if (cluster_membership && !will_participate) {
-    // stop the thread
-    cluster_membership.reset();
+  if (will_participate && !quiesce_db_thread.is_started()) {
+    // start the thread
+    dout(5) << "starting the db mgr thread at epoch: " << new_membership.epoch << dendl;
+    db_thread_should_exit = false;
+    quiesce_db_thread.create("quiesce_db_mgr");
+  } else {
     submit_condition.notify_all();
-    lock.unlock();
-    ceph_assert(quiesce_db_thread.is_started());
-    dout(5) << "stopping the db mgr thread at epoch: " << new_membership.epoch << dendl;
-    quiesce_db_thread.join();
-  } else if (will_participate) {
-    if (!cluster_membership) {
-      // start the thread
-      dout(5) << "starting the db mgr thread at epoch: " << new_membership.epoch << dendl;
-      quiesce_db_thread.create("quiesce_db_mgr");
-    } else {
-      submit_condition.notify_all();
-    }
-    if (inject_request) {
-      pending_requests.push_front(inject_request);
-    }
+  }
+
+  if (inject_request) {
+    pending_requests.push_front(inject_request);
+  }
+
+  if (will_participate) {
     cluster_membership = new_membership;
-    
-    std::lock_guard lc(agent_mutex);
-    if (agent_callback) {
-        agent_callback->if_newer = {0, 0};
-    }
+  } else {
+    cluster_membership.reset();
   }
 
-  if (!will_participate && inject_request) {
-    inject_request->complete(-EPERM);
+  std::lock_guard lc(agent_mutex);
+  if (agent_callback) {
+      agent_callback->if_newer = {0, 0};
   }
 }
 
-bool QuiesceDbManager::membership_upkeep()
+std::pair<QuiesceDbManager::IsMemberBool, QuiesceDbManager::ShouldExitBool> QuiesceDbManager::membership_upkeep()
 {
   if (cluster_membership && cluster_membership->epoch == membership.epoch) {
     // no changes
-    return true;
+    return {true, db_thread_should_exit};
   }
 
   bool was_leader = membership.epoch > 0 && membership.leader == membership.me;
@@ -206,7 +202,7 @@ bool QuiesceDbManager::membership_upkeep()
       << std::boolalpha << was_leader << "->" << is_leader << std::noboolalpha
       << " members:" << cluster_membership->members << dendl;
   } else {
-    dout(10) << "shutdown! was_leader: " << was_leader << dendl;
+    dout(10) << "not a member! was_leader: " << was_leader << dendl;
   }
 
   if (is_leader) {
@@ -238,18 +234,24 @@ bool QuiesceDbManager::membership_upkeep()
       done_requests[await_ctx.req_ctx] = EINPROGRESS;
     }
     awaits.clear();
-    // reject pending requests
+    // reject pending requests as not leader
     while (!pending_requests.empty()) {
-      done_requests[pending_requests.front()] = EPERM;
+      done_requests[pending_requests.front()] = ENOTTY;
       pending_requests.pop_front();
     }
   }
 
   if (cluster_membership) {
     membership = *cluster_membership;
+    dout(15) << "Updated membership" << dendl;
+  } else {
+    membership.epoch = 0;
+    peers.clear();
+    awaits.clear();
+    db.clear();
   }
 
-  return cluster_membership.has_value();
+  return { cluster_membership.has_value(), db_thread_should_exit };
 }
 
 QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates)&& db_updates)
@@ -291,7 +293,7 @@ QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates
   if (db.set_version > update.db_version.set_version) {
     dout(3) << "got an older version of DB from the leader: " << db.set_version << " > " << update.db_version.set_version << dendl;
     dout(3) << "discarding the DB" << dendl;
-    db.reset();
+    db.clear();
   } else {
     for (auto& [qs_id, qs] : update.sets) {
       db.sets.insert_or_assign(qs_id, std::move(qs));
diff --git a/src/mds/QuiesceDbManager.h b/src/mds/QuiesceDbManager.h
index 08c8392d9815..98d0b84fc24b 100644
--- a/src/mds/QuiesceDbManager.h
+++ b/src/mds/QuiesceDbManager.h
@@ -49,7 +49,19 @@ class QuiesceDbManager {
     QuiesceDbManager() : quiesce_db_thread(this) {};
     virtual ~QuiesceDbManager()
     {
+      shutdown();
+    }
+
+    void shutdown() {
       update_membership({});
+
+      if (quiesce_db_thread.is_started()) {
+        submit_mutex.lock();
+        db_thread_should_exit = true;
+        submit_condition.notify_all();
+        submit_mutex.unlock();
+        quiesce_db_thread.join();
+      }
     }
 
     // This will reset the manager state
@@ -191,6 +203,7 @@ class QuiesceDbManager {
     std::queue<QuiesceDbPeerListing> pending_db_updates;
     std::queue<QuiesceDbPeerAck> pending_acks;
     std::deque<RequestContext*> pending_requests;
+    bool db_thread_should_exit = false;
 
     class QuiesceDbThread : public Thread {
       public:
@@ -220,7 +233,7 @@ class QuiesceDbManager {
       QuiesceTimeInterval get_age() const {
         return QuiesceClock::now() - time_zero;
       }
-      void reset() { 
+      void clear() { 
         set_version = 0; 
         sets.clear();
         time_zero = QuiesceClock::now();
@@ -257,23 +270,11 @@ class QuiesceDbManager {
     std::unordered_map<RequestContext*, int> done_requests;
 
     void* quiesce_db_thread_main();
-
-    void db_thread_enter() {
-      // this will invalidate the membership, see membership_upkeep()
-      membership.epoch = 0;
-      peers.clear();
-      awaits.clear();
-      done_requests.clear();
-      db.reset();
-    }
-
-    void db_thread_exit() {
-      complete_requests();
-    }
-
     bool db_thread_has_work() const;
 
-    bool membership_upkeep();
+    using IsMemberBool = bool;
+    using ShouldExitBool = bool;
+    std::pair<IsMemberBool, ShouldExitBool> membership_upkeep();
 
     QuiesceTimeInterval replica_upkeep(decltype(pending_db_updates)&& db_updates);
     bool leader_bootstrap(decltype(pending_db_updates)&& db_updates, QuiesceTimeInterval &next_event_at_age);
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
index f930f6c042a6..2ffba9778831 100644
--- a/src/test/mds/TestQuiesceDb.cc
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -230,7 +230,7 @@ class QuiesceDbTest: public testing::Test {
         response.clear();
         request.reset(c);
 
-        int rr = -1;
+        int rr = -ENOTTY;
 
         for (auto& [rank, mgr] : parent.managers) {
           if (!(rr = mgr->submit_request(this))) {
@@ -238,10 +238,8 @@ class QuiesceDbTest: public testing::Test {
           }
         }
 
-        if (rr == EPERM) {
-          // change the error to something never returned for a request
-          // EPIPE seems reasonable as we couldn't find the leader to send the command to
-          complete(EPIPE);
+        if (rr) {
+          complete(rr);
           return false;
         }
 
@@ -358,6 +356,7 @@ TEST_F(QuiesceDbTest, ManagerStartup) {
   ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
   ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(2) }));
   ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
+  managers[mds_gid_t(2)]->shutdown();
   ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2) }));
   ASSERT_EQ(OK(), run_request_for(100, [](auto& r) {}));
 }
@@ -1328,9 +1327,9 @@ TEST_F(QuiesceDbTest, LeaderShutdown)
 
   ASSERT_EQ(managers.at(mds_gid_t(1))->internal_pending_requests().size(), pending_requests.size());
 
-  // reset the membership of the manager
+  // shutdown the manager
   // this will block until the db thread exits
-  managers.at(mds_gid_t(1))->update_membership({});
+  managers.at(mds_gid_t(1))->shutdown();
 
   // as of now all requests must have finished
   while(!outstanding_awaits.empty()) {
@@ -1341,7 +1340,7 @@ TEST_F(QuiesceDbTest, LeaderShutdown)
 
   while (!pending_requests.empty()) {
     auto& r = *pending_requests.front();
-    EXPECT_EQ(ERR(EPERM), r.check_result());
+    EXPECT_EQ(ERR(ENOTTY), r.check_result());
     pending_requests.pop();
   }
 }

From a9662eee864296da4b9d90b2577ac29db41d0f5c Mon Sep 17 00:00:00 2001
From: Bernard Landon <bernard@lndn.ch>
Date: Fri, 1 Mar 2024 22:18:32 +0100
Subject: [PATCH 2183/2492] mgr/cephadm/services/ingress make HAProxy to listen
 on IPv4 and IPv6 by default

By default, HAProxy will only bind to IPv4 addresses preventing the ingress
service to be used in dual stack or mixed IPv4/IPv6 environments.
This patch changes the default so HAProxy would bind both on IPv4 and IPv6 addresses.

Signed-off-by: Bernard Landon <bernard@lndn.ch>
---
 src/pybind/mgr/cephadm/services/ingress.py    | 4 ++--
 src/pybind/mgr/cephadm/tests/test_services.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 55be3045466a..ad6729049122 100644
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -169,9 +169,9 @@ def haproxy_generate_config(
         if spec.enable_haproxy_protocol:
             server_opts.append("send-proxy-v2")
         logger.debug("enabled default server opts: %r", server_opts)
-        ip = '*' if spec.virtual_ips_list else str(spec.virtual_ip).split('/')[0] or daemon_spec.ip or '*'
+        ip = '[..]' if spec.virtual_ips_list else str(spec.virtual_ip).split('/')[0] or daemon_spec.ip or '[..]'
         frontend_port = daemon_spec.ports[0] if daemon_spec.ports else spec.frontend_port
-        if ip != '*' and frontend_port:
+        if ip != '[..]' and frontend_port:
             daemon_spec.port_ips = {str(frontend_port): ip}
         haproxy_conf = self.mgr.template.render(
             'services/ingress/haproxy.cfg.j2',
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index f0b5360e6e30..c716d7d182f0 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -2053,7 +2053,7 @@ def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOr
                                 'maxconn                 8000\n'
                                 '\nfrontend stats\n    '
                                 'mode http\n    '
-                                'bind *:8999\n    '
+                                'bind [..]:8999\n    '
                                 'bind 1.2.3.7:8999\n    '
                                 'stats enable\n    '
                                 'stats uri /stats\n    '
@@ -2062,7 +2062,7 @@ def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOr
                                 'http-request use-service prometheus-exporter if { path /metrics }\n    '
                                 'monitor-uri /health\n'
                                 '\nfrontend frontend\n    '
-                                'bind *:8089\n    '
+                                'bind [..]:8089\n    '
                                 'default_backend backend\n\n'
                                 'backend backend\n    '
                                 'option forwardfor\n    '

From 4ac0b7df3c15140bc9d904a1e12fac482c7130be Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 6 Mar 2024 14:04:46 -0500
Subject: [PATCH 2184/2492] qa/rgw: teach rgw.py to create an optional realm

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/tasks/rgw.py | 86 ++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 15 deletions(-)

diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py
index fef38e505f30..2e44c0bad53f 100644
--- a/qa/tasks/rgw.py
+++ b/qa/tasks/rgw.py
@@ -300,6 +300,43 @@ def assign_endpoints(ctx, config, default_cert):
 
     return role_endpoints
 
+@contextlib.contextmanager
+def create_realm(ctx, clients):
+    if ctx.rgw.realm:
+        log.info('Creating realm {}'.format(ctx.rgw.realm))
+
+        client = next(iter(clients))
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        cluster_name, daemon_type, client_id = teuthology.split_role(client)
+
+        # create the realm/zonegroup/zone and set as default
+        rgwadmin(ctx, client,
+                 cmd=['realm', 'create',
+                      '--rgw-realm', ctx.rgw.realm,
+                      '--default'],
+                 check_status=True)
+        rgwadmin(ctx, client,
+                 cmd=['zonegroup', 'create',
+                      '--rgw-realm', ctx.rgw.realm,
+                      '--rgw-zonegroup', ctx.rgw.zonegroup,
+                      '--master', '--default'],
+                 check_status=True)
+        rgwadmin(ctx, client,
+                 cmd=['zone', 'create',
+                      '--rgw-realm', ctx.rgw.realm,
+                      '--rgw-zonegroup', ctx.rgw.zonegroup,
+                      '--rgw-zone', ctx.rgw.zone,
+                      '--master', '--default'],
+                 check_status=True)
+
+        rgwadmin(ctx, client,
+                 cmd=['period', 'update', '--commit',
+                      '--rgw-realm', ctx.rgw.realm,
+                      '--rgw-zonegroup', ctx.rgw.zonegroup,
+                      '--rgw-zone', ctx.rgw.zone],
+                 check_status=True)
+    yield
+
 @contextlib.contextmanager
 def create_pools(ctx, clients):
     """Create replicated or erasure coded data pools for rgw."""
@@ -308,7 +345,7 @@ def create_pools(ctx, clients):
     for client in clients:
         log.debug("Obtaining remote for client {}".format(client))
         (remote,) = ctx.cluster.only(client).remotes.keys()
-        data_pool = 'default.rgw.buckets.data'
+        data_pool = '{}.rgw.buckets.data'.format(ctx.rgw.zone)
         cluster_name, daemon_type, client_id = teuthology.split_role(client)
 
         if ctx.rgw.ec_data_pool:
@@ -317,7 +354,7 @@ def create_pools(ctx, clients):
         else:
             create_replicated_pool(remote, data_pool, ctx.rgw.data_pool_pg_size, cluster_name, 'rgw')
 
-        index_pool = 'default.rgw.buckets.index'
+        index_pool = '{}.rgw.buckets.index'.format(ctx.rgw.zone)
         create_replicated_pool(remote, index_pool, ctx.rgw.index_pool_pg_size, cluster_name, 'rgw')
 
         if ctx.rgw.cache_pools:
@@ -331,12 +368,13 @@ def configure_compression(ctx, clients, compression):
     """ set a compression type in the default zone placement """
     log.info('Configuring compression type = %s', compression)
     for client in clients:
-        # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
-        # issue a 'radosgw-admin user list' command to trigger this
-        rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
+        if not ctx.rgw.realm:
+            # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
+            # issue a 'radosgw-admin user list' command to trigger this
+            rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
 
         rgwadmin(ctx, client,
-                cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
+                cmd=['zone', 'placement', 'modify', '--rgw-zone', ctx.rgw.zone,
                      '--placement-id', 'default-placement',
                      '--compression', compression],
                 check_status=True)
@@ -345,12 +383,13 @@ def configure_compression(ctx, clients, compression):
 @contextlib.contextmanager
 def disable_inline_data(ctx, clients):
     for client in clients:
-        # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
-        # issue a 'radosgw-admin user list' command to trigger this
-        rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
+        if not ctx.rgw.realm:
+            # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
+            # issue a 'radosgw-admin user list' command to trigger this
+            rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
 
         rgwadmin(ctx, client,
-                cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
+                cmd=['zone', 'placement', 'modify', '--rgw-zone', ctx.rgw.zone,
                      '--placement-id', 'default-placement',
                      '--placement-inline-data', 'false'],
                 check_status=True)
@@ -375,21 +414,22 @@ def configure_storage_classes(ctx, clients, storage_classes):
     sc = [s.strip() for s in storage_classes.split(',')]
 
     for client in clients:
-        # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
-        # issue a 'radosgw-admin user list' command to trigger this
-        rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
+        if not ctx.rgw.realm:
+            # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
+            # issue a 'radosgw-admin user list' command to trigger this
+            rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
 
         for storage_class in sc:
             log.info('Configuring storage class type = %s', storage_class)
             rgwadmin(ctx, client,
                     cmd=['zonegroup', 'placement', 'add',
-                        '--rgw-zone', 'default',
+                        '--rgw-zone', ctx.rgw.zone,
                         '--placement-id', 'default-placement',
                         '--storage-class', storage_class],
                     check_status=True)
             rgwadmin(ctx, client,
                     cmd=['zone', 'placement', 'add',
-                        '--rgw-zone', 'default',
+                        '--rgw-zone', ctx.rgw.zone,
                         '--placement-id', 'default-placement',
                         '--storage-class', storage_class,
                         '--data-pool', 'default.rgw.buckets.data.' + storage_class.lower()],
@@ -429,6 +469,15 @@ def task(ctx, config):
             client.3:
               valgrind: [--tool=memcheck]
 
+    To create a custom realm, zonegroup and zone:
+
+        tasks:
+        - ceph:
+        - rgw:
+            realm: MyRealm
+            zonegroup: MyZoneGroup
+            zone: MyZone
+
     To configure data or index pool pg_size:
 
         overrides:
@@ -463,6 +512,9 @@ def task(ctx, config):
     ctx.rgw.index_pool_pg_size = config.pop('index_pool_pg_size', 64)
     ctx.rgw.datacache = bool(config.pop('datacache', False))
     ctx.rgw.datacache_path = config.pop('datacache_path', None)
+    ctx.rgw.realm = config.pop('realm', None)
+    ctx.rgw.zonegroup = config.pop('zonegroup', 'default')
+    ctx.rgw.zone = config.pop('zone', 'default')
     ctx.rgw.config = config
 
     log.debug("config is {}".format(config))
@@ -473,6 +525,10 @@ def task(ctx, config):
     subtasks = [
         lambda: create_pools(ctx=ctx, clients=clients),
     ]
+    if ctx.rgw.realm:
+        subtasks.extend([
+            lambda: create_realm(ctx=ctx, clients=clients),
+        ])
     if ctx.rgw.compression_type:
         subtasks.extend([
             lambda: configure_compression(ctx=ctx, clients=clients,

From 6ec3fd9c62ffd4bf1c150b169089b94453622a91 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 6 Mar 2024 15:42:58 +0800
Subject: [PATCH 2185/2492] test/cls_rbd/test_cls_rbd: change to check for
 4KB-aligned sparse read

SeaStore::fiemap() can only supported 4096-aligned sparse read.

Co-authored-by: Ilya Dryomov <idryomov@gmail.com>
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/test/cls_rbd/test_cls_rbd.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc
index 2f553a0f4338..21311302711a 100644
--- a/src/test/cls_rbd/test_cls_rbd.cc
+++ b/src/test/cls_rbd/test_cls_rbd.cc
@@ -74,19 +74,19 @@ static bool is_sparse_read_supported(librados::IoCtx &ioctx,
                                      const std::string &oid) {
   EXPECT_EQ(0, ioctx.create(oid, true));
   bufferlist inbl;
-  inbl.append(std::string(1, 'X'));
-  EXPECT_EQ(0, ioctx.write(oid, inbl, inbl.length(), 1));
-  EXPECT_EQ(0, ioctx.write(oid, inbl, inbl.length(), 3));
+  inbl.append(std::string(4096, 'X'));
+  EXPECT_EQ(0, ioctx.write(oid, inbl, inbl.length(), 4096));
+  EXPECT_EQ(0, ioctx.write(oid, inbl, inbl.length(), 4096 * 3));
 
   std::map<uint64_t, uint64_t> m;
   bufferlist outbl;
-  int r = ioctx.sparse_read(oid, m, outbl, 4, 0);
+  int r = ioctx.sparse_read(oid, m, outbl, 4096 * 4, 0);
   ioctx.remove(oid);
 
   int expected_r = 2;
-  std::map<uint64_t, uint64_t> expected_m = {{1, 1}, {3, 1}};
+  std::map<uint64_t, uint64_t> expected_m = {{4096, 4096}, {4096 * 3, 4096}};
   bufferlist expected_outbl;
-  expected_outbl.append(std::string(2, 'X'));
+  expected_outbl.append(std::string(4096 * 2, 'X'));
 
   return (r == expected_r && m == expected_m &&
           outbl.contents_equal(expected_outbl));
@@ -205,7 +205,7 @@ TEST_F(TestClsRbd, sparse_copyup)
   // create some data to write
   inbl.append(std::string(4096, '1'));
   inbl.append(std::string(4096, '2'));
-  m = {{1024, 4096}, {8192, 4096}};
+  m = {{4096, 4096}, {4096 * 3, 4096}};
 
   // copyup to nonexistent object should create new object
   ioctx.remove(oid);
@@ -214,9 +214,9 @@ TEST_F(TestClsRbd, sparse_copyup)
   // and its contents should match
   bufferlist outbl;
   bufferlist expected_outbl;
-  expected_outbl.append(std::string(1024, '\0'));
+  expected_outbl.append(std::string(4096, '\0'));
   expected_outbl.append(std::string(4096, '1'));
-  expected_outbl.append(std::string(8192 - 4096 - 1024, '\0'));
+  expected_outbl.append(std::string(4096, '\0'));
   expected_outbl.append(std::string(4096, '2'));
   ASSERT_EQ((int)expected_outbl.length(),
             ioctx.read(oid, outbl, expected_outbl.length() + 1, 0));

From dee319e61204b2ee9ac13562c2c7075ef0f2ea4b Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 7 Mar 2024 13:01:47 +1000
Subject: [PATCH 2186/2492] doc/start: add Slack invite link

Add a link to the ceph-storage Slack invitation page. Previously the
link went to a plain old "this is the ceph-storage Slack" page that did
not direct the reader to sign up.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/start/get-involved.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/start/get-involved.rst b/doc/start/get-involved.rst
index b0c1a275ed11..4acc57b8c315 100644
--- a/doc/start/get-involved.rst
+++ b/doc/start/get-involved.rst
@@ -33,7 +33,7 @@ These are exciting times in the Ceph community! Get involved!
 |                      | ``#cephfs`` dedicated for Ceph developers.      |                                               |
 +----------------------+-------------------------------------------------+-----------------------------------------------+
 | **Slack**            | Since 2023, the Upstream Ceph community has     |                                               |
-|                      | maintained a community in Slack. Ask your Ceph- | https://ceph-storage.slack.com/               |
+|                      | maintained a community in Slack. Ask your Ceph- | - `Ceph Slack Invite`_                        |
 |                      | related questions in Ceph's Slack channels.     |                                               |
 +----------------------+-------------------------------------------------+-----------------------------------------------+
 | **User List**        | Ask and answer user-related questions by        |                                               |
@@ -100,4 +100,4 @@ These are exciting times in the Ceph community! Get involved!
 .. _Blog: http://ceph.com/community/blog/
 .. _Tracker: http://tracker.ceph.com/
 .. _Ceph Source Code: http://github.com/ceph/ceph
-
+.. _Ceph Slack Invite: https://ceph-storage.slack.com/join/shared_invite/zt-2b91em8b6-NQOKhGYReEIrE28OVncnLQ#/shared-invite/email

From 0558ef8b1a16060c0cd424635315db25f5c68351 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 13:42:51 +0800
Subject: [PATCH 2187/2492] crimson/os/seastore/cache: drop the unnecessary
 friend fixed_kv_btree

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/btree/fixed_kv_btree.h |  6 +++---
 src/crimson/os/seastore/cache.h                | 16 ++++++----------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h
index b91a4e2d07bf..2d758ef4be2d 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_btree.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h
@@ -579,12 +579,12 @@ class FixedKVBtree {
           if constexpr (
             std::is_base_of_v<typename internal_node_t::base_t, child_node_t>)
           {
-            assert(!c.cache.query_cache(i->get_val(), nullptr));
+            assert(!c.cache.test_query_cache(i->get_val()));
           } else {
             if constexpr (leaf_has_children) {
               assert(i->get_val().pladdr.is_paddr()
-                ? (bool)!c.cache.query_cache(
-                    i->get_val().pladdr.get_paddr(), nullptr)
+                ? (bool)!c.cache.test_query_cache(
+                    i->get_val().pladdr.get_paddr())
                 : true);
             }
           }
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 75463f465fe1..5093c79b81c1 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -600,6 +600,12 @@ class Cache {
     return epm.get_block_size();
   }
 
+// Interfaces only for tests.
+public:
+  CachedExtentRef test_query_cache(paddr_t offset) {
+    return query_cache(offset, nullptr);
+  }
+
 private:
   // This is a workaround std::move_only_function not being available,
   // not really worth generalizing at this time.
@@ -1732,16 +1738,6 @@ class Cache {
       return CachedExtentRef();
     }
   }
-
-  template <
-    typename node_key_t,
-    typename node_val_t,
-    typename internal_node_t,
-    typename leaf_node_t,
-    typename pin_t,
-    size_t node_size,
-    bool leaf_has_children>
-  friend class FixedKVBtree;
 };
 using CacheRef = std::unique_ptr<Cache>;
 

From 09c50063a3763fc5d29f604a9daeb844252a31da Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 7 Mar 2024 14:13:54 +0530
Subject: [PATCH 2188/2492] mgr/dashboard: disable applitools e2e

Temporarily disabling this so the CI could turn green. Meanwhile I'll
research for a proper way to handle the applitools e2es which I'll track
on https://tracker.ceph.com/issues/64783

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../frontend/cypress/e2e/visualTests/dashboard.vrt-spec.ts      | 2 +-
 .../frontend/cypress/e2e/visualTests/login.vrt-spec.ts          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/dashboard.vrt-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/dashboard.vrt-spec.ts
index 450cff871b53..13802982041d 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/dashboard.vrt-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/dashboard.vrt-spec.ts
@@ -1,6 +1,6 @@
 import { LoginPageHelper } from '../ui/login.po';
 
-describe('Dashboard Landing Page', () => {
+describe.skip('Dashboard Landing Page', () => {
   const login = new LoginPageHelper();
 
   beforeEach(() => {
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/login.vrt-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/login.vrt-spec.ts
index ea74f1d0f748..bc41c29c67f4 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/login.vrt-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/visualTests/login.vrt-spec.ts
@@ -1,4 +1,4 @@
-describe('Login Page', () => {
+describe.skip('Login Page', () => {
   beforeEach(() => {
     cy.visit('#/login');
     cy.eyesOpen({

From e8b9ab73f409d5d9a51100f5dd9922dfcab5dc0e Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Wed, 6 Mar 2024 13:49:43 +0800
Subject: [PATCH 2189/2492] crimson/os/seastore/collection_manager: allow
 empty-delta-buffer collection nodes

Fixes: https://tracker.ceph.com/issues/64728
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 .../collection_manager/collection_flat_node.h     | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/seastore/collection_manager/collection_flat_node.h b/src/crimson/os/seastore/collection_manager/collection_flat_node.h
index 2690fb5fdf46..aa1e71356130 100644
--- a/src/crimson/os/seastore/collection_manager/collection_flat_node.h
+++ b/src/crimson/os/seastore/collection_manager/collection_flat_node.h
@@ -153,10 +153,19 @@ struct CollectionNode
   }
 
   ceph::bufferlist get_delta() final {
-    assert(!delta_buffer.empty());
     ceph::bufferlist bl;
-    encode(delta_buffer, bl);
-    delta_buffer.clear();
+    // FIXME: CollectionNodes are always first mutated and
+    // 	      then checked whether they have enough space,
+    // 	      and if not, new ones will be created and the
+    // 	      mutation_pending ones are left untouched.
+    //
+    // 	      The above order should be reversed, nodes should
+    // 	      be mutated only if there are enough space for new
+    // 	      entries.
+    if (!delta_buffer.empty()) {
+      encode(delta_buffer, bl);
+      delta_buffer.clear();
+    }
     return bl;
   }
 

From 13a771931b9ce49d7d8cad8ea256a60cc4319bf2 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 7 Mar 2024 09:14:03 +0000
Subject: [PATCH 2190/2492] doc/dev/crimson: add Code Walkthroughs

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 doc/dev/crimson/crimson.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/dev/crimson/crimson.rst b/doc/dev/crimson/crimson.rst
index ea00ceebf56f..f447ce96d5ef 100644
--- a/doc/dev/crimson/crimson.rst
+++ b/doc/dev/crimson/crimson.rst
@@ -475,3 +475,10 @@ addresses in the backtrace::
   [root@3deb50a8ad51 ~]# dnf install -q -y file
   [root@3deb50a8ad51 ~]# python3 seastar-addr2line -e /usr/bin/crimson-osd
   # paste the backtrace here
+
+Code Walkthroughs
+=================
+
+* `Ceph Code Walkthroughs: Crimson <https://www.youtube.com/watch?v=rtkrHk6grsg>`_
+
+* `Ceph Code Walkthroughs: SeaStore <https://www.youtube.com/watch?v=0rr5oWDE2Ck>`_

From 785634262b6c39df44f3e88ab326421f75b06c98 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 15:30:39 +0800
Subject: [PATCH 2191/2492] crimson/os/seastore: remove unecessary public
 interfaces

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/cache.h         | 2 +-
 src/crimson/os/seastore/cached_extent.h | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 5093c79b81c1..86e63aa6c11e 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -1343,6 +1343,7 @@ class Cache {
     return stats.omap_tree_depth;
   }
 
+private:
   /// Update lru for access to ref
   void touch_extent(
       CachedExtent &ext,
@@ -1355,7 +1356,6 @@ class Cache {
     }
   }
 
-private:
   ExtentPlacementManager& epm;
   RootBlockRef root;               ///< ref to current root
   ExtentIndex extents;             ///< set of live extents
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 4cb7e2b490cb..35753d101bd2 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1244,8 +1244,6 @@ class LogicalCachedExtent : public ChildableCachedExtent {
 
   std::ostream &_print_detail(std::ostream &out) const final;
 
-  void on_replace_prior(Transaction &t) final;
-
   struct modified_region_t {
     extent_len_t offset;
     extent_len_t len;
@@ -1257,9 +1255,12 @@ class LogicalCachedExtent : public ChildableCachedExtent {
   virtual void clear_modified_region() {}
 
   virtual ~LogicalCachedExtent();
+
 protected:
+  void on_replace_prior(Transaction &t) final;
 
   virtual void apply_delta(const ceph::bufferlist &bl) = 0;
+
   virtual std::ostream &print_detail_l(std::ostream &out) const {
     return out;
   }

From 05c916ba46b9edeb0551e1629373c64c91bcdd5f Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 15:34:06 +0800
Subject: [PATCH 2192/2492] crimson/os/cache: cleanup get_extent(_by_type)

Distinguish caching vs absent get_extent interfaces, and misc related
cleanups.

After the lba parent-child pointer optimization, the absent path should
be used.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/cache.cc              |  27 +-
 src/crimson/os/seastore/cache.h               | 301 +++++++-----------
 .../crimson/seastore/test_seastore_cache.cc   |   2 +-
 3 files changed, 138 insertions(+), 192 deletions(-)

diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc
index 0e5eeb96c17e..880967031544 100644
--- a/src/crimson/os/seastore/cache.cc
+++ b/src/crimson/os/seastore/cache.cc
@@ -1850,7 +1850,7 @@ Cache::replay_delta(
     };
     auto extent_fut = (delta.pversion == 0 ?
       // replay is not included by the cache hit metrics
-      _get_extent_by_type(
+      do_get_caching_extent_by_type(
         delta.type,
         delta.paddr,
         delta.laddr,
@@ -2012,7 +2012,8 @@ Cache::get_root_ret Cache::get_root(Transaction &t)
   }
 }
 
-Cache::get_extent_ertr::future<CachedExtentRef> Cache::_get_extent_by_type(
+Cache::get_extent_ertr::future<CachedExtentRef>
+Cache::do_get_caching_extent_by_type(
   extent_types_t type,
   paddr_t offset,
   laddr_t laddr,
@@ -2034,55 +2035,55 @@ Cache::get_extent_ertr::future<CachedExtentRef> Cache::_get_extent_by_type(
       ceph_assert(0 == "ROOT is never directly read");
       return get_extent_ertr::make_ready_future<CachedExtentRef>();
     case extent_types_t::BACKREF_INTERNAL:
-      return get_extent<backref::BackrefInternalNode>(
+      return do_get_caching_extent<backref::BackrefInternalNode>(
 	offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::BACKREF_LEAF:
-      return get_extent<backref::BackrefLeafNode>(
+      return do_get_caching_extent<backref::BackrefLeafNode>(
 	offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::LADDR_INTERNAL:
-      return get_extent<lba_manager::btree::LBAInternalNode>(
+      return do_get_caching_extent<lba_manager::btree::LBAInternalNode>(
 	offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::LADDR_LEAF:
-      return get_extent<lba_manager::btree::LBALeafNode>(
+      return do_get_caching_extent<lba_manager::btree::LBALeafNode>(
 	offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::OMAP_INNER:
-      return get_extent<omap_manager::OMapInnerNode>(
+      return do_get_caching_extent<omap_manager::OMapInnerNode>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
         return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::OMAP_LEAF:
-      return get_extent<omap_manager::OMapLeafNode>(
+      return do_get_caching_extent<omap_manager::OMapLeafNode>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
         return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::COLL_BLOCK:
-      return get_extent<collection_manager::CollectionNode>(
+      return do_get_caching_extent<collection_manager::CollectionNode>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
         return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::ONODE_BLOCK_STAGED:
-      return get_extent<onode::SeastoreNodeExtent>(
+      return do_get_caching_extent<onode::SeastoreNodeExtent>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::OBJECT_DATA_BLOCK:
-      return get_extent<ObjectDataBlock>(
+      return do_get_caching_extent<ObjectDataBlock>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
@@ -2091,13 +2092,13 @@ Cache::get_extent_ertr::future<CachedExtentRef> Cache::_get_extent_by_type(
       ceph_assert(0 == "impossible");
       return get_extent_ertr::make_ready_future<CachedExtentRef>();
     case extent_types_t::TEST_BLOCK:
-      return get_extent<TestBlock>(
+      return do_get_caching_extent<TestBlock>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
       });
     case extent_types_t::TEST_BLOCK_PHYSICAL:
-      return get_extent<TestBlockPhysical>(
+      return do_get_caching_extent<TestBlockPhysical>(
           offset, length, p_metric_key, std::move(extent_init_func), std::move(on_cache)
       ).safe_then([](auto extent) {
 	return CachedExtentRef(extent.detach(), false /* add_ref */);
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 86e63aa6c11e..082241d408fa 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -267,109 +267,6 @@ class Cache {
     return t.root;
   }
 
-  /**
-   * get_extent
-   *
-   * returns ref to extent at offset~length of type T either from
-   * - extent_set if already in cache
-   * - disk
-   */
-  using src_ext_t = std::pair<Transaction::src_t, extent_types_t>;
-  using get_extent_ertr = base_ertr;
-  template <typename T>
-  using get_extent_ret = get_extent_ertr::future<TCachedExtentRef<T>>;
-  template <typename T, typename Func, typename OnCache>
-  get_extent_ret<T> get_extent(
-    paddr_t offset,                ///< [in] starting addr
-    extent_len_t length,           ///< [in] length
-    const src_ext_t* p_src_ext,    ///< [in] cache query metric key
-    Func &&extent_init_func,       ///< [in] init func for extent
-    OnCache &&on_cache
-  ) {
-    LOG_PREFIX(Cache::get_extent);
-    auto cached = query_cache(offset, p_src_ext);
-    if (!cached) {
-      auto ret = CachedExtent::make_cached_extent_ref<T>(
-        alloc_cache_buf(length));
-      ret->init(CachedExtent::extent_state_t::CLEAN_PENDING,
-                offset,
-                PLACEMENT_HINT_NULL,
-                NULL_GENERATION,
-		TRANS_ID_NULL);
-      SUBDEBUG(seastore_cache,
-          "{} {}~{} is absent, add extent and reading ... -- {}",
-          T::TYPE, offset, length, *ret);
-      const auto p_src = p_src_ext ? &p_src_ext->first : nullptr;
-      add_extent(ret, p_src);
-      on_cache(*ret);
-      extent_init_func(*ret);
-      return read_extent<T>(
-	std::move(ret));
-    }
-
-    // extent PRESENT in cache
-    if (cached->get_type() == extent_types_t::RETIRED_PLACEHOLDER) {
-      auto ret = CachedExtent::make_cached_extent_ref<T>(
-        alloc_cache_buf(length));
-      ret->init(CachedExtent::extent_state_t::CLEAN_PENDING,
-                offset,
-                PLACEMENT_HINT_NULL,
-                NULL_GENERATION,
-		TRANS_ID_NULL);
-      SUBDEBUG(seastore_cache,
-          "{} {}~{} is absent(placeholder), reading ... -- {}",
-          T::TYPE, offset, length, *ret);
-      extents.replace(*ret, *cached);
-      on_cache(*ret);
-
-      // replace placeholder in transactions
-      while (!cached->transactions.empty()) {
-        auto t = cached->transactions.begin()->t;
-        t->replace_placeholder(*cached, *ret);
-      }
-
-      cached->state = CachedExtent::extent_state_t::INVALID;
-      extent_init_func(*ret);
-      return read_extent<T>(
-	std::move(ret));
-    } else if (!cached->is_fully_loaded()) {
-      auto ret = TCachedExtentRef<T>(static_cast<T*>(cached.get()));
-      on_cache(*ret);
-      SUBDEBUG(seastore_cache,
-        "{} {}~{} is present without been fully loaded, reading ... -- {}",
-        T::TYPE, offset, length, *ret);
-      auto bp = alloc_cache_buf(length);
-      ret->set_bptr(std::move(bp));
-      return read_extent<T>(
-        std::move(ret));
-    } else {
-      SUBTRACE(seastore_cache,
-          "{} {}~{} is present in cache -- {}",
-          T::TYPE, offset, length, *cached);
-      auto ret = TCachedExtentRef<T>(static_cast<T*>(cached.get()));
-      on_cache(*ret);
-      return ret->wait_io(
-      ).then([ret=std::move(ret)]() mutable
-	     -> get_extent_ret<T> {
-        // ret may be invalid, caller must check
-        return get_extent_ret<T>(
-          get_extent_ertr::ready_future_marker{},
-          std::move(ret));
-      });
-    }
-  }
-  template <typename T>
-  get_extent_ret<T> get_extent(
-    paddr_t offset,                ///< [in] starting addr
-    extent_len_t length,           ///< [in] length
-    const src_ext_t* p_metric_key  ///< [in] cache query metric key
-  ) {
-    return get_extent<T>(
-      offset, length, p_metric_key,
-      [](T &){}, [](T &) {});
-  }
-
-
   /**
    * get_extent_if_cached
    *
@@ -435,7 +332,7 @@ class Cache {
   }
 
   /**
-   * get_extent
+   * get_caching_extent
    *
    * returns ref to extent at offset~length of type T either from
    * - t if modified by t
@@ -443,16 +340,19 @@ class Cache {
    * - disk
    *
    * t *must not* have retired offset
+   *
+   * Note, the current implementation leverages parent-child
+   * pointers in LBA instead, so it should only be called in tests.
    */
   using get_extent_iertr = base_iertr;
-  template <typename T, typename Func>
-  get_extent_iertr::future<TCachedExtentRef<T>> get_extent(
+  template <typename T>
+  get_extent_iertr::future<TCachedExtentRef<T>>
+  get_caching_extent(
     Transaction &t,
     paddr_t offset,
-    extent_len_t length,
-    Func &&extent_init_func) {
+    extent_len_t length) {
     CachedExtentRef ret;
-    LOG_PREFIX(Cache::get_extent);
+    LOG_PREFIX(Cache::get_caching_extent);
     auto result = t.get_extent(offset, &ret);
     if (result == Transaction::get_extent_ret::RETIRED) {
       SUBERRORT(seastore_cache, "{} {}~{} is retired on t -- {}",
@@ -485,9 +385,9 @@ class Cache {
       };
       auto metric_key = std::make_pair(t.get_src(), T::TYPE);
       return trans_intr::make_interruptible(
-        get_extent<T>(
-	  offset, length, &metric_key,
-	  std::forward<Func>(extent_init_func), std::move(f))
+        do_get_caching_extent<T>(
+          offset, length, &metric_key,
+          [](T &){}, std::move(f))
       );
     }
   }
@@ -495,9 +395,7 @@ class Cache {
   /*
    * get_absent_extent
    *
-   * Mostly the same as Cache::get_extent(), with the only difference
-   * that get_absent_extent won't search the transaction's context for
-   * the specific CachedExtent
+   * The extent in query is supposed to be absent in Cache.
    */
   template <typename T, typename Func>
   get_extent_iertr::future<TCachedExtentRef<T>> get_absent_extent(
@@ -524,20 +422,12 @@ class Cache {
     };
     auto metric_key = std::make_pair(t.get_src(), T::TYPE);
     return trans_intr::make_interruptible(
-      get_extent<T>(
+      do_get_caching_extent<T>(
 	offset, length, &metric_key,
 	std::forward<Func>(extent_init_func), std::move(f))
     );
   }
 
-  template <typename T>
-  get_extent_iertr::future<TCachedExtentRef<T>> get_extent(
-    Transaction &t,
-    paddr_t offset,
-    extent_len_t length) {
-    return get_extent<T>(t, offset, length, [](T &){});
-  }
-
   /*
    * get_absent_extent
    *
@@ -553,7 +443,9 @@ class Cache {
     return get_absent_extent<T>(t, offset, length, [](T &){});
   }
 
-  get_extent_ertr::future<CachedExtentRef> get_extent_viewable_by_trans(
+  using get_extent_ertr = base_ertr;
+  get_extent_ertr::future<CachedExtentRef>
+  get_extent_viewable_by_trans(
     Transaction &t,
     CachedExtentRef extent)
   {
@@ -586,7 +478,10 @@ class Cache {
   }
 
   template <typename T>
-  get_extent_ertr::future<TCachedExtentRef<T>> get_extent_viewable_by_trans(
+  using read_extent_ret = get_extent_ertr::future<TCachedExtentRef<T>>;
+
+  template <typename T>
+  read_extent_ret<T> get_extent_viewable_by_trans(
     Transaction &t,
     TCachedExtentRef<T> extent)
   {
@@ -607,6 +502,96 @@ class Cache {
   }
 
 private:
+  /**
+   * do_get_caching_extent
+   *
+   * returns ref to extent at offset~length of type T either from
+   * - extent_set if already in cache
+   * - disk
+   */
+  using src_ext_t = std::pair<Transaction::src_t, extent_types_t>;
+  template <typename T, typename Func, typename OnCache>
+  read_extent_ret<T> do_get_caching_extent(
+    paddr_t offset,                ///< [in] starting addr
+    extent_len_t length,           ///< [in] length
+    const src_ext_t* p_src_ext,    ///< [in] cache query metric key
+    Func &&extent_init_func,       ///< [in] init func for extent
+    OnCache &&on_cache
+  ) {
+    LOG_PREFIX(Cache::do_get_caching_extent);
+    auto cached = query_cache(offset, p_src_ext);
+    if (!cached) {
+      auto ret = CachedExtent::make_cached_extent_ref<T>(
+        alloc_cache_buf(length));
+      ret->init(CachedExtent::extent_state_t::CLEAN_PENDING,
+                offset,
+                PLACEMENT_HINT_NULL,
+                NULL_GENERATION,
+		TRANS_ID_NULL);
+      SUBDEBUG(seastore_cache,
+          "{} {}~{} is absent, add extent and reading ... -- {}",
+          T::TYPE, offset, length, *ret);
+      const auto p_src = p_src_ext ? &p_src_ext->first : nullptr;
+      add_extent(ret, p_src);
+      on_cache(*ret);
+      extent_init_func(*ret);
+      return read_extent<T>(
+	std::move(ret));
+    }
+
+    // extent PRESENT in cache
+    if (cached->get_type() == extent_types_t::RETIRED_PLACEHOLDER) {
+      auto ret = CachedExtent::make_cached_extent_ref<T>(
+        alloc_cache_buf(length));
+      ret->init(CachedExtent::extent_state_t::CLEAN_PENDING,
+                offset,
+                PLACEMENT_HINT_NULL,
+                NULL_GENERATION,
+		TRANS_ID_NULL);
+      SUBDEBUG(seastore_cache,
+          "{} {}~{} is absent(placeholder), reading ... -- {}",
+          T::TYPE, offset, length, *ret);
+      extents.replace(*ret, *cached);
+      on_cache(*ret);
+
+      // replace placeholder in transactions
+      while (!cached->transactions.empty()) {
+        auto t = cached->transactions.begin()->t;
+        t->replace_placeholder(*cached, *ret);
+      }
+
+      cached->state = CachedExtent::extent_state_t::INVALID;
+      extent_init_func(*ret);
+      return read_extent<T>(
+	std::move(ret));
+    } else if (!cached->is_fully_loaded()) {
+      auto ret = TCachedExtentRef<T>(static_cast<T*>(cached.get()));
+      on_cache(*ret);
+      SUBDEBUG(seastore_cache,
+        "{} {}~{} is present without been fully loaded, reading ... -- {}",
+        T::TYPE, offset, length, *ret);
+      auto bp = alloc_cache_buf(length);
+      ret->set_bptr(std::move(bp));
+      return read_extent<T>(
+        std::move(ret));
+    } else {
+      SUBTRACE(seastore_cache,
+          "{} {}~{} is present in cache -- {}",
+          T::TYPE, offset, length, *cached);
+      auto ret = TCachedExtentRef<T>(static_cast<T*>(cached.get()));
+      on_cache(*ret);
+      return ret->wait_io(
+      ).then([ret=std::move(ret)]() mutable
+	     -> read_extent_ret<T> {
+        // ret may be invalid, caller must check
+        return read_extent_ret<T>(
+          get_extent_ertr::ready_future_marker{},
+          std::move(ret));
+      });
+    }
+  }
+
+
   // This is a workaround std::move_only_function not being available,
   // not really worth generalizing at this time.
   class extent_init_func_t {
@@ -633,7 +618,9 @@ class Cache {
       return (*wrapped)(extent);
     }
   };
-  get_extent_ertr::future<CachedExtentRef> _get_extent_by_type(
+
+  get_extent_ertr::future<CachedExtentRef>
+  do_get_caching_extent_by_type(
     extent_types_t type,
     paddr_t offset,
     laddr_t laddr,
@@ -646,7 +633,7 @@ class Cache {
   using get_extent_by_type_iertr = get_extent_iertr;
   using get_extent_by_type_ret = get_extent_by_type_iertr::future<
     CachedExtentRef>;
-  get_extent_by_type_ret _get_extent_by_type(
+  get_extent_by_type_ret get_caching_extent_by_type(
     Transaction &t,
     extent_types_t type,
     paddr_t offset,
@@ -654,7 +641,7 @@ class Cache {
     extent_len_t length,
     extent_init_func_t &&extent_init_func
   ) {
-    LOG_PREFIX(Cache::get_extent_by_type);
+    LOG_PREFIX(Cache::get_caching_extent_by_type);
     CachedExtentRef ret;
     auto status = t.get_extent(offset, &ret);
     if (status == Transaction::get_extent_ret::RETIRED) {
@@ -687,7 +674,7 @@ class Cache {
       };
       auto src = t.get_src();
       return trans_intr::make_interruptible(
-	_get_extent_by_type(
+	do_get_caching_extent_by_type(
 	  type, offset, laddr, length, &src,
 	  std::move(extent_init_func), std::move(f))
       );
@@ -721,7 +708,7 @@ class Cache {
     };
     auto src = t.get_src();
     return trans_intr::make_interruptible(
-      _get_extent_by_type(
+      do_get_caching_extent_by_type(
 	type, offset, laddr, length, &src,
 	std::move(extent_init_func), std::move(f))
     );
@@ -768,36 +755,13 @@ class Cache {
   }
 
 public:
-  /**
-   * get_extent_by_type
+  /*
+   * get_absent_extent_by_type
    *
    * Based on type, instantiate the correct concrete type
    * and read in the extent at location offset~length.
-   */
-  template <typename Func>
-  get_extent_by_type_ret get_extent_by_type(
-    Transaction &t,         ///< [in] transaction
-    extent_types_t type,    ///< [in] type tag
-    paddr_t offset,         ///< [in] starting addr
-    laddr_t laddr,          ///< [in] logical address if logical
-    extent_len_t length,    ///< [in] length
-    Func &&extent_init_func ///< [in] extent init func
-  ) {
-    return _get_extent_by_type(
-      t,
-      type,
-      offset,
-      laddr,
-      length,
-      extent_init_func_t(std::forward<Func>(extent_init_func)));
-  }
-
-  /*
-   * get_absent_extent_by_type
    *
-   * Mostly the same as Cache::get_extent_by_type(), with the only difference
-   * that get_absent_extent_by_type won't search the transaction's context for
-   * the specific CachedExtent
+   * The extent in query is supposed to be absent in Cache.
    */
   template <typename Func>
   get_extent_by_type_ret get_absent_extent_by_type(
@@ -817,25 +781,6 @@ class Cache {
       extent_init_func_t(std::forward<Func>(extent_init_func)));
   }
 
-  get_extent_by_type_ret get_extent_by_type(
-    Transaction &t,
-    extent_types_t type,
-    paddr_t offset,
-    laddr_t laddr,
-    extent_len_t length
-  ) {
-    return get_extent_by_type(
-      t, type, offset, laddr, length, [](CachedExtent &) {});
-  }
-
-
-  /*
-   * get_absent_extent_by_type
-   *
-   * Mostly the same as Cache::get_extent_by_type(), with the only difference
-   * that get_absent_extent_by_type won't search the transaction's context for
-   * the specific CachedExtent
-   */
   get_extent_by_type_ret get_absent_extent_by_type(
     Transaction &t,
     extent_types_t type,
@@ -1678,7 +1623,7 @@ class Cache {
   void on_transaction_destruct(Transaction& t);
 
   template <typename T>
-  get_extent_ret<T> read_extent(
+  read_extent_ret<T> read_extent(
     TCachedExtentRef<T>&& extent
   ) {
     assert(extent->state == CachedExtent::extent_state_t::CLEAN_PENDING ||
@@ -1712,7 +1657,7 @@ class Cache {
       },
       get_extent_ertr::pass_further{},
       crimson::ct_error::assert_all{
-        "Cache::get_extent: invalid error"
+        "Cache::read_extent: invalid error"
       }
     );
   }
diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc
index 66c9899538c8..2b0b546b159a 100644
--- a/src/test/crimson/seastore/test_seastore_cache.cc
+++ b/src/test/crimson/seastore/test_seastore_cache.cc
@@ -76,7 +76,7 @@ struct cache_test_t : public seastar_test_suite_t {
     return with_trans_intr(
       t,
       [this](auto &&... args) {
-	return cache->get_extent<T>(args...);
+	return cache->get_caching_extent<T>(args...);
       },
       std::forward<Args>(args)...);
   }

From 1c13850f24dbb90c33a12c6da338956c2e83811b Mon Sep 17 00:00:00 2001
From: Yuval Lifshitz <ylifshit@redhat.com>
Date: Tue, 5 Mar 2024 10:14:06 +0000
Subject: [PATCH 2193/2492] rgw/kafka: set message timeout to 5 seconds

also increase the idle timeout to 30 seconds.
test instructions:
https://gist.github.com/yuvalif/33487bff19883e3409caa8a843a0b353

Fixes: https://tracker.ceph.com/issues/64710

Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
---
 src/common/options/rgw.yaml.in | 12 +++++++++++-
 src/rgw/rgw_kafka.cc           |  9 ++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 6fab43e5589f..a7af43ae05f5 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3983,7 +3983,7 @@ options:
     are sent to it for more than the time defined.
     Note that the connection will not be considered idle, even if it is down,
     as long as there are attempts to send messages to it.
-  default: 30
+  default: 300
   services:
   - rgw
   with_legacy: true
@@ -3999,3 +3999,13 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_kafka_message_timeout
+  type: uint 
+  level: advanced
+  desc: This is the maximum time in milliseconds to deliver a message (including retries)
+  long_desc: Delivery error occurs when the message timeout is exceeded.
+    Value must be greater than zero, if set to zero, a value of 1 millisecond will be used.
+  default: 5000
+  services:
+  - rgw
+  with_legacy: true
diff --git a/src/rgw/rgw_kafka.cc b/src/rgw/rgw_kafka.cc
index 0d6e77375014..3b5b1c7011b0 100644
--- a/src/rgw/rgw_kafka.cc
+++ b/src/rgw/rgw_kafka.cc
@@ -210,9 +210,16 @@ bool new_producer(connection_t* conn) {
     return false;
   }
 
+  // set message timeout
+  // according to documentation, value of zero will expire the message based on retries.
+  // however, testing with librdkafka v1.6.1 did not expire the message in that case. hence, a value of zero is changed to 1ms
+  constexpr std::uint64_t min_message_timeout = 1;
+  const auto message_timeout = std::max(min_message_timeout, conn->cct->_conf->rgw_kafka_message_timeout);
+  if (rd_kafka_conf_set(conn->temp_conf, "message.timeout.ms", 
+        std::to_string(message_timeout).c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
   // get list of brokers based on the bootstrap broker
   if (rd_kafka_conf_set(conn->temp_conf, "bootstrap.servers", conn->broker.c_str(), errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) goto conf_error;
-
+  
   if (conn->use_ssl) {
     if (!conn->user.empty()) {
       // use SSL+SASL

From d16480c36449ac118cdacf58d8cd0d9fe74050ac Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Wed, 28 Feb 2024 17:34:37 +0530
Subject: [PATCH 2194/2492] tools/cephfs: Fix first-damage to include missing
 dirs

Don't skip root and lost+found directories while
checking for damaged metadata.

Fixes: https://tracker.ceph.com/issues/64615
Signed-off-by: Kotresh HR <khiremat@redhat.com>
---
 src/tools/cephfs/first-damage.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/tools/cephfs/first-damage.py b/src/tools/cephfs/first-damage.py
index 0479dc8cb2a6..23dc50329769 100644
--- a/src/tools/cephfs/first-damage.py
+++ b/src/tools/cephfs/first-damage.py
@@ -60,6 +60,8 @@
 REPAIR_NOSNAP = None
 
 CEPH_NOSNAP = 0xfffffffe # int32 -2
+ROOT_INODE  = "1.00000000"
+LOST_FOUND_INODE  = "4.00000000"
 
 DIR_PATTERN = re.compile(r'[0-9a-fA-F]{8,}\.[0-9a-fA-F]+')
 
@@ -67,7 +69,7 @@
 
 def traverse(MEMO, ioctx):
     for o in ioctx.list_objects():
-        if not DIR_PATTERN.fullmatch(o.key):
+        if not DIR_PATTERN.fullmatch(o.key) and o.key not in [ROOT_INODE, LOST_FOUND_INODE]:
             log.debug("skipping %s", o.key)
             continue
         elif o.key in CACHE:

From 5e7a78cd900891611fd5463523d8bb22b62831db Mon Sep 17 00:00:00 2001
From: Marc Singer <marc@singer.services>
Date: Thu, 7 Mar 2024 18:46:21 +0100
Subject: [PATCH 2195/2492] rgw: allow user disabling presigned urls in rgw
 configuration Fixes: https://tracker.ceph.com/issues/64797

Signed-off-by: Marc Singer <marc@singer.services>
---
 src/common/options/rgw.yaml.in |  8 ++++++++
 src/rgw/rgw_auth.cc            |  7 ++++++-
 src/rgw/rgw_common.h           |  1 +
 src/rgw/rgw_rest_s3.cc         | 17 +++++++++++------
 4 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 6fab43e5589f..ffed94c95ebd 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -892,6 +892,14 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_s3_auth_disable_signature_url
+  type: bool
+  level: advanced
+  desc: Should authentification with presigned URLs be disabled
+  long_desc: 'If enabled, any request that is presigned with either V2 or V4 signature will be denied'
+  default: false
+  services:
+  - rgw
 - name: rgw_barbican_url
   type: str
   level: advanced
diff --git a/src/rgw/rgw_auth.cc b/src/rgw/rgw_auth.cc
index 4b3f33e9c70e..5d98933063c8 100644
--- a/src/rgw/rgw_auth.cc
+++ b/src/rgw/rgw_auth.cc
@@ -299,11 +299,16 @@ rgw::auth::Strategy::apply(const DoutPrefixProvider *dpp, const rgw::auth::Strat
        * nullptr inside. */
       ldpp_dout(dpp, 5) << "Failed the auth strategy, reason="
                        << result.get_reason() << dendl;
-      //Special handling for expired pre-signed URL
+      // Special handling for expired pre-signed URL
       if (result.get_reason() == ERR_PRESIGNED_URL_EXPIRED) {
         result = result_t::deny(-EPERM);
         set_req_state_err(s, -EPERM, "The pre-signed URL has expired");
       }
+      // Special handling for disabled presigned URL
+      if (result.get_reason() == ERR_PRESIGNED_URL_DISABLED) {
+        result = result_t::deny(-EPERM);
+        set_req_state_err(s, -EPERM, "Presigned URLs are disabled by admin");
+      }
       return result.get_reason();
     }
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index a1b93b26dfcd..1898d544dc08 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -308,6 +308,7 @@ static inline const char* to_mime_type(const RGWFormat f)
 #define ERR_INVALID_BUCKET_STATE                         2221
 #define ERR_INVALID_OBJECT_STATE			 2222
 #define ERR_PRESIGNED_URL_EXPIRED			 2223
+#define ERR_PRESIGNED_URL_DISABLED     2224
 
 #define ERR_BUSY_RESHARDING      2300
 #define ERR_NO_SUCH_ENTITY       2301
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index bb43029f7766..42fe289a0eb5 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5562,13 +5562,18 @@ AWSGeneralAbstractor::get_auth_data(const req_state* const s) const
   AwsRoute route;
   std::tie(version, route) = discover_aws_flavour(s->info);
 
-  if (version == AwsVersion::V2) {
-    return get_auth_data_v2(s);
-  } else if (version == AwsVersion::V4) {
-    return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
+  if (! s->cct->_conf->rgw_s3_auth_disable_signature_url) {
+    if (version == AwsVersion::V2) {
+      return get_auth_data_v2(s);
+    } else if (version == AwsVersion::V4) {
+      return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
+    } else {
+      /* FIXME(rzarzynski): handle anon user. */
+      throw -EINVAL;
+    }
   } else {
-    /* FIXME(rzarzynski): handle anon user. */
-    throw -EINVAL;
+    ldpp_dout(s, 0) << "Presigned URLs are disabled by admin" << dendl;
+    throw -ERR_PRESIGNED_URL_DISABLED;
   }
 }
 

From b6df68210850a93196d21b926509df53721e17dc Mon Sep 17 00:00:00 2001
From: Marc Singer <marc@singer.services>
Date: Thu, 7 Mar 2024 19:19:11 +0100
Subject: [PATCH 2196/2492] fix: comments from code review

Signed-off-by: Marc Singer <marc@singer.services>
---
 src/common/options/rgw.yaml.in |  3 ++-
 src/rgw/rgw_rest_s3.cc         | 23 ++++++++++++-----------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index ffed94c95ebd..fbcea7052beb 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -895,11 +895,12 @@ options:
 - name: rgw_s3_auth_disable_signature_url
   type: bool
   level: advanced
-  desc: Should authentification with presigned URLs be disabled
+  desc: Should authentication with presigned URLs be disabled
   long_desc: 'If enabled, any request that is presigned with either V2 or V4 signature will be denied'
   default: false
   services:
   - rgw
+  with_legacy: true
 - name: rgw_barbican_url
   type: str
   level: advanced
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 42fe289a0eb5..ed5091b302f9 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -5562,19 +5562,20 @@ AWSGeneralAbstractor::get_auth_data(const req_state* const s) const
   AwsRoute route;
   std::tie(version, route) = discover_aws_flavour(s->info);
 
-  if (! s->cct->_conf->rgw_s3_auth_disable_signature_url) {
-    if (version == AwsVersion::V2) {
-      return get_auth_data_v2(s);
-    } else if (version == AwsVersion::V4) {
-      return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
-    } else {
-      /* FIXME(rzarzynski): handle anon user. */
-      throw -EINVAL;
-    }
-  } else {
-    ldpp_dout(s, 0) << "Presigned URLs are disabled by admin" << dendl;
+  if (s->cct->_conf->rgw_s3_auth_disable_signature_url) {
+    ldpp_dout(s, 10) << "Presigned URLs are disabled by admin" << dendl;
     throw -ERR_PRESIGNED_URL_DISABLED;
   }
+  
+  if (version == AwsVersion::V2) {
+    return get_auth_data_v2(s);
+  } else if (version == AwsVersion::V4) {
+    return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
+  } else {
+    /* FIXME(rzarzynski): handle anon user. */
+    throw -EINVAL;
+  }
+
 }
 
 boost::optional<std::string>

From 7ef94cdb1c97f537561a7c8bb226d9d22ff709a8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Thu, 7 Mar 2024 18:07:38 -0500
Subject: [PATCH 2197/2492] qa/rgw: notification suite creates a realm

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/notifications/overrides.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/qa/suites/rgw/notifications/overrides.yaml b/qa/suites/rgw/notifications/overrides.yaml
index 26de9381d126..461a17e5bf1c 100644
--- a/qa/suites/rgw/notifications/overrides.yaml
+++ b/qa/suites/rgw/notifications/overrides.yaml
@@ -9,6 +9,5 @@ overrides:
         rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
         rgw crypt require ssl: false
   rgw:
-    storage classes: LUKEWARM, FROZEN
-  s3tests:
+    realm: default
     storage classes: LUKEWARM, FROZEN

From eaaf72253123de1a66f163f651046817faa97a1a Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 8 Mar 2024 03:29:50 +1000
Subject: [PATCH 2198/2492] doc/rados: restore PGcalc tool

Restore the PGcalc tool to the documentation suite.

Co-authored-by: Josh Durgin <jdurgin@redhat.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/_static/js/pgcalc.js                  | 357 ++++++++++++++++++++++
 doc/rados/operations/index.rst            |   1 +
 doc/rados/operations/pgcalc/index.rst     |  68 +++++
 doc/rados/operations/placement-groups.rst |   2 +
 4 files changed, 428 insertions(+)
 create mode 100644 doc/_static/js/pgcalc.js
 create mode 100644 doc/rados/operations/pgcalc/index.rst

diff --git a/doc/_static/js/pgcalc.js b/doc/_static/js/pgcalc.js
new file mode 100644
index 000000000000..e13c30895fcf
--- /dev/null
+++ b/doc/_static/js/pgcalc.js
@@ -0,0 +1,357 @@
+var _____WB$wombat$assign$function_____ = function(name) {return (self._wb_wombat && self._wb_wombat.local_init && self._wb_wombat.local_init(name)) || self[name]; };
+if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source = obj; return this; } }
+{
+  let window = _____WB$wombat$assign$function_____("window");
+  let self = _____WB$wombat$assign$function_____("self");
+  let document = _____WB$wombat$assign$function_____("document");
+  let location = _____WB$wombat$assign$function_____("location");
+  let top = _____WB$wombat$assign$function_____("top");
+  let parent = _____WB$wombat$assign$function_____("parent");
+  let frames = _____WB$wombat$assign$function_____("frames");
+  let opener = _____WB$wombat$assign$function_____("opener");
+
+var pow2belowThreshold = 0.25
+var key_values={};
+key_values['poolName']		={'name':'Pool Name','default':'newPool','description': 'Name of the pool in question.  Typical pool names are included below.', 'width':'30%; text-align: left'};
+key_values['size']		={'name':'Size','default': 3, 'description': 'Number of replicas the pool will have. Default value of 3 is pre-filled.', 'width':'10%', 'global':1};
+key_values['osdNum']		={'name':'OSD #','default': 100, 'description': 'Number of OSDs which this Pool will have PGs in. Typically, this is the entire Cluster OSD count, but could be less based on CRUSH rules. (e.g. Separate SSD and SATA disk sets)', 'width':'10%', 'global':1};
+key_values['percData']		={'name':'%Data', 'default': 5, 'description': 'This value represents the approximate percentage of data which will be contained in this pool for that specific OSD set. Examples are pre-filled below for guidance.','width':'10%'};
+key_values['targPGsPerOSD']	={'name':'Target PGs per OSD', 'default':100, 'description': 'This value should be populated based on the following guidance:', 'width':'10%', 'global':1, 'options': [ ['100','If the cluster OSD count is not expected to increase in the foreseeable future.'], ['200', 'If the cluster OSD count is expected to increase (up to double the size) in the foreseeable future.']]}
+
+var notes ={
+	'totalPerc':'<b>"Total Data Percentage"</b> below table should be a multiple of 100%.',
+	'totalPGs':'<b>"Total PG Count"</b> below table will be the count of Primary PG copies. However, when calculating total PGs per OSD average, you must include all copies.',
+	'noDecrease':'It\'s also important to know that the PG count can be increased, but <b>NEVER</b> decreased without destroying / recreating the pool. However, increasing the PG Count of a pool is one of the most impactful events in a Ceph Cluster, and should be avoided for production clusters if possible.',
+};
+
+var presetTables={};
+presetTables['All-in-One']=[
+	{ 'poolName' : 'rbd', 'size' : '3', 'osdNum' : '100', 'percData' : '100', 'targPGsPerOSD' : '100'},
+];
+presetTables['OpenStack']=[
+	{ 'poolName' : 'cinder-backup', 'size' : '3', 'osdNum' : '100', 'percData' : '25', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'cinder-volumes', 'size' : '3', 'osdNum' : '100', 'percData' : '53', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'ephemeral-vms', 'size' : '3', 'osdNum' : '100', 'percData' : '15', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'glance-images', 'size' : '3', 'osdNum' : '100', 'percData' : '7', 'targPGsPerOSD' : '100'},
+];
+presetTables['OpenStack w RGW - Jewel and later']=[
+        { 'poolName' : '.rgw.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.control', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.data.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.gc', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.intent-log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.meta', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.usage', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.keys', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.email', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.swift', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.uid', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.extra', 'size' : '3', 'osdNum' : '100', 'percData' : '1.0', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.index', 'size' : '3', 'osdNum' : '100', 'percData' : '3.0', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.data', 'size' : '3', 'osdNum' : '100', 'percData' : '19', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'cinder-backup', 'size' : '3', 'osdNum' : '100', 'percData' : '18', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'cinder-volumes', 'size' : '3', 'osdNum' : '100', 'percData' : '42.8', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'ephemeral-vms', 'size' : '3', 'osdNum' : '100', 'percData' : '10', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'glance-images', 'size' : '3', 'osdNum' : '100', 'percData' : '5', 'targPGsPerOSD' : '100'},
+];
+
+presetTables['Rados Gateway Only - Jewel and later']=[
+        { 'poolName' : '.rgw.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.control', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.data.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.gc', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.intent-log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.meta', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.usage', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.keys', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.email', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.swift', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.users.uid', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.extra', 'size' : '3', 'osdNum' : '100', 'percData' : '1.0', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.index', 'size' : '3', 'osdNum' : '100', 'percData' : '3.0', 'targPGsPerOSD' : '100'},
+        { 'poolName' : 'default.rgw.buckets.data', 'size' : '3', 'osdNum' : '100', 'percData' : '94.8', 'targPGsPerOSD' : '100'},
+];
+
+presetTables['OpenStack w RGW - Infernalis and earlier']=[
+	{ 'poolName' : '.intent-log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets', 'size' : '3', 'osdNum' : '100', 'percData' : '18', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets.extra', 'size' : '3', 'osdNum' : '100', 'percData' : '1.0', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets.index', 'size' : '3', 'osdNum' : '100', 'percData' : '3.0', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.control', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.gc', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.usage', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.email', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.swift', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.uid', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'cinder-backup', 'size' : '3', 'osdNum' : '100', 'percData' : '19', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'cinder-volumes', 'size' : '3', 'osdNum' : '100', 'percData' : '42.9', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'ephemeral-vms', 'size' : '3', 'osdNum' : '100', 'percData' : '10', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'glance-images', 'size' : '3', 'osdNum' : '100', 'percData' : '5', 'targPGsPerOSD' : '100'},
+];
+
+presetTables['Rados Gateway Only - Infernalis and earlier']=[
+	{ 'poolName' : '.intent-log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.log', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets', 'size' : '3', 'osdNum' : '100', 'percData' : '94.9', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets.extra', 'size' : '3', 'osdNum' : '100', 'percData' : '1.0', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.buckets.index', 'size' : '3', 'osdNum' : '100', 'percData' : '3.0', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.control', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.gc', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.rgw.root', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.usage', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.email', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.swift', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : '.users.uid', 'size' : '3', 'osdNum' : '100', 'percData' : '0.1', 'targPGsPerOSD' : '100'},
+];
+presetTables['RBD and libRados']=[
+	{ 'poolName' : 'rbd', 'size' : '3', 'osdNum' : '100', 'percData' : '75', 'targPGsPerOSD' : '100'},
+	{ 'poolName' : 'myObjects', 'size' : '3', 'osdNum' : '100', 'percData' : '25', 'targPGsPerOSD' : '100'},
+];
+
+$(function() {
+	$("#presetType").on("change",changePreset);
+	$("#btnAddPool").on("click",addPool);
+	$("#btnGenCommands").on("click",generateCommands);
+	$.each(presetTables,function(index,value) {
+		selIndex='';
+		if ( index == 'OpenStack w RGW - Jewel and later' )
+			selIndex=' selected';
+		$("#presetType").append("<option value=\""+index+"\""+selIndex+">"+index+"</option>");
+	});
+	changePreset();
+	$("#beforeTable").html("<fieldset id='keyFieldset'><legend>Key</legend><dl class='table-display' id='keyDL'></dl></fieldset>");
+	$.each(key_values, function(index, value) {
+		pre='';
+		post='';
+		if ('global' in value) {
+			pre='<a href="javascript://" onClick="globalChange(\''+index+'\');" title="Change the \''+value['name']+'\' parameter globally">';
+			post='</a>'
+		}
+
+		var dlAdd="<dt id='dt_"+index+"'>"+pre+value['name']+post+"</dt><dd id='dd_"+index+"'>"+value['description'];
+		if ( 'options' in value ) {
+			dlAdd+="<dl class='sub-table'>";
+			$.each(value['options'], function (subIndex, subValue) {
+				dlAdd+="<dt><a href=\"javascript://\" onClick=\"massUpdate('"+index+"','"+subValue[0]+"');\" title=\"Set all '"+value['name']+"' fields to '"+subValue[0]+"'.\">"+subValue[0]+"</a></dt><dd>"+subValue[1]+"</dd>";
+			});
+			dlAdd+="</dl>";
+		}
+		dlAdd+="</dd>";
+		$("#keyDL").append(dlAdd);
+	});
+	$("#afterTable").html("<fieldset id='notesFieldset'><legend>Notes</legend><ul id='notesUL'>\n<ul></fieldset>");
+	$.each(notes,function(index, value) {
+		$("#notesUL").append("\t<li id=\"li_"+index+"\">"+value+"</li>\n");
+	});
+
+});
+
+function changePreset() {
+	resetTable();
+	fillTable($("#presetType").val());
+}
+
+function resetTable() {
+	$("#pgsperpool").html("");
+	$("#pgsperpool").append("<tr id='headerRow'>\n</tr>\n");
+	$("#headerRow").append("\t<th>&nbsp;</th>\n");
+	var fieldCount=0;
+	var percDataIndex=0;
+	$.each(key_values, function(index, value) {
+		fieldCount++;
+		pre='';
+		post='';
+		var widthAdd='';
+		if ( index == 'percData' )
+			percDataIndex=fieldCount;
+		if ('width' in value)
+			widthAdd=' style=\'width: '+value['width']+'\'';
+		if ('global' in value) {
+			pre='<a href="javascript://" onClick="globalChange(\''+index+'\');" title="Change the \''+value['name']+'\' parameter globally">';
+			post='</a>'
+		}
+		$("#headerRow").append("\t<th"+widthAdd+">"+pre+value['name']+post+"</th>\n");
+	});
+	percDataIndex++;
+	$("#headerRow").append("\t<th class='center'>Suggested PG Count</th>\n");
+	$("#pgsperpool").append("<tr id='totalRow'><td colspan='"+percDataIndex+"' id='percTotal' style='text-align: right; margin-right: 10px;'><strong>Total Data Percentage:</strong> <span id='percTotalValue'>0</span>%</td><td>&nbsp;</td><td id='pgTotal' class='bold pgcount' style='text-align: right;'>PG Total Count: <span id='pgTotalValue'>0</span></td></tr>");
+}
+
+function nearestPow2( aSize ){
+	var tmp=Math.pow(2, Math.round(Math.log(aSize)/Math.log(2)));
+	if(tmp<(aSize*(1-pow2belowThreshold)))
+		tmp*=2;
+	return tmp;
+}
+
+function globalChange(field) {
+	dialogHTML='<div title="Change \''+key_values[field]['name']+'\' Globally"><form>';
+	dialogHTML+='<label for="value">New '+key_values[field]['name']+' value:</label><br />\n';
+	dialogHTML+='<input type="text" name="globalValue" id="globalValue" value="'+$("#row0_"+field+"_input").val()+'" style="text-align: right;"/>';
+	dialogHTML+='<input type="hidden" name="globalField" id="globalField" value="'+field+'"/>';
+	dialogHTML+='<input type="submit" tabindex="-1" style="position:absolute; top:-1000px">';
+	dialogHTML+='</form>';
+	globalDialog=$(dialogHTML).dialog({
+		autoOpen: true,
+		width: 350,
+		show: 'fold',
+		hide: 'fold',
+		modal: true,
+		buttons: {
+			"Update Value": function() { massUpdate($("#globalField").val(),$("#globalValue").val()); globalDialog.dialog("close"); setTimeout(function() { globalDialog.dialog("destroy"); }, 1000); },
+			"Cancel": function() { globalDialog.dialog("close"); setTimeout(function() { globalDialog.dialog("destroy"); }, 1000); }
+		}
+	});
+}
+
+var rowCount=0;
+function fillTable(presetType) {
+	rowCount=0;
+	$.each(presetTables[presetType], function(index,value) {
+		addTableRow(value);
+	});
+}
+
+function addPool() {
+	dialogHTML='<div title="Add Pool"><form>';
+	$.each(key_values, function(index,value) {
+		dialogHTML+='<br /><label for="new'+index+'">'+value['name']+':</label><br />\n';
+		classAdd='right';
+		if ( index == 'poolName' )
+			classAdd='left';
+		dialogHTML+='<input type="text" name="new'+index+'" id="new'+index+'" value="'+value['default']+'" class="'+classAdd+'"/><br />';
+	});
+	dialogHTML+='<input type="submit" tabindex="-1" style="position:absolute; top:-1000px">';
+	dialogHTML+='</form>';
+	addPoolDialog=$(dialogHTML).dialog({
+		autoOpen: true,
+		width: 350,
+		show: 'fold',
+		hide: 'fold',
+		modal: true,
+		buttons: {
+			"Add Pool": function() {
+				var newPoolValues={};
+				$.each(key_values,function(index,value) {
+					newPoolValues[index]=$("#new"+index).val();
+				});
+				addTableRow(newPoolValues);
+				addPoolDialog.dialog("close");
+				setTimeout(function() { addPoolDialog.dialog("destroy"); }, 1000); },
+			"Cancel": function() { addPoolDialog.dialog("close"); setTimeout(function() { addPoolDialog.dialog("destroy"); }, 1000); }
+		}
+	});
+
+//		addTableRow({'poolName':'newPool','size':3, 'osdNum':100,'targPGsPerOSD': 100, 'percData':0});
+}
+
+function addTableRow(rowValues) {
+	rowAdd="<tr id='row"+rowCount+"'>\n";
+	rowAdd+="\t<td width='15px' class='inputColor'><a href='javascript://' title='Remove Pool' onClick='$(\"#row"+rowCount+"\").remove();updateTotals();'><span class='ui-icon ui-icon-trash'></span></a></td>\n";
+	$.each(key_values, function(index,value) {
+		classAdd=' center';
+		modifier='';
+		if ( index == 'percData' ) {
+			classAdd='" style="text-align: right;';
+	//		modifier=' %';
+		} else if ( index == 'poolName' )
+			classAdd=' left';
+		rowAdd+="\t<td id=\"row"+rowCount+"_"+index+"\"><input type=\"text\" class=\"inputColor "+index+classAdd+"\" id=\"row"+rowCount+"_"+index+"_input\" value=\""+rowValues[index]+"\" onFocus=\"focusMe("+rowCount+",'"+index+"');\" onKeyUp=\"keyMe("+rowCount+",'"+index+"');\" onBlur=\"blurMe("+rowCount+",'"+index+"');\">"+modifier+"</td>\n";
+	});
+	rowAdd+="\t<td id=\"row"+rowCount+"_pgCount\" class='pgcount' style='text-align: right;'>0</td></tr>";
+	$("#totalRow").before(rowAdd);
+	updatePGCount(rowCount);
+	$("[id$='percData_input']").each(function() { var fieldVal=parseFloat($(this).val()); $(this).val(fieldVal.toFixed(2)); });
+	rowCount++;
+}
+
+function updatePGCount(rowID) {
+	if(rowID==-1) {
+		for(var i=0;i<rowCount;i++) {
+			updatePGCount(i);
+		}
+	} else {
+		minValue=nearestPow2(Math.floor($("#row"+rowID+"_osdNum_input").val()/$("#row"+rowID+"_size_input").val())+1);
+		if(minValue<$("#row"+rowID+"_osdNum_input").val())
+			minValue*=2;
+		calcValue=nearestPow2(Math.floor(($("#row"+rowID+"_targPGsPerOSD_input").val()*$("#row"+rowID+"_osdNum_input").val()*$("#row"+rowID+"_percData_input").val())/(100*$("#row"+rowID+"_size_input").val())));
+		if(minValue>calcValue)
+			$("#row"+rowID+"_pgCount").html(minValue);
+		else
+			$("#row"+rowID+"_pgCount").html(calcValue);
+	}
+	updateTotals();
+}
+
+function focusMe(rowID,field) {
+	$("#row"+rowID+"_"+field+"_input").toggleClass('inputColor');
+	$("#row"+rowID+"_"+field+"_input").toggleClass('highlightColor');
+	$("#dt_"+field).toggleClass('highlightColor');
+	$("#dd_"+field).toggleClass('highlightColor');
+	updatePGCount(rowID);
+}
+
+function blurMe(rowID,field) {
+	focusMe(rowID,field);
+	$("[id$='percData_input']").each(function() { var fieldVal=parseFloat($(this).val()); $(this).val(fieldVal.toFixed(2)); });
+}
+
+function keyMe(rowID,field) {
+	updatePGCount(rowID);
+}
+
+function massUpdate(field,value) {
+	$("[id$='_"+field+"_input']").val(value);
+	key_values[field]['default']=value;
+	updatePGCount(-1);
+}
+
+function updateTotals() {
+	var totalPerc=0;
+	var totalPGs=0;
+	$("[id$='percData_input']").each(function() {
+		totalPerc+=parseFloat($(this).val());
+		if ( parseFloat($(this).val()) > 100 ) 
+			$(this).addClass('ui-state-error');
+		else
+			$(this).removeClass('ui-state-error');
+	});
+	$("[id$='_pgCount']").each(function() { 
+		totalPGs+=parseInt($(this).html()); 
+	});
+	$("#percTotalValue").html(totalPerc.toFixed(2));
+	$("#pgTotalValue").html(totalPGs);
+	if(parseFloat(totalPerc.toFixed(2)) % 100 != 0) {
+		$("#percTotalValue").addClass('ui-state-error');
+		$("#li_totalPerc").addClass('ui-state-error');
+	} else {
+		$("#percTotalValue").removeClass('ui-state-error');
+		$("#li_totalPerc").removeClass('ui-state-error');
+	}
+	$("#commandCode").html("");
+}
+
+function generateCommands() {
+	outputCommands="## Note: The 'while' loops below pause between pools to allow all\n\
+##       PGs to be created.  This is a safety mechanism to prevent\n\
+##       saturating the Monitor nodes.\n\
+## -------------------------------------------------------------------\n\n";
+	for(i=0;i<rowCount;i++) {
+		console.log(i);
+		outputCommands+="ceph osd pool create "+$("#row"+i+"_poolName_input").val()+" "+$("#row"+i+"_pgCount").html()+"\n";
+		outputCommands+="ceph osd pool set "+$("#row"+i+"_poolName_input").val()+" size "+$("#row"+i+"_size_input").val()+"\n";
+		outputCommands+="while [ $(ceph -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done\n\n";
+	}
+	window.location.href = "data:application/download," + encodeURIComponent(outputCommands);
+}
+
+
+}
diff --git a/doc/rados/operations/index.rst b/doc/rados/operations/index.rst
index 15525c1d35e3..91301382da46 100644
--- a/doc/rados/operations/index.rst
+++ b/doc/rados/operations/index.rst
@@ -21,6 +21,7 @@ and, monitoring an operating cluster.
 	monitoring-osd-pg
 	user-management
 	pg-repair
+        pgcalc/index
 
 .. raw:: html
 
diff --git a/doc/rados/operations/pgcalc/index.rst b/doc/rados/operations/pgcalc/index.rst
new file mode 100644
index 000000000000..1aed873914a0
--- /dev/null
+++ b/doc/rados/operations/pgcalc/index.rst
@@ -0,0 +1,68 @@
+.. _pgcalc:
+
+  
+=======
+PG Calc
+=======
+
+
+.. raw:: html
+
+
+   <link rel="stylesheet" id="wp-job-manager-job-listings-css" href="https://web.archive.org/web/20230614135557cs_/https://old.ceph.com/wp-content/plugins/wp-job-manager/assets/dist/css/job-listings.css" type="text/css" media="all"/>
+   <link rel="stylesheet" id="ceph/googlefont-css" href="https://web.archive.org/web/20230614135557cs_/https://fonts.googleapis.com/css?family=Raleway%3A300%2C400%2C700&amp;ver=5.7.2" type="text/css" media="all"/>
+   <link rel="stylesheet" id="Stylesheet-css" href="https://web.archive.org/web/20230614135557cs_/https://old.ceph.com/wp-content/themes/cephTheme/Resources/Styles/style.min.css" type="text/css" media="all"/>
+   <link rel="stylesheet" id="tablepress-default-css" href="https://web.archive.org/web/20230614135557cs_/https://old.ceph.com/wp-content/plugins/tablepress/css/default.min.css" type="text/css" media="all"/>
+   <link rel="stylesheet" id="jetpack_css-css" href="https://web.archive.org/web/20230614135557cs_/https://old.ceph.com/wp-content/plugins/jetpack/css/jetpack.css" type="text/css" media="all"/>
+   <script type="text/javascript" src="https://web.archive.org/web/20230614135557js_/https://old.ceph.com/wp-content/themes/cephTheme/foundation_framework/js/vendor/jquery.js" id="jquery-js"></script>
+
+   <link rel="stylesheet" href="https://web.archive.org/web/20230614135557cs_/https://ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/themes/smoothness/jquery-ui.css"/>
+   <link rel="stylesheet" href="https://web.archive.org/web/20230614135557cs_/https://old.ceph.com/pgcalc_assets/pgcalc.css"/>
+   <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/jquery-ui.min.js"></script>
+
+        <script src="../../../_static/js/pgcalc.js"></script>
+        	<div id="pgcalcdiv">
+                <div id="instructions">
+                <h2>Ceph PGs per Pool Calculator</h2><br/><fieldset><legend>Instructions</legend>
+                <ol>
+                        <li>Confirm your understanding of the fields by reading through the Key below.</li>
+                        <li>Select a <b>"Ceph Use Case"</b> from the drop down menu.</li>
+                        <li>Adjust the values in the <span class="inputColor addBorder" style="font-weight: bold;">"Green"</span> shaded fields below.<br/>
+                                <b>Tip:</b> Headers can be clicked to change the value throughout the table.</li>
+                        <li>You will see the Suggested PG Count update based on your inputs.</li>
+                        <li>Click the <b>"Add Pool"</b> button to create a new line for a new pool.</li>
+                        <li>Click the <span class="ui-icon ui-icon-trash" style="display:inline-block;"></span> icon to delete the specific Pool.</li>
+                        <li>For more details on the logic used and some important details, see the area below the table.</li>
+                        <li>Once all values have been adjusted, click the <b>"Generate Commands"</b> button to get the pool creation commands.</li>
+                </ol></fieldset>
+                </div>
+                <div id="beforeTable"></div>
+                <br/>
+                <p class="validateTips">&nbsp;</p>
+                <label for="presetType">Ceph Use Case Selector:</label><br/><select id="presetType"></select><button style="margin-left: 200px;" id="btnAddPool" type="button">Add Pool</button><button type="button" id="btnGenCommands" download="commands.txt">Generate Commands</button>
+                <div id="pgsPerPoolTable">
+                        <table id="pgsperpool">
+                        </table>
+                </div> <!-- id = pgsPerPoolTable -->
+                <br/>
+                <div id="afterTable"></div>
+                <div id="countLogic"><fieldset><legend>Logic behind Suggested PG Count</legend>
+                        <br/>
+                        <div class="upperFormula">( Target PGs per OSD ) x ( OSD # ) x ( %Data )</div>
+                        <div class="lowerFormula">( Size )</div>
+                        <ol id="countLogicList">
+                                <li>If the value of the above calculation is less than the value of <b>( OSD# ) / ( Size )</b>, then the value is updated to the value of <b>( OSD# ) / ( Size )</b>.  This is to ensure even load / data distribution by allocating at least one Primary or Secondary PG to every OSD for every Pool.</li>
+                                <li>The output value is then rounded to the <b>nearest power of 2</b>.<br/><b>Tip:</b> The nearest power of 2 provides a marginal improvement in efficiency of the <a href="https://web.archive.org/web/20230614135557/http://ceph.com/docs/master/rados/operations/crush-map/" title="CRUSH Map Details">CRUSH</a> algorithm.</li>
+                                <li>If the nearest power of 2 is more than <b>25%</b> below the original value, the next higher power of 2 is used.</li>
+                        </ol>
+                        <b>Objective</b>
+                        <ul><li>The objective of this calculation and the target ranges noted in the &quot;Key&quot; section above are to ensure that there are sufficient Placement Groups for even data distribution throughout the cluster, while not going high enough on the PG per OSD ratio to cause problems during Recovery and/or Backfill operations.</li></ul>
+                        <b>Effects of enpty or non-active pools:</b>
+                        <ul>
+                                <li>Empty or otherwise non-active pools should not be considered helpful toward even data distribution throughout the cluster.</li>
+                                <li>However, the PGs associated with these empty / non-active pools still consume memory and CPU overhead.</li>
+                        </ul>
+                </fieldset>
+                </div>
+                <div id="commands" title="Pool Creation Commands"><code><pre id="commandCode"></pre></code></div>
+                </div>
diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index 84dad884e22b..3875f7cc8145 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -647,6 +647,8 @@ more time for peering.
 Setting the Number of PGs
 =========================
 
+:ref:`Placement Group Link <pgcalc>`
+
 Setting the initial number of PGs in a pool must be done at the time you create
 the pool. See `Create a Pool`_ for details. 
 

From 014d4468fce5aadfbc63bd9651c0c66db0bf7f49 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Fri, 1 Mar 2024 12:56:25 +0530
Subject: [PATCH 2199/2492] mgr/dashboard: Locking improvements in bucket
 create form

Fixes https://tracker.ceph.com/issues/64658
- Addition of help texts
- Addition of info/warnings related to modes and versioning
- change of Locking section layout
- renaming locking to 'Object Locking'
- changes default retention period to 10
- edit bucket only shows lock when its enabled

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../rgw-bucket-form.component.html            | 87 +++++++++++++------
 .../rgw-bucket-form.component.spec.ts         | 12 ++-
 .../rgw-bucket-form.component.ts              |  2 +-
 .../shared/components/components.module.ts    |  7 +-
 .../help-text/help-text.component.html        |  3 +
 .../help-text/help-text.component.scss        |  3 +
 .../help-text/help-text.component.spec.ts     | 22 +++++
 .../help-text/help-text.component.ts          |  8 ++
 8 files changed, 115 insertions(+), 29 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.ts

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index fd085a74d2d0..41b7f1162977 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -213,30 +213,34 @@
           </div>
         </fieldset>
 
-        <!-- Locking -->
-        <fieldset>
+        <!-- Object Locking -->
+        <fieldset *ngIf="!editing || (editing && bucketForm.getValue('lock_enabled'))">
           <legend class="cd-header"
-                  i18n>Locking</legend>
-
-          <!-- Locking enabled -->
+                  i18n>
+            Object Locking
+            <cd-help-text class="bc-legend-help">
+                Store objects using a write-once-read-many (WORM) model to help you prevent objects from being deleted or overwritten for a fixed amount of time or indefinitely.
+                Object Locking works only in versioned buckets.
+            </cd-help-text>
+          </legend>
+          <!-- Object Locking enable -->
           <div class="form-group row">
-            <div class="cd-col-form-offset">
-              <div class="custom-control custom-checkbox">
-                <input class="custom-control-input"
-                       id="lock_enabled"
-                       formControlName="lock_enabled"
-                       type="checkbox">
-                <label class="custom-control-label"
-                       for="lock_enabled"
-                       i18n>Enabled</label>
-                <cd-helper>
-                  <span i18n>Enables locking for the objects in the bucket. Locking can only be enabled while creating a bucket.</span>
-                </cd-helper>
-              </div>
+            <label class="cd-col-form-label pt-0"
+                   for="lock_enabled"
+                   i18n>
+                    Enable
+            </label>
+            <div class="cd-col-form-input">
+              <input class="form-check-input"
+                     id="lock_enabled"
+                     formControlName="lock_enabled"
+                     type="checkbox"/>
+              <cd-help-text>
+                <span i18n>Enables locking for the objects in the bucket. Locking can only be enabled while creating a bucket.</span>
+              </cd-help-text>
             </div>
           </div>
-
-          <!-- Locking mode -->
+          <!-- Object Locking mode -->
           <div *ngIf="bucketForm.getValue('lock_enabled')"
                class="form-group row">
             <label class="cd-col-form-label"
@@ -248,27 +252,42 @@
                       name="lock_mode"
                       id="lock_mode">
                 <option i18n
-                        value="COMPLIANCE">Compliance</option>
+                        value="COMPLIANCE" >
+                  Compliance
+                </option>
                 <option i18n
-                        value="GOVERNANCE">Governance</option>
+                        value="GOVERNANCE">
+                  Governance
+                </option>
               </select>
+              <cd-help-text>
+                <span *ngIf="bucketForm.getValue('lock_mode') === 'COMPLIANCE'"
+                      i18n>
+                  In COMPLIANCE an object version cannot be overwritten or deleted for the duration of the period.
+                </span>
+                <span *ngIf="bucketForm.getValue('lock_mode') === 'GOVERNANCE'"
+                      i18n>
+                  In GOVERNANCE mode, users cannot overwrite or delete an object version or alter its lock settings unless they have special permissions.
+                </span>
+              </cd-help-text>
             </div>
           </div>
-
           <!-- Retention period (days) -->
           <div *ngIf="bucketForm.getValue('lock_enabled')"
                class="form-group row">
             <label class="cd-col-form-label"
                    for="lock_retention_period_days">
               <ng-container i18n>Days</ng-container>
-              <cd-helper i18n>The number of days that you want to specify for the default retention period that will be applied to new objects placed in this bucket.</cd-helper>
             </label>
             <div class="cd-col-form-input">
               <input class="form-control"
                      type="number"
                      id="lock_retention_period_days"
                      formControlName="lock_retention_period_days"
-                     min="0">
+                     min="1">
+              <cd-help-text>
+                <span i18n>The number of days that you want to specify for the default retention period that will be applied to new objects placed in this bucket.</span>
+              </cd-help-text>
               <span class="invalid-feedback"
                     *ngIf="bucketForm.showError('lock_retention_period_days', frm, 'pattern')"
                     i18n>The entered value must be a positive integer.</span>
@@ -277,6 +296,24 @@
                     i18n>Retention Days must be a positive integer.</span>
             </div>
           </div>
+          <!-- Alerts -->
+          <div class="form-group row">
+            <div class="cd-col-form-label"></div>
+            <div class="cd-col-form-input">
+              <cd-alert-panel
+                type="info"
+                *ngIf="bucketForm.getValue('lock_enabled')"
+                class="me-1"
+                i18n-title>
+                  Bucket Versioning can't be disabled when Object Locking is enabled.
+              </cd-alert-panel>
+              <cd-alert-panel
+                type="warning"
+                *ngIf="bucketForm.getValue('lock_enabled')">
+                  Enabling Object Locking will allow the configuration of GOVERNANCE or COMPLIANCE modes, which will help ensure that an object version cannot be overwritten or deleted for the specified period.
+              </cd-alert-panel>
+            </div>
+          </div>
         </fieldset>
 
         <fieldset>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts
index 704d7918465d..44318eda88e1 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.spec.ts
@@ -272,10 +272,20 @@ describe('RgwBucketFormComponent', () => {
       expect(control.disabled).toBeTruthy();
     });
 
-    it('should have the "lockDays" error', () => {
+    it('should not have the "lockDays" error for 10 days', () => {
       formHelper.setValue('lock_enabled', true);
       const control = component.bucketForm.get('lock_retention_period_days');
       control.updateValueAndValidity();
+      expect(control.value).toBe(10);
+      expect(control.invalid).toBeFalsy();
+      formHelper.expectValid(control);
+    });
+
+    it('should have the "lockDays" error for 0 days', () => {
+      formHelper.setValue('lock_enabled', true);
+      formHelper.setValue('lock_retention_period_days', 0);
+      const control = component.bucketForm.get('lock_retention_period_days');
+      control.updateValueAndValidity();
       expect(control.value).toBe(0);
       expect(control.invalid).toBeTruthy();
       formHelper.expectError(control, 'lockDays');
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index 3b3a50e88f82..6857874411ae 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -150,7 +150,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
         ]
       ],
       lock_mode: ['COMPLIANCE'],
-      lock_retention_period_days: [0, [CdValidators.number(false), lockDaysValidator]],
+      lock_retention_period_days: [10, [CdValidators.number(false), lockDaysValidator]],
       bucket_policy: ['{}', CdValidators.json()],
       grantee: [Grantee.Owner, [Validators.required]],
       aclPermission: [[aclPermission.FullControl], [Validators.required]]
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index 867ef3b16d87..caaebb89a6a5 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -54,6 +54,7 @@ import { CardRowComponent } from './card-row/card-row.component';
 import { CodeBlockComponent } from './code-block/code-block.component';
 import { VerticalNavigationComponent } from './vertical-navigation/vertical-navigation.component';
 import { CardGroupComponent } from './card-group/card-group.component';
+import { HelpTextComponent } from './help-text/help-text.component';
 
 @NgModule({
   imports: [
@@ -111,7 +112,8 @@ import { CardGroupComponent } from './card-group/card-group.component';
     CardRowComponent,
     CodeBlockComponent,
     VerticalNavigationComponent,
-    CardGroupComponent
+    CardGroupComponent,
+    HelpTextComponent
   ],
   providers: [],
   exports: [
@@ -146,7 +148,8 @@ import { CardGroupComponent } from './card-group/card-group.component';
     CardRowComponent,
     CodeBlockComponent,
     VerticalNavigationComponent,
-    CardGroupComponent
+    CardGroupComponent,
+    HelpTextComponent
   ]
 })
 export class ComponentsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.html
new file mode 100644
index 000000000000..e8d0d6e3d19a
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.html
@@ -0,0 +1,3 @@
+  <div class="form-text text-muted">
+    <ng-content></ng-content>
+  </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.scss
new file mode 100644
index 000000000000..f7be01cd929b
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.scss
@@ -0,0 +1,3 @@
+::ng-deep legend .text-muted {
+  font-size: small;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.spec.ts
new file mode 100644
index 000000000000..10a86418f8dc
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.spec.ts
@@ -0,0 +1,22 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { HelpTextComponent } from './help-text.component';
+
+describe('HelpTextComponent', () => {
+  let component: HelpTextComponent;
+  let fixture: ComponentFixture<HelpTextComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [HelpTextComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(HelpTextComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.ts
new file mode 100644
index 000000000000..60fed74aa556
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/help-text/help-text.component.ts
@@ -0,0 +1,8 @@
+import { Component } from '@angular/core';
+
+@Component({
+  selector: 'cd-help-text',
+  templateUrl: './help-text.component.html',
+  styleUrls: ['./help-text.component.scss']
+})
+export class HelpTextComponent {}

From 06cf024341284cad62100920958ade131a970902 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 15:46:34 +0800
Subject: [PATCH 2200/2492] crimson/os/cache: get_extent_viewable_by_trans, the
 extent should be valid

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/cache.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 082241d408fa..28471bbcd7ea 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -449,6 +449,7 @@ class Cache {
     Transaction &t,
     CachedExtentRef extent)
   {
+    assert(extent->is_valid());
     auto p_extent = extent->get_transactional_view(t);
     if (!p_extent->is_pending_in_trans(t.get_trans_id())) {
       t.add_to_read_set(p_extent);

From 8bca062d04e95565c13fdb365f344dceae3a4c1f Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 16:20:07 +0800
Subject: [PATCH 2201/2492] crimson/os/seastore: reuse
 link_phy_tree_root_node()

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/root_block.cc | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/crimson/os/seastore/root_block.cc b/src/crimson/os/seastore/root_block.cc
index dc928e81b0fa..5b8e0ca5d1f8 100644
--- a/src/crimson/os/seastore/root_block.cc
+++ b/src/crimson/os/seastore/root_block.cc
@@ -10,16 +10,22 @@ namespace crimson::os::seastore {
 void RootBlock::on_replace_prior(Transaction &t) {
   if (!lba_root_node) {
     auto &prior = static_cast<RootBlock&>(*get_prior_instance());
-    lba_root_node = prior.lba_root_node;
-    if (lba_root_node) {
-      ((lba_manager::btree::LBANode*)lba_root_node)->root_block = this;
+    if (prior.lba_root_node) {
+      RootBlockRef this_ref = this;
+      link_phy_tree_root_node(
+        this_ref,
+        static_cast<lba_manager::btree::LBANode*>(prior.lba_root_node)
+      );
     }
   }
   if (!backref_root_node) {
     auto &prior = static_cast<RootBlock&>(*get_prior_instance());
-    backref_root_node = prior.backref_root_node;
-    if (backref_root_node) {
-      ((backref::BackrefNode*)backref_root_node)->root_block = this;
+    if (prior.backref_root_node) {
+      RootBlockRef this_ref = this;
+      link_phy_tree_root_node(
+        this_ref,
+        static_cast<backref::BackrefNode*>(prior.backref_root_node)
+      );
     }
   }
 }

From 3df6477809f8aabacbf9850a09527c2887263949 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Thu, 7 Mar 2024 16:41:25 +0800
Subject: [PATCH 2202/2492] crimson/os/seastore/../fixed_kv_node: add comment
 to copy-from-source

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/btree/fixed_kv_node.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index 6ad55e3b9b32..544727f30aac 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -430,6 +430,8 @@ struct FixedKVNode : ChildableCachedExtent {
 	// the foreign key is preserved
 	if (!child) {
 	  child = source.children[foreign_it.get_offset()];
+	  // child can be either valid if present, nullptr if absent,
+	  // or RESERVATION_PTR.
 	}
 	foreign_it++;
 	local_it++;

From bc97b82dcb1b00023a5dca149209f6871fb34a0e Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 8 Mar 2024 15:39:40 +0800
Subject: [PATCH 2203/2492] test/crimson: drop cout from seastore tests

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/test/crimson/seastore/test_seastore.cc          | 13 -------------
 .../crimson/seastore/test_transaction_manager.cc    |  1 -
 2 files changed, 14 deletions(-)

diff --git a/src/test/crimson/seastore/test_seastore.cc b/src/test/crimson/seastore/test_seastore.cc
index 1e0028b97aca..d3a21261e27a 100644
--- a/src/test/crimson/seastore/test_seastore.cc
+++ b/src/test/crimson/seastore/test_seastore.cc
@@ -794,26 +794,20 @@ TEST_P(seastore_test_t, clone_aligned_extents)
     test_obj.write(*sharded_seastore, 0, 4096, 'a');
 
     test_obj.clone(*sharded_seastore, 10);
-    std::cout << "reading origin after clone10" << std::endl;
     test_obj.read(*sharded_seastore, 0, 4096);
     test_obj.write(*sharded_seastore, 0, 4096, 'b');
     test_obj.write(*sharded_seastore, 4096, 4096, 'c');
-    std::cout << "reading origin after clone10 and write" << std::endl;
     test_obj.read(*sharded_seastore, 0, 8192);
     auto clone_obj10 = test_obj.get_clone(10);
-    std::cout << "reading clone after clone10 and write" << std::endl;
     clone_obj10.read(*sharded_seastore, 0, 8192);
 
     test_obj.clone(*sharded_seastore, 20);
-    std::cout << "reading origin after clone20" << std::endl;
     test_obj.read(*sharded_seastore, 0, 4096);
     test_obj.write(*sharded_seastore, 0, 4096, 'd');
     test_obj.write(*sharded_seastore, 4096, 4096, 'e');
     test_obj.write(*sharded_seastore, 8192, 4096, 'f');
-    std::cout << "reading origin after clone20 and write" << std::endl;
     test_obj.read(*sharded_seastore, 0, 12288);
     auto clone_obj20 = test_obj.get_clone(20);
-    std::cout << "reading clone after clone20 and write" << std::endl;
     clone_obj10.read(*sharded_seastore, 0, 12288);
     clone_obj20.read(*sharded_seastore, 0, 12288);
   });
@@ -829,31 +823,25 @@ TEST_P(seastore_test_t, clone_unaligned_extents)
 
     test_obj.clone(*sharded_seastore, 10);
     test_obj.write(*sharded_seastore, 4096, 12288, 'd');
-    std::cout << "reading origin after clone10 and write" << std::endl;
     test_obj.read(*sharded_seastore, 0, 24576);
 
     auto clone_obj10 = test_obj.get_clone(10);
-    std::cout << "reading clone after clone10 and write" << std::endl;
     clone_obj10.read(*sharded_seastore, 0, 24576);
 
     test_obj.clone(*sharded_seastore, 20);
     test_obj.write(*sharded_seastore, 8192, 12288, 'e');
-    std::cout << "reading origin after clone20 and write" << std::endl;
     test_obj.read(*sharded_seastore, 0, 24576);
 
     auto clone_obj20 = test_obj.get_clone(20);
-    std::cout << "reading clone after clone20 and write" << std::endl;
     clone_obj10.read(*sharded_seastore, 0, 24576);
     clone_obj20.read(*sharded_seastore, 0, 24576);
 
     test_obj.write(*sharded_seastore, 0, 24576, 'f');
     test_obj.clone(*sharded_seastore, 30);
     test_obj.write(*sharded_seastore, 8192, 4096, 'g');
-    std::cout << "reading origin after clone30 and write" << std::endl;
     test_obj.read(*sharded_seastore, 0, 24576);
 
     auto clone_obj30 = test_obj.get_clone(30);
-    std::cout << "reading clone after clone30 and write" << std::endl;
     clone_obj10.read(*sharded_seastore, 0, 24576);
     clone_obj20.read(*sharded_seastore, 0, 24576);
     clone_obj30.read(*sharded_seastore, 0, 24576);
@@ -909,7 +897,6 @@ TEST_P(seastore_test_t, attr)
     EXPECT_EQ(attrs.find(SS_ATTR), attrs.end());
     EXPECT_EQ(attrs.find("test_key"), attrs.end());
 
-    std::cout << "test_key passed" << std::endl;
     //create OI_ATTR with len > onode_layout_t::MAX_OI_LENGTH, rm OI_ATTR
     //create SS_ATTR with len > onode_layout_t::MAX_SS_LENGTH, rm SS_ATTR
     char oi_array[onode_layout_t::MAX_OI_LENGTH + 1] = {'a'};
diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc
index 869eae716cd6..de8dce930ddc 100644
--- a/src/test/crimson/seastore/test_transaction_manager.cc
+++ b/src/test/crimson/seastore/test_transaction_manager.cc
@@ -1723,7 +1723,6 @@ TEST_P(tm_random_block_device_test_t, scatter_allocation)
     auto t = create_transaction();
     for (int i = 0; i < 1991; i++) {
       auto extents = alloc_extents(t, ADDR + i * 16384, 16384, 'a');
-      std::cout << "num of extents: " << extents.size() << std::endl;
     }
     alloc_extents_deemed_fail(t, ADDR + 1991 * 16384, 16384, 'a');
     check_mappings(t);

From 2d2df50e24d5a4dc3806b725e8b43cc0fcd1e3c4 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 8 Mar 2024 15:48:06 +0800
Subject: [PATCH 2204/2492] crimson/os/seastore/transaction_manager: minor
 refinement to remap_pin() asserts

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/os/seastore/transaction_manager.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 28524592eaef..f11967fd6042 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -425,12 +425,17 @@ class TransactionManager : public ExtentCallbackInterface {
     for (auto &remap : remaps) {
       auto remap_offset = remap.offset;
       auto remap_len = remap.len;
+      assert(remap_len > 0);
       total_remap_len += remap.len;
-      ceph_assert(remap_offset >= (last_offset + last_len));
+      assert(remap_offset >= (last_offset + last_len));
       last_offset = remap_offset;
       last_len = remap_len;
     }
-    ceph_assert(total_remap_len < original_len);
+    if (remaps.size() == 1) {
+      assert(total_remap_len < original_len);
+    } else {
+      assert(total_remap_len <= original_len);
+    }
 #endif
 
     // FIXME: paddr can be absolute and pending

From 2a112acd0a40ceed173e3b168ea30da5c31b27ad Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 16 Feb 2024 13:28:06 +0530
Subject: [PATCH 2205/2492] mgr/cephadm: mount grafana dashboards from the path

For this, the grafana dashboards are installed directly to the
containers and later, cephadm picks it up to mount the dashboards to the
grafana container

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 ceph.spec.in                                   |  3 +++
 debian/rules                                   |  1 +
 src/cephadm/cephadm.py                         |  4 +++-
 src/cephadm/cephadmlib/constants.py            |  2 +-
 src/cephadm/cephadmlib/daemons/monitoring.py   |  3 +++
 src/pybind/mgr/cephadm/module.py               |  9 ++++++++-
 src/pybind/mgr/cephadm/services/monitoring.py  | 18 ++++++++++++++++++
 .../services/grafana/providers.yml.j2          | 13 +++++++++++++
 8 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 src/pybind/mgr/cephadm/templates/services/grafana/providers.yml.j2

diff --git a/ceph.spec.in b/ceph.spec.in
index ed3674655277..24b8b1f6123f 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -1572,6 +1572,9 @@ mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rbd-mirror
 # prometheus alerts
 install -m 644 -D monitoring/ceph-mixin/prometheus_alerts.yml %{buildroot}/etc/prometheus/ceph/ceph_default_alerts.yml
 
+# grafana charts
+install -m 644 -D monitoring/ceph-mixin/dashboards_out/* %{buildroot}/etc/grafana/dashboards/ceph-dashboard/
+
 # SNMP MIB
 install -m 644 -D -t %{buildroot}%{_datadir}/snmp/mibs monitoring/snmp/CEPH-MIB.txt
 
diff --git a/debian/rules b/debian/rules
index ed7f4a255ed4..442ea28d4e0b 100755
--- a/debian/rules
+++ b/debian/rules
@@ -77,6 +77,7 @@ override_dh_auto_install:
 	install -D -m 755 src/tools/rbd_nbd/rbd-nbd_quiesce $(DESTDIR)/usr/libexec/rbd-nbd/rbd-nbd_quiesce
 
 	install -m 644 -D monitoring/ceph-mixin/prometheus_alerts.yml $(DESTDIR)/etc/prometheus/ceph/ceph_default_alerts.yml
+	install -m 644 -D monitoring/ceph-mixin/dashboards_out/* ${DESTDIR}/etc/grafana/dashboards/ceph-dashboard
 
 # doc/changelog is a directory, which confuses dh_installchangelogs
 override_dh_installchangelogs:
diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 5a7ed9e76656..47842d09da17 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -736,8 +736,10 @@ def create_daemon_dirs(
             makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755)
             makedirs(os.path.join(data_dir_root, config_dir, 'certs'), uid, gid, 0o755)
             makedirs(os.path.join(data_dir_root, config_dir, 'provisioning/datasources'), uid, gid, 0o755)
-            makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755)
+            makedirs(os.path.join(data_dir_root, config_dir, 'provisioning/dashboards'), uid, gid, 0o755)
+            makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o472)
             touch(os.path.join(data_dir_root, 'data', 'grafana.db'), uid, gid)
+            recursive_chown(os.path.join(data_dir_root, 'data'), uid, gid)
         elif daemon_type == 'alertmanager':
             data_dir_root = ident.data_dir(ctx.data_dir)
             config_dir = 'etc/alertmanager'
diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index dfa660f48986..e53de90d0858 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -9,7 +9,7 @@
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.5.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'
-DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.12'
+DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/grafana:9.4.12'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
 DEFAULT_NVMEOF_IMAGE = 'quay.io/ceph/nvmeof:latest'
diff --git a/src/cephadm/cephadmlib/daemons/monitoring.py b/src/cephadm/cephadmlib/daemons/monitoring.py
index e0667853dd7b..2ba6da289c28 100644
--- a/src/cephadm/cephadmlib/daemons/monitoring.py
+++ b/src/cephadm/cephadmlib/daemons/monitoring.py
@@ -333,6 +333,9 @@ def _get_container_mounts(self, data_dir: str) -> Dict[str, str]:
             mounts[
                 os.path.join(data_dir, 'etc/grafana/provisioning/datasources')
             ] = '/etc/grafana/provisioning/datasources:Z'
+            mounts[
+                os.path.join(data_dir, 'etc/grafana/provisioning/dashboards')
+            ] = '/etc/grafana/provisioning/dashboards:Z'
             mounts[
                 os.path.join(data_dir, 'etc/grafana/certs')
             ] = '/etc/grafana/certs:Z'
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 87f7024bb255..fc60de433ccc 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -121,7 +121,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_LOKI_IMAGE = 'docker.io/grafana/loki:2.4.0'
 DEFAULT_PROMTAIL_IMAGE = 'docker.io/grafana/promtail:2.4.0'
 DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.25.0'
-DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:9.4.12'
+DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/grafana:9.4.12'
 DEFAULT_HAPROXY_IMAGE = 'quay.io/ceph/haproxy:2.3'
 DEFAULT_KEEPALIVED_IMAGE = 'quay.io/ceph/keepalived:2.2.4'
 DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'
@@ -333,6 +333,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             default='/etc/prometheus/ceph/ceph_default_alerts.yml',
             desc='location of alerts to include in prometheus deployments',
         ),
+        Option(
+            'grafana_dashboards_path',
+            type='str',
+            default='/etc/grafana/dashboards/ceph-dashboard/',
+            desc='location of dashboards to include in grafana deployments',
+        ),
         Option(
             'migration_current',
             type='int',
@@ -549,6 +555,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.allow_ptrace = False
             self.container_init = True
             self.prometheus_alerts_path = ''
+            self.grafana_dashboards_path = ''
             self.migration_current: Optional[int] = None
             self.config_dashboard = True
             self.manage_etc_ceph_ceph_conf = True
diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py
index 114c848608a3..28da2f7558bc 100644
--- a/src/pybind/mgr/cephadm/services/monitoring.py
+++ b/src/pybind/mgr/cephadm/services/monitoring.py
@@ -80,6 +80,14 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
             self.mgr.check_mon_command(
                 {'prefix': 'dashboard set-grafana-api-password'}, inbuf=spec.initial_admin_password)
 
+        # the path of the grafana dashboards are assumed from the providers.yml.j2 file by grafana
+        grafana_dashboards_path = self.mgr.grafana_dashboards_path or '/etc/grafana/dashboards/ceph-dashboard/'
+        grafana_providers = self.mgr.template.render(
+            'services/grafana/providers.yml.j2', {
+                'grafana_dashboards_path': grafana_dashboards_path
+            }
+        )
+
         cert, pkey = self.prepare_certificates(daemon_spec)
         config_file = {
             'files': {
@@ -87,8 +95,18 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
                 'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources,
                 'certs/cert_file': '# generated by cephadm\n%s' % cert,
                 'certs/cert_key': '# generated by cephadm\n%s' % pkey,
+                'provisioning/dashboards/default.yml': grafana_providers
             }
         }
+
+        # include dashboards, if present in the container
+        if os.path.exists(grafana_dashboards_path):
+            files = os.listdir(grafana_dashboards_path)
+            for file_name in files:
+                with open(os.path.join(grafana_dashboards_path, file_name), 'r', encoding='utf-8') as f:
+                    dashboard = f.read()
+                    config_file['files'][f'/etc/grafana/provisioning/dashboards/{file_name}'] = dashboard
+
         return config_file, sorted(deps)
 
     def prepare_certificates(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[str, str]:
diff --git a/src/pybind/mgr/cephadm/templates/services/grafana/providers.yml.j2 b/src/pybind/mgr/cephadm/templates/services/grafana/providers.yml.j2
new file mode 100644
index 000000000000..6c284daeb67d
--- /dev/null
+++ b/src/pybind/mgr/cephadm/templates/services/grafana/providers.yml.j2
@@ -0,0 +1,13 @@
+# {{ cephadm_managed }}
+apiVersion: 1
+
+providers:
+  - name: 'Ceph Dashboard'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 3
+    editable: false
+    options:
+      path: '/etc/grafana/provisioning/dashboards'

From 057f09e4810d88d268b7f74e9b441f1f23b41b07 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Fri, 8 Mar 2024 15:51:15 +0530
Subject: [PATCH 2206/2492] mgr/cephadm: adapting unit tests to include
 provider file

Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/cephadm/tests/test_services.py | 46 ++++++++++++++++++-
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index dbab022058d4..cd59fcd03561 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -1106,6 +1106,20 @@ def test_grafana_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator)
                         # generated by cephadm\n{grafana_cert}""").lstrip(),
                     'certs/cert_key': dedent(f"""
                         # generated by cephadm\n{grafana_key}""").lstrip(),
+                    'provisioning/dashboards/default.yml': dedent("""
+                        # This file is generated by cephadm.
+                        apiVersion: 1
+
+                        providers:
+                          - name: 'Ceph Dashboard'
+                            orgId: 1
+                            folder: ''
+                            type: file
+                            disableDeletion: false
+                            updateIntervalSeconds: 3
+                            editable: false
+                            options:
+                              path: '/etc/grafana/provisioning/dashboards'""").lstrip(),
                 }
 
                 _run_cephadm.assert_called_with(
@@ -1184,7 +1198,21 @@ def test_grafana_initial_admin_pw(self, cephadm_module: CephadmOrchestrator):
                                     '    isDefault: false\n'
                                     '    editable: false',
                                 'certs/cert_file': ANY,
-                                'certs/cert_key': ANY}}, ['secure_monitoring_stack:False'])
+                                'certs/cert_key': ANY,
+                                'provisioning/dashboards/default.yml':
+                                    '# This file is generated by cephadm.\n'
+                                    'apiVersion: 1\n\n'
+                                    'providers:\n'
+                                    "  - name: 'Ceph Dashboard'\n"
+                                    '    orgId: 1\n'
+                                    "    folder: ''\n"
+                                    '    type: file\n'
+                                    '    disableDeletion: false\n'
+                                    '    updateIntervalSeconds: 3\n'
+                                    '    editable: false\n'
+                                    '    options:\n'
+                                    "      path: '/etc/grafana/provisioning/dashboards'"
+                            }}, ['secure_monitoring_stack:False'])
 
     @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}'))
     def test_grafana_no_anon_access(self, cephadm_module: CephadmOrchestrator):
@@ -1232,7 +1260,21 @@ def test_grafana_no_anon_access(self, cephadm_module: CephadmOrchestrator):
                                     '    isDefault: false\n'
                                     '    editable: false',
                                 'certs/cert_file': ANY,
-                                'certs/cert_key': ANY}}, ['secure_monitoring_stack:False'])
+                                'certs/cert_key': ANY,
+                                'provisioning/dashboards/default.yml':
+                                    '# This file is generated by cephadm.\n'
+                                    'apiVersion: 1\n\n'
+                                    'providers:\n'
+                                    "  - name: 'Ceph Dashboard'\n"
+                                    '    orgId: 1\n'
+                                    "    folder: ''\n"
+                                    '    type: file\n'
+                                    '    disableDeletion: false\n'
+                                    '    updateIntervalSeconds: 3\n'
+                                    '    editable: false\n'
+                                    '    options:\n'
+                                    "      path: '/etc/grafana/provisioning/dashboards'"
+                            }}, ['secure_monitoring_stack:False'])
 
     @patch("cephadm.serve.CephadmServe._run_cephadm")
     def test_monitoring_ports(self, _run_cephadm, cephadm_module: CephadmOrchestrator):

From 82d100ad264c35d79262c1983a8005d8d4791855 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Thu, 7 Mar 2024 01:52:16 +0530
Subject: [PATCH 2207/2492] mgr/dashboard: handle infinite values for pools

Fixes https://tracker.ceph.com/issues/64724

Issue:
======
Json parsing is failing because of Infinity values present in pools
meteadata. "read_balance": {"score_acting": Infinity, "score_stable":
Infinity,}
Due to this entire pool list is not rendered.

Fix:
====
Added a handler for checking "inf" values and replacing them with a
string "Infinity" so that json parsing does not fail on frontend.

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 src/pybind/mgr/dashboard/controllers/pool.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/controllers/pool.py b/src/pybind/mgr/dashboard/controllers/pool.py
index 1e2e04e1b14d..9310d2f97aab 100644
--- a/src/pybind/mgr/dashboard/controllers/pool.py
+++ b/src/pybind/mgr/dashboard/controllers/pool.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import math
 import time
 from typing import Any, Dict, Iterable, List, Optional, Union, cast
 
@@ -101,7 +102,7 @@ def _serialize_pool(pool, attrs):
 
         crush_rules = {r['rule_id']: r["rule_name"] for r in mgr.get('osd_map_crush')['rules']}
 
-        res: Dict[Union[int, str], Union[str, List[Any]]] = {}
+        res: Dict[Union[int, str], Union[str, List[Any], Dict[str, Any]]] = {}
         for attr in attrs:
             if attr not in pool:
                 continue
@@ -111,6 +112,15 @@ def _serialize_pool(pool, attrs):
                 res[attr] = crush_rules[pool[attr]]
             elif attr == 'application_metadata':
                 res[attr] = list(pool[attr].keys())
+            # handle infinity values
+            elif attr == 'read_balance' and isinstance(pool[attr], dict):
+                read_balance: Dict[str, Any] = {}
+                for key, value in pool[attr].items():
+                    if isinstance(value, float) and math.isinf(value):
+                        read_balance[key] = "Infinity"
+                    else:
+                        read_balance[key] = value
+                res[attr] = read_balance
             else:
                 res[attr] = pool[attr]
 

From 1b77baea8dd0781fa897ec6c1f1e06c57a265ed7 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Fri, 8 Mar 2024 11:40:41 +0000
Subject: [PATCH 2208/2492] mgr/dashboard: fix snap schedule list toggle cols

Added isInvisible property to CdColumnTable interface to hide column
from 'toggle columns' drop down checkboxes.

Fixes: https://tracker.ceph.com/issues/64813

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-snapshotschedule-list.component.ts             | 6 +++---
 .../src/app/shared/datatable/table/table.component.html   | 8 +++++---
 .../frontend/src/app/shared/models/cd-table-column.ts     | 5 +++++
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index ecc2036d5dd3..f6f372dcffc4 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -143,12 +143,12 @@ export class CephfsSnapshotscheduleListComponent
 
     this.columns = [
       { prop: 'pathForSelection', name: $localize`Path`, flexGrow: 3, cellTemplate: this.pathTpl },
-      { prop: 'path', isHidden: true },
+      { prop: 'path', isHidden: true, isInvisible: true },
       { prop: 'subvol', name: $localize`Subvolume`, cellTemplate: this.subvolTpl },
       { prop: 'scheduleCopy', name: $localize`Repeat interval` },
-      { prop: 'schedule', isHidden: true },
+      { prop: 'schedule', isHidden: true, isInvisible: true },
       { prop: 'retentionCopy', name: $localize`Retention policy`, cellTemplate: this.retentionTpl },
-      { prop: 'retention', isHidden: true },
+      { prop: 'retention', isHidden: true, isInvisible: true },
       { prop: 'created_count', name: $localize`Created Count` },
       { prop: 'pruned_count', name: $localize`Deleted Count` },
       { prop: 'start', name: $localize`Start time`, cellTransformation: CellTemplate.timeAgo },
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html
index 01cc1fbc8d92..f977273b0cf6 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table/table.component.html
@@ -103,9 +103,10 @@
         </button>
         <div ngbDropdownMenu>
           <ng-container *ngFor="let column of columns">
-            <button ngbDropdownItem
-                    *ngIf="column.name !== ''"
-                    (click)="toggleColumn(column); false;">
+            <ng-container *ngIf="!column?.isInvisible">
+              <button ngbDropdownItem
+                      *ngIf="column.name !== ''"
+                      (click)="toggleColumn(column); false;">
               <div class="custom-control custom-checkbox py-0">
                 <input class="custom-control-input"
                        type="checkbox"
@@ -116,6 +117,7 @@
                        for="{{ column.prop }}{{ tableName }}">{{ column.name }}</label>
               </div>
             </button>
+            </ng-container>
           </ng-container>
         </div>
       </div>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-column.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-column.ts
index 4ed5fdd588fe..e81eeb14490b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-column.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/cd-table-column.ts
@@ -35,4 +35,9 @@ export interface CdTableColumn extends TableColumn {
    * a customize function if that's not desired. Return true to include a row.
    */
   filterPredicate?: (row: any, value: any) => boolean;
+
+  /**
+   * Hides a column from the 'toggle columns' drop down checkboxes
+   */
+  isInvisible?: boolean;
 }

From 5b11fb314abefea390c5663c33e9e1ace6179974 Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sun, 18 Feb 2024 11:46:15 +0100
Subject: [PATCH 2209/2492] librados/snap_set_diff: ignore truncates above size
 at start

Because currently calc_snap_set_diff() only ever appends to the running
diff, an excessive (either too large or completely bogus) zero extent
is reported in cases where an object is first expanded (with a snapshot
taken at that point) and then truncated but still above the size of the
object as of the starting snapshot.

Fixes: https://tracker.ceph.com/issues/63770
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librados/snap_set_diff.cc                 |  40 +-
 src/test/librbd/io/test_mock_ObjectRequest.cc | 426 ++++++++++++++++++
 2 files changed, 451 insertions(+), 15 deletions(-)

diff --git a/src/librados/snap_set_diff.cc b/src/librados/snap_set_diff.cc
index 0029bcd64780..42d78eb0392c 100644
--- a/src/librados/snap_set_diff.cc
+++ b/src/librados/snap_set_diff.cc
@@ -86,26 +86,36 @@ void calc_snap_set_diff(CephContext *cct, const librados::snap_set_t& snap_set,
       return;
     }
 
-    // start with the max(this size, next size), and subtract off any
-    // overlap
+    // start with the largest possible diff to next, and subtract off
+    // any overlap
     const vector<pair<uint64_t, uint64_t> > *overlap = &r->overlap;
     interval_set<uint64_t> diff_to_next;
-    uint64_t max_size = r->size;
+    uint64_t diff_boundary;
+    uint64_t prev_size = r->size;
     ++r;
     if (r != snap_set.clones.end()) {
-      if (r->size > max_size)
-	max_size = r->size;
-    }
-    if (max_size)
-      diff_to_next.insert(0, max_size);
-    for (vector<pair<uint64_t, uint64_t> >::const_iterator p = overlap->begin();
-	 p != overlap->end();
-	 ++p) {
-      diff_to_next.erase(p->first, p->second);
+      if (r->size >= prev_size) {
+        diff_boundary = r->size;
+      } else if (prev_size <= start_size) {
+        // truncated range below size at start
+        diff_boundary = prev_size;
+      } else {
+        // truncated range (partially) above size at start -- drop that
+        // part from the running diff
+        diff_boundary = std::max(r->size, start_size);
+        ldout(cct, 20) << "  no more diff beyond " << diff_boundary << dendl;
+        diff->erase(diff_boundary, prev_size - diff_boundary);
+      }
+      if (diff_boundary) {
+        diff_to_next.insert(0, diff_boundary);
+      }
+      for (auto p = overlap->begin(); p != overlap->end(); ++p) {
+        diff_to_next.erase(p->first, p->second);
+      }
+      ldout(cct, 20) << "  diff_to_next " << diff_to_next << dendl;
+      diff->union_of(diff_to_next);
+      ldout(cct, 20) << "  diff now " << *diff << dendl;
     }
-    ldout(cct, 20) << "  diff_to_next " << diff_to_next << dendl;
-    diff->union_of(diff_to_next);
-    ldout(cct, 20) << "  diff now " << *diff << dendl;
   }
 
   if (r != snap_set.clones.end()) {
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index 50ee5b718fad..cd553d2a509c 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -1784,6 +1784,432 @@ TEST_F(TestMockIoObjectRequest, ListSnaps) {
   ASSERT_EQ(expected_snapshot_delta, snapshot_delta);
 }
 
+TEST_F(TestMockIoObjectRequest, ListSnapsGrowFromSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 4;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2048}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 3072;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_DATA, 1024});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_DATA, 1024});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsTruncateFromSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 4;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 1536}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 1536;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1536, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1536, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsTruncateFromBelowSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4, 5};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 5;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 1536}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 5;
+  clone_info.snaps = {5};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 1024}};
+  clone_info.size = 1536;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 1024;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1024, 1024, {SPARSE_EXTENT_STATE_ZEROED, 1024});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, 5, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{5,5}].insert(
+      1536, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1024, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsTruncateStraddlingSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4, 5};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 5;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2048}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 5;
+  clone_info.snaps = {5};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 1536}};
+  clone_info.size = 3072;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 1536;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1536, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, 5, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{5,5}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_DATA, 1024});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1536, 1536, {SPARSE_EXTENT_STATE_ZEROED, 1536});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsTruncateToSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4, 5};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 5;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2048}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 5;
+  clone_info.snaps = {5};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2048}};
+  clone_info.size = 3072;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, 5, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{5,5}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_DATA, 1024});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_ZEROED, 1024});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsTruncateToAboveSizeAtStart) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3, 4, 5};
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 5;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 512}};
+  clone_info.size = 512;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 4;
+  clone_info.snaps = {4};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2048}};
+  clone_info.size = 2048;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = 5;
+  clone_info.snaps = {5};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{{0, 2560}};
+  clone_info.size = 3072;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = 2560;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {4, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      2048, 512, {SPARSE_EXTENT_STATE_DATA, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{0, mock_image_ctx.layout.object_size}},
+      {3, 4, 5, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{4,4}].insert(
+      512, 1536, {SPARSE_EXTENT_STATE_DATA, 1536});
+    expected_snapshot_delta[{5,5}].insert(
+      2048, 1024, {SPARSE_EXTENT_STATE_DATA, 1024});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      2560, 512, {SPARSE_EXTENT_STATE_ZEROED, 512});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
 TEST_F(TestMockIoObjectRequest, ListSnapsENOENT) {
   librbd::ImageCtx *ictx;
   ASSERT_EQ(0, open_image(m_image_name, &ictx));

From 0ebca5fe22d976f1a17f8078f20a47c49a057edb Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Wed, 28 Feb 2024 14:20:16 +0100
Subject: [PATCH 2210/2492] librbd: don't clip expanded diff on truncate in
 ObjectListSnapsRequest

If the diff was expanded due to LIST_SNAPS_FLAG_WHOLE_OBJECT, clipping
it when handling a truncate is wrong -- when subtracting that interval,
we either split the expanded extent into two or chop off a piece of it.
However the point of LIST_SNAPS_FLAG_WHOLE_OBJECT is to report a single
extent covering the entire object.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/librbd/io/ObjectRequest.cc                |   3 +-
 src/test/librbd/io/test_mock_ObjectRequest.cc | 123 ++++++++++++++++++
 2 files changed, 125 insertions(+), 1 deletion(-)

diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc
index 428f0fbdf584..a7748623037e 100644
--- a/src/librbd/io/ObjectRequest.cc
+++ b/src/librbd/io/ObjectRequest.cc
@@ -865,7 +865,8 @@ void ObjectListSnapsRequest<I>::handle_list_snaps(int r) {
 
     // clip diff to size of object (in case it was truncated)
     interval_set<uint64_t> zero_interval;
-    if (end_size < prev_end_size) {
+    if (end_size < prev_end_size &&
+        (m_list_snaps_flags & LIST_SNAPS_FLAG_WHOLE_OBJECT) == 0) {
       zero_interval.insert(end_size, prev_end_size - end_size);
       zero_interval.intersection_of(object_interval);
 
diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc
index cd553d2a509c..55c1e096ab7b 100644
--- a/src/test/librbd/io/test_mock_ObjectRequest.cc
+++ b/src/test/librbd/io/test_mock_ObjectRequest.cc
@@ -2410,6 +2410,129 @@ TEST_F(TestMockIoObjectRequest, ListSnapsWholeObject) {
   }
 }
 
+TEST_F(TestMockIoObjectRequest, ListSnapsWholeObjectTruncate) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3};
+
+  InSequence seq;
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 3;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = std::vector<std::pair<uint64_t,uint64_t>>{
+    {2, mock_image_ctx.layout.object_size - 4}};
+  clone_info.size = mock_image_ctx.layout.object_size;
+  snap_set.clones.push_back(clone_info);
+
+  clone_info.cloneid = CEPH_NOSNAP;
+  clone_info.snaps = {};
+  clone_info.overlap = {};
+  clone_info.size = mock_image_ctx.layout.object_size - 2;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{1, mock_image_ctx.layout.object_size - 2}},
+      {3, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1, 1, {SPARSE_EXTENT_STATE_DATA, 1});
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      mock_image_ctx.layout.object_size - 2, 1,
+      {SPARSE_EXTENT_STATE_ZEROED, 1});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{1, mock_image_ctx.layout.object_size - 2}},
+      {3, CEPH_NOSNAP}, LIST_SNAPS_FLAG_WHOLE_OBJECT, {}, &snapshot_delta,
+      &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1, mock_image_ctx.layout.object_size - 2,
+      {SPARSE_EXTENT_STATE_DATA, mock_image_ctx.layout.object_size - 2});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
+TEST_F(TestMockIoObjectRequest, ListSnapsWholeObjectRemove) {
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockTestImageCtx mock_image_ctx(*ictx);
+  mock_image_ctx.snaps = {3};
+
+  InSequence seq;
+
+  librados::snap_set_t snap_set;
+  snap_set.seq = 3;
+  librados::clone_info_t clone_info;
+
+  clone_info.cloneid = 3;
+  clone_info.snaps = {3};
+  clone_info.overlap = {};
+  clone_info.size = mock_image_ctx.layout.object_size - 2;
+  snap_set.clones.push_back(clone_info);
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{1, mock_image_ctx.layout.object_size - 2}},
+      {3, CEPH_NOSNAP}, 0, {}, &snapshot_delta, &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1, mock_image_ctx.layout.object_size - 3,
+      {SPARSE_EXTENT_STATE_ZEROED, mock_image_ctx.layout.object_size - 3});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+
+  expect_list_snaps(mock_image_ctx, snap_set, 0);
+
+  {
+    SnapshotDelta snapshot_delta;
+    C_SaferCond ctx;
+    auto req = MockObjectListSnapsRequest::create(
+      &mock_image_ctx, 0, {{1, mock_image_ctx.layout.object_size - 2}},
+      {3, CEPH_NOSNAP}, LIST_SNAPS_FLAG_WHOLE_OBJECT, {}, &snapshot_delta,
+      &ctx);
+    req->send();
+    ASSERT_EQ(0, ctx.wait());
+
+    SnapshotDelta expected_snapshot_delta;
+    expected_snapshot_delta[{CEPH_NOSNAP,CEPH_NOSNAP}].insert(
+      1, mock_image_ctx.layout.object_size - 2,
+      {SPARSE_EXTENT_STATE_ZEROED, mock_image_ctx.layout.object_size - 2});
+    EXPECT_EQ(expected_snapshot_delta, snapshot_delta);
+  }
+}
+
 TEST_F(TestMockIoObjectRequest, ListSnapsWholeObjectEndSize) {
   librbd::ImageCtx *ictx;
   ASSERT_EQ(0, open_image(m_image_name, &ictx));

From d84485ec534ccf42327242a9141906f4881c030e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Tue, 5 Mar 2024 21:05:22 +0100
Subject: [PATCH 2211/2492] test/librbd: add DiffIterateTruncate and
 DiffIterateWriteAndTruncate

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 src/test/librbd/test_librbd.cc | 315 +++++++++++++++++++++++++++++++++
 1 file changed, 315 insertions(+)

diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc
index 94420a64286a..420a9f6ae72a 100644
--- a/src/test/librbd/test_librbd.cc
+++ b/src/test/librbd/test_librbd.cc
@@ -7862,6 +7862,321 @@ TYPED_TEST(DiffIterateTest, DiffIterateDiscard)
   ASSERT_PASSED(this->validate_object_map, image);
 }
 
+TYPED_TEST(DiffIterateTest, DiffIterateTruncate)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_STRIPINGV2));
+
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
+
+  librbd::RBD rbd;
+  librbd::Image image;
+  int order = 22;
+  std::string name = this->get_temp_image_name();
+  uint64_t size = 20 << 20;
+
+  ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+
+  uint64_t object_size = 0;
+  if (this->whole_object) {
+    object_size = 1 << order;
+  }
+
+  ASSERT_EQ(0, image.snap_create("snap0"));
+
+  ceph::bufferlist bl;
+  bl.append(std::string(512 << 10, '1'));
+  ASSERT_EQ(512 << 10, image.write(0, 512 << 10, bl));
+  ASSERT_EQ(0, image.snap_create("snap1"));
+  ASSERT_EQ(512 << 10, image.write(512 << 10, 512 << 10, bl));
+  ASSERT_EQ(0, image.snap_create("snap2"));
+
+  std::vector<diff_extent> extents;
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 1024 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 1024 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(512 << 10, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_EQ(256 << 10, image.discard(768 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 768 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 768 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(512 << 10, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(768 << 10, 256 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(256 << 10, image.discard(512 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object &&
+      (is_feature_enabled(RBD_FEATURE_OBJECT_MAP) ||
+       is_feature_enabled(RBD_FEATURE_FAST_DIFF))) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+    extents.clear();
+  } else {
+    ASSERT_EQ(0u, extents.size());
+  }
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(512 << 10, 512 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(256 << 10, image.discard(256 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(256 << 10, 256 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(256 << 10, 768 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+
+  ASSERT_EQ(256 << 10, image.discard(0, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object &&
+      (is_feature_enabled(RBD_FEATURE_OBJECT_MAP) ||
+       is_feature_enabled(RBD_FEATURE_FAST_DIFF))) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+    extents.clear();
+  } else {
+    ASSERT_EQ(0u, extents.size());
+  }
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object &&
+      (is_feature_enabled(RBD_FEATURE_OBJECT_MAP) ||
+       is_feature_enabled(RBD_FEATURE_FAST_DIFF))) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+    extents.clear();
+  } else {
+    ASSERT_EQ(0u, extents.size());
+  }
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 512 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 1024 << 10, this->whole_object, object_size),
+            extents[0]);
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+}
+
+TYPED_TEST(DiffIterateTest, DiffIterateWriteAndTruncate)
+{
+  REQUIRE(!is_feature_enabled(RBD_FEATURE_STRIPINGV2));
+
+  librados::IoCtx ioctx;
+  ASSERT_EQ(0, this->_rados.ioctx_create(this->m_pool_name.c_str(), ioctx));
+
+  librbd::RBD rbd;
+  librbd::Image image;
+  int order = 22;
+  std::string name = this->get_temp_image_name();
+  uint64_t size = 20 << 20;
+
+  ASSERT_EQ(0, create_image_pp(rbd, ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rbd.open(ioctx, image, name.c_str(), NULL));
+
+  uint64_t object_size = 0;
+  if (this->whole_object) {
+    object_size = 1 << order;
+  }
+
+  ASSERT_EQ(0, image.snap_create("snap0"));
+
+  ceph::bufferlist bl;
+  bl.append(std::string(512 << 10, '1'));
+  ASSERT_EQ(512 << 10, image.write(0, 512 << 10, bl));
+  ASSERT_EQ(0, image.snap_create("snap1"));
+  ASSERT_EQ(512 << 10, image.write(512 << 10, 512 << 10, bl));
+  ASSERT_EQ(0, image.snap_create("snap2"));
+
+  std::vector<diff_extent> extents;
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 1024 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 1024 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(512 << 10, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(0u, extents.size());
+
+  ASSERT_EQ(1 << 10, image.write(767 << 10, 1 << 10, bl));
+  ASSERT_EQ(256 << 10, image.discard(768 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 768 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 768 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(512 << 10, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+  } else {
+    ASSERT_EQ(2u, extents.size());
+    ASSERT_EQ(diff_extent(767 << 10, 1 << 10, true, object_size), extents[0]);
+    ASSERT_EQ(diff_extent(768 << 10, 256 << 10, false, object_size),
+              extents[1]);
+  }
+  extents.clear();
+
+  ASSERT_EQ(2 << 10, image.write(510 << 10, 2 << 10, bl));
+  ASSERT_EQ(256 << 10, image.discard(512 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 512 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(510 << 10, 2 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+  } else {
+    ASSERT_EQ(2u, extents.size());
+    ASSERT_EQ(diff_extent(510 << 10, 2 << 10, true, object_size), extents[0]);
+    ASSERT_EQ(diff_extent(512 << 10, 512 << 10, false, object_size),
+              extents[1]);
+  }
+  extents.clear();
+
+  ASSERT_EQ(3 << 10, image.write(253 << 10, 3 << 10, bl));
+  ASSERT_EQ(256 << 10, image.discard(256 << 10, 256 << 10));
+  ASSERT_EQ(0, image.diff_iterate2(NULL, 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap0", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  ASSERT_EQ(1u, extents.size());
+  ASSERT_EQ(diff_extent(0, 256 << 10, true, object_size), extents[0]);
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap1", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+  } else {
+    ASSERT_EQ(2u, extents.size());
+    ASSERT_EQ(diff_extent(253 << 10, 3 << 10, true, object_size), extents[0]);
+    ASSERT_EQ(diff_extent(256 << 10, 256 << 10, false, object_size),
+              extents[1]);
+  }
+  extents.clear();
+  ASSERT_EQ(0, image.diff_iterate2("snap2", 0, size, true, this->whole_object,
+                                   vector_iterate_cb, &extents));
+  if (this->whole_object) {
+    ASSERT_EQ(1u, extents.size());
+    ASSERT_EQ(diff_extent(0, 0, true, object_size), extents[0]);
+  } else {
+    ASSERT_EQ(2u, extents.size());
+    ASSERT_EQ(diff_extent(253 << 10, 3 << 10, true, object_size), extents[0]);
+    ASSERT_EQ(diff_extent(256 << 10, 768 << 10, false, object_size),
+              extents[1]);
+  }
+  extents.clear();
+
+  ASSERT_PASSED(this->validate_object_map, image);
+}
+
 TYPED_TEST(DiffIterateTest, DiffIterateStress)
 {
   REQUIRE(!is_rbd_pwl_enabled((CephContext *)this->_rados.cct()));

From c8c698978e270744af9393a2126b6f9e6029d4c4 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 8 Mar 2024 14:39:17 -0500
Subject: [PATCH 2212/2492] qa: fix syntax bug with invalid 60 minute
 specification

Use 8 minute cycles instead.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 36 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index 80b5b14fcabf..52280d9fec81 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -56,12 +56,12 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
 00 05 * * 0   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 201
-10 05 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-20 05 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-30 05 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-40 05 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-50 05 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-60 05 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+08 05 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+16 05 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+24 05 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+32 05 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+40 05 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+48 05 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
 
 
 ## squid branch runs - twice weekly
@@ -70,12 +70,12 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
 00 06 * * 0,4 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 101
-10 06 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-20 06 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-30 06 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-40 06 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-50 06 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-60 06 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+08 06 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+16 06 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+24 06 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+32 06 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+40 06 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+48 06 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
 
 
 ## reef branch runs - weekly
@@ -84,12 +84,12 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
 00 07 * * 0   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 181
-10 07 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-20 07 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-30 07 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-40 07 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-50 07 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-60 07 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+08 07 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+16 07 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+24 07 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+32 07 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+40 07 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+48 07 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
 
 
 ###  The suite below must run on bare-metal because it's performance suite and run 3 times to produce more data points

From b31e061cc087b31a7e9e841dd21e7403a2197378 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sat, 9 Mar 2024 03:17:59 +1000
Subject: [PATCH 2213/2492] doc/glossary: add "Crimson" entry

Add a "Crimson" entry to the glossary.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/dev/crimson/index.rst |  2 ++
 doc/glossary.rst          | 12 ++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/doc/dev/crimson/index.rst b/doc/dev/crimson/index.rst
index 55f071825d7e..9790a9640c2e 100644
--- a/doc/dev/crimson/index.rst
+++ b/doc/dev/crimson/index.rst
@@ -1,3 +1,5 @@
+.. _crimson_dev_doc:
+
 ===============================
 Crimson developer documentation
 ===============================
diff --git a/doc/glossary.rst b/doc/glossary.rst
index 262b89598c50..988285009d27 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -213,6 +213,18 @@
                 Ceph cluster. See :ref:`the "Cluster Map" section of the
                 Architecture document<architecture_cluster_map>` for details.
 
+        Crimson
+                A next-generation OSD architecture whose core aim is the
+                reduction of latency costs incurred due to cross-core
+                communications. A re-design of the OSD that reduces lock
+                contention by reducing communication between shards in the data
+                path. Crimson improves upon the performance of classic Ceph
+                OSDs by eliminating reliance on thread pools. See `Crimson:
+                Next-generation Ceph OSD for Multi-core Scalability
+                <https://ceph.io/en/news/blog/2023/crimson-multi-core-scalability/>`_.
+                See the :ref:`Crimson developer
+                documentation<crimson_dev_doc>`.
+
 	CRUSH
                 **C**\ontrolled **R**\eplication **U**\nder **S**\calable
                 **H**\ashing. The algorithm that Ceph uses to compute object

From 2223c898533aedd9e7a3983494ea39e9c547201a Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 10 Mar 2024 20:43:52 +1000
Subject: [PATCH 2214/2492] doc/glossary: improve "Crimson" entry

Improve the glossary entry for "Crimson" in accordance with Anthony
D'Atri's suggestions here:
https://github.com/ceph/ceph/pull/56068#discussion_r1518580402

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index 988285009d27..bca2346a2127 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -214,9 +214,9 @@
                 Architecture document<architecture_cluster_map>` for details.
 
         Crimson
-                A next-generation OSD architecture whose core aim is the
+                A next-generation OSD architecture whose main aim is the
                 reduction of latency costs incurred due to cross-core
-                communications. A re-design of the OSD that reduces lock
+                communications. A re-design of the OSD reduces lock
                 contention by reducing communication between shards in the data
                 path. Crimson improves upon the performance of classic Ceph
                 OSDs by eliminating reliance on thread pools. See `Crimson:

From 543fe9f1fbb5257693b83b8002f31625cafbce56 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Thu, 7 Mar 2024 19:48:54 +0800
Subject: [PATCH 2215/2492] msg: do not abort when driver->del_event() returns
 -ENOENT

when shutting down a connection, we call into
`EpollDriver::del_event(..., EVENT_READABLE | EVENT_WRITABLE)`, and
its caller, `EventCenter::delete_file_event()` considers a negative
return value from this function a signal of bug and aborts in that
case. but in linux, if a nic is hot unplugged, all the socket file
descriptors associated with it are closed, and we would have following
chain:

__fput() -> eventpoll_release() -> eventpoll_release_file() -> __ep_remove()

in __ep_remove(), the epitem representing the fd is removed from
the list. so if we perform the cleanup when shutting down the
TCP connection, and try to unregister the fd from the interest list,
-ENOENT is returned.

librbd is using EpollDriver as well, and it sits at the client side.
the machine on which librbd is running could unplug its NIC without
shutting down librbd first. so, if librbd happen to be reading/writing
to the socket associated with the NIC being unplugged, there are
chances that librbd could crash due to the `ceph_abort_msg()` call
in `EventCenter::delete_file_event()`. but this is not a fatal error,
as we are unregistering the fd anyway.

in this change, in order to avoid the crash, we don't consider it a
bug if `driver->del_event()` returns -ENOENT anymore.

Fixes: https://tracker.ceph.com/issues/64788

Co-Authored-by: Zhang Jiao <zhangjiao@cmss.chinamobile.com>
Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/msg/async/Event.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/msg/async/Event.cc b/src/msg/async/Event.cc
index 4662e42bd144..926fdcdb1cc3 100644
--- a/src/msg/async/Event.cc
+++ b/src/msg/async/Event.cc
@@ -16,6 +16,7 @@
 
 #include "include/compat.h"
 #include "common/errno.h"
+#include <cerrno>
 #include "Event.h"
 
 #ifdef HAVE_DPDK
@@ -285,7 +286,10 @@ void EventCenter::delete_file_event(int fd, int mask)
     return ;
 
   int r = driver->del_event(fd, event->mask, mask);
-  if (r < 0) {
+  if (r < 0 && r != -ENOENT) {
+    // if the socket fd is closed by the underlying nic driver, the
+    // corresponding epoll item would be removed from the interest list, that'd
+    // lead to ENOENT when removing the fd from the list.
     // see create_file_event
     ceph_abort_msg("BUG!");
   }

From cbad3e4517e91476b19a8766f98240c81df40a36 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Thu, 7 Mar 2024 09:49:34 +0000
Subject: [PATCH 2216/2492] crimson/osd/osd: handle_scrub_message on any core

After enabling multicore cluster messenger, there is no
guarantee which core will handle the request.
See: OSDSingletonState::send_to_osd()

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/crimson/osd/osd.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 5c02b2895c2d..939fbc59beb8 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -1268,7 +1268,6 @@ seastar::future<> OSD::handle_scrub_message(
   crimson::net::ConnectionRef conn,
   Ref<MOSDFastDispatchOp> m)
 {
-  ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
   return pg_shard_manager.start_pg_operation<
     crimson::osd::ScrubMessage
     >(m, conn, m->get_min_epoch(), m->get_spg()).second;

From 7d5ea23bd2da1684821e170c4acd4a5e6572c5cb Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Sun, 3 Mar 2024 07:50:43 +0200
Subject: [PATCH 2217/2492] osd: avoid calling a virtual function in a ctor

Cleaning up a clang-tidy warning.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/osd_types.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 2c6028f9cb41..b69a6d56d83e 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -3886,7 +3886,7 @@ class pi_compact_rep : public PastIntervals::interval_rep {
     bool ec_pool,
     std::list<PastIntervals::pg_interval_t> &&intervals) {
     for (auto &&i: intervals)
-      add_interval(ec_pool, i);
+      pi_compact_rep::add_interval(ec_pool, i);
   }
 public:
   pi_compact_rep() = default;

From 166a2362378b1ff93e43f483f354c428fd6cef9e Mon Sep 17 00:00:00 2001
From: Ilya Dryomov <idryomov@gmail.com>
Date: Sat, 9 Mar 2024 22:53:44 +0100
Subject: [PATCH 2218/2492] qa/workunits/rbd: switch rbd-mirror workunits to
 bash

By making use of here strings in commit ea3a567f7f03 ("qa/workunits:
make wait_for_status_in_pool_dir() reentrant") we grew a dependency on
bash.

Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
---
 qa/workunits/rbd/compare_mirror_image_alternate_primary.sh | 2 +-
 qa/workunits/rbd/compare_mirror_images.sh                  | 2 +-
 qa/workunits/rbd/rbd_mirror.sh                             | 4 +++-
 qa/workunits/rbd/rbd_mirror_bootstrap.sh                   | 4 +++-
 qa/workunits/rbd/rbd_mirror_fsx_compare.sh                 | 4 +++-
 qa/workunits/rbd/rbd_mirror_fsx_prepare.sh                 | 4 +++-
 qa/workunits/rbd/rbd_mirror_ha.sh                          | 4 +++-
 qa/workunits/rbd/rbd_mirror_helpers.sh                     | 2 +-
 qa/workunits/rbd/rbd_mirror_stress.sh                      | 4 +++-
 9 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh b/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
index 338f43f1e530..78a390230388 100755
--- a/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
+++ b/qa/workunits/rbd/compare_mirror_image_alternate_primary.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 
diff --git a/qa/workunits/rbd/compare_mirror_images.sh b/qa/workunits/rbd/compare_mirror_images.sh
index 1b3cc300de98..342a1ebc4e7b 100755
--- a/qa/workunits/rbd/compare_mirror_images.sh
+++ b/qa/workunits/rbd/compare_mirror_images.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 
diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
index bc3237dfc089..1cda355039eb 100755
--- a/qa/workunits/rbd/rbd_mirror.sh
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror.sh - test rbd-mirror daemon in snapshot or journal mirroring mode
 #
@@ -13,6 +13,8 @@
 # socket, temporary files, and launches rbd-mirror daemon.
 #
 
+set -ex
+
 if [ "${#}" -gt 0 ]; then
   echo "unnecessary arguments: ${@}"
   exit 100
diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh
index 6ef06f2b82cb..f4c1070bc95b 100755
--- a/qa/workunits/rbd/rbd_mirror_bootstrap.sh
+++ b/qa/workunits/rbd/rbd_mirror_bootstrap.sh
@@ -1,8 +1,10 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror_bootstrap.sh - test peer bootstrap create/import
 #
 
+set -ex
+
 RBD_MIRROR_MANUAL_PEERS=1
 RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-1}
 . $(dirname $0)/rbd_mirror_helpers.sh
diff --git a/qa/workunits/rbd/rbd_mirror_fsx_compare.sh b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh
index 0ba3c97d7519..79c36546d4fb 100755
--- a/qa/workunits/rbd/rbd_mirror_fsx_compare.sh
+++ b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh
@@ -1,10 +1,12 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror_fsx_compare.sh - test rbd-mirror daemon under FSX workload
 #
 # The script is used to compare FSX-generated images between two clusters.
 #
 
+set -ex
+
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 trap 'cleanup $?' INT TERM EXIT
diff --git a/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh
index d988987ba42a..6daadbbb4501 100755
--- a/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh
+++ b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh
@@ -1,10 +1,12 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror_fsx_prepare.sh - test rbd-mirror daemon under FSX workload
 #
 # The script is used to compare FSX-generated images between two clusters.
 #
 
+set -ex
+
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 setup
diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh
index 37739a83da02..1e43712a6315 100755
--- a/qa/workunits/rbd/rbd_mirror_ha.sh
+++ b/qa/workunits/rbd/rbd_mirror_ha.sh
@@ -1,8 +1,10 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode
 #
 
+set -ex
+
 RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7}
 
 . $(dirname $0)/rbd_mirror_helpers.sh
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
index d7b4abc4d8f2..b6e1fa87de2a 100755
--- a/qa/workunits/rbd/rbd_mirror_helpers.sh
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #
 # rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions
 #
diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
index 5426f495db10..baf0c9f1a8f8 100755
--- a/qa/workunits/rbd/rbd_mirror_stress.sh
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -ex
+#!/usr/bin/env bash
 #
 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
 #
@@ -8,6 +8,8 @@
 #                             tool during the many image test
 #
 
+set -ex
+
 IMAGE_COUNT=50
 export LOCKDEP=0
 

From c8e37d1c6f3e2d6279c098aae47a2e3ab8a57a8a Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 11 Mar 2024 09:48:36 +0800
Subject: [PATCH 2219/2492] ceph.spec.in: use gts_version instead of gts_prefix

since we are going to use different gcc-toolset versions on rhel8 and
on rhel9, and the subpackage names of gcc-toolset changed across
gts 11 and gts 13, let's prepare for this change.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 ceph.spec.in | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 38dd579868cb..46d9b5e86bc7 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -141,7 +141,7 @@
 %{!?python3_pkgversion: %global python3_pkgversion 3}
 %{!?python3_version_nodots: %global python3_version_nodots 3}
 %{!?python3_version: %global python3_version 3}
-%{!?gts_prefix: %global gts_prefix gcc-toolset-11}
+%{!?gts_version: %global gts_version 11}
 
 %if ! 0%{?suse_version}
 # use multi-threaded xz compression: xz level 7 using ncpus threads
@@ -230,9 +230,9 @@ BuildRequires:	gcc-c++ >= 11
 BuildRequires: gcc11-c++
 %endif
 %if 0%{?rhel} == 8
-BuildRequires:	%{gts_prefix}-gcc-c++
-BuildRequires:	%{gts_prefix}-build
-BuildRequires:	%{gts_prefix}-libatomic-devel
+BuildRequires:	gcc-toolset-%{gts_version}-gcc-c++
+BuildRequires:	gcc-toolset-%{gts_version}-build
+BuildRequires:	gcc-toolset-%{gts_version}-libatomic-devel
 %endif
 %if 0%{?fedora} || 0%{?rhel} == 9 || 0%{?openEuler}
 BuildRequires:  libatomic
@@ -359,10 +359,10 @@ BuildRequires:  libasan
 BuildRequires:  protobuf-devel
 BuildRequires:  protobuf-compiler
 %if 0%{?rhel} == 8
-BuildRequires:  %{gts_prefix}-annobin
-BuildRequires:  %{gts_prefix}-annobin-plugin-gcc
-BuildRequires:  %{gts_prefix}-libubsan-devel
-BuildRequires:  %{gts_prefix}-libasan-devel
+BuildRequires:  gcc-toolset-%{gts_version}-annobin
+BuildRequires:  gcc-toolset-%{gts_version}-annobin-plugin-gcc
+BuildRequires:  gcc-toolset-%{gts_version}-libubsan-devel
+BuildRequires:  gcc-toolset-%{gts_version}-libasan-devel
 %endif
 %endif
 #################################################################################

From f5f2e4cc10e56dcb8cf3d72e497a5e400bb324ca Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sat, 9 Mar 2024 12:15:06 +0800
Subject: [PATCH 2220/2492] ceph.spec.in: set `CMAKE_EXE_LINKER_FLAGS` to
 `-lstdc++`

because the gcc-toolset-13 always links against the `libstdc++_nonshared.a`, which misses the
simple symbols. so we have to link against libstdc++ explicitly to assure that the
executables have access to them when compiling with the gcc-toolset toolchain.

see
```
/opt/rh/gcc-toolset-13/root/usr/bin/c++ -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -m64 -march=x86-64-v2 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -rdynamic -pie CMakeFiles/ceph_scratchtool.dir/scratchtool.c.o -o ../../bin/ceph_scratchtool  -Wl,-rpath,/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.0.0-1977-gb0c1f7e8/rpm/el9/BUILD/ceph-19.0.0-1977-gb0c1f7e8/redhat-linux-build/lib: ../../lib/librados.so.2.0.0 ../../lib/libglobal.a ../../lib/libceph-common.so.2 ../../lib/libjson_spirit.a ../../lib/libcommon_utf8.a ../../lib/liberasure_code.a ../../lib/libextblkdev.a -lcap ../../boost/lib/libboost_thread.a ../../boost/lib/libboost_chrono.a ../../boost/lib/libboost_atomic.a ../../boost/lib/libboost_system.a ../../boost/lib/libboost_random.a ../../boost/lib/libboost_program_options.a ../../boost/lib/libboost_date_time.a ../../boost/lib/libboost_iostreams.a ../../boost/lib/libboost_regex.a /usr/lib64/libblkid.so /usr/lib64/libcrypto.so /usr/lib64/libudev.so /usr/lib64/libibverbs.so /usr/lib64/librdmacm.so /usr/lib64/libz.so ../opentelemetry-cpp/sdk/src/trace/libopentelemetry_trace.a ../opentelemetry-cpp/sdk/src/resource/libopentelemetry_resources.a ../opentelemetry-cpp/sdk/src/common/libopentelemetry_common.a ../opentelemetry-cpp/exporters/jaeger/libopentelemetry_exporter_jaeger_trace.a ../opentelemetry-cpp/ext/src/http/client/curl/libopentelemetry_http_client_curl.a /usr/lib64/libcurl.so /usr/lib64/libthrift.so -ldl /usr/lib64/librt.a -lresolv  -Wl,--as-needed -latomic
/opt/rh/gcc-toolset-13/root/usr/libexec/gcc/x86_64-redhat-linux/13/ld: /opt/rh/gcc-toolset-13/root/usr/lib/gcc/x86_64-redhat-linux/13/libstdc++_nonshared.a(ios_init.o): undefined reference to symbol '_ZNSt8ios_base4InitD1Ev@@GLIBCXX_3.4'
/opt/rh/gcc-toolset-13/root/usr/libexec/gcc/x86_64-redhat-linux/13/ld:
/usr/lib64/libstdc++.so.6: error adding symbols: DSO missing from command line
```

this change prepares us for switching to gts-13.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 ceph.spec.in | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 46d9b5e86bc7..b8c67a62ead5 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -1375,6 +1375,9 @@ cmake .. \
 %if 0%{?suse_version} == 1500
     -DCMAKE_C_COMPILER=gcc-11 \
     -DCMAKE_CXX_COMPILER=g++-11 \
+%endif
+%if 0%{?gts_version} == 13
+    -DCMAKE_EXE_LINKER_FLAGS=-lstdc++ \
 %endif
     -DCMAKE_INSTALL_PREFIX=%{_prefix} \
     -DCMAKE_INSTALL_LIBDIR:PATH=%{_libdir} \

From 9fb51bb537e8bd9ea60633091acfc48a58262c3b Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sun, 10 Mar 2024 15:49:35 -0400
Subject: [PATCH 2221/2492] doc/cephadm: improve host-management.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/cephadm/host-management.rst | 58 ++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 27 deletions(-)

diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst
index c1aa88cd60aa..87f201fba65a 100644
--- a/doc/cephadm/host-management.rst
+++ b/doc/cephadm/host-management.rst
@@ -74,9 +74,9 @@ To add each new host to the cluster, perform two steps:
       ceph orch host add host2 10.10.0.102
       ceph orch host add host3 10.10.0.103
 
-   It is best to explicitly provide the host IP address.  If an IP is
+   It is best to explicitly provide the host IP address.  If an address is
    not provided, then the host name will be immediately resolved via
-   DNS and that IP will be used.
+   DNS and the result will be used.
 
    One or more labels can also be included to immediately label the
    new host.  For example, by default the ``_admin`` label will make
@@ -104,7 +104,7 @@ To drain all daemons from a host, run a command of the following form:
 The ``_no_schedule`` and ``_no_conf_keyring`` labels will be applied to the
 host. See :ref:`cephadm-special-host-labels`.
 
-If you only want to drain daemons but leave managed ceph conf and keyring
+If you want to drain daemons but leave managed `ceph.conf` and keyring
 files on the host, you may pass the ``--keep-conf-keyring`` flag to the
 drain command.
 
@@ -115,7 +115,8 @@ drain command.
 This will apply the ``_no_schedule`` label to the host but not the
 ``_no_conf_keyring`` label.
 
-All OSDs on the host will be scheduled to be removed. You can check the progress of the OSD removal operation with the following command:
+All OSDs on the host will be scheduled to be removed. You can check
+progress of the OSD removal operation with the following command:
 
 .. prompt:: bash #
 
@@ -148,7 +149,7 @@ cluster by running the following command:
 Offline host removal
 --------------------
 
-Even if a host is offline and can not be recovered, it can be removed from the
+If a host is offline and can not be recovered, it can be removed from the
 cluster by running a command of the following form:
 
 .. prompt:: bash #
@@ -250,8 +251,8 @@ Rescanning Host Devices
 =======================
 
 Some servers and external enclosures may not register device removal or insertion with the
-kernel. In these scenarios, you'll need to perform a host rescan. A rescan is typically
-non-disruptive, and can be performed with the following CLI command:
+kernel. In these scenarios, you'll need to perform a device rescan on the appropriate host.
+A rescan is typically non-disruptive, and can be performed with the following CLI command:
 
 .. prompt:: bash #
 
@@ -316,40 +317,40 @@ create a new CRUSH host located in the specified hierarchy.
 
   The ``location`` attribute will be only affect the initial CRUSH location. Subsequent
   changes of the ``location`` property will be ignored. Also, removing a host will not remove
-  any CRUSH buckets unless the ``--rm-crush-entry`` flag is provided to the ``orch host rm`` command
+  an associated CRUSH bucket unless the ``--rm-crush-entry`` flag is provided to the ``orch host rm`` command
 
 See also :ref:`crush_map_default_types`.
 
 Removing a host from the CRUSH map
 ==================================
 
-The ``ceph orch host rm`` command has support for removing the bucket entry for the host
-in the CRUSH map. This is done by providing the ``--rm-crush-entry`` flag.
+The ``ceph orch host rm`` command has support for removing the associated host bucket
+from the CRUSH map. This is done by providing the ``--rm-crush-entry`` flag.
 
 .. prompt:: bash [ceph:root@host1/]#
 
    ceph orch host rm host1 --rm-crush-entry
 
-When this flag is specified, cephadm will attempt to remove the bucket entry
-for the host from the CRUSH map as part of the host removal process. Note that if
+When this flag is specified, cephadm will attempt to remove the host bucket
+from the CRUSH map as part of the host removal process. Note that if
 it fails to do so, cephadm will report the failure and the host will remain under
 cephadm control.
 
 .. note:: 
 
-  The removal from the CRUSH map will fail if there are OSDs deployed on the
+  Removal from the CRUSH map will fail if there are OSDs deployed on the
   host. If you would like to remove all the host's OSDs as well, please start
   by using  the ``ceph orch host drain`` command to do so. Once the OSDs
-  are all gone, then you may have cephadm remove the CRUSH entry along with the
-  host using the ``--rm-crush-entry`` flag.
+  have been removed, then you may direct cephadm remove the CRUSH bucket
+  along with the host using the ``--rm-crush-entry`` flag.
 
 OS Tuning Profiles
 ==================
 
-Cephadm can be used to manage operating-system-tuning profiles that apply sets
-of sysctl settings to sets of hosts. 
+Cephadm can be used to manage operating system tuning profiles that apply
+``sysctl`` settings to sets of hosts. 
 
-Create a YAML spec file in the following format:
+To do so, create a YAML spec file in the following format:
 
 .. code-block:: yaml
 
@@ -368,18 +369,21 @@ Apply the tuning profile with the following command:
 
    ceph orch tuned-profile apply -i <tuned-profile-file-name>
 
-This profile is written to ``/etc/sysctl.d/`` on each host that matches the
-hosts specified in the placement block of the yaml, and ``sysctl --system`` is
+This profile is written to a file under ``/etc/sysctl.d/`` on each host
+specified in the ``placement`` block, then ``sysctl --system`` is
 run on the host.
 
 .. note::
 
   The exact filename that the profile is written to within ``/etc/sysctl.d/``
   is ``<profile-name>-cephadm-tuned-profile.conf``, where ``<profile-name>`` is
-  the ``profile_name`` setting that you specify in the YAML spec. Because
+  the ``profile_name`` setting that you specify in the YAML spec. We suggest
+  naming these profiles following the usual ``sysctl.d`` `NN-xxxxx` convention. Because
   sysctl settings are applied in lexicographical order (sorted by the filename
-  in which the setting is specified), you may want to set the ``profile_name``
-  in your spec so that it is applied before or after other conf files.
+  in which the setting is specified), you may want to carefully choose
+  the ``profile_name`` in your spec so that it is applied before or after other
+  conf files.  Careful selection ensures that values supplied here override or
+  do not override those in other ``sysctl.d`` files as desired.
 
 .. note::
 
@@ -388,7 +392,7 @@ run on the host.
 
 .. note::
 
-  Applying tuned profiles is idempotent when the ``--no-overwrite`` option is
+  Applying tuning profiles is idempotent when the ``--no-overwrite`` option is
   passed. Moreover, if the ``--no-overwrite`` option is passed, existing
   profiles with the same name are not overwritten.
 
@@ -548,7 +552,7 @@ There are two ways to customize this configuration for your environment:
 
    We do *not recommend* this approach.  The path name must be
    visible to *any* mgr daemon, and cephadm runs all daemons as
-   containers. That means that the file either need to be placed
+   containers. That means that the file must either be placed
    inside a customized container image for your deployment, or
    manually distributed to the mgr data directory
    (``/var/lib/ceph/<cluster-fsid>/mgr.<id>`` on the host, visible at
@@ -601,8 +605,8 @@ Note that ``man hostname`` recommends ``hostname`` to return the bare
 host name:
 
     The FQDN (Fully Qualified Domain Name) of the system is the
-    name that the resolver(3) returns for the host name, such as,
-    ursula.example.com. It is usually the hostname followed by the DNS
+    name that the resolver(3) returns for the host name, for example
+    ``ursula.example.com``. It is usually the short hostname followed by the DNS
     domain name (the part after the first dot). You can check the FQDN
     using ``hostname --fqdn`` or the domain name using ``dnsdomainname``.
 

From 7b7e5d4221389ce077780450d78a4a1e6a3d0a43 Mon Sep 17 00:00:00 2001
From: Jos Collin <jcollin@redhat.com>
Date: Thu, 29 Feb 2024 16:20:03 +0530
Subject: [PATCH 2222/2492] qa: enhance labeled perf counters test for
 cephfs-mirror

Implements checks for labeled perf counters in the appropriate tests.
This patch verifies snaps_synced, snaps_renamed, snaps_deleted and sync_failures metrics are
updated correctly based on the tests.

Fixes: https://tracker.ceph.com/issues/64486
Signed-off-by: Jos Collin <jcollin@redhat.com>
---
 qa/tasks/cephfs/test_mirroring.py | 111 +++++++++++++++++++++++++++++-
 1 file changed, 110 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/test_mirroring.py b/qa/tasks/cephfs/test_mirroring.py
index bad2d2daffdc..6e57df5d0a84 100644
--- a/qa/tasks/cephfs/test_mirroring.py
+++ b/qa/tasks/cephfs/test_mirroring.py
@@ -546,6 +546,10 @@ def test_cephfs_mirror_stats(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        first = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+
         # take a snapshot
         self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
 
@@ -554,6 +558,11 @@ def test_cephfs_mirror_stats(self):
                                "client.mirror_remote@ceph", '/d0', 'snap0', 1)
         self.verify_snapshot('d0', 'snap0')
 
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        second = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(second["counters"]["snaps_synced"], first["counters"]["snaps_synced"])
+
         # some more IO
         self.mount_a.run_shell(["mkdir", "d0/d00"])
         self.mount_a.run_shell(["mkdir", "d0/d01"])
@@ -569,6 +578,11 @@ def test_cephfs_mirror_stats(self):
                                "client.mirror_remote@ceph", '/d0', 'snap1', 2)
         self.verify_snapshot('d0', 'snap1')
 
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        third = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(third["counters"]["snaps_synced"], second["counters"]["snaps_synced"])
+
         # delete a snapshot
         self.mount_a.run_shell(["rmdir", "d0/.snap/snap0"])
 
@@ -577,6 +591,10 @@ def test_cephfs_mirror_stats(self):
         self.assertTrue('snap0' not in snap_list)
         self.check_peer_status_deleted_snap(self.primary_fs_name, self.primary_fs_id,
                                             "client.mirror_remote@ceph", '/d0', 1)
+        # check snaps_deleted
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        fourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(fourth["counters"]["snaps_deleted"], third["counters"]["snaps_deleted"])
 
         # rename a snapshot
         self.mount_a.run_shell(["mv", "d0/.snap/snap1", "d0/.snap/snap2"])
@@ -587,6 +605,10 @@ def test_cephfs_mirror_stats(self):
         self.assertTrue('snap2' in snap_list)
         self.check_peer_status_renamed_snap(self.primary_fs_name, self.primary_fs_id,
                                             "client.mirror_remote@ceph", '/d0', 1)
+        # check snaps_renamed
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        fifth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(fifth["counters"]["snaps_renamed"], fourth["counters"]["snaps_renamed"])
 
         self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -626,6 +648,12 @@ def test_cephfs_mirror_cancel_sync(self):
 
         snap_list = self.mount_b.ls(path='d0/.snap')
         self.assertTrue('snap0' not in snap_list)
+
+        # check sync_failures
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vmirror_peers = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vmirror_peers["counters"]["sync_failures"], 0)
+
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
     def test_cephfs_mirror_restart_sync_on_blocklist(self):
@@ -655,6 +683,10 @@ def test_cephfs_mirror_restart_sync_on_blocklist(self):
         # fetch rados address for blacklist check
         rados_inst = self.get_mirror_rados_addr(self.primary_fs_name, self.primary_fs_id)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+
         # take a snapshot
         self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
 
@@ -683,6 +715,10 @@ def test_cephfs_mirror_restart_sync_on_blocklist(self):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/d0', 'snap0', expected_snap_count=1)
         self.verify_snapshot('d0', 'snap0')
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
 
         self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -691,6 +727,10 @@ def test_cephfs_mirror_failed_sync_with_correction(self):
         self.enable_mirroring(self.primary_fs_name, self.primary_fs_id)
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+
         # add a non-existent directory for synchronization
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
 
@@ -707,6 +747,10 @@ def test_cephfs_mirror_failed_sync_with_correction(self):
         time.sleep(120)
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/d0', 'snap0', 1)
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
     def test_cephfs_mirror_service_daemon_status(self):
@@ -760,7 +804,7 @@ def test_mirroring_init_failure(self):
         self.disable_mirroring_module()
 
         # enable mirroring through mon interface -- this should result in the mirror daemon
-        # failing to enable mirroring due to absence of `cephfs_mirorr` index object.
+        # failing to enable mirroring due to absence of `cephfs_mirror` index object.
         self.run_ceph_cmd("fs", "mirror", "enable", self.primary_fs_name)
 
         with safe_while(sleep=5, tries=10, action='wait for failed state') as proceed:
@@ -888,6 +932,10 @@ def test_cephfs_mirror_symlink_sync(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+
         # take a snapshot
         self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
 
@@ -896,6 +944,10 @@ def test_cephfs_mirror_symlink_sync(self):
                                "client.mirror_remote@ceph", '/d0', 'snap0', 1)
         self.verify_snapshot('d0', 'snap0')
 
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
         self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0')
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
@@ -907,12 +959,20 @@ def test_cephfs_mirror_with_parent_snapshot(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3')
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+
         # take a snapshot
         self.mount_a.run_shell(["mkdir", "d0/d1/d2/d3/.snap/snap0"])
 
         time.sleep(30)
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap0', 1)
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
 
         # create snapshots in parent directories
         self.mount_a.run_shell(["mkdir", "d0/.snap/snap_d0"])
@@ -924,12 +984,20 @@ def test_cephfs_mirror_with_parent_snapshot(self):
         time.sleep(30)
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap1', 2)
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
 
         self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap0"])
         self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap1"])
         time.sleep(15)
         self.check_peer_status_deleted_snap(self.primary_fs_name, self.primary_fs_id,
                                             "client.mirror_remote@ceph", '/d0/d1/d2/d3', 2)
+        # check snaps_deleted
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vfourth["counters"]["snaps_deleted"], vthird["counters"]["snaps_deleted"])
 
         self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3')
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
@@ -1033,6 +1101,9 @@ def exec_git_cmd(cmd_list):
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
         self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}')
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
         self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a'])
 
         # full copy, takes time
@@ -1040,6 +1111,10 @@ def exec_git_cmd(cmd_list):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1)
         self.verify_snapshot(repo_path, 'snap_a')
+        # check snaps_synced
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
 
         # create some diff
         num = random.randint(5, 20)
@@ -1052,6 +1127,9 @@ def exec_git_cmd(cmd_list):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2)
         self.verify_snapshot(repo_path, 'snap_b')
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
 
         # diff again, this time back to HEAD
         log.debug('resetting to HEAD')
@@ -1063,6 +1141,9 @@ def exec_git_cmd(cmd_list):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", f'/{repo_path}', 'snap_c', 3)
         self.verify_snapshot(repo_path, 'snap_c')
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfourth = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vfourth["counters"]["snaps_synced"], vthird["counters"]["snaps_synced"])
 
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
@@ -1133,11 +1214,18 @@ def verify_types(dirname, fnames, snap_name):
         while turns != len(typs):
             snapname = f'snap_{turns}'
             cleanup_and_create_with_type('d0', fnames)
+            # dump perf counters
+            res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+            vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
             self.mount_a.run_shell(['mkdir', f'd0/.snap/{snapname}'])
             time.sleep(30)
             self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                    "client.mirror_remote@ceph", '/d0', snapname, turns+1)
             verify_types('d0', fnames, snapname)
+            res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+            vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+            self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
+
             # next type
             typs.rotate(1)
             turns += 1
@@ -1184,6 +1272,9 @@ def exec_git_cmd(cmd_list):
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
         self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}')
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vfirst = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
         self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a'])
 
         # full copy, takes time
@@ -1191,6 +1282,9 @@ def exec_git_cmd(cmd_list):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1)
         self.verify_snapshot(repo_path, 'snap_a')
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vsecond = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vsecond["counters"]["snaps_synced"], vfirst["counters"]["snaps_synced"])
 
         # create some diff
         num = random.randint(60, 100)
@@ -1207,6 +1301,9 @@ def exec_git_cmd(cmd_list):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2)
         self.verify_snapshot(repo_path, 'snap_b')
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vthird = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vthird["counters"]["snaps_synced"], vsecond["counters"]["snaps_synced"])
 
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
@@ -1265,6 +1362,9 @@ def test_cephfs_mirror_cancel_mirroring_and_readd(self):
         self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d2')
         self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name)
 
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vbefore = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
         # take snapshots
         log.debug('taking snapshots')
         self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"])
@@ -1326,6 +1426,10 @@ def test_cephfs_mirror_cancel_mirroring_and_readd(self):
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/d2', 'snap0', 1)
         self.verify_snapshot('d2', 'snap0')
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vafter = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        self.assertGreater(vafter["counters"]["snaps_synced"], vbefore["counters"]["snaps_synced"])
+        self.assertGreater(vafter["counters"]["snaps_deleted"], vbefore["counters"]["snaps_deleted"])
 
         self.disable_mirroring(self.primary_fs_name, self.primary_fs_id)
 
@@ -1354,6 +1458,11 @@ def test_local_and_remote_dir_root_mode(self):
         time.sleep(60)
         self.check_peer_status(self.primary_fs_name, self.primary_fs_id,
                                "client.mirror_remote@ceph", '/l1', 'snap0', 1)
+        # dump perf counters
+        res = self.mirror_daemon_command(f'counter dump for fs: {self.primary_fs_name}', 'counter', 'dump')
+        vmirror_peers = res[TestMirroring.PERF_COUNTER_KEY_NAME_CEPHFS_MIRROR_PEER][0]
+        snaps_synced = vmirror_peers["counters"]["snaps_synced"]
+        self.assertEqual(snaps_synced, 1, f"Mismatch snaps_synced: {snaps_synced} vs 1")
 
         mode_local = self.mount_a.run_shell(["stat", "--format=%A", "l1"]).stdout.getvalue().strip()
         mode_remote = self.mount_b.run_shell(["stat", "--format=%A", "l1"]).stdout.getvalue().strip()

From 70245c5b207eca6349920d1877ea237ec687f93d Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Tue, 5 Mar 2024 15:58:05 +0800
Subject: [PATCH 2223/2492] crimson/os/seastore: handle OP_COLL_MOVE_RENAME

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 53 ++++++++++++++++++++++++++++-
 src/crimson/os/seastore/seastore.h  |  4 +++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 961fd04bd438..e29fd4c0ea67 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -1347,7 +1347,9 @@ SeaStore::Shard::_do_transaction_step(
       o = get_onode;
       d_onodes[op->oid] = get_onode;
     }
-    if (op->op == Transaction::OP_CLONE && !d_onodes[op->dest_oid]) {
+    if ((op->op == Transaction::OP_CLONE
+	  || op->op == Transaction::OP_COLL_MOVE_RENAME)
+	&& !d_onodes[op->dest_oid]) {
       //TODO: use when_all_succeed after making onode tree
       //      support parallel extents loading
       return onode_manager->get_or_create_onode(
@@ -1471,6 +1473,20 @@ SeaStore::Shard::_do_transaction_step(
 	  i.get_oid(op->dest_oid));
 	return _clone(ctx, onodes[op->oid], d_onodes[op->dest_oid]);
       }
+      case Transaction::OP_COLL_MOVE_RENAME:
+      {
+	ceph_assert(op->cid == op->dest_cid);
+	TRACET("renaming {} to {}",
+	  *ctx.transaction,
+	  i.get_oid(op->oid),
+	  i.get_oid(op->dest_oid));
+	return _rename(
+	  ctx, onodes[op->oid], d_onodes[op->dest_oid]
+	).si_then([&onodes, &d_onodes, op] {
+	  onodes[op->oid].reset();
+	  d_onodes[op->oid].reset();
+	});
+      }
       default:
         ERROR("bad op {}", static_cast<unsigned>(op->op));
         return crimson::ct_error::input_output_error::make();
@@ -1504,6 +1520,41 @@ SeaStore::Shard::_do_transaction_step(
   );
 }
 
+SeaStore::Shard::tm_ret
+SeaStore::Shard::_rename(
+  internal_context_t &ctx,
+  OnodeRef &onode,
+  OnodeRef &d_onode)
+{
+  auto olayout = onode->get_layout();
+  uint32_t size = olayout.size;
+  auto omap_root = olayout.omap_root.get(
+    d_onode->get_metadata_hint(device->get_block_size()));
+  auto xattr_root = olayout.xattr_root.get(
+    d_onode->get_metadata_hint(device->get_block_size()));
+  auto object_data = olayout.object_data.get();
+  auto oi_bl = ceph::bufferlist::static_from_mem(
+    &olayout.oi[0],
+    (uint32_t)olayout.oi_size);
+  auto ss_bl = ceph::bufferlist::static_from_mem(
+    &olayout.ss[0],
+    (uint32_t)olayout.ss_size);
+
+  d_onode->update_onode_size(*ctx.transaction, size);
+  d_onode->update_omap_root(*ctx.transaction, omap_root);
+  d_onode->update_xattr_root(*ctx.transaction, xattr_root);
+  d_onode->update_object_data(*ctx.transaction, object_data);
+  d_onode->update_object_info(*ctx.transaction, oi_bl);
+  d_onode->update_snapset(*ctx.transaction, ss_bl);
+  return onode_manager->erase_onode(
+    *ctx.transaction, onode
+  ).handle_error_interruptible(
+    crimson::ct_error::input_output_error::pass_further(),
+    crimson::ct_error::assert_all{
+      "Invalid error in SeaStore::_rename"}
+  );
+}
+
 SeaStore::Shard::tm_ret
 SeaStore::Shard::_remove_omaps(
   internal_context_t &ctx,
diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index fe1aa143e1c1..585ce735dd28 100644
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -377,6 +377,10 @@ class SeaStore final : public FuturizedStore {
       internal_context_t &ctx,
       OnodeRef &onode,
       OnodeRef &d_onode);
+    tm_ret _rename(
+      internal_context_t &ctx,
+      OnodeRef &onode,
+      OnodeRef &d_onode);
     tm_ret _zero(
       internal_context_t &ctx,
       OnodeRef &onode,

From 2520927590ac48783cb35feb570285ec066964a4 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 7 Mar 2024 17:06:13 +0800
Subject: [PATCH 2224/2492] test/crimson/seastore: add the test case for
 renaming objects

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/test/crimson/seastore/test_seastore.cc | 32 ++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/src/test/crimson/seastore/test_seastore.cc b/src/test/crimson/seastore/test_seastore.cc
index 1e0028b97aca..a252cbfd5782 100644
--- a/src/test/crimson/seastore/test_seastore.cc
+++ b/src/test/crimson/seastore/test_seastore.cc
@@ -262,6 +262,19 @@ struct seastore_test_t :
       return clone_obj;
     }
 
+    void rename(
+      SeaStoreShard &sharded_seastore,
+      object_state_t &other) {
+      CTransaction t;
+      t.collection_move_rename(cid, oid, cid, other.oid);
+      sharded_seastore.do_transaction(
+	coll,
+	std::move(t)).get0();
+      other.contents = contents;
+      other.omap = omap;
+      other.clone_contents = clone_contents;
+    }
+
     void write(
       SeaStoreShard &sharded_seastore,
       uint64_t offset,
@@ -787,6 +800,25 @@ TEST_P(seastore_test_t, omap_test_simple)
   });
 }
 
+TEST_P(seastore_test_t, rename)
+{
+  run_async([this] {
+    auto &test_obj = get_object(make_oid(0));
+    test_obj.write(*sharded_seastore, 0, 4096, 'a');
+    test_obj.set_omap(
+      *sharded_seastore,
+      "asdf",
+      make_bufferlist(128));
+    auto test_other = object_state_t{
+      test_obj.cid, 
+      test_obj.coll,
+      ghobject_t(hobject_t(sobject_t(std::string("object_1"), CEPH_NOSNAP)))};
+    test_obj.rename(*sharded_seastore, test_other);
+    test_other.read(*sharded_seastore, 0, 4096);
+    test_other.check_omap(*sharded_seastore);
+  });
+}
+
 TEST_P(seastore_test_t, clone_aligned_extents)
 {
   run_async([this] {

From a164f5f6d8f5fa287f938db5675ea0814c39380e Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Fri, 8 Mar 2024 04:03:03 +0000
Subject: [PATCH 2225/2492] ceph.spec.in: disable lto to work around
 gcc-toolset-13 linker bug

gcc-toolset-13 lto can trigger a linker bug resulting in a segfault in
SafeTimer (and perhaps elsewhere).  See
https://tracker.ceph.com/issues/63867 for details.  This patch disables
lto for now now so that we can switch to gcc-toolset-13.

Fixes: https://tracker.ceph.com/issues/63867
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 ceph.spec.in | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index b8c67a62ead5..6e5f1a0ec2e7 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -143,6 +143,19 @@
 %{!?python3_version: %global python3_version 3}
 %{!?gts_version: %global gts_version 11}
 
+# gcc-toolset-13 seems to trigger a linker bug resulting in a segfault in SafeTimer
+# and perhaps elsewhere.  For now, let's just disable it.  See
+# ceph bug https://tracker.ceph.com/issues/63867
+# and
+# gcc bug https://bugzilla.redhat.com/show_bug.cgi?id=2241339
+# for details.
+#
+# Also disable lto on systems that do not support symver attribute
+# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200 for details
+%if 0%{?gts_version} == 13 || (0%{?rhel} && 0%{?rhel} < 9)  || ( 0%{?suse_version} && 0%{?suse_version} <= 1500 )
+%define _lto_cflags %{nil}
+%endif
+
 %if ! 0%{?suse_version}
 # use multi-threaded xz compression: xz level 7 using ncpus threads
 %global _source_payload w7T%{_smp_build_ncpus}.xzdio
@@ -1334,11 +1347,6 @@ This package provides a Ceph hardware monitoring agent.
 %autosetup -p1 -n @TARBALL_BASENAME@
 
 %build
-# Disable lto on systems that do not support symver attribute
-# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200 for details
-%if ( 0%{?rhel} && 0%{?rhel} < 9 ) || ( 0%{?suse_version} && 0%{?suse_version} <= 1500 )
-%define _lto_cflags %{nil}
-%endif
 
 %if 0%{with cephfs_java}
 # Find jni.h

From 55da694632505cbcbb07d5a743f7809daab13d41 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sat, 2 Mar 2024 12:38:00 +0800
Subject: [PATCH 2226/2492] ceph.spec.in: use gcc-toolset 13 for crimson build

since gts 13 is out, and GCC-13 brings better support of C++20 coroutines,
and because clang prefers using a newer gts when its gcc is around. so
let's bump up the gcc-toolset from 11 to 13 when building crimson. because
gcc-toolset-13 LTO triggers a linker bug resulting in a segfault in SafeTimer,
see https://tracker.ceph.com/issues/63867 , we cannot switch the classic
build to gts 13 without proving that it does not incur performance
regressions.

since annobin plugin package was renamed to
gcc-toolset-13-gcc-plugin-annobin, let's update its name accordingly.

and use -runtime subpackage instead of -build, as
macros.gcc-toolset-13-enable is now located in -runtime subpackage
since devtoolset12

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 ceph.spec.in | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index 6e5f1a0ec2e7..f8ec72acef76 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -141,7 +141,11 @@
 %{!?python3_pkgversion: %global python3_pkgversion 3}
 %{!?python3_version_nodots: %global python3_version_nodots 3}
 %{!?python3_version: %global python3_version 3}
+%if 0%{with seastar}
+%{!?gts_version: %global gts_version 13}
+%else
 %{!?gts_version: %global gts_version 11}
+%endif
 
 # gcc-toolset-13 seems to trigger a linker bug resulting in a segfault in SafeTimer
 # and perhaps elsewhere.  For now, let's just disable it.  See
@@ -186,9 +190,17 @@
 # do not provide gcc-annobin.so anymore, despite that they provide annobin.so. but
 # redhat-rpm-config still passes -fplugin=gcc-annobin to the compiler.
 %undefine _annotated_build
-%if 0%{?rhel} == 8 && 0%{?enable_devtoolset11:1}
+%if 0%{?rhel} == 8
+%if 0%{gts_version} == 13
+%if 0%{?enable_devtoolset13:1}
+%enable_devtoolset13
+%endif
+%else
+%if 0%{?enable_devtoolset11:1}
 %enable_devtoolset11
 %endif
+%endif
+%endif
 
 #################################################################################
 # main package definition
@@ -244,7 +256,11 @@ BuildRequires: gcc11-c++
 %endif
 %if 0%{?rhel} == 8
 BuildRequires:	gcc-toolset-%{gts_version}-gcc-c++
+%if 0%{?gts_version} >= 12
+BuildRequires:	gcc-toolset-%{gts_version}-runtime
+%else
 BuildRequires:	gcc-toolset-%{gts_version}-build
+%endif
 BuildRequires:	gcc-toolset-%{gts_version}-libatomic-devel
 %endif
 %if 0%{?fedora} || 0%{?rhel} == 9 || 0%{?openEuler}
@@ -372,8 +388,12 @@ BuildRequires:  libasan
 BuildRequires:  protobuf-devel
 BuildRequires:  protobuf-compiler
 %if 0%{?rhel} == 8
+%if 0%{?gts_version} >= 12
+BuildRequires:  gcc-toolset-%{gts_version}-gcc-plugin-annobin
+%else
 BuildRequires:  gcc-toolset-%{gts_version}-annobin
 BuildRequires:  gcc-toolset-%{gts_version}-annobin-plugin-gcc
+%endif
 BuildRequires:  gcc-toolset-%{gts_version}-libubsan-devel
 BuildRequires:  gcc-toolset-%{gts_version}-libasan-devel
 %endif

From 913d127c8116ef6f0066d75cfdbba3811939936d Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sat, 2 Mar 2024 12:44:56 +0800
Subject: [PATCH 2227/2492] ceph.spec.in: use gcc-toolset when with seastar or
 rhel 8

both RHEL8 and RHEL9 have gcc-toolset 13, and we need to use gts-13
for building crimson, so let's enable it when building crimson,
and we need to use gts-11 when building on RHEL7. hence this change.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 ceph.spec.in | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ceph.spec.in b/ceph.spec.in
index f8ec72acef76..52a8faad5fff 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -144,8 +144,10 @@
 %if 0%{with seastar}
 %{!?gts_version: %global gts_version 13}
 %else
+%if 0%{?rhel} == 8
 %{!?gts_version: %global gts_version 11}
 %endif
+%endif
 
 # gcc-toolset-13 seems to trigger a linker bug resulting in a segfault in SafeTimer
 # and perhaps elsewhere.  For now, let's just disable it.  See
@@ -190,7 +192,7 @@
 # do not provide gcc-annobin.so anymore, despite that they provide annobin.so. but
 # redhat-rpm-config still passes -fplugin=gcc-annobin to the compiler.
 %undefine _annotated_build
-%if 0%{?rhel} == 8
+%if 0%{?gts_version} > 0
 %if 0%{gts_version} == 13
 %if 0%{?enable_devtoolset13:1}
 %enable_devtoolset13
@@ -254,7 +256,7 @@ BuildRequires:	gcc-c++ >= 11
 %if 0%{?suse_version} == 1500
 BuildRequires: gcc11-c++
 %endif
-%if 0%{?rhel} == 8
+%if 0%{?gts_version} > 0
 BuildRequires:	gcc-toolset-%{gts_version}-gcc-c++
 %if 0%{?gts_version} >= 12
 BuildRequires:	gcc-toolset-%{gts_version}-runtime
@@ -387,7 +389,7 @@ BuildRequires:  libubsan
 BuildRequires:  libasan
 BuildRequires:  protobuf-devel
 BuildRequires:  protobuf-compiler
-%if 0%{?rhel} == 8
+%if 0%{?gts_version} > 0
 %if 0%{?gts_version} >= 12
 BuildRequires:  gcc-toolset-%{gts_version}-gcc-plugin-annobin
 %else

From 800dd29e60fcd2bcd27db56d3fe45c58ddf10c8a Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:04:47 -0400
Subject: [PATCH 2228/2492] doc/cephadm: Improve multiple files

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/cephadm/adoption.rst        | 63 ++++++++++++++++---------------
 doc/cephadm/client-setup.rst    | 36 ++++++++++--------
 doc/cephadm/compatibility.rst   |  4 +-
 doc/cephadm/install.rst         | 67 +++++++++++++++++----------------
 doc/cephadm/operations.rst      | 53 +++++++++++++-------------
 doc/cephadm/troubleshooting.rst | 11 +++---
 doc/cephadm/upgrade.rst         | 14 +++----
 7 files changed, 129 insertions(+), 119 deletions(-)

diff --git a/doc/cephadm/adoption.rst b/doc/cephadm/adoption.rst
index 86254a16cd41..2ebce606c4f0 100644
--- a/doc/cephadm/adoption.rst
+++ b/doc/cephadm/adoption.rst
@@ -22,20 +22,20 @@ Preparation
 #. Make sure that the ``cephadm`` command line tool is available on each host
    in the existing cluster.  See :ref:`get-cephadm` to learn how.
 
-#. Prepare each host for use by ``cephadm`` by running this command:
+#. Prepare each host for use by ``cephadm`` by running this command on that host:
 
    .. prompt:: bash #
 
       cephadm prepare-host
 
 #. Choose a version of Ceph to use for the conversion. This procedure will work
-   with any release of Ceph that is Octopus (15.2.z) or later, inclusive.  The
+   with any release of Ceph that is Octopus (15.2.z) or later.  The
    latest stable release of Ceph is the default. You might be upgrading from an
    earlier Ceph release at the same time that you're performing this
-   conversion; if you are upgrading from an earlier release, make sure to
+   conversion.  If you are upgrading from an earlier release, make sure to
    follow any upgrade-related instructions for that release.
 
-   Pass the image to cephadm with the following command:
+   Pass the Ceph container image to cephadm with the following command:
 
    .. prompt:: bash #
 
@@ -50,25 +50,27 @@ Preparation
 
       cephadm ls
 
-   Before starting the conversion process, ``cephadm ls`` shows all existing
-   daemons to have a style of ``legacy``. As the adoption process progresses,
-   adopted daemons will appear with a style of ``cephadm:v1``.
+   Before starting the conversion process, ``cephadm ls`` reports all existing
+   daemons with the style ``legacy``. As the adoption process progresses,
+   adopted daemons will appear with the style ``cephadm:v1``.
 
 
 Adoption process
 ----------------
 
-#. Make sure that the ceph configuration has been migrated to use the cluster
-   config database.  If the ``/etc/ceph/ceph.conf`` is identical on each host,
-   then the following command can be run on one single host and will affect all
-   hosts:
+#. Make sure that the ceph configuration has been migrated to use the cluster's
+   central config database.  If ``/etc/ceph/ceph.conf`` is identical on all
+   hosts, then the following command can be run on one host and will take
+   effect for all hosts:
 
    .. prompt:: bash #
 
       ceph config assimilate-conf -i /etc/ceph/ceph.conf
 
    If there are configuration variations between hosts, you will need to repeat
-   this command on each host. During this adoption process, view the cluster's
+   this command on each host, taking care that if there are conflicting option
+   settings across hosts, the values from the last host will be used. During this
+   adoption process, view the cluster's central
    configuration to confirm that it is complete by running the following
    command:
 
@@ -76,36 +78,36 @@ Adoption process
 
       ceph config dump
 
-#. Adopt each monitor:
+#. Adopt each Monitor:
 
    .. prompt:: bash #
 
       cephadm adopt --style legacy --name mon.<hostname>
 
-   Each legacy monitor should stop, quickly restart as a cephadm
+   Each legacy Monitor will stop, quickly restart as a cephadm
    container, and rejoin the quorum.
 
-#. Adopt each manager:
+#. Adopt each Manager:
 
    .. prompt:: bash #
 
       cephadm adopt --style legacy --name mgr.<hostname>
 
-#. Enable cephadm:
+#. Enable cephadm orchestration:
 
    .. prompt:: bash #
 
       ceph mgr module enable cephadm
       ceph orch set backend cephadm
 
-#. Generate an SSH key:
+#. Generate an SSH key for cephadm:
 
    .. prompt:: bash #
 
       ceph cephadm generate-key
       ceph cephadm get-pub-key > ~/ceph.pub
 
-#. Install the cluster SSH key on each host in the cluster:
+#. Install the cephadm SSH key on each host in the cluster:
 
    .. prompt:: bash #
 
@@ -118,9 +120,10 @@ Adoption process
      SSH keys.
 
    .. note::
-     It is also possible to have cephadm use a non-root user to SSH 
+     It is also possible to arrange for cephadm to use a non-root user to SSH 
      into cluster hosts. This user needs to have passwordless sudo access.
-     Use ``ceph cephadm set-user <user>`` and copy the SSH key to that user.
+     Use ``ceph cephadm set-user <user>`` and copy the SSH key to that user's
+     home directory on each host.
      See :ref:`cephadm-ssh-user`
 
 #. Tell cephadm which hosts to manage:
@@ -129,10 +132,10 @@ Adoption process
 
       ceph orch host add <hostname> [ip-address]
 
-   This will perform a ``cephadm check-host`` on each host before adding it;
-   this check ensures that the host is functioning properly. The IP address
-   argument is recommended; if not provided, then the host name will be resolved
-   via DNS.
+   This will run ``cephadm check-host`` on each host before adding it.
+   This check ensures that the host is functioning properly. The IP address
+   argument is recommended. If the address is not provided, then the host name
+   will be resolved via DNS.
 
 #. Verify that the adopted monitor and manager daemons are visible:
 
@@ -153,8 +156,8 @@ Adoption process
       cephadm adopt --style legacy --name osd.1
       cephadm adopt --style legacy --name osd.2
 
-#. Redeploy MDS daemons by telling cephadm how many daemons to run for
-   each file system. List file systems by name with the command ``ceph fs
+#. Redeploy CephFS MDS daemons (if deployed) by telling cephadm how many daemons to run for
+   each file system. List CephFS file systems by name with the command ``ceph fs
    ls``. Run the following command on the master nodes to redeploy the MDS
    daemons:
 
@@ -189,19 +192,19 @@ Adoption process
       systemctl stop ceph-mds.target
       rm -rf /var/lib/ceph/mds/ceph-*
 
-#. Redeploy RGW daemons. Cephadm manages RGW daemons by zone. For each
-   zone, deploy new RGW daemons with cephadm:
+#. Redeploy Ceph Object Gateway RGW daemons if deployed. Cephadm manages RGW
+   daemons by zone. For each zone, deploy new RGW daemons with cephadm:
 
    .. prompt:: bash #
 
       ceph orch apply rgw <svc_id> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>]
 
    where *<placement>* can be a simple daemon count, or a list of
-   specific hosts (see :ref:`orchestrator-cli-placement-spec`), and the
+   specific hosts (see :ref:`orchestrator-cli-placement-spec`). The
    zone and realm arguments are needed only for a multisite setup.
 
    After the daemons have started and you have confirmed that they are
-   functioning, stop and remove the old, legacy daemons:
+   functioning, stop and remove the legacy daemons:
 
    .. prompt:: bash #
 
diff --git a/doc/cephadm/client-setup.rst b/doc/cephadm/client-setup.rst
index f98ba798b5fd..0f38773b12bd 100644
--- a/doc/cephadm/client-setup.rst
+++ b/doc/cephadm/client-setup.rst
@@ -1,36 +1,36 @@
 =======================
 Basic Ceph Client Setup
 =======================
-Client machines require some basic configuration to interact with
-Ceph clusters. This section describes how to configure a client machine
-so that it can interact with a Ceph cluster.
+Client hosts require basic configuration to interact with
+Ceph clusters. This section describes how to perform this configuration.
 
 .. note:: 
-   Most client machines need to install only the `ceph-common` package
-   and its dependencies. Such a setup supplies the basic `ceph` and
-   `rados` commands, as well as other commands including `mount.ceph`
-   and `rbd`.
+   Most client hosts need to install only the ``ceph-common`` package
+   and its dependencies. Such an installation supplies the basic ``ceph`` and
+   ``rados`` commands, as well as other commands including ``mount.ceph``
+   and ``rbd``.
 
 Config File Setup
 =================
-Client machines usually require smaller configuration files (here
-sometimes called "config files") than do full-fledged cluster members.
+Client hosts  usually require smaller configuration files (here
+sometimes called "config files") than do back-end cluster hosts.
 To generate a minimal config file, log into a host that has been
-configured as a client or that is running a cluster daemon, and then run the following command:
+configured as a client or that is running a cluster daemon, then
+run the following command:
 
 .. prompt:: bash #
 
   ceph config generate-minimal-conf
 
 This command generates a minimal config file that tells the client how
-to reach the Ceph monitors. The contents of this file should usually 
-be installed in ``/etc/ceph/ceph.conf``.
+to reach the Ceph Monitors. This file should usually 
+be copied to ``/etc/ceph/ceph.conf`` on each client host.
 
 Keyring Setup
 =============
 Most Ceph clusters run with authentication enabled. This means that
-the client needs keys in order to communicate with the machines in the
-cluster. To generate a keyring file with credentials for `client.fs`,
+the client needs keys in order to communicate with Ceph daemons.
+To generate a keyring file with credentials for ``client.fs``,
 log into an running cluster member and run the following command:
 
 .. prompt:: bash $
@@ -40,6 +40,10 @@ log into an running cluster member and run the following command:
 The resulting output is directed into a keyring file, typically
 ``/etc/ceph/ceph.keyring``.
 
-To gain a broader understanding of client keyring distribution and administration, you should read :ref:`client_keyrings_and_configs`.
+To gain a broader understanding of client keyring distribution and administration,
+you should read :ref:`client_keyrings_and_configs`.
 
-To see an example that explains how to distribute ``ceph.conf`` configuration files to hosts that are tagged with the ``bare_config`` label, you should read the section called "Distributing ceph.conf to hosts tagged with bare_config" in the section called :ref:`etc_ceph_conf_distribution`.
+To see an example that explains how to distribute ``ceph.conf`` configuration
+files to hosts that are tagged with the ``bare_config`` label, you should read
+the subsection named "Distributing ceph.conf to hosts tagged with bare_config"
+under the heading :ref:`etc_ceph_conf_distribution`.
diff --git a/doc/cephadm/compatibility.rst b/doc/cephadm/compatibility.rst
index 46ab62a62726..8dd301f1a222 100644
--- a/doc/cephadm/compatibility.rst
+++ b/doc/cephadm/compatibility.rst
@@ -30,8 +30,8 @@ This table shows which version pairs are expected to work or not work together:
 
 .. note::
 
-  While not all podman versions have been actively tested against
-  all Ceph versions, there are no known issues with using podman
+  While not all Podman versions have been actively tested against
+  all Ceph versions, there are no known issues with using Podman
   version 3.0 or greater with Ceph Quincy and later releases.
 
 .. warning:: 
diff --git a/doc/cephadm/install.rst b/doc/cephadm/install.rst
index 52023ae83514..640c2fe0db6c 100644
--- a/doc/cephadm/install.rst
+++ b/doc/cephadm/install.rst
@@ -4,7 +4,7 @@
 Deploying a new Ceph cluster
 ============================
 
-Cephadm creates a new Ceph cluster by "bootstrapping" on a single
+Cephadm creates a new Ceph cluster by bootstrapping a single
 host, expanding the cluster to encompass any additional hosts, and
 then deploying the needed services.
 
@@ -18,7 +18,7 @@ Requirements
 - Python 3
 - Systemd
 - Podman or Docker for running containers
-- Time synchronization (such as chrony or NTP)
+- Time synchronization (such as Chrony or the legacy ``ntpd``)
 - LVM2 for provisioning storage devices
 
 Any modern Linux distribution should be sufficient.  Dependencies
@@ -47,9 +47,9 @@ up-to-date cephadm. There are two ways to get the initial ``cephadm``:
    Choose either the distribution-specific method or the curl-based method. Do
    not attempt to use both these methods on one system.
 
-.. note:: Recent versions of cephadm are based on a compilation of source files.
+.. note:: Recent versions of cephadm are distributed as an executable compiled from source code.
    Unlike for earlier versions of Ceph it is no longer sufficient to copy a
-   single source file from Ceph's git tree and run it. If you wish to run
+   single script from Ceph's git tree and run it. If you wish to run
    cephadm using a development version you should create your own build of
    cephadm. See :ref:`compiling-cephadm` for details on how to create your own
    standalone cephadm executable.
@@ -94,9 +94,9 @@ that case, you can install cephadm directly. For example:
 curl-based installation
 -----------------------
 
-* First, determine what version of Ceph you will need. You can use the releases
+* First, determine what version of Ceph you wish to install. You can use the releases
   page to find the `latest active releases <https://docs.ceph.com/en/latest/releases/#active-releases>`_.
-  For example, we might look at that page and find that ``18.2.0`` is the latest
+  For example, we might find that ``18.2.1`` is the latest
   active release.
 
 * Use ``curl`` to fetch a build of cephadm for that release.
@@ -122,7 +122,7 @@ curl-based installation
 * If you encounter any issues with running cephadm due to errors including
   the message ``bad interpreter``, then you may not have Python or
   the correct version of Python installed. The cephadm tool requires Python 3.6
-  and above. You can manually run cephadm with a particular version of Python by
+  or later. You can manually run cephadm with a particular version of Python by
   prefixing the command with your installed Python version. For example:
 
   .. prompt:: bash #
@@ -130,8 +130,8 @@ curl-based installation
 
      python3.8 ./cephadm <arguments...>
 
-* Although the standalone cephadm is sufficient to get a cluster started, it is
-  convenient to have the ``cephadm`` command installed on the host.  To install
+* Although the standalone cephadm is sufficient to bootstrap a cluster, it is
+  best to have the ``cephadm`` command installed on the host.  To install
   the packages that provide the ``cephadm`` command, run the following
   commands:
 
@@ -162,7 +162,7 @@ What to know before you bootstrap
 The first step in creating a new Ceph cluster is running the ``cephadm
 bootstrap`` command on the Ceph cluster's first host. The act of running the
 ``cephadm bootstrap`` command on the Ceph cluster's first host creates the Ceph
-cluster's first "monitor daemon", and that monitor daemon needs an IP address.
+cluster's first Monitor daemon.
 You must pass the IP address of the Ceph cluster's first host to the ``ceph
 bootstrap`` command, so you'll need to know the IP address of that host.
 
@@ -183,13 +183,13 @@ Run the ``ceph bootstrap`` command:
 
 This command will:
 
-* Create a monitor and manager daemon for the new cluster on the local
+* Create a Monitor and a Manager daemon for the new cluster on the local
   host.
 * Generate a new SSH key for the Ceph cluster and add it to the root
   user's ``/root/.ssh/authorized_keys`` file.
 * Write a copy of the public key to ``/etc/ceph/ceph.pub``.
 * Write a minimal configuration file to ``/etc/ceph/ceph.conf``. This
-  file is needed to communicate with the new cluster.
+  file is needed to communicate with Ceph daemons.
 * Write a copy of the ``client.admin`` administrative (privileged!)
   secret key to ``/etc/ceph/ceph.client.admin.keyring``.
 * Add the ``_admin`` label to the bootstrap host.  By default, any host
@@ -201,7 +201,7 @@ This command will:
 Further information about cephadm bootstrap
 -------------------------------------------
 
-The default bootstrap behavior will work for most users. But if you'd like
+The default bootstrap process will work for most users. But if you'd like
 immediately to know more about ``cephadm bootstrap``, read the list below.
 
 Also, you can run ``cephadm bootstrap -h`` to see all of ``cephadm``'s
@@ -212,15 +212,15 @@ available options.
   journald.  If you want Ceph to write traditional log files to ``/var/log/ceph/$fsid``,
   use the ``--log-to-file`` option during bootstrap.
 
-* Larger Ceph clusters perform better when (external to the Ceph cluster)
+* Larger Ceph clusters perform best when (external to the Ceph cluster)
   public network traffic is separated from (internal to the Ceph cluster)
   cluster traffic. The internal cluster traffic handles replication, recovery,
   and heartbeats between OSD daemons.  You can define the :ref:`cluster
   network<cluster-network>` by supplying the ``--cluster-network`` option to the ``bootstrap``
-  subcommand. This parameter must define a subnet in CIDR notation (for example
+  subcommand. This parameter must be a subnet in CIDR notation (for example
   ``10.90.90.0/24`` or ``fe80::/64``).
 
-* ``cephadm bootstrap`` writes to ``/etc/ceph`` the files needed to access
+* ``cephadm bootstrap`` writes to ``/etc/ceph`` files needed to access
   the new cluster. This central location makes it possible for Ceph
   packages installed on the host (e.g., packages that give access to the
   cephadm command line interface) to find these files.
@@ -241,12 +241,12 @@ available options.
       EOF
       $ ./cephadm bootstrap --config initial-ceph.conf ...
 
-* The ``--ssh-user *<user>*`` option makes it possible to choose which SSH
+* The ``--ssh-user *<user>*`` option makes it possible to designate which SSH
   user cephadm will use to connect to hosts. The associated SSH key will be
   added to ``/home/*<user>*/.ssh/authorized_keys``. The user that you
   designate with this option must have passwordless sudo access.
 
-* If you are using a container on an authenticated registry that requires
+* If you are using a container image from a registry that requires
   login, you may add the argument:
 
   * ``--registry-json <path to json file>``
@@ -257,7 +257,7 @@ available options.
 
   Cephadm will attempt to log in to this registry so it can pull your container
   and then store the login info in its config database. Other hosts added to
-  the cluster will then also be able to make use of the authenticated registry.
+  the cluster will then also be able to make use of the authenticated container registry.
 
 * See :ref:`cephadm-deployment-scenarios` for additional examples for using ``cephadm bootstrap``.
 
@@ -322,7 +322,7 @@ Add all hosts to the cluster by following the instructions in
 
 By default, a ``ceph.conf`` file and a copy of the ``client.admin`` keyring are
 maintained in ``/etc/ceph`` on all hosts that have the ``_admin`` label. This
-label is initially applied only to the bootstrap host. We usually recommend
+label is initially applied only to the bootstrap host. We recommend
 that one or more other hosts be given the ``_admin`` label so that the Ceph CLI
 (for example, via ``cephadm shell``) is easily accessible on multiple hosts. To add
 the ``_admin`` label to additional host(s), run a command of the following form:
@@ -335,9 +335,10 @@ the ``_admin`` label to additional host(s), run a command of the following form:
 Adding additional MONs
 ======================
 
-A typical Ceph cluster has three or five monitor daemons spread
+A typical Ceph cluster has three or five Monitor daemons spread
 across different hosts.  We recommend deploying five
-monitors if there are five or more nodes in your cluster.
+Monitors if there are five or more nodes in your cluster. Most clusters do not
+benefit from seven or more Monitors.
 
 Please follow :ref:`deploy_additional_monitors` to deploy additional MONs.
 
@@ -362,12 +363,12 @@ See :ref:`osd_autotune`.
 
 To deploy hyperconverged Ceph with TripleO, please refer to the TripleO documentation: `Scenario: Deploy Hyperconverged Ceph <https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/features/cephadm.html#scenario-deploy-hyperconverged-ceph>`_
 
-In other cases where the cluster hardware is not exclusively used by Ceph (hyperconverged),
+In other cases where the cluster hardware is not exclusively used by Ceph (converged infrastructure),
 reduce the memory consumption of Ceph like so:
 
   .. prompt:: bash #
 
-    # hyperconverged only:
+    # converged only:
     ceph config set mgr mgr/cephadm/autotune_memory_target_ratio 0.2
 
 Then enable memory autotuning:
@@ -396,9 +397,11 @@ Different deployment scenarios
 Single host
 -----------
 
-To configure a Ceph cluster to run on a single host, use the
-``--single-host-defaults`` flag when bootstrapping. For use cases of this, see
-:ref:`one-node-cluster`.
+To deploy a Ceph cluster running on a single host, use the
+``--single-host-defaults`` flag when bootstrapping. For use cases, see
+:ref:`one-node-cluster`. Such clusters are generally not suitable for
+production.
+
 
 The ``--single-host-defaults`` flag sets the following configuration options::
 
@@ -415,8 +418,8 @@ Deployment in an isolated environment
 -------------------------------------
 
 You might need to install cephadm in an environment that is not connected
-directly to the internet (such an environment is also called an "isolated
-environment"). This can be done if a custom container registry is used. Either
+directly to the Internet (an "isolated" or "airgapped"
+environment). This requires the use of a custom container registry. Either
 of two kinds of custom container registry can be used in this scenario: (1) a
 Podman-based or Docker-based insecure registry, or (2) a secure registry.
 
@@ -565,9 +568,9 @@ in order to have cephadm use them for SSHing between cluster hosts
 Note that this setup does not require installing the corresponding public key
 from the private key passed to bootstrap on other nodes. In fact, cephadm will
 reject the ``--ssh-public-key`` argument when passed along with ``--ssh-signed-cert``.
-Not because having the public key breaks anything, but because it is not at all needed
-for this setup and it helps bootstrap differentiate if the user wants the CA signed
-keys setup or standard pubkey encryption. What this means is, SSH key rotation
+This is not because having the public key breaks anything, but rather because it is not at all needed
+and helps the bootstrap command differentiate if the user wants the CA signed
+keys setup or standard pubkey encryption. What this means is that SSH key rotation
 would simply be a matter of getting another key signed by the same CA and providing
 cephadm with the new private key and signed cert. No additional distribution of
 keys to cluster nodes is needed after the initial setup of the CA key as a trusted key,
diff --git a/doc/cephadm/operations.rst b/doc/cephadm/operations.rst
index d6323c04e0c3..c2c8f8b6d28a 100644
--- a/doc/cephadm/operations.rst
+++ b/doc/cephadm/operations.rst
@@ -397,15 +397,15 @@ You can disable this health warning by running the following command:
 
 Cluster Configuration Checks
 ----------------------------
-Cephadm periodically scans each of the hosts in the cluster in order
-to understand the state of the OS, disks, NICs etc. These facts can
-then be analysed for consistency across the hosts in the cluster to
+Cephadm periodically scans each host in the cluster in order
+to understand the state of the OS, disks, network interfacess etc. This information can
+then be analyzed for consistency across the hosts in the cluster to
 identify any configuration anomalies.
 
 Enabling Cluster Configuration Checks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The configuration checks are an **optional** feature, and are enabled
+These configuration checks are an **optional** feature, and are enabled
 by running the following command:
 
 .. prompt:: bash #
@@ -415,7 +415,7 @@ by running the following command:
 States Returned by Cluster Configuration Checks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The configuration checks are triggered after each host scan (1m). The
+Configuration checks are triggered after each host scan. The
 cephadm log entries will show the current state and outcome of the
 configuration checks as follows:
 
@@ -452,14 +452,14 @@ To list all the configuration checks and their current states, run the following
   # ceph cephadm config-check ls
 
     NAME             HEALTHCHECK                      STATUS   DESCRIPTION
-  kernel_security  CEPHADM_CHECK_KERNEL_LSM         enabled  checks SELINUX/Apparmor profiles are consistent across cluster hosts
-  os_subscription  CEPHADM_CHECK_SUBSCRIPTION       enabled  checks subscription states are consistent for all cluster hosts
-  public_network   CEPHADM_CHECK_PUBLIC_MEMBERSHIP  enabled  check that all hosts have a NIC on the Ceph public_network
+  kernel_security  CEPHADM_CHECK_KERNEL_LSM         enabled  check that SELINUX/Apparmor profiles are consistent across cluster hosts
+  os_subscription  CEPHADM_CHECK_SUBSCRIPTION       enabled  check that subscription states are consistent for all cluster hosts
+  public_network   CEPHADM_CHECK_PUBLIC_MEMBERSHIP  enabled  check that all hosts have a network interface on the Ceph public_network
   osd_mtu_size     CEPHADM_CHECK_MTU                enabled  check that OSD hosts share a common MTU setting
-  osd_linkspeed    CEPHADM_CHECK_LINKSPEED          enabled  check that OSD hosts share a common linkspeed
-  network_missing  CEPHADM_CHECK_NETWORK_MISSING    enabled  checks that the cluster/public networks defined exist on the Ceph hosts
-  ceph_release     CEPHADM_CHECK_CEPH_RELEASE       enabled  check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active)
-  kernel_version   CEPHADM_CHECK_KERNEL_VERSION     enabled  checks that the MAJ.MIN of the kernel on Ceph hosts is consistent
+  osd_linkspeed    CEPHADM_CHECK_LINKSPEED          enabled  check that OSD hosts share a common network link speed
+  network_missing  CEPHADM_CHECK_NETWORK_MISSING    enabled  check that the cluster/public networks as defined exist on the Ceph hosts
+  ceph_release     CEPHADM_CHECK_CEPH_RELEASE       enabled  check for Ceph version consistency: all Ceph daemons should be the same release unless upgrade is in progress
+  kernel_version   CEPHADM_CHECK_KERNEL_VERSION     enabled  checks that the maj.min version of the kernel is consistent across Ceph hosts
 
 The name of each configuration check can be used to enable or disable a specific check by running a command of the following form:
 :
@@ -483,31 +483,31 @@ flagged as an anomaly and a healthcheck (WARNING) state raised.
 
 CEPHADM_CHECK_SUBSCRIPTION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
-This check relates to the status of vendor subscription. This check is
-performed only for hosts using RHEL, but helps to confirm that all hosts are
+This check relates to the status of OS vendor subscription. This check is
+performed only for hosts using RHEL and helps to confirm that all hosts are
 covered by an active subscription, which ensures that patches and updates are
 available.
 
 CEPHADM_CHECK_PUBLIC_MEMBERSHIP
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-All members of the cluster should have NICs configured on at least one of the
+All members of the cluster should have a network interface configured on at least one of the
 public network subnets. Hosts that are not on the public network will rely on
 routing, which may affect performance.
 
 CEPHADM_CHECK_MTU
 ~~~~~~~~~~~~~~~~~
-The MTU of the NICs on OSDs can be a key factor in consistent performance. This
+The MTU of the network interfaces on OSD hosts can be a key factor in consistent performance. This
 check examines hosts that are running OSD services to ensure that the MTU is
-configured consistently within the cluster. This is determined by establishing
+configured consistently within the cluster. This is determined by determining
 the MTU setting that the majority of hosts is using. Any anomalies result in a
-Ceph health check.
+health check.
 
 CEPHADM_CHECK_LINKSPEED
 ~~~~~~~~~~~~~~~~~~~~~~~
-This check is similar to the MTU check. Linkspeed consistency is a factor in
-consistent cluster performance, just as the MTU of the NICs on the OSDs is.
-This check determines the linkspeed shared by the majority of OSD hosts, and a
-health check is run for any hosts that are set at a lower linkspeed rate.
+This check is similar to the MTU check. Link speed consistency is a factor in
+consistent cluster performance, as is the MTU of the OSD node network interfaces.
+This check determines the link speed shared by the majority of OSD hosts, and a
+health check is run for any hosts that are set at a lower link speed rate.
 
 CEPHADM_CHECK_NETWORK_MISSING
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -517,15 +517,14 @@ a health check is raised.
 
 CEPHADM_CHECK_CEPH_RELEASE
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
-Under normal operations, the Ceph cluster runs daemons under the same ceph
-release (that is, the Ceph cluster runs all daemons under (for example)
-Octopus).  This check determines the active release for each daemon, and
+Under normal operations, the Ceph cluster runs daemons that are of the same Ceph
+release (for example, Reef).  This check determines the active release for each daemon, and
 reports any anomalies as a healthcheck. *This check is bypassed if an upgrade
-process is active within the cluster.*
+is in process.*
 
 CEPHADM_CHECK_KERNEL_VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The OS kernel version (maj.min) is checked for consistency across the hosts.
+The OS kernel version (maj.min) is checked for consistency across hosts.
 The kernel version of the majority of the hosts is used as the basis for
 identifying anomalies.
 
diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
index c0bf6f9a00a6..a7afaa108c84 100644
--- a/doc/cephadm/troubleshooting.rst
+++ b/doc/cephadm/troubleshooting.rst
@@ -32,7 +32,7 @@ completely by running the following commands:
   ceph orch set backend ''
   ceph mgr module disable cephadm
 
-These commands disable all of the ``ceph orch ...`` CLI commands. All
+These commands disable all ``ceph orch ...`` CLI commands. All
 previously deployed daemon containers continue to run and will start just as
 they were before you ran these commands.
 
@@ -56,7 +56,7 @@ following form:
 
   ceph orch ls --service_name=<service-name> --format yaml
 
-This will return something in the following form:
+This will return information in the following form:
 
 .. code-block:: yaml
 
@@ -252,16 +252,17 @@ For more detail on operations of this kind, see
 Accessing the Admin Socket
 --------------------------
 
-Each Ceph daemon provides an admin socket that bypasses the MONs (See
-:ref:`rados-monitoring-using-admin-socket`).
+Each Ceph daemon provides an admin socket that allows runtime option setting and statistic reading. See
+:ref:`rados-monitoring-using-admin-socket`.
 
 #. To access the admin socket, enter the daemon container on the host::
 
    [root@mon1 ~]# cephadm enter --name <daemon-name>
 
-#. Run a command of the following form to see the admin socket's configuration::
+#. Run a command of the following forms to see the admin socket's configuration and other available actions::
   
    [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok config show
+   [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok help
 
 Running Various Ceph Tools
 --------------------------------
diff --git a/doc/cephadm/upgrade.rst b/doc/cephadm/upgrade.rst
index e0a9f610ae2a..9bb1a6b4da43 100644
--- a/doc/cephadm/upgrade.rst
+++ b/doc/cephadm/upgrade.rst
@@ -2,7 +2,7 @@
 Upgrading Ceph
 ==============
 
-Cephadm can safely upgrade Ceph from one bugfix release to the next.  For
+Cephadm can safely upgrade Ceph from one point release to the next.  For
 example, you can upgrade from v15.2.0 (the first Octopus release) to the next
 point release, v15.2.1.
 
@@ -137,25 +137,25 @@ UPGRADE_NO_STANDBY_MGR
 ----------------------
 
 This alert (``UPGRADE_NO_STANDBY_MGR``) means that Ceph does not detect an
-active standby manager daemon. In order to proceed with the upgrade, Ceph
-requires an active standby manager daemon (which you can think of in this
+active standby Manager daemon. In order to proceed with the upgrade, Ceph
+requires an active standby Manager daemon (which you can think of in this
 context as "a second manager").
 
-You can ensure that Cephadm is configured to run 2 (or more) managers by
+You can ensure that Cephadm is configured to run two (or more) Managers by
 running the following command:
 
 .. prompt:: bash #
 
   ceph orch apply mgr 2  # or more
 
-You can check the status of existing mgr daemons by running the following
+You can check the status of existing Manager daemons by running the following
 command:
 
 .. prompt:: bash #
 
   ceph orch ps --daemon-type mgr
 
-If an existing mgr daemon has stopped, you can try to restart it by running the
+If an existing Manager daemon has stopped, you can try to restart it by running the
 following command: 
 
 .. prompt:: bash #
@@ -183,7 +183,7 @@ Using customized container images
 =================================
 
 For most users, upgrading requires nothing more complicated than specifying the
-Ceph version number to upgrade to.  In such cases, cephadm locates the specific
+Ceph version to which to upgrade.  In such cases, cephadm locates the specific
 Ceph container image to use by combining the ``container_image_base``
 configuration option (default: ``docker.io/ceph/ceph``) with a tag of
 ``vX.Y.Z``.

From a2942f01ae9bde76c6d562374a0bd8aceeee317e Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Mon, 11 Mar 2024 15:09:57 +0000
Subject: [PATCH 2229/2492] mgr/dashboard: fix snap schedule time format

Fixes: https://tracker.ceph.com/issues/64831

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-snapshotschedule-form.component.ts      | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 076c99014a6b..0ef68f7d529c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -222,13 +222,17 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     this.cd.detectChanges();
   }
 
+  convertNumberToString(input: number, length = 2, format = '0'): string {
+    return padStart(input.toString(), length, format);
+  }
+
   parseDatetime(date: NgbDateStruct, time?: NgbTimeStruct): string {
     if (!date || !time) return null;
-    return `${date.year}-${padStart(date.month.toString(), 2, '0')}-${padStart(
-      date.day.toString(),
-      2,
-      '0'
-    )}T${time.hour || '00'}:${time.minute || '00'}:${time.second || '00'}`;
+    return `${date.year}-${this.convertNumberToString(date.month)}-${this.convertNumberToString(
+      date.day
+    )}T${this.convertNumberToString(time.hour)}:${this.convertNumberToString(
+      time.minute
+    )}:${this.convertNumberToString(time.second)}`;
   }
   parseSchedule(interval: number, frequency: string): string {
     return `${interval}${frequency}`;

From 6d6837834bd70692fd714c8c7fddd6314bc76d3d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sun, 10 Mar 2024 21:21:38 -0400
Subject: [PATCH 2230/2492] qa/nightlies: simplify cron_wrapper and log to
 syslog

Instead of logging to a separate file which must be rotated manually, use the
syslog (journald on teuthology.front).

Keep a temporary unlinked file around in case we want to dump that to the
cronjob stdout for a mail status.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/nightlies/cron_wrapper | 52 +++++++++------------------------------
 1 file changed, 11 insertions(+), 41 deletions(-)

diff --git a/qa/nightlies/cron_wrapper b/qa/nightlies/cron_wrapper
index 29b4483e99c0..f159cea5bb61 100755
--- a/qa/nightlies/cron_wrapper
+++ b/qa/nightlies/cron_wrapper
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# /nightlies/cron_wrapper.sh
 
 # check for no argument case and stop
 if [ -z $1 ]; then
@@ -7,47 +6,18 @@ if [ -z $1 ]; then
   exit 1
 fi
 
-# set permanent $LOG file var
-LOG="/var/log/crontab-nightlies-log/crontab.log"
-# set $LOG_LOCKED_ERR in case locking failed
-LOG_LOCK_ERR="/var/log/crontab-nightlies-log/crontab_lock_problem.$$"
+# Make a temporary unlinked file to hold the stdout/stderr
+T=$(mktemp)
+exec 10>"$T"
+exec 11<"$T"
+rm -f "$T"
 
-# temp files to store stdout and stderr
-# named with the PID of this script in their name so they'll be unique
-STDERR="/var/tmp/stderr.$$"
-STDOUT="/var/tmp/stdout.$$"
-
-# $STDOUT and $STDERR are removed when the script exits for any reason
-trap  "rm -f $STDOUT $STDERR" 0
-
-# run a command from this script's argument
-# redirect stdout to $STDOUT file and redirect stderr to $STDERR file
-
-DATE=$(date)
-echo -n "$DATE: "  >> $STDOUT
-echo "Running command: $@" >> $STDOUT
-"$@" > $STDOUT 2> $STDERR
-
-# get return code from the command run
+# Forward to syslog (journald)
+printf 'Running command: %s' "$*" | logger
+"$@" |& tee >(logger) >&10
 code=$?
 
-if [ $code != 0 ] ; then
-        # echoing to stdout/stderr makes cron send email
-        echo "stdout:"
-        cat $STDOUT
-        echo "stderr:"
-        cat $STDERR
-else
-        # normal exit: just log stdout
-
-	# lock $LOG with file descriptor 200
-	exec 200>>$LOG
-	# if $LOG is locked by other process - wait for 20 sec
-	flock -w 20 200 || LOG=$LOG_LOCK_ERR
-	echo "stdout:" >> $LOG
-	cat $STDOUT >> $LOG
-	echo "stderr:" >> $LOG
-	cat $STDERR >> $LOG
-	# unlock
-	flock -u 200
+if [ "$code" != 0 ] ; then
+    printf 'teuthology cronjob encountered error:\n'
+    head -n 10000 <&11
 fi

From 941eace0fc48bdd64b0bca920abeea17c07b1f21 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sun, 10 Mar 2024 11:22:09 -0400
Subject: [PATCH 2231/2492] qa/crontab: correct script paths and environment

At some point the links to the shell scripts in ceph.git were broken in the
$HOME for [1]. Unless a run was done manually with `teuthology-suite` in the
crontab, the job was basically skipped. This is probably one of the reasons
nightlies fell out of use.

I've updated the home directory according to the document (comments) in this
change. The teuthology user now has persistent clones of ceph.git and
teuthology.git. The clones are updated daily by this same crontab.

Instead of using a link in teuthology's $HOME/bin to the scripts used in this
crontab, we just have a cron variable referencing where the script should be in
the ceph.git/teuthology.git clone. Adding to this, the .bash_environment file
sources the virtualenv activate script instead of adding the teuthology binary
directory to its $PATH.

I've updated the hour for these jobs to actually be done "nightly". The first
set of jobs will be scheduled around 4pm EST. Additionally, it was necessary to
include --force-priority as some jobs are below the priority thresholds.

[1] teuthology@teuthology.front.sepia.ceph.com.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 98 +++++++++++++++++++---------------
 1 file changed, 56 insertions(+), 42 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index 52280d9fec81..e6cbc2f9cbaa 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -1,4 +1,18 @@
-PATH=/home/teuthology/src/teuthology_main/virtualenv/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+# nightlies are run as teuthology@teuthology.front.sepia.ceph.com
+# Dependent data in that user's $HOME:
+#   - ~/ceph                     : a checkout of https://github.com/ceph/ceph.git
+#   - ~/teuthology               : a checkout of https://github.com/ceph/teuthology.git
+#   - ~/teuthology/virtualenv    : a virtualenv created by ./bootstrap (in teuthology.git)
+#   - ~/.bash_environment        : non-interactive shell configuration, including: `source ~/teuthology/virtualenv/bin/activate`
+
+SHELL=/bin/bash
+
+# This is necessary when running bash non-interactively.
+BASH_ENV=/home/teuthology/.bash_environment
+
+CW=/home/teuthology/ceph/qa/nightlies/cron_wrapper
+SCHEDULE=/home/teuthology/ceph/qa/machine_types/schedule_subset.sh
+
 TEUTH_CEPH_REPO='https://github.com/ceph/ceph.git'
 TEUTH_SUITE_REPO='https://github.com/ceph/ceph.git'
 MAILTO="ceph-infra@redhat.com;yweinste@redhat.com"
@@ -17,10 +31,10 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 
 # Ensure teuthology is up-to-date
-@daily cd /home/teuthology/src/teuthology_main && /home/teuthology/bin/cron_wrapper git pull
-@daily cd /home/teuthology/src/git.ceph.com_ceph_main && /home/teuthology/bin/cron_wrapper git pull
+@daily cd /home/teuthology/ceph && $CW git pull -q
+@daily cd /home/teuthology/teuthology && $CW git pull -q && $CW ./bootstrap
 # Ensure ceph-sepia-secrets is up-to-date
-*/5 * * * *  cd /home/teuthology/ceph-sepia-secrets && /home/teuthology/bin/cron_wrapper git pull
+*/5 * * * *  cd /home/teuthology/ceph-sepia-secrets && $CW git pull -q
 
 
 #Publish this crontab to the Tracker page http://tracker.ceph.com/projects/ceph-releases/wiki/Crontab
@@ -32,22 +46,22 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 
 ## ********** smoke tests on main and release branches
-0 1  * * 0,2,4 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-0 2  * * 0     CEPH_BRANCH=squid; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-0 3  * * 0     CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
-0 4  * * 0     CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL -p 70
+00 05  * * 0,2,4 CEPH_BRANCH=main;   MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
+08 05  * * 0     CEPH_BRANCH=squid;  MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
+16 05  * * 0     CEPH_BRANCH=reef;   MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
+24 05  * * 0     CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
 
 
 ## ********** windows tests on main branch - weekly
-# 00 03 * * 1 CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s windows -k distro -e $CEPH_QA_EMAIL
+# 00 03 * * 1 CEPH_BRANCH=main; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s windows -k distro -e $CEPH_QA_EMAIL
 
 
 ## ********** crimson tests on main branch - weekly
-# 01 01 * * 0 CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=crimson-rados; KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
+# 01 01 * * 0 CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=crimson-rados; KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL
 
 
 ## ********** teuthology/nop on main branch - daily
-0 0 * * * CEPH_BRANCH=main; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 10 -m $MACHINE_NAME -s teuthology/nop -k distro -e $CEPH_QA_EMAIL -p 50
+@daily CEPH_BRANCH=main; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 10 -m $MACHINE_NAME -s teuthology/nop -k distro -e $CEPH_QA_EMAIL --force-priority -p 50
 
 
 ## main branch runs - weekly
@@ -55,13 +69,13 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 05 * * 0   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 201
-08 05 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-16 05 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-24 05 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-32 05 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-40 05 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
-48 05 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 200
+00 20 * * 0   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 201
+08 20 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+16 20 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+24 20 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+32 20 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+40 20 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+48 20 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
 
 
 ## squid branch runs - twice weekly
@@ -69,13 +83,13 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 06 * * 0,4 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 101
-08 06 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-16 06 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-24 06 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-32 06 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-40 06 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
-48 06 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 100
+00 21 * * 0,4 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 101
+08 21 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
+16 21 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
+24 21 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
+32 21 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
+40 21 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
+48 21 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
 
 
 ## reef branch runs - weekly
@@ -83,17 +97,17 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 07 * * 0   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 181
-08 07 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-16 07 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-24 07 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-32 07 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-40 07 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
-48 07 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 180
+00 22 * * 0   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 181
+08 22 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+16 22 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+24 22 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+32 22 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+40 22 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+48 22 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
 
 
 ###  The suite below must run on bare-metal because it's performance suite and run 3 times to produce more data points
-# 57 03 * * 6 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s perf-basic -k distro -e $CEPH_QA_EMAIL -N 3
+# 57 03 * * 6 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s perf-basic -k distro -e $CEPH_QA_EMAIL -N 3
 
 
 ##########################
@@ -104,24 +118,24 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 ## !!!! the client suites below MUST use --suite-branch octopus, pacific (see https://tracker.ceph.com/issues/24021)
 
-20 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 175
-25 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git -p 175
-30 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:octopus-x $CEPH_QA_EMAIL distro -p 175
-35 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:pacific-x $CEPH_QA_EMAIL distro -p 175
-40 01 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL -p 175
+08 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git --force-priority -p 175
+16 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git --force-priority -p 175
+24 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW $SCHEDULE 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:octopus-x $CEPH_QA_EMAIL distro --force-priority -p 175
+32 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW $SCHEDULE 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:pacific-x $CEPH_QA_EMAIL distro --force-priority -p 175
+40 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL --force-priority -p 175
 
 ### upgrade runs for reef release
 ###### on smithi
 
-20 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x -k distro -e $CEPH_QA_EMAIL -p 170
-25 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL -p 170
+08 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x -k distro -e $CEPH_QA_EMAIL --force-priority -p 170
+16 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL --force-priority -p 170
 
 ### upgrade runs for squid release
 ###### on smithi
 
-00 00 * * 5   CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=upgrade;         KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 165
+08 02 * * 5   CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=upgrade;         KERNEL=distro;  $CW $SCHEDULE 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 165
 
 ### upgrade runs for main
 ###### on smithi
 
-00 00 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=upgrade;          KERNEL=distro;  /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 160
+08 03 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=upgrade;          KERNEL=distro;  $CW $SCHEDULE 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 160

From bc4cc4e4d64ac38edcf353cbe025d7c80bbb515e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 11 Mar 2024 10:08:15 -0400
Subject: [PATCH 2232/2492] qa/nightlies: move schedule_subset script alongside
 cron_wrapper

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/{machine_types => nightlies}/schedule_subset.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename qa/{machine_types => nightlies}/schedule_subset.sh (100%)

diff --git a/qa/machine_types/schedule_subset.sh b/qa/nightlies/schedule_subset.sh
similarity index 100%
rename from qa/machine_types/schedule_subset.sh
rename to qa/nightlies/schedule_subset.sh

From acbc26ec1273e5d4a6fa95501d94c54b84ed858b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 11 Mar 2024 10:46:05 -0400
Subject: [PATCH 2233/2492] qa/crontab: simplify and minimize argument
 specification

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs  | 87 +++++++++++++++++----------------
 qa/nightlies/schedule_subset.sh | 30 ++++++------
 2 files changed, 59 insertions(+), 58 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index e6cbc2f9cbaa..be28b3900e63 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -5,19 +5,22 @@
 #   - ~/teuthology/virtualenv    : a virtualenv created by ./bootstrap (in teuthology.git)
 #   - ~/.bash_environment        : non-interactive shell configuration, including: `source ~/teuthology/virtualenv/bin/activate`
 
-SHELL=/bin/bash
+SHELL="/bin/bash"
 
 # This is necessary when running bash non-interactively.
-BASH_ENV=/home/teuthology/.bash_environment
+BASH_ENV="/home/teuthology/.bash_environment"
 
-CW=/home/teuthology/ceph/qa/nightlies/cron_wrapper
-SCHEDULE=/home/teuthology/ceph/qa/machine_types/schedule_subset.sh
 
-TEUTH_CEPH_REPO='https://github.com/ceph/ceph.git'
-TEUTH_SUITE_REPO='https://github.com/ceph/ceph.git'
+TEUTH_CEPH_REPO="https://github.com/ceph/ceph.git"
+TEUTH_SUITE_REPO="https://github.com/ceph/ceph.git"
 MAILTO="ceph-infra@redhat.com;yweinste@redhat.com"
 CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
+CW="/home/teuthology/ceph/qa/nightlies/cron_wrapper"
+SS="/home/teuthology/ceph/qa/nightlies/schedule_subset.sh"
+# default/common arguments added by schedule_subset.sh
+TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ceph.com/ceph.git --suite-repo=https://git.ceph.com/ceph.git --machine-type smithi --force-priority"
+
 ### !!!!!!!!!!!!!!!!!!!!!!!!!!
 ## THIS CRONTAB MUST NOT BE EDITED MANUALLY !!!!
 ## AUTOMATED CRONTAB UPDATING
@@ -46,11 +49,10 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 
 ## ********** smoke tests on main and release branches
-00 05  * * 0,2,4 CEPH_BRANCH=main;   MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
-08 05  * * 0     CEPH_BRANCH=squid;  MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
-16 05  * * 0     CEPH_BRANCH=reef;   MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
-24 05  * * 0     CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -m $MACHINE_NAME -s smoke -k distro -e $CEPH_QA_EMAIL --force-priority -p 70
-
+00 05  * * 0,2,4   $CW $SS      1 --ceph main    --suite smoke -p 70
+08 05  * * 0       $CW $SS      1 --ceph squid   --suite smoke -p 70
+16 05  * * 0       $CW $SS      1 --ceph reef    --suite smoke -p 70
+24 05  * * 0       $CW $SS      1 --ceph quincy  --suite smoke -p 70
 
 ## ********** windows tests on main branch - weekly
 # 00 03 * * 1 CEPH_BRANCH=main; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s windows -k distro -e $CEPH_QA_EMAIL
@@ -61,7 +63,7 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 
 ## ********** teuthology/nop on main branch - daily
-@daily CEPH_BRANCH=main; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 10 -m $MACHINE_NAME -s teuthology/nop -k distro -e $CEPH_QA_EMAIL --force-priority -p 50
+@daily             $CW $SS      1 --ceph main --suite teuthology/nop -p 50
 
 
 ## main branch runs - weekly
@@ -69,13 +71,13 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 20 * * 0   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 201
-08 20 * * 1   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
-16 20 * * 2   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
-24 20 * * 3   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
-32 20 * * 4   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
-40 20 * * 5   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
-48 20 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 200
+00 20 * * 0        $CW $SS 100000 --ceph main --suite      rados -p 201
+08 20 * * 1        $CW $SS     64 --ceph main --suite       orch -p 200
+16 20 * * 2        $CW $SS    128 --ceph main --suite        rbd -p 200
+24 20 * * 3        $CW $SS    512 --ceph main --suite         fs -p 200
+32 20 * * 4        $CW $SS      4 --ceph main --suite powercycle -p 200
+40 20 * * 5        $CW $SS      1 --ceph main --suite        rgw -p 200
+48 20 * * 6        $CW $SS      4 --ceph main --suite       krbd -p 200 --kernel testing
 
 
 ## squid branch runs - twice weekly
@@ -83,27 +85,26 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 21 * * 0,4 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 101
-08 21 * * 1,5 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-16 21 * * 2,6 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-24 21 * * 3,0 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-32 21 * * 4,1 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-40 21 * * 5,2 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-48 21 * * 6,3 CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 100
-
+00 21 * * 0,4      $CW $SS 100000 --ceph squid --suite      rados -p 101
+08 21 * * 1,5      $CW $SS     64 --ceph squid --suite       orch -p 100
+16 21 * * 2,6      $CW $SS    128 --ceph squid --suite        rbd -p 100
+24 21 * * 3,0      $CW $SS    512 --ceph squid --suite         fs -p 100
+32 21 * * 4,1      $CW $SS      4 --ceph squid --suite powercycle -p 100
+40 21 * * 5,2      $CW $SS      1 --ceph squid --suite        rgw -p 100
+48 21 * * 6,3      $CW $SS      4 --ceph squid --suite       krbd -p 100 --kernel testing
 
 ## reef branch runs - weekly
 ## suites rados and rbd use --subset arg and must be call with schedule_subset.sh
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 22 * * 0   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rados;         KERNEL=distro;  $CW $SCHEDULE 100000 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 181
-08 22 * * 1   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=orch;          KERNEL=distro;  $CW $SCHEDULE     64 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
-16 22 * * 2   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rbd;           KERNEL=distro;  $CW $SCHEDULE    128 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
-24 22 * * 3   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=fs;            KERNEL=distro;  $CW $SCHEDULE    512 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
-32 22 * * 4   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=powercycle;    KERNEL=distro;  $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
-40 22 * * 5   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=rgw;           KERNEL=distro;  $CW $SCHEDULE      1 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
-48 22 * * 6   CEPH_BRANCH=reef; MACHINE_NAME=smithi; SUITE_NAME=krbd;          KERNEL=testing; $CW $SCHEDULE      4 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 180
+00 22 * * 0        $CW $SS 100000 --ceph reef --suite      rados -p 181
+08 22 * * 1        $CW $SS     64 --ceph reef --suite       orch -p 180
+16 22 * * 2        $CW $SS    128 --ceph reef --suite        rbd -p 180
+24 22 * * 3        $CW $SS    512 --ceph reef --suite         fs -p 180
+32 22 * * 4        $CW $SS      4 --ceph reef --suite powercycle -p 180
+40 22 * * 5        $CW $SS      1 --ceph reef --suite        rgw -p 180
+48 22 * * 6        $CW $SS      4 --ceph reef --suite       krbd -p 180 --kernel testing
 
 
 ###  The suite below must run on bare-metal because it's performance suite and run 3 times to produce more data points
@@ -118,24 +119,24 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 
 ## !!!! the client suites below MUST use --suite-branch octopus, pacific (see https://tracker.ceph.com/issues/24021)
 
-08 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch octopus --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git --force-priority -p 175
-16 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-pacific-quincy -k distro -e $CEPH_QA_EMAIL --suite-branch pacific --suite-repo https://github.com/ceph/ceph.git --ceph-repo https://github.com/ceph/ceph.git --force-priority -p 175
-24 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW $SCHEDULE 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:octopus-x $CEPH_QA_EMAIL distro --force-priority -p 175
-32 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW $SCHEDULE 120000 $CEPH_BRANCH $MACHINE_NAME upgrade:pacific-x $CEPH_QA_EMAIL distro --force-priority -p 175
-40 00 * * 1 CEPH_BRANCH=quincy; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade/quincy-p2p -k distro -e $CEPH_QA_EMAIL --force-priority -p 175
+08 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-octopus-quincy --suite-branch octopus -p 175
+16 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-pacific-quincy --suite-branch pacific -p 175
+24 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:octopus-x -p 175
+32 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:pacific-x -p 175
+40 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade/quincy-p2p -p 175
 
 ### upgrade runs for reef release
 ###### on smithi
 
-08 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:pacific-x -k distro -e $CEPH_QA_EMAIL --force-priority -p 170
-16 01 * * 3 CEPH_BRANCH=reef; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade:quincy-x -k distro -e $CEPH_QA_EMAIL --force-priority -p 170
+08 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:pacific-x -p 170
+16 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:quincy-x  -p 170
 
 ### upgrade runs for squid release
 ###### on smithi
 
-08 02 * * 5   CEPH_BRANCH=squid; MACHINE_NAME=smithi; SUITE_NAME=upgrade;         KERNEL=distro;  $CW $SCHEDULE 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 165
+08 02 * * 5        $CW $SS     32 --ceph squid --suite upgrade -p 165
 
 ### upgrade runs for main
 ###### on smithi
 
-08 03 * * 6   CEPH_BRANCH=main; MACHINE_NAME=smithi; SUITE_NAME=upgrade;          KERNEL=distro;  $CW $SCHEDULE 32 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL --force-priority -p 160
+08 03 * * 6        $CW $SS     32 --ceph main --suite upgrade -p 160
diff --git a/qa/nightlies/schedule_subset.sh b/qa/nightlies/schedule_subset.sh
index 7f18c81ef672..7bf926c1716f 100755
--- a/qa/nightlies/schedule_subset.sh
+++ b/qa/nightlies/schedule_subset.sh
@@ -1,20 +1,20 @@
-#!/bin/bash -e
+#!/bin/bash
 
-#command line => CEPH_BRANCH=<branch>; MACHINE_NAME=<machine_type>; SUITE_NAME=<suite>; ../schedule_subset.sh <day_of_week> $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL <$FILTER>
+set -e
+
+function prun {
+  printf '%s\n' "$*" >&2
+  "$@"
+}
 
 partitions="$1"
 shift
-branch="$1"
-shift
-machine="$1"
-shift
-suite="$1"
-shift
-email="$1"
-shift
-kernel="$1"
-shift
-# rest of arguments passed directly to teuthology-suite
 
-echo "Scheduling $branch branch"
-teuthology-suite -v -c "$branch" -m "$machine" -k "$kernel" -s "$suite" --ceph-repo https://git.ceph.com/ceph.git --suite-repo https://git.ceph.com/ceph.git --subset "$((RANDOM % partitions))/$partitions" --newest 100 -e "$email" "$@"
+ARGS=()
+ARGS+=("--subset=$((RANDOM % partitions))/$partitions")
+
+if [ -n "$CEPH_QA_EMAIL" ]; then
+  ARGS+=("--email=$CEPH_QA_EMAIL")
+fi
+
+prun teuthology-suite "${ARGS[@]}" $TEUTHOLOGY_SUITE_ARGS "$@"

From 156a41b7ecd1f82c9916697110186b66124b2ccf Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:35:02 -0500
Subject: [PATCH 2234/2492] cephadm: black format agent.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/agent.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadmlib/agent.py b/src/cephadm/cephadmlib/agent.py
index 71924c39ccfe..330ea6945f34 100644
--- a/src/cephadm/cephadmlib/agent.py
+++ b/src/cephadm/cephadmlib/agent.py
@@ -6,13 +6,14 @@
 logger = logging.getLogger()
 
 
-def http_query(addr: str = '',
-               port: str = '',
-               data: Optional[bytes] = None,
-               endpoint: str = '',
-               ssl_ctx: Optional[Any] = None,
-               timeout: Optional[int] = 10) -> Tuple[int, str]:
-
+def http_query(
+    addr: str = '',
+    port: str = '',
+    data: Optional[bytes] = None,
+    endpoint: str = '',
+    ssl_ctx: Optional[Any] = None,
+    timeout: Optional[int] = 10,
+) -> Tuple[int, str]:
     url = f'https://{addr}:{port}{endpoint}'
     logger.debug(f'sending query to {url}')
     try:

From dcea1c8c7af07f5a59052a21326cd49e6b34d4fd Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:35:30 -0500
Subject: [PATCH 2235/2492] cephadm: black format container_daemon_form.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_daemon_form.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/container_daemon_form.py b/src/cephadm/cephadmlib/container_daemon_form.py
index cab57871b8de..8696c9cbd66f 100644
--- a/src/cephadm/cephadmlib/container_daemon_form.py
+++ b/src/cephadm/cephadmlib/container_daemon_form.py
@@ -40,7 +40,9 @@ def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
         """
         return []
 
-    def sidecar_containers(self, ctx: CephadmContext) -> List[SidecarContainer]:
+    def sidecar_containers(
+        self, ctx: CephadmContext
+    ) -> List[SidecarContainer]:
         """Returns a list of sidecar containers that should be executed along
         with the primary service container.
         """
@@ -167,7 +169,9 @@ def daemon_to_container(
         ctx.container_engine.update_mounts(ctx, container_mounts)
     if auto_podman_args and _is_podman:
         container_args.extend(
-            ctx.container_engine.service_args(ctx, daemon.identity.service_name)
+            ctx.container_engine.service_args(
+                ctx, daemon.identity.service_name
+            )
         )
 
     return CephContainer.for_daemon(

From 275b44df1a41708622a736f89c22abf29bc0a558 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:35:46 -0500
Subject: [PATCH 2236/2492] cephadm: black format container_types.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 2af16eb47504..01fbb41d2392 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -147,13 +147,7 @@ def build_engine_run_args(self) -> List[str]:
             [],
         )
 
-        return (
-            cmd_args
-            + self.container_args
-            + envs
-            + vols
-            + binds
-        )
+        return cmd_args + self.container_args + envs + vols + binds
 
     def build_run_cmd(self) -> List[str]:
         return (
@@ -190,7 +184,7 @@ def from_container(
         cls,
         other: 'BasicContainer',
         *,
-        ident: Optional[DaemonIdentity] = None
+        ident: Optional[DaemonIdentity] = None,
     ) -> 'BasicContainer':
         return cls(
             other.ctx,
@@ -582,7 +576,6 @@ def extract_uid_gid(
     img: str = '',
     file_path: Union[str, List[str]] = '/var/lib/ceph',
 ) -> Tuple[int, int]:
-
     if not img:
         img = ctx.image
 

From 33d2bdf269376f17c9153942e6cda9c26118d94e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:36:11 -0500
Subject: [PATCH 2237/2492] cephadm: black format firewalld.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/firewalld.py | 80 ++++++++++++++++++++++-------
 1 file changed, 61 insertions(+), 19 deletions(-)

diff --git a/src/cephadm/cephadmlib/firewalld.py b/src/cephadm/cephadmlib/firewalld.py
index f47e7e71d4dc..ea035790d652 100644
--- a/src/cephadm/cephadmlib/firewalld.py
+++ b/src/cephadm/cephadmlib/firewalld.py
@@ -14,7 +14,6 @@
 
 
 class Firewalld(object):
-
     # for specifying ports we should always open when opening
     # ports for a daemon of that type. Main use case is for ports
     # that we should open when deploying the daemon type but that
@@ -51,26 +50,44 @@ def check(self):
     def enable_service_for(self, svc: str) -> None:
         assert svc, 'service name not provided'
         if not self.available:
-            logger.debug('Not possible to enable service <%s>. firewalld.service is not available' % svc)
+            logger.debug(
+                'Not possible to enable service <%s>. firewalld.service is not available'
+                % svc
+            )
             return
 
         if not self.cmd:
             raise RuntimeError('command not defined')
 
-        out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-service', svc], verbosity=CallVerbosity.DEBUG)
+        out, err, ret = call(
+            self.ctx,
+            [self.cmd, '--permanent', '--query-service', svc],
+            verbosity=CallVerbosity.DEBUG,
+        )
         if ret:
-            logger.info('Enabling firewalld service %s in current zone...' % svc)
-            out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-service', svc])
+            logger.info(
+                'Enabling firewalld service %s in current zone...' % svc
+            )
+            out, err, ret = call(
+                self.ctx, [self.cmd, '--permanent', '--add-service', svc]
+            )
             if ret:
                 raise RuntimeError(
-                    'unable to add service %s to current zone: %s' % (svc, err))
+                    'unable to add service %s to current zone: %s'
+                    % (svc, err)
+                )
         else:
-            logger.debug('firewalld service %s is enabled in current zone' % svc)
+            logger.debug(
+                'firewalld service %s is enabled in current zone' % svc
+            )
 
     def open_ports(self, fw_ports):
         # type: (List[int]) -> None
         if not self.available:
-            logger.debug('Not possible to open ports <%s>. firewalld.service is not available' % fw_ports)
+            logger.debug(
+                'Not possible to open ports <%s>. firewalld.service is not available'
+                % fw_ports
+            )
             return
 
         if not self.cmd:
@@ -78,20 +95,36 @@ def open_ports(self, fw_ports):
 
         for port in fw_ports:
             tcp_port = str(port) + '/tcp'
-            out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG)
+            out, err, ret = call(
+                self.ctx,
+                [self.cmd, '--permanent', '--query-port', tcp_port],
+                verbosity=CallVerbosity.DEBUG,
+            )
             if ret:
-                logger.info('Enabling firewalld port %s in current zone...' % tcp_port)
-                out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-port', tcp_port])
+                logger.info(
+                    'Enabling firewalld port %s in current zone...' % tcp_port
+                )
+                out, err, ret = call(
+                    self.ctx,
+                    [self.cmd, '--permanent', '--add-port', tcp_port],
+                )
                 if ret:
-                    raise RuntimeError('unable to add port %s to current zone: %s' %
-                                       (tcp_port, err))
+                    raise RuntimeError(
+                        'unable to add port %s to current zone: %s'
+                        % (tcp_port, err)
+                    )
             else:
-                logger.debug('firewalld port %s is enabled in current zone' % tcp_port)
+                logger.debug(
+                    'firewalld port %s is enabled in current zone' % tcp_port
+                )
 
     def close_ports(self, fw_ports):
         # type: (List[int]) -> None
         if not self.available:
-            logger.debug('Not possible to close ports <%s>. firewalld.service is not available' % fw_ports)
+            logger.debug(
+                'Not possible to close ports <%s>. firewalld.service is not available'
+                % fw_ports
+            )
             return
 
         if not self.cmd:
@@ -99,13 +132,22 @@ def close_ports(self, fw_ports):
 
         for port in fw_ports:
             tcp_port = str(port) + '/tcp'
-            out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG)
+            out, err, ret = call(
+                self.ctx,
+                [self.cmd, '--permanent', '--query-port', tcp_port],
+                verbosity=CallVerbosity.DEBUG,
+            )
             if not ret:
                 logger.info('Disabling port %s in current zone...' % tcp_port)
-                out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--remove-port', tcp_port])
+                out, err, ret = call(
+                    self.ctx,
+                    [self.cmd, '--permanent', '--remove-port', tcp_port],
+                )
                 if ret:
-                    raise RuntimeError('unable to remove port %s from current zone: %s' %
-                                       (tcp_port, err))
+                    raise RuntimeError(
+                        'unable to remove port %s from current zone: %s'
+                        % (tcp_port, err)
+                    )
                 else:
                     logger.info(f'Port {tcp_port} disabled')
             else:

From 18d5723dabd574c31c9696244a7dfe9a9eaabb15 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:36:36 -0500
Subject: [PATCH 2238/2492] cephadm: black format sysctl.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/sysctl.py | 43 ++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/cephadm/cephadmlib/sysctl.py b/src/cephadm/cephadmlib/sysctl.py
index 66a8b0c5ff3e..6c9693ee96ac 100644
--- a/src/cephadm/cephadmlib/sysctl.py
+++ b/src/cephadm/cephadmlib/sysctl.py
@@ -16,10 +16,13 @@
 logger = logging.getLogger()
 
 
-def install_sysctl(ctx: CephadmContext, fsid: str, daemon: DaemonForm) -> None:
+def install_sysctl(
+    ctx: CephadmContext, fsid: str, daemon: DaemonForm
+) -> None:
     """
     Set up sysctl settings
     """
+
     def _write(conf: Path, lines: List[str]) -> None:
         lines = [
             '# created by cephadm',
@@ -54,11 +57,14 @@ def sysctl_get(ctx: CephadmContext, variable: str) -> Union[str, None]:
     return out or None
 
 
-def filter_sysctl_settings(ctx: CephadmContext, lines: List[str]) -> List[str]:
+def filter_sysctl_settings(
+    ctx: CephadmContext, lines: List[str]
+) -> List[str]:
     """
     Given a list of sysctl settings, examine the system's current configuration
     and return those which are not currently set as described.
     """
+
     def test_setting(desired_line: str) -> bool:
         # Remove any comments
         comment_start = desired_line.find('#')
@@ -67,11 +73,14 @@ def test_setting(desired_line: str) -> bool:
         desired_line = desired_line.strip()
         if not desired_line or desired_line.isspace():
             return False
-        setting, desired_value = map(lambda s: s.strip(), desired_line.split('='))
+        setting, desired_value = map(
+            lambda s: s.strip(), desired_line.split('=')
+        )
         if not setting or not desired_value:
             return False
         actual_value = sysctl_get(ctx, setting)
         return desired_value != actual_value
+
     return list(filter(test_setting, lines))
 
 
@@ -81,36 +90,50 @@ def migrate_sysctl_dir(ctx: CephadmContext, fsid: str) -> None:
     This moves it to '/etc/sysctl.d'.
     """
     deprecated_location: str = '/usr/lib/sysctl.d'
-    deprecated_confs: List[str] = glob(f'{deprecated_location}/90-ceph-{fsid}-*.conf')
+    deprecated_confs: List[str] = glob(
+        f'{deprecated_location}/90-ceph-{fsid}-*.conf'
+    )
     if not deprecated_confs:
         return
 
     file_count: int = len(deprecated_confs)
-    logger.info(f'Found sysctl {file_count} files in deprecated location {deprecated_location}. Starting Migration.')
+    logger.info(
+        f'Found sysctl {file_count} files in deprecated location {deprecated_location}. Starting Migration.'
+    )
     for conf in deprecated_confs:
         try:
             shutil.move(conf, ctx.sysctl_dir)
             file_count -= 1
         except shutil.Error as err:
             if str(err).endswith('already exists'):
-                logger.warning(f'Destination file already exists. Deleting {conf}.')
+                logger.warning(
+                    f'Destination file already exists. Deleting {conf}.'
+                )
                 try:
                     os.unlink(conf)
                     file_count -= 1
                 except OSError as del_err:
                     logger.warning(f'Could not remove {conf}: {del_err}.')
             else:
-                logger.warning(f'Could not move {conf} from {deprecated_location} to {ctx.sysctl_dir}: {err}')
+                logger.warning(
+                    f'Could not move {conf} from {deprecated_location} to {ctx.sysctl_dir}: {err}'
+                )
 
     # Log successful migration
     if file_count == 0:
-        logger.info(f'Successfully migrated sysctl config to {ctx.sysctl_dir}.')
+        logger.info(
+            f'Successfully migrated sysctl config to {ctx.sysctl_dir}.'
+        )
         return
 
     # Log partially successful / unsuccessful migration
     files_processed: int = len(deprecated_confs)
     if file_count < files_processed:
-        status: str = f'partially successful (failed {file_count}/{files_processed})'
+        status: str = (
+            f'partially successful (failed {file_count}/{files_processed})'
+        )
     elif file_count == files_processed:
         status = 'unsuccessful'
-    logger.warning(f'Migration of sysctl configuration {status}. You may want to perform a migration manually.')
+    logger.warning(
+        f'Migration of sysctl configuration {status}. You may want to perform a migration manually.'
+    )

From 48767dc8f9b1bc673e362beecbfcb0424bf4fcd0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:36:58 -0500
Subject: [PATCH 2239/2492] cephadm: black format systemd_unit.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/systemd_unit.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/systemd_unit.py b/src/cephadm/cephadmlib/systemd_unit.py
index b0f62b9cbfac..d3543174a8df 100644
--- a/src/cephadm/cephadmlib/systemd_unit.py
+++ b/src/cephadm/cephadmlib/systemd_unit.py
@@ -78,7 +78,10 @@ def _write_init_containers_unit_file(
 
 
 def _write_sidecar_unit_file(
-    dest: IO, ctx: CephadmContext, primary: DaemonIdentity, sidecar: DaemonSubIdentity
+    dest: IO,
+    ctx: CephadmContext,
+    primary: DaemonIdentity,
+    sidecar: DaemonSubIdentity,
 ) -> None:
     has_docker_engine = isinstance(ctx.container_engine, Docker)
     has_podman_engine = isinstance(ctx.container_engine, Podman)
@@ -128,9 +131,7 @@ def _install_extended_systemd_services(
             difh = estack.enter_context(
                 write_new(pinfo.drop_in_file, perms=None)
             )
-            _write_drop_in(
-                difh, ctx, identity, enable_init_containers, sids
-            )
+            _write_drop_in(difh, ctx, identity, enable_init_containers, sids)
 
 
 def _get_unit_file(ctx: CephadmContext, fsid: str) -> str:

From 3cd5c237837aaa1841f2675cfe08886606b4dc62 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:37:14 -0500
Subject: [PATCH 2240/2492] cephadm: black format daemons/ceph.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/ceph.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/daemons/ceph.py b/src/cephadm/cephadmlib/daemons/ceph.py
index 45562fe4fdd4..e6392876cc60 100644
--- a/src/cephadm/cephadmlib/daemons/ceph.py
+++ b/src/cephadm/cephadmlib/daemons/ceph.py
@@ -327,7 +327,9 @@ def get_daemon_args(self) -> List[str]:
 
     def validate(self) -> None:
         if not os.path.isdir(self.sock_dir):
-            raise Error(f'Desired sock dir for ceph-exporter is not directory: {self.sock_dir}')
+            raise Error(
+                f'Desired sock dir for ceph-exporter is not directory: {self.sock_dir}'
+            )
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = daemon_to_container(ctx, self)

From 23777993bf61ae9f781eb6908f700e3904e62351 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:37:27 -0500
Subject: [PATCH 2241/2492] cephadm: black format daemons/node_proxy.py

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/node_proxy.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/node_proxy.py b/src/cephadm/cephadmlib/daemons/node_proxy.py
index c197ded3ff6c..f7f0097e7b85 100644
--- a/src/cephadm/cephadmlib/daemons/node_proxy.py
+++ b/src/cephadm/cephadmlib/daemons/node_proxy.py
@@ -81,9 +81,17 @@ def customize_container_mounts(
         data_dir = self.identity.data_dir(ctx.data_dir)
         # TODO: update this when we have the actual location
         # in the ceph container we are going to keep node-proxy
-        mounts.update({os.path.join(data_dir, 'node-proxy.json'): '/usr/share/ceph/node-proxy.json:z'})
+        mounts.update(
+            {
+                os.path.join(
+                    data_dir, 'node-proxy.json'
+                ): '/usr/share/ceph/node-proxy.json:z'
+            }
+        )
 
-    def customize_process_args(self, ctx: CephadmContext, args: List[str]) -> None:
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
         # TODO: this corresponds with the mount location of
         # the config in _get_container_mounts above. They
         # will both need to be updated when we have a proper

From 155a91bfb0b6fdf9563a907b2aa5740e00046e6e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:37:49 -0500
Subject: [PATCH 2242/2492] cephadm: use check-black environment, drop fix

A few months ago when we were starting the cephadm refactoring effort in
earnest we agreed to use `black` to format the python files we moved
into `cephadmlib` (while leaving the formatting of cephadm.py as-is).
We created tox rules to check or apply black formatting but promptly
got lazy and stopped running the checks manually - they were not run
automatically.

Change the default environment list to call check-black to start
enforcing the use of black style formatting. At the same time, remove
the `fix` environment entirely. It is liable to leave uncommitted changes
in your working tree when run - unlike check-black. This moves cephadm
practices closer to ones I've seen widely used in other python
codebases.

Note that cephadm.py remains outside of this rule - it only requires
pep8 formatting (via flake8). cephadm.py should continue to be
disassembled and components moved into cephadmlib over time. flake8 also remains
in effect for cephadmlib, it's fast, finds obvious mistakes and does
some code-level linting (like unused vars) that black does not.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tox.ini | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 4871c186b3e3..8c3728ff4c9b 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -2,8 +2,8 @@
 envlist =
     py3
     mypy
-    fix
     flake8
+    check-black
 skipsdist = true
 
 [flake8]
@@ -50,14 +50,6 @@ deps =
     -c{toxinidir}/../mypy-constrains.txt
 commands = mypy --config-file ../mypy.ini {posargs:cephadm.py cephadmlib}
 
-[testenv:fix]
-basepython = python3
-deps =
-    autopep8
-commands =
-    python --version
-    autopep8 {[autopep8]addopts} {posargs: cephadm.py}
-
 [testenv:flake8]
 basepython = python3
 allowlist_externals = bash

From 2552ac9a7d479ee27cc3241b5e20f3cd85fd521c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 9 Mar 2024 11:50:11 -0500
Subject: [PATCH 2243/2492] cephadm: reorder tox env to run fastest tasks first

Reorder the tox environments so that the simplest, fastest running tools
execute first. This can be handy when running tox locally (say in a
pre-commit workflow) and you get to see a formatting or mypy error
before the slower unit tests begin you can abort the tests and fix up
the formatting instead of waiting for all unit tests to run only to
find out you need to make changes to fix formatting problems.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 8c3728ff4c9b..a6b2601c76e2 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -1,9 +1,9 @@
 [tox]
 envlist =
-    py3
-    mypy
     flake8
+    mypy
     check-black
+    py3
 skipsdist = true
 
 [flake8]

From ebedfa7ef0858b52eb7cd652f67bbd8eef6e9674 Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Tue, 27 Feb 2024 15:07:17 +0800
Subject: [PATCH 2244/2492] CMakeLists: disable Seastar_IO_URING to fix seastar
 unittest timeout

As pr https://github.com/ceph/ceph/pull/55787 bump liburing from 0.7 to 2.5.
with liburing-dev (2.1) installed on ubuntu jammy, Seastar_IO_URING will be set ON,
seastar will be builded with liburing-dev. Ceph bluestore use build_uring in file Builduring.cmake
to build liburing version 2.5 and set URING_INCLUDE_DIR to
/home/jenkins-build/build/workspace/ceph-pull-requests/build/src/liburing/src/include/liburing/,
seastar use URING_INCLUDE_DIR(version is 2.5) to build, but seastar link liburing.so to system
liburing-dev package(version 2.1). The liburing head file seastar building and liburing binary
seastar linked is mismatched.
I have downgraded the liburing version in file 'cmake/modules/Builduring.cmake' to liburing-2.1,
the seastar unittests work fine, no timeout.

Fixes: https://tracker.ceph.com/issues/64789

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 src/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e44cecca5cf5..90c9c48e06e1 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -380,6 +380,7 @@ if(WITH_SEASTAR)
   endmacro ()
   set(Seastar_API_LEVEL "6" CACHE STRING "" FORCE)
   set(Seastar_HWLOC OFF CACHE BOOL "" FORCE)
+  set(Seastar_IO_URING OFF CACHE BOOL "" FORCE)
   set(Seastar_STD_OPTIONAL_VARIANT_STRINGVIEW ON CACHE BOOL "" FORCE)
   if(Seastar_DPDK)
     find_package(dpdk QUIET)

From 75327c5b56591c6a29ad47745df24d16320f5a99 Mon Sep 17 00:00:00 2001
From: Jiffin Tony Thottan <thottanjiffin@gmail.com>
Date: Wed, 2 Aug 2023 13:23:12 +0530
Subject: [PATCH 2245/2492] cephadm: adjust the ingress ha proxy health check
 interval

Currently health checker uses default value of 2s, it is send list
bucket request for every 2s. This seems to be frequent and need to
adjust properly. Hence introducing new setting health_check_interval in
the ingress spec for haproxy.

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

Apply suggestions from code review

Co-authored-by: Adam King <47704447+adk3798@users.noreply.github.com>
Signed-off-by: Jiffin Tony Thottan  <thottanjiffin@gmail.com>
---
 doc/cephadm/services/rgw.rst                           |  5 +++++
 src/pybind/mgr/cephadm/services/ingress.py             |  1 +
 .../cephadm/templates/services/ingress/haproxy.cfg.j2  |  2 +-
 src/pybind/mgr/cephadm/tests/test_services.py          |  6 +++---
 src/python-common/ceph/deployment/service_spec.py      | 10 ++++++++++
 5 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
index 20ec39a88dd1..ed0b149365a5 100644
--- a/doc/cephadm/services/rgw.rst
+++ b/doc/cephadm/services/rgw.rst
@@ -246,6 +246,7 @@ It is a yaml format file with the following properties:
       virtual_interface_networks: [ ... ]       # optional: list of CIDR networks
       use_keepalived_multicast: <bool>          # optional: Default is False.
       vrrp_interface_network: <string>/<string> # optional: ex: 192.168.20.0/24
+      health_check_interval: <string>           # optional: Default is 2s.
       ssl_cert: |                               # optional: SSL certificate and key
         -----BEGIN CERTIFICATE-----
         ...
@@ -273,6 +274,7 @@ It is a yaml format file with the following properties:
       monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
       virtual_interface_networks: [ ... ] # optional: list of CIDR networks
       first_virtual_router_id: <integer>  # optional: default 50
+      health_check_interval: <string>     # optional: Default is 2s.
       ssl_cert: |                         # optional: SSL certificate and key
         -----BEGIN CERTIFICATE-----
         ...
@@ -321,6 +323,9 @@ where the properties of this service specification are:
     keepalived will have different virtual_router_id. In the case of using ``virtual_ips_list``,
     each IP will create its own virtual router. So the first one will have ``first_virtual_router_id``,
     second one will have ``first_virtual_router_id`` + 1, etc. Valid values go from 1 to 255.
+* ``health_check_interval``
+    Default is 2 seconds. This parameter can be used to set the interval between health checks
+    for the haproxy with the backend servers.
 
 .. _ingress-virtual-ip:
 
diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py
index 55be3045466a..5edd2517dc01 100644
--- a/src/pybind/mgr/cephadm/services/ingress.py
+++ b/src/pybind/mgr/cephadm/services/ingress.py
@@ -187,6 +187,7 @@ def haproxy_generate_config(
                 'monitor_port': daemon_spec.ports[1] if daemon_spec.ports else spec.monitor_port,
                 'local_host_ip': host_ip,
                 'default_server_opts': server_opts,
+                'health_check_interval': spec.health_check_interval or '2s',
             }
         )
         config_files = {
diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
index c114a8cba115..9a0309ab409d 100644
--- a/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
+++ b/src/pybind/mgr/cephadm/templates/services/ingress/haproxy.cfg.j2
@@ -74,7 +74,7 @@ backend backend
     balance static-rr
     option httpchk HEAD / HTTP/1.0
     {% for server in servers %}
-    server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100
+    server {{ server.name }} {{ server.ip }}:{{ server.port }} check weight 100 inter {{ health_check_interval }}
     {% endfor %}
 {% endif %}
 {% if mode == 'tcp' %}
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index 30b121cc3726..8998d6f238e2 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -1854,7 +1854,7 @@ def test_ingress_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator)
                                 'balance static-rr\n    '
                                 'option httpchk HEAD / HTTP/1.0\n    '
                                 'server '
-                                + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100\n'
+                                + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n'
                         }
                 }
 
@@ -1979,7 +1979,7 @@ def test_ingress_config_ssl_rgw(self, _run_cephadm, cephadm_module: CephadmOrche
                                 'balance static-rr\n    '
                                 'option httpchk HEAD / HTTP/1.0\n    '
                                 'server '
-                                + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100\n'
+                                + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100 inter 2s\n'
                         }
                 }
 
@@ -2103,7 +2103,7 @@ def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOr
                                 'balance static-rr\n    '
                                 'option httpchk HEAD / HTTP/1.0\n    '
                                 'server '
-                                + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100\n'
+                                + haproxy_generated_conf[1][0] + ' 1.2.3.7:80 check weight 100 inter 2s\n'
                         }
                 }
 
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index db7beac4dc22..7edd14ce28e5 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1463,6 +1463,7 @@ def __init__(self,
                  extra_container_args: Optional[GeneralArgList] = None,
                  extra_entrypoint_args: Optional[GeneralArgList] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
+                 health_check_interval: Optional[str] = None,
                  ):
         assert service_type == 'ingress'
 
@@ -1495,6 +1496,8 @@ def __init__(self,
         self.ssl = ssl
         self.keepalive_only = keepalive_only
         self.enable_haproxy_protocol = enable_haproxy_protocol
+        self.health_check_interval = health_check_interval.strip(
+        ) if health_check_interval else None
 
     def get_port_start(self) -> List[int]:
         ports = []
@@ -1525,6 +1528,13 @@ def validate(self) -> None:
         if self.virtual_ip is not None and self.virtual_ips_list is not None:
             raise SpecValidationError(
                 'Cannot add ingress: Single and multiple virtual IPs specified')
+        if self.health_check_interval:
+            valid_units = ['s', 'm', 'h']
+            m = re.search(rf"^(\d+)({'|'.join(valid_units)})$", self.health_check_interval)
+            if not m:
+                raise SpecValidationError(
+                    f'Cannot add ingress: Invalid health_check_interval specified. '
+                    f'Valid units are: {valid_units}')
 
 
 yaml.add_representer(IngressSpec, ServiceSpec.yaml_representer)

From 96a5cbb9aa47ed1a2b51e720475ebc08012cb4cc Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 19 Feb 2024 09:01:48 -0500
Subject: [PATCH 2246/2492] rgw_lc:  replace strftime w/fmt and chrono:calendar

It's reliably claimed that std::strftime is not
mt-safe, and this would be a likely root cause of
intermittent scrambled expiration header output cases
that have been reported.

Fixes: https://tracker.ceph.com/issues/63973
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc           | 18 ++++++------------
 src/test/rgw/test_rgw_lc.cc | 12 ++++++++++++
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index f98131a59e6a..18e2340c9d55 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab ft=cpp
 
+#include <fmt/chrono.h>
 #include <string.h>
 #include <iostream>
 #include <map>
@@ -2816,18 +2817,11 @@ std::string s3_expiration_header(
 
   // cond format header
   if (expiration_date && rule_id) {
-    // Fri, 23 Dec 2012 00:00:00 GMT
-    char exp_buf[100];
-    time_t exp = ceph::real_clock::to_time_t(*expiration_date);
-    if (std::strftime(exp_buf, sizeof(exp_buf),
-		      "%a, %d %b %Y %T %Z", std::gmtime(&exp))) {
-      hdr = fmt::format("expiry-date=\"{0}\", rule-id=\"{1}\"", exp_buf,
-			*rule_id);
-    } else {
-      ldpp_dout(dpp, 0) << __func__ <<
-	"() strftime of life cycle expiration header failed"
-			<< dendl;
-    }
+    auto exp = ceph::real_clock::to_time_t(*expiration_date);
+    // Fri, 21 Dec 2012 00:00:00 GMT
+    auto exp_str = fmt::format("{:%a, %d %b %Y %T %Z}", fmt::gmtime(exp));
+    hdr = fmt::format("expiry-date=\"{0}\", rule-id=\"{1}\"", exp_str,
+		      *rule_id);
   }
 
   return hdr;
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index d10b482cbfce..057deccbe207 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -7,6 +7,7 @@
 #include <gtest/gtest.h>
 #include <string>
 #include <vector>
+#include <chrono>
 #include <stdexcept>
 
 static const char* xmldoc_1 =
@@ -107,6 +108,17 @@ TEST(TestLCFilterInvalidAnd, XMLDoc3)
   ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
 }
 
+TEST(ExpHdr, ReplaceStrftime)
+{
+  using namespace std::chrono;
+
+  constexpr auto dec21 = year(2012)/12/21;
+  auto exp = sys_days(dec21) + 9h + 13min + 7s ;
+  auto exp_str = fmt::format("{:%a, %d %b %Y %T %Z}", fmt::gmtime(exp));
+  std::cout << "exp_str: " << exp_str << std::endl;
+  ASSERT_EQ(exp_str, "Fri, 21 Dec 2012 09:13:07 GMT");
+}
+
 struct LCWorkTimeTests : ::testing::Test
 {
    CephContext* cct;

From 0f0c3936689483b09c350f833087f06f00b19449 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 11 Mar 2024 12:54:01 -0500
Subject: [PATCH 2247/2492] osd/scrub: handle 'release' events sent during
 'scrub abort'

Scenario:
- the replica is reserved;
- the Primary initiates a chunk operation;
- the replica is in ReplicaActive/ReplicaActiveOp/ReplicaBuildingMap
- 'no-scrub' is set, and the Primary sends a 'release' event to the
  replica.

Desired behavior:
- the replica aborts the chunk operation and transitions to
  ReplicaReserved;
- the 'release' event is delivered in the new state.

Fixes: https://tracker.ceph.com/issues/64827
Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_machine.cc | 10 +++++++++-
 src/osd/scrubber/scrub_machine.h  | 10 ++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc
index ce4196e1ebbb..33da35cd8c85 100644
--- a/src/osd/scrubber/scrub_machine.cc
+++ b/src/osd/scrubber/scrub_machine.cc
@@ -886,7 +886,7 @@ void ReplicaActive::clear_remote_reservation(bool warn_if_no_reservation)
   dout(10) << fmt::format(
 		  "ReplicaActive::clear_remote_reservation(): "
 		  "pending_reservation_nonce {}, reservation_granted {}",
-		  reservation_granted, pending_reservation_nonce)
+		  pending_reservation_nonce, reservation_granted)
 	   << dendl;
   if (reservation_granted || pending_reservation_nonce) {
     m_osds->get_scrub_reserver().cancel_reservation(pg_id);
@@ -1149,6 +1149,14 @@ sc::result ReplicaActiveOp::react(const StartReplica&)
   return transit<ReplicaActiveOp>();
 }
 
+sc::result ReplicaActiveOp::react(const ReplicaRelease& ev)
+{
+  dout(10) << "ReplicaActiveOp::react(const ReplicaRelease&)" << dendl;
+  post_event(ev);
+  return transit<sc::shallow_history<ReplicaReserved>>();
+}
+
+
 // ------------- ReplicaActive/ReplicaWaitUpdates ------------------------
 
 ReplicaWaitUpdates::ReplicaWaitUpdates(my_context ctx)
diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h
index 254e7861ed95..d56059c6bf87 100644
--- a/src/osd/scrubber/scrub_machine.h
+++ b/src/osd/scrubber/scrub_machine.h
@@ -1047,6 +1047,7 @@ struct ReplicaActiveOp
 
   using reactions = mpl::list<
       sc::custom_reaction<StartReplica>,
+      sc::custom_reaction<ReplicaRelease>,
       sc::transition<FullReset, ReplicaIdle>>;
 
   /**
@@ -1060,6 +1061,15 @@ struct ReplicaActiveOp
    * - and we should log this unexpected scenario clearly in the cluster log.
    */
   sc::result react(const StartReplica&);
+
+  /**
+   * a 'release' was send by the primary. Possible scenario: 'no-scrub'
+   * abort. Our two-steps reaction:
+   * - we exit the 'ActiveOp' state, and
+   * - we make sure the 'release' is remembered, to be handled by the state
+   *   we would transition into (which should be ReplicaReserved).
+   */
+  sc::result react(const ReplicaRelease&);
 };
 
 /*

From 19a9f79b5f218d8cb757579bf8c0f5d610dde3a8 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Fri, 1 Mar 2024 13:43:12 +0100
Subject: [PATCH 2248/2492] mgr/dashboard: ceph authenticate user from fs

Fixes: https://tracker.ceph.com/issues/64660
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       |  23 +++
 .../cephfs-auth-modal.component.html          | 166 ++++++++++++++++++
 .../cephfs-auth-modal.component.scss          |   0
 .../cephfs-auth-modal.component.spec.ts       |  29 +++
 .../cephfs-auth-modal.component.ts            | 129 ++++++++++++++
 .../cephfs-list/cephfs-list.component.ts      |  19 ++
 .../cephfs-tabs/cephfs-tabs.component.html    |   1 +
 .../src/app/ceph/cephfs/cephfs.module.ts      |   4 +-
 .../src/app/shared/api/cephfs.service.ts      |   9 +
 .../src/app/shared/constants/app.constants.ts |   2 +
 .../shared/services/task-message.service.ts   |   7 +
 src/pybind/mgr/dashboard/openapi.yaml         |  52 ++++++
 12 files changed, 440 insertions(+), 1 deletion(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.ts

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index e1e271b69dcc..6410a73785ec 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -101,6 +101,29 @@ def rename(self, name: str, new_name: str):
                 component='cephfs')
         return f'Volume {name} renamed successfully to {new_name}'
 
+    @UpdatePermission
+    @Endpoint('PUT')
+    @EndpointDoc("Set Ceph authentication capabilities for the specified user ID in the given path",
+                 parameters={
+                     'fs_name': (str, 'File system name'),
+                     'client_id': (str, 'Cephx user ID'),
+                     'caps': (str, 'Path and given capabilities'),
+                     'root_squash': (str, 'File System Identifier'),
+
+                 })
+    def auth(self, fs_name: str, client_id: int, caps: List[str], root_squash: bool):
+        if root_squash:
+            caps.insert(2, 'root_squash')
+        error_code, _, err = mgr.mon_command({'prefix': 'fs authorize',
+                                              'filesystem': fs_name,
+                                              'entity': client_id,
+                                              'caps': caps})
+        if error_code != 0:
+            raise DashboardException(
+                msg=f'Error setting authorization for {client_id} with {caps}: {err}',
+                component='cephfs')
+        return f'Updated {client_id} authorization successfully'
+
     def get(self, fs_id):
         fs_id = self.fs_id_to_int(fs_id)
         return self.fs_status(fs_id)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.html
new file mode 100644
index 000000000000..290504bf3a86
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.html
@@ -0,0 +1,166 @@
+<cd-modal [modalRef]="activeModal">
+  <ng-container i18n="form title"
+                class="modal-title">{{ action | titlecase }} {{ resource | upperFirst }}</ng-container>
+  <ng-container class="modal-content"
+                *cdFormLoading="loading">
+    <form name="form"
+          #formDir="ngForm"
+          [formGroup]="form">
+      <div class="modal-body">
+
+        <!-- FsName -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="userId"
+                 i18n>Fs name
+          </label>
+          <div class="cd-col-form-input">
+            <input id="fsName"
+                   name="fsName"
+                   type="text"
+                   class="form-control"
+                   formControlName="fsName">
+            <span class="invalid-feedback"
+                  *ngIf="form.showError('fsName', formDir, 'required')"
+                  i18n>This field is required!</span>
+          </div>
+        </div>
+
+        <!-- UserId -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="userId"
+                 i18n>User ID
+            <cd-helper>
+              You can manage users from
+              <a routerLink="/ceph-users"
+                 (click)="closeModal()">Ceph Users</a>
+              page
+            </cd-helper>
+          </label>
+          <div class="cd-col-form-input">
+            <div class="input-group">
+              <span class="input-group-text"
+                    for="userId"
+                    i18n>client.
+              </span>
+              <input id="userId"
+                     name="userId"
+                     type="text"
+                     class="form-control"
+                     formControlName="userId">
+              <span class="invalid-feedback"
+                    *ngIf="form.showError('userId', formDir, 'required')"
+                    i18n>This field is required!</span>
+            </div>
+          </div>
+        </div>
+
+        <!-- Directory -->
+        <div class="form-group row">
+          <label class="cd-col-form-label required"
+                 for="directory"
+                 i18n>Directory
+            <cd-helper>Path to restrict access to</cd-helper>
+          </label>
+          <div class="cd-col-form-input">
+            <input id="typeahead-http"
+                   i18n
+                   type="text"
+                   class="form-control"
+                   disabled="directoryStore.isLoading"
+                   formControlName="directory"
+                   [ngbTypeahead]="search"
+                   [placeholder]="directoryStore.isLoading ? 'Loading directories' : 'Directory search'"
+                   i18n-placeholder>
+            <div *ngIf="directoryStore.isLoading">
+              <i [ngClass]="[icons.spinner, icons.spin, 'mt-2', 'me-2']"></i>
+            </div>
+            <span class="invalid-feedback"
+                  *ngIf="form.showError('directory', formDir, 'required')"
+                  i18n>This field is required!</span>
+          </div>
+        </div>
+
+        <!-- Permissions -->
+        <div class="form-group row">
+          <label i18n
+                 class="cd-col-form-label"
+                 for="permissions">Permissons</label>
+          <div class="cd-col-form-input">
+
+            <!-- Read -->
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="read"
+                     formControlName="read"
+                     type="checkbox">
+              <label class="custom-control-label"
+                     for="read"
+                     i18n>Read
+              </label>
+              <cd-helper i18n>Read permission is the minimum givable access</cd-helper>
+            </div>
+
+            <!-- Write -->
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="write"
+                     formControlName="write"
+                     type="checkbox"
+                     (change)="toggleFormControl()">
+              <label class="custom-control-label"
+                     for="write"
+                     i18n>Write
+              </label>
+            </div>
+
+            <!-- Quota -->
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="quota"
+                     formControlName="quota"
+                     type="checkbox">
+              <label class="custom-control-label"
+                     for="quota"
+                     i18n>Quota
+              </label>
+              <cd-helper i18n>Permission to set layouts or quotas, write access needed</cd-helper>
+            </div>
+
+            <!-- Snapshot -->
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="snapshot"
+                     formControlName="snapshot"
+                     type="checkbox">
+              <label class="custom-control-label"
+                     for="snapshot"
+                     i18n>Snapshot
+              </label>
+              <cd-helper i18n>Permission to create or delete snapshots, write access needed</cd-helper>
+            </div>
+
+            <!-- Root Squash -->
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="rootSquash"
+                     formControlName="rootSquash"
+                     type="checkbox">
+              <label class="custom-control-label"
+                     for="rootSquash"
+                     i18n>Root Squash
+              </label>
+              <cd-helper>Safety measure to prevent scenarios such as accidental sudo rm -rf /path</cd-helper>
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="modal-footer">
+        <cd-form-button-panel (submitActionEvent)="onSubmit()"
+                              [form]="form"
+                              [submitText]="(action | titlecase)"></cd-form-button-panel>
+      </div>
+    </form>
+  </ng-container>
+</cd-modal>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.scss
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.spec.ts
new file mode 100644
index 000000000000..0f0ab89cbcc8
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.spec.ts
@@ -0,0 +1,29 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { CephfsAuthModalComponent } from './cephfs-auth-modal.component';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+import { ToastrModule } from 'ngx-toastr';
+import { SharedModule } from '~/app/shared/shared.module';
+import { ReactiveFormsModule } from '@angular/forms';
+
+describe('CephfsAuthModalComponent', () => {
+  let component: CephfsAuthModalComponent;
+  let fixture: ComponentFixture<CephfsAuthModalComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [CephfsAuthModalComponent],
+      imports: [HttpClientTestingModule, SharedModule, ReactiveFormsModule, ToastrModule.forRoot()],
+      providers: [NgbActiveModal]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(CephfsAuthModalComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.ts
new file mode 100644
index 000000000000..211f2c662c3e
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component.ts
@@ -0,0 +1,129 @@
+import { Component, OnInit } from '@angular/core';
+import { FormControl, Validators } from '@angular/forms';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { OperatorFunction, Observable, of } from 'rxjs';
+import { debounceTime, distinctUntilChanged, switchMap, catchError } from 'rxjs/operators';
+import { CephfsService } from '~/app/shared/api/cephfs.service';
+import { DirectoryStoreService } from '~/app/shared/api/directory-store.service';
+import { ActionLabelsI18n } from '~/app/shared/constants/app.constants';
+import { Icons } from '~/app/shared/enum/icons.enum';
+import { CdForm } from '~/app/shared/forms/cd-form';
+import { CdFormGroup } from '~/app/shared/forms/cd-form-group';
+import { FinishedTask } from '~/app/shared/models/finished-task';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
+
+const DEBOUNCE_TIMER = 300;
+
+@Component({
+  selector: 'cd-cephfs-auth-modal',
+  templateUrl: './cephfs-auth-modal.component.html',
+  styleUrls: ['./cephfs-auth-modal.component.scss']
+})
+export class CephfsAuthModalComponent extends CdForm implements OnInit {
+  fsName: string;
+  id: number;
+  subvolumeGroup: string;
+  subvolume: string;
+  isDefaultSubvolumeGroup = false;
+  isSubvolume = false;
+  form: CdFormGroup;
+  action: string;
+  resource: string;
+  icons = Icons;
+
+  constructor(
+    public activeModal: NgbActiveModal,
+    private actionLabels: ActionLabelsI18n,
+    public directoryStore: DirectoryStoreService,
+    private cephfsService: CephfsService,
+    private taskWrapper: TaskWrapperService
+  ) {
+    super();
+    this.action = this.actionLabels.UPDATE;
+    this.resource = $localize`access`;
+  }
+
+  ngOnInit() {
+    this.directoryStore.loadDirectories(this.id, '/', 3);
+    this.createForm();
+    this.loadingReady();
+  }
+
+  createForm() {
+    this.form = new CdFormGroup({
+      fsName: new FormControl(
+        { value: this.fsName, disabled: true },
+        {
+          validators: [Validators.required]
+        }
+      ),
+      directory: new FormControl(undefined, {
+        updateOn: 'blur',
+        validators: [Validators.required]
+      }),
+      userId: new FormControl(undefined, {
+        validators: [Validators.required]
+      }),
+      read: new FormControl(
+        { value: true, disabled: true },
+        {
+          validators: [Validators.required]
+        }
+      ),
+      write: new FormControl(undefined),
+      snapshot: new FormControl({ value: false, disabled: true }),
+      quota: new FormControl({ value: false, disabled: true }),
+      rootSquash: new FormControl(undefined)
+    });
+  }
+
+  search: OperatorFunction<string, readonly string[]> = (input: Observable<string>) =>
+    input.pipe(
+      debounceTime(DEBOUNCE_TIMER),
+      distinctUntilChanged(),
+      switchMap((term) =>
+        this.directoryStore.search(term, this.id).pipe(
+          catchError(() => {
+            return of([]);
+          })
+        )
+      )
+    );
+
+  closeModal() {
+    this.activeModal.close();
+  }
+
+  onSubmit() {
+    const clientId: number = this.form.getValue('userId');
+    const caps: string[] = [this.form.getValue('directory'), this.transformPermissions()];
+    const rootSquash: boolean = this.form.getValue('rootSquash');
+    this.taskWrapper
+      .wrapTaskAroundCall({
+        task: new FinishedTask('cephfs/auth', {
+          clientId: clientId
+        }),
+        call: this.cephfsService.setAuth(this.fsName, clientId, caps, rootSquash)
+      })
+      .subscribe({
+        error: () => this.form.setErrors({ cdSubmitButton: true }),
+        complete: () => {
+          this.activeModal.close();
+        }
+      });
+  }
+
+  transformPermissions(): string {
+    const write = this.form.getValue('write');
+    const snapshot = this.form.getValue('snapshot');
+    const quota = this.form.getValue('quota');
+    return `r${write ? 'w' : ''}${quota ? 'p' : ''}${snapshot ? 's' : ''}`;
+  }
+
+  toggleFormControl() {
+    const snapshot = this.form.get('snapshot');
+    const quota = this.form.get('quota');
+    snapshot.disabled ? snapshot.enable() : snapshot.disable();
+    quota.disabled ? quota.enable() : quota.disable();
+  }
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
index 0943ed82574e..2957401d86aa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-list/cephfs-list.component.ts
@@ -25,6 +25,7 @@ import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
 import { CephfsMountDetailsComponent } from '../cephfs-mount-details/cephfs-mount-details.component';
 import { map, switchMap } from 'rxjs/operators';
 import { HealthService } from '~/app/shared/api/health.service';
+import { CephfsAuthModalComponent } from '~/app/ceph/cephfs/cephfs-auth-modal/cephfs-auth-modal.component';
 
 const BASE_URL = 'cephfs';
 
@@ -95,6 +96,12 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
         click: () =>
           this.router.navigate([this.urlBuilder.getEdit(String(this.selection.first().id))])
       },
+      {
+        name: this.actionLabels.AUTHORIZE,
+        permission: 'update',
+        icon: Icons.edit,
+        click: () => this.authorizeModal()
+      },
       {
         name: this.actionLabels.ATTACH,
         permission: 'read',
@@ -187,4 +194,16 @@ export class CephfsListComponent extends ListWithDetails implements OnInit {
 
     return true;
   }
+
+  authorizeModal() {
+    const selectedFileSystem = this.selection?.selected?.[0];
+    this.modalService.show(
+      CephfsAuthModalComponent,
+      {
+        fsName: selectedFileSystem.mdsmap['fs_name'],
+        id: selectedFileSystem.id
+      },
+      { size: 'lg' }
+    );
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
index a840692ed767..4581cc0b3c92 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-tabs/cephfs-tabs.component.html
@@ -29,6 +29,7 @@
         <cd-cephfs-subvolume-list
           [fsName]="selection.mdsmap.fs_name"
           [pools]="details.pools"
+          [id]="id"
         ></cd-cephfs-subvolume-list>
       </ng-template>
     </ng-container>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
index 14481d838224..78081faa1fec 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs.module.ts
@@ -31,6 +31,7 @@ import { DataTableModule } from '../../shared/datatable/datatable.module';
 import { CephfsSubvolumeSnapshotsFormComponent } from './cephfs-subvolume-snapshots-list/cephfs-subvolume-snapshots-form/cephfs-subvolume-snapshots-form.component';
 import { CephfsSnapshotscheduleFormComponent } from './cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component';
 import { CephfsMountDetailsComponent } from './cephfs-mount-details/cephfs-mount-details.component';
+import { CephfsAuthModalComponent } from './cephfs-auth-modal/cephfs-auth-modal.component';
 
 @NgModule({
   imports: [
@@ -66,7 +67,8 @@ import { CephfsMountDetailsComponent } from './cephfs-mount-details/cephfs-mount
     CephfsSnapshotscheduleListComponent,
     CephfsSnapshotscheduleFormComponent,
     CephfsSubvolumeSnapshotsFormComponent,
-    CephfsMountDetailsComponent
+    CephfsMountDetailsComponent,
+    CephfsAuthModalComponent
   ]
 })
 export class CephfsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
index ab43343f9edd..2d49de37c08a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs.service.ts
@@ -108,4 +108,13 @@ export class CephfsService {
       observe: 'response'
     });
   }
+
+  setAuth(fsName: string, clientId: number, caps: string[], rootSquash: boolean) {
+    return this.http.put(`${this.baseURL}/auth`, {
+      fs_name: fsName,
+      client_id: `client.${clientId}`,
+      caps: caps,
+      root_squash: rootSquash
+    });
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
index 876e22bbc1b6..185c778bc1bd 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts
@@ -147,6 +147,7 @@ export class ActionLabelsI18n {
   CONNECT: string;
   DISCONNECT: string;
   RECONNECT: string;
+  AUTHORIZE: string;
 
   constructor() {
     /* Create a new item */
@@ -206,6 +207,7 @@ export class ActionLabelsI18n {
     this.FLAGS = $localize`Flags`;
     this.ENTER_MAINTENANCE = $localize`Enter Maintenance`;
     this.EXIT_MAINTENANCE = $localize`Exit Maintenance`;
+    this.AUTHORIZE = $localize`Authorize`;
 
     this.START_DRAIN = $localize`Start Drain`;
     this.STOP_DRAIN = $localize`Stop Drain`;
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index 4fbcc09d0902..9aa9f02af616 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -369,6 +369,9 @@ export class TaskMessageService {
     'cephfs/edit': this.newTaskMessage(this.commonOperations.update, (metadata) =>
       this.volume(metadata)
     ),
+    'cephfs/auth': this.newTaskMessage(this.commonOperations.update, (metadata) =>
+      this.auth(metadata)
+    ),
     'cephfs/remove': this.newTaskMessage(this.commonOperations.remove, (metadata) =>
       this.volume(metadata)
     ),
@@ -475,6 +478,10 @@ export class TaskMessageService {
     return $localize`'${metadata.volumeName}'`;
   }
 
+  auth(metadata: any) {
+    return $localize`client.${metadata.clientId} authorization successfully`;
+  }
+
   subvolume(metadata: any) {
     return $localize`subvolume '${metadata.subVolumeName}'`;
   }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index c59532c7243d..f3d4f3607f33 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -1681,6 +1681,58 @@ paths:
       - jwt: []
       tags:
       - Cephfs
+  /api/cephfs/auth:
+    put:
+      parameters: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                caps:
+                  description: Path and given capabilities
+                  type: string
+                client_id:
+                  description: Cephx user ID
+                  type: string
+                fs_name:
+                  description: File system name
+                  type: string
+                root_squash:
+                  description: File System Identifier
+                  type: string
+              required:
+              - fs_name
+              - client_id
+              - caps
+              - root_squash
+              type: object
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource updated.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Set Ceph authentication capabilities for the specified user ID in the
+        given path
+      tags:
+      - Cephfs
   /api/cephfs/remove/{name}:
     delete:
       parameters:

From 30d02940267b0757bfc4cdffe8832198f7de76cc Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Tue, 12 Mar 2024 18:17:25 +0200
Subject: [PATCH 2249/2492] mds: Locker.cc: removing an unused variable

clearing a compiler warning.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/mds/Locker.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 0247be6db5df..9ba2b7284af1 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -4318,19 +4318,17 @@ void Locker::issue_client_lease(CDentry *dn, CInode *in, const MDRequestRef& mdr
 
 void Locker::revoke_client_leases(SimpleLock *lock)
 {
-  int n = 0;
   CDentry *dn = static_cast<CDentry*>(lock->get_parent());
   for (map<client_t, ClientLease*>::iterator p = dn->client_lease_map.begin();
        p != dn->client_lease_map.end();
        ++p) {
     ClientLease *l = p->second;
-    
-    n++;
+
     ceph_assert(lock->get_type() == CEPH_LOCK_DN);
 
     CDentry *dn = static_cast<CDentry*>(lock->get_parent());
     int mask = 1 | CEPH_LOCK_DN; // old and new bits
-    
+
     // i should also revoke the dir ICONTENT lease, if they have it!
     CInode *diri = dn->get_dir()->get_inode();
     auto lease = make_message<MClientLease>(CEPH_MDS_LEASE_REVOKE, l->seq, mask, diri->ino(), diri->first, CEPH_NOSNAP, dn->get_name());

From 560dc982eec4fbd7438cf61b353187543774b29b Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Tue, 12 Mar 2024 16:54:21 +0000
Subject: [PATCH 2250/2492] common/hobject: fix hobject_t::to_str()

Introduced in: e462f76aedbb89e8db57dc2324d4f5e9fe54cf9e
Causing the following failures:
```
[  FAILED  ] SnapMapperTest.CheckObjectKeyFormat
[  FAILED  ] SnapMapperTest.CheckRawKeyFormat
```
with the following error:
```
2024-03-09T17:31:57.117 INFO:teuthology.orchestra.run.smithi060.stdout:Expected equality of these values:
2024-03-09T17:31:57.117 INFO:teuthology.orchestra.run.smithi060.stdout:  object_key
2024-03-09T17:31:57.117 INFO:teuthology.orchestra.run.smithi060.stdout:    Which is: "OBJ_.1_7FFFFFFFFFFFFFEB.76543210.FFFFFFFFFFFFFFEC.test%uobject..namespace"
2024-03-09T17:31:57.117 INFO:teuthology.orchestra.run.smithi060.stdout:  test_object_key
2024-03-09T17:31:57.117 INFO:teuthology.orchestra.run.smithi060.stdout:    Which is: "OBJ_.1_7FFFFFFFFFFFFFEB.76543210.ffffffffffffffec.test%uobject..namespace"
```

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 src/common/hobject.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/hobject.cc b/src/common/hobject.cc
index 9f8e67f10b8b..01a117c70849 100644
--- a/src/common/hobject.cc
+++ b/src/common/hobject.cc
@@ -94,7 +94,7 @@ string hobject_t::to_str() const
     out = fmt::format(FMT_COMPILE("{:016X}.{:08X}.snapdir."), poolid, revhash);
   } else {
     out = fmt::format(
-	FMT_COMPILE("{:016X}.{:08X}.{:X}."), poolid, revhash,
+	FMT_COMPILE("{:016X}.{:08X}.{:x}."), poolid, revhash,
 	(unsigned long long)snap);
   }
 

From a14dad152c651ab8048f9e10898d4d2cc61b0d42 Mon Sep 17 00:00:00 2001
From: "J. Eric Ivancich" <ivancich@redhat.com>
Date: Mon, 11 Mar 2024 17:19:40 -0400
Subject: [PATCH 2251/2492] rgw: rgw-restore-bucket-index -- sort uses
 specified temp dir

The sort command sometimes makes use of temporary files. When the user
specifies a directory to be used for temp files, have the sort command
use that same directory.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
---
 src/rgw/rgw-restore-bucket-index | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/rgw/rgw-restore-bucket-index b/src/rgw/rgw-restore-bucket-index
index 3b00ac9767a1..a38d97068a23 100755
--- a/src/rgw/rgw-restore-bucket-index
+++ b/src/rgw/rgw-restore-bucket-index
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# version 2024-03-04
+# version 2024-03-11
 
 # rgw-restore-bucket-index is an EXPERIMENTAL tool to use in case
 # bucket index entries for objects in the bucket are somehow lost. It
@@ -65,7 +65,7 @@ EOF
 # cleans all temporary files
 clean() {
   if [ "$clean_temps" == 1 ] ;then
-    rm -f $bkt_entry $temp_file_list
+    rm -f $bkt_entry $temp_file_list \
        $zone_info $olh_info_enc $olh_info_json
   fi
 }
@@ -272,7 +272,7 @@ handle_versioned() {
 		test_temp_space
 		rados -p $pool stat2 $obj --object-locator "$loc"
 	    done | # output of stat2, which includes mtime
-	    sort -k 3 | # stat2 but sorted by mtime earlier to later
+	    sort -T $temp_dir -k 3 | # stat2 but sorted by mtime earlier to later
 	    grep -v -e "$filter_out_last_instance" | # remove the final instance in case it's not last
 
 	    # sed 1) removes pool and marker, 2) removes indicator of

From 4ee1c761e50e3c3feed89ccc5c450ac96681b778 Mon Sep 17 00:00:00 2001
From: Florent Carli <florent.carli@rte-france.com>
Date: Tue, 12 Mar 2024 18:31:16 +0100
Subject: [PATCH 2252/2492] cephadm.py: add timemaster to timesync services
 list

On debian/ubuntu, if you need PTP, it's possible to use the linuxptp package for time-synchonization.
In that case the systemd service is called timemaster and is a wrapper for chrony/ntpd/phc2sys/ptp4l.

Signed-off-by: Florent Carli <florent.carli@rte-france.com>
---
 src/cephadm/cephadm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 21523cd09ea0..ced33caee527 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4418,6 +4418,7 @@ def check_time_sync(ctx, enabler=None):
         'ntp.service',  # 18.04 (at least)
         'ntpsec.service',  # 20.04 (at least) / buster
         'openntpd.service',  # ubuntu / debian
+        'timemaster.service',  # linuxptp on ubuntu/debian
     ]
     if not check_units(ctx, units, enabler):
         logger.warning('No time sync service is running; checked for %s' % units)

From d6770f9b28726295558b225f3ee6e80707689eec Mon Sep 17 00:00:00 2001
From: Laura Flores <lflores@ibm.com>
Date: Tue, 12 Mar 2024 15:27:42 -0500
Subject: [PATCH 2253/2492] qa/suites/rados/singleton: add POOL_APP_NOT_ENABLED
 to ignorelist

Applies to the mon-config-keys.yaml file.

Fixes: https://tracker.ceph.com/issues/64725
Signed-off-by: Laura Flores <lflores@ibm.com>
---
 qa/suites/rados/singleton/all/mon-config-keys.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/suites/rados/singleton/all/mon-config-keys.yaml b/qa/suites/rados/singleton/all/mon-config-keys.yaml
index 117b6d055496..d5d1853afe02 100644
--- a/qa/suites/rados/singleton/all/mon-config-keys.yaml
+++ b/qa/suites/rados/singleton/all/mon-config-keys.yaml
@@ -16,6 +16,8 @@ tasks:
 - ceph:
     pre-mgr-commands:
       - sudo ceph config set mgr mgr_pool false --force
+    log-ignorelist:
+    - \(POOL_APP_NOT_ENABLED\)
 - workunit:
     clients:
       all:

From 46721071ffe01393ef22a4ef6f4c65f52b6d45cc Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Thu, 7 Mar 2024 16:47:51 +0530
Subject: [PATCH 2254/2492] qa: Add test for first damage of lost+found

Fixes: https://tracker.ceph.com/issues/64615
Signed-off-by: Kotresh HR <khiremat@redhat.com>
---
 .../fs/damage/test-first-damage-lost-found.sh | 196 ++++++++++++++++++
 1 file changed, 196 insertions(+)
 create mode 100755 qa/workunits/fs/damage/test-first-damage-lost-found.sh

diff --git a/qa/workunits/fs/damage/test-first-damage-lost-found.sh b/qa/workunits/fs/damage/test-first-damage-lost-found.sh
new file mode 100755
index 000000000000..2c532c0b108c
--- /dev/null
+++ b/qa/workunits/fs/damage/test-first-damage-lost-found.sh
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+set -ex
+
+FIRST_DAMAGE="first-damage.py"
+FS=cephfs
+METADATA_POOL=cephfs.a.meta
+DATA_POOL=cephfs.a.data
+MOUNT=/mnt1
+PYTHON=python3
+
+function usage {
+  printf '%s: [--fs=<fs_name>] [--metadata-pool=<pool>] [--first-damage=</path/to/first-damage.py>]\n'
+  exit 1
+}
+
+
+function create {
+  ceph config set mds mds_bal_fragment_dirs 0
+  mkdir dir1
+  DIR1_INODE=$(stat -c '%i' dir1)
+  touch dir1/file1
+  DIR1_FILE1_INODE=$(stat -c '%i' dir1/file1)
+}
+
+function flush {
+  ceph tell mds."$FS":0 flush journal
+}
+
+function damage_backtrace {
+  flush
+  ceph fs fail "$FS"
+  sleep 5
+
+  cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
+  # required here as the flush would re-write the below deleted omap
+  cephfs-journal-tool --rank="$FS":0 journal reset
+
+  #remove dir1/file1 omap entry from metadata pool
+  local DIS=$(printf '%llx.%08llx' "$DIR1_INODE" 0)
+  rados --pool="$METADATA_POOL" rmomapkey "$DIS" "file1_head"
+
+  #remove backtrace
+  local FIS=$(printf '%llx.%08llx' "$DIR1_FILE1_INODE" 0)
+  rados --pool="$DATA_POOL" rmxattr "$FIS" "parent"
+
+  ceph fs set "$FS" joinable true
+  sleep 5
+}
+
+function damage_lost_found {
+  flush
+  ceph fs fail "$FS"
+  sleep 5
+  local IS=$(printf '%llx.%08llx' "1" 0)
+
+  local T=$(mktemp -p /tmp)
+  # nuke head version of "lost+found"
+  rados --pool="$METADATA_POOL" getomapval "$IS" lost+found_head "$T"
+  printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
+  rados --pool="$METADATA_POOL" setomapval "$IS" lost+found_head --input-file="$T"
+  ceph fs set "$FS" joinable true
+  sleep 5
+}
+
+function recover_damaged_backtrace_file {
+  flush
+  ceph fs fail "$FS"
+  sleep 5
+
+  cephfs-journal-tool --rank="$FS":0 journal reset
+
+  #creates lost+found directory and recovers the damaged backtrace file
+  cephfs-data-scan cleanup
+  cephfs-data-scan init
+  cephfs-data-scan scan_extents
+  cephfs-data-scan scan_inodes
+  cephfs-data-scan scan_links
+
+  ceph fs set "$FS" joinable true
+  sleep 5
+}
+
+function recover {
+  flush
+  ceph fs fail "$FS"
+  sleep 5
+  cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
+  cephfs-journal-tool --rank="$FS":0 journal reset
+  "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
+  "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap  "$METADATA_POOL"
+  "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
+  ceph fs set "$FS" joinable true
+  sleep 5
+}
+
+function check_lost_found {
+  stat lost+found || exit 2
+}
+function check {
+  if stat lost+found; then
+    echo should be gone
+    exit 1
+  fi
+}
+
+function mount {
+  #sudo --preserve-env=CEPH_CONF bin/mount.ceph :/ "$MOUNT" -o name=admin,noshare
+  sudo bin/ceph-fuse -c ./ceph.conf /mnt1
+  df -h "$MOUNT"
+}
+
+function main {
+  eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:,python:' -- "$@")
+
+  while [ "$#" -gt 0 ]; do
+      echo "$*"
+      echo "$1"
+      case "$1" in
+          -h|--help)
+              usage
+              ;;
+          --fs)
+              FS="$2"
+              shift 2
+              ;;
+          --metadata-pool)
+              METADATA_POOL="$2"
+              shift 2
+              ;;
+          --mount)
+              MOUNT="$2"
+              shift 2
+              ;;
+          --first-damage)
+              FIRST_DAMAGE="$2"
+              shift 2
+              ;;
+          --python)
+              PYTHON="$2"
+              shift 2
+              ;;
+          --)
+              shift
+              break
+              ;;
+          *)
+              usage
+              ;;
+      esac
+  done
+
+  mount
+
+  pushd "$MOUNT"
+  create
+  popd
+
+  sudo umount -f "$MOUNT"
+
+  # flush dentries/inodes to omap
+  flush
+
+  damage_backtrace
+  # creates lost+found directory
+  recover_damaged_backtrace_file
+
+  sleep 5 # for mds to join
+  mount
+  pushd "$MOUNT"
+  sleep 5 # wait for mount to complete
+
+  # check lost+found is created
+  check_lost_found
+  popd
+  sudo umount -f "$MOUNT"
+  # flush dentries/inodes to omap
+  flush
+
+  # damage lost+found directory
+  damage_lost_found
+  recover
+
+  mount
+
+  pushd "$MOUNT"
+  sleep 5 # wait for mount to complete
+
+  #check 'lost+found' dentry should be gone
+  check
+  popd
+
+  sudo umount -f "$MOUNT"
+}
+
+main "$@"

From b1d828d1d2f31c02f225bb375d915353582d158a Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 13 Mar 2024 13:32:59 +0000
Subject: [PATCH 2255/2492] node-proxy: fix RedFishClient.logout() method

the endpoint passed down to util.query() is wrong:
is passes the full url (scheme://addr:port/path) where it should only
pass the path. The cause is that RedFishClient.login() basically stores
the value of the Location header in `self.location`.

The consequence of this is that it makes the client unable to properly logout.

Fixes: https://tracker.ceph.com/issues/64894

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/redfish_client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
index 08ee4170dcc6..d75c3db1e218 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
@@ -43,7 +43,8 @@ def login(self) -> None:
                 self.log.error(msg)
                 raise RuntimeError
             self.token = _headers['X-Auth-Token']
-            self.location = _headers['Location']
+            location_endpoint: str = _headers['Location'].split('/', 3)[-1:][0]
+            self.location = f'/{location_endpoint}'
 
     def is_logged_in(self) -> bool:
         self.log.debug(f'Checking token validity for {self.url}')

From 1abd3e659ff59d7b0c5c2360a665c395a748e3ff Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Wed, 13 Mar 2024 15:04:28 +0100
Subject: [PATCH 2256/2492] ceph-volume: add missing import

This adds the missing import `List` (from typing).

Probably missed by 36c83bf54243.

Fixes: https://tracker.ceph.com/issues/64898

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-volume/ceph_volume/devices/lvm/zap.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ceph-volume/ceph_volume/devices/lvm/zap.py b/src/ceph-volume/ceph_volume/devices/lvm/zap.py
index 7d808a5f4fc3..f512a3bde1d7 100644
--- a/src/ceph-volume/ceph_volume/devices/lvm/zap.py
+++ b/src/ceph-volume/ceph_volume/devices/lvm/zap.py
@@ -10,6 +10,7 @@
 from ceph_volume.util import system, encryption, disk, arg_validators, str_to_int, merge_dict
 from ceph_volume.util.device import Device
 from ceph_volume.systemd import systemctl
+from typing import List
 
 logger = logging.getLogger(__name__)
 mlogger = terminal.MultiLogger(__name__)

From 7a2e324a6e1c3e145d3b1e04e6f006defbe0e0b4 Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Wed, 13 Mar 2024 17:27:49 +0530
Subject: [PATCH 2257/2492] doc/rbd: add map information for clone images to
 rbd-encryption.rst

Add information on the arguments required when mapping the
formatted clone of an encrypted parent image.

Co-authored-by: Zac Dover <zac.dover@proton.me>
Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 doc/rbd/rbd-encryption.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/rbd/rbd-encryption.rst b/doc/rbd/rbd-encryption.rst
index 3f37a8b1cf85..a13839efe077 100644
--- a/doc/rbd/rbd-encryption.rst
+++ b/doc/rbd/rbd-encryption.rst
@@ -240,6 +240,18 @@ The same applies to creating a formatted clone of an unformatted
 (plaintext) image since an unformatted image does not have a header at
 all.
 
+To map a formatted clone, provide encryption formats and passphrases
+for the clone itself and all of its explicitly formatted parent images.
+The order in which ``encryption-format`` and ``encryption-passphrase-file``
+options should be provided is based on the image hierarchy: start with
+that of the cloned image, then its parent and so on.
+
+Here is an example of a command that maps a formatted clone:
+
+.. prompt:: bash #
+
+   rbd device map -t nbd -o encryption-passphrase-file=clone-passphrase.bin -o encryption-passphrase-file=passphrase.bin mypool/myclone
+
 .. _journal feature: ../rbd-mirroring/#enable-image-journaling-feature
 .. _Supported Formats: #supported-formats
 .. _rbd-nbd: ../../man/8/rbd-nbd

From d513f1ca69d38f0d7eb40a2d68275caed322cc68 Mon Sep 17 00:00:00 2001
From: Lorenz Bausch <info@lorenzbausch.de>
Date: Wed, 13 Mar 2024 17:26:51 +0100
Subject: [PATCH 2258/2492] doc: cephadm/services/osd: fix typo

Signed-off-by: Lorenz Bausch <info@lorenzbausch.de>
---
 doc/cephadm/services/osd.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst
index d9c30249b41e..440af8ce0071 100644
--- a/doc/cephadm/services/osd.rst
+++ b/doc/cephadm/services/osd.rst
@@ -234,7 +234,7 @@ Remove an OSD
 
 Removing an OSD from a cluster involves two steps:
 
-#. evacuating all placement groups (PGs) from the cluster
+#. evacuating all placement groups (PGs) from the OSD
 #. removing the PG-free OSD from the cluster
 
 The following command performs these two steps:

From bb51d78efa68a3c5460d520e93b1da6e4c543253 Mon Sep 17 00:00:00 2001
From: Teoman ONAY <tonay@ibm.com>
Date: Wed, 13 Mar 2024 17:23:00 +0100
Subject: [PATCH 2259/2492] cephadm: bootstrap fails if no container engine
 installed

When boostrapping a new cluster, if no container engine is present,
the process stops asking to install one. One of the bootstrap steps,
is to run command_prepare_host which takes care of installing the
missing dependencies but it didn't reach this step.

Fixes: https://tracker.ceph.com/issues/64899

Signed-off-by: Teoman ONAY <tonay@ibm.com>
---
 src/cephadm/cephadm.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 21523cd09ea0..95a6d18ee3bc 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -4472,6 +4472,7 @@ def command_prepare_host(ctx: CephadmContext) -> None:
         if not pkg:
             pkg = create_packager(ctx)
         pkg.install_podman()
+        ctx.container_engine = find_container_engine(ctx)
 
     logger.info('Verifying lvm2 is present...')
     if not find_executable('lvcreate'):
@@ -5508,7 +5509,8 @@ def main() -> None:
                     command_prepare_host,
                     command_add_repo,
                     command_rm_repo,
-                    command_install
+                    command_install,
+                    command_bootstrap
                 ]:
             check_container_engine(ctx)
         # command handler

From d5ac275694bce35d37d7226b2f63934e3c277cbd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Mar 2024 18:12:42 -0400
Subject: [PATCH 2260/2492] doc/radosgw: move the Zone Features section to its
 own page

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/index.rst         |  1 +
 doc/radosgw/multisite.rst     | 97 ----------------------------------
 doc/radosgw/zone-features.rst | 98 +++++++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 97 deletions(-)
 create mode 100644 doc/radosgw/zone-features.rst

diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst
index f53917ad59bc..5afd5b676179 100644
--- a/doc/radosgw/index.rst
+++ b/doc/radosgw/index.rst
@@ -43,6 +43,7 @@ Cluster with one API and then retrieve that data with the other API.
 
    HTTP Frontends <frontends>
    Multisite Configuration <multisite>
+   Zone Features <zone-features>
    Pool Placement and Storage Classes <placement>
    Multisite Sync Policy Configuration <multisite-sync-policy>
    Configuring Pools <pools>
diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
index 7c03a4d14066..6a21b7479e6f 100644
--- a/doc/radosgw/multisite.rst
+++ b/doc/radosgw/multisite.rst
@@ -1599,102 +1599,5 @@ instance.
 +-------------------------------------+-----------------------------------+---------+-----------------------+
 
 
-Zone Features
-=============
-
-Some multisite features require support from all zones before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
-
-On creation of new zones and zonegroups, all known features are supported and some features (see table below) are enabled by default. After upgrading an existing multisite configuration, however, new features must be enabled manually.
-
-Supported Features
-------------------
-
-+-----------------------------------+---------+----------+
-| Feature                           | Release | Default  |
-+===================================+=========+==========+
-| :ref:`feature_resharding`         | Reef    | Enabled  |
-+-----------------------------------+---------+----------+
-| :ref:`feature_compress_encrypted` | Reef    | Disabled |
-+-----------------------------------+---------+----------+
-
-.. _feature_resharding:
-
-resharding
-~~~~~~~~~~
-
-This feature allows buckets to be resharded in a multisite configuration
-without interrupting the replication of their objects. When
-``rgw_dynamic_resharding`` is enabled, it runs on each zone independently, and
-zones may choose different shard counts for the same bucket. When buckets are
-resharded manually with ``radosgw-admin bucket reshard``, only that zone's
-bucket is modified. A zone feature should only be marked as supported after all
-of its RGWs and OSDs have upgraded.
-
-.. note:: Dynamic resharding is not supported in multisite deployments prior to
-   the Reef release.
-
-
-.. _feature_compress_encrypted:
-
-compress-encrypted
-~~~~~~~~~~~~~~~~~~
-
-This feature enables support for combining `Server-Side Encryption`_ and
-`Compression`_ on the same object. Object data gets compressed before encryption.
-Prior to Reef, multisite would not replicate such objects correctly, so all zones
-must upgrade to Reef or later before enabling.
-
-.. warning:: The compression ratio may leak information about the encrypted data,
-   and allow attackers to distinguish whether two same-sized objects might contain
-   the same data. Due to these security considerations, this feature is disabled
-   by default.
-
-Commands
---------
-
-Add support for a zone feature
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-On the cluster that contains the given zone:
-
-.. prompt:: bash $
-
-   radosgw-admin zone modify --rgw-zone={zone-name} --enable-feature={feature-name}
-   radosgw-admin period update --commit
-
-
-Remove support for a zone feature
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-On the cluster that contains the given zone:
-
-.. prompt:: bash $
-
-   radosgw-admin zone modify --rgw-zone={zone-name} --disable-feature={feature-name}
-   radosgw-admin period update --commit
-
-Enable a zonegroup feature
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-On any cluster in the realm:
-
-.. prompt:: bash $
-
-   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --enable-feature={feature-name}
-   radosgw-admin period update --commit
-
-Disable a zonegroup feature
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-On any cluster in the realm:
-
-.. prompt:: bash $
-
-   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --disable-feature={feature-name}
-   radosgw-admin period update --commit
-
-
 .. _`Pools`: ../pools
 .. _`Sync Policy Config`: ../multisite-sync-policy
-.. _`Server-Side Encryption`: ../encryption
-.. _`Compression`: ../compression
diff --git a/doc/radosgw/zone-features.rst b/doc/radosgw/zone-features.rst
new file mode 100644
index 000000000000..d397af9db003
--- /dev/null
+++ b/doc/radosgw/zone-features.rst
@@ -0,0 +1,98 @@
+=============
+Zone Features
+=============
+
+Some multisite features require support from all zones before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
+
+On creation of new zones and zonegroups, all known features are supported and some features (see table below) are enabled by default. After upgrading an existing multisite configuration, however, new features must be enabled manually.
+
+Supported Features
+------------------
+
++-----------------------------------+---------+----------+
+| Feature                           | Release | Default  |
++===================================+=========+==========+
+| :ref:`feature_resharding`         | Reef    | Enabled  |
++-----------------------------------+---------+----------+
+| :ref:`feature_compress_encrypted` | Reef    | Disabled |
++-----------------------------------+---------+----------+
+
+.. _feature_resharding:
+
+resharding
+~~~~~~~~~~
+
+This feature allows buckets to be resharded in a multisite configuration
+without interrupting the replication of their objects. When
+``rgw_dynamic_resharding`` is enabled, it runs on each zone independently, and
+zones may choose different shard counts for the same bucket. When buckets are
+resharded manually with ``radosgw-admin bucket reshard``, only that zone's
+bucket is modified. A zone feature should only be marked as supported after all
+of its RGWs and OSDs have upgraded.
+
+.. note:: Dynamic resharding is not supported in multisite deployments prior to
+   the Reef release.
+
+
+.. _feature_compress_encrypted:
+
+compress-encrypted
+~~~~~~~~~~~~~~~~~~
+
+This feature enables support for combining `Server-Side Encryption`_ and
+`Compression`_ on the same object. Object data gets compressed before encryption.
+Prior to Reef, multisite would not replicate such objects correctly, so all zones
+must upgrade to Reef or later before enabling.
+
+.. warning:: The compression ratio may leak information about the encrypted data,
+   and allow attackers to distinguish whether two same-sized objects might contain
+   the same data. Due to these security considerations, this feature is disabled
+   by default.
+
+Commands
+--------
+
+Add support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+Remove support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Enable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Disable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+.. _`Server-Side Encryption`: ../encryption
+.. _`Compression`: ../compression

From 0a97eb89e1d9c99e96fd215e9dc31febc2e44de5 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Mar 2024 18:20:55 -0400
Subject: [PATCH 2261/2492] doc/radosgw: generalize description of Zone
 Features

so they make sense outside of multisite configurations

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/zone-features.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/radosgw/zone-features.rst b/doc/radosgw/zone-features.rst
index d397af9db003..b439fd9dfd26 100644
--- a/doc/radosgw/zone-features.rst
+++ b/doc/radosgw/zone-features.rst
@@ -2,9 +2,9 @@
 Zone Features
 =============
 
-Some multisite features require support from all zones before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
+Some features require support from all cooperating radosgws before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
 
-On creation of new zones and zonegroups, all known features are supported and some features (see table below) are enabled by default. After upgrading an existing multisite configuration, however, new features must be enabled manually.
+On creation of new zones and zonegroups, all known features are supported and some features (see table below) are enabled by default. After upgrading an existing zone, however, new features must be enabled manually.
 
 Supported Features
 ------------------
@@ -62,6 +62,9 @@ On the cluster that contains the given zone:
    radosgw-admin zone modify --rgw-zone={zone-name} --enable-feature={feature-name}
    radosgw-admin period update --commit
 
+.. note:: The ``period update`` command only works if the zone belongs to a realm.
+   Otherwise, all radosgws will need to restart before they notice the change.
+
 
 Remove support for a zone feature
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From a4e2a598aeeec6eeeae00d5c81443ab8e091a337 Mon Sep 17 00:00:00 2001
From: Chen Yuanrun <114801934+chenyuanrun@users.noreply.github.com>
Date: Mon, 29 Jan 2024 10:04:59 +0000
Subject: [PATCH 2262/2492] librados: use CEPH_OSD_FLAG_FULL_FORCE for
 IoCtxImpl::remove

librados::OPERATION_FULL_FORCE should be translated to
CEPH_OSD_FLAG_FULL_FORCE before calling IoCtxImpl::remove().

Fixes: https://tracker.ceph.com/issues/64558
Signed-off-by: Chen Yuanrun <chen-yuanrun@foxmail.com>
---
 src/librados/IoCtxImpl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index d66b56560f9c..947d8a4930aa 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -1235,7 +1235,7 @@ int librados::IoCtxImpl::remove(const object_t& oid)
   ::ObjectOperation op;
   prepare_assert_ops(&op);
   op.remove();
-  return operate(oid, &op, nullptr, librados::OPERATION_FULL_FORCE);
+  return operate(oid, &op, nullptr, CEPH_OSD_FLAG_FULL_FORCE);
 }
 
 int librados::IoCtxImpl::remove(const object_t& oid, int flags)

From 2a93a8e837a42559f8a81c6fd9274b24f4fdf7f6 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 14 Mar 2024 16:29:09 +1000
Subject: [PATCH 2263/2492] doc/glossary: add "librados" entry

Add a "librados" entry to the glossary.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/glossary.rst                 | 7 +++++++
 doc/rados/api/librados-intro.rst | 2 ++
 doc/rados/api/python.rst         | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/doc/glossary.rst b/doc/glossary.rst
index bca2346a2127..84f1ac810ad1 100644
--- a/doc/glossary.rst
+++ b/doc/glossary.rst
@@ -269,6 +269,13 @@
         Hybrid OSD  
                 Refers to an OSD that has both HDD and SSD drives.
 
+        librados
+                An API that can be used to create a custom interface to a Ceph
+                storage cluster. ``librados`` makes it possible to interact
+                with Ceph Monitors and with OSDs. See :ref:`Introduction to
+                librados <librados-intro>`. See :ref:`librados (Python)
+                <librados-python>`.
+
 	LVM tags
                 **L**\ogical **V**\olume **M**\anager tags. Extensible metadata
                 for LVM volumes and groups. They are used to store
diff --git a/doc/rados/api/librados-intro.rst b/doc/rados/api/librados-intro.rst
index 5174188b4c56..b863efc9e681 100644
--- a/doc/rados/api/librados-intro.rst
+++ b/doc/rados/api/librados-intro.rst
@@ -1,3 +1,5 @@
+.. _librados-intro:
+
 ==========================
  Introduction to librados
 ==========================
diff --git a/doc/rados/api/python.rst b/doc/rados/api/python.rst
index 346653a3d2e2..60bdfa4da061 100644
--- a/doc/rados/api/python.rst
+++ b/doc/rados/api/python.rst
@@ -1,3 +1,5 @@
+.. _librados-python:
+
 ===================
  Librados (Python)
 ===================

From 3061eb367001fd13f75980fac1084639ab7c0abb Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Thu, 14 Mar 2024 14:25:08 +0530
Subject: [PATCH 2264/2492] Allow DELETE method api request, X-TOTAL-COUNT
 header in CORS config in dashboard

Fixes: https://tracker.ceph.com/issues/64913

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/tools.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py
index 5619b5bb90d5..51ed9c471aac 100644
--- a/src/pybind/mgr/dashboard/tools.py
+++ b/src/pybind/mgr/dashboard/tools.py
@@ -873,7 +873,7 @@ def cors_tool():
     cross_origin_url_list = [url.strip() for url in cross_origin_urls.split(',')]
     if req_header_cross_origin_url in cross_origin_url_list:
         resp_head['Access-Control-Allow-Origin'] = req_header_cross_origin_url
-    resp_head['Access-Control-Expose-Headers'] = 'GET, POST'
+    resp_head['Access-Control-Expose-Headers'] = 'GET, POST, X-Total-Count'
     resp_head['Access-Control-Allow-Credentials'] = 'true'
 
     # Non-simple CORS preflight request; short-circuit the normal handler.
@@ -883,7 +883,7 @@ def cors_tool():
             resp_head['Access-Control-Allow-Origin'] = req_header_origin_url
         ac_method = req_head.get('Access-Control-Request-Method', None)
 
-        allowed_methods = ['GET', 'POST', 'PUT']
+        allowed_methods = ['GET', 'POST', 'PUT', 'DELETE']
         allowed_headers = [
             'Content-Type',
             'Authorization',

From a4313e77a31c5165254068b53dfac3d277b0a6a8 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Mon, 23 Oct 2023 14:57:33 -0400
Subject: [PATCH 2265/2492] rgwlc: implement NewerNoncurrentVersions

Per AWS doc, this value controls "how many noncurrent versions
Amazon S3 will retain." [1]  We understand this to mean, retain
NewerNoncurrentVersions of any object, regardless of expiration.

Removed unused RGWLifecycleConfiguration::has_same_action() (cleanup)

[1] https://docs.aws.amazon.com/AmazonS3/latest/API/API_NoncurrentVersionExpiration.html.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc           | 62 ++++++++++++++++---------------------
 src/rgw/rgw_lc.h            | 18 ++++++++---
 src/rgw/rgw_lc_s3.cc        |  4 +++
 src/test/rgw/test_rgw_lc.cc | 30 ++++++++++++++++++
 4 files changed, 74 insertions(+), 40 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index c4888c630c36..feee38d1f75c 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -122,7 +122,9 @@ bool RGWLifecycleConfiguration::_add_rule(const LCRule& rule)
     op.expiration_date = ceph::from_iso_8601(rule.get_expiration().get_date());
   }
   if (rule.get_noncur_expiration().has_days()) {
-    op.noncur_expiration = rule.get_noncur_expiration().get_days();
+    const auto& exp = rule.get_noncur_expiration();
+    op.noncur_expiration = exp.get_days();
+    op.newer_noncurrent = exp.get_newer();
   }
   if (rule.get_mp_expiration().has_days()) {
     op.mp_expiration = rule.get_mp_expiration().get_days();
@@ -182,33 +184,6 @@ int RGWLifecycleConfiguration::check_and_add_rule(const LCRule& rule)
   return 0;
 }
 
-bool RGWLifecycleConfiguration::has_same_action(const lc_op& first,
-						const lc_op& second) {
-  if ((first.expiration > 0 || first.expiration_date != boost::none) && 
-    (second.expiration > 0 || second.expiration_date != boost::none)) {
-    return true;
-  } else if (first.noncur_expiration > 0 && second.noncur_expiration > 0) {
-    return true;
-  } else if (first.mp_expiration > 0 && second.mp_expiration > 0) {
-    return true;
-  } else if (!first.transitions.empty() && !second.transitions.empty()) {
-    for (auto &elem : first.transitions) {
-      if (second.transitions.find(elem.first) != second.transitions.end()) {
-        return true;
-      }
-    }
-  } else if (!first.noncur_transitions.empty() &&
-	     !second.noncur_transitions.empty()) {
-    for (auto &elem : first.noncur_transitions) {
-      if (second.noncur_transitions.find(elem.first) !=
-	  second.noncur_transitions.end()) {
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
 /* Formerly, this method checked for duplicate rules using an invalid
  * method (prefix uniqueness). */
 bool RGWLifecycleConfiguration::valid() 
@@ -373,13 +348,14 @@ class LCObjsLister {
   string prefix;
   vector<rgw_bucket_dir_entry>::iterator obj_iter;
   rgw_bucket_dir_entry pre_obj;
+  uint64_t num_noncurrent{0};
   int64_t delay_ms;
 
 public:
   LCObjsLister(rgw::sal::Driver* _driver, rgw::sal::Bucket* _bucket) :
       driver(_driver), bucket(_bucket) {
     list_params.list_versions = bucket->versioned();
-    list_params.allow_unordered = true;
+    list_params.allow_unordered = true; // XXX can be unconditionally true, so long as all versions of one object are assured to be on one shard and always ordered on that shard (true today in RADOS)
     delay_ms = driver->ctx()->_conf.get_val<int64_t>("rgw_lc_thread_delay");
   }
 
@@ -426,6 +402,13 @@ class LCObjsLister {
       }
       delay();
     }
+
+    if (obj_iter->key.name == pre_obj.key.name) {
+      ++num_noncurrent;
+    } else {
+      num_noncurrent = 0; // XXX the first element must be current or a delete marker (?)
+    }
+
     /* returning address of entry in objs */
     *obj = &(*obj_iter);
     return obj_iter != list_results.objs.end();
@@ -451,6 +434,7 @@ class LCObjsLister {
     return ((obj_iter + 1)->key.name);
   }
 
+  uint64_t get_num_noncurrent() { return num_noncurrent; }
 }; /* LCObjsLister */
 
 struct op_env {
@@ -477,6 +461,7 @@ struct lc_op_ctx {
   op_env env;
   rgw_bucket_dir_entry o;
   boost::optional<std::string> next_key_name;
+  uint64_t num_noncurrent;
   ceph::real_time effective_mtime;
 
   rgw::sal::Driver* driver;
@@ -493,10 +478,11 @@ struct lc_op_ctx {
 
   lc_op_ctx(op_env& env, rgw_bucket_dir_entry& o,
 	    boost::optional<std::string> next_key_name,
+	    uint64_t num_noncurrent,
 	    ceph::real_time effective_mtime,
 	    const DoutPrefixProvider *dpp, WorkQ* wq)
     : cct(env.driver->ctx()), env(env), o(o), next_key_name(next_key_name),
-      effective_mtime(effective_mtime),
+      num_noncurrent(num_noncurrent), effective_mtime(effective_mtime),
       driver(env.driver), bucket(env.bucket), op(env.op), ol(env.ol),
       octx(env.driver), dpp(dpp), wq(wq)
     {
@@ -654,6 +640,7 @@ class LCOpRule {
 
   op_env env;
   boost::optional<std::string> next_key_name;
+  uint64_t num_noncurrent;
   ceph::real_time effective_mtime;
 
   std::vector<shared_ptr<LCOpFilter> > filters; // n.b., sharing ovhd
@@ -1198,10 +1185,12 @@ class LCOpAction_NonCurrentExpiration : public LCOpAction {
 				      exp_time);
 
     ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
-		      << is_expired << " "
-		      << oc.wq->thr_name() << dendl;
+		       << is_expired << " " << ": num_noncurrent="
+		       << oc.num_noncurrent
+		       << oc.wq->thr_name() << dendl;
 
     return is_expired &&
+      (oc.num_noncurrent > oc.op.newer_noncurrent) &&
       pass_object_lock_check(oc.driver, oc.obj.get(), dpp);
   }
 
@@ -1211,9 +1200,9 @@ class LCOpAction_NonCurrentExpiration : public LCOpAction {
 			       rgw::notify::ObjectExpirationNoncurrent);
     if (r < 0) {
       ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj (non-current expiration) " 
-		       << oc.bucket << ":" << o.key
-		       << " " << cpp_strerror(r)
-		       << " " << oc.wq->thr_name() << dendl;
+			   << oc.bucket << ":" << o.key
+			   << " " << cpp_strerror(r)
+			   << " " << oc.wq->thr_name() << dendl;
       return r;
     }
     if (perfcounter) {
@@ -1567,6 +1556,7 @@ void LCOpRule::build()
 void LCOpRule::update()
 {
   next_key_name = env.ol.next_key_name();
+  num_noncurrent = env.ol.get_num_noncurrent();
   effective_mtime = env.ol.get_prev_obj().meta.mtime;
 }
 
@@ -1574,7 +1564,7 @@ int LCOpRule::process(rgw_bucket_dir_entry& o,
 		      const DoutPrefixProvider *dpp,
 		      WorkQ* wq)
 {
-  lc_op_ctx ctx(env, o, next_key_name, effective_mtime, dpp, wq);
+  lc_op_ctx ctx(env, o, next_key_name, num_noncurrent, effective_mtime, dpp, wq);
   shared_ptr<LCOpAction> *selected = nullptr; // n.b., req'd by sharing
   real_time exp;
 
diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index bd8efd9b6d03..edfb661c56b4 100644
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -44,26 +44,31 @@ class LCExpiration
   std::string days;
   //At present only current object has expiration date
   std::string date;
+  std::string newer_noncurrent;
 public:
   LCExpiration() {}
   LCExpiration(const std::string& _days, const std::string& _date) : days(_days), date(_date) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(3, 2, bl);
+    ENCODE_START(4, 2, bl);
     encode(days, bl);
     encode(date, bl);
+    encode(newer_noncurrent, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(4, 2, 2, bl);
     decode(days, bl);
     if (struct_v >= 3) {
       decode(date, bl);
+      if (struct_v >= 4) {
+	decode(newer_noncurrent, bl);
+      }
     }
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
-//  static void generate_test_instances(list<ACLOwner*>& o);
+  //  static void generate_test_instances(list<ACLOwner*>& o);
   void set_days(const std::string& _days) { days = _days; }
   std::string get_days_str() const {
     return days;
@@ -72,6 +77,11 @@ class LCExpiration
   bool has_days() const {
     return !days.empty();
   }
+  void set_newer(const std::string& _newer) { newer_noncurrent = _newer; }
+  int get_newer() const {return atoi(newer_noncurrent.c_str()); }
+  bool has_newer() const {
+    return !newer_noncurrent.empty();
+  }
   void set_date(const std::string& _date) { date = _date; }
   std::string get_date() const {
     return date;
@@ -440,6 +450,7 @@ struct lc_op
   bool dm_expiration{false};
   int expiration{0};
   int noncur_expiration{0};
+  int newer_noncurrent{0};
   int mp_expiration{0};
   boost::optional<ceph::real_time> expiration_date;
   boost::optional<RGWObjTags> obj_tags;
@@ -463,7 +474,6 @@ class RGWLifecycleConfiguration
   std::multimap<std::string, lc_op> prefix_map;
   std::multimap<std::string, LCRule> rule_map;
   bool _add_rule(const LCRule& rule);
-  bool has_same_action(const lc_op& first, const lc_op& second);
 public:
   explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
   RGWLifecycleConfiguration() : cct(NULL) {}
diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index 1e1e16b7147f..e0b4337e5097 100644
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -64,11 +64,15 @@ void LCExpiration_S3::decode_xml(XMLObj *obj)
 
 void LCNoncurExpiration_S3::decode_xml(XMLObj *obj)
 {
+  RGWXMLDecoder::decode_xml("NewerNoncurrentVersions", newer_noncurrent, obj);
   RGWXMLDecoder::decode_xml("NoncurrentDays", days, obj, true);
 }
 
 void LCNoncurExpiration_S3::dump_xml(Formatter *f) const
 {
+  if(has_newer()) {
+    encode_xml("NewerNoncurrentVersions", newer_noncurrent, f);
+  }
   encode_xml("NoncurrentDays", days, f);
 }
 
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 057deccbe207..f7a8c75d95ae 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -117,6 +117,36 @@ TEST(ExpHdr, ReplaceStrftime)
   auto exp_str = fmt::format("{:%a, %d %b %Y %T %Z}", fmt::gmtime(exp));
   std::cout << "exp_str: " << exp_str << std::endl;
   ASSERT_EQ(exp_str, "Fri, 21 Dec 2012 09:13:07 GMT");
+
+}
+
+static const char *xmldoc_4 =
+R"(<Rule>
+        <ID>noncur-cleanup-rule</ID>
+        <Filter>
+           <Prefix></Prefix>
+        </Filter>
+        <Status>Enabled</Status>
+       <NoncurrentVersionExpiration>
+            <NewerNoncurrentVersions>5</NewerNoncurrentVersions>
+            <NoncurrentDays>365</NoncurrentDays>
+       </NoncurrentVersionExpiration>
+    </Rule>
+)";
+
+TEST(TestLCConfigurationDecoder, XMLDoc4)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  ASSERT_TRUE(parser.parse(xmldoc_4, strlen(xmldoc_4), 1));
+  LCRule_S3 rule;
+  auto result = RGWXMLDecoder::decode_xml("Rule", rule, &parser, true);
+  ASSERT_TRUE(result);
+  /* check results */
+  ASSERT_TRUE(rule.is_enabled());
+  const auto& noncur_expiration = rule.get_noncur_expiration();
+  ASSERT_EQ(noncur_expiration.get_days(), 365);
+  ASSERT_EQ(noncur_expiration.get_newer(), 5);
 }
 
 struct LCWorkTimeTests : ::testing::Test

From c9b19be15362c63526e92f4d968f942a40cab150 Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Tue, 24 Oct 2023 17:59:34 -0400
Subject: [PATCH 2266/2492] rgwlc: implement Filter size limit checks

i.e., this implements ObjectSizeGreaterThan and ObjectSizeLessThan
for supported expiration and transition actions.

Fixes: https://tracker.ceph.com/issues/63304

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_lc.cc           | 76 ++++++++++++++++++++++++++++++-------
 src/rgw/rgw_lc.h            | 50 +++++++++++++++++++++---
 src/rgw/rgw_lc_s3.cc        | 27 +++++++++----
 src/test/rgw/test_rgw_lc.cc | 35 +++++++++++++++++
 4 files changed, 163 insertions(+), 25 deletions(-)

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index feee38d1f75c..7169e09edbc3 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -155,10 +155,17 @@ bool RGWLifecycleConfiguration::_add_rule(const LCRule& rule)
   } else {
     prefix = rule.get_prefix();
   }
-  if (rule.get_filter().has_tags()){
-    op.obj_tags = rule.get_filter().get_tags();
+  const auto& filter = rule.get_filter();
+  if (filter.has_tags()){
+    op.obj_tags = filter.get_tags();
   }
-  op.rule_flags = rule.get_filter().get_flags();
+  if (filter.has_size_gt()) {
+    op.size_gt = filter.get_size_gt();
+  }
+  if (filter.has_size_lt()) {
+    op.size_lt = filter.get_size_lt();
+  }
+  op.rule_flags = filter.get_flags();
   prefix_map.emplace(std::move(prefix), std::move(op));
   return true;
 }
@@ -262,8 +269,9 @@ static inline std::ostream& operator<<(std::ostream &os, rgw::sal::Lifecycle::LC
   return os;
 }
 
-static bool obj_has_expired(const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime, int days,
-			    ceph::real_time *expire_time = nullptr)
+static bool obj_has_expired(
+  const DoutPrefixProvider *dpp, CephContext *cct, ceph::real_time mtime,
+  int days, ceph::real_time *expire_time = nullptr)
 {
   double timediff, cmp;
   utime_t base_time;
@@ -497,6 +505,37 @@ struct lc_op_ctx {
 }; /* lc_op_ctx */
 
 
+static bool pass_size_limit_checks(const DoutPrefixProvider *dpp, lc_op_ctx& oc) {
+
+  const auto& op = oc.op;
+  if (op.size_gt || op.size_lt) {
+    int ret{0};
+    auto& bucket = oc.bucket;
+    auto& o = oc.o;
+    std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(o.key);
+
+    RGWObjState *obj_state{nullptr};
+    ret = obj->get_obj_state(dpp, &obj_state, null_yield, true);
+    if (ret < 0) {
+      return false;
+    }
+
+    bool gt_p{true};
+    bool lt_p{true};
+
+    if (op.size_gt) {
+      gt_p = (obj_state->size > op.size_gt.get());
+    }
+    if (op.size_lt) {
+      lt_p = (obj_state->size < op.size_lt.get());
+    }
+
+    return gt_p && lt_p;
+  } /* require size check */
+
+  return true;
+}
+
 static std::string lc_id = "rgw lifecycle";
 static std::string lc_req_id = "0";
 
@@ -1122,10 +1161,14 @@ class LCOpAction_CurrentExpiration : public LCOpAction {
       is_expired = obj_has_expired(dpp, oc.cct, mtime, op.expiration, exp_time);
     }
 
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+
     ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
-		      << (int)is_expired << " "
-		      << oc.wq->thr_name() << dendl;
-    return is_expired;
+		       << (int)is_expired << " size_check_p: "
+		       << size_check_p << " "
+		       << oc.wq->thr_name() << dendl;
+
+    return is_expired && size_check_p;
   }
 
   int process(lc_op_ctx& oc) override {
@@ -1183,14 +1226,18 @@ class LCOpAction_NonCurrentExpiration : public LCOpAction {
     int expiration = oc.op.noncur_expiration;
     bool is_expired = obj_has_expired(dpp, oc.cct, oc.effective_mtime, expiration,
 				      exp_time);
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+    auto newer_noncurrent_p = (oc.num_noncurrent > oc.op.newer_noncurrent);
 
     ldpp_dout(dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
 		       << is_expired << " " << ": num_noncurrent="
-		       << oc.num_noncurrent
+		       << oc.num_noncurrent << " size_check_p: "
+		       << size_check_p << " newer_noncurrent_p: "
+		       << newer_noncurrent_p << " "
 		       << oc.wq->thr_name() << dendl;
 
     return is_expired &&
-      (oc.num_noncurrent > oc.op.newer_noncurrent) &&
+      (oc.num_noncurrent > oc.op.newer_noncurrent) && size_check_p &&
       pass_object_lock_check(oc.driver, oc.obj.get(), dpp);
   }
 
@@ -1299,15 +1346,18 @@ class LCOpAction_Transition : public LCOpAction {
       is_expired = obj_has_expired(dpp, oc.cct, mtime, transition.days, exp_time);
     }
 
+    auto size_check_p = pass_size_limit_checks(dpp, oc);
+
     ldpp_dout(oc.dpp, 20) << __func__ << "(): key=" << o.key << ": is_expired="
-		      << is_expired << " "
-		      << oc.wq->thr_name() << dendl;
+			  << is_expired << " " << " size_check_p: "
+			  << size_check_p << " "
+			  << oc.wq->thr_name() << dendl;
 
     need_to_process =
       (rgw_placement_rule::get_canonical_storage_class(o.meta.storage_class) !=
        transition.storage_class);
 
-    return is_expired;
+    return is_expired && size_check_p;
   }
 
   bool should_process() override {
diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index edfb661c56b4..11a09c36be6b 100644
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -206,6 +206,8 @@ class LCFilter
 
 protected:
   std::string prefix;
+  std::string size_gt;
+  std::string size_lt;
   RGWObjTags obj_tags;
   uint32_t flags;
 
@@ -227,13 +229,15 @@ class LCFilter
   }
 
   bool empty() const {
-    return !(has_prefix() || has_tags() || has_flags());
+    return !(has_prefix() || has_tags() || has_flags() ||
+	     has_size_rule());
   }
 
   // Determine if we need AND tag when creating xml
   bool has_multi_condition() const {
-    if (obj_tags.count() + int(has_prefix()) + int(has_flags()) > 1) // Prefix is a member of Filter
-      return true;
+    if (obj_tags.count() + int(has_prefix()) + int(has_flags()) + int(has_size_rule()) > 1) {
+	return true;
+    }
     return false;
   }
 
@@ -245,6 +249,34 @@ class LCFilter
     return !obj_tags.empty();
   }
 
+  bool has_size_gt() const {
+    return !(size_gt.empty());
+  }
+
+  bool has_size_lt() const {
+    return !(size_lt.empty());
+  }
+
+  bool has_size_rule() const {
+    return (has_size_gt() || has_size_lt());
+  }
+
+  uint64_t get_size_gt() const {
+    uint64_t sz{0};
+    try {
+      sz = uint64_t(std::stoull(size_gt));
+    } catch (...) {}
+    return sz;
+  }
+
+  uint64_t get_size_lt() const {
+    uint64_t sz{0};
+    try {
+      sz = uint64_t(std::stoull(size_lt));
+    } catch (...) {}
+    return sz;
+  }
+
   bool has_flags() const {
     return !(flags == uint32_t(LCFlagType::none));
   }
@@ -254,10 +286,12 @@ class LCFilter
   }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(3, 1, bl);
+    ENCODE_START(4, 1, bl);
     encode(prefix, bl);
     encode(obj_tags, bl);
     encode(flags, bl);
+    encode(size_gt, bl);
+    encode(size_lt, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
@@ -267,6 +301,10 @@ class LCFilter
       decode(obj_tags, bl);
       if (struct_v >= 3) {
 	decode(flags, bl);
+	if (struct_v >= 4) {
+	  decode(size_gt, bl);
+	  decode(size_lt, bl);
+	}
       }
     }
     DECODE_FINISH(bl);
@@ -450,8 +488,10 @@ struct lc_op
   bool dm_expiration{false};
   int expiration{0};
   int noncur_expiration{0};
-  int newer_noncurrent{0};
+  uint64_t newer_noncurrent{0};
   int mp_expiration{0};
+  boost::optional<uint64_t> size_gt;
+  boost::optional<uint64_t> size_lt;
   boost::optional<ceph::real_time> expiration_date;
   boost::optional<RGWObjTags> obj_tags;
   std::map<std::string, transition_action> transitions;
diff --git a/src/rgw/rgw_lc_s3.cc b/src/rgw/rgw_lc_s3.cc
index e0b4337e5097..77d563b8be4a 100644
--- a/src/rgw/rgw_lc_s3.cc
+++ b/src/rgw/rgw_lc_s3.cc
@@ -132,6 +132,12 @@ void LCFilter_S3::dump_xml(Formatter *f) const
       encode_xml("ArchiveZone", "", f);
     }
   }
+  if (has_size_gt()) {
+    encode_xml("ObjectSizeGreaterThanw", size_gt, f);
+  }
+  if (has_size_lt()) {
+    encode_xml("ObjectSizeLessThan", size_lt, f);
+  }
   if (multi) {
     f->close_section(); // And
   }
@@ -160,6 +166,13 @@ void LCFilter_S3::decode_xml(XMLObj *obj)
     flags |= make_flag(LCFlagType::ArchiveZone);
   }
 
+  RGWXMLDecoder::decode_xml("ObjectSizeGreaterThan", size_gt, o, false);
+  RGWXMLDecoder::decode_xml("ObjectSizeLessThan", size_lt, o, false);
+  if (has_size_gt() && has_size_lt() &&
+      (size_lt <= size_gt)) {
+    throw RGWXMLDecoder::err("Filter maximum object size must be larger than the minimum object size");
+  }
+
   obj_tags.clear(); // why is this needed?
   auto tags_iter = o->find("Tag");
   while (auto tag_xml = tags_iter.get_next()){
@@ -222,6 +235,13 @@ void LCRule_S3::decode_xml(XMLObj *obj)
 
   RGWXMLDecoder::decode_xml("ID", id, obj);
 
+  if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
+    throw RGWXMLDecoder::err("missing Status in Rule");
+  }
+  if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
+    throw RGWXMLDecoder::err("bad Status in Rule");
+  }
+
   LCFilter_S3 filter_s3;
   if (!RGWXMLDecoder::decode_xml("Filter", filter_s3, obj)) {
     // Ideally the following code should be deprecated and we should return
@@ -238,13 +258,6 @@ void LCRule_S3::decode_xml(XMLObj *obj)
   }
   filter = (LCFilter)filter_s3;
 
-  if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
-    throw RGWXMLDecoder::err("missing Status in Filter");
-  }
-  if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
-    throw RGWXMLDecoder::err("bad Status in Filter");
-  }
-
   LCExpiration_S3 s3_expiration;
   LCNoncurExpiration_S3 s3_noncur_expiration;
   LCMPExpiration_S3 s3_mp_expiration;
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index f7a8c75d95ae..7d24db622938 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -149,6 +149,41 @@ TEST(TestLCConfigurationDecoder, XMLDoc4)
   ASSERT_EQ(noncur_expiration.get_newer(), 5);
 }
 
+static const char *xmldoc_5 =
+R"(<Rule>
+        <ID>expire-gt</ID>
+        <Expiration>
+            <Days>365</Days>
+        </Expiration>
+        <Filter>
+           <And>
+           <Prefix></Prefix>
+           <ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan>
+           <ObjectSizeLessThan>65536</ObjectSizeLessThan>
+           </And>
+        </Filter>
+        <Status>Enabled</Status>
+    </Rule>
+)";
+
+TEST(TestLCConfigurationDecoder, XMLDoc5)
+{
+  RGWXMLDecoder::XMLParser parser;
+  ASSERT_TRUE(parser.init());
+  auto result1 = parser.parse(xmldoc_5, strlen(xmldoc_5), 1);
+  ASSERT_TRUE(result1);
+  LCRule_S3 rule;
+  auto result2 = RGWXMLDecoder::decode_xml("Rule", rule, &parser, true);
+  ASSERT_TRUE(result2);
+  /* check results */
+  ASSERT_TRUE(rule.is_enabled());
+  const auto& expiration = rule.get_expiration();
+  ASSERT_EQ(expiration.get_days(), 365);
+  const auto& filter = rule.get_filter();
+  ASSERT_EQ(filter.get_size_gt(), 1024);
+  ASSERT_EQ(filter.get_size_lt(), 65536);
+}
+
 struct LCWorkTimeTests : ::testing::Test
 {
    CephContext* cct;

From db724a8fef183f6f5200bc0e93428a19c69d1342 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Thu, 14 Mar 2024 16:14:25 +0000
Subject: [PATCH 2267/2492] mgr/dashboard: fixed nfs attach command

Fixes: https://tracker.ceph.com/issues/64933

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-mount-details/cephfs-mount-details.component.ts      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
index af0b93a96af6..76fd81aa9413 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-mount-details/cephfs-mount-details.component.ts
@@ -21,7 +21,7 @@ export class CephfsMountDetailsComponent implements OnInit, OnDestroy {
   ngOnInit(): void {
     this.mount = `sudo mount -t ceph <CLIENT_USER>@${this.mountData?.fsId}.${this.mountData?.fsName}=${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
     this.fuse = `sudo ceph-fuse  ${this.MOUNT_DIRECTORY} -r ${this.mountData?.rootPath} --client_mds_namespace=${this.mountData?.fsName}`;
-    this.nfs = `sudo mount -t nfs -o port=<PORT> <IP of active_mds daemon>:${this.mountData?.rootPath} ${this.MOUNT_DIRECTORY}`;
+    this.nfs = `sudo mount -t nfs -o port=<PORT> <IP of active_mds daemon>:<export_name> ${this.MOUNT_DIRECTORY}`;
   }
 
   ngOnDestroy(): void {

From 7060146d6a609f89354a54da87ea4a399b9a43c6 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 14 Mar 2024 14:59:23 -0400
Subject: [PATCH 2268/2492] doc/dev: update teuthology priority guidance

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 ...sts-integration-testing-teuthology-intro.rst | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst
index 3cbe51241cc9..c4d03aa7b776 100644
--- a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst
+++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-intro.rst
@@ -47,13 +47,22 @@ also reported on the `ceph-qa mailing list <https://ceph.com/irc/>`_.
 Testing Priority
 ----------------
 
-In brief: in the ``teuthology-suite`` command option ``-p <N>``, set the value of ``<N>`` to a number lower than 1000. An explanation of why follows.
+In brief: in the ``teuthology-suite`` command option ``-p <N>``, set the value
+of ``<N>`` to a number less than 500. An explanation of why follows.
 
 The ``teuthology-suite`` command includes an option ``-p <N>``. This option specifies the priority of the jobs submitted to the queue. The lower the value of ``N``, the higher the priority.
 
-The default value of ``N`` is ``1000``. This is the same priority value given to the nightly tests (the nightlies). Often, the volume of testing done during the nightly tests is so great that the full number of nightly tests do not get run during the time allotted for their run.
+The default value of ``N`` is ``1000``. Most nightly tests automatically
+scheduled by ``teuthology@teuthology.front.sepia.ceph.com`` are run with ``N >=
+500``. Some critical nightly tests are given higher priority, such as smoke
+tests or QA runs for an imminent major releases.
 
-Set the value of ``N`` lower than ``1000``, or your tests will not have priority over the nightly tests. This means that they might never run.
+.. note:: Often the volume of testing done during the nightly tests is so great
+          that the full number of nightly tests do not get run during the time allotted
+          for their run.
+
+Set the value of ``N`` lower than ``500`` or your tests will not have priority
+over the nightly tests. This means that they might never run.
 
 Select your job's priority (the value of ``N``) in accordance with the following guidelines:
 
@@ -79,7 +88,7 @@ Select your job's priority (the value of ``N``) in accordance with the following
    * - **150 <= N < 200**
      - Use this priority for 100 jobs or fewer that test a particular feature
        or fix.  Results are available in about 24 hours.
-   * - **200 <= N < 1000**
+   * - **200 <= N < 500**
      - Use this priority for large test runs.  Results are available in about a
        week.
 

From 416c079d45affb99cbf00fc9d3ff2c35379048f3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 14 Mar 2024 14:59:36 -0400
Subject: [PATCH 2269/2492] qa/crontab: use historically normal priorities for
 nightlies

Stop using --force-priority except when necessary.

Squid still gets elevated priority due to the increased attention with the
imminent release.

I've differentiated the priorities some in that release branches should get
higher priority than the main branch and that older release branches should be
prioritized over newer ones. Finally, upgrade tests should be prioritized over
other nightlies.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/crontab/teuthology-cronjobs | 74 +++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 36 deletions(-)

diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs
index be28b3900e63..a09f7d9cc19b 100644
--- a/qa/crontab/teuthology-cronjobs
+++ b/qa/crontab/teuthology-cronjobs
@@ -19,7 +19,7 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io"
 CW="/home/teuthology/ceph/qa/nightlies/cron_wrapper"
 SS="/home/teuthology/ceph/qa/nightlies/schedule_subset.sh"
 # default/common arguments added by schedule_subset.sh
-TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ceph.com/ceph.git --suite-repo=https://git.ceph.com/ceph.git --machine-type smithi --force-priority"
+TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ceph.com/ceph.git --suite-repo=https://git.ceph.com/ceph.git --machine-type smithi"
 
 ### !!!!!!!!!!!!!!!!!!!!!!!!!!
 ## THIS CRONTAB MUST NOT BE EDITED MANUALLY !!!!
@@ -49,10 +49,10 @@ TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ce
 
 
 ## ********** smoke tests on main and release branches
-00 05  * * 0,2,4   $CW $SS      1 --ceph main    --suite smoke -p 70
-08 05  * * 0       $CW $SS      1 --ceph squid   --suite smoke -p 70
-16 05  * * 0       $CW $SS      1 --ceph reef    --suite smoke -p 70
-24 05  * * 0       $CW $SS      1 --ceph quincy  --suite smoke -p 70
+00 05  * * 0,2,4   $CW $SS      1 --ceph main    --suite smoke -p 100 --force-priority
+08 05  * * 0       $CW $SS      1 --ceph squid   --suite smoke -p 100 --force-priority
+16 05  * * 0       $CW $SS      1 --ceph reef    --suite smoke -p 100 --force-priority
+24 05  * * 0       $CW $SS      1 --ceph quincy  --suite smoke -p 100 --force-priority
 
 ## ********** windows tests on main branch - weekly
 # 00 03 * * 1 CEPH_BRANCH=main; MACHINE_NAME=smithi; $CW teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s windows -k distro -e $CEPH_QA_EMAIL
@@ -63,7 +63,7 @@ TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ce
 
 
 ## ********** teuthology/nop on main branch - daily
-@daily             $CW $SS      1 --ceph main --suite teuthology/nop -p 50
+@daily             $CW $SS      1 --ceph main --suite teuthology/nop -p 1 --force-priority
 
 
 ## main branch runs - weekly
@@ -71,13 +71,13 @@ TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ce
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 20 * * 0        $CW $SS 100000 --ceph main --suite      rados -p 201
-08 20 * * 1        $CW $SS     64 --ceph main --suite       orch -p 200
-16 20 * * 2        $CW $SS    128 --ceph main --suite        rbd -p 200
-24 20 * * 3        $CW $SS    512 --ceph main --suite         fs -p 200
-32 20 * * 4        $CW $SS      4 --ceph main --suite powercycle -p 200
-40 20 * * 5        $CW $SS      1 --ceph main --suite        rgw -p 200
-48 20 * * 6        $CW $SS      4 --ceph main --suite       krbd -p 200 --kernel testing
+00 20 * * 0        $CW $SS 100000 --ceph main --suite      rados -p 951
+08 20 * * 1        $CW $SS     64 --ceph main --suite       orch -p 950
+16 20 * * 2        $CW $SS    128 --ceph main --suite        rbd -p 950
+24 20 * * 3        $CW $SS    512 --ceph main --suite         fs -p 950
+32 20 * * 4        $CW $SS      4 --ceph main --suite powercycle -p 950
+40 20 * * 5        $CW $SS      1 --ceph main --suite        rgw -p 950
+48 20 * * 6        $CW $SS      4 --ceph main --suite       krbd -p 950 --kernel testing
 
 
 ## squid branch runs - twice weekly
@@ -85,26 +85,27 @@ TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ce
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 21 * * 0,4      $CW $SS 100000 --ceph squid --suite      rados -p 101
-08 21 * * 1,5      $CW $SS     64 --ceph squid --suite       orch -p 100
-16 21 * * 2,6      $CW $SS    128 --ceph squid --suite        rbd -p 100
-24 21 * * 3,0      $CW $SS    512 --ceph squid --suite         fs -p 100
-32 21 * * 4,1      $CW $SS      4 --ceph squid --suite powercycle -p 100
-40 21 * * 5,2      $CW $SS      1 --ceph squid --suite        rgw -p 100
-48 21 * * 6,3      $CW $SS      4 --ceph squid --suite       krbd -p 100 --kernel testing
+#                                                                 -p 94-
+00 21 * * 0,4      $CW $SS 100000 --ceph squid --suite      rados -p 101 --force-priority
+08 21 * * 1,5      $CW $SS     64 --ceph squid --suite       orch -p 100 --force-priority
+16 21 * * 2,6      $CW $SS    128 --ceph squid --suite        rbd -p 100 --force-priority
+24 21 * * 3,0      $CW $SS    512 --ceph squid --suite         fs -p 100 --force-priority
+32 21 * * 4,1      $CW $SS      4 --ceph squid --suite powercycle -p 100 --force-priority
+40 21 * * 5,2      $CW $SS      1 --ceph squid --suite        rgw -p 100 --force-priority
+48 21 * * 6,3      $CW $SS      4 --ceph squid --suite       krbd -p 100 --force-priority --kernel testing
 
 ## reef branch runs - weekly
 ## suites rados and rbd use --subset arg and must be call with schedule_subset.sh
 ## see script in https://github.com/ceph/ceph/tree/main/qa/machine_types
 
 # rados is massive and difficult to bring down to less than 300 jobs, use one higher priority
-00 22 * * 0        $CW $SS 100000 --ceph reef --suite      rados -p 181
-08 22 * * 1        $CW $SS     64 --ceph reef --suite       orch -p 180
-16 22 * * 2        $CW $SS    128 --ceph reef --suite        rbd -p 180
-24 22 * * 3        $CW $SS    512 --ceph reef --suite         fs -p 180
-32 22 * * 4        $CW $SS      4 --ceph reef --suite powercycle -p 180
-40 22 * * 5        $CW $SS      1 --ceph reef --suite        rgw -p 180
-48 22 * * 6        $CW $SS      4 --ceph reef --suite       krbd -p 180 --kernel testing
+00 22 * * 0        $CW $SS 100000 --ceph reef --suite      rados -p 931
+08 22 * * 1        $CW $SS     64 --ceph reef --suite       orch -p 930
+16 22 * * 2        $CW $SS    128 --ceph reef --suite        rbd -p 930
+24 22 * * 3        $CW $SS    512 --ceph reef --suite         fs -p 930
+32 22 * * 4        $CW $SS      4 --ceph reef --suite powercycle -p 930
+40 22 * * 5        $CW $SS      1 --ceph reef --suite        rgw -p 930
+48 22 * * 6        $CW $SS      4 --ceph reef --suite       krbd -p 930 --kernel testing
 
 
 ###  The suite below must run on bare-metal because it's performance suite and run 3 times to produce more data points
@@ -119,24 +120,25 @@ TEUTHOLOGY_SUITE_ARGS="--non-interactive --newest=100 --ceph-repo=https://git.ce
 
 ## !!!! the client suites below MUST use --suite-branch octopus, pacific (see https://tracker.ceph.com/issues/24021)
 
-08 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-octopus-quincy --suite-branch octopus -p 175
-16 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-pacific-quincy --suite-branch pacific -p 175
-24 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:octopus-x -p 175
-32 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:pacific-x -p 175
-40 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade/quincy-p2p -p 175
+08 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-octopus-quincy --suite-branch octopus -p 820
+16 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade-clients/client-upgrade-pacific-quincy --suite-branch pacific -p 820
+24 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:octopus-x -p 820
+32 00 * * 1        $CW $SS 120000 --ceph quincy --suite upgrade:pacific-x -p 820
+40 00 * * 1        $CW $SS      1 --ceph quincy --suite upgrade/quincy-p2p -p 820
 
 ### upgrade runs for reef release
 ###### on smithi
 
-08 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:pacific-x -p 170
-16 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:quincy-x  -p 170
+08 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:pacific-x -p 830
+16 01 * * 3        $CW $SS      1 --ceph reef --suite upgrade:quincy-x  -p 830
 
 ### upgrade runs for squid release
 ###### on smithi
 
-08 02 * * 5        $CW $SS     32 --ceph squid --suite upgrade -p 165
+#                                                              -p 840
+08 02 * * 5        $CW $SS     32 --ceph squid --suite upgrade -p 100 --force-priority
 
 ### upgrade runs for main
 ###### on smithi
 
-08 03 * * 6        $CW $SS     32 --ceph main --suite upgrade -p 160
+08 03 * * 6        $CW $SS     32 --ceph main --suite upgrade -p 850

From d34c1879c8886ec7f19c7a93490c4736ae9a6d20 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 14 Mar 2024 18:37:23 +1000
Subject: [PATCH 2270/2492] doc/rbd: add clone mapping command

Add a command that explains how to map a formatted clone when the parent
image and the formatted clone have different encryption types.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rbd/rbd-encryption.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rbd/rbd-encryption.rst b/doc/rbd/rbd-encryption.rst
index a13839efe077..e9c788ecb283 100644
--- a/doc/rbd/rbd-encryption.rst
+++ b/doc/rbd/rbd-encryption.rst
@@ -250,7 +250,7 @@ Here is an example of a command that maps a formatted clone:
 
 .. prompt:: bash #
 
-   rbd device map -t nbd -o encryption-passphrase-file=clone-passphrase.bin -o encryption-passphrase-file=passphrase.bin mypool/myclone
+   rbd device map -t nbd -o encryption-passphrase-file=clone-passphrase.bin,encryption-passphrase-file=passphrase.bin mypool/myclone
 
 .. _journal feature: ../rbd-mirroring/#enable-image-journaling-feature
 .. _Supported Formats: #supported-formats

From 0c37788f18cd51a712a2481f8f58a7ee00d359ae Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Tue, 27 Feb 2024 15:59:15 -0500
Subject: [PATCH 2271/2492] rgw/notification: Fix the filter_rules to be array
 vs dict in json output.

FilterRules when processed as dict in json, emits samy key name for prefix, suffix causing failure while parsing the json notification output.
So change the type FilterRules from JsonDict to Array while dumping in json.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 src/rgw/rgw_pubsub.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc
index 7031c2363f0b..79d9f989cc4f 100644
--- a/src/rgw/rgw_pubsub.cc
+++ b/src/rgw/rgw_pubsub.cc
@@ -23,20 +23,24 @@ void set_event_id(std::string& id, const std::string& hash, const utime_t& ts) {
 }
 
 void rgw_s3_key_filter::dump(Formatter *f) const {
+  if (!has_content()) {
+    return;
+  }
+  f->open_array_section("FilterRules");
   if (!prefix_rule.empty()) {
-    f->open_object_section("FilterRule");
+    f->open_object_section("");
     ::encode_json("Name", "prefix", f);
     ::encode_json("Value", prefix_rule, f);
     f->close_section();
   }
   if (!suffix_rule.empty()) {
-    f->open_object_section("FilterRule");
+    f->open_object_section("");
     ::encode_json("Name", "suffix", f);
     ::encode_json("Value", suffix_rule, f);
     f->close_section();
   }
   if (!regex_rule.empty()) {
-    f->open_object_section("FilterRule");
+    f->open_object_section("");
     ::encode_json("Name", "regex", f);
     ::encode_json("Value", regex_rule, f);
     f->close_section();
@@ -97,8 +101,12 @@ bool rgw_s3_key_filter::has_content() const {
 }
 
 void rgw_s3_key_value_filter::dump(Formatter *f) const {
+  if (!has_content()) {
+    return;
+  }
+  f->open_array_section("FilterRules");
   for (const auto& key_value : kv) {
-    f->open_object_section("FilterRule");
+    f->open_object_section("");
     ::encode_json("Name", key_value.first, f);
     ::encode_json("Value", key_value.second, f);
     f->close_section();

From ea6611327c9aa678c524e0d71e954bd735d60789 Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Tue, 27 Feb 2024 14:06:47 -0500
Subject: [PATCH 2272/2492] rgw/notification: Make the Lifecycle events AWS
 compatible

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 doc/radosgw/s3-notification-compatibility.rst |   8 ++
 src/rgw/driver/daos/rgw_sal_daos.cc           |  16 ++-
 src/rgw/driver/daos/rgw_sal_daos.h            |  18 ++-
 src/rgw/driver/motr/rgw_sal_motr.cc           |  19 ++-
 src/rgw/driver/motr/rgw_sal_motr.h            |  23 ++-
 src/rgw/driver/posix/rgw_sal_posix.cc         |  20 +--
 src/rgw/driver/posix/rgw_sal_posix.h          |  18 +--
 src/rgw/driver/rados/rgw_cr_rados.cc          |  44 +++---
 src/rgw/driver/rados/rgw_notify.cc            | 132 +++++++++---------
 src/rgw/driver/rados/rgw_notify.h             |  18 ++-
 src/rgw/driver/rados/rgw_sal_rados.cc         |  20 ++-
 src/rgw/driver/rados/rgw_sal_rados.h          |  54 +++++--
 src/rgw/rgw_lc.cc                             |  67 +++++----
 src/rgw/rgw_notify_event_type.cc              |  16 +++
 src/rgw/rgw_notify_event_type.h               |   8 +-
 src/rgw/rgw_sal.h                             |  12 +-
 src/rgw/rgw_sal_dbstore.cc                    |  20 +--
 src/rgw/rgw_sal_dbstore.h                     |  28 ++--
 src/rgw/rgw_sal_filter.cc                     |  27 ++--
 src/rgw/rgw_sal_filter.h                      |  14 +-
 src/rgw/rgw_sal_store.h                       |  13 +-
 src/test/rgw/bucket_notification/test_bn.py   |  12 +-
 22 files changed, 382 insertions(+), 225 deletions(-)

diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
index cced60924d09..b29989977b23 100644
--- a/doc/radosgw/s3-notification-compatibility.rst
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -97,6 +97,14 @@ Event Types
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectLifecycle:Transition:NonCurrent``           | Ceph extension                          |
 +--------------------------------------------------------+-----------------------------------------+
+| ``s3:LifecycleExpiration:*``                           | Supported. Equivalent to: s3:LifecycleExpiration:Delete, s3:LifecycleExpiration:DeleteMarkerCreated|
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:LifecycleExpiration:Delete``                      | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:Current.|
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:LifecycleExpiration:DeleteMarkerCreated``         | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:DeleteMarker.|
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:LifecycleTransition``                             | Supported. Equivalent to: s3:ObjectLifecycle:Transition:Current|
++--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectSynced:*``                                  | Ceph extension                          |
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectSynced:Create``                             | Ceph Extension                          |
diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index 43ae35be4265..a6136d905998 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -2271,16 +2271,22 @@ std::unique_ptr<Lifecycle> DaosStore::get_lifecycle(void) {
 std::unique_ptr<Notification> DaosStore::get_notification(
     rgw::sal::Object* obj, rgw::sal::Object* src_obj, struct req_state* s,
     rgw::notify::EventType event_type, const std::string* object_name) {
-  return std::make_unique<DaosNotification>(obj, src_obj, event_type);
+  rgw::notify::EventTypeList event_types = {event_type};
+  return std::make_unique<DaosNotification>(obj, src_obj, event_types);
 }
 
 std::unique_ptr<Notification> DaosStore::get_notification(
-    const DoutPrefixProvider* dpp, Object* obj, Object* src_obj,
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-    std::string& _user_id, std::string& _user_tenant, std::string& _req_id,
+    const DoutPrefixProvider* dpp,
+    Object* obj,
+    Object* src_obj,
+    const rgw::notify::EventTypeList& event_types,
+    rgw::sal::Bucket* _bucket,
+    std::string& _user_id,
+    std::string& _user_tenant,
+    std::string& _req_id,
     optional_yield y) {
   ldpp_dout(dpp, 20) << "get_notification" << dendl;
-  return std::make_unique<DaosNotification>(obj, src_obj, event_type);
+  return std::make_unique<DaosNotification>(obj, src_obj, event_types);
 }
 
 int DaosStore::log_usage(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index c5cfefc222d1..2d74f9c17cc7 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -135,8 +135,10 @@ WRITE_CLASS_ENCODER(DaosUserInfo);
 
 class DaosNotification : public StoreNotification {
  public:
-  DaosNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type)
-      : StoreNotification(_obj, _src_obj, _type) {}
+  DaosNotification(Object* _obj,
+                   Object* _src_obj,
+                   const rgw::notify::EventTypeList& _types)
+      : StoreNotification(_obj, _src_obj, _types) {}
   ~DaosNotification() = default;
 
   virtual int publish_reserve(const DoutPrefixProvider* dpp,
@@ -911,10 +913,14 @@ class DaosStore : public StoreDriver {
       rgw::notify::EventType event_type, optional_yield y,
       const std::string* object_name = nullptr) override;
   virtual std::unique_ptr<Notification> get_notification(
-      const DoutPrefixProvider* dpp, rgw::sal::Object* obj,
-      rgw::sal::Object* src_obj, rgw::notify::EventType event_type,
-      rgw::sal::Bucket* _bucket, std::string& _user_id,
-      std::string& _user_tenant, std::string& _req_id,
+      const DoutPrefixProvider* dpp,
+      rgw::sal::Object* obj,
+      rgw::sal::Object* src_obj,
+      const rgw::notify::EventTypeList& event_types,
+      rgw::sal::Bucket* _bucket,
+      std::string& _user_id,
+      std::string& _user_tenant,
+      std::string& _req_id,
       optional_yield y) override;
   virtual RGWLC* get_rgwlc(void) override { return NULL; }
   virtual RGWCoroutinesManagerRegistry* get_cr_registry() override {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index 55a396436167..3ee60c9c4d5d 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -3289,14 +3289,21 @@ std::unique_ptr<Lifecycle> MotrStore::get_lifecycle(void)
 std::unique_ptr<Notification> MotrStore::get_notification(Object* obj, Object* src_obj, req_state* s,
     rgw::notify::EventType event_type, optional_yield y, const string* object_name)
 {
-  return std::make_unique<MotrNotification>(obj, src_obj, event_type);
+  const rgw::notify::EventTypeList event_types = {event_type};
+  return std::make_unique<MotrNotification>(obj, src_obj, event_types);
 }
 
-std::unique_ptr<Notification>  MotrStore::get_notification(const DoutPrefixProvider* dpp, Object* obj,
-        Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-        std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y)
-{
-  return std::make_unique<MotrNotification>(obj, src_obj, event_type);
+std::unique_ptr<Notification> MotrStore::get_notification(
+    const DoutPrefixProvider* dpp,
+    Object* obj,
+    Object* src_obj,
+    const rgw::notify::EventTypeList& event_types,
+    rgw::sal::Bucket* _bucket,
+    std::string& _user_id,
+    std::string& _user_tenant,
+    std::string& _req_id,
+    optional_yield y) {
+  return std::make_unique<MotrNotification>(obj, src_obj, event_types);
 }
 
 int MotrStore::log_usage(const DoutPrefixProvider *dpp, map<rgw_user_bucket, RGWUsageBatch>& usage_info)
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index 3cc3b37fa9ad..63c2b9d9dc80 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -191,11 +191,13 @@ WRITE_CLASS_ENCODER(MotrAccessKey);
 
 class MotrNotification : public StoreNotification {
   public:
-    MotrNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) :
-        StoreNotification(_obj, _src_obj, _type) {}
-    ~MotrNotification() = default;
+  MotrNotification(Object* _obj,
+                   Object* _src_obj,
+                   const rgw::notify::EventTypeList& _types)
+      : StoreNotification(_obj, _src_obj, _types) {}
+  ~MotrNotification() = default;
 
-    virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;}
+  virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;}
     virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size,
 			       const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; }
 };
@@ -1006,9 +1008,16 @@ class MotrStore : public StoreDriver {
     virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
     virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj,
         req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override;
-    virtual std::unique_ptr<Notification> get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj,
-        rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-        std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) override;
+    virtual std::unique_ptr<Notification> get_notification(
+        const DoutPrefixProvider* dpp,
+        rgw::sal::Object* obj,
+        rgw::sal::Object* src_obj,
+        const rgw::notify::EventTypeList& event_types,
+        rgw::sal::Bucket* _bucket,
+        std::string& _user_id,
+        std::string& _user_tenant,
+        std::string& _req_id,
+        optional_yield y) override;
     virtual RGWLC* get_rgwlc(void) override { return NULL; }
     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; }
 
diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc
index 8ca6a09275bd..a8b3fe2f3e34 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.cc
+++ b/src/rgw/driver/posix/rgw_sal_posix.cc
@@ -417,14 +417,18 @@ std::unique_ptr<Notification> POSIXDriver::get_notification(rgw::sal::Object* ob
   return next->get_notification(obj, src_obj, s, event_type, y, object_name);
 }
 
-std::unique_ptr<Notification> POSIXDriver::get_notification(const DoutPrefixProvider* dpp,
-                              rgw::sal::Object* obj, rgw::sal::Object* src_obj,
-                              rgw::notify::EventType event_type,
-                              rgw::sal::Bucket* _bucket,
-                              std::string& _user_id, std::string& _user_tenant,
-                              std::string& _req_id, optional_yield y)
-{
-  return next->get_notification(dpp, obj, src_obj, event_type, _bucket, _user_id, _user_tenant, _req_id, y);
+std::unique_ptr<Notification> POSIXDriver::get_notification(
+    const DoutPrefixProvider* dpp,
+    rgw::sal::Object* obj,
+    rgw::sal::Object* src_obj,
+    const rgw::notify::EventTypeList& event_types,
+    rgw::sal::Bucket* _bucket,
+    std::string& _user_id,
+    std::string& _user_tenant,
+    std::string& _req_id,
+    optional_yield y) {
+  return next->get_notification(dpp, obj, src_obj, event_types, _bucket,
+                                _user_id, _user_tenant, _req_id, y);
 }
 
 int POSIXDriver::close()
diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h
index 61ca13dc76f5..ed7630a7d65a 100644
--- a/src/rgw/driver/posix/rgw_sal_posix.h
+++ b/src/rgw/driver/posix/rgw_sal_posix.h
@@ -82,15 +82,15 @@ class POSIXDriver : public FilterDriver {
 				 const std::string* object_name=nullptr) override;
 
   virtual std::unique_ptr<Notification> get_notification(
-                                  const DoutPrefixProvider* dpp,
-                                  rgw::sal::Object* obj,
-                                  rgw::sal::Object* src_obj,
-                                  rgw::notify::EventType event_type,
-                                  rgw::sal::Bucket* _bucket,
-                                  std::string& _user_id,
-                                  std::string& _user_tenant,
-                                  std::string& _req_id,
-                                  optional_yield y) override;
+      const DoutPrefixProvider* dpp,
+      rgw::sal::Object* obj,
+      rgw::sal::Object* src_obj,
+      const rgw::notify::EventTypeList& event_type,
+      rgw::sal::Bucket* _bucket,
+      std::string& _user_id,
+      std::string& _user_tenant,
+      std::string& _req_id,
+      optional_yield y) override;
 
   /* Internal APIs */
   int get_root_fd() { return root_fd; }
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index fb5722528c39..a6d4719e7186 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -787,40 +787,52 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
         // send notification that object was successfully synced
         std::string user_id = "rgw sync";
         std::string req_id = "0";
-        		
+
         RGWObjTags obj_tags;
         auto iter = attrs.find(RGW_ATTR_TAGS);
         if (iter != attrs.end()) {
           try {
             auto it = iter->second.cbegin();
             obj_tags.decode(it);
-          } catch (buffer::error &err) {
-            ldpp_dout(dpp, 1) << "ERROR: " << __func__ << ": caught buffer::error couldn't decode TagSet " << dendl;
+          } catch (buffer::error& err) {
+            ldpp_dout(dpp, 1)
+                << "ERROR: " << __func__
+                << ": caught buffer::error couldn't decode TagSet " << dendl;
           }
         }
 
-        // NOTE: we create a mutable copy of bucket.get_tenant as the get_notification function expects a std::string&, not const
+        // NOTE: we create a mutable copy of bucket.get_tenant as the
+        // get_notification function expects a std::string&, not const
         std::string tenant(dest_bucket.get_tenant());
 
-        std::unique_ptr<rgw::sal::Notification> notify 
-                 = store->get_notification(dpp, &dest_obj, nullptr, rgw::notify::ObjectSyncedCreate,
-                  &dest_bucket, user_id,
-                  tenant,
-                  req_id, null_yield);
-
-        auto notify_res = static_cast<rgw::sal::RadosNotification*>(notify.get())->get_reservation();
-        int ret = rgw::notify::publish_reserve(dpp, *store->svc()->site, rgw::notify::ObjectSyncedCreate, notify_res, &obj_tags);
+        std::unique_ptr<rgw::sal::Notification> notify =
+            store->get_notification(
+                dpp, &dest_obj, nullptr, {rgw::notify::ObjectSyncedCreate},
+                &dest_bucket, user_id, tenant, req_id, null_yield);
+
+        auto notify_res =
+            static_cast<rgw::sal::RadosNotification*>(notify.get())
+                ->get_reservation();
+        int ret = rgw::notify::publish_reserve(
+            dpp, *store->svc()->site, {rgw::notify::ObjectSyncedCreate},
+            notify_res, &obj_tags);
         if (ret < 0) {
-          ldpp_dout(dpp, 1) << "ERROR: reserving notification failed, with error: " << ret << dendl;
+          ldpp_dout(dpp, 1)
+              << "ERROR: reserving notification failed, with error: " << ret
+              << dendl;
           // no need to return, the sync already happened
         } else {
-          ret = rgw::notify::publish_commit(&dest_obj, *bytes_transferred, ceph::real_clock::now(), etag, dest_obj.get_instance(), rgw::notify::ObjectSyncedCreate, notify_res, dpp);
+          ret = rgw::notify::publish_commit(
+              &dest_obj, *bytes_transferred, ceph::real_clock::now(), etag,
+              dest_obj.get_instance(), notify_res, dpp);
           if (ret < 0) {
-            ldpp_dout(dpp, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl;
+            ldpp_dout(dpp, 1)
+                << "ERROR: publishing notification failed, with error: " << ret
+                << dendl;
           }
         }
       }
-      
+
       if (counters) {
         if (bytes_transferred) {
           counters->inc(sync_counters::l_fetch, *bytes_transferred);
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 3fe441eec14e..b2e91e3a77af 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -981,12 +981,11 @@ static inline bool notification_match(reservation_t& res,
   return true;
 }
 
-  int publish_reserve(const DoutPrefixProvider* dpp,
-		      const SiteConfig& site,
-		      EventType event_type,
-		      reservation_t& res,
-		      const RGWObjTags* req_tags)
-{
+int publish_reserve(const DoutPrefixProvider* dpp,
+                    const SiteConfig& site,
+                    const EventTypeList& event_types,
+                    reservation_t& res,
+                    const RGWObjTags* req_tags) {
   rgw_pubsub_bucket_topics bucket_topics;
   if (all_zonegroups_support(site, zone_features::notification_v2) &&
       res.store->stat_topics_v1(res.user_tenant, res.yield, res.dpp) == -ENOENT) {
@@ -1020,64 +1019,71 @@ static inline bool notification_match(reservation_t& res,
   for (const auto& bucket_topic : bucket_topics.topics) {
     const rgw_pubsub_topic_filter& topic_filter = bucket_topic.second;
     const rgw_pubsub_topic& topic_cfg = topic_filter.topic;
-    if (!notification_match(res, topic_filter, event_type, req_tags)) {
-      // notification does not apply to req_state
-      continue;
-    }
-    ldpp_dout(res.dpp, 20) << "INFO: notification: '" << topic_filter.s3_id <<
-        "' on topic: '" << topic_cfg.dest.arn_topic << 
-        "' and bucket: '" << res.bucket->get_name() <<
-        "' (unique topic: '" << topic_cfg.name <<
-        "') apply to event of type: '" << to_string(event_type) << "'" << dendl;
-
-    cls_2pc_reservation::id_t res_id = cls_2pc_reservation::NO_ID;
-    if (topic_cfg.dest.persistent) {
-      // TODO: take default reservation size from conf
-      constexpr auto DEFAULT_RESERVATION = 4*1024U; // 4K
-      res.size = DEFAULT_RESERVATION;
-      librados::ObjectWriteOperation op;
-      bufferlist obl;
-      int rval;
-      const auto& queue_name = topic_cfg.dest.arn_topic;
-      cls_2pc_queue_reserve(op, res.size, 1, &obl, &rval);
-      auto ret = rgw_rados_operate(
-	res.dpp, res.store->getRados()->get_notif_pool_ctx(),
-	queue_name, &op, res.yield, librados::OPERATION_RETURNVEC);
-      if (ret < 0) {
-        ldpp_dout(res.dpp, 1) <<
-	  "ERROR: failed to reserve notification on queue: "
-			      << queue_name << ". error: " << ret << dendl;
-        // if no space is left in queue we ask client to slow down
-        return (ret == -ENOSPC) ? -ERR_RATE_LIMITED : ret;
+    for (auto& event_type : event_types) {
+      if (!notification_match(res, topic_filter, event_type, req_tags)) {
+        // notification does not apply to req_state
+        continue;
       }
-      ret = cls_2pc_queue_reserve_result(obl, res_id);
-      if (ret < 0) {
-        ldpp_dout(res.dpp, 1) << "ERROR: failed to parse reservation id. error: " << ret << dendl;
-        return ret;
+      ldpp_dout(res.dpp, 20)
+          << "INFO: notification: '" << topic_filter.s3_id << "' on topic: '"
+          << topic_cfg.dest.arn_topic << "' and bucket: '"
+          << res.bucket->get_name() << "' (unique topic: '" << topic_cfg.name
+          << "') apply to event of type: '" << to_string(event_type) << "'"
+          << dendl;
+
+      cls_2pc_reservation::id_t res_id = cls_2pc_reservation::NO_ID;
+      if (topic_cfg.dest.persistent) {
+        // TODO: take default reservation size from conf
+        constexpr auto DEFAULT_RESERVATION = 4 * 1024U;  // 4K
+        res.size = DEFAULT_RESERVATION;
+        librados::ObjectWriteOperation op;
+        bufferlist obl;
+        int rval;
+        const auto& queue_name = topic_cfg.dest.arn_topic;
+        cls_2pc_queue_reserve(op, res.size, 1, &obl, &rval);
+        auto ret = rgw_rados_operate(
+            res.dpp, res.store->getRados()->get_notif_pool_ctx(), queue_name,
+            &op, res.yield, librados::OPERATION_RETURNVEC);
+        if (ret < 0) {
+          ldpp_dout(res.dpp, 1)
+              << "ERROR: failed to reserve notification on queue: "
+              << queue_name << ". error: " << ret << dendl;
+          // if no space is left in queue we ask client to slow down
+          return (ret == -ENOSPC) ? -ERR_RATE_LIMITED : ret;
+        }
+        ret = cls_2pc_queue_reserve_result(obl, res_id);
+        if (ret < 0) {
+          ldpp_dout(res.dpp, 1)
+              << "ERROR: failed to parse reservation id. error: " << ret
+              << dendl;
+          return ret;
+        }
       }
-    }
-    // load the topic,if there is change in topic config while it's stored in
-    // notification.
-    rgw_pubsub_topic result;
-    const RGWPubSub ps(res.store, res.user_tenant, site);
-    auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr);
-    if (ret < 0) {
-      ldpp_dout(res.dpp, 1)
-          << "INFO: failed to load topic: " << topic_cfg.name
-          << ". error: " << ret
-          << " while reserving persistent notification event" << dendl;
-      if (ret == -ENOENT) {
-        // either the topic is deleted but the corresponding notification still
-        // exist or in v2 mode the notification could have synced first but
-        // topic is not synced yet.
-        return 0;
+      // load the topic,if there is change in topic config while it's stored in
+      // notification.
+      rgw_pubsub_topic result;
+      const RGWPubSub ps(res.store, res.user_tenant, site);
+      auto ret =
+          ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr);
+      if (ret < 0) {
+        ldpp_dout(res.dpp, 1)
+            << "INFO: failed to load topic: " << topic_cfg.name
+            << ". error: " << ret
+            << " while reserving persistent notification event" << dendl;
+        if (ret == -ENOENT) {
+          // either the topic is deleted but the corresponding notification
+          // still exist or in v2 mode the notification could have synced first
+          // but topic is not synced yet.
+          return 0;
+        }
+        ldpp_dout(res.dpp, 1)
+            << "WARN: Using the stored topic from bucket notification struct."
+            << dendl;
+        res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id,
+                                event_type);
+      } else {
+        res.topics.emplace_back(topic_filter.s3_id, result, res_id, event_type);
       }
-      ldpp_dout(res.dpp, 1)
-          << "WARN: Using the stored topic from bucket notification struct."
-          << dendl;
-      res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id);
-    } else {
-      res.topics.emplace_back(topic_filter.s3_id, result, res_id);
     }
   }
   return 0;
@@ -1088,7 +1094,6 @@ int publish_commit(rgw::sal::Object* obj,
 		   const ceph::real_time& mtime,
 		   const std::string& etag,
 		   const std::string& version,
-		   EventType event_type,
 		   reservation_t& res,
 		   const DoutPrefixProvider* dpp)
 {
@@ -1099,7 +1104,8 @@ int publish_commit(rgw::sal::Object* obj,
       continue;
     }
     event_entry_t event_entry;
-    populate_event(res, obj, size, mtime, etag, version, event_type, event_entry.event);
+    populate_event(res, obj, size, mtime, etag, version, topic.event_type,
+                   event_entry.event);
     event_entry.event.configurationId = topic.configurationId;
     event_entry.event.opaque_data = topic.cfg.opaque_data;
     if (topic.cfg.dest.persistent) { 
diff --git a/src/rgw/driver/rados/rgw_notify.h b/src/rgw/driver/rados/rgw_notify.h
index 4f2f57341ecb..ec8117c2f573 100644
--- a/src/rgw/driver/rados/rgw_notify.h
+++ b/src/rgw/driver/rados/rgw_notify.h
@@ -49,13 +49,18 @@ int remove_persistent_topic(const DoutPrefixProvider* dpp, librados::IoCtx& rado
 struct reservation_t {
   struct topic_t {
     topic_t(const std::string& _configurationId, const rgw_pubsub_topic& _cfg,
-	    cls_2pc_reservation::id_t _res_id) :
-      configurationId(_configurationId), cfg(_cfg), res_id(_res_id) {}
+            cls_2pc_reservation::id_t _res_id,
+            rgw::notify::EventType _event_type)
+        : configurationId(_configurationId),
+          cfg(_cfg),
+          res_id(_res_id),
+          event_type(_event_type) {}
 
     const std::string configurationId;
     const rgw_pubsub_topic cfg;
     // res_id is reset after topic is committed/aborted
     cls_2pc_reservation::id_t res_id;
+    rgw::notify::EventType event_type;
   };
 
   const DoutPrefixProvider* const dpp;
@@ -112,10 +117,10 @@ struct rgw_topic_stats {
 
 // create a reservation on the 2-phase-commit queue
 int publish_reserve(const DoutPrefixProvider *dpp,
-		      const SiteConfig& site,
-		      EventType event_type,
-		      reservation_t& reservation,
-		      const RGWObjTags* req_tags);
+                    const SiteConfig& site,
+                    const EventTypeList& event_types,
+                    reservation_t& reservation,
+                    const RGWObjTags* req_tags);
 
 // commit the reservation to the queue
 int publish_commit(rgw::sal::Object* obj,
@@ -123,7 +128,6 @@ int publish_commit(rgw::sal::Object* obj,
         const ceph::real_time& mtime, 
         const std::string& etag, 
         const std::string& version,
-        EventType event_type,
         reservation_t& reservation,
         const DoutPrefixProvider *dpp);
 
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 193dfb007e80..597f4f1ccf29 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1084,9 +1084,19 @@ std::unique_ptr<Notification> RadosStore::get_notification(
   return std::make_unique<RadosNotification>(s, this, obj, src_obj, s, event_type, y, object_name);
 }
 
-std::unique_ptr<Notification> RadosStore::get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y)
-{
-  return std::make_unique<RadosNotification>(dpp, this, obj, src_obj, event_type, _bucket, _user_id, _user_tenant, _req_id, y);
+std::unique_ptr<Notification> RadosStore::get_notification(
+    const DoutPrefixProvider* dpp,
+    rgw::sal::Object* obj,
+    rgw::sal::Object* src_obj,
+    const rgw::notify::EventTypeList& event_types,
+    rgw::sal::Bucket* _bucket,
+    std::string& _user_id,
+    std::string& _user_tenant,
+    std::string& _req_id,
+    optional_yield y) {
+  return std::make_unique<RadosNotification>(dpp, this, obj, src_obj,
+                                             event_types, _bucket, _user_id,
+                                             _user_tenant, _req_id, y);
 }
 
 std::string RadosStore::topics_oid(const std::string& tenant) const {
@@ -3007,13 +3017,13 @@ std::unique_ptr<LCSerializer> RadosLifecycle::get_serializer(const std::string&
 
 int RadosNotification::publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags)
 {
-  return rgw::notify::publish_reserve(dpp, *store->svc()->site, event_type, res, obj_tags);
+  return rgw::notify::publish_reserve(dpp, *store->svc()->site, event_types, res, obj_tags);
 }
 
 int RadosNotification::publish_commit(const DoutPrefixProvider* dpp, uint64_t size,
 				     const ceph::real_time& mtime, const std::string& etag, const std::string& version)
 {
-  return rgw::notify::publish_commit(obj, size, mtime, etag, version, event_type, res, dpp);
+  return rgw::notify::publish_commit(obj, size, mtime, etag, version, res, dpp);
 }
 
 int RadosAtomicWriter::prepare(optional_yield y)
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index c97d5e1832d4..4e71045cda1e 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -155,9 +155,15 @@ class RadosStore : public StoreDriver {
     virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
     virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override;
     virtual std::unique_ptr<Notification> get_notification(
-    const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, 
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant,
-    std::string& _req_id, optional_yield y) override;
+        const DoutPrefixProvider* dpp,
+        rgw::sal::Object* obj,
+        rgw::sal::Object* src_obj,
+        const rgw::notify::EventTypeList& event_types,
+        rgw::sal::Bucket* _bucket,
+        std::string& _user_id,
+        std::string& _user_tenant,
+        std::string& _req_id,
+        optional_yield y) override;
     int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) override;
     int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) override;
@@ -733,13 +739,41 @@ class RadosNotification : public StoreNotification {
   rgw::notify::reservation_t res;
 
   public:
-    RadosNotification(const DoutPrefixProvider* _dpp, RadosStore* _store, Object* _obj, Object* _src_obj, req_state* _s, rgw::notify::EventType _type, optional_yield y, const std::string* object_name) :
-      StoreNotification(_obj, _src_obj, _type), store(_store), res(_dpp, _store, _s, _obj, _src_obj, object_name, y) { }
-
-    RadosNotification(const DoutPrefixProvider* _dpp, RadosStore* _store, Object* _obj, Object* _src_obj, rgw::notify::EventType _type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) :
-      StoreNotification(_obj, _src_obj, _type), store(_store), res(_dpp, _store, _obj, _src_obj, _bucket, _user_id, _user_tenant, _req_id, y) {}
-
-    ~RadosNotification() = default;
+  RadosNotification(const DoutPrefixProvider* _dpp,
+                    RadosStore* _store,
+                    Object* _obj,
+                    Object* _src_obj,
+                    req_state* _s,
+                    rgw::notify::EventType _type,
+                    optional_yield y,
+                    const std::string* object_name)
+      : StoreNotification(_obj, _src_obj, {_type}),
+        store(_store),
+        res(_dpp, _store, _s, _obj, _src_obj, object_name, y) {}
+
+   RadosNotification(const DoutPrefixProvider* _dpp,
+                     RadosStore* _store,
+                     Object* _obj,
+                     Object* _src_obj,
+                     const rgw::notify::EventTypeList& _types,
+                     rgw::sal::Bucket* _bucket,
+                     std::string& _user_id,
+                     std::string& _user_tenant,
+                     std::string& _req_id,
+                     optional_yield y)
+       : StoreNotification(_obj, _src_obj, _types),
+         store(_store),
+         res(_dpp,
+             _store,
+             _obj,
+             _src_obj,
+             _bucket,
+             _user_id,
+             _user_tenant,
+             _req_id,
+             y) {}
+
+   ~RadosNotification() = default;
 
     rgw::notify::reservation_t& get_reservation(void) {
       return res;
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 560c1e6c7b51..a75ce3a71c29 100644
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -535,10 +535,10 @@ static bool zonegroup_lc_check(const DoutPrefixProvider *dpp, rgw::sal::Zone* zo
   });
 }
 
-static int remove_expired_obj(
-  const DoutPrefixProvider *dpp, lc_op_ctx& oc, bool remove_indeed,
-  rgw::notify::EventType event_type)
-{
+static int remove_expired_obj(const DoutPrefixProvider* dpp,
+                              lc_op_ctx& oc,
+                              bool remove_indeed,
+                              const rgw::notify::EventTypeList& event_types) {
   auto& driver = oc.driver;
   auto& bucket_info = oc.bucket->get_info();
   auto& o = oc.o;
@@ -575,10 +575,9 @@ static int remove_expired_obj(
   del_op->params.unmod_since = meta.mtime;
 
   // notification supported only for RADOS driver for now
-  notify = driver->get_notification(dpp, obj.get(), nullptr, event_type,
-				   oc.bucket, lc_id,
-				   const_cast<std::string&>(oc.bucket->get_tenant()),
-				   lc_req_id, null_yield);
+  notify = driver->get_notification(
+      dpp, obj.get(), nullptr, event_types, oc.bucket, lc_id,
+      const_cast<std::string&>(oc.bucket->get_tenant()), lc_req_id, null_yield);
 
   ret = notify->publish_reserve(dpp, nullptr);
   if ( ret < 0) {
@@ -871,10 +870,9 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
 
       std::unique_ptr<rgw::sal::Notification> notify
 	= driver->get_notification(
-	  this, sal_obj.get(), nullptr, event_type,
-	  target, lc_id,
-	  const_cast<std::string&>(target->get_tenant()),
-	  lc_req_id, null_yield);
+          this, sal_obj.get(), nullptr, {event_type}, target, lc_id,
+          const_cast<std::string&>(target->get_tenant()), lc_req_id,
+          null_yield);
       auto version_id = obj.key.instance;
 
       ret = notify->publish_reserve(this, nullptr);
@@ -1144,8 +1142,10 @@ class LCOpAction_CurrentExpiration : public LCOpAction {
     auto& o = oc.o;
     int r;
     if (o.is_delete_marker()) {
-      r = remove_expired_obj(oc.dpp, oc, true,
-			     rgw::notify::ObjectExpirationDeleteMarker);
+      r = remove_expired_obj(
+          oc.dpp, oc, true,
+          {rgw::notify::ObjectExpirationDeleteMarker,
+           rgw::notify::LifecycleExpirationDeleteMarkerCreated});
       if (r < 0) {
 	ldpp_dout(oc.dpp, 0) << "ERROR: current is-dm remove_expired_obj "
 			 << oc.bucket << ":" << o.key
@@ -1159,7 +1159,8 @@ class LCOpAction_CurrentExpiration : public LCOpAction {
     } else {
       /* ! o.is_delete_marker() */
       r = remove_expired_obj(oc.dpp, oc, !oc.bucket->versioned(),
-			     rgw::notify::ObjectExpirationCurrent);
+                             {rgw::notify::ObjectExpirationCurrent,
+                              rgw::notify::LifecycleExpirationDelete});
       if (r < 0) {
 	ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj "
 			 << oc.bucket << ":" << o.key
@@ -1207,7 +1208,7 @@ class LCOpAction_NonCurrentExpiration : public LCOpAction {
   int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r = remove_expired_obj(oc.dpp, oc, true,
-			       rgw::notify::ObjectExpirationNoncurrent);
+                               {rgw::notify::ObjectExpirationNoncurrent});
     if (r < 0) {
       ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj (non-current expiration) " 
 		       << oc.bucket << ":" << o.key
@@ -1252,7 +1253,8 @@ class LCOpAction_DMExpiration : public LCOpAction {
   int process(lc_op_ctx& oc) override {
     auto& o = oc.o;
     int r = remove_expired_obj(oc.dpp, oc, true,
-			       rgw::notify::ObjectExpirationDeleteMarker);
+        {rgw::notify::ObjectExpirationDeleteMarker,
+         rgw::notify::LifecycleExpirationDeleteMarkerCreated});
     if (r < 0) {
       ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj (delete marker expiration) "
 		       << oc.bucket << ":" << o.key
@@ -1330,21 +1332,23 @@ class LCOpAction_Transition : public LCOpAction {
     /* If bucket is versioned, create delete_marker for current version
      */
     if (! oc.bucket->versioned()) {
-      ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransition);
+      ret =
+          remove_expired_obj(oc.dpp, oc, true, {rgw::notify::ObjectTransition});
       ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key
                             << ") not versioned flags: " << oc.o.flags << dendl;
     } else {
       /* versioned */
       if (oc.o.is_current() && !oc.o.is_delete_marker()) {
         ret = remove_expired_obj(oc.dpp, oc, false,
-                                 rgw::notify::ObjectTransitionCurrent);
+                                 {rgw::notify::ObjectTransitionCurrent,
+                                  rgw::notify::LifecycleTransition});
         ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key
                               << ") current & not delete_marker"
                               << " versioned_epoch:  " << oc.o.versioned_epoch
                               << "flags: " << oc.o.flags << dendl;
       } else {
         ret = remove_expired_obj(oc.dpp, oc, true,
-                                 rgw::notify::ObjectTransitionNoncurrent);
+                                 {rgw::notify::ObjectTransitionNoncurrent});
         ldpp_dout(oc.dpp, 20)
             << "delete_tier_obj Object(key:" << oc.o.key << ") not current "
             << "versioned_epoch:  " << oc.o.versioned_epoch
@@ -1371,17 +1375,20 @@ class LCOpAction_Transition : public LCOpAction {
       return ret;
     }
 
-    const auto event_type = (bucket->versioned() &&
-			     oc.o.is_current() && !oc.o.is_delete_marker()) ?
-      rgw::notify::ObjectTransitionCurrent :
-      rgw::notify::ObjectTransitionNoncurrent;
+    rgw::notify::EventTypeList event_types;
+    if (bucket->versioned() && oc.o.is_current() && !oc.o.is_delete_marker()) {
+      event_types.insert(event_types.end(),
+                         {rgw::notify::ObjectTransitionCurrent,
+                          rgw::notify::LifecycleTransition});
+    } else {
+      event_types.push_back(rgw::notify::ObjectTransitionNoncurrent);
+    }
 
-    std::unique_ptr<rgw::sal::Notification> notify
-      = oc.driver->get_notification(
-	oc.dpp, obj.get(), nullptr, event_type,
-	bucket, lc_id,
-	const_cast<std::string&>(oc.bucket->get_tenant()),
-	lc_req_id, null_yield);
+    std::unique_ptr<rgw::sal::Notification> notify =
+        oc.driver->get_notification(
+            oc.dpp, obj.get(), nullptr, event_types, bucket, lc_id,
+            const_cast<std::string&>(oc.bucket->get_tenant()), lc_req_id,
+            null_yield);
     auto version_id = oc.o.key.instance;
 
     ret = notify->publish_reserve(oc.dpp, nullptr);
diff --git a/src/rgw/rgw_notify_event_type.cc b/src/rgw/rgw_notify_event_type.cc
index 7a0ef9568e15..d36e10c9c2d3 100644
--- a/src/rgw/rgw_notify_event_type.cc
+++ b/src/rgw/rgw_notify_event_type.cc
@@ -50,6 +50,14 @@ namespace rgw::notify {
       return "s3:ObjectSynced:Delete";
     case ObjectSyncedDeletionMarkerCreated:
       return "s3:ObjectSynced:DeletionMarkerCreated";
+    case LifecycleExpiration:
+      return "s3:LifecycleExpiration:*";
+    case LifecycleExpirationDelete:
+      return "s3:LifecycleExpiration:Delete";
+    case LifecycleExpirationDeleteMarkerCreated:
+      return "s3:LifecycleExpiration:DeleteMarkerCreated";
+    case LifecycleTransition:
+      return "s3:LifecycleTransition";
     case UnknownEvent:
         return "s3:UnknownEvent";
     }
@@ -103,6 +111,14 @@ namespace rgw::notify {
         return ObjectSyncedDelete;
     if (s == "s3:ObjectSynced:DeletionMarkerCreated")
         return ObjectSyncedDeletionMarkerCreated;
+    if (s == "s3:LifecycleExpiration:*")
+      return LifecycleExpiration;
+    if (s == "s3:LifecycleExpiration:Delete")
+      return LifecycleExpirationDelete;
+    if (s == "s3:LifecycleExpiration:DeleteMarkerCreated")
+      return LifecycleExpirationDeleteMarkerCreated;
+    if (s == "s3:LifecycleTransition")
+      return LifecycleTransition;
     return UnknownEvent;
   }
 
diff --git a/src/rgw/rgw_notify_event_type.h b/src/rgw/rgw_notify_event_type.h
index 4fe1b5c90c62..d7f70682d162 100644
--- a/src/rgw/rgw_notify_event_type.h
+++ b/src/rgw/rgw_notify_event_type.h
@@ -29,8 +29,12 @@ namespace rgw::notify {
     ObjectSyncedCreate                   = 0x10000,
     ObjectSyncedDelete                   = 0x20000,
     ObjectSyncedDeletionMarkerCreated    = 0x40000,
-    UnknownEvent                         = 0x100000
-  };
+    LifecycleExpiration                    = 0xF00000,
+    LifecycleExpirationDelete              = 0x100000,
+    LifecycleExpirationDeleteMarkerCreated = 0x200000,
+    LifecycleTransition                    = 0xF000000,
+    UnknownEvent                           = 0x10000000
+};
 
   using EventTypeList = std::vector<EventType>;
 
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 7202d9c90dca..e21a6180f8b4 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -302,9 +302,15 @@ class Driver {
         rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) = 0;
     /** No-req_state variant (e.g., rgwlc) */
     virtual std::unique_ptr<Notification> get_notification(
-    const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj,
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant,
-    std::string& _req_id, optional_yield y) = 0;
+        const DoutPrefixProvider* dpp,
+        rgw::sal::Object* obj,
+        rgw::sal::Object* src_obj,
+        const rgw::notify::EventTypeList& event_types,
+        rgw::sal::Bucket* _bucket,
+        std::string& _user_id,
+        std::string& _user_tenant,
+        std::string& _req_id,
+        optional_yield y) = 0;
     /** Read the topic config entry into @a data and (optionally) @a objv_tracker */
     virtual int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) = 0;
diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc
index 2dc9fc95d105..0019178f640f 100644
--- a/src/rgw/rgw_sal_dbstore.cc
+++ b/src/rgw/rgw_sal_dbstore.cc
@@ -1662,17 +1662,21 @@ namespace rgw::sal {
     rgw::notify::EventType event_type, optional_yield y,
     const std::string* object_name)
   {
-    return std::make_unique<DBNotification>(obj, src_obj, event_type);
+    rgw::notify::EventTypeList event_types = {event_type};
+    return std::make_unique<DBNotification>(obj, src_obj, event_types);
   }
 
   std::unique_ptr<Notification> DBStore::get_notification(
-    const DoutPrefixProvider* dpp, rgw::sal::Object* obj,
-    rgw::sal::Object* src_obj,
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-    std::string& _user_id, std::string& _user_tenant, std::string& _req_id,
-    optional_yield y)
-  {
-    return std::make_unique<DBNotification>(obj, src_obj, event_type);
+      const DoutPrefixProvider* dpp,
+      rgw::sal::Object* obj,
+      rgw::sal::Object* src_obj,
+      const rgw::notify::EventTypeList& event_types,
+      rgw::sal::Bucket* _bucket,
+      std::string& _user_id,
+      std::string& _user_tenant,
+      std::string& _req_id,
+      optional_yield y) {
+    return std::make_unique<DBNotification>(obj, src_obj, event_types);
   }
 
   RGWLC* DBStore::get_rgwlc(void) {
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
index 3c0c7c765198..6ce6398d0626 100644
--- a/src/rgw/rgw_sal_dbstore.h
+++ b/src/rgw/rgw_sal_dbstore.h
@@ -63,11 +63,13 @@ class DBLifecycle : public StoreLifecycle {
 class DBNotification : public StoreNotification {
 protected:
   public:
-  DBNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type)
-    : StoreNotification(_obj, _src_obj, _type) {}
-    ~DBNotification() = default;
+ DBNotification(Object* _obj,
+                Object* _src_obj,
+                const rgw::notify::EventTypeList& _types)
+     : StoreNotification(_obj, _src_obj, _types) {}
+ ~DBNotification() = default;
 
-    virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;}
+ virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;}
     virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size,
 			       const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; }
 };
@@ -765,13 +767,17 @@ class DBNotification : public StoreNotification {
     rgw::notify::EventType event_type, optional_yield y, const std::string* object_name) override;
 
   virtual std::unique_ptr<Notification> get_notification(
-    const DoutPrefixProvider* dpp, rgw::sal::Object* obj,
-    rgw::sal::Object* src_obj,
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-    std::string& _user_id, std::string& _user_tenant, std::string& _req_id,
-    optional_yield y) override;
-
-      virtual RGWLC* get_rgwlc(void) override;
+      const DoutPrefixProvider* dpp,
+      rgw::sal::Object* obj,
+      rgw::sal::Object* src_obj,
+      const rgw::notify::EventTypeList& event_types,
+      rgw::sal::Bucket* _bucket,
+      std::string& _user_id,
+      std::string& _user_tenant,
+      std::string& _req_id,
+      optional_yield y) override;
+
+  virtual RGWLC* get_rgwlc(void) override;
       virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; }
       virtual int log_usage(const DoutPrefixProvider *dpp, std::map<rgw_user_bucket, RGWUsageBatch>& usage_info, optional_yield y) override;
       virtual int log_op(const DoutPrefixProvider *dpp, std::string& oid, bufferlist& bl) override;
diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc
index 6e0ca8ccd821..94b922acff48 100644
--- a/src/rgw/rgw_sal_filter.cc
+++ b/src/rgw/rgw_sal_filter.cc
@@ -228,20 +228,19 @@ std::unique_ptr<Notification> FilterDriver::get_notification(rgw::sal::Object* o
   return std::make_unique<FilterNotification>(std::move(n));
 }
 
-std::unique_ptr<Notification> FilterDriver::get_notification(const DoutPrefixProvider* dpp,
-				rgw::sal::Object* obj, rgw::sal::Object* src_obj,
-				rgw::notify::EventType event_type,
-				rgw::sal::Bucket* _bucket, std::string& _user_id,
-				std::string& _user_tenant, std::string& _req_id,
-				optional_yield y)
-{
-  std::unique_ptr<Notification> n = next->get_notification(dpp, nextObject(obj),
-							   nextObject(src_obj),
-							   event_type,
-							   nextBucket(_bucket),
-							   _user_id,
-							   _user_tenant,
-							   _req_id, y);
+std::unique_ptr<Notification> FilterDriver::get_notification(
+    const DoutPrefixProvider* dpp,
+    rgw::sal::Object* obj,
+    rgw::sal::Object* src_obj,
+    const rgw::notify::EventTypeList& event_types,
+    rgw::sal::Bucket* _bucket,
+    std::string& _user_id,
+    std::string& _user_tenant,
+    std::string& _req_id,
+    optional_yield y) {
+  std::unique_ptr<Notification> n = next->get_notification(
+      dpp, nextObject(obj), nextObject(src_obj), event_types,
+      nextBucket(_bucket), _user_id, _user_tenant, _req_id, y);
   return std::make_unique<FilterNotification>(std::move(n));
 }
 
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 5095f675f162..b07179aa44e5 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -176,11 +176,15 @@ class FilterDriver : public Driver {
 				 rgw::notify::EventType event_type, optional_yield y,
 				 const std::string* object_name=nullptr) override;
   virtual std::unique_ptr<Notification> get_notification(
-    const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj,
-
-    rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket,
-    std::string& _user_id, std::string& _user_tenant,
-    std::string& _req_id, optional_yield y) override;
+      const DoutPrefixProvider* dpp,
+      rgw::sal::Object* obj,
+      rgw::sal::Object* src_obj,
+      const rgw::notify::EventTypeList& event_types,
+      rgw::sal::Bucket* _bucket,
+      std::string& _user_id,
+      std::string& _user_tenant,
+      std::string& _req_id,
+      optional_yield y) override;
 
   int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
       optional_yield y, const DoutPrefixProvider *dpp) override {
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index 23fc3eb76856..1ba44bc02ec4 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -408,14 +408,15 @@ class StoreNotification : public Notification {
 protected:
   Object* obj;
   Object* src_obj;
-  rgw::notify::EventType event_type;
+  rgw::notify::EventTypeList event_types;
 
-  public:
-    StoreNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type)
-      : obj(_obj), src_obj(_src_obj), event_type(_type)
-    {}
+ public:
+  StoreNotification(Object* _obj,
+                    Object* _src_obj,
+                    rgw::notify::EventTypeList _types)
+      : obj(_obj), src_obj(_src_obj), event_types(std::move(_types)) {}
 
-    virtual ~StoreNotification() = default;
+  virtual ~StoreNotification() = default;
 };
 
 class StoreWriter : public Writer {
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 8e9ebd126c63..c9049dfd1f8f 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -1907,7 +1907,8 @@ def test_ps_s3_lifecycle_on_master():
     notification_name = bucket_name + NOTIFICATION_SUFFIX
     topic_conf_list = [{'Id': notification_name,
                         'TopicArn': topic_arn,
-                        'Events': ['s3:ObjectLifecycle:Expiration:*']
+                        'Events': ['s3:ObjectLifecycle:Expiration:*',
+                                   's3:LifecycleExpiration:*']
                        }]
     s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list)
     response, status = s3_notification_conf.set_config()
@@ -1927,14 +1928,14 @@ def test_ps_s3_lifecycle_on_master():
 
     time_diff = time.time() - start_time
     print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-    
+
     # create lifecycle policy
     client = boto3.client('s3',
             endpoint_url='http://'+conn.host+':'+str(conn.port),
             aws_access_key_id=conn.aws_access_key_id,
             aws_secret_access_key=conn.aws_secret_access_key)
     yesterday = datetime.date.today() - datetime.timedelta(days=1)
-    response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, 
+    response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name,
             LifecycleConfiguration={'Rules': [
                 {
                     'ID': 'rule1',
@@ -1956,8 +1957,11 @@ def test_ps_s3_lifecycle_on_master():
     print('total number of objects: ' + str(len(keys)))
     event_keys = []
     events = http_server.get_and_reset_events()
+    assert_equal(number_of_objects * 2, len(events))
     for event in events:
-        assert_equal(event['Records'][0]['eventName'], 'ObjectLifecycle:Expiration:Current')
+        assert_in(event['Records'][0]['eventName'],
+                  ['LifecycleExpiration:Delete',
+                   'ObjectLifecycle:Expiration:Current'])
         event_keys.append(event['Records'][0]['s3']['object']['key'])
     for key in keys:
         key_found = False

From 3ea110bb89a7f666de25f8a128fa8bbb231e304f Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Tue, 27 Feb 2024 14:24:43 -0500
Subject: [PATCH 2273/2492] rgw/notification: Make the Replication events to be
 aws compatible.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 doc/radosgw/s3-notification-compatibility.rst |  8 ++++++++
 src/rgw/driver/rados/rgw_cr_rados.cc          | 11 +++++++----
 src/rgw/rgw_notify_event_type.cc              | 16 ++++++++++++++++
 src/rgw/rgw_notify_event_type.h               |  6 +++++-
 4 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
index b29989977b23..31b218456bd8 100644
--- a/doc/radosgw/s3-notification-compatibility.rst
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -113,6 +113,14 @@ Event Types
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectSynced:DeletionMarkerCreated``              | Defined, Ceph extension (not generated) |
 +--------------------------------------------------------+-----------------------------------------+
+| ``s3:Replication:*``                                   | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:Replication:Create``                              | Supported                               |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:Replication:Delete``                              | Defined, Supported (not generated)      |
++--------------------------------------------------------+-----------------------------------------+
+| ``s3:Replication:DeletionMarkerCreated``               | Defined, Supported (not generated)      |
++--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectRestore:Post``                              | Not applicable                          |
 +--------------------------------------------------------+-----------------------------------------+
 | ``s3:ObjectRestore:Complete``                          | Not applicable                          |
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index a6d4719e7186..7e4164ff7dde 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -806,15 +806,18 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
         std::string tenant(dest_bucket.get_tenant());
 
         std::unique_ptr<rgw::sal::Notification> notify =
-            store->get_notification(
-                dpp, &dest_obj, nullptr, {rgw::notify::ObjectSyncedCreate},
-                &dest_bucket, user_id, tenant, req_id, null_yield);
+            store->get_notification(dpp, &dest_obj, nullptr,
+                                    {rgw::notify::ObjectSyncedCreate,
+                                     rgw::notify::ReplicationCreate},
+                                    &dest_bucket, user_id, tenant, req_id,
+                                    null_yield);
 
         auto notify_res =
             static_cast<rgw::sal::RadosNotification*>(notify.get())
                 ->get_reservation();
         int ret = rgw::notify::publish_reserve(
-            dpp, *store->svc()->site, {rgw::notify::ObjectSyncedCreate},
+            dpp, *store->svc()->site,
+            {rgw::notify::ObjectSyncedCreate, rgw::notify::ReplicationCreate},
             notify_res, &obj_tags);
         if (ret < 0) {
           ldpp_dout(dpp, 1)
diff --git a/src/rgw/rgw_notify_event_type.cc b/src/rgw/rgw_notify_event_type.cc
index d36e10c9c2d3..9baa5df798bd 100644
--- a/src/rgw/rgw_notify_event_type.cc
+++ b/src/rgw/rgw_notify_event_type.cc
@@ -58,6 +58,14 @@ namespace rgw::notify {
       return "s3:LifecycleExpiration:DeleteMarkerCreated";
     case LifecycleTransition:
       return "s3:LifecycleTransition";
+    case Replication:
+      return "s3:Replication:*";
+    case ReplicationCreate:
+      return "s3:Replication:Create";
+    case ReplicationDelete:
+      return "s3:Replication:Delete";
+    case ReplicationDeletionMarkerCreated:
+      return "s3:Replication:DeletionMarkerCreated";
     case UnknownEvent:
         return "s3:UnknownEvent";
     }
@@ -119,6 +127,14 @@ namespace rgw::notify {
       return LifecycleExpirationDeleteMarkerCreated;
     if (s == "s3:LifecycleTransition")
       return LifecycleTransition;
+    if (s == "s3:Replication:*")
+      return Replication;
+    if (s == "s3:Replication:Create")
+      return ReplicationCreate;
+    if (s == "s3:Replication:Delete")
+      return ReplicationDelete;
+    if (s == "s3:Replication:DeletionMarkerCreated")
+      return ReplicationDeletionMarkerCreated;
     return UnknownEvent;
   }
 
diff --git a/src/rgw/rgw_notify_event_type.h b/src/rgw/rgw_notify_event_type.h
index d7f70682d162..34a73d229979 100644
--- a/src/rgw/rgw_notify_event_type.h
+++ b/src/rgw/rgw_notify_event_type.h
@@ -33,7 +33,11 @@ namespace rgw::notify {
     LifecycleExpirationDelete              = 0x100000,
     LifecycleExpirationDeleteMarkerCreated = 0x200000,
     LifecycleTransition                    = 0xF000000,
-    UnknownEvent                           = 0x10000000
+    Replication                            = 0xF0000000,
+    ReplicationCreate                      = 0x10000000,
+    ReplicationDelete                      = 0x20000000,
+    ReplicationDeletionMarkerCreated       = 0x40000000,
+    UnknownEvent                           = 0x100000000
 };
 
   using EventTypeList = std::vector<EventType>;

From 37069ac96b53d2d4927b9d6f743790847861ec8e Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Tue, 27 Feb 2024 15:11:48 -0500
Subject: [PATCH 2274/2492] rgw/notification: Support generating multisite sync
 delete events.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 doc/radosgw/s3-notification-compatibility.rst | 131 ++++++++++--------
 src/rgw/driver/rados/rgw_cr_rados.cc          | 107 +++++++-------
 2 files changed, 127 insertions(+), 111 deletions(-)

diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
index 31b218456bd8..b6bf460f6f4e 100644
--- a/doc/radosgw/s3-notification-compatibility.rst
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -66,67 +66,76 @@ However, the ``requestParameters.sourceIPAddress`` field will be sent empty.
 Event Types
 -----------
 
-+--------------------------------------------------------+-----------------------------------------+
-| Event                                                  | Note                                    |
-+========================================================+=========================================+
-| ``s3:ObjectCreated:*``                                 | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectCreated:Put``                               | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectCreated:Post``                              | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectCreated:Copy``                              | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectCreated:CompleteMultipartUpload``           | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectRemoved:*``                                 | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectRemoved:Delete``                            | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectRemoved:DeleteMarkerCreated``               | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Expiration:Current``              | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Expiration:NonCurrent``           | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Expiration:DeleteMarker``         | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Expiration:AbortMultipartUpload`` | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Transition:Current``              | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectLifecycle:Transition:NonCurrent``           | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:LifecycleExpiration:*``                           | Supported. Equivalent to: s3:LifecycleExpiration:Delete, s3:LifecycleExpiration:DeleteMarkerCreated|
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:LifecycleExpiration:Delete``                      | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:Current.|
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:LifecycleExpiration:DeleteMarkerCreated``         | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:DeleteMarker.|
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:LifecycleTransition``                             | Supported. Equivalent to: s3:ObjectLifecycle:Transition:Current|
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectSynced:*``                                  | Ceph extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectSynced:Create``                             | Ceph Extension                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectSynced:Delete``                             | Defined, Ceph extension (not generated) |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectSynced:DeletionMarkerCreated``              | Defined, Ceph extension (not generated) |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:Replication:*``                                   | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:Replication:Create``                              | Supported                               |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:Replication:Delete``                              | Defined, Supported (not generated)      |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:Replication:DeletionMarkerCreated``               | Defined, Supported (not generated)      |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectRestore:Post``                              | Not applicable                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ObjectRestore:Complete``                          | Not applicable                          |
-+--------------------------------------------------------+-----------------------------------------+
-| ``s3:ReducedRedundancyLostObject``                     | Not applicable                          |
-+--------------------------------------------------------+-----------------------------------------+
++--------------------------------------------------------+-------------------------------------------+
+| Event                                                  | Note                                      |
++========================================================+===========================================+
+| ``s3:ObjectCreated:*``                                 | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectCreated:Put``                               | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectCreated:Post``                              | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectCreated:Copy``                              | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectCreated:CompleteMultipartUpload``           | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectRemoved:*``                                 | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectRemoved:Delete``                            | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectRemoved:DeleteMarkerCreated``               | Supported                                 |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:Current``              | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:NonCurrent``           | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:DeleteMarker``         | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Expiration:AbortMultipartUpload`` | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Transition:Current``              | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectLifecycle:Transition:NonCurrent``           | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:LifecycleExpiration:*``                           | Supported. Equivalent to                  |
+|                                                        | s3:LifecycleExpiration:Delete,            |
+|                                                        | s3:LifecycleExpiration:DeleteMarkerCreated|
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:LifecycleExpiration:Delete``                      | Supported. Equivalent to                  |
+|                                                        | s3:ObjectLifecycle:Expiration:Current     |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:LifecycleExpiration:DeleteMarkerCreated``         | Supported. Equivalent to                  |
+|                                                        | s3:ObjectLifecycle:Expiration:DeleteMarker|
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:LifecycleTransition``                             | Supported. Equivalent to                  |
+|                                                        | s3:ObjectLifecycle:Transition:Current     |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectSynced:*``                                  | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectSynced:Create``                             | Ceph Extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectSynced:Delete``                             | Ceph extension                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectSynced:DeletionMarkerCreated``              | Defined, Ceph extension (not generated)   |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:Replication:*``                                   | Supported. Equivalent to                  |
+|                                                        | s3:ObjectSynced:Create,                   |
+|                                                        | s3:ObjectSynced:Delete                    |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:Replication:Create``                              | Supported. Equivalent to                  |
+|                                                        | s3:ObjectSynced:Create                    |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:Replication:Delete``                              | Supported. Equivalent to                  |
+|                                                        | s3:ObjectSynced:Delete                    |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:Replication:DeletionMarkerCreated``               | Defined, Supported (not generated)        |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectRestore:Post``                              | Not applicable                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ObjectRestore:Complete``                          | Not applicable                            |
++--------------------------------------------------------+-------------------------------------------+
+| ``s3:ReducedRedundancyLostObject``                     | Not applicable                            |
++--------------------------------------------------------+-------------------------------------------+
 
 .. note:: 
 
diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index 7e4164ff7dde..a717b971a896 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -717,6 +717,55 @@ int RGWRadosBILogTrimCR::request_complete()
   return r;
 }
 
+int send_sync_notification(const DoutPrefixProvider* dpp,
+                           rgw::sal::RadosStore* store,
+                           rgw::sal::Bucket* bucket,
+                           rgw::sal::Object* obj,
+                           const rgw::sal::Attrs& attrs,
+                           uint64_t obj_size,
+                           const rgw::notify::EventTypeList& event_types) {
+  // send notification that object was successfully synced
+  std::string user_id = "rgw sync";
+  std::string req_id = "0";
+
+  RGWObjTags obj_tags;
+  auto iter = attrs.find(RGW_ATTR_TAGS);
+  if (iter != attrs.end()) {
+    try {
+      auto it = iter->second.cbegin();
+      obj_tags.decode(it);
+    } catch (buffer::error& err) {
+      ldpp_dout(dpp, 1) << "ERROR: " << __func__
+                        << ": caught buffer::error couldn't decode TagSet "
+                        << dendl;
+      return -EIO;
+    }
+  }
+  rgw::notify::reservation_t notify_res(dpp, store, obj, nullptr, bucket,
+                                        user_id, bucket->get_tenant(), req_id,
+                                        null_yield);
+  int ret = rgw::notify::publish_reserve(dpp, *store->svc()->site, event_types,
+                                         notify_res, &obj_tags);
+  if (ret < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: reserving notification failed, with error: "
+                      << ret << dendl;
+  } else {
+    std::string etag;
+    const auto iter = attrs.find(RGW_ATTR_ETAG);
+    if (iter != attrs.end()) {
+      etag = iter->second.to_str();
+    }
+    ret =
+        rgw::notify::publish_commit(obj, obj_size, ceph::real_clock::now(),
+                                    etag, obj->get_instance(), notify_res, dpp);
+    if (ret < 0) {
+      ldpp_dout(dpp, 1) << "ERROR: publishing notification failed, with error: "
+                        << ret << dendl;
+    }
+  }
+  return ret;
+}
+
 int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
 {
   RGWObjectCtx obj_ctx(store);
@@ -784,56 +833,9 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp)
   } else {
       // r >= 0
       if (bytes_transferred) {
-        // send notification that object was successfully synced
-        std::string user_id = "rgw sync";
-        std::string req_id = "0";
-
-        RGWObjTags obj_tags;
-        auto iter = attrs.find(RGW_ATTR_TAGS);
-        if (iter != attrs.end()) {
-          try {
-            auto it = iter->second.cbegin();
-            obj_tags.decode(it);
-          } catch (buffer::error& err) {
-            ldpp_dout(dpp, 1)
-                << "ERROR: " << __func__
-                << ": caught buffer::error couldn't decode TagSet " << dendl;
-          }
-        }
-
-        // NOTE: we create a mutable copy of bucket.get_tenant as the
-        // get_notification function expects a std::string&, not const
-        std::string tenant(dest_bucket.get_tenant());
-
-        std::unique_ptr<rgw::sal::Notification> notify =
-            store->get_notification(dpp, &dest_obj, nullptr,
-                                    {rgw::notify::ObjectSyncedCreate,
-                                     rgw::notify::ReplicationCreate},
-                                    &dest_bucket, user_id, tenant, req_id,
-                                    null_yield);
-
-        auto notify_res =
-            static_cast<rgw::sal::RadosNotification*>(notify.get())
-                ->get_reservation();
-        int ret = rgw::notify::publish_reserve(
-            dpp, *store->svc()->site,
-            {rgw::notify::ObjectSyncedCreate, rgw::notify::ReplicationCreate},
-            notify_res, &obj_tags);
-        if (ret < 0) {
-          ldpp_dout(dpp, 1)
-              << "ERROR: reserving notification failed, with error: " << ret
-              << dendl;
-          // no need to return, the sync already happened
-        } else {
-          ret = rgw::notify::publish_commit(
-              &dest_obj, *bytes_transferred, ceph::real_clock::now(), etag,
-              dest_obj.get_instance(), notify_res, dpp);
-          if (ret < 0) {
-            ldpp_dout(dpp, 1)
-                << "ERROR: publishing notification failed, with error: " << ret
-                << dendl;
-          }
-        }
+        send_sync_notification(
+            dpp, store, &dest_bucket, &dest_obj, attrs, *bytes_transferred,
+            {rgw::notify::ObjectSyncedCreate, rgw::notify::ReplicationCreate});
       }
 
       if (counters) {
@@ -940,6 +942,11 @@ int RGWAsyncRemoveObj::_send_request(const DoutPrefixProvider *dpp)
   ret = del_op->delete_obj(dpp, null_yield, true);
   if (ret < 0) {
     ldpp_dout(dpp, 20) << __func__ << "(): delete_obj() obj=" << obj << " returned ret=" << ret << dendl;
+  } else {
+    send_sync_notification(
+        dpp, store, bucket.get(), obj.get(), obj->get_attrs(),
+        obj->get_obj_size(),
+        {rgw::notify::ObjectSyncedDelete, rgw::notify::ReplicationDelete});
   }
   return ret;
 }

From 57602aad351b21dde06db8632a9c8fdc2ab541d6 Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 14 Mar 2024 13:52:51 -0700
Subject: [PATCH 2275/2492] ceph.spec.in: add runtime protobuf dependency for
 crimson-osd inherited from seastar

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 ceph.spec.in | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ceph.spec.in b/ceph.spec.in
index 38dd579868cb..0abbb11709ce 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -887,6 +887,9 @@ Provides:	ceph-test:/usr/bin/ceph-osdomap-tool
 Requires:	ceph-base = %{_epoch_prefix}%{version}-%{release}
 Requires:	sudo
 Requires:	libstoragemgmt
+%if 0%{with seastar}
+Requires:	protobuf
+%endif
 %if 0%{?weak_deps}
 Recommends:	ceph-volume = %{_epoch_prefix}%{version}-%{release}
 %endif

From ab4c5daead7f26d41028625453d50bb58d3b02be Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Thu, 14 Mar 2024 13:58:09 -0700
Subject: [PATCH 2276/2492] debian/control: add runtime protobuf dependency for
 crimson-osd inherited from seastar

Signed-off-by: Samuel Just <sjust@redhat.com>
---
 debian/control | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/control b/debian/control
index 318a822386ef..5038dc99e0f0 100644
--- a/debian/control
+++ b/debian/control
@@ -392,6 +392,7 @@ Depends: ceph-base (= ${binary:Version}),
          ${misc:Depends},
          ${python3:Depends},
          ${shlibs:Depends},
+         libprotobuf23 <pkg.ceph.crimson>,
 Replaces: ceph (<< 10),
           ceph-test (<< 12.2.2-14),
           ceph-osd (<< 17.0.0)

From f97a16cfba2dbd4ccb34ab812ca14a98a8aa3d92 Mon Sep 17 00:00:00 2001
From: Zack Cerza <zack@redhat.com>
Date: Thu, 14 Mar 2024 15:43:36 -0600
Subject: [PATCH 2277/2492] do_cmake.sh: Use value of ARGS at runtime

The "Building Ceph" section of README.md instructs the user to run:

    ARGS="-DCMAKE_C_COMPILER=gcc-7" ./do_cmake.sh

But, very early on do_cmake.sh was doing ARGS="-GNinja", ignoring the existing
 value.

Signed-off-by: Zack Cerza <zack@redhat.com>
---
 do_cmake.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/do_cmake.sh b/do_cmake.sh
index 90f8a3381df3..8177e7263960 100755
--- a/do_cmake.sh
+++ b/do_cmake.sh
@@ -14,7 +14,7 @@ if [ -e $BUILD_DIR ]; then
 fi
 
 PYBUILD="3"
-ARGS="-GNinja"
+ARGS="${ARGS} -GNinja"
 if [ -r /etc/os-release ]; then
   source /etc/os-release
   case "$ID" in

From 45981167aeee2c7b980539ce4890be4c5feafd7a Mon Sep 17 00:00:00 2001
From: luo rixin <luorixin@huawei.com>
Date: Fri, 23 Feb 2024 09:01:57 +0800
Subject: [PATCH 2278/2492] test/encoding/check-generated: parallel testing by
 type

Fixes: https://tracker.ceph.com/issues/64210

Signed-off-by: luo rixin <luorixin@huawei.com>
---
 src/test/encoding/check-generated.sh | 189 +++++++++++++++------------
 1 file changed, 108 insertions(+), 81 deletions(-)

diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh
index 2569bc1a5969..aae5bd48f759 100755
--- a/src/test/encoding/check-generated.sh
+++ b/src/test/encoding/check-generated.sh
@@ -6,93 +6,120 @@ source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
 
 dir=$1
 
-tmp1=`mktemp /tmp/typ-XXXXXXXXX`
-tmp2=`mktemp /tmp/typ-XXXXXXXXX`
-tmp3=`mktemp /tmp/typ-XXXXXXXXX`
-tmp4=`mktemp /tmp/typ-XXXXXXXXX`
+test_selected_type() {
+    local type=$1
+    local result_dir=$2
+    local failed=0
+    local numtests=0
+    local pids=""
+    local result=$(mktemp ${result_dir}/$type-result-XXXXXXXX)
+    local tmp1=$(mktemp /tmp/typ-XXXXXXXX)
+    local tmp2=$(mktemp /tmp/typ-XXXXXXXX)
+    local tmp3=$(mktemp /tmp/typ-XXXXXXXX)
+    local tmp4=$(mktemp /tmp/typ-XXXXXXXX)
+    local num=$(ceph-dencoder type $type count_tests)
+    local deterministic=0
+    if ceph-dencoder type "$type" is_deterministic; then
+        deterministic=1
+    fi
 
-failed=0
-numtests=0
+    echo "$num $type"
+    for n in $(seq 1 1 $num 2>/dev/null); do
+        run_in_background pids save_stdout "$tmp1" ceph-dencoder type "$type" select_test "$n" dump_json
+        run_in_background pids save_stdout "$tmp2" ceph-dencoder type "$type" select_test "$n" encode decode dump_json
+        run_in_background pids save_stdout "$tmp3" ceph-dencoder type "$type" select_test "$n" copy dump_json
+        run_in_background pids save_stdout "$tmp4" ceph-dencoder type "$type" select_test "$n" copy_ctor dump_json
+        wait_background pids
+
+        if [ $? -ne 0 ]; then
+            echo "**** $type test $n encode+decode check failed ****"
+            echo "   ceph-dencoder type $type select_test $n encode decode"
+            failed=$(($failed + 3))
+            continue
+        fi
+
+        # nondeterministic classes may dump nondeterministically.  compare
+        # the sorted json output.  this is a weaker test, but is better
+        # than nothing.
+        if [ $deterministic -eq 0 ]; then
+            echo "  sorting json output for nondeterministic object"
+            for f in $tmp1 $tmp2 $tmp3 $tmp4; do
+                sort $f | sed 's/,$//' > $f.new
+                mv $f.new $f
+            done
+        fi
+
+        if ! cmp $tmp1 $tmp2; then
+            echo "**** $type test $n dump_json check failed ****"
+            echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+            echo "   ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2"
+            diff $tmp1 $tmp2
+            failed=$(($failed + 1))
+        fi
+
+        if ! cmp $tmp1 $tmp3; then
+            echo "**** $type test $n copy dump_json check failed ****"
+            echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+            echo "   ceph-dencoder type $type select_test $n copy dump_json > $tmp2"
+            diff $tmp1 $tmp2
+            failed=$(($failed + 1))
+        fi
+
+        if ! cmp $tmp1 $tmp4; then
+            echo "**** $type test $n copy_ctor dump_json check failed ****"
+            echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+            echo "   ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2"
+            diff $tmp1 $tmp2
+            failed=$(($failed + 1))
+        fi
+
+        if [ $deterministic -ne 0 ]; then
+            run_in_background pids ceph-dencoder type "$type" select_test $n encode export "$tmp1"
+            run_in_background pids ceph-dencoder type "$type" select_test $n encode decode encode export "$tmp2"
+            wait_background pids
+
+            if ! cmp $tmp1 $tmp2; then
+                echo "**** $type test $n binary reencode check failed ****"
+                echo "   ceph-dencoder type $type select_test $n encode export $tmp1"
+                echo "   ceph-dencoder type $type select_test $n encode decode encode export $tmp2"
+                diff <(hexdump -C $tmp1) <(hexdump -C $tmp2)
+                failed=$(($failed + 1))
+            fi
+        fi
+
+        numtests=$(($numtests + 3))
+    done
+    rm -f $tmp1 $tmp2 $tmp3 $tmp4
+    echo -e "numtests: $numtests\nfailed: $failed" > $result
+}
+
+# Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
+# processors
+if [ $(uname) == FreeBSD -o $(uname) == Darwin ]; then
+    NPROC=$(sysctl -n hw.ncpu)
+    max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
+else
+    max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
+fi
+
+results=$(mktemp -d /tmp/check-generated-result-XXXXXXXX)
+running_jobs=0
+pids=""
 echo "checking ceph-dencoder generated test instances..."
 echo "numgen type"
 while read type; do
-    num=`ceph-dencoder type $type count_tests`
-    echo "$num $type"
-    for n in `seq 1 1 $num 2>/dev/null`; do
-
-	pids=""
-	run_in_background pids save_stdout "$tmp1" ceph-dencoder type "$type" select_test "$n" dump_json
-	run_in_background pids save_stdout "$tmp2" ceph-dencoder type "$type" select_test "$n" encode decode dump_json
-	run_in_background pids save_stdout "$tmp3" ceph-dencoder type "$type" select_test "$n" copy dump_json
-	run_in_background pids save_stdout "$tmp4" ceph-dencoder type "$type" select_test "$n" copy_ctor dump_json
-	wait_background pids
-
-	if [ $? -ne 0 ]; then
-	    echo "**** $type test $n encode+decode check failed ****"
-	    echo "   ceph-dencoder type $type select_test $n encode decode"
-	    failed=$(($failed + 3))
-	    continue
-	fi
-
-	# nondeterministic classes may dump nondeterministically.  compare
-	# the sorted json output.  this is a weaker test, but is better
-	# than nothing.
-	deterministic=0
-	if ceph-dencoder type "$type" is_deterministic; then
-	    deterministic=1
-	fi
-
-	if [ $deterministic -eq 0 ]; then
-	    echo "  sorting json output for nondeterministic object"
-	    for f in $tmp1 $tmp2 $tmp3 $tmp4; do
-		sort $f | sed 's/,$//' > $f.new
-		mv $f.new $f
-	    done
-	fi
-
-	if ! cmp $tmp1 $tmp2; then
-	    echo "**** $type test $n dump_json check failed ****"
-	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
-	    echo "   ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2"
-	    diff $tmp1 $tmp2
-	    failed=$(($failed + 1))
-	fi
-
-	if ! cmp $tmp1 $tmp3; then
-	    echo "**** $type test $n copy dump_json check failed ****"
-	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
-	    echo "   ceph-dencoder type $type select_test $n copy dump_json > $tmp2"
-	    diff $tmp1 $tmp2
-	    failed=$(($failed + 1))
-	fi
-
-	if ! cmp $tmp1 $tmp4; then
-	    echo "**** $type test $n copy_ctor dump_json check failed ****"
-	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
-	    echo "   ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2"
-	    diff $tmp1 $tmp2
-	    failed=$(($failed + 1))
-	fi
-
-	if [ $deterministic -ne 0 ]; then
-	    run_in_background pids ceph-dencoder type "$type" select_test $n encode export "$tmp1"
-	    run_in_background pids ceph-dencoder type "$type" select_test $n encode decode encode export "$tmp2"
-	    wait_background pids
-
-	    if ! cmp $tmp1 $tmp2; then
-		echo "**** $type test $n binary reencode check failed ****"
-		echo "   ceph-dencoder type $type select_test $n encode export $tmp1"
-		echo "   ceph-dencoder type $type select_test $n encode decode encode export $tmp2"
-		diff <(hexdump -C $tmp1) <(hexdump -C $tmp2)
-		failed=$(($failed + 1))
-	    fi
-	fi
-
-	numtests=$(($numtests + 3))
-    done
+    run_in_background pids test_selected_type $type $results
+    running_jobs=$(($running_jobs + 1))
+    if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then
+        wait_background pids
+        running_jobs=0
+    fi
 done < <(ceph-dencoder list_types)
+wait_background pids
 
-rm -f $tmp1 $tmp2 $tmp3 $tmp4
+numtests=$(cat $results/* 2>/dev/null | grep "numtests: " | awk '{sum += $2} END {print sum}')
+failed=$(cat $results/* 2>/dev/null | grep "failed: " | awk '{sum += $2} END {print sum}')
+rm -rf $results
 
 if [ $failed -gt 0 ]; then
     echo "FAILED $failed / $numtests tests."

From 61d7f8821fa06cf48911230228687abdf747dba2 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Fri, 15 Mar 2024 19:48:33 +0800
Subject: [PATCH 2279/2492] test/crimson/sesastore/test_block: add constructor
 to construct non-fully-loaded TestBlocks

Before this commit, all TestBlocks are fully-loaded whether they do have
data or not.

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/test/crimson/seastore/test_block.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h
index 3bf119f77260..d605497ae934 100644
--- a/src/test/crimson/seastore/test_block.h
+++ b/src/test/crimson/seastore/test_block.h
@@ -55,6 +55,8 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent {
     : LogicalCachedExtent(std::move(ptr)) {}
   TestBlock(const TestBlock &other)
     : LogicalCachedExtent(other), modified_region(other.modified_region) {}
+  TestBlock(extent_len_t length)
+    : LogicalCachedExtent(length) {}
 
   CachedExtentRef duplicate_for_write(Transaction&) final {
     return CachedExtentRef(new TestBlock(*this));

From c74c1210fbe53f2a7df3191ca150b615888c6566 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 15 Mar 2024 09:37:48 -0400
Subject: [PATCH 2280/2492] qa: change log-whitelist to log-ignorelist

Fixes: https://tracker.ceph.com/issues/64947
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/suites/fs/functional/tasks/snap-schedule.yaml         | 2 +-
 qa/suites/fs/functional/tasks/snap_schedule_snapdir.yaml | 2 +-
 qa/suites/netsplit/ceph.yaml                             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/qa/suites/fs/functional/tasks/snap-schedule.yaml b/qa/suites/fs/functional/tasks/snap-schedule.yaml
index f2e62b050491..26922abeda4b 100644
--- a/qa/suites/fs/functional/tasks/snap-schedule.yaml
+++ b/qa/suites/fs/functional/tasks/snap-schedule.yaml
@@ -6,7 +6,7 @@ overrides:
         debug ms: 1
         debug finisher: 20
         debug client: 20
-    log-whitelist:
+    log-ignorelist:
       - OSD full dropping all updates
       - OSD near full
       - pausewr flag
diff --git a/qa/suites/fs/functional/tasks/snap_schedule_snapdir.yaml b/qa/suites/fs/functional/tasks/snap_schedule_snapdir.yaml
index 7bbcf000fe78..2a175dbf157d 100644
--- a/qa/suites/fs/functional/tasks/snap_schedule_snapdir.yaml
+++ b/qa/suites/fs/functional/tasks/snap_schedule_snapdir.yaml
@@ -6,7 +6,7 @@ overrides:
         debug ms: 1
         debug finisher: 20
         debug client: 20
-    log-whitelist:
+    log-ignorelist:
       - OSD full dropping all updates
       - OSD near full
       - pausewr flag
diff --git a/qa/suites/netsplit/ceph.yaml b/qa/suites/netsplit/ceph.yaml
index ddf54b3a3787..7bdb78c9e73f 100644
--- a/qa/suites/netsplit/ceph.yaml
+++ b/qa/suites/netsplit/ceph.yaml
@@ -11,7 +11,7 @@ overrides:
         mon osdmap full prune interval: 2
         mon osdmap full prune txsize: 2
 # thrashing monitors may make mgr have trouble w/ its keepalive
-    log-whitelist:
+    log-ignorelist:
       - overall HEALTH_
       - \(MGR_DOWN\)
       - \(MON_DOWN\)

From 30d97250f945e0afdfe9ca52a97a42922091d4dd Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Mar 2024 18:35:40 -0400
Subject: [PATCH 2281/2492] doc/radosgw: document the notification_v2 zone
 feature

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 doc/radosgw/notifications.rst |  4 ++++
 doc/radosgw/zone-features.rst | 16 ++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
index 655f3bbc39e3..aa9a55ef966e 100644
--- a/doc/radosgw/notifications.rst
+++ b/doc/radosgw/notifications.rst
@@ -4,6 +4,10 @@ Bucket Notifications
 
 .. versionadded:: Nautilus
 
+.. versionchanged:: Squid
+   A new "v2" format for Topic and Notification metadata can be enabled with
+   the :ref:`feature_notification_v2` zone feature.
+
 .. contents::
 
 Bucket notifications provide a mechanism for sending information out of radosgw
diff --git a/doc/radosgw/zone-features.rst b/doc/radosgw/zone-features.rst
index b439fd9dfd26..5b5986527ab6 100644
--- a/doc/radosgw/zone-features.rst
+++ b/doc/radosgw/zone-features.rst
@@ -16,6 +16,8 @@ Supported Features
 +-----------------------------------+---------+----------+
 | :ref:`feature_compress_encrypted` | Reef    | Disabled |
 +-----------------------------------+---------+----------+
+| :ref:`feature_notification_v2`    | Squid   | Enabled  |
++-----------------------------------+---------+----------+
 
 .. _feature_resharding:
 
@@ -49,6 +51,20 @@ must upgrade to Reef or later before enabling.
    the same data. Due to these security considerations, this feature is disabled
    by default.
 
+
+.. _feature_notification_v2:
+
+notification_v2
+~~~~~~~~~~~~~~~
+
+This feature opts in to a new "v2" metadata format for bucket notifications and
+topics. Unlike "v1", this format is supported by multisite replication and can
+scale to many topics.
+
+Once this feature is enabled on all zonegroups in the realm, a background process
+will convert existing v1 topics and bucket notifications into their v2 format.
+
+
 Commands
 --------
 

From 4f0ff0f80ba0ef6273c31345e93244ccc37cfa69 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 13 Mar 2024 18:47:35 -0400
Subject: [PATCH 2282/2492] PendingReleaseNotes: announce the notification_v2
 feature and its migration

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 PendingReleaseNotes | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index 801546c0b577..6b5c33500bc6 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -13,6 +13,16 @@
 * Tracing: The blkin tracing feature (see https://docs.ceph.com/en/reef/dev/blkin/)
   is now deprecated in favor of Opentracing (https://docs.ceph.com/en/reef/dev/developer_guide/jaegertracing/)
   and will be removed in a later release.
+* RGW: Introducing a new data layout for the Topic metadata associated with S3
+  Bucket Notifications, where each Topic is stored as a separate RADOS object
+  and the bucket notification configuration is stored in a bucket attribute.
+  This new representation supports multisite replication via metadata sync and
+  can scale to many topics. This is on by default for new deployments, but is
+  is not enabled by default on upgrade. Once all radosgws have upgraded (on all
+  zones in a multisite configuration), the ``notification_v2`` zone feature can
+  be enabled to migrate to the new format. See
+  https://docs.ceph.com/en/squid/radosgw/zone-features for details. The "v1"
+  format is now considered deprecated and may be removed after 2 major releases.
 * CEPHFS: MDS evicts clients which are not advancing their request tids which causes
   a large buildup of session metadata resulting in the MDS going read-only due to
   the RADOS operation exceeding the size threshold. `mds_session_metadata_threshold`

From f11cdb1e1820cc8febbf227532efa11696916e2c Mon Sep 17 00:00:00 2001
From: Matt Benjamin <mbenjamin@redhat.com>
Date: Wed, 13 Mar 2024 20:19:01 -0400
Subject: [PATCH 2283/2492] rgw_file: fix mv/rename cases broken by zipper
 integration

There were two problems.  First, leaf object names must be
expressed as fully-qualified to the bucket as input to the
copy-object step.  Second, handle s->object in the same step
indicates the being-created destination object of the copy,
this was correct in the original zipper change but broken
later.

* add a rename/mv unit test

Tests for the following cases added:

1. move between two sub-directory paths in a single bucket
2. move between two names at the top level of a single bucket
3. move between sub-directory paths in different buckets (cross-bucket rename)

Fixes: https://tracker.ceph.com/issues/64950

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 qa/workunits/rgw/test_librgw_file.sh |   4 +
 src/rgw/rgw_file_int.h               |  12 +-
 src/rgw/rgw_op.cc                    |   3 +-
 src/test/CMakeLists.txt              |  15 ++
 src/test/librgw_file_rename.cc       | 302 +++++++++++++++++++++++++++
 5 files changed, 330 insertions(+), 6 deletions(-)
 create mode 100644 src/test/librgw_file_rename.cc

diff --git a/qa/workunits/rgw/test_librgw_file.sh b/qa/workunits/rgw/test_librgw_file.sh
index 2fa4c7f2ae18..8a0f952ad63c 100755
--- a/qa/workunits/rgw/test_librgw_file.sh
+++ b/qa/workunits/rgw/test_librgw_file.sh
@@ -65,4 +65,8 @@ ceph_test_librgw_file_gp ${K} --get --stat --put --create
 echo "phase 5.2"
 ceph_test_librgw_file_gp ${K} --delete
 
+# rename tests
+echo "phase 6.1"
+ceph_test_librgw_file_rename ${K} --create
+
 exit 0
diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h
index d50846a255cb..d61ef33c8a88 100644
--- a/src/rgw/rgw_file_int.h
+++ b/src/rgw/rgw_file_int.h
@@ -2574,6 +2574,8 @@ class RGWCopyObjRequest : public RGWLibRequest,
   RGWFileHandle* dst_parent;
   const std::string& src_name;
   const std::string& dst_name;
+  std::string src_obj_name;
+  std::string dst_obj_name;
 
   RGWCopyObjRequest(CephContext* _cct, std::unique_ptr<rgw::sal::User> _user,
 		    RGWFileHandle* _src_parent, RGWFileHandle* _dst_parent,
@@ -2607,13 +2609,14 @@ class RGWCopyObjRequest : public RGWLibRequest,
     state->src_bucket_name = src_parent->bucket_name();
     state->bucket_name = dst_parent->bucket_name();
 
-    std::string dest_obj_name = dst_parent->format_child_name(dst_name, false);
+    src_obj_name = src_parent->format_child_name(src_name, false /* is_dir */);
+    dst_obj_name = dst_parent->format_child_name(dst_name, false);
 
-    int rc = valid_s3_object_name(dest_obj_name);
+    int rc = valid_s3_object_name(dst_obj_name);
     if (rc != 0)
       return rc;
 
-    state->object = RGWHandler::driver->get_object(rgw_obj_key(dest_obj_name));
+    state->object = RGWHandler::driver->get_object(rgw_obj_key(dst_obj_name));
 
     /* XXX and fixup key attr (could optimize w/string ref and
      * dest_obj_name) */
@@ -2638,9 +2641,8 @@ class RGWCopyObjRequest : public RGWLibRequest,
     /* we don't have (any) headers, so just create default ACLs */
     dest_policy.create_default(s->owner.id, s->owner.display_name);
     /* src_object required before RGWCopyObj::verify_permissions() */
-    rgw_obj_key k = rgw_obj_key(src_name);
+    rgw_obj_key k = rgw_obj_key(src_obj_name);
     s->src_object = s->bucket->get_object(k);
-    s->object = s->src_object->clone(); // needed to avoid trap at rgw_op.cc:5150
     return 0;
   }
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index b15c22accbf6..92b01e08c0bb 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -5497,7 +5497,8 @@ int RGWCopyObj::verify_permission(optional_yield y)
     rgw_placement_rule src_placement;
 
     /* check source object permissions */
-    op_ret = read_obj_policy(this, driver, s, src_bucket->get_info(), src_bucket->get_attrs(), src_acl, &src_placement.storage_class,
+    op_ret = read_obj_policy(this, driver, s, src_bucket->get_info(),
+			     src_bucket->get_attrs(), src_acl, &src_placement.storage_class,
 			     src_policy, src_bucket.get(), s->src_object.get(), y);
     if (op_ret < 0) {
       return op_ret;
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index ecd7dddda8d3..63598971bbdb 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -287,6 +287,7 @@ install(TARGETS ceph_test_librgw_file DESTINATION ${CMAKE_INSTALL_BINDIR})
 add_dependencies(ceph_test_librgw_file
   ceph_test_librgw_file_cd
   ceph_test_librgw_file_gp
+  ceph_test_librgw_file_rename
   ceph_test_librgw_file_nfsns
   ceph_test_librgw_file_aw
   ceph_test_librgw_file_marker)
@@ -391,6 +392,20 @@ target_link_libraries(ceph_test_librgw_file_xattr
   )
 target_link_libraries(ceph_test_librgw_file_xattr spawn)
 
+# ceph_test_librgw_file_rename (mv/rename tests)
+add_executable(ceph_test_librgw_file_rename
+  librgw_file_rename.cc
+  )
+target_link_libraries(ceph_test_librgw_file_rename
+  rgw
+  librados
+  ceph-common
+  ${UNITTEST_LIBS}
+  ${EXTRALIBS}
+  ${ALLOC_LIBS}
+  )
+install(TARGETS ceph_test_librgw_file_rename DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 # ceph_test_rgw_token
 add_executable(ceph_test_rgw_token
   test_rgw_token.cc
diff --git a/src/test/librgw_file_rename.cc b/src/test/librgw_file_rename.cc
new file mode 100644
index 000000000000..b4a5c758cbec
--- /dev/null
+++ b/src/test/librgw_file_rename.cc
@@ -0,0 +1,302 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <stdint.h>
+#include <tuple>
+#include <iostream>
+
+#include "include/rados/librgw.h"
+#include "include/rados/rgw_file.h"
+
+#include "gtest/gtest.h"
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+namespace {
+  librgw_t rgw = nullptr;
+  string userid("testuser");
+  string access_key("");
+  string secret_key("");
+  struct rgw_fs *fs = nullptr;
+
+  uint32_t owner_uid = 867;
+  uint32_t owner_gid = 5309;
+  uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;
+
+  bool do_create = false;
+  bool do_delete = false;
+
+  string bucket1_name = "wyndemere";
+  string bucket2_name = "galahad";
+  string obj_name1 = "tommy1";
+  string obj_name2 = "ricky1";
+  string obj_name3 = "zoot";
+
+  struct rgw_file_handle* bucket1_fh = nullptr;
+  struct rgw_file_handle* bucket2_fh = nullptr;
+  struct rgw_file_handle* object_fh = nullptr;
+
+  string subdir1_name = "meep";
+  struct rgw_file_handle* subdir1_fh;
+
+  string subdir2_name = "mork";
+  struct rgw_file_handle* subdir2_fh;
+  
+  struct {
+    int argc;
+    char **argv;
+  } saved_args;
+}
+
+TEST(LibRGW, INIT) {
+  int ret = librgw_create(&rgw, saved_args.argc, saved_args.argv);
+  ASSERT_EQ(ret, 0);
+  ASSERT_NE(rgw, nullptr);
+}
+
+TEST(LibRGW, MOUNT) {
+  int ret = rgw_mount2(rgw, userid.c_str(), access_key.c_str(),
+                       secret_key.c_str(), "/", &fs, RGW_MOUNT_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  ASSERT_NE(fs, nullptr);
+}
+
+TEST(LibRGW, CREATE_BUCKETS) {
+  if (do_create) {
+    struct stat st;
+    int ret{0};
+
+    st.st_uid = owner_uid;
+    st.st_gid = owner_gid;
+    st.st_mode = 755;
+
+    ret = rgw_mkdir(fs, fs->root_fh, bucket1_name.c_str(), &st, create_mask,
+			&bucket1_fh, RGW_MKDIR_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+    ret = rgw_fh_rele(fs, bucket1_fh, 0 /* flags */);
+    ASSERT_EQ(ret, 0);
+
+    ret = rgw_mkdir(fs, fs->root_fh, bucket2_name.c_str(), &st, create_mask,
+		    &bucket2_fh, RGW_MKDIR_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+    ret = rgw_fh_rele(fs, bucket2_fh, 0 /* flags */);
+    ASSERT_EQ(ret, 0); 
+  }
+}
+
+TEST(LibRGW, LOOKUP_BUCKETS) {
+  int ret{0};
+  ret = rgw_lookup(fs, fs->root_fh, bucket1_name.c_str(), &bucket1_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+
+  ret = rgw_lookup(fs, fs->root_fh, bucket2_name.c_str(), &bucket2_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+static inline
+int make_object(struct rgw_file_handle* parent_fh, const string& name) {
+  int ret{0};
+  ret = rgw_lookup(fs, parent_fh, name.c_str(), &object_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
+  ret = rgw_open(fs, object_fh, 0 /* posix flags */, 0 /* flags */);
+
+  size_t nbytes;
+  string data = "hi mom";
+  ret = rgw_write(fs, object_fh, 0, data.length(), &nbytes,
+		  (void*) data.c_str(), RGW_WRITE_FLAG_NONE);
+  /* commit write transaction */
+  ret = rgw_close(fs, object_fh, 0 /* flags */);
+  return ret;
+}
+
+TEST(LibRGW, TOPDIR_RENAME) {
+  /* rename a file directly residing at the bucket */
+  int ret{0};
+
+  ret = make_object(bucket1_fh, obj_name1);
+  ASSERT_EQ(ret, 0);
+
+  /* now move it */
+  ret = rgw_rename(fs,
+		   bucket1_fh, obj_name1.c_str(),
+		   bucket1_fh, obj_name2.c_str(),
+		   0 /* flags */);
+  ASSERT_EQ(ret, 0);
+
+  /* now check the result */
+  struct rgw_file_handle* name2_fh;
+  ret = rgw_lookup(fs, bucket1_fh, obj_name2.c_str(), &name2_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  /* release file handle */
+  ret = rgw_fh_rele(fs, name2_fh, 0 /* flags */);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, SUBDIR_RENAME) {
+  int ret{0};
+
+  if (do_create) {
+      struct stat st;
+
+      st.st_uid = owner_uid;
+      st.st_gid = owner_gid;
+      st.st_mode = 755;
+
+      ret = rgw_mkdir(fs, bucket1_fh, subdir1_name.c_str(), &st, create_mask,
+		      &subdir1_fh, RGW_MKDIR_FLAG_NONE);
+      ASSERT_EQ(ret, 0);
+
+      ret = make_object(subdir1_fh, obj_name1);
+      ASSERT_EQ(ret, 0);
+  } else {
+    ret = rgw_lookup(fs, bucket1_fh, subdir1_name.c_str(), &subdir1_fh,
+		     nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+
+  /* now move it */
+  ret = rgw_rename(fs,
+		   subdir1_fh, obj_name1.c_str(),
+		   subdir1_fh, obj_name2.c_str(),
+		   0 /* flags */);
+  ASSERT_EQ(ret, 0);
+  /* now check the result */
+  struct rgw_file_handle* name2_fh;
+  ret = rgw_lookup(fs, subdir1_fh, obj_name2.c_str(), &name2_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  /* release file handle */
+  ret = rgw_fh_rele(fs, name2_fh, 0 /* flags */);
+  ASSERT_EQ(ret, 0);
+  /* we'll re-use subdir1_fh */
+}
+
+TEST(LibRGW, CROSS_BUCKET_RENAME) {
+  /* rename a file across bucket boundaries */
+  int ret{0};
+
+  if (do_create) {
+      struct stat st;
+
+      st.st_uid = owner_uid;
+      st.st_gid = owner_gid;
+      st.st_mode = 755;
+
+      ret = rgw_mkdir(fs, bucket2_fh, subdir2_name.c_str(), &st, create_mask,
+		      &subdir2_fh, RGW_MKDIR_FLAG_NONE); // galahad/mork
+      ASSERT_EQ(ret, 0);
+
+      ret = make_object(subdir1_fh, obj_name1); // wyndemere/meep/tommy1
+      ASSERT_EQ(ret, 0);
+  } else {
+    ret = rgw_lookup(fs, bucket2_fh, subdir2_name.c_str(), &subdir2_fh,
+		     nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+    ASSERT_EQ(ret, 0);
+  }
+
+  /* now move it -- subdir2 is directory mork in bucket galahad */
+  ret = rgw_rename(fs,
+		   subdir1_fh, obj_name1.c_str(),
+		   subdir2_fh, obj_name3.c_str(),
+		   0 /* flags */);
+  ASSERT_EQ(ret, 0);
+  /* now check the result */
+  struct rgw_file_handle* name3_fh; // galahad/mork/zoot
+  ret = rgw_lookup(fs, subdir2_fh, obj_name3.c_str(), &name3_fh,
+		   nullptr, 0, RGW_LOOKUP_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+  /* release file handle */
+  ret = rgw_fh_rele(fs, name3_fh, 0 /* flags */);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, CLEANUP) {
+  // do nothing
+}
+
+TEST(LibRGW, UMOUNT) {
+  if (! fs)
+    return;
+
+  int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
+  ASSERT_EQ(ret, 0);
+}
+
+TEST(LibRGW, SHUTDOWN) {
+  librgw_shutdown(rgw);
+}
+
+int main(int argc, char *argv[])
+{
+  auto args = argv_to_vec(argc, argv);
+  env_to_vec(args);
+
+  char* v = getenv("AWS_ACCESS_KEY_ID");
+  if (v) {
+    access_key = v;
+  }
+
+  v = getenv("AWS_SECRET_ACCESS_KEY");
+  if (v) {
+    secret_key = v;
+  }
+
+  string val;
+  for (auto arg_iter = args.begin(); arg_iter != args.end();) {
+    if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
+			      (char*) nullptr)) {
+      access_key = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
+				     (char*) nullptr)) {
+      secret_key = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--userid",
+				     (char*) nullptr)) {
+      userid = val;
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
+				     (char*) nullptr)) {
+      owner_uid = std::stoi(val);
+    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--gid",
+				     (char*) nullptr)) {
+      owner_gid = std::stoi(val);
+    } else if (ceph_argparse_flag(args, arg_iter, "--create",
+					    (char*) nullptr)) {
+      do_create = true;
+    } else if (ceph_argparse_flag(args, arg_iter, "--delete",
+					    (char*) nullptr)) {
+      do_delete = true;
+    } else {
+      ++arg_iter;
+    }
+  }
+
+  /* don't accidentally run as anonymous */
+  if ((access_key == "") ||
+      (secret_key == "")) {
+    std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
+    return EPERM;
+  }
+
+  saved_args.argc = argc;
+  saved_args.argv = argv;
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

From 64803e1ced57d64b758927c3977bb4a4d1769180 Mon Sep 17 00:00:00 2001
From: Joshua Baergen <jbaergen@digitalocean.com>
Date: Tue, 12 Sep 2023 14:05:01 -0400
Subject: [PATCH 2284/2492] rgw: Add missing empty checks to the split string
 in is_string_in_set().

In certain cases, where a user misconfigures a CORS rule, the entirety
of the string can be token characters (or, at least, the string before
and after a given token is all token characters), but != "*". If the
misconfigured string includes "*" we'll try to split the string and we
assume that we can pop the list of string elements when "*" isn't
first/last, but get_str_list() won't return anything for token-only
substrings and thus 'ssplit' will have fewer elements than would be
expected for a correct rule. In the case of an empty list, front() has
undefined behaviour; in our experience, it often results in a huge
allocation attempt because the code tries to copy the string into a
local variable 'sl'.

An example of this misconfiguration (and thus a reproduction case) is
configuring an origin of " *".

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
---
 src/rgw/rgw_cors.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/rgw/rgw_cors.cc b/src/rgw/rgw_cors.cc
index e41abf8ccb44..bb80e2b58db2 100644
--- a/src/rgw/rgw_cors.cc
+++ b/src/rgw/rgw_cors.cc
@@ -121,6 +121,8 @@ static bool is_string_in_set(set<string>& s, string h) {
       
       get_str_list((*it), "* \t", ssplit);
       if (off != 0) {
+        if (ssplit.empty())
+          continue;
         string sl = ssplit.front();
         flen = sl.length();
         dout(10) << "Finding " << sl << ", in " << h << ", at offset 0" << dendl;
@@ -129,6 +131,8 @@ static bool is_string_in_set(set<string>& s, string h) {
         ssplit.pop_front();
       }
       if (off != ((*it).length() - 1)) {
+        if (ssplit.empty())
+          continue;
         string sl = ssplit.front();
         dout(10) << "Finding " << sl << ", in " << h 
           << ", at offset not less than " << flen << dendl;

From d7ccf26983c41344a12f33b2a30fc79b65cc548f Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@ibm.com>
Date: Fri, 15 Mar 2024 14:20:29 +0000
Subject: [PATCH 2285/2492] node-proxy: support more Location value formats

After some tests, it turns out that depending on the hardware,
the header 'Location' which is returned by the server after logged can be different.
I could notice the following:

either:

Location: scheme://address:port/redfish/v1/SessionService/Session

or

Location: /redfish/v1/SessionService/Session

a previous tracker [1] was opened because I thought only the first one existed, which is wrong.

[1] https://tracker.ceph.com/issues/64894

Signed-off-by: Guillaume Abrioux <gabrioux@ibm.com>
---
 src/ceph-node-proxy/ceph_node_proxy/redfish_client.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
index d75c3db1e218..64a4e44dfe3d 100644
--- a/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
+++ b/src/ceph-node-proxy/ceph_node_proxy/redfish_client.py
@@ -30,6 +30,7 @@ def login(self) -> None:
             oob_credentials = json.dumps({'UserName': self.username,
                                           'Password': self.password})
             headers = {'Content-Type': 'application/json'}
+            location_endpoint: str = ''
 
             try:
                 _headers, _data, _status_code = self.query(data=oob_credentials,
@@ -43,8 +44,14 @@ def login(self) -> None:
                 self.log.error(msg)
                 raise RuntimeError
             self.token = _headers['X-Auth-Token']
-            location_endpoint: str = _headers['Location'].split('/', 3)[-1:][0]
-            self.location = f'/{location_endpoint}'
+            if _headers['Location'].startswith('http'):
+                # We assume the value has the following format:
+                # scheme://address:port/redfish/v1/SessionService/Session
+                location_endpoint = f"/{_headers['Location'].split('/', 3)[-1:][0]}"
+            else:
+                location_endpoint = _headers['Location']
+            self.location = location_endpoint
+            self.log.info(f'Logged in to {self.url}, Received header "Location": {self.location}')
 
     def is_logged_in(self) -> bool:
         self.log.debug(f'Checking token validity for {self.url}')

From f9c280ce593fdde2914a93262fb3d9bc525f7947 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 11 Mar 2024 14:44:17 -0400
Subject: [PATCH 2286/2492] cephadm: allow list_daemons for only a specific
 daemon

At the moment, my thoughts are to use this internally
in the binary for when we need infor from list_daemons
but only for a specific daemon. I could also see wanting
this just on the command line to get info on a certain
daemon, so I've added it as a flag for `cephadm ls` as well

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index 21523cd09ea0..b3ff564ec2f4 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -3378,12 +3378,17 @@ def serialize_sets(obj: Any) -> Any:
 def command_ls(ctx):
     # type: (CephadmContext) -> None
     ls = list_daemons(ctx, detail=not ctx.no_detail,
-                      legacy_dir=ctx.legacy_dir)
+                      legacy_dir=ctx.legacy_dir,
+                      daemon_name=ctx.name)
     print(json.dumps(ls, indent=4))
 
 
-def list_daemons(ctx, detail=True, legacy_dir=None):
-    # type: (CephadmContext, bool, Optional[str]) -> List[Dict[str, str]]
+def list_daemons(
+    ctx: CephadmContext,
+    detail: bool = True,
+    legacy_dir: Optional[str] = None,
+    daemon_name: Optional[str] = None,
+) -> List[Dict[str, str]]:
     host_version: Optional[str] = None
     ls = []
     container_path = ctx.container_engine.path
@@ -3452,6 +3457,8 @@ def list_daemons(ctx, detail=True, legacy_dir=None):
                 for j in os.listdir(os.path.join(data_dir, i)):
                     if '.' in j and os.path.isdir(os.path.join(data_dir, fsid, j)):
                         name = j
+                        if daemon_name and name != daemon_name:
+                            continue
                         (daemon_type, daemon_id) = j.split('.', 1)
                         unit_name = get_unit_name(fsid,
                                                   daemon_type,
@@ -4880,6 +4887,9 @@ def _get_parser():
         '--legacy-dir',
         default='/',
         help='base directory for legacy daemon data')
+    parser_ls.add_argument(
+        '--name', '-n',
+        help='Only get data for specific daemon. Format of daemon name: (type.id)')
 
     parser_list_networks = subparsers.add_parser(
         'list-networks', help='list IP networks')

From dc5e1ea20c31e1089966ab89370d82e24d138524 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Tue, 12 Mar 2024 10:26:18 -0400
Subject: [PATCH 2287/2492] cephadm: fix `cephadm shell --name <daemon-name>`
 for stopped/failed daemon

This previously would always try to use 'podman
inspect' on the running container of the daemon,
but this doesn't work if the daemon is stopped
or failed. Doing this for stopped/failed daemons
is a valid use case as we recommend cephadm shell
with --name for running debugging tools (often
for OSDs)

Fixes: https://tracker.ceph.com/issues/64879

Signed-off-by: Adam King <adking@redhat.com>
---
 src/cephadm/cephadm.py            | 76 ++++++++++++++++++++++++++++---
 src/cephadm/tests/test_cephadm.py | 75 ++++++++++++++++++++++++++++++
 2 files changed, 144 insertions(+), 7 deletions(-)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index b3ff564ec2f4..3b2157cdc322 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -483,17 +483,66 @@ def daemon_name_or_type(daemon: Dict[str, str]) -> str:
     if by_name and '.' not in daemon_filter:
         logger.warning(f'Trying to get container info using invalid daemon name {daemon_filter}')
         return None
-    daemons = list_daemons(ctx, detail=False)
-    matching_daemons = [d for d in daemons if daemon_name_or_type(d) == daemon_filter and d['fsid'] == ctx.fsid]
+    if by_name:
+        matching_daemons = _get_matching_daemons_by_name(ctx, daemon_filter)
+    else:
+        # NOTE: we are passing detail=False here as in this case where we are not
+        # doing it by_name, we really only need the names of the daemons. Additionally,
+        # when not doing it by_name, we are getting the info for all daemons on the
+        # host, and doing this with detail=True tends to be slow.
+        daemons = list_daemons(ctx, detail=False)
+        matching_daemons = [d for d in daemons if daemon_name_or_type(d) == daemon_filter and d['fsid'] == ctx.fsid]
     if matching_daemons:
-        d_type, d_id = matching_daemons[0]['name'].split('.', 1)
-        out, _, code = get_container_stats(ctx, ctx.container_engine.path, ctx.fsid, d_type, d_id)
-        if not code:
-            (container_id, image_name, image_id, start, version) = out.strip().split(',')
-            return ContainerInfo(container_id, image_name, image_id, start, version)
+        if (
+            by_name
+            and 'state' in matching_daemons[0]
+            and matching_daemons[0]['state'] != 'running'
+            and 'container_image_name' in matching_daemons[0]
+            and matching_daemons[0]['container_image_name']
+        ):
+            # this daemon contianer is not running so the regular `podman/docker inspect` on the
+            # container will not help us. If we have the image name from the list_daemons output
+            # we can try that.
+            image_name = matching_daemons[0]['container_image_name']
+            out, _, code = get_container_stats_by_image_name(ctx, ctx.container_engine.path, image_name)
+            if not code:
+                # keep in mind, the daemon container is not running, so no container id here
+                (image_id, start, version) = out.strip().split(',')
+                return ContainerInfo(
+                    container_id='',
+                    image_name=image_name,
+                    image_id=image_id,
+                    start=start,
+                    version=version)
+        else:
+            d_type, d_id = matching_daemons[0]['name'].split('.', 1)
+            out, _, code = get_container_stats(ctx, ctx.container_engine.path, ctx.fsid, d_type, d_id)
+            if not code:
+                (container_id, image_name, image_id, start, version) = out.strip().split(',')
+                return ContainerInfo(container_id, image_name, image_id, start, version)
     return None
 
 
+def _get_matching_daemons_by_name(ctx: CephadmContext, daemon_filter: str) -> List[Dict[str, str]]:
+    # NOTE: we are not passing detail=False to this list_daemons call
+    # as we want the container_image name in the case where we are
+    # doing this by name and this is skipped when detail=False
+    matching_daemons = list_daemons(ctx, daemon_name=daemon_filter)
+    if len(matching_daemons) > 1:
+        logger.warning(f'Found multiple daemons sharing same name: {daemon_filter}')
+        # Take the first daemon we find that is actually running, or just the
+        # first in the list if none are running
+        matched_daemon = None
+        for d in matching_daemons:
+            if 'state' in d and d['state'] == 'running':
+                matched_daemon = d
+                break
+        if not matched_daemon:
+            matched_daemon = matching_daemons[0]
+        matching_daemons = [matched_daemon]
+    return matching_daemons
+
+
 def infer_local_ceph_image(ctx: CephadmContext, container_path: str) -> Optional[str]:
     """
      Infer the local ceph image based on the following priority criteria:
@@ -3656,6 +3705,7 @@ def get_daemon_description(ctx, fsid, name, detail=False, legacy_dir=None):
 
 
 def get_container_stats(ctx: CephadmContext, container_path: str, fsid: str, daemon_type: str, daemon_id: str) -> Tuple[str, str, int]:
+    """returns container id, image name, image id, created time, and ceph version if available"""
     c = CephContainer.for_daemon(
         ctx, DaemonIdentity(fsid, daemon_type, daemon_id), 'bash'
     )
@@ -3671,6 +3721,18 @@ def get_container_stats(ctx: CephadmContext, container_path: str, fsid: str, dae
             break
     return out, err, code
 
+
+def get_container_stats_by_image_name(ctx: CephadmContext, container_path: str, image_name: str) -> Tuple[str, str, int]:
+    """returns image id, created time, and ceph version if available"""
+    out, err, code = '', '', -1
+    cmd = [
+        container_path, 'image', 'inspect',
+        '--format', '{{.Id}},{{.Created}},{{index .Config.Labels "io.ceph.version"}}',
+        image_name
+    ]
+    out, err, code = call(ctx, cmd, verbosity=CallVerbosity.QUIET)
+    return out, err, code
+
 ##################################
 
 
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 9c6caf0092b7..59d360b15b84 100644
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -1,5 +1,6 @@
 # type: ignore
 
+import copy
 import errno
 import json
 import mock
@@ -784,6 +785,80 @@ def test_get_container_info(self, _logger, daemon_filter, by_name, daemon_list,
             with mock.patch('cephadm.get_container_stats', return_value=container_stats):
                 assert _cephadm.get_container_info(ctx, daemon_filter, by_name) == output
 
+    @mock.patch('cephadm.list_daemons')
+    @mock.patch('cephadm.get_container_stats')
+    @mock.patch('cephadm.get_container_stats_by_image_name')
+    def test_get_container_info_daemon_down(self, _get_stats_by_name, _get_stats, _list_daemons):
+        ctx = _cephadm.CephadmContext()
+        ctx.fsid = '5e39c134-dfc5-11ee-a344-5254000ee071'
+        ctx.container_engine = mock_podman()
+
+        # list_daemons output taken from cephadm ls of an
+        # OSD that was stopped, with subsititutions
+        # true -> True
+        # null -> None
+        down_osd_json = {
+                "style": "cephadm:v1",
+                "name": "osd.2",
+                "fsid": "5e39c134-dfc5-11ee-a344-5254000ee071",
+                "systemd_unit": "ceph-5e39c134-dfc5-11ee-a344-5254000ee071@osd.2",
+                "enabled": True,
+                "state": "stopped",
+                "service_name": "osd.foo",
+                "ports": [],
+                "ip": None,
+                "deployed_by": [
+                    "quay.io/adk3798/ceph@sha256:7da0af22ce45aac97dff00125af590506d8e36ab97d78e5175149643562bfb0b"
+                ],
+                "rank": None,
+                "rank_generation": None,
+                "extra_container_args": None,
+                "extra_entrypoint_args": None,
+                "memory_request": None,
+                "memory_limit": None,
+                "container_id": None,
+                "container_image_name": "quay.io/adk3798/ceph@sha256:7da0af22ce45aac97dff00125af590506d8e36ab97d78e5175149643562bfb0b",
+                "container_image_id": None,
+                "container_image_digests": None,
+                "version": None,
+                "started": None,
+                "created": "2024-03-11T17:17:49.533757Z",
+                "deployed": "2024-03-11T17:37:23.520061Z",
+                "configured": "2024-03-11T17:37:28.494075Z"
+        }
+        _list_daemons.return_value = [down_osd_json]
+        _get_stats_by_name.return_value = (('a03c201ff4080204949932f367545cd381c4acee0d48dbc15f2eac1e35f22318,'
+                                   '2023-11-28 21:34:38.045413692 +0000 UTC,'),
+                                   '', 0)
+
+        expected_container_info = _cephadm.ContainerInfo(
+            container_id='',
+            image_name='quay.io/adk3798/ceph@sha256:7da0af22ce45aac97dff00125af590506d8e36ab97d78e5175149643562bfb0b',
+            image_id='a03c201ff4080204949932f367545cd381c4acee0d48dbc15f2eac1e35f22318',
+            start='2023-11-28 21:34:38.045413692 +0000 UTC',
+            version='')
+
+        assert _cephadm.get_container_info(ctx, 'osd.2', by_name=True) == expected_container_info
+        assert not _get_stats.called, 'only get_container_stats_by_image_name should have been called'
+
+        # If there is one down and one up daemon of the same name, it should use the up one
+        # In this case, we would be using the running container to get the image, so
+        # all the info will come from the return value of get_container_stats, rather
+        # than it partially being taken from the list_daemons output
+        up_osd_json = copy.deepcopy(down_osd_json)
+        up_osd_json['state'] = 'running'
+        _get_stats.return_value = (('container_id,image_name,image_id,the_past,'), '', 0)
+        _list_daemons.return_value = [down_osd_json, up_osd_json]
+
+        expected_container_info = _cephadm.ContainerInfo(
+            container_id='container_id',
+            image_name='image_name',
+            image_id='image_id',
+            start='the_past',
+            version='')
+
+        assert _cephadm.get_container_info(ctx, 'osd.2', by_name=True) == expected_container_info
+
     def test_should_log_to_journald(self):
         from cephadmlib import context_getters
 

From 129c10b4da518b8843ac0bc5397276eb695d4033 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Wed, 13 Mar 2024 15:30:25 -0400
Subject: [PATCH 2288/2492] mgr/cephadm: refresh public_network for config
 checks before checking

The place it was being run before meant it would only grab the
public_network setting once at startup of the module. This meant
if a user changed the setting, which they are likely to do if they
get the warning, cephadm would ignore the change and continue
reporting that the hosts don't match up with the old setting
for the public_network. This moves the call to refresh the
setting to right before we actually run the checks. It does
mean we'll do the `ceph config dump --format json` call
each serve loop iteration, but I've found that only tends
to take a few milliseconds, which is nothing compared to
the time to refresh other things we check during the serve
loop.

I additionally modified the use of this option to use
the attribute on the mgr, rather than calling
`get_module_option`. This was just to get it more in
line with how we tend to handle other config options

Fixes: https://tracker.ceph.com/issues/64902

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/configchecks.py            | 2 +-
 src/pybind/mgr/cephadm/module.py                  | 3 ++-
 src/pybind/mgr/cephadm/serve.py                   | 5 +++--
 src/pybind/mgr/cephadm/tests/test_configchecks.py | 5 ++---
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/cephadm/configchecks.py b/src/pybind/mgr/cephadm/configchecks.py
index 38cde7806afb..6e442c4a3ce8 100644
--- a/src/pybind/mgr/cephadm/configchecks.py
+++ b/src/pybind/mgr/cephadm/configchecks.py
@@ -674,7 +674,7 @@ def _process_hosts(self) -> None:
             self.host_to_role[hostname] = list(self.mgr.cache.get_daemon_types(hostname))
 
     def run_checks(self) -> None:
-        checks_enabled = self.mgr.get_module_option('config_checks_enabled')
+        checks_enabled = self.mgr.config_checks_enabled
         if checks_enabled is not True:
             return
 
diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 467950afea33..538f7606bb71 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -560,6 +560,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.registry_password: Optional[str] = None
             self.registry_insecure: bool = False
             self.use_repo_digest = True
+            self.config_checks_enabled = False
             self.default_registry = ''
             self.autotune_memory_target_ratio = 0.0
             self.autotune_interval = 0
@@ -1371,7 +1372,7 @@ def _config_checks_list(self, format: Format = Format.plain) -> HandleCommandRes
     @orchestrator._cli_read_command('cephadm config-check status')
     def _config_check_status(self) -> HandleCommandResult:
         """Show whether the configuration checker feature is enabled/disabled"""
-        status = self.get_module_option('config_checks_enabled')
+        status = self.config_checks_enabled
         return HandleCommandResult(stdout="Enabled" if status else "Disabled")
 
     @orchestrator._cli_write_command('cephadm config-check enable')
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 27775087d05d..6bd3c5222c1e 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -67,7 +67,6 @@ def serve(self) -> None:
         of cephadm. This loop will then attempt to apply this new state.
         """
         self.log.debug("serve starting")
-        self.mgr.config_checker.load_network_config()
 
         while self.mgr.run:
             self.log.debug("serve loop start")
@@ -322,7 +321,9 @@ def refresh(host: str) -> None:
         self.mgr.agent_helpers._update_agent_down_healthcheck(agents_down)
         self.mgr.http_server.config_update()
 
-        self.mgr.config_checker.run_checks()
+        if self.mgr.config_checks_enabled:
+            self.mgr.config_checker.load_network_config()
+            self.mgr.config_checker.run_checks()
 
         for k in [
                 'CEPHADM_HOST_CHECK_FAILED',
diff --git a/src/pybind/mgr/cephadm/tests/test_configchecks.py b/src/pybind/mgr/cephadm/tests/test_configchecks.py
index 3cae0a27d5b8..ff1e2186109f 100644
--- a/src/pybind/mgr/cephadm/tests/test_configchecks.py
+++ b/src/pybind/mgr/cephadm/tests/test_configchecks.py
@@ -238,6 +238,7 @@ def __init__(self):
         self.default_version = 'quincy'
         self.version_overrides = {}
         self.daemon_to_host = {}
+        self.config_checks_enabled = True
 
         self.cache = HostCache(self)
         self.upgrade = CephadmUpgrade(self)
@@ -623,9 +624,7 @@ def test_skip_release_during_upgrade(self, mgr):
         assert 'ceph_release' in checker.skipped_checks
 
     def test_skip_when_disabled(self, mgr):
-        mgr.module_option.update({
-            "config_checks_enabled": "false"
-        })
+        mgr.config_checks_enabled = False
         checker = CephadmConfigChecks(mgr)
         checker.cluster_network_list = []
         checker.public_network_list = ['10.9.64.0/24']

From b0275929c07ebcfba3e7877020f182c6d8b74435 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 16 Mar 2024 18:34:43 +0000
Subject: [PATCH 2289/2492] mgr/dashboard: bump follow-redirects

Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.3 to 1.15.6.
- [Release notes](https://github.com/follow-redirects/follow-redirects/releases)
- [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.3...v1.15.6)

---
updated-dependencies:
- dependency-name: follow-redirects
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .../mgr/dashboard/frontend/package-lock.json  | 73 +------------------
 1 file changed, 3 insertions(+), 70 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json
index 8e6a9c4018fd..ecb9f0f587dc 100644
--- a/src/pybind/mgr/dashboard/frontend/package-lock.json
+++ b/src/pybind/mgr/dashboard/frontend/package-lock.json
@@ -4315,49 +4315,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/@compodoc/live-server/node_modules/cors": {
-      "version": "2.8.5",
-      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
-      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
-      "dev": true,
-      "dependencies": {
-        "object-assign": "^4",
-        "vary": "^1"
-      },
-      "engines": {
-        "node": ">= 0.10"
-      }
-    },
-    "node_modules/@compodoc/live-server/node_modules/debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-      "dev": true,
-      "dependencies": {
-        "ms": "2.0.0"
-      }
-    },
-    "node_modules/@compodoc/live-server/node_modules/debug/node_modules/ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
-      "dev": true
-    },
-    "node_modules/@compodoc/live-server/node_modules/ms": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-      "dev": true
-    },
-    "node_modules/@compodoc/live-server/node_modules/object-assign": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/@compodoc/live-server/node_modules/open": {
       "version": "8.4.0",
       "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
@@ -4375,30 +4332,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/@compodoc/live-server/node_modules/send": {
-      "version": "0.18.0",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
-      "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
-      "dev": true,
-      "dependencies": {
-        "debug": "2.6.9",
-        "depd": "2.0.0",
-        "destroy": "1.2.0",
-        "encodeurl": "~1.0.2",
-        "escape-html": "~1.0.3",
-        "etag": "~1.8.1",
-        "fresh": "0.5.2",
-        "http-errors": "2.0.0",
-        "mime": "1.6.0",
-        "ms": "2.1.3",
-        "on-finished": "2.4.1",
-        "range-parser": "~1.2.1",
-        "statuses": "2.0.1"
-      },
-      "engines": {
-        "node": ">= 0.8.0"
-      }
-    },
     "node_modules/@compodoc/ngd-core": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/@compodoc/ngd-core/-/ngd-core-2.1.1.tgz",
@@ -15888,9 +15821,9 @@
       "dev": true
     },
     "node_modules/follow-redirects": {
-      "version": "1.15.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz",
-      "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==",
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
       "funding": [
         {
           "type": "individual",

From 233b670704d15c3a69b8d91885b60f3fb101ec58 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Sun, 17 Mar 2024 19:27:11 +1000
Subject: [PATCH 2290/2492] doc/rados: update common.rst

Clarify some overwritten paragraphs in
doc/rados/configuration/common.rst. Cf.
https://github.com/ceph/ceph/pull/50795/files#r1153897231.

Reviewed-by: Anthony D'Atri <anthony.datri@gmail.com>
Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/configuration/common.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/doc/rados/configuration/common.rst b/doc/rados/configuration/common.rst
index 0b373f469b67..c397f4e52acc 100644
--- a/doc/rados/configuration/common.rst
+++ b/doc/rados/configuration/common.rst
@@ -123,11 +123,10 @@ OSD host, run the following commands:
     ssh {osd-host}
     sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
 
-The ``osd_data`` path ought to lead to a mount point that has mounted on it a
-device that is distinct from the device that contains the operating system and
-the daemons. To use a device distinct from the device that contains the
+The ``osd_data`` path must lead to a device that is not shared with the
+operating system. To use a device other than the device that contains the
 operating system and the daemons, prepare it for use with Ceph and mount it on
-the directory you just created by running the following commands:
+the directory you just created by running commands of the following form:
 
 .. prompt:: bash $
 

From a811ef0d813570752aaffcf38dc18cc6778b72a9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 14 Feb 2024 11:35:57 -0500
Subject: [PATCH 2291/2492] mgr/cephadm: make remote command execution
 auditable

Update ssh.py and other code using it to only allow commands wrapped
in particular python types as executables on the remote hosts.
By using a specific type for remote executables we make the code more
auditable, avoiding the possibility of executing arbitrary strings
as commands with sudo. This is all enforced by mypy's type checking.

The result is a list of commands that the cephadm mgr module may
execute on a remote host using sudo:
```
$ git ls-files -z | xargs -0 grep 'RemoteExecutable(' -d skip -h | grep
-v '(str)' | sed -e 's/.*RemoteExecutable(//' -e 's/)//' -e 's/,$//'
'which'
'/usr/bin/cephadm'
python
'chmod'
'ls'
'sysctl'
'chown'
'mkdir'
'mv'
'touch'
'rm'
'true'
```
Note that *python* is special as it is based on the output of which and
may vary from OS to OS. The quoted items are used exactly as named.
Only the binary at `/usr/bin/cephadm` _or_ the dynamically discovered
python3 binary will be used. This depends on a configuration option for
the cephadm module.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/offline_watcher.py     |   5 +-
 src/pybind/mgr/cephadm/serve.py               |  23 +++-
 src/pybind/mgr/cephadm/ssh.py                 | 127 +++++++++++++++---
 .../mgr/cephadm/tests/test_tuned_profiles.py  |  30 ++++-
 src/pybind/mgr/cephadm/tuned_profiles.py      |  11 +-
 5 files changed, 161 insertions(+), 35 deletions(-)

diff --git a/src/pybind/mgr/cephadm/offline_watcher.py b/src/pybind/mgr/cephadm/offline_watcher.py
index 2b7751dfc34d..4aa07e2f584a 100644
--- a/src/pybind/mgr/cephadm/offline_watcher.py
+++ b/src/pybind/mgr/cephadm/offline_watcher.py
@@ -4,6 +4,8 @@
 import multiprocessing as mp
 import threading
 
+from . import ssh
+
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
 
@@ -38,7 +40,8 @@ def run(self) -> None:
     def check_host(self, host: str) -> None:
         if host not in self.mgr.offline_hosts:
             try:
-                self.mgr.ssh.check_execute_command(host, ['true'], log_command=self.mgr.log_refresh_metadata)
+                rcmd = ssh.RemoteCommand(ssh.Executables.TRUE)
+                self.mgr.ssh.check_execute_command(host, rcmd, log_command=self.mgr.log_refresh_metadata)
             except Exception:
                 logger.debug(f'OfflineHostDetector: detected {host} to be offline')
                 # kick serve loop in case corrective action must be taken for offline host
diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py
index 27775087d05d..761174fc6c0f 100644
--- a/src/pybind/mgr/cephadm/serve.py
+++ b/src/pybind/mgr/cephadm/serve.py
@@ -34,6 +34,7 @@
 
 from . import utils
 from . import exchange
+from . import ssh
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -42,6 +43,9 @@
 
 REQUIRES_POST_ACTIONS = ['grafana', 'iscsi', 'prometheus', 'alertmanager', 'rgw', 'nvmeof']
 
+WHICH = ssh.RemoteExecutable('which')
+CEPHADM_EXE = ssh.RemoteExecutable('/usr/bin/cephadm')
+
 
 class CephadmServe:
     """
@@ -1271,7 +1275,7 @@ def _write_client_files(self,
             if path == '/etc/ceph/ceph.conf':
                 continue
             self.log.info(f'Removing {host}:{path}')
-            cmd = ['rm', '-f', path]
+            cmd = ssh.RemoteCommand(ssh.Executables.RM, ['-f', path])
             self.mgr.ssh.check_execute_command(host, cmd)
             updated_files = True
             self.mgr.cache.removed_client_file(host, path)
@@ -1620,15 +1624,24 @@ async def _run_cephadm(self,
             if stdin and 'agent' not in str(entity):
                 self.log.debug('stdin: %s' % stdin)
 
-            cmd = ['which', 'python3']
+            cmd = ssh.RemoteCommand(WHICH, ['python3'])
             python = await self.mgr.ssh._check_execute_command(host, cmd, addr=addr)
-            cmd = [python, self.mgr.cephadm_binary_path] + final_args
+            # N.B. because the python3 executable is based on the results of the
+            # which command we can not know it ahead of time and must be converted
+            # into a RemoteExecutable.
+            cmd = ssh.RemoteCommand(
+                ssh.RemoteExecutable(python),
+                [self.mgr.cephadm_binary_path] + final_args
+            )
 
             try:
                 out, err, code = await self.mgr.ssh._execute_command(
                     host, cmd, stdin=stdin, addr=addr)
                 if code == 2:
-                    ls_cmd = ['ls', self.mgr.cephadm_binary_path]
+                    ls_cmd = ssh.RemoteCommand(
+                        ssh.Executables.LS,
+                        [self.mgr.cephadm_binary_path]
+                    )
                     out_ls, err_ls, code_ls = await self.mgr.ssh._execute_command(host, ls_cmd, addr=addr,
                                                                                   log_command=log_output)
                     if code_ls == 2:
@@ -1649,7 +1662,7 @@ async def _run_cephadm(self,
 
         elif self.mgr.mode == 'cephadm-package':
             try:
-                cmd = ['/usr/bin/cephadm'] + final_args
+                cmd = ssh.RemoteCommand(CEPHADM_EXE, final_args)
                 out, err, code = await self.mgr.ssh._execute_command(
                     host, cmd, stdin=stdin, addr=addr)
             except Exception as e:
diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index 7460fc159d7b..f0d507dfe895 100644
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -1,3 +1,4 @@
+import enum
 import logging
 import os
 import asyncio
@@ -44,6 +45,77 @@ def __init__(self, message: str, hostname: str, addr: str) -> None:
 """
 
 
+class RemoteExecutable(str):
+    pass
+
+
+class RemoteCommand:
+    exe: RemoteExecutable
+    args: List[str]
+
+    def __init__(self, exe: RemoteExecutable, args: Optional[List[str]] = None) -> None:
+        self.exe = exe
+        self.args = args or []
+
+    def __iter__(self) -> Iterator[str]:
+        yield str(self.exe)
+        for arg in self.args:
+            yield arg
+
+    def quoted(self) -> Iterator[str]:
+        return (quote(a) for a in self)
+
+    def __str__(self) -> str:
+        return " ".join(self.quoted())
+
+    def __repr__(self) -> str:
+        # handy when debugging tests
+        return f'<RemoteCommand>({self.exe!r}, {self.args!r})'
+
+    def __eq__(self, other: object) -> bool:
+        # handy when working with unit tests
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return other.exe == self.exe and other.args == self.args
+
+
+class RemoteSudoCommand(RemoteCommand):
+    use_sudo: bool = True
+
+    def __init__(
+        self, exe: RemoteExecutable, args: List[str], use_sudo: bool = True
+    ) -> None:
+        super().__init__(exe, args)
+        self.use_sudo = use_sudo
+
+    def __iter__(self) -> Iterator[str]:
+        if self.use_sudo:
+            yield 'sudo'
+        for a in super().__iter__():
+            yield a
+
+    @classmethod
+    def wrap(
+        cls, other: RemoteCommand, use_sudo: bool = True
+    ) -> 'RemoteSudoCommand':
+        return cls(other.exe, other.args, use_sudo)
+
+
+class Executables(RemoteExecutable, enum.Enum):
+    CHMOD = RemoteExecutable('chmod')
+    CHOWN = RemoteExecutable('chown')
+    LS = RemoteExecutable('ls')
+    MKDIR = RemoteExecutable('mkdir')
+    MV = RemoteExecutable('mv')
+    RM = RemoteExecutable('rm')
+    SYSCTL = RemoteExecutable('sysctl')
+    TOUCH = RemoteExecutable('touch')
+    TRUE = RemoteExecutable('true')
+
+    def __str__(self) -> str:
+        return self.value
+
+
 class EventLoopThread(Thread):
 
     def __init__(self) -> None:
@@ -152,24 +224,27 @@ def remote_connection(self,
 
     async def _execute_command(self,
                                host: str,
-                               cmd_components: List[str],
+                               cmd_components: RemoteCommand,
                                stdin: Optional[str] = None,
                                addr: Optional[str] = None,
                                log_command: Optional[bool] = True,
                                ) -> Tuple[str, str, int]:
 
         conn = await self._remote_connection(host, addr)
-        sudo_prefix = "sudo " if self.mgr.ssh_user != 'root' else ""
-        cmd = sudo_prefix + " ".join(quote(x) for x in cmd_components)
+        use_sudo = (self.mgr.ssh_user != 'root')
+        rcmd = RemoteSudoCommand.wrap(cmd_components, use_sudo=use_sudo)
         try:
             address = addr or self.mgr.inventory.get_addr(host)
         except Exception:
             address = host
         if log_command:
-            logger.debug(f'Running command: {cmd}')
+            logger.debug(f'Running command: {rcmd}')
         try:
-            r = await conn.run(f'{sudo_prefix}true', check=True, timeout=5)  # host quick check
-            r = await conn.run(cmd, input=stdin)
+            test_cmd = RemoteSudoCommand(
+                Executables.TRUE, [], use_sudo=use_sudo
+            )
+            r = await conn.run(str(test_cmd), check=True, timeout=5)  # host quick check
+            r = await conn.run(str(rcmd), input=stdin)
         # handle these Exceptions otherwise you might get a weird error like
         # TypeError: __init__() missing 1 required positional argument: 'reason' (due to the asyncssh error interacting with raise_if_exception)
         except asyncssh.ChannelOpenError as e:
@@ -179,13 +254,13 @@ async def _execute_command(self,
             self.mgr.offline_hosts.add(host)
             raise HostConnectionError(f'Unable to reach remote host {host}. {str(e)}', host, address)
         except asyncssh.ProcessError as e:
-            msg = f"Cannot execute the command '{cmd}' on the {host}. {str(e.stderr)}."
+            msg = f"Cannot execute the command '{rcmd}' on the {host}. {str(e.stderr)}."
             logger.debug(msg)
             await self._reset_con(host)
             self.mgr.offline_hosts.add(host)
             raise HostConnectionError(msg, host, address)
         except Exception as e:
-            msg = f"Generic error while executing command '{cmd}' on the host {host}. {str(e)}."
+            msg = f"Generic error while executing command '{rcmd}' on the host {host}. {str(e)}."
             logger.debug(msg)
             await self._reset_con(host)
             self.mgr.offline_hosts.add(host)
@@ -209,7 +284,7 @@ def _rstrip(v: Union[bytes, str, None]) -> str:
 
     def execute_command(self,
                         host: str,
-                        cmd: List[str],
+                        cmd: RemoteCommand,
                         stdin: Optional[str] = None,
                         addr: Optional[str] = None,
                         log_command: Optional[bool] = True
@@ -219,7 +294,7 @@ def execute_command(self,
 
     async def _check_execute_command(self,
                                      host: str,
-                                     cmd: List[str],
+                                     cmd: RemoteCommand,
                                      stdin: Optional[str] = None,
                                      addr: Optional[str] = None,
                                      log_command: Optional[bool] = True
@@ -233,7 +308,7 @@ async def _check_execute_command(self,
 
     def check_execute_command(self,
                               host: str,
-                              cmd: List[str],
+                              cmd: RemoteCommand,
                               stdin: Optional[str] = None,
                               addr: Optional[str] = None,
                               log_command: Optional[bool] = True,
@@ -253,14 +328,22 @@ async def _write_remote_file(self,
         try:
             cephadm_tmp_dir = f"/tmp/cephadm-{self.mgr._cluster_fsid}"
             dirname = os.path.dirname(path)
-            await self._check_execute_command(host, ['mkdir', '-p', dirname], addr=addr)
-            await self._check_execute_command(host, ['mkdir', '-p', cephadm_tmp_dir + dirname], addr=addr)
+            mkdir = RemoteCommand(Executables.MKDIR, ['-p', dirname])
+            await self._check_execute_command(host, mkdir, addr=addr)
+            mkdir2 = RemoteCommand(Executables.MKDIR, ['-p', cephadm_tmp_dir + dirname])
+            await self._check_execute_command(host, mkdir2, addr=addr)
             tmp_path = cephadm_tmp_dir + path + '.new'
-            await self._check_execute_command(host, ['touch', tmp_path], addr=addr)
+            touch = RemoteCommand(Executables.TOUCH, [tmp_path])
+            await self._check_execute_command(host, touch, addr=addr)
             if self.mgr.ssh_user != 'root':
                 assert self.mgr.ssh_user
-                await self._check_execute_command(host, ['chown', '-R', self.mgr.ssh_user, cephadm_tmp_dir], addr=addr)
-                await self._check_execute_command(host, ['chmod', str(644), tmp_path], addr=addr)
+                chown = RemoteCommand(
+                    Executables.CHOWN,
+                    ['-R', self.mgr.ssh_user, cephadm_tmp_dir]
+                )
+                await self._check_execute_command(host, chown, addr=addr)
+                chmod = RemoteCommand(Executables.CHMOD, [str(644), tmp_path])
+                await self._check_execute_command(host, chmod, addr=addr)
             with NamedTemporaryFile(prefix='cephadm-write-remote-file-') as f:
                 os.fchmod(f.fileno(), 0o600)
                 f.write(content)
@@ -270,9 +353,15 @@ async def _write_remote_file(self,
                     await sftp.put(f.name, tmp_path)
             if uid is not None and gid is not None and mode is not None:
                 # shlex quote takes str or byte object, not int
-                await self._check_execute_command(host, ['chown', '-R', str(uid) + ':' + str(gid), tmp_path], addr=addr)
-                await self._check_execute_command(host, ['chmod', oct(mode)[2:], tmp_path], addr=addr)
-            await self._check_execute_command(host, ['mv', tmp_path, path], addr=addr)
+                chown = RemoteCommand(
+                    Executables.CHOWN,
+                    ['-R', str(uid) + ':' + str(gid), tmp_path]
+                )
+                await self._check_execute_command(host, chown, addr=addr)
+                chmod = RemoteCommand(Executables.CHMOD, [oct(mode)[2:], tmp_path])
+                await self._check_execute_command(host, chmod, addr=addr)
+            mv = RemoteCommand(Executables.MV, [tmp_path, path])
+            await self._check_execute_command(host, mv, addr=addr)
         except Exception as e:
             msg = f"Unable to write {host}:{path}: {e}"
             logger.exception(msg)
diff --git a/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py b/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py
index 66feaee31949..9db971f6f216 100644
--- a/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py
+++ b/src/pybind/mgr/cephadm/tests/test_tuned_profiles.py
@@ -5,7 +5,7 @@
 from cephadm.inventory import TunedProfileStore
 from ceph.utils import datetime_now
 from ceph.deployment.service_spec import TunedProfileSpec, PlacementSpec
-from cephadm.ssh import SSHManager
+from cephadm.ssh import SSHManager, RemoteCommand, Executables
 from orchestrator import HostSpec
 
 from typing import List, Dict
@@ -148,10 +148,26 @@ def test_rm_stray_tuned_profiles(self, _check_execute_command):
         tp = TunedProfileUtils(mgr)
         tp._remove_stray_tuned_profiles('a', self.profiles_to_calls(tp, [self.tspec1, self.tspec2]))
         calls = [
-            mock.call('a', ['ls', SYSCTL_DIR], log_command=False),
-            mock.call('a', ['rm', '-f', f'{SYSCTL_DIR}/p3-cephadm-tuned-profile.conf']),
-            mock.call('a', ['rm', '-f', f'{SYSCTL_DIR}/who-cephadm-tuned-profile.conf']),
-            mock.call('a', ['sysctl', '--system'])
+            mock.call(
+                'a', RemoteCommand(Executables.LS, [SYSCTL_DIR]), log_command=False
+            ),
+            mock.call(
+                'a',
+                RemoteCommand(
+                    Executables.RM,
+                    ['-f', f'{SYSCTL_DIR}/p3-cephadm-tuned-profile.conf']
+                )
+            ),
+            mock.call(
+                'a',
+                RemoteCommand(
+                    Executables.RM,
+                    ['-f', f'{SYSCTL_DIR}/who-cephadm-tuned-profile.conf']
+                )
+            ),
+            mock.call(
+                'a', RemoteCommand(Executables.SYSCTL, ['--system'])
+            ),
         ]
         _check_execute_command.assert_has_calls(calls, any_order=True)
 
@@ -170,7 +186,9 @@ def test_write_tuned_profiles(self, _write_remote_file, _check_execute_command):
                       profiles)
         tp = TunedProfileUtils(mgr)
         tp._write_tuned_profiles('a', self.profiles_to_calls(tp, [self.tspec1, self.tspec2]))
-        _check_execute_command.assert_called_with('a', ['sysctl', '--system'])
+        _check_execute_command.assert_called_with(
+            'a', RemoteCommand(Executables.SYSCTL, ['--system'])
+        )
         _write_remote_file.assert_called_with(
             'a', f'{SYSCTL_DIR}/p2-cephadm-tuned-profile.conf', tp._profile_to_str(self.tspec2).encode('utf-8'))
 
diff --git a/src/pybind/mgr/cephadm/tuned_profiles.py b/src/pybind/mgr/cephadm/tuned_profiles.py
index 8ec30bd536be..7a37d9379044 100644
--- a/src/pybind/mgr/cephadm/tuned_profiles.py
+++ b/src/pybind/mgr/cephadm/tuned_profiles.py
@@ -3,6 +3,7 @@
 from ceph.utils import datetime_now
 from .schedule import HostAssignment
 from ceph.deployment.service_spec import ServiceSpec, TunedProfileSpec
+from . import ssh
 
 if TYPE_CHECKING:
     from cephadm.module import CephadmOrchestrator
@@ -11,6 +12,8 @@
 
 SYSCTL_DIR = '/etc/sysctl.d'
 
+SYSCTL_SYSTEM_CMD = ssh.RemoteCommand(ssh.Executables.SYSCTL, ['--system'])
+
 
 class TunedProfileUtils():
     def __init__(self, mgr: "CephadmOrchestrator") -> None:
@@ -69,7 +72,7 @@ def _remove_stray_tuned_profiles(self, host: str, profiles: List[Dict[str, str]]
         """
         if self.mgr.cache.is_host_unreachable(host):
             return
-        cmd = ['ls', SYSCTL_DIR]
+        cmd = ssh.RemoteCommand(ssh.Executables.LS, [SYSCTL_DIR])
         found_files = self.mgr.ssh.check_execute_command(host, cmd, log_command=self.mgr.log_refresh_metadata).split('\n')
         found_files = [s.strip() for s in found_files]
         profile_names: List[str] = sum([[*p] for p in profiles], [])  # extract all profiles names
@@ -81,11 +84,11 @@ def _remove_stray_tuned_profiles(self, host: str, profiles: List[Dict[str, str]]
                 continue
             if file not in expected_files:
                 logger.info(f'Removing stray tuned profile file {file}')
-                cmd = ['rm', '-f', f'{SYSCTL_DIR}/{file}']
+                cmd = ssh.RemoteCommand(ssh.Executables.RM, ['-f', f'{SYSCTL_DIR}/{file}'])
                 self.mgr.ssh.check_execute_command(host, cmd)
                 updated = True
         if updated:
-            self.mgr.ssh.check_execute_command(host, ['sysctl', '--system'])
+            self.mgr.ssh.check_execute_command(host, SYSCTL_SYSTEM_CMD)
 
     def _write_tuned_profiles(self, host: str, profiles: List[Dict[str, str]]) -> None:
         if self.mgr.cache.is_host_unreachable(host):
@@ -99,5 +102,5 @@ def _write_tuned_profiles(self, host: str, profiles: List[Dict[str, str]]) -> No
                     self.mgr.ssh.write_remote_file(host, profile_filename, content.encode('utf-8'))
                     updated = True
         if updated:
-            self.mgr.ssh.check_execute_command(host, ['sysctl', '--system'])
+            self.mgr.ssh.check_execute_command(host, SYSCTL_SYSTEM_CMD)
         self.mgr.cache.last_tuned_profile_update[host] = datetime_now()

From d9fb2840fc34534fc157cd2d9563c2c1a5ebe407 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 8 Mar 2024 13:19:39 -0500
Subject: [PATCH 2292/2492] mgr/cephadm: add test to ensure the list of remote
 commands is known

Add a test file to help ensure the audit of remote commands is kept up
to date.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadm/tests/test_remote_executables.py  | 262 ++++++++++++++++++
 1 file changed, 262 insertions(+)
 create mode 100644 src/pybind/mgr/cephadm/tests/test_remote_executables.py

diff --git a/src/pybind/mgr/cephadm/tests/test_remote_executables.py b/src/pybind/mgr/cephadm/tests/test_remote_executables.py
new file mode 100644
index 000000000000..a98f81a8df1e
--- /dev/null
+++ b/src/pybind/mgr/cephadm/tests/test_remote_executables.py
@@ -0,0 +1,262 @@
+#!/usr/bin/python3
+"""Test to ensure remote suodable executables are audited.
+
+This file can be used in one of two ways:
+* as a "unit test" executed by pytest
+* as a script that can report on or check expected remote executables
+
+It is designed to act as a method of ensuring that the executables that we run
+on remote nodes under sudo are explicitly known. The types defined in ssh.py
+act as both a way to audit the commands (via this script) and that we don't
+lose track of what can be run - by relying on mypy.
+
+The unit test mode integrates into pytest for convenience, it really acts as a
+static check that scans the source code of the cephadm mgr module.  NOTE: the
+file's test script mode is sensitive to it's location in the source tree. If
+files get moved this script may need to be updated.
+
+The test asserts that the `EXPECTED` list matches all tools that may be
+executed remotely under sudo.
+
+This file can also be run as a script. When supplied with a directory or list
+of files it will read the sources and report on all remote executables found.
+When run with the `--check` option it performs the same job as the unit test
+mode but outputs a report in a more human readable format.
+
+If the commands the manager module can execute remotely change the `EXPECTED`
+this must be updated to match to ensure the change is being done deliberately.
+Any corresponding documentation should also be updated.
+
+Note that ideally the EXPECTED list should shrink and not grow.  Any changes to
+the list may cause administrators of the ceph cluster to have to make manual
+changes to the system prior/during upgrade!
+"""
+import ast
+import os
+import pathlib
+import sys
+
+ssh_py = 'ssh.py'
+serve_py = 'serve.py'
+
+# IMPORTANT - please read the entire module docstring before changing
+# the list below.
+EXPECTED = [
+    # - value | is_constant | filename -
+    # constant executables
+    ('/usr/bin/cephadm', True, serve_py),
+    ('chmod', True, ssh_py),
+    ('chown', True, ssh_py),
+    ('ls', True, ssh_py),
+    ('mkdir', True, ssh_py),
+    ('mv', True, ssh_py),
+    ('rm', True, ssh_py),
+    ('sysctl', True, ssh_py),
+    ('touch', True, ssh_py),
+    ('true', True, ssh_py),
+    ('which', True, serve_py),
+    # variable executables
+    ('python', False, serve_py),
+]
+
+
+def test_expected_remote_executables():
+    import pytest
+
+    if sys.version_info < (3, 8):
+        pytest.skip("python 3.8 or later required")
+
+    self_path = pathlib.Path(__file__).resolve()
+    # test is sensitive to where it is in the source tree. it expects to be in
+    # a tests directory under the cephadm mgr module. if stuff gets moved
+    # around this test will likely start failing and need to be updated
+    remote_exes = find_sudoable_exes_in_files(
+        _file_paths([self_path.parent.parent])
+    )
+    unexpected, gone = diff_remote_exes(remote_exes)
+    unexpected_msgs = gone_msgs = []
+    if unexpected or gone:
+        unexpected_msgs, gone_msgs = format_diff(
+            unexpected, gone, remote_exes
+        )
+    assert not unexpected_msgs, unexpected_msgs
+    assert not gone_msgs, gone_msgs
+
+
+def _essential(v):
+    """Convert a remote exe dict to a tuple with only the essential fields."""
+    return (v["value"], v["is_constant"], v["filename"].split("/")[-1])
+
+
+def _names(node):
+    if isinstance(node, ast.Name):
+        return [node.id]
+    if isinstance(node, ast.Attribute):
+        vn = _names(node.value)
+        return vn + [node.attr]
+    if isinstance(node, ast.Call):
+        return _names(node.func)
+    if isinstance(node, ast.Constant):
+        return [repr(node.value)]
+    if isinstance(node, ast.JoinedStr):
+        return [f"<JoinedStr: {node.values!r}>"]
+    if isinstance(node, ast.Subscript):
+        return [f"<Subscript: {node.value}{node.slice}>"]
+    raise ValueError(f"_names: unexpected type: {node}")
+
+
+def _arg_kind(node):
+    assert isinstance(node, ast.Call)
+    assert len(node.args) == 1
+    arg = node.args[0]
+    if isinstance(arg, ast.Constant):
+        return str(arg.value), True
+    names = _names(arg)
+    return ".".join(names), False
+
+
+class ExecutableVisitor(ast.NodeVisitor):
+    def __init__(self):
+        super().__init__()
+        self.remote_executables = []
+
+    def visit_Call(self, node):
+        names = _names(node)
+        if names[-1] == 'RemoteExecutable':
+            value, is_constant = _arg_kind(node)
+            self.remote_executables.append(
+                dict(value=value, is_constant=is_constant, lineno=node.lineno)
+            )
+        self.generic_visit(node)
+
+
+def find_sudoable_exes(tree):
+    ev = ExecutableVisitor()
+    ev.visit(tree)
+    return ev.remote_executables
+
+
+def find_sudoable_exes_in_files(files):
+    out = []
+    for file in files:
+        with open(file) as fh:
+            source = fh.read()
+        tree = ast.parse(source, fh.name)
+        rmes = find_sudoable_exes(tree)
+        for rme in rmes:
+            rme['filename'] = str(file)
+        out.extend(rmes)
+    return out
+
+
+def diff_remote_exes(remote_exes):
+    expected = set(EXPECTED)
+    current = {_essential(v) for v in remote_exes}
+    unexpected = current - expected
+    gone = expected - current
+    return unexpected, gone
+
+
+def format_diff(unexpected, gone, remote_exes):
+    current = {_essential(v): v for v in remote_exes}
+    unexpected_msgs = []
+    for val, is_constant, fn in unexpected:
+        vn = 'constant' if is_constant else 'variable'
+        vq = repr(val) if is_constant else val
+        # info is needed for full filename/linenumber and is only relevant for
+        # found (unexpected) entries
+        info = current[(val, is_constant, fn)]
+        unexpected_msgs.append(
+            f'{vn} {vq} in {info["filename"]}:{info["lineno"]} not tracked'
+        )
+    gone_msgs = []
+    for val, is_constant, fn in gone:
+        vn = 'constant' if is_constant else 'variable'
+        vq = repr(val) if is_constant else val
+        gone_msgs.append(f"{vn} {vq} expected in {fn} not found")
+    return unexpected_msgs, gone_msgs
+
+
+def report_remote_exes(remote_exes):
+    for rme in remote_exes:
+        clabel = 'CONSTANT' if rme["is_constant"] else "VARIABLE"
+        print(
+            "{clabel:10} {value:10} {filename}:{lineno}".format(
+                clabel=clabel, **rme
+            )
+        )
+
+
+def report_compare_remote_exes(remote_exes):
+    import textwrap
+
+    unexpected, gone = diff_remote_exes(remote_exes)
+    if not (unexpected or gone):
+        print('No issues detected')
+        sys.exit(0)
+    unexpected_msgs, gone_msgs = format_diff(unexpected, gone, remote_exes)
+    if unexpected_msgs:
+        desc = textwrap.wrap(
+            "One or more remote executable has been detected in the source"
+            " files that is not tracked in the test. If this change is"
+            " intended you must update the test AND update any corresponding"
+            " documentation.",
+            76,
+        )
+        for line in desc:
+            print(line)
+        print('-' * 76)
+    for msg in unexpected_msgs:
+        print(f'* {msg}')
+    if unexpected_msgs:
+        print()
+    if gone_msgs:
+        desc = textwrap.wrap(
+            "One or more remote executable that is expected to appear"
+            " in the source files has not been detected."
+            " If this change is intended you must update the test AND update"
+            " any corresponding documentation.",
+            76,
+        )
+        for line in desc:
+            print(line)
+        print('-' * 76)
+    for msg in gone_msgs:
+        print(f'* {msg}')
+
+    sys.exit(1)
+
+
+def _file_paths(src_paths):
+    files = set()
+    for path in src_paths:
+        if path.is_file():
+            files.add(path)
+            continue
+        for d, ds, fs in os.walk(path):
+            if 'tests' in ds:
+                ds.remove('tests')
+            dpath = pathlib.Path(d)
+            for fn in fs:
+                if fn.endswith('.py'):
+                    files.add(dpath / fn)
+    return files
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--check', action='store_true')
+    parser.add_argument('PATH', nargs='+', type=pathlib.Path)
+    cli = parser.parse_args()
+
+    remote_exes = find_sudoable_exes_in_files(_file_paths(cli.PATH))
+    if cli.check:
+        report_compare_remote_exes(remote_exes)
+    else:
+        report_remote_exes(remote_exes)
+
+
+if __name__ == '__main__':
+    main()

From e357d2f99a14969b6f1a089680a9f16dc3bb0e54 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 14 Mar 2024 14:02:17 -0400
Subject: [PATCH 2293/2492] mgr/cephadm: add a simple unit test for
 RemoteCommand class

Converting a remote command to something that other libs uses requires
converting the enum to a string. Python behavior in the area varies
across versions so add a unit test that verifies the conversion
behaves as intended.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/tests/test_ssh.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/pybind/mgr/cephadm/tests/test_ssh.py b/src/pybind/mgr/cephadm/tests/test_ssh.py
index 29f01b6c7972..44ef3d429b75 100644
--- a/src/pybind/mgr/cephadm/tests/test_ssh.py
+++ b/src/pybind/mgr/cephadm/tests/test_ssh.py
@@ -103,3 +103,14 @@ def run_test(host, conn, expected_error):
 class TestWithoutSSH:
     def test_can_run(self, cephadm_module: CephadmOrchestrator):
         assert cephadm_module.can_run() == (False, "loading asyncssh library:No module named 'asyncssh'")
+
+
+def test_remote_command():
+    from cephadm.ssh import RemoteCommand, Executables
+
+    assert list(RemoteCommand(Executables.TRUE)) == ['true']
+    assert list(RemoteCommand(Executables.RM, ['-rf', '/tmp/blat'])) == [
+        'rm',
+        '-rf',
+        '/tmp/blat',
+    ]

From 18efb3b74249f7035bfa77ad6925328c16ab0ce9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 8 Mar 2024 13:56:19 -0500
Subject: [PATCH 2294/2492] doc/cephdam: document limiting passwordless sudo
 commands

Based on the previous commits making the remote executables auditable
and explicit, document the admin's ability to restrict password-less
sudo access to only the set of commands cephadm actually uses.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 doc/cephadm/operations.rst | 45 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/doc/cephadm/operations.rst b/doc/cephadm/operations.rst
index c2c8f8b6d28a..4ec28bc1c362 100644
--- a/doc/cephadm/operations.rst
+++ b/doc/cephadm/operations.rst
@@ -658,6 +658,51 @@ For example, to distribute configs to hosts with the ``bare_config`` label, run
 
 (See :ref:`orchestrator-cli-placement-spec` for more information about placement specs.)
 
+
+Limiting Password-less sudo Access
+==================================
+
+By default, the cephadm install guide recommends enabling password-less
+``sudo`` for the cephadm user. This option is the most flexible and
+future-proof but may not be preferred in all environments. An administrator can
+restrict ``sudo`` to only running an exact list of commands without password
+access.  Note that this list may change between Ceph versions and
+administrators choosing this option should read the release notes and review
+this list in the destination version of the Ceph documentation. If the list
+differs one must extend the list of password-less ``sudo`` commands prior to
+upgrade.
+
+Commands requiring password-less sudo support:
+
+  - ``chmod``
+  - ``chown``
+  - ``ls``
+  - ``mkdir``
+  - ``mv``
+  - ``rm``
+  - ``sysctl``
+  - ``touch``
+  - ``true``
+  - ``which`` (see note)
+  - ``/usr/bin/cephadm`` or python executable (see note)
+
+.. note:: Typically cephadm will execute ``which`` to determine what python3
+   command is available and then use the command returned by ``which`` in
+   subsequent commands.
+   Before configuring ``sudo`` run ``which python3`` to determine what
+   python command to add to the ``sudo`` configuration.
+   In some rare configurations ``/usr/bin/cephadm`` will be used instead.
+
+
+Configuring the ``sudoers`` file can be performed using a tool like ``visudo``
+and adding or replacing a user configuration line such as the following:
+
+.. code-block::
+
+  # assuming the cephadm user is named "ceph"
+  ceph ALL=(ALL) NOPASSWD:/usr/bin/chmod,/usr/bin/chown,/usr/bin/ls,/usr/bin/mkdir,/usr/bin/mv,/usr/bin/rm,/usr/sbin/sysctl,/usr/bin/touch,/usr/bin/true,/usr/bin/which,/usr/bin/cephadm,/usr/bin/python3
+
+
 Purging a cluster
 =================
 

From e1c8e08961e8199f1da74dc9d1eb2a940e2908d5 Mon Sep 17 00:00:00 2001
From: Leonid Usov <leonid.usov@ibm.com>
Date: Sun, 17 Mar 2024 12:22:34 -0400
Subject: [PATCH 2295/2492] mds/quiesce-db: always clear the db if a membership
 is lost

Fixes: https://tracker.ceph.com/issues/64912
Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/QuiesceDbManager.cc   | 36 ++++++++++++++++++++---------------
 src/mds/QuiesceDbManager.h    |  1 +
 src/test/mds/TestQuiesceDb.cc | 10 ++--------
 3 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/mds/QuiesceDbManager.cc b/src/mds/QuiesceDbManager.cc
index 6fccaacf10c4..bcbcbeed0b14 100644
--- a/src/mds/QuiesceDbManager.cc
+++ b/src/mds/QuiesceDbManager.cc
@@ -168,18 +168,23 @@ void QuiesceDbManager::update_membership(const QuiesceClusterMembership& new_mem
     dout(5) << "starting the db mgr thread at epoch: " << new_membership.epoch << dendl;
     db_thread_should_exit = false;
     quiesce_db_thread.create("quiesce_db_mgr");
-  } else {
+  } else if (quiesce_db_thread.is_started()) {
     submit_condition.notify_all();
   }
 
   if (inject_request) {
-    pending_requests.push_front(inject_request);
+    if (will_participate || quiesce_db_thread.is_started()) {
+      pending_requests.push_front(inject_request);
+    } else {
+      inject_request->complete(ENOTTY);
+    }
   }
 
   if (will_participate) {
     cluster_membership = new_membership;
   } else {
     cluster_membership.reset();
+    db_thread_should_clear_db = true;
   }
 
   std::lock_guard lc(agent_mutex);
@@ -188,8 +193,20 @@ void QuiesceDbManager::update_membership(const QuiesceClusterMembership& new_mem
   }
 }
 
-std::pair<QuiesceDbManager::IsMemberBool, QuiesceDbManager::ShouldExitBool> QuiesceDbManager::membership_upkeep()
+std::pair<QuiesceDbManager::IsMemberBool, QuiesceDbManager::ShouldExitBool>
+QuiesceDbManager::membership_upkeep()
 {
+  if (db_thread_should_clear_db) {
+    dout(5) << "a reset of the db has been requested" << dendl;
+    db_thread_should_clear_db = false;
+    membership.epoch = 0;
+    // clear the peers to bootstrap from scratch if we are the leader
+    peers.clear();
+    // reset the db
+    db.clear();
+    // not clearing awaits and requests, they will be handled below
+  }
+
   if (cluster_membership && cluster_membership->epoch == membership.epoch) {
     // no changes
     return {true, db_thread_should_exit};
@@ -218,12 +235,6 @@ std::pair<QuiesceDbManager::IsMemberBool, QuiesceDbManager::ShouldExitBool> Quie
     for (auto peer : cluster_membership->members) {
       peers.try_emplace(peer);
     }
-
-    if (db.set_version == 0) {
-      db.time_zero = QuiesceClock::now();
-      db.sets.clear();
-    }
-
   } else {
     peers.clear();
     // abort awaits with EINPROGRESS
@@ -244,11 +255,6 @@ std::pair<QuiesceDbManager::IsMemberBool, QuiesceDbManager::ShouldExitBool> Quie
   if (cluster_membership) {
     membership = *cluster_membership;
     dout(15) << "Updated membership" << dendl;
-  } else {
-    membership.epoch = 0;
-    peers.clear();
-    awaits.clear();
-    db.clear();
   }
 
   return { cluster_membership.has_value(), db_thread_should_exit };
@@ -291,7 +297,7 @@ QuiesceTimeInterval QuiesceDbManager::replica_upkeep(decltype(pending_db_updates
   db.time_zero = time_zero;
 
   if (db.set_version > update.db_version.set_version) {
-    dout(3) << "got an older version of DB from the leader: " << db.set_version << " > " << update.db_version.set_version << dendl;
+    dout(3) << "got an older version of DB from the leader: " << update.db_version.set_version << " < " << db.set_version << dendl;
     dout(3) << "discarding the DB" << dendl;
     db.clear();
   } else {
diff --git a/src/mds/QuiesceDbManager.h b/src/mds/QuiesceDbManager.h
index 98d0b84fc24b..2fdc605ac846 100644
--- a/src/mds/QuiesceDbManager.h
+++ b/src/mds/QuiesceDbManager.h
@@ -204,6 +204,7 @@ class QuiesceDbManager {
     std::queue<QuiesceDbPeerAck> pending_acks;
     std::deque<RequestContext*> pending_requests;
     bool db_thread_should_exit = false;
+    bool db_thread_should_clear_db = true;
 
     class QuiesceDbThread : public Thread {
       public:
diff --git a/src/test/mds/TestQuiesceDb.cc b/src/test/mds/TestQuiesceDb.cc
index 2ffba9778831..cf05aaa038d0 100644
--- a/src/test/mds/TestQuiesceDb.cc
+++ b/src/test/mds/TestQuiesceDb.cc
@@ -1571,7 +1571,7 @@ TEST_F(QuiesceDbTest, MultiRankRecovery)
   ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
 
   // we expect the db to be populated since the new leader must have discovered newer versions
-  // we expect the sets to become quiescing since there's at least one member that's not acking (the new one)
+  // we expect the sets to become quiesced since all members are now acking
   EXPECT_EQ(OK(), run_request([](auto& r) {
     r.set_id = "set1";
     r.await = sec(1);
@@ -1598,15 +1598,9 @@ TEST_F(QuiesceDbTest, MultiRankRecovery)
   });
 
   // add back a quiescing peer
-  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3)}));
-
-  EXPECT_EQ(OK(), run_request([](auto& r) {}));
-  ASSERT_EQ(2, last_request->response.sets.size());
-  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);
-  EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set2").rstate.state);
+  ASSERT_NO_FATAL_FAILURE(configure_cluster({ mds_gid_t(1), mds_gid_t(2), mds_gid_t(3) }));
 
   EXPECT_EQ(std::future_status::ready, did_ack3.wait_for(std::chrono::milliseconds(2000)));
-
   EXPECT_EQ(OK(), run_request([](auto& r) {}));
   ASSERT_EQ(2, last_request->response.sets.size());
   EXPECT_EQ(QS_QUIESCED, last_request->response.sets.at("set1").rstate.state);

From f3eb489520fd4fae057e61275d16c6c8fd596f3f Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 18 Mar 2024 09:32:39 +0530
Subject: [PATCH 2296/2492] docs/rbd: fix typo in arg name

Replace "{image-}" with "{image-id}" in the "rbd trash rm"
command description.

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 doc/rbd/rados-rbd-cmds.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/rbd/rados-rbd-cmds.rst b/doc/rbd/rados-rbd-cmds.rst
index 0bbcb261160d..489e117e5409 100644
--- a/doc/rbd/rados-rbd-cmds.rst
+++ b/doc/rbd/rados-rbd-cmds.rst
@@ -250,13 +250,13 @@ Removing a Deferred Block Device from a Pool
 --------------------------------------------
 
 To remove a deferred block device from a pool, run the following command but
-replace ``{image-}`` with the ID of the image to be removed, and replace
+replace ``{image-id}`` with the ID of the image to be removed, and replace
 ``{pool-name}`` with the name of the pool from which the image is to be
 removed:
 
 .. prompt:: bash $
 
-   rbd trash rm {pool-name}/{image-}
+   rbd trash rm {pool-name}/{image-id}
 
 For example:
 

From ebf2f60f784728c04d8ec59015d666bafcef8218 Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 18 Mar 2024 09:41:10 +0530
Subject: [PATCH 2297/2492] docs/rbd: replace introspect with inspect

Replace "introspect" with "inspect" in the rbd basic commands
description.

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 doc/rbd/rados-rbd-cmds.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rbd/rados-rbd-cmds.rst b/doc/rbd/rados-rbd-cmds.rst
index 489e117e5409..470fac61845a 100644
--- a/doc/rbd/rados-rbd-cmds.rst
+++ b/doc/rbd/rados-rbd-cmds.rst
@@ -4,7 +4,7 @@
 
 .. index:: Ceph Block Device; image management
 
-The ``rbd`` command enables you to create, list, introspect and remove block
+The ``rbd`` command enables you to create, list, inspect and remove block
 device images. You can also use it to clone images, create snapshots,
 rollback an image to a snapshot, view a snapshot, etc. For details on using
 the ``rbd`` command, see `RBD – Manage RADOS Block Device (RBD) Images`_ for

From 4fd5c134536d652ae1f9e05ecf52cb81adb3b850 Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 18 Mar 2024 10:04:44 +0530
Subject: [PATCH 2298/2492] docs/rbd: typo in "retrieving image information"

Replace "for the image" with "of the image".

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 doc/rbd/rados-rbd-cmds.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/rbd/rados-rbd-cmds.rst b/doc/rbd/rados-rbd-cmds.rst
index 470fac61845a..a290dc1e5a37 100644
--- a/doc/rbd/rados-rbd-cmds.rst
+++ b/doc/rbd/rados-rbd-cmds.rst
@@ -139,7 +139,7 @@ Retrieving Image Information
 ============================
 
 To retrieve information from a particular image, run the following command, but
-replace ``{image-name}`` with the name for the image:
+replace ``{image-name}`` with the name of the image:
 
 .. prompt:: bash $
 

From 35a3e6d8edd97ae6ff237e3e39f9edefc4d48cbf Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Sat, 16 Mar 2024 16:54:22 +0530
Subject: [PATCH 2299/2492] mgr/dashboard: securely store remote cluster token

Instead of using the localStorage use cookies for storing the token more
securely

Fixes: https://tracker.ceph.com/issues/64958
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../mgr/dashboard/frontend/package-lock.json  | 18 ++++++++++++++++
 .../mgr/dashboard/frontend/package.json       |  1 +
 .../multi-cluster-list.component.ts           |  5 ++++-
 .../navigation/navigation.component.ts        | 16 ++++++++++----
 .../services/api-interceptor.service.ts       |  7 ++++---
 .../shared/services/cookie.service.spec.ts    | 16 ++++++++++++++
 .../src/app/shared/services/cookie.service.ts | 21 +++++++++++++++++++
 7 files changed, 76 insertions(+), 8 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.ts

diff --git a/src/pybind/mgr/dashboard/frontend/package-lock.json b/src/pybind/mgr/dashboard/frontend/package-lock.json
index 8e6a9c4018fd..1af58a5101ce 100644
--- a/src/pybind/mgr/dashboard/frontend/package-lock.json
+++ b/src/pybind/mgr/dashboard/frontend/package-lock.json
@@ -38,6 +38,7 @@
         "ng-block-ui": "3.0.2",
         "ng-click-outside": "7.0.0",
         "ng2-charts": "4.1.1",
+        "ngx-cookie-service": "17.1.0",
         "ngx-pipe-function": "1.0.0",
         "ngx-toastr": "17.0.2",
         "rxjs": "6.6.3",
@@ -25048,6 +25049,23 @@
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
+    "node_modules/ngx-cookie-service": {
+      "version": "17.1.0",
+      "resolved": "https://registry.npmjs.org/ngx-cookie-service/-/ngx-cookie-service-17.1.0.tgz",
+      "integrity": "sha512-m4YI9IEgTaEBDMCz7oeVsO6UX14EmCzg29cTL6yxW8f7oye9wv56egi+3C4wAVSRPkI+cWlqnIOr+XyHwYQYmg==",
+      "dependencies": {
+        "tslib": "^2.6.2"
+      },
+      "peerDependencies": {
+        "@angular/common": "^17.0.0",
+        "@angular/core": "^17.0.0"
+      }
+    },
+    "node_modules/ngx-cookie-service/node_modules/tslib": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
+      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
+    },
     "node_modules/ngx-pipe-function": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/ngx-pipe-function/-/ngx-pipe-function-1.0.0.tgz",
diff --git a/src/pybind/mgr/dashboard/frontend/package.json b/src/pybind/mgr/dashboard/frontend/package.json
index fd4f7a0bff6b..f72dbdbe4125 100644
--- a/src/pybind/mgr/dashboard/frontend/package.json
+++ b/src/pybind/mgr/dashboard/frontend/package.json
@@ -72,6 +72,7 @@
     "ng-block-ui": "3.0.2",
     "ng-click-outside": "7.0.0",
     "ng2-charts": "4.1.1",
+    "ngx-cookie-service": "17.1.0",
     "ngx-pipe-function": "1.0.0",
     "ngx-toastr": "17.0.2",
     "rxjs": "6.6.3",
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
index 4496b3a34ceb..07238019c199 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
@@ -18,6 +18,7 @@ import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
 import { MultiCluster } from '~/app/shared/models/multi-cluster';
 import { SummaryService } from '~/app/shared/services/summary.service';
 import { Router } from '@angular/router';
+import { CookiesService } from '~/app/shared/services/cookie.service';
 
 @Component({
   selector: 'cd-multi-cluster-list',
@@ -48,7 +49,8 @@ export class MultiClusterListComponent {
     public actionLabels: ActionLabelsI18n,
     private notificationService: NotificationService,
     private authStorageService: AuthStorageService,
-    private modalService: ModalService
+    private modalService: ModalService,
+    private cookieService: CookiesService
   ) {
     this.tableActions = [
       {
@@ -189,6 +191,7 @@ export class MultiClusterListComponent {
       itemNames: [cluster['cluster_alias'] + ' - ' + cluster['user']],
       submitAction: () =>
         this.multiClusterService.deleteCluster(cluster['name'], cluster['user']).subscribe(() => {
+          this.cookieService.deleteToken(`${cluster['name']}-${cluster['user']}`);
           this.modalRef.close();
           this.notificationService.show(
             NotificationType.success,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index 4ae8d1897e27..8973843872bc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -9,6 +9,7 @@ import { Icons } from '~/app/shared/enum/icons.enum';
 import { MultiCluster } from '~/app/shared/models/multi-cluster';
 import { Permissions } from '~/app/shared/models/permissions';
 import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
+import { CookiesService } from '~/app/shared/services/cookie.service';
 import {
   FeatureTogglesMap$,
   FeatureTogglesService
@@ -56,7 +57,8 @@ export class NavigationComponent implements OnInit, OnDestroy {
     private featureToggles: FeatureTogglesService,
     private telemetryNotificationService: TelemetryNotificationService,
     public prometheusAlertService: PrometheusAlertService,
-    private motdNotificationService: MotdNotificationService
+    private motdNotificationService: MotdNotificationService,
+    private cookieService: CookiesService
   ) {
     this.permissions = this.authStorageService.getPermissions();
     this.enabledFeature$ = this.featureToggles.get();
@@ -178,7 +180,12 @@ export class NavigationComponent implements OnInit, OnDestroy {
   onClusterSelection(value: object) {
     this.multiClusterService.setCluster(value).subscribe(
       (resp: any) => {
-        localStorage.setItem('cluster_api_url', value['url']);
+        if (value['cluster_alias'] === 'local-cluster') {
+          localStorage.setItem('cluster_api_url', '');
+        } else {
+          localStorage.setItem('current_cluster_name', `${value['name']}-${value['user']}`);
+          localStorage.setItem('cluster_api_url', value['url']);
+        }
         this.selectedCluster = this.clustersMap.get(`${value['url']}-${value['user']}`) || {};
         const clustersConfig = resp['config'];
         if (clustersConfig && typeof clustersConfig === 'object') {
@@ -192,9 +199,10 @@ export class NavigationComponent implements OnInit, OnDestroy {
 
               if (
                 clusterName === this.selectedCluster['name'] &&
-                clusterUser === this.selectedCluster['user']
+                clusterUser === this.selectedCluster['user'] &&
+                clusterDetails['cluster_alias'] !== 'local-cluster'
               ) {
-                localStorage.setItem('token_of_selected_cluster', clusterToken);
+                this.cookieService.setToken(`${clusterName}-${clusterUser}`, clusterToken);
               }
             });
           });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
index 5cbd99911fb2..d1e98487789b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
@@ -20,6 +20,7 @@ import { NotificationService } from './notification.service';
 import { MultiClusterService } from '../api/multi-cluster.service';
 import { SummaryService } from './summary.service';
 import { AuthStorageService } from './auth-storage.service';
+import { CookiesService } from './cookie.service';
 
 export class CdHttpErrorResponse extends HttpErrorResponse {
   preventDefault: Function;
@@ -37,7 +38,8 @@ export class ApiInterceptorService implements HttpInterceptor {
     public notificationService: NotificationService,
     private summaryService: SummaryService,
     private authStorageService: AuthStorageService,
-    private multiClusterService: MultiClusterService
+    private multiClusterService: MultiClusterService,
+    private cookieService: CookiesService
   ) {
     this.multiClusterService.subscribe((resp: any) => {
       const clustersConfig = resp['config'];
@@ -91,14 +93,13 @@ export class ApiInterceptorService implements HttpInterceptor {
       'api/multi-cluster/auth'
     ];
 
-    const token = localStorage.getItem('token_of_selected_cluster');
-
     if (
       !currentRoute.includes('login') &&
       !ALWAYS_TO_HUB_APIs.includes(request.url) &&
       apiUrl &&
       !apiUrl.includes(origin)
     ) {
+      const token = this.cookieService.getToken(localStorage.getItem('current_cluster_name'));
       reqWithVersion = reqWithVersion.clone({
         url: `${apiUrl}${reqWithVersion.url}`,
         setHeaders: {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.spec.ts
new file mode 100644
index 000000000000..49ab3d05ebcd
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.spec.ts
@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { CookiesService } from './cookie.service';
+
+describe('CookieService', () => {
+  let service: CookiesService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(CookiesService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.ts
new file mode 100644
index 000000000000..cdbbadeb1cfe
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/cookie.service.ts
@@ -0,0 +1,21 @@
+import { Injectable } from '@angular/core';
+import { CookieService } from 'ngx-cookie-service';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class CookiesService {
+  constructor(private cookieService: CookieService) {}
+
+  setToken(name: string, token: string) {
+    this.cookieService.set(name, token, null, null, null, true, 'Strict');
+  }
+
+  getToken(name: string): string {
+    return this.cookieService.get(name);
+  }
+
+  deleteToken(name: string) {
+    this.cookieService.delete(name);
+  }
+}

From 7aa972def9f53fd9ed2f708aa87ce83362765cbf Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 18 Mar 2024 10:54:24 +0800
Subject: [PATCH 2300/2492] crimson/os/seastore/btree: check for reserved ptrs
 when determining children stability

Fixes: https://tracker.ceph.com/issues/64957
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/btree/fixed_kv_btree.h |  8 ++++++--
 src/crimson/os/seastore/btree/fixed_kv_node.cc |  6 +++++-
 src/crimson/os/seastore/btree/fixed_kv_node.h  | 11 ++++++++---
 src/crimson/os/seastore/cached_extent.h        |  2 ++
 4 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/crimson/os/seastore/btree/fixed_kv_btree.h b/src/crimson/os/seastore/btree/fixed_kv_btree.h
index 2d758ef4be2d..f15682621fb7 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_btree.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_btree.h
@@ -15,8 +15,6 @@
 #include "crimson/os/seastore/btree/btree_range_pin.h"
 #include "crimson/os/seastore/root_block.h"
 
-#define RESERVATION_PTR reinterpret_cast<ChildableCachedExtent*>(0x1)
-
 namespace crimson::os::seastore::lba_manager::btree {
 struct lba_map_val_t;
 }
@@ -25,6 +23,12 @@ namespace crimson::os::seastore {
 
 bool is_valid_child_ptr(ChildableCachedExtent* child);
 
+bool is_reserved_ptr(ChildableCachedExtent* child);
+
+inline ChildableCachedExtent* get_reserved_ptr() {
+  return (ChildableCachedExtent*)0x1;
+}
+
 template <typename T>
 phy_tree_root_t& get_phy_tree_root(root_t& r);
 
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.cc b/src/crimson/os/seastore/btree/fixed_kv_node.cc
index 00aceab92b38..94783a010910 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.cc
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.cc
@@ -6,7 +6,11 @@
 namespace crimson::os::seastore {
 
 bool is_valid_child_ptr(ChildableCachedExtent* child) {
-  return child != nullptr && child != RESERVATION_PTR;
+  return child != nullptr && child != get_reserved_ptr();
+}
+
+bool is_reserved_ptr(ChildableCachedExtent* child) {
+  return child == get_reserved_ptr();
 }
 
 } // namespace crimson::os::seastore
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index 544727f30aac..de3569c0c5bc 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -134,7 +134,7 @@ struct FixedKVNode : ChildableCachedExtent {
       // copy sources when committing this lba node, because
       // we rely on pointers' "nullness" to avoid copying
       // pointers for updated values
-      children[offset] = RESERVATION_PTR;
+      children[offset] = get_reserved_ptr();
     }
   }
 
@@ -431,7 +431,7 @@ struct FixedKVNode : ChildableCachedExtent {
 	if (!child) {
 	  child = source.children[foreign_it.get_offset()];
 	  // child can be either valid if present, nullptr if absent,
-	  // or RESERVATION_PTR.
+	  // or reserved ptr.
 	}
 	foreign_it++;
 	local_it++;
@@ -972,6 +972,7 @@ struct FixedKVLeafNode
   get_child_ret_t<LogicalCachedExtent>
   get_logical_child(op_context_t<NODE_KEY> c, uint16_t pos) final {
     auto child = this->children[pos];
+    ceph_assert(!is_reserved_ptr(child));
     if (is_valid_child_ptr(child)) {
       ceph_assert(child->is_logical());
       return c.cache.template get_extent_viewable_by_trans<
@@ -996,9 +997,13 @@ struct FixedKVLeafNode
   // children are considered stable if any of the following case is true:
   // 1. The child extent is absent in cache
   // 2. The child extent is stable
+  //
+  // For reserved mappings, the return values are undefined.
   bool is_child_stable(uint16_t pos) const final {
     auto child = this->children[pos];
-    if (is_valid_child_ptr(child)) {
+    if (is_reserved_ptr(child)) {
+      return true;
+    } else if (is_valid_child_ptr(child)) {
       ceph_assert(child->is_logical());
       return child->is_stable();
     } else if (this->is_pending()) {
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 35753d101bd2..18ddbd95796c 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -1062,6 +1062,8 @@ class PhysicalNodeMapping {
     child_pos->link_child(c);
   }
 
+  // For reserved mappings, the return values are
+  // undefined although it won't crash
   virtual bool is_stable() const = 0;
   virtual bool is_clone() const = 0;
   bool is_zero_reserved() const {

From b1c59ca077597e14a053d47d4961b12ba7a2a1b0 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Mon, 18 Mar 2024 17:24:02 +0800
Subject: [PATCH 2301/2492] crimson/os/seastore/cached_extent: add comments to
 elaborate why MUTATION_PENDING and under-io extents are "stable"

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/cached_extent.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 18ddbd95796c..730a0ace9a0b 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -426,6 +426,9 @@ class CachedExtent
   /// Returns true if extent is stable and shared among transactions
   bool is_stable() const {
     return is_stable_written() ||
+	   // MUTATION_PENDING and under-io extents are to-be-stable extents,
+	   // for the sake of caveats that checks the correctness of extents
+	   // states, we consider them stable.
            (is_mutation_pending() &&
             is_pending_io());
   }

From 7f253ce5aa98018b5b209fd6e062078d151ad567 Mon Sep 17 00:00:00 2001
From: myoungwon oh <ohmyoungwon@gmail.com>
Date: Mon, 18 Mar 2024 06:48:07 +0000
Subject: [PATCH 2302/2492] crimson/os/seastore: cache metadata during trimming
 to prevent from disk read

I encountered continous disk reads during trimming even though there are sufficient
cache available, in 4K random write test with RBM (RBD).
This is because metadata is note cached if its source is background transaction
within touch_extent(). So, seastore, including the trimming process,  needs to
constantly retrieve metadata (e.g., BACKREF_LEAF).

Signed-off-by: Myoungwon Oh <myoungwon.oh@samsung.com>
---
 src/crimson/os/seastore/cache.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 75463f465fe1..829d57da1931 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -1342,8 +1342,11 @@ class Cache {
       CachedExtent &ext,
       const Transaction::src_t* p_src=nullptr)
   {
-    if (p_src && is_background_transaction(*p_src))
+    if (p_src &&
+	is_background_transaction(*p_src) &&
+	is_logical_type(ext.get_type())) {
       return;
+    }
     if (ext.is_stable_clean() && !ext.is_placeholder()) {
       lru.move_to_top(ext);
     }

From 5dcff6a4b8d835fc55e454af977dc5ebad99d37f Mon Sep 17 00:00:00 2001
From: N Balachandran <nibalach@redhat.com>
Date: Mon, 18 Mar 2024 17:52:47 +0530
Subject: [PATCH 2303/2492] doc/rbd: minor changes to the rbd man page

Fixes typos and grammar for some commands. Adds
additional details for some commandds.

Signed-off-by: N Balachandran <nibalach@redhat.com>
---
 doc/man/8/rbd.rst | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
index 5e7e691c8ca8..881e891d76b2 100644
--- a/doc/man/8/rbd.rst
+++ b/doc/man/8/rbd.rst
@@ -333,7 +333,7 @@ Commands
   be specified.
 
 :command:`flatten` [--encryption-format *encryption-format* --encryption-passphrase-file *passphrase-file*]... *image-spec*
-  If image is a clone, copy all shared blocks from the parent snapshot and
+  If the image is a clone, copy all shared blocks from the parent snapshot and
   make the child independent of the parent, severing the link between
   parent snap and child.  The parent snapshot can be unprotected and
   deleted if it has no further dependent clones.
@@ -390,7 +390,7 @@ Commands
   Set metadata key with the value. They will displayed in `image-meta list`.
 
 :command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
-  Create a new image and imports its data from path (use - for
+  Create a new image and import its data from path (use - for
   stdin).  The import operation will try to create sparse rbd images 
   if possible.  For import from stdin, the sparsification unit is
   the data block size of the destination image (object size).
@@ -402,14 +402,14 @@ Commands
   of image, but also the snapshots and other properties, such as image_order, features.
 
 :command:`import-diff` *src-path* *image-spec*
-  Import an incremental diff of an image and applies it to the current image.  If the diff
+  Import an incremental diff of an image and apply it to the current image.  If the diff
   was generated relative to a start snapshot, we verify that snapshot already exists before
   continuing.  If there was an end snapshot we verify it does not already exist before
   applying the changes, and create the snapshot when we are done.
   
 :command:`info` *image-spec* | *snap-spec*
   Will dump information (such as size and object size) about a specific rbd image.
-  If image is a clone, information about its parent is also displayed.
+  If the image is a clone, information about its parent is also displayed.
   If a snapshot is specified, whether it is protected is shown as well.
 
 :command:`journal client disconnect` *journal-spec*
@@ -472,7 +472,7 @@ Commands
   the destination image are lost.
 
 :command:`migration commit` *image-spec*
-  Commit image migration. This step is run after a successful migration
+  Commit image migration. This step is run after successful migration
   prepare and migration execute steps and removes the source image data.
 
 :command:`migration execute` *image-spec*
@@ -499,14 +499,12 @@ Commands
 :command:`mirror image disable` [--force] *image-spec*
   Disable RBD mirroring for an image. If the mirroring is
   configured in ``image`` mode for the image's pool, then it
-  can be explicitly disabled mirroring for each image within
-  the pool.
+  must be disabled for each image individually.
 
 :command:`mirror image enable` *image-spec* *mode*
   Enable RBD mirroring for an image. If the mirroring is
   configured in ``image`` mode for the image's pool, then it
-  can be explicitly enabled mirroring for each image within
-  the pool.
+  must be enabled for each image individually.
 
   The mirror image mode can either be ``journal`` (default) or
   ``snapshot``. The ``journal`` mode requires the RBD journaling
@@ -523,7 +521,7 @@ Commands
 
 :command:`mirror pool demote` [*pool-name*]
   Demote all primary images within a pool to non-primary.
-  Every mirroring enabled image will demoted in the pool.
+  Every mirror-enabled image in the pool will be demoted.
 
 :command:`mirror pool disable` [*pool-name*]
   Disable RBD mirroring by default within a pool. When mirroring
@@ -551,7 +549,7 @@ Commands
 
   The default for *remote client name* is "client.admin".
 
-  This requires mirroring mode is enabled.
+  This requires mirroring to be enabled on the pool.
 
 :command:`mirror pool peer remove` [*pool-name*] *uuid*
   Remove a mirroring peer from a pool. The peer uuid is available
@@ -564,12 +562,12 @@ Commands
 
 :command:`mirror pool promote` [--force] [*pool-name*]
   Promote all non-primary images within a pool to primary.
-  Every mirroring enabled image will promoted in the pool.
+  Every mirror-enabled image in the pool will be promoted.
 
 :command:`mirror pool status` [--verbose] [*pool-name*]
   Show status for all mirrored images in the pool.
-  With --verbose, also show additionally output status
-  details for every mirroring image in the pool.
+  With ``--verbose``, show additional output status
+  details for every mirror-enabled image in the pool.
 
 :command:`mirror snapshot schedule add` [-p | --pool *pool*] [--namespace *namespace*] [--image *image*] *interval* [*start-time*]
   Add mirror snapshot schedule.
@@ -603,7 +601,7 @@ Commands
   specified to rebuild an invalid object map for a snapshot.
 
 :command:`pool init` [*pool-name*] [--force]
-  Initialize pool for use by RBD. Newly created pools must initialized
+  Initialize pool for use by RBD. Newly created pools must be initialized
   prior to use.
 
 :command:`resize` (-s | --size *size-in-M/G/T*) [--allow-shrink] [--encryption-format *encryption-format* --encryption-passphrase-file *passphrase-file*]... *image-spec*
@@ -615,7 +613,7 @@ Commands
   snapshots, this fails and nothing is deleted.
 
 :command:`snap create` *snap-spec*
-  Create a new snapshot. Requires the snapshot name parameter specified.
+  Create a new snapshot. Requires the snapshot name parameter to be specified.
 
 :command:`snap limit clear` *image-spec*
   Remove any previously set limit on the number of snapshots allowed on
@@ -625,7 +623,7 @@ Commands
   Set a limit for the number of snapshots allowed on an image.
 
 :command:`snap ls` *image-spec*
-  Dump the list of snapshots inside a specific image.
+  Dump the list of snapshots of a specific image.
 
 :command:`snap protect` *snap-spec*
   Protect a snapshot from deletion, so that clones can be made of it
@@ -668,9 +666,11 @@ Commands
 :command:`trash ls` [*pool-name*]
   List all entries from trash.
 
-:command:`trash mv` *image-spec*
+:command:`trash mv` [--expires-at <expires-at>] *image-spec*
   Move an image to the trash. Images, even ones actively in-use by 
-  clones, can be moved to the trash and deleted at a later time.
+  clones, can be moved to the trash and deleted at a later time. Use
+  ``--expires-at`` to set the expiration time of an image after which
+  it's allowed to be removed.
 
 :command:`trash purge` [*pool-name*]
   Remove all expired images from trash.
@@ -678,10 +678,10 @@ Commands
 :command:`trash restore` *image-id*  
   Restore an image from trash.
 
-:command:`trash rm` *image-id* 
-  Delete an image from trash. If image deferment time has not expired
-  you can not removed it unless use force. But an actively in-use by clones 
-  or has snapshots can not be removed.
+:command:`trash rm` [--force] *image-id*
+  Delete an image from trash. If the image deferment time has not expired
+  it can be removed using ``--force``. An image that is actively in-use by clones
+  or has snapshots cannot be removed.
 
 :command:`trash purge schedule add` [-p | --pool *pool*] [--namespace *namespace*] *interval* [*start-time*]
   Add trash purge schedule.

From ead944be18781ddccd52af5aa5cff414d15bb39d Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
Date: Sun, 10 Mar 2024 17:36:37 -0400
Subject: [PATCH 2304/2492] doc/cephadm/services: Improve monitoring.rst

Signed-off-by: Anthony D'Atri <anthonyeleven@users.noreply.github.com>
---
 doc/cephadm/services/monitoring.rst | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst
index 985661469ff6..87092dc5cd5d 100644
--- a/doc/cephadm/services/monitoring.rst
+++ b/doc/cephadm/services/monitoring.rst
@@ -86,29 +86,33 @@ steps below:
 Enabling security for the monitoring stack
 ----------------------------------------------
 
-By default, in a cephadm managed cluster, the monitoring components are set up and configured without incorporating any security measures.
-While this setup might suffice for certain deployments, other users with stricter security needs may find it necessary to protect their
-monitoring stack against unauthorized access to metrics and data. In such cases, cephadm relies on a specific configuration parameter,
+By default, in a cephadm-managed cluster, the monitoring components are set up and configured without enabling security measures.
+While this suffices for certain deployments, others with strict security needs may find it necessary to protect the
+monitoring stack against unauthorized access. In such cases, cephadm relies on a specific configuration parameter,
 `mgr/cephadm/secure_monitoring_stack`, which toggles the security settings for all monitoring components. To activate security
-measures, users must set this variable to true, as following:
+measures, set this option to ``true`` with a command of the following form:
 
    .. prompt:: bash #
 
      ceph config set mgr mgr/cephadm/secure_monitoring_stack true
 
-This configuration change will trigger a sequence of reconfigurations across all monitoring daemons, typically requiring
+This change will trigger a sequence of reconfigurations across all monitoring daemons, typically requiring
 few minutes until all components are fully operational. The updated secure configuration includes the following modifications:
 
-#. Prometheus: basic authentication is requiered to access the web portal and TLS is enabled for secure communication.
-#. Alertmanager: basic authentication is requiered to access the web portal and TLS is enabled for secure communication.
+#. Prometheus: basic authentication is required to access the web portal and TLS is enabled for secure communication.
+#. Alertmanager: basic authentication is required to access the web portal and TLS is enabled for secure communication.
 #. Node Exporter: TLS is enabled for secure communication.
 #. Grafana: TLS is enabled and authentication is requiered to access the datasource information.
 
-In this secure setup, users will need to setup authentication (username/password) for both Prometheus and Alertmanager. By default user/password are
-set to admin/admin. The user can change these value through the commands `orch prometheus set-credentials` and `orch alertmanager set-credentials`
-respectively. These commands offer the flexibility to input the username/password either as parameters or via a JSON file, which enhances security. Additionally,
-Cephadm provides commands such as `orch prometheus get-credentials` and `orch alertmanager get-credentials` to retrieve the currently configured credentials such
-as default values.
+In this secure setup, users will need to setup authentication
+(username/password) for both Prometheus and Alertmanager. By default the
+username and password are set to ``admin``/``admin``. The user can change these
+value with the commands ``ceph orch prometheus set-credentials`` and ``ceph
+orch alertmanager set-credentials`` respectively. These commands offer the
+flexibility to input the username/password either as parameters or via a JSON
+file, which enhances security. Additionally, Cephadm provides the commands
+`orch prometheus get-credentials` and `orch alertmanager get-credentials` to
+retrieve the current credentials.
 
 .. _cephadm-monitoring-centralized-logs:
 

From 9234fbcbe65b227b7e9b1d639fda1bf3b5e7a92a Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 18 Mar 2024 10:47:59 -0400
Subject: [PATCH 2305/2492] qa/rgw: ignore POOL_APP_NOT_ENABLED in rgw/lua
 subsuite

> cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log

Signed-off-by: Casey Bodley <cbodley@redhat.com>
---
 qa/suites/rgw/lua/ignore-pg-availability.yaml | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 qa/suites/rgw/lua/ignore-pg-availability.yaml

diff --git a/qa/suites/rgw/lua/ignore-pg-availability.yaml b/qa/suites/rgw/lua/ignore-pg-availability.yaml
new file mode 120000
index 000000000000..32340b1fa8be
--- /dev/null
+++ b/qa/suites/rgw/lua/ignore-pg-availability.yaml
@@ -0,0 +1 @@
+.qa/rgw/ignore-pg-availability.yaml
\ No newline at end of file

From 6d561092faf1af5158f438af2a585197bf44683e Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Mon, 19 Feb 2024 08:50:41 -0600
Subject: [PATCH 2306/2492] osd: improved get_objects_by_prefixes() ergonomics

Improved call signatures for get_next_objects_to_trim() &
get_objects_by_prefixes().

Also: as as get_next_objects_to_trim() has only a single
failure mode, we should not try to handle two distinct failures
in its callers' code.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 .../osd/osd_operations/snaptrim_event.cc      | 21 +++------
 src/osd/PrimaryLogPG.cc                       | 20 +++------
 src/osd/SnapMapper.cc                         | 45 +++++++++----------
 src/osd/SnapMapper.h                          | 14 +++---
 src/test/test_snap_mapper.cc                  | 18 ++++----
 5 files changed, 49 insertions(+), 69 deletions(-)

diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index 1a2292ae90fd..aaa432e295f9 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -88,28 +88,19 @@ SnapTrimEvent::start()
         client_pp().process);
     }).then_interruptible([&shard_services, this] {
       return interruptor::async([this] {
-        std::vector<hobject_t> to_trim;
         using crimson::common::local_conf;
         const auto max =
           local_conf().get_val<uint64_t>("osd_pg_max_concurrent_snap_trims");
         // we need to look for at least 1 snaptrim, otherwise we'll misinterpret
-        // the ENOENT below and erase snapid.
-        int r = snap_mapper.get_next_objects_to_trim(
+        // the nullopt below and erase snapid.
+        auto to_trim = snap_mapper.get_next_objects_to_trim(
           snapid,
-          max,
-          &to_trim);
-        if (r == -ENOENT) {
-          to_trim.clear(); // paranoia
-          return to_trim;
-        } else if (r != 0) {
-          logger().error("{}: get_next_objects_to_trim returned {}",
-                         *this, cpp_strerror(r));
-          ceph_abort_msg("get_next_objects_to_trim returned an invalid code");
-        } else {
-          assert(!to_trim.empty());
+          max);
+        if (!to_trim.has_value()) {
+          return std::vector<hobject_t>{};
         }
         logger().debug("{}: async almost done line {}", *this, __LINE__);
-        return to_trim;
+        return std::move(*to_trim);
       }).then_interruptible([&shard_services, this] (const auto& to_trim) {
         if (to_trim.empty()) {
           // the legit ENOENT -> done
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index f3b89440425d..62b8aad27d5b 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -15707,23 +15707,16 @@ boost::statechart::result PrimaryLogPG::AwaitAsyncWork::react(const DoSnapWork&)
 
   ldout(pg->cct, 10) << "AwaitAsyncWork: trimming snap " << snap_to_trim << dendl;
 
-  vector<hobject_t> to_trim;
   unsigned max = pg->cct->_conf->osd_pg_max_concurrent_snap_trims;
   // we need to look for at least 1 snaptrim, otherwise we'll misinterpret
   // the ENOENT below and erase snap_to_trim.
   ceph_assert(max > 0);
-  to_trim.reserve(max);
-  int r = pg->snap_mapper.get_next_objects_to_trim(
-    snap_to_trim,
-    max,
-    &to_trim);
-  if (r != 0 && r != -ENOENT) {
-    lderr(pg->cct) << "get_next_objects_to_trim returned "
-		   << cpp_strerror(r) << dendl;
-    ceph_abort_msg("get_next_objects_to_trim returned an invalid code");
-  } else if (r == -ENOENT) {
+
+  auto to_trim =
+      pg->snap_mapper.get_next_objects_to_trim(snap_to_trim, max);
+  if (!to_trim.has_value()) {
     // Done!
-    ldout(pg->cct, 10) << "got ENOENT" << dendl;
+    ldout(pg->cct, 10) << "no more entries to trim" << dendl;
 
     pg->snap_trimq.erase(snap_to_trim);
 
@@ -15754,9 +15747,8 @@ boost::statechart::result PrimaryLogPG::AwaitAsyncWork::react(const DoSnapWork&)
     pg->set_snaptrim_duration();
     return transit< NotTrimming >();
   }
-  ceph_assert(!to_trim.empty());
 
-  for (auto &&object: to_trim) {
+  for (auto &&object: *to_trim) {
     // Get next
     ldout(pg->cct, 10) << "AwaitAsyncWork react trimming " << object << dendl;
     OpContextUPtr ctx;
diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index c16848272209..e6a2b4d7d557 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -576,27 +576,27 @@ void SnapMapper::reset_prefix_itr(snapid_t snap, const char *s)
   prefix_itr      = prefixes.begin();
 }
 
-void SnapMapper::get_objects_by_prefixes(
+vector<hobject_t> SnapMapper::get_objects_by_prefixes(
   snapid_t snap,
-  unsigned max,
-  vector<hobject_t> *out)
+  unsigned max)
 {
+  vector<hobject_t> out;
+
   /// maintain the prefix_itr between calls to avoid searching depleted prefixes
   for ( ; prefix_itr != prefixes.end(); prefix_itr++) {
-    string prefix(get_prefix(pool, snap) + *prefix_itr);
+    const string prefix(get_prefix(pool, snap) + *prefix_itr);
     string pos = prefix;
-    while (out->size() < max) {
+    while (out.size() < max) {
       pair<string, ceph::buffer::list> next;
       // access RocksDB (an expensive operation!)
       int r = backend.get_next(pos, &next);
       dout(20) << __func__ << " get_next(" << pos << ") returns " << r
 	       << " " << next << dendl;
       if (r != 0) {
-	return; // Done
+	return out; // Done
       }
 
-      if (next.first.substr(0, prefix.size()) !=
-	  prefix) {
+      if (!next.first.starts_with(prefix)) {
 	// TBD: we access the DB twice for the first object of each iterator...
 	break; // Done with this prefix
       }
@@ -608,24 +608,22 @@ void SnapMapper::get_objects_by_prefixes(
       ceph_assert(next_decoded.first == snap);
       ceph_assert(check(next_decoded.second));
 
-      out->push_back(next_decoded.second);
+      out.push_back(next_decoded.second);
       pos = next.first;
     }
 
-    if (out->size() >= max) {
-      return;
+    if (out.size() >= max) {
+      return out;
     }
   }
+  return out;
 }
 
-int SnapMapper::get_next_objects_to_trim(
+std::optional<vector<hobject_t>> SnapMapper::get_next_objects_to_trim(
   snapid_t snap,
-  unsigned max,
-  vector<hobject_t> *out)
+  unsigned max)
 {
   dout(20) << __func__ << "::snapid=" << snap << dendl;
-  ceph_assert(out);
-  ceph_assert(out->empty());
 
   // if max would be 0, we return ENOENT and the caller would mistakenly
   // trim the snaptrim queue
@@ -649,25 +647,26 @@ int SnapMapper::get_next_objects_to_trim(
   // For more info see PG::filter_snapc()
   //
   // We still like to be extra careful and run one extra loop over all prefixes
-  get_objects_by_prefixes(snap, max, out);
-  if (unlikely(out->size() == 0)) {
+  auto objs = get_objects_by_prefixes(snap, max);
+  if (unlikely(objs.size() == 0)) {
     reset_prefix_itr(snap, "Second pass trim");
-    get_objects_by_prefixes(snap, max, out);
+    objs = get_objects_by_prefixes(snap, max);
 
-    if (unlikely(out->size() > 0)) {
+    if (unlikely(objs.size() > 0)) {
       derr << __func__ << "::New Clone-Objects were added to Snap " << snap
 	   << " after trimming was started" << dendl;
     }
     reset_prefix_itr(CEPH_NOSNAP, "Trim was completed successfully");
   }
 
-  if (out->size() == 0) {
-    return -ENOENT;
+  if (objs.size() == 0) {
+    return std::nullopt;
   } else {
-    return 0;
+    return objs;
   }
 }
 
+
 int SnapMapper::remove_oid(
   const hobject_t &oid,
   MapCacher::Transaction<std::string, ceph::buffer::list> *t)
diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h
index 70d1b6c39add..a90faa8e84fc 100644
--- a/src/osd/SnapMapper.h
+++ b/src/osd/SnapMapper.h
@@ -291,11 +291,10 @@ class SnapMapper : public Scrub::SnapMapReaderI {
   tl::expected<object_snaps, SnapMapReaderI::result_t> get_snaps_common(
     const hobject_t &hoid) const;
 
-  /// file @out vector with the first objects with @snap as a snap
-  void get_objects_by_prefixes(
+  /// \returns vector with the first objects with @snap as a snap
+  std::vector<hobject_t> get_objects_by_prefixes(
     snapid_t snap,
-    unsigned max,
-    std::vector<hobject_t> *out);
+    unsigned max);
 
   std::set<std::string>           prefixes;
   // maintain a current active prefix
@@ -373,11 +372,10 @@ class SnapMapper : public Scrub::SnapMapReaderI {
     );
 
   /// Returns first object with snap as a snap
-  int get_next_objects_to_trim(
+  std::optional<std::vector<hobject_t>> get_next_objects_to_trim(
     snapid_t snap,              ///< [in] snap to check
-    unsigned max,               ///< [in] max to get
-    std::vector<hobject_t> *out      ///< [out] next objects to trim (must be empty)
-    );  ///< @return error, -ENOENT if no more objects
+    unsigned max                ///< [in] max to get
+    );  ///< @return nullopt if no more objects
 
   /// Remove mapping for oid
   int remove_oid(
diff --git a/src/test/test_snap_mapper.cc b/src/test/test_snap_mapper.cc
index 9fe726afb1f9..a47d2538c3a6 100644
--- a/src/test/test_snap_mapper.cc
+++ b/src/test/test_snap_mapper.cc
@@ -558,18 +558,18 @@ class MapperVerifier {
   // must be called with lock held to protect access to
   // snap_to_hobject and hobject_to_snap
   int trim_snap(snapid_t snapid, unsigned max_count, vector<hobject_t> & out) {
-    set<hobject_t>&   hobjects = snap_to_hobject[snapid];
-    vector<hobject_t> hoids;
-    int ret = mapper->get_next_objects_to_trim(snapid, max_count, &hoids);
-    if (ret == 0) {
-      out.insert(out.end(), hoids.begin(), hoids.end());
-      for (auto &&hoid: hoids) {
+
+    set<hobject_t>& hobjects = snap_to_hobject[snapid];
+    auto hoids = mapper->get_next_objects_to_trim(snapid, max_count);
+    if (hoids.has_value()) {
+      out.insert(out.end(), hoids->begin(), hoids->end());
+      for (auto &&hoid: *hoids) {
 	ceph_assert(!hoid.is_max());
 	ceph_assert(hobjects.count(hoid));
 	hobjects.erase(hoid);
 
 	map<hobject_t, set<snapid_t>>::iterator j = hobject_to_snap.find(hoid);
-	ceph_assert(j->second.count(snapid));
+	ceph_assert(j->second.contains(snapid));
 	set<snapid_t> old_snaps(j->second);
 	j->second.erase(snapid);
 
@@ -587,9 +587,9 @@ class MapperVerifier {
 	}
 	hoid = hobject_t::get_max();
       }
-      hoids.clear();
+      return 0;
     }
-    return ret;
+    return -1;
   }
 
   // must be called with lock held to protect access to

From 9ef43722ad94ea688fa7f70daab6dd41fac6ee53 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Mon, 12 Feb 2024 09:31:52 +0000
Subject: [PATCH 2307/2492] osd/SnapMapper: add common logs

Signed-off-by: Matan Breizman <mbreizma@redhat.com>
(cherry picked from commit f302be018ebef365d8876d14185c4bdb246f22d8)
---
 src/osd/SnapMapper.cc | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc
index e6a2b4d7d557..2a7ee5e51cc4 100644
--- a/src/osd/SnapMapper.cc
+++ b/src/osd/SnapMapper.cc
@@ -591,18 +591,22 @@ vector<hobject_t> SnapMapper::get_objects_by_prefixes(
       // access RocksDB (an expensive operation!)
       int r = backend.get_next(pos, &next);
       dout(20) << __func__ << " get_next(" << pos << ") returns " << r
-	       << " " << next << dendl;
+	       << " " << next.first << dendl;
       if (r != 0) {
 	return out; // Done
       }
 
-      if (!next.first.starts_with(prefix)) {
+      ceph_assert(is_mapping(next.first));
+
+      if (auto next_prefix = next.first.substr(0, prefix.size());
+          next_prefix != prefix) {
 	// TBD: we access the DB twice for the first object of each iterator...
+	dout(20) << fmt::format("{}: breaking, prefix expected {} got {}",
+	                        __func__, prefix, next_prefix)
+	         << dendl;
 	break; // Done with this prefix
       }
 
-      ceph_assert(is_mapping(next.first));
-
       dout(20) << __func__ << " " << next.first << dendl;
       pair<snapid_t, hobject_t> next_decoded(from_raw(next));
       ceph_assert(next_decoded.first == snap);
@@ -613,6 +617,9 @@ vector<hobject_t> SnapMapper::get_objects_by_prefixes(
     }
 
     if (out.size() >= max) {
+      dout(20) << fmt::format("{}: reached max of: {} returning",
+                              __func__, out.size())
+               << dendl;
       return out;
     }
   }

From b623594c44df1c7d75fa7a8d4e8b90f8829d249f Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Mon, 18 Mar 2024 23:26:36 +0000
Subject: [PATCH 2308/2492] crimson/osd/osd.h: declare osdmap after
 OSDSingletonState

Otherwise, destructing the osdmap cached_map_t tries to manipulate
OSDSingletonState::osdmaps after it has already been destroyed.

Fixes: https://tracker.ceph.com/issues/64935
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index db30ad0ec737..fa3b0293072b 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -80,7 +80,6 @@ class OSD final : public crimson::net::Dispatcher,
   std::unique_ptr<crimson::mgr::Client> mgrc;
 
   // TODO: use a wrapper for ObjectStore
-  OSDMapService::cached_map_t osdmap;
   crimson::os::FuturizedStore& store;
 
   /// _first_ epoch we were marked up (after this process started)
@@ -122,6 +121,8 @@ class OSD final : public crimson::net::Dispatcher,
   seastar::sharded<OSDState> osd_states;
   seastar::sharded<ShardServices> shard_services;
 
+  OSDMapService::cached_map_t osdmap;
+
   crimson::osd::PGShardManager pg_shard_manager;
 
   std::unique_ptr<Heartbeat> heartbeat;

From 1e1ac13c74331998d8b9cb0477df50eb0b75a8ab Mon Sep 17 00:00:00 2001
From: julpark <yonhyun@gmail.com>
Date: Mon, 18 Mar 2024 19:01:18 -0700
Subject: [PATCH 2309/2492] MDS metrics typo fix

Signed-off-by: julpark <yonhyun@gmail.com>
---
 src/mds/MetricAggregator.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MetricAggregator.cc b/src/mds/MetricAggregator.cc
index 9765d4d5b517..f2e520792b5a 100644
--- a/src/mds/MetricAggregator.cc
+++ b/src/mds/MetricAggregator.cc
@@ -68,7 +68,7 @@ int MetricAggregator::init() {
 							{"id", stringify(mds->get_global_id())}});
   PerfCountersBuilder plb(m_cct, labels, l_mds_client_metrics_start, l_mds_client_metrics_last);
   plb.add_u64(l_mds_client_metrics_num_clients,
-	      "num_clients", "Numer of client sessions", "mcli", PerfCountersBuilder::PRIO_CRITICAL);
+	      "num_clients", "Number of client sessions", "mcli", PerfCountersBuilder::PRIO_CRITICAL);
   m_perf_counters = plb.create_perf_counters();
   m_cct->get_perfcounters_collection()->add(m_perf_counters);
 

From b35ed71e23ac5eabfed1c47361ee499bab77cd14 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 19 Mar 2024 11:05:11 +0800
Subject: [PATCH 2310/2492] crimson/osd/pg_map/PGShardMapping: cleanups

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/pg_map.cc | 50 ++++++++++++++++++++++-----------------
 src/crimson/osd/pg_map.h  |  4 +++-
 2 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index c0a81f300b3d..c9cd09da0649 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -14,50 +14,55 @@ namespace crimson::osd {
 
 seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
   spg_t pgid,
-  core_id_t core)
+  core_id_t core_expected)
 {
   LOG_PREFIX(PGShardMapping::get_or_create_pg_mapping);
   auto find_iter = pg_to_core.find(pgid);
   if (find_iter != pg_to_core.end()) {
     ceph_assert_always(find_iter->second != NULL_CORE);
-    if (core != NULL_CORE) {
-      ceph_assert_always(find_iter->second == core);
+    if (core_expected != NULL_CORE) {
+      ceph_assert_always(find_iter->second == core_expected);
     }
     return seastar::make_ready_future<core_id_t>(find_iter->second);
   } else {
-    return container().invoke_on(0,[pgid, core, FNAME]
-      (auto &primary_mapping) {
-      auto [insert_iter, inserted] = primary_mapping.pg_to_core.emplace(pgid, core);
+    return container().invoke_on(
+        0, [pgid, core_expected, FNAME](auto &primary_mapping) {
+      auto [insert_iter, inserted] =
+        primary_mapping.pg_to_core.emplace(pgid, core_expected);
       ceph_assert_always(inserted);
       ceph_assert_always(primary_mapping.core_to_num_pgs.size() > 0);
       std::map<core_id_t, unsigned>::iterator core_iter;
-      if (core == NULL_CORE) {
+      if (core_expected == NULL_CORE) {
         core_iter = std::min_element(
           primary_mapping.core_to_num_pgs.begin(),
           primary_mapping.core_to_num_pgs.end(),
-            [](const auto &left, const auto &right) {
+          [](const auto &left, const auto &right) {
             return left.second < right.second;
-        });
+          }
+        );
+        core_expected = core_iter->first;
       } else {
-        core_iter = primary_mapping.core_to_num_pgs.find(core);
+        core_iter = primary_mapping.core_to_num_pgs.find(core_expected);
       }
+      assert(core_expected != NULL_CORE);
       ceph_assert_always(primary_mapping.core_to_num_pgs.end() != core_iter);
-      insert_iter->second = core_iter->first;
+      insert_iter->second = core_expected;
       core_iter->second++;
-      DEBUG("mapping pg {} to core: {} with num_pgs of: {}",
-            pgid, insert_iter->second, core_iter->second);
+      DEBUG("mapping pg {} to core {} (primary) with num_pgs {}",
+            pgid, core_expected, core_iter->second);
       return primary_mapping.container().invoke_on_others(
-        [pgid = insert_iter->first, core = insert_iter->second, FNAME]
-        (auto &other_mapping) {
-        ceph_assert_always(core != NULL_CORE);
-        auto [insert_iter, inserted] = other_mapping.pg_to_core.emplace(pgid, core);
+          [pgid, core_expected, FNAME](auto &other_mapping) {
+        auto [insert_iter, inserted] =
+          other_mapping.pg_to_core.emplace(pgid, core_expected);
         ceph_assert_always(inserted);
-        DEBUG("mapping pg {} to core: {}", pgid, core);
+        DEBUG("mapping pg {} to core {} (others)",
+              pgid, core_expected);
       });
     }).then([this, pgid, FNAME] {
       auto find_iter = pg_to_core.find(pgid);
       ceph_assert_always(find_iter != pg_to_core.end());
-      DEBUG("returning pg {} mapping to core {}", pgid, find_iter->second);
+      DEBUG("returning pg {} mapping to core {}",
+            pgid, find_iter->second);
       return seastar::make_ready_future<core_id_t>(find_iter->second);
     });
   }
@@ -66,7 +71,8 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
 seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
   LOG_PREFIX(PGShardMapping::remove_pg_mapping);
   DEBUG("{}", pgid);
-  return container().invoke_on(0, [pgid, FNAME](auto &primary_mapping) {
+  return container().invoke_on(
+      0, [pgid, FNAME](auto &primary_mapping) {
     auto iter = primary_mapping.pg_to_core.find(pgid);
     ceph_assert_always(iter != primary_mapping.pg_to_core.end());
     ceph_assert_always(iter->second != NULL_CORE);
@@ -75,14 +81,14 @@ seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
     ceph_assert_always(count_iter->second > 0);
     --(count_iter->second);
     primary_mapping.pg_to_core.erase(iter);
-    DEBUG("pg {} mapping erased", pgid);
+    DEBUG("pg {} mapping erased (primary)", pgid);
     return primary_mapping.container().invoke_on_others(
       [pgid, FNAME](auto &other_mapping) {
       auto iter = other_mapping.pg_to_core.find(pgid);
       ceph_assert_always(iter != other_mapping.pg_to_core.end());
       ceph_assert_always(iter->second != NULL_CORE);
       other_mapping.pg_to_core.erase(iter);
-      DEBUG("pg {} mapping erased", pgid);
+      DEBUG("pg {} mapping erased (others)", pgid);
     });
   });
 }
diff --git a/src/crimson/osd/pg_map.h b/src/crimson/osd/pg_map.h
index 2ea3f239b9b6..ff33b5d8b858 100644
--- a/src/crimson/osd/pg_map.h
+++ b/src/crimson/osd/pg_map.h
@@ -37,7 +37,7 @@ class PGShardMapping : public seastar::peering_sharded_service<PGShardMapping> {
   /// Returns mapping for pgid, creates new one if it doesn't already exist
   seastar::future<core_id_t> get_or_create_pg_mapping(
     spg_t pgid,
-    core_id_t core = NULL_CORE);
+    core_id_t core_expected = NULL_CORE);
 
   /// Remove pgid mapping
   seastar::future<> remove_pg_mapping(spg_t pgid);
@@ -60,7 +60,9 @@ class PGShardMapping : public seastar::peering_sharded_service<PGShardMapping> {
   }
 
 private:
+  // only in shard 0
   std::map<core_id_t, unsigned> core_to_num_pgs;
+  // per-shard, updated by shard 0
   std::map<spg_t, core_id_t> pg_to_core;
 };
 

From 7582834d90c048765fbc0f7431e04eccc5e5ee68 Mon Sep 17 00:00:00 2001
From: Sachin Punadikar <sachin.punadikar@ibm.com>
Date: Tue, 19 Mar 2024 05:41:53 -0400
Subject: [PATCH 2311/2492] vstart: Ganesha should not be started in DEBUG mode

Currently vstart script, deploy NFS Ganesha in debug mode. Enabling
DEBUG mode for Ganesha leads to logging lot of debug messages, which may
not be required all the time. One can enable DEBUG mode on need basis.
Hence removing the default DEBUG mode.

Signed-off-by: Sachin Punadikar <sachin.punadikar@ibm.com>
---
 src/vstart.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vstart.sh b/src/vstart.sh
index dbb44cb434b2..06264325f82e 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1497,7 +1497,7 @@ EOF
         prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace add $name
         prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace
 
-        prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_DEBUG
+        prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_EVENT
 
         # Wait few seconds for grace period to be removed
         sleep 2

From 80e1207f4b536fe6edbc81e61cbf951e135eba54 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Wed, 21 Feb 2024 13:02:19 +0000
Subject: [PATCH 2312/2492] mgr/dashboard: fix retention add for subvolume

- Added parameters for subvolume and subvolume group when adding a new
snap schedule.
- Added call to remove retention policies when removing a snap schedule
  in case it is the last one with same path

Fixes: https://tracker.ceph.com/issues/64524

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../mgr/dashboard/controllers/cephfs.py       | 35 +++++++++++++++++--
 .../cephfs-snapshotschedule-form.component.ts |  3 +-
 .../cephfs-snapshotschedule-list.component.ts | 13 ++++++-
 .../api/cephfs-snapshot-schedule.service.ts   | 20 +++++++++--
 src/pybind/mgr/dashboard/openapi.yaml         |  5 +++
 5 files changed, 69 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/cephfs.py b/src/pybind/mgr/dashboard/controllers/cephfs.py
index 6410a73785ec..4210746fbd1f 100644
--- a/src/pybind/mgr/dashboard/controllers/cephfs.py
+++ b/src/pybind/mgr/dashboard/controllers/cephfs.py
@@ -1029,7 +1029,9 @@ def create(self, fs: str, path: str, snap_schedule: str, start: str, retention_p
                                                                     path,
                                                                     retention_spec_or_period,
                                                                     retention_count,
-                                                                    fs)
+                                                                    fs,
+                                                                    subvol,
+                                                                    group)
                 if error_code_retention != 0:
                     raise DashboardException(
                         f'Failed to add retention policy for path {path}: {err_retention}'
@@ -1071,7 +1073,36 @@ def editRetentionPolicies(method, retention_policy):
 
     @RESTController.Resource('DELETE')
     def delete_snapshot(self, fs: str, path: str, schedule: str, start: str,
-                        subvol=None, group=None):
+                        retention_policy=None, subvol=None, group=None):
+        if retention_policy:
+            # check if there are other snap schedules for this exact same path
+            error_code, out, err = mgr.remote('snap_schedule', 'snap_schedule_list',
+                                              path, False, fs, subvol, group, 'plain')
+
+            if error_code != 0:
+                raise DashboardException(
+                    f'Failed to get snapshot schedule list for path {path}: {err}'
+                )
+            # only remove the retention policies if there no other snap schedules for this path
+            snapshot_schedule_list = out.split('\n')
+            if len(snapshot_schedule_list) <= 1:
+                retention_policies = retention_policy.split('|')
+                for retention in retention_policies:
+                    retention_count = retention.split('-')[0]
+                    retention_spec_or_period = retention.split('-')[1]
+                    error_code, _, err = mgr.remote('snap_schedule',
+                                                    'snap_schedule_retention_rm',
+                                                    path,
+                                                    retention_spec_or_period,
+                                                    retention_count,
+                                                    fs,
+                                                    subvol,
+                                                    group)
+                    if error_code != 0:
+                        raise DashboardException(
+                            f'Failed to remove retention policy for path {path}: {err}'
+                        )
+        # remove snap schedule
         error_code, _, err = mgr.remote('snap_schedule',
                                         'snap_schedule_rm',
                                         path,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 0ef68f7d529c..22fa33f0fe1c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -389,7 +389,8 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
               frm.get('directory').value,
               this.fsName,
               retentionList,
-              this.retentionPoliciesToRemove?.map?.((rp) => rp.retentionFrequency) || []
+              this.retentionPoliciesToRemove?.map?.((rp) => rp.retentionFrequency) || [],
+              !!this.subvolume
             )
             .pipe(
               map(({ exists, errorIndex }) => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
index f6f372dcffc4..9a131a1e80b7 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-list/cephfs-snapshotschedule-list.component.ts
@@ -291,7 +291,17 @@ export class CephfsSnapshotscheduleListComponent
   }
 
   deleteSnapshotSchedule() {
-    const { path, start, fs, schedule, subvol, group } = this.selection.first();
+    const { path, start, fs, schedule, subvol, group, retention } = this.selection.first();
+    const retentionPolicy = retention
+      ?.split(/\s/gi)
+      ?.filter((r: string) => !!r)
+      ?.map((r: string) => {
+        const frequency = r.substring(r.length - 1);
+        const interval = r.substring(0, r.length - 1);
+        return `${interval}-${frequency}`;
+      })
+      ?.join('|')
+      ?.toLocaleLowerCase();
 
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
       itemDescription: $localize`snapshot schedule`,
@@ -305,6 +315,7 @@ export class CephfsSnapshotscheduleListComponent
             schedule,
             start,
             fs,
+            retentionPolicy,
             subvol,
             group
           })
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
index 93c04dc38ed9..ade935a9299a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cephfs-snapshot-schedule.service.ts
@@ -48,10 +48,21 @@ export class CephfsSnapshotScheduleService {
     );
   }
 
-  delete({ fs, path, schedule, start, subvol, group }: Record<string, any>): Observable<any> {
+  delete({
+    fs,
+    path,
+    schedule,
+    start,
+    retentionPolicy,
+    subvol,
+    group
+  }: Record<string, any>): Observable<any> {
     let deleteUrl = `${this.baseURL}/snapshot/schedule/${fs}/${encodeURIComponent(
       path
     )}/delete_snapshot?schedule=${schedule}&start=${encodeURIComponent(start)}`;
+    if (retentionPolicy) {
+      deleteUrl += `&retention_policy=${retentionPolicy}`;
+    }
     if (subvol && group) {
       deleteUrl += `&subvol=${encodeURIComponent(subvol)}&group=${encodeURIComponent(group)}`;
     }
@@ -81,13 +92,16 @@ export class CephfsSnapshotScheduleService {
     path: string,
     fs: string,
     retentionFrequencies: string[],
-    retentionFrequenciesRemoved: string[] = []
+    retentionFrequenciesRemoved: string[] = [],
+    isSubvolume = false
   ): Observable<{ exists: boolean; errorIndex: number }> {
     return this.getSnapshotSchedule(path, fs, false).pipe(
       map((response) => {
         let errorIndex = -1;
         let exists = false;
-        const index = response.findIndex((x) => x.path === path);
+        const index = response.findIndex((x) =>
+          isSubvolume ? x.path.startsWith(path) : x.path === path
+        );
         const result = retentionFrequencies?.length
           ? intersection(
               Object.keys(response?.[index]?.retention).filter(
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index f3d4f3607f33..5244d4a983b2 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -2081,6 +2081,11 @@ paths:
         required: true
         schema:
           type: string
+      - allowEmptyValue: true
+        in: query
+        name: retention_policy
+        schema:
+          type: string
       - allowEmptyValue: true
         in: query
         name: subvol

From 4182362c1195f3f322ae50dc0054518e62fea73a Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 18 Mar 2024 11:03:23 -0400
Subject: [PATCH 2313/2492] mon: do not log MON_DOWN if monitor uptime is less
 than threshold

Fixes: https://tracker.ceph.com/issues/64968
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/options/mon.yaml.in |  9 +++++++++
 src/mon/HealthMonitor.cc       |  9 +++++++--
 src/mon/Monitor.cc             |  1 +
 src/mon/Monitor.h              | 12 ++++++++++++
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
index ff8813c982f9..379917445c0f 100644
--- a/src/common/options/mon.yaml.in
+++ b/src/common/options/mon.yaml.in
@@ -55,6 +55,15 @@ options:
   default: 1_min
   services:
   - mon
+- name: mon_down_uptime_grace
+  type: secs
+  level: advanced
+  desc: Period in seconds that the cluster may have a mon down after this (leader) monitor comes up.
+  default: 1_min
+  services:
+  - mon
+  flags:
+  - runtime
 - name: mon_mgr_beacon_grace
   type: secs
   level: advanced
diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 4d2303d09fbc..45563f87d3d0 100644
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -803,9 +803,14 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks)
 {
   int max = mon.monmap->size();
   int actual = mon.get_quorum().size();
-  const auto now = ceph::real_clock::now();
+  const auto rcnow = ceph::real_clock::now();
+  const auto created = mon.monmap->created.to_real_time();
+  const auto mcnow = ceph::coarse_mono_clock::now();
+  const auto starttime = mon.get_starttime();
+
   if (actual < max &&
-      now > mon.monmap->created.to_real_time() + g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace")) {
+      (rcnow - created) > g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace") &&
+      (mcnow - starttime) > g_conf().get_val<std::chrono::seconds>("mon_down_uptime_grace")) {
     ostringstream ss;
     ss << (max-actual) << "/" << max << " mons down, quorum "
        << mon.get_quorum_names();
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index a0fe3fa038e6..a70bfbe33c9d 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -2672,6 +2672,7 @@ void Monitor::get_mon_status(Formatter *f)
   f->dump_int("rank", rank);
   f->dump_string("state", get_state_name());
   f->dump_int("election_epoch", get_epoch());
+  f->dump_int("uptime", get_uptime().count());
 
   f->open_array_section("quorum");
   for (set<int>::iterator p = quorum.begin(); p != quorum.end(); ++p) {
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index ed1e2a4e8a95..13afacafde7d 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -1099,6 +1099,18 @@ class Monitor : public Dispatcher,
   }
 
   bool is_keyring_required();
+
+public:
+  ceph::coarse_mono_time get_starttime() const {
+    return starttime;
+  }
+  std::chrono::milliseconds get_uptime() const {
+    auto now = ceph::coarse_mono_clock::now();
+    return std::chrono::duration_cast<std::chrono::milliseconds>(now-starttime);
+  }
+
+private:
+  ceph::coarse_mono_time const starttime = coarse_mono_clock::now();
 };
 
 #define CEPH_MON_FEATURE_INCOMPAT_BASE CompatSet::Feature (1, "initial feature set (~v.18)")

From 23de8e318f56b3052873fdd3744d313423e871bf Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 18 Mar 2024 15:55:04 -0400
Subject: [PATCH 2314/2492] qa: update dashboard schema for mon_status

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/mgr/dashboard/test_health.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index 3ebea97e3d29..5b181968633c 100644
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -268,7 +268,8 @@ def test_full_health(self):
                 'state': str,
                 # @TODO: What type should be expected here?
                 'sync_provider': JList(JAny(none=True)),
-                'stretch_mode': bool
+                'stretch_mode': bool,
+                'uptime': int,
             }),
             'osd_map': JObj({
                 # @TODO: define schema for crush map and osd_metadata, among

From 57b9e161f44d2c685b4a0ab68aff1b43bfd63e66 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 18 Mar 2024 20:38:08 -0400
Subject: [PATCH 2315/2492] qa: extend mon timeout coming up after mondb
 creation

Fixes: https://tracker.ceph.com/issues/64968
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/conf/mon.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/cephfs/conf/mon.yaml b/qa/cephfs/conf/mon.yaml
index eea56004a40b..e32f0fd35473 100644
--- a/qa/cephfs/conf/mon.yaml
+++ b/qa/cephfs/conf/mon.yaml
@@ -3,3 +3,5 @@ overrides:
     conf:
       mon:
         mon op complaint time: 120
+        # cephadm can take up to 5 minutes to bring up remaining mons
+        mon down mkfs grace: 300

From ae96ed4afb288fbc5a7ed818ba556c21418c631c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 18 Mar 2024 14:32:01 -0400
Subject: [PATCH 2316/2492] qa/cephfs: stop ignoring MON_DOWN globally

This should hopefully no longer be necessary with the addition of
mon_down_uptime_grace.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
index c9c46cc48f41..41ba84f04db5 100644
--- a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
+++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
@@ -7,4 +7,3 @@ overrides:
       - but it is still running
 # MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a'
       - is not responding
-      - MON_DOWN

From 629ba7bd349d48cdaa6d094751e7cfce651ba2bc Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Tue, 19 Mar 2024 13:43:10 +0000
Subject: [PATCH 2317/2492] mon, doc: overriding ec profile requires
 --yes-i-really-mean-it

This is per https://tracker.ceph.com/issues/64333#note-17 describing
driving factors of a catastrophic cluster failure.

Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
 doc/rados/operations/erasure-code-profile.rst | 4 +++-
 src/mon/OSDMonitor.cc                         | 7 +++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/doc/rados/operations/erasure-code-profile.rst b/doc/rados/operations/erasure-code-profile.rst
index 947b34c1f006..a8f006398fa3 100644
--- a/doc/rados/operations/erasure-code-profile.rst
+++ b/doc/rados/operations/erasure-code-profile.rst
@@ -96,7 +96,9 @@ Where:
 ``--force``
 
 :Description: Override an existing profile by the same name, and allow
-              setting a non-4K-aligned stripe_unit.
+              setting a non-4K-aligned stripe_unit. Overriding an existing
+              profile can be dangerous, and thus ``--yes-i-really-mean-it``
+              must be used as well.
 
 :Type: String
 :Required: No.
diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index c3429b5fd2ed..94f88f54c4e9 100644
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -11305,6 +11305,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 	  err = 0;
 	  goto reply_no_propose;
 	}
+	bool force_no_fake = false;
+	cmd_getval(cmdmap, "yes_i_really_mean_it", force_no_fake);
 	if (!force) {
 	  err = -EPERM;
 	  ss << "will not override erasure code profile " << name
@@ -11313,6 +11315,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 	     << " is different from the proposed profile "
 	     << profile_map;
 	  goto reply_no_propose;
+	} else if (!force_no_fake) {
+	  err = -EPERM;
+	  ss << "overriding erasure code profile can be DANGEROUS"
+	     << "; add --yes-i-really-mean-it to do it anyway";
+	  goto reply_no_propose;
 	}
       }
 

From 131a70754a6bfbbb2d1481ba74436953c6ecc226 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 19 Mar 2024 20:27:13 +0530
Subject: [PATCH 2318/2492] mgr/dashboard: rm warning/error threshold for cpu
 usage

for multi-core cpu's the value can be more than 100% so it doesn't make
sense to show warning/error when the usage is at or more than 100%.
hence removing it

Fixes: https://tracker.ceph.com/issues/58838
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../service-daemon-list/service-daemon-list.component.html   | 5 +----
 .../app/shared/components/usage-bar/usage-bar.component.html | 2 +-
 .../app/shared/components/usage-bar/usage-bar.component.ts   | 4 ++--
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-daemon-list/service-daemon-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-daemon-list/service-daemon-list.component.html
index c5c173044ea4..273ea0b338a1 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-daemon-list/service-daemon-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-daemon-list/service-daemon-list.component.html
@@ -94,9 +94,6 @@
   <cd-usage-bar [total]="total"
                 [calculatePerc]="false"
                 [used]="row.cpu_percentage"
-                [isBinary]="false"
-                [warningThreshold]="warningThreshold"
-                [errorThreshold]="errorThreshold">
+                [isBinary]="false">
   </cd-usage-bar>
 </ng-template>
-
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
index 9a0a3398a3c0..e29b0dd8b4be 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -29,7 +29,7 @@
      data-placement="left"
      [ngbTooltip]="usageTooltipTpl">
   <div class="progress-bar bg-info"
-       [ngClass]="{'bg-warning': (warningThreshold >= 0) && (usedPercentage/100 >= warningThreshold), 'bg-danger': (errorThreshold >= 0) && (usedPercentage/100 >= errorThreshold)}"
+       [ngClass]="{'bg-warning': warningThreshold && (warningThreshold >= 0) && (usedPercentage/100 >= warningThreshold), 'bg-danger': errorThreshold && (errorThreshold >= 0) && (usedPercentage/100 >= errorThreshold)}"
        role="progressbar"
        [attr.aria-label]="{ title }"
        i18n-aria-label="The title of this usage bar is { title }"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
index 58bd7d4a46dd..f147227eb1bc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
@@ -13,9 +13,9 @@ export class UsageBarComponent implements OnChanges {
   @Input()
   used: any;
   @Input()
-  warningThreshold: number;
+  warningThreshold?: number;
   @Input()
-  errorThreshold: number;
+  errorThreshold?: number;
   @Input()
   isBinary = true;
   @Input()

From d2d844b60be5b0881844cecf7b0a54124061152b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 11:01:48 -0400
Subject: [PATCH 2319/2492] qa/cephfs: add mgr debugging

Fixes: https://tracker.ceph.com/issues/64985
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/conf/mgr.yaml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 qa/cephfs/conf/mgr.yaml

diff --git a/qa/cephfs/conf/mgr.yaml b/qa/cephfs/conf/mgr.yaml
new file mode 100644
index 000000000000..fb6e9b09fa15
--- /dev/null
+++ b/qa/cephfs/conf/mgr.yaml
@@ -0,0 +1,7 @@
+overrides:
+  ceph:
+    conf:
+      mgr:
+        debug client: 20
+        debug mgr: 20
+        debug ms: 1

From 73d215510971e5fd0b27ac12e172684783765e4b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 11:08:17 -0400
Subject: [PATCH 2320/2492] qa/cephfs: ignorelist clog of MDS_UP_LESS_THAN_MAX

Fixes: https://tracker.ceph.com/issues/64986
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/ignorelist_health.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
index aef498f92c60..c9a422261ec5 100644
--- a/qa/cephfs/overrides/ignorelist_health.yaml
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -10,6 +10,7 @@ overrides:
       - MDS_FAILED
       - MDS_INSUFFICIENT_STANDBY
       - MDS_UP_LESS_THAN_MAX
+      - filesystem is online with fewer MDS than max_mds
       - POOL_APP_NOT_ENABLED
       - overall HEALTH_
       - Replacing daemon

From ae8756246e18c3dec5fdebc2a9ff98bc9806dca5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 09:52:44 -0400
Subject: [PATCH 2321/2492] qa/suites/fs/nfs: use standard health ignorelist

Fixes: https://tracker.ceph.com/issues/64987
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/suites/fs/nfs/overrides/ignorelist_health.yaml | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)
 mode change 100644 => 120000 qa/suites/fs/nfs/overrides/ignorelist_health.yaml

diff --git a/qa/suites/fs/nfs/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs/overrides/ignorelist_health.yaml
deleted file mode 100644
index 8bfe4dc6f618..000000000000
--- a/qa/suites/fs/nfs/overrides/ignorelist_health.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-overrides:
-  ceph:
-    log-ignorelist:
-      - overall HEALTH_
-      - \(FS_DEGRADED\)
-      - \(MDS_FAILED\)
-      - \(MDS_DEGRADED\)
-      - \(FS_WITH_FAILED_MDS\)
-      - \(MDS_DAMAGE\)
-      - \(MDS_ALL_DOWN\)
-      - \(MDS_UP_LESS_THAN_MAX\)
-      - \(FS_INLINE_DATA_DEPRECATED\)
-      - \(OSD_DOWN\)
diff --git a/qa/suites/fs/nfs/overrides/ignorelist_health.yaml b/qa/suites/fs/nfs/overrides/ignorelist_health.yaml
new file mode 120000
index 000000000000..5cb891a95c3c
--- /dev/null
+++ b/qa/suites/fs/nfs/overrides/ignorelist_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/ignorelist_health.yaml
\ No newline at end of file

From 00651cfac230a759389a595b41bb220474f5f2a7 Mon Sep 17 00:00:00 2001
From: Vallari Agrawal <val.agl002@gmail.com>
Date: Tue, 20 Feb 2024 13:14:32 +0530
Subject: [PATCH 2322/2492] qa/suite/rbd/nvmeof: Deploy multiple gateways and
 namespaces

1. Deploy 2 gateways on different nodes, then check for multi-path.
    To add another gateway, only "roles" need to be changed in job yaml.
2. Create "n" nvmeof namespaces, configured by 'namespaces_count'
3. Rename qa/suites/rbd/nvmeof/cluster/fixed-3.yaml to fixed-4.yaml
    which contains 2 gateways and 2 initiators.

Signed-off-by: Vallari Agrawal <val.agl002@gmail.com>
---
 .../cluster/{fixed-3.yaml => fixed-4.yaml}    |  2 +
 qa/suites/rbd/nvmeof/conf                     |  1 +
 .../nvmeof/workloads/nvmeof_initiator.yaml    | 30 +++++---
 qa/tasks/nvmeof.py                            | 70 +++++++++----------
 qa/workunits/rbd/nvmeof_basic_tests.sh        | 16 +++--
 qa/workunits/rbd/nvmeof_fio_test.sh           | 53 ++++++++++++--
 qa/workunits/rbd/nvmeof_initiator.sh          | 27 -------
 qa/workunits/rbd/nvmeof_setup_subsystem.sh    | 38 ++++++++++
 8 files changed, 151 insertions(+), 86 deletions(-)
 rename qa/suites/rbd/nvmeof/cluster/{fixed-3.yaml => fixed-4.yaml} (81%)
 create mode 120000 qa/suites/rbd/nvmeof/conf
 delete mode 100755 qa/workunits/rbd/nvmeof_initiator.sh
 create mode 100755 qa/workunits/rbd/nvmeof_setup_subsystem.sh

diff --git a/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml b/qa/suites/rbd/nvmeof/cluster/fixed-4.yaml
similarity index 81%
rename from qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
rename to qa/suites/rbd/nvmeof/cluster/fixed-4.yaml
index f417079e31a1..d57e9fc47be3 100644
--- a/qa/suites/rbd/nvmeof/cluster/fixed-3.yaml
+++ b/qa/suites/rbd/nvmeof/cluster/fixed-4.yaml
@@ -12,4 +12,6 @@ roles:
   - osd.3
   - osd.4
   - client.1
+  - ceph.nvmeof.nvmeof.b
 - - client.2
+- - client.3
diff --git a/qa/suites/rbd/nvmeof/conf b/qa/suites/rbd/nvmeof/conf
new file mode 120000
index 000000000000..4bc0fe86c630
--- /dev/null
+++ b/qa/suites/rbd/nvmeof/conf
@@ -0,0 +1 @@
+.qa/rbd/conf
\ No newline at end of file
diff --git a/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
index bbb9b0ab5f2a..5a4143ffa917 100644
--- a/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
+++ b/qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml
@@ -1,15 +1,13 @@
 tasks:
 - nvmeof:
     client: client.0
-    version: latest  # "default" uses packaged version; change to test specific nvmeof images, example "latest"
+    version: default  # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
     rbd:
-        pool_name: mypool
-        image_name: myimage
+      pool_name: mypool
+      image_name_prefix: myimage
     gateway_config:
-        source: host.a 
-        target: client.2
-        vars:
-            cli_version: latest
+      namespaces_count: 128
+      cli_version: latest
 
 - cephadm.wait_for_service:
     service: nvmeof.mypool
@@ -18,10 +16,22 @@ tasks:
     no_coverage_and_limits: true
     clients:
       client.2:
-        - rbd/nvmeof_initiator.sh
+        - rbd/nvmeof_setup_subsystem.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.2:
+        - rbd/nvmeof_basic_tests.sh
+        - rbd/nvmeof_fio_test.sh --start_ns 1 --end_ns 64 --rbd_iostat
+      client.3:
         - rbd/nvmeof_basic_tests.sh
-        - rbd/nvmeof_fio_test.sh
+        - rbd/nvmeof_fio_test.sh --start_ns 65 --end_ns 128
     env:
       RBD_POOL: mypool
-      RBD_IMAGE: myimage
       IOSTAT_INTERVAL: '10'
+      RUNTIME: '600'
diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py
index b75d00d93ae8..7e9afe789466 100644
--- a/qa/tasks/nvmeof.py
+++ b/qa/tasks/nvmeof.py
@@ -20,13 +20,10 @@ class Nvmeof(Task):
             version: default
             rbd:
                 pool_name: mypool
-                image_name: myimage
                 rbd_size: 1024
             gateway_config:
-                source: host.a 
-                target: client.2
-                vars:
-                    cli_version: latest
+                namespaces_count: 10
+                cli_version: latest
                     
     """
 
@@ -54,17 +51,17 @@ def _set_defaults(self):
 
         rbd_config = self.config.get('rbd', {})
         self.poolname = rbd_config.get('pool_name', 'mypool')
-        self.rbd_image_name = rbd_config.get('image_name', 'myimage')
+        self.image_name_prefix = rbd_config.get('image_name_prefix', 'myimage')
         self.rbd_size = rbd_config.get('rbd_size', 1024*8)
 
         gateway_config = self.config.get('gateway_config', {})
-        conf_vars = gateway_config.get('vars', {})
-        self.cli_image = conf_vars.get('cli_version', 'latest')
-        self.bdev = conf_vars.get('bdev', 'mybdev')
-        self.serial = conf_vars.get('serial', 'SPDK00000000000001')
-        self.nqn = conf_vars.get('nqn', 'nqn.2016-06.io.spdk:cnode1')
-        self.port = conf_vars.get('port', '4420')
-        self.srport = conf_vars.get('srport', '5500')
+        self.namespaces_count = gateway_config.get('namespaces_count', 1)
+        self.cli_image = gateway_config.get('cli_version', 'latest')
+        self.bdev = gateway_config.get('bdev', 'mybdev')
+        self.serial = gateway_config.get('serial', 'SPDK00000000000001')
+        self.nqn = gateway_config.get('nqn', 'nqn.2016-06.io.spdk:cnode1')
+        self.port = gateway_config.get('port', '4420')
+        self.srport = gateway_config.get('srport', '5500')
 
     def deploy_nvmeof(self):
         """
@@ -97,7 +94,6 @@ def deploy_nvmeof(self):
                 ])
 
             poolname = self.poolname
-            imagename = self.rbd_image_name
 
             log.info(f'[nvmeof]: ceph osd pool create {poolname}')
             _shell(self.ctx, self.cluster_name, self.remote, [
@@ -115,10 +111,13 @@ def deploy_nvmeof(self):
                 '--placement', str(len(nodes)) + ';' + ';'.join(nodes)
             ])
 
-            log.info(f'[nvmeof]: rbd create {poolname}/{imagename} --size {self.rbd_size}')
-            _shell(self.ctx, self.cluster_name, self.remote, [
-                'rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}'
-            ])
+            log.info(f'[nvmeof]: creating {self.namespaces_count} images')
+            for i in range(1, int(self.namespaces_count) + 1):
+                imagename = self.image_name_prefix + str(i)
+                log.info(f'[nvmeof]: rbd create {poolname}/{imagename} --size {self.rbd_size}')
+                _shell(self.ctx, self.cluster_name, self.remote, [
+                    'rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}'
+                ])
 
         for role, i in daemons.items():
             remote, id_ = i
@@ -134,34 +133,29 @@ def deploy_nvmeof(self):
         
     def set_gateway_cfg(self):
         log.info('[nvmeof]: running set_gateway_cfg...')
-        gateway_config = self.config.get('gateway_config', {})
-        source_host = gateway_config.get('source')
-        target_host = gateway_config.get('target')
-        if not (source_host and target_host):
-            raise ConfigError('gateway_config requires "source" and "target"')
-        remote = list(self.ctx.cluster.only(source_host).remotes.keys())[0]
-        ip_address = remote.ip_address
-        gateway_name = ""
+        ip_address = self.remote.ip_address
+        gateway_names = []
+        gateway_ips = []
         nvmeof_daemons = self.ctx.daemons.iter_daemons_of_role('nvmeof', cluster=self.cluster_name)
         for daemon in nvmeof_daemons:
-            if ip_address == daemon.remote.ip_address:
-                gateway_name = daemon.name()
+            gateway_names += [daemon.name()]
+            gateway_ips += [daemon.remote.ip_address]
         conf_data = dedent(f"""
-            NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
-            NVMEOF_GATEWAY_NAME={gateway_name}
+            NVMEOF_GATEWAY_IP_ADDRESSES={",".join(gateway_ips)}
+            NVMEOF_GATEWAY_NAMES={",".join(gateway_names)}
+            NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS={ip_address}
             NVMEOF_CLI_IMAGE="quay.io/ceph/nvmeof-cli:{self.cli_image}"
-            NVMEOF_BDEV={self.bdev}
-            NVMEOF_SERIAL={self.serial}
+            NVMEOF_NAMESPACES_COUNT={self.namespaces_count}
             NVMEOF_NQN={self.nqn}
             NVMEOF_PORT={self.port}
             NVMEOF_SRPORT={self.srport}
             """)
-        target_remote = list(self.ctx.cluster.only(target_host).remotes.keys())[0]
-        target_remote.write_file(
-            path=conf_file,
-            data=conf_data,
-            sudo=True
-        )
+        for remote in self.ctx.cluster.remotes.keys():
+            remote.write_file(
+                path=conf_file,
+                data=conf_data,
+                sudo=True
+            )
         log.info("[nvmeof]: executed set_gateway_cfg successfully!")
 
 
diff --git a/qa/workunits/rbd/nvmeof_basic_tests.sh b/qa/workunits/rbd/nvmeof_basic_tests.sh
index 878e043fbeb5..d92eed9b6f02 100755
--- a/qa/workunits/rbd/nvmeof_basic_tests.sh
+++ b/qa/workunits/rbd/nvmeof_basic_tests.sh
@@ -1,11 +1,16 @@
 #!/bin/bash -x
 
+sudo modprobe nvme-fabrics
+sudo modprobe nvme-tcp
+sudo dnf install nvme-cli -y
+sudo lsmod | grep nvme
+
 source /etc/ceph/nvmeof.env
 SPDK_CONTROLLER="SPDK bdev Controller"
 DISCOVERY_PORT="8009"
 
 discovery() {
-    output=$(sudo nvme discover -t tcp -a $NVMEOF_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT)
+    output=$(sudo nvme discover -t tcp -a $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT)
     expected_discovery_stdout="subtype: nvme subsystem"
     if ! echo "$output" | grep -q "$expected_discovery_stdout"; then
         return 1
@@ -13,7 +18,7 @@ discovery() {
 }
 
 connect() {
-    sudo nvme connect -t tcp --traddr $NVMEOF_GATEWAY_IP_ADDRESS -s $NVMEOF_PORT -n $NVMEOF_NQN
+    sudo nvme connect -t tcp --traddr $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS -s $NVMEOF_PORT -n $NVMEOF_NQN
     output=$(sudo nvme list)
     if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
         return 1
@@ -29,7 +34,7 @@ disconnect_all() {
 }
 
 connect_all() {
-    sudo nvme connect-all --traddr=$NVMEOF_GATEWAY_IP_ADDRESS --transport=tcp
+    sudo nvme connect-all --traddr=$NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --transport=tcp
     output=$(sudo nvme list)
     if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
         return 1
@@ -39,7 +44,7 @@ connect_all() {
 list_subsys() {
     expected_count=$1
     output=$(sudo nvme list-subsys --output-format=json)
-    multipath=$(echo $output | grep -c '"tcp"')
+    multipath=$(echo $output | grep -o '"tcp"' | wc -l)
     if [ "$multipath" -ne "$expected_count" ]; then
         return 1
     fi
@@ -65,7 +70,8 @@ test_run list_subsys 1
 test_run disconnect_all
 test_run list_subsys 0
 test_run connect_all
-test_run list_subsys 1
+gateway_count=$(( $(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | tr -cd ',' | wc -c) + 1))
+test_run list_subsys $gateway_count
 
 
 echo "-------------Test Summary-------------"
diff --git a/qa/workunits/rbd/nvmeof_fio_test.sh b/qa/workunits/rbd/nvmeof_fio_test.sh
index bacc15e83eb7..5e4bad98a642 100755
--- a/qa/workunits/rbd/nvmeof_fio_test.sh
+++ b/qa/workunits/rbd/nvmeof_fio_test.sh
@@ -3,11 +3,46 @@
 sudo yum -y install fio
 sudo yum -y install sysstat
 
+namespace_range_start=
+namespace_range_end=
+rbd_iostat=false
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --start_ns)
+            namespace_range_start=$2
+            shift 2
+            ;;
+        --end_ns)
+            namespace_range_end=$2
+            shift 2
+            ;;
+        --rbd_iostat)
+            rbd_iostat=true
+            shift
+            ;;
+        *)
+            exit 100	# Internal error
+            ;;
+    esac
+done
+
 fio_file=$(mktemp -t nvmeof-fio-XXXX)
-drives_list=$(sudo nvme list --output-format=json | jq -r '.Devices | .[] | select(.ModelNumber == "SPDK bdev Controller") | .DevicePath')
+all_drives_list=$(sudo nvme list --output-format=json | 
+    jq -r '.Devices | sort_by(.NameSpace) | .[] | select(.ModelNumber == "SPDK bdev Controller") | .DevicePath')
+
+# When the script is passed --start_ns and --end_ns (example: `nvmeof_fio_test.sh --start_ns 1 --end_ns 3`), 
+# then fio runs on namespaces only in the defined range (which is 1 to 3 here). 
+# So if `nvme list` has 5 namespaces with "SPDK Controller", then fio will 
+# run on first 3 namespaces here.
+if [ "$namespace_range_start" ] || [ "$namespace_range_end" ]; then
+    selected_drives=$(echo "${all_drives_list[@]}" | sed -n "${namespace_range_start},${namespace_range_end}p")
+else
+    selected_drives="${all_drives_list[@]}"
+fi
+
 
 RUNTIME=${RUNTIME:-600}
-# IOSTAT_INTERVAL=10
 
 
 cat >> $fio_file <<EOF
@@ -19,18 +54,24 @@ size=${SIZE:-1G}
 time_based=1
 runtime=$RUNTIME
 rw=${RW:-randrw}
-filename=$(echo "$drives_list" | tr '\n' ':' | sed 's/:$//')
+filename=$(echo "$selected_drives" | tr '\n' ':' | sed 's/:$//')
 verify=md5
 verify_fatal=1
+direct=1
 EOF
 
-fio --showcmd $fio_file
-sudo fio $fio_file &
+echo "[nvmeof] starting fio test..."
 
 if [ -n "$IOSTAT_INTERVAL" ]; then
     iostat_count=$(( RUNTIME / IOSTAT_INTERVAL ))
-    iostat -d $IOSTAT_INTERVAL $iostat_count -h 
+    iostat -d -p $selected_drives $IOSTAT_INTERVAL $iostat_count -h &
 fi
+if [ "$rbd_iostat" = true  ]; then
+    iterations=$(( RUNTIME / 5 ))
+    rbd perf image iostat $RBD_POOL --iterations $iterations &
+fi
+fio --showcmd $fio_file
+sudo fio $fio_file 
 wait
 
 echo "[nvmeof] fio test successful!"
diff --git a/qa/workunits/rbd/nvmeof_initiator.sh b/qa/workunits/rbd/nvmeof_initiator.sh
deleted file mode 100755
index 8f79238083b2..000000000000
--- a/qa/workunits/rbd/nvmeof_initiator.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-sudo modprobe nvme-fabrics
-sudo modprobe nvme-tcp
-sudo dnf install nvme-cli -y
-
-source /etc/ceph/nvmeof.env
-
-# RBD_POOL and RBD_IMAGE is intended to be set from yaml, 'mypool' and 'myimage' are defaults
-RBD_POOL="${RBD_POOL:-mypool}"
-RBD_IMAGE="${RBD_IMAGE:-myimage}"
-
-HOSTNAME=$(hostname)
-sudo podman images
-sudo podman ps
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem list
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem add --subsystem $NVMEOF_NQN
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT namespace add --subsystem $NVMEOF_NQN --rbd-pool $RBD_POOL --rbd-image $RBD_IMAGE
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT listener add --subsystem $NVMEOF_NQN --gateway-name client.$NVMEOF_GATEWAY_NAME --traddr $NVMEOF_GATEWAY_IP_ADDRESS --trsvcid $NVMEOF_PORT
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT host add --subsystem $NVMEOF_NQN --host "*"
-sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem list
-sudo lsmod | grep nvme
-sudo nvme list
-
-echo "[nvmeof] Initiator setup done"
\ No newline at end of file
diff --git a/qa/workunits/rbd/nvmeof_setup_subsystem.sh b/qa/workunits/rbd/nvmeof_setup_subsystem.sh
new file mode 100755
index 000000000000..938b10c4af32
--- /dev/null
+++ b/qa/workunits/rbd/nvmeof_setup_subsystem.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -ex
+
+
+source /etc/ceph/nvmeof.env
+
+# Set these in job yaml
+RBD_POOL="${RBD_POOL:-mypool}"
+RBD_IMAGE_PREFIX="${RBD_IMAGE_PREFIX:-myimage}"
+
+HOSTNAME=$(hostname)
+sudo podman images
+sudo podman ps
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT --format json subsystem list
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem add --subsystem $NVMEOF_NQN
+
+# add all namespaces
+for i in $(seq 1 $NVMEOF_NAMESPACES_COUNT); do
+    image="${RBD_IMAGE_PREFIX}${i}"
+    sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT namespace add --subsystem $NVMEOF_NQN --rbd-pool $RBD_POOL --rbd-image $image
+done
+
+# add all gateway listeners
+IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES"
+IFS=',' read -ra gateway_names <<< "$NVMEOF_GATEWAY_NAMES"
+for i in "${!gateway_ips[@]}"
+do
+    ip="${gateway_ips[i]}"
+    name="${gateway_names[i]}"
+    echo "Adding gateway listener $index with IP ${ip} and name ${name}"
+    sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT listener add --subsystem $NVMEOF_NQN --gateway-name client.$name --traddr $ip --trsvcid $NVMEOF_PORT
+    sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT --format json subsystem list
+done
+
+sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT host add --subsystem $NVMEOF_NQN --host "*"
+
+echo "[nvmeof] Subsystem setup done"

From 429967917b4f9b5f71214fa3c7718e780728249b Mon Sep 17 00:00:00 2001
From: kchheda3 <kchheda3@bloomberg.net>
Date: Wed, 6 Mar 2024 18:13:57 -0500
Subject: [PATCH 2323/2492] rgw/notification: add rgw notification specific
 debug log subsystem.

decorate the events with event details while logging.

Signed-off-by: kchheda3 <kchheda3@bloomberg.net>
---
 doc/rados/troubleshooting/log-and-debug.rst   |  2 +
 .../perf/settings/optimized.yaml              |  1 +
 qa/suites/perf-basic/settings/optimized.yaml  |  1 +
 qa/suites/rados/perf/settings/optimized.yaml  |  1 +
 qa/suites/rgw/multisite/overrides.yaml        |  1 +
 src/common/subsys.h                           |  1 +
 src/rgw/driver/rados/rgw_notify.cc            | 83 +++++++++++--------
 src/rgw/rgw_amqp.cc                           |  2 +-
 src/rgw/rgw_kafka.cc                          |  2 +-
 9 files changed, 58 insertions(+), 36 deletions(-)

diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
index a553269cb8e3..56bf448432fa 100644
--- a/doc/rados/troubleshooting/log-and-debug.rst
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -302,6 +302,8 @@ to their default level or to a level suitable for normal operations.
 +--------------------------+-----------+--------------+
 | ``rgw lifecycle``        |     1     |      5       |
 +--------------------------+-----------+--------------+
+| ``rgw notification``     |     1     |      5       |
++--------------------------+-----------+--------------+
 | ``javaclient``           |     1     |      5       |
 +--------------------------+-----------+--------------+
 | ``asok``                 |     1     |      5       |
diff --git a/qa/suites/crimson-rados/perf/settings/optimized.yaml b/qa/suites/crimson-rados/perf/settings/optimized.yaml
index deefc61d3d83..65a8c7f56ea3 100644
--- a/qa/suites/crimson-rados/perf/settings/optimized.yaml
+++ b/qa/suites/crimson-rados/perf/settings/optimized.yaml
@@ -55,6 +55,7 @@ overrides:
         debug rgw: "0/0"
         debug rgw sync: "0/0"
         debug rgw lifecycle: "0/0"
+        debug rgw notification: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/perf-basic/settings/optimized.yaml b/qa/suites/perf-basic/settings/optimized.yaml
index 3983ed9d26e6..bba1ddcf7dd8 100644
--- a/qa/suites/perf-basic/settings/optimized.yaml
+++ b/qa/suites/perf-basic/settings/optimized.yaml
@@ -59,6 +59,7 @@ overrides:
         debug rgw: "0/0"
         debug rgw sync: "0/0"
         debug rgw lifecycle: "0/0"
+        debug rgw notification: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/rados/perf/settings/optimized.yaml b/qa/suites/rados/perf/settings/optimized.yaml
index 8f05f59163ac..46351c0d578a 100644
--- a/qa/suites/rados/perf/settings/optimized.yaml
+++ b/qa/suites/rados/perf/settings/optimized.yaml
@@ -55,6 +55,7 @@ overrides:
         debug rgw: "0/0"
         debug rgw sync: "0/0"
         debug rgw lifecycle: "0/0"
+        debug rgw notification: "0/0"
         debug civetweb: "0/0"
         debug javaclient: "0/0"
         debug asok: "0/0"
diff --git a/qa/suites/rgw/multisite/overrides.yaml b/qa/suites/rgw/multisite/overrides.yaml
index 8f0d376f7d82..13b498d167f6 100644
--- a/qa/suites/rgw/multisite/overrides.yaml
+++ b/qa/suites/rgw/multisite/overrides.yaml
@@ -7,6 +7,7 @@ overrides:
         debug rgw: 20
         debug rgw sync: 20
         debug rgw lifecycle: 20
+        debug rgw notification: 20
         rgw crypt s3 kms backend: testing
         rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=
         rgw crypt require ssl: false
diff --git a/src/common/subsys.h b/src/common/subsys.h
index d52018c880dd..6022c1fcaf1f 100644
--- a/src/common/subsys.h
+++ b/src/common/subsys.h
@@ -65,6 +65,7 @@ SUBSYS(rgw_access, 1, 5)
 SUBSYS(rgw_dbstore, 1, 5)
 SUBSYS(rgw_flight, 1, 5)
 SUBSYS(rgw_lifecycle, 1, 5)
+SUBSYS(rgw_notification, 1, 5)
 SUBSYS(javaclient, 1, 5)
 SUBSYS(asok, 1, 5)
 SUBSYS(throttle, 1, 1)
diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc
index 9baed182ed94..4be0524587e0 100644
--- a/src/rgw/driver/rados/rgw_notify.cc
+++ b/src/rgw/driver/rados/rgw_notify.cc
@@ -18,7 +18,7 @@
 #include "common/dout.h"
 #include <chrono>
 
-#define dout_subsys ceph_subsys_rgw
+#define dout_subsys ceph_subsys_rgw_notification
 
 namespace rgw::notify {
 
@@ -66,6 +66,16 @@ struct event_entry_t {
 };
 WRITE_CLASS_ENCODER(event_entry_t)
 
+static inline std::ostream& operator<<(std::ostream& out,
+                                       const event_entry_t& e) {
+  return out << "notification id: '" << e.event.configurationId
+             << "', topic: '" << e.arn_topic
+             << "', endpoint: '" << e.push_endpoint
+             << "', bucket_owner: '" << e.event.bucket_ownerIdentity
+             << "', bucket: '" << e.event.bucket_name
+             << "', object: '" << e.event.object_key
+             << "', event type: '" << e.event.eventName << "'";
+}
 
 struct persistency_tracker {
   ceph::coarse_real_time last_retry_time {ceph::coarse_real_clock::zero()};
@@ -244,15 +254,12 @@ class Manager : public DoutPrefixProvider {
     if ( (topic_persistency_ttl != 0 && event_entry.creation_time != ceph::coarse_real_clock::zero() &&
          time_now - event_entry.creation_time > std::chrono::seconds(topic_persistency_ttl))
          || ( topic_persistency_max_retries != 0 && entry_persistency_tracker.retires_num >  topic_persistency_max_retries) ) {
-      ldpp_dout(this, 1) << "Expiring entry for topic= "
-                         << event_entry.arn_topic << " bucket_owner= "
-                         << event_entry.event.bucket_ownerIdentity
-                         << " bucket= " << event_entry.event.bucket_name
-                         << " object_name= " << event_entry.event.object_key
-                         << " entry retry_number="
+      ldpp_dout(this, 1) << "WARNING: Expiring entry marker: " << entry.marker
+                         << " for event with " << event_entry
+                         << " entry retry_number: "
                          << entry_persistency_tracker.retires_num
-                         << " creation_time=" << event_entry.creation_time
-                         << " time_now=" << time_now << dendl;
+                         << " creation_time: " << event_entry.creation_time
+                         << " time_now: " << time_now << dendl;
       return EntryProcessingResult::Expired;
     }
     if (time_now - entry_persistency_tracker.last_retry_time < std::chrono::seconds(topic_persistency_sleep_duration) ) {
@@ -261,7 +268,10 @@ class Manager : public DoutPrefixProvider {
 
     ++entry_persistency_tracker.retires_num;
     entry_persistency_tracker.last_retry_time = time_now;
-    ldpp_dout(this, 20) << "Processing entry retry_number=" << entry_persistency_tracker.retires_num << " time=" << dendl;
+    ldpp_dout(this, 20) << "Processing event entry with " << event_entry
+                        << " retry_number: "
+                        << entry_persistency_tracker.retires_num
+                        << " current time: " << time_now << dendl;
     try {
       // TODO move endpoint creation to queue level
       const auto push_endpoint = RGWPubSubEndpoint::create(event_entry.push_endpoint, event_entry.arn_topic,
@@ -271,12 +281,14 @@ class Manager : public DoutPrefixProvider {
         " for entry: " << entry.marker << dendl;
       const auto ret = push_endpoint->send_to_completion_async(cct, event_entry.event, optional_yield(io_context, yield));
       if (ret < 0) {
-        ldpp_dout(this, 5) << "WARNING: push entry: " << entry.marker << " to endpoint: " << event_entry.push_endpoint 
-          << " failed. error: " << ret << " (will retry)" << dendl;
+        ldpp_dout(this, 5) << "WARNING: push entry marker: " << entry.marker
+                           << " failed. error: " << ret
+                           << " (will retry) for event with " << event_entry
+                           << dendl;
         return EntryProcessingResult::Failure;
       } else {
-        ldpp_dout(this, 20) << "INFO: push entry: " << entry.marker << " to endpoint: " << event_entry.push_endpoint 
-          << " ok" <<  dendl;
+        ldpp_dout(this, 5) << "INFO: push entry marker: " << entry.marker
+                           << " ok for event with " << event_entry << dendl;
         if (perfcounter) perfcounter->inc(l_rgw_pubsub_push_ok);
         return EntryProcessingResult::Successful;
       }
@@ -304,12 +316,15 @@ class Manager : public DoutPrefixProvider {
       auto ret = rgw_rados_operate(this, rados_store.getRados()->get_notif_pool_ctx(), queue_name, &op, optional_yield(io_context, yield));
       if (ret == -ENOENT) {
         // queue was deleted
-        ldpp_dout(this, 5) << "INFO: queue: " 
-          << queue_name << ". was removed. cleanup will stop" << dendl;
+        ldpp_dout(this, 10) << "INFO: queue: " << queue_name
+                            << ". was removed. cleanup will stop" << dendl;
         return;
       }
       if (ret == -EBUSY) {
-        ldpp_dout(this, 5) << "WARNING: queue: " << queue_name << " ownership moved to another daemon. processing will stop" << dendl;
+        ldpp_dout(this, 10)
+            << "WARNING: queue: " << queue_name
+            << " ownership moved to another daemon. processing will stop"
+            << dendl;
         return;
       }
       if (ret < 0) {
@@ -367,13 +382,16 @@ class Manager : public DoutPrefixProvider {
         if (ret == -ENOENT) {
           // queue was deleted
           topics_persistency_tracker.erase(queue_name);
-          ldpp_dout(this, 5) << "INFO: queue: " 
-            << queue_name << ". was removed. processing will stop" << dendl;
+          ldpp_dout(this, 10) << "INFO: queue: " << queue_name
+                              << ". was removed. processing will stop" << dendl;
           return;
         }
         if (ret == -EBUSY) {
           topics_persistency_tracker.erase(queue_name);
-          ldpp_dout(this, 5) << "WARNING: queue: " << queue_name << " ownership moved to another daemon. processing will stop" << dendl;
+          ldpp_dout(this, 10)
+              << "WARNING: queue: " << queue_name
+              << " ownership moved to another daemon. processing will stop"
+              << dendl;
           return;
         }
         if (ret < 0) {
@@ -468,11 +486,15 @@ class Manager : public DoutPrefixProvider {
         auto ret = rgw_rados_operate(this, rados_ioctx, queue_name, &op, optional_yield(io_context, yield));
         if (ret == -ENOENT) {
           // queue was deleted
-          ldpp_dout(this, 5) << "INFO: queue: " << queue_name << ". was removed. processing will stop" << dendl;
+          ldpp_dout(this, 10) << "INFO: queue: " << queue_name
+                              << ". was removed. processing will stop" << dendl;
           return;
         }
         if (ret == -EBUSY) {
-          ldpp_dout(this, 5) << "WARNING: queue: " << queue_name << " ownership moved to another daemon. processing will stop" << dendl;
+          ldpp_dout(this, 10)
+              << "WARNING: queue: " << queue_name
+              << " ownership moved to another daemon. processing will stop"
+              << dendl;
           return;
         }
         if (ret < 0) {
@@ -1133,23 +1155,16 @@ int publish_commit(rgw::sal::Object* obj,
 	  dpp->get_cct(), event_entry.event, res.yield);
         if (ret < 0) {
           ldpp_dout(dpp, 1)
-              << "ERROR: push to endpoint " << topic.cfg.dest.push_endpoint
-              << " bucket: " << event_entry.event.bucket_name
-              << " bucket_owner: " << event_entry.event.bucket_ownerIdentity
-              << " object_name: " << event_entry.event.object_key
-              << " failed. error: " << ret << dendl;
+              << "ERROR: failed to push sync notification event with error: "
+              << ret << " for event with " << event_entry << dendl;
           if (perfcounter) perfcounter->inc(l_rgw_pubsub_push_failed);
           return ret;
         }
         if (perfcounter) perfcounter->inc(l_rgw_pubsub_push_ok);
       } catch (const RGWPubSubEndpoint::configuration_error& e) {
-        ldpp_dout(dpp, 1) << "ERROR: failed to create push endpoint: "
-                          << topic.cfg.dest.push_endpoint
-                          << " bucket: " << event_entry.event.bucket_name
-                          << " bucket_owner: "
-                          << event_entry.event.bucket_ownerIdentity
-                          << " object_name: " << event_entry.event.object_key
-                          << ". error: " << e.what() << dendl;
+        ldpp_dout(dpp, 1) << "ERROR: failed to create push endpoint for sync "
+                             "notification event  with  error: "
+                          << e.what() << " event with " << event_entry << dendl;
         if (perfcounter) perfcounter->inc(l_rgw_pubsub_push_failed);
         return -EINVAL;
       }
diff --git a/src/rgw/rgw_amqp.cc b/src/rgw/rgw_amqp.cc
index 67cbbe5c6771..46ab9c575bd6 100644
--- a/src/rgw/rgw_amqp.cc
+++ b/src/rgw/rgw_amqp.cc
@@ -20,7 +20,7 @@
 #include "common/dout.h"
 #include <openssl/ssl.h>
 
-#define dout_subsys ceph_subsys_rgw
+#define dout_subsys ceph_subsys_rgw_notification
 
 // TODO investigation, not necessarily issues:
 // (1) in case of single threaded writer context use spsc_queue
diff --git a/src/rgw/rgw_kafka.cc b/src/rgw/rgw_kafka.cc
index 0d6e77375014..65a1aed0cf91 100644
--- a/src/rgw/rgw_kafka.cc
+++ b/src/rgw/rgw_kafka.cc
@@ -16,7 +16,7 @@
 #include <boost/lockfree/queue.hpp>
 #include "common/dout.h"
 
-#define dout_subsys ceph_subsys_rgw
+#define dout_subsys ceph_subsys_rgw_notification
 
 // TODO investigation, not necessarily issues:
 // (1) in case of single threaded writer context use spsc_queue

From a30cbabefe15022284f9e6b6316a204879a17fc0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 14:33:20 -0500
Subject: [PATCH 2324/2492] mgr/cephadm: fix test failure on newer python

Tests that touch this enum fail for me locally but pass in the CI. This
seems to be due to new enum related behavior in Python 3.11.
See: https://blog.pecar.me/python-enum
Instead of fixing it as suggested in the above blog, adding a __str__
method works on all python versions I care to know about.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py
index 3aedfbd86f00..1ba3e48454a2 100644
--- a/src/pybind/mgr/cephadm/utils.py
+++ b/src/pybind/mgr/cephadm/utils.py
@@ -57,6 +57,9 @@ class SpecialHostLabels(str, Enum):
     def to_json(self) -> str:
         return self.value
 
+    def __str__(self) -> str:
+        return self.value
+
 
 def name_to_config_section(name: str) -> ConfEntity:
     """

From 6873a5c3840c929c9b76b413cdbb29bd8ea9274b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 30 Jan 2024 14:15:19 -0500
Subject: [PATCH 2325/2492] mypy-constrains.txt: bump mypy up to version 1.1.1

In PR #55255 I updated various files to be valid on newer versions of
mypy. Update the constrains file to specify mypy 1.1.1. This certainly
is not the newest version of mypy, but because ceph uses various older
versions of python I am being cautious and trying to only incrementally
update the mypy version. This version fixed an issue or two I hit while
working on a prototype manager module compared to the previous version
in the mypy-constrains.txt file.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/mypy-constrains.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mypy-constrains.txt b/src/mypy-constrains.txt
index a181a87bad4e..7810870804ed 100644
--- a/src/mypy-constrains.txt
+++ b/src/mypy-constrains.txt
@@ -2,7 +2,7 @@
 # Unfortunately this means we have to manually update those 
 # packages regularly. 
 
-mypy==0.981
+mypy==1.1.1
 
 # global
 types-python-dateutil==0.1.3

From 27187de7824a0a98123faabb9bcef3c883cf3c2a Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 18 Mar 2024 16:15:34 -0400
Subject: [PATCH 2326/2492] pybind/mgr: remove spurious commas from tox envlist

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini
index 6270dbdca51e..13aae3820a23 100644
--- a/src/pybind/mgr/tox.ini
+++ b/src/pybind/mgr/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 envlist =
-    py3,
-    mypy,
+    py3
+    mypy
     fix
     flake8
     jinjalint

From 928c54683e17946df23aed4ecb3b0bc9c7c60c9c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 18 Mar 2024 16:16:05 -0400
Subject: [PATCH 2327/2492] pybind/mgr: fix mypy env to allow use with >1
 python version

tox supports automatically running tools with different python versions
if you name your envs properly. Continue to permit the use of `mypy` as
an env (as the default) but also allow a selection of python
versions like `py38-mypy`.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/tox.ini | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini
index 13aae3820a23..62f3ec07ca80 100644
--- a/src/pybind/mgr/tox.ini
+++ b/src/pybind/mgr/tox.ini
@@ -65,12 +65,11 @@ commands =
     pytest {posargs:cephadm/tests/test_ssh.py}
 
 
-[testenv:mypy]
+[testenv:{,py37-,py38-,py39-,py310-}mypy]
 setenv =
     MYPYPATH = {toxinidir}/..
 passenv =
     MYPYPATH
-basepython = python3
 deps =
     -rrequirements.txt
     -c{toxinidir}/../../mypy-constrains.txt

From 40906ddf705f5cce89289f470f5947dda929dff7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 18 Mar 2024 16:17:45 -0400
Subject: [PATCH 2328/2492] pybind/mgr: sort and fix indent in modules list in
 mypy env

In the tox.ini file the modules validated by mypy are explicitly listed.
Fix one item which was indented differently than the others.
Sort the list.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini
index 62f3ec07ca80..082d0dbd53c2 100644
--- a/src/pybind/mgr/tox.ini
+++ b/src/pybind/mgr/tox.ini
@@ -103,10 +103,10 @@ commands =
            -m progress \
            -m prometheus \
            -m rbd_support \
-	   -m rgw \
+           -m rgw \
            -m rook \
-           -m snap_schedule \
            -m selftest \
+           -m snap_schedule \
            -m stats \
            -m status \
            -m telegraf \

From 3276ad571640bc91fe290832a0001bd96f1f26de Mon Sep 17 00:00:00 2001
From: Samuel Just <sjust@redhat.com>
Date: Tue, 19 Mar 2024 23:11:48 +0000
Subject: [PATCH 2329/2492] crimson/.../scrub_events: fix progress lifetime in
 deep_scan_object

seastar::repeat may or may not move the passed action after the first
invocation, so we can't actually rely on references to variables
captured by value being stable.  Instead, allocate with std::make_unique
and move into a finally lambda.

Fixes: https://tracker.ceph.com/issues/64996
Signed-off-by: Samuel Just <sjust@redhat.com>
---
 src/crimson/osd/osd_operations/scrub_events.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/crimson/osd/osd_operations/scrub_events.cc b/src/crimson/osd/osd_operations/scrub_events.cc
index a7b782a71000..dbe22a086bb4 100644
--- a/src/crimson/osd/osd_operations/scrub_events.cc
+++ b/src/crimson/osd/osd_operations/scrub_events.cc
@@ -258,9 +258,10 @@ ScrubScan::ifut<> ScrubScan::deep_scan_object(
   DEBUGDPP("obj: {}", pg, obj);
   using crimson::common::local_conf;
   auto &entry = ret.objects[obj.hobj];
+  auto progress_ref = std::make_unique<obj_scrub_progress_t>();
+  auto &progress = *progress_ref;
   return interruptor::repeat(
-    [FNAME, this, progress = obj_scrub_progress_t(),
-     &obj, &entry, &pg]() mutable
+    [FNAME, this, &progress, &obj, &entry, &pg]()
     -> interruptible_future<seastar::stop_iteration> {
       if (progress.offset) {
 	DEBUGDPP("op: {}, obj: {}, progress: {} scanning data",
@@ -272,7 +273,10 @@ ScrubScan::ifut<> ScrubScan::deep_scan_object(
 	  obj,
 	  *(progress.offset),
 	  stride
-	).safe_then([stride, &progress, &entry](auto bl) {
+	).safe_then([this, FNAME, stride, &obj, &progress, &entry, &pg](auto bl) {
+	  size_t offset = *progress.offset;
+	  DEBUGDPP("op: {}, obj: {}, progress: {} got offset {}",
+		   pg, *this, obj, progress, offset);
 	  progress.data_hash << bl;
 	  if (bl.length() < stride) {
 	    progress.offset = std::nullopt;
@@ -373,7 +377,7 @@ ScrubScan::ifut<> ScrubScan::deep_scan_object(
 	  seastar::make_ready_future<seastar::stop_iteration>(
 	    seastar::stop_iteration::yes));
       }
-    });
+    }).finally([progress_ref=std::move(progress_ref)] {});
 }
 
 template class ScrubAsyncOpT<ScrubScan>;

From de2e66723586d2e9c5fd838e5199547b0d2d5f4f Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Wed, 20 Mar 2024 15:20:57 +0800
Subject: [PATCH 2330/2492] crimson/osd/shard_services: fix log

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/shard_services.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index c694dba7f110..a8da40210e1e 100644
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -183,7 +183,7 @@ seastar::future<> OSDSingletonState::osdmap_subscribe(
   version_t epoch, bool force_request)
 {
   LOG_PREFIX(OSDSingletonState::osdmap_subscribe);
-  INFO("epoch {}");
+  INFO("epoch {}", epoch);
   if (monc.sub_want_increment("osdmap", epoch, CEPH_SUBSCRIBE_ONETIME) ||
       force_request) {
     return monc.renew_subs();

From ceeb1ca4b38dc307e493b04a59d5d07fed23971f Mon Sep 17 00:00:00 2001
From: molpako <35787116+molpako@users.noreply.github.com>
Date: Wed, 20 Mar 2024 18:43:42 +0900
Subject: [PATCH 2331/2492] doc/cephfs: fix architecture link to correct
 relative path

Signed-off-by: molpako <35787116+molpako@users.noreply.github.com>
---
 doc/cephfs/cephfs-io-path.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cephfs/cephfs-io-path.rst b/doc/cephfs/cephfs-io-path.rst
index 8c7810ba0a4e..d5ae17197039 100644
--- a/doc/cephfs/cephfs-io-path.rst
+++ b/doc/cephfs/cephfs-io-path.rst
@@ -47,4 +47,4 @@ client cache.
             |         MDSs        | -=-------> |         OSDs       |
             +---------------------+            +--------------------+
 
-.. _Architecture: ../architecture
+.. _Architecture: ../../architecture

From a60f82566551e731987b4a02d1bb2bf4cfb9a9ce Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 14 Mar 2024 07:58:36 +0100
Subject: [PATCH 2332/2492] mgr/dashboard: fix empty tags

Fixes: https://tracker.ceph.com/issues/64911
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index 6857874411ae..e92cf952c655 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -61,6 +61,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
   kmsVaultConfig = false;
   s3VaultConfig = false;
   tags: Record<string, string>[] = [];
+  dirtyTags = false;
   tagConfig = [
     {
       attribute: 'key'
@@ -437,6 +438,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
 
   deleteTag(index: number) {
     this.tags.splice(index, 1);
+    this.dirtyTags = true;
     this.bucketForm.markAsDirty();
     this.bucketForm.updateValueAndValidity();
   }
@@ -447,11 +449,13 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
     } else {
       this.tags.push(tag);
     }
+    this.dirtyTags = true;
     this.bucketForm.markAsDirty();
     this.bucketForm.updateValueAndValidity();
   }
 
   private tagsToXML(tags: Record<string, string>[]): string {
+    if (!this.dirtyTags && tags.length === 0) return '';
     let xml = '<Tagging><TagSet>';
     for (const tag of tags) {
       xml += '<Tag>';

From a1a093aced774dfc54adfb9f9a9013f3f6a06f0f Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Fri, 8 Mar 2024 06:49:28 +0100
Subject: [PATCH 2333/2492] mgr/dashboard: open api swagger in new tab

Fixes: https://tracker.ceph.com/issues/62350
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../dashboard-help/dashboard-help.component.html  | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/dashboard-help/dashboard-help.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/dashboard-help/dashboard-help.component.html
index d7f7d137768c..34ff79a115e8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/dashboard-help/dashboard-help.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/dashboard-help/dashboard-help.component.html
@@ -10,15 +10,18 @@
   </a>
   <div ngbDropdownMenu>
     <a ngbDropdownItem
-       class="text-capitalize"
        [ngClass]="{'disabled': !docsUrl}"
        href="{{ docsUrl }}"
        target="_blank"
-       i18n>documentation</a>
-    <button ngbDropdownItem
-            routerLink="/api-docs"
-            target="_blank"
-            i18n>API</button>
+       i18n>Documentation
+      <i class="fa fa-external-link"></i>
+    </a>
+    <a ngbDropdownItem
+       routerLink="/api-docs"
+       target="_blank"
+       i18n>API
+      <i class="fa fa-external-link"></i>
+    </a>
     <button ngbDropdownItem
             (click)="openAboutModal()"
             i18n>About</button>

From 62456f384b64a6e17382e91930f8f607cb15aa49 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Wed, 20 Mar 2024 06:35:17 -0500
Subject: [PATCH 2334/2492] osd/scrub: no "slow response" warning for queued
 reservations

When sending a "queued reservation" request (the new default), the OSD
will not respond until the reservation is granted.  This can take a
long time, and the client should not be warned about a "slow response".

Fixes: https://tracker.ceph.com/issues/64869

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/osd/scrubber/scrub_reservations.cc | 16 +---------------
 src/osd/scrubber/scrub_reservations.h  |  4 ----
 2 files changed, 1 insertion(+), 19 deletions(-)

diff --git a/src/osd/scrubber/scrub_reservations.cc b/src/osd/scrubber/scrub_reservations.cc
index 284d90290578..b9cc2d096ad0 100644
--- a/src/osd/scrubber/scrub_reservations.cc
+++ b/src/osd/scrubber/scrub_reservations.cc
@@ -60,13 +60,9 @@ ReplicaReservations::ReplicaReservations(
     dout(10) << "high-priority scrub - no reservations needed" << dendl;
     m_perf_set.inc(scrbcnt_resrv_skipped);
   } else {
-    m_process_started_at = ScrubClock::now();
-
     // send out the 1'st request (unless we have no replicas)
+    m_process_started_at = ScrubClock::now();
     send_next_reservation_or_complete();
-    m_slow_response_warn_timeout =
-	m_scrubber.get_pg_cct()->_conf.get_val<milliseconds>(
-	    "osd_scrub_slow_reservation_response");
   }
 }
 
@@ -174,16 +170,6 @@ bool ReplicaReservations::handle_reserve_grant(
   }
 
   auto elapsed = ScrubClock::now() - m_last_request_sent_at;
-
-  // log a warning if the response was slow to arrive
-  if ((m_slow_response_warn_timeout > 0ms) &&
-      (elapsed > m_slow_response_warn_timeout)) {
-    m_osds->clog->warn() << fmt::format(
-	"slow reservation response from {} ({}ms)", from,
-	duration_cast<milliseconds>(elapsed).count());
-    // prevent additional warnings
-    m_slow_response_warn_timeout = 0ms;
-  }
   dout(10) << fmt::format(
 		  "(e:{} nonce:{}) granted by {} ({} of {}) in {}ms",
 		  msg.map_epoch, msg.reservation_nonce, from,
diff --git a/src/osd/scrubber/scrub_reservations.h b/src/osd/scrubber/scrub_reservations.h
index 2e13760ffa1f..173b23d7db52 100644
--- a/src/osd/scrubber/scrub_reservations.h
+++ b/src/osd/scrubber/scrub_reservations.h
@@ -90,10 +90,6 @@ class ReplicaReservations {
    */
   reservation_nonce_t& m_last_request_sent_nonce;
 
-  /// the 'slow response' timeout (in milliseconds) - as configured.
-  /// Doubles as a 'do once' flag for the warning.
-  std::chrono::milliseconds m_slow_response_warn_timeout;
-
   /// access to the performance counters container relevant to this scrub
   /// parameters
   PerfCounters& m_perf_set;

From 0bd8d3b6bc8ecbe99e22545549a63de3444e5f75 Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Tue, 19 Mar 2024 13:13:17 +0000
Subject: [PATCH 2335/2492] mgr/dashboard: fix M retention frequency display

Fix snap schedule list monthly retention frequency display

Fixes: https://tracker.ceph.com/issues/64982

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../src/app/shared/enum/retention-frequency.enum.ts         | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
index 193418a1783c..b24a337ee1cf 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/retention-frequency.enum.ts
@@ -1,8 +1,9 @@
 export enum RetentionFrequency {
+  Minutely = 'm',
   Hourly = 'h',
   Daily = 'd',
   Weekly = 'w',
-  Monthly = 'm',
+  Monthly = 'M',
   Yearly = 'y',
   'lastest snapshots' = 'n'
 }
@@ -11,7 +12,8 @@ export enum RetentionFrequencyCopy {
   h = 'Hourly',
   d = 'Daily',
   w = 'Weekly',
-  m = 'Monthly',
+  M = 'Monthly',
+  m = 'Minutely',
   y = 'Yearly',
   n = 'lastest snapshots'
 }

From 58923668320ba373312c68a8869a1089da41d804 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 Nov 2023 14:15:14 -0500
Subject: [PATCH 2336/2492] include: use larger int for large gathers

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/include/Context.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/include/Context.h b/src/include/Context.h
index bef85ca5b52f..b89c4fe8220a 100644
--- a/src/include/Context.h
+++ b/src/include/Context.h
@@ -310,8 +310,8 @@ class C_GatherBase {
 #ifdef DEBUG_GATHER
   std::set<ContextType*> waitfor;
 #endif
-  int sub_created_count = 0;
-  int sub_existing_count = 0;
+  uint64_t sub_created_count = 0;
+  uint64_t sub_existing_count = 0;
   mutable ceph::recursive_mutex lock =
     ceph::make_recursive_mutex("C_GatherBase::lock"); // disable lockdep
   bool activated = false;

From 770ab517c2b05045c09af3d0a56b6446bd20bb58 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 16 Feb 2024 20:43:06 -0500
Subject: [PATCH 2337/2492] common: use more efficient vector for stack

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/Formatter.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/common/Formatter.h b/src/common/Formatter.h
index 1919b018a67c..cddfdab0c4f0 100644
--- a/src/common/Formatter.h
+++ b/src/common/Formatter.h
@@ -13,6 +13,7 @@
 #include <stdarg.h>
 #include <sstream>
 #include <map>
+#include <vector>
 
 namespace ceph {
 
@@ -204,7 +205,7 @@ namespace ceph {
     copyable_sstream m_ss;
     copyable_sstream m_pending_string;
     std::string m_pending_name;
-    std::list<json_formatter_stack_entry_d> m_stack;
+    std::vector<json_formatter_stack_entry_d> m_stack;
     bool m_is_pending_string;
     bool m_line_break_enabled = false;
   };

From 8928a1266fc6bc492dac34253edb36c045e491e5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 17 Feb 2024 10:22:29 -0500
Subject: [PATCH 2338/2492] common: add JSONFormatterFile class

To stream JSON to a file for collection (for testing).  The idea here that
trying to send a very large JSON result over `ceph tell` is best to be avoided.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/Formatter.cc | 53 +++++++++++-----------
 src/common/Formatter.h  | 99 ++++++++++++++++++++++++++++++++++-------
 2 files changed, 108 insertions(+), 44 deletions(-)

diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc
index f121afa07a3e..b43e04a8bd09 100644
--- a/src/common/Formatter.cc
+++ b/src/common/Formatter.cc
@@ -148,12 +148,6 @@ void Formatter::dump_format_unquoted(std::string_view name, const char *fmt, ...
 
 // -----------------------
 
-JSONFormatter::JSONFormatter(bool p)
-: m_pretty(p), m_is_pending_string(false)
-{
-  reset();
-}
-
 void JSONFormatter::flush(std::ostream& os)
 {
   finish_pending_string();
@@ -175,30 +169,33 @@ void JSONFormatter::reset()
 
 void JSONFormatter::print_comma(json_formatter_stack_entry_d& entry)
 {
+  auto& ss = get_ss();
   if (entry.size) {
     if (m_pretty) {
-      m_ss << ",\n";
+      ss << ",\n";
       for (unsigned i = 1; i < m_stack.size(); i++)
-        m_ss << "    ";
+        ss << "    ";
     } else {
-      m_ss << ",";
+      ss << ",";
     }
   } else if (m_pretty) {
-    m_ss << "\n";
+    ss << "\n";
     for (unsigned i = 1; i < m_stack.size(); i++)
-      m_ss << "    ";
+      ss << "    ";
   }
   if (m_pretty && entry.is_array)
-    m_ss << "    ";
+    ss << "    ";
 }
 
 void JSONFormatter::print_quoted_string(std::string_view s)
 {
-  m_ss << '\"' << json_stream_escaper(s) << '\"';
+  auto& ss = get_ss();
+  ss << '\"' << json_stream_escaper(s) << '\"';
 }
 
 void JSONFormatter::print_name(std::string_view name)
 {
+  auto& ss = get_ss();
   finish_pending_string();
   if (m_stack.empty())
     return;
@@ -206,19 +203,20 @@ void JSONFormatter::print_name(std::string_view name)
   print_comma(entry);
   if (!entry.is_array) {
     if (m_pretty) {
-      m_ss << "    ";
+      ss << "    ";
     }
-    m_ss << "\"" << name << "\"";
+    ss << "\"" << name << "\"";
     if (m_pretty)
-      m_ss << ": ";
+      ss << ": ";
     else
-      m_ss << ':';
+      ss << ':';
   }
   ++entry.size;
 }
 
 void JSONFormatter::open_section(std::string_view name, const char *ns, bool is_array)
 {
+  auto& ss = get_ss();
   if (handle_open_section(name, ns, is_array)) {
     return;
   }
@@ -230,9 +228,9 @@ void JSONFormatter::open_section(std::string_view name, const char *ns, bool is_
     print_name(name);
   }
   if (is_array)
-    m_ss << '[';
+    ss << '[';
   else
-    m_ss << '{';
+    ss << '{';
 
   json_formatter_stack_entry_d n;
   n.is_array = is_array;
@@ -261,7 +259,7 @@ void JSONFormatter::open_object_section_in_ns(std::string_view name, const char
 
 void JSONFormatter::close_section()
 {
-
+  auto& ss = get_ss();
   if (handle_close_section()) {
     return;
   }
@@ -270,14 +268,14 @@ void JSONFormatter::close_section()
 
   struct json_formatter_stack_entry_d& entry = m_stack.back();
   if (m_pretty && entry.size) {
-    m_ss << "\n";
+    ss << "\n";
     for (unsigned i = 1; i < m_stack.size(); i++)
-      m_ss << "    ";
+      ss << "    ";
   }
-  m_ss << (entry.is_array ? ']' : '}');
+  ss << (entry.is_array ? ']' : '}');
   m_stack.pop_back();
   if (m_pretty && m_stack.empty())
-    m_ss << "\n";
+    ss << "\n";
 }
 
 void JSONFormatter::finish_pending_string()
@@ -300,12 +298,13 @@ void JSONFormatter::add_value(std::string_view name, T val)
 
 void JSONFormatter::add_value(std::string_view name, std::string_view val, bool quoted)
 {
+  auto& ss = get_ss();
   if (handle_value(name, val, quoted)) {
     return;
   }
   print_name(name);
   if (!quoted) {
-    m_ss << val;
+    ss << val;
   } else {
     print_quoted_string(val);
   }
@@ -354,12 +353,12 @@ void JSONFormatter::dump_format_va(std::string_view name, const char *ns, bool q
 
 int JSONFormatter::get_len() const
 {
-  return m_ss.str().size();
+  return m_ss.tellp();
 }
 
 void JSONFormatter::write_raw_data(const char *data)
 {
-  m_ss << data;
+  get_ss() << data;
 }
 
 const char *XMLFormatter::XML_1_DTD =
diff --git a/src/common/Formatter.h b/src/common/Formatter.h
index cddfdab0c4f0..65ffb0a68551 100644
--- a/src/common/Formatter.h
+++ b/src/common/Formatter.h
@@ -7,6 +7,7 @@
 #include "include/buffer_fwd.h"
 
 #include <deque>
+#include <fstream>
 #include <list>
 #include <memory>
 #include <vector>
@@ -130,21 +131,52 @@ namespace ceph {
     virtual void write_bin_data(const char* buff, int buf_len);
   };
 
-  class copyable_sstream : public std::stringstream {
+  class JSONFormatter : public Formatter {
   public:
-    copyable_sstream() {}
-    copyable_sstream(const copyable_sstream& rhs) {
-      str(rhs.str());
+    explicit JSONFormatter(bool p = false) : m_pretty(p) {}
+    JSONFormatter(const JSONFormatter& f) :
+      m_pretty(f.m_pretty),
+      m_pending_name(f.m_pending_name),
+      m_stack(f.m_stack),
+      m_is_pending_string(f.m_is_pending_string),
+      m_line_break_enabled(f.m_line_break_enabled)
+    {
+      m_ss.str(f.m_ss.str());
+      m_pending_string.str(f.m_pending_string.str());
+    }
+    JSONFormatter(JSONFormatter&& f) :
+      m_pretty(f.m_pretty),
+      m_ss(std::move(f.m_ss)),
+      m_pending_string(std::move(f.m_pending_string)),
+      m_pending_name(f.m_pending_name),
+      m_stack(std::move(f.m_stack)),
+      m_is_pending_string(f.m_is_pending_string),
+      m_line_break_enabled(f.m_line_break_enabled)
+    {
     }
-    copyable_sstream& operator=(const copyable_sstream& rhs) {
-      str(rhs.str());
+    JSONFormatter& operator=(const JSONFormatter& f)
+    {
+      m_pretty = f.m_pretty;
+      m_ss.str(f.m_ss.str());
+      m_pending_string.str(f.m_pending_string.str());
+      m_pending_name = f.m_pending_name;
+      m_stack = f.m_stack;
+      m_is_pending_string = f.m_is_pending_string;
+      m_line_break_enabled = f.m_line_break_enabled;
       return *this;
     }
-  };
 
-  class JSONFormatter : public Formatter {
-  public:
-    explicit JSONFormatter(bool p = false);
+    JSONFormatter& operator=(JSONFormatter&& f)
+    {
+      m_pretty = f.m_pretty;
+      m_ss = std::move(f.m_ss);
+      m_pending_string = std::move(f.m_pending_string);
+      m_pending_name = f.m_pending_name;
+      m_stack = std::move(f.m_stack);
+      m_is_pending_string = f.m_is_pending_string;
+      m_line_break_enabled = f.m_line_break_enabled;
+      return *this;
+    }
 
     void set_status(int status, const char* status_name) override {};
     void output_header() override {};
@@ -183,15 +215,18 @@ namespace ceph {
 
     int stack_size() { return m_stack.size(); }
 
+    virtual std::ostream& get_ss() {
+      return m_ss;
+    }
+
   private:
 
     struct json_formatter_stack_entry_d {
-      int size;
-      bool is_array;
-      json_formatter_stack_entry_d() : size(0), is_array(false) { }
+      int size = 0;
+      bool is_array = false;
     };
 
-    bool m_pretty;
+    bool m_pretty = false;
     void open_section(std::string_view name, const char *ns, bool is_array);
     void print_quoted_string(std::string_view s);
     void print_name(std::string_view name);
@@ -202,14 +237,44 @@ namespace ceph {
     void add_value(std::string_view name, T val);
     void add_value(std::string_view name, std::string_view val, bool quoted);
 
-    copyable_sstream m_ss;
-    copyable_sstream m_pending_string;
+    mutable std::stringstream m_ss; // mutable for get_len
+    std::stringstream m_pending_string;
     std::string m_pending_name;
     std::vector<json_formatter_stack_entry_d> m_stack;
-    bool m_is_pending_string;
+    bool m_is_pending_string = false;
     bool m_line_break_enabled = false;
   };
 
+  class JSONFormatterFile : public JSONFormatter {
+public:
+    JSONFormatterFile(const std::string& path, bool pretty=false) :
+      JSONFormatter(pretty),
+      path(path),
+      file(path, std::ios::out | std::ios::trunc)
+    {
+    }
+    ~JSONFormatterFile() {
+      file.flush();
+    }
+
+    void reset() override {
+      JSONFormatter::reset();
+      file = std::ofstream(path, std::ios::out | std::ios::trunc);
+    }
+    int get_len() const override {
+      return file.tellp();
+    }
+
+protected:
+    std::ostream& get_ss() override {
+      return file;
+    }
+
+private:
+    std::string path;
+    mutable std::ofstream file; // mutable for get_len
+  };
+
   template <class T>
   void add_value(std::string_view name, T val);
 

From cdef9b6dd0a4d0fe39843c5a8198295d14b1c2e7 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 16 Nov 2023 12:07:32 -0500
Subject: [PATCH 2339/2492] common: simplify loop

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/TrackedOp.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/common/TrackedOp.cc b/src/common/TrackedOp.cc
index 32a1ab472a89..0de21e38743a 100644
--- a/src/common/TrackedOp.cc
+++ b/src/common/TrackedOp.cc
@@ -207,14 +207,11 @@ void OpHistory::dump_slow_ops(utime_t now, Formatter *f, set<string> filters)
   f->dump_int("threshold to keep", history_slow_op_threshold.load());
   {
     f->open_array_section("Ops");
-    for (set<pair<utime_t, TrackedOpRef> >::const_iterator i =
-	   slow_op.begin();
-	 i != slow_op.end();
-	 ++i) {
-      if (!i->second->filter_out(filters))
+    for (const auto& [t, op] : slow_op) {
+      if (!op->filter_out(filters))
         continue;
       f->open_object_section("Op");
-      i->second->dump(now, f, OpTracker::default_dumper);
+      op->dump(now, f, OpTracker::default_dumper);
       f->close_section();
     }
     f->close_section();

From c036b151465a2ecb8bf24943e1030988f7e6c6ce Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 16 Nov 2023 12:36:25 -0500
Subject: [PATCH 2340/2492] common: support long running ops without slow
 warnings

A quiesce op in the MDS may run for an indeterminate amount of time. Warnings
should not be generated.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/TrackedOp.cc |  6 +++++-
 src/common/TrackedOp.h  | 23 +++++++++++++++++++++--
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/common/TrackedOp.cc b/src/common/TrackedOp.cc
index 0de21e38743a..8de8c7d1ff41 100644
--- a/src/common/TrackedOp.cc
+++ b/src/common/TrackedOp.cc
@@ -207,7 +207,7 @@ void OpHistory::dump_slow_ops(utime_t now, Formatter *f, set<string> filters)
   f->dump_int("threshold to keep", history_slow_op_threshold.load());
   {
     f->open_array_section("Ops");
-    for (const auto& [t, op] : slow_op) {
+    for ([[maybe_unused]] const auto& [t, op] : slow_op) {
       if (!op->filter_out(filters))
         continue;
       f->open_object_section("Op");
@@ -388,6 +388,9 @@ bool OpTracker::with_slow_ops_in_flight(utime_t* oldest_secs,
       // no more slow ops in flight
       return false;
     }
+    if (op.is_continuous()) {
+      return true; /* skip reporting */
+    }
     if (!op.warn_interval_multiplier)
       return true;
     slow++;
@@ -502,6 +505,7 @@ void TrackedOp::dump(utime_t now, Formatter *f, OpTracker::dumper lambda) const
   f->dump_stream("initiated_at") << get_initiated();
   f->dump_float("age", now - get_initiated());
   f->dump_float("duration", get_duration());
+  f->dump_bool("continuous", is_continuous());
   {
     f->open_object_section("type_data");
     lambda(*this, f);
diff --git a/src/common/TrackedOp.h b/src/common/TrackedOp.h
index 238f1c7ac7c5..1a921309ca01 100644
--- a/src/common/TrackedOp.h
+++ b/src/common/TrackedOp.h
@@ -207,6 +207,9 @@ class OpTracker {
   template <typename T, typename U>
   typename T::Ref create_request(U params)
   {
+    constexpr bool has_is_continuous = requires(U u) {
+      { u->is_continuous() } -> std::same_as<bool>;
+    };
     typename T::Ref retval(new T(params, this));
     retval->tracking_start();
     if (is_tracking()) {
@@ -215,18 +218,25 @@ class OpTracker {
       retval->mark_event("all_read", params->get_recv_complete_stamp());
       retval->mark_event("dispatched", params->get_dispatch_stamp());
     }
+    if constexpr (has_is_continuous) {
+      if (params->is_continuous()) {
+        retval->mark_continuous();
+      }
+    }
 
     return retval;
   }
 };
 
 class TrackedOp : public boost::intrusive::list_base_hook<> {
-private:
+public:
   friend class OpHistory;
   friend class OpTracker;
 
-  boost::intrusive::list_member_hook<> tracker_item;
+  static const uint64_t FLAG_CONTINUOUS = (1<<1);
 
+private:
+  boost::intrusive::list_member_hook<> tracker_item;
 public:
   typedef boost::intrusive::list<
   TrackedOp,
@@ -243,6 +253,7 @@ class TrackedOp : public boost::intrusive::list_base_hook<> {
     }
   };
 
+
 protected:
   OpTracker *tracker;          ///< the tracker we are associated with
   std::atomic_int nref = {0};  ///< ref count
@@ -281,6 +292,14 @@ class TrackedOp : public boost::intrusive::list_base_hook<> {
     STATE_HISTORY
   };
   std::atomic<int> state = {STATE_UNTRACKED};
+  uint64_t flags = 0;
+
+  void mark_continuous() {
+    flags |= FLAG_CONTINUOUS;
+  }
+  bool is_continuous() const {
+    return flags & FLAG_CONTINUOUS;
+  }
 
   TrackedOp(OpTracker *_tracker, const utime_t& initiated) :
     tracker(_tracker),

From 92df0fcc649345f438d0d7cc6fab658c02060f07 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 16 Jan 2024 13:59:47 -0500
Subject: [PATCH 2341/2492] common: provide a template for object dumps

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/dout.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/common/dout.h b/src/common/dout.h
index 6516060c5438..8d05b12fbe2b 100644
--- a/src/common/dout.h
+++ b/src/common/dout.h
@@ -44,6 +44,18 @@ inline std::ostream& operator<<(std::ostream& out, _bad_endl_use_dendl_t) {
   return out;
 }
 
+template<typename T>
+concept HasPrint = requires(T t, std::ostream& u) {
+  { t.print(u) } -> std::same_as<void>;
+};
+
+template<typename T> requires HasPrint<T>
+static inline std::ostream& operator<<(std::ostream& out, T&& t)
+{
+  t.print(out);
+  return out;
+}
+
 class DoutPrefixProvider {
 public:
   virtual std::ostream& gen_prefix(std::ostream& out) const = 0;

From ed5987883a411d090d1820ef00453e8147bc1753 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 7 Feb 2024 14:59:06 -0500
Subject: [PATCH 2342/2492] mds: remove dead method

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 30e004b97a6e..990951b5531e 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -418,7 +418,6 @@ class MDCache {
     return active_requests.count(rid);
   }
   MDRequestRef request_get(metareqid_t rid);
-  void request_pin_ref(const MDRequestRef& r, CInode *ref, std::vector<CDentry*>& trace);
   void request_finish(const MDRequestRef& mdr);
   void request_forward(const MDRequestRef& mdr, mds_rank_t mds, int port=0);
   void dispatch_request(const MDRequestRef& mdr);

From 944d00ca1fcafb39e45cb431c63f00e71a7976df Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 8 Feb 2024 12:30:38 -0500
Subject: [PATCH 2343/2492] mds: allow specifying rdlock position

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Mutation.h | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index bc83f2191514..18266c447c31 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -86,30 +86,33 @@ struct MutationImpl : public TrackedOp {
       reserve(32);
     }
 
-    void add_rdlock(SimpleLock *lock) {
-      emplace_back(lock, LockOp::RDLOCK);
+    void add_rdlock(SimpleLock *lock, int idx=-1) {
+      add_lock(LockOp(lock, LockOp::RDLOCK), idx);
     }
     void erase_rdlock(SimpleLock *lock);
     void add_xlock(SimpleLock *lock, int idx=-1) {
-      if (idx >= 0)
-	emplace(cbegin() + idx, lock, LockOp::XLOCK);
-      else
-	emplace_back(lock, LockOp::XLOCK);
+      add_lock(LockOp(lock, LockOp::XLOCK), idx);
     }
     void add_wrlock(SimpleLock *lock, int idx=-1) {
-      if (idx >= 0)
-	emplace(cbegin() + idx, lock, LockOp::WRLOCK);
-      else
-	emplace_back(lock, LockOp::WRLOCK);
+      add_lock(LockOp(lock, LockOp::WRLOCK), idx);
     }
     void add_remote_wrlock(SimpleLock *lock, mds_rank_t rank) {
       ceph_assert(rank != MDS_RANK_NONE);
-      emplace_back(lock, LockOp::REMOTE_WRLOCK, rank);
+      add_lock(LockOp(lock, LockOp::REMOTE_WRLOCK, rank), -1);
     }
     void lock_scatter_gather(SimpleLock *lock) {
-      emplace_back(lock, LockOp::WRLOCK | LockOp::STATE_PIN);
+      add_lock(LockOp(lock, LockOp::WRLOCK | LockOp::STATE_PIN), -1);
     }
     void sort_and_merge();
+
+  protected:
+    void add_lock(LockOp op, int idx) {
+      if (idx >= 0) {
+	emplace(cbegin() + idx, std::move(op));
+      } else {
+	emplace_back(std::move(op));
+      }
+    }
   };
 
   using lock_set = std::set<LockOp>;

From 32a544f9a71aa68547db5bb0fdd517cafbf7d928 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 5 Jan 2024 15:06:20 -0500
Subject: [PATCH 2344/2492] mds: add comment on locking order rules

From a past commit.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index f79e806494bf..5022b23dd0f3 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8240,6 +8240,24 @@ void MDCache::dispatch(const cref_t<Message> &m)
   }
 }
 
+/**
+ * In 246f647566095c173e5e0e54661696cea230f96e, an updated rule for locking order
+ * was established (differing from past strategies):
+ *
+ *  [The helper function is for requests that operate on two paths. It
+ *  ensures that the two paths get locks in proper order.] The rule is:
+ *
+ *   1. Lock directory inodes or dentries according to which trees they
+ *      are under. Lock objects under fs root before objects under mdsdir.
+ *   2. Lock directory inodes or dentries according to their depth, in
+ *      ascending order.
+ *   3. Lock directory inodes or dentries according to inode numbers or
+ *      dentries' parent inode numbers, in ascending order.
+ *   4. Lock dentries in the same directory in order of their keys.
+ *   5. Lock non-directory inodes according to inode numbers, in ascending
+ *      order.
+ */
+
 int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
                            const filepath& path, int flags,
                            vector<CDentry*> *pdnvec, CInode **pin)

From d1765e621611fef5517c2f2227d7786ae41e2001 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 23 Jan 2024 21:27:18 -0500
Subject: [PATCH 2345/2492] mds: add request result to mutation for analysis by
 tests

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Mutation.cc | 1 +
 src/mds/Mutation.h  | 3 +++
 src/mds/Server.cc   | 1 +
 3 files changed, 5 insertions(+)

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index b52e04a63936..9b71e244e08a 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -472,6 +472,7 @@ void MDRequestImpl::print(ostream &out) const
 void MDRequestImpl::_dump(Formatter *f, bool has_mds_lock) const
 {
   std::lock_guard l(lock);
+  f->dump_int("result", result);
   f->dump_string("flag_point", _get_state_string());
   f->dump_object("reqid", reqid);
   if (client_request) {
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 18266c447c31..1a308c423f51 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -15,6 +15,8 @@
 #ifndef CEPH_MDS_MUTATION_H
 #define CEPH_MDS_MUTATION_H
 
+#include <limits>
+
 #include "include/interval_set.h"
 #include "include/elist.h"
 #include "include/filepath.h"
@@ -227,6 +229,7 @@ struct MutationImpl : public TrackedOp {
   void _dump_op_descriptor(std::ostream& stream) const override;
 
   metareqid_t reqid;
+  int result = std::numeric_limits<int>::min();
   __u32 attempt = 0;      // which attempt for this request
   LogSegment *ls = nullptr;  // the log segment i'm committing to
 
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index ed42ac439d82..598452c32fd4 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2073,6 +2073,7 @@ void Server::submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin, const MDRe
  */
 void Server::respond_to_request(const MDRequestRef& mdr, int r)
 {
+  mdr->result = r;
   if (mdr->client_request) {
     if (mdr->is_batch_head()) {
       dout(20) << __func__ << " batch head " << *mdr << dendl;

From d1b066cdbd1ec65b620996cb7f4da7006dfcb7d0 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 16 Jan 2024 14:01:54 -0500
Subject: [PATCH 2346/2492] mds: use new insert template via print

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/LogEvent.h         |   5 --
 src/mds/MDSCacheObject.h   |   5 --
 src/mds/Mutation.h         |   5 --
 src/mds/SimpleLock.h       |   5 --
 src/mds/events/EMetaBlob.h |   5 --
 src/mds/mdstypes.h         | 125 +++++++++++++++++--------------------
 6 files changed, 57 insertions(+), 93 deletions(-)

diff --git a/src/mds/LogEvent.h b/src/mds/LogEvent.h
index 49955290fd69..1e748d904fc8 100644
--- a/src/mds/LogEvent.h
+++ b/src/mds/LogEvent.h
@@ -128,9 +128,4 @@ class LogEvent {
   LogSegment *_segment = nullptr;
 };
 
-inline std::ostream& operator<<(std::ostream& out, const LogEvent &le) {
-  le.print(out);
-  return out;
-}
-
 #endif
diff --git a/src/mds/MDSCacheObject.h b/src/mds/MDSCacheObject.h
index 8710102b70d9..7da2a67c3a27 100644
--- a/src/mds/MDSCacheObject.h
+++ b/src/mds/MDSCacheObject.h
@@ -326,11 +326,6 @@ class MDSCacheObject {
   static uint64_t last_wait_seq;
 };
 
-inline std::ostream& operator<<(std::ostream& out, const MDSCacheObject& o) {
-  o.print(out);
-  return out;
-}
-
 inline std::ostream& operator<<(std::ostream& out, const mdsco_db_line_prefix& o) {
   o.object->print_db_line_prefix(out);
   return out;
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 1a308c423f51..74444d548ff0 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -543,9 +543,4 @@ struct MDLockCache : public MutationImpl {
 typedef boost::intrusive_ptr<MutationImpl> MutationRef;
 typedef boost::intrusive_ptr<MDRequestImpl> MDRequestRef;
 
-inline std::ostream& operator<<(std::ostream &out, const MutationImpl &mut)
-{
-  mut.print(out);
-  return out;
-}
 #endif
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 2a7a5fc806e7..33c091b6244e 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -660,9 +660,4 @@ class SimpleLock {
 };
 WRITE_CLASS_ENCODER(SimpleLock)
 
-inline std::ostream& operator<<(std::ostream& out, const SimpleLock& l) 
-{
-  l.print(out);
-  return out;
-}
 #endif
diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h
index 6c995dddeb02..02607ce8ba13 100644
--- a/src/mds/events/EMetaBlob.h
+++ b/src/mds/events/EMetaBlob.h
@@ -611,9 +611,4 @@ WRITE_CLASS_ENCODER(EMetaBlob::remotebit)
 WRITE_CLASS_ENCODER(EMetaBlob::nullbit)
 WRITE_CLASS_ENCODER_FEATURES(EMetaBlob::dirlump)
 
-inline std::ostream& operator<<(std::ostream& out, const EMetaBlob& t) {
-  t.print(out);
-  return out;
-}
-
 #endif
diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index 3381d44c95f1..a39ec5abbd5d 100644
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -85,12 +85,13 @@ class mds_role_t {
     return (rank == MDS_RANK_NONE);
   }
 
+  void print(std::ostream& out) const {
+    out << fscid << ":" << rank;
+  }
+
   fs_cluster_id_t fscid = FS_CLUSTER_ID_NONE;
   mds_rank_t rank = MDS_RANK_NONE;
 };
-inline std::ostream& operator<<(std::ostream& out, const mds_role_t& role) {
-  return out << role.fscid << ":" << role.rank;
-}
 
 // CAPS
 inline std::string gcap_string(int cap)
@@ -277,15 +278,15 @@ struct old_rstat_t {
   void dump(ceph::Formatter *f) const;
   static void generate_test_instances(std::list<old_rstat_t*>& ls);
 
+  void print(std::ostream& out) const {
+    out << "old_rstat(first " << first << " " << rstat << " " << accounted_rstat << ")";
+  }
+
   snapid_t first;
   nest_info_t rstat, accounted_rstat;
 };
 WRITE_CLASS_ENCODER(old_rstat_t)
 
-inline std::ostream& operator<<(std::ostream& out, const old_rstat_t& o) {
-  return out << "old_rstat(first " << o.first << " " << o.rstat << " " << o.accounted_rstat << ")";
-}
-
 class feature_bitset_t {
 public:
   typedef uint64_t block_type;
@@ -351,11 +352,6 @@ class feature_bitset_t {
 };
 WRITE_CLASS_ENCODER(feature_bitset_t)
 
-inline std::ostream& operator<<(std::ostream& out, const feature_bitset_t& s) {
-  s.print(out);
-  return out;
-}
-
 struct metric_spec_t {
   metric_spec_t() {}
   metric_spec_t(const metric_spec_t& other) :
@@ -394,11 +390,6 @@ struct metric_spec_t {
 };
 WRITE_CLASS_ENCODER(metric_spec_t)
 
-inline std::ostream& operator<<(std::ostream& out, const metric_spec_t& mst) {
-  mst.print(out);
-  return out;
-}
-
 /*
  * client_metadata_t
  */
@@ -480,6 +471,10 @@ struct dentry_key_t {
   dentry_key_t(snapid_t s, std::string_view n, __u32 h=0) :
     snapid(s), name(n), hash(h) {}
 
+  void print(std::ostream& out) const {
+    out << "(" << name << "," << snapid << ")";
+  }
+
   bool is_valid() { return name.length() || snapid; }
 
   // encode into something that can be decoded as a string.
@@ -530,11 +525,6 @@ struct dentry_key_t {
   __u32 hash = 0;
 };
 
-inline std::ostream& operator<<(std::ostream& out, const dentry_key_t &k)
-{
-  return out << "(" << k.name << "," << k.snapid << ")";
-}
-
 inline bool operator<(const dentry_key_t& k1, const dentry_key_t& k2)
 {
   /*
@@ -556,6 +546,10 @@ struct string_snap_t {
   string_snap_t() {}
   string_snap_t(std::string_view n, snapid_t s) : name(n), snapid(s) {}
 
+  void print(std::ostream& out) const {
+    out << "(" << name << "," << snapid << ")";
+  }
+
   int compare(const string_snap_t& r) const {
     int ret = name.compare(r.name);
     if (ret)
@@ -584,11 +578,6 @@ inline bool operator<(const string_snap_t& l, const string_snap_t& r) {
   return c < 0 || (c == 0 && l.snapid < r.snapid);
 }
 
-inline std::ostream& operator<<(std::ostream& out, const string_snap_t &k)
-{
-  return out << "(" << k.name << "," << k.snapid << ")";
-}
-
 /*
  * mds_table_pending_t
  *
@@ -622,16 +611,15 @@ struct metareqid_t {
     decode(tid, p);
   }
   void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const {
+    out << name << ":" << tid;
+  }
 
   entity_name_t name;
   uint64_t tid = 0;
 };
 WRITE_CLASS_ENCODER(metareqid_t)
 
-inline std::ostream& operator<<(std::ostream& out, const metareqid_t& r) {
-  return out << r.name << ":" << r.tid;
-}
-
 inline bool operator==(const metareqid_t& l, const metareqid_t& r) {
   return (l.name == r.name) && (l.tid == r.tid);
 }
@@ -761,6 +749,13 @@ struct dirfrag_t {
   dirfrag_t() {}
   dirfrag_t(inodeno_t i, frag_t f) : ino(i), frag(f) { }
 
+  void print(std::ostream& out) const {
+    out << ino;
+    if (!frag.is_root()) {
+      out << "." << frag;
+    }
+  }
+
   void encode(ceph::buffer::list& bl) const {
     using ceph::encode;
     encode(ino, bl);
@@ -777,11 +772,6 @@ struct dirfrag_t {
 };
 WRITE_CLASS_ENCODER(dirfrag_t)
 
-inline std::ostream& operator<<(std::ostream& out, const dirfrag_t &df) {
-  out << df.ino;
-  if (!df.frag.is_root()) out << "." << df.frag;
-  return out;
-}
 inline bool operator<(dirfrag_t l, dirfrag_t r) {
   if (l.ino < r.ino) return true;
   if (l.ino == r.ino && l.frag < r.frag) return true;
@@ -875,6 +865,18 @@ class dirfrag_load_vec_t {
   }
   void dump(ceph::Formatter *f) const;
   void dump(ceph::Formatter *f, const DecayRate& rate) const;
+  void print(std::ostream& out) const {
+    CachedStackStringStream css;
+    *css << std::setprecision(1) << std::fixed
+         << "[pop"
+            " IRD:" << vec[0]
+         << " IWR:" << vec[1]
+         << " RDR:" << vec[2]
+         << " FET:" << vec[3]
+         << " STR:" << vec[4]
+         << " *LOAD:" << meta_load() << "]";
+    out << css->strv();
+  }
   static void generate_test_instances(std::list<dirfrag_load_vec_t*>& ls);
 
   const DecayCounter &get(int t) const {
@@ -916,7 +918,6 @@ class dirfrag_load_vec_t {
   }
 
 private:
-  friend inline std::ostream& operator<<(std::ostream& out, const dirfrag_load_vec_t& dl);
   std::array<DecayCounter, NUM> vec;
 };
 
@@ -927,20 +928,6 @@ inline void decode(dirfrag_load_vec_t& c, ceph::buffer::list::const_iterator &p)
   c.decode(p);
 }
 
-inline std::ostream& operator<<(std::ostream& out, const dirfrag_load_vec_t& dl)
-{
-  CachedStackStringStream css;
-  *css << std::setprecision(1) << std::fixed
-     << "[pop"
-        " IRD:" << dl.vec[0]
-     << " IWR:" << dl.vec[1]
-     << " RDR:" << dl.vec[2]
-     << " FET:" << dl.vec[3]
-     << " STR:" << dl.vec[4]
-     << " *LOAD:" << dl.meta_load() << "]";
-  return out << css->strv();
-}
-
 struct mds_load_t {
   using clock = dirfrag_load_vec_t::clock;
   using time = dirfrag_load_vec_t::time;
@@ -951,6 +938,15 @@ struct mds_load_t {
   mds_load_t() : auth(DecayRate()), all(DecayRate()) {}
   mds_load_t(const DecayRate &rate) : auth(rate), all(rate) {}
 
+  void print(std::ostream& out) const {
+    out << "mdsload<" << auth << "/" << all
+        << ", req " << req_rate
+        << ", hr " << cache_hit_rate
+        << ", qlen " << queue_len
+	<< ", cpu " << cpu_load_avg
+        << ">";
+  }
+
   double req_rate = 0.0;
   double cache_hit_rate = 0.0;
   double queue_len = 0.0;
@@ -970,16 +966,6 @@ inline void decode(mds_load_t &c, ceph::buffer::list::const_iterator &p) {
   c.decode(p);
 }
 
-inline std::ostream& operator<<(std::ostream& out, const mds_load_t& load)
-{
-  return out << "mdsload<" << load.auth << "/" << load.all
-             << ", req " << load.req_rate 
-             << ", hr " << load.cache_hit_rate
-             << ", qlen " << load.queue_len
-	     << ", cpu " << load.cpu_load_avg
-             << ">";
-}
-
 // ================================================================
 typedef std::pair<mds_rank_t, mds_rank_t> mds_authority_t;
 
@@ -997,6 +983,16 @@ class MDSCacheObjectInfo {
   void encode(ceph::buffer::list& bl) const;
   void decode(ceph::buffer::list::const_iterator& bl);
   void dump(ceph::Formatter *f) const;
+  void print(std::ostream& out) const {
+    if (ino) {
+      out << ino << "." << snapid;
+    } else if (dname.length()) {
+      out << dirfrag << "/" << dname
+          << " snap " << snapid;
+    } else {
+      out << dirfrag;
+    }
+  }
   static void generate_test_instances(std::list<MDSCacheObjectInfo*>& ls);
 
   inodeno_t ino = 0;
@@ -1005,13 +1001,6 @@ class MDSCacheObjectInfo {
   snapid_t snapid;
 };
 
-inline std::ostream& operator<<(std::ostream& out, const MDSCacheObjectInfo &info) {
-  if (info.ino) return out << info.ino << "." << info.snapid;
-  if (info.dname.length()) return out << info.dirfrag << "/" << info.dname
-    << " snap " << info.snapid;
-  return out << info.dirfrag;
-}
-
 inline bool operator==(const MDSCacheObjectInfo& l, const MDSCacheObjectInfo& r) {
   if (l.ino || r.ino)
     return l.ino == r.ino && l.snapid == r.snapid;

From 8ddf631886ce4ac34f74e9e62eacc2c33c6e445f Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 16 Jan 2024 14:02:41 -0500
Subject: [PATCH 2347/2492] mds: add LockOp::print method for debugging

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Mutation.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 74444d548ff0..2eb566e07bdf 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -78,6 +78,16 @@ struct MutationImpl : public TrackedOp {
       return lock < r.lock;
     }
 
+    void print(std::ostream& out) const {
+      CachedStackStringStream css;
+      *css << "0x" << std::hex << flags;
+      out << "LockOp(l=" << *lock << ",f=" << css->strv();
+      if (wrlock_target != MDS_RANK_NONE) {
+        out << ",wt=" << wrlock_target;
+      }
+      out << ")";
+    }
+
     SimpleLock* lock;
     mutable unsigned flags;
     mutable mds_rank_t wrlock_target;

From 3057ec9572dd2c1805db09ffd5eaac8ab40a45b2 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 16 Jan 2024 14:03:29 -0500
Subject: [PATCH 2348/2492] mds: dump locks passed to Locker::acquire_locks

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 9ba2b7284af1..a0b818971bbe 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -226,6 +226,11 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 			   bool auth_pin_nonblocking)
 {
   dout(10) << "acquire_locks " << *mdr << dendl;
+  dout(20) << " lov = " << lov << dendl;
+  if (auth_pin_freeze) {
+    dout(20) << " auth_pin_freeze = " << *auth_pin_freeze << dendl;
+  }
+  dout(20) << " auth_pin_nonblocking=" << auth_pin_nonblocking << dendl;
 
   MarkEventOnDestruct marker(mdr, "failed to acquire_locks");
 

From 180979d7014a6c3799bb7282d90d6e6af860978a Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 4 Jan 2024 12:24:29 -0500
Subject: [PATCH 2349/2492] mds: simplify assert

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index a0b818971bbe..5aed88f6f4b7 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -149,8 +149,7 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
   bool found_locked = false;
   bool found_layout = false;
 
-  if (want_layout)
-    ceph_assert(n == 0);
+  ceph_assert(!want_layout || n == 0);
 
   client_t client = mdr->get_client();
 

From 401e6f49fc2e1c725c8e89e23cdc8afcf5f92876 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 9 Jan 2024 21:56:20 -0500
Subject: [PATCH 2350/2492] mds: call Locker method using this

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 5aed88f6f4b7..bccb71ab8073 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -274,7 +274,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 	if (wait) {
 	  dout(10) << " must xlock " << *lock << " " << *object
 		   << ", waiting for cluster recovered" << dendl;
-	  mds->locker->drop_locks(mdr.get(), NULL);
+	  drop_locks(mdr.get(), NULL);
 	  mdr->drop_local_auth_pins();
 	  mds->wait_for_cluster_recovered(new C_MDS_RetryRequest(mdcache, mdr));
 	  return false;

From d74d3aadd08ad37a74bf3a9ade18d0b6670c92a0 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 9 Jan 2024 14:01:17 -0500
Subject: [PATCH 2351/2492] mds: refactor if into switch

To make it easier to read and match other constructs in this code.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 54 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index bccb71ab8073..47e1680d0b6d 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1720,9 +1720,13 @@ bool Locker::rdlock_try_set(MutationImpl::LockOpVec& lov, MutationRef& mut)
 
 void Locker::wrlock_force(SimpleLock *lock, MutationRef& mut)
 {
-  if (lock->get_type() == CEPH_LOCK_IVERSION ||
-      lock->get_type() == CEPH_LOCK_DVERSION)
-    return local_wrlock_grab(static_cast<LocalLockC*>(lock), mut);
+  switch  (lock->get_type()) {
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      return local_wrlock_grab(static_cast<LocalLockC*>(lock), mut);
+    default:
+      break;
+  }
 
   dout(7) << "wrlock_force  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
@@ -1767,9 +1771,13 @@ bool Locker::wrlock_try(SimpleLock *lock, const MutationRef& mut, client_t clien
 bool Locker::wrlock_start(const MutationImpl::LockOp &op, const MDRequestRef& mut)
 {
   SimpleLock *lock = op.lock;
-  if (lock->get_type() == CEPH_LOCK_IVERSION ||
-      lock->get_type() == CEPH_LOCK_DVERSION)
-    return local_wrlock_start(static_cast<LocalLockC*>(lock), mut);
+  switch  (lock->get_type()) {
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      return local_wrlock_start(static_cast<LocalLockC*>(lock), mut);
+    default:
+      break;
+  }
 
   dout(10) << "wrlock_start " << *lock << " on " << *lock->get_parent() << dendl;
 
@@ -1828,9 +1836,15 @@ void Locker::wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
   ceph_assert(it->is_wrlock());
   SimpleLock* lock = it->lock;
 
-  if (lock->get_type() == CEPH_LOCK_IVERSION ||
-      lock->get_type() == CEPH_LOCK_DVERSION)
-    return local_wrlock_finish(it, mut);
+  switch  (lock->get_type()) {
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      return local_wrlock_finish(it, mut);
+    default:
+      break;
+  }
+
+
 
   dout(7) << "wrlock_finish on " << *lock << " on " << *lock->get_parent() << dendl;
   lock->put_wrlock();
@@ -1910,9 +1924,14 @@ void Locker::remote_wrlock_finish(const MutationImpl::lock_iterator& it, Mutatio
 
 bool Locker::xlock_start(SimpleLock *lock, const MDRequestRef& mut)
 {
-  if (lock->get_type() == CEPH_LOCK_IVERSION ||
-      lock->get_type() == CEPH_LOCK_DVERSION)
-    return local_xlock_start(static_cast<LocalLockC*>(lock), mut);
+  switch  (lock->get_type()) {
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      return local_xlock_start(static_cast<LocalLockC*>(lock), mut);
+    default:
+      break;
+  }
+
 
   dout(7) << "xlock_start on " << *lock << " on " << *lock->get_parent() << dendl;
   client_t client = mut->get_client();
@@ -2031,9 +2050,14 @@ void Locker::xlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *m
   ceph_assert(it->is_xlock());
   SimpleLock *lock = it->lock;
 
-  if (lock->get_type() == CEPH_LOCK_IVERSION ||
-      lock->get_type() == CEPH_LOCK_DVERSION)
-    return local_xlock_finish(it, mut);
+  switch  (lock->get_type()) {
+    case CEPH_LOCK_DVERSION:
+    case CEPH_LOCK_IVERSION:
+      return local_xlock_finish(it, mut);
+    default:
+      break;
+  }
+
 
   dout(10) << "xlock_finish on " << *lock << " " << *lock->get_parent() << dendl;
 

From 6fa38e4e4ff230ca2a1c6d2de63d561d7cfee826 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 9 Jan 2024 14:06:39 -0500
Subject: [PATCH 2352/2492] mds: annotate mdr with try_rdlock_snap_layout
 failure

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 47e1680d0b6d..e40e1c9b8f82 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -161,6 +161,7 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 
     if (!found_locked) {
       if (!t->snaplock.can_rdlock(client)) {
+        mdr->mark_event("failed to acquire snap lock");
 	t->snaplock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
 	goto failed;
       }
@@ -171,6 +172,7 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
     if (want_layout && !found_layout) {
       if (!mdr->is_rdlocked(&t->policylock)) {
 	if (!t->policylock.can_rdlock(client)) {
+          mdr->mark_event("failed to acquire policy lock");
 	  t->policylock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
 	  goto failed;
 	}

From 9858e01abd57458b04f87fcaeb432ab7edb6ac07 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 8 Jan 2024 23:53:24 -0500
Subject: [PATCH 2353/2492] mds: make LockType fields const

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CDentry.cc    |  4 ++--
 src/mds/CDentry.h     |  4 ++--
 src/mds/CInode.cc     | 21 +++++++++++----------
 src/mds/CInode.h      | 20 ++++++++++----------
 src/mds/LocalLockC.h  |  2 +-
 src/mds/ScatterLock.h |  2 +-
 src/mds/SimpleLock.h  |  4 ++--
 7 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/src/mds/CDentry.cc b/src/mds/CDentry.cc
index 6722f0f2a367..8694dfc72a2e 100644
--- a/src/mds/CDentry.cc
+++ b/src/mds/CDentry.cc
@@ -38,8 +38,8 @@ ostream& CDentry::print_db_line_prefix(ostream& out) const
   return out << ceph_clock_now() << " mds." << dir->mdcache->mds->get_nodeid() << ".cache.den(" << dir->ino() << " " << name << ") ";
 }
 
-LockType CDentry::lock_type(CEPH_LOCK_DN);
-LockType CDentry::versionlock_type(CEPH_LOCK_DVERSION);
+const LockType CDentry::lock_type(CEPH_LOCK_DN);
+const LockType CDentry::versionlock_type(CEPH_LOCK_DVERSION);
 
 
 // CDentry
diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h
index 1c2b6f892cec..2566395d1856 100644
--- a/src/mds/CDentry.h
+++ b/src/mds/CDentry.h
@@ -367,8 +367,8 @@ class CDentry : public MDSCacheObject, public LRUObject, public Counter<CDentry>
   elist<CDentry*>::item item_stray;
 
   // lock
-  static LockType lock_type;
-  static LockType versionlock_type;
+  static const LockType lock_type;
+  static const LockType versionlock_type;
 
   SimpleLock lock; // FIXME referenced containers not in mempool
   LocalLockC versionlock; // FIXME referenced containers not in mempool
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 6ef7d240a0ed..7e9185ff5cc6 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -91,16 +91,17 @@ class CInodeIOContext : public MDSIOContextBase
 
 sr_t* const CInode::projected_inode::UNDEF_SRNODE = (sr_t*)(unsigned long)-1;
 
-LockType CInode::versionlock_type(CEPH_LOCK_IVERSION);
-LockType CInode::authlock_type(CEPH_LOCK_IAUTH);
-LockType CInode::linklock_type(CEPH_LOCK_ILINK);
-LockType CInode::dirfragtreelock_type(CEPH_LOCK_IDFT);
-LockType CInode::filelock_type(CEPH_LOCK_IFILE);
-LockType CInode::xattrlock_type(CEPH_LOCK_IXATTR);
-LockType CInode::snaplock_type(CEPH_LOCK_ISNAP);
-LockType CInode::nestlock_type(CEPH_LOCK_INEST);
-LockType CInode::flocklock_type(CEPH_LOCK_IFLOCK);
-LockType CInode::policylock_type(CEPH_LOCK_IPOLICY);
+const LockType CInode::quiescelock_type(CEPH_LOCK_IQUIESCE);
+const LockType CInode::versionlock_type(CEPH_LOCK_IVERSION);
+const LockType CInode::authlock_type(CEPH_LOCK_IAUTH);
+const LockType CInode::linklock_type(CEPH_LOCK_ILINK);
+const LockType CInode::dirfragtreelock_type(CEPH_LOCK_IDFT);
+const LockType CInode::filelock_type(CEPH_LOCK_IFILE);
+const LockType CInode::xattrlock_type(CEPH_LOCK_IXATTR);
+const LockType CInode::snaplock_type(CEPH_LOCK_ISNAP);
+const LockType CInode::nestlock_type(CEPH_LOCK_INEST);
+const LockType CInode::flocklock_type(CEPH_LOCK_IFLOCK);
+const LockType CInode::policylock_type(CEPH_LOCK_IPOLICY);
 
 std::string_view CInode::pin_name(int p) const
 {
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 979b451742cb..242288d5ec33 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -1089,16 +1089,16 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   elist<CInode*>::item item_pop_lru;
 
   // -- locks --
-  static LockType versionlock_type;
-  static LockType authlock_type;
-  static LockType linklock_type;
-  static LockType dirfragtreelock_type;
-  static LockType filelock_type;
-  static LockType xattrlock_type;
-  static LockType snaplock_type;
-  static LockType nestlock_type;
-  static LockType flocklock_type;
-  static LockType policylock_type;
+  static const LockType versionlock_type;
+  static const LockType authlock_type;
+  static const LockType linklock_type;
+  static const LockType dirfragtreelock_type;
+  static const LockType filelock_type;
+  static const LockType xattrlock_type;
+  static const LockType snaplock_type;
+  static const LockType nestlock_type;
+  static const LockType flocklock_type;
+  static const LockType policylock_type;
 
   // FIXME not part of mempool
   LocalLockC  versionlock;
diff --git a/src/mds/LocalLockC.h b/src/mds/LocalLockC.h
index 96cea93ebd05..4d26a1c3e142 100644
--- a/src/mds/LocalLockC.h
+++ b/src/mds/LocalLockC.h
@@ -20,7 +20,7 @@
 
 class LocalLockC : public SimpleLock {
 public:
-  LocalLockC(MDSCacheObject *o, LockType *t) : 
+  LocalLockC(MDSCacheObject *o, const LockType *t) :
     SimpleLock(o, t) {
     set_state(LOCK_LOCK); // always.
   }
diff --git a/src/mds/ScatterLock.h b/src/mds/ScatterLock.h
index f654fd2ff319..d82487480ba1 100644
--- a/src/mds/ScatterLock.h
+++ b/src/mds/ScatterLock.h
@@ -22,7 +22,7 @@
 
 class ScatterLock : public SimpleLock {
 public:
-  ScatterLock(MDSCacheObject *o, LockType *lt) :
+  ScatterLock(MDSCacheObject *o, const LockType *lt) :
     SimpleLock(o, lt) {}
   ~ScatterLock() override {
     ceph_assert(!_more);
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 33c091b6244e..3ef474c19a4b 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -173,7 +173,7 @@ class SimpleLock {
     }
   }
 
-  SimpleLock(MDSCacheObject *o, LockType *lt) :
+  SimpleLock(MDSCacheObject *o, const LockType *lt) :
     type(lt),
     parent(o)
   {}
@@ -599,7 +599,7 @@ class SimpleLock {
     out << ")";
   }
 
-  LockType *type;
+  const LockType *type;
 
 protected:
   // parent (what i lock)

From ba7d1f5f2a2e4355155e00173b0d2ade562a8945 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 Nov 2023 09:38:03 -0500
Subject: [PATCH 2354/2492] mds: implement Mutation::dump method

So we can dump it from asok commands.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Mutation.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 2eb566e07bdf..297524a28c02 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -235,7 +235,9 @@ struct MutationImpl : public TrackedOp {
     out << "mutation(" << this << ")";
   }
 
-  virtual void dump(ceph::Formatter *f) const {}
+  virtual void dump(ceph::Formatter *f) const {
+    _dump(f);
+  }
   void _dump_op_descriptor(std::ostream& stream) const override;
 
   metareqid_t reqid;

From 1513327c1a55614ac7e2f1a6da0de06e5fd2214d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 13 Oct 2023 20:27:02 -0400
Subject: [PATCH 2355/2492] mds: remove redundant parenthesis

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 5022b23dd0f3..84e743af82fc 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8619,7 +8619,7 @@ int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
 	discover = true;
       }
 
-      if ((discover)) {
+      if (discover) {
 	dout(7) << "traverse: discover from " << path[depth] << " from " << *curdir << dendl;
 	discover_path(curdir, snapid, path.postfixpath(depth), cf.build(),
 		      path_locked);

From f8a26a61305ba8b9fbb9ce47b5221e3adb754c9b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 Nov 2023 13:19:47 -0500
Subject: [PATCH 2356/2492] mds: add more information to debug message

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Server.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 598452c32fd4..186882caed1b 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2076,13 +2076,13 @@ void Server::respond_to_request(const MDRequestRef& mdr, int r)
   mdr->result = r;
   if (mdr->client_request) {
     if (mdr->is_batch_head()) {
-      dout(20) << __func__ << " batch head " << *mdr << dendl;
+      dout(20) << __func__ << ": batch head " << *mdr << dendl;
       mdr->release_batch_op()->respond(r);
     } else {
      reply_client_request(mdr, make_message<MClientReply>(*mdr->client_request, r));
     }
   } else if (mdr->internal_op > -1) {
-    dout(10) << "respond_to_request on internal request " << mdr << dendl;
+    dout(10) << __func__ << ": completing with result " << cpp_strerror(r) << " on internal " << *mdr << dendl;
     if (!mdr->internal_op_finish)
       ceph_abort_msg("trying to respond to internal op without finisher");
     mdr->internal_op_finish->complete(r);

From eaea93e05018294e893e0e5241b3293e10971f5e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 31 Jan 2024 14:28:14 -0500
Subject: [PATCH 2357/2492] mds: print internal_request filepaths if present

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Mutation.cc | 20 ++++++++++++++++++--
 src/mds/Mutation.h  |  5 +++--
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 9b71e244e08a..f866f663359c 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -287,6 +287,12 @@ MDRequestImpl::More* MDRequestImpl::more()
   return _more;
 }
 
+MDRequestImpl::More const* MDRequestImpl::more() const
+{
+  ceph_assert(_more);
+  return _more;
+}
+
 bool MDRequestImpl::has_more() const
 {
   return _more != nullptr;
@@ -373,14 +379,14 @@ void MDRequestImpl::drop_local_auth_pins()
   MutationImpl::drop_local_auth_pins();
 }
 
-const filepath& MDRequestImpl::get_filepath()
+const filepath& MDRequestImpl::get_filepath() const
 {
   if (client_request)
     return client_request->get_filepath();
   return more()->filepath1;
 }
 
-const filepath& MDRequestImpl::get_filepath2()
+const filepath& MDRequestImpl::get_filepath2() const
 {
   if (client_request)
     return client_request->get_filepath2();
@@ -549,6 +555,16 @@ void MDRequestImpl::_dump_op_descriptor(ostream& os) const
     os << "peer_request:" << reqid;
   } else if (internal_op >= 0) {
     os << "internal op " << ceph_mds_op_name(internal_op) << ":" << reqid;
+    if (has_more()) {
+      auto& fp = get_filepath();
+      if (!fp.empty()) {
+        os << " fp=" << fp;
+      }
+      auto& fp2 = get_filepath2();
+      if (!fp2.empty()) {
+        os << " fp2=" << fp2;
+      }
+    }
   } else {
     // drat, it's triggered by a peer request, but we don't have a message
     // FIXME
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 297524a28c02..a8a89e1ef3ff 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -389,6 +389,7 @@ struct MDRequestImpl : public MutationImpl {
   ~MDRequestImpl() override;
   
   More* more();
+  More const* more() const;
   bool has_more() const;
   bool has_witnesses();
   bool peer_did_prepare();
@@ -400,8 +401,8 @@ struct MDRequestImpl : public MutationImpl {
   void drop_local_auth_pins();
   void set_ambiguous_auth(CInode *inode);
   void clear_ambiguous_auth();
-  const filepath& get_filepath();
-  const filepath& get_filepath2();
+  const filepath& get_filepath() const;
+  const filepath& get_filepath2() const;
   void set_filepath(const filepath& fp);
   void set_filepath2(const filepath& fp);
   bool is_queued_for_replay() const;

From a1303a52af7504c3748a75ade9619c7a6c454831 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 17 Feb 2024 10:23:43 -0500
Subject: [PATCH 2358/2492] mds: add path argument to `ops` and `dump tree` to
 stream result to local file

This file can be collected during testing to avoid sending a large JSON result
over the `ceph tell`/asok interface.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSDaemon.cc | 10 ++++----
 src/mds/MDSRank.cc   | 54 +++++++++++++++++++++++++++++++++-----------
 2 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index e69bacf8f49d..e59419d1533e 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -260,9 +260,10 @@ void MDSDaemon::set_up_admin_socket()
 				     "show the ops currently in flight");
   ceph_assert(r == 0);
   r = admin_socket->register_command("ops "
-				     "name=flags,type=CephChoices,strings=locks,n=N,req=false ",
-                                     asok_hook,
-				     "show the ops currently in flight");
+				     "name=flags,type=CephChoices,strings=locks,n=N,req=false "
+				     "name=path,type=CephString,req=false "
+                                     ,asok_hook
+				     ,"show the ops currently in flight");
   ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",
       asok_hook,
@@ -342,7 +343,8 @@ void MDSDaemon::set_up_admin_socket()
   ceph_assert(r == 0);
   r = admin_socket->register_command("dump tree "
 				     "name=root,type=CephString,req=true "
-				     "name=depth,type=CephInt,req=false ",
+				     "name=depth,type=CephInt,req=false "
+				     "name=path,type=CephString,req=false ",
 				     asok_hook,
 				     "dump metadata cache for subtree");
   ceph_assert(r == 0);
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 791765ddd661..e8486b1344b1 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2664,6 +2664,8 @@ void MDSRankDispatcher::handle_asok_command(
   } else if (command == "ops") {
     vector<string> flags;
     cmd_getval(cmdmap, "flags", flags);
+    string path;
+    cmd_getval(cmdmap, "path", path);
     std::unique_lock l(mds_lock, std::defer_lock);
     auto lambda = OpTracker::default_dumper;
     if (flags.size()) {
@@ -2678,8 +2680,18 @@ void MDSRankDispatcher::handle_asok_command(
       };
       l.lock();
     }
-    if (!op_tracker.dump_ops_in_flight(f, false, {""}, false, lambda)) {
-      *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+    if (!path.empty()) {
+      auto ff = JSONFormatterFile(path, false);
+      if (!op_tracker.dump_ops_in_flight(&ff, false, {""}, false, lambda)) {
+        *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+      }
+      f->open_object_section("result");
+      f->dump_string("path", path);
+      f->close_section();
+    } else {
+      if (!op_tracker.dump_ops_in_flight(f, false, {""}, false, lambda)) {
+        *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+      }
     }
   } else if (command == "dump_blocked_ops") {
     if (!op_tracker.dump_ops_in_flight(f, true)) {
@@ -3206,23 +3218,39 @@ int MDSRank::_command_export_dir(
 
 void MDSRank::command_dump_tree(const cmdmap_t &cmdmap, std::ostream &ss, Formatter *f) 
 {
+  std::string path;
+  cmd_getval(cmdmap, "path", path);
   std::string root;
-  int64_t depth;
   cmd_getval(cmdmap, "root", root);
+  int64_t depth;
+  if (!cmd_getval(cmdmap, "depth", depth)) {
+    depth = -1;
+  }
   if (root.empty()) {
     root = "/";
   }
-  if (!cmd_getval(cmdmap, "depth", depth))
-    depth = -1;
-  std::lock_guard l(mds_lock);
-  CInode *in = mdcache->cache_traverse(filepath(root.c_str()));
-  if (!in) {
-    ss << "inode for path '" << filepath(root.c_str()) << "' is not in cache";
-    return;
+
+  auto dump = [&](Formatter *f) {
+    std::lock_guard l(mds_lock);
+    CInode *in = mdcache->cache_traverse(filepath(root.c_str()));
+    if (!in) {
+      ss << "inode for path '" << filepath(root.c_str()) << "' is not in cache";
+      return;
+    }
+    f->open_array_section("inodes");
+    mdcache->dump_tree(in, 0, depth, f);
+    f->close_section();
+  };
+
+  if (!path.empty()) {
+    auto ff = JSONFormatterFile(path, false);
+    dump(&ff);
+    f->open_object_section("result");
+    f->dump_string("path", path);
+    f->close_section();
+  } else {
+    dump(f);
   }
-  f->open_array_section("inodes");
-  mdcache->dump_tree(in, 0, depth, f);
-  f->close_section();
 }
 
 CDir *MDSRank::_command_dirfrag_get(

From 9956f9ad78a3b6ebe520dcd10e33a59c361c4d91 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 13 Nov 2023 14:23:54 -0500
Subject: [PATCH 2359/2492] mds: add command to kill request

Fixes: https://tracker.ceph.com/issues/63374
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc   |  5 +++++
 src/mds/MDSDaemon.cc |  5 +++++
 src/mds/MDSRank.cc   | 30 ++++++++++++++++++++++++++++++
 src/mds/mdstypes.h   | 17 +++++++++++++++++
 4 files changed, 57 insertions(+)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 84e743af82fc..38df8e3b03e3 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -9904,6 +9904,11 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
 
 void MDCache::request_kill(const MDRequestRef& mdr)
 {
+  if (mdr->killed) {
+    /* ignore duplicate kills */
+    return;
+  }
+
   // rollback peer requests is tricky. just let the request proceed.
   if (mdr->has_more() &&
       (!mdr->more()->witnessed.empty() || !mdr->more()->waiting_on_peer.empty())) {
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index e59419d1533e..850e9e700c41 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -265,6 +265,11 @@ void MDSDaemon::set_up_admin_socket()
                                      ,asok_hook
 				     ,"show the ops currently in flight");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("op kill "
+                                     "name=id,type=CephString,req=true",
+				     asok_hook,
+				     "kill op");
+  ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",
       asok_hook,
       "show the blocked ops currently in flight");
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index e8486b1344b1..8ab77f23a66f 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2693,6 +2693,36 @@ void MDSRankDispatcher::handle_asok_command(
         *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
       }
     }
+  } else if (command == "op kill") {
+    std::string id;
+    if(!cmd_getval(cmdmap, "id", id)) {
+      *css << "malformed id";
+      r = -CEPHFS_EINVAL;
+      goto out;
+    }
+    metareqid_t mrid;
+    try {
+      mrid = metareqid_t(id);
+    } catch (const std::exception& e) {
+      *css << "malformed id: " << e.what();
+      r = -CEPHFS_EINVAL;
+      goto out;
+    }
+    std::lock_guard l(mds_lock);
+    if (!mdcache->have_request(mrid)) {
+      *css << "request does not exist";
+      r = -CEPHFS_ENOENT;
+      goto out;
+    }
+    auto mdr = mdcache->request_get(mrid);
+    {
+      f->open_object_section("result");
+      f->dump_int("result", 0);
+      f->dump_object("request", *mdr);
+      f->close_section();
+    }
+    mdcache->request_kill(mdr);
+    r = 0;
   } else if (command == "dump_blocked_ops") {
     if (!op_tracker.dump_ops_in_flight(f, true)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index a39ec5abbd5d..272ae76cac18 100644
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -8,6 +8,7 @@
 #include <ostream>
 #include <set>
 #include <map>
+#include <string>
 #include <string_view>
 
 #include "common/config.h"
@@ -600,6 +601,22 @@ WRITE_CLASS_ENCODER(mds_table_pending_t)
 struct metareqid_t {
   metareqid_t() {}
   metareqid_t(entity_name_t n, ceph_tid_t t) : name(n), tid(t) {}
+  metareqid_t(std::string_view sv) {
+    auto p = sv.find(':');
+    if (p == std::string::npos) {
+      throw std::invalid_argument("invalid format: expected colon");
+    }
+    if (!name.parse(sv.substr(0, p))) {
+      throw std::invalid_argument("invalid format: invalid entity name");
+    }
+    try {
+      tid = std::stoul(std::string(sv.substr(p+1)), nullptr, 0);
+    } catch (const std::invalid_argument& e) {
+      throw std::invalid_argument("invalid format: tid is not a number");
+    } catch (const std::out_of_range& e) {
+      throw std::invalid_argument("invalid format: tid is out of range");
+    }
+  }
   void encode(ceph::buffer::list& bl) const {
     using ceph::encode;
     encode(name, bl);

From b72e153d2b36956e97b8686aeb6aaacffe928750 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 13 Feb 2024 15:58:19 -0500
Subject: [PATCH 2360/2492] mds: avoid killing dead requests

This is important when you have:

* sub-op A finishes and resumes parent
* parent kills all sub-ops (including sub-op A)

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 4 +++-
 src/mds/Mutation.h | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 38df8e3b03e3..179c461f8072 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -9864,6 +9864,8 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
 {
   dout(15) << "request_cleanup " << *mdr << dendl;
 
+  mdr->dead = true;
+
   if (mdr->has_more()) {
     if (mdr->more()->is_ambiguous_auth)
       mdr->clear_ambiguous_auth();
@@ -9904,7 +9906,7 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
 
 void MDCache::request_kill(const MDRequestRef& mdr)
 {
-  if (mdr->killed) {
+  if (mdr->killed || mdr->dead) {
     /* ignore duplicate kills */
     return;
   }
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index a8a89e1ef3ff..6e51275fbc15 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -277,6 +277,7 @@ struct MutationImpl : public TrackedOp {
   bool committing = false;
   bool aborted = false;
   bool killed = false;
+  bool dead = false;
 
   // for applying projected inode changes
   std::set<MDSCacheObject*> projected_nodes;

From ba3dac94e6a90ed76970fef89b1a82dcedb4ec2b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 Nov 2023 13:50:53 -0500
Subject: [PATCH 2361/2492] mds: complete internal op if killed

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 179c461f8072..d2c118675745 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -9938,6 +9938,10 @@ void MDCache::request_kill(const MDRequestRef& mdr)
     mdr->item_session_request.remove_myself();
   } else {
     dout(10) << "request_kill " << *mdr << dendl;
+    if (mdr->internal_op_finish) {
+      mdr->internal_op_finish->complete(-CEPHFS_ECANCELED);
+      mdr->internal_op_finish = nullptr;
+    }
     request_cleanup(mdr);
   }
 }

From c944eadaf6987a812e738459214872af1ff7678a Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 13 Feb 2024 16:07:37 -0500
Subject: [PATCH 2362/2492] mds: finish request before completing internal req

Importantly, this marks the request dead so it cannot be killed.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Server.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 186882caed1b..b27cd2bdbc24 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2083,10 +2083,11 @@ void Server::respond_to_request(const MDRequestRef& mdr, int r)
     }
   } else if (mdr->internal_op > -1) {
     dout(10) << __func__ << ": completing with result " << cpp_strerror(r) << " on internal " << *mdr << dendl;
-    if (!mdr->internal_op_finish)
+    auto c = mdr->internal_op_finish;
+    if (!c)
       ceph_abort_msg("trying to respond to internal op without finisher");
-    mdr->internal_op_finish->complete(r);
     mdcache->request_finish(mdr);
+    c->complete(r);
   }
 }
 

From 3197a975a521654500173175c6e856c54df497ef Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 9 Jan 2024 13:55:48 -0500
Subject: [PATCH 2363/2492] mds: add command to get specific op

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSDaemon.cc |  6 ++++++
 src/mds/MDSRank.cc   | 46 ++++++++++++++++++++++++++++++++++++++++++++
 src/mds/Mutation.h   |  1 +
 3 files changed, 53 insertions(+)

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 850e9e700c41..3f29e756b4d2 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -270,6 +270,12 @@ void MDSDaemon::set_up_admin_socket()
 				     asok_hook,
 				     "kill op");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("op get "
+				     "name=flags,type=CephChoices,strings=locks,n=N,req=false "
+                                     "name=id,type=CephString,req=true",
+				     asok_hook,
+				     "get op");
+  ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",
       asok_hook,
       "show the blocked ops currently in flight");
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 8ab77f23a66f..b51148eedcb7 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2693,6 +2693,52 @@ void MDSRankDispatcher::handle_asok_command(
         *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
       }
     }
+  } else if (command == "op get") {
+    vector<string> flags;
+    cmd_getval(cmdmap, "flags", flags);
+
+    std::string id;
+    if(!cmd_getval(cmdmap, "id", id)) {
+      *css << "malformed id";
+      r = -CEPHFS_EINVAL;
+      goto out;
+    }
+    metareqid_t mrid;
+    try {
+      mrid = metareqid_t(id);
+    } catch (const std::exception& e) {
+      *css << "malformed id: " << e.what();
+      r = -CEPHFS_EINVAL;
+      goto out;
+    }
+
+    auto dumper = OpTracker::default_dumper;
+    if (flags.size()) {
+      if (flags.size()) {
+        /* use std::function if we actually want to capture flags someday */
+        dumper = [](const TrackedOp& op, Formatter* f) {
+          auto* req = dynamic_cast<const MDRequestImpl*>(&op);
+          if (req) {
+            req->dump_with_mds_lock(f);
+          } else {
+            op.dump_type(f);
+          }
+        };
+      }
+    }
+
+    std::lock_guard l(mds_lock);
+    if (!mdcache->have_request(mrid)) {
+      *css << "request does not exist";
+      r = -CEPHFS_ENOENT;
+      goto out;
+    }
+    auto mdr = mdcache->request_get(mrid);
+
+    f->open_object_section("op");
+    mdr->dump(ceph_clock_now(), f, dumper);
+    f->close_section();
+    r = 0;
   } else if (command == "op kill") {
     std::string id;
     if(!cmd_getval(cmdmap, "id", id)) {
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 6e51275fbc15..ded6271f99ee 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -238,6 +238,7 @@ struct MutationImpl : public TrackedOp {
   virtual void dump(ceph::Formatter *f) const {
     _dump(f);
   }
+  using TrackedOp::dump;
   void _dump_op_descriptor(std::ostream& stream) const override;
 
   metareqid_t reqid;

From a9964a7ccc4394f923fb0f1c76eb8fa03fe8733d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 13 Feb 2024 11:07:26 -0500
Subject: [PATCH 2364/2492] mds: provide mechanism to authpin while freezing

When a subtree is freezing, it's no longer possible to acquire new authpins.
This is a problem when a compound request like quiescing a subtree is trying to
acquire authpins for each sub-op. This creates a situation where some quiesce
sub-ops complete with authpins (thereby preventing the tree from becoming
"frozen") and new sub-ops cannot acquire authpins (because the tree is
"freezing"). To circumvent this, allow some authpin requests to proceed if
FLAG_BYPASSFREEZING is set.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CDentry.cc             |  4 ++--
 src/mds/CDentry.h              |  2 +-
 src/mds/CDir.cc                | 15 ++++++++++++---
 src/mds/CDir.h                 |  2 +-
 src/mds/CInode.cc              | 11 +++++++++--
 src/mds/CInode.h               |  2 +-
 src/mds/MDSCacheObject.h       |  2 +-
 src/mds/Mutation.cc            |  4 ++--
 src/mds/Mutation.h             |  2 +-
 src/mds/Server.cc              |  6 +++++-
 src/messages/MMDSPeerRequest.h |  3 +++
 11 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/src/mds/CDentry.cc b/src/mds/CDentry.cc
index 8694dfc72a2e..9f5f1a49f55e 100644
--- a/src/mds/CDentry.cc
+++ b/src/mds/CDentry.cc
@@ -368,10 +368,10 @@ int CDentry::get_num_dir_auth_pins() const
   return auth_pins;
 }
 
-bool CDentry::can_auth_pin(int *err_ret) const
+bool CDentry::can_auth_pin(int *err_ret, bool bypassfreezing) const
 {
   ceph_assert(dir);
-  return dir->can_auth_pin(err_ret);
+  return dir->can_auth_pin(err_ret, bypassfreezing);
 }
 
 void CDentry::auth_pin(void *by)
diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h
index 2566395d1856..837ce2b8818d 100644
--- a/src/mds/CDentry.h
+++ b/src/mds/CDentry.h
@@ -222,7 +222,7 @@ class CDentry : public MDSCacheObject, public LRUObject, public Counter<CDentry>
   void _put() override;
 
   // auth pins
-  bool can_auth_pin(int *err_ret=nullptr) const override;
+  bool can_auth_pin(int *err_ret=nullptr, bool bypassfreezing=false) const override;
   void auth_pin(void *by) override;
   void auth_unpin(void *by) override;
   void adjust_nested_auth_pins(int diradj, void *by);
diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index a8aaf11c0512..c65dee271fc8 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -3445,16 +3445,25 @@ void CDir::adjust_freeze_after_rename(CDir *dir)
   mdcache->mds->queue_waiters(unfreeze_waiters);
 }
 
-bool CDir::can_auth_pin(int *err_ret) const
+bool CDir::can_auth_pin(int *err_ret, bool bypassfreezing) const
 {
   int err;
   if (!is_auth()) {
     err = ERR_NOT_AUTH;
-  } else if (is_freezing_dir() || is_frozen_dir()) {
+  } else if (is_freezing_dir()) {
+    if (bypassfreezing) {
+      dout(20) << "allowing authpin with freezing" << dendl;
+      err = 0;
+    } else {
+      err = ERR_FRAGMENTING_DIR;
+    }
+  } else if (is_frozen_dir()) {
     err = ERR_FRAGMENTING_DIR;
   } else {
     auto p = is_freezing_or_frozen_tree();
-    if (p.first || p.second) {
+    if (p.first && !bypassfreezing) {
+      err = ERR_EXPORTING_TREE;
+    } else if (p.second) {
       err = ERR_EXPORTING_TREE;
     } else {
       err = 0;
diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 7cc4dc7ffcf8..234da01b10c9 100644
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -526,7 +526,7 @@ class CDir : public MDSCacheObject, public Counter<CDir> {
   void abort_import();
 
   // -- auth pins --
-  bool can_auth_pin(int *err_ret=nullptr) const override;
+  bool can_auth_pin(int *err_ret=nullptr, bool bypassfreezing=false) const override;
   int get_auth_pins() const { return auth_pins; }
   int get_dir_auth_pins() const { return dir_auth_pins; }
   void auth_pin(void *who) override;
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 7e9185ff5cc6..6d15908e5455 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2929,11 +2929,18 @@ void CInode::clear_ambiguous_auth()
 }
 
 // auth_pins
-bool CInode::can_auth_pin(int *err_ret) const {
+bool CInode::can_auth_pin(int *err_ret, bool bypassfreezing) const {
   int err;
   if (!is_auth()) {
     err = ERR_NOT_AUTH;
-  } else if (is_freezing_inode() || is_frozen_inode() || is_frozen_auth_pin()) {
+  } else if (is_freezing_inode()) {
+    if (bypassfreezing) {
+      dout(20) << "allowing authpin with freezing" << dendl;
+      err = 0;
+    } else {
+      err = ERR_EXPORTING_INODE;
+    }
+  } else if (is_frozen_inode() || is_frozen_auth_pin()) {
     err = ERR_EXPORTING_INODE;
   } else {
     if (parent)
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 242288d5ec33..7d91becb97b9 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -926,7 +926,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   mds_authority_t authority() const override;
 
   // -- auth pins --
-  bool can_auth_pin(int *err_ret=nullptr) const override;
+  bool can_auth_pin(int *err_ret=nullptr, bool bypassfreezing=false) const override;
   void auth_pin(void *by) override;
   void auth_unpin(void *by) override;
 
diff --git a/src/mds/MDSCacheObject.h b/src/mds/MDSCacheObject.h
index 7da2a67c3a27..8a319b4404d9 100644
--- a/src/mds/MDSCacheObject.h
+++ b/src/mds/MDSCacheObject.h
@@ -210,7 +210,7 @@ class MDSCacheObject {
     ERR_FRAGMENTING_DIR,
     ERR_EXPORTING_INODE,
   };
-  virtual bool can_auth_pin(int *err_code=nullptr) const = 0;
+  virtual bool can_auth_pin(int *err_code=nullptr, bool bypassfreezing=false) const = 0;
   virtual void auth_pin(void *who) = 0;
   virtual void auth_unpin(void *who) = 0;
   virtual bool is_frozen() const = 0;
diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index f866f663359c..ecc455de08db 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -364,9 +364,9 @@ void MDRequestImpl::clear_ambiguous_auth()
   more()->is_ambiguous_auth = false;
 }
 
-bool MDRequestImpl::can_auth_pin(MDSCacheObject *object)
+bool MDRequestImpl::can_auth_pin(MDSCacheObject *object, bool bypassfreezing)
 {
-  return object->can_auth_pin() ||
+  return object->can_auth_pin(nullptr, bypassfreezing) ||
          (is_auth_pinned(object) && has_more() &&
 	  more()->is_freeze_authpin &&
 	  more()->rename_inode == object);
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index ded6271f99ee..fcb0b5a21ad8 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -399,7 +399,7 @@ struct MDRequestImpl : public MutationImpl {
   bool freeze_auth_pin(CInode *inode);
   void unfreeze_auth_pin(bool clear_inode=false);
   void set_remote_frozen_auth_pin(CInode *inode);
-  bool can_auth_pin(MDSCacheObject *object);
+  bool can_auth_pin(MDSCacheObject *object, bool bypassfreezing=false);
   void drop_local_auth_pins();
   void set_ambiguous_auth(CInode *inode);
   void clear_ambiguous_auth();
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index b27cd2bdbc24..5749bfa7f263 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -3149,9 +3149,13 @@ void Server::handle_peer_auth_pin(const MDRequestRef& mdr)
   list<MDSCacheObject*> objects;
   CInode *auth_pin_freeze = NULL;
   bool nonblocking = mdr->peer_request->is_nonblocking();
+  bool bypassfreezing = mdr->peer_request->is_bypassfreezing();
   bool fail = false, wouldblock = false, readonly = false;
   ref_t<MMDSPeerRequest> reply;
 
+  dout(15) << " nonblocking=" << nonblocking
+           << " bypassfreezing=" << bypassfreezing << dendl;
+
   if (mdcache->is_readonly()) {
     dout(10) << " read-only FS" << dendl;
     readonly = true;
@@ -3183,7 +3187,7 @@ void Server::handle_peer_auth_pin(const MDRequestRef& mdr)
       }
       if (mdr->is_auth_pinned(obj))
 	continue;
-      if (!mdr->can_auth_pin(obj)) {
+      if (!mdr->can_auth_pin(obj, bypassfreezing)) {
 	if (nonblocking) {
 	  dout(10) << " can't auth_pin (freezing?) " << *obj << " nonblocking" << dendl;
 	  fail = true;
diff --git a/src/messages/MMDSPeerRequest.h b/src/messages/MMDSPeerRequest.h
index 1799ab361a11..1a007364d4ab 100644
--- a/src/messages/MMDSPeerRequest.h
+++ b/src/messages/MMDSPeerRequest.h
@@ -106,6 +106,7 @@ class MMDSPeerRequest final : public MMDSOp {
   static constexpr unsigned FLAG_INTERRUPTED	= 1<<5;
   static constexpr unsigned FLAG_NOTIFYBLOCKING	= 1<<6;
   static constexpr unsigned FLAG_REQBLOCKED	= 1<<7;
+  static constexpr unsigned FLAG_BYPASSFREEZING = 1<<8;
 
   // for locking
   __u16 lock_type;  // lock object type
@@ -160,6 +161,8 @@ class MMDSPeerRequest final : public MMDSOp {
   void clear_notify_blocking() const { flags &= ~FLAG_NOTIFYBLOCKING; }
   bool is_req_blocked() const { return (flags & FLAG_REQBLOCKED); }
   void mark_req_blocked() { flags |= FLAG_REQBLOCKED; }
+  bool is_bypassfreezing() const { return (flags & FLAG_BYPASSFREEZING); }
+  void mark_bypassfreezing() { flags |= FLAG_BYPASSFREEZING; }
 
   void set_lock_type(int t) { lock_type = t; }
   const ceph::buffer::list& get_lock_data() const { return inode_export; }

From 84b33ea9fe6268adfaca1e17b282e68788e38bf4 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 20 Feb 2024 17:08:32 -0500
Subject: [PATCH 2365/2492] mds: use 128 bits for waiters on MDSCacheObject

Adding a new inode lock will overflow inode wait bits into the MDSCacheObject
wait bits. Make space for the quiescelock.

This includes a minor refactor to no longer attempt scoping the set of masks we
test in MDSCacheObject::waiting when calling MDSCacheObject::is_waiter_for.
This optimization wasn't worth the overhead and would be awkard to keep as
std::bitset cannot be used as a key for a std::multimap (easily).  Instead, we
use the sequence number as a key which helps us to avoid allocating another map
whenever we call MDSCacheObject::take_waiting.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CDir.h            |  3 +-
 src/mds/CInode.h          |  3 +-
 src/mds/MDSCacheObject.cc | 53 +++++++++++------------------------
 src/mds/MDSCacheObject.h  | 59 ++++++++++++++++++++++++++++-----------
 src/mds/SimpleLock.cc     | 24 ++++++++--------
 src/mds/SimpleLock.h      | 25 +++++++++++++----
 6 files changed, 95 insertions(+), 72 deletions(-)

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 234da01b10c9..b70725f714ed 100644
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -182,10 +182,11 @@ class CDir : public MDSCacheObject, public Counter<CDir> {
   static const uint64_t WAIT_COMPLETE     = (1<<1);  // wait for complete dir contents
   static const uint64_t WAIT_FROZEN       = (1<<2);  // auth pins removed
   static const uint64_t WAIT_CREATED	  = (1<<3);  // new dirfrag is logged
+  static const uint64_t WAIT_BITS         = 4;
 
   static const int WAIT_DNLOCK_OFFSET = 4;
 
-  static const uint64_t WAIT_ANY_MASK = (uint64_t)(-1);
+  static const uint64_t WAIT_ANY_MASK = ((1ul << WAIT_BITS) - 1);
   static const uint64_t WAIT_ATSUBTREEROOT = (WAIT_SINGLEAUTH);
 
   // -- dump flags --
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 7d91becb97b9..1030b7466238 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -398,8 +398,9 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   static const uint64_t WAIT_FROZEN      = (1<<1);
   static const uint64_t WAIT_TRUNC       = (1<<2);
   static const uint64_t WAIT_FLOCK       = (1<<3);
+  static const uint64_t WAIT_BITS        = 4;
   
-  static const uint64_t WAIT_ANY_MASK	= (uint64_t)(-1);
+  static const uint64_t WAIT_ANY_MASK    = ((1ul << WAIT_BITS) - 1);
 
   // misc
   static const unsigned EXPORT_NONCE = 1; // nonce given to replicas created by export
diff --git a/src/mds/MDSCacheObject.cc b/src/mds/MDSCacheObject.cc
index 626623a81288..2a7367824039 100644
--- a/src/mds/MDSCacheObject.cc
+++ b/src/mds/MDSCacheObject.cc
@@ -23,7 +23,7 @@ std::string_view MDSCacheObject::generic_pin_name(int p) const {
   }
 }
 
-void MDSCacheObject::finish_waiting(uint64_t mask, int result) {
+void MDSCacheObject::finish_waiting(waitmask_t mask, int result) {
   MDSContext::vec finished;
   take_waiting(mask, finished);
   finish_contexts(g_ceph_context, finished, result);
@@ -85,52 +85,31 @@ void MDSCacheObject::dump_states(ceph::Formatter *f) const
     f->dump_string("state", "rejoinundef");
 }
 
-bool MDSCacheObject::is_waiter_for(uint64_t mask, uint64_t min) {
-  if (!min) {
-    min = mask;
-    while (min & (min-1))  // if more than one bit is set
-      min &= min-1;        //  clear LSB
-  }
-  for (auto p = waiting.lower_bound(min); p != waiting.end(); ++p) {
-    if (p->first & mask) return true;
-    if (p->first > mask) return false;
+bool MDSCacheObject::is_waiter_for(waitmask_t mask) {
+  for ([[maybe_unused]] auto& [seq, waiter] : waiting) {
+    if ((waiter.mask & mask).any()) {
+      return true;
+    }
   }
   return false;
 }
 
-void MDSCacheObject::take_waiting(uint64_t mask, MDSContext::vec& ls) {
-  if (waiting.empty()) return;
-
-  // process ordered waiters in the same order that they were added.
-  std::map<uint64_t, MDSContext*> ordered_waiters;
-
+void MDSCacheObject::take_waiting(waitmask_t mask, MDSContext::vec& ls) {
+  if (waiting.empty()) {
+    return;
+  }
   for (auto it = waiting.begin(); it != waiting.end(); ) {
-    if (it->first & mask) {
-        if (it->second.first > 0) {
-          ordered_waiters.insert(it->second);
-        } else {
-          ls.push_back(it->second.second);
-        }
-//      pdout(10,g_conf()->debug_mds) << (mdsco_db_line_prefix(this))
-//                                 << "take_waiting mask " << hex << mask << dec << " took " << it->second
-//                                 << " tag " << hex << it->first << dec
-//                                 << " on " << *this
-//                                 << dendl;
-        waiting.erase(it++);
+    auto& waiter = it->second;
+    if ((waiter.mask & mask).any()) {
+      ls.push_back(waiter.c);
+      it = waiting.erase(it);
     } else {
-//      pdout(10,g_conf()->debug_mds) << "take_waiting mask " << hex << mask << dec << " SKIPPING " << it->second
-//                                 << " tag " << hex << it->first << dec
-//                                 << " on " << *this 
-//                                 << dendl;
-        ++it;
+      ++it;
     }
   }
-  for (auto it = ordered_waiters.begin(); it != ordered_waiters.end(); ++it) {
-    ls.push_back(it->second);
-  }
   if (waiting.empty()) {
     put(PIN_WAITER);
-    waiting.clear();
+    waiting.clear(); // free internal map
   }
 }
 
diff --git a/src/mds/MDSCacheObject.h b/src/mds/MDSCacheObject.h
index 8a319b4404d9..6fa40ea7f8d6 100644
--- a/src/mds/MDSCacheObject.h
+++ b/src/mds/MDSCacheObject.h
@@ -58,6 +58,16 @@ class MDSCacheObject {
  public:
   typedef mempool::mds_co::compact_map<mds_rank_t,unsigned> replica_map_type;
 
+  /* Mask for waiters. It is 128 bits to accomodate lock waiters. Its layout:
+   *
+   *  Most-significant                 Least significant
+   * [   SimpleLock 64 bits  |   MDSCacheObject 64 bits ]
+   *
+   * It is organized this way for simplicity not for compactness and because
+   * the total bits will be >64 bits.
+   */
+  using waitmask_t = std::bitset<128>;
+
   struct ptr_lt {
     bool operator()(const MDSCacheObject* l, const MDSCacheObject* r) const {
       return l->is_lt(r);
@@ -261,27 +271,39 @@ class MDSCacheObject {
   unsigned get_replica_nonce() const { return replica_nonce; }
   void set_replica_nonce(unsigned n) { replica_nonce = n; }
 
-  bool is_waiter_for(uint64_t mask, uint64_t min=0);
+  /* A uint64_t mask is accepted to accomodate existing code that expects the
+   * mask to actually be a 64 bit integer.
+   */
+  bool is_waiter_for(uint64_t mask) {
+    return is_waiter_for(waitmask_t(mask));
+  }
+  bool is_waiter_for(waitmask_t mask);
+
   virtual void add_waiter(uint64_t mask, MDSContext *c) {
+    add_waiter(waitmask_t(mask), c);
+  }
+  void add_waiter(waitmask_t mask, MDSContext *c) {
     if (waiting.empty())
       get(PIN_WAITER);
 
-    uint64_t seq = 0;
-    if (mask & WAIT_ORDERED) {
+    waiter_seq_t seq = 0;
+    if ((mask & waitmask_t(WAIT_ORDERED)).any()) {
       seq = ++last_wait_seq;
-      mask &= ~WAIT_ORDERED;
+      mask &= ~waitmask_t(WAIT_ORDERED);
+    } else {
+      /* always at the front */
+      seq = 0;
     }
-    waiting.insert(std::pair<uint64_t, std::pair<uint64_t, MDSContext*> >(
-			    mask,
-			    std::pair<uint64_t, MDSContext*>(seq, c)));
-//    pdout(10,g_conf()->debug_mds) << (mdsco_db_line_prefix(this)) 
-//			       << "add_waiter " << hex << mask << dec << " " << c
-//			       << " on " << *this
-//			       << dendl;
-    
+    waiting.insert(std::pair<waiter_seq_t, waiter>(seq, waiter(mask, c)));
+  }
+  virtual void take_waiting(uint64_t mask, MDSContext::vec& ls) {
+    take_waiting(waitmask_t(mask), ls);
   }
-  virtual void take_waiting(uint64_t mask, MDSContext::vec& ls);
-  void finish_waiting(uint64_t mask, int result = 0);
+  void take_waiting(waitmask_t mask, MDSContext::vec& ls);
+  void finish_waiting(uint64_t mask, int result = 0) {
+    finish_waiting(waitmask_t(mask), result);
+  }
+  void finish_waiting(waitmask_t mask, int result = 0);
 
   // ---------------------------------------------
   // locking
@@ -322,8 +344,13 @@ class MDSCacheObject {
   // ---------------------------------------------
   // waiting
  private:
-  mempool::mds_co::compact_multimap<uint64_t, std::pair<uint64_t, MDSContext*>> waiting;
-  static uint64_t last_wait_seq;
+  using waiter_seq_t = uint64_t;
+  struct waiter {
+    waitmask_t mask;
+    MDSContext* c;
+  };
+  mempool::mds_co::compact_multimap<waiter_seq_t, struct waiter> waiting;
+  static waiter_seq_t last_wait_seq;
 };
 
 inline std::ostream& operator<<(std::ostream& out, const mdsco_db_line_prefix& o) {
diff --git a/src/mds/SimpleLock.cc b/src/mds/SimpleLock.cc
index b23915f94521..4dea69f2b5d1 100644
--- a/src/mds/SimpleLock.cc
+++ b/src/mds/SimpleLock.cc
@@ -45,18 +45,18 @@ void SimpleLock::dump(ceph::Formatter *f) const {
 
 int SimpleLock::get_wait_shift() const {
   switch (get_type()) {
-    case CEPH_LOCK_DN:       return 8;
-    case CEPH_LOCK_DVERSION: return 8 + 1*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IAUTH:    return 8 + 2*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_ILINK:    return 8 + 3*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IDFT:     return 8 + 4*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IFILE:    return 8 + 5*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IVERSION: return 8 + 6*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IXATTR:   return 8 + 7*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_ISNAP:    return 8 + 8*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_INEST:    return 8 + 9*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IFLOCK:   return 8 +10*SimpleLock::WAIT_BITS;
-    case CEPH_LOCK_IPOLICY:  return 8 +11*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_DN:       return 0;
+    case CEPH_LOCK_DVERSION: return 1*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IAUTH:    return 2*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_ILINK:    return 3*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IDFT:     return 4*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IFILE:    return 5*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IVERSION: return 6*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IXATTR:   return 7*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_ISNAP:    return 8*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_INEST:    return 9*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IFLOCK:   return 10*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IPOLICY:  return 11*SimpleLock::WAIT_BITS;
     default:
       ceph_abort();
   }
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 3ef474c19a4b..8d3ea32e2442 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -200,7 +200,6 @@ class SimpleLock {
   int get_type() const { return type->type; }
   const sm_t* get_sm() const { return type->sm; }
 
-  int get_wait_shift() const;
   int get_cap_shift() const;
   int get_cap_mask() const;
 
@@ -210,17 +209,33 @@ class SimpleLock {
   void encode_locked_state(ceph::buffer::list& bl) {
     parent->encode_lock_state(type->type, bl);
   }
+
+  using waitmask_t = MDSCacheObject::waitmask_t;
+  static constexpr auto WAIT_ORDERED = waitmask_t(MDSCacheObject::WAIT_ORDERED);
+  int get_wait_shift() const;
+  MDSCacheObject::waitmask_t getmask(uint64_t mask) const {
+    /* See definition of MDSCacheObject::waitmask_t for waiter bits reserved
+     * for SimpleLock.
+     */
+    static constexpr int simplelock_shift = 64;
+    auto waitmask = waitmask_t(mask);
+    int shift = get_wait_shift();
+    ceph_assert(shift < 64);
+    shift += simplelock_shift;
+    waitmask <<= shift;
+    return waitmask;
+  }
   void finish_waiters(uint64_t mask, int r=0) {
-    parent->finish_waiting(mask << get_wait_shift(), r);
+    parent->finish_waiting(getmask(mask), r);
   }
   void take_waiting(uint64_t mask, MDSContext::vec& ls) {
-    parent->take_waiting(mask << get_wait_shift(), ls);
+    parent->take_waiting(getmask(mask), ls);
   }
   void add_waiter(uint64_t mask, MDSContext *c) {
-    parent->add_waiter((mask << get_wait_shift()) | MDSCacheObject::WAIT_ORDERED, c);
+    parent->add_waiter(getmask(mask) | WAIT_ORDERED, c);
   }
   bool is_waiter_for(uint64_t mask) const {
-    return parent->is_waiter_for(mask << get_wait_shift());
+    return parent->is_waiter_for(getmask(mask));
   }
 
   bool is_cached() const {

From a3e541f9992e5e264a8d9ecffd69787d722f88a0 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 9 Jan 2024 14:04:45 -0500
Subject: [PATCH 2366/2492] mds: add new inode quiescelock

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 doc/dev/mds_internals/locking.rst |   1 +
 src/include/ceph_fs.h             |  30 ++++---
 src/mds/CInode.cc                 |  44 ++++++++--
 src/mds/CInode.h                  |  27 +++---
 src/mds/Locker.cc                 | 141 +++++++++++++++++++++++++-----
 src/mds/Locker.h                  |   5 +-
 src/mds/Migrator.cc               |   5 ++
 src/mds/Server.cc                 |   7 +-
 src/mds/SimpleLock.cc             |   1 +
 src/mds/SimpleLock.h              |   2 +
 10 files changed, 208 insertions(+), 55 deletions(-)

diff --git a/doc/dev/mds_internals/locking.rst b/doc/dev/mds_internals/locking.rst
index cfd934f3f31a..4d21b895bea4 100644
--- a/doc/dev/mds_internals/locking.rst
+++ b/doc/dev/mds_internals/locking.rst
@@ -17,6 +17,7 @@ MDS defines a handful of lock types associated with different metadata for an in
 
   CEPH_LOCK_DN       - dentry
   CEPH_LOCK_DVERSION - dentry version
+  CEPH_LOCK_IQUIESCE - inode quiesce lock (a type of superlock)
   CEPH_LOCK_IVERSION - inode version
   CEPH_LOCK_IAUTH    - mode, uid, gid
   CEPH_LOCK_ILINK    - nlink
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 245421680265..016645ab5b02 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -333,18 +333,24 @@ extern const char *ceph_mds_state_name(int s);
  */
 #define CEPH_LOCK_DN          (1 << 0)
 #define CEPH_LOCK_DVERSION    (1 << 1)
-#define CEPH_LOCK_ISNAP       (1 << 4)  /* snapshot lock. MDS internal */
-#define CEPH_LOCK_IPOLICY     (1 << 5)  /* policy lock on dirs. MDS internal */
-#define CEPH_LOCK_IFILE       (1 << 6)
-#define CEPH_LOCK_INEST       (1 << 7)  /* mds internal */
-#define CEPH_LOCK_IDFT        (1 << 8)  /* dir frag tree */
-#define CEPH_LOCK_IAUTH       (1 << 9)
-#define CEPH_LOCK_ILINK       (1 << 10)
-#define CEPH_LOCK_IXATTR      (1 << 11)
-#define CEPH_LOCK_IFLOCK      (1 << 12)  /* advisory file locks */
-#define CEPH_LOCK_IVERSION    (1 << 13)  /* mds internal */
-
-#define CEPH_LOCK_IFIRST      CEPH_LOCK_ISNAP
+#define CEPH_LOCK_IQUIESCE    (1 << 4)  /* mds internal */
+#define CEPH_LOCK_ISNAP       (1 << 5)  /* snapshot lock. MDS internal */
+#define CEPH_LOCK_IPOLICY     (1 << 6)  /* policy lock on dirs. MDS internal */
+#define CEPH_LOCK_IFILE       (1 << 7)
+#define CEPH_LOCK_INEST       (1 << 8)  /* mds internal */
+#define CEPH_LOCK_IDFT        (1 << 9)  /* dir frag tree */
+#define CEPH_LOCK_IAUTH       (1 << 10)
+#define CEPH_LOCK_ILINK       (1 << 11)
+#define CEPH_LOCK_IXATTR      (1 << 12)
+#define CEPH_LOCK_IFLOCK      (1 << 13)  /* advisory file locks */
+#define CEPH_LOCK_IVERSION    (1 << 14)  /* mds internal */
+
+#define CEPH_LOCK_IFIRST      CEPH_LOCK_IQUIESCE
+#define CEPH_LOCK_ILAST       CEPH_LOCK_IVERSION
+
+static inline bool is_inode_lock(int l) {
+  return (CEPH_LOCK_IFIRST <= l && l <= CEPH_LOCK_ILAST);
+}
 
 
 /* client_session ops */
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 6d15908e5455..911bedf408fe 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -257,6 +257,8 @@ ostream& operator<<(ostream& out, const CInode& in)
     out << " " << in.xattrlock;
   if (!in.versionlock.is_sync_and_unlocked())  
     out << " " << in.versionlock;
+  if (!in.quiescelock.is_sync_and_unlocked())
+    out << " " << in.quiescelock;
 
   // hack: spit out crap on which clients have caps
   if (in.get_inode()->client_ranges.size())
@@ -324,6 +326,7 @@ CInode::CInode(MDCache *c, bool auth, snapid_t f, snapid_t l) :
     item_dirty_dirfrag_nest(this),
     item_dirty_dirfrag_dirfragtree(this),
     pop(c->decayrate),
+    quiescelock(this, &quiescelock_type),
     versionlock(this, &versionlock_type),
     authlock(this, &authlock_type),
     linklock(this, &linklock_type),
@@ -1660,6 +1663,7 @@ SimpleLock* CInode::get_lock(int type)
     case CEPH_LOCK_INEST: return &nestlock;
     case CEPH_LOCK_IFLOCK: return &flocklock;
     case CEPH_LOCK_IPOLICY: return &policylock;
+    case CEPH_LOCK_IQUIESCE: return &quiescelock;
   }
   return 0;
 }
@@ -2203,6 +2207,13 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
   case CEPH_LOCK_IPOLICY:
     encode_lock_ipolicy(bl);
     break;
+
+  case CEPH_LOCK_IQUIESCE: {
+    ENCODE_START(1, 1, bl);
+    /* skeleton */
+    ENCODE_FINISH(bl);
+    break;
+  }
   
   default:
     ceph_abort();
@@ -2270,6 +2281,13 @@ void CInode::decode_lock_state(int type, const bufferlist& bl)
     decode_lock_ipolicy(p);
     break;
 
+  case CEPH_LOCK_IQUIESCE: {
+    DECODE_START(1, p);
+    /* skeleton */
+    DECODE_FINISH(p);
+    break;
+  }
+
   default:
     ceph_abort();
   }
@@ -4264,8 +4282,8 @@ void CInode::_encode_locks_full(bufferlist& bl)
   encode(nestlock, bl);
   encode(flocklock, bl);
   encode(policylock, bl);
-
   encode(loner_cap, bl);
+  encode(quiescelock, bl);
 }
 void CInode::_decode_locks_full(bufferlist::const_iterator& p)
 {
@@ -4279,15 +4297,15 @@ void CInode::_decode_locks_full(bufferlist::const_iterator& p)
   decode(nestlock, p);
   decode(flocklock, p);
   decode(policylock, p);
-
   decode(loner_cap, p);
   set_loner_cap(loner_cap);
   want_loner_cap = loner_cap;  // for now, we'll eval() shortly.
+  decode(quiescelock, p);
 }
 
 void CInode::_encode_locks_state_for_replica(bufferlist& bl, bool need_recover)
 {
-  ENCODE_START(1, 1, bl);
+  ENCODE_START(2, 1, bl);
   authlock.encode_state_for_replica(bl);
   linklock.encode_state_for_replica(bl);
   dirfragtreelock.encode_state_for_replica(bl);
@@ -4298,11 +4316,13 @@ void CInode::_encode_locks_state_for_replica(bufferlist& bl, bool need_recover)
   flocklock.encode_state_for_replica(bl);
   policylock.encode_state_for_replica(bl);
   encode(need_recover, bl);
+  quiescelock.encode_state_for_replica(bl);
   ENCODE_FINISH(bl);
 }
 
 void CInode::_encode_locks_state_for_rejoin(bufferlist& bl, int rep)
 {
+  // TODO versioning?
   authlock.encode_state_for_replica(bl);
   linklock.encode_state_for_replica(bl);
   dirfragtreelock.encode_state_for_rejoin(bl, rep);
@@ -4312,11 +4332,12 @@ void CInode::_encode_locks_state_for_rejoin(bufferlist& bl, int rep)
   snaplock.encode_state_for_replica(bl);
   flocklock.encode_state_for_replica(bl);
   policylock.encode_state_for_replica(bl);
+  quiescelock.encode_state_for_replica(bl);
 }
 
 void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool is_new)
 {
-  DECODE_START(1, p);
+  DECODE_START(2, p);
   authlock.decode_state(p, is_new);
   linklock.decode_state(p, is_new);
   dirfragtreelock.decode_state(p, is_new);
@@ -4329,6 +4350,11 @@ void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool
 
   bool need_recover;
   decode(need_recover, p);
+
+  if (struct_v >= 2) {
+    quiescelock.decode_state(p, is_new);
+  }
+
   if (need_recover && is_new) {
     // Auth mds replicated this inode while it's recovering. Auth mds may take xlock on the lock
     // and change the object when replaying unsafe requests.
@@ -4341,6 +4367,7 @@ void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool
     snaplock.mark_need_recover();
     flocklock.mark_need_recover();
     policylock.mark_need_recover();
+    quiescelock.mark_need_recover();
   }
   DECODE_FINISH(p);
 }
@@ -4356,6 +4383,7 @@ void CInode::_decode_locks_rejoin(bufferlist::const_iterator& p, MDSContext::vec
   snaplock.decode_state_rejoin(p, waiters, survivor);
   flocklock.decode_state_rejoin(p, waiters, survivor);
   policylock.decode_state_rejoin(p, waiters, survivor);
+  quiescelock.decode_state_rejoin(p, waiters, survivor);
 
   if (!dirfragtreelock.is_stable() && !dirfragtreelock.is_wrlocked())
     eval_locks.push_back(&dirfragtreelock);
@@ -4370,7 +4398,7 @@ void CInode::_decode_locks_rejoin(bufferlist::const_iterator& p, MDSContext::vec
 
 void CInode::encode_export(bufferlist& bl)
 {
-  ENCODE_START(5, 4, bl);
+  ENCODE_START(6, 6, bl);
   _encode_base(bl, mdcache->mds->mdsmap->get_up_features());
 
   encode(state, bl);
@@ -4421,7 +4449,7 @@ void CInode::finish_export()
 void CInode::decode_import(bufferlist::const_iterator& p,
 			   LogSegment *ls)
 {
-  DECODE_START(5, p);
+  DECODE_START(6, p);
 
   _decode_base(p);
 
@@ -5099,6 +5127,10 @@ void CInode::dump(Formatter *f, int flags) const
     f->open_object_section("policylock");
     policylock.dump(f);
     f->close_section();
+
+    f->open_object_section("quiescelock");
+    quiescelock.dump(f);
+    f->close_section();
   }
 
   if (flags & DUMP_STATE) {
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 1030b7466238..1aaa5eadb5f5 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -1100,18 +1100,23 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   static const LockType nestlock_type;
   static const LockType flocklock_type;
   static const LockType policylock_type;
+  static const LockType quiescelock_type;
 
-  // FIXME not part of mempool
-  LocalLockC  versionlock;
-  SimpleLock authlock;
-  SimpleLock linklock;
-  ScatterLock dirfragtreelock;
-  ScatterLock filelock;
-  SimpleLock xattrlock;
-  SimpleLock snaplock;
-  ScatterLock nestlock;
-  SimpleLock flocklock;
-  SimpleLock policylock;
+  /* Please consult doc/dev/mds_internals/quiesce.rst for information about the
+   * quiescelock.
+   */
+
+  SimpleLock quiescelock; // FIXME not part of mempool
+  LocalLockC versionlock; // FIXME not part of mempool
+  SimpleLock authlock; // FIXME not part of mempool
+  SimpleLock linklock; // FIXME not part of mempool
+  ScatterLock dirfragtreelock; // FIXME not part of mempool
+  ScatterLock filelock; // FIXME not part of mempool
+  SimpleLock xattrlock; // FIXME not part of mempool
+  SimpleLock snaplock; // FIXME not part of mempool
+  ScatterLock nestlock; // FIXME not part of mempool
+  SimpleLock flocklock; // FIXME not part of mempool
+  SimpleLock policylock; // FIXME not part of mempool
 
   // -- caps -- (new)
   // client caps
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index e40e1c9b8f82..897f2ec5d03c 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -148,6 +148,8 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
   int depth = -1;
   bool found_locked = false;
   bool found_layout = false;
+  bool dropped_locks = false;
+  std::string_view err;
 
   ceph_assert(!want_layout || n == 0);
 
@@ -156,12 +158,13 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
   CInode *t = in;
   while (true) {
     ++depth;
+
     if (!found_locked && mdr->is_rdlocked(&t->snaplock))
       found_locked = true;
 
     if (!found_locked) {
       if (!t->snaplock.can_rdlock(client)) {
-        mdr->mark_event("failed to acquire snap lock");
+        err = "failed to acquire snap lock"sv;
 	t->snaplock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
 	goto failed;
       }
@@ -172,7 +175,7 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
     if (want_layout && !found_layout) {
       if (!mdr->is_rdlocked(&t->policylock)) {
 	if (!t->policylock.can_rdlock(client)) {
-          mdr->mark_event("failed to acquire policy lock");
+          err = "failed to acquire policy lock"sv;
 	  t->policylock.add_waiter(SimpleLock::WAIT_RD, new C_MDS_RetryRequest(mdcache, mdr));
 	  goto failed;
 	}
@@ -200,8 +203,11 @@ bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 failed:
   dout(10) << __func__ << " failed" << dendl;
 
-  drop_locks(mdr.get(), nullptr);
-  mdr->drop_local_auth_pins();
+  mdr->mark_event(err);
+  if (!dropped_locks) {
+    drop_locks(mdr.get(), nullptr);
+    mdr->drop_local_auth_pins();
+  }
   return false;
 }
 
@@ -224,7 +230,8 @@ struct MarkEventOnDestruct {
 bool Locker::acquire_locks(const MDRequestRef& mdr,
 			   MutationImpl::LockOpVec& lov,
 			   CInode *auth_pin_freeze,
-			   bool auth_pin_nonblocking)
+			   bool auth_pin_nonblocking,
+                           bool skip_quiesce)
 {
   dout(10) << "acquire_locks " << *mdr << dendl;
   dout(20) << " lov = " << lov << dendl;
@@ -242,10 +249,12 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
     mustpin.insert(auth_pin_freeze);
 
   // xlocks
+  bool need_quiescelock = !skip_quiesce;
   for (size_t i = 0; i < lov.size(); ++i) {
     auto& p = lov[i];
     SimpleLock *lock = p.lock;
     MDSCacheObject *object = lock->get_parent();
+    auto t = lock->get_type();
 
     if (p.is_xlock()) {
       if ((lock->get_type() == CEPH_LOCK_ISNAP ||
@@ -301,19 +310,30 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 	  lov.add_xlock(&dn->versionlock, i + 1);
 	}
       }
-      if (lock->get_type() >= CEPH_LOCK_IFIRST && lock->get_type() != CEPH_LOCK_IVERSION) {
-	// inode version lock?
-	CInode *in = static_cast<CInode*>(object);
-	if (!in->is_auth())
-	  continue;
-	if (mdr->is_leader()) {
-	  // leader.  wrlock versionlock so we can pipeline inode updates to journal.
-	  lov.add_wrlock(&in->versionlock, i + 1);
-	} else {
-	  // peer.  exclusively lock the inode version (i.e. block other journal updates).
-	  // this makes rollback safe.
-	  lov.add_xlock(&in->versionlock, i + 1);
-	}
+      if (is_inode_lock(t)) {
+        switch (t) {
+          case CEPH_LOCK_IVERSION:
+          case CEPH_LOCK_IQUIESCE:
+            break;
+          default:
+	    CInode *in = static_cast<CInode*>(object);
+            if (need_quiescelock) {
+              need_quiescelock = false;
+              lov.add_rdlock(&in->quiescelock, i + 1);
+            }
+	    if (!in->is_auth())
+	      continue;
+	    // inode version lock?
+	    if (mdr->is_leader()) {
+	      // leader.  wrlock versionlock so we can pipeline inode updates to journal.
+	      lov.add_wrlock(&in->versionlock, i + 1);
+	    } else {
+	      // peer.  exclusively lock the inode version (i.e. block other journal updates).
+	      // this makes rollback safe.
+	      lov.add_xlock(&in->versionlock, i + 1);
+	    }
+            break;
+        }
       }
     } else if (p.is_wrlock()) {
       dout(20) << " must wrlock " << *lock << " " << *object << dendl;
@@ -327,12 +347,21 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 		 << " in case we need to request a scatter" << dendl;
 	mustpin.insert(object);
       }
+      if (need_quiescelock && is_inode_lock(t) && t != CEPH_LOCK_IQUIESCE) {
+	CInode *in = static_cast<CInode*>(object);
+        lov.add_rdlock(&in->quiescelock, i + 1);
+        need_quiescelock = false;
+      }
     } else if (p.is_remote_wrlock()) {
       dout(20) << " must remote_wrlock on mds." << p.wrlock_target << " "
 	       << *lock << " " << *object << dendl;
       mustpin.insert(object);
+      if (need_quiescelock && is_inode_lock(t) && t != CEPH_LOCK_IQUIESCE) {
+	CInode *in = static_cast<CInode*>(object);
+        lov.add_rdlock(&in->quiescelock, i + 1);
+        need_quiescelock = false;
+      }
     } else if (p.is_rdlock()) {
-
       dout(20) << " must rdlock " << *lock << " " << *object << dendl;
       if (object->is_auth()) {
 	mustpin.insert(object);
@@ -342,6 +371,33 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 		 << " in case we need to request a rdlock" << dendl;
 	mustpin.insert(object);
       }
+
+      /* We treat rdlocks differently when adding the quiescelock. If the lock
+       * can be acquired immediately for reading without waiting
+       * (SimpleLock::can_rdlock), then skip adding the quiescelock. This is to
+       * allow some rdonly operations (like lookup) to proceed without blocking
+       * on the exclusively locked quiescelock. This is safe from deadlock (due
+       * to lock ordering) when Locker::acquire_locks is called more than once
+       * with different LockOpVectors for a given inode (already a dangerous
+       * thing to do) where there may be a wrlock/xlock in one set but not the
+       * other. The reason is simple: if Locker::acquire_locks ever adds the
+       * quiescelock, it is always the first lock to be acquired, and if it is
+       * xlocked, then all locks are dropped (s.f.
+       * Locker::handle_quiesce_failure). So adding the quiescelock can never
+       * contribute to deadlock.
+       */
+
+      if (need_quiescelock && !mdr->is_rdlocked(lock)) {
+        /* Can we get the lock without waiting? */
+        if (!lock->can_rdlock(client)) {
+          /* To prevent deadlock where an op holds a parent snaplock
+           * (Locker::try_rdlock_snap_layout), add quiescelock.
+           */
+	  CInode *in = static_cast<CInode*>(object);
+          lov.add_rdlock(&in->quiescelock, i + 1);
+          need_quiescelock = false;
+        }
+      }
     } else {
       ceph_assert(0 == "locker unknown lock operation");
     }
@@ -508,6 +564,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
   // make sure they match currently acquired locks.
   for (const auto& p : lov) {
     auto lock = p.lock;
+    auto t = lock->get_type();
     if (p.is_xlock()) {
       if (mdr->is_xlocked(lock)) {
 	dout(10) << " already xlocked " << *lock << " " << *lock->get_parent() << dendl;
@@ -516,7 +573,11 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
       if (mdr->locking && lock != mdr->locking)
 	cancel_locking(mdr.get(), &issue_set);
       if (!xlock_start(lock, mdr)) {
-	marker.message = "failed to xlock, waiting";
+        if (t == CEPH_LOCK_IQUIESCE) {
+          handle_quiesce_failure(mdr, marker.message);
+        } else {
+	  marker.message = "failed to xlock, waiting";
+        }
 	goto out;
       }
       dout(10) << " got xlock on " << *lock << " " << *lock->get_parent() << dendl;
@@ -542,18 +603,26 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 	if (p.is_remote_wrlock()) {
 	  // nowait if we have already gotten remote wrlock
 	  if (!wrlock_try(lock, mdr, _client)) {
-	    marker.message = "failed to wrlock, dropping remote wrlock and waiting";
 	    // can't take the wrlock because the scatter lock is gathering. need to
 	    // release the remote wrlock, so that the gathering process can finish.
 	    ceph_assert(it != mdr->locks.end());
 	    remote_wrlock_finish(it, mdr.get());
 	    remote_wrlock_start(lock, p.wrlock_target, mdr);
+            if (t == CEPH_LOCK_IQUIESCE) {
+              handle_quiesce_failure(mdr, marker.message);
+            } else {
+	      marker.message = "failed to wrlock, dropping remote wrlock and waiting";
+            }
 	    goto out;
 	  }
 	} else {
 	  if (!wrlock_start(p, mdr)) {
 	    ceph_assert(!p.is_remote_wrlock());
-	    marker.message = "failed to wrlock, waiting";
+            if (t == CEPH_LOCK_IQUIESCE) {
+              handle_quiesce_failure(mdr, marker.message);
+            } else {
+	      marker.message = "failed to wrlock, waiting";
+            }
 	    goto out;
 	  }
 	}
@@ -584,7 +653,11 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
       }
 
       if (!rdlock_start(lock, mdr)) {
-	marker.message = "failed to rdlock, waiting";
+        if (t == CEPH_LOCK_IQUIESCE) {
+          handle_quiesce_failure(mdr, marker.message);
+        } else {
+	  marker.message = "failed to rdlock, waiting";
+        }
 	goto out;
       }
       dout(10) << " got rdlock on " << *lock << " " << *lock->get_parent() << dendl;
@@ -600,6 +673,23 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
   return result;
 }
 
+/* Dropping *all* locks here is necessary so parent directory
+ * snap/layout/quiesce locks are unlocked for a future mksnap.  This is the
+ * primary purpose of the new quiescelock. An op, e.g. getattr, cannot block
+ * waiting for another lock held by quiesce_subvolume_inode, e.g. filelock,
+ * which will prevent a mksnap on a subvolume inode (because getattr will
+ * already have gotten parent snaplocks, see Locker::try_rdlock_snap_layout).
+ */
+
+void Locker::handle_quiesce_failure(const MDRequestRef& mdr, std::string_view& marker)
+{
+  dout(10) << " failed to acquire quiesce lock; dropping all locks" << dendl;
+  marker = "failed to acquire quiesce lock"sv;
+  drop_locks(mdr.get(), NULL);
+  mdr->drop_local_auth_pins();
+}
+
+
 void Locker::notify_freeze_waiter(MDSCacheObject *o)
 {
   CDir *dir = NULL;
@@ -1336,6 +1426,8 @@ bool Locker::eval(CInode *in, int mask, bool caps_imported)
     eval_any(&in->flocklock, &need_issue, &finishers, caps_imported);
   if (mask & CEPH_LOCK_IPOLICY)
     eval_any(&in->policylock, &need_issue, &finishers, caps_imported);
+  if (mask & CEPH_LOCK_IQUIESCE)
+    eval_any(&in->quiescelock, &need_issue, &finishers, caps_imported);
 
   // drop loner?
   if (in->is_auth() && in->is_head() && in->get_wanted_loner() != in->get_loner()) {
@@ -4418,6 +4510,7 @@ SimpleLock *Locker::get_lock(int lock_type, const MDSCacheObjectInfo &info)
   case CEPH_LOCK_ISNAP:
   case CEPH_LOCK_IFLOCK:
   case CEPH_LOCK_IPOLICY:
+  case CEPH_LOCK_IQUIESCE:
     {
       CInode *in = mdcache->get_inode(info.ino, info.snapid);
       if (!in) {
@@ -4434,6 +4527,7 @@ SimpleLock *Locker::get_lock(int lock_type, const MDSCacheObjectInfo &info)
       case CEPH_LOCK_ISNAP: return &in->snaplock;
       case CEPH_LOCK_IFLOCK: return &in->flocklock;
       case CEPH_LOCK_IPOLICY: return &in->policylock;
+      case CEPH_LOCK_IQUIESCE: return &in->quiescelock;
       }
     }
 
@@ -4459,6 +4553,7 @@ void Locker::handle_lock(const cref_t<MLock> &m)
 
   switch (lock->get_type()) {
   case CEPH_LOCK_DN:
+  case CEPH_LOCK_IQUIESCE:
   case CEPH_LOCK_IAUTH:
   case CEPH_LOCK_ILINK:
   case CEPH_LOCK_ISNAP:
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index eed421ba0db9..e3ecdf8131b1 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -54,7 +54,8 @@ class Locker {
   bool acquire_locks(const MDRequestRef& mdr,
 		     MutationImpl::LockOpVec& lov,
 		     CInode *auth_pin_freeze=NULL,
-		     bool auth_pin_nonblocking=false);
+		     bool auth_pin_nonblocking=false,
+                     bool skip_quiesce=false);
 
   bool try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 			      int n=0, bool want_layout=false);
@@ -259,6 +260,8 @@ class Locker {
   friend class LockerContext;
   friend class LockerLogContext;
 
+  void handle_quiesce_failure(const MDRequestRef& mdr, std::string_view& marker);
+
   bool any_late_revoking_caps(xlist<Capability*> const &revoking, double timeout) const;
   uint64_t calc_new_max_size(const CInode::inode_const_ptr& pi, uint64_t size);
   __u32 get_xattr_total_length(CInode::mempool_xattr_map &xattr);
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index a3e9d31ded14..b5690f4e2d78 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -1698,6 +1698,7 @@ void Migrator::finish_export_inode(CInode *in, mds_rank_t peer,
   in->snaplock.export_twiddle();
   in->flocklock.export_twiddle();
   in->policylock.export_twiddle();
+  in->quiescelock.export_twiddle();
   
   // mark auth
   ceph_assert(in->is_auth());
@@ -3245,6 +3246,10 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::const_iterator& blp,
   if (in->policylock.is_stable() &&
       in->policylock.get_state() != LOCK_SYNC)
       mds->locker->try_eval(&in->policylock, NULL);
+
+  if (in->quiescelock.is_stable() &&
+      in->quiescelock.get_state() != LOCK_SYNC)
+      mds->locker->try_eval(&in->quiescelock, NULL);
 }
 
 void Migrator::decode_import_inode_caps(CInode *in, bool auth_cap,
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 5749bfa7f263..d3d14a61ce6f 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -3076,8 +3076,9 @@ void Server::dispatch_peer_request(const MDRequestRef& mdr)
 	  replycode = MMDSPeerRequest::OP_WRLOCKACK;
 	  break;
 	}
-	
-	if (!mds->locker->acquire_locks(mdr, lov))
+
+        // don't add quiescelock, let the peer acquire that rdlock themselves
+	if (!mds->locker->acquire_locks(mdr, lov, nullptr, {}, false, true))
 	  return;
 	
 	// ack
@@ -4890,6 +4891,8 @@ void Server::handle_client_readdir(const MDRequestRef& mdr)
       return;
   }
 
+  /* readdir can add dentries to cache: acquire the quiescelock */
+  lov.add_rdlock(&diri->quiescelock);
   lov.add_rdlock(&diri->filelock);
   lov.add_rdlock(&diri->dirfragtreelock);
 
diff --git a/src/mds/SimpleLock.cc b/src/mds/SimpleLock.cc
index 4dea69f2b5d1..43f710ec68a5 100644
--- a/src/mds/SimpleLock.cc
+++ b/src/mds/SimpleLock.cc
@@ -57,6 +57,7 @@ int SimpleLock::get_wait_shift() const {
     case CEPH_LOCK_INEST:    return 9*SimpleLock::WAIT_BITS;
     case CEPH_LOCK_IFLOCK:   return 10*SimpleLock::WAIT_BITS;
     case CEPH_LOCK_IPOLICY:  return 11*SimpleLock::WAIT_BITS;
+    case CEPH_LOCK_IQUIESCE: return 12*SimpleLock::WAIT_BITS;
     default:
       ceph_abort();
   }
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 8d3ea32e2442..d0b6578302d7 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -41,6 +41,7 @@ struct LockType {
   explicit LockType(int t) : type(t) {
     switch (type) {
     case CEPH_LOCK_DN:
+    case CEPH_LOCK_IQUIESCE:
     case CEPH_LOCK_IAUTH:
     case CEPH_LOCK_ILINK:
     case CEPH_LOCK_IXATTR:
@@ -150,6 +151,7 @@ class SimpleLock {
       case CEPH_LOCK_ISNAP: return "isnap";
       case CEPH_LOCK_IFLOCK: return "iflock";
       case CEPH_LOCK_IPOLICY: return "ipolicy";
+      case CEPH_LOCK_IQUIESCE: return "iquiesce";
       default: return "unknown";
     }
   }

From 7c59c3749021e9113dfb777c197f70709f416ce6 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 10 Jan 2024 14:08:11 -0500
Subject: [PATCH 2367/2492] mds: pretty print mutation when dumping lock

Now looks like:

    ... (iquiesce xlock x=1 by request(mds.0:2785 nref=6)) ...

instead of:

    ... (iquiesce xlock x=1 by 0xdeadbeef)) ...

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/SimpleLock.cc | 26 ++++++++++++++++++++++++++
 src/mds/SimpleLock.h  | 23 +----------------------
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/src/mds/SimpleLock.cc b/src/mds/SimpleLock.cc
index 43f710ec68a5..70d277bd10c0 100644
--- a/src/mds/SimpleLock.cc
+++ b/src/mds/SimpleLock.cc
@@ -108,3 +108,29 @@ std::vector<MDLockCache*> SimpleLock::get_active_caches() {
   }
   return result;
 }
+
+void SimpleLock::_print(std::ostream& out) const
+{
+  out << get_lock_type_name(get_type()) << " ";
+  out << get_state_name(get_state());
+  if (!get_gather_set().empty())
+    out << " g=" << get_gather_set();
+  if (is_leased())
+    out << " l";
+  if (is_rdlocked())
+    out << " r=" << get_num_rdlocks();
+  if (is_wrlocked())
+    out << " w=" << get_num_wrlocks();
+  if (is_xlocked()) {
+    out << " x=" << get_num_xlocks();
+    if (auto mut = get_xlock_by(); mut) {
+      out << " by " << *mut;
+    }
+  }
+#if 0
+  if (is_stable())
+    out << " stable";
+  else
+    out << " unstable";
+#endif
+}
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index d0b6578302d7..f0fef93e1c60 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -581,28 +581,7 @@ class SimpleLock {
     return false;
   }
 
-  void _print(std::ostream& out) const {
-    out << get_lock_type_name(get_type()) << " ";
-    out << get_state_name(get_state());
-    if (!get_gather_set().empty())
-      out << " g=" << get_gather_set();
-    if (is_leased())
-      out << " l";
-    if (is_rdlocked()) 
-      out << " r=" << get_num_rdlocks();
-    if (is_wrlocked()) 
-      out << " w=" << get_num_wrlocks();
-    if (is_xlocked()) {
-      out << " x=" << get_num_xlocks();
-      if (get_xlock_by())
-	out << " by " << get_xlock_by();
-    }
-    /*if (is_stable())
-      out << " stable";
-    else
-      out << " unstable";
-    */
-  }
+  void _print(std::ostream& out) const;
 
   /**
    * Write bare values (caller must be in an object section)

From 3980454db49cbd70c42a502b832262552d9c9c74 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 27 Feb 2024 15:18:21 -0500
Subject: [PATCH 2368/2492] mds: print all SimpleLock flags in debug output

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/SimpleLock.cc | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/mds/SimpleLock.cc b/src/mds/SimpleLock.cc
index 70d277bd10c0..da266e30dab7 100644
--- a/src/mds/SimpleLock.cc
+++ b/src/mds/SimpleLock.cc
@@ -115,8 +115,18 @@ void SimpleLock::_print(std::ostream& out) const
   out << get_state_name(get_state());
   if (!get_gather_set().empty())
     out << " g=" << get_gather_set();
-  if (is_leased())
-    out << " l";
+  {
+    std::string flags;
+    if (is_leased())
+      flags += "l";
+    if (is_cached())
+      flags += "c";
+    if (needs_recover())
+      flags += "r";
+    if (!flags.empty()) {
+      out << " " << flags;
+    }
+  }
   if (is_rdlocked())
     out << " r=" << get_num_rdlocks();
   if (is_wrlocked())

From 06916dd2a7f30874d29b03e0cd998370cf595f58 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 Nov 2023 13:17:57 -0500
Subject: [PATCH 2369/2492] mds: add quiesce op

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

Fixes: https://tracker.ceph.com/issues/63664
---
 src/common/ceph_strings.cc     |   2 +
 src/common/options/mds.yaml.in |  18 ++
 src/include/ceph_fs.h          |   4 +-
 src/include/fs_types.h         |   1 +
 src/mds/Locker.cc              |  32 +++-
 src/mds/Locker.h               |   2 +
 src/mds/MDCache.cc             | 330 ++++++++++++++++++++++++++++++++-
 src/mds/MDCache.h              |  20 +-
 src/mds/MDSDaemon.cc           |   6 +
 src/mds/MDSRank.cc             |  51 +++++
 src/mds/MDSRank.h              |   1 +
 src/mds/Migrator.cc            |   2 +-
 src/mds/Mutation.h             |   6 +
 13 files changed, 461 insertions(+), 14 deletions(-)

diff --git a/src/common/ceph_strings.cc b/src/common/ceph_strings.cc
index 5103b1a3f83c..e36df170febc 100644
--- a/src/common/ceph_strings.cc
+++ b/src/common/ceph_strings.cc
@@ -311,6 +311,8 @@ const char *ceph_mds_op_name(int op)
 	case CEPH_MDS_OP_ENQUEUE_SCRUB: return "enqueue_scrub";
 	case CEPH_MDS_OP_REPAIR_FRAGSTATS: return "repair_fragstats";
 	case CEPH_MDS_OP_REPAIR_INODESTATS: return "repair_inodestats";
+	case CEPH_MDS_OP_QUIESCE_PATH: return "quiesce_path";
+	case CEPH_MDS_OP_QUIESCE_INODE: return "quiesce_inode";
 	}
 	return "???";
 }
diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 7bd958f5951f..6b7ef89080a0 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -74,6 +74,24 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_cache_quiesce_delay
+  type: millisecs
+  level: dev
+  desc: delay before starting recursive quiesce inode operations
+  default: 0
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_cache_quiesce_splitauth
+  type: bool
+  level: advanced
+  desc: allow recursive quiesce across auth boundaries
+  default: false
+  services:
+  - mds
+  flags:
+  - runtime
 - name: mds_cache_release_free_interval
   type: secs
   level: dev
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 016645ab5b02..49b45f26eb35 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -436,7 +436,9 @@ enum {
 	CEPH_MDS_OP_ENQUEUE_SCRUB  = 0x01503,
 	CEPH_MDS_OP_REPAIR_FRAGSTATS = 0x01504,
 	CEPH_MDS_OP_REPAIR_INODESTATS = 0x01505,
-	CEPH_MDS_OP_RDLOCK_FRAGSSTATS = 0x01507
+	CEPH_MDS_OP_RDLOCK_FRAGSSTATS = 0x01507,
+	CEPH_MDS_OP_QUIESCE_PATH = 0x01508,
+	CEPH_MDS_OP_QUIESCE_INODE = 0x01509,
 };
 
 #define IS_CEPH_MDS_OP_NEWINODE(op) (op == CEPH_MDS_OP_CREATE     || \
diff --git a/src/include/fs_types.h b/src/include/fs_types.h
index 606b9c6503d4..5be4f74ebfd3 100644
--- a/src/include/fs_types.h
+++ b/src/include/fs_types.h
@@ -48,6 +48,7 @@ class JSONObj;
 #define CEPHFS_EFAULT          14
 #define CEPHFS_EISCONN         106
 #define CEPHFS_EMULTIHOP       72
+#define CEPHFS_EINPROGRESS     115
 
 // taken from linux kernel: include/uapi/linux/fcntl.h
 #define CEPHFS_AT_FDCWD        -100    /* Special value used to indicate
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 897f2ec5d03c..7b6b8266700a 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -229,7 +229,8 @@ struct MarkEventOnDestruct {
  * on the appropriate wait list */
 bool Locker::acquire_locks(const MDRequestRef& mdr,
 			   MutationImpl::LockOpVec& lov,
-			   CInode *auth_pin_freeze,
+			   CInode* auth_pin_freeze,
+                           std::set<MDSCacheObject*> mustpin,
 			   bool auth_pin_nonblocking,
                            bool skip_quiesce)
 {
@@ -244,7 +245,6 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 
   client_t client = mdr->get_client();
 
-  set<MDSCacheObject*> mustpin;  // items to authpin
   if (auth_pin_freeze)
     mustpin.insert(auth_pin_freeze);
 
@@ -447,7 +447,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
       continue;
     }
     int err = 0;
-    if (!object->can_auth_pin(&err)) {
+    if (!object->can_auth_pin(&err, skip_quiesce)) {
       if (mdr->lock_cache) {
 	CDir *dir;
 	if (CInode *in = dynamic_cast<CInode*>(object)) {
@@ -544,6 +544,8 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
       }
       if (auth_pin_nonblocking)
 	req->mark_nonblocking();
+      if (skip_quiesce)
+	req->mark_bypassfreezing();
       else if (!mdr->locks.empty())
 	req->mark_notify_blocking();
 
@@ -676,7 +678,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 /* Dropping *all* locks here is necessary so parent directory
  * snap/layout/quiesce locks are unlocked for a future mksnap.  This is the
  * primary purpose of the new quiescelock. An op, e.g. getattr, cannot block
- * waiting for another lock held by quiesce_subvolume_inode, e.g. filelock,
+ * waiting for another lock held by quiesce_inode, e.g. filelock,
  * which will prevent a mksnap on a subvolume inode (because getattr will
  * already have gotten parent snaplocks, see Locker::try_rdlock_snap_layout).
  */
@@ -863,6 +865,28 @@ void Locker::drop_rdlocks_for_early_reply(MutationImpl *mut)
   issue_caps_set(need_issue);
 }
 
+void Locker::drop_rdlock(MutationImpl* mut, SimpleLock* what)
+{
+  dout(20) << __func__ << ": " << *what << dendl;
+
+  for (auto it = mut->locks.begin(); it != mut->locks.end(); ++it) {
+    auto* lock = it->lock;
+    if (lock == what) {
+      dout(20) << __func__ << ": found lock " << *lock << dendl;
+      ceph_assert(it->is_rdlock());
+      bool ni = false;
+      rdlock_finish(it, mut, &ni);
+      if (ni) {
+        set<CInode*> need_issue;
+        need_issue.insert(static_cast<CInode*>(lock->get_parent()));
+        issue_caps_set(need_issue);
+      }
+      return;
+    }
+  }
+  dout(20) << __func__ << ": not found!" << dendl;
+}
+
 void Locker::drop_locks_for_fragment_unfreeze(MutationImpl *mut)
 {
   set<CInode*> need_issue;
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index e3ecdf8131b1..cfc0d9ace9a1 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -54,6 +54,7 @@ class Locker {
   bool acquire_locks(const MDRequestRef& mdr,
 		     MutationImpl::LockOpVec& lov,
 		     CInode *auth_pin_freeze=NULL,
+                     std::set<MDSCacheObject*> mustpin = {},
 		     bool auth_pin_nonblocking=false,
                      bool skip_quiesce=false);
 
@@ -66,6 +67,7 @@ class Locker {
   void set_xlocks_done(MutationImpl *mut, bool skip_dentry=false);
   void drop_non_rdlocks(MutationImpl *mut, std::set<CInode*> *pneed_issue=0);
   void drop_rdlocks_for_early_reply(MutationImpl *mut);
+  void drop_rdlock(MutationImpl* mut, SimpleLock* what);
   void drop_locks_for_fragment_unfreeze(MutationImpl *mut);
 
   int get_cap_bit_for_lock_cache(int op);
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index d2c118675745..bc0271251c64 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -9663,8 +9663,18 @@ MDRequestRef MDCache::request_start_internal(int op)
   params.all_read = now;
   params.dispatched = now;
   params.internal_op = op;
-  MDRequestRef mdr =
-      mds->op_tracker.create_request<MDRequestImpl,MDRequestImpl::Params*>(&params);
+
+  switch (op) {
+    case CEPH_MDS_OP_QUIESCE_PATH:
+    case CEPH_MDS_OP_QUIESCE_INODE:
+      params.continuous = true;
+      break;
+    default:
+      params.continuous = false;
+      break;
+  }
+
+  MDRequestRef mdr = mds->op_tracker.create_request<MDRequestImpl,MDRequestImpl::Params*>(&params);
 
   if (active_requests.count(mdr->reqid)) {
     auto& _mdr = active_requests[mdr->reqid];
@@ -9708,6 +9718,12 @@ void MDCache::request_finish(const MDRequestRef& mdr)
   }
 
   switch(mdr->internal_op) {
+    case CEPH_MDS_OP_QUIESCE_PATH:
+      logger->inc(l_mdss_ireq_quiesce_path);
+      break;
+    case CEPH_MDS_OP_QUIESCE_INODE:
+      logger->inc(l_mdss_ireq_quiesce_inode);
+      break;
     case CEPH_MDS_OP_FRAGMENTDIR:
       logger->inc(l_mdss_ireq_fragmentdir);
       break;
@@ -9765,6 +9781,12 @@ void MDCache::dispatch_request(const MDRequestRef& mdr)
     mds->server->dispatch_peer_request(mdr);
   } else {
     switch (mdr->internal_op) {
+    case CEPH_MDS_OP_QUIESCE_PATH:
+      dispatch_quiesce_path(mdr);
+      break;
+    case CEPH_MDS_OP_QUIESCE_INODE:
+      dispatch_quiesce_inode(mdr);
+      break;
     case CEPH_MDS_OP_FRAGMENTDIR:
       dispatch_fragment_dir(mdr);
       break;
@@ -9871,6 +9893,24 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
       mdr->clear_ambiguous_auth();
     if (!mdr->more()->waiting_for_finish.empty())
       mds->queue_waiters(mdr->more()->waiting_for_finish);
+    for (auto& [in, reqid] : mdr->more()->quiesce_ops) {
+      if (auto it = active_requests.find(reqid); it != active_requests.end()) {
+        auto qimdr = it->second;
+        dout(20) << "killing quiesce op " << *qimdr << dendl;
+        request_kill(qimdr);
+      }
+    }
+  }
+
+  if (mdr->internal_op == CEPH_MDS_OP_QUIESCE_PATH) {
+    /* This construction is obviously not performant but it's rarely done and only for subvolumes */
+    for (auto it = quiesced_subvolumes.begin(); it != quiesced_subvolumes.end();) {
+      if (it->second == mdr) {
+        it = quiesced_subvolumes.erase(it);
+      } else {
+        ++it;
+      }
+    }
   }
 
   request_drop_locks(mdr);
@@ -9930,6 +9970,18 @@ void MDCache::request_kill(const MDRequestRef& mdr)
     return;
   }
 
+  /* quiesce ops are all completed via request_kill */
+  switch(mdr->internal_op) {
+    case CEPH_MDS_OP_QUIESCE_PATH:
+      logger->inc(l_mdss_ireq_quiesce_path);
+      break;
+    case CEPH_MDS_OP_QUIESCE_INODE:
+      logger->inc(l_mdss_ireq_quiesce_inode);
+      break;
+    default:
+      break;
+  }
+
   mdr->killed = true;
   mdr->mark_event("killing request");
 
@@ -11992,7 +12044,7 @@ void MDCache::dispatch_fragment_dir(const MDRequestRef& mdr)
     // prevent a racing gather on any other scatterlocks too
     lov.lock_scatter_gather(&diri->nestlock);
     lov.lock_scatter_gather(&diri->filelock);
-    if (!mds->locker->acquire_locks(mdr, lov, NULL, true)) {
+    if (!mds->locker->acquire_locks(mdr, lov, NULL, {}, true)) {
       if (!mdr->aborted)
 	return;
     }
@@ -13380,6 +13432,10 @@ void MDCache::register_perfcounters()
                         "Stray dentries migrated");
 
     // low prio internal request stats
+    pcb.add_u64_counter(l_mdss_ireq_quiesce_path, "ireq_quiesce_path",
+                        "Internal Request type quiesce subvolume");
+    pcb.add_u64_counter(l_mdss_ireq_quiesce_inode, "ireq_quiesce_inode",
+                        "Internal Request type quiesce subvolume inode");
     pcb.add_u64_counter(l_mdss_ireq_enqueue_scrub, "ireq_enqueue_scrub",
                         "Internal Request type enqueue scrub");
     pcb.add_u64_counter(l_mdss_ireq_exportdir, "ireq_exportdir",
@@ -13448,6 +13504,274 @@ void MDCache::clear_dirty_bits_for_stray(CInode* diri) {
   }
 }
 
+void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
+{
+  if (mdr->killed) {
+    dout(20) << __func__ << " " << *mdr << " not dispatching killed " << *mdr << dendl;
+    return;
+  } else if (mdr->internal_op_finish == nullptr) {
+    dout(20) << __func__ << " " << *mdr << " already finished quiesce" << dendl;
+    return;
+  }
+
+  auto* qfinisher = static_cast<C_MDS_QuiescePath*>(mdr->internal_op_private);
+  auto delay = qfinisher->delay;
+  auto splitauth = qfinisher->splitauth;
+  auto& qs = *qfinisher->qs;
+  auto qrmdr = qfinisher->mdr;
+
+  CInode *in = get_inode(mdr->get_filepath().get_ino());
+  if (in == nullptr) {
+    qs.add_failed(mdr, -CEPHFS_ENOENT);
+    mds->server->respond_to_request(mdr, -CEPHFS_ENOENT);
+    return;
+  }
+  const bool is_root = (mdr->get_filepath().get_ino() == mdr->get_filepath2().get_ino());
+
+  dout(20) << __func__ << " " << *mdr << " quiescing " << *in << dendl;
+
+
+  {
+    /* Acquire authpins on `in` to prevent migrations after this rank considers
+     * it (and its children) quiesced.
+     */
+
+    MutationImpl::LockOpVec lov;
+    if (!mds->locker->acquire_locks(mdr, lov, nullptr, {in}, false, true)) {
+      return;
+    }
+  }
+
+  /* TODO: Consider:
+   *
+   *  rank0 is auth for /foo
+   *  rank1 quiesces /foo with no dirents in cache (and stops)
+   *  rank0 begins quiescing /foo
+   *  rank0 exports a dirfrag of /foo/bar to rank1 (/foo/bar is not authpinned by rank1 nor by rank0 (yet))
+   *  rank1 discovers relevant paths in /foo/bar
+   *  rank1 now has /foo/bar in cache and may issue caps / execute operations
+   *
+   * The solution is probably to have rank1 mark /foo has STATE_QUIESCED and reject export ops from rank0.
+   */
+
+  if (in->is_auth()) {
+    /* Acquire rdlocks on anything which prevents writing.
+     *
+     * Because files are treated specially allowing multiple reader/writers, we
+     * need an xlock here to recall all write caps. This unfortunately means
+     * there can be no readers.
+     *
+     * The xlock on the quiescelock is important to prevent future requests
+     * from blocking on other inode locks while holding path traversal locks.
+     * See dev doc doc/dev/mds_internals/quiesce.rst for more details.
+     */
+
+    MutationImpl::LockOpVec lov;
+    lov.add_rdlock(&in->authlock);
+    lov.add_rdlock(&in->dirfragtreelock);
+    lov.add_rdlock(&in->filelock);
+    lov.add_rdlock(&in->linklock);
+    lov.add_rdlock(&in->nestlock);
+    lov.add_rdlock(&in->policylock);
+    // N.B.: NO xlock/wrlock on quiescelock; we need to allow access to mksnap/lookup
+    // This is an unfortunate inconsistency. It may be possible to circumvent
+    // this issue by having those ops acquire the quiscelock only if necessary.
+    if (is_root) {
+      lov.add_rdlock(&in->quiescelock);
+    } else {
+      lov.add_xlock(&in->quiescelock); /* !! */
+    }
+    lov.add_rdlock(&in->snaplock);
+    lov.add_rdlock(&in->xattrlock);
+    if (!mds->locker->acquire_locks(mdr, lov, nullptr, {in}, false, true)) {
+      return;
+    }
+  } else if (!splitauth) {
+    dout(5) << "auth is split and splitauth is false: " << *in << dendl;
+    qs.add_failed(mdr, -CEPHFS_EPERM);
+    mds->server->respond_to_request(mdr, -CEPHFS_EPERM);
+    return;
+  }
+
+  if (in->is_dir()) {
+    for (auto& dir : in->get_dirfrags()) {
+      if (!dir->is_auth() && !splitauth) {
+        dout(5) << "auth is split and splitauth is false: " << *dir << dendl;
+        qs.add_failed(mdr, -CEPHFS_EPERM);
+        mds->server->respond_to_request(mdr, -CEPHFS_EPERM);
+        return;
+      }
+    }
+    MDSGatherBuilder gather(g_ceph_context, new C_MDS_RetryRequest(this, mdr));
+    auto& qops = qrmdr->more()->quiesce_ops;
+    for (auto& dir : in->get_dirfrags()) {
+      for (auto& [dnk, dn] : *dir) {
+        auto* in = dn->get_projected_inode();
+        if (!in) {
+          continue;
+        }
+
+        if (auto it = qops.find(in); it != qops.end()) {
+          dout(25) << __func__ << ": existing quiesce metareqid: "  << it->second << dendl;
+          if (auto reqit = active_requests.find(it->second); reqit != active_requests.end()) {
+            auto& qimdr = reqit->second;
+            dout(25) << __func__ << ": found in-progress " << qimdr << dendl;
+            continue;
+          }
+        }
+        dout(10) << __func__ << ": scheduling op to quiesce " << *in << dendl;
+
+        MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
+        qimdr->set_filepath(filepath(in->ino()));
+        qimdr->internal_op_finish = gather.new_sub();
+        qimdr->internal_op_private = qfinisher;
+        qops[in] = qimdr->reqid;
+        qs.inc_inodes();
+        if (delay > 0ms) {
+          mds->timer.add_event_after(delay, new LambdaContext([cache=this,qimdr](int r) {
+            cache->dispatch_request(qimdr);
+          }));
+        } else {
+          dispatch_request(qimdr);
+        }
+        if (!(qs.inc_heartbeat_count() % mds->heartbeat_reset_grace())) {
+          mds->heartbeat_reset();
+        }
+      }
+    }
+    if (gather.has_subs()) {
+      dout(20) << __func__ << ": waiting for sub-ops to gather" << dendl;
+      gather.activate();
+      return;
+    }
+  }
+
+  if (in->is_auth()) {
+    dout(10) << __func__ << " " << *mdr << " quiesce complete of " << *in << dendl;
+    mdr->mark_event("quiesce complete");
+  } else {
+    dout(10) << __func__ << " " << *mdr << " non-auth quiesce complete of " << *in << dendl;
+    mdr->mark_event("quiesce complete for non-auth inode");
+  }
+
+  qs.inc_inodes_quiesced();
+  mdr->internal_op_finish->complete(0);
+  mdr->internal_op_finish = nullptr;
+
+  /* do not respond/complete so locks are not lost, parent request will complete */
+}
+
+void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
+{
+  if (mdr->killed) {
+    dout(20) << __func__ << " not dispatching killed " << *mdr << dendl;
+    return;
+  }
+
+  if (!mds->is_active()) {
+    dout(20) << __func__ << " is not active!" << dendl;
+    mds->server->respond_to_request(mdr, -CEPHFS_EAGAIN);
+    return;
+  }
+
+  ceph_assert(mdr->internal_op_finish);
+
+  dout(5) << __func__ << ": dispatching" << dendl;
+
+  C_MDS_QuiescePath* qfinisher = static_cast<C_MDS_QuiescePath*>(mdr->internal_op_finish);
+  auto& qs = *qfinisher->qs;
+  auto delay = qfinisher->delay = g_conf().get_val<std::chrono::milliseconds>("mds_cache_quiesce_delay");
+  auto splitauth = qfinisher->splitauth = g_conf().get_val<bool>("mds_cache_quiesce_splitauth");
+
+  CInode* diri = nullptr;
+  CF_MDS_RetryRequestFactory cf(this, mdr, true);
+  static const int ptflags = 0
+    | MDS_TRAVERSE_DISCOVER
+    | MDS_TRAVERSE_RDLOCK_PATH
+    | MDS_TRAVERSE_WANT_INODE
+    ;
+  int r = path_traverse(mdr, cf, mdr->get_filepath(), ptflags, nullptr, &diri);
+  if (r > 0)
+    return;
+  if (r < 0) {
+    mds->server->respond_to_request(mdr, r);
+    return;
+  }
+
+  if (!diri->is_dir()) {
+    dout(5) << __func__ << ": file is not a directory" << dendl;
+    mds->server->respond_to_request(mdr, -CEPHFS_ENOTDIR);
+    return;
+  }
+
+  if (auto [it, inserted] = quiesced_subvolumes.try_emplace(diri->ino(), mdr); !inserted) {
+    if (!it->second) {
+      it->second = mdr;
+    } else if (it->second != mdr) {
+      dout(5) << __func__ << ": quiesce operation already in flight: " << it->second << dendl;
+      mds->server->respond_to_request(mdr, -CEPHFS_EINPROGRESS);
+      return;
+    }
+  }
+
+  qfinisher->mdr = mdr;
+
+  for (auto& [qimdr, rc] : qs.get_failed()) {
+    dout(5) << __func__ << ": op " << *qimdr << " failed with " << rc << "!" << dendl;
+    mds->server->respond_to_request(mdr, rc);
+    return;
+  }
+
+  if (!diri->is_auth() && !splitauth) {
+    dout(5) << __func__ << ": skipping recursive quiesce of path for non-auth inode" << dendl;
+    mdr->mark_event("quiesce complete for non-auth tree");
+  } else if (auto& qops = mdr->more()->quiesce_ops; qops.count(diri) == 0) {
+    MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
+    qimdr->set_filepath(filepath(diri->ino()));
+    qimdr->set_filepath2(filepath(diri->ino())); /* is_root! */
+    qimdr->internal_op_finish = new C_MDS_RetryRequest(this, mdr);
+    qimdr->internal_op_private = qfinisher;
+    qops[diri] = qimdr->reqid;
+    qs.inc_inodes();
+    if (delay > 0ms) {
+      mds->timer.add_event_after(delay, new LambdaContext([cache=this,qimdr](int r) {
+        cache->dispatch_request(qimdr);
+      }));
+    } else {
+      dispatch_request(qimdr);
+    }
+    return;
+  } else {
+    dout(5) << __func__ << ": fully quiesced "  << *diri << dendl;
+    mdr->mark_event("quiesce complete");
+  }
+
+  if (qfinisher) {
+    qfinisher->complete(0);
+    mdr->internal_op_finish = nullptr;
+  }
+  mdr->result = 0;
+
+  /* caller kills this op */
+}
+
+MDRequestRef MDCache::quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *f, std::chrono::milliseconds delay) {
+  MDRequestRef mdr = request_start_internal(CEPH_MDS_OP_QUIESCE_PATH);
+  mdr->set_filepath(p);
+  mdr->internal_op_finish = c;
+
+  if (delay > 0ms) {
+    mds->timer.add_event_after(delay, new LambdaContext([cache=this,mdr=mdr](int r) {
+      cache->dispatch_request(mdr);
+    }));
+  } else {
+    dispatch_request(mdr);
+  }
+
+  return mdr;
+}
+
+
 bool MDCache::dump_inode(Formatter *f, uint64_t number) {
   CInode *in = get_inode(number);
   if (!in) {
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 990951b5531e..ba91a24e1451 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -108,6 +108,8 @@ enum {
   // How many inodes ever completed size recovery
   l_mdc_recovery_completed,
 
+  l_mdss_ireq_quiesce_path,
+  l_mdss_ireq_quiesce_inode,
   l_mdss_ireq_enqueue_scrub,
   l_mdss_ireq_exportdir,
   l_mdss_ireq_flush,
@@ -535,6 +537,9 @@ class MDCache {
     void inc_inodes_quiesced() {
       inodes_quiesced++;
     }
+    uint64_t inc_heartbeat_count() {
+      return ++heartbeat_count;
+    }
     uint64_t get_inodes() const {
       return inodes;
     }
@@ -564,6 +569,7 @@ class MDCache {
       f->close_section();
     }
 private:
+    uint64_t heartbeat_count = 0;
     uint64_t inodes = 0;
     uint64_t inodes_quiesced = 0;
     std::map<MDRequestRef, int> failed;
@@ -574,7 +580,7 @@ class MDCache {
       MDSInternalContext(c->mds), cache(c), finisher(_finisher) {}
     ~C_MDS_QuiescePath() {
       if (finisher) {
-        finisher->complete(-ECANCELED);
+        finisher->complete(-CEPHFS_ECANCELED);
         finisher = nullptr;
       }
     }
@@ -587,12 +593,14 @@ class MDCache {
         finisher = nullptr;
       }
     }
-    QuiesceStatistics qs;
+    std::shared_ptr<QuiesceStatistics> qs = std::make_shared<QuiesceStatistics>();
+    std::chrono::milliseconds delay = 0ms;
+    bool splitauth = false;
     MDCache *cache;
     MDRequestRef mdr;
     Context* finisher = nullptr;
   };
-  MDRequestRef quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *f = nullptr, std::chrono::milliseconds delay = 0ms) { c->complete(-ENOTSUP); return nullptr; }
+  MDRequestRef quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *f = nullptr, std::chrono::milliseconds delay = 0ms);
 
   void clean_open_file_lists();
   void dump_openfiles(Formatter *f);
@@ -1435,8 +1443,8 @@ class MDCache {
   void finish_uncommitted_fragment(dirfrag_t basedirfrag, int op);
   void rollback_uncommitted_fragment(dirfrag_t basedirfrag, frag_vec_t&& old_frags);
 
-  void dispatch_quiesce_path(const MDRequestRef& mdr) { }
-  void dispatch_quiesce_inode(const MDRequestRef& mdr) { }
+  void dispatch_quiesce_path(const MDRequestRef& mdr);
+  void dispatch_quiesce_inode(const MDRequestRef& mdr);
 
   void upkeep_main(void);
 
@@ -1479,6 +1487,8 @@ class MDCache {
   std::atomic<bool> upkeep_trim_shutdown{false};
 
   uint64_t kill_shutdown_at = 0;
+
+  std::map<inodeno_t, MDRequestRef> quiesced_subvolumes;
 };
 
 class C_MDS_RetryRequest : public MDSInternalContext {
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 3f29e756b4d2..fa430a30bc47 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -352,6 +352,12 @@ void MDSDaemon::set_up_admin_socket()
                                      asok_hook,
                                      "show cache status");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("quiesce path"
+                                     " name=path,type=CephString,req=true"
+                                     " name=wait,type=CephBool,req=false"
+				     ,asok_hook
+				     ,"quiesce a subtree");
+  ceph_assert(r == 0);
   r = admin_socket->register_command("dump tree "
 				     "name=root,type=CephString,req=true "
 				     "name=depth,type=CephInt,req=false "
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index b51148eedcb7..209de7cb86e3 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3001,6 +3001,8 @@ void MDSRankDispatcher::handle_asok_command(
   } else if (command == "cache status") {
     std::lock_guard l(mds_lock);
     mdcache->cache_status(f);
+  } else if (command == "quiesce path") {
+    r = command_quiesce_path(f, cmdmap, *css);
   } else if (command == "dump tree") {
     command_dump_tree(cmdmap, *css, f);
   } else if (command == "dump loads") {
@@ -3483,6 +3485,55 @@ void MDSRank::command_openfiles_ls(Formatter *f)
   mdcache->dump_openfiles(f);
 }
 
+class C_MDS_QuiescePathCommand : public MDCache::C_MDS_QuiescePath {
+public:
+  C_MDS_QuiescePathCommand(MDCache* cache, Context* fin) : C_MDS_QuiescePath(cache), finisher(fin) {}
+  void finish(int rc) override {
+    if (finisher) {
+      finisher->complete(rc);
+      finisher = nullptr;
+    }
+  }
+private:
+  Context* finisher = nullptr;
+};
+
+int MDSRank::command_quiesce_path(Formatter* f, const cmdmap_t& cmdmap, std::ostream& ss)
+{
+  std::string path;
+  {
+    bool got = cmd_getval(cmdmap, "path", path);
+    if (!got) {
+      ss << "missing path";
+      return -CEPHFS_EINVAL;
+    }
+  }
+
+  bool wait = false;
+  cmd_getval(cmdmap, "wait", wait);
+
+  C_SaferCond cond;
+  auto* finisher = new C_MDS_QuiescePathCommand(mdcache, wait ? &cond : nullptr);
+  auto qs = finisher->qs;
+  MDRequestRef mdr;
+  f->open_object_section("quiesce");
+  {
+    std::lock_guard l(mds_lock);
+    mdr = mdcache->quiesce_path(filepath(path), finisher, f);
+    if (!wait) {
+      f->dump_object("op", *mdr);
+    }
+  }
+  if (wait) {
+    cond.wait();
+    std::lock_guard l(mds_lock);
+    f->dump_object("op", *mdr);
+  }
+  f->dump_object("state", *qs);
+  f->close_section();
+  return 0;
+}
+
 void MDSRank::command_dump_inode(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss)
 {
   std::lock_guard l(mds_lock);
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 83fc3109fdb4..ea33b011ee52 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -530,6 +530,7 @@ class MDSRank {
         std::ostream &ss);
     void command_openfiles_ls(Formatter *f);
     void command_dump_tree(const cmdmap_t &cmdmap, std::ostream &ss, Formatter *f);
+    int command_quiesce_path(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss);
     void command_dump_inode(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss);
     void command_dump_dir(Formatter *f, const cmdmap_t &cmdmap, std::ostream &ss);
     void command_cache_drop(uint64_t timeout, Formatter *f, Context *on_finish);
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index b5690f4e2d78..dc898543d155 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -1067,7 +1067,7 @@ void Migrator::dispatch_export_dir(const MDRequestRef& mdr, int count)
     }
     lov.add_rdlock(&dir->get_inode()->dirfragtreelock);
 
-    if (!mds->locker->acquire_locks(mdr, lov, nullptr, true)) {
+    if (!mds->locker->acquire_locks(mdr, lov, nullptr, {}, true)) {
       if (mdr->aborted)
 	export_try_cancel(dir);
       return;
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index fcb0b5a21ad8..4b9d0d7528c8 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -349,6 +349,8 @@ struct MDRequestImpl : public MutationImpl {
 
     MDSContext::vec waiting_for_finish;
 
+    std::map<CInode*, metareqid_t> quiesce_ops;
+
     // export & fragment
     CDir* export_dir = nullptr;
     dirfrag_t fragment_base;
@@ -374,6 +376,9 @@ struct MDRequestImpl : public MutationImpl {
     const utime_t& get_dispatch_stamp() const {
       return dispatched;
     }
+    bool is_continuous() const {
+      return continuous;
+    }
     metareqid_t reqid;
     __u32 attempt = 0;
     ceph::cref_t<MClientRequest> client_req;
@@ -382,6 +387,7 @@ struct MDRequestImpl : public MutationImpl {
     utime_t initiated;
     utime_t throttled, all_read, dispatched;
     int internal_op = -1;
+    bool continuous = false;
   };
   MDRequestImpl(const Params* params, OpTracker *tracker) :
     MutationImpl(tracker, params->initiated,

From 92b5bcb44c0eacbc4c449a418350c0ff0506b7b5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 21 Feb 2024 14:08:13 -0500
Subject: [PATCH 2370/2492] mds: skip non-head inodes for quiesce

These are accessible only via snapshots.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index bc0271251c64..2d2b0273e2c4 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13607,7 +13607,7 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
     for (auto& dir : in->get_dirfrags()) {
       for (auto& [dnk, dn] : *dir) {
         auto* in = dn->get_projected_inode();
-        if (!in) {
+        if (!in || !in->is_head()) {
           continue;
         }
 

From af073d2fba48a80a533e2a1be2f21716d56aa26c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 31 Oct 2023 13:27:16 -0400
Subject: [PATCH 2371/2492] mds: add quiesce set feature flag

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

Fixes: https://tracker.ceph.com/issues/63664
---
 src/mds/MDSMap.cc | 2 ++
 src/mds/MDSMap.h  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc
index bedeed165ab9..cd5cb3a98a7b 100644
--- a/src/mds/MDSMap.cc
+++ b/src/mds/MDSMap.cc
@@ -53,6 +53,7 @@ CompatSet MDSMap::get_compat_set_all() {
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2);
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_SNAPREALM_V2);
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_MINORLOGSEGMENTS);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_QUIESCE_SUBVOLUMES);
 
   return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
 }
@@ -71,6 +72,7 @@ CompatSet MDSMap::get_compat_set_default() {
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2);
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_SNAPREALM_V2);
   feature_incompat.insert(MDS_FEATURE_INCOMPAT_MINORLOGSEGMENTS);
+  feature_incompat.insert(MDS_FEATURE_INCOMPAT_QUIESCE_SUBVOLUMES);
 
   return CompatSet(feature_compat, feature_ro_compat, feature_incompat);
 }
diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index c2cb9b009ff5..9ba6377da3f4 100644
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -49,6 +49,7 @@ static inline const auto MDS_FEATURE_INCOMPAT_NOANCHOR = CompatSet::Feature(8, "
 static inline const auto MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 = CompatSet::Feature(9, "file layout v2");
 static inline const auto MDS_FEATURE_INCOMPAT_SNAPREALM_V2 = CompatSet::Feature(10, "snaprealm v2");
 static inline const auto MDS_FEATURE_INCOMPAT_MINORLOGSEGMENTS = CompatSet::Feature(11, "minor log segments");
+static inline const auto MDS_FEATURE_INCOMPAT_QUIESCE_SUBVOLUMES = CompatSet::Feature(12, "quiesce subvolumes");
 
 #define MDS_FS_NAME_DEFAULT "cephfs"
 

From f4eec857fc8bda4980b9046cde6314fa005122c2 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 21 Feb 2024 10:03:14 -0500
Subject: [PATCH 2372/2492] mds: add counter to throttle quiesce

So a storm of quiesce operations do not affect normal MDS operations.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/common/options/mds.yaml.in | 27 +++++++++++++++++++++++++++
 src/mds/MDCache.cc             | 23 ++++++++++++++++++++++-
 src/mds/MDCache.h              |  3 +++
 src/mds/MDSRank.cc             |  3 +++
 4 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 6b7ef89080a0..d25e7b52edb0 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -163,6 +163,33 @@ options:
   - mds
   flags:
   - runtime
+- name: mds_cache_quiesce_decay_rate
+  type: float
+  level: advanced
+  desc: decay rate for quiescing inodes throttle
+  default: 1
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_cache_quiesce_threshold
+  type: size
+  level: advanced
+  desc: threshold for number of inodes that can be quiesced
+  default: 512_K
+  services:
+  - mds
+  flags:
+  - runtime
+- name: mds_cache_quiesce_sleep
+  type: millisecs
+  level: advanced
+  desc: sleep time for request after passing quiesce threshold
+  default: 200
+  services:
+  - mds
+  flags:
+  - runtime
 - name: mds_max_file_recover
   type: uint
   level: advanced
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 2d2b0273e2c4..8a3dd21d57be 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -122,7 +122,10 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   filer(m->objecter, m->finisher),
   stray_manager(m, purge_queue_),
   recovery_queue(m),
-  trim_counter(g_conf().get_val<double>("mds_cache_trim_decay_rate"))
+  trim_counter(g_conf().get_val<double>("mds_cache_trim_decay_rate")),
+  quiesce_counter(g_conf().get_val<double>("mds_cache_trim_decay_rate")),
+  quiesce_threshold(g_conf().get_val<Option::size_t>("mds_cache_quiesce_threshold")),
+  quiesce_sleep(g_conf().get_val<std::chrono::milliseconds>("mds_cache_quiesce_sleep"))
 {
   migrator.reset(new Migrator(mds, this));
 
@@ -195,6 +198,15 @@ void MDCache::handle_conf_change(const std::set<std::string>& changed, const MDS
     cache_health_threshold = g_conf().get_val<double>("mds_health_cache_threshold");
   if (changed.count("mds_cache_mid"))
     lru.lru_set_midpoint(g_conf().get_val<double>("mds_cache_mid"));
+  if (changed.count("mds_cache_quiesce_decay_rate")) {
+    quiesce_counter = DecayCounter(g_conf().get_val<double>("mds_cache_quiesce_decay_rate"));
+  }
+  if (changed.count("mds_cache_quiesce_threshold")) {
+    quiesce_threshold = g_conf().get_val<Option::size_t>("mds_cache_quiesce_threshold");
+  }
+  if (changed.count("mds_cache_quiesce_sleep")) {
+    quiesce_sleep = g_conf().get_val<std::chrono::milliseconds>("mds_cache_quiesce_sleep");
+  }
   if (changed.count("mds_cache_trim_decay_rate")) {
     trim_counter = DecayCounter(g_conf().get_val<double>("mds_cache_trim_decay_rate"));
   }
@@ -13530,6 +13542,15 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
 
   dout(20) << __func__ << " " << *mdr << " quiescing " << *in << dendl;
 
+  if (quiesce_counter.get() > quiesce_threshold) {
+    dout(20) << __func__
+             << " quiesce counter " << quiesce_counter
+             << " threshold (" << quiesce_threshold
+             << ") reached: scheduling retry" << dendl;
+    mds->timer.add_event_after(quiesce_sleep, new C_MDS_RetryRequest(this, mdr));
+    return;
+  }
+  quiesce_counter.hit();
 
   {
     /* Acquire authpins on `in` to prevent migrations after this rank considers
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index ba91a24e1451..ab34d238eafb 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1489,6 +1489,9 @@ class MDCache {
   uint64_t kill_shutdown_at = 0;
 
   std::map<inodeno_t, MDRequestRef> quiesced_subvolumes;
+  DecayCounter quiesce_counter;
+  uint64_t quiesce_threshold;
+  std::chrono::milliseconds quiesce_sleep;
 };
 
 class C_MDS_RetryRequest : public MDSInternalContext {
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 209de7cb86e3..c9e50d79fa20 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -4020,6 +4020,9 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_cache_memory_limit",
     "mds_cache_mid",
     "mds_cache_reservation",
+    "mds_cache_quiesce_decay_rate",
+    "mds_cache_quiesce_threshold",
+    "mds_cache_quiesce_sleep",
     "mds_cache_trim_decay_rate",
     "mds_cap_acquisition_throttle_retry_request_time",
     "mds_cap_revoke_eviction_timeout",

From fb05dd1806fc0edf2e4c0908aae114d57099a6d6 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 26 Jan 2024 16:34:41 -0500
Subject: [PATCH 2373/2492] mds: convert encoded ephemeral dist pin to flags

To use space efficiently, convert this field to a 8 bit flags field that is
backward compatible.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/include/cephfs/types.h | 46 +++++++++++++++++++++++++++++++++-----
 src/mds/CInode.cc          | 18 +++++++--------
 src/mds/CInode.h           |  2 +-
 src/mds/Server.cc          |  2 +-
 4 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 1b66929240b6..5ab5c229a063 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -394,6 +394,8 @@ struct inode_t {
    */
   using client_range_map = std::map<client_t,client_writeable_range_t,std::less<client_t>,Allocator<std::pair<const client_t,client_writeable_range_t>>>;
 
+  static const uint8_t F_EPHEMERAL_DISTRIBUTED_PIN = (1<<0);
+
   inode_t()
   {
     clear_layout();
@@ -482,6 +484,23 @@ struct inode_t {
     old_pools.insert(l);
   }
 
+  void set_flag(bool v, uint8_t flag) {
+    if (v) {
+      flags |= flag;
+    } else {
+      flags &= ~(flag);
+    }
+  }
+  bool get_flag(uint8_t flag) const {
+    return flags&flag;
+  }
+  void set_ephemeral_distributed_pin(bool v) {
+    set_flag(v, F_EPHEMERAL_DISTRIBUTED_PIN);
+  }
+  bool get_ephemeral_distributed_pin() const {
+    return get_flag(F_EPHEMERAL_DISTRIBUTED_PIN);
+  }
+
   void encode(ceph::buffer::list &bl, uint64_t features) const;
   void decode(ceph::buffer::list::const_iterator& bl);
   void dump(ceph::Formatter *f) const;
@@ -548,7 +567,23 @@ struct inode_t {
   mds_rank_t export_pin = MDS_RANK_NONE;
 
   double export_ephemeral_random_pin = 0;
-  bool export_ephemeral_distributed_pin = false;
+  /**
+   * N.B. previously this was a bool for distributed_ephemeral_pin which is
+   * encoded as a __u8. We take advantage of that to harness the remaining 7
+   * bits to avoid adding yet another field to this struct. This is safe also
+   * because the integral conversion of a bool to int (__u8) is well-defined
+   * per the standard as 0 (false) and 1 (true):
+   *
+   *     [conv.integral]
+   *     If the destination type is bool, see [conv.bool]. If the source type is
+   *     bool, the value false is converted to zero and the value true is converted
+   *     to one.
+   *
+   * So we can be certain the other bits have not be set during
+   * encoding/decoding due to implementation defined compiler behavior.
+   *
+   */
+  uint8_t flags = 0;
 
   // special stuff
   version_t version = 0;           // auth only
@@ -630,12 +665,13 @@ void inode_t<Allocator>::encode(ceph::buffer::list &bl, uint64_t features) const
   encode(export_pin, bl);
 
   encode(export_ephemeral_random_pin, bl);
-  encode(export_ephemeral_distributed_pin, bl);
+  encode(flags, bl);
 
   encode(!fscrypt_auth.empty(), bl);
   encode(fscrypt_auth, bl);
   encode(fscrypt_file, bl);
   encode(fscrypt_last_block, bl);
+
   ENCODE_FINISH(bl);
 }
 
@@ -734,10 +770,10 @@ void inode_t<Allocator>::decode(ceph::buffer::list::const_iterator &p)
 
   if (struct_v >= 16) {
     decode(export_ephemeral_random_pin, p);
-    decode(export_ephemeral_distributed_pin, p);
+    decode(flags, p);
   } else {
     export_ephemeral_random_pin = 0;
-    export_ephemeral_distributed_pin = false;
+    flags = 0;
   }
 
   if (struct_v >= 17) {
@@ -791,7 +827,7 @@ void inode_t<Allocator>::dump(ceph::Formatter *f) const
   f->dump_unsigned("change_attr", change_attr);
   f->dump_int("export_pin", export_pin);
   f->dump_int("export_ephemeral_random_pin", export_ephemeral_random_pin);
-  f->dump_bool("export_ephemeral_distributed_pin", export_ephemeral_distributed_pin);
+  f->dump_bool("export_ephemeral_distributed_pin", get_ephemeral_distributed_pin());
 
   f->open_array_section("client_ranges");
   for (const auto &p : client_ranges) {
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 911bedf408fe..8225e4fbb5a7 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -493,8 +493,8 @@ void CInode::pop_and_dirty_projected_inode(LogSegment *ls, const MutationRef& mu
 
   bool pool_updated = get_inode()->layout.pool_id != front.inode->layout.pool_id;
   bool pin_updated = (get_inode()->export_pin != front.inode->export_pin) ||
-		     (get_inode()->export_ephemeral_distributed_pin !=
-		      front.inode->export_ephemeral_distributed_pin);
+		     (get_inode()->get_ephemeral_distributed_pin() !=
+		      front.inode->get_ephemeral_distributed_pin());
 
   reset_inode(std::move(front.inode));
   if (front.xattrs != get_xattrs())
@@ -2130,7 +2130,7 @@ void CInode::encode_lock_ipolicy(bufferlist& bl)
     encode(get_inode()->layout, bl, mdcache->mds->mdsmap->get_up_features());
     encode(get_inode()->quota, bl);
     encode(get_inode()->export_pin, bl);
-    encode(get_inode()->export_ephemeral_distributed_pin, bl);
+    encode(get_inode()->flags, bl);
     encode(get_inode()->export_ephemeral_random_pin, bl);
   }
   ENCODE_FINISH(bl);
@@ -2151,15 +2151,15 @@ void CInode::decode_lock_ipolicy(bufferlist::const_iterator& p)
     decode(_inode->quota, p);
     decode(_inode->export_pin, p);
     if (struct_v >= 2) {
-      decode(_inode->export_ephemeral_distributed_pin, p);
+      decode(_inode->flags, p);
       decode(_inode->export_ephemeral_random_pin, p);
     }
   }
   DECODE_FINISH(p);
 
   bool pin_updated = (get_inode()->export_pin != _inode->export_pin) ||
-		     (get_inode()->export_ephemeral_distributed_pin !=
-		      _inode->export_ephemeral_distributed_pin);
+		     (get_inode()->get_ephemeral_distributed_pin() !=
+		      _inode->get_ephemeral_distributed_pin());
   reset_inode(std::move(_inode));
   maybe_export_pin(pin_updated);
 }
@@ -5443,7 +5443,7 @@ void CInode::setxattr_ephemeral_rand(double probability)
 void CInode::setxattr_ephemeral_dist(bool val)
 {
   ceph_assert(is_dir());
-  _get_projected_inode()->export_ephemeral_distributed_pin = val;
+  _get_projected_inode()->set_ephemeral_distributed_pin(val);
 }
 
 void CInode::set_export_pin(mds_rank_t rank)
@@ -5479,7 +5479,7 @@ mds_rank_t CInode::get_export_pin(bool inherit) const
 
     if (in->get_inode()->export_pin >= 0) {
       return in->get_inode()->export_pin;
-    } else if (in->get_inode()->export_ephemeral_distributed_pin &&
+    } else if (in->get_inode()->get_ephemeral_distributed_pin() &&
 	       mdcache->get_export_ephemeral_distributed_config()) {
       if (in != this)
 	return mdcache->hash_into_rank_bucket(in->ino(), dir->get_frag());
@@ -5545,7 +5545,7 @@ double CInode::get_ephemeral_rand() const
      * random pin set.
      */
     if (in->get_inode()->export_pin >= 0 ||
-	in->get_inode()->export_ephemeral_distributed_pin)
+	in->get_inode()->get_ephemeral_distributed_pin())
       return 0.0;
 
     in = pdn->get_dir()->inode;
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 1aaa5eadb5f5..10fabc402d6e 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -1025,7 +1025,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
 
   bool has_ephemeral_policy() const {
     return get_inode()->export_ephemeral_random_pin > 0.0 ||
-           get_inode()->export_ephemeral_distributed_pin;
+           get_inode()->get_ephemeral_distributed_pin();
   }
   bool is_ephemerally_pinned() const {
     return state_test(STATE_DISTEPHEMERALPIN) ||
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index d3d14a61ce6f..3bc4f9196b6d 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -6885,7 +6885,7 @@ void Server::handle_client_getvxattr(const MDRequestRef& mdr)
     } else if (xattr_name == "ceph.dir.pin.random"sv) {
       *css << cur->get_projected_inode()->export_ephemeral_random_pin;
     } else if (xattr_name == "ceph.dir.pin.distributed"sv) {
-      *css << cur->get_projected_inode()->export_ephemeral_distributed_pin;
+      *css << cur->get_projected_inode()->get_ephemeral_distributed_pin();
     } else {
       // otherwise respond as invalid request
       // since we only handle ceph vxattrs here

From e2529fc74808b102b1ca85a3ee6644160da32e60 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 26 Jan 2024 16:36:53 -0500
Subject: [PATCH 2374/2492] mds: add vxattr to block quiesce on an inode

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/include/cephfs/types.h |  8 ++++++
 src/mds/CInode.cc          |  3 +++
 src/mds/MDCache.cc         | 11 ++++++++
 src/mds/MDCache.h          |  8 ++++++
 src/mds/Server.cc          | 53 ++++++++++++++++++++++++++++++++++++++
 src/mds/Server.h           |  3 +++
 6 files changed, 86 insertions(+)

diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h
index 5ab5c229a063..108878794f75 100644
--- a/src/include/cephfs/types.h
+++ b/src/include/cephfs/types.h
@@ -395,6 +395,7 @@ struct inode_t {
   using client_range_map = std::map<client_t,client_writeable_range_t,std::less<client_t>,Allocator<std::pair<const client_t,client_writeable_range_t>>>;
 
   static const uint8_t F_EPHEMERAL_DISTRIBUTED_PIN = (1<<0);
+  static const uint8_t F_QUIESCE_BLOCK             = (1<<1);
 
   inode_t()
   {
@@ -500,6 +501,12 @@ struct inode_t {
   bool get_ephemeral_distributed_pin() const {
     return get_flag(F_EPHEMERAL_DISTRIBUTED_PIN);
   }
+  void set_quiesce_block(bool v) {
+    set_flag(v, F_QUIESCE_BLOCK);
+  }
+  bool get_quiesce_block() const {
+    return get_flag(F_QUIESCE_BLOCK);
+  }
 
   void encode(ceph::buffer::list &bl, uint64_t features) const;
   void decode(ceph::buffer::list::const_iterator& bl);
@@ -828,6 +835,7 @@ void inode_t<Allocator>::dump(ceph::Formatter *f) const
   f->dump_int("export_pin", export_pin);
   f->dump_int("export_ephemeral_random_pin", export_ephemeral_random_pin);
   f->dump_bool("export_ephemeral_distributed_pin", get_ephemeral_distributed_pin());
+  f->dump_bool("quiesce_block", get_quiesce_block());
 
   f->open_array_section("client_ranges");
   for (const auto &p : client_ranges) {
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 8225e4fbb5a7..ed9a74f4435b 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -310,6 +310,9 @@ ostream& operator<<(ostream& out, const CInode& in)
   if (in.state_test(CInode::STATE_RANDEPHEMERALPIN)) {
     out << " randepin";
   }
+  if (in.get_inode()->get_quiesce_block()) {
+    out << " qblock";
+  }
 
   out << " " << &in;
   out << "]";
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 8a3dd21d57be..a6df73510692 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13614,6 +13614,17 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
     return;
   }
 
+  if (in->get_projected_inode()->get_quiesce_block()) {
+    dout(10) << __func__ << " quiesce is blocked for this inode; dropping locks!" << dendl;
+    mdr->mark_event("quiesce blocked");
+    mds->locker->drop_locks(mdr.get());
+    /* keep authpins! */
+    qs.inc_inodes_blocked();
+    mdr->internal_op_finish->complete(0);
+    mdr->internal_op_finish = nullptr;
+    return;
+  }
+
   if (in->is_dir()) {
     for (auto& dir : in->get_dirfrags()) {
       if (!dir->is_auth() && !splitauth) {
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index ab34d238eafb..92a6be6d2a08 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -540,12 +540,18 @@ class MDCache {
     uint64_t inc_heartbeat_count() {
       return ++heartbeat_count;
     }
+    void inc_inodes_blocked() {
+      inodes_blocked++;
+    }
     uint64_t get_inodes() const {
       return inodes;
     }
     uint64_t get_inodes_quiesced() const {
       return inodes_quiesced;
     }
+    uint64_t get_inodes_blocked() const {
+      return inodes_blocked;
+    }
     void add_failed(const MDRequestRef& mdr, int rc) {
       failed[mdr] = rc;
     }
@@ -559,6 +565,7 @@ class MDCache {
     void dump(Formatter* f) const {
       f->dump_unsigned("inodes", inodes);
       f->dump_unsigned("inodes_quiesced", inodes_quiesced);
+      f->dump_unsigned("inodes_blocked", inodes_blocked);
       f->open_array_section("failed");
       for (auto& [mdr, rc] : failed) {
         f->open_object_section("failure");
@@ -572,6 +579,7 @@ class MDCache {
     uint64_t heartbeat_count = 0;
     uint64_t inodes = 0;
     uint64_t inodes_quiesced = 0;
+    uint64_t inodes_blocked = 0;
     std::map<MDRequestRef, int> failed;
   };
   class C_MDS_QuiescePath : public MDSInternalContext {
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 3bc4f9196b6d..04330cd308d1 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -6175,6 +6175,57 @@ void Server::handle_set_vxattr(const MDRequestRef& mdr, CInode *cur)
 
     client_t exclude_ct = mdr->get_client();
     mdcache->broadcast_quota_to_client(cur, exclude_ct, true);
+  } else if (name == "ceph.quiesce.block"sv) {
+    bool val;
+    try {
+      val = boost::lexical_cast<bool>(value);
+    } catch (boost::bad_lexical_cast const&) {
+      dout(10) << "bad vxattr value, unable to parse bool for " << name << dendl;
+      respond_to_request(mdr, -CEPHFS_EINVAL);
+      return;
+    }
+
+    /* Verify it's not already marked with lighter weight
+     * rdlock.
+     */
+    if (!mdr->more()->rdonly_checks) {
+      if (!(mdr->locking_state & MutationImpl::ALL_LOCKED)) {
+        lov.add_rdlock(&cur->policylock);
+        if (!mds->locker->acquire_locks(mdr, lov))
+          return;
+        mdr->locking_state |= MutationImpl::ALL_LOCKED;
+      }
+      bool is_blocked = cur->get_projected_inode()->get_quiesce_block();
+      if (is_blocked == val) {
+        dout(20) << "already F_QUIESCE_BLOCK set" << dendl;
+        respond_to_request(mdr, 0);
+        return;
+      }
+      mdr->more()->rdonly_checks = true;
+    }
+
+    if ((mdr->locking_state & MutationImpl::ALL_LOCKED) && !mdr->is_xlocked(&cur->policylock)) {
+      /* drop the rdlock and acquire xlocks */
+      dout(20) << "dropping rdlocks" << dendl;
+      mds->locker->drop_locks(mdr.get());
+      if (!xlock_policylock(mdr, cur, false, true))
+        return;
+    }
+
+    /* repeat rdonly checks in case changed between rdlock -> xlock */
+    bool is_blocked = cur->get_projected_inode()->get_quiesce_block();
+    if (is_blocked == val) {
+      dout(20) << "already F_QUIESCE_BLOCK set" << dendl;
+      respond_to_request(mdr, 0);
+      return;
+    }
+
+    auto pi = cur->project_inode(mdr);
+    pi.inode->set_quiesce_block(val);
+    dout(20) << (val ? "setting" : "unsetting") << " F_QUIESCE_BLOCK on ino: " << cur->ino() << dendl;
+
+    mdr->no_early_reply = true;
+    pip = pi.inode.get();
   } else if (name == "ceph.dir.subvolume"sv) {
     if (!cur->is_dir()) {
       respond_to_request(mdr, -CEPHFS_EINVAL);
@@ -6879,6 +6930,8 @@ void Server::handle_client_getvxattr(const MDRequestRef& mdr)
     } else {
       r = -CEPHFS_ENODATA; // no such attribute
     }
+  } else if (xattr_name == "ceph.quiesce.block"sv) {
+    *css << cur->get_projected_inode()->get_quiesce_block();
   } else if (xattr_name.substr(0, 12) == "ceph.dir.pin"sv) {
     if (xattr_name == "ceph.dir.pin"sv) {
       *css << cur->get_projected_inode()->export_pin;
diff --git a/src/mds/Server.h b/src/mds/Server.h
index 63d05ea32abb..152ab1b70199 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -17,6 +17,8 @@
 
 #include <string_view>
 
+using namespace std::literals::string_view_literals;
+
 #include <common/DecayCounter.h>
 
 #include "include/common_fwd.h"
@@ -441,6 +443,7 @@ class Server {
     return xattr_name.rfind("ceph.dir.layout", 0) == 0 ||
            xattr_name.rfind("ceph.file.layout", 0) == 0 ||
            xattr_name.rfind("ceph.quota", 0) == 0 ||
+           xattr_name == "ceph.quiesce.block"sv ||
            xattr_name == "ceph.dir.subvolume" ||
            xattr_name == "ceph.dir.pin" ||
            xattr_name == "ceph.dir.pin.random" ||

From 0308f215ae7a6392de4dbd658c6d4334c1c98e9d Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 26 Jan 2024 16:37:15 -0500
Subject: [PATCH 2375/2492] pybind/mgr/volumes: block quiesce for critical
 .meta file

This file must remain accessible during quiesce to allow snapshots to be taken
on a subvolume.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../fs/operations/versions/subvolume_base.py        | 13 +++++++++++++
 .../volumes/fs/operations/versions/subvolume_v1.py  |  3 +++
 .../volumes/fs/operations/versions/subvolume_v2.py  |  3 +++
 3 files changed, 19 insertions(+)

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
index 8fbe177e5f4b..a8afcc368073 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
@@ -69,6 +69,19 @@ def base_path(self):
     def config_path(self):
         return os.path.join(self.base_path, b".meta")
 
+    def mark_meta(self):
+        """
+        Set "ceph.quiesce.block" flag on the .meta file inode.  It must remain
+        available while a subvolume is quiesced in order to allow some
+        interactions with the subvolume, snapshots in particular.
+        """
+        try:
+            self.fs.setxattr(self.config_path, 'ceph.quiesce.block', b'1', 0)
+        except cephfs.InvalidValue:
+            raise VolumeException(-errno.EINVAL, "invalid value specified for ceph.quiesce.block")
+        except cephfs.Error as e:
+            raise VolumeException(-e.args[0], e.args[1])
+
     @property
     def legacy_dir(self):
         return (os.path.join(self.vol_spec.base_dir.encode('utf-8'),
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
index 90f35a4c90b3..5f81a6305053 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
@@ -111,6 +111,7 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
             qpath = subvol_path.decode('utf-8')
             self.init_config(SubvolumeV1.VERSION, subvolume_type, qpath, initial_state)
             self.mark_subvolume()
+            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 log.info("cleaning up subvolume with path: {0}".format(self.subvolname))
@@ -180,6 +181,7 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
             # create directory and set attributes
             self.fs.mkdirs(subvol_path, attrs.get("mode"))
             self.mark_subvolume()
+            self.mark_meta()
             self.set_attrs(subvol_path, attrs)
 
             # persist subvolume metadata and clone source
@@ -244,6 +246,7 @@ def open(self, op_type):
             st = self.fs.stat(subvol_path)
             # unconditionally mark as subvolume, to handle pre-existing subvolumes without the mark
             self.mark_subvolume()
+            self.mark_meta()
 
             self.uid = int(st.st_uid)
             self.gid = int(st.st_gid)
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
index 55d7f945b775..089d5ae1c018 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
@@ -189,6 +189,7 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
 
             # Create the subvolume metadata file which manages auth-ids if it doesn't exist
             self.auth_mdata_mgr.create_subvolume_metadata_file(self.group.groupname, self.subvolname)
+            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 self._remove_on_failure(subvol_path, retained)
@@ -243,6 +244,7 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
                 self.metadata_mgr.init(SubvolumeV2.VERSION, subvolume_type.value, qpath, initial_state.value)
             self.add_clone_source(source_volname, source_subvolume, snapname)
             self.metadata_mgr.flush()
+            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 self._remove_on_failure(subvol_path, retained)
@@ -300,6 +302,7 @@ def open(self, op_type):
             self.metadata_mgr.refresh()
             # unconditionally mark as subvolume, to handle pre-existing subvolumes without the mark
             self.mark_subvolume()
+            self.mark_meta()
 
             etype = self.subvol_type
             if op_type not in self.allowed_ops_by_type(etype):

From 51c9b86bf1f7be41f568ef1594b305133cd8b118 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 10 Jan 2024 22:08:13 -0500
Subject: [PATCH 2376/2492] doc: add dev docs for quiesce protocol

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 doc/dev/mds_internals/quiesce.rst | 93 +++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 doc/dev/mds_internals/quiesce.rst

diff --git a/doc/dev/mds_internals/quiesce.rst b/doc/dev/mds_internals/quiesce.rst
new file mode 100644
index 000000000000..72f3a6ddd27d
--- /dev/null
+++ b/doc/dev/mds_internals/quiesce.rst
@@ -0,0 +1,93 @@
+MDS Quiesce Protocol
+====================
+
+The MDS quiesce protocol is a mechanism for "quiescing" (quieting) a tree
+in a file system, stopping all write (and most read) I/O.
+
+The purpose of this API is to prevent multiple clients from interleaving reads
+and writes across an eventually consistent snapshot barrier where out-of-band
+communication exists between clients. This communication can lead to clients
+wrongly believing they've reached a checkpoint that is mutually recoverable to
+via a snapshot.
+
+Mechanism
+---------
+
+The MDS quiesces I/O using a new ``quiesce_path`` internal request that obtains
+appropriate locks on the root of a tree and then launches a series of
+sub-requests for locking other inodes in the tree. The locks obtained will
+force clients to release caps and in-progress client/MDS requests to complete.
+
+The sub-requests launched are ``quiesce_inode`` internal requests which simply
+lock the inode, if the MDS is authoritative for the inode.  Generally, these
+are rdlocks (read locks) on each inode metadata lock but the ``filelock`` is
+xlocked (exclusively locked) because its type allows for multiple readers and
+writers. Additionally, a new ``quiescelock`` is exclusively locked (more on
+that next).
+
+Because the ``quiesce_inode`` request will xlock the ``filelock`` and
+``quiescelock``, it only does so if run on the authoritative MDS. It is
+expected that the glue layer on top of the quiesce protocol will execute the
+same ``quiesce_path`` operation on each MDS rank.  This allows each rank which
+may be authoritative for part of the tree to lock all inodes it is
+authoritative for.
+
+
+Inode Quiescelock
+-----------------
+
+The ``quiescelock`` is a new lock for inodes which supports quiescing I/O.  It
+is a type of superlock where every client or MDS operation which accesses an
+inode lock will also implicitly acquire the ``quiescelock`` (readonly). In
+general, this lock is never held except for reading. When a subtree is
+quiesced, the ``quiesce_inode`` internal operation will hold ``quiescelock``
+exclusively, thereby denying the **new** acquisition of any other inode lock.
+The ``quiescelock`` must be ordered before all other locks (see
+``src/include/ceph_fs.h`` for ordering) in order to act as this superlock.
+
+The reason for this lock is to prevent an operation from blocking on acquiring
+locks held by ``quiesce_inode`` while still holding locks obtained
+during path traversal. Notably, the important locks are the ``snaplock`` and
+``policylock`` obtained via ``Locker::try_rdlock_snap_layout`` on all parents
+of the root inode of the request (the ``ino`` in the ``filepath`` struct). If
+that operation waits with those locks held, then a future ``mksnap`` on the
+root inode will be impossible.
+
+.. note:: The ``mksnap`` RPC only acquires a wrlock (write lock) on the
+          ``snaplock`` for the inode to be snapshotted.
+
+The way ``quiescelock`` helps prevent this is by being the first **mandatory**
+lock acquired and the special handling when it cannot be acquired: all locks
+held by the operation are dropped and the operation waits for the
+``quiescelock`` to be available.  The lock is mandatory in that all inode locks
+automatically include (add) the ``quiescelock`` when calling
+``Locker::acquire_locks``. So the expected normal flow is that an operation
+like ``getattr`` will perform its path traversal, acquiring parent and dentry
+locks, then attempt to acquire locks on the inode necessary for the requested
+client caps. The operation will fail to acquire the automatically included
+``quiescelock``, add itself to the ``quiescelock`` wait list, and then drop all
+held locks.
+
+There is a divergence in locking behavior for the root of the subvolume. The
+``quiescelock`` is only locked read-only. This allows the inode to be accessed
+by operations like ``mksnap`` which will implicitly acquire the ``quiescelock``
+read-only when locking the ``snaplock`` for writing. Additionally, if
+``Locker::acquire_locks`` will only acquire read locks without waiting, then it
+will skip the read-only lock on ``quiescelock``. This is to allow some forms of
+``lookup`` nececessary for snapshot management (e.g. volumes plugin) at higher
+layers.
+
+
+Readable quiesced tree
+----------------------
+
+It may be desirable to allow readers to continue accessing a quiesced
+subvolume. One way to do that is to have a separate superlock (yuck) for read
+access, say ``quiescerlock``. If a "readable" quiesce is performed, then
+``quiescerlock`` is not xlocked by ``quiesce_inode``. Read locks on
+other (non-quiesce) locks will acquire a read lock only on ``quiescerlock`` and
+no longer on ``quiescelock``.  Write locks would try to acquire both
+``quiescelock`` and ``quiescerlock`` (since writes may also read).
+
+Ideally, it may be a new lock type could be used to handle both cases but no
+such lock type yet exists.

From c3707b99c38ab153149684488276c76be6186d8f Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 6 Mar 2024 22:21:24 -0500
Subject: [PATCH 2377/2492] qa: simplify run_shell argument processing

No functional changes.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/mount.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 5263c5bfac47..007be38e223f 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -776,15 +776,14 @@ def run_python(self, pyscript, py_version='python3', sudo=False, timeout=None):
         p.wait()
         return p.stdout.getvalue().strip()
 
-    def run_shell(self, args, timeout=300, **kwargs):
-        omit_sudo = kwargs.pop('omit_sudo', False)
-        cwd = kwargs.pop('cwd', self.mountpoint)
-        stdout = kwargs.pop('stdout', StringIO())
-        stderr = kwargs.pop('stderr', StringIO())
-
-        return self.client_remote.run(args=args, cwd=cwd, timeout=timeout,
-                                      stdout=stdout, stderr=stderr,
-                                      omit_sudo=omit_sudo, **kwargs)
+    def run_shell(self, args, **kwargs):
+        kwargs.setdefault('cwd', self.mountpoint)
+        kwargs.setdefault('omit_sudo', False)
+        kwargs.setdefault('stdout', StringIO())
+        kwargs.setdefault('stderr', StringIO())
+        kwargs.setdefault('timeout', 300)
+
+        return self.client_remote.run(args=args, **kwargs)
 
     def run_shell_payload(self, payload, **kwargs):
         kwargs['args'] = ["bash", "-c", Raw(f"'{payload}'")]

From 4cfc5b802f4dcc4dec563b6dbc662feefb8ae951 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 23 Jan 2024 21:25:35 -0500
Subject: [PATCH 2378/2492] qa: use stdin-killer to timeout run_shell_payload

- simplify argument processing / forwarding
- use stdin-killer to kill all sub-processes of the shell
- do not needlessly use run_shell to execute the command as it adds a timeout
  to the stdout/stderr processing
- provide a stdin (PIPE) by default otherwise teuthology's code closes stdin
  and triggers stdin-killer to timeout the shell.
- use a 15 minute timeout by default

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/mount.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 007be38e223f..32f81551f99c 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -785,12 +785,25 @@ def run_shell(self, args, **kwargs):
 
         return self.client_remote.run(args=args, **kwargs)
 
-    def run_shell_payload(self, payload, **kwargs):
-        kwargs['args'] = ["bash", "-c", Raw(f"'{payload}'")]
+    def run_shell_payload(self, payload, wait=True, timeout=900, **kwargs):
+        kwargs.setdefault('cwd', self.mountpoint)
+        kwargs.setdefault('omit_sudo', False)
+        kwargs.setdefault('stdout', StringIO())
+        kwargs.setdefault('stderr', StringIO())
+        kwargs.setdefault('stdin', run.PIPE)
+        args = []
         if kwargs.pop('sudo', False):
-            kwargs['args'].insert(0, 'sudo')
+            args.append('sudo')
             kwargs['omit_sudo'] = False
-        return self.run_shell(**kwargs)
+        args.append("stdin-killer")
+        if timeout is not None:
+            args.append(f"--timeout={timeout}")
+        args += ("--", "bash", "-c", Raw(f"'{payload}'"))
+        p = self.client_remote.run(args=args, wait=False, **kwargs)
+        if wait:
+            p.stdin.close()
+            p.wait()
+        return p
 
     def run_as_user(self, **kwargs):
         """

From a1ac3e03a0227fc386f4341fc55b9a2c2a58d0bc Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 17 Feb 2024 10:26:14 -0500
Subject: [PATCH 2379/2492] qa: detect partial migrations during large config
 of dist epin

This method would wrongly "succeed" when looking for setup of distributed
ephemerally pinned directory fragments. If the migrator splits a subtree during
the course of migration (to reduce the migration size) then the operation may
not actually be complete.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/cephfs_test_case.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephfs/cephfs_test_case.py b/qa/tasks/cephfs/cephfs_test_case.py
index dea8a310e31b..7334a5972173 100644
--- a/qa/tasks/cephfs/cephfs_test_case.py
+++ b/qa/tasks/cephfs/cephfs_test_case.py
@@ -339,6 +339,7 @@ def _get_subtrees(self, status=None, rank=None, path=None):
                             subtrees = []
                             for r in self.fs.get_ranks(status=status):
                                 s = self.fs.rank_asok(["get", "subtrees"], status=status, rank=r['rank'])
+                                log.debug(f"{json.dumps(s, indent=2)}")
                                 s = filter(lambda s: s['auth_first'] == r['rank'] and s['auth_second'] == -2, s)
                                 subtrees += s
                         else:
@@ -384,11 +385,15 @@ def _wait_distributed_subtrees(self, count, status=None, rank=None, path=None):
             with contextutil.safe_while(sleep=5, tries=20) as proceed:
                 while proceed():
                     subtrees = self._get_subtrees(status=status, rank=rank, path=path)
-                    subtrees = list(filter(lambda s: s['distributed_ephemeral_pin'] == True and
-                                                     s['auth_first'] == s['export_pin_target'],
-                                           subtrees))
-                    log.info(f"len={len(subtrees)} {subtrees}")
+                    dist = list(filter(lambda s: s['distributed_ephemeral_pin'] == True and
+                                                 s['auth_first'] == s['export_pin_target'],
+                                       subtrees))
+                    log.info(f"len={len(dist)}\n{json.dumps(dist, indent=2)}")
+
                     if len(subtrees) >= count:
+                        if len(subtrees) > len(dist):
+                            # partial migration
+                            continue
                         return subtrees
         except contextutil.MaxWhileTries as e:
             raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) from e

From 7cf14fb10a528761dd3139d813834207c8233145 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 23 Jan 2024 21:26:46 -0500
Subject: [PATCH 2380/2492] qa: add quiesce protocol unit tests

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

Fixes: https://tracker.ceph.com/issues/63664
---
 qa/tasks/cephfs/filesystem.py   |  25 +-
 qa/tasks/cephfs/test_quiesce.py | 623 ++++++++++++++++++++++++++++++++
 2 files changed, 647 insertions(+), 1 deletion(-)
 create mode 100644 qa/tasks/cephfs/test_quiesce.py

diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 4532aaf280c5..f952a2740a79 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -1314,7 +1314,9 @@ def rank_asok(self, command, rank=0, status=None, timeout=None):
         info = self.get_rank(rank=rank, status=status)
         return self.json_asok(command, 'mds', info['name'], timeout=timeout)
 
-    def rank_tell(self, command, rank=0, status=None, timeout=120):
+    def rank_tell(self, command, rank=None, status=None, timeout=120):
+        if rank is None:
+            rank = 0
         try:
             out = self.get_ceph_cmd_stdout("tell", f"mds.{self.id}:{rank}", *command, timeout=timeout)
             return json.loads(out)
@@ -1778,3 +1780,24 @@ def get_damage(self, rank=None):
             return result
         else:
             return self.rank_tell(['damage', 'ls'], rank=rank)
+
+    def get_ops(self, locks=False, path=None, rank=None, status=None):
+        name = self.get_rank(rank=rank, status=status)['name']
+        cmd = ['ops']
+        if locks:
+            cmd.append('--flags=locks')
+        if path:
+            cmd.append(f'--path={path}')
+        J = self.rank_tell(cmd, rank=rank)
+        if path:
+            mds_remote = self.mon_manager.find_remote('mds', name)
+            blob = misc.get_file(mds_remote, path, sudo=True).decode('utf-8')
+            log.debug(f"read {len(blob)}B of ops")
+            J = json.loads(blob)
+        return J
+
+    def get_op(self, reqid, rank=None):
+        return self.rank_tell(['op', 'get', reqid], rank=rank)
+
+    def kill_op(self, reqid, rank=None):
+        return self.rank_tell(['op', 'kill', reqid], rank=rank)
diff --git a/qa/tasks/cephfs/test_quiesce.py b/qa/tasks/cephfs/test_quiesce.py
new file mode 100644
index 000000000000..12f4f7971cb0
--- /dev/null
+++ b/qa/tasks/cephfs/test_quiesce.py
@@ -0,0 +1,623 @@
+import json
+import logging
+import os
+import re
+import secrets
+import tempfile
+import unittest
+from io import StringIO
+import os.path
+from time import sleep
+
+from teuthology.contextutil import safe_while
+
+from tasks.cephfs.cephfs_test_case import CephFSTestCase
+
+log = logging.getLogger(__name__)
+
+INODE_RE = re.compile(r'\[inode 0x([0-9a-fA-F]+)')
+FP_RE = re.compile(r'fp=#0x([0-9a-fA-F]+)(\S*)')
+
+# MDS uses linux defines:
+S_IFMT   = 0o0170000
+S_IFSOCK =  0o140000
+S_IFLNK  =  0o120000
+S_IFREG  =  0o100000
+S_IFBLK  =  0o060000
+S_IFDIR  =  0o040000
+S_IFCHR  =  0o020000
+S_IFIFO  =  0o010000
+S_ISUID  =  0o004000
+S_ISGID  =  0o002000
+S_ISVTX  =  0o001000
+
+class QuiesceTestCase(CephFSTestCase):
+    """
+    Test case for quiescing subvolumes.
+    """
+
+    CLIENTS_REQUIRED = 2
+    MDSS_REQUIRED = 1
+
+    QUIESCE_SUBVOLUME = "subvol_quiesce"
+
+    def setUp(self):
+        super().setUp()
+        self.run_ceph_cmd(f'fs subvolume create {self.fs.name} {self.QUIESCE_SUBVOLUME} --mode=777')
+        p = self.run_ceph_cmd(f'fs subvolume getpath {self.fs.name} {self.QUIESCE_SUBVOLUME}', stdout=StringIO())
+        self.mntpnt = p.stdout.getvalue().strip()
+        self.subvolume = self.mntpnt
+
+    def tearDown(self):
+        # restart fs so quiesce commands clean up and commands are left unkillable
+        self.fs.fail()
+        self.fs.set_joinable()
+        self.fs.wait_for_daemons()
+        super().tearDown()
+
+    def _configure_subvolume(self):
+        for m in self.mounts:
+            m.umount_wait()
+        for m in self.mounts:
+            m.update_attrs(cephfs_mntpt = self.mntpnt)
+            m.mount()
+
+    CLIENT_WORKLOAD = """
+        set -ex
+        pushd `mktemp -d -p .`
+        cp -a /usr .
+        popd
+    """
+    def _client_background_workload(self):
+       for m in self.mounts:
+           p = m.run_shell_payload(self.CLIENT_WORKLOAD, wait=False, stderr=StringIO(), timeout=1)
+           m.background_procs.append(p)
+
+    def _wait_for_quiesce_complete(self, reqid, rank=0, path=None):
+        op = None
+        try:
+            with safe_while(sleep=1, tries=120, action='wait for quiesce completion') as proceed:
+                while proceed():
+                    op = self.fs.get_op(reqid, rank=rank)
+                    log.debug(f"op:\n{op}")
+                    self.assertEqual(op['type_data']['op_name'], 'quiesce_path')
+                    if op['type_data']['flag_point'] in (self.FP_QUIESCE_COMPLETE, self.FP_QUIESCE_COMPLETE_NON_AUTH_TREE):
+                        return op
+        except:
+            log.info(f"op:\n{op}")
+            if path is not None:
+                cache = self.fs.read_cache(path, rank=rank)
+                (fd, path) = tempfile.mkstemp()
+                with os.fdopen(fd, "wt") as f:
+                    f.write(f"{json.dumps(cache, indent=2)}")
+                    log.error(f"cache written to {path}")
+            (fd, path) = tempfile.mkstemp()
+            with os.fdopen(fd, "wt") as f:
+                ops = self.fs.get_ops(locks=True, rank=rank)
+                f.write(f"{json.dumps(ops, indent=2)}")
+                log.error(f"ops written to {path}")
+            raise
+
+    FP_QUIESCE_COMPLETE = 'quiesce complete'
+    FP_QUIESCE_BLOCKED = 'quiesce blocked'
+    FP_QUIESCE_COMPLETE_NON_AUTH = 'quiesce complete for non-auth inode'
+    FP_QUIESCE_COMPLETE_NON_AUTH_TREE = 'quiesce complete for non-auth tree'
+    def _verify_quiesce(self, rank=0, root=None, splitauth=False):
+        if root is None:
+            root = self.subvolume
+
+        name = self.fs.get_rank(rank=rank)['name']
+        root_ino = self.fs.read_cache(root, depth=0, rank=rank)[0]['ino']
+        ops = self.fs.get_ops(locks=True, rank=rank, path=f"/tmp/mds.{rank}-ops")
+        quiesce_inode_ops = {}
+        skipped_nonauth = False
+
+        count_q = 0
+        count_qb = 0
+        count_qna = 0
+
+        for op in ops['ops']:
+            try:
+                log.debug(f"op = {op}")
+                type_data = op['type_data']
+                flag_point = type_data['flag_point']
+                op_type = type_data['op_type']
+                if op_type == 'client_request' or op_type == 'peer_request':
+                    continue
+                op_name = type_data['op_name']
+                if op_name == "quiesce_path":
+                    self.assertIn(flag_point, (self.FP_QUIESCE_COMPLETE, self.FP_QUIESCE_COMPLETE_NON_AUTH_TREE))
+                    if flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH_TREE:
+                        skipped_nonauth = True
+                elif op_name == "quiesce_inode":
+                    # get the inode number
+                    op_description = op['description']
+                    m = FP_RE.search(op_description)
+                    self.assertIsNotNone(m)
+                    if len(m.group(2)) == 0:
+                        ino = int(m.group(1), 16)
+                    else:
+                        self.assertEqual(int(m.group(1)), 1)
+                        fp = m.group(2)
+                        dump = self.fs.read_cache(fp, depth=0, rank=rank)
+                        ino = dump[0]['ino']
+                    self.assertNotIn(ino, quiesce_inode_ops)
+
+                    self.assertIn(flag_point, (self.FP_QUIESCE_COMPLETE, self.FP_QUIESCE_BLOCKED, self.FP_QUIESCE_COMPLETE_NON_AUTH))
+
+                    locks = type_data['locks']
+                    if flag_point == self.FP_QUIESCE_BLOCKED:
+                        count_qb += 1
+                        self.assertEqual(locks, [])
+                    elif flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH:
+                        count_qna += 1
+                        #self.assertEqual(len(locks), 1)
+                        #lock = locks[0]
+                        #lock_type = lock['lock']['type']
+                        #self.assertEqual(lock_type, "iauth")
+                        #object_string = lock['object_string']
+                        #m = INODE_RE.match(object_string)
+                        #self.assertIsNotNone(m)
+                        #self.assertEqual(ino, int(m.group(1), 16))
+                    else:
+                        count_q += 1
+                        for lock in locks:
+                            lock_type = lock['lock']['type']
+                            if lock_type.startswith('i'):
+                                object_string = lock['object_string']
+                                m = INODE_RE.match(object_string)
+                                self.assertIsNotNone(m)
+                                self.assertEqual(ino, int(m.group(1), 16))
+                        self.assertIsNotNone(ino)
+                    quiesce_inode_ops[ino] = op
+            except:
+                log.error(f"op:\n{json.dumps(op, indent=2)}")
+                raise
+
+        log.info(f"q = {count_q}; qb = {count_qb}; qna = {count_qna}")
+
+        if skipped_nonauth:
+            return
+
+        for ino, op in quiesce_inode_ops.items():
+            log.debug(f"{ino}: {op['description']}")
+
+        # now verify all files in cache have an op
+        cache = self.fs.read_cache(root, rank=rank)
+        visited = set()
+        locks_expected = set([
+          "iquiesce",
+          "isnap",
+          "ipolicy",
+          "ifile",
+          "inest",
+          "idft",
+          "iauth",
+          "ilink",
+          "ixattr",
+        ])
+        for inode in cache:
+            ino = inode['ino']
+            visited.add(ino)
+            mode = inode['mode']
+            self.assertIn(ino, quiesce_inode_ops)
+            op = quiesce_inode_ops[ino]
+            type_data = op['type_data']
+            flag_point = type_data['flag_point']
+            try:
+                locks_seen = set()
+                lock_type = None
+                op_name = type_data['op_name']
+                for lock in op['type_data']['locks']:
+                    lock_type = lock['lock']['type']
+                    if lock_type == "iquiesce":
+                        if ino == root_ino:
+                            self.assertEqual(lock['flags'], 1)
+                            self.assertEqual(lock['lock']['state'], 'sync')
+                        else:
+                            self.assertEqual(lock['flags'], 4)
+                            self.assertEqual(lock['lock']['state'], 'xlock')
+                    elif lock_type == "isnap":
+                        self.assertEqual(lock['flags'], 1)
+                        self.assertEqual(lock['lock']['state'][:4], 'sync')
+                    elif lock_type == "ifile":
+                        self.assertEqual(lock['flags'], 1)
+                        self.assertEqual(lock['lock']['state'][:4], 'sync')
+                    elif lock_type in ("ipolicy", "inest", "idft", "iauth", "ilink", "ixattr"):
+                        self.assertEqual(lock['flags'], 1)
+                        self.assertEqual(lock['lock']['state'][:4], 'sync')
+                    else:
+                        # no iflock
+                        self.assertFalse(lock_type.startswith("i"))
+                    if flag_point == self.FP_QUIESCE_COMPLETE and lock_type.startswith("i"):
+                        #if op_name == "quiesce_inode":
+                        #    self.assertTrue(lock['object']['is_auth'])
+                        locks_seen.add(lock_type)
+                try:
+                    if flag_point == self.FP_QUIESCE_BLOCKED:
+                        self.assertTrue(inode['quiesce_block'])
+                        self.assertEqual(set(), locks_seen)
+                    elif flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH:
+                        self.assertFalse(inode['quiesce_block'])
+                        self.assertEqual(set(), locks_seen)
+                    else:
+                        self.assertFalse(inode['quiesce_block'])
+                        self.assertEqual(locks_expected, locks_seen)
+                except:
+                    log.error(f"{sorted(locks_expected)} != {sorted(locks_seen)}")
+                    raise
+            except:
+                log.error(f"inode:\n{json.dumps(inode, indent=2)}")
+                log.error(f"op:\n{json.dumps(op, indent=2)}")
+                log.error(f"lock_type: {lock_type}")
+                raise
+        try:
+            self.assertEqual(visited, quiesce_inode_ops.keys())
+        except:
+            log.error(f"cache:\n{json.dumps(cache, indent=2)}")
+            log.error(f"ops:\n{json.dumps(quiesce_inode_ops, indent=2)}")
+
+        # check request/cap count is stopped
+        # count inodes under /usr and count subops!
+
+    def reqid_tostr(self, reqid):
+        return f"{reqid['entity']['type']}.{reqid['entity']['num']}:{reqid['tid']}"
+
+class TestQuiesce(QuiesceTestCase):
+    """
+    Single rank functional tests.
+    """
+
+    def test_quiesce_path_workload(self):
+        """
+        That a quiesce op can be created and verified while a workload is running.
+        """
+
+        self._configure_subvolume()
+        self._client_background_workload()
+
+        sleep(secrets.randbelow(30)+10)
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+
+        self._verify_quiesce()
+
+    def test_quiesce_path_snap(self):
+        """
+        That a snapshot can be taken on a quiesced subvolume.
+        """
+
+        self._configure_subvolume()
+        self._client_background_workload()
+
+        sleep(secrets.randbelow(30)+10)
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+
+        #path = os.path.normpath(os.path.join(self.mntpnt, ".."))
+        #p = self.fs.run_client_payload(f"mkdir {path}/.snap/foo && ls {path}/.snap/", stdout=StringIO())
+        #p = self.mount_a.run_shell_payload(f"mkdir ../.snap/foo && ls ../.snap/", stdout=StringIO())
+        self.run_ceph_cmd(f'fs subvolume snapshot create {self.fs.name} {self.QUIESCE_SUBVOLUME} foo')
+        p = self.run_ceph_cmd(f'fs subvolume snapshot ls {self.fs.name} {self.QUIESCE_SUBVOLUME}', stdout=StringIO())
+        log.debug(f"{p.stdout.getvalue()}")
+
+    def test_quiesce_path_create(self):
+        """
+        That a quiesce op can be created and verified.
+        """
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce()
+
+    def test_quiesce_path_kill(self):
+        """
+        That killing a quiesce op also kills its subops
+        ("quiesce_inode").
+        """
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce()
+        ops = self.fs.get_ops()
+        quiesce_inode = 0
+        for op in ops['ops']:
+            op_name = op['type_data'].get('op_name', None)
+            if op_name == "quiesce_inode":
+                quiesce_inode += 1
+        log.debug(f"there are {quiesce_inode} quiesce_path_inode requests")
+        self.assertLess(0, quiesce_inode)
+        J = self.fs.kill_op(reqid)
+        log.debug(f"{J}")
+        ops = self.fs.get_ops()
+        for op in ops['ops']:
+            op_name = op['type_data'].get('op_name', None)
+            self.assertNotIn(op_name, ('quiesce_path', 'quiesce_inode'))
+
+    def test_quiesce_path_release(self):
+        """
+        That killing the quiesce op properly releases the subvolume so that
+        client IO proceeds.
+        """
+
+        self._configure_subvolume()
+        self._client_background_workload()
+
+        P = self.fs.rank_tell(["ops"])
+        log.debug(f"{P}")
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+
+        P = self.fs.rank_tell(["ops"])
+        log.debug(f"{P}")
+
+        self.fs.kill_op(reqid)
+
+        P = self.fs.rank_tell(["perf", "dump"])
+        log.debug(f"{P}")
+        requests = P['mds']['request']
+        replies = P['mds']['reply']
+        grants = P['mds']['ceph_cap_op_grant']
+
+        def resumed():
+            P = self.fs.rank_tell(["perf", "dump"])
+            log.debug(f"{P}")
+            try:
+                self.assertLess(requests, P['mds']['request'])
+                self.assertLess(replies, P['mds']['reply'])
+                self.assertLess(grants, P['mds']['ceph_cap_op_grant'])
+                return True
+            except AssertionError:
+                return False
+
+        self.wait_until_true(resumed, 60)
+
+        P = self.fs.rank_tell(["ops"])
+        log.debug(f"{P}")
+
+    def test_quiesce_path_link_terminal(self):
+        """
+        That quiesce on path with an terminal link fails with ENOTDIR even
+        pointing to a valid subvolume.
+        """
+
+        self._configure_subvolume()
+
+        self.mount_a.run_shell_payload("ln -s ../.. subvol_quiesce")
+        path = self.mount_a.cephfs_mntpt + "/subvol_quiesce"
+
+        J = self.fs.rank_tell(["quiesce", "path", path, '--wait'])
+        log.debug(f"{J}")
+        self.assertEqual(J['op']['result'], -20) # ENOTDIR: the link is not a directory
+
+    def test_quiesce_path_link_intermediate(self):
+        """
+        That quiesce on path with an intermediate link fails with ENOTDIR.
+        """
+
+        self._configure_subvolume()
+
+        self.mount_a.run_shell_payload("ln -s ../../.. _nogroup")
+        path = self.mount_a.cephfs_mntpt + "/_nogroup/" + self.QUIESCE_SUBVOLUME
+
+        J = self.fs.rank_tell(["quiesce", "path", path, '--wait'])
+        log.debug(f"{J}")
+        self.assertEqual(J['op']['result'], -20) # ENOTDIR: path_traverse: the intermediate link is not a directory
+
+    def test_quiesce_path_notsubvol(self):
+        """
+        That quiesce on a directory under a subvolume is valid.
+        """
+
+        self._configure_subvolume()
+
+        self.mount_a.run_shell_payload("mkdir dir")
+        path = self.mount_a.cephfs_mntpt + "/dir"
+
+        J = self.fs.rank_tell(["quiesce", "path", path, '--wait'])
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce(root=path)
+
+    def test_quiesce_path_regfile(self):
+        """
+        That quiesce on a regular file fails with ENOTDIR.
+        """
+
+        self._configure_subvolume()
+
+        self.mount_a.run_shell_payload("touch file")
+        path = self.mount_a.cephfs_mntpt + "/file"
+
+        J = self.fs.rank_tell(["quiesce", "path", path, '--wait'])
+        log.debug(f"{J}")
+        self.assertEqual(J['op']['result'], -20) # ENOTDIR
+
+    def test_quiesce_path_dup(self):
+        """
+        That two identical quiesce ops will result in one failing with
+        EINPROGRESS.
+        """
+
+        self._configure_subvolume()
+
+        op1 = self.fs.rank_tell(["quiesce", "path", self.subvolume])['op']
+        op1_reqid = self.reqid_tostr(op1['reqid'])
+        op2 = self.fs.rank_tell(["quiesce", "path", self.subvolume, '--wait'])['op']
+        op1 = self.fs.get_op(op1_reqid)['type_data'] # for possible dup result
+        log.debug(f"op1 = {op1}")
+        log.debug(f"op2 = {op2}")
+        self.assertIn(op1['flag_point'], (self.FP_QUIESCE_COMPLETE, 'cleaned up request'))
+        self.assertIn(op2['flag_point'], (self.FP_QUIESCE_COMPLETE, 'cleaned up request'))
+        self.assertTrue(op1['result'] == -115 or op2['result'] == -115) # EINPROGRESS
+
+    def test_quiesce_blocked(self):
+        """
+        That a file with ceph.quiesce.block is not quiesced.
+        """
+
+        self._configure_subvolume()
+
+        self.mount_a.run_shell_payload("touch file")
+        self.mount_a.setfattr("file", "ceph.quiesce.block", "1")
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume, '--wait'])
+        log.debug(f"{J}")
+        self.assertEqual(J['op']['result'], 0)
+        self.assertEqual(J['state']['inodes_blocked'], 1)
+        self._verify_quiesce(root=self.subvolume)
+
+    def test_quiesce_slow(self):
+        """
+        That a subvolume is quiesced when artificially slowed down.
+        """
+
+        self.config_set('mds', 'mds_cache_quiesce_delay', '2000')
+        self._configure_subvolume()
+        self._client_background_workload()
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        log.debug(f"{J}")
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce(root=self.subvolume)
+
+    # TODO test lookup leaf file/dir after quiesce
+    # TODO ditto path_traverse
+
+class TestQuiesceMultiRank(QuiesceTestCase):
+    """
+    Tests for quiescing subvolumes on multiple ranks.
+    """
+
+    MDSS_REQUIRED = 2
+
+    CLIENT_WORKLOAD = """
+        set -ex
+        for ((i = 0; i < 10; ++i)); do
+            (
+                pushd `mktemp -d -p .`
+                touch file
+                sleep 5 # for export
+                cp -a /usr .
+                popd
+            ) &
+        done
+        wait
+    """
+
+    def setUp(self):
+        super().setUp()
+        self.fs.set_max_mds(2)
+        status = self.fs.wait_for_daemons()
+        self.mds_map = self.fs.get_mds_map(status=status)
+        self.ranks = list(range(self.mds_map['max_mds']))
+
+    def test_quiesce_path_splitauth(self):
+        """
+        That quiesce fails (by default) if auth is split on a path.
+        """
+
+        self._configure_subvolume()
+        self.mount_a.setfattr(".", "ceph.dir.pin.distributed", "1")
+        self._client_background_workload()
+        self._wait_distributed_subtrees(2*2, rank="all", path=self.mntpnt)
+
+        op = self.fs.rank_tell(["quiesce", "path", self.subvolume, '--wait'], rank=0)['op']
+        self.assertEqual(op['result'], -1) # EPERM
+
+
+    def test_quiesce_path_multirank(self):
+        """
+        That quiesce may complete with two ranks and a basic workload.
+        """
+
+        self._configure_subvolume()
+        self.mount_a.setfattr(".", "ceph.dir.pin.distributed", "1")
+        self._client_background_workload()
+        self._wait_distributed_subtrees(2*2, rank="all", path=self.mntpnt)
+
+        sleep(secrets.randbelow(30)+10)
+
+        p = self.mount_a.run_shell_payload("ls", stdout=StringIO())
+        dirs = p.stdout.getvalue().strip().split()
+
+        ops = []
+        for d in dirs:
+            path = os.path.join(self.mntpnt, d)
+            for r in self.ranks:
+                op = self.fs.rank_tell(["quiesce", "path", path], rank=r)['op']
+                reqid = self.reqid_tostr(op['reqid'])
+                log.info(f"created {reqid}")
+                ops.append((r, op, path))
+        for rank, op, path in ops:
+            reqid = self.reqid_tostr(op['reqid'])
+            log.debug(f"waiting for ({rank}, {reqid})")
+            op = self._wait_for_quiesce_complete(reqid, rank=rank, path=path)
+        # FIXME _verify_quiesce needs adjustment for multiple quiesce
+        #for rank, op, path in ops:
+        #    self._verify_quiesce(root=path, rank=rank)
+
+    # TODO: test for quiesce_counter
+
+class TestQuiesceSplitAuth(QuiesceTestCase):
+    """
+    Tests for quiescing subvolumes on multiple ranks with split auth.
+    """
+
+    MDSS_REQUIRED = 2
+
+    CLIENT_WORKLOAD = """
+        set -ex
+        for ((i = 0; i < 10; ++i)); do
+            (
+                pushd `mktemp -d -p .`
+                touch file
+                sleep 5 # for export
+                cp -a /usr .
+                popd
+            ) &
+        done
+        wait
+    """
+
+    def setUp(self):
+        super().setUp()
+        self.config_set('mds', 'mds_export_ephemeral_random_max', '0.75')
+        self.config_set('mds', 'mds_cache_quiesce_splitauth', 'true')
+        self.fs.set_max_mds(2)
+        status = self.fs.wait_for_daemons()
+        self.mds_map = self.fs.get_mds_map(status=status)
+        self.ranks = list(range(self.mds_map['max_mds']))
+
+    @unittest.skip("splitauth is experimental")
+    def test_quiesce_path_multirank_exports(self):
+        """
+        That quiesce may complete with two ranks and a basic workload.
+        """
+
+        self.config_set('mds', 'mds_cache_quiesce_delay', '4000')
+        self._configure_subvolume()
+        self.mount_a.setfattr(".", "ceph.dir.pin.random", "0.5")
+        self._client_background_workload()
+
+        sleep(2)
+
+        op0 = self.fs.rank_tell(["quiesce", "path", self.subvolume], rank=0)['op']
+        op1 = self.fs.rank_tell(["quiesce", "path", self.subvolume], rank=1)['op']
+        reqid0 = self.reqid_tostr(op0['reqid'])
+        reqid1 = self.reqid_tostr(op1['reqid'])
+        op0 = self._wait_for_quiesce_complete(reqid0, rank=0)
+        op1 = self._wait_for_quiesce_complete(reqid1, rank=1)
+        log.debug(f"op0 = {op0}")
+        log.debug(f"op1 = {op1}")
+        self._verify_quiesce(rank=0)
+        self._verify_quiesce(rank=1)

From 51a04479726776c2852159fb9cfe28e60fe74d54 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 13 Feb 2024 12:37:07 -0500
Subject: [PATCH 2381/2492] qa: run quiesce unit tests in fs:functional

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

Fixes: https://tracker.ceph.com/issues/63664
---
 qa/suites/fs/functional/tasks/quiesce.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 qa/suites/fs/functional/tasks/quiesce.yaml

diff --git a/qa/suites/fs/functional/tasks/quiesce.yaml b/qa/suites/fs/functional/tasks/quiesce.yaml
new file mode 100644
index 000000000000..117ed7362c6b
--- /dev/null
+++ b/qa/suites/fs/functional/tasks/quiesce.yaml
@@ -0,0 +1,9 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - slow request
+tasks:
+  - cephfs_test_runner:
+      fail_on_skip: false
+      modules:
+        - tasks.cephfs.test_quiesce

From 7fa8bc8b29f22f3cbe3a15c34f86003ed7c73088 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 27 Feb 2024 15:17:28 -0500
Subject: [PATCH 2382/2492] mds: quiescelock as local lock + cap masking

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/common/options/mds.yaml.in |   2 +-
 src/mds/CDir.cc                |   5 +
 src/mds/CInode.cc              |  43 ++++--
 src/mds/CInode.h               |   5 +-
 src/mds/Locker.cc              | 105 +++++++-------
 src/mds/Locker.h               |   2 +-
 src/mds/MDCache.cc             | 245 ++++++++++++++++++---------------
 src/mds/MDCache.h              |  21 ++-
 src/mds/MDSCacheObject.h       |   2 +-
 src/mds/Migrator.cc            |  16 +--
 src/mds/Server.cc              |   3 +-
 src/mds/SimpleLock.h           |   2 +-
 12 files changed, 256 insertions(+), 195 deletions(-)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index d25e7b52edb0..e8116f275237 100644
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -87,7 +87,7 @@ options:
   type: bool
   level: advanced
   desc: allow recursive quiesce across auth boundaries
-  default: false
+  default: true
   services:
   - mds
   flags:
diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index c65dee271fc8..025e4d17b81f 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -639,6 +639,11 @@ void CDir::link_inode_work( CDentry *dn, CInode *in)
     in->snaprealm->adjust_parent();
   else if (in->is_any_caps())
     in->move_to_realm(inode->find_snaprealm());
+
+  bool is_quiesced = inode->is_quiesced();
+  if (is_quiesced) {
+    mdcache->add_quiesce(inode, in);
+  }
 }
 
 void CDir::unlink_inode(CDentry *dn, bool adjust_lru)
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index ed9a74f4435b..7b5bfc6133cb 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -255,9 +255,9 @@ ostream& operator<<(ostream& out, const CInode& in)
     out << " " << in.filelock;
   if (!in.xattrlock.is_sync_and_unlocked())
     out << " " << in.xattrlock;
-  if (!in.versionlock.is_sync_and_unlocked())  
+  if (in.versionlock.is_locked())
     out << " " << in.versionlock;
-  if (!in.quiescelock.is_sync_and_unlocked())
+  if (in.quiescelock.is_locked())
     out << " " << in.quiescelock;
 
   // hack: spit out crap on which clients have caps
@@ -2882,6 +2882,7 @@ bool CInode::freeze_inode(int auth_pin_allowance)
   const static int lock_types[] = {
     CEPH_LOCK_IVERSION, CEPH_LOCK_IFILE, CEPH_LOCK_IAUTH, CEPH_LOCK_ILINK, CEPH_LOCK_IDFT,
     CEPH_LOCK_IXATTR, CEPH_LOCK_ISNAP, CEPH_LOCK_INEST, CEPH_LOCK_IFLOCK, CEPH_LOCK_IPOLICY, 0
+    //TODO: add iquiesce here?
   };
   for (int i = 0; lock_types[i]; ++i) {
     auto lock = get_lock(lock_types[i]);
@@ -3532,13 +3533,23 @@ void CInode::export_client_caps(map<client_t,Capability::Export>& cl)
   }
 }
 
+int CInode::get_caps_quiesce_mask() const
+{
+  if (is_quiesced()) {
+    // what we allow to our clients for a quiesced node
+    return CEPH_CAP_ANY_RD | CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_BUFFER | CEPH_CAP_PIN;
+  } else {
+    return CEPH_CAP_ANY;
+  }
+}
+
   // caps allowed
 int CInode::get_caps_liked() const
 {
   if (is_dir())
-    return CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED;  // but not, say, FILE_RD|WR|WRBUFFER
+    return get_caps_quiesce_mask() & (CEPH_CAP_PIN | CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_SHARED); // but not, say, FILE_RD|WR|WRBUFFER
   else
-    return CEPH_CAP_ANY & ~CEPH_CAP_FILE_LAZYIO;
+    return get_caps_quiesce_mask() & (CEPH_CAP_ANY & ~CEPH_CAP_FILE_LAZYIO);
 }
 
 int CInode::get_caps_allowed_ever() const
@@ -3558,30 +3569,33 @@ int CInode::get_caps_allowed_ever() const
 
 int CInode::get_caps_allowed_by_type(int type) const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     CEPH_CAP_PIN |
     (filelock.gcaps_allowed(type) << filelock.get_cap_shift()) |
     (authlock.gcaps_allowed(type) << authlock.get_cap_shift()) |
     (xattrlock.gcaps_allowed(type) << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_allowed(type) << linklock.get_cap_shift());
+    (linklock.gcaps_allowed(type) << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_caps_careful() const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     (filelock.gcaps_careful() << filelock.get_cap_shift()) |
     (authlock.gcaps_careful() << authlock.get_cap_shift()) |
     (xattrlock.gcaps_careful() << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_careful() << linklock.get_cap_shift());
+    (linklock.gcaps_careful() << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_xlocker_mask(client_t client) const
 {
-  return 
+  return get_caps_quiesce_mask() & (
     (filelock.gcaps_xlocker_mask(client) << filelock.get_cap_shift()) |
     (authlock.gcaps_xlocker_mask(client) << authlock.get_cap_shift()) |
     (xattrlock.gcaps_xlocker_mask(client) << xattrlock.get_cap_shift()) |
-    (linklock.gcaps_xlocker_mask(client) << linklock.get_cap_shift());
+    (linklock.gcaps_xlocker_mask(client) << linklock.get_cap_shift())
+  );
 }
 
 int CInode::get_caps_allowed_for_client(Session *session, Capability *cap,
@@ -3679,6 +3693,14 @@ int CInode::get_caps_wanted(int *ploner, int *pother, int shift, int mask) const
       other |= p.second;
       //cout << " get_caps_wanted mds " << it->first << " " << cap_string(it->second) << endl;
     }
+
+  // we adjust wanted caps to prevent unnecessary lock transitions
+  // don't worry, when the quiesce lock is dropped
+  // the whole thing will get evaluated again, with a fixed mask
+  loner &= get_caps_quiesce_mask();
+  other &= get_caps_quiesce_mask();
+  w &= get_caps_quiesce_mask();
+
   if (ploner) *ploner = (loner >> shift) & mask;
   if (pother) *pother = (other >> shift) & mask;
   return (w >> shift) & mask;
@@ -4370,7 +4392,6 @@ void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool
     snaplock.mark_need_recover();
     flocklock.mark_need_recover();
     policylock.mark_need_recover();
-    quiescelock.mark_need_recover();
   }
   DECODE_FINISH(p);
 }
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 10fabc402d6e..06cde6589487 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -662,6 +662,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   bool is_file() const    { return get_inode()->is_file(); }
   bool is_symlink() const { return get_inode()->is_symlink(); }
   bool is_dir() const     { return get_inode()->is_dir(); }
+  bool is_quiesced() const { return quiescelock.is_xlocked(); }
 
   bool is_head() const { return last == CEPH_NOSNAP; }
 
@@ -867,6 +868,8 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   int count_nonstale_caps();
   bool multiple_nonstale_caps();
 
+  int get_caps_quiesce_mask() const;
+
   bool is_any_caps() { return !client_caps.empty(); }
   bool is_any_nonstale_caps() { return count_nonstale_caps(); }
 
@@ -1106,7 +1109,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
    * quiescelock.
    */
 
-  SimpleLock quiescelock; // FIXME not part of mempool
+  LocalLockC quiescelock; // FIXME not part of mempool
   LocalLockC versionlock; // FIXME not part of mempool
   SimpleLock authlock; // FIXME not part of mempool
   SimpleLock linklock; // FIXME not part of mempool
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 7b6b8266700a..16e138edded2 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -316,10 +316,12 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
           case CEPH_LOCK_IQUIESCE:
             break;
           default:
-	    CInode *in = static_cast<CInode*>(object);
-            if (need_quiescelock) {
+            CInode* in = static_cast<CInode*>(object);
+            if (need_quiescelock && (lock->get_cap_shift() > 0)) {
+              dout(20) << "need shared quiesce lock for " << p << " on " << SimpleLock::get_lock_type_name(t) << " of " << in << dendl;
               need_quiescelock = false;
-              lov.add_rdlock(&in->quiescelock, i + 1);
+              CInode *in = static_cast<CInode*>(object);
+              lov.add_wrlock(&in->quiescelock, i + 1);
             }
 	    if (!in->is_auth())
 	      continue;
@@ -347,19 +349,21 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 		 << " in case we need to request a scatter" << dendl;
 	mustpin.insert(object);
       }
-      if (need_quiescelock && is_inode_lock(t) && t != CEPH_LOCK_IQUIESCE) {
-	CInode *in = static_cast<CInode*>(object);
-        lov.add_rdlock(&in->quiescelock, i + 1);
+      if (need_quiescelock && (lock->get_cap_shift() > 0)) {
+        dout(20) << "need shared quiesce lock for " << p << " on " << SimpleLock::get_lock_type_name(t) << " of " << object << dendl;
         need_quiescelock = false;
+        CInode *in = static_cast<CInode*>(object);
+        lov.add_wrlock(&in->quiescelock, i + 1);
       }
     } else if (p.is_remote_wrlock()) {
       dout(20) << " must remote_wrlock on mds." << p.wrlock_target << " "
 	       << *lock << " " << *object << dendl;
       mustpin.insert(object);
-      if (need_quiescelock && is_inode_lock(t) && t != CEPH_LOCK_IQUIESCE) {
-	CInode *in = static_cast<CInode*>(object);
-        lov.add_rdlock(&in->quiescelock, i + 1);
+      if (need_quiescelock && (lock->get_cap_shift() > 0)) {
+        dout(20) << "need shared quiesce lock for " << p << " on " << SimpleLock::get_lock_type_name(t) << " of " << object << dendl;
         need_quiescelock = false;
+        CInode *in = static_cast<CInode*>(object);
+        lov.add_wrlock(&in->quiescelock, i + 1);
       }
     } else if (p.is_rdlock()) {
       dout(20) << " must rdlock " << *lock << " " << *object << dendl;
@@ -371,33 +375,6 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 		 << " in case we need to request a rdlock" << dendl;
 	mustpin.insert(object);
       }
-
-      /* We treat rdlocks differently when adding the quiescelock. If the lock
-       * can be acquired immediately for reading without waiting
-       * (SimpleLock::can_rdlock), then skip adding the quiescelock. This is to
-       * allow some rdonly operations (like lookup) to proceed without blocking
-       * on the exclusively locked quiescelock. This is safe from deadlock (due
-       * to lock ordering) when Locker::acquire_locks is called more than once
-       * with different LockOpVectors for a given inode (already a dangerous
-       * thing to do) where there may be a wrlock/xlock in one set but not the
-       * other. The reason is simple: if Locker::acquire_locks ever adds the
-       * quiescelock, it is always the first lock to be acquired, and if it is
-       * xlocked, then all locks are dropped (s.f.
-       * Locker::handle_quiesce_failure). So adding the quiescelock can never
-       * contribute to deadlock.
-       */
-
-      if (need_quiescelock && !mdr->is_rdlocked(lock)) {
-        /* Can we get the lock without waiting? */
-        if (!lock->can_rdlock(client)) {
-          /* To prevent deadlock where an op holds a parent snaplock
-           * (Locker::try_rdlock_snap_layout), add quiescelock.
-           */
-	  CInode *in = static_cast<CInode*>(object);
-          lov.add_rdlock(&in->quiescelock, i + 1);
-          need_quiescelock = false;
-        }
-      }
     } else {
       ceph_assert(0 == "locker unknown lock operation");
     }
@@ -578,7 +555,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
         if (t == CEPH_LOCK_IQUIESCE) {
           handle_quiesce_failure(mdr, marker.message);
         } else {
-	  marker.message = "failed to xlock, waiting";
+          marker.message = "failed to xlock, waiting";
         }
 	goto out;
       }
@@ -593,6 +570,7 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 	    cancel_locking(mdr.get(), &issue_set);
 	  marker.message = "waiting for remote wrlocks";
 	  remote_wrlock_start(lock, p.wrlock_target, mdr);
+          marker.message = "failed to wrlock, dropping remote wrlock and waiting";
 	  goto out;
 	}
       }
@@ -604,17 +582,13 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
 	client_t _client = p.is_state_pin() ? lock->get_excl_client() : client;
 	if (p.is_remote_wrlock()) {
 	  // nowait if we have already gotten remote wrlock
+          ceph_assert(t != CEPH_LOCK_IQUIESCE);
 	  if (!wrlock_try(lock, mdr, _client)) {
 	    // can't take the wrlock because the scatter lock is gathering. need to
 	    // release the remote wrlock, so that the gathering process can finish.
 	    ceph_assert(it != mdr->locks.end());
 	    remote_wrlock_finish(it, mdr.get());
 	    remote_wrlock_start(lock, p.wrlock_target, mdr);
-            if (t == CEPH_LOCK_IQUIESCE) {
-              handle_quiesce_failure(mdr, marker.message);
-            } else {
-	      marker.message = "failed to wrlock, dropping remote wrlock and waiting";
-            }
 	    goto out;
 	  }
 	} else {
@@ -655,11 +629,8 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
       }
 
       if (!rdlock_start(lock, mdr)) {
-        if (t == CEPH_LOCK_IQUIESCE) {
-          handle_quiesce_failure(mdr, marker.message);
-        } else {
-	  marker.message = "failed to rdlock, waiting";
-        }
+        ceph_assert(t != CEPH_LOCK_IQUIESCE); // rdlock is undefined for LocalLock
+        marker.message = "failed to rdlock, waiting";
 	goto out;
       }
       dout(10) << " got rdlock on " << *lock << " " << *lock->get_parent() << dendl;
@@ -865,7 +836,7 @@ void Locker::drop_rdlocks_for_early_reply(MutationImpl *mut)
   issue_caps_set(need_issue);
 }
 
-void Locker::drop_rdlock(MutationImpl* mut, SimpleLock* what)
+void Locker::drop_lock(MutationImpl* mut, SimpleLock* what)
 {
   dout(20) << __func__ << ": " << *what << dendl;
 
@@ -873,9 +844,14 @@ void Locker::drop_rdlock(MutationImpl* mut, SimpleLock* what)
     auto* lock = it->lock;
     if (lock == what) {
       dout(20) << __func__ << ": found lock " << *lock << dendl;
-      ceph_assert(it->is_rdlock());
       bool ni = false;
-      rdlock_finish(it, mut, &ni);
+      if (it->is_xlock()) {
+        xlock_finish(it, mut, &ni);
+      } else if (it->is_wrlock()) {
+        wrlock_finish(it, mut, &ni);
+      } else if (it->is_rdlock()) {
+        rdlock_finish(it, mut, &ni);
+      }
       if (ni) {
         set<CInode*> need_issue;
         need_issue.insert(static_cast<CInode*>(lock->get_parent()));
@@ -1450,8 +1426,7 @@ bool Locker::eval(CInode *in, int mask, bool caps_imported)
     eval_any(&in->flocklock, &need_issue, &finishers, caps_imported);
   if (mask & CEPH_LOCK_IPOLICY)
     eval_any(&in->policylock, &need_issue, &finishers, caps_imported);
-  if (mask & CEPH_LOCK_IQUIESCE)
-    eval_any(&in->quiescelock, &need_issue, &finishers, caps_imported);
+  // LocalLocks should not be eval'd
 
   // drop loner?
   if (in->is_auth() && in->is_head() && in->get_wanted_loner() != in->get_loner()) {
@@ -1841,6 +1816,7 @@ void Locker::wrlock_force(SimpleLock *lock, MutationRef& mut)
   switch  (lock->get_type()) {
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       return local_wrlock_grab(static_cast<LocalLockC*>(lock), mut);
     default:
       break;
@@ -1892,6 +1868,7 @@ bool Locker::wrlock_start(const MutationImpl::LockOp &op, const MDRequestRef& mu
   switch  (lock->get_type()) {
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       return local_wrlock_start(static_cast<LocalLockC*>(lock), mut);
     default:
       break;
@@ -1957,6 +1934,7 @@ void Locker::wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
   switch  (lock->get_type()) {
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       return local_wrlock_finish(it, mut);
     default:
       break;
@@ -2045,6 +2023,7 @@ bool Locker::xlock_start(SimpleLock *lock, const MDRequestRef& mut)
   switch  (lock->get_type()) {
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       return local_xlock_start(static_cast<LocalLockC*>(lock), mut);
     default:
       break;
@@ -2171,6 +2150,7 @@ void Locker::xlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *m
   switch  (lock->get_type()) {
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       return local_xlock_finish(it, mut);
     default:
       break;
@@ -3965,7 +3945,7 @@ bool Locker::_do_cap_update(CInode *in, Capability *cap,
   if (in->is_file()) {
     bool forced_change_max = false;
     dout(20) << "inode is file" << dendl;
-    if (cap && ((cap->issued() | cap->wanted()) & CEPH_CAP_ANY_FILE_WR)) {
+    if (cap && ((cap->issued() | cap->wanted()) & CEPH_CAP_ANY_FILE_WR & in->get_caps_quiesce_mask())) {
       dout(20) << "client has write caps; m->get_max_size="
                << m->get_max_size() << "; old_max=" << old_max << dendl;
       if (m->get_max_size() > new_max) {
@@ -5481,7 +5461,6 @@ void Locker::local_wrlock_grab(LocalLockC *lock, MutationRef& mut)
   dout(7) << "local_wrlock_grab  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
   
-  ceph_assert(lock->get_parent()->is_auth());
   ceph_assert(lock->can_wrlock());
   lock->get_wrlock(mut->get_client());
 
@@ -5494,7 +5473,6 @@ bool Locker::local_wrlock_start(LocalLockC *lock, const MDRequestRef& mut)
   dout(7) << "local_wrlock_start  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
   
-  ceph_assert(lock->get_parent()->is_auth());
   if (lock->can_wrlock()) {
     lock->get_wrlock(mut->get_client());
     auto it = mut->emplace_lock(lock, MutationImpl::LockOp::WRLOCK);
@@ -5526,7 +5504,9 @@ bool Locker::local_xlock_start(LocalLockC *lock, const MDRequestRef& mut)
   dout(7) << "local_xlock_start  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
   
-  ceph_assert(lock->get_parent()->is_auth());
+  if (lock->is_cached()) {
+    invalidate_lock_caches(lock);
+  }
   if (!lock->can_xlock_local()) {
     lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
     return false;
@@ -5546,9 +5526,18 @@ void Locker::local_xlock_finish(const MutationImpl::lock_iterator& it, MutationI
   lock->put_xlock();
   mut->locks.erase(it);
 
-  lock->finish_waiters(SimpleLock::WAIT_STABLE | 
-		       SimpleLock::WAIT_WR | 
+  lock->finish_waiters(SimpleLock::WAIT_STABLE |
+		       SimpleLock::WAIT_WR |
 		       SimpleLock::WAIT_RD);
+
+  /* Evaluate locks only after finishing waiters so blocked requests have a
+   * chance to acquire ifile/ixattr/etc. before issuing back caps.
+   */
+  if (lock->get_type() == CEPH_LOCK_IQUIESCE) {
+    auto in = static_cast<CInode*>(lock->get_parent());
+    // reevaluate everything related related to caps
+    eval(in, CEPH_LOCK_IFILE | CEPH_LOCK_IAUTH | CEPH_LOCK_ILINK | CEPH_LOCK_IXATTR, true);
+  }
 }
 
 
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index cfc0d9ace9a1..0a500f09be15 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -67,7 +67,7 @@ class Locker {
   void set_xlocks_done(MutationImpl *mut, bool skip_dentry=false);
   void drop_non_rdlocks(MutationImpl *mut, std::set<CInode*> *pneed_issue=0);
   void drop_rdlocks_for_early_reply(MutationImpl *mut);
-  void drop_rdlock(MutationImpl* mut, SimpleLock* what);
+  void drop_lock(MutationImpl* mut, SimpleLock* what);
   void drop_locks_for_fragment_unfreeze(MutationImpl *mut);
 
   int get_cap_bit_for_lock_cache(int op);
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index a6df73510692..e21d2d4d0ecd 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -17,6 +17,7 @@
 #include <string>
 #include <string_view>
 #include <map>
+#include <memory>
 
 #include "MDCache.h"
 #include "MDSRank.h"
@@ -116,6 +117,14 @@ class MDCacheLogContext : public virtual MDSLogContextBase {
   explicit MDCacheLogContext(MDCache *mdc_) : mdcache(mdc_) {}
 };
 
+struct QuiesceInodeState {
+  MDRequestRef qrmdr;
+  std::shared_ptr<MDCache::QuiesceStatistics> qs;
+  std::chrono::milliseconds delay = 0ms;
+  bool splitauth = false;
+};
+using QuiesceInodeStateRef = std::shared_ptr<QuiesceInodeState>;
+
 MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   mds(m),
   open_file_table(m),
@@ -9787,6 +9796,10 @@ void MDCache::request_forward(const MDRequestRef& mdr, mds_rank_t who, int port)
 
 void MDCache::dispatch_request(const MDRequestRef& mdr)
 {
+  if (mdr->dead) {
+    dout(20) << __func__ << ": dead " << *mdr << dendl;
+    return;
+  }
   if (mdr->client_request) {
     mds->server->dispatch_client_request(mdr);
   } else if (mdr->peer_request) {
@@ -9905,24 +9918,28 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
       mdr->clear_ambiguous_auth();
     if (!mdr->more()->waiting_for_finish.empty())
       mds->queue_waiters(mdr->more()->waiting_for_finish);
+    uint64_t count = 0;
     for (auto& [in, reqid] : mdr->more()->quiesce_ops) {
       if (auto it = active_requests.find(reqid); it != active_requests.end()) {
         auto qimdr = it->second;
         dout(20) << "killing quiesce op " << *qimdr << dendl;
         request_kill(qimdr);
+        if (!(++count % mds->heartbeat_reset_grace())) {
+          mds->heartbeat_reset();
+        }
       }
     }
   }
 
-  if (mdr->internal_op == CEPH_MDS_OP_QUIESCE_PATH) {
-    /* This construction is obviously not performant but it's rarely done and only for subvolumes */
-    for (auto it = quiesced_subvolumes.begin(); it != quiesced_subvolumes.end();) {
-      if (it->second == mdr) {
-        it = quiesced_subvolumes.erase(it);
-      } else {
-        ++it;
-      }
+  switch(mdr->internal_op) {
+    case CEPH_MDS_OP_QUIESCE_INODE: {
+      auto* qisp = static_cast<QuiesceInodeStateRef*>(mdr->internal_op_private);
+      delete qisp;
+      mdr->internal_op_private = nullptr;
+      break;
     }
+    default:
+      break;
   }
 
   request_drop_locks(mdr);
@@ -13518,27 +13535,28 @@ void MDCache::clear_dirty_bits_for_stray(CInode* diri) {
 
 void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
 {
-  if (mdr->killed) {
-    dout(20) << __func__ << " " << *mdr << " not dispatching killed " << *mdr << dendl;
-    return;
-  } else if (mdr->internal_op_finish == nullptr) {
+  if (mdr->internal_op_finish == nullptr) {
     dout(20) << __func__ << " " << *mdr << " already finished quiesce" << dendl;
     return;
   }
 
-  auto* qfinisher = static_cast<C_MDS_QuiescePath*>(mdr->internal_op_private);
-  auto delay = qfinisher->delay;
-  auto splitauth = qfinisher->splitauth;
-  auto& qs = *qfinisher->qs;
-  auto qrmdr = qfinisher->mdr;
+  auto& qis = *static_cast<QuiesceInodeStateRef*>(mdr->internal_op_private);
+  auto& qrmdr = qis->qrmdr;
+  auto& qops = qrmdr->more()->quiesce_ops;
+  auto& qs = *qis->qs;
+  auto& delay = qis->delay;
+  auto& splitauth = qis->splitauth;
 
   CInode *in = get_inode(mdr->get_filepath().get_ino());
   if (in == nullptr) {
-    qs.add_failed(mdr, -CEPHFS_ENOENT);
-    mds->server->respond_to_request(mdr, -CEPHFS_ENOENT);
+    /* It has been trimmed from cache before we could acquire locks/pins, complete quietly. */
+    qops.erase(in); // allow a future try if it comes back into cache
+    qs.inc_inodes_dropped();
+    mds->server->respond_to_request(mdr, 0);
     return;
   }
-  const bool is_root = (mdr->get_filepath().get_ino() == mdr->get_filepath2().get_ino());
+
+  [[maybe_unused]] const bool is_root = (mdr->get_filepath().get_ino() == mdr->get_filepath2().get_ino());
 
   dout(20) << __func__ << " " << *mdr << " quiescing " << *in << dendl;
 
@@ -13552,77 +13570,69 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
   }
   quiesce_counter.hit();
 
-  {
-    /* Acquire authpins on `in` to prevent migrations after this rank considers
-     * it (and its children) quiesced.
-     */
-
-    MutationImpl::LockOpVec lov;
-    if (!mds->locker->acquire_locks(mdr, lov, nullptr, {in}, false, true)) {
-      return;
-    }
-  }
-
-  /* TODO: Consider:
+  /* Acquire cap-related locks. This pushes the lock state in the right
+   * direction.
+   *
+   * The quiescelock, once held exclusively, prevents future operations from
+   * acquiring the cap-related locks. It also prevents issuance of caps to
+   * clients.
    *
-   *  rank0 is auth for /foo
-   *  rank1 quiesces /foo with no dirents in cache (and stops)
-   *  rank0 begins quiescing /foo
-   *  rank0 exports a dirfrag of /foo/bar to rank1 (/foo/bar is not authpinned by rank1 nor by rank0 (yet))
-   *  rank1 discovers relevant paths in /foo/bar
-   *  rank1 now has /foo/bar in cache and may issue caps / execute operations
+   * Because we're dropping the cap-related locks, other MDS can still complete
+   * operations requiring those locks or issue caps. This protocol necessarily
+   * requires that all ranks cooperatively quiesce the same tree. This is
+   * effected through the QuiesceDB.
    *
-   * The solution is probably to have rank1 mark /foo has STATE_QUIESCED and reject export ops from rank0.
+   * The xlock on the quiescelock is important to prevent future requests from
+   * blocking on other inode locks while holding path traversal locks.  See dev
+   * doc doc/dev/mds_internals/quiesce.rst for more details.
    */
 
-  if (in->is_auth()) {
-    /* Acquire rdlocks on anything which prevents writing.
-     *
-     * Because files are treated specially allowing multiple reader/writers, we
-     * need an xlock here to recall all write caps. This unfortunately means
-     * there can be no readers.
-     *
-     * The xlock on the quiescelock is important to prevent future requests
-     * from blocking on other inode locks while holding path traversal locks.
-     * See dev doc doc/dev/mds_internals/quiesce.rst for more details.
-     */
+  ceph_assert(in->is_auth() || splitauth);
+
+  if (in->is_quiesced()) {
+    auto qimdr = get_quiesce_inode_op(in);
+    if (qimdr != mdr) {
+      dout(5) << __func__ << ": already quiesced by " << *qimdr << dendl;
+      qs.add_failed(mdr, -CEPHFS_EINPROGRESS);
+      mds->server->respond_to_request(mdr, -CEPHFS_EINPROGRESS);
+      return;
+    }
+  }
 
+  if (!(mdr->locking_state & MutationImpl::ALL_LOCKED)) {
     MutationImpl::LockOpVec lov;
     lov.add_rdlock(&in->authlock);
-    lov.add_rdlock(&in->dirfragtreelock);
     lov.add_rdlock(&in->filelock);
     lov.add_rdlock(&in->linklock);
-    lov.add_rdlock(&in->nestlock);
-    lov.add_rdlock(&in->policylock);
-    // N.B.: NO xlock/wrlock on quiescelock; we need to allow access to mksnap/lookup
-    // This is an unfortunate inconsistency. It may be possible to circumvent
-    // this issue by having those ops acquire the quiscelock only if necessary.
-    if (is_root) {
-      lov.add_rdlock(&in->quiescelock);
-    } else {
-      lov.add_xlock(&in->quiescelock); /* !! */
-    }
-    lov.add_rdlock(&in->snaplock);
+    lov.add_xlock(&in->quiescelock); /* !! */
     lov.add_rdlock(&in->xattrlock);
     if (!mds->locker->acquire_locks(mdr, lov, nullptr, {in}, false, true)) {
       return;
     }
-  } else if (!splitauth) {
-    dout(5) << "auth is split and splitauth is false: " << *in << dendl;
-    qs.add_failed(mdr, -CEPHFS_EPERM);
-    mds->server->respond_to_request(mdr, -CEPHFS_EPERM);
-    return;
-  }
+    mdr->locking_state |= MutationImpl::ALL_LOCKED;
+
+    if (in->get_projected_inode()->get_quiesce_block()) {
+      dout(10) << __func__ << " quiesce is blocked for this inode; dropping locks!" << dendl;
+      mdr->mark_event("quiesce blocked");
+      mds->locker->drop_locks(mdr.get());
+      /* keep authpins! */
+      qs.inc_inodes_blocked();
+      auto* c = mdr->internal_op_finish;
+      mdr->internal_op_finish = nullptr; // prevent ::request_kill recursion
+      c->complete(0);
+      return;
+    }
 
-  if (in->get_projected_inode()->get_quiesce_block()) {
-    dout(10) << __func__ << " quiesce is blocked for this inode; dropping locks!" << dendl;
-    mdr->mark_event("quiesce blocked");
-    mds->locker->drop_locks(mdr.get());
-    /* keep authpins! */
-    qs.inc_inodes_blocked();
-    mdr->internal_op_finish->complete(0);
-    mdr->internal_op_finish = nullptr;
-    return;
+    if (splitauth) {
+      /* Once we have the queiscelock, we no longer need these locks.  However,
+       * if splitauth==false, the replicas do not try quiescing so we must keep
+       * them locked.
+       */
+      mds->locker->drop_lock(mdr.get(), &in->authlock);
+      mds->locker->drop_lock(mdr.get(), &in->filelock);
+      mds->locker->drop_lock(mdr.get(), &in->linklock);
+      mds->locker->drop_lock(mdr.get(), &in->xattrlock);
+    }
   }
 
   if (in->is_dir()) {
@@ -13635,7 +13645,6 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
       }
     }
     MDSGatherBuilder gather(g_ceph_context, new C_MDS_RetryRequest(this, mdr));
-    auto& qops = qrmdr->more()->quiesce_ops;
     for (auto& dir : in->get_dirfrags()) {
       for (auto& [dnk, dn] : *dir) {
         auto* in = dn->get_projected_inode();
@@ -13645,18 +13654,14 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
 
         if (auto it = qops.find(in); it != qops.end()) {
           dout(25) << __func__ << ": existing quiesce metareqid: "  << it->second << dendl;
-          if (auto reqit = active_requests.find(it->second); reqit != active_requests.end()) {
-            auto& qimdr = reqit->second;
-            dout(25) << __func__ << ": found in-progress " << qimdr << dendl;
-            continue;
-          }
+          continue;
         }
         dout(10) << __func__ << ": scheduling op to quiesce " << *in << dendl;
 
         MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
         qimdr->set_filepath(filepath(in->ino()));
         qimdr->internal_op_finish = gather.new_sub();
-        qimdr->internal_op_private = qfinisher;
+        qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
         qops[in] = qimdr->reqid;
         qs.inc_inodes();
         if (delay > 0ms) {
@@ -13687,19 +13692,43 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
   }
 
   qs.inc_inodes_quiesced();
-  mdr->internal_op_finish->complete(0);
-  mdr->internal_op_finish = nullptr;
+  auto* c = mdr->internal_op_finish;
+  mdr->internal_op_finish = nullptr; // prevent ::request_kill recursion
+  c->complete(0);
 
   /* do not respond/complete so locks are not lost, parent request will complete */
 }
 
-void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
+void MDCache::add_quiesce(CInode* parent, CInode* in)
 {
-  if (mdr->killed) {
-    dout(20) << __func__ << " not dispatching killed " << *mdr << dendl;
+  ceph_assert(parent->is_quiesced());
+  auto mdr = get_quiesce_inode_op(parent);
+
+  auto& qis = *static_cast<QuiesceInodeStateRef*>(mdr->internal_op_private);
+  auto& qrmdr = qis->qrmdr;
+  auto& qs = *qis->qs;
+  auto& qops = qrmdr->more()->quiesce_ops;
+
+  if (auto it = qops.find(in); it != qops.end()) {
+    dout(25) << __func__ << ": existing quiesce metareqid: "  << it->second << dendl;
     return;
   }
+  dout(10) << __func__ << ": scheduling op to quiesce " << *in << dendl;
 
+  MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
+  qimdr->set_filepath(filepath(in->ino()));
+  qimdr->internal_op_finish = new LambdaContext([](int r) {});
+  qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
+  qops[in] = qimdr->reqid;
+  qs.inc_inodes();
+  dispatch_request(qimdr);
+  if (!(qs.inc_heartbeat_count() % mds->heartbeat_reset_grace())) {
+    mds->heartbeat_reset();
+  }
+}
+
+void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
+{
   if (!mds->is_active()) {
     dout(20) << __func__ << " is not active!" << dendl;
     mds->server->respond_to_request(mdr, -CEPHFS_EAGAIN);
@@ -13712,8 +13741,11 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
 
   C_MDS_QuiescePath* qfinisher = static_cast<C_MDS_QuiescePath*>(mdr->internal_op_finish);
   auto& qs = *qfinisher->qs;
-  auto delay = qfinisher->delay = g_conf().get_val<std::chrono::milliseconds>("mds_cache_quiesce_delay");
-  auto splitauth = qfinisher->splitauth = g_conf().get_val<bool>("mds_cache_quiesce_splitauth");
+  auto delay = g_conf().get_val<std::chrono::milliseconds>("mds_cache_quiesce_delay");
+  auto splitauth = g_conf().get_val<bool>("mds_cache_quiesce_splitauth");
+
+  QuiesceInodeStateRef qis = std::make_shared<QuiesceInodeState>();
+  *qis = {mdr, qfinisher->qs, delay, splitauth};
 
   CInode* diri = nullptr;
   CF_MDS_RetryRequestFactory cf(this, mdr, true);
@@ -13736,22 +13768,18 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
     return;
   }
 
-  if (auto [it, inserted] = quiesced_subvolumes.try_emplace(diri->ino(), mdr); !inserted) {
-    if (!it->second) {
-      it->second = mdr;
-    } else if (it->second != mdr) {
-      dout(5) << __func__ << ": quiesce operation already in flight: " << it->second << dendl;
-      mds->server->respond_to_request(mdr, -CEPHFS_EINPROGRESS);
-      return;
-    }
-  }
-
   qfinisher->mdr = mdr;
 
-  for (auto& [qimdr, rc] : qs.get_failed()) {
-    dout(5) << __func__ << ": op " << *qimdr << " failed with " << rc << "!" << dendl;
-    mds->server->respond_to_request(mdr, rc);
-    return;
+  {
+    int myrc = 0;
+    for (auto& [qimdr, rc] : qs.get_failed()) {
+      dout(5) << __func__ << ": op " << *qimdr << " failed with " << rc << "!" << dendl;
+      myrc = rc;
+    }
+    if (myrc) {
+      mds->server->respond_to_request(mdr, myrc);
+      return;
+    }
   }
 
   if (!diri->is_auth() && !splitauth) {
@@ -13762,7 +13790,7 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
     qimdr->set_filepath(filepath(diri->ino()));
     qimdr->set_filepath2(filepath(diri->ino())); /* is_root! */
     qimdr->internal_op_finish = new C_MDS_RetryRequest(this, mdr);
-    qimdr->internal_op_private = qfinisher;
+    qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
     qops[diri] = qimdr->reqid;
     qs.inc_inodes();
     if (delay > 0ms) {
@@ -13779,8 +13807,9 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
   }
 
   if (qfinisher) {
-    qfinisher->complete(0);
-    mdr->internal_op_finish = nullptr;
+    auto* c = mdr->internal_op_finish;
+    mdr->internal_op_finish = nullptr; // prevent ::request_kill recursion
+    c->complete(0);
   }
   mdr->result = 0;
 
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 92a6be6d2a08..963784ced038 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -543,6 +543,9 @@ class MDCache {
     void inc_inodes_blocked() {
       inodes_blocked++;
     }
+    void inc_inodes_dropped() {
+      inodes_dropped++;
+    }
     uint64_t get_inodes() const {
       return inodes;
     }
@@ -566,6 +569,7 @@ class MDCache {
       f->dump_unsigned("inodes", inodes);
       f->dump_unsigned("inodes_quiesced", inodes_quiesced);
       f->dump_unsigned("inodes_blocked", inodes_blocked);
+      f->dump_unsigned("inodes_dropped", inodes_dropped);
       f->open_array_section("failed");
       for (auto& [mdr, rc] : failed) {
         f->open_object_section("failure");
@@ -580,6 +584,7 @@ class MDCache {
     uint64_t inodes = 0;
     uint64_t inodes_quiesced = 0;
     uint64_t inodes_blocked = 0;
+    uint64_t inodes_dropped = 0;
     std::map<MDRequestRef, int> failed;
   };
   class C_MDS_QuiescePath : public MDSInternalContext {
@@ -602,13 +607,24 @@ class MDCache {
       }
     }
     std::shared_ptr<QuiesceStatistics> qs = std::make_shared<QuiesceStatistics>();
-    std::chrono::milliseconds delay = 0ms;
-    bool splitauth = false;
     MDCache *cache;
     MDRequestRef mdr;
     Context* finisher = nullptr;
   };
   MDRequestRef quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *f = nullptr, std::chrono::milliseconds delay = 0ms);
+  MDRequestRef get_quiesce_inode_op(CInode* in) {
+    if (in->is_quiesced()) {
+      auto mut = in->quiescelock.get_xlock_by();
+      ceph_assert(mut); /* that would be weird */
+      auto* mdr = dynamic_cast<MDRequestImpl*>(mut.get());
+      ceph_assert(mdr); /* also would be weird */
+      ceph_assert(mdr->internal_op == CEPH_MDS_OP_QUIESCE_INODE);
+      return MDRequestRef(mdr);
+    } else {
+      return MDRequestRef();
+    }
+  }
+  void add_quiesce(CInode* parent, CInode* in);
 
   void clean_open_file_lists();
   void dump_openfiles(Formatter *f);
@@ -1496,7 +1512,6 @@ class MDCache {
 
   uint64_t kill_shutdown_at = 0;
 
-  std::map<inodeno_t, MDRequestRef> quiesced_subvolumes;
   DecayCounter quiesce_counter;
   uint64_t quiesce_threshold;
   std::chrono::milliseconds quiesce_sleep;
diff --git a/src/mds/MDSCacheObject.h b/src/mds/MDSCacheObject.h
index 6fa40ea7f8d6..7bc820e4234b 100644
--- a/src/mds/MDSCacheObject.h
+++ b/src/mds/MDSCacheObject.h
@@ -294,7 +294,7 @@ class MDSCacheObject {
       /* always at the front */
       seq = 0;
     }
-    waiting.insert(std::pair<waiter_seq_t, waiter>(seq, waiter(mask, c)));
+    waiting.insert(std::pair<waiter_seq_t, waiter>(seq, waiter{mask, c}));
   }
   virtual void take_waiting(uint64_t mask, MDSContext::vec& ls) {
     take_waiting(waitmask_t(mask), ls);
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index dc898543d155..8478bbb7fc78 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -3198,6 +3198,14 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::const_iterator& blp,
 
   DECODE_FINISH(blp);
 
+  // add inode?
+  if (added) {
+    mdcache->add_inode(in);
+    dout(10) << "added " << *in << dendl;
+  } else {
+    dout(10) << "  had " << *in << dendl;
+  }
+
   // link before state  -- or not!  -sage
   if (dn->get_linkage()->get_inode() != in) {
     ceph_assert(!dn->get_linkage()->get_inode());
@@ -3207,14 +3215,6 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::const_iterator& blp,
   if (in->is_dir())
     dn->dir->pop_lru_subdirs.push_back(&in->item_pop_lru);
  
-  // add inode?
-  if (added) {
-    mdcache->add_inode(in);
-    dout(10) << "added " << *in << dendl;
-  } else {
-    dout(10) << "  had " << *in << dendl;
-  }
-
   if (in->get_inode()->is_dirty_rstat())
     in->mark_dirty_rstat();
 
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 04330cd308d1..aae51735d914 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -3077,7 +3077,7 @@ void Server::dispatch_peer_request(const MDRequestRef& mdr)
 	  break;
 	}
 
-        // don't add quiescelock, let the peer acquire that rdlock themselves
+        // don't add quiescelock, let the peer acquire that lock themselves
 	if (!mds->locker->acquire_locks(mdr, lov, nullptr, {}, false, true))
 	  return;
 	
@@ -4892,7 +4892,6 @@ void Server::handle_client_readdir(const MDRequestRef& mdr)
   }
 
   /* readdir can add dentries to cache: acquire the quiescelock */
-  lov.add_rdlock(&diri->quiescelock);
   lov.add_rdlock(&diri->filelock);
   lov.add_rdlock(&diri->dirfragtreelock);
 
diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index f0fef93e1c60..6f1d049ea0aa 100644
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -41,7 +41,6 @@ struct LockType {
   explicit LockType(int t) : type(t) {
     switch (type) {
     case CEPH_LOCK_DN:
-    case CEPH_LOCK_IQUIESCE:
     case CEPH_LOCK_IAUTH:
     case CEPH_LOCK_ILINK:
     case CEPH_LOCK_IXATTR:
@@ -59,6 +58,7 @@ struct LockType {
       break;
     case CEPH_LOCK_DVERSION:
     case CEPH_LOCK_IVERSION:
+    case CEPH_LOCK_IQUIESCE:
       sm = &sm_locallock;
       break;
     default:

From 7e75a9e96343b2acf8a2c77b71563dadbdefd37b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 5 Mar 2024 14:13:42 -0500
Subject: [PATCH 2383/2492] mds: remove quiescelock handling for SimpleLock
 type

This is no longer necessary with the conversion to LocalLock.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/CInode.cc   | 48 ++++++++++++++++-----------------------------
 src/mds/Migrator.cc |  5 -----
 2 files changed, 17 insertions(+), 36 deletions(-)

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 7b5bfc6133cb..d66c9d27a7d9 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2210,13 +2210,6 @@ void CInode::encode_lock_state(int type, bufferlist& bl)
   case CEPH_LOCK_IPOLICY:
     encode_lock_ipolicy(bl);
     break;
-
-  case CEPH_LOCK_IQUIESCE: {
-    ENCODE_START(1, 1, bl);
-    /* skeleton */
-    ENCODE_FINISH(bl);
-    break;
-  }
   
   default:
     ceph_abort();
@@ -2284,13 +2277,6 @@ void CInode::decode_lock_state(int type, const bufferlist& bl)
     decode_lock_ipolicy(p);
     break;
 
-  case CEPH_LOCK_IQUIESCE: {
-    DECODE_START(1, p);
-    /* skeleton */
-    DECODE_FINISH(p);
-    break;
-  }
-
   default:
     ceph_abort();
   }
@@ -2879,10 +2865,19 @@ bool CInode::freeze_inode(int auth_pin_allowance)
   if (!dir->lock_caches_with_auth_pins.empty())
     mdcache->mds->locker->invalidate_lock_caches(dir);
 
-  const static int lock_types[] = {
-    CEPH_LOCK_IVERSION, CEPH_LOCK_IFILE, CEPH_LOCK_IAUTH, CEPH_LOCK_ILINK, CEPH_LOCK_IDFT,
-    CEPH_LOCK_IXATTR, CEPH_LOCK_ISNAP, CEPH_LOCK_INEST, CEPH_LOCK_IFLOCK, CEPH_LOCK_IPOLICY, 0
-    //TODO: add iquiesce here?
+  static const int lock_types[] = {
+    CEPH_LOCK_IQUIESCE,
+    CEPH_LOCK_IVERSION,
+    CEPH_LOCK_IFILE,
+    CEPH_LOCK_IAUTH,
+    CEPH_LOCK_ILINK,
+    CEPH_LOCK_IDFT,
+    CEPH_LOCK_IXATTR,
+    CEPH_LOCK_ISNAP,
+    CEPH_LOCK_INEST,
+    CEPH_LOCK_IFLOCK,
+    CEPH_LOCK_IPOLICY,
+    0
   };
   for (int i = 0; lock_types[i]; ++i) {
     auto lock = get_lock(lock_types[i]);
@@ -4308,7 +4303,6 @@ void CInode::_encode_locks_full(bufferlist& bl)
   encode(flocklock, bl);
   encode(policylock, bl);
   encode(loner_cap, bl);
-  encode(quiescelock, bl);
 }
 void CInode::_decode_locks_full(bufferlist::const_iterator& p)
 {
@@ -4325,12 +4319,11 @@ void CInode::_decode_locks_full(bufferlist::const_iterator& p)
   decode(loner_cap, p);
   set_loner_cap(loner_cap);
   want_loner_cap = loner_cap;  // for now, we'll eval() shortly.
-  decode(quiescelock, p);
 }
 
 void CInode::_encode_locks_state_for_replica(bufferlist& bl, bool need_recover)
 {
-  ENCODE_START(2, 1, bl);
+  ENCODE_START(1, 1, bl);
   authlock.encode_state_for_replica(bl);
   linklock.encode_state_for_replica(bl);
   dirfragtreelock.encode_state_for_replica(bl);
@@ -4341,7 +4334,6 @@ void CInode::_encode_locks_state_for_replica(bufferlist& bl, bool need_recover)
   flocklock.encode_state_for_replica(bl);
   policylock.encode_state_for_replica(bl);
   encode(need_recover, bl);
-  quiescelock.encode_state_for_replica(bl);
   ENCODE_FINISH(bl);
 }
 
@@ -4357,12 +4349,11 @@ void CInode::_encode_locks_state_for_rejoin(bufferlist& bl, int rep)
   snaplock.encode_state_for_replica(bl);
   flocklock.encode_state_for_replica(bl);
   policylock.encode_state_for_replica(bl);
-  quiescelock.encode_state_for_replica(bl);
 }
 
 void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool is_new)
 {
-  DECODE_START(2, p);
+  DECODE_START(1, p);
   authlock.decode_state(p, is_new);
   linklock.decode_state(p, is_new);
   dirfragtreelock.decode_state(p, is_new);
@@ -4376,10 +4367,6 @@ void CInode::_decode_locks_state_for_replica(bufferlist::const_iterator& p, bool
   bool need_recover;
   decode(need_recover, p);
 
-  if (struct_v >= 2) {
-    quiescelock.decode_state(p, is_new);
-  }
-
   if (need_recover && is_new) {
     // Auth mds replicated this inode while it's recovering. Auth mds may take xlock on the lock
     // and change the object when replaying unsafe requests.
@@ -4407,7 +4394,6 @@ void CInode::_decode_locks_rejoin(bufferlist::const_iterator& p, MDSContext::vec
   snaplock.decode_state_rejoin(p, waiters, survivor);
   flocklock.decode_state_rejoin(p, waiters, survivor);
   policylock.decode_state_rejoin(p, waiters, survivor);
-  quiescelock.decode_state_rejoin(p, waiters, survivor);
 
   if (!dirfragtreelock.is_stable() && !dirfragtreelock.is_wrlocked())
     eval_locks.push_back(&dirfragtreelock);
@@ -4422,7 +4408,7 @@ void CInode::_decode_locks_rejoin(bufferlist::const_iterator& p, MDSContext::vec
 
 void CInode::encode_export(bufferlist& bl)
 {
-  ENCODE_START(6, 6, bl);
+  ENCODE_START(5, 4, bl);
   _encode_base(bl, mdcache->mds->mdsmap->get_up_features());
 
   encode(state, bl);
@@ -4473,7 +4459,7 @@ void CInode::finish_export()
 void CInode::decode_import(bufferlist::const_iterator& p,
 			   LogSegment *ls)
 {
-  DECODE_START(6, p);
+  DECODE_START(5, p);
 
   _decode_base(p);
 
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 8478bbb7fc78..d6b554c77681 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -1698,7 +1698,6 @@ void Migrator::finish_export_inode(CInode *in, mds_rank_t peer,
   in->snaplock.export_twiddle();
   in->flocklock.export_twiddle();
   in->policylock.export_twiddle();
-  in->quiescelock.export_twiddle();
   
   // mark auth
   ceph_assert(in->is_auth());
@@ -3246,10 +3245,6 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist::const_iterator& blp,
   if (in->policylock.is_stable() &&
       in->policylock.get_state() != LOCK_SYNC)
       mds->locker->try_eval(&in->policylock, NULL);
-
-  if (in->quiescelock.is_stable() &&
-      in->quiescelock.get_state() != LOCK_SYNC)
-      mds->locker->try_eval(&in->quiescelock, NULL);
 }
 
 void Migrator::decode_import_inode_caps(CInode *in, bool auth_cap,

From c90cc2d04bb2d6d4e558fe268bcd241ea7723880 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 7 Mar 2024 14:30:35 -0500
Subject: [PATCH 2384/2492] mds: dispatch quiesce_inode ops after dir traversal

Mostly to avoid the possibility of the CDir::items changing during iteration
when calling sub-ops.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index e21d2d4d0ecd..83fffabd5099 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13645,6 +13645,7 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
       }
     }
     MDSGatherBuilder gather(g_ceph_context, new C_MDS_RetryRequest(this, mdr));
+    std::vector<MDRequestRef> todispatch;
     for (auto& dir : in->get_dirfrags()) {
       for (auto& [dnk, dn] : *dir) {
         auto* in = dn->get_projected_inode();
@@ -13669,13 +13670,16 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
             cache->dispatch_request(qimdr);
           }));
         } else {
-          dispatch_request(qimdr);
-        }
-        if (!(qs.inc_heartbeat_count() % mds->heartbeat_reset_grace())) {
-          mds->heartbeat_reset();
+          todispatch.push_back(qimdr);
         }
       }
     }
+    for (auto& qimdr : todispatch) {
+      dispatch_request(qimdr);
+      if (!(qs.inc_heartbeat_count() % mds->heartbeat_reset_grace())) {
+        mds->heartbeat_reset();
+      }
+    }
     if (gather.has_subs()) {
       dout(20) << __func__ << ": waiting for sub-ops to gather" << dendl;
       gather.activate();

From 1fa9c25d70c666259c300fc95e54ec70e8fd35a5 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 7 Mar 2024 14:40:58 -0500
Subject: [PATCH 2385/2492] mds: use inodeno_t to track quiesce requests

If a CInode is removed from cache before the quiesce_inode request can process
it (and pin it in cache), a new CInode may be created with the same address.
That pointer still exists in MutationImpl::quiesce_ops and would prevent
issuing a quiesce_inode op for the new inode.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 38 +++++++++++++++++++++++---------------
 src/mds/Mutation.h |  2 +-
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 83fffabd5099..cd08d798dfba 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13547,16 +13547,18 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
   auto& delay = qis->delay;
   auto& splitauth = qis->splitauth;
 
-  CInode *in = get_inode(mdr->get_filepath().get_ino());
+  auto ino = mdr->get_filepath().get_ino();
+  CInode *in = get_inode(ino);
   if (in == nullptr) {
+    dout(20) << " failed to lookup " << ino << dendl;
     /* It has been trimmed from cache before we could acquire locks/pins, complete quietly. */
-    qops.erase(in); // allow a future try if it comes back into cache
+    qops.erase(ino); // allow a future try if it comes back into cache
     qs.inc_inodes_dropped();
     mds->server->respond_to_request(mdr, 0);
     return;
   }
 
-  [[maybe_unused]] const bool is_root = (mdr->get_filepath().get_ino() == mdr->get_filepath2().get_ino());
+  [[maybe_unused]] const bool is_root = (ino == mdr->get_filepath2().get_ino());
 
   dout(20) << __func__ << " " << *mdr << " quiescing " << *in << dendl;
 
@@ -13647,14 +13649,18 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
     MDSGatherBuilder gather(g_ceph_context, new C_MDS_RetryRequest(this, mdr));
     std::vector<MDRequestRef> todispatch;
     for (auto& dir : in->get_dirfrags()) {
+      dout(25) << " iterating " << *dir << dendl;
       for (auto& [dnk, dn] : *dir) {
+        dout(25) << " evaluating (" << dnk << ", " << *dn << ")" << dendl;
         auto* in = dn->get_projected_inode();
-        if (!in || !in->is_head()) {
+        if (!in) {
+          dout(25) << " skipping dentry: " << *dn << dendl;
           continue;
-        }
-
-        if (auto it = qops.find(in); it != qops.end()) {
-          dout(25) << __func__ << ": existing quiesce metareqid: "  << it->second << dendl;
+        } else if (!in->is_head()) {
+          dout(25) << " skipping non-head inode: " << *in << dendl;
+          continue;
+        } else if (auto it = qops.find(in->ino()); it != qops.end()) {
+          dout(25) << " existing quiesce metareqid: "  << it->second << dendl;
           continue;
         }
         dout(10) << __func__ << ": scheduling op to quiesce " << *in << dendl;
@@ -13663,7 +13669,7 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
         qimdr->set_filepath(filepath(in->ino()));
         qimdr->internal_op_finish = gather.new_sub();
         qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
-        qops[in] = qimdr->reqid;
+        qops[in->ino()] = qimdr->reqid;
         qs.inc_inodes();
         if (delay > 0ms) {
           mds->timer.add_event_after(delay, new LambdaContext([cache=this,qimdr](int r) {
@@ -13713,7 +13719,7 @@ void MDCache::add_quiesce(CInode* parent, CInode* in)
   auto& qs = *qis->qs;
   auto& qops = qrmdr->more()->quiesce_ops;
 
-  if (auto it = qops.find(in); it != qops.end()) {
+  if (auto it = qops.find(in->ino()); it != qops.end()) {
     dout(25) << __func__ << ": existing quiesce metareqid: "  << it->second << dendl;
     return;
   }
@@ -13723,7 +13729,7 @@ void MDCache::add_quiesce(CInode* parent, CInode* in)
   qimdr->set_filepath(filepath(in->ino()));
   qimdr->internal_op_finish = new LambdaContext([](int r) {});
   qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
-  qops[in] = qimdr->reqid;
+  qops[in->ino()] = qimdr->reqid;
   qs.inc_inodes();
   dispatch_request(qimdr);
   if (!(qs.inc_heartbeat_count() % mds->heartbeat_reset_grace())) {
@@ -13766,6 +13772,8 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
     return;
   }
 
+  auto dirino = diri->ino();
+
   if (!diri->is_dir()) {
     dout(5) << __func__ << ": file is not a directory" << dendl;
     mds->server->respond_to_request(mdr, -CEPHFS_ENOTDIR);
@@ -13789,13 +13797,13 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
   if (!diri->is_auth() && !splitauth) {
     dout(5) << __func__ << ": skipping recursive quiesce of path for non-auth inode" << dendl;
     mdr->mark_event("quiesce complete for non-auth tree");
-  } else if (auto& qops = mdr->more()->quiesce_ops; qops.count(diri) == 0) {
+  } else if (auto& qops = mdr->more()->quiesce_ops; qops.count(dirino) == 0) {
     MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
-    qimdr->set_filepath(filepath(diri->ino()));
-    qimdr->set_filepath2(filepath(diri->ino())); /* is_root! */
+    qimdr->set_filepath(filepath(dirino));
+    qimdr->set_filepath2(filepath(dirino)); /* is_root! */
     qimdr->internal_op_finish = new C_MDS_RetryRequest(this, mdr);
     qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
-    qops[diri] = qimdr->reqid;
+    qops[dirino] = qimdr->reqid;
     qs.inc_inodes();
     if (delay > 0ms) {
       mds->timer.add_event_after(delay, new LambdaContext([cache=this,qimdr](int r) {
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 4b9d0d7528c8..6a11144362ec 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -349,7 +349,7 @@ struct MDRequestImpl : public MutationImpl {
 
     MDSContext::vec waiting_for_finish;
 
-    std::map<CInode*, metareqid_t> quiesce_ops;
+    std::map<inodeno_t, metareqid_t> quiesce_ops;
 
     // export & fragment
     CDir* export_dir = nullptr;

From d86a5c2ab76b740da235aeb1f5a994a84bb3515f Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 2 Mar 2024 22:22:19 -0500
Subject: [PATCH 2386/2492] mds: print lock cache during invalidation

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc   |  4 ++++
 src/mds/Mutation.cc | 15 +++++++++++++++
 src/mds/Mutation.h  |  1 +
 3 files changed, 20 insertions(+)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 16e138edded2..19ead4c4647d 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -895,6 +895,8 @@ class C_MDL_DropCache : public LockerContext {
 
 void Locker::put_lock_cache(MDLockCache* lock_cache)
 {
+  dout(20) << __func__ << ": " << *lock_cache << dendl;
+
   ceph_assert(lock_cache->ref > 0);
   if (--lock_cache->ref > 0)
     return;
@@ -928,6 +930,8 @@ int Locker::get_cap_bit_for_lock_cache(int op)
 
 void Locker::invalidate_lock_cache(MDLockCache *lock_cache)
 {
+  dout(15) << __func__ << ": " << *lock_cache << dendl;
+
   ceph_assert(lock_cache->item_cap_lock_cache.is_on_list());
   if (lock_cache->invalidating) {
     ceph_assert(!lock_cache->client_cap);
diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index ecc455de08db..6ff87183914b 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -624,3 +624,18 @@ void MDLockCache::detach_dirfrags()
   }
   items_dir.reset();
 }
+
+void MDLockCache::print(std::ostream& out) const {
+  out << "MDLockCache(o=" << ceph_mds_op_name(opcode)
+      << " diri=" << diri->ino();
+  if (client_cap) {
+    out << " c=" << client_cap->get_client();
+  } else {
+    out << " c=(nil)";
+  }
+  out << " r=" << ref;
+  if (invalidating) {
+    out << " invalidating";
+  }
+  out << ")";
+}
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 6a11144362ec..7f985651e37d 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -538,6 +538,7 @@ struct MDLockCache : public MutationImpl {
     return dir_layout;
   }
 
+  void print(std::ostream& out) const;
   void attach_locks();
   void attach_dirfrags(std::vector<CDir*>&& dfv);
   void detach_locks();

From 772f04afdf87d25de8102b7f6aa7e0a242cd566e Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 27 Feb 2024 15:24:26 -0500
Subject: [PATCH 2387/2492] mds: avoid issuing exclusive caps to clients
 lacking w caps

Signed-off-by: Leonid Usov <leonid.usov@ibm.com>
---
 src/mds/Locker.cc | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 19ead4c4647d..300b94904e64 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2965,10 +2965,13 @@ bool Locker::check_inode_max_size(CInode *in, bool force_wrlock,
   } else if (!force_wrlock && !in->filelock.can_wrlock(in->get_loner())) {
     // lock?
     if (in->filelock.is_stable()) {
-      if (in->get_target_loner() >= 0)
-	file_excl(&in->filelock);
-      else
-	simple_lock(&in->filelock);
+      auto wanted = in->get_caps_wanted();
+      if (in->get_target_loner() >= 0 && (wanted & CEPH_CAP_ANY_FILE_WR)) {
+        dout(10) << "check_inode_max_size requesting file_excl for wanted caps " << ccap_string(wanted) << " " << *in << dendl;
+        file_excl(&in->filelock);
+      } else {
+        simple_lock(&in->filelock);
+      }
     }
     if (!in->filelock.can_wrlock(in->get_loner())) {
       dout(10) << "check_inode_max_size can't wrlock, waiting on " << *in << dendl;
@@ -5584,7 +5587,7 @@ void Locker::file_eval(ScatterLock *lock, bool *need_issue)
             << " other_issued=" << gcap_string(other_issued)
 	    << " xlocker_issued=" << gcap_string(xlocker_issued)
 	    << dendl;
-    if (!((loner_wanted|loner_issued) & (CEPH_CAP_GEXCL|CEPH_CAP_GWR|CEPH_CAP_GBUFFER)) ||
+    if (!((loner_wanted|loner_issued) & (CEPH_CAP_ANY_FILE_WR >> CEPH_CAP_SFILE)) ||
 	(other_wanted & (CEPH_CAP_GEXCL|CEPH_CAP_GWR|CEPH_CAP_GRD)) ||
 	(in->is_dir() && in->multiple_nonstale_caps())) {  // FIXME.. :/
       dout(20) << " should lose it" << dendl;
@@ -5617,7 +5620,7 @@ void Locker::file_eval(ScatterLock *lock, bool *need_issue)
 	   in->get_target_loner() >= 0 &&
 	   (in->is_dir() ?
 	    !in->has_subtree_or_exporting_dirfrag() :
-	    (wanted & (CEPH_CAP_GEXCL|CEPH_CAP_GWR|CEPH_CAP_GBUFFER)))) {
+	    (wanted & (CEPH_CAP_ANY_FILE_WR >> CEPH_CAP_SFILE)))) {
     dout(7) << "file_eval stable, bump to loner " << *lock
 	    << " on " << *lock->get_parent() << dendl;
     file_excl(lock, need_issue);

From 1ec6817cc082b379fba86bcd0a73ff950d7d1152 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 1 Mar 2024 22:01:48 -0500
Subject: [PATCH 2388/2492] mds: block import discover when parent directory
 inode is quiesced

This is to prevent two racing ranks quiescing some root from exporting a tree
under a completely quiesced directory (inode). The state of that imported tree
may take time to quiesce and cause the root to be QUIESCED before all inodes
under it are actually quiesced.

If a dirfrag to be imported is discovered before the parent is quiesced, then
the quiesce traversal will issue a quiesce_inode op normally for parent which
will attempt to authpin the parent. That will block if the export is still
in-progress (causing quiesce to wait for the export to finish or abort).

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc  | 35 ++++++++++++++++++++++++-----------
 src/mds/MDCache.h   |  1 +
 src/mds/Migrator.cc | 21 +++++++++++++++------
 3 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index cd08d798dfba..d62d93e16fd8 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8293,6 +8293,7 @@ int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
   bool rdlock_path = (flags & MDS_TRAVERSE_RDLOCK_PATH);
   bool xlock_dentry = (flags & MDS_TRAVERSE_XLOCK_DENTRY);
   bool rdlock_authlock = (flags & MDS_TRAVERSE_RDLOCK_AUTHLOCK);
+  bool forimport = (flags & MDS_TRAVERSE_IMPORT);
 
   if (forward)
     ceph_assert(mdr);  // forward requires a request
@@ -8403,11 +8404,17 @@ int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
         curdir = cur->get_or_open_dirfrag(this, fg);
       } else {
         // discover?
-	dout(10) << "traverse: need dirfrag " << fg << ", doing discover from " << *cur << dendl;
-	discover_path(cur, snapid, path.postfixpath(depth), cf.build(),
-		      path_locked);
-	if (mds->logger) mds->logger->inc(l_mds_traverse_discover);
-        return 1;
+        if (forimport && cur->is_quiesced()) {
+          /* block discover for import */
+          dout(5) << __func__ << ": blocking discover due to quiesced parent: " << *cur << dendl;
+          return -CEPHFS_EAGAIN;
+        } else {
+	  dout(10) << "traverse: need dirfrag " << fg << ", doing discover from " << *cur << dendl;
+	  discover_path(cur, snapid, path.postfixpath(depth), cf.build(),
+		        path_locked);
+	  if (mds->logger) mds->logger->inc(l_mds_traverse_discover);
+          return 1;
+        }
       }
     }
     ceph_assert(curdir);
@@ -8641,12 +8648,18 @@ int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
       }
 
       if (discover) {
-	dout(7) << "traverse: discover from " << path[depth] << " from " << *curdir << dendl;
-	discover_path(curdir, snapid, path.postfixpath(depth), cf.build(),
-		      path_locked);
-	if (mds->logger) mds->logger->inc(l_mds_traverse_discover);
-        return 1;
-      } 
+        if (forimport && cur->is_quiesced()) {
+          /* block discover for import */
+          dout(5) << __func__ << ": blocking discover due to quiesced parent: " << *cur << dendl;
+          return -CEPHFS_EAGAIN;
+        } else {
+	  dout(7) << "traverse: discover from " << path[depth] << " from " << *curdir << dendl;
+	  discover_path(curdir, snapid, path.postfixpath(depth), cf.build(),
+		        path_locked);
+	  if (mds->logger) mds->logger->inc(l_mds_traverse_discover);
+          return 1;
+        }
+      }
       if (forward) {
         // forward
         dout(7) << "traverse: not auth for " << path << " in " << *curdir << dendl;
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 963784ced038..7ac4f9562788 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -133,6 +133,7 @@ static const int MDS_TRAVERSE_XLOCK_DENTRY	= (1 << 8);
 static const int MDS_TRAVERSE_RDLOCK_AUTHLOCK	= (1 << 9);
 static const int MDS_TRAVERSE_CHECK_LOCKCACHE	= (1 << 10);
 static const int MDS_TRAVERSE_WANT_INODE	= (1 << 11);
+static const int MDS_TRAVERSE_IMPORT            = (1 << 12);
 
 
 // flags for predirty_journal_parents()
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index d6b554c77681..abb6dc4397b3 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -1223,7 +1223,7 @@ void Migrator::handle_export_discover_ack(const cref_t<MExportDirDiscoverAck> &m
       ceph_assert(g_conf()->mds_kill_export_at != 3);
 
     } else {
-      dout(7) << "peer failed to discover (not active?), canceling" << dendl;
+      dout(7) << "peer failed to discover (not active or quiesced), canceling" << dendl;
       export_try_cancel(dir, false);
     }
   }
@@ -2309,13 +2309,22 @@ void Migrator::handle_export_discover(const cref_t<MExportDirDiscover> &m, bool
     filepath fpath(m->get_path());
     vector<CDentry*> trace;
     MDRequestRef null_ref;
-    int r = mdcache->path_traverse(null_ref, cf, fpath,
-				   MDS_TRAVERSE_DISCOVER | MDS_TRAVERSE_PATH_LOCKED,
-				   &trace);
+    static constexpr int flags = 0
+       | MDS_TRAVERSE_DISCOVER
+       | MDS_TRAVERSE_PATH_LOCKED
+       | MDS_TRAVERSE_IMPORT;
+    int r = mdcache->path_traverse(null_ref, cf, fpath, flags, &trace);
     if (r > 0) return;
     if (r < 0) {
-      dout(7) << "failed to discover or not dir " << m->get_path() << ", NAK" << dendl;
-      ceph_abort();    // this shouldn't happen if the auth pins its path properly!!!!
+      if (r == -CEPHFS_EAGAIN) {
+        dout(5) << "blocking import during quiesce" << dendl;
+        import_reverse_discovering(df);
+        mds->send_message_mds(make_message<MExportDirDiscoverAck>(df, m->get_tid(), false), from);
+        return;
+      } else {
+        dout(7) << "failed to discover or not dir " << m->get_path() << ", NAK" << dendl;
+        ceph_abort();    // this shouldn't happen if the auth pins its path properly!!!!
+      }
     }
 
     ceph_abort(); // this shouldn't happen; the get_inode above would have succeeded.

From bddd3c72014dc7aa4bb4f16e4633111a0f2e0bb0 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 5 Mar 2024 12:19:42 -0500
Subject: [PATCH 2389/2492] mds: prevent new wrlocks on LocalLock if there
 exists any xlock waiter

Otherwise, an xlock waiter can become starved as a LocalLock supports multiple
writers.

Strictly speaking, a new lock state would be appropriate for this but we cheat
frequently with the LocalLock -- there is only one state. All transition checks
are already manually performed by the Locker.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/LocalLockC.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mds/LocalLockC.h b/src/mds/LocalLockC.h
index 4d26a1c3e142..a4df37bbcfed 100644
--- a/src/mds/LocalLockC.h
+++ b/src/mds/LocalLockC.h
@@ -34,7 +34,7 @@ class LocalLockC : public SimpleLock {
   }
 
   bool can_wrlock() const {
-    return !is_xlocked();
+    return !is_xlocked() && !is_waiter_for(SimpleLock::WAIT_XLOCK);
   }
   void get_wrlock(client_t client) {
     ceph_assert(can_wrlock());

From f813b8c511a50a7fab673c2b99587f00fb506323 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 4 Mar 2024 15:44:45 -0500
Subject: [PATCH 2390/2492] mds: use XLOCK_WAIT For local lock xlockers

This avoids waking up all waiters when only WAIT_XLOCK waiters should wake.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 300b94904e64..6a72f2d3e4dc 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -5486,7 +5486,7 @@ bool Locker::local_wrlock_start(LocalLockC *lock, const MDRequestRef& mut)
     ceph_assert(it->is_wrlock());
     return true;
   } else {
-    lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    lock->add_waiter(SimpleLock::WAIT_WR, new C_MDS_RetryRequest(mdcache, mut));
     return false;
   }
 }
@@ -5500,9 +5500,9 @@ void Locker::local_wrlock_finish(const MutationImpl::lock_iterator& it, Mutation
   lock->put_wrlock();
   mut->locks.erase(it);
   if (lock->get_num_wrlocks() == 0) {
-    lock->finish_waiters(SimpleLock::WAIT_STABLE |
-                         SimpleLock::WAIT_WR |
-                         SimpleLock::WAIT_RD);
+    /* wrlocks do not wait unless an xlocker is waiting */
+    ceph_assert(!lock->is_waiter_for(SimpleLock::WAIT_WR) || lock->is_waiter_for(SimpleLock::WAIT_XLOCK));
+    lock->finish_waiters(SimpleLock::WAIT_XLOCK);
   }
 }
 
@@ -5515,7 +5515,7 @@ bool Locker::local_xlock_start(LocalLockC *lock, const MDRequestRef& mut)
     invalidate_lock_caches(lock);
   }
   if (!lock->can_xlock_local()) {
-    lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mut));
+    lock->add_waiter(SimpleLock::WAIT_XLOCK, new C_MDS_RetryRequest(mdcache, mut));
     return false;
   }
 
@@ -5533,7 +5533,7 @@ void Locker::local_xlock_finish(const MutationImpl::lock_iterator& it, MutationI
   lock->put_xlock();
   mut->locks.erase(it);
 
-  lock->finish_waiters(SimpleLock::WAIT_STABLE |
+  lock->finish_waiters(SimpleLock::WAIT_XLOCK |
 		       SimpleLock::WAIT_WR |
 		       SimpleLock::WAIT_RD);
 

From 9b4a93f2b7b902e56a41f7e50fd21b8046134ef6 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 5 Mar 2024 12:27:00 -0500
Subject: [PATCH 2391/2492] mds: prevent new lock cache cons when invalidating
 an existing one

The previous scheme invalidated a lock cache and then immediately removed it
from its Capability list. The lock cache would eventually be deleted but a new
one could be constructed shortly after. The main reason for this is that simply
invalidating the lock cache does not drive a state change in the local locks
preventing new writers. This is mostly important for acquiring the quiescelock.

This commit also corrects a bug where a MDLockCache would be created for a
given opcode type (like create) when the capability does not have the issued
cap (CEPH_CAP_DIR_CREATE). The bug would not cause any negative side-effects
but would hold locks unnecessarily when only MDS ops (and not the client
executing ops asynchronously) are acquiring the locks.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/Locker.cc   | 77 +++++++++++++++++++++++++--------------------
 src/mds/Mutation.cc | 13 ++++++++
 src/mds/Mutation.h  | 13 +++++++-
 3 files changed, 68 insertions(+), 35 deletions(-)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 6a72f2d3e4dc..d0b5ac67c685 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -903,6 +903,10 @@ void Locker::put_lock_cache(MDLockCache* lock_cache)
 
   ceph_assert(lock_cache->invalidating);
 
+  /* The lock cache is only removed from the cap's list when the cap is removed
+   * or the lock cache is finally deleted.
+   */
+  lock_cache->item_cap_lock_cache.remove_myself();
   lock_cache->detach_locks();
 
   CInode *diri = lock_cache->get_dir_inode();
@@ -915,44 +919,37 @@ void Locker::put_lock_cache(MDLockCache* lock_cache)
   mds->queue_waiter(new C_MDL_DropCache(this, lock_cache));
 }
 
-int Locker::get_cap_bit_for_lock_cache(int op)
-{
-  switch(op) {
-    case CEPH_MDS_OP_CREATE:
-      return CEPH_CAP_DIR_CREATE;
-    case CEPH_MDS_OP_UNLINK:
-      return CEPH_CAP_DIR_UNLINK;
-    default:
-      ceph_assert(0 == "unsupported operation");
-      return 0;
-  }
-}
-
 void Locker::invalidate_lock_cache(MDLockCache *lock_cache)
 {
   dout(15) << __func__ << ": " << *lock_cache << dendl;
 
-  ceph_assert(lock_cache->item_cap_lock_cache.is_on_list());
-  if (lock_cache->invalidating) {
-    ceph_assert(!lock_cache->client_cap);
-  } else {
+  if (!lock_cache->invalidating) {
     lock_cache->invalidating = true;
     lock_cache->detach_dirfrags();
   }
 
   Capability *cap = lock_cache->client_cap;
   if (cap) {
-    int cap_bit = get_cap_bit_for_lock_cache(lock_cache->opcode);
+    int cap_bit = lock_cache->get_cap_bit();
     cap->clear_lock_cache_allowed(cap_bit);
-    if (cap->issued() & cap_bit)
+    if (cap->issued() & cap_bit) {
       issue_caps(lock_cache->get_dir_inode(), cap);
-    else
+    } else {
       cap = nullptr;
+    }
+  } else {
+    /* cap is removed but the lock_cache may not yet be 0 ref */
+    lock_cache->item_cap_lock_cache.remove_myself();
   }
-
   if (!cap) {
-    lock_cache->item_cap_lock_cache.remove_myself();
-    put_lock_cache(lock_cache);
+    /* the cap is removed or we lost relevant op rights */
+    if (lock_cache->cap_ref) {
+      put_lock_cache(lock_cache);
+      lock_cache->cap_ref = false;
+    }
+    /* N.B.: the lock cache may still be associated with the cap even when
+     * invalidated so a new lock cache is not created.
+     */
   }
 }
 
@@ -961,12 +958,10 @@ void Locker::eval_lock_caches(Capability *cap)
   for (auto p = cap->lock_caches.begin(); !p.end(); ) {
     MDLockCache *lock_cache = *p;
     ++p;
-    if (!lock_cache->invalidating)
-      continue;
-    int cap_bit = get_cap_bit_for_lock_cache(lock_cache->opcode);
+    int cap_bit = lock_cache->get_cap_bit();
     if (!(cap->issued() & cap_bit)) {
-      lock_cache->item_cap_lock_cache.remove_myself();
-      put_lock_cache(lock_cache);
+      dout(20) << __func__ << ": lost " << ccap_string(cap_bit) << " on " << *lock_cache << dendl;
+      invalidate_lock_cache(lock_cache);
     }
   }
 }
@@ -999,7 +994,7 @@ void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layou
 
   client_t client = mdr->get_client();
   int opcode = mdr->client_request->get_op();
-  dout(10) << "create_lock_cache for client." << client << "/" << ceph_mds_op_name(opcode)<< " on " << *diri << dendl;
+  dout(10) << __func__ << ": for client." << client << "/" << ceph_mds_op_name(opcode)<< " on " << *diri << dendl;
 
   if (!diri->is_auth()) {
     dout(10) << " dir inode is not auth, noop" << dendl;
@@ -1024,6 +1019,12 @@ void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layou
     }
   }
 
+  int cap_bit = MDLockCache::get_cap_bit_for_lock_cache(opcode);
+  if (!(cap->issued() & cap_bit)) {
+    dout(10) << " client cap lacks rights for lock cache: " << ccap_string(cap_bit) << dendl;
+    return;
+  }
+
   set<MDSCacheObject*> ancestors;
   for (CInode *in = diri; ; ) {
     CDentry *pdn = in->get_projected_parent_dn();
@@ -1081,7 +1082,7 @@ void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layou
   auto lock_cache = new MDLockCache(cap, opcode);
   if (dir_layout)
     lock_cache->set_dir_layout(*dir_layout);
-  cap->set_lock_cache_allowed(get_cap_bit_for_lock_cache(opcode));
+  cap->set_lock_cache_allowed(lock_cache->get_cap_bit());
 
   for (auto dir : dfv) {
     // prevent subtree migration
@@ -1135,8 +1136,10 @@ void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layou
   }
   lock_cache->attach_locks();
 
-  lock_cache->ref++;
+  dout(20) << __func__ << ": created " << *lock_cache << dendl;
+
   mdr->lock_cache = lock_cache;
+  lock_cache->ref++; /* for mdr */
 }
 
 bool Locker::find_and_attach_lock_cache(const MDRequestRef& mdr, CInode *diri)
@@ -1151,10 +1154,10 @@ bool Locker::find_and_attach_lock_cache(const MDRequestRef& mdr, CInode *diri)
   int opcode = mdr->client_request->get_op();
   for (auto p = cap->lock_caches.begin(); !p.end(); ++p) {
     MDLockCache *lock_cache = *p;
-    if (lock_cache->opcode == opcode) {
-      dout(10) << "found lock cache for " << ceph_mds_op_name(opcode) << " on " << *diri << dendl;
+    if (lock_cache->opcode == opcode && lock_cache->attachable()) {
+      dout(10) << "found lock cache " << *lock_cache << " on " << *diri << dendl;
       mdr->lock_cache = lock_cache;
-      mdr->lock_cache->ref++;
+      mdr->lock_cache->ref++; /* for mdr */
       return true;
     }
   }
@@ -2454,6 +2457,12 @@ int Locker::get_allowed_caps(CInode *in, Capability *cap,
 
 int Locker::issue_caps(CInode *in, Capability *only_cap)
 {
+  dout(20) << __func__ << ": " << *in;
+  if (only_cap) {
+    *_dout << " for " << only_cap->get_client();
+  }
+  *_dout << dendl;
+
   // count conflicts with
   int nissued = 0;
   int all_allowed = -1, loner_allowed = -1, xlocker_allowed = -1;
diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 6ff87183914b..e94a988703f0 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -639,3 +639,16 @@ void MDLockCache::print(std::ostream& out) const {
   }
   out << ")";
 }
+
+int MDLockCache::get_cap_bit_for_lock_cache(int opcode)
+{
+  switch(opcode) {
+    case CEPH_MDS_OP_CREATE:
+      return CEPH_CAP_DIR_CREATE;
+    case CEPH_MDS_OP_UNLINK:
+      return CEPH_CAP_DIR_UNLINK;
+    default:
+      ceph_abort("unsupported opcode");
+      return 0;
+  }
+}
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 7f985651e37d..d7d74de82dfd 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -543,10 +543,20 @@ struct MDLockCache : public MutationImpl {
   void attach_dirfrags(std::vector<CDir*>&& dfv);
   void detach_locks();
   void detach_dirfrags();
+  /* Is the lock cache still attached to a capability and does that capability
+   * still have issued the rights (create/unlink) associated with the cap?
+   */
+  bool attachable() const {
+    return client_cap && cap_ref;
+  }
+  static int get_cap_bit_for_lock_cache(int opcode);
+  int get_cap_bit() const {
+    return get_cap_bit_for_lock_cache(opcode);
+  }
 
   CInode *diri;
   Capability *client_cap;
-  int opcode;
+  const int opcode;
   file_layout_t dir_layout;
 
   elist<MDLockCache*>::item item_cap_lock_cache;
@@ -560,6 +570,7 @@ struct MDLockCache : public MutationImpl {
 
   int ref = 1;
   bool invalidating = false;
+  bool cap_ref = true; /* does the cap still have issued&cap_bit ? */
 };
 
 typedef boost::intrusive_ptr<MutationImpl> MutationRef;

From 667c9a9c5af8d342ce6da5e942efb219b67d980b Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 17:08:54 -0400
Subject: [PATCH 2392/2492] mds: remove is_root indication on quiesce_inode op

This is no longer necessary with the change to a LocalLock quiescelock.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDCache.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index d62d93e16fd8..127721f085bb 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -13571,8 +13571,6 @@ void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
     return;
   }
 
-  [[maybe_unused]] const bool is_root = (ino == mdr->get_filepath2().get_ino());
-
   dout(20) << __func__ << " " << *mdr << " quiescing " << *in << dendl;
 
   if (quiesce_counter.get() > quiesce_threshold) {
@@ -13813,7 +13811,6 @@ void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
   } else if (auto& qops = mdr->more()->quiesce_ops; qops.count(dirino) == 0) {
     MDRequestRef qimdr = request_start_internal(CEPH_MDS_OP_QUIESCE_INODE);
     qimdr->set_filepath(filepath(dirino));
-    qimdr->set_filepath2(filepath(dirino)); /* is_root! */
     qimdr->internal_op_finish = new C_MDS_RetryRequest(this, mdr);
     qimdr->internal_op_private = new QuiesceInodeStateRef(qis);
     qops[dirino] = qimdr->reqid;

From 92f1ed3ed1fdba8a292dd4f625acd2b23bd6fdec Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 5 Mar 2024 14:22:53 -0500
Subject: [PATCH 2393/2492] Revert "pybind/mgr/volumes: block quiesce for
 critical .meta file"

This reverts commit 16e50abbd42a4b4f740816ba5f60bc259a6a96bf.

This flag is no longer necessary as the volumes plugin issues quiesce calls
against the data (i.e. root) directory of the subvolume rather than the
subvolume directory (with its associated .meta file).

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 .../fs/operations/versions/subvolume_base.py        | 13 -------------
 .../volumes/fs/operations/versions/subvolume_v1.py  |  3 ---
 .../volumes/fs/operations/versions/subvolume_v2.py  |  3 ---
 3 files changed, 19 deletions(-)

diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
index a8afcc368073..8fbe177e5f4b 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py
@@ -69,19 +69,6 @@ def base_path(self):
     def config_path(self):
         return os.path.join(self.base_path, b".meta")
 
-    def mark_meta(self):
-        """
-        Set "ceph.quiesce.block" flag on the .meta file inode.  It must remain
-        available while a subvolume is quiesced in order to allow some
-        interactions with the subvolume, snapshots in particular.
-        """
-        try:
-            self.fs.setxattr(self.config_path, 'ceph.quiesce.block', b'1', 0)
-        except cephfs.InvalidValue:
-            raise VolumeException(-errno.EINVAL, "invalid value specified for ceph.quiesce.block")
-        except cephfs.Error as e:
-            raise VolumeException(-e.args[0], e.args[1])
-
     @property
     def legacy_dir(self):
         return (os.path.join(self.vol_spec.base_dir.encode('utf-8'),
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
index 5f81a6305053..90f35a4c90b3 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py
@@ -111,7 +111,6 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
             qpath = subvol_path.decode('utf-8')
             self.init_config(SubvolumeV1.VERSION, subvolume_type, qpath, initial_state)
             self.mark_subvolume()
-            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 log.info("cleaning up subvolume with path: {0}".format(self.subvolname))
@@ -181,7 +180,6 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
             # create directory and set attributes
             self.fs.mkdirs(subvol_path, attrs.get("mode"))
             self.mark_subvolume()
-            self.mark_meta()
             self.set_attrs(subvol_path, attrs)
 
             # persist subvolume metadata and clone source
@@ -246,7 +244,6 @@ def open(self, op_type):
             st = self.fs.stat(subvol_path)
             # unconditionally mark as subvolume, to handle pre-existing subvolumes without the mark
             self.mark_subvolume()
-            self.mark_meta()
 
             self.uid = int(st.st_uid)
             self.gid = int(st.st_gid)
diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
index 089d5ae1c018..55d7f945b775 100644
--- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
+++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py
@@ -189,7 +189,6 @@ def create(self, size, isolate_nspace, pool, mode, uid, gid):
 
             # Create the subvolume metadata file which manages auth-ids if it doesn't exist
             self.auth_mdata_mgr.create_subvolume_metadata_file(self.group.groupname, self.subvolname)
-            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 self._remove_on_failure(subvol_path, retained)
@@ -244,7 +243,6 @@ def create_clone(self, pool, source_volname, source_subvolume, snapname):
                 self.metadata_mgr.init(SubvolumeV2.VERSION, subvolume_type.value, qpath, initial_state.value)
             self.add_clone_source(source_volname, source_subvolume, snapname)
             self.metadata_mgr.flush()
-            self.mark_meta()
         except (VolumeException, MetadataMgrException, cephfs.Error) as e:
             try:
                 self._remove_on_failure(subvol_path, retained)
@@ -302,7 +300,6 @@ def open(self, op_type):
             self.metadata_mgr.refresh()
             # unconditionally mark as subvolume, to handle pre-existing subvolumes without the mark
             self.mark_subvolume()
-            self.mark_meta()
 
             etype = self.subvol_type
             if op_type not in self.allowed_ops_by_type(etype):

From 464ef51525d673d38187bb9a841c4802c0681e0f Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 1 Mar 2024 21:49:53 -0500
Subject: [PATCH 2394/2492] qa: simplify calls to (rank|mds)_(tell|asok)

Instead of requiring the caller to put the arguments in a list, allow passing
as regular arguments.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/filesystem.py | 43 ++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index f952a2740a79..9a231c65f354 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -278,7 +278,8 @@ def get_nonnumeric_values(value):
                 j = json.loads(response_data.replace('inf', 'Infinity'),
                             parse_constant=get_nonnumeric_values)
             except json.decoder.JSONDecodeError:
-                raise RuntimeError(response_data) # assume it is an error message, pass it up
+                log.error(f"could not decode:\n{response_data}")
+                raise
             
             pretty = json.dumps(j, sort_keys=True, indent=2)
             log.debug(f"_json_asok output\n{pretty}")
@@ -1298,38 +1299,44 @@ def get_journal_version(self):
 
         return version
 
-    def mds_asok(self, command, mds_id=None, timeout=None):
+    def mds_asok(self, *args, mds_id=None, **kwargs):
         if mds_id is None:
-            return self.rank_asok(command, timeout=timeout)
+            return self.rank_asok(*args, **kwargs)
+        if len(args) == 1 and isinstance(args[0], (tuple, list)):
+            args = list(args[0])
 
-        return self.json_asok(command, 'mds', mds_id, timeout=timeout)
+        kwargs.pop('status', None) # not useful
+        return self.json_asok(list(args), 'mds', mds_id, **kwargs)
 
-    def mds_tell(self, command, mds_id=None):
+    def mds_tell(self, *args, mds_id=None, **kwargs):
         if mds_id is None:
-            return self.rank_tell(command)
-
-        return json.loads(self.get_ceph_cmd_stdout("tell", f"mds.{mds_id}", *command))
-
-    def rank_asok(self, command, rank=0, status=None, timeout=None):
-        info = self.get_rank(rank=rank, status=status)
-        return self.json_asok(command, 'mds', info['name'], timeout=timeout)
+            return self.rank_tell(*args, **kwargs)
+        if len(args) == 1 and isinstance(args[0], (tuple, list)):
+            args = list(args[0])
 
-    def rank_tell(self, command, rank=None, status=None, timeout=120):
-        if rank is None:
-            rank = 0
+        kwargs.pop('status', None) # not useful
         try:
-            out = self.get_ceph_cmd_stdout("tell", f"mds.{self.id}:{rank}", *command, timeout=timeout)
+            out = self.get_ceph_cmd_stdout("tell", f"mds.{mds_id}", *args, **kwargs)
             return json.loads(out)
         except json.decoder.JSONDecodeError:
             log.error("could not decode: {}".format(out))
             raise
 
-    def ranks_tell(self, command, status=None):
+    def rank_asok(self, *args, rank=0, status=None, **kwargs):
+        info = self.get_rank(rank=rank, status=status)
+        return self.mds_asok(*args, mds_id=info['name'], **kwargs)
+
+    def rank_tell(self, *args, rank=None, **kwargs):
+        if rank is None:
+            rank = 0
+        return self.mds_tell(*args, mds_id=f"{self.id}:{rank}", **kwargs)
+
+    def ranks_tell(self, *args, status=None):
         if status is None:
             status = self.status()
         out = []
         for r in status.get_ranks(self.id):
-            result = self.rank_tell(command, rank=r['rank'], status=status)
+            result = self.rank_tell(*args, rank=r['rank'], status=status)
             out.append((r['rank'], result))
         return sorted(out)
 

From 71fd9a34a911745957b8dbc4a904ab37616678b8 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 14 Mar 2024 10:37:12 -0400
Subject: [PATCH 2395/2492] qa: use kwarg for rank parameter

Otherwise it gets included in the *args list. This is necessary after commit
`qa: simplify calls to (rank|mds)_(tell|asok)`.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/cephfs_test_case.py   |  2 +-
 qa/tasks/cephfs/filesystem.py         |  2 +-
 qa/tasks/cephfs/test_damage.py        |  4 ++--
 qa/tasks/cephfs/test_data_scan.py     | 18 +++++++++---------
 qa/tasks/cephfs/test_forward_scrub.py |  2 +-
 qa/tasks/cephfs/test_multimds_misc.py |  6 +++---
 qa/tasks/cephfs/test_scrub_checks.py  |  8 ++++----
 qa/tasks/cephfs/test_strays.py        |  8 ++++----
 8 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/qa/tasks/cephfs/cephfs_test_case.py b/qa/tasks/cephfs/cephfs_test_case.py
index 7334a5972173..6f46bb7734e9 100644
--- a/qa/tasks/cephfs/cephfs_test_case.py
+++ b/qa/tasks/cephfs/cephfs_test_case.py
@@ -194,7 +194,7 @@ def setUp(self):
         # Load an config settings of interest
         for setting in self.LOAD_SETTINGS:
             setattr(self, setting, float(self.fs.mds_asok(
-                ['config', 'get', setting], list(self.mds_cluster.mds_ids)[0]
+                ['config', 'get', setting], mds_id=list(self.mds_cluster.mds_ids)[0]
             )[setting]))
 
         self.configs_set = set()
diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 9a231c65f354..0e57973b6124 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -604,7 +604,7 @@ def set_flag(self, var, *args):
 
     def set_config(self, opt, val, rank=0, status=None):
         command = ["config", "set", opt, val]
-        self.rank_asok(command, rank, status=status)
+        self.rank_asok(command, rank=rank, status=status)
 
     def set_allow_multifs(self, yes=True):
         self.set_flag("enable_multiple", yes)
diff --git a/qa/tasks/cephfs/test_damage.py b/qa/tasks/cephfs/test_damage.py
index 339b0e6c0556..ebf6dc03b649 100644
--- a/qa/tasks/cephfs/test_damage.py
+++ b/qa/tasks/cephfs/test_damage.py
@@ -51,7 +51,7 @@ def test_object_deletion(self):
         # to avoid waiting through reconnect on every MDS start.
         self.mount_a.umount_wait()
         for mds_name in self.fs.get_active_names():
-            self.fs.mds_asok(["flush", "journal"], mds_name)
+            self.fs.mds_asok(["flush", "journal"], mds_id=mds_name)
 
         self.fs.fail()
 
@@ -387,7 +387,7 @@ def test_damaged_dentry(self):
 
         self.mount_a.umount_wait()
         for mds_name in self.fs.get_active_names():
-            self.fs.mds_asok(["flush", "journal"], mds_name)
+            self.fs.mds_asok(["flush", "journal"], mds_id=mds_name)
 
         self.fs.fail()
 
diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py
index 49c609c78af4..6533ac98a2d6 100644
--- a/qa/tasks/cephfs/test_data_scan.py
+++ b/qa/tasks/cephfs/test_data_scan.py
@@ -549,11 +549,11 @@ def test_fragmented_injection(self):
 
         # Ensure that one directory is fragmented
         mds_id = self.fs.get_active_names()[0]
-        self.fs.mds_asok(["dirfrag", "split", "/subdir", "0/0", "1"], mds_id)
+        self.fs.mds_asok(["dirfrag", "split", "/subdir", "0/0", "1"], mds_id=mds_id)
 
         # Flush journal and stop MDS
         self.mount_a.umount_wait()
-        self.fs.mds_asok(["flush", "journal"], mds_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds_id)
         self.fs.fail()
 
         # Pick a dentry and wipe out its key
@@ -596,8 +596,8 @@ def test_fragmented_injection(self):
         # Finally, close the loop by checking our injected dentry survives a merge
         mds_id = self.fs.get_active_names()[0]
         self.mount_a.ls("subdir")  # Do an ls to ensure both frags are in cache so the merge will work
-        self.fs.mds_asok(["dirfrag", "merge", "/subdir", "0/0"], mds_id)
-        self.fs.mds_asok(["flush", "journal"], mds_id)
+        self.fs.mds_asok(["dirfrag", "merge", "/subdir", "0/0"], mds_id=mds_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds_id)
         frag_obj_id = "{0:x}.00000000".format(dir_ino)
         keys = self._dirfrag_keys(frag_obj_id)
         self.assertListEqual(sorted(keys), sorted(["%s_head" % f for f in file_names]))
@@ -667,7 +667,7 @@ def test_rebuild_linkage(self):
         self.mount_a.run_shell(["ln", "testdir1/file1", "testdir2/link2"])
 
         mds_id = self.fs.get_active_names()[0]
-        self.fs.mds_asok(["flush", "journal"], mds_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds_id)
 
         dirfrag1_keys = self._dirfrag_keys(dirfrag1_oid)
 
@@ -687,7 +687,7 @@ def test_rebuild_linkage(self):
         self.mount_a.run_shell(["touch", "testdir1/file1"])
         self.mount_a.umount_wait()
 
-        self.fs.mds_asok(["flush", "journal"], mds_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds_id)
         self.fs.fail()
 
         # repair linkage errors
@@ -738,8 +738,8 @@ def test_rebuild_inotable(self):
 
         self.mount_a.umount_wait()
 
-        self.fs.mds_asok(["flush", "journal"], mds0_id)
-        self.fs.mds_asok(["flush", "journal"], mds1_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds0_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds1_id)
         self.fs.fail()
 
         self.fs.radosm(["rm", "mds0_inotable"])
@@ -777,7 +777,7 @@ def test_rebuild_snaptable(self):
         self.mount_a.umount_wait()
 
         mds0_id = self.fs.get_active_names()[0]
-        self.fs.mds_asok(["flush", "journal"], mds0_id)
+        self.fs.mds_asok(["flush", "journal"], mds_id=mds0_id)
 
         # wait for mds to update removed snaps
         time.sleep(10)
diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py
index 334a73e1cda8..12a0fa6dafaf 100644
--- a/qa/tasks/cephfs/test_forward_scrub.py
+++ b/qa/tasks/cephfs/test_forward_scrub.py
@@ -325,7 +325,7 @@ def test_health_status_after_dentry_repair(self):
 
         self.mount_a.umount_wait()
         for mds_name in self.fs.get_active_names():
-            self.fs.mds_asok(["flush", "journal"], mds_name)
+            self.fs.mds_asok(["flush", "journal"], mds_id=mds_name)
 
         self.fs.fail()
 
diff --git a/qa/tasks/cephfs/test_multimds_misc.py b/qa/tasks/cephfs/test_multimds_misc.py
index e0e46fb24c09..66dcbceddc96 100644
--- a/qa/tasks/cephfs/test_multimds_misc.py
+++ b/qa/tasks/cephfs/test_multimds_misc.py
@@ -70,14 +70,14 @@ def _setup_subtrees(self):
         self._wait_subtrees([('/d1/d2/d3/d4', 1), ('/d1/d2/d3/d4/d5/d6', 2)], status, 2)
 
         for rank in range(3):
-            self.fs.rank_tell(["flush", "journal"], rank)
+            self.fs.rank_tell(["flush", "journal"], rank=rank)
 
     def test_apply_tag(self):
         self._setup_subtrees()
         inos = self._find_path_inos('d1/d2/d3/')
 
         tag = "tag123"
-        out_json = self.fs.rank_tell(["tag", "path", "/d1/d2/d3", tag], 0)
+        out_json = self.fs.rank_tell(["tag", "path", "/d1/d2/d3", tag], rank=0)
         self.assertNotEqual(out_json, None)
         self.assertEqual(out_json["return_code"], 0)
         self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
@@ -103,7 +103,7 @@ def test_scrub_backtrace(self):
         self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
 
         def _check_damage(mds_rank, inos):
-            all_damage = self.fs.rank_tell(["damage", "ls"], mds_rank)
+            all_damage = self.fs.rank_tell(["damage", "ls"], rank=mds_rank)
             damage = [d for d in all_damage if d['ino'] in inos and d['damage_type'] == "backtrace"]
             return len(damage) >= len(inos)
 
diff --git a/qa/tasks/cephfs/test_scrub_checks.py b/qa/tasks/cephfs/test_scrub_checks.py
index f17a6ceb1153..473b8bc11b08 100644
--- a/qa/tasks/cephfs/test_scrub_checks.py
+++ b/qa/tasks/cephfs/test_scrub_checks.py
@@ -278,7 +278,7 @@ def _checks(self, run_seq):
         command = "scrub start {file}".format(file=test_new_file)
 
         def _check_and_clear_damage(ino, dtype):
-            all_damage = self.fs.rank_tell(["damage", "ls"], mds_rank)
+            all_damage = self.fs.rank_tell(["damage", "ls"], rank=mds_rank)
             damage = [d for d in all_damage if d['ino'] == ino and d['damage_type'] == dtype]
             for d in damage:
                 self.run_ceph_cmd(
@@ -308,7 +308,7 @@ def scrub_with_stray_evaluation(self, fs, mnt, path, flag, files=2000,
         mnt.run_shell(["mkdir", f"{client_path}/.snap/snap1-{test_dir}"])
         mnt.run_shell(f"find {client_path}/ -type f -delete")
         mnt.run_shell(["rmdir", f"{client_path}/.snap/snap1-{test_dir}"])
-        perf_dump = fs.rank_tell(["perf", "dump"], 0)
+        perf_dump = fs.rank_tell(["perf", "dump"], rank=0)
         self.assertNotEqual(perf_dump.get('mds_cache').get('num_strays'),
                             0, "mdcache.num_strays is zero")
 
@@ -322,7 +322,7 @@ def scrub_with_stray_evaluation(self, fs, mnt, path, flag, files=2000,
         self.assertEqual(
             fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
 
-        perf_dump = fs.rank_tell(["perf", "dump"], 0)
+        perf_dump = fs.rank_tell(["perf", "dump"], rank=0)
         self.assertEqual(int(perf_dump.get('mds_cache').get('num_strays')),
                          0, "mdcache.num_strays is non-zero")
 
@@ -390,7 +390,7 @@ def tell_command(self, mds_rank, command, validator):
         log.info("Running command '{command}'".format(command=command))
 
         command_list = command.split()
-        jout = self.fs.rank_tell(command_list, mds_rank)
+        jout = self.fs.rank_tell(command_list, rank=mds_rank)
 
         log.info("command '{command}' returned '{jout}'".format(
                      command=command, jout=jout))
diff --git a/qa/tasks/cephfs/test_strays.py b/qa/tasks/cephfs/test_strays.py
index 11701dc28368..274cc238c325 100644
--- a/qa/tasks/cephfs/test_strays.py
+++ b/qa/tasks/cephfs/test_strays.py
@@ -681,8 +681,8 @@ def test_migration_on_shutdown(self):
 
         # empty mds cache. otherwise mds reintegrates stray when unlink finishes
         self.mount_a.umount_wait()
-        self.fs.mds_asok(['flush', 'journal'], rank_1_id)
-        self.fs.mds_asok(['cache', 'drop'], rank_1_id)
+        self.fs.mds_asok(['flush', 'journal'], mds_id=rank_1_id)
+        self.fs.mds_asok(['cache', 'drop'], mds_id=rank_1_id)
 
         self.mount_a.mount_wait()
         self.mount_a.run_shell(["rm", "-f", "dir_1/original"])
@@ -726,8 +726,8 @@ def test_migrate_unlinked_dir(self):
         self.assertEqual(self.get_mdc_stat("strays_enqueued", mds_id=rank_1_id), 0)
 
         # Test loading unlinked dir into cache
-        self.fs.mds_asok(['flush', 'journal'], rank_1_id)
-        self.fs.mds_asok(['cache', 'drop'], rank_1_id)
+        self.fs.mds_asok(['flush', 'journal'], mds_id=rank_1_id)
+        self.fs.mds_asok(['cache', 'drop'], mds_id=rank_1_id)
 
         # Shut down rank 1
         self.fs.set_max_mds(1)

From bb71023ced2d97a2590b98c4b820e7af6f1801b3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 1 Mar 2024 21:51:36 -0500
Subject: [PATCH 2396/2492] qa: refactor CephFSMount.kill_background to
 optionally kill all background jobs

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/mount.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 32f81551f99c..ba4874cc92da 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -1380,11 +1380,8 @@ def create_n_files(self, fs_path, count, sync=False, dirsync=False,
         self.run_python(pyscript)
 
     def teardown(self):
-        for p in self.background_procs:
-            log.info("Terminating background process")
-            self._kill_background(p)
-
-        self.background_procs = []
+        log.info("Terminating background process")
+        self.kill_background()
 
     def _kill_background(self, p):
         if p.stdin:
@@ -1394,13 +1391,16 @@ def _kill_background(self, p):
             except (CommandFailedError, ConnectionLostError):
                 pass
 
-    def kill_background(self, p):
+    def kill_background(self, p=None):
         """
         For a process that was returned by one of the _background member functions,
         kill it hard.
         """
-        self._kill_background(p)
-        self.background_procs.remove(p)
+        procs = [p] if p is not None else self.background_procs
+        for p in procs:
+            log.debug(f"terminating {p}")
+            self._kill_background(p)
+            self.background_procs.remove(p)
 
     def send_signal(self, signal):
         signal = signal.lower()

From 561965131eefb092163fa000fda7da9afcdcfcd4 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 2 Mar 2024 21:19:58 -0500
Subject: [PATCH 2397/2492] qa: set archive path in vstart_runner

For storing misc. test artifacts.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/vstart_runner.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index 8d9dc6f1845c..7aa9ac680c2c 100644
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -1002,6 +1002,7 @@ def __init__(self):
                           stdout=StringIO()).stdout.getvalue()
 
         cluster_name = 'ceph'
+        self.archive = "./"
         self.config = {'cluster': cluster_name}
         self.ceph = {cluster_name: Namespace()}
         self.ceph[cluster_name].fsid = FSID

From c0f0997e17e5639499bde09a82e8fbeef95ce006 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 27 Feb 2024 15:27:02 -0500
Subject: [PATCH 2398/2492] qa: update quiesce tests for control via locallock

- Verify multirank quiesce incl. caps issued.
- Unset splitauth experimental

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/filesystem.py   |  15 +-
 qa/tasks/cephfs/test_quiesce.py | 300 +++++++++++++++++++++-----------
 2 files changed, 205 insertions(+), 110 deletions(-)

diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 0e57973b6124..3f8d9b067d4e 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -1347,14 +1347,21 @@ def ranks_perf(self, f, status=None):
             out.append((rank, f(perf)))
         return out
 
-    def read_cache(self, path, depth=None, rank=None):
-        cmd = ["dump", "tree", path]
+    def read_cache(self, root, depth=None, path=None, rank=0, status=None):
+        name = self.get_rank(rank=rank, status=status)['name']
+        cmd = ["dump", "tree", root]
         if depth is not None:
             cmd.append(depth.__str__())
-        result = self.rank_asok(cmd, rank=rank)
+        if path:
+            cmd.append(f'--path={path}')
+        result = self.rank_asok(cmd, rank=rank, status=status)
         if result is None or len(result) == 0:
             raise RuntimeError("Path not found in cache: {0}".format(path))
-
+        if path:
+            mds_remote = self.mon_manager.find_remote('mds', name)
+            blob = misc.get_file(mds_remote, path, sudo=True).decode('utf-8')
+            log.debug(f"read {len(blob)}B of cache")
+            result = json.loads(blob)
         return result
 
     def wait_for_state(self, goal_state, reject=None, timeout=None, mds_id=None, rank=None):
diff --git a/qa/tasks/cephfs/test_quiesce.py b/qa/tasks/cephfs/test_quiesce.py
index 12f4f7971cb0..5ec86259fdde 100644
--- a/qa/tasks/cephfs/test_quiesce.py
+++ b/qa/tasks/cephfs/test_quiesce.py
@@ -1,3 +1,4 @@
+import errno
 import json
 import logging
 import os
@@ -16,6 +17,7 @@
 log = logging.getLogger(__name__)
 
 INODE_RE = re.compile(r'\[inode 0x([0-9a-fA-F]+)')
+CAP_RE = re.compile(r'(p)?(A[sx]+)?(L[sx]+)?(X[sx]+)?(F[sxrwcbl]+)?')
 FP_RE = re.compile(r'fp=#0x([0-9a-fA-F]+)(\S*)')
 
 # MDS uses linux defines:
@@ -43,10 +45,14 @@ class QuiesceTestCase(CephFSTestCase):
 
     def setUp(self):
         super().setUp()
+        self.config_set('mds', 'debug_mds', '25')
+        self.config_set('mds', 'mds_cache_quiesce_splitauth', 'true')
         self.run_ceph_cmd(f'fs subvolume create {self.fs.name} {self.QUIESCE_SUBVOLUME} --mode=777')
         p = self.run_ceph_cmd(f'fs subvolume getpath {self.fs.name} {self.QUIESCE_SUBVOLUME}', stdout=StringIO())
         self.mntpnt = p.stdout.getvalue().strip()
         self.subvolume = self.mntpnt
+        self.splitauth = True
+        self.archive = os.path.join(self.ctx.archive, 'quiesce')
 
     def tearDown(self):
         # restart fs so quiesce commands clean up and commands are left unkillable
@@ -55,6 +61,14 @@ def tearDown(self):
         self.fs.wait_for_daemons()
         super().tearDown()
 
+    def _make_archive(self):
+        log.info(f"making archive directory {self.archive}")
+        try:
+            os.mkdir(self.archive)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
     def _configure_subvolume(self):
         for m in self.mounts:
             m.umount_wait()
@@ -73,11 +87,17 @@ def _client_background_workload(self):
            p = m.run_shell_payload(self.CLIENT_WORKLOAD, wait=False, stderr=StringIO(), timeout=1)
            m.background_procs.append(p)
 
-    def _wait_for_quiesce_complete(self, reqid, rank=0, path=None):
+    def _wait_for_quiesce_complete(self, reqid, rank=0, path=None, status=None, timeout=120):
+        if path is None:
+            path = self.subvolume
+        if status is None:
+            status = self.fs.status()
         op = None
         try:
-            with safe_while(sleep=1, tries=120, action='wait for quiesce completion') as proceed:
+            with safe_while(sleep=1, tries=timeout, action='wait for quiesce completion') as proceed:
                 while proceed():
+                    if self.fs.status().hadfailover(status):
+                        raise RuntimeError("failover occurred")
                     op = self.fs.get_op(reqid, rank=rank)
                     log.debug(f"op:\n{op}")
                     self.assertEqual(op['type_data']['op_name'], 'quiesce_path')
@@ -85,15 +105,16 @@ def _wait_for_quiesce_complete(self, reqid, rank=0, path=None):
                         return op
         except:
             log.info(f"op:\n{op}")
-            if path is not None:
-                cache = self.fs.read_cache(path, rank=rank)
-                (fd, path) = tempfile.mkstemp()
-                with os.fdopen(fd, "wt") as f:
-                    f.write(f"{json.dumps(cache, indent=2)}")
-                    log.error(f"cache written to {path}")
-            (fd, path) = tempfile.mkstemp()
+            self._make_archive()
+            cache = self.fs.read_cache(path, rank=rank, path=f"/tmp/mds.{rank}-cache", status=status)
+            (fd, path) = tempfile.mkstemp(prefix="cache", dir=self.archive)
+            with os.fdopen(fd, "wt") as f:
+                os.fchmod(fd, 0o644)
+                f.write(f"{json.dumps(cache, indent=2)}")
+                log.error(f"cache written to {path}")
+            ops = self.fs.get_ops(locks=True, rank=rank, path=f"/tmp/mds.{rank}-ops", status=status)
+            (fd, path) = tempfile.mkstemp(prefix="ops", dir=self.archive)
             with os.fdopen(fd, "wt") as f:
-                ops = self.fs.get_ops(locks=True, rank=rank)
                 f.write(f"{json.dumps(ops, indent=2)}")
                 log.error(f"ops written to {path}")
             raise
@@ -102,36 +123,46 @@ def _wait_for_quiesce_complete(self, reqid, rank=0, path=None):
     FP_QUIESCE_BLOCKED = 'quiesce blocked'
     FP_QUIESCE_COMPLETE_NON_AUTH = 'quiesce complete for non-auth inode'
     FP_QUIESCE_COMPLETE_NON_AUTH_TREE = 'quiesce complete for non-auth tree'
-    def _verify_quiesce(self, rank=0, root=None, splitauth=False):
+    def _verify_quiesce(self, rank=0, root=None, splitauth=None, status=None):
         if root is None:
             root = self.subvolume
+        if splitauth is None:
+            splitauth = self.splitauth
+        if status is None:
+            status = self.fs.status()
 
-        name = self.fs.get_rank(rank=rank)['name']
-        root_ino = self.fs.read_cache(root, depth=0, rank=rank)[0]['ino']
-        ops = self.fs.get_ops(locks=True, rank=rank, path=f"/tmp/mds.{rank}-ops")
+        root_inode = self.fs.read_cache(root, depth=0, rank=rank, status=status)[0]
+        ops = self.fs.get_ops(locks=True, rank=rank, path=f"/tmp/mds.{rank}-ops", status=status)
         quiesce_inode_ops = {}
-        skipped_nonauth = False
 
-        count_q = 0
-        count_qb = 0
-        count_qna = 0
+        count_qp = 0
+        count_qi = 0
+        count_qib = 0
+        count_qina = 0
 
         for op in ops['ops']:
             try:
-                log.debug(f"op = {op}")
                 type_data = op['type_data']
                 flag_point = type_data['flag_point']
                 op_type = type_data['op_type']
                 if op_type == 'client_request' or op_type == 'peer_request':
                     continue
                 op_name = type_data['op_name']
+                op_description = op['description']
                 if op_name == "quiesce_path":
                     self.assertIn(flag_point, (self.FP_QUIESCE_COMPLETE, self.FP_QUIESCE_COMPLETE_NON_AUTH_TREE))
                     if flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH_TREE:
-                        skipped_nonauth = True
+                        self.assertFalse(splitauth)
+                        m = FP_RE.search(op_description)
+                        self.assertEqual(int(m.group(1)), 1)
+                        fp = m.group(2)
+                        if os.path.realpath(root) == os.path.realpath(fp):
+                            self.assertFalse(root_inode['is_auth'])
+                            log.debug("rank is not auth for tree and !splitauth")
+                            return
+                    count_qp += 1
                 elif op_name == "quiesce_inode":
                     # get the inode number
-                    op_description = op['description']
                     m = FP_RE.search(op_description)
                     self.assertIsNotNone(m)
                     if len(m.group(2)) == 0:
@@ -139,7 +170,7 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
                     else:
                         self.assertEqual(int(m.group(1)), 1)
                         fp = m.group(2)
-                        dump = self.fs.read_cache(fp, depth=0, rank=rank)
+                        dump = self.fs.read_cache(fp, depth=0, rank=rank, status=status)
                         ino = dump[0]['ino']
                     self.assertNotIn(ino, quiesce_inode_ops)
 
@@ -147,10 +178,10 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
 
                     locks = type_data['locks']
                     if flag_point == self.FP_QUIESCE_BLOCKED:
-                        count_qb += 1
+                        count_qib += 1
                         self.assertEqual(locks, [])
                     elif flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH:
-                        count_qna += 1
+                        count_qina += 1
                         #self.assertEqual(len(locks), 1)
                         #lock = locks[0]
                         #lock_type = lock['lock']['type']
@@ -160,7 +191,7 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
                         #self.assertIsNotNone(m)
                         #self.assertEqual(ino, int(m.group(1), 16))
                     else:
-                        count_q += 1
+                        count_qi += 1
                         for lock in locks:
                             lock_type = lock['lock']['type']
                             if lock_type.startswith('i'):
@@ -174,88 +205,106 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=False):
                 log.error(f"op:\n{json.dumps(op, indent=2)}")
                 raise
 
-        log.info(f"q = {count_q}; qb = {count_qb}; qna = {count_qna}")
-
-        if skipped_nonauth:
-            return
-
-        for ino, op in quiesce_inode_ops.items():
-            log.debug(f"{ino}: {op['description']}")
+        log.info(f"qp = {count_qp}; qi = {count_qi}; qib = {count_qib}; qina = {count_qina}")
 
         # now verify all files in cache have an op
-        cache = self.fs.read_cache(root, rank=rank)
+        cache = self.fs.read_cache(root, rank=rank, path=f"/tmp/mds.{rank}-cache", status=status)
         visited = set()
         locks_expected = set([
           "iquiesce",
-          "isnap",
-          "ipolicy",
-          "ifile",
-          "inest",
-          "idft",
-          "iauth",
-          "ilink",
-          "ixattr",
         ])
-        for inode in cache:
-            ino = inode['ino']
-            visited.add(ino)
-            mode = inode['mode']
-            self.assertIn(ino, quiesce_inode_ops)
-            op = quiesce_inode_ops[ino]
-            type_data = op['type_data']
-            flag_point = type_data['flag_point']
-            try:
-                locks_seen = set()
-                lock_type = None
-                op_name = type_data['op_name']
-                for lock in op['type_data']['locks']:
-                    lock_type = lock['lock']['type']
-                    if lock_type == "iquiesce":
-                        if ino == root_ino:
+        if not splitauth:
+            locks_expected.add('iauth')
+            locks_expected.add('ifile')
+            locks_expected.add('ilink')
+            locks_expected.add('ixattr')
+        try:
+            inos = set()
+            for inode in cache:
+                ino = inode['ino']
+                auth = inode['is_auth']
+                if not auth and not splitauth:
+                    continue
+                inos.add(ino)
+            self.assertLessEqual(set(inos), set(quiesce_inode_ops.keys()))
+            for inode in cache:
+                ino = inode['ino']
+                auth = inode['is_auth']
+                if not auth and not splitauth:
+                    continue
+                visited.add(ino)
+                self.assertIn(ino, quiesce_inode_ops.keys())
+                op = quiesce_inode_ops[ino]
+                type_data = op['type_data']
+                flag_point = type_data['flag_point']
+                try:
+                    locks_seen = set()
+                    lock_type = None
+                    op_name = type_data['op_name']
+                    for lock in op['type_data']['locks']:
+                        lock_type = lock['lock']['type']
+                        if lock_type == "iquiesce":
+                            self.assertEqual(lock['flags'], 4)
+                            self.assertEqual(lock['lock']['state'], 'lock')
+                            self.assertEqual(lock['lock']['num_xlocks'], 1)
+                        elif lock_type in ("ifile", "iauth", "ilink", "ixattr"):
+                            self.assertFalse(splitauth)
                             self.assertEqual(lock['flags'], 1)
-                            self.assertEqual(lock['lock']['state'], 'sync')
+                            self.assertEqual(lock['lock']['state'][:4], 'sync')
                         else:
-                            self.assertEqual(lock['flags'], 4)
-                            self.assertEqual(lock['lock']['state'], 'xlock')
-                    elif lock_type == "isnap":
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
-                    elif lock_type == "ifile":
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
-                    elif lock_type in ("ipolicy", "inest", "idft", "iauth", "ilink", "ixattr"):
-                        self.assertEqual(lock['flags'], 1)
-                        self.assertEqual(lock['lock']['state'][:4], 'sync')
-                    else:
-                        # no iflock
-                        self.assertFalse(lock_type.startswith("i"))
-                    if flag_point == self.FP_QUIESCE_COMPLETE and lock_type.startswith("i"):
-                        #if op_name == "quiesce_inode":
-                        #    self.assertTrue(lock['object']['is_auth'])
-                        locks_seen.add(lock_type)
-                try:
-                    if flag_point == self.FP_QUIESCE_BLOCKED:
-                        self.assertTrue(inode['quiesce_block'])
-                        self.assertEqual(set(), locks_seen)
-                    elif flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH:
-                        self.assertFalse(inode['quiesce_block'])
-                        self.assertEqual(set(), locks_seen)
-                    else:
-                        self.assertFalse(inode['quiesce_block'])
-                        self.assertEqual(locks_expected, locks_seen)
+                            # no other locks
+                            self.assertFalse(lock_type.startswith("i"))
+                        if flag_point == self.FP_QUIESCE_COMPLETE and lock_type.startswith("i"):
+                            #if op_name == "quiesce_inode":
+                            #    self.assertTrue(lock['object']['is_auth'])
+                            locks_seen.add(lock_type)
+                    try:
+                        if flag_point == self.FP_QUIESCE_BLOCKED:
+                            self.assertTrue(inode['quiesce_block'])
+                            self.assertEqual(set(), locks_seen)
+                        elif flag_point == self.FP_QUIESCE_COMPLETE_NON_AUTH:
+                            self.assertFalse(inode['quiesce_block'])
+                            self.assertEqual(set(), locks_seen)
+                        elif flag_point == self.FP_QUIESCE_COMPLETE:
+                            self.assertFalse(inode['quiesce_block'])
+                            self.assertEqual(locks_expected, locks_seen)
+                        else:
+                            self.fail(f"unexpected flag_point: {flag_point}")
+                    except:
+                        log.error(f"{sorted(locks_expected)} != {sorted(locks_seen)}")
+                        raise
+                    if flag_point in (self.FP_QUIESCE_COMPLETE_NON_AUTH, self.FP_QUIESCE_COMPLETE):
+                        for cap in inode['client_caps']:
+                            issued = cap['issued']
+                            m = CAP_RE.match(issued)
+                            if m is None:
+                                log.error(f"failed to parse client cap: {issued}")
+                                self.assertIsNotNone(m)
+                            g = m.groups()
+                            if g[1] is not None:
+                                # Ax?
+                                self.assertNotIn('x', g[1])
+                            if g[2] is not None:
+                                # Lx?
+                                self.assertNotIn('x', g[2])
+                            if g[3] is not None:
+                                # Xx?
+                                self.assertNotIn('x', g[3])
+                            if g[4] is not None:
+                                # Fxw?
+                                self.assertNotIn('x', g[4])
+                                self.assertNotIn('w', g[4])
                 except:
-                    log.error(f"{sorted(locks_expected)} != {sorted(locks_seen)}")
+                    log.error(f"inode:\n{json.dumps(inode, indent=2)}")
+                    log.error(f"op:\n{json.dumps(op, indent=2)}")
+                    log.error(f"lock_type: {lock_type}")
                     raise
-            except:
-                log.error(f"inode:\n{json.dumps(inode, indent=2)}")
-                log.error(f"op:\n{json.dumps(op, indent=2)}")
-                log.error(f"lock_type: {lock_type}")
-                raise
-        try:
-            self.assertEqual(visited, quiesce_inode_ops.keys())
+            if count_qp == 1:
+                self.assertEqual(visited, quiesce_inode_ops.keys())
         except:
             log.error(f"cache:\n{json.dumps(cache, indent=2)}")
             log.error(f"ops:\n{json.dumps(quiesce_inode_ops, indent=2)}")
+            raise
 
         # check request/cap count is stopped
         # count inodes under /usr and count subops!
@@ -424,7 +473,7 @@ def test_quiesce_path_notsubvol(self):
 
         J = self.fs.rank_tell(["quiesce", "path", path, '--wait'])
         reqid = self.reqid_tostr(J['op']['reqid'])
-        self._wait_for_quiesce_complete(reqid)
+        self._wait_for_quiesce_complete(reqid, path=path)
         self._verify_quiesce(root=path)
 
     def test_quiesce_path_regfile(self):
@@ -490,8 +539,41 @@ def test_quiesce_slow(self):
         self._wait_for_quiesce_complete(reqid)
         self._verify_quiesce(root=self.subvolume)
 
-    # TODO test lookup leaf file/dir after quiesce
-    # TODO ditto path_traverse
+    def test_quiesce_find(self):
+        """
+        That a `find` can be executed on a quiesced path.
+        """
+
+        # build a tree
+        self._configure_subvolume()
+        self._client_background_workload()
+        sleep(secrets.randbelow(20)+10)
+        for m in self.mounts:
+            m.kill_background()
+            m.remount() # drop all caps
+
+        # drop cache
+        self.fs.rank_tell(["cache", "drop"])
+
+        J = self.fs.rank_tell(["quiesce", "path", self.subvolume])
+        log.debug(f"{J}")
+        reqid = self.reqid_tostr(J['op']['reqid'])
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce(root=self.subvolume)
+
+        p = self.fs.rank_tell("perf", "dump")
+        dfc1 = p['mds']['dir_fetch_complete']
+
+        # now try `find`
+        self.mount_a.run_shell_payload('find -printf ""', timeout=300)
+
+        p = self.fs.rank_tell("perf", "dump")
+        dfc2 = p['mds']['dir_fetch_complete']
+        self.assertGreater(dfc2, dfc1)
+
+        self._wait_for_quiesce_complete(reqid)
+        self._verify_quiesce(root=self.subvolume)
+
 
 class TestQuiesceMultiRank(QuiesceTestCase):
     """
@@ -520,12 +602,18 @@ def setUp(self):
         status = self.fs.wait_for_daemons()
         self.mds_map = self.fs.get_mds_map(status=status)
         self.ranks = list(range(self.mds_map['max_mds']))
+        # mds_cache_quiesce_splitauth is now true by default but maintain
+        # manually as well.
+        self.config_set('mds', 'mds_cache_quiesce_splitauth', 'true')
+        self.splitauth = True
 
+    @unittest.skip("!splitauth")
     def test_quiesce_path_splitauth(self):
         """
         That quiesce fails (by default) if auth is split on a path.
         """
 
+        self.config_set('mds', 'mds_cache_quiesce_splitauth', 'false')
         self._configure_subvolume()
         self.mount_a.setfattr(".", "ceph.dir.pin.distributed", "1")
         self._client_background_workload()
@@ -544,6 +632,7 @@ def test_quiesce_path_multirank(self):
         self.mount_a.setfattr(".", "ceph.dir.pin.distributed", "1")
         self._client_background_workload()
         self._wait_distributed_subtrees(2*2, rank="all", path=self.mntpnt)
+        status = self.fs.status()
 
         sleep(secrets.randbelow(30)+10)
 
@@ -561,10 +650,9 @@ def test_quiesce_path_multirank(self):
         for rank, op, path in ops:
             reqid = self.reqid_tostr(op['reqid'])
             log.debug(f"waiting for ({rank}, {reqid})")
-            op = self._wait_for_quiesce_complete(reqid, rank=rank, path=path)
-        # FIXME _verify_quiesce needs adjustment for multiple quiesce
-        #for rank, op, path in ops:
-        #    self._verify_quiesce(root=path, rank=rank)
+            op = self._wait_for_quiesce_complete(reqid, rank=rank, path=path, status=status)
+        for rank, op, path in ops:
+            self._verify_quiesce(root=path, rank=rank, status=status)
 
     # TODO: test for quiesce_counter
 
@@ -593,12 +681,12 @@ def setUp(self):
         super().setUp()
         self.config_set('mds', 'mds_export_ephemeral_random_max', '0.75')
         self.config_set('mds', 'mds_cache_quiesce_splitauth', 'true')
+        self.splitauth = True
         self.fs.set_max_mds(2)
         status = self.fs.wait_for_daemons()
         self.mds_map = self.fs.get_mds_map(status=status)
         self.ranks = list(range(self.mds_map['max_mds']))
 
-    @unittest.skip("splitauth is experimental")
     def test_quiesce_path_multirank_exports(self):
         """
         That quiesce may complete with two ranks and a basic workload.
@@ -615,9 +703,9 @@ def test_quiesce_path_multirank_exports(self):
         op1 = self.fs.rank_tell(["quiesce", "path", self.subvolume], rank=1)['op']
         reqid0 = self.reqid_tostr(op0['reqid'])
         reqid1 = self.reqid_tostr(op1['reqid'])
-        op0 = self._wait_for_quiesce_complete(reqid0, rank=0)
-        op1 = self._wait_for_quiesce_complete(reqid1, rank=1)
+        op0 = self._wait_for_quiesce_complete(reqid0, rank=0, timeout=300)
+        op1 = self._wait_for_quiesce_complete(reqid1, rank=1, timeout=300)
         log.debug(f"op0 = {op0}")
         log.debug(f"op1 = {op1}")
-        self._verify_quiesce(rank=0)
-        self._verify_quiesce(rank=1)
+        self._verify_quiesce(rank=0, splitauth=True)
+        self._verify_quiesce(rank=1, splitauth=True)

From 48766b336d6d21051221f3cd33a5dd9b97df7740 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 7 Mar 2024 22:50:54 -0500
Subject: [PATCH 2399/2492] qa: wrap quiesce verification to dump debugging on
 error

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/tasks/cephfs/test_quiesce.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/qa/tasks/cephfs/test_quiesce.py b/qa/tasks/cephfs/test_quiesce.py
index 5ec86259fdde..ddb923d8c3aa 100644
--- a/qa/tasks/cephfs/test_quiesce.py
+++ b/qa/tasks/cephfs/test_quiesce.py
@@ -133,6 +133,23 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=None, status=None):
 
         root_inode = self.fs.read_cache(root, depth=0, rank=rank, status=status)[0]
         ops = self.fs.get_ops(locks=True, rank=rank, path=f"/tmp/mds.{rank}-ops", status=status)
+        cache = self.fs.read_cache(root, rank=rank, path=f"/tmp/mds.{rank}-cache", status=status)
+        try:
+            return self._verify_quiesce_wrapped(rank, status, root, root_inode, ops, cache, splitauth)
+        except:
+            self._make_archive()
+            (fd, path) = tempfile.mkstemp(prefix="cache", dir=self.archive)
+            with os.fdopen(fd, "wt") as f:
+                os.fchmod(fd, 0o644)
+                f.write(f"{json.dumps(cache, indent=2)}")
+                log.error(f"cache written to {path}")
+            (fd, path) = tempfile.mkstemp(prefix="ops", dir=self.archive)
+            with os.fdopen(fd, "wt") as f:
+                f.write(f"{json.dumps(ops, indent=2)}")
+                log.error(f"ops written to {path}")
+            raise
+
+    def _verify_quiesce_wrapped(self, rank, status, root, root_inode, ops, cache, splitauth):
         quiesce_inode_ops = {}
 
         count_qp = 0
@@ -208,7 +225,6 @@ def _verify_quiesce(self, rank=0, root=None, splitauth=None, status=None):
         log.info(f"qp = {count_qp}; qi = {count_qi}; qib = {count_qib}; qina = {count_qina}")
 
         # now verify all files in cache have an op
-        cache = self.fs.read_cache(root, rank=rank, path=f"/tmp/mds.{rank}-cache", status=status)
         visited = set()
         locks_expected = set([
           "iquiesce",

From 719d30d2774ab05bd9f92b7902487aec859c5d99 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 14 Mar 2024 14:29:58 -0400
Subject: [PATCH 2400/2492] doc/dev: update quiesce developer document

To include changes relating to it now being a local lock that prevents mutable
caps.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 doc/dev/mds_internals/quiesce.rst | 154 +++++++++++++++++-------------
 1 file changed, 90 insertions(+), 64 deletions(-)

diff --git a/doc/dev/mds_internals/quiesce.rst b/doc/dev/mds_internals/quiesce.rst
index 72f3a6ddd27d..7dd6bcf9086f 100644
--- a/doc/dev/mds_internals/quiesce.rst
+++ b/doc/dev/mds_internals/quiesce.rst
@@ -1,8 +1,8 @@
 MDS Quiesce Protocol
 ====================
 
-The MDS quiesce protocol is a mechanism for "quiescing" (quieting) a tree
-in a file system, stopping all write (and most read) I/O.
+The MDS quiesce protocol is a mechanism for "quiescing" (quieting) a tree in a
+file system, stopping all write (and sometimes incidentally read) I/O.
 
 The purpose of this API is to prevent multiple clients from interleaving reads
 and writes across an eventually consistent snapshot barrier where out-of-band
@@ -10,6 +10,11 @@ communication exists between clients. This communication can lead to clients
 wrongly believing they've reached a checkpoint that is mutually recoverable to
 via a snapshot.
 
+.. note:: This is documentation for the low-level mechanism in the MDS for
+          quiescing a tree of files. The higher-level QuiesceDb is the
+          intended API for clients to effect a quiesce.
+
+
 Mechanism
 ---------
 
@@ -18,76 +23,97 @@ appropriate locks on the root of a tree and then launches a series of
 sub-requests for locking other inodes in the tree. The locks obtained will
 force clients to release caps and in-progress client/MDS requests to complete.
 
-The sub-requests launched are ``quiesce_inode`` internal requests which simply
-lock the inode, if the MDS is authoritative for the inode.  Generally, these
-are rdlocks (read locks) on each inode metadata lock but the ``filelock`` is
-xlocked (exclusively locked) because its type allows for multiple readers and
-writers. Additionally, a new ``quiescelock`` is exclusively locked (more on
-that next).
+The sub-requests launched are ``quiesce_inode`` internal requests. These will
+obtain "cap-related" locks which control capability state, including the
+``filelock``, ``authlock``, ``linklock``, and ``xattrlock``. Additionally, the
+new local lock ``quiescelock`` is acquired. More information on that lock in
+the next section.
+
+Locks that are not cap-related are skipped because they do not control typical
+and durable metadata state. Additionally, only Capabilities can give a client
+local control of a file's metadata or data.
+
+Once all locks have been acquired, the cap-related locks are released and the
+``quiescelock`` is relied on to prevent issuing Capabilities to clients for the
+cap-related locks. This is controlled primarily by ``CInode:get_caps_*``
+methods. Releasing these locks is necessary to allow other ranks with the
+replicated inode to quiesce without lock state transitions resulting in
+deadlock. For example, a client wanting ``Xx`` on an inode will trigger a
+``xattrlock`` in ``LOCK_SYNC`` state to transition to ``LOCK_SYNC_EXCL``.  That
+state would not allow another rank to acquire ``xattrlock`` for reading,
+thereby creating deadlock, subject to quiesce timeout/expiration. (Quiesce
+cannot complete until all ranks quiesce the tree.)
 
-Because the ``quiesce_inode`` request will xlock the ``filelock`` and
-``quiescelock``, it only does so if run on the authoritative MDS. It is
-expected that the glue layer on top of the quiesce protocol will execute the
-same ``quiesce_path`` operation on each MDS rank.  This allows each rank which
-may be authoritative for part of the tree to lock all inodes it is
-authoritative for.
+Finally, if the inode is a directory, the ``quiesce_inode`` operation traverses
+all directory fragments and issues new ``quiesce_inode`` requests for any child
+inodes.
 
 
 Inode Quiescelock
 -----------------
 
-The ``quiescelock`` is a new lock for inodes which supports quiescing I/O.  It
-is a type of superlock where every client or MDS operation which accesses an
-inode lock will also implicitly acquire the ``quiescelock`` (readonly). In
-general, this lock is never held except for reading. When a subtree is
-quiesced, the ``quiesce_inode`` internal operation will hold ``quiescelock``
-exclusively, thereby denying the **new** acquisition of any other inode lock.
-The ``quiescelock`` must be ordered before all other locks (see
-``src/include/ceph_fs.h`` for ordering) in order to act as this superlock.
-
-The reason for this lock is to prevent an operation from blocking on acquiring
-locks held by ``quiesce_inode`` while still holding locks obtained
-during path traversal. Notably, the important locks are the ``snaplock`` and
-``policylock`` obtained via ``Locker::try_rdlock_snap_layout`` on all parents
-of the root inode of the request (the ``ino`` in the ``filepath`` struct). If
-that operation waits with those locks held, then a future ``mksnap`` on the
-root inode will be impossible.
+The ``quiescelock`` is a new local lock for inodes which supports quiescing
+I/O.  It is a type of superlock where every client or MDS operation which
+requires a wrlock or xlock on a "cap-related" inode lock will also implicitly
+acquire a wrlock on the ``quiescelock``.
+
+.. note:: A local lock supports multiple writers and only one exclusive locker. No read locks.
+
+During normal operation in the MDS, the ``quiescelock`` is never held except
+for writing. However, when a subtree is quiesced, the ``quiesce_inode``
+internal operation will hold ``quiescelock`` exclusively for the entire
+lifetime of the ``quiesce_inode`` operation. This will deny the **new**
+acquisition of any other cap-related inode lock.  The ``quiescelock`` must be ordered
+before all other locks (see ``src/include/ceph_fs.h`` for ordering) in order to
+act as this superlock.
+
+One primary reason for this ``quiescelock`` is to prevent a client request from
+blocking on acquiring locks held by ``quiesce_inode`` (e.g. ``filelock`` or
+``quiescelock``) while still holding locks obtained during normal path
+traversal. Notably, the important locks are the ``snaplock`` and ``policylock``
+obtained via ``Locker::try_rdlock_snap_layout`` on all parents of the root
+inode of the request (the ``ino`` in the ``filepath`` struct). If that
+operation waits with those locks held, then a future ``mksnap`` on the root
+inode will be impossible.
 
 .. note:: The ``mksnap`` RPC only acquires a wrlock (write lock) on the
           ``snaplock`` for the inode to be snapshotted.
 
 The way ``quiescelock`` helps prevent this is by being the first **mandatory**
-lock acquired and the special handling when it cannot be acquired: all locks
-held by the operation are dropped and the operation waits for the
-``quiescelock`` to be available.  The lock is mandatory in that all inode locks
-automatically include (add) the ``quiescelock`` when calling
-``Locker::acquire_locks``. So the expected normal flow is that an operation
-like ``getattr`` will perform its path traversal, acquiring parent and dentry
-locks, then attempt to acquire locks on the inode necessary for the requested
-client caps. The operation will fail to acquire the automatically included
-``quiescelock``, add itself to the ``quiescelock`` wait list, and then drop all
-held locks.
-
-There is a divergence in locking behavior for the root of the subvolume. The
-``quiescelock`` is only locked read-only. This allows the inode to be accessed
-by operations like ``mksnap`` which will implicitly acquire the ``quiescelock``
-read-only when locking the ``snaplock`` for writing. Additionally, if
-``Locker::acquire_locks`` will only acquire read locks without waiting, then it
-will skip the read-only lock on ``quiescelock``. This is to allow some forms of
-``lookup`` nececessary for snapshot management (e.g. volumes plugin) at higher
-layers.
-
-
-Readable quiesced tree
-----------------------
-
-It may be desirable to allow readers to continue accessing a quiesced
-subvolume. One way to do that is to have a separate superlock (yuck) for read
-access, say ``quiescerlock``. If a "readable" quiesce is performed, then
-``quiescerlock`` is not xlocked by ``quiesce_inode``. Read locks on
-other (non-quiesce) locks will acquire a read lock only on ``quiescerlock`` and
-no longer on ``quiescelock``.  Write locks would try to acquire both
-``quiescelock`` and ``quiescerlock`` (since writes may also read).
-
-Ideally, it may be a new lock type could be used to handle both cases but no
-such lock type yet exists.
+lock acquired when acquiring a wrlock or xlock on a cap-related lock.
+Additionally, there is also special handling when it cannot be acquired: all
+locks held by the operation are dropped and the operation waits for the
+``quiescelock`` to be available. The lock is mandatory in that a call to
+``Locker::acquire_locks`` with a wrlock/xlock on a cap-related lock  will
+automatically include (add) the ``quiescelock``.
+
+So, the expected normal flow is that an operation like ``mkdir`` will perform
+its path traversal, acquiring parent and dentry locks, then attempt to acquire
+locks on the parent inode necessary for the creation of a dentry. The operation
+will fail to acquire a wrlock on the automatically included ``quiescelock``,
+add itself to the ``quiescelock`` wait list, and then drop all held locks.
+
+
+Lookups and Exports
+-------------------
+
+Quiescing a tree results in a number of ``quiesce_inode`` operations for each
+inode under the tree. Those operations have a shared lifetime tied to the
+parent ``quiesce_path`` operation. So, once operations complete quiesce (but do
+not finish and release locks), the operations sit with locks held and do not
+monitor the state of the tree. This means we need to handle cases where new
+metadata is imported.
+
+If an inode is fetched via a directory ``lookup`` or ``readdir``, the MDS will
+check if its parent is quiesced (i.e. is the parent directory ``quiescelock``
+xlocked?). If so, the MDS will immediately issue an dispatch a
+``quiesce_inode`` operation for that inode. Because it's a fresh inode, the
+operation will immediately succeed and prevent the client from being issued
+inappropriate capabailities.
+
+The second case is handling subtree imports from another rank. This is
+problematic since the subtree import may have inodes with inappropriate state
+that would invalidate the guarantees of the reportedly "quiesced" tree. To
+avoid this, importer MDS will skip discovery of the root inode for an import if
+it encounters a directory inode that is quiesced. If skipped, the rank
+will send a NAK message back to the exporter which will abort the export.

From 00584a5619ba1c20d0c017f7326c8e9bd376e324 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 19 Mar 2024 13:28:53 -0400
Subject: [PATCH 2401/2492] script: allow use of run_tox.sh with default tox
 envlist

Add a "magic" --tox-envs value to force the run_tox.sh script to
defer to the tox.ini file's envlist rather than specifying a
custom env list. Pass `--tox-envs __tox_defaults__` to the script
to use whatever the tox.ini defaults are.

Adding a magic value like this avoids the need to make breaking changes
(or any changes) to the current cmake "code" used to invoke the
tox tests from make check. It's a little hacky but the funny looking
value should make it clear that it's special. In particular it avoids
adding new conditionals to the cmake files.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/run_tox.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/script/run_tox.sh b/src/script/run_tox.sh
index 9d45d8b9246c..f2f71a60f292 100755
--- a/src/script/run_tox.sh
+++ b/src/script/run_tox.sh
@@ -125,7 +125,11 @@ function main() {
     export CEPH_BUILD_DIR=$build_dir
     # use the wheelhouse prepared by install-deps.sh
     export PIP_FIND_LINKS="$tox_path/wheelhouse"
-    tox -c $tox_path/tox.ini -e "$tox_envs" "$@"
+    tox_cmd=(tox -c $tox_path/tox.ini)
+    if [ "$tox_envs" != "__tox_defaults__" ]; then
+        tox_cmd+=("-e" "$tox_envs")
+    fi
+    "${tox_cmd[@]}" "$@"
 }
 
 main "$@"

From d232ef57c99e47112c7efb0311ed686803705669 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 19 Mar 2024 13:33:55 -0400
Subject: [PATCH 2402/2492] cephadm: use tox.ini envlist defaults when run via
 cmake

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cephadm/CMakeLists.txt b/src/cephadm/CMakeLists.txt
index c33572dd4efb..c8b7c74a985a 100644
--- a/src/cephadm/CMakeLists.txt
+++ b/src/cephadm/CMakeLists.txt
@@ -1,6 +1,6 @@
 if(WITH_TESTS)
   include(AddCephTest)
-  add_tox_test(cephadm TOX_ENVS py3 mypy flake8)
+  add_tox_test(cephadm TOX_ENVS __tox_defaults__)
 endif()
 
 set(bin_target_file ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/cephadm)

From a92c2a47dd9afffb5a6aa9d1e29c788ee0141526 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 19 Mar 2024 13:43:09 -0400
Subject: [PATCH 2403/2492] cephadm: remove stray config section for autopep8

The cephadm tree has dropped autopep8 in favor of black. Remove this
unused config section.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tox.ini | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index a6b2601c76e2..106879b02f5a 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -21,15 +21,6 @@ exclude =
     .eggs
 statistics = True
 
-[autopep8]
-addopts =
-    --max-line-length {[flake8]max-line-length} \
-    --ignore "{[flake8]ignore}" \
-    --exclude "{[flake8]exclude}" \
-    --in-place \
-    --recursive \
-    --ignore-local-config
-
 [testenv]
 skip_install=true
 deps =

From 4e423259eadee61a897324ea2d3be6803f0506a1 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 19 Mar 2024 13:43:50 -0400
Subject: [PATCH 2404/2492] cephadm: add format-black reminder comment

Add a comment to tox.ini for people who might not know about the
relatively new `check-black` and `format-black` envs. While check-black
is part of the default envlist (as it is read-only) someone may not
know about the read-write version `format-black`.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 106879b02f5a..8ad1d297f385 100644
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -5,6 +5,8 @@ envlist =
     check-black
     py3
 skipsdist = true
+# REMINDER: run `tox -e format-black` to apply black formatting
+# with the exact same specs as `check-black` expects.
 
 [flake8]
 max-line-length = 100

From a2cf1285449b88e8a053161ba8055891c51f94c2 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Fri, 1 Mar 2024 12:56:25 +0530
Subject: [PATCH 2405/2492] mgr/dashboard:Mark placement targets as
 non-required in bucket form

Fixes https://tracker.ceph.com/issues/64708

- adds info text explaining about placement targets
- adds fieldset to security, policy and tags for a11y
- removes placement target as required field
- when no placement target provided, request omits placement_target
  field
- hides placement atrgets ection for edit form since its not possible to
  edit after creation
- updates rgw-bucket.service.spec.ts unit test
- minor cleanups related to unncesseary css present, and changes in help
  text for object locking
- updated e2e tests for buckets to incorporate changes

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../frontend/cypress/e2e/rgw/buckets.po.ts    |  13 +-
 .../rgw-bucket-form.component.html            | 277 +++++++++---------
 .../rgw-bucket-form.component.ts              |   2 +-
 .../app/shared/api/rgw-bucket.service.spec.ts |   4 +-
 .../src/app/shared/api/rgw-bucket.service.ts  |  39 +--
 5 files changed, 163 insertions(+), 172 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
index d121608490f9..53756f0a2569 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
@@ -41,7 +41,6 @@ export class BucketsPageHelper extends PageHelper {
 
     // Select bucket placement target:
     this.selectPlacementTarget(placementTarget);
-    cy.get('#placement-target').should('have.class', 'ng-valid');
 
     if (isLocking) {
       cy.get('#lock_enabled').click({ force: true });
@@ -70,7 +69,8 @@ export class BucketsPageHelper extends PageHelper {
   edit(name: string, new_owner: string, isLocking = false) {
     this.navigateEdit(name);
 
-    cy.get('input[name=placement-target]').should('have.value', 'default-placement');
+    // Placement target is not allowed to be edited and should be hidden
+    cy.get('input[name=placement-target]').should('not.exist');
     this.selectOwner(new_owner);
 
     // If object locking is enabled versioning shouldn't be visible
@@ -171,15 +171,6 @@ export class BucketsPageHelper extends PageHelper {
     // Check that error message was printed under owner drop down field
     cy.get('#owner + .invalid-feedback').should('have.text', 'This field is required.');
 
-    // Check invalid placement target input
-    this.selectOwner(BucketsPageHelper.USERS[1]);
-    // The drop down error message will not appear unless a valid option is previously selected.
-    this.selectPlacementTarget('default-placement');
-    this.selectPlacementTarget('-- Select a placement target --');
-    cy.get('@nameInputField').click(); // Trigger validation
-    cy.get('#placement-target').should('have.class', 'ng-invalid');
-    cy.get('#placement-target + .invalid-feedback').should('have.text', 'This field is required.');
-
     // Clicks the Create Bucket button but the page doesn't move.
     // Done by testing for the breadcrumb
     cy.contains('button', 'Create Bucket').click(); // Clicks Create Bucket button
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index 41b7f1162977..108b1ba6ce56 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -95,38 +95,30 @@
         </div>
 
         <!-- Placement target -->
-        <div class="form-group row">
+        <div class="form-group row"
+             *ngIf="!editing">
           <label class="cd-col-form-label"
-                 [ngClass]="{required: !editing}"
                  for="placement-target"
                  i18n>Placement target</label>
           <div class="cd-col-form-input">
-            <ng-template #placementTargetSelect>
-              <select id="placement-target"
-                      name="placement-target"
-                      formControlName="placement-target"
-                      class="form-select">
-                <option i18n
-                        *ngIf="placementTargets === null"
-                        [ngValue]="null">Loading...</option>
-                <option i18n
-                        *ngIf="placementTargets !== null"
-                        [ngValue]="null">-- Select a placement target --</option>
-                <option *ngFor="let placementTarget of placementTargets"
-                        [value]="placementTarget.name">{{ placementTarget.description }}</option>
-              </select>
-              <span class="invalid-feedback"
-                    *ngIf="bucketForm.showError('placement-target', frm, 'required')"
-                    i18n>This field is required.</span>
-            </ng-template>
-            <ng-container *ngIf="editing; else placementTargetSelect">
-              <input id="placement-target"
-                     name="placement-target"
-                     formControlName="placement-target"
-                     class="form-control"
-                     type="text"
-                     readonly>
-            </ng-container>
+            <select id="placement-target"
+                    name="placement-target"
+                    formControlName="placement-target"
+                    class="form-select">
+              <option i18n
+                      *ngIf="placementTargets === null"
+                      [ngValue]="null">Loading...</option>
+              <option i18n
+                      *ngIf="placementTargets !== null"
+                      [ngValue]="null">-- Select a placement target --</option>
+              <option *ngFor="let placementTarget of placementTargets"
+                      [value]="placementTarget.name">{{ placementTarget.description }}</option>
+            </select>
+            <cd-help-text>
+              <span i18n>
+                When creating a bucket, a placement target can be provided as part of the LocationConstraint to override the default placement targets from the user and zonegroup.
+              </span>
+            </cd-help-text>
           </div>
         </div>
 
@@ -218,8 +210,8 @@
           <legend class="cd-header"
                   i18n>
             Object Locking
-            <cd-help-text class="bc-legend-help">
-                Store objects using a write-once-read-many (WORM) model to help you prevent objects from being deleted or overwritten for a fixed amount of time or indefinitely.
+            <cd-help-text>
+                Store objects using a write-once-read-many (WORM) model to prevent objects from being deleted or overwritten for a fixed amount of time or indefinitely.
                 Object Locking works only in versioned buckets.
             </cd-help-text>
           </legend>
@@ -316,6 +308,7 @@
           </div>
         </fieldset>
 
+        <!-- Security -->
         <fieldset>
           <legend class="cd-header"
                   i18n>Security</legend>
@@ -423,135 +416,139 @@
         </fieldset>
 
         <!-- Tags -->
-        <legend class="cd-header"
-                i18n>Tags
-          <cd-helper>Tagging gives you a way to categorize storage</cd-helper>
-        </legend>
-        <span *ngFor="let tag of tags; let i=index;">
-          <ng-container *ngTemplateOutlet="tagTpl; context:{index: i, tag: tag}"></ng-container>
-        </span>
+        <fieldset>
+          <legend class="cd-header"
+                  i18n>Tags
+            <cd-helper>Tagging gives you a way to categorize storage</cd-helper>
+          </legend>
+          <span *ngFor="let tag of tags; let i=index;">
+            <ng-container *ngTemplateOutlet="tagTpl; context:{index: i, tag: tag}"></ng-container>
+          </span>
 
-        <div class="row">
-          <div class="col-12">
-            <strong *ngIf="tags.length > 19"
-                    class="text-warning"
-                    i18n>Maximum of 20 tags reached</strong>
-            <button type="button"
-                    id="add-tag"
-                    class="btn btn-light float-end my-3"
-                    [disabled]="tags.length > 19"
-                    (click)="showTagModal()">
-              <i [ngClass]="[icons.add]"></i>
-              <ng-container i18n>Add tag</ng-container>
-            </button>
+          <div class="row">
+            <div class="col-12">
+              <strong *ngIf="tags.length > 19"
+                      class="text-warning"
+                      i18n>Maximum of 20 tags reached</strong>
+              <button type="button"
+                      id="add-tag"
+                      class="btn btn-light float-end my-3"
+                      [disabled]="tags.length > 19"
+                      (click)="showTagModal()">
+                <i [ngClass]="[icons.add]"></i>
+                <ng-container i18n>Add tag</ng-container>
+              </button>
+            </div>
           </div>
-        </div>
+        </fieldset>
 
         <!-- Policies -->
-        <legend class="cd-header"
-                i18n>Policies
-        </legend>
-        <div class="row">
-          <div class="col-12">
-            <div class="form-group row">
+        <fieldset>
+          <legend class="cd-header"
+                  i18n>Policies
+          </legend>
+          <div class="row">
+            <div class="col-12">
+              <div class="form-group row">
 
-              <!-- Bucket policy -->
-              <label i18n
-                     class="cd-col-form-label"
-                     for="id">Bucket policy</label>
-              <div class="cd-col-form-input">
-                <textarea #bucketPolicyTextArea
-                          class="form-control resize-vertical"
-                          id="bucket_policy"
-                          formControlName="bucket_policy"
-                          (change)="bucketPolicyOnChange()">
-                </textarea>
-                <span class="invalid-feedback"
-                      *ngIf="bucketForm.showError('bucket_policy', frm, 'invalidJson')"
-                      i18n>Invalid json text</span>
-                <button type="button"
-                        id="clear-bucket-policy"
-                        class="btn btn-light my-3"
-                        (click)="clearBucketPolicy()"
-                        i18n>
-                  <i [ngClass]="[icons.destroy]"></i>
-                  Clear
-                </button>
-                <div class="btn-group float-end"
-                     role="group"
-                     aria-label="bucket-policy-helpers">
-                  <button type="button"
-                          id="example-generator-button"
-                          class="btn btn-light my-3"
-                          (click)="openUrl('https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-bucket-policies.html?icmpid=docs_amazons3_console')"
-                          i18n>
-                    <i [ngClass]="[icons.externalUrl]"></i>
-                    Policy examples
-                  </button>
+                <!-- Bucket policy -->
+                <label i18n
+                       class="cd-col-form-label"
+                       for="id">Bucket policy</label>
+                <div class="cd-col-form-input">
+                  <textarea #bucketPolicyTextArea
+                            class="form-control resize-vertical"
+                            id="bucket_policy"
+                            formControlName="bucket_policy"
+                            (change)="bucketPolicyOnChange()">
+                  </textarea>
+                  <span class="invalid-feedback"
+                        *ngIf="bucketForm.showError('bucket_policy', frm, 'invalidJson')"
+                        i18n>Invalid json text</span>
                   <button type="button"
-                          id="example-generator-button"
+                          id="clear-bucket-policy"
                           class="btn btn-light my-3"
-                          (click)="openUrl('https://awspolicygen.s3.amazonaws.com/policygen.html')"
+                          (click)="clearBucketPolicy()"
                           i18n>
-                    <i [ngClass]="[icons.externalUrl]"></i>
-                    Policy generator
+                    <i [ngClass]="[icons.destroy]"></i>
+                    Clear
                   </button>
+                  <div class="btn-group float-end"
+                       role="group"
+                       aria-label="bucket-policy-helpers">
+                    <button type="button"
+                            id="example-generator-button"
+                            class="btn btn-light my-3"
+                            (click)="openUrl('https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-bucket-policies.html?icmpid=docs_amazons3_console')"
+                            i18n>
+                      <i [ngClass]="[icons.externalUrl]"></i>
+                      Policy examples
+                    </button>
+                    <button type="button"
+                            id="example-generator-button"
+                            class="btn btn-light my-3"
+                            (click)="openUrl('https://awspolicygen.s3.amazonaws.com/policygen.html')"
+                            i18n>
+                      <i [ngClass]="[icons.externalUrl]"></i>
+                      Policy generator
+                    </button>
+                  </div>
                 </div>
               </div>
-            </div>
 
-            <div class="form-group row">
+              <div class="form-group row">
 
-              <!-- ACL -->
-              <label class="cd-col-form-label"
-                     i18n>ACL
-                <cd-helper>Any changes to the ACL will overwrite previous one.
-                  You can choose any of the available options to modify the spcified user group.</cd-helper>
-              </label>
-              <div class="cd-col-form-input">
-                <div class="input-group">
-                  <span class="input-group-text"
-                        for="grantee"
-                        i18n>Grantee
-                    <cd-helper>Select a grantee (user group) to modify it's permisions</cd-helper>
-                  </span>
-                  <select id="grantee"
-                          name="grantee"
-                          class="form-input form-select"
-                          formControlName="grantee"
-                          (change)="onSelectionFilter()">
-                    <option *ngFor="let item of grantees"
-                            [value]="item"
-                            i18n>{{ item }}</option>
-                  </select>
-                  <span class="invalid-feedback"
-                        *ngIf="bucketForm.showError('grantee', frm, 'required')"
-                        i18n>This field is required.</span>
-                  <span class="input-group-text"
-                        for="aclPermission"
-                        i18n>Permissions
-                    <cd-helper>Select the permision to give to the selected grantee.
-                        Regardless, the owner of the bucket will always have
-                          FULL CONTROL access</cd-helper>
+                <!-- ACL -->
+                <label class="cd-col-form-label"
+                       i18n>ACL
+                  <cd-helper>Any changes to the ACL will overwrite previous one.
+                    You can choose any of the available options to modify the spcified user group.</cd-helper>
+                </label>
+                <div class="cd-col-form-input">
+                  <div class="input-group">
+                    <span class="input-group-text"
+                          for="grantee"
+                          i18n>Grantee
+                      <cd-helper>Select a grantee (user group) to modify it's permisions</cd-helper>
                     </span>
-                  <select id="aclPermission"
-                          name="aclPermission"
-                          class="form-input form-select"
-                          formControlName="aclPermission">
-                    <option *ngFor="let permission of aclPermissions"
-                            [value]="permission"
-                            i18n>{{ permission }}</option>
-                  </select>
-                  <span class="invalid-feedback"
-                        *ngIf="bucketForm.showError('aclPermission', frm, 'required')"
-                        i18n>This field is required.</span>
+                    <select id="grantee"
+                            name="grantee"
+                            class="form-input form-select"
+                            formControlName="grantee"
+                            (change)="onSelectionFilter()">
+                      <option *ngFor="let item of grantees"
+                              [value]="item"
+                              i18n>{{ item }}</option>
+                    </select>
+                    <span class="invalid-feedback"
+                          *ngIf="bucketForm.showError('grantee', frm, 'required')"
+                          i18n>This field is required.</span>
+                    <span class="input-group-text"
+                          for="aclPermission"
+                          i18n>Permissions
+                      <cd-helper>Select the permision to give to the selected grantee.
+                          Regardless, the owner of the bucket will always have
+                            FULL CONTROL access</cd-helper>
+                      </span>
+                    <select id="aclPermission"
+                            name="aclPermission"
+                            class="form-input form-select"
+                            formControlName="aclPermission">
+                      <option *ngFor="let permission of aclPermissions"
+                              [value]="permission"
+                              i18n>{{ permission }}</option>
+                    </select>
+                    <span class="invalid-feedback"
+                          *ngIf="bucketForm.showError('aclPermission', frm, 'required')"
+                          i18n>This field is required.</span>
+                  </div>
                 </div>
               </div>
             </div>
           </div>
-        </div>
-
+        </fieldset>
       </div>
+
       <div class="card-footer">
         <cd-form-button-panel (submitActionEvent)="submit()"
                               [form]="bucketForm"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
index 6857874411ae..061dee0225c8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.ts
@@ -125,7 +125,7 @@ export class RgwBucketFormComponent extends CdForm implements OnInit, AfterViewC
       ],
       owner: [null, [Validators.required]],
       kms_provider: ['vault'],
-      'placement-target': [null, this.editing ? [] : [Validators.required]],
+      'placement-target': [null],
       versioning: [null],
       'mfa-delete': [null],
       'mfa-token-serial': [''],
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
index f40fd17cfc08..eaed2c4abac2 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.spec.ts
@@ -53,7 +53,7 @@ describe('RgwBucketService', () => {
         'foo',
         'bar',
         'default',
-        'default-placement',
+        null,
         false,
         'COMPLIANCE',
         '5',
@@ -66,7 +66,7 @@ describe('RgwBucketService', () => {
       )
       .subscribe();
     const req = httpTesting.expectOne(
-      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&placement_target=default-placement&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&bucket_policy=null&canned_acl=private&${RgwHelper.DAEMON_QUERY_PARAM}`
+      `api/rgw/bucket?bucket=foo&uid=bar&zonegroup=default&lock_enabled=false&lock_mode=COMPLIANCE&lock_retention_period_days=5&encryption_state=true&encryption_type=aws%253Akms&key_id=qwerty1&tags=null&bucket_policy=null&canned_acl=private&${RgwHelper.DAEMON_QUERY_PARAM}`
     );
     expect(req.request.method).toBe('POST');
   });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
index e8e123bf9f25..ddeeadf5e49b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/rgw-bucket.service.ts
@@ -65,25 +65,28 @@ export class RgwBucketService extends ApiClient {
     cannedAcl: string
   ) {
     return this.rgwDaemonService.request((params: HttpParams) => {
+      const paramsObject = {
+        bucket,
+        uid,
+        zonegroup,
+        lock_enabled: String(lockEnabled),
+        lock_mode,
+        lock_retention_period_days,
+        encryption_state: String(encryption_state),
+        encryption_type,
+        key_id,
+        tags: tags,
+        bucket_policy: bucketPolicy,
+        canned_acl: cannedAcl,
+        daemon_name: params.get('daemon_name')
+      };
+
+      if (placementTarget) {
+        paramsObject['placement_target'] = placementTarget;
+      }
+
       return this.http.post(this.url, null, {
-        params: new HttpParams({
-          fromObject: {
-            bucket,
-            uid,
-            zonegroup,
-            placement_target: placementTarget,
-            lock_enabled: String(lockEnabled),
-            lock_mode,
-            lock_retention_period_days,
-            encryption_state: String(encryption_state),
-            encryption_type,
-            key_id,
-            tags: tags,
-            bucket_policy: bucketPolicy,
-            canned_acl: cannedAcl,
-            daemon_name: params.get('daemon_name')
-          }
-        })
+        params: new HttpParams({ fromObject: paramsObject })
       });
     });
   }

From ab1dd05300622a4c79712555aa81b3eb4ccd9d3c Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Tue, 19 Mar 2024 13:12:51 +0000
Subject: [PATCH 2406/2492] run-make: remove WITH_ZBD

This commit remove 'WITH_ZBD' in run-make.sh because of: https://github.com/ceph/ceph/pull/55054

Fixes: https://tracker.ceph.com/issues/64962
Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/script/run-make.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 42d8a94aee1f..5914b7c5294f 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -111,9 +111,6 @@ EOM
     if [ $WITH_SEASTAR ]; then
         cmake_opts+=" -DWITH_SEASTAR=ON"
     fi
-    if [ $WITH_ZBD ]; then
-        cmake_opts+=" -DWITH_ZBD=ON"
-    fi
     if [ $WITH_RBD_RWL ]; then
         cmake_opts+=" -DWITH_RBD_RWL=ON"
     fi

From 2da1812a1de7e32395cb95c2e0ebeea689c1b810 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Thu, 21 Mar 2024 22:48:10 +1000
Subject: [PATCH 2407/2492] doc/start: improve MDS explanation

Remove a prolix "on behalf of" and streamline some sentences explaining
the benefits of MDS.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/start/intro.rst | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/doc/start/intro.rst b/doc/start/intro.rst
index 9881a7205487..c67b9a23ba87 100644
--- a/doc/start/intro.rst
+++ b/doc/start/intro.rst
@@ -51,11 +51,10 @@ The Ceph Metadata Server is necessary to run Ceph File System clients.
   heartbeat. At least three Ceph OSDs are normally required for 
   redundancy and high availability.
 
-- **MDSs**: A :term:`Ceph Metadata Server` (MDS, ``ceph-mds``) stores
-  metadata on behalf of the :term:`Ceph File System` (i.e., Ceph Block
-  Devices and Ceph Object Storage do not use MDS). Ceph Metadata
-  Servers allow POSIX file system users to execute basic commands (like
-  ``ls``, ``find``, etc.) without placing an enormous burden on the
+- **MDSs**: A :term:`Ceph Metadata Server` (MDS, ``ceph-mds``) stores metadata
+  for the :term:`Ceph File System`. Ceph Block Devices and Ceph Object Storage
+  do not use MDS. Ceph Metadata Servers allow POSIX-file-system users to run
+  basic commands (like ``ls``, ``find``, etc.) without placing a burden on the
   Ceph Storage Cluster.
 
 Ceph stores data as objects within logical storage pools. Using the

From 40d5064baa41af5b33823f05f5ae50a28ff33cf6 Mon Sep 17 00:00:00 2001
From: Ronen Friedman <rfriedma@redhat.com>
Date: Thu, 21 Mar 2024 17:06:13 +0200
Subject: [PATCH 2408/2492] osd/scrub: removing the slow reservation response
 configuration option

Removing 'osd_scrub_slow_reservation_response', as it is not in use
anymore.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>
---
 src/common/options/osd.yaml.in | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in
index 598cf4f50fd0..bc5d53577050 100644
--- a/src/common/options/osd.yaml.in
+++ b/src/common/options/osd.yaml.in
@@ -520,17 +520,6 @@ options:
     stats (inc. scrub/block duration) every this many seconds.
   default: 120
   with_legacy: false
-- name: osd_scrub_slow_reservation_response
-  type: millisecs
-  level: advanced
-  desc: Maximum wait (milliseconds) for scrub reservations before issuing a cluster-log warning
-  long_desc: Waiting too long for a replica to respond to scrub resource reservation request.
-    Disable by setting to a very large value.
-  default: 30000
-  min: 500
-  see_also:
-  - osd_scrub_reservation_timeout
-  with_legacy: false
 - name: osd_scrub_reservation_timeout
   type: millisecs
   level: advanced
@@ -540,8 +529,6 @@ options:
     to a very large value.
   default: 300000
   min: 2000
-  see_also:
-  - osd_scrub_slow_reservation_response
   with_legacy: false
 - name: osd_scrub_disable_reservation_queuing
   type: bool

From 86a592730b09ab2cfaf4100907366050c932f30a Mon Sep 17 00:00:00 2001
From: Gil Bregman <gbregman@il.ibm.com>
Date: Thu, 21 Mar 2024 11:07:54 +0200
Subject: [PATCH 2409/2492] python-common/ceph/deployment: add nvmeof
 configuration fields

Fixes: https://tracker.ceph.com/issues/64821

Signed-off-by: Gil Bregman <gbregman@il.ibm.com>
---
 .../ceph/deployment/service_spec.py           | 96 +++++++++++++++++--
 1 file changed, 87 insertions(+), 9 deletions(-)

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 24f5c646461b..767e1e6f4685 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1267,21 +1267,41 @@ def __init__(self,
                  port: Optional[int] = None,
                  pool: Optional[str] = None,
                  enable_auth: bool = False,
-                 min_controller_id: Optional[str] = '1',
-                 max_controller_id: Optional[str] = '65519',
+                 state_update_notify: Optional[bool] = True,
+                 state_update_interval_sec: Optional[int] = 5,
                  enable_spdk_discovery_controller: Optional[bool] = False,
+                 omap_file_lock_duration: Optional[int] = 60,
+                 omap_file_lock_retries: Optional[int] = 15,
+                 omap_file_lock_retry_sleep_interval: Optional[int] = 5,
+                 omap_file_update_reloads: Optional[int] = 10,
+                 enable_prometheus_exporter: Optional[bool] = True,
+                 bdevs_per_cluster: Optional[int] = 32,
+                 verify_nqns: Optional[bool] = True,
                  server_key: Optional[str] = None,
                  server_cert: Optional[str] = None,
                  client_key: Optional[str] = None,
                  client_cert: Optional[str] = None,
                  spdk_path: Optional[str] = None,
                  tgt_path: Optional[str] = None,
-                 timeout: Optional[int] = 60,
+                 spdk_timeout: Optional[float] = 60.0,
+                 spdk_log_level: Optional[str] = 'WARNING',
+                 rpc_socket_dir: Optional[str] = '/var/tmp/',
+                 rpc_socket_name: Optional[str] = 'spdk.sock',
                  conn_retries: Optional[int] = 10,
                  transports: Optional[str] = 'tcp',
                  transport_tcp_options: Optional[Dict[str, int]] =
                  {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
                  tgt_cmd_extra_args: Optional[str] = None,
+                 discovery_port: Optional[int] = None,
+                 log_level: Optional[str] = 'INFO',
+                 log_files_enabled: Optional[bool] = True,
+                 log_files_rotation_enabled: Optional[bool] = True,
+                 verbose_log_messages: Optional[bool] = True,
+                 max_log_file_size_in_mb: Optional[int] = 10,
+                 max_log_files_count: Optional[int] = 20,
+                 max_log_directory_backups: Optional[int] = 10,
+                 log_directory: Optional[str] = '/var/log/ceph/',
+                 monitor_timeout: Optional[float] = 1.0,
                  placement: Optional[PlacementSpec] = None,
                  unmanaged: bool = False,
                  preview_only: bool = False,
@@ -1310,12 +1330,26 @@ def __init__(self,
         self.group = group
         #: ``enable_auth`` enables user authentication on nvmeof gateway
         self.enable_auth = enable_auth
-        #: ``min_controller_id`` minimum controller id used by SPDK, essential for multipath
-        self.min_controller_id = min_controller_id
-        #: ``max_controller_id`` maximum controller id used by SPDK, essential for multipath
-        self.max_controller_id = max_controller_id
+        #: ``state_update_notify`` enables automatic update from OMAP in nvmeof gateway
+        self.state_update_notify = state_update_notify
+        #: ``state_update_interval_sec`` number of seconds to check for updates in OMAP
+        self.state_update_interval_sec = state_update_interval_sec
         #: ``enable_spdk_discovery_controller`` SPDK or ceph-nvmeof discovery service
         self.enable_spdk_discovery_controller = enable_spdk_discovery_controller
+        #: ``enable_prometheus_exporter`` enables Prometheus exporter
+        self.enable_prometheus_exporter = enable_prometheus_exporter
+        #: ``verify_nqns`` enables verification of subsystem and host NQNs for validity
+        self.verify_nqns = verify_nqns
+        #: ``omap_file_lock_duration`` number of seconds before automatically unlock OMAP file lock
+        self.omap_file_lock_duration = omap_file_lock_duration
+        #: ``omap_file_lock_retries`` number of retries to lock OMAP file before giving up
+        self.omap_file_lock_retries = omap_file_lock_retries
+        #: ``omap_file_lock_retry_sleep_interval`` seconds to wait before retrying to lock OMAP
+        self.omap_file_lock_retry_sleep_interval = omap_file_lock_retry_sleep_interval
+        #: ``omap_file_update_reloads`` number of attempt to reload OMAP when it differs from local
+        self.omap_file_update_reloads = omap_file_update_reloads
+        #: ``bdevs_per_cluster`` number of bdevs per cluster
+        self.bdevs_per_cluster = bdevs_per_cluster
         #: ``server_key`` gateway server key
         self.server_key = server_key or './server.key'
         #: ``server_cert`` gateway server certificate
@@ -1328,8 +1362,14 @@ def __init__(self,
         self.spdk_path = spdk_path or '/usr/local/bin/nvmf_tgt'
         #: ``tgt_path`` nvmeof target path
         self.tgt_path = tgt_path or '/usr/local/bin/nvmf_tgt'
-        #: ``timeout`` ceph connectivity timeout
-        self.timeout = timeout
+        #: ``spdk_timeout`` SPDK connectivity timeout
+        self.spdk_timeout = spdk_timeout
+        #: ``spdk_log_level`` the SPDK log level
+        self.spdk_log_level = spdk_log_level or 'WARNING'
+        #: ``rpc_socket_dir`` the SPDK socket file directory
+        self.rpc_socket_dir = rpc_socket_dir or '/var/tmp/'
+        #: ``rpc_socket_name`` the SPDK socket file name
+        self.rpc_socket_name = rpc_socket_name or 'spdk.sock'
         #: ``conn_retries`` ceph connection retries number
         self.conn_retries = conn_retries
         #: ``transports`` tcp
@@ -1338,6 +1378,26 @@ def __init__(self,
         self.transport_tcp_options: Optional[Dict[str, int]] = transport_tcp_options
         #: ``tgt_cmd_extra_args`` extra arguments for the nvmf_tgt process
         self.tgt_cmd_extra_args = tgt_cmd_extra_args
+        #: ``discovery_port`` port of the discovery service
+        self.discovery_port = discovery_port or 8009
+        #: ``log_level`` the nvmeof gateway log level
+        self.log_level = log_level or 'INFO'
+        #: ``log_files_enabled`` enables the usage of files to keep the nameof gateway log
+        self.log_files_enabled = log_files_enabled
+        #: ``log_files_rotation_enabled`` enables rotation of log files when pass the size limit
+        self.log_files_rotation_enabled = log_files_rotation_enabled
+        #: ``verbose_log_messages`` add more details to the nvmeof gateway log message
+        self.verbose_log_messages = verbose_log_messages
+        #: ``max_log_file_size_in_mb`` max size in MB before starting a new log file
+        self.max_log_file_size_in_mb = max_log_file_size_in_mb
+        #: ``max_log_files_count`` max log files to keep before overriding them
+        self.max_log_files_count = max_log_files_count
+        #: ``max_log_directory_backups`` max directories for old gateways with same name to keep
+        self.max_log_directory_backups = max_log_directory_backups
+        #: ``log_directory`` directory for keeping nameof gateway log files
+        self.log_directory = log_directory or '/var/log/ceph/'
+        #: ``monitor_timeout`` monitor connectivity timeout
+        self.monitor_timeout = monitor_timeout
 
     def get_port_start(self) -> List[int]:
         return [5500, 4420, 8009]
@@ -1357,6 +1417,24 @@ def validate(self) -> None:
         if self.transports not in ['tcp']:
             raise SpecValidationError('Invalid transport. Valid values are tcp')
 
+        if self.log_level:
+            if self.log_level not in ['debug', 'DEBUG',
+                                      'info', 'INFO',
+                                      'warning', 'WARNING',
+                                      'error', 'ERROR',
+                                      'critical', 'CRITICAL']:
+                raise SpecValidationError(
+                    'Invalid log level. Valid values are: debug, info, warning, error, critial')
+
+        if self.spdk_log_level:
+            if self.spdk_log_level not in ['debug', 'DEBUG',
+                                           'info', 'INFO',
+                                           'warning', 'WARNING',
+                                           'error', 'ERROR',
+                                           'notice', 'NOTICE']:
+                raise SpecValidationError(
+                    'Invalid SPDK log level. Valid values are: DEBUG, INFO, WARNING, ERROR, NOTICE')
+
 
 yaml.add_representer(NvmeofServiceSpec, ServiceSpec.yaml_representer)
 

From a16ac81c43be884f2dd7fc8b0566be5aa35b7ace Mon Sep 17 00:00:00 2001
From: Gil Bregman <gbregman@il.ibm.com>
Date: Thu, 21 Mar 2024 11:06:52 +0200
Subject: [PATCH 2410/2492] mgr/cephadm: add nvmeof configuration fields

Fixes: https://tracker.ceph.com/issues/64821

Signed-off-by: Gil Bregman <gbregman@il.ibm.com>
---
 src/pybind/mgr/cephadm/services/nvmeof.py     |  5 ++-
 .../services/nvmeof/ceph-nvmeof.conf.j2       | 41 +++++++++++++++----
 src/pybind/mgr/cephadm/tests/test_services.py | 34 ++++++++++++---
 3 files changed, 63 insertions(+), 17 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index 6cd1f4604d31..d133ad75d252 100644
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -43,8 +43,9 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
             'name': name,
             'addr': host_ip,
             'port': spec.port,
-            'log_level': 'WARN',
-            'rpc_socket': '/var/tmp/spdk.sock',
+            'spdk_log_level': 'WARNING',
+            'rpc_socket_dir': '/var/tmp/',
+            'rpc_socket_name': 'spdk.sock',
             'transport_tcp_options': transport_tcp_options,
             'rados_id': rados_id
         }
diff --git a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
index 711af0ee724a..fad99ad50e8f 100644
--- a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
@@ -3,16 +3,33 @@
 name = {{ name }}
 group = {{ spec.group if spec.group is not none else '' }}
 addr = {{ addr }}
-port = {{ port }}
+port = {{ spec.port }}
 enable_auth = {{ spec.enable_auth }}
-state_update_notify = True
-state_update_interval_sec = 5
-min_controller_id = {{ spec.min_controller_id }}
-max_controller_id = {{ spec.max_controller_id }}
+state_update_notify = {{ spec.state_update_notify }}
+state_update_interval_sec = {{ spec.state_update_interval_sec }}
 enable_spdk_discovery_controller = {{ spec.enable_spdk_discovery_controller }}
-enable_prometheus_exporter = True
+enable_prometheus_exporter = {{ spec.enable_prometheus_exporter }}
 prometheus_exporter_ssl = False
 prometheus_port = 10008
+verify_nqns = {{ spec.verify_nqns }}
+omap_file_lock_duration = {{ spec.omap_file_lock_duration }}
+omap_file_lock_retries = {{ spec.omap_file_lock_retries }}
+omap_file_lock_retry_sleep_interval = {{ spec.omap_file_lock_retry_sleep_interval }}
+omap_file_update_reloads = {{ spec.omap_file_update_reloads }}
+
+[gateway-logs]
+log_level = {{ spec.log_level }}
+log_files_enabled = {{ spec.log_files_enabled }}
+log_files_rotation_enabled = {{ spec.log_files_rotation_enabled }}
+verbose_log_messages = {{ spec.verbose_log_messages }}
+max_log_file_size_in_mb = {{ spec.max_log_file_size_in_mb }}
+max_log_files_count = {{ spec.max_log_files_count }}
+max_log_directory_backups = {{ spec.max_log_directory_backups }}
+log_directory = {{ spec.log_directory }}
+
+[discovery]
+addr = {{ addr }}
+port = {{ spec.discovery_port }}
 
 [ceph]
 pool = {{ spec.pool }}
@@ -27,9 +44,11 @@ client_cert = {{ spec.client_cert }}
 
 [spdk]
 tgt_path = {{ spec.tgt_path }}
-rpc_socket = {{ rpc_socket }}
-timeout = {{ spec.timeout }}
-log_level = {{ log_level }}
+rpc_socket_dir = {{ spec.rpc_socket_dir }}
+rpc_socket_name = {{ spec.rpc_socket_name }}
+timeout = {{ spec.spdk_timeout }}
+bdevs_per_cluster = {{ spec.bdevs_per_cluster }}
+log_level = {{ spec.spdk_log_level }}
 conn_retries = {{ spec.conn_retries }}
 transports = {{ spec.transports }}
 {% if transport_tcp_options %}
@@ -38,3 +57,7 @@ transport_tcp_options = {{ transport_tcp_options }}
 {% if spec.tgt_cmd_extra_args %}
 tgt_cmd_extra_args = {{ spec.tgt_cmd_extra_args }}
 {% endif %}
+
+[monitor]
+timeout = {{ spec.monitor_timeout }}
+
diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index cb7f618d37b4..48a9c3f06186 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -390,12 +390,29 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
 enable_auth = False
 state_update_notify = True
 state_update_interval_sec = 5
-min_controller_id = 1
-max_controller_id = 65519
 enable_spdk_discovery_controller = False
 enable_prometheus_exporter = True
 prometheus_exporter_ssl = False
 prometheus_port = 10008
+verify_nqns = True
+omap_file_lock_duration = 60
+omap_file_lock_retries = 15
+omap_file_lock_retry_sleep_interval = 5
+omap_file_update_reloads = 10
+
+[gateway-logs]
+log_level = INFO
+log_files_enabled = True
+log_files_rotation_enabled = True
+verbose_log_messages = True
+max_log_file_size_in_mb = 10
+max_log_files_count = 20
+max_log_directory_backups = 10
+log_directory = /var/log/ceph/
+
+[discovery]
+addr = 192.168.100.100
+port = 8009
 
 [ceph]
 pool = {pool}
@@ -410,13 +427,18 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
 
 [spdk]
 tgt_path = /usr/local/bin/nvmf_tgt
-rpc_socket = /var/tmp/spdk.sock
-timeout = 60
-log_level = WARN
+rpc_socket_dir = /var/tmp/
+rpc_socket_name = spdk.sock
+timeout = 60.0
+bdevs_per_cluster = 32
+log_level = WARNING
 conn_retries = 10
 transports = tcp
 transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}}
-tgt_cmd_extra_args = {tgt_cmd_extra_args}\n"""
+tgt_cmd_extra_args = {tgt_cmd_extra_args}
+
+[monitor]
+timeout = 1.0\n"""
 
         with with_host(cephadm_module, 'test'):
             with with_service(cephadm_module, NvmeofServiceSpec(service_id=pool,

From a19f2b6e3c4b51b605be1b5683c0ddad2b9d959f Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Thu, 21 Mar 2024 16:13:48 +0000
Subject: [PATCH 2411/2492] mgr/dashboard: fix subvolume group edit size

Removed the 'infinite' string which was set when no value was provided

Fixes: https://tracker.ceph.com/issues/65049

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-subvolumegroup-form.component.ts                    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
index accbb61b7eee..ea677e5d6834 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-subvolumegroup-form/cephfs-subvolumegroup-form.component.ts
@@ -146,7 +146,6 @@ export class CephfsSubvolumegroupFormComponent extends CdForm implements OnInit
     const gid = this.subvolumegroupForm.getValue('gid');
     const mode = this.formatter.toOctalPermission(this.subvolumegroupForm.getValue('mode'));
     if (this.isEdit) {
-      const editSize = size === 0 ? 'infinite' : size;
       this.taskWrapper
         .wrapTaskAroundCall({
           task: new FinishedTask('cephfs/subvolume/group/' + URLVerbs.EDIT, {
@@ -156,7 +155,7 @@ export class CephfsSubvolumegroupFormComponent extends CdForm implements OnInit
             this.fsName,
             subvolumegroupName,
             pool,
-            String(editSize),
+            String(size),
             uid,
             gid,
             mode

From 2a95bc7746a55256bc681d7edf7bba142f116708 Mon Sep 17 00:00:00 2001
From: Avan Thakkar <athakkar@redhat.com>
Date: Fri, 8 Mar 2024 12:45:44 +0530
Subject: [PATCH 2412/2492] mgr/dashboard: add support for NFSv3 exports

Fixes: https://tracker.ceph.com/issues/64812
Signed-off-by: Avan Thakkar <athakkar@redhat.com>
---
 .../ceph/nfs/nfs-form/nfs-form.component.html | 23 +++++++++----
 .../nfs/nfs-form/nfs-form.component.spec.ts   | 32 +++++++++++++++++--
 .../ceph/nfs/nfs-form/nfs-form.component.ts   | 25 +++++++++++++--
 3 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.html
index 82c97e322958..b9593a5c2680 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.html
@@ -213,26 +213,37 @@
                  for="protocols"
                  i18n>NFS Protocol</label>
           <div class="cd-col-form-input">
+            <div class="custom-control custom-checkbox">
+              <input type="checkbox"
+                     class="custom-control-input"
+                     formControlName="protocolNfsv3"
+                     name="protocolNfsv3"
+                     id="protocolNfsv3">
+              <label for="protocolNfsv3"
+                     class="custom-control-label"
+                     i18n>NFSv3</label>
+            </div>
             <div class="custom-control custom-checkbox">
               <input type="checkbox"
                      class="custom-control-input"
                      formControlName="protocolNfsv4"
                      name="protocolNfsv4"
-                     id="protocolNfsv4"
-                     disabled>
-              <label i18n
+                     id="protocolNfsv4">
+              <label for="protocolNfsv4"
                      class="custom-control-label"
-                     for="protocolNfsv4">NFSv4</label>
+                     i18n>NFSv4</label>
             </div>
             <span class="invalid-feedback"
-                  *ngIf="nfsForm.showError('protocolNfsv4', formDir, 'required')"
+                  *ngIf="nfsForm.showError('protocolNfsv3', formDir, 'required') ||
+                  nfsForm.showError('protocolNfsv4', formDir, 'required')"
                   i18n>This field is required.</span>
+            <hr>
           </div>
         </div>
 
         <!-- Pseudo -->
         <div class="form-group row"
-             *ngIf="nfsForm.getValue('protocolNfsv4')">
+             *ngIf="nfsForm.getValue('protocolNfsv4') || nfsForm.getValue('protocolNfsv3')">
           <label class="cd-col-form-label"
                  for="pseudo">
             <span class="required"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts
index 62efec423d36..65267a157916 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts
@@ -99,6 +99,7 @@ describe('NfsFormComponent', () => {
       fsal: { fs_name: 'a', name: 'CEPH' },
       path: '/',
       protocolNfsv4: true,
+      protocolNfsv3: true,
       pseudo: '',
       sec_label_xattr: 'security.selinux',
       security_label: false,
@@ -121,8 +122,9 @@ describe('NfsFormComponent', () => {
     expect(component.nfsForm.get('cluster_id').disabled).toBeTruthy();
   });
 
-  it('should mark NFSv4 protocol as enabled always', () => {
+  it('should mark NFSv4 & NFSv3 protocols as enabled always', () => {
     expect(component.nfsForm.get('protocolNfsv4')).toBeTruthy();
+    expect(component.nfsForm.get('protocolNfsv3')).toBeTruthy();
   });
 
   it('should match backend squash values with ui values', () => {
@@ -142,6 +144,7 @@ describe('NfsFormComponent', () => {
         fsal: { name: 'CEPH', fs_name: 1 },
         path: '/foo',
         protocolNfsv4: true,
+        protocolNfsv3: true,
         pseudo: '/baz',
         squash: 'no_root_squash',
         transportTCP: true,
@@ -157,6 +160,31 @@ describe('NfsFormComponent', () => {
       component.nfsForm.patchValue({ export_id: 1 });
       component.submitAction();
 
+      const req = httpTesting.expectOne('api/nfs-ganesha/export/cluster1/1');
+      expect(req.request.method).toBe('PUT');
+      expect(req.request.body).toEqual({
+        access_type: 'RW',
+        clients: [],
+        cluster_id: 'cluster1',
+        export_id: 1,
+        fsal: { fs_name: 1, name: 'CEPH', sec_label_xattr: null },
+        path: '/foo',
+        protocols: [3, 4],
+        pseudo: '/baz',
+        security_label: false,
+        squash: 'no_root_squash',
+        transports: ['TCP', 'UDP']
+      });
+    });
+
+    it('should call update with selected nfs protocol', () => {
+      activatedRoute.setParams({ cluster_id: 'cluster1', export_id: '1' });
+      component.isEdit = true;
+      component.cluster_id = 'cluster1';
+      component.export_id = '1';
+      component.nfsForm.patchValue({ export_id: 1, protocolNfsv3: false });
+      component.submitAction();
+
       const req = httpTesting.expectOne('api/nfs-ganesha/export/cluster1/1');
       expect(req.request.method).toBe('PUT');
       expect(req.request.body).toEqual({
@@ -190,7 +218,7 @@ describe('NfsFormComponent', () => {
           sec_label_xattr: null
         },
         path: '/foo',
-        protocols: [4],
+        protocols: [3, 4],
         pseudo: '/baz',
         security_label: false,
         squash: 'no_root_squash',
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts
index 540b7bfe64be..0543a9eb7abb 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts
@@ -156,10 +156,23 @@ export class NfsFormComponent extends CdForm implements OnInit {
         })
       }),
       path: new UntypedFormControl('/'),
-      protocolNfsv4: new UntypedFormControl(true),
+      protocolNfsv3: new UntypedFormControl(true, {
+        validators: [
+          CdValidators.requiredIf({ protocolNfsv4: false }, (value: boolean) => {
+            return !value;
+          })
+        ]
+      }),
+      protocolNfsv4: new UntypedFormControl(true, {
+        validators: [
+          CdValidators.requiredIf({ protocolNfsv3: false }, (value: boolean) => {
+            return !value;
+          })
+        ]
+      }),
       pseudo: new UntypedFormControl('', {
         validators: [
-          CdValidators.requiredIf({ protocolNfsv4: true }),
+          CdValidators.requiredIf({ protocolNfsv4: true, protocolNfsv3: true }),
           Validators.pattern('^/[^><|&()]*$')
         ]
       }),
@@ -194,6 +207,7 @@ export class NfsFormComponent extends CdForm implements OnInit {
     }
 
     res.protocolNfsv4 = res.protocols.indexOf(4) !== -1;
+    res.protocolNfsv3 = res.protocols.indexOf(3) !== -1;
     delete res.protocols;
 
     res.transportTCP = res.transports.indexOf('TCP') !== -1;
@@ -471,11 +485,16 @@ export class NfsFormComponent extends CdForm implements OnInit {
     }
 
     requestModel.protocols = [];
+    if (requestModel.protocolNfsv3) {
+      requestModel.protocols.push(3);
+    }
     if (requestModel.protocolNfsv4) {
       requestModel.protocols.push(4);
-    } else {
+    }
+    if (!requestModel.protocolNfsv3 && !requestModel.protocolNfsv4) {
       requestModel.pseudo = null;
     }
+    delete requestModel.protocolNfsv3;
     delete requestModel.protocolNfsv4;
 
     requestModel.transports = [];

From 5d300db825316db25706111bbe577d41d41ab4b3 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Fri, 22 Mar 2024 08:12:16 +1000
Subject: [PATCH 2413/2492] doc/start: link to mon map command

Link to the "ceph mon stat" command when "Intro to Ceph" document first
mentions Monitor Maps.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/rados/operations/monitoring.rst |  2 ++
 doc/start/intro.rst                 | 14 +++++++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/doc/rados/operations/monitoring.rst b/doc/rados/operations/monitoring.rst
index 2343e7e1f59c..f9af6f03d16e 100644
--- a/doc/rados/operations/monitoring.rst
+++ b/doc/rados/operations/monitoring.rst
@@ -517,6 +517,8 @@ multiple monitors are running to ensure proper functioning of your Ceph
 cluster. Check monitor status regularly in order to ensure that all of the
 monitors are running.
 
+.. _display-mon-map:
+
 To display the monitor map, run the following command:
 
 .. prompt:: bash $
diff --git a/doc/start/intro.rst b/doc/start/intro.rst
index 9881a7205487..6a0d1d318a8f 100644
--- a/doc/start/intro.rst
+++ b/doc/start/intro.rst
@@ -27,13 +27,13 @@ The Ceph Metadata Server is necessary to run Ceph File System clients.
             |      OSDs     | | Monitors   | |  Managers  | |      MDSs     |
             +---------------+ +------------+ +------------+ +---------------+
 
-- **Monitors**: A :term:`Ceph Monitor` (``ceph-mon``) maintains maps
-  of the cluster state, including the monitor map, manager map, the
-  OSD map, the MDS map, and the CRUSH map.  These maps are critical 
-  cluster state required for Ceph daemons to coordinate with each other.  
-  Monitors are also responsible for managing authentication between 
-  daemons and clients.  At least three monitors are normally required 
-  for redundancy and high availability.
+- **Monitors**: A :term:`Ceph Monitor` (``ceph-mon``) maintains maps of the
+  cluster state, including the :ref:`monitor map<display-mon-map>`, manager
+  map, the OSD map, the MDS map, and the CRUSH map.  These maps are critical
+  cluster state required for Ceph daemons to coordinate with each other.
+  Monitors are also responsible for managing authentication between daemons and
+  clients.  At least three monitors are normally required for redundancy and
+  high availability.
 
 - **Managers**: A :term:`Ceph Manager` daemon (``ceph-mgr``) is
   responsible for keeping track of runtime metrics and the current

From b6fa001cdb49a4db6c469bb9f80b9baee7b91ae4 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sun, 3 Dec 2023 11:01:05 -0500
Subject: [PATCH 2414/2492] cephadm: add generic methods for sharing namespaces
 across containers

In the future, some sidecar containers will need to share namespaces
with the primary container (or each other). Make it easy to set this up
by creating a enable_shared_namespaces function and Namespace enum.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/container_types.py | 63 ++++++++++++++++++++++-
 src/cephadm/tests/test_util_funcs.py      | 62 ++++++++++++++++++++++
 2 files changed, 124 insertions(+), 1 deletion(-)

diff --git a/src/cephadm/cephadmlib/container_types.py b/src/cephadm/cephadmlib/container_types.py
index 01fbb41d2392..665c4d89652a 100644
--- a/src/cephadm/cephadmlib/container_types.py
+++ b/src/cephadm/cephadmlib/container_types.py
@@ -1,9 +1,11 @@
 # container_types.py - container instance wrapper types
 
 import copy
+import enum
+import functools
 import os
 
-from typing import Dict, List, Optional, Any, Union, Tuple, cast
+from typing import Dict, List, Optional, Any, Union, Tuple, Iterable, cast
 
 from .call_wrappers import call, call_throws, CallVerbosity
 from .constants import DEFAULT_TIMEOUT
@@ -599,3 +601,62 @@ def extract_uid_gid(
         raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}')
 
     raise RuntimeError('uid/gid not found')
+
+
+@functools.lru_cache()
+def _opt_key(value: str) -> str:
+    """Return a (long) option stripped of its value."""
+    return value.split('=', 1)[0]
+
+
+def _replace_container_arg(args: List[str], new_arg: str) -> None:
+    """Remove and replace arguments that have the same `--xyz` part as
+    the given `new_arg`. If new_arg is expected to have a value it
+    must be part of the new_arg string following an equal sign (`=`).
+    The existing arg may be a single or two strings in the input list.
+    """
+    key = _opt_key(new_arg)
+    has_value = key != new_arg
+    try:
+        idx = [_opt_key(v) for v in args].index(key)
+        if '=' in args[idx] or not has_value:
+            del args[idx]
+        else:
+            del args[idx]
+            del args[idx]
+    except ValueError:
+        pass
+    args.append(new_arg)
+
+
+class Namespace(enum.Enum):
+    """General container namespace control options."""
+
+    cgroupns = 'cgroupns'
+    cgroup = 'cgroupns'  # alias
+    ipc = 'ipc'
+    network = 'network'
+    pid = 'pid'
+    userns = 'userns'
+    user = 'userns'  # alias
+    uts = 'uts'
+
+    def to_option(self, value: str) -> str:
+        return f'--{self}={value}'
+
+    def __str__(self) -> str:
+        return self.value
+
+
+def enable_shared_namespaces(
+    args: List[str],
+    name: str,
+    ns: Iterable[Namespace],
+) -> None:
+    """Update the args list to contain options that enable container namespace
+    sharing where name is the name/id of the target container and ns is a list
+    or set of namespaces that should be shared.
+    """
+    cc = f'container:{name}'
+    for n in ns:
+        _replace_container_arg(args, n.to_option(cc))
diff --git a/src/cephadm/tests/test_util_funcs.py b/src/cephadm/tests/test_util_funcs.py
index 6b5380711f33..aa64d54b0734 100644
--- a/src/cephadm/tests/test_util_funcs.py
+++ b/src/cephadm/tests/test_util_funcs.py
@@ -906,3 +906,65 @@ def test_daemon_sub_id_systemd_names():
     )
     with pytest.raises(ValueError):
         dsi.service_name
+
+
+@pytest.mark.parametrize(
+    "args,new_arg,expected",
+    [
+        (['--foo=77'], '--bar', ['--foo=77', '--bar']),
+        (['--foo=77'], '--foo=12', ['--foo=12']),
+        (
+            ['--foo=77', '--quux=later', '--range=2-5'],
+            '--quux=now',
+            ['--foo=77', '--range=2-5', '--quux=now'],
+        ),
+        (
+            ['--foo=77', '--quux', 'later', '--range=2-5'],
+            '--quux=now',
+            ['--foo=77', '--range=2-5', '--quux=now'],
+        ),
+        (
+            ['--foo=77', '--quux', 'later', '--range=2-5'],
+            '--jiffy',
+            ['--foo=77', '--quux', 'later', '--range=2-5', '--jiffy'],
+        ),
+        (
+            ['--foo=77', '--quux=buff', '--range=2-5'],
+            '--quux',
+            ['--foo=77', '--range=2-5', '--quux'],
+        ),
+    ],
+)
+def test_replace_container_args(args, new_arg, expected):
+    from cephadmlib.container_types import _replace_container_arg
+
+    _args = list(args)  # preserve the input so test input is not mutated
+    _replace_container_arg(_args, new_arg)
+    assert _args == expected
+
+
+
+def test_enable_shared_namespaces():
+    from cephadmlib.container_types import enable_shared_namespaces, Namespace
+
+    args = []
+    enable_shared_namespaces(args, 'c001d00d', {Namespace.ipc})
+    assert args == ['--ipc=container:c001d00d']
+
+    enable_shared_namespaces(
+        args, 'c001d00d', [Namespace.uts, Namespace.network]
+    )
+    assert args == [
+        '--ipc=container:c001d00d',
+        '--uts=container:c001d00d',
+        '--network=container:c001d00d',
+    ]
+
+    enable_shared_namespaces(
+        args, 'badd33d5', [Namespace.network]
+    )
+    assert args == [
+        '--ipc=container:c001d00d',
+        '--uts=container:c001d00d',
+        '--network=container:badd33d5',
+    ]

From d373edf0d6126ab6672f115e439d5bc14f983336 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 21 Nov 2023 17:11:37 -0500
Subject: [PATCH 2415/2492] cephadm: add a default constant value for samba
 server container image

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/constants.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cephadm/cephadmlib/constants.py b/src/cephadm/cephadmlib/constants.py
index 6ca5d39ed9de..e5c7de8cc43d 100644
--- a/src/cephadm/cephadmlib/constants.py
+++ b/src/cephadm/cephadmlib/constants.py
@@ -18,6 +18,7 @@
 DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29'
 DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29'
 DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29'
+DEFAULT_SMB_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64'
 DEFAULT_REGISTRY = 'docker.io'  # normalize unqualified digests to this
 # ------------------------------------------------------------------------------
 

From 0169fd945e7a1d5ea9dc3969e257cd0ecce0c1c6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 6 Dec 2023 15:14:31 -0500
Subject: [PATCH 2416/2492] cephadm: add an SMB daemon module and classes

Add an incomplete but largely viable SMB/Samba container daemon form
implementation to cephadm. Currently unused but it lays out some of the
basics needed to create smb sharing using samba containers under cephadm
orchestration.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadm.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py
index cb86af7df406..6257fb11d131 100755
--- a/src/cephadm/cephadm.py
+++ b/src/cephadm/cephadm.py
@@ -173,6 +173,7 @@
     Keepalived,
     Monitoring,
     NFSGanesha,
+    SMB,
     SNMPGateway,
     Tracing,
     NodeProxy,
@@ -227,6 +228,7 @@ def get_supported_daemons():
     supported_daemons.append(SNMPGateway.daemon_type)
     supported_daemons.extend(Tracing.components)
     supported_daemons.append(NodeProxy.daemon_type)
+    supported_daemons.append(SMB.daemon_type)
     assert len(supported_daemons) == len(set(supported_daemons))
     return supported_daemons
 

From f86e7106a48bd64a36c67377c780c6cf1521bdb5 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 6 Dec 2023 15:14:32 -0500
Subject: [PATCH 2417/2492] cephadm: import and enable deployment of SMB daemon
 class

Enable the use of the SMB container daemon form class by importing, and
thus registering, it. Note that the only way to invoke this feature is
by hand rolling some JSON to feed to the `ceph _orch deploy` command.
Connecting this with the cephadm mgr module is left as a future task.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/__init__.py |   2 +
 src/cephadm/cephadmlib/daemons/smb.py      | 442 +++++++++++++++++++++
 2 files changed, 444 insertions(+)
 create mode 100644 src/cephadm/cephadmlib/daemons/smb.py

diff --git a/src/cephadm/cephadmlib/daemons/__init__.py b/src/cephadm/cephadmlib/daemons/__init__.py
index 29f150694832..1a9d2d568bcf 100644
--- a/src/cephadm/cephadmlib/daemons/__init__.py
+++ b/src/cephadm/cephadmlib/daemons/__init__.py
@@ -5,6 +5,7 @@
 from .monitoring import Monitoring
 from .nfs import NFSGanesha
 from .nvmeof import CephNvmeof
+from .smb import SMB
 from .snmp import SNMPGateway
 from .tracing import Tracing
 from .node_proxy import NodeProxy
@@ -20,6 +21,7 @@
     'Monitoring',
     'NFSGanesha',
     'OSD',
+    'SMB',
     'SNMPGateway',
     'Tracing',
     'NodeProxy',
diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py
new file mode 100644
index 000000000000..00103ac8e4b4
--- /dev/null
+++ b/src/cephadm/cephadmlib/daemons/smb.py
@@ -0,0 +1,442 @@
+import enum
+import json
+import pathlib
+import logging
+
+from typing import List, Dict, Tuple, Optional, Any
+
+from .. import context_getters
+from .. import daemon_form
+from .. import data_utils
+from .. import deployment_utils
+from .. import file_utils
+from ..constants import DEFAULT_SMB_IMAGE
+from ..container_daemon_form import ContainerDaemonForm, daemon_to_container
+from ..container_engines import Podman
+from ..container_types import (
+    CephContainer,
+    InitContainer,
+    Namespace,
+    SidecarContainer,
+    enable_shared_namespaces,
+)
+from ..context import CephadmContext
+from ..daemon_identity import DaemonIdentity, DaemonSubIdentity
+from ..deploy import DeploymentType
+from ..exceptions import Error
+from ..net_utils import EndPoint
+
+
+logger = logging.getLogger()
+
+
+class Features(enum.Enum):
+    DOMAIN = 'domain'
+    CLUSTERED = 'clustered'
+
+    @classmethod
+    def valid(cls, value: str) -> bool:
+        # workaround for older python versions
+        try:
+            cls(value)
+            return True
+        except ValueError:
+            return False
+
+
+class Config:
+    instance_id: str
+    source_config: str
+    samba_debug_level: int
+    debug_delay: int
+    domain_member: bool
+    clustered: bool
+    join_sources: List[str]
+    custom_dns: List[str]
+    smb_port: int
+    ceph_config_entity: str
+
+    def __init__(
+        self,
+        *,
+        instance_id: str,
+        source_config: str,
+        domain_member: bool,
+        clustered: bool,
+        samba_debug_level: int = 0,
+        debug_delay: int = 0,
+        join_sources: Optional[List[str]] = None,
+        custom_dns: Optional[List[str]] = None,
+        smb_port: int = 0,
+        ceph_config_entity: str = 'client.admin',
+    ) -> None:
+        self.instance_id = instance_id
+        self.source_config = source_config
+        self.domain_member = domain_member
+        self.clustered = clustered
+        self.samba_debug_level = samba_debug_level
+        self.debug_delay = debug_delay
+        self.join_sources = join_sources or []
+        self.custom_dns = custom_dns or []
+        self.smb_port = smb_port
+        self.ceph_config_entity = ceph_config_entity
+
+    def __str__(self) -> str:
+        return (
+            f'SMB Config[id={self.instance_id},'
+            f' source_config={self.source_config},'
+            f' domain_member={self.domain_member},'
+            f' clustered={self.clustered}]'
+        )
+
+
+class SambaContainerCommon:
+    def __init__(
+        self,
+        cfg: Config,
+    ) -> None:
+        self.cfg = cfg
+
+    def name(self) -> str:
+        raise NotImplementedError('samba container name')
+
+    def envs(self) -> Dict[str, str]:
+        cfg_uris = [self.cfg.source_config]
+        environ = {
+            'SAMBA_CONTAINER_ID': self.cfg.instance_id,
+            'SAMBACC_CONFIG': json.dumps(cfg_uris),
+        }
+        if self.cfg.ceph_config_entity:
+            environ['SAMBACC_CEPH_ID'] = f'name={self.cfg.ceph_config_entity}'
+        return environ
+
+    def envs_list(self) -> List[str]:
+        return [f'{k}={v}' for (k, v) in self.envs().items()]
+
+    def args(self) -> List[str]:
+        args = []
+        if self.cfg.samba_debug_level:
+            args.append(f'--samba-debug-level={self.cfg.samba_debug_level}')
+        if self.cfg.debug_delay:
+            args.append(f'--debug-delay={self.cfg.debug_delay}')
+        return args
+
+    def container_args(self) -> List[str]:
+        return []
+
+
+class SMBDContainer(SambaContainerCommon):
+    def name(self) -> str:
+        return 'smbd'
+
+    def args(self) -> List[str]:
+        return super().args() + ['run', 'smbd']
+
+    def container_args(self) -> List[str]:
+        cargs = []
+        if self.cfg.smb_port:
+            cargs.append(f'--publish={self.cfg.smb_port}:{self.cfg.smb_port}')
+        return cargs
+
+
+class WinbindContainer(SambaContainerCommon):
+    def name(self) -> str:
+        return 'winbindd'
+
+    def args(self) -> List[str]:
+        return super().args() + ['run', 'winbindd']
+
+
+class ConfigInitContainer(SambaContainerCommon):
+    def name(self) -> str:
+        return 'config'
+
+    def args(self) -> List[str]:
+        return super().args() + ['init']
+
+
+class MustJoinContainer(SambaContainerCommon):
+    def name(self) -> str:
+        return 'mustjoin'
+
+    def args(self) -> List[str]:
+        args = super().args() + ['must-join']
+        for join_src in self.cfg.join_sources:
+            args.append(f'-j{join_src}')
+        return args
+
+    def container_args(self) -> List[str]:
+        cargs = []
+        for dns in self.cfg.custom_dns:
+            cargs.append(f'--dns={dns}')
+        return cargs
+
+
+class ConfigWatchContainer(SambaContainerCommon):
+    def name(self) -> str:
+        return 'configwatch'
+
+    def args(self) -> List[str]:
+        return super().args() + ['update-config', '--watch']
+
+
+class ContainerLayout:
+    init_containers: List[SambaContainerCommon]
+    primary: SambaContainerCommon
+    supplemental: List[SambaContainerCommon]
+
+    def __init__(
+        self,
+        init_containers: List[SambaContainerCommon],
+        primary: SambaContainerCommon,
+        supplemental: List[SambaContainerCommon],
+    ) -> None:
+        self.init_containers = init_containers
+        self.primary = primary
+        self.supplemental = supplemental
+
+
+@daemon_form.register
+class SMB(ContainerDaemonForm):
+    """Provides a form for SMB containers."""
+
+    daemon_type = 'smb'
+    default_image = DEFAULT_SMB_IMAGE
+
+    @classmethod
+    def for_daemon_type(cls, daemon_type: str) -> bool:
+        return cls.daemon_type == daemon_type
+
+    def __init__(self, ctx: CephadmContext, ident: DaemonIdentity):
+        assert ident.daemon_type == self.daemon_type
+        self._identity = ident
+        self._instance_cfg: Optional[Config] = None
+        self._files: Dict[str, str] = {}
+        self._raw_configs: Dict[str, Any] = context_getters.fetch_configs(ctx)
+        self._config_keyring = context_getters.get_config_and_keyring(ctx)
+        self._cached_layout: Optional[ContainerLayout] = None
+        self.smb_port = 445
+        logger.debug('Created SMB ContainerDaemonForm instance')
+
+    def validate(self) -> None:
+        if self._instance_cfg is not None:
+            return
+
+        configs = self._raw_configs
+        instance_id = configs.get('cluster_id', '')
+        source_config = configs.get('config_uri', '')
+        join_sources = configs.get('join_sources', [])
+        custom_dns = configs.get('custom_dns', [])
+        instance_features = configs.get('features', [])
+        files = data_utils.dict_get(configs, 'files', {})
+        ceph_config_entity = configs.get('config_auth_entity', '')
+
+        if not instance_id:
+            raise Error('invalid instance (cluster) id')
+        if not source_config:
+            raise Error('invalid configuration source uri')
+        invalid_features = {
+            f for f in instance_features if not Features.valid(f)
+        }
+        if invalid_features:
+            raise Error(
+                f'invalid instance features: {", ".join(invalid_features)}'
+            )
+        if Features.CLUSTERED.value in instance_features:
+            raise NotImplementedError('clustered instance')
+
+        self._instance_cfg = Config(
+            instance_id=instance_id,
+            source_config=source_config,
+            join_sources=join_sources,
+            custom_dns=custom_dns,
+            domain_member=Features.DOMAIN.value in instance_features,
+            clustered=Features.CLUSTERED.value in instance_features,
+            samba_debug_level=6,
+            smb_port=self.smb_port,
+            ceph_config_entity=ceph_config_entity,
+        )
+        self._files = files
+        logger.debug('SMB Instance Config: %s', self._instance_cfg)
+        logger.debug('Configured files: %s', self._files)
+
+    @property
+    def _cfg(self) -> Config:
+        self.validate()
+        assert self._instance_cfg
+        return self._instance_cfg
+
+    @property
+    def instance_id(self) -> str:
+        return self._cfg.instance_id
+
+    @property
+    def source_config(self) -> str:
+        return self._cfg.source_config
+
+    @classmethod
+    def create(cls, ctx: CephadmContext, ident: DaemonIdentity) -> 'SMB':
+        return cls(ctx, ident)
+
+    @property
+    def identity(self) -> DaemonIdentity:
+        return self._identity
+
+    def uid_gid(self, ctx: CephadmContext) -> Tuple[int, int]:
+        return 0, 0
+
+    def config_and_keyring(
+        self, ctx: CephadmContext
+    ) -> Tuple[Optional[str], Optional[str]]:
+        return self._config_keyring
+
+    def _layout(self) -> ContainerLayout:
+        if self._cached_layout:
+            return self._cached_layout
+        init_ctrs: List[SambaContainerCommon] = []
+        ctrs: List[SambaContainerCommon] = []
+
+        init_ctrs.append(ConfigInitContainer(self._cfg))
+        ctrs.append(ConfigWatchContainer(self._cfg))
+
+        if self._cfg.domain_member:
+            init_ctrs.append(MustJoinContainer(self._cfg))
+            ctrs.append(WinbindContainer(self._cfg))
+
+        smbd = SMBDContainer(self._cfg)
+        self._cached_layout = ContainerLayout(init_ctrs, smbd, ctrs)
+        return self._cached_layout
+
+    def _to_init_container(
+        self, ctx: CephadmContext, smb_ctr: SambaContainerCommon
+    ) -> InitContainer:
+        volume_mounts: Dict[str, str] = {}
+        container_args: List[str] = smb_ctr.container_args()
+        self.customize_container_mounts(ctx, volume_mounts)
+        # XXX: is this needed? if so, can this be simplified
+        if isinstance(ctx.container_engine, Podman):
+            ctx.container_engine.update_mounts(ctx, volume_mounts)
+        identity = DaemonSubIdentity.from_parent(
+            self.identity, smb_ctr.name()
+        )
+        return InitContainer(
+            ctx,
+            entrypoint='',
+            image=ctx.image or self.default_image,
+            identity=identity,
+            args=smb_ctr.args(),
+            container_args=container_args,
+            envs=smb_ctr.envs_list(),
+            volume_mounts=volume_mounts,
+        )
+
+    def _to_sidecar_container(
+        self, ctx: CephadmContext, smb_ctr: SambaContainerCommon
+    ) -> SidecarContainer:
+        volume_mounts: Dict[str, str] = {}
+        container_args: List[str] = smb_ctr.container_args()
+        self.customize_container_mounts(ctx, volume_mounts)
+        shared_ns = {
+            Namespace.ipc,
+            Namespace.network,
+            Namespace.pid,
+        }
+        if isinstance(ctx.container_engine, Podman):
+            # XXX: is this needed? if so, can this be simplified
+            ctx.container_engine.update_mounts(ctx, volume_mounts)
+            # docker doesn't support sharing the uts namespace with other
+            # containers. It may not be entirely needed on podman but it gives
+            # me warm fuzzies to make sure it gets shared.
+            shared_ns.add(Namespace.uts)
+        enable_shared_namespaces(
+            container_args, self.identity.container_name, shared_ns
+        )
+        identity = DaemonSubIdentity.from_parent(
+            self.identity, smb_ctr.name()
+        )
+        return SidecarContainer(
+            ctx,
+            entrypoint='',
+            image=ctx.image or self.default_image,
+            identity=identity,
+            container_args=container_args,
+            args=smb_ctr.args(),
+            envs=smb_ctr.envs_list(),
+            volume_mounts=volume_mounts,
+            init=False,
+            remove=True,
+        )
+
+    def container(self, ctx: CephadmContext) -> CephContainer:
+        ctr = daemon_to_container(ctx, self, host_network=False)
+        # We want to share the IPC ns between the samba containers for one
+        # instance.  Cephadm's default, host ipc, is not what we want.
+        # Unsetting it works fine for podman but docker (on ubuntu 22.04) needs
+        # to be expliclty told that ipc of the primary container must be
+        # shareable.
+        ctr.ipc = 'shareable'
+        return deployment_utils.to_deployment_container(ctx, ctr)
+
+    def init_containers(self, ctx: CephadmContext) -> List[InitContainer]:
+        return [
+            self._to_init_container(ctx, smb_ctr)
+            for smb_ctr in self._layout().init_containers
+        ]
+
+    def sidecar_containers(
+        self, ctx: CephadmContext
+    ) -> List[SidecarContainer]:
+        return [
+            self._to_sidecar_container(ctx, smb_ctr)
+            for smb_ctr in self._layout().supplemental
+        ]
+
+    def customize_container_envs(
+        self, ctx: CephadmContext, envs: List[str]
+    ) -> None:
+        clayout = self._layout()
+        envs.extend(clayout.primary.envs_list())
+
+    def customize_process_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        clayout = self._layout()
+        args.extend(clayout.primary.args())
+
+    def customize_container_args(
+        self, ctx: CephadmContext, args: List[str]
+    ) -> None:
+        args.extend(self._layout().primary.container_args())
+
+    def customize_container_mounts(
+        self,
+        ctx: CephadmContext,
+        mounts: Dict[str, str],
+    ) -> None:
+        self.validate()
+        data_dir = pathlib.Path(self.identity.data_dir(ctx.data_dir))
+        etc_samba_ctr = str(data_dir / 'etc-samba-container')
+        lib_samba = str(data_dir / 'lib-samba')
+        run_samba = str(data_dir / 'run')
+        config = str(data_dir / 'config')
+        keyring = str(data_dir / 'keyring')
+        mounts[etc_samba_ctr] = '/etc/samba/container:z'
+        mounts[lib_samba] = '/var/lib/samba:z'
+        mounts[run_samba] = '/run:z'  # TODO: make this a shared tmpfs
+        mounts[config] = '/etc/ceph/ceph.conf:z'
+        mounts[keyring] = '/etc/ceph/keyring:z'
+
+    def customize_container_endpoints(
+        self, endpoints: List[EndPoint], deployment_type: DeploymentType
+    ) -> None:
+        if not any(ep.port == self.smb_port for ep in endpoints):
+            endpoints.append(EndPoint('0.0.0.0', self.smb_port))
+
+    def prepare_data_dir(self, data_dir: str, uid: int, gid: int) -> None:
+        self.validate()
+        ddir = pathlib.Path(data_dir)
+        file_utils.makedirs(ddir / 'etc-samba-container', uid, gid, 0o770)
+        file_utils.makedirs(ddir / 'lib-samba', uid, gid, 0o770)
+        file_utils.makedirs(ddir / 'run', uid, gid, 0o770)
+        if self._files:
+            file_utils.populate_files(data_dir, self._files, uid, gid)

From 3b0f33188abef97d8e6ff774b03947ba23e180d0 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 6 Dec 2023 15:14:32 -0500
Subject: [PATCH 2418/2492] cephadm: add a basic deployment test for an smb
 daemon

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/tests/test_deploy.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/cephadm/tests/test_deploy.py b/src/cephadm/tests/test_deploy.py
index 26d5c56b7563..0be62ad021eb 100644
--- a/src/cephadm/tests/test_deploy.py
+++ b/src/cephadm/tests/test_deploy.py
@@ -534,3 +534,33 @@ def test_deploy_and_rm_iscsi(cephadm_fs, funkypatch):
     assert not drop_in.exists()
     assert not drop_in.parent.exists()
     assert not tcmu_sidecar.exists()
+
+
+def test_deploy_smb_container(cephadm_fs, funkypatch):
+    mocks = _common_patches(funkypatch)
+    fsid = 'b01dbeef-701d-9abe-0000-e1e5a47004a7'
+    with with_cephadm_ctx([]) as ctx:
+        ctx.container_engine = mock_podman()
+        ctx.fsid = fsid
+        ctx.name = 'smb.b01s'
+        ctx.image = 'quay.io/essembee/samba-server:latest'
+        ctx.reconfig = False
+        ctx.config_blobs = {
+            'cluster_id': 'smb1',
+            'config_uri': 'http://localhost:9876/smb.json',
+            'config': 'SAMPLE',
+            'keyring': 'SOMETHING',
+        }
+        _cephadm._common_deploy(ctx)
+
+    basedir = pathlib.Path(f'/var/lib/ceph/{fsid}/smb.b01s')
+    assert basedir.is_dir()
+    with open(basedir / 'unit.run') as f:
+        runfile_lines = f.read().splitlines()
+    assert 'podman' in runfile_lines[-1]
+    assert runfile_lines[-1].endswith('quay.io/essembee/samba-server:latest --samba-debug-level=6 run smbd')
+    assert f'-v {basedir}/etc-samba-container:/etc/samba/container:z' in runfile_lines[-1]
+    assert f'-v {basedir}/lib-samba:/var/lib/samba:z' in runfile_lines[-1]
+    assert '-e SAMBA_CONTAINER_ID=smb1' in runfile_lines[-1]
+    assert '-e \'SAMBACC_CONFIG=["http://localhost:9876/smb.json"]\'' in runfile_lines[-1]
+    assert '--publish' in runfile_lines[-1]

From f8160ed11b046bc747bbf045173fb222688ed669 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 16 Jan 2024 15:37:27 -0500
Subject: [PATCH 2419/2492] cephadm: fix issue joining to ad by using a virtual
 hostname

The not-a-real-fqdn hostname that the containers got were causing
performance issues joining AD (and running testjoin and winbind).
Define a virtual hostname that can be passed in from the service or
automatically derived from the system's hostname.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/cephadm/cephadmlib/daemons/smb.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/smb.py b/src/cephadm/cephadmlib/daemons/smb.py
index 00103ac8e4b4..bd92b97e7f19 100644
--- a/src/cephadm/cephadmlib/daemons/smb.py
+++ b/src/cephadm/cephadmlib/daemons/smb.py
@@ -1,7 +1,8 @@
 import enum
 import json
-import pathlib
 import logging
+import pathlib
+import socket
 
 from typing import List, Dict, Tuple, Optional, Any
 
@@ -55,6 +56,7 @@ class Config:
     custom_dns: List[str]
     smb_port: int
     ceph_config_entity: str
+    vhostname: str
 
     def __init__(
         self,
@@ -69,6 +71,7 @@ def __init__(
         custom_dns: Optional[List[str]] = None,
         smb_port: int = 0,
         ceph_config_entity: str = 'client.admin',
+        vhostname: str = '',
     ) -> None:
         self.instance_id = instance_id
         self.source_config = source_config
@@ -80,6 +83,7 @@ def __init__(
         self.custom_dns = custom_dns or []
         self.smb_port = smb_port
         self.ceph_config_entity = ceph_config_entity
+        self.vhostname = vhostname
 
     def __str__(self) -> str:
         return (
@@ -90,6 +94,15 @@ def __str__(self) -> str:
         )
 
 
+def _container_dns_args(cfg: Config) -> List[str]:
+    cargs = []
+    for dns in cfg.custom_dns:
+        cargs.append(f'--dns={dns}')
+    if cfg.vhostname:
+        cargs.append(f'--hostname={cfg.vhostname}')
+    return cargs
+
+
 class SambaContainerCommon:
     def __init__(
         self,
@@ -136,6 +149,7 @@ def container_args(self) -> List[str]:
         cargs = []
         if self.cfg.smb_port:
             cargs.append(f'--publish={self.cfg.smb_port}:{self.cfg.smb_port}')
+        cargs.extend(_container_dns_args(self.cfg))
         return cargs
 
 
@@ -166,9 +180,7 @@ def args(self) -> List[str]:
         return args
 
     def container_args(self) -> List[str]:
-        cargs = []
-        for dns in self.cfg.custom_dns:
-            cargs.append(f'--dns={dns}')
+        cargs = _container_dns_args(self.cfg)
         return cargs
 
 
@@ -230,6 +242,7 @@ def validate(self) -> None:
         instance_features = configs.get('features', [])
         files = data_utils.dict_get(configs, 'files', {})
         ceph_config_entity = configs.get('config_auth_entity', '')
+        vhostname = configs.get('virtual_hostname', '')
 
         if not instance_id:
             raise Error('invalid instance (cluster) id')
@@ -244,6 +257,11 @@ def validate(self) -> None:
             )
         if Features.CLUSTERED.value in instance_features:
             raise NotImplementedError('clustered instance')
+        if not vhostname:
+            # if a virtual hostname is not provided, generate one by prefixing
+            # the cluster/instanced id to the system hostname
+            hname = socket.getfqdn()
+            vhostname = f'{instance_id}-{hname}'
 
         self._instance_cfg = Config(
             instance_id=instance_id,
@@ -255,6 +273,7 @@ def validate(self) -> None:
             samba_debug_level=6,
             smb_port=self.smb_port,
             ceph_config_entity=ceph_config_entity,
+            vhostname=vhostname,
         )
         self._files = files
         logger.debug('SMB Instance Config: %s', self._instance_cfg)

From 07b44900e8c8bef434e848bf7aa84d18d78d8bd6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 14:33:20 -0500
Subject: [PATCH 2420/2492] mgr/cephadm: fix test failure on newer python

Tests that touch this enum fail for me locally but pass in the CI. This
seems to be due to new enum related behavior in Python 3.11.
See: https://blog.pecar.me/python-enum
Instead of fixing it as suggested in the above blog, adding a __str__
method works on all python versions I care to know about.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py
index 3aedfbd86f00..1ba3e48454a2 100644
--- a/src/pybind/mgr/cephadm/utils.py
+++ b/src/pybind/mgr/cephadm/utils.py
@@ -57,6 +57,9 @@ class SpecialHostLabels(str, Enum):
     def to_json(self) -> str:
         return self.value
 
+    def __str__(self) -> str:
+        return self.value
+
 
 def name_to_config_section(name: str) -> ConfEntity:
     """

From 96456aaf46d13ee29529c4fb031a90f0e4a795a8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 14:37:17 -0500
Subject: [PATCH 2421/2492] mgr/orchestrator: clean up import style

In the seemingly never-ending fight against line continuations and just
blatting tons of stuff onto single lines another small victory is won.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/orchestrator/module.py | 33 +++++++++++++++++++++------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 96a2d9104012..3e80621ef838 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -26,13 +26,32 @@
 from mgr_module import MgrModule, HandleCommandResult, Option
 from object_format import Format
 
-from ._interface import OrchestratorClientMixin, DeviceLightLoc, _cli_read_command, \
-    raise_if_exception, _cli_write_command, OrchestratorError, \
-    NoOrchestrator, OrchestratorValidationError, NFSServiceSpec, \
-    RGWSpec, InventoryFilter, InventoryHost, HostSpec, CLICommandMeta, \
-    ServiceDescription, DaemonDescription, IscsiServiceSpec, json_to_generic_spec, \
-    GenericSpec, DaemonDescriptionStatus, SNMPGatewaySpec, MDSSpec, TunedProfileSpec, \
-    NvmeofServiceSpec
+from ._interface import (
+    CLICommandMeta,
+    DaemonDescription,
+    DaemonDescriptionStatus,
+    DeviceLightLoc,
+    GenericSpec,
+    HostSpec,
+    InventoryFilter,
+    InventoryHost,
+    IscsiServiceSpec,
+    MDSSpec,
+    NFSServiceSpec,
+    NoOrchestrator,
+    NvmeofServiceSpec,
+    OrchestratorClientMixin,
+    OrchestratorError,
+    OrchestratorValidationError,
+    RGWSpec,
+    SNMPGatewaySpec,
+    ServiceDescription,
+    TunedProfileSpec,
+    _cli_read_command,
+    _cli_write_command,
+    json_to_generic_spec,
+    raise_if_exception,
+)
 
 
 def nice_delta(now: datetime.datetime, t: Optional[datetime.datetime], suffix: str = '') -> str:

From 35028e15789dc3600143a434301625d094b24475 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 16:05:27 -0500
Subject: [PATCH 2422/2492] mgr/orchestrator: fix the sorting of the imports

While ceph doesn't enforce sorted imports I prefer them when possible. I
had once sorted these imports but then nvmeof came along an ruined
things. Put nvmeof back in it's place.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/orchestrator/_interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index f0fb2c429069..7e1b57466e3b 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -38,11 +38,11 @@ class Protocol:  # type: ignore
     IscsiServiceSpec,
     MDSSpec,
     NFSServiceSpec,
+    NvmeofServiceSpec,
     RGWSpec,
     SNMPGatewaySpec,
     ServiceSpec,
     TunedProfileSpec,
-    NvmeofServiceSpec
 )
 from ceph.deployment.drive_group import DriveGroupSpec
 from ceph.deployment.hostspec import HostSpec, SpecValidationError

From a500f42d1a6faf5dc2607dd3ec425c5af7585128 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 15:49:12 -0500
Subject: [PATCH 2423/2492] mgr/cephadm: reformat the _service_classes variable

Reformat the _service_classes variable so that it uses a multi-line list
with a single item on each line in a more black-ish style that is more
readable (especially if you use code-folding wisely).
Sort the list while we're at it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 33 ++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 697fad0f05ad..d30f2e5b5b6c 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -661,12 +661,33 @@ def __init__(self, *args: Any, **kwargs: Any):
         self.migration = Migrations(self)
 
         _service_classes: Sequence[Type[CephadmService]] = [
-            OSDService, NFSService, MonService, MgrService, MdsService,
-            RgwService, RbdMirrorService, GrafanaService, AlertmanagerService,
-            PrometheusService, NodeExporterService, LokiService, PromtailService, CrashService, IscsiService,
-            IngressService, CustomContainerService, CephfsMirrorService, NvmeofService,
-            CephadmAgent, CephExporterService, SNMPGatewayService, ElasticSearchService,
-            JaegerQueryService, JaegerAgentService, JaegerCollectorService, NodeProxy
+            AlertmanagerService,
+            CephExporterService,
+            CephadmAgent,
+            CephfsMirrorService,
+            CrashService,
+            CustomContainerService,
+            ElasticSearchService,
+            GrafanaService,
+            IngressService,
+            IscsiService,
+            JaegerAgentService,
+            JaegerCollectorService,
+            JaegerQueryService,
+            LokiService,
+            MdsService,
+            MgrService,
+            MonService,
+            NFSService,
+            NodeExporterService,
+            NodeProxy,
+            NvmeofService,
+            OSDService,
+            PrometheusService,
+            PromtailService,
+            RbdMirrorService,
+            RgwService,
+            SNMPGatewayService,
         ]
 
         # https://github.com/python/mypy/issues/8993

From 41e2b27817c783f3b4b142441ed827e1827482d6 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 5 Jan 2024 10:24:10 -0500
Subject: [PATCH 2424/2492] mgr/cephadm: refactor keyring simplification out of
 get_keyring_with_caps

Refactor get_keyring_with_caps such that the keyring simplification code
is moved into a new function that can be used in other locations.
get_keyring_with_caps will now call the new function to return the
simplified & consistent keyring output.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../mgr/cephadm/services/cephadmservice.py    | 35 ++++++++++---------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py
index d211bbaa309f..6e3ee927341c 100644
--- a/src/pybind/mgr/cephadm/services/cephadmservice.py
+++ b/src/pybind/mgr/cephadm/services/cephadmservice.py
@@ -55,6 +55,24 @@ def get_auth_entity(daemon_type: str, daemon_id: str, host: str = "") -> AuthEnt
         raise OrchestratorError(f"unknown daemon type {daemon_type}")
 
 
+def simplified_keyring(entity: str, contents: str) -> str:
+    # strip down keyring
+    #  - don't include caps (auth get includes them; get-or-create does not)
+    #  - use pending key if present
+    key = None
+    for line in contents.splitlines():
+        if ' = ' not in line:
+            continue
+        line = line.strip()
+        (ls, rs) = line.split(' = ', 1)
+        if ls == 'key' and not key:
+            key = rs
+        if ls == 'pending key':
+            key = rs
+    keyring = f'[{entity}]\nkey = {key}\n'
+    return keyring
+
+
 class CephadmDaemonDeploySpec:
     # typing.NamedTuple + Generic is broken in py36
     def __init__(self, host: str, daemon_id: str,
@@ -307,22 +325,7 @@ def get_keyring_with_caps(self, entity: AuthEntity, caps: List[str]) -> str:
             })
             if err:
                 raise OrchestratorError(f"Unable to fetch keyring for {entity}: {err}")
-
-        # strip down keyring
-        #  - don't include caps (auth get includes them; get-or-create does not)
-        #  - use pending key if present
-        key = None
-        for line in keyring.splitlines():
-            if ' = ' not in line:
-                continue
-            line = line.strip()
-            (ls, rs) = line.split(' = ', 1)
-            if ls == 'key' and not key:
-                key = rs
-            if ls == 'pending key':
-                key = rs
-        keyring = f'[{entity}]\nkey = {key}\n'
-        return keyring
+        return simplified_keyring(entity, keyring)
 
     def _inventory_get_fqdn(self, hostname: str) -> str:
         """Get a host's FQDN with its hostname.

From 4fc2697fb1e4dc71b480db275aa4e54c2b66d018 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 19:20:45 -0500
Subject: [PATCH 2425/2492] python-common: reformat ServiceSpec class level
 service type lists

Reformat the ServiceSpec classes properties KNOWN_SERVICE_TYPES and
REQUIRES_SERVICE_ID. These were previously strings that were converted
to lists via a call to split. With a string there's very little a human
or a tool can do to validate the content. Changing these into proper
lists in the source code brings clarity of intent and the ability to
analyze the code. Because there's no semantic difference what services
are listed where (this means the type could probably be a set - a quest
for another day) I also took the opportunity to sort the contents of the
lists and add some basic comments for what these lists are for.

It also removes the use of (ugly, IMO) line continuations. The downside
is that it makes more total lines, but if that bugs you - use code
folding :-).

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../ceph/deployment/service_spec.py           | 52 ++++++++++++++++---
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 24f5c646461b..c1fb11c60d78 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -753,12 +753,52 @@ class ServiceSpec(object):
     This structure is supposed to be enough information to
     start the services.
     """
-    KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi nvmeof loki promtail mds mgr mon nfs ' \
-                          'node-exporter osd prometheus rbd-mirror rgw agent ceph-exporter ' \
-                          'container ingress cephfs-mirror snmp-gateway jaeger-tracing ' \
-                          'elasticsearch jaeger-agent jaeger-collector jaeger-query ' \
-                          'node-proxy'.split()
-    REQUIRES_SERVICE_ID = 'iscsi nvmeof mds nfs rgw container ingress '.split()
+
+    # list of all service type names that a ServiceSpec can be cast info
+    KNOWN_SERVICE_TYPES = [
+        'agent',
+        'alertmanager',
+        'ceph-exporter',
+        'cephfs-mirror',
+        'container',
+        'crash',
+        'elasticsearch',
+        'grafana',
+        'ingress',
+        'iscsi',
+        'jaeger-agent',
+        'jaeger-collector',
+        'jaeger-query',
+        'jaeger-tracing',
+        'loki',
+        'mds',
+        'mgr',
+        'mon',
+        'nfs',
+        'node-exporter',
+        'node-proxy',
+        'nvmeof',
+        'osd',
+        'prometheus',
+        'promtail',
+        'rbd-mirror',
+        'rgw',
+        'snmp-gateway',
+    ]
+
+    # list of all service type names that require/get assigned a service_id value.
+    # if a service is not listed here it *will not* be assigned a service_id even
+    # if it is present in the JSON/YAML input
+    REQUIRES_SERVICE_ID = [
+        'container',
+        'ingress',
+        'iscsi',
+        'mds',
+        'nfs',
+        'nvmeof',
+        'rgw',
+    ]
+
     MANAGED_CONFIG_OPTIONS = [
         'mds_join_fs',
     ]

From 4f655c5e1894812ad983111276f188b4fd61aebe Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 16:10:37 -0500
Subject: [PATCH 2426/2492] python-common: define a new SMBSpec service spec
 type

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../ceph/deployment/service_spec.py           | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index c1fb11c60d78..5d2e410e58c9 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -783,6 +783,7 @@ class ServiceSpec(object):
         'promtail',
         'rbd-mirror',
         'rgw',
+        'smb',
         'snmp-gateway',
     ]
 
@@ -797,6 +798,7 @@ class ServiceSpec(object):
         'nfs',
         'nvmeof',
         'rgw',
+        'smb',
     ]
 
     MANAGED_CONFIG_OPTIONS = [
@@ -830,6 +832,7 @@ def _cls(cls: Type[ServiceSpecT], service_type: str) -> Type[ServiceSpecT]:
             'jaeger-collector': TracingSpec,
             'jaeger-query': TracingSpec,
             'jaeger-tracing': TracingSpec,
+            SMBSpec.service_type: SMBSpec,
         }.get(service_type, cls)
         if ret == ServiceSpec and not service_type:
             raise SpecValidationError('Spec needs a "service_type" key.')
@@ -2372,3 +2375,85 @@ def validate(self) -> None:
 
 
 yaml.add_representer(CephExporterSpec, ServiceSpec.yaml_representer)
+
+
+class SMBSpec(ServiceSpec):
+    service_type = 'smb'
+    _valid_features = {'domain'}
+
+    def __init__(
+        self,
+        # --- common service spec args ---
+        service_type: str = 'smb',
+        service_id: Optional[str] = None,
+        placement: Optional[PlacementSpec] = None,
+        count: Optional[int] = None,
+        config: Optional[Dict[str, str]] = None,
+        unmanaged: bool = False,
+        preview_only: bool = False,
+        networks: Optional[List[str]] = None,
+        # --- smb specific values ---
+        # cluster_id - a name identifying the smb "cluster" this daemon
+        # is part of. A cluster may be made up of one or more services
+        # sharing a common configuration.
+        cluster_id: str = '',
+        # features - a list of terms enabling specific deployment features.
+        # terms include: 'domain' to enable Active Dir. Domain membership.
+        features: Optional[List[str]] = None,
+        # config_uri - a pseudo-uri that resolves to a configuration source
+        # that the samba-container can load. A ceph based samba container will
+        # be typically storing configuration in rados (rados:// prefix)
+        config_uri: str = '',
+        # join_sources - a list of pseudo-uris that resolve to a (JSON) blob
+        # containing data the samba-container can use to join a domain. A ceph
+        # based samba container may typically use a rados uri or a mon
+        # config-key store uri (example:
+        # `rados:mon-config-key:smb/config/mycluster/join1.json`).
+        join_sources: Optional[List[str]] = None,
+        # custom_dns -  a list of IP addresses that will be set up as custom
+        # dns servers for the samba container.
+        custom_dns: Optional[List[str]] = None,
+        # include_ceph_users - A list of ceph auth entity names that will be
+        # automatically added to the ceph keyring provided to the samba
+        # container.
+        include_ceph_users: Optional[List[str]] = None,
+        # --- genearal tweaks ---
+        extra_container_args: Optional[GeneralArgList] = None,
+        extra_entrypoint_args: Optional[GeneralArgList] = None,
+        custom_configs: Optional[List[CustomConfig]] = None,
+    ) -> None:
+        if service_type != self.service_type:
+            raise ValueError(f'invalid service_type: {service_type!r}')
+        super().__init__(
+            self.service_type,
+            service_id=service_id,
+            placement=placement,
+            count=count,
+            config=config,
+            unmanaged=unmanaged,
+            preview_only=preview_only,
+            networks=networks,
+            extra_container_args=extra_container_args,
+            extra_entrypoint_args=extra_entrypoint_args,
+            custom_configs=custom_configs,
+        )
+        self.cluster_id = cluster_id
+        self.features = features or []
+        self.config_uri = config_uri
+        self.join_sources = join_sources or []
+        self.custom_dns = custom_dns or []
+        self.include_ceph_users = include_ceph_users or []
+        self.validate()
+
+    def validate(self) -> None:
+        if not self.cluster_id:
+            raise ValueError('a valid cluster_id is required')
+        if not self.config_uri:
+            raise ValueError('a valid config_uri is required')
+        if self.features:
+            invalid = set(self.features).difference(self._valid_features)
+            if invalid:
+                raise ValueError(f'invalid feature flags: {", ".join(invalid)}')
+
+
+yaml.add_representer(SMBSpec, ServiceSpec.yaml_representer)

From a88cf505a051298996b6be99d6d55a91a7684467 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 16:10:11 -0500
Subject: [PATCH 2427/2492] mgr/cephadm: add a new smb ceph service subclass

Will be used in a later commit to implement deploying smb instances.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/services/smb.py | 123 +++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 src/pybind/mgr/cephadm/services/smb.py

diff --git a/src/pybind/mgr/cephadm/services/smb.py b/src/pybind/mgr/cephadm/services/smb.py
new file mode 100644
index 000000000000..920c4ef02f74
--- /dev/null
+++ b/src/pybind/mgr/cephadm/services/smb.py
@@ -0,0 +1,123 @@
+import logging
+from typing import Any, Dict, Iterator, List, Tuple, cast
+
+from ceph.deployment.service_spec import ServiceSpec, SMBSpec
+
+from orchestrator import DaemonDescription
+from .cephadmservice import (
+    AuthEntity,
+    CephService,
+    CephadmDaemonDeploySpec,
+    simplified_keyring,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class SMBService(CephService):
+    TYPE = 'smb'
+
+    def config(self, spec: ServiceSpec) -> None:
+        assert self.TYPE == spec.service_type
+        logger.warning('config is a no-op')
+
+    def prepare_create(
+        self, daemon_spec: CephadmDaemonDeploySpec
+    ) -> CephadmDaemonDeploySpec:
+        assert self.TYPE == daemon_spec.daemon_type
+        logger.debug('smb prepare_create')
+        daemon_spec.final_config, daemon_spec.deps = self.generate_config(
+            daemon_spec
+        )
+        return daemon_spec
+
+    def generate_config(
+        self, daemon_spec: CephadmDaemonDeploySpec
+    ) -> Tuple[Dict[str, Any], List[str]]:
+        logger.debug('smb generate_config')
+        assert self.TYPE == daemon_spec.daemon_type
+        smb_spec = cast(
+            SMBSpec, self.mgr.spec_store[daemon_spec.service_name].spec
+        )
+        config_blobs: Dict[str, Any] = {}
+
+        config_blobs['cluster_id'] = smb_spec.cluster_id
+        config_blobs['features'] = smb_spec.features
+        config_blobs['config_uri'] = smb_spec.config_uri
+        if smb_spec.join_sources:
+            config_blobs['join_sources'] = smb_spec.join_sources
+        if smb_spec.custom_dns:
+            config_blobs['custom_dns'] = smb_spec.custom_dns
+        ceph_users = smb_spec.include_ceph_users or []
+        config_blobs.update(
+            self._ceph_config_and_keyring_for(
+                smb_spec, daemon_spec.daemon_id, ceph_users
+            )
+        )
+        logger.debug('smb generate_config: %r', config_blobs)
+        return config_blobs, []
+
+    def config_dashboard(
+        self, daemon_descrs: List[DaemonDescription]
+    ) -> None:
+        # TODO ???
+        logger.warning('config_dashboard is a no-op')
+
+    def get_auth_entity(self, daemon_id: str, host: str = "") -> AuthEntity:
+        # We want a clear, distinct auth entity for fetching the config versus
+        # data path access.
+        return AuthEntity(f'client.{self.TYPE}.config.{daemon_id}')
+
+    def _rados_uri_to_pool(self, uri: str) -> str:
+        """Given a psudo-uri possibly pointing to an object in a pool, return
+        the name of the pool if a rados uri, otherwise return an empty string.
+        """
+        if not uri.startswith('rados://'):
+            return ''
+        pool = uri[8:].lstrip('/').split('/')[0]
+        logger.debug('extracted pool %r from uri %r', pool, uri)
+        return pool
+
+    def _allow_config_key_command(self, name: str) -> str:
+        # permit the samba container config access to the mon config key store
+        # with keys like smb/config/<cluster_id>/*.
+        return f'allow command "config-key get" with "key" prefix "smb/config/{name}/"'
+
+    def _pools_in_spec(self, smb_spec: SMBSpec) -> Iterator[str]:
+        uris = [smb_spec.config_uri]
+        uris.extend(smb_spec.join_sources or [])
+        for uri in uris:
+            pool = self._rados_uri_to_pool(uri)
+            if pool:
+                yield pool
+
+    def _key_for_user(self, entity: str) -> str:
+        ret, keyring, err = self.mgr.mon_command({
+            'prefix': 'auth get',
+            'entity': entity,
+        })
+        if ret != 0:
+            raise ValueError(f'no auth key for user: {entity!r}')
+        return '\n' + simplified_keyring(entity, keyring)
+
+    def _ceph_config_and_keyring_for(
+        self, smb_spec: SMBSpec, daemon_id: str, ceph_users: List[str]
+    ) -> Dict[str, str]:
+        ackc = self._allow_config_key_command(smb_spec.cluster_id)
+        wanted_caps = ['mon', f'allow r, {ackc}']
+        pools = list(self._pools_in_spec(smb_spec))
+        if pools:
+            wanted_caps.append('osd')
+            wanted_caps.append(
+                ', '.join(f'allow r pool={pool}' for pool in pools)
+            )
+        entity = self.get_auth_entity(daemon_id)
+        keyring = self.get_keyring_with_caps(entity, wanted_caps)
+        # add additional data-path users to the ceph keyring
+        for ceph_user in ceph_users:
+            keyring += self._key_for_user(ceph_user)
+        return {
+            'config': self.mgr.get_minimal_ceph_conf(),
+            'keyring': keyring,
+            'config_auth_entity': entity,
+        }

From c5e4912fd5fd43e21525952ca1d295dac10e2bbe Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 4 Jan 2024 16:38:08 -0500
Subject: [PATCH 2428/2492] mgr/cepahdm: add various touch points to enable smb
 service

Add the smb service by name or by type to one of the many, many touch
points in the orchestrator and cephadm packages needed to get the
orchestrator aware of smb.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/module.py          |  9 +++++-
 src/pybind/mgr/cephadm/utils.py           |  4 +--
 src/pybind/mgr/orchestrator/_interface.py | 12 ++++++--
 src/pybind/mgr/orchestrator/module.py     | 37 +++++++++++++++++++++++
 4 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index d30f2e5b5b6c..8e8137591722 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -76,6 +76,7 @@
     NodeExporterService, SNMPGatewayService, LokiService, PromtailService
 from .services.jaeger import ElasticSearchService, JaegerAgentService, JaegerCollectorService, JaegerQueryService
 from .services.node_proxy import NodeProxy
+from .services.smb import SMBService
 from .schedule import HostAssignment
 from .inventory import Inventory, SpecStore, HostCache, AgentCache, EventStore, \
     ClientKeyringStore, ClientKeyringSpec, TunedProfileStore, NodeProxyCache
@@ -687,6 +688,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             PromtailService,
             RbdMirrorService,
             RgwService,
+            SMBService,
             SNMPGatewayService,
         ]
 
@@ -3275,7 +3277,8 @@ def _apply_service_spec(self, spec: ServiceSpec) -> str:
                 'elasticsearch': PlacementSpec(count=1),
                 'jaeger-agent': PlacementSpec(host_pattern='*'),
                 'jaeger-collector': PlacementSpec(count=1),
-                'jaeger-query': PlacementSpec(count=1)
+                'jaeger-query': PlacementSpec(count=1),
+                SMBService.TYPE: PlacementSpec(count=1),
             }
             spec.placement = defaults[spec.service_type]
         elif spec.service_type in ['mon', 'mgr'] and \
@@ -3405,6 +3408,10 @@ def apply_container(self, spec: ServiceSpec) -> str:
     def apply_snmp_gateway(self, spec: ServiceSpec) -> str:
         return self._apply(spec)
 
+    @handle_orch_error
+    def apply_smb(self, spec: ServiceSpec) -> str:
+        return self._apply(spec)
+
     @handle_orch_error
     def set_unmanaged(self, service_name: str, value: bool) -> str:
         return self.spec_store.set_unmanaged(service_name, value)
diff --git a/src/pybind/mgr/cephadm/utils.py b/src/pybind/mgr/cephadm/utils.py
index 1ba3e48454a2..3673fbf621cb 100644
--- a/src/pybind/mgr/cephadm/utils.py
+++ b/src/pybind/mgr/cephadm/utils.py
@@ -36,7 +36,7 @@ class CephadmNoImage(Enum):
 # these daemons do not use the ceph image. There are other daemons
 # that also don't use the ceph image, but we only care about those
 # that are part of the upgrade order here
-NON_CEPH_IMAGE_TYPES = MONITORING_STACK_TYPES + ['nvmeof']
+NON_CEPH_IMAGE_TYPES = MONITORING_STACK_TYPES + ['nvmeof', 'smb']
 
 # Used for _run_cephadm used for check-host etc that don't require an --image parameter
 cephadmNoImage = CephadmNoImage.token
@@ -66,7 +66,7 @@ def name_to_config_section(name: str) -> ConfEntity:
     Map from daemon names to ceph entity names (as seen in config)
     """
     daemon_type = name.split('.', 1)[0]
-    if daemon_type in ['rgw', 'rbd-mirror', 'nfs', 'crash', 'iscsi', 'ceph-exporter', 'nvmeof']:
+    if daemon_type in ['rgw', 'rbd-mirror', 'nfs', 'crash', 'iscsi', 'ceph-exporter', 'nvmeof', 'smb']:
         return ConfEntity('client.' + name)
     elif daemon_type in ['mon', 'osd', 'mds', 'mgr', 'client']:
         return ConfEntity(name)
diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py
index 7e1b57466e3b..f5ddbe6515b2 100644
--- a/src/pybind/mgr/orchestrator/_interface.py
+++ b/src/pybind/mgr/orchestrator/_interface.py
@@ -40,6 +40,7 @@ class Protocol:  # type: ignore
     NFSServiceSpec,
     NvmeofServiceSpec,
     RGWSpec,
+    SMBSpec,
     SNMPGatewaySpec,
     ServiceSpec,
     TunedProfileSpec,
@@ -582,6 +583,7 @@ def apply(self, specs: Sequence["GenericSpec"], no_overwrite: bool = False) -> L
             'ingress': self.apply_ingress,
             'snmp-gateway': self.apply_snmp_gateway,
             'host': self.add_host,
+            'smb': self.apply_smb,
         }
 
         def merge(l: OrchResult[List[str]], r: OrchResult[str]) -> OrchResult[List[str]]:  # noqa: E741
@@ -819,6 +821,10 @@ def apply_snmp_gateway(self, spec: SNMPGatewaySpec) -> OrchResult[str]:
         """Update an existing snmp gateway service"""
         raise NotImplementedError()
 
+    def apply_smb(self, spec: SMBSpec) -> OrchResult[str]:
+        """Update a smb gateway service"""
+        raise NotImplementedError()
+
     def apply_tuned_profiles(self, specs: List[TunedProfileSpec], no_overwrite: bool) -> OrchResult[str]:
         """Add or update an existing tuned profile"""
         raise NotImplementedError()
@@ -917,7 +923,8 @@ def daemon_type_to_service(dtype: str) -> str:
         'elasticsearch': 'elasticsearch',
         'jaeger-agent': 'jaeger-agent',
         'jaeger-collector': 'jaeger-collector',
-        'jaeger-query': 'jaeger-query'
+        'jaeger-query': 'jaeger-query',
+        'smb': 'smb',
     }
     return mapping[dtype]
 
@@ -951,7 +958,8 @@ def service_to_daemon_types(stype: str) -> List[str]:
         'jaeger-agent': ['jaeger-agent'],
         'jaeger-collector': ['jaeger-collector'],
         'jaeger-query': ['jaeger-query'],
-        'jaeger-tracing': ['elasticsearch', 'jaeger-query', 'jaeger-collector', 'jaeger-agent']
+        'jaeger-tracing': ['elasticsearch', 'jaeger-query', 'jaeger-collector', 'jaeger-agent'],
+        'smb': ['smb'],
     }
     return mapping[stype]
 
diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py
index 3e80621ef838..d1ba6ea86755 100644
--- a/src/pybind/mgr/orchestrator/module.py
+++ b/src/pybind/mgr/orchestrator/module.py
@@ -44,6 +44,7 @@
     OrchestratorError,
     OrchestratorValidationError,
     RGWSpec,
+    SMBSpec,
     SNMPGatewaySpec,
     ServiceDescription,
     TunedProfileSpec,
@@ -1820,6 +1821,42 @@ def _apply_jaeger(self,
         specs: List[ServiceSpec] = spec.get_tracing_specs()
         return self._apply_misc(specs, dry_run, format, no_overwrite)
 
+    @_cli_write_command('orch apply smb')
+    def _apply_smb(
+        self,
+        cluster_id: str,
+        config_uri: str,
+        features: str = '',
+        join_sources: Optional[List[str]] = None,
+        custom_dns: Optional[List[str]] = None,
+        include_ceph_users: Optional[List[str]] = None,
+        placement: Optional[str] = None,
+        unmanaged: bool = False,
+        dry_run: bool = False,
+        format: Format = Format.plain,
+        no_overwrite: bool = False,
+    ) -> HandleCommandResult:
+        """Apply an SMB network file system gateway service configuration."""
+
+        _features = features.replace(',', ' ').split()
+        spec = SMBSpec(
+            service_id=cluster_id,
+            placement=PlacementSpec.from_string(placement),
+            unmanaged=unmanaged,
+            preview_only=dry_run,
+            cluster_id=cluster_id,
+            features=_features,
+            config_uri=config_uri,
+            join_sources=join_sources,
+            custom_dns=custom_dns,
+            include_ceph_users=include_ceph_users,
+        )
+
+        spec.validate()  # force any validation exceptions to be caught correctly
+        # The previous comment makes no sense to JJM. But when in rome.
+
+        return self._apply_misc([spec], dry_run, format, no_overwrite)
+
     @_cli_write_command('orch set-unmanaged')
     def _set_unmanaged(self, service_name: str) -> HandleCommandResult:
         """Set 'unmanaged: true' for the given service name"""

From 3985325e6983e3440ebb73a5b328071c096ac027 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 15 Dec 2023 13:15:19 -0500
Subject: [PATCH 2429/2492] mgr/cephadm: add the samba container image for smb
 daemons

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 8e8137591722..8771c9a1e8c0 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -132,6 +132,7 @@ def os_exit_noop(status: int) -> None:
 DEFAULT_JAEGER_COLLECTOR_IMAGE = 'quay.io/jaegertracing/jaeger-collector:1.29'
 DEFAULT_JAEGER_AGENT_IMAGE = 'quay.io/jaegertracing/jaeger-agent:1.29'
 DEFAULT_JAEGER_QUERY_IMAGE = 'quay.io/jaegertracing/jaeger-query:1.29'
+DEFAULT_SAMBA_IMAGE = 'quay.io/samba.org/samba-server:devbuilds-centos-amd64'
 # ------------------------------------------------------------------------------
 
 
@@ -288,6 +289,11 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
             default=DEFAULT_JAEGER_QUERY_IMAGE,
             desc='Jaeger query container image',
         ),
+        Option(
+            'container_image_samba',
+            default=DEFAULT_SAMBA_IMAGE,
+            desc='Samba/SMB container image',
+        ),
         Option(
             'warn_on_stray_hosts',
             type='bool',
@@ -552,6 +558,7 @@ def __init__(self, *args: Any, **kwargs: Any):
             self.container_image_jaeger_agent = ''
             self.container_image_jaeger_collector = ''
             self.container_image_jaeger_query = ''
+            self.container_image_samba = ''
             self.warn_on_stray_hosts = True
             self.warn_on_stray_daemons = True
             self.warn_on_failed_host_check = True
@@ -1617,6 +1624,8 @@ def _get_container_image(self, daemon_name: str) -> Optional[str]:
             image = None
         elif daemon_type == 'snmp-gateway':
             image = self.container_image_snmp_gateway
+        elif daemon_type == SMBService.TYPE:
+            image = self.container_image_samba
         else:
             assert False, daemon_type
 

From 0847ee2ee4531f2bb02ee3cc2a290c4c10fe4330 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 5 Jan 2024 10:45:08 -0500
Subject: [PATCH 2430/2492] mgr/cephadm: simplify _get_container_image a bit

Because the "if-ladder" was only ever assigning a single variable with
a value it can be directly replaced by a dict & dict-lookup which is
much more succinct.
Also take the opportunity to sort the (non-comment) lines as there's
no meaning to the previous order and this makes it easier for a reader
to scan through.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/module.py | 61 +++++++++++++-------------------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index 8771c9a1e8c0..b93cb7f3096f 100644
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1591,43 +1591,32 @@ def _get_container_image(self, daemon_name: str) -> Optional[str]:
                 utils.name_to_config_section(daemon_name),
                 'container_image'
             )).strip()
-        elif daemon_type == 'prometheus':
-            image = self.container_image_prometheus
-        elif daemon_type == 'nvmeof':
-            image = self.container_image_nvmeof
-        elif daemon_type == 'grafana':
-            image = self.container_image_grafana
-        elif daemon_type == 'alertmanager':
-            image = self.container_image_alertmanager
-        elif daemon_type == 'node-exporter':
-            image = self.container_image_node_exporter
-        elif daemon_type == 'loki':
-            image = self.container_image_loki
-        elif daemon_type == 'promtail':
-            image = self.container_image_promtail
-        elif daemon_type == 'haproxy':
-            image = self.container_image_haproxy
-        elif daemon_type == 'keepalived':
-            image = self.container_image_keepalived
-        elif daemon_type == 'elasticsearch':
-            image = self.container_image_elasticsearch
-        elif daemon_type == 'jaeger-agent':
-            image = self.container_image_jaeger_agent
-        elif daemon_type == 'jaeger-collector':
-            image = self.container_image_jaeger_collector
-        elif daemon_type == 'jaeger-query':
-            image = self.container_image_jaeger_query
-        elif daemon_type == CustomContainerService.TYPE:
-            # The image can't be resolved, the necessary information
-            # is only available when a container is deployed (given
-            # via spec).
-            image = None
-        elif daemon_type == 'snmp-gateway':
-            image = self.container_image_snmp_gateway
-        elif daemon_type == SMBService.TYPE:
-            image = self.container_image_samba
         else:
-            assert False, daemon_type
+            images = {
+                'alertmanager': self.container_image_alertmanager,
+                'elasticsearch': self.container_image_elasticsearch,
+                'grafana': self.container_image_grafana,
+                'haproxy': self.container_image_haproxy,
+                'jaeger-agent': self.container_image_jaeger_agent,
+                'jaeger-collector': self.container_image_jaeger_collector,
+                'jaeger-query': self.container_image_jaeger_query,
+                'keepalived': self.container_image_keepalived,
+                'loki': self.container_image_loki,
+                'node-exporter': self.container_image_node_exporter,
+                'nvmeof': self.container_image_nvmeof,
+                'prometheus': self.container_image_prometheus,
+                'promtail': self.container_image_promtail,
+                'snmp-gateway': self.container_image_snmp_gateway,
+                # The image can't be resolved here, the necessary information
+                # is only available when a container is deployed (given
+                # via spec).
+                CustomContainerService.TYPE: None,
+                SMBService.TYPE: self.container_image_samba,
+            }
+            try:
+                image = images[daemon_type]
+            except KeyError:
+                raise ValueError(f'no image for {daemon_type}')
 
         self.log.debug('%s container image %s' % (daemon_name, image))
 

From 9a58843dde69d2016be019d3e501d70022720e11 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 13 Dec 2023 19:36:46 -0500
Subject: [PATCH 2431/2492] mgr/cephadm: add some tests for the new smb service

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/pybind/mgr/cephadm/tests/test_services.py | 120 ++++++++++++++++++
 1 file changed, 120 insertions(+)

diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py
index cb7f618d37b4..48123b8ea174 100644
--- a/src/pybind/mgr/cephadm/tests/test_services.py
+++ b/src/pybind/mgr/cephadm/tests/test_services.py
@@ -17,6 +17,7 @@
 from cephadm.services.osd import OSDService
 from cephadm.services.monitoring import GrafanaService, AlertmanagerService, PrometheusService, \
     NodeExporterService, LokiService, PromtailService
+from cephadm.services.smb import SMBSpec
 from cephadm.module import CephadmOrchestrator
 from ceph.deployment.service_spec import (
     AlertManagerSpec,
@@ -2984,3 +2985,122 @@ def test_deploy_custom_container_with_init_ctrs(
                     [],
                     stdin=json.dumps(expected),
                 )
+
+
+class TestSMB:
+    @patch("cephadm.module.CephadmOrchestrator.get_unique_name")
+    @patch("cephadm.serve.CephadmServe._run_cephadm")
+    def test_deploy_smb(
+        self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator
+    ):
+        _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+        _get_uname.return_value = 'tango.briskly'
+
+        spec = SMBSpec(
+            cluster_id='foxtrot',
+            config_uri='rados://.smb/foxtrot/config.json',
+        )
+
+        expected = {
+            'fsid': 'fsid',
+            'name': 'smb.tango.briskly',
+            'image': '',
+            'deploy_arguments': [],
+            'params': {},
+            'meta': {
+                'service_name': 'smb',
+                'ports': [],
+                'ip': None,
+                'deployed_by': [],
+                'rank': None,
+                'rank_generation': None,
+                'extra_container_args': None,
+                'extra_entrypoint_args': None,
+            },
+            'config_blobs': {
+                'cluster_id': 'foxtrot',
+                'features': [],
+                'config_uri': 'rados://.smb/foxtrot/config.json',
+                'config': '',
+                'keyring': '[client.smb.config.tango.briskly]\nkey = None\n',
+                'config_auth_entity': 'client.smb.config.tango.briskly',
+            },
+        }
+        with with_host(cephadm_module, 'hostx'):
+            with with_service(cephadm_module, spec):
+                _run_cephadm.assert_called_with(
+                    'hostx',
+                    'smb.tango.briskly',
+                    ['_orch', 'deploy'],
+                    [],
+                    stdin=json.dumps(expected),
+                )
+
+    @patch("cephadm.module.CephadmOrchestrator.get_unique_name")
+    @patch("cephadm.serve.CephadmServe._run_cephadm")
+    def test_deploy_smb_join_dns(
+        self, _run_cephadm, _get_uname, cephadm_module: CephadmOrchestrator
+    ):
+        _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+        _get_uname.return_value = 'tango.briskly'
+
+        spec = SMBSpec(
+            cluster_id='foxtrot',
+            features=['domain'],
+            config_uri='rados://.smb/foxtrot/config2.json',
+            join_sources=[
+                'rados://.smb/foxtrot/join1.json',
+                'rados:mon-config-key:smb/config/foxtrot/join2.json',
+            ],
+            custom_dns=['10.8.88.103'],
+            include_ceph_users=[
+                'client.smb.fs.cephfs.share1',
+                'client.smb.fs.cephfs.share2',
+                'client.smb.fs.fs2.share3',
+            ],
+        )
+
+        expected = {
+            'fsid': 'fsid',
+            'name': 'smb.tango.briskly',
+            'image': '',
+            'deploy_arguments': [],
+            'params': {},
+            'meta': {
+                'service_name': 'smb',
+                'ports': [],
+                'ip': None,
+                'deployed_by': [],
+                'rank': None,
+                'rank_generation': None,
+                'extra_container_args': None,
+                'extra_entrypoint_args': None,
+            },
+            'config_blobs': {
+                'cluster_id': 'foxtrot',
+                'features': ['domain'],
+                'config_uri': 'rados://.smb/foxtrot/config2.json',
+                'join_sources': [
+                    'rados://.smb/foxtrot/join1.json',
+                    'rados:mon-config-key:smb/config/foxtrot/join2.json',
+                ],
+                'custom_dns': ['10.8.88.103'],
+                'config': '',
+                'keyring': (
+                    '[client.smb.config.tango.briskly]\nkey = None\n\n'
+                    '[client.smb.fs.cephfs.share1]\nkey = None\n\n'
+                    '[client.smb.fs.cephfs.share2]\nkey = None\n\n'
+                    '[client.smb.fs.fs2.share3]\nkey = None\n'
+                ),
+                'config_auth_entity': 'client.smb.config.tango.briskly',
+            },
+        }
+        with with_host(cephadm_module, 'hostx'):
+            with with_service(cephadm_module, spec):
+                _run_cephadm.assert_called_with(
+                    'hostx',
+                    'smb.tango.briskly',
+                    ['_orch', 'deploy'],
+                    [],
+                    stdin=json.dumps(expected),
+                )

From 4e897de3225aa8624c3e0188236f85f523088ea2 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 10 Feb 2024 09:06:53 -0500
Subject: [PATCH 2432/2492] doc/cephadm: add a file documenting the smb service

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 doc/cephadm/services/index.rst |   1 +
 doc/cephadm/services/smb.rst   | 205 +++++++++++++++++++++++++++++++++
 2 files changed, 206 insertions(+)
 create mode 100644 doc/cephadm/services/smb.rst

diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst
index c1da5d15f895..df8b3b421699 100644
--- a/doc/cephadm/services/index.rst
+++ b/doc/cephadm/services/index.rst
@@ -19,6 +19,7 @@ for details on individual services:
     monitoring
     snmp-gateway
     tracing
+    smb
 
 Service Status
 ==============
diff --git a/doc/cephadm/services/smb.rst b/doc/cephadm/services/smb.rst
new file mode 100644
index 000000000000..8ca70c5baf23
--- /dev/null
+++ b/doc/cephadm/services/smb.rst
@@ -0,0 +1,205 @@
+.. _deploy-cephadm-smb-samba:
+
+===========
+SMB Service
+===========
+
+.. note:: Only the SMB3 protocol is supported.
+
+.. warning::
+
+    SMB support is under active development and many features may be
+    missing or immature. Additionally, a Manager module to automate
+    SMB clusters and SMB shares is in development. Once that feature
+    is developed it will be the preferred method for managing
+    SMB on ceph.
+
+
+Deploying Samba Containers
+==========================
+
+Cephadm deploys `Samba <http://www.samba.org>`_ servers using container images
+built by the `samba-container project <http://github.com/samba-in-kubernetes/samba-container>`_.
+
+In order to host SMB Shares with access to CephFS file systems, deploy
+Samba Containers with the following command:
+
+.. prompt:: bash #
+
+    orch apply smb <cluster_id> <config_uri> [--features ...] [--placement ...] ...
+
+There are a number of additional parameters that the command accepts. See
+the Service Specification for a description of these options.
+
+Service Specification
+=====================
+
+An SMB Service can be applied using a specification. An example in YAML follows:
+
+.. code-block:: yaml
+
+    service_type: smb
+    service_id: tango
+    placement:
+      hosts:
+        - ceph0
+    spec:
+      cluster_id: tango
+      features:
+        - domain
+      config_uri: rados://.smb/tango/scc.toml
+      custom_dns:
+        - "192.168.76.204"
+      join_sources:
+        - "rados:mon-config-key:smb/config/tango/join1.json"
+      include_ceph_users:
+        - client.smb.fs.cluster.tango
+
+The specification can then be applied by running the following command:
+
+.. prompt:: bash #
+
+   ceph orch apply -i smb.yaml
+
+
+Service Spec Options
+--------------------
+
+Fields specific to the ``spec`` section of the SMB Service are described below.
+
+cluster_id
+    A short name identifying the SMB "cluster". In this case a cluster is
+    simply a management unit of one or more Samba services sharing a common
+    configuration, and may not provide actual clustering or availability
+    mechanisms.
+
+features
+    A list of pre-defined terms enabling specific deployment characteristics.
+    An empty list is valid. Supported terms:
+
+    * ``domain``: Enable domain member mode
+
+config_uri
+    A string containing a (standard or de-facto) URI that identifies a
+    configuration source that should be loaded by the samba-container as the
+    primary configuration file.
+    Supported URI schemes include ``http:``, ``https:``, ``rados:``, and
+    ``rados:mon-config-key:``.
+
+join_sources
+    A list of strings with (standard or de-facto) URI values that will
+    be used to identify where authentication data that will be used to
+    perform domain joins are located. Each join source is tried in sequence
+    until one succeeds.
+    See ``config_uri`` for the supported list of URI schemes.
+
+custom_dns
+    A list of IP addresses that will be used as the DNS servers for a Samba
+    container. This features allows Samba Containers to integrate with
+    Active Directory even if the Ceph host nodes are not tied into the Active
+    Directory DNS domain(s).
+
+include_ceph_users:
+    A list of cephx user (aka entity) names that the Samba Containers may use.
+    The cephx keys for each user in the list will automatically be added to
+    the keyring in the container.
+
+
+Configuring an SMB Service
+--------------------------
+
+.. warning::
+
+   A Manager module for SMB is under active development. Once that module
+   is available it will be the preferred method for managing Samba on Ceph
+   in an end-to-end manner. The following discussion is provided for the sake
+   of completeness and to explain how the software layers interact.
+
+Creating an SMB Service spec is not sufficient for complete operation of a
+Samba Container on Ceph. It is important to create valid configurations and
+place them in locations that the container can read. The complete specification
+of these configurations is out of scope for this document. You can refer to the
+`documentation for Samba <https://wiki.samba.org/index.php/Main_Page>`_ as
+well as the `samba server container
+<https://github.com/samba-in-kubernetes/samba-container/blob/master/docs/server.md>`_
+and the `configuation file
+<https://github.com/samba-in-kubernetes/sambacc/blob/master/docs/configuration.md>`_
+it accepts.
+
+When one has composed a configuration it should be stored in a location
+that the Samba Container can access. The recommended approach for running
+Samba Containers within Ceph orchestration is to store the configuration
+in the Ceph cluster. There are two ways to store the configuration
+in ceph:
+
+RADOS
+~~~~~
+
+A configuration file can be stored as a RADOS object in a pool
+named ``.smb``. Within the pool there should be a namespace named after the
+``cluster_id`` value. The URI used to identify this resource should be
+constructed like ``rados://.smb/<cluster_id>/<object_name>``. Example:
+``rados://.smb/tango/config.json``.
+
+The containers are automatically deployed with cephx keys allowing access to
+resources in these pools and namespaces. As long as this scheme is used
+no additional configuration to read the object is needed.
+
+To copy a configuration file to a RADOS pool, use the ``rados`` command line
+tool. For example:
+
+.. prompt:: bash #
+
+    # assuming your config file is /tmp/config.json
+    rados --pool=.smb --namespace=tango put config.json /tmp/config.json
+
+MON Key/Value Store
+~~~~~~~~~~~~~~~~~~~
+
+A configuration file can be stored as a value in the Ceph Monitor Key/Value
+store.  The key must be named after the cluster like so:
+``smb/config/<cluster_id>/<name>``.  This results in a URI that can be used to
+identify this configuration constructed like
+``rados:mon-config-key:smb/config/<cluster_id>/<name>``.
+Example: ``rados:mon-config-key:smb/config/tango/config.json``.
+
+The containers are automatically deployed with cephx keys allowing access to
+resources with the key-prefix ``smb/config/<cluster_id>/``. As long as this
+scheme is used no additional configuration to read the value is needed.
+
+To copy a configuration file into the Key/Value store use the ``ceph config-key
+put ...`` tool. For example:
+
+.. prompt:: bash #
+
+    # assuming your config file is /tmp/config.json
+    ceph config-key set smb/config/tango/config.json -i /tmp/config.json
+
+
+HTTP/HTTPS
+~~~~~~~~~~
+
+A configuration file can be stored on an HTTP(S) server and automatically read
+by the Samba Container. Managing a configuration file on HTTP(S) is left as an
+exercise for the reader.
+
+.. note:: All URI schemes are supported by parameters that accept URIs. Each
+   scheme has different performance and security characteristics.
+
+
+Limitations
+===========
+
+A non-exhaustive list of important limitations for the SMB service follows:
+
+* DNS is a critical component of Active Directory. If one is configuring the
+  SMB service for domain membership, either the Ceph host node must be
+  configured so that it can resolve the Active Directory (AD) domain or the
+  ``custom_dns`` option may be used. In both cases DNS hosts for the AD domain
+  must still be reachable from whatever network segment the ceph cluster is on.
+* Proper clustering/high-availability/"transparent state migration" is not yet
+  supported. If a placement causes more than service to be created these
+  services will act independently and may lead to unexpected behavior if clients
+  access the same files at once.
+* Services must bind to TCP port 445. Running multiple SMB services on the same
+  node is not yet supported and will trigger a port-in-use conflict.

From a99dc99589a0b81792abab1d14db6aad3ef9f2b8 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 20 Feb 2024 18:28:58 -0500
Subject: [PATCH 2433/2492] qa/tasks: add a new cephadm task for setting up
 samba ad dc

Add a new task function to cephadm.py that sets up a container running
the Samba based domain controller on a node using podman or docker.
Much of the function actually deals with disabling systemd-resolved
because that service conflicts with the DNS server component of the DC.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 250 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 250 insertions(+)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 248ce68e12f1..927cf6c4599f 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -9,6 +9,7 @@
 import logging
 import os
 import re
+import time
 import uuid
 import yaml
 
@@ -24,6 +25,7 @@
 from teuthology.orchestra import run
 from teuthology.orchestra.daemon import DaemonGroup
 from teuthology.config import config as teuth_config
+from teuthology.exceptions import ConfigError, CommandFailedError
 from textwrap import dedent
 from tasks.cephfs.filesystem import MDSCluster, Filesystem
 from tasks.util import chacra
@@ -1746,6 +1748,254 @@ def initialize_config(ctx, config):
     yield
 
 
+def _disable_systemd_resolved(ctx, remote):
+    r = remote.run(args=['ss', '-lunH'], stdout=StringIO())
+    # this heuristic tries to detect if systemd-resolved is running
+    if '%lo:53' not in r.stdout.getvalue():
+        return
+    log.info('Disabling systemd-resolved on %s', remote.shortname)
+    # Samba AD DC container DNS support conflicts with resolved stub
+    # resolver when using host networking. And we want host networking
+    # because it is the simplest thing to set up.  We therefore will turn
+    # off the stub resolver.
+    r = remote.run(
+        args=['sudo', 'cat', '/etc/systemd/resolved.conf'],
+        stdout=StringIO(),
+    )
+    resolved_conf = r.stdout.getvalue()
+    setattr(ctx, 'orig_resolved_conf', resolved_conf)
+    new_resolved_conf = (
+        resolved_conf + '\n# EDITED BY TEUTHOLOGY: deploy_samba_ad_dc\n'
+    )
+    if '[Resolve]' not in new_resolved_conf.splitlines():
+        new_resolved_conf += '[Resolve]\n'
+    new_resolved_conf += 'DNSStubListener=no\n'
+    remote.write_file(
+        path='/etc/systemd/resolved.conf',
+        data=new_resolved_conf,
+        sudo=True,
+    )
+    remote.run(args=['sudo', 'systemctl', 'restart', 'systemd-resolved'])
+    r = remote.run(args=['ss', '-lunH'], stdout=StringIO())
+    assert '%lo:53' not in r.stdout.getvalue()
+    # because docker is a big fat persistent deamon, we need to bounce it
+    # after resolved is restarted
+    remote.run(args=['sudo', 'systemctl', 'restart', 'docker'])
+
+
+def _reset_systemd_resolved(ctx, remote):
+    orig_resolved_conf = getattr(ctx, 'orig_resolved_conf', None)
+    if not orig_resolved_conf:
+        return  # no orig_resolved_conf means nothing to reset
+    log.info('Resetting systemd-resolved state on %s', remote.shortname)
+    remote.write_file(
+        path='/etc/systemd/resolved.conf',
+        data=orig_resolved_conf,
+        sudo=True,
+    )
+    remote.run(args=['sudo', 'systemctl', 'restart', 'systemd-resolved'])
+    setattr(ctx, 'orig_resolved_conf', None)
+
+
+def _samba_ad_dc_conf(ctx, remote, cengine):
+    # this config has not been tested outside of smithi nodes. it's possible
+    # that this will break when used elsewhere because we have to list
+    # interfaces explicitly. Later I may add a feature to sambacc to exclude
+    # known-unwanted interfaces that having to specify known good interfaces.
+    cf = {
+        "samba-container-config": "v0",
+        "configs": {
+            "demo": {
+                "instance_features": ["addc"],
+                "domain_settings": "sink",
+                "instance_name": "dc1",
+            }
+        },
+        "domain_settings": {
+            "sink": {
+                "realm": "DOMAIN1.SINK.TEST",
+                "short_domain": "DOMAIN1",
+                "admin_password": "Passw0rd",
+                "interfaces": {
+                    "exclude_pattern": "^docker[0-9]+$",
+                },
+            }
+        },
+        "domain_groups": {
+            "sink": [
+                {"name": "supervisors"},
+                {"name": "employees"},
+                {"name": "characters"},
+                {"name": "bulk"},
+            ]
+        },
+        "domain_users": {
+            "sink": [
+                {
+                    "name": "bwayne",
+                    "password": "1115Rose.",
+                    "given_name": "Bruce",
+                    "surname": "Wayne",
+                    "member_of": ["supervisors", "characters", "employees"],
+                },
+                {
+                    "name": "ckent",
+                    "password": "1115Rose.",
+                    "given_name": "Clark",
+                    "surname": "Kent",
+                    "member_of": ["characters", "employees"],
+                },
+                {
+                    "name": "user0",
+                    "password": "1115Rose.",
+                    "given_name": "George0",
+                    "surname": "Hue-Sir",
+                    "member_of": ["bulk"],
+                },
+                {
+                    "name": "user1",
+                    "password": "1115Rose.",
+                    "given_name": "George1",
+                    "surname": "Hue-Sir",
+                    "member_of": ["bulk"],
+                },
+                {
+                    "name": "user2",
+                    "password": "1115Rose.",
+                    "given_name": "George2",
+                    "surname": "Hue-Sir",
+                    "member_of": ["bulk"],
+                },
+                {
+                    "name": "user3",
+                    "password": "1115Rose.",
+                    "given_name": "George3",
+                    "surname": "Hue-Sir",
+                    "member_of": ["bulk"],
+                },
+            ]
+        },
+    }
+    cf_json = json.dumps(cf)
+    remote.run(args=['sudo', 'mkdir', '-p', '/var/tmp/samba'])
+    remote.write_file(
+        path='/var/tmp/samba/container.json', data=cf_json, sudo=True
+    )
+    return [
+        '--volume=/var/tmp/samba:/etc/samba-container:ro',
+        '-eSAMBACC_CONFIG=/etc/samba-container/container.json',
+    ]
+
+
+@contextlib.contextmanager
+def deploy_samba_ad_dc(ctx, config):
+    role = config.get('role')
+    ad_dc_image = config.get(
+        'ad_dc_image', 'quay.io/samba.org/samba-ad-server:latest'
+    )
+    samba_client_image = config.get(
+        'samba_client_image', 'quay.io/samba.org/samba-client:latest'
+    )
+    test_user_pass = config.get('test_user_pass', 'DOMAIN1\\ckent%1115Rose.')
+    if not role:
+        raise ConfigError(
+            "you must specify a role to allocate a host for the AD DC"
+        )
+    (remote,) = ctx.cluster.only(role).remotes.keys()
+    ip = remote.ssh.get_transport().getpeername()[0]
+    cengine = 'podman'
+    try:
+        log.info("Testing if podman is available")
+        remote.run(args=['sudo', cengine, '--help'])
+    except CommandFailedError:
+        log.info("Failed to find podman. Using docker")
+        cengine = 'docker'
+    remote.run(args=['sudo', cengine, 'pull', ad_dc_image])
+    remote.run(args=['sudo', cengine, 'pull', samba_client_image])
+    _disable_systemd_resolved(ctx, remote)
+    remote.run(
+        args=[
+            'sudo',
+            'mkdir',
+            '-p',
+            '/var/lib/samba/container/logs',
+            '/var/lib/samba/container/data',
+        ]
+    )
+    remote.run(
+        args=[
+            'sudo',
+            cengine,
+            'run',
+            '-d',
+            '--name=samba-ad',
+            '--network=host',
+            '--privileged',
+        ]
+        + _samba_ad_dc_conf(ctx, remote, cengine)
+        + [ad_dc_image]
+    )
+
+    # test that the ad dc is running and basically works
+    connected = False
+    samba_client_container_cmd = [
+        'sudo',
+        cengine,
+        'run',
+        '--rm',
+        '--net=host',
+        f'--dns={ip}',
+        '-eKRB5_CONFIG=/dev/null',
+        samba_client_image,
+    ]
+    for idx in range(10):
+        time.sleep((2 ** (1 + idx)) / 8)
+        log.info("Probing SMB status of DC %s, idx=%s", ip, idx)
+        cmd = samba_client_container_cmd + [
+            'smbclient',
+            '-U',
+            test_user_pass,
+            '//domain1.sink.test/sysvol',
+            '-c',
+            'ls',
+        ]
+        try:
+            remote.run(args=cmd)
+            connected = True
+            log.info("SMB status probe succeeded")
+            break
+        except CommandFailedError:
+            pass
+    if not connected:
+        raise RuntimeError('failed to connect to AD DC SMB share')
+
+    setattr(ctx, 'samba_ad_dc_ip', ip)
+    setattr(ctx, 'samba_client_container_cmd', samba_client_container_cmd)
+    try:
+        yield
+    finally:
+        try:
+            remote.run(args=['sudo', cengine, 'stop', 'samba-ad'])
+        except CommandFailedError:
+            log.error("Failed to stop samba-ad container")
+        try:
+            remote.run(args=['sudo', cengine, 'rm', 'samba-ad'])
+        except CommandFailedError:
+            log.error("Failed to remove samba-ad container")
+        remote.run(
+            args=[
+                'sudo',
+                'rm',
+                '-rf',
+                '/var/lib/samba/container/logs',
+                '/var/lib/samba/container/data',
+            ]
+        )
+        _reset_systemd_resolved(ctx, remote)
+        setattr(ctx, 'samba_ad_dc_ip', None)
+        setattr(ctx, 'samba_client_container_cmd', None)
+
+
 @contextlib.contextmanager
 def task(ctx, config):
     """

From 2a917e23ca6b3d2a4b90a40a07a1b5ae299b3924 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 24 Feb 2024 14:26:36 -0500
Subject: [PATCH 2434/2492] qa/tasks: allow passing stdin string to cephadm
 shell commands

There are cases where I want to pass some large-ish strings to ceph
commands executed via cephadm shell. Allow items within the commands
list to be dicts containing a command (as before) and an optional
stdin variable. This change also supports possible future extensions as
well.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 927cf6c4599f..daf36fb84cc5 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1421,10 +1421,17 @@ def shell(ctx, config):
         (remote,) = ctx.cluster.only(role).remotes.keys()
         log.info('Running commands on role %s host %s', role, remote.name)
         if isinstance(cmd, list):
-            for c in cmd:
-                _shell(ctx, cluster_name, remote,
-                       ['bash', '-c', c],
-                       extra_cephadm_args=args)
+            for cobj in cmd:
+                sh_cmd, stdin = _shell_command(cobj)
+                _shell(
+                    ctx,
+                    cluster_name,
+                    remote,
+                    ['bash', '-c', sh_cmd],
+                    extra_cephadm_args=args,
+                    stdin=stdin,
+                )
+
         else:
             assert isinstance(cmd, str)
             _shell(ctx, cluster_name, remote,
@@ -1432,6 +1439,16 @@ def shell(ctx, config):
                    extra_cephadm_args=args)
 
 
+def _shell_command(obj):
+    if isinstance(obj, str):
+        return obj, None
+    if isinstance(obj, dict):
+        cmd = obj['cmd']
+        stdin = obj.get('stdin', None)
+        return cmd, stdin
+    raise ValueError(f'invalid command item: {obj!r}')
+
+
 def apply(ctx, config):
     """
     Apply spec

From 3ec0bfa9eb5daf6b8b0101a50d0323918cdb7d31 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 26 Feb 2024 13:47:04 -0500
Subject: [PATCH 2435/2492] qa/tasks: add a cephadm.exclude role

Add a cephadm.exclude role that excludes a test node from cluster setup
and related commands. I need this as I have  test node that will be set
up as an AD Domain Controller for testing Samba and do not want that
node to be have *any* other services running on it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 52 ++++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index daf36fb84cc5..a7069c19fe7b 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -112,6 +112,19 @@ def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs):
     )
 
 
+def _cephadm_remotes(ctx, log_excluded=False):
+    out = []
+    for remote, roles in ctx.cluster.remotes.items():
+        if any(r.startswith('cephadm.exclude') for r in roles):
+            if log_excluded:
+                log.info(
+                    f'Remote {remote.shortname} excluded from cephadm cluster by role'
+                )
+            continue
+        out.append((remote, roles))
+    return out
+
+
 def build_initial_config(ctx, config):
     cluster_name = config['cluster']
 
@@ -138,7 +151,7 @@ def distribute_iscsi_gateway_cfg(ctx, conf_data):
     These will help in iscsi clients with finding trusted_ip_list.
     """
     log.info('Distributing iscsi-gateway.cfg...')
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         remote.write_file(
             path='/etc/ceph/iscsi-gateway.cfg',
             data=conf_data,
@@ -367,13 +380,14 @@ def _fetch_stable_branch_cephadm_from_chacra(ctx, config, cluster_name):
 
 def _rm_cluster(ctx, cluster_name):
     log.info('Removing cluster...')
-    ctx.cluster.run(args=[
-        'sudo',
-        ctx.cephadm,
-        'rm-cluster',
-        '--fsid', ctx.ceph[cluster_name].fsid,
-        '--force',
-    ])
+    for remote, _ in _cephadm_remotes(ctx):
+        remote.run(args=[
+            'sudo',
+            ctx.cephadm,
+            'rm-cluster',
+            '--fsid', ctx.ceph[cluster_name].fsid,
+            '--force',
+        ])
 
 
 def _rm_cephadm(ctx):
@@ -785,7 +799,7 @@ def ceph_bootstrap(ctx, config):
                    check_status=False)
 
         # add other hosts
-        for remote in ctx.cluster.remotes.keys():
+        for remote, roles in _cephadm_remotes(ctx, log_excluded=True):
             if remote == bootstrap_remote:
                 continue
 
@@ -869,7 +883,7 @@ def ceph_mons(ctx, config):
             # This is the old way of adding mons that works with the (early) octopus
             # cephadm scheduler.
             num_mons = 1
-            for remote, roles in ctx.cluster.remotes.items():
+            for remote, roles in _cephadm_remotes(ctx):
                 for mon in [r for r in roles
                             if teuthology.is_type('mon', cluster_name)(r)]:
                     c_, _, id_ = teuthology.split_role(mon)
@@ -908,7 +922,7 @@ def ceph_mons(ctx, config):
                                 break
         else:
             nodes = []
-            for remote, roles in ctx.cluster.remotes.items():
+            for remote, roles in _cephadm_remotes(ctx):
                 for mon in [r for r in roles
                             if teuthology.is_type('mon', cluster_name)(r)]:
                     c_, _, id_ = teuthology.split_role(mon)
@@ -982,7 +996,7 @@ def ceph_mgrs(ctx, config):
     try:
         nodes = []
         daemons = {}
-        for remote, roles in ctx.cluster.remotes.items():
+        for remote, roles in _cephadm_remotes(ctx):
             for mgr in [r for r in roles
                         if teuthology.is_type('mgr', cluster_name)(r)]:
                 c_, _, id_ = teuthology.split_role(mgr)
@@ -1027,7 +1041,7 @@ def ceph_osds(ctx, config):
         # provision OSDs in numeric order
         id_to_remote = {}
         devs_by_remote = {}
-        for remote, roles in ctx.cluster.remotes.items():
+        for remote, roles in _cephadm_remotes(ctx):
             devs_by_remote[remote] = teuthology.get_scratch_devices(remote)
             for osd in [r for r in roles
                         if teuthology.is_type('osd', cluster_name)(r)]:
@@ -1111,7 +1125,7 @@ def ceph_mdss(ctx, config):
 
     nodes = []
     daemons = {}
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         for role in [r for r in roles
                     if teuthology.is_type('mds', cluster_name)(r)]:
             c_, _, id_ = teuthology.split_role(role)
@@ -1188,7 +1202,7 @@ def ceph_monitoring(daemon_type, ctx, config):
 
     nodes = []
     daemons = {}
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         for role in [r for r in roles
                     if teuthology.is_type(daemon_type, cluster_name)(r)]:
             c_, _, id_ = teuthology.split_role(role)
@@ -1224,7 +1238,7 @@ def ceph_rgw(ctx, config):
 
     nodes = {}
     daemons = {}
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         for role in [r for r in roles
                     if teuthology.is_type('rgw', cluster_name)(r)]:
             c_, _, id_ = teuthology.split_role(role)
@@ -1267,7 +1281,7 @@ def ceph_iscsi(ctx, config):
     daemons = {}
     ips = []
 
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         for role in [r for r in roles
                      if teuthology.is_type('iscsi', cluster_name)(r)]:
             c_, _, id_ = teuthology.split_role(role)
@@ -1619,7 +1633,7 @@ def distribute_config_and_admin_keyring(ctx, config):
     """
     cluster_name = config['cluster']
     log.info('Distributing (final) config and client.admin keyring...')
-    for remote, roles in ctx.cluster.remotes.items():
+    for remote, roles in _cephadm_remotes(ctx):
         remote.write_file(
             '/etc/ceph/{}.conf'.format(cluster_name),
             ctx.ceph[cluster_name].config_file,
@@ -1712,7 +1726,7 @@ def initialize_config(ctx, config):
 
     # mon ips
     log.info('Choosing monitor IPs and ports...')
-    remotes_and_roles = ctx.cluster.remotes.items()
+    remotes_and_roles = _cephadm_remotes(ctx)
     ips = [host for (host, port) in
            (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)]
 

From 1ed66542ef27f5382d82b7cc60e76c2fe6d30fc9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 26 Feb 2024 16:16:57 -0500
Subject: [PATCH 2436/2492] qa/tasks: a new cephadm exec task similar to
 vip.exec but generalized

Add a new cephadm.exec task that works similarly to the existing
vip.exec but instead of only considering VIP related string replacements
it uses that templating feature that was recently added to the
cephadm module for generalized string templating.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index a7069c19fe7b..ca6e33d13169 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1463,6 +1463,41 @@ def _shell_command(obj):
     raise ValueError(f'invalid command item: {obj!r}')
 
 
+def exec(ctx, config):
+    """
+    This is similar to the standard 'exec' task, but does template substitutions.
+
+    TODO: this should probably be moved out of cephadm.py as it's pretty generic.
+    """
+    assert isinstance(config, dict), "task exec got invalid config"
+
+    testdir = teuthology.get_testdir(ctx)
+
+    if 'all-roles' in config and len(config) == 1:
+        a = config['all-roles']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles if not id_.startswith('host.'))
+    elif 'all-hosts' in config and len(config) == 1:
+        a = config['all-hosts']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
+
+    for role, ls in config.items():
+        (remote,) = ctx.cluster.only(role).remotes.keys()
+        log.info('Running commands on role %s host %s', role, remote.name)
+        for c in ls:
+            c.replace('$TESTDIR', testdir)
+            remote.run(
+                args=[
+                    'sudo',
+                    'TESTDIR={tdir}'.format(tdir=testdir),
+                    'bash',
+                    '-ex',
+                    '-c',
+                    _template_transform(ctx, config, c)],
+                )
+
+
 def apply(ctx, config):
     """
     Apply spec

From 361cbd46b9c5431452c742edd3ca7f0280e4161b Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 26 Feb 2024 16:17:22 -0500
Subject: [PATCH 2437/2492] qa/tasks: add a template filter to map a role name
 to a remote

Add a `role_to_remote` template filter function that has the ability to
map a role name to a remote. Attributes of the remote can then be
used to get the actual node ip or name.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index ca6e33d13169..69510c645e3a 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -75,6 +75,7 @@ def _template_transform(ctx, config, target):
     if jenv is None:
         loader = jinja2.BaseLoader()
         jenv = jinja2.Environment(loader=loader)
+        jenv.filters['role_to_remote'] = _role_to_remote
         setattr(ctx, '_jinja_env', jenv)
     rctx = dict(ctx=ctx, config=config, cluster_name=config.get('cluster', ''))
     _vip_vars(rctx)
@@ -94,6 +95,16 @@ def _vip_vars(rctx):
             rctx[f'VIP{idx}'] = str(vip)
 
 
+@jinja2.pass_context
+def _role_to_remote(rctx, role):
+    """Return the first remote matching the given role."""
+    ctx = rctx['ctx']
+    for remote, roles in ctx.cluster.remotes.items():
+        if role in roles:
+            return remote
+    return None
+
+
 def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs):
     teuthology.get_testdir(ctx)
     return remote.run(

From bf1607a4a14e92a745cd8d7e743e5f81b7d407b7 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 27 Feb 2024 09:44:51 -0500
Subject: [PATCH 2438/2492] qa/tasks: reduce duplicated code

All `exec`-style function in teuthology appear to have a transformation
block that expands names like `all-roles` and `all-hosts`. With the new
cephadm.exec task that block appeared twice in cephadm.py. This change
removes the duplication by creating an _expand_roles function that
can be called from the command executing functions.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 69510c645e3a..fdc9fdcd7392 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1420,6 +1420,18 @@ def stop(ctx, config):
     yield
 
 
+def _expand_roles(ctx, config):
+    if 'all-roles' in config and len(config) == 1:
+        a = config['all-roles']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles if not id_.startswith('host.'))
+    elif 'all-hosts' in config and len(config) == 1:
+        a = config['all-hosts']
+        roles = teuthology.all_roles(ctx.cluster)
+        config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
+    return config
+
+
 def shell(ctx, config):
     """
     Execute (shell) commands
@@ -1432,15 +1444,7 @@ def shell(ctx, config):
     for k in config.pop('volumes', []):
         args.extend(['-v', k])
 
-    if 'all-roles' in config and len(config) == 1:
-        a = config['all-roles']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if not id_.startswith('host.'))
-    elif 'all-hosts' in config and len(config) == 1:
-        a = config['all-hosts']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
-
+    config = _expand_roles(ctx, config)
     config = _template_transform(ctx, config, config)
     for role, cmd in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()
@@ -1481,18 +1485,8 @@ def exec(ctx, config):
     TODO: this should probably be moved out of cephadm.py as it's pretty generic.
     """
     assert isinstance(config, dict), "task exec got invalid config"
-
     testdir = teuthology.get_testdir(ctx)
-
-    if 'all-roles' in config and len(config) == 1:
-        a = config['all-roles']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if not id_.startswith('host.'))
-    elif 'all-hosts' in config and len(config) == 1:
-        a = config['all-hosts']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
-
+    config = _expand_roles(ctx, config)
     for role, ls in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()
         log.info('Running commands on role %s host %s', role, remote.name)

From 96704903f273ca0ee597dd819d9aadd1616625ed Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 27 Feb 2024 09:48:25 -0500
Subject: [PATCH 2439/2492] qa/tasks: add error condition to exec functions

Looking at the code that expands `all-roles` and `all-hosts` there's no
proper error checking for when these values appear but there are >1
top-level roles in the task config. If a user does this it'll fail
but in a somewhat unclear manner. Add a new condition that raises a
clear exception in this case hopefully saving someone future debugging
time.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index fdc9fdcd7392..66d51aff2f54 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1429,6 +1429,10 @@ def _expand_roles(ctx, config):
         a = config['all-hosts']
         roles = teuthology.all_roles(ctx.cluster)
         config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
+    elif 'all-roles' in config or 'all-hosts' in config:
+        raise ValueError(
+            'all-roles/all-hosts may not be combined with any other roles'
+        )
     return config
 
 

From 1f3001eef670a6bd1ff47cc11e459058a523b388 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Sat, 24 Feb 2024 10:52:53 -0500
Subject: [PATCH 2440/2492] qa/suites/orch: add a new smb service cephadm
 sub-suite and test

Start a new subdir under cephadm suite for the new smb service
that cephadm can deploy. Add one new test that checks that a
smb service with domain membership can be deployed and connect
to it with smbclient from the samba client container image.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/suites/orch/cephadm/smb/%                  |  0
 qa/suites/orch/cephadm/smb/.qa                |  1 +
 qa/suites/orch/cephadm/smb/0-distro           |  1 +
 qa/suites/orch/cephadm/smb/tasks/.qa          |  1 +
 .../cephadm/smb/tasks/deploy_smb_domain.yaml  | 88 +++++++++++++++++++
 5 files changed, 91 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/smb/%
 create mode 120000 qa/suites/orch/cephadm/smb/.qa
 create mode 120000 qa/suites/orch/cephadm/smb/0-distro
 create mode 120000 qa/suites/orch/cephadm/smb/tasks/.qa
 create mode 100644 qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml

diff --git a/qa/suites/orch/cephadm/smb/% b/qa/suites/orch/cephadm/smb/%
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/orch/cephadm/smb/.qa b/qa/suites/orch/cephadm/smb/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smb/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smb/0-distro b/qa/suites/orch/cephadm/smb/0-distro
new file mode 120000
index 000000000000..66187855738e
--- /dev/null
+++ b/qa/suites/orch/cephadm/smb/0-distro
@@ -0,0 +1 @@
+.qa/distros/supported-container-hosts
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smb/tasks/.qa b/qa/suites/orch/cephadm/smb/tasks/.qa
new file mode 120000
index 000000000000..fea2489fdf6d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smb/tasks/.qa
@@ -0,0 +1 @@
+../.qa
\ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml
new file mode 100644
index 000000000000..7662c9d6a468
--- /dev/null
+++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_domain.yaml
@@ -0,0 +1,88 @@
+roles:
+# Test is for basic smb deployment & functionality. one node cluster is OK
+- - host.a
+  - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - client.0
+# Reserve a host for acting as a domain controller
+- - host.b
+  - cephadm.exclude
+tasks:
+- cephadm.deploy_samba_ad_dc:
+    role: host.b
+- cephadm:
+
+- cephadm.shell:
+    host.a:
+      - ceph fs volume create cephfs
+- cephadm.wait_for_service:
+    service: mds.cephfs
+
+- cephadm.shell:
+    host.a:
+      # create a subvolume so we can verify that we're sharing something
+      - cmd: ceph fs subvolumegroup create cephfs g1
+      - cmd: ceph fs subvolume create cephfs sub1 --group-name=g1 --mode=0777
+      # Create a user access the file system from samba
+      - cmd: ceph fs authorize cephfs client.smbdata / rw
+      # Create a rados pool and store the config in it
+      - cmd: ceph osd pool create .smb --yes-i-really-mean-it
+      - cmd: ceph osd pool application enable .smb smb
+      - cmd: rados --pool=.smb --namespace=admem1 put conf.toml /dev/stdin
+        stdin: |
+          samba-container-config = "v0"
+          [configs.admem1]
+          shares = ["share1"]
+          globals = ["default", "domain"]
+          instance_name = "SAMBA"
+          [shares.share1.options]
+          "vfs objects" = "ceph"
+          path = "/"
+          "ceph:config_file" = "/etc/ceph/ceph.conf"
+          "ceph:user_id" = "smbdata"
+          "kernel share modes" = "no"
+          "read only" = "no"
+          "browseable" = "yes"
+          [globals.default.options]
+          "server min protocol" = "SMB2"
+          "load printers" = "no"
+          "printing" = "bsd"
+          "printcap name" = "/dev/null"
+          "disable spoolss" = "yes"
+          "guest ok" = "no"
+          [globals.domain.options]
+          security = "ads"
+          workgroup = "DOMAIN1"
+          realm = "domain1.sink.test"
+          "idmap config * : backend" = "autorid"
+          "idmap config * : range" = "2000-9999999"
+      # Store the join auth user/pass in the config-key store
+      - cmd: ceph config-key set smb/config/admem1/join1.json  -i -
+        stdin: |
+          {"username": "Administrator", "password": "Passw0rd"}
+
+- cephadm.apply:
+    specs:
+      - service_type: smb
+        service_id: admem1
+        placement:
+          count: 1
+        cluster_id: admem1
+        features:
+          - domain
+        config_uri: "rados://.smb/admem1/conf.toml"
+        custom_dns:
+          - "{{ctx.samba_ad_dc_ip}}"
+        join_sources:
+          - "rados:mon-config-key:smb/config/admem1/join1.json"
+        include_ceph_users:
+          - "client.smbdata"
+- cephadm.wait_for_service:
+    service: smb.admem1
+
+- cephadm.exec:
+    host.b:
+      - sleep 30
+      - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U DOMAIN1\\\\ckent%1115Rose. //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls"

From b2197e43b5eeb326b2e5498a06a2e13f4a532b87 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Fri, 15 Mar 2024 13:48:35 -0400
Subject: [PATCH 2441/2492] qa/tasks: add a cephadm samba container helper func
 independent of AD DC

To have the standalone (non-AD) server test function similarly to the AD
member server test we need to set a variable for samba client container
command similar to how the AD setup command does it.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 qa/tasks/cephadm.py | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 66d51aff2f54..008d8c94d55b 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -1962,6 +1962,44 @@ def _samba_ad_dc_conf(ctx, remote, cengine):
     ]
 
 
+@contextlib.contextmanager
+def configure_samba_client_container(ctx, config):
+    # TODO: deduplicate logic between this task and deploy_samba_ad_dc
+    role = config.get('role')
+    samba_client_image = config.get(
+        'samba_client_image', 'quay.io/samba.org/samba-client:latest'
+    )
+    if not role:
+        raise ConfigError(
+            "you must specify a role to discover container engine / pull image"
+        )
+    (remote,) = ctx.cluster.only(role).remotes.keys()
+    cengine = 'podman'
+    try:
+        log.info("Testing if podman is available")
+        remote.run(args=['sudo', cengine, '--help'])
+    except CommandFailedError:
+        log.info("Failed to find podman. Using docker")
+        cengine = 'docker'
+
+    remote.run(args=['sudo', cengine, 'pull', samba_client_image])
+    samba_client_container_cmd = [
+        'sudo',
+        cengine,
+        'run',
+        '--rm',
+        '--net=host',
+        '-eKRB5_CONFIG=/dev/null',
+        samba_client_image,
+    ]
+
+    setattr(ctx, 'samba_client_container_cmd', samba_client_container_cmd)
+    try:
+        yield
+    finally:
+        setattr(ctx, 'samba_client_container_cmd', None)
+
+
 @contextlib.contextmanager
 def deploy_samba_ad_dc(ctx, config):
     role = config.get('role')

From 8bb5fb69648f497da80c97011e171dff23c5130d Mon Sep 17 00:00:00 2001
From: Shachar Sharon <ssharon@redhat.com>
Date: Wed, 13 Mar 2024 16:43:29 +0200
Subject: [PATCH 2442/2492] qa/suites/orch: add minimal smb non-AD test

Test minimal SMB deployment over CephFS, using local users (non-AD).
Upon successful deployment run minima smbclient command ('ls') to probe
Samba's share liveness.

Co-authored-by: John Mulligan <jmulligan@redhat.com>
Signed-off-by: Shachar Sharon <ssharon@redhat.com>
Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 .../cephadm/smb/tasks/deploy_smb_basic.yaml   | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml

diff --git a/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml
new file mode 100644
index 000000000000..a64591ce8f82
--- /dev/null
+++ b/qa/suites/orch/cephadm/smb/tasks/deploy_smb_basic.yaml
@@ -0,0 +1,77 @@
+---
+roles:
+  # Test is for basic smb deployment & functionality. one node cluster is OK
+  - - host.a
+    - mon.a
+    - mgr.x
+    - osd.0
+    - osd.1
+    - client.0
+  # Reserve a host for acting as a test client
+  - - host.b
+    - cephadm.exclude
+tasks:
+  # TODO: (jjm) I don't think `install` is necessary for this file. Remove?
+  - install:
+  - cephadm.configure_samba_client_container:
+      role: host.b
+  - cephadm:
+  - cephadm.shell:
+      host.a:
+        - ceph fs volume create cephfs
+  - cephadm.wait_for_service:
+      service: mds.cephfs
+  - cephadm.shell:
+      host.a:
+        # create a subvolume so we can verify that we're sharing something
+        - cmd: ceph fs subvolumegroup create cephfs g1
+        - cmd: ceph fs subvolume create cephfs sub1 --group-name=g1 --mode=0777
+        # Create a user access the file system from samba
+        - cmd: ceph fs authorize cephfs client.smbdata / rw
+        # Create a rados pool and store the config in it
+        - cmd: ceph osd pool create .smb --yes-i-really-mean-it
+        - cmd: ceph osd pool application enable .smb smb
+        - cmd: rados --pool=.smb --namespace=saserv1 put conf.toml /dev/stdin
+          stdin: |
+            samba-container-config = "v0"
+            [configs.saserv1]
+            shares = ["share1"]
+            globals = ["default", "domain"]
+            instance_name = "SAMBA"
+            [shares.share1.options]
+            "vfs objects" = "ceph"
+            path = "/"
+            "ceph:config_file" = "/etc/ceph/ceph.conf"
+            "ceph:user_id" = "smbdata"
+            "kernel share modes" = "no"
+            "read only" = "no"
+            "browseable" = "yes"
+            [globals.default.options]
+            "server min protocol" = "SMB2"
+            "load printers" = "no"
+            "printing" = "bsd"
+            "printcap name" = "/dev/null"
+            "disable spoolss" = "yes"
+            "guest ok" = "no"
+            [globals.domain.options]
+            security = "USER"
+            workgroup = "STANDALONE1"
+            [[users.all_entries]]
+            name = "smbuser1"
+            password = "insecure321"
+  - cephadm.apply:
+      specs:
+        - service_type: smb
+          service_id: saserv1
+          placement:
+            count: 1
+          cluster_id: saserv1
+          config_uri: "rados://.smb/saserv1/conf.toml"
+          include_ceph_users:
+            - "client.smbdata"
+  - cephadm.wait_for_service:
+      service: smb.saserv1
+  - cephadm.exec:
+      host.b:
+        - sleep 30
+        - "{{ctx.samba_client_container_cmd|join(' ')}} smbclient -U smbuser1%insecure321 //{{'host.a'|role_to_remote|attr('ip_address')}}/share1 -c ls"

From 593ab1ad41467898c57c44baeb947bd238cba645 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Fri, 22 Mar 2024 03:12:09 +0000
Subject: [PATCH 2443/2492] doc/install: fix typos in openEuler-installation
 doc

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 doc/install/get-packages.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/install/get-packages.rst b/doc/install/get-packages.rst
index ca716831075f..a677bed68380 100644
--- a/doc/install/get-packages.rst
+++ b/doc/install/get-packages.rst
@@ -254,8 +254,8 @@ There's no need to add another package repository manually.
 openEuler
 ^^^^^^^^^
 
-There are two major versions supported in normal openEuler repositories. They are ceph 12.2.8 in openEuler-20.03-LTS series and ceph 16.2.7 in openEuler-22.03-LTS series. There’s no need to add another package repository manually.
-You can install ceph just by executing the following:
+There are two Ceph releases supported in normal openEuler repositories. They are Ceph 12.2.8 in the openEuler-20.03-LTS series and Ceph 16.2.7 in the openEuler-22.03-LTS series. There’s no need to add another package repository manually.
+You can install Ceph by executing the following:
 
 .. prompt:: bash $
 

From 7dcd74ac098201d2ea6f9d1c723f9944ade6bf33 Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xuxuehan@qianxin.com>
Date: Thu, 21 Mar 2024 09:54:42 +0800
Subject: [PATCH 2444/2492] crimson/os/seastore/btree: always check the
 stability of extents within the current transaction's view

Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
---
 src/crimson/os/seastore/btree/btree_range_pin.cc |  2 +-
 src/crimson/os/seastore/btree/fixed_kv_node.h    | 14 +++++++++-----
 src/crimson/os/seastore/cache.h                  |  9 +++++++++
 src/crimson/os/seastore/cached_extent.h          | 10 +++++-----
 4 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/crimson/os/seastore/btree/btree_range_pin.cc b/src/crimson/os/seastore/btree/btree_range_pin.cc
index 1fe79eafa817..3da1cac970ae 100644
--- a/src/crimson/os/seastore/btree/btree_range_pin.cc
+++ b/src/crimson/os/seastore/btree/btree_range_pin.cc
@@ -29,7 +29,7 @@ bool BtreeNodeMapping<key_t, val_t>::is_stable() const
   assert(parent->is_valid());
   assert(pos != std::numeric_limits<uint16_t>::max());
   auto &p = (FixedKVNode<key_t>&)*parent;
-  return p.is_child_stable(pos);
+  return p.is_child_stable(ctx, pos);
 }
 
 template class BtreeNodeMapping<laddr_t, paddr_t>;
diff --git a/src/crimson/os/seastore/btree/fixed_kv_node.h b/src/crimson/os/seastore/btree/fixed_kv_node.h
index de3569c0c5bc..ab8775ad8aa8 100644
--- a/src/crimson/os/seastore/btree/fixed_kv_node.h
+++ b/src/crimson/os/seastore/btree/fixed_kv_node.h
@@ -229,13 +229,14 @@ struct FixedKVNode : ChildableCachedExtent {
   virtual get_child_ret_t<LogicalCachedExtent>
   get_logical_child(op_context_t<node_key_t> c, uint16_t pos) = 0;
 
-  virtual bool is_child_stable(uint16_t pos) const = 0;
+  virtual bool is_child_stable(op_context_t<node_key_t>, uint16_t pos) const = 0;
 
   template <typename T, typename iter_t>
   get_child_ret_t<T> get_child(op_context_t<node_key_t> c, iter_t iter) {
     auto pos = iter.get_offset();
     assert(children.capacity());
     auto child = children[pos];
+    ceph_assert(!is_reserved_ptr(child));
     if (is_valid_child_ptr(child)) {
       ceph_assert(child->get_type() == T::TYPE);
       return c.cache.template get_extent_viewable_by_trans<T>(c.trans, (T*)child);
@@ -596,7 +597,7 @@ struct FixedKVInternalNode
     return get_child_ret_t<LogicalCachedExtent>(child_pos_t(nullptr, 0));
   }
 
-  bool is_child_stable(uint16_t pos) const final {
+  bool is_child_stable(op_context_t<NODE_KEY>, uint16_t pos) const final {
     ceph_abort("impossible");
     return false;
   }
@@ -999,13 +1000,16 @@ struct FixedKVLeafNode
   // 2. The child extent is stable
   //
   // For reserved mappings, the return values are undefined.
-  bool is_child_stable(uint16_t pos) const final {
+  bool is_child_stable(op_context_t<NODE_KEY> c, uint16_t pos) const final {
     auto child = this->children[pos];
     if (is_reserved_ptr(child)) {
       return true;
     } else if (is_valid_child_ptr(child)) {
       ceph_assert(child->is_logical());
-      return child->is_stable();
+      ceph_assert(
+	child->is_pending_in_trans(c.trans.get_trans_id())
+	|| this->is_stable_written());
+      return c.cache.is_viewable_extent_stable(c.trans, child);
     } else if (this->is_pending()) {
       auto key = this->iter_idx(pos).get_key();
       auto &sparent = this->get_stable_for_key(key);
@@ -1013,7 +1017,7 @@ struct FixedKVLeafNode
       auto child = sparent.children[spos];
       if (is_valid_child_ptr(child)) {
 	ceph_assert(child->is_logical());
-	return child->is_stable();
+	return c.cache.is_viewable_extent_stable(c.trans, child);
       } else {
 	return true;
       }
diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h
index 28471bbcd7ea..c3dbda1dd8ad 100644
--- a/src/crimson/os/seastore/cache.h
+++ b/src/crimson/os/seastore/cache.h
@@ -443,6 +443,15 @@ class Cache {
     return get_absent_extent<T>(t, offset, length, [](T &){});
   }
 
+  bool is_viewable_extent_stable(
+    Transaction &t,
+    CachedExtentRef extent)
+  {
+    assert(extent);
+    auto view = extent->get_transactional_view(t);
+    return view->is_stable();
+  }
+
   using get_extent_ertr = base_ertr;
   get_extent_ertr::future<CachedExtentRef>
   get_extent_viewable_by_trans(
diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h
index 730a0ace9a0b..fee7ca74514c 100644
--- a/src/crimson/os/seastore/cached_extent.h
+++ b/src/crimson/os/seastore/cached_extent.h
@@ -618,6 +618,11 @@ class CachedExtent
     return last_committed_crc;
   }
 
+  /// Returns true if the extent part of the open transaction
+  bool is_pending_in_trans(transaction_id_t id) const {
+    return is_pending() && pending_for_transaction == id;
+  }
+
 private:
   template <typename T>
   friend class read_set_item_t;
@@ -648,11 +653,6 @@ class CachedExtent
     ptr = nptr;
   }
 
-  /// Returns true if the extent part of the open transaction
-  bool is_pending_in_trans(transaction_id_t id) const {
-    return is_pending() && pending_for_transaction == id;
-  }
-
   /// hook for intrusive ref list (mainly dirty or lru list)
   boost::intrusive::list_member_hook<> primary_ref_list_hook;
   using primary_ref_list_member_options = boost::intrusive::member_hook<

From 6aa7d356d3acfea1bc92f65e4d27c4b4bc10a6e2 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Fri, 8 Mar 2024 06:40:16 +0100
Subject: [PATCH 2445/2492] mgr/dashboard: add a notification when deleting rgw
 bucket

Fixes: https://tracker.ceph.com/issues/64855
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../rgw-bucket-list.component.html            |  7 +++
 .../rgw-bucket-list.component.spec.ts         |  4 +-
 .../rgw-bucket-list.component.ts              | 55 ++++++++++++-------
 .../shared/services/task-message.service.ts   |  6 ++
 4 files changed, 50 insertions(+), 22 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.html
index b5e75841afe6..d05e54f854ff 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.html
@@ -42,3 +42,10 @@
   <ng-template #noObjectQuota
                i18n>No Limit</ng-template>
 </ng-template>
+
+<ng-template #deleteTpl>
+  <cd-alert-panel type="danger"
+                  i18n>
+    Buckets might still have underlying data depending on your bucket configuration
+  </cd-alert-panel>
+</ng-template>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.spec.ts
index 58d6fa983d38..3aca92fb414e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.spec.ts
@@ -12,6 +12,7 @@ import { SharedModule } from '~/app/shared/shared.module';
 import { configureTestBed, PermissionHelper } from '~/testing/unit-test-helper';
 import { RgwBucketDetailsComponent } from '../rgw-bucket-details/rgw-bucket-details.component';
 import { RgwBucketListComponent } from './rgw-bucket-list.component';
+import { ToastrModule } from 'ngx-toastr';
 
 describe('RgwBucketListComponent', () => {
   let component: RgwBucketListComponent;
@@ -26,7 +27,8 @@ describe('RgwBucketListComponent', () => {
       RouterTestingModule,
       SharedModule,
       NgbNavModule,
-      HttpClientTestingModule
+      HttpClientTestingModule,
+      ToastrModule.forRoot()
     ]
   });
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.ts
index 58adf6ab08fa..cbef751c4567 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-list/rgw-bucket-list.component.ts
@@ -13,11 +13,13 @@ import { CdTableAction } from '~/app/shared/models/cd-table-action';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
 import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context';
 import { CdTableSelection } from '~/app/shared/models/cd-table-selection';
+import { FinishedTask } from '~/app/shared/models/finished-task';
 import { Permission } from '~/app/shared/models/permissions';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
 import { DimlessPipe } from '~/app/shared/pipes/dimless.pipe';
 import { AuthStorageService } from '~/app/shared/services/auth-storage.service';
 import { ModalService } from '~/app/shared/services/modal.service';
+import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service';
 import { URLBuilderService } from '~/app/shared/services/url-builder.service';
 
 const BASE_URL = 'rgw/bucket';
@@ -35,6 +37,8 @@ export class RgwBucketListComponent extends ListWithDetails implements OnInit {
   bucketSizeTpl: TemplateRef<any>;
   @ViewChild('bucketObjectTpl', { static: true })
   bucketObjectTpl: TemplateRef<any>;
+  @ViewChild('deleteTpl', { static: true })
+  deleteTpl: TemplateRef<any>;
 
   permission: Permission;
   tableActions: CdTableAction[];
@@ -51,7 +55,8 @@ export class RgwBucketListComponent extends ListWithDetails implements OnInit {
     private modalService: ModalService,
     private urlBuilder: URLBuilderService,
     public actionLabels: ActionLabelsI18n,
-    protected ngZone: NgZone
+    protected ngZone: NgZone,
+    private taskWrapper: TaskWrapperService
   ) {
     super(ngZone);
   }
@@ -156,31 +161,39 @@ export class RgwBucketListComponent extends ListWithDetails implements OnInit {
   }
 
   deleteAction() {
+    const itemNames = this.selection.selected.map((bucket: any) => bucket['bid']);
     this.modalService.show(CriticalConfirmationModalComponent, {
       itemDescription: this.selection.hasSingleSelection ? $localize`bucket` : $localize`buckets`,
-      itemNames: this.selection.selected.map((bucket: any) => bucket['bid']),
+      itemNames: itemNames,
+      bodyTemplate: this.deleteTpl,
       submitActionObservable: () => {
         return new Observable((observer: Subscriber<any>) => {
-          // Delete all selected data table rows.
-          observableForkJoin(
-            this.selection.selected.map((bucket: any) => {
-              return this.rgwBucketService.delete(bucket.bid);
+          this.taskWrapper
+            .wrapTaskAroundCall({
+              task: new FinishedTask('rgw/bucket/delete', {
+                bucket_names: itemNames
+              }),
+              call: observableForkJoin(
+                this.selection.selected.map((bucket: any) => {
+                  return this.rgwBucketService.delete(bucket.bid);
+                })
+              )
             })
-          ).subscribe({
-            error: (error) => {
-              // Forward the error to the observer.
-              observer.error(error);
-              // Reload the data table content because some deletions might
-              // have been executed successfully in the meanwhile.
-              this.table.refreshBtn();
-            },
-            complete: () => {
-              // Notify the observer that we are done.
-              observer.complete();
-              // Reload the data table content.
-              this.table.refreshBtn();
-            }
-          });
+            .subscribe({
+              error: (error: any) => {
+                // Forward the error to the observer.
+                observer.error(error);
+                // Reload the data table content because some deletions might
+                // have been executed successfully in the meanwhile.
+                this.table.refreshBtn();
+              },
+              complete: () => {
+                // Notify the observer that we are done.
+                observer.complete();
+                // Reload the data table content.
+                this.table.refreshBtn();
+              }
+            });
         });
       }
     });
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
index 4fbcc09d0902..f3aed9704951 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts
@@ -318,6 +318,12 @@ export class TaskMessageService {
       this.rbd_mirroring.pool_peer,
       () => ({})
     ),
+    // RGW operations
+    'rgw/bucket/delete': this.newTaskMessage(this.commonOperations.delete, (metadata) => {
+      return $localize`${
+        metadata.bucket_names.length > 1 ? 'selected buckets' : metadata.bucket_names[0]
+      }`;
+    }),
     // iSCSI target tasks
     'iscsi/target/create': this.newTaskMessage(this.commonOperations.create, (metadata) =>
       this.iscsiTarget(metadata)

From 93974820ceb58bdde12074f2b9ab750696f51a51 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 19 Mar 2024 20:39:09 +0530
Subject: [PATCH 2446/2492] mgr/dashboard: update grafana api url when
 switching cluster

do a force refresh on the settings service so it picks up the new api
url of grafana

Fixes: https://tracker.ceph.com/issues/65052
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../navigation/navigation/navigation.component.ts    | 12 +++++++++++-
 .../frontend/src/app/shared/api/settings.service.ts  |  9 +++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index 8973843872bc..de8b0a267db8 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -4,6 +4,7 @@ import { Router } from '@angular/router';
 import * as _ from 'lodash';
 import { Subscription } from 'rxjs';
 import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
+import { SettingsService } from '~/app/shared/api/settings.service';
 
 import { Icons } from '~/app/shared/enum/icons.enum';
 import { MultiCluster } from '~/app/shared/models/multi-cluster';
@@ -58,7 +59,8 @@ export class NavigationComponent implements OnInit, OnDestroy {
     private telemetryNotificationService: TelemetryNotificationService,
     public prometheusAlertService: PrometheusAlertService,
     private motdNotificationService: MotdNotificationService,
-    private cookieService: CookiesService
+    private cookieService: CookiesService,
+    private settingsService: SettingsService
   ) {
     this.permissions = this.authStorageService.getPermissions();
     this.enabledFeature$ = this.featureToggles.get();
@@ -212,6 +214,14 @@ export class NavigationComponent implements OnInit, OnDestroy {
       () => {
         this.multiClusterService.refresh();
         this.summaryService.refresh();
+
+        // force refresh grafana api url to get the correct url for the selected cluster
+        this.settingsService.ifSettingConfigured(
+          'api/grafana/url',
+          () => {},
+          () => {},
+          true
+        );
         const currentRoute = this.router.url.split('?')[0];
         if (currentRoute.includes('dashboard')) {
           this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/settings.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/settings.service.ts
index 1e53fa064930..9aeb79906c9f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/settings.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/settings.service.ts
@@ -35,9 +35,14 @@ export class SettingsService {
     );
   }
 
-  ifSettingConfigured(url: string, fn: (value?: string) => void, elseFn?: () => void): void {
+  ifSettingConfigured(
+    url: string,
+    fn: (value?: string) => void,
+    elseFn?: () => void,
+    forceRefresh = false
+  ): void {
     const setting = this.settings[url];
-    if (setting === undefined) {
+    if (forceRefresh || setting === undefined) {
       this.http.get(url).subscribe(
         (data: any) => {
           this.settings[url] = this.getSettingsValue(data);

From b6dbf0f24d189d5b75148f07f1dc7f332d23b64a Mon Sep 17 00:00:00 2001
From: Ivo Almeida <ialmeida@redhat.com>
Date: Wed, 20 Mar 2024 10:35:52 +0000
Subject: [PATCH 2447/2492] mgr/dashboard: show full subvol path snap schedule

Show full subvolume path when adding snap schedule

Fixes: https://tracker.ceph.com/issues/65000

Signed-off-by: Ivo Almeida <ialmeida@redhat.com>
---
 .../cephfs-snapshotschedule-form.component.ts | 76 ++++++++++++++-----
 1 file changed, 56 insertions(+), 20 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
index 0ef68f7d529c..99c4a5b0a424 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-snapshotschedule-form/cephfs-snapshotschedule-form.component.ts
@@ -3,7 +3,17 @@ import { AbstractControl, FormArray, FormControl, FormGroup, Validators } from '
 import { NgbActiveModal, NgbDateStruct, NgbTimeStruct } from '@ng-bootstrap/ng-bootstrap';
 import { padStart, uniq } from 'lodash';
 import { Observable, OperatorFunction, of, timer } from 'rxjs';
-import { catchError, debounceTime, distinctUntilChanged, map, switchMap } from 'rxjs/operators';
+import {
+  catchError,
+  debounceTime,
+  distinctUntilChanged,
+  filter,
+  map,
+  mergeMap,
+  pluck,
+  switchMap,
+  tap
+} from 'rxjs/operators';
 import { CephfsSnapshotScheduleService } from '~/app/shared/api/cephfs-snapshot-schedule.service';
 import { CephfsSubvolumeService } from '~/app/shared/api/cephfs-subvolume.service';
 import { DirectoryStoreService } from '~/app/shared/api/directory-store.service';
@@ -91,25 +101,51 @@ export class CephfsSnapshotscheduleFormComponent extends CdForm implements OnIni
     this.directoryStore.loadDirectories(this.id, '/', 3);
     this.createForm();
     this.isEdit ? this.populateForm() : this.loadingReady();
-    this.snapScheduleForm.get('directory').valueChanges.subscribe({
-      next: (value: string) => {
-        this.subvolumeGroup = value?.split?.('/')?.[2];
-        this.subvolume = value?.split?.('/')?.[3];
-        this.subvolumeService
-          .exists(
-            this.subvolume,
-            this.fsName,
-            this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP ? '' : this.subvolumeGroup
-          )
-          .subscribe({
-            next: (exists: boolean) => {
-              this.isSubvolume = exists;
-              this.isDefaultSubvolumeGroup =
-                exists && this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP;
-            }
-          });
-      }
-    });
+
+    this.snapScheduleForm
+      .get('directory')
+      .valueChanges.pipe(
+        filter(() => !this.isEdit),
+        debounceTime(DEBOUNCE_TIMER),
+        tap(() => {
+          this.isSubvolume = false;
+        }),
+        tap((value: string) => {
+          this.subvolumeGroup = value?.split?.('/')?.[2];
+          this.subvolume = value?.split?.('/')?.[3];
+        }),
+        filter(() => !!this.subvolume && !!this.subvolumeGroup),
+        mergeMap(() =>
+          this.subvolumeService
+            .exists(
+              this.subvolume,
+              this.fsName,
+              this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP ? '' : this.subvolumeGroup
+            )
+            .pipe(
+              tap((exists: boolean) => (this.isSubvolume = exists)),
+              tap(
+                (exists: boolean) =>
+                  (this.isDefaultSubvolumeGroup =
+                    exists && this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP)
+              )
+            )
+        ),
+        filter((exists: boolean) => exists),
+        mergeMap(() =>
+          this.subvolumeService
+            .info(
+              this.fsName,
+              this.subvolume,
+              this.subvolumeGroup === DEFAULT_SUBVOLUME_GROUP ? '' : this.subvolumeGroup
+            )
+            .pipe(pluck('path'))
+        ),
+        filter((path: string) => path !== this.snapScheduleForm.get('directory').value)
+      )
+      .subscribe({
+        next: (path: string) => this.snapScheduleForm.get('directory').setValue(path)
+      });
   }
 
   get retentionPolicies() {

From db438cf990cb7cb1ebbe99cfdf31f1fc285747ab Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 09:56:20 -0400
Subject: [PATCH 2448/2492] qa/cephfs: add probabilistic ignorelist for
 pg_health

PG_AVAILABILITY/PG_DEGRADED warnings are dominating fs runs. We want the
underlying issue fixed but it cannot continue to fail all of our tests 100% of
the time. Use a probabilistic addition of these warnings to the ignorelist.

Fixes: https://tracker.ceph.com/issues/64984
Related-to: https://tracker.ceph.com/issues/52624
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/overrides/pg_health.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 qa/cephfs/overrides/pg_health.yaml

diff --git a/qa/cephfs/overrides/pg_health.yaml b/qa/cephfs/overrides/pg_health.yaml
new file mode 100644
index 000000000000..defd70eeba86
--- /dev/null
+++ b/qa/cephfs/overrides/pg_health.yaml
@@ -0,0 +1,14 @@
+# To prevent PG_AVAILABILITY failures from dominating runs, we usually ignore
+# these warnings. However, we'd like to see the cause of these warnings to be
+# fixed so occasionally allow the warning to fail the job.
+#
+# https://tracker.ceph.com/issues/52624
+
+teuthology:
+  premerge: if math.random() < 0.05 then reject() end
+overrides:
+  ceph:
+    log-ignorelist:
+      - PG_AVAILABILITY
+      - PG_DEGRADED
+      - Reduced data availability

From b690263a6c5904dcfa1d62e7ac08a67dd11a7829 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Mar 2024 10:19:15 -0400
Subject: [PATCH 2449/2492] qa/suites/fs: link in pg_health ignorelist

Fixes: https://tracker.ceph.com/issues/64984
Related-to: https://tracker.ceph.com/issues/52624
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/suites/fs/32bits/overrides/pg_health.yaml        | 1 +
 qa/suites/fs/fscrypt/overrides/pg_health.yaml       | 1 +
 qa/suites/fs/full/overrides/pg_health.yaml          | 1 +
 qa/suites/fs/functional/overrides/pg_health.yaml    | 1 +
 qa/suites/fs/libcephfs/overrides/pg_health.yaml     | 1 +
 qa/suites/fs/mirror-ha/overrides/pg_health.yaml     | 1 +
 qa/suites/fs/mirror/overrides/pg_health.yaml        | 1 +
 qa/suites/fs/mixed-clients/overrides/pg_health.yaml | 1 +
 qa/suites/fs/multiclient/overrides/pg_health.yaml   | 1 +
 qa/suites/fs/multifs/overrides/pg_health.yaml       | 1 +
 qa/suites/fs/nfs/overrides/+                        | 0
 qa/suites/fs/nfs/overrides/pg_health.yaml           | 1 +
 qa/suites/fs/permission/overrides/pg_health.yaml    | 1 +
 qa/suites/fs/shell/overrides/pg_health.yaml         | 1 +
 qa/suites/fs/snaps/overrides/pg_health.yaml         | 1 +
 qa/suites/fs/top/overrides/+                        | 0
 qa/suites/fs/top/overrides/pg_health.yaml           | 1 +
 qa/suites/fs/traceless/overrides/pg_health.yaml     | 1 +
 qa/suites/fs/verify/overrides/pg_health.yaml        | 1 +
 qa/suites/fs/volumes/overrides/pg_health.yaml       | 1 +
 qa/suites/fs/workload/overrides/pg_health.yaml      | 1 +
 21 files changed, 19 insertions(+)
 create mode 120000 qa/suites/fs/32bits/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/fscrypt/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/full/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/functional/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/libcephfs/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/mirror-ha/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/mirror/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/mixed-clients/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/multiclient/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/multifs/overrides/pg_health.yaml
 create mode 100644 qa/suites/fs/nfs/overrides/+
 create mode 120000 qa/suites/fs/nfs/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/permission/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/shell/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/snaps/overrides/pg_health.yaml
 create mode 100644 qa/suites/fs/top/overrides/+
 create mode 120000 qa/suites/fs/top/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/traceless/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/verify/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/volumes/overrides/pg_health.yaml
 create mode 120000 qa/suites/fs/workload/overrides/pg_health.yaml

diff --git a/qa/suites/fs/32bits/overrides/pg_health.yaml b/qa/suites/fs/32bits/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/32bits/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/fscrypt/overrides/pg_health.yaml b/qa/suites/fs/fscrypt/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/fscrypt/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/full/overrides/pg_health.yaml b/qa/suites/fs/full/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/full/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/functional/overrides/pg_health.yaml b/qa/suites/fs/functional/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/functional/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/libcephfs/overrides/pg_health.yaml b/qa/suites/fs/libcephfs/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/libcephfs/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/mirror-ha/overrides/pg_health.yaml b/qa/suites/fs/mirror-ha/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/mirror-ha/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/mirror/overrides/pg_health.yaml b/qa/suites/fs/mirror/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/mirror/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/mixed-clients/overrides/pg_health.yaml b/qa/suites/fs/mixed-clients/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/mixed-clients/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/multiclient/overrides/pg_health.yaml b/qa/suites/fs/multiclient/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/multiclient/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/multifs/overrides/pg_health.yaml b/qa/suites/fs/multifs/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/multifs/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/nfs/overrides/+ b/qa/suites/fs/nfs/overrides/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/nfs/overrides/pg_health.yaml b/qa/suites/fs/nfs/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/nfs/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/permission/overrides/pg_health.yaml b/qa/suites/fs/permission/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/permission/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/shell/overrides/pg_health.yaml b/qa/suites/fs/shell/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/shell/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/snaps/overrides/pg_health.yaml b/qa/suites/fs/snaps/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/snaps/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/top/overrides/+ b/qa/suites/fs/top/overrides/+
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/qa/suites/fs/top/overrides/pg_health.yaml b/qa/suites/fs/top/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/top/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/traceless/overrides/pg_health.yaml b/qa/suites/fs/traceless/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/traceless/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/verify/overrides/pg_health.yaml b/qa/suites/fs/verify/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/verify/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/volumes/overrides/pg_health.yaml b/qa/suites/fs/volumes/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/volumes/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/workload/overrides/pg_health.yaml b/qa/suites/fs/workload/overrides/pg_health.yaml
new file mode 120000
index 000000000000..5b6be3a65b62
--- /dev/null
+++ b/qa/suites/fs/workload/overrides/pg_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/pg_health.yaml
\ No newline at end of file

From eb7b5c800fd03c3e6e699c3281ed897fa48c9085 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Wed, 20 Mar 2024 21:52:11 -0400
Subject: [PATCH 2450/2492] qa/suites/fs/workload: enable snap_schedule early

Otherwise it can cause client evictions when the volumes plugin has mounted
cephfs.

Fixes: https://tracker.ceph.com/issues/64988
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 qa/cephfs/begin/3-modules.yaml               | 19 +++++++++++++++++++
 qa/suites/fs/workload/begin/3-modules.yaml   |  1 +
 qa/suites/fs/workload/tasks/3-snaps/yes.yaml | 10 +++++++---
 3 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 qa/cephfs/begin/3-modules.yaml
 create mode 120000 qa/suites/fs/workload/begin/3-modules.yaml

diff --git a/qa/cephfs/begin/3-modules.yaml b/qa/cephfs/begin/3-modules.yaml
new file mode 100644
index 000000000000..259473425693
--- /dev/null
+++ b/qa/cephfs/begin/3-modules.yaml
@@ -0,0 +1,19 @@
+# Enable mgr modules now before any CephFS mounts are created by the mgr.  This
+# avoids the potential race of the mgr mounting CephFS and then getting failed
+# over by the monitors before the monitors have a chance to note the new client
+# session from the mgr beacon. In that case, the monitors will not blocklist
+# that client mount automatically so the MDS will eventually do the eviction
+# (and create a cluster log warning which we want to avoid).
+#
+# Note: ideally the mgr would gently stop mgr modules before respawning so that
+# the client mounts can be unmounted but this caused issues historically with
+# modules like the dashboard so an abrupt restart was chosen instead.
+
+mgrmodules:
+  sequential:
+    - print: "Enabling mgr modules"
+    # other fragments append to this
+
+tasks:
+  - sequential:
+      - mgrmodules
diff --git a/qa/suites/fs/workload/begin/3-modules.yaml b/qa/suites/fs/workload/begin/3-modules.yaml
new file mode 120000
index 000000000000..1eba706a59dc
--- /dev/null
+++ b/qa/suites/fs/workload/begin/3-modules.yaml
@@ -0,0 +1 @@
+.qa/cephfs/begin/3-modules.yaml
\ No newline at end of file
diff --git a/qa/suites/fs/workload/tasks/3-snaps/yes.yaml b/qa/suites/fs/workload/tasks/3-snaps/yes.yaml
index 1a6a6cc60142..69f53768df81 100644
--- a/qa/suites/fs/workload/tasks/3-snaps/yes.yaml
+++ b/qa/suites/fs/workload/tasks/3-snaps/yes.yaml
@@ -1,3 +1,10 @@
+mgrmodules:
+  sequential:
+    - exec:
+        mon.a:
+          - ceph mgr module enable snap_schedule
+          - ceph config set mgr mgr/snap_schedule/allow_m_granularity true
+          - ceph config set mgr mgr/snap_schedule/dump_on_update true
 overrides:
   ceph:
     conf:
@@ -12,9 +19,6 @@ overrides:
 tasks:
 - exec:
     mon.a:
-      - ceph mgr module enable snap_schedule
-      - ceph config set mgr mgr/snap_schedule/allow_m_granularity true
-      - ceph config set mgr mgr/snap_schedule/dump_on_update true
       - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m
       - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h
       - ceph fs snap-schedule status --fs=cephfs --path=/

From 4adf48998d97f8199bcb86c8d8994bb5989c198a Mon Sep 17 00:00:00 2001
From: Ernesto Puerta <epuertat@redhat.com>
Date: Wed, 13 Mar 2024 14:06:10 +0100
Subject: [PATCH 2451/2492] mgr/dashboard: fix NVMeoF API

* Update NVMe-oF gRPC Proto to 1.0.0
* Error handling,
* Missing PATCH for certain namespace ops (resize, set QoS, set balance
  groups),
* Stop bypassing gRPC payloads and validate those in the back-end,
* Fix incorrect HTTP 1.1 semantics for some POST/DELETE and URIs.
* Catch errors/exceptions.
* Clean-up EndpointDoc Params
* Run Black linter.
* Remove most of NVMeoFClient glue code between gRPC and controller.
* Fix namespace delete endpoint by exposing trsvcid
* nvmeof io_stats support

Fixes: https://tracker.ceph.com/issues/64890
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
---
 .../mgr/dashboard/controllers/__init__.py     |   3 +-
 src/pybind/mgr/dashboard/controllers/_docs.py |   9 +-
 .../mgr/dashboard/controllers/_endpoint.py    |   8 +-
 .../dashboard/controllers/_rest_controller.py |   4 +-
 src/pybind/mgr/dashboard/controllers/auth.py  |   1 +
 src/pybind/mgr/dashboard/controllers/docs.py  |   6 +-
 .../mgr/dashboard/controllers/nvmeof.py       | 519 ++++++++++++------
 src/pybind/mgr/dashboard/model/nvmeof.py      |  92 ++++
 src/pybind/mgr/dashboard/openapi.yaml         | 444 +++++++++++----
 src/pybind/mgr/dashboard/requirements.txt     |   2 +-
 .../mgr/dashboard/services/exception.py       |   9 +-
 .../mgr/dashboard/services/nvmeof_client.py   | 223 ++++----
 .../dashboard/services/proto/gateway.proto    |   8 +-
 .../dashboard/services/proto/gateway_pb2.py   | 237 ++++----
 14 files changed, 1072 insertions(+), 493 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/model/nvmeof.py

diff --git a/src/pybind/mgr/dashboard/controllers/__init__.py b/src/pybind/mgr/dashboard/controllers/__init__.py
index af3f276ebfa2..3db5da5d3235 100755
--- a/src/pybind/mgr/dashboard/controllers/__init__.py
+++ b/src/pybind/mgr/dashboard/controllers/__init__.py
@@ -2,7 +2,7 @@
 from ._auth import ControllerAuthMixin
 from ._base_controller import BaseController
 from ._crud import CRUDCollectionMethod, CRUDEndpoint, CRUDResourceMethod, SecretStr
-from ._docs import APIDoc, EndpointDoc
+from ._docs import APIDoc, EndpointDoc, Param
 from ._endpoint import Endpoint, Proxy
 from ._helpers import ENDPOINT_MAP, allow_empty_body, \
     generate_controller_routes, json_error_page, validate_ceph_type
@@ -23,6 +23,7 @@
     'Task',
     'ControllerAuthMixin',
     'EndpointDoc',
+    'Param',
     'APIDoc',
     'allow_empty_body',
     'ENDPOINT_MAP',
diff --git a/src/pybind/mgr/dashboard/controllers/_docs.py b/src/pybind/mgr/dashboard/controllers/_docs.py
index 5bd7a5a7a6ea..7301875f6b4f 100644
--- a/src/pybind/mgr/dashboard/controllers/_docs.py
+++ b/src/pybind/mgr/dashboard/controllers/_docs.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, Union
 
 from ..api.doc import SchemaInput, SchemaType
 
@@ -115,6 +115,13 @@ def __call__(self, func: Any) -> Any:
         return func
 
 
+class Param(NamedTuple):
+    type: Union[Type, List[Type]]
+    description: str
+    optional: bool = False
+    default: Optional[Any] = None
+
+
 class APIDoc(object):
     def __init__(self, description="", group=""):
         self.tag = group
diff --git a/src/pybind/mgr/dashboard/controllers/_endpoint.py b/src/pybind/mgr/dashboard/controllers/_endpoint.py
index fccab89c3497..6d77dcdcea4a 100644
--- a/src/pybind/mgr/dashboard/controllers/_endpoint.py
+++ b/src/pybind/mgr/dashboard/controllers/_endpoint.py
@@ -12,9 +12,9 @@ def __init__(self, method=None, path=None, path_params=None, query_params=None,
         if method is None:
             method = 'GET'
         elif not isinstance(method, str) or \
-                method.upper() not in ['GET', 'POST', 'DELETE', 'PUT']:
+                method.upper() not in ['GET', 'POST', 'DELETE', 'PUT', 'PATCH']:
             raise TypeError("Possible values for method are: 'GET', 'POST', "
-                            "'DELETE', or 'PUT'")
+                            "'DELETE', 'PUT', 'PATCH'")
 
         method = method.upper()
 
@@ -25,7 +25,7 @@ def __init__(self, method=None, path=None, path_params=None, query_params=None,
                                 " path parameters by default".format(method))
 
         if path_params is None:
-            if method in ['POST', 'PUT']:
+            if method in ['POST', 'PUT', 'PATCH']:
                 path_params = []
 
         if query_params is None:
@@ -41,7 +41,7 @@ def __init__(self, method=None, path=None, path_params=None, query_params=None,
         self.version = version
 
     def __call__(self, func):
-        if self.method in ['POST', 'PUT']:
+        if self.method in ['POST', 'PUT', 'PATCH']:
             func_params = _get_function_params(func)
             for param in func_params:
                 if param['name'] in self.path_params and not param['required']:
diff --git a/src/pybind/mgr/dashboard/controllers/_rest_controller.py b/src/pybind/mgr/dashboard/controllers/_rest_controller.py
index 0224c366f3bc..c3bd07cf807b 100644
--- a/src/pybind/mgr/dashboard/controllers/_rest_controller.py
+++ b/src/pybind/mgr/dashboard/controllers/_rest_controller.py
@@ -49,6 +49,7 @@ class RESTController(BaseController, skip_registry=True):
         'GET': Permission.READ,
         'POST': Permission.CREATE,
         'PUT': Permission.UPDATE,
+        'PATCH': Permission.UPDATE,
         'DELETE': Permission.DELETE
     }
 
@@ -60,7 +61,8 @@ class RESTController(BaseController, skip_registry=True):
         ('get', {'method': 'GET', 'resource': True, 'status': 200, 'version': APIVersion.DEFAULT}),
         ('delete', {'method': 'DELETE', 'resource': True, 'status': 204, 'version': APIVersion.DEFAULT}),  # noqa E501 #pylint: disable=line-too-long
         ('set', {'method': 'PUT', 'resource': True, 'status': 200, 'version': APIVersion.DEFAULT}),
-        ('singleton_set', {'method': 'PUT', 'resource': False, 'status': 200, 'version': APIVersion.DEFAULT})  # noqa E501 #pylint: disable=line-too-long
+        ('singleton_set', {'method': 'PUT', 'resource': False, 'status': 200, 'version': APIVersion.DEFAULT}),  # noqa E501 #pylint: disable=line-too-long
+        ('update', {'method': 'PATCH', 'resource': True, 'status': 200, 'version': APIVersion.DEFAULT})  # noqa E501 #pylint: disable=line-too-long
     ])
 
     @classmethod
diff --git a/src/pybind/mgr/dashboard/controllers/auth.py b/src/pybind/mgr/dashboard/controllers/auth.py
index e8bb4bbef8e1..eca87a9a84ed 100644
--- a/src/pybind/mgr/dashboard/controllers/auth.py
+++ b/src/pybind/mgr/dashboard/controllers/auth.py
@@ -38,6 +38,7 @@ class Auth(RESTController, ControllerAuthMixin):
     Provide authenticates and returns JWT token.
     """
     # pylint: disable=R0912
+
     def create(self, username, password):
         user_data = AuthManager.authenticate(username, password)
         user_perms, pwd_expiration_date, pwd_update_required = None, None, None
diff --git a/src/pybind/mgr/dashboard/controllers/docs.py b/src/pybind/mgr/dashboard/controllers/docs.py
index 2ade4ef9bad4..9ecb513621a1 100644
--- a/src/pybind/mgr/dashboard/controllers/docs.py
+++ b/src/pybind/mgr/dashboard/controllers/docs.py
@@ -215,7 +215,7 @@ def _gen_responses(cls, method, resp_object=None,
             resp['201'] = {'description': "Resource created.",
                            'content': {version.to_mime_type():
                                        {'type': 'object'}}}
-        if method.lower() == 'put':
+        if method.lower() in ['put', 'patch']:
             resp['200'] = {'description': "Resource updated.",
                            'content': {version.to_mime_type():
                                        {'type': 'object'}}}
@@ -315,7 +315,7 @@ def set_request_body_param(cls, endpoint_param, method, methods, p_info):
     @classmethod
     def gen_paths(cls, all_endpoints):
         # pylint: disable=R0912
-        method_order = ['get', 'post', 'put', 'delete']
+        method_order = ['get', 'post', 'put', 'patch', 'delete']
         paths = {}
         for path, endpoints in sorted(list(ENDPOINT_MAP.items()),
                                       key=lambda p: p[0]):
@@ -344,7 +344,7 @@ def gen_paths(cls, all_endpoints):
                 if summary:
                     methods[method.lower()]['summary'] = summary
 
-                if method.lower() in ['post', 'put']:
+                if method.lower() in ['post', 'put', 'patch']:
                     cls.set_request_body_param(endpoint.body_params, method, methods, p_info)
                     cls.set_request_body_param(endpoint.query_params, method, methods, p_info)
 
diff --git a/src/pybind/mgr/dashboard/controllers/nvmeof.py b/src/pybind/mgr/dashboard/controllers/nvmeof.py
index b485a5aa17b2..a3b4925b0eee 100644
--- a/src/pybind/mgr/dashboard/controllers/nvmeof.py
+++ b/src/pybind/mgr/dashboard/controllers/nvmeof.py
@@ -1,167 +1,370 @@
 # -*- coding: utf-8 -*-
-import json
 from typing import Optional
 
+from ..model import nvmeof as model
 from ..security import Scope
-from . import APIDoc, APIRouter, CreatePermission, DeletePermission, Endpoint, \
-    EndpointDoc, ReadPermission, RESTController
+from ..tools import str_to_bool
+from . import APIDoc, APIRouter, Endpoint, EndpointDoc, Param, ReadPermission, RESTController
 
 try:
-    from google.protobuf.json_format import MessageToJson
-
-    from ..services.nvmeof_client import NVMeoFClient
+    from ..services.nvmeof_client import NVMeoFClient, empty_response, \
+        handle_nvmeof_error, map_collection, map_model
 except ImportError:
-    MessageToJson = None
+    pass
 else:
-    @APIRouter('/nvmeof/namespace', Scope.NVME_OF)
-    @APIDoc('NVMe-oF Namespace Management API', 'NVMe-oF')
-    class NvmeofNamespace(RESTController):
-        @ReadPermission
-        @EndpointDoc('List all NVMeoF namespaces',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
-                     })
-        def list(self, subsystem_nqn: str):
-            response = MessageToJson(NVMeoFClient().list_namespaces(subsystem_nqn))
-            return json.loads(response)
-
-        @CreatePermission
-        @EndpointDoc('Create a new NVMeoF namespace',
-                     parameters={
-                         'rbd_pool': (str, 'RBD pool name'),
-                         'rbd_image': (str, 'RBD image name'),
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'create_image': (bool, 'Create RBD image'),
-                         'image_size': (int, 'RBD image size'),
-                         'block_size': (int, 'NVMeoF namespace block size')
-                     })
-        def create(self, rbd_pool: str, rbd_image: str, subsystem_nqn: str,
-                   create_image: Optional[bool] = True, image_size: Optional[int] = 1024,
-                   block_size: int = 512):
-            response = NVMeoFClient().create_namespace(rbd_pool, rbd_image,
-                                                       subsystem_nqn, block_size,
-                                                       create_image, image_size)
-            return json.loads(MessageToJson(response))
-
-        @Endpoint('DELETE', path='{subsystem_nqn}')
-        @EndpointDoc('Delete an existing NVMeoF namespace',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
-                     })
-        @DeletePermission
-        def delete(self, subsystem_nqn: str):
-            response = NVMeoFClient().delete_namespace(subsystem_nqn)
-            return json.loads(MessageToJson(response))
-
-    @APIRouter('/nvmeof/subsystem', Scope.NVME_OF)
-    @APIDoc('NVMe-oF Subsystem Management API', 'NVMe-oF')
-    class NvmeofSubsystem(RESTController):
-        @ReadPermission
-        @EndpointDoc("List all NVMeoF subsystems",
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                     })
-        @ReadPermission
-        def list(self, subsystem_nqn: Optional[str] = None):
-            response = MessageToJson(NVMeoFClient().list_subsystems(
-                subsystem_nqn=subsystem_nqn))
-
-            return json.loads(response)
-
-        @CreatePermission
-        @EndpointDoc('Create a new NVMeoF subsystem',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'serial_number': (str, 'NVMeoF subsystem serial number'),
-                         'max_namespaces': (int, 'Maximum number of namespaces')
-                     })
-        def create(self, subsystem_nqn: str):
-            response = NVMeoFClient().create_subsystem(subsystem_nqn)
-            return json.loads(MessageToJson(response))
-
-        @DeletePermission
-        @Endpoint('DELETE', path='{subsystem_nqn}')
-        @EndpointDoc('Delete an existing NVMeoF subsystem',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'force': (bool, 'Force delete')
-                     })
-        def delete(self, subsystem_nqn: str, force: Optional[bool] = False):
-            response = NVMeoFClient().delete_subsystem(subsystem_nqn, force)
-            return json.loads(MessageToJson(response))
-
-    @APIRouter('/nvmeof/hosts', Scope.NVME_OF)
-    @APIDoc('NVMe-oF Host Management API', 'NVMe-oF')
-    class NvmeofHost(RESTController):
-        @ReadPermission
-        @EndpointDoc('List all allowed hosts for an NVMeoF subsystem',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
-                     })
-        def list(self, subsystem_nqn: str):
-            response = MessageToJson(NVMeoFClient().list_hosts(subsystem_nqn))
-            return json.loads(response)
-
-        @CreatePermission
-        @EndpointDoc('Allow hosts to access an NVMeoF subsystem',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'host_nqn': (str, 'NVMeoF host NQN')
-                     })
-        def create(self, subsystem_nqn: str, host_nqn: str):
-            response = NVMeoFClient().add_host(subsystem_nqn, host_nqn)
-            return json.loads(MessageToJson(response))
-
-        @DeletePermission
-        @EndpointDoc('Disallow hosts from accessing an NVMeoF subsystem',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN'),
-                         'host_nqn': (str, 'NVMeoF host NQN')
-                     })
-        def delete(self, subsystem_nqn: str, host_nqn: str):
-            response = NVMeoFClient().remove_host(subsystem_nqn, host_nqn)
-            return json.loads(MessageToJson(response))
-
-    @APIRouter('/nvmeof/listener', Scope.NVME_OF)
-    @APIDoc('NVMe-oF Listener Management API', 'NVMe-oF')
-    class NvmeofListener(RESTController):
-        @ReadPermission
-        @EndpointDoc('List all NVMeoF listeners',
-                     parameters={
-                         'subsystem_nqn': (str, 'NVMeoF subsystem NQN')
-                     })
-        def list(self, subsystem_nqn: str):
-            response = MessageToJson(NVMeoFClient().list_listeners(subsystem_nqn))
-            return json.loads(response)
-
-        @CreatePermission
-        @EndpointDoc('Create a new NVMeoF listener',
-                     parameters={
-                         'nqn': (str, 'NVMeoF subsystem NQN'),
-                         'gateway': (str, 'NVMeoF gateway'),
-                         'traddr': (str, 'NVMeoF transport address')
-                     })
-        def create(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            response = NVMeoFClient().create_listener(nqn, gateway, traddr)
-            return json.loads(MessageToJson(response))
-
-        @DeletePermission
-        @EndpointDoc('Delete an existing NVMeoF listener',
-                     parameters={
-                         'nqn': (str, 'NVMeoF subsystem NQN'),
-                         'gateway': (str, 'NVMeoF gateway'),
-                         'traddr': (str, 'NVMeoF transport address')
-                     })
-        def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            response = NVMeoFClient().delete_listener(nqn, gateway, traddr)
-            return json.loads(MessageToJson(response))
-
-    @APIRouter('/nvmeof/gateway', Scope.NVME_OF)
-    @APIDoc('NVMe-oF Gateway Management API', 'NVMe-oF')
-    class NvmeofGateway(RESTController):
+    @APIRouter("/nvmeof/gateway", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Gateway Management API", "NVMe-oF Gateway")
+    class NVMeoFGateway(RESTController):
+        @EndpointDoc("Get information about the NVMeoF gateway")
+        @map_model(model.GatewayInfo)
+        @handle_nvmeof_error
+        def list(self):
+            return NVMeoFClient().stub.get_gateway_info(
+                NVMeoFClient.pb2.get_gateway_info_req()
+            )
+
+    @APIRouter("/nvmeof/subsystem", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Subsystem Management API", "NVMe-oF Subsystem")
+    class NVMeoFSubsystem(RESTController):
+        @EndpointDoc("List all NVMeoF subsystems")
+        @map_collection(model.Subsystem, pick="subsystems")
+        @handle_nvmeof_error
+        def list(self):
+            return NVMeoFClient().stub.list_subsystems(
+                NVMeoFClient.pb2.list_subsystems_req()
+            )
+
+        @EndpointDoc(
+            "Get information from a specific NVMeoF subsystem",
+            parameters={"nqn": Param(str, "NVMeoF subsystem NQN")},
+        )
+        @map_model(model.Subsystem, first="subsystems")
+        @handle_nvmeof_error
+        def get(self, nqn: str):
+            return NVMeoFClient().stub.list_subsystems(
+                NVMeoFClient.pb2.list_subsystems_req(subsystem_nqn=nqn)
+            )
+
+        @EndpointDoc(
+            "Create a new NVMeoF subsystem",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "max_namespaces": Param(int, "Maximum number of namespaces", True, 256),
+                "enable_ha": Param(bool, "Enable high availability"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def create(self, nqn: str, enable_ha: bool, max_namespaces: int = 256):
+            return NVMeoFClient().stub.create_subsystem(
+                NVMeoFClient.pb2.create_subsystem_req(
+                    subsystem_nqn=nqn, max_namespaces=max_namespaces, enable_ha=enable_ha
+                )
+            )
+
+        @EndpointDoc(
+            "Delete an existing NVMeoF subsystem",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "force": Param(bool, "Force delete", "false"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def delete(self, nqn: str, force: Optional[str] = "false"):
+            return NVMeoFClient().stub.delete_subsystem(
+                NVMeoFClient.pb2.delete_subsystem_req(
+                    subsystem_nqn=nqn, force=str_to_bool(force)
+                )
+            )
+
+    @APIRouter("/nvmeof/subsystem/{nqn}/listener", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Subsystem Listener Management API", "NVMe-oF Subsystem Listener")
+    class NVMeoFListener(RESTController):
+        @EndpointDoc(
+            "List all NVMeoF listeners",
+            parameters={"nqn": Param(str, "NVMeoF subsystem NQN")},
+        )
+        @map_collection(model.Listener, pick="listeners")
+        @handle_nvmeof_error
+        def list(self, nqn: str):
+            return NVMeoFClient().stub.list_listeners(
+                NVMeoFClient.pb2.list_listeners_req(subsystem=nqn)
+            )
+
+        @EndpointDoc(
+            "Create a new NVMeoF listener",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "gateway": Param(str, "NVMeoF gateway"),
+                "traddr": Param(str, "NVMeoF transport address"),
+                "trsvcid": Param(int, "NVMeoF transport service port"),
+                "adrfam": Param(str, "NVMeoF address family"),
+                "trtype": Param(str, "NVMeoF transport type"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def create(
+            self,
+            nqn: str,
+            gateway: str,
+            traddr: str,
+            trsvcid: Optional[int] = 4420,
+            adrfam: Optional[str] = "ipv4",
+        ):
+            return NVMeoFClient().stub.create_listener(
+                NVMeoFClient.pb2.create_listener_req(
+                    nqn=nqn,
+                    gateway_name=gateway,
+                    traddr=traddr,
+                    trsvcid=trsvcid,
+                    adrfam=adrfam,
+                )
+            )
+
+        @EndpointDoc(
+            "Delete an existing NVMeoF listener",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "gateway": Param(str, "NVMeoF gateway"),
+                "traddr": Param(str, "NVMeoF transport address"),
+                "trsvid": Param(int, "NVMeoF transport service port"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def delete(self, nqn: str, gateway: str, traddr: Optional[str] = None,
+                   trsvcid: Optional[int] = 4420):
+            return NVMeoFClient().stub.delete_listener(
+                NVMeoFClient.pb2.delete_listener_req(
+                    nqn=nqn, gateway_name=gateway, traddr=traddr, trsvcid=int(trsvcid)
+                )
+            )
+
+    @APIRouter("/nvmeof/subsystem/{nqn}/namespace", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Subsystem Namespace Management API", "NVMe-oF Subsystem Namespace")
+    class NVMeoFNamespace(RESTController):
+        @EndpointDoc(
+            "List all NVMeoF namespaces in a subsystem",
+            parameters={"nqn": Param(str, "NVMeoF subsystem NQN")},
+        )
+        @map_collection(model.Namespace, pick="namespaces")
+        @handle_nvmeof_error
+        def list(self, nqn: str):
+            return NVMeoFClient().stub.list_namespaces(
+                NVMeoFClient.pb2.list_namespaces_req(subsystem=nqn)
+            )
+
+        @EndpointDoc(
+            "Get info from specified NVMeoF namespace",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "nsid": Param(str, "NVMeoF Namespace ID"),
+            },
+        )
+        @map_model(model.Namespace, first="namespaces")
+        @handle_nvmeof_error
+        def get(self, nqn: str, nsid: str):
+            return NVMeoFClient().stub.list_namespaces(
+                NVMeoFClient.pb2.list_namespaces_req(subsystem=nqn, nsid=int(nsid))
+            )
+
         @ReadPermission
-        @Endpoint()
-        @EndpointDoc('List all NVMeoF gateways')
-        def info(self):
-            response = MessageToJson(NVMeoFClient().gateway_info())
-            return json.loads(response)
+        @Endpoint('GET', '{nsid}/io_stats')
+        @EndpointDoc(
+            "Get IO stats from specified NVMeoF namespace",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "nsid": Param(str, "NVMeoF Namespace ID"),
+            },
+        )
+        @map_model(model.NamespaceIOStats)
+        @handle_nvmeof_error
+        def io_stats(self, nqn: str, nsid: str):
+            return NVMeoFClient().stub.namespace_get_io_stats(
+                NVMeoFClient.pb2.namespace_get_io_stats_req(
+                    subsystem_nqn=nqn, nsid=int(nsid))
+            )
+
+        @EndpointDoc(
+            "Create a new NVMeoF namespace",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "rbd_pool": Param(str, "RBD pool name"),
+                "rbd_image_name": Param(str, "RBD image name"),
+                "create_image": Param(bool, "Create RBD image"),
+                "size": Param(int, "RBD image size"),
+                "block_size": Param(int, "NVMeoF namespace block size"),
+                "load_balancing_group": Param(int, "Load balancing group"),
+            },
+        )
+        @map_model(model.NamespaceCreation)
+        @handle_nvmeof_error
+        def create(
+            self,
+            nqn: str,
+            rbd_image_name: str,
+            rbd_pool: str = "rbd",
+            create_image: Optional[bool] = True,
+            size: Optional[int] = 1024,
+            block_size: int = 512,
+            load_balancing_group: Optional[int] = None,
+        ):
+            return NVMeoFClient().stub.namespace_add(
+                NVMeoFClient.pb2.namespace_add_req(
+                    subsystem_nqn=nqn,
+                    rbd_image_name=rbd_image_name,
+                    rbd_pool_name=rbd_pool,
+                    block_size=block_size,
+                    create_image=create_image,
+                    size=size,
+                    anagrpid=load_balancing_group,
+                )
+            )
+
+        @EndpointDoc(
+            "Update an existing NVMeoF namespace",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "nsid": Param(str, "NVMeoF Namespace ID"),
+                "rbd_image_size": Param(int, "RBD image size"),
+                "load_balancing_group": Param(int, "Load balancing group"),
+                "rw_ios_per_second": Param(int, "Read/Write IOPS"),
+                "rw_mbytes_per_second": Param(int, "Read/Write MB/s"),
+                "r_mbytes_per_second": Param(int, "Read MB/s"),
+                "w_mbytes_per_second": Param(int, "Write MB/s"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def update(
+            self,
+            nqn: str,
+            nsid: str,
+            rbd_image_size: Optional[int] = None,
+            load_balancing_group: Optional[int] = None,
+            rw_ios_per_second: Optional[int] = None,
+            rw_mbytes_per_second: Optional[int] = None,
+            r_mbytes_per_second: Optional[int] = None,
+            w_mbytes_per_second: Optional[int] = None,
+        ):
+            if rbd_image_size:
+                mib = 1024 * 1024
+                new_size_mib = int((rbd_image_size + mib - 1) / mib)
+
+                response = NVMeoFClient().stub.namespace_resize(
+                    NVMeoFClient.pb2.namespace_resize_req(
+                        subsystem_nqn=nqn, nsid=int(nsid), new_size=new_size_mib
+                    )
+                )
+                if response.status != 0:
+                    return response
+
+            if load_balancing_group:
+                response = NVMeoFClient().stub.namespace_change_load_balancing_group(
+                    NVMeoFClient.pb2.namespace_change_load_balancing_group_req(
+                        subsystem_nqn=nqn, nsid=int(nsid), anagrpid=load_balancing_group
+                    )
+                )
+                if response.status != 0:
+                    return response
+
+            if (
+                rw_ios_per_second
+                or rw_mbytes_per_second
+                or r_mbytes_per_second
+                or w_mbytes_per_second
+            ):
+                response = NVMeoFClient().stub.namespace_set_qos_limits(
+                    NVMeoFClient.pb2.namespace_set_qos_req(
+                        subsystem_nqn=nqn,
+                        nsid=int(nsid),
+                        rw_ios_per_second=rw_ios_per_second,
+                        rw_mbytes_per_second=rw_mbytes_per_second,
+                        r_mbytes_per_second=r_mbytes_per_second,
+                        w_mbytes_per_second=w_mbytes_per_second,
+                    )
+                )
+                if response.status != 0:
+                    return response
+
+            return response
+
+        @EndpointDoc(
+            "Delete an existing NVMeoF namespace",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "nsid": Param(str, "NVMeoF Namespace ID"),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def delete(self, nqn: str, nsid: str):
+            return NVMeoFClient().stub.namespace_delete(
+                NVMeoFClient.pb2.namespace_delete_req(subsystem_nqn=nqn, nsid=int(nsid))
+            )
+
+    @APIRouter("/nvmeof/subsystem/{nqn}/host", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Subsystem Host Allowlist Management API",
+            "NVMe-oF Subsystem Host Allowlist")
+    class NVMeoFHost(RESTController):
+        @EndpointDoc(
+            "List all allowed hosts for an NVMeoF subsystem",
+            parameters={"nqn": Param(str, "NVMeoF subsystem NQN")},
+        )
+        @map_collection(
+            model.Host,
+            pick="hosts",
+            # Display the "allow any host" option as another host item
+            finalize=lambda i, o: [model.Host(nqn="*")._asdict()] + o
+            if i.allow_any_host
+            else o,
+        )
+        @handle_nvmeof_error
+        def list(self, nqn: str):
+            return NVMeoFClient().stub.list_hosts(
+                NVMeoFClient.pb2.list_hosts_req(subsystem=nqn)
+            )
+
+        @EndpointDoc(
+            "Allow hosts to access an NVMeoF subsystem",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "host_nqn": Param(str, 'NVMeoF host NQN. Use "*" to allow any host.'),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def create(self, nqn: str, host_nqn: str):
+            return NVMeoFClient().stub.add_host(
+                NVMeoFClient.pb2.add_host_req(subsystem_nqn=nqn, host_nqn=host_nqn)
+            )
+
+        @EndpointDoc(
+            "Disallow hosts from accessing an NVMeoF subsystem",
+            parameters={
+                "nqn": Param(str, "NVMeoF subsystem NQN"),
+                "host_nqn": Param(str, 'NVMeoF host NQN. Use "*" to disallow any host.'),
+            },
+        )
+        @empty_response
+        @handle_nvmeof_error
+        def delete(self, nqn: str, host_nqn: str):
+            return NVMeoFClient().stub.remove_host(
+                NVMeoFClient.pb2.remove_host_req(subsystem_nqn=nqn, host_nqn=host_nqn)
+            )
+
+    @APIRouter("/nvmeof/subsystem/{nqn}/connection", Scope.NVME_OF)
+    @APIDoc("NVMe-oF Subsystem Connection Management API", "NVMe-oF Subsystem Connection")
+    class NVMeoFConnection(RESTController):
+        @EndpointDoc(
+            "List all NVMeoF Subsystem Connections",
+            parameters={"nqn": Param(str, "NVMeoF subsystem NQN")},
+        )
+        @map_collection(model.Connection, pick="connections")
+        @handle_nvmeof_error
+        def list(self, nqn: str):
+            return NVMeoFClient().stub.list_connections(
+                NVMeoFClient.pb2.list_connections_req(subsystem=nqn)
+            )
diff --git a/src/pybind/mgr/dashboard/model/nvmeof.py b/src/pybind/mgr/dashboard/model/nvmeof.py
new file mode 100644
index 000000000000..543ae6afddde
--- /dev/null
+++ b/src/pybind/mgr/dashboard/model/nvmeof.py
@@ -0,0 +1,92 @@
+from typing import NamedTuple, Optional
+
+
+class GatewayInfo(NamedTuple):
+    cli_version: str
+    version: str
+    name: str
+    group: str
+    addr: str
+    port: int
+    load_balancing_group: int
+    spdk_version: Optional[str] = ""
+
+
+class Subsystem(NamedTuple):
+    nqn: str
+    enable_ha: bool
+    serial_number: str
+    model_number: str
+    min_cntlid: int
+    max_cntlid: int
+    namespace_count: int
+    subtype: str
+    max_namespaces: int
+
+
+class Connection(NamedTuple):
+    traddr: str
+    trsvcid: int
+    trtype: str
+    adrfam: int
+    connected: bool
+    qpairs_count: int
+    controller_id: int
+
+
+class NamespaceCreation(NamedTuple):
+    nsid: int
+
+
+class Namespace(NamedTuple):
+    nsid: Optional[int]
+    uuid: Optional[str]
+    bdev_name: str
+    rbd_image_name: str
+    rbd_pool_name: str
+    load_balancing_group: int
+    rbd_image_size: int
+    block_size: int
+    rw_ios_per_second: int
+    rw_mbytes_per_second: int
+    r_mbytes_per_second: int
+    w_mbytes_per_second: int
+
+
+class NamespaceIOStats(NamedTuple):
+    nsid: int
+    uuid: str
+    bdev_name: str
+    tick_rate: int
+    ticks: int
+    bytes_read: int
+    num_read_ops: int
+    bytes_written: int
+    num_write_ops: int
+    bytes_unmapped: int
+    num_unmap_ops: int
+    read_latency_ticks: int
+    max_read_latency_ticks: int
+    min_read_latency_ticks: int
+    write_latency_ticks: int
+    max_write_latency_ticks: int
+    min_write_latency_ticks: int
+    unmap_latency_ticks: int
+    max_unmap_latency_ticks: int
+    min_unmap_latency_ticks: int
+    copy_latency_ticks: int
+    max_copy_latency_ticks: int
+    min_copy_latency_ticks: int
+    # io_error: List[int]
+
+
+class Listener(NamedTuple):
+    gateway_name: str
+    trtype: str
+    traddr: str
+    adrfam: Optional[str] = "ipv4"
+    trsvcid: Optional[int] = 4420
+
+
+class Host(NamedTuple):
+    nqn: str
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index c59532c7243d..de3a9df0338c 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -7995,7 +7995,7 @@ paths:
       summary: Updates an NFS-Ganesha export
       tags:
       - NFS-Ganesha
-  /api/nvmeof/gateway/info:
+  /api/nvmeof/gateway:
     get:
       parameters: []
       responses:
@@ -8015,10 +8015,32 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: List all NVMeoF gateways
+      summary: Get information about the NVMeoF gateway
       tags:
-      - NVMe-oF
-  /api/nvmeof/hosts:
+      - NVMe-oF Gateway
+  /api/nvmeof/subsystem:
+    get:
+      parameters: []
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: List all NVMeoF subsystems
+      tags:
+      - NVMe-oF Subsystem
     post:
       parameters: []
       requestBody:
@@ -8026,15 +8048,19 @@ paths:
           application/json:
             schema:
               properties:
-                host_nqn:
-                  description: NVMeoF host NQN
-                  type: string
-                subsystem_nqn:
+                enable_ha:
+                  description: Enable high availability
+                  type: boolean
+                max_namespaces:
+                  default: 256
+                  description: Maximum number of namespaces
+                  type: integer
+                nqn:
                   description: NVMeoF subsystem NQN
                   type: string
               required:
-              - subsystem_nqn
-              - host_nqn
+              - nqn
+              - enable_ha
               type: object
       responses:
         '201':
@@ -8058,15 +8084,112 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: Allow hosts to access an NVMeoF subsystem
+      summary: Create a new NVMeoF subsystem
+      tags:
+      - NVMe-oF Subsystem
+  /api/nvmeof/subsystem/{nqn}:
+    delete:
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      - default: 'false'
+        description: Force delete
+        in: query
+        name: force
+        schema:
+          type: boolean
+      responses:
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '204':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource deleted.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Delete an existing NVMeoF subsystem
       tags:
-      - NVMe-oF
-  /api/nvmeof/hosts/{subsystem_nqn}:
+      - NVMe-oF Subsystem
     get:
       parameters:
       - description: NVMeoF subsystem NQN
         in: path
-        name: subsystem_nqn
+        name: nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Get information from a specific NVMeoF subsystem
+      tags:
+      - NVMe-oF Subsystem
+  /api/nvmeof/subsystem/{nqn}/connection:
+    get:
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: List all NVMeoF Subsystem Connections
+      tags:
+      - NVMe-oF Subsystem Connection
+  /api/nvmeof/subsystem/{nqn}/host:
+    get:
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
         required: true
         schema:
           type: string
@@ -8089,17 +8212,61 @@ paths:
       - jwt: []
       summary: List all allowed hosts for an NVMeoF subsystem
       tags:
-      - NVMe-oF
-  /api/nvmeof/hosts/{subsystem_nqn}/{host_nqn}:
+      - NVMe-oF Subsystem Host Allowlist
+    post:
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              properties:
+                host_nqn:
+                  description: NVMeoF host NQN. Use "*" to allow any host.
+                  type: string
+              required:
+              - host_nqn
+              type: object
+      responses:
+        '201':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Resource created.
+        '202':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: Operation is still executing. Please check the task queue.
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: Allow hosts to access an NVMeoF subsystem
+      tags:
+      - NVMe-oF Subsystem Host Allowlist
+  /api/nvmeof/subsystem/{nqn}/host/{host_nqn}:
     delete:
       parameters:
       - description: NVMeoF subsystem NQN
         in: path
-        name: subsystem_nqn
+        name: nqn
         required: true
         schema:
           type: string
-      - description: NVMeoF host NQN
+      - description: NVMeoF host NQN. Use "*" to disallow any host.
         in: path
         name: host_nqn
         required: true
@@ -8129,13 +8296,13 @@ paths:
       - jwt: []
       summary: Disallow hosts from accessing an NVMeoF subsystem
       tags:
-      - NVMe-oF
-  /api/nvmeof/listener:
+      - NVMe-oF Subsystem Host Allowlist
+  /api/nvmeof/subsystem/{nqn}/listener:
     get:
       parameters:
       - description: NVMeoF subsystem NQN
-        in: query
-        name: subsystem_nqn
+        in: path
+        name: nqn
         required: true
         schema:
           type: string
@@ -8158,26 +8325,37 @@ paths:
       - jwt: []
       summary: List all NVMeoF listeners
       tags:
-      - NVMe-oF
+      - NVMe-oF Subsystem Listener
     post:
-      parameters: []
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
       requestBody:
         content:
           application/json:
             schema:
               properties:
+                adrfam:
+                  default: ipv4
+                  description: NVMeoF address family
+                  type: string
                 gateway:
                   description: NVMeoF gateway
                   type: string
-                nqn:
-                  description: NVMeoF subsystem NQN
-                  type: string
                 traddr:
                   description: NVMeoF transport address
                   type: string
+                trsvcid:
+                  default: 4420
+                  description: NVMeoF transport service port
+                  type: integer
               required:
-              - nqn
               - gateway
+              - traddr
               type: object
       responses:
         '201':
@@ -8203,8 +8381,8 @@ paths:
       - jwt: []
       summary: Create a new NVMeoF listener
       tags:
-      - NVMe-oF
-  /api/nvmeof/listener/{nqn}/{gateway}:
+      - NVMe-oF Subsystem Listener
+  /api/nvmeof/subsystem/{nqn}/listener/{gateway}:
     delete:
       parameters:
       - description: NVMeoF subsystem NQN
@@ -8225,6 +8403,11 @@ paths:
         name: traddr
         schema:
           type: string
+      - default: 4420
+        in: query
+        name: trsvcid
+        schema:
+          type: integer
       responses:
         '202':
           content:
@@ -8249,10 +8432,44 @@ paths:
       - jwt: []
       summary: Delete an existing NVMeoF listener
       tags:
-      - NVMe-oF
-  /api/nvmeof/namespace:
+      - NVMe-oF Subsystem Listener
+  /api/nvmeof/subsystem/{nqn}/namespace:
+    get:
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          content:
+            application/vnd.ceph.api.v1.0+json:
+              type: object
+          description: OK
+        '400':
+          description: Operation exception. Please check the response body for details.
+        '401':
+          description: Unauthenticated access. Please login first.
+        '403':
+          description: Unauthorized access. Please check your permissions.
+        '500':
+          description: Unexpected error. Please check the response body for the stack
+            trace.
+      security:
+      - jwt: []
+      summary: List all NVMeoF namespaces in a subsystem
+      tags:
+      - NVMe-oF Subsystem Namespace
     post:
-      parameters: []
+      parameters:
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
       requestBody:
         content:
           application/json:
@@ -8266,23 +8483,22 @@ paths:
                   default: true
                   description: Create RBD image
                   type: boolean
-                image_size:
-                  default: 1024
-                  description: RBD image size
+                load_balancing_group:
+                  description: Load balancing group
                   type: integer
-                rbd_image:
+                rbd_image_name:
                   description: RBD image name
                   type: string
                 rbd_pool:
+                  default: rbd
                   description: RBD pool name
                   type: string
-                subsystem_nqn:
-                  description: NVMeoF subsystem NQN
-                  type: string
+                size:
+                  default: 1024
+                  description: RBD image size
+                  type: integer
               required:
-              - rbd_pool
-              - rbd_image
-              - subsystem_nqn
+              - rbd_image_name
               type: object
       responses:
         '201':
@@ -8308,13 +8524,19 @@ paths:
       - jwt: []
       summary: Create a new NVMeoF namespace
       tags:
-      - NVMe-oF
-  /api/nvmeof/namespace/{subsystem_nqn}:
+      - NVMe-oF Subsystem Namespace
+  /api/nvmeof/subsystem/{nqn}/namespace/{nsid}:
     delete:
       parameters:
       - description: NVMeoF subsystem NQN
         in: path
-        name: subsystem_nqn
+        name: nqn
+        required: true
+        schema:
+          type: string
+      - description: NVMeoF Namespace ID
+        in: path
+        name: nsid
         required: true
         schema:
           type: string
@@ -8342,12 +8564,18 @@ paths:
       - jwt: []
       summary: Delete an existing NVMeoF namespace
       tags:
-      - NVMe-oF
+      - NVMe-oF Subsystem Namespace
     get:
       parameters:
       - description: NVMeoF subsystem NQN
         in: path
-        name: subsystem_nqn
+        name: nqn
+        required: true
+        schema:
+          type: string
+      - description: NVMeoF Namespace ID
+        in: path
+        name: nsid
         required: true
         schema:
           type: string
@@ -8368,62 +8596,53 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: List all NVMeoF namespaces
+      summary: Get info from specified NVMeoF namespace
       tags:
-      - NVMe-oF
-  /api/nvmeof/subsystem:
-    get:
+      - NVMe-oF Subsystem Namespace
+    patch:
       parameters:
-      - allowEmptyValue: true
-        description: NVMeoF subsystem NQN
-        in: query
-        name: subsystem_nqn
+      - description: NVMeoF subsystem NQN
+        in: path
+        name: nqn
+        required: true
+        schema:
+          type: string
+      - description: NVMeoF Namespace ID
+        in: path
+        name: nsid
+        required: true
         schema:
           type: string
-      responses:
-        '200':
-          content:
-            application/vnd.ceph.api.v1.0+json:
-              type: object
-          description: OK
-        '400':
-          description: Operation exception. Please check the response body for details.
-        '401':
-          description: Unauthenticated access. Please login first.
-        '403':
-          description: Unauthorized access. Please check your permissions.
-        '500':
-          description: Unexpected error. Please check the response body for the stack
-            trace.
-      security:
-      - jwt: []
-      summary: List all NVMeoF subsystems
-      tags:
-      - NVMe-oF
-    post:
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               properties:
-                subsystem_nqn:
-                  description: NVMeoF subsystem NQN
-                  type: string
-              required:
-              - subsystem_nqn
+                load_balancing_group:
+                  description: Load balancing group
+                  type: integer
+                r_mbytes_per_second:
+                  description: Read MB/s
+                  type: integer
+                rbd_image_size:
+                  description: RBD image size
+                  type: integer
+                rw_ios_per_second:
+                  description: Read/Write IOPS
+                  type: integer
+                rw_mbytes_per_second:
+                  description: Read/Write MB/s
+                  type: integer
+                w_mbytes_per_second:
+                  description: Write MB/s
+                  type: integer
               type: object
       responses:
-        '201':
-          content:
-            application/vnd.ceph.api.v1.0+json:
-              type: object
-          description: Resource created.
-        '202':
+        '200':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: Operation is still executing. Please check the task queue.
+          description: Resource updated.
         '400':
           description: Operation exception. Please check the response body for details.
         '401':
@@ -8435,35 +8654,30 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: Create a new NVMeoF subsystem
+      summary: Update an existing NVMeoF namespace
       tags:
-      - NVMe-oF
-  /api/nvmeof/subsystem/{subsystem_nqn}:
-    delete:
+      - NVMe-oF Subsystem Namespace
+  /api/nvmeof/subsystem/{nqn}/namespace/{nsid}/io_stats:
+    get:
       parameters:
       - description: NVMeoF subsystem NQN
         in: path
-        name: subsystem_nqn
+        name: nqn
         required: true
         schema:
           type: string
-      - default: false
-        description: Force delete
-        in: query
-        name: force
+      - description: NVMeoF Namespace ID
+        in: path
+        name: nsid
+        required: true
         schema:
-          type: boolean
+          type: string
       responses:
-        '202':
-          content:
-            application/vnd.ceph.api.v1.0+json:
-              type: object
-          description: Operation is still executing. Please check the task queue.
-        '204':
+        '200':
           content:
             application/vnd.ceph.api.v1.0+json:
               type: object
-          description: Resource deleted.
+          description: OK
         '400':
           description: Operation exception. Please check the response body for details.
         '401':
@@ -8475,9 +8689,9 @@ paths:
             trace.
       security:
       - jwt: []
-      summary: Delete an existing NVMeoF subsystem
+      summary: Get IO stats from specified NVMeoF namespace
       tags:
-      - NVMe-oF
+      - NVMe-oF Subsystem Namespace
   /api/osd:
     get:
       parameters: []
@@ -14373,7 +14587,17 @@ tags:
 - description: NFS-Ganesha Cluster Management API
   name: NFS-Ganesha
 - description: NVMe-oF Gateway Management API
-  name: NVMe-oF
+  name: NVMe-oF Gateway
+- description: NVMe-oF Subsystem Management API
+  name: NVMe-oF Subsystem
+- description: NVMe-oF Subsystem Connection Management API
+  name: NVMe-oF Subsystem Connection
+- description: NVMe-oF Subsystem Host Allowlist Management API
+  name: NVMe-oF Subsystem Host Allowlist
+- description: NVMe-oF Subsystem Listener Management API
+  name: NVMe-oF Subsystem Listener
+- description: NVMe-oF Subsystem Namespace Management API
+  name: NVMe-oF Subsystem Namespace
 - description: OSD management API
   name: OSD
 - description: OSD Perf Counters Management API
diff --git a/src/pybind/mgr/dashboard/requirements.txt b/src/pybind/mgr/dashboard/requirements.txt
index f7fc52bf92d8..b5c78ac8bec4 100644
--- a/src/pybind/mgr/dashboard/requirements.txt
+++ b/src/pybind/mgr/dashboard/requirements.txt
@@ -6,7 +6,7 @@ requests
 Routes
 -e ../../../python-common
 prettytable
-pytest==8.0.2
+pytest==7.0.1
 pyyaml
 natsort
 setuptools
diff --git a/src/pybind/mgr/dashboard/services/exception.py b/src/pybind/mgr/dashboard/services/exception.py
index c39209569dbd..e8e019b0f71e 100644
--- a/src/pybind/mgr/dashboard/services/exception.py
+++ b/src/pybind/mgr/dashboard/services/exception.py
@@ -49,9 +49,14 @@ def dashboard_exception_handler(handler, *args, **kwargs):
     except (cherrypy.HTTPRedirect, cherrypy.NotFound, cherrypy.HTTPError):
         raise
     except (ViewCacheNoDataException, DashboardException) as error:
-        logger.exception('Dashboard Exception')
+        http_status = getattr(error, 'status', 400)
+        cherrypy.response.status = http_status
         cherrypy.response.headers['Content-Type'] = 'application/json'
-        cherrypy.response.status = getattr(error, 'status', 400)
+
+        if http_status >= 500:
+            logger.exception('Dashboard Exception')
+        else:
+            logger.info('Dashboard Exception: %s', error)
         return json.dumps(serialize_dashboard_exception(error)).encode('utf-8')
     except Exception as error:
         logger.exception('Internal Server Error')
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index c51d7e80edb7..5dee7dfcfbc3 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -1,121 +1,134 @@
+import functools
 import logging
-from typing import Optional
+from collections.abc import Iterable
+from typing import Any, Callable, Dict, List, NamedTuple, Optional, Type
 
+from ..exceptions import DashboardException
 from .nvmeof_conf import NvmeofGatewaysConfig
 
-logger = logging.getLogger('nvmeof_client')
+logger = logging.getLogger("nvmeof_client")
 
 try:
-    import grpc
+    import grpc  # type: ignore
+    import grpc._channel  # type: ignore
+    from google.protobuf.message import Message  # type: ignore
 
     from .proto import gateway_pb2 as pb2
     from .proto import gateway_pb2_grpc as pb2_grpc
 except ImportError:
     grpc = None
 else:
+
     class NVMeoFClient(object):
+        pb2 = pb2
+
         def __init__(self):
-            logger.info('Initiating nvmeof gateway connection...')
-
-            self.gateway_addr = list(NvmeofGatewaysConfig.get_gateways_config()[
-                                     'gateways'].values())[0]['service_url']
-            self.channel = grpc.insecure_channel(
-                '{}'.format(self.gateway_addr)
-            )
-            logger.info('Found nvmeof gateway at %s', self.gateway_addr)
+            logger.info("Initiating nvmeof gateway connection...")
+
+            self.gateway_addr = list(
+                NvmeofGatewaysConfig.get_gateways_config()["gateways"].values()
+            )[0]["service_url"]
+            self.channel = grpc.insecure_channel("{}".format(self.gateway_addr))
+            logger.info("Found nvmeof gateway at %s", self.gateway_addr)
             self.stub = pb2_grpc.GatewayStub(self.channel)
 
-        def list_subsystems(self, subsystem_nqn: Optional[str] = None):
-            return self.stub.list_subsystems(pb2.list_subsystems_req(
-                subsystem_nqn=subsystem_nqn
-            ))
-
-        def create_subsystem(self, subsystem_nqn: str):
-            return self.stub.create_subsystem(pb2.create_subsystem_req(
-                subsystem_nqn=subsystem_nqn
-            ))
-
-        def delete_subsystem(self, subsystem_nqn: str, force: Optional[bool] = False):
-            return self.stub.delete_subsystem(pb2.delete_subsystem_req(
-                subsystem_nqn=subsystem_nqn,
-                force=force
-            ))
-
-        def list_namespaces(self, subsystem_nqn: str):
-            return self.stub.list_namespaces(pb2.list_namespaces_req(
-                subsystem=subsystem_nqn
-            ))
-
-        def create_namespace(self, rbd_pool_name: str, rbd_image_name: str,
-                             subsystem_nqn: str, block_size: int = 512,
-                             create_image: Optional[bool] = True,
-                             size: Optional[int] = 1024):
-            return self.stub.namespace_add(pb2.namespace_add_req(
-                rbd_pool_name=rbd_pool_name,
-                rbd_image_name=rbd_image_name,
-                subsystem_nqn=subsystem_nqn,
-                block_size=block_size,
-                create_image=create_image,
-                size=size
-            ))
-
-        def delete_namespace(self, subsystem_nqn: str):
-            return self.stub.namespace_delete(pb2.namespace_delete_req(
-                subsystem_nqn=subsystem_nqn
-            ))
-
-        def list_hosts(self, subsystem_nqn: str):
-            return self.stub.list_hosts(pb2.list_hosts_req(
-                subsystem=subsystem_nqn
-            ))
-
-        def add_host(self, subsystem_nqn: str, host_nqn: str):
-            return self.stub.add_host(pb2.add_host_req(
-                subsystem_nqn=subsystem_nqn,
-                host_nqn=host_nqn
-            ))
-
-        def remove_host(self, subsystem_nqn: str, host_nqn: str):
-            return self.stub.remove_host(pb2.remove_host_req(
-                subsystem_nqn=subsystem_nqn,
-                host_nqn=host_nqn
-            ))
-
-        def list_listeners(self, subsystem_nqn: str):
-            return self.stub.list_listeners(pb2.list_listeners_req(
-                subsystem=subsystem_nqn
-            ))
-
-        def create_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            if traddr is None:
-                addr = self.gateway_addr
-                ip_address, _ = addr.split(':')
-                traddr = self._escape_address_if_ipv6(ip_address)
-
-            req = pb2.create_listener_req(
-                nqn=nqn,
-                gateway_name=gateway,
-                traddr=traddr
-            )
-            return self.stub.create_listener(req)
-
-        def delete_listener(self, nqn: str, gateway: str, traddr: Optional[str] = None):
-            if traddr is None:
-                addr = self.gateway_addr
-                ip_address, _ = addr.split(':')
-                traddr = self._escape_address_if_ipv6(ip_address)
-
-            return self.stub.delete_listener(pb2.delete_listener_req(
-                nqn=nqn,
-                gateway_name=gateway,
-                traddr=traddr
-            ))
-
-        def gateway_info(self):
-            return self.stub.get_gateway_info(pb2.get_gateway_info_req())
-
-        def _escape_address_if_ipv6(self, addr):
-            ret_addr = addr
-            if ":" in addr and not addr.strip().startswith("["):
-                ret_addr = f"[{addr}]"
-            return ret_addr
+    def make_namedtuple_from_object(cls: Type[NamedTuple], obj: Any) -> NamedTuple:
+        return cls(
+            **{
+                field: getattr(obj, field)
+                for field in cls._fields
+                if hasattr(obj, field)
+            }
+        )  # type: ignore
+
+    Model = Dict[str, Any]
+
+    def map_model(
+        model: Type[NamedTuple],
+        first: Optional[str] = None,
+    ) -> Callable[..., Callable[..., Model]]:
+        def decorator(func: Callable[..., Message]) -> Callable[..., Model]:
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs) -> Model:
+                message = func(*args, **kwargs)
+                if first:
+                    try:
+                        message = getattr(message, first)[0]
+                    except IndexError:
+                        raise DashboardException(
+                            msg="Not Found", http_status_code=404, component="nvmeof"
+                        )
+
+                return make_namedtuple_from_object(model, message)._asdict()
+
+            return wrapper
+
+        return decorator
+
+    Collection = List[Model]
+
+    def map_collection(
+        model: Type[NamedTuple],
+        pick: str,
+        finalize: Optional[Callable[[Message, Collection], Collection]] = None,
+    ) -> Callable[..., Callable[..., Collection]]:
+        def decorator(func: Callable[..., Message]) -> Callable[..., Collection]:
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs) -> Collection:
+                message = func(*args, **kwargs)
+                collection: Iterable = getattr(message, pick)
+                out = [
+                    make_namedtuple_from_object(model, i)._asdict() for i in collection
+                ]
+                if finalize:
+                    return finalize(message, out)
+                return out
+
+            return wrapper
+
+        return decorator
+
+    import errno
+
+    NVMeoFError2HTTP = {
+        # errno errors
+        errno.EPERM: 403,  # 1
+        errno.ENOENT: 404,  # 2
+        errno.EACCES: 403,  # 13
+        errno.EEXIST: 409,  # 17
+        errno.ENODEV: 404,  # 19
+        # JSONRPC Spec: https://www.jsonrpc.org/specification#error_object
+        -32602: 422,  # Invalid Params
+        -32603: 500,  # Internal Error
+    }
+
+    def handle_nvmeof_error(func: Callable[..., Message]) -> Callable[..., Message]:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Message:
+            try:
+                response = func(*args, **kwargs)
+            except grpc._channel._InactiveRpcError as e:  # pylint: disable=protected-access
+                raise DashboardException(
+                    msg=e.details(),
+                    code=e.code(),
+                    http_status_code=504,
+                    component="nvmeof",
+                )
+
+            if response.status != 0:
+                raise DashboardException(
+                    msg=response.error_message,
+                    code=response.status,
+                    http_status_code=NVMeoFError2HTTP.get(response.status, 400),
+                    component="nvmeof",
+                )
+            return response
+
+        return wrapper
+
+    def empty_response(func: Callable[..., Message]) -> Callable[..., None]:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> None:
+            func(*args, **kwargs)
+
+        return wrapper
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway.proto b/src/pybind/mgr/dashboard/services/proto/gateway.proto
index 81e2b7205ceb..b2698872a90c 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway.proto
+++ b/src/pybind/mgr/dashboard/services/proto/gateway.proto
@@ -105,7 +105,7 @@ message namespace_add_req {
 	optional string uuid = 6;
 	optional int32 anagrpid = 7;
 	optional bool create_image = 8;
-	optional uint32 size = 9;
+	optional uint64 size = 9;
 	optional bool force = 10;
 }
 
@@ -113,7 +113,7 @@ message namespace_resize_req {
 	string subsystem_nqn = 1;
 	optional uint32 nsid = 2;
 	optional string uuid = 3;
-	uint32 new_size = 4;
+	uint64 new_size = 4;
 }
 
 message namespace_get_io_stats_req {
@@ -319,6 +319,7 @@ message subsystem_cli {
 	uint32 max_cntlid = 6;
 	uint32 namespace_count = 7;
 	string subtype = 8;
+	uint32 max_namespaces = 9;
 }
 
 message gateway_info {
@@ -332,6 +333,7 @@ message gateway_info {
 	int32 status = 8;
 	string error_message = 9;
 	optional string spdk_version = 10;
+        uint32 load_balancing_group = 11;
 }
 
 message cli_version {
@@ -453,4 +455,4 @@ message spdk_nvmf_log_flags_and_level_info {
 	repeated spdk_log_flag_info nvmf_log_flags = 3;
 	LogLevel log_level = 4;
 	LogLevel log_print_level = 5;
-}
\ No newline at end of file
+}
diff --git a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
index 6bd40aaee473..6d807fc27a76 100644
--- a/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
+++ b/src/pybind/mgr/dashboard/services/proto/gateway_pb2.py
@@ -19,7 +19,7 @@
     syntax='proto3',
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
-    serialized_pb=b'\n\rgateway.proto\"\x91\x02\n\x11namespace_add_req\x12\x15\n\rrbd_pool_name\x18\x01 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x11\n\x04nsid\x18\x04 \x01(\rH\x00\x88\x01\x01\x12\x12\n\nblock_size\x18\x05 \x01(\r\x12\x11\n\x04uuid\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x07 \x01(\x05H\x02\x88\x01\x01\x12\x19\n\x0c\x63reate_image\x18\x08 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04size\x18\t \x01(\rH\x04\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x0f\n\r_create_imageB\x07\n\x05_size\"w\n\x14namespace_resize_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08new_size\x18\x04 \x01(\rB\x07\n\x05_nsidB\x07\n\x05_uuid\"k\n\x1anamespace_get_io_stats_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\xcc\x02\n\x15namespace_set_qos_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x1e\n\x11rw_ios_per_second\x18\x04 \x01(\x04H\x02\x88\x01\x01\x12!\n\x14rw_mbytes_per_second\x18\x05 \x01(\x04H\x03\x88\x01\x01\x12 \n\x13r_mbytes_per_second\x18\x06 \x01(\x04H\x04\x88\x01\x01\x12 \n\x13w_mbytes_per_second\x18\x07 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x14\n\x12_rw_ios_per_secondB\x17\n\x15_rw_mbytes_per_secondB\x16\n\x14_r_mbytes_per_secondB\x16\n\x14_w_mbytes_per_second\"\x8c\x01\n)namespace_change_load_balancing_group_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08\x61nagrpid\x18\x04 \x01(\x05\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"e\n\x14namespace_delete_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\x87\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x1b\n\x0emax_namespaces\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x11\n\tenable_ha\x18\x04 \x01(\x08\x42\x11\n\x0f_max_namespaces\"K\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x12\n\x05\x66orce\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x08\n\x06_force\"`\n\x13list_namespaces_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"#\n\x0elist_hosts_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\")\n\x14list_connections_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"\x9a\x01\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\x9a\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\'\n\x12list_listeners_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"q\n\x13list_subsystems_req\x12\x1a\n\rsubsystem_nqn\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rserial_number\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x10\n\x0e_subsystem_nqnB\x10\n\x0e_serial_number\"\x14\n\x12get_subsystems_req\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"@\n\x14get_gateway_info_req\x12\x18\n\x0b\x63li_version\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_cli_version\"<\n\x0f\x61na_group_state\x12\x0e\n\x06grp_id\x18\x01 \x01(\r\x12\x19\n\x05state\x18\x02 \x01(\x0e\x32\n.ana_state\"?\n\x0enqn_ana_states\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12 \n\x06states\x18\x02 \x03(\x0b\x32\x10.ana_group_state\"+\n\x08\x61na_info\x12\x1f\n\x06states\x18\x01 \x03(\x0b\x32\x0f.nqn_ana_states\"3\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\"B\n\x0bnsid_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0c\n\x04nsid\x18\x03 \x01(\r\"1\n\x0fsubsystems_info\x12\x1e\n\nsubsystems\x18\x01 \x03(\x0b\x32\n.subsystem\"\xfc\x02\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0f\n\x07subtype\x18\x02 \x01(\t\x12)\n\x10listen_addresses\x18\x03 \x03(\x0b\x32\x0f.listen_address\x12\x14\n\x05hosts\x18\x04 \x03(\x0b\x32\x05.host\x12\x16\n\x0e\x61llow_any_host\x18\x05 \x01(\x08\x12\x1a\n\rserial_number\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0cmodel_number\x18\x07 \x01(\tH\x01\x88\x01\x01\x12\x1b\n\x0emax_namespaces\x18\x08 \x01(\rH\x02\x88\x01\x01\x12\x17\n\nmin_cntlid\x18\t \x01(\rH\x03\x88\x01\x01\x12\x17\n\nmax_cntlid\x18\n \x01(\rH\x04\x88\x01\x01\x12\x1e\n\nnamespaces\x18\x0b \x03(\x0b\x32\n.namespaceB\x10\n\x0e_serial_numberB\x0f\n\r_model_numberB\x11\n\x0f_max_namespacesB\r\n\x0b_min_cntlidB\r\n\x0b_max_cntlid\"w\n\x0elisten_address\x12\x0e\n\x06trtype\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x64rfam\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12\x0f\n\x07trsvcid\x18\x04 \x01(\t\x12\x16\n\ttransport\x18\x05 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_transport\"\xc9\x01\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\tbdev_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05nguid\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x12\n\x05nonce\x18\x07 \x01(\tH\x04\x88\x01\x01\x42\x0c\n\n_bdev_nameB\x08\n\x06_nguidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x08\n\x06_nonce\"`\n\x13subsystems_info_cli\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\"\n\nsubsystems\x18\x03 \x03(\x0b\x32\x0e.subsystem_cli\"\xae\x01\n\rsubsystem_cli\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x11\n\tenable_ha\x18\x02 \x01(\x08\x12\x15\n\rserial_number\x18\x03 \x01(\t\x12\x14\n\x0cmodel_number\x18\x04 \x01(\t\x12\x12\n\nmin_cntlid\x18\x05 \x01(\r\x12\x12\n\nmax_cntlid\x18\x06 \x01(\r\x12\x17\n\x0fnamespace_count\x18\x07 \x01(\r\x12\x0f\n\x07subtype\x18\x08 \x01(\t\"\xd5\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05group\x18\x04 \x01(\t\x12\x0c\n\x04\x61\x64\x64r\x18\x05 \x01(\t\x12\x0c\n\x04port\x18\x06 \x01(\t\x12\x13\n\x0b\x62ool_status\x18\x07 \x01(\x08\x12\x0e\n\x06status\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\x12\x19\n\x0cspdk_version\x18\n \x01(\tH\x00\x88\x01\x01\x42\x0f\n\r_spdk_version\"E\n\x0b\x63li_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"D\n\ngw_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"v\n\rlistener_info\x12\x14\n\x0cgateway_name\x18\x01 \x01(\t\x12\x0e\n\x06trtype\x18\x02 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\r\"Z\n\x0elisteners_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12!\n\tlisteners\x18\x03 \x03(\x0b\x32\x0e.listener_info\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"x\n\nhosts_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x16\n\x0e\x61llow_any_host\x18\x03 \x01(\x08\x12\x15\n\rsubsystem_nqn\x18\x04 \x01(\t\x12\x14\n\x05hosts\x18\x05 \x03(\x0b\x32\x05.host\"\xaa\x01\n\nconnection\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0e\n\x06traddr\x18\x02 \x01(\t\x12\x0f\n\x07trsvcid\x18\x03 \x01(\r\x12\x0e\n\x06trtype\x18\x04 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamily\x12\x11\n\tconnected\x18\x06 \x01(\x08\x12\x14\n\x0cqpairs_count\x18\x07 \x01(\x05\x12\x15\n\rcontroller_id\x18\x08 \x01(\x05\"r\n\x10\x63onnections_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12 \n\x0b\x63onnections\x18\x04 \x03(\x0b\x32\x0b.connection\"\xaa\x02\n\rnamespace_cli\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x04 \x01(\t\x12\x1c\n\x14load_balancing_group\x18\x05 \x01(\r\x12\x12\n\nblock_size\x18\x06 \x01(\r\x12\x16\n\x0erbd_image_size\x18\x07 \x01(\x04\x12\x0c\n\x04uuid\x18\x08 \x01(\t\x12\x19\n\x11rw_ios_per_second\x18\t \x01(\x04\x12\x1c\n\x14rw_mbytes_per_second\x18\n \x01(\x04\x12\x1b\n\x13r_mbytes_per_second\x18\x0b \x01(\x04\x12\x1b\n\x13w_mbytes_per_second\x18\x0c \x01(\x04\"s\n\x0fnamespaces_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\"\n\nnamespaces\x18\x04 \x03(\x0b\x32\x0e.namespace_cli\"\xb7\x05\n\x17namespace_io_stats_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x0c\n\x04nsid\x18\x04 \x01(\r\x12\x0c\n\x04uuid\x18\x05 \x01(\t\x12\x11\n\tbdev_name\x18\x06 \x01(\t\x12\x11\n\ttick_rate\x18\x07 \x01(\x04\x12\r\n\x05ticks\x18\x08 \x01(\x04\x12\x12\n\nbytes_read\x18\t \x01(\x04\x12\x14\n\x0cnum_read_ops\x18\n \x01(\x04\x12\x15\n\rbytes_written\x18\x0b \x01(\x04\x12\x15\n\rnum_write_ops\x18\x0c \x01(\x04\x12\x16\n\x0e\x62ytes_unmapped\x18\r \x01(\x04\x12\x15\n\rnum_unmap_ops\x18\x0e \x01(\x04\x12\x1a\n\x12read_latency_ticks\x18\x0f \x01(\x04\x12\x1e\n\x16max_read_latency_ticks\x18\x10 \x01(\x04\x12\x1e\n\x16min_read_latency_ticks\x18\x11 \x01(\x04\x12\x1b\n\x13write_latency_ticks\x18\x12 \x01(\x04\x12\x1f\n\x17max_write_latency_ticks\x18\x13 \x01(\x04\x12\x1f\n\x17min_write_latency_ticks\x18\x14 \x01(\x04\x12\x1b\n\x13unmap_latency_ticks\x18\x15 \x01(\x04\x12\x1f\n\x17max_unmap_latency_ticks\x18\x16 \x01(\x04\x12\x1f\n\x17min_unmap_latency_ticks\x18\x17 \x01(\x04\x12\x1a\n\x12\x63opy_latency_ticks\x18\x18 \x01(\x04\x12\x1e\n\x16max_copy_latency_ticks\x18\x19 \x01(\x04\x12\x1e\n\x16min_copy_latency_ticks\x18\x1a \x01(\x04\x12\x10\n\x08io_error\x18\x1b \x03(\r\"3\n\x12spdk_log_flag_info\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x02 \x01(\x08\"\xba\x01\n\"spdk_nvmf_log_flags_and_level_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12+\n\x0envmf_log_flags\x18\x03 \x03(\x0b\x32\x13.spdk_log_flag_info\x12\x1c\n\tlog_level\x18\x04 \x01(\x0e\x32\t.LogLevel\x12\"\n\x0flog_print_level\x18\x05 \x01(\x0e\x32\t.LogLevel*#\n\rAddressFamily\x12\x08\n\x04ipv4\x10\x00\x12\x08\n\x04ipv6\x10\x01*C\n\x08LogLevel\x12\t\n\x05\x45RROR\x10\x00\x12\x0b\n\x07WARNING\x10\x01\x12\n\n\x06NOTICE\x10\x02\x12\x08\n\x04INFO\x10\x03\x12\t\n\x05\x44\x45\x42UG\x10\x04*J\n\tana_state\x12\t\n\x05UNSET\x10\x00\x12\r\n\tOPTIMIZED\x10\x01\x12\x11\n\rNON_OPTIMIZED\x10\x02\x12\x10\n\x0cINACCESSIBLE\x10\x03\x32\xaa\x0b\n\x07Gateway\x12\x33\n\rnamespace_add\x12\x12.namespace_add_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12;\n\x0flist_namespaces\x12\x14.list_namespaces_req\x1a\x10.namespaces_info\"\x00\x12\x38\n\x10namespace_resize\x12\x15.namespace_resize_req\x1a\x0b.req_status\"\x00\x12Q\n\x16namespace_get_io_stats\x12\x1b.namespace_get_io_stats_req\x1a\x18.namespace_io_stats_info\"\x00\x12\x41\n\x18namespace_set_qos_limits\x12\x16.namespace_set_qos_req\x1a\x0b.req_status\"\x00\x12\x62\n%namespace_change_load_balancing_group\x12*.namespace_change_load_balancing_group_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10namespace_delete\x12\x15.namespace_delete_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12,\n\nlist_hosts\x12\x0f.list_hosts_req\x1a\x0b.hosts_info\"\x00\x12>\n\x10list_connections\x12\x15.list_connections_req\x1a\x11.connections_info\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x38\n\x0elist_listeners\x12\x13.list_listeners_req\x1a\x0f.listeners_info\"\x00\x12?\n\x0flist_subsystems\x12\x14.list_subsystems_req\x1a\x14.subsystems_info_cli\"\x00\x12\x39\n\x0eget_subsystems\x12\x13.get_subsystems_req\x1a\x10.subsystems_info\"\x00\x12)\n\rset_ana_state\x12\t.ana_info\x1a\x0b.req_status\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3'
+    serialized_pb=b'\n\rgateway.proto\"\xaf\x02\n\x11namespace_add_req\x12\x15\n\rrbd_pool_name\x18\x01 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x11\n\x04nsid\x18\x04 \x01(\rH\x00\x88\x01\x01\x12\x12\n\nblock_size\x18\x05 \x01(\r\x12\x11\n\x04uuid\x18\x06 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x07 \x01(\x05H\x02\x88\x01\x01\x12\x19\n\x0c\x63reate_image\x18\x08 \x01(\x08H\x03\x88\x01\x01\x12\x11\n\x04size\x18\t \x01(\x04H\x04\x88\x01\x01\x12\x12\n\x05\x66orce\x18\n \x01(\x08H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x0f\n\r_create_imageB\x07\n\x05_sizeB\x08\n\x06_force\"w\n\x14namespace_resize_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08new_size\x18\x04 \x01(\x04\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"k\n\x1anamespace_get_io_stats_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\xcc\x02\n\x15namespace_set_qos_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x1e\n\x11rw_ios_per_second\x18\x04 \x01(\x04H\x02\x88\x01\x01\x12!\n\x14rw_mbytes_per_second\x18\x05 \x01(\x04H\x03\x88\x01\x01\x12 \n\x13r_mbytes_per_second\x18\x06 \x01(\x04H\x04\x88\x01\x01\x12 \n\x13w_mbytes_per_second\x18\x07 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuidB\x14\n\x12_rw_ios_per_secondB\x17\n\x15_rw_mbytes_per_secondB\x16\n\x14_r_mbytes_per_secondB\x16\n\x14_w_mbytes_per_second\"\x8c\x01\n)namespace_change_load_balancing_group_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08\x61nagrpid\x18\x04 \x01(\x05\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"e\n\x14namespace_delete_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"\x87\x01\n\x14\x63reate_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x15\n\rserial_number\x18\x02 \x01(\t\x12\x1b\n\x0emax_namespaces\x18\x03 \x01(\rH\x00\x88\x01\x01\x12\x11\n\tenable_ha\x18\x04 \x01(\x08\x42\x11\n\x0f_max_namespaces\"K\n\x14\x64\x65lete_subsystem_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x12\n\x05\x66orce\x18\x02 \x01(\x08H\x00\x88\x01\x01\x42\x08\n\x06_force\"`\n\x13list_namespaces_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\x12\x11\n\x04nsid\x18\x02 \x01(\rH\x00\x88\x01\x01\x12\x11\n\x04uuid\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x07\n\x05_nsidB\x07\n\x05_uuid\"7\n\x0c\x61\x64\x64_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\":\n\x0fremove_host_req\x12\x15\n\rsubsystem_nqn\x18\x01 \x01(\t\x12\x10\n\x08host_nqn\x18\x02 \x01(\t\"#\n\x0elist_hosts_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\")\n\x14list_connections_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"\x9a\x01\n\x13\x63reate_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\x9a\x01\n\x13\x64\x65lete_listener_req\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x14\n\x0cgateway_name\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12#\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamilyH\x00\x88\x01\x01\x12\x14\n\x07trsvcid\x18\x06 \x01(\rH\x01\x88\x01\x01\x42\t\n\x07_adrfamB\n\n\x08_trsvcid\"\'\n\x12list_listeners_req\x12\x11\n\tsubsystem\x18\x01 \x01(\t\"q\n\x13list_subsystems_req\x12\x1a\n\rsubsystem_nqn\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rserial_number\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x10\n\x0e_subsystem_nqnB\x10\n\x0e_serial_number\"\x14\n\x12get_subsystems_req\"\'\n%get_spdk_nvmf_log_flags_and_level_req\"\x1c\n\x1a\x64isable_spdk_nvmf_logs_req\"~\n\x16set_spdk_nvmf_logs_req\x12!\n\tlog_level\x18\x01 \x01(\x0e\x32\t.LogLevelH\x00\x88\x01\x01\x12#\n\x0bprint_level\x18\x02 \x01(\x0e\x32\t.LogLevelH\x01\x88\x01\x01\x42\x0c\n\n_log_levelB\x0e\n\x0c_print_level\"@\n\x14get_gateway_info_req\x12\x18\n\x0b\x63li_version\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_cli_version\"<\n\x0f\x61na_group_state\x12\x0e\n\x06grp_id\x18\x01 \x01(\r\x12\x19\n\x05state\x18\x02 \x01(\x0e\x32\n.ana_state\"?\n\x0enqn_ana_states\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12 \n\x06states\x18\x02 \x03(\x0b\x32\x10.ana_group_state\"+\n\x08\x61na_info\x12\x1f\n\x06states\x18\x01 \x03(\x0b\x32\x0f.nqn_ana_states\"3\n\nreq_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\"B\n\x0bnsid_status\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0c\n\x04nsid\x18\x03 \x01(\r\"1\n\x0fsubsystems_info\x12\x1e\n\nsubsystems\x18\x01 \x03(\x0b\x32\n.subsystem\"\xfc\x02\n\tsubsystem\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0f\n\x07subtype\x18\x02 \x01(\t\x12)\n\x10listen_addresses\x18\x03 \x03(\x0b\x32\x0f.listen_address\x12\x14\n\x05hosts\x18\x04 \x03(\x0b\x32\x05.host\x12\x16\n\x0e\x61llow_any_host\x18\x05 \x01(\x08\x12\x1a\n\rserial_number\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0cmodel_number\x18\x07 \x01(\tH\x01\x88\x01\x01\x12\x1b\n\x0emax_namespaces\x18\x08 \x01(\rH\x02\x88\x01\x01\x12\x17\n\nmin_cntlid\x18\t \x01(\rH\x03\x88\x01\x01\x12\x17\n\nmax_cntlid\x18\n \x01(\rH\x04\x88\x01\x01\x12\x1e\n\nnamespaces\x18\x0b \x03(\x0b\x32\n.namespaceB\x10\n\x0e_serial_numberB\x0f\n\r_model_numberB\x11\n\x0f_max_namespacesB\r\n\x0b_min_cntlidB\r\n\x0b_max_cntlid\"w\n\x0elisten_address\x12\x0e\n\x06trtype\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x64rfam\x18\x02 \x01(\t\x12\x0e\n\x06traddr\x18\x03 \x01(\t\x12\x0f\n\x07trsvcid\x18\x04 \x01(\t\x12\x16\n\ttransport\x18\x05 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_transport\"\xc9\x01\n\tnamespace\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\tbdev_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05nguid\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x11\n\x04uuid\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x15\n\x08\x61nagrpid\x18\x06 \x01(\rH\x03\x88\x01\x01\x12\x12\n\x05nonce\x18\x07 \x01(\tH\x04\x88\x01\x01\x42\x0c\n\n_bdev_nameB\x08\n\x06_nguidB\x07\n\x05_uuidB\x0b\n\t_anagrpidB\x08\n\x06_nonce\"`\n\x13subsystems_info_cli\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\"\n\nsubsystems\x18\x03 \x03(\x0b\x32\x0e.subsystem_cli\"\xc6\x01\n\rsubsystem_cli\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x11\n\tenable_ha\x18\x02 \x01(\x08\x12\x15\n\rserial_number\x18\x03 \x01(\t\x12\x14\n\x0cmodel_number\x18\x04 \x01(\t\x12\x12\n\nmin_cntlid\x18\x05 \x01(\r\x12\x12\n\nmax_cntlid\x18\x06 \x01(\r\x12\x17\n\x0fnamespace_count\x18\x07 \x01(\r\x12\x0f\n\x07subtype\x18\x08 \x01(\t\x12\x16\n\x0emax_namespaces\x18\t \x01(\r\"\xf3\x01\n\x0cgateway_info\x12\x13\n\x0b\x63li_version\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05group\x18\x04 \x01(\t\x12\x0c\n\x04\x61\x64\x64r\x18\x05 \x01(\t\x12\x0c\n\x04port\x18\x06 \x01(\t\x12\x13\n\x0b\x62ool_status\x18\x07 \x01(\x08\x12\x0e\n\x06status\x18\x08 \x01(\x05\x12\x15\n\rerror_message\x18\t \x01(\t\x12\x19\n\x0cspdk_version\x18\n \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x14load_balancing_group\x18\x0b \x01(\rB\x0f\n\r_spdk_version\"E\n\x0b\x63li_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"D\n\ngw_version\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\"v\n\rlistener_info\x12\x14\n\x0cgateway_name\x18\x01 \x01(\t\x12\x0e\n\x06trtype\x18\x02 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x03 \x01(\x0e\x32\x0e.AddressFamily\x12\x0e\n\x06traddr\x18\x04 \x01(\t\x12\x0f\n\x07trsvcid\x18\x05 \x01(\r\"Z\n\x0elisteners_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12!\n\tlisteners\x18\x03 \x03(\x0b\x32\x0e.listener_info\"\x13\n\x04host\x12\x0b\n\x03nqn\x18\x01 \x01(\t\"x\n\nhosts_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x16\n\x0e\x61llow_any_host\x18\x03 \x01(\x08\x12\x15\n\rsubsystem_nqn\x18\x04 \x01(\t\x12\x14\n\x05hosts\x18\x05 \x03(\x0b\x32\x05.host\"\xaa\x01\n\nconnection\x12\x0b\n\x03nqn\x18\x01 \x01(\t\x12\x0e\n\x06traddr\x18\x02 \x01(\t\x12\x0f\n\x07trsvcid\x18\x03 \x01(\r\x12\x0e\n\x06trtype\x18\x04 \x01(\t\x12\x1e\n\x06\x61\x64rfam\x18\x05 \x01(\x0e\x32\x0e.AddressFamily\x12\x11\n\tconnected\x18\x06 \x01(\x08\x12\x14\n\x0cqpairs_count\x18\x07 \x01(\x05\x12\x15\n\rcontroller_id\x18\x08 \x01(\x05\"r\n\x10\x63onnections_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12 \n\x0b\x63onnections\x18\x04 \x03(\x0b\x32\x0b.connection\"\xaa\x02\n\rnamespace_cli\x12\x0c\n\x04nsid\x18\x01 \x01(\r\x12\x11\n\tbdev_name\x18\x02 \x01(\t\x12\x16\n\x0erbd_image_name\x18\x03 \x01(\t\x12\x15\n\rrbd_pool_name\x18\x04 \x01(\t\x12\x1c\n\x14load_balancing_group\x18\x05 \x01(\r\x12\x12\n\nblock_size\x18\x06 \x01(\r\x12\x16\n\x0erbd_image_size\x18\x07 \x01(\x04\x12\x0c\n\x04uuid\x18\x08 \x01(\t\x12\x19\n\x11rw_ios_per_second\x18\t \x01(\x04\x12\x1c\n\x14rw_mbytes_per_second\x18\n \x01(\x04\x12\x1b\n\x13r_mbytes_per_second\x18\x0b \x01(\x04\x12\x1b\n\x13w_mbytes_per_second\x18\x0c \x01(\x04\"s\n\x0fnamespaces_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\"\n\nnamespaces\x18\x04 \x03(\x0b\x32\x0e.namespace_cli\"\xb7\x05\n\x17namespace_io_stats_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12\x15\n\rsubsystem_nqn\x18\x03 \x01(\t\x12\x0c\n\x04nsid\x18\x04 \x01(\r\x12\x0c\n\x04uuid\x18\x05 \x01(\t\x12\x11\n\tbdev_name\x18\x06 \x01(\t\x12\x11\n\ttick_rate\x18\x07 \x01(\x04\x12\r\n\x05ticks\x18\x08 \x01(\x04\x12\x12\n\nbytes_read\x18\t \x01(\x04\x12\x14\n\x0cnum_read_ops\x18\n \x01(\x04\x12\x15\n\rbytes_written\x18\x0b \x01(\x04\x12\x15\n\rnum_write_ops\x18\x0c \x01(\x04\x12\x16\n\x0e\x62ytes_unmapped\x18\r \x01(\x04\x12\x15\n\rnum_unmap_ops\x18\x0e \x01(\x04\x12\x1a\n\x12read_latency_ticks\x18\x0f \x01(\x04\x12\x1e\n\x16max_read_latency_ticks\x18\x10 \x01(\x04\x12\x1e\n\x16min_read_latency_ticks\x18\x11 \x01(\x04\x12\x1b\n\x13write_latency_ticks\x18\x12 \x01(\x04\x12\x1f\n\x17max_write_latency_ticks\x18\x13 \x01(\x04\x12\x1f\n\x17min_write_latency_ticks\x18\x14 \x01(\x04\x12\x1b\n\x13unmap_latency_ticks\x18\x15 \x01(\x04\x12\x1f\n\x17max_unmap_latency_ticks\x18\x16 \x01(\x04\x12\x1f\n\x17min_unmap_latency_ticks\x18\x17 \x01(\x04\x12\x1a\n\x12\x63opy_latency_ticks\x18\x18 \x01(\x04\x12\x1e\n\x16max_copy_latency_ticks\x18\x19 \x01(\x04\x12\x1e\n\x16min_copy_latency_ticks\x18\x1a \x01(\x04\x12\x10\n\x08io_error\x18\x1b \x03(\r\"3\n\x12spdk_log_flag_info\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x65nabled\x18\x02 \x01(\x08\"\xba\x01\n\"spdk_nvmf_log_flags_and_level_info\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x15\n\rerror_message\x18\x02 \x01(\t\x12+\n\x0envmf_log_flags\x18\x03 \x03(\x0b\x32\x13.spdk_log_flag_info\x12\x1c\n\tlog_level\x18\x04 \x01(\x0e\x32\t.LogLevel\x12\"\n\x0flog_print_level\x18\x05 \x01(\x0e\x32\t.LogLevel*#\n\rAddressFamily\x12\x08\n\x04ipv4\x10\x00\x12\x08\n\x04ipv6\x10\x01*C\n\x08LogLevel\x12\t\n\x05\x45RROR\x10\x00\x12\x0b\n\x07WARNING\x10\x01\x12\n\n\x06NOTICE\x10\x02\x12\x08\n\x04INFO\x10\x03\x12\t\n\x05\x44\x45\x42UG\x10\x04*J\n\tana_state\x12\t\n\x05UNSET\x10\x00\x12\r\n\tOPTIMIZED\x10\x01\x12\x11\n\rNON_OPTIMIZED\x10\x02\x12\x10\n\x0cINACCESSIBLE\x10\x03\x32\xaa\x0b\n\x07Gateway\x12\x33\n\rnamespace_add\x12\x12.namespace_add_req\x1a\x0c.nsid_status\"\x00\x12\x38\n\x10\x63reate_subsystem\x12\x15.create_subsystem_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10\x64\x65lete_subsystem\x12\x15.delete_subsystem_req\x1a\x0b.req_status\"\x00\x12;\n\x0flist_namespaces\x12\x14.list_namespaces_req\x1a\x10.namespaces_info\"\x00\x12\x38\n\x10namespace_resize\x12\x15.namespace_resize_req\x1a\x0b.req_status\"\x00\x12Q\n\x16namespace_get_io_stats\x12\x1b.namespace_get_io_stats_req\x1a\x18.namespace_io_stats_info\"\x00\x12\x41\n\x18namespace_set_qos_limits\x12\x16.namespace_set_qos_req\x1a\x0b.req_status\"\x00\x12\x62\n%namespace_change_load_balancing_group\x12*.namespace_change_load_balancing_group_req\x1a\x0b.req_status\"\x00\x12\x38\n\x10namespace_delete\x12\x15.namespace_delete_req\x1a\x0b.req_status\"\x00\x12(\n\x08\x61\x64\x64_host\x12\r.add_host_req\x1a\x0b.req_status\"\x00\x12.\n\x0bremove_host\x12\x10.remove_host_req\x1a\x0b.req_status\"\x00\x12,\n\nlist_hosts\x12\x0f.list_hosts_req\x1a\x0b.hosts_info\"\x00\x12>\n\x10list_connections\x12\x15.list_connections_req\x1a\x11.connections_info\"\x00\x12\x36\n\x0f\x63reate_listener\x12\x14.create_listener_req\x1a\x0b.req_status\"\x00\x12\x36\n\x0f\x64\x65lete_listener\x12\x14.delete_listener_req\x1a\x0b.req_status\"\x00\x12\x38\n\x0elist_listeners\x12\x13.list_listeners_req\x1a\x0f.listeners_info\"\x00\x12?\n\x0flist_subsystems\x12\x14.list_subsystems_req\x1a\x14.subsystems_info_cli\"\x00\x12\x39\n\x0eget_subsystems\x12\x13.get_subsystems_req\x1a\x10.subsystems_info\"\x00\x12)\n\rset_ana_state\x12\t.ana_info\x1a\x0b.req_status\"\x00\x12r\n!get_spdk_nvmf_log_flags_and_level\x12&.get_spdk_nvmf_log_flags_and_level_req\x1a#.spdk_nvmf_log_flags_and_level_info\"\x00\x12\x44\n\x16\x64isable_spdk_nvmf_logs\x12\x1b.disable_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12<\n\x12set_spdk_nvmf_logs\x12\x17.set_spdk_nvmf_logs_req\x1a\x0b.req_status\"\x00\x12:\n\x10get_gateway_info\x12\x15.get_gateway_info_req\x1a\r.gateway_info\"\x00\x62\x06proto3'
 )
 
 _ADDRESSFAMILY = _descriptor.EnumDescriptor(
@@ -42,8 +42,8 @@
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=6057,
-    serialized_end=6092,
+    serialized_start=6141,
+    serialized_end=6176,
 )
 _sym_db.RegisterEnumDescriptor(_ADDRESSFAMILY)
 
@@ -83,8 +83,8 @@
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=6094,
-    serialized_end=6161,
+    serialized_start=6178,
+    serialized_end=6245,
 )
 _sym_db.RegisterEnumDescriptor(_LOGLEVEL)
 
@@ -119,8 +119,8 @@
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=6163,
-    serialized_end=6237,
+    serialized_start=6247,
+    serialized_end=6321,
 )
 _sym_db.RegisterEnumDescriptor(_ANA_STATE)
 
@@ -204,11 +204,18 @@
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
             name='size', full_name='namespace_add_req.size', index=8,
-            number=9, type=13, cpp_type=3, label=1,
+            number=9, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='force', full_name='namespace_add_req.force', index=9,
+            number=10, type=8, cpp_type=7, label=1,
+            has_default_value=False, default_value=False,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -245,9 +252,14 @@
             index=4, containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[]),
+        _descriptor.OneofDescriptor(
+            name='_force', full_name='namespace_add_req._force',
+            index=5, containing_type=None,
+            create_key=_descriptor._internal_create_key,
+            fields=[]),
     ],
     serialized_start=18,
-    serialized_end=291,
+    serialized_end=321,
 )
 
 
@@ -282,7 +294,7 @@
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
         _descriptor.FieldDescriptor(
             name='new_size', full_name='namespace_resize_req.new_size', index=3,
-            number=4, type=13, cpp_type=3, label=1,
+            number=4, type=4, cpp_type=4, label=1,
             has_default_value=False, default_value=0,
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
@@ -309,8 +321,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=293,
-    serialized_end=412,
+    serialized_start=323,
+    serialized_end=442,
 )
 
 
@@ -365,8 +377,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=414,
-    serialized_end=521,
+    serialized_start=444,
+    serialized_end=551,
 )
 
 
@@ -469,8 +481,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=524,
-    serialized_end=856,
+    serialized_start=554,
+    serialized_end=886,
 )
 
 
@@ -532,8 +544,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=859,
-    serialized_end=999,
+    serialized_start=889,
+    serialized_end=1029,
 )
 
 
@@ -588,8 +600,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1001,
-    serialized_end=1102,
+    serialized_start=1031,
+    serialized_end=1132,
 )
 
 
@@ -646,8 +658,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1105,
-    serialized_end=1240,
+    serialized_start=1135,
+    serialized_end=1270,
 )
 
 
@@ -690,8 +702,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1242,
-    serialized_end=1317,
+    serialized_start=1272,
+    serialized_end=1347,
 )
 
 
@@ -746,8 +758,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1319,
-    serialized_end=1415,
+    serialized_start=1349,
+    serialized_end=1445,
 )
 
 
@@ -785,8 +797,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1417,
-    serialized_end=1472,
+    serialized_start=1447,
+    serialized_end=1502,
 )
 
 
@@ -824,8 +836,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1474,
-    serialized_end=1532,
+    serialized_start=1504,
+    serialized_end=1562,
 )
 
 
@@ -856,8 +868,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1534,
-    serialized_end=1569,
+    serialized_start=1564,
+    serialized_end=1599,
 )
 
 
@@ -888,8 +900,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1571,
-    serialized_end=1612,
+    serialized_start=1601,
+    serialized_end=1642,
 )
 
 
@@ -958,8 +970,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1615,
-    serialized_end=1769,
+    serialized_start=1645,
+    serialized_end=1799,
 )
 
 
@@ -1028,8 +1040,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1772,
-    serialized_end=1926,
+    serialized_start=1802,
+    serialized_end=1956,
 )
 
 
@@ -1060,8 +1072,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=1928,
-    serialized_end=1967,
+    serialized_start=1958,
+    serialized_end=1997,
 )
 
 
@@ -1109,8 +1121,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=1969,
-    serialized_end=2082,
+    serialized_start=1999,
+    serialized_end=2112,
 )
 
 
@@ -1134,8 +1146,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2084,
-    serialized_end=2104,
+    serialized_start=2114,
+    serialized_end=2134,
 )
 
 
@@ -1159,8 +1171,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2106,
-    serialized_end=2145,
+    serialized_start=2136,
+    serialized_end=2175,
 )
 
 
@@ -1184,8 +1196,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2147,
-    serialized_end=2175,
+    serialized_start=2177,
+    serialized_end=2205,
 )
 
 
@@ -1233,8 +1245,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=2177,
-    serialized_end=2303,
+    serialized_start=2207,
+    serialized_end=2333,
 )
 
 
@@ -1270,8 +1282,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=2305,
-    serialized_end=2369,
+    serialized_start=2335,
+    serialized_end=2399,
 )
 
 
@@ -1309,8 +1321,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2371,
-    serialized_end=2431,
+    serialized_start=2401,
+    serialized_end=2461,
 )
 
 
@@ -1348,8 +1360,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2433,
-    serialized_end=2496,
+    serialized_start=2463,
+    serialized_end=2526,
 )
 
 
@@ -1380,8 +1392,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2498,
-    serialized_end=2541,
+    serialized_start=2528,
+    serialized_end=2571,
 )
 
 
@@ -1419,8 +1431,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2543,
-    serialized_end=2594,
+    serialized_start=2573,
+    serialized_end=2624,
 )
 
 
@@ -1465,8 +1477,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2596,
-    serialized_end=2662,
+    serialized_start=2626,
+    serialized_end=2692,
 )
 
 
@@ -1497,8 +1509,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=2664,
-    serialized_end=2713,
+    serialized_start=2694,
+    serialized_end=2743,
 )
 
 
@@ -1624,8 +1636,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=2716,
-    serialized_end=3096,
+    serialized_start=2746,
+    serialized_end=3126,
 )
 
 
@@ -1689,8 +1701,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=3098,
-    serialized_end=3217,
+    serialized_start=3128,
+    serialized_end=3247,
 )
 
 
@@ -1788,8 +1800,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=3220,
-    serialized_end=3421,
+    serialized_start=3250,
+    serialized_end=3451,
 )
 
 
@@ -1834,8 +1846,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3423,
-    serialized_end=3519,
+    serialized_start=3453,
+    serialized_end=3549,
 )
 
 
@@ -1903,6 +1915,13 @@
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='max_namespaces', full_name='subsystem_cli.max_namespaces', index=8,
+            number=9, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -1915,8 +1934,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3522,
-    serialized_end=3696,
+    serialized_start=3552,
+    serialized_end=3750,
 )
 
 
@@ -1998,6 +2017,13 @@
             message_type=None, enum_type=None, containing_type=None,
             is_extension=False, extension_scope=None,
             serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+        _descriptor.FieldDescriptor(
+            name='load_balancing_group', full_name='gateway_info.load_balancing_group', index=10,
+            number=11, type=13, cpp_type=3, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None,
+            serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     ],
     extensions=[
     ],
@@ -2015,8 +2041,8 @@
             create_key=_descriptor._internal_create_key,
             fields=[]),
     ],
-    serialized_start=3699,
-    serialized_end=3912,
+    serialized_start=3753,
+    serialized_end=3996,
 )
 
 
@@ -2061,8 +2087,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3914,
-    serialized_end=3983,
+    serialized_start=3998,
+    serialized_end=4067,
 )
 
 
@@ -2107,8 +2133,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=3985,
-    serialized_end=4053,
+    serialized_start=4069,
+    serialized_end=4137,
 )
 
 
@@ -2167,8 +2193,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4055,
-    serialized_end=4173,
+    serialized_start=4139,
+    serialized_end=4257,
 )
 
 
@@ -2213,8 +2239,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4175,
-    serialized_end=4265,
+    serialized_start=4259,
+    serialized_end=4349,
 )
 
 
@@ -2245,8 +2271,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4267,
-    serialized_end=4286,
+    serialized_start=4351,
+    serialized_end=4370,
 )
 
 
@@ -2305,8 +2331,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4288,
-    serialized_end=4408,
+    serialized_start=4372,
+    serialized_end=4492,
 )
 
 
@@ -2386,8 +2412,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4411,
-    serialized_end=4581,
+    serialized_start=4495,
+    serialized_end=4665,
 )
 
 
@@ -2439,8 +2465,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4583,
-    serialized_end=4697,
+    serialized_start=4667,
+    serialized_end=4781,
 )
 
 
@@ -2548,8 +2574,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=4700,
-    serialized_end=4998,
+    serialized_start=4784,
+    serialized_end=5082,
 )
 
 
@@ -2601,8 +2627,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5000,
-    serialized_end=5115,
+    serialized_start=5084,
+    serialized_end=5199,
 )
 
 
@@ -2815,8 +2841,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5118,
-    serialized_end=5813,
+    serialized_start=5202,
+    serialized_end=5897,
 )
 
 
@@ -2854,8 +2880,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5815,
-    serialized_end=5866,
+    serialized_start=5899,
+    serialized_end=5950,
 )
 
 
@@ -2914,8 +2940,8 @@
     extension_ranges=[],
     oneofs=[
     ],
-    serialized_start=5869,
-    serialized_end=6055,
+    serialized_start=5953,
+    serialized_end=6139,
 )
 
 _NAMESPACE_ADD_REQ.oneofs_by_name['_nsid'].fields.append(
@@ -2933,6 +2959,9 @@
 _NAMESPACE_ADD_REQ.oneofs_by_name['_size'].fields.append(
     _NAMESPACE_ADD_REQ.fields_by_name['size'])
 _NAMESPACE_ADD_REQ.fields_by_name['size'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_size']
+_NAMESPACE_ADD_REQ.oneofs_by_name['_force'].fields.append(
+    _NAMESPACE_ADD_REQ.fields_by_name['force'])
+_NAMESPACE_ADD_REQ.fields_by_name['force'].containing_oneof = _NAMESPACE_ADD_REQ.oneofs_by_name['_force']
 _NAMESPACE_RESIZE_REQ.oneofs_by_name['_nsid'].fields.append(
     _NAMESPACE_RESIZE_REQ.fields_by_name['nsid'])
 _NAMESPACE_RESIZE_REQ.fields_by_name['nsid'].containing_oneof = _NAMESPACE_RESIZE_REQ.oneofs_by_name['_nsid']
@@ -3462,8 +3491,8 @@
     index=0,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
-    serialized_start=6240,
-    serialized_end=7690,
+    serialized_start=6324,
+    serialized_end=7774,
     methods=[
         _descriptor.MethodDescriptor(
             name='namespace_add',

From 9951a3a7f314c7cd1eb147fb1b398ac6c5212806 Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Thu, 21 Mar 2024 08:56:58 +0100
Subject: [PATCH 2452/2492] mgr/dashboard: sanitize dashboard user creation

Fixes: https://tracker.ceph.com/issues/65030
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 .../mgr/dashboard/frontend/src/app/core/auth/auth.module.ts  | 5 +++--
 .../src/app/core/auth/user-form/user-form.component.html     | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/auth.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/auth.module.ts
index 74583431c97a..eedcdd686021 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/auth.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/auth.module.ts
@@ -3,7 +3,7 @@ import { NgModule } from '@angular/core';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { RouterModule, Routes } from '@angular/router';
 
-import { NgbNavModule, NgbPopoverModule } from '@ng-bootstrap/ng-bootstrap';
+import { NgbModule, NgbNavModule, NgbPopoverModule } from '@ng-bootstrap/ng-bootstrap';
 import { NgxPipeFunctionModule } from 'ngx-pipe-function';
 
 import { ActionLabels, URLVerbs } from '~/app/shared/constants/app.constants';
@@ -27,7 +27,8 @@ import { UserTabsComponent } from './user-tabs/user-tabs.component';
     NgbNavModule,
     NgbPopoverModule,
     NgxPipeFunctionModule,
-    RouterModule
+    RouterModule,
+    NgbModule
   ],
   declarations: [
     LoginComponent,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/user-form/user-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/user-form/user-form.component.html
index ddb0e6ab8aeb..4169d54c39fa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/user-form/user-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/user-form/user-form.component.html
@@ -23,7 +23,10 @@
                    name="username"
                    formControlName="username"
                    autocomplete="off"
-                   autofocus>
+                   autofocus
+                   ngbTooltip="White spaces at the beginning and end will be trimmed"
+                   i18n-ngbTooltip
+                   cdTrim>
             <span class="invalid-feedback"
                   *ngIf="userForm.showError('username', formDir, 'required')"
                   i18n>This field is required.</span>

From 6a7f2b0fda74aedb213067abe5c0a64b68fbbdc7 Mon Sep 17 00:00:00 2001
From: Matan Breizman <mbreizma@redhat.com>
Date: Wed, 13 Mar 2024 14:19:59 +0000
Subject: [PATCH 2453/2492] doc/dev: osd_internals/snaps.rst: add clone_overlap
 doc

snapdir concept was removed in: https://github.com/ceph/ceph/pull/17579

Co-authored-by: Zac Dover <zac.dover@proton.me>
Signed-off-by: Matan Breizman <mbreizma@redhat.com>
---
 doc/dev/osd_internals/manifest.rst |   2 +
 doc/dev/osd_internals/snaps.rst    | 121 +++++++++++++++++++++++++++--
 2 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/doc/dev/osd_internals/manifest.rst b/doc/dev/osd_internals/manifest.rst
index 7be4350ead88..43c23fa71e91 100644
--- a/doc/dev/osd_internals/manifest.rst
+++ b/doc/dev/osd_internals/manifest.rst
@@ -218,6 +218,8 @@ we may want to exploit.
 The dedup-tool needs to be updated to use ``LIST_SNAPS`` to discover
 clones as part of leak detection.
 
+.. _osd-make-writeable:
+
 An important question is how we deal with the fact that many clones
 will frequently have references to the same backing chunks at the same
 offset.  In particular, ``make_writeable`` will generally create a clone
diff --git a/doc/dev/osd_internals/snaps.rst b/doc/dev/osd_internals/snaps.rst
index 5ebd0884af4f..736d0add5208 100644
--- a/doc/dev/osd_internals/snaps.rst
+++ b/doc/dev/osd_internals/snaps.rst
@@ -23,12 +23,11 @@ The difference between *pool snaps* and *self managed snaps* from the
 OSD's point of view lies in whether the *SnapContext* comes to the OSD
 via the client's MOSDOp or via the most recent OSDMap.
 
-See OSD::make_writeable
+See :ref:`manifest.rst <osd-make-writeable>` for more information.
 
 Ondisk Structures
 -----------------
-Each object has in the PG collection a *head* object (or *snapdir*, which we
-will come to shortly) and possibly a set of *clone* objects.
+Each object has in the PG collection a *head* object and possibly a set of *clone* objects.
 Each hobject_t has a snap field.  For the *head* (the only writeable version
 of an object), the snap field is set to CEPH_NOSNAP.  For the *clones*, the
 snap field is set to the *seq* of the *SnapContext* at their creation.
@@ -47,8 +46,12 @@ The *head* object contains a *SnapSet* encoded in an attribute, which tracks
   3. Overlapping intervals between clones for tracking space usage
   4. Clone size
 
-If the *head* is deleted while there are still clones, a *snapdir* object
-is created instead to house the *SnapSet*.
+The *head* can't be deleted while there are still clones. Instead, it is
+marked as whiteout (``object_info_t::FLAG_WHITEOUT``) in order to house the
+*SnapSet* contained in it.
+In that case, the *head* object no longer logically exists.
+
+See: should_whiteout()
 
 Additionally, the *object_info_t* on each clone includes a vector of snaps
 for which clone is defined.
@@ -126,3 +129,111 @@ up to 8 prefixes need to be checked to determine all hobjects in a particular
 snap for a particular PG.  Upon split, the prefixes to check on the parent
 are adjusted such that only the objects remaining in the PG will be visible.
 The children will immediately have the correct mapping.
+
+clone_overlap
+-------------
+Each SnapSet attached to the *head* object contains the overlapping intervals
+between clone objects for optimizing space.
+The overlapping intervals are stored within the ``clone_overlap`` map, each element in the
+map stores the snap ID and the corresponding overlap with the next newest clone.
+
+See the following example using a 4 byte object:
+
++--------+---------+
+| object | content |
++========+=========+
+| head   | [AAAA]  |
++--------+---------+
+
+listsnaps output is as follows:
+
++---------+-------+------+---------+
+| cloneid | snaps | size | overlap |
++=========+=======+======+=========+
+| head    | -     | 4    |         |
++---------+-------+------+---------+
+
+After taking a snapshot (ID 1) and re-writing the first 2 bytes of the object,
+the clone created will overlap with the new *head* object in its last 2 bytes.
+
++------------+---------+
+| object     | content |
++============+=========+
+| head       | [BBAA]  |
++------------+---------+
+| clone ID 1 | [AAAA]  |
++------------+---------+
+
++---------+-------+------+---------+
+| cloneid | snaps | size | overlap |
++=========+=======+======+=========+
+| 1       | 1     | 4    | [2~2]   |
++---------+-------+------+---------+
+| head    | -     | 4    |         |
++---------+-------+------+---------+
+
+By taking another snapshot (ID 2) and this time re-writing only the first 1 byte of the object,
+the clone created (ID 2) will overlap with the new *head* object in its last 3 bytes.
+While the oldest clone (ID 1) will overlap with the newest clone in its last 2 bytes.
+
++------------+---------+
+| object     | content |
++============+=========+
+| head       | [CBAA]  |
++------------+---------+
+| clone ID 2 | [BBAA]  |
++------------+---------+
+| clone ID 1 | [AAAA]  |
++------------+---------+
+
++---------+-------+------+---------+
+| cloneid | snaps | size | overlap |
++=========+=======+======+=========+
+| 1       | 1     | 4    | [2~2]   |
++---------+-------+------+---------+
+| 2       | 2     | 4    | [1~3]   |
++---------+-------+------+---------+
+| head    | -     | 4    |         |
++---------+-------+------+---------+
+
+If the *head* object will be completely re-written by re-writing 4 bytes,
+the only existing overlap that will remain will be between the two clones.
+
++------------+---------+
+| object     | content |
++============+=========+
+| head       | [DDDD]  |
++------------+---------+
+| clone ID 2 | [BBAA]  |
++------------+---------+
+| clone ID 1 | [AAAA]  |
++------------+---------+
+
++---------+-------+------+---------+
+| cloneid | snaps | size | overlap |
++=========+=======+======+=========+
+| 1       | 1     | 4    | [2~2]   |
++---------+-------+------+---------+
+| 2       | 2     | 4    |         |
++---------+-------+------+---------+
+| head    | -     | 4    |         |
++---------+-------+------+---------+
+
+Lastly, after the last snap (ID 2) is removed and snaptrim kicks in,
+no overlapping intervals will remain:
+
++------------+---------+
+| object     | content |
++============+=========+
+| head       | [DDDD]  |
++------------+---------+
+| clone ID 1 | [AAAA]  |
++------------+---------+
+
++---------+-------+------+---------+
+| cloneid | snaps | size | overlap |
++=========+=======+======+=========+
+| 1       | 1     | 4    |         |
++---------+-------+------+---------+
+| head    | -     | 4    |         |
++---------+-------+------+---------+

From e7eff3cf8738ddd378c2fd70075ac61dbac3cf55 Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Mon, 25 Mar 2024 07:19:24 +1000
Subject: [PATCH 2454/2492] doc/start: improve MDS explanation

Improve the explanation of MDS as requested by Anthony D'Atri here:

https://github.com/ceph/ceph/pull/56367#discussion_r1536667249

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 doc/start/intro.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/doc/start/intro.rst b/doc/start/intro.rst
index 4964779f94c5..390f1b2d8d52 100644
--- a/doc/start/intro.rst
+++ b/doc/start/intro.rst
@@ -52,10 +52,9 @@ The Ceph Metadata Server is necessary to run Ceph File System clients.
   redundancy and high availability.
 
 - **MDSs**: A :term:`Ceph Metadata Server` (MDS, ``ceph-mds``) stores metadata
-  for the :term:`Ceph File System`. Ceph Block Devices and Ceph Object Storage
-  do not use MDS. Ceph Metadata Servers allow POSIX-file-system users to run
-  basic commands (like ``ls``, ``find``, etc.) without placing a burden on the
-  Ceph Storage Cluster.
+  for the :term:`Ceph File System`. Ceph Metadata Servers allow CephFS users to
+  run basic commands (like ``ls``, ``find``, etc.) without placing a burden on
+  the Ceph Storage Cluster.
 
 Ceph stores data as objects within logical storage pools. Using the
 :term:`CRUSH` algorithm, Ceph calculates which placement group (PG) should

From dc53b767e3f48b66b7b2b794d4c53cdc9475e57e Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 25 Mar 2024 07:37:58 +0800
Subject: [PATCH 2455/2492] test: do not increase ref when creating
 intrusive_ptr<CephContext>

before this change, we increment the refcount when constructing
`cct` instrusive_ptr, but nobody owns this smart pointer. also,
`CephContext` 's constructor set its refcount to 1. so, when the
test finishes, the refcount is 1, and this leads to a leakage of
the `CephContext` instance, this not only annoys ASan, and defeats
the purpose of 14d878c8.
```
Indirect leak of 10880000 byte(s) in 1 object(s) allocated from:
    #0 0x5564d173537d in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_ipaddr+0x19b37d) (BuildId: 45c0c7f28b253c04fcb7bb1a43aed52a5526d734)
    #1 0x7fe7f2ccd189 in __gnu_cxx::new_allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:127:27
    #2 0x7fe7f2ccc563 in std::allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:185:32
    #3 0x7fe7f2ccc563 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::allocate(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2396:39
    #4 0x7fe7f2ccc2c0 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::initialize_buffer(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2494:18
    #5 0x7fe7f2cc6192 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::circular_buffer(unsigned long, std::allocator<ceph::logging::ConcreteEntry> const&) /opt/ceph/include/boost/circular_buffer/base.hpp:1039:9
    #6 0x7fe7f2cb91e4 in ceph::logging::Log::Log(ceph::logging::SubsystemMap const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/log/Log.cc:53:5
    #7 0x7fe7f1f8f96d in ceph::common::CephContext::CephContext(unsigned int, ceph::common::CephContext::create_options const&) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:729:16
    #8 0x7fe7f1f8e93b in ceph::common::CephContext::CephContext(unsigned int, code_environment_t, int) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:697:5
    #9 0x5564d1752eb9 in pick_address_find_ip_in_subnet_list_Test::TestBody() /home/jenkins-build/build/workspace/ceph-pull-requests/src/test/test_ipaddr.cc:706:47
    #10 0x5564d18694d6 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #11 0x5564d1820fc2 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #12 0x5564d17d19dc in testing::Test::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2680:5
    #13 0x5564d17d3a12 in testing::TestInfo::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2858:11
    #14 0x5564d17d504b in testing::TestSuite::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:3012:28
    #15 0x5564d17f24d8 in testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5723:44
    #16 0x5564d1871d06 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #17 0x5564d1827932 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #18 0x5564d17f1862 in testing::UnitTest::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5306:10
    #19 0x5564d1775d80 in RUN_ALL_TESTS() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/include/gtest/gtest.h:2486:46
    #20 0x5564d1775d11 in main /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googlemock/src/gmock_main.cc:70:10
```

so, in this change, we do not increase the refcount when
creating cct.

the same applies to `test/common/test_fault_injector.cc`.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/common/test_fault_injector.cc | 5 +++--
 src/test/test_ipaddr.cc                | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test/common/test_fault_injector.cc b/src/test/common/test_fault_injector.cc
index dfa147478ea5..9b5eb1595bc6 100644
--- a/src/test/common/test_fault_injector.cc
+++ b/src/test/common/test_fault_injector.cc
@@ -40,8 +40,9 @@ class Fixture : public testing::Test {
  protected:
   void SetUp() override {
     CephInitParameters params(CEPH_ENTITY_TYPE_CLIENT);
-    cct = common_preinit(params, CODE_ENVIRONMENT_UTILITY,
-                         CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+    cct.reset(common_preinit(params, CODE_ENVIRONMENT_UTILITY,
+			     CINIT_FLAG_NO_DEFAULT_CONFIG_FILE),
+	      false);
     prefix.emplace(cct.get(), ceph_subsys_context);
   }
   void TearDown() override {
diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc
index bc8dbef70d7e..8b362df9880d 100644
--- a/src/test/test_ipaddr.cc
+++ b/src/test/test_ipaddr.cc
@@ -703,7 +703,7 @@ TEST(pick_address, find_ip_in_subnet_list)
   ipv4(&a_two, "10.2.1.123");
   ipv6(&a_three, "2001:1234:5678:90ab::cdef");
 
-  boost::intrusive_ptr<CephContext> cct = new CephContext(CEPH_ENTITY_TYPE_OSD);
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_OSD), false};
 
   // match by network
   result = find_ip_in_subnet_list(

From e316c2e481d4afd446b5baf477487b7505966414 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 25 Mar 2024 07:43:26 +0800
Subject: [PATCH 2456/2492] test/common: avoid leakage of CephContext

before this change, in test_util.cc, we increment the refcount of
when constructing it. but at that moment, nobody really owns it.
also, `CephContext` 's refcount is set to 1 in its constructor.
so, we should not do this. otherwise, the created `CephContext`
is leaked as LeakSanitizer rightly points out:
```
Indirect leak of 10880000 byte(s) in 1 object(s) allocated from:
    #0 0x5632320d27ed in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_util+0x1917ed) (BuildId: ff1df1455bd07b651ad580584a17ea204afeb36e)
    #1 0x7ff9d535b189 in __gnu_cxx::new_allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:127:27
    #2 0x7ff9d535a563 in std::allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:185:32
    #3 0x7ff9d535a563 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::allocate(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2396:39
    #4 0x7ff9d535a2c0 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::initialize_buffer(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2494:18
    #5 0x7ff9d5354192 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::circular_buffer(unsigned long, std::allocator<ceph::logging::ConcreteEntry> const&) /opt/ceph/include/boost/circular_buffer/base.hpp:1039:9
    #6 0x7ff9d53471e4 in ceph::logging::Log::Log(ceph::logging::SubsystemMap const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/log/Log.cc:53:5
    #7 0x7ff9d461d96d in ceph::common::CephContext::CephContext(unsigned int, ceph::common::CephContext::create_options const&) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:729:16
    #8 0x7ff9d461c93b in ceph::common::CephContext::CephContext(unsigned int, code_environment_t, int) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:697:5
    #9 0x5632320d52e0 in util_collect_sys_info_Test::TestBody() /home/jenkins-build/build/workspace/ceph-pull-requests/src/test/common/test_util.cc:34:27
    #10 0x563232205c16 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #11 0x5632321c2742 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #12 0x5632321736dc in testing::Test::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2680:5
```
in this change, instead of using a raw pointer, let's
use `boost::intrusive_ptr<CephContext>` to manage the lifecyle
of `CephContext`, this also address the leakage reported by
LeakSanitizer.

the same applies to common/test_context.cc

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/common/test_context.cc | 5 ++---
 src/test/common/test_util.cc    | 8 ++++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/test/common/test_context.cc b/src/test/common/test_context.cc
index 2c846a9ae724..889d000da85c 100644
--- a/src/test/common/test_context.cc
+++ b/src/test/common/test_context.cc
@@ -30,7 +30,7 @@ using namespace std;
 
 TEST(CephContext, do_command)
 {
-  CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
 
   cct->_conf->cluster = "ceph";
 
@@ -89,12 +89,11 @@ TEST(CephContext, do_command)
     string s(out.c_str(), out.length());
     EXPECT_EQ("<config_diff_get><diff><key><default></default><override>" + value + "</override><final>value</final></key><rbd_default_features><default>61</default><final>61</final></rbd_default_features><rbd_qos_exclude_ops><default>0</default><final>0</final></rbd_qos_exclude_ops></diff></config_diff_get>", s);
   }
-  cct->put();
 }
 
 TEST(CephContext, experimental_features)
 {
-  CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
 
   cct->_conf->cluster = "ceph";
 
diff --git a/src/test/common/test_util.cc b/src/test/common/test_util.cc
index 91ac771f8048..8eb989d3047b 100644
--- a/src/test/common/test_util.cc
+++ b/src/test/common/test_util.cc
@@ -31,12 +31,12 @@ TEST(util, collect_sys_info)
 
   map<string, string> sys_info;
 
-  CephContext *cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
-  collect_sys_info(&sys_info, cct);
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
+
+  collect_sys_info(&sys_info, cct.get());
 
   ASSERT_TRUE(sys_info.find("distro") != sys_info.end());
   ASSERT_TRUE(sys_info.find("distro_description") != sys_info.end());
-
-  cct->put();
 }
+
 #endif

From 4deebd6294cb38db92bc7475fec38f26c8663f5c Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 25 Mar 2024 11:19:35 +0800
Subject: [PATCH 2457/2492] test/common: do not leak in MemoryIsZeroSmallTest

before this change, we allocate memory chunks with specified
size using `new []`, but we never free them. when testing with
LeakSanitizer enabled, it rightly points identifies the leakage:

```
Direct leak of 8754 byte(s) in 184 object(s) allocated from:
    #0 0x55c0b2470f0d in operator new[](unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_memory+0x196f0d) (BuildId: d3267dd8819427b804c4729e0467dbe7601fb321)
    #1 0x55c0b247456c in MemoryIsZeroSmallTest_MemoryIsZeroTestSmall_Test::TestBody() /home/jenkins-build/build/workspace/ceph-pull-requests/src/test/common/test_memory.cc:33:18
    #2 0x55c0b2598ee6 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #3 0x55c0b2553b92 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #4 0x55c0b25049dc in testing::Test::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2680:5
    #5 0x55c0b2506a12 in testing::TestInfo::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2858:11
    #6 0x55c0b250804b in testing::TestSuite::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:3012:28
    #7 0x55c0b25254d8 in testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5723:44
    #8 0x55c0b25a16f6 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #9 0x55c0b255a502 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #10 0x55c0b2524862 in testing::UnitTest::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5306:10
    #11 0x55c0b24ab4c0 in RUN_ALL_TESTS() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/include/gtest/gtest.h:2486:46
    #12 0x55c0b24ab451 in main /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googlemock/src/gmock_main.cc:70:10
    #13 0x7f45e065ad8f in __libc_start_call_main csu/../sysdeps/nptl/libc_start_call_main.h:58:16
```

in this change, we free the allocate memory.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/common/test_memory.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/common/test_memory.cc b/src/test/common/test_memory.cc
index 05992e5897c6..9adcbbd1baba 100644
--- a/src/test/common/test_memory.cc
+++ b/src/test/common/test_memory.cc
@@ -30,11 +30,11 @@ TEST_P(MemoryIsZeroBigTest, MemoryIsZeroTestBig) {
 TEST_P(MemoryIsZeroSmallTest, MemoryIsZeroTestSmall) {
   size_t size = GetParam();
   for (size_t i = 0; i < size; i++) {
-    char* data = new char[size]();
-    EXPECT_TRUE(mem_is_zero(data, size));
+    auto data = std::make_unique<char[]>(size);
+    EXPECT_TRUE(mem_is_zero(data.get(), size));
 
     data[i] = 'a';
-    EXPECT_FALSE(mem_is_zero(data, size));
+    EXPECT_FALSE(mem_is_zero(data.get(), size));
   }
 }
 

From 5b731beb2aa75a2348445eb62f0e5935f798ba2b Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Wed, 20 Mar 2024 11:21:03 +0530
Subject: [PATCH 2458/2492] mgr/dashboard: introduce cli commands to set cors
 urls

The benefit is that this will take care of cherrypy reconfiguration

Fixes: https://tracker.ceph.com/issues/65104
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 src/pybind/mgr/dashboard/module.py | 34 ++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
index 837841976564..2149537c157f 100644
--- a/src/pybind/mgr/dashboard/module.py
+++ b/src/pybind/mgr/dashboard/module.py
@@ -456,6 +456,40 @@ def unset_login_banner(self):
         mgr.set_store('custom_login_banner', None)
         return HandleCommandResult(stdout='Login banner removed')
 
+    # allow cors by setting cross_origin_url
+    # the value is a comma separated list of URLs
+    @CLIWriteCommand("dashboard set-cross-origin-url")
+    def set_cross_origin_url(self, value: str):
+        cross_origin_urls = self.get_module_option('cross_origin_url', '')
+        cross_origin_urls_list = [url.strip()
+                                  for url in cross_origin_urls.split(',')]  # type: ignore
+        urls = [v.strip() for v in value.split(',')]
+        for url in urls:
+            if url in cross_origin_urls_list:
+                return -errno.EINVAL, '', 'Cross-origin URL already set'
+            cross_origin_urls_list.append(url)
+        self.set_module_option('cross_origin_url', ','.join(cross_origin_urls_list))
+        configure_cors()
+        return 0, 'Cross-origin URL set', ''
+
+    @CLIReadCommand("dashboard get-cross-origin-url")
+    def get_cross_origin_url(self):
+        urls = self.get_module_option('cross_origin_url', '')
+        if urls:
+            return HandleCommandResult(stdout=urls)  # type: ignore
+        return HandleCommandResult(stdout='No cross-origin URL set')
+
+    @CLIReadCommand("dashboard rm-cross-origin-url")
+    def rm_cross_origin_url(self, value: str):
+        urls = self.get_module_option('cross_origin_url', '')
+        urls_list = [url.strip() for url in urls.split(',')]  # type: ignore
+        if value not in urls_list:
+            return -errno.EINVAL, '', 'Cross-origin URL not set'
+        urls_list.remove(value)
+        self.set_module_option('cross_origin_url', ','.join(urls_list))
+        configure_cors()
+        return 0, 'Cross-origin URL removed', ''
+
     def handle_command(self, inbuf, cmd):
         # pylint: disable=too-many-return-statements
         res = handle_option_command(cmd, inbuf)

From b658bb9c32ad31608921e77603003c004ab0f15b Mon Sep 17 00:00:00 2001
From: Pedro Gonzalez Gomez <pegonzal@redhat.com>
Date: Fri, 22 Mar 2024 15:20:48 +0100
Subject: [PATCH 2459/2492] mgr/dashboard: add system users to rgw user form

Fixes: https://tracker.ceph.com/issues/65074
Signed-off-by: Pedro Gonzalez Gomez <pegonzal@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/rgw.py      |  8 ++++++--
 .../rgw-user-details.component.html              |  4 ++--
 .../rgw-user-details.component.spec.ts           |  6 +++---
 .../rgw-user-form/rgw-user-form.component.html   | 16 ++++++++++++++++
 .../rgw-user-form.component.spec.ts              | 15 +++++++++++----
 .../rgw/rgw-user-form/rgw-user-form.component.ts |  6 ++++--
 src/pybind/mgr/dashboard/openapi.yaml            |  4 ++++
 7 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py
index 3238901c9fd4..4df642a9c733 100644
--- a/src/pybind/mgr/dashboard/controllers/rgw.py
+++ b/src/pybind/mgr/dashboard/controllers/rgw.py
@@ -581,7 +581,7 @@ def get_emails(self, daemon_name=None):
 
     @allow_empty_body
     def create(self, uid, display_name, email=None, max_buckets=None,
-               suspended=None, generate_key=None, access_key=None,
+               system=None, suspended=None, generate_key=None, access_key=None,
                secret_key=None, daemon_name=None):
         params = {'uid': uid}
         if display_name is not None:
@@ -590,6 +590,8 @@ def create(self, uid, display_name, email=None, max_buckets=None,
             params['email'] = email
         if max_buckets is not None:
             params['max-buckets'] = max_buckets
+        if system is not None:
+            params['system'] = system
         if suspended is not None:
             params['suspended'] = suspended
         if generate_key is not None:
@@ -604,7 +606,7 @@ def create(self, uid, display_name, email=None, max_buckets=None,
 
     @allow_empty_body
     def set(self, uid, display_name=None, email=None, max_buckets=None,
-            suspended=None, daemon_name=None):
+            system=None, suspended=None, daemon_name=None):
         params = {'uid': uid}
         if display_name is not None:
             params['display-name'] = display_name
@@ -612,6 +614,8 @@ def set(self, uid, display_name=None, email=None, max_buckets=None,
             params['email'] = email
         if max_buckets is not None:
             params['max-buckets'] = max_buckets
+        if system is not None:
+            params['system'] = system
         if suspended is not None:
             params['suspended'] = suspended
         result = self.proxy(daemon_name, 'POST', 'user', params)
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.html
index 27162404a62b..49aff08e3875 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.html
@@ -60,8 +60,8 @@
         </tr>
         <tr>
           <td i18n
-              class="bold">System</td>
-          <td>{{ user.system === 'true' | booleanText }}</td>
+              class="bold">System user</td>
+          <td>{{ user.system | booleanText }}</td>
         </tr>
         <tr>
           <td i18n
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.spec.ts
index 7b203eb9ccaa..27bb6cc46ef0 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-details/rgw-user-details.component.spec.ts
@@ -29,7 +29,7 @@ describe('RgwUserDetailsComponent', () => {
   });
 
   it('should show correct "System" info', () => {
-    component.selection = { uid: '', email: '', system: 'true', keys: [], swift_keys: [] };
+    component.selection = { uid: '', email: '', system: true, keys: [], swift_keys: [] };
 
     component.ngOnChanges();
     fixture.detectChanges();
@@ -37,10 +37,10 @@ describe('RgwUserDetailsComponent', () => {
     const detailsTab = fixture.debugElement.nativeElement.querySelectorAll(
       '.table.table-striped.table-bordered tr td'
     );
-    expect(detailsTab[10].textContent).toEqual('System');
+    expect(detailsTab[10].textContent).toEqual('System user');
     expect(detailsTab[11].textContent).toEqual('Yes');
 
-    component.selection.system = 'false';
+    component.selection.system = false;
     component.ngOnChanges();
     fixture.detectChanges();
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.html
index 9fec45dfe01a..69e9b4ca29ba 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.html
@@ -164,6 +164,22 @@
           </div>
         </div>
 
+        <!-- System User -->
+        <div class="form-group row">
+          <div class="cd-col-form-offset">
+            <div class="custom-control custom-checkbox">
+              <input class="custom-control-input"
+                     id="system"
+                     type="checkbox"
+                     formControlName="system">
+              <label class="custom-control-label"
+                     for="system"
+                     i18n>System user</label>
+              <cd-helper i18n>System users are distinct from regular users, they are used by the RGW service to perform administrative tasks, manage buckets and objects</cd-helper>
+            </div>
+          </div>
+        </div>
+
         <!-- S3 key -->
         <fieldset *ngIf="!editing">
           <legend i18n>S3 key</legend>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.spec.ts
index 15665d53bb95..64afa205e692 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.spec.ts
@@ -187,6 +187,7 @@ describe('RgwUserFormComponent', () => {
         max_buckets: -1,
         secret_key: '',
         suspended: false,
+        system: false,
         uid: null
       });
     });
@@ -200,7 +201,8 @@ describe('RgwUserFormComponent', () => {
         display_name: null,
         email: null,
         max_buckets: -1,
-        suspended: false
+        suspended: false,
+        system: false
       });
     });
 
@@ -216,6 +218,7 @@ describe('RgwUserFormComponent', () => {
         max_buckets: 0,
         secret_key: '',
         suspended: false,
+        system: false,
         uid: null
       });
     });
@@ -229,7 +232,8 @@ describe('RgwUserFormComponent', () => {
         display_name: null,
         email: null,
         max_buckets: 0,
-        suspended: false
+        suspended: false,
+        system: false
       });
     });
 
@@ -246,6 +250,7 @@ describe('RgwUserFormComponent', () => {
         max_buckets: 100,
         secret_key: '',
         suspended: false,
+        system: false,
         uid: null
       });
     });
@@ -260,7 +265,8 @@ describe('RgwUserFormComponent', () => {
         display_name: null,
         email: null,
         max_buckets: 100,
-        suspended: false
+        suspended: false,
+        system: false
       });
     });
   });
@@ -283,7 +289,8 @@ describe('RgwUserFormComponent', () => {
         display_name: null,
         email: '',
         max_buckets: 1000,
-        suspended: false
+        suspended: false,
+        system: false
       });
     });
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.ts
index 9d4e1ce601b7..b2cb7d6518d9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-user-form/rgw-user-form.component.ts
@@ -109,6 +109,7 @@ export class RgwUserFormComponent extends CdForm implements OnInit {
         1000,
         [CdValidators.requiredIf({ max_buckets_mode: '1' }), CdValidators.number(false)]
       ],
+      system: [false],
       suspended: [false],
       // S3 key
       generate_key: [true],
@@ -577,7 +578,7 @@ export class RgwUserFormComponent extends CdForm implements OnInit {
    * @return {Boolean} Returns TRUE if the general user settings have been modified.
    */
   private _isGeneralDirty(): boolean {
-    return ['display_name', 'email', 'max_buckets_mode', 'max_buckets', 'suspended'].some(
+    return ['display_name', 'email', 'max_buckets_mode', 'max_buckets', 'system', 'suspended'].some(
       (path) => {
         return this.userForm.get(path).dirty;
       }
@@ -624,6 +625,7 @@ export class RgwUserFormComponent extends CdForm implements OnInit {
     const result = {
       uid: this.getUID(),
       display_name: this.userForm.getValue('display_name'),
+      system: this.userForm.getValue('system'),
       suspended: this.userForm.getValue('suspended'),
       email: '',
       max_buckets: this.userForm.getValue('max_buckets'),
@@ -658,7 +660,7 @@ export class RgwUserFormComponent extends CdForm implements OnInit {
    */
   private _getUpdateArgs() {
     const result: Record<string, any> = {};
-    const keys = ['display_name', 'email', 'max_buckets', 'suspended'];
+    const keys = ['display_name', 'email', 'max_buckets', 'system', 'suspended'];
     for (const key of keys) {
       result[key] = this.userForm.getValue(key);
     }
diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml
index 5244d4a983b2..9a14dcf2c0ef 100644
--- a/src/pybind/mgr/dashboard/openapi.yaml
+++ b/src/pybind/mgr/dashboard/openapi.yaml
@@ -11297,6 +11297,8 @@ paths:
                   type: string
                 suspended:
                   type: string
+                system:
+                  type: string
                 uid:
                   type: string
               required:
@@ -11449,6 +11451,8 @@ paths:
                   type: string
                 suspended:
                   type: string
+                system:
+                  type: string
               type: object
       responses:
         '200':

From 2422817b7578da44090ea5e2f540c505bbce1412 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Wed, 20 Mar 2024 15:25:33 +0800
Subject: [PATCH 2460/2492] crimson/os/pg_map: allow multiple shards to create
 new pg mappings at the same time

Also:
* Better detections in case of inconsistent racings, such as:
  * The new mapping is creating towards different cores.
  * Mapping creation is racing with its eracing.
  * Multiple shards are erasing the same mapping at the same time.
* Add more logs to debug in case of unexpected issues.

Fixes: https://tracker.ceph.com/issues/64934
Fixes: https://tracker.ceph.com/issues/64009

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/osd/pg_map.cc | 159 ++++++++++++++++++++++++++------------
 1 file changed, 111 insertions(+), 48 deletions(-)

diff --git a/src/crimson/osd/pg_map.cc b/src/crimson/osd/pg_map.cc
index c9cd09da0649..2f5c0660cc38 100644
--- a/src/crimson/osd/pg_map.cc
+++ b/src/crimson/osd/pg_map.cc
@@ -19,75 +19,138 @@ seastar::future<core_id_t> PGShardMapping::get_or_create_pg_mapping(
   LOG_PREFIX(PGShardMapping::get_or_create_pg_mapping);
   auto find_iter = pg_to_core.find(pgid);
   if (find_iter != pg_to_core.end()) {
-    ceph_assert_always(find_iter->second != NULL_CORE);
-    if (core_expected != NULL_CORE) {
-      ceph_assert_always(find_iter->second == core_expected);
+    auto core_found = find_iter->second;
+    assert(core_found != NULL_CORE);
+    if (core_expected != NULL_CORE && core_expected != core_found) {
+      ERROR("the mapping is inconsistent for pg {}: core {}, expected {}",
+            pgid, core_found, core_expected);
+      ceph_abort("The pg mapping is inconsistent!");
     }
-    return seastar::make_ready_future<core_id_t>(find_iter->second);
+    return seastar::make_ready_future<core_id_t>(core_found);
   } else {
+    DEBUG("calling primary to add mapping for pg {} to the expected core {}",
+          pgid, core_expected);
     return container().invoke_on(
         0, [pgid, core_expected, FNAME](auto &primary_mapping) {
-      auto [insert_iter, inserted] =
-        primary_mapping.pg_to_core.emplace(pgid, core_expected);
-      ceph_assert_always(inserted);
-      ceph_assert_always(primary_mapping.core_to_num_pgs.size() > 0);
-      std::map<core_id_t, unsigned>::iterator core_iter;
-      if (core_expected == NULL_CORE) {
-        core_iter = std::min_element(
-          primary_mapping.core_to_num_pgs.begin(),
-          primary_mapping.core_to_num_pgs.end(),
-          [](const auto &left, const auto &right) {
-            return left.second < right.second;
+      auto core_to_update = core_expected;
+      auto find_iter = primary_mapping.pg_to_core.find(pgid);
+      if (find_iter != primary_mapping.pg_to_core.end()) {
+        // this pgid was already mapped within primary_mapping, assert that the
+        // mapping is consistent and avoid emplacing once again.
+        auto core_found = find_iter->second;
+        assert(core_found != NULL_CORE);
+        if (core_expected != NULL_CORE) {
+          if (core_expected != core_found) {
+            ERROR("the mapping is inconsistent for pg {} (primary): core {}, expected {}",
+                  pgid, core_found, core_expected);
+            ceph_abort("The pg mapping is inconsistent!");
           }
-        );
-        core_expected = core_iter->first;
-      } else {
-        core_iter = primary_mapping.core_to_num_pgs.find(core_expected);
+          // core_expected == core_found
+          DEBUG("mapping pg {} to core {} (primary): already mapped and expected",
+                pgid, core_to_update);
+        } else { // core_expected == NULL_CORE
+          core_to_update = core_found;
+          DEBUG("mapping pg {} to core {} (primary): already mapped",
+                pgid, core_to_update);
+        }
+        // proceed to broadcast core_to_update
+      } else { // find_iter == primary_mapping.pg_to_core.end()
+        // this pgid isn't mapped within primary_mapping,
+        // add the mapping and ajust core_to_num_pgs
+        ceph_assert_always(primary_mapping.core_to_num_pgs.size() > 0);
+        std::map<core_id_t, unsigned>::iterator count_iter;
+        if (core_expected == NULL_CORE) {
+          count_iter = std::min_element(
+            primary_mapping.core_to_num_pgs.begin(),
+            primary_mapping.core_to_num_pgs.end(),
+            [](const auto &left, const auto &right) {
+              return left.second < right.second;
+            }
+          );
+          core_to_update = count_iter->first;
+        } else { // core_expected != NULL_CORE
+          count_iter = primary_mapping.core_to_num_pgs.find(core_to_update);
+        }
+        ceph_assert_always(primary_mapping.core_to_num_pgs.end() != count_iter);
+        ++(count_iter->second);
+        auto [insert_iter, inserted] =
+          primary_mapping.pg_to_core.emplace(pgid, core_to_update);
+        assert(inserted);
+        DEBUG("mapping pg {} to core {} (primary): num_pgs {}",
+              pgid, core_to_update, count_iter->second);
       }
-      assert(core_expected != NULL_CORE);
-      ceph_assert_always(primary_mapping.core_to_num_pgs.end() != core_iter);
-      insert_iter->second = core_expected;
-      core_iter->second++;
-      DEBUG("mapping pg {} to core {} (primary) with num_pgs {}",
-            pgid, core_expected, core_iter->second);
+      assert(core_to_update != NULL_CORE);
       return primary_mapping.container().invoke_on_others(
-          [pgid, core_expected, FNAME](auto &other_mapping) {
-        auto [insert_iter, inserted] =
-          other_mapping.pg_to_core.emplace(pgid, core_expected);
-        ceph_assert_always(inserted);
-        DEBUG("mapping pg {} to core {} (others)",
-              pgid, core_expected);
+          [pgid, core_to_update, FNAME](auto &other_mapping) {
+        auto find_iter = other_mapping.pg_to_core.find(pgid);
+        if (find_iter == other_mapping.pg_to_core.end()) {
+          DEBUG("mapping pg {} to core {} (others)",
+                pgid, core_to_update);
+          auto [insert_iter, inserted] =
+            other_mapping.pg_to_core.emplace(pgid, core_to_update);
+          assert(inserted);
+        } else {
+          auto core_found = find_iter->second;
+          if (core_found != core_to_update) {
+            ERROR("the mapping is inconsistent for pg {} (others): core {}, expected {}",
+                  pgid, core_found, core_to_update);
+            ceph_abort("The pg mapping is inconsistent!");
+          }
+          DEBUG("mapping pg {} to core {} (others): already mapped",
+                pgid, core_to_update);
+        }
       });
-    }).then([this, pgid, FNAME] {
+    }).then([this, pgid, core_expected, FNAME] {
       auto find_iter = pg_to_core.find(pgid);
-      ceph_assert_always(find_iter != pg_to_core.end());
-      DEBUG("returning pg {} mapping to core {}",
-            pgid, find_iter->second);
-      return seastar::make_ready_future<core_id_t>(find_iter->second);
+      if (find_iter == pg_to_core.end()) {
+        ERROR("the mapping is inconsistent for pg {}: core not found, expected {}",
+              pgid, core_expected);
+        ceph_abort("The pg mapping is inconsistent!");
+      }
+      auto core_found = find_iter->second;
+      if (core_expected != NULL_CORE && core_found != core_expected) {
+        ERROR("the mapping is inconsistent for pg {}: core {}, expected {}",
+              pgid, core_found, core_expected);
+        ceph_abort("The pg mapping is inconsistent!");
+      }
+      DEBUG("returning pg {} mapping to core {} after broadcasted",
+            pgid, core_found);
+      return seastar::make_ready_future<core_id_t>(core_found);
     });
   }
 }
 
 seastar::future<> PGShardMapping::remove_pg_mapping(spg_t pgid) {
   LOG_PREFIX(PGShardMapping::remove_pg_mapping);
-  DEBUG("{}", pgid);
+  auto find_iter = pg_to_core.find(pgid);
+  if (find_iter == pg_to_core.end()) {
+    ERROR("trying to remove non-exist mapping for pg {}", pgid);
+    ceph_abort("The pg mapping is inconsistent!");
+  }
+  DEBUG("calling primary to remove mapping for pg {}", pgid);
   return container().invoke_on(
       0, [pgid, FNAME](auto &primary_mapping) {
-    auto iter = primary_mapping.pg_to_core.find(pgid);
-    ceph_assert_always(iter != primary_mapping.pg_to_core.end());
-    ceph_assert_always(iter->second != NULL_CORE);
-    auto count_iter = primary_mapping.core_to_num_pgs.find(iter->second);
-    ceph_assert_always(count_iter != primary_mapping.core_to_num_pgs.end());
-    ceph_assert_always(count_iter->second > 0);
+    auto find_iter = primary_mapping.pg_to_core.find(pgid);
+    if (find_iter == primary_mapping.pg_to_core.end()) {
+      ERROR("trying to remove non-exist mapping for pg {} (primary)", pgid);
+      ceph_abort("The pg mapping is inconsistent!");
+    }
+    assert(find_iter->second != NULL_CORE);
+    auto count_iter = primary_mapping.core_to_num_pgs.find(find_iter->second);
+    assert(count_iter != primary_mapping.core_to_num_pgs.end());
+    assert(count_iter->second > 0);
     --(count_iter->second);
-    primary_mapping.pg_to_core.erase(iter);
+    primary_mapping.pg_to_core.erase(find_iter);
     DEBUG("pg {} mapping erased (primary)", pgid);
     return primary_mapping.container().invoke_on_others(
       [pgid, FNAME](auto &other_mapping) {
-      auto iter = other_mapping.pg_to_core.find(pgid);
-      ceph_assert_always(iter != other_mapping.pg_to_core.end());
-      ceph_assert_always(iter->second != NULL_CORE);
-      other_mapping.pg_to_core.erase(iter);
+      auto find_iter = other_mapping.pg_to_core.find(pgid);
+      if (find_iter == other_mapping.pg_to_core.end()) {
+        ERROR("trying to remove non-exist mapping for pg {} (others)", pgid);
+        ceph_abort("The pg mapping is inconsistent!");
+      }
+      assert(find_iter->second != NULL_CORE);
+      other_mapping.pg_to_core.erase(find_iter);
       DEBUG("pg {} mapping erased (others)", pgid);
     });
   });

From 4a180f559a3fff1615664898603fc832cfd8d72c Mon Sep 17 00:00:00 2001
From: junxiang Mu <1948535941@qq.com>
Date: Thu, 21 Mar 2024 22:44:49 -0400
Subject: [PATCH 2461/2492] crimson/osd/osd_operations/snaptrim_event: update
 PG's stats

Fixes: https://tracker.ceph.com/issues/63307
Signed-off-by: junxiang Mu <1948535941@qq.com>
---
 src/crimson/osd/ops_executer.cc                  |  4 +---
 src/crimson/osd/osd_operations/snaptrim_event.cc | 15 ++++++++-------
 src/crimson/osd/osd_operations/snaptrim_event.h  |  2 --
 src/crimson/osd/pg.cc                            |  9 +++++++++
 src/crimson/osd/pg.h                             |  3 +++
 5 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/crimson/osd/ops_executer.cc b/src/crimson/osd/ops_executer.cc
index 7e11868d0abb..f9c7c7d3b8b3 100644
--- a/src/crimson/osd/ops_executer.cc
+++ b/src/crimson/osd/ops_executer.cc
@@ -1049,9 +1049,7 @@ ObjectContextRef OpsExecuter::prepare_clone(
 
 void OpsExecuter::apply_stats()
 {
-  pg->get_peering_state().apply_op_stats(get_target(), delta_stats);
-  pg->scrubber.handle_op_stats(get_target(), delta_stats);
-  pg->publish_stats_to_osd();
+  pg->apply_stats(get_target(), delta_stats);
 }
 
 OpsExecuter::OpsExecuter(Ref<PG> pg,
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.cc b/src/crimson/osd/osd_operations/snaptrim_event.cc
index aaa432e295f9..d75b8e4461a8 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.cc
+++ b/src/crimson/osd/osd_operations/snaptrim_event.cc
@@ -381,7 +381,6 @@ SnapTrimObjSubEvent::remove_or_update(
   }
 
   return seastar::do_with(ceph::os::Transaction{}, [=, this](auto &txn) {
-    int64_t num_objects_before_trim = delta_stats.num_objects;
     osd_op_p.at_version = pg->get_next_version();
     auto ret = remove_or_update_iertr::now();
     if (new_snaps.empty()) {
@@ -396,8 +395,7 @@ SnapTrimObjSubEvent::remove_or_update(
       ret = adjust_snaps(obc, head_obc, new_snaps, txn);
     }
     return std::move(ret).si_then(
-      [&txn, obc, num_objects_before_trim,
-      head_obc=std::move(head_obc), this]() mutable {
+      [&txn, obc, head_obc=std::move(head_obc), this]() mutable {
       // save head snapset
       logger().debug("{}: {} new snapset {} on {}",
 		     *this, coid, head_obc->ssc->snapset, head_obc->obs.oi);
@@ -407,11 +405,14 @@ SnapTrimObjSubEvent::remove_or_update(
 	update_head(obc, head_obc, txn);
       }
       // Stats reporting - Set number of objects trimmed
-      if (num_objects_before_trim > delta_stats.num_objects) {
-	//int64_t num_objects_trimmed =
-	//  num_objects_before_trim - delta_stats.num_objects;
-	//add_objects_trimmed_count(num_objects_trimmed);
+      if (delta_stats.num_objects < 0) {
+        int64_t num_objects_trimmed = std::abs(delta_stats.num_objects);
+        pg->get_peering_state().update_stats_wo_resched(
+          [num_objects_trimmed](auto &history, auto &stats) {
+          stats.objects_trimmed += num_objects_trimmed;
+        });
       }
+      pg->apply_stats(coid, delta_stats);
     }).si_then(
       [&txn] () mutable {
       return std::move(txn);
diff --git a/src/crimson/osd/osd_operations/snaptrim_event.h b/src/crimson/osd/osd_operations/snaptrim_event.h
index 23f8be26c841..66132543b13d 100644
--- a/src/crimson/osd/osd_operations/snaptrim_event.h
+++ b/src/crimson/osd/osd_operations/snaptrim_event.h
@@ -134,8 +134,6 @@ class SnapTrimObjSubEvent : public PhasedOperationT<SnapTrimObjSubEvent> {
   CommonPGPipeline& client_pp();
 
 private:
-  /* TODO: we don't actually update the PG's stats
-   * https://tracker.ceph.com/issues/63307 */
   object_stat_sum_t delta_stats;
 
   snap_trim_obj_subevent_ret_t remove_clone(
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 8c7dd32f19b7..5809c370e433 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -212,6 +212,15 @@ pg_stat_t PG::get_stats() const
   return pg_stats.value_or(pg_stat_t{});
 }
 
+void PG::apply_stats(
+  const hobject_t &soid,
+  const object_stat_sum_t &delta_stats)
+{
+  peering_state.apply_op_stats(soid, delta_stats);
+  scrubber.handle_op_stats(soid, delta_stats);
+  publish_stats_to_osd();
+}
+
 void PG::queue_check_readable(epoch_t last_peering_reset, ceph::timespan delay)
 {
   // handle the peering event in the background
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 0552d49f2ad4..c8cb4010cf06 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -664,6 +664,9 @@ class PG : public boost::intrusive_ref_counter<
   void publish_stats_to_osd() final;
   void clear_publish_stats() final;
   pg_stat_t get_stats() const;
+  void apply_stats(
+    const hobject_t &soid,
+    const object_stat_sum_t &delta_stats);
 
 private:
   std::optional<pg_stat_t> pg_stats;

From 3cd39e3582ed830f86c55d2203ece72fa0f9426b Mon Sep 17 00:00:00 2001
From: Taha Jahangir <mtjahangir@gmail.com>
Date: Mon, 25 Mar 2024 13:32:12 +0330
Subject: [PATCH 2462/2492] docs/rados: remove incorrect ceph command

The removed line was (incorrectly changed) output of the previous command.

Signed-off-by: Taha Jahangir <mtjahangir@gmail.com>
---
 doc/rados/operations/placement-groups.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
index 3875f7cc8145..5fcbc6034952 100644
--- a/doc/rados/operations/placement-groups.rst
+++ b/doc/rados/operations/placement-groups.rst
@@ -183,7 +183,6 @@ The output will resemble the following::
    .. prompt:: bash #
 
       ceph osd pool set .mgr crush_rule replicated-ssd
-      ceph osd pool set pool 1 crush_rule to replicated-ssd
 
    This intervention will result in a small amount of backfill, but
    typically this traffic completes quickly.

From 012b701d1d5a25577dbd864620fa947cd13441fb Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Mon, 25 Mar 2024 10:13:24 -0400
Subject: [PATCH 2463/2492] vstart: require debugging switch for all mds debug
 configs

mds_debug_subtrees wrecks vstart cluster performance particularly egregiously.
Do not turn it on by default.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/vstart.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/vstart.sh b/src/vstart.sh
index 06264325f82e..682f475a5d4d 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -1632,11 +1632,6 @@ osd_debug_op_order = true
 osd_debug_misdirected_ops = true
 osd_copyfrom_max_chunk = 524288
 
-[mds]
-mds_debug_frag = true
-mds_debug_auth_pins = true
-mds_debug_subtrees = true
-
 [mgr]
 mgr/telemetry/nag = false
 mgr/telemetry/enable = false
@@ -1673,6 +1668,9 @@ debug_monc = 20
 debug_mgrc = 20
 mds_debug_scatterstat = true
 mds_verify_scatter = true
+mds_debug_frag = true
+mds_debug_auth_pins = true
+mds_debug_subtrees = true
 EOF
     fi
     if [ "$cephadm" -gt 0 ]; then

From 2ee77b8bd62df3d202b2dbe28a4609bcde58e6fc Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 25 Mar 2024 22:56:28 +0800
Subject: [PATCH 2464/2492] test/rgw/test_rgw_iam_policy: do not increase ref
 when creating intrusive_ptr<CephContext>

before this change, we increment the refcount when constructing
`cct` instrusive_ptr, but nobody owns this smart pointer. also,
`CephContext` 's constructor set its refcount to 1. so, when the
test finishes, the refcount is 1, and this leads to a leakage of
the `CephContext` instance. and LeakSanitizer points this out:

```
Indirect leak of 10880000 byte(s) in 1 object(s) allocated from:
    #0 0xaaaac359c7c8 in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests-arm64/build/bin/unittest_rgw_iam_policy+0x211c7c8) (BuildId: 060fadb10da261b52fd5757c7b1e9812d34542f1)
    #1 0xffff96f764e4 in __gnu_cxx::new_allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:127:27
    #2 0xffff96f757cc in std::allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:185:32
    #3 0xffff96f757cc in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::allocate(unsigned long) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/build/boost/include/boost/circular_buffer/base.hpp:2396:39
    #4 0xffff96f75500 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::initialize_buffer(unsigned long) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/build/boost/include/boost/circular_buffer/base.hpp:2494:18
    #5 0xffff96f6ec4c in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::circular_buffer(unsigned long, std::allocator<ceph::logging::ConcreteEntry> const&) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/build/boost/include/boost/circular_buffer/base.hpp:1039:9
    #6 0xffff96f63528 in ceph::logging::Log::Log(ceph::logging::SubsystemMap const*) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/log/Log.cc:53:5
    #7 0xffff96045300 in ceph::common::CephContext::CephContext(unsigned int, ceph::common::CephContext::create_options const&) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/common/ceph_context.cc:729:16
    #8 0xffff960446ec in ceph::common::CephContext::CephContext(unsigned int, code_environment_t, int) /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/common/ceph_context.cc:697:5
    #9 0xaaaac3629238 in IPPolicyTest::IPPolicyTest() /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/test/rgw/test_rgw_iam_policy.cc:864:15
    #10 0xaaaac3628da0 in IPPolicyTest_MaskedIPOperations_Test::IPPolicyTest_MaskedIPOperations_Test() /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/test/rgw/test_rgw_iam_policy.cc:869:1
    #11 0xaaaac3628d3c in testing::internal::TestFactoryImpl<IPPolicyTest_MaskedIPOperations_Test>::CreateTest() /home/jenkins-build/build/workspace/ceph-pull-requests-arm64/src/googletest/googletest/include/gtest/internal/gtest-internal.h:472:44
```

so, in this change, we do not increase the refcount when creating cct.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/rgw/test_rgw_iam_policy.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc
index dac4d2cef388..d65ef9185e1e 100644
--- a/src/test/rgw/test_rgw_iam_policy.cc
+++ b/src/test/rgw/test_rgw_iam_policy.cc
@@ -159,7 +159,7 @@ class PolicyTest : public ::testing::Test {
   static string example7;
 public:
   PolicyTest() {
-    cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
+    cct.reset(new CephContext(CEPH_ENTITY_TYPE_CLIENT), false);
   }
 };
 
@@ -857,7 +857,7 @@ class IPPolicyTest : public ::testing::Test {
   const rgw::IAM::MaskedIP allowedIPv6Range = { true, rgw::IAM::Address("00100000000000010000110110111000100001011010001100000000000000000000000000000000100010100010111000000011011100000111001100110000"), 124 };
 public:
   IPPolicyTest() {
-    cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT);
+    cct.reset(new CephContext(CEPH_ENTITY_TYPE_CLIENT), false);
   }
 };
 const string IPPolicyTest::arbitrary_tenant = "arbitrary_tenant";

From 5f7c6d58cc90ff139c1f8f8b950ef37d8597f92f Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 25 Mar 2024 12:13:32 -0400
Subject: [PATCH 2465/2492] mgr/rgw: add support to modify zonegroup parameters

This is being done with `radosgw-admin zonegroup set`
rather than `radosgw-admin zonegroup modify` as I don't
think the hostnames parameter (which is the primary focus
for this specific change) can be set using the modify
command. The nice bit about that is it should in theory
make it easy to extend this to allow setting other parameters
to be modified in the zonegroup in the future.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/rgw/module.py                  | 18 +++++--
 .../ceph/deployment/service_spec.py           |  4 +-
 src/python-common/ceph/rgw/rgwam_core.py      | 52 +++++++++++++++++--
 3 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/src/pybind/mgr/rgw/module.py b/src/pybind/mgr/rgw/module.py
index f48e2e09fc32..1b589541932b 100644
--- a/src/pybind/mgr/rgw/module.py
+++ b/src/pybind/mgr/rgw/module.py
@@ -28,7 +28,7 @@
         from typing_extensions import Protocol
 
     class MgrModuleProtocol(Protocol):
-        def tool_exec(self, args: List[str]) -> Tuple[int, str, str]:
+        def tool_exec(self, args: List[str], timeout: int = 10, stdin: Optional[bytes] = None) -> Tuple[int, str, str]:
             ...
 
         def apply_rgw(self, spec: RGWSpec) -> OrchResult[str]:
@@ -66,9 +66,9 @@ class RGWAMOrchMgr(RGWAMEnvMgr):
     def __init__(self, mgr: MgrModuleProtocol):
         self.mgr = mgr
 
-    def tool_exec(self, prog: str, args: List[str]) -> Tuple[List[str], int, str, str]:
+    def tool_exec(self, prog: str, args: List[str], stdin: Optional[bytes] = None) -> Tuple[List[str], int, str, str]:
         cmd = [prog] + args
-        rc, stdout, stderr = self.mgr.tool_exec(args=cmd)
+        rc, stdout, stderr = self.mgr.tool_exec(args=cmd, stdin=stdin)
         return cmd, rc, stdout, stderr
 
     def apply_rgw(self, spec: RGWSpec) -> None:
@@ -286,6 +286,18 @@ def update_zone_info(self, realm_name: str, zonegroup_name: str, zone_name: str,
             self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message))
             return HandleCommandResult(retval=e.retcode, stdout=e.stdout, stderr=e.stderr)
 
+    @CLICommand('rgw zonegroup modify', perm='rw')
+    def update_zonegroup_info(self, realm_name: str, zonegroup_name: str, zone_name: str, hostnames: List[str]) -> HandleCommandResult:
+        try:
+            retval, out, err = RGWAM(self.env).zonegroup_modify(realm_name,
+                                                                zonegroup_name,
+                                                                zone_name,
+                                                                hostnames)
+            return HandleCommandResult(retval, 'Zonegroup updated successfully', '')
+        except RGWAMException as e:
+            self.log.error('cmd run exception: (%d) %s' % (e.retcode, e.message))
+            return HandleCommandResult(retval=e.retcode, stdout=e.stdout, stderr=e.stderr)
+
     @CLICommand('rgw zone create', perm='rw')
     @check_orchestrator
     def _cmd_rgw_zone_create(self,
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 767e1e6f4685..1917e0048025 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1174,10 +1174,11 @@ def __init__(self,
                  rgw_realm_token: Optional[str] = None,
                  update_endpoints: Optional[bool] = False,
                  zone_endpoints: Optional[str] = None,  # comma separated endpoints list
+                 zonegroup_hostnames: Optional[str] = None,
                  rgw_user_counters_cache: Optional[bool] = False,
                  rgw_user_counters_cache_size: Optional[int] = None,
                  rgw_bucket_counters_cache: Optional[bool] = False,
-                 rgw_bucket_counters_cache_size: Optional[int] = None
+                 rgw_bucket_counters_cache_size: Optional[int] = None,
                  ):
         assert service_type == 'rgw', service_type
 
@@ -1217,6 +1218,7 @@ def __init__(self,
         self.rgw_realm_token = rgw_realm_token
         self.update_endpoints = update_endpoints
         self.zone_endpoints = zone_endpoints
+        self.zonegroup_hostnames = zonegroup_hostnames
 
         #: To track op metrics by user config value rgw_user_counters_cache must be set to true
         self.rgw_user_counters_cache = rgw_user_counters_cache
diff --git a/src/python-common/ceph/rgw/rgwam_core.py b/src/python-common/ceph/rgw/rgwam_core.py
index 7041ea1544f0..333f49015852 100644
--- a/src/python-common/ceph/rgw/rgwam_core.py
+++ b/src/python-common/ceph/rgw/rgwam_core.py
@@ -149,11 +149,12 @@ def __init__(self, prog, zone_env: ZoneEnv):
             opt_arg(self.cmd_suffix, '--rgw-zone', zone_env.zone.name)
             opt_arg(self.cmd_suffix, '--zone-id', zone_env.zone.id)
 
-    def run(self, cmd):
+    def run(self, cmd, stdin=None):
         args = cmd + self.cmd_suffix
-        cmd, returncode, stdout, stderr = self.mgr.tool_exec(self.prog, args)
+        cmd, returncode, stdout, stderr = self.mgr.tool_exec(self.prog, args, stdin)
 
         log.debug('cmd=%s' % str(cmd))
+        log.debug(f'stdin={stdin}')
         log.debug('stdout=%s' % stdout)
 
         if returncode != 0:
@@ -174,8 +175,8 @@ class RGWAdminJSONCmd(RGWAdminCmd):
     def __init__(self, zone_env: ZoneEnv):
         super().__init__(zone_env)
 
-    def run(self, cmd):
-        stdout, _ = RGWAdminCmd.run(self, cmd)
+    def run(self, cmd, stdin=None):
+        stdout, _ = RGWAdminCmd.run(self, cmd, stdin)
 
         return json.loads(stdout)
 
@@ -237,9 +238,13 @@ def list(self):
     def get(self, zonegroup: EntityKey = None):
         ze = ZoneEnv(self.env)
         params = ['zonegroup', 'get']
-        opt_arg(params, '--rgw-zonegroup', zonegroup)
         return RGWAdminJSONCmd(ze).run(params)
 
+    def set(self, zonegroup: EntityKey, zg_json: str):
+        ze = ZoneEnv(self.env)
+        params = ['zonegroup', 'set']
+        return RGWAdminJSONCmd(ze).run(params, stdin=zg_json.encode('utf-8'))
+
     def create(self, realm: EntityKey, zg: EntityKey = None, endpoints=None, is_master=True):
         ze = ZoneEnv(self.env, realm=realm).init_zg(zg, gen=True)
 
@@ -724,6 +729,43 @@ def zone_modify(self, realm_name, zonegroup_name, zone_name, endpoints, realm_to
 
         return (0, success_message, '')
 
+    def zonegroup_modify(self, realm_name, zonegroup_name, zone_name, hostnames):
+        if realm_name is None:
+            raise RGWAMException('Realm name is a mandatory parameter')
+        if zone_name is None:
+            raise RGWAMException('Zone name is a mandatory parameter')
+        if zonegroup_name is None:
+            raise RGWAMException('Zonegroup name is a mandatory parameter')
+
+        realm = EntityName(realm_name)
+        zone = EntityName(zone_name)
+        period_info = self.period_op().get(realm)
+        period = RGWPeriod(period_info)
+        logging.info('Period: ' + period.id)
+        zonegroup = period.find_zonegroup_by_name(zonegroup_name)
+        if not zonegroup:
+            raise RGWAMException(f'zonegroup {zonegroup_name} not found')
+        zg = EntityName(zonegroup.name)
+        zg_json = self.zonegroup_op().get(zg)
+
+        if hostnames:
+            zg_json['hostnames'] = hostnames
+
+        try:
+            self.zonegroup_op().set(zg, json.dumps(zg_json))
+        except RGWAMException as e:
+            raise RGWAMException('failed to set zonegroup', e)
+
+        try:
+            period_info = self.period_op().update(realm, zg, zone, True)
+        except RGWAMException as e:
+            raise RGWAMException('failed to update period', e)
+
+        period = RGWPeriod(period_info)
+        logging.debug(period.to_json())
+
+        return (0, f'Modified zonegroup {zonegroup_name} of realm {realm_name}', '')
+
     def get_realms_info(self):
         realms_info = []
         for realm_name in self.realm_op().list():

From 3ba87243edb811033d15dcd95322f0517d701f12 Mon Sep 17 00:00:00 2001
From: Adam King <adking@redhat.com>
Date: Mon, 25 Mar 2024 13:47:12 -0400
Subject: [PATCH 2466/2492] mgr/cephadm: update zonegroup w/
 zonegroup_hostnames from rgw spec

If users provide this field to the rgw spec, we need to modify
the zonegroup to includes these hostnames. This most just makes
use of another commit that added the `ceph rgw zonegroup modify`
command to the rgw module for this specific purpose.

Signed-off-by: Adam King <adking@redhat.com>
---
 src/pybind/mgr/cephadm/services/cephadmservice.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py
index d211bbaa309f..b786255ee1e0 100644
--- a/src/pybind/mgr/cephadm/services/cephadmservice.py
+++ b/src/pybind/mgr/cephadm/services/cephadmservice.py
@@ -965,6 +965,17 @@ def config(self, spec: RGWSpec) -> None:  # type: ignore
                 'val': cert_data,
             })
 
+        if spec.zonegroup_hostnames:
+            zg_update_cmd = {
+                'prefix': 'rgw zonegroup modify',
+                'realm_name': spec.rgw_realm,
+                'zonegroup_name': spec.rgw_zonegroup,
+                'zone_name': spec.rgw_zone,
+                'hostnames': spec.zonegroup_hostnames,
+            }
+            logger.debug(f'rgw cmd: {zg_update_cmd}')
+            ret, out, err = self.mgr.check_mon_command(zg_update_cmd)
+
         # TODO: fail, if we don't have a spec
         logger.info('Saving service %s spec with placement %s' % (
             spec.service_name(), spec.placement.pretty_str()))

From 99b496580b1e791d2640eab7665f21fac49fa0f3 Mon Sep 17 00:00:00 2001
From: Rongqi Sun <sunrongqi@huawei.com>
Date: Fri, 22 Mar 2024 07:39:02 +0000
Subject: [PATCH 2467/2492] run-make: do not pass '-DWITH_FMT_VERSION' to cmake

Clean up this opt because 'WITH_FMT_VERSION' is removed from
src/CMakeLists.txt.
See: https://github.com/ceph/ceph/blob/686dd3d838b6de6bf1101f0b61f729426b0bccad/src/CMakeLists.txt

Fixes: https://tracker.ceph.com/issues/65067
Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
---
 src/script/run-make.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 42d8a94aee1f..ba436f4b59ef 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -28,7 +28,6 @@ function clean_up_after_myself() {
 }
 
 function detect_ceph_dev_pkgs() {
-    local cmake_opts="-DWITH_FMT_VERSION=9.0.0"
     local boost_root=/opt/ceph
     if test -f $boost_root/include/boost/config.hpp; then
         cmake_opts+=" -DWITH_SYSTEM_BOOST=ON -DBOOST_ROOT=$boost_root"

From ffece5ff61b930696f9cb6a709131f90e797850e Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 26 Mar 2024 10:50:16 +0800
Subject: [PATCH 2468/2492] test/mon/test_config_map: free CephContext created
 with new

before this change, we create a new cct instance with `new`, but
we never free this instance after done with it. and LeakSanitizer
points this out:

```
Indirect leak of 10880000 byte(s) in 1 object(s) allocated from:
    #0 0x561afe148fed in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_config_map+0x1c2fed) (BuildId: 3ce9eeed38cee335628fa74fdd08cd215b15019e)
    #1 0x7f37dc9ac189 in __gnu_cxx::new_allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:127:27
    #2 0x7f37dc9ab563 in std::allocator<ceph::logging::ConcreteEntry>::allocate(unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:185:32
    #3 0x7f37dc9ab563 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::allocate(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2396:39
    #4 0x7f37dc9ab2c0 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::initialize_buffer(unsigned long) /opt/ceph/include/boost/circular_buffer/base.hpp:2494:18
    #5 0x7f37dc9a5192 in boost::circular_buffer<ceph::logging::ConcreteEntry, std::allocator<ceph::logging::ConcreteEntry> >::circular_buffer(unsigned long, std::allocator<ceph::logging::ConcreteEntry> const&) /opt/ceph/include/boost/circular_buffer/base.hpp:1039:9
    #6 0x7f37dc9981e4 in ceph::logging::Log::Log(ceph::logging::SubsystemMap const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/log/Log.cc:53:5
    #7 0x7f37dbc6e96d in ceph::common::CephContext::CephContext(unsigned int, ceph::common::CephContext::create_options const&) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:729:16
    #8 0x7f37dbc6d93b in ceph::common::CephContext::CephContext(unsigned int, code_environment_t, int) /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/ceph_context.cc:697:5
    #9 0x561afe14e983 in ConfigMap_add_option_Test::TestBody() /home/jenkins-build/build/workspace/ceph-pull-requests/src/test/mon/test_config_map.cc:58:18
    #10 0x561afe2689b6 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #11 0x561afe221262 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #12 0x561afe1d1f7c in testing::Test::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2680:5
    #13 0x561afe1d3fb2 in testing::TestInfo::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2858:11
    #14 0x561afe1d55eb in testing::TestSuite::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:3012:28
    #15 0x561afe1f2a78 in testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5723:44
    #16 0x561afe2711e6 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #17 0x561afe227bd2 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #18 0x561afe1f1e02 in testing::UnitTest::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5306:10
    #19 0x561afe176ec0 in RUN_ALL_TESTS() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/include/gtest/gtest.h:2486:46
    #20 0x561afe176e51 in main /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googlemock/src/gmock_main.cc:70:10
    #21 0x7f37d9397d8f in __libc_start_call_main csu/../sysdeps/nptl/libc_start_call_main.h:58:16
```

so in this change, we manage the `CephContext` pointer with a smart
pointer. because the size of CephContext could be large, we don't create
it on stack.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/mon/test_config_map.cc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/test/mon/test_config_map.cc b/src/test/mon/test_config_map.cc
index ab148a10bca6..75fbe1ce526e 100644
--- a/src/test/mon/test_config_map.cc
+++ b/src/test/mon/test_config_map.cc
@@ -55,11 +55,11 @@ TEST(ConfigMap, parse_key)
 TEST(ConfigMap, add_option)
 {
   ConfigMap cm;
-  auto cct = new CephContext(CEPH_ENTITY_TYPE_MON);
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
   int r;
 
   r = cm.add_option(
-    cct, "foo", "global", "fooval",
+    cct.get(), "foo", "global", "fooval",
     [&](const std::string& name) {
       return nullptr;
     });
@@ -67,7 +67,7 @@ TEST(ConfigMap, add_option)
   ASSERT_EQ(1, cm.global.options.size());
 
   r = cm.add_option(
-    cct, "foo", "mon", "fooval",
+    cct.get(), "foo", "mon", "fooval",
     [&](const std::string& name) {
       return nullptr;
     });
@@ -76,7 +76,7 @@ TEST(ConfigMap, add_option)
   ASSERT_EQ(1, cm.by_type["mon"].options.size());
   
   r = cm.add_option(
-    cct, "foo", "mon.a", "fooval",
+    cct.get(), "foo", "mon.a", "fooval",
     [&](const std::string& name) {
       return nullptr;
     });
@@ -89,14 +89,14 @@ TEST(ConfigMap, add_option)
 TEST(ConfigMap, result_sections)
 {
   ConfigMap cm;
-  auto cct = new CephContext(CEPH_ENTITY_TYPE_MON);
+  boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
   auto crush = new CrushWrapper;
   crush->finalize();
 
   int r;
 
   r = cm.add_option(
-    cct, "foo", "global", "g",
+    cct.get(), "foo", "global", "g",
     [&](const std::string& name) {
       return nullptr;
     });
@@ -104,7 +104,7 @@ TEST(ConfigMap, result_sections)
   ASSERT_EQ(1, cm.global.options.size());
 
   r = cm.add_option(
-    cct, "foo", "mon", "m",
+    cct.get(), "foo", "mon", "m",
     [&](const std::string& name) {
       return nullptr;
     });
@@ -113,7 +113,7 @@ TEST(ConfigMap, result_sections)
   ASSERT_EQ(1, cm.by_type["mon"].options.size());
 
   r = cm.add_option(
-    cct, "foo", "mon.a", "a",
+    cct.get(), "foo", "mon.a", "a",
     [&](const std::string& name) {
       return nullptr;
     });

From ec64c3e61ed4c4e08d042f231b3130a5b3f1df4d Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Tue, 26 Mar 2024 11:03:38 +0800
Subject: [PATCH 2469/2492] test/mon/test_config_map: free CrushWrapper created
 with new

before this change, we create a new CrushWrapper instance with `new`, but
we never free this instance after done with it. and LeakSanitizer
points this out:

```
Direct leak of 544 byte(s) in 1 object(s) allocated from:
    #0 0x561afe148fed in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_config_map+0x1c2fed) (BuildId: 3ce9eeed38cee335628fa74fdd08cd215b15019e)
    #1 0x561afe151cbd in ConfigMap_result_sections_Test::TestBody() /home/jenkins-build/build/workspace/ceph-pull-requests/src/test/mon/test_config_map.cc:93:16
    #2 0x561afe2689b6 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #3 0x561afe221262 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #4 0x561afe1d1f7c in testing::Test::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2680:5
    #5 0x561afe1d3fb2 in testing::TestInfo::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2858:11
    #6 0x561afe1d55eb in testing::TestSuite::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:3012:28
    #7 0x561afe1f2a78 in testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5723:44
    #8 0x561afe2711e6 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2605:10
    #9 0x561afe227bd2 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:2641:14
    #10 0x561afe1f1e02 in testing::UnitTest::Run() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/src/gtest.cc:5306:10
    #11 0x561afe176ec0 in RUN_ALL_TESTS() /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googletest/include/gtest/gtest.h:2486:46
    #12 0x561afe176e51 in main /home/jenkins-build/build/workspace/ceph-pull-requests/src/googletest/googlemock/src/gmock_main.cc:70:10
    #13 0x7f37d9397d8f in __libc_start_call_main csu/../sysdeps/nptl/libc_start_call_main.h:58:16
```

so in this change, we manage the `CrushWrapper` pointer with a smart
pointer. because the size of `CrushWrapper` is relatively large, we
don't create it on stack.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/test/mon/test_config_map.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/test/mon/test_config_map.cc b/src/test/mon/test_config_map.cc
index 75fbe1ce526e..e147db9779af 100644
--- a/src/test/mon/test_config_map.cc
+++ b/src/test/mon/test_config_map.cc
@@ -4,6 +4,7 @@
 #include "mon/ConfigMap.h"
 
 #include <iostream>
+#include <memory>
 #include <string>
 #include "crush/CrushWrapper.h"
 #include "common/ceph_context.h"
@@ -90,7 +91,7 @@ TEST(ConfigMap, result_sections)
 {
   ConfigMap cm;
   boost::intrusive_ptr<CephContext> cct{new CephContext(CEPH_ENTITY_TYPE_CLIENT), false};
-  auto crush = new CrushWrapper;
+  auto crush = std::make_unique<CrushWrapper>();
   crush->finalize();
 
   int r;
@@ -124,19 +125,19 @@ TEST(ConfigMap, result_sections)
   EntityName n;
   n.set(CEPH_ENTITY_TYPE_MON, "a");
   auto c = cm.generate_entity_map(
-    n, {}, crush, "none", nullptr);
+    n, {}, crush.get(), "none", nullptr);
   ASSERT_EQ(1, c.size());
   ASSERT_EQ("a", c["foo"]);
 
   n.set(CEPH_ENTITY_TYPE_MON, "b");
   c = cm.generate_entity_map(
-    n, {}, crush, "none", nullptr);
+    n, {}, crush.get(), "none", nullptr);
   ASSERT_EQ(1, c.size());
   ASSERT_EQ("m", c["foo"]);
 
   n.set(CEPH_ENTITY_TYPE_MDS, "c");
   c = cm.generate_entity_map(
-    n, {}, crush, "none", nullptr);
+    n, {}, crush.get(), "none", nullptr);
   ASSERT_EQ(1, c.size());
   ASSERT_EQ("g", c["foo"]);
 }

From 7a08bdd264ba81beec74be08d8041167300e49a5 Mon Sep 17 00:00:00 2001
From: Aashish Sharma
 <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com>
Date: Mon, 26 Feb 2024 09:47:34 +0530
Subject: [PATCH 2470/2492] mgr/dashboards: multi-cluster improvements and bug
 fixes

Fixes: https://tracker.ceph.com/issues/64880

Signed-off-by: Aashish Sharma <aasharma@redhat.com>
---
 src/pybind/mgr/dashboard/controllers/auth.py  |  87 ++++++------
 .../multi-cluster-form.component.html         |  39 +++++-
 .../multi-cluster-form.component.ts           |  12 +-
 .../multi-cluster-list.component.ts           |  34 ++---
 .../multi-cluster.component.html              |  24 +++-
 .../multi-cluster/multi-cluster.component.ts  | 129 ++++++++++++++++--
 .../dashboard-area-chart.component.html       |   2 +-
 .../dashboard-area-chart.component.spec.ts    |  10 +-
 .../dashboard-area-chart.component.ts         |   7 +
 .../dashboard-pie/dashboard-pie.component.ts  |   6 +-
 .../navigation/navigation.component.ts        |  14 +-
 .../app/shared/api/multi-cluster.service.ts   |  47 ++++++-
 .../src/app/shared/api/prometheus.service.ts  |  87 +++++-------
 ...critical-confirmation-modal.component.html |   6 +
 .../critical-confirmation-modal.component.ts  |   1 +
 .../app/shared/enum/dashboard-promqls.enum.ts |  20 ++-
 .../services/api-interceptor.service.ts       |  53 +------
 17 files changed, 370 insertions(+), 208 deletions(-)

diff --git a/src/pybind/mgr/dashboard/controllers/auth.py b/src/pybind/mgr/dashboard/controllers/auth.py
index eca87a9a84ed..c2287ef51a80 100644
--- a/src/pybind/mgr/dashboard/controllers/auth.py
+++ b/src/pybind/mgr/dashboard/controllers/auth.py
@@ -54,50 +54,6 @@ def create(self, username, password):
                 pwd_expiration_date = user_data.get('pwdExpirationDate', None)
                 pwd_update_required = user_data.get('pwdUpdateRequired', False)
 
-            if isinstance(Settings.MULTICLUSTER_CONFIG, str):
-                try:
-                    item_to_dict = json.loads(Settings.MULTICLUSTER_CONFIG)
-                except json.JSONDecodeError:
-                    item_to_dict = {}
-                multicluster_config = item_to_dict.copy()
-            else:
-                multicluster_config = Settings.MULTICLUSTER_CONFIG.copy()
-            try:
-                if fsid in multicluster_config['config']:
-                    existing_entries = multicluster_config['config'][fsid]
-                    if not any(entry['user'] == username for entry in existing_entries):
-                        existing_entries.append({
-                            "name": fsid,
-                            "url": origin,
-                            "cluster_alias": "local-cluster",
-                            "user": username
-                        })
-                else:
-                    multicluster_config['config'][fsid] = [{
-                        "name": fsid,
-                        "url": origin,
-                        "cluster_alias": "local-cluster",
-                        "user": username
-                    }]
-
-            except KeyError:
-                multicluster_config = {
-                    'current_url': origin,
-                    'current_user': username,
-                    'hub_url': origin,
-                    'config': {
-                        fsid: [
-                            {
-                                "name": fsid,
-                                "url": origin,
-                                "cluster_alias": "local-cluster",
-                                "user": username
-                            }
-                        ]
-                    }
-                }
-            Settings.MULTICLUSTER_CONFIG = multicluster_config
-
             if user_perms is not None:
                 url_prefix = 'https' if mgr.get_localized_module_option('ssl') else 'http'
 
@@ -110,6 +66,49 @@ def create(self, username, password):
                 token = token.decode('utf-8') if isinstance(token, bytes) else token
 
                 self._set_token_cookie(url_prefix, token)
+                if isinstance(Settings.MULTICLUSTER_CONFIG, str):
+                    try:
+                        item_to_dict = json.loads(Settings.MULTICLUSTER_CONFIG)
+                    except json.JSONDecodeError:
+                        item_to_dict = {}
+                    multicluster_config = item_to_dict.copy()
+                else:
+                    multicluster_config = Settings.MULTICLUSTER_CONFIG.copy()
+                try:
+                    if fsid in multicluster_config['config']:
+                        existing_entries = multicluster_config['config'][fsid]
+                        if not any((entry['user'] == username or entry['cluster_alias'] == 'local-cluster') for entry in existing_entries):  # noqa E501 #pylint: disable=line-too-long
+                            existing_entries.append({
+                                "name": fsid,
+                                "url": origin,
+                                "cluster_alias": "local-cluster",
+                                "user": username
+                            })
+                    else:
+                        multicluster_config['config'][fsid] = [{
+                            "name": fsid,
+                            "url": origin,
+                            "cluster_alias": "local-cluster",
+                            "user": username
+                        }]
+
+                except KeyError:
+                    multicluster_config = {
+                        'current_url': origin,
+                        'current_user': username,
+                        'hub_url': origin,
+                        'config': {
+                            fsid: [
+                                {
+                                    "name": fsid,
+                                    "url": origin,
+                                    "cluster_alias": "local-cluster",
+                                    "user": username
+                                }
+                            ]
+                        }
+                    }
+                Settings.MULTICLUSTER_CONFIG = multicluster_config
                 return {
                     'token': token,
                     'username': username,
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
index a2d36e4232aa..68af8c1672dc 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.html
@@ -20,7 +20,7 @@
         <cd-alert-panel type="info"
                         spacingClass="mb-3"
                         i18n
-                        *ngIf="connectionVerified !== undefined && !connectionVerified && connectionMessage === 'Connection refused'">
+                        *ngIf="connectionVerified !== undefined && !connectionVerified && connectionMessage === 'Connection refused' || remoteClusterForm.getValue('showToken')">
         <p>You need to set this cluster's url as the cross origin url in the remote cluster you are trying to connect.
           You can do it by running this CLI command in your remote cluster and proceed with authentication via token.</p>
           <cd-code-block [codes]="[crossOriginCmd]"></cd-code-block>
@@ -29,7 +29,14 @@
           <label class="cd-col-form-label required"
                  for="remoteClusterUrl"
                  i18n>Cluster API URL
-            <cd-helper>Enter the Dashboard API URL. You can retrieve it from the CLI with: <b>ceph mgr services</b></cd-helper>
+            <cd-helper>
+              <span>
+                <p>Enter the Dashboard API URL. You can retrieve it from the CLI with: <b>{{ clusterApiUrlCmd }} </b>
+                  <cd-copy-2-clipboard-button [source]="clusterApiUrlCmd"
+                                              [byId]="false"></cd-copy-2-clipboard-button>
+                </p>
+              </span>
+            </cd-helper>
           </label>
           <div class="cd-col-form-input">
             <input class="form-control"
@@ -110,6 +117,32 @@
             </span>
           </div>
         </div>
+        <div class="form-group row"
+             *ngIf="remoteClusterForm.getValue('showToken') && action !== 'edit'">
+          <label class="cd-col-form-label required"
+                 for="prometheusApiUrl"
+                 i18n>Prometheus API URL
+            <cd-helper>
+              <span>
+                <p>Enter the Prometheus API URL. You can retrieve it from the CLI with: <b>{{ prometheusApiUrlCmd }} </b>
+                  <cd-copy-2-clipboard-button [source]="prometheusApiUrlCmd"
+                                              [byId]="false"></cd-copy-2-clipboard-button>
+                </p>
+              </span>
+            </cd-helper>
+          </label>
+          <div class="cd-col-form-input">
+            <input id="prometheusApiUrl"
+                   name="prometheusApiUrl"
+                   class="form-control"
+                   type="text"
+                   formControlName="prometheusApiUrl">
+            <span class="invalid-feedback"
+                  *ngIf="remoteClusterForm.showError('prometheusApiUrl', frm, 'required')"
+                  i18n>This field is required.
+            </span>
+          </div>
+        </div>
         <div class="form-group row"
              *ngIf="!remoteClusterForm.getValue('showToken') && !showCrossOriginError && action !== 'edit'">
           <label class="cd-col-form-label required"
@@ -211,7 +244,7 @@
             </div>
           </div>
         <div class="form-group row"
-             *ngIf="!showCrossOriginError && action !== 'edit' && !remoteClusterForm.getValue('showToken')">
+             *ngIf="!showCrossOriginError && action !== 'edit' && !remoteClusterForm.getValue('showToken') && !connectionVerified">
           <div class="cd-col-form-offset">
             <div class="custom-control">
               <button class="btn btn-primary"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
index ee39a51d4700..b5b5f9ca2e16 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-form/multi-cluster-form.component.ts
@@ -22,13 +22,15 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
   readonly endpoints = /^((https?:\/\/)|(www.))(?:([a-zA-Z]+)|(\d+\.\d+.\d+.\d+)):\d{2,5}\/?$/;
   readonly ipv4Rgx = /^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/i;
   readonly ipv6Rgx = /^(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4}$/i;
+  clusterApiUrlCmd = 'ceph mgr services';
+  prometheusApiUrlCmd = 'ceph config get mgr mgr/dashboard/PROMETHEUS_API_HOST';
+  crossOriginCmd = `ceph dashboard set-cross-origin-url ${window.location.origin}`;
   remoteClusterForm: CdFormGroup;
   showToken = false;
   connectionVerified: boolean;
   connectionMessage = '';
   private subs = new Subscription();
   showCrossOriginError = false;
-  crossOriginCmd: string;
   action: string;
   cluster: MultiCluster;
   clustersData: MultiCluster[];
@@ -99,6 +101,11 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
           showToken: true
         })
       ]),
+      prometheusApiUrl: new FormControl('', [
+        CdValidators.requiredIf({
+          showToken: true
+        })
+      ]),
       password: new FormControl('', []),
       remoteClusterUrl: new FormControl(null, {
         validators: [
@@ -152,6 +159,7 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
     const url = this.remoteClusterForm.getValue('remoteClusterUrl');
     const updatedUrl = url.endsWith('/') ? url.slice(0, -1) : url;
     const clusterAlias = this.remoteClusterForm.getValue('clusterAlias');
+    const prometheusApiUrl = this.remoteClusterForm.getValue('prometheusApiUrl');
     const username = this.remoteClusterForm.getValue('username');
     const password = this.remoteClusterForm.getValue('password');
     const token = this.remoteClusterForm.getValue('apiToken');
@@ -210,6 +218,7 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
             token,
             window.location.origin,
             clusterFsid,
+            prometheusApiUrl,
             ssl,
             ssl_certificate
           )
@@ -256,7 +265,6 @@ export class MultiClusterFormComponent implements OnInit, OnDestroy {
               this.connectionVerified = false;
               this.showCrossOriginError = true;
               this.connectionMessage = resp;
-              this.crossOriginCmd = `ceph config set mgr mgr/dashboard/cross_origin_url ${window.location.origin} `;
               this.notificationService.show(
                 NotificationType.error,
                 $localize`Connection to the cluster failed`
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
index 07238019c199..da5e08dfebea 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster-list/multi-cluster-list.component.ts
@@ -16,7 +16,6 @@ import { NotificationService } from '~/app/shared/services/notification.service'
 import { NotificationType } from '~/app/shared/enum/notification-type.enum';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
 import { MultiCluster } from '~/app/shared/models/multi-cluster';
-import { SummaryService } from '~/app/shared/services/summary.service';
 import { Router } from '@angular/router';
 import { CookiesService } from '~/app/shared/services/cookie.service';
 
@@ -41,11 +40,12 @@ export class MultiClusterListComponent {
   clustersTokenMap: Map<string, string> = new Map<string, string>();
   newData: any;
   modalRef: NgbModalRef;
+  hubUrl: string;
+  currentUrl: string;
 
   constructor(
     private multiClusterService: MultiClusterService,
     private router: Router,
-    private summaryService: SummaryService,
     public actionLabels: ActionLabelsI18n,
     private notificationService: NotificationService,
     private authStorageService: AuthStorageService,
@@ -57,6 +57,7 @@ export class MultiClusterListComponent {
         permission: 'create',
         icon: Icons.add,
         name: this.actionLabels.CONNECT,
+        disable: (selection: CdTableSelection) => this.getDisable('connect', selection),
         click: () => this.openRemoteClusterInfoModal('connect')
       },
       {
@@ -87,6 +88,8 @@ export class MultiClusterListComponent {
   ngOnInit(): void {
     this.multiClusterService.subscribe((resp: object) => {
       if (resp && resp['config']) {
+        this.hubUrl = resp['hub_url'];
+        this.currentUrl = resp['current_url'];
         const clusterDetailsArray = Object.values(resp['config']).flat();
         this.data = clusterDetailsArray;
         this.checkClusterConnectionStatus();
@@ -139,7 +142,7 @@ export class MultiClusterListComponent {
   checkClusterConnectionStatus() {
     if (this.clusterTokenStatus && this.data) {
       this.data.forEach((cluster: MultiCluster) => {
-        const clusterStatus = this.clusterTokenStatus[cluster.name];
+        const clusterStatus = this.clusterTokenStatus[cluster.name.trim()];
 
         if (clusterStatus !== undefined) {
           cluster.cluster_connection_status = clusterStatus.status;
@@ -164,18 +167,10 @@ export class MultiClusterListComponent {
       size: 'xl'
     });
     this.bsModalRef.componentInstance.submitAction.subscribe(() => {
-      this.multiClusterService.refresh();
-      this.summaryService.refresh();
       const currentRoute = this.router.url.split('?')[0];
-      if (currentRoute.includes('dashboard')) {
-        this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
-          this.router.navigate([currentRoute]);
-        });
-      } else {
-        this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
-          this.router.navigate([currentRoute]);
-        });
-      }
+      this.multiClusterService.refreshMultiCluster(currentRoute);
+      this.checkClusterConnectionStatus();
+      this.multiClusterService.isClusterAdded(true);
     });
   }
 
@@ -186,28 +181,35 @@ export class MultiClusterListComponent {
   openDeleteClusterModal() {
     const cluster = this.selection.first();
     this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, {
+      infoMessage: $localize`Please note that the data for the disconnected cluster will be visible for a duration of ~ 5 minutes. After this period, it will be automatically removed.`,
       actionDescription: $localize`Disconnect`,
       itemDescription: $localize`Cluster`,
       itemNames: [cluster['cluster_alias'] + ' - ' + cluster['user']],
       submitAction: () =>
         this.multiClusterService.deleteCluster(cluster['name'], cluster['user']).subscribe(() => {
           this.cookieService.deleteToken(`${cluster['name']}-${cluster['user']}`);
+          this.multiClusterService.showPrometheusDelayMessage(true);
           this.modalRef.close();
           this.notificationService.show(
             NotificationType.success,
             $localize`Disconnected cluster '${cluster['cluster_alias']}'`
           );
+          const currentRoute = this.router.url.split('?')[0];
+          this.multiClusterService.refreshMultiCluster(currentRoute);
         })
     });
   }
 
   getDisable(action: string, selection: CdTableSelection): string | boolean {
-    if (!selection.hasSelection) {
+    if (this.hubUrl !== this.currentUrl) {
+      return $localize`Please switch to the local-cluster to ${action} a remote cluster`;
+    }
+    if (!selection.hasSelection && action !== 'connect') {
       return $localize`Please select one or more clusters to ${action}`;
     }
     if (selection.hasSingleSelection) {
       const cluster = selection.first();
-      if (cluster['cluster_alias'] === 'local-cluster') {
+      if (cluster['cluster_alias'] === 'local-cluster' && action !== 'connect') {
         return $localize`Cannot ${action} local cluster`;
       }
     }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
index 0542b1868bae..46d6bab35a9c 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.html
@@ -42,6 +42,8 @@ <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortles
         <i class="mx-auto d-block"
            [ngClass]="[icons.large3x, icons.spinner, icons.spin]">
         </i>
+        <p class="text-center mt-3"
+           i18n>Loading data, Please wait...</p>
       </div>
     </div>
   </div>
@@ -50,6 +52,16 @@ <h4 class="mt-3">Upgrade your current cluster to a multi-cluster setup effortles
 <div class="container-fluid h-100 p-4"
      *ngIf="isMultiCluster; else emptyCluster">
   <ng-container *ngIf="!loading; else loadingTpl">
+    <cd-alert-panel type="info"
+                    spacingClass="mb-3"
+                    [showTitle]="false"
+                    size="slim"
+                    *ngIf="showDeletionMessage"
+                    (dismissed)="onDismissed()"
+                    [dismissible]="true"
+                    i18n>
+      <p>Please note that the data for the disconnected cluster will be visible for a duration of ~ 5 minutes. After this period, it will be automatically removed.</p>
+    </cd-alert-panel>
     <cd-card-group>
       <div class="col-lg-4">
         <div class="row">
@@ -112,7 +124,7 @@ <h3 [ngClass]="{'text-danger': connectionErrorsCount > 0}">
                    i18n-title
                    class="col-sm-6 m-0 p-0 ps-2 pe-2"
                    aria-label="Total number of hosts"
-                   *ngIf="queriesResults['TOTAL_HOSTS'][0][1] !== '0'">
+                   *ngIf="queriesResults['TOTAL_HOSTS'] && queriesResults['TOTAL_HOSTS'][0]">
             <span class="text-center">
               <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
             </span>
@@ -142,10 +154,11 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                  aria-label="Cluster Utilization card"
                  *ngIf="clusters">
           <div class="ms-4 me-4 mt-0">
-            <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
+            <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event, 'clusterUtilization')">
             </cd-dashboard-time-selector>
             <cd-dashboard-area-chart  chartTitle="Capacity"
                                       [labelsArray]="capacityLabels"
+                                      isMultiCluster="true"
                                       dataUnits="B"
                                       [dataArray]="capacityValues"
                                       [truncateLabel]="true"
@@ -155,6 +168,7 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                                      [labelsArray]="iopsLabels"
                                      dataUnits=""
                                      decimals="0"
+                                     isMultiCluster="true"
                                      [dataArray]="iopsValues"
                                      [truncateLabel]="true"
                                      *ngIf="iopsLabels && iopsValues">
@@ -163,6 +177,7 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                                      [labelsArray]="throughputLabels"
                                      dataUnits="B/s"
                                      decimals="2"
+                                     isMultiCluster="true"
                                      [dataArray]="throughputValues"
                                      [truncateLabel]="true"
                                      *ngIf="throughputLabels && throughputLabels">
@@ -190,11 +205,12 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                    aria-label="Pools Utilization card"
                    *ngIf="clusters">
             <div class="ms-4 me-4 mt-0">
-              <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event)">
+              <cd-dashboard-time-selector (selectedTime)="getPrometheusData($event, 'poolUtilization')">
               </cd-dashboard-time-selector>
               <cd-dashboard-area-chart  chartTitle="Capacity"
                                         [labelsArray]="poolCapacityLabels"
                                         dataUnits="B"
+                                        isMultiCluster="true"
                                         [dataArray]="poolCapacityValues"
                                         *ngIf="poolCapacityLabels && poolCapacityValues"
                                         [truncateLabel]="true">
@@ -203,6 +219,7 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                                        [labelsArray]="poolIOPSLabels"
                                        dataUnits=""
                                        decimals="0"
+                                       isMultiCluster="true"
                                        [dataArray]="poolIOPSValues"
                                        *ngIf="poolIOPSLabels && poolIOPSValues"
                                        [truncateLabel]="true">
@@ -211,6 +228,7 @@ <h3>{{ queriesResults['TOTAL_HOSTS'][0][1] }}</h3>
                                        [labelsArray]="poolThroughputLabels"
                                        dataUnits="B/s"
                                        decimals="2"
+                                       isMultiCluster="true"
                                        [dataArray]="poolThroughputValues"
                                        *ngIf="poolThroughputLabels && poolThroughputValues"
                                        [truncateLabel]="true">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
index ab8b413e7362..18dc80406169 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/multi-cluster/multi-cluster.component.ts
@@ -1,4 +1,4 @@
-import { Component, OnInit, ViewChild } from '@angular/core';
+import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core';
 import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap';
 import { Subscription } from 'rxjs';
 import { MultiClusterService } from '~/app/shared/api/multi-cluster.service';
@@ -6,17 +6,23 @@ import { Icons } from '~/app/shared/enum/icons.enum';
 import { ModalService } from '~/app/shared/services/modal.service';
 import { MultiClusterFormComponent } from './multi-cluster-form/multi-cluster-form.component';
 import { PrometheusService } from '~/app/shared/api/prometheus.service';
-import { MultiClusterPromqls as queries } from '~/app/shared/enum/dashboard-promqls.enum';
 import { CdTableColumn } from '~/app/shared/models/cd-table-column';
 import { CellTemplate } from '~/app/shared/enum/cell-template.enum';
 import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe';
+import { Router } from '@angular/router';
+
+import {
+  MultiClusterPromqls as allQueries,
+  MultiClusterPromqlsForClusterUtilization as ClusterUltilizationQueries,
+  MultiClusterPromqlsForPoolUtilization as PoolUltilizationQueries
+} from '~/app/shared/enum/dashboard-promqls.enum';
 
 @Component({
   selector: 'cd-multi-cluster',
   templateUrl: './multi-cluster.component.html',
   styleUrls: ['./multi-cluster.component.scss']
 })
-export class MultiClusterComponent implements OnInit {
+export class MultiClusterComponent implements OnInit, OnDestroy {
   COUNT_OF_UTILIZATION_CHARTS = 5;
 
   @ViewChild('nameTpl', { static: true })
@@ -56,7 +62,7 @@ export class MultiClusterComponent implements OnInit {
   isMultiCluster = true;
   clusterTokenStatus: object = {};
   localClusterName: string;
-  clusters: any;
+  clusters: any = [];
   connectionErrorsCount = 0;
 
   capacityLabels: string[] = [];
@@ -72,13 +78,38 @@ export class MultiClusterComponent implements OnInit {
   poolIOPSValues: string[] = [];
   poolCapacityValues: string[] = [];
   poolThroughputValues: string[] = [];
+  showDeletionMessage = false;
+  isClusterAdded = false;
+  selectedQueries: any;
+  PROMETHEUS_DELAY = 20000;
+  LOAD_DELAY = 5000;
+  CLUSTERS_REFRESH_INTERVAL = 30000;
+  interval: NodeJS.Timer;
+  selectedTime: any;
+  multiClusterQueries: any = {};
 
   constructor(
     private multiClusterService: MultiClusterService,
     private modalService: ModalService,
+    private router: Router,
     private prometheusService: PrometheusService,
     private dimlessBinaryPipe: DimlessBinaryPipe
-  ) {}
+  ) {
+    this.multiClusterQueries = {
+      cluster: {
+        queries: ClusterUltilizationQueries,
+        selectedTime: this.prometheusService.lastHourDateObject
+      },
+      pool: {
+        queries: PoolUltilizationQueries,
+        selectedTime: this.prometheusService.lastHourDateObject
+      },
+      all: {
+        queries: allQueries,
+        selectedTime: this.prometheusService.lastHourDateObject
+      }
+    };
+  }
 
   ngOnInit(): void {
     this.columns = [
@@ -160,7 +191,24 @@ export class MultiClusterComponent implements OnInit {
         this.clusterTokenStatus = resp;
       })
     );
-    this.getPrometheusData(this.prometheusService.lastHourDateObject);
+
+    this.isClusterAdded = this.multiClusterService.isClusterAdded();
+
+    if (this.isClusterAdded) {
+      setTimeout(() => {
+        this.getPrometheusData(this.prometheusService.lastHourDateObject);
+        this.multiClusterService.isClusterAdded(false);
+      }, this.PROMETHEUS_DELAY);
+    } else {
+      this.showDeletionMessage = this.multiClusterService.showPrometheusDelayMessage();
+      if (this.showDeletionMessage) {
+        setTimeout(() => {
+          this.getPrometheusData(this.prometheusService.lastHourDateObject);
+        }, this.LOAD_DELAY);
+      } else {
+        this.getPrometheusData(this.prometheusService.lastHourDateObject);
+      }
+    }
   }
 
   openRemoteClusterInfoModal() {
@@ -170,17 +218,69 @@ export class MultiClusterComponent implements OnInit {
     this.bsModalRef = this.modalService.show(MultiClusterFormComponent, initialState, {
       size: 'lg'
     });
+    this.bsModalRef.componentInstance.submitAction.subscribe(() => {
+      this.loading = true;
+      setTimeout(() => {
+        const currentRoute = this.router.url.split('?')[0];
+        this.multiClusterService.refreshMultiCluster(currentRoute);
+        this.getPrometheusData(this.prometheusService.lastHourDateObject);
+      }, this.PROMETHEUS_DELAY);
+    });
   }
 
-  getPrometheusData(selectedTime: any) {
+  getPrometheusData(selectedTime: any, selectedQueries?: string) {
+    const validRangeQueries = [
+      'CLUSTER_CAPACITY_UTILIZATION',
+      'CLUSTER_IOPS_UTILIZATION',
+      'CLUSTER_THROUGHPUT_UTILIZATION',
+      'POOL_CAPACITY_UTILIZATION',
+      'POOL_IOPS_UTILIZATION',
+      'POOL_THROUGHPUT_UTILIZATION'
+    ];
+    const validQueries = [
+      'ALERTS',
+      'MGR_METADATA',
+      'HEALTH_STATUS',
+      'TOTAL_CAPACITY',
+      'USED_CAPACITY',
+      'POOLS',
+      'OSDS',
+      'CLUSTER_CAPACITY_UTILIZATION',
+      'CLUSTER_IOPS_UTILIZATION',
+      'CLUSTER_THROUGHPUT_UTILIZATION',
+      'POOL_CAPACITY_UTILIZATION',
+      'POOL_IOPS_UTILIZATION',
+      'POOL_THROUGHPUT_UTILIZATION',
+      'HOSTS',
+      'CLUSTER_ALERTS'
+    ];
+
+    if (selectedQueries) {
+      if (selectedQueries === 'poolUtilization') {
+        this.multiClusterQueries.pool['selectedTime'] = selectedTime;
+      }
+
+      if (selectedQueries === 'clusterUtilization') {
+        this.multiClusterQueries.cluster.selectedTime = selectedTime;
+      }
+    }
+
     this.prometheusService
-      .getMultiClusterQueriesData(selectedTime, queries, this.queriesResults)
+      .getMultiClusterQueriesData(
+        this.queriesResults,
+        validQueries,
+        validRangeQueries,
+        this.multiClusterQueries
+      )
       .subscribe((data: any) => {
         this.queriesResults = data;
         this.loading = false;
         this.alerts = this.queriesResults.ALERTS;
         this.getAlertsInfo();
         this.getClustersInfo();
+        this.interval = setInterval(() => {
+          this.getClustersInfo();
+        }, this.CLUSTERS_REFRESH_INTERVAL);
       });
   }
 
@@ -224,7 +324,6 @@ export class MultiClusterComponent implements OnInit {
     }
 
     const clusters: ClusterInfo[] = [];
-
     this.queriesResults.TOTAL_CAPACITY?.forEach((totalCapacityMetric: any) => {
       const clusterName = totalCapacityMetric.metric.cluster;
       const totalCapacity = parseInt(totalCapacityMetric.value[1]);
@@ -240,7 +339,7 @@ export class MultiClusterComponent implements OnInit {
       const available_capacity = totalCapacity - usedCapacity;
 
       clusters.push({
-        cluster: clusterName,
+        cluster: clusterName.trim(),
         status,
         alert,
         total_capacity: totalCapacity,
@@ -318,7 +417,6 @@ export class MultiClusterComponent implements OnInit {
       }
       labels.push(label);
     }
-    // console.log(labels)
     return labels;
   }
 
@@ -329,4 +427,13 @@ export class MultiClusterComponent implements OnInit {
     }
     return values;
   }
+
+  onDismissed() {
+    this.showDeletionMessage = false;
+    this.multiClusterService.showPrometheusDelayMessage(false);
+  }
+
+  ngOnDestroy(): void {
+    clearInterval(this.interval);
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
index 2b4878e995d2..99a71181b70e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.html
@@ -24,7 +24,7 @@
   </div>
 
   <div class="col-9 d-flex flex-column">
-    <div class="chart mt-3">
+    <div [ngClass]="{'chart mt-3': !isMultiCluster, 'mt-3': isMultiCluster}">
       <canvas baseChart
               [datasets]="chartData.dataset"
               [options]="options"
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
index 80cdb1ea2369..bcfd5a892b67 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.spec.ts
@@ -63,22 +63,22 @@ describe('DashboardAreaChartComponent', () => {
 
   it('should set label', () => {
     component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
-    expect(component.chartData.dataset[0].label).toEqual('Read');
+    expect(component.chartData.dataset[0].label).toEqual('Total');
     expect(component.chartData.dataset[1].label).toEqual('Write');
-    expect(component.chartData.dataset[2].label).toEqual('Total');
+    expect(component.chartData.dataset[2].label).toEqual('Read');
   });
 
   it('should transform and update data', () => {
     component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
     expect(component.chartData.dataset[0].data).toEqual([
-      { x: 1000, y: 110 },
-      { x: 3000, y: 130 }
+      { x: 5000, y: 150 },
+      { x: 6000, y: 160 }
     ]);
   });
 
   it('should set currentData to last value', () => {
     component.ngOnChanges({ dataArray: new SimpleChange(null, component.dataArray, false) });
-    expect(component.currentChartData.dataset[0].currentData).toBe('130');
+    expect(component.currentChartData.dataset[0].currentData).toBe('160');
   });
 
   it('should keep data units consistency', () => {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
index 607a3b7d51ad..8a267b4782aa 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-area-chart/dashboard-area-chart.component.ts
@@ -31,6 +31,8 @@ export class DashboardAreaChartComponent implements OnChanges {
   decimals?: number = 1;
   @Input()
   truncateLabel = false;
+  @Input()
+  isMultiCluster?: boolean = false;
 
   currentDataUnits: string;
   currentData: number;
@@ -214,8 +216,13 @@ export class DashboardAreaChartComponent implements OnChanges {
           [this.maxConvertedValue, this.maxConvertedValueUnits] = this.convertUnits(
             this.maxValue
           ).split(' ');
+          this.currentChartData.dataset[index]['currentDataValue'] = currentDataValue;
         }
       }
+      this.currentChartData.dataset.sort(
+        (a: { currentDataValue: string }, b: { currentDataValue: string }) =>
+          parseFloat(b['currentDataValue']) - parseFloat(a['currentDataValue'])
+      );
     }
 
     if (this.chart) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
index b0c253c33e9d..fa194024db9e 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard-v3/dashboard-pie/dashboard-pie.component.ts
@@ -157,8 +157,10 @@ export class DashboardPieComponent implements OnChanges, OnInit {
   private prepareRawUsage(chart: Record<string, any>, data: Record<string, any>) {
     const nearFullRatioPercent = this.lowThreshold * 100;
     const fullRatioPercent = this.highThreshold * 100;
-    const percentAvailable = this.calcPercentage(data.max - data.current, data.max);
-    const percentUsed = this.calcPercentage(data.current, data.max);
+    const max = typeof data.max === 'string' ? parseFloat(data.max) : data.max;
+    const current = typeof data.current === 'string' ? parseFloat(data.current) : data.current;
+    const percentAvailable = this.calcPercentage(max - current, max);
+    const percentUsed = this.calcPercentage(current, max);
 
     if (fullRatioPercent >= 0 && percentUsed >= fullRatioPercent) {
       this.color = 'chart-color-red';
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
index de8b0a267db8..6f52dc6cf339 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts
@@ -71,6 +71,7 @@ export class NavigationComponent implements OnInit, OnDestroy {
       this.multiClusterService.subscribe((resp: object) => {
         const clustersConfig = resp['config'];
         if (clustersConfig) {
+          this.clustersMap.clear();
           Object.keys(clustersConfig).forEach((clusterKey: string) => {
             const clusterDetailsList = clustersConfig[clusterKey];
             clusterDetailsList.forEach((clusterDetails: MultiCluster) => {
@@ -212,9 +213,6 @@ export class NavigationComponent implements OnInit, OnDestroy {
       },
       () => {},
       () => {
-        this.multiClusterService.refresh();
-        this.summaryService.refresh();
-
         // force refresh grafana api url to get the correct url for the selected cluster
         this.settingsService.ifSettingConfigured(
           'api/grafana/url',
@@ -223,15 +221,7 @@ export class NavigationComponent implements OnInit, OnDestroy {
           true
         );
         const currentRoute = this.router.url.split('?')[0];
-        if (currentRoute.includes('dashboard')) {
-          this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
-            this.router.navigate([currentRoute]);
-          });
-        } else {
-          this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
-            this.router.navigate([currentRoute]);
-          });
-        }
+        this.multiClusterService.refreshMultiCluster(currentRoute);
       }
     );
   }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
index ffb312de4d9d..5e6ab6e3606f 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/multi-cluster.service.ts
@@ -3,6 +3,8 @@ import { Injectable } from '@angular/core';
 import { BehaviorSubject, Observable, Subscription } from 'rxjs';
 import { TimerService } from '../services/timer.service';
 import { filter } from 'rxjs/operators';
+import { SummaryService } from '../services/summary.service';
+import { Router } from '@angular/router';
 
 @Injectable({
   providedIn: 'root'
@@ -13,7 +15,14 @@ export class MultiClusterService {
   msData$ = this.msSource.asObservable();
   private tokenStatusSource = new BehaviorSubject<any>(null);
   tokenStatusSource$ = this.tokenStatusSource.asObservable();
-  constructor(private http: HttpClient, private timerService: TimerService) {}
+  showDeletionMessage = false;
+  isClusterAddedFlag = false;
+  constructor(
+    private http: HttpClient,
+    private timerService: TimerService,
+    private summaryService: SummaryService,
+    private router: Router
+  ) {}
 
   startPolling(): Subscription {
     return this.timerService
@@ -41,7 +50,9 @@ export class MultiClusterService {
 
         if (tempMap.size > 0) {
           clustersTokenMap = tempMap;
-          dataSubscription.unsubscribe();
+          if (dataSubscription) {
+            dataSubscription.unsubscribe();
+          }
           this.checkAndStartTimer(clustersTokenMap);
         }
       }
@@ -95,6 +106,7 @@ export class MultiClusterService {
     token = '',
     hub_url = '',
     clusterFsid = '',
+    prometheusApiUrl = '',
     ssl = false,
     cert = ''
   ) {
@@ -106,6 +118,7 @@ export class MultiClusterService {
       token,
       hub_url,
       cluster_fsid: clusterFsid,
+      prometheus_api_url: prometheusApiUrl,
       ssl_verify: ssl,
       ssl_certificate: cert
     });
@@ -119,7 +132,7 @@ export class MultiClusterService {
     ssl = false,
     cert = ''
   ) {
-    return this.http.post('api/multi-cluster/reconnect_cluster', {
+    return this.http.put('api/multi-cluster/reconnect_cluster', {
       url,
       username,
       password,
@@ -169,4 +182,32 @@ export class MultiClusterService {
 
     return this.http.get<object>('api/multi-cluster/check_token_status', { params });
   }
+
+  showPrometheusDelayMessage(showDeletionMessage?: boolean) {
+    if (showDeletionMessage !== undefined) {
+      this.showDeletionMessage = showDeletionMessage;
+    }
+    return this.showDeletionMessage;
+  }
+
+  isClusterAdded(isClusterAddedFlag?: boolean) {
+    if (isClusterAddedFlag !== undefined) {
+      this.isClusterAddedFlag = isClusterAddedFlag;
+    }
+    return this.isClusterAddedFlag;
+  }
+
+  refreshMultiCluster(currentRoute: string) {
+    this.refresh();
+    this.summaryService.refresh();
+    if (currentRoute.includes('dashboard')) {
+      this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
+        this.router.navigate([currentRoute]);
+      });
+    } else {
+      this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
+        this.router.navigate([currentRoute]);
+      });
+    }
+  }
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
index b7db0bc2f3cc..eaa1696dc873 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts
@@ -200,7 +200,12 @@ export class PrometheusService {
     return this.http.get<any>(`${this.baseURL}/data`, { params });
   }
 
-  getMultiClusterQueriesData(selectedTime: any, queries: any, queriesResults: any) {
+  getMultiClusterQueriesData(
+    queriesResults: any,
+    validQueries: string[],
+    validRangeQueries: string[],
+    multiClusterQueries: any
+  ) {
     return new Observable((observer) => {
       this.ifPrometheusConfigured(() => {
         if (this.timerGetPrometheusDataSub) {
@@ -208,63 +213,45 @@ export class PrometheusService {
         }
 
         this.timerGetPrometheusDataSub = timer(0, this.timerTime).subscribe(() => {
-          selectedTime = this.updateTimeStamp(selectedTime);
+          let requests: any[] = [];
+          let queryNames: string[] = [];
 
-          const requests = [];
-          for (const queryName in queries) {
-            if (queries.hasOwnProperty(queryName)) {
-              const validRangeQueries1 = [
-                'CLUSTER_CAPACITY_UTILIZATION',
-                'CLUSTER_IOPS_UTILIZATION',
-                'CLUSTER_THROUGHPUT_UTILIZATION',
-                'POOL_CAPACITY_UTILIZATION',
-                'POOL_IOPS_UTILIZATION',
-                'POOL_THROUGHPUT_UTILIZATION'
-              ];
-              if (validRangeQueries1.includes(queryName)) {
-                const query = queries[queryName];
-                const request = this.getMultiClusterQueryRangeData({
-                  params: encodeURIComponent(query),
-                  start: selectedTime['start'],
-                  end: selectedTime['end'],
-                  step: selectedTime['step']
-                });
-                requests.push(request);
-              } else {
-                const query = queries[queryName];
-                const request = this.getMultiClusterData({
-                  params: encodeURIComponent(query),
-                  start: selectedTime['start'],
-                  end: selectedTime['end'],
-                  step: selectedTime['step']
-                });
-                requests.push(request);
+          Object.entries(multiClusterQueries).forEach(([key, _value]) => {
+            for (const queryName in multiClusterQueries[key].queries) {
+              if (multiClusterQueries[key].queries.hasOwnProperty(queryName)) {
+                const query = multiClusterQueries[key].queries[queryName];
+                const start = this.updateTimeStamp(multiClusterQueries[key].selectedTime)['start'];
+                const end = this.updateTimeStamp(multiClusterQueries[key].selectedTime)['end'];
+                const step = this.updateTimeStamp(multiClusterQueries[key].selectedTime)['step'];
+
+                if (validRangeQueries.includes(queryName)) {
+                  const request = this.getMultiClusterQueryRangeData({
+                    params: encodeURIComponent(query),
+                    start,
+                    end,
+                    step
+                  });
+                  requests.push(request);
+                  queryNames.push(queryName);
+                } else {
+                  const request = this.getMultiClusterData({
+                    params: encodeURIComponent(query),
+                    start,
+                    end,
+                    step
+                  });
+                  requests.push(request);
+                  queryNames.push(queryName);
+                }
               }
             }
-          }
+          });
 
           forkJoin(requests).subscribe(
             (responses: any[]) => {
               for (let i = 0; i < responses.length; i++) {
                 const data = responses[i];
-                const queryName = Object.keys(queries)[i];
-                const validQueries = [
-                  'ALERTS',
-                  'MGR_METADATA',
-                  'HEALTH_STATUS',
-                  'TOTAL_CAPACITY',
-                  'USED_CAPACITY',
-                  'POOLS',
-                  'OSDS',
-                  'CLUSTER_CAPACITY_UTILIZATION',
-                  'CLUSTER_IOPS_UTILIZATION',
-                  'CLUSTER_THROUGHPUT_UTILIZATION',
-                  'POOL_CAPACITY_UTILIZATION',
-                  'POOL_IOPS_UTILIZATION',
-                  'POOL_THROUGHPUT_UTILIZATION',
-                  'HOSTS',
-                  'CLUSTER_ALERTS'
-                ];
+                const queryName = queryNames[i];
                 if (data.result.length) {
                   if (validQueries.includes(queryName)) {
                     queriesResults[queryName] = data.result;
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.html
index cc2eded0e3b8..41b0ed5ddb7b 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.html
@@ -10,6 +10,12 @@
           [formGroup]="deletionForm"
           novalidate>
       <div class="modal-body">
+        <cd-alert-panel *ngIf="infoMessage"
+                        type="info"
+                        spacingClass="mb-3"
+                        i18n>
+          <p>{{ infoMessage }}</p>
+        </cd-alert-panel>
         <ng-container *ngTemplateOutlet="bodyTemplate; context: bodyContext"></ng-container>
         <div class="question">
           <span *ngIf="itemNames; else noNames">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.ts
index 406f992a9df2..65c0a4f5ee2d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component.ts
@@ -25,6 +25,7 @@ export class CriticalConfirmationModalComponent implements OnInit {
   itemDescription: 'entry';
   itemNames: string[];
   actionDescription = 'delete';
+  infoMessage: string;
 
   childFormGroup: CdFormGroup;
   childFormGroupTemplate: TemplateRef<any>;
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
index 9a85d108a6b5..117cc943d51a 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/dashboard-promqls.enum.ts
@@ -36,11 +36,17 @@ export enum MultiClusterPromqls {
   ALERTS = 'ALERTS{alertstate="firing"}',
   HOSTS = 'sum by (hostname, cluster) (group by (hostname, cluster) (ceph_osd_metadata)) or vector(0)',
   TOTAL_HOSTS = 'count by (cluster) (ceph_osd_metadata) or vector(0)',
-  CLUSTER_ALERTS = 'count by (cluster) (ALERTS{alertstate="firing"}) or vector(0)',
-  CLUSTER_CAPACITY_UTILIZATION = 'topk(2, ceph_cluster_total_used_bytes)',
-  CLUSTER_IOPS_UTILIZATION = 'topk(2, sum by (cluster) (rate(ceph_pool_wr[1m])) + sum by (cluster) (rate(ceph_pool_rd[1m])) )',
-  CLUSTER_THROUGHPUT_UTILIZATION = 'topk(2, sum by (cluster) (rate(ceph_pool_wr_bytes[1m])) + sum by (cluster) (rate(ceph_pool_rd_bytes[1m])) )',
-  POOL_CAPACITY_UTILIZATION = 'topk(2, ceph_pool_bytes_used/ceph_pool_max_avail * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata)',
-  POOL_IOPS_UTILIZATION = 'topk(2, (rate(ceph_pool_rd[1m]) + rate(ceph_pool_wr[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )',
-  POOL_THROUGHPUT_UTILIZATION = 'topk(2, (irate(ceph_pool_rd_bytes[1m]) + irate(ceph_pool_wr_bytes[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )'
+  CLUSTER_ALERTS = 'count by (cluster) (ALERTS{alertstate="firing"}) or vector(0)'
+}
+
+export enum MultiClusterPromqlsForClusterUtilization {
+  CLUSTER_CAPACITY_UTILIZATION = 'topk(5, ceph_cluster_total_used_bytes)',
+  CLUSTER_IOPS_UTILIZATION = 'topk(5, sum by (cluster) (rate(ceph_pool_wr[1m])) + sum by (cluster) (rate(ceph_pool_rd[1m])) )',
+  CLUSTER_THROUGHPUT_UTILIZATION = 'topk(5, sum by (cluster) (rate(ceph_pool_wr_bytes[1m])) + sum by (cluster) (rate(ceph_pool_rd_bytes[1m])) )'
+}
+
+export enum MultiClusterPromqlsForPoolUtilization {
+  POOL_CAPACITY_UTILIZATION = 'topk(5, ceph_pool_bytes_used/ceph_pool_max_avail * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata)',
+  POOL_IOPS_UTILIZATION = 'topk(5, (rate(ceph_pool_rd[1m]) + rate(ceph_pool_wr[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )',
+  POOL_THROUGHPUT_UTILIZATION = 'topk(5, (irate(ceph_pool_rd_bytes[1m]) + irate(ceph_pool_wr_bytes[1m])) * on(pool_id, cluster) group_left(instance, name) ceph_pool_metadata )'
 }
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
index d1e98487789b..84a630346d47 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/api-interceptor.service.ts
@@ -17,8 +17,6 @@ import { NotificationType } from '../enum/notification-type.enum';
 import { CdNotificationConfig } from '../models/cd-notification';
 import { FinishedTask } from '../models/finished-task';
 import { NotificationService } from './notification.service';
-import { MultiClusterService } from '../api/multi-cluster.service';
-import { SummaryService } from './summary.service';
 import { AuthStorageService } from './auth-storage.service';
 import { CookiesService } from './cookie.service';
 
@@ -31,37 +29,12 @@ export class CdHttpErrorResponse extends HttpErrorResponse {
   providedIn: 'root'
 })
 export class ApiInterceptorService implements HttpInterceptor {
-  localClusterDetails: object;
-  dashboardClustersMap: Map<string, string> = new Map<string, string>();
   constructor(
     private router: Router,
     public notificationService: NotificationService,
-    private summaryService: SummaryService,
     private authStorageService: AuthStorageService,
-    private multiClusterService: MultiClusterService,
     private cookieService: CookiesService
-  ) {
-    this.multiClusterService.subscribe((resp: any) => {
-      const clustersConfig = resp['config'];
-      const hub_url = resp['hub_url'];
-      if (clustersConfig) {
-        Object.keys(clustersConfig).forEach((clusterKey: string) => {
-          const clusterDetailsList = clustersConfig[clusterKey];
-
-          clusterDetailsList.forEach((clusterDetails: any) => {
-            const clusterUrl = clusterDetails['url'];
-            const clusterName = clusterDetails['name'];
-
-            this.dashboardClustersMap.set(clusterUrl, clusterName);
-
-            if (clusterDetails['url'] === hub_url) {
-              this.localClusterDetails = clusterDetails;
-            }
-          });
-        });
-      }
-    });
-  }
+  ) {}
 
   intercept(request: HttpRequest<any>, next: HttpHandler): Observable<HttpEvent<any>> {
     const acceptHeader = request.headers.get('Accept');
@@ -88,8 +61,8 @@ export class ApiInterceptorService implements HttpInterceptor {
     const ALWAYS_TO_HUB_APIs = [
       'api/auth/login',
       'api/auth/logout',
-      'api/multi-cluster/get_config',
       'api/multi-cluster/set_config',
+      'api/multi-cluster/get_config',
       'api/multi-cluster/auth'
     ];
 
@@ -132,26 +105,8 @@ export class ApiInterceptorService implements HttpInterceptor {
               timeoutId = this.notificationService.notifyTask(finishedTask);
               break;
             case 401:
-              if (this.dashboardClustersMap.size > 1) {
-                this.multiClusterService.setCluster(this.localClusterDetails).subscribe(() => {
-                  localStorage.setItem('cluster_api_url', this.localClusterDetails['url']);
-                });
-                this.multiClusterService.refresh();
-                this.summaryService.refresh();
-                const currentRoute = this.router.url.split('?')[0];
-                if (currentRoute.includes('dashboard')) {
-                  this.router.navigateByUrl('/pool', { skipLocationChange: true }).then(() => {
-                    this.router.navigate([currentRoute]);
-                  });
-                } else {
-                  this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {
-                    this.router.navigate([currentRoute]);
-                  });
-                }
-              } else {
-                this.authStorageService.remove();
-                this.router.navigate(['/login']);
-              }
+              this.authStorageService.remove();
+              this.router.navigate(['/login']);
               break;
             case 403:
               this.router.navigate(['error'], {

From 1b5ca67ef8e9de48221d52b6e46f89aad1a731eb Mon Sep 17 00:00:00 2001
From: Zhang Song <zhangsong02@qianxin.com>
Date: Tue, 26 Mar 2024 14:15:35 +0800
Subject: [PATCH 2471/2492] crimson/os/seastore: fix use after free error

Signed-off-by: Zhang Song <zhangsong02@qianxin.com>
---
 src/crimson/os/seastore/seastore.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index e29fd4c0ea67..ac38a978e0db 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -253,7 +253,7 @@ SeaStore::mount_ertr::future<> SeaStore::mount()
   ceph_assert(seastar::this_shard_id() == primary_core);
   return device->mount(
   ).safe_then([this] {
-    auto sec_devices = device->get_sharded_device().get_secondary_devices();
+    auto &sec_devices = device->get_sharded_device().get_secondary_devices();
     return crimson::do_for_each(sec_devices, [this](auto& device_entry) {
       device_id_t id = device_entry.first;
       magic_t magic = device_entry.second.magic;

From d0979e9c5223231a0fd3ef6408323d2fd5cd2ac5 Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Fri, 1 Mar 2024 12:56:25 +0530
Subject: [PATCH 2472/2492] mgr/dashboard: Add advanced fieldset component

Fixes https://tracker.ceph.com/issues/65024

- adds a new shared component for displaying advanced fields in /shared
- utilizes that component in bucket form
- placement targets are under advanced fieldset
- minor help text changes in object locking help text
- updated e2e tests

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../workflow/10-nfs-exports.e2e-spec.ts       |  2 +-
 .../cypress/e2e/rgw/buckets.e2e-spec.ts       |  6 +-
 .../frontend/cypress/e2e/rgw/buckets.po.ts    | 10 +---
 .../rgw-bucket-form.component.html            | 58 ++++++++++---------
 .../shared/components/components.module.ts    |  7 ++-
 .../form-advanced-fieldset.component.html     | 22 +++++++
 .../form-advanced-fieldset.component.scss     | 22 +++++++
 .../form-advanced-fieldset.component.spec.ts  | 22 +++++++
 .../form-advanced-fieldset.component.ts       | 10 ++++
 9 files changed, 117 insertions(+), 42 deletions(-)
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.html
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.scss
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.spec.ts
 create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.ts

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/workflow/10-nfs-exports.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/workflow/10-nfs-exports.e2e-spec.ts
index 6380e5a13e6e..fdd96d7e975e 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/workflow/10-nfs-exports.e2e-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/orchestrator/workflow/10-nfs-exports.e2e-spec.ts
@@ -41,7 +41,7 @@ describe('nfsExport page', () => {
 
     it('should create a nfs-export with RGW backend', () => {
       buckets.navigateTo('create');
-      buckets.create(bucketName, 'dashboard', 'default-placement');
+      buckets.create(bucketName, 'dashboard');
 
       nfsExport.navigateTo();
       nfsExport.existTableCell(rgwPseudo, false);
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.e2e-spec.ts
index 99c0732fc6e6..8b05c309f695 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.e2e-spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.e2e-spec.ts
@@ -18,7 +18,7 @@ describe('RGW buckets page', () => {
   describe('create, edit & delete bucket tests', () => {
     it('should create bucket', () => {
       buckets.navigateTo('create');
-      buckets.create(bucket_name, BucketsPageHelper.USERS[0], 'default-placement');
+      buckets.create(bucket_name, BucketsPageHelper.USERS[0]);
       buckets.getFirstTableCell(bucket_name).should('exist');
     });
 
@@ -38,7 +38,7 @@ describe('RGW buckets page', () => {
 
     it('should create bucket with object locking enabled', () => {
       buckets.navigateTo('create');
-      buckets.create(bucket_name, BucketsPageHelper.USERS[0], 'default-placement', true);
+      buckets.create(bucket_name, BucketsPageHelper.USERS[0], true);
       buckets.getFirstTableCell(bucket_name).should('exist');
     });
 
@@ -57,7 +57,7 @@ describe('RGW buckets page', () => {
 
     it('should test invalid input in edit owner field', () => {
       buckets.navigateTo('create');
-      buckets.create(bucket_name, BucketsPageHelper.USERS[0], 'default-placement');
+      buckets.create(bucket_name, BucketsPageHelper.USERS[0]);
       buckets.testInvalidEdit(bucket_name);
       buckets.navigateTo();
       buckets.delete(bucket_name);
diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
index 53756f0a2569..8d87826da779 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
@@ -22,16 +22,12 @@ export class BucketsPageHelper extends PageHelper {
     return this.selectOption('owner', owner);
   }
 
-  private selectPlacementTarget(placementTarget: string) {
-    return this.selectOption('placement-target', placementTarget);
-  }
-
   private selectLockMode(lockMode: string) {
     return this.selectOption('lock_mode', lockMode);
   }
 
   @PageHelper.restrictTo(pages.create.url)
-  create(name: string, owner: string, placementTarget: string, isLocking = false) {
+  create(name: string, owner: string, isLocking = false) {
     // Enter in bucket name
     cy.get('#bid').type(name);
 
@@ -39,9 +35,6 @@ export class BucketsPageHelper extends PageHelper {
     this.selectOwner(owner);
     cy.get('#owner').should('have.class', 'ng-valid');
 
-    // Select bucket placement target:
-    this.selectPlacementTarget(placementTarget);
-
     if (isLocking) {
       cy.get('#lock_enabled').click({ force: true });
       // Select lock mode:
@@ -71,6 +64,7 @@ export class BucketsPageHelper extends PageHelper {
 
     // Placement target is not allowed to be edited and should be hidden
     cy.get('input[name=placement-target]').should('not.exist');
+
     this.selectOwner(new_owner);
 
     // If object locking is enabled versioning shouldn't be visible
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index 108b1ba6ce56..74b4915cd9c9 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -94,34 +94,6 @@
           </div>
         </div>
 
-        <!-- Placement target -->
-        <div class="form-group row"
-             *ngIf="!editing">
-          <label class="cd-col-form-label"
-                 for="placement-target"
-                 i18n>Placement target</label>
-          <div class="cd-col-form-input">
-            <select id="placement-target"
-                    name="placement-target"
-                    formControlName="placement-target"
-                    class="form-select">
-              <option i18n
-                      *ngIf="placementTargets === null"
-                      [ngValue]="null">Loading...</option>
-              <option i18n
-                      *ngIf="placementTargets !== null"
-                      [ngValue]="null">-- Select a placement target --</option>
-              <option *ngFor="let placementTarget of placementTargets"
-                      [value]="placementTarget.name">{{ placementTarget.description }}</option>
-            </select>
-            <cd-help-text>
-              <span i18n>
-                When creating a bucket, a placement target can be provided as part of the LocationConstraint to override the default placement targets from the user and zonegroup.
-              </span>
-            </cd-help-text>
-          </div>
-        </div>
-
         <!-- Versioning -->
         <fieldset *ngIf="editing">
           <legend class="cd-header"
@@ -547,6 +519,36 @@
             </div>
           </div>
         </fieldset>
+
+        <!--Advanced-->
+        <cd-form-advanced-fieldset *ngIf="!editing">
+          <!-- Placement target -->
+          <div class="form-group row">
+            <label class="cd-col-form-label"
+                   for="placement-target"
+                   i18n>Placement target</label>
+            <div class="cd-col-form-input">
+              <select id="placement-target"
+                      name="placement-target"
+                      formControlName="placement-target"
+                      class="form-select">
+                <option i18n
+                        *ngIf="placementTargets === null"
+                        [ngValue]="null">Loading...</option>
+                <option i18n
+                        *ngIf="placementTargets !== null"
+                        [ngValue]="null">-- Select a placement target --</option>
+                <option *ngFor="let placementTarget of placementTargets"
+                        [value]="placementTarget.name">{{ placementTarget.description }}</option>
+              </select>
+              <cd-help-text>
+                <span i18n>
+                  When creating a bucket, a placement target can be provided as part of the LocationConstraint to override the default placement targets from the user and zonegroup.
+                </span>
+              </cd-help-text>
+            </div>
+          </div>
+        </cd-form-advanced-fieldset>
       </div>
 
       <div class="card-footer">
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
index caaebb89a6a5..d70776dfd868 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts
@@ -55,6 +55,7 @@ import { CodeBlockComponent } from './code-block/code-block.component';
 import { VerticalNavigationComponent } from './vertical-navigation/vertical-navigation.component';
 import { CardGroupComponent } from './card-group/card-group.component';
 import { HelpTextComponent } from './help-text/help-text.component';
+import { FormAdvancedFieldsetComponent } from './form-advanced-fieldset/form-advanced-fieldset.component';
 
 @NgModule({
   imports: [
@@ -113,7 +114,8 @@ import { HelpTextComponent } from './help-text/help-text.component';
     CodeBlockComponent,
     VerticalNavigationComponent,
     CardGroupComponent,
-    HelpTextComponent
+    HelpTextComponent,
+    FormAdvancedFieldsetComponent
   ],
   providers: [],
   exports: [
@@ -149,7 +151,8 @@ import { HelpTextComponent } from './help-text/help-text.component';
     CodeBlockComponent,
     VerticalNavigationComponent,
     CardGroupComponent,
-    HelpTextComponent
+    HelpTextComponent,
+    FormAdvancedFieldsetComponent
   ]
 })
 export class ComponentsModule {}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.html
new file mode 100644
index 000000000000..133bc8440e67
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.html
@@ -0,0 +1,22 @@
+<fieldset>
+  <legend class="cd-header">
+    <div class="accordion accordion-flush">
+      <button class="accordion-button cd-form-advanced-fieldset-buttton p-0"
+              type="button"
+              id="advanced-fieldset"
+              aria-label="toggle advanced mode"
+              [ngClass]="{collapsed: !showAdvanced}"
+              (click)="showAdvanced = !showAdvanced"
+              i18n>Advanced
+      </button>
+    </div>
+  </legend>
+  <div class="accordion-collapse collapse"
+       [ngClass]="{show: showAdvanced}">
+    <div class="accordion-body">
+      <div class="card-body">
+        <ng-content></ng-content>
+      </div>
+    </div>
+  </div>
+</fieldset>
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.scss
new file mode 100644
index 000000000000..e3bc7aaed0ca
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.scss
@@ -0,0 +1,22 @@
+/*
+  Overrides the active and focus states of bootstrap accordion,
+  in order to align the accordion css with rest of the form.
+
+  Summary of overrides:
+  - font-size was 1 rem
+  - color was blue when higlighted
+  - border,outlines and box-shadow were set
+  - accordion down button in active form was a blue svg icon
+
+ */
+.cd-form-advanced-fieldset-buttton {
+  --bs-accordion-btn-active-icon: var(--bs-accordion-btn-icon);
+  background-color: inherit;
+  border: hidden;
+  border-color: inherit;
+  box-shadow: none;
+  color: inherit;
+  font-size: inherit;
+  outline: 0;
+  text-decoration: none;
+}
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.spec.ts
new file mode 100644
index 000000000000..de2c48c48d74
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.spec.ts
@@ -0,0 +1,22 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { FormAdvancedFieldsetComponent } from './form-advanced-fieldset.component';
+
+describe('FormAdvancedFieldsetComponent', () => {
+  let component: FormAdvancedFieldsetComponent;
+  let fixture: ComponentFixture<FormAdvancedFieldsetComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [FormAdvancedFieldsetComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(FormAdvancedFieldsetComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.ts
new file mode 100644
index 000000000000..e4b71d10e657
--- /dev/null
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/form-advanced-fieldset/form-advanced-fieldset.component.ts
@@ -0,0 +1,10 @@
+import { Component } from '@angular/core';
+
+@Component({
+  selector: 'cd-form-advanced-fieldset',
+  templateUrl: './form-advanced-fieldset.component.html',
+  styleUrls: ['./form-advanced-fieldset.component.scss']
+})
+export class FormAdvancedFieldsetComponent {
+  showAdvanced: boolean = false;
+}

From 96b21d7cf5bb698102dc0cfce6fef05204b82d2e Mon Sep 17 00:00:00 2001
From: Afreen <afreen23.git@gmail.com>
Date: Mon, 25 Mar 2024 14:09:08 +0530
Subject: [PATCH 2473/2492] mgr/dashboard:Update encryption and tags in bucket
 form

Fixes https://tracker.ceph.com/issues/65110

- renamed Security to Encryption
- aligned security fieldset with rest of the form using `cd-help-text`
  and aligning it with object locking in UX
- aligned security fieldset top use `cd-help-text`
- changed help text of Tags

Signed-off-by: Afreen <afreen23.git@gmail.com>
---
 .../frontend/cypress/e2e/rgw/buckets.po.ts    |  1 -
 .../rgw-bucket-form.component.html            | 44 +++++++++----------
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
index 53756f0a2569..51fa3de4987f 100644
--- a/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
+++ b/src/pybind/mgr/dashboard/frontend/cypress/e2e/rgw/buckets.po.ts
@@ -58,7 +58,6 @@ export class BucketsPageHelper extends PageHelper {
 
   @PageHelper.restrictTo(pages.create.url)
   checkForDefaultEncryption() {
-    cy.get("cd-helper[aria-label='toggle encryption helper']").click();
     cy.get("a[aria-label='click here']").click();
     cy.get('cd-modal').within(() => {
       cy.get('input[id=s3Enabled]').should('be.checked');
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
index 108b1ba6ce56..ce3776e726af 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/rgw/rgw-bucket-form/rgw-bucket-form.component.html
@@ -308,30 +308,30 @@
           </div>
         </fieldset>
 
-        <!-- Security -->
+        <!-- Encryption -->
         <fieldset>
           <legend class="cd-header"
-                  i18n>Security</legend>
+                  i18n>Encryption</legend>
           <div class="form-group row">
-            <div class="cd-col-form-offset">
-              <div class="custom-control custom-checkbox">
-                <input class="form-check-input"
-                       id="encryption_enabled"
-                       name="encryption_enabled"
-                       formControlName="encryption_enabled"
-                       type="checkbox"
-                       [attr.disabled]="!kmsVaultConfig && !s3VaultConfig ? true : null">
-                <label class="form-check-label"
-                       for="encryption_enabled"
-                       i18n>Encryption</label>
-                <cd-helper aria-label="toggle encryption helper">
-                  <span i18n>Enables encryption for the objects in the bucket.
-                     To enable encryption on a bucket you need to set the configuration values for SSE-S3 or SSE-KMS.
-                     To set the configuration values <a href="#/rgw/bucket/create"
-                                                        (click)="openConfigModal()"
-                                                        aria-label="click here">Click here</a></span>
-                </cd-helper>
-              </div>
+            <label class="cd-col-form-label pt-0"
+                   for="encryption_enabled"
+                   i18n>
+                    Enable
+            </label>
+            <div class="cd-col-form-input">
+              <input class="form-check-input"
+                     id="encryption_enabled"
+                     name="encryption_enabled"
+                     formControlName="encryption_enabled"
+                     type="checkbox"
+                     [attr.disabled]="!kmsVaultConfig && !s3VaultConfig ? true : null"/>
+              <cd-help-text aria-label="encryption helper">
+                <span i18n>Enables encryption for the objects in the bucket.
+                    To enable encryption on a bucket you need to set the configuration values for SSE-S3 or SSE-KMS.
+                    To set the configuration values <a href="#/rgw/bucket/create"
+                                                       (click)="openConfigModal()"
+                                                       aria-label="click here">Click here</a></span>
+              </cd-help-text>
             </div>
           </div>
 
@@ -419,7 +419,7 @@
         <fieldset>
           <legend class="cd-header"
                   i18n>Tags
-            <cd-helper>Tagging gives you a way to categorize storage</cd-helper>
+            <cd-help-text>Tagging provides a way to categorize storage</cd-help-text>
           </legend>
           <span *ngFor="let tag of tags; let i=index;">
             <ng-container *ngTemplateOutlet="tagTpl; context:{index: i, tag: tag}"></ng-container>

From 242318452c8111717e49b896c7f9cf0ec5f5b262 Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Tue, 26 Mar 2024 13:23:39 +0530
Subject: [PATCH 2474/2492] mgr/dashboard: fix cephfs name validation

allow volume name to start with dot (.)

Fixes: https://tracker.ceph.com/issues/65143
Signed-off-by: Nizamudeen A <nia@redhat.com>
---
 .../ceph/cephfs/cephfs-form/cephfs-form.component.html |  2 +-
 .../cephfs/cephfs-form/cephfs-form.component.spec.ts   | 10 +++++++++-
 .../ceph/cephfs/cephfs-form/cephfs-form.component.ts   |  5 ++++-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
index f8d0fa803204..958a4bff2a2d 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.html
@@ -47,7 +47,7 @@
                   i18n>This field is required!</span>
             <span *ngIf="form.showError('name', formDir, 'pattern')"
                   class="invalid-feedback"
-                  i18n>File System name should start with a letter and can only contain letters, numbers, '.', '-' or '_'</span>
+                  i18n>File System name should start with a letter or dot (.) and can only contain letters, numbers, '.', '-' or '_'</span>
           </div>
         </div>
 
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
index 520f726d5553..3bcedd1cd669 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.spec.ts
@@ -42,7 +42,15 @@ describe('CephfsVolumeFormComponent', () => {
   });
 
   it('should validate proper names', fakeAsync(() => {
-    const validNames = ['test', 'test1234', 'test_1234', 'test-1234', 'test.1234', 'test12test'];
+    const validNames = [
+      'test',
+      'test1234',
+      'test_1234',
+      'test-1234',
+      'test.1234',
+      'test12test',
+      '.test'
+    ];
     const invalidNames = ['1234', 'test@', 'test)'];
 
     for (const validName of validNames) {
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
index b0f90979c252..dbbe522fa0a1 100644
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cephfs/cephfs-form/cephfs-form.component.ts
@@ -87,7 +87,10 @@ export class CephfsVolumeFormComponent extends CdForm implements OnInit {
     });
     this.form = this.formBuilder.group({
       name: new FormControl('', {
-        validators: [Validators.pattern(/^[a-zA-Z][.A-Za-z0-9_-]+$/), Validators.required]
+        validators: [
+          Validators.pattern(/^(?:[.][A-Za-z0-9_-]+|[A-Za-z][.A-Za-z0-9_-]*)$/),
+          Validators.required
+        ]
       }),
       placement: ['hosts'],
       hosts: [[]],

From ce444d473ce0fd178e7df2984f147f46e0dbf921 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 17 Mar 2024 18:53:03 +0800
Subject: [PATCH 2475/2492] cmake: quote a list using quotes

otherwised we'd have

```
CMake Error at cmake/modules/FindSanitizers.cmake:17 (if):
  if given arguments:

    "address" "IN_LIST" "address" "thread" "undefined_behavior" "OR" "leak" "IN_LIST" "address" "thread" "undefined_behavior"

  Unknown arguments specified
```

when enabling TSan with WITH_TSAN=ON.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/FindSanitizers.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/FindSanitizers.cmake b/cmake/modules/FindSanitizers.cmake
index bfb99821a9bd..1401ca2442bf 100644
--- a/cmake/modules/FindSanitizers.cmake
+++ b/cmake/modules/FindSanitizers.cmake
@@ -14,8 +14,8 @@ foreach(component ${Sanitizers_FIND_COMPONENTS})
   elseif(component STREQUAL "leak")
     set(Sanitizers_leak_COMPILE_OPTIONS "-fsanitize=leak")
   elseif(component STREQUAL "thread")
-    if ("address" IN_LIST ${Sanitizers_FIND_COMPONENTS} OR
-        "leak" IN_LIST ${Sanitizers_FIND_COMPONENTS})
+    if ("address" IN_LIST "${Sanitizers_FIND_COMPONENTS}" OR
+        "leak" IN_LIST "${Sanitizers_FIND_COMPONENTS}")
       message(SEND_ERROR "Cannot combine -fsanitize-leak w/ -fsanitize-thread")
     elseif(NOT CMAKE_POSITION_INDEPENDENT_CODE)
       message(SEND_ERROR "TSan requires all code to be position independent")

From a13a61d63dbaa0bf294a34da3d016291dfdc5668 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 17 Mar 2024 19:35:04 +0800
Subject: [PATCH 2476/2492] cmake: error out on UBSan error

so we can be alerted if UBSan identify something wrong.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/AddCephTest.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index ccd3f8dee0b5..45bb26aa6155 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -19,6 +19,12 @@ function(add_ceph_test test_name test_path)
     PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
     PYTHONPATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cython_modules/lib.3:${CMAKE_SOURCE_DIR}/src/pybind
     CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV})
+  if(WITH_UBSAN)
+    set_property(TEST ${test_name}
+      APPEND
+      PROPERTY ENVIRONMENT
+      UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1)
+  endif()
   set_property(TEST ${test_name}
     PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
   # Crimson seastar unittest always run with --smp N to start N threads. By default, crimson seastar unittest

From 0dc61a4be335073e9a3047cf52ec3d12e4244f4d Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 17 Mar 2024 21:36:23 +0800
Subject: [PATCH 2477/2492] cmake: do not override CMAKE_EXE_LINKER_FLAGS

instead of overriding CMAKE_EXE_LINKER_FLAGS, let's append to it.
so that the existing `CMAKE_EXE_LINKER_FLAGS` is not overriden.

this should enable us to build with Clang and with sanitizer(s) enabled.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 90c9c48e06e1..149bdc45562f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -143,7 +143,7 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
   if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12) # require >= clang-12
     message(FATAL_ERROR "C++20 support requires a minimum Clang version of 12.")
   endif()
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_EXPORTS_C_FLAG}")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " ${CMAKE_EXE_EXPORTS_C_FLAG}")
   string(APPEND CMAKE_LINKER_FLAGS " -rdynamic -export-dynamic ${CMAKE_EXE_EXPORTS_C_FLAG}")
   string(PREPEND CMAKE_CXX_FLAGS_DEBUG "-g ")
   add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-Wno-inconsistent-missing-override>)

From d22734f6cb0e11dff0d94d212f47867fe06ab0fd Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 17 Mar 2024 23:40:27 +0800
Subject: [PATCH 2478/2492] cmake: add ${CMAKE_SHARED_LINKER_FLAGS} to LDFLAGS
 when building python extensions

if sanitizers are enabled, we have to populate the required link flags
to python extensions's building workflow. otherwise ld would fail to
link like:
```
/usr/bin/ld: /home/jenkins-build/build/workspace/ceph-pull-requests/build/lib/libceph-common.so.2: undefined reference to `__asan_stack_free_10'
```

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/Distutils.cmake | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/Distutils.cmake b/cmake/modules/Distutils.cmake
index daaae4ba63fd..f3d6c41e7317 100644
--- a/cmake/modules/Distutils.cmake
+++ b/cmake/modules/Distutils.cmake
@@ -73,6 +73,8 @@ function(distutils_add_cython_module target name src)
   set(PY_CC ${compiler_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1})
   set(PY_CXX ${compiler_launcher} ${CMAKE_CXX_COMPILER} ${cxx_compiler_arg1})
   set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared")
+  string(REPLACE " " ";" PY_LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
+  list(APPEND PY_LDFLAGS -L${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
   execute_process(COMMAND "${Python3_EXECUTABLE}" -c
     "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"
@@ -98,7 +100,7 @@ function(distutils_add_cython_module target name src)
     CXX="${PY_CXX}"
     LDSHARED="${PY_LDSHARED}"
     OPT=\"-DNDEBUG -g -fwrapv -O2 -w\"
-    LDFLAGS=-L${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
+    LDFLAGS="${PY_LDFLAGS}"
     CYTHON_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}
     CEPH_LIBDIR=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
     ${Python3_EXECUTABLE} ${setup_py}
@@ -130,7 +132,7 @@ function(distutils_install_cython_module name)
                         -D'void0=dead_function\(void\)' \
                         -D'__Pyx_check_single_interpreter\(ARG\)=ARG\#\#0' \
                         ${CFLAG_DISABLE_VTA}\")
-    set(ENV{LDFLAGS} \"-L${CMAKE_LIBRARY_OUTPUT_DIRECTORY}\")
+    set(ENV{LDFLAGS} \"${PY_LDFLAGS}\")
     set(ENV{CYTHON_BUILD_DIR} \"${CMAKE_CURRENT_BINARY_DIR}\")
     set(ENV{CEPH_LIBDIR} \"${CMAKE_LIBRARY_OUTPUT_DIRECTORY}\")
 

From f68248944815acbf5fb08ef66f36da6d5baef824 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 24 Mar 2024 20:23:00 +0800
Subject: [PATCH 2479/2492] pybind: use LDFLAGS in env variable when check for
 building env

when building python bindings extensions using Cython, we first perform
sanity checks by building an executable and linking it against the
shared library of the C language binding. if the executable builds,
we consider the sanity test passes.

before this change, we don't add `LDFLAGS` specified in environmental
variable to the ldflags when building the executable. but we need to
link against libasan, so that the symbols used by, for instance,
librados.so, can be found by the executable.

in this change, we always check `LDFLAGS`, and add it to the ldflags
when building the executable for testing, if it's set.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 src/pybind/cephfs/setup.py | 5 +++++
 src/pybind/rados/setup.py  | 5 +++++
 src/pybind/rbd/setup.py    | 5 +++++
 src/pybind/rgw/setup.py    | 5 +++++
 4 files changed, 20 insertions(+)

diff --git a/src/pybind/cephfs/setup.py b/src/pybind/cephfs/setup.py
index f6c2025f75d3..956a4e7008c6 100755
--- a/src/pybind/cephfs/setup.py
+++ b/src/pybind/cephfs/setup.py
@@ -117,10 +117,15 @@ def check_sanity():
             extra_preargs=['-iquote{path}'.format(path=os.path.join(CEPH_SRC_DIR, 'include'))]
         )
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            extra_postargs = ldflags.split()
+        else:
+            extra_postargs = None
         compiler.link_executable(
             objects=link_objects,
             output_progname=os.path.join(tmp_dir, 'cephfs_dummy'),
             libraries=['cephfs'],
+            extra_postargs=extra_postargs,
             output_dir=tmp_dir,
         )
 
diff --git a/src/pybind/rados/setup.py b/src/pybind/rados/setup.py
index 62b54d26b6c8..000189b7da13 100755
--- a/src/pybind/rados/setup.py
+++ b/src/pybind/rados/setup.py
@@ -112,10 +112,15 @@ def check_sanity():
             sources=[tmp_file],
             output_dir=tmp_dir
         )
+        if ldflags := os.environ.get('LDFLAGS'):
+            extra_postargs = ldflags.split()
+        else:
+            extra_postargs = None
         compiler.link_executable(
             objects=link_objects,
             output_progname=os.path.join(tmp_dir, 'rados_dummy'),
             libraries=['rados'],
+            extra_postargs=extra_postargs,
             output_dir=tmp_dir,
         )
 
diff --git a/src/pybind/rbd/setup.py b/src/pybind/rbd/setup.py
index eeb33c73d49b..b03d1c3fc957 100755
--- a/src/pybind/rbd/setup.py
+++ b/src/pybind/rbd/setup.py
@@ -116,10 +116,15 @@ def check_sanity():
             output_dir=tmp_dir
         )
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            extra_postargs = ldflags.split()
+        else:
+            extra_postargs = None
         compiler.link_executable(
             objects=link_objects,
             output_progname=os.path.join(tmp_dir, 'rbd_dummy'),
             libraries=['rbd', 'rados'],
+            extra_postargs=extra_postargs,
             output_dir=tmp_dir,
         )
 
diff --git a/src/pybind/rgw/setup.py b/src/pybind/rgw/setup.py
index ed45399d3946..eb8fc554db6e 100755
--- a/src/pybind/rgw/setup.py
+++ b/src/pybind/rgw/setup.py
@@ -116,10 +116,15 @@ def check_sanity():
             output_dir=tmp_dir,
         )
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            extra_postargs = ldflags.split()
+        else:
+            extra_postargs = None
         compiler.link_executable(
             objects=link_objects,
             output_progname=os.path.join(tmp_dir, 'rgw_dummy'),
             libraries=['rgw', 'rados'],
+            extra_postargs=extra_postargs,
             output_dir=tmp_dir,
         )
 

From a837f494b5a82555313e7424adfaee96910a4673 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 24 Mar 2024 21:40:28 +0800
Subject: [PATCH 2480/2492] cmake: prevent ASAN_OPTIONS from
 detect_odr_violation

turns out we have multiple copies of following symbol defined by rbd executable:
```
AddressSanitizer: odr-violation: global 'ceph::buffer::list::always_empty_bptr' at /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/buffer.cc:1267:34
```
before addressing it. let's disable this warning.

Refs https://tracker.ceph.com/issues/65098

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/AddCephTest.cmake | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index 45bb26aa6155..4593070fe17d 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -25,6 +25,15 @@ function(add_ceph_test test_name test_path)
       PROPERTY ENVIRONMENT
       UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1)
   endif()
+  if(WITH_ASAN)
+    # AddressSanitizer: odr-violation: global 'ceph::buffer::list::always_empty_bptr' at
+    # /home/jenkins-build/build/workspace/ceph-pull-requests/src/common/buffer.cc:1267:34
+    # see https://tracker.ceph.com/issues/65098
+    set_property(TEST ${test_name}
+      APPEND
+      PROPERTY ENVIRONMENT
+      ASAN_OPTIONS=detect_odr_violation=0)
+  endif()
   set_property(TEST ${test_name}
     PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
   # Crimson seastar unittest always run with --smp N to start N threads. By default, crimson seastar unittest

From 02155b4ffc34c530cd42b8bc4c10540426df75e2 Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Sun, 24 Mar 2024 23:20:38 +0800
Subject: [PATCH 2481/2492] cmake: suppress LeakSanitizer reports of known
 leaks

there are known leaks, which are tracked by qa/lsan.suppr, in Ceph.
so let's reuse it so we don't see them when running unit test with
ASan enabled.

see also https://clang.llvm.org/docs/AddressSanitizer.html#issue-suppression

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 cmake/modules/AddCephTest.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index 4593070fe17d..ab4dc63ca32a 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -32,7 +32,8 @@ function(add_ceph_test test_name test_path)
     set_property(TEST ${test_name}
       APPEND
       PROPERTY ENVIRONMENT
-      ASAN_OPTIONS=detect_odr_violation=0)
+      ASAN_OPTIONS=detect_odr_violation=0
+      LSAN_OPTIONS=suppressions=${CMAKE_SOURCE_DIR}/qa/lsan.supp)
   endif()
   set_property(TEST ${test_name}
     PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})

From 1a80165e4acae2a9c2c47c9f490018462a266f6e Mon Sep 17 00:00:00 2001
From: Kefu Chai <tchaikov@gmail.com>
Date: Mon, 25 Mar 2024 00:15:15 +0800
Subject: [PATCH 2482/2492] qa/lsan.supp: suppress MallocExtension::Initialize

LeakSanitizer reports
```
==688591==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 45 byte(s) in 1 object(s) allocated from:
    #0 0x55f8dd9969dd in operator new(unsigned long) (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest_fastbmap_allocator+0x1f89dd) (BuildId: cac39eac8ef1e8774f9dd48e6e3f677fdd864776)
    #1 0x55f8dd99c730 in __gnu_cxx::new_allocator<char>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:127:27
    #2 0x55f8dd99c690 in std::allocator<char>::allocate(unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:185:32
    #3 0x55f8dd99c690 in std::allocator_traits<std::allocator<char> >::allocate(std::allocator<char>&, unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:464:20
    #4 0x55f8dd99c393 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_create(unsigned long&, unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:153:14
    #5 0x55f8dda96a6c in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:307:21
    #6 0x55f8dda96852 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:395:8
    #7 0x7f4a751ab6f0 in
    MallocExtension::Initialize() (/lib/x86_64-linux-gnu/libtcmalloc.so.4+0x2a6f0) (BuildId:
    eeef3d1257388a806e122398dbce3157ee568ef4)
```

this is a global object allocated by the allocator, so we can suppress this report.

Signed-off-by: Kefu Chai <tchaikov@gmail.com>
---
 qa/lsan.supp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qa/lsan.supp b/qa/lsan.supp
index f63c4cf0e1e4..c7d6cf59ed11 100644
--- a/qa/lsan.supp
+++ b/qa/lsan.supp
@@ -3,6 +3,9 @@
 # LSAN_OPTIONS="suppressions=../qa/lsan.supp"
 # export ASAN_OPTIONS="detect_odr_violation=0"
 
+# gperftools allocates a singleton of MallocExtension and never frees it
+leak:^MallocExtension::Initialize
+
 # from perfglue/heap_profiler.cc
 # gperftools allocates a singleton and never frees it
 leak:^InitModule

From b605167a3ad2bb6c87b1f87e84c53c02ad7e2366 Mon Sep 17 00:00:00 2001
From: Leonid Chernin <lechernin@gmail.com>
Date: Tue, 17 Oct 2023 13:25:07 +0000
Subject: [PATCH 2483/2492] mon: add NVMe-oF gateway monitor and HA
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- gateway submodule

Fixes: https://tracker.ceph.com/issues/64777

This PR adds high availability support for the nvmeof Ceph service. High availability means that even in the case that a certain GW is down, there will be another available path for the initiator to be able to continue the IO through another GW. High availability is achieved by running nvmeof service consisting of at least 2 nvmeof GWs in the Ceph cluster. Every GW will be seen by the host (initiator) as a separate path to the nvme namespaces (volumes).

The implementation consists of the following main modules:

- NVMeofGWMon - a PaxosService. It is a monitor that tracks the status of the nvmeof running services, and take actions in case that services fail, and in case services restored.
- NVMeofGwMonitorClient – It is an agent that is running as a part of each nvmeof GW. It is sending beacons to the monitor to signal that the GW is alive. As a part of the beacon, the client also sends information about the service. This information is used by the monitor to take decisions and perform some operations.
- MNVMeofGwBeacon – It is a structure used by the client and the monitor to send/recv the beacons.
- MNVMeofGwMap – The map is tracking the nvmeof GWs status. It also defines what should be the new role of every GW. So in the events of GWs go down or GWs restored, the map will reflect the new role of each GW resulted by these events. The map is distributed to the NVMeofGwMonitorClient on each GW, and it knows to update the GW with the required changes.

It is also adding 3 new mon commands:
- nvme-gw create
- nvme-gw delete
- nvme-gw show

The commands are used by the ceph adm to update the monitor that a new GW is deployed. The monitor will update the map accordingly and will start tracking this GW until it is deleted.

additional commits:
(cherry picked from commit 4f543c150503fc9ee3c4d840b6eb5ed43e15905b)
(cherry picked from commit 7af4d96530531b1458e5d02bd9f706bb8c30c2ea)
(cherry picked from commit 249951b5545d857861562ef7a90f876cb8f3d778)
(cherry picked from commit f704d0c84257701fcbf8891dd5f285c890343219)

Signed-off-by: Leonid Chernin <lechernin@gmail.com>
Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 .gitmodules                                   |   5 +
 PendingReleaseNotes                           |   8 +
 ceph.spec.in                                  |  15 +
 doc/nvmeof/beacon.puml                        |  63 ++
 doc/nvmeof/gateway-state.puml                 |  41 ++
 install-deps.sh                               |   5 +
 src/CMakeLists.txt                            | 112 ++++
 src/ceph_nvmeof_monitor_client.cc             |  79 +++
 src/common/options/mon.yaml.in                |  15 +
 src/messages/MNVMeofGwBeacon.h                | 132 ++++
 src/messages/MNVMeofGwMap.h                   |  62 ++
 src/mon/CMakeLists.txt                        |   2 +
 src/mon/MonCommands.h                         |  19 +-
 src/mon/Monitor.cc                            |  21 +-
 src/mon/Monitor.h                             |   5 +
 src/mon/NVMeofGwMap.cc                        | 598 +++++++++++++++++
 src/mon/NVMeofGwMap.h                         |  92 +++
 src/mon/NVMeofGwMon.cc                        | 546 ++++++++++++++++
 src/mon/NVMeofGwMon.h                         |  94 +++
 src/mon/NVMeofGwSerialize.h                   | 602 ++++++++++++++++++
 src/mon/NVMeofGwTypes.h                       | 167 +++++
 src/mon/mon_types.h                           |   1 +
 src/msg/Message.cc                            |  10 +
 src/msg/Message.h                             |  10 +-
 src/nvmeof/NVMeofGwClient.cc                  |  32 +
 src/nvmeof/NVMeofGwClient.h                   |  40 ++
 src/nvmeof/NVMeofGwMonitorClient.cc           | 384 +++++++++++
 src/nvmeof/NVMeofGwMonitorClient.h            |  83 +++
 src/nvmeof/NVMeofGwMonitorGroupClient.cc      |  25 +
 src/nvmeof/NVMeofGwMonitorGroupClient.h       |  39 ++
 src/nvmeof/gateway                            |   1 +
 src/pybind/mgr/cephadm/services/nvmeof.py     |  33 +-
 .../services/nvmeof/ceph-nvmeof.conf.j2       |   2 +-
 .../ceph/deployment/service_spec.py           |   2 +-
 src/test/CMakeLists.txt                       |   8 +
 src/test/test_nvmeof_mon_encoding.cc          | 197 ++++++
 36 files changed, 3543 insertions(+), 7 deletions(-)
 create mode 100644 doc/nvmeof/beacon.puml
 create mode 100644 doc/nvmeof/gateway-state.puml
 create mode 100644 src/ceph_nvmeof_monitor_client.cc
 create mode 100644 src/messages/MNVMeofGwBeacon.h
 create mode 100644 src/messages/MNVMeofGwMap.h
 create mode 100755 src/mon/NVMeofGwMap.cc
 create mode 100755 src/mon/NVMeofGwMap.h
 create mode 100644 src/mon/NVMeofGwMon.cc
 create mode 100644 src/mon/NVMeofGwMon.h
 create mode 100755 src/mon/NVMeofGwSerialize.h
 create mode 100755 src/mon/NVMeofGwTypes.h
 create mode 100644 src/nvmeof/NVMeofGwClient.cc
 create mode 100644 src/nvmeof/NVMeofGwClient.h
 create mode 100644 src/nvmeof/NVMeofGwMonitorClient.cc
 create mode 100644 src/nvmeof/NVMeofGwMonitorClient.h
 create mode 100644 src/nvmeof/NVMeofGwMonitorGroupClient.cc
 create mode 100644 src/nvmeof/NVMeofGwMonitorGroupClient.h
 create mode 160000 src/nvmeof/gateway
 create mode 100644 src/test/test_nvmeof_mon_encoding.cc

diff --git a/.gitmodules b/.gitmodules
index 30307592b565..a17fb06572c8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -81,3 +81,8 @@
 [submodule "src/qatzip"]
 	path = src/qatzip
 	url = https://github.com/intel/qatzip.git
+[submodule "src/nvmeof/gateway"]
+	path = src/nvmeof/gateway
+	url = https://github.com/ceph/ceph-nvmeof.git
+	fetchRecurseSubmodules = false
+	shallow = true
diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index d5ec98fb6b50..eaf3e20f7b18 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -409,3 +409,11 @@ Relevant tracker: https://tracker.ceph.com/issues/57090
 set using the `fs set` command. This flag prevents using a standby for another
 file system (join_fs = X) when standby for the current filesystem is not available.
 Relevant tracker: https://tracker.ceph.com/issues/61599
+* mon: add NVMe-oF gateway monitor and HA
+  This PR adds high availability support for the nvmeof Ceph service. High availability
+means that even in the case that a certain GW is down, there will be another available
+path for the initiator to be able to continue the IO through another GW.
+It is also adding 2 new mon commands, to notify monitor about the gateway creation/deletion:
+  - nvme-gw create
+  - nvme-gw delete
+Relevant tracker: https://tracker.ceph.com/issues/64777
diff --git a/ceph.spec.in b/ceph.spec.in
index f4da47eda989..d0037c7d4edc 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -250,6 +250,7 @@ BuildRequires:	gperf
 BuildRequires:  cmake > 3.5
 BuildRequires:	fuse-devel
 BuildRequires:	git
+BuildRequires:	grpc-devel
 %if 0%{?fedora} || 0%{?suse_version} > 1500 || 0%{?rhel} == 9 || 0%{?openEuler}
 BuildRequires:	gcc-c++ >= 11
 %endif
@@ -642,6 +643,17 @@ system. One or more instances of ceph-mon form a Paxos part-time
 parliament cluster that provides extremely reliable and durable storage
 of cluster membership, configuration, and state.
 
+%package mon-client-nvmeof
+Summary:	Ceph NVMeoF Gateway Monitor Client
+%if 0%{?suse_version}
+Group:		System/Filesystems
+%endif
+Provides:	ceph-test:/usr/bin/ceph-nvmeof-monitor-client
+Requires:	librados2 = %{_epoch_prefix}%{version}-%{release}
+%description mon-client-nvmeof
+Ceph NVMeoF Gateway Monitor Client distributes Paxos ANA info
+to NVMeoF Gateway and provides beacons to the monitor daemon
+
 %package mgr
 Summary:        Ceph Manager Daemon
 %if 0%{?suse_version}
@@ -2071,6 +2083,9 @@ if [ $1 -ge 1 ] ; then
   fi
 fi
 
+%files mon-client-nvmeof
+%{_bindir}/ceph-nvmeof-monitor-client
+
 %files fuse
 %{_bindir}/ceph-fuse
 %{_mandir}/man8/ceph-fuse.8*
diff --git a/doc/nvmeof/beacon.puml b/doc/nvmeof/beacon.puml
new file mode 100644
index 000000000000..c4279b77e75d
--- /dev/null
+++ b/doc/nvmeof/beacon.puml
@@ -0,0 +1,63 @@
+@startuml
+
+package "MNVMeofGwBeacon" #DDDDDD {
+
+    class Beacon {
+        gw_id: String
+        gw_pool: String
+        gw_group: String
+        availability: Availability
+        subsystems: Subsystem[*]
+    }
+
+    note top
+        NVMeoF gateways regularly transmit
+        a Beacon message to the monitor,
+        conveying their current state.
+    end note
+
+    class Subsystem {
+        nqn: String
+        listeners: Listener[*]
+        namespaces: Namespace[*]
+    }
+
+    class Listener {
+        address: String
+        family: String
+        service: Decimal
+        
+    }
+
+    class Namespace {
+        nonce: String
+        ana_grp_id: Decimal
+    }
+
+    note right of Namespace::nonce
+        <b>RADOS</b> connection
+    end note
+
+    enum Availability {
+        CREATED
+        AVAILABLE
+        INACCESSIBLE
+    }
+
+    note right of Availability::CREATED
+        The initial state, 
+        until a <b>MNVMeofGwMap</b>
+        that contains the entry for 
+        this gateway
+    end note
+
+    Beacon::availability --> "1" Availability
+    Beacon::subsystems "1" *-- "n" Subsystem : contains
+
+    Subsystem::listeners "1" *-- "n" Listener : contains
+    Subsystem::namespaces "1" *-- "n" Namespace : contains
+}
+@enduml
+
+
+
diff --git a/doc/nvmeof/gateway-state.puml b/doc/nvmeof/gateway-state.puml
new file mode 100644
index 000000000000..dbfb6143ba70
--- /dev/null
+++ b/doc/nvmeof/gateway-state.puml
@@ -0,0 +1,41 @@
+@startuml
+
+package "MNVMeofGwMap" #DDDDDD {
+
+    class GatewayState {
+        group_id: Decimal
+        ana_info: SubsystemAnaInfo[*]
+    }
+
+    note top
+        The data structure distributed by the Monitors to NVMEoF gateways includes
+        a mapping of the <b>GatewayState</b> objects.
+
+        NVMeoF gateways are segmented into groups identified
+        by <b>(pool, group)</b> pair, with each gateway uniquely
+        identified by a gateway <b>ID</b>.
+    end note
+
+    note right of GatewayState::group_id
+        Identification of this gateway within its group
+    end note
+
+    class SubsystemAnaInfo {
+        nqn: String
+        ana_state: AnaState[0..n]
+    }
+
+    note right of SubsystemAnaInfo
+        Denotes the condition of the ANA groups
+        of the subsystem identified by <b>nqn</b>.
+    end note
+
+    enum AnaState {
+        OPTIMIZED
+        INACCESSIBLE
+    }
+
+    GatewayState "1" *-- "many" SubsystemAnaInfo : contains
+    SubsystemAnaInfo::ana_state "1" *-- "n" AnaState : contains
+}
+@enduml
diff --git a/install-deps.sh b/install-deps.sh
index b303493a2e51..df3008e87c28 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -517,6 +517,11 @@ else
                 $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION
                 $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org*
                 if test $ID = centos -a $MAJOR_VERSION = 8 ; then
+                    # for grpc-devel
+                    # See https://copr.fedorainfracloud.org/coprs/ceph/grpc/
+                    # epel is enabled for all major versions couple of lines above
+                    $SUDO dnf copr enable -y ceph/grpc
+
                     # Enable 'powertools' or 'PowerTools' repo
                     $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}')
                     dts_ver=11
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 149bdc45562f..ede78f56cf45 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -303,6 +303,12 @@ endif(WITH_BLKIN)
 
 if(WITH_JAEGER)
   find_package(thrift 0.13.0 REQUIRED)
+
+  if(EXISTS "/etc/redhat-release" OR EXISTS "/etc/fedora-release")
+    # absl is installed as grpc build dependency on RPM based systems
+    add_definitions(-DHAVE_ABSEIL)
+  endif()
+
   include(BuildOpentelemetry)
   build_opentelemetry()
   add_library(jaeger_base INTERFACE)
@@ -870,6 +876,112 @@ if(WITH_FUSE)
   install(PROGRAMS mount.fuse.ceph DESTINATION ${CMAKE_INSTALL_SBINDIR})
 endif(WITH_FUSE)
 
+# NVMEOF GATEWAY MONITOR CLIENT
+# Supported on RPM-based platforms only, depends on grpc devel libraries/tools
+if(EXISTS "/etc/redhat-release" OR EXISTS "/etc/fedora-release")
+  option(WITH_NVMEOF_GATEWAY_MONITOR_CLIENT "build nvmeof gateway monitor client" ON)
+else()
+  option(WITH_NVMEOF_GATEWAY_MONITOR_CLIENT "build nvmeof gateway monitor client" OFF)
+endif()
+
+if(WITH_NVMEOF_GATEWAY_MONITOR_CLIENT)
+
+  # Find Protobuf installation
+  # Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
+  option(protobuf_MODULE_COMPATIBLE TRUE)
+  find_package(Protobuf REQUIRED)
+
+  set(_REFLECTION grpc++_reflection)
+  if(CMAKE_CROSSCOMPILING)
+    find_program(_PROTOBUF_PROTOC protoc)
+  else()
+    set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+  endif()
+
+  # Find gRPC installation
+  # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+  find_package(gRPC CONFIG REQUIRED)
+  message(STATUS "Using gRPC ${gRPC_VERSION}")
+  set(_GRPC_GRPCPP gRPC::grpc++)
+  if(CMAKE_CROSSCOMPILING)
+    find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+  else()
+    set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+  endif()
+
+  # Gateway Proto file
+  get_filename_component(nvmeof_gateway_proto "nvmeof/gateway/control/proto/gateway.proto" ABSOLUTE)
+  get_filename_component(nvmeof_gateway_proto_path "${nvmeof_gateway_proto}" PATH)
+
+  # Generated sources
+  set(nvmeof_gateway_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/gateway.pb.cc")
+  set(nvmeof_gateway_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/gateway.pb.h")
+  set(nvmeof_gateway_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/gateway.grpc.pb.cc")
+  set(nvmeof_gateway_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/gateway.grpc.pb.h")
+
+  add_custom_command(
+        OUTPUT "${nvmeof_gateway_proto_srcs}" "${nvmeof_gateway_proto_hdrs}" "${nvmeof_gateway_grpc_srcs}" "${nvmeof_gateway_grpc_hdrs}"
+        COMMAND ${_PROTOBUF_PROTOC}
+        ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+          --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+          -I "${nvmeof_gateway_proto_path}"
+          --experimental_allow_proto3_optional
+          --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+          "${nvmeof_gateway_proto}"
+        DEPENDS "${nvmeof_gateway_proto}")
+
+
+  # Monitor Proto file
+  get_filename_component(nvmeof_monitor_proto "nvmeof/gateway/control/proto/monitor.proto" ABSOLUTE)
+  get_filename_component(nvmeof_monitor_proto_path "${nvmeof_monitor_proto}" PATH)
+
+  # Generated sources
+  set(nvmeof_monitor_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/monitor.pb.cc")
+  set(nvmeof_monitor_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/monitor.pb.h")
+  set(nvmeof_monitor_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/monitor.grpc.pb.cc")
+  set(nvmeof_monitor_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/monitor.grpc.pb.h")
+
+  add_custom_command(
+        OUTPUT "${nvmeof_monitor_proto_srcs}" "${nvmeof_monitor_proto_hdrs}" "${nvmeof_monitor_grpc_srcs}" "${nvmeof_monitor_grpc_hdrs}"
+        COMMAND ${_PROTOBUF_PROTOC}
+        ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+          --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+          -I "${nvmeof_monitor_proto_path}"
+          --experimental_allow_proto3_optional
+          --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+          "${nvmeof_monitor_proto}"
+        DEPENDS "${nvmeof_monitor_proto}")
+
+  # Include generated *.pb.h files
+  include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+  set(ceph_nvmeof_monitor_client_srcs
+    ${nvmeof_gateway_proto_srcs}
+    ${nvmeof_gateway_proto_hdrs}
+    ${nvmeof_gateway_grpc_srcs}
+    ${nvmeof_gateway_grpc_hdrs}
+    ${nvmeof_monitor_proto_srcs}
+    ${nvmeof_monitor_proto_hdrs}
+    ${nvmeof_monitor_grpc_srcs}
+    ${nvmeof_monitor_grpc_hdrs}
+    ceph_nvmeof_monitor_client.cc
+    nvmeof/NVMeofGwClient.cc
+    nvmeof/NVMeofGwMonitorGroupClient.cc
+    nvmeof/NVMeofGwMonitorClient.cc)
+  add_executable(ceph-nvmeof-monitor-client ${ceph_nvmeof_monitor_client_srcs})
+  add_dependencies(ceph-nvmeof-monitor-client ceph-common)
+  target_link_libraries(ceph-nvmeof-monitor-client
+    client
+    mon
+    global-static
+    ceph-common
+    ${_REFLECTION}
+    ${_GRPC_GRPCPP}
+    )
+  install(TARGETS ceph-nvmeof-monitor-client DESTINATION bin)
+endif()
+# END OF NVMEOF GATEWAY MONITOR CLIENT
+
 if(WITH_DOKAN)
   add_subdirectory(dokan)
 endif(WITH_DOKAN)
diff --git a/src/ceph_nvmeof_monitor_client.cc b/src/ceph_nvmeof_monitor_client.cc
new file mode 100644
index 000000000000..05457998cb8b
--- /dev/null
+++ b/src/ceph_nvmeof_monitor_client.cc
@@ -0,0 +1,79 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM Inc
+ *
+ * Author: Alexander Indenbaum <aindenba@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <pthread.h>
+
+#include "include/types.h"
+#include "include/compat.h"
+#include "common/config.h"
+#include "common/ceph_argparse.h"
+#include "common/errno.h"
+#include "common/pick_address.h"
+#include "global/global_init.h"
+
+#include "nvmeof/NVMeofGwMonitorClient.h"
+
+static void usage()
+{
+  std::cout << "usage: ceph-nvmeof-monitor-client\n"
+               "        --gateway-name <GW_NAME>\n"
+               "        --gateway-address <GW_ADDRESS>\n"
+               "        --gateway-pool <CEPH_POOL>\n"
+               "        --gateway-group <GW_GROUP>\n"
+               "        --monitor-group-address <MONITOR_GROUP_ADDRESS>\n"
+               "        [flags]\n"
+	    << std::endl;
+  generic_server_usage();
+}
+
+/**
+ * A short main() which just instantiates a Nvme and
+ * hands over control to that.
+ */
+int main(int argc, const char **argv)
+{
+  ceph_pthread_setname(pthread_self(), "ceph-nvmeof-monitor-client");
+
+  auto args = argv_to_vec(argc, argv);
+  if (args.empty()) {
+    std::cerr << argv[0] << ": -h or --help for usage" << std::endl;
+    exit(1);
+  }
+  if (ceph_argparse_need_usage(args)) {
+    usage();
+    exit(0);
+  }
+
+  auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
+                         CODE_ENVIRONMENT_UTILITY, // maybe later use CODE_ENVIRONMENT_DAEMON,
+			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+
+  pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC);
+
+  global_init_daemonize(g_ceph_context);
+  global_init_chdir(g_ceph_context);
+  common_init_finish(g_ceph_context);
+
+  NVMeofGwMonitorClient gw_monitor_client(argc, argv);
+  int rc = gw_monitor_client.init();
+  if (rc != 0) {
+      std::cerr << "Error in initialization: " << cpp_strerror(rc) << std::endl;
+      return rc;
+  }
+
+  return gw_monitor_client.main(args);
+}
+
diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
index 379917445c0f..627dbb417314 100644
--- a/src/common/options/mon.yaml.in
+++ b/src/common/options/mon.yaml.in
@@ -72,6 +72,14 @@ options:
   default: 30
   services:
   - mon
+- name: mon_nvmeofgw_beacon_grace
+  type: secs
+  level: advanced
+  desc: Period in seconds from last beacon to monitor marking a  NVMeoF gateway as
+    failed
+  default: 10
+  services:
+  - mon
 - name: mon_mgr_inactive_grace
   type: int
   level: advanced
@@ -1347,3 +1355,10 @@ options:
   with_legacy: true
   see_also:
   - osd_heartbeat_use_min_delay_socket
+- name: nvmeof_mon_client_tick_period
+  type: secs
+  level: advanced
+  desc: Period in seconds of nvmeof gateway beacon messages to monitor
+  default: 2
+  services:
+  - mon
diff --git a/src/messages/MNVMeofGwBeacon.h b/src/messages/MNVMeofGwBeacon.h
new file mode 100644
index 000000000000..202d77a78b87
--- /dev/null
+++ b/src/messages/MNVMeofGwBeacon.h
@@ -0,0 +1,132 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_NVMEOFGWBEACON_H
+#define CEPH_NVMEOFGWBEACON_H
+
+#include <cstddef>
+#include <vector>
+#include "messages/PaxosServiceMessage.h"
+#include "mon/MonCommand.h"
+#include "mon/NVMeofGwMap.h"
+#include "include/types.h"
+
+class MNVMeofGwBeacon final : public PaxosServiceMessage {
+private:
+  static constexpr int HEAD_VERSION = 1;
+  static constexpr int COMPAT_VERSION = 1;
+
+protected:
+    std::string       gw_id;
+    std::string       gw_pool;
+    std::string       gw_group;
+    BeaconSubsystems  subsystems;                           // gateway susbsystem and their state machine states
+    GW_AVAILABILITY_E availability;                         // in absence of  beacon  heartbeat messages it becomes inavailable
+    epoch_t           last_osd_epoch;
+    epoch_t           last_gwmap_epoch;
+
+public:
+  MNVMeofGwBeacon()
+    : PaxosServiceMessage{MSG_MNVMEOF_GW_BEACON, 0, HEAD_VERSION, COMPAT_VERSION}
+  {
+    set_priority(CEPH_MSG_PRIO_HIGH);
+  }
+
+  MNVMeofGwBeacon(const std::string &gw_id_,
+        const std::string& gw_pool_,
+        const std::string& gw_group_,
+        const BeaconSubsystems& subsystems_,
+        const GW_AVAILABILITY_E& availability_,
+        const epoch_t& last_osd_epoch_,
+        const epoch_t& last_gwmap_epoch_
+  )
+    : PaxosServiceMessage{MSG_MNVMEOF_GW_BEACON, 0, HEAD_VERSION, COMPAT_VERSION},
+      gw_id(gw_id_), gw_pool(gw_pool_), gw_group(gw_group_), subsystems(subsystems_),
+      availability(availability_), last_osd_epoch(last_osd_epoch_), last_gwmap_epoch(last_gwmap_epoch_)
+  {
+    set_priority(CEPH_MSG_PRIO_HIGH);
+  }
+
+  const std::string& get_gw_id() const { return gw_id; }
+  const std::string& get_gw_pool() const { return gw_pool; }
+  const std::string& get_gw_group() const { return gw_group; }
+  NvmeAnaNonceMap get_nonce_map() const {
+    NvmeAnaNonceMap nonce_map;
+    for (const auto& sub: subsystems) {
+      for (const auto& ns: sub.namespaces) {
+        auto& nonce_vec = nonce_map[ns.anagrpid-1];//Converting   ana groups to offsets
+        if (std::find(nonce_vec.begin(), nonce_vec.end(), ns.nonce) == nonce_vec.end())
+          nonce_vec.push_back(ns.nonce);
+      }
+    }
+    return nonce_map;
+  }
+
+  const GW_AVAILABILITY_E& get_availability()   const   { return availability; }
+  const epoch_t&           get_last_osd_epoch() const   { return last_osd_epoch; }
+  const epoch_t&           get_last_gwmap_epoch() const { return last_gwmap_epoch; }
+  const BeaconSubsystems&  get_subsystems()     const   { return subsystems; };
+
+private:
+  ~MNVMeofGwBeacon() final {}
+
+public:
+
+  std::string_view get_type_name() const override { return "nvmeofgwbeacon"; }
+
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    paxos_encode();
+    encode(gw_id, payload);
+    encode(gw_pool, payload);
+    encode(gw_group, payload);
+    encode((uint32_t)subsystems.size(), payload);
+    for (const auto& st: subsystems) {
+      encode(st, payload);
+    }
+    encode((uint32_t)availability, payload);
+    encode(last_osd_epoch, payload);
+    encode(last_gwmap_epoch, payload);
+  }
+
+  void decode_payload() override {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    
+    paxos_decode(p);
+    decode(gw_id, p);
+    decode(gw_pool, p);
+    decode(gw_group, p);
+    uint32_t n;
+    decode(n, p);
+    subsystems.clear();
+    for (uint32_t i = 0; i < n; i++) {
+      BeaconSubsystem sub;
+      decode(sub, p);
+      subsystems.push_back(sub);
+    }
+    uint32_t tmp;
+    decode(tmp, p);
+    availability = static_cast<GW_AVAILABILITY_E>(tmp);
+    decode(last_osd_epoch, p);
+    decode(last_gwmap_epoch, p);
+  }
+
+private:
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+};
+
+
+#endif
diff --git a/src/messages/MNVMeofGwMap.h b/src/messages/MNVMeofGwMap.h
new file mode 100644
index 000000000000..add2554b137f
--- /dev/null
+++ b/src/messages/MNVMeofGwMap.h
@@ -0,0 +1,62 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_MNVMEOFGWMAP_H
+#define CEPH_MNVMEOFGWMAP_H
+
+#include "msg/Message.h"
+#include "mon/NVMeofGwMap.h"
+
+class MNVMeofGwMap final : public Message {
+protected:
+  std::map<NvmeGroupKey, NvmeGwMap> map;
+  epoch_t                           gwmap_epoch;
+
+public:
+  const std::map<NvmeGroupKey, NvmeGwMap>& get_map() {return map;}
+  const epoch_t& get_gwmap_epoch() {return gwmap_epoch;}
+
+private:
+  MNVMeofGwMap() :
+    Message{MSG_MNVMEOF_GW_MAP} {}
+  MNVMeofGwMap(const NVMeofGwMap &map_) :
+    Message{MSG_MNVMEOF_GW_MAP}, gwmap_epoch(map_.epoch)
+  {
+    map_.to_gmap(map);
+  }
+  ~MNVMeofGwMap() final {}
+
+public:
+  std::string_view get_type_name() const override { return "nvmeofgwmap"; }
+
+  void decode_payload() override {
+    auto p = payload.cbegin();
+    decode(gwmap_epoch, p);
+    decode(map, p);
+  }
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    encode(gwmap_epoch, payload);
+    encode(map, payload);
+  }
+private:
+  using RefCountedObject::put;
+  using RefCountedObject::get;
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+  template<class T, typename... Args>
+  friend MURef<T> crimson::make_message(Args&&... args);
+};
+
+#endif
diff --git a/src/mon/CMakeLists.txt b/src/mon/CMakeLists.txt
index 3f2b8605344b..94b5d55bd3e3 100644
--- a/src/mon/CMakeLists.txt
+++ b/src/mon/CMakeLists.txt
@@ -21,6 +21,8 @@ set(lib_mon_srcs
   ConnectionTracker.cc
   HealthMonitor.cc
   KVMonitor.cc
+  NVMeofGwMon.cc
+  NVMeofGwMap.cc
   ../mds/MDSAuthCaps.cc
   ../mgr/mgr_commands.cc
   ../osd/OSDCap.cc
diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 14bb3602c9b5..3adac47b28ef 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1366,8 +1366,25 @@ COMMAND("config generate-minimal-conf",
 	"Generate a minimal ceph.conf file",
 	"config", "r")
 
+/* NVMeofGwMon*/
+COMMAND("nvme-gw create"
+    " name=id,type=CephString"
+    " name=pool,type=CephString"
+    " name=group,type=CephString",
+    "create nvmeof gateway id for (pool, group)",
+    "mgr", "rw")
+COMMAND("nvme-gw delete"
+    " name=id,type=CephString"
+    " name=pool,type=CephString"
+    " name=group,type=CephString",
+    "delete nvmeof gateway id for (pool, group)",
+    "mgr", "rw")
 
-
+COMMAND("nvme-gw show"
+   " name=pool,type=CephString"
+   " name=group,type=CephString",
+   " show nvmeof gateways within (pool, group)",
+   "mgr", "rw")
 
 // these are tell commands that were implemented as CLI commands in
 // the broken pre-octopus way that we want to allow to work when a
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index a70bfbe33c9d..34182abb174b 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -84,6 +84,7 @@
 #include "MgrStatMonitor.h"
 #include "ConfigMonitor.h"
 #include "KVMonitor.h"
+#include "NVMeofGwMon.h"
 #include "mon/HealthMonitor.h"
 #include "common/config.h"
 #include "common/cmdparse.h"
@@ -247,6 +248,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
   paxos_service[PAXOS_HEALTH].reset(new HealthMonitor(*this, *paxos, "health"));
   paxos_service[PAXOS_CONFIG].reset(new ConfigMonitor(*this, *paxos, "config"));
   paxos_service[PAXOS_KV].reset(new KVMonitor(*this, *paxos, "kv"));
+  paxos_service[PAXOS_NVMEGW].reset(new NVMeofGwMon(*this, *paxos, "nvmeofgw"));
 
   bool r = mon_caps.parse("allow *", NULL);
   ceph_assert(r);
@@ -3617,7 +3619,11 @@ void Monitor::handle_command(MonOpRequestRef op)
     mgrmon()->dispatch(op);
     return;
   }
-
+  if (module == "nvme-gw"){
+      nvmegwmon()->dispatch(op);
+      dout(10) << " Dispatching module " << module << " to NVMeofGwMon"  << dendl;
+      return;
+  }
   if (prefix == "fsid") {
     if (f) {
       f->open_object_section("fsid");
@@ -4446,6 +4452,7 @@ void Monitor::_ms_dispatch(Message *m)
   }
 
   MonOpRequestRef op = op_tracker.create_request<MonOpRequest>(m);
+  dout(10) << "Received message: " << op->get_req()->get_type() << dendl;
   bool src_is_mon = op->is_src_mon();
   op->mark_event("mon:_ms_dispatch");
   MonSession *s = op->get_session();
@@ -4551,6 +4558,9 @@ void Monitor::_ms_dispatch(Message *m)
 void Monitor::dispatch_op(MonOpRequestRef op)
 {
   op->mark_event("mon:dispatch_op");
+
+  dout(10) << "Received message: " << op->get_req()->get_type() << dendl;
+
   MonSession *s = op->get_session();
   ceph_assert(s);
   if (s->closed) {
@@ -4664,6 +4674,11 @@ void Monitor::dispatch_op(MonOpRequestRef op)
       paxos_service[PAXOS_MGR]->dispatch(op);
       return;
 
+    case MSG_MNVMEOF_GW_BEACON:
+       paxos_service[PAXOS_NVMEGW]->dispatch(op);
+       return;
+
+
     // MgrStat
     case MSG_MON_MGR_REPORT:
     case CEPH_MSG_STATFS:
@@ -5351,6 +5366,10 @@ void Monitor::handle_subscribe(MonOpRequestRef op)
     } else if (p->first.find("kv:") == 0) {
       kvmon()->check_sub(s->sub_map[p->first]);
     }
+    else if (p->first == "NVMeofGw") {
+        dout(10) << "NVMeofGw->check_sub " << dendl;
+        nvmegwmon()->check_sub(s->sub_map[p->first]);
+    }
   }
 
   if (reply) {
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 13afacafde7d..0f8481eea6dc 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -712,6 +712,11 @@ class Monitor : public Dispatcher,
     return (class KVMonitor*) paxos_service[PAXOS_KV].get();
   }
 
+  class NVMeofGwMon *nvmegwmon() {
+      return (class NVMeofGwMon*) paxos_service[PAXOS_NVMEGW].get();
+  }
+
+
   friend class Paxos;
   friend class OSDMonitor;
   friend class MDSMonitor;
diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
new file mode 100755
index 000000000000..869c3784ca1d
--- /dev/null
+++ b/src/mon/NVMeofGwMap.cc
@@ -0,0 +1,598 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include <boost/tokenizer.hpp>
+#include "include/stringify.h"
+#include "NVMeofGwMon.h"
+#include "NVMeofGwMap.h"
+#include "OSDMonitor.h"
+
+using std::map;
+using std::make_pair;
+using std::ostream;
+using std::ostringstream;
+using std::string;
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix *_dout << "nvmeofgw " << __PRETTY_FUNCTION__ << " "
+
+void NVMeofGwMap::to_gmap(std::map<NvmeGroupKey, NvmeGwMap>& Gmap) const {
+    Gmap.clear();
+    for (const auto& created_map_pair: Created_gws) {
+        const auto& group_key = created_map_pair.first;
+        const NvmeGwCreatedMap& gw_created_map = created_map_pair.second;
+        for (const auto& gw_created_pair: gw_created_map) {
+            const auto& gw_id = gw_created_pair.first;
+            const auto& gw_created  = gw_created_pair.second;
+
+            auto gw_state = NvmeGwState(gw_created.ana_grp_id, epoch);
+            for (const auto& sub: gw_created.subsystems) {
+                gw_state.subsystems.insert({sub.nqn, NqnState(sub.nqn, gw_created.sm_state, gw_created )});
+            }
+            Gmap[group_key][gw_id] = gw_state;
+        }
+    }
+}
+
+int  NVMeofGwMap::cfg_add_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key) {
+    // Calculate allocated group bitmask
+    bool allocated[MAX_SUPPORTED_ANA_GROUPS] = {false};
+    for (auto& itr: Created_gws[group_key]) {
+        allocated[itr.second.ana_grp_id] = true;
+        if(itr.first == gw_id) {
+            dout(1) << __func__ << " ERROR create GW: already exists in map " << gw_id << dendl;
+            return -EEXIST ;
+        }
+    }
+
+    // Allocate the new group id
+    for(int i=0; i<=MAX_SUPPORTED_ANA_GROUPS; i++) {
+        if (allocated[i] == false) {
+            NvmeGwCreated gw_created(i);
+            Created_gws[group_key][gw_id] = gw_created;
+            dout(4) << __func__ << "Created GWS:  " << Created_gws  <<  dendl;
+            return 0;
+        }
+    }
+    dout(1) << __func__ << " ERROR create GW: " << gw_id << "   ANA groupId was not allocated "   << dendl;
+    return -EINVAL;
+}
+
+int NVMeofGwMap::cfg_delete_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key) {
+    int rc = 0;
+    for (auto& gws_states: Created_gws[group_key]) {
+
+        if (gws_states.first == gw_id) {
+            auto& state = gws_states.second;
+            for(int i=0; i<MAX_SUPPORTED_ANA_GROUPS; i++){
+                bool modified;
+                fsm_handle_gw_delete (gw_id, group_key, state.sm_state[i], i, modified);
+            }
+            dout(4) << " Delete GW :"<< gw_id  << " ANA grpid: " << state.ana_grp_id  << dendl;
+            Gmetadata[group_key].erase(gw_id);
+            if(Gmetadata[group_key].size() == 0)
+                Gmetadata.erase(group_key);
+
+            Created_gws[group_key].erase(gw_id);
+            if(Created_gws[group_key].size() == 0)
+                Created_gws.erase(group_key);
+            return rc;
+        }
+    }
+
+    return -EINVAL;
+}
+
+
+int NVMeofGwMap::process_gw_map_gw_down(const NvmeGwId &gw_id, const NvmeGroupKey& group_key,
+                                            bool &propose_pending) {
+    int rc = 0;
+    auto& gws_states = Created_gws[group_key];
+    auto  gw_state = gws_states.find(gw_id);
+    if (gw_state != gws_states.end()) {
+        dout(4) << "GW down " << gw_id << dendl;
+        auto& st = gw_state->second;
+        st.availability = GW_AVAILABILITY_E::GW_UNAVAILABLE;
+        for (NvmeAnaGrpId i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i ++) {
+            fsm_handle_gw_down (gw_id, group_key, st.sm_state[i], i, propose_pending);
+            st.standby_state(i);
+        }
+    }
+    else {
+        dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map " << gw_id << dendl;
+        rc = -EINVAL;
+    }
+    return rc;
+}
+
+
+void NVMeofGwMap::process_gw_map_ka(const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  epoch_t& last_osd_epoch, bool &propose_pending)
+{
+    auto& gws_states = Created_gws[group_key];
+    auto  gw_state = gws_states.find(gw_id);
+    ceph_assert (gw_state != gws_states.end());
+    auto& st = gw_state->second;
+    dout(20)  << "KA beacon from the GW " << gw_id << " in state " << (int)st.availability << dendl;
+
+    if (st.availability == GW_AVAILABILITY_E::GW_CREATED) {
+        // first time appears - allow IO traffic for this GW
+        st.availability = GW_AVAILABILITY_E::GW_AVAILABLE;
+        for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) st.sm_state[i] = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
+        if (st.ana_grp_id != REDUNDANT_GW_ANA_GROUP_ID) { // not a redundand GW
+            st.active_state(st.ana_grp_id);
+        }
+        propose_pending = true;
+    }
+    else if (st.availability == GW_AVAILABILITY_E::GW_UNAVAILABLE) {
+        st.availability = GW_AVAILABILITY_E::GW_AVAILABLE;
+        if (st.ana_grp_id == REDUNDANT_GW_ANA_GROUP_ID) {
+            for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) st.sm_state[i] = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
+            propose_pending = true; //TODO  try to find the 1st GW overloaded by ANA groups and start  failback for ANA group that it is not an owner of
+        }
+        else {
+            //========= prepare to Failback to this GW =========
+            // find the GW that took over on the group st.ana_grp_id
+            find_failback_gw(gw_id, group_key, propose_pending);
+        }
+    }
+    else if (st.availability == GW_AVAILABILITY_E::GW_AVAILABLE) {
+        for(int i=0; i<MAX_SUPPORTED_ANA_GROUPS; i++){
+          fsm_handle_gw_alive (gw_id, group_key, gw_state->second, st.sm_state[i], i, last_osd_epoch, propose_pending);
+        }
+    }
+}
+
+
+void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
+{
+    propose = false;
+    for (auto& group_state: Created_gws) {
+        auto& group_key = group_state.first;
+        auto& gws_states = group_state.second;
+
+            for (auto& gw_state : gws_states) { // loop for GWs inside nqn group
+                auto& gw_id = gw_state.first;
+                NvmeGwCreated& state = gw_state.second;
+
+                //1. Failover missed : is there is a GW in unavailable state? if yes, is its ANA group handled by some other GW?
+                if (state.availability == GW_AVAILABILITY_E::GW_UNAVAILABLE && state.ana_grp_id != REDUNDANT_GW_ANA_GROUP_ID) {
+                    auto found_gw_for_ana_group = false;
+                    for (auto& gw_state2 : gws_states) {
+                        NvmeGwCreated& state2 = gw_state2.second;
+                        if (state2.availability == GW_AVAILABILITY_E::GW_AVAILABLE && state2.sm_state[state.ana_grp_id] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE) {
+                            found_gw_for_ana_group = true; // dout(4) << "Found GW " << ptr2.first << " that handles ANA grp " << (int)state->optimized_ana_group_id << dendl;
+                            break;
+                        }
+                    }
+                    if (found_gw_for_ana_group == false) { //choose the GW for handle ana group
+                        dout(4)<< "Was not found the GW " << " that handles ANA grp " << (int)state.ana_grp_id << " find candidate "<< dendl;
+
+                        for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++)
+                            find_failover_candidate( gw_id, group_key, i, propose );
+                    }
+                }
+
+                //2. Failback missed: Check this GW is Available and Standby and no other GW is doing Failback to it
+                else if (state.availability == GW_AVAILABILITY_E::GW_AVAILABLE
+                            && state.ana_grp_id != REDUNDANT_GW_ANA_GROUP_ID &&
+                            state.sm_state[state.ana_grp_id] == GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE)
+                {
+                    find_failback_gw(gw_id, group_key, propose);
+                }
+            }
+    }
+}
+
+
+void  NVMeofGwMap::set_failover_gw_for_ANA_group(const NvmeGwId &failed_gw_id, const NvmeGroupKey& group_key, const NvmeGwId &gw_id,  NvmeAnaGrpId ANA_groupid)
+{
+    NvmeGwCreated& gw_state = Created_gws[group_key][gw_id];
+    gw_state.failover_peer[ANA_groupid] = failed_gw_id;
+    epoch_t epoch;
+    dout(4) << "Found failower GW " << gw_id << " for ANA group " << (int)ANA_groupid << dendl;
+    int rc = blocklist_gw (failed_gw_id, group_key, ANA_groupid, epoch, true);
+    if(rc){
+        gw_state.active_state(ANA_groupid); //TODO check whether it is valid to  start failover when nonces are empty !
+        //ceph_assert(false);
+    }
+    else{
+        gw_state.sm_state[ANA_groupid] = GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL;
+        gw_state.blocklist_data[ANA_groupid].osd_epoch = epoch;
+        gw_state.blocklist_data[ANA_groupid].is_failover = true;
+        start_timer(gw_id, group_key, ANA_groupid, 30); //start Failover preparation timer
+    }
+}
+
+void  NVMeofGwMap::find_failback_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key,   bool &propose)
+{
+    auto& gws_states = Created_gws[group_key];
+    auto& gw_state = Created_gws[group_key][gw_id];
+    bool do_failback = false;
+
+    dout(4) << "Find failback GW for GW " << gw_id << dendl;
+    for (auto& gw_state_it: gws_states) {
+        auto& st = gw_state_it.second;
+        if (st.sm_state[gw_state.ana_grp_id] != GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE) {// some other gw owns or owned the desired ana-group
+            do_failback = true;// if candidate is in state ACTIVE for the desired ana-group, then failback starts immediately, otherwise need to wait
+            dout(4) << "Found some gw " << gw_state_it.first  <<  " in state " << st.sm_state[gw_state.ana_grp_id]  << dendl;
+            break;
+        }
+    }
+
+    if (do_failback == false) {
+        // No other gw currently performs some activity with desired ana group of coming-up GW - so it just takes over on the group
+        dout(4)  << "Failback GW candidate was not found, just set Optimized to group " << gw_state.ana_grp_id << " to GW " << gw_id << dendl;
+        gw_state.active_state(gw_state.ana_grp_id);
+        propose = true;
+        return;
+    }
+    //try to do_failback
+    for (auto& gw_state_it: gws_states) {
+        auto& failback_gw_id = gw_state_it.first;
+        auto& st = gw_state_it.second;
+        if (st.sm_state[gw_state.ana_grp_id] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE) {
+            ceph_assert(st.failover_peer[gw_state.ana_grp_id] == gw_id);
+
+            dout(4)  << "Found Failback GW " << failback_gw_id << " that previously took over the ANAGRP " << gw_state.ana_grp_id << " of the available GW " << gw_id << dendl;
+            st.sm_state[gw_state.ana_grp_id] = GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED;
+            start_timer(failback_gw_id, group_key, gw_state.ana_grp_id, 3);// Add timestamp of start Failback preparation
+            gw_state.sm_state[gw_state.ana_grp_id] = GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED;
+            propose = true;
+            break;
+        }
+    }
+}
+
+
+// TODO When decision to change ANA state of group is prepared, need to consider that last seen FSM state is "approved" - means it was returned in beacon alone with map version
+void  NVMeofGwMap::find_failover_candidate(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid, bool &propose_pending)
+{
+    dout(4) <<__func__<< " " << gw_id << dendl;
+    #define ILLEGAL_GW_ID " "
+    #define MIN_NUM_ANA_GROUPS 0xFFF
+    int min_num_ana_groups_in_gw = 0;
+    int current_ana_groups_in_gw = 0;
+    NvmeGwId min_loaded_gw_id = ILLEGAL_GW_ID;
+
+    auto& gws_states = Created_gws[group_key];
+
+    auto gw_state = gws_states.find(gw_id);
+    ceph_assert(gw_state != gws_states.end());
+
+    // this GW may handle several ANA groups and  for each of them need to found the candidate GW
+    if (gw_state->second.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE || gw_state->second.ana_grp_id == grpid) {
+
+        for (auto& found_gw_state: gws_states) { // for all the gateways of the subsystem
+            auto st = found_gw_state.second;
+            if(st.sm_state[grpid] ==  GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL){   // some GW already started failover/failback on this group
+               dout(4) << "Failover" << st.blocklist_data[grpid].is_failover <<  " already started for the group " << grpid <<  " by GW " << found_gw_state.first << dendl;
+               gw_state->second.standby_state(grpid);
+               return ;
+            }
+        }
+
+        // Find a GW that takes over the ANA group(s)
+        min_num_ana_groups_in_gw = MIN_NUM_ANA_GROUPS;
+        min_loaded_gw_id = ILLEGAL_GW_ID;
+        for (auto& found_gw_state: gws_states) { // for all the gateways of the subsystem
+            auto st = found_gw_state.second;
+            if (st.availability == GW_AVAILABILITY_E::GW_AVAILABLE) {
+                current_ana_groups_in_gw = 0;
+                for (int j = 0; j < MAX_SUPPORTED_ANA_GROUPS; j++) {
+                    if (st.sm_state[j] == GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED || st.sm_state[j] == GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED
+                                                                                          || st.sm_state[j] == GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL){
+                        current_ana_groups_in_gw = 0xFFFF;
+                        break; // dont take into account   GWs in the transitive state
+                    }
+                    else if (st.sm_state[j] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE)
+                        //dout(4) << " process GW down " << current_ana_groups_in_gw << dendl;
+                        current_ana_groups_in_gw++; // how many ANA groups are handled by this GW
+                    }
+
+                    if (min_num_ana_groups_in_gw > current_ana_groups_in_gw) {
+                        min_num_ana_groups_in_gw = current_ana_groups_in_gw;
+                        min_loaded_gw_id = found_gw_state.first;
+                        dout(4) << "choose: gw-id  min_ana_groups " << min_loaded_gw_id << current_ana_groups_in_gw << " min " << min_num_ana_groups_in_gw << dendl;
+                    }
+                }
+            }
+            if (min_loaded_gw_id != ILLEGAL_GW_ID) {
+                propose_pending = true;
+                set_failover_gw_for_ANA_group(gw_id, group_key, min_loaded_gw_id, grpid);
+            }
+            else {
+                if (gw_state->second.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE){// not found candidate but map changed.
+                    propose_pending = true;
+                    dout(4) << "gw down no candidate found " << dendl;
+                }
+            }
+            gw_state->second.standby_state(grpid);
+        }
+}
+
+void NVMeofGwMap::fsm_handle_gw_alive (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  NvmeGwCreated & gw_state, GW_STATES_PER_AGROUP_E state, NvmeAnaGrpId grpid, epoch_t& last_osd_epoch, bool &map_modified)
+{
+    switch (state) {
+    case GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL:
+    {
+        int timer_val = get_timer(gw_id, group_key, grpid);
+        NvmeGwCreated& gw_map = Created_gws[group_key][gw_id];
+            if(gw_map.blocklist_data[grpid].osd_epoch <= last_osd_epoch){
+                dout(4) << "is-failover: " << gw_map.blocklist_data[grpid].is_failover << " osd epoch changed from " << gw_map.blocklist_data[grpid].osd_epoch << " to "<< last_osd_epoch
+                                       << " Ana-grp: " << grpid  << " timer:" << timer_val << dendl;
+                gw_state.active_state(grpid);                   // Failover Gw still alive and guaranteed that
+                cancel_timer(gw_id, group_key, grpid);          // ana group wouldnt be taken back  during blocklist wait period
+                map_modified = true;
+            }
+            else{
+                dout(20) << "osd epoch not changed from " <<  gw_map.blocklist_data[grpid].osd_epoch << " to "<< last_osd_epoch
+                                                   << " Ana-grp: " << grpid  << " timer:" << timer_val << dendl;
+            }
+    }
+    break;
+
+    default:
+        break;
+    }
+}
+
+ void NVMeofGwMap::fsm_handle_gw_down(const NvmeGwId &gw_id, const NvmeGroupKey& group_key,   GW_STATES_PER_AGROUP_E state, NvmeAnaGrpId grpid,  bool &map_modified)
+ {
+    switch (state)
+    {
+        case GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE:
+        case GW_STATES_PER_AGROUP_E::GW_IDLE_STATE:
+            // nothing to do
+            break;
+
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL:
+        {
+            cancel_timer(gw_id, group_key, grpid);
+        }break;
+
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED:
+            cancel_timer(gw_id, group_key,  grpid);
+
+            for (auto& gw_st: Created_gws[group_key]) {
+                auto& st = gw_st.second;
+                if (st.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED) { // found GW   that was intended for  Failback for this ana grp
+                    dout(4) << "Warning: Outgoing Failback when GW is down back - to rollback it"  <<" GW "  <<gw_id << "for ANA Group " << grpid << dendl;
+                    st.standby_state(grpid);
+                    map_modified = true;
+                    break;
+                }
+            }
+            break;
+
+        case GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED:
+            // nothing to do - let failback timer expire
+            break;
+
+        case GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE:
+        {
+            find_failover_candidate( gw_id, group_key, grpid, map_modified);
+        }
+        break;
+
+        default:{
+            ceph_assert(false);
+        }
+
+    }
+ }
+
+
+void NVMeofGwMap::fsm_handle_gw_delete (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,
+     GW_STATES_PER_AGROUP_E state , NvmeAnaGrpId grpid, bool &map_modified) {
+    switch (state)
+    {
+        case GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE:
+        case GW_STATES_PER_AGROUP_E::GW_IDLE_STATE:
+        case GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED:
+        {
+            NvmeGwCreated& gw_state = Created_gws[group_key][gw_id];
+
+            if (grpid == gw_state.ana_grp_id) {// Try to find GW that temporary owns my group - if found, this GW should pass to standby for  this group
+                auto& gateway_states = Created_gws[group_key];
+                for (auto& gs: gateway_states) {
+                    if (gs.second.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE  || gs.second.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED){
+                        gs.second.standby_state(grpid);
+                        map_modified = true;
+                        if (gs.second.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED)
+                            cancel_timer(gs.first, group_key, grpid);
+                        break;
+                    }
+                }
+            }
+        }
+        break;
+
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL:
+        {
+            NvmeGwCreated& gw_state = Created_gws[group_key][gw_id];
+            cancel_timer(gw_id, group_key, grpid);
+            map_modified = true;
+            gw_state.standby_state(grpid);
+        }
+        break;
+
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED:
+        {
+            cancel_timer(gw_id, group_key, grpid);
+            for (auto& nqn_gws_state: Created_gws[group_key]) {
+                auto& st = nqn_gws_state.second;
+
+                if (st.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED) { // found GW   that was intended for  Failback for this ana grp
+                    dout(4) << "Warning: Outgoing Failback when GW is deleted - to rollback it" << " GW " << gw_id << "for ANA Group " << grpid << dendl;
+                    st.standby_state(grpid);
+                    map_modified = true;
+                    break;
+                }
+            }
+        }
+        break;
+
+        case GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE:
+        {
+            NvmeGwCreated& gw_state = Created_gws[group_key][gw_id];
+            map_modified = true;
+            gw_state.standby_state(grpid);
+        }
+        break;
+
+        default: {
+            ceph_assert(false);
+        }
+    }
+}
+
+void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid,  bool &map_modified)
+{
+    auto& fbp_gw_state = Created_gws[group_key][gw_id];// GW in Fail-back preparation state fbp
+    bool grp_owner_found = false;
+    if (fbp_gw_state.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED) {
+        for (auto& gw_state: Created_gws[group_key]) {
+            auto& st = gw_state.second;
+            if (st.ana_grp_id == grpid){// group owner
+                grp_owner_found = true;
+                if( ! (fbp_gw_state.last_gw_map_epoch_valid  && st.last_gw_map_epoch_valid) ){
+                   //Timer is not cancelled so it would expire over and over as long as both gws are not updated
+                   dout(1) << "gw " << gw_id  <<" or gw " << gw_state.first  << "map epochs are not updated "<< dendl;
+                   return;
+                }
+                cancel_timer(gw_id, group_key, grpid);
+                if (st.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED && st.availability == GW_AVAILABILITY_E::GW_AVAILABLE )
+                {
+                    fbp_gw_state.standby_state(grpid);// Previous failover GW  set to standby
+                    st.active_state(grpid);
+                    dout(4)  << "Expired Failback-preparation timer from GW " << gw_id << " ANA groupId "<< grpid << dendl;
+                    map_modified = true;
+                    break;
+                }
+                else if(st.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE  &&  st.availability == GW_AVAILABILITY_E::GW_AVAILABLE) {
+                   st.active_state(grpid);// GW failed and started during the persistency interval
+                   dout(4)  << "Failback unsuccessfull. GW: " << gw_state.first << "becomes Active for the ana group " << grpid  << dendl;
+                }
+                fbp_gw_state.standby_state(grpid);
+                dout(4) << "Failback unsuccessfull GW: " << gw_id << "becomes standby for the ana group " << grpid  << dendl;
+                map_modified = true;
+                break;
+            }
+       }
+      ceph_assert(grp_owner_found);// when  GW group owner is deleted the fbk gw is put to standby
+    }
+    else if(fbp_gw_state.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL){
+        dout(1) << " Expired GW_WAIT_FAILOVER_PREPARED timer from GW " << gw_id << " ANA groupId: "<< grpid << dendl;
+        ceph_assert(false);
+    }
+}
+
+NvmeGwCreated& NVMeofGwMap::find_already_created_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key)
+{
+    auto& group_gws = Created_gws[group_key];
+    auto  gw_it = group_gws.find(gw_id);
+    ceph_assert(gw_it != group_gws.end());//should not happen
+    return gw_it->second;
+}
+
+struct CMonRequestProposal : public Context {
+  NVMeofGwMap *m;
+  CMonRequestProposal(NVMeofGwMap *mon) : m(mon) {}
+  void finish(int r) {
+      dout(4) << "osdmon is  writable? " << m->mon->osdmon()->is_writeable() << dendl;
+      if(m->mon->osdmon()->is_writeable()){
+        m->mon->nvmegwmon()->request_proposal(m->mon->osdmon());
+      }
+      else {
+          m->mon->osdmon()->wait_for_writeable_ctx( new CMonRequestProposal(m));
+      }
+  }
+};
+
+int NVMeofGwMap::blocklist_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid, epoch_t &epoch, bool failover)
+{
+    NvmeGwCreated& gw_map =  Created_gws[group_key][gw_id];  //find_already_created_gw(gw_id, group_key);
+
+     if (gw_map.nonce_map[grpid].size() > 0){
+        NvmeNonceVector &nonce_vector = gw_map.nonce_map[grpid];;
+        std::string str = "[";
+        entity_addrvec_t addr_vect;
+
+        double d = g_conf().get_val<double>("mon_osd_blocklist_default_expire");
+        utime_t expires = ceph_clock_now();
+        expires += d;
+        dout(4) << " blocklist timestamp " << expires << dendl;
+        for(auto &it: nonce_vector ){
+            if(str != "[") str += ",";
+            str += it;
+        }
+        str += "]";
+        bool rc = addr_vect.parse(&str[0]);
+        dout(10) << str << " rc " << rc <<  " network vector: " << addr_vect << " " << addr_vect.size() << dendl;
+        ceph_assert(rc);
+        epoch = mon->osdmon()->blocklist(addr_vect, expires);
+
+        if (!mon->osdmon()->is_writeable()) {
+            dout(4) << "osdmon is not writable, waiting " << dendl;
+            mon->osdmon()->wait_for_writeable_ctx( new CMonRequestProposal(this ));// return false;
+        }
+        else  mon->nvmegwmon()->request_proposal(mon->osdmon());
+        dout(4) << str << " mon->osdmon()->blocklist:  " << epoch <<  " address vector: " << addr_vect << " " << addr_vect.size() << dendl;
+    }
+    else{
+        dout(1) << "Error: No nonces context present for gw: " <<gw_id  << " ANA group: " << grpid << dendl;
+        return 1;
+    }
+    return 0;
+}
+
+void NVMeofGwMap::update_active_timers( bool &propose_pending ){
+
+    //dout(4) << __func__  <<  " called,  p_monitor: " << mon << dendl;
+    const auto now = std::chrono::system_clock::now();
+    for (auto& group_md: Gmetadata) {
+        auto& group_key = group_md.first;
+        auto& pool = group_key.first;
+        auto& group = group_key.second;
+        for (auto& gw_md: group_md.second) {
+            auto& gw_id = gw_md.first;
+            auto& md = gw_md.second;
+            for (size_t ana_grpid = 0; ana_grpid < MAX_SUPPORTED_ANA_GROUPS; ana_grpid ++) {
+                if (md.data[ana_grpid].timer_started == 0) continue;
+                dout(20) << "Checking timer for GW " << gw_id << " ANA GRP " << ana_grpid<< " value(seconds): "<< (int)md.data[ana_grpid].timer_value << dendl;
+                if(now >= md.data[ana_grpid].end_time){
+                    fsm_handle_to_expired (gw_id, std::make_pair(pool, group), ana_grpid, propose_pending);
+                }
+            }
+        }
+    }
+}
+
+
+void NVMeofGwMap::start_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid, uint8_t value_sec) {
+    Gmetadata[group_key][gw_id].data[anagrpid].timer_started = 1;
+    Gmetadata[group_key][gw_id].data[anagrpid].timer_value = value_sec;
+    dout(4) << "start timer for ana " << anagrpid << " gw " << gw_id << "value sec " << (int)value_sec << dendl;
+    const auto now = std::chrono::system_clock::now();
+    Gmetadata[group_key][gw_id].data[anagrpid].end_time = now + std::chrono::seconds(value_sec);
+}
+
+int  NVMeofGwMap::get_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid) {
+    auto timer = Gmetadata[group_key][gw_id].data[anagrpid].timer_value;
+    return timer;
+}
+
+void NVMeofGwMap::cancel_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid) {
+    Gmetadata[group_key][gw_id].data[anagrpid].timer_started = 0;
+}
diff --git a/src/mon/NVMeofGwMap.h b/src/mon/NVMeofGwMap.h
new file mode 100755
index 000000000000..bf46b31dc07e
--- /dev/null
+++ b/src/mon/NVMeofGwMap.h
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef MON_NVMEOFGWMAP_H_
+#define MON_NVMEOFGWMAP_H_
+#include <map>
+#include <iostream>
+#include "include/encoding.h"
+#include "include/utime.h"
+#include "common/Formatter.h"
+#include "common/ceph_releases.h"
+#include "common/version.h"
+#include "common/options.h"
+#include "common/Clock.h"
+#include "msg/Message.h"
+#include "common/ceph_time.h"
+#include "NVMeofGwTypes.h"
+
+using ceph::coarse_mono_clock;
+class Monitor;
+/*-------------------*/
+class NVMeofGwMap
+{
+public:
+    Monitor*                            mon           = NULL;
+    epoch_t                             epoch         = 0;      // epoch is for Paxos synchronization  mechanizm
+    bool                                delay_propose = false;
+
+    std::map<NvmeGroupKey, NvmeGwCreatedMap>  Created_gws;
+    std::map<NvmeGroupKey, NvmeGwMetaDataMap> Gmetadata;
+    void to_gmap(std::map<NvmeGroupKey, NvmeGwMap>& Gmap) const;
+
+    int   cfg_add_gw                    (const NvmeGwId &gw_id, const NvmeGroupKey& group_key);
+    int   cfg_delete_gw                 (const NvmeGwId &gw_id, const NvmeGroupKey& group_key);
+    void  process_gw_map_ka             (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, epoch_t& last_osd_epoch,  bool &propose_pending);
+    int   process_gw_map_gw_down        (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, bool &propose_pending);
+    void  update_active_timers          (bool &propose_pending);
+    void  handle_abandoned_ana_groups   (bool &propose_pending);
+    void  handle_removed_subsystems     (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, const std::vector<NvmeNqnId> &current_subsystems, bool &propose_pending);
+    void  start_timer (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid, uint8_t value);
+private:
+    NvmeGwCreated&   find_already_created_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key);
+    void fsm_handle_gw_down    (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  GW_STATES_PER_AGROUP_E state, NvmeAnaGrpId grpid,  bool &map_modified);
+    void fsm_handle_gw_delete  (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  GW_STATES_PER_AGROUP_E state, NvmeAnaGrpId grpid,  bool &map_modified);
+    void fsm_handle_gw_alive   (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  NvmeGwCreated & gw_state, GW_STATES_PER_AGROUP_E state,
+                                                                                   NvmeAnaGrpId grpid, epoch_t& last_osd_epoch, bool &map_modified);
+    void fsm_handle_to_expired (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  NvmeAnaGrpId grpid,  bool &map_modified);
+
+    void find_failover_candidate(const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  NvmeAnaGrpId grpid, bool &propose_pending);
+    void find_failback_gw       (const NvmeGwId &gw_id, const NvmeGroupKey& group_key,  bool &propose_pending);
+    void set_failover_gw_for_ANA_group (const NvmeGwId &failed_gw_id, const NvmeGroupKey& group_key, const NvmeGwId &gw_id,
+                                                                                                     NvmeAnaGrpId groupid);
+    int  blocklist_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId ANA_groupid, epoch_t &epoch, bool failover);
+
+    int  get_timer   (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
+    void cancel_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
+
+public:
+    void encode(ceph::buffer::list &bl) const {
+        using ceph::encode;
+        ENCODE_START(1, 1, bl);
+        encode(epoch, bl);// global map epoch
+
+        encode(Created_gws, bl); //Encode created GWs
+        encode(Gmetadata, bl);
+        ENCODE_FINISH(bl);
+    }
+
+    void decode(ceph::buffer::list::const_iterator &bl) {
+        using ceph::decode;
+        DECODE_START(1, bl);
+        decode(epoch, bl);
+
+        decode(Created_gws, bl);
+        decode(Gmetadata, bl);
+        DECODE_FINISH(bl);
+    }
+};
+
+#include "NVMeofGwSerialize.h"
+
+#endif /* SRC_MON_NVMEOFGWMAP_H_ */
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
new file mode 100644
index 000000000000..bdbdec716dab
--- /dev/null
+++ b/src/mon/NVMeofGwMon.cc
@@ -0,0 +1,546 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include <boost/tokenizer.hpp>
+#include "include/stringify.h"
+#include "NVMeofGwMon.h"
+#include "messages/MNVMeofGwBeacon.h"
+#include "messages/MNVMeofGwMap.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix *_dout << "nvmeofgw " << __PRETTY_FUNCTION__ << " "
+
+using std::string;
+
+void NVMeofGwMon::init(){
+    dout(4) <<  "called " << dendl;
+    g_conf().add_observer(this);
+}
+
+void NVMeofGwMon::on_restart(){
+    dout(4) <<  "called " << dendl;
+    last_beacon.clear();
+    last_tick = ceph::coarse_mono_clock::now();
+}
+
+
+void NVMeofGwMon::synchronize_last_beacon(){
+    dout(4) <<  "called " << dendl;
+    last_beacon.clear();
+    last_tick = ceph::coarse_mono_clock::now();
+    // Initialize last_beacon to identify transitions of available  GWs to unavailable state
+    for (const auto& created_map_pair: pending_map.Created_gws) {
+      const auto& group_key = created_map_pair.first;
+      const NvmeGwCreatedMap& gw_created_map = created_map_pair.second;
+      for (const auto& gw_created_pair: gw_created_map) {
+          const auto& gw_id = gw_created_pair.first;
+          if (gw_created_pair.second.availability == GW_AVAILABILITY_E::GW_AVAILABLE){
+             dout(4) << "synchronize last_beacon for  GW :" << gw_id << dendl;
+             LastBeacon lb = {gw_id, group_key};
+             last_beacon[lb] = last_tick;
+          }
+      }
+    }
+}
+
+void NVMeofGwMon::on_shutdown() {
+     g_conf().remove_observer(this);
+}
+
+void NVMeofGwMon::tick(){
+    if (!is_active() || !mon.is_leader()){
+        dout(10) << "NVMeofGwMon leader : " << mon.is_leader() << "active : " << is_active()  << dendl;
+        last_leader = false;
+        return;
+    }
+    bool _propose_pending = false;
+  
+    const auto now = ceph::coarse_mono_clock::now();
+    const auto nvmegw_beacon_grace = g_conf().get_val<std::chrono::seconds>("mon_nvmeofgw_beacon_grace"); 
+    dout(15) <<  "NVMeofGwMon leader got a real tick, pending epoch "<< pending_map.epoch     << dendl;
+
+    const auto mgr_tick_period = g_conf().get_val<std::chrono::seconds>("mgr_tick_period");
+
+    if (last_tick != ceph::coarse_mono_clock::zero()
+          && (now - last_tick > (nvmegw_beacon_grace - mgr_tick_period))) {
+        // This case handles either local slowness (calls being delayed
+        // for whatever reason) or cluster election slowness (a long gap
+        // between calls while an election happened)
+        dout(4) << ": resetting beacon timeouts due to mon delay "
+                "(slow election?) of " << now - last_tick << " seconds" << dendl;
+        for (auto &i : last_beacon) {
+          i.second = now;
+        }
+    }
+
+    last_tick = now;
+    bool propose = false;
+
+    pending_map.update_active_timers(propose);  // Periodic: check active FSM timers
+    _propose_pending |= propose;
+
+
+    //handle exception of tick overdued in order to avoid false detection of overdued beacons , see MgrMonitor::tick
+    const auto cutoff = now - nvmegw_beacon_grace;
+    for(auto &itr : last_beacon){// Pass over all the stored beacons
+        auto& lb = itr.first;
+        auto last_beacon_time = itr.second;
+        if(last_beacon_time < cutoff){
+            dout(4) << "beacon timeout for GW " << lb.gw_id << dendl;
+            pending_map.process_gw_map_gw_down( lb.gw_id, lb.group_key, propose);
+            _propose_pending |= propose;
+            last_beacon.erase(lb);
+        }
+        else {
+           dout(20) << "beacon live for GW key: " << lb.gw_id << dendl;
+        }
+    }
+
+    pending_map.handle_abandoned_ana_groups(propose); // Periodic: take care of not handled ANA groups
+    _propose_pending |= propose;
+
+    if(_propose_pending){
+       //pending_map.delay_propose = true; // not to send map to clients immediately in "update_from_paxos"
+       dout(4) << "propose pending " <<dendl;
+       propose_pending();
+    }
+
+    // if propose_pending returned true , call propose_pending method of the paxosService
+    // todo understand the logic of paxos.plugged for sending several propose_pending see MgrMonitor::tick
+}
+
+const char **NVMeofGwMon::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    NULL
+  };
+  return KEYS;
+}
+
+void NVMeofGwMon::handle_conf_change(const ConfigProxy& conf,
+                                    const std::set<std::string> &changed)
+{
+  dout(4) << "changed " << changed << dendl;
+}
+
+void NVMeofGwMon::create_pending(){
+
+    pending_map = map;// deep copy of the object
+    // TODO  since "pending_map"  can be reset  each time during paxos re-election even in the middle of the changes ...
+    pending_map.epoch++;
+    dout(4) << " pending " << pending_map  << dendl;
+    if(last_leader == false){ // peon becomes leader and gets updated map , need to synchronize the last_beacon
+        synchronize_last_beacon();
+        last_leader = true;
+    }
+}
+
+void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t){
+
+    dout(10) <<  dendl;
+    ceph_assert(get_last_committed() + 1 == pending_map.epoch);
+    bufferlist bl;
+    pending_map.encode(bl);
+    put_version(t, pending_map.epoch, bl);
+    put_last_committed(t, pending_map.epoch);
+}
+
+void NVMeofGwMon::update_from_paxos(bool *need_bootstrap){
+    version_t version = get_last_committed();
+
+    //dout(4) <<  MY_MON_PREFFIX << __func__ << " version "  << version  << " map.epoch " << map.epoch << dendl;
+
+    if (version != map.epoch) {
+        dout(4) << " NVMeGW loading version " << version  << " " << map.epoch << dendl;
+
+        bufferlist bl;
+        int err = get_version(version, bl);
+        ceph_assert(err == 0);
+
+        auto p = bl.cbegin();
+        map.decode(p);
+        if(!mon.is_leader()) {
+            dout(4) << "leader map: " << map <<  dendl;
+        }
+        check_subs(true);
+    }
+}
+
+void NVMeofGwMon::check_sub(Subscription *sub)
+{
+   /* MgrMonitor::check_sub*/
+    //if (sub->type == "NVMeofGw") {
+    dout(10) << "sub->next , map-epoch " << sub->next << " " << map.epoch << dendl;
+    if (sub->next <= map.epoch)
+    {
+      dout(4) << "Sending map to subscriber " << sub->session->con << " " << sub->session->con->get_peer_addr() << dendl;
+      sub->session->con->send_message2(make_message<MNVMeofGwMap>(map));
+
+      if (sub->onetime) {
+        mon.session_map.remove_sub(sub);
+      } else {
+        sub->next = map.epoch + 1;
+      }
+    }
+}
+
+void NVMeofGwMon::check_subs(bool t)
+{
+  const std::string type = "NVMeofGw";
+  dout(4) <<  "count " << mon.session_map.subs.count(type) << dendl;
+
+  if (mon.session_map.subs.count(type) == 0){
+      return;
+  }
+  for (auto sub : *(mon.session_map.subs[type])){
+    check_sub(sub);
+  }
+}
+
+bool NVMeofGwMon::preprocess_query(MonOpRequestRef op){
+    dout(20) << dendl;
+
+    auto m = op->get_req<PaxosServiceMessage>();
+      switch (m->get_type()) {
+        case MSG_MNVMEOF_GW_BEACON:
+          return preprocess_beacon(op);
+
+        case MSG_MON_COMMAND:
+          try {
+        return preprocess_command(op);
+          } catch (const bad_cmd_get& e) {
+          bufferlist bl;
+          mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+          return true;
+        }
+
+        default:
+          mon.no_reply(op);
+          derr << "Unhandled message type " << m->get_type() << dendl;
+          return true;
+      }
+    return false;
+}
+
+bool NVMeofGwMon::prepare_update(MonOpRequestRef op){
+    auto m = op->get_req<PaxosServiceMessage>();
+      switch (m->get_type()) {
+        case MSG_MNVMEOF_GW_BEACON:
+          return prepare_beacon(op);
+
+        case MSG_MON_COMMAND:
+          try {
+        return prepare_command(op);
+          } catch (const bad_cmd_get& e) {
+        bufferlist bl;
+        mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed());
+        return false; /* nothing to propose! */
+          }
+
+        default:
+          mon.no_reply(op);
+          dout(1) << "Unhandled message type " << m->get_type() << dendl;
+          return false; /* nothing to propose! */
+      }
+    return true;
+}
+
+bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
+{
+    dout(4) << dendl;
+    auto m = op->get_req<MMonCommand>();
+    std::stringstream ss;
+    bufferlist rdata;
+
+    cmdmap_t cmdmap;
+    if (!cmdmap_from_json(m->cmd, &cmdmap, ss))
+    {
+        string rs = ss.str();
+        dout(1) << "Invalid command "  << m->cmd << "Error " << rs << dendl;
+        mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+        return true;
+    }
+
+    string prefix;
+    cmd_getval(cmdmap, "prefix", prefix);
+    dout(4) << "MonCommand : "<< prefix <<  dendl;
+
+   /* MonSession *session = op->get_session();
+    if (!session)
+    {
+        dout(4) << "MonCommand : "<< prefix << " access denied due to lack of session" <<  dendl;
+        mon.reply_command(op, -EACCES, "access denied", rdata,
+                          get_last_committed());
+        return true;
+    }
+   */
+    string format = cmd_getval_or<string>(cmdmap, "format", "plain");
+    boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+    // TODO   need to check formatter per preffix  - if f is NULL
+
+    return false;
+}
+
+bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
+{
+    dout(4)  << dendl;
+    auto m = op->get_req<MMonCommand>();
+    int rc;
+    std::stringstream ss;
+    bufferlist rdata;
+    string rs;
+    int err = 0;
+    cmdmap_t cmdmap;
+
+    if (!cmdmap_from_json(m->cmd, &cmdmap, ss))
+    {
+        string rs = ss.str();
+        mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed());
+        return true;
+    }
+
+    MonSession *session = op->get_session();
+    if (!session)
+    {
+        mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
+        return true;
+    }
+
+    string format = cmd_getval_or<string>(cmdmap, "format", "plain");
+    boost::scoped_ptr<Formatter> f(Formatter::create(format));
+
+    const auto prefix = cmd_getval_or<string>(cmdmap, "prefix", string{});
+
+    dout(4) << "MonCommand : "<< prefix <<  dendl;
+    if( prefix == "nvme-gw create" || prefix == "nvme-gw delete" ) {
+        std::string id, pool, group;
+
+        cmd_getval(cmdmap, "id", id);
+        cmd_getval(cmdmap, "pool", pool);
+        cmd_getval(cmdmap, "group", group);
+        auto group_key = std::make_pair(pool, group);
+        dout(4) << " id "<< id <<" pool "<< pool << " group "<< group << dendl;
+        if(prefix == "nvme-gw create"){
+            rc = pending_map.cfg_add_gw(id, group_key);
+            ceph_assert(rc!= -EINVAL);
+        }
+        else{
+            rc = pending_map.cfg_delete_gw(id, group_key);
+            if(rc== -EINVAL){
+                dout (1) << "Error: GW not found in the database " << id << " " << pool << " " << group << "  rc " << rc << dendl;
+                err = rc;
+                ss.str("");
+            }
+        }
+        if((rc != -EEXIST) && (rc != -EINVAL)){
+            //propose pending would be generated by the PaxosService
+            goto update;
+        }
+        else {
+            goto reply_no_propose;
+        }
+    }
+    else if ( prefix == "nvme-gw show" ){
+        std::string  pool, group;
+        if (!f) {
+          f.reset(Formatter::create(format, "json-pretty", "json-pretty"));
+        }
+        cmd_getval(cmdmap, "pool", pool);
+        cmd_getval(cmdmap, "group", group);
+        auto group_key = std::make_pair(pool, group);
+        dout(4) <<"nvme-gw show  pool "<< pool << " group "<< group << dendl;
+
+        if( map.Created_gws[group_key].size()){
+            f->open_object_section("common");
+            f->dump_string("pool", pool);
+            f->dump_string("group", group);
+            f->dump_unsigned("num gws", map.Created_gws[group_key].size());
+            ss <<"[ ";
+            NvmeAnaGrpId anas[MAX_SUPPORTED_ANA_GROUPS];
+            int i = 0;
+            for (auto& gw_created_pair: map.Created_gws[group_key]) {
+                 auto& st = gw_created_pair.second;
+                 ss << st.ana_grp_id+1 << " ";
+                 anas[i++] = st.ana_grp_id;
+            }
+            ss << "]";
+            f->dump_string("Anagrp list", ss.str());
+            f->close_section();
+
+            for (auto& gw_created_pair: map.Created_gws[group_key]) {
+                 auto& gw_id = gw_created_pair.first;
+                 auto& state = gw_created_pair.second;
+                 f->open_object_section("stat");
+                 f->dump_string("gw-id", gw_id);
+                 f->dump_unsigned("anagrp-id",state.ana_grp_id+1);
+                 f->dump_unsigned("last-gw_map-epoch-valid",state.last_gw_map_epoch_valid);
+                 std::stringstream  ss1;
+                 ss1 << state.availability;
+                 f->dump_string("Availability", ss1.str());
+                 ss1.str("");
+                 for (size_t i = 0; i < map.Created_gws[group_key].size(); i++) {
+                         ss1 << " " << anas[i]+1 <<": " << state.sm_state[anas[i]] << ",";
+                 }
+                 f->dump_string("ana states", ss1.str());
+                 f->close_section();
+            }
+            f->flush(rdata);
+            ss.str("");
+        }
+        else {
+            ss << "num_gws  0";
+        }
+    }
+
+  reply_no_propose:
+    getline(ss, rs);
+    if (err < 0 && rs.length() == 0)
+    {
+        rs = cpp_strerror(err);
+        dout(1) << "Error command  err : "<< err  << " rs-len: " << rs.length() <<  dendl;
+    }
+    mon.reply_command(op, err, rs, rdata, get_last_committed());
+    return false; /* nothing to propose */
+
+  update:
+    getline(ss, rs);
+    wait_for_commit(op, new Monitor::C_Command(mon, op, 0, rs,
+                            get_last_committed() + 1));
+    return true;
+}
+
+
+bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op){
+    //dout(4)   << dendl;
+    auto m = op->get_req<MNVMeofGwBeacon>();
+    const BeaconSubsystems& sub = m->get_subsystems();
+     //mon.no_reply(op); // we never reply to beacons
+     dout(15) << "beacon from " << m->get_type() << " GW : " << m->get_gw_id()  << " num subsystems " << sub.size() <<  dendl;
+     MonSession *session = op->get_session();
+     if (!session){
+         dout(4) << "beacon no session "  << dendl;
+         return true;
+     }
+
+    return false; // allways  return false to call leader's prepare beacon
+}
+
+
+//#define BYPASS_GW_CREATE_CLI
+
+bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
+    //dout(4)  << dendl;
+    auto m = op->get_req<MNVMeofGwBeacon>();
+
+    dout(20) << "availability " <<  m->get_availability() << " GW : " << m->get_gw_id() <<
+        " osdmap_epoch " << m->get_last_osd_epoch() << " subsystems " << m->get_subsystems() << dendl;
+
+    NvmeGwId gw_id = m->get_gw_id();
+    NvmeGroupKey group_key = std::make_pair(m->get_gw_pool(),  m->get_gw_group());
+    GW_AVAILABILITY_E  avail = m->get_availability();
+    bool propose = false;
+    bool nonce_propose = false;
+    bool timer_propose = false;
+    bool gw_created = true;
+    NVMeofGwMap ack_map;
+    auto& group_gws = map.Created_gws[group_key];
+    auto gw = group_gws.find(gw_id);
+    const BeaconSubsystems& sub = m->get_subsystems();
+
+    if (avail == GW_AVAILABILITY_E::GW_CREATED){
+        if (gw == group_gws.end()) {
+           gw_created = false;
+           dout(4) << "Warning: GW " << gw_id << " group_key " << group_key << " was not found in the  map.Created_gws "<< map.Created_gws <<dendl;
+        }
+        goto set_propose;
+    }
+
+    // At this stage the gw has to be in the Created_gws
+    if(gw == group_gws.end()){
+        dout(1) << "Error : Administratively deleted GW sends beacon " << gw_id <<dendl;
+        goto false_return; // not sending ack to this beacon
+    }
+
+    // deep copy the whole nonce map of this GW
+    if(m->get_nonce_map().size()) {
+        if(pending_map.Created_gws[group_key][gw_id].nonce_map != m->get_nonce_map())
+        {
+            dout(4) << "nonce map of GW  changed , propose pending " << gw_id << dendl;
+            pending_map.Created_gws[group_key][gw_id].nonce_map = m->get_nonce_map();
+            dout(4) << "nonce map of GW " << gw_id << " "<< pending_map.Created_gws[group_key][gw_id].nonce_map  << dendl;
+            nonce_propose = true;
+        }
+    }
+    else  {
+        dout(4) << "Warning: received empty nonce map in the beacon of GW " << gw_id << " "<< dendl;
+    }
+
+    //pending_map.handle_removed_subsystems(gw_id, group_key, configured_subsystems, propose);
+
+    //if  no subsystem configured set gw as avail = GW_AVAILABILITY_E::GW_UNAVAILABLE
+
+    if(sub.size() == 0) {
+        avail = GW_AVAILABILITY_E::GW_UNAVAILABLE;
+    }
+    pending_map.Created_gws[group_key][gw_id].subsystems =  sub;
+    pending_map.Created_gws[group_key][gw_id].last_gw_map_epoch_valid = ( map.epoch == m->get_last_gwmap_epoch() );
+    if( pending_map.Created_gws[group_key][gw_id].last_gw_map_epoch_valid == false ){
+      dout(1) <<  "map epoch of gw is not up-to-date " << gw_id << " epoch " << map.epoch << " beacon_epoch " << m->get_last_gwmap_epoch() <<  dendl;
+    }
+    if(avail == GW_AVAILABILITY_E::GW_AVAILABLE)
+    {
+        //dout(4) <<"subsystems from beacon " << pending_map.Created_gws << dendl;
+        auto now = ceph::coarse_mono_clock::now();
+        // check pending_map.epoch vs m->get_version() - if different - drop the beacon
+
+        LastBeacon lb = {gw_id, group_key};
+        last_beacon[lb] = now;
+        epoch_t last_osd_epoch = m->get_last_osd_epoch();
+        pending_map.process_gw_map_ka(gw_id, group_key, last_osd_epoch, propose);
+    }
+    else if(avail == GW_AVAILABILITY_E::GW_UNAVAILABLE){ // state set by GW client application
+        //  TODO: remove from last_beacon if found . if gw was found in last_beacon call process_gw_map_gw_down
+
+        LastBeacon lb = {gw_id, group_key};
+
+        auto it = last_beacon.find(lb);
+        if (it != last_beacon.end()){
+            last_beacon.erase(lb);
+            pending_map.process_gw_map_gw_down(gw_id, group_key, propose);
+        }
+    }
+    pending_map.update_active_timers(timer_propose);  // Periodic: check active FSM timers
+    propose |= timer_propose;
+    propose |= nonce_propose;
+
+set_propose:
+    if(!propose) {
+      if(gw_created){
+          ack_map.Created_gws[group_key][gw_id] = map.Created_gws[group_key][gw_id];// respond with a map slice correspondent to the same GW
+      }
+      ack_map.epoch = map.epoch;
+      dout(20) << "ack_map " << ack_map <<dendl;
+      auto msg = make_message<MNVMeofGwMap>(ack_map);
+      mon.send_reply(op, msg.detach());
+    }
+false_return:
+    if (propose){
+      dout(4) << "decision in prepare_beacon" <<dendl;
+      return true;
+    }
+    else 
+     return false; // if no changes are need in the map
+}
diff --git a/src/mon/NVMeofGwMon.h b/src/mon/NVMeofGwMon.h
new file mode 100644
index 000000000000..33313e805d01
--- /dev/null
+++ b/src/mon/NVMeofGwMon.h
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef  MON_NVMEGWMONITOR_H_
+#define  MON_NVMEGWMONITOR_H_
+
+#include "PaxosService.h"
+#include "NVMeofGwMap.h"
+
+struct LastBeacon {
+    NvmeGwId gw_id;
+    NvmeGroupKey group_key;
+
+    // Comparison operators to allow usage as a map key
+    bool operator<(const LastBeacon& other) const {
+        if (gw_id != other.gw_id) return gw_id < other.gw_id;
+        return group_key < other.group_key;
+    }
+
+    bool operator==(const LastBeacon& other) const {
+        return gw_id == other.gw_id &&
+		group_key == other.group_key;
+    }
+};
+
+class NVMeofGwMon: public PaxosService,
+                   public md_config_obs_t
+{
+    NVMeofGwMap map;  //NVMeGWMap
+    NVMeofGwMap pending_map;
+
+    std::map<LastBeacon, ceph::coarse_mono_clock::time_point> last_beacon;
+
+    // when the mon was not updating us for some period (e.g. during slow
+    // election) to reset last_beacon timeouts
+    ceph::coarse_mono_clock::time_point last_tick;
+
+    std::vector<MonCommand> command_descs;
+    std::vector<MonCommand> pending_command_descs;
+
+public:
+    NVMeofGwMon(Monitor &mn, Paxos &p, const std::string& service_name): PaxosService(mn, p, service_name) {map.mon = &mn; last_leader = false;}
+    ~NVMeofGwMon() override {}
+
+
+    // config observer
+    const char** get_tracked_conf_keys() const override;
+    void handle_conf_change(const ConfigProxy& conf, const std::set<std::string> &changed) override;
+
+    // 3 pure virtual methods of the paxosService
+    void create_initial()override{};
+    void create_pending()override ;
+    void encode_pending(MonitorDBStore::TransactionRef t)override ;
+
+    void init() override;
+    void on_shutdown() override;
+    void on_restart() override;
+    void update_from_paxos(bool *need_bootstrap) override;
+
+
+    bool preprocess_query(MonOpRequestRef op) override;
+    bool prepare_update(MonOpRequestRef op) override;
+
+    bool preprocess_command(MonOpRequestRef op);
+    bool prepare_command(MonOpRequestRef op);
+
+    void encode_full(MonitorDBStore::TransactionRef t) override { }
+
+    bool preprocess_beacon(MonOpRequestRef op);
+    bool prepare_beacon(MonOpRequestRef op);
+
+    void tick() override;
+    void print_summary(ceph::Formatter *f, std::ostream *ss) const;
+
+    void check_subs(bool type);
+    void check_sub(Subscription *sub);
+
+private:
+    bool last_leader;
+    void synchronize_last_beacon();
+
+};
+
+#endif /* MON_NVMEGWMONITOR_H_ */
diff --git a/src/mon/NVMeofGwSerialize.h b/src/mon/NVMeofGwSerialize.h
new file mode 100755
index 000000000000..0c8414e6b7f5
--- /dev/null
+++ b/src/mon/NVMeofGwSerialize.h
@@ -0,0 +1,602 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+#ifndef MON_NVMEOFGWSERIALIZE_H_
+#define MON_NVMEOFGWSERIALIZE_H_
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define MODULE_PREFFIX "nvmeofgw "
+#define dout_prefix *_dout << MODULE_PREFFIX << __PRETTY_FUNCTION__ << " "
+
+inline std::ostream& operator<<(std::ostream& os, const GW_EXPORTED_STATES_PER_AGROUP_E value) {
+    switch (value) {
+        case GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_OPTIMIZED_STATE: os << "OPTIMIZED "; break;
+        case GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_INACCESSIBLE_STATE: os << "INACCESSIBLE "; break;
+        default: os << "Invalid " << (int)value << " ";
+    }
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const GW_STATES_PER_AGROUP_E value) {
+    switch (value) {
+        case GW_STATES_PER_AGROUP_E::GW_IDLE_STATE:                  os << "IDLE "; break;
+        case GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE:               os << "STANDBY "; break;
+        case GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE:                os << "ACTIVE "; break;
+        case GW_STATES_PER_AGROUP_E::GW_OWNER_WAIT_FAILBACK_PREPARED: os << "OWNER_FAILBACK_PREPARED "; break;
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED:      os << "WAIT_FAILBACK_PREPARED "; break;
+        case GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL:       os <<   "WAIT_BLOCKLIST_CMPL "; break;
+        default: os << "Invalid " << (int)value << " ";
+    }
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const GW_AVAILABILITY_E value) {
+    switch (value) {
+
+        case GW_AVAILABILITY_E::GW_CREATED: os << "CREATED"; break;
+        case GW_AVAILABILITY_E::GW_AVAILABLE: os << "AVAILABLE"; break;
+        case GW_AVAILABILITY_E::GW_UNAVAILABLE: os << "UNAVAILABLE"; break;
+
+        default: os << "Invalid " << (int)value << " ";
+    }
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const SM_STATE value) {
+    os << "SM_STATE [ ";
+    for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) os << value[i];
+    os << "]";
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const BeaconNamespace value) {
+    os << "BeaconNamespace( anagrpid:" << value.anagrpid << ", nonce:" << value.nonce <<" )";
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const BeaconListener value) {
+    os << "BeaconListener( addrfam:" << value.address_family
+                        << ", addr:" << value.address
+                        << ", svcid:" << value.svcid << " )";
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const BeaconSubsystem value) {
+    os << "BeaconSubsystem( nqn:" << value.nqn << ", listeners [ ";
+    for (const auto& list: value.listeners) os << list << " ";
+    os << "] namespaces [ ";
+    for (const auto& ns: value.namespaces) os << ns << " ";
+    os << "] )";
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NqnState value) {
+    os << "NqnState( nqn: " << value.nqn << ", " << value.ana_state << " )";
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeGwState value) {
+    os <<  "NvmeGwState { group id: " << value.group_id <<  " gw_map_epoch " <<  value.gw_map_epoch
+        << " GwSubsystems: [ ";
+    for (const auto& sub: value.subsystems) os << sub.second << " ";
+    os << " ] }";
+
+    return os;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeGroupKey value) {
+    os << "NvmeGroupKey {" << value.first << "," << value.second << "}";
+    return os;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeGwMap value) {
+    os << "NvmeGwMap ";
+    for (auto& gw_state: value) {
+        os << "\n" << MODULE_PREFFIX <<" { == gw_id: " << gw_state.first << " -> " <<  gw_state.second << "}";
+    }
+    os << "}";
+
+    return os;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeNonceVector value) {
+    for (auto & nonces : value) {
+        os <<  nonces << " ";
+    }
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeAnaNonceMap value) {
+    if(value.size()) os << "\n" << MODULE_PREFFIX;
+    for (auto &nonce_map : value) {
+        os  << "  ana_grp: " << nonce_map.first  << " [ " << nonce_map.second << "]\n"<< MODULE_PREFFIX ;
+    }
+    return os;
+}
+
+inline std::ostream& print_gw_created_t(std::ostream& os, const NvmeGwCreated value, size_t num_ana_groups, NvmeAnaGrpId *anas) {
+    os << "==Internal map ==NvmeGwCreated { ana_group_id " << value.ana_grp_id << " osd_epochs: ";
+    for(size_t i = 0; i < num_ana_groups; i ++){
+        os << " " << anas[i] <<": " << value.blocklist_data[anas[i]].osd_epoch << ":" <<value.blocklist_data[anas[i]].is_failover ;
+    }
+    os << "\n" << MODULE_PREFFIX << "nonces: " << value.nonce_map << " }";
+    os << "\n" << MODULE_PREFFIX << "saved-nonces: " << value.copied_nonce_map << " }";
+    for (size_t i = 0; i < num_ana_groups; i++) {
+        os << " " << anas[i] <<": " << value.sm_state[anas[i]] << ",";
+    }
+    os <<  "]\n"<< MODULE_PREFFIX << " failover peers ";
+    for (size_t i = 0; i < num_ana_groups; i++) {
+        os << anas[i] <<": "  << value.failover_peer[anas[i]] << ",";
+    }
+    os << "]\n"<< MODULE_PREFFIX << "availability " << value.availability << "]";
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeGwCreated value) {
+    os << "==Internal map ==G W_CREATED_T { ana_group_id " << value.ana_grp_id << " osd_epochs: ";
+    for(int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i ++){
+        os << " " << value.blocklist_data[i].osd_epoch;
+    }
+    os << "\n" << MODULE_PREFFIX << "nonces: " << value.nonce_map << " }";
+
+    for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) {
+        os << value.sm_state[i] << ",";
+    }
+    os <<  "]\n"<< MODULE_PREFFIX << " failover peers ";
+    for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) {
+        os << value.failover_peer[i] << ",";
+    }
+    os <<  "]\n"<< MODULE_PREFFIX << " beacon-subsystems ";
+    for (const auto& sub: value.subsystems) {
+        os << sub << ",";
+    }
+
+    os << "]\n"<< MODULE_PREFFIX << "availability " << value.availability << "]";
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NvmeGwCreatedMap value) {
+    if(value.size()) os << "\n" << MODULE_PREFFIX;;
+    NvmeAnaGrpId anas[MAX_SUPPORTED_ANA_GROUPS];
+    int i = 0;
+    for(auto &it: value ){
+       anas[i++] = it.second.ana_grp_id; // effective ana groups for these GWs within group_pool
+    }
+    for (auto &gw_created_map : value) {
+        os  <<  "gw_id: " << gw_created_map.first  << " [ " ;//<< gw_created_map.second << "] \n"<< MODULE_PREFFIX;
+        print_gw_created_t(os, gw_created_map.second,  value.size(), anas);
+        os << "] \n"<< MODULE_PREFFIX;
+    }
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NVMeofGwMap value) {
+    os << "NVMeofGwMap [ Created_gws: ";
+    for (auto& group_gws: value.Created_gws) {
+        os <<  "\n" <<  MODULE_PREFFIX  << "{ " << group_gws.first << " } -> { " << group_gws.second << " }";
+    }
+    os << "]";
+    return os;
+}
+
+inline void encode(const ANA_STATE& st,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode((uint32_t)st.size(), bl);
+    for (const auto& gr: st){
+        encode((uint32_t)gr.first, bl);
+        encode((uint32_t)gr.second, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(ANA_STATE& st, ceph::buffer::list::const_iterator &bl) {
+    uint32_t n;
+    DECODE_START(1, bl);
+    decode(n, bl);
+    st.resize(n);
+    for (uint32_t i = 0; i < n; i++) {
+        uint32_t a, b;
+        decode(a, bl);
+        decode(b, bl);
+        st[i].first  = (GW_EXPORTED_STATES_PER_AGROUP_E)a;
+        st[i].second = (epoch_t)b;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const GwSubsystems& subsystems,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode((uint32_t)subsystems.size(), bl);
+    for (const auto& sub: subsystems){
+        encode(sub.second.nqn, bl);
+        encode(sub.second.ana_state, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline  void decode(GwSubsystems& subsystems,  ceph::bufferlist::const_iterator& bl) {
+  uint32_t num_subsystems;
+  DECODE_START(1, bl);
+  decode(num_subsystems, bl);
+  subsystems.clear();
+  for (uint32_t i=0; i<num_subsystems; i++){
+     std::string  nqn;
+     decode(nqn, bl);
+     ANA_STATE st;
+     decode(st, bl);
+     subsystems.insert({nqn, NqnState(nqn, st)});
+  }
+  DECODE_FINISH(bl);
+}
+
+inline void encode(const NvmeGwState& state,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode(state.group_id, bl);
+    encode(state.gw_map_epoch, bl);
+    encode (state.subsystems, bl);
+    ENCODE_FINISH(bl);
+}
+
+inline  void decode(NvmeGwState& state,  ceph::bufferlist::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(state.group_id, bl);
+    decode(state.gw_map_epoch, bl);
+    decode(state.subsystems, bl);
+    DECODE_FINISH(bl);
+}
+
+inline  void encode(const NvmeGwMetaData& state,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode((uint32_t)MAX_SUPPORTED_ANA_GROUPS, bl);
+    for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+        int    tick = state.data[i].timer_started;
+        uint8_t val = state.data[i].timer_value;
+        encode(tick, bl);
+        encode(val,  bl);
+        auto endtime = state.data[i].end_time;
+        // Convert the time point to milliseconds since the epoch
+        uint64_t  millisecondsSinceEpoch = std::chrono::duration_cast<std::chrono::milliseconds>(endtime.time_since_epoch()).count();
+        encode(millisecondsSinceEpoch , bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline  void decode(NvmeGwMetaData& state,  ceph::bufferlist::const_iterator& bl) {
+    uint32_t s;
+      
+    DECODE_START(1, bl);
+    decode(s, bl);
+    ceph_assert(s == (uint32_t)MAX_SUPPORTED_ANA_GROUPS);
+    for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+        int tick;
+        uint8_t val;
+        decode(tick, bl);
+        decode(val,  bl);
+        state.data[i].timer_started = tick;
+        state.data[i].timer_value = val;
+        uint64_t milliseconds;
+        decode(milliseconds, bl);
+        auto duration = std::chrono::milliseconds(milliseconds);
+        state.data[i].end_time = std::chrono::time_point<std::chrono::system_clock>(duration);
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const NvmeAnaNonceMap& nonce_map,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode((uint32_t)nonce_map.size(), bl);
+    for (auto& ana_group_nonces : nonce_map) {
+        encode(ana_group_nonces.first, bl); // ana group id
+        encode ((uint32_t)ana_group_nonces.second.size(), bl); // encode the vector size
+        for (auto& nonce: ana_group_nonces.second) encode(nonce, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(NvmeAnaNonceMap& nonce_map, ceph::buffer::list::const_iterator &bl) {
+    uint32_t map_size;
+    NvmeAnaGrpId ana_grp_id;
+    uint32_t vector_size;
+    std::string nonce;
+    DECODE_START(1, bl);
+    decode(map_size, bl);
+    for(uint32_t i = 0; i<map_size; i++){
+        decode(ana_grp_id, bl);
+        decode(vector_size,bl);
+        for(uint32_t j = 0; j < vector_size; j++){
+            decode (nonce, bl);
+            nonce_map[ana_grp_id].push_back(nonce);
+        }
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const NvmeGwCreatedMap& gws,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode ((uint32_t)gws.size(), bl); // number of gws in the group
+    for(auto& gw : gws){
+        encode(gw.first, bl);// GW_id
+        encode(gw.second.ana_grp_id, bl); // GW owns this group-id
+
+        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+            encode((uint32_t)(gw.second.sm_state[i]), bl);
+        }
+        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+            encode((gw.second.failover_peer[i]), bl);
+        }
+        encode((uint32_t)gw.second.availability, bl);
+        encode((uint32_t)gw.second.last_gw_map_epoch_valid, bl);
+        encode(gw.second.subsystems, bl);
+
+        for(int i=0; i< MAX_SUPPORTED_ANA_GROUPS; i++){
+            encode(gw.second.blocklist_data[i].osd_epoch, bl);
+            encode(gw.second.blocklist_data[i].is_failover, bl);
+        }
+        encode(gw.second.nonce_map, bl);
+        encode(gw.second.copied_nonce_map, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(NvmeGwCreatedMap& gws, ceph::buffer::list::const_iterator &bl) {
+    gws.clear();
+    uint32_t num_created_gws;
+    DECODE_START(1, bl);
+    decode(num_created_gws, bl);
+
+    for(uint32_t i = 0; i<num_created_gws; i++){
+        std::string gw_name;
+        decode(gw_name, bl);
+        NvmeAnaGrpId ana_grp_id;
+        decode(ana_grp_id, bl);
+
+        NvmeGwCreated gw_created(ana_grp_id);
+        uint32_t sm_state;
+        NvmeGwId peer_name;
+        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+            decode(sm_state, bl);
+            gw_created.sm_state[i] = (GW_STATES_PER_AGROUP_E)  sm_state;
+        }
+        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
+            decode(peer_name, bl);
+            gw_created.failover_peer[i] = peer_name;
+        }
+        uint32_t avail;
+        decode(avail, bl);
+        gw_created.availability = (GW_AVAILABILITY_E)avail;
+        uint32_t gwmap_epoch;
+        decode(gwmap_epoch, bl);
+        gw_created.last_gw_map_epoch_valid = (bool)gwmap_epoch;
+        BeaconSubsystems   subsystems;
+        decode(subsystems, bl);
+        gw_created.subsystems = subsystems;
+
+        for(int i=0; i< MAX_SUPPORTED_ANA_GROUPS; i++){
+            decode(gw_created.blocklist_data[i].osd_epoch, bl);
+            decode(gw_created.blocklist_data[i].is_failover, bl);
+        }
+        decode(gw_created.nonce_map, bl);
+        decode(gw_created.copied_nonce_map, bl);
+
+        gws[gw_name] = gw_created;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const std::map<NvmeGroupKey, NvmeGwCreatedMap>& created_gws,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode ((uint32_t)created_gws.size(), bl); // number of groups
+    for (auto& group_gws: created_gws) {
+        auto& group_key = group_gws.first;
+        encode(group_key.first, bl); // pool
+        encode(group_key.second, bl); // group
+
+        auto& gws = group_gws.second;
+        encode (gws, bl); // encode group gws
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(std::map<NvmeGroupKey, NvmeGwCreatedMap>& created_gws, ceph::buffer::list::const_iterator &bl) {
+    created_gws.clear();
+    uint32_t ngroups;
+    DECODE_START(1, bl);
+    decode(ngroups, bl);
+    for(uint32_t i = 0; i<ngroups; i++){
+        std::string pool, group;
+        decode(pool, bl);
+        decode(group, bl);
+        NvmeGwCreatedMap cmap;
+        decode(cmap, bl);
+        created_gws[std::make_pair(pool, group)] = cmap;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const NvmeGwMap& subsyst_gwmap,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode((uint32_t)subsyst_gwmap.size(), bl);
+    for (auto& subsyst: subsyst_gwmap) {
+        encode(subsyst.first, bl);
+        encode(subsyst.second, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(NvmeGwMap& subsyst_gwmap, ceph::buffer::list::const_iterator &bl) {
+    subsyst_gwmap.clear();
+    uint32_t num_gws;
+    DECODE_START(1, bl);
+    decode(num_gws, bl);
+
+    for (uint32_t i = 0; i < num_gws; i++) {
+        NvmeGwId gw_id;
+        decode(gw_id, bl);
+        NvmeGwState gw_st;
+        decode(gw_st, bl);
+        subsyst_gwmap[gw_id] = gw_st;
+    }
+    DECODE_FINISH(bl);
+}
+
+// Start encode  NvmeGroupKey, GMAP
+inline void encode(const std::map<NvmeGroupKey, NvmeGwMap>& gmap,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode ((uint32_t)gmap.size(), bl); // number of groups
+    for (auto& group_state: gmap) {
+        auto& group_key = group_state.first;
+        encode(group_key.first, bl); // pool
+        encode(group_key.second, bl); // group
+        encode(group_state.second, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+// Start decode NvmeGroupKey, NvmeGwMap
+inline void decode(std::map<NvmeGroupKey, NvmeGwMap>& gmap, ceph::buffer::list::const_iterator &bl) {
+    gmap.clear();
+    uint32_t ngroups;
+    DECODE_START(1, bl);
+    decode(ngroups, bl);
+    for(uint32_t i = 0; i<ngroups; i++){
+        std::string pool, group;
+        decode(pool, bl);
+        decode(group, bl);
+        NvmeGwMap grp_map;
+        decode(grp_map, bl);
+        gmap[std::make_pair(pool, group)] = grp_map;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const std::map<NvmeGroupKey, NvmeGwMetaDataMap>& gmetadata,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode ((uint32_t)gmetadata.size(), bl); // number of groups
+    for (auto& group_md: gmetadata) {
+        auto& group_key = group_md.first;
+        encode(group_key.first, bl); // pool
+        encode(group_key.second, bl); // group
+
+        encode(group_md.second, bl);
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(std::map<NvmeGroupKey, NvmeGwMetaDataMap>& gmetadata, ceph::buffer::list::const_iterator &bl) {
+    gmetadata.clear();
+    uint32_t ngroups;
+    DECODE_START(1, bl);
+    decode(ngroups, bl);
+    for(uint32_t i = 0; i<ngroups; i++){
+        std::string pool, group;
+        decode(pool, bl);
+        decode(group, bl);
+        NvmeGwMetaDataMap gmd;
+        decode(gmd, bl);
+        gmetadata[std::make_pair(pool, group)] = gmd;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const NvmeGwMetaDataMap& group_md,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode ((uint32_t)group_md.size(), bl); // number of groups
+    for (auto& gw_md: group_md) {
+        encode(gw_md.first, bl); // gw
+        encode(gw_md.second, bl); //  map of this gw
+    }
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(NvmeGwMetaDataMap& md, ceph::buffer::list::const_iterator &bl) {
+    uint32_t num_gws;
+    DECODE_START(1, bl);
+    decode(num_gws, bl);
+    for (uint32_t i = 0; i < num_gws; i++) {
+        std::string gw_id;
+        decode(gw_id, bl);
+        NvmeGwMetaData gw_meta;
+        decode(gw_meta, bl);
+        md[gw_id] = gw_meta;
+    }
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const BeaconNamespace& ns,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode(ns.anagrpid, bl);
+    encode(ns.nonce, bl);
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(BeaconNamespace& ns, ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(1, bl);
+    decode(ns.anagrpid, bl);
+    decode(ns.nonce, bl);
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const BeaconListener& ls,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode(ls.address_family, bl);
+    encode(ls.address, bl);
+    encode(ls.svcid, bl);
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(BeaconListener& ls, ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(1, bl);
+    decode(ls.address_family, bl);
+    decode(ls.address, bl);
+    decode(ls.svcid, bl);
+    DECODE_FINISH(bl);
+}
+
+inline void encode(const BeaconSubsystem& sub,  ceph::bufferlist &bl) {
+    ENCODE_START(1, 1, bl);
+    encode(sub.nqn, bl);
+    encode((uint32_t)sub.listeners.size(), bl);
+    for (const auto& ls: sub.listeners)
+        encode(ls, bl);
+    encode((uint32_t)sub.namespaces.size(), bl);
+    for (const auto& ns: sub.namespaces)
+        encode(ns, bl);
+    ENCODE_FINISH(bl);
+}
+
+inline void decode(BeaconSubsystem& sub, ceph::buffer::list::const_iterator &bl) {
+    DECODE_START(1, bl);
+    decode(sub.nqn, bl);
+    uint32_t s;
+    sub.listeners.clear();
+    decode(s, bl);
+    for (uint32_t i = 0; i < s; i++) {
+        BeaconListener ls;
+        decode(ls, bl);
+        sub.listeners.push_back(ls);
+    }
+
+    sub.namespaces.clear();
+    decode(s, bl);
+    for (uint32_t i = 0; i < s; i++) {
+        BeaconNamespace ns;
+        decode(ns, bl);
+        sub.namespaces.push_back(ns);
+    }
+    DECODE_FINISH(bl);
+}
+
+
+#undef dout_subsys
+#endif /* SRC_MON_NVMEOFGWSERIALIZEP_H_ */
diff --git a/src/mon/NVMeofGwTypes.h b/src/mon/NVMeofGwTypes.h
new file mode 100755
index 000000000000..5eb1a3ba622b
--- /dev/null
+++ b/src/mon/NVMeofGwTypes.h
@@ -0,0 +1,167 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#ifndef MON_NVMEOFGWTYPES_H_
+#define MON_NVMEOFGWTYPES_H_
+#include <string>
+#include <iomanip>
+#include <map>
+#include <iostream>
+
+using NvmeGwId      = std::string;
+using NvmeGroupKey  = std::pair<std::string, std::string>;
+using NvmeNqnId     = std::string;
+using NvmeAnaGrpId  = uint32_t;
+
+
+enum class GW_STATES_PER_AGROUP_E {
+    GW_IDLE_STATE = 0, //invalid state
+    GW_STANDBY_STATE,
+    GW_ACTIVE_STATE,
+    GW_OWNER_WAIT_FAILBACK_PREPARED,
+    GW_WAIT_FAILBACK_PREPARED,
+    GW_WAIT_BLOCKLIST_CMPL
+};
+
+enum class GW_EXPORTED_STATES_PER_AGROUP_E {
+    GW_EXPORTED_OPTIMIZED_STATE = 0,
+    GW_EXPORTED_INACCESSIBLE_STATE
+};
+
+enum class GW_AVAILABILITY_E {
+    GW_CREATED = 0,
+    GW_AVAILABLE,
+    GW_UNAVAILABLE,
+    GW_DELETED
+};
+
+#define MAX_SUPPORTED_ANA_GROUPS 16
+#define REDUNDANT_GW_ANA_GROUP_ID 0xFF
+
+typedef GW_STATES_PER_AGROUP_E          SM_STATE         [MAX_SUPPORTED_ANA_GROUPS];
+
+using ANA_STATE = std::vector<std::pair<GW_EXPORTED_STATES_PER_AGROUP_E, epoch_t>>;
+
+struct BeaconNamespace {
+    NvmeAnaGrpId anagrpid;
+    std::string  nonce;
+};
+
+struct BeaconListener {
+    std::string address_family; // IPv4 or IPv6
+    std::string address;        //
+    std::string svcid;          // port
+};
+
+struct BeaconSubsystem {
+    NvmeNqnId nqn;
+    std::list<BeaconListener>  listeners;
+    std::list<BeaconNamespace> namespaces;
+};
+
+using BeaconSubsystems = std::list<BeaconSubsystem>;
+
+using NvmeNonceVector    = std::vector<std::string>;
+using NvmeAnaNonceMap  = std::map <NvmeAnaGrpId, NvmeNonceVector>;
+
+struct NvmeGwCreated {
+    NvmeAnaGrpId       ana_grp_id;                    // ana-group-id allocated for this GW, GW owns this group-id
+    GW_AVAILABILITY_E  availability;                  // in absence of  beacon  heartbeat messages it becomes inavailable
+    bool               last_gw_map_epoch_valid;       // "true" if the last epoch seen by the gw-client is up-to-date
+    BeaconSubsystems   subsystems;                    // gateway susbsystem and their state machine states
+    NvmeAnaNonceMap    nonce_map;
+    NvmeAnaNonceMap    copied_nonce_map;
+    SM_STATE           sm_state;                      // state machine states per ANA group
+    NvmeGwId           failover_peer[MAX_SUPPORTED_ANA_GROUPS];
+    struct{
+       epoch_t     osd_epoch;
+       bool        is_failover;
+    }blocklist_data[MAX_SUPPORTED_ANA_GROUPS];
+
+    NvmeGwCreated(): ana_grp_id(REDUNDANT_GW_ANA_GROUP_ID) {};
+
+    NvmeGwCreated(NvmeAnaGrpId id): ana_grp_id(id), availability(GW_AVAILABILITY_E::GW_CREATED), last_gw_map_epoch_valid(false)
+    {
+        for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++){
+            sm_state[i] = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
+            failover_peer[i]  = "";
+            blocklist_data[i].osd_epoch = 0;
+            blocklist_data[i].is_failover = true;
+        }
+    };
+
+    void standby_state(NvmeAnaGrpId grpid) {
+           sm_state[grpid]       = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
+           failover_peer[grpid]  = "";
+    };
+    void active_state(NvmeAnaGrpId grpid) {
+           sm_state[grpid]       = GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE;
+           blocklist_data[grpid].osd_epoch = 0;
+    };
+};
+
+struct NqnState {
+    std::string   nqn;          // subsystem NQN
+    ANA_STATE     ana_state;    // subsystem's ANA state
+
+    // constructors
+    NqnState(const std::string& _nqn, const ANA_STATE& _ana_state):
+        nqn(_nqn), ana_state(_ana_state)  {}
+    NqnState(const std::string& _nqn, const SM_STATE& sm_state, const NvmeGwCreated & gw_created) : nqn(_nqn)  {
+        for (int i=0; i < MAX_SUPPORTED_ANA_GROUPS; i++){
+            std::pair<GW_EXPORTED_STATES_PER_AGROUP_E, epoch_t> state_pair;
+            state_pair.first = (  sm_state[i] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE
+			       || sm_state[i] == GW_STATES_PER_AGROUP_E::GW_WAIT_BLOCKLIST_CMPL)
+                           ? GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_OPTIMIZED_STATE
+                           : GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_INACCESSIBLE_STATE;
+            state_pair.second = gw_created.blocklist_data[i].osd_epoch;
+            ana_state.push_back(state_pair);
+        }
+    }
+};
+
+typedef std::map<NvmeNqnId, NqnState> GwSubsystems;
+
+struct NvmeGwState {
+    NvmeAnaGrpId              group_id;
+    epoch_t                   gw_map_epoch;
+    GwSubsystems              subsystems;
+
+    NvmeGwState(NvmeAnaGrpId id, epoch_t epoch):
+        group_id(id),
+        gw_map_epoch(epoch)
+    {};
+
+    NvmeGwState() : NvmeGwState(REDUNDANT_GW_ANA_GROUP_ID, 0) {};
+};
+
+struct NvmeGwMetaData {
+   struct{
+      uint32_t     timer_started; // statemachine timer(timestamp) set in some state
+      uint8_t      timer_value;
+      std::chrono::system_clock::time_point end_time;
+   } data[MAX_SUPPORTED_ANA_GROUPS];
+
+    NvmeGwMetaData() {
+        for (int i=0; i<MAX_SUPPORTED_ANA_GROUPS; i++){
+            data[i].timer_started = 0;
+            data[i].timer_value = 0;
+        }
+    };
+};
+
+using NvmeGwMap             = std::map<NvmeGwId, NvmeGwState>;
+using NvmeGwMetaDataMap     = std::map<NvmeGwId, NvmeGwMetaData>;
+using NvmeGwCreatedMap      = std::map<NvmeGwId, NvmeGwCreated>;
+
+#endif /* SRC_MON_NVMEOFGWTYPES_H_ */
diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index 3429a8e99916..9dd2797852d4 100644
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -36,6 +36,7 @@ enum {
   PAXOS_HEALTH,
   PAXOS_CONFIG,
   PAXOS_KV,
+  PAXOS_NVMEGW,
   PAXOS_NUM
 };
 
diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 22208d2d1f42..f649e0f3d3ee 100644
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -219,6 +219,9 @@
 #include "messages/MOSDPGUpdateLogMissing.h"
 #include "messages/MOSDPGUpdateLogMissingReply.h"
 
+#include "messages/MNVMeofGwBeacon.h"
+#include "messages/MNVMeofGwMap.h"
+
 #ifdef WITH_BLKIN
 #include "Messenger.h"
 #endif
@@ -885,6 +888,10 @@ Message *decode_message(CephContext *cct,
     m = make_message<MMgrBeacon>();
     break;
 
+  case MSG_MNVMEOF_GW_BEACON:
+    m = make_message<MNVMeofGwBeacon>();
+  break;
+
   case MSG_MON_MGR_REPORT:
     m = make_message<MMonMgrReport>();
     break;
@@ -944,6 +951,9 @@ Message *decode_message(CephContext *cct,
     m = make_message<MMonHealthChecks>();
     break;
 
+  case MSG_MNVMEOF_GW_MAP:
+    m = make_message<MNVMeofGwMap>();
+    break;
     // -- simple messages without payload --
 
   case CEPH_MSG_SHUTDOWN:
diff --git a/src/msg/Message.h b/src/msg/Message.h
index 15eb3feadced..4a01175f46b0 100644
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -230,7 +230,7 @@
 #define MSG_MGR_MAP               0x704
 
 // *** ceph-mon(MgrMonitor) -> ceph-mgr
-#define MSG_MGR_DIGEST               0x705
+#define MSG_MGR_DIGEST            0x705
 // *** cephmgr -> ceph-mon
 #define MSG_MON_MGR_REPORT        0x706
 #define MSG_SERVICE_MAP           0x707
@@ -240,7 +240,13 @@
 #define MSG_MGR_COMMAND_REPLY     0x70a
 
 // *** ceph-mgr <-> MON daemons ***
-#define MSG_MGR_UPDATE     0x70b
+#define MSG_MGR_UPDATE            0x70b
+
+// *** nvmeof mon -> gw daemons ***
+#define MSG_MNVMEOF_GW_MAP        0x800
+
+// *** gw daemons -> nvmeof mon  ***
+#define MSG_MNVMEOF_GW_BEACON     0x801
 
 // ======================================================
 
diff --git a/src/nvmeof/NVMeofGwClient.cc b/src/nvmeof/NVMeofGwClient.cc
new file mode 100644
index 000000000000..c82423de5158
--- /dev/null
+++ b/src/nvmeof/NVMeofGwClient.cc
@@ -0,0 +1,32 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "NVMeofGwClient.h"
+
+bool NVMeofGwClient::get_subsystems(subsystems_info& reply) {
+  get_subsystems_req request;
+  ClientContext context;
+
+  Status status = stub_->get_subsystems(&context, request, &reply);
+
+  return status.ok();
+}
+
+bool NVMeofGwClient::set_ana_state(const ana_info& info) {
+  req_status reply;
+  ClientContext context;
+
+  Status status = stub_->set_ana_state(&context, info, &reply);
+
+  return status.ok() && reply.status();
+}
diff --git a/src/nvmeof/NVMeofGwClient.h b/src/nvmeof/NVMeofGwClient.h
new file mode 100644
index 000000000000..022485251d6b
--- /dev/null
+++ b/src/nvmeof/NVMeofGwClient.h
@@ -0,0 +1,40 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+
+#ifndef  __NVMEOFGWCLIENT_H__
+#define  __NVMEOFGWCLIENT_H__
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include <grpcpp/grpcpp.h>
+
+#include "gateway.grpc.pb.h"
+
+using grpc::Channel;
+using grpc::ClientContext;
+using grpc::Status;
+
+class NVMeofGwClient {
+ public:
+  NVMeofGwClient(std::shared_ptr<Channel> channel)
+      : stub_(Gateway::NewStub(channel)) {}
+
+  bool get_subsystems(subsystems_info& reply);
+  bool set_ana_state(const ana_info& info);
+
+ private:
+  std::unique_ptr<Gateway::Stub> stub_;
+};
+#endif
diff --git a/src/nvmeof/NVMeofGwMonitorClient.cc b/src/nvmeof/NVMeofGwMonitorClient.cc
new file mode 100644
index 000000000000..28330e432eaf
--- /dev/null
+++ b/src/nvmeof/NVMeofGwMonitorClient.cc
@@ -0,0 +1,384 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023,2024 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include <boost/algorithm/string/replace.hpp>
+
+#include "common/errno.h"
+#include "common/signal.h"
+#include "common/ceph_argparse.h"
+#include "include/compat.h"
+
+#include "include/stringify.h"
+#include "global/global_context.h"
+#include "global/signal_handler.h"
+
+
+#include "messages/MNVMeofGwBeacon.h"
+#include "messages/MNVMeofGwMap.h"
+#include "NVMeofGwMonitorClient.h"
+#include "NVMeofGwClient.h"
+#include "NVMeofGwMonitorGroupClient.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix *_dout << "nvmeofgw " << __PRETTY_FUNCTION__ << " "
+
+NVMeofGwMonitorClient::NVMeofGwMonitorClient(int argc, const char **argv) :
+  Dispatcher(g_ceph_context),
+  osdmap_epoch(0),
+  gwmap_epoch(0),
+  monc{g_ceph_context, poolctx},
+  client_messenger(Messenger::create(g_ceph_context, "async", entity_name_t::CLIENT(-1), "client", getpid())),
+  objecter{g_ceph_context, client_messenger.get(), &monc, poolctx},
+  client{client_messenger.get(), &monc, &objecter},
+  timer(g_ceph_context, lock),
+  orig_argc(argc),
+  orig_argv(argv)
+{
+}
+
+NVMeofGwMonitorClient::~NVMeofGwMonitorClient() = default;
+
+const char** NVMeofGwMonitorClient::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    NULL
+  };
+  return KEYS;
+}
+
+int NVMeofGwMonitorClient::init()
+{
+  dout(0) << dendl;
+  std::string val;
+  auto args = argv_to_vec(orig_argc, orig_argv);
+
+  for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) {
+    if (ceph_argparse_double_dash(args, i)) {
+      break;
+    } else if (ceph_argparse_witharg(args, i, &val, "--gateway-name", (char*)NULL)) {
+      name = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--gateway-pool", (char*)NULL)) {
+      pool = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--gateway-group", (char*)NULL)) {
+      group = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--gateway-address", (char*)NULL)) {
+      gateway_address = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--monitor-group-address", (char*)NULL)) {
+      monitor_address = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--server-key", (char*)NULL)) {
+      server_key = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--server-cert", (char*)NULL)) {
+      server_cert = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--client-cert", (char*)NULL)) {
+      client_cert = val;
+    } else {
+      ++i;
+    }
+  }
+
+  dout(0) << "gateway name: " << name <<
+    " pool:" << pool <<
+    " group:" << group <<
+    " address: " << gateway_address << dendl;
+  ceph_assert(name != "" && pool != "" && gateway_address != "" && monitor_address != "");
+
+  // todo
+  ceph_assert(server_key == "" && server_cert == "" && client_cert == "");
+
+  init_async_signal_handler();
+  register_async_signal_handler(SIGHUP, sighup_handler);
+
+  std::lock_guard l(lock);
+
+  // Initialize Messenger
+  client_messenger->add_dispatcher_tail(this);
+  client_messenger->add_dispatcher_head(&objecter);
+  client_messenger->add_dispatcher_tail(&client);
+  client_messenger->start();
+
+  poolctx.start(2);
+
+  // Initialize MonClient
+  if (monc.build_initial_monmap() < 0) {
+    client_messenger->shutdown();
+    client_messenger->wait();
+    return -1;
+  }
+
+  monc.sub_want("NVMeofGw", 0, 0);
+  monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD
+      |CEPH_ENTITY_TYPE_MDS|CEPH_ENTITY_TYPE_MGR);
+  monc.set_messenger(client_messenger.get());
+
+  // We must register our config callback before calling init(), so
+  // that we see the initial configuration message
+  monc.register_config_callback([this](const std::string &k, const std::string &v){
+      // leaving this for debugging purposes
+      dout(10) << "nvmeof config_callback: " << k << " : " << v << dendl;
+      
+      return false;
+    });
+  monc.register_config_notify_callback([this]() {
+      dout(4) << "nvmeof monc config notify callback" << dendl;
+    });
+  dout(4) << "nvmeof Registered monc callback" << dendl;
+
+  int r = monc.init();
+  if (r < 0) {
+    monc.shutdown();
+    client_messenger->shutdown();
+    client_messenger->wait();
+    return r;
+  }
+  dout(0) << "nvmeof Registered monc callback" << dendl;
+
+  r = monc.authenticate();
+  if (r < 0) {
+    derr << "Authentication failed, did you specify an ID with a valid keyring?" << dendl;
+    monc.shutdown();
+    client_messenger->shutdown();
+    client_messenger->wait();
+    return r;
+  }
+  dout(0) << "monc.authentication done" << dendl;
+  monc.set_passthrough_monmap();
+
+  client_t whoami = monc.get_global_id();
+  client_messenger->set_myname(entity_name_t::MGR(whoami.v));
+  objecter.set_client_incarnation(0);
+  objecter.init();
+  objecter.enable_blocklist_events();
+  objecter.start();
+  client.init();
+  timer.init();
+
+  tick();
+
+  dout(0) << "Complete." << dendl;
+  return 0;
+}
+
+static bool get_gw_state(const char* desc, const std::map<NvmeGroupKey, NvmeGwMap>& m, const NvmeGroupKey& group_key, const NvmeGwId& gw_id, NvmeGwState& out)
+{
+  auto gw_group = m.find(group_key);
+  if (gw_group == m.end()) {
+    dout(0) << "can not find group (" << group_key.first << "," << group_key.second << ") "  << desc << " map: " << m << dendl;
+    return false;
+  }
+  auto gw_state = gw_group->second.find(gw_id);
+  if (gw_state == gw_group->second.end()) {
+    dout(0) << "can not find gw id: " << gw_id << " in " << desc << "group: " << gw_group->second  << dendl;
+    return false;
+  }
+  out = gw_state->second;
+  return true;
+}
+
+void NVMeofGwMonitorClient::send_beacon()
+{
+  ceph_assert(ceph_mutex_is_locked_by_me(lock));
+  //dout(0) << "sending beacon as gid " << monc.get_global_id() << dendl;
+  GW_AVAILABILITY_E gw_availability = GW_AVAILABILITY_E::GW_CREATED;
+  BeaconSubsystems subs;
+  NVMeofGwClient gw_client(
+     grpc::CreateChannel(gateway_address, grpc::InsecureChannelCredentials()));
+  subsystems_info gw_subsystems;
+  bool ok = gw_client.get_subsystems(gw_subsystems);
+  if (ok) {
+    for (int i = 0; i < gw_subsystems.subsystems_size(); i++) {
+      const subsystem& sub = gw_subsystems.subsystems(i);
+      BeaconSubsystem bsub;
+      bsub.nqn = sub.nqn();
+      for (int j = 0; j < sub.namespaces_size(); j++) {
+        const auto& ns = sub.namespaces(j);
+        BeaconNamespace bns = {ns.anagrpid(), ns.nonce()};
+        bsub.namespaces.push_back(bns);
+      }
+      for (int k = 0; k < sub.listen_addresses_size(); k++) {
+        const auto& ls = sub.listen_addresses(k);
+        BeaconListener bls = { ls.adrfam(), ls.traddr(), ls.trsvcid() };
+        bsub.listeners.push_back(bls);
+      }
+      subs.push_back(bsub);
+    }
+  }
+
+  auto group_key = std::make_pair(pool, group);
+  NvmeGwState old_gw_state;
+  // if already got gateway state in the map
+  if (get_gw_state("old map", map, group_key, name, old_gw_state))
+    gw_availability = ok ? GW_AVAILABILITY_E::GW_AVAILABLE : GW_AVAILABILITY_E::GW_UNAVAILABLE;
+  dout(0) << "sending beacon as gid " << monc.get_global_id() << " availability " << (int)gw_availability <<
+    " osdmap_epoch " << osdmap_epoch << " gwmap_epoch " << gwmap_epoch << dendl;
+  auto m = ceph::make_message<MNVMeofGwBeacon>(
+      name,
+      pool,
+      group,
+      subs,
+      gw_availability,
+      osdmap_epoch,
+      gwmap_epoch);
+  monc.send_mon_message(std::move(m));
+}
+
+void NVMeofGwMonitorClient::tick()
+{
+  dout(0) << dendl;
+  send_beacon();
+
+  timer.add_event_after(
+      g_conf().get_val<std::chrono::seconds>("nvmeof_mon_client_tick_period").count(),
+      new LambdaContext([this](int r){
+          tick();
+      }
+  ));
+}
+
+void NVMeofGwMonitorClient::shutdown()
+{
+  std::lock_guard l(lock);
+
+  dout(4) << "nvmeof Shutting down" << dendl;
+
+
+  // stop sending beacon first, I use monc to talk with monitors
+  timer.shutdown();
+  // client uses monc and objecter
+  client.shutdown();
+  // Stop asio threads, so leftover events won't call into shut down
+  // monclient/objecter.
+  poolctx.finish();
+  // stop monc
+  monc.shutdown();
+
+  // objecter is used by monc
+  objecter.shutdown();
+  // client_messenger is used by all of them, so stop it in the end
+  client_messenger->shutdown();
+}
+
+void NVMeofGwMonitorClient::handle_nvmeof_gw_map(ceph::ref_t<MNVMeofGwMap> nmap)
+{
+  auto &new_map = nmap->get_map();
+  gwmap_epoch = nmap->get_gwmap_epoch();
+  auto group_key = std::make_pair(pool, group);
+  dout(0) << "handle nvmeof gw map: " << new_map << dendl;
+
+  NvmeGwState old_gw_state;
+  auto got_old_gw_state = get_gw_state("old map", map, group_key, name, old_gw_state); 
+  NvmeGwState new_gw_state;
+  auto got_new_gw_state = get_gw_state("new map", new_map, group_key, name, new_gw_state); 
+
+  if (!got_old_gw_state) {
+    if (!got_new_gw_state) {
+      dout(0) << "Can not find new gw state" << dendl;
+      return;
+    }
+    bool set_group_id = false;
+    while (!set_group_id) {
+      NVMeofGwMonitorGroupClient monitor_group_client(
+          grpc::CreateChannel(monitor_address, grpc::InsecureChannelCredentials()));
+      dout(0) << "GRPC set_group_id: " <<  new_gw_state.group_id << dendl;
+      set_group_id = monitor_group_client.set_group_id( new_gw_state.group_id);
+      if (!set_group_id) {
+	      dout(0) << "GRPC set_group_id failed" << dendl;
+	      usleep(1000); // TODO: conf options
+      }
+    }
+  }
+
+  // Make sure we do not get out of order state changes from the monitor
+  if (got_old_gw_state && got_new_gw_state) {
+    dout(0) << "got_old_gw_state: " << old_gw_state << "got_new_gw_state: " << new_gw_state << dendl;
+    ceph_assert(new_gw_state.gw_map_epoch >= old_gw_state.gw_map_epoch);
+  }
+
+  // Gather all state changes
+  ana_info ai;
+  epoch_t max_blocklist_epoch = 0;
+  for (const auto& nqn_state_pair: new_gw_state.subsystems) {
+    auto& sub = nqn_state_pair.second;
+    const auto& nqn = nqn_state_pair.first;
+    nqn_ana_states nas;
+    nas.set_nqn(nqn);
+    for (NvmeAnaGrpId  ana_grp_index = 0; ana_grp_index < sub.ana_state.size(); ana_grp_index++) {
+      const auto& old_nqn_state_pair = old_gw_state.subsystems.find(nqn);
+      auto found_old_nqn_state = (old_nqn_state_pair != old_gw_state.subsystems.end());
+      auto new_group_state = sub.ana_state[ana_grp_index];
+
+      // if no state change detected for this nqn, group id
+      if (got_old_gw_state && found_old_nqn_state &&
+           new_group_state == old_nqn_state_pair->second.ana_state[ana_grp_index]) {
+        continue;
+      }
+      ana_group_state gs;
+      gs.set_grp_id(ana_grp_index + 1); // offset by 1, index 0 is ANAGRP1
+      const auto& new_agroup_state = new_group_state.first;
+      const epoch_t& blocklist_epoch = new_group_state.second;
+
+      if (new_agroup_state == GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_OPTIMIZED_STATE &&
+          blocklist_epoch != 0) {
+        if (blocklist_epoch > max_blocklist_epoch) max_blocklist_epoch = blocklist_epoch;
+      }
+      gs.set_state(new_agroup_state == GW_EXPORTED_STATES_PER_AGROUP_E::GW_EXPORTED_OPTIMIZED_STATE ? OPTIMIZED : INACCESSIBLE); // Set the ANA state
+      nas.mutable_states()->Add(std::move(gs));
+      dout(0) << " grpid " << (ana_grp_index + 1) << " state: " << new_gw_state << dendl;
+    }
+    if (nas.states_size()) ai.mutable_states()->Add(std::move(nas));
+  }
+
+  // if there is state change, notify the gateway
+  if (ai.states_size()) {
+    bool set_ana_state = false;
+    while (!set_ana_state) {
+      NVMeofGwClient gw_client(
+          grpc::CreateChannel(gateway_address, grpc::InsecureChannelCredentials()));
+      set_ana_state = gw_client.set_ana_state(ai);
+      if (!set_ana_state) {
+	dout(0) << "GRPC set_ana_state failed" << dendl;
+	usleep(1000); // TODO conf option
+      }
+    }
+    // Update latest accepted osdmap epoch, for beacons
+    if (max_blocklist_epoch > osdmap_epoch) {
+      osdmap_epoch = max_blocklist_epoch;
+      dout(0) << "Ready for blocklist osd map epoch: " << osdmap_epoch << dendl;
+    }
+  }
+  map = new_map;
+}
+
+bool NVMeofGwMonitorClient::ms_dispatch2(const ref_t<Message>& m)
+{
+  std::lock_guard l(lock);
+  dout(0) << "got map type " << m->get_type() << dendl;
+
+  if (m->get_type() == MSG_MNVMEOF_GW_MAP) {
+    handle_nvmeof_gw_map(ref_cast<MNVMeofGwMap>(m));
+  }
+  bool handled = false;
+  return handled;
+}
+
+int NVMeofGwMonitorClient::main(std::vector<const char *> args)
+{
+  client_messenger->wait();
+
+  // Disable signal handlers
+  unregister_async_signal_handler(SIGHUP, sighup_handler);
+  shutdown_async_signal_handler();
+
+  return 0;
+}
diff --git a/src/nvmeof/NVMeofGwMonitorClient.h b/src/nvmeof/NVMeofGwMonitorClient.h
new file mode 100644
index 000000000000..1c4c58f5d2f0
--- /dev/null
+++ b/src/nvmeof/NVMeofGwMonitorClient.h
@@ -0,0 +1,83 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023,2024 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+
+#ifndef NVMEOFGWMONITORCLIENT_H_
+#define NVMEOFGWMONITORCLIENT_H_
+
+#include "auth/Auth.h"
+#include "common/async/context_pool.h"
+#include "common/Finisher.h"
+#include "common/Timer.h"
+#include "common/LogClient.h"
+
+#include "client/Client.h"
+#include "mon/MonClient.h"
+#include "osdc/Objecter.h"
+#include "messages/MNVMeofGwMap.h"
+
+class NVMeofGwMonitorClient: public Dispatcher,
+		   public md_config_obs_t {
+private:
+  std::string name;
+  std::string pool;
+  std::string group;
+  std::string gateway_address;
+  std::string monitor_address;
+  std::string server_key;
+  std::string server_cert;
+  std::string client_cert;
+  epoch_t     osdmap_epoch; // last awaited osdmap_epoch
+  epoch_t     gwmap_epoch; // last received gw map epoch
+
+protected:
+  ceph::async::io_context_pool poolctx;
+  MonClient monc;
+  std::unique_ptr<Messenger> client_messenger;
+  Objecter objecter;
+  Client client;
+  std::map<NvmeGroupKey, NvmeGwMap> map;
+  ceph::mutex lock = ceph::make_mutex("NVMeofGw::lock");
+  SafeTimer timer;
+
+  int orig_argc;
+  const char **orig_argv;
+
+  void send_config_beacon(); 
+  void send_beacon();
+ 
+public:
+  NVMeofGwMonitorClient(int argc, const char **argv);
+  ~NVMeofGwMonitorClient() override;
+
+  // Dispatcher interface
+  bool ms_dispatch2(const ceph::ref_t<Message>& m) override;
+  bool ms_handle_reset(Connection *con) override { return false; }
+  void ms_handle_remote_reset(Connection *con) override {}
+  bool ms_handle_refused(Connection *con) override { return false; };
+
+  // config observer bits
+  const char** get_tracked_conf_keys() const override;
+  void handle_conf_change(const ConfigProxy& conf,
+			  const std::set <std::string> &changed) override {};
+
+  int init();
+  void shutdown();
+  int main(std::vector<const char *> args);
+  void tick();
+
+  void handle_nvmeof_gw_map(ceph::ref_t<MNVMeofGwMap> m);
+};
+
+#endif
+
diff --git a/src/nvmeof/NVMeofGwMonitorGroupClient.cc b/src/nvmeof/NVMeofGwMonitorGroupClient.cc
new file mode 100644
index 000000000000..27ed7b134816
--- /dev/null
+++ b/src/nvmeof/NVMeofGwMonitorGroupClient.cc
@@ -0,0 +1,25 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "NVMeofGwMonitorGroupClient.h"
+
+bool NVMeofGwMonitorGroupClient::set_group_id(const uint32_t& id) {
+  group_id_req request;
+  request.set_id(id);
+  google::protobuf::Empty reply;
+  ClientContext context;
+
+  Status status = stub_->group_id(&context, request, &reply);
+
+  return status.ok();
+}
diff --git a/src/nvmeof/NVMeofGwMonitorGroupClient.h b/src/nvmeof/NVMeofGwMonitorGroupClient.h
new file mode 100644
index 000000000000..805e182c15c1
--- /dev/null
+++ b/src/nvmeof/NVMeofGwMonitorGroupClient.h
@@ -0,0 +1,39 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+
+#ifndef  __NVMEOFGWMONITORGROUPCLIENT_H__
+#define  __NVMEOFGWMONITORGROUPCLIENT_H__
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include <grpcpp/grpcpp.h>
+
+#include "monitor.grpc.pb.h"
+
+using grpc::Channel;
+using grpc::ClientContext;
+using grpc::Status;
+
+class NVMeofGwMonitorGroupClient {
+ public:
+  NVMeofGwMonitorGroupClient(std::shared_ptr<Channel> channel)
+      : stub_(MonitorGroup::NewStub(channel)) {}
+
+  bool set_group_id(const uint32_t& id);
+
+ private:
+  std::unique_ptr<MonitorGroup::Stub> stub_;
+};
+#endif
diff --git a/src/nvmeof/gateway b/src/nvmeof/gateway
new file mode 160000
index 000000000000..322a86f7348a
--- /dev/null
+++ b/src/nvmeof/gateway
@@ -0,0 +1 @@
+Subproject commit 322a86f7348af1bc173f01e6cc4b64e9a8075727
diff --git a/src/pybind/mgr/cephadm/services/nvmeof.py b/src/pybind/mgr/cephadm/services/nvmeof.py
index d133ad75d252..73a81eac96ec 100644
--- a/src/pybind/mgr/cephadm/services/nvmeof.py
+++ b/src/pybind/mgr/cephadm/services/nvmeof.py
@@ -21,6 +21,9 @@ class NvmeofService(CephService):
     def config(self, spec: NvmeofServiceSpec) -> None:  # type: ignore
         assert self.TYPE == spec.service_type
         assert spec.pool
+        self.pool = spec.pool
+        assert spec.group is not None
+        self.group = spec.group
         self.mgr._check_pool_exists(spec.pool, spec.service_name())
 
     def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
@@ -55,6 +58,19 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         daemon_spec.extra_files = {'ceph-nvmeof.conf': gw_conf}
         daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
         daemon_spec.deps = []
+        # Notify monitor about this gateway creation
+        cmd = {
+            'prefix': 'nvme-gw create',
+            'id': name,
+            'group': spec.group,
+            'pool': spec.pool
+        }
+        _, _, err = self.mgr.mon_command(cmd)
+        # if send command failed, raise assertion exception, failing the daemon creation
+        assert not err, f"Unable to send monitor command {cmd}, error {err}"
+        if not hasattr(self, 'gws'):
+            self.gws = {} # id -> name map of gateways for this service.
+        self.gws[nvmeof_gw_id] = name # add to map of service's gateway names
         return daemon_spec
 
     def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None:
@@ -125,7 +141,22 @@ def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None
         if not ret:
             logger.info(f'{daemon.hostname} removed from nvmeof gateways dashboard config')
 
-        # and any certificates being used for mTLS
+        # Assert configured
+        assert self.pool
+        assert self.group is not None
+        assert daemon.daemon_id in self.gws
+        name = self.gws[daemon.daemon_id]
+        self.gws.pop(daemon.daemon_id)
+        # Notify monitor about this gateway deletion
+        cmd = {
+            'prefix': 'nvme-gw delete',
+            'id': name,
+            'group': self.group,
+            'pool': self.pool
+        }
+        _, _, err = self.mgr.mon_command(cmd)
+        if err:
+            self.mgr.log.error(f"Unable to send monitor command {cmd}, error {err}")
 
     def purge(self, service_name: str) -> None:
         """Removes configuration
diff --git a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
index fad99ad50e8f..ad70af317b16 100644
--- a/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
+++ b/src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2
@@ -1,7 +1,7 @@
 # {{ cephadm_managed }}
 [gateway]
 name = {{ name }}
-group = {{ spec.group if spec.group is not none else '' }}
+group = {{ spec.group }}
 addr = {{ addr }}
 port = {{ spec.port }}
 enable_auth = {{ spec.enable_auth }}
diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py
index 1440ef579b45..576d61907393 100644
--- a/src/python-common/ceph/deployment/service_spec.py
+++ b/src/python-common/ceph/deployment/service_spec.py
@@ -1372,7 +1372,7 @@ def __init__(self,
         #: ``name`` name of the nvmeof gateway
         self.name = name
         #: ``group`` name of the nvmeof gateway
-        self.group = group
+        self.group = group or ''
         #: ``enable_auth`` enables user authentication on nvmeof gateway
         self.enable_auth = enable_auth
         #: ``state_update_notify`` enables automatic update from OMAP in nvmeof gateway
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 63598971bbdb..654852af2f09 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -1005,3 +1005,11 @@ add_ceph_unittest(unittest_weighted_shuffle)
 add_executable(unittest_intarith test_intarith.cc)
 add_ceph_unittest(unittest_intarith)
 #make check ends here
+
+# test_nvmeof_mon_encoding
+add_executable(test_nvmeof_mon_encoding
+  test_nvmeof_mon_encoding.cc
+  )
+target_link_libraries(test_nvmeof_mon_encoding
+  mon ceph-common global-static
+  )
diff --git a/src/test/test_nvmeof_mon_encoding.cc b/src/test/test_nvmeof_mon_encoding.cc
new file mode 100644
index 000000000000..9d84b58397bb
--- /dev/null
+++ b/src/test/test_nvmeof_mon_encoding.cc
@@ -0,0 +1,197 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2024 IBM, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <iostream>
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "include/ceph_assert.h"
+#include "global/global_init.h"
+#include "mon/NVMeofGwMon.h"
+#include "messages/MNVMeofGwMap.h"
+#include "messages/MNVMeofGwBeacon.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix *_dout
+
+using namespace std;
+
+void test_NVMeofGwMap() {
+  dout(0) << __func__ << "\n\n" << dendl;
+
+  NVMeofGwMap pending_map;
+  std::string pool = "pool1";
+  std::string group = "grp1";
+  auto group_key = std::make_pair(pool, group);
+  pending_map.cfg_add_gw("GW1" ,group_key);
+  pending_map.cfg_add_gw("GW2" ,group_key);
+  pending_map.cfg_add_gw("GW3" ,group_key);
+  NvmeNonceVector new_nonces = {"abc", "def","hij"};
+  pending_map.Created_gws[group_key]["GW1"].nonce_map[1] = new_nonces;
+  for(int i=0; i< MAX_SUPPORTED_ANA_GROUPS; i++){
+    pending_map.Created_gws[group_key]["GW1"].blocklist_data[i].osd_epoch = i*2;
+    pending_map.Created_gws[group_key]["GW1"].blocklist_data[i].is_failover = false;
+  }
+
+  pending_map.Created_gws[group_key]["GW2"].nonce_map[2] = new_nonces;
+  dout(0) << pending_map << dendl;
+
+  ceph::buffer::list bl;
+  pending_map.encode(bl);
+  auto p = bl.cbegin();
+  pending_map.decode(p);
+  dout(0) << "Dump map after decode encode:" <<dendl;
+  dout(0) << pending_map << dendl;
+}
+
+void test_MNVMeofGwMap() {
+  dout(0) << __func__ << "\n\n" << dendl;
+  std::map<NvmeGroupKey, NvmeGwMap> map;
+
+  std::string pool = "pool1";
+  std::string group = "grp1";
+  std::string gw_id = "GW1";
+  NvmeGwState state(1, 32);
+  std::string nqn = "nqn";
+  ANA_STATE ana_state;
+  NqnState nqn_state(nqn, ana_state);
+  state.subsystems.insert({nqn, nqn_state});
+
+  auto group_key = std::make_pair(pool, group);
+  map[group_key][gw_id] = state;
+
+
+
+  ceph::buffer::list bl;
+  encode(map, bl);
+  dout(0) << "encode: " << map << dendl;
+  decode(map, bl);
+  dout(0) << "decode: " << map << dendl;
+
+  BeaconSubsystem sub = { nqn, {}, {} };
+  NVMeofGwMap pending_map;
+  pending_map.cfg_add_gw("GW1" ,group_key);
+  pending_map.cfg_add_gw("GW2" ,group_key);
+  pending_map.cfg_add_gw("GW3" ,group_key);
+  NvmeNonceVector new_nonces = {"abc", "def","hij"};
+  pending_map.Created_gws[group_key]["GW1"].nonce_map[1] = new_nonces;
+  pending_map.Created_gws[group_key]["GW1"].copied_nonce_map[1] = new_nonces;
+  pending_map.Created_gws[group_key]["GW1"].subsystems.push_back(sub);
+  for(int i=0; i< MAX_SUPPORTED_ANA_GROUPS; i++){
+    pending_map.Created_gws[group_key]["GW1"].blocklist_data[i].osd_epoch = i*2;
+    pending_map.Created_gws[group_key]["GW1"].blocklist_data[i].is_failover = false;
+  }
+
+  pending_map.Created_gws[group_key]["GW2"].nonce_map[2] = new_nonces;
+  dout(0) << "False pending map: " << pending_map << dendl;
+
+  auto msg = make_message<MNVMeofGwMap>(pending_map);
+  msg->encode_payload(0);
+  msg->decode_payload();
+  dout(0) << "decode msg: " << *msg << dendl;
+
+  dout(0)   << "\n == Test GW Delete ==" << dendl;
+  pending_map.cfg_delete_gw("GW1" ,group_key);
+  dout(0) << "deleted GW1 " << pending_map << dendl;
+
+  pending_map.cfg_delete_gw("GW1" ,group_key);
+  dout(0) << "duplicated delete of GW1 " << pending_map << dendl;
+
+  pending_map.cfg_delete_gw("GW2" ,group_key);
+  dout(0) << "deleted GW2 " << pending_map << dendl;
+
+  dout(0) << "delete of wrong gw id" << dendl;
+  pending_map.cfg_delete_gw("wow" ,group_key);
+
+  pending_map.cfg_delete_gw("GW3" ,group_key);
+  dout(0) << "deleted GW3 . we should see the empty map " << pending_map << dendl;
+
+
+}
+
+void test_MNVMeofGwBeacon() {
+  std::string gw_id = "GW";
+  std::string gw_pool = "pool";
+  std::string gw_group = "group";
+  GW_AVAILABILITY_E availability = GW_AVAILABILITY_E::GW_AVAILABLE;
+  std::string nqn = "nqn";
+  BeaconSubsystem sub = { nqn, {}, {} };
+  BeaconSubsystems subs = { sub };
+  epoch_t osd_epoch = 17;
+  epoch_t gwmap_epoch = 42;
+
+  auto msg = make_message<MNVMeofGwBeacon>(
+      gw_id,
+      gw_pool,
+      gw_group,
+      subs,
+      availability,
+      osd_epoch,
+      gwmap_epoch);
+  msg->encode_payload(0);
+  msg->decode_payload();
+  dout(0) << "decode msg: " << *msg << dendl;
+  ceph_assert(msg->get_gw_id() == gw_id);
+  ceph_assert(msg->get_gw_pool() == gw_pool);
+  ceph_assert(msg->get_gw_group() == gw_group);
+  ceph_assert(msg->get_availability() == availability);
+  ceph_assert(msg->get_last_osd_epoch() == osd_epoch);
+  ceph_assert(msg->get_last_gwmap_epoch() == gwmap_epoch);
+  const auto& dsubs = msg->get_subsystems();
+  auto it = std::find_if(dsubs.begin(), dsubs.end(),
+                           [&nqn](const auto& element) {
+                               return element.nqn == nqn;
+                           });
+  ceph_assert(it != dsubs.end());
+}
+
+void test_NVMeofGwTimers()
+{
+    NVMeofGwMap pending_map;
+    //pending_map.Gmetadata;
+    const NvmeGroupKey group_key = std::make_pair("a","b");
+    std::string gwid = "GW1";
+    NvmeAnaGrpId  grpid = 2;
+    pending_map.start_timer(gwid, group_key, grpid, 30);
+    auto end_time  = pending_map.Gmetadata[group_key][gwid].data[grpid].end_time;
+    uint64_t  millisecondsSinceEpoch = std::chrono::duration_cast<std::chrono::milliseconds>(end_time.time_since_epoch()).count();
+    dout(0) << "Metadata milliseconds " << millisecondsSinceEpoch << " " << (int)pending_map.Gmetadata[group_key][gwid].data[grpid].timer_value << dendl;
+    ceph::buffer::list bl;
+    pending_map.encode(bl);
+    auto p = bl.cbegin();
+    pending_map.decode(p);
+
+    end_time  = pending_map.Gmetadata[group_key][gwid].data[2].end_time;
+    millisecondsSinceEpoch = std::chrono::duration_cast<std::chrono::milliseconds>(end_time.time_since_epoch()).count();
+    dout(0) << "After encode decode Metadata milliseconds " << millisecondsSinceEpoch << " " <<  (int)pending_map.Gmetadata[group_key][gwid].data[grpid].timer_value<<dendl;
+
+}
+
+int main(int argc, const char **argv)
+{
+  // Init ceph
+  auto args = argv_to_vec(argc, argv);
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+                         CODE_ENVIRONMENT_UTILITY,
+                         CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+
+  // Run tests
+  test_NVMeofGwMap();
+  test_MNVMeofGwMap();
+  test_MNVMeofGwBeacon();
+  test_NVMeofGwTimers();
+}
+

From 0c5bb79671104f700eb199115d89fbd150e528b3 Mon Sep 17 00:00:00 2001
From: Alexander Indenbaum <aindenba@redhat.com>
Date: Thu, 28 Mar 2024 07:20:11 +0000
Subject: [PATCH 2484/2492] pybind: support RHEL8, whose default python is
 python 3.6.8, original commit f68248944815acbf5fb08ef66f36da6d5baef824

It seems the shaman CentOS8 builders, which serve as the base for container images, aren't current with the ":=" syntax. Amend to support RHEL8, whose default python is python 3.6.8.

Error example from https://shaman.ceph.com/builds/ceph/ceph-nvmeof-mon/27188b78bdbaa3c33de518b8514906de629e774a/default/388913/

  File "/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/19.0.0-2505-g27188b78/rpm/el8/BUILD/ceph-19.0.0-2505-g27188b78/src/pybind/cephfs/setup.py", line 120
    if ldflags := os.environ.get('LDFLAGS'):
                ^
SyntaxError: invalid syntax

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 src/pybind/cephfs/setup.py | 3 ++-
 src/pybind/rados/setup.py  | 3 ++-
 src/pybind/rbd/setup.py    | 3 ++-
 src/pybind/rgw/setup.py    | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/pybind/cephfs/setup.py b/src/pybind/cephfs/setup.py
index 956a4e7008c6..e6547103c705 100755
--- a/src/pybind/cephfs/setup.py
+++ b/src/pybind/cephfs/setup.py
@@ -117,7 +117,8 @@ def check_sanity():
             extra_preargs=['-iquote{path}'.format(path=os.path.join(CEPH_SRC_DIR, 'include'))]
         )
 
-        if ldflags := os.environ.get('LDFLAGS'):
+        ldflags = os.environ.get('LDFLAGS')
+        if ldflags:
             extra_postargs = ldflags.split()
         else:
             extra_postargs = None
diff --git a/src/pybind/rados/setup.py b/src/pybind/rados/setup.py
index 000189b7da13..cd99fb457c17 100755
--- a/src/pybind/rados/setup.py
+++ b/src/pybind/rados/setup.py
@@ -112,7 +112,8 @@ def check_sanity():
             sources=[tmp_file],
             output_dir=tmp_dir
         )
-        if ldflags := os.environ.get('LDFLAGS'):
+        ldflags = os.environ.get('LDFLAGS')
+        if ldflags:
             extra_postargs = ldflags.split()
         else:
             extra_postargs = None
diff --git a/src/pybind/rbd/setup.py b/src/pybind/rbd/setup.py
index b03d1c3fc957..4f247273796e 100755
--- a/src/pybind/rbd/setup.py
+++ b/src/pybind/rbd/setup.py
@@ -116,7 +116,8 @@ def check_sanity():
             output_dir=tmp_dir
         )
 
-        if ldflags := os.environ.get('LDFLAGS'):
+        ldflags = os.environ.get('LDFLAGS')
+        if ldflags:
             extra_postargs = ldflags.split()
         else:
             extra_postargs = None
diff --git a/src/pybind/rgw/setup.py b/src/pybind/rgw/setup.py
index eb8fc554db6e..a39f8d4e5453 100755
--- a/src/pybind/rgw/setup.py
+++ b/src/pybind/rgw/setup.py
@@ -116,7 +116,8 @@ def check_sanity():
             output_dir=tmp_dir,
         )
 
-        if ldflags := os.environ.get('LDFLAGS'):
+        ldflags = os.environ.get('LDFLAGS')
+        if ldflags:
             extra_postargs = ldflags.split()
         else:
             extra_postargs = None

From 1461a95ba828851e8550d06da6e515ce54912eae Mon Sep 17 00:00:00 2001
From: Leonid Chernin <leonidc@il.ibm.com>
Date: Sun, 31 Mar 2024 13:37:26 +0000
Subject: [PATCH 2485/2492] ceph-nvmeof-mon fixes

- remove observer from NVMeofMon
  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit a20fb90357990abac6d2696949e48ef4fa5a7880)

- fix assert issue in find_failback_gw - failover_peers array was removed
  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit 4c6a9b3ed194eb7837e6b341aba227042e497078)

- try to resolve slow ops
  It seems that if monitor preprocess_query(MonOpRequestRef op) returns
  true, then  mon.reply_command() should be called. Remove path which
  returns true without mon.reply_command() call.
  (cherry picked from commit 99a345cb0d70851addc33aa3fa7fe07ed5d67b94)

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 src/mon/NVMeofGwMap.cc      |  5 +----
 src/mon/NVMeofGwMon.cc      | 26 ++------------------------
 src/mon/NVMeofGwSerialize.h | 17 ++---------------
 src/mon/NVMeofGwTypes.h     |  3 ---
 4 files changed, 5 insertions(+), 46 deletions(-)

diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
index 869c3784ca1d..4a495758565f 100755
--- a/src/mon/NVMeofGwMap.cc
+++ b/src/mon/NVMeofGwMap.cc
@@ -198,9 +198,8 @@ void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
 void  NVMeofGwMap::set_failover_gw_for_ANA_group(const NvmeGwId &failed_gw_id, const NvmeGroupKey& group_key, const NvmeGwId &gw_id,  NvmeAnaGrpId ANA_groupid)
 {
     NvmeGwCreated& gw_state = Created_gws[group_key][gw_id];
-    gw_state.failover_peer[ANA_groupid] = failed_gw_id;
     epoch_t epoch;
-    dout(4) << "Found failower GW " << gw_id << " for ANA group " << (int)ANA_groupid << dendl;
+    dout(4) << "Found failover GW " << gw_id << " for ANA group " << (int)ANA_groupid << dendl;
     int rc = blocklist_gw (failed_gw_id, group_key, ANA_groupid, epoch, true);
     if(rc){
         gw_state.active_state(ANA_groupid); //TODO check whether it is valid to  start failover when nonces are empty !
@@ -242,8 +241,6 @@ void  NVMeofGwMap::find_failback_gw(const NvmeGwId &gw_id, const NvmeGroupKey& g
         auto& failback_gw_id = gw_state_it.first;
         auto& st = gw_state_it.second;
         if (st.sm_state[gw_state.ana_grp_id] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE) {
-            ceph_assert(st.failover_peer[gw_state.ana_grp_id] == gw_id);
-
             dout(4)  << "Found Failback GW " << failback_gw_id << " that previously took over the ANAGRP " << gw_state.ana_grp_id << " of the available GW " << gw_id << dendl;
             st.sm_state[gw_state.ana_grp_id] = GW_STATES_PER_AGROUP_E::GW_WAIT_FAILBACK_PREPARED;
             start_timer(failback_gw_id, group_key, gw_state.ana_grp_id, 3);// Add timestamp of start Failback preparation
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index bdbdec716dab..fd8dc3c2477c 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -26,7 +26,6 @@ using std::string;
 
 void NVMeofGwMon::init(){
     dout(4) <<  "called " << dendl;
-    g_conf().add_observer(this);
 }
 
 void NVMeofGwMon::on_restart(){
@@ -56,7 +55,7 @@ void NVMeofGwMon::synchronize_last_beacon(){
 }
 
 void NVMeofGwMon::on_shutdown() {
-     g_conf().remove_observer(this);
+    dout(4) <<  "called " << dendl;
 }
 
 void NVMeofGwMon::tick(){
@@ -277,20 +276,6 @@ bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
     cmd_getval(cmdmap, "prefix", prefix);
     dout(4) << "MonCommand : "<< prefix <<  dendl;
 
-   /* MonSession *session = op->get_session();
-    if (!session)
-    {
-        dout(4) << "MonCommand : "<< prefix << " access denied due to lack of session" <<  dendl;
-        mon.reply_command(op, -EACCES, "access denied", rdata,
-                          get_last_committed());
-        return true;
-    }
-   */
-    string format = cmd_getval_or<string>(cmdmap, "format", "plain");
-    boost::scoped_ptr<Formatter> f(Formatter::create(format));
-
-    // TODO   need to check formatter per preffix  - if f is NULL
-
     return false;
 }
 
@@ -424,16 +409,9 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
 
 
 bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op){
-    //dout(4)   << dendl;
     auto m = op->get_req<MNVMeofGwBeacon>();
     const BeaconSubsystems& sub = m->get_subsystems();
-     //mon.no_reply(op); // we never reply to beacons
-     dout(15) << "beacon from " << m->get_type() << " GW : " << m->get_gw_id()  << " num subsystems " << sub.size() <<  dendl;
-     MonSession *session = op->get_session();
-     if (!session){
-         dout(4) << "beacon no session "  << dendl;
-         return true;
-     }
+    dout(15) << "beacon from " << m->get_type() << " GW : " << m->get_gw_id()  << " num subsystems " << sub.size() <<  dendl;
 
     return false; // allways  return false to call leader's prepare beacon
 }
diff --git a/src/mon/NVMeofGwSerialize.h b/src/mon/NVMeofGwSerialize.h
index 0c8414e6b7f5..905d9090d1e0 100755
--- a/src/mon/NVMeofGwSerialize.h
+++ b/src/mon/NVMeofGwSerialize.h
@@ -134,10 +134,7 @@ inline std::ostream& print_gw_created_t(std::ostream& os, const NvmeGwCreated va
     for (size_t i = 0; i < num_ana_groups; i++) {
         os << " " << anas[i] <<": " << value.sm_state[anas[i]] << ",";
     }
-    os <<  "]\n"<< MODULE_PREFFIX << " failover peers ";
-    for (size_t i = 0; i < num_ana_groups; i++) {
-        os << anas[i] <<": "  << value.failover_peer[anas[i]] << ",";
-    }
+
     os << "]\n"<< MODULE_PREFFIX << "availability " << value.availability << "]";
 
     return os;
@@ -153,10 +150,7 @@ inline std::ostream& operator<<(std::ostream& os, const NvmeGwCreated value) {
     for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) {
         os << value.sm_state[i] << ",";
     }
-    os <<  "]\n"<< MODULE_PREFFIX << " failover peers ";
-    for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++) {
-        os << value.failover_peer[i] << ",";
-    }
+
     os <<  "]\n"<< MODULE_PREFFIX << " beacon-subsystems ";
     for (const auto& sub: value.subsystems) {
         os << sub << ",";
@@ -333,9 +327,6 @@ inline void encode(const NvmeGwCreatedMap& gws,  ceph::bufferlist &bl) {
         for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
             encode((uint32_t)(gw.second.sm_state[i]), bl);
         }
-        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
-            encode((gw.second.failover_peer[i]), bl);
-        }
         encode((uint32_t)gw.second.availability, bl);
         encode((uint32_t)gw.second.last_gw_map_epoch_valid, bl);
         encode(gw.second.subsystems, bl);
@@ -369,10 +360,6 @@ inline void decode(NvmeGwCreatedMap& gws, ceph::buffer::list::const_iterator &bl
             decode(sm_state, bl);
             gw_created.sm_state[i] = (GW_STATES_PER_AGROUP_E)  sm_state;
         }
-        for(int i = 0; i <MAX_SUPPORTED_ANA_GROUPS; i ++){
-            decode(peer_name, bl);
-            gw_created.failover_peer[i] = peer_name;
-        }
         uint32_t avail;
         decode(avail, bl);
         gw_created.availability = (GW_AVAILABILITY_E)avail;
diff --git a/src/mon/NVMeofGwTypes.h b/src/mon/NVMeofGwTypes.h
index 5eb1a3ba622b..5539b2eaf61b 100755
--- a/src/mon/NVMeofGwTypes.h
+++ b/src/mon/NVMeofGwTypes.h
@@ -82,7 +82,6 @@ struct NvmeGwCreated {
     NvmeAnaNonceMap    nonce_map;
     NvmeAnaNonceMap    copied_nonce_map;
     SM_STATE           sm_state;                      // state machine states per ANA group
-    NvmeGwId           failover_peer[MAX_SUPPORTED_ANA_GROUPS];
     struct{
        epoch_t     osd_epoch;
        bool        is_failover;
@@ -94,7 +93,6 @@ struct NvmeGwCreated {
     {
         for (int i = 0; i < MAX_SUPPORTED_ANA_GROUPS; i++){
             sm_state[i] = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
-            failover_peer[i]  = "";
             blocklist_data[i].osd_epoch = 0;
             blocklist_data[i].is_failover = true;
         }
@@ -102,7 +100,6 @@ struct NvmeGwCreated {
 
     void standby_state(NvmeAnaGrpId grpid) {
            sm_state[grpid]       = GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE;
-           failover_peer[grpid]  = "";
     };
     void active_state(NvmeAnaGrpId grpid) {
            sm_state[grpid]       = GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE;

From cbbf1ecbcebc6b25e17d2fbc1b7203efc53ae1ae Mon Sep 17 00:00:00 2001
From: Alexander Indenbaum <aindenba@redhat.com>
Date: Tue, 2 Apr 2024 10:36:04 +0000
Subject: [PATCH 2486/2492] Clean up

- NVMeofGwTypes: remove copied_nonce_map cleanup
- NVMeofGwMon.h: fix weird spacing
- NVMeofGwMon.cc: dout(4) to dout(10)
- NVMeofGwMon.cc: rm commented out code

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 src/mon/NVMeofGwMon.cc               | 65 +++++++++++-----------------
 src/mon/NVMeofGwMon.h                |  6 +--
 src/mon/NVMeofGwSerialize.h          |  3 --
 src/mon/NVMeofGwTypes.h              |  1 -
 src/test/test_nvmeof_mon_encoding.cc |  1 -
 5 files changed, 29 insertions(+), 47 deletions(-)

diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index fd8dc3c2477c..3f8218dbb73a 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -25,18 +25,18 @@
 using std::string;
 
 void NVMeofGwMon::init(){
-    dout(4) <<  "called " << dendl;
+    dout(10) <<  "called " << dendl;
 }
 
 void NVMeofGwMon::on_restart(){
-    dout(4) <<  "called " << dendl;
+    dout(10) <<  "called " << dendl;
     last_beacon.clear();
     last_tick = ceph::coarse_mono_clock::now();
 }
 
 
 void NVMeofGwMon::synchronize_last_beacon(){
-    dout(4) <<  "called " << dendl;
+    dout(10) <<  "called " << dendl;
     last_beacon.clear();
     last_tick = ceph::coarse_mono_clock::now();
     // Initialize last_beacon to identify transitions of available  GWs to unavailable state
@@ -46,7 +46,7 @@ void NVMeofGwMon::synchronize_last_beacon(){
       for (const auto& gw_created_pair: gw_created_map) {
           const auto& gw_id = gw_created_pair.first;
           if (gw_created_pair.second.availability == GW_AVAILABILITY_E::GW_AVAILABLE){
-             dout(4) << "synchronize last_beacon for  GW :" << gw_id << dendl;
+             dout(10) << "synchronize last_beacon for  GW :" << gw_id << dendl;
              LastBeacon lb = {gw_id, group_key};
              last_beacon[lb] = last_tick;
           }
@@ -55,7 +55,7 @@ void NVMeofGwMon::synchronize_last_beacon(){
 }
 
 void NVMeofGwMon::on_shutdown() {
-    dout(4) <<  "called " << dendl;
+    dout(10) <<  "called " << dendl;
 }
 
 void NVMeofGwMon::tick(){
@@ -77,7 +77,7 @@ void NVMeofGwMon::tick(){
         // This case handles either local slowness (calls being delayed
         // for whatever reason) or cluster election slowness (a long gap
         // between calls while an election happened)
-        dout(4) << ": resetting beacon timeouts due to mon delay "
+        dout(10) << ": resetting beacon timeouts due to mon delay "
                 "(slow election?) of " << now - last_tick << " seconds" << dendl;
         for (auto &i : last_beacon) {
           i.second = now;
@@ -97,7 +97,7 @@ void NVMeofGwMon::tick(){
         auto& lb = itr.first;
         auto last_beacon_time = itr.second;
         if(last_beacon_time < cutoff){
-            dout(4) << "beacon timeout for GW " << lb.gw_id << dendl;
+            dout(10) << "beacon timeout for GW " << lb.gw_id << dendl;
             pending_map.process_gw_map_gw_down( lb.gw_id, lb.group_key, propose);
             _propose_pending |= propose;
             last_beacon.erase(lb);
@@ -111,13 +111,9 @@ void NVMeofGwMon::tick(){
     _propose_pending |= propose;
 
     if(_propose_pending){
-       //pending_map.delay_propose = true; // not to send map to clients immediately in "update_from_paxos"
-       dout(4) << "propose pending " <<dendl;
+       dout(10) << "propose pending " <<dendl;
        propose_pending();
     }
-
-    // if propose_pending returned true , call propose_pending method of the paxosService
-    // todo understand the logic of paxos.plugged for sending several propose_pending see MgrMonitor::tick
 }
 
 const char **NVMeofGwMon::get_tracked_conf_keys() const
@@ -131,15 +127,14 @@ const char **NVMeofGwMon::get_tracked_conf_keys() const
 void NVMeofGwMon::handle_conf_change(const ConfigProxy& conf,
                                     const std::set<std::string> &changed)
 {
-  dout(4) << "changed " << changed << dendl;
+  dout(10) << "changed " << changed << dendl;
 }
 
 void NVMeofGwMon::create_pending(){
 
     pending_map = map;// deep copy of the object
-    // TODO  since "pending_map"  can be reset  each time during paxos re-election even in the middle of the changes ...
     pending_map.epoch++;
-    dout(4) << " pending " << pending_map  << dendl;
+    dout(10) << " pending " << pending_map  << dendl;
     if(last_leader == false){ // peon becomes leader and gets updated map , need to synchronize the last_beacon
         synchronize_last_beacon();
         last_leader = true;
@@ -159,10 +154,10 @@ void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t){
 void NVMeofGwMon::update_from_paxos(bool *need_bootstrap){
     version_t version = get_last_committed();
 
-    //dout(4) <<  MY_MON_PREFFIX << __func__ << " version "  << version  << " map.epoch " << map.epoch << dendl;
+    //dout(10) <<  MY_MON_PREFFIX << __func__ << " version "  << version  << " map.epoch " << map.epoch << dendl;
 
     if (version != map.epoch) {
-        dout(4) << " NVMeGW loading version " << version  << " " << map.epoch << dendl;
+        dout(10) << " NVMeGW loading version " << version  << " " << map.epoch << dendl;
 
         bufferlist bl;
         int err = get_version(version, bl);
@@ -171,7 +166,7 @@ void NVMeofGwMon::update_from_paxos(bool *need_bootstrap){
         auto p = bl.cbegin();
         map.decode(p);
         if(!mon.is_leader()) {
-            dout(4) << "leader map: " << map <<  dendl;
+            dout(10) << "leader map: " << map <<  dendl;
         }
         check_subs(true);
     }
@@ -184,7 +179,7 @@ void NVMeofGwMon::check_sub(Subscription *sub)
     dout(10) << "sub->next , map-epoch " << sub->next << " " << map.epoch << dendl;
     if (sub->next <= map.epoch)
     {
-      dout(4) << "Sending map to subscriber " << sub->session->con << " " << sub->session->con->get_peer_addr() << dendl;
+      dout(10) << "Sending map to subscriber " << sub->session->con << " " << sub->session->con->get_peer_addr() << dendl;
       sub->session->con->send_message2(make_message<MNVMeofGwMap>(map));
 
       if (sub->onetime) {
@@ -198,7 +193,7 @@ void NVMeofGwMon::check_sub(Subscription *sub)
 void NVMeofGwMon::check_subs(bool t)
 {
   const std::string type = "NVMeofGw";
-  dout(4) <<  "count " << mon.session_map.subs.count(type) << dendl;
+  dout(10) <<  "count " << mon.session_map.subs.count(type) << dendl;
 
   if (mon.session_map.subs.count(type) == 0){
       return;
@@ -258,7 +253,7 @@ bool NVMeofGwMon::prepare_update(MonOpRequestRef op){
 
 bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
 {
-    dout(4) << dendl;
+    dout(10) << dendl;
     auto m = op->get_req<MMonCommand>();
     std::stringstream ss;
     bufferlist rdata;
@@ -274,14 +269,14 @@ bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
 
     string prefix;
     cmd_getval(cmdmap, "prefix", prefix);
-    dout(4) << "MonCommand : "<< prefix <<  dendl;
+    dout(10) << "MonCommand : "<< prefix <<  dendl;
 
     return false;
 }
 
 bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
 {
-    dout(4)  << dendl;
+    dout(10)  << dendl;
     auto m = op->get_req<MMonCommand>();
     int rc;
     std::stringstream ss;
@@ -309,7 +304,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
 
     const auto prefix = cmd_getval_or<string>(cmdmap, "prefix", string{});
 
-    dout(4) << "MonCommand : "<< prefix <<  dendl;
+    dout(10) << "MonCommand : "<< prefix <<  dendl;
     if( prefix == "nvme-gw create" || prefix == "nvme-gw delete" ) {
         std::string id, pool, group;
 
@@ -317,7 +312,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
         cmd_getval(cmdmap, "pool", pool);
         cmd_getval(cmdmap, "group", group);
         auto group_key = std::make_pair(pool, group);
-        dout(4) << " id "<< id <<" pool "<< pool << " group "<< group << dendl;
+        dout(10) << " id "<< id <<" pool "<< pool << " group "<< group << dendl;
         if(prefix == "nvme-gw create"){
             rc = pending_map.cfg_add_gw(id, group_key);
             ceph_assert(rc!= -EINVAL);
@@ -346,7 +341,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
         cmd_getval(cmdmap, "pool", pool);
         cmd_getval(cmdmap, "group", group);
         auto group_key = std::make_pair(pool, group);
-        dout(4) <<"nvme-gw show  pool "<< pool << " group "<< group << dendl;
+        dout(10) <<"nvme-gw show  pool "<< pool << " group "<< group << dendl;
 
         if( map.Created_gws[group_key].size()){
             f->open_object_section("common");
@@ -417,10 +412,7 @@ bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op){
 }
 
 
-//#define BYPASS_GW_CREATE_CLI
-
 bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
-    //dout(4)  << dendl;
     auto m = op->get_req<MNVMeofGwBeacon>();
 
     dout(20) << "availability " <<  m->get_availability() << " GW : " << m->get_gw_id() <<
@@ -441,7 +433,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
     if (avail == GW_AVAILABILITY_E::GW_CREATED){
         if (gw == group_gws.end()) {
            gw_created = false;
-           dout(4) << "Warning: GW " << gw_id << " group_key " << group_key << " was not found in the  map.Created_gws "<< map.Created_gws <<dendl;
+           dout(10) << "Warning: GW " << gw_id << " group_key " << group_key << " was not found in the  map.Created_gws "<< map.Created_gws <<dendl;
         }
         goto set_propose;
     }
@@ -456,20 +448,16 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
     if(m->get_nonce_map().size()) {
         if(pending_map.Created_gws[group_key][gw_id].nonce_map != m->get_nonce_map())
         {
-            dout(4) << "nonce map of GW  changed , propose pending " << gw_id << dendl;
+            dout(10) << "nonce map of GW  changed , propose pending " << gw_id << dendl;
             pending_map.Created_gws[group_key][gw_id].nonce_map = m->get_nonce_map();
-            dout(4) << "nonce map of GW " << gw_id << " "<< pending_map.Created_gws[group_key][gw_id].nonce_map  << dendl;
+            dout(10) << "nonce map of GW " << gw_id << " "<< pending_map.Created_gws[group_key][gw_id].nonce_map  << dendl;
             nonce_propose = true;
         }
     }
     else  {
-        dout(4) << "Warning: received empty nonce map in the beacon of GW " << gw_id << " "<< dendl;
+        dout(10) << "Warning: received empty nonce map in the beacon of GW " << gw_id << " "<< dendl;
     }
 
-    //pending_map.handle_removed_subsystems(gw_id, group_key, configured_subsystems, propose);
-
-    //if  no subsystem configured set gw as avail = GW_AVAILABILITY_E::GW_UNAVAILABLE
-
     if(sub.size() == 0) {
         avail = GW_AVAILABILITY_E::GW_UNAVAILABLE;
     }
@@ -480,7 +468,6 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
     }
     if(avail == GW_AVAILABILITY_E::GW_AVAILABLE)
     {
-        //dout(4) <<"subsystems from beacon " << pending_map.Created_gws << dendl;
         auto now = ceph::coarse_mono_clock::now();
         // check pending_map.epoch vs m->get_version() - if different - drop the beacon
 
@@ -516,7 +503,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
     }
 false_return:
     if (propose){
-      dout(4) << "decision in prepare_beacon" <<dendl;
+      dout(10) << "decision in prepare_beacon" <<dendl;
       return true;
     }
     else 
diff --git a/src/mon/NVMeofGwMon.h b/src/mon/NVMeofGwMon.h
index 33313e805d01..9bf63c0dd7d1 100644
--- a/src/mon/NVMeofGwMon.h
+++ b/src/mon/NVMeofGwMon.h
@@ -58,9 +58,9 @@ class NVMeofGwMon: public PaxosService,
     void handle_conf_change(const ConfigProxy& conf, const std::set<std::string> &changed) override;
 
     // 3 pure virtual methods of the paxosService
-    void create_initial()override{};
-    void create_pending()override ;
-    void encode_pending(MonitorDBStore::TransactionRef t)override ;
+    void create_initial() override {};
+    void create_pending() override;
+    void encode_pending(MonitorDBStore::TransactionRef t) override;
 
     void init() override;
     void on_shutdown() override;
diff --git a/src/mon/NVMeofGwSerialize.h b/src/mon/NVMeofGwSerialize.h
index 905d9090d1e0..153da333cb3d 100755
--- a/src/mon/NVMeofGwSerialize.h
+++ b/src/mon/NVMeofGwSerialize.h
@@ -130,7 +130,6 @@ inline std::ostream& print_gw_created_t(std::ostream& os, const NvmeGwCreated va
         os << " " << anas[i] <<": " << value.blocklist_data[anas[i]].osd_epoch << ":" <<value.blocklist_data[anas[i]].is_failover ;
     }
     os << "\n" << MODULE_PREFFIX << "nonces: " << value.nonce_map << " }";
-    os << "\n" << MODULE_PREFFIX << "saved-nonces: " << value.copied_nonce_map << " }";
     for (size_t i = 0; i < num_ana_groups; i++) {
         os << " " << anas[i] <<": " << value.sm_state[anas[i]] << ",";
     }
@@ -336,7 +335,6 @@ inline void encode(const NvmeGwCreatedMap& gws,  ceph::bufferlist &bl) {
             encode(gw.second.blocklist_data[i].is_failover, bl);
         }
         encode(gw.second.nonce_map, bl);
-        encode(gw.second.copied_nonce_map, bl);
     }
     ENCODE_FINISH(bl);
 }
@@ -375,7 +373,6 @@ inline void decode(NvmeGwCreatedMap& gws, ceph::buffer::list::const_iterator &bl
             decode(gw_created.blocklist_data[i].is_failover, bl);
         }
         decode(gw_created.nonce_map, bl);
-        decode(gw_created.copied_nonce_map, bl);
 
         gws[gw_name] = gw_created;
     }
diff --git a/src/mon/NVMeofGwTypes.h b/src/mon/NVMeofGwTypes.h
index 5539b2eaf61b..6642fdab4f0b 100755
--- a/src/mon/NVMeofGwTypes.h
+++ b/src/mon/NVMeofGwTypes.h
@@ -80,7 +80,6 @@ struct NvmeGwCreated {
     bool               last_gw_map_epoch_valid;       // "true" if the last epoch seen by the gw-client is up-to-date
     BeaconSubsystems   subsystems;                    // gateway susbsystem and their state machine states
     NvmeAnaNonceMap    nonce_map;
-    NvmeAnaNonceMap    copied_nonce_map;
     SM_STATE           sm_state;                      // state machine states per ANA group
     struct{
        epoch_t     osd_epoch;
diff --git a/src/test/test_nvmeof_mon_encoding.cc b/src/test/test_nvmeof_mon_encoding.cc
index 9d84b58397bb..5e2ee918ab64 100644
--- a/src/test/test_nvmeof_mon_encoding.cc
+++ b/src/test/test_nvmeof_mon_encoding.cc
@@ -87,7 +87,6 @@ void test_MNVMeofGwMap() {
   pending_map.cfg_add_gw("GW3" ,group_key);
   NvmeNonceVector new_nonces = {"abc", "def","hij"};
   pending_map.Created_gws[group_key]["GW1"].nonce_map[1] = new_nonces;
-  pending_map.Created_gws[group_key]["GW1"].copied_nonce_map[1] = new_nonces;
   pending_map.Created_gws[group_key]["GW1"].subsystems.push_back(sub);
   for(int i=0; i< MAX_SUPPORTED_ANA_GROUPS; i++){
     pending_map.Created_gws[group_key]["GW1"].blocklist_data[i].osd_epoch = i*2;

From 20bd67ee3dc6c4038f40a71c170683d2488efd1f Mon Sep 17 00:00:00 2001
From: Leonid Chernin <leonidc@il.ibm.com>
Date: Wed, 3 Apr 2024 07:59:36 +0000
Subject: [PATCH 2487/2492] Remove last_leader logic, some  cleanups in
 NVMeofGwMon.h

---
 src/mon/NVMeofGwMon.cc | 12 +++---------
 src/mon/NVMeofGwMon.h  |  6 +-----
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index 3f8218dbb73a..fe7150b13c16 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -32,15 +32,14 @@ void NVMeofGwMon::on_restart(){
     dout(10) <<  "called " << dendl;
     last_beacon.clear();
     last_tick = ceph::coarse_mono_clock::now();
+    synchronize_last_beacon();
 }
 
 
 void NVMeofGwMon::synchronize_last_beacon(){
-    dout(10) <<  "called " << dendl;
-    last_beacon.clear();
-    last_tick = ceph::coarse_mono_clock::now();
+    dout(10) <<  "called, is leader : " << mon.is_leader()  <<" active " << is_active()  << dendl;
     // Initialize last_beacon to identify transitions of available  GWs to unavailable state
-    for (const auto& created_map_pair: pending_map.Created_gws) {
+    for (const auto& created_map_pair: map.Created_gws) {
       const auto& group_key = created_map_pair.first;
       const NvmeGwCreatedMap& gw_created_map = created_map_pair.second;
       for (const auto& gw_created_pair: gw_created_map) {
@@ -61,7 +60,6 @@ void NVMeofGwMon::on_shutdown() {
 void NVMeofGwMon::tick(){
     if (!is_active() || !mon.is_leader()){
         dout(10) << "NVMeofGwMon leader : " << mon.is_leader() << "active : " << is_active()  << dendl;
-        last_leader = false;
         return;
     }
     bool _propose_pending = false;
@@ -135,10 +133,6 @@ void NVMeofGwMon::create_pending(){
     pending_map = map;// deep copy of the object
     pending_map.epoch++;
     dout(10) << " pending " << pending_map  << dendl;
-    if(last_leader == false){ // peon becomes leader and gets updated map , need to synchronize the last_beacon
-        synchronize_last_beacon();
-        last_leader = true;
-    }
 }
 
 void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t){
diff --git a/src/mon/NVMeofGwMon.h b/src/mon/NVMeofGwMon.h
index 9bf63c0dd7d1..6078b5b54b3e 100644
--- a/src/mon/NVMeofGwMon.h
+++ b/src/mon/NVMeofGwMon.h
@@ -45,11 +45,8 @@ class NVMeofGwMon: public PaxosService,
     // election) to reset last_beacon timeouts
     ceph::coarse_mono_clock::time_point last_tick;
 
-    std::vector<MonCommand> command_descs;
-    std::vector<MonCommand> pending_command_descs;
-
 public:
-    NVMeofGwMon(Monitor &mn, Paxos &p, const std::string& service_name): PaxosService(mn, p, service_name) {map.mon = &mn; last_leader = false;}
+    NVMeofGwMon(Monitor &mn, Paxos &p, const std::string& service_name): PaxosService(mn, p, service_name) {map.mon = &mn;}
     ~NVMeofGwMon() override {}
 
 
@@ -86,7 +83,6 @@ class NVMeofGwMon: public PaxosService,
     void check_sub(Subscription *sub);
 
 private:
-    bool last_leader;
     void synchronize_last_beacon();
 
 };

From 726050aa3fd1ce2b21539fdd632b16e880bf2946 Mon Sep 17 00:00:00 2001
From: Roy Sahar <royswi@gmail.com>
Date: Wed, 3 Apr 2024 12:21:05 +0300
Subject: [PATCH 2488/2492] Ceph NVMEoF Gateway - add mtls folder for nvmeof
 container

Signed-off-by: Roy Sahar <royswi@gmail.com>
---
 src/cephadm/cephadmlib/daemons/nvmeof.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/cephadm/cephadmlib/daemons/nvmeof.py b/src/cephadm/cephadmlib/daemons/nvmeof.py
index f22147c775c3..289ec1129e4a 100644
--- a/src/cephadm/cephadmlib/daemons/nvmeof.py
+++ b/src/cephadm/cephadmlib/daemons/nvmeof.py
@@ -63,7 +63,7 @@ def identity(self) -> DaemonIdentity:
         return DaemonIdentity(self.fsid, self.daemon_type, self.daemon_id)
 
     @staticmethod
-    def _get_container_mounts(data_dir: str, log_dir: str) -> Dict[str, str]:
+    def _get_container_mounts(data_dir: str, log_dir: str, mtls_dir: Optional[str] = None) -> Dict[str, str]:
         mounts = dict()
         mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
         mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
@@ -74,6 +74,8 @@ def _get_container_mounts(data_dir: str, log_dir: str) -> Dict[str, str]:
         mounts['/dev/hugepages'] = '/dev/hugepages'
         mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         mounts[log_dir] = '/var/log/ceph:z'
+        if mtls_dir:
+            mounts[mtls_dir] = '/src/mtls:z'
         return mounts
 
     def customize_container_mounts(
@@ -81,7 +83,11 @@ def customize_container_mounts(
     ) -> None:
         data_dir = self.identity.data_dir(ctx.data_dir)
         log_dir = os.path.join(ctx.log_dir, self.identity.fsid)
-        mounts.update(self._get_container_mounts(data_dir, log_dir))
+        mtls_dir = os.path.join(ctx.data_dir, self.identity.fsid, 'mtls')
+        if os.path.exists(mtls_dir):
+            mounts.update(self._get_container_mounts(data_dir, log_dir, mtls_dir=mtls_dir))
+        else:
+            mounts.update(self._get_container_mounts(data_dir, log_dir))
 
     def customize_container_binds(
         self, ctx: CephadmContext, binds: List[List[str]]

From 12ef028e30def7f77ac937868a61e7891cb9e8f2 Mon Sep 17 00:00:00 2001
From: Leonid Chernin <leonidc@il.ibm.com>
Date: Wed, 3 Apr 2024 11:30:50 +0000
Subject: [PATCH 2489/2492] fix slow ops

---
 src/mon/NVMeofGwMon.cc | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index fe7150b13c16..e4428b1784d9 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -487,13 +487,16 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
 
 set_propose:
     if(!propose) {
-      if(gw_created){
-          ack_map.Created_gws[group_key][gw_id] = map.Created_gws[group_key][gw_id];// respond with a map slice correspondent to the same GW
-      }
-      ack_map.epoch = map.epoch;
-      dout(20) << "ack_map " << ack_map <<dendl;
-      auto msg = make_message<MNVMeofGwMap>(ack_map);
-      mon.send_reply(op, msg.detach());
+       if(gw_created){
+           ack_map.Created_gws[group_key][gw_id] = map.Created_gws[group_key][gw_id];// respond with a map slice correspondent to the same GW
+       }
+       ack_map.epoch = map.epoch;
+       dout(20) << "ack_map " << ack_map <<dendl;
+       auto msg = make_message<MNVMeofGwMap>(ack_map);
+       mon.send_reply(op, msg.detach());
+    }
+    else {
+       mon.no_reply(op);
     }
 false_return:
     if (propose){

From e50ed54b74e84e301edd1c017604709c6d2821e6 Mon Sep 17 00:00:00 2001
From: Leonid Chernin <leonidc@il.ibm.com>
Date: Thu, 4 Apr 2024 17:06:50 +0000
Subject: [PATCH 2490/2492] ceph-nvmeof-mon fixes

- fix access denied message upon calling nvme-gw show from the gw

  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit 912e980cec762ddc4900ad701a7b1c35ba600580)

- modify access to r for nvme-gw show command

  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit 4794b2b255339f5f2fc9acb8cbf26e4598b082fd)

- gw-nvme show is per mon

  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit 6232ec868d9cae4f27db805c13cf10205ae88a73)
---
 src/mon/MonCommands.h  | 2 +-
 src/mon/NVMeofGwMon.cc | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 3adac47b28ef..a56ee0d83951 100644
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1384,7 +1384,7 @@ COMMAND("nvme-gw show"
    " name=pool,type=CephString"
    " name=group,type=CephString",
    " show nvmeof gateways within (pool, group)",
-   "mgr", "rw")
+   "mon", "r")
 
 // these are tell commands that were implemented as CLI commands in
 // the broken pre-octopus way that we want to allow to work when a
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index e4428b1784d9..3be1aa038502 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -286,13 +286,6 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
         return true;
     }
 
-    MonSession *session = op->get_session();
-    if (!session)
-    {
-        mon.reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
-        return true;
-    }
-
     string format = cmd_getval_or<string>(cmdmap, "format", "plain");
     boost::scoped_ptr<Formatter> f(Formatter::create(format));
 

From 1c8eaaed171338ca430801cd2a0de955426cce1d Mon Sep 17 00:00:00 2001
From: Alexander Indenbaum <aindenba@redhat.com>
Date: Mon, 8 Apr 2024 18:18:59 +0000
Subject: [PATCH 2491/2492] ceph-nvmeof-mon: fixes

- gw subsystems update
  propose state change when subsystems reported by the gateway changes.
  otherwise, only the initially reported subsystem is handled, and any
  new subsystem listener ana groups could remain "unreachable".

  example of the bug: https://github.com/ceph/ceph-nvmeof/actions/runs/8603572675/job/23576286745?pr=560

  Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
  (cherry picked from commit 7ad25a319dd3386aa97a535965e41b97cf44c875)

- fixing  issue : same group set to Active on 2 gws

  Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
  (cherry picked from commit f2d3d29695b43481b7c8ef46c3d200e9bb0a9764)

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>
---
 src/mon/NVMeofGwMap.cc              |  7 ++++---
 src/mon/NVMeofGwMon.cc              |  8 +++++++-
 src/mon/NVMeofGwTypes.h             | 20 ++++++++++++++++++++
 src/nvmeof/NVMeofGwMonitorClient.cc |  1 -
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
index 4a495758565f..c6c4cb6a3232 100755
--- a/src/mon/NVMeofGwMap.cc
+++ b/src/mon/NVMeofGwMap.cc
@@ -108,6 +108,7 @@ int NVMeofGwMap::process_gw_map_gw_down(const NvmeGwId &gw_id, const NvmeGroupKe
             fsm_handle_gw_down (gw_id, group_key, st.sm_state[i], i, propose_pending);
             st.standby_state(i);
         }
+        propose_pending = true; // map should reflect that gw becames unavailable
     }
     else {
         dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map " << gw_id << dendl;
@@ -478,11 +479,11 @@ void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKe
                     break;
                 }
                 else if(st.sm_state[grpid] == GW_STATES_PER_AGROUP_E::GW_STANDBY_STATE  &&  st.availability == GW_AVAILABILITY_E::GW_AVAILABLE) {
-                   st.active_state(grpid);// GW failed and started during the persistency interval
-                   dout(4)  << "Failback unsuccessfull. GW: " << gw_state.first << "becomes Active for the ana group " << grpid  << dendl;
+                   st.standby_state(grpid);// GW failed during the persistency interval
+                   dout(4)  << "Failback unsuccessfull. GW: " << gw_state.first << " becomes Standby for the ANA groupId " << grpid  << dendl;
                 }
                 fbp_gw_state.standby_state(grpid);
-                dout(4) << "Failback unsuccessfull GW: " << gw_id << "becomes standby for the ana group " << grpid  << dendl;
+                dout(4) << "Failback unsuccessfull GW: " << gw_id << " becomes Standby for the ANA groupId " << grpid  << dendl;
                 map_modified = true;
                 break;
             }
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index 3be1aa038502..073e238793be 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -448,7 +448,13 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
     if(sub.size() == 0) {
         avail = GW_AVAILABILITY_E::GW_UNAVAILABLE;
     }
-    pending_map.Created_gws[group_key][gw_id].subsystems =  sub;
+    if(pending_map.Created_gws[group_key][gw_id].subsystems != sub)
+    {
+        dout(10) << "subsystems of GW changed, propose pending " << gw_id << dendl;
+        pending_map.Created_gws[group_key][gw_id].subsystems =  sub;
+        dout(10) << "subsystems of GW " << gw_id << " "<< pending_map.Created_gws[group_key][gw_id].subsystems << dendl;
+        nonce_propose = true;
+    }
     pending_map.Created_gws[group_key][gw_id].last_gw_map_epoch_valid = ( map.epoch == m->get_last_gwmap_epoch() );
     if( pending_map.Created_gws[group_key][gw_id].last_gw_map_epoch_valid == false ){
       dout(1) <<  "map epoch of gw is not up-to-date " << gw_id << " epoch " << map.epoch << " beacon_epoch " << m->get_last_gwmap_epoch() <<  dendl;
diff --git a/src/mon/NVMeofGwTypes.h b/src/mon/NVMeofGwTypes.h
index 6642fdab4f0b..f97dbc0b829b 100755
--- a/src/mon/NVMeofGwTypes.h
+++ b/src/mon/NVMeofGwTypes.h
@@ -55,18 +55,38 @@ using ANA_STATE = std::vector<std::pair<GW_EXPORTED_STATES_PER_AGROUP_E, epoch_t
 struct BeaconNamespace {
     NvmeAnaGrpId anagrpid;
     std::string  nonce;
+
+    // Define the equality operator
+    bool operator==(const BeaconNamespace& other) const {
+        return anagrpid == other.anagrpid &&
+               nonce == other.nonce;
+    }
 };
 
 struct BeaconListener {
     std::string address_family; // IPv4 or IPv6
     std::string address;        //
     std::string svcid;          // port
+
+    // Define the equality operator
+    bool operator==(const BeaconListener& other) const {
+        return address_family == other.address_family &&
+               address == other.address &&
+               svcid == other.svcid;
+    }
 };
 
 struct BeaconSubsystem {
     NvmeNqnId nqn;
     std::list<BeaconListener>  listeners;
     std::list<BeaconNamespace> namespaces;
+
+    // Define the equality operator
+    bool operator==(const BeaconSubsystem& other) const {
+        return nqn == other.nqn &&
+               listeners == other.listeners &&
+               namespaces == other.namespaces;
+    }
 };
 
 using BeaconSubsystems = std::list<BeaconSubsystem>;
diff --git a/src/nvmeof/NVMeofGwMonitorClient.cc b/src/nvmeof/NVMeofGwMonitorClient.cc
index 28330e432eaf..1893baaaf53d 100644
--- a/src/nvmeof/NVMeofGwMonitorClient.cc
+++ b/src/nvmeof/NVMeofGwMonitorClient.cc
@@ -189,7 +189,6 @@ static bool get_gw_state(const char* desc, const std::map<NvmeGroupKey, NvmeGwMa
 void NVMeofGwMonitorClient::send_beacon()
 {
   ceph_assert(ceph_mutex_is_locked_by_me(lock));
-  //dout(0) << "sending beacon as gid " << monc.get_global_id() << dendl;
   GW_AVAILABILITY_E gw_availability = GW_AVAILABILITY_E::GW_CREATED;
   BeaconSubsystems subs;
   NVMeofGwClient gw_client(

From 78ac9ff669e20c40f341ed941a3b8784f05ff405 Mon Sep 17 00:00:00 2001
From: Leonid Chernin <leonidc@il.ibm.com>
Date: Thu, 18 Apr 2024 10:02:21 +0000
Subject: [PATCH 2492/2492] add validation of the map

Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
---
 src/mon/NVMeofGwMap.cc | 35 +++++++++++++++++++++++++++++++++++
 src/mon/NVMeofGwMap.h  |  1 +
 src/mon/NVMeofGwMon.cc |  1 +
 3 files changed, 37 insertions(+)

diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
index c6c4cb6a3232..3cba68a2536b 100755
--- a/src/mon/NVMeofGwMap.cc
+++ b/src/mon/NVMeofGwMap.cc
@@ -109,6 +109,7 @@ int NVMeofGwMap::process_gw_map_gw_down(const NvmeGwId &gw_id, const NvmeGroupKe
             st.standby_state(i);
         }
         propose_pending = true; // map should reflect that gw becames unavailable
+        if (propose_pending) validate_gw_map(group_key);
     }
     else {
         dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map " << gw_id << dendl;
@@ -152,6 +153,7 @@ void NVMeofGwMap::process_gw_map_ka(const NvmeGwId &gw_id, const NvmeGroupKey& g
           fsm_handle_gw_alive (gw_id, group_key, gw_state->second, st.sm_state[i], i, last_osd_epoch, propose_pending);
         }
     }
+    if (propose_pending) validate_gw_map(group_key);
 }
 
 
@@ -192,6 +194,9 @@ void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
                     find_failback_gw(gw_id, group_key, propose);
                 }
             }
+            if (propose) {
+                validate_gw_map(group_key);
+            }
     }
 }
 
@@ -453,6 +458,7 @@ void NVMeofGwMap::fsm_handle_gw_delete (const NvmeGwId &gw_id, const NvmeGroupKe
             ceph_assert(false);
         }
     }
+    if (map_modified) validate_gw_map(group_key);
 }
 
 void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid,  bool &map_modified)
@@ -494,6 +500,7 @@ void NVMeofGwMap::fsm_handle_to_expired(const NvmeGwId &gw_id, const NvmeGroupKe
         dout(1) << " Expired GW_WAIT_FAILOVER_PREPARED timer from GW " << gw_id << " ANA groupId: "<< grpid << dendl;
         ceph_assert(false);
     }
+    if (map_modified) validate_gw_map(group_key);
 }
 
 NvmeGwCreated& NVMeofGwMap::find_already_created_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key)
@@ -555,6 +562,34 @@ int NVMeofGwMap::blocklist_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_k
     return 0;
 }
 
+void  NVMeofGwMap::validate_gw_map(const NvmeGroupKey& group_key)
+{
+   NvmeAnaGrpId anas[MAX_SUPPORTED_ANA_GROUPS];
+   int i = 0;
+   int max_groups = 0;
+   for (auto& gw_created_pair: Created_gws[group_key]) {
+        auto& st = gw_created_pair.second;
+        anas[i++] = st.ana_grp_id;
+   }
+   max_groups = i;
+   for(int i = 0; i < max_groups; i++)
+   {
+       int ana_group = anas[i];
+       int count = 0;
+       for (auto& gw_created_pair: Created_gws[group_key]) {
+           auto& st = gw_created_pair.second;
+           if (st.sm_state[ana_group] == GW_STATES_PER_AGROUP_E::GW_ACTIVE_STATE){
+               count ++;
+               if(count == 2) {
+                   dout(1) << "number active states per ana-group " << ana_group << "more than 1 in pool-group " << group_key << dendl;
+                   dout(1) << Created_gws[group_key] << dendl;
+                   ceph_assert(false);
+               }
+           }
+       }
+   }
+}
+
 void NVMeofGwMap::update_active_timers( bool &propose_pending ){
 
     //dout(4) << __func__  <<  " called,  p_monitor: " << mon << dendl;
diff --git a/src/mon/NVMeofGwMap.h b/src/mon/NVMeofGwMap.h
index bf46b31dc07e..51c80111ed40 100755
--- a/src/mon/NVMeofGwMap.h
+++ b/src/mon/NVMeofGwMap.h
@@ -64,6 +64,7 @@ class NVMeofGwMap
 
     int  get_timer   (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
     void cancel_timer(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid);
+    void validate_gw_map(const NvmeGroupKey& group_key);
 
 public:
     void encode(ceph::buffer::list &bl) const {
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
index 073e238793be..d9e7e6f9dd29 100644
--- a/src/mon/NVMeofGwMon.cc
+++ b/src/mon/NVMeofGwMon.cc
@@ -332,6 +332,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
 
         if( map.Created_gws[group_key].size()){
             f->open_object_section("common");
+            f->dump_unsigned("epoch", map.epoch);
             f->dump_string("pool", pool);
             f->dump_string("group", group);
             f->dump_unsigned("num gws", map.Created_gws[group_key].size());